summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-10 21:30:40 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-10 21:30:40 +0000
commit133a45c109da5310add55824db21af5239951f93 (patch)
treeba6ac4c0a950a0dda56451944315d66409923918
parentInitial commit. (diff)
downloadrspamd-133a45c109da5310add55824db21af5239951f93.tar.xz
rspamd-133a45c109da5310add55824db21af5239951f93.zip
Adding upstream version 3.8.1.upstream/3.8.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--.clang-format67
-rw-r--r--.drone.jsonnet310
-rw-r--r--.drone.yml563
-rw-r--r--.eslintrc.json76
-rw-r--r--.github/FUNDING.yml4
-rw-r--r--.github/ISSUE_TEMPLATE.md1
-rw-r--r--.github/ISSUE_TEMPLATE/bug_report.md44
-rw-r--r--.github/ISSUE_TEMPLATE/feature_request.md32
-rw-r--r--.github/ISSUE_TEMPLATE/support_question.md15
-rw-r--r--.github/stale.yml17
-rw-r--r--.gitignore26
-rw-r--r--.luacheckrc84
-rw-r--r--.overcommit.yml38
-rw-r--r--.stylelintrc.json27
-rw-r--r--.tidyallrc22
-rw-r--r--AUTHORS.md30
-rw-r--r--CMakeLists.txt806
-rw-r--r--CONTRIBUTING.md128
-rw-r--r--ChangeLog6482
-rw-r--r--LICENSE.md170
-rw-r--r--README.md54
-rw-r--r--blas-config.h.in11
-rw-r--r--clang-plugin/CMakeLists.txt42
-rw-r--r--clang-plugin/plugin.cc69
-rw-r--r--clang-plugin/printf_check.cc822
-rw-r--r--clang-plugin/printf_check.h39
-rw-r--r--cmake/ArchDep.cmake102
-rw-r--r--cmake/AsmOp.cmake19
-rw-r--r--cmake/CompilerWarnings.cmake89
-rw-r--r--cmake/FindArch.cmake127
-rw-r--r--cmake/FindRagel.cmake103
-rw-r--r--cmake/Hyperscan.cmake8
-rw-r--r--cmake/OSDep.cmake65
-rw-r--r--cmake/Openblas.cmake119
-rw-r--r--cmake/PVS-Studio.cmake547
-rw-r--r--cmake/Paths.cmake72
-rw-r--r--cmake/ProcessPackage.cmake122
-rw-r--r--cmake/Sanitizer.cmake75
-rw-r--r--cmake/Toolset.cmake252
-rw-r--r--conf/actions.conf29
-rw-r--r--conf/cgp.inc17
-rw-r--r--conf/common.conf40
-rw-r--r--conf/composites.conf193
-rw-r--r--conf/groups.conf126
-rw-r--r--conf/lang_detection.inc28
-rw-r--r--conf/logging.inc35
-rw-r--r--conf/maps.d/dmarc_whitelist.inc70
-rw-r--r--conf/maps.d/maillist.inc176
-rw-r--r--conf/maps.d/mid.inc22
-rw-r--r--conf/maps.d/mime_types.inc1538
-rw-r--r--conf/maps.d/redirectors.inc1045
-rw-r--r--conf/maps.d/spf_dkim_whitelist.inc233
-rw-r--r--conf/maps.d/surbl-whitelist.inc830
-rw-r--r--conf/metrics.conf24
-rw-r--r--conf/modules.conf17
-rw-r--r--conf/modules.d/antivirus.conf58
-rw-r--r--conf/modules.d/arc.conf72
-rw-r--r--conf/modules.d/asn.conf29
-rw-r--r--conf/modules.d/aws_s3.conf27
-rw-r--r--conf/modules.d/bimi.conf30
-rw-r--r--conf/modules.d/chartable.conf21
-rw-r--r--conf/modules.d/clickhouse.conf59
-rw-r--r--conf/modules.d/dcc.conf28
-rw-r--r--conf/modules.d/dkim.conf25
-rw-r--r--conf/modules.d/dkim_signing.conf77
-rw-r--r--conf/modules.d/dmarc.conf19
-rw-r--r--conf/modules.d/elastic.conf35
-rw-r--r--conf/modules.d/emails.conf8
-rw-r--r--conf/modules.d/external_relay.conf22
-rw-r--r--conf/modules.d/external_services.conf92
-rw-r--r--conf/modules.d/force_actions.conf22
-rw-r--r--conf/modules.d/forged_recipients.conf22
-rw-r--r--conf/modules.d/fuzzy_check.conf49
-rw-r--r--conf/modules.d/greylist.conf35
-rw-r--r--conf/modules.d/hfilter.conf26
-rw-r--r--conf/modules.d/history_redis.conf25
-rw-r--r--conf/modules.d/http_headers.conf22
-rw-r--r--conf/modules.d/known_senders.conf31
-rw-r--r--conf/modules.d/maillist.conf20
-rw-r--r--conf/modules.d/metadata_exporter.conf24
-rw-r--r--conf/modules.d/metric_exporter.conf21
-rw-r--r--conf/modules.d/mid.conf28
-rw-r--r--conf/modules.d/milter_headers.conf29
-rw-r--r--conf/modules.d/mime_types.conf41
-rw-r--r--conf/modules.d/multimap.conf182
-rw-r--r--conf/modules.d/mx_check.conf43
-rw-r--r--conf/modules.d/neural.conf35
-rw-r--r--conf/modules.d/once_received.conf26
-rw-r--r--conf/modules.d/p0f.conf45
-rw-r--r--conf/modules.d/phishing.conf48
-rw-r--r--conf/modules.d/ratelimit.conf44
-rw-r--r--conf/modules.d/rbl.conf347
-rw-r--r--conf/modules.d/redis.conf27
-rw-r--r--conf/modules.d/regexp.conf21
-rw-r--r--conf/modules.d/replies.conf30
-rw-r--r--conf/modules.d/reputation.conf30
-rw-r--r--conf/modules.d/rspamd_update.conf26
-rw-r--r--conf/modules.d/spamassassin.conf26
-rw-r--r--conf/modules.d/spamtrap.conf53
-rw-r--r--conf/modules.d/spf.conf22
-rw-r--r--conf/modules.d/surbl.conf9
-rw-r--r--conf/modules.d/trie.conf37
-rw-r--r--conf/modules.d/url_redirector.conf27
-rw-r--r--conf/modules.d/whitelist.conf66
-rw-r--r--conf/options.inc72
-rw-r--r--conf/rspamd.conf73
-rw-r--r--conf/scores.d/content_group.conf51
-rw-r--r--conf/scores.d/fuzzy_group.conf37
-rw-r--r--conf/scores.d/headers_group.conf77
-rw-r--r--conf/scores.d/hfilter_group.conf133
-rw-r--r--conf/scores.d/mime_types_group.conf78
-rw-r--r--conf/scores.d/mua_group.conf25
-rw-r--r--conf/scores.d/phishing_group.conf57
-rw-r--r--conf/scores.d/policies_group.conf147
-rw-r--r--conf/scores.d/rbl_group.conf250
-rw-r--r--conf/scores.d/statistics_group.conf29
-rw-r--r--conf/scores.d/subject_group.conf23
-rw-r--r--conf/scores.d/surbl_group.conf259
-rw-r--r--conf/scores.d/whitelist_group.conf63
-rw-r--r--conf/settings.conf66
-rw-r--r--conf/statistic.conf59
-rw-r--r--conf/worker-controller.inc19
-rw-r--r--conf/worker-fuzzy.inc22
-rw-r--r--conf/worker-normal.inc15
-rw-r--r--conf/worker-proxy.inc35
-rw-r--r--config.h.in416
-rw-r--r--contrib/DEPENDENCY_INFO.md41
-rw-r--r--contrib/aho-corasick/CMakeLists.txt14
-rw-r--r--contrib/aho-corasick/LICENSE165
-rw-r--r--contrib/aho-corasick/README.md67
-rw-r--r--contrib/aho-corasick/_acism.h114
-rw-r--r--contrib/aho-corasick/acism.c124
-rw-r--r--contrib/aho-corasick/acism.h53
-rw-r--r--contrib/aho-corasick/acism_create.c382
-rw-r--r--contrib/ankerl/LICENSE21
-rw-r--r--contrib/ankerl/svector.h999
-rw-r--r--contrib/ankerl/unordered_dense.h2032
-rw-r--r--contrib/backward-cpp/BackwardConfig.cmake249
-rw-r--r--contrib/backward-cpp/CMakeLists.txt88
-rw-r--r--contrib/backward-cpp/backward.cpp42
-rw-r--r--contrib/backward-cpp/backward.hpp4471
-rw-r--r--contrib/cdb/CMakeLists.txt8
-rw-r--r--contrib/cdb/cdb.h161
-rw-r--r--contrib/cdb/cdb_find.c147
-rw-r--r--contrib/cdb/cdb_init.c140
-rw-r--r--contrib/cdb/cdb_make.c524
-rw-r--r--contrib/doctest/CMakeLists.txt63
-rw-r--r--contrib/doctest/LICENSE.txt21
-rw-r--r--contrib/doctest/README.md153
-rw-r--r--contrib/doctest/doctest/doctest.h6580
-rw-r--r--contrib/doctest/doctest/extensions/doctest_mpi.h120
-rw-r--r--contrib/doctest/doctest/extensions/doctest_util.h34
-rw-r--r--contrib/doctest/doctest/extensions/mpi_reporter.h236
-rw-r--r--contrib/doctest/doctest/parts/doctest.cpp3866
-rw-r--r--contrib/doctest/doctest/parts/doctest_fwd.h2706
-rw-r--r--contrib/elastic/kibana.json138
-rw-r--r--contrib/elastic/rspamd_template.json149
-rw-r--r--contrib/exim/dlfunc-json/README10
-rw-r--r--contrib/exim/dlfunc-json/exim-example.txt75
-rw-r--r--contrib/exim/dlfunc-json/rspamd.c576
-rw-r--r--contrib/exim/local_scan.c700
-rw-r--r--contrib/exim/local_scan.c.in377
-rw-r--r--contrib/exim/patch-exim-src_spam.c.diff338
-rw-r--r--contrib/exim/patch-exim-src_spam.c.diff.exim-4.85.diff332
-rw-r--r--contrib/exim/shutdown.patch14
-rw-r--r--contrib/expected/COPYING121
-rw-r--r--contrib/expected/expected.hpp2326
-rw-r--r--contrib/fastutf8/CMakeLists.txt11
-rw-r--r--contrib/fastutf8/LICENSE22
-rw-r--r--contrib/fastutf8/avx2.c314
-rw-r--r--contrib/fastutf8/fastutf8.c160
-rw-r--r--contrib/fastutf8/fastutf8.h65
-rw-r--r--contrib/fastutf8/sse41.c272
-rw-r--r--contrib/fmt/LICENSE.rst27
-rw-r--r--contrib/fmt/README.rst506
-rw-r--r--contrib/fmt/include/fmt/args.h234
-rw-r--r--contrib/fmt/include/fmt/chrono.h2267
-rw-r--r--contrib/fmt/include/fmt/color.h633
-rw-r--r--contrib/fmt/include/fmt/compile.h607
-rw-r--r--contrib/fmt/include/fmt/core.h2951
-rw-r--r--contrib/fmt/include/fmt/format-inl.h1681
-rw-r--r--contrib/fmt/include/fmt/format.h4735
-rw-r--r--contrib/fmt/include/fmt/locale.h2
-rw-r--r--contrib/fmt/include/fmt/os.h451
-rw-r--r--contrib/fmt/include/fmt/ostream.h209
-rw-r--r--contrib/fmt/include/fmt/posix.h2
-rw-r--r--contrib/fmt/include/fmt/printf.h679
-rw-r--r--contrib/fmt/include/fmt/ranges.h732
-rw-r--r--contrib/fmt/include/fmt/std.h349
-rw-r--r--contrib/fmt/include/fmt/xchar.h259
-rw-r--r--contrib/fpconv/CMakeLists.txt7
-rw-r--r--contrib/fpconv/LICENSE23
-rw-r--r--contrib/fpconv/fpconv.c480
-rw-r--r--contrib/fpconv/fpconv.h35
-rw-r--r--contrib/fpconv/powers.h87
-rw-r--r--contrib/frozen/AUTHORS3
-rw-r--r--contrib/frozen/CMakeLists.txt12
-rw-r--r--contrib/frozen/LICENSE202
-rw-r--r--contrib/frozen/include/frozen/algorithm.h197
-rw-r--r--contrib/frozen/include/frozen/bits/algorithms.h229
-rw-r--r--contrib/frozen/include/frozen/bits/basic_types.h200
-rw-r--r--contrib/frozen/include/frozen/bits/constexpr_assert.h40
-rw-r--r--contrib/frozen/include/frozen/bits/defines.h58
-rw-r--r--contrib/frozen/include/frozen/bits/elsa.h50
-rw-r--r--contrib/frozen/include/frozen/bits/exceptions.h39
-rw-r--r--contrib/frozen/include/frozen/bits/pmh.h240
-rw-r--r--contrib/frozen/include/frozen/bits/version.h30
-rw-r--r--contrib/frozen/include/frozen/map.h323
-rw-r--r--contrib/frozen/include/frozen/random.h90
-rw-r--r--contrib/frozen/include/frozen/set.h220
-rw-r--r--contrib/frozen/include/frozen/string.h152
-rw-r--r--contrib/frozen/include/frozen/unordered_map.h197
-rw-r--r--contrib/frozen/include/frozen/unordered_set.h147
-rw-r--r--contrib/fu2/LICENSE.txt23
-rw-r--r--contrib/fu2/include/function2/function2.hpp1792
-rw-r--r--contrib/google-ced/CMakeLists.txt26
-rw-r--r--contrib/google-ced/LICENSE202
-rw-r--r--contrib/google-ced/ced_c.cc25
-rw-r--r--contrib/google-ced/ced_c.h32
-rw-r--r--contrib/google-ced/compact_enc_det.cc5719
-rw-r--r--contrib/google-ced/compact_enc_det.h83
-rw-r--r--contrib/google-ced/compact_enc_det_generated_tables.h6326
-rw-r--r--contrib/google-ced/compact_enc_det_generated_tables2.h856
-rw-r--r--contrib/google-ced/compact_enc_det_hint_code.cc169
-rw-r--r--contrib/google-ced/compact_enc_det_hint_code.h45
-rw-r--r--contrib/google-ced/detail_head_string.inc152
-rw-r--r--contrib/google-ced/util/basictypes.h331
-rw-r--r--contrib/google-ced/util/case_insensitive_hash.h88
-rw-r--r--contrib/google-ced/util/commandlineflags.h39
-rw-r--r--contrib/google-ced/util/encodings/encodings.cc891
-rw-r--r--contrib/google-ced/util/encodings/encodings.h299
-rw-r--r--contrib/google-ced/util/encodings/encodings.pb.h181
-rw-r--r--contrib/google-ced/util/encodings/encodings_unittest.cc34
-rw-r--r--contrib/google-ced/util/languages/languages.cc349
-rw-r--r--contrib/google-ced/util/languages/languages.h381
-rw-r--r--contrib/google-ced/util/languages/languages.pb.h191
-rw-r--r--contrib/google-ced/util/logging.h25
-rw-r--r--contrib/google-ced/util/port.h53
-rw-r--r--contrib/google-ced/util/string_util.h61
-rw-r--r--contrib/google-ced/util/varsetter.h66
-rw-r--r--contrib/hiredis/CMakeLists.txt11
-rw-r--r--contrib/hiredis/COPYING29
-rw-r--r--contrib/hiredis/README.md392
-rw-r--r--contrib/hiredis/adapters/libev.h147
-rw-r--r--contrib/hiredis/adapters/libevent.h108
-rw-r--r--contrib/hiredis/async.c693
-rw-r--r--contrib/hiredis/async.h135
-rw-r--r--contrib/hiredis/dict.c340
-rw-r--r--contrib/hiredis/dict.h126
-rw-r--r--contrib/hiredis/fmacros.h21
-rw-r--r--contrib/hiredis/hiredis.c1025
-rw-r--r--contrib/hiredis/hiredis.h242
-rw-r--r--contrib/hiredis/net.c464
-rw-r--r--contrib/hiredis/net.h53
-rw-r--r--contrib/hiredis/read.c525
-rw-r--r--contrib/hiredis/read.h116
-rw-r--r--contrib/hiredis/sds.c1095
-rw-r--r--contrib/hiredis/sds.h108
-rw-r--r--contrib/http-parser/CMakeLists.txt8
-rw-r--r--contrib/http-parser/LICENSE-MIT23
-rw-r--r--contrib/http-parser/http_parser.c2268
-rw-r--r--contrib/http-parser/http_parser.h321
-rw-r--r--contrib/kann/CMakeLists.txt16
-rw-r--r--contrib/kann/LICENSE.txt24
-rw-r--r--contrib/kann/kann.c992
-rw-r--r--contrib/kann/kann.h240
-rw-r--r--contrib/kann/kautodiff.c2460
-rw-r--r--contrib/kann/kautodiff.h256
-rw-r--r--contrib/languages-data/af.json1
-rw-r--r--contrib/languages-data/an.json1
-rw-r--r--contrib/languages-data/ar.json1
-rw-r--r--contrib/languages-data/bg.json1
-rw-r--r--contrib/languages-data/bn.json1
-rw-r--r--contrib/languages-data/br.json1
-rw-r--r--contrib/languages-data/cs.json1
-rw-r--r--contrib/languages-data/cy.json1
-rw-r--r--contrib/languages-data/da.json1
-rw-r--r--contrib/languages-data/de.json1
-rw-r--r--contrib/languages-data/en.json1
-rw-r--r--contrib/languages-data/es.json1
-rw-r--r--contrib/languages-data/et.json1
-rw-r--r--contrib/languages-data/eu.json1
-rw-r--r--contrib/languages-data/fa.json1
-rw-r--r--contrib/languages-data/fi.json1
-rw-r--r--contrib/languages-data/fr.json1
-rw-r--r--contrib/languages-data/ga.json1
-rw-r--r--contrib/languages-data/hi.json1
-rw-r--r--contrib/languages-data/hr.json1
-rw-r--r--contrib/languages-data/hu.json1
-rw-r--r--contrib/languages-data/id.json1
-rw-r--r--contrib/languages-data/is.json1
-rw-r--r--contrib/languages-data/it.json1
-rw-r--r--contrib/languages-data/lt.json1
-rw-r--r--contrib/languages-data/lv.json1
-rw-r--r--contrib/languages-data/mr.json1
-rw-r--r--contrib/languages-data/ms.json1
-rw-r--r--contrib/languages-data/ne.json1
-rw-r--r--contrib/languages-data/nl.json1
-rw-r--r--contrib/languages-data/no.json1
-rw-r--r--contrib/languages-data/pa.json1
-rw-r--r--contrib/languages-data/pl.json1
-rw-r--r--contrib/languages-data/pt.json1
-rw-r--r--contrib/languages-data/ro.json1
-rw-r--r--contrib/languages-data/ru.json1
-rw-r--r--contrib/languages-data/sl.json1
-rw-r--r--contrib/languages-data/so.json1
-rw-r--r--contrib/languages-data/sq.json1
-rw-r--r--contrib/languages-data/sr.json1
-rw-r--r--contrib/languages-data/stop_words4153
-rw-r--r--contrib/languages-data/sv.json1
-rw-r--r--contrib/languages-data/sw.json1
-rw-r--r--contrib/languages-data/tr.json1
-rw-r--r--contrib/languages-data/uk.json1
-rw-r--r--contrib/languages-data/ur.json1
-rw-r--r--contrib/languages-data/vi.json1
-rw-r--r--contrib/lc-btrie/CMakeLists.txt6
-rw-r--r--contrib/lc-btrie/btrie.c2644
-rw-r--r--contrib/lc-btrie/btrie.h83
-rw-r--r--contrib/libev/CMakeLists.txt82
-rw-r--r--contrib/libev/Changes529
-rw-r--r--contrib/libev/LICENSE37
-rw-r--r--contrib/libev/config.h.in112
-rw-r--r--contrib/libev/ev++.h816
-rw-r--r--contrib/libev/ev.c5678
-rw-r--r--contrib/libev/ev.h849
-rw-r--r--contrib/libev/ev_epoll.c298
-rw-r--r--contrib/libev/ev_iouring.c697
-rw-r--r--contrib/libev/ev_kqueue.c224
-rw-r--r--contrib/libev/ev_linuxaio.c620
-rw-r--r--contrib/libev/ev_poll.c156
-rw-r--r--contrib/libev/ev_port.c192
-rw-r--r--contrib/libev/ev_select.c316
-rw-r--r--contrib/libev/ev_vars.h249
-rw-r--r--contrib/libev/ev_win32.c162
-rw-r--r--contrib/libev/ev_wrap.h272
-rw-r--r--contrib/libottery/CMakeLists.txt11
-rw-r--r--contrib/libottery/aes_cryptobox.c181
-rw-r--r--contrib/libottery/chacha_cryptobox.c64
-rw-r--r--contrib/libottery/chacha_merged.c218
-rw-r--r--contrib/libottery/chacha_merged_ecrypt.h133
-rw-r--r--contrib/libottery/ottery-internal.h335
-rw-r--r--contrib/libottery/ottery-threading.h96
-rw-r--r--contrib/libottery/ottery.c847
-rw-r--r--contrib/libottery/ottery.h143
-rw-r--r--contrib/libottery/ottery_common.h351
-rw-r--r--contrib/libottery/ottery_cpuinfo.c88
-rw-r--r--contrib/libottery/ottery_entropy.c112
-rw-r--r--contrib/libottery/ottery_entropy_cryptgenrandom.c51
-rw-r--r--contrib/libottery/ottery_entropy_egd.c75
-rw-r--r--contrib/libottery/ottery_entropy_rdrand.c58
-rw-r--r--contrib/libottery/ottery_entropy_urandom.c117
-rw-r--r--contrib/libottery/ottery_global.c116
-rw-r--r--contrib/libottery/ottery_nolock.h190
-rw-r--r--contrib/libottery/ottery_st.h184
-rw-r--r--contrib/libottery/ottery_version.h.in28
-rw-r--r--contrib/librdns/CMakeLists.txt11
-rw-r--r--contrib/librdns/compression.c168
-rw-r--r--contrib/librdns/compression.h48
-rw-r--r--contrib/librdns/curve.c890
-rw-r--r--contrib/librdns/dns_private.h338
-rw-r--r--contrib/librdns/logger.c53
-rw-r--r--contrib/librdns/logger.h48
-rw-r--r--contrib/librdns/packet.c288
-rw-r--r--contrib/librdns/packet.h61
-rw-r--r--contrib/librdns/parse.c461
-rw-r--r--contrib/librdns/parse.h65
-rw-r--r--contrib/librdns/punycode.c303
-rw-r--r--contrib/librdns/punycode.h59
-rw-r--r--contrib/librdns/rdns.h493
-rw-r--r--contrib/librdns/rdns_curve.h75
-rw-r--r--contrib/librdns/rdns_ev.h235
-rw-r--r--contrib/librdns/rdns_event.h231
-rw-r--r--contrib/librdns/ref.h71
-rw-r--r--contrib/librdns/resolver.c1577
-rw-r--r--contrib/librdns/upstream.h282
-rw-r--r--contrib/librdns/util.c1035
-rw-r--r--contrib/librdns/util.h99
-rw-r--r--contrib/libucl/CMakeLists.txt19
-rw-r--r--contrib/libucl/README.md397
-rw-r--r--contrib/libucl/khash.h682
-rw-r--r--contrib/libucl/kvec.h161
-rw-r--r--contrib/libucl/lua_ucl.c1574
-rw-r--r--contrib/libucl/lua_ucl.h109
-rw-r--r--contrib/libucl/tree.h219
-rw-r--r--contrib/libucl/ucl.h1650
-rw-r--r--contrib/libucl/ucl_chartable.h268
-rw-r--r--contrib/libucl/ucl_emitter.c1189
-rw-r--r--contrib/libucl/ucl_emitter_streamline.c173
-rw-r--r--contrib/libucl/ucl_hash.c498
-rw-r--r--contrib/libucl/ucl_hash.h114
-rw-r--r--contrib/libucl/ucl_internal.h666
-rw-r--r--contrib/libucl/ucl_msgpack.c1615
-rw-r--r--contrib/libucl/ucl_parser.c3212
-rw-r--r--contrib/libucl/ucl_schema.c1104
-rw-r--r--contrib/libucl/ucl_sexp.c226
-rw-r--r--contrib/libucl/ucl_util.c3952
-rw-r--r--contrib/lua-argparse/LICENSE20
-rw-r--r--contrib/lua-argparse/argparse.lua2100
-rw-r--r--contrib/lua-bit/CMakeLists.txt4
-rw-r--r--contrib/lua-bit/bit.c198
-rw-r--r--contrib/lua-fun/COPYING.md27
-rw-r--r--contrib/lua-fun/fun.lua1076
-rw-r--r--contrib/lua-lpeg/CMakeLists.txt9
-rw-r--r--contrib/lua-lpeg/LICENSE19
-rw-r--r--contrib/lua-lpeg/lpcap.c555
-rw-r--r--contrib/lua-lpeg/lpcap.h57
-rw-r--r--contrib/lua-lpeg/lpcode.c1014
-rw-r--r--contrib/lua-lpeg/lpcode.h40
-rw-r--r--contrib/lua-lpeg/lpegre.lua267
-rw-r--r--contrib/lua-lpeg/lpprint.c244
-rw-r--r--contrib/lua-lpeg/lpprint.h36
-rw-r--r--contrib/lua-lpeg/lptree.c1319
-rw-r--r--contrib/lua-lpeg/lptree.h77
-rw-r--r--contrib/lua-lpeg/lptypes.h154
-rw-r--r--contrib/lua-lpeg/lpvm.c383
-rw-r--r--contrib/lua-lpeg/lpvm.h58
-rw-r--r--contrib/lua-lupa/LICENSE21
-rw-r--r--contrib/lua-lupa/README.md179
-rw-r--r--contrib/lua-lupa/lupa.lua1810
-rw-r--r--contrib/lua-tableshape/LICENSE19
-rw-r--r--contrib/lua-tableshape/tableshape.lua2324
-rw-r--r--contrib/mumhash/mum.h392
-rw-r--r--contrib/publicsuffix/effective_tld_names.dat14206
-rw-r--r--contrib/publicsuffix/idn.pl20
-rw-r--r--contrib/replxx/CMakeLists.txt82
-rw-r--r--contrib/replxx/LICENSE.md63
-rw-r--r--contrib/replxx/README.md119
-rw-r--r--contrib/replxx/include/replxx.h575
-rw-r--r--contrib/replxx/include/replxx.hxx615
-rw-r--r--contrib/replxx/src/conversion.cxx134
-rw-r--r--contrib/replxx/src/conversion.hxx35
-rw-r--r--contrib/replxx/src/escape.cxx890
-rw-r--r--contrib/replxx/src/escape.hxx37
-rw-r--r--contrib/replxx/src/history.cxx402
-rw-r--r--contrib/replxx/src/history.hxx141
-rw-r--r--contrib/replxx/src/killring.hxx78
-rw-r--r--contrib/replxx/src/prompt.cxx144
-rw-r--r--contrib/replxx/src/prompt.hxx45
-rw-r--r--contrib/replxx/src/replxx.cxx648
-rw-r--r--contrib/replxx/src/replxx_impl.cxx2248
-rw-r--r--contrib/replxx/src/replxx_impl.hxx280
-rw-r--r--contrib/replxx/src/terminal.cxx742
-rw-r--r--contrib/replxx/src/terminal.hxx94
-rw-r--r--contrib/replxx/src/unicodestring.hxx201
-rw-r--r--contrib/replxx/src/utf8string.hxx94
-rw-r--r--contrib/replxx/src/util.cxx158
-rw-r--r--contrib/replxx/src/util.hxx25
-rw-r--r--contrib/replxx/src/wcwidth.cpp296
-rw-r--r--contrib/replxx/src/windows.cxx144
-rw-r--r--contrib/replxx/src/windows.hxx44
-rw-r--r--contrib/snowball/.gitignore5
-rw-r--r--contrib/snowball/.travis.yml4
-rw-r--r--contrib/snowball/AUTHORS27
-rw-r--r--contrib/snowball/CMakeLists.txt70
-rw-r--r--contrib/snowball/NEWS407
-rw-r--r--contrib/snowball/README5
-rw-r--r--contrib/snowball/algorithms/arabic.sbl561
-rw-r--r--contrib/snowball/algorithms/basque.sbl149
-rw-r--r--contrib/snowball/algorithms/catalan.sbl202
-rw-r--r--contrib/snowball/algorithms/danish.sbl93
-rw-r--r--contrib/snowball/algorithms/dutch.sbl164
-rw-r--r--contrib/snowball/algorithms/english.sbl229
-rw-r--r--contrib/snowball/algorithms/finnish.sbl197
-rw-r--r--contrib/snowball/algorithms/french.sbl254
-rw-r--r--contrib/snowball/algorithms/german.sbl139
-rw-r--r--contrib/snowball/algorithms/german2.sbl145
-rw-r--r--contrib/snowball/algorithms/greek.sbl706
-rw-r--r--contrib/snowball/algorithms/hindi.sbl323
-rw-r--r--contrib/snowball/algorithms/hungarian.sbl241
-rw-r--r--contrib/snowball/algorithms/indonesian.sbl192
-rw-r--r--contrib/snowball/algorithms/irish.sbl151
-rw-r--r--contrib/snowball/algorithms/italian.sbl195
-rw-r--r--contrib/snowball/algorithms/kraaij_pohlmann.sbl240
-rw-r--r--contrib/snowball/algorithms/lithuanian.sbl373
-rw-r--r--contrib/snowball/algorithms/lovins.sbl208
-rw-r--r--contrib/snowball/algorithms/nepali.sbl92
-rw-r--r--contrib/snowball/algorithms/norwegian.sbl80
-rw-r--r--contrib/snowball/algorithms/porter.sbl139
-rw-r--r--contrib/snowball/algorithms/portuguese.sbl218
-rw-r--r--contrib/snowball/algorithms/romanian.sbl236
-rw-r--r--contrib/snowball/algorithms/russian.sbl221
-rw-r--r--contrib/snowball/algorithms/serbian.sbl2378
-rw-r--r--contrib/snowball/algorithms/spanish.sbl230
-rw-r--r--contrib/snowball/algorithms/swedish.sbl72
-rw-r--r--contrib/snowball/algorithms/tamil.sbl405
-rw-r--r--contrib/snowball/algorithms/turkish.sbl470
-rw-r--r--contrib/snowball/charsets/ISO-8859-2.sbl98
-rw-r--r--contrib/snowball/charsets/KOI8-R.sbl74
-rw-r--r--contrib/snowball/charsets/cp850.sbl130
-rw-r--r--contrib/snowball/compiler/analyser.c1380
-rw-r--r--contrib/snowball/compiler/driver.c574
-rw-r--r--contrib/snowball/compiler/generator.c1725
-rw-r--r--contrib/snowball/compiler/header.h411
-rw-r--r--contrib/snowball/compiler/space.c287
-rw-r--r--contrib/snowball/compiler/syswords.h86
-rw-r--r--contrib/snowball/compiler/syswords2.h13
-rw-r--r--contrib/snowball/compiler/tokeniser.c567
-rw-r--r--contrib/snowball/doc/TODO15
-rw-r--r--contrib/snowball/doc/libstemmer_c_README125
-rw-r--r--contrib/snowball/doc/libstemmer_java_README40
-rw-r--r--contrib/snowball/include/libstemmer.h78
-rw-r--r--contrib/snowball/libstemmer/libstemmer_c.in96
-rwxr-xr-xcontrib/snowball/libstemmer/mkmodules.pl267
-rw-r--r--contrib/snowball/libstemmer/modules.txt58
-rw-r--r--contrib/snowball/libstemmer/modules_utf8.txt49
-rw-r--r--contrib/snowball/runtime/api.c58
-rw-r--r--contrib/snowball/runtime/api.h32
-rw-r--r--contrib/snowball/runtime/header.h59
-rw-r--r--contrib/snowball/runtime/utilities.c503
-rw-r--r--contrib/t1ha/CMakeLists.txt6
-rw-r--r--contrib/t1ha/LICENSE21
-rw-r--r--contrib/t1ha/t1ha.h321
-rw-r--r--contrib/t1ha/t1ha1.c158
-rw-r--r--contrib/t1ha/t1ha2.c326
-rw-r--r--contrib/t1ha/t1ha_bits.h1171
-rw-r--r--contrib/uthash/uthash.h951
-rw-r--r--contrib/uthash/utlist.h766
-rw-r--r--contrib/uthash/utstring.h415
-rw-r--r--contrib/xxhash/CMakeLists.txt13
-rw-r--r--contrib/xxhash/LICENSE24
-rw-r--r--contrib/xxhash/xxh3.h55
-rw-r--r--contrib/xxhash/xxhash.c43
-rw-r--r--contrib/xxhash/xxhash.h5580
-rw-r--r--contrib/zstd/CHANGELOG555
-rw-r--r--contrib/zstd/CMakeLists.txt27
-rw-r--r--contrib/zstd/LICENSE30
-rw-r--r--contrib/zstd/PATENTS33
-rw-r--r--contrib/zstd/README.md94
-rw-r--r--contrib/zstd/bits.h175
-rw-r--r--contrib/zstd/bitstream.h437
-rw-r--r--contrib/zstd/clevels.h134
-rw-r--r--contrib/zstd/compiler.h354
-rw-r--r--contrib/zstd/cpu.h213
-rw-r--r--contrib/zstd/debug.c24
-rw-r--r--contrib/zstd/debug.h107
-rw-r--r--contrib/zstd/divsufsort.c1913
-rw-r--r--contrib/zstd/divsufsort.h67
-rw-r--r--contrib/zstd/entropy_common.c340
-rw-r--r--contrib/zstd/error_private.c63
-rw-r--r--contrib/zstd/error_private.h159
-rw-r--r--contrib/zstd/error_public.h59
-rw-r--r--contrib/zstd/fse.h639
-rw-r--r--contrib/zstd/fse_compress.c624
-rw-r--r--contrib/zstd/fse_decompress.c311
-rw-r--r--contrib/zstd/hist.c181
-rw-r--r--contrib/zstd/hist.h75
-rw-r--r--contrib/zstd/huf.h273
-rw-r--r--contrib/zstd/huf_compress.c1435
-rw-r--r--contrib/zstd/huf_decompress.c1882
-rw-r--r--contrib/zstd/mem.h435
-rw-r--r--contrib/zstd/pool.c371
-rw-r--r--contrib/zstd/pool.h90
-rw-r--r--contrib/zstd/portability_macros.h156
-rw-r--r--contrib/zstd/zdict.h474
-rw-r--r--contrib/zstd/zstd.h2974
-rw-r--r--contrib/zstd/zstd_common.c83
-rw-r--r--contrib/zstd/zstd_compress.c6927
-rw-r--r--contrib/zstd/zstd_compress.h307
-rw-r--r--contrib/zstd/zstd_compress_internal.h1478
-rw-r--r--contrib/zstd/zstd_compress_literals.c235
-rw-r--r--contrib/zstd/zstd_compress_literals.h39
-rw-r--r--contrib/zstd/zstd_compress_sequences.c442
-rw-r--r--contrib/zstd/zstd_compress_sequences.h54
-rw-r--r--contrib/zstd/zstd_compress_superblock.c577
-rw-r--r--contrib/zstd/zstd_compress_superblock.h32
-rw-r--r--contrib/zstd/zstd_cwksp.h678
-rw-r--r--contrib/zstd/zstd_ddict.c243
-rw-r--r--contrib/zstd/zstd_ddict.h44
-rw-r--r--contrib/zstd/zstd_decompress.c2352
-rw-r--r--contrib/zstd/zstd_decompress_block.c2193
-rw-r--r--contrib/zstd/zstd_decompress_block.h68
-rw-r--r--contrib/zstd/zstd_decompress_internal.h238
-rw-r--r--contrib/zstd/zstd_deps.h111
-rw-r--r--contrib/zstd/zstd_double_fast.c758
-rw-r--r--contrib/zstd/zstd_double_fast.h39
-rw-r--r--contrib/zstd/zstd_errors.h114
-rw-r--r--contrib/zstd/zstd_fast.c960
-rw-r--r--contrib/zstd/zstd_fast.h38
-rw-r--r--contrib/zstd/zstd_internal.h397
-rw-r--r--contrib/zstd/zstd_lazy.c2127
-rw-r--r--contrib/zstd/zstd_lazy.h127
-rw-r--r--contrib/zstd/zstd_ldm.c724
-rw-r--r--contrib/zstd/zstd_ldm.h117
-rw-r--r--contrib/zstd/zstd_ldm_geartab.h106
-rw-r--r--contrib/zstd/zstd_opt.c1470
-rw-r--r--contrib/zstd/zstd_opt.h56
-rw-r--r--contrib/zstd/zstd_trace.h163
-rwxr-xr-xdist.sh29
-rw-r--r--doc/Makefile94
-rw-r--r--doc/doxydown/.gitignore19
-rw-r--r--doc/doxydown/LICENSE21
-rw-r--r--doc/doxydown/README.md139
-rwxr-xr-xdoc/doxydown/doxydown.pl624
-rw-r--r--doc/rspamadm.1134
-rw-r--r--doc/rspamadm.1.md85
-rw-r--r--doc/rspamc.1307
-rw-r--r--doc/rspamc.1.md173
-rw-r--r--doc/rspamd.878
-rw-r--r--doc/rspamd.8.md57
-rw-r--r--docker/README.md3
-rw-r--r--freebsd/cmake/PkgCreate.cmake56
-rwxr-xr-xfreebsd/rspamd.sh.in52
-rw-r--r--interface/README.md51
-rw-r--r--interface/apple-touch-icon.pngbin0 -> 435 bytes
-rw-r--r--interface/browserconfig.xml9
-rw-r--r--interface/css/FooTable.Glyphicons.css62
-rw-r--r--interface/css/bootstrap.min.css6
-rw-r--r--interface/css/codejar-linenumbers.css17
-rw-r--r--interface/css/d3evolution.css66
-rw-r--r--interface/css/d3pie.css49
-rw-r--r--interface/css/font-glyphicons.css805
-rw-r--r--interface/css/footable.standalone.min.css1
-rw-r--r--interface/css/nprogress.css74
-rw-r--r--interface/css/prism.css4
-rw-r--r--interface/css/rspamd.css598
-rw-r--r--interface/css/svg-with-js.min.css5
-rw-r--r--interface/favicon-16x16.pngbin0 -> 257 bytes
-rw-r--r--interface/favicon-32x32.pngbin0 -> 300 bytes
-rw-r--r--interface/favicon.icobin0 -> 10734 bytes
-rw-r--r--interface/fonts/glyphicons-halflings-regular.ttfbin0 -> 45404 bytes
-rw-r--r--interface/fonts/glyphicons-halflings-regular.woffbin0 -> 23424 bytes
-rw-r--r--interface/fonts/glyphicons-halflings-regular.woff2bin0 -> 18028 bytes
-rw-r--r--interface/img/asc.pngbin0 -> 128 bytes
-rw-r--r--interface/img/desc.pngbin0 -> 128 bytes
-rw-r--r--interface/img/rspamd_logo_navbar.pngbin0 -> 38028 bytes
-rw-r--r--interface/index.html737
-rw-r--r--interface/js/app/common.js233
-rw-r--r--interface/js/app/config.js246
-rw-r--r--interface/js/app/graph.js252
-rw-r--r--interface/js/app/history.js306
-rw-r--r--interface/js/app/libft.js524
-rw-r--r--interface/js/app/rspamd.js486
-rw-r--r--interface/js/app/selectors.js145
-rw-r--r--interface/js/app/stats.js372
-rw-r--r--interface/js/app/symbols.js260
-rw-r--r--interface/js/app/upload.js243
-rw-r--r--interface/js/lib/bootstrap.bundle.min.js7
-rw-r--r--interface/js/lib/codejar-linenumbers.min.js6
-rw-r--r--interface/js/lib/codejar.min.js5
-rw-r--r--interface/js/lib/d3.min.js2
-rw-r--r--interface/js/lib/d3evolution.min.js5
-rw-r--r--interface/js/lib/d3pie.min.js5
-rw-r--r--interface/js/lib/fontawesome.min.js5
-rw-r--r--interface/js/lib/footable.min.js10
-rw-r--r--interface/js/lib/jquery-3.7.1.min.js2
-rw-r--r--interface/js/lib/jquery.stickytabs.min.js9
-rw-r--r--interface/js/lib/nprogress.min.js4
-rw-r--r--interface/js/lib/prism.js5
-rw-r--r--interface/js/lib/require.min.js5
-rw-r--r--interface/js/lib/solid.min.js5
-rw-r--r--interface/js/lib/visibility.min.js5
-rw-r--r--interface/js/main.js68
-rw-r--r--interface/mstile-150x150.pngbin0 -> 711 bytes
-rw-r--r--interface/safari-pinned-tab.svg1
-rw-r--r--lua_style.md733
-rw-r--r--lualib/ansicolors.lua68
-rw-r--r--lualib/global_functions.lua56
-rw-r--r--lualib/lua_auth_results.lua301
-rw-r--r--lualib/lua_aws.lua300
-rw-r--r--lualib/lua_bayes_learn.lua151
-rw-r--r--lualib/lua_bayes_redis.lua244
-rw-r--r--lualib/lua_cfg_transform.lua634
-rw-r--r--lualib/lua_cfg_utils.lua84
-rw-r--r--lualib/lua_clickhouse.lua547
-rw-r--r--lualib/lua_content/ical.lua105
-rw-r--r--lualib/lua_content/init.lua109
-rw-r--r--lualib/lua_content/pdf.lua1424
-rw-r--r--lualib/lua_content/vcard.lua84
-rw-r--r--lualib/lua_dkim_tools.lua742
-rw-r--r--lualib/lua_ffi/common.lua45
-rw-r--r--lualib/lua_ffi/dkim.lua144
-rw-r--r--lualib/lua_ffi/init.lua59
-rw-r--r--lualib/lua_ffi/linalg.lua87
-rw-r--r--lualib/lua_ffi/spf.lua143
-rw-r--r--lualib/lua_fuzzy.lua355
-rw-r--r--lualib/lua_lexer.lua163
-rw-r--r--lualib/lua_magic/heuristics.lua605
-rw-r--r--lualib/lua_magic/init.lua388
-rw-r--r--lualib/lua_magic/patterns.lua471
-rw-r--r--lualib/lua_magic/types.lua327
-rw-r--r--lualib/lua_maps.lua612
-rw-r--r--lualib/lua_maps_expressions.lua219
-rw-r--r--lualib/lua_meta.lua549
-rw-r--r--lualib/lua_mime.lua760
-rw-r--r--lualib/lua_mime_types.lua745
-rw-r--r--lualib/lua_redis.lua1817
-rw-r--r--lualib/lua_scanners/avast.lua304
-rw-r--r--lualib/lua_scanners/clamav.lua193
-rw-r--r--lualib/lua_scanners/cloudmark.lua372
-rw-r--r--lualib/lua_scanners/common.lua539
-rw-r--r--lualib/lua_scanners/dcc.lua313
-rw-r--r--lualib/lua_scanners/fprot.lua181
-rw-r--r--lualib/lua_scanners/icap.lua713
-rw-r--r--lualib/lua_scanners/init.lua75
-rw-r--r--lualib/lua_scanners/kaspersky_av.lua197
-rw-r--r--lualib/lua_scanners/kaspersky_se.lua287
-rw-r--r--lualib/lua_scanners/oletools.lua369
-rw-r--r--lualib/lua_scanners/p0f.lua227
-rw-r--r--lualib/lua_scanners/pyzor.lua206
-rw-r--r--lualib/lua_scanners/razor.lua181
-rw-r--r--lualib/lua_scanners/savapi.lua261
-rw-r--r--lualib/lua_scanners/sophos.lua192
-rw-r--r--lualib/lua_scanners/spamassassin.lua213
-rw-r--r--lualib/lua_scanners/vadesecure.lua351
-rw-r--r--lualib/lua_scanners/virustotal.lua214
-rw-r--r--lualib/lua_selectors/common.lua95
-rw-r--r--lualib/lua_selectors/extractors.lua565
-rw-r--r--lualib/lua_selectors/init.lua668
-rw-r--r--lualib/lua_selectors/maps.lua19
-rw-r--r--lualib/lua_selectors/transforms.lua571
-rw-r--r--lualib/lua_settings.lua309
-rw-r--r--lualib/lua_smtp.lua201
-rw-r--r--lualib/lua_stat.lua869
-rw-r--r--lualib/lua_tcp_sync.lua213
-rw-r--r--lualib/lua_urls_compose.lua286
-rw-r--r--lualib/lua_util.lua1639
-rw-r--r--lualib/lua_verdict.lua208
-rw-r--r--lualib/plugins/dmarc.lua359
-rw-r--r--lualib/plugins/neural.lua892
-rw-r--r--lualib/plugins/rbl.lua232
-rw-r--r--lualib/plugins_stats.lua48
-rw-r--r--lualib/redis_scripts/bayes_cache_check.lua20
-rw-r--r--lualib/redis_scripts/bayes_cache_learn.lua61
-rw-r--r--lualib/redis_scripts/bayes_classify.lua37
-rw-r--r--lualib/redis_scripts/bayes_learn.lua44
-rw-r--r--lualib/redis_scripts/bayes_stat.lua19
-rw-r--r--lualib/redis_scripts/neural_maybe_invalidate.lua25
-rw-r--r--lualib/redis_scripts/neural_maybe_lock.lua19
-rw-r--r--lualib/redis_scripts/neural_save_unlock.lua24
-rw-r--r--lualib/redis_scripts/neural_train_size.lua24
-rw-r--r--lualib/redis_scripts/ratelimit_check.lua85
-rw-r--r--lualib/redis_scripts/ratelimit_cleanup_pending.lua33
-rw-r--r--lualib/redis_scripts/ratelimit_update.lua93
-rw-r--r--lualib/rspamadm/clickhouse.lua528
-rw-r--r--lualib/rspamadm/configgraph.lua172
-rw-r--r--lualib/rspamadm/confighelp.lua123
-rw-r--r--lualib/rspamadm/configwizard.lua849
-rw-r--r--lualib/rspamadm/cookie.lua125
-rw-r--r--lualib/rspamadm/corpus_test.lua185
-rw-r--r--lualib/rspamadm/dkim_keygen.lua178
-rw-r--r--lualib/rspamadm/dmarc_report.lua737
-rw-r--r--lualib/rspamadm/dns_tool.lua232
-rw-r--r--lualib/rspamadm/fuzzy_convert.lua208
-rw-r--r--lualib/rspamadm/fuzzy_ping.lua259
-rw-r--r--lualib/rspamadm/fuzzy_stat.lua366
-rw-r--r--lualib/rspamadm/grep.lua174
-rw-r--r--lualib/rspamadm/keypair.lua508
-rw-r--r--lualib/rspamadm/mime.lua1012
-rw-r--r--lualib/rspamadm/neural_test.lua228
-rw-r--r--lualib/rspamadm/publicsuffix.lua82
-rw-r--r--lualib/rspamadm/stat_convert.lua38
-rw-r--r--lualib/rspamadm/statistics_dump.lua544
-rw-r--r--lualib/rspamadm/template.lua131
-rw-r--r--lualib/rspamadm/vault.lua579
-rw-r--r--package.json14
-rw-r--r--rpm/80-rspamd.preset1
-rw-r--r--rpm/rspamd.logrotate11
-rw-r--r--rpm/rspamd.spec223
-rw-r--r--rspamd.service17
-rw-r--r--rules/bitcoin.lua237
-rw-r--r--rules/bounce.lua117
-rw-r--r--rules/content.lua118
-rw-r--r--rules/controller/fuzzy.lua46
-rw-r--r--rules/controller/init.lua67
-rw-r--r--rules/controller/maps.lua220
-rw-r--r--rules/controller/neural.lua70
-rw-r--r--rules/controller/selectors.lua73
-rw-r--r--rules/forwarding.lua163
-rw-r--r--rules/headers_checks.lua1174
-rw-r--r--rules/html.lua462
-rw-r--r--rules/mid.lua131
-rw-r--r--rules/misc.lua864
-rw-r--r--rules/parts.lua11
-rw-r--r--rules/regexp/compromised_hosts.lua223
-rw-r--r--rules/regexp/headers.lua1046
-rw-r--r--rules/regexp/misc.lua117
-rw-r--r--rules/regexp/upstream_spam_filters.lua60
-rw-r--r--rules/rspamd.lua71
-rw-r--r--rules/subject_checks.lua70
-rwxr-xr-xset-version.sh14
-rw-r--r--src/CMakeLists.txt246
-rw-r--r--src/client/CMakeLists.txt16
-rw-r--r--src/client/rspamc.cxx2380
-rw-r--r--src/client/rspamdclient.c498
-rw-r--r--src/client/rspamdclient.h106
-rw-r--r--src/controller.c4288
-rw-r--r--src/fuzzy_storage.c3287
-rw-r--r--src/hs_helper.c426
-rw-r--r--src/libcryptobox/AsmOpt.cmake0
-rw-r--r--src/libcryptobox/CMakeLists.txt41
-rw-r--r--src/libcryptobox/base64/avx2.c287
-rw-r--r--src/libcryptobox/base64/base64.c445
-rw-r--r--src/libcryptobox/base64/base64.h31
-rw-r--r--src/libcryptobox/base64/ref.c241
-rw-r--r--src/libcryptobox/base64/sse42.c268
-rw-r--r--src/libcryptobox/catena/LICENSE20
-rw-r--r--src/libcryptobox/catena/README.md18
-rw-r--r--src/libcryptobox/catena/catena.c444
-rw-r--r--src/libcryptobox/catena/catena.h62
-rw-r--r--src/libcryptobox/chacha20/avx.S614
-rw-r--r--src/libcryptobox/chacha20/avx2.S1018
-rw-r--r--src/libcryptobox/chacha20/chacha.c262
-rw-r--r--src/libcryptobox/chacha20/chacha.h87
-rw-r--r--src/libcryptobox/chacha20/constants.S6
-rw-r--r--src/libcryptobox/chacha20/ref.c272
-rw-r--r--src/libcryptobox/chacha20/sse2.S734
-rw-r--r--src/libcryptobox/cryptobox.c1778
-rw-r--r--src/libcryptobox/cryptobox.h437
-rw-r--r--src/libcryptobox/keypair.c1021
-rw-r--r--src/libcryptobox/keypair.h317
-rw-r--r--src/libcryptobox/keypair_private.h143
-rw-r--r--src/libcryptobox/keypairs_cache.c141
-rw-r--r--src/libcryptobox/keypairs_cache.h57
-rw-r--r--src/libcryptobox/macro.S176
-rw-r--r--src/libcryptobox/platform_config.h.in16
-rw-r--r--src/libmime/CMakeLists.txt19
-rw-r--r--src/libmime/archives.c2057
-rw-r--r--src/libmime/archives.h72
-rw-r--r--src/libmime/content_type.c884
-rw-r--r--src/libmime/content_type.h130
-rw-r--r--src/libmime/email_addr.c563
-rw-r--r--src/libmime/email_addr.h97
-rw-r--r--src/libmime/images.c718
-rw-r--r--src/libmime/images.h76
-rw-r--r--src/libmime/lang_detection.c2103
-rw-r--r--src/libmime/lang_detection.h110
-rw-r--r--src/libmime/lang_detection_fasttext.cxx269
-rw-r--r--src/libmime/lang_detection_fasttext.h91
-rw-r--r--src/libmime/message.c1732
-rw-r--r--src/libmime/message.h239
-rw-r--r--src/libmime/mime_encoding.c864
-rw-r--r--src/libmime/mime_encoding.h148
-rw-r--r--src/libmime/mime_encoding_list.h1577
-rw-r--r--src/libmime/mime_expressions.c2392
-rw-r--r--src/libmime/mime_expressions.h65
-rw-r--r--src/libmime/mime_headers.c1441
-rw-r--r--src/libmime/mime_headers.h200
-rw-r--r--src/libmime/mime_parser.c1758
-rw-r--r--src/libmime/mime_parser.h46
-rw-r--r--src/libmime/mime_string.cxx167
-rw-r--r--src/libmime/mime_string.hxx670
-rw-r--r--src/libmime/received.cxx1017
-rw-r--r--src/libmime/received.h68
-rw-r--r--src/libmime/received.hxx314
-rw-r--r--src/libmime/scan_result.c1106
-rw-r--r--src/libmime/scan_result.h250
-rw-r--r--src/libmime/scan_result_private.h55
-rw-r--r--src/libmime/smtp_parsers.h51
-rw-r--r--src/libserver/CMakeLists.txt52
-rw-r--r--src/libserver/async_session.c364
-rw-r--r--src/libserver/async_session.h121
-rw-r--r--src/libserver/backtrace.cxx61
-rw-r--r--src/libserver/cfg_file.h889
-rw-r--r--src/libserver/cfg_file_private.h41
-rw-r--r--src/libserver/cfg_rcl.cxx4110
-rw-r--r--src/libserver/cfg_rcl.h476
-rw-r--r--src/libserver/cfg_utils.cxx2955
-rw-r--r--src/libserver/composites/composites.cxx989
-rw-r--r--src/libserver/composites/composites.h64
-rw-r--r--src/libserver/composites/composites_internal.hxx112
-rw-r--r--src/libserver/composites/composites_manager.cxx330
-rw-r--r--src/libserver/css/CMakeLists.txt9
-rw-r--r--src/libserver/css/css.cxx227
-rw-r--r--src/libserver/css/css.hxx68
-rw-r--r--src/libserver/css/css_colors_list.hxx738
-rw-r--r--src/libserver/css/css_parser.cxx892
-rw-r--r--src/libserver/css/css_parser.hxx244
-rw-r--r--src/libserver/css/css_property.cxx69
-rw-r--r--src/libserver/css/css_property.hxx172
-rw-r--r--src/libserver/css/css_rule.cxx531
-rw-r--r--src/libserver/css/css_rule.hxx153
-rw-r--r--src/libserver/css/css_rule_parser.rl27
-rw-r--r--src/libserver/css/css_selector.cxx226
-rw-r--r--src/libserver/css/css_selector.hxx134
-rw-r--r--src/libserver/css/css_selector_parser.rl27
-rw-r--r--src/libserver/css/css_style.hxx66
-rw-r--r--src/libserver/css/css_syntax.rl110
-rw-r--r--src/libserver/css/css_tokeniser.cxx836
-rw-r--r--src/libserver/css/css_tokeniser.hxx215
-rw-r--r--src/libserver/css/css_util.cxx157
-rw-r--r--src/libserver/css/css_util.hxx37
-rw-r--r--src/libserver/css/css_value.cxx449
-rw-r--r--src/libserver/css/css_value.hxx174
-rw-r--r--src/libserver/css/parse_error.hxx61
-rw-r--r--src/libserver/dkim.c3588
-rw-r--r--src/libserver/dkim.h298
-rw-r--r--src/libserver/dns.c1124
-rw-r--r--src/libserver/dns.h110
-rw-r--r--src/libserver/dynamic_cfg.c743
-rw-r--r--src/libserver/dynamic_cfg.h81
-rw-r--r--src/libserver/fuzzy_backend/fuzzy_backend.c560
-rw-r--r--src/libserver/fuzzy_backend/fuzzy_backend.h131
-rw-r--r--src/libserver/fuzzy_backend/fuzzy_backend_redis.c1666
-rw-r--r--src/libserver/fuzzy_backend/fuzzy_backend_redis.h67
-rw-r--r--src/libserver/fuzzy_backend/fuzzy_backend_sqlite.c1029
-rw-r--r--src/libserver/fuzzy_backend/fuzzy_backend_sqlite.h107
-rw-r--r--src/libserver/fuzzy_wire.h154
-rw-r--r--src/libserver/html/html.cxx2393
-rw-r--r--src/libserver/html/html.h137
-rw-r--r--src/libserver/html/html.hxx146
-rw-r--r--src/libserver/html/html_block.hxx358
-rw-r--r--src/libserver/html/html_entities.cxx2644
-rw-r--r--src/libserver/html/html_entities.hxx31
-rw-r--r--src/libserver/html/html_tag.hxx159
-rw-r--r--src/libserver/html/html_tag_defs.hxx194
-rw-r--r--src/libserver/html/html_tags.h176
-rw-r--r--src/libserver/html/html_tests.cxx304
-rw-r--r--src/libserver/html/html_url.cxx496
-rw-r--r--src/libserver/html/html_url.hxx68
-rw-r--r--src/libserver/http/http_connection.c2649
-rw-r--r--src/libserver/http/http_connection.h320
-rw-r--r--src/libserver/http/http_context.c670
-rw-r--r--src/libserver/http/http_context.h122
-rw-r--r--src/libserver/http/http_message.c725
-rw-r--r--src/libserver/http/http_message.h254
-rw-r--r--src/libserver/http/http_private.h129
-rw-r--r--src/libserver/http/http_router.c559
-rw-r--r--src/libserver/http/http_router.h149
-rw-r--r--src/libserver/http/http_util.c295
-rw-r--r--src/libserver/http/http_util.h47
-rw-r--r--src/libserver/hyperscan_tools.cxx627
-rw-r--r--src/libserver/hyperscan_tools.h77
-rw-r--r--src/libserver/logger.h403
-rw-r--r--src/libserver/logger/logger.c1319
-rw-r--r--src/libserver/logger/logger_console.c211
-rw-r--r--src/libserver/logger/logger_file.c510
-rw-r--r--src/libserver/logger/logger_private.h218
-rw-r--r--src/libserver/logger/logger_syslog.c143
-rw-r--r--src/libserver/maps/map.c3195
-rw-r--r--src/libserver/maps/map.h168
-rw-r--r--src/libserver/maps/map_helpers.c1845
-rw-r--r--src/libserver/maps/map_helpers.h269
-rw-r--r--src/libserver/maps/map_private.h226
-rw-r--r--src/libserver/mempool_vars_internal.h47
-rw-r--r--src/libserver/milter.c2232
-rw-r--r--src/libserver/milter.h188
-rw-r--r--src/libserver/milter_internal.h176
-rw-r--r--src/libserver/monitored.c735
-rw-r--r--src/libserver/monitored.h161
-rw-r--r--src/libserver/protocol.c2185
-rw-r--r--src/libserver/protocol.h130
-rw-r--r--src/libserver/protocol_internal.h99
-rw-r--r--src/libserver/re_cache.c2712
-rw-r--r--src/libserver/re_cache.h212
-rw-r--r--src/libserver/redis_pool.cxx663
-rw-r--r--src/libserver/redis_pool.h91
-rw-r--r--src/libserver/roll_history.c432
-rw-r--r--src/libserver/roll_history.h98
-rw-r--r--src/libserver/rspamd_control.c1334
-rw-r--r--src/libserver/rspamd_control.h328
-rw-r--r--src/libserver/rspamd_symcache.h578
-rw-r--r--src/libserver/spf.c2799
-rw-r--r--src/libserver/spf.h159
-rw-r--r--src/libserver/ssl_util.c1133
-rw-r--r--src/libserver/ssl_util.h120
-rw-r--r--src/libserver/symcache/symcache_c.cxx715
-rw-r--r--src/libserver/symcache/symcache_id_list.hxx95
-rw-r--r--src/libserver/symcache/symcache_impl.cxx1316
-rw-r--r--src/libserver/symcache/symcache_internal.hxx652
-rw-r--r--src/libserver/symcache/symcache_item.cxx652
-rw-r--r--src/libserver/symcache/symcache_item.hxx561
-rw-r--r--src/libserver/symcache/symcache_periodic.hxx89
-rw-r--r--src/libserver/symcache/symcache_runtime.cxx823
-rw-r--r--src/libserver/symcache/symcache_runtime.hxx209
-rw-r--r--src/libserver/task.c1975
-rw-r--r--src/libserver/task.h392
-rw-r--r--src/libserver/url.c4365
-rw-r--r--src/libserver/url.h430
-rw-r--r--src/libserver/worker_util.c2313
-rw-r--r--src/libserver/worker_util.h334
-rw-r--r--src/libstat/CMakeLists.txt25
-rw-r--r--src/libstat/backends/backends.h127
-rw-r--r--src/libstat/backends/cdb_backend.cxx491
-rw-r--r--src/libstat/backends/http_backend.cxx440
-rw-r--r--src/libstat/backends/mmaped_file.c1113
-rw-r--r--src/libstat/backends/redis_backend.cxx1132
-rw-r--r--src/libstat/backends/sqlite3_backend.c907
-rw-r--r--src/libstat/classifiers/bayes.c551
-rw-r--r--src/libstat/classifiers/classifiers.h109
-rw-r--r--src/libstat/classifiers/lua_classifier.c237
-rw-r--r--src/libstat/learn_cache/learn_cache.h79
-rw-r--r--src/libstat/learn_cache/redis_cache.cxx254
-rw-r--r--src/libstat/learn_cache/sqlite3_cache.c274
-rw-r--r--src/libstat/stat_api.h147
-rw-r--r--src/libstat/stat_config.c603
-rw-r--r--src/libstat/stat_internal.h134
-rw-r--r--src/libstat/stat_process.c1250
-rw-r--r--src/libstat/tokenizers/osb.c424
-rw-r--r--src/libstat/tokenizers/tokenizers.c955
-rw-r--r--src/libstat/tokenizers/tokenizers.h100
-rw-r--r--src/libutil/CMakeLists.txt24
-rw-r--r--src/libutil/addr.c2049
-rw-r--r--src/libutil/addr.h356
-rw-r--r--src/libutil/cxx/error.hxx161
-rw-r--r--src/libutil/cxx/file_util.cxx457
-rw-r--r--src/libutil/cxx/file_util.hxx312
-rw-r--r--src/libutil/cxx/hash_util.hxx109
-rw-r--r--src/libutil/cxx/local_shared_ptr.hxx440
-rw-r--r--src/libutil/cxx/utf8_util.cxx421
-rw-r--r--src/libutil/cxx/utf8_util.h85
-rw-r--r--src/libutil/cxx/util.hxx238
-rw-r--r--src/libutil/cxx/util_tests.cxx82
-rw-r--r--src/libutil/expression.c1635
-rw-r--r--src/libutil/expression.h173
-rw-r--r--src/libutil/fstring.c482
-rw-r--r--src/libutil/fstring.h231
-rw-r--r--src/libutil/hash.c716
-rw-r--r--src/libutil/hash.h114
-rw-r--r--src/libutil/heap.c197
-rw-r--r--src/libutil/heap.h97
-rw-r--r--src/libutil/libev_helper.c111
-rw-r--r--src/libutil/libev_helper.h86
-rw-r--r--src/libutil/mem_pool.c1327
-rw-r--r--src/libutil/mem_pool.h470
-rw-r--r--src/libutil/mem_pool_internal.h92
-rw-r--r--src/libutil/multipattern.c821
-rw-r--r--src/libutil/multipattern.h173
-rw-r--r--src/libutil/printf.c1097
-rw-r--r--src/libutil/printf.h96
-rw-r--r--src/libutil/radix.c434
-rw-r--r--src/libutil/radix.h123
-rw-r--r--src/libutil/ref.h91
-rw-r--r--src/libutil/regexp.c1359
-rw-r--r--src/libutil/regexp.h276
-rw-r--r--src/libutil/rrd.c1502
-rw-r--r--src/libutil/rrd.h362
-rw-r--r--src/libutil/shingles.c412
-rw-r--r--src/libutil/shingles.h101
-rw-r--r--src/libutil/sqlite_utils.c620
-rw-r--r--src/libutil/sqlite_utils.h90
-rw-r--r--src/libutil/str_util.c3886
-rw-r--r--src/libutil/str_util.h565
-rw-r--r--src/libutil/unix-std.h79
-rw-r--r--src/libutil/upstream.c1761
-rw-r--r--src/libutil/upstream.h344
-rw-r--r--src/libutil/uthash_strcase.h91
-rw-r--r--src/libutil/util.c2746
-rw-r--r--src/libutil/util.h581
-rw-r--r--src/lua/CMakeLists.txt39
-rw-r--r--src/lua/lua_cdb.c391
-rw-r--r--src/lua/lua_cfg_file.c157
-rw-r--r--src/lua/lua_classifier.c230
-rw-r--r--src/lua/lua_common.c2659
-rw-r--r--src/lua/lua_common.h729
-rw-r--r--src/lua/lua_compress.c622
-rw-r--r--src/lua/lua_compress.h37
-rw-r--r--src/lua/lua_config.c4780
-rw-r--r--src/lua/lua_cryptobox.c3065
-rw-r--r--src/lua/lua_dns.c198
-rw-r--r--src/lua/lua_dns_resolver.c754
-rw-r--r--src/lua/lua_dns_resolver.h15
-rw-r--r--src/lua/lua_expression.c512
-rw-r--r--src/lua/lua_html.cxx738
-rw-r--r--src/lua/lua_http.c1270
-rw-r--r--src/lua/lua_ip.c637
-rw-r--r--src/lua/lua_kann.c1361
-rw-r--r--src/lua/lua_logger.c1068
-rw-r--r--src/lua/lua_map.c1421
-rw-r--r--src/lua/lua_map.h38
-rw-r--r--src/lua/lua_mempool.c612
-rw-r--r--src/lua/lua_mimepart.c2304
-rw-r--r--src/lua/lua_parsers.c410
-rw-r--r--src/lua/lua_parsers.h88
-rw-r--r--src/lua/lua_redis.c1662
-rw-r--r--src/lua/lua_regexp.c858
-rw-r--r--src/lua/lua_rsa.c867
-rw-r--r--src/lua/lua_spf.c620
-rw-r--r--src/lua/lua_sqlite3.c379
-rw-r--r--src/lua/lua_task.c7295
-rw-r--r--src/lua/lua_tcp.c2566
-rw-r--r--src/lua/lua_tensor.c817
-rw-r--r--src/lua/lua_tensor.h34
-rw-r--r--src/lua/lua_text.c1789
-rw-r--r--src/lua/lua_thread_pool.cxx369
-rw-r--r--src/lua/lua_thread_pool.h194
-rw-r--r--src/lua/lua_trie.c500
-rw-r--r--src/lua/lua_udp.c594
-rw-r--r--src/lua/lua_upstream.c672
-rw-r--r--src/lua/lua_url.c1481
-rw-r--r--src/lua/lua_url.h87
-rw-r--r--src/lua/lua_util.c3585
-rw-r--r--src/lua/lua_worker.c883
-rw-r--r--src/lua/lua_xmlrpc.c796
-rw-r--r--src/lua/rspamd.luadoc124
-rw-r--r--src/plugins/chartable.cxx2122
-rw-r--r--src/plugins/dkim_check.c1620
-rw-r--r--src/plugins/fuzzy_check.c4695
-rw-r--r--src/plugins/lua/antivirus.lua348
-rw-r--r--src/plugins/lua/arc.lua853
-rw-r--r--src/plugins/lua/asn.lua168
-rw-r--r--src/plugins/lua/aws_s3.lua269
-rw-r--r--src/plugins/lua/bayes_expiry.lua503
-rw-r--r--src/plugins/lua/bimi.lua391
-rw-r--r--src/plugins/lua/clickhouse.lua1556
-rw-r--r--src/plugins/lua/clustering.lua322
-rw-r--r--src/plugins/lua/dcc.lua119
-rw-r--r--src/plugins/lua/dkim_signing.lua186
-rw-r--r--src/plugins/lua/dmarc.lua685
-rw-r--r--src/plugins/lua/dynamic_conf.lua333
-rw-r--r--src/plugins/lua/elastic.lua544
-rw-r--r--src/plugins/lua/emails.lua4
-rw-r--r--src/plugins/lua/external_relay.lua285
-rw-r--r--src/plugins/lua/external_services.lua408
-rw-r--r--src/plugins/lua/force_actions.lua227
-rw-r--r--src/plugins/lua/forged_recipients.lua183
-rw-r--r--src/plugins/lua/fuzzy_collect.lua193
-rw-r--r--src/plugins/lua/greylist.lua542
-rw-r--r--src/plugins/lua/hfilter.lua622
-rw-r--r--src/plugins/lua/history_redis.lua314
-rw-r--r--src/plugins/lua/http_headers.lua198
-rw-r--r--src/plugins/lua/ip_score.lua4
-rw-r--r--src/plugins/lua/known_senders.lua245
-rw-r--r--src/plugins/lua/maillist.lua235
-rw-r--r--src/plugins/lua/maps_stats.lua133
-rw-r--r--src/plugins/lua/metadata_exporter.lua707
-rw-r--r--src/plugins/lua/metric_exporter.lua252
-rw-r--r--src/plugins/lua/mid.lua123
-rw-r--r--src/plugins/lua/milter_headers.lua762
-rw-r--r--src/plugins/lua/mime_types.lua737
-rw-r--r--src/plugins/lua/multimap.lua1403
-rw-r--r--src/plugins/lua/mx_check.lua392
-rw-r--r--src/plugins/lua/neural.lua1000
-rw-r--r--src/plugins/lua/once_received.lua230
-rw-r--r--src/plugins/lua/p0f.lua124
-rw-r--r--src/plugins/lua/phishing.lua667
-rw-r--r--src/plugins/lua/ratelimit.lua868
-rw-r--r--src/plugins/lua/rbl.lua1425
-rw-r--r--src/plugins/lua/replies.lua328
-rw-r--r--src/plugins/lua/reputation.lua1390
-rw-r--r--src/plugins/lua/rspamd_update.lua161
-rw-r--r--src/plugins/lua/settings.lua1437
-rw-r--r--src/plugins/lua/spamassassin.lua1774
-rw-r--r--src/plugins/lua/spamtrap.lua200
-rw-r--r--src/plugins/lua/spf.lua242
-rw-r--r--src/plugins/lua/trie.lua184
-rw-r--r--src/plugins/lua/url_redirector.lua422
-rw-r--r--src/plugins/lua/whitelist.lua443
-rw-r--r--src/plugins/regexp.c564
-rw-r--r--src/ragel/content_disposition.rl36
-rw-r--r--src/ragel/content_disposition_parser.rl126
-rw-r--r--src/ragel/content_type.rl36
-rw-r--r--src/ragel/rfc2047_parser.rl85
-rw-r--r--src/ragel/smtp_addr_parser.rl103
-rw-r--r--src/ragel/smtp_address.rl36
-rw-r--r--src/ragel/smtp_base.rl44
-rw-r--r--src/ragel/smtp_date.rl230
-rw-r--r--src/ragel/smtp_date_parser.rl47
-rw-r--r--src/ragel/smtp_ip.rl36
-rw-r--r--src/ragel/smtp_ip_parser.rl56
-rw-r--r--src/rspamadm/CMakeLists.txt31
-rw-r--r--src/rspamadm/commands.c280
-rw-r--r--src/rspamadm/configdump.c555
-rw-r--r--src/rspamadm/confighelp.c301
-rw-r--r--src/rspamadm/configtest.c190
-rw-r--r--src/rspamadm/control.c258
-rw-r--r--src/rspamadm/fuzzy_convert.c161
-rw-r--r--src/rspamadm/lua_repl.c1026
-rw-r--r--src/rspamadm/pw.c392
-rw-r--r--src/rspamadm/rspamadm.c621
-rw-r--r--src/rspamadm/rspamadm.h92
-rw-r--r--src/rspamadm/signtool.c623
-rw-r--r--src/rspamadm/stat_convert.c262
-rw-r--r--src/rspamd.c1729
-rw-r--r--src/rspamd.h378
-rw-r--r--src/rspamd_proxy.c2441
-rw-r--r--src/worker.c537
-rw-r--r--src/worker_private.h72
-rw-r--r--test/.depends6
-rw-r--r--test/CMakeLists.txt50
-rw-r--r--test/coverage.md58
-rw-r--r--test/functional/cases/001_merged/100_general.robot61
-rw-r--r--test/functional/cases/001_merged/101_lua.robot50
-rw-r--r--test/functional/cases/001_merged/102_multimap.robot436
-rw-r--r--test/functional/cases/001_merged/104_get_from.robot57
-rw-r--r--test/functional/cases/001_merged/105_mimetypes.robot74
-rw-r--r--test/functional/cases/001_merged/106_mid.robot36
-rw-r--r--test/functional/cases/001_merged/114_phishing.robot26
-rw-r--r--test/functional/cases/001_merged/115_dmarc.robot101
-rw-r--r--test/functional/cases/001_merged/117_spf.robot157
-rw-r--r--test/functional/cases/001_merged/160_antivirus.robot151
-rw-r--r--test/functional/cases/001_merged/240_redis.robot16
-rw-r--r--test/functional/cases/001_merged/250_dns.robot21
-rw-r--r--test/functional/cases/001_merged/270_selector.robot19
-rw-r--r--test/functional/cases/001_merged/280_rules.robot139
-rw-r--r--test/functional/cases/001_merged/281_fnames.robot13
-rw-r--r--test/functional/cases/001_merged/290_greylist.robot25
-rw-r--r--test/functional/cases/001_merged/300_rbl.robot98
-rw-r--r--test/functional/cases/001_merged/310_udp.robot40
-rw-r--r--test/functional/cases/001_merged/321_arc_check.robot19
-rw-r--r--test/functional/cases/001_merged/340_surbl.robot182
-rw-r--r--test/functional/cases/001_merged/350_magic.robot67
-rw-r--r--test/functional/cases/001_merged/__init__.robot28
-rw-r--r--test/functional/cases/101_lua.robot35
-rw-r--r--test/functional/cases/103_password.robot71
-rw-r--r--test/functional/cases/108_settings.robot268
-rw-r--r--test/functional/cases/109_composites.robot78
-rw-r--r--test/functional/cases/110_statistics/090-peruser.robot15
-rw-r--r--test/functional/cases/110_statistics/100-redis-keyed-siphash.robot16
-rw-r--r--test/functional/cases/110_statistics/110-redis-keyed-xxhash.robot16
-rw-r--r--test/functional/cases/110_statistics/200-redis-plain-siphash.robot15
-rw-r--r--test/functional/cases/110_statistics/210-redis-plain-xxhash.robot15
-rw-r--r--test/functional/cases/110_statistics/lib.robot69
-rw-r--r--test/functional/cases/116_dkim.robot59
-rw-r--r--test/functional/cases/120_fuzzy/encrypted.robot14
-rw-r--r--test/functional/cases/120_fuzzy/fasthash-keyed.robot14
-rw-r--r--test/functional/cases/120_fuzzy/fasthash.robot14
-rw-r--r--test/functional/cases/120_fuzzy/general.robot17
-rw-r--r--test/functional/cases/120_fuzzy/lib.robot181
-rw-r--r--test/functional/cases/120_fuzzy/mumhash-keyed.robot14
-rw-r--r--test/functional/cases/120_fuzzy/mumhash.robot14
-rw-r--r--test/functional/cases/120_fuzzy/siphash-keyed.robot14
-rw-r--r--test/functional/cases/120_fuzzy/siphash.robot14
-rw-r--r--test/functional/cases/120_fuzzy/xxhash-keyed.robot15
-rw-r--r--test/functional/cases/120_fuzzy/xxhash.robot15
-rw-r--r--test/functional/cases/121_json/100_preresult.robot23
-rw-r--r--test/functional/cases/121_json/101_simple.robot23
-rw-r--r--test/functional/cases/121_json/lib.robot22
-rw-r--r--test/functional/cases/123_whitelist.robot79
-rw-r--r--test/functional/cases/125_map_reload.robot47
-rw-r--r--test/functional/cases/131_dkim_signing/001_simple.robot35
-rw-r--r--test/functional/cases/131_dkim_signing/002_redis.robot31
-rw-r--r--test/functional/cases/131_dkim_signing/003_eddsa.robot30
-rw-r--r--test/functional/cases/131_dkim_signing/004_invalidate_key.robot50
-rw-r--r--test/functional/cases/131_dkim_signing/005_multiple.robot20
-rw-r--r--test/functional/cases/131_dkim_signing/006_milter.robot31
-rw-r--r--test/functional/cases/131_dkim_signing/007_sign_maps.robot25
-rw-r--r--test/functional/cases/135_spamassassin.robot34
-rw-r--r--test/functional/cases/140_proxy.robot50
-rw-r--r--test/functional/cases/150_rspamadm.robot45
-rw-r--r--test/functional/cases/151_rspamadm_async.robot54
-rw-r--r--test/functional/cases/161_p0f.robot97
-rw-r--r--test/functional/cases/162_url_redirector.robot34
-rw-r--r--test/functional/cases/180_milter.robot40
-rw-r--r--test/functional/cases/210_clickhouse/001_migration.robot91
-rw-r--r--test/functional/cases/210_clickhouse/clickhouse.py78
-rw-r--r--test/functional/cases/220_http.robot74
-rw-r--r--test/functional/cases/230_tcp.robot75
-rw-r--r--test/functional/cases/231_tcp_down.robot28
-rw-r--r--test/functional/cases/241_redis_is_dead.robot26
-rw-r--r--test/functional/cases/260_regex.robot33
-rw-r--r--test/functional/cases/320_arc_signing/001_simple.robot31
-rw-r--r--test/functional/cases/320_arc_signing/002_redis.robot29
-rw-r--r--test/functional/cases/330_neural/001_autotrain.robot63
-rw-r--r--test/functional/cases/330_neural/002_manualtrain.robot72
-rw-r--r--test/functional/cases/360_force_actions.robot44
-rw-r--r--test/functional/cases/380_external_relay.robot48
-rw-r--r--test/functional/cases/400_known_senders.robot36
-rw-r--r--test/functional/cases/410_logging/000_console/000_systemd_logger.robot24
-rw-r--r--test/functional/cases/410_logging/000_console/001_timestamps.robot24
-rw-r--r--test/functional/cases/410_logging/001_file/000_json.robot17
-rw-r--r--test/functional/cases/__init__.robot19
-rw-r--r--test/functional/configs/arc_signing/redis.conf10
-rw-r--r--test/functional/configs/arc_signing/simple.conf10
-rw-r--r--test/functional/configs/clickhouse-config.xml70
-rw-r--r--test/functional/configs/clickhouse-users.xml109
-rw-r--r--test/functional/configs/clickhouse.conf63
-rw-r--r--test/functional/configs/composites.conf87
-rw-r--r--test/functional/configs/dkim-eddsa.key1
-rw-r--r--test/functional/configs/dkim.conf66
-rw-r--r--test/functional/configs/dkim.key16
-rw-r--r--test/functional/configs/dkim_signing/eddsa.conf8
-rw-r--r--test/functional/configs/dkim_signing/invalidate.conf8
-rw-r--r--test/functional/configs/dkim_signing/milter.conf76
-rw-r--r--test/functional/configs/dkim_signing/multiple.conf17
-rw-r--r--test/functional/configs/dkim_signing/redis.conf10
-rw-r--r--test/functional/configs/dkim_signing/sign_maps.conf11
-rw-r--r--test/functional/configs/dkim_signing/simple.conf9
-rw-r--r--test/functional/configs/dynamic.conf117
-rw-r--r--test/functional/configs/empty.conf0
-rw-r--r--test/functional/configs/force_actions.conf88
-rw-r--r--test/functional/configs/fuzzy-encryption-key.conf2
-rw-r--r--test/functional/configs/fuzzy.conf95
-rw-r--r--test/functional/configs/known_senders-local.conf4
-rw-r--r--test/functional/configs/known_senders.conf7
-rw-r--r--test/functional/configs/loggingtest-local.conf5
-rw-r--r--test/functional/configs/loggingtest.conf3
-rw-r--r--test/functional/configs/lua_script.conf25
-rw-r--r--test/functional/configs/lua_test.conf52
-rw-r--r--test/functional/configs/maps/domains.cdbbin0 -> 2084 bytes
-rw-r--r--test/functional/configs/maps/domains.list5
-rw-r--r--test/functional/configs/maps/domains.list.216
-rw-r--r--test/functional/configs/maps/dynamic_symbols.map2
-rw-r--r--test/functional/configs/maps/external_relay.hostname_map3
-rw-r--r--test/functional/configs/maps/external_relay.ip_map2
-rw-r--r--test/functional/configs/maps/external_relay.user_map2
-rw-r--r--test/functional/configs/maps/external_relay_ip.list1
-rw-r--r--test/functional/configs/maps/ip.list4
-rw-r--r--test/functional/configs/maps/ip2.list3
-rw-r--r--test/functional/configs/maps/known_senders_domains.map2
-rw-r--r--test/functional/configs/maps/map.list3
-rw-r--r--test/functional/configs/maps/mid.list2
-rw-r--r--test/functional/configs/maps/mime_types.wl1
-rw-r--r--test/functional/configs/maps/multiple.list12
-rw-r--r--test/functional/configs/maps/rcvd.list1
-rw-r--r--test/functional/configs/maps/rcvd2.list2
-rw-r--r--test/functional/configs/maps/redir.map3
-rw-r--r--test/functional/configs/maps/regexp.list5
-rw-r--r--test/functional/configs/maps/strict.phishing1
-rw-r--r--test/functional/configs/maps/stricter.phishing1
-rw-r--r--test/functional/configs/maps/top.list2
-rw-r--r--test/functional/configs/maps/url_compose_map.list3
-rw-r--r--test/functional/configs/maps/url_compose_map_for_mails.list3
-rw-r--r--test/functional/configs/maps/users.list1
-rw-r--r--test/functional/configs/maps/utf.list1
-rw-r--r--test/functional/configs/merged-local.conf972
-rw-r--r--test/functional/configs/merged-override.conf435
-rw-r--r--test/functional/configs/merged.conf39
-rw-r--r--test/functional/configs/milter.conf61
-rw-r--r--test/functional/configs/neural.conf83
-rw-r--r--test/functional/configs/neural_noauto.conf85
-rw-r--r--test/functional/configs/nginx.conf20
-rw-r--r--test/functional/configs/p0f.conf13
-rw-r--r--test/functional/configs/password.conf45
-rw-r--r--test/functional/configs/plugins.conf770
-rw-r--r--test/functional/configs/proxy.conf26
-rw-r--r--test/functional/configs/redis-server.conf7
-rw-r--r--test/functional/configs/redis.conf7
-rw-r--r--test/functional/configs/regexp.conf64
-rw-r--r--test/functional/configs/settings.conf117
-rw-r--r--test/functional/configs/spamassassin.conf7
-rw-r--r--test/functional/configs/spamassassin.rules97
-rw-r--r--test/functional/configs/stats.conf85
-rw-r--r--test/functional/configs/trivial.conf50
-rw-r--r--test/functional/configs/url_redirector.conf8
-rw-r--r--test/functional/configs/url_tags.conf9
-rw-r--r--test/functional/configs/whitelist.conf83
-rw-r--r--test/functional/data/bayes.ham.sqlite3bin0 -> 102400 bytes
-rw-r--r--test/functional/data/bayes.spam.sqlite3bin0 -> 327680 bytes
-rw-r--r--test/functional/data/initial_schema/data.rspamd.sql56
-rw-r--r--test/functional/data/initial_schema/data.rspamd_asn.sql66
-rw-r--r--test/functional/data/initial_schema/data.rspamd_attachments.sql10
-rw-r--r--test/functional/data/initial_schema/data.rspamd_emails.sql6
-rw-r--r--test/functional/data/initial_schema/data.rspamd_symbols.sql10
-rw-r--r--test/functional/data/initial_schema/data.rspamd_urls.sql10
-rw-r--r--test/functional/data/initial_schema/schema.sql70
-rw-r--r--test/functional/data/schema_2/data.rspamd.sql56
-rw-r--r--test/functional/data/schema_2/schema.sql43
-rw-r--r--test/functional/lib/.gitignore1
-rw-r--r--test/functional/lib/rspamd.py357
-rw-r--r--test/functional/lib/rspamd.robot398
-rw-r--r--test/functional/lib/vars.py30
-rw-r--r--test/functional/lua/composites.lua140
-rw-r--r--test/functional/lua/conditions.lua22
-rw-r--r--test/functional/lua/deps.lua30
-rw-r--r--test/functional/lua/dns.lua53
-rw-r--r--test/functional/lua/external_relay.lua10
-rw-r--r--test/functional/lua/flags.lua35
-rw-r--r--test/functional/lua/get_from.lua10
-rw-r--r--test/functional/lua/hashes.lua74
-rw-r--r--test/functional/lua/http.lua169
-rw-r--r--test/functional/lua/magic.lua14
-rw-r--r--test/functional/lua/mapreload.lua20
-rw-r--r--test/functional/lua/maps_kv.lua94
-rw-r--r--test/functional/lua/miltertest/combined.lua35
-rw-r--r--test/functional/lua/miltertest/data.lua26
-rw-r--r--test/functional/lua/miltertest/data_dkim.lua23
-rw-r--r--test/functional/lua/miltertest/dkim_many.lua11
-rw-r--r--test/functional/lua/miltertest/dkim_one.lua11
-rw-r--r--test/functional/lua/miltertest/lib.lua119
-rw-r--r--test/functional/lua/miltertest/mt1.lua11
-rw-r--r--test/functional/lua/miltertest/mt2.lua11
-rw-r--r--test/functional/lua/miltertest/mt3.lua12
-rw-r--r--test/functional/lua/miltertest/mt4.lua11
-rw-r--r--test/functional/lua/neural.lua64
-rw-r--r--test/functional/lua/option_order.lua15
-rw-r--r--test/functional/lua/params.lua80
-rw-r--r--test/functional/lua/prepostfilters.lua65
-rw-r--r--test/functional/lua/preresult.lua3
-rw-r--r--test/functional/lua/recipients.lua14
-rw-r--r--test/functional/lua/redis.lua113
-rw-r--r--test/functional/lua/regex_test.lua12
-rw-r--r--test/functional/lua/remove_result.lua26
-rw-r--r--test/functional/lua/rspamadm/test_batch.lua4
-rw-r--r--test/functional/lua/rspamadm/test_dns_client.lua30
-rw-r--r--test/functional/lua/rspamadm/test_message_callback.lua5
-rw-r--r--test/functional/lua/rspamadm/test_redis_client.lua40
-rw-r--r--test/functional/lua/rspamadm/test_tcp_client.lua64
-rw-r--r--test/functional/lua/rspamadm/test_verbose.lua3
-rw-r--r--test/functional/lua/selector_test.lua23
-rw-r--r--test/functional/lua/settings.lua68
-rw-r--r--test/functional/lua/simple.lua7
-rw-r--r--test/functional/lua/tcp.lua279
-rw-r--r--test/functional/lua/test_coverage.lua46
-rw-r--r--test/functional/lua/test_fname.lua12
-rw-r--r--test/functional/lua/tlds.lua58
-rw-r--r--test/functional/lua/udp.lua81
-rw-r--r--test/functional/messages/arcallow.eml107
-rw-r--r--test/functional/messages/arcbad.eml194
-rw-r--r--test/functional/messages/bad_ext.dotted_file_name.eml9
-rw-r--r--test/functional/messages/bad_message.eml4270
-rw-r--r--test/functional/messages/badboundary.eml17
-rw-r--r--test/functional/messages/broken_richtext.eml27
-rw-r--r--test/functional/messages/btc.eml22
-rw-r--r--test/functional/messages/btc2.eml22
-rw-r--r--test/functional/messages/btc3.eml22
-rw-r--r--test/functional/messages/btc4.eml26
-rw-r--r--test/functional/messages/content_url.eml202
-rw-r--r--test/functional/messages/currency.eml5
-rw-r--r--test/functional/messages/dmarc/bad_dkim1.eml17
-rw-r--r--test/functional/messages/dmarc/bad_dkim2.eml17
-rw-r--r--test/functional/messages/dmarc/bad_dkim3.eml17
-rw-r--r--test/functional/messages/dmarc/bad_dkim4.eml17
-rw-r--r--test/functional/messages/dmarc/bad_dkim_highsecure.eml61
-rw-r--r--test/functional/messages/dmarc/bad_dkim_rspamd.eml61
-rw-r--r--test/functional/messages/dmarc/dkim_unknown_tags.eml19
-rw-r--r--test/functional/messages/dmarc/dmarc_tmpfail.eml17
-rw-r--r--test/functional/messages/dmarc/fail_none.eml3
-rw-r--r--test/functional/messages/dmarc/fail_none1.eml3
-rw-r--r--test/functional/messages/dmarc/fail_quarantine.eml3
-rw-r--r--test/functional/messages/dmarc/fail_reject.eml3
-rw-r--r--test/functional/messages/dmarc/good_dkim_highsecure.eml60
-rw-r--r--test/functional/messages/dmarc/good_dkim_rspamd.eml60
-rw-r--r--test/functional/messages/dmarc/no_dkim_valid_spf.eml48
-rw-r--r--test/functional/messages/dmarc/onsubdomain_fail.eml3
-rw-r--r--test/functional/messages/dmarc/onsubdomain_fail_alignment.eml11
-rw-r--r--test/functional/messages/dmarc/onsubdomain_pass_relaxed.eml11
-rw-r--r--test/functional/messages/dmarc/pass_none.eml18
-rw-r--r--test/functional/messages/dmarc/pct_none.eml17
-rw-r--r--test/functional/messages/dmarc/pct_none1.eml17
-rw-r--r--test/functional/messages/dmarc/subdomain_fail_none.eml3
-rw-r--r--test/functional/messages/dmarc/subdomain_fail_quarantine.eml3
-rw-r--r--test/functional/messages/dmarc/subdomain_fail_reject.eml3
-rw-r--r--test/functional/messages/ed25519-broken.eml26
-rw-r--r--test/functional/messages/ed25519.eml26
-rw-r--r--test/functional/messages/emailbltext.eml4
-rw-r--r--test/functional/messages/emailbltext2.eml3
-rw-r--r--test/functional/messages/emailbltext3.eml4
-rw-r--r--test/functional/messages/emails1.eml8
-rw-r--r--test/functional/messages/empty-plain-text.eml15
-rw-r--r--test/functional/messages/empty_part.eml22
-rw-r--r--test/functional/messages/exe_attm.eml22
-rw-r--r--test/functional/messages/external_relay.eml16
-rw-r--r--test/functional/messages/f.zip.001.eml11
-rw-r--r--test/functional/messages/f.zip.gz.eml11
-rw-r--r--test/functional/messages/fname.eml29
-rw-r--r--test/functional/messages/freemail.eml9
-rw-r--r--test/functional/messages/freemailcc.eml10
-rw-r--r--test/functional/messages/from/from.eml2
-rw-r--r--test/functional/messages/from/from_comment.eml2
-rw-r--r--test/functional/messages/from/from_dn.eml2
-rw-r--r--test/functional/messages/from/from_dn_base64.eml2
-rw-r--r--test/functional/messages/from/from_dn_comment.eml2
-rw-r--r--test/functional/messages/from/from_quoted_dn.eml2
-rw-r--r--test/functional/messages/from/from_quoted_dn_comment.eml2
-rw-r--r--test/functional/messages/from/from_quoted_dn_middle.eml2
-rw-r--r--test/functional/messages/from/from_quoted_dn_middle_inner.eml2
-rw-r--r--test/functional/messages/fws_fn.eml30
-rw-r--r--test/functional/messages/fws_fp.eml56
-rw-r--r--test/functional/messages/fws_tp.eml56
-rw-r--r--test/functional/messages/gargantua.eml23505
-rw-r--r--test/functional/messages/gtube.eml56
-rw-r--r--test/functional/messages/ham.eml269
-rw-r--r--test/functional/messages/ics.eml53
-rw-r--r--test/functional/messages/invalid_mid_allowed.eml10
-rw-r--r--test/functional/messages/mailadr.eml4
-rw-r--r--test/functional/messages/mailadr2.eml4
-rw-r--r--test/functional/messages/newlines.eml15
-rw-r--r--test/functional/messages/next2last-digits_in_brackets.eml9
-rw-r--r--test/functional/messages/next2last-digits_in_parens.eml9
-rw-r--r--test/functional/messages/next2last-doublebad.eml9
-rw-r--r--test/functional/messages/numeric_urls.eml226
-rw-r--r--test/functional/messages/pdf_encrypted.eml692
-rw-r--r--test/functional/messages/pdf_js.eml1800
-rw-r--r--test/functional/messages/phish_sender.eml15
-rw-r--r--test/functional/messages/phish_sender2.eml16
-rw-r--r--test/functional/messages/phish_sender3.eml15
-rw-r--r--test/functional/messages/phish_sender4.eml16
-rw-r--r--test/functional/messages/phish_sender5.eml15
-rw-r--r--test/functional/messages/phish_sender6.eml14
-rw-r--r--test/functional/messages/phishing1.eml3
-rw-r--r--test/functional/messages/phishing2.eml3
-rw-r--r--test/functional/messages/phishing3.eml3
-rw-r--r--test/functional/messages/priority.eml5
-rw-r--r--test/functional/messages/rar-date-bad-ext.eml10
-rw-r--r--test/functional/messages/rar4.eml10
-rw-r--r--test/functional/messages/rcvd7.eml34
-rw-r--r--test/functional/messages/received1.eml16
-rw-r--r--test/functional/messages/received2.eml16
-rw-r--r--test/functional/messages/received3.eml16
-rw-r--r--test/functional/messages/received4.eml16
-rw-r--r--test/functional/messages/received5.eml13
-rw-r--r--test/functional/messages/received6.eml17
-rw-r--r--test/functional/messages/received7.eml17
-rw-r--r--test/functional/messages/redir.eml6
-rw-r--r--test/functional/messages/replyto.eml5
-rw-r--r--test/functional/messages/replyto2.eml5
-rw-r--r--test/functional/messages/replyto_addr_eq_from.eml5
-rw-r--r--test/functional/messages/replytosubdomain.eml5
-rw-r--r--test/functional/messages/spam.eml760
-rw-r--r--test/functional/messages/spam_message.eml59
-rw-r--r--test/functional/messages/spam_message.eml.fuzzy134
-rw-r--r--test/functional/messages/subject1.eml3
-rw-r--r--test/functional/messages/url1.eml3
-rw-r--r--test/functional/messages/url10.eml6
-rw-r--r--test/functional/messages/url11.eml4
-rw-r--r--test/functional/messages/url12.eml4
-rw-r--r--test/functional/messages/url13.eml4
-rw-r--r--test/functional/messages/url14.eml4
-rw-r--r--test/functional/messages/url15.eml3
-rw-r--r--test/functional/messages/url16.eml5
-rw-r--r--test/functional/messages/url2.eml3
-rw-r--r--test/functional/messages/url3.eml3
-rw-r--r--test/functional/messages/url4.eml4
-rw-r--r--test/functional/messages/url5.eml3
-rw-r--r--test/functional/messages/url6.eml3
-rw-r--r--test/functional/messages/url7.eml3
-rw-r--r--test/functional/messages/url8.eml3
-rw-r--r--test/functional/messages/url9.eml3
-rw-r--r--test/functional/messages/urlimage.eml5
-rw-r--r--test/functional/messages/urlinsubject.eml7
-rw-r--r--test/functional/messages/urlinsubjectencoded.eml9
-rw-r--r--test/functional/messages/utf.eml15
-rw-r--r--test/functional/messages/whitelist.eml3
-rw-r--r--test/functional/messages/yand_forward.eml31
-rw-r--r--test/functional/messages/zerofont.eml20
-rw-r--r--test/functional/messages/zip-doublebad.eml12
-rw-r--r--test/functional/messages/zip.eml11
-rwxr-xr-xtest/functional/util/dummy_avast.py53
-rwxr-xr-xtest/functional/util/dummy_clam.py50
-rwxr-xr-xtest/functional/util/dummy_fprot.py54
-rwxr-xr-xtest/functional/util/dummy_http.py145
-rw-r--r--test/functional/util/dummy_killer.py30
-rwxr-xr-xtest/functional/util/dummy_p0f.py98
-rwxr-xr-xtest/functional/util/dummy_ssl.py66
-rwxr-xr-xtest/functional/util/dummy_udp.py26
-rw-r--r--test/functional/util/nn_unpack.lua16
-rw-r--r--test/functional/util/server.pem46
-rw-r--r--test/lua/compat_env.lua391
-rw-r--r--test/lua/pcall_test.lua45
-rw-r--r--test/lua/rspamd_assertions.lua138
-rw-r--r--test/lua/rspamd_test_helper.lua45
-rw-r--r--test/lua/telescope.lua621
-rw-r--r--test/lua/tests.lua51
-rw-r--r--test/lua/unit/addr.lua46
-rw-r--r--test/lua/unit/base32.lua55
-rw-r--r--test/lua/unit/base64.lua194
-rw-r--r--test/lua/unit/compression.lua58
-rw-r--r--test/lua/unit/expressions.lua111
-rw-r--r--test/lua/unit/folding.lua66
-rw-r--r--test/lua/unit/fpconv.lua97
-rw-r--r--test/lua/unit/html.lua113
-rw-r--r--test/lua/unit/kann.lua46
-rw-r--r--test/lua/unit/logger.lua27
-rw-r--r--test/lua/unit/lua_mime.message_to_ucl.lua330
-rw-r--r--test/lua/unit/lua_util.extract_specific_urls.lua345
-rw-r--r--test/lua/unit/lua_util.misc.lua61
-rw-r--r--test/lua/unit/mempool.lua47
-rw-r--r--test/lua/unit/quoted_printable.lua164
-rw-r--r--test/lua/unit/regxep.lua90
-rw-r--r--test/lua/unit/rfc2047.lua92
-rw-r--r--test/lua/unit/rsa.lua50
-rw-r--r--test/lua/unit/rspamd_resolver.lua31
-rw-r--r--test/lua/unit/rspamd_text.lua79
-rw-r--r--test/lua/unit/rspamd_util.lua136
-rw-r--r--test/lua/unit/selectors.combined.lua130
-rw-r--r--test/lua/unit/selectors.custom.lua81
-rw-r--r--test/lua/unit/selectors.lua472
-rw-r--r--test/lua/unit/selectors.negative.lua113
-rw-r--r--test/lua/unit/smtp_addr.lua110
-rw-r--r--test/lua/unit/smtp_date.lua58
-rw-r--r--test/lua/unit/sqlite3.lua50
-rw-r--r--test/lua/unit/task.lua162
-rw-r--r--test/lua/unit/test.data10
-rw-r--r--test/lua/unit/test_tld.dat20
-rw-r--r--test/lua/unit/testkey.pub9
-rw-r--r--test/lua/unit/testkey.sec27
-rw-r--r--test/lua/unit/tokenizer.lua81
-rw-r--r--test/lua/unit/trie.lua81
-rw-r--r--test/lua/unit/url.lua253
-rw-r--r--test/lua/unit/utf.lua207
-rw-r--r--test/rspamd_cryptobox_test.c347
-rw-r--r--test/rspamd_cxx_local_ptr.hxx344
-rw-r--r--test/rspamd_cxx_unit.cxx86
-rw-r--r--test/rspamd_cxx_unit_dkim.hxx173
-rw-r--r--test/rspamd_cxx_unit_utils.hxx209
-rw-r--r--test/rspamd_dkim_test.c86
-rw-r--r--test/rspamd_dns_test.c103
-rw-r--r--test/rspamd_heap_test.c182
-rw-r--r--test/rspamd_lua_pcall_vs_resume_test.c160
-rw-r--r--test/rspamd_lua_test.c151
-rw-r--r--test/rspamd_mem_pool_test.c35
-rw-r--r--test/rspamd_radix_test.c376
-rw-r--r--test/rspamd_rrd_test.c101
-rw-r--r--test/rspamd_shingles_test.c339
-rw-r--r--test/rspamd_statfile_test.c47
-rw-r--r--test/rspamd_test_suite.c95
-rw-r--r--test/rspamd_upstream_test.c182
-rw-r--r--test/rspamd_url_test.c60
-rw-r--r--test/test.cfg16
-rw-r--r--test/test.cfg.sig1
-rw-r--r--test/tests.h51
-rwxr-xr-xtest/tools/dump_coveralls.py66
-rwxr-xr-xtest/tools/gcov_coveralls.py206
-rwxr-xr-xtest/tools/http_put.py51
-rwxr-xr-xtest/tools/merge_coveralls.py207
-rw-r--r--utils/CMakeLists.txt22
-rw-r--r--utils/asn.pl331
-rw-r--r--utils/base64.c89
-rw-r--r--utils/cgp_rspamd.pl357
-rw-r--r--utils/classifier_test.pl539
-rwxr-xr-xutils/fann_train.pl247
-rw-r--r--utils/rspamd_http_bench.c411
-rw-r--r--utils/rspamd_http_server.c300
-rwxr-xr-xutils/rspamd_stats.pl1018
-rw-r--r--utils/sa_trivial_convert.lua443
1605 files changed, 604977 insertions, 0 deletions
diff --git a/.clang-format b/.clang-format
new file mode 100644
index 0000000..89beadc
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,67 @@
+# Generated from CLion C/C++ Code Style settings
+BasedOnStyle: LLVM
+AccessModifierOffset: -4
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: None
+AlignOperands: Align
+AllowAllArgumentsOnNextLine: false
+AllowAllConstructorInitializersOnNextLine: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: Always
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: Always
+AllowShortLambdasOnASingleLine: All
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterReturnType: None
+AlwaysBreakTemplateDeclarations: Yes
+BreakBeforeBraces: Custom
+BraceWrapping:
+ AfterCaseLabel: false
+ AfterClass: false
+ AfterControlStatement: Never
+ AfterEnum: false
+ AfterFunction: true
+ AfterNamespace: false
+ AfterUnion: false
+ BeforeCatch: false
+ BeforeElse: true
+ IndentBraces: false
+ SplitEmptyFunction: true
+ SplitEmptyRecord: true
+BreakBeforeBinaryOperators: None
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializers: BeforeColon
+BreakInheritanceList: BeforeColon
+ColumnLimit: 0
+CompactNamespaces: false
+ContinuationIndentWidth: 4
+IndentCaseLabels: false
+IndentPPDirectives: None
+IndentWidth: 4
+KeepEmptyLinesAtTheStartOfBlocks: true
+MaxEmptyLinesToKeep: 2
+NamespaceIndentation: None
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PointerAlignment: Right
+ReflowComments: false
+SortIncludes: Never
+SpaceAfterCStyleCast: true
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 0
+SpacesInAngles: false
+SpacesInCStyleCastParentheses: false
+SpacesInContainerLiterals: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+TabWidth: 4
+UseTab: ForContinuationAndIndentation
diff --git a/.drone.jsonnet b/.drone.jsonnet
new file mode 100644
index 0000000..8d41197
--- /dev/null
+++ b/.drone.jsonnet
@@ -0,0 +1,310 @@
+local docker_pipeline = {
+ kind: 'pipeline',
+ type: 'docker',
+};
+
+local default_trigger_events_ex_pr = [
+ 'push',
+ 'tag',
+ 'custom',
+];
+
+local default_trigger = {
+ trigger: {
+ event: default_trigger_events_ex_pr + ['pull_request'],
+ },
+};
+
+local platform(arch) = {
+ platform: {
+ os: 'linux',
+ arch: arch,
+ },
+};
+
+local coveralls_attribs = {
+ branch: [
+ 'master',
+ ],
+ event: [
+ 'push',
+ 'tag',
+ ],
+};
+
+local close_coveralls_trigger = {
+ trigger: coveralls_attribs { status: ['success', 'failure'] },
+};
+
+local coveralls_when = {
+ when: coveralls_attribs,
+};
+
+local notify_pipeline = {
+ name: 'notify',
+ depends_on: [
+ 'default-amd64',
+ 'default-arm64',
+ 'default-noarch',
+ ],
+ steps: [
+ {
+ name: 'notify',
+ image: 'drillster/drone-email',
+ pull: 'if-not-exists',
+ settings: {
+ from: 'noreply@rspamd.com',
+ host: {
+ from_secret: 'email_host',
+ },
+ username: {
+ from_secret: 'email_username',
+ },
+ password: {
+ from_secret: 'email_password',
+ },
+ },
+ },
+ ],
+ trigger: {
+ event: default_trigger_events_ex_pr,
+ status: [
+ 'failure',
+ ],
+ },
+} + docker_pipeline;
+
+local pipeline(arch) = {
+ local rspamd_volumes = {
+ volumes: [
+ {
+ name: 'rspamd',
+ path: '/rspamd',
+ },
+ ],
+ },
+ local hyperscan_altroot = if (arch) == 'amd64' then '' else '-DHYPERSCAN_ROOT_DIR=/vectorscan',
+ depends_on: [
+ ],
+ name: 'default-' + arch,
+ steps: [
+ {
+ name: 'prepare',
+ image: 'ubuntu:22.04',
+ pull: 'if-not-exists',
+ commands: [
+ 'install -d -o nobody -g nogroup /rspamd/build /rspamd/install /rspamd/fedora/build /rspamd/fedora/install',
+ ],
+ } + rspamd_volumes,
+ {
+ name: 'build',
+ image: 'rspamd/ci:ubuntu-build',
+ pull: 'always',
+ depends_on: [
+ 'prepare',
+ ],
+ commands: [
+ 'test "$(id -un)" = nobody',
+ 'cd /rspamd/build',
+ 'cmake -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DENABLE_HYPERSCAN=ON ' + hyperscan_altroot + ' -GNinja $DRONE_WORKSPACE\n',
+ 'ncpu=$(getconf _NPROCESSORS_ONLN)',
+ 'ninja -j $ncpu install',
+ 'ninja -j $ncpu rspamd-test',
+ 'ninja -j $ncpu rspamd-test-cxx',
+ ],
+ } + rspamd_volumes,
+ {
+ name: 'build-clang',
+ image: 'rspamd/ci:fedora-build',
+ pull: 'always',
+ depends_on: [
+ 'prepare',
+ ],
+ commands: [
+ 'test "$(id -un)" = nobody',
+ 'cd /rspamd/fedora/build',
+ "export LDFLAGS='-fuse-ld=lld'",
+ 'export ASAN_OPTIONS=detect_leaks=0',
+ 'cmake -DCMAKE_INSTALL_PREFIX=/rspamd/fedora/install -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_CLANG_PLUGIN=ON -DENABLE_FULL_DEBUG=ON -DENABLE_HYPERSCAN=ON ' + hyperscan_altroot + ' -DSANITIZE=address $DRONE_WORKSPACE\n',
+ 'ncpu=$(getconf _NPROCESSORS_ONLN)',
+ 'make -j $ncpu install',
+ 'make -j $ncpu rspamd-test',
+ 'make -j $ncpu rspamd-test-cxx',
+ ],
+ } + rspamd_volumes,
+ {
+ name: 'rspamd-test',
+ image: 'rspamd/ci:ubuntu-test',
+ pull: 'always',
+ depends_on: [
+ 'build',
+ ],
+ commands: [
+ 'test "$(id -un)" = nobody',
+ 'ulimit -c unlimited',
+ 'cd /rspamd/build/test',
+ 'set +e',
+ 'env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?',
+ 'set -e',
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test.core ./rspamd-test; exit $EXIT_CODE; fi; if [ $EXIT_CODE -ne 0 ]; then exit $EXIT_CODE; fi\n",
+ 'luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun',
+ 'set +e',
+ './rspamd-test-cxx -s; EXIT_CODE=$?',
+ 'set -e',
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n",
+ 'exit $EXIT_CODE',
+ ],
+ } + rspamd_volumes,
+ {
+ name: 'test-fedora-clang',
+ image: 'rspamd/ci:fedora-test',
+ pull: 'always',
+ depends_on: [
+ 'build-clang',
+ ],
+ commands: [
+ 'test "$(id -un)" = nobody',
+ 'ulimit -c 2097152',
+ 'ulimit -s unlimited',
+ 'export ASAN_OPTIONS="detect_leaks=0:print_stacktrace=1:disable_coredump=0"',
+ 'export UBSAN_OPTIONS="print_stacktrace=1:print_summary=0:log_path=/tmp/ubsan"',
+ 'cd /rspamd/fedora/build/test',
+ 'set +e',
+ 'env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?',
+ 'set -e',
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'bt' -c /var/tmp/*.rspamd-test.core ./rspamd-test; fi\n",
+ 'set +e',
+ './rspamd-test-cxx -s; EXIT_CODE=$?',
+ 'set -e',
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n",
+ 'cat /tmp/ubsan.* || true',
+ 'exit $EXIT_CODE',
+ ],
+ } + rspamd_volumes,
+ {
+ name: 'functional',
+ image: 'rspamd/ci:ubuntu-test-func',
+ pull: 'always',
+ depends_on: [
+ 'build',
+ ],
+ commands: [
+ 'cd /rspamd/build',
+ 'ulimit -c unlimited',
+ 'ulimit -s unlimited',
+ 'set +e',
+ 'RSPAMD_INSTALLROOT=/rspamd/install robot --removekeywords wuks --exclude isbroken $DRONE_WORKSPACE/test/functional/cases; EXIT_CODE=$?',
+ 'set -e',
+ 'if [ -n "$HTTP_PUT_AUTH" ]; then $DRONE_WORKSPACE/test/tools/http_put.py log.html report.html https://$DRONE_SYSTEM_HOSTNAME/testlogs/$DRONE_REPO/${DRONE_BUILD_NUMBER}-' + arch + '/; fi\n',
+ "core_files=$(find /var/tmp/ -name '*.core')",
+ "for core in $core_files; do exe=$(gdb --batch -ex 'info proc mappings' -c $core | tail -1 | awk '{print $5}'); gdb --batch -ex 'bt' -c $core $exe; echo '---'; done\n",
+ 'exit $EXIT_CODE',
+ ],
+ environment: {
+ HTTP_PUT_AUTH: {
+ from_secret: 'http_put_auth',
+ },
+ },
+ } + rspamd_volumes,
+ {
+ name: 'send-coverage',
+ image: 'rspamd/ci:ubuntu-test',
+ pull: 'if-not-exists',
+ depends_on: [
+ 'functional',
+ 'rspamd-test',
+ ],
+ commands: [
+ 'cd /rspamd/build',
+ '$DRONE_WORKSPACE/test/tools/gcov_coveralls.py --exclude test --prefix /rspamd/build --prefix $DRONE_WORKSPACE --out coverage.c.json',
+ 'luacov-coveralls -o coverage.functional.lua.json --dryrun',
+ '$DRONE_WORKSPACE/test/tools/merge_coveralls.py --parallel --root $DRONE_WORKSPACE --input coverage.c.json unit_test_lua.json coverage.functional.lua.json --token=$COVERALLS_REPO_TOKEN',
+ ],
+ environment: {
+ COVERALLS_REPO_TOKEN: {
+ from_secret: 'coveralls_repo_token',
+ },
+ },
+ } + coveralls_when + rspamd_volumes,
+ ],
+ volumes: [
+ {
+ name: 'rspamd',
+ temp: {},
+ },
+ ],
+} + platform(arch) + default_trigger + docker_pipeline;
+
+local close_coveralls = {
+ name: 'close_coveralls',
+ depends_on: [
+ 'default-amd64',
+ 'default-arm64',
+ ],
+ steps: [
+ {
+ name: 'close_coveralls',
+ image: 'rspamd/ci:ubuntu-test-func',
+ pull: 'always',
+ commands: [
+ '$DRONE_WORKSPACE/test/tools/merge_coveralls.py --parallel-close --token=$COVERALLS_REPO_TOKEN',
+ ],
+ environment: {
+ COVERALLS_REPO_TOKEN: {
+ from_secret: 'coveralls_repo_token',
+ },
+ },
+ },
+ ],
+} + close_coveralls_trigger + docker_pipeline;
+
+local noarch_pipeline = {
+ name: 'default-noarch',
+ steps: [
+ {
+ name: 'perl-tidyall',
+ image: 'rspamd/ci:perl-tidyall',
+ pull: 'if-not-exists',
+ failure: 'ignore',
+ commands: [
+ 'tidyall --version',
+ 'perltidy --version | head -1',
+ 'tidyall --all --check-only --no-cache --data-dir /tmp/tidyall',
+ ],
+ },
+ {
+ name: 'eslint',
+ image: 'node:18-alpine',
+ pull: 'if-not-exists',
+ failure: 'ignore',
+ commands: [
+ 'npm install',
+ 'npm ls',
+ './node_modules/.bin/eslint ./',
+ './node_modules/.bin/stylelint ./**/*.css ./**/*.html ./**/*.js',
+ ],
+ },
+ {
+ name: 'luacheck',
+ image: 'pipelinecomponents/luacheck',
+ pull: 'if-not-exists',
+ commands: [
+ 'luacheck -q --no-color .',
+ ],
+ },
+ ],
+} + default_trigger + docker_pipeline;
+
+local signature = {
+ kind: 'signature',
+ hmac: '0000000000000000000000000000000000000000000000000000000000000000',
+};
+
+[
+ pipeline('amd64'),
+ pipeline('arm64'),
+ close_coveralls,
+ noarch_pipeline,
+ notify_pipeline,
+ signature,
+]
diff --git a/.drone.yml b/.drone.yml
new file mode 100644
index 0000000..cad3689
--- /dev/null
+++ b/.drone.yml
@@ -0,0 +1,563 @@
+---
+{
+ "depends_on": [ ],
+ "kind": "pipeline",
+ "name": "default-amd64",
+ "platform": {
+ "arch": "amd64",
+ "os": "linux"
+ },
+ "steps": [
+ {
+ "commands": [
+ "install -d -o nobody -g nogroup /rspamd/build /rspamd/install /rspamd/fedora/build /rspamd/fedora/install"
+ ],
+ "image": "ubuntu:22.04",
+ "name": "prepare",
+ "pull": "if-not-exists",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "test \"$(id -un)\" = nobody",
+ "cd /rspamd/build",
+ "cmake -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DENABLE_HYPERSCAN=ON -GNinja $DRONE_WORKSPACE\n",
+ "ncpu=$(getconf _NPROCESSORS_ONLN)",
+ "ninja -j $ncpu install",
+ "ninja -j $ncpu rspamd-test",
+ "ninja -j $ncpu rspamd-test-cxx"
+ ],
+ "depends_on": [
+ "prepare"
+ ],
+ "image": "rspamd/ci:ubuntu-build",
+ "name": "build",
+ "pull": "always",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "test \"$(id -un)\" = nobody",
+ "cd /rspamd/fedora/build",
+ "export LDFLAGS='-fuse-ld=lld'",
+ "export ASAN_OPTIONS=detect_leaks=0",
+ "cmake -DCMAKE_INSTALL_PREFIX=/rspamd/fedora/install -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_CLANG_PLUGIN=ON -DENABLE_FULL_DEBUG=ON -DENABLE_HYPERSCAN=ON -DSANITIZE=address $DRONE_WORKSPACE\n",
+ "ncpu=$(getconf _NPROCESSORS_ONLN)",
+ "make -j $ncpu install",
+ "make -j $ncpu rspamd-test",
+ "make -j $ncpu rspamd-test-cxx"
+ ],
+ "depends_on": [
+ "prepare"
+ ],
+ "image": "rspamd/ci:fedora-build",
+ "name": "build-clang",
+ "pull": "always",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "test \"$(id -un)\" = nobody",
+ "ulimit -c unlimited",
+ "cd /rspamd/build/test",
+ "set +e",
+ "env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?",
+ "set -e",
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test.core ./rspamd-test; exit $EXIT_CODE; fi; if [ $EXIT_CODE -ne 0 ]; then exit $EXIT_CODE; fi\n",
+ "luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun",
+ "set +e",
+ "./rspamd-test-cxx -s; EXIT_CODE=$?",
+ "set -e",
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n",
+ "exit $EXIT_CODE"
+ ],
+ "depends_on": [
+ "build"
+ ],
+ "image": "rspamd/ci:ubuntu-test",
+ "name": "rspamd-test",
+ "pull": "always",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "test \"$(id -un)\" = nobody",
+ "ulimit -c 2097152",
+ "ulimit -s unlimited",
+ "export ASAN_OPTIONS=\"detect_leaks=0:print_stacktrace=1:disable_coredump=0\"",
+ "export UBSAN_OPTIONS=\"print_stacktrace=1:print_summary=0:log_path=/tmp/ubsan\"",
+ "cd /rspamd/fedora/build/test",
+ "set +e",
+ "env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?",
+ "set -e",
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'bt' -c /var/tmp/*.rspamd-test.core ./rspamd-test; fi\n",
+ "set +e",
+ "./rspamd-test-cxx -s; EXIT_CODE=$?",
+ "set -e",
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n",
+ "cat /tmp/ubsan.* || true",
+ "exit $EXIT_CODE"
+ ],
+ "depends_on": [
+ "build-clang"
+ ],
+ "image": "rspamd/ci:fedora-test",
+ "name": "test-fedora-clang",
+ "pull": "always",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "cd /rspamd/build",
+ "ulimit -c unlimited",
+ "ulimit -s unlimited",
+ "set +e",
+ "RSPAMD_INSTALLROOT=/rspamd/install robot --removekeywords wuks --exclude isbroken $DRONE_WORKSPACE/test/functional/cases; EXIT_CODE=$?",
+ "set -e",
+ "if [ -n \"$HTTP_PUT_AUTH\" ]; then $DRONE_WORKSPACE/test/tools/http_put.py log.html report.html https://$DRONE_SYSTEM_HOSTNAME/testlogs/$DRONE_REPO/${DRONE_BUILD_NUMBER}-amd64/; fi\n",
+ "core_files=$(find /var/tmp/ -name '*.core')",
+ "for core in $core_files; do exe=$(gdb --batch -ex 'info proc mappings' -c $core | tail -1 | awk '{print $5}'); gdb --batch -ex 'bt' -c $core $exe; echo '---'; done\n",
+ "exit $EXIT_CODE"
+ ],
+ "depends_on": [
+ "build"
+ ],
+ "environment": {
+ "HTTP_PUT_AUTH": {
+ "from_secret": "http_put_auth"
+ }
+ },
+ "image": "rspamd/ci:ubuntu-test-func",
+ "name": "functional",
+ "pull": "always",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "cd /rspamd/build",
+ "$DRONE_WORKSPACE/test/tools/gcov_coveralls.py --exclude test --prefix /rspamd/build --prefix $DRONE_WORKSPACE --out coverage.c.json",
+ "luacov-coveralls -o coverage.functional.lua.json --dryrun",
+ "$DRONE_WORKSPACE/test/tools/merge_coveralls.py --parallel --root $DRONE_WORKSPACE --input coverage.c.json unit_test_lua.json coverage.functional.lua.json --token=$COVERALLS_REPO_TOKEN"
+ ],
+ "depends_on": [
+ "functional",
+ "rspamd-test"
+ ],
+ "environment": {
+ "COVERALLS_REPO_TOKEN": {
+ "from_secret": "coveralls_repo_token"
+ }
+ },
+ "image": "rspamd/ci:ubuntu-test",
+ "name": "send-coverage",
+ "pull": "if-not-exists",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ],
+ "when": {
+ "branch": [
+ "master"
+ ],
+ "event": [
+ "push",
+ "tag"
+ ]
+ }
+ }
+ ],
+ "trigger": {
+ "event": [
+ "push",
+ "tag",
+ "custom",
+ "pull_request"
+ ]
+ },
+ "type": "docker",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "temp": { }
+ }
+ ]
+}
+---
+{
+ "depends_on": [ ],
+ "kind": "pipeline",
+ "name": "default-arm64",
+ "platform": {
+ "arch": "arm64",
+ "os": "linux"
+ },
+ "steps": [
+ {
+ "commands": [
+ "install -d -o nobody -g nogroup /rspamd/build /rspamd/install /rspamd/fedora/build /rspamd/fedora/install"
+ ],
+ "image": "ubuntu:22.04",
+ "name": "prepare",
+ "pull": "if-not-exists",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "test \"$(id -un)\" = nobody",
+ "cd /rspamd/build",
+ "cmake -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DENABLE_HYPERSCAN=ON -DHYPERSCAN_ROOT_DIR=/vectorscan -GNinja $DRONE_WORKSPACE\n",
+ "ncpu=$(getconf _NPROCESSORS_ONLN)",
+ "ninja -j $ncpu install",
+ "ninja -j $ncpu rspamd-test",
+ "ninja -j $ncpu rspamd-test-cxx"
+ ],
+ "depends_on": [
+ "prepare"
+ ],
+ "image": "rspamd/ci:ubuntu-build",
+ "name": "build",
+ "pull": "always",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "test \"$(id -un)\" = nobody",
+ "cd /rspamd/fedora/build",
+ "export LDFLAGS='-fuse-ld=lld'",
+ "export ASAN_OPTIONS=detect_leaks=0",
+ "cmake -DCMAKE_INSTALL_PREFIX=/rspamd/fedora/install -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_CLANG_PLUGIN=ON -DENABLE_FULL_DEBUG=ON -DENABLE_HYPERSCAN=ON -DHYPERSCAN_ROOT_DIR=/vectorscan -DSANITIZE=address $DRONE_WORKSPACE\n",
+ "ncpu=$(getconf _NPROCESSORS_ONLN)",
+ "make -j $ncpu install",
+ "make -j $ncpu rspamd-test",
+ "make -j $ncpu rspamd-test-cxx"
+ ],
+ "depends_on": [
+ "prepare"
+ ],
+ "image": "rspamd/ci:fedora-build",
+ "name": "build-clang",
+ "pull": "always",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "test \"$(id -un)\" = nobody",
+ "ulimit -c unlimited",
+ "cd /rspamd/build/test",
+ "set +e",
+ "env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?",
+ "set -e",
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test.core ./rspamd-test; exit $EXIT_CODE; fi; if [ $EXIT_CODE -ne 0 ]; then exit $EXIT_CODE; fi\n",
+ "luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun",
+ "set +e",
+ "./rspamd-test-cxx -s; EXIT_CODE=$?",
+ "set -e",
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n",
+ "exit $EXIT_CODE"
+ ],
+ "depends_on": [
+ "build"
+ ],
+ "image": "rspamd/ci:ubuntu-test",
+ "name": "rspamd-test",
+ "pull": "always",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "test \"$(id -un)\" = nobody",
+ "ulimit -c 2097152",
+ "ulimit -s unlimited",
+ "export ASAN_OPTIONS=\"detect_leaks=0:print_stacktrace=1:disable_coredump=0\"",
+ "export UBSAN_OPTIONS=\"print_stacktrace=1:print_summary=0:log_path=/tmp/ubsan\"",
+ "cd /rspamd/fedora/build/test",
+ "set +e",
+ "env RSPAMD_LUA_EXPENSIVE_TESTS=1 ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?",
+ "set -e",
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'bt' -c /var/tmp/*.rspamd-test.core ./rspamd-test; fi\n",
+ "set +e",
+ "./rspamd-test-cxx -s; EXIT_CODE=$?",
+ "set -e",
+ "if [ $EXIT_CODE -gt 128 ]; then gdb --batch -ex 'thread apply all bt full' -c /var/tmp/*.rspamd-test-cxx.core ./rspamd-test-cxx; exit $EXIT_CODE; fi\n",
+ "cat /tmp/ubsan.* || true",
+ "exit $EXIT_CODE"
+ ],
+ "depends_on": [
+ "build-clang"
+ ],
+ "image": "rspamd/ci:fedora-test",
+ "name": "test-fedora-clang",
+ "pull": "always",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "cd /rspamd/build",
+ "ulimit -c unlimited",
+ "ulimit -s unlimited",
+ "set +e",
+ "RSPAMD_INSTALLROOT=/rspamd/install robot --removekeywords wuks --exclude isbroken $DRONE_WORKSPACE/test/functional/cases; EXIT_CODE=$?",
+ "set -e",
+ "if [ -n \"$HTTP_PUT_AUTH\" ]; then $DRONE_WORKSPACE/test/tools/http_put.py log.html report.html https://$DRONE_SYSTEM_HOSTNAME/testlogs/$DRONE_REPO/${DRONE_BUILD_NUMBER}-arm64/; fi\n",
+ "core_files=$(find /var/tmp/ -name '*.core')",
+ "for core in $core_files; do exe=$(gdb --batch -ex 'info proc mappings' -c $core | tail -1 | awk '{print $5}'); gdb --batch -ex 'bt' -c $core $exe; echo '---'; done\n",
+ "exit $EXIT_CODE"
+ ],
+ "depends_on": [
+ "build"
+ ],
+ "environment": {
+ "HTTP_PUT_AUTH": {
+ "from_secret": "http_put_auth"
+ }
+ },
+ "image": "rspamd/ci:ubuntu-test-func",
+ "name": "functional",
+ "pull": "always",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ]
+ },
+ {
+ "commands": [
+ "cd /rspamd/build",
+ "$DRONE_WORKSPACE/test/tools/gcov_coveralls.py --exclude test --prefix /rspamd/build --prefix $DRONE_WORKSPACE --out coverage.c.json",
+ "luacov-coveralls -o coverage.functional.lua.json --dryrun",
+ "$DRONE_WORKSPACE/test/tools/merge_coveralls.py --parallel --root $DRONE_WORKSPACE --input coverage.c.json unit_test_lua.json coverage.functional.lua.json --token=$COVERALLS_REPO_TOKEN"
+ ],
+ "depends_on": [
+ "functional",
+ "rspamd-test"
+ ],
+ "environment": {
+ "COVERALLS_REPO_TOKEN": {
+ "from_secret": "coveralls_repo_token"
+ }
+ },
+ "image": "rspamd/ci:ubuntu-test",
+ "name": "send-coverage",
+ "pull": "if-not-exists",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "path": "/rspamd"
+ }
+ ],
+ "when": {
+ "branch": [
+ "master"
+ ],
+ "event": [
+ "push",
+ "tag"
+ ]
+ }
+ }
+ ],
+ "trigger": {
+ "event": [
+ "push",
+ "tag",
+ "custom",
+ "pull_request"
+ ]
+ },
+ "type": "docker",
+ "volumes": [
+ {
+ "name": "rspamd",
+ "temp": { }
+ }
+ ]
+}
+---
+{
+ "depends_on": [
+ "default-amd64",
+ "default-arm64"
+ ],
+ "kind": "pipeline",
+ "name": "close_coveralls",
+ "steps": [
+ {
+ "commands": [
+ "$DRONE_WORKSPACE/test/tools/merge_coveralls.py --parallel-close --token=$COVERALLS_REPO_TOKEN"
+ ],
+ "environment": {
+ "COVERALLS_REPO_TOKEN": {
+ "from_secret": "coveralls_repo_token"
+ }
+ },
+ "image": "rspamd/ci:ubuntu-test-func",
+ "name": "close_coveralls",
+ "pull": "always"
+ }
+ ],
+ "trigger": {
+ "branch": [
+ "master"
+ ],
+ "event": [
+ "push",
+ "tag"
+ ],
+ "status": [
+ "success",
+ "failure"
+ ]
+ },
+ "type": "docker"
+}
+---
+{
+ "kind": "pipeline",
+ "name": "default-noarch",
+ "steps": [
+ {
+ "commands": [
+ "tidyall --version",
+ "perltidy --version | head -1",
+ "tidyall --all --check-only --no-cache --data-dir /tmp/tidyall"
+ ],
+ "failure": "ignore",
+ "image": "rspamd/ci:perl-tidyall",
+ "name": "perl-tidyall",
+ "pull": "if-not-exists"
+ },
+ {
+ "commands": [
+ "npm install",
+ "npm ls",
+ "./node_modules/.bin/eslint ./",
+ "./node_modules/.bin/stylelint ./**/*.css ./**/*.html ./**/*.js"
+ ],
+ "failure": "ignore",
+ "image": "node:18-alpine",
+ "name": "eslint",
+ "pull": "if-not-exists"
+ },
+ {
+ "commands": [
+ "luacheck -q --no-color ."
+ ],
+ "image": "pipelinecomponents/luacheck",
+ "name": "luacheck",
+ "pull": "if-not-exists"
+ }
+ ],
+ "trigger": {
+ "event": [
+ "push",
+ "tag",
+ "custom",
+ "pull_request"
+ ]
+ },
+ "type": "docker"
+}
+---
+{
+ "depends_on": [
+ "default-amd64",
+ "default-arm64",
+ "default-noarch"
+ ],
+ "kind": "pipeline",
+ "name": "notify",
+ "steps": [
+ {
+ "image": "drillster/drone-email",
+ "name": "notify",
+ "pull": "if-not-exists",
+ "settings": {
+ "from": "noreply@rspamd.com",
+ "host": {
+ "from_secret": "email_host"
+ },
+ "password": {
+ "from_secret": "email_password"
+ },
+ "username": {
+ "from_secret": "email_username"
+ }
+ }
+ }
+ ],
+ "trigger": {
+ "event": [
+ "push",
+ "tag",
+ "custom"
+ ],
+ "status": [
+ "failure"
+ ]
+ },
+ "type": "docker"
+}
+---
+{
+ "hmac": "2351718d9a562ea71ff344fb39fcf4ad5dae5b9694219b933c1b63a8b87d2aa5",
+ "kind": "signature"
+}
+...
diff --git a/.eslintrc.json b/.eslintrc.json
new file mode 100644
index 0000000..b9dbe16
--- /dev/null
+++ b/.eslintrc.json
@@ -0,0 +1,76 @@
+{
+ "env": {
+ "browser": true,
+ "es6": true
+ },
+ "extends": [
+ "eslint:all",
+ "plugin:@stylistic/all-extends"
+ ],
+ "globals": {
+ "define": false
+ },
+ "parserOptions": {
+ "ecmaVersion": 2016
+ },
+ "plugins": [
+ "@stylistic"
+ ],
+ "rules": {
+ "camelcase": "off",
+ "capitalized-comments": "off",
+ "curly": ["error", "multi-line"],
+ "func-names": "off",
+ "func-style": ["error", "declaration"],
+ "id-length": ["error", { "min": 1 }],
+ "line-comment-position": "off",
+ "logical-assignment-operators": ["error", "never"],
+ "max-params": ["warn", 6],
+ "max-statements": ["warn", 55],
+ "multiline-comment-style": "off",
+ "no-continue": "off",
+ "no-inline-comments": "off",
+ "no-magic-numbers": "off",
+ "no-negated-condition": "off",
+ "no-plusplus": "off",
+ "no-ternary": "off",
+ "object-shorthand": "off",
+ "one-var": ["error", { "initialized": "never" }],
+ "prefer-named-capture-group": "off",
+ "prefer-object-has-own": "off",
+ "prefer-spread": "off",
+ "prefer-template": "off",
+ "require-unicode-regexp": "off",
+
+ "@stylistic/array-bracket-newline": ["error", "consistent"],
+ "@stylistic/array-element-newline": "off",
+ "@stylistic/brace-style": ["error", "1tbs", { "allowSingleLine": true }],
+ "@stylistic/comma-dangle": ["error", "only-multiline"],
+ "@stylistic/dot-location": ["error", "property"],
+ "@stylistic/function-call-argument-newline": "off",
+ "@stylistic/max-len": ["error", { "code": 128 }],
+ "@stylistic/max-statements-per-line": ["error", { "max": 2 }],
+ "@stylistic/multiline-ternary": ["error", "always-multiline"],
+ "@stylistic/newline-per-chained-call": ["error", { "ignoreChainWithDepth": 5 }],
+ "@stylistic/no-extra-parens": ["error", "functions"],
+ "@stylistic/object-property-newline": ["error", { "allowAllPropertiesOnSameLine": true }],
+ "@stylistic/padded-blocks": ["error", "never"],
+ "@stylistic/quote-props" : ["error", "consistent-as-needed"],
+ "@stylistic/quotes": ["error", "double", { "avoidEscape": true }],
+ "@stylistic/semi": ["error", "always"],
+ "@stylistic/space-before-function-paren": ["error", {
+ "anonymous": "always",
+ "named": "never"
+ }],
+
+
+ // Temporarily disabled rules
+ "max-lines": "off",
+ "max-lines-per-function": "off",
+ "no-invalid-this": "off",
+ "sort-keys": "off",
+
+ "@stylistic/function-paren-newline": "off",
+ "@stylistic/indent-binary-ops": "off"
+ }
+}
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
new file mode 100644
index 0000000..ea05c64
--- /dev/null
+++ b/.github/FUNDING.yml
@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: vstakhov
+open_collective: rspamd_project
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
new file mode 100644
index 0000000..3cb5540
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE.md
@@ -0,0 +1 @@
+Please raise issues via the [new interface](https://github.com/rspamd/rspamd/issues/new/choose)
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 0000000..d34d9eb
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,44 @@
+---
+name: Bug report
+about: Create a report to let us know about issue in Rspamd
+title: "[BUG]"
+labels: bug
+assignees: ''
+
+---
+
+<!--
+
+Do you want to ask a question? Are you looking for support? Here are the places where you can get what you need: https://rspamd.com/support.html
+
+-->
+
+### Prerequisites
+
+* [ ] Put an X between the brackets on this line if you have done all of the following:
+ * Read about bug reporting in general: https://rspamd.com/doc/faq.html#how-to-report-bugs-found-in-rspamd
+ * Enabled relevant debugging logs: https://rspamd.com/doc/faq.html#how-to-debug-some-module-in-rspamd
+ * Checked the FAQs about Core files in case of fatal crash: https://rspamd.com/doc/faq.html#how-to-figure-out-why-rspamd-process-crashed
+ * Tried ASAN package and obtained the ASAN report (if possible): https://rspamd.com/doc/faq.html#asan-builds
+ * Checked that your issue isn't already filed: https://github.com/issues?utf8=%E2%9C%93&q=is%3Aissue+user%3Arspamd
+ * Checked that there is not already an experimental package or master branch
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+### Steps to Reproduce
+
+1. [First Step]
+2. [Second Step]
+3. [and so on...]
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+### Versions
+
+You can get this information from copy and pasting the output of `rspamd --version` from the command line. Also, please include the **OS** and what version and architecture including the generation of CPU model (e.g. Haswell) where applicable, of the OS you're running.
+
+### Additional Information
+
+Any additional information, configuration or data that might be necessary to reproduce the issue.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 0000000..46ed15e
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,32 @@
+---
+name: Feature request
+about: Suggest an idea for Rspamd
+title: "[Feature]"
+labels: enhancement
+assignees: ''
+
+---
+
+<!--
+
+Do you want to ask a question? Are you looking for support? Here are the places where you can get what you need: https://rspamd.com/support.html
+
+Feature requests are welcomed. Before opening a feature request, please take a moment to find out whether your idea fits with the scope and goals of the project. It’s up to you to make a strong case to convince the project maintainers the merits of the feature. Please provide as much detail and context as possible.
+
+-->
+
+## Summary
+
+One paragraph explanation of the feature.
+
+## Motivation
+
+Why are we doing this? What use cases does it support? What is the expected outcome?
+
+## Describe alternatives you've considered
+
+A clear and concise description of the alternative solutions you've considered. Be sure to explain why Rspamd's existing customizability isn't suitable for this feature.
+
+## Additional context
+
+Add any other context about the feature request here.
diff --git a/.github/ISSUE_TEMPLATE/support_question.md b/.github/ISSUE_TEMPLATE/support_question.md
new file mode 100644
index 0000000..8740783
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/support_question.md
@@ -0,0 +1,15 @@
+---
+name: Support Question
+about: If you have a question 💬, please check out our Mailing lists or other support channels!
+
+---
+
+--------------^ Click "Preview" for a nicer view!
+We primarily use GitHub as an issue tracker; for usage and support questions, please check out these resources below. Thanks!
+
+---
+
+* Mailing list: https://lists.rspamd.com/mailman/listinfo (you can subscribe on that page or browse the archives)
+* Telegram channel: http://t.me/rspamd
+* Also have a look at the following page for more information on how to get support:
+ https://rspamd.com/support.html
diff --git a/.github/stale.yml b/.github/stale.yml
new file mode 100644
index 0000000..dc90e5a
--- /dev/null
+++ b/.github/stale.yml
@@ -0,0 +1,17 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 60
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels:
+ - pinned
+ - security
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+ This issue has been automatically marked as stale because it has not had
+ recent activity. It will be closed if no further activity occurs. Thank you
+ for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..cf6f4eb
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,26 @@
+# Code::TidyAll
+/.tidyall.d/
+.idea
+# Logs and databases #
+######################
+*.log
+*.sqlite3-*
+# Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.pyc
+*.o
+*.so
+luacov.stats.out
+# OS generated files #
+######################
+*.orig
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
diff --git a/.luacheckrc b/.luacheckrc
new file mode 100644
index 0000000..7e48b8e
--- /dev/null
+++ b/.luacheckrc
@@ -0,0 +1,84 @@
+codes = true
+std = 'min'
+
+exclude_files = {
+ '/**/contrib/**',
+ '/**/test/lua/**',
+ '/**/test/functional/lua/miltertest/**',
+ '/**/test/functional/lua/rspamadm/**',
+ '.git/**/',
+}
+
+globals = {
+ 'classifiers',
+ 'config',
+ 'confighelp',
+ 'rspamd_classifiers',
+ 'rspamd_config',
+ 'rspamd_count_metatokens',
+ 'rspamd_gen_metatokens',
+ 'rspamd_parse_redis_server',
+ 'rspamd_paths',
+ 'rspamd_env',
+ 'rspamd_plugins',
+ 'rspamd_redis_make_request',
+ 'rspamd_str_split',
+ 'rspamd_version',
+ 'rspamd_maps',
+ 'rspamd_plugins_state',
+ 'rspamadm',
+ 'loadstring',
+ 'rspamadm_ev_base',
+ 'rspamadm_session',
+ 'rspamadm_dns_resolver',
+ 'jit',
+ 'table.unpack',
+ 'table.clear',
+ 'unpack',
+}
+
+ignore = {
+ '212', -- unused argument
+ '612', -- trailing whitespace
+ '311', -- value assigned to variable X is unused
+}
+
+files['/**/src/plugins/lua/spamassassin.lua'].globals = {
+ 'ffi',
+ 'jit',
+}
+
+files['/**/src/plugins/lua/greylist.lua'].globals = {
+ 'math.ifloor',
+}
+files['/**/src/plugins/lua/reputation.lua'].globals = {
+ 'math.tanh',
+}
+
+
+files['/**/lualib/lua_redis.lua'].globals = {
+ 'rspamadm_ev_base',
+}
+
+files['/**/lualib/redis_scripts/**'].globals = {
+ 'redis',
+ 'KEYS',
+ 'cjson',
+ 'cmsgpack',
+}
+
+files['/**/src/rspamadm/*'].globals = {
+ 'ansicolors',
+ 'getopt',
+}
+
+files['test/functional/lua/test_coverage.lua'].globals = {
+ '__GLOBAL_COVERAGE_WATCHDOG'
+}
+
+files['/**/rules/'].ignore = {'631'}
+files['/**/test/functional/'].ignore = {'631'}
+
+max_string_line_length = 500
+max_comment_line_length = 500
+max_line_length = 140
diff --git a/.overcommit.yml b/.overcommit.yml
new file mode 100644
index 0000000..d26d3de
--- /dev/null
+++ b/.overcommit.yml
@@ -0,0 +1,38 @@
+# Use this file to configure the Overcommit hooks you wish to use. This will
+# extend the default configuration defined in:
+# https://github.com/sds/overcommit/blob/master/config/default.yml
+#
+# At the topmost level of this YAML file is a key representing type of hook
+# being run (e.g. pre-commit, commit-msg, etc.). Within each type you can
+# customize each hook, such as whether to only run it on certain files (via
+# `include`), whether to only display output if it fails (via `quiet`), etc.
+#
+# For a complete list of hooks, see:
+# https://github.com/sds/overcommit/tree/master/lib/overcommit/hook
+#
+# For a complete list of options that you can use to customize hooks, see:
+# https://github.com/sds/overcommit#configuration
+#
+# Uncomment the following lines to make the configuration take effect.
+
+PreCommit:
+ TrailingWhitespace:
+ enabled: true
+ AuthorEmail:
+ enabled: true
+ AuthorName:
+ enabled: true
+ LineEndings:
+ enabled: true
+ LuaCheck:
+ enabled: true
+ command: ['luacheck', 'lualib', 'src/plugins/lua']
+ ClangFormat:
+ enabled: true
+ command: ['git', 'clang-format', '--diff']
+#PostCheckout:
+# ALL: # Special hook name that customizes all hooks of this type
+# quiet: true # Change all post-checkout hooks to only display output on failure
+#
+# IndexTags:
+# enabled: true # Generate a tags file with `ctags` each time HEAD changes
diff --git a/.stylelintrc.json b/.stylelintrc.json
new file mode 100644
index 0000000..f1e63b1
--- /dev/null
+++ b/.stylelintrc.json
@@ -0,0 +1,27 @@
+{
+ "extends": "stylelint-config-standard",
+ "overrides": [
+ {
+ "files": ["**/*.html"],
+ "customSyntax": "postcss-html"
+ }
+ ],
+ "ignoreFiles": [
+ "**/*.min.css",
+ "**/*.js",
+ "interface/css/d3evolution.css",
+ "interface/css/codejar-linenumbers.css",
+ "interface/css/nprogress.css",
+ "interface/css/prism.css"
+ ],
+ "rules": {
+ "alpha-value-notation": null,
+ "at-rule-empty-line-before": null,
+ "color-hex-length": null,
+ "comment-empty-line-before": null,
+ "media-feature-range-notation": "prefix",
+ "property-no-vendor-prefix": null,
+ "rule-empty-line-before": null,
+ "value-no-vendor-prefix": null
+ }
+}
diff --git a/.tidyallrc b/.tidyallrc
new file mode 100644
index 0000000..e948435
--- /dev/null
+++ b/.tidyallrc
@@ -0,0 +1,22 @@
+; Run "tidyall -a" to process all files.
+; Run "tidyall -g" to process all added or modified files in the current git working directory.
+
+; Ignore third-party code
+ignore = contrib/**/* doc/doxydown/doxydown.pl
+
+[PerlCritic]
+select = **/*.{pl,pl.in,pm,t}
+ignore = utils/{cgp_rspamd.pl,redirector.pl.in,classifier_test.pl}
+
+[PodChecker]
+select = **/*.{pl,pl.in,pm,pod}
+
+;[PodSpell]
+;select = **/*.{pl,pl.in,pm,pod}
+
+[PodTidy]
+select = **/*.{pl,pl.in,pm,pod}
+columns = 120
+
+[Test::Vars]
+select = **/*.{pl,pl.in,pm,t}
diff --git a/AUTHORS.md b/AUTHORS.md
new file mode 100644
index 0000000..a079066
--- /dev/null
+++ b/AUTHORS.md
@@ -0,0 +1,30 @@
+# Project Authors and Contributors
+
+Rspamd was created by [Vsevolod Stakhov](https://github.com/vstakov).
+
+## Authors
+
+* [Vsevolod Stakhov](https://github.com/vstakov)
+
+## Developers
+
+* [Alexander Moisseev](https://github.com/moisseev)
+* [Andrew Lewis](https://github.com/fatalbanana)
+
+## Alumni Developers
+
+* [Mikhail Galanin](https://github.com/negram)
+
+## Significant contributors
+
+* [Steve Freegard](https://github.com/smfreegard)
+* [Alexey Savelyev](https://github.com/AlexeySa)
+* [Anna Stakhova](https://github.com/AnnaStakhova)
+* [Anton Yuzhaninov](https://github.com/citrin)
+
+<https://github.com/rspamd/rspamd/graphs/contributors>
+
+## Special thanks to
+
+* [Mimecast](https://mimecast.com)
+* [Locaweb](https://locaweb.com.br)
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000..af9f9ec
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,806 @@
+#
+# Rspamd - rapid antispam system
+#
+# Cmake configuration file
+#
+
+############################# INITIAL SECTION #############################################
+CMAKE_MINIMUM_REQUIRED(VERSION 3.12 FATAL_ERROR)
+
+SET(RSPAMD_VERSION_MAJOR 3)
+SET(RSPAMD_VERSION_MINOR 8)
+SET(RSPAMD_VERSION_PATCH 1)
+
+# Keep two digits all the time
+SET(RSPAMD_VERSION_MAJOR_NUM ${RSPAMD_VERSION_MAJOR}0)
+SET(RSPAMD_VERSION_MINOR_NUM ${RSPAMD_VERSION_MINOR}0)
+SET(RSPAMD_VERSION_PATCH_NUM ${RSPAMD_VERSION_PATCH}0)
+
+IF (GIT_ID)
+ SET(GIT_VERSION 1)
+ SET(RSPAMD_ID "${GIT_ID}")
+ENDIF ()
+
+SET(RSPAMD_VERSION "${RSPAMD_VERSION_MAJOR}.${RSPAMD_VERSION_MINOR}.${RSPAMD_VERSION_PATCH}")
+
+PROJECT(rspamd VERSION "${RSPAMD_VERSION}" LANGUAGES C CXX ASM)
+
+CMAKE_POLICY(SET CMP0075 NEW)
+
+# This is supported merely with cmake 3.1
+SET(CMAKE_C_STANDARD 11)
+SET(CMAKE_CXX_STANDARD 20)
+SET(CMAKE_C_STANDARD_REQUIRED ON)
+SET(CMAKE_CXX_STANDARD_REQUIRED ON)
+LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/")
+
+SET(RSPAMD_MASTER_SITE_URL "https://rspamd.com")
+
+IF (NOT RSPAMD_USER)
+ SET(RSPAMD_USER "nobody")
+ SET(RSPAMD_GROUP "nobody")
+ENDIF (NOT RSPAMD_USER)
+
+# Default for SysV Init
+SET(RSPAMD_WORKER_NORMAL "*:11333")
+SET(RSPAMD_WORKER_CONTROLLER "*:11334")
+
+############################# OPTIONS SECTION #############################################
+OPTION(ENABLE_LUAJIT "Link with libluajit [default: ON]" ON)
+OPTION(ENABLE_URL_INCLUDE "Enable urls in ucl includes (requires libcurl or libfetch) [default: OFF]" OFF)
+OPTION(NO_SHARED "Build internal libs static [default: ON]" ON)
+OPTION(INSTALL_WEBUI "Install web interface [default: ON]" ON)
+OPTION(WANT_SYSTEMD_UNITS "Install systemd unit files on Linux [default: OFF]" OFF)
+OPTION(ENABLE_SNOWBALL "Enable snowball stemmer [default: ON]" ON)
+OPTION(ENABLE_CLANG_PLUGIN "Enable clang static analysing plugin [default: OFF]" OFF)
+OPTION(ENABLE_PCRE2 "Enable pcre2 instead of pcre [default: ON]" ON)
+OPTION(ENABLE_JEMALLOC "Build rspamd with jemalloc allocator [default: OFF]" OFF)
+OPTION(ENABLE_UTILS "Build rspamd internal utils [default: OFF]" OFF)
+OPTION(ENABLE_LIBUNWIND "Obsoleted [default: OFF]" OFF)
+OPTION(ENABLE_LUA_TRACE "Trace all Lua C API invocations [default: OFF]" OFF)
+OPTION(ENABLE_LUA_REPL "Enables Lua repl (requires C++11 compiler) [default: ON]" ON)
+OPTION(ENABLE_FASTTEXT "Link with FastText library [default: OFF]" OFF)
+OPTION(ENABLE_BACKWARD "Build rspamd with backward-cpp stacktrace [default: ON]" ON)
+OPTION(SYSTEM_ZSTD "Use system zstd instead of bundled one [default: OFF]" OFF)
+OPTION(SYSTEM_FMT "Use system fmt instead of bundled one [default: OFF]" OFF)
+OPTION(SYSTEM_DOCTEST "Use system doctest instead of bundled one [default: OFF]" OFF)
+OPTION(SYSTEM_XXHASH "Use system xxhash instead of bundled one [default: OFF]" OFF)
+
+############################# INCLUDE SECTION #############################################
+
+INCLUDE(CheckIncludeFiles)
+INCLUDE(CheckFunctionExists)
+INCLUDE(CheckSymbolExists)
+INCLUDE(CheckCSourceCompiles)
+INCLUDE(CheckCSourceRuns)
+INCLUDE(CheckLibraryExists)
+INCLUDE(CheckCCompilerFlag)
+INCLUDE(CMakeParseArguments)
+INCLUDE(FindArch)
+INCLUDE(AsmOp)
+INCLUDE(FindRagel)
+INCLUDE(ProcessPackage)
+
+IF (NOT RAGEL_FOUND)
+ MESSAGE(FATAL_ERROR "Ragel is required to build rspamd")
+ENDIF ()
+
+FIND_PACKAGE(PkgConfig REQUIRED)
+FIND_PACKAGE(Perl REQUIRED)
+
+option(SANITIZE "Enable sanitizer: address, memory, undefined, leak (comma separated list)" "")
+INCLUDE(Toolset)
+INCLUDE(Sanitizer)
+
+INCLUDE(ArchDep)
+INCLUDE(Paths)
+
+IF (ENABLE_PCRE2 MATCHES "ON")
+ SET(WITH_PCRE2 1)
+ # For utf8 API
+ LIST(APPEND CMAKE_REQUIRED_DEFINITIONS "-DPCRE2_CODE_UNIT_WIDTH=8")
+ENDIF ()
+############################# CONFIG SECTION #############################################
+# Initial set
+
+# Prefer local include dirs to system ones
+INCLUDE_DIRECTORIES("${CMAKE_SOURCE_DIR}/"
+ "${CMAKE_SOURCE_DIR}/src"
+ "${CMAKE_SOURCE_DIR}/src/libutil"
+ "${CMAKE_SOURCE_DIR}/src/libserver"
+ "${CMAKE_SOURCE_DIR}/src/libmime"
+ "${CMAKE_SOURCE_DIR}/src/libstat"
+ "${CMAKE_SOURCE_DIR}/src/libcryptobox"
+ "${CMAKE_SOURCE_DIR}/contrib/libucl"
+ "${CMAKE_SOURCE_DIR}/contrib/replxx/include"
+ "${CMAKE_SOURCE_DIR}/contrib/uthash"
+ "${CMAKE_SOURCE_DIR}/contrib/http-parser"
+ "${CMAKE_SOURCE_DIR}/contrib/fpconv"
+ "${CMAKE_SOURCE_DIR}/contrib/libottery"
+ "${CMAKE_SOURCE_DIR}/contrib/cdb"
+ "${CMAKE_SOURCE_DIR}/contrib/snowball/include"
+ "${CMAKE_SOURCE_DIR}/contrib/librdns"
+ "${CMAKE_SOURCE_DIR}/contrib/aho-corasick"
+ "${CMAKE_SOURCE_DIR}/contrib/lc-btrie"
+ "${CMAKE_SOURCE_DIR}/contrib/lua-lpeg"
+ "${CMAKE_SOURCE_DIR}/contrib/frozen/include"
+ "${CMAKE_SOURCE_DIR}/contrib/fu2/include"
+ "${CMAKE_BINARY_DIR}/src" #Stored in the binary dir
+ "${CMAKE_BINARY_DIR}/src/libcryptobox")
+
+SET(TAR "tar")
+INCLUDE(OSDep)
+
+# Now find libraries and headers
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "m")
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "pthread")
+
+IF (ENABLE_LUAJIT MATCHES "ON")
+ ProcessPackage(LIBLUAJIT LIBRARY "luajit"
+ "luajit-2.1"
+ "luajit2.1"
+ "luajit-2.0"
+ "luajit2.0"
+ "luajit-5.1"
+ INCLUDE luajit.h INCLUDE_SUFFIXES
+ "include/luajit-2.1"
+ "include/luajit-2.0"
+ "include/luajit"
+ ROOT ${LUA_ROOT}
+ MODULES luajit)
+ SET(WITH_LUAJIT 1)
+ELSE (ENABLE_LUAJIT MATCHES "ON")
+
+ ProcessPackage(LIBLUA LIBRARY "lua"
+ "lua-5.3"
+ LIB_SUFFIXES "lua5.3"
+ INCLUDE lua.h INCLUDE_SUFFIXES
+ "include/lua-5.3"
+ "include/lua5.3"
+ "include/lua53"
+ "include/lua"
+ ROOT ${LUA_ROOT}
+ MODULES lua53
+ OPTIONAL)
+
+ IF (NOT WITH_LIBLUA)
+ ProcessPackage(LIBLUA LIBRARY "lua"
+ "lua-5.4"
+ LIB_SUFFIXES "lua5.4"
+ INCLUDE lua.h INCLUDE_SUFFIXES
+ "include/lua-5.4"
+ "include/lua5.4"
+ "include/lua54"
+ "include/lua"
+ ROOT ${LUA_ROOT}
+ MODULES lua54
+ OPTIONAL)
+ IF (NOT WITH_LIBLUA)
+ ProcessPackage(LIBLUA LIBRARY "lua"
+ "lua-5.2"
+ LIB_SUFFIXES "lua5.2"
+ INCLUDE lua.h INCLUDE_SUFFIXES
+ "include/lua-5.2"
+ "include/lua5.2"
+ "include/lua52"
+ "include/lua"
+ ROOT ${LUA_ROOT}
+ MODULES lua52
+ OPTIONAL)
+
+ IF (NOT WITH_LIBLUA)
+ ProcessPackage(LIBLUA LIBRARY "lua"
+ "lua-5.1"
+ INCLUDE lua.h INCLUDE_SUFFIXES
+ "include/lua-5.1"
+ "include/lua5.1"
+ "include/lua51"
+ "include/lua"
+ ROOT ${LUA_ROOT}
+ MODULES lua51)
+ ENDIF ()
+ ENDIF ()
+ ENDIF ()
+ENDIF (ENABLE_LUAJIT MATCHES "ON")
+
+IF (ENABLE_JEMALLOC MATCHES "ON" AND NOT SANITIZE)
+ ProcessPackage(JEMALLOC LIBRARY jemalloc_pic jemalloc INCLUDE jemalloc.h INCLUDE_SUFFIXES include/jemalloc
+ ROOT ${JEMALLOC_ROOT_DIR})
+ SET(WITH_JEMALLOC "1")
+ENDIF ()
+
+ProcessPackage(GLIB2 LIBRARY glib-2.0 INCLUDE glib.h
+ INCLUDE_SUFFIXES include/glib include/glib-2.0
+ ROOT ${GLIB_ROOT_DIR} MODULES glib-2.0>=2.28)
+
+IF (ENABLE_PCRE2 MATCHES "ON")
+ ProcessPackage(PCRE LIBRARY pcre2 pcre2-8 INCLUDE pcre2.h INCLUDE_SUFFIXES include/pcre2
+ ROOT ${PCRE_ROOT_DIR} MODULES pcre2 pcre2-8 libpcre2 libpcre2-8)
+ELSE ()
+ ProcessPackage(PCRE LIBRARY pcre INCLUDE pcre.h INCLUDE_SUFFIXES include/pcre
+ ROOT ${PCRE_ROOT_DIR} MODULES pcre libpcre pcre3 libpcre3)
+ENDIF ()
+
+ProcessPackage(SQLITE3 LIBRARY sqlite3 INCLUDE sqlite3.h INCLUDE_SUFFIXES include/sqlite3 include/sqlite
+ ROOT ${SQLITE3_ROOT_DIR} MODULES sqlite3 sqlite)
+ProcessPackage(ICUDATA LIBRARY icudata INCLUDE unicode/ucnv.h
+ ROOT ${ICU_ROOT_DIR} MODULES icu-uc)
+ProcessPackage(ICUC LIBRARY icuuc INCLUDE unicode/ucnv.h
+ ROOT ${ICU_ROOT_DIR} MODULES icu-uc)
+ProcessPackage(ICUIO LIBRARY icuio INCLUDE unicode/ucnv.h
+ ROOT ${ICU_ROOT_DIR} MODULES icu-io)
+ProcessPackage(ICUI18N LIBRARY icui18n INCLUDE unicode/ucnv.h
+ ROOT ${ICU_ROOT_DIR} MODULES icu-i18n)
+ProcessPackage(LIBCRYPT LIBRARY crypto INCLUDE openssl/evp.h
+ ROOT ${OPENSSL_ROOT_DIR} MODULES openssl libcrypt)
+ProcessPackage(LIBSSL LIBRARY ssl INCLUDE openssl/ssl.h
+ ROOT ${OPENSSL_ROOT_DIR} MODULES openssl libssl)
+ProcessPackage(LIBZ LIBRARY z INCLUDE zlib.h INCLUDE_SUFFIXES include/zlib
+ ROOT ${LIBZ_ROOT_DIR} MODULES z)
+ProcessPackage(SODIUM LIBRARY sodium INCLUDE sodium.h
+ INCLUDE_SUFFIXES include/libsodium include/sodium
+ ROOT ${LIBSODIUM_ROOT_DIR} MODULES libsodium>=1.0.0)
+
+if (ENABLE_FASTTEXT MATCHES "ON")
+ ProcessPackage(FASTTEXT LIBRARY fasttext INCLUDE fasttext/fasttext.h
+ ROOT ${FASTTEXT_ROOT_DIR} MODULES fasttext)
+ SET(WITH_FASTTEXT "1")
+endif ()
+
+include(CompilerWarnings)
+include(Hyperscan)
+include(Openblas)
+
+IF (ENABLE_LUA_TRACE)
+ SET(WITH_LUA_TRACE 1)
+ENDIF (ENABLE_LUA_TRACE)
+
+SET(CMAKE_C_FLAGS "${CMAKE_C_OPT_FLAGS} ${CMAKE_C_FLAGS}")
+SET(CMAKE_CXX_FLAGS "${CMAKE_C_OPT_FLAGS} ${CMAKE_CXX_FLAGS}")
+
+ADD_DEFINITIONS(-DHAVE_CONFIG_H)
+ADD_DEFINITIONS(-DDOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS)
+ADD_DEFINITIONS(-DFMT_HEADER_ONLY)
+# Workaround for https://github.com/onqtam/doctest/issues/356
+ADD_DEFINITIONS(-DDOCTEST_CONFIG_USE_STD_HEADERS)
+ADD_DEFINITIONS(-DU_CHARSET_IS_UTF8)
+
+# Check platform specific includes
+CHECK_INCLUDE_FILES(sys/types.h HAVE_SYS_TYPES_H)
+CHECK_INCLUDE_FILES(sys/uio.h HAVE_SYS_UIO_H)
+
+CHECK_INCLUDE_FILES(fcntl.h HAVE_FCNTL_H)
+CHECK_INCLUDE_FILES(math.h HAVE_MATH_H)
+CHECK_INCLUDE_FILES(stdio.h HAVE_STDIO_H)
+CHECK_INCLUDE_FILES(stdlib.h HAVE_STDLIB_H)
+CHECK_INCLUDE_FILES(string.h HAVE_STRING_H)
+CHECK_INCLUDE_FILES(strings.h HAVE_STRINGS_H)
+CHECK_INCLUDE_FILES(time.h HAVE_TIME_H)
+CHECK_INCLUDE_FILES(unistd.h HAVE_UNISTD_H)
+CHECK_INCLUDE_FILES(stdint.h HAVE_STDINT_H)
+CHECK_INCLUDE_FILES(inttypes.h HAVE_INTTYPES_H)
+CHECK_INCLUDE_FILES(stdbool.h HAVE_STDBOOL_H)
+CHECK_INCLUDE_FILES(endian.h HAVE_ENDIAN_H)
+CHECK_INCLUDE_FILES(sys/endian.h HAVE_SYS_ENDIAN_H)
+CHECK_INCLUDE_FILES(machine/endian.h HAVE_MACHINE_ENDIAN_H)
+CHECK_INCLUDE_FILES(sys/socket.h HAVE_SYS_SOCKET_H)
+CHECK_INCLUDE_FILES(sys/mman.h HAVE_SYS_MMAN_H)
+CHECK_INCLUDE_FILES(sys/un.h HAVE_SYS_UN_H)
+CHECK_INCLUDE_FILES(sys/stat.h HAVE_SYS_STAT_H)
+CHECK_INCLUDE_FILES(sys/wait.h HAVE_SYS_WAIT_H)
+CHECK_INCLUDE_FILES(sys/param.h HAVE_SYS_PARAM_H)
+CHECK_INCLUDE_FILES(sys/file.h HAVE_SYS_FILE_H)
+CHECK_INCLUDE_FILES(sys/resource.h HAVE_SYS_RESOURCE_H)
+CHECK_INCLUDE_FILES(netinet/in.h HAVE_NETINET_IN_H)
+CHECK_INCLUDE_FILES(netinet/tcp.h HAVE_NETINET_TCP_H)
+CHECK_INCLUDE_FILES(arpa/inet.h HAVE_ARPA_INET_H)
+CHECK_INCLUDE_FILES(netdb.h HAVE_NETDB_H)
+CHECK_INCLUDE_FILES(syslog.h HAVE_SYSLOG_H)
+CHECK_INCLUDE_FILES(siginfo.h HAVE_SIGINFO_H)
+CHECK_INCLUDE_FILES(locale.h HAVE_LOCALE_H)
+CHECK_INCLUDE_FILES(libgen.h HAVE_LIBGEN_H)
+CHECK_INCLUDE_FILES(pwd.h HAVE_PWD_H)
+CHECK_INCLUDE_FILES(grp.h HAVE_GRP_H)
+CHECK_INCLUDE_FILES(glob.h HAVE_GLOB_H)
+CHECK_INCLUDE_FILES(poll.h HAVE_POLL_H)
+CHECK_INCLUDE_FILES(readpassphrase.h HAVE_READPASSPHRASE_H)
+CHECK_INCLUDE_FILES(termios.h HAVE_TERMIOS_H)
+CHECK_INCLUDE_FILES(paths.h HAVE_PATHS_H)
+CHECK_INCLUDE_FILES(ctype.h HAVE_CTYPE_H)
+CHECK_INCLUDE_FILES(unistd.h HAVE_UNISTD_H)
+CHECK_INCLUDE_FILES(cpuid.h HAVE_CPUID_H)
+CHECK_INCLUDE_FILES(dirent.h HAVE_DIRENT_H)
+CHECK_INCLUDE_FILES(ucontext.h HAVE_UCONTEXT_H)
+CHECK_INCLUDE_FILES(sys/ucontext.h HAVE_SYS_UCONTEXT_H) # OSX specific
+
+# Check platform API
+IF(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ # setproctitle is broken badly in Linux, never try it
+ CHECK_FUNCTION_EXISTS(setproctitle HAVE_SETPROCTITLE)
+ENDIF()
+CHECK_FUNCTION_EXISTS(getpagesize HAVE_GETPAGESIZE)
+CHECK_FUNCTION_EXISTS(nanosleep HAVE_NANOSLEEP)
+CHECK_FUNCTION_EXISTS(flock HAVE_FLOCK)
+CHECK_LIBRARY_EXISTS(m tanh "" HAVE_TANH)
+CHECK_FUNCTION_EXISTS(mkstemp HAVE_MKSTEMP)
+CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME)
+
+# Check macros
+CHECK_SYMBOL_EXISTS(PATH_MAX limits.h HAVE_PATH_MAX)
+CHECK_SYMBOL_EXISTS(MAXPATHLEN sys/param.h HAVE_MAXPATHLEN)
+CHECK_SYMBOL_EXISTS(MAP_ANON sys/mman.h HAVE_MMAP_ANON)
+CHECK_SYMBOL_EXISTS(IPV6_V6ONLY "sys/socket.h;netinet/in.h" HAVE_IPV6_V6ONLY)
+CHECK_SYMBOL_EXISTS(posix_fallocate fcntl.h HAVE_POSIX_FALLOCATE)
+CHECK_SYMBOL_EXISTS(fallocate fcntl.h HAVE_FALLOCATE)
+CHECK_SYMBOL_EXISTS(_SC_NPROCESSORS_ONLN unistd.h HAVE_SC_NPROCESSORS_ONLN)
+CHECK_SYMBOL_EXISTS(setbit sys/param.h PARAM_H_HAS_BITSET)
+CHECK_SYMBOL_EXISTS(getaddrinfo "sys/types.h;sys/socket.h;netdb.h" HAVE_GETADDRINFO)
+CHECK_SYMBOL_EXISTS(sched_yield "sched.h" HAVE_SCHED_YIELD)
+CHECK_SYMBOL_EXISTS(nftw "sys/types.h;ftw.h" HAVE_NFTW)
+CHECK_SYMBOL_EXISTS(memrchr "string.h" HAVE_MEMRCHR)
+IF (ENABLE_PCRE2 MATCHES "ON")
+ LIST(APPEND CMAKE_REQUIRED_INCLUDES "${PCRE_INCLUDE}")
+ CHECK_SYMBOL_EXISTS(PCRE2_CONFIG_JIT "pcre2.h" HAVE_PCRE_JIT)
+ELSE ()
+ LIST(APPEND CMAKE_REQUIRED_INCLUDES "${PCRE_INCLUDE}")
+ CHECK_SYMBOL_EXISTS(PCRE_CONFIG_JIT "pcre.h" HAVE_PCRE_JIT)
+ENDIF ()
+CHECK_SYMBOL_EXISTS(SOCK_SEQPACKET "sys/types.h;sys/socket.h" HAVE_SOCK_SEQPACKET)
+CHECK_SYMBOL_EXISTS(O_NOFOLLOW "sys/types.h;sys/fcntl.h" HAVE_ONOFOLLOW)
+CHECK_SYMBOL_EXISTS(O_CLOEXEC "sys/types.h;sys/fcntl.h" HAVE_OCLOEXEC)
+
+# OpenSSL specific stuff
+LIST(APPEND CMAKE_REQUIRED_INCLUDES "${LIBSSL_INCLUDE}")
+IF (LIBCRYPT_LIBRARY_PATH)
+ SET(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};-L${LIBCRYPT_LIBRARY_PATH};${LIBCRYPT_LIBRARY}")
+ SET(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};-L${LIBSSL_LIBRARY_PATH};${LIBSSL_LIBRARY}")
+ELSE ()
+ SET(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};-lcrypt;-lssl")
+ENDIF ()
+
+CHECK_SYMBOL_EXISTS(SSL_set_tlsext_host_name "openssl/ssl.h" HAVE_SSL_TLSEXT_HOSTNAME)
+CHECK_SYMBOL_EXISTS(FIPS_mode "openssl/crypto.h" HAVE_FIPS_MODE)
+
+CHECK_SYMBOL_EXISTS(dirfd "sys/types.h;unistd.h;dirent.h" HAVE_DIRFD)
+CHECK_SYMBOL_EXISTS(fpathconf "sys/types.h;unistd.h" HAVE_FPATHCONF)
+CHECK_SYMBOL_EXISTS(sigaltstack "signal.h" HAVE_SIGALTSTACK)
+CHECK_SYMBOL_EXISTS(open_memstream "stdio.h" HAVE_OPENMEMSTREAM)
+CHECK_SYMBOL_EXISTS(fmemopen "stdio.h" HAVE_FMEMOPEN)
+CHECK_SYMBOL_EXISTS(clock_getcpuclockid "sys/types.h;time.h" HAVE_CLOCK_GETCPUCLOCKID)
+CHECK_SYMBOL_EXISTS(RUSAGE_SELF "sys/types.h;sys/resource.h" HAVE_RUSAGE_SELF)
+CHECK_SYMBOL_EXISTS(ffsll "strings.h" HAVE_FFSLL)
+
+IF (ENABLE_PCRE2 MATCHES "ON")
+ IF (HAVE_PCRE_JIT)
+ SET(HAVE_PCRE_JIT_FAST 1)
+ ENDIF ()
+ELSE ()
+ LIST(APPEND CMAKE_REQUIRED_INCLUDES "${PCRE_INCLUDE}")
+ IF (PCRE_LIBRARY_PATH)
+ SET(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};-L${PCRE_LIBRARY_PATH};${PCRE_LIBRARY}")
+ ELSE (PCRE_LIBRARY_PATH)
+ SET(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};-lpcre")
+ ENDIF (PCRE_LIBRARY_PATH)
+ # Some PCRE implementations are lacking of pcre_jit_exec fast path
+ SET(_PCRE_FAST_TEST "
+#include \"pcre.h\"
+int main (void)
+{
+ int rc;
+ int ovector[30];
+ pcre *re;
+ pcre_extra *extra;
+ pcre_jit_stack *jit_stack;
+
+ re = pcre_compile(\"abc\", 0, NULL, NULL, NULL);
+ extra = pcre_study(re, PCRE_STUDY_JIT_COMPILE, NULL);
+ jit_stack = pcre_jit_stack_alloc(32*1024, 512*1024);
+ pcre_assign_jit_stack(extra, NULL, jit_stack);
+ rc = pcre_jit_exec(re, extra, \"abc\", 3, 0, 0, ovector, 30, jit_stack);
+
+ return rc;
+}
+")
+ CHECK_C_SOURCE_COMPILES("${_PCRE_FAST_TEST}" HAVE_PCRE_JIT_FAST)
+ IF (HAVE_PCRE_JIT_FAST)
+ MESSAGE(STATUS "pcre_jit_exec is supported")
+ ELSE (HAVE_PCRE_JIT_FAST)
+ MESSAGE(STATUS "pcre_jit_exec is -NOT- supported")
+ ENDIF (HAVE_PCRE_JIT_FAST)
+ENDIF ()
+
+CHECK_C_COMPILER_FLAG(-fPIC SUPPORT_FPIC)
+IF (SUPPORT_FPIC)
+ ADD_COMPILE_OPTIONS("-fPIC")
+ENDIF (SUPPORT_FPIC)
+
+FILE(WRITE ${CMAKE_BINARY_DIR}/pthread_setpshared.c "
+#include <pthread.h>
+#include <stdlib.h>
+int main (void)
+{
+ pthread_mutexattr_t mattr;
+ if (pthread_mutexattr_init(&mattr) != 0) return 0;
+ if (pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED) != 0) return 0;
+ if (pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST) != 0) return 0;
+ return 1;
+}
+")
+TRY_RUN(_CAN_RUN _CAN_COMPILE
+ "${CMAKE_BINARY_DIR}" "${CMAKE_BINARY_DIR}/pthread_setpshared.c"
+ CMAKE_FLAGS CMAKE_C_FLAGS="-pthread")
+IF (_CAN_RUN EQUAL 1)
+ SET(HAVE_PTHREAD_PROCESS_SHARED 1 CACHE INTERNAL "")
+ENDIF (_CAN_RUN EQUAL 1)
+IF (HAVE_PTHREAD_PROCESS_SHARED)
+ MESSAGE(STATUS "pthread_mutexattr_setpshared is supported")
+ELSE (HAVE_PTHREAD_PROCESS_SHARED)
+ MESSAGE(STATUS "pthread_mutexattr_setpshared is -NOT- supported")
+ENDIF (HAVE_PTHREAD_PROCESS_SHARED)
+
+IF (NOT HAVE_GETADDRINFO)
+ MESSAGE(FATAL_ERROR "Your system does not support getaddrinfo call, please consider upgrading it to run rspamd")
+ENDIF (NOT HAVE_GETADDRINFO)
+IF (HAVE_SIGINFO_H)
+ CHECK_SYMBOL_EXISTS(SA_SIGINFO "signal.h;siginfo.h" HAVE_SA_SIGINFO)
+ELSE (HAVE_SIGINFO_H)
+ CHECK_SYMBOL_EXISTS(SA_SIGINFO "signal.h" HAVE_SA_SIGINFO)
+ENDIF (HAVE_SIGINFO_H)
+
+IF (NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+ IF (HAVE_CLOCK_GETTIME)
+ CHECK_SYMBOL_EXISTS(CLOCK_PROCESS_CPUTIME_ID time.h HAVE_CLOCK_PROCESS_CPUTIME_ID)
+ CHECK_SYMBOL_EXISTS(CLOCK_VIRTUAL time.h HAVE_CLOCK_VIRTUAL)
+ ELSE (HAVE_CLOCK_GETTIME)
+ CHECK_INCLUDE_FILES(sys/timeb.h HAVE_SYS_TIMEB_H)
+ ENDIF (HAVE_CLOCK_GETTIME)
+ENDIF (NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+
+CHECK_C_SOURCE_RUNS("
+#include <stdbool.h>
+int main(int argc, char **argv) {
+ int a = 0, b = 0;
+ if (__atomic_compare_exchange_n(&a, &b, 1, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {
+ return 0;
+ }
+ return -1;
+}
+" HAVE_ATOMIC_BUILTINS)
+
+IF (NOT HAVE_ATOMIC_BUILTINS)
+ MESSAGE(STATUS "atomic builtins are -NOT- supported")
+ELSE ()
+ MESSAGE(STATUS "atomic builtins are supported")
+ENDIF ()
+
+CHECK_LIBRARY_EXISTS(atomic __atomic_fetch_add_4 "" HAVE_LIBATOMIC)
+IF (HAVE_LIBATOMIC)
+ list(APPEND CMAKE_REQUIRED_LIBRARIES "atomic")
+endif ()
+
+
+CHECK_C_SOURCE_RUNS("
+#include <x86intrin.h>
+int main(int argc, char **argv) {
+ __builtin_ia32_lfence ();
+ if (__builtin_ia32_rdtsc()) {
+ return 0;
+ }
+ return -1;
+}
+" HAVE_RDTSC)
+
+IF (NOT HAVE_RDTSC)
+ MESSAGE(STATUS "rdtsc intrinsic is -NOT- supported")
+ELSE ()
+ MESSAGE(STATUS "rdtsc intrinsic is supported")
+ENDIF ()
+
+IF (CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ # In linux, we need to mount /run/shm to test which could be unavailable
+ # on a build system. On the other hand, we know that linux has stupid
+ # but compatible shmem support, so we assume this macro as true
+ SET(HAVE_SANE_SHMEM 1)
+ CHECK_C_SOURCE_COMPILES("#define _GNU_SOURCE
+ #include <sys/socket.h>
+ int main (int argc, char **argv) {
+ return ((int*)(&recvmmsg))[argc];
+ }" HAVE_RECVMMSG)
+ CHECK_C_SOURCE_COMPILES("#define _GNU_SOURCE
+ #include <fcntl.h>
+ int main (int argc, char **argv) {
+ return ((int*)(&readahead))[argc];
+ }" HAVE_READAHEAD)
+ELSE ()
+ CHECK_C_SOURCE_RUNS("
+ #include <sys/mman.h>
+ #include <fcntl.h>
+ #include <unistd.h>
+ #define TEST_NAME \"/test-shmem-work\"
+ int
+ main (int argc, char **argv)
+ {
+ int fd;
+
+ fd = shm_open (TEST_NAME, O_RDWR | O_CREAT | O_EXCL, 00600);
+ if (fd == -1) {
+ return -1;
+ }
+ if (ftruncate (fd, 100) == -1) {
+ shm_unlink (TEST_NAME);
+ close (fd);
+ return -1;
+ }
+
+ if (ftruncate (fd, 200) == -1) {
+ shm_unlink (TEST_NAME);
+ close (fd);
+ return -1;
+ }
+ if (ftruncate (fd, 300) == -1) {
+ shm_unlink (TEST_NAME);
+ close (fd);
+ return -1;
+ }
+
+ close (fd);
+ shm_unlink (TEST_NAME);
+ return 0;
+ }
+ " HAVE_SANE_SHMEM)
+ IF (NOT HAVE_SANE_SHMEM)
+ MESSAGE(STATUS "shmem support is NOT compatible with POSIX")
+ ELSE ()
+ MESSAGE(STATUS "shmem support is compatible with POSIX")
+ ENDIF ()
+ENDIF ()
+
+IF (ENABLE_URL_INCLUDE MATCHES "ON")
+ FIND_LIBRARY(LIBFETCH_LIBRARY HINTS "${RSPAMD_SEARCH_PATH}"
+ NAMES fetch PATHS PATH_SUFFIXES lib64 lib
+ PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS}
+ DOC "Path where the libfetch library can be found")
+ IF (LIBFETCH_LIBRARY)
+ FIND_FILE(HAVE_FETCH_H HINTS "${RSPAMD_SEARCH_PATH}"
+ NAMES fetch.h
+ PATH_SUFFIXES include
+ PATHS ${RSPAMD_DEFAULT_INCLUDE_PATHS}
+ DOC "Path to libfetch header")
+ ELSE (LIBFETCH_LIBRARY)
+ # Try to find libcurl
+ ProcessPackage(CURL LIBRARY curl INCLUDE curl.h INCLUDE_SUFFIXES include/curl
+ ROOT ${CURL_ROOT})
+ IF (NOT WITH_CURL)
+ MESSAGE(WARNING "Neither libcurl nor libfetch were found, no support of URL includes in configuration")
+ ENDIF (NOT WITH_CURL)
+ ENDIF (LIBFETCH_LIBRARY)
+ENDIF (ENABLE_URL_INCLUDE MATCHES "ON")
+
+IF (NOT DESTDIR)
+ SET(DESTDIR $ENV{DESTDIR})
+ENDIF (NOT DESTDIR)
+
+
+################################ SUBDIRS SECTION ###########################
+ADD_SUBDIRECTORY(contrib/hiredis)
+INCLUDE_DIRECTORIES(BEFORE "${CMAKE_SOURCE_DIR}/contrib/hiredis")
+
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}")
+IF (HAVE_FETCH_H)
+ LIST(APPEND RSPAMD_REQUIRED_LIBRARIES fetch)
+ENDIF (HAVE_FETCH_H)
+
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "${LUA_LIBRARY}")
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ucl)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rdns)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES ottery)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES xxhash)
+
+IF (SYSTEM_XXHASH MATCHES "OFF")
+ ADD_SUBDIRECTORY(contrib/xxhash)
+ INCLUDE_DIRECTORIES("${CMAKE_SOURCE_DIR}/contrib/xxhash")
+ELSE ()
+ ProcessPackage(XXHASH LIBRARY xxhash INCLUDE xxhash.h
+ ROOT ${LIBXXHASH_ROOT_DIR} MODULES xxhash libxxhash)
+ENDIF ()
+ADD_SUBDIRECTORY(contrib/cdb)
+ADD_SUBDIRECTORY(contrib/http-parser)
+ADD_SUBDIRECTORY(contrib/fpconv)
+ADD_SUBDIRECTORY(contrib/lc-btrie)
+ADD_SUBDIRECTORY(contrib/libottery)
+IF (SYSTEM_ZSTD MATCHES "OFF")
+ ADD_SUBDIRECTORY(contrib/zstd)
+ELSE ()
+ ProcessPackage(LIBZSTD LIBRARY zstd INCLUDE zstd.h
+ ROOT ${LIBZSTD_ROOT_DIR} MODULES zstd libzstd)
+ ADD_DEFINITIONS(-DSYS_ZSTD)
+ENDIF ()
+IF (ENABLE_SNOWBALL MATCHES "ON")
+ ADD_SUBDIRECTORY(contrib/snowball)
+ SET(WITH_SNOWBALL 1)
+ENDIF ()
+ADD_SUBDIRECTORY(contrib/libucl)
+ADD_SUBDIRECTORY(contrib/librdns)
+ADD_SUBDIRECTORY(contrib/aho-corasick)
+ADD_SUBDIRECTORY(contrib/lua-lpeg)
+ADD_SUBDIRECTORY(contrib/t1ha)
+ADD_SUBDIRECTORY(contrib/libev)
+ADD_SUBDIRECTORY(contrib/kann)
+ADD_SUBDIRECTORY(contrib/fastutf8)
+ADD_SUBDIRECTORY(contrib/google-ced)
+IF (ENABLE_BACKWARD MATCHES "ON")
+ ADD_SUBDIRECTORY(contrib/backward-cpp)
+ message(STATUS "Backward-cpp config: ${BACKWARD_DEFINITIONS}")
+ELSE ()
+ set(BACKWARD_ENABLE)
+ macro(add_backward target)
+ # do nothing
+ endmacro()
+ENDIF ()
+IF (BACKWARD_LIBRARIES)
+ message(STATUS "Backward-cpp libraries: ${BACKWARD_LIBRARIES}")
+ENDIF ()
+
+IF (SYSTEM_FMT MATCHES "OFF")
+ INCLUDE_DIRECTORIES("${CMAKE_SOURCE_DIR}/contrib/fmt/include")
+ELSE ()
+ find_package(fmt)
+ENDIF ()
+IF (SYSTEM_DOCTEST MATCHES "OFF")
+ ADD_SUBDIRECTORY(contrib/doctest)
+ INCLUDE_DIRECTORIES("${CMAKE_SOURCE_DIR}/contrib/doctest")
+ELSE ()
+ find_package(doctest)
+ENDIF ()
+
+IF (NOT WITH_LUAJIT)
+ ADD_SUBDIRECTORY(contrib/lua-bit)
+ENDIF ()
+
+IF (ENABLE_LUA_REPL MATCHES "ON")
+ ADD_SUBDIRECTORY(contrib/replxx)
+ SET(WITH_LUA_REPL 1)
+ LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-replxx)
+ENDIF ()
+
+IF (ENABLE_SNOWBALL MATCHES "ON")
+ LIST(APPEND RSPAMD_REQUIRED_LIBRARIES stemmer)
+ENDIF ()
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-hiredis)
+
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-actrie)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-t1ha)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-ev)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-kann)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-ced)
+
+IF (ENABLE_CLANG_PLUGIN MATCHES "ON")
+ ADD_SUBDIRECTORY(clang-plugin)
+ENDIF ()
+
+ADD_SUBDIRECTORY(src)
+ADD_SUBDIRECTORY(test)
+ADD_SUBDIRECTORY(utils)
+
+############################ TARGETS SECTION ###############################
+
+CONFIGURE_FILE(config.h.in src/config.h)
+
+##################### INSTALLATION ##########################################
+
+# Binaries
+
+# Configs
+INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${CONFDIR})")
+INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${SHAREDIR})")
+INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${LUALIBDIR})")
+INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${PLUGINSDIR})")
+INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${RULESDIR})")
+
+# Install configs only if they are unchanged
+LIST(LENGTH CONFFILES CONFLIST_COUNT)
+MATH(EXPR CONFLIST_MAX ${CONFLIST_COUNT}-1)
+
+FILE(GLOB_RECURSE CONF_FILES RELATIVE "${CMAKE_SOURCE_DIR}/conf" CONFIGURE_DEPENDS
+ "${CMAKE_SOURCE_DIR}/conf/*.conf"
+ "${CMAKE_SOURCE_DIR}/conf/*.inc")
+FOREACH (CONF_FILE ${CONF_FILES})
+ GET_FILENAME_COMPONENT(_rp ${CONF_FILE} PATH)
+ INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${CONFDIR}/${_rp})")
+ INSTALL(FILES "${CMAKE_CURRENT_SOURCE_DIR}/conf/${CONF_FILE}"
+ DESTINATION ${CONFDIR}/${_rp})
+ENDFOREACH (CONF_FILE)
+
+# Lua plugins
+
+FILE(GLOB LUA_PLUGINS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/src/plugins/lua" CONFIGURE_DEPENDS
+ "${CMAKE_CURRENT_SOURCE_DIR}/src/plugins/lua/*.lua")
+FOREACH (LUA_PLUGIN ${LUA_PLUGINS})
+ GET_FILENAME_COMPONENT(_rp ${LUA_PLUGIN} PATH)
+ INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${PLUGINSDIR}/${_rp})")
+ INSTALL(FILES "src/plugins/lua/${LUA_PLUGIN}" DESTINATION ${PLUGINSDIR}/${_rp})
+ENDFOREACH (LUA_PLUGIN)
+
+
+# Install TLD list
+INSTALL(FILES "contrib/publicsuffix/effective_tld_names.dat" DESTINATION
+ "${SHAREDIR}")
+
+# Install languages data
+INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${SHAREDIR}/languages)")
+FILE(GLOB LANGUAGES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/contrib/languages-data/*.json")
+FOREACH (_LANG ${LANGUAGES})
+ INSTALL(FILES "${_LANG}" DESTINATION ${SHAREDIR}/languages)
+ENDFOREACH ()
+INSTALL(FILES "${CMAKE_CURRENT_SOURCE_DIR}/contrib/languages-data/stop_words" DESTINATION ${SHAREDIR}/languages)
+
+# Lua config
+FILE(GLOB_RECURSE LUA_CONFIGS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/rules" CONFIGURE_DEPENDS
+ "${CMAKE_CURRENT_SOURCE_DIR}/rules/*.lua")
+FOREACH (LUA_CONF ${LUA_CONFIGS})
+ GET_FILENAME_COMPONENT(_rp ${LUA_CONF} PATH)
+ INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${RULESDIR}/${_rp})")
+ INSTALL(FILES "rules/${LUA_CONF}" DESTINATION ${RULESDIR}/${_rp})
+ENDFOREACH (LUA_CONF)
+
+# Lua libs
+FILE(GLOB_RECURSE LUA_LIBS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/lualib" CONFIGURE_DEPENDS
+ "${CMAKE_CURRENT_SOURCE_DIR}/lualib/*.lua")
+FOREACH (LUA_LIB ${LUA_LIBS})
+ GET_FILENAME_COMPONENT(_rp ${LUA_LIB} PATH)
+ INSTALL(CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${LUALIBDIR}/${_rp})")
+ INSTALL(FILES "lualib/${LUA_LIB}" DESTINATION ${LUALIBDIR}/${_rp})
+ENDFOREACH (LUA_LIB)
+
+# Install lua fun library
+INSTALL(FILES "contrib/lua-fun/fun.lua" DESTINATION ${LUALIBDIR})
+INSTALL(FILES "contrib/lua-argparse/argparse.lua" DESTINATION ${LUALIBDIR})
+INSTALL(FILES "contrib/lua-tableshape/tableshape.lua" DESTINATION ${LUALIBDIR})
+INSTALL(FILES "contrib/lua-lupa/lupa.lua" DESTINATION ${LUALIBDIR})
+INSTALL(FILES "contrib/lua-lpeg/lpegre.lua" DESTINATION ${LUALIBDIR})
+
+# systemd unit
+IF (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND WANT_SYSTEMD_UNITS MATCHES "ON")
+ INSTALL(FILES "rspamd.service" DESTINATION ${SYSTEMDDIR})
+ENDIF (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND WANT_SYSTEMD_UNITS MATCHES "ON")
+
+# Manual pages
+INSTALL(FILES "doc/rspamd.8" DESTINATION ${MANDIR}/man8)
+INSTALL(FILES "doc/rspamc.1" DESTINATION ${MANDIR}/man1)
+INSTALL(FILES "doc/rspamadm.1" DESTINATION ${MANDIR}/man1)
+
+# Utils
+INSTALL(PROGRAMS "utils/rspamd_stats.pl" RENAME rspamd_stats DESTINATION bin)
+
+# Install webui
+IF (INSTALL_WEBUI MATCHES "ON")
+ INSTALL(DIRECTORY "interface/" DESTINATION ${WWWDIR} PATTERN ".git" EXCLUDE)
+ENDIF (INSTALL_WEBUI MATCHES "ON")
+
+
+INSTALL(DIRECTORY "contrib/elastic/" DESTINATION "${SHAREDIR}/elastic" PATTERN ".git" EXCLUDE)
+
+ADD_CUSTOM_TARGET(dist ${CMAKE_SOURCE_DIR}/dist.sh
+ "${CMAKE_BINARY_DIR}/rspamd-${RSPAMD_VERSION}.tar.xz" "${TAR}"
+ COMMENT "Create source distribution"
+ WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
+IF (NOT DEBIAN_BUILD)
+ ADD_CUSTOM_TARGET(check DEPENDS rspamd-test-cxx rspamd-test)
+ ADD_CUSTOM_TARGET(run-test DEPENDS check
+ COMMAND test/rspamd-test-cxx
+ COMMAND sh -c 'LUA_PATH="${CMAKE_SOURCE_DIR}/lualib/?.lua\;${CMAKE_SOURCE_DIR}/lualib/?/?.lua\;${CMAKE_SOURCE_DIR}/lualib/?/init.lua\;${CMAKE_SOURCE_DIR}/contrib/lua-?/?.lua"
+ test/rspamd-test -p /rspamd/lua')
+ENDIF (NOT DEBIAN_BUILD)
+
+
+# PVS Studio
+find_program(_PVS_STUDIO "pvs-studio-analyzer")
+
+if (_PVS_STUDIO)
+ include(PVS-Studio)
+ pvs_studio_add_target(TARGET ${PROJECT_NAME}.analyze
+ ANALYZE ${PROJECT_NAME} rspamd-server rspamadm rspamc
+ OUTPUT FORMAT errorfile
+ LOG target_${PROJECT_NAME}.err)
+endif ()
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..61790d5
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,128 @@
+# Contributing to Rspamd
+
+:tada: First off, thanks for taking the time to contribute! :tada:
+
+The following is a set of guidelines for contributing to Rspamd and its packages, which are hosted in
+the [Rspamd Organization](https://github.com/rspamd) on GitHub. These are mostly guidelines, not rules. Use your best
+judgment, and feel free to propose changes to this document in a pull request. This contribution policy is heavily
+inspired by [Atom editor](https://github.com/atom/atom).
+
+#### Table Of Contents
+
+[I don't want to read this whole thing, I just have a question](#i-dont-want-to-read-this-whole-thing-i-just-have-a-question)
+
+[How Can I Contribute?](#how-can-i-contribute)
+
+* [Reporting Bugs](#reporting-bugs)
+
+[Styleguides](#styleguides)
+
+* [Git Commit Messages](#git-commit-messages)
+* [Lua style guide](#lua-styleguide)
+
+## I don't want to read this whole thing I just have a question
+
+> **Note:** Please don't file an issue to ask a question. You'll get faster results by using the resources below.
+
+We have an official site with a detailed FAQ and various community support resources.
+
+* [Support channels explained](https://rspamd.com/support.html)
+* [Rspamd FAQ](https://rspamd.com/doc/faq.html)
+
+The best way to ask a question and to get a relevant reply is the mailing list:
+
+* [Join Rspamd mailing lists](https://lists.rspamd.com/)
+
+If chat is more your speed, you can join the Rspamd developers and users using Telegram or IRC:
+
+* [Join Rspamd telegram channel](http://t.me/rspamd)
+* [Join Rspamd IRC channel](https://oftc.net/):
+ * server: ircs://irc.oftc.net:6697
+ * channel: #rspamd
+
+Please bear in mind that even though telegram/irc are chat services, sometimes it takes several hours for community
+members to respond &mdash; please be patient!
+
+## How Can I Contribute?
+
+### Reporting Bugs
+
+This section guides you through submitting a bug report for Rspamd. Following these guidelines helps maintainers and the
+community understand your report :pencil:, reproduce the behavior :computer: :computer:, and find related reports :
+mag_right:.
+
+When you are creating a bug report, please [include as many details as possible](#how-do-i-submit-a-good-bug-report).
+Fill out the required template, the information it asks for helps us resolve issues faster.
+
+> **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new
+> issue and include a link to the original issue in the body of your new one.
+
+#### Before Submitting A Bug Report
+
+* Read about bug reporting in general: https://rspamd.com/doc/faq.html#how-to-report-bugs-found-in-rspamd
+* Enable relevant debugging logs: https://rspamd.com/doc/faq.html#how-to-debug-some-module-in-rspamd
+* Check the FAQs about Core files in case of fatal
+ crash: https://rspamd.com/doc/faq.html#how-to-figure-out-why-rspamd-process-crashed
+* Check that your issue isn't already filed: https://github.com/issues?utf8=%E2%9C%93&q=is%3Aissue+user%3Arspamd
+* Check that there is not already an experimental package or master branch
+
+#### How Do I Submit A (Good) Bug Report?
+
+Explain the problem and include additional details to help maintainers reproduce the problem:
+
+* **Use a clear and descriptive title** for the issue to identify the problem.
+* **Describe the exact steps which reproduce the problem** in as many details as possible. For example, start by
+ explaining how you started Rspamd, e.g. which custom configuration are you using, or what message have you scanned.
+ When listing steps, **don't just say what you did, but explain how you did it**.
+* **Provide specific examples to demonstrate the steps**. Include links to files or GitHub projects, or copy/pasteable
+ snippets, which you use in those examples. If you're providing snippets in the issue,
+ use [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
+* **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that
+ behavior.
+* **Explain which behavior you expected to see instead and why.**
+* **If you're reporting that Rspamd crashed**, include a crash report with a stack trace from the operating
+ system: https://rspamd.com/doc/faq.html#how-to-figure-out-why-rspamd-process-crashed
+* **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened and
+ share more information using the guidelines below.
+
+Provide more context by answering these questions:
+
+* **Did the problem start happening recently** (e.g. after updating to a new version of Rspamd) or was this always a
+ problem?
+* If the problem started happening recently, **can you reproduce the problem in an older version of Rspamd?** What's the
+ most recent version in which the problem doesn't happen? You can download older versions of Rspamd
+ from [the releases page](https://github.com/rspamd/rspamd/releases).
+* **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which
+ conditions it normally happens.
+* If the problem is related to scanning messages, **does the problem happen for all messages or only some?**
+
+Include details about your configuration and environment:
+
+* **Which version of Rspamd are you using?** Use `rspamd --version` to get the detailed information about the version
+ and the compile flags used.
+* **What's the name and version of the OS you're using**?
+* **What hardware are you using, including CPU generation**, e.g. Intel Haswell or ArmV7? If you have `gcc` installed,
+ that could be achieved by the following command: `gcc -march=native -Q --help=target|grep march`. In Linux, you can
+ also check `/proc/cpuinfo` file for the required details.
+
+## Styleguides
+
+### Git Commit Messages
+
+* Use the present tense ("Add feature" not "Added feature")
+* Use the imperative mood ("Move cursor to..." not "Moves cursor to...")
+* Limit the first line to 72 characters or less (without tag)
+* Reference issues and pull requests liberally after the first line
+* Consider starting the commit message with an applicable tag:
+ * [Minor] - minor issue/improvement not worth to mention in ChangeLog
+ * [Feature] - a significant feature
+ * [Fix] - bug fix
+ * [CritFix] - critical bug fix worth a new version releasing
+ * [Rework] - some significant logic rework
+ * [Conf] - configuration change
+ * [Rules] - rules change
+
+### Lua styleguide
+
+Please use the following [Lua style guide](lua_style.md) when contributing changes to Lua code. This guide is both
+applicable for rules, libraries and plugins.
diff --git a/ChangeLog b/ChangeLog
new file mode 100644
index 0000000..1b3e867
--- /dev/null
+++ b/ChangeLog
@@ -0,0 +1,6482 @@
+3.8.1: 25 Jan 2024
+ * [Fix] Fix headers insertion in the ordered list
+ * [Fix] Fix learn error propagation
+ * [Fix] Fix learning with long prefixes
+ * [Fix] Fix potential double free by expclicitly zeroing pointers
+ * [Fix] Fix storing of the bayes tokens
+ * [Fix] Use short comments rather than long one
+
+3.8.0: 19 Jan 2024
+ * [Project] Rspamadm fuzzyping command
+ * [Project] Use Redis scripts for Bayes statistics and cache
+ * [Project] Support JSON logging
+ * [CritFix] - Fix reported length of logging structure
+ * [Feature] Allow to set `max_users` for Redis bayes backend
+ * [Feature] Escape JSON when needed
+ * [Feature] Proxy: Allow `encrypted_only` option
+ * [Feature] Reiterate on gtube patterns
+ * [Feature] Support ping command in fuzzy storage
+ * [Feature] Support suppressing DMARC reporting for particular recipients
+ * [Feature] rbl: support disabling or replacing url_whitelist per RBL
+ * [Fix] Another try to fix setproctitle
+ * [Fix] Cleanup session on exit
+ * [Fix] Do not cleanup hyperscan files unless new ones are loaded
+ * [Fix] Fix bad access when printing pending events on failure path
+ * [Fix] Fix issues with the raw header on header insertion/modification
+ * [Fix] Fix overflow in fuzzy_stats command
+ * [Fix] Fix some corner cases when parsing maps with no newlines
+ * [Fix] Fix various issues with canonicalisation of the paths
+ * [Fix] Fix wrong raw_len usage
+ * [Fix] Make words selection random deterministic upon content
+ * [Fix] Properly set config field when creating tasks from Lua
+ * [Fix] Really fix the language detector statistical heuristic
+ * [Fix] Set loaded variable explicitly
+ * [Fix] dkim_signing: siging_table: lowercase before lookup
+ * [Fix] known_senders: fix config handling
+ * [Fix] rbl: fix `exclude_local`
+
+3.7.5: 15 Dec 2023
+ * Add support for loongarch
+ * [Fix] Fix rspamadm lua logger
+ * [Fix] Fix systemd logging
+ * [Fix] dkim_signing: siging_table: lowercase before lookup
+ * [Minor] Fix build with no hyperscan
+ * [Minor] Try improve test stability
+
+3.7.4: 14 Nov 2023
+ * [Enhancement] Add composite rule for suspicious URLs in suspicious messages
+ * [Fix] Another try to fix setproctitle
+ * [Fix] Do not cleanup hyperscan files unless new ones are loaded
+ * [Fix] Fix various issues with canonicalisation of the paths
+ * [Fix] Properly set config field when creating tasks from Lua
+ * [Fix] Set loaded variable explicitly
+ * [Fix] known_senders: fix config handling
+ * [Fix] rbl: fix `exclude_local`
+ * [Minor] Add missing <algorithm> include for gcc 14
+ * [Minor] Add rule for messages missing both X-Mailer and User-Agent header
+ * [Minor] Bundle fasttext as linux distributives are just useless
+ * [Minor] Enable fasttext on RPM based linux
+ * [Minor] Forgot it in one more place
+ * [Minor] Improve FREEMAIL_AFF capture rates
+ * [Minor] Limit fasttext to amd64 only
+ * [Minor] Properly search for fasttext include
+ * [Minor] Treat *.zpaq attachments as archives and harmful
+ * [Minor] Update to 3.7.4
+ * [Minor] force_actions: set a group for symbols
+ * [WebUI] Update map editor
+
+3.7.3: 27 Oct 2023
+ * [Fix] Emergency fix for the hyperscan path error
+
+3.7.2: 26 Oct 2023
+ * [Feature] rbl: support checking returncodes by CIDR
+ * [Feature] rbl: support checking returncodes by regex
+ * [Feature] rbl: support globbed return codes
+ * [Fix] DMARC reporting: fix reporting for subdomains
+ * [Fix] Deal with fmtlib exceptions properly
+ * [Fix] backport fix for dlfcn.h from backward-cpp
+ * [Rules] Blank spam detection
+
+3.7.1: 07 Oct 2023
+ * [CritFix] Fix leak in `gzip` function
+ * [Feature] Add ICAP Content-Type and Filename
+ * [Feature] Add `logging`->`task_max_elts` option
+ * [Feature] Add utility to split string like stuff for C++ code
+ * [Feature] Allow to set HTTP auth parameters for the maps
+ * [Feature] Check for plugin configuration errors on `configtest`
+ * [Feature] `known_senders` plugin
+ * [Feature] Use backward-cpp instead of manual libunwind stuff
+ * [Feature] rbl: support checking numeric URLs in isolation
+ * [Fix] CMakeLists.txt remove whitespace added by linter as it makes tests fail
+ * [Fix] Change Date: header location to conform with RFC
+ * [Fix] Correct format pattern for RE tree tempfile name
+ * [Fix] Correct format string for unw_word_t
+ * [Fix] Do not accept invalid ucl object types
+ * [Fix] Do not pollute public headers with libev internals
+ * [Fix] Do not set output type if list application failed
+ * [Fix] Fix `url:set_redirected` method
+ * [Fix] Fix format string and some length issues
+ * [Fix] Fix grammar definition for content-disposition attributes
+ * [Fix] Fix lua schema enrichment logic for Redis params
+ * [Fix] Fix lua stack corruption when logging large tables
+ * [Fix] Fix merge table utility
+ * [Fix] Fix output of non-RSA DKIM keys
+ * [Fix] Fix some corner cases of single-host urls parsing
+ * [Fix] Fix various issues in the `url_redirector` plugin
+ * [Fix] MISSING_MIMEOLE: avoid matching messages from Android GMail app (#4561)
+ * [Fix] Prevent DNSWL sabotage
+ * [Fix] Try to fix unzip function
+ * [Fix] rbl: really fix dependency registration when symbols_prefixes is used
+ * [Fix] rspamadm mime: arguments beginning with letter `t`
+ * [Rework] Breaking: return back to semver
+ * [Rework] Move rcl logic to C++
+
+3.6: 03 Aug 2023
+ * [Conf] Add `one_shot` to some specific multimap rules
+ * [Conf] Add language detection configuration
+ * [Conf] Add missing attributes for the language detection configuration
+ * [Conf] Remove outdated composite rules
+ * [Feature] Add `sentinel_password` option
+ * [Feature] Add ability to deny specific fuzzy flags by default
+ * [Feature] Add controller endpoint to get fuzzy hashes from messages
+ * [Feature] Add extra symbol when URL redirector reaches nested limit
+ * [Feature] Add function to transliterate utf8 to ascii with some normalisation
+ * [Feature] Add html parsing limit
+ * [Feature] Add order to urls structure
+ * [Feature] Add some missing functions to `lua_rsa` library
+ * [Feature] Allow fuzzy workers to exchange blocked information
+ * [Feature] Allow to have weak flags in fuzzy storage
+ * [Feature] Allow to read options from maps in the multimap plugin
+ * [Feature] Allow to use other methods when fasttext detection is enabled
+ * [Feature] Count stats per key per flag
+ * [Feature] Finish all features of dkim_keygen in Lua
+ * [Feature] Khash: Allow static initialisation
+ * [Feature] Maps: Add on_load support
+ * [Feature] Preliminary implementation of dynamic composites
+ * [Feature] Process HTML parts before text ones
+ * [Feature] Reorganise struct rspamd_url to be 64 bytes size
+ * [Feature] Save fuzzy ratelimit buckets
+ * [Feature] Use in-place deflation for strings
+ * [Feature] external_relay: add ip_map strategy
+ * [Fix] Avoid race between config new/free by using a counter
+ * [Fix] Do not use `rspamadm.dkim_keygen`
+ * [Fix] Feed fasttext language model with the pre-tokenized words
+ * [Fix] Fix `rspamd_has_only_html_part`
+ * [Fix] Fix an old issue with order of destruction race between redis pool and lua
+ * [Fix] Fix format string usage
+ * [Fix] Fix parsing due to old bug revealed
+ * [Fix] Fix parsing of the mask values that are invalid
+ * [Fix] Ignore non-unique stop words
+ * [Fix] Include the last character when parsing the last header with no value
+ * [Fix] More fixes to fuzzystat
+ * [Fix] Set proper counter
+ * [Fix] Try harder to clean pending bucket
+ * [Fix] Try harder to remove bad hyperscan files
+ * [Fix] Update stats before encryption...
+ * [Fix] dmarc gramar - allow spaces before ";"
+ * [Fix] rbl: fix dependency registration when symbols_prefixes is used
+ * [Fix] remove obsolete rspamd-redirector files
+ * [Fix] test external_relay: count should always be the last rule, as it have no matching condition
+ * [Project] Allow to register multimap symbols dynamically
+ * [Project] Implement fasttext language detection
+ * [Rework] Default max shots must not influence options
+ * [Rework] Write dkim keygen tool in lua
+ * [Rules] Add thread hijacking composite rule
+
+3.5: 17 Mar 2023
+ * [Conf] Clarify `timeout` in rspamd_proxy
+ * [Conf] Fix hashbl
+ * [Conf] Add SURBL hashbl support (WIP)
+ * [Conf] RBL: Fix selector
+ * [Conf] Reduce the default timeout
+ * [CritFix] Deserialise hyperscan to the page-aligned space to prevent alignment issues
+ * [CritFix] Fill path field in hyperscan notice command
+ * [Feature] Add `thresholds` field to the scan result
+ * [Feature] Add possibility to execute lua scripts for blocked fuzzy clients
+ * [Feature] Add preliminary support of the external maps in the multimap plugin
+ * [Feature] Allow to build a map by combining tuples of selectors
+ * [Feature] Allow to query external maps for settings
+ * [Feature] Allow to specify `selector_alias` in the maps definition
+ * [Feature] Enable Mime part filters on antivirus module
+ * [Feature] Improve ratelimit redis scripts
+ * [Feature] Selectors: Add specific_urls_filter_map extractor
+ * [Feature] Some rework of the selectors framework
+ * [Fix] Add O_CREAT flag when creating a file
+ * [Fix] Add a database check function unless we have anything from Hyperscan
+ * [Fix] Add hyperscan databases sanity check
+ * [Fix] Add workaround for ENOBUFS error on sending
+ * [Fix] Do not do `lstat` when we are creating file
+ * [Fix] Finally get rid of RSPAMD_USE_47BIT_LIGHTUSERDATA_HACK
+ * [Fix] Fix boundaries that contain only dashes
+ * [Fix] Fix off-by-one error in css tokenizer
+ * [Fix] Fix url reputation plugin
+ * [Fix] Fix usage of the Redis config schema as `extra_fields`
+ * [Fix] Further checks for the hs_scratch_alloc
+ * [Fix] Honor group flag for one shot
+ * [Fix] Normalize glob paths to avoid hash table misses
+ * [Fix] Ratelimit: Use unpack for `HMGET` return value
+ * [Fix] Rbl: Fix helo check pipeline
+ * [Fix] Replace broken strict_domains with phishing_exceptions
+ * [Fix] Restore `strict_domains` support
+ * [Fix] Return true from has_urls(true) if only emails are present
+ * [Fix] Rework lists applications
+ * [Fix] Set symcache item in coroutine calls
+ * [Fix] Treat `hs_allocate_scratch` errors as non-fatal
+ * [Fix] Treat hostnames with no dots as eSLD of their own
+ * [Fix] received: filtering of artificial header
+ * [Project] Add external maps support
+ * [Rework] Breaking: Do not report soft reject in history
+ * [Rework] Convert chartable plugin to c++ for convenience
+ * [Rework] Use a different approach for customization of the settings
+ * [Rules] Mid: Add MID_END_EQ_FROM_USER_PART rule
+
+3.4: 01 Nov 2022
+ * [CritFix] Restore compatibility with the integrations and headers alterations
+ * [Feature] Milter_headers: Add `x-rspamd-action` routine
+ * [Feature] Share hyperscan database among processes
+ * [Fix] Another corner case in url parsing
+ * [Fix] Another fix for the enable password
+ * [Fix] Another try to fix close method in lua_tcp
+ * [Fix] Fix additional fields in the Redis schema
+ * [Fix] Fix emoji joiner FP
+ * [Fix] Fix favicon.ico Content-Type header
+ * [Fix] Fix hang when close is used
+ * [Fix] Lua_tcp: Sigh, another try to fix `close` invocation
+ * [Fix] Mx_check: Cache the fact of a missing MX record
+ * [Fix] Try to fix parsing of the unencoded `>` characters in html attributes
+ * [Fix] Try to fix the case where password == enable_password
+ * [Project] (Re)implement hyperscan caching
+ * [Project] Rework cleanup
+ * [Project] Synchronize hyperscan caches via the main process
+ * [Rework] Convert multipattern to use hyperscan tools
+ * [Rework] Make http normalize path function a generic function
+ * [Rework] Split locked and unlocked files, as mmap does not need flock normally
+ * [Rework] Start movement of the hyperscan related routines into a single unit
+ * [Rework] Store the current worker, so other libraries could use this information
+ * [Rework] Use blocking socket for IPC between main and workers
+ * [Rework] Use more predictable size for commands buffers
+ * [Rules] Do not insert ONCE_RECEIVED_STRICT on RDNS missing
+ * [Rules] Reduce score of HTTP_TO_HTTPS - subject to remove completely
+
+3.3: 01 Oct 2022
+ * [Conf] Add missing groups for whitelist module symbols
+ * [CritFix] Neural: Fix keys regression after #3968
+ * [Feature] Accept upstream in lua_tcp
+ * [Feature] Add ability to statically maintain disabled/enabled patterns
+ * [Feature] Add function to store upstreams for HTTP urls
+ * [Feature] Allow augmentations set in Lua API
+ * [Feature] Allow lua_http module to accept upstreams
+ * [Feature] Allow to limit write access to fuzzy storage by key
+ * [Feature] Allow to sort symbols output
+ * [Feature] Check content for binary stuff before dumping it to Lua
+ * [Feature] Implement symbols augmentations
+ * [Fix] Add missing flags
+ * [Fix] Add more sanity checks for rua in dmarc_report
+ * [Fix] Adjust length of the fuzzy checks for short text parts
+ * [Fix] Another try to fix add headers compatibility logic
+ * [Fix] Another try to fix race condition in the runtime destruction
+ * [Fix] Avoid cyclic references in symcache and fix memory leaks
+ * [Fix] Avoid overriding IP with Sender IP
+ * [Fix] BAD_REP_POLICIES did not trigger when message was classified as spam by Bayes
+ * [Fix] Bind AF_UNIX DGRAM client connection to annonymous address
+ * [Fix] Disable IPv6 lookups for Blocklist.de RBL
+ * [Fix] Distinguish dynamic and static items
+ * [Fix] Dkim: Ignore unknown DKIM kv pairs as stated in RFC
+ * [Fix] Dmarc report: Use local timezone instead of GMT
+ * [Fix] Do not exclude authenticated users from URIBL lookups
+ * [Fix] Empty envelopes should not be emitted as arrays (json+messagepack) when populated envelopes are objects. This greatly complicates decoding in strictly typed languages.
+ * [Fix] External_relay: Restore the originating hostname check
+ * [Fix] Fix DKIM keys with spaces still allowing errors on invalid base64
+ * [Fix] Fix copying of sockaddr_un addresses
+ * [Fix] Fix crash with cname replies
+ * [Fix] Fix dependencies propagation
+ * [Fix] Fix iteration over milter headers
+ * [Fix] Fix ordering when sorting symcache
+ * [Fix] Fix reading of the cached maps
+ * [Fix] Fix several issues with the HTTP keepalive parsing
+ * [Fix] Fix stack smashing
+ * [Fix] Fix synchronous auth/select in lua_redis
+ * [Fix] Fix various symcache issues
+ * [Fix] Ignore all (I hope) unknown DKIM signature KV pairs
+ * [Fix] Ignore directories in RarV5 archives
+ * [Fix] Libucl: avoid memory leak on objects merging
+ * [Fix] Lua_tcp: Another try to fix closing logic
+ * [Fix] Mempool: Fix alloc_array function to actually multiply nmembers by size
+ * [Fix] Only check allowed fuzzy worker update ips for non-unix sockets
+ * [Fix] Plug memory leak in regexp destruction with pcre2
+ * [Fix] Properly check the original email flag
+ * [Fix] Properly deal with `get_symbol/get_metric_symbol` ambiguity
+ * [Fix] Properly parse expressions atoms
+ * [Fix] Properly set `Host` in rspamd_proxy
+ * [Fix] Rbl: Fix received positioned checks
+ * [Fix] Remove check for a score with no symbol being registered
+ * [Fix] Same fix for lua_tcp
+ * [Fix] Skip cname records when processing SPF records
+ * [Fix] Skip sending dmarc reports in no-opt mode fixes https://github.com/rspamd/rspamd/issues/4241
+ * [Fix] Stop slow timer on task destruction
+ * [Fix] Symcache: Do not use C style comparators in C++ sorts
+ * [Fix] Try to avoid a corner case for `@` pattern
+ * [Fix] Try to fix dkim reputation adjustements
+ * [Fix] Try to fix passthrough results processing logic
+ * [Fix] Try to fix the mess with read only flag
+ * [Fix] Upstreams: Don't ignore revive_time config option
+ * [Fix] Use proper format string, sigh...
+ * [Fix] Use space category in ragel automata to resolve space characters
+ * [Fix] Zstd: Fix compression with the new Zstd API
+ * [Fix] milter_headers: Header fields may be inserted at wrong position.
+ * [Project] Add experimental HTTP statistics backend
+ * [Project] Add more methods for symbols addition
+ * [Project] Add raii_sink file helper
+ * [Project] Add some more methods
+ * [Project] Add symbols processing methods
+ * [Project] Allow `=` separated augmentations to be treated as kv pairs
+ * [Project] Allow to extract augmentation values
+ * [Project] Few more methods
+ * [Project] Fix on conditions
+ * [Project] Further efforts to make a more consistent architecture
+ * [Project] Further rework
+ * [Project] Further rework tracking
+ * [Project] Further split of the code
+ * [Project] Get rid of C style ctors/dtors
+ * [Project] Http_stat: Notice statfiles when creating runtime
+ * [Project] Implement dynamic items lookup and processing
+ * [Project] Implement item finalization
+ * [Project] Implement more methods
+ * [Project] Implement runtime creation
+ * [Project] Implement settings processing + some neats
+ * [Project] Implement some conditions checks
+ * [Project] Implement validation logic
+ * [Project] More methods
+ * [Project] Move runtime cache part to a separate unit
+ * [Project] Move some more methods
+ * [Project] Re-implement counters method
+ * [Project] Reimplement dependencies processing
+ * [Project] Remove obsoleted methods
+ * [Project] Remove old code (finally)
+ * [Project] Rework symbols execution
+ * [Project] Some more adjustments in symbols registration
+ * [Project] Start rewrite symcache in c++
+ * [Project] Support augmentations with values
+ * [Project] Symcache: Use ordered filters to avoid extra lookups
+ * [Rework] Another movement
+ * [Rework] Augmentations can now imply flags
+ * [Rework] Further steps
+ * [Rework] Further work on deps processing
+ * [Rework] Implement cache resorting
+ * [Rework] Isolate disable/enable logic for the configuration ucl objects
+ * [Rework] Move item implementation to a separate header
+ * [Rework] Multimap: Avoid prefilters usage where augmentations can be used
+ * [Rework] Pass upstream when sending TCP requests
+ * [Rework] Re-implement cache sorting
+ * [Rework] Reimplement saving/loading the cache items
+ * [Rework] Reiterate on priorities
+ * [Rework] Rework files structure
+ * [Rework] Rewrite rspamc in C++
+ * [Rework] Simplify scores check and extend it to pre/post filters
+ * [Rework] Switch minimum C++ standard version to C++20
+ * [Rework] Try to fix the mess with types & flags
+ * [Rework] Use another version of hash table from the same author
+ * [Rework] Use dynamic items for calling callbacks
+ * [Rework] Use dynamic items in the callbacks
+ * [Rework] Use hash map for id->symbol mappings
+ * [Rework] Use khash instead of uthash in rdns compression logic
+
+3.2: 26 Mar 2022
+ * [Conf] Score MIME_OBFUSCATED_ARCHIVE to 8 points
+ * [Conf] Set one_shot for URIBL rules by default
+ * [CritFix] Fix upstreams name resolution when there is also a port
+ * [Feature] Add ROC feature to neural network plugin
+ * [Feature] Add public suffic compilation utility
+ * [Feature] Add support of Cloudmark
+ * [Feature] Allow hyperscan for ppc64, as vectorscan now suports it.
+ * [Feature] Allow to skip DNS resolution for keep-alive connections
+ * [Feature] Aws_s3: Allow to store large parts separately
+ * [Feature] BIMI: Add preliminary version of the BIMI plugin
+ * [Feature] JSON endpoint for querying maps
+ * [Feature] Lua_magic: Add a sane CSV heuristic
+ * [Feature] Lua_mime: Add schema for message transfer
+ * [Feature] Output average scan time in /stat endpoint
+ * [Feature] Show average scan time in `rspamc stat` output
+ * [Fix] Add guards to avoid race condition on TCP connection
+ * [Fix] Allow spaces in DKIM key records
+ * [Fix] Apply the similar fix to the url_reputation
+ * [Fix] Avoid overwriting whitelisted_signers_map
+ * [Fix] Backport PR from libucl
+ * [Fix] Clear SSL errors
+ * [Fix] ClickHouse cleanup of old partitions
+ * [Fix] Do not double call error handler on ssl errors in the timeout path
+ * [Fix] Do not forget to clear pointers on IOC reset
+ * [Fix] External_relay: Remove useless check of the map value
+ * [Fix] Find suspicious url encodings that could break url extraction
+ * [Fix] Fix HTTP(s) client timeout
+ * [Fix] Fix exclude flags setting
+ * [Fix] Fix expanding of the variables
+ * [Fix] Fix host header usage in lua_http
+ * [Fix] Fix http maps shared memory cache cleanup
+ * [Fix] Fix logic in HTML processing FSM
+ * [Fix] Fix parsing of the compound mailto urls
+ * [Fix] Fix processing captures from pcre2
+ * [Fix] Fix removing from khash
+ * [Fix] Fix stuctured headers pushing
+ * [Fix] Further fix for i386 compilation
+ * [Fix] Improve duplicate settings error reporting
+ * [Fix] Lua: task:remove_result didn't work in some cases
+ * [Fix] Output service parts as well
+ * [Fix] Phishing: Deal with phishing + redirected URL
+ * [Fix] Phishing: Fix finding domains in the phishing map
+ * [Fix] Plug memory leak by using mempool for a copied address
+ * [Fix] Properly find the request and the number of requested entries
+ * [Fix] Rbl: Fix inversed logic of the url_full_hostname
+ * [Fix] Read file maps if they were not pre-read during preload
+ * [Fix] Restrict x86_64 assembly to x86_64
+ * [Fix] Return a real number of recipients when dealing with aliases
+ * [Fix] Rework unshedule DNS request function
+ * [Fix] Support definition of ungrouped symbol in conf file, use group info from lua or other conf file
+ * [Fix] Unschedule DNS request when clearing IO channel
+ * [Fix] When checking for phishing, we need to convert punicode -> UTF8, not vice versa
+ * [Fix] lua_cfg_transform - actions without score (discard)
+ * [Fix] lua_cfg_transform - silly break break actions
+ * [Fix] ratelimit - symbol per bucket
+ * [Project] BIMI: Fix helper integration issues
+ * [Project] Further DNS over TCP architecturing
+ * [Project] Rdns: Add more functions for TCP based requests
+ * [Project] Rdns: Add preliminary reading logic for TCP channels
+ * [Project] Rdns: Add reaper for inactive TCP connections
+ * [Project] Rdns: Add timeout logic for TCP requests
+ * [Project] Rdns: Do not treat TCP channels failure as fatal
+ * [Project] Rdns: Fix TCP connection mess
+ * [Project] Rdns: Fix TCP stuff cleanup
+ * [Project] Rdns: Fix various ownership issues
+ * [Project] Rdns: Implement TCP writing logic
+ * [Project] Rdns: Initial support of TCP IO channels
+ * [Project] Rdns: More fixes in TCP handling
+ * [Project] Rdns: Restore the previous EDNS0 size
+ * [Project] Rdns: Send truncated replies via TCP
+ * [Project] Rdns: Unregister TCP requests
+ * [Rework] Allow to restore SSL handlers after keepalive pooling
+ * [Rework] Allow to set a different behaviour for actions from settings
+ * [Rework] Include SSL flag into keepalive hash
+ * [Rework] Make `rspamadm dmarc_report` default behaviour more sane
+ * [Rework] Mempool: Use explicit alignment
+ * [Rework] Rdns: Use faster and more compact hash table for DNS requests
+ * [Rework] Rework SSL flag operations
+ * [Rework] Take disabled flag into account
+ * [Rework] Timeouts are now global per event and not reseted by IO activity
+ * [Rework] Use xxh3 as a default hash and fix memory/alignment issues
+ * [Rules] Fix old rules to stop global functions usage
+ * [Rules] Fix symbol for DKIM temporary failure
+ * [Rules] Remove ancient and inefficient rules
+ * [Rules] Slightly reduce MULTIPLE_FROM score
+
+3.1: 01 Nov 2021
+ * [Feature] Add junk_threshold for autolearn
+ * [Feature] Add neural test command
+ * [Feature] Antivirus: Allow to set fake eicar patterns for testing AV engines
+ * [Feature] Lua_cdb: Add cdb building interface
+ * [Feature] Ratelimit: Add per bucket configurations
+ * [Feature] S3: Allow to store structured data in messagepack
+ * [Fix] Add concept of uncancellable events to prevent use-after-free
+ * [Fix] Add temporary guard to prevent linked list exploitation
+ * [Fix] Another rework of the ucl hashing
+ * [Fix] Another try to fix references safety
+ * [Fix] Another try to fix rspamd_text passing in the selectors
+ * [Fix] Avoid copy for received structure as it has raw C pointers
+ * [Fix] Avoid dangling reference
+ * [Fix] Correctly check numeric URLs in URL DNS lists
+ * [Fix] Delete the correct pointer type
+ * [Fix] Dmarc: Always lowercase domain
+ * [Fix] Fix compilation of the hyperscan databases with errors
+ * [Fix] Fix hash table lookup
+ * [Fix] Fix http message flag shift
+ * [Fix] Fix parsing of the from_hostname when it is an IP address
+ * [Fix] Fix parsing of the unquoted attributes in HTML
+ * [Fix] Fix passing of rspamd_text in selectors pipelines
+ * [Fix] Fix rubbish QP sequences decoding
+ * [Fix] Fix some complicated case with the closing tags parsing
+ * [Fix] Fix the case when l tag is too small
+ * [Fix] Html: Fix the case where only bgcolor is explicitly set
+ * [Fix] Libucl: Fix deletion from ucl objects
+ * [Fix] Namespace and add metadata for OpenMetrics, fix interleaving
+ * [Fix] Plug memory leak in http settings reload
+ * [Fix] Preserve SPF top record in the mempool variable
+ * [Fix] Remove aarch64 GC64 workaround
+ * [Fix] Remove bogus G_LIKELY
+ * [Fix] Spf: Do not parse non TXT DNS replies as TXT replies
+ * [Fix] Try to use on_connect/on_disconnect callbacks to handle internal Redis failures
+ * [Fix] buffer overflow in rspamc counters
+ * [Fix] fix static building
+ * [Fix] lua_scanners - message_min_words logic
+ * [Fix] src/lua/lua_mimepart.c: fix null dereference
+ * [Project] Add constant iterators
+ * [Project] Add helper library to handle mime strings in a more safe matter
+ * [Project] Add preliminary support of CDB bayes dump
+ * [Project] Add trim operations
+ * [Project] Allow mempool allocated mime strings
+ * [Project] Cdb: Finish backend implementation
+ * [Project] Cdb: Fix configuration load
+ * [Project] Cdb: Use shared data between cdb statfiles
+ * [Project] Cdb: continue statistics backend implementation
+ * [Project] Finish received headers rework part
+ * [Project] Move C++ specific declarations to C++ header
+ * [Project] Rework received headers parsing to C++
+ * [Project] Start using of the new received structure
+ * [Project] Start work on cdb backend
+ * [Rework] Further rework of the redis pool
+ * [Rework] Redis_pool: fix issues found
+ * [Rework] Rework learn and add classify condition
+ * [Rework] Save invisible content to a separate buffer
+ * [Rework] Start rewriting of the redis pool logic
+ * [Rules] Improve zero font rule
+
+3.0: 19 Aug 2021
+ * [Conf] Align ARC scores with DKIM scores
+ * [CritFix] Neural: Fix sorting application
+ * [Feature] Add a simple dumper for bayes tokens
+ * [Feature] Add lua_maps.fill_config_maps function
+ * [Feature] Add preliminary exporter to AWS S3
+ * [Feature] Add preliminary restore bayes support
+ * [Feature] Add race condition protection against hs_helper restarts
+ * [Feature] Add rspamd_utf8_strcmp utility
+ * [Feature] Add zstd streaming API
+ * [Feature] Allow to log severity level explicitly
+ * [Feature] Allow to save and show attachment name when inserting AV scan results
+ * [Feature] Allow to sort urls for Lua
+ * [Feature] Allow to specify different timeouts/retransmits for fuzzy rules
+ * [Feature] Aws_s3: Allow to compress data stored
+ * [Feature] CMakeLists.txt: Change check and run-test to use rspamd-test-cxx * fixes #3807
+ * [Feature] Dmarc_report: allow sending reports in batches
+ * [Feature] Fuzzy_check: Allow to disable subject when making short text hash
+ * [Feature] Lua_cryptobox: Add keyed ssl hash functions via HMAC
+ * [Feature] Lua_task: Add get_urls_filtered method
+ * [Feature] Make monitored checks less frequent
+ * [Feature] Milter_headers: Add x-rspamd-pre-result header
+ * [Feature] Neural: Allow to balance FP/FN for the network
+ * [Feature] Ppopagate monitored errors from rbl module
+ * [Feature] Pyzor calculate score dynamically Count - WL-Count of default_score in percent
+ * [Feature] Rbl: Distinguish flattened and non-flattened selectors in RBL requests
+ * [Feature] Re-add pyzor support
+ * [Feature] Settings: add ip_map check and rework structure slightly
+ * [Feature] Spamassassin: Allow to set the default priority for SA scores
+ * [Feature] Strip smtp comments from message id
+ * [Feature] add SYSTEM_ZSTD cmake option To use the system zstd instead on the bundled version
+ * [Feature] external_relay plugin
+ * [Feature] rspamadm clickhouse neural_train subcommand
+ * [Fix] #3400 milter_headers: fix inverted logic for extended_headers_rcpt
+ * [Fix] ASN: fix _FAIL symbol for when main symbol is disabled
+ * [Fix] Add a special logic for text part with no text extraction
+ * [Fix] Add diacritics flag for several eu languages
+ * [Fix] Another FSM fix to accomodate possibility of multiple consequent ?
+ * [Fix] Avoid curse of dynamic array referencing
+ * [Fix] Avoid reinitialising neural settings
+ * [Fix] Check remain before processing TXT records
+ * [Fix] Enable error multiplier on http errors
+ * [Fix] Finally rework parsing entities logic
+ * [Fix] Fix '==' parsing in the content type attributes parser
+ * [Fix] Fix IPv6 expansion for SPF macros
+ * [Fix] Fix Mozilla Message-ID detection
+ * [Fix] Fix an edge case in BITCOIN_ADDR rule
+ * [Fix] Fix brain-damaged behaviour when http request has a custom Host header
+ * [Fix] Fix check of limits in email address parsing
+ * [Fix] Fix copy&paste error and rework
+ * [Fix] Fix expressions logic for and/or and float values
+ * [Fix] Fix fuzzy retransmits
+ * [Fix] Fix http maps with no or invalid expires data
+ * [Fix] Fix last quote character parsing in the content-type state machine
+ * [Fix] Fix normalisation flags propagation
+ * [Fix] Fix overflow when appending many broken tags
+ * [Fix] Fix parsing of rfc2047 tokens with '?' inside
+ * [Fix] Fix phishing flag set
+ * [Fix] Fix rfc2047 embedded into rfc2231 pieces in special headers
+ * [Fix] Fix round-robin rotation
+ * [Fix] Fix searching for symbols
+ * [Fix] Fix storing of the regexps inside variant
+ * [Fix] Fix tokenization near exceptions
+ * [Fix] Fix visibility calculations
+ * [Fix] Html: Attach inline tags to the structure
+ * [Fix] Html: Do not treat empty tags as block tags
+ * [Fix] Ical: Do not extract urls from all flags using merely specific ones
+ * [Fix] Initialise symcache even if it cannot be loaded properly
+ * [Fix] Lua_fuzzy: Remove text parts check when checking image dimensions
+ * [Fix] Lua_maps: Fix adjustments for the map type in the complex map definitions
+ * [Fix] Lua_task: Fix deleted symbols in has_symbol/get_symbol
+ * [Fix] Move metric and symcache link from validation to the init stage
+ * [Fix] Oletools: Another try to fix table sorting
+ * [Fix] One more default behaviour fix
+ * [Fix] Phishing: Rework urls processing
+ * [Fix] RBL: was missing some config schema
+ * [Fix] Replies: Fix 'Reply-To' handling in task:get_reply_sender
+ * [Fix] Rework metrics handling
+ * [Fix] Save symcache on exit
+ * [Fix] Selectors: Filter nil elements in lists
+ * [Fix] Selectors: Properly fix implicit tostring for nils
+ * [Fix] Try to fix some broken code in DMARC reporting plugin
+ * [Fix] Urls: Fix processing of html urls when it comes to the flags
+ * [Fix] Use proper buffer length
+ * [Fix] Various visibility fixes
+ * [Fix]: ASN: dns cb func should also return in case of an error
+ * [Project] Add a simple css rule definition
+ * [Project] Add css style skeleton
+ * [Project] Add css syntax (adopted from ebnf)
+ * [Project] Add css_selectors
+ * [Project] Add doctest unit testing library
+ * [Project] Add expected library
+ * [Project] Add fmt library for simple string ops
+ * [Project] Add fu2 library to better functions abstractions
+ * [Project] Add hashing method
+ * [Project] Add parsers skeleton
+ * [Project] Add preliminary support of vcard parser
+ * [Project] Add process exceptions for invisible text
+ * [Project] Add some methods for css parser
+ * [Project] Allow static libstdc++
+ * [Project] Another whitespace hack
+ * [Project] CSS: Various fixes in the declarations and values parsing
+ * [Project] Cpp: Add robin-hood hash map library
+ * [Project] Css: Add AST debug
+ * [Project] Css: Add colors conversion functions
+ * [Project] Css: Add dimensions handling
+ * [Project] Css: Add display value support
+ * [Project] Css: Add frozen library from https://github.com/serge-sans-paille/frozen/
+ * [Project] Css: Add opacity support
+ * [Project] Css: Add parser helpers to simplify debugging
+ * [Project] Css: Add preliminary stylesheet support
+ * [Project] Css: Add rules processing functions and tests
+ * [Project] Css: Add simple selectors unit tests
+ * [Project] Css: Add some c++ unit tests
+ * [Project] Css: Add some debug methods
+ * [Project] Css: Add some debug statements for the css parser
+ * [Project] Css: Add some logical skeleton for declarations parser
+ * [Project] Css: Add url/function tokens
+ * [Project] Css: Allow at rules parsing
+ * [Project] Css: Declarations parsing logic skeleton
+ * [Project] Css: Enable conditional css parsing support from the HTML parser
+ * [Project] Css: Finish generic lexer cases
+ * [Project] Css: Fix HSL conversion
+ * [Project] Css: Fix minus parsing
+ * [Project] Css: Fix parser consumers nesting
+ * [Project] Css: Fix parsing of the qualified rules
+ * [Project] Css: Fix rules merging
+ * [Project] Css: Further fixes to lexer
+ * [Project] Css: Further steps to parse css colors + rework
+ * [Project] Css: Further work on parser's methods
+ * [Project] Css: Implement backlog of css tokens
+ * [Project] Css: Implement numbers and ident parsers
+ * [Project] Css: Implement simple css selectors lookup
+ * [Project] Css: Implement styles merging
+ * [Project] Css: Make debug strings json like to simplify tests
+ * [Project] Css: Minor adjustments
+ * [Project] Css: More meat to the lexer
+ * [Project] Css: Move some of the tests to the doctest
+ * [Project] Css: Projected a parser
+ * [Project] Css: Properties attachment logic
+ * [Project] Css: Remove ragel from build targets (maybe keep for reference)
+ * [Project] Css: Rework css block structure
+ * [Project] Css: Rework flags of css properties
+ * [Project] Css: Rework tokens structure
+ * [Project] Css: Several fixes + tests
+ * [Project] Css: Simplify checks
+ * [Project] Css: Simplify debug code
+ * [Project] Css: Start css selectors parsing logic
+ * [Project] Css: Start semantic parsing for rules
+ * [Project] Css: Start stylesheet implementation
+ * [Project] Css: Tidy up lambdas
+ * [Project] Css: rework tokeniser
+ * [Project] Dmarc: Add dmarc report tool (WIP)
+ * [Project] Dmarc: Add munging configuration
+ * [Project] Dmarc: Add preliminary munging logic
+ * [Project] Dmarc: Fix header removal
+ * [Project] Dmarc: Fix munging logic
+ * [Project] Dmarc: Use full recipient address instead of a domain map
+ * [Project] Dmarc: Use zlists for dmarc reports
+ * [Project] Dmarc_report: Add message generation logic
+ * [Project] Dmarc_report: Add preliminary sending support
+ * [Project] Fix lua bindings
+ * [Project] Fix xml/sgml tags processing
+ * [Project] Handle new modification
+ * [Project] Html/CSS: Add transform from a CSS rule to html block
+ * [Project] Html/CSS: Link html and css styles
+ * [Project] Html/CSS: Switch styles parsing to css parser
+ * [Project] Html/Css: Fix some issues found
+ * [Project] Html/Css: Implement visibility rules for a block
+ * [Project] Html: Add more tests cases and fix some more corner issues
+ * [Project] Html: Add rows display type support
+ * [Project] Html: Allow decode entities function to normalise spaces + unit tests
+ * [Project] Html: Another rework of the tags structure
+ * [Project] Html: Another try to fix unbalanced cases
+ * [Project] Html: Fix crossing spans
+ * [Project] Html: Fix parent propagation
+ * [Project] Html: Further rework of the html parsing stuff
+ * [Project] Html: Implement logic for tags pairing
+ * [Project] Html: Implement rawtext state machine
+ * [Project] Html: Insert closing tags as well :(
+ * [Project] Html: More fixes
+ * [Project] Html: More fixes
+ * [Project] Html: More spaces logic fixes
+ * [Project] Html: One more attempt to write text content
+ * [Project] Html: Replace \0 in html content
+ * [Project] Html: Rework img/a tags handling
+ * [Project] Html: Rework propagation method
+ * [Project] Html: Rework tags placement
+ * [Project] Html: Rework transparency logic
+ * [Project] Html: Support 'hidden' attribute
+ * [Project] Html: Try another approach to append tags content
+ * [Project] Html: Try to deal with bad unknown tags properly
+ * [Project] Lua_aws: Add canonicalisation utility
+ * [Project] Lua_aws: Add function to produce AWS Authorisation header
+ * [Project] Lua_aws: Implement request signing
+ * [Project] Lua_mime: Add lua_mime.modify_headers routine
+ * [Project] Lua_task: Add modify_header method
+ * [Project] Lua_task: Allow to extract modified headers
+ * [Project] Make unescape code public for unit testing
+ * [Project] More fixes for closed tags
+ * [Project] More fixes to calculations
+ * [Project] Rework API for the modified headers
+ * [Project] Rework html visibility rule
+ * [Project] Skeleton of the css library
+ * [Project] Start headers modification API structure
+ * [Project] Start working on AWS Lua API
+ * [Project] Use lua_mime to modify headers
+ * [Project] Use modified headers on dkim signing
+ * [Project] Use string_view to constexpr variant unpacking
+ * [Rework] Add composites manager concept
+ * [Rework] Add tags definitions
+ * [Rework] Allow C code to be compiled with C++ compiler
+ * [Rework] Clickhouse: Store url flags
+ * [Rework] Composites: Rewrite the composites logic
+ * [Rework] Composites: Start rework of the composites framework
+ * [Rework] Dmarc: Move check policy function to the common utils
+ * [Rework] Dmarc: Rework reports keys structure
+ * [Rework] Further work to make html content private
+ * [Rework] Html/CSS: Remove css C bindings as they are useless now
+ * [Rework] Html/CSS: Rework Lua bindings
+ * [Rework] Html/Css: Start rework of the html blocks
+ * [Rework] Html: Add images processing logic
+ * [Rework] Html: Add traverse function
+ * [Rework] Html: Another steps to get rid of gnode
+ * [Rework] Html: Convert to variant
+ * [Rework] Html: Deal with the utf_content part
+ * [Rework] Html: Final rework part for the html processing code
+ * [Rework] Html: Fix Lua bindings
+ * [Rework] Html: Forgot to add the internal include
+ * [Rework] Html: Further html urls rework
+ * [Rework] Html: Further rework of the tags content extraction
+ * [Rework] Html: Make parameters as a vector again
+ * [Rework] Html: Move blocks part
+ * [Rework] Html: Move images processing stuff
+ * [Rework] Html: Rework lua bindings
+ * [Rework] Html: Start html text extraction rework
+ * [Rework] Html: Start refactoring of the html tags handling
+ * [Rework] Html: Start removing of GNode stuff
+ * [Rework] Html: Start rework of the html content structure
+ * [Rework] Lua_magic: Try to detect text parts with 8bit characters for non-utf8 encodings
+ * [Rework] Move HTML url functions and rework them
+ * [Rework] Move and adopt entities handling logic
+ * [Rework] Move common and rarely used dmarc code to the library
+ * [Rework] Move compression routines outside of rspamd_util library
+ * [Rework] Move entities/tags handling
+ * [Rework] Phishing: Split from redirectors usage
+ * [Rework] Redesign html blocks propagation logic
+ * [Rework] Remove tag name string
+ * [Rework] Rename phished url to a linked url
+ * [Rework] Reorganize dmarc plugin and remove unsupported reporting code
+ * [Rework] Reputation: Use more flexible types in get/set functions
+ * [Rework] Require proper C++ environment for Rspamd build
+ * [Rework] Rework extended urls output
+ * [Rework] Rework tags parsing machine
+ * [Rework] Slightly improve old regexp API
+ * [Rework] Start conversion of the redis pool code to c++
+ * [Rework] Try to resolve failed upstreams more agressively
+ * [Rework] Use C++ utf8 library with unit tests to trim whitespaces
+ * [Rework] Use C++ version for unicode normalisation
+ * [Rework] Use C++ version of the lua threads pool
+ * [Rules] Add raw addresses to MULTIPLE_FROM options
+ * [Rules] Another fix to HTTP_TO_HTTPS rule
+ * [Rules] Do not trigger HTML_SHORT_LINK_IMG on external images
+ * [Rules] Extend FORGED_X_MAILER
+ * [Rules] Extend OLD_X_MAILER
+ * [Rules] Fix CTYPE_MIXED_BOGUS for text attachments
+ * [Rules] Fix FPs for CTYPE_MIXED_BOGUS
+ * [Rules] Fix HTTP_TO_HTTPS rule
+ * [Rules] Fix HTTP_TO_HTTPS rule
+ * [Rules] Fix zerofont rule (partially)
+ * [Rules] Micro-optimize X_PHP_EVAL
+ * [Rules] Reduce default weight for R_MISSING_CHARSET
+
+2.7: 08 Jan 2021
+ * [Conf] Add R_DKIM_PERMFAIL to the metric
+ * [CritFix] Dkim: Fix simple canonicalisation if multiple signatures are presented
+ * [CritFix] Fix controller paths normalisation
+ * [Feature] Add INVALID_DATE rule
+ * [Feature] Add controller endpoint for training neural
+ * [Feature] Add sanity checks for actions thresholds
+ * [Feature] Add support of '==' and '!=' in Rspamd expressions
+ * [Feature] Composites: Improve composite atoms parser
+ * [Feature] Docker: use Debian slim variant
+ * [Feature] Elastic: Add some missing fields
+ * [Feature] Extract text from img alt attributes
+ * [Feature] Improve charset detection logic
+ * [Feature] Lua_clickhouse: Add optional row callback for large selections
+ * [Feature] Lua_dns_resolver: Add idna_convert_utf8 method
+ * [Feature] Lua_mime: Add ability to do multipattern replacement
+ * [Feature] Lua_trie: Allow to report start of the match
+ * [Feature] Multimap: support adding map values as extra options
+ * [Feature] Neural: Move PCA learning to a subprocess
+ * [Feature] RBL: support matching content/image URLs only
+ * [Feature] RBL: support use of multiple selectors
+ * [Feature] Reputation: Allow to specify ip masks
+ * [Feature] Support SMIME signed messages container
+ * [Feature] Support multiple conditions for symbols
+ * [Feature] Support ping in milter mode
+ * [Feature] Support rspamd_text in selector regexps
+ * [Feature] Use own daemonization routine
+ * [Feature] Vadesecure: Implement settings_outbound feature as recommended by Vade
+ * [Feature] `rspamadm clickhouse` command
+ * [Feature] allow hyperscan for aarch64
+ * [Fix] Allow to set priorities between post init scripts
+ * [Fix] Allow to use maps for strings that are not zero terminated
+ * [Fix] Apply max_lua_urls limit for emails as well
+ * [Fix] Arc: Fix CV check on signing
+ * [Fix] Arc: Fix signing of the broken ARC chains
+ * [Fix] Clickhouse: escape carriage return
+ * [Fix] Composites: Allow partial match
+ * [Fix] Deduct type of a table methods
+ * [Fix] Do not load errored hyperscan database
+ * [Fix] Do not process links in ignored html tags
+ * [Fix] Fix ClamAV result for cached encrypted file (#3395)
+ * [Fix] Fix canonicalisation when l= tag is presented
+ * [Fix] Fix flag shift
+ * [Fix] Fix handling of skip/skip_process http flags
+ * [Fix] Fix html attachments checks
+ * [Fix] Fix issue with pushing binary formats to Lua strings
+ * [Fix] Fix logging for rspamadm
+ * [Fix] Fix off-by-one with init check
+ * [Fix] Fix parsing of escape characters in quoted pairs
+ * [Fix] Fix pushing ucl strings with \0 inside
+ * [Fix] Fix quoted-printable soft newlines bugged case
+ * [Fix] Fix settings in case actions are set to null (#3415)
+ * [Fix] Fix several issues with auth results producing
+ * [Fix] Fix smtp comments exclusion
+ * [Fix] Fix smtp date syntax definition
+ * [Fix] Fix substring search in case if srchlen == inlen
+ * [Fix] Fix text selectors
+ * [Fix] Honour `systemd` setting when logging to console (#3514)
+ * [Fix] Html: Add entities collisions prevention logic (e.g. for mathml entities)
+ * [Fix] Lua_auth_results: Quote potentially bad values in AR header
+ * [Fix] Multimap: Fix flags usage
+ * [Fix] Multimap: Fix scoring for combined maps
+ * [Fix] Plug GList * leak in redis pool
+ * [Fix] RBL: allow for multiple matches of the same label if types are different
+ * [Fix] Rely on libev checks for file maps
+ * [Fix] Restore simple dkim canonicalisation mode
+ * [Fix] Return MimeCharset as we work with emails...
+ * [Fix] Spamassassin: Fix pcre_only flags
+ * [Fix] Spamassassin: Preserve 'pcre_only' flag when dealing with regexp replacements
+ * [Fix] Try to fix GError leak
+ * [Fix] Try to fix a mess with settings loading by adding priorities
+ * [Fix] Try to move setings initialisation to a later stage
+ * [Fix] Use dup fd in milter handler to avoid races with the proxy
+ * [Fix] Use message pointer to avoid obsolete data to be cached
+ * [Project] Rbl: Migrate to `checks`
+ * [Project] Rbl: Move config code outside of the plugin
+ * [Project] Ressurect empty prefilters as connection filters
+ * [Project] Support connection filters registration from Lua
+ * [Rework] Add final cleanup logic
+ * [Rework] Add preliminary support of hyperscan caching for re maps
+ * [Rework] Add stale cache removal
+ * [Rework] Clickhouse: Improve performance
+ * [Rework] Distinguish between strict config test mode
+ * [Rework] Furhter logging improvements
+ * [Rework] Milter_headers: improve extended_headers_rcpt support
+ * [Rework] Move parsers to a separate lua library
+ * [Rework] Neural: Skip composite symbols
+ * [Rework] Rbl: Rework defaults logic
+ * [Rework] Some tunes to cache saving
+ * [Rework] Track maps origins
+ * [Rework] Use full crypto hash for regexp maps
+ * [Rules] Remove broken rule
+
+2.6: 30 Sep 2020
+ * [Conf] Add missing symbols
+ * [Conf] Add missing symbols
+ * [Conf] Fix fat-fingers typo
+ * [Conf] Fix wrong comment in options.inc
+ * [Conf] Neural: Fix the default name for max_trains
+ * [Conf] Register a known symbol
+ * [Conf] Spf: Add R_SPF_PERMFAIL symbol
+ * [CritFix] Arc: Fix ARC validation for chains of signatures
+ * [CritFix] Distinguish socketpairs between different fuzzy workers
+ * [CritFix] Fix IDNA dots parsing
+ * [CritFix] Fix test assertion method
+ * [CritFix] Fix usage of crypto_sign it should be crypto_sign_detached!
+ * [Feature] Add BOUNCE rule
+ * [Feature] Add controller plugins support and selectors plugin
+ * [Feature] Add maps query method
+ * [Feature] Add minimal delay to fuzzy storage
+ * [Feature] Add multiple base32 alphabets for decoding
+ * [Feature] Add preliminary support of BCH addresses
+ * [Feature] Add query_specific endpoint
+ * [Feature] Allow multiple base32 encodings in Lua API
+ * [Feature] Allow to specify nonces manually
+ * [Feature] Controller: Allow to pass query arguments to the lua webui plugins
+ * [Feature] Fuzzy_check: Add gen_hashes command
+ * [Feature] Fuzzy_check: Add weight_threshold option for fuzzy rules
+ * [Feature] Implement address retry on connection failure
+ * [Feature] Improve limits in pdf scanning
+ * [Feature] Initial support of subscribe command in lua_redis
+ * [Feature] Lua_cryptobox: Add secretbox API
+ * [Feature] Lua_text: Add encoding methods
+ * [Feature] Milter_headers: Allow to activate routines via users settings
+ * [Feature] PDF: Add timeouts for expensive operations
+ * [Feature] Preliminary maps addon for controller
+ * [Feature] Split pdf processing object and output object to allow GC
+ * [Feature] Support BLIS blas library
+ * [Feature] Support input vectorisation by recvmmsg call
+ * [Feature] Support multiple base32 alphabets
+ * [Feature] add queueid, uid, messageid and specific symbols to selectors [Minor] use only selectors to fill vars in force_actions message
+ * [Feature] allow variables in force_actions messages
+ * [Feature] extend lua api
+ * [Fix] #3249
+ * [Fix] Allow to adjust neurons in the hidden layer
+ * [Fix] Another try to fix email names parsing
+ * [Fix] Arc: Allow to reuse authentication results when doing multi-stage signing
+ * [Fix] Arc: Fix bug with arc chains verification where i>1
+ * [Fix] Arc: Sort headers by their i= value
+ * [Fix] Change neural plugin's loss function
+ * [Fix] Deal with double eqsigns when decoding headers
+ * [Fix] Default ANN names in clickhouse
+ * [Fix] Disable reuseport for TCP sockets as it causes too many troubles
+ * [Fix] Disable text detection heuristics for encrypted parts
+ * [Fix] Distinguish DKIM keys by md5
+ * [Fix] Distinguish type from flags in register_symbol
+ * [Fix] Dmarc: Unbreak reporting after cf2ae3292ac93da8b6e0624b48a62828a51803c9
+ * [Fix] Do not flag pre-result of virus scanners as least if action is reject
+ * [Fix] Do not use GC64 workaround on 32bit platforms, omg
+ * [Fix] Exclude damaged urls from html parser
+ * [Fix] Fix FREEMAIL_REPLYTO_NEQ_FROM_DOM
+ * [Fix] Fix FROM_NEQ_ENVFROM
+ * [Fix] Fix FWD_GOOGLE rule (#1815)
+ * [Fix] Fix adding of the empty archive file for gzip
+ * [Fix] Fix aliases in forged recipients and limit number of iterations
+ * [Fix] Fix authentication results insertion
+ * [Fix] Fix calling of methods in selectors
+ * [Fix] Fix clen length for hiredis...
+ * [Fix] Fix endless loop if broken arc chain has been found
+ * [Fix] Fix false - operation
+ * [Fix] Fix get_urls table invocation
+ * [Fix] Fix group based composites
+ * [Fix] Fix headers passing in rspamd_proxy
+ * [Fix] Fix incomplete utf8 sequences handling
+ * [Fix] Fix lua_next invocation
+ * [Fix] Fix lua_parse_symbol_type function logic
+ * [Fix] Fix multiple listen configuration
+ * [Fix] Fix occasional encryption of the cached data
+ * [Fix] Fix parsing boundaries with spaces
+ * [Fix] Fix passing of methods arguments
+ * [Fix] Fix poor man allocator algorithm
+ * [Fix] Fix regexp selector and add flattening
+ * [Fix] Fix rfc base32 encode ordering (skip inverse bits)
+ * [Fix] Fix rfc based base32 decoding
+ * [Fix] Fix sockets leak in the client
+ * [Fix] Fix storing of the original smtp from
+ * [Fix] Fix types check and types usage in lua_cryptobox
+ * [Fix] Fix unused results
+ * [Fix] Fuzzy_check: Disable shingles for short texts (really)
+ * [Fix] Ical: Fix identation grammar
+ * [Fix] Improve part:is_attachment logic
+ * [Fix] Mmap return value must be checked versus MAP_FAILED
+ * [Fix] One more fix to skip images that are not urls
+ * [Fix] Pdf: Support some weird objects with no newline before endobj
+ * [Fix] Rbl: Fix ignore_defaults in conjunction with ignore_whitelists
+ * [Fix] Restore support for `for` and `id` parts in received headers
+ * [Fix] Segmentation fault in contrib/lua-lpeg/lpvm.c on ppc64el
+ * [Fix] Skip spaces at the boundary end
+ * [Fix] Slashing fix: fix captures matching API
+ * [Fix] Spamassassin: Rework metas processing
+ * [Fix] Store reference of upstream list in upstreams objects
+ * [Fix] Understand utf8 in content-disposition parser
+ * [Fix] Unify selectors digest functions
+ * [Fix] Use `abs` value when checking composites
+ * [Fix] Use strict IDNA for utf8 DNS names + add sanity checks for DNS names
+ * [Fix] Use unsigned char and better support of utf8 in ragel parser
+ * [Fix] add missing selector_cache declaration
+ * [Project] Add `L` flag for regexps to save start of the match in Hyperscan
+ * [Project] Add `lower` method to lua_text
+ * [Project] Add a simple matrix Lua library
+ * [Project] Add implicit bitcoincash prefix
+ * [Project] Add linalg ffi library for prototyping
+ * [Project] Add methods to append data to fuzzy requests
+ * [Project] Add routine to call a generic lua function
+ * [Project] Add ssyev method interface
+ * [Project] Add tensors index method
+ * [Project] Add text:sub method
+ * [Project] Allow rspamd_text based selectors
+ * [Project] Allow to specify re_conditions for regular expressions
+ * [Project] Attach extensions to the binary fuzzy commands
+ * [Project] Bitcoin: BTC cash addresses needs some checksum validation
+ * [Project] Cleanup the redis script
+ * [Project] Convert bitcoin rules to the new regexp conditions feature
+ * [Project] Detect memrchr in systems that supports it
+ * [Project] Do not listen sockets in the main process
+ * [Project] Implement 'probabilistic' learn mode for ANN
+ * [Project] Implement BTC polymod in C as it requires 64 bit ops
+ * [Project] Implement bitcoin cash validation in a proper way
+ * [Project] Implement extensions logic for fuzzy storage
+ * [Project] Implement symbols insertion in multiple results mode
+ * [Project] Lua_text: Add method memchr
+ * [Project] Neural: Add PCA loading logic
+ * [Project] Neural: Fix PCA based learning
+ * [Project] Neural: Fix matrix gemm
+ * [Project] Neural: Further PCA fixes
+ * [Project] Neural: Implement PCA in learning
+ * [Project] Neural: Implement PCA learning
+ * [Project] Neural: Implement PCA on ANN forward
+ * [Project] Neural: Implement PCA serialisation
+ * [Project] Neural: Start PCA implementation
+ * [Project] Neural: Use C version of scatter matrix producing
+ * [Project] Preliminary support of lua conditions for regexps
+ * [Project] Preliminary usage of the reuseport
+ * [Project] Process composites separately for each shadow result
+ * [Project] Remove old code
+ * [Project] Rework scan result functions to support shadow results
+ * [Project] Rework some more functions to work with shadow results
+ * [Project] Some more fixes
+ * [Project] Start results chain implementation
+ * [Project] Support fun iterators on rspamd_text objects
+ * [Project] Support multiply, minus and divide operators in expressions
+ * [Project] Tensor: Move scatter matrix calculation to C
+ * [Rework] Allow to specify exat metric result when adding a symbol
+ * [Rework] Change and improve openblas detection and usage
+ * [Rework] Close listen sockets in main after fork
+ * [Rework] Further rework of lua urls extraction API
+ * [Rework] Lua_cryptobox: Allow to store output of the hash function
+ * [Rework] Lua_task: Add more methods to deal with shadow results
+ * [Rework] Modernize logging for expressions
+ * [Rework] Remove empty prefilters feature - we are not prepared...
+ * [Rework] Remove old FindLua module, disable lua fallback when LuaJIT is enabled
+ * [Rework] Rework and refactor forged recipients plugin
+ * [Rework] Rework expressions processing
+ * [Rework] Rework fuzzy commands processing
+ * [Rework] Rework url flags handling API
+ * [Rework] Rework urls extraction
+ * [Rework] Split operations processing and add more debug logs
+ * [Rework] Update zstd to 1.4.5
+ * [Rework] Use google-ced instead of libicu chardet as the former sucks
+ * [Rework] add alias util:parse_addr for util:parse_mail_address
+ * [Rework] get rid of util:parse_addr duplicating the util:parse_mail_address, replace where used
+ * [Rules] Allow prefix for bitcoin cash addresses
+ * [Rules] More fixes for bitcoin cash addresses decoding
+ * [Rules] Refactor bleach32 addresses handling
+
+2.5: 01 Apr 2020
+ * [Conf] Mark Rspamd emailbl as ignore whitelist
+ * [Conf] RBL: Add missing emails = true option
+ * [Feature] Add support for scripts in fuzzy storage
+ * [Feature] Arc: Add whitelisted_signers_map option
+ * [Feature] Implement hosts file processing
+ * [Feature] Neural: Introduce classes bias that allows non-equal classes learning
+ * [Feature] Update libev to 4.33
+ * [Fix] Another brain damage html standard adoptions
+ * [Fix] Another fix for brain damaged obs-fws state
+ * [Fix] Fix flags that caused force_actions failure
+ * [Fix] Fix logging issue
+ * [Fix] Fix lua symbols scores registration when config does not define scores
+ * [Fix] Fix opaque maps logic
+ * [Fix] Fix parsing of the html tags with no spaces after attributes
+ * [Fix] Fix some corner cases in urls parsing, add limits
+ * [Fix] Fix tlds extraction if custom composition rules are used
+ * [Fix] Fix variables replacement in mempool
+ * [Fix] Improve base64 detection
+ * [Fix] Normalize dynamic scores in ANN correctly
+ * [Fix] Plug memory leak introduced by #3153
+ * [Fix] Stat_redis_backend: Fix memory leak and simplify learn path
+ * [Fix] Try hard to deal with ghost workers
+ * [Fix] metadata_exporter default formatter
+ * [Rework] Change the way to extract URLs when dealing with alternative parts
+ * [Rework] Fix various url extraction issues
+ * [Rework] Re cache: Load compiled hyperscan in the main process as well
+ * [Rework] Re cache: Load hyperscan early
+ * [Rework] Rework URL structure: adjust tld part
+ * [Rework] Rework URL structure: host field
+ * [Rework] Rework URL structure: more structure optimisations
+ * [Rework] Rework URL structure: user field
+ * [Rework] URL: Another update for urls extraction logic
+ * [Rework] Urls: Improve query urls handling
+ * [Rework] Urls: adopt html related stuff
+ * [Rework] Urls: more rework of the urls sets
+ * [Rework] Urls: process query urls in HTML urls correctly
+ * [Rework] Urls: rework urls hash structure
+ * [Rework] Urls: update lua libraries
+ * [Rework] Use multiple search tries for different url extraction types
+
+2.4: 26 Feb 2020
+ * [CritFix] Fix parsing of the content type attributes
+ * [Feature] Clickhouse: Add extra columns support
+ * [Feature] Rbl: Add url_compose_map option for RBL rules
+ * [Fix] 'R' flag is for all headers regexp
+ * [Fix] Allow to reset settings id from Lua (e.g. because of the priority)
+ * [Fix] Avoid collisions in mempool variables by changing fuzzy caching logic
+ * [Fix] Avoid strdup usage for symbols options
+ * [Fix] Do not trust stat(2) it lies
+ * [Fix] Filter all options for symbols to have sane characters
+ * [Fix] Fix all headers iteration
+ * [Fix] Fix allowed_settings for neural
+ * [Fix] Fix listen socket parsing
+ * [Fix] Fix maps expressions evaluation
+ * [Fix] Fix sentinel connections leak by using async connections
+ * [Fix] Fix smtp message on passthrough result
+ * [Fix] Fix tld compositon rules
+ * [Fix] Fuzzy_storage: Do not check for shingles if a direct hash has been found
+ * [Fix] Lua_mime: Do not perform QP encoding for 7bit parts
+ * [Fix] Neural: Distinguish missing symbols from symbols with low scores
+ * [Fix] Support listening on systemd sockets by name
+ * [Project] Add lua_urls_compose library
+ * [Project] Allow to set a custom log function to the logger
+ * [Project] CDB maps: Start making cdb a first class citizen
+ * [Project] Clickhouse: Add extra columns concept
+ * [Project] Fix urls composition rules, add unit tests
+ * [Project] Unify cdb maps
+ * [Rework] Logger infrastructure rework
+ * [Rework] Refactor libraries structure
+ * [Rework] Rework SSL caching
+ * [Rework] Update snowball stemmer to 2.0 and remove all crap aside of UTF8
+
+2.3: 04 Feb 2020
+ * [Conf] SPF is no longer a C module
+ * [Conf] Update spamtrap map path example
+ * [CritFix] Fix html entities decoding
+ * [CritFix] Fix re cache when mix of pcre and hyperscan is used
+ * [Feature] Allow milter code to deal with multiple headers
+ * [Feature] Antivirus: Add avast support
+ * [Feature] Dkim_signing: Allow to sign via milter_headers
+ * [Feature] Implement content hashes
+ * [Feature] Lua_text: Add regexp split iterator method
+ * [Feature] Lua_text: Implement flattening of the input tables
+ * [Feature] Send quit command to Redis
+ * [Feature] Speed up is_ascii function
+ * [Feature] Spf: Add external_relay option
+ * [Fix] Avoid double escaping
+ * [Fix] Fix O(N^2) algorithm
+ * [Fix] Fix arc seal validation
+ * [Fix] Fix base tag processing according to stupid HTML renderer behaviour
+ * [Fix] Fix dealing with `\0` in ucl strings and JSON
+ * [Fix] Fix gpg parts misdetection
+ * [Fix] Fix ignored symbols exporting
+ * [Fix] Fix processing of numeric url's
+ * [Fix] Fix processing of the closed tcp connections
+ * [Fix] Fix regexp type check for pcre2
+ * [Fix] Fix urls encode function
+ * [Fix] Fix urls shifting when doing decode to include separators
+ * [Fix] Fix white on white rule and add is_leaf flag
+ * [Fix] Further fixes in charset detection
+ * [Fix] Ignore diacritics in chartable module for specific languages
+ * [Fix] Limit size of symbols options by max_opts_len option
+ * [Fix] More fixes in html tag content calculations
+ * [Fix] Plug memory leak in fuzzy storage
+ * [Fix] Process high priority settings even if settings/id has been specified
+ * [Fix] Select a different upstream on last retransmit
+ * [Fix] Treat soft hyphen as zero width space
+ * [Fix] Try harder to watch the lifetime of the key_stat
+ * [Fix] Use ipv6-mapped-ipv4 addresses in radix trie
+ * [Project] Add logic to break execution when processing symbols*
+ * [Project] Add methods to set specific content for mime parts from Lua
+ * [Project] Lua_content: support PDF files
+ * [Project] Move dns_tool to using of the rspamd_spf from FFI module
+ * [Project] Preliminary SPF plugin in Lua
+ * [Project] Show debug stat for memory pool
+ * [Project] Some rework about specific data that is now tagged
+ * [Project] Start reworking of the mempool structure
+ * [Rework] Allow to add userdata as symbols options
+ * [Rework] Change mime part specifics handling
+ * [Rework] Move LRU SPF cache from spf plugin
+ * [Rework] Rework HTML tags content attachment
+ * [Rework] Rework options hash structure
+ * [Rework] Start lua_content library
+ * [Rework] Stop using of uthash for http headers
+ * [Rework] Use faster hashing approach for memory pools variables
+ * [Rules] Add PDF related rules
+
+2.2: 19 Nov 2019
+ * [Conf] Antivirus: Fix the default config
+ * [Feature] Add verdict library in lua
+ * [Feature] Allow exception when choosing upstream
+ * [Feature] Allow to disable symbols from the metric config
+ * [Feature] Allow to limit maps per specific worker
+ * [Feature] Always validate Rspamd protocol output
+ * [Feature] Antivirus: Add preliminary virustotal support
+ * [Feature] Clickhouse: Rework Clickhouse collection logic
+ * [Feature] Improve base64 usage
+ * [Feature] Shutdown timeout is now associated with task timeout
+ * [Fix] #3129 Multiple classifiers on redis working incorrectly
+ * [Fix] Allow real upstreams configuration
+ * [Fix] Another try to fix slow callbacks and timers
+ * [Fix] Check results of write message as SSL can bork them
+ * [Fix] Clickhouse: Avoid potential races in collection
+ * [Fix] Clickhouse: Fix periodic script
+ * [Fix] Fail DNS upstream on each retransmit attempt
+ * [Fix] Fix consistent hashing when upstreams are marked inactive
+ * [Fix] Fix issues found
+ * [Fix] Fix off-by-one in retries for the proxy
+ * [Fix] Fix termination
+ * [Fix] Fix upstreams exclusion logic
+ * [Fix] Fix utf8 validation for symbols options and empty strings
+ * [Fix] Oops, fix maps reload
+ * [Fix] Rbl: Allow utf8 lookups for IDN domains
+ * [Fix] Sigh, another try to fix brain-damaged openssl
+ * [Project] Add fast utf8 validation library
+ * [Project] Use own utf8 validation instead of glib
+ * [Rework] Another phase of finish actions rework
+ * [Rework] Further cmake system rework
+ * [Rework] Further isolation of the controller's functions
+ * [Rework] Make cmake structure more modular
+ * [Rework] Move cmake modules to a dedicated path
+ * [Rework] Replace controller functions by any scanner worker if needed
+ * [Rework] Rework final scripts logic
+ * [Rework] Rewrite rspamd_str_make_utf_valid function
+
+2.1: 28 Oct 2019
+ * [Conf] Update neural.conf
+ * [CritFix] Fix dkim verification for multiple headers listed
+ * [Feature] Add support of uudecode
+ * [Feature] Allow to explicitly set events backend
+ * [Feature] Implement configurable limits for SPF lookups
+ * [Feature] Lua_scanners: Use lua magic for inclusion/exclusion logic
+ * [Feature] Multimap: Do not check files in office archives
+ * [Feature] Neural: Add sampling when storing training vectors
+ * [Feature] SPF: Allow to disable AAAA checks in configuration
+ * [Feature] Spf: Add limits configuration support
+ * [Feature] Store etag in cached HTTP maps + better logging
+ * [Feature] Support segwit BTC addresses, fix LTC verification
+ * [Feature] Support uuencoding
+ * [Fix] Add configurable number of threads for OpenBLAS
+ * [Fix] Add workaround for ragel 7 in hyperscan related maps code
+ * [Fix] Another fix for numeric urls parsing
+ * [Fix] Correct EMA time calculations
+ * [Fix] Do not treat archives as text
+ * [Fix] Do not use strdup on data extracted from lua
+ * [Fix] Fix a failure calcuating URL reputation.
+ * [Fix] Fix crash due to constructors init order
+ * [Fix] Fix crash on parts with no cd
+ * [Fix] Fix empty prefilters that require mime structures
+ * [Fix] Fix event loop creation
+ * [Fix] Fix issues sending DMARC reports.
+ * [Fix] Fix misprint
+ * [Fix] Fix saving of the file maps
+ * [Fix] Fix size calculations when converting from utf16
+ * [Fix] Fix support of disable_monitoring in rbl
+ * [Fix] Fix use-after-free
+ * [Fix] Fix zip files check to relax requirements
+ * [Fix] Important hiredis fixes
+ * [Fix] Lot's of fixes in maps check logic
+ * [Fix] Lua_tcp: Deal with temporary fails on write
+ * [Fix] Lua_tcp: Make write errors fatal and rework error handlers
+ * [Fix] Meta: Filter some more values
+ * [Fix] Neural: Add protection agains infinities
+ * [Fix] Oops, fix math.huge invocation
+ * [Fix] Plug memory leak
+ * [Fix] Sigh, another email to string fix
+ * [Fix] Try to fix another ownership race in ssl connection
+ * [Fix] Uuencode: Fix parsing of corrupted uuencode
+ * [Fix] lua_scanners - razor rename need_check function
+ * [Rework] Require CMake 3.9 to work, remove manual lto crap
+
+2.0: 11 Oct 2019
+ * [Conf] Add BROKEN_HEADERS_MAILLIST composite
+ * [Conf] Add path to greylist-whitelist-domains.inc
+ * [Conf] Clarify documentation in the config files
+ * [Conf] Introduce maps.d directories
+ * [Conf] Log settings id by default
+ * [Conf] Make LEAKED_PASSWORD_SCAM a composite rule again
+ * [Conf] Move all surbl/emails rules to rbl
+ * [Conf] Register new Spamhaus codes
+ * [Conf] Remove configs for deleted modules
+ * [Conf] Remove surbl parts, fix hash_format attribute
+ * [Conf] Show autolearn sample
+ * [Conf] Slashing: Change default stats backend to Redis
+ * [Conf] Surbl: Utilise new `check_emails` option
+ * [Conf] Update header
+ * [Conf] Use multi-prefixes RBLs in the default config
+ * [CritFix] Deal with case-sensivity in Content-Disposition parser
+ * [CritFix] Eliminate old endpoint
+ * [CritFix] Fix case sensivity when parsing Content-Type
+ * [CritFix] Fix loading of DKIM public keys
+ * [CritFix] Fix procesing of urls
+ * [CritFix] Fix whitelisting when both spf and dkim are required to be valid
+ * [CritFix] Langdet: Fix language detection where no stop words found
+ * [Feature] Add description to the groups
+ * [Feature] Add limit for number of URLs in Lua
+ * [Feature] Add logging of groups to the log_format
+ * [Feature] Add lua_smtp library
+ * [Feature] Add maps cache and type refinement
+ * [Feature] Add p0f scanner
+ * [Feature] Adopt emails module to use lua_maps
+ * [Feature] Allow options matching in composites
+ * [Feature] Allow selectors in rbl module
+ * [Feature] Allow to output group results
+ * [Feature] Asn: Allow to use bgpdump when NET::MRT is broken
+ * [Feature] Calculate tokens occurrences distribution
+ * [Feature] Clickhouse: Add authenticated user and settings id columns
+ * [Feature] Clickhouse: Store groups data
+ * [Feature] Clickhouse: Utilise LowCardinality feature
+ * [Feature] Implement Redis prefixes registration logic
+ * [Feature] Implement settings id propagation between deps
+ * [Feature] Improve AV results caching
+ * [Feature] Improve autolearning
+ * [Feature] Improve logging locking logic (remove it actually)
+ * [Feature] Improve settings processing
+ * [Feature] Langdet: Limit number of stop words to be checked
+ * [Feature] Libucl: Allow to sort keys in ucl objects
+ * [Feature] Lua_config: Extend get symbols method
+ * [Feature] Lua_maps: Allow static maps for key-value pairs
+ * [Feature] Lua_mimepart: Add function filter_words
+ * [Feature] Lua_selectors: Add `words` selector
+ * [Feature] Lua_selectors: Add sort and uniq transform functions
+ * [Feature] Lua_selectors: Allow table arguments for selectors
+ * [Feature] Lua_tcp: Add preliminary support of SSL connections
+ * [Feature] Lua_trie: More flexible API
+ * [Feature] Lua_util: Add filter_specific_url function
+ * [Feature] Lua_util: table_digest can now recursively traverse tables
+ * [Feature] Maillist: Improve detection
+ * [Feature] Maps: Allow caching for complex maps
+ * [Feature] Monitored: Support random lookups
+ * [Feature] Multimap: Add combined maps prototype
+ * [Feature] Multimap: Add dependend maps via redis keys selectors
+ * [Feature] Multimap: Allow multiple email addresses matches
+ * [Feature] Multimap: Also check detected charset when do filename checks
+ * [Feature] Output number of messages processed to proctitle
+ * [Feature] Perform clean SSL shutdown
+ * [Feature] Performance: Do not use base64 SIMD version for bad inputs
+ * [Feature] RBL: Support bit results in replies
+ * [Feature] RBL: Support type specific prefixes
+ * [Feature] Ratelimit: Consider number of SMTP recipients
+ * [Feature] Rbl: Add ability to check urls
+ * [Feature] Rbl: Add resolve_ip based RBLs
+ * [Feature] Rbl: Make config checks much more strict
+ * [Feature] Rbl: Support per-rule whitelists
+ * [Feature] Rbl: Support process script
+ * [Feature] Rbl: Support replyto addresses
+ * [Feature] SURBL: Allow to check email domains
+ * [Feature] Selectors: Add `list` generator
+ * [Feature] Selectors: Add `specific_urls` extractor
+ * [Feature] Selectors: Add flatten function
+ * [Feature] Selectors: Support filter_map and apply_map functions
+ * [Feature] Store Clickhouse data outside of lua alloc
+ * [Feature] Support caching for encrypted files and macros
+ * [Feature] Support images when extracting urls
+ * [Feature] Support more hyperscan flags
+ * [Feature] Support protocol flags
+ * [Feature] URL: Apply stringprep to hostnames to filter garbage
+ * [Feature] Upstreams: Add lazy resolving logic to all upstreams
+ * [Feature] Upstreams: Set noresolve flag on numeric upstreams
+ * [Feature] Use `scores` in apply section
+ * [Feature] Use maps logic from lua_maps for multimap
+ * [Feature] Use random monitored in rbl module
+ * [Feature] lua_scanners - add Razor support
+ * [Fix] Add another safe-guard in urls processing
+ * [Fix] Add debug to ssl, fixed write hangs
+ * [Fix] Add missing groups to C callback symbols
+ * [Fix] Add more checks for ghosts symbols
+ * [Fix] Allow to enable or add new actions via settings
+ * [Fix] Allow to set 0 size for spf/dkim caches
+ * [Fix] Another bunch of fixes towards protocol mess
+ * [Fix] Another fix to deal with bad URLs
+ * [Fix] Arc: Another bunch of fixes for arc signing
+ * [Fix] Arc: More arc signing fixes
+ * [Fix] Avoid another overflow in fpconv
+ * [Fix] Clickhouse: Fix quoting
+ * [Fix] Clickhouse: Fix retention query quoting
+ * [Fix] Distinguish empty and non-empty prefilters
+ * [Fix] Distinguish remote and local addrs parsing
+ * [Fix] Do not assert if length of sig is bad, just fail verification
+ * [Fix] Do not assert if we have broken mime boundary in the headers
+ * [Fix] Do not call implicit strlen to avoid issues
+ * [Fix] Do not count images urls when checking url regexps for compatibility
+ * [Fix] Do not output rbl suffix in symbol option
+ * [Fix] Do not use config pool to avoid issues with double reload
+ * [Fix] Do not use ephemeral string
+ * [Fix] Do not use lightuserdata for traceback
+ * [Fix] Do not use priority in metric registration
+ * [Fix] Emails: Check email sanity before testing on BL
+ * [Fix] Emails: Fix misprint in key name
+ * [Fix] Escape utf in regexp to dodge ragel/hyperscan issue
+ * [Fix] Extend task_timeout to postfilters stage
+ * [Fix] Fix ARC signing after fixing another bug in it...
+ * [Fix] Fix AV scan logic
+ * [Fix] Fix DMARC_NA behaviour in case of no valid policies
+ * [Fix] Fix LRU hash iteration logic
+ * [Fix] Fix alignment mess
+ * [Fix] Fix configuring symbols without scores
+ * [Fix] Fix disabling of the actions
+ * [Fix] Fix dkim signing exceptions
+ * [Fix] Fix embedded images linking logic
+ * [Fix] Fix events leak
+ * [Fix] Fix eviction corner case
+ * [Fix] Fix fuzzy image score calculation #2962
+ * [Fix] Fix hang in fuzzy_learn when explicit rotation is set
+ * [Fix] Fix headers propagation logic
+ * [Fix] Fix hearbeats restart issue
+ * [Fix] Fix history reset
+ * [Fix] Fix log parameter
+ * [Fix] Fix lua_ip_equal logic
+ * [Fix] Fix more issues with nested messages + tests
+ * [Fix] Fix normalization of non-alphabet based languages
+ * [Fix] Fix offsets when parsing message/rfc822 in multipart
+ * [Fix] Fix options in rbl symbols
+ * [Fix] Fix out of bound access in lua logger
+ * [Fix] Fix out-of-bound read in qp decode
+ * [Fix] Fix parent CTE propagation
+ * [Fix] Fix parsing of the received headers with empty part
+ * [Fix] Fix pending checks for events
+ * [Fix] Fix printing of NULL pointer with fixed length
+ * [Fix] Fix race condition in watcher handler
+ * [Fix] Fix read-after-end in quoted printable decoding
+ * [Fix] Fix redis sentinel support
+ * [Fix] Fix registry leak in case of DNS errors
+ * [Fix] Fix reload logic
+ * [Fix] Fix sending of large entries via HTTPS
+ * [Fix] Fix settings reload
+ * [Fix] Fix some more corner cases for fpconv
+ * [Fix] Fix trie code when there are regexps and Hyperscan is absent
+ * [Fix] Further fixes to printing of the FP numbers
+ * [Fix] Fuzzy_check: Fix timeouts
+ * [Fix] Grrr, fix empty ip case
+ * [Fix] Html: Fix processing of fjlig entity
+ * [Fix] Lang_det: Try better to distinguish Chinese and Japanese
+ * [Fix] Lua_mime: Fix reversed extensions map
+ * [Fix] Lua_task: Fix message-less API
+ * [Fix] Lua_tcp: Report connection failures
+ * [Fix] Lua_tcp: Various fixes and debugging improvements
+ * [Fix] Metadata_exporter: This plugin is idempotent not a postfilter
+ * [Fix] More fixes to extract_specific_urls
+ * [Fix] More stages fixes
+ * [Fix] Neural: Another bunch of fixes
+ * [Fix] Neural: use version in ANN key profile
+ * [Fix] Postpone lua state destruction to allow lua dtors to be used
+ * [Fix] Prefer surbl/emails rule on rbl to preserve compatibility
+ * [Fix] RBL: Fix behaviour of emails_domainonly
+ * [Fix] Ratelimit: Fix dynamic score
+ * [Fix] Rbl: Fix emailbl functions
+ * [Fix] Really fix hyperscan workaround
+ * [Fix] Set sanity limits for pcre2
+ * [Fix] Settings: Fix settings check flags
+ * [Fix] Sort keys when getting data from Lua when filling rules
+ * [Fix] Statistics: Do not query Redis tokens when there are no learns
+ * [Fix] Stop IO event on write finished in http connection
+ * [Fix] Use heuristically detected text parts data
+ * [Fix] Various fixes to QP encoding algorithm
+ * [Fix] Various fixes to SSL state machine handler
+ * [Fix] Various fixes to asn module
+ * [Fix] Workaround for empty charset in rfc2231 encoding
+ * [Project] Switch from torch to KANN
+ * [Project] Add heartbeat events
+ * [Project] Add preliminary support of the Kaspersky Scan Engine
+ * [Project] Add preliminary version of maps expressions
+ * [Project] Add preprocessed settings to the config structure
+ * [Project] Add simple forward propagation function
+ * [Project] Add small helpers for migration simplifications
+ * [Project] Allow to replace body in milter
+ * [Project] Bundle libev
+ * [Project] First refactoring step libevent->libev
+ * [Project] Implement syntax highlighting for Lua
+ * [Project] Lua_magic: Adopt lua_magic stuff in mime_types
+ * [Project] Remove libfann, gd and other unsupported stuff
+ * [Project] Remove torch
+ * [Project] Rework upstreams
+ * [Rework] Allow execution of async events when hs compiles regexps
+ * [Rework] Bayes expiry: eliminate `default` expiration mode
+ * [Rework] Dkim: Remove signing code
+ * [Rework] Dkim_signing: Move sign condition to dkim_signing
+ * [Rework] Do not lowercase all data send to ClickHouse
+ * [Rework] Drop url tags
+ * [Rework] Eliminate lua_squeeze as it has shown no improvements
+ * [Rework] Eliminate virtual scan time as it is useless
+ * [Rework] Lua core: Use lightuserdata to index classes
+ * [Rework] Lua_util: Another rework for extract_specific_urls
+ * [Rework] Migrate from ip_score to reputation
+ * [Rework] Move mime modification functions to lua_mime library
+ * [Rework] Rbl: Major whitelisting logic rework
+ * [Rework] Remove deprecated plugins
+ * [Rework] Remove log helper worker
+ * [Rework] Remove rspamd.classifiers.lua
+ * [Rework] Rename filter.h to a more sane name
+ * [Rework] Reorganise selectors implementation
+ * [Rework] Replace linenoise with replxx
+ * [Rework] Reputation: Remove ipnet from the ip reputation
+ * [Rework] Reputation: Slashing - change name of symbols
+ * [Rework] Rework children operations
+ * [Rework] Rework config reload
+ * [Rework] Rework expression API
+ * [Rework] Rework image urls processing
+ * [Rework] Rework initialisation to reduce static leaks count
+ * [Rework] Rework request headers processing
+ * [Rework] Slashing: Change versioning schema - move to 2.0
+ * [Rework] Slashing: Turn off postfilters when passthrough result is set
+ * [Rework] Start moving to replxx
+ * [Rework] Stop support of signed HTTP maps to simplify code
+ * [Rework] Store ASN as UInt32 in ClickHouse
+ * [Rework] Url_redirector: Rewrite plugin
+ * [Rework] Use a dedicated library for autolearn
+ * [Rework] Use libsodium instead of hand crafted crypto implementations
+ * [Rework] Use opaque structure to store a table of mime headers
+ * [Rules] Add dedicated bitcoin addresses filter rule
+ * [Rules] Add more detection to LEAKED_PASSWORD_SCAM
+ * [Rules] Catch LTC addresses
+ * [Rules] Reduce weight of RSPAMD_EMAILBL
+ * [Rules] Rework LEAKED_PASSWORD_SCAM rule one more time
+
+1.9.4: 23 May 2019
+ * [CritFix] Fix case sensitivity when parsing Content-Type
+ * [Fix] Arc: Another bunch of fixes for arc signing
+ * [Fix] Arc: More arc signing fixes
+ * [Fix] Avoid another overflow in fpconv
+ * [Fix] Fix ARC signing after fixing another bug in it...
+ * [Fix] Fix dkim signing exceptions
+ * [Fix] Fix some more corner cases for fpconv
+ * [Fix] Further fixes to printing of the FP numbers
+ * [Fix] Ratelimit: Fix dynamic score
+
+1.9.3: 13 May 2019
+ * [Conf] Add IP_SCORE_FREEMAIL composite rule
+ * [Feature] Add cryptobox method to generate dkim keypairs
+ * [Feature] Add fast hashes to lua cryptobox hash
+ * [Feature] Add least passthrough results
+ * [Feature] Allow oversign if exists mode
+ * [Feature] Clickhouse: Modernise table initial schema
+ * [Feature] Implement IUF interface for specific fast hashes
+ * [Feature] Lua_util: Allow to obfuscate different fields
+ * [Feature] Tune memory management in Rspamd and Lua
+ * [Fix] Avoid buffer overflow when printing long lua strings
+ * [Fix] Change the default oversigning headers to a more sane list
+ * [Fix] Clickhouse: Do not store digest as it is not needed now
+ * [Fix] Clickhouse: Fix lots of storage issues
+ * [Fix] Clickhouse: Support custom actions
+ * [Fix] Deny URLs where hostname is bogus
+ * [Fix] Do not blacklist mail by SPF/DMARC for local/authed users
+ * [Fix] Fix DoS caused by bug in glib
+ * [Fix] Fix UCL parsing of the multiline strings
+ * [Fix] Fix buffer overflow when printing small floats
+ * [Fix] Fix init code for servers keypairs cache
+ * [Fix] Fix issue with urls with no tld (e.g. IP)
+ * [Fix] Fix memory in arc signing logic
+ * [Fix] Fix memory leak in language detector during reloads
+ * [Fix] Fix mixed case content type processing
+ * [Fix] Fix processing of the ip urls in file
+ * [Fix] Fix use after free
+ * [Fix] HTML: Fix `size` attribute processing
+ * [Fix] Hum, it seems that 99ff1c8 was not correct
+ * [Fix] Lua_task: Fix task:get_from method
+ * [Fix] Preserve fd when mapping file to scan
+ * [Fix] Re-use milter_headers settings when doing arc signing
+ * [Fix] Set dmarc force action as least action
+ * [Fix] Switch to GMT
+ * [Fix] allow PKCS7 signatures to be text/plain, too
+ * [Project] Add initial version of the vault management tool
+ * [Project] Add vault support for DKIM and ARC signing
+ * [Project] Implement keys rotation in the vault
+ * [Project] Improve dkim keys generation for vault
+ * [Project] Improve keys creation in rspamadm vault
+ * [Rework] Move lua_worker to a dedicated unit
+ * [WebUI] Add URL fragments (#) support
+ * [WebUI] Fix AJAX request URL
+
+1.9.2: 16 Apr 2019
+ * [Conf] Allow to load users plugins from plugins.d
+ * [Conf] oversign openpgp and autocrypt headers
+ * [Feature] Add SPF FFI library for Lua
+ * [Feature] Add more verbosity for SPF caching
+ * [Feature] Antivirus: Handle encrypted files specially
+ * [Feature] Clickhouse: Slashing - add new fields to CH
+ * [Feature] Dkim_signing: Add OpenDKIM like signing_table and key_table
+ * [Feature] Dkim_signing: Allow to use new options as maps
+ * [Feature] Import fpconv library
+ * [Feature] Lua_maps: Allow static regexp and glob maps
+ * [Feature] Parse ical files
+ * [Feature] Rspamadm: Add dns_tool utility
+ * [Feature] Store SPF records digests
+ * [Feature] Use fpconv girsu2 implementation for printing floats
+ * [Fix] Clickhouse: Use integer seconds when inserting rows
+ * [Fix] Fix floating point printing
+ * [Fix] Fix processing of embedded urls
+ * [Fix] Lua_clickhouse: Fix CH errors processing
+ * [Fix] Make spf digest stable
+ * [Fix] Properly detect encrypted files in zip archives
+ * [Fix] Slashing: Store times in GMT timezone in ClickHouse
+ * [Rules] Add additional conditions to perform BTC checks
+ * [Rules] Fix pay-to-hash addresses validation
+
+1.9.1: 5 Apr 2019
+ * [Conf] Add vendor groups for symbols
+ * [Feature] Add `rspamadm template` command
+ * [Feature] Allow to add messages from settings
+ * [Feature] Allow unconnected DNS servers operations
+ * [Feature] Check limits after being set, migrate to uint64
+ * [Feature] Greylist: Allow to disable greylisting depending on symbols
+ * [Feature] Improve lua binary strings output
+ * [Feature] Mime_types: Implement user configurable extension filters
+ * [Feature] Mime_types: When no extension defined, detect it by content
+ * [Feature] Preprocess config files using jinja templates
+ * [Feature] Replies: Filter replies sender to limit whitelisting to direct messages
+ * [Feature] Treat all tags with HREF as a potential hyperlinks
+ * [Feature] Validate BTC addresses in LEAKED_PASSWORD_SCAM
+ * [Fix] Add crash safety for HTTP async routines
+ * [Fix] Another fix for Redis sentinel
+ * [Fix] Clickhouse: Fix table schema upload
+ * [Fix] Core: Fix squeezed dependencies handling for virtual symbols
+ * [Fix] Finally fix default parameters parsing in actions section
+ * [Fix] Fix ES sending logic (restore from coroutines mess)
+ * [Fix] Fix finishing script for clickhouse collection
+ * [Fix] Fix priority for regexp symbols registriation
+ * [Fix] Fix various issues found by PVS Studio
+ * [Fix] Initialize lua debugging earlier
+ * [Fix] Neural: Fix training
+ * [Fix] Rework cached Redis logic to avoid sentinels breaking
+ * [Fix] SURBL: Fix regression in surbl module
+ * [Fix] Fix double signing in the milter
+ * [Project] Add support of HTTP proxy in requests
+ * [Rework] Change lua global variables registration
+ * [Rework] Rework HTML content urls extraction
+ * [Rework] Start rework of aliasing in Rspamd
+ * [WebUI] Combine Scan and Learning into one tab
+ * [WebUI] Fix symbol score input type
+ * [WebUI] Show grayed out pie
+ * [WebUI] Update Throughput summary values dynamically
+
+1.9.0: 12 Mar 2019
+ * [Conf] Add missing includes
+ * [Conf] Move to options
+ * [Conf] Rbl: DWL is actually special whitelist
+ * [Conf] Relax some uribl rules
+ * [Conf] Remove abuse.ch
+ * [CritFix] Html: Entities are not valid within tag params values
+ * [Feature] Add `rspamadm mime sign` tool
+ * [Feature] Add configgraph utility
+ * [Feature] Add dedicated ZW spaces detection for URLs
+ * [Feature] Add flag to url object when visible part is url_like
+ * [Feature] Add method task:lookup_words
+ * [Feature] Add pyzor support (by crosenberg)
+ * [Feature] Allow to add upstream watchers to Lua API
+ * [Feature] Allow to set rewrite subject pattern from settings
+ * [Feature] Better escaping of unicode
+ * [Feature] Clickhouse: Allow to store subject in Clickhouse
+ * [Feature] Core: Add QP encoding utility
+ * [Feature] Core: Add libmagic detection for all parts
+ * [Feature] Core: Add support for gzip archives
+ * [Feature] Core: Allow to construct scan tasks from raw data
+ * [Feature] Core: Detect charset in archived files
+ * [Feature] Core: Ignore and mark invisible spaces
+ * [Feature] Core: Normalise zero-width spaces in urls
+ * [Feature] Core: Process data urls for images
+ * [Feature] Core: Relax quoted-printable encoding
+ * [Feature] Core: Support RFC2231 encoding in headers
+ * [Feature] Core: Support telephone URLs
+ * [Feature] Core: allow to emit soft reject on task timeout
+ * [Feature] DCC: Add bulkness and reputation checks to dcc
+ * [Feature] Elastic: Modernize plugin
+ * [Feature] Export visible part of url to lua
+ * [Feature] Fuzzy_storage: add preliminary support of rate limits
+ * [Feature] HTML: Specially treat data urls in HTML
+ * [Feature] Implement event watchers for upstreams
+ * [Feature] Implement includes tracing in Lua
+ * [Feature] Improve dkim part in configwizard
+ * [Feature] Lua_scanners: Add VadeSecure engine support
+ * [Feature] Lua_task: Add flexible method to get specific urls
+ * [Feature] Mime_types: Add MIME_BAD_UNICODE rule
+ * [Feature] Mime_types: Use detected content type as well
+ * [Feature] Plugins: Add preliminary version of the external services plugin
+ * [Feature] Query sentinel on master errors
+ * [Feature] Regexp: Allow local lua functions in Rspamd regexp module
+ * [Feature] Rspamadm: Allow to append footers to plain messages
+ * [Feature] Rspamadm: Allow to rewrite headers in messages
+ * [Feature] Selectors: Add `ipmask` processor
+ * [Feature] Settings: Allow hostname match
+ * [Feature] Settings: Allow local when selecting settings
+ * [Feature] Settings: Allow multiple selectors
+ * [Feature] Settings: Allow to inverse conditions
+ * [Feature] Support User-Agent in HTTP requests
+ * [Feature] Support ed25519 dkim keys generation
+ * [Feature] Try to filter bad unicode types during normalisation
+ * [Feature] external_services - oletools (olefy) support
+ * [Feature] lua_scanners - icap protocol support
+ * [Feature] lua_scanners - spamassassin spam scanner
+ * [Fix] Add filter for absurdic URLs
+ * [Fix] Add some more cases for Received header
+ * [Fix] Allow to disable/enable composite symbols
+ * [Fix] Arc: Use a separated list of headers for arc signing
+ * [Fix] Archive: Final fixes for 7z archives
+ * [Fix] Clickhouse: Fix database usage
+ * [Fix] Controller: Make save stats timer persistent
+ * [Fix] Core: Detect encrypted rarv5 archives
+ * [Fix] Core: Don't detect language twice
+ * [Fix] Core: Fix address rotation bug
+ * [Fix] Core: Fix content calculations for message parts
+ * [Fix] Core: Fix emails comments parsing and other issues
+ * [Fix] Core: Fix etags support
+ * [Fix] Core: Fix headers folding on the last token
+ * [Fix] Core: Fix iso-8859-16 encoding
+ * [Fix] Core: Fix log_urls flag (and encrypted logging)
+ * [Fix] Core: Fix part length when dealing with boundaries
+ * [Fix] Core: Fix parts distance calculations
+ * [Fix] Core: Fix processing of NDNs of certain type
+ * [Fix] Core: Implement logic to find some bad characters in URLs
+ * [Fix] Core: treat nodes with ttl properly in lru cache
+ * [Fix] Fix Content-Type parsing
+ * [Fix] Fix HTTP headers signing case
+ * [Fix] Fix control interface
+ * [Fix] Fix deletion of the duplicate headers
+ * [Fix] Fix emails filtering in emails module
+ * [Fix] Fix greylisting log message and logic
+ * [Fix] Fix issues with storing of the accepted addr in rspamd control
+ * [Fix] Fix maps object update race condition
+ * [Fix] Fix memor leaks and whitespace processing
+ * [Fix] Fix processing of null bytes in headers
+ * [Fix] Fix rcpt_mime and from_mime in user settings
+ * [Fix] Fix rfc2047 decoding for CD headers
+ * [Fix] Fix rfc2231 for Content-Disposition header
+ * [Fix] Fix setting of the subject pattern in config
+ * [Fix] Greylist: fix records checking
+ * [Fix] HTML: Another HTML comments exception fix
+ * [Fix] HTML: Another entities decoding logic fix
+ * [Fix] HTML: Fix HTML comments with many dashes
+ * [Fix] HTML: Fix entities in HTML attributes
+ * [Fix] HTML: Fix some more SGML tags issues
+ * [Fix] Ignore whitespaces at the end of value in DKIM records
+ * [Fix] MID module: Fix DKIM domain matching
+ * [Fix] Milter_headers: Fix remove_upstream_spam_flag and modernise config
+ * [Fix] Mime_parser: Fix issue with parsing of the trailing garbadge
+ * [Fix] Mime_parser: Fix parsing of mime parts without closing boundary
+ * [Fix] Multimap: Fix operating with userdata
+ * [Fix] Process orphaned `symbols` section
+ * [Fix] Rdns: Fix multiple replies in fake replies
+ * [Fix] Rework groups scores definitions
+ * [Fix] Set proper element when reading data from Sentinel
+ * [Fix] Set rspamd user to initialise supplementary groups on reload
+ * [Fix] Settings: Fix selectors usage
+ * [Fix] Sort data received from Sentinel to avoid constant replacing
+ * [Fix] groups.conf - filename typo
+ * [Fix] lua_scanner - oletools typos, logging
+ * [Fix] lua_scanners - actions and symbol_fail
+ * [Fix] lua_scanners - fix luacheck
+ * [Fix] lua_scanners - kaspersky - response with fname
+ * [Fix] lua_scanners - savapi redis prefix
+ * [Fix] tests - antivirus - fprot symbols
+ * [Project] Add concept of flexible actions
+ * [Project] Add heuristical from parser to received parser
+ * [Project] Add new flags to clickhouse, redis and elastic exporters
+ * [Project] Attach new received parser
+ * [Project] Fallback to callbacks from coroutines
+ * [Project] Implement keep-alive support in lua_http
+ * [Project] Lua_udp: Implement fully functional client
+ * [Project] Plug keepalive knobs into http connection handling
+ * [Project] Rspamadm: Add `modify` tool
+ * [Rework] Convert rspamd-server to a shared library
+ * [Rework] Dcc: Rework DCC plugin
+ * [Rework] Enable explicit coroutines symbols
+ * [Rework] Rework telephone urls parsing logic
+ * [Rework] Rewrite RBL module
+ * [Rework] Settings: Rework settings check
+ * [Rework] Slashing: Distinguish lualibdir, pluginsdir and sharedir
+ * [Rework] Unify task_timeout
+ * [Rework] Use VEX instructions in assembly, relocate
+ * [WebUI] Notify user if uploaded data was not learned
+ * [WebUI] Remove redundant condition
+
+1.8.3: 03 Dec 2018
+ * [CritFix] Make flags mutually exclusive for mime parts
+ * [CritFix] Strictly deny unencoded bad utf8 sequences in headers
+ * [Feature] Add Kaspersky antivirus support
+ * [Feature] Add method to get dkim results
+ * [Feature] Add more words regexp classes
+ * [Feature] Allow to choose words format in `rspamadm mime`
+ * [Feature] Allow to get all types of words from Lua
+ * [Feature] Allow to get task flags in C expressions
+ * [Feature] Allow to require encryption when accepting connections
+ * [Feature] Ignore bogus whitespaces in the words
+ * [Feature] Implement more strict configuration tests
+ * [Feature] Improve SPF results in Authentication-Results
+ * [Feature] Support ClickHouse database
+ * [Fix] Add failsafety for utf8 regexps
+ * [Fix] Do not trigger BROKEN_CONTENT_TYPE on innocent text parts
+ * [Fix] Emit error if connection has been terminated with no stop pattern
+ * [Fix] Fix boundaries checks in embedded messages
+ * [Fix] Fix double free
+ * [Fix] Perform policy downgrade on sample out, add tests
+ * [Fix] Properly escape utf8 regexps in hyperscan mode
+ * [Fix] Selectors - attachments args condition
+ * [Fix] Some fixes for raw parts
+ * [Fix] Treat learning errors as non-fatal
+ * [Fix] Use tld when looking for DKIM domains
+ * [Project] Words unicode structure rework
+ * [Project] Add preliminary Redis Sentinel support
+ * [Project] Improve Authentication-Results header
+ * [Project] Rework DKIM checks results
+ * [Project] Use more generalised API to produce meta words
+
+1.8.2: 19 Nov 2018
+ * [Conf] Add DWL support in the default configuration
+ * [Conf] Disable rspamd_update by default (again)
+ * [Conf] Fix configuration sample for ratelimit
+ * [CritFix] Disable broken url tags by default
+ * [CritFix] Fix \0 processing when doing RSA sign
+ * [CritFix] Fix adding symbols to their primary groups
+ * [Feature] Add `rspamadm cookie` utility
+ * [Feature] Add specialised functions for generating encrypted cookies
+ * [Feature] Add support of cookies in replies module
+ * [Feature] Add support of words regexps
+ * [Feature] Allow to add 3rd party clang plugins
+ * [Feature] Allow to create lua regexps from glob or plain patterns
+ * [Feature] Allow to set custom limits for upstream lists
+ * [Feature] Detect orphaned parts and attach them to message
+ * [Feature] Filter tokens in bayes
+ * [Feature] Fold b= value when doing arc sealing
+ * [Feature] Ignore cookies in the future and too old in the past
+ * [Feature] Skip stop words in statistics
+ * [Feature] Store stop words and allow to query them
+ * [Feature] Support query arguments in controller's custom commands
+ * [Feature] Tune upstream limits in Rspamd proxy
+ * [Feature] Use different callback symbols for different uribls
+ * [Feature] Write DKIM selector in dkim allow/reject symbols
+ * [Fix] Add obs_fws state support to eoh state machine
+ * [Fix] Add sanity check when applying mime boundaries heuristic
+ * [Fix] Antivirus - virus names with 0 were recognized as tables
+ * [Fix] Disable headernames in bayes temporarily
+ * [Fix] Do not allow syntax errors in include files...
+ * [Fix] Do not allow to merge an object with an array (or vice versa)
+ * [Fix] Don't perform forged recipients check for missing recipients
+ * [Fix] Fix DKIM based RBLs
+ * [Fix] Fix actrie implementation (sync from upstream), fixed OOB read
+ * [Fix] Fix explicit methods call in selectors
+ * [Fix] Fix extraction of additional parts
+ * [Fix] Fix finalization for internal plugins
+ * [Fix] Fix override_defaults function
+ * [Fix] Fix squeezed symbols when using settings
+ * [Fix] Fix urls insertion in Clickhouse module
+ * [Fix] Furhter fixes to ratelimits logic
+ * [Fix] Ignore signatures when looking for boundaries
+ * [Fix] Properly set learned count
+ * [Fix] Really fix ratelimits configuration and work
+ * [Fix] Remove ambigious format flag from printf
+ * [Fix] Restore URLs exporting in ClickHouse plugin
+ * [Fix] Rework bayes calculations...
+ * [Fix] Switch from chi-square to naive for large Fisher value
+ * [Fix] Treat normal password as enable password if there is no enable password
+ * [Fix] Use proper syntax for making DNS requests
+ * [Fix] Various fixes in embedded plugins
+ * [Project] Change fuzzy check selection logic to lua_fuzzy library
+ * [Project] Rework async events and symbols
+ * [Project] Move all metatokens in Bayes to lua_stat from C
+ * [WebUI] Add history rows per page control
+
+1.8.1: 16 Oct 2018
+ * [CritFix] Fix options insertion
+ * [CritFix] Fix words decay one more time (affects long messages)
+ * [CritFix] Increase default words_decay
+ * [CritFix] Plug memory leak in redis pool
+ * [Feature] Add `check_violation` feature to DKIM/ARC signing
+ * [Feature] Add only unique elements to Clickhouse url arrays
+ * [Feature] Allow `g+:` and `g-:` composite atoms
+ * [Feature] Allow dkim domains check in surbl
+ * [Feature] Allow maps with HTTP auth
+ * [Feature] Allow to disable actions by users settings
+ * [Feature] Extend whitelisting options
+ * [Feature] Store url object in images
+ * [Feature] Use verdict instead of the plain action in plugins
+ * [Fix] Allow to call fstring append with NULL string
+ * [Fix] DCC - luacheck
+ * [Fix] Do not load torch on each rspamadm invocation
+ * [Fix] Fix boundaries detection and rework stop words algorithm
+ * [Fix] Fix dependencies for DNS_SIGNED symbol
+ * [Fix] Fix errors when dealing with dynamic rates/bursts in Ratelimit
+ * [Fix] Fix groups mess
+ * [Fix] Fix groups mess
+ * [Fix] Fix parsing address with comments
+ * [Fix] Fix resolving in DMARC reports
+ * [Fix] Fix various issues with parsing of the received headers
+ * [Fix] Fix watchers issue in lua_tcp when doing no resolving
+ * [Fix] Plug memory leak in language detector (affects reloads)
+ * [Fix] Remove one letter stop words
+ * [Fix] Slashing: backport chunk logic from libucl
+ * [Fix] Stop libevent from using cached time in rspamadm
+ * [Fix] Try to fix watchers chaining
+ * [Fix] Various fixes in redis sync interface
+ * [Fix] ip_score - respect check_authed and check_local settings from config
+ * [Project] Rework passthrough actions
+ * [Project] Clustering module
+ * [Rework] Always create result for a task
+ * [Rework] Completely rewrite DMARC checks logic
+ * [Rework] Rework and fix whitelist plugin
+ * [WebUI] Add symbols sorting buttons
+ * [WebUI] Change symbols order without updating history
+ * [WebUI] Colorize symbols
+ * [WebUI] Do not display password form when secure_ip is set
+ * [WebUI] Fix symbol description tooltips display
+ * [WebUI] History: add sorting by symbol score value
+
+1.8.0: 24 Sep 2018
+ * [Feature] Add arguments schemas to processors and extractors
+ * [Feature] Add functional selectors library
+ * [Feature] Add generic selector to reputation module
+ * [Feature] Add more ratelimits: by digest, by attachments data, by filenames
+ * [Feature] Add preliminary stop words detection support
+ * [Feature] Add pure Lua debugm function
+ * [Feature] Add schema validation for Redis settings
+ * [Feature] Add selectors combine function
+ * [Feature] Add some recursion protection to lua logger
+ * [Feature] Add support for Lua API tracing
+ * [Feature] Allow to apply schema to arguments
+ * [Feature] Allow to get dkim signing data directly from HTTP headers
+ * [Feature] Allow to reuse existing authentication results
+ * [Feature] Cache selectors results in re runtime
+ * [Feature] Implement new text tokenizer based on libicu
+ * [Feature] Integrate selectors framework to multimap
+ * [Feature] Relax FORGED_RECIPIENTS
+ * [Feature] Support (almost) all html entities
+ * [Feature] Support adding and deletion of recipients in the milter block
+ * [Feature] Support gathering HTTP body from fragments in lua_http
+ * [Feature] Support multi flag in regexp and glob maps
+ * [Feature] Support selectors in ratelimit module
+ * [Feature] Support selectors in settings
+ * [Feature] Use khash in HTML parser
+ * [Feature] Use pure Lua debugm function
+ * [Fix] Add fail-safety for destroying sessions
+ * [Fix] Allow to add result-less fake DNS records
+ * [Fix] Another try to fix race conditions on config unload
+ * [Fix] Call Lua callback on DNS timeouts
+ * [Fix] Deprecate task:inc_dns_req as it is redundant
+ * [Fix] Do not allow events deletions on cleanup
+ * [Fix] Do not try to process skipped messages
+ * [Fix] Fix HTTP requests with no body
+ * [Fix] Fix another cleanup race condition
+ * [Fix] Fix bug in processing of pcre regexps
+ * [Fix] Fix byte array allocation in the pool
+ * [Fix] Fix crashes on task cleanup
+ * [Fix] Fix dynamic buckets in ratelimits
+ * [Fix] Fix endless loop when waiting for Rspamd to stop
+ * [Fix] Fix lua_util.str_split in case of delimiters set
+ * [Fix] Fix more issues with watching of async events
+ * [Fix] Fix stop words detection and loading logic
+ * [Fix] Fix various corner cases for language detection
+ * [Fix] Fix watchers in lua_tcp
+ * [Fix] Fix words decay algorithm
+ * [Fix] Implement watchers replacement to handle nested calls
+ * [Fix] Save faked code into fake dns record
+ * [Fix] Show the proper frame when using lua_util.debugm
+ * [Fix] Use fake dns records in tests
+ * [Fix] Use unicode replacements for HTML entities
+ * [Fix] fixed "cannot find dependency on symbol 1" issue when using replaced symbols in spamassassin rules
+ * [Fix] partition_id is not available in old versions of CH
+ * [Project] Add implicit conversion logic to selectors
+ * [Project] Add initial support for selectors in regexps
+ * [Project] Add method concept
+ * [Project] Further changes in unicode operations
+ * [Project] Implement Clickhouse migrations
+ * [Project] Implement implicit conversions to userdata
+ * [Project] Implement insert method
+ * [Project] Implement selectors registration for regular expressions
+ * [Project] Implement selectors support in re_cache
+ * [Project] Improve language detector: cleanup unused files, categorize
+ * [Project] Migrate CH data to a fat table
+ * [Project] Rework selectors logic
+ * [Project] Start Clickhouse utilities library
+ * [Project] Start unicode rework
+ * [Project] coroutine threaded model for API calls: thread pool
+ * [Rework] Move phishtank to a DNS based service
+ * [Rework] Rework Clickhouse plugin to use the new API
+ * [Rework] Rework language detector
+ * [Rework] Rework utf content processing in text parts
+ * [WebUI] Add progress bar for AJAX requests
+ * [WebUI] Avoid errors table reinitialization
+ * [WebUI] Avoid history table reinitialization
+ * [WebUI] Avoid throughput summary table reinitialization
+ * [WebUI] Destroy summary table on disconnect
+ * [WebUI] Fix "auth" request URL
+ * [WebUI] Fix disabling and hiding controls on page reload
+ * [WebUI] Fix maps loading from neighbours
+ * [WebUI] Fix symbols sorting by score
+ * [WebUI] Fix tables destroying
+ * [WebUI] Fix throughput data consolidation
+ * [WebUI] Fix upload buttons disabling
+ * [WebUI] Notify user on module loading failure
+ * [WebUI] Update FooTable 3.1.4 -> 3.1.6
+
+1.7.9: 01 Aug 2018
+ * [CritFix] Fix caseless comparison of equal length strings
+ * [Feature] Add HTTP basic auth support to elastic and clickhouse plugins
+ * [Feature] Add SPF selector to reputation
+ * [Feature] Add support of the fallback backends for HTTP maps
+ * [Feature] Allow to print full mime structure when extracting mime data
+ * [Feature] Allow to split symbols in reputation plugin
+ * [Feature] Check attachments only on AV scanners in attachments_only mode
+ * [Feature] Disable all SSL checks if ssl_no_verify flag is set
+ * [Feature] Implement parsing of scoped IPv6 addresses
+ * [Feature] Improve `rspamc counters` output
+ * [Fix] Add sanity checks when expanding SPF macros
+ * [Fix] Allow to parse SA rules with no spaces around =~ (dirty hack)
+ * [Fix] Avoid one extra byte writing
+ * [Fix] Deal with direct hash table
+ * [Fix] Detect empty text part as text, not HTML
+ * [Fix] Do not reduce map watch timeout for mixed http/file maps
+ * [Fix] Fix HTML part detection heuristic
+ * [Fix] Fix double free in redirectors cleanup
+ * [Fix] Fix legacy history handling in the controller
+ * [Fix] Fix messages insertion
+ * [Fix] Fix sending string method
+ * [Fix] Fix statconver command line arguments
+ * [Fix] Fixed argument checking for being null
+ * [Fix] Fixed issues reported by luacheck
+ * [Fix] Freeze updates queue when do actual storage update
+ * [Fix] HTTP map hash is per-backend and not per-map
+ * [Fix] Plug memory leak in fuzzy updates
+ * [Fix] Prefer 'MTA-Name' when producing authentication results
+ * [Fix] Replace bad unicode sequences instead of stopping on them
+ * [Fix] Set classifier version on learning
+ * [Project] Reworked ratelimits
+ * [Project] Apply topological sorting for symbols in Rspamd
+ * [Project] Remove global contexts from C modules
+ * [Project] Move performance critical hash tables to khash
+ * [WebUI] Avoid unused indexes
+ * [WebUI] Do not execute `on_success` callback
+ * [WebUI] Fix history reset for "All SERVERS" (#2346)
+ * [WebUI] Fix query URL for selected server
+ * [WebUI] Fix symbols display in legacy history,
+ * [WebUI] Hide symbols order selector for legacy history
+ * [WebUI] Refactor query functions into one
+ * [WebUI] Remove previously-attached event handlers
+ * [WebUI] Save symbols to the selected server
+ * [WebUI] Unify arguments of query functions
+ * [WebUI] Use common query functions to get graph data
+ * [WebUI] Use common query functions to save symbols
+
+1.7.8: 12 Jul 2018
+ * [Feature] Add more extended statistics about fuzzy updates
+ * [Feature] Add more non-conformant Received headers support
+ * [Feature] Add preliminary function to get fuzzy hashes from text in Lua
+ * [Feature] Allow to configure AV module rejection message
+ * [Feature] Implement fuzzy hashes extraction in mime tool
+ * [Feature] Improve WHITE_ON_WHITE rule
+ * [Feature] Improve integer -> string conversion
+ * [Feature] Reuse maps in multimap module more aggressively
+ * [Fix] Avoid race condition in skip map as pool lifetime is not enough
+ * [Fix] Eliminate all specific C plugins pools
+ * [Fix] Fix DKIM check rule if DNS is unavailable
+ * [Fix] Fix build where ucontext is defined in ucontext.h
+ * [Fix] Fix crash in base url handling
+ * [Fix] Fix descriptors leak in sqlite3 locking code
+ * [Fix] Fix messages quarantine
+ * [Fix] Fix padded numbers printing
+ * [Fix] Fix race condition on maps reinit
+ * [Fix] Fix regexp functions when no data is passed
+ * [Fix] Fix specific urls extraction
+ * [Fix] Fix styles propagation
+ * [Fix] Improve resetting of the limit buckets
+ * [Fix] Initialize sqlite3 properly
+ * [Fix] Work with broken resolvers in resolv.conf
+ * [Project] Implement HTTP maps caching
+ * [Project] Refresh fuzzy hashes when matched
+ * [Project] Add logic to deduplicate fuzzy updates queue
+ * [WebUI] Add missed declarations
+ * [WebUI] Avoid using "undefined" property
+ * [WebUI] Do not accept passwords containing control characters
+ * [WebUI] Do not redeclare variables
+ * [WebUI] Enable strict mode,
+ * [WebUI] Fix variable assignment
+ * [WebUI] Initialize variables at declaration
+ * [WebUI] Remove duplicated path from RequireJS config
+ * [WebUI] Remove unused block
+ * [WebUI] Remove unused variable
+ * [WebUI] Remove unused variables
+ * [WebUI] Use self-explanatory notation
+ * [WebUI] Use type-safe equality operators
+
+1.7.7: 02 Jul 2018
+ * [CritFix] Check NM part of pubkey to match it with rotating keypairs
+ * [CritFix] Do not overwrite PID of the main process
+ * [CritFix] Fix maps after reload
+ * [CritFix] Fix maps race conditions on reload
+ * [CritFix] Fix shmem leak in encrypting proxy mode
+ * [Feature] Add a concept of ignored symbols to avoid race conditions
+ * [Feature] Add ability to print bayes tokens in rspamadm mime
+ * [Feature] Add method to get statistical tokens in Lua API
+ * [Feature] Add preliminary mime stat command
+ * [Feature] Add rspamadm mime tool
+ * [Feature] Add urls extraction tool
+ * [Feature] Address ZeroFont exploit
+ * [Feature] Allow rspamadm mime to process multiple files
+ * [Feature] Allow to extract words in `rspamadm mime`
+ * [Feature] Allow to print mime part data
+ * [Feature] Allow to show HTML structure on extraction
+ * [Feature] Distinguish IP failures from connection failures
+ * [Feature] Improve output for mime command
+ * [Feature] Improve styles propagation
+ * [Feature] Main process crash will now cleanup all children
+ * [Feature] Preload file and static maps in main process
+ * [Feature] Print stack trace on crash
+ * [Feature] Process font size in HTML parser
+ * [Feature] Propagate content length of invisible tags
+ * [Feature] Read ordinary file maps in chunks to be more safe on rewrites
+ * [Feature] Support base tag in HTML
+ * [Feature] Support more size suffixes when parsing HTML styles
+ * [Feature] Support opacity style
+ * [Fix] Another fix for nested composites
+ * [Fix] Fill nm id in keypairs cache code
+ * [Fix] Fix colors alpha channel handling
+ * [Fix] Fix destruction logic
+ * [Fix] Fix double free
+ * [Fix] Fix maps preload logic
+ * [Fix] Fix nested composites process
+ * [Fix] Fix proxying of Exim connections
+ * [Fix] Fix reload crash
+ * [Fix] Fix rspamadm -l command
+ * [Fix] Update ed25519 signing schema
+ * [WebUI] Stop using "const" declaration
+ * [WebUI] Update RequireJS to 2.3.5
+
+1.7.6: 15 Jun 2018
+ * [CritFix] Fix multiple neural networks support
+ * [Feature] Add decryption function to keypair command
+ * [Feature] Add gzip compression for HTTP requests in elastic module
+ * [Feature] Add gzip methods to lua util
+ * [Feature] Add maps based on Top Level Domains
+ * [Feature] Add pubkey checks for dkim_signing
+ * [Feature] Add support of fake DNS records
+ * [Feature] Add tool to encrypt files
+ * [Feature] Allow to add symbols using settings directly
+ * [Feature] Allow to match private and public keys for DKIM signatures
+ * [Feature] Allow to set task flags via settings
+ * [Feature] Allow to specify fake DNS address from the config
+ * [Feature] Implement signatures verification using rspamadm keypair
+ * [Feature] Implement signing using `rspamadm keypair`
+ * [Feature] Improve error reporting for DKIM key access issues
+ * [Feature] Provide $HOSTNAME variable in UCL
+ * [Feature] Rework levenshtein distance computation
+ * [Feature] Split message parsing and processing
+ * [Feature] Support ED25519 DKIM signatures
+ * [Feature] Support encrypted configs in UCL
+ * [Feature] Suppress duplicate warning on very large radix tries
+ * [Feature] Use OSB to combine header names
+ * [Fix] Cleanup maps data on shutdown
+ * [Fix] Fix '~' behaviour in composites
+ * [Fix] Fix HTTP maps updates
+ * [Fix] Fix NIST signatures
+ * [Fix] Fix RFC822 comments when processing a mime address
+ * [Fix] Fix double free
+ * [Fix] Fix dynamic settings application
+ * [Fix] Fix for CommuniGate Pro maillist
+ * [Fix] Fix keypair creation method to actually create keypair...
+ * [Fix] Fix matching patterns with no paths
+ * [Fix] Fix memory leak in parsing comments
+ * [Fix] Fix parsing of urls with numeric password
+ * [Fix] Fix plugins intialisation in configwizard
+ * [Fix] Fix potential crash on reload
+ * [Fix] Fix potential race condition for a finished HTTP connections
+ * [Fix] Fix race-condition leak on processes reload
+ * [Fix] Fix signing in openssl mode
+ * [Fix] Free language detector structures
+ * [Fix] Relax alignment requirements
+ * [Fix] Send DMARC reports compressed
+ * [Fix] Try to fix leak in dmarc module
+ * [Fix] Try to plug memory leak in metric exporter
+ * [Project] Convert rspamadm subcommands to Lua
+ * [WebUI] Display smtp sender/recipient in history
+ * [WebUI] Fix elements disabling in "Symbols" tab
+ * [WebUI] Limit recipients list in history column to 3
+ * [WebUI] Match envelope and mime addresses following in arbitrary order
+ * [WebUI] Update column header
+ * [WebUI] Wrap addresses in history
+
+1.7.5: 18 May 2018
+ * [Conf] Add MSBL proposed return codes
+ * [Conf] Add additional groups for policies
+ * [CritFix] Do not use volatile Lua strings as UCL keys
+ * [Feature] Add ability to add fuzzy hashes to headers
+ * [Feature] Add function to extract most meaningful urls
+ * [Feature] Add rule to block mixed text and encrypted parts
+ * [Feature] Allow multiple groups for symbols
+ * [Feature] Allow to disable lua squeezing logic
+ * [Feature] Allow to get multipart children in Lua
+ * [Feature] Allow to insert multiple headers from milter headers
+ * [Feature] Allow to print scores in subject and further extensions
+ * [Feature] Be more error-prone in squeezed rules
+ * [Feature] Support multiple return codes in emails module
+ * [Feature] Use EMA for calculating averages
+ * [Feature] Use common jit cache for all regexps
+ * [Feature] support for CommuniGate Pro self-generated messages
+ * [Fix] Allow to have multiple values for headers as arrays
+ * [Fix] Do not open sockets for disabled workers
+ * [Fix] Fix AuthservId
+ * [Fix] Fix base64 folding in Lua API
+ * [Fix] Fix build on non-x86 platforms
+ * [Fix] Fix cached maps logic
+ * [Fix] Fix compatibility with old maps query logic
+ * [Fix] Fix crash if skip_map is used
+ * [Fix] Fix importing static maps from UCL
+ * [Fix] Fix parsing of unix sockets
+ * [Fix] Fix raw_mime regexp on HTML part with no text content
+ * [Fix] Fix tables logging
+ * [Fix] Fix vertical tab handling in libucl
+ * [Fix] Try to fix frequency counters
+ * [Fix] Use better sharding for ip_score
+ * [Fix] Use multiple results from SURBL DNS reply
+ * [Fix] When doing AV scan select a different server for retransmit
+
+1.7.4: 01 May 2018
+ * [Conf] Major stock config updates:
+ - Workers are now specified in a new format worker "type" { ... }
+ - Enable fuzzy worker to simplify local fuzzy storages configuration
+ - Bind all workers to localhost by default to avoid security flaws
+ * [Conf] Make more sane fuzzy_check default settings
+ * [CritFix] Fix ucl escape for bad symbols
+ * [Feature] Add failure symbol for AV module
+ * [Feature] Add lazy expiration mode for new classifier schema
+ * [Feature] Add preliminary version of maps stats plugin
+ * [Feature] Allow to block fuzzy requests from specific networks
+ * [Feature] Allow to change `expire` of live statistics
+ * [Feature] Distinguish AV failure from clean result
+ * [Feature] Further improvements of language detector by using khash
+ * [Feature] Further optimization of the lang_detection
+ * [Feature] Implement cluster-aware bayes expiry
+ * [Feature] Implement exclude patterns in rspamc
+ * [Feature] Implement glob maps in addition to regexp maps
+ * [Feature] Implement map statistics function for lua API
+ * [Feature] Implement stop symbols for Clickhouse collection
+ * [Feature] Support recipients separated by commas
+ * [Feature] Try harder to upload scripts to the Redis server
+ * [Feature] Upgrade t1ha distribution
+ * [Feature] use_domain_sign_inbound
+ * [Feature] Use scores from maps if `symbols_set` is not defined
+ * [Fix] Add resolving version of radix map helper
+ * [Fix] Check URL before adding implicit prefix
+ * [Fix] Do not check pid/state when using PRNG
+ * [Fix] Fix CentOS logrotate script for systemd
+ * [Fix] Fix slash + dot in urls
+ * [Fix] Fix systemd version of the logrotate script
+ * [Fix] Propagate key when import implicit array from Lua
+ * [Fix] Strip spaces from map keys and values
+ * [Fix] Try to fix a specific case when processing milter protocol
+ * [Fix] Try to fix crash when a tcp connection cannot be set
+ * [Fix] Typo use_domain_local --> use_domain_sign_local
+ * [Fix] Various fixes to once_received module
+ * [Project] Store hits counters for map elements
+
+1.7.3: 10 Apr 2018
+ * [CritFix] Plug bad memory leak in protocol reply
+ * [Feature] Add avx2 codec for base64
+ * [Feature] Add method to receive all URL flags from Lua API
+ * [Feature] Allow to fold headers on stop characters
+ * [Feature] Allow to set lua_cpath from options
+ * [Feature] Allow to specify custom rejection message in milter
+ * [Feature] Deal with unnormalised Unicode obfuscation
+ * [Feature] Do not detect language twice for relative parts
+ * [Feature] Implement oversigning feature
+ * [Feature] Implement silent logging level to minimize noise in logs
+ * [Feature] Improve URL_IN_SUBJECT rule
+ * [Feature] Use hashing to reduce redis attack surface
+ * [Fix] Add oversigning for the most important headers
+ * [Fix] add 'rewrite subject' to History dropdown
+ * [Fix] Another fix in folding algorithm
+ * [Fix] Do not call multimap addr for parts of addr if filter is presented
+ * [Fix] Do not clean hostname on generic reset
+ * [Fix] Do not create pid file in no-fork mode
+ * [Fix] Fix fold_after case to preserve multiple spaces
+ * [Fix] Fix folding and folding tests
+ * [Fix] Fix hostname usage in milter mode
+ * [Fix] Fix lua RSA verify and its tests
+ * [Fix] Fix metadata exporter send_mail backend (#2124)
+ * [Fix] Fix processing of '\v' in libucl
+ * [Fix] Fix shemaless URLs detection
+ * [Fix] Fix support of multiple headers in sign_header
+ * [Fix] Fix usage of util.parse_mail_address
+ * [Fix] Fix weights of dynamic squeezed rules
+ * [Fix] Leak from bucket before checking the burst
+ * [Fix] Stop using own localtime as DST could be messy in many cases
+ * [Fix] Treat unnormalised URLs as obscured
+ * [Rework] Restore leaky bucket model in ratelimit plugin
+ * [WebUI] Add messages total to throughput summary
+ * [WebUI] Add symbols order selector to history
+ * [WebUI] Config: Load list on demand
+ * [WebUI] Fix modalBody for maps that appear more than once
+ * [WebUI] History: Fix Tooltips on paging, filtering and sorting
+ * [WebUI] Remove a previously-attached event handler
+ * [WebUI] Update D3 to v5.0.0 and jQuery to v3.3.1
+
+1.7.2: 23 Mar 2018
+ * [Feature] Store emails in Clickhouse
+ * [Feature] Support single quotes in config
+ * [Feature] Use templates when publishing CH schema
+ * [Feature] Improve Docker image
+ * [Fix] Add rounding when printing a lot of FP variables
+ * [Fix] Allow to disable certain actions by assigning null to them
+ * [Fix] Disable results caching
+ * [Fix] Fix disabling of squeezed symbols
+ * [Fix] Fix scan time set
+ * [Fix] Rework logic of actions setting
+ * [Fix] Try to fix various Lua stack issues
+ * [WebUI] Add link tag for favicon.ico
+ * [WebUI] Display hostname:port/path in the page title
+
+1.7.1: 20 Mar 2018
+ * [CritFix] Fix lowercase comparison
+ * [CritFix] Timezone defines seconds WEST UTC not East
+ * [Feature] Add filename to log format
+ * [Feature] Add lua rules squeezing
+ * [Feature] Add related symbols analysis to rspamd_stats
+ * [Feature] Remove upstream `X-Spam: Yes` header by default
+ * [Feature] rspamd_stats: Output progress info on STDERR
+ * [Feature] Whitelist for emails module
+ * [Fix] Do not allow dependencies on self
+ * [Fix] Do not cache metric result
+ * [Fix] Do not trust all issuers as a client certificate
+ * [Fix] Fix dependencies in lua squeeze
+ * [Fix] Fix enabling/disabling squeezed rules
+ * [Fix] Fix enabling/disabling symbols
+ * [Fix] Fix external dependencies
+ * [Fix] Fix processing of a single compressed file
+ * [Fix] Fix some typos
+ * [Fix] Fix various modules in case of empty message
+ * [Fix] Handle callbacks that returns table of options
+ * [Fix] Improve cached action interaction
+ * [Fix] Make dynamic conf more NaN aware
+ * [Fix] Never hide actions from WebUI `configuration` tab
+ * [Project] Implementation of Lua rules squeezing
+
+1.7.0: 12 Mar 2018
+ * [Conf] Add bayes_expiry as explicit module
+ * [Conf] Adjust names and weights for neural network plugin
+ * [Conf] Change updates url
+ * [Conf] Default statistics is stored in Redis now
+ * [Conf] Disable fann_redis module by default
+ * [Conf] Fix default elastic configuration
+ * [Conf] Fix double quote position
+ * [Conf] Massive config rework for new structure of symbols and scores
+ * [Conf] Rename Rambler BLs as they are now Rspamd's ones
+ * [Conf] Use dedicated rspamd.com subdomains
+ * [Conf] Use more data from rspamd.com fuzzy storage
+ * [CritFix] Add sanity guards for badly broken HTML
+ * [CritFix] Another errors path handling fix
+ * [CritFix] Another portion of tokenization fixes
+ * [CritFix] Do not send reject messages after set reply
+ * [CritFix] Fix ARC chain verification
+ * [CritFix] Fix crash in milter errors handler
+ * [CritFix] Fix memory leak in spf caching logic
+ * [CritFix] Fix milter commands pipelining
+ * [CritFix] Fix newlines detection
+ * [CritFix] Fix semicolons parsing in the content type
+ * [CritFix] Plug memory leak in zstd protocol compression
+ * [Feature] Add ability to match score in force_actions module
+ * [Feature] Add aes-rng PRF to libottery
+ * [Feature] Add 'composites' debug module
+ * [Feature] Add concept of experimental modules
+ * [Feature] Add DKIM trace symbol
+ * [Feature] Add EBL to the default config
+ * [Feature] Add expected ip check for emails plugin
+ * [Feature] Add framework to manage Redis scripts
+ * [Feature] Add framing for the new reputation generic plugin
+ * [Feature] Add function to show plugins stat
+ * [Feature] Add gzip compression support for clickhouse module
+ * [Feature] Add gzip compression support for rspamd controller
+ * [Feature] Add gzip support when sending lua http requests
+ * [Feature] Add json output for rspamd_stats
+ * [Feature] Add method to do a synchronous Redis connection
+ * [Feature] Add method to get all content-type attributes in Lua
+ * [Feature] Add `-m` flag to configdump to show modules states
+ * [Feature] Add mime types to extensions map
+ * [Feature] Add more features to rescore utility
+ * [Feature] Add more gtube like patterns to test other spam actions
+ * [Feature] Add more metafunctions, improve logging
+ * [Feature] Add more text attributes
+ * [Feature] Add new configwizard command to rspamadm
+ * [Feature] Add new tooling for stats conversation
+ * [Feature] Add old groups migration tool
+ * [Feature] Add plugins state variable
+ * [Feature] Add preliminary ecdsa keys support in DKIM
+ * [Feature] Add preliminary support of idempotent symbols
+ * [Feature] Add Redis server wizard
+ * [Feature] Add routine to convert old style stats to a new one
+ * [Feature] Add some sanity checks for actions and controller
+ * [Feature] Add statistic convertation module to configwizard
+ * [Feature] Add sugestions logic to mempool allocator
+ * [Feature] Add support of config transform in Lua
+ * [Feature] Add timeout to rspamc when doing corpus test
+ * [Feature] Add tooling to convert bayes schemas
+ * [Feature] Add torch conditional to configuration
+ * [Feature] Add torch-decisiontree package
+ * [Feature] Add torch-optim contrib package
+ * [Feature] Add TTL autodetection
+ * [Feature] Add urls reputation to the reputation framework
+ * [Feature] Allow floating and negative values in expressions limits
+ * [Feature] Allow multiple CTs in full extensions map
+ * [Feature] Allow multiple fann rules
+ * [Feature] Allow randomly select User-Agent from a list
+ * [Feature] Allow rspamadm commands to export methods in Lua
+ * [Feature] Allow rule specific min_bytes in fuzzy check
+ * [Feature] Allow to adjust symbols scores from Lua
+ * [Feature] Allow to attach stat signature to messages
+ * [Feature] Allow to change SMTP from via milter headers
+ * [Feature] Allow to configure monitored
+ * [Feature] Allow to create directories in Lua API
+ * [Feature] Allow to disable torch and skip train samples for ANN
+ * [Feature] Allow to discard messages dynamically
+ * [Feature] Allow to enable/disable languages from the detector
+ * [Feature] Allow to generate DKIM keys from rspamadm API
+ * [Feature] Allow to get CPU flags from Lua
+ * [Feature] Allow to have high precision timestamps in logs
+ * [Feature] Allow to insert headers into specific position
+ * [Feature] Allow to limit redirector requests per task
+ * [Feature] Allow to load and use dynamic ANNs with torch
+ * [Feature] Allow to quarantine rejected messages using milter interface
+ * [Feature] Allow to receive signing keys from mempool vars
+ * [Feature] Allow to reserve elements in libucl
+ * [Feature] Allow to reuse signal handlers chains
+ * [Feature] Allow to set custom mempool variables from settings
+ * [Feature] Allow to set headers from settings
+ * [Feature] Allow to set Settings-Id for all connections
+ * [Feature] Allow to skip real action and add a header instead
+ * [Feature] Allow to skip specific hashes in fuzzy storage
+ * [Feature] Allow to spawn asynchronous processes from Lua
+ * [Feature] Allow to specify number of threads for ANN learning
+ * [Feature] Allow to use global lua maps in settings
+ * [Feature] Allow to use postfilters in composites
+ * [Feature] Allow to verify signatures from HTTP headers in maps
+ * [Feature] Antivirus: ordered pattern matches
+ * [Feature] Authentication-Results: support hiding usernames
+ * [Feature] Automatically create tables in clickhouse
+ * [Feature] Catch next-to-last bad extension
+ * [Feature] Check cached maps more frequently
+ * [Feature] Check groups sanity
+ * [Feature] Deal with obscured URLs with @ symbols
+ * [Feature] Enhance task:store_in_file method
+ * [Feature] Export password encryption routines to Redis
+ * [Feature] Filter nan and inf when adding scores
+ * [Feature] Finalize 7zip files support
+ * [Feature] Further improvements in language detection
+ * [Feature] Further improvements in language detection algorithm
+ * [Feature] Generic key name expansion for Redis keys
+ * [Feature] Hash whitelist for fuzzy_check
+ * [Feature] Implement bayes signatures storage
+ * [Feature] Implement buckets for Redis backend
+ * [Feature] Implement DKIM reputation adjustments
+ * [Feature] Implement forked workers children monitoring
+ * [Feature] Implement headers flags in mime parser
+ * [Feature] Implement l1/l2 regularization against the current weights
+ * [Feature] Implement manual ANN train mode
+ * [Feature] Implement per-user ANN support
+ * [Feature] Implement torch based ANN learning
+ * [Feature] Implement upstreams logic for clickhouse exporter
+ * [Feature] Import torch to Rspamd...
+ * [Feature] Improve allocation policy when interacting with Lua
+ * [Feature] Improve Lua/C interaction in history_redis
+ * [Feature] Improve multiple fuzzy results combining
+ * [Feature] Improve parsing of DKIM keys: parse algorithm
+ * [Feature] Improve subprocesses termination handle
+ * [Feature] Improve symbol type parsing in Lua API
+ * [Feature] Metadata Exporter: e-Mail Alerts: support multiple recipients; alerting senders/recipients/users (#1600)
+ * [Feature] Milter headers: support adding/removing arbitrary headers from config
+ * [Feature] More metatokens
+ * [Feature] Multimap: checking of symbol options
+ * [Feature] Multimap: template URL filter
+ * [Feature] New bayes expiry plugin
+ * [Feature] Periodically save rspamd stats to disk
+ * [Feature] Preliminary import of the elasticsearch module
+ * [Feature] Ratelimit: allow full addresses in whitelisted_rcpts
+ * [Feature] Ratelimit: support fetching limits from Redis
+ * [Feature] RBL: received: filtering by position & flags
+ * [Feature] Read global maps for lua
+ * [Feature] Redis settings: support checking multiple keys
+ * [Feature] Rework fann plugin to be a normal post-filter
+ * [Feature] Rework logging configuration for rspamadm case
+ * [Feature] Rework short hashes generation to avoid FP
+ * [Feature] Save real ucl types when exporting to Lua
+ * [Feature] Set TCP_NODELAY for milter sockets
+ * [Feature] Setup DKIM signing from configwizard
+ * [Feature] Skip certain symbols from ANN classify
+ * [Feature] Store plugins state
+ * [Feature] Support etag for HTTP maps
+ * [Feature] Support Expires header when using HTTP maps
+ * [Feature] Support sending given header multiple times in lua_http
+ * [Feature] Support sha512 in DKIM signatures
+ * [Feature] Try to detect HTML messages better
+ * [Feature] Use array instead of queue to reduce memory fragmentation
+ * [Feature] Use controller port by default when connecting to local IP
+ * [Feature] Use rdtsc where possible
+ * [Fix] Actively load skip hashes map in fuzzy storage
+ * [Fix] Add another workaround to display history properly
+ * [Fix] Add definition for old glib compatibility method
+ * [Fix] Add missing rspamadm control options to help
+ * [Fix] Add workaround for IPv6 in sendmail
+ * [Fix] Add workaround for system with non-XSI compatible tzset
+ * [Fix] Allow oversigning in DKIM signatures
+ * [Fix] Allow to check negative scores in force_actions
+ * [Fix] Allow to have negative actions limits
+ * [Fix] Allow to set any layers number for fann rules
+ * [Fix] Another fix for rdtcs
+ * [Fix] Another fix to lua xmlrpc
+ * [Fix] Another try to deal with #1998
+ * [Fix] Another try to fix #1998
+ * [Fix] Another try to fix threading in torch
+ * [Fix] Apply language detection when adding fuzzy hashes
+ * [Fix] ARC: Fix Lua 5.3 compatibility; timestamp should be integer
+ * [Fix] Authentication Results: Fix SPF smtp.mail_from
+ * [Fix] Auth-Results: Multiple DKIM signatures
+ * [Fix] Avoid changing content-transfer-encoding header's value
+ * [Fix] Better handling of the legacy protocol
+ * [Fix] Check decoded headers sanity (e.g. by excluding \0)
+ * [Fix] Check for magic when checking for an archive
+ * [Fix] Cleanup mess with groups
+ * [Fix] Clickhouse: Insertion in the symbols table
+ * [Fix] Crash in URL processing
+ * [Fix] Deal with another case when processing exceptions
+ * [Fix] Deal with deeply nested messages more aggressively
+ * [Fix] Deal with nan and inf encoding in json/ucl
+ * [Fix] Deal with non-key arguments in lua_redis.exec_script
+ * [Fix] Deal with unknown weight
+ * [Fix] Deal with URLs with no slashes after protocol
+ * [Fix] Deal with URLs wrapped in [] in text parts
+ * [Fix] Deal with zero scores symbols
+ * [Fix] Default monitoring domain for surbl plugin
+ * [Fix] Delay upstream re-resolving when one upstream is defined
+ * [Fix] Detection of maillist optimized and fixed
+ * [Fix] DKIM signing: allow for auth_only to be false
+ * [Fix] DMARC: require report_settings for sending reports only
+ * [Fix] Do not allow garbadge when checking url domain
+ * [Fix] Do not cache SPF records with PTR elements
+ * [Fix] Do not constantly re-resolve failed upstreams with a single element
+ * [Fix] Do not crash if no words defined
+ * [Fix] Do not crash on empty subtype
+ * [Fix] Do not expose spamtrap messages to SMTP reply
+ * [Fix] Do not fail rbl plugin when there are no received or emails
+ * [Fix] Do not ignore short words
+ * [Fix] Do not include idempotent/nostat symbols to checksum
+ * [Fix] Do not override groups when converting metrics
+ * [Fix] Do not override unix socket group when group comes before owner
+ * [Fix] Do not skip the last character
+ * [Fix] Do not spawn too many workers by default
+ * [Fix] Do not stop monitored on dns errors
+ * [Fix] Do not stop parsing headers on bad IP header
+ * [Fix] Do not strip last character in the last word
+ * [Fix] Do not treat script content as text
+ * [Fix] Do not try to connect to non-supported addresses
+ * [Fix] Do not try to dereference last character
+ * [Fix] Do not try to sign unknown domains
+ * [Fix] Don't use whitelist/greylist maps as regexp, but as map
+ * [Fix] Erase unknown HTML entities
+ * [Fix] Exim Received header protocol parsing
+ * [Fix] First load selector_map and path_map. And only return false when domain not found if try_fallback is false
+ * [Fix] Fix a lot of FP in chartable in mixed languages
+ * [Fix] Fix ANN checks
+ * [Fix] Fix ANN loading logic
+ * [Fix] Fix another tokenization issue
+ * [Fix] Fix autolearn parameters reading
+ * [Fix] Fix bad archive characters stripping
+ * [Fix] Fix bad extension check
+ * [Fix] Fix bayes schema conversion
+ * [Fix] Fix blacklists and DMARC in whitelist
+ * [Fix] Fix brain-damaged torch build system
+ * [Fix] Fix build on FreeBSD
+ * [Fix] Fix clickhouse exporter
+ * [Fix] Fix clickhouse schema
+ * [Fix] Fix comparision
+ * [Fix] Fix composites processing
+ * [Fix] Fix connecting to a unix socket in rspamadm statconvert
+ * [Fix] Fix couple of warnings
+ * [Fix] Fix crashes in the rspamd_control path
+ * [Fix] Fix deletion from hash
+ * [Fix] Fix DKIM forgeries via multiple headers
+ * [Fix] FIx dynamic conf plugin
+ * [Fix] Fix emails detection
+ * [Fix] Fix empty headers simple canonicalization
+ * [Fix] Fix empty threshold check in greylisting module
+ * [Fix] Fix encrypted legacy reply in fuzzy storage
+ * [Fix] Fix enormous scores for R_WHITE_ON_WHITE
+ * [Fix] Fix exceptions list in surbl
+ * [Fix] Fix *_EXCESS_BASE64 rules
+ * [Fix] Fix expire rounding
+ * [Fix] Fix extra hits in PCRE mode for regular expressions
+ * [Fix] Fix format strings
+ * [Fix] Fix get_content method
+ * [Fix] Fix groups override when defining symbols
+ * [Fix] Fix learned count in new schema
+ * [Fix] Fix learn errors propagation
+ * [Fix] Fix loading of per-user redis backend for statistics
+ * [Fix] Fix logging buffer corruption in case of repeated messages
+ * [Fix] Fix lua cached elements invalidation
+ * [Fix] Fix merging of the implicit arrays
+ * [Fix] Fix mime_types scoring
+ * [Fix] Fix multiple headers in DKIM headers list
+ * [Fix] Fix null callee case in clang plugin
+ * [Fix] Fix obscured url in format user@@example.com
+ * [Fix] Fix parsing of the per-user script
+ * [Fix] Fix priorities in rspamd_update, disable rules execution
+ * [Fix] Fix processing of closed tags
+ * [Fix] Fix processing of idempotent rules when autolearn fails
+ * [Fix] Fix processing of multipart parts with no headers
+ * [Fix] Fix processing of skip-hashes in fuzzy storage
+ * [Fix] Fix PTR processing in SPF
+ * [Fix] Fix pushing country to clickhouse asn table
+ * [Fix] Fix random forests module
+ * [Fix] Fix real IP parsing for some strange Exim received
+ * [Fix] Fix Redis timeout setup
+ * [Fix] Fix reload crash when hyperscan is enabled
+ * [Fix] Fix reusing of redis connection after exec
+ * [Fix] Fix sanity checks on macro value
+ * [Fix] Fix setting of path and cpath for Lua
+ * [Fix] Fix setting of signals when spawning a thread
+ * [Fix] Fix text splitting: stack overflow (too many captures)
+ * [Fix] Fix ticks processing
+ * [Fix] Fix upstream addrs updating
+ * [Fix] Fix urls/emails distinguishing found in queries
+ * [Fix] Fix user settings check
+ * [Fix] Fix variable increment
+ * [Fix] Fix various issues in stat_convert
+ * [Fix] F-PROT Antivirus infection string for all known occurences
+ * [Fix] F-PROT Antivirus: only check return code to determine infection
+ * [Fix] Further fixes around floating point expressions
+ * [Fix] Further fixes to ANN module
+ * [Fix] Further fixes to rescore tool
+ * [Fix] Further fixes to support ES 6
+ * [Fix] Further tokenization fixes
+ * [Fix] Greylisting set phase is not idempotent
+ * [Fix] Handle proxy copy errors
+ * [Fix] Header checks: Fix get_raw_header method
+ * [Fix] Header checks: REPLYTO_UNPARSEABLE rule
+ * [Fix] Kill spawned processes on termination
+ * [Fix] Load skip map from all processes as shared cache is unavailable
+ * [Fix] Lowercase HTTP headers to make them searchable from Lua
+ * [Fix] Lowercase words
+ * [Fix] Lua_http: freeing
+ * [Fix] Lua: lpeg to be loaded with rspamd_lua_add_preload, to avoid "rspamd_config_read: rcl parse error: cannot init lua file […] module 'lpeg' not found"
+ * [Fix] Map absence is not an error
+ * [Fix] Metadata exporter: check IP sanity
+ * [Fix] Milter headers: custom headers: removing headers
+ * [Fix] Milter headers: skip_local / skip_authenticated settings
+ * [Fix] Milter headers: X-Spamd-Result header if X-Virus ran first
+ * [Fix] mime_types: fix next-to-last extension length check
+ * [Fix] More hacks to deal with old configs
+ * [Fix] Move composites second pass to the dedicated stage
+ * [Fix] Multimap: received: filtering of artificial header
+ * [Fix] Multiple fixes in torch based ANN plugins
+ * [Fix] Once more (#1879) fix bad extension check
+ * [Fix] Optimize rspamd_fstring_t reallocations
+ * [Fix] options.local_networks setting
+ * [Fix] Parse HREF urls without explicit prefix
+ * [Fix] Plan new event on HTTP errors
+ * [Fix] Plug another possible memory leak
+ * [Fix] Plug memory leak
+ * [Fix] Plug memory leak in lua_tcp
+ * [Fix] Plug memory leak when setting email addresses from Lua
+ * [Fix] Propagate learn/stat errors more precisely
+ * [Fix] Ratelimit: fix whitelisted_rcpts matching
+ * [Fix] Ratelimit: lowercase email addresses
+ * [Fix] RBL: received: deal with missing data (#1965)
+ * [Fix] Rebalance and slightly rework MX check plugin
+ * [Fix] Redis key expansion: EVAL: deal with strings
+ * [Fix] Redis script loading in DMARC; URL tags; URL reputation
+ * [Fix] Reject invalid bh for DKIM signatures earlier
+ * [Fix] Relax pem signature detection
+ * [Fix] Relax unicode properties requirements for chartable module
+ * [Fix] Remove extra noise from dkim and arc signing
+ * [Fix] Remove hop-by-hop headers in proxy
+ * [Fix] Remove incorrect method `task:set_metric_subject`
+ * [Fix] Replace space like characters in headers with plain space
+ * [Fix] Restore old style ratelimits support
+ * [Fix] Rework elasticsearch plugin
+ * [Fix] Rewriting subjects via force actions module
+ * [Fix] RPM postinstall
+ * [Fix] Sanitize IP in history redis
+ * [Fix] Select the correct signature when doing simple canon
+ * [Fix] Set CLOEXEC flag on files opened
+ * [Fix] Setting check_local / check_authed in plugins (#1954)
+ * [Fix] Settings: avoid checking invalid IP (#1981)
+ * [Fix] Settings: header: deal with multiple settings (#1988)
+ * [Fix] Skip checks if both extensions are not bad
+ * [Fix] Skip nostat tokens when get number of tokens
+ * [Fix] Some more fixes towards emails detection
+ * [Fix] SpamAssassin: Fail check_freemail_header if regexp didn't match
+ * [Fix] Stop using of g_slice...
+ * [Fix] Switch rspamadm logging to message level
+ * [Fix] Symbol 'FANNR_SPAM' has its score defined..
+ * [Fix] Table parameter for rspamd_config:add_doc()
+ * [Fix] Treat 'rewrite subject' as spam action
+ * [Fix] Try harder in passing IPv6 addresses
+ * [Fix] Try harder to find rfc822 notifications
+ * [Fix] Try harder to find urls
+ * [Fix] Use decoded values when parsing mime addresses
+ * [Fix] Use full URL when making an HTTP request
+ * [Fix] Use greylisting threshold in greylisting module
+ * [Fix] Use n_words attribute from ngramms
+ * [Fix] Use raw urls when sending requests to redirector
+ * [Fix] Use the right boolean operator on error check
+ * [Fix] Use weight from map for fuzzy scoring
+ * [Fix] Various fixes to elastic plugin
+ * [Fix] Various fixes to fann_redis instantiation
+ * [Fix] Various improvements in language detection
+ * [Fix] Virus infection string for F-PROT Antivirus
+ * [Fix] Virus infetction string for F-PROT Antivirus
+ * [Fix] WebUI: use relative path for savemap (#1943)
+ * [Fix] WHITE_ON_WHITE: Ensure score is matched to part that fired the rule
+ * [Fix] Write configuration changes as UCL config
+ * [Project] Add detection logic for words
+ * [Project] Add fast debug logging infrastructure
+ * [Project] Add more flags to languages
+ * [Project] Add n-gramms data files
+ * [Project] Add ngramms frequencies detector
+ * [Project] Add random words selection logic
+ * [Project] Add unigramms to language detection as well
+ * [Project] Convert all C modules to fast debug infrastructure
+ * [Project] Detect some languages based on unicode script
+ * [Project] Enable fast debug lookup for some modules
+ * [Project] Enable language detector init in scanner workers
+ * [Project] Further improvements to language detector
+ * [Project] Implement logic of ngramms application
+ * [Project] Improve weighting in lang_detection
+ * [Project] Initialize language detector
+ * [Project] Preliminary version of ngramms based language detector
+ * [Project] Preliminary version of the new stat_convert
+ * [Project] Remove old language detector
+ * [Project] Rework language detection ngramms structure
+ * [Project] Start language detection project
+ * [Project] Start rework of language detection to improve quality
+ * [Project] Use fast debug logging check
+ * [Rework] Add frame for new reputation based IP score module
+ * [Rework] Continue stat_convert rework task
+ * [Rework] Implement new version of fuzzy replies
+ * [Rework] Improve readability of xmlrpc API
+ * [Rework] Kill metrics!11
+ * [Rework] Ratelimit module
+ * [Rework] Rename fann_redis to neural plugin
+ * [Rework] Reorganize mime_types module
+ * [Rework] Rework rescore utility
+ * [Rework] Rewrite model and learning logic for rescore
+ * [Rework] Run post-loads when all initialization is completed
+ * [Rework] Simplify lua path initialization
+ * [Rework] Start major stat_convert rework
+ * [Rework] Start mempool fragmentation reduce project
+ * [Rework] Start moving of fann redis to torch
+ * [Rework] Stop embedding rspamadm scripts into C
+ * [Rework] Use floating point arithmetics in Rspamd expressions
+ * [Rework] Use frequencies distribution in language detector
+ * [Rules] Penalise R_BAD_CTE_7BIT for utf8 messages
+ * [WebUI] Compact graph selectors
+ * [WebUI] Escape strings inside HTML in history
+ * [WebUI] Fix message count in throughput summary (#1724)
+ * [WebUI] Fix NaNs display on Throughput graph
+ * [WebUI] Migrate widgets to D3 v4
+ * [WebUI] Restore passwordless login support (#2003)
+ * [WebUI] Show symbol descriptions as tooltips in history
+ * [WebUI] Stop using commas in pie chart tooltips
+ * [WebUI] Update D3 and jQuery
+ * [WebUI] Update D3Evolution 1.0.0 -> 1.1.0
+
+1.6.6: 16 Feb 2018
+ * [CritFix] Add sanity guards for badly broken HTML
+ * [CritFix] Another errors path handling fix
+ * [CritFix] Fix ARC chain verification
+ * [CritFix] Fix crash in milter errors handler
+ * [Feature] Allow to insert headers into specific position
+ * [Feature] Allow to receive signing keys from mempool vars
+ * [Feature] Authentication-Results: support hiding usernames
+ * [Fix] Another try to deal with #1998
+ * [Fix] Another try to fix #1998
+ * [Fix] Better handling of the legacy protocol
+ * [Fix] Check decoded headers sanity (e.g. by excluding \0)
+ * [Fix] Deal with nan and inf encoding in json/ucl
+ * [Fix] Deal with URLs wrapped in [] in text parts
+ * [Fix] DKIM signing: allow for auth_only to be false
+ * [Fix] Do not crash on empty subtype
+ * [Fix] Do not fail rbl plugin when there are no received or emails
+ * [Fix] Do not skip the last character
+ * [Fix] Do not try to dereference last character
+ * [Fix] Do not try to sign unknown domains
+ * [Fix] Exim Received header protocol parsing
+ * [Fix] First load selector_map and path_map. And only return false when domain not found if try_fallback is false
+ * [Fix] Fix bad archive characters stripping
+ * [Fix] Fix comparision
+ * [Fix] Fix connecting to a unix socket in rspamadm statconvert
+ * [Fix] Fix empty headers simple canonicalization
+ * [Fix] Fix extra hits in PCRE mode for regular expressions
+ * [Fix] Fix parsing of the per-user script
+ * [Fix] Fix processing of skip-hashes in fuzzy storage
+ * [Fix] Fix Redis timeout setup
+ * [Fix] Fix sanity checks on macro value
+ * [Fix] Fix text splitting: stack overflow (too many captures)
+ * [Fix] Fix urls/emails distinguishing found in queries
+ * [Fix] F-PROT Antivirus: only check return code to determine infection
+ * [Fix] Metadata exporter: check IP sanity
+ * [Fix] Multimap: received: filtering of artificial header
+ * [Fix] Plan new event on HTTP errors
+ * [Fix] Plug another possible memory leak
+ * [Fix] Remove hop-by-hop headers in proxy
+ * [Fix] Sanitize IP in history redis
+ * [Fix] Setting check_local / check_authed in plugins (#1954)
+ * [Fix] Settings: avoid checking invalid IP (#1981)
+ * [Fix] Try harder in passing IPv6 addresses
+ * [Fix] WebUI: use relative path for savemap (#1943)
+ * [WebUI] Fix message count in throughput summary (#1724)
+ * [WebUI] Fix NaNs display on Throughput graph
+ * [WebUI] Restore passwordless login support (#2003)
+
+1.6.5: 22 Oct 2017
+ * [CritFix] Another portion of tokenization fixes
+ * [CritFix] Fix memory leak in spf caching logic
+ * [CritFix] Fix milter commands pipelining
+ * [CritFix] Fix newlines detection
+ * [Feature] Filter nan and inf when adding scores
+ * [Feature] Implement headers flags in mime parser
+ * [Feature] Support Expires header when using HTTP maps
+ * [Fix] Actively load skip hashes map in fuzzy storage
+ * [Fix] Add workaround for IPv6 in sendmail
+ * [Fix] Authentication Results: Fix SPF smtp.mail_from
+ * [Fix] Check for magic when checking for an archive
+ * [Fix] Deal with another case when processing exceptions
+ * [Fix] Deal with URLs with no slashes after protocol
+ * [Fix] Do not allow garbadge when checking url domain
+ * [Fix] Do not ignore short words
+ * [Fix] Do not strip last character in the last word
+ * [Fix] Do not treat script content as text
+ * [Fix] Erase unknown HTML entities
+ * [Fix] Fix another tokenization issue
+ * [Fix] Fix DKIM forgeries via multiple headers
+ * [Fix] Fix emails detection
+ * [Fix] Fix empty threshold check in greylisting module
+ * [Fix] Fix enormous scores for R_WHITE_ON_WHITE
+ * [Fix] Fix loading of per-user redis backend for statistics
+ * [Fix] Fix multiple headers in DKIM headers list
+ * [Fix] Fix obscured url in format user@@example.com
+ * [Fix] Further tokenization fixes
+ * [Fix] Load skip map from all processes as shared cache is unavailable
+ * [Fix] Lowercase words
+ * [Fix] Milter headers: skip_local / skip_authenticated settings
+ * [Fix] Milter headers: X-Spamd-Result header if X-Virus ran first
+ * [Fix] Ratelimit: fix whitelisted_rcpts matching
+ * [Fix] Some more fixes towards emails detection
+ * [Fix] SpamAssassin: Fail check_freemail_header if regexp didn't match
+ * [Fix] Use greylisting threshold in greylisting module
+
+1.6.4: 10 Sep 2017
+ * [Feature] Add method to get all content-type attributes in Lua
+ * [Feature] Add some sanity checks for actions and controller
+ * [Feature] Allow randomly select User-Agent from a list
+ * [Feature] Deal with obscured URLs with @ symbols
+ * [Feature] Milter headers: support adding/removing arbitrary headers from config
+ * [Fix] Add another workaround to display history properly
+ * [Fix] Add missing rspamadm control options to help
+ * [Fix] Auth-Results: Multiple DKIM signatures
+ * [Fix] Crash in URL processing
+ * [Fix] Default monitoring domain for surbl plugin
+ * [Fix] Detection of maillist optimized and fixed
+ * [Fix] Do not cache SPF records with PTR elements
+ * [Fix] Fix blacklists and DMARC in whitelist
+ * [Fix] Fix exceptions list in surbl
+ * [Fix] Fix processing of closed tags
+ * [Fix] Fix PTR processing in SPF
+ * [Fix] Lowercase HTTP headers to make them searchable from Lua
+ * [Fix] options.local_networks setting
+ * [Fix] Ratelimit: lowercase email addresses
+ * [Fix] Rebalance and slightly rework MX check plugin
+ * [Fix] Redis script loading in DMARC; URL tags; URL reputation
+ * [Fix] Reject invalid bh for DKIM signatures earlier
+ * [Fix] Remove incorrect method `task:set_metric_subject`
+ * [Fix] Rewriting subjects via force actions module
+ * [Fix] RPM postinstall
+ * [Fix] Treat 'rewrite subject' as spam action
+ * [Fix] Try harder to find urls
+ * [Fix] Use full URL when making an HTTP request
+ * [Fix] Use raw urls when sending requests to redirector
+ * [Fix] Use weight from map for fuzzy scoring
+ * [Rules] Penalise R_BAD_CTE_7BIT for utf8 messages
+
+1.6.3: 26 Jul 2017
+ * [CritFix] Fix semicolons parsing in the content type
+ * [Feature] Add EBL to the default config
+ * [Feature] Allow to configure monitored
+ * [Feature] Allow to skip specific hashes in fuzzy storage
+ * [Feature] Multimap: checking of symbol options
+ * [Feature] Redis settings: support checking multiple keys
+ * [Fix] ARC: Fix Lua 5.3 compatibility; timestamp should be integer
+ * [Fix] Avoid changing content-transfer-encoding header's value
+ * [Fix] Don't use whitelist/greylist maps as regexp, but as map
+ * [Fix] Fix get_content method
+ * [Fix] Header checks: Fix get_raw_header method
+ * [Fix] Header checks: REPLYTO_UNPARSEABLE rule
+ * [Fix] Lua_http: freeing
+ * [Fix] Milter headers: custom headers: removing headers
+ * [Fix] Parse HREF urls without explicit prefix
+ * [Fix] WHITE_ON_WHITE: Ensure score is matched to part that fired the rule
+ * [WebUI] Escape strings inside HTML in history
+
+1.6.2: 08 Jul 2017
+ * [Conf] Remove Rambler email bl for now
+ * [Conf] Switch RAMBLER_URIBL to a locally managed source
+ * [CritFix] Switch from ragel to C for Content-Type parsing
+ * [Feature] Add `-e` option for lua_repl
+ * [Feature] Add per-domain emails normalisation rules
+ * [Feature] Add sessions cache to debug dangling sessions
+ * [Feature] Add short_text_direct_hash for fuzzy check module
+ * [Feature] Add text_part:get_stats function
+ * [Feature] Allow to add custom processing script for surbl
+ * [Feature] Allow to check reply-to email
+ * [Feature] Allow to customize spam header, remove existing spam headers
+ * [Feature] Allow to disable specific workers in the config
+ * [Feature] Allow to discard messages instead of rejection
+ * [Feature] Allow to specify custom delimiter in emails plugin
+ * [Feature] Allow to specify custom User-Agent for rspamc
+ * [Feature] Allow to store symbols data in Clickhouse
+ * [Feature] Allow to use HTTPS when connecting to Clickhouse
+ * [Feature] Enable sessions cache tracking for milter connections
+ * [Feature] Implement per-line mode in lua_repl (like `perl -p`)
+ * [Feature] Implement rdns-curve plugin based on rspamd cryptobox
+ * [Feature] Improve maps cached data lifetime
+ * [Feature] Improve maps checking frequency
+ * [Feature] Improve monitored timeouts logic
+ * [Feature] milter_headers: add `extended_headers_rcpt` option
+ * [Feature] Milter headers: Add X-Spam-Flag to rmilter-compatibility headers
+ * [Feature] Milter headers: remove-header routine
+ * [Feature] Multimap: received filters for extracting TLDs from hostnames
+ * [Feature] Normalize email aliases in emails module
+ * [Feature] Re-add rambler email bl (as hashed list)
+ * [Feature] Reload file maps more frequently
+ * [Feature] Rework newlines strip parser one more time
+ * [Feature] Skip updates for messages scanned via controller
+ * [Feature] Split long DKIM public keys
+ * [Feature] Store more data when stripping newlines
+ * [Feature] Support SPF macros transformations
+ * [Feature] Support suppressing DMARC reports for some domains
+ * [Fix] Add missing `break` statement
+ * [Fix] Allow modifiers in SPF macros
+ * [Fix] DKIM sign tools: edge-cases around use_esld
+ * [Fix] Do not cache SPF records with macros
+ * [Fix] Do not overwrite score when setting pre-action
+ * [Fix] Fix comparison logic
+ * [Fix] Fix DKIM base64 folding for milter flagged messages
+ * [Fix] Fix emails module configuration
+ * [Fix] Fix folding for arc headers when milter interface is used
+ * [Fix] Fix gmail dots removal
+ * [Fix] Fix rspamc detection in greylist module
+ * [Fix] Fix some more issues with HTTP maps
+ * [Fix] Milter sessions can live forever
+ * [Fix] Normalize fuzzy probability better
+ * [Fix] Plug memory leak
+ * [Fix] RBL: Fixed hashed email address lookups
+ * [Fix] Try to deal with brain-damaged milter behaviour
+ * [Fix] Use `\n` to fold headers for milter
+ * [Rework] Allow to use custom callback for monitored checks
+ * [Rework] Further steps towards one process monitoring
+ * [Rework] Send health checks from a single worker
+ * [WebUI] Round-up throughput summary values
+
+1.6.1: 14 Jun 2017
+ * [Fix] Allow to init resolver without rspamd_config
+ * [Fix] Do not crash when resolver failed to initialize
+ * [Fix] Fix abstract context layout
+ * [Fix] Fix CGP helper reply parsing
+ * [Fix] Fix crashes when socket write errors occur
+ * [Fix] Fix parsing IPv6 nameservers in resolv.conf
+ * [Fix] Milter: Don't defer on "greylist" action
+
+1.6.0: 12 Jun 2017
+ * [Conf] Add rspamd_proxy to the default configuration set
+ * [Conf] Add sample arc module config
+ * [Conf] Do away with systemd specifics completely
+ * [Conf] Increase min_bytes to avoid FP
+ * [Conf] Remove ratelimits from default configuration
+ * [CritFix] Fix accepting on IPv6 sockets
+ * [CritFix] Fix corruption when multiple fuzzy are defined
+ * [CritFix] Fix learn condition in fuzzy check
+ * [CritFix] Fix memory leak in fuzzy check
+ * [CritFix] Fix memory leak in maps scheduling
+ * [CritFix] Paese the last character in DKIM signature correctly
+ * [CritFix] Zero fill sockaddr_un
+ * [Feature] Add ability to add doc strings by example
+ * [Feature] Add API to verify DKIM (and ARC) signatures
+ * [Feature] Add compression/decompression to proxy
+ * [Feature] Add count to url structure
+ * [Feature] Add initial support of the new protocol reply
+ * [Feature] Add Lua plugin spamtrap
+ * [Feature] Add `monitored_address` for rbls
+ * [Feature] Add new schema for bayes tokens
+ * [Feature] Add preliminary ARC support to dkim code
+ * [Feature] Add preliminary support of ARC signing
+ * [Feature] Add rules to detect bad 8bit characters in From and To
+ * [Feature] Add scanning support for milter protocol
+ * [Feature] Add support for bidirectional symbols in rspamd_stats
+ * [Feature] Add support for static maps
+ * [Feature] Add support of maps with multiple regexps matches
+ * [Feature] Add `text_multiplier` param
+ * [Feature] Add the preliminary ARC plugin
+ * [Feature] Add top redirector targets rank
+ * [Feature] Allow async events to be registered from LUA rules
+ * [Feature] Allow storing bayes tokens in Redis
+ * [Feature] Allow to exclude specific domains from mx check
+ * [Feature] Allow to have a stack of watcher finalisers
+ * [Feature] Allow to pass hostname to `-i` flag in Rspamc
+ * [Feature] Allow to set custom user agent in url redirector
+ * [Feature] Allow to use custom callback when parsing resolv.conf
+ * [Feature] Allow to use domain from authenticated user
+ * [Feature] Bayes expiry plugin
+ * [Feature] Check dkim sign keys for modifications
+ * [Feature] DKIM signing: sign_networks/local address specific use_domain settings
+ * [Feature] DMARC: Support excluding domains from sampling
+ * [Feature] Expire processing items for URL redirector aggressively
+ * [Feature] Fix surbl monitored for IP lists, add `monitored_domain` option
+ * [Feature] Implement caching for dkim body hashes
+ * [Feature] Implement milter protocol scan reply
+ * [Feature] Improve omograph phishing detection
+ * [Feature] Initial support of self-scan in Rspamd proxy
+ * [Feature] Keep track of headers in milter interface
+ * [Feature] Milter headers: better controls for local/authenticated
+ * [Feature] Multimap: email:domain:tld filter
+ * [Feature] Preliminary DMARC reporting implementation
+ * [Feature] Reuse stemmers in the cache
+ * [Feature] Rework confighelp to load Lua plugins
+ * [Feature] Rework hfilter to use hyperscan if possible
+ * [Feature] Rework lua RSA API
+ * [Feature] Rmilter_headers: approximate rmilter's extended_spam_headers
+ * [Feature] Start integration of milter support in proxy
+ * [Feature] Store average words length and short words count
+ * [Feature] Store hash of headers order and names
+ * [Feature] Support MTA name header
+ * [Feature] Support multiple types of dkim signing in Lua
+ * [Feature] Support numeric arguments for Redis requests
+ * [Feature] Use headers hash in bayes metatokens
+ * [Feature] Use normal resolv.conf rules of rotation in Rspamd
+ * [Feature] Use version 2 proto for checking messages
+ * [Fix] Allow to follow symlinks when safe
+ * [Fix] Append MX name for authentication results as required
+ * [Fix] Change default text multiplier from 0.5 to 2.0
+ * [Fix] Check min_bytes for images as well
+ * [Fix] Deal with 7bit charsets properly
+ * [Fix] Deal with 8bit characters in email addresses
+ * [Fix] Deal with unpaired <a> tags
+ * [Fix] Detect confighelp in plugins initialisation
+ * [Fix] Disable certain checks for utf spoof detection
+ * [Fix] DKIM Signing: avoid nil index when From header is missing
+ * [Fix] Do not add exact hashes from different parts
+ * [Fix] Do not check DMARC if SPF or DKIM were not checked
+ * [Fix] Do not check URLs that are resolved to be redirected
+ * [Fix] Do not set bayes probability if we don't use it
+ * [Fix] Do not stop on illegal unicode points - replace them
+ * [Fix] Fix another race condition in arc checks
+ * [Fix] Fix arc count logic
+ * [Fix] Fix ARC signing
+ * [Fix] Fix brain-damaged spamc protocol for now
+ * [Fix] Fix calling for peak functions
+ * [Fix] Fix couple of issues in FORWARDED rule
+ * [Fix] Fix CTE propagation from parent containers to children parts
+ * [Fix] Fix errors processing in the controller
+ * [Fix] Fix format string in milter
+ * [Fix] Fix issues in SPF macros parsing
+ * [Fix] Fix logging format string
+ * [Fix] Fix logic of cached passwords check
+ * [Fix] Fix lowercasing of stemmed words
+ * [Fix] Fix LRU elements removal
+ * [Fix] Fix memory leak when accepting from unix sockets
+ * [Fix] Fix milter connections persistence
+ * [Fix] Fix objects merging in UCL
+ * [Fix] Fix order of operations to avoid race condition
+ * [Fix] Fix parsing of long regexp types
+ * [Fix] Fix passing data to log helper when many symbols defined
+ * [Fix] Fix pools management for milter session
+ * [Fix] Fix processing of the watchers
+ * [Fix] Fix queue id macro in milter
+ * [Fix] Fix R_BAD_CTE_7BIT rule
+ * [Fix] Fix Redis timeout set
+ * [Fix] Fix REPLYTO_UNPARSEABLE rule
+ * [Fix] Fix setting of email address
+ * [Fix] Fix some more issues about duplicated fuzzy requests
+ * [Fix] Fix spamc support in rspamd proxy
+ * [Fix] Fix syntax error in spamtrap plugin
+ * [Fix] Fix url counts for href urls
+ * [Fix] Fix url handling in the protocol
+ * [Fix] Multimap: Received IP filters with Redis
+ * [Fix] Oops, fix d9d0fa5e86db2f4470d34395a233b450478b2f60
+ * [Fix] Parse rgb[a](x,x,x[,x]) css colors
+ * [Fix] Phishing: strict_domains
+ * [Fix] Reduce maps aggressiveness
+ * [Fix] Reresolve upstreams even if there is a single server there
+ * [Fix] Rspamadm grep: Disable Lua patterns in string search by default
+ * [Fix] Skip text parts when checking binary parts in fuzzy check
+ * [Fix] Support v2 checks in controller
+ * [Fix] Treat empty address as valid
+ * [Fix] Try harder to detect CTE
+ * [Fix] Try to deal with v4 mapped to v6 addresses on accept
+ * [Fix] Use dkim signing callback properly
+ * [Fix] Use non-volatile memory for storing data
+ * [Fix] Use static maps instead of ugly hack for radix_from_config
+ * [Fix] Use the same pool for related sessions
+ * [Rework] Continue modularisation for lua library
+ * [Rework] Initial milter protocol support
+ * [Rework] Make log pipes worker agnostic, add scanners API
+ * [Rework] Move authentication results generation to a separate routine
+ * [Rework] Move common DKIM functions to a separate lua module
+ * [Rework] Move global functions to a separate directory
+ * [Rework] Prepare dkim module for ARC checks
+ * [Rework] Propagate ucl variables from the command line
+ * [Rework] Remove multiple metrics support from Rspamd
+ * [Rework] Stop using name 'rmilter' for the modern protocol
+ * [Rework] Use LFU algorithm in LRU cache
+ * [Rules] Fix received TLS rules
+ * [Rules] Improve URL_COUNT_ODD rule
+ * [WebUI] Fix add header filter in history
+ * [WebUI] Use modern protocol for checking messages
+
+1.5.9:
+ * [Conf] Increase min_bytes to avoid FP
+ * [Conf] Remove ratelimits from default configuration
+ * [CritFix] Fix accepting on IPv6 sockets
+ * [CritFix] Zero fill sockaddr_un
+ * [Feature] Add `text_multiplier` param
+ * [Fix] Check min_bytes for images as well
+ * [Fix] Do not add exact hashes from different parts
+ * [Fix] Fix memory leak when accepting from unix sockets
+ * [Fix] Fix some more issues about duplicated fuzzy requests
+ * [Fix] Phishing: strict_domains
+ * [Fix] Skip text parts when checking binary parts in fuzzy check
+ * [Minor] Add the same duplicates protection for all fuzzy hashes types
+ * [Minor] Fix braces
+ * [Minor] Fix test
+ * [Minor] SPOOF_DISPLAY_NAME: Use all SMTP/MIME recipients
+ * [Minor] Validate assumed spoofed display name domains to contain a dot
+
+
+1.5.8:
+ * [CritFix] Fix memory leak in fuzzy check
+ * [CritFix] Fix memory leak in maps scheduling
+ * [Feature] Multimap: email:domain:tld filter
+ * [Fix] DKIM Signing: avoid nil index when From header is missing
+ * [Fix] Do not set bayes probability if we don't use it
+ * [Fix] Do not stop on illegal unicode points - replace them
+ * [Fix] Fix brain-damaged spamc protocol for now
+ * [Fix] Fix Redis timeout set
+ * [Fix] Fix spamc support in rspamd proxy
+ * [Fix] Multimap: Received IP filters with Redis
+ * [Fix] Parse rgb[a](x,x,x[,x]) css colors
+ * [Fix] Reresolve upstreams even if there is a single server there
+ * [Fix] Treat empty address as valid
+ * [Fix] Try harder to detect CTE
+ * [Fix] Try to deal with v4 mapped to v6 addresses on accept
+ * [Minor] Add `wsf` and `hta` bad archive extensions
+ * [Minor] Fix configuration option
+ * [Minor] Fix result parsing for SAVAPI
+ * [Minor] Further logging improvements
+ * [Minor] Improve logging of errors
+ * [Minor] Prevent MID_CONTAINS_FROM from firing on empty address
+ * [Minor] Reduce digit->number transmission penalty
+ * [Minor] Relax CTYPE_MISSING_DISPOSITION rule
+
+
+1.5.7:
+ * [CritFix] Fix corruption when multiple fuzzy are defined
+ * [CritFix] Fix learn condition in fuzzy check
+ * [Feature] Add rules to detect bad 8bit characters in From and To
+ * [Feature] DKIM signing: sign_networks/local address specific use_domain settings
+ * [Feature] Support numeric arguments for Redis requests
+ * [Fix] Deal with 8bit characters in email addresses
+ * [Fix] Fix couple of issues in FORWARDED rule
+ * [Fix] Fix passing data to log helper when many symbols defined
+ * [Fix] Fix R_BAD_CTE_7BIT rule
+ * [Fix] Fix REPLYTO_UNPARSEABLE rule
+ * [Fix] Fix setting of email address
+ * [Fix] Rspamadm grep: Disable Lua patterns in string search by default
+ * [Minor] Add Lua 5.3 workaround
+ * [Minor] Add lua methods to detect if a part has 8bit characters
+ * [Minor] Allow session-less lua dns requests
+ * [Minor] Allow to append greylist end time to message reported
+ * [Minor] Avoid `nil` table
+ * [Minor] Disable dkim_signing if redis is specified but not configured
+ * [Minor] Fix build with pcre2
+ * [Minor] Fix rule
+ * [Minor] Fix warnings
+ * [Minor] Format floating point number
+ * [Minor] Push email flags to the lua API
+ * [Minor] Silence some warnings
+ * [Minor] Silence warning
+ * [Minor] Try all hostname regexps to find the most significant one
+ * [WebUI] Fix add header filter in history
+
+1.5.6:
+ * [Feature] Add unigramms support in bayes
+ * [Feature] Allow configurable sign headers for DKIM
+ * [Feature] Allow to add unigramm metatokens from Lua
+ * [Feature] DKIM Signing: envelope match exception for local IPs
+ * [Feature] UCL: register parser variables from Lua
+ * [Fix] Always try to adjust filename
+ * [Fix] Do extra copy to ensure that original content is never touched
+ * [Fix] Fix SPOOF_REPLYTO rule
+ * [Fix] Ignore Rmilter added Received
+ * [Fix] More fixes for hashed email dnsbls
+ * [Fix] Plug memory leak in chartable module
+ * [WebUI] Display multiple alerts at once
+
+
+1.5.5:
+ * [CritFix] Fix classifier learning with Redis backend
+ * [CritFix] Fix issue when parsing encoded rfc822/messages
+ * [Feature] Add escaped version of lua_ucl import
+ * [Feature] Add task:headers_foreach function
+ * [Feature] Allow to process filenames from content type
+ * [Feature] Allow to query hashed emails
+ * [Feature] Ignore bayes with mostly metatokens or with too few text
+ * [Feature] Probabilistically skip metatokens
+ * [Feature] Retrieve all virus names from SAVAPI
+ * [Feature] Rework classifiers lua metatokens
+ * [Feature] Store headers order
+ * [Feature] Store text tokens inside bayes tokens
+ * [Feature] Use cached shingles keys
+ * [Fix] Add missing score normalisation for HFILTER_URL_ONLY
+ * [Fix] Avoid lookup in absent hash
+ * [Fix] Check return values from Lua functions called from C
+ * [Fix] Do not count sending and loading time in rspamc
+ * [Fix] Escape json strings for controller rejplies from Lua
+ * [Fix] Fix archive scans for savapi
+ * [Fix] Fix domain_only emails RBL
+ * [Fix] Fix ip_score map configuration
+ * [Fix] Fix JSON output for history_redis
+ * [Fix] Fix one character length substrings search
+ * [Fix] Fix parsing of non-RFC compatible Exim received
+ * [Fix] Fix parsing of options for workers with the same type
+ * [Fix] Fix processing of small tokens vectors
+ * [Fix] Fix rfc2047 tokenization
+ * [Fix] Fix typo
+ * [Fix] More fixes for inplace decoding
+ * [Fix] Try to avoid modifications of the original data
+ * [Fix] URL redirector: Fix call to is_redirector
+ * [Rework] Set token data as uint64_t instead of chars array
+ * [WebUI] Check if neighbours' history backend versions match
+ * [WebUI] Disable phrase connectors replacement in history filtering
+ * [WebUI] Disable phrase connectors replacement in symbols filtering
+ * [WebUI] Do not hide messages with bad subject, just replace it with '???'
+ * [WebUI] Fix error message
+ * [WebUI] Fix history v2 display
+ * [WebUI] Fix legacy history
+ * [WebUI] history: break To address lists on commas
+ * [WebUI] Increase default timeout to 20 seconds
+ * [WebUI] Save some history table space
+
+1.5.4:
+ * [Conf] Add history_redis default configuration
+ * [Feature] Add spoofed rules
+ * [Feature] Add URL_IN_SUBJECT rule
+ * [Feature] Allow to get task's subject
+ * [Feature] Allow to specify maximum number of shots for symbols
+ * [Feature] Distinguish URLs found in Subject
+ * [Feature] Memoize LPEG grammars
+ * [Feature] Parse else parts in SA rules
+ * [Feature] Process subject for mixed characters
+ * [Feature] Resolve url chains in url_redirector module
+ * [Feature] Stat greylisted messages as greylisted not soft-rejected
+ * [Feature] Support checking for redirector in Lua SURBL
+ * [Feature] Support tag_exists SA function
+ * [Feature] Work with broken rfc2047 tokens
+ * [Fix] Check all watcher's dependencies
+ * [Fix] Do not compile hyperscan with no SSSE3 support
+ * [Fix] Do not crash if cannot decode qp encoded part
+ * [Fix] Fix dependencies of DKIM when multiple signatures are found
+ * [Fix] Fix lists in whitelist plugin
+ * [Fix] Fix one-shot symbols weight calculations
+ * [Fix] Fix options and shots match
+ * [Fix] Fix order of symbol options
+ * [Fix] Fix parsing of dot at the end of the address
+ * [Fix] Fix parsing of lua table arguments
+ * [Fix] Fix processing of subject words
+ * [Fix] Fix string split memoization
+ * [Fix] Fix templates grammar usage
+ * [Fix] Fix various issues related to Lua stack manipulation
+ * [Fix] Force actions: Use postfilter if we have honor_action / require_action
+ * [Fix] Further fixes to avoid PHISHING FP
+ * [Fix] Preserve order of options in symbols
+ * [Fix] Rspamadm grep: deal with unusually-formatted logs
+ * [Fix] Use hostname suffix when dealing with history
+ * [Rework] Remove outdated SA rules
+ * [WebUI] Add flexible columns
+ * [WebUI] Add footable
+ * [WebUI] Add sender, recipients and subject columns
+ * [WebUI] Allow message-id break
+ * [WebUI] Fix history clustering
+ * [WebUI] Fix history display
+ * [WebUI] Fix sorting
+ * [WebUI] Humanize sizes
+ * [WebUI] Initial move towards footable
+ * [WebUI] Remove datatables
+ * [WebUI] Replace `.values` method with `.map`
+ * [WebUI] Rework v2 symbols display
+ * [WebUI] Try to normalize frequencies
+ * [WebUI] Unbreak WebUI
+ * [WebUI] Use Footable to draw Throughput summary table
+
+1.5.3:
+ * [Conf] Add composite for hacked wordpress phishing
+ * [CritFix] Fix base64 decoding when there are unparseable characters
+ * [Feature] Additional symbol metadata in metadata exporter
+ * [Feature] Add method to get protocol reply from Lua
+ * [Feature] Add symbols when tagged rcpt/sender are normalised
+ * [Feature] Add task:get_symbols_all() function
+ * [Feature] Allow multiple formats of DKIM signing key
+ * [Feature] Allow to cache and use flexible protocol reply
+ * [Feature] Allow to set one_shot flag from register_symbol
+ * [Feature] Allow to skip certain types of hashes when learning fuzzy
+ * [Feature] Cache and insert scan time into the protocol
+ * [Feature] Detect newlines in rspamc --mime
+ * [Feature] DKIM signing: support use of maps
+ * [Feature] Greylist: Support excluding low-scoring messages from greylisting
+ * [Feature] Implement lua history in controller
+ * [Feature] Implement redis history querying
+ * [Feature] Preliminary implementation of redis history plugin
+ * [Feature] Support using request headers in settings
+ * [Fix] Change default template to deal with non-ASCII characters
+ * [Fix] Deal with lists of maps in whitelist module
+ * [Fix] DKIM signing: use domain-specific signing key
+ * [Fix] Do not reallocate completed zstd buffer
+ * [Fix] Do not use local_addrs in proxy
+ * [Fix] Fix crash when resolver is undefined
+ * [Fix] Fix double free when closing lua_tcp connections
+ * [Fix] Fix for lua 5.3
+ * [Fix] Fix freeing of arrays iterators
+ * [Fix] Fix issue with task:get_symbol and symbols with no metric
+ * [Fix] Fix log line duplication in `rspamadm grep`
+ * [Fix] Fix memory corruption on termination
+ * [Fix] Fix out-of-bound access in base64 decode
+ * [Fix] Fix ratelimit + greylisting
+ * [Fix] Fix subject rewriting
+ * [Fix] Fix task:set_recipients function
+ * [Fix] Fix URI_COUNT_ODD rule
+ * [Fix] Follow the traditional symbols conventions in RCPT_COUNT rule
+ * [Fix] Greylist: Suppress greylist action for whitelisted hosts too
+ * [Fix] Metadata exporter: use rule-specific settings for emails
+ * [Fix] Properly set missing fields in exporter
+ * [Fix] Proxy: max_retries option
+ * [Fix] RCPT_COUNT fixes
+ * [Fix] Rework HAS_X_PRIO rule to match symbols conventions
+ * [Fix] Update issues in ac-trie
+ * [Fix] Use optimised base64 decoding in DKIM
+ * [WebUI] Add preliminary v2 history parser
+ * [WebUI] Allow different history parsers
+ * [WebUI] Display symbols
+ * [WebUI] Rework history v2 function
+
+1.5.2:
+ * [Conf] Add default config for spamassasssin plugin
+ * [Conf] Add default configuration for antivirus module
+ * [Conf] Add dkim signing docs
+ * [Conf] Add mx_check default config
+ * [Conf] Add replies config
+ * [Conf] Add trie default config
+ * [Feature] Add heuristic to find text parts in files
+ * [Feature] Add rule to detect broken content type
+ * [Feature] Allow to extract CTE in Lua API
+ * [Feature] Allow to set from address for a lua_task
+ * [Feature] Allow to set recipients of a task from Lua
+ * [Feature] Enchance text_part:get_content method
+ * [Feature] Remove + aliases from emails
+ * [Feature] Support rmilter block and dkim signature in CGP helper
+ * [Feature] Support running event loop from Lua
+ * [Fix] Antivirus: use scanner-specific redis prefix
+ * [Fix] Couple of fixes for DKIM signing module
+ * [Fix] Distinguish missing and broken mandatory headers
+ * [Fix] Do more heuristical detection for missing CTE
+ * [Fix] Do not resort cache on each check
+ * [Fix] Fix CGP escaping
+ * [Fix] Fix MISSING_MIME_VERSION rule for plain messages
+ * [Fix] Fix parsing of cte in expressions
+ * [Fix] Fix partial matches in rspamadm grep
+ * [Fix] Fix setting class on style field
+ * [WebUI] Auto-switch Throughput units to `msg/min` for very low rate
+ * [WebUI] Update D3Evolution to 0.0.2
+
+1.5.1:
+ * [CritFix] Fix processing of stop_patterns with `\0` character
+ * [CritFix] Fix setting of raw key for signing
+ * [Fix] Fix lua exports from plugins during reload
+ * [Fix] Fix prefilters action scores
+ * [Fix] Fix symbols processing order
+ * [Minor] Help cmake find gthread
+ * [Minor] Some cmake fixes
+
+1.5.0:
+ * [Conf] Add configurations for asn, clickhouse and dcc
+ * [Conf] Add default config for url redirector plugin
+ * [Conf] Add the default config for greylist module
+ * [Conf] Allow to edit all local maps from WebUI by default
+ * [CritFix] Deal with absent headers in DKIM
+ * [CritFix] Do not trust remote shingles count
+ * [CritFix] Fix bad memory leak in TLS certificates validation
+ * [CritFix] Fix critical memory issues with radix maps
+ * [CritFix] Fix descriptors leak on reload
+ * [CritFix] Fix headers selection in DKIM verification
+ * [CritFix] Fix parsing of boundaries that end with `--`
+ * [CritFix] Repair PTR_ARRAY_FOREACH macro
+ * [Feature] Add CORS support to the controller
+ * [Feature] Add FROM_NAME_EXCESS_SPACE rule
+ * [Feature] Add REPLYTO_EMAIL_HAS_TITLE rule
+ * [Feature] Add `caseless_hash` method to `lua_util`
+ * [Feature] Add `rip` keyword to ratelimit module
+ * [Feature] Add a simple benchmark for content type parsing
+ * [Feature] Add boundaries parsing in content type
+ * [Feature] Add charset detection for text parts
+ * [Feature] Add content disposition parser
+ * [Feature] Add fallback if too many updates are failing
+ * [Feature] Add function to convert struct tm to time using timezone
+ * [Feature] Add function to normalize HTTP paths
+ * [Feature] Add fuzzy collection plugin
+ * [Feature] Add fuzzy logic for images
+ * [Feature] Add gmime parser to mime_tool
+ * [Feature] Add heuristic to detect broken messages
+ * [Feature] Add heuristic to find displayed URLs
+ * [Feature] Add heuristic to process broken email addresses
+ * [Feature] Add images normalization
+ * [Feature] Add mechanism for disabling composites (Fixes #1270)
+ * [Feature] Add method to create regexp from a glob pattern
+ * [Feature] Add mime encoding manipulation routines
+ * [Feature] Add mime tool to explore messages
+ * [Feature] Add more meta tokens from received headers
+ * [Feature] Add neighbours option to support Rspamd cluster in WebUI
+ * [Feature] Add new function to parse mime addresses
+ * [Feature] Add new methods for lua_tcp
+ * [Feature] Add own headers decoding routine
+ * [Feature] Add own routine to generate a message id
+ * [Feature] Add parser for SMTP date
+ * [Feature] Add per-task lua cache to reuse 'heavy' objects
+ * [Feature] Add plugins list path in WebUI
+ * [Feature] Add preliminary multipart support
+ * [Feature] Add preliminary version of DKIM signing module
+ * [Feature] Add profiling support in client output
+ * [Feature] Add rfc2047 grammar
+ * [Feature] Add rfc2047 variant for QP decoding
+ * [Feature] Add rmilter_headers module (Fixes #1227)
+ * [Feature] Add sse42 version of base64 decoding
+ * [Feature] Add ssse3 and avx2 base64 decoders
+ * [Feature] Add support of libgd
+ * [Feature] Add the preliminary version of redirects resolver in Lua
+ * [Feature] Add ucl_object_iterate_full function
+ * [Feature] Add url encoding function
+ * [Feature] Allow SOA requests in lua dns
+ * [Feature] Allow custom parse types in lua ucl
+ * [Feature] Allow plugins to register webui handlers
+ * [Feature] Allow to add options explicitly to symbols
+ * [Feature] Allow to call a callback when symbol frequency is on peak
+ * [Feature] Allow to call redirector script from SURBL
+ * [Feature] Allow to create variable length dkim keys
+ * [Feature] Allow to have module specific options for Redis in plugins
+ * [Feature] Allow to pass sign key directly from Lua
+ * [Feature] Allow to register configuration docs from Lua API
+ * [Feature] Allow to return options as a table
+ * [Feature] Allow to set peak callbacks from Lua
+ * [Feature] Allow to specify custom method for a message
+ * [Feature] Allow to store dkim keys in Redis
+ * [Feature] Allow to store messages in files
+ * [Feature] Apply DCT using AAN for fuzzy signature
+ * [Feature] Avira SAVAPI support
+ * [Feature] Cache and simplify DCT and jpeg decode
+ * [Feature] Cache libicu converters
+ * [Feature] Detect URLs with suspicious omographs
+ * [Feature] Do not increase score for duplicate options
+ * [Feature] Do not trust CTE, check base64 and qp strictly
+ * [Feature] Dynamic reputation in URL reputation plugin
+ * [Feature] Extend redis lock when learning spawned
+ * [Feature] Filter non-utf chars from all decoded headers
+ * [Feature] Fix phishing detection for IDNA urls
+ * [Feature] Ignore bad symbols on base64 decoding
+ * [Feature] Ignore too wide elements in SPF
+ * [Feature] Implement fuzzy collection mode
+ * [Feature] Implement helo maps in multimap
+ * [Feature] Implement human readable buckets configuration
+ * [Feature] Implement min-hash shingles for DCT data from images
+ * [Feature] Implement new algorithm for fuzzy hashes of images
+ * [Feature] Implement new unicode normalizer
+ * [Feature] Implement quoted printable decoding
+ * [Feature] Implement received headers flags
+ * [Feature] Implement rspamdgrep tool
+ * [Feature] Implement sane checksum for config file
+ * [Feature] Implement url tags concept
+ * [Feature] Improve detection of omographs using libicu
+ * [Feature] Improve url redirector module
+ * [Feature] Multimap: Received header processing
+ * [Feature] Multiple improvements in the maps
+ * [Feature] New URL filters in multimap
+ * [Feature] Plugin to force actions on selected symbols
+ * [Feature] RBL module: support hashing for emails and helo RBL
+ * [Feature] Reuse URL tags in SURBL module
+ * [Feature] Rework RRD ds count, add conversion path
+ * [Feature] Rework surbl module to avoid extra redirector calls
+ * [Feature] Send config id to the WebUI
+ * [Feature] Simplify HTTPCrypt client support
+ * [Feature] Skip processing for large images
+ * [Feature] Start collection only mode implementation for fuzzy storage
+ * [Feature] Start import of the optimized base64 decode
+ * [Feature] Store all received headers in lua
+ * [Feature] Store relational order of all headers in a message
+ * [Feature] Support DKIM signing in Lua plugins
+ * [Feature] Support HTTPCrypt client in lua_http
+ * [Feature] Support setting SMTP message in multimap
+ * [Feature] Support setting metric subject from Lua
+ * [Feature] Support setting subject in force actions module
+ * [Feature] Treat v6 mapped addresses as v4 addresses
+ * [Feature] URL reputation plugin
+ * [Feature] Use Redis instead of memcached in URLs redirector
+ * [Feature] Use Rspamd rfc2047 decoder instead of gmime one
+ * [Feature] Use a different normalization for fuzzy images
+ * [Feature] Use normalized images in fuzzy hashes
+ * [Feature] Use own code for parsing of date
+ * [Feature] Use shingles for images fuzzying
+ * [Feature] Use t1ha for hashes, allow inlining
+ * [Feature] Use t1ha instead of metrohash and xxhash32
+ * [Feature] Various new features in metadata exporter module
+ * [Feature] rmilter_headers: authentication-results (#78)
+ * [Fix] Add additional check to mark redis connection inactive
+ * [Fix] Add packed attribute for protocol structure
+ * [Fix] Adopt OMOGRAPH_URL rule
+ * [Fix] Allow static maps
+ * [Fix] Allow to disable classifiers checks using settings and conditions
+ * [Fix] Another try to fix 0 length maps
+ * [Fix] Another try to fix corruption during maps reload
+ * [Fix] Another try to fix descriptors leak
+ * [Fix] Another try to fix reload and logger
+ * [Fix] Antivirus module: register virtual symbols for patterns
+ * [Fix] Avoid extensive reallocs
+ * [Fix] Avoid mempool leak in SA plugin on reload
+ * [Fix] Avoid race condition on saving cache and reload
+ * [Fix] Avoid reusing g_error (Fixes #1262)
+ * [Fix] Break pool connection on fatal redis errors
+ * [Fix] Check for NaN properly
+ * [Fix] Couple of fixes for date parsing
+ * [Fix] Date header timezone adjustments (#1279)
+ * [Fix] Deal with EOF properly
+ * [Fix] Decode filename in content disposition
+ * [Fix] Disable fuzzy images by default
+ * [Fix] Disable zero-copy mode for text parts to avoid crashes
+ * [Fix] Do not destroy session when not all finish scripts are done
+ * [Fix] Do not greyscale images
+ * [Fix] Do not leave parent-less workers processes on fatal errors
+ * [Fix] Do not lowercase Content-Disposition to perform decoding
+ * [Fix] Do not penalize characters just after numeric prefix
+ * [Fix] Do not refork workers that are intended to die
+ * [Fix] Do not set pre-result and update records for no `Queue-ID` messages
+ * [Fix] Do not skip post-filters when pre-filters have set some results
+ * [Fix] Do not stop symbols planning if async events are pending
+ * [Fix] Do not try to set keys for unencrypted requests in proxy
+ * [Fix] Encode URLs according to rfc3986
+ * [Fix] Encode URLs before sending them to the protocol
+ * [Fix] Filter bad characters from message id
+ * [Fix] Fix CTE detection heuristic
+ * [Fix] Fix Content-Type in HTTP requests
+ * [Fix] Fix IDN eslds phishing checks
+ * [Fix] Fix adding maps from config in Lua
+ * [Fix] Fix another reload memory issue
+ * [Fix] Fix argument returned on redis backend errors
+ * [Fix] Fix assertion in graph handling
+ * [Fix] Fix body trie matching
+ * [Fix] Fix build
+ * [Fix] Fix byte array expansion during toutf8 conversion
+ * [Fix] Fix charset normalisation
+ * [Fix] Fix checking of DKIM bodies that needs just `\n` to be added
+ * [Fix] Fix couple of cornercases with email addresses
+ * [Fix] Fix couple of issues
+ * [Fix] Fix dependencies tracking for callback symbols
+ * [Fix] Fix detection of jpeg size
+ * [Fix] Fix errors handling in fuzzy backend initialization
+ * [Fix] Fix fuzzy hashes count
+ * [Fix] Fix globbing and convert lists to arrays in fuzzy_check
+ * [Fix] Fix heuristical CTE detection for QP encoding
+ * [Fix] Fix ignoring of bad text parts
+ * [Fix] Fix indexes in array access, interleave loop
+ * [Fix] Fix int64 -> double conversion
+ * [Fix] Fix invalid memory access on reload
+ * [Fix] Fix issues with empty updates
+ * [Fix] Fix issues with quoted-printable encoding
+ * [Fix] Fix keys names
+ * [Fix] Fix lots of issues in mime parser code
+ * [Fix] Fix lua maps load
+ * [Fix] Fix macro name
+ * [Fix] Fix mas group score calculations
+ * [Fix] Fix matching of the same patterns from different tries
+ * [Fix] Fix memory corruprtion and leak
+ * [Fix] Fix memory leak in HTTP maps
+ * [Fix] Fix memory leak in expression destroying
+ * [Fix] Fix memory leak in parsing of mime names
+ * [Fix] Fix memory leak in safe ucl iterators
+ * [Fix] Fix memory leak on reload in plugins
+ * [Fix] Fix modules reconfigure on reload
+ * [Fix] Fix monitored setup fro URLBLs with IP addresses
+ * [Fix] Fix name of var
+ * [Fix] Fix new rrd updates
+ * [Fix] Fix out of bounds access
+ * [Fix] Fix parsing messages with no body
+ * [Fix] Fix parsing of '=' character in headers
+ * [Fix] Fix parsing of messages with no content type
+ * [Fix] Fix plugins callbacks in webui
+ * [Fix] Fix possible memory corruption in redis pool
+ * [Fix] Fix probability calculations for fuzzy redis backend
+ * [Fix] Fix processing errors in lua_tcp
+ * [Fix] Fix processing of emails with name only
+ * [Fix] Fix processing of non-multipart messages
+ * [Fix] Fix processing of parts with no valid content type
+ * [Fix] Fix race condition in SIGUSR2 handler
+ * [Fix] Fix redis options parsing when no redis servers are defined
+ * [Fix] Fix reload and hyperscan ready event
+ * [Fix] Fix reload memory issue
+ * [Fix] Fix rra_ptr conversion
+ * [Fix] Fix rrd file conversion
+ * [Fix] Fix setting of content-type attributes
+ * [Fix] Fix signing headers creation in DKIM
+ * [Fix] Fix stddev calculations
+ * [Fix] Fix surbl plugin to work with composite maps
+ * [Fix] Fix timezones parsing
+ * [Fix] Fix tokens usage
+ * [Fix] Fix urls and emails hashes
+ * [Fix] Fix usage of unsafe ucl iterators
+ * [Fix] Fix work with broken utf8 tokens
+ * [Fix] Fix writing of user to roll history
+ * [Fix] Forgotten worker
+ * [Fix] Further memory leaks fixes
+ * [Fix] Ignore lua metatokens in bayes for now
+ * [Fix] Improve OMOGRAPH_URL rule
+ * [Fix] Lua IP from string should be invalid if parsing failed
+ * [Fix] Miltiple fixes to new lua_tcp, add debugging
+ * [Fix] More fixes for iterators cleanup
+ * [Fix] More fixes to logger initialization
+ * [Fix] More heuristic fixes for phishing detection
+ * [Fix] More leaks eliminated
+ * [Fix] More leaks...
+ * [Fix] More random fixes for reload...
+ * [Fix] Multimap: Fixes for email filters
+ * [Fix] Multiple fixes for fann module
+ * [Fix] Multiple memory corruption fixes
+ * [Fix] Normalize path in HTTP router
+ * [Fix] Plug memory leak
+ * [Fix] Plug memory leak in adding radix trees
+ * [Fix] Plug memory leak in configuration parser
+ * [Fix] Plug memory leak in expressions parsing during reload
+ * [Fix] Plug memory leak in learning fuzzy storage
+ * [Fix] Plug memory leak in lua_tcp
+ * [Fix] Plug reload leaks
+ * [Fix] Plug termination memory leaks
+ * [Fix] Really increase lock lifetime
+ * [Fix] Replies module: fix symbol weight
+ * [Fix] Restore content type params related functions
+ * [Fix] Set task's subject from mime subject
+ * [Fix] Sigh, one more reload leak
+ * [Fix] Simplify images shingles
+ * [Fix] Some more memory issues are fixed
+ * [Fix] Stop hardcoding of lua in C
+ * [Fix] Stop processing of bad parts as text parts
+ * [Fix] Strictly filter bad characters when emittin json
+ * [Fix] Strings returned from lua are ephemeral
+ * [Fix] Support unix sockets for lua redis
+ * [Fix] Try to fix issues with reloading config
+ * [Fix] Try to fix race condition in redis_pool
+ * [Fix] Use checksum to avoid intersection between different ANNs
+ * [Fix] Use rspamd hashes in embedded ucl
+ * [Fix] Use sane default rewrite subject (*** SPAM *** %s)
+ * [Fix] Various collection mode fixes
+ * [Fix] Various fixes to mime parser
+ * [Fix] Various reload leak fixing
+ * [Fix] Whitelist certain extensions from archive checks
+ * [Rework] Add preliminary implementation of the mime parser
+ * [Rework] Adopt code for the new options
+ * [Rework] Change logger setup interface
+ * [Rework] Composite configuration (#1270)
+ * [Rework] Finally remove gmime dependency from Rspamd
+ * [Rework] Further fixes to symbols frequencies
+ * [Rework] Implement content type parser for mime
+ * [Rework] Kill all InternetAddressList usages
+ * [Rework] Multiple fixes for symbols cache statistics
+ * [Rework] Refactor struct names
+ * [Rework] Rework images fuzzy hashes algorithm
+ * [Rework] Rework lua_tcp to allow TCP dialog
+ * [Rework] Start massive rework to get rid of gmime
+ * [Rework] Start new approach for multiparts parsing
+ * [Rework] Start rework of mime addresses
+ * [Rework] Start rework of symbols cache updates
+ * [Rework] Start switching to libicu
+ * [Rework] Use a special structure for stats tokens
+ * [Rework] Use hash tables for symbols options
+ * [Rework] Use libicu instead of iconv for conversions
+ * [Rework] Use new scheme to parse mime parts
+ * [WebUI] Add Access-Control-Allow-Origin for cluster management
+ * [WebUI] Add Throughput graph autorefreshing (#820)
+ * [WebUI] Add Visibility.js library
+ * [WebUI] Add basic cluster support to Throughput tab
+ * [WebUI] Add graph legend entries for new DSes
+ * [WebUI] Add graph tab
+ * [WebUI] Add neighbours RRD data consolidation
+ * [WebUI] Add preliminary save symbols clustering
+ * [WebUI] Add server selector to navbar
+ * [WebUI] Add soft reject to auth stats
+ * [WebUI] Add summary to the Throughput tab
+ * [WebUI] Allow to save maps on the cluster
+ * [WebUI] Avoid extra graph redraw and alerts glitching
+ * [WebUI] Be more generous with AJAX timeout
+ * [WebUI] Disable error ring loading in `read only` mode
+ * [WebUI] Enclose table header cells with `tr`s
+ * [WebUI] Finish interface rework
+ * [WebUI] Fix RRD summary pie chart position
+ * [WebUI] Fix `All SERVERS` graph fot just one available server
+ * [WebUI] Fix case when no cluster is defined
+ * [WebUI] Fix compatibility with non-ES6 compliant browsers
+ * [WebUI] Fix config ID
+ * [WebUI] Fix configuration page partially
+ * [WebUI] Fix disabled state
+ * [WebUI] Fix graph dataset selector initialization
+ * [WebUI] Fix graph selectors state resetting
+ * [WebUI] Fix mouse events on throughput summary table area
+ * [WebUI] Fix multiple JS issues
+ * [WebUI] Fix pie chart displaying
+ * [WebUI] Fix read only
+ * [WebUI] Fix read only2
+ * [WebUI] Fix retarded datatables
+ * [WebUI] Fix soft reject in pie chart
+ * [WebUI] Fix stat widgets timers multiplication on `Refresh` click
+ * [WebUI] Fix symbols config
+ * [WebUI] Fix various errors with login form
+ * [WebUI] Further fixes
+ * [WebUI] Hide learning tab in read-only mode
+ * [WebUI] Initial clusters support
+ * [WebUI] Make legend entry colours more contrast
+ * [WebUI] Move configuration tab to a separate module
+ * [WebUI] Move history tab
+ * [WebUI] Move symbols config as well
+ * [WebUI] New sec to time function
+ * [WebUI] Prevent multiple clicks on `Refresh`
+ * [WebUI] RRD summary: Hide inner labels of tiny pie sectors
+ * [WebUI] RRD summary: Respect undefined values
+ * [WebUI] Reduce font size of graph's legend
+ * [WebUI] Remove orphaned font duplicates
+ * [WebUI] Remove unused code
+ * [WebUI] Replace spinner with animated glyphicon
+ * [WebUI] Reset refresh timer on server switching
+ * [WebUI] Rework interface to use requirejs
+ * [WebUI] Rework neighbours query function
+ * [WebUI] Separate attributes by space
+ * [WebUI] Set focus to password field (#1230)
+ * [WebUI] Simplify neighbours table populating
+ * [WebUI] Start rework of modules
+ * [WebUI] Stop stats refreshing if the page is hidden
+ * [WebUI] Turn d3pie's stuff into a reusable function,
+ * [WebUI] Unify send data functions
+ * [WebUI] Update D3Evolution to 0.0.1
+ * [WebUI] Update d3.js
+ * [WebUI] Update datatables to work with the requirejs
+ * [WebUI] Use unified tab click event handler,
+ * [WebUI] clusters for the chart
+ * [WebUI] fix uptime
+
+1.4.2:
+ * [CritFix] Deal with absent headers in DKIM
+ * [CritFix] Do not trust remote shingles count
+ * [CritFix] Fix headers selection in DKIM verification
+ * [Feature] Add EXT_CSS rule
+ * [Feature] Add toggle for disabling SURBLs
+ * [Feature] Extend redis lock when learning spawned
+ * [Feature] Parse <link> HTML tags
+ * [Fix] Avoid reusing g_error (Fixes #1262)
+ * [Fix] Do not reset loaded ANN when learning is requested
+ * [Fix] Fix another issue with external deps in SA
+ * [Fix] Fix body trie matching
+ * [Fix] Fix checking of DKIM bodies that needs just `\n` to be added
+ * [Fix] Fix fuzzy hashes count
+ * [Fix] Fix keys names
+ * [Fix] Fix length calculations for url encoded urls
+ * [Fix] Fix matching of the same patterns from different tries
+ * [Fix] Fix name of var
+ * [Fix] Fix parsing of URLs with spaces and other bad chars
+ * [Fix] Fix probability calculations for fuzzy redis backend
+ * [Fix] Fix signing headers creation in DKIM
+ * [Fix] Plug memory leak
+ * [Fix] Really fix chained SA dependencies
+ * [Fix] Really increase lock lifetime
+ * [Fix] Use checksum to avoid intersection between different ANNs
+ * [Fix] Use rspamd hashes in embedded ucl
+ * [Fix] Yet another change for testing external deps
+
+1.4.1:
+ * [Feature] ASN support in Clickhouse module
+ * [Feature] Add clickhouse plugin
+ * [Feature] Add generic tool to add universal maps for lua modules
+ * [Feature] Add logger.debugm to debug lua modules
+ * [Feature] Allow to register metrics symbols using register_symbol
+ * [Feature] Allow to specify prefix for fann_redis
+ * [Feature] Clickhouse: support different masks for IPv4/IPv6
+ * [Feature] Support forcing action in antivirus plugin
+ * [Fix] Add handling of regexp maps
+ * [Fix] Allow backslashes in http urls
+ * [Fix] Avoid mapping of empty files
+ * [Fix] Do not load tld file to speed up rspamadm
+ * [Fix] Do not resolve numeric IP addresses due to ipv6 insanity
+ * [Fix] Filter incorrect training data
+ * [Fix] Fix Fuzzyconvert tool when password or DB is given
+ * [Fix] Fix build with custom glib/gmime
+ * [Fix] Fix converting of learn count from sqlite to redis
+ * [Fix] Fix crashes with invalid received and task:set_from_ip
+ * [Fix] Fix external dependencies for SA module
+ * [Fix] Fix fann_redis when number of scores has been changed
+ * [Fix] Fix hyperscan usage for non compatible platforms
+ * [Fix] Fix loading of maps from UCL objects
+ * [Fix] Fix memory leak for task-less redis requests
+ * [Fix] Fix mid module with new maps syntax
+ * [Fix] Fix parsing of URLs with username
+ * [Fix] Fix re cache initialisation
+ * [Fix] Fix replacements to sanitize '%' character
+ * [Fix] Fix set and regexp like static maps
+ * [Fix] Fix some issues in redis settings
+ * [Fix] Fix static IP maps
+ * [Fix] Fix total learns counter for redis stats
+ * [Fix] Fix usage of config during reload
+ * [Fix] Fix various warnings and issues
+ * [Fix] Invalidate ANN if training data is incorrect
+ * [Fix] Miltiple fixes to fann_redis module
+ * [Fix] More fixes for URLs with backslashes
+ * [Fix] Properly get options for ip_score module
+ * [Fix] Relax requirements for Received as gmail cannot RFC
+ * [Fix] Remove or fix hyperscan incompatible regexps
+ * [Fix] Settings: correctly read redis config
+ * [Rework] Rework lua logger interface slightly
+ * [Rework] Use new maps add function
+
+1.4.0:
+ * [CritFix] Add guards for inactive redis connections
+ * [CritFix] Another fix for proxying files using rspamd_proxy
+ * [CritFix] Cleanup inactive redis connections
+ * [CritFix] Do not sometimes try to exec posfilters before classification
+ * [CritFix] Fix application of IPv6 mask
+ * [CritFix] Fix chunked encoding when reading messages
+ * [CritFix] Fix file mode for rspamd_proxy
+ * [CritFix] Fix hyperscan compilation on regexp change
+ * [CritFix] Fix issue with finding of end of lines pointers
+ * [CritFix] Fix iteration over headers array (introduced in 1.4)
+ * [CritFix] Fix processing of learned tokens count for redis backend
+ * [CritFix] Fix race condition in checking of cached maps
+ * [CritFix] Fix workers scripts by sharing workers configs
+ * [CritFix] Introduce raw content to text parts
+ * [CritFix] Plug memory leak and potential memory corruption
+ * [Feature] Adaptive ratelimits
+ * [Feature] Add ASN -> rbldnsd script for asn.rspamd.com
+ * [Feature] Add DMARC_NA symbol
+ * [Feature] Add F-Prot support to antivirus module
+ * [Feature] Add HTTP backend to metadata exporter
+ * [Feature] Add Lua API module for monitored objects
+ * [Feature] Add R_DKIM_NA / R_SPF_NA / AUTH_NA symbols
+ * [Feature] Add R_DKIM_PERMFAIL symbol
+ * [Feature] Add R_SPF_PERMFAIL symbol
+ * [Feature] Add Sophos antivirus support
+ * [Feature] Add ZSTD compression to Lua API
+ * [Feature] Add `mid` Lua module
+ * [Feature] Add `one_param` flag for metric symbols
+ * [Feature] Add a generic lua classifier
+ * [Feature] Add a very basic interface to access workers data from on_load
+ * [Feature] Add ability to delete a hash by its data to fuzzy_check plugin
+ * [Feature] Add ability to enable/disable symbols via dynamic_conf
+ * [Feature] Add ability to lookup settings by key
+ * [Feature] Add common way to disable Lua modules
+ * [Feature] Add compression support to rspamd client
+ * [Feature] Add condition to do antiviral check
+ * [Feature] Add configuration for lua classifiers
+ * [Feature] Add configuration knobs for the errors circular buffer
+ * [Feature] Add decompression support in rspamd client
+ * [Feature] Add errors exporter to the controller
+ * [Feature] Add expected value for monitored DNS resources
+ * [Feature] Add exporter from error ringbuf to ucl
+ * [Feature] Add extended version for fann creation function
+ * [Feature] Add ffi friendly version of process_regexp function
+ * [Feature] Add frequency and time display to webui
+ * [Feature] Add fuzzy_delhash command to rspamc client
+ * [Feature] Add implementation of redis connections pool
+ * [Feature] Add latency and offline time monitoring
+ * [Feature] Add learning support for lua classifiers
+ * [Feature] Add max-size and timeout options to CGP helper
+ * [Feature] Add method to enable/disable symbols in config
+ * [Feature] Add methods to get metric's actions and symbols from Lua
+ * [Feature] Add mmap support to lua_text
+ * [Feature] Add monitored object for surbl plugin
+ * [Feature] Add more exceptions to surbl whitelist
+ * [Feature] Add more meta-tokens to bayes
+ * [Feature] Add neural net classifier to fann_scores module
+ * [Feature] Add neural net serialization/deserialization
+ * [Feature] Add new dynamic conf module
+ * [Feature] Add periodic events support for lua_config
+ * [Feature] Add plugin to check MX'es for the sender's domain
+ * [Feature] Add preliminary monitored module
+ * [Feature] Add preliminary support of dynamic conf updates in Redis
+ * [Feature] Add preliminary version of clamav plugin
+ * [Feature] Add redis cache to asn module
+ * [Feature] Add replies compression
+ * [Feature] Add spamhaus DROP dnsbl
+ * [Feature] Add support for dictionary in client compression
+ * [Feature] Add support for fuzzy learn and unlearn from lua
+ * [Feature] Add support for input encryption
+ * [Feature] Add support of min_learns to neural net classifier
+ * [Feature] Add termination callbacks for workers
+ * [Feature] Add user-agent for rspamc
+ * [Feature] Add utility to perform classifier tests
+ * [Feature] Add zstd compression library
+ * [Feature] Allow HTTPS requests in lua_http
+ * [Feature] Allow conditions for pre and postfilters
+ * [Feature] Allow custom functions for ratelimits
+ * [Feature] Allow for excluding messages from AV scanning based on size
+ * [Feature] Allow for getting worker stats from Lua
+ * [Feature] Allow getting task UID from Lua
+ * [Feature] Allow parsing of mailbox messages from the commandline
+ * [Feature] Allow plugins to publish their lua API via rspamd_plugins
+ * [Feature] Allow to compare other systems with Rspamd
+ * [Feature] Allow to execute Lua scripts by controller
+ * [Feature] Allow to have a function to set custom greylist message
+ * [Feature] Allow to iterate over multiple tags
+ * [Feature] Allow to pass extra data from plugins to log helper
+ * [Feature] Allow to plan new periodics at different time
+ * [Feature] Allow to reset hashes
+ * [Feature] Allow to run rspamadm lua just as a lua interpreter
+ * [Feature] Allow to store settings in redis
+ * [Feature] Allow to update dynamic conf in Redis
+ * [Feature] Allow to use dictionaries for compression
+ * [Feature] Allow to use md5, sha1, sha256, sha384 and sha512 hashes in Lua
+ * [Feature] Allow whitelisting by IP for greylisting plugin
+ * [Feature] Antivirus: Support whitelists & pattern-matching sig names
+ * [Feature] Backport pack/unpack routines from Lua 5.3
+ * [Feature] Check settings with equal priopities in alphabetical order
+ * [Feature] Compress neural net in redis
+ * [Feature] Consider more tags when doing WHITE_ON_WHITE rule
+ * [Feature] Descriptive options for DMARC failure symbols
+ * [Feature] Descriptive options for RBL symbols
+ * [Feature] Enable configuration for monitored objects
+ * [Feature] Execute on_load scripts with ev_base ready
+ * [Feature] Fann scores now uses metadata from a message
+ * [Feature] Implement FANN threaded learning
+ * [Feature] Implement classifying for lua classifiers
+ * [Feature] Implement finish scripts for worker processes
+ * [Feature] Implement monitoring for DNS resources
+ * [Feature] Implement real priorities for pre and post filters
+ * [Feature] Insert two symbols: FANN_HAM and FANN_SPAM instead of one
+ * [Feature] Module to push metadata/messages to redis pubsub
+ * [Feature] Monitor RBL records
+ * [Feature] Move fann_classifier to a separate plugin
+ * [Feature] Normalize all ANN inputs
+ * [Feature] Preliminary version of metric exporter module
+ * [Feature] Preserve decompression context between tasks
+ * [Feature] Ratelimit: Support dynamic bucket size/leak rate
+ * [Feature] Relax FORGED_RECIPIENTS: allow senders to BCC themselves
+ * [Feature] Remove symbols weights on composites processing
+ * [Feature] Return symbol scores when getting resulting symbols
+ * [Feature] Rework lua tcp module
+ * [Feature] Rule to detect some obvious X-PHP-Originating-Script forgeries
+ * [Feature] Rule to identify some X-PHP-Script forgeries
+ * [Feature] Rules for scoring Google Message-ID fixes
+ * [Feature] Send hashes values to reply
+ * [Feature] Set expire for dmarc reports
+ * [Feature] Stop using cymru zone as it is unstable
+ * [Feature] Stop using of GLists for headers, improve performance
+ * [Feature] Store `for` in task:get_received_headers
+ * [Feature] Store `for` part in received headers
+ * [Feature] Store enabled flag for webui session
+ * [Feature] Store error messages in ring buffer
+ * [Feature] Support compressed maps
+ * [Feature] Support excluding selected users from ratelimits
+ * [Feature] Support looking up NS records in lua_dns
+ * [Feature] Support modern style SURBL configuration
+ * [Feature] Support multiple hashes in delhash path
+ * [Feature] Support new messages in rspamc
+ * [Feature] Support requests without reads in lua_tcp
+ * [Feature] Support setting task message from Lua
+ * [Feature] Track visibility of HTML elements
+ * [Feature] Try to add CRLF when checking DKIM
+ * [Feature] Try to guess line endings when folding headers
+ * [Feature] Try to improve normalization function for bayes
+ * [Feature] Use FFI to optimize SA module
+ * [Feature] Use length based arguments for redis, allow lua_text as arg
+ * [Feature] Use more layers for fann and another normalization
+ * [Feature] User-defined ratelimits
+ * [Feature] Utility to convert fuzzy storage from sqlite to redis
+ * [Feature] Yield DMARC_DNSFAIL on lookup failure
+ * [Fix] Adopt fuzzy storage for flexible backends
+ * [Fix] Allow plain IP addresses in Rspamd maps
+ * [Fix] Another fix for brain-damaged hiredis
+ * [Fix] Another fix for rdns write errors
+ * [Fix] Another fix for rdns_make_request_full invocation
+ * [Fix] Another fix in DKIM canonicalization
+ * [Fix] Another memory leak plugged
+ * [Fix] Another try to deal with posix idiotizm
+ * [Fix] Another try to fix RDNS events processing logic
+ * [Fix] Avoid double frees in HEAD requests
+ * [Fix] Avoid extra symbols for RBLs
+ * [Fix] Banish table.maxn from Lua parts
+ * [Fix] Check for socket error before connection in lua_tcp
+ * [Fix] Correctly propagate redis timeouts to Lua
+ * [Fix] Do not add extra newline in MIME mode
+ * [Fix] Do not be cheated by system hiredis
+ * [Fix] Do not classify when a message has not enough tokens
+ * [Fix] Do not crash on redis errors
+ * [Fix] Do not distinguish NXDOMAIN and NOREC for monitored
+ * [Fix] Do not replan retransmits if merely one server is defined
+ * [Fix] Do not use headers to calculate messages digests
+ * [Fix] Don't force action in replies module for authenticated users/local networks
+ * [Fix] Explicitly ban default passwords in webui
+ * [Fix] Finally fix ambiguity between parsed and resolved spf elts
+ * [Fix] Fix 'decoded' value in task:get_header_full()
+ * [Fix] Fix DKIM calculations
+ * [Fix] Fix DKIM signing for messages with no newline at the end
+ * [Fix] Fix DNS request in monitored
+ * [Fix] Fix DNS write errors processing
+ * [Fix] Fix HTTP methods other than GET and POST
+ * [Fix] Fix PERMFAIL for v6/v4 ambiguities
+ * [Fix] Fix absurdic scores for HFILTER_URL_ONLY
+ * [Fix] Fix actions in rolling history
+ * [Fix] Fix actrie patterns
+ * [Fix] Fix applying of lua dynamic confg
+ * [Fix] Fix autolearning errors and redis cache
+ * [Fix] Fix bayes learn_condition
+ * [Fix] Fix build with the recent OpenSSL
+ * [Fix] Fix caching and compressed maps
+ * [Fix] Fix check plain text part
+ * [Fix] Fix crash on OpenBSD in `url_email_start`
+ * [Fix] Fix double free in SPF
+ * [Fix] Fix extraction of shingles from redis fuzzy storage
+ * [Fix] Fix false sharing for symbols in the cache
+ * [Fix] Fix float usage in util:get_time
+ * [Fix] Fix folding algorithm to deal with empty tokens
+ * [Fix] Fix format string
+ * [Fix] Fix format string usage in controller errors handling
+ * [Fix] Fix handling of '\0' in lua_tcp
+ * [Fix] Fix handling of HTTP HEAD methods
+ * [Fix] Fix hash creation
+ * [Fix] Fix hiredis stupidity
+ * [Fix] Fix implicit settings module settingsup
+ * [Fix] Fix interaction with lua GC to avoid craches
+ * [Fix] Fix ip_score module registration
+ * [Fix] Fix issue with empty messages and dkim
+ * [Fix] Fix issues with CGP helper
+ * [Fix] Fix issues with the recent SPF changes
+ * [Fix] Fix key name to load ANN correctly
+ * [Fix] Fix lua tcp module by saving `do_read` in callback data
+ * [Fix] Fix memory leak in client when using compression
+ * [Fix] Fix min_learns option
+ * [Fix] Fix on_finish scripts and async handlers
+ * [Fix] Fix options for SPF dnsfail symbol
+ * [Fix] Fix parsing includes and redirects in SPF
+ * [Fix] Fix parsing of lua comments with empty lines
+ * [Fix] Fix parsing of unquoted HTML attributes
+ * [Fix] Fix periodic events and redis
+ * [Fix] Fix processing of fuzzy learns from Lua
+ * [Fix] Fix processing of redirect in SPF includes
+ * [Fix] Fix processing of symbols when reject limit is reached
+ * [Fix] Fix refcounts when map is specified by IP
+ * [Fix] Fix rspamd{session} class in Lua API
+ * [Fix] Fix setting ratelimit key for 'ip' bucket
+ * [Fix] Fix some cases of TLD urls detector
+ * [Fix] Fix statconvert tool
+ * [Fix] Fix stats for backend-less classifiers
+ * [Fix] Fix training script for fann_redis
+ * [Fix] Fix variable in ann module
+ * [Fix] Fix various errors in lua dynamic conf plugin
+ * [Fix] Forget old ANN when max_usages is reached to avoid overtrain
+ * [Fix] Further canonicalization fixes
+ * [Fix] Further fixes for fann_redis prefixes
+ * [Fix] Handle failures for inactive pooled connections
+ * [Fix] Improve multimap info message
+ * [Fix] More fixes in ANN loading
+ * [Fix] More fixes to fann_redis
+ * [Fix] More issues in fann_redis
+ * [Fix] More spaces fix in DKIM signature
+ * [Fix] Multiple fixes to asn script, add IPv6 support
+ * [Fix] Multiple issues in fann_redis
+ * [Fix] No greylist rejected messages
+ * [Fix] One more attempt to fix lua_redis
+ * [Fix] One more check for readdir...
+ * [Fix] Params should be treated as a hash
+ * [Fix] Plug memory leak in regexp desctructor
+ * [Fix] Process headers only once
+ * [Fix] Properly handle nil values in ratelimit plugin
+ * [Fix] Really fix redis shingles check
+ * [Fix] Remove fann with incorrect layers count
+ * [Fix] Remove mentions of deleted include
+ * [Fix] Remove some incompatible functions
+ * [Fix] Settings: fix `authenticated` parameter (#886)
+ * [Fix] Skip MX check for authenticated users and local networks
+ * [Fix] Slightly fix ANN routines
+ * [Fix] Stop caching records with DNS failures
+ * [Fix] Treat all errors in redis_pool as fatal errors for a connection
+ * [Fix] Try avoid false-positives in HEADER_FORGED_MDN rule
+ * [Fix] Try to avoid race condition when using rrd
+ * [Fix] Try to reload redis scripts if they are missing
+ * [Fix] Unbreak once_received skipping for local networks
+ * [Fix] Unlock ANN on error
+ * [Fix] Use memmove for overlapping regions
+ * [Fix] Use real size instead of displayed for core limits
+ * [Fix] Use the correct macro to get the size of control
+ * [Fix] Various fixes for errors ringbuffer
+ * [Fix] Yield R_SPF_DNSFAIL if lookup of included record fails
+ * [Fix] mid: fix map initialization
+ * [Fix] mid: handle incorrect rgexps in the map
+ * [Rework] Add extract training data function to fann_redis
+ * [Rework] Add preliminary train tests
+ * [Rework] Add redis storage feature to fann_redis
+ * [Rework] Adopt fuzzy storage for abstract backend
+ * [Rework] Adopt plugins
+ * [Rework] First reiteration on fann scores
+ * [Rework] Implement loading/invalidating
+ * [Rework] Make lua_redis task agnostic
+ * [Rework] Make rspamd protocol messages useful
+ * [Rework] Massive removal of legacy code
+ * [Rework] More cleanup actions
+ * [Rework] Remove legacy code never used for classifiers
+ * [Rework] Remove outdated and unused lua_session module
+ * [Rework] Reorganize fuzzy backend structure
+ * [Rework] Reorganize the internal backend structure
+ * [Rework] Restore old fann_scores, move common parts
+ * [Rework] Rework and simplify rbl plugin
+ * [Rework] Rework parsing of DMARC records
+
+1.3.4:
+ * [Feature] ASN module; support matching ASN/country in multimap
+ * [Feature] Add SPF method in spf return result
+ * [Feature] Add Yandex and Mail.ru forwarding rules
+ * [Feature] Add mempool maps in multimap
+ * [Feature] Add rule for identifying mail sent by eval()'d PHP code
+ * [Feature] Add support of stub DNSSEC resolver to rdns
+ * [Feature] Add task:get_digest method
+ * [Feature] Allow for more fine-grained scoring for ip_score
+ * [Feature] Allow to get digest of a mime part from lua
+ * [Feature] Allow to print message digest in logs
+ * [Feature] Fold DKIM-Signature header
+ * [Feature] Implement encrypted logs
+ * [Feature] Log URLs encrypted if we have log encryption pubkey
+ * [Feature] Pass authenticated bit to lua
+ * [Feature] Read redis backend statistics configuration from global section
+ * [Feature] Show the exact value matched for multima symbols
+ * [Feature] Store task checksum
+ * [Fix] Avoid setting limits when required elements are missing
+ * [Fix] DMARC: Fix alignment checking for subdomains
+ * [Fix] DMARC: deal with missing and spurious spaces
+ * [Fix] Defer insertion of results in ip_score to avoid skewing stats
+ * [Fix] Disable DMARC for local/authorized mail
+ * [Fix] Fix handling of proxied headers in controller
+ * [Fix] Fix hex printing of strings
+ * [Fix] Fix issue with spaces in maps
+ * [Fix] Fix parsing of forwarded IP
+ * [Fix] Fix reload in some plugins and workers
+ * [Fix] Fix reloading on SIGHUP
+ * [Fix] Fix some border cases for DKIM canonicalization
+ * [Fix] Fix url maps
+ * [Fix] Make dnssec configurable option disabled by default for now
+ * [Fix] rspamadm statconvert: force db to be a string
+ * [Fix] rspamadm statconvert: use db/password for learn cache
+ * [Rework] Rework flags in rspamd logger
+
+1.3.3:
+ * [CritFix] Check hyperscan cache sanity before loading
+ * [CritFix] Fix setting of fuzzy keys (completely breaks fuzzy storage)
+ * [Feature] Add SARBL (sarbl.org) uribl
+ * [Feature] Add `--search-pattern` option to rspamd_stats
+ * [Feature] Add some sanity check for very long from/to log elements
+ * [Feature] Allow to create hashes from string in a single step
+ * [Feature] Fix order of pre and postfilters
+ * [Feature] Improve lua URLs API
+ * [Feature] Improve message about fuzzy rules
+ * [Feature] Pre-calculate blake2 digest for all parts
+ * [Feature] Print radix duplicate keys as IP addresses
+ * [Feature] Simple mechanism for disabling RBLs in local.d/rbl.conf
+ * [Feature] Use faster hash function for fuzzy storage
+ * [Feature] rspamd_stats: support log directory reading
+ * [Fix] Add sanity check for url filters
+ * [Fix] Do not show rmilter section as a fake metric in rspamc
+ * [Fix] Fix URL filters
+ * [Fix] Fix a stupid mistake in util.strequal_caseless
+ * [Fix] Fix blake2b hash of the string "rspamd"
+ * [Fix] Fix filename maps filter
+ * [Fix] Fix finding tld in util.get_tld
+ * [Fix] Fix multimap content filters
+ * [Fix] Fix returning boolean from Lua
+ * [Fix] Fix returning of REDIS_NIL
+ * [Fix] Try to deal with multiple workers terminated
+ * [Fix] Use forced DNS request when calling for lua_http
+ * [Rework] Rework multimap filters, add redis maps
+
+1.3.2:
+ * [Feature] Add a special symbol for SPF DNS errors: R_SPF_DNSFAIL
+ * [Feature] Add correlations report in fuzzy stats
+ * [Feature] Add experimental CGP integration
+ * [Feature] Add method to get urls length in a text part
+ * [Feature] Add new methods to lua_html to access HTML tags
+ * [Feature] Allow all types of symbols to be added via __newindex method
+ * [Feature] Allow to create settings for authenticated users
+ * [Feature] Allow to get block content for HTML tags
+ * [Feature] Improve DNS failures when dealing with SPF
+ * [Feature] Properly implement R_WHITE_ON_WHITE rule
+ * [Feature] Remove old ugly rules
+ * [Feature] Rspamc can now add dkim signature in mime mode
+ * [Feature] Store content length for HTML tags
+ * [Feature] Support reacher set of HTML colors
+ * [Feature] Try to avoid FP for low contrast fonts detection
+ * [Fix] Add missing HTML colors
+ * [Fix] Add spaces to dkim signature to allow folding
+ * [Fix] Avoid returning NaN as score on scan
+ * [Fix] Decode entitles in href parts
+ * [Fix] Do not cache SPF records with DNS errors
+ * [Fix] Do not crash on cyclic depends
+ * [Fix] Do not insert HELO/HOSTNAME unknown when they are not passed
+ * [Fix] Do not set absent hostname to "unknown"
+ * [Fix] Do not stress redis with KEYS command (#791)
+ * [Fix] Fix DMARC_BAD_POLICY symbol
+ * [Fix] Fix HFILTER_URL module
+ * [Fix] Fix HFILTER_URL_ONELINE rule
+ * [Fix] Fix buffering in CGP integration
+ * [Fix] Fix colors propagation from parent nodes
+ * [Fix] Fix confusing OpenSSL API usage of i2d_RSAPublicKey
+ * [Fix] Fix dependencies id sanity check
+ * [Fix] Fix folding for semicolon separated tokens
+ * [Fix] Fix largest possible TLD behaviour
+ * [Fix] Fix last token folding
+ * [Fix] Fix length calculations in white on white rule
+ * [Fix] Fix multiple request headers structure
+ * [Fix] Fix multiple values headers freeing
+ * [Fix] Fix parsing of background color
+ * [Fix] Fix printing from field in log_urls
+ * [Fix] Fix processing of last element of DMARC policies
+ * [Fix] Further fixes for HTML colors
+ * [Fix] Further fixes for multiple values headers
+ * [Fix] Further fixes for white on white rule
+ * [Fix] Further fixes in HTML tags parsing
+ * [Fix] Ignore content type/subtype case
+ * [Fix] Increase score of R_WHITE_ON_WHITE
+ * [Fix] Parse CGP envelope data
+ * [Fix] Propagate colors in HTML
+ * [Fix] Restore multiple values headers in protocol
+ * [Fix] Restore multiple values in headers processing
+ * [Fix] Some more changes to tag's content length calculations
+ * [Fix] Some more fixes for low contrast fonts detector
+ * [Fix] SpamAssassin plugin: support check_freemail_header('EnvelopeFrom', [..])
+ * [Fix] Trigger HTML_SHORT_LINK_IMG on any external image
+ * [Fix] rspamd_stats: remove deprecated defined(@array)
+
+1.3.1:
+ * [CritFix] Fix catena passwords validation
+ * [CritFix] Fix crash when the first received is faked
+ * [Feature] Add DMARC_BAD_POLICY symbol when DMARC policy was invalid
+ * [Feature] Allow for matching hostnames in multimap (#773)
+ * [Feature] Allow for setting action based on DMARC disposition
+ * [Feature] Allow limiting of the inbound message size
+ * [Feature] Allow maps with multiple symbols and scores
+ * [Feature] Allow regexps in the emails maps
+ * [Feature] Allow to register metric symbols from multimap
+ * [Feature] Allow to reset redis tokens instead of appendig values
+ * [Feature] Allow to store strings in radix maps
+ * [Feature] Check UTF validity when there are utf regexps in a map
+ * [Feature] Correctly work when there is no hard reject action
+ * [Feature] Implement dependencies for maps
+ * [Fix] Another effort to unbreak sqlite locking
+ * [Fix] Avoid crash when closing mmapped file
+ * [Fix] Do not break history on NaN in required score
+ * [Fix] Ensure that hyperscan cache written is written properly
+ * [Fix] Filter NaN from scores in history
+ * [Fix] Fix DNSBL maps
+ * [Fix] Fix another locking issue in sqlite
+ * [Fix] Fix another locking issue with mapped files
+ * [Fix] Fix deadlock in mmaped file stats
+ * [Fix] Fix dependencies in multimap plugin
+ * [Fix] Fix emails module configuration
+ * [Fix] Fix greylist plugin (#755)
+ * [Fix] Fix greylisting plugin variable usage
+ * [Fix] Fix installed permissions for rspamd_stats
+ * [Fix] Fix locking in mmapped statistics
+ * [Fix] Fix paths in tests
+ * [Fix] Fix prefilter mode for multimap
+ * [Fix] Forgot to commit leftover changes
+ * [Fix] Really fix local.d includes
+ * [Fix] Restore selective greylisting behaviour
+ * [Fix] Set max size on per connection basis
+ * [Fix] Use temporary storage for hyperscan cache
+ * [Rework] Remove systemd socket activation
+
+1.3.0:
+ * [CritFix] Fix SA rawbody processing - exclude top part
+ * [CritFix] Fix decoding of UTF HTML entitles
+ * [CritFix] Fix encrypted fuzzy requests
+ * [CritFix] Fix leak of shared memory fds and files
+ * [CritFix] Fix levenshtein distance calculations
+ * [CritFix] Fix mime headers processing
+ * [CritFix] Fix parsing of URLs in texts
+ * [CritFix] Fix parsing of missing classes
+ * [CritFix] Fix redis structure by adding {NULL, NULL} member
+ * [CritFix] Fix some more URL detector issues
+ * [CritFix] Fix systemd sockets activation
+ * [CritFix] Fix unencrypted passwords processing in the controller
+ * [CritFix] Fix writing CDPs to the database
+ * [CritFix] Fix writing of encrypted HTTP requests
+ * [CritFix] Plug memory leak in dkim module
+ * [CritFix] Plug memory leak in headers getting code
+ * [CritFix] Pre-filters and post-filters were completely broken
+ * [CritFix] Properly support SA body regexps
+ * [CritFix] Really skip filters in case of pre-result set
+ * [CritFix] Restore the intended pre-filters behaviour
+ * [Rework] Adopt new maps code
+ * [Rework] Compile ragel sources when building rspamd
+ * [Rework] Finish rework for the rest of places that use HTTP
+ * [Rework] Fix DKIM headers canonicalization
+ * [Rework] Fix lua maps API
+ * [Rework] Import linenoise for line editing
+ * [Rework] Include config structure to all rcl handlers
+ * [Rework] Make chartable module useful
+ * [Rework] Move http internal structures to a private header
+ * [Rework] Partly fix controller
+ * [Rework] Remove dedicated images list
+ * [Rework] Rename http proxy to rspamd proxy
+ * [Rework] Rename mime parts structures
+ * [Rework] Rework HTTP code
+ * [Rework] Rework exceptions and newlines processing
+ * [Rework] Rework pre and postfilters system
+ * [Rework] Separate method to close backend connections
+ * [Rework] Start the complete maps rework
+ * [Rework] Use dynamically generated ragel C sources
+ * [Feature] Add 'blacklist' and 'strict' modes for whitelists
+ * [Feature] Add 'symbols_enabled' and 'groups_enabled' to settings
+ * [Feature] Add ESMTPSA received type
+ * [Feature] Add a simple script to evaluate rspamd rules in the logs
+ * [Feature] Add a simple tool to generate DKIM keys
+ * [Feature] Add a trivial heuristic for codepages
+ * [Feature] Add and use mumhash for non-crypto hashing
+ * [Feature] Add better method to check lua userdata types
+ * [Feature] Add better zip files search algorithm
+ * [Feature] Add catena PBKDF function
+ * [Feature] Add configuration knobs for in and out parser scripts
+ * [Feature] Add content filtering support to multimap
+ * [Feature] Add different timeouts for proxy connections
+ * [Feature] Add dot commands for lua REPL:
+ * [Feature] Add execution of lua global functions script
+ * [Feature] Add function for pretty printing of inet addresses
+ * [Feature] Add function to convert fstring_t to c string
+ * [Feature] Add function to create temporary shared memory mapping
+ * [Feature] Add function to generate random hex string
+ * [Feature] Add generic fucnction to parse IP maps
+ * [Feature] Add initial version of HTTP lua repl
+ * [Feature] Add learn condition to the default configuration
+ * [Feature] Add learn conditions for classifiers
+ * [Feature] Add limit for dkim signatures to be checked
+ * [Feature] Add locking routines for lua_util
+ * [Feature] Add lua API for getting info from archives
+ * [Feature] Add lua utility to decode URL encoding
+ * [Feature] Add method to copy message from http connection
+ * [Feature] Add mirrors feature
+ * [Feature] Add more algorithms for shingles generation
+ * [Feature] Add more domains to redirectors list
+ * [Feature] Add more encodingsto cryptobox hash API
+ * [Feature] Add more file utilities to lua_util
+ * [Feature] Add more functions to extract data from text parts
+ * [Feature] Add more methods to get headers from a task
+ * [Feature] Add more methods to init http message body
+ * [Feature] Add more options for redis config parsing function
+ * [Feature] Add new representation of email address
+ * [Feature] Add new symbols to filter bad extensions in messages
+ * [Feature] Add new utility methods to mimepart object
+ * [Feature] Add openphish support to rspamd phishing module
+ * [Feature] Add parsers for SMTP address in ragel
+ * [Feature] Add parsing of mirror hosts for fuzzy storage
+ * [Feature] Add parsing scripts for master connection as well
+ * [Feature] Add preliminary greylist plugin
+ * [Feature] Add preliminary phishtank support
+ * [Feature] Add preliminary rarv5 support
+ * [Feature] Add preliminary version of ssl toolbox
+ * [Feature] Add protection against open files limit and accepting sockets
+ * [Feature] Add rar v4 support
+ * [Feature] Add reading scripts for master connection
+ * [Feature] Add replies plugin
+ * [Feature] Add results parsing code
+ * [Feature] Add routines to compare and check pubkeys
+ * [Feature] Add simpler versions of refcounts
+ * [Feature] Add some time manipulation functions for lua APi
+ * [Feature] Add support for non-standard BATV signatures
+ * [Feature] Add support of address with at-domain list
+ * [Feature] Add support to search archives by magic
+ * [Feature] Add task:get_rawbody method
+ * [Feature] Add test to check shared memory support sanity
+ * [Feature] Add the initial version of LUA repl to rspamadm
+ * [Feature] Add throughput graph for RRD backend to WebUI
+ * [Feature] Add universal function to make a proper redis request
+ * [Feature] Add universal function to parse redis servers for plugins
+ * [Feature] Add util.unlink function
+ * [Feature] Add utility function to return random number from 0 to 1
+ * [Feature] Add utility method to convert ftok to C string
+ * [Feature] Add utility to map shared memory segments
+ * [Feature] Add versions to fuzzy storage
+ * [Feature] Add workaround for legacy clients in rspamd proxy
+ * [Feature] Add workaround for systems without sane shmem support
+ * [Feature] Add xoroshiro+ fast rng for non-crypto purposes
+ * [Feature] Adopt plugins for new maps API
+ * [Feature] Allow SPF to be checked for empty tasks
+ * [Feature] Allow binary patterns in lua_trie
+ * [Feature] Allow catena encrypted passwords in controller
+ * [Feature] Allow client ip match in the settings
+ * [Feature] Allow easy adding and overriding of fuzzy rules
+ * [Feature] Allow empty tasks to be processed
+ * [Feature] Allow hostnames in IP maps
+ * [Feature] Allow https maps
+ * [Feature] Allow multiple PBKDF types in `rspamadm pw`
+ * [Feature] Allow named fuzzy rules
+ * [Feature] Allow non zero terminated patterns in multipattern
+ * [Feature] Allow partial hash updates
+ * [Feature] Allow pipelining for redis.make_request
+ * [Feature] Allow sending empty requests using client
+ * [Feature] Allow setting fuzzy flag by symbol not by value
+ * [Feature] Allow setting scores and actions from lua
+ * [Feature] Allow shared memory simple http client
+ * [Feature] Allow static lua files in any parts of rspamd sources
+ * [Feature] Allow to change flag from fuzzy learn condition
+ * [Feature] Allow to check rspamd_text using maps
+ * [Feature] Allow to disable composite rules from settings
+ * [Feature] Allow to disable some modules from common redis setup
+ * [Feature] Allow to extract ucl_object from lua using common API
+ * [Feature] Allow to get settings and settings id hash from lua_task
+ * [Feature] Allow to have specific settings for mirrored traffic
+ * [Feature] Allow to open message from a shared memory segment
+ * [Feature] Allow to parse pubkeys from the rcl config
+ * [Feature] Allow to pass extradata from rspamd to rmilter
+ * [Feature] Allow to query storage about number of fuzzy hashes stored
+ * [Feature] Allow to read logs without symbols scores
+ * [Feature] Allow to read password from console for rspamc
+ * [Feature] Allow to set ciphers and CA paths in config
+ * [Feature] Allow to skip some initialization phases to speed up rspamadm
+ * [Feature] Allow underscore separated names in settings
+ * [Feature] Allow versioning for sqlite databases
+ * [Feature] Always allow to terminate rspamd
+ * [Feature] Better deal with backend errors
+ * [Feature] Better lua_redis logging
+ * [Feature] Configure CA path and ciphers
+ * [Feature] Create a dedicated parser to strip newlines
+ * [Feature] Deduplicate the same urls in multimap module
+ * [Feature] Distinguish luajit from lua
+ * [Feature] Do not print garbadge in --compact output
+ * [Feature] Dynamically detect if a CPU is incompatible with hyperscan
+ * [Feature] Enable forced resolving for some lua plugins
+ * [Feature] Enable rrd by default
+ * [Feature] Enable workaround for exim
+ * [Feature] Fix task functions to work without rspamd_config
+ * [Feature] Further improvements to chartable module
+ * [Feature] Further micro-optimizations for hashing and shingles
+ * [Feature] Further relax parser
+ * [Feature] Humanize numbers in stats widgets
+ * [Feature] Implement HTTPS client
+ * [Feature] Implement SSL support in http client
+ * [Feature] Implement body rules for the trie plugin
+ * [Feature] Implement braced regexp quantifiers
+ * [Feature] Implement compare scripts for mirrors results
+ * [Feature] Implement compare scripts setup
+ * [Feature] Implement composites policies
+ * [Feature] Implement conditional learning for classifiers
+ * [Feature] Implement constructing of map from UCL
+ * [Feature] Implement dkim signing in dkim check plugin
+ * [Feature] Implement fuzzy storage updates
+ * [Feature] Implement fuzzy updates push protocol
+ * [Feature] Implement https maps
+ * [Feature] Implement inter-process maps caching
+ * [Feature] Implement mapping of remote flags to local flags
+ * [Feature] Implement mirroring in rspamd proxy
+ * [Feature] Implement multi-flags fuzzy replies
+ * [Feature] Implement multiple-sources fuzzy storage
+ * [Feature] Implement order of pre/post filters
+ * [Feature] Implement partial deleting for multi-flags
+ * [Feature] Implement pipelining for redis async interface
+ * [Feature] Implement ragel parser for received headers
+ * [Feature] Implement reading of messages to shared memory
+ * [Feature] Implement refcount for messages
+ * [Feature] Implement retransmits for master connection
+ * [Feature] Implement zero-copy mode for HTTP reading
+ * [Feature] Improve SPF domain detection logic
+ * [Feature] Improve config:register_symbol function
+ * [Feature] Improve error report for type mismatch in lua
+ * [Feature] Improve fstrings API
+ * [Feature] Improve getting SMTP data from lua_task
+ * [Feature] Improve levenshtein distance function
+ * [Feature] Improve logging in proxy and add refcounts
+ * [Feature] Improve logging lua types
+ * [Feature] Improve master/slave logging
+ * [Feature] Improve phishing plugin
+ * [Feature] Improve phishtank and openphish support
+ * [Feature] Improve ragel build target
+ * [Feature] Improve statistics script
+ * [Feature] Initialize ssl library to use SSL connections
+ * [Feature] Interpolate points sent to webui
+ * [Feature] Limit logging of elements that could have too many items
+ * [Feature] Lock ANN file when loading
+ * [Feature] New abstract hashing API in cryptobox
+ * [Feature] Normalize quoted addresses
+ * [Feature] Now cryptobox lua API accepts rspamd text as input
+ * [Feature] Optimize alignment to speed up hashing
+ * [Feature] Parse received date and ESMTPA proto
+ * [Feature] Parse received timestamp
+ * [Feature] Pass settings id to log helper
+ * [Feature] Pass settings id to lua script from log helper
+ * [Feature] Perform files expansion on proxying
+ * [Feature] Preliminary implementation of fuzzy master/slave updates
+ * [Feature] Print userdata using tostring if possible
+ * [Feature] Propagate HTTP result string when using proxy
+ * [Feature] Properly implement unweighted round-robin algorithm
+ * [Feature] Reduce number of timers queries
+ * [Feature] Rework and improve fuzzy storage
+ * [Feature] Rework dns resolving API for lua, add 'forced' option
+ * [Feature] Rework fann module to understand settings
+ * [Feature] Rework listening system to allow multiple socket types per worker
+ * [Feature] Rework ratelimit module to set expiration
+ * [Feature] Save bayes probability in memory pool var
+ * [Feature] Save settings id hash for convenience
+ * [Feature] Search for SSL_set_tlsext_host_name support
+ * [Feature] Send DKIM signature to protocol reply
+ * [Feature] Show DKIM signature in rspamc client
+ * [Feature] Show symbols description in scan output
+ * [Feature] Sign message merely after DKIM check
+ * [Feature] Simplify machines by assuming that headers are unfolded
+ * [Feature] Sort symbols when displaying them in log
+ * [Feature] Split main connection from mirrored connections
+ * [Feature] Start moving to the new email address structure
+ * [Feature] Store HTTP headers in a hash table
+ * [Feature] Store more information about compressed files
+ * [Feature] Store raw headers value to use them in DKIM
+ * [Feature] Store text parts content with newlines stripped
+ * [Feature] Support DKIM signing
+ * [Feature] Support EXIF jpeg images
+ * [Feature] Support archive files list extraction
+ * [Feature] Support archives when matching patterns in multimap
+ * [Feature] Support premium/academic feed for openphish
+ * [Feature] Support rspamd_updates via https
+ * [Feature] Supprort FQDNs in phishing module maps
+ * [Feature] Try to read on fuzzy timeout to avoid fake timeouts
+ * [Feature] Try to select the optimal possible function for input
+ * [Feature] Unescape and unquote smtp addresses
+ * [Feature] Update fuzzy timestamp when adding value
+ * [Feature] Update mumhash
+ * [Feature] Use -flto if possible when optimizations are enabled
+ * [Feature] Use extended map types in lua map, unify code
+ * [Feature] Use file lock in logger to avoid deadlocks
+ * [Feature] Use generic global string split function
+ * [Feature] Use metrohash as well
+ * [Feature] Use mumhash by default for incremental hashing
+ * [Feature] Use mumhash for words hashing
+ * [Feature] Use new ip parsing API
+ * [Feature] Use new maps API for local addrs
+ * [Feature] Use new ragel parser in message parsing code
+ * [Feature] Use new received parser instead of old one
+ * [Feature] Use new redis API in DMARC plugin
+ * [Feature] Use new redis API in greylist plugin
+ * [Feature] Use new redis API in ip_score plugin
+ * [Feature] Use new redis API in ratelimit plugin
+ * [Feature] Use new redis API in replies plugin
+ * [Feature] Use new version of register_symbol in rspamd plugins
+ * [Feature] Use offset when passing shm to deal with encrypted requests
+ * [Feature] Use one pass to remove newlines and store their positions
+ * [Feature] Use rspamd specific type checks for userdata
+ * [Feature] Use shared memory storage for http maps
+ * [Feature] Use universal redis definitions in rspamd plugins
+ * [Feature] Various improvements in greylist module
+ * [Feature] Wait for sqlite if locked when switching to WAL mode
+ * [Fix] Add filenames sanity filtering for mime types
+ * [Fix] Add guards for empty parts
+ * [Fix] Add missing types
+ * [Fix] Add more priority for config file symbols registered from UCL
+ * [Fix] Add sanity checks when compiling regexp maps
+ * [Fix] Add spaces instead of newlines to the normalized content
+ * [Fix] Add workaround for ancient openssl
+ * [Fix] Add workaround for gmime CTE stupidity
+ * [Fix] Add workaround for hex digits
+ * [Fix] Adjust MISSING_MIMEOLE score
+ * [Fix] Adjust body/body_buf when stealing encrypted message
+ * [Fix] Adopt lua task API for the new email addresses structure
+ * [Fix] Allow for disabling DMARC reporting when Redis is configured
+ * [Fix] Always register openphish and phishtank virtual symbols
+ * [Fix] Always use shmem on linux
+ * [Fix] Another change of newlines policy
+ * [Fix] Another d3evolution update
+ * [Fix] Another fix for exim workaround
+ * [Fix] Another fix for legacy clients
+ * [Fix] Another fix for maps scheduling
+ * [Fix] Another fix for marking upstreams inactive
+ * [Fix] Another fix for postfilters
+ * [Fix] Another fix for redis timeouts
+ * [Fix] Avoid `table.getn` method as it has been removed in lua 5.3
+ * [Fix] Avoid double hashing for images
+ * [Fix] Avoid linking with actrie if hyperscan is enabled
+ * [Fix] Check copy result when sending message to mirrors
+ * [Fix] Cleanup message when assiging body
+ * [Fix] Cleanup stack from global vars
+ * [Fix] Correctly parse query type
+ * [Fix] Disable all symbols if enable_groups is found in settings
+ * [Fix] Disable fts as it is completely broken in bloody linux
+ * [Fix] Disable multiple autolearn checks
+ * [Fix] Disallow updates by default
+ * [Fix] Do not abort when cannot load a map
+ * [Fix] Do not check recursion for non-DNS SPF record types
+ * [Fix] Do not delete uninitialized events
+ * [Fix] Do not die if shmem_mkstemp fails
+ * [Fix] Do not die when no metrics defined
+ * [Fix] Do not even try pcre in case of regexp maps
+ * [Fix] Do not greylist messages if redis has failed somehow
+ * [Fix] Do not greylist on rejection
+ * [Fix] Do not leave temporary maps cached
+ * [Fix] Do not output meaningless errors
+ * [Fix] Do not send NaN in json
+ * [Fix] Don't mix hyperscan and pcre processing within a same task
+ * [Fix] Finally rework and simplify redis backend for statistics
+ * [Fix] Fix Exim shutdown patch
+ * [Fix] Fix JIT compilation for PCRE2 expressions
+ * [Fix] Fix JIT usage for PCRE2
+ * [Fix] Fix REPL output
+ * [Fix] Fix SMTP address parsing machine
+ * [Fix] Fix UTF8 mode in PCRE2
+ * [Fix] Fix a stupid misprint in word 'phishing'
+ * [Fix] Fix accepting the first update when local idx is -1
+ * [Fix] Fix adding maps from ucl
+ * [Fix] Fix adding upstream to an active queue
+ * [Fix] Fix and rescore R_PARTS_DIFFER logic
+ * [Fix] Fix body rules in SA plugin
+ * [Fix] Fix body start position
+ * [Fix] Fix border case in urls detector
+ * [Fix] Fix border cases for incremental hashing
+ * [Fix] Fix caseless uthash application
+ * [Fix] Fix chartable issue with starting digits
+ * [Fix] Fix client_ip in users settings
+ * [Fix] Fix compilation issue
+ * [Fix] Fix conditional learning
+ * [Fix] Fix crash on empty maps
+ * [Fix] Fix creating of URLs from LUA
+ * [Fix] Fix creating of temporary shmem segment
+ * [Fix] Fix creation of mmapped statfiles
+ * [Fix] Fix descriptors leak on shmem detaching
+ * [Fix] Fix detaching of shared memory segments
+ * [Fix] Fix detection of URLs in text parts
+ * [Fix] Fix directories processing for rspamc
+ * [Fix] Fix displaying of rewrite subject in WebUI
+ * [Fix] Fix dkim private keys operations
+ * [Fix] Fix dkim signing
+ * [Fix] Fix dynamic scoring of subject symbols
+ * [Fix] Fix email address build
+ * [Fix] Fix encrypted proxy requests
+ * [Fix] Fix errors counting in upstreams
+ * [Fix] Fix errors handling in the proxy
+ * [Fix] Fix event bases for IO events
+ * [Fix] Fix events handling when scheduling map wacth
+ * [Fix] Fix fann rewrite
+ * [Fix] Fix files fallback for shmem transfer
+ * [Fix] Fix fuzzy adding in webui
+ * [Fix] Fix fuzzy storage encrypted mirroring
+ * [Fix] Fix fuzzy storage sync replies
+ * [Fix] Fix handling of the same words
+ * [Fix] Fix inserting values to the sources list
+ * [Fix] Fix ipv6 mask application
+ * [Fix] Fix issue with missing recipients
+ * [Fix] Fix issues with multiple returns from lua
+ * [Fix] Fix learning for non-existent backend
+ * [Fix] Fix legacy clients support in proxy
+ * [Fix] Fix length calculations for shared memory segments
+ * [Fix] Fix listening on UDP sockets
+ * [Fix] Fix loading of file maps
+ * [Fix] Fix long regexp flags (e.g. {sa_body})
+ * [Fix] Fix lua compare function init
+ * [Fix] Fix maps descriptions
+ * [Fix] Fix maps locking
+ * [Fix] Fix max_train setup in ANN module
+ * [Fix] Fix memory corruption
+ * [Fix] Fix memory leak in unsigned maps reading
+ * [Fix] Fix misprints for lto usage
+ * [Fix] Fix more issues with scripts processing
+ * [Fix] Fix next-to-last extension length check
+ * [Fix] Fix openssl initialization
+ * [Fix] Fix order of arguments
+ * [Fix] Fix order of initialization
+ * [Fix] Fix parser
+ * [Fix] Fix parsing of binary tries
+ * [Fix] Fix parsing of braced IPv6 addresses
+ * [Fix] Fix parsing of nested braces in SMTP comments
+ * [Fix] Fix parsing of rarv5 archives
+ * [Fix] Fix periodic scheduling when a map is not modified
+ * [Fix] Fix possible FP in TRACKER_ID rule
+ * [Fix] Fix post-filters processing
+ * [Fix] Fix potential NULL dereference
+ * [Fix] Fix processing of <br> and <hr> tags
+ * [Fix] Fix processing of addresses in protocol
+ * [Fix] Fix processing of messages without received headers
+ * [Fix] Fix proxying issue for unconnected sessions
+ * [Fix] Fix proxying of the encrypted messages
+ * [Fix] Fix race condition with shared memory by refcounts
+ * [Fix] Fix ratelimit initialization
+ * [Fix] Fix redis set request in replies plugin
+ * [Fix] Fix redis timeout events handling
+ * [Fix] Fix redis timeouts processing logic
+ * [Fix] Fix refcounts in lua_redis
+ * [Fix] Fix results checking if no master connection is active
+ * [Fix] Fix return value for couple of lua functions
+ * [Fix] Fix round-robin selection when upstreams have no weight
+ * [Fix] Fix rows calculation in graph
+ * [Fix] Fix rspamd_redis_make_request syntax in replies plugin
+ * [Fix] Fix scheduling of locked map events
+ * [Fix] Fix scores detection
+ * [Fix] Fix searching for newline positions
+ * [Fix] Fix secure_ip setting in controller
+ * [Fix] Fix sending data to graph command
+ * [Fix] Fix setting of score for parts differ
+ * [Fix] Fix setting of the lua top
+ * [Fix] Fix setting path for lua
+ * [Fix] Fix setting path for phishtank
+ * [Fix] Fix settings application
+ * [Fix] Fix shm_open call as described in POSIX
+ * [Fix] Fix size of length in fuzzy mirror wire protocol
+ * [Fix] Fix smtp grammar issues
+ * [Fix] Fix some issues with redis API
+ * [Fix] Fix some issues with retries in the proxy
+ * [Fix] Fix stack growing
+ * [Fix] Fix start of body detection in DKIM
+ * [Fix] Fix state on timeout
+ * [Fix] Fix stats script
+ * [Fix] Fix substring search when there are '\0' in strings
+ * [Fix] Fix symbol name for spf soft fail
+ * [Fix] Fix symbol type's check
+ * [Fix] Fix symbols registration and execution
+ * [Fix] Fix the case of multiple values keys
+ * [Fix] Fix the default symbol names according to metric
+ * [Fix] Fix timeout setup on learning
+ * [Fix] Fix timeouts in redis cache processing
+ * [Fix] Fix timeouts processing in lua_redis
+ * [Fix] Fix upstreams interaction for rspamd proxy
+ * [Fix] Fix usage of rdns reply structure
+ * [Fix] Fix varargs loop
+ * [Fix] Fix whitelists and blacklists in SA rules
+ * [Fix] Fix write servers setup for redis
+ * [Fix] Fix writing of HTTP messages
+ * [Fix] Force rspamd to upgrade fuzzy db on opening
+ * [Fix] Free the correct pointer
+ * [Fix] Further fixes for lto and static linking
+ * [Fix] Further fixes for surbl extensions map
+ * [Fix] Further fixes in maps code
+ * [Fix] Further improvements to error messages in fuzzy check
+ * [Fix] Further tweaks to redis garbadge collection
+ * [Fix] Groups are now case insensitive
+ * [Fix] Handle log pipe read errors
+ * [Fix] Handle nested dependencies in SpamAssassin plugin
+ * [Fix] Implement new automata to skip empty lines for dkim signing
+ * [Fix] Improve error messages on fuzzy add
+ * [Fix] Improve lua redis handling
+ * [Fix] Improve phishing module logging
+ * [Fix] Improve printing of fuzzy errors
+ * [Fix] Improve rrd diagnostic errors
+ * [Fix] Improve strcase hash used in uthash
+ * [Fix] Include fuzzy key to distinguish storages with different keys
+ * [Fix] Include slave cluster name into http request
+ * [Fix] Include some more information about archives
+ * [Fix] Indicate upstream error on timeout
+ * [Fix] Initialize hash tables array to avoid crashes
+ * [Fix] Initialize parser scripts properly
+ * [Fix] Initialize vars to avoid warnings
+ * [Fix] Inverse logic for saving ANN
+ * [Fix] Link lpeg to rspamd lua library
+ * [Fix] Make extension checks case-insensitive
+ * [Fix] Mark expired hashes as not found and not as zero flag
+ * [Fix] Match archive name as well
+ * [Fix] More and more fixes to redis states
+ * [Fix] More fixes about shared memory in proxy
+ * [Fix] More fixes for redis refcounts
+ * [Fix] More fixes to end of headers detection
+ * [Fix] More fixes to events logic
+ * [Fix] More fixes to multi-flag fuzzy storage
+ * [Fix] More fixes to parts distance calculations
+ * [Fix] More guards for redis free
+ * [Fix] One more fix in redis destructor
+ * [Fix] One more try to fix redis
+ * [Fix] PIE is required for static build
+ * [Fix] Partial fix for mmap'd statistics tests
+ * [Fix] Plug memory leak in proxy
+ * [Fix] Properly detect end of headers position
+ * [Fix] Properly init and free session structures
+ * [Fix] Reduce PRECEDENCE_BULK rule weight
+ * [Fix] Reduce the default thresholds for learning
+ * [Fix] Remove Type=forking from systemd unit file (#709)
+ * [Fix] Remove bad FANN file to save computational resources
+ * [Fix] Remove event before closing of fd to avoid race conditions
+ * [Fix] Remove parsing of 'from' variable in redis backend
+ * [Fix] Remove some bad domains from whitelists
+ * [Fix] Repair optional dependencies
+ * [Fix] Reset master connection when retransmitting scan request
+ * [Fix] Restore ONCE_RECEIVED symbol
+ * [Fix] Restore compatibility with old lua API behaviour
+ * [Fix] Restore redis runtime state
+ * [Fix] Reverse options when received
+ * [Fix] Send updates to mirrors only if we have some changes
+ * [Fix] Set host attribute properly when making HTTP request from lua
+ * [Fix] Set terminated state before calling of async free
+ * [Fix] Simplify MISSING_MIMEOLE rule
+ * [Fix] Simplify state machine by ignoring multiple spaces
+ * [Fix] Skip setting RPATH for static builds
+ * [Fix] Slightly reduce weights of rules with high FP rate
+ * [Fix] Some fixes to libmagic initialization
+ * [Fix] Some more fixes to ratelimit plugin
+ * [Fix] Strip '\r\n' properly
+ * [Fix] Switch hashes to mumhash
+ * [Fix] Treat NaN values properly in graph command
+ * [Fix] Try to avoid FP when checking for phished URLs
+ * [Fix] Try to avoid recursive events deletions
+ * [Fix] Try to fix false positive URL detections in text parts
+ * [Fix] Try to fix issue in redis stats backend when task is closed
+ * [Fix] Try to fix proxying of stupid spamc protocol to HTTP mirrors
+ * [Fix] Try to fix redis crashes
+ * [Fix] Try to fix upstreams with one element
+ * [Fix] Try to handle multiline history in a more sane way
+ * [Fix] Unbreak build on gcc < 4.9
+ * [Fix] Update RPM spec/sources (#700)
+ * [Fix] Update d3evolution version
+ * [Fix] Update mumhash implementation
+ * [Fix] Use custom error function for pre and post filters
+ * [Fix] Use new postfilters and prefilters API in the plugins
+ * [Fix] Use non-blocking mode for systemd sockets
+ * [Fix] Use shared memory merely for local backends in the proxy
+ * [Fix] Use watchers for spf plugin
+ * [Fix] Varioud fixes to the maps code
+
+1.2.8:
+ * Another fix for exim workaround (#637)
+ * Fix unencrypted passwords processing in the controller
+ * Fix setting path for lua (#652)
+ * Fix usage of rdns reply structure (#654)
+ * Use file lock in logger to avoid deadlocks
+ * Add `application/octet-stream` mime type for `pdf` extension (by @moisseev)
+ * Implement new automata to skip empty lines for dkim signing (#651)
+ * Fix parsing of missing classes
+ * Clarify some rspamc arguments (by @fatalbanana)
+ * Correct suppress spelling
+
+1.2.7:
+ * Slightly reduce weights of rules with high FP rate
+ * Add workround for rspamd-1.3
+ * Fix possible FP in TRACKER_ID rule
+ * Simplify MISSING_MIMEOLE rule
+ * Add workaround for gmime CTE stupidity
+ * Fix mime headers processing
+ * Fix false positive URL detections in text parts
+ * Fix Exim shutdown patch
+ * Enable workaround for exim mailbox format
+ * Backport shingles static test
+ * Fix levenshtein distance calculations
+ * Fix max_train setup in ANN module
+ * Fix redis structure by adding {NULL, NULL} member
+ * Fix build with unmodified LibreSSL opensslv.h
+ * Repair optional dependencies
+ * Really skip filters in case of pre-result set
+ * Restore the intended pre-filters behaviour
+ * Fix ipv6 mask application
+
+1.2.6:
+ * Fix parsing of URLs in texts
+ * Fix creating of URLs from LUA
+ * Fix some more URL detector issues
+ * Fix unit tests
+ * Fix JIT compilation for PCRE2 expressions
+ * Fix JIT usage for PCRE2
+ * Fix UTF8 mode in PCRE2
+ * Add workaround for pre-historic compilers (#605)
+ * Fix and rescore R_PARTS_DIFFER logic
+ * Properly set lua paths for tests
+ * Fix SA rawbody processing - exclude top part
+ * Store text parts content with newlines stripped
+ * Properly support SA body regexps
+ * Fix body rules in SA plugin
+ * Fix setting of score for parts differ
+ * More fixes to parts distance calculations
+ - Use hashed words instead of full words for speed
+ - Improve levenstein distance calculations and penalise replaces
+ - Always return number from 0 to 1
+ - Use g_malloc instead of alloca
+ * Fix percents output in R_PARTS_DIFFER
+ * Plug memory leak in dkim module
+ * Plug minor memory leak in regexps creation
+
+1.2.5:
+ * Plug an important memory leak in headers getting code
+ * Remove some bad domains from whitelists
+
+1.2.4:
+ * Implement new multipattern matcher that uses hyperscan if possible
+ * Use mutlipattern for lua_trie code
+ * Add utility methods for multipattern
+ * Use multipattern in url matcher
+ * Add escape functions for hyperscan
+ * Allow to optimize lua -> C transition by flattening table args
+ * Optimize hot paths in SA plugin
+ * Optimize rspamd_re_cache_type_from_string
+ * Allow empty tries
+ * Fix extraction of URLs from Subject
+ * Allow to have different flags for different patterns in multipattern
+ * Add common directory for hyperscan cache to config
+ * Implement caching for hyperscan multipattern
+ * Attach domain part to `R_SUSPICIOUS_URL` (by @moisseev)
+ * Allow multipattern scans to be nested for the case of hyperscan
+ * Simplify SURBL redirector search code and avoid ac_trie
+ * Add two way substring search algorithm
+ * Avoid acism usage to find gtube pattern
+ * Fix processing of empty headers
+ * Allow to disable pthread mutexes on broken platforms
+ * Make web interface not send password in query strings (#585) by @fatalbanana
+ * Add maximum delay to ratelimit module
+ * Backport fix for empty files inclusion from libucl
+ * Fix settings id setup
+ * Add min_learns option to classifiers
+ * Use more clever to utf8 conversion strategy
+ * Fix disabling of virtual symbols in the settings
+ * Rework settings to work properly in metric-less configuration
+ * Set the default limit for classifier
+ * Fix ttl based expiration from LRU cache
+ * Rework DKIM module to use OpenSSL for digests
+ * Fix mailto urls parsing with hyperscan
+ * Do not set obscured flag for urls starting with spaces
+ * Fix crash on redis learn
+ * Fix ratelimit ctime setting
+
+1.2.3:
+ * New DCC module (by @smfreegard)
+ * Rework whitelist module:
+ - Now we check different elements for different checks
+ - MIME from for DMARC
+ - DKIM signature domain for DKIM
+ - SMTP from or HELO for SPF
+ * Fix regexps results combination (*critical*)
+ * Fix issue with expressions processing (*critical*)
+ * Optimize strlcpy for aligned input
+ * Add support of half-closed connection in lua_tcp
+ * Allow to print compact json in client
+ * Save required score in history (#581)
+ * Allow to attach file descriptors to control commands
+ * Allow to send descriptors from workers to main
+ * Allow to attach fd when broadcasting to workers
+ * Implement log pipe feature for rspamd logs analysis
+ * Add `log_helper` worker
+ * Add `URIBL_SBL_CSS` (by @smfreegard)
+ * Add worker scripts functionality
+ * Add on load hooks for rspamd_config
+ * Add lua scripts for log_helper worker
+ * Add generic maillist detector (#584)
+ * Implement FANN autolearn using log_helper worker
+ * Rework metrics configuration to allow includes
+ * Change default value of forced removal in composite rules
+ * Allow to use assembly version of blake2b on x86 cpu
+ * Use less precise (but faster) clock if possible
+ * Insert redirected URL to the urls list
+ * Allow to get and set callback data for rspamd symbols
+ * Add binary heap implementation
+ * Use binary heap for expire algorithms in the hash
+ * Use `least frequent used` expiration strategy
+ * Allow to get mime headers from a task
+ * Add support for mime headers in `regexp` module
+ * Update Exim patches (by @fatalbanana)
+ * Allow building rspamd with jemalloc
+ * Save multipart boundaries
+ * SA plugin changes:
+ - Properly handle MIME headers
+ - Fix eval:check_for_missing_to_header rule
+ - Implement SA compatible body regexps
+ - Use sabody rules in SA plugin
+ * LUA API changes:
+ - Add util.get_ticks function
+ - Add util.stat function
+ - Add task:get_symbols_numeric method
+ - Add method to get number of symbols in the cache
+ - Add lua methods to get redirected urls
+ - Allow to get callbacks for lua symbols
+ - Add config:set_symbol_callback function
+
+1.2.2:
+ * Use HTTP Content-Type on non mime input if possible
+ * Save log level when compressing log messages
+ * Further rework of composite rules (add '^' prefix)
+ * Add tracking for rspamd expressions
+ * Store actions limits in metric result
+ * Fix parsing of include/redirect with many records in SPF
+ * Add method to disable symbols execution in the cache
+ * Allow to disable checks from settings
+ * Allow to select settings by id in HTTP query
+ * Find URLs with '\r' and '\n' inside href attribute
+ * Implement vectored mode for hyperscan (experimental)
+ * Improve client connection errors diagnostics
+ * Allow to edit new files with signtool
+ * Improve hashes performance on 32 bit platforms
+ * Fix sorting of limits
+ * Remove slow and unused rules `INVALID_EXIM_RECEIVED*`
+ * Add expression:process_traced lua method
+ * Allow tables in task:insert_result
+ * Save trace for SA metas
+ * Do not parse broken TLD parts in URLs
+ * Investigate many border cases in URLs parser
+
+1.2.1:
+ * Add list support to `mime types` module configuration (by @moisseev)
+ * Allow symbols params to be printed in logs
+ * Fix `MIME_BAD_ATTACHMENT` false positives for MDN/DSN
+ * Fix crashes on arm32
+ * Do not classify message if some class is missing
+ * Fix cryptobox cleanup
+ * Remove multipart/report from bad mime types (#569)
+ * Improve logging for fuzzy hashes
+ * Show map URLs in webui
+ * Sort symbols in webui
+
+1.2.0:
+ * New dynamic updates plugin
+ * Regular expressions map support
+ * Faster radix trie algorithm
+ * Faster siphash for AVX2 supporing CPUs (used in fuzzy hashes)
+ * PCRE2 support
+ * Allow quoted and slashed keys in map
+ * Add proper support of DNS resolvers balancing (#552)
+ * Rework includes and configuration system for better local changes support
+ * New keypairs framework for signing and encryption
+ * Added support for dynamic modules and workers
+ * Allow to dump configuration with help comments
+ * Rework once_received module
+ - Fix priority for `good_hosts`
+ - If a good host has been found do not add once_received symbols
+ - Fix priorities for strict once_received
+ - Add ability to whitelist IP addresses
+ * Implement support of signed maps for HTTP and file maps
+ * Add command to sync fuzzy storage (#533)
+ * Rework system of symbols and actions registration
+ It is possible now to use priorities when adding symbols to metrics and
+ override scores for symbols with lower priority with the scores with
+ high priority.
+ * Add auth support and db selection for redis stats
+ * Improve composite rules application
+ * Add ignore_received option
+ * Fix critical issue with inconsistent resorting
+ * Fix `all` in spf redirects
+ * Add punycoded versions for IDN domains (#554)
+ * Improve sorting order for symbols cache
+ * Add lockless logging for processes management
+ * Allow to specify flags for metric symbols
+ * Load images height and width from style attribute (#538)
+ * Override DNS requests limits for SPF and DKIM
+ * Fix resetting symbols to their default values in WebUI
+ * Improve configuration agility for redis stats
+ * Allow to set db and password for redis in stat_convert
+ * Import the latest libucl
+ * LUA API changes:
+ - Add rspamd_version function to LUA API
+ - Add lua_cryptobox module
+ - Add lua_map module
+ - Add task:set_metric_action lua API method
+ - Fix race condition in lua_tcp module
+ - Fix a lot of issues in lua_redis module
+ - Rework and abstract lua maps API
+ - Add util.strlen_utf8 lua function
+ - Add lua functions for caseless comparison
+ - Allow optional symbols registration
+ - Add config:add_map table form method, add regexp maps
+ - Add task:has_urls method
+ - Add task:has_flag method
+ - Add html tags methods to lua_html
+ - Add task:get_dns_req
+ * Plugins changes:
+ - Add support for WLBLEval SA plugin
+ - Use caseless comparison in SA and DMARC plugins
+ - Allow SA plugin to set scores for rspamd symbols
+ - Add regexp maps support to multimap
+ - Allow filenames match in multimap
+ - Add more filters for the existing map types
+ - Fix html images rules to reduce FP rates
+ * New rules:
+ - LONG_SUBJ - too long subject
+ - MIME_BAD_ATTACHMENT - bad attachment type
+ - RDNS_NONE - no reverse DNS record for sender's IP
+ - Fix MISSING_MIMEOLE rule for modern OE
+ * Many other bugfixes, memory leaks plugs thanks to:
+ - Coverity scan
+ - New gcc-6 warnings
+ - valgrind manual iterations
+ * Documentation improvements:
+ - FAQ list: https://rspamd.com/doc/faq.html
+ - Reworked quick start guide
+ - Added documentation for all active modules
+ * Other changes:
+ - Dropped Ubuntu Vivid support
+ - Added Ubuntu Xenial support
+ - Rework build system for rspamd and rmilter
+
+1.1.4:
+ * Print traceback on lua errors in lua config
+ * Fix leaks in lua error paths
+ * Improve 'R_EMPTY_IMAGE' rule
+ * Fix metas memoization in SA plugin
+ * Properly set `flag` in fuzzy replies
+ * Fix arguments order
+ * Fix issue with out-of-boundary reading
+ * Fix issues found by coverity
+ * Same result checking error found by coverity
+ * Fix varargs processing (found by coverity)
+ * Fix error in printing hex
+ * Reduce weights for some hfilter patterns
+ * Add aliases for task:get_from_ip:
+ - task:get_addr
+ - task:get_from_addr
+ - task:get_ip
+ * Rework once_received module
+ - Fix priority for `good_hosts`
+ - If a good host has been found do not add once_received symbols
+ - Fix priorities for strict once_received
+ - Add ability to whitelist IP addresses
+ * Fix `MISSING_MIMEOLE` rule for modern OE
+ * Treat meta tags as embedded tags (#501)
+
+1.1.3:
+ * Fix DSN rules when SMTP from is unavailable
+ * Fix statconvert routine to avoid lua module usage
+ * Set a sane quark for configtest to avoid NULL to be printed in logs
+ * Support c11 if available
+ * Fix parsing of ip:port strings
+ * Add more diagnostic for lua subr errors
+ * Fix task:set_from_ip lua method
+ * Add basic routines for digital signatures
+ * Add tool for digital signatures
+ * Add plain open file API method for atomic open
+ * Fix parsing nested braces inside logger vars
+ * Pre filters now actually skip processing
+ * Add pre-filter mode for multimap
+ * Switch to apache 2 license
+
+1.1.2:
+ * Fix stat_cache closing
+ * Add checkpoints to sqlite3 learn cache
+ * Do not recompile lua generated headers all the time
+ * Increase number of messages learned
+ * Fix issues with dual stack and hfilter
+ * Disable MID checks for hfilter by default
+ * Fix cache definitions in multiple classifier and no type
+ * Don't crash if learn cache failed to initialize
+ * Fix googlegroups support in maillist plugin
+ * Rework flags LUA API:
+ - Allow to check for a specific flag
+ - Add `learn_spam`, `learn_ham` and `broken_headers` flags
+ - Unify internal functions
+ * Add `BROKEN_HEADERS` rule
+ * Add support for forged confirmation headers (by @AdUser)
+ * Allow `any`, `mime` and `smtp` for get_from/get_recipients
+ * Add mime types checking plugin
+ * Add rule to detect spammers attempts to cheat mime parsing
+ * Rework parsing of IP addresses in configuration (better IPv6 support)
+ * Add `util.parse_mail_address` function to LUA API
+ * Add lua sqlite3 module
+ * Implement synchronous redis call
+ * Ratelimit: avoid possible indexing of nil value (Fixes #498) (by @fatalbanana)
+ * Add stat_convert command to convert stats tokens from sqlite3 to redis
+ * Implement redis advanced lua api with pipelining
+ * Fix memory leak on redis stat (#500)
+ * Fix user/language learn count in sqlite statistics (#496) (by @fatalbanana)
+ * Fix build with custom pcre
+ * Fix fuzzy relearning (#498)
+ * Improve planning of asynchronous tasks
+ * Show slow rules in log
+ * Add warning for slow regexps
+ * Add base32 decode/encode routines to lua util
+ * Allow converting of learn cache from sqlite to redis
+ * Add methods to check if a messages has from/rcpts
+ * Improve and fix multimap plugin:
+ - Restore 'header' maps
+ - Add filters for headers
+ - Add 'email:addr', 'email:user', 'email:domain' and 'email:name' filters
+ - Add generic regexp filters
+ * Disable reload command in rc scripts
+ * Improve runtime CPU dispatcher for libcryptobox
+ * Add preliminary support of digital signatures via ed25519
+ * Add detection for RDRAND support
+ * Print configuration of crypto on start
+ * A in SPF presumes AAAA lookup as well
+
+1.1.1:
+ * Fix duplicated XBL symbol
+ * Reduce log severity for ratelimit missing servers
+ * Fix XBL composite to avoid duplicate symbols
+ * Reduce weight of URL_ONLY rule due to FP rate
+ * Disable fuzzy hashes from the metadata for now
+ * Fix processing of empty messages (#486)
+ * Always treat DNS timeouts as temporary fail for SPF
+ * Fix issue with SPF double IP stack (#483)
+ * Use X-Forwarded-For when checking secure_ip (#488)
+ * Fix hash calculation for sqlite stats
+ * Fix memory corruption on punycode
+ * Fix strings allocation in punycode
+ * Fix error message (#491)
+
+1.1.0:
+ * Incompatible change: sqlite3 and per_user behaviour:
+ Now both redis and sqlite3 follows the common principles for per-user
+ statistics:
+ 1) If per-user statistics is enabled check per-user tokens ONLY
+ 2) If per-user statistics is not enabled then check common tokens ONLY
+ If you need old behaviour, then you'd need to use separate classifier
+ for per-user statistics.
+ * Implement redis statistics backend and cache
+ * Implement autolearning for statistics
+ * Reworked statistics architecture from scratch
+ * Add hyperscan (https://github.com/01org/hyperscan) engine for regular
+ expressions:
+ - add lazy loader for hyperscan databases
+ - rework regexp cache to have joint pcre/hyperscan scanning
+ - implement hyperscan pre-filter support
+ - add compilation guards for bad expressions
+ - implement `rspamadm control recompile` command
+ - implement hyperscan cache monitoring
+ - slides: <https://highsecure.ru/rspamd-hyperscan.pdf>
+ * Implement flexible task logging
+ * Rework fuzzy worker:
+ - it is now possible to run multiple fuzzy workers;
+ - implement lazy writing as sqlite3 is bad at concurrent writing;
+ - add retries for simple sql commands in fuzzy backend;
+ - use fine-grained transactions for fuzzy;
+ - implement new multi-pubkeys mode;
+ - allow encrypted only storages;
+ - rework statistics for fuzzy;
+ - add `rspamadm control fuzzystat` command for extended statistics;
+ - implement human readable output for the previous command;
+ - add condition script for learning fuzzy storage;
+ * Various fixes to SPF:
+ - fix `redirect` records;
+ - fix domains when parsing mx/ptr/a records in includes/redirects;
+ - fix issues with multiple addresses in SPF records;
+ - ignore SPF results in case of DNS failure;
+ - adjust TTL of records when resolving subelements of SPF records;
+ - always select `v=spf1` line if it is available
+ - do not cache records with DNS failure in subrequests;
+ - ignore records with temporary fails during subrequests resolving;
+ - fix `RDNS_RC_NOREC` support;
+ * Add clang plugin for static analysis:
+ - implement static checks for `rspamd_printf` format strings;
+ * Add 'allow_raw_input' option for non-mime messages
+ * Recognize types using libmagic
+ * Fix parsing of IPv6 received headers
+ * Add new interface of communication between workers in rspamd
+ * Add support for named socketpairs
+ * Don't write URLs by default as it is too verbose
+ * Set status for HTTP replies
+ * Try load `rspamd.conf.override`
+ * Implement words decaying for text parts to limit many checks
+ * Improve support of SA rules and plugins:
+ - add check_for_shifted_date and check_for_missing_to_header eval rules;
+ - add 'check_relays_unparseable' support;
+ - add `check_for_mime('mime_attachement')` function;
+ - use new re_cache interface for all SA rules;
+ - add support for `Mail::SpamAssassin::Plugin::MIMEHeader`;
+ - add support of 'special' SA headers to `exists` function;
+ - fix issue when SA metas contain other metas;
+ - fix freemail rules;
+ * Many fixes to the URL parser
+ * Match any newline character in regexps
+ * Fix resolving of upstreams and detection of poor IPv6 configurations
+ * Parse upstreams selection algorithm from the configuration line
+ * Add `reresolve` command to the control interface
+ * Generate fuzzy hashes from task metadata (URLs and headers)
+ * Add method to check if IP is local and `local_addrs` option
+ * Implement forced timeout for delayed filters
+ * Disable fast path of pcre-jit as it seems to be broken
+ * Bayes fixes:
+ - new normalizer function;
+ - really use weights of tokens from the OSB algorithm;
+ - restore multiple classifiers support;
+ * Rules changes:
+ - add `R_SUSPICIOUS_URL` rule that detects obfuscated URL's;
+ - improve empty image rule;
+ - rework `FORGED_RECIPIENTS` rule;
+ - reduce weight of `SUSPICIOUS_RECIPS`;
+ - fix `*_NORESOLVE_MX` symbols in hfilter;
+ - add `SUBJ_ALL_CAPS` rule with support of UTF8
+ - add spamhaus SBL to uribl
+ - fix `SUSPICIOUS_RECIPS` and `SORTED_RECIPS` rules
+ - remove `R_TO_SEEMS_AUTO` as it generates a lot of FP;
+ - add new Message-ID regexp for Thunderbird (by @moisseev);
+ * Plugins changes:
+ - allow ratelimit plugin to set symbol instead of pre-result
+ - support IP DNS black lists for URIBL (e.g spamhaus SBL);
+ - drop deprecated SURBL bits (by @fatalbanana)
+ - rename `JP_SURBL_MULTI` to `ABUSE_SURBL` (by @fatalbanana)
+ - add `SURBL_BLOCKED` (by @fatalbanana)
+ - add `CR_SURBL`
+ - SURBL: allow fallthrough to default symbol (by @fatalbanana)
+ - Settings: fix IP match (by @fatalbanana)
+ - SURBL: add missing symbols to metric (by @fatalbanana)
+ - allow processing images urls for SURBL
+ - unconditionally disable SPF for authenticated users and local networks
+ * Rework ratelimit plugin
+ - switch to `rates` instead of old and stupid strings to setup;
+ - check if a bucket is zero and disable the corresponding limits'
+ - turn off all buckets by default;
+ - check either `rcpt` or `user` buckets, not all together'
+ - document new `rates` and `symbol` options;
+ - inform user about what buckets are used in the configuration;
+ * Add neural network **experimental** plugin
+ * Add a sample script to learn neural network from rspamd logs
+ * Add documentation strings support to rspamd:
+ - add strings for the main configuration options;
+ - document workers options;
+ - add internal plugin options;
+ - create `rspamadm confighelp` routine;
+ - implement human readable output for the previous command;
+ - add subtree search support;
+ - add keyword search support;
+ * Documentation improvements, tutorials section, statistics description
+ * Many other minor and major bugfixes not noted here
+
+1.0.11:
+ * Fix spf redirects
+ * Fix domains when parsing mx/ptr/a records in includes/redirects
+ * Fix unfolded base64 encoding
+ * Fix GError use-after-free
+ * Do not rewrite the original url when using redirector
+ * Fix parsing of fragment in urls
+ * Fix processing of HTML tags
+ * Improve empty image rule
+ * Avoid long double type
+ * Fix tokens weights in OSB algorithm
+ * Improve debugging for bayes
+
+1.0.10:
+ * Fix settings application (#416)
+ * Fix another issue with fixed strings
+ * Fix hash function invocation
+ * Use the proper string for make_dns_request in lua_http
+ * Fix scan time output
+ * Update webui:
+ - fix labels for greylisting
+ - fix dimension of scan time
+
+1.0.9:
+ * Emergency fix in keyed blake2 to fix fuzzy hashes and encrypted password
+ * Support passwords longer than 64 symbols
+
+1.0.8:
+ * Add function to traverse AST atoms
+ * Allow dependencies on rspamd symbols for SA metas
+ * Fix memory corruption when timeout is removed in fuzzy check
+ * Fix encrypted fuzzy add processing
+ * Avoid use-after-free in controller session destructor
+ * Use session pool instead of task pool in fuzzy check
+ * Fix assembly in i386 mode (#413, #412)
+
+1.0.7:
+ * Plugged memory leaks in internet address object & html parser
+ * Fixed static build
+ * Fixed multiple sigchld processing
+ * Fixed deletion of signal events after event processing loop
+ * Fixed build on ARM (#404 - reported by @Gottox)
+ * Fixed setting the default mask for SPF.
+ * Fixed sanitisation of HTTP query values
+ * Fixed parsing of the last header in encrypted HTTP messages
+ * Additions and fixes for test suite & benchmarks
+ * Added openssl aes-256-gcm support to libcryptobox & HTTP server
+ * Implemented support for starting multiple HTTP servers
+ * Implemented batch accept in HTTP server
+ * Added module to get data from HTTP headers (#285 - reported by @msimerson)
+ * Added `rspamadm control` command
+ * Added ability to sort counters output.
+ * Added ability to specify custom headers for rspamc client
+ * Fix architecture detection
+ * Converted history storage to the UCL format
+ * Allow flexible number of rows in history
+ * Fix action badges in WebUI
+ * Add universal cryptobox hash API
+ * Migrated to the optimized blake2b implementation adopted from Andrew Moon
+ * Allow explicit loading of specific modules
+ * Always load settings module
+ * Allow to add symbols from settings
+ * Fix double free in the controller fuzzy learn command
+ * Avoid endless loop when cannot open sqlite db
+ * Updated libucl
+
+1.0.6:
+ * Fix build on i386
+ * Update CentOS7 service file patch (by @fatalbanana)
+ * Fix path to rspamadm in Debian init script (by @fatalbanana)
+ * Fix broken '_SC_GETPW_R_SIZE_MAX' on FreeBSD
+ * Fix portability issues
+ * Use cryptobox chacha for libottery
+ * Better support of 32 bit builds
+ * Fix header name tokens setup
+ * Fix levenstein distance method for words
+ * Add workaround for old libevent (#400)
+ * Fix microseconds in termination timer
+ * Fix some more issues with fixed strings
+ * Explicitly test CPU instructions even after CPUID call
+ * Do not check out of boundary memory
+ * Do not output broken emails
+ * Fix unknown symbols registration
+ * Handle SIGILL using longjmp
+ * Block signals when exiting event loop
+ * Fix incorrect allocation size
+ * Slightly optimize alignment
+ * Restore rspamd -t for compatibility
+ * Add more sanity checks for emails
+
+1.0.5:
+ * Add rspamd control interface:
+ - support `stat` command to get runtime stats of rspamd workers
+ - support `reload` command to reload runtime elements (e.g. sqlite3 databases)
+ * Rework curve25519 library for modular design:
+ - add Sandy2x implementation by Tung Chou
+ - fix CPU detection for variables loading assembly
+ - add testing for curve25519 ECDH
+ * New fixed strings library
+ * Add `R_SUSPICIOUS_IMAGES` rule
+ * Enable mmap in sqlite3
+ * Use new strings in the HTTP code
+ * Improve google perftools invocation
+ * Improve performance profiling in http test
+ * Reorganize includes to reduce namespace pollution
+ * Allow specific sections printing in configdump command
+ * Rework workers signals handlers to be chained if needed
+ * Update socketpair utility function
+ * Add control_path option for rspamd control protocol
+ * Fix ownership when listening on UNIX sockets
+ * Rework signals processing in main
+ * Remove extra tools from rspamd (they live in rspamadm now)
+ * Remove global rspamd_main
+ * Add global timeout for the overall task processing (8 seconds by default)
+ * Sanitize NULL values for fuzzy backend
+ * Store NM between encrypt/decrypt
+ * Add textpart:get_words_count method
+ * Fix generic DNS request in lua
+ * Tune hfilter weights
+ * Add support of IPv6 in hfilter
+ * Fix parsing of HTTP headers with IP addresses
+ * Sync with the recent libucl
+ * Various minor bugfixes
+
+1.0.4:
+ * Add configdump routine to rspamadm
+ * Implement retransmits for fuzzy_check plugin
+ * Fix events processing for learning anf checking fuzzy hashes
+ * Avoid dependency on unneeded and uncompatible glib include
+ * Add `historyreset` command to the controller
+ * Fix loading of tokenizer config from dump (#389)
+ * Add sorting hints for the history
+ * Allow custom lua scripts for users/languages extraction (#388)
+ * Do not add FORGED_RECIPIENTS when 'To' is missing (#387)
+ * Do not add R_UNDISC_RCPT when 'To' is missing (#387)
+ * Add encryption to fuzzy check plugin
+ * Add encryption for fuzzy storage
+ * Add new epoch for encrypted fuzzy request
+ * Add encryption for `rspamd.com` storage
+ * Remove gmime processing for LDA mode as it is deadly broken
+ * Add routine to find end of headers position in mime messages
+ * Fix LDA headers folding
+ * Init libraries in rspamc client as well to avoid locale issues
+ * Avoid collision with locally installed includes
+ * Allocate and free memory with the same allocator in rspamadm (#385)
+ * Preserve expired fuzzy hashes counter
+ * Improvements in webui:
+ - Add favicon.ico
+ - Rework history table
+ - Fix sorting for the history
+ - Migrate to bootstrap 3 and jquery 2
+ - Fix css bugs
+ - Add glyphicons
+ - Add reset history
+ - Improve history buttons
+ - Redraw graph to avoid display issues
+ - Webui is now MIT licensed to match licensing policy of rspamd
+
+1.0.3:
+ * Fix piechart clean slice (#380)
+ * Fix controller crashes when GString is reallocated (#381)
+ * Correctly set locale before start
+ * Set C locale for numeric values
+ * Add rspamadm routine:
+ - add `pw` command to manage passwords
+ - add `help` command for displaying help
+ - add `configtest` command to check configuration files
+ - add `keypair` command for generating encryption keys
+ - add `fuzzy_merge` routine to merge fuzzy sqlite databases
+ - add a simple manual page for rspamadm
+ * Allow metric registration for composite expressions
+ * Add strict mode for configtest
+ * Add logger counters
+ * Save and show learned messages count (#383)
+ * Add `no_stat` flag
+ * Add `task:set_flag` and `task:get_flags` (#382)
+ * Enable foreign keys in sqlite3
+ * Remove orphaned shingles from fuzzy storage
+ * Optimize synchronization steps for fuzzy storage
+ * Allow delayed conditions registration
+ * Add lua API for conditions registering
+
+1.0.2:
+ * Fix critical bug in webui that prevents password from being sent
+ * Rework webui view:
+ - Switch to d3.js for graphs
+ - Improve piechart look
+ - Rework colors for piechart
+ - Fix layout for symbols
+ - Fix refresh button
+ * Add descriptions for whitelist maps
+ * Fix build on arm (#379)
+ * Fix issue with the last element in the radix trie
+ * Add more tests for radix trie algorithm
+ * Allow to extract URLs from query strings of other URLs (#361)
+ * Initialize rrd fields before writing to file
+ * Fix double free if no password has been specified
+
+1.0.1:
+ * Add writing to rrd from the controller
+ * Fixed lots of bugs in rrd code
+ * Adopt new DNS API in hfilter plugin (by @AlexeySa)
+ * Allow only one controller process to manage rrd file
+ * Set event base for fuzzy calls
+ * Improve fuzzy IO errors logging
+ * Add rra extraction function to rrd library
+ * Add graph handler to the controller
+ * Cache correct passwords to avoid too high CPU usage when working with webui
+ * Controller sockets are owned by router do not export them to task
+ * Optimize logging by skipping hash table search if it's empty
+ * Fix loading issue with broken statfiles
+ * Print assertions from glib to rspamd logger
+ * Load legacy `lua/rspamd.local.lua`
+ * Update webui with some fixes to learning and scanning
+
+1.0.0:
+ * Rework symbols processing:
+ - Improve sorting logic for symbols
+ - Organize processing into multiple stages
+ - Added asynchronous watchers for symbols
+ - Added ability to organize dependencies between symbols
+ * Fixed URL redirector:
+ - Use optimized POE loop
+ - Organize dependencies
+ - Fix startup
+ * New sqlite3 backend:
+ - Allow to have per-languages and per-user statistics
+ - Allow sqlite3 to be used as statistics backend
+ * Store tokenizer configuration within statfiles
+ * Improve bayes statistics:
+ - Use headers and images metainformation in bayes
+ - Suggest using of pre-processed tokens for statistics
+ - Fix tokens normalization for OSB algorithm
+ * Rewrite url parsing:
+ - Fix numerous issues with url extraction and normalization
+ - Fix mailto urls
+ * Fix settings plugin to allow custom actions scores
+ * Improve rbl plugin
+ * Allow capturing patterns in rspamd lua regexp library
+ * Add GTUBE support
+ * Fix spamc legacy support
+ * Add DKIM support to RBL module
+ * Fix issues with multiple DKIM signatures
+ * Fix issue if rspamd cannot create statfiles (#331)
+ * Rework parts and task structure:
+ - Now text_parts, parts and received are arrays
+ - Pre-allocate arrays with some reasonable defaults
+ - Use arrays instead of lists in plugins and checks
+ - Remove unused fields from task structure
+ - Rework mime_foreach callback function
+ - Remove deprecated scan_milliseconds field
+ * Add ip_score plugin support (not enabled by default):
+ - Can check for asn/country and network using DNS lookups
+ - Can store and load reputation from redis server
+ * Improve PARTS_DIFFER rule to count merely different words
+ * New HTML parser:
+ - Parses HTML parts using a set of state machines
+ - Extracts useful data and exports it to lua functions:
+ + Styles
+ + Images
+ + URLs
+ + Colors
+ + Structure elements
+ - Added HTML rules for some checks
+ * New version of LUA DNS API
+ * Table versions of many functions in LUA API
+ * Improve rspamc client:
+ - Print execution time
+ - Allow executing of external commands and passing output to them
+ - Allow mime output mode when rspamc alters message according to rspamd
+ checks and send it to an external command or stdout
+ * Allow scanning of local files using HTTP requests
+ * Rework configuration system:
+ - Rules are now moved from the $CONFDIR to $RULESDIR to avoid ambiguity
+ - All modules configurations are now split in $CONFDIR/modules.d/* to
+ simplify upgrades
+ - Move hfilter to plugins
+ - Allow plugins and rules to define default scores to simplify metrics
+ setup
+ - Include overrides for all modules to honor local/automatic parameters
+ - Tune scores for many modules
+ * Rework and enable DMARC plugin
+ * Add whitelist plugin for SPF/DKIM/DMARC based whitelisting
+ * Add some common domains to whitelists shipped with rspamd
+ * Rework logging:
+ - Now each log entry supports module name and a `tag`. Tag is used to
+ identify unique objects (such as tasks) when checking log files
+ - It is possible to turn on debugging for the specific modules
+ - Systemd logging is fixed
+ * Improve spamassassin plugin.
+ - Now headers are matched more like SA
+ - Improve support of Message-ID
+ - Add support of ToCc header type
+ - Fix :addr and :name in headers regexps
+ * Resurrect rrd support code
+ * Save controller stats between restarts
+ * Fixed tonns of bugs
+ * Added tonns of minor improvements and features
+ * Added more unit tests
+ * Create functional tests framework
+ * Added documentation for missing modules
+ * Added rpm/deb repositories and scripts
+ * Updated WebUI and libucl externals
+
+0.9.10:
+ * Do not dereference null pointer on learning.
+ * Fix some extreme cases in BAYES.
+ * Add a workaround to avoid bad HTML messages breaking.
+ * Build with -O2 flags by default.
+ * Add constraints to limit DNS requests count per task.
+ * Add workaround for SURBL DNS flood.
+ * Set error if rspamd cannot learn anything.
+
+0.9.9:
+ * Don't use RWL_SPAMHAUS_WL (unknown result) for whitelisting (by @fatalbanana)
+ * Import updated public suffix list (by @fatalbanana)
+ * Remove debug message
+ * Fix settings (by @fatalbanana)
+ * Remove duplicated symbol registration
+ * Use WAL for fuzzy storage
+ * RBL fixes (by @fatalbanana):
+ - silence errors;
+ - yield unknown results from RBLs;
+ - fix scoring for DNSWL;
+ - fix use of RBL name as symbol;
+ - ignore RBL names that would not be yielded;
+ * Support captures in regular expressions
+ * Add captures support to lua_regexp
+ * Support dist on FreeBSD and Darwin
+ * Add RCVD_IN_DNSWL_NONE as whitelisting exclusion (by @fatalbanana)
+ * Multiple fixes to URL detection:
+ - support port definition;
+ - fix query and path recognition;
+ - fix parsing of multiple slashes in URL;
+ - fix parsing query just after port;
+ - fix path field in `url:to_table` method;
+ - improve support of IP based URLs.
+ * Set ignore_whitelists = true for RECEIVED_SPAMHAUS_XBL (by @fatalbanana)
+ * Add GTUBE support
+ * Ignore User header in SA mode
+
+0.9.8:
+ * Fix critical bug in bayes classifier (#305)
+ * Fix critical bug in RBL module (by @fatalbanana)
+ * Fix and rework settings plugin.
+ * Fix get_all_opts for a case of non-iterable options.
+ * Use tld for redirector's matching.
+
+0.9.7:
+ * Add whitelist_exception setting to RBL module (by @fatalbanana)
+ * Don't use RWL_MAILSPIKE_POSSIBLE or DNSWL_BLOCKED for whitelisting (by
+ @fatalbanana)
+ * Fix extreme cases in bayes classifier.
+ * Fix parsing of urls with '?' at the end of hostname.
+ * Update interface.
+ * Fix number of issues with webui interaction.
+ * Fix saving maps.
+ * Allow user@ and @domain matches in multimap.
+ * Fix issues with bounces From processing.
+ * Fix abs/fabs misuse.
+ * Fix builds on suse and arch linux distributions.
+
+0.9.6:
+ * Fix memory leak if mime cannot be parsed.
+ * Fix dkim cache expiration.
+ * Fix issues with redirector HTTP response.
+ * Fix abnormal connection closing with certains messages with a high score
+ (issue #296)
+ * Fix redirector installation.
+ * Use specific POE loop for some systems.
+ * Fix number of issues in URL redirector.
+ * Fix selecting URLs for sending to redirector.
+
+0.9.5:
+ * Avoid double free when extending HTTP message.
+ * Fix double free if multiple classifiers are defined.
+ * Fix misprint in spamassassin plugin.
+ * Fix cpuid invocation on i386.
+ * Fix ownership issues for zero-copy decode.
+ * Allow __len metamethod on rspamd{text}.
+ * Add base64 decoding lua utility.
+ * Fix build on FreeBSD
+ * Skip spaces at the beginning of mime messages.
+ * DBL_ABUSE_REDIR should not have significant weight.
+ * Allow to split by lua_regexp rspamd{text} objects.
+ * Allow to specify custom stop pattern for lua_tcp.
+
+0.9.4:
+ * Fix critical bugs in tokenization algorithm
+ * Write unit tests for tokenization
+ * Add documentation for lua_tcp
+ * Switch off legacy tokenization by default.
+ * Fix critical bugs in words normalization
+ * Add lua bindings to tokenizer.
+ * Implement storing of HTTP headers inside task
+ * Add lua API to accerss HTTP headers data
+ * Implemented base64 encoding suitable for MIME
+ * Use caseless hash and equal functions for HTTP request headers.
+ * Improve debian architectures support (by @dottedmag)
+
+0.9.3:
+ * Revert incorrect regexp change that broke the default rules
+ * Fix lua_tcp module
+
+0.9.2:
+ * Fix error on spawning unique workers.
+ * Add preliminary version of generic LUA TCP requests API.
+ * Use lua 5.1 if luajit is not available (Arm64, PowerPC, s390x etc)
+ * Fix fuzzy mime strings with only type.
+ * Improve thunderbird sanity checks.
+ * Fix critical bug on matching regular expressions.
+ * Make hiredis optional dependency.
+ * Fix multiple bugs in daemon reloading
+
+0.9.1:
+ * Restore utf8 validation for regular expressions to avoid crashes
+ * Fix symbols displaying in the interface
+ * Add symbol groups to the interface
+ * Fix maps ID parsing in the controller
+ * Add multimap and regexp modules documentation
+ * Backport fixes from libucl
+ * Fix debian package (by @dottedmag)
+ * Rework XXH32 invocations
+
+0.9.0:
+ * Add support of the fast and secure protocol level encryption:
+ - curve25519 is used for key exchange;
+ - chacha20/poly1305 cryptobox construction for bulk encryption;
+ - zero latency overhead;
+ - encrypting and balancing HTTP proxy worker
+ * Rework expressions and create new expressions library:
+ - aggressive optimizations based on the abstract syntax tree;
+ - abstract expressions support (regular expressions, functions, lua modules
+ composites and so on)
+ - New comparison and '+' operators support
+ - New greedy algorithm to minimize execution time of expressions and
+ all symbols
+ - Dynamic expressions benchmark and reoptimizations
+ * Many improvements to the LUA API:
+ - reworked logger module allowing to do pretty print of the most of lua
+ types (including tables and userdata classes)
+ - reworked lua redis and lua HTTP to support more features
+ - added opaque type for passing large text chunks without copying
+ - new regexp module with many auxiliary functions (e.g. `re:split`)
+ * LuaJIT is now the default requirement for rspamd allowing to speed up lua
+ execution by a large margin (however, plain lua is still supported)
+ * New plugins:
+ - spamassassin rules plugin that allows to load and re-use the most of
+ SA rules natively
+ - DMARC plugin that evaluates SPF and DKIM policies to the domain policies
+ - many old plugins has been reworked to implement new features and improve
+ stability
+ * New aho-corasic trie implementation from @mischasan that allows to load and
+ use hundreds of thousands of patterns with no influence on load
+ * Support of PCRE JIT and PCRE JIT fast path modes that significantly improves
+ the performance of regular expressions if supported by PCRE
+ * New URLs parser and extractor:
+ - removed legacy code that was useless for url finding
+ - reworked algorithms of URL parsing for more precise and accurate results
+ - added top-level-domains tree from http://publicsuffix.org
+ - improved emails parsing
+ - removed many phishing false positives due to TLD tree check
+ * New statistics infrastructure:
+ - created a separate layer of statistic library
+ - improved OSB-Bayes by re-weighting tokens according to the original
+ academic paper and `crm114` implementation, which reduced false positives
+ rate significantly
+ - created learn cache to avoid double learning of statistics and providing
+ an efficient way to re-learn class for a message
+ - created abstract layers for different statistics backends
+ - implemented new tokenization algorithms with fast or secure (siphash)
+ hashes to generate statistics features
+ * Reworked utf8 tokenization that previously corrupted all UTF8 words (minor
+ incompatibility with old fuzzy hashes with utf-8 symbols)
+ * SPF module has been completely rewritten to support complex cases of
+ `include` and `redirect` within SPF records
+ * DKIM module now supports multiple signatures
+ * Controller passwords can now be stored encrypted by `PBKDF2-HMAC` in the
+ configuration file
+ * Many hand-written HTTP clients has been replaced with the common rspamd
+ http module
+ * New test framework:
+ - import lua `telescope` test framework
+ - add unit tests for many rspamd modules and routines
+ - create a unit test for each possible bug found
+ - use luajit ffi for testing C code
+ - added preliminary support of functional testing by creating tasks from lua
+ * Randomize hash seed to avoid certain hash tables vulnerabilities
+ * Documentation improvements:
+ - added documentation for the vast majority of rspamd modules
+ - added documentation for rspamd protocol
+ - added documentation for the most of rspamd LUA extensions
+ * Fixed tonns of bugs and memory leaks
+ * Added tonns of minor features
+
+0.8.3:
+ * Various critical fixes in distribution (by @dottedmag and @fatalbanana)
+ * Fixed bugs in url detector to parse certain patterns
+ * Add default host and helo for a client
+ * Some sanity checks for tokenizer and classifier
+ * Reiterate on systemd support
+ * Fix missing symbol registration
+ * Add support of spamc compatible output
+ * Filter double-dots in rbl.lua validate_dns (by @fatalbanana)
+ * Update ucl submodule due to critical bugfix
+
+0.8.2:
+ * Create fuzzy db if it does not exist
+ * Fix: Centos init script: configtest() (by @AlexeySa)
+ * Enable one_shot for RECEIVED_SPAMHAUS_XBL - Fixes #102 (by @fatalbanana)
+ * Update Exim patch (by @fatalbanana)
+ * Fix processing of unix sockets.
+ * Allow applying settings to authenticated users (by @fatalbanana)
+ * Make settings priorities work as documented (by @fatalbanana)
+ * Fix race condition in symbols planner
+ * Add DNSWL_BLOCKED symbol (by @fatalbanana)
+ * Make Exim pass usernames to rspamd (by @fatalbanana)
+ * Update RBL module (by @fatalbanana):
+ - fix indentation;
+ - collapse loops;
+ - avoid calling for un-needed information;
+ - allow disabling RBLs for authenticated users
+ * once_received.lua: Fix indentation & add exclusion for authenticated users (by @fatalbanana)
+ * hfilter.lua: Add exclusion for authenticated users (by @AlexeySa)
+ * Updates to hfilter rules (by @AlexeySa)
+ * Set empty <> user or addr for msgs without FROM (by @eneq123)
+ * Fix: attempt to index field '?' (a nil value) (by @eneq123)
+ * Fix: if not exist Date-header (by @AlexeySa)
+ * Add task:get_content() method.
+ * rbl.lua: Ignore private IP space (by @fatalbanana)
+ * Allow to check radix maps from lua by rspamd{ip}
+ * Make local exclusions configurable per-RBL (by @fatalbanana)
+ * Add rspamd_config:radix_from_config() (by @fatalbanana)
+ * Support emails dnsbl in rbl (by @fatalbanana)
+ * Complete rework of url extraction logic
+ * Allow customizations for unix sockets. (fixes #182)
+ * Set lua path according to rspamd settings.
+ * Import lua-functional for plugins stuff.
+ * Completely rewrite multimap plugin in functional style.
+ * Fix FORGED_MUA_THUNDERBIRD_MSGID (fixes #186)
+ * Check IPv6 addresses at dnswl.org and Spamhaus whitelist (by @fatalbanana)
+ * Add lowercase utility for utf8 strings.
+ * Various fixes to build system
+ * Updated debian configuration infrastructure (by @dottedmag)
+
+0.8.1:
+ * Add sqlite and perl as dependencies for RPM/Debian packages (by @fatalbanana)
+ * Remove whitelist.lua from RPM file list (by @fatalbanana)
+ * Make Exim pass hostnames to rspamd (by @fatalbanana)
+ * Fix building on Fedora (by @fatalbanana)
+ * Add toggle for disabling installation of systemd units on Linux (by @fatalbanana)
+ * Fix double format rounding that caused output corruption (reported by @fatalbanana)
+ * Revert broken change for destructors ordering that led to memory corruption
+ * Do not reset symbols case of settings if parsed from lua (reported by @andrejzverev)
+ * Fix build on SunOS (by @wiedi)
+ * Fix multiple crashes on broken DKIM DNS records
+ * Fix critical issue with composites weights removing
+ * Fix memory corruption in composites processing code
+ * Ignore non-SPF TXT records when parsing SPF includes
+
+0.8.0:
+ * New fuzzy check logic:
+ - use shingles algorithm for fuzzy matching
+ - use blake2 instead of md5 for larger output space
+ - combine fuzzy and strict matching
+ - allow to organize private storages by means of keys
+ - preserve compatibility with previous versions
+ * New fuzzy storage:
+ - use sqlite instead of own memory based hash tables
+ - rework commands interface
+ - add conversion from the old format
+ - add fuzzy match by shignles
+ - support old rspamd versions
+ * Add lemmatizing for words used in fuzzy hashes that allows to improve match
+ quality by using of the first forms of all words
+ * Rework language detection
+ * Fix several critical bugs, memory leaks and deadlocks:
+ - memory leak in HTML nodes parsing
+ - deadlock in logger code
+ - deadlock in signals processing
+ - crashes in fuzzy_storage
+ - crashes in tokenizers if the input was empty
+ * Import new libucl with several bugfixes and improvements
+ * Support listening on ipv6 addresses only
+ * Fix macro expansion in SPF module
+ * Several bugfixes in DKIM module
+ * Add load headers support for mime parts to the lua API
+ * Add documentation for:
+ - workers in general
+ - fuzzy_storage worker
+ - fuzzy_check plugin
+ - mimepart and textpart lua API modules
+
+0.7.6:
+ * Apply boundary fix for dkim simple canonization
+ * Fix ping command
+ * Return nil if header was not found in lua_task
+ * Fix hang in upstreams revive logic
+ * Decode entitles when normalizing HTML parts
+ * Fix logic of finding URLs in HTML parts
+ * Do not include \0 into length of text when performing conversion to utf8
+ * Fix raw vs parsed reperesentations
+ Raw parts are now:
+ - decoded b64/qp, but *NOT* converted to utf-8
+ Processed parts are now:
+ - converted to UTF-8
+ - normalized if needed (e.g. HTML tags are stripped)
+ * Rework DKIM canonization to line based
+ * Fix fuzzy hashes adding
+ * Use more specific hash function for fuzzy
+ * Fix leaking of iconv descriptors
+ * Fix PTR resolving in lua resolver
+ * Rework spf module.
+ - Copy data to memory pool as cached record might be destroyed causing
+ freed memory being passed to the protocol output (use after free)
+ - Allow SPF_NEUTRAL policy to be handled separately
+ - Add R_SPF_NEUTRAL to the default config
+ * Rework `register_symbols` function
+ * Allow to disable components of hfilter
+
+0.7.5:
+ * Fix owner when creating folder /run/rspamd (by @sfirmery)
+ * Fix IP validity checks
+ * Decode URLs obtained from HTML tags
+ * Fix crash with unweighted upstreams
+ * Stop processing headers in parts
+ * Set sockaddr.sa_family properly when connectig to upstreams
+ * Fix reload issues in surbl and fuzzy_check (reported by @citrin)
+ * Fix timeouts in redirector
+ * Improve lua errors reporting
+ * Fix lua closures processing in libucl
+ * Rework calling of lua functions from regexp module
+ * Choose raw regexp for raw headers
+ * Rework conversion to utf since glib one is broken
+ * Ignore SGML style tags in html
+ * Fix old bug with non-capturing https urls
+ * Fix memory corruption on fuzzy reload (reported by @citrin)
+ * Fix percents display in rspamc
+ * Fix buffer update for DKIM
+ * Do not validate utf for raw headers
+
+0.7.4:
+ * Fix build under *BSD
+ * Detect HAN unicode script
+ * Implement language detection heuristic for text parts
+ * Fix time output in history
+ * Improve piechart coloring
+ * Fix \r\n conversion in DKIM module (reported by @citrin)
+ * Try to detect systems with no IPv6 support
+ * Fix multiple/single values in use settings (reported by @citrin)
+ * Rework IP addresses in upstreams:
+ - Select ipv4/unix addresses if they exist and use ipv6 for ipv6 only
+ upstreams (since the support of ipv6 is poor in many OSes and
+ environments)
+ - Free IP list on upstream destruction
+ - Add test cases for addresses selection
+ - Allow adding of free form IP addresses to upstreams
+ * Fix endianness in lua_radix search (reported by @citrin)
+ * Soft shutdown should also set wanna_die flag (reported by @citrin)
+ * Stop use-after-free in event loop termination
+ * Fix processing of very short messages in DKIM (reported by @citrin)
+ * Detect systems without shared mutexes
+ * Fix issues with PTR and MX elements in SPF parser (reported by @citrin)
+
+0.7.3:
+ * New upstreams code:
+ - simplify upstreams API;
+ - unify strings parsing in upstreams definition;
+ - add configuration options for the upstreams;
+ - for failed upstreams re-resolve their addresses;
+ - use all resolved addresses for an upstream (round-robin);
+ - implement stable hashing and use it by default for upstreams;
+ - add unit test for upstreams module.
+ * Rework signals processing in all rspamd workers:
+ - signals are now processed in the event loop;
+ - implement the most common signal handlers for all workers;
+ - add callbacks for workers specific signal handlers
+ * Fix critical issue with fuzzy storage:
+ Fuzzy stroage could not save any hashes on termination due to bugged
+ signals handling
+ * Fix roll history IP storage
+ * Rework ipv4/ipv6 handling in parsing addresses:
+ - turn off support of IPV6_V6ONLY socket option;
+ - create ipv6 socket prior to ipv4 one to handle systems with v6/v4
+ sockets enabled (Linux)
+ * Remove CBL as it's wholly included in Spamhaus XBL (by @fatalbanana)
+ * Remove nszones.com fake RBL (by @citrin)
+ * Fix upstreams interaction for fuzzy_check
+ * Verify spf PTR records (reported by @citrin)
+ * Fix spf MX records parsing
+ * Add compatibility for old libevent (by @yellowman)
+ * Sync bugfixes from libucl
+
+0.7.2:
+ * Convert all maps to the compressed radix trie
+ * Allow IPv6 addresses in IP maps
+ * Remove dynamic items support from symbols cache
+ * Allow hex encoded output of strings
+ * Fix bug with control connections count
+ * Process fuzzy weight correctly (reported by @fatalbanana)
+ * Remove extra reference retain of http connection on error
+ * Remove deprecated options from the default config
+ * Add `one_shot` attr to metric's symbols
+ * Doc: add documentation for metrics
+ * Add Upstart job to debian packaging (by @CameronNemo)
+ * Config: improve SURBL symbols descriptions (by @citrin)
+ * Config: reflect SURBL changes (by @citrin):
+ - Outblaze removed, malware moved to separate list:
+ http://www.surbl.org/news/internal/MW-malware-sublist-added-to-multi
+ * Fix C modules initialization on restart
+ * Treat single IP as a single IP in radix lists (reported by @citrin)
+ * Do not touch file and core limits if not asked explicitly (reported by @citrin)
+ * Improve logging for fuzzy errors
+ * Block SIGPIPE for HTTP writing
+ * Doc: update manual pages
+ * Fix HTTP connection termination
+ * Reduce default number of parallel requests to 8
+ * Sync with libucl include features
+
+0.7.1:
+ * Fix typo in stat output.
+ * Fix issues with includes crossing with the system includes
+ * Restore testing framework
+ * Add radix trie test suite
+ * Implement new path-compressed radix trie.
+ - The performance benefit over the old algorithm is about 1.5 times.
+ - Memory usage is significantly lower as well.
+ - Now radix trie can accept any IPv4/IPv6 values
+ * Various improvements to the memory pools code
+ * Fix writing reply to a client when no filters are defined
+ * Write base32 encoded fuzzy
+ * Fix 'soft reject' action
+ * Fix rspamd reload and modules reconfiguration
+ * Fix subject rewriting for the default subject
+ * Fix states for processing task and pre-filters
+ * Fix issues with connection closing
+ * Fix crashes in rdns
+ * Fix ratelimit pre-filter
+ * Update exim patch.
+ - Update to the recent exim version
+ - Strip extra leading src/ from the patch
+ - Remove sendfile since it was broken
+ - Fix rspamd spam report for exim
+ * Improve documentation
+
+0.7.0:
+ * Use HTTP protocol for all operatiosn
+ * Webui worker is now removed and controller works as webui
+ * Allow to serve static files via controller by option `static_dir`
+ * Rspamd interface is now a part of rspamd
+ * Rspamc client has been rewritten to use HTTP and non-blocking mode
+ allowing to start multiple operations simultaneously (see `-n` option)
+ * Lua API was completely reworked to satisfy modern standards of LUA:
+ * Module `lua-message` was removed
+ * Reduced number of superglobals registered by rspamd
+ * Many functions has been redesigned
+ * Symbols registration is now more convenient
+ * Users settings has been rewritten as lua plugin
+ * Reworked headers system as gmime's based one misses many headers and is
+ very slow to get headers values
+ * Reorganized code and removed embedded jannsson by using UCL for all json
+ parsing
+ * Migrated to `librdns` for DNS resolving that improves concurrency for
+ DNS requests significantly
+ * Fixed tonns of bugs in MIME processing
+ * Improved metrcis and default symbol's weights
+ * Added new RBL's
+ * Fixed a number of issues in the modules
+ * Removed several memory leaks found
+ * Fix unicode processing
+ * Fix fuzzy checking for unicode parts
+ * Significantly improve documentation and especially LUA API docs
+ * See migration notes at https://rspamd.com/doc/migration.html
+
+0.6.8:
+ * Controller now listen for localhost and not for 127.0.0.1 by default
+ * Allow FCrDNS-style RBL lookups (by @fatalbanana)
+ * Reduce threshold for parts_differ function.
+ * Fix hostname lookup for rdns rbl (by @AlexeySa)
+ * Fix HFILTER_URL_ONELINE to reduce false positive rate.
+ * Fix whitelist module.
+ * Allow override system predefined settings without touching system ones
+ by .try_include macro (by @andrejzverev)
+ * Check for [ip.address]-style HELO and suppress lookups. (by
+ @fatalbanana)
+ * Optimize hfilter (by @AlexeySa)
+ * Fix issue with random numbers generator in dns.
+ * Use more clever time values to setup entropy.
+ * Synced with the recent libucl.
+
+0.6.7:
+ * Use ChaCha20 for DNS generator (more secure DNS id)
+ * Unknown symbols now has zero weight and not 1.0
+ * Fix fuzzy hashes expire time
+ * Fix critical issue in statfiles on FreeBSD 9 (and some other platforms)
+ * Add .include_map macro to ucl parser
+ * Update libucl
+ * Fix headers end detection for DKIM module
+ * Fix a bug in received headers parser
+ * Validate IP addresses before pushing them to lua
+ * Start new documentation project
+ * Fixed tonns of other minor bugs
+ * Start to prepare for 0.7 with HTTP protocol and new settings
+
+0.6.6:
+ * Removed issue with BUFSIZ limitation in the controller output
+ * Simplify logging symbols escaping
+ * Adjusted weights for several rules
+ * Improve spamhaus rbl support
+ * Removed PBL for received headers checks
+ * Added hfilter module that performs various HELO and IP checks.
+ * Rspamd can now be reloaded by HUP signal
+ * Fuzzy storage should expire hashes properly
+ * Build system has been reworked for better supportof pkg-config
+ * Various minor bugfixes
+
+0.6.5:
+ * Fixed critical bug in DNS resolver, introduced in 0.6.4
+ * Improved multimap and rbl plugins to skip
+ * Add dns_sockets option for tuning sockets per server in DNS resolver
+ * Improved packages for rspamd
+
+0.6.4:
+ * Added io channels for DNS request to balance load and reduce id
+ collisions chance
+ * Fixed a bug in SPF filter that may cause core dump in specific
+ circumstances
+ * FIxed default config for rbl module
+ * It is possible to get a list of rspamc commands with their descriptions
+ * Added SORBS bl to the default config
+ * 2tld file for surbl module has been significantly extended
+ * Perl modules has been removed from the code.
+ * Fixed an issue in libucl when parsing macros
+
+0.6.3:
+ * Fixed issues with DNS:
+ - labels decompression algorithm was fixed;
+ - added resolve_mx to lua API;
+ - fixed modules that use DNS.
+ * Lua modules once_received and emails reworked for new resolver API and UCL.
+ * Debian package was polished.
+ * Fixed a bug in fuzzy_check module that prevents correct processing messages
+ without valid parts.
+
+0.6.2:
+ * Fuzzy check module has been reworked:
+ - now fuzzy_check operates with a group of rules, that define which
+ servers sre to be checked;
+ - it is possible to specify read_only groups to avoid learning errors;
+ - turn fuzzy_check to one_shot mode permanently;
+ - fuzzy_check module configuration is now incompatible with the previous
+ versions.
+ * Imported bugfixes from libucl.
+ * Fixed whitelist plugin.
+ * Fixed statfiles resizing.
+ * Improved logging initialization order.
+ * Fixed race condition in the controller worker.
+
+0.6.1:
+ * Critical bugfixes:
+ - fixed build system;
+ - fixed in_class setting in bayes learning;
+
+0.6.0:
+ * Use UCL instead xml for configuration (https://github.com/vstakhov/libucl)
+ * Fix statistics module normalization
+ * Rework the many modules for the new configuration:
+ - surbl has incompatible configuration;
+ - fuzzy_check has incompatible configuration;
+ - multimap has now new configuration;
+ - received_rbl is removed and replaced by rbl module.
+ * Removed deprecated options:
+ - statfile_pool_size;
+ - action and required/reject score for a metric.
+ * Simplify build system and unify configuration for all platforms.
+ * Improved URL detector (reduced false positive rate).
+ * Lua 5.2 is now the default and fully supported version.
+ * Tons of bugfixes and minor improvements.
+
+0.5.6:
+ * Fix bug with counters incrementing in rolling history
+ * Detect expl and exp2l as some systems do not have it
+ * Support input streams without Content-Length
+ * Implement counters output via rspamc and controller interface
+ * Fix bug with udp sockets in fuzzy storage
+
+0.5.5:
+ * New bayes normalizator based on inverse chi-square function
+ * Various fixes to fuzzy storage
+ * Allow update fuzzy storage only from specific IP addresses
+ * Better support of IPv6 and address selection algorithms
+ * Add CentOS spec file
+
+0.5.4:
+ * Fixed issues with diff algorithm
+ * Added support of RRD statistics
+ * Add webui worker for interface interaction
+ * Fix a lot of issues with dynamic conf
+ * Fix critical memory leak in settings code
+ * Improve stability of the system
+
+0.5.3:
+ * Added dynamic options
+ * Added advanced metaclassfication
+ * Added RESTfull API for controller
+ * Improved hashing algorithms
+ * Various fixes for rspamc client:
+ - allow interacting with unix sockets
+ - librspamdclient major cleanup
+ - bayes is now default classifier
+0.5.2:
+ * Added lua bindings for:
+ - basic mime parts, that allows checkign attachements for example;
+ - DNS resolver;
+ * Existing lua bindings now works without task object allowing to use them
+ in custom code.
+ * Threads system was reworked to avoid global lua interpreter lock.
+ * DKIM module now converts all line endings to CRLF how opendkim does.
+ * URL detector is now more accurate for text parts.
+ * Several critical bugs and memory leaks were fixed.
+
+0.5.1:
+ * Added lua worker type to handle network connections in lua.
+ * Added lua bindings for async_session, IO dispatcher, memory_pool, and
+ worker.
+ * Composites can now uses other composites in expressions.
+ * Fixes for debian package and for FreeBSD >= 9.1
+ * Add support of gmime-2.6 if gmime-2.4 is not found.
+ * Improve url detection and phishing detection.
+ * Add lua mime_part library to get an access to all message part
+ attributes (like filename, length, type).
+
+0.5.0:
+ * Added SMTP lightweight balancing proxy with XCLIENT support.
+ * Added lua bindings for upstreams objects and API.
+ * New pre-filters are implemented to support initial checking for messages.
+ * Added ratelimit plugin that uses redis protocol to store data.
+ * Added ipv6 support to spf and some other modules.
+ * Unbreak spf plugin.
+ * Allow options with the same name be threated as list.
+ * DKIM plugin an parsing code was added.
+ * Separate build system to put logic in several shared libraries.
+ * Many bugfixes.
+
+0.3.2:
+ * Add error handling for regexps
+ * Fix quit command in controller interface
+ * Write symbols weights to rspamc output
+ * Improve logic of selecting rspamc version
+ * Do not try to parse broken DNS replies
+ * Add 'raw' flag to FROM_EXCESS_BASE64 rule (requested by citrin)
+ * Output message id in rspamc reply
+ * Fix inserting composite symbol
+ * Fix output of log line
+ * Document composites
+ * Add logging for fuzzy checks
+ * Add logging for learning
+ * Improve logic of learning messages - do not learn more than specific threshold
+ * Fix inserting results for symbols that were incorrectly (for example more than 1 time) defined in config file
+ * Do not output control characters if output is not terminal
+ * Fix some logic errors in learning
+ * Consider lua plugins errors as fatal configuration errors
+ * Fix writing message id during fuzzy_add command
+ * Display weight of symbols correctly
+ * Fixes to winnow learning
+ * One more try to improve accuracy of winnow algorithm
+ * Add bayesian classifier (initial version)
+ * Remove normalizer as it is winnow specific thing, so all statistic algorithms now returns value from 0 to 1
+ * Some fixes to fuzzy hashes expiration:
+ * Fix assertion while look up value in NULL hash (found by cirtin)
+ * Fix normalization for systems that have not tanhl function
+ * Ignore rfc822 group addresses
+ * Move images library to core rspamd
+ * Add lua api to access images properties
+ * Add post filters to lua API - filters that would be called after all message's processing
+ * Add ability to check for specified symbol in task results from lua
+ * Add ability to check for metric's results from lua
+ * Add ability to learn specified statfile form lua
+ * Add ability to extract filename and size of images from lua
+ * Fix assertion while extracting internet address
+ * Fix races in fuzzy storage
+ * Make spf parser case insensitive
+ * Add ability to check hashes of selected mime types
+ * Add ability to set minimum size in bytes for mime types
+ * Add ability to set minimum dimensions for images
+ * Assume all text/* content types as text/plain
+ * Fix getting data wrapper for gmime24
+ * Many fixes to statfile synchronization system
+ * Fixed statfile pool initialization and synchronization with disk
+ * Prepare 0.3.2
+ * Fix Mail::Rspamd::Config for new rspamd features
+ * Use Mail::Rspamd::Config in rspamc client
+ * Write user's name to rspamd log
+ * Prepare rspamd build infrastructure for creating rpm and deb packages
+ * Fix depends
+ * Add start script for linux systems
+ * Fix shared usage of statfiles
+ * Add invalidation of statfiles in case of learning, so now statfiles
+ * This should fix shared usage of statfile pool by several processes
+ * Fix misprint (reported by az)
+ * Fix stupid error when all checks can be done in a single pass
+ * New trie based url scanner (based on libcamel)
+ * Small fixes to rspamd perl client
+ * Write fuzzy hashes info to log
+ * Add trie interface to lua api
+ * Explain sample config and cleanup it
+
+0.3.1:
+ * Add modules documentation
+ * Continue implementing smtp proxy
+ * Implement new learning system, now rspamd should be much more intelligent while learning messages
+ * Convert statistic sums to use long double for counters
+ * Use hyperbolic tangent for internal normalizer
+ * In classify normalize result after comparing, not before
+ * New symbols sorter
+ * Fix strict aliasing while compiling with optimization
+ * Fix tanhl detection for platforms that have not implementation of it
+ * Remove several compile warnings
+ * Add experimental support of dynamic rules to regexp module
+ * Document views configuration
+ * Several fixes to documentation
+ * Add more logic for dynamic rules
+ * Add documentation for dynamic rules
+ * Add ability to make negations in networks in dynamic rules
+ * Clean up cache items correctly
+ * Implement basic SMTP dialog:
+ * Implement interaction with smtp upstream (with support of XCLIENT)
+ * Check messages received via smtp proxy
+ * Add support for sendfile in io dispatcher
+ * Fix issues with compatibility of worker_task and smtp proxy
+ * Proxy DATA command
+ * Fix SMTP
+ * Change metric logic
+ * Completely remove lex/yacc readers for config
+ * Make common sense of metric/action and symbols
+ * Sync changes with all plugins
+ * Incorrectly removed in previous commit
+ * Fix misprint (by Andrej Zverev)
+ * announce the "password" keyword in usage list
+ * Implement initial version of greylisting triplets storage
+ * Fix issues with smtp worker
+ * Fix QUIT command in SMTP worker
+ * Some fixes about new metrics system (may be incomplete)
+ * Get weights of symbol from default metric for symbols cache
+ * Fix setting task->from/task->rctp in smtp client
+ * Copy from and rcpt correctly
+ * Some performance improvements to IO dispatcher (do not drain the whole buffer after a single line readed)
+ * Fix smtp data input
+ * Fix misprint
+ * Add limit of maximum allowed smtp session errors
+ * New logic of SURBL module:
+ * Use system mkstemp(3) on systems where it is available as glib implementation
+ * Try to fix memmove issues in io dispatcher
+ * Remove debug from SURBL module
+ * Rewrite buffered input for line policy (again)
+ * Fix issue with links that are ip addresses in numeric form in surbl
+ * On Darwin use BSD style sendfile definition
+ * Reorganize platform specific knobs in CMakeLists
+ * Use gettimeofday on systems that have not clock_getres
+ * Use ftime for dns trans id generation on systems without clock_getres
+ * Darwin sendfile(2) support
+ * TIMEDB->TIMEB
+ * More to previous commit
+ * Pass env from main() arguments instead of platform specific global environ
+ * Fix compatibility issues
+ * Fix -lintl detection
+ * Init some variables to avoid problems
+ * Remove garbadge (gnome terminal sucks)
+ * Add more information about why we drop smtp connection
+ * Fix mkstemp call
+ * Send to upstream QUIT command at the end of session
+ * Check return value of each rspamd_dispatcher_write as in case of write errors sessions can be destroyed early
+ * Fix states in smtp dialog
+ * Use rspamd_snprintf instead of libc one
+ * Fix URLS command
+ * Fix reconfigure process of surbl module
+ * Fix destroying smtp session (unmap memory and do not delete pool early)
+ * Delete pool after using its variables
+ * Delay timer must be registered in async session to correctly handle connection termination
+ * Register dns requests in session too
+ * Make session before registering events
+ * Remove events in handlers
+ * Add ability to set filters for smtp worker for each smtp stage
+ * Add very initial version of DNS resolver (many things to be done)
+ * Announce weights and sync
+ * Fix few typo
+ * Understand short names of facility in logging config
+ * Add ability to make whitelist for spf checks
+ * Misprint != -> ==
+ * Handle lua tag in way that it is not required to write additional text:
+ * Strip all starting whitespace symbols from xml texts
+ * Fix stupid bug in calculating buffer length while reading file maps
+ * Add resolv.conf parsing into dns.c
+ * Fix microseconds<->milliseconds conversions
+ * Take callback argument in Mail::Rspamd::Client for processing files and directories
+ * Print results if rspamc is called for a directory
+ * Fix stupid error with surbl module reconfig (another one, blame me)
+ * Do not show duplicate urls in url header
+ * Fix detection of numeric urls (reported by citrin)
+ * Write real time of message's scan to log (not only virtual)
+ * Fix chartable module in utf mode
+ * Fix parsing of some broken urls
+ * Add ability to test regexp with 'T' flag
+ * Write more code for DNS resolver:
+ * Make DNS resolver working
+ * Many improvements to rspamd test suite: now it CAN be used for testing rspamd functionality
+ * Write DNS resolver tests
+ * Fix issues with memory_pool mutexes and with creating of statfiles
+ * Forgotten in previous commit
+ * Add support for parsing SPF and SRV records
+ * Fix PTR parsing
+ * Add tests
+ * Make SURBL module to use rspamd dns module
+ * Several fixes to DNS logic
+ * Remove evdns and use only rspamd resolver
+ * Very hard to detect problem with race among error in socket and destroying task while we are writing to socket and go through a hash table
+ * Fix resolving in smtp module
+ * Init events before configuring resolver in smtp worker
+ * Set resolver inside task
+ * Fix reload signal (reported by citrin)
+ * Some improvements to redirector
+ * Call has_forked method to inform POE about fork
+ * Fix lua DNS code
+ * Decompress labels in DNS packets more strictly
+ * Fix some problems with TXT records
+ * Try to fix removing of DNS events
+ * Do not insert unparsed RR's into reply
+ * Calling callbacks may cause destroying session from which we are calling callback so we MUST call callback as the latest action
+ * Fix check_smtp_data function
+ * Add ability to make views by recipient
+ * Add ability to set metric's action from config file
+ * Fix bug with writing garbadge if message has no urls or no messages
+ * Fix bug with incorrect behaviour of compare_parts_distance function
+ * Add ability to assign several actions to one metric
+ * Report action in rspamc protocol
+ * Mail::Rspamd::Client and rspamc can now understand Action header too
+ * Write action to log as well
+ * Make valgrind happy about comparing symbols
+ * Add more debug to comparing parts distance function
+ * Write action even if message has no symbols
+ * Make improvements to HTML entities decoder: now it replaces entities with common characters and
+ * Add -d option to force debug output
+ * Assume 7bit as default transfer encoding
+ * Do not overwrite lua plugins and configs if they already exists in target directory
+ * Improve logging
+ * Write queue id to log
+ * Remove test messages from cmake
+ * Reopen log file by USR1 signal
+ * Add reopenlog method to FreeBSD rc script
+ * Adopt foreach for cmake 2.6
+ * Fix to rc script
+ * Do not try to resolve names with several dots in a row
+ * Fix surbl request formatting for ip addresses
+ * Handle cases of broken requests
+ * Fix problems with parsing compressed names
+ * Fix TXT records parsing
+ * Fix expanding spf macros that may fail in rare cases
+ * Fix another error with early task destroying
+ * Handle empty from header
+ * Improve reopenlog command in rc script
+ * Strip trailing whitespace characters in Mail::Rspamd::Client
+ * Use ungreedy match to strip trailing whitespaces
+ * Stupid error in calculation compressed label length
+ * Some optimizations to client library
+ * Do not compare empty parts
+ * Empty and non-empty parts are always different
+ * Save in regexp cache the whole regexp with header name (if exists) and with flags
+ * Add rspamd_log variable to lua plugins to access logging functions
+ * Each part in rspamd task now can have parent part
+ * Check for parts distance only for multipart/alternative subparts
+ * Do not check attachements even if they are text (but attached as file)
+ * Do not die if write (2) returned ENOSPACE while doing logging, turn on throttling mode instead (1 write try in a second)
+ * Add ability to turn on debug for specific symbols
+ * Add ability to configure dns timeouts and dns retransmits in config file
+ * More debug
+ * Fix extracting arguments in lua logger interface
+ * Turn off debug_ip during reload if it was disabled by new config
+ * Improve lua logging
+ * Pre-init symbols cache when rereading config
+ * Fix lua representing of invalid ip (nil, not 255.255.255.255)
+ * Fix R_TO_SEEMS_AUTO rule (by citrin)
+ * Add multimap lua plugin
+ * Fix some multimap issues
+ * Try to save images hashes to fuzzy storage to stop some annoying spammers
+ * Allocate some more bytes for read buffer to avoid incorrect behavoiur
+ * Add ability to check dns black lists by multimap module
+ * Add multimap documentation
+ * Fix labels parsing
+ * Another try to save regexps in cache correctly
+ * Improve test logs for regexps
+ * Fix parsing txt records to avoid reading of uninitialized data
+ * Fix error with writing symbols cache file
+ * Fix error while working in utf mode when raw regexps was not created properly
+ * Do not add extra byte while converting text to utf
+ * Add error handling for regexps
diff --git a/LICENSE.md b/LICENSE.md
new file mode 100644
index 0000000..537fde8
--- /dev/null
+++ b/LICENSE.md
@@ -0,0 +1,170 @@
+Apache License
+==============
+
+_Version 2.0, January 2004_
+_&lt;<http://www.apache.org/licenses/>&gt;_
+
+### Terms and Conditions for use, reproduction, and distribution
+
+#### 1. Definitions
+
+“License†shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+“Licensor†shall mean the copyright owner or entity authorized by the copyright
+owner that is granting the License.
+
+“Legal Entity†shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, “control†means **(i)** the power, direct or
+indirect, to cause the direction or management of such entity, whether by
+contract or otherwise, or **(ii)** ownership of fifty percent (50%) or more of the
+outstanding shares, or **(iii)** beneficial ownership of such entity.
+
+“You†(or “Yourâ€) shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+“Source†form shall mean the preferred form for making modifications, including
+but not limited to software source code, documentation source, and configuration
+files.
+
+“Object†form shall mean any form resulting from mechanical transformation or
+translation of a Source form, including but not limited to compiled object code,
+generated documentation, and conversions to other media types.
+
+“Work†shall mean the work of authorship, whether in Source or Object form, made
+available under the License, as indicated by a copyright notice that is included
+in or attached to the work (an example is provided in the Appendix below).
+
+“Derivative Works†shall mean any work, whether in Source or Object form, that
+is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative Works
+shall not include works that remain separable from, or merely link (or bind by
+name) to the interfaces of, the Work and Derivative Works thereof.
+
+“Contribution†shall mean any work of authorship, including the original version
+of the Work and any modifications or additions to that Work or Derivative Works
+thereof, that is intentionally submitted to Licensor for inclusion in the Work
+by the copyright owner or by an individual or Legal Entity authorized to submit
+on behalf of the copyright owner. For the purposes of this definition,
+“submitted†means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems, and
+issue tracking systems that are managed by, or on behalf of, the Licensor for
+the purpose of discussing and improving the Work, but excluding communication
+that is conspicuously marked or otherwise designated in writing by the copyright
+owner as “Not a Contribution.â€
+
+“Contributor†shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+#### 2. Grant of Copyright License
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the Work and such
+Derivative Works in Source or Object form.
+
+#### 3. Grant of Patent License
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable (except as stated in this section) patent license to make, have
+made, use, offer to sell, sell, import, and otherwise transfer the Work, where
+such license applies only to those patent claims licensable by such Contributor
+that are necessarily infringed by their Contribution(s) alone or by combination
+of their Contribution(s) with the Work to which such Contribution(s) was
+submitted. If You institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work or a
+Contribution incorporated within the Work constitutes direct or contributory
+patent infringement, then any patent licenses granted to You under this License
+for that Work shall terminate as of the date such litigation is filed.
+
+#### 4. Redistribution
+
+You may reproduce and distribute copies of the Work or Derivative Works thereof
+in any medium, with or without modifications, and in Source or Object form,
+provided that You meet the following conditions:
+
+* **(a)** You must give any other recipients of the Work or Derivative Works a copy of
+this License; and
+* **(b)** You must cause any modified files to carry prominent notices stating that You
+changed the files; and
+* **(c)** You must retain, in the Source form of any Derivative Works that You distribute,
+all copyright, patent, trademark, and attribution notices from the Source form
+of the Work, excluding those notices that do not pertain to any part of the
+Derivative Works; and
+* **(d)** If the Work includes a “NOTICE†text file as part of its distribution, then any
+Derivative Works that You distribute must include a readable copy of the
+attribution notices contained within such NOTICE file, excluding those notices
+that do not pertain to any part of the Derivative Works, in at least one of the
+following places: within a NOTICE text file distributed as part of the
+Derivative Works; within the Source form or documentation, if provided along
+with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents of
+the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works that
+You distribute, alongside or as an addendum to the NOTICE text from the Work,
+provided that such additional attribution notices cannot be construed as
+modifying the License.
+
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction, or
+distribution of Your modifications, or for any such Derivative Works as a whole,
+provided Your use, reproduction, and distribution of the Work otherwise complies
+with the conditions stated in this License.
+
+#### 5. Submission of Contributions
+
+Unless You explicitly state otherwise, any Contribution intentionally submitted
+for inclusion in the Work by You to the Licensor shall be under the terms and
+conditions of this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify the terms of
+any separate license agreement you may have executed with Licensor regarding
+such Contributions.
+
+#### 6. Trademarks
+
+This License does not grant permission to use the trade names, trademarks,
+service marks, or product names of the Licensor, except as required for
+reasonable and customary use in describing the origin of the Work and
+reproducing the content of the NOTICE file.
+
+#### 7. Disclaimer of Warranty
+
+Unless required by applicable law or agreed to in writing, Licensor provides the
+Work (and each Contributor provides its Contributions) on an “AS IS†BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
+including, without limitation, any warranties or conditions of TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
+solely responsible for determining the appropriateness of using or
+redistributing the Work and assume any risks associated with Your exercise of
+permissions under this License.
+
+#### 8. Limitation of Liability
+
+In no event and under no legal theory, whether in tort (including negligence),
+contract, or otherwise, unless required by applicable law (such as deliberate
+and grossly negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special, incidental,
+or consequential damages of any character arising as a result of this License or
+out of the use or inability to use the Work (including but not limited to
+damages for loss of goodwill, work stoppage, computer failure or malfunction, or
+any and all other commercial damages or losses), even if such Contributor has
+been advised of the possibility of such damages.
+
+#### 9. Accepting Warranty or Additional Liability
+
+While redistributing the Work or Derivative Works thereof, You may choose to
+offer, and charge a fee for, acceptance of support, warranty, indemnity, or
+other liability obligations and/or rights consistent with this License. However,
+in accepting such obligations, You may act only on Your own behalf and on Your
+sole responsibility, not on behalf of any other Contributor, and only if You
+agree to indemnify, defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason of your
+accepting any such warranty or additional liability.
+
+_END OF TERMS AND CONDITIONS_ \ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..23c0618
--- /dev/null
+++ b/README.md
@@ -0,0 +1,54 @@
+# <a href="https://rspamd.com"><img src="https://rspamd.com/img/rspamd_logo_black.png" alt="Rspamd" width="220px"/></a>
+
+[![DroneCI](https://ci.rspamd.com/api/badges/rspamd/rspamd/status.svg)](https://ci.rspamd.com/rspamd/rspamd)
+
+
+## Introduction
+
+[Rspamd](https://rspamd.com) is an advanced spam filtering system and email processing framework that allows evaluation of messages by a number of
+rules including regular expressions, statistical analysis and custom services
+such as URL black lists. Each message is analysed by Rspamd and given a verdict that might be used by MTA for further processing (e.g. to reject a message, or add a special header indicating spam) along with other information, such as possible DKIM signature or modifications suggested for a message.
+
+Rspamd can act as a [Milter](https://en.wikipedia.org/wiki/Milter) allowing direct interaction with popular MTA systems, such as Postfix or Sendmail.
+
+Rspamd is designed to process hundreds of messages per second simultaneously, and provides a number of
+useful features including a comprehensive [Lua API](https://rspamd.com/doc/lua/) that allows access to messages processing in various aspects as well as [asynchronous](https://rspamd.com/doc/lua/sync_async.html) network API to access external resources, such as DNS, HTTP or even generic TCP/UDP services.
+
+
+## Getting Started
+
+A good starting point to study how to install and configure Rspamd is [the quick start guide](https://rspamd.com/doc/quickstart.html).
+
+Rspamd is [packaged](https://rspamd.com/downloads.html) for the major Linux distributions, and is also available via [FreeBSD ports](https://freshports.org/mail/rspamd), NetBSD [pkgsrc](https://pkgsrc.org) and [OpenBSD ports](http://openports.se/mail/rspamd).
+
+We advice to use packages provided by Rspamd project if available for your OS instead of packages that might be provided by some Linux distributives, as they are usually out of date and does not provide the desired spam filtering quality nor supported by Rspamd project.
+
+## Spam filtering features
+
+Rspamd is shipped with various spam filtering modules and features enabled just out of the box.
+The full list of built-in modules could be found in the [Rspamd documentation](https://rspamd.com/doc/modules/).
+
+If that is not enough, Rspamd provides an extensive [Lua API](https://rspamd.com/doc/lua/) to write your own rules and plugins: <https://rspamd.com/doc/tutorials/writing_rules.html>
+
+## License
+
+This project is licensed under the Apache 2.0 License - see the [LICENSE.md](LICENSE.md) file for details
+
+## Contributing
+
+Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details on the process for submitting pull requests to us.
+
+Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in `rspamd` by you, as defined in the APACHE 2.0 license, shall be licensed as above, without any additional terms or conditions.
+
+## Authors
+
+* **Vsevolod Stakhov** - [vstakhov](https://github.com/vstakhov)
+
+See also the list of [contributors](AUTHORS.md) who participated in this project.
+
+
+## References
+
+* Home site: <https://rspamd.com>
+* Development: <https://github.com/rspamd/rspamd>
+* Site repository: <https://github.com/rspamd/rspamd.com>
diff --git a/blas-config.h.in b/blas-config.h.in
new file mode 100644
index 0000000..4f83797
--- /dev/null
+++ b/blas-config.h.in
@@ -0,0 +1,11 @@
+#ifndef RSPAMD_BLAS_CONFIG_H_IN
+#define RSPAMD_BLAS_CONFIG_H_IN
+
+#cmakedefine HAVE_CBLAS_SGEMM 1
+#cmakedefine HAVE_CBLAS_SAXPY 1
+#cmakedefine HAVE_OPENBLAS_SET_NUM_THREADS 1
+#cmakedefine HAVE_BLI_THREAD_SET_NUM_THREADS 1
+#cmakedefine HAVE_CBLAS_H 1
+#cmakedefine HAVE_CBLAS 1
+
+#endif \ No newline at end of file
diff --git a/clang-plugin/CMakeLists.txt b/clang-plugin/CMakeLists.txt
new file mode 100644
index 0000000..0d61c79
--- /dev/null
+++ b/clang-plugin/CMakeLists.txt
@@ -0,0 +1,42 @@
+
+IF (ENABLE_CLANG_PLUGIN MATCHES "ON")
+ # Clang plugin for static analysis
+ PROJECT(RspamdClangPlugin CXX)
+ if (NOT "${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
+ MESSAGE(FATAL_ERROR "Cannot build clang plugin when compiler is not clang")
+ endif ()
+ FIND_PACKAGE(LLVM REQUIRED CONFIG)
+
+ MESSAGE(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
+ MESSAGE(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
+ INCLUDE_DIRECTORIES(${LLVM_INCLUDE_DIRS})
+ ADD_DEFINITIONS(${LLVM_DEFINITIONS})
+
+ find_package(Clang REQUIRED)
+
+ set(clang_libs clang)
+
+ message(STATUS "Found LibClang in: ${CLANG_INSTALL_PREFIX}")
+ include_directories(${CLANG_INCLUDE_DIRS})
+
+ SET(CLANGPLUGINSRC plugin.cc printf_check.cc)
+
+ ADD_LIBRARY(rspamd-clang SHARED ${CLANGPLUGINSRC})
+ IF (SANITIZE)
+ set (CMAKE_C_FLAGS "")
+ set (CMAKE_CXX_FLAGS "")
+ set (CMAKE_EXE_LINKER_FLAGS "")
+ ENDIF()
+
+
+ find_library(found_LLVM LLVM HINTS ${LLVM_LIBRARY_DIRS})
+ if(found_LLVM)
+ target_link_libraries(rspamd-clang PRIVATE ${LLVM})
+ else()
+ # XXX not tested yet
+ llvm_map_components_to_libnames(llvm_libs support core)
+ target_link_libraries(rspamd-clang PRIVATE ${llvm_libs})
+ endif()
+
+ target_link_libraries(rspamd-clang PRIVATE ${clang_libs})
+ENDIF()
diff --git a/clang-plugin/plugin.cc b/clang-plugin/plugin.cc
new file mode 100644
index 0000000..9236764
--- /dev/null
+++ b/clang-plugin/plugin.cc
@@ -0,0 +1,69 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "clang/Frontend/FrontendPluginRegistry.h"
+#include "clang/AST/AST.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Sema/Sema.h"
+#include "llvm/Support/raw_ostream.h"
+#include "printf_check.h"
+
+
+using namespace clang;
+
+namespace rspamd {
+
+ class RspamdASTConsumer : public ASTConsumer {
+ CompilerInstance &Instance;
+
+ public:
+ RspamdASTConsumer (CompilerInstance &Instance)
+ : Instance (Instance)
+ {
+ }
+
+ void HandleTranslationUnit (ASTContext &context) override
+ {
+ rspamd::PrintfCheckVisitor v(&context, Instance);
+ v.TraverseDecl (context.getTranslationUnitDecl ());
+ }
+ };
+
+ class RspamdASTAction : public PluginASTAction {
+ protected:
+ std::unique_ptr <ASTConsumer> CreateASTConsumer (CompilerInstance &CI,
+ llvm::StringRef) override
+ {
+ return std::make_unique<RspamdASTConsumer> (CI);
+ }
+
+ bool ParseArgs (const CompilerInstance &CI,
+ const std::vector <std::string> &args) override
+ {
+ return true;
+ }
+
+ void PrintHelp (llvm::raw_ostream &ros)
+ {
+ ros << "Nothing here\n";
+ }
+
+ };
+
+}
+
+static FrontendPluginRegistry::Add <rspamd::RspamdASTAction>
+ X ("rspamd-ast", "rspamd ast checker");
diff --git a/clang-plugin/printf_check.cc b/clang-plugin/printf_check.cc
new file mode 100644
index 0000000..e39cec8
--- /dev/null
+++ b/clang-plugin/printf_check.cc
@@ -0,0 +1,822 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <sys/types.h>
+#include "printf_check.h"
+#include "clang/AST/AST.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+#include <sstream>
+#include <ctype.h>
+#include <signal.h>
+#include <assert.h>
+#include <cstdint>
+
+using namespace clang;
+
+namespace rspamd {
+ struct PrintfArgChecker;
+
+ static bool cstring_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool int_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool long_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool size_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool char_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool double_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool long_double_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool pointer_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool pid_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool time_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool int64_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool int32_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool tok_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool fstring_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool gstring_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ static bool gerr_arg_handler (const Expr *arg,
+ struct PrintfArgChecker *ctx);
+
+ using arg_parser_t = bool (*) (const Expr *, struct PrintfArgChecker *);
+
+ static void
+ print_error (const char *err, const Expr *e, const ASTContext *ast,
+ CompilerInstance *ci)
+ {
+ auto loc = e->getExprLoc ();
+ auto &diag = ci->getDiagnostics ();
+ auto id = diag.getCustomDiagID (DiagnosticsEngine::Error,
+ "format query error: %0");
+ diag.Report (loc, id) << err;
+ }
+
+ static void
+ print_warning (const char *err, const Expr *e, const ASTContext *ast,
+ CompilerInstance *ci)
+ {
+ auto loc = e->getExprLoc ();
+ auto &diag = ci->getDiagnostics ();
+ auto id = diag.getCustomDiagID (DiagnosticsEngine::Warning,
+ "format query warning: %0");
+ diag.Report (loc, id) << err;
+ }
+
+ static void
+ print_remark (const char *err, const Expr *e, const ASTContext *ast,
+ CompilerInstance *ci)
+ {
+ auto loc = e->getExprLoc ();
+ auto &diag = ci->getDiagnostics ();
+ auto id = diag.getCustomDiagID (DiagnosticsEngine::Remark,
+ "format query warning: %0");
+ diag.Report (loc, id) << err;
+ }
+
+ struct PrintfArgChecker {
+ private:
+ arg_parser_t parser;
+ public:
+ int width;
+ int precision;
+ bool is_unsigned;
+ ASTContext *past;
+ CompilerInstance *pci;
+
+ PrintfArgChecker (arg_parser_t _p, ASTContext *_ast, CompilerInstance *_ci) :
+ parser (_p), past (_ast), pci(_ci)
+ {
+ width = 0;
+ precision = 0;
+ is_unsigned = false;
+ }
+
+ virtual ~PrintfArgChecker ()
+ {
+ }
+
+ bool operator() (const Expr *e)
+ {
+ return parser (e, this);
+ }
+ };
+
+ class PrintfCheckVisitor::impl {
+ std::unordered_map<std::string, unsigned int> printf_functions;
+ std::unordered_set<char> format_specs;
+ ASTContext *pcontext;
+ CompilerInstance *ci;
+
+ std::unique_ptr <PrintfArgChecker> parseFlags (const std::string &flags,
+ const Expr *e)
+ {
+ auto type = flags.back ();
+
+ switch (type) {
+ case 's':
+ return std::make_unique<PrintfArgChecker> (cstring_arg_handler,
+ this->pcontext, this->ci);
+ case 'd':
+ return std::make_unique<PrintfArgChecker> (int_arg_handler,
+ this->pcontext, this->ci);
+ case 'z':
+ return std::make_unique<PrintfArgChecker> (size_arg_handler,
+ this->pcontext, this->ci);
+ case 'l':
+ return std::make_unique<PrintfArgChecker> (long_arg_handler,
+ this->pcontext, this->ci);
+ case 'f':
+ case 'g':
+ return std::make_unique<PrintfArgChecker> (double_arg_handler,
+ this->pcontext, this->ci);
+ case 'F':
+ case 'G':
+ return std::make_unique<PrintfArgChecker> (
+ long_double_arg_handler,
+ this->pcontext, this->ci);
+ case 'c':
+ return std::make_unique<PrintfArgChecker> (char_arg_handler,
+ this->pcontext, this->ci);
+ case 'p':
+ return std::make_unique<PrintfArgChecker> (pointer_arg_handler,
+ this->pcontext, this->ci);
+ case 'P':
+ return std::make_unique<PrintfArgChecker> (pid_arg_handler,
+ this->pcontext, this->ci);
+ case 't':
+ return std::make_unique<PrintfArgChecker> (time_arg_handler,
+ this->pcontext, this->ci);
+ case 'L':
+ return std::make_unique<PrintfArgChecker> (int64_arg_handler,
+ this->pcontext, this->ci);
+ case 'D':
+ return std::make_unique<PrintfArgChecker> (int32_arg_handler,
+ this->pcontext, this->ci);
+ case 'T':
+ return std::make_unique<PrintfArgChecker> (tok_arg_handler,
+ this->pcontext, this->ci);
+ case 'V':
+ return std::make_unique<PrintfArgChecker> (fstring_arg_handler,
+ this->pcontext, this->ci);
+ case 'v':
+ return std::make_unique<PrintfArgChecker> (gstring_arg_handler,
+ this->pcontext, this->ci);
+ case 'e':
+ return std::make_unique<PrintfArgChecker> (gerr_arg_handler,
+ this->pcontext, this->ci);
+ default: {
+ auto err_msg = std::string ("unknown parser flag: ") + type;
+ print_warning (err_msg.c_str(),
+ e, this->pcontext, this->ci);
+ break;
+ }
+ }
+
+ return nullptr;
+ }
+
+ std::shared_ptr <std::vector<PrintfArgChecker>>
+ genParsers (const StringRef query, const Expr *e)
+ {
+ enum {
+ ignore_chars = 0,
+ read_percent,
+ read_width,
+ read_precision,
+ read_arg
+ } state = ignore_chars;
+ int width, precision;
+ std::string flags;
+
+ auto res = std::make_shared<std::vector<PrintfArgChecker> > ();
+
+ for (auto citer = query.begin(); citer != query.end(); ++citer) {
+ auto c = *citer;
+
+ switch (state) {
+ case ignore_chars:
+ if (c == '%') {
+ state = read_percent;
+ flags.clear ();
+ width = precision = 0;
+ }
+ break;
+ case read_percent:
+ if (isdigit (c)) {
+ state = read_width;
+ width = c - '0';
+ }
+ else if (c == '.') {
+ state = read_precision;
+ precision = c - '0';
+ }
+ else if (c == '*') {
+ /* %*s - need integer argument */
+ res->emplace_back (int_arg_handler, this->pcontext,
+ this->ci);
+
+ if (*std::next (citer) == '.') {
+ ++citer;
+ state = read_precision;
+ }
+ else {
+ state = read_arg;
+ }
+ }
+ else if (c == '%') {
+ /* Percent character, ignore */
+ state = ignore_chars;
+ }
+ else {
+ // Rewind iter
+ --citer;
+ state = read_arg;
+ }
+ break;
+ case read_width:
+ if (isdigit (c)) {
+ width *= 10;
+ width += c - '0';
+ }
+ else if (c == '.') {
+ state = read_precision;
+ precision = c - '0';
+ }
+ else {
+ // Rewind iter
+ --citer;
+ state = read_arg;
+ }
+ break;
+ case read_precision:
+ if (isdigit (c)) {
+ precision *= 10;
+ precision += c - '0';
+ }
+ else if (c == '*') {
+ res->emplace_back (int_arg_handler, this->pcontext,
+ this->ci);
+ state = read_arg;
+ }
+ else {
+ // Rewind iter
+ --citer;
+ state = read_arg;
+ }
+ break;
+ case read_arg:
+ auto found = format_specs.find (c);
+ if (found != format_specs.end () || !isalpha (c)) {
+
+ if (isalpha (c)) {
+ flags.push_back (c);
+ }
+
+ auto handler = parseFlags (flags, e);
+
+ if (handler) {
+ auto handler_copy = *handler;
+ handler_copy.precision = precision;
+ handler_copy.width = width;
+ res->emplace_back (std::move (handler_copy));
+ }
+ else {
+ return nullptr;
+ }
+
+ if (c == '%') {
+ state = read_percent;
+ }
+ else {
+ state = ignore_chars;
+ }
+ flags.clear ();
+ width = precision = 0;
+ }
+ else {
+ flags.push_back (c);
+ }
+ break;
+ }
+ }
+
+ if (state == read_arg) {
+ auto handler = parseFlags (flags, e);
+
+ if (handler) {
+ auto handler_copy = *handler;
+ handler_copy.precision = precision;
+ handler_copy.width = width;
+ res->emplace_back (std::move (handler_copy));
+ }
+ else {
+ return nullptr;
+ }
+ }
+
+ return res;
+ }
+
+ public:
+ impl (ASTContext *_ctx, clang::CompilerInstance &_ci)
+ : pcontext (_ctx), ci(&_ci)
+ {
+ /* name -> format string position */
+ printf_functions = {
+ {"rspamd_printf", 0},
+ {"rspamd_default_log_function", 4},
+ {"rspamd_snprintf", 2},
+ {"rspamd_fprintf", 1},
+ {"rspamd_printf_gstring", 1},
+ {"rspamd_printf_fstring", 1},
+ {"rspamd_conditional_debug_fast", 6},
+ };
+
+ format_specs = {
+ 's', 'd', 'l', 'L', 'v', 'V', 'f', 'F', 'g', 'G',
+ 'T', 'z', 'D', 'c', 'p', 'P', 'e'
+ };
+ };
+
+ bool VisitCallExpr (CallExpr *E)
+ {
+ if (E->getCalleeDecl () == nullptr) {
+ print_remark ("cannot get callee decl",
+ E, this->pcontext, this->ci);
+ return true;
+ }
+ auto callee = dyn_cast<NamedDecl> (E->getCalleeDecl ());
+ if (callee == NULL) {
+ print_remark ("cannot get named callee decl",
+ E, this->pcontext, this->ci);
+ return true;
+ }
+
+ auto fname = callee->getNameAsString ();
+
+ auto pos_it = printf_functions.find (fname);
+
+ if (pos_it != printf_functions.end ()) {
+ const auto args = E->getArgs ();
+ auto pos = pos_it->second;
+ auto query = args[pos];
+
+ if (!query->isEvaluatable (*pcontext)) {
+ print_remark ("cannot evaluate query",
+ E, this->pcontext, this->ci);
+ /* It is not assumed to be an error */
+ return true;
+ }
+
+ clang::Expr::EvalResult r;
+
+ if (!query->EvaluateAsRValue (r, *pcontext)) {
+ print_warning ("cannot evaluate rvalue of query",
+ E, this->pcontext, this->ci);
+ return false;
+ }
+
+ auto qval = dyn_cast<StringLiteral> (
+ r.Val.getLValueBase ().get<const Expr *> ());
+ if (!qval) {
+ print_warning ("bad or absent query string",
+ E, this->pcontext, this->ci);
+ return false;
+ }
+
+ auto parsers = genParsers (qval->getString (), E);
+
+ if (parsers) {
+ if (parsers->size () != E->getNumArgs () - (pos + 1)) {
+ std::ostringstream err_buf;
+ err_buf << "number of arguments for " << fname
+ << " mismatches query string '" <<
+ qval->getString ().str ()
+ << "', expected " << parsers->size () <<
+ " args"
+ << ", got " <<
+ (E->getNumArgs () - (pos + 1))
+ << " args";
+ print_error (err_buf.str().c_str(), E, this->pcontext, this->ci);
+
+ return false;
+ }
+ else {
+ for (auto i = pos + 1; i < E->getNumArgs (); i++) {
+ auto arg = args[i];
+
+ if (arg) {
+ if (!parsers->at (i - (pos + 1)) (arg)) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ return true;
+ }
+ };
+
+ PrintfCheckVisitor::PrintfCheckVisitor (ASTContext *ctx,
+ clang::CompilerInstance &ci) :
+ pimpl{new impl (ctx, ci)}
+ {
+ }
+
+ PrintfCheckVisitor::~PrintfCheckVisitor ()
+ {
+ }
+
+ bool PrintfCheckVisitor::VisitCallExpr (clang::CallExpr *E)
+ {
+ return pimpl->VisitCallExpr (E);
+ }
+
+ /* Type handlers */
+ static bool
+ cstring_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ auto type = arg->getType ().split ().Ty;
+
+ if (!type->isPointerType ()) {
+ auto err_msg = std::string ("bad string argument for %s: ") +
+ arg->getType ().getAsString ();
+ print_error (err_msg.c_str(),
+ arg, ctx->past, ctx->pci);
+ return false;
+ }
+
+ auto ptr_type = type->getPointeeType ().split ().Ty;
+
+ if (!ptr_type->isCharType ()) {
+ /* We might have gchar * here */
+ auto desugared_type = ptr_type->getUnqualifiedDesugaredType ();
+ auto desugared_ptr_type = type->getUnqualifiedDesugaredType ();
+
+ if (!desugared_type || (!desugared_type->isCharType () &&
+ !desugared_ptr_type->isVoidPointerType ())) {
+ if (desugared_type) {
+ desugared_type->dump ();
+ }
+ auto err_msg = std::string ("bad string argument for %s: ") +
+ arg->getType ().getAsString ();
+ print_error (err_msg.c_str(),
+ arg, ctx->past, ctx->pci);
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ static bool
+ check_builtin_type (const Expr *arg, struct PrintfArgChecker *ctx,
+ const std::vector <BuiltinType::Kind> &k, const std::string &fmt)
+ {
+ auto type = arg->getType ().split ().Ty;
+
+ auto desugared_type = type->getUnqualifiedDesugaredType ();
+
+ if (!desugared_type->isBuiltinType ()) {
+ auto err_msg = std::string ("not a builtin type for ") + fmt + " arg: " +
+ arg->getType ().getAsString ();
+ print_error (err_msg.c_str(),
+ arg, ctx->past, ctx->pci);
+ return false;
+ }
+
+ auto builtin_type = dyn_cast<BuiltinType> (desugared_type);
+ auto kind = builtin_type->getKind ();
+ auto found = false;
+
+ for (auto kk : k) {
+ if (kind == kk) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ auto err_msg = std::string ("bad argument for ") +
+ fmt + " arg: " +
+ arg->getType ().getAsString () +
+ ", resolved as: " +
+ builtin_type->getNameAsCString (ctx->past->getPrintingPolicy ());
+ print_error (err_msg.c_str(),
+ arg, ctx->past, ctx->pci);
+ return false;
+ }
+
+ return true;
+ }
+
+ static bool
+ int_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::UInt,
+ BuiltinType::Kind::Int},
+ "%d or *");
+ }
+
+ static bool
+ long_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::ULong,
+ BuiltinType::Kind::Long},
+ "%l");
+ }
+
+ static bool
+ char_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::UChar,
+ BuiltinType::Kind::SChar,
+ BuiltinType::Kind::Int}, // Because of char -> int propagation
+ "%c");
+ }
+
+ static bool
+ size_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ if (sizeof (size_t) == sizeof (long)) {
+ if (sizeof (long long) == sizeof (long)) {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::ULong,
+ BuiltinType::Kind::Long,
+ BuiltinType::Kind::LongLong,
+ BuiltinType::Kind::ULongLong},
+ "%z");
+ }
+ else {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::ULong,
+ BuiltinType::Kind::Long},
+ "%z");
+ }
+ }
+ else if (sizeof (size_t) == sizeof (int)) {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::UInt,
+ BuiltinType::Kind::Int},
+ "%z");
+ }
+ else {
+ assert (0);
+ }
+
+ return true;
+ }
+
+ static bool
+ double_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::Double},
+ "%f or %g");
+ }
+
+ static bool
+ long_double_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::LongDouble},
+ "%F or %G");
+ }
+
+ static bool
+ pid_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ if (sizeof (pid_t) == sizeof (long)) {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::ULong,
+ BuiltinType::Kind::Long},
+ "%P");
+ }
+ else if (sizeof (pid_t) == sizeof (int)) {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::UInt,
+ BuiltinType::Kind::Int},
+ "%P");
+ }
+ else {
+ assert (0);
+ }
+ }
+
+ static bool
+ time_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ if (sizeof (time_t) == sizeof (long)) {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::ULong,
+ BuiltinType::Kind::Long},
+ "%t");
+ }
+ else if (sizeof (time_t) == sizeof (int)) {
+ return check_builtin_type (arg,
+ ctx,
+ {BuiltinType::Kind::UInt,
+ BuiltinType::Kind::Int},
+ "%t");
+ }
+ else {
+ assert (0);
+ }
+ }
+
+ static bool
+ pointer_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ auto type = arg->getType ().split ().Ty;
+
+ if (!type->isPointerType ()) {
+ auto err_msg = std::string ("bad pointer argument for %p: ") +
+ arg->getType ().getAsString ();
+ print_error (err_msg.c_str(),
+ arg, ctx->past, ctx->pci);
+ return false;
+ }
+
+ return true;
+ }
+
+ static bool
+ int64_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ std::vector <BuiltinType::Kind> check;
+
+ if (sizeof (int64_t) == sizeof (long long)) {
+ check.push_back (BuiltinType::Kind::ULongLong);
+ check.push_back (BuiltinType::Kind::LongLong);
+ }
+ if (sizeof (int64_t) == sizeof (long)) {
+ check.push_back (BuiltinType::Kind::ULong);
+ check.push_back (BuiltinType::Kind::Long);
+ }
+
+ return check_builtin_type (arg,
+ ctx,
+ check,
+ "%L");
+
+ return true;
+ }
+
+ static bool
+ int32_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ std::vector < BuiltinType::Kind> check;
+
+ if (sizeof (int32_t) == sizeof (long)) {
+ check.push_back (BuiltinType::Kind::ULong);
+ check.push_back (BuiltinType::Kind::Long);
+ }
+ if (sizeof (int32_t) == sizeof (int)) {
+ check.push_back (BuiltinType::Kind::UInt);
+ check.push_back (BuiltinType::Kind::Int);
+ }
+
+ return check_builtin_type (arg,
+ ctx,
+ check,
+ "%D");
+
+ return true;
+ }
+
+ static bool
+ check_struct_type (const Expr *arg, struct PrintfArgChecker *ctx,
+ const std::string &sname, const std::string &fmt)
+ {
+ auto type = arg->getType ().split ().Ty;
+
+ if (!type->isPointerType ()) {
+ auto err_msg = std::string ("non pointer argument for %s: ") +
+ arg->getType ().getAsString ();
+ print_error (err_msg.c_str(),
+ arg, ctx->past, ctx->pci);
+ return false;
+ }
+
+ auto ptr_type = type->getPointeeType ().split ().Ty;
+ auto desugared_type = ptr_type->getUnqualifiedDesugaredType ();
+
+ if (!desugared_type->isRecordType ()) {
+ auto err_msg = std::string ("not a record type for ") + fmt + " arg: " +
+ arg->getType ().getAsString ();
+ print_error (err_msg.c_str(),
+ arg, ctx->past, ctx->pci);
+ return false;
+ }
+
+ auto struct_type = desugared_type->getAsStructureType ();
+ auto struct_decl = struct_type->getDecl ();
+ auto struct_def = struct_decl->getNameAsString ();
+
+ if (struct_def != sname) {
+ auto err_msg = std::string ("bad argument '") + struct_def + "' for "
+ + fmt + " arg: " +
+ arg->getType ().getAsString ();
+ print_error (err_msg.c_str(),
+ arg, ctx->past, ctx->pci);
+ return false;
+ }
+
+ return true;
+ }
+
+ static bool
+ tok_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ return check_struct_type (arg,
+ ctx,
+ "f_str_tok",
+ "%T");
+ }
+
+ static bool
+ fstring_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ return check_struct_type (arg,
+ ctx,
+ "f_str_s",
+ "%V");
+ }
+
+ static bool
+ gstring_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ return check_struct_type (arg,
+ ctx,
+ "_GString",
+ "%v");
+ }
+
+ static bool
+ gerr_arg_handler (const Expr *arg, struct PrintfArgChecker *ctx)
+ {
+ return check_struct_type (arg,
+ ctx,
+ "_GError",
+ "%e");
+ }
+}
diff --git a/clang-plugin/printf_check.h b/clang-plugin/printf_check.h
new file mode 100644
index 0000000..6e0b1d1
--- /dev/null
+++ b/clang-plugin/printf_check.h
@@ -0,0 +1,39 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_PRINTF_CHECK_H
+#define RSPAMD_PRINTF_CHECK_H
+
+#include <memory>
+#include "clang/AST/AST.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/AST/Expr.h"
+
+namespace rspamd {
+
+class PrintfCheckVisitor : public clang::RecursiveASTVisitor<PrintfCheckVisitor> {
+ class impl;
+ std::unique_ptr<impl> pimpl;
+
+public:
+ PrintfCheckVisitor(clang::ASTContext *ctx, clang::CompilerInstance &ci);
+ virtual ~PrintfCheckVisitor(void);
+ bool VisitCallExpr(clang::CallExpr *E);
+};
+
+}// namespace rspamd
+
+#endif
diff --git a/cmake/ArchDep.cmake b/cmake/ArchDep.cmake
new file mode 100644
index 0000000..8271709
--- /dev/null
+++ b/cmake/ArchDep.cmake
@@ -0,0 +1,102 @@
+TARGET_ARCHITECTURE(ARCH)
+
+SET(ASM_CODE "
+ .macro TEST1 op
+ \\op %eax, %eax
+ .endm
+ TEST1 xorl
+ ")
+ASM_OP(HAVE_SLASHMACRO "slash macro convention")
+
+SET(ASM_CODE "
+ .macro TEST1 op
+ $0 %eax, %eax
+ .endm
+ TEST1 xorl
+ ")
+ASM_OP(HAVE_DOLLARMACRO "dollar macro convention")
+
+# For now we support only x86_64/i386 architecture with optimizations
+IF("${ARCH}" STREQUAL "x86_64" OR "${ARCH}" STREQUAL "i386")
+ IF(NOT HAVE_SLASHMACRO AND NOT HAVE_DOLLARMACRO)
+ MESSAGE(FATAL_ERROR "Your assembler cannot compile macros, please check your CMakeFiles/CMakeError.log")
+ ENDIF()
+
+ SET(ASM_CODE "vpaddq %ymm0, %ymm0, %ymm0")
+ ASM_OP(HAVE_AVX2 "avx2")
+ # Handle broken compilers, sigh...
+ IF(HAVE_AVX2)
+ CHECK_C_SOURCE_COMPILES(
+ "
+#include <stddef.h>
+#pragma GCC push_options
+#pragma GCC target(\"avx2\")
+#ifndef __SSE2__
+#define __SSE2__
+#endif
+#ifndef __SSE__
+#define __SSE__
+#endif
+#ifndef __SSE4_2__
+#define __SSE4_2__
+#endif
+#ifndef __SSE4_1__
+#define __SSE4_1__
+#endif
+#ifndef __SSEE3__
+#define __SSEE3__
+#endif
+#ifndef __AVX__
+#define __AVX__
+#endif
+#ifndef __AVX2__
+#define __AVX2__
+#endif
+
+#ifndef __clang__
+#if __GNUC__ < 6
+#error Broken due to compiler bug
+#endif
+#endif
+
+#include <immintrin.h>
+static void foo(const char* a) __attribute__((__target__(\"avx2\")));
+static void foo(const char* a)
+{
+ __m256i str = _mm256_loadu_si256((__m256i *)a);
+ __m256i t = _mm256_loadu_si256((__m256i *)a + 1);
+ _mm256_add_epi8(str, t);
+}
+int main(int argc, char** argv) {
+ foo(argv[0]);
+}" HAVE_AVX2_C_COMPILER)
+ IF(NOT HAVE_AVX2_C_COMPILER)
+ MESSAGE(STATUS "Your compiler has broken AVX2 support")
+ UNSET(HAVE_AVX2 CACHE)
+ ENDIF()
+ ENDIF()
+ SET(ASM_CODE "vpaddq %xmm0, %xmm0, %xmm0")
+ ASM_OP(HAVE_AVX "avx")
+ SET(ASM_CODE "pmuludq %xmm0, %xmm0")
+ ASM_OP(HAVE_SSE2 "sse2")
+ SET(ASM_CODE "lddqu 0(%esi), %xmm0")
+ ASM_OP(HAVE_SSE3 "sse3")
+ SET(ASM_CODE "pshufb %xmm0, %xmm0")
+ ASM_OP(HAVE_SSSE3 "ssse3")
+ SET(ASM_CODE "pblendw \$0, %xmm0, %xmm0")
+ ASM_OP(HAVE_SSE41 "sse41")
+ SET(ASM_CODE "crc32 %eax, %eax")
+ ASM_OP(HAVE_SSE42 "sse42")
+ENDIF()
+
+IF ("${ARCH}" STREQUAL "x86_64")
+ MESSAGE(STATUS "Enable sse2 on x86_64 architecture")
+ IF((CMAKE_C_COMPILER_ID MATCHES "GNU") OR (CMAKE_C_COMPILER_ID MATCHES "Clang"))
+ ADD_COMPILE_OPTIONS(-msse2)
+ ADD_COMPILE_OPTIONS(-m64)
+ ELSEIF(CMAKE_C_COMPILER_ID MATCHES "Intel")
+ ADD_COMPILE_OPTIONS(/QxSSE2)
+ ELSEIF((CMAKE_C_COMPILER_ID MATCHES "MSVC"))
+ ADD_COMPILE_OPTIONS(/arch:SSE2)
+ ENDIF()
+ENDIF()
diff --git a/cmake/AsmOp.cmake b/cmake/AsmOp.cmake
new file mode 100644
index 0000000..bcf9d99
--- /dev/null
+++ b/cmake/AsmOp.cmake
@@ -0,0 +1,19 @@
+# Check for assembler option specified
+
+MACRO(asm_op output_var description)
+ IF(NOT ${output_var})
+ file(WRITE "${CMAKE_BINARY_DIR}/asm.S" "${ASM_CODE}")
+ try_compile(HAVE_OP
+ "${CMAKE_BINARY_DIR}"
+ "${CMAKE_BINARY_DIR}/asm.S"
+ CMAKE_FLAGS "-DCMAKE_ASM_LINK_EXECUTABLE='echo not linking now...'")
+
+ if(HAVE_OP)
+ MESSAGE(STATUS "Compilation of ${description} asm set is supported")
+ else()
+ MESSAGE(STATUS "Compilation of ${description} asm set is -NOT- supported")
+ endif()
+
+ set(${output_var} "${HAVE_OP}" CACHE INTERNAL "${description}")
+ ENDIF()
+ENDMACRO()
diff --git a/cmake/CompilerWarnings.cmake b/cmake/CompilerWarnings.cmake
new file mode 100644
index 0000000..9092457
--- /dev/null
+++ b/cmake/CompilerWarnings.cmake
@@ -0,0 +1,89 @@
+CHECK_C_COMPILER_FLAG(-Wall SUPPORT_WALL)
+CHECK_C_COMPILER_FLAG(-Wextra SUPPORT_WEXTRA)
+CHECK_C_COMPILER_FLAG(-Wpointer-arith SUPPORT_WPOINTER)
+CHECK_C_COMPILER_FLAG(-Wno-unused-parameter SUPPORT_WPARAM)
+CHECK_C_COMPILER_FLAG(-Wno-unused-function SUPPORT_WFUNCTION)
+CHECK_C_COMPILER_FLAG(-Wno-strict-aliasing SUPPORT_WSTRICT_ALIASING)
+CHECK_C_COMPILER_FLAG(-Wunused-variable SUPPORT_WUNUSED_VAR)
+CHECK_C_COMPILER_FLAG(-Wno-pointer-sign SUPPORT_WPOINTER_SIGN)
+CHECK_C_COMPILER_FLAG(-Wno-sign-compare SUPPORT_WSIGN_COMPARE)
+CHECK_C_COMPILER_FLAG(-Wstrict-prototypes SUPPORT_WSTRICT_PROTOTYPES)
+CHECK_C_COMPILER_FLAG(-pedantic SUPPORT_PEDANTIC_FLAG)
+CHECK_C_COMPILER_FLAG(-Wno-unused-const-variable SUPPORT_WNO_UNUSED_CONST)
+CHECK_C_COMPILER_FLAG(-Wmissing-noreturn SUPPORT_WMISSING_NORETURN)
+CHECK_C_COMPILER_FLAG(-Wmissing-format-attribute SUPPORT_WMISSING_FORMAT_ATTRIBUTE)
+# GCC 6 specific
+CHECK_C_COMPILER_FLAG(-Wnull-dereference SUPPORT_WNULL_DEREFERENCE)
+CHECK_C_COMPILER_FLAG(-Wduplicated-cond SUPPORT_WDUPLICATED_COND)
+# GCC 7 specific
+CHECK_C_COMPILER_FLAG(-Wimplicit-fallthrough SUPPORT_WIMPLICIT_FALLTHROUGH)
+# Special check for deprecated declarations, as since OpenSSL 3.0 they
+# just poison output for no good reason
+CHECK_C_COMPILER_FLAG(-Wdeprecated-declarations SUPPORT_WDEPRECATED_DECLARATIONS)
+# Disable -Wsuggest-attribute=format: it is too noisy with FPs around fmt C++ library
+CHECK_C_COMPILER_FLAG(-Wsuggest-attribute SUPPORT_WSUGGEST_ATTRIBUTE)
+
+IF(SUPPORT_WEXTRA)
+ ADD_COMPILE_OPTIONS("-Wextra")
+ENDIF(SUPPORT_WEXTRA)
+IF(SUPPORT_WALL)
+ ADD_COMPILE_OPTIONS("-Wall")
+ENDIF(SUPPORT_WALL)
+IF(SUPPORT_WPOINTER)
+ ADD_COMPILE_OPTIONS("-Wpointer-arith")
+ENDIF(SUPPORT_WPOINTER)
+IF(SUPPORT_WPARAM)
+ ADD_COMPILE_OPTIONS("-Wno-unused-parameter")
+ENDIF(SUPPORT_WPARAM)
+IF(SUPPORT_WFUNCTION)
+ ADD_COMPILE_OPTIONS("-Wno-unused-function")
+ENDIF(SUPPORT_WFUNCTION)
+IF(SUPPORT_WUNUSED_VAR)
+ ADD_COMPILE_OPTIONS("-Wunused-variable")
+ENDIF(SUPPORT_WUNUSED_VAR)
+IF(SUPPORT_WPOINTER_SIGN)
+ # only valid for C
+ ADD_COMPILE_OPTIONS($<$<COMPILE_LANGUAGE:C>:-Wno-pointer-sign>)
+ENDIF(SUPPORT_WPOINTER_SIGN)
+IF(SUPPORT_WSTRICT_PROTOTYPES)
+ # only valid for C
+ ADD_COMPILE_OPTIONS($<$<COMPILE_LANGUAGE:C>:-Wstrict-prototypes>)
+ENDIF(SUPPORT_WSTRICT_PROTOTYPES)
+IF(SUPPORT_WSTRICT_ALIASING)
+ ADD_COMPILE_OPTIONS("-Wno-strict-aliasing")
+ ADD_COMPILE_OPTIONS("-fno-strict-aliasing")
+ENDIF(SUPPORT_WSTRICT_ALIASING)
+#IF(SUPPORT_PEDANTIC_FLAG)
+# SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -pedantic")
+#ENDIF(SUPPORT_PEDANTIC_FLAG)
+IF(SUPPORT_WNULL_DEREFERENCE)
+ ADD_COMPILE_OPTIONS("-Wnull-dereference")
+ENDIF()
+IF(SUPPORT_WDUPLICATED_COND)
+ ADD_COMPILE_OPTIONS("-Wduplicated-cond")
+ENDIF()
+IF(SUPPORT_WLOGICAL_OP)
+ ADD_COMPILE_OPTIONS("-Wlogical-op")
+ENDIF()
+IF(SUPPORT_WNO_UNUSED_CONST)
+ ADD_COMPILE_OPTIONS("-Wno-unused-const-variable")
+ENDIF()
+IF(SUPPORT_WSIGN_COMPARE)
+ ADD_COMPILE_OPTIONS("-Wno-sign-compare")
+ENDIF()
+IF(SUPPORT_WIMPLICIT_FALLTHROUGH)
+ ADD_COMPILE_OPTIONS("-Wno-implicit-fallthrough")
+ENDIF(SUPPORT_WIMPLICIT_FALLTHROUGH)
+IF(SUPPORT_WMISSING_NORETURN)
+ ADD_COMPILE_OPTIONS("-Wmissing-noreturn")
+ENDIF(SUPPORT_WMISSING_NORETURN)
+IF(SUPPORT_WMISSING_FORMAT_ATTRIBUTE)
+ ADD_COMPILE_OPTIONS("-Wmissing-format-attribute")
+ENDIF(SUPPORT_WMISSING_FORMAT_ATTRIBUTE)
+IF(SUPPORT_WSUGGEST_ATTRIBUTE)
+ ADD_COMPILE_OPTIONS("-Wno-suggest-attribute=format")
+ENDIF()
+
+IF(SUPPORT_WDEPRECATED_DECLARATIONS)
+ ADD_COMPILE_OPTIONS("-Wno-deprecated-declarations")
+ENDIF()
diff --git a/cmake/FindArch.cmake b/cmake/FindArch.cmake
new file mode 100644
index 0000000..e172207
--- /dev/null
+++ b/cmake/FindArch.cmake
@@ -0,0 +1,127 @@
+set(archdetect_c_code "
+#if defined(__arm__) || defined(__TARGET_ARCH_ARM)
+ #if defined(__ARM_ARCH_7__) \\
+ || defined(__ARM_ARCH_7A__) \\
+ || defined(__ARM_ARCH_7R__) \\
+ || defined(__ARM_ARCH_7M__) \\
+ || (defined(__TARGET_ARCH_ARM) && __TARGET_ARCH_ARM-0 >= 7)
+ #error cmake_ARCH armv7
+ #elif defined(__ARM_ARCH_6__) \\
+ || defined(__ARM_ARCH_6J__) \\
+ || defined(__ARM_ARCH_6T2__) \\
+ || defined(__ARM_ARCH_6Z__) \\
+ || defined(__ARM_ARCH_6K__) \\
+ || defined(__ARM_ARCH_6ZK__) \\
+ || defined(__ARM_ARCH_6M__) \\
+ || (defined(__TARGET_ARCH_ARM) && __TARGET_ARCH_ARM-0 >= 6)
+ #error cmake_ARCH armv6
+ #elif defined(__ARM_ARCH_5TEJ__) \\
+ || (defined(__TARGET_ARCH_ARM) && __TARGET_ARCH_ARM-0 >= 5)
+ #error cmake_ARCH armv5
+ #else
+ #error cmake_ARCH arm
+ #endif
+#elif defined(__i386) || defined(__i386__) || defined(_M_IX86)
+ #error cmake_ARCH i386
+#elif defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64)
+ #error cmake_ARCH x86_64
+#elif defined(__ia64) || defined(__ia64__) || defined(_M_IA64)
+ #error cmake_ARCH ia64
+#elif defined(__ppc__) || defined(__ppc) || defined(__powerpc__) \\
+ || defined(_ARCH_COM) || defined(_ARCH_PWR) || defined(_ARCH_PPC) \\
+ || defined(_M_MPPC) || defined(_M_PPC)
+ #if defined(__ppc64__) || defined(__powerpc64__) || defined(__64BIT__)
+ #error cmake_ARCH ppc64
+ #else
+ #error cmake_ARCH ppc
+ #endif
+#elif defined(__loongarch__) || defined(__loongarch64)
+ #error cmake_ARCH loongarch64
+#endif
+
+#error cmake_ARCH unknown
+")
+
+# Set ppc_support to TRUE before including this file or ppc and ppc64
+# will be treated as invalid architectures since they are no longer supported by Apple
+
+function(target_architecture output_var)
+ if(APPLE AND CMAKE_OSX_ARCHITECTURES)
+ # On OS X we use CMAKE_OSX_ARCHITECTURES *if* it was set
+ # First let's normalize the order of the values
+
+ # Note that it's not possible to compile PowerPC applications if you are using
+ # the OS X SDK version 10.6 or later - you'll need 10.4/10.5 for that, so we
+ # disable it by default
+ # See this page for more information:
+ # http://stackoverflow.com/questions/5333490/how-can-we-restore-ppc-ppc64-as-well-as-full-10-4-10-5-sdk-support-to-xcode-4
+
+ # Architecture defaults to i386 or ppc on OS X 10.5 and earlier, depending on the CPU type detected at runtime.
+ # On OS X 10.6+ the default is x86_64 if the CPU supports it, i386 otherwise.
+
+ foreach(osx_arch ${CMAKE_OSX_ARCHITECTURES})
+ if("${osx_arch}" STREQUAL "ppc" AND ppc_support)
+ set(osx_arch_ppc TRUE)
+ elseif("${osx_arch}" STREQUAL "i386")
+ set(osx_arch_i386 TRUE)
+ elseif("${osx_arch}" STREQUAL "x86_64")
+ set(osx_arch_x86_64 TRUE)
+ elseif("${osx_arch}" STREQUAL "ppc64" AND ppc_support)
+ set(osx_arch_ppc64 TRUE)
+ else()
+ message(FATAL_ERROR "Invalid OS X arch name: ${osx_arch}")
+ endif()
+ endforeach()
+
+ # Now add all the architectures in our normalized order
+ if(osx_arch_ppc)
+ list(APPEND ARCH ppc)
+ endif()
+
+ if(osx_arch_i386)
+ list(APPEND ARCH i386)
+ endif()
+
+ if(osx_arch_x86_64)
+ list(APPEND ARCH x86_64)
+ endif()
+
+ if(osx_arch_ppc64)
+ list(APPEND ARCH ppc64)
+ endif()
+ else()
+ file(WRITE "${CMAKE_BINARY_DIR}/arch.c" "${archdetect_c_code}")
+
+ # Detect the architecture in a rather creative way...
+ # This compiles a small C program which is a series of ifdefs that selects a
+ # particular #error preprocessor directive whose message string contains the
+ # target architecture. The program will always fail to compile (both because
+ # file is not a valid C program, and obviously because of the presence of the
+ # #error preprocessor directives... but by exploiting the preprocessor in this
+ # way, we can detect the correct target architecture even when cross-compiling,
+ # since the program itself never needs to be run (only the compiler/preprocessor)
+ try_run(
+ run_result_unused
+ compile_result_unused
+ "${CMAKE_BINARY_DIR}"
+ "${CMAKE_BINARY_DIR}/arch.c"
+ COMPILE_OUTPUT_VARIABLE ARCH
+ CMAKE_FLAGS CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
+ )
+
+ # Parse the architecture name from the compiler output
+ string(REGEX MATCH "cmake_ARCH ([a-zA-Z0-9_]+)" ARCH "${ARCH}")
+
+ # Get rid of the value marker leaving just the architecture name
+ string(REPLACE "cmake_ARCH " "" ARCH "${ARCH}")
+
+ # If we are compiling with an unknown architecture this variable should
+ # already be set to "unknown" but in the case that it's empty (i.e. due
+ # to a typo in the code), then set it to unknown
+ if (NOT ARCH)
+ set(ARCH unknown)
+ endif()
+ endif()
+
+ set(${output_var} "${ARCH}" PARENT_SCOPE)
+endfunction()
diff --git a/cmake/FindRagel.cmake b/cmake/FindRagel.cmake
new file mode 100644
index 0000000..f43298e
--- /dev/null
+++ b/cmake/FindRagel.cmake
@@ -0,0 +1,103 @@
+# - Find Ragel executable and provides macros to generate custom build rules
+# The module defines the following variables:
+#
+# RAGEL_EXECUTABLE - path to the ragel program
+# RAGEL_VERSION - version of ragel
+# RAGEL_FOUND - true if the program was found
+#
+# If ragel is found, the module defines the macros:
+#
+# RAGEL_TARGET(<Name> INPUTS <inputs> OUTPUT <output>
+# [COMPILE_FLAGS <string>] [DEPENDS <depends>])
+#
+# which will create a custom rule to generate a state machine. <RagelInp> is
+# the path to a Ragel file. <CodeOutput> is the name of the source file
+# generated by ragel. If COMPILE_FLAGS option is specified, the next
+# parameter is added in the ragel command line.
+#
+# The macro defines a set of variables:
+# RAGEL_${Name}_DEFINED - true is the macro ran successfully
+# RAGEL_${Name}_INPUT - The input source file, an alias for <RagelInp>
+# RAGEL_${Name}_OUTPUT_SOURCE - The source file generated by ragel
+# RAGEL_${Name}_OUTPUT_HEADER - The header file generated by ragel
+# RAGEL_${Name}_OUTPUTS - The sources files generated by ragel
+# RAGEL_${Name}_COMPILE_FLAGS - Options used in the ragel command line
+#
+# ====================================================================
+# Example:
+#
+# find_package(RAGEL) # or e.g.: find_package(RAGEL 6.6 REQUIRED)
+# RAGEL_TARGET(MyMachine machine.rl ${CMAKE_CURRENT_BINARY_DIR}/machine.cc)
+# add_executable(Foo main.cc ${RAGEL_MyMachine_OUTPUTS})
+# ====================================================================
+
+# 2014-02-09, Georg Sauthoff <mail@georg.so>
+#
+# I don't think that these few lines are even copyrightable material,
+# but I am fine with using the BSD/MIT/GPL license on it ...
+#
+# I've used following references:
+# http://www.cmake.org/cmake/help/v2.8.12/cmake.html
+# /usr/share/cmake/Modules/FindFLEX.cmake
+# /usr/share/cmake/Modules/FindBISON.cmake
+
+find_program(RAGEL_EXECUTABLE NAMES ragel DOC "path to the ragel executable")
+mark_as_advanced(RAGEL_EXECUTABLE)
+
+if(RAGEL_EXECUTABLE)
+
+ execute_process(COMMAND ${RAGEL_EXECUTABLE} --version
+ OUTPUT_VARIABLE RAGEL_version_output
+ ERROR_VARIABLE RAGEL_version_error
+ RESULT_VARIABLE RAGEL_version_result
+ OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+ if(${RAGEL_version_result} EQUAL 0)
+ string(REGEX REPLACE "^Ragel State Machine Compiler version ([^ ]+) .*$"
+ "\\1"
+ RAGEL_VERSION "${RAGEL_version_output}")
+ else()
+ message(SEND_ERROR
+ "Command \"${RAGEL_EXECUTABLE} --version\" failed with output:
+${RAGEL_version_error}")
+ endif()
+
+ #============================================================
+ # RAGEL_TARGET (public macro)
+ #============================================================
+ #
+ macro(RAGEL_TARGET Name)
+ CMAKE_PARSE_ARGUMENTS(RAGEL "" "OUTPUT"
+ "INPUTS;DEPENDS;COMPILE_FLAGS" ${ARGN})
+
+ file(RELATIVE_PATH RAGEL_OUTPUT_RELATIVE "${CMAKE_CURRENT_BINARY_DIR}"
+ "${RAGEL_OUTPUT}")
+ file(RELATIVE_PATH RAGEL_INPUTS_RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
+ "${RAGEL_INPUTS}")
+
+ add_custom_command(OUTPUT ${RAGEL_OUTPUT}
+ COMMAND ${RAGEL_EXECUTABLE}
+ ARGS ${RAGEL_COMPILE_FLAGS}
+ -o${RAGEL_OUTPUT} ${RAGEL_INPUTS}
+ DEPENDS ${RAGEL_INPUTS} ${RAGEL_DEPENDS}
+ COMMENT
+ "[RAGEL][${Name}] Compiling state machine with Ragel ${RAGEL_VERSION} -> ${RAGEL_OUTPUT}"
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+ get_filename_component(src_target ${RAGEL_INPUTS} NAME_WE)
+ add_custom_target(ragel_${src_target} DEPENDS ${RAGEL_OUTPUT})
+ set_source_files_properties(${RAGEL_OUTPUT} PROPERTIES GENERATED TRUE)
+
+ set(RAGEL_${Name}_DEFINED TRUE)
+ set(RAGEL_${Name}_OUTPUTS ${RAGEL_OUTPUT})
+ set(RAGEL_${Name}_INPUT ${RAGEL_INPUTS})
+ set(RAGEL_${Name}_COMPILE_FLAGS ${RAGEL_COMPILE_FLAGS})
+ endmacro()
+
+endif()
+
+# use this include when module file is located under /usr/share/cmake/Modules
+#include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake)
+# use this include when module file is located in build tree
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(RAGEL REQUIRED_VARS RAGEL_EXECUTABLE
+ VERSION_VAR RAGEL_VERSION)
diff --git a/cmake/Hyperscan.cmake b/cmake/Hyperscan.cmake
new file mode 100644
index 0000000..dc19f49
--- /dev/null
+++ b/cmake/Hyperscan.cmake
@@ -0,0 +1,8 @@
+option (ENABLE_HYPERSCAN "Enable hyperscan for fast regexp processing [default: OFF]" OFF)
+
+if (ENABLE_HYPERSCAN MATCHES "ON")
+ ProcessPackage (HYPERSCAN LIBRARY hs INCLUDE hs.h INCLUDE_SUFFIXES
+ hs include/hs
+ ROOT ${HYPERSCAN_ROOT_DIR} MODULES libhs)
+ set (WITH_HYPERSCAN 1)
+endif ()
diff --git a/cmake/OSDep.cmake b/cmake/OSDep.cmake
new file mode 100644
index 0000000..78f6549
--- /dev/null
+++ b/cmake/OSDep.cmake
@@ -0,0 +1,65 @@
+# Platform specific configuration
+IF(CMAKE_SYSTEM_NAME MATCHES "^.*BSD$|DragonFly")
+ ADD_COMPILE_OPTIONS(-DFREEBSD -D_BSD_SOURCE)
+ SET(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D_BSD_SOURCE")
+ CONFIGURE_FILE(freebsd/rspamd.sh.in freebsd/rspamd @ONLY)
+ MESSAGE(STATUS "Configuring for BSD system")
+ # Find util library
+ ProcessPackage(LIBUTIL LIBRARY util INCLUDE libutil.h
+ ROOT ${LIBUTIL_ROOT_DIR} OPTIONAL)
+ IF(WITH_LIBUTIL)
+ SET(HAVE_LIBUTIL_H 1)
+ LIST(APPEND CMAKE_REQUIRED_LIBRARIES util)
+ CHECK_FUNCTION_EXISTS(pidfile_open HAVE_PIDFILE)
+ CHECK_FUNCTION_EXISTS(pidfile_fileno HAVE_PIDFILE_FILENO)
+ ENDIF()
+ IF(CMAKE_SYSTEM_NAME MATCHES "^NetBSD$")
+ LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt)
+ ENDIF()
+ SET(TAR "gtar")
+ENDIF()
+
+IF(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+ ADD_COMPILE_OPTIONS(-D_BSD_SOURCE -DDARWIN)
+ SET(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS} -undefined dynamic_lookup")
+ IF(ENABLE_LUAJIT MATCHES "ON")
+ SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pagezero_size 10000 -image_base 100000000")
+ ENDIF(ENABLE_LUAJIT MATCHES "ON")
+ MESSAGE(STATUS "Configuring for Darwin")
+ SET(TAR "gnutar")
+ SET(CMAKE_FIND_FRAMEWORK "NEVER")
+ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+
+IF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ ADD_COMPILE_OPTIONS(-D_GNU_SOURCE -DLINUX)
+ SET(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D_GNU_SOURCE")
+ # Workaround with architecture specific includes
+ #IF(IS_DIRECTORY "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/")
+ # INCLUDE_DIRECTORIES("/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/")
+ # LIST(APPEND CMAKE_REQUIRED_INCLUDES "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/")
+ #ENDIF(IS_DIRECTORY "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/")
+
+ LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt)
+ LIST(APPEND CMAKE_REQUIRED_LIBRARIES dl)
+ LIST(APPEND CMAKE_REQUIRED_LIBRARIES resolv)
+ MESSAGE(STATUS "Configuring for Linux")
+ IF(EXISTS "/etc/debian_version")
+ SET(LINUX_START_SCRIPT "rspamd_debian.in")
+ ELSE(EXISTS "/etc/debian_version")
+ SET(LINUX_START_SCRIPT "rspamd_rh.in")
+ ENDIF(EXISTS "/etc/debian_version")
+ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+
+IF(CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+ ADD_COMPILE_OPTIONS(-D__EXTENSIONS__ -DSOLARIS -D_POSIX_SOURCE -D_POSIX_C_SOURCE=200112)
+ LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt)
+ LIST(APPEND CMAKE_REQUIRED_LIBRARIES dl)
+ LIST(APPEND CMAKE_REQUIRED_LIBRARIES resolv)
+ LIST(APPEND CMAKE_REQUIRED_LIBRARIES nsl)
+ LIST(APPEND CMAKE_REQUIRED_LIBRARIES socket)
+ LIST(APPEND CMAKE_REQUIRED_LIBRARIES umem)
+ # Ugly hack, but FindOpenSSL on Solaris does not link with libcrypto
+ SET(CMAKE_VERBOSE_MAKEFILE ON)
+ SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE)
+ SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib:${RSPAMD_LIBDIR}")
+ENDIF(CMAKE_SYSTEM_NAME STREQUAL "SunOS") \ No newline at end of file
diff --git a/cmake/Openblas.cmake b/cmake/Openblas.cmake
new file mode 100644
index 0000000..da62363
--- /dev/null
+++ b/cmake/Openblas.cmake
@@ -0,0 +1,119 @@
+option (ENABLE_BLAS "Enable openblas for fast neural network processing [default: OFF]" OFF)
+
+IF(ENABLE_BLAS MATCHES "ON")
+ ProcessPackage(BLAS OPTIONAL_INCLUDE LIBRARY openblas blas blis
+ INCLUDE cblas.h INCLUDE_SUFFIXES include/openblas
+ include/blas
+ include/blis
+ ROOT ${BLAS_ROOT_DIR}
+ LIB_OUTPUT BLAS_REQUIRED_LIBRARIES)
+ ProcessPackage(BLAS_LAPACK OPTIONAL_INCLUDE LIBRARY lapack
+ INCLUDE cblas.h INCLUDE_SUFFIXES include/openblas
+ include/blas
+ include/blis
+ ROOT ${BLAS_ROOT_DIR}
+ LIB_OUTPUT BLAS_REQUIRED_LIBRARIES)
+ENDIF()
+
+IF(WITH_BLAS)
+ MESSAGE(STATUS "Use openblas to accelerate kann")
+ IF(NOT BLAS_INCLUDE)
+ FIND_FILE(HAVE_CBLAS_H HINTS "${RSPAMD_SEARCH_PATH}"
+ NAMES cblas.h
+ DOC "Path to cblas.h header")
+ IF(NOT HAVE_CBLAS_H)
+ MESSAGE(STATUS "Blas header cblas.h has not been found, use internal workaround")
+ ELSE()
+ SET(HAVE_CBLAS_H 1)
+ ENDIF()
+ ELSE()
+ SET(HAVE_CBLAS_H 1)
+ ENDIF()
+ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/sgemm.c" "
+#include <stddef.h>
+enum CBLAS_ORDER {CblasRowMajor=101, CblasColMajor=102 };
+enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112 };
+extern void cblas_sgemm(const enum CBLAS_ORDER Order,
+ const enum CBLAS_TRANSPOSE TA,
+ const enum CBLAS_TRANSPOSE TB,
+ const int M, const int N, const int K,
+ const float alpha, const float *A, const int lda,
+ const float *B, const int ldb, const float beta,
+ float *C, const int ldc);
+int main(int argc, char **argv)
+{
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, 0, 0, 0, 0, NULL, 0,
+ NULL, 0, 0, NULL, 0);
+ return 0;
+}
+")
+ try_compile(HAVE_CBLAS_SGEMM
+ ${CMAKE_CURRENT_BINARY_DIR}
+ "${CMAKE_CURRENT_BINARY_DIR}/sgemm.c"
+ COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS}
+ LINK_LIBRARIES ${BLAS_REQUIRED_LIBRARIES}
+ OUTPUT_VARIABLE SGEMM_ERR)
+ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/saxpy.c" "
+#include <stddef.h>
+extern void cblas_saxpy(const int __N,
+ const float __alpha, const float *__X, const int __incX, float *__Y, const int __incY);
+int main(int argc, char **argv)
+{
+ cblas_saxpy(0, 0, NULL, 0, NULL, 0);
+ return 0;
+}
+")
+ try_compile(HAVE_CBLAS_SAXPY
+ ${CMAKE_CURRENT_BINARY_DIR}
+ "${CMAKE_CURRENT_BINARY_DIR}/saxpy.c"
+ COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS}
+ LINK_LIBRARIES ${BLAS_REQUIRED_LIBRARIES}
+ OUTPUT_VARIABLE SAXPY_ERR)
+
+ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/openblas_set_num_threads.c" "
+#include <stddef.h>
+extern void openblas_set_num_threads(int num_threads);
+int main(int argc, char **argv)
+{
+ openblas_set_num_threads(1);
+ return 0;
+}
+")
+ try_compile(HAVE_OPENBLAS_SET_NUM_THREADS
+ ${CMAKE_CURRENT_BINARY_DIR}
+ "${CMAKE_CURRENT_BINARY_DIR}/openblas_set_num_threads.c"
+ COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS}
+ LINK_LIBRARIES ${BLAS_REQUIRED_LIBRARIES}
+ OUTPUT_VARIABLE OPENBLAS_SET_NUM_THREADS_ERR)
+
+ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/bli_thread_set_num_threads.c" "
+#include <stddef.h>
+extern void bli_thread_set_num_threads(int num_threads);
+int main(int argc, char **argv)
+{
+ bli_thread_set_num_threads(1);
+ return 0;
+}
+")
+ try_compile(HAVE_BLI_THREAD_SET_NUM_THREADS
+ ${CMAKE_CURRENT_BINARY_DIR}
+ "${CMAKE_CURRENT_BINARY_DIR}/bli_thread_set_num_threads.c"
+ COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS}
+ LINK_LIBRARIES ${BLAS_REQUIRED_LIBRARIES}
+ OUTPUT_VARIABLE BLI_SET_NUM_THREADS_ERR)
+ # Cmake is just brain damaged
+ #CHECK_LIBRARY_EXISTS(${BLAS_REQUIRED_LIBRARIES} cblas_sgemm "" HAVE_CBLAS_SGEMM)
+ if(HAVE_CBLAS_SGEMM)
+ MESSAGE(STATUS "Blas has CBLAS sgemm")
+ else()
+ MESSAGE(STATUS "Blas has -NOT- CBLAS sgemm, use internal workaround: ${SGEMM_ERR}")
+ endif()
+ if(HAVE_CBLAS_SAXPY)
+ MESSAGE(STATUS "Blas has CBLAS saxpy")
+ else()
+ MESSAGE(STATUS "Blas has -NOT- CBLAS saxpy, use internal workaround: ${SAXPY_ERR}")
+ endif()
+ SET(HAVE_CBLAS 1)
+ENDIF(WITH_BLAS)
+
+CONFIGURE_FILE("${CMAKE_SOURCE_DIR}/blas-config.h.in" "${CMAKE_BINARY_DIR}/src/blas-config.h") \ No newline at end of file
diff --git a/cmake/PVS-Studio.cmake b/cmake/PVS-Studio.cmake
new file mode 100644
index 0000000..6001f33
--- /dev/null
+++ b/cmake/PVS-Studio.cmake
@@ -0,0 +1,547 @@
+# 2006-2008 (c) Viva64.com Team
+# 2008-2018 (c) OOO "Program Verification Systems"
+#
+# Version 12
+# Apache 2.0 license
+
+cmake_minimum_required(VERSION 2.8.12)
+cmake_policy(SET CMP0054 NEW)
+
+if (PVS_STUDIO_AS_SCRIPT)
+ # This code runs at build time.
+ # It executes pvs-studio-analyzer and propagates its return value.
+
+ set(in_cl_params FALSE)
+ set(additional_args)
+
+ foreach (arg ${PVS_STUDIO_COMMAND})
+ if (NOT in_cl_params)
+ if ("${arg}" STREQUAL "--cl-params")
+ set(in_cl_params TRUE)
+ endif ()
+ else ()
+ # A workaround for macOS frameworks (e.g. QtWidgets.framework)
+ # You can test this workaround on this project: https://github.com/easyaspi314/MidiEditor/tree/gba
+ if (APPLE AND "${arg}" MATCHES "^-I(.*)\\.framework$")
+ STRING(REGEX REPLACE "^-I(.*)\\.framework$" "\\1.framework" framework "${arg}")
+ if (IS_ABSOLUTE "${framework}")
+ get_filename_component(framework "${framework}" DIRECTORY)
+ list(APPEND additional_args "-iframework")
+ list(APPEND additional_args "${framework}")
+ endif ()
+ endif ()
+ endif ()
+ endforeach ()
+
+ execute_process(COMMAND ${PVS_STUDIO_COMMAND} ${additional_args}
+ ERROR_VARIABLE error
+ RESULT_VARIABLE result)
+
+ set(stderr_type "")
+
+ if (result)
+ set(stderr_type FATAL_ERROR)
+ endif ()
+
+ if (result OR error)
+ message(${stderr_type} "${error}")
+ endif ()
+
+ return()
+endif ()
+
+if(__PVS_STUDIO_INCLUDED)
+ return()
+endif()
+set(__PVS_STUDIO_INCLUDED TRUE)
+
+set(PVS_STUDIO_SCRIPT "${CMAKE_CURRENT_LIST_FILE}")
+
+function (pvs_studio_log TEXT)
+ if (PVS_STUDIO_DEBUG)
+ message("PVS-Studio: ${TEXT}")
+ endif ()
+endfunction ()
+
+function (pvs_studio_relative_path VAR ROOT FILEPATH)
+ set("${VAR}" "${FILEPATH}" PARENT_SCOPE)
+ if ("${FILEPATH}" MATCHES "^/.*$" OR "${FILEPATH}" MATCHES "^.:/.*$")
+ file(RELATIVE_PATH RPATH "${ROOT}" "${FILEPATH}")
+ if (NOT "${RPATH}" MATCHES "^\\.\\..*$")
+ set("${VAR}" "${RPATH}" PARENT_SCOPE)
+ endif ()
+ endif ()
+endfunction ()
+
+function (pvs_studio_join_path VAR DIR1 DIR2)
+ if ("${DIR2}" MATCHES "^(/|~|.:/).*$" OR "${DIR1}" STREQUAL "")
+ set("${VAR}" "${DIR2}" PARENT_SCOPE)
+ else ()
+ set("${VAR}" "${DIR1}/${DIR2}" PARENT_SCOPE)
+ endif ()
+endfunction ()
+
+macro (pvs_studio_append_flags_from_property CXX C DIR PREFIX)
+ if (NOT "${PROPERTY}" STREQUAL "NOTFOUND" AND NOT "${PROPERTY}" STREQUAL "PROPERTY-NOTFOUND")
+ foreach (PROP ${PROPERTY})
+ pvs_studio_join_path(PROP "${DIR}" "${PROP}")
+
+ if (APPLE AND "${PREFIX}" STREQUAL "-I" AND IS_ABSOLUTE "${PROP}" AND "${PROP}" MATCHES "\\.framework$")
+ get_filename_component(FRAMEWORK "${PROP}" DIRECTORY)
+ list(APPEND "${CXX}" "-iframework")
+ list(APPEND "${CXX}" "${FRAMEWORK}")
+ list(APPEND "${C}" "-iframework")
+ list(APPEND "${C}" "${FRAMEWORK}")
+ pvs_studio_log("framework: ${FRAMEWORK}")
+ elseif (NOT "${PROP}" STREQUAL "")
+ list(APPEND "${CXX}" "${PREFIX}${PROP}")
+ list(APPEND "${C}" "${PREFIX}${PROP}")
+ endif()
+ endforeach ()
+ endif ()
+endmacro ()
+
+macro (pvs_studio_append_standard_flag FLAGS STANDARD)
+ if ("${STANDARD}" MATCHES "^(99|11|14|17)$")
+ if ("${PVS_STUDIO_PREPROCESSOR}" MATCHES "gcc|clang")
+ list(APPEND "${FLAGS}" "-std=c++${STANDARD}")
+ endif ()
+ endif ()
+endmacro ()
+
+function (pvs_studio_set_directory_flags DIRECTORY CXX C)
+ set(CXX_FLAGS "${${CXX}}")
+ set(C_FLAGS "${${C}}")
+
+ get_directory_property(PROPERTY DIRECTORY "${DIRECTORY}" INCLUDE_DIRECTORIES)
+ pvs_studio_append_flags_from_property(CXX_FLAGS C_FLAGS "${DIRECTORY}" "-I")
+
+ get_directory_property(PROPERTY DIRECTORY "${DIRECTORY}" COMPILE_DEFINITIONS)
+ pvs_studio_append_flags_from_property(CXX_FLAGS C_FLAGS "" "-D")
+
+ set("${CXX}" "${CXX_FLAGS}" PARENT_SCOPE)
+ set("${C}" "${C_FLAGS}" PARENT_SCOPE)
+endfunction ()
+
+function (pvs_studio_set_target_flags TARGET CXX C)
+ set(CXX_FLAGS "${${CXX}}")
+ set(C_FLAGS "${${C}}")
+
+ set(prop_incdirs "$<TARGET_PROPERTY:${TARGET},INCLUDE_DIRECTORIES>")
+ list(APPEND CXX_FLAGS "$<$<BOOL:${prop_incdirs}>:-I$<JOIN:${prop_incdirs},$<SEMICOLON>-I>>")
+ list(APPEND C_FLAGS "$<$<BOOL:${prop_incdirs}>:-I$<JOIN:${prop_incdirs},$<SEMICOLON>-I>>")
+
+ set(prop_compdefs "$<TARGET_PROPERTY:${TARGET},COMPILE_DEFINITIONS>")
+ list(APPEND CXX_FLAGS "$<$<BOOL:${prop_compdefs}>:-D$<JOIN:${prop_compdefs},$<SEMICOLON>-D>>")
+ list(APPEND C_FLAGS "$<$<BOOL:${prop_compdefs}>:-D$<JOIN:${prop_compdefs},$<SEMICOLON>-D>>")
+
+ set("${CXX}" "${CXX_FLAGS}" PARENT_SCOPE)
+ set("${C}" "${C_FLAGS}" PARENT_SCOPE)
+endfunction ()
+
+function (pvs_studio_set_source_file_flags SOURCE)
+ set(LANGUAGE "")
+
+ string(TOLOWER "${SOURCE}" SOURCE_LOWER)
+ if ("${LANGUAGE}" STREQUAL "" AND "${SOURCE_LOWER}" MATCHES "^.*\\.(c|cpp|cc|cx|cxx|cp|c\\+\\+)$")
+ if ("${SOURCE}" MATCHES "^.*\\.c$")
+ set(LANGUAGE C)
+ else ()
+ set(LANGUAGE CXX)
+ endif ()
+ endif ()
+
+ if ("${LANGUAGE}" STREQUAL "C")
+ set(CL_PARAMS ${PVS_STUDIO_C_FLAGS} ${PVS_STUDIO_TARGET_C_FLAGS} -DPVS_STUDIO)
+ elseif ("${LANGUAGE}" STREQUAL "CXX")
+ set(CL_PARAMS ${PVS_STUDIO_CXX_FLAGS} ${PVS_STUDIO_TARGET_CXX_FLAGS} -DPVS_STUDIO)
+ endif ()
+
+ set(PVS_STUDIO_LANGUAGE "${LANGUAGE}" PARENT_SCOPE)
+ set(PVS_STUDIO_CL_PARAMS "${CL_PARAMS}" PARENT_SCOPE)
+endfunction ()
+
+function (pvs_studio_analyze_file SOURCE SOURCE_DIR BINARY_DIR)
+ set(PLOGS ${PVS_STUDIO_PLOGS})
+ pvs_studio_set_source_file_flags("${SOURCE}")
+
+ get_filename_component(SOURCE "${SOURCE}" REALPATH)
+
+ get_source_file_property(PROPERTY "${SOURCE}" HEADER_FILE_ONLY)
+ if (PROPERTY)
+ return()
+ endif ()
+
+ pvs_studio_relative_path(SOURCE_RELATIVE "${SOURCE_DIR}" "${SOURCE}")
+ pvs_studio_join_path(SOURCE "${SOURCE_DIR}" "${SOURCE}")
+
+ set(LOG "${BINARY_DIR}/PVS-Studio/${SOURCE_RELATIVE}.plog")
+ get_filename_component(LOG "${LOG}" REALPATH)
+ get_filename_component(PARENT_DIR "${LOG}" DIRECTORY)
+
+ if (EXISTS "${SOURCE}" AND NOT TARGET "${LOG}" AND NOT "${PVS_STUDIO_LANGUAGE}" STREQUAL "")
+ # A workaround to support implicit dependencies for ninja generators.
+ set(depPvsArg)
+ set(depCommandArg)
+ if (CMAKE_VERSION VERSION_GREATER 3.6 AND "${CMAKE_GENERATOR}" STREQUAL "Ninja")
+ pvs_studio_relative_path(relLog "${CMAKE_BINARY_DIR}" "${LOG}")
+ set(depPvsArg --dep-file "${LOG}.d" --dep-file-target "${relLog}")
+ set(depCommandArg DEPFILE "${LOG}.d")
+ endif ()
+
+ # https://public.kitware.com/Bug/print_bug_page.php?bug_id=14353
+ # https://public.kitware.com/Bug/file/5436/expand_command.cmake
+ #
+ # It is a workaround to expand generator expressions.
+ set(cmdline "${PVS_STUDIO_BIN}" analyze
+ --output-file "${LOG}"
+ --source-file "${SOURCE}"
+ ${depPvsArg}
+ ${PVS_STUDIO_ARGS}
+ --cl-params "${PVS_STUDIO_CL_PARAMS}" "${SOURCE}")
+
+ string(REPLACE ";" "$<SEMICOLON>" cmdline "${cmdline}")
+ set(pvscmd "${CMAKE_COMMAND}"
+ -D PVS_STUDIO_AS_SCRIPT=TRUE
+ -D "PVS_STUDIO_COMMAND=${cmdline}"
+ -P "${PVS_STUDIO_SCRIPT}"
+ )
+
+ add_custom_command(OUTPUT "${LOG}"
+ COMMAND "${CMAKE_COMMAND}" -E make_directory "${PARENT_DIR}"
+ COMMAND "${CMAKE_COMMAND}" -E remove_directory "${LOG}"
+ COMMAND ${pvscmd}
+ WORKING_DIRECTORY "${BINARY_DIR}"
+ DEPENDS "${SOURCE}" "${PVS_STUDIO_CONFIG}"
+ IMPLICIT_DEPENDS "${PVS_STUDIO_LANGUAGE}" "${SOURCE}"
+ ${depCommandArg}
+ VERBATIM
+ COMMENT "Analyzing ${PVS_STUDIO_LANGUAGE} file ${SOURCE_RELATIVE}")
+ list(APPEND PLOGS "${LOG}")
+ endif ()
+ set(PVS_STUDIO_PLOGS "${PLOGS}" PARENT_SCOPE)
+endfunction ()
+
+function (pvs_studio_analyze_target TARGET DIR)
+ set(PVS_STUDIO_PLOGS "${PVS_STUDIO_PLOGS}")
+ set(PVS_STUDIO_TARGET_CXX_FLAGS "")
+ set(PVS_STUDIO_TARGET_C_FLAGS "")
+
+ get_target_property(PROPERTY "${TARGET}" SOURCES)
+ pvs_studio_relative_path(BINARY_DIR "${CMAKE_SOURCE_DIR}" "${DIR}")
+ if ("${BINARY_DIR}" MATCHES "^/.*$")
+ pvs_studio_join_path(BINARY_DIR "${CMAKE_BINARY_DIR}" "PVS-Studio/__${BINARY_DIR}")
+ else ()
+ pvs_studio_join_path(BINARY_DIR "${CMAKE_BINARY_DIR}" "${BINARY_DIR}")
+ endif ()
+
+ file(MAKE_DIRECTORY "${BINARY_DIR}")
+
+ pvs_studio_set_directory_flags("${DIR}" PVS_STUDIO_TARGET_CXX_FLAGS PVS_STUDIO_TARGET_C_FLAGS)
+ pvs_studio_set_target_flags("${TARGET}" PVS_STUDIO_TARGET_CXX_FLAGS PVS_STUDIO_TARGET_C_FLAGS)
+
+ if (NOT "${PROPERTY}" STREQUAL "NOTFOUND" AND NOT "${PROPERTY}" STREQUAL "PROPERTY-NOTFOUND")
+ foreach (SOURCE ${PROPERTY})
+ pvs_studio_join_path(SOURCE "${DIR}" "${SOURCE}")
+ pvs_studio_analyze_file("${SOURCE}" "${DIR}" "${BINARY_DIR}")
+ endforeach ()
+ endif ()
+
+ set(PVS_STUDIO_PLOGS "${PVS_STUDIO_PLOGS}" PARENT_SCOPE)
+endfunction ()
+
+set(PVS_STUDIO_RECURSIVE_TARGETS)
+set(PVS_STUDIO_RECURSIVE_TARGETS_NEW)
+
+macro(pvs_studio_get_recursive_targets TARGET)
+ get_target_property(libs "${TARGET}" LINK_LIBRARIES)
+ foreach (lib IN LISTS libs)
+ list(FIND PVS_STUDIO_RECURSIVE_TARGETS "${lib}" index)
+ if (TARGET "${lib}" AND "${index}" STREQUAL -1)
+ get_target_property(target_type "${lib}" TYPE)
+ if (NOT "${target_type}" STREQUAL "INTERFACE_LIBRARY")
+ list(APPEND PVS_STUDIO_RECURSIVE_TARGETS "${lib}")
+ list(APPEND PVS_STUDIO_RECURSIVE_TARGETS_NEW "${lib}")
+ pvs_studio_get_recursive_targets("${lib}")
+ endif ()
+ endif ()
+ endforeach ()
+endmacro()
+
+option(PVS_STUDIO_DISABLE OFF "Disable PVS-Studio targets")
+option(PVS_STUDIO_DEBUG OFF "Add debug info")
+
+# pvs_studio_add_target
+# Target options:
+# ALL add PVS-Studio target to default build (default: off)
+# TARGET target name of analysis target (default: pvs)
+# ANALYZE targets... targets to analyze
+# RECURSIVE analyze target's dependencies (requires CMake 3.5+)
+# COMPILE_COMMANDS use compile_commands.json instead of targets (specified by the 'ANALYZE' option) to determine files for analysis
+# (set CMAKE_EXPORT_COMPILE_COMMANDS, available only for Makefile and Ninja generators)
+#
+# Output options:
+# OUTPUT prints report to stdout
+# LOG path path to report (default: ${CMAKE_CURRENT_BINARY_DIR}/PVS-Studio.log)
+# FORMAT format format of report
+# MODE mode analyzers/levels filter (default: GA:1,2)
+# HIDE_HELP do not print help message
+#
+# Analyzer options:
+# PLATFORM name linux32/linux64 (default: linux64)
+# PREPROCESSOR name preprocessor type: gcc/clang (default: auto detected)
+# LICENSE path path to PVS-Studio.lic (default: ~/.config/PVS-Studio/PVS-Studio.lic)
+# CONFIG path path to PVS-Studio.cfg
+# CFG_TEXT text embedded PVS-Studio.cfg
+# KEEP_COMBINED_PLOG do not delete combined plog file *.pvs.raw for further processing with plog-converter
+#
+# Misc options:
+# DEPENDS targets.. additional target dependencies
+# SOURCES path... list of source files to analyze
+# BIN path path to pvs-studio-analyzer (Unix) or CompilerCommandsAnalyzer.exe (Windows)
+# CONVERTER path path to plog-converter (Unix) or HtmlGenerator.exe (Windows)
+# C_FLAGS flags... additional C_FLAGS
+# CXX_FLAGS flags... additional CXX_FLAGS
+# ARGS args... additional pvs-studio-analyzer/CompilerCommandsAnalyzer.exe flags
+function (pvs_studio_add_target)
+ macro (default VAR VALUE)
+ if ("${${VAR}}" STREQUAL "")
+ set("${VAR}" "${VALUE}")
+ endif ()
+ endmacro ()
+
+ set(PVS_STUDIO_SUPPORTED_PREPROCESSORS "gcc|clang|visualcpp")
+ if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+ set(DEFAULT_PREPROCESSOR "clang")
+ elseif (MSVC)
+ set(DEFAULT_PREPROCESSOR "visualcpp")
+ else ()
+ set(DEFAULT_PREPROCESSOR "gcc")
+ endif ()
+
+ set(OPTIONAL OUTPUT ALL RECURSIVE HIDE_HELP KEEP_COMBINED_PLOG COMPILE_COMMANDS)
+ set(SINGLE LICENSE CONFIG TARGET LOG FORMAT BIN CONVERTER PLATFORM PREPROCESSOR CFG_TEXT)
+ set(MULTI SOURCES C_FLAGS CXX_FLAGS ARGS DEPENDS ANALYZE MODE)
+ cmake_parse_arguments(PVS_STUDIO "${OPTIONAL}" "${SINGLE}" "${MULTI}" ${ARGN})
+
+ if ("${PVS_STUDIO_CONFIG}" STREQUAL "" OR NOT "${PVS_STUDIO_CFG_TEXT}" STREQUAL "")
+ set(PVS_STUDIO_EMPTY_CONFIG ON)
+ else ()
+ set(PVS_STUDIO_EMPTY_CONFIG OFF)
+ endif ()
+
+ default(PVS_STUDIO_CFG_TEXT "analysis-mode=31")
+ default(PVS_STUDIO_CONFIG "${CMAKE_BINARY_DIR}/PVS-Studio.cfg")
+ default(PVS_STUDIO_C_FLAGS "")
+ default(PVS_STUDIO_CXX_FLAGS "")
+ default(PVS_STUDIO_TARGET "pvs")
+ default(PVS_STUDIO_LOG "PVS-Studio.log")
+
+ set(PATHS)
+ if (WIN32)
+ set(ROOT "PROGRAMFILES(X86)")
+ set(ROOT "$ENV{${ROOT}}/PVS-Studio")
+ string(REPLACE \\ / ROOT "${ROOT}")
+
+ if (EXISTS "${ROOT}")
+ set(PATHS "${ROOT}")
+ endif ()
+
+ default(PVS_STUDIO_BIN "CompilerCommandsAnalyzer.exe")
+ default(PVS_STUDIO_CONVERTER "HtmlGenerator.exe")
+ else ()
+ default(PVS_STUDIO_BIN "pvs-studio-analyzer")
+ default(PVS_STUDIO_CONVERTER "plog-converter")
+ endif ()
+
+ find_program(PVS_STUDIO_BIN_PATH "${PVS_STUDIO_BIN}" ${PATHS})
+ set(PVS_STUDIO_BIN "${PVS_STUDIO_BIN_PATH}")
+
+ if (NOT EXISTS "${PVS_STUDIO_BIN}")
+ message(FATAL_ERROR "pvs-studio-analyzer is not found")
+ endif ()
+
+ find_program(PVS_STUDIO_CONVERTER_PATH "${PVS_STUDIO_CONVERTER}" ${PATHS})
+ set(PVS_STUDIO_CONVERTER "${PVS_STUDIO_CONVERTER_PATH}")
+
+ if (NOT EXISTS "${PVS_STUDIO_CONVERTER}")
+ message(FATAL_ERROR "plog-converter is not found")
+ endif ()
+
+ default(PVS_STUDIO_MODE "GA:1,2")
+ default(PVS_STUDIO_PREPROCESSOR "${DEFAULT_PREPROCESSOR}")
+ if (WIN32)
+ default(PVS_STUDIO_PLATFORM "x64")
+ else ()
+ default(PVS_STUDIO_PLATFORM "linux64")
+ endif ()
+
+ string(REPLACE ";" "+" PVS_STUDIO_MODE "${PVS_STUDIO_MODE}")
+
+ if (PVS_STUDIO_EMPTY_CONFIG)
+ set(PVS_STUDIO_CONFIG_COMMAND "${CMAKE_COMMAND}" -E echo "${PVS_STUDIO_CFG_TEXT}" > "${PVS_STUDIO_CONFIG}")
+ else ()
+ set(PVS_STUDIO_CONFIG_COMMAND "${CMAKE_COMMAND}" -E touch "${PVS_STUDIO_CONFIG}")
+ endif ()
+
+ add_custom_command(OUTPUT "${PVS_STUDIO_CONFIG}"
+ COMMAND ${PVS_STUDIO_CONFIG_COMMAND}
+ WORKING_DIRECTORY "${BINARY_DIR}"
+ COMMENT "Generating PVS-Studio.cfg")
+
+ if (NOT "${PVS_STUDIO_PREPROCESSOR}" MATCHES "^${PVS_STUDIO_SUPPORTED_PREPROCESSORS}$")
+ message(FATAL_ERROR "Preprocessor ${PVS_STUDIO_PREPROCESSOR} isn't supported. Available options: ${PVS_STUDIO_SUPPORTED_PREPROCESSORS}.")
+ endif ()
+
+ pvs_studio_append_standard_flag(PVS_STUDIO_CXX_FLAGS "${CMAKE_CXX_STANDARD}")
+ pvs_studio_set_directory_flags("${CMAKE_CURRENT_SOURCE_DIR}" PVS_STUDIO_CXX_FLAGS PVS_STUDIO_C_FLAGS)
+
+ if (NOT "${PVS_STUDIO_LICENSE}" STREQUAL "")
+ pvs_studio_join_path(PVS_STUDIO_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}" "${PVS_STUDIO_LICENSE}")
+ list(APPEND PVS_STUDIO_ARGS --lic-file "${PVS_STUDIO_LICENSE}")
+ endif ()
+
+ list(APPEND PVS_STUDIO_ARGS --cfg "${PVS_STUDIO_CONFIG}"
+ --platform "${PVS_STUDIO_PLATFORM}"
+ --preprocessor "${PVS_STUDIO_PREPROCESSOR}")
+
+ if (NOT "${CMAKE_CXX_COMPILER}" STREQUAL "")
+ list(APPEND PVS_STUDIO_ARGS --cxx "${CMAKE_CXX_COMPILER}")
+ endif ()
+
+ if (NOT "${CMAKE_C_COMPILER}" STREQUAL "")
+ list(APPEND PVS_STUDIO_ARGS --cc "${CMAKE_C_COMPILER}")
+ endif ()
+
+ set(PVS_STUDIO_PLOGS "")
+
+ set(PVS_STUDIO_RECURSIVE_TARGETS_NEW)
+ if (${PVS_STUDIO_RECURSIVE})
+ foreach (TARGET IN LISTS PVS_STUDIO_ANALYZE)
+ list(APPEND PVS_STUDIO_RECURSIVE_TARGETS_NEW "${TARGET}")
+ pvs_studio_get_recursive_targets("${TARGET}")
+ endforeach ()
+ endif ()
+
+ set(inc_path)
+
+ foreach (TARGET ${PVS_STUDIO_ANALYZE})
+ set(DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+ string(FIND "${TARGET}" ":" DELIM)
+ if ("${DELIM}" GREATER "-1")
+ math(EXPR DELIMI "${DELIM}+1")
+ string(SUBSTRING "${TARGET}" "${DELIMI}" "-1" DIR)
+ string(SUBSTRING "${TARGET}" "0" "${DELIM}" TARGET)
+ pvs_studio_join_path(DIR "${CMAKE_CURRENT_SOURCE_DIR}" "${DIR}")
+ else ()
+ get_target_property(TARGET_SOURCE_DIR "${TARGET}" SOURCE_DIR)
+ if (EXISTS "${TARGET_SOURCE_DIR}")
+ set(DIR "${TARGET_SOURCE_DIR}")
+ endif ()
+ endif ()
+ pvs_studio_analyze_target("${TARGET}" "${DIR}")
+ list(APPEND PVS_STUDIO_DEPENDS "${TARGET}")
+
+ if ("${inc_path}" STREQUAL "")
+ set(inc_path "$<TARGET_PROPERTY:${TARGET},INCLUDE_DIRECTORIES>")
+ else ()
+ set(inc_path "${inc_path}$<SEMICOLON>$<TARGET_PROPERTY:${TARGET},INCLUDE_DIRECTORIES>")
+ endif ()
+ endforeach ()
+
+ foreach (TARGET ${PVS_STUDIO_RECURSIVE_TARGETS_NEW})
+ set(DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+ get_target_property(TARGET_SOURCE_DIR "${TARGET}" SOURCE_DIR)
+ if (EXISTS "${TARGET_SOURCE_DIR}")
+ set(DIR "${TARGET_SOURCE_DIR}")
+ endif ()
+ pvs_studio_analyze_target("${TARGET}" "${DIR}")
+ list(APPEND PVS_STUDIO_DEPENDS "${TARGET}")
+ endforeach ()
+
+ set(PVS_STUDIO_TARGET_CXX_FLAGS "")
+ set(PVS_STUDIO_TARGET_C_FLAGS "")
+ foreach (SOURCE ${PVS_STUDIO_SOURCES})
+ pvs_studio_analyze_file("${SOURCE}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
+ endforeach ()
+
+ if (PVS_STUDIO_COMPILE_COMMANDS)
+ set(COMPILE_COMMANDS_LOG "${PVS_STUDIO_LOG}.pvs.analyzer.raw")
+ if (NOT CMAKE_EXPORT_COMPILE_COMMANDS)
+ message(FATAL_ERROR "You should set CMAKE_EXPORT_COMPILE_COMMANDS to TRUE")
+ endif ()
+ add_custom_command(
+ OUTPUT "${COMPILE_COMMANDS_LOG}"
+ COMMAND "${PVS_STUDIO_BIN}" analyze -i
+ --output-file "${COMPILE_COMMANDS_LOG}.always"
+ ${PVS_STUDIO_ARGS}
+ COMMENT "Analyzing with PVS-Studio"
+ WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+ DEPENDS "${PVS_STUDIO_CONFIG}"
+ )
+ list(APPEND PVS_STUDIO_PLOGS_LOGS "${COMPILE_COMMANDS_LOG}.always")
+ list(APPEND PVS_STUDIO_PLOGS_DEPENDENCIES "${COMPILE_COMMANDS_LOG}")
+ endif ()
+
+ pvs_studio_relative_path(LOG_RELATIVE "${CMAKE_BINARY_DIR}" "${PVS_STUDIO_LOG}")
+ if (PVS_STUDIO_PLOGS OR PVS_STUDIO_COMPILE_COMMANDS)
+ if (WIN32)
+ string(REPLACE / \\ PVS_STUDIO_PLOGS "${PVS_STUDIO_PLOGS}")
+ endif ()
+ if (WIN32)
+ set(COMMANDS COMMAND type ${PVS_STUDIO_PLOGS} ${PVS_STUDIO_PLOGS_LOGS} > "${PVS_STUDIO_LOG}" 2>nul)
+ else ()
+ set(COMMANDS COMMAND cat ${PVS_STUDIO_PLOGS} ${PVS_STUDIO_PLOGS_LOGS} > "${PVS_STUDIO_LOG}")
+ endif ()
+ set(COMMENT "Generating ${LOG_RELATIVE}")
+ if (NOT "${PVS_STUDIO_FORMAT}" STREQUAL "" OR PVS_STUDIO_OUTPUT)
+ if ("${PVS_STUDIO_FORMAT}" STREQUAL "")
+ set(PVS_STUDIO_FORMAT "errorfile")
+ endif ()
+ list(APPEND COMMANDS
+ COMMAND "${CMAKE_COMMAND}" -E remove -f "${PVS_STUDIO_LOG}.pvs.raw"
+ COMMAND "${CMAKE_COMMAND}" -E rename "${PVS_STUDIO_LOG}" "${PVS_STUDIO_LOG}.pvs.raw"
+ COMMAND "${PVS_STUDIO_CONVERTER}" -t "${PVS_STUDIO_FORMAT}" "${PVS_STUDIO_LOG}.pvs.raw" -o "${PVS_STUDIO_LOG}" -a "${PVS_STUDIO_MODE}"
+ )
+ if(NOT PVS_STUDIO_KEEP_COMBINED_PLOG)
+ list(APPEND COMMANDS COMMAND "${CMAKE_COMMAND}" -E remove -f "${PVS_STUDIO_LOG}.pvs.raw")
+ endif()
+ endif ()
+ else ()
+ set(COMMANDS COMMAND "${CMAKE_COMMAND}" -E touch "${PVS_STUDIO_LOG}")
+ set(COMMENT "Generating ${LOG_RELATIVE}: no sources found")
+ endif ()
+
+ if (WIN32)
+ string(REPLACE / \\ PVS_STUDIO_LOG "${PVS_STUDIO_LOG}")
+ endif ()
+
+ add_custom_command(OUTPUT "${PVS_STUDIO_LOG}"
+ ${COMMANDS}
+ COMMENT "${COMMENT}"
+ DEPENDS ${PVS_STUDIO_PLOGS} ${PVS_STUDIO_PLOGS_DEPENDENCIES}
+ WORKING_DIRECTORY "${CMAKE_BINARY_DIR}")
+
+ if (PVS_STUDIO_ALL)
+ set(ALL "ALL")
+ else ()
+ set(ALL "")
+ endif ()
+
+ if (PVS_STUDIO_OUTPUT)
+ if (PVS_STUDIO_HIDE_HELP AND NOT WIN32)
+ set(COMMANDS COMMAND grep -v " error: Help:" ${PVS_STUDIO_LOG} 1>&2 || exit 0)
+ elseif (WIN32)
+ set(COMMANDS COMMAND type "${PVS_STUDIO_LOG}" 1>&2)
+ else ()
+ set(COMMANDS COMMAND cat "${PVS_STUDIO_LOG}" 1>&2)
+ endif()
+ else ()
+ set(COMMANDS "")
+ endif ()
+
+ add_custom_target("${PVS_STUDIO_TARGET}" ${ALL} ${COMMANDS} WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" DEPENDS ${PVS_STUDIO_DEPENDS} "${PVS_STUDIO_LOG}")
+
+ # A workaround to add implicit dependencies of source files from include directories
+ set_target_properties("${PVS_STUDIO_TARGET}" PROPERTIES INCLUDE_DIRECTORIES "${inc_path}")
+endfunction () \ No newline at end of file
diff --git a/cmake/Paths.cmake b/cmake/Paths.cmake
new file mode 100644
index 0000000..858cdc2
--- /dev/null
+++ b/cmake/Paths.cmake
@@ -0,0 +1,72 @@
+# Now CMAKE_INSTALL_PREFIX is a base prefix for everything
+# CONFDIR - for configuration
+# LOCAL_CONFDIR - for local configuration
+# MANDIR - for manual pages
+# RUNDIR - for runtime files
+# DBDIR - for static files
+# LOGDIR - for log files
+
+IF(NOT CONFDIR)
+ SET(CONFDIR "${CMAKE_INSTALL_PREFIX}/etc/rspamd")
+ENDIF(NOT CONFDIR)
+
+IF(NOT LOCAL_CONFDIR)
+ SET(LOCAL_CONFDIR "${CONFDIR}")
+ENDIF(NOT LOCAL_CONFDIR)
+
+IF(NOT MANDIR)
+ SET(MANDIR "${CMAKE_INSTALL_PREFIX}/share/man")
+ENDIF(NOT MANDIR)
+
+IF(NOT RUNDIR)
+ SET(RUNDIR "/var/run/rspamd")
+ENDIF(NOT RUNDIR)
+
+IF(NOT DBDIR)
+ SET(DBDIR "/var/lib/rspamd")
+ENDIF(NOT DBDIR)
+
+IF(NOT LOGDIR)
+ SET(LOGDIR "/var/log/rspamd")
+ENDIF(NOT LOGDIR)
+
+IF(NOT SHAREDIR)
+ SET(SHAREDIR "${CMAKE_INSTALL_PREFIX}/share/rspamd")
+ENDIF(NOT SHAREDIR)
+
+IF(NOT LUALIBDIR)
+ SET(LUALIBDIR "${SHAREDIR}/lualib")
+ENDIF(NOT LUALIBDIR)
+
+IF(NOT PLUGINSDIR)
+ SET(PLUGINSDIR "${SHAREDIR}/plugins")
+ENDIF(NOT PLUGINSDIR)
+
+IF(NOT RULESDIR)
+ SET(RULESDIR "${SHAREDIR}/rules")
+ENDIF(NOT RULESDIR)
+
+IF(NOT WWWDIR)
+ SET(WWWDIR "${SHAREDIR}/www")
+ENDIF(NOT WWWDIR)
+
+# Set libdir
+IF(NOT LIBDIR)
+ SET(RSPAMD_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/rspamd")
+ELSE(NOT LIBDIR)
+ SET(RSPAMD_LIBDIR "${LIBDIR}")
+ENDIF(NOT LIBDIR)
+SET(CMAKE_MACOSX_RPATH ON)
+SET(CMAKE_INSTALL_RPATH "${RSPAMD_LIBDIR}")
+
+# Set includedir
+IF(NOT INCLUDEDIR)
+ SET(INCLUDEDIR include/rspamd)
+ENDIF(NOT INCLUDEDIR)
+
+IF(NOT SYSTEMDDIR)
+ SET(SYSTEMDDIR ${CMAKE_INSTALL_PREFIX}/lib/systemd/system)
+ENDIF(NOT SYSTEMDDIR)
+
+SET(RSPAMD_DEFAULT_INCLUDE_PATHS "/opt;/usr;/usr/local;/opt/local;/usr/pkg;/opt/csw;/sw")
+SET(RSPAMD_DEFAULT_LIBRARY_PATHS "/usr/local;/usr/pkg;/usr;/Library/Frameworks;/sw;/opt/local;/opt/csw;/opt")
diff --git a/cmake/ProcessPackage.cmake b/cmake/ProcessPackage.cmake
new file mode 100644
index 0000000..a46fd85
--- /dev/null
+++ b/cmake/ProcessPackage.cmake
@@ -0,0 +1,122 @@
+# Process required package by using FindPackage and calling for INCLUDE_DIRECTORIES and
+# setting list of required libraries
+# Usage:
+# ProcessPackage(VAR [OPTIONAL] [ROOT path] [INCLUDE path]
+# [LIBRARY path] [INCLUDE_SUFFIXES path1 path2 ...] [LIB_SUFFIXES path1 path2 ...]
+# [MODULES module1 module2 ...])
+# params:
+# OPTIONAL - do not fail if a package has not been found
+# ROOT - defines root directory for a package
+# INCLUDE - name of the include file to check
+# LIBRARY - name of the library to check
+# INCLUDE_SUFFIXES - list of include suffixes (relative to ROOT)
+# LIB_SUFFIXES - list of library suffixes
+# MODULES - modules to search using pkg_config
+MACRO(ProcessPackage PKG_NAME)
+
+ CMAKE_PARSE_ARGUMENTS(PKG "OPTIONAL;OPTIONAL_INCLUDE" "ROOT;INCLUDE"
+ "LIBRARY;INCLUDE_SUFFIXES;LIB_SUFFIXES;MODULES;LIB_OUTPUT" ${ARGN})
+
+ IF(NOT PKG_LIBRARY)
+ SET(PKG_LIBRARY "${PKG_NAME}")
+ ENDIF()
+ IF(NOT PKG_INCLUDE)
+ SET(PKG_INCLUDE "${PKG_NAME}.h")
+ ENDIF()
+ IF(NOT PKG_LIB_OUTPUT)
+ SET(PKG_LIB_OUTPUT RSPAMD_REQUIRED_LIBRARIES)
+ ENDIF()
+
+ IF(NOT PKG_ROOT AND PKG_MODULES)
+ PKG_SEARCH_MODULE(${PKG_NAME} ${PKG_MODULES})
+ ENDIF()
+
+ IF(${PKG_NAME}_FOUND)
+ MESSAGE(STATUS "Found package ${PKG_NAME} in pkg-config modules ${PKG_MODULES}")
+ SET(WITH_${PKG_NAME} 1 CACHE INTERNAL "")
+ IF(ENABLE_STATIC MATCHES "ON")
+ SET(_XPREFIX "${PKG_NAME}_STATIC")
+ ELSE(ENABLE_STATIC MATCHES "ON")
+ SET(_XPREFIX "${PKG_NAME}")
+ ENDIF(ENABLE_STATIC MATCHES "ON")
+ FOREACH(_arg ${${_XPREFIX}_INCLUDE_DIRS})
+ INCLUDE_DIRECTORIES("${_arg}")
+ SET(${PKG_NAME}_INCLUDE "${_arg}" CACHE INTERNAL "")
+ ENDFOREACH(_arg ${${_XPREFIX}_INCLUDE_DIRS})
+ FOREACH(_arg ${${_XPREFIX}_LIBRARY_DIRS})
+ LINK_DIRECTORIES("${_arg}")
+ SET(${PKG_NAME}_LIBRARY_PATH "${_arg}" CACHE INTERNAL "")
+ ENDFOREACH(_arg ${${_XPREFIX}_LIBRARY_DIRS})
+ # Handle other CFLAGS and LDFLAGS
+ FOREACH(_arg ${${_XPREFIX}_CFLAGS_OTHER})
+ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_arg}")
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_arg}")
+ ENDFOREACH(_arg ${${_XPREFIX}_CFLAGS_OTHER})
+ FOREACH(_arg ${${_XPREFIX}_LDFLAGS_OTHER})
+ SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${_arg}")
+ ENDFOREACH(_arg ${${_XPREFIX}_LDFLAGS_OTHER})
+ LIST(APPEND ${PKG_LIB_OUTPUT} "${${_XPREFIX}_LIBRARIES}")
+ INCLUDE_DIRECTORIES(${${_XPREFIX}_INCLUDEDIR})
+ ELSE()
+ IF(NOT ${PKG_NAME}_GUESSED)
+ # Try some more heuristic
+ FIND_LIBRARY(_lib NAMES ${PKG_LIBRARY}
+ HINTS ${PKG_ROOT} ${RSPAMD_SEARCH_PATH}
+ PATH_SUFFIXES ${PKG_LIB_SUFFIXES} lib64 lib
+ PATHS ${RSPAMD_DEFAULT_LIBRARY_PATHS})
+ IF(NOT _lib)
+ IF(PKG_OPTIONAL)
+ MESSAGE(STATUS "Cannot find library ${PKG_LIBRARY} for package ${PKG_NAME}, ignoring")
+ ELSE()
+ MESSAGE(FATAL_ERROR "Cannot find library ${PKG_LIBRARY} for package ${PKG_NAME}")
+ ENDIF()
+ ENDIF(NOT _lib)
+
+ FIND_PATH(_incl ${PKG_INCLUDE}
+ HINTS ${PKG_ROOT} ${RSPAMD_SEARCH_PATH}
+ PATH_SUFFIXES ${PKG_INCLUDE_SUFFIXES} include
+ PATHS {RSPAMD_DEFAULT_INCLUDE_PATHS})
+ IF(NOT _incl)
+ IF(PKG_OPTIONAL OR PKG_OPTIONAL_INCLUDE)
+ MESSAGE(STATUS "Cannot find header ${PKG_INCLUDE} for package ${PKG_NAME}")
+ ELSE()
+ MESSAGE(FATAL_ERROR "Cannot find header ${PKG_INCLUDE} for package ${PKG_NAME}")
+ ENDIF()
+ ELSE()
+ STRING(REGEX REPLACE "/[^/]+$" "" _incl_path "${PKG_INCLUDE}")
+ STRING(REGEX REPLACE "${_incl_path}/$" "" _stripped_incl "${_incl}")
+ INCLUDE_DIRECTORIES("${_stripped_incl}")
+ SET(${PKG_NAME}_INCLUDE "${_stripped_incl}" CACHE INTERNAL "")
+ ENDIF(NOT _incl)
+
+ IF(_lib)
+ # We need to apply heuristic to find the real dir name
+ GET_FILENAME_COMPONENT(_lib_path "${_lib}" PATH)
+ LINK_DIRECTORIES("${_lib_path}")
+ LIST(APPEND ${PKG_LIB_OUTPUT} ${_lib})
+ SET(${PKG_NAME}_LIBRARY_PATH "${_lib_path}" CACHE INTERNAL "")
+ SET(${PKG_NAME}_LIBRARY "${_lib}" CACHE INTERNAL "")
+ ENDIF()
+
+ IF(_incl AND _lib)
+ MESSAGE(STATUS "Found package ${PKG_NAME} in '${_lib_path}' (${_lib}) and '${_stripped_incl}' (${PKG_INCLUDE}).")
+ SET(${PKG_NAME}_GUESSED 1 CACHE INTERNAL "")
+ SET(WITH_${PKG_NAME} 1 CACHE INTERNAL "")
+ ELSEIF(_lib)
+ IF(PKG_OPTIONAL_INCLUDE)
+ SET(${PKG_NAME}_GUESSED 1 INTERNAL "")
+ SET(WITH_${PKG_NAME} 1 INTERNAL "")
+ ENDIF()
+ MESSAGE(STATUS "Found incomplete package ${PKG_NAME} in '${_lib_path}' (${_lib}); no includes.")
+ ENDIF()
+ ELSE()
+ MESSAGE(STATUS "Found package ${PKG_NAME} (cached)")
+ INCLUDE_DIRECTORIES("${${PKG_NAME}_INCLUDE}")
+ LINK_DIRECTORIES("${${PKG_NAME}_LIBRARY_PATH}")
+ LIST(APPEND ${PKG_LIB_OUTPUT} "${${PKG_NAME}_LIBRARY}")
+ ENDIF()
+ ENDIF(${PKG_NAME}_FOUND)
+
+ UNSET(_lib CACHE)
+ UNSET(_incl CACHE)
+ENDMACRO(ProcessPackage name) \ No newline at end of file
diff --git a/cmake/Sanitizer.cmake b/cmake/Sanitizer.cmake
new file mode 100644
index 0000000..c258706
--- /dev/null
+++ b/cmake/Sanitizer.cmake
@@ -0,0 +1,75 @@
+# Ported from Clickhouse: https://github.com/ClickHouse/ClickHouse/blob/master/cmake/sanitize.cmake
+
+set (SAN_FLAGS "${SAN_FLAGS} -g -fno-omit-frame-pointer -DSANITIZER")
+# O1 is normally set by clang, and -Og by gcc
+if (COMPILER_GCC)
+ if (ENABLE_FULL_DEBUG MATCHES "ON")
+ set (SAN_FLAGS "${SAN_FLAGS} -O0")
+ else()
+ set (SAN_FLAGS "${SAN_FLAGS} -Og")
+ endif()
+else ()
+ if (ENABLE_FULL_DEBUG MATCHES "ON")
+ set (SAN_FLAGS "${SAN_FLAGS} -O0")
+ else()
+ set (SAN_FLAGS "${SAN_FLAGS} -O1")
+ endif()
+endif ()
+if (SANITIZE)
+ if (ENABLE_JEMALLOC MATCHES "ON")
+ message (STATUS "Jemalloc support is useless in case of build with sanitizers")
+ set (ENABLE_JEMALLOC "OFF")
+ endif ()
+
+ string(REPLACE "," ";" SANITIZE_LIST ${SANITIZE})
+ foreach(SANITIZE ${SANITIZE_LIST})
+ if (SANITIZE STREQUAL "address")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope")
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope")
+ if (COMPILER_GCC)
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -static-libasan")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libasan")
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libasan")
+ endif ()
+
+ elseif (SANITIZE STREQUAL "leak")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=leak")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=leak")
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=leak")
+
+ elseif (SANITIZE STREQUAL "memory")
+ set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls")
+
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${MSAN_FLAGS}")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${MSAN_FLAGS}")
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory")
+
+ if (COMPILER_GCC)
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -static-libmsan")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libmsan")
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libmsan")
+ endif ()
+
+ elseif (SANITIZE STREQUAL "undefined")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all")
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
+
+ if (COMPILER_GCC)
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -static-libubsan")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libubsan")
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan")
+ endif ()
+ else ()
+ message (FATAL_ERROR "Unknown sanitizer type: ${SANITIZE}")
+ endif ()
+ endforeach ()
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS}")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS}")
+ message (STATUS "Add sanitizer: ${SANITIZE}")
+ # Disable sanitizing on make stage e.g. for snowball compiler
+ set (ENV{ASAN_OPTIONS} "detect_leaks=0")
+ message (STATUS "Sanitizer CFLAGS: ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
+ message (STATUS "Sanitizer CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
+endif() \ No newline at end of file
diff --git a/cmake/Toolset.cmake b/cmake/Toolset.cmake
new file mode 100644
index 0000000..4e7017d
--- /dev/null
+++ b/cmake/Toolset.cmake
@@ -0,0 +1,252 @@
+option (ENABLE_FAST_MATH "Build rspamd with fast math compiler flag [default: ON]" ON)
+option (ENABLE_ANALYZER "Build rspamd with static analyzer [default: OFF]" OFF)
+option (ENABLE_STATIC_LIBCXX "Build rspamd with static lib(std)c++ [default: OFF]" OFF)
+option (ENABLE_COMPILE_TIME "Show compile time [default: OFF]" OFF)
+option (ENABLE_LIBCXX "Use libc++ instead of libstdc++" OFF)
+
+if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
+ SET (COMPILER_GCC 1)
+elseif(CMAKE_C_COMPILER_ID MATCHES "Clang|AppleClang")
+ SET (COMPILER_CLANG 1)
+endif()
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+SET (COMPILER_FAST_MATH "")
+if (ENABLE_FAST_MATH MATCHES "ON")
+ # We need to keep nans and infinities, so cannot keep all fast math there
+ IF (COMPILER_CLANG)
+ SET (COMPILER_FAST_MATH "-fassociative-math -freciprocal-math -fno-signed-zeros -ffp-contract=fast")
+ ELSE()
+ SET (COMPILER_FAST_MATH "-funsafe-math-optimizations -fno-math-errno")
+ ENDIF ()
+endif ()
+
+if (CMAKE_GENERATOR STREQUAL "Ninja")
+ # Turn on colored output. https://github.com/ninja-build/ninja/wiki/FAQ
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
+endif ()
+
+if (COMPILER_GCC)
+ # Require minimum version of gcc
+ set (GCC_MINIMUM_VERSION 8)
+ if (CMAKE_C_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9)
+ message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}.")
+ endif ()
+ if (ENABLE_LIBCXX MATCHES "ON")
+ # XXX: too complicated to implement for now
+ endif ()
+elseif (COMPILER_CLANG)
+ # Require minimum version of clang
+ set (CLANG_MINIMUM_VERSION 7)
+ if (CMAKE_C_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
+ message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.")
+ endif ()
+ # Hack to fix try_compile
+ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-command-line-argument")
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-command-line-argument")
+ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-ignored-optimization-argument")
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-ignored-optimization-argument")
+ if (ENABLE_LIBCXX MATCHES "AUTO")
+ include(CheckCXXSourceCompiles)
+ set(OLD_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+ check_cxx_source_compiles("
+#include <version>
+int main() { return 0; }
+ " HAVE_LIBCXX )
+ if (HAVE_LIBCXX)
+ SET(ENABLE_LIBCXX "ON")
+ else()
+ SET(ENABLE_LIBCXX "OFF")
+ endif()
+ set(CMAKE_CXX_FLAGS "${OLD_CMAKE_CXX_FLAGS}")
+ endif()
+ if (ENABLE_LIBCXX MATCHES "ON")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+ set(CLANG_DEFAULT_CXX_STDLIB "libc++")
+ endif ()
+else ()
+ message (WARNING "You are using an unsupported compiler ${CMAKE_C_COMPILER_ID}. Compilation has only been tested with Clang 4+ and GCC 4+.")
+endif ()
+
+option(LINKER_NAME "Linker name or full path")
+
+find_program(LLD_PATH NAMES "ld.lld" "lld")
+find_program(GOLD_PATH NAMES "ld.gold" "gold")
+
+if(NOT LINKER_NAME)
+ if(LLD_PATH)
+ if (COMPILER_CLANG)
+ set(LINKER_NAME "lld")
+ elseif(NOT SANITIZE)
+ if(GOLD_PATH)
+ set(LINKER_NAME "gold")
+ else()
+ message(STATUS "Use generic 'ld' as a linker: gold not found")
+ endif()
+ else()
+ message(STATUS "Use generic 'ld' as a linker: sanitizers are enabled")
+ endif()
+ elseif(GOLD_PATH)
+ set(LINKER_NAME "gold")
+ else()
+ message(STATUS "Use generic 'ld' as a linker")
+ endif()
+endif()
+
+if(LINKER_NAME)
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
+ set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
+
+ message(STATUS "Using custom linker by name: ${LINKER_NAME}")
+endif ()
+
+option (ENABLE_STATIC "Enable static compiling [default: OFF]" OFF)
+
+if (ENABLE_STATIC MATCHES "ON")
+ MESSAGE(STATUS "Static build of rspamd implies that the target binary will be *GPL* licensed")
+ SET(GPL_RSPAMD_BINARY 1)
+ SET(CMAKE_SKIP_INSTALL_RPATH ON)
+ SET(BUILD_STATIC 1)
+ SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+ SET(BUILD_SHARED_LIBS OFF)
+ SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static")
+ SET(LINK_TYPE "STATIC")
+ SET(NO_SHARED "ON")
+ # Dirty hack for cmake
+ SET(CMAKE_EXE_LINK_DYNAMIC_C_FLAGS) # remove -Wl,-Bdynamic
+ SET(CMAKE_EXE_LINK_DYNAMIC_CXX_FLAGS)
+ SET(CMAKE_SHARED_LIBRARY_C_FLAGS) # remove -fPIC
+ SET(CMAKE_SHARED_LIBRARY_CXX_FLAGS)
+ SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS) # remove -rdynamic
+ SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS)
+else ()
+ if (NO_SHARED MATCHES "OFF")
+ SET(LINK_TYPE "SHARED")
+ else ()
+ SET(LINK_TYPE "STATIC")
+ endif ()
+endif ()
+
+
+# Legacy options support
+option (ENABLE_COVERAGE "Build rspamd with code coverage options [default: OFF]" OFF)
+option (ENABLE_OPTIMIZATION "Enable extra optimizations [default: OFF]" OFF)
+option (SKIP_RELINK_RPATH "Skip relinking and full RPATH for the install tree" OFF)
+option (ENABLE_FULL_DEBUG "Build rspamd with all possible debug [default: OFF]" OFF)
+
+if(NOT CMAKE_BUILD_TYPE)
+ if (ENABLE_FULL_DEBUG MATCHES "ON")
+ set(CMAKE_BUILD_TYPE Debug CACHE STRING "" FORCE)
+ endif()
+ if (ENABLE_COVERAGE MATCHES "ON")
+ set(CMAKE_BUILD_TYPE Coverage CACHE STRING "" FORCE)
+ endif()
+ if (ENABLE_OPTIMIZATION MATCHES "ON")
+ set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE)
+ endif()
+endif()
+
+if (CMAKE_CONFIGURATION_TYPES) # multiconfig generator?
+ set (CMAKE_CONFIGURATION_TYPES "Debug;RelWithDebInfo;Release;Coverage" CACHE STRING "" FORCE)
+else()
+ if (NOT CMAKE_BUILD_TYPE)
+ if (NOT SANITIZE)
+ message(STATUS "Defaulting to release build.")
+ set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE)
+ else ()
+ message(STATUS "Defaulting to debug build due to sanitizers being enabled.")
+ set(CMAKE_BUILD_TYPE Debug CACHE STRING "" FORCE)
+ endif ()
+ endif()
+ set_property(CACHE CMAKE_BUILD_TYPE PROPERTY HELPSTRING "Choose the type of build")
+ set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug;Release;Coverage")
+endif()
+
+string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
+message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE_UC}")
+set(CMAKE_C_FLAGS_COVERAGE "${CMAKE_C_FLAGS} -O1 --coverage -fno-inline -fno-default-inline -fno-inline-small-functions ${COMPILER_FAST_MATH}")
+set(CMAKE_CXX_FLAGS_COVERAGE "${CMAKE_CXX_FLAGS} -O1 --coverage -fno-inline -fno-default-inline -fno-inline-small-functions ${COMPILER_FAST_MATH}")
+
+if (COMPILER_GCC)
+ # GCC flags
+ set (COMPILER_DEBUG_FLAGS "-g -ggdb -g3 -ggdb3")
+ set (CXX_COMMON_FLAGS "")
+ set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELEASE} -O2 ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS}")
+ set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELEASE} -O2 ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS} ${CXX_COMMON_FLAGS}")
+
+ set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 ${COMPILER_FAST_MATH} -fomit-frame-pointer")
+ set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 ${COMPILER_FAST_MATH} -fomit-frame-pointer ${CXX_COMMON_FLAGS}")
+
+ if (ENABLE_FULL_DEBUG MATCHES "ON")
+ if (ENABLE_ANALYZER MATCHES "ON")
+ # Check support of -fanalyzer
+ CHECK_C_COMPILER_FLAG(-fanalyzer SUPPORT_FANALYZER)
+ if (SUPPORT_FANALYZER)
+ set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -fanalyzer")
+ #set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fanalyzer")
+ endif()
+ endif ()
+ set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 ${COMPILER_DEBUG_FLAGS}")
+ set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${COMPILER_DEBUG_FLAGS} ${CXX_COMMON_FLAGS}")
+ else()
+ set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Og ${COMPILER_DEBUG_FLAGS}")
+ set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Og ${COMPILER_DEBUG_FLAGS} ${CXX_COMMON_FLAGS}")
+ endif()
+elseif (COMPILER_CLANG)
+ # Clang flags
+ set (COMPILER_DEBUG_FLAGS "-g -glldb -gdwarf-aranges -gdwarf-4")
+ set (CXX_COMMON_FLAGS "")
+ set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O2 -fomit-frame-pointer ${COMPILER_FAST_MATH}")
+ set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O2 -fomit-frame-pointer ${COMPILER_FAST_MATH} ${CXX_COMMON_FLAGS}")
+
+ set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELEASE} -O2 ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS}")
+ set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELEASE} -O2 ${COMPILER_FAST_MATH} ${COMPILER_DEBUG_FLAGS} ${CXX_COMMON_FLAGS}")
+
+ set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 ${COMPILER_DEBUG_FLAGS}")
+ set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${COMPILER_DEBUG_FLAGS} ${CXX_COMMON_FLAGS}")
+endif()
+
+
+if (CMAKE_BUILD_TYPE_UC MATCHES "RELEASE|RELWITHDEBINFO")
+ set(ENABLE_LTO_INIT ON)
+else()
+ set(ENABLE_LTO_INIT OFF)
+endif()
+option(ENABLE_LTO "Build rspamd with Link Time Optimization if supported [default: ${ENABLE_LTO_INIT}]" ${ENABLE_LTO_INIT})
+
+if (CMAKE_BUILD_TYPE_UC MATCHES "COVERAGE")
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage")
+ message (STATUS "IPO not enabled for COVERAGE build")
+elseif (ENABLE_LTO)
+ if (${CMAKE_VERSION} VERSION_GREATER "3.9.0")
+ cmake_policy (SET CMP0069 NEW)
+ include (CheckIPOSupported)
+ check_ipo_supported (RESULT SUPPORT_LTO OUTPUT LTO_DIAG )
+ if (SUPPORT_LTO)
+ set (CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+ message (STATUS "Enable IPO for the ${CMAKE_BUILD_TYPE} build")
+ else ()
+ message(WARNING "IPO is not supported: ${LTO_DIAG}")
+ endif ()
+ endif ()
+else ()
+ message (STATUS "IPO not enabled for the ${CMAKE_BUILD_TYPE} build")
+endif ()
+
+if (ENABLE_STATIC_LIBCXX MATCHES "ON")
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++")
+ set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libstdc++")
+endif()
+
+if (ENABLE_COMPILE_TIME MATCHES "ON")
+ set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "/usr/bin/time")
+endif()
+
+message (STATUS "Final CFLAGS: ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
+message (STATUS "Final CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
+message (STATUS "Final link flags for shlib: ${CMAKE_SHARED_LINKER_FLAGS}")
+message (STATUS "Final link flags for exe: ${CMAKE_EXE_LINKER_FLAGS}")
+
diff --git a/conf/actions.conf b/conf/actions.conf
new file mode 100644
index 0000000..8be3823
--- /dev/null
+++ b/conf/actions.conf
@@ -0,0 +1,29 @@
+# Actions settings
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/actions.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/actions.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# See also https://rspamd.com/doc/faq.html#what-are-rspamd-actions for actions definition
+
+actions {
+ reject = 15; # Reject when reaching this score
+ add_header = 6; # Add header when reaching this score
+ greylist = 4; # Apply greylisting when reaching this score (will emit `soft reject action`)
+
+ #unknown_weight = 1.0; # Enable if need to set score for all symbols implicitly
+ # Each new symbol is added multiplied by gf^N, where N is the number of spammy symbols
+ #grow_factor = 1.1;
+ # Set rewrite subject to this value (%s is replaced by the original subject)
+ #subject = "***SPAM*** %s"
+
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/actions.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/actions.conf"
+}
diff --git a/conf/cgp.inc b/conf/cgp.inc
new file mode 100644
index 0000000..436713c
--- /dev/null
+++ b/conf/cgp.inc
@@ -0,0 +1,17 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# This file defines some specific settings that are applicable merely when using
+# CommuniGate Pro MTA and it's specific integration
+
+arc {
+ sign_networks = [127.2.4.7];
+}
+
+dkim_signing {
+ sign_networks = [127.2.4.7];
+}
+
+options {
+ local_addrs = [127.2.4.7];
+}
diff --git a/conf/common.conf b/conf/common.conf
new file mode 100644
index 0000000..023fd51
--- /dev/null
+++ b/conf/common.conf
@@ -0,0 +1,40 @@
+# A common rspamd configuration file (should never ever be changed)
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+lua = "$RULESDIR/rspamd.lua"
+
+.include "$CONFDIR/metrics.conf"
+.include "$CONFDIR/actions.conf"
+.include "$CONFDIR/groups.conf"
+.include "$CONFDIR/composites.conf"
+
+.include "$CONFDIR/statistic.conf"
+
+.include "$CONFDIR/modules.conf"
+
+# Include users settings
+.include "$CONFDIR/settings.conf"
+
+# User local settings
+.include(try=true) "$LOCAL_CONFDIR/rspamd.conf.local"
+.include(try=true,priority=10) "$LOCAL_CONFDIR/rspamd.conf.local.override"
+.include(try=true,priority=10) "$LOCAL_CONFDIR/rspamd.conf.override"
+
+modules {
+ path = "${PLUGINSDIR}";
+ fallback_path = "${SHAREDIR}/lua"; # Legacy path
+ try_path = "${LOCAL_CONFDIR}/plugins.d/"; # User plugins
+}
diff --git a/conf/composites.conf b/conf/composites.conf
new file mode 100644
index 0000000..e38d64e
--- /dev/null
+++ b/conf/composites.conf
@@ -0,0 +1,193 @@
+# Composites setup
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/composites.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/composites.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html and
+# https://rspamd.com/doc/configuration/composites.html for details
+
+composites {
+
+ SHORT_PART_BAD_HEADERS {
+ expression = "MISSING_ESSENTIAL_HEADERS & SINGLE_SHORT_PART";
+ group = "blankspam";
+ policy = "leave";
+ score = 7.0;
+ }
+ FORGED_RECIPIENTS_MAILLIST {
+ expression = "FORGED_RECIPIENTS & -MAILLIST";
+ }
+ FORGED_SENDER_MAILLIST {
+ expression = "FORGED_SENDER & -MAILLIST";
+ }
+ FORGED_SENDER_FORWARDING {
+ expression = "FORGED_SENDER & g:forwarding";
+ description = "Forged sender, but message is forwarded";
+ policy = "remove_weight";
+ }
+ SPF_FAIL_FORWARDING {
+ expression = "g:forwarding & (R_SPF_SOFTFAIL | R_SPF_FAIL)";
+ policy = "remove_weight";
+ }
+ DMARC_POLICY_ALLOW_WITH_FAILURES {
+ expression = "DMARC_POLICY_ALLOW & (R_SPF_SOFTFAIL | R_SPF_FAIL | R_DKIM_REJECT)";
+ policy = "remove_weight";
+ }
+ FORGED_RECIPIENTS_FORWARDING {
+ expression = "FORGED_RECIPIENTS & g:forwarding";
+ policy = "remove_weight";
+ }
+ FORGED_SENDER_VERP_SRS {
+ expression = "FORGED_SENDER & (ENVFROM_PRVS | ENVFROM_VERP)";
+ }
+ FORGED_MUA_MAILLIST {
+ expression = "g:mua & -MAILLIST";
+ }
+ AUTH_NA {
+ expression = "R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA";
+ score = 1.0;
+ policy = "remove_weight";
+ description = "Authenticating message via SPF/DKIM/DMARC/ARC not available";
+ }
+ AUTH_NA_OR_FAIL {
+ expression = "!(R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA) & (R_DKIM_NA | R_DKIM_TEMPFAIL | R_DKIM_PERMFAIL) & (R_SPF_NA | R_SPF_DNSFAIL) & DMARC_NA & (ARC_NA | ARC_DNSFAIL)";
+ score = 1.0;
+ policy = "remove_weight";
+ description = "No authenticating method SPF/DKIM/DMARC/ARC was successful";
+ }
+ BOUNCE_NO_AUTH {
+ expression = "(AUTH_NA | AUTH_NA_OR_FAIL) & (BOUNCE | SUBJ_BOUNCE_WORDS)";
+ score = 1.0;
+ }
+ DKIM_MIXED {
+ expression = "-R_DKIM_ALLOW & (R_DKIM_TEMPFAIL | R_DKIM_PERMFAIL | R_DKIM_REJECT)"
+ policy = "remove_weight";
+ }
+ APPLE_MAILER_COMMON {
+ description = "Message was sent by 'Apple Mail' and has common symbols in place";
+ expression = "APPLE_MAILER & MV_CASE";
+ }
+ APPLE_IOS_MAILER_COMMON {
+ description = "Message was sent by 'Apple iOS Mail' and has common symbols in place";
+ expression = "APPLE_IOS_MAILER & (MV_CASE | MIME_MA_MISSING_TEXT)";
+ }
+ HACKED_WP_PHISHING {
+ expression = "(HAS_X_POS | HAS_PHPMAILER_SIG) & HAS_WP_URI & (PHISHING | CRACKED_SURBL | PH_SURBL_MULTI | DBL_PHISH | DBL_ABUSE_PHISH | URIBL_BLACK | PHISHED_OPENPHISH | PHISHED_PHISHTANK)";
+ description = "Phish message sent by hacked Wordpress instance";
+ policy = "leave";
+ }
+ COMPROMISED_ACCT_BULK {
+ expression = "(HAS_XOIP | RCVD_FROM_SMTP_AUTH) & DCC_BULK";
+ description = "Likely to be from a compromised account";
+ score = 3.0;
+ policy = "leave";
+ }
+ UNDISC_RCPTS_BULK {
+ expression = "DCC_BULK & (MISSING_TO | R_UNDISC_RCPT)";
+ description = "Missing or undisclosed recipients with a bulk signature";
+ score = 3.0;
+ policy = "leave";
+ }
+ RCVD_UNAUTH_PBL {
+ expression = "RECEIVED_SPAMHAUS_PBL & !RCVD_VIA_SMTP_AUTH";
+ description = "Relayed through Spamhaus PBL IP without sufficient authentication (possibly indicating an open relay)";
+ score = 2.0;
+ policy = "leave";
+ }
+ RCVD_DKIM_ARC_DNSWL_MED {
+ expression = "(R_DKIM_ALLOW | ARC_ALLOW) & RCVD_IN_DNSWL_MED";
+ description = "Sufficiently DKIM/ARC signed and received from IP with medium trust at DNSWL";
+ score = -0.5;
+ policy = "leave";
+ }
+ RCVD_DKIM_ARC_DNSWL_HI {
+ expression = "(R_DKIM_ALLOW | ARC_ALLOW) & RCVD_IN_DNSWL_HI";
+ description = "Sufficiently DKIM/ARC signed and received from IP with high trust at DNSWL";
+ score = -1.0;
+ policy = "leave";
+ }
+ AUTOGEN_PHP_SPAMMY {
+ expression = "(HAS_X_POS | HAS_PHPMAILER_SIG | HAS_X_PHP_SCRIPT) & (SUBJECT_ENDS_QUESTION | SUBJECT_ENDS_EXCLAIM | MANY_INVISIBLE_PARTS)";
+ description = "Message was generated by PHP script and contains some spam indicators";
+ score = 1.0;
+ policy = "leave";
+ }
+ PHISH_EMOTION {
+ expression = "(PHISHING | DBL_PHISH | PHISHED_OPENPHISH | PHISHED_PHISHTANK) & (SUBJECT_ENDS_QUESTION | SUBJECT_ENDS_EXCLAIM)";
+ description = "Phish message with subject trying to address users emotion";
+ score = 1.0;
+ policy = "leave";
+ }
+ HAS_ANON_DOMAIN {
+ expression = "HAS_GUC_PROXY_URI | URIBL_RED | DBL_ABUSE_REDIR | HAS_ONION_URI";
+ description = "Contains one or more domains trying to disguise owner/destination";
+ score = 0.1;
+ policy = "leave";
+ }
+ BAD_REP_POLICIES {
+ description = "Contains valid policies but are also marked by fuzzy/bayes/SURBL/RBL";
+ expression = "(~g-:policies) & (-g+:fuzzy | -g+:statistics | -g+:surbl | -g+:rbl)";
+ score = 0.1;
+ }
+ VIOLATED_DIRECT_SPF {
+ description = "Has no Received (or no trusted received relays) and SPF policy fails or soft fails";
+ expression = "(R_SPF_FAIL | R_SPF_SOFTFAIL) & (RCVD_COUNT_ZERO | RCVD_NO_TLS_LAST)";
+ policy = "leave";
+ score = 3.5;
+ }
+ IP_SCORE_FREEMAIL {
+ description = "Negate IP_SCORE when message comes from FreeMail";
+ expression = "FREEMAIL_FROM & SENDER_REP_SPAM";
+ score = 0.0;
+ policy = "remove_weight";
+ }
+ BROKEN_HEADERS_MAILLIST {
+ description = "Negate BROKEN_HEADERS when message comes via some mailing list";
+ expression = "BROKEN_HEADERS & -MAILLIST";
+ score = 0.0;
+ policy = "remove_weight";
+ }
+ LEAKED_PASSWORD_SCAM {
+ description = "Contains BTC wallet address and scam patterns";
+ expression = "BITCOIN_ADDR & (LEAKED_PASSWORD_SCAM_RE | R_MIXED_CHARSET | R_EMPTY_IMAGE)";
+ policy = "leave";
+ score = 7.0;
+ group = "scams";
+ }
+ FREEMAIL_AFF {
+ expression = "(FREEMAIL_FROM | FREEMAIL_ENVFROM | FREEMAIL_REPLYTO) & (TO_DN_RECIPIENTS | R_UNDISC_RCPT) & (INTRODUCTION | FROM_NAME_HAS_TITLE | FREEMAIL_REPLYTO_NEQ_FROM_DOM | SUBJECT_HAS_CURRENCY)";
+ score = 4.0;
+ policy = "leave";
+ description = "Message exhibits strong characteristics of advance fee fraud (AFF a/k/a '419' spam) involving freemail addresses";
+ }
+ REDIRECTOR_URL_ONLY {
+ expression = "HFILTER_URL_ONLY & REDIRECTOR_URL";
+ score = 1.0;
+ policy = "leave";
+ description = "Message only contains a redirector URL";
+ }
+ THREAD_HIJACKING_FROM_INJECTOR {
+ expression = "FAKE_REPLY & RCVD_VIA_SMTP_AUTH & (!RECEIVED_SPAMHAUS_PBL | RECEIVED_SPAMHAUS_XBL | RECEIVED_SPAMHAUS_SBL)";
+ score = 2.0;
+ policy = "leave";
+ description = "Fake reply exhibiting characteristics of being injected into a compromised mail server, possibly e-mail thread hijacking";
+ group = "compromised_hosts";
+ }
+ SUSPICIOUS_URL_IN_SUSPICIOUS_MESSAGE {
+ expression = "(REDIRECTOR_URL | HAS_ANON_DOMAIN | HAS_IPFS_GATEWAY_URL) & (-g+:fuzzy | -g+:statistics | -g+:surbl | -g+:rbl)";
+ score = 1.0;
+ policy = "leave";
+ description = "Message contains redirector, anonymous or IPFS gateway URL and is marked by fuzzy/bayes/SURBL/RBL";
+ }
+
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/composites.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/composites.conf"
+}
diff --git a/conf/groups.conf b/conf/groups.conf
new file mode 100644
index 0000000..2aeb4ed
--- /dev/null
+++ b/conf/groups.conf
@@ -0,0 +1,126 @@
+# Symbols groups setup
+# Each individual group lives in scores.d/<GROUPNAME>
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/groups.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/groups.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# See also: https://rspamd.com/doc/faq.html#how-to-change-score-for-some-symbol
+
+group "headers" {
+ .include "$CONFDIR/scores.d/headers_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/headers_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/headers_group.conf"
+}
+
+group "subject" {
+ .include "$CONFDIR/scores.d/subject_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/subject_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/subject_group.conf"
+}
+
+group "mua" {
+ .include "$CONFDIR/scores.d/mua_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/mua_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/mua_group.conf"
+}
+
+group "rbl" {
+ .include "$CONFDIR/scores.d/rbl_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/rbl_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/rbl_group.conf"
+}
+
+group "statistics" {
+ .include "$CONFDIR/scores.d/statistics_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/statistics_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/statistics_group.conf"
+}
+
+group "fuzzy" {
+ .include "$CONFDIR/scores.d/fuzzy_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/fuzzy_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/fuzzy_group.conf"
+}
+
+# SPF + DKIM + DMARC + ARC
+group "policies" {
+ .include "$CONFDIR/scores.d/policies_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/policies_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/policies_group.conf"
+}
+
+group "whitelist" {
+ .include "$CONFDIR/scores.d/whitelist_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/whitelist_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/whitelist_group.conf"
+}
+
+group "surbl" {
+ .include "$CONFDIR/scores.d/surbl_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/surbl_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/surbl_group.conf"
+}
+
+group "phishing" {
+ .include "$CONFDIR/scores.d/phishing_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/phishing_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/phishing_group.conf"
+}
+
+group "hfilter" {
+ .include "$CONFDIR/scores.d/hfilter_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/hfilter_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/hfilter_group.conf"
+}
+
+group "mime_types" {
+ .include "$CONFDIR/scores.d/mime_types_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/mime_types_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/mime_types_group.conf"
+}
+
+# Used to limit maximum score
+group "excessqp" {
+ max_score = 2.4;
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/excessqp_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/excessqp_group.conf"
+}
+group "excessb64" {
+ max_score = 3.0;
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/excessb64_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/excessb64_group.conf"
+}
+
+# Not defined by default
+group "neural" {
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/neural_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/neural_group.conf"
+}
+
+group "antivirus" {
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/antivirus_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/antivirus_group.conf"
+}
+
+group "external_services" {
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/external_services_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/external_services_group.conf"
+}
+
+group "content" {
+ .include "$CONFDIR/scores.d/content_group.conf"
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/content_group.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/content_group.conf"
+}
+
+.include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/groups.conf"
+.include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/groups.conf"
diff --git a/conf/lang_detection.inc b/conf/lang_detection.inc
new file mode 100644
index 0000000..16bfa3f
--- /dev/null
+++ b/conf/lang_detection.inc
@@ -0,0 +1,28 @@
+# Language detection configuration
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/lang_detection.inc' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/lang_detection.inc' to strictly override all
+# parameters defined inside this section
+
+# Path to the languages shared data
+# languages = "${SHAREDIR}/languages"
+
+# Limit in words to treat text as short for language detection
+# short_text_limit = 10
+
+# Enable only specific languages
+# languages_enable = ["en", "de"]
+
+# Disable specific languages
+# languages_disable = ["fr", "es"]
+
+# Use the following fasttext model for language detection (if Fasttext support is compiled in)
+# fasttext_model = "${SHAREDIR}/languages/fasttext_model.ftz"
+
+# Prefer fasttext over all other methods
+# prefer_fasttext = true;
+
diff --git a/conf/logging.inc b/conf/logging.inc
new file mode 100644
index 0000000..00dba39
--- /dev/null
+++ b/conf/logging.inc
@@ -0,0 +1,35 @@
+# Logging configuration
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/logging.inc' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/logging.inc' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Relevant documentation: https://rspamd.com/doc/configuration/logging.html
+
+level = "info";
+log_format =<<EOD
+id: <$mid>,$if_qid{ qid: <$>,}$if_ip{ ip: $,}$if_user{ user: $,}$if_smtp_from{ from: <$>,}
+(default: $is_spam ($action): [$scores] [$symbols_scores_params]),
+len: $len, time: $time_real, dns req: $dns_req,
+digest: <$digest>$if_smtp_rcpts{, rcpts: <$>}$if_mime_rcpts{, mime_rcpts: <$>}$if_filename{, file: $}$if_forced_action{, forced: $}$if_settings_id{, settings_id: $}
+EOD
+
+
+# Show statistics for regular expressions
+log_re_cache = true;
+
+# Can be used for console logging
+color = false;
+
+# Log with microseconds resolution
+log_usec = false;
+
+# Enable debug for specific modules (e.g. `debug_modules = ["dkim", "re_cache"];`)
+debug_modules = []
diff --git a/conf/maps.d/dmarc_whitelist.inc b/conf/maps.d/dmarc_whitelist.inc
new file mode 100644
index 0000000..ca8c214
--- /dev/null
+++ b/conf/maps.d/dmarc_whitelist.inc
@@ -0,0 +1,70 @@
+# Domains with valid DMARC and DKIM
+# These domains come from some top visited sites that have restrictive dmarc policy
+
+4chan.org
+adp.com
+advice.hmrc.gov.uk
+airbnb.com
+airtel.in
+alibaba.com
+aliexpress.com
+alipay.com
+americanexpress.com
+avg.com
+avito.ru
+badoo.com
+battle.net
+beeline.ru
+booking.com
+box.com
+chase.com
+consultant.ru
+dhl.com
+emarsys.com
+esetnod32.ru
+evernote.com
+facebook.com
+facebookmail.com
+fedex.com
+force.com
+garant.ru
+gosuslugi.ru
+hh.ru
+hmrc.gov.uk
+instagram.com
+linkedin.com
+livejournal.com
+lufthansa-group.com
+mackeeper.com
+megafon.ru
+mercadolibre.com.ar
+mercadolivre.com.br
+messenger.com
+mvideo.ru
+neobux.com
+netflix.com
+newegg.com
+nic.ru
+paypal.com
+pch.com
+pinterest.com
+rostelecom.ru
+rt.ru
+sberbank.ru
+sportmaster.ru
+squarespace.com
+subscribe.ru
+superjob.ru
+twitter.com
+uber.com
+ulmart.ru
+ups.com
+usps.com
+utair.ru
+verizonwireless.com
+vk.com
+vkrugudruzei.ru
+westernunion.com
+whatsapp.com
+zendesk.com
+zomato.com
diff --git a/conf/maps.d/maillist.inc b/conf/maps.d/maillist.inc
new file mode 100644
index 0000000..b2de86d
--- /dev/null
+++ b/conf/maps.d/maillist.inc
@@ -0,0 +1,176 @@
+1c-bitrix.ru
+360.cn
+360.com
+activeby.net
+adobe.com
+aeroflot.ru
+alibaba.com
+aliexpress.com
+alipay.com
+amazon.co.jp
+amazon.com
+amazon.co.uk
+amazon.de
+amazon.in
+apple.com
+ask.com
+avito.ru
+b2b-center.ru
+baby.ru
+babysfera.ru
+baidu.com
+beeline.ru
+bing.com
+blogger.com
+blogspot.com
+booking.com
+chase.com
+cnn.com
+comodo.com
+comodogroup.com
+comodo.net
+dating.ru
+dmir.ru
+dropbox.com
+ebay.com
+electrozon.ru
+e-moskva.ru
+etp-micex.ru
+exist.ru
+fabrikant.ru
+facebook.com
+fc2.com
+flipkart.com
+free-lance.ru
+github.com
+github.net
+gmw.cn
+go.com
+google.ca
+google.co.id
+google.co.in
+google.co.jp
+google.com.au
+google.com.br
+google.com.hk
+google.com.mx
+google.com.tr
+google.co.uk
+google.de
+google.fr
+google.it
+google.pl
+google.ru
+googleusercontent.com
+gosuslugi.ru
+gov.ru
+habramail.net
+hao123.com
+hh.ru
+imdb.com
+imgur.com
+instagram.com
+ispsystem.com
+ispsystem.net
+ispsystem.ru
+jobinmoscow.ru
+job.ru
+linkedin.com
+live.com
+livejournal.com
+mailgun.com
+mailgun.net
+mail.mtml.ru
+mamba.ru
+megafon.ru
+microsoft.com
+microsoftonline.com
+mnogo.ru
+mobilelement.ru
+moesk.ru
+molotok.ru
+mos.ru
+msn.com
+mts.ru
+nalog.ru
+naukanet.ru
+netflix.com
+nic.ru
+nix.ru
+nytimes.com
+odesk.com
+odnoklassniki.ru
+office.com
+ok.ru
+osmp.ru
+outbrain.com
+ozon.ru
+paypal.com
+pinterest.com
+pixnet.net
+pornhub.com
+professionali.ru
+psport.ru
+qiwi.com
+qiwi.ru
+qq.com
+quickpay.ru
+r01.ru
+rabota.ru
+rbkmoney.ru
+reddit.com
+reg.ru
+regtime.net
+returnpath.net
+roboxchange.com
+roseltorg.ru
+rosreestr.ru
+rostelecom.ru
+rp-system.ru
+rts-tender.ru
+rtstender.ru
+russianpost.ru
+sberbank-ast.ru
+sberbank.ru
+senderscore.net
+shopotam.ru
+sipnet.ru
+sitesoft.ru
+skype.com
+smartsndr.com
+sohu.com
+soso.com
+stackoverflow.com
+startcomca.com
+subscribe.ru
+superjob.ru
+taobao.com
+tario.ru
+t.co
+tks.ru
+tmall.com
+topface.com
+tumblr.com
+tutu.ru
+twitter.com
+ulmart.ru
+usndr.com
+vedomosti.ru
+vk.com
+vkrugudruzei.ru
+webnames.ru
+weibo.com
+whatsapp.com
+wikipedia.org
+wmtransfer.com
+wordpress.com
+wosign.com
+xhamster.com
+xvideos.com
+yahoo.co.jp
+yahoo.com
+yamoney.ru
+youtube.com
+zakazrf.ru
+ztel.ru
+zzap.ru
diff --git a/conf/maps.d/mid.inc b/conf/maps.d/mid.inc
new file mode 100644
index 0000000..aeb5454
--- /dev/null
+++ b/conf/maps.d/mid.inc
@@ -0,0 +1,22 @@
+# Known invalid or missed Message-IDs
+# 'domain' 'Message-ID regexp'
+
+alibaba.com /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}-0$/
+aliexpress.com /^(?:(?:[0-9]{14,15}|[a-z]{4}UTT_[0-9]{5,6}_(?:[0-9]{8}_)?)[$])?[a-f0-9]{32}$/
+beeline.ru /^<[A-z0-9+]{18}>$/
+noreply.esphere.ru
+noreply.etprf.ru
+rkn.gov.ru
+garant.ru
+is-zakupki.com
+mirtesen.ru
+fcod.nalog.ru
+otc.ru
+qiwi.ru
+client.rostelecom.ru
+sberbank-ast.ru
+crm.taxcom.ru
+wildberries.ru /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/
+promo.wildberries.ru /^[A-F0-9]{8}(?:-[A-F0-9]{4}){3}-[A-F0-9]{12}$/
+taxi.yandex.ru /^[a-f0-9]{32}$/
+online.zcts.ru
diff --git a/conf/maps.d/mime_types.inc b/conf/maps.d/mime_types.inc
new file mode 100644
index 0000000..7dd5d89
--- /dev/null
+++ b/conf/maps.d/mime_types.inc
@@ -0,0 +1,1538 @@
+# perl -ne 'if(/([^\d.\s\[o][^\d.\[\]\s\/A-Z]{3,}\/[^\s\/]+)/) { print "$1 0\n"; }'
+application/1d-interleaved-parityfec 0
+application/3gpdash-qoe-report+xml 0
+application/3gpp-ims+xml 0
+application/A2L 0
+application/AML 0
+application/ATF 0
+application/ATFX 0
+application/ATXML 0
+application/CALS-1840 0
+application/CDFX+XML 0
+application/CEA 0
+application/CSTAdata+xml 0
+application/DCD 0
+application/DII 0
+application/DIT 0
+application/EDI-X12 0
+application/EDI-consent 0
+application/EDIFACT 0
+application/EmergencyCallData.Comment+xml 0
+application/EmergencyCallData.DeviceInfo+xml 0
+application/EmergencyCallData.ProviderInfo+xml 0
+application/EmergencyCallData.ServiceInfo+xml 0
+application/EmergencyCallData.SubscriberInfo+xml 0
+application/H224 0
+application/IOTP 0
+application/ISUP 0
+application/LXF 0
+application/MF4 0
+application/ODA 0
+application/ODX 0
+application/PDX 0
+application/QSIG 0
+application/SGML 0
+application/activemessage 0
+application/alto-costmap+json 0
+application/alto-costmapfilter+json 0
+application/alto-directory+json 0
+application/alto-endpointcost+json 0
+application/alto-endpointcostparams+json 0
+application/alto-endpointprop+json 0
+application/alto-endpointpropparams+json 0
+application/alto-error+json 0
+application/alto-networkmap+json 0
+application/alto-networkmapfilter+json 0
+application/andrew-inset 0
+application/applefile 0
+application/atom+xml 0
+application/atomcat+xml 0
+application/atomdeleted+xml 0
+application/atomicmail 0
+application/atomsvc+xml 0
+application/auth-policy+xml 0
+application/bacnet-xdd+zip 0
+application/batch-SMTP 0
+application/beep+xml 0
+application/calendar+json 0
+application/calendar+xml 0
+application/call-completion 0
+application/cbor 0
+application/ccmp+xml 0
+application/ccxml+xml 0
+application/cdmi-capability 0
+application/cdmi-container 0
+application/cdmi-domain 0
+application/cdmi-object 0
+application/cdmi-queue 0
+application/cdni 0
+application/cea-2018+xml 0
+application/cellml+xml 0
+application/cfw 0
+application/cms 0
+application/cnrp+xml 0
+application/coap-group+json 0
+application/commonground 0
+application/conference-info+xml 0
+application/cpl+xml 0
+application/csrattrs 0
+application/csta+xml 0
+application/csvm+json 0
+application/cybercash 0
+application/dash+xml 0
+application/dashdelta 0
+application/davmount+xml 0
+application/dca-rft 0
+application/dec-dx 0
+application/dialog-info+xml 0
+application/dicom 0
+application/dns 0
+application/dskpp+xml 0
+application/dssc+der 0
+application/dssc+xml 0
+application/dvcs 0
+application/ecmascript 0
+application/emotionml+xml 0
+application/encaprtp 0
+application/epp+xml 0
+application/epub+zip 0
+application/eshop 0
+application/example 0
+application/fastinfoset 0
+application/fastsoap 0
+application/fdt+xml 0
+application/fits 0
+application/font-sfnt 0
+application/font-tdpfr 0
+application/font-woff 0
+application/framework-attributes+xml 0
+application/gzip 0
+application/held+xml 0
+application/http 0
+application/hyperstudio 0
+application/ibe-key-request+xml 0
+application/ibe-pkg-reply+xml 0
+application/ibe-pp-data 0
+application/ics 0
+application/iges 0
+application/im-iscomposing+xml 0
+application/index 0
+application/index-obj 0
+application/index.cmd 0
+application/index.response 0
+application/index.vnd 0
+application/inkml+xml 0
+application/ipfix 0
+application/ipp 0
+application/its+xml 0
+application/javascript 0
+application/jose 0
+application/jose+json 0
+application/jrd+json 0
+application/json 0
+application/json-patch+json 0
+application/json-seq 0
+application/jwk+json 0
+application/jwk-set+json 0
+application/jwt 0
+application/kpml-request+xml 0
+application/kpml-response+xml 0
+application/ld+json 0
+application/link-format 0
+application/load-control+xml 0
+application/lost+xml 0
+application/lostsync+xml 0
+application/mac-binhex40 0
+application/macwriteii 0
+application/mads+xml 0
+application/marc 0
+application/marcxml+xml 0
+application/mathematica 0
+application/mbms-associated-procedure-description+xml 0
+application/mbms-deregister+xml 0
+application/mbms-envelope+xml 0
+application/mbms-msk+xml 0
+application/mbms-msk-response+xml 0
+application/mbms-protection-description+xml 0
+application/mbms-reception-report+xml 0
+application/mbms-register+xml 0
+application/mbms-register-response+xml 0
+application/mbms-schedule+xml 0
+application/mbms-user-service-description+xml 0
+application/mbox 0
+application/media-policy-dataset+xml 0
+application/media_control+xml 0
+application/mediaservercontrol+xml 0
+application/merge-patch+json 0
+application/metalink4+xml 0
+application/mets+xml 0
+application/mikey 0
+application/mods+xml 0
+application/moss-keys 0
+application/moss-signature 0
+application/mosskey-data 0
+application/mosskey-request 0
+application/mp21 0
+application/mp4 0
+application/mpeg4-generic 0
+application/mpeg4-iod 0
+application/mpeg4-iod-xmt 0
+application/mrb-consumer+xml 0
+application/mrb-publish+xml 0
+application/msc-ivr+xml 0
+application/msc-mixer+xml 0
+application/msword 0
+application/mxf 0
+application/nasdata 0
+application/news-checkgroups 0
+application/news-groupinfo 0
+application/news-transmission 0
+application/nlsml+xml 0
+application/nss 0
+application/ocsp-request 0
+application/ocsp-response 0
+application/octet-stream 0
+application/oebps-package+xml 0
+application/ogg 0
+application/oxps 0
+application/p2p-overlay+xml 0
+application/patch-ops-error+xml 0
+application/pdf 0
+application/pgp-encrypted 0
+application/pgp-keys 0
+application/pgp-signature 0
+application/pidf+xml 0
+application/pidf-diff+xml 0
+application/pkcs10 0
+application/pkcs12 0
+application/pkcs7-mime -1
+application/pkcs7-signature -1
+application/pkcs8 0
+application/pkix-attr-cert 0
+application/pkix-cert 0
+application/pkix-crl 0
+application/pkix-pkipath 0
+application/pkixcmp 0
+application/pls+xml 0
+application/poc-settings+xml 0
+application/postscript 0
+application/ppsp-tracker+json 0
+application/provenance+xml 0
+application/prs.alvestrand.titrax-sheet 0
+application/prs.cww 0
+application/prs.hpub+zip 0
+application/prs.nprend 0
+application/prs.plucker 0
+application/prs.rdf-xml-crypt 0
+application/prs.xsf+xml 0
+application/pskc+xml 0
+application/raptorfec 0
+application/rdap+json 0
+application/rdf+xml 0
+application/reginfo+xml 0
+application/relax-ng-compact-syntax 0
+application/remote-printing 0
+application/reputon+json 0
+application/resource-lists+xml 0
+application/resource-lists-diff+xml 0
+application/rfc+xml 0
+application/riscos 0
+application/rlmi+xml 0
+application/rls-services+xml 0
+application/rpki-ghostbusters 0
+application/rpki-manifest 0
+application/rpki-roa 0
+application/rpki-updown 0
+application/rtf 0
+application/rtploopback 0
+application/rtx 0
+application/samlassertion+xml 0
+application/samlmetadata+xml 0
+application/sbml+xml 0
+application/scaip+xml 0
+application/scim+json 0
+application/scvp-cv-request 0
+application/scvp-cv-response 0
+application/scvp-vp-request 0
+application/scvp-vp-response 0
+application/sdp 0
+application/sep+xml 0
+application/sep-exi 0
+application/session-info 0
+application/set-payment 0
+application/set-payment-initiation 0
+application/set-registration 0
+application/set-registration-initiation 0
+application/sgml-open-catalog 0
+application/shf+xml 0
+application/sieve 0
+application/simple-filter+xml 0
+application/simple-message-summary 0
+application/simpleSymbolContainer 0
+application/slate 0
+application/smil+xml 0
+application/smpte336m 0
+application/soap+fastinfoset 0
+application/soap+xml 0
+application/spirits-event+xml 0
+application/sql 0
+application/srgs 0
+application/srgs+xml 0
+application/sru+xml 0
+application/ssml+xml 0
+application/tamp-apex-update 0
+application/tamp-apex-update-confirm 0
+application/tamp-community-update 0
+application/tamp-community-update-confirm 0
+application/tamp-error 0
+application/tamp-sequence-adjust 0
+application/tamp-sequence-adjust-confirm 0
+application/tamp-status-query 0
+application/tamp-status-response 0
+application/tamp-update 0
+application/tamp-update-confirm 0
+application/tei+xml 0
+application/thraud+xml 0
+application/timestamp-query 0
+application/timestamp-reply 0
+application/timestamped-data 0
+application/ttml+xml 0
+application/tve-trigger 0
+application/ulpfec 0
+application/urc-grpsheet+xml 0
+application/urc-ressheet+xml 0
+application/urc-targetdesc+xml 0
+application/urc-uisocketdesc+xml 0
+application/vcard+json 0
+application/vcard+xml 0
+application/vemmi 0
+application/vnd-acucobol 0
+application/vnd-curl 0
+application/vnd-dart 0
+application/vnd-dxr 0
+application/vnd-fdf 0
+application/vnd-mif 0
+application/vnd-sema 0
+application/vnd-wap-wmlc 0
+application/vnd.3M.Post-it-Notes 0
+application/vnd.3gpp-prose+xml 0
+application/vnd.3gpp-prose-pc3ch+xml 0
+application/vnd.3gpp.SRVCC-info+xml 0
+application/vnd.3gpp.access-transfer-events+xml 0
+application/vnd.3gpp.bsf+xml 0
+application/vnd.3gpp.mid-call+xml 0
+application/vnd.3gpp.pic-bw-large 0
+application/vnd.3gpp.pic-bw-small 0
+application/vnd.3gpp.pic-bw-var 0
+application/vnd.3gpp.sms 0
+application/vnd.3gpp.srvcc-ext+xml 0
+application/vnd.3gpp.state-and-event-info+xml 0
+application/vnd.3gpp.ussd+xml 0
+application/vnd.3gpp2.bcmcsinfo+xml 0
+application/vnd.3gpp2.sms 0
+application/vnd.3gpp2.tcap 0
+application/vnd.FloGraphIt 0
+application/vnd.HandHeld-Entertainment+xml 0
+application/vnd.Kinar 0
+application/vnd.MFER 0
+application/vnd.Mobius.DAF 0
+application/vnd.Mobius.DIS 0
+application/vnd.Mobius.MBK 0
+application/vnd.Mobius.MQY 0
+application/vnd.Mobius.MSL 0
+application/vnd.Mobius.PLC 0
+application/vnd.Mobius.TXF 0
+application/vnd.Quark.QuarkXPress 0
+application/vnd.SimTech-MindMapper 0
+application/vnd.accpac.simply.aso 0
+application/vnd.accpac.simply.imp 0
+application/vnd.acucorp 0
+application/vnd.adobe.flash-movie 0
+application/vnd.adobe.formscentral.fcdt 0
+application/vnd.adobe.fxp 0
+application/vnd.adobe.partial-upload 0
+application/vnd.adobe.xdp+xml 0
+application/vnd.adobe.xfdf 0
+application/vnd.aether.imp 0
+application/vnd.ah-barcode 0
+application/vnd.ahead.space 0
+application/vnd.airzip.filesecure.azf 0
+application/vnd.airzip.filesecure.azs 0
+application/vnd.americandynamics.acc 0
+application/vnd.amiga.ami 0
+application/vnd.amundsen.maze+xml 0
+application/vnd.anki 0
+application/vnd.anser-web-certificate-issue-initiation 0
+application/vnd.antix.game-component 0
+application/vnd.apache.thrift.binary 0
+application/vnd.apache.thrift.compact 0
+application/vnd.apache.thrift.json 0
+application/vnd.api+json 0
+application/vnd.apple.installer+xml 0
+application/vnd.apple.mpegurl 0
+application/vnd.arastra.swi 0
+application/vnd.aristanetworks.swi 0
+application/vnd.artsquare 0
+application/vnd.astraea-software.iota 0
+application/vnd.audiograph 0
+application/vnd.autopackage 0
+application/vnd.avistar+xml 0
+application/vnd.balsamiq.bmml+xml 0
+application/vnd.balsamiq.bmpr 0
+application/vnd.bekitzur-stech+json 0
+application/vnd.biopax.rdf+xml 0
+application/vnd.blueice.multipass 0
+application/vnd.bluetooth.ep.oob 0
+application/vnd.bluetooth.le.oob 0
+application/vnd.bmi 0
+application/vnd.businessobjects 0
+application/vnd.cab-jscript 0
+application/vnd.canon-cpdl 0
+application/vnd.canon-lips 0
+application/vnd.cendio.thinlinc.clientconf 0
+application/vnd.century-systems.tcp_stream 0
+application/vnd.chemdraw+xml 0
+application/vnd.chipnuts.karaoke-mmd 0
+application/vnd.cinderella 0
+application/vnd.cirpack.isdn-ext 0
+application/vnd.citationstyles.style+xml 0
+application/vnd.claymore 0
+application/vnd.cloanto.rp9 0
+application/vnd.clonk.c4group 0
+application/vnd.cluetrust.cartomobile-config 0
+application/vnd.cluetrust.cartomobile-config-pkg 0
+application/vnd.coffeescript 0
+application/vnd.collection+json 0
+application/vnd.collection.doc+json 0
+application/vnd.collection.next+json 0
+application/vnd.commerce-battelle 0
+application/vnd.commonspace 0
+application/vnd.contact.cmsg 0
+application/vnd.cosmocaller 0
+application/vnd.crick.clicker 0
+application/vnd.crick.clicker.keyboard 0
+application/vnd.crick.clicker.palette 0
+application/vnd.crick.clicker.template 0
+application/vnd.crick.clicker.wordbank 0
+application/vnd.criticaltools.wbs+xml 0
+application/vnd.ctc-posml 0
+application/vnd.ctct.ws+xml 0
+application/vnd.cups-pdf 0
+application/vnd.cups-postscript 0
+application/vnd.cups-ppd 0
+application/vnd.cups-raster 0
+application/vnd.cups-raw 0
+application/vnd.cyan.dean.root+xml 0
+application/vnd.cybank 0
+application/vnd.data-vision.rdz 0
+application/vnd.debian.binary-package 0
+application/vnd.dece-zip 0
+application/vnd.dece.data 0
+application/vnd.dece.ttml+xml 0
+application/vnd.dece.unspecified 0
+application/vnd.denovo.fcselayout-link 0
+application/vnd.desmume-movie 0
+application/vnd.dir-bi.plate-dl-nosuffix 0
+application/vnd.dm.delegation+xml 0
+application/vnd.dna 0
+application/vnd.document+json 0
+application/vnd.dolby.mobile.1 0
+application/vnd.dolby.mobile.2 0
+application/vnd.doremir.scorecloud-binary-document 0
+application/vnd.dpgraph 0
+application/vnd.dreamfactory 0
+application/vnd.drive+json 0
+application/vnd.dtg.local 0
+application/vnd.dtg.local-html 0
+application/vnd.dtg.local.flash 0
+application/vnd.dvb.ait 0
+application/vnd.dvb.dvbj 0
+application/vnd.dvb.esgcontainer 0
+application/vnd.dvb.ipdcdftnotifaccess 0
+application/vnd.dvb.ipdcesgaccess 0
+application/vnd.dvb.ipdcesgaccess2 0
+application/vnd.dvb.ipdcesgpdd 0
+application/vnd.dvb.ipdcroaming 0
+application/vnd.dvb.iptv.alfec-base 0
+application/vnd.dvb.iptv.alfec-enhancement 0
+application/vnd.dvb.notif-aggregate-root+xml 0
+application/vnd.dvb.notif-container+xml 0
+application/vnd.dvb.notif-generic+xml 0
+application/vnd.dvb.notif-ia-msglist+xml 0
+application/vnd.dvb.notif-ia-registration-request+xml 0
+application/vnd.dvb.notif-ia-registration-response+xml 0
+application/vnd.dvb.notif-init+xml 0
+application/vnd.dvb.pfr 0
+application/vnd.dvb_service 0
+application/vnd.dynageo 0
+application/vnd.dzr 0
+application/vnd.easykaraoke.cdgdownload 0
+application/vnd.ecdis-update 0
+application/vnd.ecowin.chart 0
+application/vnd.ecowin.filerequest 0
+application/vnd.ecowin.fileupdate 0
+application/vnd.ecowin.series 0
+application/vnd.ecowin.seriesrequest 0
+application/vnd.ecowin.seriesupdate 0
+application/vnd.emclient.accessrequest+xml 0
+application/vnd.enliven 0
+application/vnd.enphase.envoy 0
+application/vnd.eprints.data+xml 0
+application/vnd.epson.esf 0
+application/vnd.epson.msf 0
+application/vnd.epson.quickanime 0
+application/vnd.epson.salt 0
+application/vnd.epson.ssf 0
+application/vnd.ericsson.quickcall 0
+application/vnd.eszigno3+xml 0
+application/vnd.etsi.aoc+xml 0
+application/vnd.etsi.asic-e+zip 0
+application/vnd.etsi.asic-s+zip 0
+application/vnd.etsi.cug+xml 0
+application/vnd.etsi.iptvcommand+xml 0
+application/vnd.etsi.iptvdiscovery+xml 0
+application/vnd.etsi.iptvprofile+xml 0
+application/vnd.etsi.iptvsad-bc+xml 0
+application/vnd.etsi.iptvsad-cod+xml 0
+application/vnd.etsi.iptvsad-npvr+xml 0
+application/vnd.etsi.iptvservice+xml 0
+application/vnd.etsi.iptvsync+xml 0
+application/vnd.etsi.iptvueprofile+xml 0
+application/vnd.etsi.mcid+xml 0
+application/vnd.etsi.mheg5 0
+application/vnd.etsi.overload-control-policy-dataset+xml 0
+application/vnd.etsi.pstn+xml 0
+application/vnd.etsi.sci+xml 0
+application/vnd.etsi.simservs+xml 0
+application/vnd.etsi.timestamp-token 0
+application/vnd.etsi.tsl+xml 0
+application/vnd.etsi.tsl.der 0
+application/vnd.eudora.data 0
+application/vnd.ezpix-album 0
+application/vnd.ezpix-package 0
+application/vnd.f-secure.mobile 0
+application/vnd.fastcopy-disk-image 0
+application/vnd.fdsn.mseed 0
+application/vnd.fdsn.seed 0
+application/vnd.ffsns 0
+application/vnd.filmit.zfc 0
+application/vnd.fints 0
+application/vnd.firemonkeys.cloudcell 0
+application/vnd.fluxtime.clip 0
+application/vnd.font-fontforge-sfd 0
+application/vnd.framemaker 0
+application/vnd.frogans.fnc 0
+application/vnd.frogans.ltf 0
+application/vnd.fsc.weblaunch 0
+application/vnd.fujitsu.oasys 0
+application/vnd.fujitsu.oasys2 0
+application/vnd.fujitsu.oasys3 0
+application/vnd.fujitsu.oasysgp 0
+application/vnd.fujitsu.oasysprs 0
+application/vnd.fujixerox.ART-EX 0
+application/vnd.fujixerox.ART4 0
+application/vnd.fujixerox.HBPL 0
+application/vnd.fujixerox.ddd 0
+application/vnd.fujixerox.docuworks 0
+application/vnd.fujixerox.docuworks.binder 0
+application/vnd.fujixerox.docuworks.container 0
+application/vnd.fut-misnet 0
+application/vnd.fuzzysheet 0
+application/vnd.genomatix.tuxedo 0
+application/vnd.geo+json 0
+application/vnd.geocube+xml 0
+application/vnd.geogebra.file 0
+application/vnd.geogebra.tool 0
+application/vnd.geometry-explorer 0
+application/vnd.geonext 0
+application/vnd.geoplan 0
+application/vnd.geospace 0
+application/vnd.gerber 0
+application/vnd.globalplatform.card-content-mgt 0
+application/vnd.globalplatform.card-content-mgt-response 0
+application/vnd.gmx 0
+application/vnd.google-earth.kml+xml 0
+application/vnd.google-earth.kmz 0
+application/vnd.gov.sk.e-form+xml 0
+application/vnd.gov.sk.e-form+zip 0
+application/vnd.gov.sk.xmldatacontainer+xml 0
+application/vnd.grafeq 0
+application/vnd.gridmp 0
+application/vnd.groove-account 0
+application/vnd.groove-help 0
+application/vnd.groove-identity-message 0
+application/vnd.groove-injector 0
+application/vnd.groove-tool-message 0
+application/vnd.groove-tool-template 0
+application/vnd.groove-vcard 0
+application/vnd.hal+json 0
+application/vnd.hal+xml 0
+application/vnd.hbci 0
+application/vnd.hcl-bireports 0
+application/vnd.hdt 0
+application/vnd.heroku+json 0
+application/vnd.hhe.lesson-player 0
+application/vnd.hp-HPGL 0
+application/vnd.hp-PCL 0
+application/vnd.hp-PCLXL 0
+application/vnd.hp-hpid 0
+application/vnd.hp-hps 0
+application/vnd.hp-jlyt 0
+application/vnd.httphone 0
+application/vnd.hydrostatix.sof-data 0
+application/vnd.hyperdrive+json 0
+application/vnd.hzn-3d-crossword 0
+application/vnd.ibm.MiniPay 0
+application/vnd.ibm.afplinedata 0
+application/vnd.ibm.electronic-media 0
+application/vnd.ibm.modcap 0
+application/vnd.ibm.rights-management 0
+application/vnd.ibm.secure-container 0
+application/vnd.iccprofile 0
+application/vnd.ieee.1905 0
+application/vnd.igloader 0
+application/vnd.immervision-ivp 0
+application/vnd.immervision-ivu 0
+application/vnd.ims.imsccv1p1 0
+application/vnd.ims.imsccv1p2 0
+application/vnd.ims.imsccv1p3 0
+application/vnd.ims.lis.v2.result+json 0
+application/vnd.ims.lti.v2.toolconsumerprofile+json 0
+application/vnd.ims.lti.v2.toolproxy+json 0
+application/vnd.ims.lti.v2.toolproxy.id+json 0
+application/vnd.ims.lti.v2.toolsettings+json 0
+application/vnd.ims.lti.v2.toolsettings.simple+json 0
+application/vnd.informedcontrol.rms+xml 0
+application/vnd.infotech.project 0
+application/vnd.infotech.project+xml 0
+application/vnd.innopath.wamp.notification 0
+application/vnd.insors.igm 0
+application/vnd.intercon.formnet 0
+application/vnd.intergeo 0
+application/vnd.intertrust.digibox 0
+application/vnd.intertrust.nncp 0
+application/vnd.intu.qbo 0
+application/vnd.intu.qfx 0
+application/vnd.iptc.g2.catalogitem+xml 0
+application/vnd.iptc.g2.conceptitem+xml 0
+application/vnd.iptc.g2.knowledgeitem+xml 0
+application/vnd.iptc.g2.newsitem+xml 0
+application/vnd.iptc.g2.newsmessage+xml 0
+application/vnd.iptc.g2.packageitem+xml 0
+application/vnd.iptc.g2.planningitem+xml 0
+application/vnd.ipunplugged.rcprofile 0
+application/vnd.irepository.package+xml 0
+application/vnd.is-xpr 0
+application/vnd.isac.fcs 0
+application/vnd.jam 0
+application/vnd.japannet-directory-service 0
+application/vnd.japannet-jpnstore-wakeup 0
+application/vnd.japannet-payment-wakeup 0
+application/vnd.japannet-registration 0
+application/vnd.japannet-registration-wakeup 0
+application/vnd.japannet-setstore-wakeup 0
+application/vnd.japannet-verification 0
+application/vnd.japannet-verification-wakeup 0
+application/vnd.jcp.javame.midlet-rms 0
+application/vnd.jisp 0
+application/vnd.joost.joda-archive 0
+application/vnd.jsk.isdn-ngn 0
+application/vnd.kahootz 0
+application/vnd.kde.karbon 0
+application/vnd.kde.kchart 0
+application/vnd.kde.kformula 0
+application/vnd.kde.kivio 0
+application/vnd.kde.kontour 0
+application/vnd.kde.kpresenter 0
+application/vnd.kde.kspread 0
+application/vnd.kde.kword 0
+application/vnd.kenameaapp 0
+application/vnd.kidspiration 0
+application/vnd.koan 0
+application/vnd.kodak-descriptor 0
+application/vnd.las.las+xml 0
+application/vnd.liberty-request+xml 0
+application/vnd.llamagraphics.life-balance.desktop 0
+application/vnd.llamagraphics.life-balance.exchange+xml 0
+application/vnd.lotus-1-2-3 0
+application/vnd.lotus-approach 0
+application/vnd.lotus-freelance 0
+application/vnd.lotus-notes 0
+application/vnd.lotus-organizer 0
+application/vnd.lotus-screencam 0
+application/vnd.lotus-wordpro 0
+application/vnd.macports.portpkg 0
+application/vnd.mapbox-vector-tile 0
+application/vnd.marlin.drm.actiontoken+xml 0
+application/vnd.marlin.drm.conftoken+xml 0
+application/vnd.marlin.drm.license+xml 0
+application/vnd.marlin.drm.mdcf 0
+application/vnd.mason+json 0
+application/vnd.maxmind.maxmind-db 0
+application/vnd.mcd 0
+application/vnd.medcalcdata 0
+application/vnd.mediastation.cdkey 0
+application/vnd.meridian-slingshot 0
+application/vnd.mfmp 0
+application/vnd.micro+json 0
+application/vnd.micrografx-igx 0
+application/vnd.micrografx.flo 0
+application/vnd.microsoft.portable-executable 0
+application/vnd.miele+json 0
+application/vnd.minisoft-hp3000-save 0
+application/vnd.mitsubishi.misty-guard.trustweb 0
+application/vnd.mophun.application 0
+application/vnd.mophun.certificate 0
+application/vnd.motorola.flexsuite 0
+application/vnd.motorola.flexsuite.adsi 0
+application/vnd.motorola.flexsuite.fis 0
+application/vnd.motorola.flexsuite.gotap 0
+application/vnd.motorola.flexsuite.kmr 0
+application/vnd.motorola.flexsuite.ttc 0
+application/vnd.motorola.flexsuite.wem 0
+application/vnd.motorola.iprm 0
+application/vnd.mozilla.xul+xml 0
+application/vnd.ms-3mfdocument 0
+application/vnd.ms-PrintDeviceCapabilities+xml 0
+application/vnd.ms-artgalry 0
+application/vnd.ms-asf 0
+application/vnd.ms-cab-compressed 0
+application/vnd.ms-excel 0
+application/vnd.ms-excel.addin.macroEnabled.12 0
+application/vnd.ms-excel.sheet.binary.macroEnabled.12 0
+application/vnd.ms-excel.sheet.macroEnabled.12 0
+application/vnd.ms-excel.template.macroEnabled.12 0
+application/vnd.ms-fontobject 0
+application/vnd.ms-htmlhelp 0
+application/vnd.ms-ims 0
+application/vnd.ms-lrm 0
+application/vnd.ms-office.activeX+xml 0
+application/vnd.ms-officetheme 0
+application/vnd.ms-playready.initiator+xml 0
+application/vnd.ms-powerpoint 0
+application/vnd.ms-powerpoint.addin.macroEnabled.12 0
+application/vnd.ms-powerpoint.presentation.macroEnabled.12 0
+application/vnd.ms-powerpoint.slide.macroEnabled.12 0
+application/vnd.ms-powerpoint.slideshow.macroEnabled.12 0
+application/vnd.ms-powerpoint.template.macroEnabled.12 0
+application/vnd.ms-project 0
+application/vnd.ms-tnef 0
+application/vnd.ms-windows.devicepairing 0
+application/vnd.ms-windows.nwprinting.oob 0
+application/vnd.ms-windows.printerpairing 0
+application/vnd.ms-windows.wsd.oob 0
+application/vnd.ms-wmdrm.lic-chlg-req 0
+application/vnd.ms-wmdrm.lic-resp 0
+application/vnd.ms-wmdrm.meter-chlg-req 0
+application/vnd.ms-wmdrm.meter-resp 0
+application/vnd.ms-word.document.macroEnabled.12 0
+application/vnd.ms-word.template.macroEnabled.12 0
+application/vnd.ms-works 0
+application/vnd.ms-wpl 0
+application/vnd.ms-xpsdocument 0
+application/vnd.msa-disk-image 0
+application/vnd.mseq 0
+application/vnd.msign 0
+application/vnd.multiad.creator 0
+application/vnd.multiad.creator.cif 0
+application/vnd.music-niff 0
+application/vnd.musician 0
+application/vnd.muvee.style 0
+application/vnd.mynfc 0
+application/vnd.ncd.control 0
+application/vnd.ncd.reference 0
+application/vnd.nervana 0
+application/vnd.netfpx 0
+application/vnd.neurolanguage.nlu 0
+application/vnd.nintendo.nitro.rom 0
+application/vnd.nintendo.snes.rom 0
+application/vnd.nitf 0
+application/vnd.noblenet-directory 0
+application/vnd.noblenet-sealer 0
+application/vnd.noblenet-web 0
+application/vnd.nokia.catalogs 0
+application/vnd.nokia.conml+wbxml 0
+application/vnd.nokia.conml+xml 0
+application/vnd.nokia.iSDS-radio-presets 0
+application/vnd.nokia.iptv.config+xml 0
+application/vnd.nokia.landmark+wbxml 0
+application/vnd.nokia.landmark+xml 0
+application/vnd.nokia.landmarkcollection+xml 0
+application/vnd.nokia.n-gage.ac+xml 0
+application/vnd.nokia.n-gage.data 0
+application/vnd.nokia.n-gage.symbian.install 0
+application/vnd.nokia.ncd 0
+application/vnd.nokia.pcd+wbxml 0
+application/vnd.nokia.pcd+xml 0
+application/vnd.nokia.radio-preset 0
+application/vnd.nokia.radio-presets 0
+application/vnd.novadigm.EDM 0
+application/vnd.novadigm.EDX 0
+application/vnd.novadigm.EXT 0
+application/vnd.ntt-local.content-share 0
+application/vnd.ntt-local.file-transfer 0
+application/vnd.ntt-local.ogw_remote-access 0
+application/vnd.ntt-local.sip-ta_remote 0
+application/vnd.ntt-local.sip-ta_tcp_stream 0
+application/vnd.oasis.opendocument.chart 0
+application/vnd.oasis.opendocument.chart-template 0
+application/vnd.oasis.opendocument.database 0
+application/vnd.oasis.opendocument.formula 0
+application/vnd.oasis.opendocument.formula-template 0
+application/vnd.oasis.opendocument.graphics 0
+application/vnd.oasis.opendocument.graphics-template 0
+application/vnd.oasis.opendocument.image 0
+application/vnd.oasis.opendocument.image-template 0
+application/vnd.oasis.opendocument.presentation 0
+application/vnd.oasis.opendocument.presentation-template 0
+application/vnd.oasis.opendocument.spreadsheet 0
+application/vnd.oasis.opendocument.spreadsheet-template 0
+application/vnd.oasis.opendocument.text 0
+application/vnd.oasis.opendocument.text-master 0
+application/vnd.oasis.opendocument.text-template 0
+application/vnd.oasis.opendocument.text-web 0
+application/vnd.obn 0
+application/vnd.oftn.l10n+json 0
+application/vnd.oipf.contentaccessdownload+xml 0
+application/vnd.oipf.contentaccessstreaming+xml 0
+application/vnd.oipf.cspg-hexbinary 0
+application/vnd.oipf.dae.svg+xml 0
+application/vnd.oipf.dae.xhtml+xml 0
+application/vnd.oipf.mippvcontrolmessage+xml 0
+application/vnd.oipf.pae.gem 0
+application/vnd.oipf.spdiscovery+xml 0
+application/vnd.oipf.spdlist+xml 0
+application/vnd.oipf.ueprofile+xml 0
+application/vnd.oipf.userprofile+xml 0
+application/vnd.olpc-sugar 0
+application/vnd.oma-scws-config 0
+application/vnd.oma-scws-http-request 0
+application/vnd.oma-scws-http-response 0
+application/vnd.oma.bcast.associated-procedure-parameter+xml 0
+application/vnd.oma.bcast.drm-trigger+xml 0
+application/vnd.oma.bcast.imd+xml 0
+application/vnd.oma.bcast.ltkm 0
+application/vnd.oma.bcast.notification+xml 0
+application/vnd.oma.bcast.provisioningtrigger 0
+application/vnd.oma.bcast.sgboot 0
+application/vnd.oma.bcast.sgdd+xml 0
+application/vnd.oma.bcast.sgdu 0
+application/vnd.oma.bcast.simple-symbol-container 0
+application/vnd.oma.bcast.smartcard-trigger+xml 0
+application/vnd.oma.bcast.sprov+xml 0
+application/vnd.oma.bcast.stkm 0
+application/vnd.oma.cab-address-book+xml 0
+application/vnd.oma.cab-feature-handler+xml 0
+application/vnd.oma.cab-pcc+xml 0
+application/vnd.oma.cab-subs-invite+xml 0
+application/vnd.oma.cab-user-prefs+xml 0
+application/vnd.oma.dcd 0
+application/vnd.oma.dcdc 0
+application/vnd.oma.dd2+xml 0
+application/vnd.oma.drm.risd+xml 0
+application/vnd.oma.group-usage-list+xml 0
+application/vnd.oma.pal+xml 0
+application/vnd.oma.poc.detailed-progress-report+xml 0
+application/vnd.oma.poc.final-report+xml 0
+application/vnd.oma.poc.groups+xml 0
+application/vnd.oma.poc.invocation-descriptor+xml 0
+application/vnd.oma.poc.optimized-progress-report+xml 0
+application/vnd.oma.push 0
+application/vnd.oma.scidm.messages+xml 0
+application/vnd.oma.xcap-directory+xml 0
+application/vnd.omads-email+xml 0
+application/vnd.omads-file+xml 0
+application/vnd.omads-folder+xml 0
+application/vnd.omaloc-supl-init 0
+application/vnd.openblox.game+xml 0
+application/vnd.openblox.game-binary 0
+application/vnd.openeye.oeb 0
+application/vnd.openxmlformats-officedocument.custom-properties+xml 0
+application/vnd.openxmlformats-officedocument.customXmlProperties+xml 0
+application/vnd.openxmlformats-officedocument.drawing+xml 0
+application/vnd.openxmlformats-officedocument.drawingml.chart+xml 0
+application/vnd.openxmlformats-officedocument.drawingml.chartshapes+xml 0
+application/vnd.openxmlformats-officedocument.drawingml.diagramColors+xml 0
+application/vnd.openxmlformats-officedocument.drawingml.diagramData+xml 0
+application/vnd.openxmlformats-officedocument.drawingml.diagramLayout+xml 0
+application/vnd.openxmlformats-officedocument.drawingml.diagramStyle+xml 0
+application/vnd.openxmlformats-officedocument.extended-properties+xml 0
+application/vnd.openxmlformats-officedocument.presentationml-template 0
+application/vnd.openxmlformats-officedocument.presentationml.commentAuthors+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.comments+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.handoutMaster+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.notesMaster+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.notesSlide+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.presProps+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.presentation 0
+application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.slide 0
+application/vnd.openxmlformats-officedocument.presentationml.slide+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.slideLayout+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.slideMaster+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.slideUpdateInfo+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.slideshow 0
+application/vnd.openxmlformats-officedocument.presentationml.slideshow.main+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.tableStyles+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.tags+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.template.main+xml 0
+application/vnd.openxmlformats-officedocument.presentationml.viewProps+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml-template 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.calcChain+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.chartsheet+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.comments+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.connections+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.dialogsheet+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.externalLink+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheDefinition+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheRecords+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.pivotTable+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.queryTable+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.revisionHeaders+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.revisionLog+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.sharedStrings+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.sheet 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.sheetMetadata+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.table+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.tableSingleCells+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.template.main+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.userNames+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.volatileDependencies+xml 0
+application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml 0
+application/vnd.openxmlformats-officedocument.theme+xml 0
+application/vnd.openxmlformats-officedocument.themeOverride+xml 0
+application/vnd.openxmlformats-officedocument.vmlDrawing 0
+application/vnd.openxmlformats-officedocument.wordprocessingml-template 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.document 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.document.glossary+xml 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.endnotes+xml 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.fontTable+xml 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.footnotes+xml 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.settings+xml 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.template.main+xml 0
+application/vnd.openxmlformats-officedocument.wordprocessingml.webSettings+xml 0
+application/vnd.openxmlformats-package.core-properties+xml 0
+application/vnd.openxmlformats-package.digital-signature-xmlsignature+xml 0
+application/vnd.openxmlformats-package.relationships+xml 0
+application/vnd.oracle.resource+json 0
+application/vnd.orange.indata 0
+application/vnd.osa.netdeploy 0
+application/vnd.osgeo.mapguide.package 0
+application/vnd.osgi.bundle 0
+application/vnd.osgi.dp 0
+application/vnd.osgi.subsystem 0
+application/vnd.otps.ct-kip+xml 0
+application/vnd.oxli.countgraph 0
+application/vnd.pagerduty+json 0
+application/vnd.palm 0
+application/vnd.panoply 0
+application/vnd.paos+xml 0
+application/vnd.pawaafile 0
+application/vnd.pcos 0
+application/vnd.pg.format 0
+application/vnd.pg.osasli 0
+application/vnd.piaccess.application-licence 0
+application/vnd.picsel 0
+application/vnd.pmi.widget 0
+application/vnd.poc.group-advertisement+xml 0
+application/vnd.pocketlearn 0
+application/vnd.powerbuilder6 0
+application/vnd.powerbuilder6-s 0
+application/vnd.powerbuilder7 0
+application/vnd.powerbuilder7-s 0
+application/vnd.powerbuilder75 0
+application/vnd.powerbuilder75-s 0
+application/vnd.preminet 0
+application/vnd.previewsystems.box 0
+application/vnd.proteus.magazine 0
+application/vnd.publishare-delta-tree 0
+application/vnd.pvi.ptid1 0
+application/vnd.pwg-multiplexed 0
+application/vnd.pwg-xhtml-print+xml 0
+application/vnd.qualcomm.brew-app-res 0
+application/vnd.quobject-quoxdocument 0
+application/vnd.radisys.moml+xml 0
+application/vnd.radisys.msml+xml 0
+application/vnd.radisys.msml-audit+xml 0
+application/vnd.radisys.msml-audit-conf+xml 0
+application/vnd.radisys.msml-audit-conn+xml 0
+application/vnd.radisys.msml-audit-dialog+xml 0
+application/vnd.radisys.msml-audit-stream+xml 0
+application/vnd.radisys.msml-conf+xml 0
+application/vnd.radisys.msml-dialog+xml 0
+application/vnd.radisys.msml-dialog-base+xml 0
+application/vnd.radisys.msml-dialog-fax-detect+xml 0
+application/vnd.radisys.msml-dialog-fax-sendrecv+xml 0
+application/vnd.radisys.msml-dialog-group+xml 0
+application/vnd.radisys.msml-dialog-speech+xml 0
+application/vnd.radisys.msml-dialog-transform+xml 0
+application/vnd.rainstor.data 0
+application/vnd.rapid 0
+application/vnd.realvnc.bed 0
+application/vnd.recordare.musicxml 0
+application/vnd.recordare.musicxml+xml 0
+application/vnd.renlearn.rlprint 0
+application/vnd.rig.cryptonote 0
+application/vnd.route66.link66+xml 0
+application/vnd.rs-274x 0
+application/vnd.ruckus.download 0
+application/vnd.s3sms 0
+application/vnd.sailingtracker.track 0
+application/vnd.sbm.cid 0
+application/vnd.sbm.mid2 0
+application/vnd.scribus 0
+application/vnd.sealed-doc 0
+application/vnd.sealed-eml 0
+application/vnd.sealed-mht 0
+application/vnd.sealed-ppt 0
+application/vnd.sealed-tiff 0
+application/vnd.sealed-xls 0
+application/vnd.sealed.3df 0
+application/vnd.sealed.csf 0
+application/vnd.sealed.net 0
+application/vnd.sealedmedia.softseal-html 0
+application/vnd.sealedmedia.softseal-pdf 0
+application/vnd.seemail 0
+application/vnd.semd 0
+application/vnd.semf 0
+application/vnd.shana.informed.formdata 0
+application/vnd.shana.informed.formtemplate 0
+application/vnd.shana.informed.interchange 0
+application/vnd.shana.informed.package 0
+application/vnd.siren+json 0
+application/vnd.smaf 0
+application/vnd.smart.notebook 0
+application/vnd.smart.teacher 0
+application/vnd.software602.filler.form+xml 0
+application/vnd.software602.filler.form-xml-zip 0
+application/vnd.solent.sdkm+xml 0
+application/vnd.spotfire.dxp 0
+application/vnd.spotfire.sfs 0
+application/vnd.sss-cod 0
+application/vnd.sss-dtf 0
+application/vnd.sss-ntf 0
+application/vnd.stepmania.package 0
+application/vnd.stepmania.stepchart 0
+application/vnd.street-stream 0
+application/vnd.sun.wadl+xml 0
+application/vnd.sus-calendar 0
+application/vnd.svd 0
+application/vnd.swiftview-ics 0
+application/vnd.syncml+xml 0
+application/vnd.syncml.dm+wbxml 0
+application/vnd.syncml.dm+xml 0
+application/vnd.syncml.dm.notification 0
+application/vnd.syncml.dmddf+wbxml 0
+application/vnd.syncml.dmddf+xml 0
+application/vnd.syncml.dmtnds+wbxml 0
+application/vnd.syncml.dmtnds+xml 0
+application/vnd.syncml.ds.notification 0
+application/vnd.tao.intent-module-archive 0
+application/vnd.tcpdump.pcap 0
+application/vnd.tmd.mediaflex.api+xml 0
+application/vnd.tml 0
+application/vnd.tmobile-livetv 0
+application/vnd.trid.tpt 0
+application/vnd.triscape.mxs 0
+application/vnd.trueapp 0
+application/vnd.truedoc 0
+application/vnd.ubisoft.webplayer 0
+application/vnd.ufdl 0
+application/vnd.uiq.theme 0
+application/vnd.umajin 0
+application/vnd.unity 0
+application/vnd.uoml+xml 0
+application/vnd.uplanet.alert 0
+application/vnd.uplanet.alert-wbxml 0
+application/vnd.uplanet.bearer-choice 0
+application/vnd.uplanet.bearer-choice-wbxml 0
+application/vnd.uplanet.cacheop 0
+application/vnd.uplanet.cacheop-wbxml 0
+application/vnd.uplanet.channel 0
+application/vnd.uplanet.channel-wbxml 0
+application/vnd.uplanet.list 0
+application/vnd.uplanet.list-wbxml 0
+application/vnd.uplanet.listcmd 0
+application/vnd.uplanet.listcmd-wbxml 0
+application/vnd.uplanet.signal 0
+application/vnd.uri-map 0
+application/vnd.valve.source.material 0
+application/vnd.vcx 0
+application/vnd.vd-study 0
+application/vnd.vectorworks 0
+application/vnd.verimatrix.vcas 0
+application/vnd.vidsoft.vidconference 0
+application/vnd.visio 0
+application/vnd.visionary 0
+application/vnd.vividence.scriptfile 0
+application/vnd.vsf 0
+application/vnd.wap-slc 0
+application/vnd.wap-wbxml 0
+application/vnd.wap.sic 0
+application/vnd.wap.wmlscriptc 0
+application/vnd.webturbo 0
+application/vnd.wfa.p2p 0
+application/vnd.wfa.wsc 0
+application/vnd.windows.devicepairing 0
+application/vnd.wmc 0
+application/vnd.wmf.bootstrap 0
+application/vnd.wolfram.mathematica 0
+application/vnd.wolfram.mathematica.package 0
+application/vnd.wolfram.player 0
+application/vnd.wordperfect 0
+application/vnd.wqd 0
+application/vnd.wrq-hp3000-labelled 0
+application/vnd.wt.stf 0
+application/vnd.wv.csp+wbxml 0
+application/vnd.wv.csp+xml 0
+application/vnd.wv.ssp+xml 0
+application/vnd.xacml+json 0
+application/vnd.xara 0
+application/vnd.xfdl 0
+application/vnd.xfdl.webform 0
+application/vnd.xmi+xml 0
+application/vnd.xmpie.cpkg 0
+application/vnd.xmpie.dpkg 0
+application/vnd.xmpie.plan 0
+application/vnd.xmpie.ppkg 0
+application/vnd.xmpie.xlim 0
+application/vnd.yamaha.hv-dic 0
+application/vnd.yamaha.hv-script 0
+application/vnd.yamaha.hv-voice 0
+application/vnd.yamaha.openscoreformat 0
+application/vnd.yamaha.openscoreformat.osfpvg+xml 0
+application/vnd.yamaha.remote-setup 0
+application/vnd.yamaha.smaf-audio 0
+application/vnd.yamaha.smaf-phrase 0
+application/vnd.yamaha.through-ngn 0
+application/vnd.yamaha.tunnel-udpencap 0
+application/vnd.yaoweme 0
+application/vnd.yellowriver-custom-menu 0
+application/vnd.zul 0
+application/vnd.zzazz.deck+xml 0
+application/voicexml+xml 0
+application/vq-rtcpxr 0
+application/watcherinfo+xml 0
+application/whoispp-query 0
+application/whoispp-response 0
+application/wita 0
+application/wordperfect5.1 0
+application/wsdl+xml 0
+application/wspolicy+xml 0
+application/x-msdownload 0
+application/x-pkcs12 0
+application/x-www-form-urlencoded 0
+application/x-tar 0
+application/x-x509-ca-cert 0
+application/x-zip-compressed 0
+application/x400-bp 0
+application/xacml+xml 0
+application/xcap-att+xml 0
+application/xcap-caps+xml 0
+application/xcap-diff+xml 0
+application/xcap-el+xml 0
+application/xcap-error+xml 0
+application/xcap-ns+xml 0
+application/xcon-conference-info+xml 0
+application/xcon-conference-info-diff+xml 0
+application/xenc+xml 0
+application/xhtml+xml 0
+application/xml 0
+application/xml-dtd 0
+application/xml-external-parsed-entity 0
+application/xml-patch+xml 0
+application/xmpp+xml 0
+application/xop+xml 0
+application/xv+xml 0
+application/yang 0
+application/yin+xml 0
+application/zip 0
+application/zlib 0
+audio/1d-interleaved-parityfec 0
+audio/32kadpcm 0
+audio/3gpp 0
+audio/3gpp2 0
+audio/AMR 0
+audio/AMR-WB 0
+audio/ATRAC-ADVANCED-LOSSLESS 0
+audio/ATRAC-X 0
+audio/ATRAC3 0
+audio/BV16 0
+audio/BV32 0
+audio/CN 0
+audio/DAT12 0
+audio/DV 0
+audio/DVI4 0
+audio/EVRC 0
+audio/EVRC-QCP 0
+audio/EVRC0 0
+audio/EVRC1 0
+audio/EVRCB 0
+audio/EVRCB0 0
+audio/EVRCB1 0
+audio/EVRCNW 0
+audio/EVRCNW0 0
+audio/EVRCNW1 0
+audio/EVRCWB 0
+audio/EVRCWB0 0
+audio/EVRCWB1 0
+audio/EVS 0
+audio/G711-0 0
+audio/G719 0
+audio/G722 0
+audio/G7221 0
+audio/G723 0
+audio/G726-16 0
+audio/G726-24 0
+audio/G726-32 0
+audio/G726-40 0
+audio/G728 0
+audio/G729 0
+audio/G729D 0
+audio/G729E 0
+audio/GSM 0
+audio/GSM-EFR 0
+audio/GSM-HR-08 0
+audio/L16 0
+audio/L20 0
+audio/L24 0
+audio/L8 0
+audio/LPC 0
+audio/MP4A-LATM 0
+audio/MPA 0
+audio/PCMA 0
+audio/PCMA-WB 0
+audio/PCMU 0
+audio/PCMU-WB 0
+audio/RED 0
+audio/SMV 0
+audio/SMV-QCP 0
+audio/SMV0 0
+audio/UEMCLIP 0
+audio/VDVI 0
+audio/VMR-WB 0
+audio/ac3 0
+audio/amr-wb+ 0
+audio/aptx 0
+audio/asc 0
+audio/basic 0
+audio/clearmode 0
+audio/dls 0
+audio/dsr-es201108 0
+audio/dsr-es202050 0
+audio/dsr-es202211 0
+audio/dsr-es202212 0
+audio/eac3 0
+audio/encaprtp 0
+audio/example 0
+audio/fwdred 0
+audio/iLBC 0
+audio/ip-mr_v2.5 0
+audio/mobile-xmf 0
+audio/mp4 0
+audio/mpa-robust 0
+audio/mpeg 0
+audio/mpeg4-generic 0
+audio/ogg 0
+audio/opus 0
+audio/prs.sid 0
+audio/qcelp 0
+audio/raptorfec 0
+audio/rtp-enc-aescm128 0
+audio/rtp-midi 0
+audio/rtploopback 0
+audio/rtx 0
+audio/sp-midi 0
+audio/speex 0
+audio/t140c 0
+audio/t38 0
+audio/telephone-event 0
+audio/tone 0
+audio/ulpfec 0
+audio/vnd.3gpp.iufp 0
+audio/vnd.4SB 0
+audio/vnd.CELP 0
+audio/vnd.audiokoz 0
+audio/vnd.cisco.nse 0
+audio/vnd.cmles.radio-events 0
+audio/vnd.cns.anp1 0
+audio/vnd.cns.inf1 0
+audio/vnd.dece.audio 0
+audio/vnd.digital-winds 0
+audio/vnd.dlna.adts 0
+audio/vnd.dolby.heaac.1 0
+audio/vnd.dolby.heaac.2 0
+audio/vnd.dolby.mlp 0
+audio/vnd.dolby.mps 0
+audio/vnd.dolby.pl2 0
+audio/vnd.dolby.pl2x 0
+audio/vnd.dolby.pl2z 0
+audio/vnd.dolby.pulse.1 0
+audio/vnd.dra 0
+audio/vnd.dts 0
+audio/vnd.dts.hd 0
+audio/vnd.dvb.file 0
+audio/vnd.everad.plj 0
+audio/vnd.hns.audio 0
+audio/vnd.lucent.voice 0
+audio/vnd.ms-playready.media.pya 0
+audio/vnd.nokia.mobile-xmf 0
+audio/vnd.nortel.vbk 0
+audio/vnd.nuera.ecelp4800 0
+audio/vnd.nuera.ecelp7470 0
+audio/vnd.nuera.ecelp9600 0
+audio/vnd.octel.sbc 0
+audio/vnd.rhetorex.32kadpcm 0
+audio/vnd.rip 0
+audio/vnd.sealedmedia.softseal-mpeg 0
+audio/vnd.vmx.cvsd 0
+audio/vorbis 0
+audio/vorbis-config 0
+image/bmp 0
+image/cgm 0
+image/example 0
+image/fits 0
+image/g3fax 0
+image/gif 0
+image/jp2 0
+image/jpeg 0
+image/jpm 0
+image/jpx 0
+image/naplps 0
+image/png 0
+image/prs.btif 0
+image/prs.pti 0
+image/pwg-raster 0
+image/t38 0
+image/tiff 0
+image/tiff-fx 0
+image/vnd-djvu 0
+image/vnd-svf 0
+image/vnd-wap-wbmp 0
+image/vnd.adobe.photoshop 0
+image/vnd.airzip.accelerator.azv 0
+image/vnd.cns.inf2 0
+image/vnd.dece.graphic 0
+image/vnd.dvb.subtitle 0
+image/vnd.dwg 0
+image/vnd.dxf 0
+image/vnd.fastbidsheet 0
+image/vnd.fpx 0
+image/vnd.fst 0
+image/vnd.fujixerox.edmics-mmr 0
+image/vnd.fujixerox.edmics-rlc 0
+image/vnd.globalgraphics.pgb 0
+image/vnd.microsoft.icon 0
+image/vnd.mix 0
+image/vnd.mozilla.apng 0
+image/vnd.ms-modi 0
+image/vnd.net-fpx 0
+image/vnd.radiance 0
+image/vnd.sealed-png 0
+image/vnd.sealedmedia.softseal-gif 0
+image/vnd.sealedmedia.softseal-jpg 0
+image/vnd.tencent.tap 0
+image/vnd.valve.source.texture 0
+image/vnd.xiff 0
+image/vnd.zbrush.pcx 0
+image/wmf 0
+image/x-wmf 0
+message/CPIM 0
+message/delivery-status 0
+message/disposition-notification 0
+message/example 0
+message/feedback-report 0
+message/global 0
+message/global-delivery-status 0
+message/global-disposition-notification 0
+message/global-headers 0
+message/http 0
+message/imdn+xml 0
+message/news 0
+message/rfc822 0
+message/rfc822-headers 0
+message/s-http 0
+message/sip 0
+message/sipfrag 0
+message/tracking-status 0
+message/vnd.si.simp 0
+message/vnd.wfa.wsc 0
+model/example 0
+model/iges 0
+model/vnd-dwf 0
+model/vnd.collada+xml 0
+model/vnd.flatland.3dml 0
+model/vnd.gdl 0
+model/vnd.gs-gdl 0
+model/vnd.gtw 0
+model/vnd.moml+xml 0
+model/vnd.mts 0
+model/vnd.opengex 0
+model/vnd.parasolid.transmit-binary 0
+model/vnd.parasolid.transmit-text 0
+model/vnd.valve.source.compiled-map 0
+model/vnd.vtu 0
+model/x3d+fastinfoset 0
+model/x3d+xml 0
+model/x3d-vrml 0
+multipart/alternative -1
+multipart/mixed -1
+multipart/appledouble 2
+multipart/byteranges 2
+multipart/encrypted -2
+multipart/example 2
+multipart/form-data 2
+multipart/header-set 2
+multipart/related -1
+multipart/report 0
+multipart/signed -2
+multipart/voice-message 2
+multipart/x-mixed-replace 2
+text/1d-interleaved-parityfec 0
+text/RED 0
+text/SGML 0
+text/cache-manifest 0
+text/calendar 0
+text/css 0
+text/csv 0
+text/csv-schema 0
+text/directory 0
+text/dns 0
+text/encaprtp 0
+text/example 0
+text/fwdred 0
+text/grammar-ref-list 0
+text/html 0
+text/jcr-cnd 0
+text/markdown 0
+text/mizar 0
+text/n3 0
+text/parameters 0
+text/provenance-notation 0
+text/prs.fallenstein.rst 0
+text/prs.lines.tag 0
+text/plain -1
+text/raptorfec 0
+text/rfc822-headers 0
+text/rtf 0
+text/rtp-enc-aescm128 0
+text/rtploopback 0
+text/rtx 0
+text/t140 0
+text/tab-separated-values 0
+text/troff 0
+text/turtle 0
+text/ulpfec 0
+text/uri-list 0
+text/vcard 0
+text/vnd-a 0
+text/vnd-curl 0
+text/vnd.DMClientScript 0
+text/vnd.IPTC.NITF 0
+text/vnd.IPTC.NewsML 0
+text/vnd.abc 0
+text/vnd.debian.copyright 0
+text/vnd.dvb.subtitle 0
+text/vnd.esmertec.theme-descriptor 0
+text/vnd.fly 0
+text/vnd.fmi.flexstor 0
+text/vnd.graphviz 0
+text/vnd.in3d.3dml 0
+text/vnd.in3d.spot 0
+text/vnd.latex-z 0
+text/vnd.motorola.reflex 0
+text/vnd.ms-mediapackage 0
+text/vnd.net2phone.commcenter.command 0
+text/vnd.radisys.msml-basic-layout 0
+text/vnd.si.uricatalogue 0
+text/vnd.sun.j2me.app-descriptor 0
+text/vnd.trolltech.linguist 0
+text/vnd.wap-wml 0
+text/vnd.wap.si 0
+text/vnd.wap.sl 0
+text/vnd.wap.wmlscript 0
+text/xml 0
+text/xml-external-parsed-entity 0
+text/x-vcard 0
+video/1d-interleaved-parityfec 0
+video/3gpp 0
+video/3gpp-tt 0
+video/3gpp2 0
+video/BMPEG 0
+video/BT656 0
+video/CelB 0
+video/DV 0
+video/H261 0
+video/H263 0
+video/H263-1998 0
+video/H263-2000 0
+video/H264 0
+video/H264-RCDO 0
+video/H264-SVC 0
+video/H265 0
+video/JPEG 0
+video/MP1S 0
+video/MP2P 0
+video/MP2T 0
+video/MP4V-ES 0
+video/MPV 0
+video/SMPTE292M 0
+video/VP8 0
+video/encaprtp 0
+video/example 0
+video/iso.segment 0
+video/jpeg2000 0
+video/mj2 0
+video/mp4 0
+video/mpeg4-generic 0
+video/nv 0
+video/ogg 0
+video/pointer 0
+video/quicktime 0
+video/raptorfec 0
+video/rtp-enc-aescm128 0
+video/rtploopback 0
+video/rtx 0
+video/ulpfec 0
+video/vc1 0
+video/vnd-mpegurl 0
+video/vnd-vivo 0
+video/vnd.CCTV 0
+video/vnd.dece-mp4 0
+video/vnd.dece.hd 0
+video/vnd.dece.mobile 0
+video/vnd.dece.pd 0
+video/vnd.dece.sd 0
+video/vnd.dece.video 0
+video/vnd.directv-mpeg 0
+video/vnd.directv.mpeg-tts 0
+video/vnd.dlna.mpeg-tts 0
+video/vnd.dvb.file 0
+video/vnd.fvt 0
+video/vnd.hns.video 0
+video/vnd.iptvforum.1dparityfec-1010 0
+video/vnd.iptvforum.1dparityfec-2005 0
+video/vnd.iptvforum.2dparityfec-1010 0
+video/vnd.iptvforum.2dparityfec-2005 0
+video/vnd.iptvforum.ttsavc 0
+video/vnd.iptvforum.ttsmpeg2 0
+video/vnd.motorola.video 0
+video/vnd.motorola.videop 0
+video/vnd.ms-playready.media.pyv 0
+video/vnd.nokia.interleaved-multimedia 0
+video/vnd.nokia.videovoip 0
+video/vnd.objectvideo 0
+video/vnd.radgamettools.bink 0
+video/vnd.radgamettools.smacker 0
+video/vnd.sealed-swf 0
+video/vnd.sealed.mpeg1 0
+video/vnd.sealed.mpeg4 0
+video/vnd.sealedmedia.softseal-mov 0
+video/vnd.uvvu-mp4 0
diff --git a/conf/maps.d/redirectors.inc b/conf/maps.d/redirectors.inc
new file mode 100644
index 0000000..c7d7f25
--- /dev/null
+++ b/conf/maps.d/redirectors.inc
@@ -0,0 +1,1045 @@
+000d.ru
+0845.com
+0c.ru
+0lv.ru
+0pen.me
+0rz.tw
+10r.us
+123url.org
+140.uz
+17q.com
+1c-bitrix.ru
+1cl.in
+1ink.in
+1ink.ru
+1iny.com
+1lik.net
+1link.in
+1url.com
+1url.in
+1-url.net
+1-url.ru
+2big.at
+2dwww.com
+2.gp
+2it.info
+2.ly
+2mb.eu
+2qu.ru
+2sms.ru
+2tu.me
+2tu.us
+2url.org
+307.to
+3fw.ru
+3le.ru
+3.ly
+3.vu
+3x.si
+4.gg
+4job.ru
+4.ly
+4ms.me
+4p5.com
+4ry.ru
+4sq.com
+4u.gd
+4url.cc
+4url.tk
+5.gp
+5link.tk
+5pl.us
+5url.net
+5z8.info
+6fr.ru
+6.ly
+6pn.com
+6url.com
+6yo.org
+70.ru
+74job.ru
+7.ly
+7ly.ru
+7pisem.ru
+7ruh.com
+7ry.us
+7xu.org
+8.ly
+8q.ro
+9mp.com
+9-n.org
+9xi.ru
+a1.tc
+a2k.in
+aa.cx
+aafter.us
+abe5.com
+access.im
+action-emails.ru
+ad4.us
+adf.ly
+adjix.com
+adsbeta.net
+ad.vu
+afx.cc
+a.gg
+ah.ae
+aipro.ru
+airs.ru
+aka-url.com
+alic.at
+all.fuseurl.com
+allshort.ru
+all-top.ru
+alturl.com
+a.md
+amzn.to
+a.nf
+apeurl.com
+api.m3653.net
+apsense.cc
+apu.sh
+ar.gy
+arm.in
+arst.ch
+atiny.me
+atto.co.za
+atu.ca
+autodesk.com
+avast.com
+avoo.net
+azc.cc
+b23.ru
+b2l.me
+backupurl.com
+bacn.me
+bai.lu
+bcool.bz
+bezurl.com
+bi.gl
+binged.it
+bin.nu
+bitby.net
+bit.do
+bit.gy
+bit.ly
+bitly.com
+biturl.net
+bit.uz
+bizj.us
+bloat.me
+bmu.li
+boi.re
+bq.ro
+bravo.ly
+briefurl.pl
+bsa.ly
+bsndsy.ru
+budurl.com
+bun.ru
+bu.tt
+byst.ro
+byyb.net
+bz9.com
+campaign-services.directcrm.ru
+canurl.com
+capello.linkatty.com
+capourl.com
+care2share.tk
+cbs.so
+cbuz.com
+cctv.ws
+cd.vg
+cektkp.com
+cha.la
+chilp.it
+chzb.gr
+cjb.net
+cjt99.tk
+clck.ru
+cliccami.info
+click2.info
+click.email4customers.com
+click.emailinfo.mail.hpe.com
+click.icptrack.com
+click-me.us
+clickthru.ca
+clickv.tk
+cli.gs
+clkit.co
+cl.lk
+cl.ly
+clme.ru
+cloakreferer.com
+clockurl.com
+clop.in
+cms.im
+cmylink.com
+cnect.us
+comyonet.com
+conta.cc
+coolestone.com
+cort.as
+cortas.elpais.com
+cot.ag
+cowurl.com
+cp.bitrix.ru
+cr.am
+createurl.com
+crks.me
+crlf.ru
+crop.im
+crum.pl
+ctvr.us
+cug.kr
+cut4.me
+cut.by
+cuthut.com
+cutt.us
+d2u.us
+d8z.ru
+dai.ly
+da.lc
+ddp.net
+decenturl.com
+delivr.com
+dev0.ru
+dft.ba
+digbig.com
+di.gd
+digg.com
+digidns.net
+din.gy
+directtrafficlink.com
+disq.us
+dld.bz
+dlvr.it
+dmanalytics1.com
+doiop.com
+do.my
+dopen.us
+dot.tk
+dr2.biz
+driz.ru
+dr.tl
+durlz.info
+easyuri.com
+easyurl.jp
+easyurl.net
+eepurl.com
+ej.uz
+elurl.com
+email.account.2gis.com
+email.mail.ostrovok.ru
+email.news.ostrovok.ru
+e.mail.ru
+emap.ws
+em.digium.com
+etdurl.com
+eweri.com
+exa.im
+f1ru.net
+fa.by
+fanta.linkatty.com
+fav.me
+fbi.pp.ua
+fb.me
+fbshare.me
+fff.to
+ff.im
+ffs.cc
+fi.gd
+fire.to
+firsturl.de
+firsturl.net
+fishurl.ru
+flane.info
+flavr.be
+flic.kr
+flq.us
+flx.im
+fly2.ws
+folo.me
+fo.my
+fon.gs
+forex-trade.be
+fqav.com
+freak.to
+freepl.us
+free-redirect.tk
+freeurl.me
+free-url-redirection.com.ru
+fur.ly
+fuseurl.com
+fuzzy.to
+fwd4.me
+fwds.me
+fwib.net
+fyad.org
+fyn.im
+g00.me
+gadaf.fi
+game-url.com
+gentleurl.net
+geteml.com
+getlink.info
+get.sh
+get.tf
+gho.co
+gig140.com
+gizmo.do
+gl.am
+glink.co
+gltw.ru
+gmetzner.de
+gmy.su
+gnu.su
+go2-url.com
+go.9nl.com
+go9.us
+goandgrab.info
+gog.tc
+go.it
+go-links.net
+golook.at
+go.ly
+good.ly
+goo.gl
+goo.pm
+go.qb.by
+goshrink.com
+gosite.in
+goto.pattayacitythailand.com
+gourl.ca
+gourl.gr
+gourl.it
+go-url.ru
+go.usa.gov
+gri.bz
+g.ro.lt
+gtgg.us
+g.ua
+gu.ma
+gurl.es
+haqm.com
+hex.io
+hhvx.com
+hiderefer.com
+hijw.com
+hi.kg
+hit.kg
+hj.to
+hlurl.com
+hmm.ph
+ho.io
+hop.clickbank.net
+hopclicks.com
+ho.pe
+hop.kz
+href.in
+hsblinks.com
+htxt.it
+hubb.me
+huff.to
+hulu.com
+hurl.me
+hurl.ws
+huuk.net
+hvmnd.org
+i2h.de
+i5.be
+icanhaz.com
+idek.net
+idelink.com
+ifree.kz
+ih3.ru
+ikeafamilynews.ru
+ilix.in
+ilnk.me
+informer.ru
+innogam.es
+ino.me
+int.kz
+ipsha.ru
+ir.pe
+is.gd
+is.gs
+issuu.com
+itshrunk.com
+its.my
+iurlz.com
+ix.lt
+ixr.be
+j3w.it
+ja.cx
+jdem.cz
+jewi.sh
+jijr.com
+jmb.tw
+j.mp
+jom.la
+joo.ru
+just.as
+juu.cc
+keep2.me
+kickurl.com
+kipq.com
+kisaurl.com
+ki.tl
+kl.am
+klck.me
+klik.sihitam.com
+klx.co
+knb.im
+kon.tl
+kore.us
+korta.nu
+kqon.com
+kr1n.ru
+krunchd.com
+krz.ch
+ktzr.us
+l24.cm
+l3ss.me
+l9k.net
+lat.ms
+lavvs.com
+lcut.us
+leeturl.net
+leto.tk
+liip.to
+liltext.com
+lin.io
+link2me.ru
+link.ac
+linkbee.com
+linkbun.ch
+linkcash.biz
+linkde.info
+linkee.com
+link.hhut.ru
+linkl.ru
+link.mail.e.glavbukh-mail.ru
+link.rengo.ru
+link.sendsay.ru
+linkslash.ca
+linkunion.de
+linkx.me
+linkyy.com
+linkzip.net
+lip.tc
+li.ru
+list-manage1.com
+list-manage2.com
+list-manage.com
+little.im
+littleurl.net
+liurl.cn
+livehoster.org
+llinks.net
+ln0.ru
+ln4.me
+lnk.by
+lnk.cm
+lnk.co
+lnkd.in
+lnk.gd
+lnk.in
+lnk.ly
+lnk.ms
+lnks.it
+lnk.sk
+lnkstts.com
+lnkurl.com
+ln-s.net
+ln-s.ru
+loh.ru
+loo.gl
+lovebyt.es
+low.cc
+l.pr
+lr.tc
+lru.jp
+lrwk.com
+ltos.ru
+lt.tl
+lul.es
+lurl.no
+lx2.net
+ly9.net
+m4u.in
+m7a.org
+macte.ch
+mail.rambler.ru
+mandrillapp.com
+mash.to
+mee.la
+merky.de
+metamark.net
+micurl.com
+migre.me
+miliuner.com
+miniurl.com
+miniurl.net.ru
+miniurl.pl
+minu.me
+minurl.fr
+minyurl.net
+minyurl.org
+mislead.in
+miud.in
+mixe.me
+mke.me
+mlcampaignru.com
+mlsendru.com
+mmt.su
+mobotix-news.com
+mo.by
+moby.to
+mockurl.com
+moourl.com
+mp77.com
+mrte.ch
+mtp.pl
+mty.in
+mug.gs
+murl.kz
+mvp.im
+mylink4u.info
+mylink.to
+myloc.me
+myooo.info
+mypaqe.com
+mypl.us
+mytinyurl.net
+myurl.in
+myurl.si
+myxx.me
+mzan.si
+n3n.in
+n3r.ru
+nbc.co
+nblo.gs
+nbold.com
+ne1.net
+netgod.tk
+neuf.tk
+newhotlink.com
+nexturl.ru
+nicesharing.com
+nik.im
+niurl.com
+nl.cr
+nn.nf
+no1.in
+no.io
+nonameno.com
+normalurl.com
+notlong.com
+not.my
+now.am
+n.pr
+nsfw.in
+nutshellurl.com
+nxy.in
+nyti.ms
+oc1.us
+oeeq.com
+oiurl.com
+o.ly
+omf.gd
+om.ly
+omoikane.net
+on.cnn.com
+on.mktw.net
+oogyah.com
+oork.com
+opurl.us
+orbita.co.il
+orz.se
+ourgplus.at
+out.houseofgaga.ru
+ovr.me
+ow.ly
+o-x.fr
+p1.fr
+pathto.net
+pb8.ru
+pburl.com
+pcw.ro
+pduda.mobi
+pechkincensor.ru
+pechkinspy.ru
+peeep.us
+peekurl.com
+pendek.in
+penting.web.id
+pfat.de
+pho.se
+phpm.ru
+php-ru.info
+pi90.com
+picourl.ru
+piks.nl
+ping.fm
+pli.gs
+plink.es
+plo.cc
+ploshadka.ru
+plugin.name
+plusphp.com
+p.ly
+pnt.me
+pobierz-film.tk
+politi.co
+ponyurl.com
+poo.pr
+post.ly
+pot.vg
+pp.gg
+ppt.cc
+pra.im
+privacy-surf.com
+proext.com
+professionali.ru
+profile.to
+prourl.de
+ptiturl.com
+p.tl
+pub.vitrue.com
+punyurl.com
+purl.org
+pvh.me
+pw.ly
+pxlz.org
+py6.ru
+pygmyurl.com
+pysper.com
+q32.ru
+qick.ws
+qid.in
+qkr.cc
+qlnk.net
+qlql.ru
+qnh.pl
+qr.cx
+qr.ee
+qrf.in
+qr.net
+qru.ru
+qte.me
+qtwk.com
+quik.in
+qurl.com
+qu.tc
+qz.bz
+rb6.me
+r.delphicomponent.ru
+rdrct.us
+rdr.to
+read.bi
+readthis.ca
+reallytinyurl.com
+redir.ec
+redirectingat.com
+redirects.ca
+redirect.subscribe.ru
+redire.ru
+redirx.com
+reduc.in
+referer.us
+retweet.cc
+retwt.me
+reurl.org
+rhwm.eu
+rickroll.it
+r.im
+ri.ms
+riz.gd
+rlu.ru
+rmse.ru
+rnd.ru
+romb.su
+r-ss.de
+rt.nu
+rubyurl.com
+ru.ly
+rurl.org
+rurl.ru
+rurls.ru
+rustech.org
+rww.tw
+s0bu.ru
+s0e.ru
+s4c.in
+s7y.us
+s8.hk
+safe.mn
+saf.li
+sami.24.gg
+sayabit.com
+s.coop
+scurtare-url.hi2.ro
+sdut.us
+securityexpert.ru
+securl.ru
+sendgrid.net
+sendit.in
+sendsay.ru
+sendurl.info
+sg4d.com
+sg-url.tk
+shadyurl.com
+share.flocktory.com
+shar.es
+shink.de
+shiturl.com
+shli.de
+shorl.com
+shortb.net
+shorten.im
+shorten.ws
+short.ie
+short.im
+shortlinks.co.uk
+short-me.com
+shortner.com
+shortn.me
+short.nr
+short.su
+short.to
+shorturl.asia
+shorturl.com
+short-url.co.uk
+shorturls.co.uk
+shortz.me
+shout.to
+show.my
+shrinkee.com
+shrinkify.com
+shrinkr.com
+shrten.com
+shrt.fr
+shrtl.com
+shrt.st
+shrunkin.com
+shx.in
+simplesite.com
+simurl.com
+sitefwd.com
+sk9.pl
+slate.me
+sli.su
+slki.ru
+sl.to
+smallr.com
+smallurl.in
+smallurl.ru
+smalur.com
+s-m.co
+smsh.me
+smurl.ca
+smurl.name
+sn9.ru
+sn.im
+snipr.com
+snipurl.com
+snurl.com
+softlinemail.ru
+sokrati.ru
+somexyz.com
+song.ly
+sorturl.net
+so.vg
+sp2.ro
+spedr.com
+speed-tester.info
+sq6.ru
+srclick.ru
+srcom.net
+srnk.net
+srs.li
+srtn.me
+starts.com
+starturl.com
+stat-pulse.com
+stnx.at
+stump.ws
+su.ly
+su.pr
+surl.co.uk
+surl.hu
+surl.me
+surs.nl
+susurl.com
+swturl.com
+t1ny.net
+ta.gd
+ta.gg
+tagiturl.com
+taourl.com
+tbd.ly
+t.cn
+t.co
+tcrn.ch
+techto.us
+tez.co
+tgl.net
+tgr.me
+tgr.ph
+th8.us
+thexyz.org
+thinfi.com
+th.ly
+thnlnk.com
+thurl.in
+tie.ly
+tighturl.com
+tin.cc
+tiniuri.com
+tiny9.com
+tinyarro.ws
+tiny.by
+tiny.cc
+tinyit.cc
+tinylink.ca
+tinylink.in
+tiny.ly
+tiny.pl
+tiny.ps
+tinyuri.ca
+tinyurl.com
+tinyurl.ru
+tki.me
+tldr.in
+tl.gd
+t.lh.com
+tlim.ru
+tllg.net
+tmi.me
+tnij.org
+tny.com
+tny.tc
+to8.cc
+togoto.us
+to.je
+to.ly
+toma.ai
+tos.co
+totc.us
+tourl.fr
+toysr.us
+tozm.com
+tpm.ly
+tra.kz
+trg.li
+trii.us
+tr.im
+trimurl.me
+trunc.it
+trusteml.com
+tty.su
+tuckinfo.com
+tux-pla.net
+tvsl.eu
+tweet.ms
+tweez.me
+twhub.com
+twirl.at
+twitclicks.com
+twitter.com
+twitterurl.net
+twiturl.de
+twiu.ru
+twurl.cc
+twurl.nl
+tyny.me
+u2s.in
+u76.org
+ub0.cc
+u.info-mg.ru
+ukril.ru
+ulk.me
+ulmart.ru
+ulo.me
+ulu.lu
+u.mavrev.com
+umenytt.se
+unfake.it
+u.nu
+updating.me
+ur1.ca
+ur3.us
+ural-tender.ru
+url2.ru
+url360.me
+url3.ru
+url4.eu
+url4.ru
+url4t.com
+url5.ru
+url66.com
+urla.ru
+url.az
+url.b24.am
+urlbit.us
+urlborg.com
+urlbrief.com
+urlcantik.com
+urlclick.ru
+url.cn
+urlcorta.es
+url.co.uk
+urlcover.com
+urlcut.com
+urldefender.com
+urldepo.ru
+url.dflix.net
+urlel.com
+urlenco.de
+urle.us
+url.g4team.com
+urlgator.com
+urlgeo.me
+urlg.in
+url-go.com
+urlgo.ru
+url.ie
+urlin.it
+urlink.eu
+urli.nl
+urlite.de
+url.lotpatrol.com
+urlmint.com
+url.mk.ua
+urlms.com
+urloid.com
+urloo.com
+urlot.com
+urlredo.com
+urlscott.com
+urls.co.za
+url.shinri.biz
+urlshorteningservicefortwitter.com
+urlshort.me
+urls.im
+urlsim.com
+urlsnip.com
+urlsp.in
+urlsqueeze.com
+urls.vg
+urltwit.com
+urlu.ms
+urlus.ru
+url.vsofte.ru
+urlx.ie
+urlxs.fr
+ur.ly
+url.yanclex.com
+urlz.at
+urlzen.com
+url-zip.com
+urlz.ro
+usat.ly
+use.my
+u.to
+uud.in
+uuu.su
+uww.me
+uyurl.com
+vani.sh
+vash-repetitor.ru
+vb.ly
+vc8.net
+vgn.am
+view.my
+vk.cc
+vl.am
+vll.me
+vog.me
+vovka.com
+vst.tv
+w3t.org
+w55.de
+wa.la
+wapo.st
+wapurl.co.uk
+warble.co
+webformyself.com
+weblist.kharkov.ua
+weburl.me
+weeclix.com
+wez.su
+whspr.it
+widg.me
+wik.ro
+wipi.es
+wlatcy-moch.tk
+wlink.me.uk
+wmturls.com
+wom.im
+wowurl.com
+wp.me
+wp.nu
+wurl.in
+wurl.us
+wuw.su
+ww.tl
+www.clickmeeting.com
+www.grandstreamnetworks.ru
+wz.ae
+x2t.com
+xaa.su
+xav.cc
+xcqv.com
+xcs.me
+xd5.net
+xdvn.net
+xew.co
+xops.fr
+xp.cm
+xr.com
+xrl.in
+xrls.tk
+xrl.us
+x-url.com
+xurl.es
+xurl.jp
+x.vu
+xvx.su
+xw6.com
+xxsurl.de
+xxw.me
+y.ahoo.it
+yatuc.com
+ydn.ru
+ye.pe
+yep.it
+yfrog.com
+yhoo.it
+yi.pe
+yiyd.com
+ysu.me
+yuarel.com
+yurl.in
+y-url.ru
+yurl.ru
+z0p.de
+z2z.ca
+zapt.in
+zazi.me
+zcom.us
+zebratelecom.ru
+zeep.in
+zi.ma
+zi.mu
+zio.in
+zipmyurl.com
+zolp.net
+zrps.info
+zti.me
+zud.me
+zurl.ws
+zxc9.com
+zzang.kr
+zz.gd
diff --git a/conf/maps.d/spf_dkim_whitelist.inc b/conf/maps.d/spf_dkim_whitelist.inc
new file mode 100644
index 0000000..fe0ddbb
--- /dev/null
+++ b/conf/maps.d/spf_dkim_whitelist.inc
@@ -0,0 +1,233 @@
+# Domains with valid SPF and DKIM
+
+4chan.org
+6pm.com
+about.com
+addthis.com
+adf.ly
+adobe.com
+adp.com
+adschemist.com
+airbnb.com
+airtel.in
+alibaba.com
+aliexpress.com 2.0
+alipay.com 2.0
+allrecipes.com
+amazon.ca
+amazon.cn
+amazon.co.jp
+amazon.com
+amazon.co.uk
+amazon.de
+amazon.es
+amazon.fr
+amazon.in
+amazon.it
+amazon.ru
+americanexpress.com
+ancestry.com
+android.com
+apple.com
+asana.com
+att.com
+autohome.com.cn
+avg.com
+aweber.com
+badoo.com
+bankofamerica.com
+basecamp.com
+battle.net
+bet365.com
+biglobe.ne.jp
+bitly.com
+bleacherreport.com
+blogger.com
+bloomberg.com
+booking.com
+box.com
+bt.com
+capitalone.com
+cdiscount.com
+change.org
+chase.com
+cisco.com
+citi.com
+constantcontact.com
+costco.com
+craigslist.org
+custhelp.com
+dell.com
+delta.com
+diply.com
+discovercard.com
+disqus.com
+dropbox.com
+drweb.com
+ebay.ca
+ebay.com
+ebay.com.au
+ebay.co.uk
+ebay.de
+ebay.fr
+ebay.in
+ebay.it
+ebay.ru
+etsy.com
+evernote.com
+expedia.com
+facebook.com
+fedex.com
+fidelity.com
+fishki.net
+flickr.com
+flirchi.com
+force.com
+freepik.com
+gap.com
+gawker.com
+github.com 2.0
+gizmodo.com
+godaddy.com
+googleadservices.com
+googleusercontent.com
+groupon.com
+hdfcbank.com
+hgtv.com
+hh.ru
+hm.com
+houzz.com
+hubspot.com
+icicibank.com
+icloud.com
+ign.com
+imgur.com
+immobilienscout24.de
+indeed.com
+indiatimes.com
+infusionsoft.com
+instagram.com
+intel.com
+irctc.co.in
+kayak.com
+kickstarter.com
+kijiji.ca
+kotaku.com
+letsencrypt.org
+libero.it
+lifehacker.com
+likes.com
+linkedin.com
+linux.com
+list-manage.com
+mackeeper.com
+mailchimp.com
+mashable.com
+match.com
+mercadolibre.com.ar
+mercadolivre.com.br
+messenger.com
+microsoft.com
+microsoftonline.com
+moikrug.ru
+mts.ru
+neobux.com
+netflix.com
+newegg.com
+nhk.or.jp
+nifty.com
+nikkeibp.co.jp
+nyaa.se
+nytimes.com
+odnoklassniki.ru
+ok.ru
+olx.ua
+overstock.com
+ozon.ru
+ozon.travel
+pandora.com
+paypal.ca
+paypal.cn
+paypal.com
+paypal.com
+paypal.co.uk
+paypal.de
+paypal.es
+paypal.fr
+paypal.it
+paypal.ru
+paytm.com
+pch.com
+pinterest.com
+porn.com
+priceline.com
+qq.com
+quora.com
+rakuten.co.jp
+reddit.com
+researchgate.net
+salesforce.com
+sciencedirect.com
+shopify.com
+slack.com
+slideshare.net
+so-net.ne.jp
+southwest.com
+spotify.com
+springer.com
+squarespace.com
+stalker.com
+steampowered.com
+stumbleupon.com
+surveymonkey.com
+swagbucks.com
+taboola.com
+taleo.net
+taobao.com
+target.com
+taringa.net
+taxi.yandex.ru
+tele2.ru
+thekitchn.com
+tokopedia.com
+trello.com
+tribunnews.com
+trulia.com
+tumblr.com
+twitter.com
+ultimate-guitar.com
+ups.com
+usaa.com
+usbank.com
+usps.com
+verizon.com
+verizonwireless.com
+vimeo.com
+vine.co
+vk.com
+vmware.com
+vtb24.ru
+wahoofitness.com
+walmart.com
+wav.tv
+wellsfargo.com
+whatsapp.com
+wikia.com
+wikimedia.org
+wikipedia.org
+wildberries.ru
+wix.com
+wordpress.com
+wordpress.org
+wp.com
+xuite.net
+xvideos.com
+yelp.com
+youtube.com
+yts.to
+zappos.com
+zendesk.com
+zippyshare.com
+zomato.com
+zulily.com
+zwift.com
diff --git a/conf/maps.d/surbl-whitelist.inc b/conf/maps.d/surbl-whitelist.inc
new file mode 100644
index 0000000..479c929
--- /dev/null
+++ b/conf/maps.d/surbl-whitelist.inc
@@ -0,0 +1,830 @@
+126.com
+163.com
+1gost.info
+1stnationalbank.com
+2o7.net
+365online.com
+4at1.com
+53.com
+5iantlavalamp.com
+abl.com.pk
+about.com
+accessbankplc.com
+adelphia.net
+adib.ae
+adobe.com
+agora-inc.com
+agoramedia.com
+aibgb.co.uk
+aib.ie
+airdriesavingsbank.com
+akamai.net
+akamaitech.net
+aldermore.co.uk
+alexa.com
+alliancebank.com.my
+alliancefg.com
+alliantcreditunion.com
+alliantcreditunion.org
+allianz.de
+allybank.com
+alterna.ca
+amazon.com
+americanexpress.ch
+americanexpress.com
+anadolubank.nl
+ancestry.com
+anpdm.com
+anz.com
+anz.co.nz
+aol.com
+apache.org
+apple.com
+arbuthnotlatham.co.uk
+arcamax.com
+asb.co.nz
+ask.com
+astrology.com
+atdmt.com
+att.net
+authorize.net
+autorambler.ru
+axisbank.co.in
+axisbank.com
+b2bbank.com
+baaderbank.de
+baidu.com
+baloise.ch
+baml.com
+banamex.com
+bancanetbsc.do
+bancanetsantacruz.com.do
+bancapulia.it
+bancarios.com
+bancastato.ch
+bancatransilvania.ro
+bancobase.com
+bancobic.ao
+bancobic.pt
+bancobpi.pt
+banco.bradesco
+bancobrasil.com.br
+bancochile.cl
+bancochile.com
+bancoestado.cl
+bancofalabella.cl
+bancofalabella.com.co
+bancofalabella.pe
+bancomer.com
+bancopopolare.it
+bancopostaclick.it
+bancoposta.it
+bancosantander.es
+bancovotorantimcartoes.com.br
+bank24.ru
+bankalhabib.com
+bankaustria.at
+bank.barclays.co.uk
+bankbgzbnpparibas.pl
+bankcardservices.co.uk
+bankcomm.com
+bankcoop.ch
+bankiabancapersonal.es
+bankia.com
+bankia.es
+bankinter.com
+bankinter.es
+bankmutual.com
+bankofamerica.com
+bankofcanada.ca
+bankofchina.com
+bankofcyprus.com
+bankofindia.co.nz
+bankofireland.com
+bank-of-ireland.co.uk
+bankofirelanduk.com
+bankofoklahoma.com
+bankofscotland.co.uk
+banksinarmas.com
+bankvonroll.ch
+bankwest.com.au
+banque-casino.fr
+banquepopulaire.fr
+banquescotia.com
+barclaycard.co.uk
+barclaycard.de
+barclaycard.es
+barclays.com
+barclays.co.uk
+barclayspartnerfinance.com
+barclays.sc
+barodanzltd.co.nz
+basler.ch
+bbandt.com
+bbc.co.uk
+bcentral.com
+bci.cl
+bcp.com.pe
+bcv.ch
+bcvs.ch
+bekb.ch
+bellevue.ch
+bellsouth.net
+bendigobank.com.au
+berliner-bank.de
+berliner-sparkasse.de
+bfanet.ao
+bfi0.com
+bgfi.com
+bgfionline.com
+bgzbnpparibas.pl
+billmelater.com
+bing.com
+bkb.ch
+bk.rw
+bks.at
+blkb.ch
+bmocm.com
+bmo.com
+bmogam.com
+bmoharris.com
+bmoharrisprivatebankingonline.com
+bmoinvestorline.com
+bmonesbittburns.com
+bnl.it
+bnpparibas.com
+bnpparibas.fr
+boc.cnnz
+bonuscard.ch
+bpe-gruposantander.com
+bpi.pt
+bpostbank.be
+bradescardonline.com.br
+bradesco.com.br
+bradescoseguranca.com.br
+bridgetrack.com
+bridgewaterbank.ca
+bsibank.com
+btrl.ro
+bt-trade.ro
+businessonline-boi.com
+bzbank.ch
+ca-cib.com
+ca-egypt.com
+cafbank.org
+cafe24.com
+cafonline.org
+caisse-epargne.com
+caisse-epargne.fr
+caixabank.com
+caixa.gov.br
+cajasur.es
+camsonline.com
+canadiandirect.com
+capitalone360.com
+capitalone.com
+capitaloneonline.co.uk
+capitecbank.co.za
+cariparma.it
+carrefour-banque.fr
+cartabcc.it
+cartabccpos.it
+cartasi.it
+ca-suisse.com
+catalunyacaixa.com
+cbg.gm
+cbonline.co.uk
+cembra.ch
+cenbank.org
+centralbank.ae
+charitybank.org
+charter.net
+chase.com
+chebanca.it
+chinatrust.com.tw
+cial.ch
+cibc.com
+cic.ch
+cimbclicks.com.my
+citibank.ae
+citibank.co.in
+citibank.com
+citibank.co.uk
+citibankonline.com
+citibusiness.com
+citicards.com
+citi.com
+citi.co.nz
+citi.eu
+citigroup.com
+citizensbank.ca
+citizensbank.com
+civibank.com
+civibank.it
+cjb.net
+classmates.com
+clickbank.net
+closebrothers.com
+closebrothers.co.uk
+clubsc.ch
+cnet.com
+cnn.com
+colpatria.com
+colpatria.com.co
+comcast.net
+com.com
+commbank.com.au
+commerzbank.com
+commerzbank.de
+com.ne.kr
+coopbank.dk
+co-operativebank.co.uk
+cornerbanca.ch
+cornercard.ch
+cornercard.com
+corner.ch
+corporate-ir.net
+cosycard.ch
+coutts.com
+cox.net
+craigslist.org
+credit-agricole.com
+credit-agricole.fr
+creditagricole.rs
+credit-suisse.com
+cs.com
+css.ch
+ctbcbank.com
+ctfs.com
+custhelp.com
+cwbank.com
+cwbankgroup.com
+cwt.ca
+cybg.com
+danskebankas.lt
+danskebank.com
+danskebank.co.uk
+danskebank.de
+danskebank.dk
+danskebank.ee
+danskebank.fi
+danskebank.ie
+danskebank.no
+datatrans.biz
+datatrans.ch
+daum.net
+db.com
+dbs.com
+dd.se
+debian.org
+dell.com
+demirbank.kg
+denizbank.com
+desjardins.ca
+desjardins.com
+deutschebank.be
+deutschebank.co.nz
+deutsche-bank.de
+diamondbank.com
+dibpak.com
+directnic.com
+directtrack.com
+discovercard.com
+discover.com
+discovery.co.za
+dnbnord.lt
+domain.com
+doubleclick.com
+dovecot.org
+dresdner-bank.de
+dsbbank.sr
+dsbl.org
+duncanlawrie.com
+earthlink.net
+easybank.at
+easylnk.com
+ebay.com
+ebay.co.uk
+ebay.de
+ebayimg.com
+ebaystatic.com
+ecobank.com
+edgesuite.net
+ediets.com
+edwardjones.com
+egroups.com
+e-gulfbank.com
+emode.com
+esunbank.com.tw
+exacttarget.com
+example.com
+example.net
+example.org
+excite.com
+facebook.com
+fedex.com
+fednetbank.com
+fidelity.com
+fidor.de
+finance.com
+finansbank.com.tr
+finasta.lt
+fineco.it
+firstbankcard.com
+firstmerit.com
+firstnational.com
+firstnationalmerchantsolutions.com
+firsttrustbank.co.uk
+flickr.com
+fnbc.ca
+fnb.co.za
+fnb-online.com
+freebsd.org
+free.fr
+friuladria.it
+f-secure.com
+garantibank.eu
+garantibank.nl
+garanti.com.tr
+gazprombank.ch
+gazprombank.ru
+generali.es
+genevoise.ch
+gentoo.org
+geocities.com
+github.com
+gkb.ch
+gmail.com
+gmx.net
+go.com
+godaddy.com
+googleadservices.com
+google.co.in
+google.com
+google.it
+google.ru
+granitbank.hu
+grisoft.com
+gtbank.com
+halifax.co.uk
+hallmark.com
+handelsbanken.se
+harrodsbank.co.uk
+hbl.com
+hblibank.com
+hblibank.com.pk
+hdfcbank.com
+heartland.co.nz
+hellenicbank.com
+hinet.net
+hkbea.com
+hlb.com.kh
+hlb.com.my
+hoaresbank.co.uk
+home.barclays
+hongleongconnect.com.kh
+hongleongconnect.com.vn
+hongleongconnect.my
+hotbar.com
+hotmail.com
+hotpop.com
+hp.com
+hsbc.com
+hsbc.com.ar
+hsbc.com.hk
+hsbc.co.nz
+hsbc.co.uk
+hypovereinsbank.co.uk
+hypovereinsbank.de
+ibm.com
+icbcnz.com
+icicibank.co.in
+icicibank.com
+icicibankprivatebanking.com
+icorner.ch
+icscards.de
+icscards.nl
+incredimail.com
+ing.be
+ing.com
+ing-diba.de
+ingdirect.ca
+ing.lu
+ing.nl
+ingvysyabank.com
+interac.ca
+investorplace.com
+iobnet.co.in
+isbank.com.tr
+isbank.de
+isbank.ge
+isbank.iq
+isbankkosova.com
+isc.org
+itau.com.br
+ivillage.com
+joingevalia.com
+jpmchase.com
+jpmorgan.com
+jsafrasarasin.com
+julianhodgebank.com
+juliusbaer.com
+juno.com
+jyskebank.dk
+kantonalbank.ch
+kernel.org
+key.com
+kiwibank.co.nz
+kotak.com
+kredytbank.pl
+kreissparkasse-schwalm-eder.de
+ksklb.de
+kutxabank.es
+laboralkutxa.com
+lacaixa.cat
+lacaixa.es
+laurentianbank.ca
+lbb.de
+lcl.com
+lcl.fr
+li.ru
+list.ru
+lists.isc.org
+lists.roundcube.net
+liveinternet.ru
+livejournal.com
+lloydsbank.com
+lloydsbankcommercial.com
+lloydsbankinggroup.com
+lloydstsb.ch
+lloydstsb.co.uk
+lombardodier.com
+loydsbank.com
+lycos.com
+m7z.net
+mac.com
+macromedia.com
+maerki-baumann.ch
+mail.com
+mail.ru
+mailscanner.info
+mandtbank.com
+manulifebank.ca
+manulifebankselect.ca
+manulife.com
+manulifeone.ca
+marketwatch.com
+mashreqbank.com
+mastercard.com
+maybank2u.com
+maybank2u.com.my
+mcafee.com
+mchsi.com
+mdmbank.com
+mechanicsbank.com
+medbank.lt
+messagelabs.com
+metrobankdirect.com
+metrobankonline.co.uk
+microsoft.com
+migbank.com
+migrosbank.ch
+military.com
+mindspring.com
+mit.edu
+mizuhobank.co.jp
+mmwarburg.lu
+monster.com
+montepio.pt
+morganstanley.com
+mozilla.com
+mps.it
+ms.com
+msn.com
+mufg.jp
+myonlineresourcecenter.com
+myonlineservices.ch
+myspace.com
+nate.com
+nationalesuisse.ch
+nationwide-communications.co.uk
+nationwide.co.uk
+nationwide-service.co.uk
+natwest.com
+navyfederal.org
+nbc.ca
+netflix.com
+netscape.com
+netscape.net
+netzero.net
+newyorkfed.org
+nibl.com.np
+nod32.com
+nordea.fi
+nordea.lt
+nordfynsbank.dk
+norisbank.de
+norman.com
+notenstein.ch
+nuvisionfederal.com
+nytimes.com
+oceanbank.com
+onlinesbi.com
+openoffice.org
+openxmlformats.org
+optonline.net
+orchardbank.com
+osdn.com
+ostsaechsische-sparkasse-dresden.de
+overstock.com
+pacbell.net
+pandasoftware.com
+passport.com
+paylife.at
+paypal.be
+paypal-brasil.com.br
+paypal.ca
+paypal.ch
+paypal.co.il
+paypal.com
+paypal.com.au
+paypal.com.br
+paypal-communication.com
+paypal-community.com
+paypal.com.mx
+paypal.com.pt
+paypal.co.uk
+paypal-customerfeedback.com
+paypal.de
+paypal-deutschland.de
+paypal.dk
+paypal.es
+paypal-exchanges.com
+paypal.fr
+paypal.it
+paypal-marketing.co.uk
+paypal-marketing.pl
+paypal.net
+paypal.nl
+paypal.no
+paypal-notify.com
+paypal-now.com
+paypalobjects.com
+paypal-opwaarderen.nl
+paypal-pages.com
+paypal.pt
+paypal.ru
+paypal.se
+paypal-search.com
+paypal-shopping.co.uk
+paypal-techsupport.com
+pbebank.com
+pcfinancial.ca
+peoplepc.com
+permanenttsb.ie
+plaxo.com
+pnc.com
+popolarevicenza.it
+postbank.de
+postepay.it
+postfinancearena.ch
+postfinance.ch
+postfinance.info
+price.ru
+prodigy.net
+publicislamicbank.com.my
+rabobank.com
+rabobank.co.nz
+rabobank.nl
+radaruol.com.br
+rahnbodmer.ch
+raiffeisenbank.rs
+raiffeisen.ch
+raiffeisen.hu
+raiffeisen.li
+raiffeisen.ru
+rambler-co.ru
+rambler.ru
+raphaelsbank.com
+rbc.com
+rbcroyalbank.com
+rbs.co.uk
+rbssecure.co.uk
+rbsworldpay.com
+rcb.at rcb.at
+real.com
+recordbank.be
+redhat.com
+rediff.com
+regiobank.nl
+regions.com
+regionsnet.com
+renasantbank.com
+rhbgroup.com
+rogersbank.com
+rogers.com
+rothschildbank.com
+rothschild.com
+royalbank.com
+rr.com
+sagepay.com
+sagepay.co.uk
+sainsburysbank.co.uk
+samba.com
+santander.cl
+santander.com
+santander.com.br
+santander.com.mx
+santandercorretora.com.br
+santander.co.uk
+santanderesfera.com.br
+santandersantiago.cl
+sarasin.ch
+sbcglobal.net
+sberbank.ch
+sbs.net.nz
+sc.com
+schoellerbank.at
+scotiabank.ca
+scotiabank.com
+scotiamocatta.com
+scotiaonline.com
+s.de
+sec.gov
+securetrustbank.com
+service-sparkasse.de
+serviciobancomer.com
+sf.net
+shawbrook.co.uk
+shaw.ca
+shkb.ch
+shockwave.com
+six-group.com
+six-payment-services.com
+skrill.com
+sls-direkt.de
+smithbarney.com
+snb.ch snb.ch
+snsbank.nl
+societegenerale.fr
+sourceforge.net
+spamcop.net
+sparda-a.de
+sparda-bank-hamburg.de
+sparda-b.de
+sparda-bw.de
+sparda-h.de
+sparda-hessen.de
+sparda-m.de
+sparda-ms.de
+sparda-n.de
+sparda-ostbayern.de
+sparda-sw.de
+sparda-verband.de
+sparda-west.de
+sparkasse.at
+sparkasse-bank-malta.com
+sparkasse-bielefeld.de
+sparkasseblog.de
+sparkasse-bochum.de
+sparkasse.ch
+sparkasse.de
+sparkasse-gera-greiz.de
+sparkasse-hamm.de
+sparkasse-heidelberg.de
+sparkasse-ingolstadt.de
+sparkasse-mittelthueringen.de
+speedera.net
+sportsline.com
+standardbank.com
+standardbank.co.za
+standardchartered.com.gh
+standardchartered.com.my
+subscribe.ru
+sun.com
+suncorpbank.com.au
+suntrust.com
+svn.apache.org
+swedbank.com
+swedbank.ee
+swedbank.lt
+swedbank.lu
+swedbank.se
+swisscanto.ch
+swisscaution.ch
+swissquote.ch
+sydbank.dk
+sympatico.ca
+taggedmail.com
+tails.nl
+tangerine.ca
+tcb-bank.com.tw
+tdbank.com
+tdcommercialbanking.com
+telus.net
+terra.com.br
+tescobank.com
+ticketmaster.com
+tinyurl.com
+tiscali.co.uk
+tns-counter.ru
+tom.com
+tone.co.nz
+t-online.de
+top4top.ru
+tsbbank.co.nz
+tsb.co.nz
+tsb.co.uk
+tumblr.com
+tux.org
+twitter.com
+ubibanca.com
+ubs.com
+ulsterbankanytimebanking.co.uk
+ulsterbank.co.uk
+unibancoconnect.pt
+unibanco.pt
+unicreditbank.lt
+unicredit.eu
+unicreditgroup.eu
+unicredit.it
+unionbankcameroon.com
+unionbank.com
+unity.co.uk
+uob.com.sg
+uobgroup.com
+uol.com.br
+ups.com
+usbank.com
+valianttrust.com
+vaudoise.ch
+venetobanca.it
+venetobanka.al
+verizon.net
+versabank.com
+videobank.it
+virginmoney.com
+visa.com.ar
+visa.com.br
+visaeurope.ch
+visaeurope.com
+viseca.ch
+vistaprint.com
+vistaprint.dk
+volksbank.de
+volkswagenbank.de
+vpbank.com
+vr.de
+vwbank.de
+w3.org
+wachovia.com
+walmart.com
+wamu.com
+wanadoo.fr
+washingtonpost.com
+weatherbug.com
+weatherbys.co.uk
+web.de
+webshots.com
+webtv.net
+wegelin.ch
+wellsfargo.com
+wellsfargoemail.com
+westernunion.ca
+westernunion.com
+westernunion.fr
+westernunion.se
+westpac.com.au
+westpac.co.nz
+wir.ch
+wordpress.com
+worldbank.org
+worldpay.com
+wsj.com
+wvb.de
+xmlsoap.org
+yacht.nl
+yahoo.ca
+yahoo.co.jp
+yahoo.co.kr
+yahoo.com
+yahoo.com.br
+yahoo.co.uk
+yahoogroups.com
+yandex.net
+yandex.ru
+ybonline.co.uk
+yimg.com
+yopi.de
+yorkshirebank.co.uk
+yourbankcard.com
+yoursite.com
+youtube.com
+zagbank.ca
+zdnet.com
+zenithbank.com
+zkb.ch
+zugerkb.ch
+# list-manage1.com # grey
+# list-manage2.com # grey
+# list-manage.com # grey
diff --git a/conf/metrics.conf b/conf/metrics.conf
new file mode 100644
index 0000000..9c4358f
--- /dev/null
+++ b/conf/metrics.conf
@@ -0,0 +1,24 @@
+# Metrics settings
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+# DEPRECATION WARNING!!
+# This file is deprecated since 1.7
+# Please use actions.conf and groups.conf files instead
+metric {
+ name = "default";
+ .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/metrics.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/metrics.conf"
+}
diff --git a/conf/modules.conf b/conf/modules.conf
new file mode 100644
index 0000000..b37da06
--- /dev/null
+++ b/conf/modules.conf
@@ -0,0 +1,17 @@
+# A common rspamd configuration file (should never ever be changed)
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+.include(glob=true) "${CONFDIR}/modules.d/*.conf" \ No newline at end of file
diff --git a/conf/modules.d/antivirus.conf b/conf/modules.d/antivirus.conf
new file mode 100644
index 0000000..b172288
--- /dev/null
+++ b/conf/modules.d/antivirus.conf
@@ -0,0 +1,58 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/antivirus.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/antivirus.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/antivirus.html
+
+antivirus {
+ # multiple scanners could be checked, for each we create a configuration block with an arbitrary name
+ #clamav {
+ # If set force this action if any virus is found (default unset: no action is forced)
+ # action = "reject";
+ # message = '${SCANNER}: virus found: "${VIRUS}"';
+ # Scan mime_parts separately - otherwise the complete mail will be transferred to AV Scanner
+ #scan_mime_parts = true;
+ # Scanning Text is suitable for some av scanner databases (e.g. Sanesecurity)
+ #scan_text_mime = false;
+ #scan_image_mime = false;
+ # If `max_size` is set, messages > n bytes in size are not scanned
+ #max_size = 20000000;
+ # symbol to add (add it to metric if you want non-zero weight)
+ #symbol = "CLAM_VIRUS";
+ # type of scanner: "clamav", "fprot", "sophos" or "savapi"
+ #type = "clamav";
+ # For "savapi" you must also specify the following variable
+ #product_id = 12345;
+ # You can enable logging for clean messages
+ #log_clean = true;
+ # servers to query (if port is unspecified, scanner-specific default is used)
+ # can be specified multiple times to pool servers
+ # can be set to a path to a unix socket
+ # Enable this in local.d/antivirus.conf
+ #servers = "127.0.0.1:3310";
+ # if `patterns` is specified virus name will be matched against provided regexes and the related
+ # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+ #patterns {
+ # symbol_name = "pattern";
+ # JUST_EICAR = '^Eicar-Test-Signature$';
+ #}
+ #patterns_fail {
+ # symbol_name = "pattern";
+ #CLAM_PROTOCOL_ERROR = '^unhandled response';
+ #}
+ # `whitelist` points to a map of signature names. Hits on these signatures are ignored.
+ #whitelist = "/etc/rspamd/antivirus.wl";
+ #}
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/antivirus.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/antivirus.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/antivirus.conf"
+}
diff --git a/conf/modules.d/arc.conf b/conf/modules.d/arc.conf
new file mode 100644
index 0000000..f26dad8
--- /dev/null
+++ b/conf/modules.d/arc.conf
@@ -0,0 +1,72 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/arc.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/arc.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/arc.html
+
+
+# To configure this module, please also check the following document:
+# https://rspamd.com/doc/tutorials/scanning_outbound.html and
+# https://rspamd.com/doc/modules/arc.html
+
+# To enable this module define the following attributes:
+# path = "${DBDIR}/arc/$domain.$selector.key";
+# OR
+# domain { ... }, if you use per-domain conf
+# OR
+# set `use_redis=true;` and define redis servers
+
+arc {
+ # If false, messages with empty envelope from are not signed
+ allow_envfrom_empty = true;
+ # If true, envelope/header domain mismatch is ignored
+ allow_hdrfrom_mismatch = true;
+ # If true, multiple from headers are allowed (but only first is used)
+ allow_hdrfrom_multiple = false;
+ # If true, username does not need to contain matching domain
+ allow_username_mismatch = false;
+ # Default path to key, can include '$domain' and '$selector' variables
+ #path = "${DBDIR}/arc/$domain.$selector.key";
+ # Default selector to use
+ selector = "arc";
+ # If false, messages from authenticated users are not selected for signing
+ sign_authenticated = false;
+ # If false, inbound messages are not selected for signing
+ sign_inbound = true;
+ # If false, messages from local networks are not selected for signing
+ sign_local = false;
+ # Symbol to add when message is signed
+ sign_symbol = "ARC_SIGNED";
+ # Whether to fallback to global config
+ try_fallback = true;
+ # Domain to use for ARC signing: can be "header", "envelope" or "recipient"
+ use_domain = "recipient";
+ # Whether to normalise domains to eSLD
+ use_esld = true;
+ # Whether to get keys from Redis
+ use_redis = false;
+ # Hash for ARC keys in Redis
+ key_prefix = "ARC_KEYS";
+
+ # Domain specific settings
+ #domain {
+ # example.com {
+ # # Private key path
+ # path = "${DBDIR}/arc/example.key";
+ # # Selector
+ # selector = "ds";
+ # }
+ #}
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/arc.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/arc.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/arc.conf"
+}
diff --git a/conf/modules.d/asn.conf b/conf/modules.d/asn.conf
new file mode 100644
index 0000000..61efd52
--- /dev/null
+++ b/conf/modules.d/asn.conf
@@ -0,0 +1,29 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/asn.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/asn.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/asn.html
+
+asn {
+ # Provider: just "rspamd" for now
+ provider_type = "rspamd";
+ # Provider-specific configuration
+ provider_info {
+ ip4 = "asn.rspamd.com";
+ ip6 = "asn6.rspamd.com";
+ }
+ # If defined, insert symbol with lookup results
+ # symbol = "ASN";
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/asn.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/asn.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/asn.conf"
+}
diff --git a/conf/modules.d/aws_s3.conf b/conf/modules.d/aws_s3.conf
new file mode 100644
index 0000000..5d78148
--- /dev/null
+++ b/conf/modules.d/aws_s3.conf
@@ -0,0 +1,27 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/aws_s3.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/aws_s3.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/aws_s3.html
+
+aws_s3 {
+ # Required attributes
+ #s3_bucket = 'xxx';
+ s3_region = 'us-east-1';
+ s3_host = 's3.amazonaws.com';
+ #s3_secret_key = 'xxx';
+ #s3_key_id = 'xxx';
+ # Enable in local.d/aws_s3.conf
+ enabled = false;
+ .include(try=true,priority=5) "${DBDIR}/dynamic/aws_s3.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/aws_s3.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/aws_s3.conf"
+}
diff --git a/conf/modules.d/bimi.conf b/conf/modules.d/bimi.conf
new file mode 100644
index 0000000..ed7be6b
--- /dev/null
+++ b/conf/modules.d/bimi.conf
@@ -0,0 +1,30 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/bimi.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/bimi.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Currently there is no documentation for this module. When it is written it will
+# be available at https://rspamd.com/doc/modules/bimi.html
+
+bimi {
+ # Required attributes
+ #helper_url = "http://127.0.0.1:3030",
+ helper_timeout = 5s;
+ helper_sync = true;
+ vmc_only = true;
+ redis_prefix = 'rs_bimi';
+ redis_min_expiry = 24h;
+
+ # Enable in local.d/bimi.conf
+ enabled = false;
+ .include(try=true,priority=5) "${DBDIR}/dynamic/bimi.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/bimi.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/bimi.conf"
+}
diff --git a/conf/modules.d/chartable.conf b/conf/modules.d/chartable.conf
new file mode 100644
index 0000000..849c8bd
--- /dev/null
+++ b/conf/modules.d/chartable.conf
@@ -0,0 +1,21 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/chartable.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/chartable.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/chartable.html
+
+chartable {
+ threshold = 0.300000;
+ symbol = "R_MIXED_CHARSET";
+ .include(try=true,priority=5) "${DBDIR}/dynamic/chartable.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/chartable.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/chartable.conf"
+}
diff --git a/conf/modules.d/clickhouse.conf b/conf/modules.d/clickhouse.conf
new file mode 100644
index 0000000..5edf710
--- /dev/null
+++ b/conf/modules.d/clickhouse.conf
@@ -0,0 +1,59 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/clickhouse.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/clickhouse.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/clickhouse.html
+
+clickhouse {
+ # Push update when 1000 records are collected (1000 if unset)
+ limit = 1000;
+ # IP:port of Clickhouse server
+ # server = "localhost:8123";
+ # Timeout to wait for response (5 seconds if unset)
+ timeout = 5;
+ # How many bits of sending IP to mask in logs for IPv4 (19 if unset)
+ ipmask = 19;
+ # How many bits of sending IP to mask in logs for IPv6 (48 if unset)
+ ipmask6 = 48;
+ # Record URL paths? (default false)
+ full_urls = false;
+ # This parameter points to a map of domain names
+ # If a message has a domain in this map in From: header and DKIM signature,
+ # record general metadata in a table named after the domain
+ #from_tables = "/etc/rspamd/clickhouse_from.map";
+ # These are symbols of other checks in Rspamd
+ # Set these if you use non-default symbol names (unlikely)
+ #bayes_spam_symbols = ["BAYES_SPAM"];
+ #bayes_ham_symbols = ["BAYES_HAM"];
+ #fann_symbols = ["FANN_SCORE"];
+ #fuzzy_symbols = ["FUZZY_DENIED"];
+ #whitelist_symbols = ["WHITELIST_DKIM", "WHITELIST_SPF_DKIM", "WHITELIST_DMARC"];
+ #dkim_allow_symbols = ["R_DKIM_ALLOW"];
+ #dkim_reject_symbols = ["R_DKIM_REJECT"];
+ #dmarc_allow_symbols = ["DMARC_POLICY_ALLOW"];
+ #dmarc_reject_symbols = ["DMARC_POLICY_REJECT", "DMARC_POLICY_QUARANTINE"];
+
+ #retention {
+ # # disabled by default
+ # enable = true;
+ # # drop | detach, please refer to ClickHouse docs for details
+ # # http://clickhouse-docs.readthedocs.io/en/latest/query_language/queries.html#manipulations-with-partitions-and-parts
+ # method = "drop";
+ # # how many month the data should be kept in ClickHouse
+ # period_months = 3;
+ # # how often run the cleanup process
+ # run_every = "7d";
+ #}
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/clickhouse.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/clickhouse.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/clickhouse.conf"
+}
diff --git a/conf/modules.d/dcc.conf b/conf/modules.d/dcc.conf
new file mode 100644
index 0000000..22ca600
--- /dev/null
+++ b/conf/modules.d/dcc.conf
@@ -0,0 +1,28 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/dcc.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/dcc.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/dcc.html
+
+dcc {
+
+ enabled = false;
+
+ # Define local socket or TCP servers in upstreams syntax
+ # When sockets and servers are definined - servers is used!
+ socket = "/var/dcc/dccifd"; # Unix socket
+ #servers = "127.0.0.1:10045" # OR TCP upstreams
+ timeout = 2s; # Timeout to wait for checks
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/dcc.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/dcc.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/dcc.conf"
+}
diff --git a/conf/modules.d/dkim.conf b/conf/modules.d/dkim.conf
new file mode 100644
index 0000000..689ecd2
--- /dev/null
+++ b/conf/modules.d/dkim.conf
@@ -0,0 +1,25 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/dkim.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/dkim.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/dkim.html
+
+dkim {
+ dkim_cache_size = 2k;
+ dkim_cache_expire = 1d;
+ time_jitter = 6h;
+ trusted_only = false;
+ skip_multi = false;
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/dkim.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/dkim.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/dkim.conf"
+}
diff --git a/conf/modules.d/dkim_signing.conf b/conf/modules.d/dkim_signing.conf
new file mode 100644
index 0000000..42cb0e2
--- /dev/null
+++ b/conf/modules.d/dkim_signing.conf
@@ -0,0 +1,77 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/dkim_signing.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/dkim_signing.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+
+
+# To configure this module, please also check the following document:
+# https://rspamd.com/doc/tutorials/scanning_outbound.html and
+# https://rspamd.com/doc/modules/dkim_signing.html
+
+# To enable this module define the following attributes:
+# path = "/var/lib/rspamd/dkim/$domain.$selector.key";
+# OR
+# domain { ... }, if you use per-domain conf
+# OR
+# set `use_redis=true;` and define redis servers
+
+dkim_signing {
+ # If false, messages with empty envelope from are not signed
+ allow_envfrom_empty = true;
+ # If true, envelope/header domain mismatch is ignored
+ allow_hdrfrom_mismatch = false;
+ # If true, multiple from headers are allowed (but only first is used)
+ allow_hdrfrom_multiple = false;
+ # If true, username does not need to contain matching domain
+ allow_username_mismatch = false;
+ # Default path to key, can include '$domain' and '$selector' variables
+ #path = "/var/lib/rspamd/dkim/$domain.$selector.key";
+ # Default selector to use
+ selector = "dkim";
+ # If false, messages from authenticated users are not selected for signing
+ sign_authenticated = true;
+ # If false, messages from local networks are not selected for signing
+ sign_local = true;
+ # Symbol to add when message is signed
+ symbol = "DKIM_SIGNED";
+ # Whether to fallback to global config
+ try_fallback = true;
+ # Domain to use for DKIM signing: can be "header" or "envelope"
+ use_domain = "header";
+ # Whether to normalise domains to eSLD
+ use_esld = true;
+ # Whether to get keys from Redis
+ use_redis = false;
+ # Hash for DKIM keys in Redis
+ key_prefix = "DKIM_KEYS";
+
+ # Domain specific settings
+ #domain {
+ # example.com {
+ # selectors [
+ # { # Private key path
+ # path = "/var/lib/rspamd/dkim/example.key";
+ # # Selector
+ # selector = "ds";
+ # },
+ # { # multiple dkim signature
+ # path = "/var/lib/rspamd/dkim/eddsa.key";
+ # selector = "eddsa";
+ # }
+ # ]
+ # }
+ #}
+
+
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/dkim_signing.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/dkim_signing.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/dkim_signing.conf"
+}
diff --git a/conf/modules.d/dmarc.conf b/conf/modules.d/dmarc.conf
new file mode 100644
index 0000000..d487041
--- /dev/null
+++ b/conf/modules.d/dmarc.conf
@@ -0,0 +1,19 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/dmarc.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/dmarc.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/dmarc.html
+
+dmarc {
+ .include(try=true,priority=5) "${DBDIR}/dynamic/dmarc.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/dmarc.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/dmarc.conf"
+}
diff --git a/conf/modules.d/elastic.conf b/conf/modules.d/elastic.conf
new file mode 100644
index 0000000..8056023
--- /dev/null
+++ b/conf/modules.d/elastic.conf
@@ -0,0 +1,35 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/elastic.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/elastic.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/elastic.html
+
+elastic {
+ # Push update when 10 records are collected (10 if unset)
+ limit = 10;
+ # IP:port of Elasticsearch server
+ #server = "localhost:9200";
+ # Timeout to wait for response (5 seconds if unset)
+ timeout = 5;
+ # Elasticsearch template file (json format)
+ #template_file = "${SHAREDIR}/elastic/rspamd_template.json";
+ # Kibana prebuild visualizations and dashboard template (json format)
+ #kibana_file = "${SHAREDIR}/elastic/kibana.json";
+ # Elasticsearch index name pattern
+ index_pattern = "rspamd-%Y.%m.%d";
+ # Dump debug information
+ debug = false;
+ # Import kibana template
+ import_kibana = false;
+ .include(try=true,priority=5) "${DBDIR}/dynamic/elastic.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/elastic.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/elastic.conf"
+}
diff --git a/conf/modules.d/emails.conf b/conf/modules.d/emails.conf
new file mode 100644
index 0000000..19f750f
--- /dev/null
+++ b/conf/modules.d/emails.conf
@@ -0,0 +1,8 @@
+# Deprecated, preserved for compatibility purposes! Use rbl module
+# Module documentation https://rspamd.com/doc/modules/rbl.html
+
+emails {
+ .include(try=true,priority=5) "${DBDIR}/dynamic/emails.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/emails.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/emails.conf"
+}
diff --git a/conf/modules.d/external_relay.conf b/conf/modules.d/external_relay.conf
new file mode 100644
index 0000000..7d52ced
--- /dev/null
+++ b/conf/modules.d/external_relay.conf
@@ -0,0 +1,22 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/external_relay.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/external_relay.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/external_relay.html
+
+external_relay {
+ # This module is default-disabled
+ enabled = false;
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/external_relay.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/external_relay.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/external_relay.conf"
+}
diff --git a/conf/modules.d/external_services.conf b/conf/modules.d/external_services.conf
new file mode 100644
index 0000000..549c7d7
--- /dev/null
+++ b/conf/modules.d/external_services.conf
@@ -0,0 +1,92 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/external_services.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/external_services.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/external_services.html
+
+external_services {
+ oletools {
+ # If set force this action if any virus is found (default unset: no action is forced)
+ # action = "reject";
+ # If set, then rejection message is set to this value (mention single quotes)
+ # If `max_size` is set, messages > n bytes in size are not scanned
+ # max_size = 20000000;
+ # log_clean = true;
+ # servers = "127.0.0.1:10050";
+ # cache_expire = 86400;
+ # scan_mime_parts = true;
+ # extended = false;
+ # if `patterns` is specified virus name will be matched against provided regexes and the related
+ # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+ patterns {
+ # symbol_name = "pattern";
+ }
+ # mime-part regex matching in content-type or filename
+ mime_parts_filter_regex {
+ #GEN1 = "application\/octet-stream";
+ DOC2 = "application\/msword";
+ DOC3 = "application\/vnd\.ms-word.*";
+ XLS = "application\/vnd\.ms-excel.*";
+ PPT = "application\/vnd\.ms-powerpoint.*";
+ GEN2 = "application\/vnd\.openxmlformats-officedocument.*";
+ }
+ # Mime-Part filename extension matching (no regex)
+ mime_parts_filter_ext {
+ doc = "doc";
+ dot = "dot";
+ docx = "docx";
+ dotx = "dotx";
+ docm = "docm";
+ dotm = "dotm";
+ xls = "xls";
+ xlt = "xlt";
+ xla = "xla";
+ xlsx = "xlsx";
+ xltx = "xltx";
+ xlsm = "xlsm";
+ xltm = "xltm";
+ xlam = "xlam";
+ xlsb = "xlsb";
+ ppt = "ppt";
+ pot = "pot";
+ pps = "pps";
+ ppa = "ppa";
+ pptx = "pptx";
+ potx = "potx";
+ ppsx = "ppsx";
+ ppam = "ppam";
+ pptm = "pptm";
+ potm = "potm";
+ ppsm = "ppsm";
+ }
+ # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+ whitelist = "/etc/rspamd/antivirus.wl";
+ }
+ dcc {
+ # If set force this action if any virus is found (default unset: no action is forced)
+ # action = "reject";
+ # If set, then rejection message is set to this value (mention single quotes)
+ # If `max_size` is set, messages > n bytes in size are not scanned
+ max_size = 20000000;
+ #servers = "127.0.0.1:10045";
+ # if `patterns` is specified virus name will be matched against provided regexes and the related
+ # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+ patterns {
+ # symbol_name = "pattern";
+ }
+ # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+ whitelist = "/etc/rspamd/antivirus.wl";
+ }
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/external_services.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/external_services.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/external_services.conf"
+}
diff --git a/conf/modules.d/force_actions.conf b/conf/modules.d/force_actions.conf
new file mode 100644
index 0000000..a753066
--- /dev/null
+++ b/conf/modules.d/force_actions.conf
@@ -0,0 +1,22 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/force_actions.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/force_actions.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/force_actions.html
+
+force_actions {
+
+ # Refer to https://rspamd.com/doc/modules/force_actions.html for information on configuration
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/force_actions.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/force_actions.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/force_actions.conf"
+}
diff --git a/conf/modules.d/forged_recipients.conf b/conf/modules.d/forged_recipients.conf
new file mode 100644
index 0000000..b480bb1
--- /dev/null
+++ b/conf/modules.d/forged_recipients.conf
@@ -0,0 +1,22 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/forged_recipients.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/forged_recipients.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/forged_recipients.html
+
+forged_recipients {
+ symbol_sender = "FORGED_SENDER";
+ symbol_rcpt = "FORGED_RECIPIENTS";
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/forged_recipients.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/forged_recipients.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/forged_recipients.conf"
+}
diff --git a/conf/modules.d/fuzzy_check.conf b/conf/modules.d/fuzzy_check.conf
new file mode 100644
index 0000000..73e280f
--- /dev/null
+++ b/conf/modules.d/fuzzy_check.conf
@@ -0,0 +1,49 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/fuzzy_check.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/fuzzy_check.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/fuzzy_check.html
+
+fuzzy_check {
+ min_bytes = 1k; # Since small parts and small attachments causes too many FP
+ timeout = 2s;
+ retransmits = 1;
+ rule "rspamd.com" {
+ algorithm = "mumhash";
+ servers = "round-robin:fuzzy1.rspamd.com:11335,fuzzy2.rspamd.com:11335";
+ encryption_key = "icy63itbhhni8bq15ntp5n5symuixf73s1kpjh6skaq4e7nx5fiy";
+ symbol = "FUZZY_UNKNOWN";
+ mime_types = ["*"];
+ max_score = 20.0;
+ read_only = yes;
+ skip_unknown = yes;
+ short_text_direct_hash = true; # If less than min_length then use direct hash
+ min_length = 64; # Minimum words count to consider shingles
+ fuzzy_map = {
+ FUZZY_DENIED {
+ max_score = 20.0;
+ flag = 1;
+ }
+ FUZZY_PROB {
+ max_score = 10.0;
+ flag = 2;
+ }
+ FUZZY_WHITE {
+ max_score = 2.0;
+ flag = 3;
+ }
+ }
+ }
+ # Include dynamic conf for the rule
+ .include(try=true,priority=5) "${DBDIR}/dynamic/fuzzy_check.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/fuzzy_check.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/fuzzy_check.conf"
+}
diff --git a/conf/modules.d/greylist.conf b/conf/modules.d/greylist.conf
new file mode 100644
index 0000000..fde9cc0
--- /dev/null
+++ b/conf/modules.d/greylist.conf
@@ -0,0 +1,35 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/greylist.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/greylist.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/greylisting.html
+
+greylist {
+ # Search "example.com" and "mail.example.com" for "mx.out.mail.example.com":
+ whitelist_domains_url = [
+ "$LOCAL_CONFDIR/local.d/greylist-whitelist-domains.inc",
+ "$LOCAL_CONFDIR/local.d/maps.d/greylist-whitelist-domains.inc",
+ ];
+
+ expire = 1d; # 1 day by default
+ timeout = 5min; # 5 minutes by default
+ key_prefix = "rg"; # default hash name
+ max_data_len = 10k; # default data limit to hash
+ message = "Try again later"; # default greylisted message
+ #symbol = "GREYLIST"; # Symbol to insert
+ action = "soft reject"; # default greylisted action
+ ipv4_mask = 19; # Mask bits for ipv4
+ ipv6_mask = 64; # Mask bits for ipv6
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/greylist.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/greylist.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/greylist.conf"
+}
diff --git a/conf/modules.d/hfilter.conf b/conf/modules.d/hfilter.conf
new file mode 100644
index 0000000..36d5f47
--- /dev/null
+++ b/conf/modules.d/hfilter.conf
@@ -0,0 +1,26 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/hfilter.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/hfilter.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/hfilter.html
+
+hfilter {
+ helo_enabled = true;
+ hostname_enabled = true;
+ url_enabled = true;
+ from_enabled = true;
+ rcpt_enabled = true;
+ mid_enabled = false;
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/hfilter.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/hfilter.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/hfilter.conf"
+}
diff --git a/conf/modules.d/history_redis.conf b/conf/modules.d/history_redis.conf
new file mode 100644
index 0000000..0d1c7f6
--- /dev/null
+++ b/conf/modules.d/history_redis.conf
@@ -0,0 +1,25 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/history_redis.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/history_redis.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/history_redis.html
+
+history_redis {
+ #servers = 127.0.0.1:6379; # Redis server to store history
+ key_prefix = "rs_history"; # Default key name
+ nrows = 200; # Default rows limit
+ compress = true; # Use zstd compression when storing data in redis
+ subject_privacy = false; # subject privacy is off
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/history_redis.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/history_redis.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/history_redis.conf"
+}
diff --git a/conf/modules.d/http_headers.conf b/conf/modules.d/http_headers.conf
new file mode 100644
index 0000000..51e5b82
--- /dev/null
+++ b/conf/modules.d/http_headers.conf
@@ -0,0 +1,22 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/http_headers.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/http_headers.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/http_headers.html
+
+http_headers {
+ # This module is default-disabled
+ enabled = false;
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/http_headers.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/http_headers.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/http_headers.conf"
+}
diff --git a/conf/modules.d/known_senders.conf b/conf/modules.d/known_senders.conf
new file mode 100644
index 0000000..80b9239
--- /dev/null
+++ b/conf/modules.d/known_senders.conf
@@ -0,0 +1,31 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/known_senders.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/known_senders.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/known_senders.html
+
+known_senders {
+ # This module is default-disabled
+ enabled = false;
+
+ # Domains to track senders
+ domains = "https://maps.rspamd.com/freemail/free.txt.zst";
+ # Maximum number of elements
+ max_senders = 100000;
+ # Maximum time to live (when not using bloom filters)
+ max_ttl = 30d;
+ # Use bloom filters (must be enabled in Redis as a plugin)
+ use_bloom = false;
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/known_senders.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/known_senders.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/known_senders.conf"
+} \ No newline at end of file
diff --git a/conf/modules.d/maillist.conf b/conf/modules.d/maillist.conf
new file mode 100644
index 0000000..3ddd0a6
--- /dev/null
+++ b/conf/modules.d/maillist.conf
@@ -0,0 +1,20 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/maillist.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/maillist.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/maillist.html
+
+maillist {
+ symbol = "MAILLIST";
+ .include(try=true,priority=5) "${DBDIR}/dynamic/maillist.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/maillist.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/maillist.conf"
+}
diff --git a/conf/modules.d/metadata_exporter.conf b/conf/modules.d/metadata_exporter.conf
new file mode 100644
index 0000000..cd62be5
--- /dev/null
+++ b/conf/modules.d/metadata_exporter.conf
@@ -0,0 +1,24 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/metadata_exporter.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/metadata_exporter.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/metadata_exporter.html
+
+metadata_exporter {
+
+ # Refer to https://rspamd.com/doc/modules/metadata_exporter.html for information on configuration
+ rules {
+ }
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/metadata_exporter.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/metadata_exporter.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/metadata_exporter.conf"
+}
diff --git a/conf/modules.d/metric_exporter.conf b/conf/modules.d/metric_exporter.conf
new file mode 100644
index 0000000..8edce1d
--- /dev/null
+++ b/conf/modules.d/metric_exporter.conf
@@ -0,0 +1,21 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/metric_exporter.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/metric_exporter.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/metric_exporter.html
+metric_exporter {
+
+ # Refer to https://rspamd.com/doc/modules/metric_exporter.html for information on configuration
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/metric_exporter.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/metric_exporter.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/metric_exporter.conf"
+}
diff --git a/conf/modules.d/mid.conf b/conf/modules.d/mid.conf
new file mode 100644
index 0000000..e214a71
--- /dev/null
+++ b/conf/modules.d/mid.conf
@@ -0,0 +1,28 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/mid.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/mid.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/mid.html
+
+mid = {
+ source = {
+ url = [
+ "https://maps.rspamd.com/rspamd/mid.inc.zst",
+ "$LOCAL_CONFDIR/local.d/maps.d/mid.inc",
+ "$LOCAL_CONFDIR/local.d/mid.inc",
+ "fallback+file://${CONFDIR}/maps.d/mid.inc"
+ ];
+ }
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/mid.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/mid.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/mid.conf"
+}
diff --git a/conf/modules.d/milter_headers.conf b/conf/modules.d/milter_headers.conf
new file mode 100644
index 0000000..eafdaed
--- /dev/null
+++ b/conf/modules.d/milter_headers.conf
@@ -0,0 +1,29 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/milter_headers.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/milter_headers.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/milter_headers.html
+
+milter_headers {
+
+ # Refer to https://rspamd.com/doc/modules/milter_headers.html for information on configuration
+
+ use = [];
+
+ # Compatibility
+ .include(try=true,priority=5) "${DBDIR}/dynamic/rmilter_headers.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/rmilter_headers.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/rmilter_headers.conf"
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/milter_headers.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/milter_headers.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/milter_headers.conf"
+}
diff --git a/conf/modules.d/mime_types.conf b/conf/modules.d/mime_types.conf
new file mode 100644
index 0000000..1f67595
--- /dev/null
+++ b/conf/modules.d/mime_types.conf
@@ -0,0 +1,41 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/mime_types.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/mime_types.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/mime_types.html
+
+mime_types {
+ file = [
+ "https://maps.rspamd.com/rspamd/mime_types.inc.zst",
+ "$LOCAL_CONFDIR/local.d/maps.d/mime_types.inc.local",
+ "${DBDIR}/mime_types.inc.local",
+ "fallback+file://${CONFDIR}/maps.d/mime_types.inc"
+ ]
+
+ # Match specific extensions to specific content types
+ extension_map = {
+ html = "text/html";
+ txt = [
+ "message/disposition-notification",
+ "text/plain",
+ "text/rfc822-headers"
+ ];
+ pdf = [
+ "application/octet-stream",
+ "application/pdf",
+ "application/x-pdf"
+ ];
+ }
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/mime_types.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/mime_types.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/mime_types.conf"
+}
diff --git a/conf/modules.d/multimap.conf b/conf/modules.d/multimap.conf
new file mode 100644
index 0000000..b707ddf
--- /dev/null
+++ b/conf/modules.d/multimap.conf
@@ -0,0 +1,182 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/multimap.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/multimap.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/multimap.html
+
+multimap {
+ redirector {
+ type = "url";
+ filter = "tld";
+ map = "https://maps.rspamd.com/rspamd/redirectors.inc.zst";
+ symbol = "REDIRECTOR_URL";
+ description = "The presence of a redirector in the mail";
+ score = 0.0;
+ one_shot = true;
+ }
+
+ # Freemail Addresses
+ freemail_envfrom {
+ type = "from";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/free.txt.zst";
+ symbol = "FREEMAIL_ENVFROM";
+ description = "Envelope From is a Freemail address";
+ score = 0.0;
+ }
+
+ freemail_envrcpt {
+ type = "rcpt";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/free.txt.zst";
+ symbol = "FREEMAIL_ENVRCPT";
+ description = "Envelope Recipient is a Freemail address";
+ score = 0.0;
+ one_shot = true;
+ }
+
+ freemail_from {
+ type = "header";
+ header = "from";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/free.txt.zst";
+ symbol = "FREEMAIL_FROM";
+ description = "From is a Freemail address";
+ score = 0.0;
+ }
+
+ freemail_to {
+ type = "header";
+ header = "To";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/free.txt.zst";
+ symbol = "FREEMAIL_TO";
+ description = "To is a Freemail address";
+ score = 0.0;
+ one_shot = true;
+ }
+
+ freemail_cc {
+ type = "header";
+ header = "Cc";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/free.txt.zst";
+ symbol = "FREEMAIL_CC";
+ description = "To is a Freemail address";
+ score = 0.0;
+ one_shot = true;
+ }
+
+ freemail_replyto {
+ type = "header";
+ header = "Reply-To";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/free.txt.zst";
+ symbol = "FREEMAIL_REPLYTO";
+ description = "Reply-To is a Freemail address";
+ score = 0.0;
+ }
+
+ # Disposable Addresses
+ disposable_envfrom {
+ type = "from";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/disposable.txt.zst";
+ symbol = "DISPOSABLE_ENVFROM";
+ description = "Envelope From is a Disposable e-mail address";
+ score = 0.0;
+ }
+
+ disposable_envrcpt {
+ type = "rcpt";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/disposable.txt.zst";
+ symbol = "DISPOSABLE_ENVRCPT";
+ description = "Envelope Recipient is a Disposable e-mail address";
+ score = 0.0;
+ one_shot = true;
+ }
+
+ disposable_from {
+ type = "header";
+ header = "from";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/disposable.txt.zst";
+ symbol = "DISPOSABLE_FROM";
+ description = "From a Disposable e-mail address";
+ score = 0.0;
+ }
+
+ disposable_to {
+ type = "header";
+ header = "To";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/disposable.txt.zst";
+ symbol = "DISPOSABLE_TO";
+ description = "To a disposable e-mail address";
+ score = 0.0;
+ one_shot = true;
+ }
+
+ disposable_cc {
+ type = "header";
+ header = "Cc";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/disposable.txt.zst";
+ symbol = "DISPOSABLE_CC";
+ description = "To a disposable e-mail address";
+ score = 0.0;
+ one_shot = true;
+ }
+
+ disposable_replyto {
+ type = "header";
+ header = "Reply-To";
+ filter = "email:domain";
+ map = "https://maps.rspamd.com/freemail/disposable.txt.zst";
+ symbol = "DISPOSABLE_REPLYTO";
+ description = "Reply-To a disposable e-mail address";
+ score = 0.0;
+ }
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/multimap.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/multimap.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/multimap.conf"
+}
+
+/* Example setup
+sender_from_whitelist_user {
+ type = "from";
+ filter = "email:user";
+ map = "file:///tmp/from.map";
+ symbol = "SENDER_FROM_WHITELIST_USER";
+ action = "accept"; # Prefilter mode
+}
+sender_from_regexp {
+ type = "header";
+ header = "from";
+ filter = 'regexp:/.*@/';
+ map = "file:///tmp/from_re.map";
+ symbol = "SENDER_FROM_REGEXP";
+}
+url_map {
+ type = "url";
+ filter = "tld";
+ map = "file:///tmp/url.map";
+ symbol = "URL_MAP";
+}
+url_tld_re {
+ type = "url";
+ filter = 'tld:regexp:/\.[^.]+$/'; # Extracts the last component of URL
+ map = "file:///tmp/url.map";
+ symbol = "URL_MAP_RE";
+}
+*/
diff --git a/conf/modules.d/mx_check.conf b/conf/modules.d/mx_check.conf
new file mode 100644
index 0000000..2068fc5
--- /dev/null
+++ b/conf/modules.d/mx_check.conf
@@ -0,0 +1,43 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/mx_check.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/mx_check.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/mx_check.html
+
+# This module is *DISABLED* by default
+# If you need to enable it, then define the following line in
+# local.d/mx_check.conf:
+#
+# enabled = true;
+#
+# You also need to define redis servers for this module
+
+mx_check {
+ # connection timeout in seconds
+ timeout = 1.0;
+ # symbol yielded if no MX is connectable
+ symbol_bad_mx = "MX_INVALID";
+ # symbol yielded if no MX is found
+ symbol_no_mx = "MX_MISSING";
+ # symbol yielded if MX is connectable
+ symbol_good_mx = "MX_GOOD";
+ # lifetime of redis cache - 1 day by default
+ expire = 86400;
+ # prefix used for redis key
+ key_prefix = "rmx";
+
+ # !!! Disabled by default !!!
+ enabled = false;
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/mx_check.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/mx_check.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/mx_check.conf"
+}
diff --git a/conf/modules.d/neural.conf b/conf/modules.d/neural.conf
new file mode 100644
index 0000000..2ab0cbe
--- /dev/null
+++ b/conf/modules.d/neural.conf
@@ -0,0 +1,35 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/neural.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/neural.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/neural.html
+
+neural {
+ #servers = 127.0.0.1:6379; # Redis server to store learning data and ANN
+
+ train {
+ max_trains = 1k; # Number of trains per epoch
+ max_usages = 20; # Number of learn iterations while ANN data is valid
+ learning_rate = 0.01; # Rate of learning (Torch only)
+ max_iterations = 25; # Maximum iterations of learning (Torch only)
+ }
+
+ timeout = 20; # Increase redis timeout
+
+ # Legacy support
+ .include(try=true,priority=5) "${DBDIR}/dynamic/fann_redis.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/fann_redis.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/fann_redis.conf"
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/neural.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/neural.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/neural.conf"
+}
diff --git a/conf/modules.d/once_received.conf b/conf/modules.d/once_received.conf
new file mode 100644
index 0000000..ab07492
--- /dev/null
+++ b/conf/modules.d/once_received.conf
@@ -0,0 +1,26 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/once_received.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/once_received.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/once_received.html
+
+once_received {
+ good_host = "mail";
+ bad_host = "static";
+ bad_host = "dynamic";
+ symbol_strict = "ONCE_RECEIVED_STRICT";
+ symbol = "ONCE_RECEIVED";
+ symbol_mx = "DIRECT_TO_MX";
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/once_received.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/once_received.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/once_received.conf"
+}
diff --git a/conf/modules.d/p0f.conf b/conf/modules.d/p0f.conf
new file mode 100644
index 0000000..049d4fe
--- /dev/null
+++ b/conf/modules.d/p0f.conf
@@ -0,0 +1,45 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/p0f.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/p0f.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/p0f.html
+
+p0f {
+ # Disable module by default
+ enabled = false;
+
+ # Path to the unix socket that p0f listens on
+ socket = '/var/run/p0f.sock';
+
+ # Connection timeout
+ timeout = 5s;
+
+ # If defined, insert symbol with lookup results
+ symbol = 'P0F';
+
+ # Patterns to match against results returned by p0f
+ # Symbol will be yielded on OS string, link type or distance matches
+ patterns = {
+ WINDOWS = '^Windows.*';
+ #DSL = '^DSL$';
+ #DISTANCE10 = '^distance:10$';
+ }
+
+ # Cache lifetime in seconds (default - 2 hours)
+ expire = 7200;
+
+ # Cache key prefix
+ prefix = 'p0f';
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/p0f.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/p0f.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/p0f.conf"
+}
diff --git a/conf/modules.d/phishing.conf b/conf/modules.d/phishing.conf
new file mode 100644
index 0000000..a6531e6
--- /dev/null
+++ b/conf/modules.d/phishing.conf
@@ -0,0 +1,48 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/phishing.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/phishing.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/phishing.html
+
+phishing {
+ symbol = "PHISHING";
+ # Disabled by default
+ openphish_enabled = false;
+ openphish_premium = false;
+ openphish_map = "https://www.openphish.com/feed.txt";
+ # Phishtank is disabled by default in the module, so let's enable it here explicitly
+ phishtank_enabled = true;
+
+ # List of excluded hosts from checks over openphish, phishtank and generic_service
+ phishing_feed_exclusion_symbol = "PHISHED_EXCLUDED";
+ # Disabled by default
+ phishing_feed_exclusion_enabled = false;
+ phishing_feed_exclusion_map = "$LOCAL_CONFDIR/local.d/maps.d/phishing_feed_exclusion.inc";
+
+ # Make exclusions for known redirectors and domains
+ exceptions = {
+ REDIRECTOR_FALSE = [
+ "https://maps.rspamd.com/rspamd/redirectors.inc.zst",
+ "$LOCAL_CONFDIR/local.d/maps.d/redirectors.inc",
+ "$LOCAL_CONFDIR/local.d/redirectors.inc",
+ "fallback+file://${CONFDIR}/maps.d/redirectors.inc"
+ ];
+ PHISHED_WHITELISTED = [
+ "glob;https://maps.rspamd.com/rspamd/phishing_whitelist.inc.zst",
+ "glob;$LOCAL_CONFDIR/local.d/maps.d/phishing_whitelist.inc",
+ "glob;$LOCAL_CONFDIR/local.d/phishing_whitelist.inc",
+ ];
+ };
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/phishing.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/phishing.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/phishing.conf"
+}
diff --git a/conf/modules.d/ratelimit.conf b/conf/modules.d/ratelimit.conf
new file mode 100644
index 0000000..d1a91d3
--- /dev/null
+++ b/conf/modules.d/ratelimit.conf
@@ -0,0 +1,44 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/ratelimit.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/ratelimit.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/ratelimit.html
+
+ratelimit {
+ #rates {
+ # Predefined ratelimit
+ #to = {
+ # bucket = {
+ # burst = 100;
+ # rate = 0.01666666666666666666; # leak 1 message per minute
+ # }
+ #}
+ # or define it with selector
+ #other_limit_alt = {
+ # selector = 'rcpts:addr.take_n(5)';
+ # bucket = {
+ # burst = 100;
+ # rate = "1 / 1m"; # leak 1 message per minute
+ # }
+ #}
+ #}
+ # If symbol is specified, then it is inserted *instead* of setting result to soft reject
+ #symbol = "R_RATELIMIT";
+
+ # If info_symbol is specified, then it is inserted next to set the result
+ #info_symbol = "R_RATELIMIT_INFO";
+
+ whitelisted_rcpts = "postmaster,mailer-daemon";
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/ratelimit.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/ratelimit.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/ratelimit.conf"
+}
diff --git a/conf/modules.d/rbl.conf b/conf/modules.d/rbl.conf
new file mode 100644
index 0000000..79dfc84
--- /dev/null
+++ b/conf/modules.d/rbl.conf
@@ -0,0 +1,347 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/rbl.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/rbl.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/rbl.html
+
+rbl {
+ default_exclude_users = true;
+ default_unknown = true;
+
+ url_whitelist = [
+ "https://maps.rspamd.com/rspamd/surbl-whitelist.inc.zst",
+ "$LOCAL_CONFDIR/local.d/maps.d/surbl-whitelist.inc.local",
+ "${DBDIR}/surbl-whitelist.inc.local",
+ "fallback+file://${CONFDIR}/maps.d/surbl-whitelist.inc"
+ ];
+
+ attached_maps = [
+ {
+ selector_alias = "surbl_hashbl_map",
+ description = "SURBL hashbl map",
+ url = "regexp;http://sa-update.surbl.org/rspamd/surbl-hashbl-map.inc",
+ }
+ ]
+
+ rbls {
+
+ spamhaus {
+ symbol = "SPAMHAUS"; # Augmented by prefixes
+ rbl = "zen.spamhaus.org";
+ # Check types
+ checks = ['received', 'from'];
+
+ symbols_prefixes = {
+ received = 'RECEIVED',
+ from = 'RBL',
+ }
+ returncodes {
+ SPAMHAUS_SBL = "127.0.0.2";
+ SPAMHAUS_CSS = "127.0.0.3";
+ SPAMHAUS_XBL = ["127.0.0.4", "127.0.0.5", "127.0.0.6", "127.0.0.7"];
+ SPAMHAUS_PBL = ["127.0.0.10", "127.0.0.11"];
+ SPAMHAUS_DROP = "127.0.0.9";
+ SPAMHAUS_BLOCKED_OPENRESOLVER = "127.255.255.254";
+ SPAMHAUS_BLOCKED= "127.255.255.255";
+ }
+ }
+
+ mailspike {
+ symbol = "MAILSPIKE";
+ rbl = "rep.mailspike.net";
+ is_whitelist = true;
+ checks = ['from'];
+ whitelist_exception = "MAILSPIKE";
+ whitelist_exception = "RWL_MAILSPIKE_GOOD";
+ whitelist_exception = "RWL_MAILSPIKE_NEUTRAL";
+ whitelist_exception = "RWL_MAILSPIKE_POSSIBLE";
+ whitelist_exception = "RBL_MAILSPIKE_WORST";
+ whitelist_exception = "RBL_MAILSPIKE_VERYBAD";
+ whitelist_exception = "RBL_MAILSPIKE_BAD";
+ returncodes {
+ RBL_MAILSPIKE_WORST = "127.0.0.10";
+ RBL_MAILSPIKE_VERYBAD = "127.0.0.11";
+ RBL_MAILSPIKE_BAD = "127.0.0.12";
+ RWL_MAILSPIKE_NEUTRAL = ["127.0.0.16", "127.0.0.15", "127.0.0.14", "127.0.0.13"];
+ RWL_MAILSPIKE_POSSIBLE = "127.0.0.17";
+ RWL_MAILSPIKE_GOOD = "127.0.0.18";
+ RWL_MAILSPIKE_VERYGOOD = "127.0.0.19";
+ RWL_MAILSPIKE_EXCELLENT = "127.0.0.20";
+ }
+ }
+
+ senderscore {
+ symbol = "RBL_SENDERSCORE";
+ checks = ['from'];
+ rbl = "bl.score.senderscore.com";
+ }
+
+ sem {
+ symbol = "RBL_SEM";
+ rbl = "bl.spameatingmonkey.net";
+ ipv6 = false;
+ checks = ['from'];
+ }
+
+ semIPv6 {
+ symbol = "RBL_SEM_IPV6";
+ rbl = "bl.ipv6.spameatingmonkey.net";
+ ipv4 = false;
+ ipv6 = true;
+ checks = ['from'];
+ }
+
+ dnswl {
+ symbol = "RCVD_IN_DNSWL";
+ rbl = "list.dnswl.org";
+ ipv6 = true;
+ checks = ['from', 'received'];
+ is_whitelist = true;
+ returncodes_matcher = "luapattern";
+ whitelist_exception = "RCVD_IN_DNSWL";
+ whitelist_exception = "RCVD_IN_DNSWL_NONE";
+ whitelist_exception = "RCVD_IN_DNSWL_LOW";
+ whitelist_exception = "DNSWL_BLOCKED";
+ returncodes {
+ RCVD_IN_DNSWL_NONE = ["127%.0%.%d%.0", "127%.0%.[02-9]%d%.0", "127%.0%.1[1-9]%.0", "127%.0%.[12]%d%d%.0"];
+ RCVD_IN_DNSWL_LOW = ["127%.0%.%d%.1", "127%.0%.[02-9]%d%.1", "127%.0%.1[1-9]%.1", "127%.0%.[12]%d%d%.1"];
+ RCVD_IN_DNSWL_MED = ["127%.0%.%d%.2", "127%.0%.[02-9]%d%.2", "127%.0%.1[1-9]%.2", "127%.0%.[12]%d%d%.2"];
+ RCVD_IN_DNSWL_HI = ["127%.0%.%d%.3", "127%.0%.[02-9]%d%.3", "127%.0%.1[1-9]%.3", "127%.0%.[12]%d%d%.3"];
+ DNSWL_BLOCKED = ["127%.0%.0%.255", "127%.0%.10%.%d+"];
+ }
+ }
+
+ # Provided by https://virusfree.cz
+ virusfree {
+ symbol = "RBL_VIRUSFREE_UNKNOWN";
+ rbl = "bip.virusfree.cz";
+ ipv6 = true;
+ checks = ['from'];
+ returncodes {
+ RBL_VIRUSFREE_BOTNET = "127.0.0.2";
+ }
+ }
+
+ nixspam {
+ symbol = "RBL_NIXSPAM";
+ rbl = "ix.dnsbl.manitu.net";
+ ipv6 = true;
+ checks = ['from'];
+ }
+
+ blocklistde {
+ symbols_prefixes = {
+ received = 'RECEIVED',
+ from = 'RBL',
+ }
+ symbol = "BLOCKLISTDE";
+ rbl = "bl.blocklist.de";
+ checks = ['from', 'received'];
+ }
+
+ # Dkim whitelist
+ dnswl_dwl {
+ symbol = "DWL_DNSWL";
+ rbl = "dwl.dnswl.org";
+ checks = ['dkim'];
+ ignore_whitelist = true;
+ returncodes_matcher = "luapattern";
+ unknown = false;
+
+ returncodes {
+ DWL_DNSWL_NONE = ["127%.0%.%d%.0", "127%.0%.[02-9]%d%.0", "127%.0%.1[1-9]%.0", "127%.0%.[12]%d%d%.0"];
+ DWL_DNSWL_LOW = ["127%.0%.%d%.1", "127%.0%.[02-9]%d%.1", "127%.0%.1[1-9]%.1", "127%.0%.[12]%d%d%.1"];
+ DWL_DNSWL_MED = ["127%.0%.%d%.2", "127%.0%.[02-9]%d%.2", "127%.0%.1[1-9]%.2", "127%.0%.[12]%d%d%.2"];
+ DWL_DNSWL_HI = ["127%.0%.%d%.3", "127%.0%.[02-9]%d%.3", "127%.0%.1[1-9]%.3", "127%.0%.[12]%d%d%.3"];
+ DWL_DNSWL_BLOCKED = ["127%.0%.0%.255", "127%.0%.10%.%d+"];
+ }
+ }
+
+ RSPAMD_EMAILBL {
+ ignore_whitelist = true;
+ ignore_url_whitelist = true;
+ ignore_defaults = true;
+ exclude_users = false;
+ emails_delimiter = ".";
+ hash_format = "base32";
+ hash_len = 32;
+ rbl = "email.rspamd.com";
+ checks = ['emails', 'replyto'];
+ hash = "blake2";
+ returncodes = {
+ RSPAMD_EMAILBL = "127.0.0.2";
+ }
+ }
+
+ MSBL_EBL {
+ ignore_whitelist = true;
+ ignore_url_whitelist = true;
+ ignore_defaults = true;
+ exclude_users = false;
+ rbl = "ebl.msbl.org";
+ checks = ['emails', 'replyto'];
+ emails_domainonly = false;
+ hash = "sha1";
+ returncodes = {
+ MSBL_EBL = [
+ "127.0.0.2",
+ "127.0.0.3"
+ ];
+ MSBL_EBL_GREY = [
+ "127.0.1.2",
+ "127.0.1.3"
+ ];
+ }
+ }
+
+ "SURBL_MULTI" {
+ ignore_defaults = true;
+ rbl = "multi.surbl.org";
+ checks = ['emails', 'dkim', 'helo', 'rdns', 'replyto', 'urls'];
+ emails_domainonly = true;
+ exclude_users = false;
+
+ returnbits = {
+ CRACKED_SURBL = 128;
+ ABUSE_SURBL = 64;
+ MW_SURBL_MULTI = 16;
+ PH_SURBL_MULTI = 8;
+ SURBL_BLOCKED = 1;
+ }
+ }
+
+ SURBL_HASHBL {
+ rbl = "hashbl.surbl.org";
+ ignore_defaults = true;
+ random_monitored = true,
+ # TODO: make limit more configurable maybe?
+ selector = "specific_urls_filter_map('surbl_hashbl_map', {limit = 10}).apply_methods('get_host', 'get_path').join_tables('/')",
+ hash = 'md5';
+ hash_len = 32;
+ returncodes_matcher = "luapattern";
+ returncodes = {
+ SURBL_HASHBL_PHISH = "127.0.0.8";
+ SURBL_HASHBL_MALWARE = "127.0.0.16";
+ SURBL_HASHBL_ABUSE = "127.0.0.64";
+ SURBL_HASHBL_CRACKED = "127.0.0.128";
+ SURBL_HASHBL_EMAIL = "127.0.1.%d+";
+ }
+ }
+
+ "URIBL_MULTI" {
+ ignore_defaults = true;
+ rbl = "multi.uribl.com";
+ checks = ['emails', 'dkim', 'helo', 'rdns', 'replyto', 'urls'];
+ emails_domainonly = true;
+ exclude_users = false;
+
+ returnbits {
+ URIBL_BLOCKED = 1;
+ URIBL_BLACK = 2;
+ URIBL_GREY = 4;
+ URIBL_RED = 8;
+ }
+ }
+
+ "RSPAMD_URIBL" {
+ ignore_defaults = true;
+ rbl = "uribl.rspamd.com";
+ checks = ['emails', 'dkim', 'urls'];
+ emails_domainonly = true;
+ hash = 'blake2';
+ hash_len = 32;
+ hash_format = 'base32';
+ exclude_users = false;
+
+ returncodes = {
+ RSPAMD_URIBL = [
+ "127.0.0.2",
+ ];
+ }
+ }
+
+ "DBL" {
+ ignore_defaults = true;
+ rbl = "dbl.spamhaus.org";
+ no_ip = true;
+ checks = ['emails', 'dkim', 'helo', 'rdns', 'replyto', 'urls'];
+ emails_domainonly = true;
+ exclude_users = false;
+
+ returncodes = {
+ # spam domain
+ DBL_SPAM = "127.0.1.2";
+ # phish domain
+ DBL_PHISH = "127.0.1.4";
+ # malware domain
+ DBL_MALWARE = "127.0.1.5";
+ # botnet C&C domain
+ DBL_BOTNET = "127.0.1.6";
+ # abused legit spam
+ DBL_ABUSE = "127.0.1.102";
+ # abused spammed redirector domain
+ DBL_ABUSE_REDIR = "127.0.1.103";
+ # abused legit phish
+ DBL_ABUSE_PHISH = "127.0.1.104";
+ # abused legit malware
+ DBL_ABUSE_MALWARE = "127.0.1.105";
+ # abused legit botnet C&C
+ DBL_ABUSE_BOTNET = "127.0.1.106";
+ # error - IP queries prohibited!
+ DBL_PROHIBIT = "127.0.1.255";
+ # issue #3074
+ DBL_BLOCKED_OPENRESOLVER = "127.255.255.254";
+ DBL_BLOCKED = "127.255.255.255";
+ }
+ }
+
+ # Not enabled by default due to privacy concerns! (see also groups.d/surbl_group.conf)
+ "SPAMHAUS_ZEN_URIBL" {
+ enabled = false;
+ rbl = "zen.spamhaus.org";
+ checks = ['emails'];
+ resolve_ip = true;
+ returncodes = {
+ URIBL_SBL = "127.0.0.2";
+ URIBL_SBL_CSS = "127.0.0.3";
+ URIBL_XBL = ["127.0.0.4", "127.0.0.5", "127.0.0.6", "127.0.0.7"];
+ URIBL_PBL = ["127.0.0.10", "127.0.0.11"];
+ URIBL_DROP = "127.0.0.9";
+ }
+ }
+
+ "SEM_URIBL_UNKNOWN" {
+ ignore_defaults = true;
+ rbl = "uribl.spameatingmonkey.net";
+ no_ip = true;
+ checks = ['emails', 'dkim', 'urls'];
+ emails_domainonly = true;
+ returnbits {
+ SEM_URIBL = 2;
+ }
+ }
+
+ "SEM_URIBL_FRESH15_UNKNOWN" {
+ ignore_defaults = true;
+ rbl = "fresh15.spameatingmonkey.net";
+ no_ip = true;
+ checks = ['emails', 'dkim', 'urls'];
+ emails_domainonly = true;
+ returnbits {
+ SEM_URIBL_FRESH15 = 2;
+ }
+ }
+ }
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/rbl.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/rbl.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/rbl.conf"
+}
diff --git a/conf/modules.d/redis.conf b/conf/modules.d/redis.conf
new file mode 100644
index 0000000..24948b1
--- /dev/null
+++ b/conf/modules.d/redis.conf
@@ -0,0 +1,27 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/redis.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/redis.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# See https://rspamd.com/doc/configuration/redis.html
+
+redis {
+ #servers = "127.0.0.1"; # Read servers (unless write_servers are unspecified)
+ #servers = "master-slave:127.0.0.1,10.0.1.1";
+ #write_servers = "127.0.0.1"; # Servers to write data
+ #disabled_modules = ["ratelimit"]; # List of modules that should not use redis from this section
+ #timeout = 1s;
+ #db = "0";
+ #username = "some_username";
+ #password = "some_password";
+ .include(try=true,priority=5) "${DBDIR}/dynamic/redis.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/redis.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/redis.conf"
+}
diff --git a/conf/modules.d/regexp.conf b/conf/modules.d/regexp.conf
new file mode 100644
index 0000000..f7d5921
--- /dev/null
+++ b/conf/modules.d/regexp.conf
@@ -0,0 +1,21 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/regexp.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/regexp.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/regexp.html
+
+regexp {
+ max_size = 1M;
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/regexp.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/regexp.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/regexp.conf"
+}
diff --git a/conf/modules.d/replies.conf b/conf/modules.d/replies.conf
new file mode 100644
index 0000000..a1a85df
--- /dev/null
+++ b/conf/modules.d/replies.conf
@@ -0,0 +1,30 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/replies.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/replies.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/replies.html
+
+replies {
+ # This setting is non-default & is required to be set
+ # Redis servers to use
+ #servers = "localhost";
+ # This setting is non-default & may be desirable
+ #action = "no action";
+ # These are default settings you may want to change
+ expire = 86400;
+ key_prefix = "rr";
+ message = "Message is reply to one we originated";
+ symbol = "REPLY";
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/replies.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/replies.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/replies.conf"
+}
diff --git a/conf/modules.d/reputation.conf b/conf/modules.d/reputation.conf
new file mode 100644
index 0000000..fe9d5f7
--- /dev/null
+++ b/conf/modules.d/reputation.conf
@@ -0,0 +1,30 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/reputation.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/reputation.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/reputation.html
+
+reputation {
+# rules {
+# SPF_REPUTATION = {
+# selector {
+# type = "spf";
+# }
+# backend {
+# type = "redis";
+# }
+# symbol = "SPF_REPUTATION";
+# }
+# }
+ .include(try=true,priority=5) "${DBDIR}/dynamic/reputation.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/reputation.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/reputation.conf"
+}
diff --git a/conf/modules.d/rspamd_update.conf b/conf/modules.d/rspamd_update.conf
new file mode 100644
index 0000000..1b998b2
--- /dev/null
+++ b/conf/modules.d/rspamd_update.conf
@@ -0,0 +1,26 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/rspamd_update.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/rspamd_update.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/rspamd_update.html
+
+# This module is disabled by default, please don't enable it unless you know
+# what are you doing!
+
+rspamd_update {
+ rules = "sign+https://updates.rspamd.com/rspamd-${BRANCH_VERSION}.ucl";
+ key = "qxuogdh5eghytji1utkkte1dn3n81c3y5twe61uzoddzwqzuxxyb";
+ enabled = false; # Disable this module by default
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/rspamd_update.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/rspamd_update.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/rspamd_update.conf"
+}
diff --git a/conf/modules.d/spamassassin.conf b/conf/modules.d/spamassassin.conf
new file mode 100644
index 0000000..79f7527
--- /dev/null
+++ b/conf/modules.d/spamassassin.conf
@@ -0,0 +1,26 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/spamassassin.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/spamassassin.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/spamassassin.html
+
+spamassassin {
+ # This config defines no SA files leaving this module efficiently disabled by default
+ #ruleset = "/path/to/file";
+ # Limit search size to 100 kilobytes for all regular expressions
+ #match_limit = 100k;
+ # Those regexp atoms will not be passed through hyperscan:
+ #pcre_only = ["RULE1", "__RULE2"];
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/spamassassin.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/spamassassin.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/spamassassin.conf"
+}
diff --git a/conf/modules.d/spamtrap.conf b/conf/modules.d/spamtrap.conf
new file mode 100644
index 0000000..d0e70f6
--- /dev/null
+++ b/conf/modules.d/spamtrap.conf
@@ -0,0 +1,53 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/spamtrap.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/spamtrap.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/spamtrap.html
+
+# This module is *DISABLED* by default
+# If you need to enable it, then define the following line in
+# local.d/spamtrap.conf:
+#
+# enabled = true;
+#
+# You also need to define redis servers for this module
+
+spamtrap {
+ # Optionally set an action
+ #action = "no action";
+ # A map file containing regexp entries for spamtrap emails and domains
+ #map = file://$LOCAL_CONFDIR/local.d/maps.d/spamtrap.map
+ # Name of the symbol
+ #symbol = "SPAMTRAP";
+ # A score for this module
+ #score = 0.0;
+ # Flag to enable fuzzy learning
+ learn_fuzzy = false;
+ # Flag to enable bayes spam learning
+ learn_spam = false;
+ # Fuzzy flag
+ #fuzzy_flag = 1;
+ # Fuzzy weight
+ #fuzzy_weight = 10.0;
+ # Redis key prefix
+ #key_prefix = 'sptr_';
+ # Skip spamtrap checks for authorized users
+ #check_authed = false;
+ # Skip spamtrap checks for local networks
+ #check_local = false;
+
+ # !!! Disabled by default !!!
+ enabled = false;
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/spamtrap.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/spamtrap.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/spamtrap.conf"
+}
diff --git a/conf/modules.d/spf.conf b/conf/modules.d/spf.conf
new file mode 100644
index 0000000..c4284bc
--- /dev/null
+++ b/conf/modules.d/spf.conf
@@ -0,0 +1,22 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/spf.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/spf.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/spf.html
+spf {
+ spf_cache_size = 2k;
+ spf_cache_expire = 1d;
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/spf.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/spf.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/spf.conf"
+
+}
diff --git a/conf/modules.d/surbl.conf b/conf/modules.d/surbl.conf
new file mode 100644
index 0000000..89b8842
--- /dev/null
+++ b/conf/modules.d/surbl.conf
@@ -0,0 +1,9 @@
+# Deprecated, use RBL module!
+surbl {
+ rules {
+ }
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/surbl.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/surbl.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/surbl.conf"
+}
diff --git a/conf/modules.d/trie.conf b/conf/modules.d/trie.conf
new file mode 100644
index 0000000..9a1e1ca
--- /dev/null
+++ b/conf/modules.d/trie.conf
@@ -0,0 +1,37 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/trie.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/trie.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/trie.html
+
+trie {
+ # Each subsection defines a single rule with associated symbol
+ # This config defines no tries leaving this module efficiently disabled by default
+# SYMBOL1 {
+# # Define rules in the file (it is *NOT* a map)
+# file = "/some/path";
+# # Raw rules search within the whole undecoded messages
+# raw = true;
+# # If we have multiple occurrences of strings from this rule
+# # then we insert a symbol multiple times
+# multi = true;
+# }
+# SYMBOL2 {
+# patterns = [
+# "pattern1",
+# "pattern2",
+# "pattern3"
+# ]
+# }
+ .include(try=true,priority=5) "${DBDIR}/dynamic/trie.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/trie.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/trie.conf"
+}
diff --git a/conf/modules.d/url_redirector.conf b/conf/modules.d/url_redirector.conf
new file mode 100644
index 0000000..da3b5bb
--- /dev/null
+++ b/conf/modules.d/url_redirector.conf
@@ -0,0 +1,27 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/url_redirector.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/url_redirector.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/url_redirector.html
+
+url_redirector {
+ expire = 1d; # 1 day by default
+ timeout = 10; # 10 seconds by default
+ nested_limit = 1; # How many redirects to follow
+ #proxy = "http://example.com:3128"; # Send request through proxy
+ key_prefix = "rdr:"; # default hash name
+ check_ssl = false; # check ssl certificates
+ max_size = 10k; # maximum body to process
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/url_redirector.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/url_redirector.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/url_redirector.conf"
+}
diff --git a/conf/modules.d/whitelist.conf b/conf/modules.d/whitelist.conf
new file mode 100644
index 0000000..a4b39aa
--- /dev/null
+++ b/conf/modules.d/whitelist.conf
@@ -0,0 +1,66 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/whitelist.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/whitelist.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at https://rspamd.com/doc/modules/whitelist.html
+
+whitelist {
+ rules {
+ "WHITELIST_SPF" = {
+ valid_spf = true;
+ domains = [
+ "$LOCAL_CONFDIR/local.d/maps.d/spf_whitelist.inc.local",
+ "${DBDIR}/spf_whitelist.inc.local",
+ "fallback+file://${CONFDIR}/maps.d/spf_whitelist.inc"
+ ];
+ score = -1.0
+ inverse_symbol = "BLACKLIST_SPF";
+ }
+
+ "WHITELIST_DKIM" = {
+ valid_dkim = true;
+ domains = [
+ "$LOCAL_CONFDIR/local.d/maps.d/dkim_whitelist.inc.local",
+ "${DBDIR}/dkim_whitelist.inc.local",
+ "fallback+file://${CONFDIR}/maps.d/dkim_whitelist.inc"
+ ];
+ score = -1.0;
+ inverse_symbol = "BLACKLIST_DKIM";
+ }
+ "WHITELIST_SPF_DKIM" = {
+ valid_spf = true;
+ valid_dkim = true;
+ domains = [
+ "https://maps.rspamd.com/rspamd/spf_dkim_whitelist.inc.zst",
+ "$LOCAL_CONFDIR/local.d/maps.d/spf_dkim_whitelist.inc.local",
+ "${DBDIR}/spf_dkim_whitelist.inc.local",
+ "fallback+file://${CONFDIR}/maps.d/spf_dkim_whitelist.inc"
+ ];
+ score = -3.0;
+ inverse_symbol = "BLACKLIST_SPF_DKIM";
+ }
+ "WHITELIST_DMARC" = {
+ valid_dmarc = true;
+ domains = [
+ "https://maps.rspamd.com/rspamd/dmarc_whitelist_new.inc.zst",
+ "$LOCAL_CONFDIR/local.d/maps.d/dmarc_whitelist.inc.local",
+ "${DBDIR}/dmarc_whitelist.inc.local",
+ "fallback+file://${CONFDIR}/maps.d/dmarc_whitelist.inc"
+ ];
+ score = -7.0;
+ inverse_symbol = "BLACKLIST_DMARC";
+ }
+ }
+
+ .include(try=true,priority=5) "${DBDIR}/dynamic/whitelist.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/whitelist.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/whitelist.conf"
+}
diff --git a/conf/options.inc b/conf/options.inc
new file mode 100644
index 0000000..d5cf60d
--- /dev/null
+++ b/conf/options.inc
@@ -0,0 +1,72 @@
+# Basic rspamd configuration
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/options.inc' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/options.inc' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Relevant documentation: https://rspamd.com/doc/configuration/options.html
+
+filters = "chartable,dkim,regexp,fuzzy_check";
+one_shot = false;
+cache_file = "$DBDIR/symbols.cache";
+# How often maps are checked (
+map_watch_interval = 5min;
+# Multiplier for watch interval for files
+map_file_watch_multiplier = 0.1;
+dynamic_conf = "$DBDIR/rspamd_dynamic";
+history_file = "$DBDIR/rspamd.history";
+check_all_filters = false;
+
+# Default settings
+dns_max_requests = 64;
+max_lua_urls = 1024;
+max_urls = 10240;
+max_recipients = 1024;
+
+dns {
+ timeout = 1s;
+ sockets = 16;
+ retransmits = 5;
+}
+tempdir = "/tmp";
+url_tld = "${SHAREDIR}/effective_tld_names.dat";
+classify_headers = [
+ "User-Agent",
+ "X-Mailer",
+ "Content-Type",
+ "X-MimeOLE",
+];
+
+control_socket = "$DBDIR/rspamd.sock mode=0600";
+history_rows = 200;
+explicit_modules = ["settings", "bayes_expiry"];
+
+# Scan messages even if they are not MIME
+allow_raw_input = true;
+
+# Start ignore words when reaching the following limit, so the total
+# amount of words processed will not be *LIKELY more than the twice of that limit
+words_decay = 600;
+
+# Write statistics about rspamd usage to the round-robin database
+rrd = "${DBDIR}/rspamd.rrd";
+
+# Write statistics for `rspamc` here
+stats_file = "${DBDIR}/stats.ucl";
+
+# Local networks
+local_addrs = [192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12, fd00::/8, 169.254.0.0/16, fe80::/10];
+hs_cache_dir = "${DBDIR}/";
+
+# Timeout for messages processing (must be larger than any internal timeout used)
+task_timeout = 8s;
+
+# Emit soft reject when timeout takes place
+soft_reject_on_timeout = false;
diff --git a/conf/rspamd.conf b/conf/rspamd.conf
new file mode 100644
index 0000000..b05a18b
--- /dev/null
+++ b/conf/rspamd.conf
@@ -0,0 +1,73 @@
+# System V init adopted top level configuration
+
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+.include "$CONFDIR/common.conf"
+
+options {
+ pidfile = "$RUNDIR/rspamd.pid";
+ .include "$CONFDIR/options.inc"
+ .include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/options.inc"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/options.inc"
+}
+
+lang_detection {
+ .include "$CONFDIR/lang_detection.inc"
+ .include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/lang_detection.inc"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/lang_detection.inc"
+}
+
+.include(try=true; duplicate=merge) "$CONFDIR/cgp.inc"
+.include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/cgp.inc"
+
+logging {
+ type = "file";
+ filename = "$LOGDIR/rspamd.log";
+ .include "$CONFDIR/logging.inc"
+ .include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/logging.inc"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/logging.inc"
+}
+
+worker "normal" {
+ bind_socket = "localhost:11333";
+ .include "$CONFDIR/worker-normal.inc"
+ .include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/worker-normal.inc"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/worker-normal.inc"
+}
+
+worker "controller" {
+ bind_socket = "localhost:11334";
+ .include "$CONFDIR/worker-controller.inc"
+ .include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/worker-controller.inc"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/worker-controller.inc"
+}
+
+worker "rspamd_proxy" {
+ bind_socket = "localhost:11332";
+ .include "$CONFDIR/worker-proxy.inc"
+ .include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/worker-proxy.inc"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/worker-proxy.inc"
+}
+
+# Local fuzzy storage is disabled by default
+
+worker "fuzzy" {
+ bind_socket = "localhost:11335";
+ count = -1; # Disable by default, see #4677 for details
+ .include "$CONFDIR/worker-fuzzy.inc"
+ .include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/worker-fuzzy.inc"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/worker-fuzzy.inc"
+}
diff --git a/conf/scores.d/content_group.conf b/conf/scores.d/content_group.conf
new file mode 100644
index 0000000..88b4765
--- /dev/null
+++ b/conf/scores.d/content_group.conf
@@ -0,0 +1,51 @@
+# Content matching rules
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "Content rules";
+
+symbols = {
+ "PDF_ENCRYPTED" {
+ weight = 0.3;
+ description = "There is an encrypted PDF in the message";
+ one_shot = true;
+ }
+ "PDF_JAVASCRIPT" {
+ weight = 0.1;
+ description = "There is an PDF with JavaScript in the message";
+ one_shot = true;
+ }
+ "PDF_SUSPICIOUS" {
+ weight = 4.5;
+ description = "There is an PDF with suspicious properties in the message";
+ one_shot = true;
+ }
+ "PDF_LONG_TRAILER" {
+ weight = 0.2;
+ description = "There is an PDF with a long trailer in the message";
+ one_shot = true;
+ }
+ "PDF_MANY_OBJECTS" {
+ weight = 0;
+ description = "There is a PDF with too many objects in the message";
+ one_shot = true;
+ }
+ "PDF_TIMEOUT" {
+ weight = 0;
+ description = "There is a PDF in the message that caused timeout in processing";
+ one_shot = true;
+ }
+}
diff --git a/conf/scores.d/fuzzy_group.conf b/conf/scores.d/fuzzy_group.conf
new file mode 100644
index 0000000..bc2f949
--- /dev/null
+++ b/conf/scores.d/fuzzy_group.conf
@@ -0,0 +1,37 @@
+# Fuzzy rules scores
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "Fuzzy hashes group";
+
+symbols = {
+ "FUZZY_UNKNOWN" {
+ weight = 5.0;
+ description = "Generic fuzzy hash match, bl.rspamd.com";
+ }
+ "FUZZY_DENIED" {
+ weight = 12.0;
+ description = "Denied fuzzy hash, bl.rspamd.com";
+ }
+ "FUZZY_PROB" {
+ weight = 5.0;
+ description = "Probable fuzzy hash, bl.rspamd.com";
+ }
+ "FUZZY_WHITE" {
+ weight = -2.1;
+ description = "Whitelisted fuzzy hash, bl.rspamd.com";
+ }
+} \ No newline at end of file
diff --git a/conf/scores.d/headers_group.conf b/conf/scores.d/headers_group.conf
new file mode 100644
index 0000000..c9b078c
--- /dev/null
+++ b/conf/scores.d/headers_group.conf
@@ -0,0 +1,77 @@
+# Headers rules scores
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "Various headers checks";
+
+max_score = 8.0;
+
+symbols = {
+ "FORGED_SENDER" {
+ weight = 0.3;
+ description = "Sender is forged (different From: header and smtp MAIL FROM: addresses)";
+ }
+ "R_MIXED_CHARSET" {
+ weight = 5.0;
+ description = "Mixed characters in a message";
+ one_shot = true;
+ }
+ "R_MIXED_CHARSET_URL" {
+ weight = 7.0;
+ description = "Mixed characters in a URL inside message";
+ one_shot = true;
+ }
+ "FORGED_RECIPIENTS" {
+ weight = 2.0;
+ description = "Recipients are not the same as RCPT TO: mail command";
+ }
+ "FORGED_RECIPIENTS_MAILLIST" {
+ weight = 0.0;
+ description = "Recipients are not the same as RCPT TO: mail command, but a message from a maillist";
+ }
+ "FORGED_SENDER_MAILLIST" {
+ weight = 0.0;
+ description = "Sender is not the same as MAIL FROM: envelope, but a message is from a maillist";
+ }
+ "ONCE_RECEIVED" {
+ weight = 0.1;
+ description = "One received header in a message";
+ }
+ "RDNS_NONE" {
+ weight = 1.0;
+ description = "Cannot resolve reverse DNS for sender's IP";
+ }
+ "RDNS_DNSFAIL" {
+ weight = 0.0;
+ description = "PTR verification DNS error";
+ }
+ "ONCE_RECEIVED_STRICT" {
+ weight = 4.0;
+ description = "One received header with 'bad' patterns inside";
+ }
+ "DIRECT_TO_MX" {
+ weight = 0.0;
+ description = "Message has been directly delivered from MUA to local MX";
+ }
+ "MAILLIST" {
+ weight = -0.2;
+ description = "Message seems to be from maillist";
+ }
+ "BOUNCE" {
+ weight = -0.1;
+ description = "(Non) Delivery Status Notification";
+ }
+}
diff --git a/conf/scores.d/hfilter_group.conf b/conf/scores.d/hfilter_group.conf
new file mode 100644
index 0000000..09fcfcd
--- /dev/null
+++ b/conf/scores.d/hfilter_group.conf
@@ -0,0 +1,133 @@
+# Host and connection rules scores
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "SMTP envelope filter";
+
+symbols = {
+ "HFILTER_HELO_BAREIP" {
+ weight = 3.0;
+ description = "Helo host is bare ip";
+ }
+ "HFILTER_HELO_BADIP" {
+ weight = 4.5;
+ description = "Helo host is very bad ip";
+ }
+ "HFILTER_HELO_1" {
+ weight = 0.5;
+ description = "Helo host checks (very low)";
+ }
+ "HFILTER_HELO_2" {
+ weight = 1.0;
+ description = "Helo host checks (low)";
+ }
+ "HFILTER_HELO_3" {
+ weight = 2.0;
+ description = "Helo host checks (medium)";
+ }
+ "HFILTER_HELO_4" {
+ weight = 2.5;
+ description = "Helo host checks (hard)";
+ }
+ "HFILTER_HELO_5" {
+ weight = 3.0;
+ description = "Helo host checks (very hard)";
+ }
+ "HFILTER_HOSTNAME_1" {
+ weight = 0.5;
+ description = "Hostname checks (very low)";
+ }
+ "HFILTER_HOSTNAME_2" {
+ weight = 1.0;
+ description = "Hostname checks (low)";
+ }
+ "HFILTER_HOSTNAME_3" {
+ weight = 2.0;
+ description = "Hostname checks (medium)";
+ }
+ "HFILTER_HOSTNAME_4" {
+ weight = 2.5;
+ description = "Hostname checks (hard)";
+ }
+ "HFILTER_HOSTNAME_5" {
+ weight = 3.0;
+ description = "Hostname checks (very hard)";
+ }
+ "HFILTER_HELO_NORESOLVE_MX" {
+ weight = 0.2;
+ description = "MX found in Helo and no resolve";
+ }
+ "HFILTER_HELO_NORES_A_OR_MX" {
+ weight = 0.3;
+ description = "Helo no resolve to A or MX";
+ }
+ "HFILTER_HELO_IP_A" {
+ weight = 1.0;
+ description = "Helo A IP != hostname IP";
+ }
+ "HFILTER_HELO_NOT_FQDN" {
+ weight = 2.0;
+ description = "Helo not FQDN";
+ }
+ "HFILTER_FROMHOST_NORESOLVE_MX" {
+ weight = 0.5;
+ description = "MX found in FROM host and no resolve";
+ }
+ "HFILTER_FROMHOST_NORES_A_OR_MX" {
+ weight = 1.5;
+ description = "FROM host no resolve to A or MX";
+ }
+ "HFILTER_FROMHOST_NOT_FQDN" {
+ weight = 3.0;
+ description = "FROM host not FQDN";
+ }
+ "HFILTER_FROM_BOUNCE" {
+ weight = 0.0;
+ description = "Bounce message";
+ }
+/*
+ # Disabled by default
+ "HFILTER_MID_NORESOLVE_MX" {
+ weight = 0.5;
+ description = "MX found in Message-id host and no resolve";
+ }
+ "HFILTER_MID_NORES_A_OR_MX" {
+ weight = 0.5;
+ name = ;
+ description = "Message-id host no resolve to A or MX";
+ }
+ "HFILTER_MID_NOT_FQDN" {
+ weight = 0.5;
+ description = "Message-id host not FQDN";
+ }
+*/
+ "HFILTER_HOSTNAME_UNKNOWN" {
+ weight = 2.5;
+ description = "Unknown client hostname (PTR or FCrDNS verification failed)";
+ }
+ "HFILTER_RCPT_BOUNCEMOREONE" {
+ weight = 1.5;
+ description = "Message from bounce and over 1 recipient";
+ }
+ "HFILTER_URL_ONLY" {
+ weight = 2.2;
+ description = "URL only in body";
+ }
+ "HFILTER_URL_ONELINE" {
+ weight = 2.5;
+ description = "One line URL and text in body";
+ }
+}
diff --git a/conf/scores.d/mime_types_group.conf b/conf/scores.d/mime_types_group.conf
new file mode 100644
index 0000000..268709e
--- /dev/null
+++ b/conf/scores.d/mime_types_group.conf
@@ -0,0 +1,78 @@
+# Mime types rules scores
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "Mime attachments rules";
+
+max_score = 10.0;
+
+symbols = {
+ "MIME_GOOD" {
+ weight = -0.1;
+ description = "Known content-type";
+ one_shot = true;
+ }
+ "MIME_BAD" {
+ weight = 1.0;
+ description = "Known bad content-type";
+ one_shot = true;
+ }
+ "MIME_UNKNOWN" {
+ weight = 0.1;
+ description = "Missing or unknown content-type";
+ one_shot = true;
+ }
+ "MIME_BAD_ATTACHMENT" {
+ weight = 4.0;
+ description = "Invalid attachment mime type";
+ one_shot = true;
+ }
+ "MIME_ENCRYPTED_ARCHIVE" {
+ weight = 2.0;
+ description = "Encrypted archive in a message";
+ one_shot = true;
+ }
+ "MIME_OBFUSCATED_ARCHIVE" {
+ weight = 8.0;
+ description = "Archive has files with clear obfuscation signs";
+ one_shot = true;
+ }
+ "MIME_EXE_IN_GEN_SPLIT_RAR" {
+ weight = 5.0;
+ description = "EXE file in RAR archive with generic split extension (e.g. .001)";
+ one_shot = true;
+ }
+ "MIME_ARCHIVE_IN_ARCHIVE" {
+ weight = 5.0;
+ description = "Archive within another archive";
+ one_shot = true;
+ }
+ "MIME_DOUBLE_BAD_EXTENSION" {
+ weight = 3.0; # This rule has dynamic weight up to 4.0
+ description = "Bad extension cloaking";
+ one_shot = true;
+ }
+ "MIME_BAD_EXTENSION" {
+ weight = 2.0; # This rule has dynamic weight up to 4.0
+ description = "Bad extension";
+ one_shot = true;
+ }
+ "MIME_BAD_UNICODE" {
+ weight = 8.0;
+ description = "Filename with known obscured unicode characters";
+ one_shot = true;
+ }
+}
diff --git a/conf/scores.d/mua_group.conf b/conf/scores.d/mua_group.conf
new file mode 100644
index 0000000..18ca013
--- /dev/null
+++ b/conf/scores.d/mua_group.conf
@@ -0,0 +1,25 @@
+# MUA rules scores
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "MUA forgeries";
+
+symbols = {
+ "FORGED_MUA_MAILLIST" {
+ weight = 0.0;
+ description = "Avoid false positives for FORGED_MUA_* in maillist";
+ }
+} \ No newline at end of file
diff --git a/conf/scores.d/phishing_group.conf b/conf/scores.d/phishing_group.conf
new file mode 100644
index 0000000..54a660a
--- /dev/null
+++ b/conf/scores.d/phishing_group.conf
@@ -0,0 +1,57 @@
+# Phishing rules scores
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "Phishing in emails";
+
+max_score = 10.0;
+
+symbols = {
+ "PHISHING" {
+ weight = 4.0;
+ description = "Phished URL";
+ one_shot = true;
+ }
+ "PHISHED_EXCLUDED" {
+ weight = 0.0;
+ description = "Phished URL found in exclusions list";
+ }
+ "PHISHED_OPENPHISH" {
+ weight = 7.0;
+ description = "Phished URL found in openphish.com";
+ }
+ "PHISHED_PHISHTANK" {
+ weight = 7.0;
+ description = "Phished URL found in phishtank.com";
+ }
+ HACKED_WP_PHISHING {
+ weight = 4.5;
+ description = "Phish message sent by hacked Wordpress instance";
+ }
+ REDIRECTOR_FALSE {
+ weight = 0.0;
+ description = "Phishing exclusion symbol for known redirectors";
+ }
+ URL_REDIRECTOR_NESTED {
+ weight = 1.0;
+ description = "URL redirector nested limit has been reached";
+ one_shot = true;
+ }
+ PHISHED_WHITELISTED {
+ weight = 0.0;
+ description = "Phishing exclusion symbol for known exceptions";
+ }
+}
diff --git a/conf/scores.d/policies_group.conf b/conf/scores.d/policies_group.conf
new file mode 100644
index 0000000..4a8bdb6
--- /dev/null
+++ b/conf/scores.d/policies_group.conf
@@ -0,0 +1,147 @@
+# Policies rules scores, includes SPF, DKIM, DMARC and ARC symbols
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "SPF, DKIM, DMARC, ARC";
+
+symbols = {
+ # SPF
+ "R_SPF_FAIL" {
+ weight = 1.0;
+ description = "SPF verification failed";
+ groups = ["spf"];
+ }
+ "R_SPF_SOFTFAIL" {
+ weight = 0.0;
+ description = "SPF verification soft-failed";
+ groups = ["spf"];
+ }
+ "R_SPF_NEUTRAL" {
+ weight = 0.0;
+ description = "SPF policy is neutral";
+ groups = ["spf"];
+ }
+ "R_SPF_ALLOW" {
+ weight = -0.2;
+ description = "SPF verification allows sending";
+ groups = ["spf"];
+ }
+ "R_SPF_DNSFAIL" {
+ weight = 0.0;
+ description = "SPF DNS failure";
+ groups = ["spf"];
+ }
+ "R_SPF_NA" {
+ weight = 0.0;
+ description = "Missing SPF record";
+ one_shot = true;
+ groups = ["spf"];
+ }
+ "R_SPF_PERMFAIL" {
+ weight = 0.0;
+ description = "SPF record is malformed or persistent DNS error";
+ groups = ["spf"];
+ }
+
+ # DKIM
+ "R_DKIM_REJECT" {
+ weight = 1.0;
+ description = "DKIM verification failed";
+ one_shot = true;
+ groups = ["dkim"];
+ }
+ "R_DKIM_TEMPFAIL" {
+ weight = 0.0;
+ description = "DKIM verification soft-failed";
+ groups = ["dkim"];
+ }
+ "R_DKIM_PERMFAIL" {
+ weight = 0.0;
+ description = "DKIM verification hard-failed (invalid)";
+ groups = ["dkim"];
+ }
+ "R_DKIM_ALLOW" {
+ weight = -0.2;
+ description = "DKIM verification succeed";
+ one_shot = true;
+ groups = ["dkim"];
+ }
+ "R_DKIM_NA" {
+ weight = 0.0;
+ description = "Missing DKIM signature";
+ one_shot = true;
+ groups = ["dkim"];
+ }
+
+ # DMARC
+ "DMARC_POLICY_ALLOW" {
+ weight = -0.5;
+ description = "DMARC permit policy";
+ groups = ["dmarc"];
+ }
+ "DMARC_POLICY_ALLOW_WITH_FAILURES" {
+ weight = -0.5;
+ description = "DMARC permit policy with DKIM/SPF failure";
+ groups = ["dmarc"];
+ }
+ "DMARC_POLICY_REJECT" {
+ weight = 2.0;
+ description = "DMARC reject policy";
+ groups = ["dmarc"];
+ }
+ "DMARC_POLICY_QUARANTINE" {
+ weight = 1.5;
+ description = "DMARC quarantine policy";
+ groups = ["dmarc"];
+ }
+ "DMARC_POLICY_SOFTFAIL" {
+ weight = 0.1;
+ description = "DMARC failed";
+ groups = ["dmarc"];
+ }
+ "DMARC_NA" {
+ weight = 0.0;
+ description = "No DMARC record";
+ groups = ["dmarc"];
+ }
+
+ # ARC
+ "ARC_ALLOW" {
+ weight = -1.0;
+ description = "ARC checks success";
+ groups = ["arc"];
+ }
+ "ARC_REJECT" {
+ weight = 1.0;
+ description = "ARC checks failed";
+ groups = ["arc"];
+ }
+ "ARC_INVALID" {
+ weight = 0.5;
+ description = "ARC structure invalid";
+ groups = ["arc"];
+ }
+ "ARC_DNSFAIL" {
+ weight = 0.0;
+ description = "ARC DNS error";
+ groups = ["arc"];
+ }
+ "ARC_NA" {
+ weight = 0.0;
+ description = "ARC signature absent";
+ groups = ["arc"];
+ }
+}
diff --git a/conf/scores.d/rbl_group.conf b/conf/scores.d/rbl_group.conf
new file mode 100644
index 0000000..ef29ed2
--- /dev/null
+++ b/conf/scores.d/rbl_group.conf
@@ -0,0 +1,250 @@
+# RBL rules scores
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "IP DNS lists";
+
+symbols = {
+
+ "DNSWL_BLOCKED" {
+ weight = 0.0;
+ description = "https://www.dnswl.org: Resolver blocked due to excessive queries";
+ groups = ["dnswl", "blocked"];
+ }
+ "RCVD_IN_DNSWL" {
+ weight = 0.0;
+ description = "Unrecognised result from https://www.dnswl.org";
+ groups = ["dnswl"];
+ }
+ "RCVD_IN_DNSWL_NONE" {
+ weight = 0.0;
+ description = "Sender listed at https://www.dnswl.org, no trust";
+ groups = ["dnswl"];
+ }
+ "RCVD_IN_DNSWL_LOW" {
+ weight = -0.1;
+ description = "Sender listed at https://www.dnswl.org, low trust";
+ groups = ["dnswl"];
+ }
+ "RCVD_IN_DNSWL_MED" {
+ weight = -0.2;
+ description = "Sender listed at https://www.dnswl.org, medium trust";
+ groups = ["dnswl"];
+ }
+ "RCVD_IN_DNSWL_HI" {
+ weight = -0.5;
+ description = "Sender listed at https://www.dnswl.org, high trust";
+ groups = ["dnswl"];
+ }
+
+ "DWL_DNSWL_BLOCKED" {
+ weight = 0.0;
+ description = "https://www.dnswl.org: Resolver blocked due to excessive queries (DWL)";
+ groups = ["dnswl", "blocked"];
+ }
+ "DWL_DNSWL" {
+ weight = 0.0;
+ description = "Unrecognised result from https://www.dnswl.org (DWL)";
+ groups = ["dnswl"];
+ }
+ "DWL_DNSWL_NONE" {
+ weight = 0.0;
+ description = "Message has a valid dkim signature originated from domain listed at https://www.dnswl.org, no trust";
+ groups = ["dnswl"];
+ }
+ "DWL_DNSWL_LOW" {
+ weight = -1.0;
+ description = "Message has a valid dkim signature originated from domain listed at https://www.dnswl.org, low trust";
+ groups = ["dnswl"];
+ }
+ "DWL_DNSWL_MED" {
+ weight = -2.0;
+ description = "Message has a valid dkim signature originated from domain listed at https://www.dnswl.org, medium trust";
+ groups = ["dnswl"];
+ }
+ "DWL_DNSWL_HI" {
+ weight = -3.5;
+ description = "Message has a valid dkim signature originated from domain listed at https://www.dnswl.org, high trust";
+ groups = ["dnswl"];
+ }
+
+ "RBL_SPAMHAUS" {
+ weight = 0.0;
+ description = "Unrecognised result from Spamhaus ZEN";
+ groups = ["spamhaus"];
+ }
+ "RBL_SPAMHAUS_SBL" {
+ weight = 4.0;
+ description = "From address is listed in Spamhaus SBL";
+ groups = ["spamhaus"];
+ }
+ "RBL_SPAMHAUS_CSS" {
+ weight = 2.0;
+ description = "From address is listed in Spamhaus CSS";
+ groups = ["spamhaus"];
+ }
+ "RBL_SPAMHAUS_XBL" {
+ weight = 4.0;
+ description = "From address is listed in Spamhaus XBL";
+ groups = ["spamhaus"];
+ }
+ "RBL_SPAMHAUS_PBL" {
+ weight = 2.0;
+ description = "From address is listed in Spamhaus PBL";
+ groups = ["spamhaus"];
+ }
+ "RBL_SPAMHAUS_DROP" {
+ weight = 7.0;
+ description = "From address is listed in Spamhaus DROP";
+ groups = ["spamhaus"];
+ }
+ "RBL_SPAMHAUS_BLOCKED_OPENRESOLVER" {
+ weight = 0.0;
+ description = "You are querying Spamhaus from an open resolver, please see https://www.spamhaus.org/returnc/pub/";
+ groups = ["spamhaus", "blocked"];
+ }
+ "RBL_SPAMHAUS_BLOCKED" {
+ weight = 0.0;
+ description = "You are exceeding the query limit, please see https://www.spamhaus.org/returnc/vol/";
+ groups = ["spamhaus", "blocked"];
+ }
+ "RECEIVED_SPAMHAUS_SBL" {
+ weight = 3.0;
+ description = "Received address is listed in Spamhaus SBL";
+ groups = ["spamhaus"];
+ one_shot = true;
+ }
+ "RECEIVED_SPAMHAUS_CSS" {
+ weight = 1.0;
+ description = "Received address is listed in Spamhaus CSS";
+ groups = ["spamhaus"];
+ one_shot = true;
+ }
+ "RECEIVED_SPAMHAUS_XBL" {
+ weight = 1.0;
+ description = "Received address is listed in Spamhaus XBL";
+ groups = ["spamhaus"];
+ one_shot = true;
+ }
+ "RECEIVED_SPAMHAUS_PBL" {
+ weight = 0.0;
+ description = "Received address is listed in Spamhaus PBL";
+ groups = ["spamhaus"];
+ one_shot = true;
+ }
+ "RECEIVED_SPAMHAUS_DROP" {
+ weight = 6.0;
+ description = "Received address is listed in Spamhaus DROP";
+ groups = ["spamhaus"];
+ one_shot = true;
+ }
+ "RECEIVED_SPAMHAUS_BLOCKED_OPENRESOLVER" {
+ weight = 0.0;
+ description = "You are querying Spamhaus from an open resolver, please see https://www.spamhaus.org/returnc/pub/";
+ groups = ["spamhaus", "blocked"];
+ }
+ "RECEIVED_SPAMHAUS_BLOCKED" {
+ weight = 0.0;
+ description = "You are exceeding the query limit, please see https://www.spamhaus.org/returnc/vol/";
+ groups = ["spamhaus", "blocked"];
+ }
+
+ "RBL_SENDERSCORE" {
+ weight = 2.0;
+ description = "From address is listed in senderscore.com BL";
+ }
+
+ "MAILSPIKE" {
+ weight = 0.0;
+ description = "Unrecognised result from Mailspike";
+ groups = ["mailspike"];
+ }
+ "RWL_MAILSPIKE_NEUTRAL" {
+ weight = 0.0;
+ description = "Neutral result from Mailspike";
+ groups = ["mailspike"];
+ }
+ "RBL_MAILSPIKE_WORST" {
+ weight = 2.0;
+ description = "From address is listed in Mailspike RBL - worst possible reputation";
+ groups = ["mailspike"];
+ }
+ "RBL_MAILSPIKE_VERYBAD" {
+ weight = 1.5;
+ description = "From address is listed in Mailspike RBL - very bad reputation";
+ groups = ["mailspike"];
+ }
+ "RBL_MAILSPIKE_BAD" {
+ weight = 1.0;
+ description = "From address is listed in Mailspike RBL - bad reputation";
+ groups = ["mailspike"];
+ }
+ "RWL_MAILSPIKE_POSSIBLE" {
+ weight = 0.0;
+ description = "From address is listed in Mailspike RWL - possibly legit";
+ groups = ["mailspike"];
+ }
+ "RWL_MAILSPIKE_GOOD" {
+ weight = -0.1;
+ description = "From address is listed in Mailspike RWL - good reputation";
+ groups = ["mailspike"];
+ }
+ "RWL_MAILSPIKE_VERYGOOD" {
+ weight = -0.2;
+ description = "From address is listed in Mailspike RWL - very good reputation";
+ groups = ["mailspike"];
+ }
+ "RWL_MAILSPIKE_EXCELLENT" {
+ weight = -0.4;
+ description = "From address is listed in Mailspike RWL - excellent reputation";
+ groups = ["mailspike"];
+ }
+
+ "RBL_SEM" {
+ weight = 1.0;
+ description = "From address is listed in Spameatingmonkey RBL";
+ groups = ["sem"];
+ }
+
+ "RBL_SEM_IPV6" {
+ weight = 1.0;
+ description = "From address is listed in Spameatingmonkey RBL (IPv6)";
+ groups = ["sem"];
+ }
+
+ "RBL_VIRUSFREE_BOTNET" {
+ weight = 2.0;
+ description = "From address is listed in virusfree.cz BL";
+ }
+
+ "RBL_NIXSPAM" {
+ weight = 4.0;
+ description = "From address is listed in NiX Spam (https://www.nixspam.net/)";
+ }
+
+ "RBL_BLOCKLISTDE" {
+ weight = 4.0;
+ description = "From address is listed in Blocklist (https://www.blocklist.de/)";
+ groups = ["blocklistde"];
+ }
+
+ "RECEIVED_BLOCKLISTDE" {
+ weight = 3.0;
+ description = "Received address is listed in Blocklist (https://www.blocklist.de/)";
+ groups = ["blocklistde"];
+ one_shot = true;
+ }
+}
diff --git a/conf/scores.d/statistics_group.conf b/conf/scores.d/statistics_group.conf
new file mode 100644
index 0000000..b46fa7a
--- /dev/null
+++ b/conf/scores.d/statistics_group.conf
@@ -0,0 +1,29 @@
+# Bayes and statistics rules scores
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "Statistical symbols";
+
+symbols = {
+ "BAYES_SPAM" {
+ weight = 5.1;
+ description = "Message probably spam, probability: ";
+ }
+ "BAYES_HAM" {
+ weight = -3.0;
+ description = "Message probably ham, probability: ";
+ }
+} \ No newline at end of file
diff --git a/conf/scores.d/subject_group.conf b/conf/scores.d/subject_group.conf
new file mode 100644
index 0000000..1cc2e0c
--- /dev/null
+++ b/conf/scores.d/subject_group.conf
@@ -0,0 +1,23 @@
+# Subject rules scores
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "Subject filters";
+
+max_score = 6.0;
+
+symbols = {
+}
diff --git a/conf/scores.d/surbl_group.conf b/conf/scores.d/surbl_group.conf
new file mode 100644
index 0000000..25e8ed7
--- /dev/null
+++ b/conf/scores.d/surbl_group.conf
@@ -0,0 +1,259 @@
+# URIBL rules scores
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "URL DNS lists";
+
+max_score = 12.5;
+
+symbols = {
+ "SURBL_BLOCKED" {
+ weight = 0.0;
+ description = "SURBL: query blocked by policy/overusage";
+ one_shot = true;
+ groups = ["surblorg", "blocked"];
+ }
+ "PH_SURBL_MULTI" {
+ weight = 7.5;
+ description = "A domain in the message is listed in SURBL as phishing";
+ one_shot = true;
+ groups = ["surblorg", "phishing"];
+ }
+ "MW_SURBL_MULTI" {
+ weight = 7.5;
+ description = "A domain in the message is listed in SURBL as malware";
+ one_shot = true;
+ groups = ["surblorg"];
+ }
+ "ABUSE_SURBL" {
+ weight = 5.0;
+ description = "A domain in the message is listed in SURBL as abused";
+ one_shot = true;
+ groups = ["surblorg"];
+ }
+ "CRACKED_SURBL" {
+ weight = 5.0;
+ description = "A domain in the message is listed in SURBL as cracked";
+ one_shot = true;
+ groups = ["surblorg"];
+ }
+
+ "RSPAMD_URIBL" {
+ weight = 4.5;
+ description = "Rspamd uribl, bl.rspamd.com";
+ one_shot = true;
+ groups = ["rspamdbl"];
+ }
+ "RSPAMD_EMAILBL" {
+ weight = 2.5;
+ description = "Rspamd emailbl, bl.rspamd.com";
+ one_shot = true;
+ groups = ["rspamdbl"];
+ }
+
+ "MSBL_EBL" {
+ weight = 7.5;
+ description = "MSBL emailbl (https://www.msbl.org/)";
+ one_shot = true;
+ groups = ["ebl"];
+ }
+
+ "MSBL_EBL_GREY" {
+ weight = 0.5; # TODO: test it
+ description = "MSBL emailbl grey list (https://www.msbl.org/)";
+ one_shot = true;
+ groups = ["ebl"];
+ }
+
+ "SEM_URIBL_UNKNOWN" {
+ weight = 0.0;
+ description = "Unrecognised result from Spameatingmonkey URIBL";
+ one_shot = true;
+ groups = ["sem"];
+ }
+ "SEM_URIBL" {
+ weight = 3.5;
+ description = "A domain in the message is listed in Spameatingmonkey URIBL";
+ one_shot = true;
+ groups = ["sem"];
+ }
+
+ "SEM_URIBL_FRESH15_UNKNOWN" {
+ weight = 0.0;
+ description = "Unrecognised result from Spameatingmonkey Fresh15 URIBL";
+ one_shot = true;
+ groups = ["sem"];
+ }
+ "SEM_URIBL_FRESH15" {
+ weight = 3.0;
+ description = "A domain in the message is listed in Spameatingmonkey Fresh15 URIBL (registered in the past 15 days, .AERO,.BIZ,.COM,.INFO,.NAME,.NET,.PRO,.SK,.TEL,.US only)";
+ one_shot = true;
+ groups = ["sem"];
+ }
+
+ "DBL" {
+ weight = 0.0;
+ description = "Unrecognised result from Spamhaus DBL";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "DBL_SPAM" {
+ weight = 6.5;
+ description = "A domain in the message is listed in Spamhaus DBL as spam";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "DBL_PHISH" {
+ weight = 7.5;
+ description = "A domain in the message is listed in Spamhaus DBL as phishing";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "DBL_MALWARE" {
+ weight = 7.5;
+ description = "A domain in the message is listed in Spamhaus DBL as malware";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "DBL_BOTNET" {
+ weight = 7.5;
+ description = "A domain in the message is listed in Spamhaus DBL as botnet C&C";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "DBL_ABUSE" {
+ weight = 5.0;
+ description = "A domain in the message is listed in Spamhaus DBL as abused legit spam";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "DBL_ABUSE_REDIR" {
+ weight = 5.0;
+ description = "A domain in the message is listed in Spamhaus DBL as spammed redirector domain";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "DBL_ABUSE_PHISH" {
+ weight = 6.5;
+ description = "A domain in the message is listed in Spamhaus DBL as abused legit phish";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "DBL_ABUSE_MALWARE" {
+ weight = 6.5;
+ description = "A domain in the message is listed in Spamhaus DBL as abused legit malware";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "DBL_ABUSE_BOTNET" {
+ weight = 6.5;
+ description = "A domain in the message is listed in Spamhaus DBL as abused legit botnet C&C";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "DBL_PROHIBIT" {
+ weight = 0.0;
+ description = "DBL uribl IP queries prohibited!";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "DBL_BLOCKED_OPENRESOLVER" {
+ weight = 0.0;
+ description = "You are querying Spamhaus from an open resolver, please see https://www.spamhaus.org/returnc/pub/";
+ one_shot = true;
+ groups = ["spamhaus", "blocked"];
+ }
+ "DBL_BLOCKED" {
+ weight = 0.0;
+ description = "You are exceeding the query limit, please see https://www.spamhaus.org/returnc/vol/";
+ one_shot = true;
+ groups = ["spamhaus", "blocked"];
+ }
+
+ "URIBL_MULTI" {
+ weight = 0.0;
+ description = "Unrecognised result from URIBL.com";
+ one_shot = true;
+ groups = ["uribl"];
+ }
+ "URIBL_BLOCKED" {
+ weight = 0.0;
+ description = "URIBL.com: query refused, likely due to policy/overusage";
+ one_shot = true;
+ groups = ["uribl", "blocked"];
+ }
+ "URIBL_BLACK" {
+ weight = 7.5;
+ description = "A domain in the message is listed in URIBL.com black";
+ one_shot = true;
+ groups = ["uribl"];
+ }
+ "URIBL_RED" {
+ weight = 3.5;
+ description = "A domain in the message is listed in URIBL.com red";
+ one_shot = true;
+ groups = ["uribl"];
+ }
+ "URIBL_GREY" {
+ weight = 1.5;
+ description = "A domain in the message is listed in URIBL.com grey";
+ one_shot = true;
+ groups = ["uribl"];
+ }
+
+ "SPAMHAUS_ZEN_URIBL" {
+ ignore = true;
+ weight = 0.0;
+ description = "Unrecognised result from Spamhaus ZEN URIBL";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "URIBL_SBL" {
+ ignore = true;
+ weight = 6.5;
+ description = "A domain in the message body resolves to an IP listed in Spamhaus SBL";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "URIBL_SBL_CSS" {
+ ignore = true;
+ weight = 5.0;
+ description = "A domain in the message body resolves to an IP listed in Spamhaus CSS";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "URIBL_XBL" {
+ ignore = true;
+ weight = 3.0;
+ description = "A domain in the message body resolves to an IP listed in Spamhaus XBL";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "URIBL_PBL" {
+ ignore = true;
+ weight = 0.01;
+ description = "A domain in the message body resolves to an IP listed in Spamhaus PBL";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+ "URIBL_DROP" {
+ ignore = true;
+ weight = 5.0;
+ description = "A domain in the message body resolves to an IP listed in Spamhaus DROP";
+ one_shot = true;
+ groups = ["spamhaus"];
+ }
+}
diff --git a/conf/scores.d/whitelist_group.conf b/conf/scores.d/whitelist_group.conf
new file mode 100644
index 0000000..c0d2044
--- /dev/null
+++ b/conf/scores.d/whitelist_group.conf
@@ -0,0 +1,63 @@
+# Whitelist rules scores
+#
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+description = "White lists group";
+
+max_score = 10.0;
+
+symbols = {
+ "WHITELIST_SPF" {
+ weight = -1.0;
+ description = "Mail comes from the whitelisted domain and has a valid SPF policy";
+ groups = ["spf"];
+ }
+ "BLACKLIST_SPF" {
+ weight = 1.0;
+ description = "Mail comes from the whitelisted domain and has no valid SPF policy";
+ groups = ["spf"];
+ }
+ "WHITELIST_DKIM" {
+ weight = -1.0;
+ description = "Mail comes from the whitelisted domain and has a valid DKIM signature";
+ groups = ["dkim"];
+ }
+ "BLACKLIST_DKIM" {
+ weight = 2.0;
+ description = "Mail comes from the whitelisted domain and has non-valid DKIM signature";
+ groups = ["dkim"];
+ }
+ "WHITELIST_SPF_DKIM" {
+ weight = -3.0;
+ description = "Mail comes from the whitelisted domain and has valid SPF and DKIM policies";
+ groups = ["spf", "dkim"];
+ }
+ "BLACKLIST_SPF_DKIM" {
+ weight = 3.0;
+ description = "Mail comes from the whitelisted domain and has no valid SPF policy or a bad DKIM signature";
+ groups = ["spf", "dkim"];
+ }
+ "WHITELIST_DMARC" {
+ weight = -7.0;
+ description = "Mail comes from the whitelisted domain and has valid DMARC and DKIM policies";
+ groups = ["dmarc", "spf", "dkim"];
+ }
+ "BLACKLIST_DMARC" {
+ weight = 6.0;
+ description = "Mail comes from the whitelisted domain and has failed DMARC and DKIM policies";
+ groups = ["dmarc", "spf", "dkim"];
+ }
+}
diff --git a/conf/settings.conf b/conf/settings.conf
new file mode 100644
index 0000000..95d71e4
--- /dev/null
+++ b/conf/settings.conf
@@ -0,0 +1,66 @@
+# Settings setup
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/settings.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/settings.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation: https://rspamd.com/doc/configuration/settings.html
+
+# If you want to use settings map, then please define something like:
+#
+# settings = "http://example.com/settings.json"
+#
+# in rspamd.conf.override
+
+settings {
+ # Some examples below (define in local.d/settings.conf without `settings {}`!)
+ #some_users {
+ # id = "some_users";
+ # priority = high;
+ # from = "@example.com";
+ # rcpt = "admin";
+ # rcpt = "/user.*/";
+ # ip = "172.16.0.0/16";
+ # user = "@example.net";
+ # request_header = {
+ # "MTA-Tag" = "\.example\.net$";
+ # }
+ # apply {
+ # symbol1 = 10.0;
+ # symbol2 = 0.0;
+ # actions {
+ # reject = 100.0;
+ # greylist = 10.0;
+ # "add header" = 5.0; # Please note the space, NOT an underscore
+ # }
+ # }
+ # Always add these symbols when settings rule has matched
+ # symbols [
+ # "symbol2", "symbol4"
+ # ]
+ #}
+ #whitelist {
+ # priority = low;
+ # rcpt = "postmaster@example.com";
+ # want_spam = yes;
+ #}
+ # Disable some checks for authenticated users
+ #authenticated {
+ # priority = high;
+ # authenticated = yes;
+ # apply {
+ # groups_disabled = ["rbl", "spf"];
+ # }
+ #}
+ # End of examples
+
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/settings.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/settings.conf"
+} \ No newline at end of file
diff --git a/conf/statistic.conf b/conf/statistic.conf
new file mode 100644
index 0000000..0ba8302
--- /dev/null
+++ b/conf/statistic.conf
@@ -0,0 +1,59 @@
+# Statistics setup
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/statistic.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/statistic.conf' to strictly override all
+# parameters defined inside this section
+#
+# If you just need to change the default bayes classifier, you can also use
+# 'local.d/classifier-bayes.conf' or 'override.d/classifier-bayes.conf'. But
+# never ever use both `statistic.conf` and `classifier-bayes.conf` locals files
+# together!
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation: https://rspamd.com/doc/configuration/statistic.html
+
+classifier "bayes" {
+ tokenizer {
+ name = "osb";
+ }
+ cache {
+ }
+ new_schema = true; # Always use new schema
+ store_tokens = false; # Redefine if storing of tokens is desired
+ signatures = false; # Store learn signatures
+ #per_user = true; # Enable per user classifier
+ min_tokens = 11;
+ backend = "redis";
+ min_learns = 200;
+
+ statfile {
+ symbol = "BAYES_HAM";
+ spam = false;
+ }
+ statfile {
+ symbol = "BAYES_SPAM";
+ spam = true;
+ }
+ learn_condition = 'return require("lua_bayes_learn").can_learn';
+
+ # Autolearn sample
+ # autolearn {
+ # spam_threshold = 6.0; # When to learn spam (score >= threshold and action is reject)
+ # junk_threshold = 4.0; # When to learn spam (score >= threshold and action is rewrite subject or add header, and has two or more positive results)
+ # ham_threshold = -0.5; # When to learn ham (score <= threshold and action is no action, and score is negative or has three or more negative results)
+ # check_balance = true; # Check spam and ham balance
+ # min_balance = 0.9; # Keep diff for spam/ham learns for at least this value
+ #}
+
+ .include(try=true; priority=1) "$LOCAL_CONFDIR/local.d/classifier-bayes.conf"
+ .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/classifier-bayes.conf"
+}
+
+.include(try=true; priority=1) "$LOCAL_CONFDIR/local.d/statistic.conf"
+.include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/statistic.conf"
diff --git a/conf/worker-controller.inc b/conf/worker-controller.inc
new file mode 100644
index 0000000..1fdffff
--- /dev/null
+++ b/conf/worker-controller.inc
@@ -0,0 +1,19 @@
+# Controller worker setup
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/worker-controller.inc' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/worker-controller.inc' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+# Module documentation: https://rspamd.com/doc/workers/controller.html
+
+count = 1;
+password = "q1";
+secure_ip = "127.0.0.1";
+secure_ip = "::1";
+static_dir = "${WWWDIR}";
diff --git a/conf/worker-fuzzy.inc b/conf/worker-fuzzy.inc
new file mode 100644
index 0000000..5cf788c
--- /dev/null
+++ b/conf/worker-fuzzy.inc
@@ -0,0 +1,22 @@
+# Fuzzy storage worker setup
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/worker-fuzzy.inc' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/worker-fuzzy.inc' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+# Module documentation: https://rspamd.com/doc/workers/fuzzy_storage.html
+
+backend = "redis";
+
+# For sqlite stuff
+#backend = "sqlite";
+#hash_file = "${DBDIR}/fuzzy.db";
+
+expire = 90d;
+allow_update = ["localhost"]; \ No newline at end of file
diff --git a/conf/worker-normal.inc b/conf/worker-normal.inc
new file mode 100644
index 0000000..285a17a
--- /dev/null
+++ b/conf/worker-normal.inc
@@ -0,0 +1,15 @@
+# Normal scanner worker setup
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/worker-normal.inc' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/worker-normal.inc' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+# Module documentation: https://rspamd.com/doc/workers/normal.html
+
+mime = true;
diff --git a/conf/worker-proxy.inc b/conf/worker-proxy.inc
new file mode 100644
index 0000000..7f67238
--- /dev/null
+++ b/conf/worker-proxy.inc
@@ -0,0 +1,35 @@
+# Proxy worker setup
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/worker-proxy.inc' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/worker-proxy.inc' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+# Module documentation: https://rspamd.com/doc/workers/rspamd_proxy.html
+
+milter = yes; # Enable milter mode
+
+# This timeout is mostly specific to the milter mode.
+# Please also bear in mind that if this timeout is longer than `task_timeout`,
+# your messages might be processed for much longer due to this timeout (this is
+# true for self-scan mode).
+# If this behaviour is not desired, then it is recommended to reduce and adjust this
+# value accordingly
+timeout = 60s;
+
+upstream "local" {
+ default = yes;
+ hosts = "localhost";
+}
+
+count = 1; # Do not spawn too many processes of this type
+max_retries = 5; # How many times master is queried in case of failure
+discard_on_reject = false; # Discard message instead of rejection
+quarantine_on_reject = false; # Tell MTA to quarantine rejected messages
+spam_header = "X-Spam"; # Use the specific spam header
+reject_message = "Spam message rejected"; # Use custom rejection message
diff --git a/config.h.in b/config.h.in
new file mode 100644
index 0000000..59c70c6
--- /dev/null
+++ b/config.h.in
@@ -0,0 +1,416 @@
+#ifndef RSPAMD_CONFIG_H_IN
+#define RSPAMD_CONFIG_H_IN
+
+
+#cmakedefine BUILD_STATIC 1
+#cmakedefine CURL_FOUND 1
+#cmakedefine DEBUG_MODE 1
+#cmakedefine GIT_VERSION 1
+#cmakedefine GLIB_HASH_COMPAT 1
+#cmakedefine GLIB_RE_COMPAT 1
+#cmakedefine GLIB_UNISCRIPT_COMPAT 1
+#cmakedefine HAVE_ARPA_INET_H 1
+#cmakedefine HAVE_ATOMIC_BUILTINS 1
+#cmakedefine HAVE_CLOCK_GETCPUCLOCKID 1
+#cmakedefine HAVE_CLOCK_GETTIME 1
+#cmakedefine HAVE_CLOCK_PROCESS_CPUTIME_ID 1
+#cmakedefine HAVE_CLOCK_VIRTUAL 1
+#cmakedefine HAVE_CPUID_H 1
+#cmakedefine HAVE_CTYPE_H 1
+#cmakedefine HAVE_DIRENT_H 1
+#cmakedefine HAVE_DIRFD 1
+#cmakedefine HAVE_ENDIAN_H 1
+#cmakedefine HAVE_FALLOCATE 1
+#cmakedefine HAVE_FCNTL_H 1
+#cmakedefine HAVE_FETCH_H 1
+#cmakedefine HAVE_FIPS_MODE 1
+#cmakedefine HAVE_FFSLL 1
+#cmakedefine HAVE_FLOCK 1
+#cmakedefine HAVE_FPATHCONF 1
+#cmakedefine HAVE_GETPAGESIZE 1
+#cmakedefine HAVE_GLOB_H 1
+#cmakedefine HAVE_GRP_H 1
+#cmakedefine HAVE_INTTYPES_H 1
+#cmakedefine HAVE_IPV6_V6ONLY 1
+#cmakedefine HAVE_LIBGEN_H 1
+#cmakedefine HAVE_LIBUTIL_H 1
+#cmakedefine HAVE_LOCALE_H 1
+#cmakedefine HAVE_MACHINE_ENDIAN_H 1
+#cmakedefine HAVE_MAXPATHLEN 1
+#cmakedefine HAVE_FMEMOPEN 1
+#cmakedefine HAVE_MEMRCHR 1
+#cmakedefine HAVE_MKSTEMP 1
+#cmakedefine HAVE_MMAP_ANON 1
+#cmakedefine HAVE_NANOSLEEP 1
+#cmakedefine HAVE_NETDB_H 1
+#cmakedefine HAVE_NETINET_IN_H 1
+#cmakedefine HAVE_NETINET_TCP_H 1
+#cmakedefine HAVE_NFTW 1
+#cmakedefine HAVE_OCLOEXEC 1
+#cmakedefine HAVE_ONOFOLLOW 1
+#cmakedefine HAVE_OPENMEMSTREAM 1
+#cmakedefine HAVE_PATH_MAX 1
+
+/* OSX has broken JIT support in PCRE, disable it */
+#cmakedefine HAVE_PCRE_JIT 1
+#cmakedefine HAVE_PCRE_JIT_FAST 1
+
+#cmakedefine HAVE_PIDFILE 1
+#cmakedefine HAVE_PIDFILE_FILENO 1
+#cmakedefine HAVE_POLL_H 1
+#cmakedefine HAVE_POSIX_FALLOCATE 1
+#cmakedefine HAVE_PTHREAD_PROCESS_SHARED 1
+#cmakedefine HAVE_PWD_H 1
+#cmakedefine HAVE_RDTSC 1
+#cmakedefine HAVE_READAHEAD 1
+#cmakedefine HAVE_READPASSPHRASE_H 1
+#cmakedefine HAVE_RECVMMSG 1
+#cmakedefine HAVE_RUSAGE_SELF 1
+#cmakedefine HAVE_SA_SIGINFO 1
+#cmakedefine HAVE_SANE_SHMEM 1
+#cmakedefine HAVE_SCHED_YIELD 1
+#cmakedefine HAVE_SC_NPROCESSORS_ONLN 1
+#cmakedefine HAVE_SETPROCTITLE 1
+#cmakedefine HAVE_SIGALTSTACK 1
+#cmakedefine HAVE_SIGINFO_H 1
+#cmakedefine HAVE_SOCK_SEQPACKET 1
+#cmakedefine HAVE_SSL_TLSEXT_HOSTNAME 1
+#cmakedefine HAVE_STDBOOL_H 1
+#cmakedefine HAVE_STDINT_H 1
+#cmakedefine HAVE_STDIO_H 1
+#cmakedefine HAVE_STDLIB_H 1
+#cmakedefine HAVE_STRINGS_H 1
+#cmakedefine HAVE_STRING_H 1
+#cmakedefine HAVE_SYSLOG_H 1
+#cmakedefine HAVE_SYS_CDEFS_H 1
+#cmakedefine HAVE_SYS_ENDIAN_H 1
+#cmakedefine HAVE_SYS_EVENTFD_H 1
+#cmakedefine HAVE_SYS_FILE_H 1
+#cmakedefine HAVE_SYS_MMAN_H 1
+#cmakedefine HAVE_SYS_PARAM_H 1
+#cmakedefine HAVE_SYS_RESOURCE_H 1
+#cmakedefine HAVE_SYS_SOCKET_H 1
+#cmakedefine HAVE_SYS_STAT_H 1
+#cmakedefine HAVE_SYS_TIMEB_H 1
+#cmakedefine HAVE_SYS_TYPES_H 1
+#cmakedefine HAVE_SYS_UCONTEXT_H 1
+#cmakedefine HAVE_SYS_UIO_H 1
+#cmakedefine HAVE_SYS_UN_H 1
+#cmakedefine HAVE_SYS_WAIT_H 1
+#cmakedefine HAVE_TANH 1
+#cmakedefine HAVE_TERMIOS_H 1
+#cmakedefine HAVE_TIME_H 1
+#cmakedefine HAVE_UCONTEXT_H 1
+#cmakedefine HAVE_UNISTD_H 1
+#cmakedefine PARAM_H_HAS_BITSET 1
+#cmakedefine WITH_GPERF_TOOLS 1
+#cmakedefine WITH_HYPERSCAN 1
+#cmakedefine WITH_JEMALLOC 1
+#cmakedefine WITH_LUA 1
+#cmakedefine WITH_LUAJIT 1
+#cmakedefine WITH_PCRE2 1
+#cmakedefine WITH_SNOWBALL 1
+#cmakedefine WITH_SQLITE 1
+#cmakedefine WITH_LUA_TRACE 1
+#cmakedefine WITH_LUA_REPL 1
+#cmakedefine WITH_FASTTEXT 1
+#cmakedefine BACKWARD_ENABLE 1
+
+#cmakedefine DISABLE_PTHREAD_MUTEX 1
+
+/* Detect endianness */
+
+#ifdef HAVE_ENDIAN_H
+ #include <endian.h>
+#elif defined(HAVE_SYS_ENDIAN_H)
+ #include <sys/endian.h>
+#elif defined(HAVE_MACHINE_ENDIAN_H)
+ #include <machine/endian.h>
+#elif defined(__sun)
+ #include <sys/byteorder.h>
+ #ifndef LITTLE_ENDIAN
+ #define LITTLE_ENDIAN 1234
+ #endif
+ #ifndef BIG_ENDIAN
+ #define BIG_ENDIAN 4321
+ #endif
+ #ifdef _LITTLE_ENDIAN
+ #define BYTE_ORDER LITTLE_ENDIAN
+ #else
+ #define BYTE_ORDER BIG_ENDIAN
+ #endif
+#endif
+
+#ifndef BYTE_ORDER
+
+#ifndef LITTLE_ENDIAN
+ #define LITTLE_ENDIAN 1234
+#endif
+#ifndef BIG_ENDIAN
+ #define BIG_ENDIAN 4321
+#endif
+
+#if defined(__BYTE_ORDER) && __BYTE_ORDER == __BIG_ENDIAN || \
+ defined(__BIG_ENDIAN__) || \
+ defined(__ARMEB__) || \
+ defined(__THUMBEB__) || \
+ defined(__AARCH64EB__) || \
+ defined(_MIBSEB) || defined(__MIBSEB) || defined(__MIBSEB__)
+#define BYTE_ORDER BIG_ENDIAN
+#elif defined(__BYTE_ORDER) && __BYTE_ORDER == __LITTLE_ENDIAN || \
+ defined(__LITTLE_ENDIAN__) || \
+ defined(__ARMEL__) || \
+ defined(__THUMBEL__) || \
+ defined(__AARCH64EL__) || \
+ defined(_MIPSEL) || defined(__MIPSEL) || defined(__MIPSEL__)
+#define BYTE_ORDER LITTLE_ENDIAN
+#else
+#error "I don't know what architecture this is!"
+#endif
+
+#endif /* BYTE_ORDER */
+
+#define RSPAMD_SHAREDIR "${SHAREDIR}"
+#define RSPAMD_CONFDIR "${CONFDIR}"
+#define RSPAMD_LOCAL_CONFDIR "${LOCAL_CONFDIR}"
+#define RSPAMD_RUNDIR "${RUNDIR}"
+#define RSPAMD_LOGDIR "${LOGDIR}"
+#define RSPAMD_DBDIR "${DBDIR}"
+#define RSPAMD_PLUGINSDIR "${PLUGINSDIR}"
+#define RSPAMD_LUALIBDIR "${LUALIBDIR}"
+#define RSPAMD_RULESDIR "${RULESDIR}"
+#define RSPAMD_WWWDIR "${WWWDIR}"
+#define RSPAMD_PREFIX "${CMAKE_INSTALL_PREFIX}"
+#define RSPAMD_LIBDIR "${RSPAMD_LIBDIR}"
+
+#define RSPAMD_VERSION_MAJOR "${RSPAMD_VERSION_MAJOR}"
+#define RSPAMD_VERSION_MINOR "${RSPAMD_VERSION_MINOR}"
+#define RSPAMD_VERSION_PATCH "${RSPAMD_VERSION_PATCH}"
+
+#define RSPAMD_VERSION_MAJOR_NUM ${RSPAMD_VERSION_MAJOR_NUM}
+#define RSPAMD_VERSION_MINOR_NUM ${RSPAMD_VERSION_MINOR_NUM}
+#define RSPAMD_VERSION_PATCH_NUM ${RSPAMD_VERSION_PATCH_NUM}
+
+#define RSPAMD_VERSION_BRANCH "${RSPAMD_VERSION_MAJOR}"
+
+#if defined(GIT_VERSION) && GIT_VERSION == 1
+# define RVERSION "${RSPAMD_VERSION}"
+# define RSPAMD_VERSION_FULL "${RSPAMD_VERSION}_${RSPAMD_ID}"
+# define RID "${RSPAMD_ID}"
+# define RSPAMD_VERSION_NUM 0x${RSPAMD_VERSION_MAJOR_NUM}${RSPAMD_VERSION_MINOR_NUM}00${RSPAMD_ID}ULL
+#else
+# define RSPAMD_VERSION_FULL "${RSPAMD_VERSION}"
+# define RVERSION "${RSPAMD_VERSION}"
+# define RSPAMD_VERSION_NUM 0x${RSPAMD_VERSION_MAJOR_NUM}${RSPAMD_VERSION_MINOR_NUM}000000000ULL
+# define RID "release"
+#endif
+
+#define RSPAMD_MASTER_SITE_URL "${RSPAMD_MASTER_SITE_URL}"
+
+#define MODULES_NUM ${RSPAMD_MODULES_NUM}
+
+/* sys/types */
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+
+/* cdefs */
+#ifdef HAVE_SYS_CDEFS_H
+#include <sys/cdefs.h>
+#endif
+
+/* sys/param */
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+
+/* stdint */
+#ifdef HAVE_STDINT_H
+#include <stdint.h>
+#elif defined(HAVE_INTTYPES_H)
+#include <inttypes.h>
+#endif
+
+/* stdbool */
+#ifdef HAVE_STDBOOL_H
+#include <stdbool.h>
+#endif
+
+/* stdlib */
+#ifdef HAVE_STDLIB_H
+#include <stdlib.h>
+#endif
+
+/* stdio */
+#ifdef HAVE_STDIO_H
+#include <stdio.h>
+#endif
+
+/* time */
+#ifdef HAVE_TIME_H
+#include <time.h>
+#endif
+
+/* string */
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#ifdef HAVE_STRINGS_H
+#include <strings.h>
+#endif
+
+#include <errno.h>
+
+#include <glib.h>
+#include <gmodule.h>
+
+#ifndef PARAM_H_HAS_BITSET
+/* Bit map related macros. */
+#ifndef NBBY
+# define NBBY 8 /* number of bits in a byte */
+#endif
+#define setbit(a, \
+ i) (((unsigned char *)(a))[(i) / NBBY] |= 1 << ((i) % NBBY))
+#define clrbit(a, \
+ i) (((unsigned char *)(a))[(i) / NBBY] &= ~(1 << ((i) % NBBY)))
+#define isset(a,i) \
+ (((const unsigned char *)(a))[(i) / NBBY] & (1 << ((i) % NBBY)))
+#define isclr(a,i) \
+ ((((const unsigned char *)(a))[(i) / NBBY] & (1 << ((i) % NBBY))) == 0)
+#endif
+
+#ifdef _MSC_VER
+# define RSPAMD_PACKED(name) \
+ __pragma(pack(push, 1)) struct name __pragma(pack(pop))
+#elif defined(__GNUC__)
+# define RSPAMD_PACKED(name) struct __attribute__((packed)) name
+#else
+# define RSPAMD_PACKED(name) struct name
+#endif
+
+#ifndef RSPAMD_ALIGNED
+#if defined(_MSC_VER)
+# define RSPAMD_ALIGNED(x) __declspec(align(x))
+# define RSPAMD_OPTIMIZE(x)
+# define RSPAMD_ALWAYS_INLINE
+# define RSPAMD_PURE_FUNCTION
+#elif defined(__GNUC__)
+# define RSPAMD_ALIGNED(x) __attribute__((aligned(x)))
+# define RSPAMD_ALWAYS_INLINE __attribute__((always_inline))
+# define RSPAMD_PURE_FUNCTION __attribute__((pure))
+#ifndef __clang__
+# define RSPAMD_OPTIMIZE(x) __attribute__((__optimize__ (x)))
+#else
+# define RSPAMD_OPTIMIZE(x)
+#endif
+#else
+/* Unknown compiler */
+# define RSPAMD_ALIGNED(x)
+# define RSPAMD_OPTIMIZE(x)
+# define RSPAMD_ALWAYS_INLINE
+# define RSPAMD_PURE_FUNCTION
+#endif
+#endif
+
+#ifndef __cplusplus
+# ifdef G_ALIGNOF
+# define RSPAMD_ALIGNOF G_ALIGNOF
+# else
+# define RSPAMD_ALIGNOF(t) _Alignof(t)
+# endif
+#else
+/* glib G_ALIGNOF nor C11 _Alignof are not good enough for C++, nuff said... */
+# define RSPAMD_ALIGNOF(t) alignof(t)
+#endif
+
+/* Address sanitizer */
+#ifdef __clang__
+# if __has_feature(address_sanitizer)
+/* emulate gcc's __SANITIZE_ADDRESS__ flag */
+# define __SANITIZE_ADDRESS__
+# define RSPAMD_NO_SANITIZE \
+ __attribute__((no_sanitize("address", "hwaddress")))
+# else
+# define RSPAMD_NO_SANITIZE
+# endif
+#elif defined(__GNUC__)
+/* GCC based */
+# if defined(__has_attribute)
+# if __has_attribute(__no_sanitize_address__)
+# define RSPAMD_NO_SANITIZE __attribute__((no_sanitize_address))
+# else
+# define RSPAMD_NO_SANITIZE
+# endif
+# else
+# define RSPAMD_NO_SANITIZE
+# endif
+#else
+# define RSPAMD_NO_SANITIZE
+#endif
+
+
+#ifndef BITSPERBYTE
+# define BITSPERBYTE (NBBY * sizeof (char))
+#endif
+#ifndef NBYTES
+# define NBYTES(nbits) (((nbits) + BITSPERBYTE - 1) / BITSPERBYTE)
+#endif
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern uint64_t ottery_rand_uint64(void);
+#define UCL_RANDOM_FUNCTION ottery_rand_uint64()
+#ifdef __cplusplus
+}
+#endif
+
+
+/* Disable slab allocator if jemalloc is already in the system */
+#if defined(WITH_JEMALLOC) || defined(__FreeBSD__) || \
+ (defined(__NetBSD__) && __NetBSD_Version__ >= 500000000)
+#if 0
+ #define g_slice_alloc(sz) g_malloc(sz)
+ #define g_slice_alloc0(sz) g_malloc0(sz)
+ #define g_slice_free1(sz, p) g_free(p)
+#endif
+#endif
+
+#ifdef __cplusplus
+ #define RSPAMD_CONSTRUCTOR(f) \
+ static void f(void) noexcept; \
+ struct f##_t_ { f##_t_(void) noexcept { f(); } }; static f##_t_ f##_; \
+ static void f(void) noexcept
+#else
+#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 7)
+ #define RSPAMD_CONSTRUCTOR(f) \
+ static void f(void) __attribute__((constructor)); \
+ static void f(void)
+ #define RSPAMD_DESTRUCTOR(f) \
+ static void f(void) __attribute__((destructor)); \
+ static void f(void)
+#else
+ /* In fact, everything else is not supported ¯\_(ツ)_/¯ */
+ #error incompatible compiler found, need gcc > 2.7 or clang
+#endif
+#endif /* __cplusplus */
+
+#ifdef __GNUC__
+#define RSPAMD_CONST_FUNCTION __attribute__ ((const))
+#else
+#define RSPAMD_CONST_FUNCTION
+#endif
+
+#ifdef __GNUC__
+#define RSPAMD_UNREACHABLE __builtin_unreachable()
+#else
+#define RSPAMD_UNREACHABLE abort()
+#endif
+
+#define HAVE_OPENSSL 1
+#define HAVE_MATH_H 1
+
+
+#endif
diff --git a/contrib/DEPENDENCY_INFO.md b/contrib/DEPENDENCY_INFO.md
new file mode 100644
index 0000000..300a38c
--- /dev/null
+++ b/contrib/DEPENDENCY_INFO.md
@@ -0,0 +1,41 @@
+# Rspamd Dependency Info
+
+| Name | Version | License | Patched | Notes |
+|------------------------|---------|---------------------|---------|---------------------|
+| aho-corasick | ? | LGPL-3.0 | YES | lowercase support |
+| cdb | 1.1.0 | Public Domain / CC0 | NO | |
+| hiredis | 0.13.3 | BSD-3-Clause | YES | many changes |
+| libev | 4.33 | BSD-2-Clause | YES | many changes |
+| lc-btrie | ? | BSD-3-Clause | YES | mempool support |
+| libottery | ? | Public Domain / CC0 | YES | many changes |
+| librdns | ? | BSD-2-Clause | YES | |
+| libucl | ? | BSD-2-Clause | YES | |
+| replxx | 6d93360 | BSD-2-Clause | YES | libicu usage |
+| lua-argparse | 0.7.1 | MIT | NO | |
+| lua-bit | 1.0.2 | MIT | YES | build fixes |
+| lua-fun | ? | MIT | YES | rspamd text |
+| lua-lpeg | 1.0 | MIT | YES | rspamd text + alloc |
+| lua-moses | ? | MIT | NO | |
+| lua-lupa | ? | MIT | NO | |
+| lua-tableshape | 2.6.0 | MIT | NO | |
+| mumhash | ? | MIT | NO | |
+| ngx-http-parser | 2.2.0 | MIT | YES | spamc support |
+| Mozilla-PublicSuffix | ? | MIT | NO | |
+| snowball | ? | BSD-3-Clause | NO | |
+| t1ha | ? | Zlib | NO | |
+| uthash | 1.9.8 | BSD | YES | |
+| xxhash | 0.8.1 | BSD | NO | |
+| zstd | 1.5.4 | BSD | YES | build fixes only |
+| google-ced | 37529e6 | Apache 2 | YES | build fixes |
+| kann | ? | MIT | YES | blas/lapack changes |
+| fpconv | ? | Boost | YES | many changes |
+| fastutf8 | ? | MIT | YES | many changes |
+| expected | v1.0 | Public Domain / CC0 | NO | |
+| frozen | 1.0.1 | Apache 2 | NO | |
+| fmt | 10.0.0 | MIT | NO | |
+| doctest | 2.4.6 | MIT | NO | |
+| function2 | 4.1.0 | Boost | NO | |
+| ankerl/svector | 1.0.2 | MIT | NO | |
+| ankerl/unordered_dense | 4.4.0 | MIT | NO | |
+| backward-cpp | 1.6 | MIT | NO | |
+
diff --git a/contrib/aho-corasick/CMakeLists.txt b/contrib/aho-corasick/CMakeLists.txt
new file mode 100644
index 0000000..f47dbed
--- /dev/null
+++ b/contrib/aho-corasick/CMakeLists.txt
@@ -0,0 +1,14 @@
+SET(AHOCORASICSRC acism_create.c
+ acism.c)
+
+IF(NOT GPL_RSPAMD_BINARY)
+ ADD_LIBRARY(rspamd-actrie SHARED ${AHOCORASICSRC})
+ target_link_libraries(rspamd-actrie glib-2.0)
+ target_link_libraries(rspamd-actrie "${RSPAMD_REQUIRED_LIBRARIES}")
+
+ INSTALL(TARGETS rspamd-actrie
+ LIBRARY DESTINATION ${RSPAMD_LIBDIR})
+ELSE()
+ ADD_LIBRARY(rspamd-actrie STATIC ${AHOCORASICSRC})
+ target_link_libraries(rspamd-actrie glib-2.0)
+ENDIF()
diff --git a/contrib/aho-corasick/LICENSE b/contrib/aho-corasick/LICENSE
new file mode 100644
index 0000000..65c5ca8
--- /dev/null
+++ b/contrib/aho-corasick/LICENSE
@@ -0,0 +1,165 @@
+ GNU LESSER GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+ This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+ 0. Additional Definitions.
+
+ As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+ "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+ An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+ A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+ The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+ The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+ 1. Exception to Section 3 of the GNU GPL.
+
+ You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+ 2. Conveying Modified Versions.
+
+ If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+ a) under this License, provided that you make a good faith effort to
+ ensure that, in the event an Application does not supply the
+ function or data, the facility still operates, and performs
+ whatever part of its purpose remains meaningful, or
+
+ b) under the GNU GPL, with none of the additional permissions of
+ this License applicable to that copy.
+
+ 3. Object Code Incorporating Material from Library Header Files.
+
+ The object code form of an Application may incorporate material from
+a header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+ a) Give prominent notice with each copy of the object code that the
+ Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the object code with a copy of the GNU GPL and this license
+ document.
+
+ 4. Combined Works.
+
+ You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+ a) Give prominent notice with each copy of the Combined Work that
+ the Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the Combined Work with a copy of the GNU GPL and this license
+ document.
+
+ c) For a Combined Work that displays copyright notices during
+ execution, include the copyright notice for the Library among
+ these notices, as well as a reference directing the user to the
+ copies of the GNU GPL and this license document.
+
+ d) Do one of the following:
+
+ 0) Convey the Minimal Corresponding Source under the terms of this
+ License, and the Corresponding Application Code in a form
+ suitable for, and under terms that permit, the user to
+ recombine or relink the Application with a modified version of
+ the Linked Version to produce a modified Combined Work, in the
+ manner specified by section 6 of the GNU GPL for conveying
+ Corresponding Source.
+
+ 1) Use a suitable shared library mechanism for linking with the
+ Library. A suitable mechanism is one that (a) uses at run time
+ a copy of the Library already present on the user's computer
+ system, and (b) will operate properly with a modified version
+ of the Library that is interface-compatible with the Linked
+ Version.
+
+ e) Provide Installation Information, but only if you would otherwise
+ be required to provide such information under section 6 of the
+ GNU GPL, and only to the extent that such information is
+ necessary to install and execute a modified version of the
+ Combined Work produced by recombining or relinking the
+ Application with a modified version of the Linked Version. (If
+ you use option 4d0, the Installation Information must accompany
+ the Minimal Corresponding Source and Corresponding Application
+ Code. If you use option 4d1, you must provide the Installation
+ Information in the manner specified by section 6 of the GNU GPL
+ for conveying Corresponding Source.)
+
+ 5. Combined Libraries.
+
+ You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+ a) Accompany the combined library with a copy of the same work based
+ on the Library, uncombined with any other library facilities,
+ conveyed under the terms of this License.
+
+ b) Give prominent notice with the combined library that part of it
+ is a work based on the Library, and explaining where to find the
+ accompanying uncombined form of the same work.
+
+ 6. Revised Versions of the GNU Lesser General Public License.
+
+ The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+ If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/contrib/aho-corasick/README.md b/contrib/aho-corasick/README.md
new file mode 100644
index 0000000..c245d6f
--- /dev/null
+++ b/contrib/aho-corasick/README.md
@@ -0,0 +1,67 @@
+aho-corasick
+==
+
+Aho-Corasick parallel string search, using interleaved arrays.
+
+Mischa Sandberg mischasan@gmail.com
+
+ACISM is an implementation of Aho-Corasick parallel string search,
+using an Interleaved State-transition Matrix.
+It combines the fastest possible Aho-Corasick implementation,
+with the smallest possible data structure (!).
+
+FEATURES
+--------
+
+* Fast. No hashing, no tree traversal; just a straight look-up equivalent to
+ matrix[state, input-byte] per input character.
+
+* Tiny. On average, the whole data structure (mostly the array) takes about 2-3 bytes per
+ input pattern byte. The original set of pattern strings can be reverse-generated from the machine.
+
+* Shareable. The state machine contains no pointers, so it can be compiled once,
+ then memory-mapped by many processes.
+
+* Searches byte vectors, not null-terminated strings.
+ Suitable for searching machine code as much as searching text.
+
+* DOS-proof. Well, that's an attribute of Aho-Corasick,
+ so no real points for that.
+
+* Stream-ready. The state can be saved between calls to search data.
+
+DOCUMENTATION
+-------------
+
+The GoogleDocs description is at http://goo.gl/lE6zG
+I originally called it "psearch", but found that name was overused by other authors.
+
+LICENSE
+-------
+
+Though I've had strong suggestions to go with BSD license, I'm going with GPL2 until I figure out
+how to keep in touch with people who download and use the code. Hence the "CONTACT ME IF..." line in the license.
+
+GETTING STARTED
+---------------
+
+Download the source, type "gmake".
+"gmake install" exports lib/libacism.a, include/acism.h and bin/acism_x.
+"acism_x.c" is a good example of calling acism_create and acism_scan/acism_more.
+
+(If you're interested in the GNUmakefile and rules.mk,
+ check my blog posts on non-recursive make, at mischasan.wordpress.com.)
+
+HISTORY
+-------
+
+The interleaved-array approach was tried and discarded in the late 70's, because the compile time was O(n^2).
+acism_create beats the problem with a "hint" array that tracks the restart points for searches.
+That, plus discarding the original idea of how to get maximal density, resulted in the tiny-fast win-win.
+
+ACKNOWLEDGEMENTS
+----------------
+
+I'd like to thank Mike Shannon, who wanted to see a machine built to make best use of L1/L2 cache.
+The change to do that doubled performance on hardware with a much larger cache than the matrix.
+Go figure.
diff --git a/contrib/aho-corasick/_acism.h b/contrib/aho-corasick/_acism.h
new file mode 100644
index 0000000..3993594
--- /dev/null
+++ b/contrib/aho-corasick/_acism.h
@@ -0,0 +1,114 @@
+/*
+** Copyright (C) 2009-2014 Mischa Sandberg <mischasan@gmail.com>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU Lesser General Public License Version 3 as
+** published by the Free Software Foundation. You may not use, modify or
+** distribute this program under any other version of the GNU Lesser General
+** Public License.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+** GNU Lesser General Public License for more details.
+**
+** You should have received a copy of the GNU Lesser General Public License
+** along with this program; if not, write to the Free Software
+** Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+#ifndef _ACISM_H
+#define _ACISM_H
+
+#include <stdint.h>
+#include <stdlib.h> // malloc
+#include <string.h> // memcpy
+
+typedef int (*qsort_cmp)(const void *, const void *);
+
+// "Width" specifier for different plats
+#if __LONG_MAX__ == 9223372036854775807LL
+# ifdef __APPLE__
+# define F64 "ll"
+# else
+# define F64 "l"
+# endif
+#elif __LONG_MAX__ == 2147483647L || defined(_LONG_LONG) || defined(__sun) // AIX 6.1 ...
+# define F64 "ll"
+#else
+//XXX Assuming F64 is "ll" for VS
+# define F64 "ll"
+#endif
+
+#ifndef ACISM_SIZE
+# define ACISM_SIZE 4
+#endif
+
+#if ACISM_SIZE == 8
+typedef uint64_t TRAN, STATE, STRNO;
+# define SYM_BITS 9U
+# define SYM_MASK 511
+# define FNO F64
+#else
+typedef uint32_t TRAN, STATE, STRNO;
+# define SYM_BITS psp->sym_bits
+# define SYM_MASK psp->sym_mask
+# define FNO
+#endif
+
+typedef uint16_t SYMBOL;
+typedef unsigned _SYMBOL; // An efficient stacklocal SYMBOL
+
+#define BACK ((SYMBOL)0)
+#define ROOT ((STATE) 0)
+
+// MATCH and SUFFIX are the top 2 bits of a TRAN:
+enum {
+ IS_MATCH = (TRAN)1 << (8*sizeof(TRAN) - 1),
+ IS_SUFFIX = (TRAN)1 << (8*sizeof(TRAN) - 2),
+ T_FLAGS = IS_MATCH | IS_SUFFIX
+};
+
+typedef struct { STATE state; STRNO strno; } STRASH;
+
+struct acism {
+ TRAN* tranv;
+ STRASH* hashv;
+ unsigned flags;
+# define IS_MMAP 1
+
+#if ACISM_SIZE < 8
+ TRAN sym_mask;
+ unsigned sym_bits;
+#endif
+ unsigned hash_mod; // search hashv starting at (state + sym) % hash_mod.
+ unsigned hash_size; // #(hashv): hash_mod plus the overflows past [hash_mod-1]
+ unsigned tran_size; // #(tranv)
+ unsigned nsyms, nchars, nstrs, maxlen;
+ SYMBOL symv[256];
+};
+
+#include "acism.h"
+
+// p_size: size of tranv + hashv
+static inline size_t p_size(ACISM const *psp)
+{ return psp->hash_size * sizeof*psp->hashv
+ + psp->tran_size * sizeof*psp->tranv; }
+
+static inline unsigned p_hash(ACISM const *psp, STATE s)
+{ return s * 107 % psp->hash_mod; }
+
+static inline void set_tranv(ACISM *psp, void *mem)
+{ psp->hashv = (STRASH*)&(psp->tranv = (TRAN*)mem)[psp->tran_size]; }
+
+// TRAN accessors. For ACISM_SIZE=8, SYM_{BITS,MASK} do not use psp.
+
+static inline TRAN p_tran(ACISM const *psp, STATE s, _SYMBOL sym)
+{ return psp->tranv[s + sym] ^ sym; }
+
+static inline _SYMBOL t_sym(ACISM const *psp, TRAN t) { (void)psp; return t & SYM_MASK; }
+static inline STATE t_next(ACISM const *psp, TRAN t) { (void)psp; return (t & ~T_FLAGS) >> SYM_BITS; }
+static inline int t_isleaf(ACISM const *psp, TRAN t) { return t_next(psp, t) >= psp->tran_size; }
+static inline int t_strno(ACISM const *psp, TRAN t) { return t_next(psp, t) - psp->tran_size; }
+static inline _SYMBOL t_valid(ACISM const *psp, TRAN t) { return !t_sym(psp, t); }
+
+#endif//_ACISM_H \ No newline at end of file
diff --git a/contrib/aho-corasick/acism.c b/contrib/aho-corasick/acism.c
new file mode 100644
index 0000000..e2b48a5
--- /dev/null
+++ b/contrib/aho-corasick/acism.c
@@ -0,0 +1,124 @@
+/*
+** Copyright (C) 2009-2014 Mischa Sandberg <mischasan@gmail.com>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU Lesser General Public License Version as
+** published by the Free Software Foundation. You may not use, modify or
+** distribute this program under any other version of the GNU Lesser General
+** Public License.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+** GNU Lesser General Public License for more details.
+**
+** You should have received a copy of the GNU Lesser General Public License
+** along with this program; if not, write to the Free Software
+** Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+#include <glib.h>
+
+#include "_acism.h"
+#include "unix-std.h"
+
+#define BACK ((SYMBOL)0)
+#define ROOT ((STATE) 0)
+extern const guchar lc_map[256];
+
+int
+acism_lookup(ac_trie_t const *psp, const char *text, size_t len,
+ ACISM_ACTION *cb, void *context, int *statep, bool caseless)
+{
+ char const *cp = text, *endp = cp + len;
+ uint8_t s;
+ STATE state = *statep;
+ int ret = 0;
+
+ while (cp < endp) {
+ s = caseless ? lc_map[(guint8)*cp++] : *cp++;
+ _SYMBOL sym = psp->symv[s];
+ if (!sym) {
+ // Input byte is not in any pattern string.
+ state = ROOT;
+ continue;
+ }
+
+ // Search for a valid transition from this (state, sym),
+ // following the backref chain.
+
+ TRAN next;
+ while (!t_valid(psp, next = p_tran(psp, state, sym)) && state != ROOT) {
+ TRAN back = p_tran(psp, state, BACK);
+ state = t_valid(psp, back) ? t_next(psp, back) : ROOT;
+ }
+
+ if (!t_valid(psp, next))
+ continue;
+
+ if (!(next & (IS_MATCH | IS_SUFFIX))) {
+ // No complete match yet; keep going.
+ state = t_next(psp, next);
+ continue;
+ }
+
+ // At this point, one or more patterns have matched.
+ // Find all matches by following the backref chain.
+ // A valid node for (sym) with no SUFFIX flag marks the
+ // end of the suffix chain.
+ // In the same backref traversal, find a new (state),
+ // if the original transition is to a leaf.
+
+ STATE s = state;
+
+ // Initially state is ROOT. The chain search saves the
+ // first state from which the next char has a transition.
+ state = t_isleaf(psp, next) ? 0 : t_next(psp, next);
+
+ while (1) {
+
+ if (t_valid(psp, next)) {
+
+ if (next & IS_MATCH) {
+ unsigned strno, ss = s + sym, i;
+ if (t_isleaf(psp, psp->tranv[ss])) {
+ strno = t_strno(psp, psp->tranv[ss]);
+ } else {
+ for (i = p_hash(psp, ss); psp->hashv[i].state != ss; ++i);
+ strno = psp->hashv[i].strno;
+ }
+
+ if ((ret = cb(strno, cp - text, context)))
+ goto EXIT;
+ }
+
+ if (!state && !t_isleaf(psp, next))
+ state = t_next(psp, next);
+ if ( state && !(next & IS_SUFFIX))
+ break;
+ }
+
+ if (s == ROOT)
+ break;
+
+ TRAN b = p_tran(psp, s, BACK);
+ s = t_valid(psp, b) ? t_next(psp, b) : ROOT;
+ next = p_tran(psp, s, sym);
+ }
+ }
+EXIT:
+ *statep = state;
+ return ret;
+}
+
+void
+acism_destroy(ac_trie_t *psp)
+{
+ if (!psp) return;
+ if (psp->flags & IS_MMAP)
+ munmap((char*)psp->tranv - sizeof(ac_trie_t),
+ sizeof(ac_trie_t) + p_size(psp));
+ else g_free(psp->tranv);
+ g_free(psp);
+}
+//EOF
diff --git a/contrib/aho-corasick/acism.h b/contrib/aho-corasick/acism.h
new file mode 100644
index 0000000..1e03176
--- /dev/null
+++ b/contrib/aho-corasick/acism.h
@@ -0,0 +1,53 @@
+/*
+** Copyright (C) 2009-2014 Mischa Sandberg <mischasan@gmail.com>
+** Copyright (C) 2015 Vsevolod Stakhov <vsevolod@rspamd.com>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU Lesser General Public License as
+** published by the Free Software Foundation. You may not use, modify or
+** distribute this program under any other version of the GNU Lesser General
+** Public License.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+** GNU Lesser General Public License for more details.
+**
+** You should have received a copy of the GNU Lesser General Public License
+** along with this program; if not, write to the Free Software
+** Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+#ifndef ACISM_H
+#define ACISM_H
+
+#include "config.h"
+// "acism" uses MEMREF {ptr,len} bytevec structs for "string" args,
+// rather than NUL-terminated "C" strings.
+
+typedef struct ac_trie_pat_s { char const *ptr; size_t len; } ac_trie_pat_t;
+
+typedef struct acism ac_trie_t;
+typedef struct acism ACISM;
+typedef struct ac_trie_pat_s MEMREF;
+
+ac_trie_t* acism_create(ac_trie_pat_t const *strv, int nstrs);
+void acism_destroy(ac_trie_t*);
+
+// For each match, acism_scan calls its ACISM_ACTION fn,
+// giving it the strv[] index of the matched string,
+// and the text[] offset of the byte PAST the end of the string.
+// If ACISM_ACTION returns 0, search continues; otherwise,
+// acism_more returns that nonzero value immediately.
+
+typedef int (ACISM_ACTION)(int strnum, int textpos, void *context);
+
+// If sequential blocks of (text) are passed to repeated acism_more calls,
+// then search continues where the previous acism_more left off --
+// string matches can cross block boundaries.
+// *state should initially be (0).
+
+int acism_lookup(ac_trie_t const *psp, const char *text, size_t len,
+ ACISM_ACTION *cb, void *context, int *statep, bool caseless);
+
+#endif//ACISM_H
diff --git a/contrib/aho-corasick/acism_create.c b/contrib/aho-corasick/acism_create.c
new file mode 100644
index 0000000..2d4439f
--- /dev/null
+++ b/contrib/aho-corasick/acism_create.c
@@ -0,0 +1,382 @@
+/*
+** Copyright (C) 2009-2014 Mischa Sandberg <mischasan@gmail.com>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU Lesser General Public License Version 3 as
+** published by the Free Software Foundation. You may not use, modify or
+** distribute this program under any other version of the GNU Lesser General
+** Public License.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+** GNU Lesser General Public License for more details.
+**
+** You should have received a copy of the GNU Lesser General Public License
+** along with this program; if not, write to the Free Software
+** Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+#include "_acism.h"
+
+typedef enum { BASE=2, USED=1 } USES;
+
+typedef struct tnode {
+ struct tnode *child, *next, *back;
+ // nrefs was used in "prune_backlinks".
+ // It will be used again in "curtail".
+ unsigned nrefs;
+ STATE state;
+ STRNO match;
+ SYMBOL sym;
+ char is_suffix; // "bool"
+} TNODE;
+
+//--------------|---------------------------------------------
+// bitwid: 1+floor(log2(u))
+static inline int bitwid(unsigned u)
+{
+ int ret = !!u;
+ if (u & 0xFFFF0000) u >>= 16, ret += 16;
+ if (u & 0x0000FF00) u >>= 8, ret += 8;
+ if (u & 0x000000F0) u >>= 4, ret += 4;
+ if (u & 0x0000000C) u >>= 2, ret += 2;
+ if (u & 0x00000002) ret++;
+ return ret;
+}
+
+static void fill_symv(ACISM*, MEMREF const*, int ns);
+static int create_tree(TNODE*, SYMBOL const*symv, MEMREF const*strv, int nstrs);
+static void add_backlinks(TNODE*, TNODE**, TNODE**);
+static int interleave(TNODE*, int nnodes, int nsyms, TNODE**, TNODE**);
+static void fill_tranv(ACISM*, TNODE const*);
+static void fill_hashv(ACISM*, TNODE const*, int nn);
+
+static TNODE* find_child(TNODE*, SYMBOL);
+
+// (ns) is either a STATE, or a (STRNO + tran_size)
+static inline void
+set_tran(ACISM *psp, STATE s, SYMBOL sym, int match, int suffix, TRAN ns)
+{
+ psp->tranv[s + sym] = sym | (match ? IS_MATCH : 0)
+ | (suffix ? IS_SUFFIX : 0)
+ | (ns << SYM_BITS);
+}
+
+// Track statistics for construction
+#ifdef ACISM_STATS
+typedef struct { long long val; const char *name; } PSSTAT;
+extern PSSTAT psstat[];
+# define NOTE(n) (psstat[__LINE__] = (PSSTAT) {n, #n})
+# define HIT(id) (psstat[__LINE__].val++, psstat[__LINE__].name = id)
+#else
+# define NOTE(n) (void)0
+# define HIT(id) (void)0
+#endif //ACISM_STATS
+
+//--------------|---------------------------------------------
+ACISM*
+acism_create(MEMREF const* strv, int nstrs)
+{
+ TNODE **v1 = NULL, **v2 = NULL;
+ ACISM *psp = g_malloc0(sizeof*psp);
+
+ fill_symv(psp, strv, nstrs);
+ TNODE *troot = g_malloc0((psp->nchars + 1) * sizeof(*troot));
+
+ int nnodes = create_tree(troot, psp->symv, strv, nstrs);
+ NOTE(nnodes);
+
+ // v1, v2: breadth-first work vectors for add_backlink and interleave.
+ int i = (nstrs + 1) * sizeof(TNODE);
+ add_backlinks(troot, v1 = g_malloc0(i), v2 = g_malloc0(i));
+
+ int nhash = 0;
+ TNODE* tp = troot + nnodes;
+ while (--tp > troot)
+ nhash += tp->match && tp->child;
+
+ // Calculate each node's offset in tranv[]:
+ psp->tran_size = interleave(troot, nnodes, psp->nsyms, v1, v2);
+ if (bitwid(psp->tran_size + nstrs - 1) + SYM_BITS > sizeof(TRAN)*8 - 2)
+ goto FAIL;
+
+ if (nhash) {
+ // Hash table is for match info of non-leaf nodes (only).
+ // Set hash_size for p_size(psp):
+ psp->hash_mod = nhash * 5 / 4 + 1;
+ // Initially oversize the table for overflows without wraparound.
+ psp->hash_size = psp->hash_mod + nhash;
+ }
+
+ set_tranv(psp, g_malloc0(p_size(psp) + sizeof(TRAN)));
+ if (!psp->tranv) goto FAIL;
+ fill_tranv(psp, troot);
+ // The root state (0) must not look like a valid backref.
+ // Any symbol value other than (0) in tranv[0] ensures that.
+ psp->tranv[0] = 1;
+
+ if (nhash) {
+ fill_hashv(psp, troot, nnodes);
+ // Adjust hash_size to include trailing overflows
+ // but trim trailing empty slots.
+ psp->hash_size = psp->hash_mod;
+ while ( psp->hashv[psp->hash_size].state) ++psp->hash_size;
+ while (!psp->hashv[psp->hash_size - 1].state) --psp->hash_size;
+ set_tranv(psp, g_realloc(psp->tranv, p_size(psp)));
+ }
+
+ // Diagnostics/statistics only:
+ psp->nstrs = nstrs;
+ for (i = psp->maxlen = 0; i < nstrs; ++i)
+ if (psp->maxlen < strv[i].len) psp->maxlen = strv[i].len;
+
+ goto DONE;
+ FAIL: acism_destroy(psp), psp = NULL;
+ DONE: free(troot), free(v1), free(v2);
+ return psp;
+}
+
+typedef struct { int freq, rank; } FRANK;
+static int frcmp(FRANK*a, FRANK*b) { return a->freq - b->freq; }
+
+static void
+fill_symv(ACISM *psp, MEMREF const *strv, int nstrs)
+{
+ int i, j;
+ FRANK frv[256];
+
+ for (i = 0; i < 256; ++i) frv[i] = (FRANK){0,i};
+ for (i = 0; i < nstrs; ++i)
+ for (psp->nchars += j = strv[i].len; --j >= 0;)
+ frv[(uint8_t)strv[i].ptr[j]].freq++;
+
+ qsort(frv, 256, sizeof*frv, (qsort_cmp)frcmp);
+
+ for (i = 256; --i >= 0 && frv[i].freq;)
+ psp->symv[frv[i].rank] = ++psp->nsyms;
+ ++psp->nsyms;
+
+#if ACISM_SIZE < 8
+ psp->sym_bits = bitwid(psp->nsyms);
+ psp->sym_mask = ~((~0u) << psp->sym_bits);
+#endif
+}
+
+static int
+create_tree(TNODE *Tree, SYMBOL const *symv, MEMREF const *strv, int nstrs)
+{
+ int i, j;
+ TNODE *nextp = Tree + 1;
+
+ for (i = 0; i < nstrs; ++i) {
+ TNODE *tp = Tree;
+
+ for (j = 0; tp->child && j < (int)strv[i].len; ++j) {
+ SYMBOL sym = symv[(uint8_t)strv[i].ptr[j]];
+
+ if (sym < tp->child->sym) {
+ // Prep to insert new node before tp->child
+ nextp->next = tp->child;
+ break;
+ }
+
+ tp = tp->child;
+ while (tp->next && sym >= tp->next->sym)
+ tp = tp->next;
+
+ // Insert new sibling after tp
+ if (sym > tp->sym) {
+ nextp->next = tp->next;
+ tp = tp->next = nextp++;
+ tp->sym = sym;
+ tp->back = Tree;
+ }
+ }
+
+ for (; j < (int) strv[i].len; ++j) {
+ tp = tp->child = nextp++;
+ tp->sym = symv[(uint8_t)strv[i].ptr[j]];
+ tp->back = Tree;
+ }
+
+ tp->match = i + 1; // Encode strno as nonzero
+ }
+
+ return nextp - Tree;
+}
+
+static void
+add_backlinks(TNODE *troot, TNODE **v1, TNODE **v2)
+{
+ TNODE *tp, **tmp;
+
+ for (tp = troot->child, tmp = v1; tp; tp = tp->next)
+ *tmp++ = tp;
+ *tmp = NULL;
+
+ while (*v1) {
+ TNODE **spp = v1, **dpp = v2, *srcp, *dstp;
+
+ while ((srcp = *spp++)) {
+ for (dstp = srcp->child; dstp; dstp = dstp->next) {
+ TNODE *bp = NULL;
+ if (dstp->child)
+ *dpp++ = dstp;
+
+ // Go through the parent (srcp) node's backlink chain,
+ // looking for a useful backlink for the child (dstp).
+ // If the parent (srcp) has a backlink to (tp),
+ // and (tp) has a child matching the transition sym
+ // for (srcp -> dstp), then it is a useful backlink
+ // for the child (dstp).
+ // Note that backlinks do not point at the suffix match;
+ // they point at the PARENT of that match.
+
+ for (tp = srcp->back; tp; tp = tp->back)
+ if ((bp = find_child(tp, dstp->sym)))
+ break;
+ if (!bp)
+ bp = troot;
+
+ dstp->back = dstp->child ? bp : tp ? tp : troot;
+ dstp->back->nrefs++;
+ dstp->is_suffix = bp->match || bp->is_suffix;
+ }
+ }
+ *dpp = 0;
+ tmp = v1; v1 = v2; v2 = tmp;
+ }
+}
+
+static int
+interleave(TNODE *troot, int nnodes, int nsyms, TNODE **v1, TNODE **v2)
+{
+ unsigned usev_size = nnodes + nsyms;
+ char *usev = g_malloc0(usev_size * sizeof(*usev));
+ STATE last_trans = 0, startv[257][2];
+ TNODE *cp, **tmp;
+
+ memset(startv, 0, nsyms * sizeof*startv);
+
+ // Iterate through one level of the Tree at a time.
+ // That srsly improves locality (L1-cache use).
+
+ v1[0] = troot, v1[1] = NULL;
+ for (; *v1; tmp = v1, v1 = v2, v2 = tmp) {
+ TNODE **srcp = v1, **dstp = v2, *tp;
+ while ((tp = *srcp++)) {
+ if (!tp->child) continue;
+
+ HIT("nonleaf");
+ if (tp->back == troot) tp->back = NULL; // simplify tests.
+ cp = tp->child;
+
+ STATE pos, *startp = &startv[cp->sym][!!tp->back];
+ while ((cp = cp->next)) {
+ STATE *newp = &startv[cp->sym][!!tp->back];
+ if (*startp < *newp) startp = newp;
+ }
+
+ // If (tp) has a backref, we need a slot at offset 0
+ // that is free as a base AND to be used (filled in).
+ char need = tp->back ? BASE|USED : BASE;
+ for (pos = *startp;; ++pos) {
+ if (usev[pos] & need) {
+ HIT("inner loop");
+ continue;
+ }
+
+ for (cp = tp->child; cp; cp = cp->next) {
+ HIT("child loop");
+ if (usev[pos + cp->sym] & USED) break;
+ }
+
+ // No child needs an in-use slot? We're done.
+ if (!cp) break;
+ }
+ tp->state = pos;
+
+ // Mark node's base and children as used:
+ usev[pos] |= need;
+ STATE last = 0; // Make compiler happy
+ int nkids = 0;
+ for (cp = tp->child; cp; *dstp++ = cp, cp = cp->next, ++nkids)
+ usev[last = pos + cp->sym] |= USED;
+
+ // This is a HEURISTIC for advancing search for other nodes
+ *startp += (pos - *startp) / nkids;
+
+ if (last_trans < last) {
+ last_trans = last;
+ if (last + nsyms >= usev_size) {
+ usev = g_realloc(usev, usev_size << 1);
+ memset(usev + usev_size, 0, usev_size);
+ usev_size <<= 1;
+ }
+ }
+ }
+
+ *dstp = NULL;
+ }
+
+ free(usev);
+ return last_trans + 1;
+}
+
+static void
+fill_hashv(ACISM *psp, TNODE const treev[], int nnodes)
+{
+ STRASH *sv = g_malloc0(psp->hash_mod * sizeof*sv), *sp = sv;
+ int i;
+
+ // First pass: insert without resolving collisions.
+ for (i = 0; i < nnodes; ++i) {
+ STATE base = treev[i].state;
+ TNODE const *tp;
+ for (tp = treev[i].child; tp; tp = tp->next) {
+ if (tp->match && tp->child) {
+ STATE state = base + tp->sym;
+ STRASH *hp = &psp->hashv[p_hash(psp, state)];
+ *(hp->state ? sp++ : hp) = (STRASH){state, tp->match - 1};
+ }
+ }
+ }
+
+ while (--sp >= sv) {
+ HIT("hash collisions");
+ for (i = p_hash(psp, sp->state); psp->hashv[i].state; ++i)
+ HIT("hash displacements");
+ psp->hashv[i] = *sp;
+ }
+
+ free(sv);
+}
+
+static void
+fill_tranv(ACISM *psp, TNODE const*tp)
+{
+ TNODE const *cp = tp->child;
+
+ if (cp && tp->back)
+ set_tran(psp, tp->state, 0, 0, 0, tp->back->state);
+
+ for (; cp; cp = cp->next) {
+ //NOTE: cp->match is (strno+1) so that !cp->match means "no match".
+ set_tran(psp, tp->state, cp->sym, cp->match, cp->is_suffix,
+ cp->child ? cp->state : cp->match - 1 + psp->tran_size);
+ if (cp->child)
+ fill_tranv(psp, cp);
+ }
+}
+
+static TNODE *
+find_child(TNODE *tp, SYMBOL sym)
+{
+ for (tp = tp->child; tp && tp->sym < sym; tp = tp->next);
+ return tp && tp->sym == sym ? tp : NULL;
+}
+
+#ifdef ACISM_STATS
+PSSTAT psstat[__LINE__] = {{__LINE__,0}};
+#endif//ACISM_STATS
+//EOF \ No newline at end of file
diff --git a/contrib/ankerl/LICENSE b/contrib/ankerl/LICENSE
new file mode 100644
index 0000000..c4d1a0e
--- /dev/null
+++ b/contrib/ankerl/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Martin Leitner-Ankerl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/contrib/ankerl/svector.h b/contrib/ankerl/svector.h
new file mode 100644
index 0000000..dbd075b
--- /dev/null
+++ b/contrib/ankerl/svector.h
@@ -0,0 +1,999 @@
+// ┌─â”┬ ┬┌─â”┌─â”┌┬â”┌─â”┬─┠Compact SVO optimized vector C++17 or higher
+// └─â”â””â”┌┘├┤ │ │ │ │├┬┘ Version 1.0.2
+// └─┘ └┘ └─┘└─┘ ┴ └─┘┴└─ https://github.com/martinus/svector
+//
+// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2022 Martin Leitner-Ankerl <martin.ankerl@gmail.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#ifndef ANKERL_SVECTOR_H
+#define ANKERL_SVECTOR_H
+
+// see https://semver.org/spec/v2.0.0.html
+#define ANKERL_SVECTOR_VERSION_MAJOR 1 // incompatible API changes
+#define ANKERL_SVECTOR_VERSION_MINOR 0 // add functionality in a backwards compatible manner
+#define ANKERL_SVECTOR_VERSION_PATCH 2 // backwards compatible bug fixes
+
+// API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/
+#define ANKERL_SVECTOR_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch
+#define ANKERL_SVECTOR_VERSION_CONCAT(major, minor, patch) ANKERL_SVECTOR_VERSION_CONCAT1(major, minor, patch)
+#define ANKERL_SVECTOR_NAMESPACE \
+ ANKERL_SVECTOR_VERSION_CONCAT(ANKERL_SVECTOR_VERSION_MAJOR, ANKERL_SVECTOR_VERSION_MINOR, ANKERL_SVECTOR_VERSION_PATCH)
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <initializer_list>
+#include <iterator>
+#include <limits>
+#include <memory>
+#include <new>
+#include <stdexcept>
+#include <type_traits>
+#include <utility>
+
+namespace ankerl {
+inline namespace ANKERL_SVECTOR_NAMESPACE {
+namespace detail {
+
+template <typename Condition, typename T = void>
+using enable_if_t = typename std::enable_if<Condition::value, T>::type;
+
+template <typename It>
+using is_input_iterator = std::is_base_of<std::input_iterator_tag, typename std::iterator_traits<It>::iterator_category>;
+
+constexpr auto round_up(size_t n, size_t multiple) -> size_t {
+ return ((n + (multiple - 1)) / multiple) * multiple;
+}
+
+template <typename T>
+constexpr auto cx_min(T a, T b) -> T {
+ return a < b ? a : b;
+}
+
+template <typename T>
+constexpr auto cx_max(T a, T b) -> T {
+ return a > b ? a : b;
+}
+
+template <typename T>
+constexpr auto alignment_of_svector() -> size_t {
+ return cx_max(sizeof(void*), std::alignment_of_v<T>);
+}
+
+/**
+ * @brief Calculates sizeof(svector<T, N>) for a given type and inline capacity
+ */
+template <typename T>
+constexpr auto size_of_svector(size_t min_inline_capacity) -> size_t {
+ // + 1 for one byte size in direct mode
+ return round_up(sizeof(T) * min_inline_capacity + 1, alignment_of_svector<T>());
+}
+
+/**
+ * @brief Calculates how many T we can actually store inside of an svector without increasing its sizeof().
+ *
+ * E.g. svector<char, 1> could store 7 bytes even though 1 is specified. This makes sure we don't waste any
+ * of the padding.
+ */
+template <typename T>
+constexpr auto automatic_capacity(size_t min_inline_capacity) -> size_t {
+ return cx_min((size_of_svector<T>(min_inline_capacity) - 1U) / sizeof(T), size_t{127});
+}
+
+/**
+ * Holds size & capacity, a glorified struct.
+ */
+class header {
+ size_t m_size{};
+ size_t const m_capacity;
+
+public:
+ inline explicit header(size_t capacity)
+ : m_capacity{capacity} {}
+
+ [[nodiscard]] inline auto size() const -> size_t {
+ return m_size;
+ }
+
+ [[nodiscard]] inline auto capacity() const -> size_t {
+ return m_capacity;
+ }
+
+ inline void size(size_t s) {
+ m_size = s;
+ }
+};
+
+/**
+ * @brief Holds header (size+capacity) plus an arbitrary number of T.
+ *
+ * To make storage compact, we don't actually store a pointer to T. We don't have to
+ * because we know exactly at which location it begins.
+ */
+template <typename T>
+struct storage : public header {
+ static constexpr auto alignment_of_t = std::alignment_of_v<T>;
+ static constexpr auto max_alignment = std::max(std::alignment_of_v<header>, std::alignment_of_v<T>);
+ static constexpr auto offset_to_data = detail::round_up(sizeof(header), alignment_of_t);
+ static_assert(max_alignment <= __STDCPP_DEFAULT_NEW_ALIGNMENT__);
+
+ explicit storage(size_t capacity)
+ : header(capacity) {}
+
+ auto data() -> T* {
+ auto ptr_to_data = reinterpret_cast<std::byte*>(this) + offset_to_data;
+ return std::launder(reinterpret_cast<T*>(ptr_to_data));
+ }
+
+ /**
+ * @brief Allocates space for storage plus capacity*T objects.
+ *
+ * Checks to make sure that allocation won't overflow.
+ *
+ * @param capacity Number of T to allocate.
+ * @return storage<T>*
+ */
+ static auto alloc(size_t capacity) -> storage<T>* {
+ // make sure we don't overflow!
+ auto mem = sizeof(T) * capacity;
+ if (mem < capacity) {
+ throw std::bad_alloc();
+ }
+ if (offset_to_data + mem < mem) {
+ throw std::bad_alloc();
+ }
+ mem += offset_to_data;
+ if (static_cast<uint64_t>(mem) > static_cast<uint64_t>(std::numeric_limits<std::ptrdiff_t>::max())) {
+ throw std::bad_alloc();
+ }
+
+ void* ptr = ::operator new(offset_to_data + sizeof(T) * capacity);
+ if (nullptr == ptr) {
+ throw std::bad_alloc();
+ }
+ // use void* to ensure we don't use an overload for T*
+ return new (ptr) storage<T>(capacity);
+ }
+};
+
+} // namespace detail
+
+template <typename T, size_t MinInlineCapacity>
+class svector {
+ static_assert(MinInlineCapacity <= 127, "sorry, can't have more than 127 direct elements");
+ static constexpr auto N = detail::automatic_capacity<T>(MinInlineCapacity);
+
+ enum class direction { direct, indirect };
+
+ /**
+ * A buffer to hold the data of the svector Depending on direct/indirect mode, the content it holds is like so:
+ *
+ * direct:
+ * m_data[0] & 1: lowest bit is 1 for direct mode.
+ * m_data[0] >> 1: size for direct mode
+ * Then 0-X bytes unused (padding), and then the actual inline T data.
+ * indirect:
+ * m_data[0] & 1: lowest bit is 0 for indirect mode
+ * m_data[0..7]: stores an uintptr_t, which points to the indirect data.
+ */
+ alignas(detail::alignment_of_svector<T>()) std::array<uint8_t, detail::size_of_svector<T>(MinInlineCapacity)> m_data;
+
+ // direct mode ///////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto is_direct() const -> bool {
+ return (m_data[0] & 1U) != 0U;
+ }
+
+ [[nodiscard]] auto direct_size() const -> size_t {
+ return m_data[0] >> 1U;
+ }
+
+ // sets size of direct mode and mode to direct too.
+ constexpr void set_direct_and_size(size_t s) {
+ m_data[0] = (s << 1U) | 1U;
+ }
+
+ [[nodiscard]] auto direct_data() -> T* {
+ return std::launder(reinterpret_cast<T*>(m_data.data() + std::alignment_of_v<T>));
+ }
+
+ // indirect mode /////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto indirect() -> detail::storage<T>* {
+ detail::storage<T>* ptr; // NOLINT(cppcoreguidelines-init-variables)
+ std::memcpy(&ptr, m_data.data(), sizeof(ptr));
+ return ptr;
+ }
+
+ [[nodiscard]] auto indirect() const -> detail::storage<T> const* {
+ return const_cast<svector*>(this)->indirect(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ void set_indirect(detail::storage<T>* ptr) {
+ std::memcpy(m_data.data(), &ptr, sizeof(ptr));
+
+ // safety check to guarantee the lowest bit is 0
+ if (is_direct()) {
+ throw std::bad_alloc(); // LCOV_EXCL_LINE
+ }
+ }
+
+ // helpers ///////////////////////////////////////////////////////////////
+
+ /**
+ * @brief Moves size objects from source_ptr to target_ptr, and destroys what remains in source_ptr.
+ *
+ * Assumes data is not overlapping
+ */
+ static void uninitialized_move_and_destroy(T* source_ptr, T* target_ptr, size_t size) {
+ if constexpr (std::is_trivially_copyable_v<T>) {
+ std::memcpy(target_ptr, source_ptr, size * sizeof(T));
+ } else {
+ std::uninitialized_move_n(source_ptr, size, target_ptr);
+ std::destroy_n(source_ptr, size);
+ }
+ }
+
+ /**
+ * @brief Reallocates all data when capacity changes.
+ *
+ * if new_capacity <= N chooses direct memory, otherwise indirect.
+ */
+ void realloc(size_t new_capacity) {
+ if (new_capacity <= N) {
+ // put everything into direct storage
+ if (is_direct()) {
+ // direct -> direct: nothing to do!
+ return;
+ }
+
+ // indirect -> direct
+ auto* storage = indirect();
+ uninitialized_move_and_destroy(storage->data(), direct_data(), storage->size());
+ set_direct_and_size(storage->size());
+ std::destroy_at(storage);
+ ::operator delete(storage);
+ } else {
+ // put everything into indirect storage
+ auto* storage = detail::storage<T>::alloc(new_capacity);
+ if (is_direct()) {
+ // direct -> indirect
+ uninitialized_move_and_destroy(data<direction::direct>(), storage->data(), size<direction::direct>());
+ storage->size(size<direction::direct>());
+ } else {
+ // indirect -> indirect
+ uninitialized_move_and_destroy(data<direction::indirect>(), storage->data(), size<direction::indirect>());
+ storage->size(size<direction::indirect>());
+ auto* storage = indirect();
+ std::destroy_at(storage);
+ ::operator delete(storage);
+ }
+ set_indirect(storage);
+ }
+ }
+
+ /**
+ * @brief Doubles starting_capacity until it is >= size_to_fit.
+ */
+ [[nodiscard]] static auto calculate_new_capacity(size_t size_to_fit, size_t starting_capacity) -> size_t {
+ if (size_to_fit > max_size()) {
+ // not enough space
+ throw std::bad_alloc();
+ }
+
+ if (size_to_fit == 0) {
+ // special handling for 0 so N==0 works
+ return starting_capacity;
+ }
+ // start with at least 1, so N==0 works
+ auto new_capacity = std::max<size_t>(1, starting_capacity);
+
+ // double capacity until its large enough, but make sure we don't overflow
+ while (new_capacity < size_to_fit && new_capacity * 2 > new_capacity) {
+ new_capacity *= 2;
+ }
+ if (new_capacity < size_to_fit) {
+ // got an overflow, set capacity to max
+ new_capacity = max_size();
+ }
+ return std::min(new_capacity, max_size());
+ }
+
+ template <direction D>
+ [[nodiscard]] auto capacity() const -> size_t {
+ if constexpr (D == direction::direct) {
+ return N;
+ } else {
+ return indirect()->capacity();
+ }
+ }
+
+ template <direction D>
+ [[nodiscard]] auto size() const -> size_t {
+ if constexpr (D == direction::direct) {
+ return direct_size();
+ } else {
+ return indirect()->size();
+ }
+ }
+
+ template <direction D>
+ void set_size(size_t s) {
+ if constexpr (D == direction::direct) {
+ set_direct_and_size(s);
+ } else {
+ indirect()->size(s);
+ }
+ }
+
+ void set_size(size_t s) {
+ if (is_direct()) {
+ set_size<direction::direct>(s);
+ } else {
+ set_size<direction::indirect>(s);
+ }
+ }
+
+ template <direction D>
+ [[nodiscard]] auto data() -> T* {
+ if constexpr (D == direction::direct) {
+ return direct_data();
+ } else {
+ return indirect()->data();
+ }
+ }
+
+ template <direction D>
+ [[nodiscard]] auto data() const -> T const* {
+ return const_cast<svector*>(this)->data<D>(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ template <direction D>
+ void pop_back() {
+ if constexpr (std::is_trivially_destructible_v<T>) {
+ set_size<D>(size<D>() - 1);
+ } else {
+ auto s = size<D>() - 1;
+ (data<D>() + s)->~T();
+ set_size<D>(s);
+ }
+ }
+
+ /**
+ * @brief We need variadic arguments so we can either use copy ctor or default ctor
+ */
+ template <direction D, class... Args>
+ void resize_after_reserve(size_t count, Args&&... args) {
+ auto current_size = size<D>();
+ if (current_size > count) {
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ auto* d = data<D>();
+ std::destroy(d + count, d + current_size);
+ }
+ } else {
+ auto* d = data<D>();
+ for (auto ptr = d + current_size, end = d + count; ptr != end; ++ptr) {
+ new (static_cast<void*>(ptr)) T(std::forward<Args>(args)...);
+ }
+ }
+ set_size<D>(count);
+ }
+
+ // Makes sure that to is not past the end iterator
+ template <direction D>
+ auto erase_checked_end(T const* cfrom, T const* to) -> T* {
+ auto* const erase_begin = const_cast<T*>(cfrom); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ auto* const container_end = data<D>() + size<D>();
+ auto* const erase_end = std::min(const_cast<T*>(to), container_end); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+
+ std::move(erase_end, container_end, erase_begin);
+ auto const num_erased = std::distance(erase_begin, erase_end);
+ std::destroy(container_end - num_erased, container_end);
+ set_size<D>(size<D>() - num_erased);
+ return erase_begin;
+ }
+
+ template <typename It>
+ void assign(It first, It last, std::input_iterator_tag /*unused*/) {
+ clear();
+
+ // TODO this can be made faster, e.g. by setting size only when finished.
+ while (first != last) {
+ push_back(*first);
+ ++first;
+ }
+ }
+
+ template <typename It>
+ void assign(It first, It last, std::forward_iterator_tag /*unused*/) {
+ clear();
+
+ auto s = std::distance(first, last);
+ reserve(s);
+ std::uninitialized_copy(first, last, data());
+ set_size(s);
+ }
+
+ // precondition: all uninitialized
+ void do_move_assign(svector&& other) {
+ if (!other.is_direct()) {
+ // take other's memory, even when empty
+ set_indirect(other.indirect());
+ } else {
+ auto* other_ptr = other.data<direction::direct>();
+ auto s = other.size<direction::direct>();
+ auto* other_end = other_ptr + s;
+
+ std::uninitialized_move(other_ptr, other_end, data<direction::direct>());
+ std::destroy(other_ptr, other_end);
+ set_size(s);
+ }
+ other.set_direct_and_size(0);
+ }
+
+ /**
+ * @brief Shifts data [source_begin, source_end( to the right, starting on target_begin.
+ *
+ * Preconditions:
+ * * contiguous memory
+ * * source_begin <= target_begin
+ * * source_end onwards is uninitialized memory
+ *
+ * Destroys then empty elements in [source_begin, source_end(
+ */
+ static void shift_right(T* source_begin, T* source_end, T* target_begin) {
+ // 1. uninitialized moves
+ auto const num_moves = std::distance(source_begin, source_end);
+ auto const target_end = target_begin + num_moves;
+ auto const num_uninitialized_move = std::min(num_moves, std::distance(source_end, target_end));
+ std::uninitialized_move(source_end - num_uninitialized_move, source_end, target_end - num_uninitialized_move);
+ std::move_backward(source_begin, source_end - num_uninitialized_move, target_end - num_uninitialized_move);
+ std::destroy(source_begin, std::min(source_end, target_begin));
+ }
+
+ template <direction D>
+ [[nodiscard]] auto make_uninitialized_space_new(size_t s, T* p, size_t count) -> T* {
+ auto target = svector();
+ // we know target is indirect because we're increasing capacity
+ target.reserve(s + count);
+
+ // move everything [begin, pos[
+ auto* target_pos = std::uninitialized_move(data<D>(), p, target.template data<direction::indirect>());
+
+ // move everything [pos, end]
+ std::uninitialized_move(p, data<D>() + s, target_pos + count);
+
+ target.template set_size<direction::indirect>(s + count);
+ *this = std::move(target);
+ return target_pos;
+ }
+
+ template <direction D>
+ [[nodiscard]] auto make_uninitialized_space(T const* pos, size_t count) -> T* {
+ auto* const p = const_cast<T*>(pos); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ auto s = size<D>();
+ if (s + count > capacity<D>()) {
+ return make_uninitialized_space_new<D>(s, p, count);
+ }
+
+ shift_right(p, data<D>() + s, p + count);
+ set_size<D>(s + count);
+ return p;
+ }
+
+ // makes space for uninitialized data of cout elements. Also updates size.
+ [[nodiscard]] auto make_uninitialized_space(T const* pos, size_t count) -> T* {
+ if (is_direct()) {
+ return make_uninitialized_space<direction::direct>(pos, count);
+ }
+ return make_uninitialized_space<direction::indirect>(pos, count);
+ }
+
+ void destroy() {
+ auto const is_dir = is_direct();
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ T* ptr = nullptr;
+ size_t s = 0;
+ if (is_dir) {
+ ptr = data<direction::direct>();
+ s = size<direction::direct>();
+ } else {
+ ptr = data<direction::indirect>();
+ s = size<direction::indirect>();
+ }
+ std::destroy_n(ptr, s);
+ }
+ if (!is_dir) {
+ auto* storage = indirect();
+ std::destroy_at(storage);
+ ::operator delete(storage);
+ }
+ set_direct_and_size(0);
+ }
+
+ // performs a const_cast so we don't need this implementation twice
+ template <direction D>
+ auto at(size_t idx) -> T& {
+ if (idx >= size<D>()) {
+ throw std::out_of_range{"svector: idx out of range"};
+ }
+ auto* ptr = const_cast<T*>(data<D>() + idx); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ return *ptr;
+ } // LCOV_EXCL_LINE why is this single } marked as not covered? gcov bug?
+
+public:
+ using value_type = T;
+ using size_type = size_t;
+ using difference_type = std::ptrdiff_t;
+ using reference = value_type&;
+ using const_reference = value_type const&;
+ using pointer = T*;
+ using const_pointer = T const*;
+ using iterator = T*;
+ using const_iterator = T const*;
+ using reverse_iterator = std::reverse_iterator<iterator>;
+ using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+
+ svector() {
+ set_direct_and_size(0);
+ }
+
+ svector(size_t count, T const& value)
+ : svector() {
+ resize(count, value);
+ }
+
+ explicit svector(size_t count)
+ : svector() {
+ reserve(count);
+ if (is_direct()) {
+ resize_after_reserve<direction::direct>(count);
+ } else {
+ resize_after_reserve<direction::indirect>(count);
+ }
+ }
+
+ template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>>
+ svector(InputIt first, InputIt last)
+ : svector() {
+ assign(first, last);
+ }
+
+ svector(svector const& other)
+ : svector() {
+ auto s = other.size();
+ reserve(s);
+ std::uninitialized_copy(other.begin(), other.end(), begin());
+ set_size(s);
+ }
+
+ svector(svector&& other) noexcept
+ : svector() {
+ do_move_assign(std::move(other));
+ }
+
+ svector(std::initializer_list<T> init)
+ : svector(init.begin(), init.end()) {}
+
+ ~svector() {
+ destroy();
+ }
+
+ void assign(size_t count, T const& value) {
+ clear();
+ resize(count, value);
+ }
+
+ template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>>
+ void assign(InputIt first, InputIt last) {
+ assign(first, last, typename std::iterator_traits<InputIt>::iterator_category());
+ }
+
+ void assign(std::initializer_list<T> l) {
+ assign(l.begin(), l.end());
+ }
+
+ auto operator=(svector const& other) -> svector& {
+ if (&other == this) {
+ return *this;
+ }
+
+ assign(other.begin(), other.end());
+ return *this;
+ }
+
+ auto operator=(svector&& other) noexcept -> svector& {
+ if (&other == this) {
+ // It doesn't seem to be required to do self-check, but let's do it anyways to be safe
+ return *this;
+ }
+ destroy();
+ do_move_assign(std::move(other));
+ return *this;
+ }
+
+ auto operator=(std::initializer_list<T> l) -> svector& {
+ assign(l.begin(), l.end());
+ return *this;
+ }
+
+ void resize(size_t count) {
+ if (count > capacity()) {
+ reserve(count);
+ }
+ if (is_direct()) {
+ resize_after_reserve<direction::direct>(count);
+ } else {
+ resize_after_reserve<direction::indirect>(count);
+ }
+ }
+
+ void resize(size_t count, T const& value) {
+ if (count > capacity()) {
+ reserve(count);
+ }
+ if (is_direct()) {
+ resize_after_reserve<direction::direct>(count, value);
+ } else {
+ resize_after_reserve<direction::indirect>(count, value);
+ }
+ }
+
+ void reserve(size_t s) {
+ auto old_capacity = capacity();
+ auto new_capacity = calculate_new_capacity(s, old_capacity);
+ if (new_capacity > old_capacity) {
+ realloc(new_capacity);
+ }
+ }
+
+ [[nodiscard]] auto capacity() const -> size_t {
+ if (is_direct()) {
+ return capacity<direction::direct>();
+ }
+ return capacity<direction::indirect>();
+ }
+
+ [[nodiscard]] auto size() const -> size_t {
+ if (is_direct()) {
+ return size<direction::direct>();
+ }
+ return size<direction::indirect>();
+ }
+
+ [[nodiscard]] auto data() -> T* {
+ if (is_direct()) {
+ return direct_data();
+ }
+ return indirect()->data();
+ }
+
+ [[nodiscard]] auto data() const -> T const* {
+ return const_cast<svector*>(this)->data(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ template <class... Args>
+ auto emplace_back(Args&&... args) -> T& {
+ size_t c; // NOLINT(cppcoreguidelines-init-variables)
+ size_t s; // NOLINT(cppcoreguidelines-init-variables)
+ bool is_dir = is_direct();
+ if (is_dir) {
+ c = capacity<direction::direct>();
+ s = size<direction::direct>();
+ } else {
+ c = capacity<direction::indirect>();
+ s = size<direction::indirect>();
+ }
+
+ if (s == c) {
+ auto new_capacity = calculate_new_capacity(s + 1, c);
+ realloc(new_capacity);
+ // reallocation happened, so we definitely are now in indirect mode
+ is_dir = false;
+ }
+
+ T* ptr; // NOLINT(cppcoreguidelines-init-variables)
+ if (is_dir) {
+ ptr = data<direction::direct>() + s;
+ set_size<direction::direct>(s + 1);
+ } else {
+ ptr = data<direction::indirect>() + s;
+ set_size<direction::indirect>(s + 1);
+ }
+ return *new (static_cast<void*>(ptr)) T(std::forward<Args>(args)...);
+ }
+
+ void push_back(T const& value) {
+ emplace_back(value);
+ }
+
+ void push_back(T&& value) {
+ emplace_back(std::move(value));
+ }
+
+ [[nodiscard]] auto operator[](size_t idx) const -> T const& {
+ return *(data() + idx);
+ }
+
+ [[nodiscard]] auto operator[](size_t idx) -> T& {
+ return *(data() + idx);
+ }
+
+ auto at(size_t idx) -> T& {
+ if (is_direct()) {
+ return at<direction::direct>(idx);
+ }
+ return at<direction::indirect>(idx);
+ }
+
+ auto at(size_t idx) const -> T const& {
+ return const_cast<svector*>(this)->at(idx); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ [[nodiscard]] auto begin() const -> T const* {
+ return data();
+ }
+
+ [[nodiscard]] auto cbegin() const -> T const* {
+ return begin();
+ }
+
+ [[nodiscard]] auto begin() -> T* {
+ return data();
+ }
+
+ [[nodiscard]] auto end() -> T* {
+ if (is_direct()) {
+ return data<direction::direct>() + size<direction::direct>();
+ }
+ return data<direction::indirect>() + size<direction::indirect>();
+ }
+
+ [[nodiscard]] auto end() const -> T const* {
+ return const_cast<svector*>(this)->end(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ [[nodiscard]] auto cend() const -> T const* {
+ return end();
+ }
+
+ [[nodiscard]] auto rbegin() -> reverse_iterator {
+ return reverse_iterator{end()};
+ }
+
+ [[nodiscard]] auto rbegin() const -> const_reverse_iterator {
+ return crbegin();
+ }
+
+ [[nodiscard]] auto crbegin() const -> const_reverse_iterator {
+ return const_reverse_iterator{end()};
+ }
+
+ [[nodiscard]] auto rend() -> reverse_iterator {
+ return reverse_iterator{begin()};
+ }
+
+ [[nodiscard]] auto rend() const -> const_reverse_iterator {
+ return crend();
+ }
+
+ [[nodiscard]] auto crend() const -> const_reverse_iterator {
+ return const_reverse_iterator{begin()};
+ }
+
+ [[nodiscard]] auto front() const -> T const& {
+ return *data();
+ }
+
+ [[nodiscard]] auto front() -> T& {
+ return *data();
+ }
+
+ [[nodiscard]] auto back() -> T& {
+ if (is_direct()) {
+ return *(data<direction::direct>() + size<direction::direct>() - 1);
+ }
+ return *(data<direction::indirect>() + size<direction::indirect>() - 1);
+ }
+
+ [[nodiscard]] auto back() const -> T const& {
+ return const_cast<svector*>(this)->back(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ void clear() {
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ std::destroy(begin(), end());
+ }
+
+ if (is_direct()) {
+ set_size<direction::direct>(0);
+ } else {
+ set_size<direction::indirect>(0);
+ }
+ }
+
+ [[nodiscard]] auto empty() const -> bool {
+ return 0U == size();
+ }
+
+ void pop_back() {
+ if (is_direct()) {
+ pop_back<direction::direct>();
+ } else {
+ pop_back<direction::indirect>();
+ }
+ }
+
+ [[nodiscard]] static auto max_size() -> size_t {
+ return std::numeric_limits<std::ptrdiff_t>::max();
+ }
+
+ void swap(svector& other) {
+ // TODO we could try to do the minimum number of moves
+ std::swap(*this, other);
+ }
+
+ void shrink_to_fit() {
+ // per the standard we wouldn't need to do anything here. But since we are so nice,
+ // let's do the shrink.
+ auto const c = capacity();
+ auto const s = size();
+ if (s >= c) {
+ return;
+ }
+
+ auto new_capacity = calculate_new_capacity(s, N);
+ if (new_capacity == c) {
+ // nothing change!
+ return;
+ }
+
+ realloc(new_capacity);
+ }
+
+ template <class... Args>
+ auto emplace(const_iterator pos, Args&&... args) -> iterator {
+ auto* p = make_uninitialized_space(pos, 1);
+ return new (static_cast<void*>(p)) T(std::forward<Args>(args)...);
+ }
+
+ auto insert(const_iterator pos, T const& value) -> iterator {
+ return emplace(pos, value);
+ }
+
+ auto insert(const_iterator pos, T&& value) -> iterator {
+ return emplace(pos, std::move(value));
+ }
+
+ auto insert(const_iterator pos, size_t count, T const& value) -> iterator {
+ auto* p = make_uninitialized_space(pos, count);
+ std::uninitialized_fill_n(p, count, value);
+ return p;
+ }
+
+ template <typename It>
+ auto insert(const_iterator pos, It first, It last, std::input_iterator_tag /*unused*/) {
+ if (!(first != last)) {
+ return const_cast<T*>(pos); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ // just input_iterator_tag makes this very slow. Let's do the same as the STL.
+ if (pos == end()) {
+ auto s = size();
+ while (first != last) {
+ emplace_back(*first);
+ ++first;
+ }
+ return begin() + s;
+ }
+
+ auto tmp = svector(first, last);
+ return insert(pos, std::make_move_iterator(tmp.begin()), std::make_move_iterator(tmp.end()));
+ }
+
+ template <typename It>
+ auto insert(const_iterator pos, It first, It last, std::forward_iterator_tag /*unused*/) -> iterator {
+ auto* p = make_uninitialized_space(pos, std::distance(first, last));
+ std::uninitialized_copy(first, last, p);
+ return p;
+ }
+
+ template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>>
+ auto insert(const_iterator pos, InputIt first, InputIt last) -> iterator {
+ return insert(pos, first, last, typename std::iterator_traits<InputIt>::iterator_category());
+ }
+
+ auto insert(const_iterator pos, std::initializer_list<T> l) -> iterator {
+ return insert(pos, l.begin(), l.end());
+ }
+
+ auto erase(const_iterator pos) -> iterator {
+ return erase(pos, pos + 1);
+ }
+
+ auto erase(const_iterator first, const_iterator last) -> iterator {
+ if (is_direct()) {
+ return erase_checked_end<direction::direct>(first, last);
+ }
+ return erase_checked_end<direction::indirect>(first, last);
+ }
+};
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator==(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return std::equal(a.begin(), a.end(), b.begin(), b.end());
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator!=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return !(a == b);
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator<(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end());
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator>=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return !(a < b);
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator>(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return std::lexicographical_compare(b.begin(), b.end(), a.begin(), a.end());
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator<=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return !(a > b);
+}
+
+} // namespace ANKERL_SVECTOR_NAMESPACE
+} // namespace ankerl
+
+// NOLINTNEXTLINE(cert-dcl58-cpp)
+namespace std {
+inline namespace ANKERL_SVECTOR_NAMESPACE {
+
+template <class T, size_t N, class U>
+constexpr auto erase(ankerl::svector<T, N>& sv, U const& value) -> typename ankerl::svector<T, N>::size_type {
+ auto* removed_begin = std::remove(sv.begin(), sv.end(), value);
+ auto num_removed = std::distance(removed_begin, sv.end());
+ sv.erase(removed_begin, sv.end());
+ return num_removed;
+}
+
+template <class T, size_t N, class Pred>
+constexpr auto erase_if(ankerl::svector<T, N>& sv, Pred pred) -> typename ankerl::svector<T, N>::size_type {
+ auto* removed_begin = std::remove_if(sv.begin(), sv.end(), pred);
+ auto num_removed = std::distance(removed_begin, sv.end());
+ sv.erase(removed_begin, sv.end());
+ return num_removed;
+}
+
+} // namespace ANKERL_SVECTOR_NAMESPACE
+} // namespace std
+
+#endif
diff --git a/contrib/ankerl/unordered_dense.h b/contrib/ankerl/unordered_dense.h
new file mode 100644
index 0000000..2aaacd6
--- /dev/null
+++ b/contrib/ankerl/unordered_dense.h
@@ -0,0 +1,2032 @@
+///////////////////////// ankerl::unordered_dense::{map, set} /////////////////////////
+
+// A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion.
+// Version 4.4.0
+// https://github.com/martinus/unordered_dense
+//
+// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2022-2023 Martin Leitner-Ankerl <martin.ankerl@gmail.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#ifndef ANKERL_UNORDERED_DENSE_H
+#define ANKERL_UNORDERED_DENSE_H
+
+// see https://semver.org/spec/v2.0.0.html
+#define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 4 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes
+#define ANKERL_UNORDERED_DENSE_VERSION_MINOR 4 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality
+#define ANKERL_UNORDERED_DENSE_VERSION_PATCH 0 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes
+
+// API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/
+
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT(major, minor, patch) ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch)
+#define ANKERL_UNORDERED_DENSE_NAMESPACE \
+ ANKERL_UNORDERED_DENSE_VERSION_CONCAT( \
+ ANKERL_UNORDERED_DENSE_VERSION_MAJOR, ANKERL_UNORDERED_DENSE_VERSION_MINOR, ANKERL_UNORDERED_DENSE_VERSION_PATCH)
+
+#if defined(_MSVC_LANG)
+# define ANKERL_UNORDERED_DENSE_CPP_VERSION _MSVC_LANG
+#else
+# define ANKERL_UNORDERED_DENSE_CPP_VERSION __cplusplus
+#endif
+
+#if defined(__GNUC__)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_PACK(decl) decl __attribute__((__packed__))
+#elif defined(_MSC_VER)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_PACK(decl) __pragma(pack(push, 1)) decl __pragma(pack(pop))
+#endif
+
+// exceptions
+#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)
+# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 1 // NOLINT(cppcoreguidelines-macro-usage)
+#else
+# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 0 // NOLINT(cppcoreguidelines-macro-usage)
+#endif
+#ifdef _MSC_VER
+# define ANKERL_UNORDERED_DENSE_NOINLINE __declspec(noinline)
+#else
+# define ANKERL_UNORDERED_DENSE_NOINLINE __attribute__((noinline))
+#endif
+
+// defined in unordered_dense.cpp
+#if !defined(ANKERL_UNORDERED_DENSE_EXPORT)
+# define ANKERL_UNORDERED_DENSE_EXPORT
+#endif
+
+#if ANKERL_UNORDERED_DENSE_CPP_VERSION < 201703L
+# error ankerl::unordered_dense requires C++17 or higher
+#else
+# include <array> // for array
+# include <cstdint> // for uint64_t, uint32_t, uint8_t, UINT64_C
+# include <cstring> // for size_t, memcpy, memset
+# include <functional> // for equal_to, hash
+# include <initializer_list> // for initializer_list
+# include <iterator> // for pair, distance
+# include <limits> // for numeric_limits
+# include <memory> // for allocator, allocator_traits, shared_ptr
+# include <optional> // for optional
+# include <stdexcept> // for out_of_range
+# include <string> // for basic_string
+# include <string_view> // for basic_string_view, hash
+# include <tuple> // for forward_as_tuple
+# include <type_traits> // for enable_if_t, declval, conditional_t, ena...
+# include <utility> // for forward, exchange, pair, as_const, piece...
+# include <vector> // for vector
+# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() == 0
+# include <cstdlib> // for abort
+# endif
+
+# if defined(__has_include)
+# if __has_include(<memory_resource>)
+# define ANKERL_UNORDERED_DENSE_PMR std::pmr // NOLINT(cppcoreguidelines-macro-usage)
+# include <memory_resource> // for polymorphic_allocator
+# elif __has_include(<experimental/memory_resource>)
+# define ANKERL_UNORDERED_DENSE_PMR std::experimental::pmr // NOLINT(cppcoreguidelines-macro-usage)
+# include <experimental/memory_resource> // for polymorphic_allocator
+# endif
+# endif
+
+# if defined(_MSC_VER) && defined(_M_X64)
+# include <intrin.h>
+# pragma intrinsic(_umul128)
+# endif
+
+# if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
+# define ANKERL_UNORDERED_DENSE_LIKELY(x) __builtin_expect(x, 1) // NOLINT(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_UNLIKELY(x) __builtin_expect(x, 0) // NOLINT(cppcoreguidelines-macro-usage)
+# else
+# define ANKERL_UNORDERED_DENSE_LIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_UNLIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage)
+# endif
+
+namespace ankerl::unordered_dense {
+inline namespace ANKERL_UNORDERED_DENSE_NAMESPACE {
+
+namespace detail {
+
+# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS()
+
+// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
+// inlinings more difficult. Throws are also generally the slow path.
+[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_key_not_found() {
+ throw std::out_of_range("ankerl::unordered_dense::map::at(): key not found");
+}
+[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_bucket_overflow() {
+ throw std::overflow_error("ankerl::unordered_dense: reached max bucket size, cannot increase size");
+}
+[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_too_many_elements() {
+ throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements");
+}
+
+# else
+
+[[noreturn]] inline void on_error_key_not_found() {
+ abort();
+}
+[[noreturn]] inline void on_error_bucket_overflow() {
+ abort();
+}
+[[noreturn]] inline void on_error_too_many_elements() {
+ abort();
+}
+
+# endif
+
+} // namespace detail
+
+// hash ///////////////////////////////////////////////////////////////////////
+
+// This is a stripped-down implementation of wyhash: https://github.com/wangyi-fudan/wyhash
+// No big-endian support (because different values on different machines don't matter),
+// hardcodes seed and the secret, reformats the code, and clang-tidy fixes.
+namespace detail::wyhash {
+
+inline void mum(uint64_t* a, uint64_t* b) {
+# if defined(__SIZEOF_INT128__)
+ __uint128_t r = *a;
+ r *= *b;
+ *a = static_cast<uint64_t>(r);
+ *b = static_cast<uint64_t>(r >> 64U);
+# elif defined(_MSC_VER) && defined(_M_X64)
+ *a = _umul128(*a, *b, b);
+# else
+ uint64_t ha = *a >> 32U;
+ uint64_t hb = *b >> 32U;
+ uint64_t la = static_cast<uint32_t>(*a);
+ uint64_t lb = static_cast<uint32_t>(*b);
+ uint64_t hi{};
+ uint64_t lo{};
+ uint64_t rh = ha * hb;
+ uint64_t rm0 = ha * lb;
+ uint64_t rm1 = hb * la;
+ uint64_t rl = la * lb;
+ uint64_t t = rl + (rm0 << 32U);
+ auto c = static_cast<uint64_t>(t < rl);
+ lo = t + (rm1 << 32U);
+ c += static_cast<uint64_t>(lo < t);
+ hi = rh + (rm0 >> 32U) + (rm1 >> 32U) + c;
+ *a = lo;
+ *b = hi;
+# endif
+}
+
+// multiply and xor mix function, aka MUM
+[[nodiscard]] inline auto mix(uint64_t a, uint64_t b) -> uint64_t {
+ mum(&a, &b);
+ return a ^ b;
+}
+
+// read functions. WARNING: we don't care about endianness, so results are different on big endian!
+[[nodiscard]] inline auto r8(const uint8_t* p) -> uint64_t {
+ uint64_t v{};
+ std::memcpy(&v, p, 8U);
+ return v;
+}
+
+[[nodiscard]] inline auto r4(const uint8_t* p) -> uint64_t {
+ uint32_t v{};
+ std::memcpy(&v, p, 4);
+ return v;
+}
+
+// reads 1, 2, or 3 bytes
+[[nodiscard]] inline auto r3(const uint8_t* p, size_t k) -> uint64_t {
+ return (static_cast<uint64_t>(p[0]) << 16U) | (static_cast<uint64_t>(p[k >> 1U]) << 8U) | p[k - 1];
+}
+
+[[maybe_unused]] [[nodiscard]] inline auto hash(void const* key, size_t len) -> uint64_t {
+ static constexpr auto secret = std::array{UINT64_C(0xa0761d6478bd642f),
+ UINT64_C(0xe7037ed1a0b428db),
+ UINT64_C(0x8ebc6af09c88c6e3),
+ UINT64_C(0x589965cc75374cc3)};
+
+ auto const* p = static_cast<uint8_t const*>(key);
+ uint64_t seed = secret[0];
+ uint64_t a{};
+ uint64_t b{};
+ if (ANKERL_UNORDERED_DENSE_LIKELY(len <= 16)) {
+ if (ANKERL_UNORDERED_DENSE_LIKELY(len >= 4)) {
+ a = (r4(p) << 32U) | r4(p + ((len >> 3U) << 2U));
+ b = (r4(p + len - 4) << 32U) | r4(p + len - 4 - ((len >> 3U) << 2U));
+ } else if (ANKERL_UNORDERED_DENSE_LIKELY(len > 0)) {
+ a = r3(p, len);
+ b = 0;
+ } else {
+ a = 0;
+ b = 0;
+ }
+ } else {
+ size_t i = len;
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 48)) {
+ uint64_t see1 = seed;
+ uint64_t see2 = seed;
+ do {
+ seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
+ see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1);
+ see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2);
+ p += 48;
+ i -= 48;
+ } while (ANKERL_UNORDERED_DENSE_LIKELY(i > 48));
+ seed ^= see1 ^ see2;
+ }
+ while (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 16)) {
+ seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
+ i -= 16;
+ p += 16;
+ }
+ a = r8(p + i - 16);
+ b = r8(p + i - 8);
+ }
+
+ return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed));
+}
+
+[[nodiscard]] inline auto hash(uint64_t x) -> uint64_t {
+ return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15));
+}
+
+} // namespace detail::wyhash
+
+ANKERL_UNORDERED_DENSE_EXPORT template <typename T, typename Enable = void>
+struct hash {
+ auto operator()(T const& obj) const noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>())))
+ -> uint64_t {
+ return std::hash<T>{}(obj);
+ }
+};
+
+template <typename CharT>
+struct hash<std::basic_string<CharT>> {
+ using is_avalanching = void;
+ auto operator()(std::basic_string<CharT> const& str) const noexcept -> uint64_t {
+ return detail::wyhash::hash(str.data(), sizeof(CharT) * str.size());
+ }
+};
+
+template <typename CharT>
+struct hash<std::basic_string_view<CharT>> {
+ using is_avalanching = void;
+ auto operator()(std::basic_string_view<CharT> const& sv) const noexcept -> uint64_t {
+ return detail::wyhash::hash(sv.data(), sizeof(CharT) * sv.size());
+ }
+};
+
+template <class T>
+struct hash<T*> {
+ using is_avalanching = void;
+ auto operator()(T* ptr) const noexcept -> uint64_t {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr));
+ }
+};
+
+template <class T>
+struct hash<std::unique_ptr<T>> {
+ using is_avalanching = void;
+ auto operator()(std::unique_ptr<T> const& ptr) const noexcept -> uint64_t {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
+ }
+};
+
+template <class T>
+struct hash<std::shared_ptr<T>> {
+ using is_avalanching = void;
+ auto operator()(std::shared_ptr<T> const& ptr) const noexcept -> uint64_t {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
+ }
+};
+
+template <typename Enum>
+struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
+ using is_avalanching = void;
+ auto operator()(Enum e) const noexcept -> uint64_t {
+ using underlying = typename std::underlying_type_t<Enum>;
+ return detail::wyhash::hash(static_cast<underlying>(e));
+ }
+};
+
+template <typename... Args>
+struct tuple_hash_helper {
+ // Converts the value into 64bit. If it is an integral type, just cast it. Mixing is doing the rest.
+ // If it isn't an integral we need to hash it.
+ template <typename Arg>
+ [[nodiscard]] constexpr static auto to64(Arg const& arg) -> uint64_t {
+ if constexpr (std::is_integral_v<Arg> || std::is_enum_v<Arg>) {
+ return static_cast<uint64_t>(arg);
+ } else {
+ return hash<Arg>{}(arg);
+ }
+ }
+
+ [[nodiscard]] static auto mix64(uint64_t state, uint64_t v) -> uint64_t {
+ return detail::wyhash::mix(state + v, uint64_t{0x9ddfea08eb382d69});
+ }
+
+ // Creates a buffer that holds all the data from each element of the tuple. If possible we memcpy the data directly. If
+ // not, we hash the object and use this for the array. Size of the array is known at compile time, and memcpy is optimized
+ // away, so filling the buffer is highly efficient. Finally, call wyhash with this buffer.
+ template <typename T, std::size_t... Idx>
+ [[nodiscard]] static auto calc_hash(T const& t, std::index_sequence<Idx...>) noexcept -> uint64_t {
+ auto h = uint64_t{};
+ ((h = mix64(h, to64(std::get<Idx>(t)))), ...);
+ return h;
+ }
+};
+
+template <typename... Args>
+struct hash<std::tuple<Args...>> : tuple_hash_helper<Args...> {
+ using is_avalanching = void;
+ auto operator()(std::tuple<Args...> const& t) const noexcept -> uint64_t {
+ return tuple_hash_helper<Args...>::calc_hash(t, std::index_sequence_for<Args...>{});
+ }
+};
+
+template <typename A, typename B>
+struct hash<std::pair<A, B>> : tuple_hash_helper<A, B> {
+ using is_avalanching = void;
+ auto operator()(std::pair<A, B> const& t) const noexcept -> uint64_t {
+ return tuple_hash_helper<A, B>::calc_hash(t, std::index_sequence_for<A, B>{});
+ }
+};
+
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_HASH_STATICCAST(T) \
+ template <> \
+ struct hash<T> { \
+ using is_avalanching = void; \
+ auto operator()(T const& obj) const noexcept -> uint64_t { \
+ return detail::wyhash::hash(static_cast<uint64_t>(obj)); \
+ } \
+ }
+
+# if defined(__GNUC__) && !defined(__clang__)
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wuseless-cast"
+# endif
+// see https://en.cppreference.com/w/cpp/utility/hash
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(bool);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(signed char);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned char);
+# if ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L && defined(__cpp_char8_t)
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char8_t);
+# endif
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char16_t);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char32_t);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(wchar_t);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(short);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned short);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(int);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned int);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long long);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long long);
+
+# if defined(__GNUC__) && !defined(__clang__)
+# pragma GCC diagnostic pop
+# endif
+
+// bucket_type //////////////////////////////////////////////////////////
+
+namespace bucket_type {
+
+struct standard {
+ static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint
+ static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint
+
+ uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
+ uint32_t m_value_idx; // index into the m_values vector.
+};
+
+ANKERL_UNORDERED_DENSE_PACK(struct big {
+ static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint
+ static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint
+
+ uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
+ size_t m_value_idx; // index into the m_values vector.
+});
+
+} // namespace bucket_type
+
+namespace detail {
+
+struct nonesuch {};
+
+template <class Default, class AlwaysVoid, template <class...> class Op, class... Args>
+struct detector {
+ using value_t = std::false_type;
+ using type = Default;
+};
+
+template <class Default, template <class...> class Op, class... Args>
+struct detector<Default, std::void_t<Op<Args...>>, Op, Args...> {
+ using value_t = std::true_type;
+ using type = Op<Args...>;
+};
+
+template <template <class...> class Op, class... Args>
+using is_detected = typename detail::detector<detail::nonesuch, void, Op, Args...>::value_t;
+
+template <template <class...> class Op, class... Args>
+constexpr bool is_detected_v = is_detected<Op, Args...>::value;
+
+template <typename T>
+using detect_avalanching = typename T::is_avalanching;
+
+template <typename T>
+using detect_is_transparent = typename T::is_transparent;
+
+template <typename T>
+using detect_iterator = typename T::iterator;
+
+template <typename T>
+using detect_reserve = decltype(std::declval<T&>().reserve(size_t{}));
+
+// enable_if helpers
+
+template <typename Mapped>
+constexpr bool is_map_v = !std::is_void_v<Mapped>;
+
+// clang-format off
+template <typename Hash, typename KeyEqual>
+constexpr bool is_transparent_v = is_detected_v<detect_is_transparent, Hash> && is_detected_v<detect_is_transparent, KeyEqual>;
+// clang-format on
+
+template <typename From, typename To1, typename To2>
+constexpr bool is_neither_convertible_v = !std::is_convertible_v<From, To1> && !std::is_convertible_v<From, To2>;
+
+template <typename T>
+constexpr bool has_reserve = is_detected_v<detect_reserve, T>;
+
+// base type for map has mapped_type
+template <class T>
+struct base_table_type_map {
+ using mapped_type = T;
+};
+
+// base type for set doesn't have mapped_type
+struct base_table_type_set {};
+
+} // namespace detail
+
+// Very much like std::deque, but faster for indexing (in most cases). As of now this doesn't implement the full std::vector
+// API, but merely what's necessary to work as an underlying container for ankerl::unordered_dense::{map, set}.
+// It allocates blocks of equal size and puts them into the m_blocks vector. That means it can grow simply by adding a new
+// block to the back of m_blocks, and doesn't double its size like an std::vector. The disadvantage is that memory is not
+// linear and thus there is one more indirection necessary for indexing.
+template <typename T, typename Allocator = std::allocator<T>, size_t MaxSegmentSizeBytes = 4096>
+class segmented_vector {
+ template <bool IsConst>
+ class iter_t;
+
+public:
+ using allocator_type = Allocator;
+ using pointer = typename std::allocator_traits<allocator_type>::pointer;
+ using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer;
+ using difference_type = typename std::allocator_traits<allocator_type>::difference_type;
+ using value_type = T;
+ using size_type = std::size_t;
+ using reference = T&;
+ using const_reference = T const&;
+ using iterator = iter_t<false>;
+ using const_iterator = iter_t<true>;
+
+private:
+ using vec_alloc = typename std::allocator_traits<Allocator>::template rebind_alloc<pointer>;
+ std::vector<pointer, vec_alloc> m_blocks{};
+ size_t m_size{};
+
+ // Calculates the maximum number for x in (s << x) <= max_val
+ static constexpr auto num_bits_closest(size_t max_val, size_t s) -> size_t {
+ auto f = size_t{0};
+ while (s << (f + 1) <= max_val) {
+ ++f;
+ }
+ return f;
+ }
+
+ using self_t = segmented_vector<T, Allocator, MaxSegmentSizeBytes>;
+ static constexpr auto num_bits = num_bits_closest(MaxSegmentSizeBytes, sizeof(T));
+ static constexpr auto num_elements_in_block = 1U << num_bits;
+ static constexpr auto mask = num_elements_in_block - 1U;
+
+ /**
+ * Iterator class doubles as const_iterator and iterator
+ */
+ template <bool IsConst>
+ class iter_t {
+ using ptr_t = typename std::conditional_t<IsConst, segmented_vector::const_pointer const*, segmented_vector::pointer*>;
+ ptr_t m_data{};
+ size_t m_idx{};
+
+ template <bool B>
+ friend class iter_t;
+
+ public:
+ using difference_type = segmented_vector::difference_type;
+ using value_type = T;
+ using reference = typename std::conditional_t<IsConst, value_type const&, value_type&>;
+ using pointer = typename std::conditional_t<IsConst, segmented_vector::const_pointer, segmented_vector::pointer>;
+ using iterator_category = std::forward_iterator_tag;
+
+ iter_t() noexcept = default;
+
+ template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
+ // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions)
+ constexpr iter_t(iter_t<OtherIsConst> const& other) noexcept
+ : m_data(other.m_data)
+ , m_idx(other.m_idx) {}
+
+ constexpr iter_t(ptr_t data, size_t idx) noexcept
+ : m_data(data)
+ , m_idx(idx) {}
+
+ template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
+ constexpr auto operator=(iter_t<OtherIsConst> const& other) noexcept -> iter_t& {
+ m_data = other.m_data;
+ m_idx = other.m_idx;
+ return *this;
+ }
+
+ constexpr auto operator++() noexcept -> iter_t& {
+ ++m_idx;
+ return *this;
+ }
+
+ constexpr auto operator+(difference_type diff) noexcept -> iter_t {
+ return {m_data, static_cast<size_t>(static_cast<difference_type>(m_idx) + diff)};
+ }
+
+ template <bool OtherIsConst>
+ constexpr auto operator-(iter_t<OtherIsConst> const& other) noexcept -> difference_type {
+ return static_cast<difference_type>(m_idx) - static_cast<difference_type>(other.m_idx);
+ }
+
+ constexpr auto operator*() const noexcept -> reference {
+ return m_data[m_idx >> num_bits][m_idx & mask];
+ }
+
+ constexpr auto operator->() const noexcept -> pointer {
+ return &m_data[m_idx >> num_bits][m_idx & mask];
+ }
+
+ template <bool O>
+ constexpr auto operator==(iter_t<O> const& o) const noexcept -> bool {
+ return m_idx == o.m_idx;
+ }
+
+ template <bool O>
+ constexpr auto operator!=(iter_t<O> const& o) const noexcept -> bool {
+ return !(*this == o);
+ }
+ };
+
+ // slow path: need to allocate a new segment every once in a while
+ void increase_capacity() {
+ auto ba = Allocator(m_blocks.get_allocator());
+ pointer block = std::allocator_traits<Allocator>::allocate(ba, num_elements_in_block);
+ m_blocks.push_back(block);
+ }
+
+ // Moves everything from other
+ void append_everything_from(segmented_vector&& other) {
+ reserve(size() + other.size());
+ for (auto&& o : other) {
+ emplace_back(std::move(o));
+ }
+ }
+
+ // Copies everything from other
+ void append_everything_from(segmented_vector const& other) {
+ reserve(size() + other.size());
+ for (auto const& o : other) {
+ emplace_back(o);
+ }
+ }
+
+ void dealloc() {
+ auto ba = Allocator(m_blocks.get_allocator());
+ for (auto ptr : m_blocks) {
+ std::allocator_traits<Allocator>::deallocate(ba, ptr, num_elements_in_block);
+ }
+ }
+
+ [[nodiscard]] static constexpr auto calc_num_blocks_for_capacity(size_t capacity) {
+ return (capacity + num_elements_in_block - 1U) / num_elements_in_block;
+ }
+
+public:
+ segmented_vector() = default;
+
+ // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions)
+ segmented_vector(Allocator alloc)
+ : m_blocks(vec_alloc(alloc)) {}
+
+ segmented_vector(segmented_vector&& other, Allocator alloc)
+ : segmented_vector(alloc) {
+ *this = std::move(other);
+ }
+
+ segmented_vector(segmented_vector const& other, Allocator alloc)
+ : m_blocks(vec_alloc(alloc)) {
+ append_everything_from(other);
+ }
+
+ segmented_vector(segmented_vector&& other) noexcept
+ : segmented_vector(std::move(other), get_allocator()) {}
+
+ segmented_vector(segmented_vector const& other) {
+ append_everything_from(other);
+ }
+
+ auto operator=(segmented_vector const& other) -> segmented_vector& {
+ if (this == &other) {
+ return *this;
+ }
+ clear();
+ append_everything_from(other);
+ return *this;
+ }
+
+ auto operator=(segmented_vector&& other) noexcept -> segmented_vector& {
+ clear();
+ dealloc();
+ if (other.get_allocator() == get_allocator()) {
+ m_blocks = std::move(other.m_blocks);
+ m_size = std::exchange(other.m_size, {});
+ } else {
+ // make sure to construct with other's allocator!
+ m_blocks = std::vector<pointer, vec_alloc>(vec_alloc(other.get_allocator()));
+ append_everything_from(std::move(other));
+ }
+ return *this;
+ }
+
+ ~segmented_vector() {
+ clear();
+ dealloc();
+ }
+
+ [[nodiscard]] constexpr auto size() const -> size_t {
+ return m_size;
+ }
+
+ [[nodiscard]] constexpr auto capacity() const -> size_t {
+ return m_blocks.size() * num_elements_in_block;
+ }
+
+ // Indexing is highly performance critical
+ [[nodiscard]] constexpr auto operator[](size_t i) const noexcept -> T const& {
+ return m_blocks[i >> num_bits][i & mask];
+ }
+
+ [[nodiscard]] constexpr auto operator[](size_t i) noexcept -> T& {
+ return m_blocks[i >> num_bits][i & mask];
+ }
+
+ [[nodiscard]] constexpr auto begin() -> iterator {
+ return {m_blocks.data(), 0U};
+ }
+ [[nodiscard]] constexpr auto begin() const -> const_iterator {
+ return {m_blocks.data(), 0U};
+ }
+ [[nodiscard]] constexpr auto cbegin() const -> const_iterator {
+ return {m_blocks.data(), 0U};
+ }
+
+ [[nodiscard]] constexpr auto end() -> iterator {
+ return {m_blocks.data(), m_size};
+ }
+ [[nodiscard]] constexpr auto end() const -> const_iterator {
+ return {m_blocks.data(), m_size};
+ }
+ [[nodiscard]] constexpr auto cend() const -> const_iterator {
+ return {m_blocks.data(), m_size};
+ }
+
+ [[nodiscard]] constexpr auto back() -> reference {
+ return operator[](m_size - 1);
+ }
+ [[nodiscard]] constexpr auto back() const -> const_reference {
+ return operator[](m_size - 1);
+ }
+
+ void pop_back() {
+ back().~T();
+ --m_size;
+ }
+
+ [[nodiscard]] auto empty() const {
+ return 0 == m_size;
+ }
+
+ void reserve(size_t new_capacity) {
+ m_blocks.reserve(calc_num_blocks_for_capacity(new_capacity));
+ while (new_capacity > capacity()) {
+ increase_capacity();
+ }
+ }
+
+ [[nodiscard]] auto get_allocator() const -> allocator_type {
+ return allocator_type{m_blocks.get_allocator()};
+ }
+
+ template <class... Args>
+ auto emplace_back(Args&&... args) -> reference {
+ if (m_size == capacity()) {
+ increase_capacity();
+ }
+ auto* ptr = static_cast<void*>(&operator[](m_size));
+ auto& ref = *new (ptr) T(std::forward<Args>(args)...);
+ ++m_size;
+ return ref;
+ }
+
+ void clear() {
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ for (size_t i = 0, s = size(); i < s; ++i) {
+ operator[](i).~T();
+ }
+ }
+ m_size = 0;
+ }
+
+ void shrink_to_fit() {
+ auto ba = Allocator(m_blocks.get_allocator());
+ auto num_blocks_required = calc_num_blocks_for_capacity(m_size);
+ while (m_blocks.size() > num_blocks_required) {
+ std::allocator_traits<Allocator>::deallocate(ba, m_blocks.back(), num_elements_in_block);
+ m_blocks.pop_back();
+ }
+ m_blocks.shrink_to_fit();
+ }
+};
+
+namespace detail {
+
+// This is it, the table. Doubles as map and set, and uses `void` for T when its used as a set.
+template <class Key,
+ class T, // when void, treat it as a set.
+ class Hash,
+ class KeyEqual,
+ class AllocatorOrContainer,
+ class Bucket,
+ bool IsSegmented>
+class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, base_table_type_set> {
+ using underlying_value_type = typename std::conditional_t<is_map_v<T>, std::pair<Key, T>, Key>;
+ using underlying_container_type = std::conditional_t<IsSegmented,
+ segmented_vector<underlying_value_type, AllocatorOrContainer>,
+ std::vector<underlying_value_type, AllocatorOrContainer>>;
+
+public:
+ using value_container_type = std::
+ conditional_t<is_detected_v<detect_iterator, AllocatorOrContainer>, AllocatorOrContainer, underlying_container_type>;
+
+private:
+ using bucket_alloc =
+ typename std::allocator_traits<typename value_container_type::allocator_type>::template rebind_alloc<Bucket>;
+ using bucket_alloc_traits = std::allocator_traits<bucket_alloc>;
+
+ static constexpr uint8_t initial_shifts = 64 - 2; // 2^(64-m_shift) number of buckets
+ static constexpr float default_max_load_factor = 0.8F;
+
+public:
+ using key_type = Key;
+ using value_type = typename value_container_type::value_type;
+ using size_type = typename value_container_type::size_type;
+ using difference_type = typename value_container_type::difference_type;
+ using hasher = Hash;
+ using key_equal = KeyEqual;
+ using allocator_type = typename value_container_type::allocator_type;
+ using reference = typename value_container_type::reference;
+ using const_reference = typename value_container_type::const_reference;
+ using pointer = typename value_container_type::pointer;
+ using const_pointer = typename value_container_type::const_pointer;
+ using const_iterator = typename value_container_type::const_iterator;
+ using iterator = std::conditional_t<is_map_v<T>, typename value_container_type::iterator, const_iterator>;
+ using bucket_type = Bucket;
+
+private:
+ using value_idx_type = decltype(Bucket::m_value_idx);
+ using dist_and_fingerprint_type = decltype(Bucket::m_dist_and_fingerprint);
+
+ static_assert(std::is_trivially_destructible_v<Bucket>, "assert there's no need to call destructor / std::destroy");
+ static_assert(std::is_trivially_copyable_v<Bucket>, "assert we can just memset / memcpy");
+
+ value_container_type m_values{}; // Contains all the key-value pairs in one densely stored container. No holes.
+ using bucket_pointer = typename std::allocator_traits<bucket_alloc>::pointer;
+ bucket_pointer m_buckets{};
+ size_t m_num_buckets = 0;
+ size_t m_max_bucket_capacity = 0;
+ float m_max_load_factor = default_max_load_factor;
+ Hash m_hash{};
+ KeyEqual m_equal{};
+ uint8_t m_shifts = initial_shifts;
+
+ [[nodiscard]] auto next(value_idx_type bucket_idx) const -> value_idx_type {
+ return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == m_num_buckets)
+ ? 0
+ : static_cast<value_idx_type>(bucket_idx + 1U);
+ }
+
+ // Helper to access bucket through pointer types
+ [[nodiscard]] static constexpr auto at(bucket_pointer bucket_ptr, size_t offset) -> Bucket& {
+ return *(bucket_ptr + static_cast<typename std::allocator_traits<bucket_alloc>::difference_type>(offset));
+ }
+
+ // use the dist_inc and dist_dec functions so that uint16_t types work without warning
+ [[nodiscard]] static constexpr auto dist_inc(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
+ return static_cast<dist_and_fingerprint_type>(x + Bucket::dist_inc);
+ }
+
+ [[nodiscard]] static constexpr auto dist_dec(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
+ return static_cast<dist_and_fingerprint_type>(x - Bucket::dist_inc);
+ }
+
+ // The goal of mixed_hash is to always produce a high quality 64bit hash.
+ template <typename K>
+ [[nodiscard]] constexpr auto mixed_hash(K const& key) const -> uint64_t {
+ if constexpr (is_detected_v<detect_avalanching, Hash>) {
+ // we know that the hash is good because is_avalanching.
+ if constexpr (sizeof(decltype(m_hash(key))) < sizeof(uint64_t)) {
+ // 32bit hash and is_avalanching => multiply with a constant to avalanche bits upwards
+ return m_hash(key) * UINT64_C(0x9ddfea08eb382d69);
+ } else {
+ // 64bit and is_avalanching => only use the hash itself.
+ return m_hash(key);
+ }
+ } else {
+ // not is_avalanching => apply wyhash
+ return wyhash::hash(m_hash(key));
+ }
+ }
+
+ [[nodiscard]] constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> dist_and_fingerprint_type {
+ return Bucket::dist_inc | (static_cast<dist_and_fingerprint_type>(hash) & Bucket::fingerprint_mask);
+ }
+
+ [[nodiscard]] constexpr auto bucket_idx_from_hash(uint64_t hash) const -> value_idx_type {
+ return static_cast<value_idx_type>(hash >> m_shifts);
+ }
+
+ [[nodiscard]] static constexpr auto get_key(value_type const& vt) -> key_type const& {
+ if constexpr (is_map_v<T>) {
+ return vt.first;
+ } else {
+ return vt;
+ }
+ }
+
+ template <typename K>
+ [[nodiscard]] auto next_while_less(K const& key) const -> Bucket {
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (dist_and_fingerprint < at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+ return {dist_and_fingerprint, bucket_idx};
+ }
+
+ void place_and_shift_up(Bucket bucket, value_idx_type place) {
+ while (0 != at(m_buckets, place).m_dist_and_fingerprint) {
+ bucket = std::exchange(at(m_buckets, place), bucket);
+ bucket.m_dist_and_fingerprint = dist_inc(bucket.m_dist_and_fingerprint);
+ place = next(place);
+ }
+ at(m_buckets, place) = bucket;
+ }
+
+ [[nodiscard]] static constexpr auto calc_num_buckets(uint8_t shifts) -> size_t {
+ return (std::min)(max_bucket_count(), size_t{1} << (64U - shifts));
+ }
+
+ [[nodiscard]] constexpr auto calc_shifts_for_size(size_t s) const -> uint8_t {
+ auto shifts = initial_shifts;
+ while (shifts > 0 && static_cast<size_t>(static_cast<float>(calc_num_buckets(shifts)) * max_load_factor()) < s) {
+ --shifts;
+ }
+ return shifts;
+ }
+
+ // assumes m_values has data, m_buckets=m_buckets_end=nullptr, m_shifts is INITIAL_SHIFTS
+ void copy_buckets(table const& other) {
+ // assumes m_values has already the correct data copied over.
+ if (empty()) {
+ // when empty, at least allocate an initial buckets and clear them.
+ allocate_buckets_from_shift();
+ clear_buckets();
+ } else {
+ m_shifts = other.m_shifts;
+ allocate_buckets_from_shift();
+ std::memcpy(m_buckets, other.m_buckets, sizeof(Bucket) * bucket_count());
+ }
+ }
+
+ /**
+ * True when no element can be added any more without increasing the size
+ */
+ [[nodiscard]] auto is_full() const -> bool {
+ return size() > m_max_bucket_capacity;
+ }
+
+ void deallocate_buckets() {
+ auto ba = bucket_alloc(m_values.get_allocator());
+ if (nullptr != m_buckets) {
+ bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
+ m_buckets = nullptr;
+ }
+ m_num_buckets = 0;
+ m_max_bucket_capacity = 0;
+ }
+
+ void allocate_buckets_from_shift() {
+ auto ba = bucket_alloc(m_values.get_allocator());
+ m_num_buckets = calc_num_buckets(m_shifts);
+ m_buckets = bucket_alloc_traits::allocate(ba, m_num_buckets);
+ if (m_num_buckets == max_bucket_count()) {
+ // reached the maximum, make sure we can use each bucket
+ m_max_bucket_capacity = max_bucket_count();
+ } else {
+ m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(m_num_buckets) * max_load_factor());
+ }
+ }
+
+ void clear_buckets() {
+ if (m_buckets != nullptr) {
+ std::memset(&*m_buckets, 0, sizeof(Bucket) * bucket_count());
+ }
+ }
+
+ void clear_and_fill_buckets_from_values() {
+ clear_buckets();
+ for (value_idx_type value_idx = 0, end_idx = static_cast<value_idx_type>(m_values.size()); value_idx < end_idx;
+ ++value_idx) {
+ auto const& key = get_key(m_values[value_idx]);
+ auto [dist_and_fingerprint, bucket] = next_while_less(key);
+
+ // we know for certain that key has not yet been inserted, so no need to check it.
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket);
+ }
+ }
+
+ void increase_size() {
+ if (m_max_bucket_capacity == max_bucket_count()) {
+ // remove the value again, we can't add it!
+ m_values.pop_back();
+ on_error_bucket_overflow();
+ }
+ --m_shifts;
+ deallocate_buckets();
+ allocate_buckets_from_shift();
+ clear_and_fill_buckets_from_values();
+ }
+
+ template <typename Op>
+ void do_erase(value_idx_type bucket_idx, Op handle_erased_value) {
+ auto const value_idx_to_remove = at(m_buckets, bucket_idx).m_value_idx;
+
+ // shift down until either empty or an element with correct spot is found
+ auto next_bucket_idx = next(bucket_idx);
+ while (at(m_buckets, next_bucket_idx).m_dist_and_fingerprint >= Bucket::dist_inc * 2) {
+ at(m_buckets, bucket_idx) = {dist_dec(at(m_buckets, next_bucket_idx).m_dist_and_fingerprint),
+ at(m_buckets, next_bucket_idx).m_value_idx};
+ bucket_idx = std::exchange(next_bucket_idx, next(next_bucket_idx));
+ }
+ at(m_buckets, bucket_idx) = {};
+ handle_erased_value(std::move(m_values[value_idx_to_remove]));
+
+ // update m_values
+ if (value_idx_to_remove != m_values.size() - 1) {
+ // no luck, we'll have to replace the value with the last one and update the index accordingly
+ auto& val = m_values[value_idx_to_remove];
+ val = std::move(m_values.back());
+
+ // update the values_idx of the moved entry. No need to play the info game, just look until we find the values_idx
+ auto mh = mixed_hash(get_key(val));
+ bucket_idx = bucket_idx_from_hash(mh);
+
+ auto const values_idx_back = static_cast<value_idx_type>(m_values.size() - 1);
+ while (values_idx_back != at(m_buckets, bucket_idx).m_value_idx) {
+ bucket_idx = next(bucket_idx);
+ }
+ at(m_buckets, bucket_idx).m_value_idx = value_idx_to_remove;
+ }
+ m_values.pop_back();
+ }
+
+ template <typename K, typename Op>
+ auto do_erase_key(K&& key, Op handle_erased_value) -> size_t {
+ if (empty()) {
+ return 0;
+ }
+
+ auto [dist_and_fingerprint, bucket_idx] = next_while_less(key);
+
+ while (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
+ !m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ if (dist_and_fingerprint != at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ return 0;
+ }
+ do_erase(bucket_idx, handle_erased_value);
+ return 1;
+ }
+
+ template <class K, class M>
+ auto do_insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> {
+ auto it_isinserted = try_emplace(std::forward<K>(key), std::forward<M>(mapped));
+ if (!it_isinserted.second) {
+ it_isinserted.first->second = std::forward<M>(mapped);
+ }
+ return it_isinserted;
+ }
+
+ template <typename... Args>
+ auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint, value_idx_type bucket_idx, Args&&... args)
+ -> std::pair<iterator, bool> {
+
+ // emplace the new value. If that throws an exception, no harm done; index is still in a valid state
+ m_values.emplace_back(std::forward<Args>(args)...);
+
+ auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) {
+ increase_size();
+ } else {
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
+ }
+
+ // place element and shift up until we find an empty spot
+ return {begin() + static_cast<difference_type>(value_idx), true};
+ }
+
+ template <typename K, typename... Args>
+ auto do_try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> {
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (true) {
+ auto* bucket = &at(m_buckets, bucket_idx);
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
+ if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return {begin() + static_cast<difference_type>(bucket->m_value_idx), false};
+ }
+ } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
+ return do_place_element(dist_and_fingerprint,
+ bucket_idx,
+ std::piecewise_construct,
+ std::forward_as_tuple(std::forward<K>(key)),
+ std::forward_as_tuple(std::forward<Args>(args)...));
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+ }
+
+ template <typename K>
+ auto do_find(K const& key) -> iterator {
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(empty())) {
+ return end();
+ }
+
+ auto mh = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(mh);
+ auto bucket_idx = bucket_idx_from_hash(mh);
+ auto* bucket = &at(m_buckets, bucket_idx);
+
+ // unrolled loop. *Always* check a few directly, then enter the loop. This is faster.
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ bucket = &at(m_buckets, bucket_idx);
+
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ bucket = &at(m_buckets, bucket_idx);
+
+ while (true) {
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
+ if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
+ }
+ } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
+ return end();
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ bucket = &at(m_buckets, bucket_idx);
+ }
+ }
+
+ template <typename K>
+ auto do_find(K const& key) const -> const_iterator {
+ return const_cast<table*>(this)->do_find(key); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto do_at(K const& key) -> Q& {
+ if (auto it = find(key); ANKERL_UNORDERED_DENSE_LIKELY(end() != it)) {
+ return it->second;
+ }
+ on_error_key_not_found();
+ }
+
+ template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto do_at(K const& key) const -> Q const& {
+ return const_cast<table*>(this)->at(key); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+public:
+ explicit table(size_t bucket_count,
+ Hash const& hash = Hash(),
+ KeyEqual const& equal = KeyEqual(),
+ allocator_type const& alloc_or_container = allocator_type())
+ : m_values(alloc_or_container)
+ , m_hash(hash)
+ , m_equal(equal) {
+ if (0 != bucket_count) {
+ reserve(bucket_count);
+ } else {
+ allocate_buckets_from_shift();
+ clear_buckets();
+ }
+ }
+
+ table()
+ : table(0) {}
+
+ table(size_t bucket_count, allocator_type const& alloc)
+ : table(bucket_count, Hash(), KeyEqual(), alloc) {}
+
+ table(size_t bucket_count, Hash const& hash, allocator_type const& alloc)
+ : table(bucket_count, hash, KeyEqual(), alloc) {}
+
+ explicit table(allocator_type const& alloc)
+ : table(0, Hash(), KeyEqual(), alloc) {}
+
+ template <class InputIt>
+ table(InputIt first,
+ InputIt last,
+ size_type bucket_count = 0,
+ Hash const& hash = Hash(),
+ KeyEqual const& equal = KeyEqual(),
+ allocator_type const& alloc = allocator_type())
+ : table(bucket_count, hash, equal, alloc) {
+ insert(first, last);
+ }
+
+ template <class InputIt>
+ table(InputIt first, InputIt last, size_type bucket_count, allocator_type const& alloc)
+ : table(first, last, bucket_count, Hash(), KeyEqual(), alloc) {}
+
+ template <class InputIt>
+ table(InputIt first, InputIt last, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
+ : table(first, last, bucket_count, hash, KeyEqual(), alloc) {}
+
+ table(table const& other)
+ : table(other, other.m_values.get_allocator()) {}
+
+ table(table const& other, allocator_type const& alloc)
+ : m_values(other.m_values, alloc)
+ , m_max_load_factor(other.m_max_load_factor)
+ , m_hash(other.m_hash)
+ , m_equal(other.m_equal) {
+ copy_buckets(other);
+ }
+
+ table(table&& other) noexcept
+ : table(std::move(other), other.m_values.get_allocator()) {}
+
+ table(table&& other, allocator_type const& alloc) noexcept
+ : m_values(alloc) {
+ *this = std::move(other);
+ }
+
+ table(std::initializer_list<value_type> ilist,
+ size_t bucket_count = 0,
+ Hash const& hash = Hash(),
+ KeyEqual const& equal = KeyEqual(),
+ allocator_type const& alloc = allocator_type())
+ : table(bucket_count, hash, equal, alloc) {
+ insert(ilist);
+ }
+
+ table(std::initializer_list<value_type> ilist, size_type bucket_count, allocator_type const& alloc)
+ : table(ilist, bucket_count, Hash(), KeyEqual(), alloc) {}
+
+ table(std::initializer_list<value_type> init, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
+ : table(init, bucket_count, hash, KeyEqual(), alloc) {}
+
+ ~table() {
+ if (nullptr != m_buckets) {
+ auto ba = bucket_alloc(m_values.get_allocator());
+ bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
+ }
+ }
+
+ auto operator=(table const& other) -> table& {
+ if (&other != this) {
+ deallocate_buckets(); // deallocate before m_values is set (might have another allocator)
+ m_values = other.m_values;
+ m_max_load_factor = other.m_max_load_factor;
+ m_hash = other.m_hash;
+ m_equal = other.m_equal;
+ m_shifts = initial_shifts;
+ copy_buckets(other);
+ }
+ return *this;
+ }
+
+ auto operator=(table&& other) noexcept(noexcept(std::is_nothrow_move_assignable_v<value_container_type> &&
+ std::is_nothrow_move_assignable_v<Hash> &&
+ std::is_nothrow_move_assignable_v<KeyEqual>)) -> table& {
+ if (&other != this) {
+ deallocate_buckets(); // deallocate before m_values is set (might have another allocator)
+ m_values = std::move(other.m_values);
+ other.m_values.clear();
+
+ // we can only reuse m_buckets when both maps have the same allocator!
+ if (get_allocator() == other.get_allocator()) {
+ m_buckets = std::exchange(other.m_buckets, nullptr);
+ m_num_buckets = std::exchange(other.m_num_buckets, 0);
+ m_max_bucket_capacity = std::exchange(other.m_max_bucket_capacity, 0);
+ m_shifts = std::exchange(other.m_shifts, initial_shifts);
+ m_max_load_factor = std::exchange(other.m_max_load_factor, default_max_load_factor);
+ m_hash = std::exchange(other.m_hash, {});
+ m_equal = std::exchange(other.m_equal, {});
+ other.allocate_buckets_from_shift();
+ other.clear_buckets();
+ } else {
+ // set max_load_factor *before* copying the other's buckets, so we have the same
+ // behavior
+ m_max_load_factor = other.m_max_load_factor;
+
+ // copy_buckets sets m_buckets, m_num_buckets, m_max_bucket_capacity, m_shifts
+ copy_buckets(other);
+ // clear's the other's buckets so other is now already usable.
+ other.clear_buckets();
+ m_hash = other.m_hash;
+ m_equal = other.m_equal;
+ }
+ // map "other" is now already usable, it's empty.
+ }
+ return *this;
+ }
+
+ auto operator=(std::initializer_list<value_type> ilist) -> table& {
+ clear();
+ insert(ilist);
+ return *this;
+ }
+
+ auto get_allocator() const noexcept -> allocator_type {
+ return m_values.get_allocator();
+ }
+
+ // iterators //////////////////////////////////////////////////////////////
+
+ auto begin() noexcept -> iterator {
+ return m_values.begin();
+ }
+
+ auto begin() const noexcept -> const_iterator {
+ return m_values.begin();
+ }
+
+ auto cbegin() const noexcept -> const_iterator {
+ return m_values.cbegin();
+ }
+
+ auto end() noexcept -> iterator {
+ return m_values.end();
+ }
+
+ auto cend() const noexcept -> const_iterator {
+ return m_values.cend();
+ }
+
+ auto end() const noexcept -> const_iterator {
+ return m_values.end();
+ }
+
+ // capacity ///////////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto empty() const noexcept -> bool {
+ return m_values.empty();
+ }
+
+ [[nodiscard]] auto size() const noexcept -> size_t {
+ return m_values.size();
+ }
+
+ [[nodiscard]] static constexpr auto max_size() noexcept -> size_t {
+ if constexpr ((std::numeric_limits<value_idx_type>::max)() == (std::numeric_limits<size_t>::max)()) {
+ return size_t{1} << (sizeof(value_idx_type) * 8 - 1);
+ } else {
+ return size_t{1} << (sizeof(value_idx_type) * 8);
+ }
+ }
+
+ // modifiers //////////////////////////////////////////////////////////////
+
+ void clear() {
+ m_values.clear();
+ clear_buckets();
+ }
+
+ auto insert(value_type const& value) -> std::pair<iterator, bool> {
+ return emplace(value);
+ }
+
+ auto insert(value_type&& value) -> std::pair<iterator, bool> {
+ return emplace(std::move(value));
+ }
+
+ template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true>
+ auto insert(P&& value) -> std::pair<iterator, bool> {
+ return emplace(std::forward<P>(value));
+ }
+
+ auto insert(const_iterator /*hint*/, value_type const& value) -> iterator {
+ return insert(value).first;
+ }
+
+ auto insert(const_iterator /*hint*/, value_type&& value) -> iterator {
+ return insert(std::move(value)).first;
+ }
+
+ template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true>
+ auto insert(const_iterator /*hint*/, P&& value) -> iterator {
+ return insert(std::forward<P>(value)).first;
+ }
+
+ template <class InputIt>
+ void insert(InputIt first, InputIt last) {
+ while (first != last) {
+ insert(*first);
+ ++first;
+ }
+ }
+
+ void insert(std::initializer_list<value_type> ilist) {
+ insert(ilist.begin(), ilist.end());
+ }
+
+ // nonstandard API: *this is emptied.
+ // Also see "A Standard flat_map" https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p0429r9.pdf
+ auto extract() && -> value_container_type {
+ return std::move(m_values);
+ }
+
+ // nonstandard API:
+ // Discards the internally held container and replaces it with the one passed. Erases non-unique elements.
+ auto replace(value_container_type&& container) {
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(container.size() > max_size())) {
+ on_error_too_many_elements();
+ }
+ auto shifts = calc_shifts_for_size(container.size());
+ if (0 == m_num_buckets || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) {
+ m_shifts = shifts;
+ deallocate_buckets();
+ allocate_buckets_from_shift();
+ }
+ clear_buckets();
+
+ m_values = std::move(container);
+
+ // can't use clear_and_fill_buckets_from_values() because container elements might not be unique
+ auto value_idx = value_idx_type{};
+
+ // loop until we reach the end of the container. duplicated entries will be replaced with back().
+ while (value_idx != static_cast<value_idx_type>(m_values.size())) {
+ auto const& key = get_key(m_values[value_idx]);
+
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ bool key_found = false;
+ while (true) {
+ auto const& bucket = at(m_buckets, bucket_idx);
+ if (dist_and_fingerprint > bucket.m_dist_and_fingerprint) {
+ break;
+ }
+ if (dist_and_fingerprint == bucket.m_dist_and_fingerprint &&
+ m_equal(key, get_key(m_values[bucket.m_value_idx]))) {
+ key_found = true;
+ break;
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ if (key_found) {
+ if (value_idx != static_cast<value_idx_type>(m_values.size() - 1)) {
+ m_values[value_idx] = std::move(m_values.back());
+ }
+ m_values.pop_back();
+ } else {
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
+ ++value_idx;
+ }
+ }
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(Key const& key, M&& mapped) -> std::pair<iterator, bool> {
+ return do_insert_or_assign(key, std::forward<M>(mapped));
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(Key&& key, M&& mapped) -> std::pair<iterator, bool> {
+ return do_insert_or_assign(std::move(key), std::forward<M>(mapped));
+ }
+
+ template <typename K,
+ typename M,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> {
+ return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped));
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(const_iterator /*hint*/, Key const& key, M&& mapped) -> iterator {
+ return do_insert_or_assign(key, std::forward<M>(mapped)).first;
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(const_iterator /*hint*/, Key&& key, M&& mapped) -> iterator {
+ return do_insert_or_assign(std::move(key), std::forward<M>(mapped)).first;
+ }
+
+ template <typename K,
+ typename M,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto insert_or_assign(const_iterator /*hint*/, K&& key, M&& mapped) -> iterator {
+ return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped)).first;
+ }
+
+ // Single arguments for unordered_set can be used without having to construct the value_type
+ template <class K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<!is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto emplace(K&& key) -> std::pair<iterator, bool> {
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
+ m_equal(key, m_values[at(m_buckets, bucket_idx).m_value_idx])) {
+ // found it, return without ever actually creating anything
+ return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ // value is new, insert element first, so when exception happens we are in a valid state
+ return do_place_element(dist_and_fingerprint, bucket_idx, std::forward<K>(key));
+ }
+
+ template <class... Args>
+ auto emplace(Args&&... args) -> std::pair<iterator, bool> {
+ // we have to instantiate the value_type to be able to access the key.
+ // 1. emplace_back the object so it is constructed. 2. If the key is already there, pop it later in the loop.
+ auto& key = get_key(m_values.emplace_back(std::forward<Args>(args)...));
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
+ m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
+ m_values.pop_back(); // value was already there, so get rid of it
+ return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ // value is new, place the bucket and shift up until we find an empty spot
+ auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) {
+ // increase_size just rehashes all the data we have in m_values
+ increase_size();
+ } else {
+ // place element and shift up until we find an empty spot
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
+ }
+ return {begin() + static_cast<difference_type>(value_idx), true};
+ }
+
+ template <class... Args>
+ auto emplace_hint(const_iterator /*hint*/, Args&&... args) -> iterator {
+ return emplace(std::forward<Args>(args)...).first;
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(Key const& key, Args&&... args) -> std::pair<iterator, bool> {
+ return do_try_emplace(key, std::forward<Args>(args)...);
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(Key&& key, Args&&... args) -> std::pair<iterator, bool> {
+ return do_try_emplace(std::move(key), std::forward<Args>(args)...);
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(const_iterator /*hint*/, Key const& key, Args&&... args) -> iterator {
+ return do_try_emplace(key, std::forward<Args>(args)...).first;
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(const_iterator /*hint*/, Key&& key, Args&&... args) -> iterator {
+ return do_try_emplace(std::move(key), std::forward<Args>(args)...).first;
+ }
+
+ template <
+ typename K,
+ typename... Args,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>,
+ bool> = true>
+ auto try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> {
+ return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...);
+ }
+
+ template <
+ typename K,
+ typename... Args,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>,
+ bool> = true>
+ auto try_emplace(const_iterator /*hint*/, K&& key, Args&&... args) -> iterator {
+ return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...).first;
+ }
+
+ auto erase(iterator it) -> iterator {
+ auto hash = mixed_hash(get_key(*it));
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin());
+ while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) {
+ bucket_idx = next(bucket_idx);
+ }
+
+ do_erase(bucket_idx, [](value_type&& /*unused*/) {
+ });
+ return begin() + static_cast<difference_type>(value_idx_to_remove);
+ }
+
+ auto extract(iterator it) -> value_type {
+ auto hash = mixed_hash(get_key(*it));
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin());
+ while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) {
+ bucket_idx = next(bucket_idx);
+ }
+
+ auto tmp = std::optional<value_type>{};
+ do_erase(bucket_idx, [&tmp](value_type&& val) {
+ tmp = std::move(val);
+ });
+ return std::move(tmp).value();
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto erase(const_iterator it) -> iterator {
+ return erase(begin() + (it - cbegin()));
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto extract(const_iterator it) -> value_type {
+ return extract(begin() + (it - cbegin()));
+ }
+
+ auto erase(const_iterator first, const_iterator last) -> iterator {
+ auto const idx_first = first - cbegin();
+ auto const idx_last = last - cbegin();
+ auto const first_to_last = std::distance(first, last);
+ auto const last_to_end = std::distance(last, cend());
+
+ // remove elements from left to right which moves elements from the end back
+ auto const mid = idx_first + (std::min)(first_to_last, last_to_end);
+ auto idx = idx_first;
+ while (idx != mid) {
+ erase(begin() + idx);
+ ++idx;
+ }
+
+ // all elements from the right are moved, now remove the last element until all done
+ idx = idx_last;
+ while (idx != mid) {
+ --idx;
+ erase(begin() + idx);
+ }
+
+ return begin() + idx_first;
+ }
+
+ auto erase(Key const& key) -> size_t {
+ return do_erase_key(key, [](value_type&& /*unused*/) {
+ });
+ }
+
+ auto extract(Key const& key) -> std::optional<value_type> {
+ auto tmp = std::optional<value_type>{};
+ do_erase_key(key, [&tmp](value_type&& val) {
+ tmp = std::move(val);
+ });
+ return tmp;
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto erase(K&& key) -> size_t {
+ return do_erase_key(std::forward<K>(key), [](value_type&& /*unused*/) {
+ });
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto extract(K&& key) -> std::optional<value_type> {
+ auto tmp = std::optional<value_type>{};
+ do_erase_key(std::forward<K>(key), [&tmp](value_type&& val) {
+ tmp = std::move(val);
+ });
+ return tmp;
+ }
+
+ void swap(table& other) noexcept(noexcept(std::is_nothrow_swappable_v<value_container_type> &&
+ std::is_nothrow_swappable_v<Hash> && std::is_nothrow_swappable_v<KeyEqual>)) {
+ using std::swap;
+ swap(other, *this);
+ }
+
+ // lookup /////////////////////////////////////////////////////////////////
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto at(key_type const& key) -> Q& {
+ return do_at(key);
+ }
+
+ template <typename K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto at(K const& key) -> Q& {
+ return do_at(key);
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto at(key_type const& key) const -> Q const& {
+ return do_at(key);
+ }
+
+ template <typename K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto at(K const& key) const -> Q const& {
+ return do_at(key);
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto operator[](Key const& key) -> Q& {
+ return try_emplace(key).first->second;
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto operator[](Key&& key) -> Q& {
+ return try_emplace(std::move(key)).first->second;
+ }
+
+ template <typename K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto operator[](K&& key) -> Q& {
+ return try_emplace(std::forward<K>(key)).first->second;
+ }
+
+ auto count(Key const& key) const -> size_t {
+ return find(key) == end() ? 0 : 1;
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto count(K const& key) const -> size_t {
+ return find(key) == end() ? 0 : 1;
+ }
+
+ auto find(Key const& key) -> iterator {
+ return do_find(key);
+ }
+
+ auto find(Key const& key) const -> const_iterator {
+ return do_find(key);
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto find(K const& key) -> iterator {
+ return do_find(key);
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto find(K const& key) const -> const_iterator {
+ return do_find(key);
+ }
+
+ auto contains(Key const& key) const -> bool {
+ return find(key) != end();
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto contains(K const& key) const -> bool {
+ return find(key) != end();
+ }
+
+ auto equal_range(Key const& key) -> std::pair<iterator, iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ auto equal_range(const Key& key) const -> std::pair<const_iterator, const_iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto equal_range(K const& key) -> std::pair<iterator, iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto equal_range(K const& key) const -> std::pair<const_iterator, const_iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ // bucket interface ///////////////////////////////////////////////////////
+
+ auto bucket_count() const noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
+ return m_num_buckets;
+ }
+
+ static constexpr auto max_bucket_count() noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
+ return max_size();
+ }
+
+ // hash policy ////////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto load_factor() const -> float {
+ return bucket_count() ? static_cast<float>(size()) / static_cast<float>(bucket_count()) : 0.0F;
+ }
+
+ [[nodiscard]] auto max_load_factor() const -> float {
+ return m_max_load_factor;
+ }
+
+ void max_load_factor(float ml) {
+ m_max_load_factor = ml;
+ if (m_num_buckets != max_bucket_count()) {
+ m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(bucket_count()) * max_load_factor());
+ }
+ }
+
+ void rehash(size_t count) {
+ count = (std::min)(count, max_size());
+ auto shifts = calc_shifts_for_size((std::max)(count, size()));
+ if (shifts != m_shifts) {
+ m_shifts = shifts;
+ deallocate_buckets();
+ m_values.shrink_to_fit();
+ allocate_buckets_from_shift();
+ clear_and_fill_buckets_from_values();
+ }
+ }
+
+ void reserve(size_t capa) {
+ capa = (std::min)(capa, max_size());
+ if constexpr (has_reserve<value_container_type>) {
+ // std::deque doesn't have reserve(). Make sure we only call when available
+ m_values.reserve(capa);
+ }
+ auto shifts = calc_shifts_for_size((std::max)(capa, size()));
+ if (0 == m_num_buckets || shifts < m_shifts) {
+ m_shifts = shifts;
+ deallocate_buckets();
+ allocate_buckets_from_shift();
+ clear_and_fill_buckets_from_values();
+ }
+ }
+
+ // observers //////////////////////////////////////////////////////////////
+
+ auto hash_function() const -> hasher {
+ return m_hash;
+ }
+
+ auto key_eq() const -> key_equal {
+ return m_equal;
+ }
+
+ // nonstandard API: expose the underlying values container
+ [[nodiscard]] auto values() const noexcept -> value_container_type const& {
+ return m_values;
+ }
+
+ // non-member functions ///////////////////////////////////////////////////
+
+ friend auto operator==(table const& a, table const& b) -> bool {
+ if (&a == &b) {
+ return true;
+ }
+ if (a.size() != b.size()) {
+ return false;
+ }
+ for (auto const& b_entry : b) {
+ auto it = a.find(get_key(b_entry));
+ if constexpr (is_map_v<T>) {
+ // map: check that key is here, then also check that value is the same
+ if (a.end() == it || !(b_entry.second == it->second)) {
+ return false;
+ }
+ } else {
+ // set: only check that the key is here
+ if (a.end() == it) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ friend auto operator!=(table const& a, table const& b) -> bool {
+ return !(a == b);
+ }
+};
+
+} // namespace detail
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
+ class Bucket = bucket_type::standard>
+using map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
+ class Bucket = bucket_type::standard>
+using segmented_map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<Key>,
+ class Bucket = bucket_type::standard>
+using set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<Key>,
+ class Bucket = bucket_type::standard>
+using segmented_set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>;
+
+# if defined(ANKERL_UNORDERED_DENSE_PMR)
+
+namespace pmr {
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using map =
+ detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using segmented_map =
+ detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, true>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using set = detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using segmented_set =
+ detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, true>;
+
+} // namespace pmr
+
+# endif
+
+// deduction guides ///////////////////////////////////////////////////////////
+
+// deduction guides for alias templates are only possible since C++20
+// see https://en.cppreference.com/w/cpp/language/class_template_argument_deduction
+
+} // namespace ANKERL_UNORDERED_DENSE_NAMESPACE
+} // namespace ankerl::unordered_dense
+
+// std extensions /////////////////////////////////////////////////////////////
+
+namespace std { // NOLINT(cert-dcl58-cpp)
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash,
+ class KeyEqual,
+ class AllocatorOrContainer,
+ class Bucket,
+ class Pred,
+ bool IsSegmented>
+// NOLINTNEXTLINE(cert-dcl58-cpp)
+auto erase_if(ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>& map,
+ Pred pred) -> size_t {
+ using map_t = ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>;
+
+ // going back to front because erase() invalidates the end iterator
+ auto const old_size = map.size();
+ auto idx = old_size;
+ while (idx) {
+ --idx;
+ auto it = map.begin() + static_cast<typename map_t::difference_type>(idx);
+ if (pred(*it)) {
+ map.erase(it);
+ }
+ }
+
+ return old_size - map.size();
+}
+
+} // namespace std
+
+#endif
+#endif
diff --git a/contrib/backward-cpp/BackwardConfig.cmake b/contrib/backward-cpp/BackwardConfig.cmake
new file mode 100644
index 0000000..4cec0be
--- /dev/null
+++ b/contrib/backward-cpp/BackwardConfig.cmake
@@ -0,0 +1,249 @@
+#
+# BackwardMacros.cmake
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+###############################################################################
+# OPTIONS
+###############################################################################
+
+set(STACK_WALKING_UNWIND TRUE CACHE BOOL
+ "Use compiler's unwind API")
+set(STACK_WALKING_BACKTRACE FALSE CACHE BOOL
+ "Use backtrace from (e)glibc for stack walking")
+set(STACK_WALKING_LIBUNWIND FALSE CACHE BOOL
+ "Use libunwind for stack walking")
+
+set(STACK_DETAILS_AUTO_DETECT TRUE CACHE BOOL
+ "Auto detect backward's stack details dependencies")
+
+set(STACK_DETAILS_BACKTRACE_SYMBOL FALSE CACHE BOOL
+ "Use backtrace from (e)glibc for symbols resolution")
+set(STACK_DETAILS_DW FALSE CACHE BOOL
+ "Use libdw to read debug info")
+set(STACK_DETAILS_BFD FALSE CACHE BOOL
+ "Use libbfd to read debug info")
+set(STACK_DETAILS_DWARF FALSE CACHE BOOL
+ "Use libdwarf/libelf to read debug info")
+
+if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND NOT DEFINED BACKWARD_TESTS)
+ # If this is a top level CMake project, we most lixely want the tests
+ set(BACKWARD_TESTS ON CACHE BOOL "Enable tests")
+else()
+ set(BACKWARD_TESTS OFF CACHE BOOL "Enable tests")
+endif()
+###############################################################################
+# CONFIGS
+###############################################################################
+include(FindPackageHandleStandardArgs)
+
+if (STACK_WALKING_LIBUNWIND)
+ # libunwind works on the macOS without having to add special include
+ # paths or libraries
+ if (NOT APPLE)
+ find_path(LIBUNWIND_INCLUDE_DIR NAMES "libunwind.h")
+ find_library(LIBUNWIND_LIBRARY unwind)
+
+ if (LIBUNWIND_LIBRARY)
+ include(CheckSymbolExists)
+ check_symbol_exists(UNW_INIT_SIGNAL_FRAME libunwind.h HAVE_UNW_INIT_SIGNAL_FRAME)
+ if (NOT HAVE_UNW_INIT_SIGNAL_FRAME)
+ message(STATUS "libunwind does not support unwinding from signal handler frames")
+ endif()
+ endif()
+
+ set(LIBUNWIND_INCLUDE_DIRS ${LIBUNWIND_INCLUDE_DIR})
+ set(LIBDWARF_LIBRARIES ${LIBUNWIND_LIBRARY})
+ find_package_handle_standard_args(libunwind DEFAULT_MSG
+ LIBUNWIND_LIBRARY LIBUNWIND_INCLUDE_DIR)
+ mark_as_advanced(LIBUNWIND_INCLUDE_DIR LIBUNWIND_LIBRARY)
+ list(APPEND _BACKWARD_LIBRARIES ${LIBUNWIND_LIBRARY})
+ endif()
+
+ # Disable other unwinders if libunwind is found
+ set(STACK_WALKING_UNWIND FALSE)
+ set(STACK_WALKING_BACKTRACE FALSE)
+endif()
+
+if (${STACK_DETAILS_AUTO_DETECT})
+ if(NOT CMAKE_VERSION VERSION_LESS 3.17)
+ set(_name_mismatched_arg NAME_MISMATCHED)
+ endif()
+ # find libdw
+ find_path(LIBDW_INCLUDE_DIR NAMES "elfutils/libdw.h" "elfutils/libdwfl.h")
+ find_library(LIBDW_LIBRARY dw)
+ set(LIBDW_INCLUDE_DIRS ${LIBDW_INCLUDE_DIR} )
+ set(LIBDW_LIBRARIES ${LIBDW_LIBRARY} )
+ find_package_handle_standard_args(libdw ${_name_mismatched_arg}
+ REQUIRED_VARS LIBDW_LIBRARY LIBDW_INCLUDE_DIR)
+ mark_as_advanced(LIBDW_INCLUDE_DIR LIBDW_LIBRARY)
+
+ # find libbfd
+ find_path(LIBBFD_INCLUDE_DIR NAMES "bfd.h")
+ find_path(LIBDL_INCLUDE_DIR NAMES "dlfcn.h")
+ find_library(LIBBFD_LIBRARY bfd)
+ find_library(LIBDL_LIBRARY dl)
+ set(LIBBFD_INCLUDE_DIRS ${LIBBFD_INCLUDE_DIR} ${LIBDL_INCLUDE_DIR})
+ set(LIBBFD_LIBRARIES ${LIBBFD_LIBRARY} ${LIBDL_LIBRARY})
+ find_package_handle_standard_args(libbfd ${_name_mismatched_arg}
+ REQUIRED_VARS LIBBFD_LIBRARY LIBBFD_INCLUDE_DIR
+ LIBDL_LIBRARY LIBDL_INCLUDE_DIR)
+ mark_as_advanced(LIBBFD_INCLUDE_DIR LIBBFD_LIBRARY
+ LIBDL_INCLUDE_DIR LIBDL_LIBRARY)
+
+ # find libdwarf
+ find_path(LIBDWARF_INCLUDE_DIR NAMES "libdwarf.h" PATH_SUFFIXES libdwarf)
+ find_path(LIBELF_INCLUDE_DIR NAMES "libelf.h")
+ find_path(LIBDL_INCLUDE_DIR NAMES "dlfcn.h")
+ find_library(LIBDWARF_LIBRARY dwarf)
+ find_library(LIBELF_LIBRARY elf)
+ find_library(LIBDL_LIBRARY dl)
+ set(LIBDWARF_INCLUDE_DIRS ${LIBDWARF_INCLUDE_DIR} ${LIBELF_INCLUDE_DIR} ${LIBDL_INCLUDE_DIR})
+ set(LIBDWARF_LIBRARIES ${LIBDWARF_LIBRARY} ${LIBELF_LIBRARY} ${LIBDL_LIBRARY})
+ find_package_handle_standard_args(libdwarf ${_name_mismatched_arg}
+ REQUIRED_VARS LIBDWARF_LIBRARY LIBDWARF_INCLUDE_DIR
+ LIBELF_LIBRARY LIBELF_INCLUDE_DIR
+ LIBDL_LIBRARY LIBDL_INCLUDE_DIR)
+ mark_as_advanced(LIBDWARF_INCLUDE_DIR LIBDWARF_LIBRARY
+ LIBELF_INCLUDE_DIR LIBELF_LIBRARY
+ LIBDL_INCLUDE_DIR LIBDL_LIBRARY)
+
+ if (LIBDW_FOUND)
+ LIST(APPEND _BACKWARD_INCLUDE_DIRS ${LIBDW_INCLUDE_DIRS})
+ LIST(APPEND _BACKWARD_LIBRARIES ${LIBDW_LIBRARIES})
+ set(STACK_DETAILS_DW TRUE)
+ set(STACK_DETAILS_BFD FALSE)
+ set(STACK_DETAILS_DWARF FALSE)
+ set(STACK_DETAILS_BACKTRACE_SYMBOL FALSE)
+ elseif(LIBBFD_FOUND)
+ LIST(APPEND _BACKWARD_INCLUDE_DIRS ${LIBBFD_INCLUDE_DIRS})
+ LIST(APPEND _BACKWARD_LIBRARIES ${LIBBFD_LIBRARIES})
+
+ # If we attempt to link against static bfd, make sure to link its dependencies, too
+ get_filename_component(bfd_lib_ext "${LIBBFD_LIBRARY}" EXT)
+ if (bfd_lib_ext STREQUAL "${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ if (NOT APPLE)
+ list(APPEND _BACKWARD_LIBRARIES iberty z)
+ endif()
+ endif()
+
+ set(STACK_DETAILS_DW FALSE)
+ set(STACK_DETAILS_BFD TRUE)
+ set(STACK_DETAILS_DWARF FALSE)
+ set(STACK_DETAILS_BACKTRACE_SYMBOL FALSE)
+ elseif(LIBDWARF_FOUND)
+ LIST(APPEND _BACKWARD_INCLUDE_DIRS ${LIBDWARF_INCLUDE_DIRS})
+ LIST(APPEND _BACKWARD_LIBRARIES ${LIBDWARF_LIBRARIES})
+
+ set(STACK_DETAILS_DW FALSE)
+ set(STACK_DETAILS_BFD FALSE)
+ set(STACK_DETAILS_DWARF TRUE)
+ set(STACK_DETAILS_BACKTRACE_SYMBOL FALSE)
+ else()
+ set(STACK_DETAILS_DW FALSE)
+ set(STACK_DETAILS_BFD FALSE)
+ set(STACK_DETAILS_DWARF FALSE)
+ set(STACK_DETAILS_BACKTRACE_SYMBOL TRUE)
+ endif()
+else()
+ if (STACK_DETAILS_DW)
+ LIST(APPEND _BACKWARD_LIBRARIES dw)
+ endif()
+
+ if (STACK_DETAILS_BFD)
+ LIST(APPEND _BACKWARD_LIBRARIES bfd dl)
+ endif()
+
+ if (STACK_DETAILS_DWARF)
+ LIST(APPEND _BACKWARD_LIBRARIES dwarf elf)
+ endif()
+endif()
+
+macro(map_definitions var_prefix define_prefix)
+ foreach(def ${ARGN})
+ if (${${var_prefix}${def}})
+ LIST(APPEND _BACKWARD_DEFINITIONS "${define_prefix}${def}=1")
+ else()
+ LIST(APPEND _BACKWARD_DEFINITIONS "${define_prefix}${def}=0")
+ endif()
+ endforeach()
+endmacro()
+
+if (NOT _BACKWARD_DEFINITIONS)
+ map_definitions("STACK_WALKING_" "BACKWARD_HAS_" UNWIND LIBUNWIND BACKTRACE)
+ map_definitions("STACK_DETAILS_" "BACKWARD_HAS_" BACKTRACE_SYMBOL DW BFD DWARF)
+endif()
+
+if(WIN32)
+ list(APPEND _BACKWARD_LIBRARIES dbghelp psapi)
+ if(MINGW)
+ set(MINGW_MSVCR_LIBRARY "msvcr90$<$<CONFIG:DEBUG>:d>" CACHE STRING "Mingw MSVC runtime import library")
+ list(APPEND _BACKWARD_LIBRARIES ${MINGW_MSVCR_LIBRARY})
+ endif()
+endif()
+
+set(BACKWARD_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}")
+
+set(BACKWARD_HAS_EXTERNAL_LIBRARIES FALSE)
+set(FIND_PACKAGE_REQUIRED_VARS BACKWARD_INCLUDE_DIR)
+if(DEFINED _BACKWARD_LIBRARIES)
+ set(BACKWARD_HAS_EXTERNAL_LIBRARIES TRUE)
+ list(APPEND FIND_PACKAGE_REQUIRED_VARS _BACKWARD_LIBRARIES)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Backward
+ REQUIRED_VARS ${FIND_PACKAGE_REQUIRED_VARS}
+)
+list(APPEND _BACKWARD_INCLUDE_DIRS ${BACKWARD_INCLUDE_DIR})
+
+macro(add_backward target)
+ target_include_directories(${target} PRIVATE ${BACKWARD_INCLUDE_DIRS})
+ set_property(TARGET ${target} APPEND PROPERTY COMPILE_DEFINITIONS ${BACKWARD_DEFINITIONS})
+ set_property(TARGET ${target} APPEND PROPERTY LINK_LIBRARIES ${BACKWARD_LIBRARIES})
+endmacro()
+
+set(BACKWARD_INCLUDE_DIRS ${_BACKWARD_INCLUDE_DIRS} CACHE INTERNAL "_BACKWARD_INCLUDE_DIRS")
+set(BACKWARD_DEFINITIONS ${_BACKWARD_DEFINITIONS} CACHE INTERNAL "BACKWARD_DEFINITIONS")
+set(BACKWARD_LIBRARIES ${_BACKWARD_LIBRARIES} CACHE INTERNAL "BACKWARD_LIBRARIES")
+mark_as_advanced(BACKWARD_INCLUDE_DIRS BACKWARD_DEFINITIONS BACKWARD_LIBRARIES)
+
+# Expand each definition in BACKWARD_DEFINITIONS to its own cmake var and export
+# to outer scope
+foreach(var ${BACKWARD_DEFINITIONS})
+ string(REPLACE "=" ";" var_as_list ${var})
+ list(GET var_as_list 0 var_name)
+ list(GET var_as_list 1 var_value)
+ set(${var_name} ${var_value})
+ mark_as_advanced(${var_name})
+endforeach()
+
+if (NOT TARGET Backward::Backward)
+ add_library(Backward::Backward INTERFACE IMPORTED)
+ set_target_properties(Backward::Backward PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${BACKWARD_INCLUDE_DIRS}"
+ INTERFACE_COMPILE_DEFINITIONS "${BACKWARD_DEFINITIONS}"
+ )
+ if(BACKWARD_HAS_EXTERNAL_LIBRARIES)
+ set_target_properties(Backward::Backward PROPERTIES
+ INTERFACE_LINK_LIBRARIES "${BACKWARD_LIBRARIES}"
+ )
+ endif()
+endif()
diff --git a/contrib/backward-cpp/CMakeLists.txt b/contrib/backward-cpp/CMakeLists.txt
new file mode 100644
index 0000000..038c505
--- /dev/null
+++ b/contrib/backward-cpp/CMakeLists.txt
@@ -0,0 +1,88 @@
+#
+# CMakeLists.txt
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+cmake_minimum_required(VERSION 3.0)
+project(backward CXX)
+
+# Introduce variables:
+# * CMAKE_INSTALL_LIBDIR
+# * CMAKE_INSTALL_BINDIR
+# * CMAKE_INSTALL_INCLUDEDIR
+include(GNUInstallDirs)
+
+include(BackwardConfig.cmake)
+
+# check if compiler is nvcc or nvcc_wrapper
+set(COMPILER_IS_NVCC false)
+get_filename_component(COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
+if (COMPILER_NAME MATCHES "^nvcc")
+ set(COMPILER_IS_NVCC true)
+endif()
+
+if (DEFINED ENV{OMPI_CXX} OR DEFINED ENV{MPICH_CXX})
+ if ( ($ENV{OMPI_CXX} MATCHES "nvcc") OR ($ENV{MPICH_CXX} MATCHES "nvcc") )
+ set(COMPILER_IS_NVCC true)
+ endif()
+endif()
+
+# set CXX standard
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+set(CMAKE_CXX_STANDARD 11)
+if (${COMPILER_IS_NVCC})
+ # GNU CXX extensions are not supported by nvcc
+ set(CMAKE_CXX_EXTENSIONS OFF)
+endif()
+
+###############################################################################
+# COMPILER FLAGS
+###############################################################################
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_COMPILER_IS_GNUCXX)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra")
+ if (NOT ${COMPILER_IS_NVCC})
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic-errors")
+ endif()
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
+endif()
+
+###############################################################################
+# BACKWARD OBJECT
+###############################################################################
+
+add_library(backward_object OBJECT backward.cpp)
+target_compile_definitions(backward_object PRIVATE ${BACKWARD_DEFINITIONS})
+target_include_directories(backward_object PRIVATE ${BACKWARD_INCLUDE_DIRS})
+set(BACKWARD_ENABLE $<TARGET_OBJECTS:backward_object> CACHE STRING
+ "Link with this object to setup backward automatically")
+
+
+###############################################################################
+# BACKWARD LIBRARY (Includes backward.cpp)
+###############################################################################
+option(BACKWARD_SHARED "Build dynamic backward-cpp shared lib" OFF)
+
+if(BACKWARD_SHARED)
+ set(libtype SHARED)
+endif()
+add_library(backward ${libtype} backward.cpp)
+target_compile_definitions(backward PUBLIC ${BACKWARD_DEFINITIONS})
+target_include_directories(backward PUBLIC ${BACKWARD_INCLUDE_DIRS}) \ No newline at end of file
diff --git a/contrib/backward-cpp/backward.cpp b/contrib/backward-cpp/backward.cpp
new file mode 100644
index 0000000..110441c
--- /dev/null
+++ b/contrib/backward-cpp/backward.cpp
@@ -0,0 +1,42 @@
+// Pick your poison.
+//
+// On GNU/Linux, you have few choices to get the most out of your stack trace.
+//
+// By default you get:
+// - object filename
+// - function name
+//
+// In order to add:
+// - source filename
+// - line and column numbers
+// - source code snippet (assuming the file is accessible)
+
+// Install one of the following libraries then uncomment one of the macro (or
+// better, add the detection of the lib and the macro definition in your build
+// system)
+
+// - apt-get install libdw-dev ...
+// - g++/clang++ -ldw ...
+// #define BACKWARD_HAS_DW 1
+
+// - apt-get install binutils-dev ...
+// - g++/clang++ -lbfd ...
+// #define BACKWARD_HAS_BFD 1
+
+// - apt-get install libdwarf-dev ...
+// - g++/clang++ -ldwarf ...
+// #define BACKWARD_HAS_DWARF 1
+
+// Regardless of the library you choose to read the debug information,
+// for potentially more detailed stack traces you can use libunwind
+// - apt-get install libunwind-dev
+// - g++/clang++ -lunwind
+// #define BACKWARD_HAS_LIBUNWIND 1
+
+#include "backward.hpp"
+
+namespace backward {
+
+backward::SignalHandling sh;
+
+} // namespace backward
diff --git a/contrib/backward-cpp/backward.hpp b/contrib/backward-cpp/backward.hpp
new file mode 100644
index 0000000..ca09b72
--- /dev/null
+++ b/contrib/backward-cpp/backward.hpp
@@ -0,0 +1,4471 @@
+/*
+ * backward.hpp
+ * Copyright 2013 Google Inc. All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef H_6B9572DA_A64B_49E6_B234_051480991C89
+#define H_6B9572DA_A64B_49E6_B234_051480991C89
+
+#ifndef __cplusplus
+#error "It's not going to compile without a C++ compiler..."
+#endif
+
+#if defined(BACKWARD_CXX11)
+#elif defined(BACKWARD_CXX98)
+#else
+#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)
+#define BACKWARD_CXX11
+#define BACKWARD_ATLEAST_CXX11
+#define BACKWARD_ATLEAST_CXX98
+#if __cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L)
+#define BACKWARD_ATLEAST_CXX17
+#endif
+#else
+#define BACKWARD_CXX98
+#define BACKWARD_ATLEAST_CXX98
+#endif
+#endif
+
+// You can define one of the following (or leave it to the auto-detection):
+//
+// #define BACKWARD_SYSTEM_LINUX
+// - specialization for linux
+//
+// #define BACKWARD_SYSTEM_DARWIN
+// - specialization for Mac OS X 10.5 and later.
+//
+// #define BACKWARD_SYSTEM_WINDOWS
+// - specialization for Windows (Clang 9 and MSVC2017)
+//
+// #define BACKWARD_SYSTEM_UNKNOWN
+// - placebo implementation, does nothing.
+//
+#if defined(BACKWARD_SYSTEM_LINUX)
+#elif defined(BACKWARD_SYSTEM_DARWIN)
+#elif defined(BACKWARD_SYSTEM_UNKNOWN)
+#elif defined(BACKWARD_SYSTEM_WINDOWS)
+#else
+#if defined(__linux) || defined(__linux__)
+#define BACKWARD_SYSTEM_LINUX
+#elif defined(__APPLE__)
+#define BACKWARD_SYSTEM_DARWIN
+#elif defined(_WIN32)
+#define BACKWARD_SYSTEM_WINDOWS
+#else
+#define BACKWARD_SYSTEM_UNKNOWN
+#endif
+#endif
+
+#define NOINLINE __attribute__((noinline))
+
+#include <algorithm>
+#include <cctype>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <limits>
+#include <new>
+#include <sstream>
+#include <streambuf>
+#include <string>
+#include <vector>
+#include <exception>
+#include <iterator>
+
+#if defined(BACKWARD_SYSTEM_LINUX)
+
+// On linux, backtrace can back-trace or "walk" the stack using the following
+// libraries:
+//
+// #define BACKWARD_HAS_UNWIND 1
+// - unwind comes from libgcc, but I saw an equivalent inside clang itself.
+// - with unwind, the stacktrace is as accurate as it can possibly be, since
+// this is used by the C++ runtine in gcc/clang for stack unwinding on
+// exception.
+// - normally libgcc is already linked to your program by default.
+//
+// #define BACKWARD_HAS_LIBUNWIND 1
+// - libunwind provides, in some cases, a more accurate stacktrace as it knows
+// to decode signal handler frames and lets us edit the context registers when
+// unwinding, allowing stack traces over bad function references.
+//
+// #define BACKWARD_HAS_BACKTRACE == 1
+// - backtrace seems to be a little bit more portable than libunwind, but on
+// linux, it uses unwind anyway, but abstract away a tiny information that is
+// sadly really important in order to get perfectly accurate stack traces.
+// - backtrace is part of the (e)glib library.
+//
+// The default is:
+// #define BACKWARD_HAS_UNWIND == 1
+//
+// Note that only one of the define should be set to 1 at a time.
+//
+#if BACKWARD_HAS_UNWIND == 1
+#elif BACKWARD_HAS_LIBUNWIND == 1
+#elif BACKWARD_HAS_BACKTRACE == 1
+#else
+#undef BACKWARD_HAS_UNWIND
+#define BACKWARD_HAS_UNWIND 1
+#undef BACKWARD_HAS_LIBUNWIND
+#define BACKWARD_HAS_LIBUNWIND 0
+#undef BACKWARD_HAS_BACKTRACE
+#define BACKWARD_HAS_BACKTRACE 0
+#endif
+
+// On linux, backward can extract detailed information about a stack trace
+// using one of the following libraries:
+//
+// #define BACKWARD_HAS_DW 1
+// - libdw gives you the most juicy details out of your stack traces:
+// - object filename
+// - function name
+// - source filename
+// - line and column numbers
+// - source code snippet (assuming the file is accessible)
+// - variable names (if not optimized out)
+// - variable values (not supported by backward-cpp)
+// - You need to link with the lib "dw":
+// - apt-get install libdw-dev
+// - g++/clang++ -ldw ...
+//
+// #define BACKWARD_HAS_BFD 1
+// - With libbfd, you get a fair amount of details:
+// - object filename
+// - function name
+// - source filename
+// - line numbers
+// - source code snippet (assuming the file is accessible)
+// - You need to link with the lib "bfd":
+// - apt-get install binutils-dev
+// - g++/clang++ -lbfd ...
+//
+// #define BACKWARD_HAS_DWARF 1
+// - libdwarf gives you the most juicy details out of your stack traces:
+// - object filename
+// - function name
+// - source filename
+// - line and column numbers
+// - source code snippet (assuming the file is accessible)
+// - variable names (if not optimized out)
+// - variable values (not supported by backward-cpp)
+// - You need to link with the lib "dwarf":
+// - apt-get install libdwarf-dev
+// - g++/clang++ -ldwarf ...
+//
+// #define BACKWARD_HAS_BACKTRACE_SYMBOL 1
+// - backtrace provides minimal details for a stack trace:
+// - object filename
+// - function name
+// - backtrace is part of the (e)glib library.
+//
+// The default is:
+// #define BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+//
+// Note that only one of the define should be set to 1 at a time.
+//
+#if BACKWARD_HAS_DW == 1
+#elif BACKWARD_HAS_BFD == 1
+#elif BACKWARD_HAS_DWARF == 1
+#elif BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+#else
+#undef BACKWARD_HAS_DW
+#define BACKWARD_HAS_DW 0
+#undef BACKWARD_HAS_BFD
+#define BACKWARD_HAS_BFD 0
+#undef BACKWARD_HAS_DWARF
+#define BACKWARD_HAS_DWARF 0
+#undef BACKWARD_HAS_BACKTRACE_SYMBOL
+#define BACKWARD_HAS_BACKTRACE_SYMBOL 1
+#endif
+
+#include <cxxabi.h>
+#include <fcntl.h>
+#ifdef __ANDROID__
+// Old Android API levels define _Unwind_Ptr in both link.h and
+// unwind.h Rename the one in link.h as we are not going to be using
+// it
+#define _Unwind_Ptr _Unwind_Ptr_Custom
+#include <link.h>
+#undef _Unwind_Ptr
+#else
+#include <link.h>
+#endif
+#if defined(__ppc__) || defined(__powerpc) || defined(__powerpc__) || \
+ defined(__POWERPC__)
+// Linux kernel header required for the struct pt_regs definition
+// to access the NIP (Next Instruction Pointer) register value
+#include <asm/ptrace.h>
+#endif
+#include <signal.h>
+#include <sys/stat.h>
+#include <syscall.h>
+#include <unistd.h>
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#undef _GNU_SOURCE
+#else
+#include <dlfcn.h>
+#endif
+
+#if BACKWARD_HAS_BFD == 1
+// NOTE: defining PACKAGE{,_VERSION} is required before including
+// bfd.h on some platforms, see also:
+// https://sourceware.org/bugzilla/show_bug.cgi?id=14243
+#ifndef PACKAGE
+#define PACKAGE
+#endif
+#ifndef PACKAGE_VERSION
+#define PACKAGE_VERSION
+#endif
+#include <bfd.h>
+#endif
+
+#if BACKWARD_HAS_DW == 1
+#include <dwarf.h>
+#include <elfutils/libdw.h>
+#include <elfutils/libdwfl.h>
+#endif
+
+#if BACKWARD_HAS_DWARF == 1
+#include <algorithm>
+#include <dwarf.h>
+#include <libdwarf.h>
+#include <libelf.h>
+#include <map>
+#endif
+
+#if (BACKWARD_HAS_BACKTRACE == 1) || (BACKWARD_HAS_BACKTRACE_SYMBOL == 1)
+// then we shall rely on backtrace
+#include <execinfo.h>
+#endif
+
+#endif // defined(BACKWARD_SYSTEM_LINUX)
+
+#if defined(BACKWARD_SYSTEM_DARWIN)
+// On Darwin, backtrace can back-trace or "walk" the stack using the following
+// libraries:
+//
+// #define BACKWARD_HAS_UNWIND 1
+// - unwind comes from libgcc, but I saw an equivalent inside clang itself.
+// - with unwind, the stacktrace is as accurate as it can possibly be, since
+// this is used by the C++ runtine in gcc/clang for stack unwinding on
+// exception.
+// - normally libgcc is already linked to your program by default.
+//
+// #define BACKWARD_HAS_LIBUNWIND 1
+// - libunwind comes from clang, which implements an API compatible version.
+// - libunwind provides, in some cases, a more accurate stacktrace as it knows
+// to decode signal handler frames and lets us edit the context registers when
+// unwinding, allowing stack traces over bad function references.
+//
+// #define BACKWARD_HAS_BACKTRACE == 1
+// - backtrace is available by default, though it does not produce as much
+// information as another library might.
+//
+// The default is:
+// #define BACKWARD_HAS_UNWIND == 1
+//
+// Note that only one of the define should be set to 1 at a time.
+//
+#if BACKWARD_HAS_UNWIND == 1
+#elif BACKWARD_HAS_BACKTRACE == 1
+#elif BACKWARD_HAS_LIBUNWIND == 1
+#else
+#undef BACKWARD_HAS_UNWIND
+#define BACKWARD_HAS_UNWIND 1
+#undef BACKWARD_HAS_BACKTRACE
+#define BACKWARD_HAS_BACKTRACE 0
+#undef BACKWARD_HAS_LIBUNWIND
+#define BACKWARD_HAS_LIBUNWIND 0
+#endif
+
+// On Darwin, backward can extract detailed information about a stack trace
+// using one of the following libraries:
+//
+// #define BACKWARD_HAS_BACKTRACE_SYMBOL 1
+// - backtrace provides minimal details for a stack trace:
+// - object filename
+// - function name
+//
+// The default is:
+// #define BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+//
+#if BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+#else
+#undef BACKWARD_HAS_BACKTRACE_SYMBOL
+#define BACKWARD_HAS_BACKTRACE_SYMBOL 1
+#endif
+
+#include <cxxabi.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <signal.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#if (BACKWARD_HAS_BACKTRACE == 1) || (BACKWARD_HAS_BACKTRACE_SYMBOL == 1)
+#include <execinfo.h>
+#endif
+#endif // defined(BACKWARD_SYSTEM_DARWIN)
+
+#if defined(BACKWARD_SYSTEM_WINDOWS)
+
+#include <condition_variable>
+#include <mutex>
+#include <thread>
+
+#include <basetsd.h>
+typedef SSIZE_T ssize_t;
+
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#include <winnt.h>
+
+#include <psapi.h>
+#include <signal.h>
+
+#ifndef __clang__
+#undef NOINLINE
+#define NOINLINE __declspec(noinline)
+#endif
+
+#ifdef _MSC_VER
+#pragma comment(lib, "psapi.lib")
+#pragma comment(lib, "dbghelp.lib")
+#endif
+
+// Comment / packing is from stackoverflow:
+// https://stackoverflow.com/questions/6205981/windows-c-stack-trace-from-a-running-app/28276227#28276227
+// Some versions of imagehlp.dll lack the proper packing directives themselves
+// so we need to do it.
+#pragma pack(push, before_imagehlp, 8)
+#include <imagehlp.h>
+#pragma pack(pop, before_imagehlp)
+
+// TODO maybe these should be undefined somewhere else?
+#undef BACKWARD_HAS_UNWIND
+#undef BACKWARD_HAS_BACKTRACE
+#if BACKWARD_HAS_PDB_SYMBOL == 1
+#else
+#undef BACKWARD_HAS_PDB_SYMBOL
+#define BACKWARD_HAS_PDB_SYMBOL 1
+#endif
+
+#endif
+
+#if BACKWARD_HAS_UNWIND == 1
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#include <unwind.h>
+#undef _GNU_SOURCE
+#else
+#include <unwind.h>
+#endif
+// while gcc's unwind.h defines something like that:
+// extern _Unwind_Ptr _Unwind_GetIP (struct _Unwind_Context *);
+// extern _Unwind_Ptr _Unwind_GetIPInfo (struct _Unwind_Context *, int *);
+//
+// clang's unwind.h defines something like this:
+// uintptr_t _Unwind_GetIP(struct _Unwind_Context* __context);
+//
+// Even if the _Unwind_GetIPInfo can be linked to, it is not declared, worse we
+// cannot just redeclare it because clang's unwind.h doesn't define _Unwind_Ptr
+// anyway.
+//
+// Luckily we can play on the fact that the guard macros have a different name:
+#ifdef __CLANG_UNWIND_H
+// In fact, this function still comes from libgcc (on my different linux boxes,
+// clang links against libgcc).
+#include <inttypes.h>
+extern "C" uintptr_t _Unwind_GetIPInfo(_Unwind_Context *, int *);
+#endif
+
+#endif // BACKWARD_HAS_UNWIND == 1
+
+#if BACKWARD_HAS_LIBUNWIND == 1
+#define UNW_LOCAL_ONLY
+#include <libunwind.h>
+#endif // BACKWARD_HAS_LIBUNWIND == 1
+
+#ifdef BACKWARD_ATLEAST_CXX11
+#include <unordered_map>
+#include <utility> // for std::swap
+namespace backward {
+namespace details {
+template <typename K, typename V> struct hashtable {
+ typedef std::unordered_map<K, V> type;
+};
+using std::move;
+} // namespace details
+} // namespace backward
+#else // NOT BACKWARD_ATLEAST_CXX11
+#define nullptr NULL
+#define override
+#include <map>
+namespace backward {
+namespace details {
+template <typename K, typename V> struct hashtable {
+ typedef std::map<K, V> type;
+};
+template <typename T> const T &move(const T &v) { return v; }
+template <typename T> T &move(T &v) { return v; }
+} // namespace details
+} // namespace backward
+#endif // BACKWARD_ATLEAST_CXX11
+
+namespace backward {
+namespace details {
+#if defined(BACKWARD_SYSTEM_WINDOWS)
+const char kBackwardPathDelimiter[] = ";";
+#else
+const char kBackwardPathDelimiter[] = ":";
+#endif
+} // namespace details
+} // namespace backward
+
+namespace backward {
+
+namespace system_tag {
+struct linux_tag; // seems that I cannot call that "linux" because the name
+// is already defined... so I am adding _tag everywhere.
+struct darwin_tag;
+struct windows_tag;
+struct unknown_tag;
+
+#if defined(BACKWARD_SYSTEM_LINUX)
+typedef linux_tag current_tag;
+#elif defined(BACKWARD_SYSTEM_DARWIN)
+typedef darwin_tag current_tag;
+#elif defined(BACKWARD_SYSTEM_WINDOWS)
+typedef windows_tag current_tag;
+#elif defined(BACKWARD_SYSTEM_UNKNOWN)
+typedef unknown_tag current_tag;
+#else
+#error "May I please get my system defines?"
+#endif
+} // namespace system_tag
+
+namespace trace_resolver_tag {
+#if defined(BACKWARD_SYSTEM_LINUX)
+struct libdw;
+struct libbfd;
+struct libdwarf;
+struct backtrace_symbol;
+
+#if BACKWARD_HAS_DW == 1
+typedef libdw current;
+#elif BACKWARD_HAS_BFD == 1
+typedef libbfd current;
+#elif BACKWARD_HAS_DWARF == 1
+typedef libdwarf current;
+#elif BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+typedef backtrace_symbol current;
+#else
+#error "You shall not pass, until you know what you want."
+#endif
+#elif defined(BACKWARD_SYSTEM_DARWIN)
+struct backtrace_symbol;
+
+#if BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+typedef backtrace_symbol current;
+#else
+#error "You shall not pass, until you know what you want."
+#endif
+#elif defined(BACKWARD_SYSTEM_WINDOWS)
+struct pdb_symbol;
+#if BACKWARD_HAS_PDB_SYMBOL == 1
+typedef pdb_symbol current;
+#else
+#error "You shall not pass, until you know what you want."
+#endif
+#endif
+} // namespace trace_resolver_tag
+
+namespace details {
+
+template <typename T> struct rm_ptr { typedef T type; };
+
+template <typename T> struct rm_ptr<T *> { typedef T type; };
+
+template <typename T> struct rm_ptr<const T *> { typedef const T type; };
+
+template <typename R, typename T, R (*F)(T)> struct deleter {
+ template <typename U> void operator()(U &ptr) const { (*F)(ptr); }
+};
+
+template <typename T> struct default_delete {
+ void operator()(T &ptr) const { delete ptr; }
+};
+
+template <typename T, typename Deleter = deleter<void, void *, &::free>>
+class handle {
+ struct dummy;
+ T _val;
+ bool _empty;
+
+#ifdef BACKWARD_ATLEAST_CXX11
+ handle(const handle &) = delete;
+ handle &operator=(const handle &) = delete;
+#endif
+
+public:
+ ~handle() {
+ if (!_empty) {
+ Deleter()(_val);
+ }
+ }
+
+ explicit handle() : _val(), _empty(true) {}
+ explicit handle(T val) : _val(val), _empty(false) {
+ if (!_val)
+ _empty = true;
+ }
+
+#ifdef BACKWARD_ATLEAST_CXX11
+ handle(handle &&from) : _empty(true) { swap(from); }
+ handle &operator=(handle &&from) {
+ swap(from);
+ return *this;
+ }
+#else
+ explicit handle(const handle &from) : _empty(true) {
+ // some sort of poor man's move semantic.
+ swap(const_cast<handle &>(from));
+ }
+ handle &operator=(const handle &from) {
+ // some sort of poor man's move semantic.
+ swap(const_cast<handle &>(from));
+ return *this;
+ }
+#endif
+
+ void reset(T new_val) {
+ handle tmp(new_val);
+ swap(tmp);
+ }
+
+ void update(T new_val) {
+ _val = new_val;
+ _empty = !static_cast<bool>(new_val);
+ }
+
+ operator const dummy *() const {
+ if (_empty) {
+ return nullptr;
+ }
+ return reinterpret_cast<const dummy *>(_val);
+ }
+ T get() { return _val; }
+ T release() {
+ _empty = true;
+ return _val;
+ }
+ void swap(handle &b) {
+ using std::swap;
+ swap(b._val, _val); // can throw, we are safe here.
+ swap(b._empty, _empty); // should not throw: if you cannot swap two
+ // bools without throwing... It's a lost cause anyway!
+ }
+
+ T &operator->() { return _val; }
+ const T &operator->() const { return _val; }
+
+ typedef typename rm_ptr<T>::type &ref_t;
+ typedef const typename rm_ptr<T>::type &const_ref_t;
+ ref_t operator*() { return *_val; }
+ const_ref_t operator*() const { return *_val; }
+ ref_t operator[](size_t idx) { return _val[idx]; }
+
+ // Watch out, we've got a badass over here
+ T *operator&() {
+ _empty = false;
+ return &_val;
+ }
+};
+
+// Default demangler implementation (do nothing).
+template <typename TAG> struct demangler_impl {
+ static std::string demangle(const char *funcname) { return funcname; }
+};
+
+#if defined(BACKWARD_SYSTEM_LINUX) || defined(BACKWARD_SYSTEM_DARWIN)
+
+template <> struct demangler_impl<system_tag::current_tag> {
+ demangler_impl() : _demangle_buffer_length(0) {}
+
+ std::string demangle(const char *funcname) {
+ using namespace details;
+ char *result = abi::__cxa_demangle(funcname, _demangle_buffer.get(),
+ &_demangle_buffer_length, nullptr);
+ if (result) {
+ _demangle_buffer.update(result);
+ return result;
+ }
+ return funcname;
+ }
+
+private:
+ details::handle<char *> _demangle_buffer;
+ size_t _demangle_buffer_length;
+};
+
+#endif // BACKWARD_SYSTEM_LINUX || BACKWARD_SYSTEM_DARWIN
+
+struct demangler : public demangler_impl<system_tag::current_tag> {};
+
+// Split a string on the platform's PATH delimiter. Example: if delimiter
+// is ":" then:
+// "" --> []
+// ":" --> ["",""]
+// "::" --> ["","",""]
+// "/a/b/c" --> ["/a/b/c"]
+// "/a/b/c:/d/e/f" --> ["/a/b/c","/d/e/f"]
+// etc.
+inline std::vector<std::string> split_source_prefixes(const std::string &s) {
+ std::vector<std::string> out;
+ size_t last = 0;
+ size_t next = 0;
+ size_t delimiter_size = sizeof(kBackwardPathDelimiter) - 1;
+ while ((next = s.find(kBackwardPathDelimiter, last)) != std::string::npos) {
+ out.push_back(s.substr(last, next - last));
+ last = next + delimiter_size;
+ }
+ if (last <= s.length()) {
+ out.push_back(s.substr(last));
+ }
+ return out;
+}
+
+} // namespace details
+
+/*************** A TRACE ***************/
+
+struct Trace {
+ void *addr;
+ size_t idx;
+
+ Trace() : addr(nullptr), idx(0) {}
+
+ explicit Trace(void *_addr, size_t _idx) : addr(_addr), idx(_idx) {}
+};
+
+struct ResolvedTrace : public Trace {
+
+ struct SourceLoc {
+ std::string function;
+ std::string filename;
+ unsigned line;
+ unsigned col;
+
+ SourceLoc() : line(0), col(0) {}
+
+ bool operator==(const SourceLoc &b) const {
+ return function == b.function && filename == b.filename &&
+ line == b.line && col == b.col;
+ }
+
+ bool operator!=(const SourceLoc &b) const { return !(*this == b); }
+ };
+
+ // In which binary object this trace is located.
+ std::string object_filename;
+
+ // The function in the object that contain the trace. This is not the same
+ // as source.function which can be an function inlined in object_function.
+ std::string object_function;
+
+ // The source location of this trace. It is possible for filename to be
+ // empty and for line/col to be invalid (value 0) if this information
+ // couldn't be deduced, for example if there is no debug information in the
+ // binary object.
+ SourceLoc source;
+
+ // An optionals list of "inliners". All the successive sources location
+ // from where the source location of the trace (the attribute right above)
+ // is inlined. It is especially useful when you compiled with optimization.
+ typedef std::vector<SourceLoc> source_locs_t;
+ source_locs_t inliners;
+
+ ResolvedTrace() : Trace() {}
+ ResolvedTrace(const Trace &mini_trace) : Trace(mini_trace) {}
+};
+
+/*************** STACK TRACE ***************/
+
+// default implemention.
+template <typename TAG> class StackTraceImpl {
+public:
+ size_t size() const { return 0; }
+ Trace operator[](size_t) const { return Trace(); }
+ size_t load_here(size_t = 0) { return 0; }
+ size_t load_from(void *, size_t = 0, void * = nullptr, void * = nullptr) {
+ return 0;
+ }
+ size_t thread_id() const { return 0; }
+ void skip_n_firsts(size_t) {}
+};
+
+class StackTraceImplBase {
+public:
+ StackTraceImplBase()
+ : _thread_id(0), _skip(0), _context(nullptr), _error_addr(nullptr) {}
+
+ size_t thread_id() const { return _thread_id; }
+
+ void skip_n_firsts(size_t n) { _skip = n; }
+
+protected:
+ void load_thread_info() {
+#ifdef BACKWARD_SYSTEM_LINUX
+#ifndef __ANDROID__
+ _thread_id = static_cast<size_t>(syscall(SYS_gettid));
+#else
+ _thread_id = static_cast<size_t>(gettid());
+#endif
+ if (_thread_id == static_cast<size_t>(getpid())) {
+ // If the thread is the main one, let's hide that.
+ // I like to keep little secret sometimes.
+ _thread_id = 0;
+ }
+#elif defined(BACKWARD_SYSTEM_DARWIN)
+ _thread_id = reinterpret_cast<size_t>(pthread_self());
+ if (pthread_main_np() == 1) {
+ // If the thread is the main one, let's hide that.
+ _thread_id = 0;
+ }
+#endif
+ }
+
+ void set_context(void *context) { _context = context; }
+ void *context() const { return _context; }
+
+ void set_error_addr(void *error_addr) { _error_addr = error_addr; }
+ void *error_addr() const { return _error_addr; }
+
+ size_t skip_n_firsts() const { return _skip; }
+
+private:
+ size_t _thread_id;
+ size_t _skip;
+ void *_context;
+ void *_error_addr;
+};
+
+class StackTraceImplHolder : public StackTraceImplBase {
+public:
+ size_t size() const {
+ return (_stacktrace.size() >= skip_n_firsts())
+ ? _stacktrace.size() - skip_n_firsts()
+ : 0;
+ }
+ Trace operator[](size_t idx) const {
+ if (idx >= size()) {
+ return Trace();
+ }
+ return Trace(_stacktrace[idx + skip_n_firsts()], idx);
+ }
+ void *const *begin() const {
+ if (size()) {
+ return &_stacktrace[skip_n_firsts()];
+ }
+ return nullptr;
+ }
+
+protected:
+ std::vector<void *> _stacktrace;
+};
+
+#if BACKWARD_HAS_UNWIND == 1
+
+namespace details {
+
+template <typename F> class Unwinder {
+public:
+ size_t operator()(F &f, size_t depth) {
+ _f = &f;
+ _index = -1;
+ _depth = depth;
+ _Unwind_Backtrace(&this->backtrace_trampoline, this);
+ return static_cast<size_t>(_index);
+ }
+
+private:
+ F *_f;
+ ssize_t _index;
+ size_t _depth;
+
+ static _Unwind_Reason_Code backtrace_trampoline(_Unwind_Context *ctx,
+ void *self) {
+ return (static_cast<Unwinder *>(self))->backtrace(ctx);
+ }
+
+ _Unwind_Reason_Code backtrace(_Unwind_Context *ctx) {
+ if (_index >= 0 && static_cast<size_t>(_index) >= _depth)
+ return _URC_END_OF_STACK;
+
+ int ip_before_instruction = 0;
+ uintptr_t ip = _Unwind_GetIPInfo(ctx, &ip_before_instruction);
+
+ if (!ip_before_instruction) {
+ // calculating 0-1 for unsigned, looks like a possible bug to sanitiziers,
+ // so let's do it explicitly:
+ if (ip == 0) {
+ ip = std::numeric_limits<uintptr_t>::max(); // set it to 0xffff... (as
+ // from casting 0-1)
+ } else {
+ ip -= 1; // else just normally decrement it (no overflow/underflow will
+ // happen)
+ }
+ }
+
+ if (_index >= 0) { // ignore first frame.
+ (*_f)(static_cast<size_t>(_index), reinterpret_cast<void *>(ip));
+ }
+ _index += 1;
+ return _URC_NO_REASON;
+ }
+};
+
+template <typename F> size_t unwind(F f, size_t depth) {
+ Unwinder<F> unwinder;
+ return unwinder(f, depth);
+}
+
+} // namespace details
+
+template <>
+class StackTraceImpl<system_tag::current_tag> : public StackTraceImplHolder {
+public:
+ NOINLINE
+ size_t load_here(size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ load_thread_info();
+ set_context(context);
+ set_error_addr(error_addr);
+ if (depth == 0) {
+ return 0;
+ }
+ _stacktrace.resize(depth);
+ size_t trace_cnt = details::unwind(callback(*this), depth);
+ _stacktrace.resize(trace_cnt);
+ skip_n_firsts(0);
+ return size();
+ }
+ size_t load_from(void *addr, size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ load_here(depth + 8, context, error_addr);
+
+ for (size_t i = 0; i < _stacktrace.size(); ++i) {
+ if (_stacktrace[i] == addr) {
+ skip_n_firsts(i);
+ break;
+ }
+ }
+
+ _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth));
+ return size();
+ }
+
+private:
+ struct callback {
+ StackTraceImpl &self;
+ callback(StackTraceImpl &_self) : self(_self) {}
+
+ void operator()(size_t idx, void *addr) { self._stacktrace[idx] = addr; }
+ };
+};
+
+#elif BACKWARD_HAS_LIBUNWIND == 1
+
+template <>
+class StackTraceImpl<system_tag::current_tag> : public StackTraceImplHolder {
+public:
+ __attribute__((noinline)) size_t load_here(size_t depth = 32,
+ void *_context = nullptr,
+ void *_error_addr = nullptr) {
+ set_context(_context);
+ set_error_addr(_error_addr);
+ load_thread_info();
+ if (depth == 0) {
+ return 0;
+ }
+ _stacktrace.resize(depth + 1);
+
+ int result = 0;
+
+ unw_context_t ctx;
+ size_t index = 0;
+
+ // Add the tail call. If the Instruction Pointer is the crash address it
+ // means we got a bad function pointer dereference, so we "unwind" the
+ // bad pointer manually by using the return address pointed to by the
+ // Stack Pointer as the Instruction Pointer and letting libunwind do
+ // the rest
+
+ if (context()) {
+ ucontext_t *uctx = reinterpret_cast<ucontext_t *>(context());
+#ifdef REG_RIP // x86_64
+ if (uctx->uc_mcontext.gregs[REG_RIP] ==
+ reinterpret_cast<greg_t>(error_addr())) {
+ uctx->uc_mcontext.gregs[REG_RIP] =
+ *reinterpret_cast<size_t *>(uctx->uc_mcontext.gregs[REG_RSP]);
+ }
+ _stacktrace[index] =
+ reinterpret_cast<void *>(uctx->uc_mcontext.gregs[REG_RIP]);
+ ++index;
+ ctx = *reinterpret_cast<unw_context_t *>(uctx);
+#elif defined(REG_EIP) // x86_32
+ if (uctx->uc_mcontext.gregs[REG_EIP] ==
+ reinterpret_cast<greg_t>(error_addr())) {
+ uctx->uc_mcontext.gregs[REG_EIP] =
+ *reinterpret_cast<size_t *>(uctx->uc_mcontext.gregs[REG_ESP]);
+ }
+ _stacktrace[index] =
+ reinterpret_cast<void *>(uctx->uc_mcontext.gregs[REG_EIP]);
+ ++index;
+ ctx = *reinterpret_cast<unw_context_t *>(uctx);
+#elif defined(__arm__)
+ // libunwind uses its own context type for ARM unwinding.
+ // Copy the registers from the signal handler's context so we can
+ // unwind
+ unw_getcontext(&ctx);
+ ctx.regs[UNW_ARM_R0] = uctx->uc_mcontext.arm_r0;
+ ctx.regs[UNW_ARM_R1] = uctx->uc_mcontext.arm_r1;
+ ctx.regs[UNW_ARM_R2] = uctx->uc_mcontext.arm_r2;
+ ctx.regs[UNW_ARM_R3] = uctx->uc_mcontext.arm_r3;
+ ctx.regs[UNW_ARM_R4] = uctx->uc_mcontext.arm_r4;
+ ctx.regs[UNW_ARM_R5] = uctx->uc_mcontext.arm_r5;
+ ctx.regs[UNW_ARM_R6] = uctx->uc_mcontext.arm_r6;
+ ctx.regs[UNW_ARM_R7] = uctx->uc_mcontext.arm_r7;
+ ctx.regs[UNW_ARM_R8] = uctx->uc_mcontext.arm_r8;
+ ctx.regs[UNW_ARM_R9] = uctx->uc_mcontext.arm_r9;
+ ctx.regs[UNW_ARM_R10] = uctx->uc_mcontext.arm_r10;
+ ctx.regs[UNW_ARM_R11] = uctx->uc_mcontext.arm_fp;
+ ctx.regs[UNW_ARM_R12] = uctx->uc_mcontext.arm_ip;
+ ctx.regs[UNW_ARM_R13] = uctx->uc_mcontext.arm_sp;
+ ctx.regs[UNW_ARM_R14] = uctx->uc_mcontext.arm_lr;
+ ctx.regs[UNW_ARM_R15] = uctx->uc_mcontext.arm_pc;
+
+ // If we have crashed in the PC use the LR instead, as this was
+ // a bad function dereference
+ if (reinterpret_cast<unsigned long>(error_addr()) ==
+ uctx->uc_mcontext.arm_pc) {
+ ctx.regs[UNW_ARM_R15] =
+ uctx->uc_mcontext.arm_lr - sizeof(unsigned long);
+ }
+ _stacktrace[index] = reinterpret_cast<void *>(ctx.regs[UNW_ARM_R15]);
+ ++index;
+#elif defined(__APPLE__) && defined(__x86_64__)
+ unw_getcontext(&ctx);
+ // OS X's implementation of libunwind uses its own context object
+ // so we need to convert the passed context to libunwind's format
+ // (information about the data layout taken from unw_getcontext.s
+ // in Apple's libunwind source
+ ctx.data[0] = uctx->uc_mcontext->__ss.__rax;
+ ctx.data[1] = uctx->uc_mcontext->__ss.__rbx;
+ ctx.data[2] = uctx->uc_mcontext->__ss.__rcx;
+ ctx.data[3] = uctx->uc_mcontext->__ss.__rdx;
+ ctx.data[4] = uctx->uc_mcontext->__ss.__rdi;
+ ctx.data[5] = uctx->uc_mcontext->__ss.__rsi;
+ ctx.data[6] = uctx->uc_mcontext->__ss.__rbp;
+ ctx.data[7] = uctx->uc_mcontext->__ss.__rsp;
+ ctx.data[8] = uctx->uc_mcontext->__ss.__r8;
+ ctx.data[9] = uctx->uc_mcontext->__ss.__r9;
+ ctx.data[10] = uctx->uc_mcontext->__ss.__r10;
+ ctx.data[11] = uctx->uc_mcontext->__ss.__r11;
+ ctx.data[12] = uctx->uc_mcontext->__ss.__r12;
+ ctx.data[13] = uctx->uc_mcontext->__ss.__r13;
+ ctx.data[14] = uctx->uc_mcontext->__ss.__r14;
+ ctx.data[15] = uctx->uc_mcontext->__ss.__r15;
+ ctx.data[16] = uctx->uc_mcontext->__ss.__rip;
+
+ // If the IP is the same as the crash address we have a bad function
+ // dereference The caller's address is pointed to by %rsp, so we
+ // dereference that value and set it to be the next frame's IP.
+ if (uctx->uc_mcontext->__ss.__rip ==
+ reinterpret_cast<__uint64_t>(error_addr())) {
+ ctx.data[16] =
+ *reinterpret_cast<__uint64_t *>(uctx->uc_mcontext->__ss.__rsp);
+ }
+ _stacktrace[index] = reinterpret_cast<void *>(ctx.data[16]);
+ ++index;
+#elif defined(__APPLE__)
+ unw_getcontext(&ctx)
+ // TODO: Convert the ucontext_t to libunwind's unw_context_t like
+ // we do in 64 bits
+ if (ctx.uc_mcontext->__ss.__eip ==
+ reinterpret_cast<greg_t>(error_addr())) {
+ ctx.uc_mcontext->__ss.__eip = ctx.uc_mcontext->__ss.__esp;
+ }
+ _stacktrace[index] =
+ reinterpret_cast<void *>(ctx.uc_mcontext->__ss.__eip);
+ ++index;
+#endif
+ }
+
+ unw_cursor_t cursor;
+ if (context()) {
+#if defined(UNW_INIT_SIGNAL_FRAME)
+ result = unw_init_local2(&cursor, &ctx, UNW_INIT_SIGNAL_FRAME);
+#else
+ result = unw_init_local(&cursor, &ctx);
+#endif
+ } else {
+ unw_getcontext(&ctx);
+ ;
+ result = unw_init_local(&cursor, &ctx);
+ }
+
+ if (result != 0)
+ return 1;
+
+ unw_word_t ip = 0;
+
+ while (index <= depth && unw_step(&cursor) > 0) {
+ result = unw_get_reg(&cursor, UNW_REG_IP, &ip);
+ if (result == 0) {
+ _stacktrace[index] = reinterpret_cast<void *>(--ip);
+ ++index;
+ }
+ }
+ --index;
+
+ _stacktrace.resize(index + 1);
+ skip_n_firsts(0);
+ return size();
+ }
+
+ size_t load_from(void *addr, size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ load_here(depth + 8, context, error_addr);
+
+ for (size_t i = 0; i < _stacktrace.size(); ++i) {
+ if (_stacktrace[i] == addr) {
+ skip_n_firsts(i);
+ _stacktrace[i] = (void *)((uintptr_t)_stacktrace[i]);
+ break;
+ }
+ }
+
+ _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth));
+ return size();
+ }
+};
+
+#elif defined(BACKWARD_HAS_BACKTRACE)
+
+template <>
+class StackTraceImpl<system_tag::current_tag> : public StackTraceImplHolder {
+public:
+ NOINLINE
+ size_t load_here(size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ set_context(context);
+ set_error_addr(error_addr);
+ load_thread_info();
+ if (depth == 0) {
+ return 0;
+ }
+ _stacktrace.resize(depth + 1);
+ size_t trace_cnt = backtrace(&_stacktrace[0], _stacktrace.size());
+ _stacktrace.resize(trace_cnt);
+ skip_n_firsts(1);
+ return size();
+ }
+
+ size_t load_from(void *addr, size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ load_here(depth + 8, context, error_addr);
+
+ for (size_t i = 0; i < _stacktrace.size(); ++i) {
+ if (_stacktrace[i] == addr) {
+ skip_n_firsts(i);
+ _stacktrace[i] = (void *)((uintptr_t)_stacktrace[i] + 1);
+ break;
+ }
+ }
+
+ _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth));
+ return size();
+ }
+};
+
+#elif defined(BACKWARD_SYSTEM_WINDOWS)
+
+template <>
+class StackTraceImpl<system_tag::current_tag> : public StackTraceImplHolder {
+public:
+ // We have to load the machine type from the image info
+ // So we first initialize the resolver, and it tells us this info
+ void set_machine_type(DWORD machine_type) { machine_type_ = machine_type; }
+ void set_context(CONTEXT *ctx) { ctx_ = ctx; }
+ void set_thread_handle(HANDLE handle) { thd_ = handle; }
+
+ NOINLINE
+ size_t load_here(size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ set_context(static_cast<CONTEXT*>(context));
+ set_error_addr(error_addr);
+ CONTEXT localCtx; // used when no context is provided
+
+ if (depth == 0) {
+ return 0;
+ }
+
+ if (!ctx_) {
+ ctx_ = &localCtx;
+ RtlCaptureContext(ctx_);
+ }
+
+ if (!thd_) {
+ thd_ = GetCurrentThread();
+ }
+
+ HANDLE process = GetCurrentProcess();
+
+ STACKFRAME64 s;
+ memset(&s, 0, sizeof(STACKFRAME64));
+
+ // TODO: 32 bit context capture
+ s.AddrStack.Mode = AddrModeFlat;
+ s.AddrFrame.Mode = AddrModeFlat;
+ s.AddrPC.Mode = AddrModeFlat;
+#ifdef _M_X64
+ s.AddrPC.Offset = ctx_->Rip;
+ s.AddrStack.Offset = ctx_->Rsp;
+ s.AddrFrame.Offset = ctx_->Rbp;
+#else
+ s.AddrPC.Offset = ctx_->Eip;
+ s.AddrStack.Offset = ctx_->Esp;
+ s.AddrFrame.Offset = ctx_->Ebp;
+#endif
+
+ if (!machine_type_) {
+#ifdef _M_X64
+ machine_type_ = IMAGE_FILE_MACHINE_AMD64;
+#else
+ machine_type_ = IMAGE_FILE_MACHINE_I386;
+#endif
+ }
+
+ for (;;) {
+ // NOTE: this only works if PDBs are already loaded!
+ SetLastError(0);
+ if (!StackWalk64(machine_type_, process, thd_, &s, ctx_, NULL,
+ SymFunctionTableAccess64, SymGetModuleBase64, NULL))
+ break;
+
+ if (s.AddrReturn.Offset == 0)
+ break;
+
+ _stacktrace.push_back(reinterpret_cast<void *>(s.AddrPC.Offset));
+
+ if (size() >= depth)
+ break;
+ }
+
+ return size();
+ }
+
+ size_t load_from(void *addr, size_t depth = 32, void *context = nullptr,
+ void *error_addr = nullptr) {
+ load_here(depth + 8, context, error_addr);
+
+ for (size_t i = 0; i < _stacktrace.size(); ++i) {
+ if (_stacktrace[i] == addr) {
+ skip_n_firsts(i);
+ break;
+ }
+ }
+
+ _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth));
+ return size();
+ }
+
+private:
+ DWORD machine_type_ = 0;
+ HANDLE thd_ = 0;
+ CONTEXT *ctx_ = nullptr;
+};
+
+#endif
+
+class StackTrace : public StackTraceImpl<system_tag::current_tag> {};
+
+/*************** TRACE RESOLVER ***************/
+
+class TraceResolverImplBase {
+public:
+ virtual ~TraceResolverImplBase() {}
+
+ virtual void load_addresses(void *const*addresses, int address_count) {
+ (void)addresses;
+ (void)address_count;
+ }
+
+ template <class ST> void load_stacktrace(ST &st) {
+ load_addresses(st.begin(), (int)st.size());
+ }
+
+ virtual ResolvedTrace resolve(ResolvedTrace t) { return t; }
+
+protected:
+ std::string demangle(const char *funcname) {
+ return _demangler.demangle(funcname);
+ }
+
+private:
+ details::demangler _demangler;
+};
+
+template <typename TAG> class TraceResolverImpl;
+
+#ifdef BACKWARD_SYSTEM_UNKNOWN
+
+template <> class TraceResolverImpl<system_tag::unknown_tag>
+ : public TraceResolverImplBase {};
+
+#endif
+
+#ifdef BACKWARD_SYSTEM_LINUX
+
+class TraceResolverLinuxBase : public TraceResolverImplBase {
+public:
+ TraceResolverLinuxBase()
+ : argv0_(get_argv0()), exec_path_(read_symlink("/proc/self/exe")) {}
+ std::string resolve_exec_path(Dl_info &symbol_info) const {
+ // mutates symbol_info.dli_fname to be filename to open and returns filename
+ // to display
+ if (symbol_info.dli_fname == argv0_) {
+ // dladdr returns argv[0] in dli_fname for symbols contained in
+ // the main executable, which is not a valid path if the
+ // executable was found by a search of the PATH environment
+ // variable; In that case, we actually open /proc/self/exe, which
+ // is always the actual executable (even if it was deleted/replaced!)
+ // but display the path that /proc/self/exe links to.
+ // However, this right away reduces probability of successful symbol
+ // resolution, because libbfd may try to find *.debug files in the
+ // same dir, in case symbols are stripped. As a result, it may try
+ // to find a file /proc/self/<exe_name>.debug, which obviously does
+ // not exist. /proc/self/exe is a last resort. First load attempt
+ // should go for the original executable file path.
+ symbol_info.dli_fname = "/proc/self/exe";
+ return exec_path_;
+ } else {
+ return symbol_info.dli_fname;
+ }
+ }
+
+private:
+ std::string argv0_;
+ std::string exec_path_;
+
+ static std::string get_argv0() {
+ std::string argv0;
+ std::ifstream ifs("/proc/self/cmdline");
+ std::getline(ifs, argv0, '\0');
+ return argv0;
+ }
+
+ static std::string read_symlink(std::string const &symlink_path) {
+ std::string path;
+ path.resize(100);
+
+ while (true) {
+ ssize_t len =
+ ::readlink(symlink_path.c_str(), &*path.begin(), path.size());
+ if (len < 0) {
+ return "";
+ }
+ if (static_cast<size_t>(len) == path.size()) {
+ path.resize(path.size() * 2);
+ } else {
+ path.resize(static_cast<std::string::size_type>(len));
+ break;
+ }
+ }
+
+ return path;
+ }
+};
+
+template <typename STACKTRACE_TAG> class TraceResolverLinuxImpl;
+
+#if BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+
+template <>
+class TraceResolverLinuxImpl<trace_resolver_tag::backtrace_symbol>
+ : public TraceResolverLinuxBase {
+public:
+ void load_addresses(void *const*addresses, int address_count) override {
+ if (address_count == 0) {
+ return;
+ }
+ _symbols.reset(backtrace_symbols(addresses, address_count));
+ }
+
+ ResolvedTrace resolve(ResolvedTrace trace) override {
+ char *filename = _symbols[trace.idx];
+ char *funcname = filename;
+ while (*funcname && *funcname != '(') {
+ funcname += 1;
+ }
+ trace.object_filename.assign(filename,
+ funcname); // ok even if funcname is the ending
+ // \0 (then we assign entire string)
+
+ if (*funcname) { // if it's not end of string (e.g. from last frame ip==0)
+ funcname += 1;
+ char *funcname_end = funcname;
+ while (*funcname_end && *funcname_end != ')' && *funcname_end != '+') {
+ funcname_end += 1;
+ }
+ *funcname_end = '\0';
+ trace.object_function = this->demangle(funcname);
+ trace.source.function = trace.object_function; // we cannot do better.
+ }
+ return trace;
+ }
+
+private:
+ details::handle<char **> _symbols;
+};
+
+#endif // BACKWARD_HAS_BACKTRACE_SYMBOL == 1
+
+#if BACKWARD_HAS_BFD == 1
+
+template <>
+class TraceResolverLinuxImpl<trace_resolver_tag::libbfd>
+ : public TraceResolverLinuxBase {
+public:
+ TraceResolverLinuxImpl() : _bfd_loaded(false) {}
+
+ ResolvedTrace resolve(ResolvedTrace trace) override {
+ Dl_info symbol_info;
+
+ // trace.addr is a virtual address in memory pointing to some code.
+ // Let's try to find from which loaded object it comes from.
+ // The loaded object can be yourself btw.
+ if (!dladdr(trace.addr, &symbol_info)) {
+ return trace; // dat broken trace...
+ }
+
+ // Now we get in symbol_info:
+ // .dli_fname:
+ // pathname of the shared object that contains the address.
+ // .dli_fbase:
+ // where the object is loaded in memory.
+ // .dli_sname:
+ // the name of the nearest symbol to trace.addr, we expect a
+ // function name.
+ // .dli_saddr:
+ // the exact address corresponding to .dli_sname.
+
+ if (symbol_info.dli_sname) {
+ trace.object_function = demangle(symbol_info.dli_sname);
+ }
+
+ if (!symbol_info.dli_fname) {
+ return trace;
+ }
+
+ trace.object_filename = resolve_exec_path(symbol_info);
+ bfd_fileobject *fobj;
+ // Before rushing to resolution need to ensure the executable
+ // file still can be used. For that compare inode numbers of
+ // what is stored by the executable's file path, and in the
+ // dli_fname, which not necessarily equals to the executable.
+ // It can be a shared library, or /proc/self/exe, and in the
+ // latter case has drawbacks. See the exec path resolution for
+ // details. In short - the dli object should be used only as
+ // the last resort.
+ // If inode numbers are equal, it is known dli_fname and the
+ // executable file are the same. This is guaranteed by Linux,
+ // because if the executable file is changed/deleted, it will
+ // be done in a new inode. The old file will be preserved in
+ // /proc/self/exe, and may even have inode 0. The latter can
+ // happen if the inode was actually reused, and the file was
+ // kept only in the main memory.
+ //
+ struct stat obj_stat;
+ struct stat dli_stat;
+ if (stat(trace.object_filename.c_str(), &obj_stat) == 0 &&
+ stat(symbol_info.dli_fname, &dli_stat) == 0 &&
+ obj_stat.st_ino == dli_stat.st_ino) {
+ // The executable file, and the shared object containing the
+ // address are the same file. Safe to use the original path.
+ // this is preferable. Libbfd will search for stripped debug
+ // symbols in the same directory.
+ fobj = load_object_with_bfd(trace.object_filename);
+ } else{
+ // The original object file was *deleted*! The only hope is
+ // that the debug symbols are either inside the shared
+ // object file, or are in the same directory, and this is
+ // not /proc/self/exe.
+ fobj = nullptr;
+ }
+ if (fobj == nullptr || !fobj->handle) {
+ fobj = load_object_with_bfd(symbol_info.dli_fname);
+ if (!fobj->handle) {
+ return trace;
+ }
+ }
+
+ find_sym_result *details_selected; // to be filled.
+
+ // trace.addr is the next instruction to be executed after returning
+ // from the nested stack frame. In C++ this usually relate to the next
+ // statement right after the function call that leaded to a new stack
+ // frame. This is not usually what you want to see when printing out a
+ // stacktrace...
+ find_sym_result details_call_site =
+ find_symbol_details(fobj, trace.addr, symbol_info.dli_fbase);
+ details_selected = &details_call_site;
+
+#if BACKWARD_HAS_UNWIND == 0
+ // ...this is why we also try to resolve the symbol that is right
+ // before the return address. If we are lucky enough, we will get the
+ // line of the function that was called. But if the code is optimized,
+ // we might get something absolutely not related since the compiler
+ // can reschedule the return address with inline functions and
+ // tail-call optimisation (among other things that I don't even know
+ // or cannot even dream about with my tiny limited brain).
+ find_sym_result details_adjusted_call_site = find_symbol_details(
+ fobj, (void *)(uintptr_t(trace.addr) - 1), symbol_info.dli_fbase);
+
+ // In debug mode, we should always get the right thing(TM).
+ if (details_call_site.found && details_adjusted_call_site.found) {
+ // Ok, we assume that details_adjusted_call_site is a better estimation.
+ details_selected = &details_adjusted_call_site;
+ trace.addr = (void *)(uintptr_t(trace.addr) - 1);
+ }
+
+ if (details_selected == &details_call_site && details_call_site.found) {
+ // we have to re-resolve the symbol in order to reset some
+ // internal state in BFD... so we can call backtrace_inliners
+ // thereafter...
+ details_call_site =
+ find_symbol_details(fobj, trace.addr, symbol_info.dli_fbase);
+ }
+#endif // BACKWARD_HAS_UNWIND
+
+ if (details_selected->found) {
+ if (details_selected->filename) {
+ trace.source.filename = details_selected->filename;
+ }
+ trace.source.line = details_selected->line;
+
+ if (details_selected->funcname) {
+ // this time we get the name of the function where the code is
+ // located, instead of the function were the address is
+ // located. In short, if the code was inlined, we get the
+ // function correspoding to the code. Else we already got in
+ // trace.function.
+ trace.source.function = demangle(details_selected->funcname);
+
+ if (!symbol_info.dli_sname) {
+ // for the case dladdr failed to find the symbol name of
+ // the function, we might as well try to put something
+ // here.
+ trace.object_function = trace.source.function;
+ }
+ }
+
+ // Maybe the source of the trace got inlined inside the function
+ // (trace.source.function). Let's see if we can get all the inlined
+ // calls along the way up to the initial call site.
+ trace.inliners = backtrace_inliners(fobj, *details_selected);
+
+#if 0
+ if (trace.inliners.size() == 0) {
+ // Maybe the trace was not inlined... or maybe it was and we
+ // are lacking the debug information. Let's try to make the
+ // world better and see if we can get the line number of the
+ // function (trace.source.function) now.
+ //
+ // We will get the location of where the function start (to be
+ // exact: the first instruction that really start the
+ // function), not where the name of the function is defined.
+ // This can be quite far away from the name of the function
+ // btw.
+ //
+ // If the source of the function is the same as the source of
+ // the trace, we cannot say if the trace was really inlined or
+ // not. However, if the filename of the source is different
+ // between the function and the trace... we can declare it as
+ // an inliner. This is not 100% accurate, but better than
+ // nothing.
+
+ if (symbol_info.dli_saddr) {
+ find_sym_result details = find_symbol_details(fobj,
+ symbol_info.dli_saddr,
+ symbol_info.dli_fbase);
+
+ if (details.found) {
+ ResolvedTrace::SourceLoc diy_inliner;
+ diy_inliner.line = details.line;
+ if (details.filename) {
+ diy_inliner.filename = details.filename;
+ }
+ if (details.funcname) {
+ diy_inliner.function = demangle(details.funcname);
+ } else {
+ diy_inliner.function = trace.source.function;
+ }
+ if (diy_inliner != trace.source) {
+ trace.inliners.push_back(diy_inliner);
+ }
+ }
+ }
+ }
+#endif
+ }
+
+ return trace;
+ }
+
+private:
+ bool _bfd_loaded;
+
+ typedef details::handle<bfd *,
+ details::deleter<bfd_boolean, bfd *, &bfd_close>>
+ bfd_handle_t;
+
+ typedef details::handle<asymbol **> bfd_symtab_t;
+
+ struct bfd_fileobject {
+ bfd_handle_t handle;
+ bfd_vma base_addr;
+ bfd_symtab_t symtab;
+ bfd_symtab_t dynamic_symtab;
+ };
+
+ typedef details::hashtable<std::string, bfd_fileobject>::type fobj_bfd_map_t;
+ fobj_bfd_map_t _fobj_bfd_map;
+
+ bfd_fileobject *load_object_with_bfd(const std::string &filename_object) {
+ using namespace details;
+
+ if (!_bfd_loaded) {
+ using namespace details;
+ bfd_init();
+ _bfd_loaded = true;
+ }
+
+ fobj_bfd_map_t::iterator it = _fobj_bfd_map.find(filename_object);
+ if (it != _fobj_bfd_map.end()) {
+ return &it->second;
+ }
+
+ // this new object is empty for now.
+ bfd_fileobject *r = &_fobj_bfd_map[filename_object];
+
+ // we do the work temporary in this one;
+ bfd_handle_t bfd_handle;
+
+ int fd = open(filename_object.c_str(), O_RDONLY);
+ bfd_handle.reset(bfd_fdopenr(filename_object.c_str(), "default", fd));
+ if (!bfd_handle) {
+ close(fd);
+ return r;
+ }
+
+ if (!bfd_check_format(bfd_handle.get(), bfd_object)) {
+ return r; // not an object? You lose.
+ }
+
+ if ((bfd_get_file_flags(bfd_handle.get()) & HAS_SYMS) == 0) {
+ return r; // that's what happen when you forget to compile in debug.
+ }
+
+ ssize_t symtab_storage_size = bfd_get_symtab_upper_bound(bfd_handle.get());
+
+ ssize_t dyn_symtab_storage_size =
+ bfd_get_dynamic_symtab_upper_bound(bfd_handle.get());
+
+ if (symtab_storage_size <= 0 && dyn_symtab_storage_size <= 0) {
+ return r; // weird, is the file is corrupted?
+ }
+
+ bfd_symtab_t symtab, dynamic_symtab;
+ ssize_t symcount = 0, dyn_symcount = 0;
+
+ if (symtab_storage_size > 0) {
+ symtab.reset(static_cast<bfd_symbol **>(
+ malloc(static_cast<size_t>(symtab_storage_size))));
+ symcount = bfd_canonicalize_symtab(bfd_handle.get(), symtab.get());
+ }
+
+ if (dyn_symtab_storage_size > 0) {
+ dynamic_symtab.reset(static_cast<bfd_symbol **>(
+ malloc(static_cast<size_t>(dyn_symtab_storage_size))));
+ dyn_symcount = bfd_canonicalize_dynamic_symtab(bfd_handle.get(),
+ dynamic_symtab.get());
+ }
+
+ if (symcount <= 0 && dyn_symcount <= 0) {
+ return r; // damned, that's a stripped file that you got there!
+ }
+
+ r->handle = move(bfd_handle);
+ r->symtab = move(symtab);
+ r->dynamic_symtab = move(dynamic_symtab);
+ return r;
+ }
+
+ struct find_sym_result {
+ bool found;
+ const char *filename;
+ const char *funcname;
+ unsigned int line;
+ };
+
+ struct find_sym_context {
+ TraceResolverLinuxImpl *self;
+ bfd_fileobject *fobj;
+ void *addr;
+ void *base_addr;
+ find_sym_result result;
+ };
+
+ find_sym_result find_symbol_details(bfd_fileobject *fobj, void *addr,
+ void *base_addr) {
+ find_sym_context context;
+ context.self = this;
+ context.fobj = fobj;
+ context.addr = addr;
+ context.base_addr = base_addr;
+ context.result.found = false;
+ bfd_map_over_sections(fobj->handle.get(), &find_in_section_trampoline,
+ static_cast<void *>(&context));
+ return context.result;
+ }
+
+ static void find_in_section_trampoline(bfd *, asection *section, void *data) {
+ find_sym_context *context = static_cast<find_sym_context *>(data);
+ context->self->find_in_section(
+ reinterpret_cast<bfd_vma>(context->addr),
+ reinterpret_cast<bfd_vma>(context->base_addr), context->fobj, section,
+ context->result);
+ }
+
+ void find_in_section(bfd_vma addr, bfd_vma base_addr, bfd_fileobject *fobj,
+ asection *section, find_sym_result &result) {
+ if (result.found)
+ return;
+
+#ifdef bfd_get_section_flags
+ if ((bfd_get_section_flags(fobj->handle.get(), section) & SEC_ALLOC) == 0)
+#else
+ if ((bfd_section_flags(section) & SEC_ALLOC) == 0)
+#endif
+ return; // a debug section is never loaded automatically.
+
+#ifdef bfd_get_section_vma
+ bfd_vma sec_addr = bfd_get_section_vma(fobj->handle.get(), section);
+#else
+ bfd_vma sec_addr = bfd_section_vma(section);
+#endif
+#ifdef bfd_get_section_size
+ bfd_size_type size = bfd_get_section_size(section);
+#else
+ bfd_size_type size = bfd_section_size(section);
+#endif
+
+ // are we in the boundaries of the section?
+ if (addr < sec_addr || addr >= sec_addr + size) {
+ addr -= base_addr; // oups, a relocated object, lets try again...
+ if (addr < sec_addr || addr >= sec_addr + size) {
+ return;
+ }
+ }
+
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
+#endif
+ if (!result.found && fobj->symtab) {
+ result.found = bfd_find_nearest_line(
+ fobj->handle.get(), section, fobj->symtab.get(), addr - sec_addr,
+ &result.filename, &result.funcname, &result.line);
+ }
+
+ if (!result.found && fobj->dynamic_symtab) {
+ result.found = bfd_find_nearest_line(
+ fobj->handle.get(), section, fobj->dynamic_symtab.get(),
+ addr - sec_addr, &result.filename, &result.funcname, &result.line);
+ }
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+ }
+
+ ResolvedTrace::source_locs_t
+ backtrace_inliners(bfd_fileobject *fobj, find_sym_result previous_result) {
+ // This function can be called ONLY after a SUCCESSFUL call to
+ // find_symbol_details. The state is global to the bfd_handle.
+ ResolvedTrace::source_locs_t results;
+ while (previous_result.found) {
+ find_sym_result result;
+ result.found = bfd_find_inliner_info(fobj->handle.get(), &result.filename,
+ &result.funcname, &result.line);
+
+ if (result
+ .found) /* and not (
+ cstrings_eq(previous_result.filename,
+ result.filename) and
+ cstrings_eq(previous_result.funcname, result.funcname)
+ and result.line == previous_result.line
+ )) */
+ {
+ ResolvedTrace::SourceLoc src_loc;
+ src_loc.line = result.line;
+ if (result.filename) {
+ src_loc.filename = result.filename;
+ }
+ if (result.funcname) {
+ src_loc.function = demangle(result.funcname);
+ }
+ results.push_back(src_loc);
+ }
+ previous_result = result;
+ }
+ return results;
+ }
+
+ bool cstrings_eq(const char *a, const char *b) {
+ if (!a || !b) {
+ return false;
+ }
+ return strcmp(a, b) == 0;
+ }
+};
+#endif // BACKWARD_HAS_BFD == 1
+
+#if BACKWARD_HAS_DW == 1
+
+template <>
+class TraceResolverLinuxImpl<trace_resolver_tag::libdw>
+ : public TraceResolverLinuxBase {
+public:
+ TraceResolverLinuxImpl() : _dwfl_handle_initialized(false) {}
+
+ ResolvedTrace resolve(ResolvedTrace trace) override {
+ using namespace details;
+
+ Dwarf_Addr trace_addr = (Dwarf_Addr)trace.addr;
+
+ if (!_dwfl_handle_initialized) {
+ // initialize dwfl...
+ _dwfl_cb.reset(new Dwfl_Callbacks);
+ _dwfl_cb->find_elf = &dwfl_linux_proc_find_elf;
+ _dwfl_cb->find_debuginfo = &dwfl_standard_find_debuginfo;
+ _dwfl_cb->debuginfo_path = 0;
+
+ _dwfl_handle.reset(dwfl_begin(_dwfl_cb.get()));
+ _dwfl_handle_initialized = true;
+
+ if (!_dwfl_handle) {
+ return trace;
+ }
+
+ // ...from the current process.
+ dwfl_report_begin(_dwfl_handle.get());
+ int r = dwfl_linux_proc_report(_dwfl_handle.get(), getpid());
+ dwfl_report_end(_dwfl_handle.get(), NULL, NULL);
+ if (r < 0) {
+ return trace;
+ }
+ }
+
+ if (!_dwfl_handle) {
+ return trace;
+ }
+
+ // find the module (binary object) that contains the trace's address.
+ // This is not using any debug information, but the addresses ranges of
+ // all the currently loaded binary object.
+ Dwfl_Module *mod = dwfl_addrmodule(_dwfl_handle.get(), trace_addr);
+ if (mod) {
+ // now that we found it, lets get the name of it, this will be the
+ // full path to the running binary or one of the loaded library.
+ const char *module_name = dwfl_module_info(mod, 0, 0, 0, 0, 0, 0, 0);
+ if (module_name) {
+ trace.object_filename = module_name;
+ }
+ // We also look after the name of the symbol, equal or before this
+ // address. This is found by walking the symtab. We should get the
+ // symbol corresponding to the function (mangled) containing the
+ // address. If the code corresponding to the address was inlined,
+ // this is the name of the out-most inliner function.
+ const char *sym_name = dwfl_module_addrname(mod, trace_addr);
+ if (sym_name) {
+ trace.object_function = demangle(sym_name);
+ }
+ }
+
+ // now let's get serious, and find out the source location (file and
+ // line number) of the address.
+
+ // This function will look in .debug_aranges for the address and map it
+ // to the location of the compilation unit DIE in .debug_info and
+ // return it.
+ Dwarf_Addr mod_bias = 0;
+ Dwarf_Die *cudie = dwfl_module_addrdie(mod, trace_addr, &mod_bias);
+
+#if 1
+ if (!cudie) {
+ // Sadly clang does not generate the section .debug_aranges, thus
+ // dwfl_module_addrdie will fail early. Clang doesn't either set
+ // the lowpc/highpc/range info for every compilation unit.
+ //
+ // So in order to save the world:
+ // for every compilation unit, we will iterate over every single
+ // DIEs. Normally functions should have a lowpc/highpc/range, which
+ // we will use to infer the compilation unit.
+
+ // note that this is probably badly inefficient.
+ while ((cudie = dwfl_module_nextcu(mod, cudie, &mod_bias))) {
+ Dwarf_Die die_mem;
+ Dwarf_Die *fundie =
+ find_fundie_by_pc(cudie, trace_addr - mod_bias, &die_mem);
+ if (fundie) {
+ break;
+ }
+ }
+ }
+#endif
+
+//#define BACKWARD_I_DO_NOT_RECOMMEND_TO_ENABLE_THIS_HORRIBLE_PIECE_OF_CODE
+#ifdef BACKWARD_I_DO_NOT_RECOMMEND_TO_ENABLE_THIS_HORRIBLE_PIECE_OF_CODE
+ if (!cudie) {
+ // If it's still not enough, lets dive deeper in the shit, and try
+ // to save the world again: for every compilation unit, we will
+ // load the corresponding .debug_line section, and see if we can
+ // find our address in it.
+
+ Dwarf_Addr cfi_bias;
+ Dwarf_CFI *cfi_cache = dwfl_module_eh_cfi(mod, &cfi_bias);
+
+ Dwarf_Addr bias;
+ while ((cudie = dwfl_module_nextcu(mod, cudie, &bias))) {
+ if (dwarf_getsrc_die(cudie, trace_addr - bias)) {
+
+ // ...but if we get a match, it might be a false positive
+ // because our (address - bias) might as well be valid in a
+ // different compilation unit. So we throw our last card on
+ // the table and lookup for the address into the .eh_frame
+ // section.
+
+ handle<Dwarf_Frame *> frame;
+ dwarf_cfi_addrframe(cfi_cache, trace_addr - cfi_bias, &frame);
+ if (frame) {
+ break;
+ }
+ }
+ }
+ }
+#endif
+
+ if (!cudie) {
+ return trace; // this time we lost the game :/
+ }
+
+ // Now that we have a compilation unit DIE, this function will be able
+ // to load the corresponding section in .debug_line (if not already
+ // loaded) and hopefully find the source location mapped to our
+ // address.
+ Dwarf_Line *srcloc = dwarf_getsrc_die(cudie, trace_addr - mod_bias);
+
+ if (srcloc) {
+ const char *srcfile = dwarf_linesrc(srcloc, 0, 0);
+ if (srcfile) {
+ trace.source.filename = srcfile;
+ }
+ int line = 0, col = 0;
+ dwarf_lineno(srcloc, &line);
+ dwarf_linecol(srcloc, &col);
+ trace.source.line = line;
+ trace.source.col = col;
+ }
+
+ deep_first_search_by_pc(cudie, trace_addr - mod_bias,
+ inliners_search_cb(trace));
+ if (trace.source.function.size() == 0) {
+ // fallback.
+ trace.source.function = trace.object_function;
+ }
+
+ return trace;
+ }
+
+private:
+ typedef details::handle<Dwfl *, details::deleter<void, Dwfl *, &dwfl_end>>
+ dwfl_handle_t;
+ details::handle<Dwfl_Callbacks *, details::default_delete<Dwfl_Callbacks *>>
+ _dwfl_cb;
+ dwfl_handle_t _dwfl_handle;
+ bool _dwfl_handle_initialized;
+
+ // defined here because in C++98, template function cannot take locally
+ // defined types... grrr.
+ struct inliners_search_cb {
+ void operator()(Dwarf_Die *die) {
+ switch (dwarf_tag(die)) {
+ const char *name;
+ case DW_TAG_subprogram:
+ if ((name = dwarf_diename(die))) {
+ trace.source.function = name;
+ }
+ break;
+
+ case DW_TAG_inlined_subroutine:
+ ResolvedTrace::SourceLoc sloc;
+ Dwarf_Attribute attr_mem;
+
+ if ((name = dwarf_diename(die))) {
+ sloc.function = name;
+ }
+ if ((name = die_call_file(die))) {
+ sloc.filename = name;
+ }
+
+ Dwarf_Word line = 0, col = 0;
+ dwarf_formudata(dwarf_attr(die, DW_AT_call_line, &attr_mem), &line);
+ dwarf_formudata(dwarf_attr(die, DW_AT_call_column, &attr_mem), &col);
+ sloc.line = (unsigned)line;
+ sloc.col = (unsigned)col;
+
+ trace.inliners.push_back(sloc);
+ break;
+ };
+ }
+ ResolvedTrace &trace;
+ inliners_search_cb(ResolvedTrace &t) : trace(t) {}
+ };
+
+ static bool die_has_pc(Dwarf_Die *die, Dwarf_Addr pc) {
+ Dwarf_Addr low, high;
+
+ // continuous range
+ if (dwarf_hasattr(die, DW_AT_low_pc) && dwarf_hasattr(die, DW_AT_high_pc)) {
+ if (dwarf_lowpc(die, &low) != 0) {
+ return false;
+ }
+ if (dwarf_highpc(die, &high) != 0) {
+ Dwarf_Attribute attr_mem;
+ Dwarf_Attribute *attr = dwarf_attr(die, DW_AT_high_pc, &attr_mem);
+ Dwarf_Word value;
+ if (dwarf_formudata(attr, &value) != 0) {
+ return false;
+ }
+ high = low + value;
+ }
+ return pc >= low && pc < high;
+ }
+
+ // non-continuous range.
+ Dwarf_Addr base;
+ ptrdiff_t offset = 0;
+ while ((offset = dwarf_ranges(die, offset, &base, &low, &high)) > 0) {
+ if (pc >= low && pc < high) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ static Dwarf_Die *find_fundie_by_pc(Dwarf_Die *parent_die, Dwarf_Addr pc,
+ Dwarf_Die *result) {
+ if (dwarf_child(parent_die, result) != 0) {
+ return 0;
+ }
+
+ Dwarf_Die *die = result;
+ do {
+ switch (dwarf_tag(die)) {
+ case DW_TAG_subprogram:
+ case DW_TAG_inlined_subroutine:
+ if (die_has_pc(die, pc)) {
+ return result;
+ }
+ };
+ bool declaration = false;
+ Dwarf_Attribute attr_mem;
+ dwarf_formflag(dwarf_attr(die, DW_AT_declaration, &attr_mem),
+ &declaration);
+ if (!declaration) {
+ // let's be curious and look deeper in the tree,
+ // function are not necessarily at the first level, but
+ // might be nested inside a namespace, structure etc.
+ Dwarf_Die die_mem;
+ Dwarf_Die *indie = find_fundie_by_pc(die, pc, &die_mem);
+ if (indie) {
+ *result = die_mem;
+ return result;
+ }
+ }
+ } while (dwarf_siblingof(die, result) == 0);
+ return 0;
+ }
+
+ template <typename CB>
+ static bool deep_first_search_by_pc(Dwarf_Die *parent_die, Dwarf_Addr pc,
+ CB cb) {
+ Dwarf_Die die_mem;
+ if (dwarf_child(parent_die, &die_mem) != 0) {
+ return false;
+ }
+
+ bool branch_has_pc = false;
+ Dwarf_Die *die = &die_mem;
+ do {
+ bool declaration = false;
+ Dwarf_Attribute attr_mem;
+ dwarf_formflag(dwarf_attr(die, DW_AT_declaration, &attr_mem),
+ &declaration);
+ if (!declaration) {
+ // let's be curious and look deeper in the tree, function are
+ // not necessarily at the first level, but might be nested
+ // inside a namespace, structure, a function, an inlined
+ // function etc.
+ branch_has_pc = deep_first_search_by_pc(die, pc, cb);
+ }
+ if (!branch_has_pc) {
+ branch_has_pc = die_has_pc(die, pc);
+ }
+ if (branch_has_pc) {
+ cb(die);
+ }
+ } while (dwarf_siblingof(die, &die_mem) == 0);
+ return branch_has_pc;
+ }
+
+ static const char *die_call_file(Dwarf_Die *die) {
+ Dwarf_Attribute attr_mem;
+ Dwarf_Word file_idx = 0;
+
+ dwarf_formudata(dwarf_attr(die, DW_AT_call_file, &attr_mem), &file_idx);
+
+ if (file_idx == 0) {
+ return 0;
+ }
+
+ Dwarf_Die die_mem;
+ Dwarf_Die *cudie = dwarf_diecu(die, &die_mem, 0, 0);
+ if (!cudie) {
+ return 0;
+ }
+
+ Dwarf_Files *files = 0;
+ size_t nfiles;
+ dwarf_getsrcfiles(cudie, &files, &nfiles);
+ if (!files) {
+ return 0;
+ }
+
+ return dwarf_filesrc(files, file_idx, 0, 0);
+ }
+};
+#endif // BACKWARD_HAS_DW == 1
+
+#if BACKWARD_HAS_DWARF == 1
+
+template <>
+class TraceResolverLinuxImpl<trace_resolver_tag::libdwarf>
+ : public TraceResolverLinuxBase {
+public:
+ TraceResolverLinuxImpl() : _dwarf_loaded(false) {}
+
+ ResolvedTrace resolve(ResolvedTrace trace) override {
+ // trace.addr is a virtual address in memory pointing to some code.
+ // Let's try to find from which loaded object it comes from.
+ // The loaded object can be yourself btw.
+
+ Dl_info symbol_info;
+ int dladdr_result = 0;
+#if defined(__GLIBC__)
+ link_map *link_map;
+ // We request the link map so we can get information about offsets
+ dladdr_result =
+ dladdr1(trace.addr, &symbol_info, reinterpret_cast<void **>(&link_map),
+ RTLD_DL_LINKMAP);
+#else
+ // Android doesn't have dladdr1. Don't use the linker map.
+ dladdr_result = dladdr(trace.addr, &symbol_info);
+#endif
+ if (!dladdr_result) {
+ return trace; // dat broken trace...
+ }
+
+ // Now we get in symbol_info:
+ // .dli_fname:
+ // pathname of the shared object that contains the address.
+ // .dli_fbase:
+ // where the object is loaded in memory.
+ // .dli_sname:
+ // the name of the nearest symbol to trace.addr, we expect a
+ // function name.
+ // .dli_saddr:
+ // the exact address corresponding to .dli_sname.
+ //
+ // And in link_map:
+ // .l_addr:
+ // difference between the address in the ELF file and the address
+ // in memory
+ // l_name:
+ // absolute pathname where the object was found
+
+ if (symbol_info.dli_sname) {
+ trace.object_function = demangle(symbol_info.dli_sname);
+ }
+
+ if (!symbol_info.dli_fname) {
+ return trace;
+ }
+
+ trace.object_filename = resolve_exec_path(symbol_info);
+ dwarf_fileobject &fobj = load_object_with_dwarf(symbol_info.dli_fname);
+ if (!fobj.dwarf_handle) {
+ return trace; // sad, we couldn't load the object :(
+ }
+
+#if defined(__GLIBC__)
+ // Convert the address to a module relative one by looking at
+ // the module's loading address in the link map
+ Dwarf_Addr address = reinterpret_cast<uintptr_t>(trace.addr) -
+ reinterpret_cast<uintptr_t>(link_map->l_addr);
+#else
+ Dwarf_Addr address = reinterpret_cast<uintptr_t>(trace.addr);
+#endif
+
+ if (trace.object_function.empty()) {
+ symbol_cache_t::iterator it = fobj.symbol_cache.lower_bound(address);
+
+ if (it != fobj.symbol_cache.end()) {
+ if (it->first != address) {
+ if (it != fobj.symbol_cache.begin()) {
+ --it;
+ }
+ }
+ trace.object_function = demangle(it->second.c_str());
+ }
+ }
+
+ // Get the Compilation Unit DIE for the address
+ Dwarf_Die die = find_die(fobj, address);
+
+ if (!die) {
+ return trace; // this time we lost the game :/
+ }
+
+ // libdwarf doesn't give us direct access to its objects, it always
+ // allocates a copy for the caller. We keep that copy alive in a cache
+ // and we deallocate it later when it's no longer required.
+ die_cache_entry &die_object = get_die_cache(fobj, die);
+ if (die_object.isEmpty())
+ return trace; // We have no line section for this DIE
+
+ die_linemap_t::iterator it = die_object.line_section.lower_bound(address);
+
+ if (it != die_object.line_section.end()) {
+ if (it->first != address) {
+ if (it == die_object.line_section.begin()) {
+ // If we are on the first item of the line section
+ // but the address does not match it means that
+ // the address is below the range of the DIE. Give up.
+ return trace;
+ } else {
+ --it;
+ }
+ }
+ } else {
+ return trace; // We didn't find the address.
+ }
+
+ // Get the Dwarf_Line that the address points to and call libdwarf
+ // to get source file, line and column info.
+ Dwarf_Line line = die_object.line_buffer[it->second];
+ Dwarf_Error error = DW_DLE_NE;
+
+ char *filename;
+ if (dwarf_linesrc(line, &filename, &error) == DW_DLV_OK) {
+ trace.source.filename = std::string(filename);
+ dwarf_dealloc(fobj.dwarf_handle.get(), filename, DW_DLA_STRING);
+ }
+
+ Dwarf_Unsigned number = 0;
+ if (dwarf_lineno(line, &number, &error) == DW_DLV_OK) {
+ trace.source.line = number;
+ } else {
+ trace.source.line = 0;
+ }
+
+ if (dwarf_lineoff_b(line, &number, &error) == DW_DLV_OK) {
+ trace.source.col = number;
+ } else {
+ trace.source.col = 0;
+ }
+
+ std::vector<std::string> namespace_stack;
+ deep_first_search_by_pc(fobj, die, address, namespace_stack,
+ inliners_search_cb(trace, fobj, die));
+
+ dwarf_dealloc(fobj.dwarf_handle.get(), die, DW_DLA_DIE);
+
+ return trace;
+ }
+
+public:
+ static int close_dwarf(Dwarf_Debug dwarf) {
+ return dwarf_finish(dwarf, NULL);
+ }
+
+private:
+ bool _dwarf_loaded;
+
+ typedef details::handle<int, details::deleter<int, int, &::close>>
+ dwarf_file_t;
+
+ typedef details::handle<Elf *, details::deleter<int, Elf *, &elf_end>>
+ dwarf_elf_t;
+
+ typedef details::handle<Dwarf_Debug,
+ details::deleter<int, Dwarf_Debug, &close_dwarf>>
+ dwarf_handle_t;
+
+ typedef std::map<Dwarf_Addr, int> die_linemap_t;
+
+ typedef std::map<Dwarf_Off, Dwarf_Off> die_specmap_t;
+
+ struct die_cache_entry {
+ die_specmap_t spec_section;
+ die_linemap_t line_section;
+ Dwarf_Line *line_buffer;
+ Dwarf_Signed line_count;
+ Dwarf_Line_Context line_context;
+
+ inline bool isEmpty() {
+ return line_buffer == NULL || line_count == 0 || line_context == NULL ||
+ line_section.empty();
+ }
+
+ die_cache_entry() : line_buffer(0), line_count(0), line_context(0) {}
+
+ ~die_cache_entry() {
+ if (line_context) {
+ dwarf_srclines_dealloc_b(line_context);
+ }
+ }
+ };
+
+ typedef std::map<Dwarf_Off, die_cache_entry> die_cache_t;
+
+ typedef std::map<uintptr_t, std::string> symbol_cache_t;
+
+ struct dwarf_fileobject {
+ dwarf_file_t file_handle;
+ dwarf_elf_t elf_handle;
+ dwarf_handle_t dwarf_handle;
+ symbol_cache_t symbol_cache;
+
+ // Die cache
+ die_cache_t die_cache;
+ die_cache_entry *current_cu;
+ };
+
+ typedef details::hashtable<std::string, dwarf_fileobject>::type
+ fobj_dwarf_map_t;
+ fobj_dwarf_map_t _fobj_dwarf_map;
+
+ static bool cstrings_eq(const char *a, const char *b) {
+ if (!a || !b) {
+ return false;
+ }
+ return strcmp(a, b) == 0;
+ }
+
+ dwarf_fileobject &load_object_with_dwarf(const std::string &filename_object) {
+
+ if (!_dwarf_loaded) {
+ // Set the ELF library operating version
+ // If that fails there's nothing we can do
+ _dwarf_loaded = elf_version(EV_CURRENT) != EV_NONE;
+ }
+
+ fobj_dwarf_map_t::iterator it = _fobj_dwarf_map.find(filename_object);
+ if (it != _fobj_dwarf_map.end()) {
+ return it->second;
+ }
+
+ // this new object is empty for now
+ dwarf_fileobject &r = _fobj_dwarf_map[filename_object];
+
+ dwarf_file_t file_handle;
+ file_handle.reset(open(filename_object.c_str(), O_RDONLY));
+ if (file_handle.get() < 0) {
+ return r;
+ }
+
+ // Try to get an ELF handle. We need to read the ELF sections
+ // because we want to see if there is a .gnu_debuglink section
+ // that points to a split debug file
+ dwarf_elf_t elf_handle;
+ elf_handle.reset(elf_begin(file_handle.get(), ELF_C_READ, NULL));
+ if (!elf_handle) {
+ return r;
+ }
+
+ const char *e_ident = elf_getident(elf_handle.get(), 0);
+ if (!e_ident) {
+ return r;
+ }
+
+ // Get the number of sections
+ // We use the new APIs as elf_getshnum is deprecated
+ size_t shdrnum = 0;
+ if (elf_getshdrnum(elf_handle.get(), &shdrnum) == -1) {
+ return r;
+ }
+
+ // Get the index to the string section
+ size_t shdrstrndx = 0;
+ if (elf_getshdrstrndx(elf_handle.get(), &shdrstrndx) == -1) {
+ return r;
+ }
+
+ std::string debuglink;
+ // Iterate through the ELF sections to try to get a gnu_debuglink
+ // note and also to cache the symbol table.
+ // We go the preprocessor way to avoid having to create templated
+ // classes or using gelf (which might throw a compiler error if 64 bit
+ // is not supported
+#define ELF_GET_DATA(ARCH) \
+ Elf_Scn *elf_section = 0; \
+ Elf_Data *elf_data = 0; \
+ Elf##ARCH##_Shdr *section_header = 0; \
+ Elf_Scn *symbol_section = 0; \
+ size_t symbol_count = 0; \
+ size_t symbol_strings = 0; \
+ Elf##ARCH##_Sym *symbol = 0; \
+ const char *section_name = 0; \
+ \
+ while ((elf_section = elf_nextscn(elf_handle.get(), elf_section)) != NULL) { \
+ section_header = elf##ARCH##_getshdr(elf_section); \
+ if (section_header == NULL) { \
+ return r; \
+ } \
+ \
+ if ((section_name = elf_strptr(elf_handle.get(), shdrstrndx, \
+ section_header->sh_name)) == NULL) { \
+ return r; \
+ } \
+ \
+ if (cstrings_eq(section_name, ".gnu_debuglink")) { \
+ elf_data = elf_getdata(elf_section, NULL); \
+ if (elf_data && elf_data->d_size > 0) { \
+ debuglink = \
+ std::string(reinterpret_cast<const char *>(elf_data->d_buf)); \
+ } \
+ } \
+ \
+ switch (section_header->sh_type) { \
+ case SHT_SYMTAB: \
+ symbol_section = elf_section; \
+ symbol_count = section_header->sh_size / section_header->sh_entsize; \
+ symbol_strings = section_header->sh_link; \
+ break; \
+ \
+ /* We use .dynsyms as a last resort, we prefer .symtab */ \
+ case SHT_DYNSYM: \
+ if (!symbol_section) { \
+ symbol_section = elf_section; \
+ symbol_count = section_header->sh_size / section_header->sh_entsize; \
+ symbol_strings = section_header->sh_link; \
+ } \
+ break; \
+ } \
+ } \
+ \
+ if (symbol_section && symbol_count && symbol_strings) { \
+ elf_data = elf_getdata(symbol_section, NULL); \
+ symbol = reinterpret_cast<Elf##ARCH##_Sym *>(elf_data->d_buf); \
+ for (size_t i = 0; i < symbol_count; ++i) { \
+ int type = ELF##ARCH##_ST_TYPE(symbol->st_info); \
+ if (type == STT_FUNC && symbol->st_value > 0) { \
+ r.symbol_cache[symbol->st_value] = std::string( \
+ elf_strptr(elf_handle.get(), symbol_strings, symbol->st_name)); \
+ } \
+ ++symbol; \
+ } \
+ }
+
+ if (e_ident[EI_CLASS] == ELFCLASS32) {
+ ELF_GET_DATA(32)
+ } else if (e_ident[EI_CLASS] == ELFCLASS64) {
+ // libelf might have been built without 64 bit support
+#if __LIBELF64
+ ELF_GET_DATA(64)
+#endif
+ }
+
+ if (!debuglink.empty()) {
+ // We have a debuglink section! Open an elf instance on that
+ // file instead. If we can't open the file, then return
+ // the elf handle we had already opened.
+ dwarf_file_t debuglink_file;
+ debuglink_file.reset(open(debuglink.c_str(), O_RDONLY));
+ if (debuglink_file.get() > 0) {
+ dwarf_elf_t debuglink_elf;
+ debuglink_elf.reset(elf_begin(debuglink_file.get(), ELF_C_READ, NULL));
+
+ // If we have a valid elf handle, return the new elf handle
+ // and file handle and discard the original ones
+ if (debuglink_elf) {
+ elf_handle = move(debuglink_elf);
+ file_handle = move(debuglink_file);
+ }
+ }
+ }
+
+ // Ok, we have a valid ELF handle, let's try to get debug symbols
+ Dwarf_Debug dwarf_debug;
+ Dwarf_Error error = DW_DLE_NE;
+ dwarf_handle_t dwarf_handle;
+
+ int dwarf_result = dwarf_elf_init(elf_handle.get(), DW_DLC_READ, NULL, NULL,
+ &dwarf_debug, &error);
+
+ // We don't do any special handling for DW_DLV_NO_ENTRY specially.
+ // If we get an error, or the file doesn't have debug information
+ // we just return.
+ if (dwarf_result != DW_DLV_OK) {
+ return r;
+ }
+
+ dwarf_handle.reset(dwarf_debug);
+
+ r.file_handle = move(file_handle);
+ r.elf_handle = move(elf_handle);
+ r.dwarf_handle = move(dwarf_handle);
+
+ return r;
+ }
+
+ die_cache_entry &get_die_cache(dwarf_fileobject &fobj, Dwarf_Die die) {
+ Dwarf_Error error = DW_DLE_NE;
+
+ // Get the die offset, we use it as the cache key
+ Dwarf_Off die_offset;
+ if (dwarf_dieoffset(die, &die_offset, &error) != DW_DLV_OK) {
+ die_offset = 0;
+ }
+
+ die_cache_t::iterator it = fobj.die_cache.find(die_offset);
+
+ if (it != fobj.die_cache.end()) {
+ fobj.current_cu = &it->second;
+ return it->second;
+ }
+
+ die_cache_entry &de = fobj.die_cache[die_offset];
+ fobj.current_cu = &de;
+
+ Dwarf_Addr line_addr;
+ Dwarf_Small table_count;
+
+ // The addresses in the line section are not fully sorted (they might
+ // be sorted by block of code belonging to the same file), which makes
+ // it necessary to do so before searching is possible.
+ //
+ // As libdwarf allocates a copy of everything, let's get the contents
+ // of the line section and keep it around. We also create a map of
+ // program counter to line table indices so we can search by address
+ // and get the line buffer index.
+ //
+ // To make things more difficult, the same address can span more than
+ // one line, so we need to keep the index pointing to the first line
+ // by using insert instead of the map's [ operator.
+
+ // Get the line context for the DIE
+ if (dwarf_srclines_b(die, 0, &table_count, &de.line_context, &error) ==
+ DW_DLV_OK) {
+ // Get the source lines for this line context, to be deallocated
+ // later
+ if (dwarf_srclines_from_linecontext(de.line_context, &de.line_buffer,
+ &de.line_count,
+ &error) == DW_DLV_OK) {
+
+ // Add all the addresses to our map
+ for (int i = 0; i < de.line_count; i++) {
+ if (dwarf_lineaddr(de.line_buffer[i], &line_addr, &error) !=
+ DW_DLV_OK) {
+ line_addr = 0;
+ }
+ de.line_section.insert(std::pair<Dwarf_Addr, int>(line_addr, i));
+ }
+ }
+ }
+
+ // For each CU, cache the function DIEs that contain the
+ // DW_AT_specification attribute. When building with -g3 the function
+ // DIEs are separated in declaration and specification, with the
+ // declaration containing only the name and parameters and the
+ // specification the low/high pc and other compiler attributes.
+ //
+ // We cache those specifications so we don't skip over the declarations,
+ // because they have no pc, and we can do namespace resolution for
+ // DWARF function names.
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ Dwarf_Die current_die = 0;
+ if (dwarf_child(die, &current_die, &error) == DW_DLV_OK) {
+ for (;;) {
+ Dwarf_Die sibling_die = 0;
+
+ Dwarf_Half tag_value;
+ dwarf_tag(current_die, &tag_value, &error);
+
+ if (tag_value == DW_TAG_subprogram ||
+ tag_value == DW_TAG_inlined_subroutine) {
+
+ Dwarf_Bool has_attr = 0;
+ if (dwarf_hasattr(current_die, DW_AT_specification, &has_attr,
+ &error) == DW_DLV_OK) {
+ if (has_attr) {
+ Dwarf_Attribute attr_mem;
+ if (dwarf_attr(current_die, DW_AT_specification, &attr_mem,
+ &error) == DW_DLV_OK) {
+ Dwarf_Off spec_offset = 0;
+ if (dwarf_formref(attr_mem, &spec_offset, &error) ==
+ DW_DLV_OK) {
+ Dwarf_Off spec_die_offset;
+ if (dwarf_dieoffset(current_die, &spec_die_offset, &error) ==
+ DW_DLV_OK) {
+ de.spec_section[spec_offset] = spec_die_offset;
+ }
+ }
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+ }
+ }
+
+ int result = dwarf_siblingof(dwarf, current_die, &sibling_die, &error);
+ if (result == DW_DLV_ERROR) {
+ break;
+ } else if (result == DW_DLV_NO_ENTRY) {
+ break;
+ }
+
+ if (current_die != die) {
+ dwarf_dealloc(dwarf, current_die, DW_DLA_DIE);
+ current_die = 0;
+ }
+
+ current_die = sibling_die;
+ }
+ }
+ return de;
+ }
+
+ static Dwarf_Die get_referenced_die(Dwarf_Debug dwarf, Dwarf_Die die,
+ Dwarf_Half attr, bool global) {
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Attribute attr_mem;
+
+ Dwarf_Die found_die = NULL;
+ if (dwarf_attr(die, attr, &attr_mem, &error) == DW_DLV_OK) {
+ Dwarf_Off offset;
+ int result = 0;
+ if (global) {
+ result = dwarf_global_formref(attr_mem, &offset, &error);
+ } else {
+ result = dwarf_formref(attr_mem, &offset, &error);
+ }
+
+ if (result == DW_DLV_OK) {
+ if (dwarf_offdie(dwarf, offset, &found_die, &error) != DW_DLV_OK) {
+ found_die = NULL;
+ }
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+ return found_die;
+ }
+
+ static std::string get_referenced_die_name(Dwarf_Debug dwarf, Dwarf_Die die,
+ Dwarf_Half attr, bool global) {
+ Dwarf_Error error = DW_DLE_NE;
+ std::string value;
+
+ Dwarf_Die found_die = get_referenced_die(dwarf, die, attr, global);
+
+ if (found_die) {
+ char *name;
+ if (dwarf_diename(found_die, &name, &error) == DW_DLV_OK) {
+ if (name) {
+ value = std::string(name);
+ }
+ dwarf_dealloc(dwarf, name, DW_DLA_STRING);
+ }
+ dwarf_dealloc(dwarf, found_die, DW_DLA_DIE);
+ }
+
+ return value;
+ }
+
+ // Returns a spec DIE linked to the passed one. The caller should
+ // deallocate the DIE
+ static Dwarf_Die get_spec_die(dwarf_fileobject &fobj, Dwarf_Die die) {
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Off die_offset;
+ if (fobj.current_cu &&
+ dwarf_die_CU_offset(die, &die_offset, &error) == DW_DLV_OK) {
+ die_specmap_t::iterator it =
+ fobj.current_cu->spec_section.find(die_offset);
+
+ // If we have a DIE that completes the current one, check if
+ // that one has the pc we are looking for
+ if (it != fobj.current_cu->spec_section.end()) {
+ Dwarf_Die spec_die = 0;
+ if (dwarf_offdie(dwarf, it->second, &spec_die, &error) == DW_DLV_OK) {
+ return spec_die;
+ }
+ }
+ }
+
+ // Maybe we have an abstract origin DIE with the function information?
+ return get_referenced_die(fobj.dwarf_handle.get(), die,
+ DW_AT_abstract_origin, true);
+ }
+
+ static bool die_has_pc(dwarf_fileobject &fobj, Dwarf_Die die, Dwarf_Addr pc) {
+ Dwarf_Addr low_pc = 0, high_pc = 0;
+ Dwarf_Half high_pc_form = 0;
+ Dwarf_Form_Class return_class;
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ bool has_lowpc = false;
+ bool has_highpc = false;
+ bool has_ranges = false;
+
+ if (dwarf_lowpc(die, &low_pc, &error) == DW_DLV_OK) {
+ // If we have a low_pc check if there is a high pc.
+ // If we don't have a high pc this might mean we have a base
+ // address for the ranges list or just an address.
+ has_lowpc = true;
+
+ if (dwarf_highpc_b(die, &high_pc, &high_pc_form, &return_class, &error) ==
+ DW_DLV_OK) {
+ // We do have a high pc. In DWARF 4+ this is an offset from the
+ // low pc, but in earlier versions it's an absolute address.
+
+ has_highpc = true;
+ // In DWARF 2/3 this would be a DW_FORM_CLASS_ADDRESS
+ if (return_class == DW_FORM_CLASS_CONSTANT) {
+ high_pc = low_pc + high_pc;
+ }
+
+ // We have low and high pc, check if our address
+ // is in that range
+ return pc >= low_pc && pc < high_pc;
+ }
+ } else {
+ // Reset the low_pc, in case dwarf_lowpc failing set it to some
+ // undefined value.
+ low_pc = 0;
+ }
+
+ // Check if DW_AT_ranges is present and search for the PC in the
+ // returned ranges list. We always add the low_pc, as it not set it will
+ // be 0, in case we had a DW_AT_low_pc and DW_AT_ranges pair
+ bool result = false;
+
+ Dwarf_Attribute attr;
+ if (dwarf_attr(die, DW_AT_ranges, &attr, &error) == DW_DLV_OK) {
+
+ Dwarf_Off offset;
+ if (dwarf_global_formref(attr, &offset, &error) == DW_DLV_OK) {
+ Dwarf_Ranges *ranges;
+ Dwarf_Signed ranges_count = 0;
+ Dwarf_Unsigned byte_count = 0;
+
+ if (dwarf_get_ranges_a(dwarf, offset, die, &ranges, &ranges_count,
+ &byte_count, &error) == DW_DLV_OK) {
+ has_ranges = ranges_count != 0;
+ for (int i = 0; i < ranges_count; i++) {
+ if (ranges[i].dwr_addr1 != 0 &&
+ pc >= ranges[i].dwr_addr1 + low_pc &&
+ pc < ranges[i].dwr_addr2 + low_pc) {
+ result = true;
+ break;
+ }
+ }
+ dwarf_ranges_dealloc(dwarf, ranges, ranges_count);
+ }
+ }
+ }
+
+ // Last attempt. We might have a single address set as low_pc.
+ if (!result && low_pc != 0 && pc == low_pc) {
+ result = true;
+ }
+
+ // If we don't have lowpc, highpc and ranges maybe this DIE is a
+ // declaration that relies on a DW_AT_specification DIE that happens
+ // later. Use the specification cache we filled when we loaded this CU.
+ if (!result && (!has_lowpc && !has_highpc && !has_ranges)) {
+ Dwarf_Die spec_die = get_spec_die(fobj, die);
+ if (spec_die) {
+ result = die_has_pc(fobj, spec_die, pc);
+ dwarf_dealloc(dwarf, spec_die, DW_DLA_DIE);
+ }
+ }
+
+ return result;
+ }
+
+ static void get_type(Dwarf_Debug dwarf, Dwarf_Die die, std::string &type) {
+ Dwarf_Error error = DW_DLE_NE;
+
+ Dwarf_Die child = 0;
+ if (dwarf_child(die, &child, &error) == DW_DLV_OK) {
+ get_type(dwarf, child, type);
+ }
+
+ if (child) {
+ type.insert(0, "::");
+ dwarf_dealloc(dwarf, child, DW_DLA_DIE);
+ }
+
+ char *name;
+ if (dwarf_diename(die, &name, &error) == DW_DLV_OK) {
+ type.insert(0, std::string(name));
+ dwarf_dealloc(dwarf, name, DW_DLA_STRING);
+ } else {
+ type.insert(0, "<unknown>");
+ }
+ }
+
+ static std::string get_type_by_signature(Dwarf_Debug dwarf, Dwarf_Die die) {
+ Dwarf_Error error = DW_DLE_NE;
+
+ Dwarf_Sig8 signature;
+ Dwarf_Bool has_attr = 0;
+ if (dwarf_hasattr(die, DW_AT_signature, &has_attr, &error) == DW_DLV_OK) {
+ if (has_attr) {
+ Dwarf_Attribute attr_mem;
+ if (dwarf_attr(die, DW_AT_signature, &attr_mem, &error) == DW_DLV_OK) {
+ if (dwarf_formsig8(attr_mem, &signature, &error) != DW_DLV_OK) {
+ return std::string("<no type signature>");
+ }
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+ }
+
+ Dwarf_Unsigned next_cu_header;
+ Dwarf_Sig8 tu_signature;
+ std::string result;
+ bool found = false;
+
+ while (dwarf_next_cu_header_d(dwarf, 0, 0, 0, 0, 0, 0, 0, &tu_signature, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+
+ if (strncmp(signature.signature, tu_signature.signature, 8) == 0) {
+ Dwarf_Die type_cu_die = 0;
+ if (dwarf_siblingof_b(dwarf, 0, 0, &type_cu_die, &error) == DW_DLV_OK) {
+ Dwarf_Die child_die = 0;
+ if (dwarf_child(type_cu_die, &child_die, &error) == DW_DLV_OK) {
+ get_type(dwarf, child_die, result);
+ found = !result.empty();
+ dwarf_dealloc(dwarf, child_die, DW_DLA_DIE);
+ }
+ dwarf_dealloc(dwarf, type_cu_die, DW_DLA_DIE);
+ }
+ }
+ }
+
+ if (found) {
+ while (dwarf_next_cu_header_d(dwarf, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+ // Reset the cu header state. Unfortunately, libdwarf's
+ // next_cu_header API keeps its own iterator per Dwarf_Debug
+ // that can't be reset. We need to keep fetching elements until
+ // the end.
+ }
+ } else {
+ // If we couldn't resolve the type just print out the signature
+ std::ostringstream string_stream;
+ string_stream << "<0x" << std::hex << std::setfill('0');
+ for (int i = 0; i < 8; ++i) {
+ string_stream << std::setw(2) << std::hex
+ << (int)(unsigned char)(signature.signature[i]);
+ }
+ string_stream << ">";
+ result = string_stream.str();
+ }
+ return result;
+ }
+
+ struct type_context_t {
+ bool is_const;
+ bool is_typedef;
+ bool has_type;
+ bool has_name;
+ std::string text;
+
+ type_context_t()
+ : is_const(false), is_typedef(false), has_type(false), has_name(false) {
+ }
+ };
+
+ // Types are resolved from right to left: we get the variable name first
+ // and then all specifiers (like const or pointer) in a chain of DW_AT_type
+ // DIEs. Call this function recursively until we get a complete type
+ // string.
+ static void set_parameter_string(dwarf_fileobject &fobj, Dwarf_Die die,
+ type_context_t &context) {
+ char *name;
+ Dwarf_Error error = DW_DLE_NE;
+
+ // typedefs contain also the base type, so we skip it and only
+ // print the typedef name
+ if (!context.is_typedef) {
+ if (dwarf_diename(die, &name, &error) == DW_DLV_OK) {
+ if (!context.text.empty()) {
+ context.text.insert(0, " ");
+ }
+ context.text.insert(0, std::string(name));
+ dwarf_dealloc(fobj.dwarf_handle.get(), name, DW_DLA_STRING);
+ }
+ } else {
+ context.is_typedef = false;
+ context.has_type = true;
+ if (context.is_const) {
+ context.text.insert(0, "const ");
+ context.is_const = false;
+ }
+ }
+
+ bool next_type_is_const = false;
+ bool is_keyword = true;
+
+ Dwarf_Half tag = 0;
+ Dwarf_Bool has_attr = 0;
+ if (dwarf_tag(die, &tag, &error) == DW_DLV_OK) {
+ switch (tag) {
+ case DW_TAG_structure_type:
+ case DW_TAG_union_type:
+ case DW_TAG_class_type:
+ case DW_TAG_enumeration_type:
+ context.has_type = true;
+ if (dwarf_hasattr(die, DW_AT_signature, &has_attr, &error) ==
+ DW_DLV_OK) {
+ // If we have a signature it means the type is defined
+ // in .debug_types, so we need to load the DIE pointed
+ // at by the signature and resolve it
+ if (has_attr) {
+ std::string type =
+ get_type_by_signature(fobj.dwarf_handle.get(), die);
+ if (context.is_const)
+ type.insert(0, "const ");
+
+ if (!context.text.empty())
+ context.text.insert(0, " ");
+ context.text.insert(0, type);
+ }
+
+ // Treat enums like typedefs, and skip printing its
+ // base type
+ context.is_typedef = (tag == DW_TAG_enumeration_type);
+ }
+ break;
+ case DW_TAG_const_type:
+ next_type_is_const = true;
+ break;
+ case DW_TAG_pointer_type:
+ context.text.insert(0, "*");
+ break;
+ case DW_TAG_reference_type:
+ context.text.insert(0, "&");
+ break;
+ case DW_TAG_restrict_type:
+ context.text.insert(0, "restrict ");
+ break;
+ case DW_TAG_rvalue_reference_type:
+ context.text.insert(0, "&&");
+ break;
+ case DW_TAG_volatile_type:
+ context.text.insert(0, "volatile ");
+ break;
+ case DW_TAG_typedef:
+ // Propagate the const-ness to the next type
+ // as typedefs are linked to its base type
+ next_type_is_const = context.is_const;
+ context.is_typedef = true;
+ context.has_type = true;
+ break;
+ case DW_TAG_base_type:
+ context.has_type = true;
+ break;
+ case DW_TAG_formal_parameter:
+ context.has_name = true;
+ break;
+ default:
+ is_keyword = false;
+ break;
+ }
+ }
+
+ if (!is_keyword && context.is_const) {
+ context.text.insert(0, "const ");
+ }
+
+ context.is_const = next_type_is_const;
+
+ Dwarf_Die ref =
+ get_referenced_die(fobj.dwarf_handle.get(), die, DW_AT_type, true);
+ if (ref) {
+ set_parameter_string(fobj, ref, context);
+ dwarf_dealloc(fobj.dwarf_handle.get(), ref, DW_DLA_DIE);
+ }
+
+ if (!context.has_type && context.has_name) {
+ context.text.insert(0, "void ");
+ context.has_type = true;
+ }
+ }
+
+ // Resolve the function return type and parameters
+ static void set_function_parameters(std::string &function_name,
+ std::vector<std::string> &ns,
+ dwarf_fileobject &fobj, Dwarf_Die die) {
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Die current_die = 0;
+ std::string parameters;
+ bool has_spec = true;
+ // Check if we have a spec DIE. If we do we use it as it contains
+ // more information, like parameter names.
+ Dwarf_Die spec_die = get_spec_die(fobj, die);
+ if (!spec_die) {
+ has_spec = false;
+ spec_die = die;
+ }
+
+ std::vector<std::string>::const_iterator it = ns.begin();
+ std::string ns_name;
+ for (it = ns.begin(); it < ns.end(); ++it) {
+ ns_name.append(*it).append("::");
+ }
+
+ if (!ns_name.empty()) {
+ function_name.insert(0, ns_name);
+ }
+
+ // See if we have a function return type. It can be either on the
+ // current die or in its spec one (usually true for inlined functions)
+ std::string return_type =
+ get_referenced_die_name(dwarf, die, DW_AT_type, true);
+ if (return_type.empty()) {
+ return_type = get_referenced_die_name(dwarf, spec_die, DW_AT_type, true);
+ }
+ if (!return_type.empty()) {
+ return_type.append(" ");
+ function_name.insert(0, return_type);
+ }
+
+ if (dwarf_child(spec_die, &current_die, &error) == DW_DLV_OK) {
+ for (;;) {
+ Dwarf_Die sibling_die = 0;
+
+ Dwarf_Half tag_value;
+ dwarf_tag(current_die, &tag_value, &error);
+
+ if (tag_value == DW_TAG_formal_parameter) {
+ // Ignore artificial (ie, compiler generated) parameters
+ bool is_artificial = false;
+ Dwarf_Attribute attr_mem;
+ if (dwarf_attr(current_die, DW_AT_artificial, &attr_mem, &error) ==
+ DW_DLV_OK) {
+ Dwarf_Bool flag = 0;
+ if (dwarf_formflag(attr_mem, &flag, &error) == DW_DLV_OK) {
+ is_artificial = flag != 0;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+
+ if (!is_artificial) {
+ type_context_t context;
+ set_parameter_string(fobj, current_die, context);
+
+ if (parameters.empty()) {
+ parameters.append("(");
+ } else {
+ parameters.append(", ");
+ }
+ parameters.append(context.text);
+ }
+ }
+
+ int result = dwarf_siblingof(dwarf, current_die, &sibling_die, &error);
+ if (result == DW_DLV_ERROR) {
+ break;
+ } else if (result == DW_DLV_NO_ENTRY) {
+ break;
+ }
+
+ if (current_die != die) {
+ dwarf_dealloc(dwarf, current_die, DW_DLA_DIE);
+ current_die = 0;
+ }
+
+ current_die = sibling_die;
+ }
+ }
+ if (parameters.empty())
+ parameters = "(";
+ parameters.append(")");
+
+ // If we got a spec DIE we need to deallocate it
+ if (has_spec)
+ dwarf_dealloc(dwarf, spec_die, DW_DLA_DIE);
+
+ function_name.append(parameters);
+ }
+
+ // defined here because in C++98, template function cannot take locally
+ // defined types... grrr.
+ struct inliners_search_cb {
+ void operator()(Dwarf_Die die, std::vector<std::string> &ns) {
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Half tag_value;
+ Dwarf_Attribute attr_mem;
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+
+ dwarf_tag(die, &tag_value, &error);
+
+ switch (tag_value) {
+ char *name;
+ case DW_TAG_subprogram:
+ if (!trace.source.function.empty())
+ break;
+ if (dwarf_diename(die, &name, &error) == DW_DLV_OK) {
+ trace.source.function = std::string(name);
+ dwarf_dealloc(dwarf, name, DW_DLA_STRING);
+ } else {
+ // We don't have a function name in this DIE.
+ // Check if there is a referenced non-defining
+ // declaration.
+ trace.source.function =
+ get_referenced_die_name(dwarf, die, DW_AT_abstract_origin, true);
+ if (trace.source.function.empty()) {
+ trace.source.function =
+ get_referenced_die_name(dwarf, die, DW_AT_specification, true);
+ }
+ }
+
+ // Append the function parameters, if available
+ set_function_parameters(trace.source.function, ns, fobj, die);
+
+ // If the object function name is empty, it's possible that
+ // there is no dynamic symbol table (maybe the executable
+ // was stripped or not built with -rdynamic). See if we have
+ // a DWARF linkage name to use instead. We try both
+ // linkage_name and MIPS_linkage_name because the MIPS tag
+ // was the unofficial one until it was adopted in DWARF4.
+ // Old gcc versions generate MIPS_linkage_name
+ if (trace.object_function.empty()) {
+ details::demangler demangler;
+
+ if (dwarf_attr(die, DW_AT_linkage_name, &attr_mem, &error) !=
+ DW_DLV_OK) {
+ if (dwarf_attr(die, DW_AT_MIPS_linkage_name, &attr_mem, &error) !=
+ DW_DLV_OK) {
+ break;
+ }
+ }
+
+ char *linkage;
+ if (dwarf_formstring(attr_mem, &linkage, &error) == DW_DLV_OK) {
+ trace.object_function = demangler.demangle(linkage);
+ dwarf_dealloc(dwarf, linkage, DW_DLA_STRING);
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+ break;
+
+ case DW_TAG_inlined_subroutine:
+ ResolvedTrace::SourceLoc sloc;
+
+ if (dwarf_diename(die, &name, &error) == DW_DLV_OK) {
+ sloc.function = std::string(name);
+ dwarf_dealloc(dwarf, name, DW_DLA_STRING);
+ } else {
+ // We don't have a name for this inlined DIE, it could
+ // be that there is an abstract origin instead.
+ // Get the DW_AT_abstract_origin value, which is a
+ // reference to the source DIE and try to get its name
+ sloc.function =
+ get_referenced_die_name(dwarf, die, DW_AT_abstract_origin, true);
+ }
+
+ set_function_parameters(sloc.function, ns, fobj, die);
+
+ std::string file = die_call_file(dwarf, die, cu_die);
+ if (!file.empty())
+ sloc.filename = file;
+
+ Dwarf_Unsigned number = 0;
+ if (dwarf_attr(die, DW_AT_call_line, &attr_mem, &error) == DW_DLV_OK) {
+ if (dwarf_formudata(attr_mem, &number, &error) == DW_DLV_OK) {
+ sloc.line = number;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+
+ if (dwarf_attr(die, DW_AT_call_column, &attr_mem, &error) ==
+ DW_DLV_OK) {
+ if (dwarf_formudata(attr_mem, &number, &error) == DW_DLV_OK) {
+ sloc.col = number;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+
+ trace.inliners.push_back(sloc);
+ break;
+ };
+ }
+ ResolvedTrace &trace;
+ dwarf_fileobject &fobj;
+ Dwarf_Die cu_die;
+ inliners_search_cb(ResolvedTrace &t, dwarf_fileobject &f, Dwarf_Die c)
+ : trace(t), fobj(f), cu_die(c) {}
+ };
+
+ static Dwarf_Die find_fundie_by_pc(dwarf_fileobject &fobj,
+ Dwarf_Die parent_die, Dwarf_Addr pc,
+ Dwarf_Die result) {
+ Dwarf_Die current_die = 0;
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+
+ if (dwarf_child(parent_die, &current_die, &error) != DW_DLV_OK) {
+ return NULL;
+ }
+
+ for (;;) {
+ Dwarf_Die sibling_die = 0;
+ Dwarf_Half tag_value;
+ dwarf_tag(current_die, &tag_value, &error);
+
+ switch (tag_value) {
+ case DW_TAG_subprogram:
+ case DW_TAG_inlined_subroutine:
+ if (die_has_pc(fobj, current_die, pc)) {
+ return current_die;
+ }
+ };
+ bool declaration = false;
+ Dwarf_Attribute attr_mem;
+ if (dwarf_attr(current_die, DW_AT_declaration, &attr_mem, &error) ==
+ DW_DLV_OK) {
+ Dwarf_Bool flag = 0;
+ if (dwarf_formflag(attr_mem, &flag, &error) == DW_DLV_OK) {
+ declaration = flag != 0;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+
+ if (!declaration) {
+ // let's be curious and look deeper in the tree, functions are
+ // not necessarily at the first level, but might be nested
+ // inside a namespace, structure, a function, an inlined
+ // function etc.
+ Dwarf_Die die_mem = 0;
+ Dwarf_Die indie = find_fundie_by_pc(fobj, current_die, pc, die_mem);
+ if (indie) {
+ result = die_mem;
+ return result;
+ }
+ }
+
+ int res = dwarf_siblingof(dwarf, current_die, &sibling_die, &error);
+ if (res == DW_DLV_ERROR) {
+ return NULL;
+ } else if (res == DW_DLV_NO_ENTRY) {
+ break;
+ }
+
+ if (current_die != parent_die) {
+ dwarf_dealloc(dwarf, current_die, DW_DLA_DIE);
+ current_die = 0;
+ }
+
+ current_die = sibling_die;
+ }
+ return NULL;
+ }
+
+ template <typename CB>
+ static bool deep_first_search_by_pc(dwarf_fileobject &fobj,
+ Dwarf_Die parent_die, Dwarf_Addr pc,
+ std::vector<std::string> &ns, CB cb) {
+ Dwarf_Die current_die = 0;
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ Dwarf_Error error = DW_DLE_NE;
+
+ if (dwarf_child(parent_die, &current_die, &error) != DW_DLV_OK) {
+ return false;
+ }
+
+ bool branch_has_pc = false;
+ bool has_namespace = false;
+ for (;;) {
+ Dwarf_Die sibling_die = 0;
+
+ Dwarf_Half tag;
+ if (dwarf_tag(current_die, &tag, &error) == DW_DLV_OK) {
+ if (tag == DW_TAG_namespace || tag == DW_TAG_class_type) {
+ char *ns_name = NULL;
+ if (dwarf_diename(current_die, &ns_name, &error) == DW_DLV_OK) {
+ if (ns_name) {
+ ns.push_back(std::string(ns_name));
+ } else {
+ ns.push_back("<unknown>");
+ }
+ dwarf_dealloc(dwarf, ns_name, DW_DLA_STRING);
+ } else {
+ ns.push_back("<unknown>");
+ }
+ has_namespace = true;
+ }
+ }
+
+ bool declaration = false;
+ Dwarf_Attribute attr_mem;
+ if (tag != DW_TAG_class_type &&
+ dwarf_attr(current_die, DW_AT_declaration, &attr_mem, &error) ==
+ DW_DLV_OK) {
+ Dwarf_Bool flag = 0;
+ if (dwarf_formflag(attr_mem, &flag, &error) == DW_DLV_OK) {
+ declaration = flag != 0;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+ }
+
+ if (!declaration) {
+ // let's be curious and look deeper in the tree, function are
+ // not necessarily at the first level, but might be nested
+ // inside a namespace, structure, a function, an inlined
+ // function etc.
+ branch_has_pc = deep_first_search_by_pc(fobj, current_die, pc, ns, cb);
+ }
+
+ if (!branch_has_pc) {
+ branch_has_pc = die_has_pc(fobj, current_die, pc);
+ }
+
+ if (branch_has_pc) {
+ cb(current_die, ns);
+ }
+
+ int result = dwarf_siblingof(dwarf, current_die, &sibling_die, &error);
+ if (result == DW_DLV_ERROR) {
+ return false;
+ } else if (result == DW_DLV_NO_ENTRY) {
+ break;
+ }
+
+ if (current_die != parent_die) {
+ dwarf_dealloc(dwarf, current_die, DW_DLA_DIE);
+ current_die = 0;
+ }
+
+ if (has_namespace) {
+ has_namespace = false;
+ ns.pop_back();
+ }
+ current_die = sibling_die;
+ }
+
+ if (has_namespace) {
+ ns.pop_back();
+ }
+ return branch_has_pc;
+ }
+
+ static std::string die_call_file(Dwarf_Debug dwarf, Dwarf_Die die,
+ Dwarf_Die cu_die) {
+ Dwarf_Attribute attr_mem;
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Unsigned file_index;
+
+ std::string file;
+
+ if (dwarf_attr(die, DW_AT_call_file, &attr_mem, &error) == DW_DLV_OK) {
+ if (dwarf_formudata(attr_mem, &file_index, &error) != DW_DLV_OK) {
+ file_index = 0;
+ }
+ dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR);
+
+ if (file_index == 0) {
+ return file;
+ }
+
+ char **srcfiles = 0;
+ Dwarf_Signed file_count = 0;
+ if (dwarf_srcfiles(cu_die, &srcfiles, &file_count, &error) == DW_DLV_OK) {
+ if (file_count > 0 && file_index <= static_cast<Dwarf_Unsigned>(file_count)) {
+ file = std::string(srcfiles[file_index - 1]);
+ }
+
+ // Deallocate all strings!
+ for (int i = 0; i < file_count; ++i) {
+ dwarf_dealloc(dwarf, srcfiles[i], DW_DLA_STRING);
+ }
+ dwarf_dealloc(dwarf, srcfiles, DW_DLA_LIST);
+ }
+ }
+ return file;
+ }
+
+ Dwarf_Die find_die(dwarf_fileobject &fobj, Dwarf_Addr addr) {
+ // Let's get to work! First see if we have a debug_aranges section so
+ // we can speed up the search
+
+ Dwarf_Debug dwarf = fobj.dwarf_handle.get();
+ Dwarf_Error error = DW_DLE_NE;
+ Dwarf_Arange *aranges;
+ Dwarf_Signed arange_count;
+
+ Dwarf_Die returnDie;
+ bool found = false;
+ if (dwarf_get_aranges(dwarf, &aranges, &arange_count, &error) !=
+ DW_DLV_OK) {
+ aranges = NULL;
+ }
+
+ if (aranges) {
+ // We have aranges. Get the one where our address is.
+ Dwarf_Arange arange;
+ if (dwarf_get_arange(aranges, arange_count, addr, &arange, &error) ==
+ DW_DLV_OK) {
+
+ // We found our address. Get the compilation-unit DIE offset
+ // represented by the given address range.
+ Dwarf_Off cu_die_offset;
+ if (dwarf_get_cu_die_offset(arange, &cu_die_offset, &error) ==
+ DW_DLV_OK) {
+ // Get the DIE at the offset returned by the aranges search.
+ // We set is_info to 1 to specify that the offset is from
+ // the .debug_info section (and not .debug_types)
+ int dwarf_result =
+ dwarf_offdie_b(dwarf, cu_die_offset, 1, &returnDie, &error);
+
+ found = dwarf_result == DW_DLV_OK;
+ }
+ dwarf_dealloc(dwarf, arange, DW_DLA_ARANGE);
+ }
+ }
+
+ if (found)
+ return returnDie; // The caller is responsible for freeing the die
+
+ // The search for aranges failed. Try to find our address by scanning
+ // all compilation units.
+ Dwarf_Unsigned next_cu_header;
+ Dwarf_Half tag = 0;
+ returnDie = 0;
+
+ while (!found &&
+ dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+
+ if (returnDie)
+ dwarf_dealloc(dwarf, returnDie, DW_DLA_DIE);
+
+ if (dwarf_siblingof(dwarf, 0, &returnDie, &error) == DW_DLV_OK) {
+ if ((dwarf_tag(returnDie, &tag, &error) == DW_DLV_OK) &&
+ tag == DW_TAG_compile_unit) {
+ if (die_has_pc(fobj, returnDie, addr)) {
+ found = true;
+ }
+ }
+ }
+ }
+
+ if (found) {
+ while (dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+ // Reset the cu header state. Libdwarf's next_cu_header API
+ // keeps its own iterator per Dwarf_Debug that can't be reset.
+ // We need to keep fetching elements until the end.
+ }
+ }
+
+ if (found)
+ return returnDie;
+
+ // We couldn't find any compilation units with ranges or a high/low pc.
+ // Try again by looking at all DIEs in all compilation units.
+ Dwarf_Die cudie;
+ while (dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+ if (dwarf_siblingof(dwarf, 0, &cudie, &error) == DW_DLV_OK) {
+ Dwarf_Die die_mem = 0;
+ Dwarf_Die resultDie = find_fundie_by_pc(fobj, cudie, addr, die_mem);
+
+ if (resultDie) {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (found) {
+ while (dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0,
+ &next_cu_header, 0, &error) == DW_DLV_OK) {
+ // Reset the cu header state. Libdwarf's next_cu_header API
+ // keeps its own iterator per Dwarf_Debug that can't be reset.
+ // We need to keep fetching elements until the end.
+ }
+ }
+
+ if (found)
+ return cudie;
+
+ // We failed.
+ return NULL;
+ }
+};
+#endif // BACKWARD_HAS_DWARF == 1
+
+template <>
+class TraceResolverImpl<system_tag::linux_tag>
+ : public TraceResolverLinuxImpl<trace_resolver_tag::current> {};
+
+#endif // BACKWARD_SYSTEM_LINUX
+
+#ifdef BACKWARD_SYSTEM_DARWIN
+
+template <typename STACKTRACE_TAG> class TraceResolverDarwinImpl;
+
+template <>
+class TraceResolverDarwinImpl<trace_resolver_tag::backtrace_symbol>
+ : public TraceResolverImplBase {
+public:
+ void load_addresses(void *const*addresses, int address_count) override {
+ if (address_count == 0) {
+ return;
+ }
+ _symbols.reset(backtrace_symbols(addresses, address_count));
+ }
+
+ ResolvedTrace resolve(ResolvedTrace trace) override {
+ // parse:
+ // <n> <file> <addr> <mangled-name> + <offset>
+ char *filename = _symbols[trace.idx];
+
+ // skip "<n> "
+ while (*filename && *filename != ' ')
+ filename++;
+ while (*filename == ' ')
+ filename++;
+
+ // find start of <mangled-name> from end (<file> may contain a space)
+ char *p = filename + strlen(filename) - 1;
+ // skip to start of " + <offset>"
+ while (p > filename && *p != ' ')
+ p--;
+ while (p > filename && *p == ' ')
+ p--;
+ while (p > filename && *p != ' ')
+ p--;
+ while (p > filename && *p == ' ')
+ p--;
+ char *funcname_end = p + 1;
+
+ // skip to start of "<manged-name>"
+ while (p > filename && *p != ' ')
+ p--;
+ char *funcname = p + 1;
+
+ // skip to start of " <addr> "
+ while (p > filename && *p == ' ')
+ p--;
+ while (p > filename && *p != ' ')
+ p--;
+ while (p > filename && *p == ' ')
+ p--;
+
+ // skip "<file>", handling the case where it contains a
+ char *filename_end = p + 1;
+ if (p == filename) {
+ // something went wrong, give up
+ filename_end = filename + strlen(filename);
+ funcname = filename_end;
+ }
+ trace.object_filename.assign(
+ filename, filename_end); // ok even if filename_end is the ending \0
+ // (then we assign entire string)
+
+ if (*funcname) { // if it's not end of string
+ *funcname_end = '\0';
+
+ trace.object_function = this->demangle(funcname);
+ trace.object_function += " ";
+ trace.object_function += (funcname_end + 1);
+ trace.source.function = trace.object_function; // we cannot do better.
+ }
+ return trace;
+ }
+
+private:
+ details::handle<char **> _symbols;
+};
+
+template <>
+class TraceResolverImpl<system_tag::darwin_tag>
+ : public TraceResolverDarwinImpl<trace_resolver_tag::current> {};
+
+#endif // BACKWARD_SYSTEM_DARWIN
+
+#ifdef BACKWARD_SYSTEM_WINDOWS
+
+// Load all symbol info
+// Based on:
+// https://stackoverflow.com/questions/6205981/windows-c-stack-trace-from-a-running-app/28276227#28276227
+
+struct module_data {
+ std::string image_name;
+ std::string module_name;
+ void *base_address;
+ DWORD load_size;
+};
+
+class get_mod_info {
+ HANDLE process;
+ static const int buffer_length = 4096;
+
+public:
+ get_mod_info(HANDLE h) : process(h) {}
+
+ module_data operator()(HMODULE module) {
+ module_data ret;
+ char temp[buffer_length];
+ MODULEINFO mi;
+
+ GetModuleInformation(process, module, &mi, sizeof(mi));
+ ret.base_address = mi.lpBaseOfDll;
+ ret.load_size = mi.SizeOfImage;
+
+ GetModuleFileNameExA(process, module, temp, sizeof(temp));
+ ret.image_name = temp;
+ GetModuleBaseNameA(process, module, temp, sizeof(temp));
+ ret.module_name = temp;
+ std::vector<char> img(ret.image_name.begin(), ret.image_name.end());
+ std::vector<char> mod(ret.module_name.begin(), ret.module_name.end());
+ SymLoadModule64(process, 0, &img[0], &mod[0], (DWORD64)ret.base_address,
+ ret.load_size);
+ return ret;
+ }
+};
+
+template <> class TraceResolverImpl<system_tag::windows_tag>
+ : public TraceResolverImplBase {
+public:
+ TraceResolverImpl() {
+
+ HANDLE process = GetCurrentProcess();
+
+ std::vector<module_data> modules;
+ DWORD cbNeeded;
+ std::vector<HMODULE> module_handles(1);
+ SymInitialize(process, NULL, false);
+ DWORD symOptions = SymGetOptions();
+ symOptions |= SYMOPT_LOAD_LINES | SYMOPT_UNDNAME;
+ SymSetOptions(symOptions);
+ EnumProcessModules(process, &module_handles[0],
+ module_handles.size() * sizeof(HMODULE), &cbNeeded);
+ module_handles.resize(cbNeeded / sizeof(HMODULE));
+ EnumProcessModules(process, &module_handles[0],
+ module_handles.size() * sizeof(HMODULE), &cbNeeded);
+ std::transform(module_handles.begin(), module_handles.end(),
+ std::back_inserter(modules), get_mod_info(process));
+ void *base = modules[0].base_address;
+ IMAGE_NT_HEADERS *h = ImageNtHeader(base);
+ image_type = h->FileHeader.Machine;
+ }
+
+ static const int max_sym_len = 255;
+ struct symbol_t {
+ SYMBOL_INFO sym;
+ char buffer[max_sym_len];
+ } sym;
+
+ DWORD64 displacement;
+
+ ResolvedTrace resolve(ResolvedTrace t) override {
+ HANDLE process = GetCurrentProcess();
+
+ char name[256];
+
+ memset(&sym, 0, sizeof(sym));
+ sym.sym.SizeOfStruct = sizeof(SYMBOL_INFO);
+ sym.sym.MaxNameLen = max_sym_len;
+
+ if (!SymFromAddr(process, (ULONG64)t.addr, &displacement, &sym.sym)) {
+ // TODO: error handling everywhere
+ char* lpMsgBuf;
+ DWORD dw = GetLastError();
+
+ if (FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+ FORMAT_MESSAGE_FROM_SYSTEM |
+ FORMAT_MESSAGE_IGNORE_INSERTS,
+ NULL, dw, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+ (char*)&lpMsgBuf, 0, NULL)) {
+ std::fprintf(stderr, "%s\n", lpMsgBuf);
+ LocalFree(lpMsgBuf);
+ }
+
+ // abort();
+ }
+ UnDecorateSymbolName(sym.sym.Name, (PSTR)name, 256, UNDNAME_COMPLETE);
+
+ DWORD offset = 0;
+ IMAGEHLP_LINE line;
+ if (SymGetLineFromAddr(process, (ULONG64)t.addr, &offset, &line)) {
+ t.object_filename = line.FileName;
+ t.source.filename = line.FileName;
+ t.source.line = line.LineNumber;
+ t.source.col = offset;
+ }
+
+ t.source.function = name;
+ t.object_filename = "";
+ t.object_function = name;
+
+ return t;
+ }
+
+ DWORD machine_type() const { return image_type; }
+
+private:
+ DWORD image_type;
+};
+
+#endif
+
+class TraceResolver : public TraceResolverImpl<system_tag::current_tag> {};
+
+/*************** CODE SNIPPET ***************/
+
+class SourceFile {
+public:
+ typedef std::vector<std::pair<unsigned, std::string>> lines_t;
+
+ SourceFile() {}
+ SourceFile(const std::string &path) {
+ // 1. If BACKWARD_CXX_SOURCE_PREFIXES is set then assume it contains
+ // a colon-separated list of path prefixes. Try prepending each
+ // to the given path until a valid file is found.
+ const std::vector<std::string> &prefixes = get_paths_from_env_variable();
+ for (size_t i = 0; i < prefixes.size(); ++i) {
+ // Double slashes (//) should not be a problem.
+ std::string new_path = prefixes[i] + '/' + path;
+ _file.reset(new std::ifstream(new_path.c_str()));
+ if (is_open())
+ break;
+ }
+ // 2. If no valid file found then fallback to opening the path as-is.
+ if (!_file || !is_open()) {
+ _file.reset(new std::ifstream(path.c_str()));
+ }
+ }
+ bool is_open() const { return _file->is_open(); }
+
+ lines_t &get_lines(unsigned line_start, unsigned line_count, lines_t &lines) {
+ using namespace std;
+ // This function make uses of the dumbest algo ever:
+ // 1) seek(0)
+ // 2) read lines one by one and discard until line_start
+ // 3) read line one by one until line_start + line_count
+ //
+ // If you are getting snippets many time from the same file, it is
+ // somewhat a waste of CPU, feel free to benchmark and propose a
+ // better solution ;)
+
+ _file->clear();
+ _file->seekg(0);
+ string line;
+ unsigned line_idx;
+
+ for (line_idx = 1; line_idx < line_start; ++line_idx) {
+ std::getline(*_file, line);
+ if (!*_file) {
+ return lines;
+ }
+ }
+
+ // think of it like a lambda in C++98 ;)
+ // but look, I will reuse it two times!
+ // What a good boy am I.
+ struct isspace {
+ bool operator()(char c) { return std::isspace(c); }
+ };
+
+ bool started = false;
+ for (; line_idx < line_start + line_count; ++line_idx) {
+ getline(*_file, line);
+ if (!*_file) {
+ return lines;
+ }
+ if (!started) {
+ if (std::find_if(line.begin(), line.end(), not_isspace()) == line.end())
+ continue;
+ started = true;
+ }
+ lines.push_back(make_pair(line_idx, line));
+ }
+
+ lines.erase(
+ std::find_if(lines.rbegin(), lines.rend(), not_isempty()).base(),
+ lines.end());
+ return lines;
+ }
+
+ lines_t get_lines(unsigned line_start, unsigned line_count) {
+ lines_t lines;
+ return get_lines(line_start, line_count, lines);
+ }
+
+ // there is no find_if_not in C++98, lets do something crappy to
+ // workaround.
+ struct not_isspace {
+ bool operator()(char c) { return !std::isspace(c); }
+ };
+ // and define this one here because C++98 is not happy with local defined
+ // struct passed to template functions, fuuuu.
+ struct not_isempty {
+ bool operator()(const lines_t::value_type &p) {
+ return !(std::find_if(p.second.begin(), p.second.end(), not_isspace()) ==
+ p.second.end());
+ }
+ };
+
+ void swap(SourceFile &b) { _file.swap(b._file); }
+
+#ifdef BACKWARD_ATLEAST_CXX11
+ SourceFile(SourceFile &&from) : _file(nullptr) { swap(from); }
+ SourceFile &operator=(SourceFile &&from) {
+ swap(from);
+ return *this;
+ }
+#else
+ explicit SourceFile(const SourceFile &from) {
+ // some sort of poor man's move semantic.
+ swap(const_cast<SourceFile &>(from));
+ }
+ SourceFile &operator=(const SourceFile &from) {
+ // some sort of poor man's move semantic.
+ swap(const_cast<SourceFile &>(from));
+ return *this;
+ }
+#endif
+
+private:
+ details::handle<std::ifstream *, details::default_delete<std::ifstream *>>
+ _file;
+
+ std::vector<std::string> get_paths_from_env_variable_impl() {
+ std::vector<std::string> paths;
+ const char *prefixes_str = std::getenv("BACKWARD_CXX_SOURCE_PREFIXES");
+ if (prefixes_str && prefixes_str[0]) {
+ paths = details::split_source_prefixes(prefixes_str);
+ }
+ return paths;
+ }
+
+ const std::vector<std::string> &get_paths_from_env_variable() {
+ static std::vector<std::string> paths = get_paths_from_env_variable_impl();
+ return paths;
+ }
+
+#ifdef BACKWARD_ATLEAST_CXX11
+ SourceFile(const SourceFile &) = delete;
+ SourceFile &operator=(const SourceFile &) = delete;
+#endif
+};
+
+class SnippetFactory {
+public:
+ typedef SourceFile::lines_t lines_t;
+
+ lines_t get_snippet(const std::string &filename, unsigned line_start,
+ unsigned context_size) {
+
+ SourceFile &src_file = get_src_file(filename);
+ unsigned start = line_start - context_size / 2;
+ return src_file.get_lines(start, context_size);
+ }
+
+ lines_t get_combined_snippet(const std::string &filename_a, unsigned line_a,
+ const std::string &filename_b, unsigned line_b,
+ unsigned context_size) {
+ SourceFile &src_file_a = get_src_file(filename_a);
+ SourceFile &src_file_b = get_src_file(filename_b);
+
+ lines_t lines =
+ src_file_a.get_lines(line_a - context_size / 4, context_size / 2);
+ src_file_b.get_lines(line_b - context_size / 4, context_size / 2, lines);
+ return lines;
+ }
+
+ lines_t get_coalesced_snippet(const std::string &filename, unsigned line_a,
+ unsigned line_b, unsigned context_size) {
+ SourceFile &src_file = get_src_file(filename);
+
+ using std::max;
+ using std::min;
+ unsigned a = min(line_a, line_b);
+ unsigned b = max(line_a, line_b);
+
+ if ((b - a) < (context_size / 3)) {
+ return src_file.get_lines((a + b - context_size + 1) / 2, context_size);
+ }
+
+ lines_t lines = src_file.get_lines(a - context_size / 4, context_size / 2);
+ src_file.get_lines(b - context_size / 4, context_size / 2, lines);
+ return lines;
+ }
+
+private:
+ typedef details::hashtable<std::string, SourceFile>::type src_files_t;
+ src_files_t _src_files;
+
+ SourceFile &get_src_file(const std::string &filename) {
+ src_files_t::iterator it = _src_files.find(filename);
+ if (it != _src_files.end()) {
+ return it->second;
+ }
+ SourceFile &new_src_file = _src_files[filename];
+ new_src_file = SourceFile(filename);
+ return new_src_file;
+ }
+};
+
+/*************** PRINTER ***************/
+
+namespace ColorMode {
+enum type { automatic, never, always };
+}
+
+class cfile_streambuf : public std::streambuf {
+public:
+ cfile_streambuf(FILE *_sink) : sink(_sink) {}
+ int_type underflow() override { return traits_type::eof(); }
+ int_type overflow(int_type ch) override {
+ if (traits_type::not_eof(ch) && fputc(ch, sink) != EOF) {
+ return ch;
+ }
+ return traits_type::eof();
+ }
+
+ std::streamsize xsputn(const char_type *s, std::streamsize count) override {
+ return static_cast<std::streamsize>(
+ fwrite(s, sizeof *s, static_cast<size_t>(count), sink));
+ }
+
+#ifdef BACKWARD_ATLEAST_CXX11
+public:
+ cfile_streambuf(const cfile_streambuf &) = delete;
+ cfile_streambuf &operator=(const cfile_streambuf &) = delete;
+#else
+private:
+ cfile_streambuf(const cfile_streambuf &);
+ cfile_streambuf &operator=(const cfile_streambuf &);
+#endif
+
+private:
+ FILE *sink;
+ std::vector<char> buffer;
+};
+
+#ifdef BACKWARD_SYSTEM_LINUX
+
+namespace Color {
+enum type { yellow = 33, purple = 35, reset = 39 };
+} // namespace Color
+
+class Colorize {
+public:
+ Colorize(std::ostream &os) : _os(os), _reset(false), _enabled(false) {}
+
+ void activate(ColorMode::type mode) { _enabled = mode == ColorMode::always; }
+
+ void activate(ColorMode::type mode, FILE *fp) { activate(mode, fileno(fp)); }
+
+ void set_color(Color::type ccode) {
+ if (!_enabled)
+ return;
+
+ // I assume that the terminal can handle basic colors. Seriously I
+ // don't want to deal with all the termcap shit.
+ _os << "\033[" << static_cast<int>(ccode) << "m";
+ _reset = (ccode != Color::reset);
+ }
+
+ ~Colorize() {
+ if (_reset) {
+ set_color(Color::reset);
+ }
+ }
+
+private:
+ void activate(ColorMode::type mode, int fd) {
+ activate(mode == ColorMode::automatic && isatty(fd) ? ColorMode::always
+ : mode);
+ }
+
+ std::ostream &_os;
+ bool _reset;
+ bool _enabled;
+};
+
+#else // ndef BACKWARD_SYSTEM_LINUX
+
+namespace Color {
+enum type { yellow = 0, purple = 0, reset = 0 };
+} // namespace Color
+
+class Colorize {
+public:
+ Colorize(std::ostream &) {}
+ void activate(ColorMode::type) {}
+ void activate(ColorMode::type, FILE *) {}
+ void set_color(Color::type) {}
+};
+
+#endif // BACKWARD_SYSTEM_LINUX
+
+class Printer {
+public:
+ bool snippet;
+ ColorMode::type color_mode;
+ bool address;
+ bool object;
+ int inliner_context_size;
+ int trace_context_size;
+
+ Printer()
+ : snippet(true), color_mode(ColorMode::automatic), address(false),
+ object(false), inliner_context_size(5), trace_context_size(7) {}
+
+ template <typename ST> FILE *print(ST &st, FILE *fp = stderr) {
+ cfile_streambuf obuf(fp);
+ std::ostream os(&obuf);
+ Colorize colorize(os);
+ colorize.activate(color_mode, fp);
+ print_stacktrace(st, os, colorize);
+ return fp;
+ }
+
+ template <typename ST> std::ostream &print(ST &st, std::ostream &os) {
+ Colorize colorize(os);
+ colorize.activate(color_mode);
+ print_stacktrace(st, os, colorize);
+ return os;
+ }
+
+ template <typename IT>
+ FILE *print(IT begin, IT end, FILE *fp = stderr, size_t thread_id = 0) {
+ cfile_streambuf obuf(fp);
+ std::ostream os(&obuf);
+ Colorize colorize(os);
+ colorize.activate(color_mode, fp);
+ print_stacktrace(begin, end, os, thread_id, colorize);
+ return fp;
+ }
+
+ template <typename IT>
+ std::ostream &print(IT begin, IT end, std::ostream &os,
+ size_t thread_id = 0) {
+ Colorize colorize(os);
+ colorize.activate(color_mode);
+ print_stacktrace(begin, end, os, thread_id, colorize);
+ return os;
+ }
+
+ TraceResolver const &resolver() const { return _resolver; }
+
+private:
+ TraceResolver _resolver;
+ SnippetFactory _snippets;
+
+ template <typename ST>
+ void print_stacktrace(ST &st, std::ostream &os, Colorize &colorize) {
+ print_header(os, st.thread_id());
+ _resolver.load_stacktrace(st);
+ for (size_t trace_idx = st.size(); trace_idx > 0; --trace_idx) {
+ print_trace(os, _resolver.resolve(st[trace_idx - 1]), colorize);
+ }
+ }
+
+ template <typename IT>
+ void print_stacktrace(IT begin, IT end, std::ostream &os, size_t thread_id,
+ Colorize &colorize) {
+ print_header(os, thread_id);
+ for (; begin != end; ++begin) {
+ print_trace(os, *begin, colorize);
+ }
+ }
+
+ void print_header(std::ostream &os, size_t thread_id) {
+ os << "Stack trace (most recent call last)";
+ if (thread_id) {
+ os << " in thread " << thread_id;
+ }
+ os << ":\n";
+ }
+
+ void print_trace(std::ostream &os, const ResolvedTrace &trace,
+ Colorize &colorize) {
+ os << "#" << std::left << std::setw(2) << trace.idx << std::right;
+ bool already_indented = true;
+
+ if (!trace.source.filename.size() || object) {
+ os << " Object \"" << trace.object_filename << "\", at " << trace.addr
+ << ", in " << trace.object_function << "\n";
+ already_indented = false;
+ }
+
+ for (size_t inliner_idx = trace.inliners.size(); inliner_idx > 0;
+ --inliner_idx) {
+ if (!already_indented) {
+ os << " ";
+ }
+ const ResolvedTrace::SourceLoc &inliner_loc =
+ trace.inliners[inliner_idx - 1];
+ print_source_loc(os, " | ", inliner_loc);
+ if (snippet) {
+ print_snippet(os, " | ", inliner_loc, colorize, Color::purple,
+ inliner_context_size);
+ }
+ already_indented = false;
+ }
+
+ if (trace.source.filename.size()) {
+ if (!already_indented) {
+ os << " ";
+ }
+ print_source_loc(os, " ", trace.source, trace.addr);
+ if (snippet) {
+ print_snippet(os, " ", trace.source, colorize, Color::yellow,
+ trace_context_size);
+ }
+ }
+ }
+
+ void print_snippet(std::ostream &os, const char *indent,
+ const ResolvedTrace::SourceLoc &source_loc,
+ Colorize &colorize, Color::type color_code,
+ int context_size) {
+ using namespace std;
+ typedef SnippetFactory::lines_t lines_t;
+
+ lines_t lines = _snippets.get_snippet(source_loc.filename, source_loc.line,
+ static_cast<unsigned>(context_size));
+
+ for (lines_t::const_iterator it = lines.begin(); it != lines.end(); ++it) {
+ if (it->first == source_loc.line) {
+ colorize.set_color(color_code);
+ os << indent << ">";
+ } else {
+ os << indent << " ";
+ }
+ os << std::setw(4) << it->first << ": " << it->second << "\n";
+ if (it->first == source_loc.line) {
+ colorize.set_color(Color::reset);
+ }
+ }
+ }
+
+ void print_source_loc(std::ostream &os, const char *indent,
+ const ResolvedTrace::SourceLoc &source_loc,
+ void *addr = nullptr) {
+ os << indent << "Source \"" << source_loc.filename << "\", line "
+ << source_loc.line << ", in " << source_loc.function;
+
+ if (address && addr != nullptr) {
+ os << " [" << addr << "]";
+ }
+ os << "\n";
+ }
+};
+
+/*************** SIGNALS HANDLING ***************/
+
+#if defined(BACKWARD_SYSTEM_LINUX) || defined(BACKWARD_SYSTEM_DARWIN)
+
+class SignalHandling {
+public:
+ static std::vector<int> make_default_signals() {
+ const int posix_signals[] = {
+ // Signals for which the default action is "Core".
+ SIGABRT, // Abort signal from abort(3)
+ SIGBUS, // Bus error (bad memory access)
+ SIGFPE, // Floating point exception
+ SIGILL, // Illegal Instruction
+ SIGIOT, // IOT trap. A synonym for SIGABRT
+ SIGQUIT, // Quit from keyboard
+ SIGSEGV, // Invalid memory reference
+ SIGSYS, // Bad argument to routine (SVr4)
+ SIGTRAP, // Trace/breakpoint trap
+ SIGXCPU, // CPU time limit exceeded (4.2BSD)
+ SIGXFSZ, // File size limit exceeded (4.2BSD)
+#if defined(BACKWARD_SYSTEM_DARWIN)
+ SIGEMT, // emulation instruction executed
+#endif
+ };
+ return std::vector<int>(posix_signals,
+ posix_signals +
+ sizeof posix_signals / sizeof posix_signals[0]);
+ }
+
+ SignalHandling(const std::vector<int> &posix_signals = make_default_signals())
+ : _loaded(false) {
+ bool success = true;
+
+ const size_t stack_size = 1024 * 1024 * 8;
+ _stack_content.reset(static_cast<char *>(malloc(stack_size)));
+ if (_stack_content) {
+ stack_t ss;
+ ss.ss_sp = _stack_content.get();
+ ss.ss_size = stack_size;
+ ss.ss_flags = 0;
+ if (sigaltstack(&ss, nullptr) < 0) {
+ success = false;
+ }
+ } else {
+ success = false;
+ }
+
+ for (size_t i = 0; i < posix_signals.size(); ++i) {
+ struct sigaction action;
+ memset(&action, 0, sizeof action);
+ action.sa_flags =
+ static_cast<int>(SA_SIGINFO | SA_ONSTACK | SA_NODEFER | SA_RESETHAND);
+ sigfillset(&action.sa_mask);
+ sigdelset(&action.sa_mask, posix_signals[i]);
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdisabled-macro-expansion"
+#endif
+ action.sa_sigaction = &sig_handler;
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+
+ int r = sigaction(posix_signals[i], &action, nullptr);
+ if (r < 0)
+ success = false;
+ }
+
+ _loaded = success;
+ }
+
+ bool loaded() const { return _loaded; }
+
+ static void handleSignal(int, siginfo_t *info, void *_ctx) {
+ ucontext_t *uctx = static_cast<ucontext_t *>(_ctx);
+
+ StackTrace st;
+ void *error_addr = nullptr;
+#ifdef REG_RIP // x86_64
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.gregs[REG_RIP]);
+#elif defined(REG_EIP) // x86_32
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.gregs[REG_EIP]);
+#elif defined(__arm__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.arm_pc);
+#elif defined(__aarch64__)
+ #if defined(__APPLE__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext->__ss.__pc);
+ #else
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.pc);
+ #endif
+#elif defined(__loongarch__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.__pc);
+#elif defined(__mips__)
+ error_addr = reinterpret_cast<void *>(
+ reinterpret_cast<struct sigcontext *>(&uctx->uc_mcontext)->sc_pc);
+#elif defined(__ppc__) || defined(__powerpc) || defined(__powerpc__) || \
+ defined(__POWERPC__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.regs->nip);
+#elif defined(__riscv)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.__gregs[REG_PC]);
+#elif defined(__s390x__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext.psw.addr);
+#elif defined(__APPLE__) && defined(__x86_64__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext->__ss.__rip);
+#elif defined(__APPLE__)
+ error_addr = reinterpret_cast<void *>(uctx->uc_mcontext->__ss.__eip);
+#else
+#warning ":/ sorry, ain't know no nothing none not of your architecture!"
+#endif
+ if (error_addr) {
+ st.load_from(error_addr, 32, reinterpret_cast<void *>(uctx),
+ info->si_addr);
+ } else {
+ st.load_here(32, reinterpret_cast<void *>(uctx), info->si_addr);
+ }
+
+ Printer printer;
+ printer.address = true;
+ printer.print(st, stderr);
+
+#if _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L
+ psiginfo(info, nullptr);
+#else
+ (void)info;
+#endif
+ }
+
+private:
+ details::handle<char *> _stack_content;
+ bool _loaded;
+
+#ifdef __GNUC__
+ __attribute__((noreturn))
+#endif
+ static void
+ sig_handler(int signo, siginfo_t *info, void *_ctx) {
+ handleSignal(signo, info, _ctx);
+
+ // try to forward the signal.
+ raise(info->si_signo);
+
+ // terminate the process immediately.
+ puts("watf? exit");
+ _exit(EXIT_FAILURE);
+ }
+};
+
+#endif // BACKWARD_SYSTEM_LINUX || BACKWARD_SYSTEM_DARWIN
+
+#ifdef BACKWARD_SYSTEM_WINDOWS
+
+class SignalHandling {
+public:
+ SignalHandling(const std::vector<int> & = std::vector<int>())
+ : reporter_thread_([]() {
+ /* We handle crashes in a utility thread:
+ backward structures and some Windows functions called here
+ need stack space, which we do not have when we encounter a
+ stack overflow.
+ To support reporting stack traces during a stack overflow,
+ we create a utility thread at startup, which waits until a
+ crash happens or the program exits normally. */
+
+ {
+ std::unique_lock<std::mutex> lk(mtx());
+ cv().wait(lk, [] { return crashed() != crash_status::running; });
+ }
+ if (crashed() == crash_status::crashed) {
+ handle_stacktrace(skip_recs());
+ }
+ {
+ std::unique_lock<std::mutex> lk(mtx());
+ crashed() = crash_status::ending;
+ }
+ cv().notify_one();
+ }) {
+ SetUnhandledExceptionFilter(crash_handler);
+
+ signal(SIGABRT, signal_handler);
+ _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+
+ std::set_terminate(&terminator);
+#ifndef BACKWARD_ATLEAST_CXX17
+ std::set_unexpected(&terminator);
+#endif
+ _set_purecall_handler(&terminator);
+ _set_invalid_parameter_handler(&invalid_parameter_handler);
+ }
+ bool loaded() const { return true; }
+
+ ~SignalHandling() {
+ {
+ std::unique_lock<std::mutex> lk(mtx());
+ crashed() = crash_status::normal_exit;
+ }
+
+ cv().notify_one();
+
+ reporter_thread_.join();
+ }
+
+private:
+ static CONTEXT *ctx() {
+ static CONTEXT data;
+ return &data;
+ }
+
+ enum class crash_status { running, crashed, normal_exit, ending };
+
+ static crash_status &crashed() {
+ static crash_status data;
+ return data;
+ }
+
+ static std::mutex &mtx() {
+ static std::mutex data;
+ return data;
+ }
+
+ static std::condition_variable &cv() {
+ static std::condition_variable data;
+ return data;
+ }
+
+ static HANDLE &thread_handle() {
+ static HANDLE handle;
+ return handle;
+ }
+
+ std::thread reporter_thread_;
+
+ // TODO: how not to hardcode these?
+ static const constexpr int signal_skip_recs =
+#ifdef __clang__
+ // With clang, RtlCaptureContext also captures the stack frame of the
+ // current function Below that, there ar 3 internal Windows functions
+ 4
+#else
+ // With MSVC cl, RtlCaptureContext misses the stack frame of the current
+ // function The first entries during StackWalk are the 3 internal Windows
+ // functions
+ 3
+#endif
+ ;
+
+ static int &skip_recs() {
+ static int data;
+ return data;
+ }
+
+ static inline void terminator() {
+ crash_handler(signal_skip_recs);
+ abort();
+ }
+
+ static inline void signal_handler(int) {
+ crash_handler(signal_skip_recs);
+ abort();
+ }
+
+ static inline void __cdecl invalid_parameter_handler(const wchar_t *,
+ const wchar_t *,
+ const wchar_t *,
+ unsigned int,
+ uintptr_t) {
+ crash_handler(signal_skip_recs);
+ abort();
+ }
+
+ NOINLINE static LONG WINAPI crash_handler(EXCEPTION_POINTERS *info) {
+ // The exception info supplies a trace from exactly where the issue was,
+ // no need to skip records
+ crash_handler(0, info->ContextRecord);
+ return EXCEPTION_CONTINUE_SEARCH;
+ }
+
+ NOINLINE static void crash_handler(int skip, CONTEXT *ct = nullptr) {
+
+ if (ct == nullptr) {
+ RtlCaptureContext(ctx());
+ } else {
+ memcpy(ctx(), ct, sizeof(CONTEXT));
+ }
+ DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
+ GetCurrentProcess(), &thread_handle(), 0, FALSE,
+ DUPLICATE_SAME_ACCESS);
+
+ skip_recs() = skip;
+
+ {
+ std::unique_lock<std::mutex> lk(mtx());
+ crashed() = crash_status::crashed;
+ }
+
+ cv().notify_one();
+
+ {
+ std::unique_lock<std::mutex> lk(mtx());
+ cv().wait(lk, [] { return crashed() != crash_status::crashed; });
+ }
+ }
+
+ static void handle_stacktrace(int skip_frames = 0) {
+ // printer creates the TraceResolver, which can supply us a machine type
+ // for stack walking. Without this, StackTrace can only guess using some
+ // macros.
+ // StackTrace also requires that the PDBs are already loaded, which is done
+ // in the constructor of TraceResolver
+ Printer printer;
+
+ StackTrace st;
+ st.set_machine_type(printer.resolver().machine_type());
+ st.set_thread_handle(thread_handle());
+ st.load_here(32 + skip_frames, ctx());
+ st.skip_n_firsts(skip_frames);
+
+ printer.address = true;
+ printer.print(st, std::cerr);
+ }
+};
+
+#endif // BACKWARD_SYSTEM_WINDOWS
+
+#ifdef BACKWARD_SYSTEM_UNKNOWN
+
+class SignalHandling {
+public:
+ SignalHandling(const std::vector<int> & = std::vector<int>()) {}
+ bool init() { return false; }
+ bool loaded() { return false; }
+};
+
+#endif // BACKWARD_SYSTEM_UNKNOWN
+
+} // namespace backward
+
+#endif /* H_GUARD */
diff --git a/contrib/cdb/CMakeLists.txt b/contrib/cdb/CMakeLists.txt
new file mode 100644
index 0000000..6b5b676
--- /dev/null
+++ b/contrib/cdb/CMakeLists.txt
@@ -0,0 +1,8 @@
+# CDB support makefile
+SET(CDBSRC cdb_init.c
+ cdb_find.c
+ cdb_make.c)
+
+ADD_LIBRARY(rspamd-cdb STATIC ${CDBSRC})
+SET_TARGET_PROPERTIES(rspamd-cdb PROPERTIES VERSION ${RSPAMD_VERSION})
+SET_TARGET_PROPERTIES(rspamd-cdb PROPERTIES COMPILE_FLAGS "-DRSPAMD_LIB") \ No newline at end of file
diff --git a/contrib/cdb/cdb.h b/contrib/cdb/cdb.h
new file mode 100644
index 0000000..8774799
--- /dev/null
+++ b/contrib/cdb/cdb.h
@@ -0,0 +1,161 @@
+/* $Id: cdb.h,v 1.10 2009-01-31 17:12:22 mjt Exp $
+ * public cdb include file
+ *
+ * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru.
+ * Public domain.
+ */
+
+#ifndef TINYCDB_VERSION
+#define TINYCDB_VERSION 0.77
+
+#include "config.h"
+#include "unix-std.h"
+#include "contrib/libev/ev.h"
+
+/*
+ * OpenBSD fix
+ */
+#ifndef EPROTO
+#define EPROTO EPROTONOSUPPORT
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef unsigned int cdbi_t; /* compatibility */
+
+/* common routines */
+unsigned cdb_hash(const void *buf, unsigned len);
+unsigned cdb_unpack(const unsigned char buf[4]);
+void cdb_pack(unsigned num, unsigned char buf[4]);
+
+struct cdb
+{
+ int cdb_fd; /* file descriptor */
+ char *filename; /* file name */
+ time_t mtime; /* mtime of cdb file */
+ struct ev_loop *loop;
+ ev_stat stat_ev; /* event structure for checking cdb for modifications */
+ ev_tstamp check_ts;
+ /* private members */
+ unsigned cdb_fsize; /* datafile size */
+ unsigned cdb_dend; /* end of data ptr */
+ const unsigned char *cdb_mem; /* mmap'ed file memory */
+ unsigned cdb_vpos, cdb_vlen; /* found data */
+ unsigned cdb_kpos, cdb_klen; /* found key */
+};
+
+#define CDB_STATIC_INIT {0,0,0,0,0,0,0,0}
+
+#define cdb_datapos(c) ((c)->cdb_vpos)
+#define cdb_datalen(c) ((c)->cdb_vlen)
+#define cdb_keypos(c) ((c)->cdb_kpos)
+#define cdb_keylen(c) ((c)->cdb_klen)
+#define cdb_fileno(c) ((c)->cdb_fd)
+
+int cdb_init(struct cdb *cdbp, int fd);
+void cdb_add_timer(struct cdb *cdbp, EV_P_ ev_tstamp seconds);
+void cdb_free(struct cdb *cdbp);
+
+int cdb_read(const struct cdb *cdbp, void *buf, unsigned len, unsigned pos);
+#define cdb_readdata(cdbp, buf) \
+ cdb_read((cdbp), (buf), cdb_datalen(cdbp), cdb_datapos(cdbp))
+#define cdb_readkey(cdbp, buf) \
+ cdb_read((cdbp), (buf), cdb_keylen(cdbp), cdb_keypos(cdbp))
+
+const void *cdb_get(const struct cdb *cdbp, unsigned len, unsigned pos);
+#define cdb_getdata(cdbp) \
+ cdb_get((cdbp), cdb_datalen(cdbp), cdb_datapos(cdbp))
+#define cdb_getkey(cdbp) \
+ cdb_get((cdbp), cdb_keylen(cdbp), cdb_keypos(cdbp))
+
+int cdb_find(struct cdb *cdbp, const void *key, unsigned klen);
+
+struct cdb_find
+{
+ struct cdb *cdb_cdbp;
+ unsigned cdb_hval;
+ const unsigned char *cdb_htp, *cdb_htab, *cdb_htend;
+ unsigned cdb_httodo;
+ const void *cdb_key;
+ unsigned cdb_klen;
+};
+
+int cdb_findinit(struct cdb_find *cdbfp, struct cdb *cdbp, const void *key,
+ unsigned klen);
+int cdb_findnext(struct cdb_find *cdbfp);
+
+#define cdb_seqinit(cptr, cdbp) ((*(cptr))=2048)
+int cdb_seqnext(unsigned *cptr, struct cdb *cdbp);
+
+/* old simple interface */
+/* open file using standard routine, then: */
+int cdb_seek(int fd, const void *key, unsigned klen, unsigned *dlenp);
+int cdb_bread(int fd, void *buf, int len);
+
+/* cdb_make */
+
+struct cdb_make
+{
+ int cdb_fd; /* file descriptor */
+
+ /* private */
+ unsigned cdb_dpos; /* data position so far */
+ unsigned cdb_rcnt; /* record count so far */
+ unsigned char cdb_buf[4096]; /* write buffer */
+ unsigned char *cdb_bpos; /* current buf position */
+ struct cdb_rl *cdb_rec[256]; /* list of arrays of record infos */
+};
+
+enum cdb_put_mode
+{
+ CDB_PUT_ADD = 0, /* add unconditionnaly, like cdb_make_add() */
+#define CDB_PUT_ADD CDB_PUT_ADD
+ CDB_FIND = CDB_PUT_ADD, CDB_PUT_REPLACE, /* replace: do not place to index OLD record */
+#define CDB_PUT_REPLACE CDB_PUT_REPLACE
+ CDB_FIND_REMOVE = CDB_PUT_REPLACE, CDB_PUT_INSERT, /* add only if not already exists */
+#define CDB_PUT_INSERT CDB_PUT_INSERT
+ CDB_PUT_WARN, /* add unconditionally but ret. 1 if exists */
+#define CDB_PUT_WARN CDB_PUT_WARN
+ CDB_PUT_REPLACE0, /* if a record exists, fill old one with zeros */
+#define CDB_PUT_REPLACE0 CDB_PUT_REPLACE0
+ CDB_FIND_FILL0 = CDB_PUT_REPLACE0
+};
+
+int cdb_make_start(struct cdb_make *cdbmp, int fd);
+int cdb_make_add(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ const void *val, unsigned vlen);
+int cdb_make_exists(struct cdb_make *cdbmp, const void *key, unsigned klen);
+int cdb_make_find(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ enum cdb_put_mode mode);
+int cdb_make_put(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ const void *val, unsigned vlen, enum cdb_put_mode mode);
+int cdb_make_finish(struct cdb_make *cdbmp);
+
+/** Private API **/
+struct cdb_rec
+{
+ unsigned hval;
+ unsigned rpos;
+};
+
+struct cdb_rl
+{
+ struct cdb_rl *next;
+ unsigned cnt;
+ struct cdb_rec rec[254];
+};
+
+int _cdb_make_write(struct cdb_make *cdbmp, const unsigned char *ptr,
+ unsigned len);
+int _cdb_make_fullwrite(int fd, const unsigned char *buf, unsigned len);
+int _cdb_make_flush(struct cdb_make *cdbmp);
+int _cdb_make_add(struct cdb_make *cdbmp, unsigned hval, const void *key,
+ unsigned klen, const void *val, unsigned vlen);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* include guard */
diff --git a/contrib/cdb/cdb_find.c b/contrib/cdb/cdb_find.c
new file mode 100644
index 0000000..cae0f18
--- /dev/null
+++ b/contrib/cdb/cdb_find.c
@@ -0,0 +1,147 @@
+/* $Id: cdb_init.c,v 1.12 2008-11-06 18:07:04 mjt Exp $
+ * cdb_init, cdb_free and cdb_read routines
+ *
+ * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru.
+ * Public domain.
+ */
+
+#include "cdb.h"
+
+int
+cdb_find(struct cdb *cdbp, const void *key, unsigned klen)
+{
+ const unsigned char *htp; /* hash table pointer */
+ const unsigned char *htab; /* hash table */
+ const unsigned char *htend; /* end of hash table */
+ unsigned httodo; /* ht bytes left to look */
+ unsigned pos, n;
+
+ unsigned hval;
+
+ if (klen >= cdbp->cdb_dend) /* if key size is too large */
+ return 0;
+
+ hval = cdb_hash (key, klen);
+
+ /* find (pos,n) hash table to use */
+ /* first 2048 bytes (toc) are always available */
+ /* (hval % 256) * 8 */
+ htp = cdbp->cdb_mem + ((hval << 3) & 2047); /* index in toc (256x8) */
+ n = cdb_unpack (htp + 4); /* table size */
+ if (!n) /* empty table */
+ return 0; /* not found */
+ httodo = n << 3; /* bytes of htab to lookup */
+ pos = cdb_unpack (htp); /* htab position */
+ if (n > (cdbp->cdb_fsize >> 3) /* overflow of httodo ? */
+ || pos < cdbp->cdb_dend /* is htab inside data section ? */
+ || pos > cdbp->cdb_fsize /* htab start within file ? */
+ || httodo > cdbp->cdb_fsize - pos) /* entrie htab within file ? */
+ return errno = EPROTO, -1;
+
+ htab = cdbp->cdb_mem + pos; /* htab pointer */
+ htend = htab + httodo; /* after end of htab */
+ /* htab starting position: rest of hval modulo htsize, 8bytes per elt */
+ htp = htab + (((hval >> 8) % n) << 3);
+
+ for (;;) {
+ pos = cdb_unpack (htp + 4); /* record position */
+ if (!pos)
+ return 0;
+ if (cdb_unpack (htp) == hval) {
+ if (pos > cdbp->cdb_dend - 8) /* key+val lengths */
+ return errno = EPROTO, -1;
+ if (cdb_unpack (cdbp->cdb_mem + pos) == klen) {
+ if (cdbp->cdb_dend - klen < pos + 8)
+ return errno = EPROTO, -1;
+ if (memcmp (key, cdbp->cdb_mem + pos + 8, klen) == 0) {
+ n = cdb_unpack (cdbp->cdb_mem + pos + 4);
+ pos += 8;
+ if (cdbp->cdb_dend < n || cdbp->cdb_dend - n < pos + klen)
+ return errno = EPROTO, -1;
+ cdbp->cdb_kpos = pos;
+ cdbp->cdb_klen = klen;
+ cdbp->cdb_vpos = pos + klen;
+ cdbp->cdb_vlen = n;
+ return 1;
+ }
+ }
+ }
+ httodo -= 8;
+ if (!httodo)
+ return 0;
+ if ((htp += 8) >= htend)
+ htp = htab;
+ }
+
+}
+
+int
+cdb_findinit(struct cdb_find *cdbfp, struct cdb *cdbp,
+ const void *key, unsigned klen)
+{
+ unsigned n, pos;
+
+ cdbfp->cdb_cdbp = cdbp;
+ cdbfp->cdb_key = key;
+ cdbfp->cdb_klen = klen;
+ cdbfp->cdb_hval = cdb_hash(key, klen);
+
+ cdbfp->cdb_htp = cdbp->cdb_mem + ((cdbfp->cdb_hval << 3) & 2047);
+ n = cdb_unpack(cdbfp->cdb_htp + 4);
+ cdbfp->cdb_httodo = n << 3;
+ if (!n)
+ return 0;
+ pos = cdb_unpack(cdbfp->cdb_htp);
+ if (n > (cdbp->cdb_fsize >> 3)
+ || pos < cdbp->cdb_dend
+ || pos > cdbp->cdb_fsize
+ || cdbfp->cdb_httodo > cdbp->cdb_fsize - pos)
+ return errno = EPROTO, -1;
+
+ cdbfp->cdb_htab = cdbp->cdb_mem + pos;
+ cdbfp->cdb_htend = cdbfp->cdb_htab + cdbfp->cdb_httodo;
+ cdbfp->cdb_htp = cdbfp->cdb_htab + (((cdbfp->cdb_hval >> 8) % n) << 3);
+
+ return 1;
+}
+
+int
+cdb_findnext(struct cdb_find *cdbfp) {
+ struct cdb *cdbp = cdbfp->cdb_cdbp;
+ unsigned pos, n;
+ unsigned klen = cdbfp->cdb_klen;
+
+ while(cdbfp->cdb_httodo) {
+ pos = cdb_unpack(cdbfp->cdb_htp + 4);
+ if (!pos)
+ return 0;
+ n = cdb_unpack(cdbfp->cdb_htp) == cdbfp->cdb_hval;
+ if ((cdbfp->cdb_htp += 8) >= cdbfp->cdb_htend)
+ cdbfp->cdb_htp = cdbfp->cdb_htab;
+ cdbfp->cdb_httodo -= 8;
+ if (n) {
+ if (pos > cdbp->cdb_fsize - 8)
+ return errno = EPROTO, -1;
+ if (cdb_unpack(cdbp->cdb_mem + pos) == klen) {
+ if (cdbp->cdb_fsize - klen < pos + 8)
+ return errno = EPROTO, -1;
+ if (memcmp(cdbfp->cdb_key,
+ cdbp->cdb_mem + pos + 8, klen) == 0) {
+ n = cdb_unpack(cdbp->cdb_mem + pos + 4);
+ pos += 8;
+ if (cdbp->cdb_fsize < n ||
+ cdbp->cdb_fsize - n < pos + klen)
+ return errno = EPROTO, -1;
+ cdbp->cdb_kpos = pos;
+ cdbp->cdb_klen = klen;
+ cdbp->cdb_vpos = pos + klen;
+ cdbp->cdb_vlen = n;
+ return 1;
+ }
+ }
+ }
+ }
+
+ return 0;
+
+}
diff --git a/contrib/cdb/cdb_init.c b/contrib/cdb/cdb_init.c
new file mode 100644
index 0000000..bfc6dd0
--- /dev/null
+++ b/contrib/cdb/cdb_init.c
@@ -0,0 +1,140 @@
+/* $Id: cdb_init.c,v 1.12 2008-11-06 18:07:04 mjt Exp $
+ * cdb_init, cdb_free and cdb_read routines
+ *
+ * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru.
+ * Public domain.
+ */
+
+#include "cdb.h"
+
+unsigned
+cdb_hash(const void *buf, unsigned len)
+{
+ register const unsigned char *p = (const unsigned char *) buf;
+ register const unsigned char *end = p + len;
+ register unsigned hash = 5381; /* start value */
+ while (p < end)
+ hash = (hash + (hash << 5)) ^ *p++;
+ return hash;
+}
+
+int
+cdb_init(struct cdb *cdbp, int fd)
+{
+ struct stat st;
+ unsigned char *mem;
+ unsigned fsize, dend;
+#ifdef _WIN32
+ HANDLE hFile, hMapping;
+#endif
+
+ /* get file size */
+ if (fstat (fd, &st) < 0)
+ return -1;
+ /* trivial sanity check: at least toc should be here */
+ if (st.st_size < 2048)
+ return errno = EPROTO, -1;
+ fsize = (unsigned) (st.st_size & 0xffffffffu);
+ /* memory-map file */
+#ifdef _WIN32
+ hFile = (HANDLE) _get_osfhandle(fd);
+ if (hFile == (HANDLE) -1)
+ return -1;
+ hMapping = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
+ if (!hMapping)
+ return -1;
+ mem = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
+ CloseHandle(hMapping);
+ if (!mem)
+ return -1;
+#else
+ mem = (unsigned char*) mmap (NULL, fsize, PROT_READ, MAP_SHARED, fd, 0);
+ if (mem == MAP_FAILED)
+ return -1;
+#endif /* _WIN32 */
+
+ cdbp->cdb_fd = fd;
+ cdbp->cdb_fsize = fsize;
+ cdbp->cdb_mem = mem;
+ cdbp->mtime = st.st_mtime;
+
+ cdbp->cdb_vpos = cdbp->cdb_vlen = 0;
+ cdbp->cdb_kpos = cdbp->cdb_klen = 0;
+ dend = cdb_unpack (mem);
+ if (dend < 2048)
+ dend = 2048;
+ else if (dend >= fsize)
+ dend = fsize;
+ cdbp->cdb_dend = dend;
+
+ return 0;
+}
+
+void
+cdb_free(struct cdb *cdbp)
+{
+ if (cdbp->cdb_mem) {
+#ifdef _WIN32
+ UnmapViewOfFile((void*) cdbp->cdb_mem);
+#else
+ munmap ((void*) cdbp->cdb_mem, cdbp->cdb_fsize);
+#endif /* _WIN32 */
+ cdbp->cdb_mem = NULL;
+ }
+ cdbp->cdb_fsize = 0;
+
+ if (cdbp->loop) {
+ ev_stat_stop (cdbp->loop, &cdbp->stat_ev);
+ }
+}
+
+const void *
+cdb_get(const struct cdb *cdbp, unsigned len, unsigned pos)
+{
+ if (pos > cdbp->cdb_fsize || cdbp->cdb_fsize - pos < len) {
+ errno = EPROTO;
+ return NULL;
+ }
+ return cdbp->cdb_mem + pos;
+}
+
+int
+cdb_read(const struct cdb *cdbp, void *buf, unsigned len, unsigned pos)
+{
+ const void *data = cdb_get (cdbp, len, pos);
+ if (!data)
+ return -1;
+ memcpy (buf, data, len);
+ return 0;
+}
+
+static void
+cdb_timer_callback (EV_P_ ev_stat *w, int revents)
+{
+ struct cdb *cdbp = w->data;
+ gint nfd;
+
+ /* Check cdb file for modifications */
+ if ((nfd = open (cdbp->filename, O_RDONLY)) != -1) {
+ if (cdbp->cdb_mem) {
+#ifdef _WIN32
+ UnmapViewOfFile((void*) cdbp->cdb_mem);
+#else
+ munmap ((void*) cdbp->cdb_mem, cdbp->cdb_fsize);
+#endif /* _WIN32 */
+ cdbp->cdb_mem = NULL;
+ }
+ (void)close (cdbp->cdb_fd);
+ cdbp->cdb_fsize = 0;
+ (void)cdb_init (cdbp, nfd);
+ }
+}
+
+void
+cdb_add_timer (struct cdb *cdbp, EV_P_ ev_tstamp seconds)
+{
+ cdbp->loop = loop;
+ ev_stat_init (&cdbp->stat_ev, cdb_timer_callback, cdbp->filename, seconds);
+ cdbp->stat_ev.data = cdbp;
+ ev_stat_start (EV_A_ &cdbp->stat_ev);
+}
diff --git a/contrib/cdb/cdb_make.c b/contrib/cdb/cdb_make.c
new file mode 100644
index 0000000..646a7e4
--- /dev/null
+++ b/contrib/cdb/cdb_make.c
@@ -0,0 +1,524 @@
+/* $Id: cdb_init.c,v 1.12 2008-11-06 18:07:04 mjt Exp $
+ * cdb_init, cdb_free and cdb_read routines
+ *
+ * This file is a part of tinycdb package by Michael Tokarev, mjt@corpit.ru.
+ * Public domain.
+ */
+
+#include "cdb.h"
+
+void cdb_pack(unsigned num, unsigned char buf[4])
+{
+ buf[0] = num & 255;
+ num >>= 8;
+ buf[1] = num & 255;
+ num >>= 8;
+ buf[2] = num & 255;
+ buf[3] = num >> 8;
+}
+
+int cdb_make_start(struct cdb_make *cdbmp, int fd)
+{
+ memset (cdbmp, 0, sizeof(*cdbmp));
+ cdbmp->cdb_fd = fd;
+ cdbmp->cdb_dpos = 2048;
+ cdbmp->cdb_bpos = cdbmp->cdb_buf + 2048;
+ return 0;
+}
+
+int
+_cdb_make_fullwrite(int fd, const unsigned char *buf, unsigned len)
+{
+ while (len) {
+ int l = write (fd, buf, len);
+ if (l > 0) {
+ len -= l;
+ buf += l;
+ }
+ else if (l < 0 && errno != EINTR)
+ return -1;
+ }
+ return 0;
+}
+
+int
+_cdb_make_flush(struct cdb_make *cdbmp)
+{
+ unsigned len = cdbmp->cdb_bpos - cdbmp->cdb_buf;
+ if (len) {
+ if (_cdb_make_fullwrite (cdbmp->cdb_fd, cdbmp->cdb_buf, len) < 0)
+ return -1;
+ cdbmp->cdb_bpos = cdbmp->cdb_buf;
+ }
+ return 0;
+}
+
+int
+_cdb_make_write(struct cdb_make *cdbmp, const unsigned char *ptr, unsigned len)
+{
+ unsigned l = sizeof(cdbmp->cdb_buf) - (cdbmp->cdb_bpos - cdbmp->cdb_buf);
+ cdbmp->cdb_dpos += len;
+ if (len > l) {
+ memcpy (cdbmp->cdb_bpos, ptr, l);
+ cdbmp->cdb_bpos += l;
+ if (_cdb_make_flush (cdbmp) < 0)
+ return -1;
+ ptr += l;
+ len -= l;
+ l = len / sizeof(cdbmp->cdb_buf);
+ if (l) {
+ l *= sizeof(cdbmp->cdb_buf);
+ if (_cdb_make_fullwrite (cdbmp->cdb_fd, ptr, l) < 0)
+ return -1;
+ ptr += l;
+ len -= l;
+ }
+ }
+ if (len) {
+ memcpy (cdbmp->cdb_bpos, ptr, len);
+ cdbmp->cdb_bpos += len;
+ }
+ return 0;
+}
+
+static int cdb_make_finish_internal(struct cdb_make *cdbmp)
+{
+ unsigned hcnt[256]; /* hash table counts */
+ unsigned hpos[256]; /* hash table positions */
+ struct cdb_rec *htab;
+ unsigned char *p;
+ struct cdb_rl *rl;
+ unsigned hsize;
+ unsigned t, i;
+
+ if (((0xffffffff - cdbmp->cdb_dpos) >> 3) < cdbmp->cdb_rcnt)
+ return errno = ENOMEM, -1;
+
+ /* count htab sizes and reorder reclists */
+ hsize = 0;
+ for (t = 0; t < 256; ++t) {
+ struct cdb_rl *rlt = NULL;
+ i = 0;
+ rl = cdbmp->cdb_rec[t];
+ while (rl) {
+ struct cdb_rl *rln = rl->next;
+ rl->next = rlt;
+ rlt = rl;
+ i += rl->cnt;
+ rl = rln;
+ }
+ cdbmp->cdb_rec[t] = rlt;
+ if (hsize < (hcnt[t] = i << 1))
+ hsize = hcnt[t];
+ }
+
+ /* allocate memory to hold max htable */
+ htab = (struct cdb_rec*) malloc ((hsize + 2) * sizeof(struct cdb_rec));
+ if (!htab)
+ return errno = ENOENT, -1;
+ p = (unsigned char *) htab;
+ htab += 2;
+
+ /* build hash tables */
+ for (t = 0; t < 256; ++t) {
+ unsigned len, hi;
+ hpos[t] = cdbmp->cdb_dpos;
+ if ((len = hcnt[t]) == 0)
+ continue;
+ for (i = 0; i < len; ++i)
+ htab[i].hval = htab[i].rpos = 0;
+ for (rl = cdbmp->cdb_rec[t]; rl; rl = rl->next)
+ for (i = 0; i < rl->cnt; ++i) {
+ hi = (rl->rec[i].hval >> 8) % len;
+ while (htab[hi].rpos)
+ if (++hi == len)
+ hi = 0;
+ htab[hi] = rl->rec[i];
+ }
+ for (i = 0; i < len; ++i) {
+ cdb_pack (htab[i].hval, p + (i << 3));
+ cdb_pack (htab[i].rpos, p + (i << 3) + 4);
+ }
+ if (_cdb_make_write (cdbmp, p, len << 3) < 0) {
+ free (p);
+ return -1;
+ }
+ }
+ free (p);
+ if (_cdb_make_flush (cdbmp) < 0)
+ return -1;
+ p = cdbmp->cdb_buf;
+ for (t = 0; t < 256; ++t) {
+ cdb_pack (hpos[t], p + (t << 3));
+ cdb_pack (hcnt[t], p + (t << 3) + 4);
+ }
+ if (lseek (cdbmp->cdb_fd, 0, 0) != 0 || _cdb_make_fullwrite (cdbmp->cdb_fd,
+ p, 2048) != 0)
+ return -1;
+
+ return 0;
+}
+
+static void cdb_make_free(struct cdb_make *cdbmp)
+{
+ unsigned t;
+ for (t = 0; t < 256; ++t) {
+ struct cdb_rl *rl = cdbmp->cdb_rec[t];
+ while (rl) {
+ struct cdb_rl *tm = rl;
+ rl = rl->next;
+ free (tm);
+ }
+ }
+}
+
+int cdb_make_finish(struct cdb_make *cdbmp)
+{
+ int r = cdb_make_finish_internal (cdbmp);
+ cdb_make_free (cdbmp);
+ return r;
+}
+
+int
+_cdb_make_add(struct cdb_make *cdbmp, unsigned hval, const void *key,
+ unsigned klen, const void *val, unsigned vlen)
+{
+ unsigned char rlen[8];
+ struct cdb_rl *rl;
+ unsigned i;
+ if (klen > 0xffffffff - (cdbmp->cdb_dpos + 8) || vlen > 0xffffffff
+ - (cdbmp->cdb_dpos + klen + 8))
+ return errno = ENOMEM, -1;
+ i = hval & 255;
+ rl = cdbmp->cdb_rec[i];
+ if (!rl || rl->cnt >= sizeof(rl->rec) / sizeof(rl->rec[0])) {
+ rl = (struct cdb_rl*) malloc (sizeof(struct cdb_rl));
+ if (!rl)
+ return errno = ENOMEM, -1;
+ rl->cnt = 0;
+ rl->next = cdbmp->cdb_rec[i];
+ cdbmp->cdb_rec[i] = rl;
+ }
+ i = rl->cnt++;
+ rl->rec[i].hval = hval;
+ rl->rec[i].rpos = cdbmp->cdb_dpos;
+ ++cdbmp->cdb_rcnt;
+ cdb_pack (klen, rlen);
+ cdb_pack (vlen, rlen + 4);
+ if (_cdb_make_write (cdbmp, rlen, 8) < 0 || _cdb_make_write (cdbmp, key,
+ klen) < 0 || _cdb_make_write (cdbmp, val, vlen) < 0)
+ return -1;
+ return 0;
+}
+
+int cdb_make_add(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ const void *val, unsigned vlen)
+{
+ return _cdb_make_add (cdbmp, cdb_hash (key, klen), key, klen, val, vlen);
+}
+
+static void fixup_rpos(struct cdb_make *cdbmp, unsigned rpos, unsigned rlen)
+{
+ unsigned i;
+ struct cdb_rl *rl;
+ register struct cdb_rec *rp, *rs;
+ for (i = 0; i < 256; ++i) {
+ for (rl = cdbmp->cdb_rec[i]; rl; rl = rl->next)
+ for (rs = rl->rec, rp = rs + rl->cnt; --rp >= rs;)
+ if (rp->rpos <= rpos)
+ goto nexthash;
+ else
+ rp->rpos -= rlen;
+ nexthash: ;
+ }
+}
+
+static int remove_record(struct cdb_make *cdbmp, unsigned rpos, unsigned rlen)
+{
+ unsigned pos, len;
+ int r, fd;
+
+ len = cdbmp->cdb_dpos - rpos - rlen;
+ cdbmp->cdb_dpos -= rlen;
+ if (!len)
+ return 0; /* it was the last record, nothing to do */
+ pos = rpos;
+ fd = cdbmp->cdb_fd;
+ do {
+ r = len > sizeof(cdbmp->cdb_buf) ? sizeof(cdbmp->cdb_buf) : len;
+ if (lseek (fd, pos + rlen, SEEK_SET) < 0 || (r = read (fd,
+ cdbmp->cdb_buf, r)) <= 0)
+ return -1;
+ if (lseek (fd, pos, SEEK_SET) < 0 || _cdb_make_fullwrite (fd,
+ cdbmp->cdb_buf, r) < 0)
+ return -1;
+ pos += r;
+ len -= r;
+ } while (len);
+ g_assert (cdbmp->cdb_dpos == pos);
+ fixup_rpos (cdbmp, rpos, rlen);
+ return 0;
+}
+
+static int zerofill_record(struct cdb_make *cdbmp, unsigned rpos, unsigned rlen)
+{
+ if (rpos + rlen == cdbmp->cdb_dpos) {
+ cdbmp->cdb_dpos = rpos;
+ return 0;
+ }
+ if (lseek (cdbmp->cdb_fd, rpos, SEEK_SET) < 0)
+ return -1;
+ memset (cdbmp->cdb_buf, 0, sizeof(cdbmp->cdb_buf));
+ cdb_pack (rlen - 8, cdbmp->cdb_buf + 4);
+ for (;;) {
+ rpos = rlen > sizeof(cdbmp->cdb_buf) ? sizeof(cdbmp->cdb_buf) : rlen;
+ if (_cdb_make_fullwrite (cdbmp->cdb_fd, cdbmp->cdb_buf, rpos) < 0)
+ return -1;
+ rlen -= rpos;
+ if (!rlen)
+ return 0;
+ memset (cdbmp->cdb_buf + 4, 0, 4);
+ }
+}
+
+/* return: 0 = not found, 1 = error, or record length */
+static unsigned match(struct cdb_make *cdbmp, unsigned pos, const char *key,
+ unsigned klen)
+{
+ int len;
+ unsigned rlen;
+ if (lseek (cdbmp->cdb_fd, pos, SEEK_SET) < 0)
+ return 1;
+ if (read (cdbmp->cdb_fd, cdbmp->cdb_buf, 8) != 8)
+ return 1;
+ if (cdb_unpack (cdbmp->cdb_buf) != klen)
+ return 0;
+
+ /* record length; check its validity */
+ rlen = cdb_unpack (cdbmp->cdb_buf + 4);
+ if (rlen > cdbmp->cdb_dpos - pos - klen - 8)
+ return errno = EPROTO, 1; /* someone changed our file? */
+ rlen += klen + 8;
+
+ while (klen) {
+ len = klen > sizeof(cdbmp->cdb_buf) ? sizeof(cdbmp->cdb_buf) : klen;
+ len = read (cdbmp->cdb_fd, cdbmp->cdb_buf, len);
+ if (len <= 0)
+ return 1;
+ if (memcmp (cdbmp->cdb_buf, key, len) != 0)
+ return 0;
+ key += len;
+ klen -= len;
+ }
+
+ return rlen;
+}
+
+static int findrec(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ unsigned hval, enum cdb_put_mode mode)
+{
+ struct cdb_rl *rl;
+ struct cdb_rec *rp, *rs;
+ unsigned r;
+ int sought = 0;
+ int ret = 0;
+ for (rl = cdbmp->cdb_rec[hval & 255]; rl; rl = rl->next)
+ for (rs = rl->rec, rp = rs + rl->cnt; --rp >= rs;) {
+ if (rp->hval != hval)
+ continue;
+ /*XXX this explicit flush may be unnecessary having
+ * smarter match() that looks into cdb_buf too, but
+ * most of a time here spent in finding hash values
+ * (above), not keys */
+ if (!sought && _cdb_make_flush (cdbmp) < 0)
+ return -1;
+ sought = 1;
+ r = match (cdbmp, rp->rpos, key, klen);
+ if (!r)
+ continue;
+ if (r == 1)
+ return -1;
+ ret = 1;
+ switch (mode)
+ {
+ case CDB_FIND_REMOVE:
+ if (remove_record (cdbmp, rp->rpos, r) < 0)
+ return -1;
+ break;
+ case CDB_FIND_FILL0:
+ if (zerofill_record (cdbmp, rp->rpos, r) < 0)
+ return -1;
+ break;
+ default:
+ goto finish;
+ }
+ memmove (rp, rp + 1, (rs + rl->cnt - 1 - rp) * sizeof(*rp));
+ --rl->cnt;
+ --cdbmp->cdb_rcnt;
+ }
+ finish: if (sought && lseek (cdbmp->cdb_fd, cdbmp->cdb_dpos, SEEK_SET) < 0)
+ return -1;
+ return ret;
+}
+
+int cdb_make_find(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ enum cdb_put_mode mode)
+{
+ return findrec (cdbmp, key, klen, cdb_hash (key, klen), mode);
+}
+
+int cdb_make_exists(struct cdb_make *cdbmp, const void *key, unsigned klen)
+{
+ return cdb_make_find (cdbmp, key, klen, CDB_FIND);
+}
+
+int cdb_make_put(struct cdb_make *cdbmp, const void *key, unsigned klen,
+ const void *val, unsigned vlen, enum cdb_put_mode mode)
+{
+ unsigned hval = cdb_hash (key, klen);
+ int r;
+
+ switch (mode)
+ {
+ case CDB_PUT_REPLACE:
+ case CDB_PUT_INSERT:
+ case CDB_PUT_WARN:
+ case CDB_PUT_REPLACE0:
+ r = findrec (cdbmp, key, klen, hval, mode);
+ if (r < 0)
+ return -1;
+ if (r && mode == CDB_PUT_INSERT)
+ return errno = EEXIST, 1;
+ break;
+
+ case CDB_PUT_ADD:
+ r = 0;
+ break;
+
+ default:
+ return errno = EINVAL, -1;
+ }
+
+ if (_cdb_make_add (cdbmp, hval, key, klen, val, vlen) < 0)
+ return -1;
+
+ return r;
+}
+
+unsigned
+cdb_unpack(const unsigned char buf[4])
+{
+ unsigned n = buf[3];
+ n <<= 8; n |= buf[2];
+ n <<= 8; n |= buf[1];
+ n <<= 8; n |= buf[0];
+ return n;
+}
+
+int
+cdb_seqnext(unsigned *cptr, struct cdb *cdbp) {
+ unsigned klen, vlen;
+ unsigned pos = *cptr;
+ unsigned dend = cdbp->cdb_dend;
+ const unsigned char *mem = cdbp->cdb_mem;
+ if (pos > dend - 8)
+ return 0;
+ klen = cdb_unpack(mem + pos);
+ vlen = cdb_unpack(mem + pos + 4);
+ pos += 8;
+ if (dend - klen < pos || dend - vlen < pos + klen)
+ return errno = EPROTO, -1;
+ cdbp->cdb_kpos = pos;
+ cdbp->cdb_klen = klen;
+ cdbp->cdb_vpos = pos + klen;
+ cdbp->cdb_vlen = vlen;
+ *cptr = pos + klen + vlen;
+ return 1;
+}
+
+/* read a chunk from file, ignoring interrupts (EINTR) */
+
+int
+cdb_bread(int fd, void *buf, int len)
+{
+ int l;
+ while(len > 0) {
+ do l = read(fd, buf, len);
+ while(l < 0 && errno == EINTR);
+ if (l <= 0) {
+ if (!l)
+ errno = EIO;
+ return -1;
+ }
+ buf = (char*)buf + l;
+ len -= l;
+ }
+ return 0;
+}
+
+/* find a given key in cdb file, seek a file pointer to it's value and
+ place data length to *dlenp. */
+
+int
+cdb_seek(int fd, const void *key, unsigned klen, unsigned *dlenp)
+{
+ unsigned htstart; /* hash table start position */
+ unsigned htsize; /* number of elements in a hash table */
+ unsigned httodo; /* hash table elements left to look */
+ unsigned hti; /* hash table index */
+ unsigned pos; /* position in a file */
+ unsigned hval; /* key's hash value */
+ unsigned char rbuf[64]; /* read buffer */
+ int needseek = 1; /* if we should seek to a hash slot */
+
+ hval = cdb_hash(key, klen);
+ pos = (hval & 0xff) << 3; /* position in TOC */
+ /* read the hash table parameters */
+ if (lseek(fd, pos, SEEK_SET) < 0 || cdb_bread(fd, rbuf, 8) < 0)
+ return -1;
+ if ((htsize = cdb_unpack(rbuf + 4)) == 0)
+ return 0;
+ hti = (hval >> 8) % htsize; /* start position in hash table */
+ httodo = htsize;
+ htstart = cdb_unpack(rbuf);
+
+ for(;;) {
+ if (needseek && lseek(fd, htstart + (hti << 3), SEEK_SET) < 0)
+ return -1;
+ if (cdb_bread(fd, rbuf, 8) < 0)
+ return -1;
+ if ((pos = cdb_unpack(rbuf + 4)) == 0) /* not found */
+ return 0;
+
+ if (cdb_unpack(rbuf) != hval) /* hash value not matched */
+ needseek = 0;
+ else { /* hash value matched */
+ if (lseek(fd, pos, SEEK_SET) < 0 || cdb_bread(fd, rbuf, 8) < 0)
+ return -1;
+ if (cdb_unpack(rbuf) == klen) { /* key length matches */
+ /* read the key from file and compare with wanted */
+ unsigned l = klen, c;
+ const char *k = (const char*)key;
+ if (dlenp)
+ *dlenp = cdb_unpack(rbuf + 4); /* save value length */
+ for(;;) {
+ if (!l) /* the whole key read and matches, return */
+ return 1;
+ c = l > sizeof(rbuf) ? sizeof(rbuf) : l;
+ if (cdb_bread(fd, rbuf, c) < 0)
+ return -1;
+ if (memcmp(rbuf, k, c) != 0) /* no, it differs, stop here */
+ break;
+ k += c; l -= c;
+ }
+ }
+ needseek = 1; /* we're looked to other place, should seek back */
+ }
+ if (!--httodo)
+ return 0;
+ if (++hti == htsize) {
+ hti = 0;
+ needseek = 1;
+ }
+ }
+}
diff --git a/contrib/doctest/CMakeLists.txt b/contrib/doctest/CMakeLists.txt
new file mode 100644
index 0000000..4a17708
--- /dev/null
+++ b/contrib/doctest/CMakeLists.txt
@@ -0,0 +1,63 @@
+cmake_minimum_required(VERSION 3.0)
+
+if(POLICY CMP0077)
+ cmake_policy(SET CMP0077 NEW)
+endif()
+
+################################################################################
+## DOCTEST
+################################################################################
+
+project(doctest VERSION 2.4.6 LANGUAGES CXX)
+
+# Determine if doctest is built as a subproject (using add_subdirectory) or if it is the main project.
+set(MAIN_PROJECT OFF)
+if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
+ set(MAIN_PROJECT ON)
+endif()
+
+option(DOCTEST_WITH_TESTS "Build tests/examples" ${MAIN_PROJECT})
+option(DOCTEST_WITH_MAIN_IN_STATIC_LIB "Build a static lib (cmake target) with a default main entry point" ON)
+option(DOCTEST_NO_INSTALL "Skip the installation process" OFF)
+option(DOCTEST_USE_STD_HEADERS "Use std headers" OFF)
+
+add_library(${PROJECT_NAME} INTERFACE)
+add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
+
+if(NOT CMAKE_VERSION VERSION_LESS 3.8)
+ target_compile_features(${PROJECT_NAME} INTERFACE cxx_std_11)
+endif()
+
+set(doctest_parts_folder "${CMAKE_CURRENT_SOURCE_DIR}/doctest/parts")
+set(doctest_folder "${CMAKE_CURRENT_SOURCE_DIR}/") # in order to have the mpi extension files, not included into the doctest.h single header
+
+if(MAIN_PROJECT)
+ # use a special hidden version of the header which directly includes the 2 parts - proper reporting of file/line locations during dev
+ target_include_directories(${PROJECT_NAME} INTERFACE
+ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/scripts/development_only/>
+ $<BUILD_INTERFACE:${doctest_parts_folder}>
+ $<BUILD_INTERFACE:${doctest_folder}>)
+
+ # add a custom target that assembles the single header when any of the parts are touched
+ add_custom_command(
+ OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/doctest/doctest.h
+ DEPENDS
+ ${doctest_parts_folder}/doctest_fwd.h
+ ${doctest_parts_folder}/doctest.cpp
+ COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/assemble_single_header.cmake
+ COMMENT "assembling the single header")
+
+ add_custom_target(assemble_single_header ALL DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/doctest/doctest.h)
+else()
+ target_include_directories(${PROJECT_NAME} INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/>)
+endif()
+
+# hack to support building on XCode 6 and 7 - propagate the definition to everything
+if(DEFINED DOCTEST_THREAD_LOCAL)
+ target_compile_definitions(${PROJECT_NAME} INTERFACE
+ DOCTEST_THREAD_LOCAL=${DOCTEST_THREAD_LOCAL})
+endif()
+
+if(DOCTEST_USE_STD_HEADERS)
+ target_compile_definitions(${PROJECT_NAME} INTERFACE DOCTEST_CONFIG_USE_STD_HEADERS)
+endif()
diff --git a/contrib/doctest/LICENSE.txt b/contrib/doctest/LICENSE.txt
new file mode 100644
index 0000000..d67bb64
--- /dev/null
+++ b/contrib/doctest/LICENSE.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016-2021 Viktor Kirilov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/contrib/doctest/README.md b/contrib/doctest/README.md
new file mode 100644
index 0000000..42148f0
--- /dev/null
+++ b/contrib/doctest/README.md
@@ -0,0 +1,153 @@
+<p align="center"><img src="scripts/data/logo/logo_1.svg"></p>
+<b>
+<table>
+ <tr>
+ <td>
+ master branch
+ </td>
+ <td>
+ Windows <a href="https://ci.appveyor.com/project/onqtam/doctest/branch/master"><img src="https://ci.appveyor.com/api/projects/status/j89qxtahyw1dp4gd/branch/master?svg=true"></a>
+ </td>
+ <td>
+ All <a href="https://github.com/onqtam/doctest/actions?query=branch%3Amaster"><img src="https://github.com/onqtam/doctest/workflows/CI/badge.svg?branch=master"></a>
+ </td>
+ <td>
+ <a href="https://coveralls.io/github/onqtam/doctest?branch=master"><img src="https://coveralls.io/repos/github/onqtam/doctest/badge.svg?branch=master"></a>
+ </td>
+ <!--
+ <td>
+ <a href="https://scan.coverity.com/projects/onqtam-doctest"><img src="https://scan.coverity.com/projects/7865/badge.svg"></a>
+ </td>
+ -->
+ </tr>
+ <tr>
+ <td>
+ dev branch
+ </td>
+ <td>
+ Windows <a href="https://ci.appveyor.com/project/onqtam/doctest/branch/dev"><img src="https://ci.appveyor.com/api/projects/status/j89qxtahyw1dp4gd/branch/dev?svg=true"></a>
+ </td>
+ <td>
+ All <a href="https://github.com/onqtam/doctest/actions?query=branch%3Adev"><img src="https://github.com/onqtam/doctest/workflows/CI/badge.svg?branch=dev"></a>
+ </td>
+ <td>
+ <a href="https://coveralls.io/github/onqtam/doctest?branch=dev"><img src="https://coveralls.io/repos/github/onqtam/doctest/badge.svg?branch=dev"></a>
+ </td>
+ <!--
+ <td>
+ </td>
+ -->
+ </tr>
+</table>
+</b>
+
+**doctest** is a new C++ testing framework but is by far the fastest both in compile times (by [**orders of magnitude**](doc/markdown/benchmarks.md)) and runtime compared to other feature-rich alternatives. It brings the ability of compiled languages such as [**D**](https://dlang.org/spec/unittest.html) / [**Rust**](https://doc.rust-lang.org/book/second-edition/ch11-00-testing.html) / [**Nim**](https://nim-lang.org/docs/unittest.html) to have tests written directly in the production code thanks to a fast, transparent and flexible test runner with a clean interface.
+
+[![Standard](https://img.shields.io/badge/c%2B%2B-11/14/17/20-blue.svg)](https://en.wikipedia.org/wiki/C%2B%2B#Standardization)
+[![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![Version](https://badge.fury.io/gh/onqtam%2Fdoctest.svg)](https://github.com/onqtam/doctest/releases)
+[![download](https://img.shields.io/badge/download%20%20-link-blue.svg)](https://raw.githubusercontent.com/onqtam/doctest/master/doctest/doctest.h)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/503/badge)](https://bestpractices.coreinfrastructure.org/projects/503)
+[![Language grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/onqtam/doctest.svg)](https://lgtm.com/projects/g/onqtam/doctest/context:cpp)
+[![Join the chat at https://gitter.im/onqtam/doctest](https://badges.gitter.im/onqtam/doctest.svg)](https://gitter.im/onqtam/doctest?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Try it online](https://img.shields.io/badge/try%20it-online-orange.svg)](https://wandbox.org/permlink/nJIibfbivG7BG7r1)
+<!--
+[![Language](https://img.shields.io/badge/language-C++-blue.svg)](https://isocpp.org/)
+[![documentation](https://img.shields.io/badge/documentation%20%20-online-blue.svg)](https://github.com/onqtam/doctest/blob/master/doc/markdown/readme.md#reference)
+-->
+
+[<img src="https://cloud.githubusercontent.com/assets/8225057/5990484/70413560-a9ab-11e4-8942-1a63607c0b00.png" align="right">](http://www.patreon.com/onqtam)
+
+The framework is and will stay free but needs your support to sustain its development. There are lots of <a href="doc/markdown/roadmap.md"><b>new features</b></a> and maintenance to do. If you work for a company using **doctest** or have the means to do so, please consider financial support. Monthly donations via Patreon and one-offs via PayPal.
+
+[<img src="https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif" align="right">](https://www.paypal.me/onqtam/10)
+
+A complete example with a self-registering test that compiles to an executable looks like this:
+
+![cover-example](scripts/data/using_doctest_888px_wide.gif)
+
+There are many C++ testing frameworks - [Catch](https://github.com/catchorg/Catch2), [Boost.Test](http://www.boost.org/doc/libs/1_64_0/libs/test/doc/html/index.html), [UnitTest++](https://github.com/unittest-cpp/unittest-cpp), [cpputest](https://github.com/cpputest/cpputest), [googletest](https://github.com/google/googletest) and many [other](https://en.wikipedia.org/wiki/List_of_unit_testing_frameworks#C.2B.2B).
+
+The **key** differences between it and other testing frameworks are that it is light and unintrusive:
+- Ultra light on compile times both in terms of [**including the header**](doc/markdown/benchmarks.md#cost-of-including-the-header) and writing [**thousands of asserts**](doc/markdown/benchmarks.md#cost-of-an-assertion-macro)
+- Doesn't produce any warnings even on the [**most aggressive**](scripts/cmake/common.cmake#L84) warning levels for **MSVC**/**GCC**/**Clang**
+- Offers a way to remove **everything** testing-related from the binary with the [**```DOCTEST_CONFIG_DISABLE```**](doc/markdown/configuration.md#doctest_config_disable) identifier
+- [**thread-safe**](doc/markdown/faq.md#is-doctest-thread-aware) - asserts (and logging) can be used from multiple threads spawned from a single test case - [**example**](examples/all_features/concurrency.cpp)
+- asserts can be used [**outside of a testing context**](doc/markdown/assertions.md#using-asserts-out-of-a-testing-context) - as a general purpose assert library - [**example**](examples/all_features/asserts_used_outside_of_tests.cpp)
+- Doesn't pollute the global namespace (everything is in namespace ```doctest```) and doesn't drag **any** headers with it
+- Very [**portable**](doc/markdown/features.md#extremely-portable) C++11 (use tag [**1.2.9**](https://github.com/onqtam/doctest/tree/1.2.9) for C++98) with over 180 different CI builds (static analysis, sanitizers...)
+- binaries (exe/dll) can use the test runner of another binary - so tests end up in a single registry - [**example**](examples/executable_dll_and_plugin/)
+
+![cost-of-including-the-framework-header](scripts/data/benchmarks/header.png)
+
+This allows the framework to be used in more ways than any other - tests can be written directly in the production code!
+
+*Tests can be considered a form of documentation and should be able to reside near the production code which they test.*
+
+- This makes the barrier for writing tests **much lower** - you don't have to: **1)** make a separate source file **2)** include a bunch of stuff in it **3)** add it to the build system and **4)** add it to source control - You can just write the tests for a class or a piece of functionality at the bottom of its source file - or even header file!
+- Tests in the production code can be thought of as documentation or up-to-date comments - showing the use of APIs
+- Testing internals that are not exposed through the public API and headers is no longer a mind-bending exercise
+- [**Test-driven development**](https://en.wikipedia.org/wiki/Test-driven_development) in C++ has never been easier!
+
+The framework can be used like any other if you don't want/need to mix production code and tests - check out the [**features**](doc/markdown/features.md).
+
+**doctest** is modeled after [**Catch**](https://github.com/catchorg/Catch2) and some parts of the code have been taken directly - check out [**the differences**](doc/markdown/faq.md#how-is-doctest-different-from-catch).
+
+[This table](https://github.com/martinmoene/catch-lest-other-comparison) compares **doctest** / [**Catch**](https://github.com/catchorg/Catch2) / [**lest**](https://github.com/martinmoene/lest) which are all very similar.
+
+Checkout the [**CppCon 2017 talk**](https://cppcon2017.sched.com/event/BgsI/mix-tests-and-production-code-with-doctest-implementing-and-using-the-fastest-modern-c-testing-framework) on [**YouTube**](https://www.youtube.com/watch?v=eH1CxEC29l8) to get a better understanding of how the framework works and read about how to use it in [**the JetBrains article**](https://blog.jetbrains.com/rscpp/better-ways-testing-with-doctest/) - highlighting the unique aspects of the framework! On a short description on how to use the framework along production code you could refer to [**this GitHub issue**](https://github.com/onqtam/doctest/issues/252). There is also an [**older article**](https://accu.org/var/uploads/journals/Overload137.pdf) in the february edition of ACCU Overload 2017.
+
+[![CppCon 2017 talk about doctest on youtube](scripts/data/youtube-cppcon-talk-thumbnail.png)](https://www.youtube.com/watch?v=eH1CxEC29l8)
+
+Documentation
+-------------
+
+Project:
+
+- [Features and design goals](doc/markdown/features.md) - the complete list of features
+- [Roadmap](doc/markdown/roadmap.md) - upcoming features
+- [Benchmarks](doc/markdown/benchmarks.md) - compile-time and runtime supremacy
+- [Contributing](CONTRIBUTING.md) - how to make a proper pull request
+- [Changelog](CHANGELOG.md) - generated changelog based on closed issues/PRs
+
+Usage:
+
+- [Tutorial](doc/markdown/tutorial.md) - make sure you have read it before the other parts of the documentation
+- [Assertion macros](doc/markdown/assertions.md)
+- [Test cases, subcases and test fixtures](doc/markdown/testcases.md)
+- [Parameterized test cases](doc/markdown/parameterized-tests.md)
+- [Command line](doc/markdown/commandline.md)
+- [Logging macros](doc/markdown/logging.md)
+- [```main()``` entry point](doc/markdown/main.md)
+- [Configuration](doc/markdown/configuration.md)
+- [String conversions](doc/markdown/stringification.md)
+- [Reporters](doc/markdown/reporters.md)
+- [Extensions](doc/markdown/extensions.md)
+- [FAQ](doc/markdown/faq.md)
+- [Build systems](doc/markdown/build-systems.md)
+- [Examples](examples)
+
+Contributing
+------------
+
+[<img src="https://cloud.githubusercontent.com/assets/8225057/5990484/70413560-a9ab-11e4-8942-1a63607c0b00.png" align="right">](http://www.patreon.com/onqtam)
+
+Support the development of the project with donations! There is a list of planned features which are all important and big - see the [**roadmap**](doc/markdown/roadmap.md). I took a break from working in the industry to make open source software so every cent is a big deal.
+
+[<img src="https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif" align="right">](https://www.paypal.me/onqtam/10)
+
+If you work for a company using **doctest** or have the means to do so, please consider financial support.
+
+Contributions in the form of issues and pull requests are welcome as well - check out the [**Contributing**](CONTRIBUTING.md) page.
+
+Stargazers over time
+------------
+
+[![Stargazers over time](https://starcharts.herokuapp.com/onqtam/doctest.svg)](https://starcharts.herokuapp.com/onqtam/doctest)
+
+Logo
+------------
+
+The [logo](scripts/data/logo) is licensed under a Creative Commons Attribution 4.0 International License. Copyright &copy; 2019 [area55git](https://github.com/area55git) &nbsp; [![License: CC BY 4.0](https://licensebuttons.net/l/by/4.0/80x15.png)](https://creativecommons.org/licenses/by/4.0/)
+
+<p align="center"><img src="scripts/data/logo/icon_2.svg"></p>
diff --git a/contrib/doctest/doctest/doctest.h b/contrib/doctest/doctest/doctest.h
new file mode 100644
index 0000000..42eb039
--- /dev/null
+++ b/contrib/doctest/doctest/doctest.h
@@ -0,0 +1,6580 @@
+// ====================================================================== lgtm [cpp/missing-header-guard]
+// == DO NOT MODIFY THIS FILE BY HAND - IT IS AUTO GENERATED BY CMAKE! ==
+// ======================================================================
+//
+// doctest.h - the lightest feature-rich C++ single-header testing framework for unit tests and TDD
+//
+// Copyright (c) 2016-2021 Viktor Kirilov
+//
+// Distributed under the MIT Software License
+// See accompanying file LICENSE.txt or copy at
+// https://opensource.org/licenses/MIT
+//
+// The documentation can be found at the library's page:
+// https://github.com/onqtam/doctest/blob/master/doc/markdown/readme.md
+//
+// =================================================================================================
+// =================================================================================================
+// =================================================================================================
+//
+// The library is heavily influenced by Catch - https://github.com/catchorg/Catch2
+// which uses the Boost Software License - Version 1.0
+// see here - https://github.com/catchorg/Catch2/blob/master/LICENSE.txt
+//
+// The concept of subcases (sections in Catch) and expression decomposition are from there.
+// Some parts of the code are taken directly:
+// - stringification - the detection of "ostream& operator<<(ostream&, const T&)" and StringMaker<>
+// - the Approx() helper class for floating point comparison
+// - colors in the console
+// - breaking into a debugger
+// - signal / SEH handling
+// - timer
+// - XmlWriter class - thanks to Phil Nash for allowing the direct reuse (AKA copy/paste)
+//
+// The expression decomposing templates are taken from lest - https://github.com/martinmoene/lest
+// which uses the Boost Software License - Version 1.0
+// see here - https://github.com/martinmoene/lest/blob/master/LICENSE.txt
+//
+// =================================================================================================
+// =================================================================================================
+// =================================================================================================
+
+#ifndef DOCTEST_LIBRARY_INCLUDED
+#define DOCTEST_LIBRARY_INCLUDED
+
+// =================================================================================================
+// == VERSION ======================================================================================
+// =================================================================================================
+
+#define DOCTEST_VERSION_MAJOR 2
+#define DOCTEST_VERSION_MINOR 4
+#define DOCTEST_VERSION_PATCH 6
+#define DOCTEST_VERSION_STR "2.4.6"
+
+#define DOCTEST_VERSION \
+ (DOCTEST_VERSION_MAJOR * 10000 + DOCTEST_VERSION_MINOR * 100 + DOCTEST_VERSION_PATCH)
+
+// =================================================================================================
+// == COMPILER VERSION =============================================================================
+// =================================================================================================
+
+// ideas for the version stuff are taken from here: https://github.com/cxxstuff/cxx_detect
+
+#define DOCTEST_COMPILER(MAJOR, MINOR, PATCH) ((MAJOR)*10000000 + (MINOR)*100000 + (PATCH))
+
+// GCC/Clang and GCC/MSVC are mutually exclusive, but Clang/MSVC are not because of clang-cl...
+#if defined(_MSC_VER) && defined(_MSC_FULL_VER)
+#if _MSC_VER == _MSC_FULL_VER / 10000
+#define DOCTEST_MSVC DOCTEST_COMPILER(_MSC_VER / 100, _MSC_VER % 100, _MSC_FULL_VER % 10000)
+#else // MSVC
+#define DOCTEST_MSVC \
+ DOCTEST_COMPILER(_MSC_VER / 100, (_MSC_FULL_VER / 100000) % 100, _MSC_FULL_VER % 100000)
+#endif // MSVC
+#endif // MSVC
+#if defined(__clang__) && defined(__clang_minor__)
+#define DOCTEST_CLANG DOCTEST_COMPILER(__clang_major__, __clang_minor__, __clang_patchlevel__)
+#elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && \
+ !defined(__INTEL_COMPILER)
+#define DOCTEST_GCC DOCTEST_COMPILER(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__)
+#endif // GCC
+
+#ifndef DOCTEST_MSVC
+#define DOCTEST_MSVC 0
+#endif // DOCTEST_MSVC
+#ifndef DOCTEST_CLANG
+#define DOCTEST_CLANG 0
+#endif // DOCTEST_CLANG
+#ifndef DOCTEST_GCC
+#define DOCTEST_GCC 0
+#endif // DOCTEST_GCC
+
+// =================================================================================================
+// == COMPILER WARNINGS HELPERS ====================================================================
+// =================================================================================================
+
+#if DOCTEST_CLANG
+#define DOCTEST_PRAGMA_TO_STR(x) _Pragma(#x)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_PUSH _Pragma("clang diagnostic push")
+#define DOCTEST_CLANG_SUPPRESS_WARNING(w) DOCTEST_PRAGMA_TO_STR(clang diagnostic ignored w)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_POP _Pragma("clang diagnostic pop")
+#define DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_PUSH DOCTEST_CLANG_SUPPRESS_WARNING(w)
+#else // DOCTEST_CLANG
+#define DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+#define DOCTEST_CLANG_SUPPRESS_WARNING(w)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_POP
+#define DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_CLANG
+
+#if DOCTEST_GCC
+#define DOCTEST_PRAGMA_TO_STR(x) _Pragma(#x)
+#define DOCTEST_GCC_SUPPRESS_WARNING_PUSH _Pragma("GCC diagnostic push")
+#define DOCTEST_GCC_SUPPRESS_WARNING(w) DOCTEST_PRAGMA_TO_STR(GCC diagnostic ignored w)
+#define DOCTEST_GCC_SUPPRESS_WARNING_POP _Pragma("GCC diagnostic pop")
+#define DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_GCC_SUPPRESS_WARNING_PUSH DOCTEST_GCC_SUPPRESS_WARNING(w)
+#else // DOCTEST_GCC
+#define DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+#define DOCTEST_GCC_SUPPRESS_WARNING(w)
+#define DOCTEST_GCC_SUPPRESS_WARNING_POP
+#define DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_GCC
+
+#if DOCTEST_MSVC
+#define DOCTEST_MSVC_SUPPRESS_WARNING_PUSH __pragma(warning(push))
+#define DOCTEST_MSVC_SUPPRESS_WARNING(w) __pragma(warning(disable : w))
+#define DOCTEST_MSVC_SUPPRESS_WARNING_POP __pragma(warning(pop))
+#define DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH DOCTEST_MSVC_SUPPRESS_WARNING(w)
+#else // DOCTEST_MSVC
+#define DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+#define DOCTEST_MSVC_SUPPRESS_WARNING(w)
+#define DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#define DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_MSVC
+
+// =================================================================================================
+// == COMPILER WARNINGS ============================================================================
+// =================================================================================================
+
+DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wnon-virtual-dtor")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wweak-vtables")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wpadded")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wdeprecated")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-prototypes")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-local-typedef")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat-pedantic")
+
+DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wpragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Weffc++")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-overflow")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-aliasing")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wctor-dtor-privacy")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-declarations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnon-virtual-dtor")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-local-typedefs")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wuseless-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-promo")
+
+DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+DOCTEST_MSVC_SUPPRESS_WARNING(4616) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4619) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4996) // The compiler encountered a deprecated declaration
+DOCTEST_MSVC_SUPPRESS_WARNING(4706) // assignment within conditional expression
+DOCTEST_MSVC_SUPPRESS_WARNING(4512) // 'class' : assignment operator could not be generated
+DOCTEST_MSVC_SUPPRESS_WARNING(4127) // conditional expression is constant
+DOCTEST_MSVC_SUPPRESS_WARNING(4820) // padding
+DOCTEST_MSVC_SUPPRESS_WARNING(4625) // copy constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4626) // assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5027) // move assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5026) // move constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4623) // default constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4640) // construction of local static object is not thread-safe
+// static analysis
+DOCTEST_MSVC_SUPPRESS_WARNING(26439) // This kind of function may not throw. Declare it 'noexcept'
+DOCTEST_MSVC_SUPPRESS_WARNING(26495) // Always initialize a member variable
+DOCTEST_MSVC_SUPPRESS_WARNING(26451) // Arithmetic overflow ...
+DOCTEST_MSVC_SUPPRESS_WARNING(26444) // Avoid unnamed objects with custom construction and dtr...
+DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum'
+
+// 4548 - expression before comma has no effect; expected expression with side - effect
+// 4265 - class has virtual functions, but destructor is not virtual
+// 4986 - exception specification does not match previous declaration
+// 4350 - behavior change: 'member1' called instead of 'member2'
+// 4668 - 'x' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif'
+// 4365 - conversion from 'int' to 'unsigned long', signed/unsigned mismatch
+// 4774 - format string expected in argument 'x' is not a string literal
+// 4820 - padding in structs
+
+// only 4 should be disabled globally:
+// - 4514 # unreferenced inline function has been removed
+// - 4571 # SEH related
+// - 4710 # function not inlined
+// - 4711 # function 'x' selected for automatic inline expansion
+
+#define DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN \
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4548) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4265) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4986) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4350) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4668) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4365) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4774) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4820) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4625) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4626) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5027) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5026) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4623) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5039) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5045) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5105)
+
+#define DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+// =================================================================================================
+// == FEATURE DETECTION ============================================================================
+// =================================================================================================
+
+// general compiler feature support table: https://en.cppreference.com/w/cpp/compiler_support
+// MSVC C++11 feature support table: https://msdn.microsoft.com/en-us/library/hh567368.aspx
+// GCC C++11 feature support table: https://gcc.gnu.org/projects/cxx-status.html
+// MSVC version table:
+// https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B#Internal_version_numbering
+// MSVC++ 14.2 (16) _MSC_VER == 1920 (Visual Studio 2019)
+// MSVC++ 14.1 (15) _MSC_VER == 1910 (Visual Studio 2017)
+// MSVC++ 14.0 _MSC_VER == 1900 (Visual Studio 2015)
+// MSVC++ 12.0 _MSC_VER == 1800 (Visual Studio 2013)
+// MSVC++ 11.0 _MSC_VER == 1700 (Visual Studio 2012)
+// MSVC++ 10.0 _MSC_VER == 1600 (Visual Studio 2010)
+// MSVC++ 9.0 _MSC_VER == 1500 (Visual Studio 2008)
+// MSVC++ 8.0 _MSC_VER == 1400 (Visual Studio 2005)
+
+#if DOCTEST_MSVC && !defined(DOCTEST_CONFIG_WINDOWS_SEH)
+#define DOCTEST_CONFIG_WINDOWS_SEH
+#endif // MSVC
+#if defined(DOCTEST_CONFIG_NO_WINDOWS_SEH) && defined(DOCTEST_CONFIG_WINDOWS_SEH)
+#undef DOCTEST_CONFIG_WINDOWS_SEH
+#endif // DOCTEST_CONFIG_NO_WINDOWS_SEH
+
+#if !defined(_WIN32) && !defined(__QNX__) && !defined(DOCTEST_CONFIG_POSIX_SIGNALS) && \
+ !defined(__EMSCRIPTEN__)
+#define DOCTEST_CONFIG_POSIX_SIGNALS
+#endif // _WIN32
+#if defined(DOCTEST_CONFIG_NO_POSIX_SIGNALS) && defined(DOCTEST_CONFIG_POSIX_SIGNALS)
+#undef DOCTEST_CONFIG_POSIX_SIGNALS
+#endif // DOCTEST_CONFIG_NO_POSIX_SIGNALS
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND)
+#define DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // no exceptions
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+#define DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#if defined(DOCTEST_CONFIG_NO_EXCEPTIONS) && !defined(DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS)
+#define DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS && !DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+
+#if defined(DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN) && !defined(DOCTEST_CONFIG_IMPLEMENT)
+#define DOCTEST_CONFIG_IMPLEMENT
+#endif // DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#if DOCTEST_MSVC
+#define DOCTEST_SYMBOL_EXPORT __declspec(dllexport)
+#define DOCTEST_SYMBOL_IMPORT __declspec(dllimport)
+#else // MSVC
+#define DOCTEST_SYMBOL_EXPORT __attribute__((dllexport))
+#define DOCTEST_SYMBOL_IMPORT __attribute__((dllimport))
+#endif // MSVC
+#else // _WIN32
+#define DOCTEST_SYMBOL_EXPORT __attribute__((visibility("default")))
+#define DOCTEST_SYMBOL_IMPORT
+#endif // _WIN32
+
+#ifdef DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#ifdef DOCTEST_CONFIG_IMPLEMENT
+#define DOCTEST_INTERFACE DOCTEST_SYMBOL_EXPORT
+#else // DOCTEST_CONFIG_IMPLEMENT
+#define DOCTEST_INTERFACE DOCTEST_SYMBOL_IMPORT
+#endif // DOCTEST_CONFIG_IMPLEMENT
+#else // DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#define DOCTEST_INTERFACE
+#endif // DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+
+#define DOCTEST_EMPTY
+
+#if DOCTEST_MSVC
+#define DOCTEST_NOINLINE __declspec(noinline)
+#define DOCTEST_UNUSED
+#define DOCTEST_ALIGNMENT(x)
+#elif DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 5, 0)
+#define DOCTEST_NOINLINE
+#define DOCTEST_UNUSED
+#define DOCTEST_ALIGNMENT(x)
+#else
+#define DOCTEST_NOINLINE __attribute__((noinline))
+#define DOCTEST_UNUSED __attribute__((unused))
+#define DOCTEST_ALIGNMENT(x) __attribute__((aligned(x)))
+#endif
+
+#ifndef DOCTEST_NORETURN
+#define DOCTEST_NORETURN [[noreturn]]
+#endif // DOCTEST_NORETURN
+
+#ifndef DOCTEST_NOEXCEPT
+#define DOCTEST_NOEXCEPT noexcept
+#endif // DOCTEST_NOEXCEPT
+
+// =================================================================================================
+// == FEATURE DETECTION END ========================================================================
+// =================================================================================================
+
+// internal macros for string concatenation and anonymous variable name generation
+#define DOCTEST_CAT_IMPL(s1, s2) s1##s2
+#define DOCTEST_CAT(s1, s2) DOCTEST_CAT_IMPL(s1, s2)
+#ifdef __COUNTER__ // not standard and may be missing for some compilers
+#define DOCTEST_ANONYMOUS(x) DOCTEST_CAT(x, __COUNTER__)
+#else // __COUNTER__
+#define DOCTEST_ANONYMOUS(x) DOCTEST_CAT(x, __LINE__)
+#endif // __COUNTER__
+
+#define DOCTEST_TOSTR(x) #x
+
+#ifndef DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+#define DOCTEST_REF_WRAP(x) x&
+#else // DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+#define DOCTEST_REF_WRAP(x) x
+#endif // DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+
+// not using __APPLE__ because... this is how Catch does it
+#ifdef __MAC_OS_X_VERSION_MIN_REQUIRED
+#define DOCTEST_PLATFORM_MAC
+#elif defined(__IPHONE_OS_VERSION_MIN_REQUIRED)
+#define DOCTEST_PLATFORM_IPHONE
+#elif defined(_WIN32)
+#define DOCTEST_PLATFORM_WINDOWS
+#else // DOCTEST_PLATFORM
+#define DOCTEST_PLATFORM_LINUX
+#endif // DOCTEST_PLATFORM
+
+#define DOCTEST_GLOBAL_NO_WARNINGS(var) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wglobal-constructors") \
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-variable") \
+ static const int var DOCTEST_UNUSED // NOLINT(fuchsia-statically-constructed-objects,cert-err58-cpp)
+#define DOCTEST_GLOBAL_NO_WARNINGS_END() DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#ifndef DOCTEST_BREAK_INTO_DEBUGGER
+// should probably take a look at https://github.com/scottt/debugbreak
+#ifdef DOCTEST_PLATFORM_LINUX
+#if defined(__GNUC__) && (defined(__i386) || defined(__x86_64))
+// Break at the location of the failing check if possible
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT (hicpp-no-assembler)
+#else
+#include <signal.h>
+#define DOCTEST_BREAK_INTO_DEBUGGER() raise(SIGTRAP)
+#endif
+#elif defined(DOCTEST_PLATFORM_MAC)
+#if defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__i386)
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT (hicpp-no-assembler)
+#else
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("brk #0"); // NOLINT (hicpp-no-assembler)
+#endif
+#elif DOCTEST_MSVC
+#define DOCTEST_BREAK_INTO_DEBUGGER() __debugbreak()
+#elif defined(__MINGW32__)
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wredundant-decls")
+extern "C" __declspec(dllimport) void __stdcall DebugBreak();
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+#define DOCTEST_BREAK_INTO_DEBUGGER() ::DebugBreak()
+#else // linux
+#define DOCTEST_BREAK_INTO_DEBUGGER() (static_cast<void>(0))
+#endif // linux
+#endif // DOCTEST_BREAK_INTO_DEBUGGER
+
+// this is kept here for backwards compatibility since the config option was changed
+#ifdef DOCTEST_CONFIG_USE_IOSFWD
+#define DOCTEST_CONFIG_USE_STD_HEADERS
+#endif // DOCTEST_CONFIG_USE_IOSFWD
+
+#ifdef DOCTEST_CONFIG_USE_STD_HEADERS
+#ifndef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#define DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#include <iosfwd>
+#include <cstddef>
+#include <ostream>
+#else // DOCTEST_CONFIG_USE_STD_HEADERS
+
+#if DOCTEST_CLANG
+// to detect if libc++ is being used with clang (the _LIBCPP_VERSION identifier)
+#include <ciso646>
+#endif // clang
+
+#ifdef _LIBCPP_VERSION
+#define DOCTEST_STD_NAMESPACE_BEGIN _LIBCPP_BEGIN_NAMESPACE_STD
+#define DOCTEST_STD_NAMESPACE_END _LIBCPP_END_NAMESPACE_STD
+#else // _LIBCPP_VERSION
+#define DOCTEST_STD_NAMESPACE_BEGIN namespace std {
+#define DOCTEST_STD_NAMESPACE_END }
+#endif // _LIBCPP_VERSION
+
+// Forward declaring 'X' in namespace std is not permitted by the C++ Standard.
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4643)
+
+DOCTEST_STD_NAMESPACE_BEGIN // NOLINT (cert-dcl58-cpp)
+typedef decltype(nullptr) nullptr_t;
+template <class charT>
+struct char_traits;
+template <>
+struct char_traits<char>;
+template <class charT, class traits>
+class basic_ostream;
+typedef basic_ostream<char, char_traits<char>> ostream;
+template <class... Types>
+class tuple;
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+template <class _Ty>
+class allocator;
+template <class _Elem, class _Traits, class _Alloc>
+class basic_string;
+using string = basic_string<char, char_traits<char>, allocator<char>>;
+#endif // VS 2019
+DOCTEST_STD_NAMESPACE_END
+
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_USE_STD_HEADERS
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#include <type_traits>
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+namespace doctest {
+
+DOCTEST_INTERFACE extern bool is_running_in_test;
+
+// A 24 byte string class (can be as small as 17 for x64 and 13 for x86) that can hold strings with length
+// of up to 23 chars on the stack before going on the heap - the last byte of the buffer is used for:
+// - "is small" bit - the highest bit - if "0" then it is small - otherwise its "1" (128)
+// - if small - capacity left before going on the heap - using the lowest 5 bits
+// - if small - 2 bits are left unused - the second and third highest ones
+// - if small - acts as a null terminator if strlen() is 23 (24 including the null terminator)
+// and the "is small" bit remains "0" ("as well as the capacity left") so its OK
+// Idea taken from this lecture about the string implementation of facebook/folly - fbstring
+// https://www.youtube.com/watch?v=kPR8h4-qZdk
+// TODO:
+// - optimizations - like not deleting memory unnecessarily in operator= and etc.
+// - resize/reserve/clear
+// - substr
+// - replace
+// - back/front
+// - iterator stuff
+// - find & friends
+// - push_back/pop_back
+// - assign/insert/erase
+// - relational operators as free functions - taking const char* as one of the params
+class DOCTEST_INTERFACE String
+{
+ static const unsigned len = 24; //!OCLINT avoid private static members
+ static const unsigned last = len - 1; //!OCLINT avoid private static members
+
+ struct view // len should be more than sizeof(view) - because of the final byte for flags
+ {
+ char* ptr;
+ unsigned size;
+ unsigned capacity;
+ };
+
+ union
+ {
+ char buf[len];
+ view data;
+ };
+
+ bool isOnStack() const { return (buf[last] & 128) == 0; }
+ void setOnHeap();
+ void setLast(unsigned in = last);
+
+ void copy(const String& other);
+
+public:
+ String();
+ ~String();
+
+ // cppcheck-suppress noExplicitConstructor
+ String(const char* in);
+ String(const char* in, unsigned in_size);
+
+ String(const String& other);
+ String& operator=(const String& other);
+
+ String& operator+=(const String& other);
+ String operator+(const String& other) const;
+
+ String(String&& other);
+ String& operator=(String&& other);
+
+ char operator[](unsigned i) const;
+ char& operator[](unsigned i);
+
+ // the only functions I'm willing to leave in the interface - available for inlining
+ const char* c_str() const { return const_cast<String*>(this)->c_str(); } // NOLINT
+ char* c_str() {
+ if(isOnStack())
+ return reinterpret_cast<char*>(buf);
+ return data.ptr;
+ }
+
+ unsigned size() const;
+ unsigned capacity() const;
+
+ int compare(const char* other, bool no_case = false) const;
+ int compare(const String& other, bool no_case = false) const;
+};
+
+DOCTEST_INTERFACE bool operator==(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator!=(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator<(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator>(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator<=(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator>=(const String& lhs, const String& rhs);
+
+DOCTEST_INTERFACE std::ostream& operator<<(std::ostream& s, const String& in);
+
+namespace Color {
+ enum Enum
+ {
+ None = 0,
+ White,
+ Red,
+ Green,
+ Blue,
+ Cyan,
+ Yellow,
+ Grey,
+
+ Bright = 0x10,
+
+ BrightRed = Bright | Red,
+ BrightGreen = Bright | Green,
+ LightGrey = Bright | Grey,
+ BrightWhite = Bright | White
+ };
+
+ DOCTEST_INTERFACE std::ostream& operator<<(std::ostream& s, Color::Enum code);
+} // namespace Color
+
+namespace assertType {
+ enum Enum
+ {
+ // macro traits
+
+ is_warn = 1,
+ is_check = 2 * is_warn,
+ is_require = 2 * is_check,
+
+ is_normal = 2 * is_require,
+ is_throws = 2 * is_normal,
+ is_throws_as = 2 * is_throws,
+ is_throws_with = 2 * is_throws_as,
+ is_nothrow = 2 * is_throws_with,
+
+ is_false = 2 * is_nothrow,
+ is_unary = 2 * is_false, // not checked anywhere - used just to distinguish the types
+
+ is_eq = 2 * is_unary,
+ is_ne = 2 * is_eq,
+
+ is_lt = 2 * is_ne,
+ is_gt = 2 * is_lt,
+
+ is_ge = 2 * is_gt,
+ is_le = 2 * is_ge,
+
+ // macro types
+
+ DT_WARN = is_normal | is_warn,
+ DT_CHECK = is_normal | is_check,
+ DT_REQUIRE = is_normal | is_require,
+
+ DT_WARN_FALSE = is_normal | is_false | is_warn,
+ DT_CHECK_FALSE = is_normal | is_false | is_check,
+ DT_REQUIRE_FALSE = is_normal | is_false | is_require,
+
+ DT_WARN_THROWS = is_throws | is_warn,
+ DT_CHECK_THROWS = is_throws | is_check,
+ DT_REQUIRE_THROWS = is_throws | is_require,
+
+ DT_WARN_THROWS_AS = is_throws_as | is_warn,
+ DT_CHECK_THROWS_AS = is_throws_as | is_check,
+ DT_REQUIRE_THROWS_AS = is_throws_as | is_require,
+
+ DT_WARN_THROWS_WITH = is_throws_with | is_warn,
+ DT_CHECK_THROWS_WITH = is_throws_with | is_check,
+ DT_REQUIRE_THROWS_WITH = is_throws_with | is_require,
+
+ DT_WARN_THROWS_WITH_AS = is_throws_with | is_throws_as | is_warn,
+ DT_CHECK_THROWS_WITH_AS = is_throws_with | is_throws_as | is_check,
+ DT_REQUIRE_THROWS_WITH_AS = is_throws_with | is_throws_as | is_require,
+
+ DT_WARN_NOTHROW = is_nothrow | is_warn,
+ DT_CHECK_NOTHROW = is_nothrow | is_check,
+ DT_REQUIRE_NOTHROW = is_nothrow | is_require,
+
+ DT_WARN_EQ = is_normal | is_eq | is_warn,
+ DT_CHECK_EQ = is_normal | is_eq | is_check,
+ DT_REQUIRE_EQ = is_normal | is_eq | is_require,
+
+ DT_WARN_NE = is_normal | is_ne | is_warn,
+ DT_CHECK_NE = is_normal | is_ne | is_check,
+ DT_REQUIRE_NE = is_normal | is_ne | is_require,
+
+ DT_WARN_GT = is_normal | is_gt | is_warn,
+ DT_CHECK_GT = is_normal | is_gt | is_check,
+ DT_REQUIRE_GT = is_normal | is_gt | is_require,
+
+ DT_WARN_LT = is_normal | is_lt | is_warn,
+ DT_CHECK_LT = is_normal | is_lt | is_check,
+ DT_REQUIRE_LT = is_normal | is_lt | is_require,
+
+ DT_WARN_GE = is_normal | is_ge | is_warn,
+ DT_CHECK_GE = is_normal | is_ge | is_check,
+ DT_REQUIRE_GE = is_normal | is_ge | is_require,
+
+ DT_WARN_LE = is_normal | is_le | is_warn,
+ DT_CHECK_LE = is_normal | is_le | is_check,
+ DT_REQUIRE_LE = is_normal | is_le | is_require,
+
+ DT_WARN_UNARY = is_normal | is_unary | is_warn,
+ DT_CHECK_UNARY = is_normal | is_unary | is_check,
+ DT_REQUIRE_UNARY = is_normal | is_unary | is_require,
+
+ DT_WARN_UNARY_FALSE = is_normal | is_false | is_unary | is_warn,
+ DT_CHECK_UNARY_FALSE = is_normal | is_false | is_unary | is_check,
+ DT_REQUIRE_UNARY_FALSE = is_normal | is_false | is_unary | is_require,
+ };
+} // namespace assertType
+
+DOCTEST_INTERFACE const char* assertString(assertType::Enum at);
+DOCTEST_INTERFACE const char* failureString(assertType::Enum at);
+DOCTEST_INTERFACE const char* skipPathFromFilename(const char* file);
+
+struct DOCTEST_INTERFACE TestCaseData
+{
+ String m_file; // the file in which the test was registered (using String - see #350)
+ unsigned m_line; // the line where the test was registered
+ const char* m_name; // name of the test case
+ const char* m_test_suite; // the test suite in which the test was added
+ const char* m_description;
+ bool m_skip;
+ bool m_no_breaks;
+ bool m_no_output;
+ bool m_may_fail;
+ bool m_should_fail;
+ int m_expected_failures;
+ double m_timeout;
+};
+
+struct DOCTEST_INTERFACE AssertData
+{
+ // common - for all asserts
+ const TestCaseData* m_test_case;
+ assertType::Enum m_at;
+ const char* m_file;
+ int m_line;
+ const char* m_expr;
+ bool m_failed;
+
+ // exception-related - for all asserts
+ bool m_threw;
+ String m_exception;
+
+ // for normal asserts
+ String m_decomp;
+
+ // for specific exception-related asserts
+ bool m_threw_as;
+ const char* m_exception_type;
+ const char* m_exception_string;
+};
+
+struct DOCTEST_INTERFACE MessageData
+{
+ String m_string;
+ const char* m_file;
+ int m_line;
+ assertType::Enum m_severity;
+};
+
+struct DOCTEST_INTERFACE SubcaseSignature
+{
+ String m_name;
+ const char* m_file;
+ int m_line;
+
+ bool operator<(const SubcaseSignature& other) const;
+};
+
+struct DOCTEST_INTERFACE IContextScope
+{
+ IContextScope();
+ virtual ~IContextScope();
+ virtual void stringify(std::ostream*) const = 0;
+};
+
+namespace detail {
+ struct DOCTEST_INTERFACE TestCase;
+} // namespace detail
+
+struct ContextOptions //!OCLINT too many fields
+{
+ std::ostream* cout; // stdout stream - std::cout by default
+ std::ostream* cerr; // stderr stream - std::cerr by default
+ String binary_name; // the test binary name
+
+ const detail::TestCase* currentTest = nullptr;
+
+ // == parameters from the command line
+ String out; // output filename
+ String order_by; // how tests should be ordered
+ unsigned rand_seed; // the seed for rand ordering
+
+ unsigned first; // the first (matching) test to be executed
+ unsigned last; // the last (matching) test to be executed
+
+ int abort_after; // stop tests after this many failed assertions
+ int subcase_filter_levels; // apply the subcase filters for the first N levels
+
+ bool success; // include successful assertions in output
+ bool case_sensitive; // if filtering should be case sensitive
+ bool exit; // if the program should be exited after the tests are ran/whatever
+ bool duration; // print the time duration of each test case
+ bool no_throw; // to skip exceptions-related assertion macros
+ bool no_exitcode; // if the framework should return 0 as the exitcode
+ bool no_run; // to not run the tests at all (can be done with an "*" exclude)
+ bool no_version; // to not print the version of the framework
+ bool no_colors; // if output to the console should be colorized
+ bool force_colors; // forces the use of colors even when a tty cannot be detected
+ bool no_breaks; // to not break into the debugger
+ bool no_skip; // don't skip test cases which are marked to be skipped
+ bool gnu_file_line; // if line numbers should be surrounded with :x: and not (x):
+ bool no_path_in_filenames; // if the path to files should be removed from the output
+ bool no_line_numbers; // if source code line numbers should be omitted from the output
+ bool no_debug_output; // no output in the debug console when a debugger is attached
+ bool no_skipped_summary; // don't print "skipped" in the summary !!! UNDOCUMENTED !!!
+ bool no_time_in_output; // omit any time/timestamps from output !!! UNDOCUMENTED !!!
+
+ bool help; // to print the help
+ bool version; // to print the version
+ bool count; // if only the count of matching tests is to be retrieved
+ bool list_test_cases; // to list all tests matching the filters
+ bool list_test_suites; // to list all suites matching the filters
+ bool list_reporters; // lists all registered reporters
+};
+
+namespace detail {
+ template <bool CONDITION, typename TYPE = void>
+ struct enable_if
+ {};
+
+ template <typename TYPE>
+ struct enable_if<true, TYPE>
+ { typedef TYPE type; };
+
+ // clang-format off
+ template<class T> struct remove_reference { typedef T type; };
+ template<class T> struct remove_reference<T&> { typedef T type; };
+ template<class T> struct remove_reference<T&&> { typedef T type; };
+
+ template<typename T, typename U = T&&> U declval(int);
+
+ template<typename T> T declval(long);
+
+ template<typename T> auto declval() DOCTEST_NOEXCEPT -> decltype(declval<T>(0)) ;
+
+ template<class T> struct is_lvalue_reference { const static bool value=false; };
+ template<class T> struct is_lvalue_reference<T&> { const static bool value=true; };
+
+ template <class T>
+ inline T&& forward(typename remove_reference<T>::type& t) DOCTEST_NOEXCEPT
+ {
+ return static_cast<T&&>(t);
+ }
+
+ template <class T>
+ inline T&& forward(typename remove_reference<T>::type&& t) DOCTEST_NOEXCEPT
+ {
+ static_assert(!is_lvalue_reference<T>::value,
+ "Can not forward an rvalue as an lvalue.");
+ return static_cast<T&&>(t);
+ }
+
+ template<class T> struct remove_const { typedef T type; };
+ template<class T> struct remove_const<const T> { typedef T type; };
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template<class T> struct is_enum : public std::is_enum<T> {};
+ template<class T> struct underlying_type : public std::underlying_type<T> {};
+#else
+ // Use compiler intrinsics
+ template<class T> struct is_enum { constexpr static bool value = __is_enum(T); };
+ template<class T> struct underlying_type { typedef __underlying_type(T) type; };
+#endif
+ // clang-format on
+
+ template <typename T>
+ struct deferred_false
+ // cppcheck-suppress unusedStructMember
+ { static const bool value = false; };
+
+ namespace has_insertion_operator_impl {
+ std::ostream &os();
+ template<class T>
+ DOCTEST_REF_WRAP(T) val();
+
+ template<class, class = void>
+ struct check {
+ static constexpr bool value = false;
+ };
+
+ template<class T>
+ struct check<T, decltype(os() << val<T>(), void())> {
+ static constexpr bool value = true;
+ };
+ } // namespace has_insertion_operator_impl
+
+ template<class T>
+ using has_insertion_operator = has_insertion_operator_impl::check<const T>;
+
+ DOCTEST_INTERFACE void my_memcpy(void* dest, const void* src, unsigned num);
+
+ DOCTEST_INTERFACE std::ostream* getTlsOss(); // returns a thread-local ostringstream
+ DOCTEST_INTERFACE String getTlsOssResult();
+
+ template <bool C>
+ struct StringMakerBase
+ {
+ template <typename T>
+ static String convert(const DOCTEST_REF_WRAP(T)) {
+ return "{?}";
+ }
+ };
+
+ template <>
+ struct StringMakerBase<true>
+ {
+ template <typename T>
+ static String convert(const DOCTEST_REF_WRAP(T) in) {
+ *getTlsOss() << in;
+ return getTlsOssResult();
+ }
+ };
+
+ DOCTEST_INTERFACE String rawMemoryToString(const void* object, unsigned size);
+
+ template <typename T>
+ String rawMemoryToString(const DOCTEST_REF_WRAP(T) object) {
+ return rawMemoryToString(&object, sizeof(object));
+ }
+
+ template <typename T>
+ const char* type_to_string() {
+ return "<>";
+ }
+} // namespace detail
+
+template <typename T>
+struct StringMaker : public detail::StringMakerBase<detail::has_insertion_operator<T>::value>
+{};
+
+template <typename T>
+struct StringMaker<T*>
+{
+ template <typename U>
+ static String convert(U* p) {
+ if(p)
+ return detail::rawMemoryToString(p);
+ return "NULL";
+ }
+};
+
+template <typename R, typename C>
+struct StringMaker<R C::*>
+{
+ static String convert(R C::*p) {
+ if(p)
+ return detail::rawMemoryToString(p);
+ return "NULL";
+ }
+};
+
+template <typename T, typename detail::enable_if<!detail::is_enum<T>::value, bool>::type = true>
+String toString(const DOCTEST_REF_WRAP(T) value) {
+ return StringMaker<T>::convert(value);
+}
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+DOCTEST_INTERFACE String toString(char* in);
+DOCTEST_INTERFACE String toString(const char* in);
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+DOCTEST_INTERFACE String toString(bool in);
+DOCTEST_INTERFACE String toString(float in);
+DOCTEST_INTERFACE String toString(double in);
+DOCTEST_INTERFACE String toString(double long in);
+
+DOCTEST_INTERFACE String toString(char in);
+DOCTEST_INTERFACE String toString(char signed in);
+DOCTEST_INTERFACE String toString(char unsigned in);
+DOCTEST_INTERFACE String toString(int short in);
+DOCTEST_INTERFACE String toString(int short unsigned in);
+DOCTEST_INTERFACE String toString(int in);
+DOCTEST_INTERFACE String toString(int unsigned in);
+DOCTEST_INTERFACE String toString(int long in);
+DOCTEST_INTERFACE String toString(int long unsigned in);
+DOCTEST_INTERFACE String toString(int long long in);
+DOCTEST_INTERFACE String toString(int long long unsigned in);
+DOCTEST_INTERFACE String toString(std::nullptr_t in);
+
+template <typename T, typename detail::enable_if<detail::is_enum<T>::value, bool>::type = true>
+String toString(const DOCTEST_REF_WRAP(T) value) {
+ typedef typename detail::underlying_type<T>::type UT;
+ return toString(static_cast<UT>(value));
+}
+
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+DOCTEST_INTERFACE String toString(const std::string& in);
+#endif // VS 2019
+
+class DOCTEST_INTERFACE Approx
+{
+public:
+ explicit Approx(double value);
+
+ Approx operator()(double value) const;
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ explicit Approx(const T& value,
+ typename detail::enable_if<std::is_constructible<double, T>::value>::type* =
+ static_cast<T*>(nullptr)) {
+ *this = Approx(static_cast<double>(value));
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ Approx& epsilon(double newEpsilon);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ typename detail::enable_if<std::is_constructible<double, T>::value, Approx&>::type epsilon(
+ const T& newEpsilon) {
+ m_epsilon = static_cast<double>(newEpsilon);
+ return *this;
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ Approx& scale(double newScale);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ typename detail::enable_if<std::is_constructible<double, T>::value, Approx&>::type scale(
+ const T& newScale) {
+ m_scale = static_cast<double>(newScale);
+ return *this;
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ // clang-format off
+ DOCTEST_INTERFACE friend bool operator==(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator==(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator!=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator!=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator<=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator<=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator>=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator>=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator< (double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator< (const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator> (double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator> (const Approx & lhs, double rhs);
+
+ DOCTEST_INTERFACE friend String toString(const Approx& in);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#define DOCTEST_APPROX_PREFIX \
+ template <typename T> friend typename detail::enable_if<std::is_constructible<double, T>::value, bool>::type
+
+ DOCTEST_APPROX_PREFIX operator==(const T& lhs, const Approx& rhs) { return operator==(double(lhs), rhs); }
+ DOCTEST_APPROX_PREFIX operator==(const Approx& lhs, const T& rhs) { return operator==(rhs, lhs); }
+ DOCTEST_APPROX_PREFIX operator!=(const T& lhs, const Approx& rhs) { return !operator==(lhs, rhs); }
+ DOCTEST_APPROX_PREFIX operator!=(const Approx& lhs, const T& rhs) { return !operator==(rhs, lhs); }
+ DOCTEST_APPROX_PREFIX operator<=(const T& lhs, const Approx& rhs) { return double(lhs) < rhs.m_value || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator<=(const Approx& lhs, const T& rhs) { return lhs.m_value < double(rhs) || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator>=(const T& lhs, const Approx& rhs) { return double(lhs) > rhs.m_value || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator>=(const Approx& lhs, const T& rhs) { return lhs.m_value > double(rhs) || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator< (const T& lhs, const Approx& rhs) { return double(lhs) < rhs.m_value && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator< (const Approx& lhs, const T& rhs) { return lhs.m_value < double(rhs) && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator> (const T& lhs, const Approx& rhs) { return double(lhs) > rhs.m_value && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator> (const Approx& lhs, const T& rhs) { return lhs.m_value > double(rhs) && lhs != rhs; }
+#undef DOCTEST_APPROX_PREFIX
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ // clang-format on
+
+private:
+ double m_epsilon;
+ double m_scale;
+ double m_value;
+};
+
+DOCTEST_INTERFACE String toString(const Approx& in);
+
+DOCTEST_INTERFACE const ContextOptions* getContextOptions();
+
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+namespace detail {
+ // clang-format off
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ template<class T> struct decay_array { typedef T type; };
+ template<class T, unsigned N> struct decay_array<T[N]> { typedef T* type; };
+ template<class T> struct decay_array<T[]> { typedef T* type; };
+
+ template<class T> struct not_char_pointer { enum { value = 1 }; };
+ template<> struct not_char_pointer<char*> { enum { value = 0 }; };
+ template<> struct not_char_pointer<const char*> { enum { value = 0 }; };
+
+ template<class T> struct can_use_op : public not_char_pointer<typename decay_array<T>::type> {};
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ // clang-format on
+
+ struct DOCTEST_INTERFACE TestFailureException
+ {
+ };
+
+ DOCTEST_INTERFACE bool checkIfShouldThrow(assertType::Enum at);
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_NORETURN
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_INTERFACE void throwException();
+
+ struct DOCTEST_INTERFACE Subcase
+ {
+ SubcaseSignature m_signature;
+ bool m_entered = false;
+
+ Subcase(const String& name, const char* file, int line);
+ ~Subcase();
+
+ operator bool() const;
+ };
+
+ template <typename L, typename R>
+ String stringifyBinaryExpr(const DOCTEST_REF_WRAP(L) lhs, const char* op,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ return toString(lhs) + op + toString(rhs);
+ }
+
+#if DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 6, 0)
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-comparison")
+#endif
+
+// This will check if there is any way it could find a operator like member or friend and uses it.
+// If not it doesn't find the operator or if the operator at global scope is defined after
+// this template, the template won't be instantiated due to SFINAE. Once the template is not
+// instantiated it can look for global operator using normal conversions.
+#define SFINAE_OP(ret,op) decltype(doctest::detail::declval<L>() op doctest::detail::declval<R>(),static_cast<ret>(0))
+
+#define DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(op, op_str, op_macro) \
+ template <typename R> \
+ DOCTEST_NOINLINE SFINAE_OP(Result,op) operator op(R&& rhs) { \
+ bool res = op_macro(doctest::detail::forward<L>(lhs), doctest::detail::forward<R>(rhs)); \
+ if(m_at & assertType::is_false) \
+ res = !res; \
+ if(!res || doctest::getContextOptions()->success) \
+ return Result(res, stringifyBinaryExpr(lhs, op_str, rhs)); \
+ return Result(res); \
+ }
+
+ // more checks could be added - like in Catch:
+ // https://github.com/catchorg/Catch2/pull/1480/files
+ // https://github.com/catchorg/Catch2/pull/1481/files
+#define DOCTEST_FORBIT_EXPRESSION(rt, op) \
+ template <typename R> \
+ rt& operator op(const R&) { \
+ static_assert(deferred_false<R>::value, \
+ "Expression Too Complex Please Rewrite As Binary Comparison!"); \
+ return *this; \
+ }
+
+ struct DOCTEST_INTERFACE Result
+ {
+ bool m_passed;
+ String m_decomp;
+
+ Result(bool passed, const String& decomposition = String());
+
+ // forbidding some expressions based on this table: https://en.cppreference.com/w/cpp/language/operator_precedence
+ DOCTEST_FORBIT_EXPRESSION(Result, &)
+ DOCTEST_FORBIT_EXPRESSION(Result, ^)
+ DOCTEST_FORBIT_EXPRESSION(Result, |)
+ DOCTEST_FORBIT_EXPRESSION(Result, &&)
+ DOCTEST_FORBIT_EXPRESSION(Result, ||)
+ DOCTEST_FORBIT_EXPRESSION(Result, ==)
+ DOCTEST_FORBIT_EXPRESSION(Result, !=)
+ DOCTEST_FORBIT_EXPRESSION(Result, <)
+ DOCTEST_FORBIT_EXPRESSION(Result, >)
+ DOCTEST_FORBIT_EXPRESSION(Result, <=)
+ DOCTEST_FORBIT_EXPRESSION(Result, >=)
+ DOCTEST_FORBIT_EXPRESSION(Result, =)
+ DOCTEST_FORBIT_EXPRESSION(Result, +=)
+ DOCTEST_FORBIT_EXPRESSION(Result, -=)
+ DOCTEST_FORBIT_EXPRESSION(Result, *=)
+ DOCTEST_FORBIT_EXPRESSION(Result, /=)
+ DOCTEST_FORBIT_EXPRESSION(Result, %=)
+ DOCTEST_FORBIT_EXPRESSION(Result, <<=)
+ DOCTEST_FORBIT_EXPRESSION(Result, >>=)
+ DOCTEST_FORBIT_EXPRESSION(Result, &=)
+ DOCTEST_FORBIT_EXPRESSION(Result, ^=)
+ DOCTEST_FORBIT_EXPRESSION(Result, |=)
+ };
+
+#ifndef DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-conversion")
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-compare")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wdouble-promotion")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wconversion")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wfloat-equal")
+
+ DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+ DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-conversion")
+ DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-compare")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wdouble-promotion")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wconversion")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal")
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+ // https://stackoverflow.com/questions/39479163 what's the difference between 4018 and 4389
+ DOCTEST_MSVC_SUPPRESS_WARNING(4388) // signed/unsigned mismatch
+ DOCTEST_MSVC_SUPPRESS_WARNING(4389) // 'operator' : signed/unsigned mismatch
+ DOCTEST_MSVC_SUPPRESS_WARNING(4018) // 'expression' : signed/unsigned mismatch
+ //DOCTEST_MSVC_SUPPRESS_WARNING(4805) // 'operation' : unsafe mix of type 'type' and type 'type' in operation
+
+#endif // DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ // clang-format off
+#ifndef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_COMPARISON_RETURN_TYPE bool
+#else // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_COMPARISON_RETURN_TYPE typename enable_if<can_use_op<L>::value || can_use_op<R>::value, bool>::type
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ inline bool eq(const char* lhs, const char* rhs) { return String(lhs) == String(rhs); }
+ inline bool ne(const char* lhs, const char* rhs) { return String(lhs) != String(rhs); }
+ inline bool lt(const char* lhs, const char* rhs) { return String(lhs) < String(rhs); }
+ inline bool gt(const char* lhs, const char* rhs) { return String(lhs) > String(rhs); }
+ inline bool le(const char* lhs, const char* rhs) { return String(lhs) <= String(rhs); }
+ inline bool ge(const char* lhs, const char* rhs) { return String(lhs) >= String(rhs); }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ // clang-format on
+
+#define DOCTEST_RELATIONAL_OP(name, op) \
+ template <typename L, typename R> \
+ DOCTEST_COMPARISON_RETURN_TYPE name(const DOCTEST_REF_WRAP(L) lhs, \
+ const DOCTEST_REF_WRAP(R) rhs) { \
+ return lhs op rhs; \
+ }
+
+ DOCTEST_RELATIONAL_OP(eq, ==)
+ DOCTEST_RELATIONAL_OP(ne, !=)
+ DOCTEST_RELATIONAL_OP(lt, <)
+ DOCTEST_RELATIONAL_OP(gt, >)
+ DOCTEST_RELATIONAL_OP(le, <=)
+ DOCTEST_RELATIONAL_OP(ge, >=)
+
+#ifndef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_CMP_EQ(l, r) l == r
+#define DOCTEST_CMP_NE(l, r) l != r
+#define DOCTEST_CMP_GT(l, r) l > r
+#define DOCTEST_CMP_LT(l, r) l < r
+#define DOCTEST_CMP_GE(l, r) l >= r
+#define DOCTEST_CMP_LE(l, r) l <= r
+#else // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_CMP_EQ(l, r) eq(l, r)
+#define DOCTEST_CMP_NE(l, r) ne(l, r)
+#define DOCTEST_CMP_GT(l, r) gt(l, r)
+#define DOCTEST_CMP_LT(l, r) lt(l, r)
+#define DOCTEST_CMP_GE(l, r) ge(l, r)
+#define DOCTEST_CMP_LE(l, r) le(l, r)
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+
+ template <typename L>
+ // cppcheck-suppress copyCtorAndEqOperator
+ struct Expression_lhs
+ {
+ L lhs;
+ assertType::Enum m_at;
+
+ explicit Expression_lhs(L&& in, assertType::Enum at)
+ : lhs(doctest::detail::forward<L>(in))
+ , m_at(at) {}
+
+ DOCTEST_NOINLINE operator Result() {
+// this is needed only foc MSVC 2015:
+// https://ci.appveyor.com/project/onqtam/doctest/builds/38181202
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4800) // 'int': forcing value to bool
+ bool res = static_cast<bool>(lhs);
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ if(m_at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ res = !res;
+
+ if(!res || getContextOptions()->success)
+ return Result(res, toString(lhs));
+ return Result(res);
+ }
+
+ /* This is required for user-defined conversions from Expression_lhs to L */
+ //operator L() const { return lhs; }
+ operator L() const { return lhs; }
+
+ // clang-format off
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(==, " == ", DOCTEST_CMP_EQ) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(!=, " != ", DOCTEST_CMP_NE) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(>, " > ", DOCTEST_CMP_GT) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(<, " < ", DOCTEST_CMP_LT) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(>=, " >= ", DOCTEST_CMP_GE) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(<=, " <= ", DOCTEST_CMP_LE) //!OCLINT bitwise operator in conditional
+ // clang-format on
+
+ // forbidding some expressions based on this table: https://en.cppreference.com/w/cpp/language/operator_precedence
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ^)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, |)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &&)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ||)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, =)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, +=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, -=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, *=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, /=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, %=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, <<=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, >>=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ^=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, |=)
+ // these 2 are unfortunate because they should be allowed - they have higher precedence over the comparisons, but the
+ // ExpressionDecomposer class uses the left shift operator to capture the left operand of the binary expression...
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, <<)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, >>)
+ };
+
+#ifndef DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+#if DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 6, 0)
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+#endif
+
+ struct DOCTEST_INTERFACE ExpressionDecomposer
+ {
+ assertType::Enum m_at;
+
+ ExpressionDecomposer(assertType::Enum at);
+
+ // The right operator for capturing expressions is "<=" instead of "<<" (based on the operator precedence table)
+ // but then there will be warnings from GCC about "-Wparentheses" and since "_Pragma()" is problematic this will stay for now...
+ // https://github.com/catchorg/Catch2/issues/870
+ // https://github.com/catchorg/Catch2/issues/565
+ template <typename L>
+ Expression_lhs<L> operator<<(L &&operand) {
+ return Expression_lhs<L>(doctest::detail::forward<L>(operand), m_at);
+ }
+ };
+
+ struct DOCTEST_INTERFACE TestSuite
+ {
+ const char* m_test_suite;
+ const char* m_description;
+ bool m_skip;
+ bool m_no_breaks;
+ bool m_no_output;
+ bool m_may_fail;
+ bool m_should_fail;
+ int m_expected_failures;
+ double m_timeout;
+
+ TestSuite& operator*(const char* in);
+
+ template <typename T>
+ TestSuite& operator*(const T& in) {
+ in.fill(*this);
+ return *this;
+ }
+ };
+
+ typedef void (*funcType)();
+
+ struct DOCTEST_INTERFACE TestCase : public TestCaseData
+ {
+ funcType m_test; // a function pointer to the test case
+
+ const char* m_type; // for templated test cases - gets appended to the real name
+ int m_template_id; // an ID used to distinguish between the different versions of a templated test case
+ String m_full_name; // contains the name (only for templated test cases!) + the template type
+
+ TestCase(funcType test, const char* file, unsigned line, const TestSuite& test_suite,
+ const char* type = "", int template_id = -1);
+
+ TestCase(const TestCase& other);
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(26434) // hides a non-virtual function
+ TestCase& operator=(const TestCase& other);
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ TestCase& operator*(const char* in);
+
+ template <typename T>
+ TestCase& operator*(const T& in) {
+ in.fill(*this);
+ return *this;
+ }
+
+ bool operator<(const TestCase& other) const;
+ };
+
+ // forward declarations of functions used by the macros
+ DOCTEST_INTERFACE int regTest(const TestCase& tc);
+ DOCTEST_INTERFACE int setTestSuite(const TestSuite& ts);
+ DOCTEST_INTERFACE bool isDebuggerActive();
+
+ template<typename T>
+ int instantiationHelper(const T&) { return 0; }
+
+ namespace binaryAssertComparison {
+ enum Enum
+ {
+ eq = 0,
+ ne,
+ gt,
+ lt,
+ ge,
+ le
+ };
+ } // namespace binaryAssertComparison
+
+ // clang-format off
+ template <int, class L, class R> struct RelationalComparator { bool operator()(const DOCTEST_REF_WRAP(L), const DOCTEST_REF_WRAP(R) ) const { return false; } };
+
+#define DOCTEST_BINARY_RELATIONAL_OP(n, op) \
+ template <class L, class R> struct RelationalComparator<n, L, R> { bool operator()(const DOCTEST_REF_WRAP(L) lhs, const DOCTEST_REF_WRAP(R) rhs) const { return op(lhs, rhs); } };
+ // clang-format on
+
+ DOCTEST_BINARY_RELATIONAL_OP(0, doctest::detail::eq)
+ DOCTEST_BINARY_RELATIONAL_OP(1, doctest::detail::ne)
+ DOCTEST_BINARY_RELATIONAL_OP(2, doctest::detail::gt)
+ DOCTEST_BINARY_RELATIONAL_OP(3, doctest::detail::lt)
+ DOCTEST_BINARY_RELATIONAL_OP(4, doctest::detail::ge)
+ DOCTEST_BINARY_RELATIONAL_OP(5, doctest::detail::le)
+
+ struct DOCTEST_INTERFACE ResultBuilder : public AssertData
+ {
+ ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr,
+ const char* exception_type = "", const char* exception_string = "");
+
+ void setResult(const Result& res);
+
+ template <int comparison, typename L, typename R>
+ DOCTEST_NOINLINE void binary_assert(const DOCTEST_REF_WRAP(L) lhs,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ m_failed = !RelationalComparator<comparison, L, R>()(lhs, rhs);
+ if(m_failed || getContextOptions()->success)
+ m_decomp = stringifyBinaryExpr(lhs, ", ", rhs);
+ }
+
+ template <typename L>
+ DOCTEST_NOINLINE void unary_assert(const DOCTEST_REF_WRAP(L) val) {
+ m_failed = !val;
+
+ if(m_at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ m_failed = !m_failed;
+
+ if(m_failed || getContextOptions()->success)
+ m_decomp = toString(val);
+ }
+
+ void translateException();
+
+ bool log();
+ void react() const;
+ };
+
+ namespace assertAction {
+ enum Enum
+ {
+ nothing = 0,
+ dbgbreak = 1,
+ shouldthrow = 2
+ };
+ } // namespace assertAction
+
+ DOCTEST_INTERFACE void failed_out_of_a_testing_context(const AssertData& ad);
+
+ DOCTEST_INTERFACE void decomp_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, Result result);
+
+#define DOCTEST_ASSERT_OUT_OF_TESTS(decomp) \
+ do { \
+ if(!is_running_in_test) { \
+ if(failed) { \
+ ResultBuilder rb(at, file, line, expr); \
+ rb.m_failed = failed; \
+ rb.m_decomp = decomp; \
+ failed_out_of_a_testing_context(rb); \
+ if(isDebuggerActive() && !getContextOptions()->no_breaks) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ if(checkIfShouldThrow(at)) \
+ throwException(); \
+ } \
+ return; \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_IN_TESTS(decomp) \
+ ResultBuilder rb(at, file, line, expr); \
+ rb.m_failed = failed; \
+ if(rb.m_failed || getContextOptions()->success) \
+ rb.m_decomp = decomp; \
+ if(rb.log()) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ if(rb.m_failed && checkIfShouldThrow(at)) \
+ throwException()
+
+ template <int comparison, typename L, typename R>
+ DOCTEST_NOINLINE void binary_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, const DOCTEST_REF_WRAP(L) lhs,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ bool failed = !RelationalComparator<comparison, L, R>()(lhs, rhs);
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(stringifyBinaryExpr(lhs, ", ", rhs));
+ DOCTEST_ASSERT_IN_TESTS(stringifyBinaryExpr(lhs, ", ", rhs));
+ }
+
+ template <typename L>
+ DOCTEST_NOINLINE void unary_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, const DOCTEST_REF_WRAP(L) val) {
+ bool failed = !val;
+
+ if(at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ failed = !failed;
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(toString(val));
+ DOCTEST_ASSERT_IN_TESTS(toString(val));
+ }
+
+ struct DOCTEST_INTERFACE IExceptionTranslator
+ {
+ IExceptionTranslator();
+ virtual ~IExceptionTranslator();
+ virtual bool translate(String&) const = 0;
+ };
+
+ template <typename T>
+ class ExceptionTranslator : public IExceptionTranslator //!OCLINT destructor of virtual class
+ {
+ public:
+ explicit ExceptionTranslator(String (*translateFunction)(T))
+ : m_translateFunction(translateFunction) {}
+
+ bool translate(String& res) const override {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ try {
+ throw; // lgtm [cpp/rethrow-no-exception]
+ // cppcheck-suppress catchExceptionByValue
+ } catch(T ex) { // NOLINT
+ res = m_translateFunction(ex); //!OCLINT parameter reassignment
+ return true;
+ } catch(...) {} //!OCLINT - empty catch statement
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ static_cast<void>(res); // to silence -Wunused-parameter
+ return false;
+ }
+
+ private:
+ String (*m_translateFunction)(T);
+ };
+
+ DOCTEST_INTERFACE void registerExceptionTranslatorImpl(const IExceptionTranslator* et);
+
+ template <bool C>
+ struct StringStreamBase
+ {
+ template <typename T>
+ static void convert(std::ostream* s, const T& in) {
+ *s << toString(in);
+ }
+
+ // always treat char* as a string in this context - no matter
+ // if DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING is defined
+ static void convert(std::ostream* s, const char* in) { *s << String(in); }
+ };
+
+ template <>
+ struct StringStreamBase<true>
+ {
+ template <typename T>
+ static void convert(std::ostream* s, const T& in) {
+ *s << in;
+ }
+ };
+
+ template <typename T>
+ struct StringStream : public StringStreamBase<has_insertion_operator<T>::value>
+ {};
+
+ template <typename T>
+ void toStream(std::ostream* s, const T& value) {
+ StringStream<T>::convert(s, value);
+ }
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char* in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, const char* in);
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ DOCTEST_INTERFACE void toStream(std::ostream* s, bool in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, float in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, double in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, double long in);
+
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char signed in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int short in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int short unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long long in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long long unsigned in);
+
+ // ContextScope base class used to allow implementing methods of ContextScope
+ // that don't depend on the template parameter in doctest.cpp.
+ class DOCTEST_INTERFACE ContextScopeBase : public IContextScope {
+ protected:
+ ContextScopeBase();
+
+ void destroy();
+ };
+
+ template <typename L> class ContextScope : public ContextScopeBase
+ {
+ const L lambda_;
+
+ public:
+ explicit ContextScope(const L &lambda) : lambda_(lambda) {}
+
+ ContextScope(ContextScope &&other) : lambda_(other.lambda_) {}
+
+ void stringify(std::ostream* s) const override { lambda_(s); }
+
+ ~ContextScope() override { destroy(); }
+ };
+
+ struct DOCTEST_INTERFACE MessageBuilder : public MessageData
+ {
+ std::ostream* m_stream;
+
+ MessageBuilder(const char* file, int line, assertType::Enum severity);
+ MessageBuilder() = delete;
+ ~MessageBuilder();
+
+ // the preferred way of chaining parameters for stringification
+ template <typename T>
+ MessageBuilder& operator,(const T& in) {
+ toStream(m_stream, in);
+ return *this;
+ }
+
+ // kept here just for backwards-compatibility - the comma operator should be preferred now
+ template <typename T>
+ MessageBuilder& operator<<(const T& in) { return this->operator,(in); }
+
+ // the `,` operator has the lowest operator precedence - if `<<` is used by the user then
+ // the `,` operator will be called last which is not what we want and thus the `*` operator
+ // is used first (has higher operator precedence compared to `<<`) so that we guarantee that
+ // an operator of the MessageBuilder class is called first before the rest of the parameters
+ template <typename T>
+ MessageBuilder& operator*(const T& in) { return this->operator,(in); }
+
+ bool log();
+ void react();
+ };
+
+ template <typename L>
+ ContextScope<L> MakeContextScope(const L &lambda) {
+ return ContextScope<L>(lambda);
+ }
+} // namespace detail
+
+#define DOCTEST_DEFINE_DECORATOR(name, type, def) \
+ struct name \
+ { \
+ type data; \
+ name(type in = def) \
+ : data(in) {} \
+ void fill(detail::TestCase& state) const { state.DOCTEST_CAT(m_, name) = data; } \
+ void fill(detail::TestSuite& state) const { state.DOCTEST_CAT(m_, name) = data; } \
+ }
+
+DOCTEST_DEFINE_DECORATOR(test_suite, const char*, "");
+DOCTEST_DEFINE_DECORATOR(description, const char*, "");
+DOCTEST_DEFINE_DECORATOR(skip, bool, true);
+DOCTEST_DEFINE_DECORATOR(no_breaks, bool, true);
+DOCTEST_DEFINE_DECORATOR(no_output, bool, true);
+DOCTEST_DEFINE_DECORATOR(timeout, double, 0);
+DOCTEST_DEFINE_DECORATOR(may_fail, bool, true);
+DOCTEST_DEFINE_DECORATOR(should_fail, bool, true);
+DOCTEST_DEFINE_DECORATOR(expected_failures, int, 0);
+
+template <typename T>
+int registerExceptionTranslator(String (*translateFunction)(T)) {
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wexit-time-destructors")
+ static detail::ExceptionTranslator<T> exceptionTranslator(translateFunction);
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ detail::registerExceptionTranslatorImpl(&exceptionTranslator);
+ return 0;
+}
+
+} // namespace doctest
+
+// in a separate namespace outside of doctest because the DOCTEST_TEST_SUITE macro
+// introduces an anonymous namespace in which getCurrentTestSuite gets overridden
+namespace doctest_detail_test_suite_ns {
+DOCTEST_INTERFACE doctest::detail::TestSuite& getCurrentTestSuite();
+} // namespace doctest_detail_test_suite_ns
+
+namespace doctest {
+#else // DOCTEST_CONFIG_DISABLE
+template <typename T>
+int registerExceptionTranslator(String (*)(T)) {
+ return 0;
+}
+#endif // DOCTEST_CONFIG_DISABLE
+
+namespace detail {
+ typedef void (*assert_handler)(const AssertData&);
+ struct ContextState;
+} // namespace detail
+
+class DOCTEST_INTERFACE Context
+{
+ detail::ContextState* p;
+
+ void parseArgs(int argc, const char* const* argv, bool withDefaults = false);
+
+public:
+ explicit Context(int argc = 0, const char* const* argv = nullptr);
+
+ ~Context();
+
+ void applyCommandLine(int argc, const char* const* argv);
+
+ void addFilter(const char* filter, const char* value);
+ void clearFilters();
+ void setOption(const char* option, int value);
+ void setOption(const char* option, const char* value);
+
+ bool shouldExit();
+
+ void setAsDefaultForAssertsOutOfTestCases();
+
+ void setAssertHandler(detail::assert_handler ah);
+
+ int run();
+};
+
+namespace TestCaseFailureReason {
+ enum Enum
+ {
+ None = 0,
+ AssertFailure = 1, // an assertion has failed in the test case
+ Exception = 2, // test case threw an exception
+ Crash = 4, // a crash...
+ TooManyFailedAsserts = 8, // the abort-after option
+ Timeout = 16, // see the timeout decorator
+ ShouldHaveFailedButDidnt = 32, // see the should_fail decorator
+ ShouldHaveFailedAndDid = 64, // see the should_fail decorator
+ DidntFailExactlyNumTimes = 128, // see the expected_failures decorator
+ FailedExactlyNumTimes = 256, // see the expected_failures decorator
+ CouldHaveFailedAndDid = 512 // see the may_fail decorator
+ };
+} // namespace TestCaseFailureReason
+
+struct DOCTEST_INTERFACE CurrentTestCaseStats
+{
+ int numAssertsCurrentTest;
+ int numAssertsFailedCurrentTest;
+ double seconds;
+ int failure_flags; // use TestCaseFailureReason::Enum
+};
+
+struct DOCTEST_INTERFACE TestCaseException
+{
+ String error_string;
+ bool is_crash;
+};
+
+struct DOCTEST_INTERFACE TestRunStats
+{
+ unsigned numTestCases;
+ unsigned numTestCasesPassingFilters;
+ unsigned numTestSuitesPassingFilters;
+ unsigned numTestCasesFailed;
+ int numAsserts;
+ int numAssertsFailed;
+};
+
+struct QueryData
+{
+ const TestRunStats* run_stats = nullptr;
+ const TestCaseData** data = nullptr;
+ unsigned num_data = 0;
+};
+
+struct DOCTEST_INTERFACE IReporter
+{
+ // The constructor has to accept "const ContextOptions&" as a single argument
+ // which has most of the options for the run + a pointer to the stdout stream
+ // Reporter(const ContextOptions& in)
+
+ // called when a query should be reported (listing test cases, printing the version, etc.)
+ virtual void report_query(const QueryData&) = 0;
+
+ // called when the whole test run starts
+ virtual void test_run_start() = 0;
+ // called when the whole test run ends (caching a pointer to the input doesn't make sense here)
+ virtual void test_run_end(const TestRunStats&) = 0;
+
+ // called when a test case is started (safe to cache a pointer to the input)
+ virtual void test_case_start(const TestCaseData&) = 0;
+ // called when a test case is reentered because of unfinished subcases (safe to cache a pointer to the input)
+ virtual void test_case_reenter(const TestCaseData&) = 0;
+ // called when a test case has ended
+ virtual void test_case_end(const CurrentTestCaseStats&) = 0;
+
+ // called when an exception is thrown from the test case (or it crashes)
+ virtual void test_case_exception(const TestCaseException&) = 0;
+
+ // called whenever a subcase is entered (don't cache pointers to the input)
+ virtual void subcase_start(const SubcaseSignature&) = 0;
+ // called whenever a subcase is exited (don't cache pointers to the input)
+ virtual void subcase_end() = 0;
+
+ // called for each assert (don't cache pointers to the input)
+ virtual void log_assert(const AssertData&) = 0;
+ // called for each message (don't cache pointers to the input)
+ virtual void log_message(const MessageData&) = 0;
+
+ // called when a test case is skipped either because it doesn't pass the filters, has a skip decorator
+ // or isn't in the execution range (between first and last) (safe to cache a pointer to the input)
+ virtual void test_case_skipped(const TestCaseData&) = 0;
+
+ // doctest will not be managing the lifetimes of reporters given to it but this would still be nice to have
+ virtual ~IReporter();
+
+ // can obtain all currently active contexts and stringify them if one wishes to do so
+ static int get_num_active_contexts();
+ static const IContextScope* const* get_active_contexts();
+
+ // can iterate through contexts which have been stringified automatically in their destructors when an exception has been thrown
+ static int get_num_stringified_contexts();
+ static const String* get_stringified_contexts();
+};
+
+namespace detail {
+ typedef IReporter* (*reporterCreatorFunc)(const ContextOptions&);
+
+ DOCTEST_INTERFACE void registerReporterImpl(const char* name, int prio, reporterCreatorFunc c, bool isReporter);
+
+ template <typename Reporter>
+ IReporter* reporterCreator(const ContextOptions& o) {
+ return new Reporter(o);
+ }
+} // namespace detail
+
+template <typename Reporter>
+int registerReporter(const char* name, int priority, bool isReporter) {
+ detail::registerReporterImpl(name, priority, detail::reporterCreator<Reporter>, isReporter);
+ return 0;
+}
+} // namespace doctest
+
+// if registering is not disabled
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+// common code in asserts - for convenience
+#define DOCTEST_ASSERT_LOG_AND_REACT(b) \
+ if(b.log()) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ b.react()
+
+#ifdef DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#define DOCTEST_WRAP_IN_TRY(x) x;
+#else // DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#define DOCTEST_WRAP_IN_TRY(x) \
+ try { \
+ x; \
+ } catch(...) { _DOCTEST_RB.translateException(); }
+#endif // DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+
+#ifdef DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+#define DOCTEST_CAST_TO_VOID(...) \
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wuseless-cast") \
+ static_cast<void>(__VA_ARGS__); \
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+#else // DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+#define DOCTEST_CAST_TO_VOID(...) __VA_ARGS__;
+#endif // DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+
+// registers the test by initializing a dummy var with a function
+#define DOCTEST_REGISTER_FUNCTION(global_prefix, f, decorators) \
+ global_prefix DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::regTest( \
+ doctest::detail::TestCase( \
+ f, __FILE__, __LINE__, \
+ doctest_detail_test_suite_ns::getCurrentTestSuite()) * \
+ decorators); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END()
+
+#define DOCTEST_IMPLEMENT_FIXTURE(der, base, func, decorators) \
+ namespace { \
+ struct der : public base \
+ { \
+ void f(); \
+ }; \
+ static void func() { \
+ der v; \
+ v.f(); \
+ } \
+ DOCTEST_REGISTER_FUNCTION(DOCTEST_EMPTY, func, decorators) \
+ } \
+ inline DOCTEST_NOINLINE void der::f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION(f, decorators) \
+ static void f(); \
+ DOCTEST_REGISTER_FUNCTION(DOCTEST_EMPTY, f, decorators) \
+ static void f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION_IN_CLASS(f, proxy, decorators) \
+ static doctest::detail::funcType proxy() { return f; } \
+ DOCTEST_REGISTER_FUNCTION(inline const, proxy(), decorators) \
+ static void f()
+
+// for registering tests
+#define DOCTEST_TEST_CASE(decorators) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), decorators)
+
+// for registering tests in classes - requires C++17 for inline variables!
+#if __cplusplus >= 201703L || (DOCTEST_MSVC >= DOCTEST_COMPILER(19, 12, 0) && _MSVC_LANG >= 201703L)
+#define DOCTEST_TEST_CASE_CLASS(decorators) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION_IN_CLASS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_PROXY_), \
+ decorators)
+#else // DOCTEST_TEST_CASE_CLASS
+#define DOCTEST_TEST_CASE_CLASS(...) \
+ TEST_CASES_CAN_BE_REGISTERED_IN_CLASSES_ONLY_IN_CPP17_MODE_OR_WITH_VS_2017_OR_NEWER
+#endif // DOCTEST_TEST_CASE_CLASS
+
+// for registering tests with a fixture
+#define DOCTEST_TEST_CASE_FIXTURE(c, decorators) \
+ DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(_DOCTEST_ANON_CLASS_), c, \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), decorators)
+
+// for converting types to strings without the <typeinfo> header and demangling
+#define DOCTEST_TYPE_TO_STRING_IMPL(...) \
+ template <> \
+ inline const char* type_to_string<__VA_ARGS__>() { \
+ return "<" #__VA_ARGS__ ">"; \
+ }
+#define DOCTEST_TYPE_TO_STRING(...) \
+ namespace doctest { namespace detail { \
+ DOCTEST_TYPE_TO_STRING_IMPL(__VA_ARGS__) \
+ } \
+ } \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, iter, func) \
+ template <typename T> \
+ static void func(); \
+ namespace { \
+ template <typename Tuple> \
+ struct iter; \
+ template <typename Type, typename... Rest> \
+ struct iter<std::tuple<Type, Rest...>> \
+ { \
+ iter(const char* file, unsigned line, int index) { \
+ doctest::detail::regTest(doctest::detail::TestCase(func<Type>, file, line, \
+ doctest_detail_test_suite_ns::getCurrentTestSuite(), \
+ doctest::detail::type_to_string<Type>(), \
+ int(line) * 1000 + index) \
+ * dec); \
+ iter<std::tuple<Rest...>>(file, line, index + 1); \
+ } \
+ }; \
+ template <> \
+ struct iter<std::tuple<>> \
+ { \
+ iter(const char*, unsigned, int) {} \
+ }; \
+ } \
+ template <typename T> \
+ static void func()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE(dec, T, id) \
+ DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, DOCTEST_CAT(id, ITERATOR), \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_))
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, anon, ...) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_CAT(anon, DUMMY)) = \
+ doctest::detail::instantiationHelper(DOCTEST_CAT(id, ITERATOR)<__VA_ARGS__>(__FILE__, __LINE__, 0));\
+ DOCTEST_GLOBAL_NO_WARNINGS_END()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), std::tuple<__VA_ARGS__>) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), __VA_ARGS__) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_IMPL(dec, T, anon, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, DOCTEST_CAT(anon, ITERATOR), anon); \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(anon, anon, std::tuple<__VA_ARGS__>) \
+ template <typename T> \
+ static void anon()
+
+#define DOCTEST_TEST_CASE_TEMPLATE(dec, T, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_IMPL(dec, T, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), __VA_ARGS__)
+
+// for subcases
+#define DOCTEST_SUBCASE(name) \
+ if(const doctest::detail::Subcase & DOCTEST_ANONYMOUS(_DOCTEST_ANON_SUBCASE_) DOCTEST_UNUSED = \
+ doctest::detail::Subcase(name, __FILE__, __LINE__))
+
+// for grouping tests in test suites by using code blocks
+#define DOCTEST_TEST_SUITE_IMPL(decorators, ns_name) \
+ namespace ns_name { namespace doctest_detail_test_suite_ns { \
+ static DOCTEST_NOINLINE doctest::detail::TestSuite& getCurrentTestSuite() { \
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4640) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wexit-time-destructors") \
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wmissing-field-initializers") \
+ static doctest::detail::TestSuite data{}; \
+ static bool inited = false; \
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP \
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP \
+ DOCTEST_GCC_SUPPRESS_WARNING_POP \
+ if(!inited) { \
+ data* decorators; \
+ inited = true; \
+ } \
+ return data; \
+ } \
+ } \
+ } \
+ namespace ns_name
+
+#define DOCTEST_TEST_SUITE(decorators) \
+ DOCTEST_TEST_SUITE_IMPL(decorators, DOCTEST_ANONYMOUS(_DOCTEST_ANON_SUITE_))
+
+// for starting a testsuite block
+#define DOCTEST_TEST_SUITE_BEGIN(decorators) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::setTestSuite(doctest::detail::TestSuite() * decorators); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for ending a testsuite block
+#define DOCTEST_TEST_SUITE_END \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::setTestSuite(doctest::detail::TestSuite() * ""); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for registering exception translators
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR_IMPL(translatorName, signature) \
+ inline doctest::String translatorName(signature); \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_)) = \
+ doctest::registerExceptionTranslator(translatorName); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ doctest::String translatorName(signature)
+
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature) \
+ DOCTEST_REGISTER_EXCEPTION_TRANSLATOR_IMPL(DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_), \
+ signature)
+
+// for registering reporters
+#define DOCTEST_REGISTER_REPORTER(name, priority, reporter) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_REPORTER_)) = \
+ doctest::registerReporter<reporter>(name, priority, true); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for registering listeners
+#define DOCTEST_REGISTER_LISTENER(name, priority, reporter) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_REPORTER_)) = \
+ doctest::registerReporter<reporter>(name, priority, false); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for logging
+#define DOCTEST_INFO(...) \
+ DOCTEST_INFO_IMPL(DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), \
+ __VA_ARGS__)
+
+#define DOCTEST_INFO_IMPL(mb_name, s_name, ...) \
+ auto DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_) = doctest::detail::MakeContextScope( \
+ [&](std::ostream* s_name) { \
+ doctest::detail::MessageBuilder mb_name(__FILE__, __LINE__, doctest::assertType::is_warn); \
+ mb_name.m_stream = s_name; \
+ mb_name * __VA_ARGS__; \
+ })
+
+#define DOCTEST_CAPTURE(x) DOCTEST_INFO(#x " := ", x)
+
+#define DOCTEST_ADD_AT_IMPL(type, file, line, mb, ...) \
+ do { \
+ doctest::detail::MessageBuilder mb(file, line, doctest::assertType::type); \
+ mb * __VA_ARGS__; \
+ DOCTEST_ASSERT_LOG_AND_REACT(mb); \
+ } while(false)
+
+// clang-format off
+#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_warn, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_check, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+#define DOCTEST_ADD_FAIL_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_require, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+// clang-format on
+
+#define DOCTEST_MESSAGE(...) DOCTEST_ADD_MESSAGE_AT(__FILE__, __LINE__, __VA_ARGS__)
+#define DOCTEST_FAIL_CHECK(...) DOCTEST_ADD_FAIL_CHECK_AT(__FILE__, __LINE__, __VA_ARGS__)
+#define DOCTEST_FAIL(...) DOCTEST_ADD_FAIL_AT(__FILE__, __LINE__, __VA_ARGS__)
+
+#define DOCTEST_TO_LVALUE(...) __VA_ARGS__ // Not removed to keep backwards compatibility.
+
+#ifndef DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_ASSERT_IMPLEMENT_2(assert_type, ...) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Woverloaded-shift-op-parentheses") \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY(_DOCTEST_RB.setResult( \
+ doctest::detail::ExpressionDecomposer(doctest::assertType::assert_type) \
+ << __VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#define DOCTEST_ASSERT_IMPLEMENT_1(assert_type, ...) \
+ do { \
+ DOCTEST_ASSERT_IMPLEMENT_2(assert_type, __VA_ARGS__); \
+ } while(false)
+
+#else // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+// necessary for <ASSERT>_MESSAGE
+#define DOCTEST_ASSERT_IMPLEMENT_2 DOCTEST_ASSERT_IMPLEMENT_1
+
+#define DOCTEST_ASSERT_IMPLEMENT_1(assert_type, ...) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Woverloaded-shift-op-parentheses") \
+ doctest::detail::decomp_assert( \
+ doctest::assertType::assert_type, __FILE__, __LINE__, #__VA_ARGS__, \
+ doctest::detail::ExpressionDecomposer(doctest::assertType::assert_type) \
+ << __VA_ARGS__) DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_WARN(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_WARN, __VA_ARGS__)
+#define DOCTEST_CHECK(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_CHECK, __VA_ARGS__)
+#define DOCTEST_REQUIRE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_REQUIRE, __VA_ARGS__)
+#define DOCTEST_WARN_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_WARN_FALSE, __VA_ARGS__)
+#define DOCTEST_CHECK_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_CHECK_FALSE, __VA_ARGS__)
+#define DOCTEST_REQUIRE_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_REQUIRE_FALSE, __VA_ARGS__)
+
+// clang-format off
+#define DOCTEST_WARN_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN, cond); } while(false)
+#define DOCTEST_CHECK_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK, cond); } while(false)
+#define DOCTEST_REQUIRE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE, cond); } while(false)
+#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN_FALSE, cond); } while(false)
+#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK_FALSE, cond); } while(false)
+#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE_FALSE, cond); } while(false)
+// clang-format on
+
+#define DOCTEST_ASSERT_THROWS_AS(expr, assert_type, message, ...) \
+ do { \
+ if(!doctest::getContextOptions()->no_throw) { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #expr, #__VA_ARGS__, message); \
+ try { \
+ DOCTEST_CAST_TO_VOID(expr) \
+ } catch(const typename doctest::detail::remove_const< \
+ typename doctest::detail::remove_reference<__VA_ARGS__>::type>::type&) { \
+ _DOCTEST_RB.translateException(); \
+ _DOCTEST_RB.m_threw_as = true; \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_THROWS_WITH(expr, expr_str, assert_type, ...) \
+ do { \
+ if(!doctest::getContextOptions()->no_throw) { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, expr_str, "", __VA_ARGS__); \
+ try { \
+ DOCTEST_CAST_TO_VOID(expr) \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_NOTHROW(assert_type, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ try { \
+ DOCTEST_CAST_TO_VOID(__VA_ARGS__) \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+// clang-format off
+#define DOCTEST_WARN_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_WARN_THROWS, "")
+#define DOCTEST_CHECK_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_CHECK_THROWS, "")
+#define DOCTEST_REQUIRE_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_REQUIRE_THROWS, "")
+
+#define DOCTEST_WARN_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_WARN_THROWS_AS, "", __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_CHECK_THROWS_AS, "", __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_REQUIRE_THROWS_AS, "", __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_WARN_THROWS_WITH, __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_CHECK_THROWS_WITH, __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_REQUIRE_THROWS_WITH, __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_WARN_THROWS_WITH_AS, message, __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_CHECK_THROWS_WITH_AS, message, __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_REQUIRE_THROWS_WITH_AS, message, __VA_ARGS__)
+
+#define DOCTEST_WARN_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_WARN_NOTHROW, __VA_ARGS__)
+#define DOCTEST_CHECK_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_CHECK_NOTHROW, __VA_ARGS__)
+#define DOCTEST_REQUIRE_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_REQUIRE_NOTHROW, __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS(expr); } while(false)
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS(expr); } while(false)
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS(expr); } while(false)
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_NOTHROW(expr); } while(false)
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_NOTHROW(expr); } while(false)
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_NOTHROW(expr); } while(false)
+// clang-format on
+
+#ifndef DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_BINARY_ASSERT(assert_type, comp, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY( \
+ _DOCTEST_RB.binary_assert<doctest::detail::binaryAssertComparison::comp>( \
+ __VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+#define DOCTEST_UNARY_ASSERT(assert_type, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY(_DOCTEST_RB.unary_assert(__VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+#else // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_BINARY_ASSERT(assert_type, comparison, ...) \
+ doctest::detail::binary_assert<doctest::detail::binaryAssertComparison::comparison>( \
+ doctest::assertType::assert_type, __FILE__, __LINE__, #__VA_ARGS__, __VA_ARGS__)
+
+#define DOCTEST_UNARY_ASSERT(assert_type, ...) \
+ doctest::detail::unary_assert(doctest::assertType::assert_type, __FILE__, __LINE__, \
+ #__VA_ARGS__, __VA_ARGS__)
+
+#endif // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_WARN_EQ(...) DOCTEST_BINARY_ASSERT(DT_WARN_EQ, eq, __VA_ARGS__)
+#define DOCTEST_CHECK_EQ(...) DOCTEST_BINARY_ASSERT(DT_CHECK_EQ, eq, __VA_ARGS__)
+#define DOCTEST_REQUIRE_EQ(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_EQ, eq, __VA_ARGS__)
+#define DOCTEST_WARN_NE(...) DOCTEST_BINARY_ASSERT(DT_WARN_NE, ne, __VA_ARGS__)
+#define DOCTEST_CHECK_NE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_NE, ne, __VA_ARGS__)
+#define DOCTEST_REQUIRE_NE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_NE, ne, __VA_ARGS__)
+#define DOCTEST_WARN_GT(...) DOCTEST_BINARY_ASSERT(DT_WARN_GT, gt, __VA_ARGS__)
+#define DOCTEST_CHECK_GT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GT, gt, __VA_ARGS__)
+#define DOCTEST_REQUIRE_GT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GT, gt, __VA_ARGS__)
+#define DOCTEST_WARN_LT(...) DOCTEST_BINARY_ASSERT(DT_WARN_LT, lt, __VA_ARGS__)
+#define DOCTEST_CHECK_LT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LT, lt, __VA_ARGS__)
+#define DOCTEST_REQUIRE_LT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LT, lt, __VA_ARGS__)
+#define DOCTEST_WARN_GE(...) DOCTEST_BINARY_ASSERT(DT_WARN_GE, ge, __VA_ARGS__)
+#define DOCTEST_CHECK_GE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GE, ge, __VA_ARGS__)
+#define DOCTEST_REQUIRE_GE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GE, ge, __VA_ARGS__)
+#define DOCTEST_WARN_LE(...) DOCTEST_BINARY_ASSERT(DT_WARN_LE, le, __VA_ARGS__)
+#define DOCTEST_CHECK_LE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LE, le, __VA_ARGS__)
+#define DOCTEST_REQUIRE_LE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LE, le, __VA_ARGS__)
+
+#define DOCTEST_WARN_UNARY(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY, __VA_ARGS__)
+#define DOCTEST_CHECK_UNARY(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY, __VA_ARGS__)
+#define DOCTEST_REQUIRE_UNARY(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY, __VA_ARGS__)
+#define DOCTEST_WARN_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY_FALSE, __VA_ARGS__)
+#define DOCTEST_CHECK_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY_FALSE, __VA_ARGS__)
+#define DOCTEST_REQUIRE_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY_FALSE, __VA_ARGS__)
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS
+
+#undef DOCTEST_WARN_THROWS
+#undef DOCTEST_CHECK_THROWS
+#undef DOCTEST_REQUIRE_THROWS
+#undef DOCTEST_WARN_THROWS_AS
+#undef DOCTEST_CHECK_THROWS_AS
+#undef DOCTEST_REQUIRE_THROWS_AS
+#undef DOCTEST_WARN_THROWS_WITH
+#undef DOCTEST_CHECK_THROWS_WITH
+#undef DOCTEST_REQUIRE_THROWS_WITH
+#undef DOCTEST_WARN_THROWS_WITH_AS
+#undef DOCTEST_CHECK_THROWS_WITH_AS
+#undef DOCTEST_REQUIRE_THROWS_WITH_AS
+#undef DOCTEST_WARN_NOTHROW
+#undef DOCTEST_CHECK_NOTHROW
+#undef DOCTEST_REQUIRE_NOTHROW
+
+#undef DOCTEST_WARN_THROWS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_MESSAGE
+#undef DOCTEST_WARN_THROWS_AS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_AS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_AS_MESSAGE
+#undef DOCTEST_WARN_THROWS_WITH_MESSAGE
+#undef DOCTEST_CHECK_THROWS_WITH_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_WITH_MESSAGE
+#undef DOCTEST_WARN_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_WARN_NOTHROW_MESSAGE
+#undef DOCTEST_CHECK_NOTHROW_MESSAGE
+#undef DOCTEST_REQUIRE_NOTHROW_MESSAGE
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#define DOCTEST_WARN_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#undef DOCTEST_REQUIRE
+#undef DOCTEST_REQUIRE_FALSE
+#undef DOCTEST_REQUIRE_MESSAGE
+#undef DOCTEST_REQUIRE_FALSE_MESSAGE
+#undef DOCTEST_REQUIRE_EQ
+#undef DOCTEST_REQUIRE_NE
+#undef DOCTEST_REQUIRE_GT
+#undef DOCTEST_REQUIRE_LT
+#undef DOCTEST_REQUIRE_GE
+#undef DOCTEST_REQUIRE_LE
+#undef DOCTEST_REQUIRE_UNARY
+#undef DOCTEST_REQUIRE_UNARY_FALSE
+
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+// =================================================================================================
+// == WHAT FOLLOWS IS VERSIONS OF THE MACROS THAT DO NOT DO ANY REGISTERING! ==
+// == THIS CAN BE ENABLED BY DEFINING DOCTEST_CONFIG_DISABLE GLOBALLY! ==
+// =================================================================================================
+#else // DOCTEST_CONFIG_DISABLE
+
+#define DOCTEST_IMPLEMENT_FIXTURE(der, base, func, name) \
+ namespace { \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ struct der : public base \
+ { void f(); }; \
+ } \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ inline void der<DOCTEST_UNUSED_TEMPLATE_TYPE>::f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION(f, name) \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ static inline void f()
+
+// for registering tests
+#define DOCTEST_TEST_CASE(name) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for registering tests in classes
+#define DOCTEST_TEST_CASE_CLASS(name) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for registering tests with a fixture
+#define DOCTEST_TEST_CASE_FIXTURE(x, name) \
+ DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(_DOCTEST_ANON_CLASS_), x, \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for converting types to strings without the <typeinfo> header and demangling
+#define DOCTEST_TYPE_TO_STRING(...) typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+#define DOCTEST_TYPE_TO_STRING_IMPL(...)
+
+// for typed tests
+#define DOCTEST_TEST_CASE_TEMPLATE(name, type, ...) \
+ template <typename type> \
+ inline void DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_)()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE(name, type, id) \
+ template <typename type> \
+ inline void DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_)()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, ...) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, ...) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for subcases
+#define DOCTEST_SUBCASE(name)
+
+// for a testsuite block
+#define DOCTEST_TEST_SUITE(name) namespace
+
+// for starting a testsuite block
+#define DOCTEST_TEST_SUITE_BEGIN(name) typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for ending a testsuite block
+#define DOCTEST_TEST_SUITE_END typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature) \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ static inline doctest::String DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_)(signature)
+
+#define DOCTEST_REGISTER_REPORTER(name, priority, reporter)
+#define DOCTEST_REGISTER_LISTENER(name, priority, reporter)
+
+#define DOCTEST_INFO(...) (static_cast<void>(0))
+#define DOCTEST_CAPTURE(x) (static_cast<void>(0))
+#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_ADD_FAIL_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_MESSAGE(...) (static_cast<void>(0))
+#define DOCTEST_FAIL_CHECK(...) (static_cast<void>(0))
+#define DOCTEST_FAIL(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN(...) (static_cast<void>(0))
+#define DOCTEST_CHECK(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_FALSE(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_EQ(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_EQ(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_EQ(...) (static_cast<void>(0))
+#define DOCTEST_WARN_NE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_GT(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_GT(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_GT(...) (static_cast<void>(0))
+#define DOCTEST_WARN_LT(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_LT(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_LT(...) (static_cast<void>(0))
+#define DOCTEST_WARN_GE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_GE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_GE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_LE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_LE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_LE(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_WARN_UNARY_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_UNARY_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_UNARY_FALSE(...) (static_cast<void>(0))
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+// clang-format off
+// KEPT FOR BACKWARDS COMPATIBILITY - FORWARDING TO THE RIGHT MACROS
+#define DOCTEST_FAST_WARN_EQ DOCTEST_WARN_EQ
+#define DOCTEST_FAST_CHECK_EQ DOCTEST_CHECK_EQ
+#define DOCTEST_FAST_REQUIRE_EQ DOCTEST_REQUIRE_EQ
+#define DOCTEST_FAST_WARN_NE DOCTEST_WARN_NE
+#define DOCTEST_FAST_CHECK_NE DOCTEST_CHECK_NE
+#define DOCTEST_FAST_REQUIRE_NE DOCTEST_REQUIRE_NE
+#define DOCTEST_FAST_WARN_GT DOCTEST_WARN_GT
+#define DOCTEST_FAST_CHECK_GT DOCTEST_CHECK_GT
+#define DOCTEST_FAST_REQUIRE_GT DOCTEST_REQUIRE_GT
+#define DOCTEST_FAST_WARN_LT DOCTEST_WARN_LT
+#define DOCTEST_FAST_CHECK_LT DOCTEST_CHECK_LT
+#define DOCTEST_FAST_REQUIRE_LT DOCTEST_REQUIRE_LT
+#define DOCTEST_FAST_WARN_GE DOCTEST_WARN_GE
+#define DOCTEST_FAST_CHECK_GE DOCTEST_CHECK_GE
+#define DOCTEST_FAST_REQUIRE_GE DOCTEST_REQUIRE_GE
+#define DOCTEST_FAST_WARN_LE DOCTEST_WARN_LE
+#define DOCTEST_FAST_CHECK_LE DOCTEST_CHECK_LE
+#define DOCTEST_FAST_REQUIRE_LE DOCTEST_REQUIRE_LE
+
+#define DOCTEST_FAST_WARN_UNARY DOCTEST_WARN_UNARY
+#define DOCTEST_FAST_CHECK_UNARY DOCTEST_CHECK_UNARY
+#define DOCTEST_FAST_REQUIRE_UNARY DOCTEST_REQUIRE_UNARY
+#define DOCTEST_FAST_WARN_UNARY_FALSE DOCTEST_WARN_UNARY_FALSE
+#define DOCTEST_FAST_CHECK_UNARY_FALSE DOCTEST_CHECK_UNARY_FALSE
+#define DOCTEST_FAST_REQUIRE_UNARY_FALSE DOCTEST_REQUIRE_UNARY_FALSE
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id,__VA_ARGS__)
+// clang-format on
+
+// BDD style macros
+// clang-format off
+#define DOCTEST_SCENARIO(name) DOCTEST_TEST_CASE(" Scenario: " name)
+#define DOCTEST_SCENARIO_CLASS(name) DOCTEST_TEST_CASE_CLASS(" Scenario: " name)
+#define DOCTEST_SCENARIO_TEMPLATE(name, T, ...) DOCTEST_TEST_CASE_TEMPLATE(" Scenario: " name, T, __VA_ARGS__)
+#define DOCTEST_SCENARIO_TEMPLATE_DEFINE(name, T, id) DOCTEST_TEST_CASE_TEMPLATE_DEFINE(" Scenario: " name, T, id)
+
+#define DOCTEST_GIVEN(name) DOCTEST_SUBCASE(" Given: " name)
+#define DOCTEST_WHEN(name) DOCTEST_SUBCASE(" When: " name)
+#define DOCTEST_AND_WHEN(name) DOCTEST_SUBCASE("And when: " name)
+#define DOCTEST_THEN(name) DOCTEST_SUBCASE(" Then: " name)
+#define DOCTEST_AND_THEN(name) DOCTEST_SUBCASE(" And: " name)
+// clang-format on
+
+// == SHORT VERSIONS OF THE MACROS
+#if !defined(DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES)
+
+#define TEST_CASE(name) DOCTEST_TEST_CASE(name)
+#define TEST_CASE_CLASS(name) DOCTEST_TEST_CASE_CLASS(name)
+#define TEST_CASE_FIXTURE(x, name) DOCTEST_TEST_CASE_FIXTURE(x, name)
+#define TYPE_TO_STRING(...) DOCTEST_TYPE_TO_STRING(__VA_ARGS__)
+#define TEST_CASE_TEMPLATE(name, T, ...) DOCTEST_TEST_CASE_TEMPLATE(name, T, __VA_ARGS__)
+#define TEST_CASE_TEMPLATE_DEFINE(name, T, id) DOCTEST_TEST_CASE_TEMPLATE_DEFINE(name, T, id)
+#define TEST_CASE_TEMPLATE_INVOKE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, __VA_ARGS__)
+#define TEST_CASE_TEMPLATE_APPLY(id, ...) DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, __VA_ARGS__)
+#define SUBCASE(name) DOCTEST_SUBCASE(name)
+#define TEST_SUITE(decorators) DOCTEST_TEST_SUITE(decorators)
+#define TEST_SUITE_BEGIN(name) DOCTEST_TEST_SUITE_BEGIN(name)
+#define TEST_SUITE_END DOCTEST_TEST_SUITE_END
+#define REGISTER_EXCEPTION_TRANSLATOR(signature) DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature)
+#define REGISTER_REPORTER(name, priority, reporter) DOCTEST_REGISTER_REPORTER(name, priority, reporter)
+#define REGISTER_LISTENER(name, priority, reporter) DOCTEST_REGISTER_LISTENER(name, priority, reporter)
+#define INFO(...) DOCTEST_INFO(__VA_ARGS__)
+#define CAPTURE(x) DOCTEST_CAPTURE(x)
+#define ADD_MESSAGE_AT(file, line, ...) DOCTEST_ADD_MESSAGE_AT(file, line, __VA_ARGS__)
+#define ADD_FAIL_CHECK_AT(file, line, ...) DOCTEST_ADD_FAIL_CHECK_AT(file, line, __VA_ARGS__)
+#define ADD_FAIL_AT(file, line, ...) DOCTEST_ADD_FAIL_AT(file, line, __VA_ARGS__)
+#define MESSAGE(...) DOCTEST_MESSAGE(__VA_ARGS__)
+#define FAIL_CHECK(...) DOCTEST_FAIL_CHECK(__VA_ARGS__)
+#define FAIL(...) DOCTEST_FAIL(__VA_ARGS__)
+#define TO_LVALUE(...) DOCTEST_TO_LVALUE(__VA_ARGS__)
+
+#define WARN(...) DOCTEST_WARN(__VA_ARGS__)
+#define WARN_FALSE(...) DOCTEST_WARN_FALSE(__VA_ARGS__)
+#define WARN_THROWS(...) DOCTEST_WARN_THROWS(__VA_ARGS__)
+#define WARN_THROWS_AS(expr, ...) DOCTEST_WARN_THROWS_AS(expr, __VA_ARGS__)
+#define WARN_THROWS_WITH(expr, ...) DOCTEST_WARN_THROWS_WITH(expr, __VA_ARGS__)
+#define WARN_THROWS_WITH_AS(expr, with, ...) DOCTEST_WARN_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define WARN_NOTHROW(...) DOCTEST_WARN_NOTHROW(__VA_ARGS__)
+#define CHECK(...) DOCTEST_CHECK(__VA_ARGS__)
+#define CHECK_FALSE(...) DOCTEST_CHECK_FALSE(__VA_ARGS__)
+#define CHECK_THROWS(...) DOCTEST_CHECK_THROWS(__VA_ARGS__)
+#define CHECK_THROWS_AS(expr, ...) DOCTEST_CHECK_THROWS_AS(expr, __VA_ARGS__)
+#define CHECK_THROWS_WITH(expr, ...) DOCTEST_CHECK_THROWS_WITH(expr, __VA_ARGS__)
+#define CHECK_THROWS_WITH_AS(expr, with, ...) DOCTEST_CHECK_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define CHECK_NOTHROW(...) DOCTEST_CHECK_NOTHROW(__VA_ARGS__)
+#define REQUIRE(...) DOCTEST_REQUIRE(__VA_ARGS__)
+#define REQUIRE_FALSE(...) DOCTEST_REQUIRE_FALSE(__VA_ARGS__)
+#define REQUIRE_THROWS(...) DOCTEST_REQUIRE_THROWS(__VA_ARGS__)
+#define REQUIRE_THROWS_AS(expr, ...) DOCTEST_REQUIRE_THROWS_AS(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH(expr, ...) DOCTEST_REQUIRE_THROWS_WITH(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_AS(expr, with, ...) DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define REQUIRE_NOTHROW(...) DOCTEST_REQUIRE_NOTHROW(__VA_ARGS__)
+
+#define WARN_MESSAGE(cond, ...) DOCTEST_WARN_MESSAGE(cond, __VA_ARGS__)
+#define WARN_FALSE_MESSAGE(cond, ...) DOCTEST_WARN_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define WARN_THROWS_MESSAGE(expr, ...) DOCTEST_WARN_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define WARN_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define WARN_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define WARN_NOTHROW_MESSAGE(expr, ...) DOCTEST_WARN_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+#define CHECK_MESSAGE(cond, ...) DOCTEST_CHECK_MESSAGE(cond, __VA_ARGS__)
+#define CHECK_FALSE_MESSAGE(cond, ...) DOCTEST_CHECK_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define CHECK_THROWS_MESSAGE(expr, ...) DOCTEST_CHECK_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define CHECK_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define CHECK_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define CHECK_NOTHROW_MESSAGE(expr, ...) DOCTEST_CHECK_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+#define REQUIRE_MESSAGE(cond, ...) DOCTEST_REQUIRE_MESSAGE(cond, __VA_ARGS__)
+#define REQUIRE_FALSE_MESSAGE(cond, ...) DOCTEST_REQUIRE_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define REQUIRE_THROWS_MESSAGE(expr, ...) DOCTEST_REQUIRE_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define REQUIRE_NOTHROW_MESSAGE(expr, ...) DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+
+#define SCENARIO(name) DOCTEST_SCENARIO(name)
+#define SCENARIO_CLASS(name) DOCTEST_SCENARIO_CLASS(name)
+#define SCENARIO_TEMPLATE(name, T, ...) DOCTEST_SCENARIO_TEMPLATE(name, T, __VA_ARGS__)
+#define SCENARIO_TEMPLATE_DEFINE(name, T, id) DOCTEST_SCENARIO_TEMPLATE_DEFINE(name, T, id)
+#define GIVEN(name) DOCTEST_GIVEN(name)
+#define WHEN(name) DOCTEST_WHEN(name)
+#define AND_WHEN(name) DOCTEST_AND_WHEN(name)
+#define THEN(name) DOCTEST_THEN(name)
+#define AND_THEN(name) DOCTEST_AND_THEN(name)
+
+#define WARN_EQ(...) DOCTEST_WARN_EQ(__VA_ARGS__)
+#define CHECK_EQ(...) DOCTEST_CHECK_EQ(__VA_ARGS__)
+#define REQUIRE_EQ(...) DOCTEST_REQUIRE_EQ(__VA_ARGS__)
+#define WARN_NE(...) DOCTEST_WARN_NE(__VA_ARGS__)
+#define CHECK_NE(...) DOCTEST_CHECK_NE(__VA_ARGS__)
+#define REQUIRE_NE(...) DOCTEST_REQUIRE_NE(__VA_ARGS__)
+#define WARN_GT(...) DOCTEST_WARN_GT(__VA_ARGS__)
+#define CHECK_GT(...) DOCTEST_CHECK_GT(__VA_ARGS__)
+#define REQUIRE_GT(...) DOCTEST_REQUIRE_GT(__VA_ARGS__)
+#define WARN_LT(...) DOCTEST_WARN_LT(__VA_ARGS__)
+#define CHECK_LT(...) DOCTEST_CHECK_LT(__VA_ARGS__)
+#define REQUIRE_LT(...) DOCTEST_REQUIRE_LT(__VA_ARGS__)
+#define WARN_GE(...) DOCTEST_WARN_GE(__VA_ARGS__)
+#define CHECK_GE(...) DOCTEST_CHECK_GE(__VA_ARGS__)
+#define REQUIRE_GE(...) DOCTEST_REQUIRE_GE(__VA_ARGS__)
+#define WARN_LE(...) DOCTEST_WARN_LE(__VA_ARGS__)
+#define CHECK_LE(...) DOCTEST_CHECK_LE(__VA_ARGS__)
+#define REQUIRE_LE(...) DOCTEST_REQUIRE_LE(__VA_ARGS__)
+#define WARN_UNARY(...) DOCTEST_WARN_UNARY(__VA_ARGS__)
+#define CHECK_UNARY(...) DOCTEST_CHECK_UNARY(__VA_ARGS__)
+#define REQUIRE_UNARY(...) DOCTEST_REQUIRE_UNARY(__VA_ARGS__)
+#define WARN_UNARY_FALSE(...) DOCTEST_WARN_UNARY_FALSE(__VA_ARGS__)
+#define CHECK_UNARY_FALSE(...) DOCTEST_CHECK_UNARY_FALSE(__VA_ARGS__)
+#define REQUIRE_UNARY_FALSE(...) DOCTEST_REQUIRE_UNARY_FALSE(__VA_ARGS__)
+
+// KEPT FOR BACKWARDS COMPATIBILITY
+#define FAST_WARN_EQ(...) DOCTEST_FAST_WARN_EQ(__VA_ARGS__)
+#define FAST_CHECK_EQ(...) DOCTEST_FAST_CHECK_EQ(__VA_ARGS__)
+#define FAST_REQUIRE_EQ(...) DOCTEST_FAST_REQUIRE_EQ(__VA_ARGS__)
+#define FAST_WARN_NE(...) DOCTEST_FAST_WARN_NE(__VA_ARGS__)
+#define FAST_CHECK_NE(...) DOCTEST_FAST_CHECK_NE(__VA_ARGS__)
+#define FAST_REQUIRE_NE(...) DOCTEST_FAST_REQUIRE_NE(__VA_ARGS__)
+#define FAST_WARN_GT(...) DOCTEST_FAST_WARN_GT(__VA_ARGS__)
+#define FAST_CHECK_GT(...) DOCTEST_FAST_CHECK_GT(__VA_ARGS__)
+#define FAST_REQUIRE_GT(...) DOCTEST_FAST_REQUIRE_GT(__VA_ARGS__)
+#define FAST_WARN_LT(...) DOCTEST_FAST_WARN_LT(__VA_ARGS__)
+#define FAST_CHECK_LT(...) DOCTEST_FAST_CHECK_LT(__VA_ARGS__)
+#define FAST_REQUIRE_LT(...) DOCTEST_FAST_REQUIRE_LT(__VA_ARGS__)
+#define FAST_WARN_GE(...) DOCTEST_FAST_WARN_GE(__VA_ARGS__)
+#define FAST_CHECK_GE(...) DOCTEST_FAST_CHECK_GE(__VA_ARGS__)
+#define FAST_REQUIRE_GE(...) DOCTEST_FAST_REQUIRE_GE(__VA_ARGS__)
+#define FAST_WARN_LE(...) DOCTEST_FAST_WARN_LE(__VA_ARGS__)
+#define FAST_CHECK_LE(...) DOCTEST_FAST_CHECK_LE(__VA_ARGS__)
+#define FAST_REQUIRE_LE(...) DOCTEST_FAST_REQUIRE_LE(__VA_ARGS__)
+
+#define FAST_WARN_UNARY(...) DOCTEST_FAST_WARN_UNARY(__VA_ARGS__)
+#define FAST_CHECK_UNARY(...) DOCTEST_FAST_CHECK_UNARY(__VA_ARGS__)
+#define FAST_REQUIRE_UNARY(...) DOCTEST_FAST_REQUIRE_UNARY(__VA_ARGS__)
+#define FAST_WARN_UNARY_FALSE(...) DOCTEST_FAST_WARN_UNARY_FALSE(__VA_ARGS__)
+#define FAST_CHECK_UNARY_FALSE(...) DOCTEST_FAST_CHECK_UNARY_FALSE(__VA_ARGS__)
+#define FAST_REQUIRE_UNARY_FALSE(...) DOCTEST_FAST_REQUIRE_UNARY_FALSE(__VA_ARGS__)
+
+#define TEST_CASE_TEMPLATE_INSTANTIATE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE(id, __VA_ARGS__)
+
+#endif // DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES
+
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+// this is here to clear the 'current test suite' for the current translation unit - at the top
+DOCTEST_TEST_SUITE_END();
+
+// add stringification for primitive/fundamental types
+namespace doctest { namespace detail {
+ DOCTEST_TYPE_TO_STRING_IMPL(bool)
+ DOCTEST_TYPE_TO_STRING_IMPL(float)
+ DOCTEST_TYPE_TO_STRING_IMPL(double)
+ DOCTEST_TYPE_TO_STRING_IMPL(long double)
+ DOCTEST_TYPE_TO_STRING_IMPL(char)
+ DOCTEST_TYPE_TO_STRING_IMPL(signed char)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned char)
+#if !DOCTEST_MSVC || defined(_NATIVE_WCHAR_T_DEFINED)
+ DOCTEST_TYPE_TO_STRING_IMPL(wchar_t)
+#endif // not MSVC or wchar_t support enabled
+ DOCTEST_TYPE_TO_STRING_IMPL(short int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned short int)
+ DOCTEST_TYPE_TO_STRING_IMPL(int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned int)
+ DOCTEST_TYPE_TO_STRING_IMPL(long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(long long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned long long int)
+}} // namespace doctest::detail
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_LIBRARY_INCLUDED
+
+#ifndef DOCTEST_SINGLE_HEADER
+#define DOCTEST_SINGLE_HEADER
+#endif // DOCTEST_SINGLE_HEADER
+
+#if defined(DOCTEST_CONFIG_IMPLEMENT) || !defined(DOCTEST_SINGLE_HEADER)
+
+#ifndef DOCTEST_SINGLE_HEADER
+#include "doctest_fwd.h"
+#endif // DOCTEST_SINGLE_HEADER
+
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-macros")
+
+#ifndef DOCTEST_LIBRARY_IMPLEMENTATION
+#define DOCTEST_LIBRARY_IMPLEMENTATION
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wpadded")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wweak-vtables")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wglobal-constructors")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-prototypes")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-conversion")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wshorten-64-to-32")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-variable-declarations")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wswitch")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wswitch-enum")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wcovered-switch-default")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-noreturn")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-local-typedef")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wdisabled-macro-expansion")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-braces")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-field-initializers")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat-pedantic")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-member-function")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wnonportable-system-include-path")
+
+DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wpragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wconversion")
+DOCTEST_GCC_SUPPRESS_WARNING("-Weffc++")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-conversion")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-overflow")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-aliasing")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-field-initializers")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-braces")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-declarations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch-enum")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch-default")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunsafe-loop-optimizations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wold-style-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-local-typedefs")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wuseless-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-function")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmultiple-inheritance")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsuggest-attribute")
+
+DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+DOCTEST_MSVC_SUPPRESS_WARNING(4616) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4619) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4996) // The compiler encountered a deprecated declaration
+DOCTEST_MSVC_SUPPRESS_WARNING(4267) // 'var' : conversion from 'x' to 'y', possible loss of data
+DOCTEST_MSVC_SUPPRESS_WARNING(4706) // assignment within conditional expression
+DOCTEST_MSVC_SUPPRESS_WARNING(4512) // 'class' : assignment operator could not be generated
+DOCTEST_MSVC_SUPPRESS_WARNING(4127) // conditional expression is constant
+DOCTEST_MSVC_SUPPRESS_WARNING(4530) // C++ exception handler used, but unwind semantics not enabled
+DOCTEST_MSVC_SUPPRESS_WARNING(4577) // 'noexcept' used with no exception handling mode specified
+DOCTEST_MSVC_SUPPRESS_WARNING(4774) // format string expected in argument is not a string literal
+DOCTEST_MSVC_SUPPRESS_WARNING(4365) // conversion from 'int' to 'unsigned', signed/unsigned mismatch
+DOCTEST_MSVC_SUPPRESS_WARNING(4820) // padding in structs
+DOCTEST_MSVC_SUPPRESS_WARNING(4640) // construction of local static object is not thread-safe
+DOCTEST_MSVC_SUPPRESS_WARNING(5039) // pointer to potentially throwing function passed to extern C
+DOCTEST_MSVC_SUPPRESS_WARNING(5045) // Spectre mitigation stuff
+DOCTEST_MSVC_SUPPRESS_WARNING(4626) // assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5027) // move assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5026) // move constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4625) // copy constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4800) // forcing value to bool 'true' or 'false' (performance warning)
+// static analysis
+DOCTEST_MSVC_SUPPRESS_WARNING(26439) // This kind of function may not throw. Declare it 'noexcept'
+DOCTEST_MSVC_SUPPRESS_WARNING(26495) // Always initialize a member variable
+DOCTEST_MSVC_SUPPRESS_WARNING(26451) // Arithmetic overflow ...
+DOCTEST_MSVC_SUPPRESS_WARNING(26444) // Avoid unnamed objects with custom construction and dtor...
+DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum'
+
+DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN
+
+// required includes - will go only in one translation unit!
+#include <ctime>
+#include <cmath>
+#include <climits>
+// borland (Embarcadero) compiler requires math.h and not cmath - https://github.com/onqtam/doctest/pull/37
+#ifdef __BORLANDC__
+#include <math.h>
+#endif // __BORLANDC__
+#include <new>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <utility>
+#include <fstream>
+#include <sstream>
+#include <iostream>
+#include <algorithm>
+#include <iomanip>
+#include <vector>
+#include <atomic>
+#include <mutex>
+#include <set>
+#include <map>
+#include <exception>
+#include <stdexcept>
+#include <csignal>
+#include <cfloat>
+#include <cctype>
+#include <cstdint>
+
+#ifdef DOCTEST_PLATFORM_MAC
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/sysctl.h>
+#endif // DOCTEST_PLATFORM_MAC
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+
+// defines for a leaner windows.h
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif // WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif // NOMINMAX
+
+// not sure what AfxWin.h is for - here I do what Catch does
+#ifdef __AFXDLL
+#include <AfxWin.h>
+#else
+#include <windows.h>
+#endif
+#include <io.h>
+
+#else // DOCTEST_PLATFORM_WINDOWS
+
+#include <sys/time.h>
+#include <unistd.h>
+
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+// this is a fix for https://github.com/onqtam/doctest/issues/348
+// https://mail.gnome.org/archives/xml/2012-January/msg00000.html
+#if !defined(HAVE_UNISTD_H) && !defined(STDOUT_FILENO)
+#define STDOUT_FILENO fileno(stdout)
+#endif // HAVE_UNISTD_H
+
+DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END
+
+// counts the number of elements in a C array
+#define DOCTEST_COUNTOF(x) (sizeof(x) / sizeof(x[0]))
+
+#ifdef DOCTEST_CONFIG_DISABLE
+#define DOCTEST_BRANCH_ON_DISABLED(if_disabled, if_not_disabled) if_disabled
+#else // DOCTEST_CONFIG_DISABLE
+#define DOCTEST_BRANCH_ON_DISABLED(if_disabled, if_not_disabled) if_not_disabled
+#endif // DOCTEST_CONFIG_DISABLE
+
+#ifndef DOCTEST_CONFIG_OPTIONS_PREFIX
+#define DOCTEST_CONFIG_OPTIONS_PREFIX "dt-"
+#endif
+
+#ifndef DOCTEST_THREAD_LOCAL
+#define DOCTEST_THREAD_LOCAL thread_local
+#endif
+
+#ifndef DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES
+#define DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES 32
+#endif
+
+#ifndef DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE
+#define DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE 64
+#endif
+
+#ifdef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+#define DOCTEST_OPTIONS_PREFIX_DISPLAY DOCTEST_CONFIG_OPTIONS_PREFIX
+#else
+#define DOCTEST_OPTIONS_PREFIX_DISPLAY ""
+#endif
+
+#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP)
+#define DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+#endif
+
+namespace doctest {
+
+bool is_running_in_test = false;
+
+namespace {
+ using namespace detail;
+ // case insensitive strcmp
+ int stricmp(const char* a, const char* b) {
+ for(;; a++, b++) {
+ const int d = tolower(*a) - tolower(*b);
+ if(d != 0 || !*a)
+ return d;
+ }
+ }
+
+ template <typename T>
+ String fpToString(T value, int precision) {
+ std::ostringstream oss;
+ oss << std::setprecision(precision) << std::fixed << value;
+ std::string d = oss.str();
+ size_t i = d.find_last_not_of('0');
+ if(i != std::string::npos && i != d.size() - 1) {
+ if(d[i] == '.')
+ i++;
+ d = d.substr(0, i + 1);
+ }
+ return d.c_str();
+ }
+
+ struct Endianness
+ {
+ enum Arch
+ {
+ Big,
+ Little
+ };
+
+ static Arch which() {
+ int x = 1;
+ // casting any data pointer to char* is allowed
+ auto ptr = reinterpret_cast<char*>(&x);
+ if(*ptr)
+ return Little;
+ return Big;
+ }
+ };
+} // namespace
+
+namespace detail {
+ void my_memcpy(void* dest, const void* src, unsigned num) { memcpy(dest, src, num); }
+
+ String rawMemoryToString(const void* object, unsigned size) {
+ // Reverse order for little endian architectures
+ int i = 0, end = static_cast<int>(size), inc = 1;
+ if(Endianness::which() == Endianness::Little) {
+ i = end - 1;
+ end = inc = -1;
+ }
+
+ unsigned const char* bytes = static_cast<unsigned const char*>(object);
+ std::ostringstream oss;
+ oss << "0x" << std::setfill('0') << std::hex;
+ for(; i != end; i += inc)
+ oss << std::setw(2) << static_cast<unsigned>(bytes[i]);
+ return oss.str().c_str();
+ }
+
+ DOCTEST_THREAD_LOCAL std::ostringstream g_oss; // NOLINT(cert-err58-cpp)
+
+ std::ostream* getTlsOss() {
+ g_oss.clear(); // there shouldn't be anything worth clearing in the flags
+ g_oss.str(""); // the slow way of resetting a string stream
+ //g_oss.seekp(0); // optimal reset - as seen here: https://stackoverflow.com/a/624291/3162383
+ return &g_oss;
+ }
+
+ String getTlsOssResult() {
+ //g_oss << std::ends; // needed - as shown here: https://stackoverflow.com/a/624291/3162383
+ return g_oss.str().c_str();
+ }
+
+#ifndef DOCTEST_CONFIG_DISABLE
+
+namespace timer_large_integer
+{
+
+#if defined(DOCTEST_PLATFORM_WINDOWS)
+ typedef ULONGLONG type;
+#else // DOCTEST_PLATFORM_WINDOWS
+ using namespace std;
+ typedef uint64_t type;
+#endif // DOCTEST_PLATFORM_WINDOWS
+}
+
+typedef timer_large_integer::type ticks_t;
+
+#ifdef DOCTEST_CONFIG_GETCURRENTTICKS
+ ticks_t getCurrentTicks() { return DOCTEST_CONFIG_GETCURRENTTICKS(); }
+#elif defined(DOCTEST_PLATFORM_WINDOWS)
+ ticks_t getCurrentTicks() {
+ static LARGE_INTEGER hz = {0}, hzo = {0};
+ if(!hz.QuadPart) {
+ QueryPerformanceFrequency(&hz);
+ QueryPerformanceCounter(&hzo);
+ }
+ LARGE_INTEGER t;
+ QueryPerformanceCounter(&t);
+ return ((t.QuadPart - hzo.QuadPart) * LONGLONG(1000000)) / hz.QuadPart;
+ }
+#else // DOCTEST_PLATFORM_WINDOWS
+ ticks_t getCurrentTicks() {
+ timeval t;
+ gettimeofday(&t, nullptr);
+ return static_cast<ticks_t>(t.tv_sec) * 1000000 + static_cast<ticks_t>(t.tv_usec);
+ }
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ struct Timer
+ {
+ void start() { m_ticks = getCurrentTicks(); }
+ unsigned int getElapsedMicroseconds() const {
+ return static_cast<unsigned int>(getCurrentTicks() - m_ticks);
+ }
+ //unsigned int getElapsedMilliseconds() const {
+ // return static_cast<unsigned int>(getElapsedMicroseconds() / 1000);
+ //}
+ double getElapsedSeconds() const { return static_cast<double>(getCurrentTicks() - m_ticks) / 1000000.0; }
+
+ private:
+ ticks_t m_ticks = 0;
+ };
+
+#ifdef DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+ template <typename T>
+ using AtomicOrMultiLaneAtomic = std::atomic<T>;
+#else // DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+ // Provides a multilane implementation of an atomic variable that supports add, sub, load,
+ // store. Instead of using a single atomic variable, this splits up into multiple ones,
+ // each sitting on a separate cache line. The goal is to provide a speedup when most
+ // operations are modifying. It achieves this with two properties:
+ //
+ // * Multiple atomics are used, so chance of congestion from the same atomic is reduced.
+ // * Each atomic sits on a separate cache line, so false sharing is reduced.
+ //
+ // The disadvantage is that there is a small overhead due to the use of TLS, and load/store
+ // is slower because all atomics have to be accessed.
+ template <typename T>
+ class MultiLaneAtomic
+ {
+ struct CacheLineAlignedAtomic
+ {
+ std::atomic<T> atomic{};
+ char padding[DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE - sizeof(std::atomic<T>)];
+ };
+ CacheLineAlignedAtomic m_atomics[DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES];
+
+ static_assert(sizeof(CacheLineAlignedAtomic) == DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE,
+ "guarantee one atomic takes exactly one cache line");
+
+ public:
+ T operator++() DOCTEST_NOEXCEPT { return fetch_add(1) + 1; }
+
+ T operator++(int) DOCTEST_NOEXCEPT { return fetch_add(1); }
+
+ T fetch_add(T arg, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ return myAtomic().fetch_add(arg, order);
+ }
+
+ T fetch_sub(T arg, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ return myAtomic().fetch_sub(arg, order);
+ }
+
+ operator T() const DOCTEST_NOEXCEPT { return load(); }
+
+ T load(std::memory_order order = std::memory_order_seq_cst) const DOCTEST_NOEXCEPT {
+ auto result = T();
+ for(auto const& c : m_atomics) {
+ result += c.atomic.load(order);
+ }
+ return result;
+ }
+
+ T operator=(T desired) DOCTEST_NOEXCEPT {
+ store(desired);
+ return desired;
+ }
+
+ void store(T desired, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ // first value becomes desired", all others become 0.
+ for(auto& c : m_atomics) {
+ c.atomic.store(desired, order);
+ desired = {};
+ }
+ }
+
+ private:
+ // Each thread has a different atomic that it operates on. If more than NumLanes threads
+ // use this, some will use the same atomic. So performance will degrate a bit, but still
+ // everything will work.
+ //
+ // The logic here is a bit tricky. The call should be as fast as possible, so that there
+ // is minimal to no overhead in determining the correct atomic for the current thread.
+ //
+ // 1. A global static counter laneCounter counts continuously up.
+ // 2. Each successive thread will use modulo operation of that counter so it gets an atomic
+ // assigned in a round-robin fashion.
+ // 3. This tlsLaneIdx is stored in the thread local data, so it is directly available with
+ // little overhead.
+ std::atomic<T>& myAtomic() DOCTEST_NOEXCEPT {
+ static std::atomic<size_t> laneCounter;
+ DOCTEST_THREAD_LOCAL size_t tlsLaneIdx =
+ laneCounter++ % DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES;
+
+ return m_atomics[tlsLaneIdx].atomic;
+ }
+ };
+
+ template <typename T>
+ using AtomicOrMultiLaneAtomic = MultiLaneAtomic<T>;
+#endif // DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+
+ // this holds both parameters from the command line and runtime data for tests
+ struct ContextState : ContextOptions, TestRunStats, CurrentTestCaseStats
+ {
+ AtomicOrMultiLaneAtomic<int> numAssertsCurrentTest_atomic;
+ AtomicOrMultiLaneAtomic<int> numAssertsFailedCurrentTest_atomic;
+
+ std::vector<std::vector<String>> filters = decltype(filters)(9); // 9 different filters
+
+ std::vector<IReporter*> reporters_currently_used;
+
+ assert_handler ah = nullptr;
+
+ Timer timer;
+
+ std::vector<String> stringifiedContexts; // logging from INFO() due to an exception
+
+ // stuff for subcases
+ std::vector<SubcaseSignature> subcasesStack;
+ std::set<decltype(subcasesStack)> subcasesPassed;
+ int subcasesCurrentMaxLevel;
+ bool should_reenter;
+ std::atomic<bool> shouldLogCurrentException;
+
+ void resetRunData() {
+ numTestCases = 0;
+ numTestCasesPassingFilters = 0;
+ numTestSuitesPassingFilters = 0;
+ numTestCasesFailed = 0;
+ numAsserts = 0;
+ numAssertsFailed = 0;
+ numAssertsCurrentTest = 0;
+ numAssertsFailedCurrentTest = 0;
+ }
+
+ void finalizeTestCaseData() {
+ seconds = timer.getElapsedSeconds();
+
+ // update the non-atomic counters
+ numAsserts += numAssertsCurrentTest_atomic;
+ numAssertsFailed += numAssertsFailedCurrentTest_atomic;
+ numAssertsCurrentTest = numAssertsCurrentTest_atomic;
+ numAssertsFailedCurrentTest = numAssertsFailedCurrentTest_atomic;
+
+ if(numAssertsFailedCurrentTest)
+ failure_flags |= TestCaseFailureReason::AssertFailure;
+
+ if(Approx(currentTest->m_timeout).epsilon(DBL_EPSILON) != 0 &&
+ Approx(seconds).epsilon(DBL_EPSILON) > currentTest->m_timeout)
+ failure_flags |= TestCaseFailureReason::Timeout;
+
+ if(currentTest->m_should_fail) {
+ if(failure_flags) {
+ failure_flags |= TestCaseFailureReason::ShouldHaveFailedAndDid;
+ } else {
+ failure_flags |= TestCaseFailureReason::ShouldHaveFailedButDidnt;
+ }
+ } else if(failure_flags && currentTest->m_may_fail) {
+ failure_flags |= TestCaseFailureReason::CouldHaveFailedAndDid;
+ } else if(currentTest->m_expected_failures > 0) {
+ if(numAssertsFailedCurrentTest == currentTest->m_expected_failures) {
+ failure_flags |= TestCaseFailureReason::FailedExactlyNumTimes;
+ } else {
+ failure_flags |= TestCaseFailureReason::DidntFailExactlyNumTimes;
+ }
+ }
+
+ bool ok_to_fail = (TestCaseFailureReason::ShouldHaveFailedAndDid & failure_flags) ||
+ (TestCaseFailureReason::CouldHaveFailedAndDid & failure_flags) ||
+ (TestCaseFailureReason::FailedExactlyNumTimes & failure_flags);
+
+ // if any subcase has failed - the whole test case has failed
+ if(failure_flags && !ok_to_fail)
+ numTestCasesFailed++;
+ }
+ };
+
+ ContextState* g_cs = nullptr;
+
+ // used to avoid locks for the debug output
+ // TODO: figure out if this is indeed necessary/correct - seems like either there still
+ // could be a race or that there wouldn't be a race even if using the context directly
+ DOCTEST_THREAD_LOCAL bool g_no_colors;
+
+#endif // DOCTEST_CONFIG_DISABLE
+} // namespace detail
+
+void String::setOnHeap() { *reinterpret_cast<unsigned char*>(&buf[last]) = 128; }
+void String::setLast(unsigned in) { buf[last] = char(in); }
+
+void String::copy(const String& other) {
+ using namespace std;
+ if(other.isOnStack()) {
+ memcpy(buf, other.buf, len);
+ } else {
+ setOnHeap();
+ data.size = other.data.size;
+ data.capacity = data.size + 1;
+ data.ptr = new char[data.capacity];
+ memcpy(data.ptr, other.data.ptr, data.size + 1);
+ }
+}
+
+String::String() {
+ buf[0] = '\0';
+ setLast();
+}
+
+String::~String() {
+ if(!isOnStack())
+ delete[] data.ptr;
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+}
+
+String::String(const char* in)
+ : String(in, strlen(in)) {}
+
+String::String(const char* in, unsigned in_size) {
+ using namespace std;
+ if(in_size <= last) {
+ memcpy(buf, in, in_size);
+ buf[in_size] = '\0';
+ setLast(last - in_size);
+ } else {
+ setOnHeap();
+ data.size = in_size;
+ data.capacity = data.size + 1;
+ data.ptr = new char[data.capacity];
+ memcpy(data.ptr, in, in_size);
+ data.ptr[in_size] = '\0';
+ }
+}
+
+String::String(const String& other) { copy(other); }
+
+String& String::operator=(const String& other) {
+ if(this != &other) {
+ if(!isOnStack())
+ delete[] data.ptr;
+
+ copy(other);
+ }
+
+ return *this;
+}
+
+String& String::operator+=(const String& other) {
+ const unsigned my_old_size = size();
+ const unsigned other_size = other.size();
+ const unsigned total_size = my_old_size + other_size;
+ using namespace std;
+ if(isOnStack()) {
+ if(total_size < len) {
+ // append to the current stack space
+ memcpy(buf + my_old_size, other.c_str(), other_size + 1);
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ setLast(last - total_size);
+ } else {
+ // alloc new chunk
+ char* temp = new char[total_size + 1];
+ // copy current data to new location before writing in the union
+ memcpy(temp, buf, my_old_size); // skip the +1 ('\0') for speed
+ // update data in union
+ setOnHeap();
+ data.size = total_size;
+ data.capacity = data.size + 1;
+ data.ptr = temp;
+ // transfer the rest of the data
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ }
+ } else {
+ if(data.capacity > total_size) {
+ // append to the current heap block
+ data.size = total_size;
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ } else {
+ // resize
+ data.capacity *= 2;
+ if(data.capacity <= total_size)
+ data.capacity = total_size + 1;
+ // alloc new chunk
+ char* temp = new char[data.capacity];
+ // copy current data to new location before releasing it
+ memcpy(temp, data.ptr, my_old_size); // skip the +1 ('\0') for speed
+ // release old chunk
+ delete[] data.ptr;
+ // update the rest of the union members
+ data.size = total_size;
+ data.ptr = temp;
+ // transfer the rest of the data
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ }
+ }
+
+ return *this;
+}
+
+// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+String String::operator+(const String& other) const { return String(*this) += other; }
+
+String::String(String&& other) {
+ using namespace std;
+ memcpy(buf, other.buf, len);
+ other.buf[0] = '\0';
+ other.setLast();
+}
+
+String& String::operator=(String&& other) {
+ using namespace std;
+ if(this != &other) {
+ if(!isOnStack())
+ delete[] data.ptr;
+ memcpy(buf, other.buf, len);
+ other.buf[0] = '\0';
+ other.setLast();
+ }
+ return *this;
+}
+
+char String::operator[](unsigned i) const {
+ return const_cast<String*>(this)->operator[](i); // NOLINT
+}
+
+char& String::operator[](unsigned i) {
+ if(isOnStack())
+ return reinterpret_cast<char*>(buf)[i];
+ return data.ptr[i];
+}
+
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wmaybe-uninitialized")
+unsigned String::size() const {
+ if(isOnStack())
+ return last - (unsigned(buf[last]) & 31); // using "last" would work only if "len" is 32
+ return data.size;
+}
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+unsigned String::capacity() const {
+ if(isOnStack())
+ return len;
+ return data.capacity;
+}
+
+int String::compare(const char* other, bool no_case) const {
+ if(no_case)
+ return doctest::stricmp(c_str(), other);
+ return std::strcmp(c_str(), other);
+}
+
+int String::compare(const String& other, bool no_case) const {
+ return compare(other.c_str(), no_case);
+}
+
+// clang-format off
+bool operator==(const String& lhs, const String& rhs) { return lhs.compare(rhs) == 0; }
+bool operator!=(const String& lhs, const String& rhs) { return lhs.compare(rhs) != 0; }
+bool operator< (const String& lhs, const String& rhs) { return lhs.compare(rhs) < 0; }
+bool operator> (const String& lhs, const String& rhs) { return lhs.compare(rhs) > 0; }
+bool operator<=(const String& lhs, const String& rhs) { return (lhs != rhs) ? lhs.compare(rhs) < 0 : true; }
+bool operator>=(const String& lhs, const String& rhs) { return (lhs != rhs) ? lhs.compare(rhs) > 0 : true; }
+// clang-format on
+
+std::ostream& operator<<(std::ostream& s, const String& in) { return s << in.c_str(); }
+
+namespace {
+ void color_to_stream(std::ostream&, Color::Enum) DOCTEST_BRANCH_ON_DISABLED({}, ;)
+} // namespace
+
+namespace Color {
+ std::ostream& operator<<(std::ostream& s, Color::Enum code) {
+ color_to_stream(s, code);
+ return s;
+ }
+} // namespace Color
+
+// clang-format off
+const char* assertString(assertType::Enum at) {
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4062) // enum 'x' in switch of enum 'y' is not handled
+ switch(at) { //!OCLINT missing default in switch statements
+ case assertType::DT_WARN : return "WARN";
+ case assertType::DT_CHECK : return "CHECK";
+ case assertType::DT_REQUIRE : return "REQUIRE";
+
+ case assertType::DT_WARN_FALSE : return "WARN_FALSE";
+ case assertType::DT_CHECK_FALSE : return "CHECK_FALSE";
+ case assertType::DT_REQUIRE_FALSE : return "REQUIRE_FALSE";
+
+ case assertType::DT_WARN_THROWS : return "WARN_THROWS";
+ case assertType::DT_CHECK_THROWS : return "CHECK_THROWS";
+ case assertType::DT_REQUIRE_THROWS : return "REQUIRE_THROWS";
+
+ case assertType::DT_WARN_THROWS_AS : return "WARN_THROWS_AS";
+ case assertType::DT_CHECK_THROWS_AS : return "CHECK_THROWS_AS";
+ case assertType::DT_REQUIRE_THROWS_AS : return "REQUIRE_THROWS_AS";
+
+ case assertType::DT_WARN_THROWS_WITH : return "WARN_THROWS_WITH";
+ case assertType::DT_CHECK_THROWS_WITH : return "CHECK_THROWS_WITH";
+ case assertType::DT_REQUIRE_THROWS_WITH : return "REQUIRE_THROWS_WITH";
+
+ case assertType::DT_WARN_THROWS_WITH_AS : return "WARN_THROWS_WITH_AS";
+ case assertType::DT_CHECK_THROWS_WITH_AS : return "CHECK_THROWS_WITH_AS";
+ case assertType::DT_REQUIRE_THROWS_WITH_AS : return "REQUIRE_THROWS_WITH_AS";
+
+ case assertType::DT_WARN_NOTHROW : return "WARN_NOTHROW";
+ case assertType::DT_CHECK_NOTHROW : return "CHECK_NOTHROW";
+ case assertType::DT_REQUIRE_NOTHROW : return "REQUIRE_NOTHROW";
+
+ case assertType::DT_WARN_EQ : return "WARN_EQ";
+ case assertType::DT_CHECK_EQ : return "CHECK_EQ";
+ case assertType::DT_REQUIRE_EQ : return "REQUIRE_EQ";
+ case assertType::DT_WARN_NE : return "WARN_NE";
+ case assertType::DT_CHECK_NE : return "CHECK_NE";
+ case assertType::DT_REQUIRE_NE : return "REQUIRE_NE";
+ case assertType::DT_WARN_GT : return "WARN_GT";
+ case assertType::DT_CHECK_GT : return "CHECK_GT";
+ case assertType::DT_REQUIRE_GT : return "REQUIRE_GT";
+ case assertType::DT_WARN_LT : return "WARN_LT";
+ case assertType::DT_CHECK_LT : return "CHECK_LT";
+ case assertType::DT_REQUIRE_LT : return "REQUIRE_LT";
+ case assertType::DT_WARN_GE : return "WARN_GE";
+ case assertType::DT_CHECK_GE : return "CHECK_GE";
+ case assertType::DT_REQUIRE_GE : return "REQUIRE_GE";
+ case assertType::DT_WARN_LE : return "WARN_LE";
+ case assertType::DT_CHECK_LE : return "CHECK_LE";
+ case assertType::DT_REQUIRE_LE : return "REQUIRE_LE";
+
+ case assertType::DT_WARN_UNARY : return "WARN_UNARY";
+ case assertType::DT_CHECK_UNARY : return "CHECK_UNARY";
+ case assertType::DT_REQUIRE_UNARY : return "REQUIRE_UNARY";
+ case assertType::DT_WARN_UNARY_FALSE : return "WARN_UNARY_FALSE";
+ case assertType::DT_CHECK_UNARY_FALSE : return "CHECK_UNARY_FALSE";
+ case assertType::DT_REQUIRE_UNARY_FALSE : return "REQUIRE_UNARY_FALSE";
+ }
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ return "";
+}
+// clang-format on
+
+const char* failureString(assertType::Enum at) {
+ if(at & assertType::is_warn) //!OCLINT bitwise operator in conditional
+ return "WARNING";
+ if(at & assertType::is_check) //!OCLINT bitwise operator in conditional
+ return "ERROR";
+ if(at & assertType::is_require) //!OCLINT bitwise operator in conditional
+ return "FATAL ERROR";
+ return "";
+}
+
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wnull-dereference")
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wnull-dereference")
+// depending on the current options this will remove the path of filenames
+const char* skipPathFromFilename(const char* file) {
+#ifndef DOCTEST_CONFIG_DISABLE
+ if(getContextOptions()->no_path_in_filenames) {
+ auto back = std::strrchr(file, '\\');
+ auto forward = std::strrchr(file, '/');
+ if(back || forward) {
+ if(back > forward)
+ forward = back;
+ return forward + 1;
+ }
+ }
+#endif // DOCTEST_CONFIG_DISABLE
+ return file;
+}
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+bool SubcaseSignature::operator<(const SubcaseSignature& other) const {
+ if(m_line != other.m_line)
+ return m_line < other.m_line;
+ if(std::strcmp(m_file, other.m_file) != 0)
+ return std::strcmp(m_file, other.m_file) < 0;
+ return m_name.compare(other.m_name) < 0;
+}
+
+IContextScope::IContextScope() = default;
+IContextScope::~IContextScope() = default;
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+String toString(char* in) { return toString(static_cast<const char*>(in)); }
+// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+String toString(const char* in) { return String("\"") + (in ? in : "{null string}") + "\""; }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+String toString(bool in) { return in ? "true" : "false"; }
+String toString(float in) { return fpToString(in, 5) + "f"; }
+String toString(double in) { return fpToString(in, 10); }
+String toString(double long in) { return fpToString(in, 15); }
+
+#define DOCTEST_TO_STRING_OVERLOAD(type, fmt) \
+ String toString(type in) { \
+ char buf[64]; \
+ std::sprintf(buf, fmt, in); \
+ return buf; \
+ }
+
+DOCTEST_TO_STRING_OVERLOAD(char, "%d")
+DOCTEST_TO_STRING_OVERLOAD(char signed, "%d")
+DOCTEST_TO_STRING_OVERLOAD(char unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int short, "%d")
+DOCTEST_TO_STRING_OVERLOAD(int short unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int, "%d")
+DOCTEST_TO_STRING_OVERLOAD(unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int long, "%ld")
+DOCTEST_TO_STRING_OVERLOAD(int long unsigned, "%lu")
+DOCTEST_TO_STRING_OVERLOAD(int long long, "%lld")
+DOCTEST_TO_STRING_OVERLOAD(int long long unsigned, "%llu")
+
+String toString(std::nullptr_t) { return "NULL"; }
+
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+String toString(const std::string& in) { return in.c_str(); }
+#endif // VS 2019
+
+Approx::Approx(double value)
+ : m_epsilon(static_cast<double>(std::numeric_limits<float>::epsilon()) * 100)
+ , m_scale(1.0)
+ , m_value(value) {}
+
+Approx Approx::operator()(double value) const {
+ Approx approx(value);
+ approx.epsilon(m_epsilon);
+ approx.scale(m_scale);
+ return approx;
+}
+
+Approx& Approx::epsilon(double newEpsilon) {
+ m_epsilon = newEpsilon;
+ return *this;
+}
+Approx& Approx::scale(double newScale) {
+ m_scale = newScale;
+ return *this;
+}
+
+bool operator==(double lhs, const Approx& rhs) {
+ // Thanks to Richard Harris for his help refining this formula
+ return std::fabs(lhs - rhs.m_value) <
+ rhs.m_epsilon * (rhs.m_scale + std::max<double>(std::fabs(lhs), std::fabs(rhs.m_value)));
+}
+bool operator==(const Approx& lhs, double rhs) { return operator==(rhs, lhs); }
+bool operator!=(double lhs, const Approx& rhs) { return !operator==(lhs, rhs); }
+bool operator!=(const Approx& lhs, double rhs) { return !operator==(rhs, lhs); }
+bool operator<=(double lhs, const Approx& rhs) { return lhs < rhs.m_value || lhs == rhs; }
+bool operator<=(const Approx& lhs, double rhs) { return lhs.m_value < rhs || lhs == rhs; }
+bool operator>=(double lhs, const Approx& rhs) { return lhs > rhs.m_value || lhs == rhs; }
+bool operator>=(const Approx& lhs, double rhs) { return lhs.m_value > rhs || lhs == rhs; }
+bool operator<(double lhs, const Approx& rhs) { return lhs < rhs.m_value && lhs != rhs; }
+bool operator<(const Approx& lhs, double rhs) { return lhs.m_value < rhs && lhs != rhs; }
+bool operator>(double lhs, const Approx& rhs) { return lhs > rhs.m_value && lhs != rhs; }
+bool operator>(const Approx& lhs, double rhs) { return lhs.m_value > rhs && lhs != rhs; }
+
+String toString(const Approx& in) {
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ return String("Approx( ") + doctest::toString(in.m_value) + " )";
+}
+const ContextOptions* getContextOptions() { return DOCTEST_BRANCH_ON_DISABLED(nullptr, g_cs); }
+
+} // namespace doctest
+
+#ifdef DOCTEST_CONFIG_DISABLE
+namespace doctest {
+Context::Context(int, const char* const*) {}
+Context::~Context() = default;
+void Context::applyCommandLine(int, const char* const*) {}
+void Context::addFilter(const char*, const char*) {}
+void Context::clearFilters() {}
+void Context::setOption(const char*, int) {}
+void Context::setOption(const char*, const char*) {}
+bool Context::shouldExit() { return false; }
+void Context::setAsDefaultForAssertsOutOfTestCases() {}
+void Context::setAssertHandler(detail::assert_handler) {}
+int Context::run() { return 0; }
+
+IReporter::~IReporter() = default;
+
+int IReporter::get_num_active_contexts() { return 0; }
+const IContextScope* const* IReporter::get_active_contexts() { return nullptr; }
+int IReporter::get_num_stringified_contexts() { return 0; }
+const String* IReporter::get_stringified_contexts() { return nullptr; }
+
+int registerReporter(const char*, int, IReporter*) { return 0; }
+
+} // namespace doctest
+#else // DOCTEST_CONFIG_DISABLE
+
+#if !defined(DOCTEST_CONFIG_COLORS_NONE)
+#if !defined(DOCTEST_CONFIG_COLORS_WINDOWS) && !defined(DOCTEST_CONFIG_COLORS_ANSI)
+#ifdef DOCTEST_PLATFORM_WINDOWS
+#define DOCTEST_CONFIG_COLORS_WINDOWS
+#else // linux
+#define DOCTEST_CONFIG_COLORS_ANSI
+#endif // platform
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS && DOCTEST_CONFIG_COLORS_ANSI
+#endif // DOCTEST_CONFIG_COLORS_NONE
+
+namespace doctest_detail_test_suite_ns {
+// holds the current test suite
+doctest::detail::TestSuite& getCurrentTestSuite() {
+ static doctest::detail::TestSuite data{};
+ return data;
+}
+} // namespace doctest_detail_test_suite_ns
+
+namespace doctest {
+namespace {
+ // the int (priority) is part of the key for automatic sorting - sadly one can register a
+ // reporter with a duplicate name and a different priority but hopefully that won't happen often :|
+ typedef std::map<std::pair<int, String>, reporterCreatorFunc> reporterMap;
+
+ reporterMap& getReporters() {
+ static reporterMap data;
+ return data;
+ }
+ reporterMap& getListeners() {
+ static reporterMap data;
+ return data;
+ }
+} // namespace
+namespace detail {
+#define DOCTEST_ITERATE_THROUGH_REPORTERS(function, ...) \
+ for(auto& curr_rep : g_cs->reporters_currently_used) \
+ curr_rep->function(__VA_ARGS__)
+
+ bool checkIfShouldThrow(assertType::Enum at) {
+ if(at & assertType::is_require) //!OCLINT bitwise operator in conditional
+ return true;
+
+ if((at & assertType::is_check) //!OCLINT bitwise operator in conditional
+ && getContextOptions()->abort_after > 0 &&
+ (g_cs->numAssertsFailed + g_cs->numAssertsFailedCurrentTest_atomic) >=
+ getContextOptions()->abort_after)
+ return true;
+
+ return false;
+ }
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_NORETURN void throwException() {
+ g_cs->shouldLogCurrentException = false;
+ throw TestFailureException();
+ } // NOLINT(cert-err60-cpp)
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ void throwException() {}
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+} // namespace detail
+
+namespace {
+ using namespace detail;
+ // matching of a string against a wildcard mask (case sensitivity configurable) taken from
+ // https://www.codeproject.com/Articles/1088/Wildcard-string-compare-globbing
+ int wildcmp(const char* str, const char* wild, bool caseSensitive) {
+ const char* cp = str;
+ const char* mp = wild;
+
+ while((*str) && (*wild != '*')) {
+ if((caseSensitive ? (*wild != *str) : (tolower(*wild) != tolower(*str))) &&
+ (*wild != '?')) {
+ return 0;
+ }
+ wild++;
+ str++;
+ }
+
+ while(*str) {
+ if(*wild == '*') {
+ if(!*++wild) {
+ return 1;
+ }
+ mp = wild;
+ cp = str + 1;
+ } else if((caseSensitive ? (*wild == *str) : (tolower(*wild) == tolower(*str))) ||
+ (*wild == '?')) {
+ wild++;
+ str++;
+ } else {
+ wild = mp; //!OCLINT parameter reassignment
+ str = cp++; //!OCLINT parameter reassignment
+ }
+ }
+
+ while(*wild == '*') {
+ wild++;
+ }
+ return !*wild;
+ }
+
+ //// C string hash function (djb2) - taken from http://www.cse.yorku.ca/~oz/hash.html
+ //unsigned hashStr(unsigned const char* str) {
+ // unsigned long hash = 5381;
+ // char c;
+ // while((c = *str++))
+ // hash = ((hash << 5) + hash) + c; // hash * 33 + c
+ // return hash;
+ //}
+
+ // checks if the name matches any of the filters (and can be configured what to do when empty)
+ bool matchesAny(const char* name, const std::vector<String>& filters, bool matchEmpty,
+ bool caseSensitive) {
+ if(filters.empty() && matchEmpty)
+ return true;
+ for(auto& curr : filters)
+ if(wildcmp(name, curr.c_str(), caseSensitive))
+ return true;
+ return false;
+ }
+} // namespace
+namespace detail {
+
+ Subcase::Subcase(const String& name, const char* file, int line)
+ : m_signature({name, file, line}) {
+ auto* s = g_cs;
+
+ // check subcase filters
+ if(s->subcasesStack.size() < size_t(s->subcase_filter_levels)) {
+ if(!matchesAny(m_signature.m_name.c_str(), s->filters[6], true, s->case_sensitive))
+ return;
+ if(matchesAny(m_signature.m_name.c_str(), s->filters[7], false, s->case_sensitive))
+ return;
+ }
+
+ // if a Subcase on the same level has already been entered
+ if(s->subcasesStack.size() < size_t(s->subcasesCurrentMaxLevel)) {
+ s->should_reenter = true;
+ return;
+ }
+
+ // push the current signature to the stack so we can check if the
+ // current stack + the current new subcase have been traversed
+ s->subcasesStack.push_back(m_signature);
+ if(s->subcasesPassed.count(s->subcasesStack) != 0) {
+ // pop - revert to previous stack since we've already passed this
+ s->subcasesStack.pop_back();
+ return;
+ }
+
+ s->subcasesCurrentMaxLevel = s->subcasesStack.size();
+ m_entered = true;
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_start, m_signature);
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+
+ Subcase::~Subcase() {
+ if(m_entered) {
+ // only mark the subcase stack as passed if no subcases have been skipped
+ if(g_cs->should_reenter == false)
+ g_cs->subcasesPassed.insert(g_cs->subcasesStack);
+ g_cs->subcasesStack.pop_back();
+
+#if defined(__cpp_lib_uncaught_exceptions) && __cpp_lib_uncaught_exceptions >= 201411L && (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200)
+ if(std::uncaught_exceptions() > 0
+#else
+ if(std::uncaught_exception()
+#endif
+ && g_cs->shouldLogCurrentException) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(
+ test_case_exception, {"exception thrown in subcase - will translate later "
+ "when the whole test case has been exited (cannot "
+ "translate while there is an active exception)",
+ false});
+ g_cs->shouldLogCurrentException = false;
+ }
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_end, DOCTEST_EMPTY);
+ }
+ }
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ Subcase::operator bool() const { return m_entered; }
+
+ Result::Result(bool passed, const String& decomposition)
+ : m_passed(passed)
+ , m_decomp(decomposition) {}
+
+ ExpressionDecomposer::ExpressionDecomposer(assertType::Enum at)
+ : m_at(at) {}
+
+ TestSuite& TestSuite::operator*(const char* in) {
+ m_test_suite = in;
+ // clear state
+ m_description = nullptr;
+ m_skip = false;
+ m_no_breaks = false;
+ m_no_output = false;
+ m_may_fail = false;
+ m_should_fail = false;
+ m_expected_failures = 0;
+ m_timeout = 0;
+ return *this;
+ }
+
+ TestCase::TestCase(funcType test, const char* file, unsigned line, const TestSuite& test_suite,
+ const char* type, int template_id) {
+ m_file = file;
+ m_line = line;
+ m_name = nullptr; // will be later overridden in operator*
+ m_test_suite = test_suite.m_test_suite;
+ m_description = test_suite.m_description;
+ m_skip = test_suite.m_skip;
+ m_no_breaks = test_suite.m_no_breaks;
+ m_no_output = test_suite.m_no_output;
+ m_may_fail = test_suite.m_may_fail;
+ m_should_fail = test_suite.m_should_fail;
+ m_expected_failures = test_suite.m_expected_failures;
+ m_timeout = test_suite.m_timeout;
+
+ m_test = test;
+ m_type = type;
+ m_template_id = template_id;
+ }
+
+ TestCase::TestCase(const TestCase& other)
+ : TestCaseData() {
+ *this = other;
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(26434) // hides a non-virtual function
+ DOCTEST_MSVC_SUPPRESS_WARNING(26437) // Do not slice
+ TestCase& TestCase::operator=(const TestCase& other) {
+ static_cast<TestCaseData&>(*this) = static_cast<const TestCaseData&>(other);
+
+ m_test = other.m_test;
+ m_type = other.m_type;
+ m_template_id = other.m_template_id;
+ m_full_name = other.m_full_name;
+
+ if(m_template_id != -1)
+ m_name = m_full_name.c_str();
+ return *this;
+ }
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ TestCase& TestCase::operator*(const char* in) {
+ m_name = in;
+ // make a new name with an appended type for templated test case
+ if(m_template_id != -1) {
+ m_full_name = String(m_name) + m_type;
+ // redirect the name to point to the newly constructed full name
+ m_name = m_full_name.c_str();
+ }
+ return *this;
+ }
+
+ bool TestCase::operator<(const TestCase& other) const {
+ // this will be used only to differentiate between test cases - not relevant for sorting
+ if(m_line != other.m_line)
+ return m_line < other.m_line;
+ const int name_cmp = strcmp(m_name, other.m_name);
+ if(name_cmp != 0)
+ return name_cmp < 0;
+ const int file_cmp = m_file.compare(other.m_file);
+ if(file_cmp != 0)
+ return file_cmp < 0;
+ return m_template_id < other.m_template_id;
+ }
+
+ // all the registered tests
+ std::set<TestCase>& getRegisteredTests() {
+ static std::set<TestCase> data;
+ return data;
+ }
+} // namespace detail
+namespace {
+ using namespace detail;
+ // for sorting tests by file/line
+ bool fileOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ // this is needed because MSVC gives different case for drive letters
+ // for __FILE__ when evaluated in a header and a source file
+ const int res = lhs->m_file.compare(rhs->m_file, bool(DOCTEST_MSVC));
+ if(res != 0)
+ return res < 0;
+ if(lhs->m_line != rhs->m_line)
+ return lhs->m_line < rhs->m_line;
+ return lhs->m_template_id < rhs->m_template_id;
+ }
+
+ // for sorting tests by suite/file/line
+ bool suiteOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ const int res = std::strcmp(lhs->m_test_suite, rhs->m_test_suite);
+ if(res != 0)
+ return res < 0;
+ return fileOrderComparator(lhs, rhs);
+ }
+
+ // for sorting tests by name/suite/file/line
+ bool nameOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ const int res = std::strcmp(lhs->m_name, rhs->m_name);
+ if(res != 0)
+ return res < 0;
+ return suiteOrderComparator(lhs, rhs);
+ }
+
+#ifdef DOCTEST_CONFIG_COLORS_WINDOWS
+ HANDLE g_stdoutHandle;
+ WORD g_origFgAttrs;
+ WORD g_origBgAttrs;
+ bool g_attrsInitted = false;
+
+ int colors_init() {
+ if(!g_attrsInitted) {
+ g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);
+ g_attrsInitted = true;
+ CONSOLE_SCREEN_BUFFER_INFO csbiInfo;
+ GetConsoleScreenBufferInfo(g_stdoutHandle, &csbiInfo);
+ g_origFgAttrs = csbiInfo.wAttributes & ~(BACKGROUND_GREEN | BACKGROUND_RED |
+ BACKGROUND_BLUE | BACKGROUND_INTENSITY);
+ g_origBgAttrs = csbiInfo.wAttributes & ~(FOREGROUND_GREEN | FOREGROUND_RED |
+ FOREGROUND_BLUE | FOREGROUND_INTENSITY);
+ }
+ return 0;
+ }
+
+ int dumy_init_console_colors = colors_init();
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ void color_to_stream(std::ostream& s, Color::Enum code) {
+ static_cast<void>(s); // for DOCTEST_CONFIG_COLORS_NONE or DOCTEST_CONFIG_COLORS_WINDOWS
+ static_cast<void>(code); // for DOCTEST_CONFIG_COLORS_NONE
+#ifdef DOCTEST_CONFIG_COLORS_ANSI
+ if(g_no_colors ||
+ (isatty(STDOUT_FILENO) == false && getContextOptions()->force_colors == false))
+ return;
+
+ auto col = "";
+ // clang-format off
+ switch(code) { //!OCLINT missing break in switch statement / unnecessary default statement in covered switch statement
+ case Color::Red: col = "[0;31m"; break;
+ case Color::Green: col = "[0;32m"; break;
+ case Color::Blue: col = "[0;34m"; break;
+ case Color::Cyan: col = "[0;36m"; break;
+ case Color::Yellow: col = "[0;33m"; break;
+ case Color::Grey: col = "[1;30m"; break;
+ case Color::LightGrey: col = "[0;37m"; break;
+ case Color::BrightRed: col = "[1;31m"; break;
+ case Color::BrightGreen: col = "[1;32m"; break;
+ case Color::BrightWhite: col = "[1;37m"; break;
+ case Color::Bright: // invalid
+ case Color::None:
+ case Color::White:
+ default: col = "[0m";
+ }
+ // clang-format on
+ s << "\033" << col;
+#endif // DOCTEST_CONFIG_COLORS_ANSI
+
+#ifdef DOCTEST_CONFIG_COLORS_WINDOWS
+ if(g_no_colors ||
+ (isatty(fileno(stdout)) == false && getContextOptions()->force_colors == false))
+ return;
+
+#define DOCTEST_SET_ATTR(x) SetConsoleTextAttribute(g_stdoutHandle, x | g_origBgAttrs)
+
+ // clang-format off
+ switch (code) {
+ case Color::White: DOCTEST_SET_ATTR(FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE); break;
+ case Color::Red: DOCTEST_SET_ATTR(FOREGROUND_RED); break;
+ case Color::Green: DOCTEST_SET_ATTR(FOREGROUND_GREEN); break;
+ case Color::Blue: DOCTEST_SET_ATTR(FOREGROUND_BLUE); break;
+ case Color::Cyan: DOCTEST_SET_ATTR(FOREGROUND_BLUE | FOREGROUND_GREEN); break;
+ case Color::Yellow: DOCTEST_SET_ATTR(FOREGROUND_RED | FOREGROUND_GREEN); break;
+ case Color::Grey: DOCTEST_SET_ATTR(0); break;
+ case Color::LightGrey: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY); break;
+ case Color::BrightRed: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_RED); break;
+ case Color::BrightGreen: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_GREEN); break;
+ case Color::BrightWhite: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE); break;
+ case Color::None:
+ case Color::Bright: // invalid
+ default: DOCTEST_SET_ATTR(g_origFgAttrs);
+ }
+ // clang-format on
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS
+ }
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+ std::vector<const IExceptionTranslator*>& getExceptionTranslators() {
+ static std::vector<const IExceptionTranslator*> data;
+ return data;
+ }
+
+ String translateActiveException() {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ String res;
+ auto& translators = getExceptionTranslators();
+ for(auto& curr : translators)
+ if(curr->translate(res))
+ return res;
+ // clang-format off
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wcatch-value")
+ try {
+ throw;
+ } catch(std::exception& ex) {
+ return ex.what();
+ } catch(std::string& msg) {
+ return msg.c_str();
+ } catch(const char* msg) {
+ return msg;
+ } catch(...) {
+ return "unknown exception";
+ }
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+// clang-format on
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ return "";
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ }
+} // namespace
+
+namespace detail {
+ // used by the macros for registering tests
+ int regTest(const TestCase& tc) {
+ getRegisteredTests().insert(tc);
+ return 0;
+ }
+
+ // sets the current test suite
+ int setTestSuite(const TestSuite& ts) {
+ doctest_detail_test_suite_ns::getCurrentTestSuite() = ts;
+ return 0;
+ }
+
+#ifdef DOCTEST_IS_DEBUGGER_ACTIVE
+ bool isDebuggerActive() { return DOCTEST_IS_DEBUGGER_ACTIVE(); }
+#else // DOCTEST_IS_DEBUGGER_ACTIVE
+#ifdef DOCTEST_PLATFORM_LINUX
+ class ErrnoGuard {
+ public:
+ ErrnoGuard() : m_oldErrno(errno) {}
+ ~ErrnoGuard() { errno = m_oldErrno; }
+ private:
+ int m_oldErrno;
+ };
+ // See the comments in Catch2 for the reasoning behind this implementation:
+ // https://github.com/catchorg/Catch2/blob/v2.13.1/include/internal/catch_debugger.cpp#L79-L102
+ bool isDebuggerActive() {
+ ErrnoGuard guard;
+ std::ifstream in("/proc/self/status");
+ for(std::string line; std::getline(in, line);) {
+ static const int PREFIX_LEN = 11;
+ if(line.compare(0, PREFIX_LEN, "TracerPid:\t") == 0) {
+ return line.length() > PREFIX_LEN && line[PREFIX_LEN] != '0';
+ }
+ }
+ return false;
+ }
+#elif defined(DOCTEST_PLATFORM_MAC)
+ // The following function is taken directly from the following technical note:
+ // https://developer.apple.com/library/archive/qa/qa1361/_index.html
+ // Returns true if the current process is being debugged (either
+ // running under the debugger or has a debugger attached post facto).
+ bool isDebuggerActive() {
+ int mib[4];
+ kinfo_proc info;
+ size_t size;
+ // Initialize the flags so that, if sysctl fails for some bizarre
+ // reason, we get a predictable result.
+ info.kp_proc.p_flag = 0;
+ // Initialize mib, which tells sysctl the info we want, in this case
+ // we're looking for information about a specific process ID.
+ mib[0] = CTL_KERN;
+ mib[1] = KERN_PROC;
+ mib[2] = KERN_PROC_PID;
+ mib[3] = getpid();
+ // Call sysctl.
+ size = sizeof(info);
+ if(sysctl(mib, DOCTEST_COUNTOF(mib), &info, &size, 0, 0) != 0) {
+ std::cerr << "\nCall to sysctl failed - unable to determine if debugger is active **\n";
+ return false;
+ }
+ // We're being debugged if the P_TRACED flag is set.
+ return ((info.kp_proc.p_flag & P_TRACED) != 0);
+ }
+#elif DOCTEST_MSVC || defined(__MINGW32__) || defined(__MINGW64__)
+ bool isDebuggerActive() { return ::IsDebuggerPresent() != 0; }
+#else
+ bool isDebuggerActive() { return false; }
+#endif // Platform
+#endif // DOCTEST_IS_DEBUGGER_ACTIVE
+
+ void registerExceptionTranslatorImpl(const IExceptionTranslator* et) {
+ if(std::find(getExceptionTranslators().begin(), getExceptionTranslators().end(), et) ==
+ getExceptionTranslators().end())
+ getExceptionTranslators().push_back(et);
+ }
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ void toStream(std::ostream* s, char* in) { *s << in; }
+ void toStream(std::ostream* s, const char* in) { *s << in; }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ void toStream(std::ostream* s, bool in) { *s << std::boolalpha << in << std::noboolalpha; }
+ void toStream(std::ostream* s, float in) { *s << in; }
+ void toStream(std::ostream* s, double in) { *s << in; }
+ void toStream(std::ostream* s, double long in) { *s << in; }
+
+ void toStream(std::ostream* s, char in) { *s << in; }
+ void toStream(std::ostream* s, char signed in) { *s << in; }
+ void toStream(std::ostream* s, char unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int short in) { *s << in; }
+ void toStream(std::ostream* s, int short unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int in) { *s << in; }
+ void toStream(std::ostream* s, int unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int long in) { *s << in; }
+ void toStream(std::ostream* s, int long unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int long long in) { *s << in; }
+ void toStream(std::ostream* s, int long long unsigned in) { *s << in; }
+
+ DOCTEST_THREAD_LOCAL std::vector<IContextScope*> g_infoContexts; // for logging with INFO()
+
+ ContextScopeBase::ContextScopeBase() {
+ g_infoContexts.push_back(this);
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+
+ // destroy cannot be inlined into the destructor because that would mean calling stringify after
+ // ContextScope has been destroyed (base class destructors run after derived class destructors).
+ // Instead, ContextScope calls this method directly from its destructor.
+ void ContextScopeBase::destroy() {
+#if defined(__cpp_lib_uncaught_exceptions) && __cpp_lib_uncaught_exceptions >= 201411L && (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200)
+ if(std::uncaught_exceptions() > 0) {
+#else
+ if(std::uncaught_exception()) {
+#endif
+ std::ostringstream s;
+ this->stringify(&s);
+ g_cs->stringifiedContexts.push_back(s.str().c_str());
+ }
+ g_infoContexts.pop_back();
+ }
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+} // namespace detail
+namespace {
+ using namespace detail;
+
+#if !defined(DOCTEST_CONFIG_POSIX_SIGNALS) && !defined(DOCTEST_CONFIG_WINDOWS_SEH)
+ struct FatalConditionHandler
+ {
+ static void reset() {}
+ static void allocateAltStackMem() {}
+ static void freeAltStackMem() {}
+ };
+#else // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+
+ void reportFatal(const std::string&);
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+
+ struct SignalDefs
+ {
+ DWORD id;
+ const char* name;
+ };
+ // There is no 1-1 mapping between signals and windows exceptions.
+ // Windows can easily distinguish between SO and SigSegV,
+ // but SigInt, SigTerm, etc are handled differently.
+ SignalDefs signalDefs[] = {
+ {static_cast<DWORD>(EXCEPTION_ILLEGAL_INSTRUCTION),
+ "SIGILL - Illegal instruction signal"},
+ {static_cast<DWORD>(EXCEPTION_STACK_OVERFLOW), "SIGSEGV - Stack overflow"},
+ {static_cast<DWORD>(EXCEPTION_ACCESS_VIOLATION),
+ "SIGSEGV - Segmentation violation signal"},
+ {static_cast<DWORD>(EXCEPTION_INT_DIVIDE_BY_ZERO), "Divide by zero error"},
+ };
+
+ struct FatalConditionHandler
+ {
+ static LONG CALLBACK handleException(PEXCEPTION_POINTERS ExceptionInfo) {
+ // Multiple threads may enter this filter/handler at once. We want the error message to be printed on the
+ // console just once no matter how many threads have crashed.
+ static std::mutex mutex;
+ static bool execute = true;
+ {
+ std::lock_guard<std::mutex> lock(mutex);
+ if(execute) {
+ bool reported = false;
+ for(size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ if(ExceptionInfo->ExceptionRecord->ExceptionCode == signalDefs[i].id) {
+ reportFatal(signalDefs[i].name);
+ reported = true;
+ break;
+ }
+ }
+ if(reported == false)
+ reportFatal("Unhandled SEH exception caught");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ }
+ execute = false;
+ }
+ std::exit(EXIT_FAILURE);
+ }
+
+ static void allocateAltStackMem() {}
+ static void freeAltStackMem() {}
+
+ FatalConditionHandler() {
+ isSet = true;
+ // 32k seems enough for doctest to handle stack overflow,
+ // but the value was found experimentally, so there is no strong guarantee
+ guaranteeSize = 32 * 1024;
+ // Register an unhandled exception filter
+ previousTop = SetUnhandledExceptionFilter(handleException);
+ // Pass in guarantee size to be filled
+ SetThreadStackGuarantee(&guaranteeSize);
+
+ // On Windows uncaught exceptions from another thread, exceptions from
+ // destructors, or calls to std::terminate are not a SEH exception
+
+ // The terminal handler gets called when:
+ // - std::terminate is called FROM THE TEST RUNNER THREAD
+ // - an exception is thrown from a destructor FROM THE TEST RUNNER THREAD
+ original_terminate_handler = std::get_terminate();
+ std::set_terminate([]() DOCTEST_NOEXCEPT {
+ reportFatal("Terminate handler called");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ std::exit(EXIT_FAILURE); // explicitly exit - otherwise the SIGABRT handler may be called as well
+ });
+
+ // SIGABRT is raised when:
+ // - std::terminate is called FROM A DIFFERENT THREAD
+ // - an exception is thrown from a destructor FROM A DIFFERENT THREAD
+ // - an uncaught exception is thrown FROM A DIFFERENT THREAD
+ prev_sigabrt_handler = std::signal(SIGABRT, [](int signal) DOCTEST_NOEXCEPT {
+ if(signal == SIGABRT) {
+ reportFatal("SIGABRT - Abort (abnormal termination) signal");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ std::exit(EXIT_FAILURE);
+ }
+ });
+
+ // The following settings are taken from google test, and more
+ // specifically from UnitTest::Run() inside of gtest.cc
+
+ // the user does not want to see pop-up dialogs about crashes
+ prev_error_mode_1 = SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT |
+ SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX);
+ // This forces the abort message to go to stderr in all circumstances.
+ prev_error_mode_2 = _set_error_mode(_OUT_TO_STDERR);
+ // In the debug version, Visual Studio pops up a separate dialog
+ // offering a choice to debug the aborted program - we want to disable that.
+ prev_abort_behavior = _set_abort_behavior(0x0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+ // In debug mode, the Windows CRT can crash with an assertion over invalid
+ // input (e.g. passing an invalid file descriptor). The default handling
+ // for these assertions is to pop up a dialog and wait for user input.
+ // Instead ask the CRT to dump such assertions to stderr non-interactively.
+ prev_report_mode = _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG);
+ prev_report_file = _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR);
+ }
+
+ static void reset() {
+ if(isSet) {
+ // Unregister handler and restore the old guarantee
+ SetUnhandledExceptionFilter(previousTop);
+ SetThreadStackGuarantee(&guaranteeSize);
+ std::set_terminate(original_terminate_handler);
+ std::signal(SIGABRT, prev_sigabrt_handler);
+ SetErrorMode(prev_error_mode_1);
+ _set_error_mode(prev_error_mode_2);
+ _set_abort_behavior(prev_abort_behavior, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+ static_cast<void>(_CrtSetReportMode(_CRT_ASSERT, prev_report_mode));
+ static_cast<void>(_CrtSetReportFile(_CRT_ASSERT, prev_report_file));
+ isSet = false;
+ }
+ }
+
+ ~FatalConditionHandler() { reset(); }
+
+ private:
+ static UINT prev_error_mode_1;
+ static int prev_error_mode_2;
+ static unsigned int prev_abort_behavior;
+ static int prev_report_mode;
+ static _HFILE prev_report_file;
+ static void (*prev_sigabrt_handler)(int);
+ static std::terminate_handler original_terminate_handler;
+ static bool isSet;
+ static ULONG guaranteeSize;
+ static LPTOP_LEVEL_EXCEPTION_FILTER previousTop;
+ };
+
+ UINT FatalConditionHandler::prev_error_mode_1;
+ int FatalConditionHandler::prev_error_mode_2;
+ unsigned int FatalConditionHandler::prev_abort_behavior;
+ int FatalConditionHandler::prev_report_mode;
+ _HFILE FatalConditionHandler::prev_report_file;
+ void (*FatalConditionHandler::prev_sigabrt_handler)(int);
+ std::terminate_handler FatalConditionHandler::original_terminate_handler;
+ bool FatalConditionHandler::isSet = false;
+ ULONG FatalConditionHandler::guaranteeSize = 0;
+ LPTOP_LEVEL_EXCEPTION_FILTER FatalConditionHandler::previousTop = nullptr;
+
+#else // DOCTEST_PLATFORM_WINDOWS
+
+ struct SignalDefs
+ {
+ int id;
+ const char* name;
+ };
+ SignalDefs signalDefs[] = {{SIGINT, "SIGINT - Terminal interrupt signal"},
+ {SIGILL, "SIGILL - Illegal instruction signal"},
+ {SIGFPE, "SIGFPE - Floating point error signal"},
+ {SIGSEGV, "SIGSEGV - Segmentation violation signal"},
+ {SIGTERM, "SIGTERM - Termination request signal"},
+ {SIGABRT, "SIGABRT - Abort (abnormal termination) signal"}};
+
+ struct FatalConditionHandler
+ {
+ static bool isSet;
+ static struct sigaction oldSigActions[DOCTEST_COUNTOF(signalDefs)];
+ static stack_t oldSigStack;
+ static size_t altStackSize;
+ static char* altStackMem;
+
+ static void handleSignal(int sig) {
+ const char* name = "<unknown signal>";
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ SignalDefs& def = signalDefs[i];
+ if(sig == def.id) {
+ name = def.name;
+ break;
+ }
+ }
+ reset();
+ reportFatal(name);
+ raise(sig);
+ }
+
+ static void allocateAltStackMem() {
+ altStackMem = new char[altStackSize];
+ }
+
+ static void freeAltStackMem() {
+ delete[] altStackMem;
+ }
+
+ FatalConditionHandler() {
+ isSet = true;
+ stack_t sigStack;
+ sigStack.ss_sp = altStackMem;
+ sigStack.ss_size = altStackSize;
+ sigStack.ss_flags = 0;
+ sigaltstack(&sigStack, &oldSigStack);
+ struct sigaction sa = {};
+ sa.sa_handler = handleSignal; // NOLINT
+ sa.sa_flags = SA_ONSTACK;
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ sigaction(signalDefs[i].id, &sa, &oldSigActions[i]);
+ }
+ }
+
+ ~FatalConditionHandler() { reset(); }
+ static void reset() {
+ if(isSet) {
+ // Set signals back to previous values -- hopefully nobody overwrote them in the meantime
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ sigaction(signalDefs[i].id, &oldSigActions[i], nullptr);
+ }
+ // Return the old stack
+ sigaltstack(&oldSigStack, nullptr);
+ isSet = false;
+ }
+ }
+ };
+
+ bool FatalConditionHandler::isSet = false;
+ struct sigaction FatalConditionHandler::oldSigActions[DOCTEST_COUNTOF(signalDefs)] = {};
+ stack_t FatalConditionHandler::oldSigStack = {};
+ size_t FatalConditionHandler::altStackSize = 4 * SIGSTKSZ;
+ char* FatalConditionHandler::altStackMem = nullptr;
+
+#endif // DOCTEST_PLATFORM_WINDOWS
+#endif // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+
+} // namespace
+
+namespace {
+ using namespace detail;
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+#define DOCTEST_OUTPUT_DEBUG_STRING(text) ::OutputDebugStringA(text)
+#else
+ // TODO: integration with XCode and other IDEs
+#define DOCTEST_OUTPUT_DEBUG_STRING(text) // NOLINT(clang-diagnostic-unused-macros)
+#endif // Platform
+
+ void addAssert(assertType::Enum at) {
+ if((at & assertType::is_warn) == 0) //!OCLINT bitwise operator in conditional
+ g_cs->numAssertsCurrentTest_atomic++;
+ }
+
+ void addFailedAssert(assertType::Enum at) {
+ if((at & assertType::is_warn) == 0) //!OCLINT bitwise operator in conditional
+ g_cs->numAssertsFailedCurrentTest_atomic++;
+ }
+
+#if defined(DOCTEST_CONFIG_POSIX_SIGNALS) || defined(DOCTEST_CONFIG_WINDOWS_SEH)
+ void reportFatal(const std::string& message) {
+ g_cs->failure_flags |= TestCaseFailureReason::Crash;
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_exception, {message.c_str(), true});
+
+ while(g_cs->subcasesStack.size()) {
+ g_cs->subcasesStack.pop_back();
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_end, DOCTEST_EMPTY);
+ }
+
+ g_cs->finalizeTestCaseData();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_end, *g_cs);
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_end, *g_cs);
+ }
+#endif // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+} // namespace
+namespace detail {
+
+ ResultBuilder::ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr,
+ const char* exception_type, const char* exception_string) {
+ m_test_case = g_cs->currentTest;
+ m_at = at;
+ m_file = file;
+ m_line = line;
+ m_expr = expr;
+ m_failed = true;
+ m_threw = false;
+ m_threw_as = false;
+ m_exception_type = exception_type;
+ m_exception_string = exception_string;
+#if DOCTEST_MSVC
+ if(m_expr[0] == ' ') // this happens when variadic macros are disabled under MSVC
+ ++m_expr;
+#endif // MSVC
+ }
+
+ void ResultBuilder::setResult(const Result& res) {
+ m_decomp = res.m_decomp;
+ m_failed = !res.m_passed;
+ }
+
+ void ResultBuilder::translateException() {
+ m_threw = true;
+ m_exception = translateActiveException();
+ }
+
+ bool ResultBuilder::log() {
+ if(m_at & assertType::is_throws) { //!OCLINT bitwise operator in conditional
+ m_failed = !m_threw;
+ } else if((m_at & assertType::is_throws_as) && (m_at & assertType::is_throws_with)) { //!OCLINT
+ m_failed = !m_threw_as || (m_exception != m_exception_string);
+ } else if(m_at & assertType::is_throws_as) { //!OCLINT bitwise operator in conditional
+ m_failed = !m_threw_as;
+ } else if(m_at & assertType::is_throws_with) { //!OCLINT bitwise operator in conditional
+ m_failed = m_exception != m_exception_string;
+ } else if(m_at & assertType::is_nothrow) { //!OCLINT bitwise operator in conditional
+ m_failed = m_threw;
+ }
+
+ if(m_exception.size())
+ m_exception = String("\"") + m_exception + "\"";
+
+ if(is_running_in_test) {
+ addAssert(m_at);
+ DOCTEST_ITERATE_THROUGH_REPORTERS(log_assert, *this);
+
+ if(m_failed)
+ addFailedAssert(m_at);
+ } else if(m_failed) {
+ failed_out_of_a_testing_context(*this);
+ }
+
+ return m_failed && isDebuggerActive() && !getContextOptions()->no_breaks &&
+ (g_cs->currentTest == nullptr || !g_cs->currentTest->m_no_breaks); // break into debugger
+ }
+
+ void ResultBuilder::react() const {
+ if(m_failed && checkIfShouldThrow(m_at))
+ throwException();
+ }
+
+ void failed_out_of_a_testing_context(const AssertData& ad) {
+ if(g_cs->ah)
+ g_cs->ah(ad);
+ else
+ std::abort();
+ }
+
+ void decomp_assert(assertType::Enum at, const char* file, int line, const char* expr,
+ Result result) {
+ bool failed = !result.m_passed;
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(result.m_decomp);
+ DOCTEST_ASSERT_IN_TESTS(result.m_decomp);
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ }
+
+ MessageBuilder::MessageBuilder(const char* file, int line, assertType::Enum severity) {
+ m_stream = getTlsOss();
+ m_file = file;
+ m_line = line;
+ m_severity = severity;
+ }
+
+ IExceptionTranslator::IExceptionTranslator() = default;
+ IExceptionTranslator::~IExceptionTranslator() = default;
+
+ bool MessageBuilder::log() {
+ m_string = getTlsOssResult();
+ DOCTEST_ITERATE_THROUGH_REPORTERS(log_message, *this);
+
+ const bool isWarn = m_severity & assertType::is_warn;
+
+ // warn is just a message in this context so we don't treat it as an assert
+ if(!isWarn) {
+ addAssert(m_severity);
+ addFailedAssert(m_severity);
+ }
+
+ return isDebuggerActive() && !getContextOptions()->no_breaks && !isWarn &&
+ (g_cs->currentTest == nullptr || !g_cs->currentTest->m_no_breaks); // break into debugger
+ }
+
+ void MessageBuilder::react() {
+ if(m_severity & assertType::is_require) //!OCLINT bitwise operator in conditional
+ throwException();
+ }
+
+ MessageBuilder::~MessageBuilder() = default;
+} // namespace detail
+namespace {
+ using namespace detail;
+
+ template <typename Ex>
+ DOCTEST_NORETURN void throw_exception(Ex const& e) {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ throw e;
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ std::cerr << "doctest will terminate because it needed to throw an exception.\n"
+ << "The message was: " << e.what() << '\n';
+ std::terminate();
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ }
+
+#ifndef DOCTEST_INTERNAL_ERROR
+#define DOCTEST_INTERNAL_ERROR(msg) \
+ throw_exception(std::logic_error( \
+ __FILE__ ":" DOCTEST_TOSTR(__LINE__) ": Internal doctest error: " msg))
+#endif // DOCTEST_INTERNAL_ERROR
+
+ // clang-format off
+
+// =================================================================================================
+// The following code has been taken verbatim from Catch2/include/internal/catch_xmlwriter.h/cpp
+// This is done so cherry-picking bug fixes is trivial - even the style/formatting is untouched.
+// =================================================================================================
+
+ class XmlEncode {
+ public:
+ enum ForWhat { ForTextNodes, ForAttributes };
+
+ XmlEncode( std::string const& str, ForWhat forWhat = ForTextNodes );
+
+ void encodeTo( std::ostream& os ) const;
+
+ friend std::ostream& operator << ( std::ostream& os, XmlEncode const& xmlEncode );
+
+ private:
+ std::string m_str;
+ ForWhat m_forWhat;
+ };
+
+ class XmlWriter {
+ public:
+
+ class ScopedElement {
+ public:
+ ScopedElement( XmlWriter* writer );
+
+ ScopedElement( ScopedElement&& other ) DOCTEST_NOEXCEPT;
+ ScopedElement& operator=( ScopedElement&& other ) DOCTEST_NOEXCEPT;
+
+ ~ScopedElement();
+
+ ScopedElement& writeText( std::string const& text, bool indent = true );
+
+ template<typename T>
+ ScopedElement& writeAttribute( std::string const& name, T const& attribute ) {
+ m_writer->writeAttribute( name, attribute );
+ return *this;
+ }
+
+ private:
+ mutable XmlWriter* m_writer = nullptr;
+ };
+
+ XmlWriter( std::ostream& os = std::cout );
+ ~XmlWriter();
+
+ XmlWriter( XmlWriter const& ) = delete;
+ XmlWriter& operator=( XmlWriter const& ) = delete;
+
+ XmlWriter& startElement( std::string const& name );
+
+ ScopedElement scopedElement( std::string const& name );
+
+ XmlWriter& endElement();
+
+ XmlWriter& writeAttribute( std::string const& name, std::string const& attribute );
+
+ XmlWriter& writeAttribute( std::string const& name, const char* attribute );
+
+ XmlWriter& writeAttribute( std::string const& name, bool attribute );
+
+ template<typename T>
+ XmlWriter& writeAttribute( std::string const& name, T const& attribute ) {
+ std::stringstream rss;
+ rss << attribute;
+ return writeAttribute( name, rss.str() );
+ }
+
+ XmlWriter& writeText( std::string const& text, bool indent = true );
+
+ //XmlWriter& writeComment( std::string const& text );
+
+ //void writeStylesheetRef( std::string const& url );
+
+ //XmlWriter& writeBlankLine();
+
+ void ensureTagClosed();
+
+ private:
+
+ void writeDeclaration();
+
+ void newlineIfNecessary();
+
+ bool m_tagIsOpen = false;
+ bool m_needsNewline = false;
+ std::vector<std::string> m_tags;
+ std::string m_indent;
+ std::ostream& m_os;
+ };
+
+// =================================================================================================
+// The following code has been taken verbatim from Catch2/include/internal/catch_xmlwriter.h/cpp
+// This is done so cherry-picking bug fixes is trivial - even the style/formatting is untouched.
+// =================================================================================================
+
+using uchar = unsigned char;
+
+namespace {
+
+ size_t trailingBytes(unsigned char c) {
+ if ((c & 0xE0) == 0xC0) {
+ return 2;
+ }
+ if ((c & 0xF0) == 0xE0) {
+ return 3;
+ }
+ if ((c & 0xF8) == 0xF0) {
+ return 4;
+ }
+ DOCTEST_INTERNAL_ERROR("Invalid multibyte utf-8 start byte encountered");
+ }
+
+ uint32_t headerValue(unsigned char c) {
+ if ((c & 0xE0) == 0xC0) {
+ return c & 0x1F;
+ }
+ if ((c & 0xF0) == 0xE0) {
+ return c & 0x0F;
+ }
+ if ((c & 0xF8) == 0xF0) {
+ return c & 0x07;
+ }
+ DOCTEST_INTERNAL_ERROR("Invalid multibyte utf-8 start byte encountered");
+ }
+
+ void hexEscapeChar(std::ostream& os, unsigned char c) {
+ std::ios_base::fmtflags f(os.flags());
+ os << "\\x"
+ << std::uppercase << std::hex << std::setfill('0') << std::setw(2)
+ << static_cast<int>(c);
+ os.flags(f);
+ }
+
+} // anonymous namespace
+
+ XmlEncode::XmlEncode( std::string const& str, ForWhat forWhat )
+ : m_str( str ),
+ m_forWhat( forWhat )
+ {}
+
+ void XmlEncode::encodeTo( std::ostream& os ) const {
+ // Apostrophe escaping not necessary if we always use " to write attributes
+ // (see: https://www.w3.org/TR/xml/#syntax)
+
+ for( std::size_t idx = 0; idx < m_str.size(); ++ idx ) {
+ uchar c = m_str[idx];
+ switch (c) {
+ case '<': os << "&lt;"; break;
+ case '&': os << "&amp;"; break;
+
+ case '>':
+ // See: https://www.w3.org/TR/xml/#syntax
+ if (idx > 2 && m_str[idx - 1] == ']' && m_str[idx - 2] == ']')
+ os << "&gt;";
+ else
+ os << c;
+ break;
+
+ case '\"':
+ if (m_forWhat == ForAttributes)
+ os << "&quot;";
+ else
+ os << c;
+ break;
+
+ default:
+ // Check for control characters and invalid utf-8
+
+ // Escape control characters in standard ascii
+ // see https://stackoverflow.com/questions/404107/why-are-control-characters-illegal-in-xml-1-0
+ if (c < 0x09 || (c > 0x0D && c < 0x20) || c == 0x7F) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ // Plain ASCII: Write it to stream
+ if (c < 0x7F) {
+ os << c;
+ break;
+ }
+
+ // UTF-8 territory
+ // Check if the encoding is valid and if it is not, hex escape bytes.
+ // Important: We do not check the exact decoded values for validity, only the encoding format
+ // First check that this bytes is a valid lead byte:
+ // This means that it is not encoded as 1111 1XXX
+ // Or as 10XX XXXX
+ if (c < 0xC0 ||
+ c >= 0xF8) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ auto encBytes = trailingBytes(c);
+ // Are there enough bytes left to avoid accessing out-of-bounds memory?
+ if (idx + encBytes - 1 >= m_str.size()) {
+ hexEscapeChar(os, c);
+ break;
+ }
+ // The header is valid, check data
+ // The next encBytes bytes must together be a valid utf-8
+ // This means: bitpattern 10XX XXXX and the extracted value is sane (ish)
+ bool valid = true;
+ uint32_t value = headerValue(c);
+ for (std::size_t n = 1; n < encBytes; ++n) {
+ uchar nc = m_str[idx + n];
+ valid &= ((nc & 0xC0) == 0x80);
+ value = (value << 6) | (nc & 0x3F);
+ }
+
+ if (
+ // Wrong bit pattern of following bytes
+ (!valid) ||
+ // Overlong encodings
+ (value < 0x80) ||
+ ( value < 0x800 && encBytes > 2) || // removed "0x80 <= value &&" because redundant
+ (0x800 < value && value < 0x10000 && encBytes > 3) ||
+ // Encoded value out of range
+ (value >= 0x110000)
+ ) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ // If we got here, this is in fact a valid(ish) utf-8 sequence
+ for (std::size_t n = 0; n < encBytes; ++n) {
+ os << m_str[idx + n];
+ }
+ idx += encBytes - 1;
+ break;
+ }
+ }
+ }
+
+ std::ostream& operator << ( std::ostream& os, XmlEncode const& xmlEncode ) {
+ xmlEncode.encodeTo( os );
+ return os;
+ }
+
+ XmlWriter::ScopedElement::ScopedElement( XmlWriter* writer )
+ : m_writer( writer )
+ {}
+
+ XmlWriter::ScopedElement::ScopedElement( ScopedElement&& other ) DOCTEST_NOEXCEPT
+ : m_writer( other.m_writer ){
+ other.m_writer = nullptr;
+ }
+ XmlWriter::ScopedElement& XmlWriter::ScopedElement::operator=( ScopedElement&& other ) DOCTEST_NOEXCEPT {
+ if ( m_writer ) {
+ m_writer->endElement();
+ }
+ m_writer = other.m_writer;
+ other.m_writer = nullptr;
+ return *this;
+ }
+
+
+ XmlWriter::ScopedElement::~ScopedElement() {
+ if( m_writer )
+ m_writer->endElement();
+ }
+
+ XmlWriter::ScopedElement& XmlWriter::ScopedElement::writeText( std::string const& text, bool indent ) {
+ m_writer->writeText( text, indent );
+ return *this;
+ }
+
+ XmlWriter::XmlWriter( std::ostream& os ) : m_os( os )
+ {
+ writeDeclaration();
+ }
+
+ XmlWriter::~XmlWriter() {
+ while( !m_tags.empty() )
+ endElement();
+ }
+
+ XmlWriter& XmlWriter::startElement( std::string const& name ) {
+ ensureTagClosed();
+ newlineIfNecessary();
+ m_os << m_indent << '<' << name;
+ m_tags.push_back( name );
+ m_indent += " ";
+ m_tagIsOpen = true;
+ return *this;
+ }
+
+ XmlWriter::ScopedElement XmlWriter::scopedElement( std::string const& name ) {
+ ScopedElement scoped( this );
+ startElement( name );
+ return scoped;
+ }
+
+ XmlWriter& XmlWriter::endElement() {
+ newlineIfNecessary();
+ m_indent = m_indent.substr( 0, m_indent.size()-2 );
+ if( m_tagIsOpen ) {
+ m_os << "/>";
+ m_tagIsOpen = false;
+ }
+ else {
+ m_os << m_indent << "</" << m_tags.back() << ">";
+ }
+ m_os << std::endl;
+ m_tags.pop_back();
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, std::string const& attribute ) {
+ if( !name.empty() && !attribute.empty() )
+ m_os << ' ' << name << "=\"" << XmlEncode( attribute, XmlEncode::ForAttributes ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, const char* attribute ) {
+ if( !name.empty() && attribute && attribute[0] != '\0' )
+ m_os << ' ' << name << "=\"" << XmlEncode( attribute, XmlEncode::ForAttributes ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, bool attribute ) {
+ m_os << ' ' << name << "=\"" << ( attribute ? "true" : "false" ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeText( std::string const& text, bool indent ) {
+ if( !text.empty() ){
+ bool tagWasOpen = m_tagIsOpen;
+ ensureTagClosed();
+ if( tagWasOpen && indent )
+ m_os << m_indent;
+ m_os << XmlEncode( text );
+ m_needsNewline = true;
+ }
+ return *this;
+ }
+
+ //XmlWriter& XmlWriter::writeComment( std::string const& text ) {
+ // ensureTagClosed();
+ // m_os << m_indent << "<!--" << text << "-->";
+ // m_needsNewline = true;
+ // return *this;
+ //}
+
+ //void XmlWriter::writeStylesheetRef( std::string const& url ) {
+ // m_os << "<?xml-stylesheet type=\"text/xsl\" href=\"" << url << "\"?>\n";
+ //}
+
+ //XmlWriter& XmlWriter::writeBlankLine() {
+ // ensureTagClosed();
+ // m_os << '\n';
+ // return *this;
+ //}
+
+ void XmlWriter::ensureTagClosed() {
+ if( m_tagIsOpen ) {
+ m_os << ">" << std::endl;
+ m_tagIsOpen = false;
+ }
+ }
+
+ void XmlWriter::writeDeclaration() {
+ m_os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+ }
+
+ void XmlWriter::newlineIfNecessary() {
+ if( m_needsNewline ) {
+ m_os << std::endl;
+ m_needsNewline = false;
+ }
+ }
+
+// =================================================================================================
+// End of copy-pasted code from Catch
+// =================================================================================================
+
+ // clang-format on
+
+ struct XmlReporter : public IReporter
+ {
+ XmlWriter xml;
+ std::mutex mutex;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc = nullptr;
+
+ XmlReporter(const ContextOptions& co)
+ : xml(*co.cout)
+ , opt(co) {}
+
+ void log_contexts() {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+ std::stringstream ss;
+ for(int i = 0; i < num_contexts; ++i) {
+ contexts[i]->stringify(&ss);
+ xml.scopedElement("Info").writeText(ss.str());
+ ss.str("");
+ }
+ }
+ }
+
+ unsigned line(unsigned l) const { return opt.no_line_numbers ? 0 : l; }
+
+ void test_case_start_impl(const TestCaseData& in) {
+ bool open_ts_tag = false;
+ if(tc != nullptr) { // we have already opened a test suite
+ if(std::strcmp(tc->m_test_suite, in.m_test_suite) != 0) {
+ xml.endElement();
+ open_ts_tag = true;
+ }
+ }
+ else {
+ open_ts_tag = true; // first test case ==> first test suite
+ }
+
+ if(open_ts_tag) {
+ xml.startElement("TestSuite");
+ xml.writeAttribute("name", in.m_test_suite);
+ }
+
+ tc = &in;
+ xml.startElement("TestCase")
+ .writeAttribute("name", in.m_name)
+ .writeAttribute("filename", skipPathFromFilename(in.m_file.c_str()))
+ .writeAttribute("line", line(in.m_line))
+ .writeAttribute("description", in.m_description);
+
+ if(Approx(in.m_timeout) != 0)
+ xml.writeAttribute("timeout", in.m_timeout);
+ if(in.m_may_fail)
+ xml.writeAttribute("may_fail", true);
+ if(in.m_should_fail)
+ xml.writeAttribute("should_fail", true);
+ }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData& in) override {
+ test_run_start();
+ if(opt.list_reporters) {
+ for(auto& curr : getListeners())
+ xml.scopedElement("Listener")
+ .writeAttribute("priority", curr.first.first)
+ .writeAttribute("name", curr.first.second);
+ for(auto& curr : getReporters())
+ xml.scopedElement("Reporter")
+ .writeAttribute("priority", curr.first.first)
+ .writeAttribute("name", curr.first.second);
+ } else if(opt.count || opt.list_test_cases) {
+ for(unsigned i = 0; i < in.num_data; ++i) {
+ xml.scopedElement("TestCase").writeAttribute("name", in.data[i]->m_name)
+ .writeAttribute("testsuite", in.data[i]->m_test_suite)
+ .writeAttribute("filename", skipPathFromFilename(in.data[i]->m_file.c_str()))
+ .writeAttribute("line", line(in.data[i]->m_line));
+ }
+ xml.scopedElement("OverallResultsTestCases")
+ .writeAttribute("unskipped", in.run_stats->numTestCasesPassingFilters);
+ } else if(opt.list_test_suites) {
+ for(unsigned i = 0; i < in.num_data; ++i)
+ xml.scopedElement("TestSuite").writeAttribute("name", in.data[i]->m_test_suite);
+ xml.scopedElement("OverallResultsTestCases")
+ .writeAttribute("unskipped", in.run_stats->numTestCasesPassingFilters);
+ xml.scopedElement("OverallResultsTestSuites")
+ .writeAttribute("unskipped", in.run_stats->numTestSuitesPassingFilters);
+ }
+ xml.endElement();
+ }
+
+ void test_run_start() override {
+ // remove .exe extension - mainly to have the same output on UNIX and Windows
+ std::string binary_name = skipPathFromFilename(opt.binary_name.c_str());
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(binary_name.rfind(".exe") != std::string::npos)
+ binary_name = binary_name.substr(0, binary_name.length() - 4);
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ xml.startElement("doctest").writeAttribute("binary", binary_name);
+ if(opt.no_version == false)
+ xml.writeAttribute("version", DOCTEST_VERSION_STR);
+
+ // only the consequential ones (TODO: filters)
+ xml.scopedElement("Options")
+ .writeAttribute("order_by", opt.order_by.c_str())
+ .writeAttribute("rand_seed", opt.rand_seed)
+ .writeAttribute("first", opt.first)
+ .writeAttribute("last", opt.last)
+ .writeAttribute("abort_after", opt.abort_after)
+ .writeAttribute("subcase_filter_levels", opt.subcase_filter_levels)
+ .writeAttribute("case_sensitive", opt.case_sensitive)
+ .writeAttribute("no_throw", opt.no_throw)
+ .writeAttribute("no_skip", opt.no_skip);
+ }
+
+ void test_run_end(const TestRunStats& p) override {
+ if(tc) // the TestSuite tag - only if there has been at least 1 test case
+ xml.endElement();
+
+ xml.scopedElement("OverallResultsAsserts")
+ .writeAttribute("successes", p.numAsserts - p.numAssertsFailed)
+ .writeAttribute("failures", p.numAssertsFailed);
+
+ xml.startElement("OverallResultsTestCases")
+ .writeAttribute("successes",
+ p.numTestCasesPassingFilters - p.numTestCasesFailed)
+ .writeAttribute("failures", p.numTestCasesFailed);
+ if(opt.no_skipped_summary == false)
+ xml.writeAttribute("skipped", p.numTestCases - p.numTestCasesPassingFilters);
+ xml.endElement();
+
+ xml.endElement();
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ test_case_start_impl(in);
+ xml.ensureTagClosed();
+ }
+
+ void test_case_reenter(const TestCaseData&) override {}
+
+ void test_case_end(const CurrentTestCaseStats& st) override {
+ xml.startElement("OverallResultsAsserts")
+ .writeAttribute("successes",
+ st.numAssertsCurrentTest - st.numAssertsFailedCurrentTest)
+ .writeAttribute("failures", st.numAssertsFailedCurrentTest);
+ if(opt.duration)
+ xml.writeAttribute("duration", st.seconds);
+ if(tc->m_expected_failures)
+ xml.writeAttribute("expected_failures", tc->m_expected_failures);
+ xml.endElement();
+
+ xml.endElement();
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.scopedElement("Exception")
+ .writeAttribute("crash", e.is_crash)
+ .writeText(e.error_string.c_str());
+ }
+
+ void subcase_start(const SubcaseSignature& in) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("SubCase")
+ .writeAttribute("name", in.m_name)
+ .writeAttribute("filename", skipPathFromFilename(in.m_file))
+ .writeAttribute("line", line(in.m_line));
+ xml.ensureTagClosed();
+ }
+
+ void subcase_end() override { xml.endElement(); }
+
+ void log_assert(const AssertData& rb) override {
+ if(!rb.m_failed && !opt.success)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("Expression")
+ .writeAttribute("success", !rb.m_failed)
+ .writeAttribute("type", assertString(rb.m_at))
+ .writeAttribute("filename", skipPathFromFilename(rb.m_file))
+ .writeAttribute("line", line(rb.m_line));
+
+ xml.scopedElement("Original").writeText(rb.m_expr);
+
+ if(rb.m_threw)
+ xml.scopedElement("Exception").writeText(rb.m_exception.c_str());
+
+ if(rb.m_at & assertType::is_throws_as)
+ xml.scopedElement("ExpectedException").writeText(rb.m_exception_type);
+ if(rb.m_at & assertType::is_throws_with)
+ xml.scopedElement("ExpectedExceptionString").writeText(rb.m_exception_string);
+ if((rb.m_at & assertType::is_normal) && !rb.m_threw)
+ xml.scopedElement("Expanded").writeText(rb.m_decomp.c_str());
+
+ log_contexts();
+
+ xml.endElement();
+ }
+
+ void log_message(const MessageData& mb) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("Message")
+ .writeAttribute("type", failureString(mb.m_severity))
+ .writeAttribute("filename", skipPathFromFilename(mb.m_file))
+ .writeAttribute("line", line(mb.m_line));
+
+ xml.scopedElement("Text").writeText(mb.m_string.c_str());
+
+ log_contexts();
+
+ xml.endElement();
+ }
+
+ void test_case_skipped(const TestCaseData& in) override {
+ if(opt.no_skipped_summary == false) {
+ test_case_start_impl(in);
+ xml.writeAttribute("skipped", "true");
+ xml.endElement();
+ }
+ }
+ };
+
+ DOCTEST_REGISTER_REPORTER("xml", 0, XmlReporter);
+
+ void fulltext_log_assert_to_stream(std::ostream& s, const AssertData& rb) {
+ if((rb.m_at & (assertType::is_throws_as | assertType::is_throws_with)) ==
+ 0) //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << " ) "
+ << Color::None;
+
+ if(rb.m_at & assertType::is_throws) { //!OCLINT bitwise operator in conditional
+ s << (rb.m_threw ? "threw as expected!" : "did NOT throw at all!") << "\n";
+ } else if((rb.m_at & assertType::is_throws_as) &&
+ (rb.m_at & assertType::is_throws_with)) { //!OCLINT
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", \""
+ << rb.m_exception_string << "\", " << rb.m_exception_type << " ) " << Color::None;
+ if(rb.m_threw) {
+ if(!rb.m_failed) {
+ s << "threw as expected!\n";
+ } else {
+ s << "threw a DIFFERENT exception! (contents: " << rb.m_exception << ")\n";
+ }
+ } else {
+ s << "did NOT throw at all!\n";
+ }
+ } else if(rb.m_at &
+ assertType::is_throws_as) { //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", "
+ << rb.m_exception_type << " ) " << Color::None
+ << (rb.m_threw ? (rb.m_threw_as ? "threw as expected!" :
+ "threw a DIFFERENT exception: ") :
+ "did NOT throw at all!")
+ << Color::Cyan << rb.m_exception << "\n";
+ } else if(rb.m_at &
+ assertType::is_throws_with) { //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", \""
+ << rb.m_exception_string << "\" ) " << Color::None
+ << (rb.m_threw ? (!rb.m_failed ? "threw as expected!" :
+ "threw a DIFFERENT exception: ") :
+ "did NOT throw at all!")
+ << Color::Cyan << rb.m_exception << "\n";
+ } else if(rb.m_at & assertType::is_nothrow) { //!OCLINT bitwise operator in conditional
+ s << (rb.m_threw ? "THREW exception: " : "didn't throw!") << Color::Cyan
+ << rb.m_exception << "\n";
+ } else {
+ s << (rb.m_threw ? "THREW exception: " :
+ (!rb.m_failed ? "is correct!\n" : "is NOT correct!\n"));
+ if(rb.m_threw)
+ s << rb.m_exception << "\n";
+ else
+ s << " values: " << assertString(rb.m_at) << "( " << rb.m_decomp << " )\n";
+ }
+ }
+
+ // TODO:
+ // - log_message()
+ // - respond to queries
+ // - honor remaining options
+ // - more attributes in tags
+ struct JUnitReporter : public IReporter
+ {
+ XmlWriter xml;
+ std::mutex mutex;
+ Timer timer;
+ std::vector<String> deepestSubcaseStackNames;
+
+ struct JUnitTestCaseData
+ {
+ static std::string getCurrentTimestamp() {
+ // Beware, this is not reentrant because of backward compatibility issues
+ // Also, UTC only, again because of backward compatibility (%z is C++11)
+ time_t rawtime;
+ std::time(&rawtime);
+ auto const timeStampSize = sizeof("2017-01-16T17:06:45Z");
+
+ std::tm timeInfo;
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ gmtime_s(&timeInfo, &rawtime);
+#else // DOCTEST_PLATFORM_WINDOWS
+ gmtime_r(&rawtime, &timeInfo);
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ char timeStamp[timeStampSize];
+ const char* const fmt = "%Y-%m-%dT%H:%M:%SZ";
+
+ std::strftime(timeStamp, timeStampSize, fmt, &timeInfo);
+ return std::string(timeStamp);
+ }
+
+ struct JUnitTestMessage
+ {
+ JUnitTestMessage(const std::string& _message, const std::string& _type, const std::string& _details)
+ : message(_message), type(_type), details(_details) {}
+
+ JUnitTestMessage(const std::string& _message, const std::string& _details)
+ : message(_message), type(), details(_details) {}
+
+ std::string message, type, details;
+ };
+
+ struct JUnitTestCase
+ {
+ JUnitTestCase(const std::string& _classname, const std::string& _name)
+ : classname(_classname), name(_name), time(0), failures() {}
+
+ std::string classname, name;
+ double time;
+ std::vector<JUnitTestMessage> failures, errors;
+ };
+
+ void add(const std::string& classname, const std::string& name) {
+ testcases.emplace_back(classname, name);
+ }
+
+ void appendSubcaseNamesToLastTestcase(std::vector<String> nameStack) {
+ for(auto& curr: nameStack)
+ if(curr.size())
+ testcases.back().name += std::string("/") + curr.c_str();
+ }
+
+ void addTime(double time) {
+ if(time < 1e-4)
+ time = 0;
+ testcases.back().time = time;
+ totalSeconds += time;
+ }
+
+ void addFailure(const std::string& message, const std::string& type, const std::string& details) {
+ testcases.back().failures.emplace_back(message, type, details);
+ ++totalFailures;
+ }
+
+ void addError(const std::string& message, const std::string& details) {
+ testcases.back().errors.emplace_back(message, details);
+ ++totalErrors;
+ }
+
+ std::vector<JUnitTestCase> testcases;
+ double totalSeconds = 0;
+ int totalErrors = 0, totalFailures = 0;
+ };
+
+ JUnitTestCaseData testCaseData;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc = nullptr;
+
+ JUnitReporter(const ContextOptions& co)
+ : xml(*co.cout)
+ , opt(co) {}
+
+ unsigned line(unsigned l) const { return opt.no_line_numbers ? 0 : l; }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData&) override {}
+
+ void test_run_start() override {}
+
+ void test_run_end(const TestRunStats& p) override {
+ // remove .exe extension - mainly to have the same output on UNIX and Windows
+ std::string binary_name = skipPathFromFilename(opt.binary_name.c_str());
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(binary_name.rfind(".exe") != std::string::npos)
+ binary_name = binary_name.substr(0, binary_name.length() - 4);
+#endif // DOCTEST_PLATFORM_WINDOWS
+ xml.startElement("testsuites");
+ xml.startElement("testsuite").writeAttribute("name", binary_name)
+ .writeAttribute("errors", testCaseData.totalErrors)
+ .writeAttribute("failures", testCaseData.totalFailures)
+ .writeAttribute("tests", p.numAsserts);
+ if(opt.no_time_in_output == false) {
+ xml.writeAttribute("time", testCaseData.totalSeconds);
+ xml.writeAttribute("timestamp", JUnitTestCaseData::getCurrentTimestamp());
+ }
+ if(opt.no_version == false)
+ xml.writeAttribute("doctest_version", DOCTEST_VERSION_STR);
+
+ for(const auto& testCase : testCaseData.testcases) {
+ xml.startElement("testcase")
+ .writeAttribute("classname", testCase.classname)
+ .writeAttribute("name", testCase.name);
+ if(opt.no_time_in_output == false)
+ xml.writeAttribute("time", testCase.time);
+ // This is not ideal, but it should be enough to mimic gtest's junit output.
+ xml.writeAttribute("status", "run");
+
+ for(const auto& failure : testCase.failures) {
+ xml.scopedElement("failure")
+ .writeAttribute("message", failure.message)
+ .writeAttribute("type", failure.type)
+ .writeText(failure.details, false);
+ }
+
+ for(const auto& error : testCase.errors) {
+ xml.scopedElement("error")
+ .writeAttribute("message", error.message)
+ .writeText(error.details);
+ }
+
+ xml.endElement();
+ }
+ xml.endElement();
+ xml.endElement();
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ testCaseData.add(skipPathFromFilename(in.m_file.c_str()), in.m_name);
+ timer.start();
+ }
+
+ void test_case_reenter(const TestCaseData& in) override {
+ testCaseData.addTime(timer.getElapsedSeconds());
+ testCaseData.appendSubcaseNamesToLastTestcase(deepestSubcaseStackNames);
+ deepestSubcaseStackNames.clear();
+
+ timer.start();
+ testCaseData.add(skipPathFromFilename(in.m_file.c_str()), in.m_name);
+ }
+
+ void test_case_end(const CurrentTestCaseStats&) override {
+ testCaseData.addTime(timer.getElapsedSeconds());
+ testCaseData.appendSubcaseNamesToLastTestcase(deepestSubcaseStackNames);
+ deepestSubcaseStackNames.clear();
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ testCaseData.addError("exception", e.error_string.c_str());
+ }
+
+ void subcase_start(const SubcaseSignature& in) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ deepestSubcaseStackNames.push_back(in.m_name);
+ }
+
+ void subcase_end() override {}
+
+ void log_assert(const AssertData& rb) override {
+ if(!rb.m_failed) // report only failures & ignore the `success` option
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ std::ostringstream os;
+ os << skipPathFromFilename(rb.m_file) << (opt.gnu_file_line ? ":" : "(")
+ << line(rb.m_line) << (opt.gnu_file_line ? ":" : "):") << std::endl;
+
+ fulltext_log_assert_to_stream(os, rb);
+ log_contexts(os);
+ testCaseData.addFailure(rb.m_decomp.c_str(), assertString(rb.m_at), os.str());
+ }
+
+ void log_message(const MessageData&) override {}
+
+ void test_case_skipped(const TestCaseData&) override {}
+
+ void log_contexts(std::ostringstream& s) {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+
+ s << " logged: ";
+ for(int i = 0; i < num_contexts; ++i) {
+ s << (i == 0 ? "" : " ");
+ contexts[i]->stringify(&s);
+ s << std::endl;
+ }
+ }
+ }
+ };
+
+ DOCTEST_REGISTER_REPORTER("junit", 0, JUnitReporter);
+
+ struct Whitespace
+ {
+ int nrSpaces;
+ explicit Whitespace(int nr)
+ : nrSpaces(nr) {}
+ };
+
+ std::ostream& operator<<(std::ostream& out, const Whitespace& ws) {
+ if(ws.nrSpaces != 0)
+ out << std::setw(ws.nrSpaces) << ' ';
+ return out;
+ }
+
+ struct ConsoleReporter : public IReporter
+ {
+ std::ostream& s;
+ bool hasLoggedCurrentTestStart;
+ std::vector<SubcaseSignature> subcasesStack;
+ size_t currentSubcaseLevel;
+ std::mutex mutex;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc;
+
+ ConsoleReporter(const ContextOptions& co)
+ : s(*co.cout)
+ , opt(co) {}
+
+ ConsoleReporter(const ContextOptions& co, std::ostream& ostr)
+ : s(ostr)
+ , opt(co) {}
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE HELPERS USED BY THE OVERRIDES OF THE VIRTUAL METHODS OF THE INTERFACE
+ // =========================================================================================
+
+ void separator_to_stream() {
+ s << Color::Yellow
+ << "==============================================================================="
+ "\n";
+ }
+
+ const char* getSuccessOrFailString(bool success, assertType::Enum at,
+ const char* success_str) {
+ if(success)
+ return success_str;
+ return failureString(at);
+ }
+
+ Color::Enum getSuccessOrFailColor(bool success, assertType::Enum at) {
+ return success ? Color::BrightGreen :
+ (at & assertType::is_warn) ? Color::Yellow : Color::Red;
+ }
+
+ void successOrFailColoredStringToStream(bool success, assertType::Enum at,
+ const char* success_str = "SUCCESS") {
+ s << getSuccessOrFailColor(success, at)
+ << getSuccessOrFailString(success, at, success_str) << ": ";
+ }
+
+ void log_contexts() {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+
+ s << Color::None << " logged: ";
+ for(int i = 0; i < num_contexts; ++i) {
+ s << (i == 0 ? "" : " ");
+ contexts[i]->stringify(&s);
+ s << "\n";
+ }
+ }
+
+ s << "\n";
+ }
+
+ // this was requested to be made virtual so users could override it
+ virtual void file_line_to_stream(const char* file, int line,
+ const char* tail = "") {
+ s << Color::LightGrey << skipPathFromFilename(file) << (opt.gnu_file_line ? ":" : "(")
+ << (opt.no_line_numbers ? 0 : line) // 0 or the real num depending on the option
+ << (opt.gnu_file_line ? ":" : "):") << tail;
+ }
+
+ void logTestStart() {
+ if(hasLoggedCurrentTestStart)
+ return;
+
+ separator_to_stream();
+ file_line_to_stream(tc->m_file.c_str(), tc->m_line, "\n");
+ if(tc->m_description)
+ s << Color::Yellow << "DESCRIPTION: " << Color::None << tc->m_description << "\n";
+ if(tc->m_test_suite && tc->m_test_suite[0] != '\0')
+ s << Color::Yellow << "TEST SUITE: " << Color::None << tc->m_test_suite << "\n";
+ if(strncmp(tc->m_name, " Scenario:", 11) != 0)
+ s << Color::Yellow << "TEST CASE: ";
+ s << Color::None << tc->m_name << "\n";
+
+ for(size_t i = 0; i < currentSubcaseLevel; ++i) {
+ if(subcasesStack[i].m_name[0] != '\0')
+ s << " " << subcasesStack[i].m_name << "\n";
+ }
+
+ if(currentSubcaseLevel != subcasesStack.size()) {
+ s << Color::Yellow << "\nDEEPEST SUBCASE STACK REACHED (DIFFERENT FROM THE CURRENT ONE):\n" << Color::None;
+ for(size_t i = 0; i < subcasesStack.size(); ++i) {
+ if(subcasesStack[i].m_name[0] != '\0')
+ s << " " << subcasesStack[i].m_name << "\n";
+ }
+ }
+
+ s << "\n";
+
+ hasLoggedCurrentTestStart = true;
+ }
+
+ void printVersion() {
+ if(opt.no_version == false)
+ s << Color::Cyan << "[doctest] " << Color::None << "doctest version is \""
+ << DOCTEST_VERSION_STR << "\"\n";
+ }
+
+ void printIntro() {
+ printVersion();
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "run with \"--" DOCTEST_OPTIONS_PREFIX_DISPLAY "help\" for options\n";
+ }
+
+ void printHelp() {
+ int sizePrefixDisplay = static_cast<int>(strlen(DOCTEST_OPTIONS_PREFIX_DISPLAY));
+ printVersion();
+ // clang-format off
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "boolean values: \"1/on/yes/true\" or \"0/off/no/false\"\n";
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "filter values: \"str1,str2,str3\" (comma separated strings)\n";
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "filters use wildcards for matching strings\n";
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "something passes a filter if any of the strings in a filter matches\n";
+#ifndef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "ALL FLAGS, OPTIONS AND FILTERS ALSO AVAILABLE WITH A \"" DOCTEST_CONFIG_OPTIONS_PREFIX "\" PREFIX!!!\n";
+#endif
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "Query flags - the program quits after them. Available:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "?, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "help, -" DOCTEST_OPTIONS_PREFIX_DISPLAY "h "
+ << Whitespace(sizePrefixDisplay*0) << "prints this message\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "v, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "version "
+ << Whitespace(sizePrefixDisplay*1) << "prints the version\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "c, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "count "
+ << Whitespace(sizePrefixDisplay*1) << "prints the number of matching tests\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ltc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-test-cases "
+ << Whitespace(sizePrefixDisplay*1) << "lists all matching tests by name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "lts, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-test-suites "
+ << Whitespace(sizePrefixDisplay*1) << "lists all matching test suites\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "lr, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-reporters "
+ << Whitespace(sizePrefixDisplay*1) << "lists all registered reporters\n\n";
+ // ================================================================================== << 79
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "The available <int>/<string> options/filters are:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-case=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tce, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-case-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sf, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "source-file=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their file\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sfe, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "source-file-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their file\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ts, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-suite=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their test suite\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tse, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-suite-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their test suite\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters subcases by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sce, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT subcases by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "r, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "reporters=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "reporters to use (console is default)\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "o, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "out=<string> "
+ << Whitespace(sizePrefixDisplay*1) << "output filename\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ob, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "order-by=<string> "
+ << Whitespace(sizePrefixDisplay*1) << "how the tests should be ordered\n";
+ s << Whitespace(sizePrefixDisplay*3) << " <string> - [file/suite/name/rand/none]\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "rs, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "rand-seed=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "seed for random ordering\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "f, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "first=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "the first test passing the filters to\n";
+ s << Whitespace(sizePrefixDisplay*3) << " execute - for range-based execution\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "l, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "last=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "the last test passing the filters to\n";
+ s << Whitespace(sizePrefixDisplay*3) << " execute - for range-based execution\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "aa, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "abort-after=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "stop after <int> failed assertions\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "scfl,--" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase-filter-levels=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "apply filters for the first <int> levels\n";
+ s << Color::Cyan << "\n[doctest] " << Color::None;
+ s << "Bool options - can be used like flags and true is assumed. Available:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "s, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "success=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "include successful assertions in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "cs, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "case-sensitive=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "filters being treated as case sensitive\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "e, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "exit=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "exits after the tests finish\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "d, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "duration=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "prints the time duration of each test\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nt, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-throw=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "skips exceptions-related assert checks\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ne, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-exitcode=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "returns (or exits) always with success\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nr, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-run=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "skips all runtime doctest operations\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nv, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-version=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "omit the framework version in the output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-colors=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "disables colors in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "fc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "force-colors=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "use colors even when not in a tty\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nb, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-breaks=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "disables breakpoints in debuggers\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ns, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-skip=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "don't skip test cases marked as skip\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "gfl, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "gnu-file-line=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << ":n: vs (n): for line numbers in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "npf, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-path-filenames=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "only filenames and no paths in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nln, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-line-numbers=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "0 instead of real line numbers in output\n";
+ // ================================================================================== << 79
+ // clang-format on
+
+ s << Color::Cyan << "\n[doctest] " << Color::None;
+ s << "for more information visit the project documentation\n\n";
+ }
+
+ void printRegisteredReporters() {
+ printVersion();
+ auto printReporters = [this] (const reporterMap& reporters, const char* type) {
+ if(reporters.size()) {
+ s << Color::Cyan << "[doctest] " << Color::None << "listing all registered " << type << "\n";
+ for(auto& curr : reporters)
+ s << "priority: " << std::setw(5) << curr.first.first
+ << " name: " << curr.first.second << "\n";
+ }
+ };
+ printReporters(getListeners(), "listeners");
+ printReporters(getReporters(), "reporters");
+ }
+
+ void list_query_results() {
+ separator_to_stream();
+ if(opt.count || opt.list_test_cases) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ } else if(opt.list_test_suites) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "test suites with unskipped test cases passing the current filters: "
+ << g_cs->numTestSuitesPassingFilters << "\n";
+ }
+ }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData& in) override {
+ if(opt.version) {
+ printVersion();
+ } else if(opt.help) {
+ printHelp();
+ } else if(opt.list_reporters) {
+ printRegisteredReporters();
+ } else if(opt.count || opt.list_test_cases) {
+ if(opt.list_test_cases) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "listing all test case names\n";
+ separator_to_stream();
+ }
+
+ for(unsigned i = 0; i < in.num_data; ++i)
+ s << Color::None << in.data[i]->m_name << "\n";
+
+ separator_to_stream();
+
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+
+ } else if(opt.list_test_suites) {
+ s << Color::Cyan << "[doctest] " << Color::None << "listing all test suites\n";
+ separator_to_stream();
+
+ for(unsigned i = 0; i < in.num_data; ++i)
+ s << Color::None << in.data[i]->m_test_suite << "\n";
+
+ separator_to_stream();
+
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "test suites with unskipped test cases passing the current filters: "
+ << g_cs->numTestSuitesPassingFilters << "\n";
+ }
+ }
+
+ void test_run_start() override { printIntro(); }
+
+ void test_run_end(const TestRunStats& p) override {
+ separator_to_stream();
+ s << std::dec;
+
+ auto totwidth = int(std::ceil(log10((std::max(p.numTestCasesPassingFilters, static_cast<unsigned>(p.numAsserts))) + 1)));
+ auto passwidth = int(std::ceil(log10((std::max(p.numTestCasesPassingFilters - p.numTestCasesFailed, static_cast<unsigned>(p.numAsserts - p.numAssertsFailed))) + 1)));
+ auto failwidth = int(std::ceil(log10((std::max(p.numTestCasesFailed, static_cast<unsigned>(p.numAssertsFailed))) + 1)));
+ const bool anythingFailed = p.numTestCasesFailed > 0 || p.numAssertsFailed > 0;
+ s << Color::Cyan << "[doctest] " << Color::None << "test cases: " << std::setw(totwidth)
+ << p.numTestCasesPassingFilters << " | "
+ << ((p.numTestCasesPassingFilters == 0 || anythingFailed) ? Color::None :
+ Color::Green)
+ << std::setw(passwidth) << p.numTestCasesPassingFilters - p.numTestCasesFailed << " passed"
+ << Color::None << " | " << (p.numTestCasesFailed > 0 ? Color::Red : Color::None)
+ << std::setw(failwidth) << p.numTestCasesFailed << " failed" << Color::None << " |";
+ if(opt.no_skipped_summary == false) {
+ const int numSkipped = p.numTestCases - p.numTestCasesPassingFilters;
+ s << " " << (numSkipped == 0 ? Color::None : Color::Yellow) << numSkipped
+ << " skipped" << Color::None;
+ }
+ s << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None << "assertions: " << std::setw(totwidth)
+ << p.numAsserts << " | "
+ << ((p.numAsserts == 0 || anythingFailed) ? Color::None : Color::Green)
+ << std::setw(passwidth) << (p.numAsserts - p.numAssertsFailed) << " passed" << Color::None
+ << " | " << (p.numAssertsFailed > 0 ? Color::Red : Color::None) << std::setw(failwidth)
+ << p.numAssertsFailed << " failed" << Color::None << " |\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "Status: " << (p.numTestCasesFailed > 0 ? Color::Red : Color::Green)
+ << ((p.numTestCasesFailed > 0) ? "FAILURE!" : "SUCCESS!") << Color::None << std::endl;
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ hasLoggedCurrentTestStart = false;
+ tc = &in;
+ subcasesStack.clear();
+ currentSubcaseLevel = 0;
+ }
+
+ void test_case_reenter(const TestCaseData&) override {
+ subcasesStack.clear();
+ }
+
+ void test_case_end(const CurrentTestCaseStats& st) override {
+ if(tc->m_no_output)
+ return;
+
+ // log the preamble of the test case only if there is something
+ // else to print - something other than that an assert has failed
+ if(opt.duration ||
+ (st.failure_flags && st.failure_flags != TestCaseFailureReason::AssertFailure))
+ logTestStart();
+
+ if(opt.duration)
+ s << Color::None << std::setprecision(6) << std::fixed << st.seconds
+ << " s: " << tc->m_name << "\n";
+
+ if(st.failure_flags & TestCaseFailureReason::Timeout)
+ s << Color::Red << "Test case exceeded time limit of " << std::setprecision(6)
+ << std::fixed << tc->m_timeout << "!\n";
+
+ if(st.failure_flags & TestCaseFailureReason::ShouldHaveFailedButDidnt) {
+ s << Color::Red << "Should have failed but didn't! Marking it as failed!\n";
+ } else if(st.failure_flags & TestCaseFailureReason::ShouldHaveFailedAndDid) {
+ s << Color::Yellow << "Failed as expected so marking it as not failed\n";
+ } else if(st.failure_flags & TestCaseFailureReason::CouldHaveFailedAndDid) {
+ s << Color::Yellow << "Allowed to fail so marking it as not failed\n";
+ } else if(st.failure_flags & TestCaseFailureReason::DidntFailExactlyNumTimes) {
+ s << Color::Red << "Didn't fail exactly " << tc->m_expected_failures
+ << " times so marking it as failed!\n";
+ } else if(st.failure_flags & TestCaseFailureReason::FailedExactlyNumTimes) {
+ s << Color::Yellow << "Failed exactly " << tc->m_expected_failures
+ << " times as expected so marking it as not failed!\n";
+ }
+ if(st.failure_flags & TestCaseFailureReason::TooManyFailedAsserts) {
+ s << Color::Red << "Aborting - too many failed asserts!\n";
+ }
+ s << Color::None; // lgtm [cpp/useless-expression]
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ if(tc->m_no_output)
+ return;
+
+ logTestStart();
+
+ file_line_to_stream(tc->m_file.c_str(), tc->m_line, " ");
+ successOrFailColoredStringToStream(false, e.is_crash ? assertType::is_require :
+ assertType::is_check);
+ s << Color::Red << (e.is_crash ? "test case CRASHED: " : "test case THREW exception: ")
+ << Color::Cyan << e.error_string << "\n";
+
+ int num_stringified_contexts = get_num_stringified_contexts();
+ if(num_stringified_contexts) {
+ auto stringified_contexts = get_stringified_contexts();
+ s << Color::None << " logged: ";
+ for(int i = num_stringified_contexts; i > 0; --i) {
+ s << (i == num_stringified_contexts ? "" : " ")
+ << stringified_contexts[i - 1] << "\n";
+ }
+ }
+ s << "\n" << Color::None;
+ }
+
+ void subcase_start(const SubcaseSignature& subc) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ subcasesStack.push_back(subc);
+ ++currentSubcaseLevel;
+ hasLoggedCurrentTestStart = false;
+ }
+
+ void subcase_end() override {
+ std::lock_guard<std::mutex> lock(mutex);
+ --currentSubcaseLevel;
+ hasLoggedCurrentTestStart = false;
+ }
+
+ void log_assert(const AssertData& rb) override {
+ if((!rb.m_failed && !opt.success) || tc->m_no_output)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ logTestStart();
+
+ file_line_to_stream(rb.m_file, rb.m_line, " ");
+ successOrFailColoredStringToStream(!rb.m_failed, rb.m_at);
+
+ fulltext_log_assert_to_stream(s, rb);
+
+ log_contexts();
+ }
+
+ void log_message(const MessageData& mb) override {
+ if(tc->m_no_output)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ logTestStart();
+
+ file_line_to_stream(mb.m_file, mb.m_line, " ");
+ s << getSuccessOrFailColor(false, mb.m_severity)
+ << getSuccessOrFailString(mb.m_severity & assertType::is_warn, mb.m_severity,
+ "MESSAGE") << ": ";
+ s << Color::None << mb.m_string << "\n";
+ log_contexts();
+ }
+
+ void test_case_skipped(const TestCaseData&) override {}
+ };
+
+ DOCTEST_REGISTER_REPORTER("console", 0, ConsoleReporter);
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ struct DebugOutputWindowReporter : public ConsoleReporter
+ {
+ DOCTEST_THREAD_LOCAL static std::ostringstream oss;
+
+ DebugOutputWindowReporter(const ContextOptions& co)
+ : ConsoleReporter(co, oss) {}
+
+#define DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(func, type, arg) \
+ void func(type arg) override { \
+ bool with_col = g_no_colors; \
+ g_no_colors = false; \
+ ConsoleReporter::func(arg); \
+ if(oss.tellp() != std::streampos{}) { \
+ DOCTEST_OUTPUT_DEBUG_STRING(oss.str().c_str()); \
+ oss.str(""); \
+ } \
+ g_no_colors = with_col; \
+ }
+
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_run_start, DOCTEST_EMPTY, DOCTEST_EMPTY)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_run_end, const TestRunStats&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_start, const TestCaseData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_reenter, const TestCaseData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_end, const CurrentTestCaseStats&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_exception, const TestCaseException&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(subcase_start, const SubcaseSignature&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(subcase_end, DOCTEST_EMPTY, DOCTEST_EMPTY)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(log_assert, const AssertData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(log_message, const MessageData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_skipped, const TestCaseData&, in)
+ };
+
+ DOCTEST_THREAD_LOCAL std::ostringstream DebugOutputWindowReporter::oss;
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ // the implementation of parseOption()
+ bool parseOptionImpl(int argc, const char* const* argv, const char* pattern, String* value) {
+ // going from the end to the beginning and stopping on the first occurrence from the end
+ for(int i = argc; i > 0; --i) {
+ auto index = i - 1;
+ auto temp = std::strstr(argv[index], pattern);
+ if(temp && (value || strlen(temp) == strlen(pattern))) { //!OCLINT prefer early exits and continue
+ // eliminate matches in which the chars before the option are not '-'
+ bool noBadCharsFound = true;
+ auto curr = argv[index];
+ while(curr != temp) {
+ if(*curr++ != '-') {
+ noBadCharsFound = false;
+ break;
+ }
+ }
+ if(noBadCharsFound && argv[index][0] == '-') {
+ if(value) {
+ // parsing the value of an option
+ temp += strlen(pattern);
+ const unsigned len = strlen(temp);
+ if(len) {
+ *value = temp;
+ return true;
+ }
+ } else {
+ // just a flag - no value
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ // parses an option and returns the string after the '=' character
+ bool parseOption(int argc, const char* const* argv, const char* pattern, String* value = nullptr,
+ const String& defaultVal = String()) {
+ if(value)
+ *value = defaultVal;
+#ifndef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ // offset (normally 3 for "dt-") to skip prefix
+ if(parseOptionImpl(argc, argv, pattern + strlen(DOCTEST_CONFIG_OPTIONS_PREFIX), value))
+ return true;
+#endif // DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ return parseOptionImpl(argc, argv, pattern, value);
+ }
+
+ // locates a flag on the command line
+ bool parseFlag(int argc, const char* const* argv, const char* pattern) {
+ return parseOption(argc, argv, pattern);
+ }
+
+ // parses a comma separated list of words after a pattern in one of the arguments in argv
+ bool parseCommaSepArgs(int argc, const char* const* argv, const char* pattern,
+ std::vector<String>& res) {
+ String filtersString;
+ if(parseOption(argc, argv, pattern, &filtersString)) {
+ // tokenize with "," as a separator
+ // cppcheck-suppress strtokCalled
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ auto pch = std::strtok(filtersString.c_str(), ","); // modifies the string
+ while(pch != nullptr) {
+ if(strlen(pch))
+ res.push_back(pch);
+ // uses the strtok() internal state to go to the next token
+ // cppcheck-suppress strtokCalled
+ pch = std::strtok(nullptr, ",");
+ }
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ return true;
+ }
+ return false;
+ }
+
+ enum optionType
+ {
+ option_bool,
+ option_int
+ };
+
+ // parses an int/bool option from the command line
+ bool parseIntOption(int argc, const char* const* argv, const char* pattern, optionType type,
+ int& res) {
+ String parsedValue;
+ if(!parseOption(argc, argv, pattern, &parsedValue))
+ return false;
+
+ if(type == 0) {
+ // boolean
+ const char positive[][5] = {"1", "true", "on", "yes"}; // 5 - strlen("true") + 1
+ const char negative[][6] = {"0", "false", "off", "no"}; // 6 - strlen("false") + 1
+
+ // if the value matches any of the positive/negative possibilities
+ for(unsigned i = 0; i < 4; i++) {
+ if(parsedValue.compare(positive[i], true) == 0) {
+ res = 1; //!OCLINT parameter reassignment
+ return true;
+ }
+ if(parsedValue.compare(negative[i], true) == 0) {
+ res = 0; //!OCLINT parameter reassignment
+ return true;
+ }
+ }
+ } else {
+ // integer
+ // TODO: change this to use std::stoi or something else! currently it uses undefined behavior - assumes '0' on failed parse...
+ int theInt = std::atoi(parsedValue.c_str()); // NOLINT
+ if(theInt != 0) {
+ res = theInt; //!OCLINT parameter reassignment
+ return true;
+ }
+ }
+ return false;
+ }
+} // namespace
+
+Context::Context(int argc, const char* const* argv)
+ : p(new detail::ContextState) {
+ parseArgs(argc, argv, true);
+ if(argc)
+ p->binary_name = argv[0];
+}
+
+Context::~Context() {
+ if(g_cs == p)
+ g_cs = nullptr;
+ delete p;
+}
+
+void Context::applyCommandLine(int argc, const char* const* argv) {
+ parseArgs(argc, argv);
+ if(argc)
+ p->binary_name = argv[0];
+}
+
+// parses args
+void Context::parseArgs(int argc, const char* const* argv, bool withDefaults) {
+ using namespace detail;
+
+ // clang-format off
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "source-file=", p->filters[0]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sf=", p->filters[0]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "source-file-exclude=",p->filters[1]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sfe=", p->filters[1]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-suite=", p->filters[2]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "ts=", p->filters[2]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-suite-exclude=", p->filters[3]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tse=", p->filters[3]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-case=", p->filters[4]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tc=", p->filters[4]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-case-exclude=", p->filters[5]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tce=", p->filters[5]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "subcase=", p->filters[6]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sc=", p->filters[6]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "subcase-exclude=", p->filters[7]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sce=", p->filters[7]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "reporters=", p->filters[8]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "r=", p->filters[8]);
+ // clang-format on
+
+ int intRes = 0;
+ String strRes;
+
+#define DOCTEST_PARSE_AS_BOOL_OR_FLAG(name, sname, var, default) \
+ if(parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", option_bool, intRes) || \
+ parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", option_bool, intRes)) \
+ p->var = static_cast<bool>(intRes); \
+ else if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name) || \
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname)) \
+ p->var = true; \
+ else if(withDefaults) \
+ p->var = default
+
+#define DOCTEST_PARSE_INT_OPTION(name, sname, var, default) \
+ if(parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", option_int, intRes) || \
+ parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", option_int, intRes)) \
+ p->var = intRes; \
+ else if(withDefaults) \
+ p->var = default
+
+#define DOCTEST_PARSE_STR_OPTION(name, sname, var, default) \
+ if(parseOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", &strRes, default) || \
+ parseOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", &strRes, default) || \
+ withDefaults) \
+ p->var = strRes
+
+ // clang-format off
+ DOCTEST_PARSE_STR_OPTION("out", "o", out, "");
+ DOCTEST_PARSE_STR_OPTION("order-by", "ob", order_by, "file");
+ DOCTEST_PARSE_INT_OPTION("rand-seed", "rs", rand_seed, 0);
+
+ DOCTEST_PARSE_INT_OPTION("first", "f", first, 0);
+ DOCTEST_PARSE_INT_OPTION("last", "l", last, UINT_MAX);
+
+ DOCTEST_PARSE_INT_OPTION("abort-after", "aa", abort_after, 0);
+ DOCTEST_PARSE_INT_OPTION("subcase-filter-levels", "scfl", subcase_filter_levels, INT_MAX);
+
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("success", "s", success, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("case-sensitive", "cs", case_sensitive, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("exit", "e", exit, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("duration", "d", duration, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-throw", "nt", no_throw, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-exitcode", "ne", no_exitcode, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-run", "nr", no_run, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-version", "nv", no_version, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-colors", "nc", no_colors, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("force-colors", "fc", force_colors, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-breaks", "nb", no_breaks, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-skip", "ns", no_skip, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("gnu-file-line", "gfl", gnu_file_line, !bool(DOCTEST_MSVC));
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-path-filenames", "npf", no_path_in_filenames, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-line-numbers", "nln", no_line_numbers, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-debug-output", "ndo", no_debug_output, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-skipped-summary", "nss", no_skipped_summary, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-time-in-output", "ntio", no_time_in_output, false);
+ // clang-format on
+
+ if(withDefaults) {
+ p->help = false;
+ p->version = false;
+ p->count = false;
+ p->list_test_cases = false;
+ p->list_test_suites = false;
+ p->list_reporters = false;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "help") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "h") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "?")) {
+ p->help = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "version") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "v")) {
+ p->version = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "count") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "c")) {
+ p->count = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-test-cases") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "ltc")) {
+ p->list_test_cases = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-test-suites") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "lts")) {
+ p->list_test_suites = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-reporters") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "lr")) {
+ p->list_reporters = true;
+ p->exit = true;
+ }
+}
+
+// allows the user to add procedurally to the filters from the command line
+void Context::addFilter(const char* filter, const char* value) { setOption(filter, value); }
+
+// allows the user to clear all filters from the command line
+void Context::clearFilters() {
+ for(auto& curr : p->filters)
+ curr.clear();
+}
+
+// allows the user to override procedurally the int/bool options from the command line
+void Context::setOption(const char* option, int value) {
+ setOption(option, toString(value).c_str());
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+}
+
+// allows the user to override procedurally the string options from the command line
+void Context::setOption(const char* option, const char* value) {
+ auto argv = String("-") + option + "=" + value;
+ auto lvalue = argv.c_str();
+ parseArgs(1, &lvalue);
+}
+
+// users should query this in their main() and exit the program if true
+bool Context::shouldExit() { return p->exit; }
+
+void Context::setAsDefaultForAssertsOutOfTestCases() { g_cs = p; }
+
+void Context::setAssertHandler(detail::assert_handler ah) { p->ah = ah; }
+
+// the main function that does all the filtering and test running
+int Context::run() {
+ using namespace detail;
+
+ // save the old context state in case such was setup - for using asserts out of a testing context
+ auto old_cs = g_cs;
+ // this is the current contest
+ g_cs = p;
+ is_running_in_test = true;
+
+ g_no_colors = p->no_colors;
+ p->resetRunData();
+
+ // stdout by default
+ p->cout = &std::cout;
+ p->cerr = &std::cerr;
+
+ // or to a file if specified
+ std::fstream fstr;
+ if(p->out.size()) {
+ fstr.open(p->out.c_str(), std::fstream::out);
+ p->cout = &fstr;
+ }
+
+ FatalConditionHandler::allocateAltStackMem();
+
+ auto cleanup_and_return = [&]() {
+ FatalConditionHandler::freeAltStackMem();
+
+ if(fstr.is_open())
+ fstr.close();
+
+ // restore context
+ g_cs = old_cs;
+ is_running_in_test = false;
+
+ // we have to free the reporters which were allocated when the run started
+ for(auto& curr : p->reporters_currently_used)
+ delete curr;
+ p->reporters_currently_used.clear();
+
+ if(p->numTestCasesFailed && !p->no_exitcode)
+ return EXIT_FAILURE;
+ return EXIT_SUCCESS;
+ };
+
+ // setup default reporter if none is given through the command line
+ if(p->filters[8].empty())
+ p->filters[8].push_back("console");
+
+ // check to see if any of the registered reporters has been selected
+ for(auto& curr : getReporters()) {
+ if(matchesAny(curr.first.second.c_str(), p->filters[8], false, p->case_sensitive))
+ p->reporters_currently_used.push_back(curr.second(*g_cs));
+ }
+
+ // TODO: check if there is nothing in reporters_currently_used
+
+ // prepend all listeners
+ for(auto& curr : getListeners())
+ p->reporters_currently_used.insert(p->reporters_currently_used.begin(), curr.second(*g_cs));
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(isDebuggerActive() && p->no_debug_output == false)
+ p->reporters_currently_used.push_back(new DebugOutputWindowReporter(*g_cs));
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ // handle version, help and no_run
+ if(p->no_run || p->version || p->help || p->list_reporters) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(report_query, QueryData());
+
+ return cleanup_and_return();
+ }
+
+ std::vector<const TestCase*> testArray;
+ for(auto& curr : getRegisteredTests())
+ testArray.push_back(&curr);
+ p->numTestCases = testArray.size();
+
+ // sort the collected records
+ if(!testArray.empty()) {
+ if(p->order_by.compare("file", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), fileOrderComparator);
+ } else if(p->order_by.compare("suite", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), suiteOrderComparator);
+ } else if(p->order_by.compare("name", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), nameOrderComparator);
+ } else if(p->order_by.compare("rand", true) == 0) {
+ std::srand(p->rand_seed);
+
+ // random_shuffle implementation
+ const auto first = &testArray[0];
+ for(size_t i = testArray.size() - 1; i > 0; --i) {
+ int idxToSwap = std::rand() % (i + 1); // NOLINT
+
+ const auto temp = first[i];
+
+ first[i] = first[idxToSwap];
+ first[idxToSwap] = temp;
+ }
+ } else if(p->order_by.compare("none", true) == 0) {
+ // means no sorting - beneficial for death tests which call into the executable
+ // with a specific test case in mind - we don't want to slow down the startup times
+ }
+ }
+
+ std::set<String> testSuitesPassingFilt;
+
+ bool query_mode = p->count || p->list_test_cases || p->list_test_suites;
+ std::vector<const TestCaseData*> queryResults;
+
+ if(!query_mode)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_start, DOCTEST_EMPTY);
+
+ // invoke the registered functions if they match the filter criteria (or just count them)
+ for(auto& curr : testArray) {
+ const auto& tc = *curr;
+
+ bool skip_me = false;
+ if(tc.m_skip && !p->no_skip)
+ skip_me = true;
+
+ if(!matchesAny(tc.m_file.c_str(), p->filters[0], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_file.c_str(), p->filters[1], false, p->case_sensitive))
+ skip_me = true;
+ if(!matchesAny(tc.m_test_suite, p->filters[2], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_test_suite, p->filters[3], false, p->case_sensitive))
+ skip_me = true;
+ if(!matchesAny(tc.m_name, p->filters[4], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_name, p->filters[5], false, p->case_sensitive))
+ skip_me = true;
+
+ if(!skip_me)
+ p->numTestCasesPassingFilters++;
+
+ // skip the test if it is not in the execution range
+ if((p->last < p->numTestCasesPassingFilters && p->first <= p->last) ||
+ (p->first > p->numTestCasesPassingFilters))
+ skip_me = true;
+
+ if(skip_me) {
+ if(!query_mode)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_skipped, tc);
+ continue;
+ }
+
+ // do not execute the test if we are to only count the number of filter passing tests
+ if(p->count)
+ continue;
+
+ // print the name of the test and don't execute it
+ if(p->list_test_cases) {
+ queryResults.push_back(&tc);
+ continue;
+ }
+
+ // print the name of the test suite if not done already and don't execute it
+ if(p->list_test_suites) {
+ if((testSuitesPassingFilt.count(tc.m_test_suite) == 0) && tc.m_test_suite[0] != '\0') {
+ queryResults.push_back(&tc);
+ testSuitesPassingFilt.insert(tc.m_test_suite);
+ p->numTestSuitesPassingFilters++;
+ }
+ continue;
+ }
+
+ // execute the test if it passes all the filtering
+ {
+ p->currentTest = &tc;
+
+ p->failure_flags = TestCaseFailureReason::None;
+ p->seconds = 0;
+
+ // reset atomic counters
+ p->numAssertsFailedCurrentTest_atomic = 0;
+ p->numAssertsCurrentTest_atomic = 0;
+
+ p->subcasesPassed.clear();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_start, tc);
+
+ p->timer.start();
+
+ bool run_test = true;
+
+ do {
+ // reset some of the fields for subcases (except for the set of fully passed ones)
+ p->should_reenter = false;
+ p->subcasesCurrentMaxLevel = 0;
+ p->subcasesStack.clear();
+
+ p->shouldLogCurrentException = true;
+
+ // reset stuff for logging with INFO()
+ p->stringifiedContexts.clear();
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ try {
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+// MSVC 2015 diagnoses fatalConditionHandler as unused (because reset() is a static method)
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4101) // unreferenced local variable
+ FatalConditionHandler fatalConditionHandler; // Handle signals
+ // execute the test
+ tc.m_test();
+ fatalConditionHandler.reset();
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ } catch(const TestFailureException&) {
+ p->failure_flags |= TestCaseFailureReason::AssertFailure;
+ } catch(...) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_exception,
+ {translateActiveException(), false});
+ p->failure_flags |= TestCaseFailureReason::Exception;
+ }
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+ // exit this loop if enough assertions have failed - even if there are more subcases
+ if(p->abort_after > 0 &&
+ p->numAssertsFailed + p->numAssertsFailedCurrentTest_atomic >= p->abort_after) {
+ run_test = false;
+ p->failure_flags |= TestCaseFailureReason::TooManyFailedAsserts;
+ }
+
+ if(p->should_reenter && run_test)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_reenter, tc);
+ if(!p->should_reenter)
+ run_test = false;
+ } while(run_test);
+
+ p->finalizeTestCaseData();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_end, *g_cs);
+
+ p->currentTest = nullptr;
+
+ // stop executing tests if enough assertions have failed
+ if(p->abort_after > 0 && p->numAssertsFailed >= p->abort_after)
+ break;
+ }
+ }
+
+ if(!query_mode) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_end, *g_cs);
+ } else {
+ QueryData qdata;
+ qdata.run_stats = g_cs;
+ qdata.data = queryResults.data();
+ qdata.num_data = unsigned(queryResults.size());
+ DOCTEST_ITERATE_THROUGH_REPORTERS(report_query, qdata);
+ }
+
+ // see these issues on the reasoning for this:
+ // - https://github.com/onqtam/doctest/issues/143#issuecomment-414418903
+ // - https://github.com/onqtam/doctest/issues/126
+ auto DOCTEST_FIX_FOR_MACOS_LIBCPP_IOSFWD_STRING_LINK_ERRORS = []() DOCTEST_NOINLINE
+ { std::cout << std::string(); };
+ DOCTEST_FIX_FOR_MACOS_LIBCPP_IOSFWD_STRING_LINK_ERRORS();
+
+ return cleanup_and_return();
+}
+
+IReporter::~IReporter() = default;
+
+int IReporter::get_num_active_contexts() { return detail::g_infoContexts.size(); }
+const IContextScope* const* IReporter::get_active_contexts() {
+ return get_num_active_contexts() ? &detail::g_infoContexts[0] : nullptr;
+}
+
+int IReporter::get_num_stringified_contexts() { return detail::g_cs->stringifiedContexts.size(); }
+const String* IReporter::get_stringified_contexts() {
+ return get_num_stringified_contexts() ? &detail::g_cs->stringifiedContexts[0] : nullptr;
+}
+
+namespace detail {
+ void registerReporterImpl(const char* name, int priority, reporterCreatorFunc c, bool isReporter) {
+ if(isReporter)
+ getReporters().insert(reporterMap::value_type(reporterMap::key_type(priority, name), c));
+ else
+ getListeners().insert(reporterMap::value_type(reporterMap::key_type(priority, name), c));
+ }
+} // namespace detail
+
+} // namespace doctest
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+#ifdef DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4007) // 'function' : must be 'attribute' - see issue #182
+int main(int argc, char** argv) { return doctest::Context(argc, argv).run(); }
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#endif // DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_LIBRARY_IMPLEMENTATION
+#endif // DOCTEST_CONFIG_IMPLEMENT
diff --git a/contrib/doctest/doctest/extensions/doctest_mpi.h b/contrib/doctest/doctest/extensions/doctest_mpi.h
new file mode 100644
index 0000000..45f82dd
--- /dev/null
+++ b/contrib/doctest/doctest/extensions/doctest_mpi.h
@@ -0,0 +1,120 @@
+#pragma once
+
+#ifdef DOCTEST_CONFIG_IMPLEMENT
+
+#include "doctest/doctest.h"
+#include "mpi_reporter.h"
+
+#else
+
+#include "mpi.h"
+#include <numeric>
+#include <vector>
+#include "doctest/doctest.h"
+#include <cassert>
+#include <string>
+
+namespace doctest {
+
+inline
+int mpi_world_nb_procs() {
+ int n;
+ MPI_Comm_size(MPI_COMM_WORLD, &n);
+ return n;
+}
+
+struct mpi_sub_comm {
+ int nb_procs;
+ int rank;
+ MPI_Comm comm;
+
+ mpi_sub_comm( mpi_sub_comm const& ) = delete;
+ mpi_sub_comm& operator=( mpi_sub_comm const& ) = delete;
+
+ mpi_sub_comm(int nb_prcs) noexcept
+ : nb_procs(nb_prcs)
+ , rank(-1)
+ , comm(MPI_COMM_NULL)
+ {
+ int comm_world_rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &comm_world_rank);
+ if (nb_procs>mpi_world_nb_procs()) {
+ if (comm_world_rank==0) {
+ MESSAGE(
+ "Unable to run test: need ", std::to_string(nb_procs), " procs",
+ " but program launched with only ", std::to_string(doctest::mpi_world_nb_procs()), "."
+ );
+ CHECK(nb_procs<=mpi_world_nb_procs());
+ }
+ } else {
+ int color = MPI_UNDEFINED;
+ if(comm_world_rank < nb_procs){
+ color = 0;
+ }
+ MPI_Comm_split(MPI_COMM_WORLD, color, comm_world_rank, &comm);
+
+ if(comm != MPI_COMM_NULL){
+ MPI_Comm_rank(comm, &rank);
+ assert(rank==comm_world_rank);
+ }
+ }
+ }
+
+ ~mpi_sub_comm() {
+ if(comm != MPI_COMM_NULL){
+ MPI_Comm_free(&comm);
+ }
+ }
+};
+
+
+template<int nb_procs, class F>
+void execute_mpi_test_case(F func) {
+ mpi_sub_comm sub(nb_procs);
+ if (sub.comm != MPI_COMM_NULL) {
+ func(sub.rank,nb_procs,sub.comm,std::integral_constant<int,nb_procs>{});
+ };
+}
+
+} // doctest
+
+
+#define DOCTEST_MPI_GEN_ASSERTION(rank_to_test, assertion, ...) \
+ static_assert(rank_to_test<test_nb_procs_as_int_constant.value,"Trying to assert on a rank greater than the number of procs of the test!"); \
+ if(rank_to_test == test_rank) assertion(__VA_ARGS__)
+
+#define DOCTEST_MPI_WARN(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_WARN,__VA_ARGS__)
+#define DOCTEST_MPI_CHECK(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_CHECK,__VA_ARGS__)
+#define DOCTEST_MPI_REQUIRE(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_REQUIRE,__VA_ARGS__)
+#define DOCTEST_MPI_WARN_FALSE(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_WARN_FALSE,__VA_ARGS__)
+#define DOCTEST_MPI_CHECK_FALSE(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_CHECK_FALSE,__VA_ARGS__)
+#define DOCTEST_MPI_REQUIRE_FALSE(rank_to_test, ...) DOCTEST_MPI_GEN_ASSERTION(rank_to_test,DOCTEST_REQUIRE_FALSE,__VA_ARGS__)
+
+#define DOCTEST_CREATE_MPI_TEST_CASE(name,nb_procs,func) \
+ static void func(DOCTEST_UNUSED int test_rank, DOCTEST_UNUSED int test_nb_procs, DOCTEST_UNUSED MPI_Comm test_comm, DOCTEST_UNUSED std::integral_constant<int,nb_procs>); \
+ TEST_CASE(name * doctest::description("MPI_TEST_CASE")) { \
+ doctest::execute_mpi_test_case<nb_procs>(func); \
+ } \
+ static void func(DOCTEST_UNUSED int test_rank, DOCTEST_UNUSED int test_nb_procs, DOCTEST_UNUSED MPI_Comm test_comm, DOCTEST_UNUSED std::integral_constant<int,nb_procs> test_nb_procs_as_int_constant)
+ // DOC: test_rank, test_nb_procs, and test_comm are available UNDER THESE SPECIFIC NAMES in the body of the unit test
+ // DOC: test_nb_procs_as_int_constant is equal to test_nb_procs, but as a compile time value
+ // (used in CHECK-like macros to assert the checked rank exists)
+
+#define DOCTEST_MPI_TEST_CASE(name,nb_procs) \
+ DOCTEST_CREATE_MPI_TEST_CASE(name,nb_procs,DOCTEST_ANONYMOUS(DOCTEST_MPI_FUNC))
+
+
+// == SHORT VERSIONS OF THE MACROS
+#if !defined(DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES)
+#define MPI_WARN DOCTEST_MPI_WARN
+#define MPI_CHECK DOCTEST_MPI_CHECK
+#define MPI_REQUIRE DOCTEST_MPI_REQUIRE
+#define MPI_WARN_FALSE DOCTEST_MPI_WARN_FALSE
+#define MPI_CHECK_FALSE DOCTEST_MPI_CHECK_FALSE
+#define MPI_REQUIRE_FALSE DOCTEST_MPI_REQUIRE_FALSE
+
+#define MPI_TEST_CASE DOCTEST_MPI_TEST_CASE
+#endif // DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES
+
+
+#endif // DOCTEST_CONFIG_IMPLEMENT
diff --git a/contrib/doctest/doctest/extensions/doctest_util.h b/contrib/doctest/doctest/extensions/doctest_util.h
new file mode 100644
index 0000000..00e1151
--- /dev/null
+++ b/contrib/doctest/doctest/extensions/doctest_util.h
@@ -0,0 +1,34 @@
+//
+// doctest_util.h - an accompanying extensions header to the main doctest.h header
+//
+// Copyright (c) 2016-2021 Viktor Kirilov
+//
+// Distributed under the MIT Software License
+// See accompanying file LICENSE.txt or copy at
+// https://opensource.org/licenses/MIT
+//
+// The documentation can be found at the library's page:
+// https://github.com/onqtam/doctest/blob/master/doc/markdown/readme.md
+//
+
+#pragma once
+
+#ifndef DOCTEST_LIBRARY_INCLUDED
+#include "../doctest.h"
+#endif
+
+#include <memory>
+#include <vector>
+#include <string>
+
+namespace doctest {
+
+ inline void applyCommandLine(doctest::Context& ctx, const std::vector<std::string>& args) {
+ auto doctest_args = std::make_unique<const char*[]>(args.size());
+ for (size_t i = 0; i < args.size(); ++i) {
+ doctest_args[i] = args[i].c_str();
+ }
+ ctx.applyCommandLine(args.size(), doctest_args.get());
+ }
+
+} // namespace doctest
diff --git a/contrib/doctest/doctest/extensions/mpi_reporter.h b/contrib/doctest/doctest/extensions/mpi_reporter.h
new file mode 100644
index 0000000..9cccb5e
--- /dev/null
+++ b/contrib/doctest/doctest/extensions/mpi_reporter.h
@@ -0,0 +1,236 @@
+#pragma once
+
+// #include <doctest/doctest.h>
+#include <fstream>
+#include <string>
+#include "mpi.h"
+
+
+#include <vector>
+#include <mutex>
+
+namespace doctest {
+
+namespace {
+
+// https://stackoverflow.com/a/11826666/1583122
+struct NullBuffer : std::streambuf {
+ int overflow(int c) { return c; }
+};
+class NullStream : public std::ostream {
+ public:
+ NullStream()
+ : std::ostream(&nullBuff)
+ {}
+ private:
+ NullBuffer nullBuff = {};
+};
+static NullStream nullStream;
+
+
+/* \brief Extends the ConsoleReporter of doctest
+ * Each process writes its results to its own file
+ * Intended to be used when a test assertion fails and the user wants to know exactly what happens on which process
+ */
+struct MpiFileReporter : public ConsoleReporter {
+ std::ofstream logfile_stream = {};
+
+ MpiFileReporter(const ContextOptions& co)
+ : ConsoleReporter(co,logfile_stream)
+ {
+ int rank = 0;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+ std::string logfile_name = "doctest_" + std::to_string(rank) + ".log";
+
+ logfile_stream = std::ofstream(logfile_name.c_str(), std::fstream::out);
+ }
+};
+
+
+/* \brief Extends the ConsoleReporter of doctest
+ * Allows to manage the execution of tests in a parallel framework
+ * All results are collected on rank 0
+ */
+struct MpiConsoleReporter : public ConsoleReporter {
+private:
+ static std::ostream& replace_by_null_if_not_rank_0(std::ostream* os) {
+ int rank = 0;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ if (rank==0) {
+ return *os;
+ } else {
+ return nullStream;
+ }
+ }
+public:
+ MpiConsoleReporter(const ContextOptions& co)
+ : ConsoleReporter(co,replace_by_null_if_not_rank_0(co.cout))
+ {}
+
+ std::string file_line_to_string(const char* file, int line,
+ const char* tail = ""){
+ std::stringstream ss;
+ ss << skipPathFromFilename(file)
+ << (opt.gnu_file_line ? ":" : "(")
+ << (opt.no_line_numbers ? 0 : line) // 0 or the real num depending on the option
+ << (opt.gnu_file_line ? ":" : "):") << tail;
+ return ss.str();
+ }
+
+ void test_run_end(const TestRunStats& p) override {
+ ConsoleReporter::test_run_end(p);
+
+ const bool anythingFailed = p.numTestCasesFailed > 0 || p.numAssertsFailed > 0;
+
+ // -----------------------------------------------------
+ // > Gather information in rank 0
+ int n_rank, rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &n_rank);
+
+ int g_numAsserts = 0;
+ int g_numAssertsFailed = 0;
+ int g_numTestCasesFailed = 0;
+
+ MPI_Reduce(&p.numAsserts , &g_numAsserts , 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&p.numAssertsFailed , &g_numAssertsFailed , 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&p.numTestCasesFailed, &g_numTestCasesFailed, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+
+ std::vector<int> numAssertsFailedByRank;
+ if(rank == 0){
+ numAssertsFailedByRank.resize(static_cast<std::size_t>(n_rank));
+ }
+
+ MPI_Gather(&p.numAssertsFailed, 1, MPI_INT, numAssertsFailedByRank.data(), 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+ if(rank == 0) {
+ separator_to_stream();
+ s << Color::Cyan << "[doctest] " << Color::None << "glob assertions: " << std::setw(6)
+ << g_numAsserts << " | "
+ << ((g_numAsserts == 0 || anythingFailed) ? Color::None : Color::Green)
+ << std::setw(6) << (g_numAsserts - g_numAssertsFailed) << " passed" << Color::None
+ << " | " << (g_numAssertsFailed > 0 ? Color::Red : Color::None) << std::setw(6)
+ << g_numAssertsFailed << " failed" << Color::None << " |\n";
+
+ separator_to_stream();
+ if(g_numAssertsFailed > 0){
+
+ s << Color::Cyan << "[doctest] " << Color::None << "fail on rank:" << std::setw(6) << "\n";
+ for(std::size_t i = 0; i < numAssertsFailedByRank.size(); ++i){
+ if( numAssertsFailedByRank[i] > 0 ){
+ s << std::setw(16) << " -> On rank [" << i << "] with " << numAssertsFailedByRank[i] << " test failed" << std::endl;
+ }
+ }
+ }
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "Status: " << (g_numTestCasesFailed > 0 ? Color::Red : Color::Green)
+ << ((g_numTestCasesFailed > 0) ? "FAILURE!" : "SUCCESS!") << Color::None << std::endl;
+ }
+ }
+
+ void test_case_end(const CurrentTestCaseStats& st) override {
+ if (is_mpi_test_case()) {
+ // function called by every rank at the end of a test
+ // if failed assertions happended, they have been sent to rank 0
+ // here rank zero gathers them and prints them all
+
+ int rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+ // Compute the number of assert with fail among all procs
+ int nb_fail_asserts_glob = 0;
+ MPI_Reduce(&st.numAssertsFailedCurrentTest, &nb_fail_asserts_glob, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+
+ if(rank == 0) {
+ MPI_Status status;
+ MPI_Status status_recv;
+
+ using id_string = std::pair<int,std::string>;
+ std::vector<id_string> msgs(static_cast<std::size_t>(nb_fail_asserts_glob));
+
+ for (std::size_t i=0; i<static_cast<std::size_t>(nb_fail_asserts_glob); ++i) {
+ MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
+
+ int count;
+ MPI_Get_count(&status, MPI_BYTE, &count);
+
+ std::string recv_msg(static_cast<std::size_t>(count),'\0');
+ void* recv_msg_data = const_cast<char*>(recv_msg.data()); // const_cast needed. Non-const .data() exists in C++11 though...
+ MPI_Recv(recv_msg_data, count, MPI_BYTE, status.MPI_SOURCE,
+ status.MPI_TAG, MPI_COMM_WORLD, &status_recv);
+
+ msgs[i] = {status.MPI_SOURCE,recv_msg};
+ }
+
+ std::sort(begin(msgs),end(msgs),[](const id_string& x, const id_string& y){ return x.first < y.first; });
+
+ // print
+ if (nb_fail_asserts_glob>0) {
+ separator_to_stream();
+ file_line_to_stream(tc->m_file.c_str(), static_cast<int>(tc->m_line), "\n");
+ if(tc->m_test_suite && tc->m_test_suite[0] != '\0')
+ s << Color::Yellow << "TEST SUITE: " << Color::None << tc->m_test_suite << "\n";
+ if(strncmp(tc->m_name, " Scenario:", 11) != 0)
+ s << Color::Yellow << "TEST CASE: ";
+ s << Color::None << tc->m_name << "\n\n";
+ for(const auto& msg : msgs) {
+ s << msg.second;
+ }
+ s << "\n";
+ }
+ }
+ }
+
+ ConsoleReporter::test_case_end(st);
+ }
+
+ bool is_mpi_test_case() const {
+ return tc->m_description != nullptr
+ && std::string(tc->m_description) == std::string("MPI_TEST_CASE");
+ }
+
+ void log_assert(const AssertData& rb) override {
+ if (!is_mpi_test_case()) {
+ ConsoleReporter::log_assert(rb);
+ } else {
+ int rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+
+ if(!rb.m_failed && !opt.success)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ std::stringstream failure_msg;
+ failure_msg << Color::Red << "On rank [" << rank << "] : " << Color::None;
+ failure_msg << file_line_to_string(rb.m_file, rb.m_line, " ");
+
+ if((rb.m_at & (assertType::is_throws_as | assertType::is_throws_with)) ==0){
+ failure_msg << Color::Cyan
+ << assertString(rb.m_at)
+ << "( " << rb.m_expr << " ) "
+ << Color::None
+
+ << (!rb.m_failed ? "is correct!\n" : "is NOT correct!\n")
+ << " values: "
+ << assertString(rb.m_at)
+ << "( " << rb.m_decomp.c_str() << " )\n";
+ }
+
+ std::string failure_str = failure_msg.str();
+ int failure_msg_size = static_cast<int>(failure_str.size());
+
+ MPI_Send(failure_str.c_str(), failure_msg_size, MPI_BYTE,
+ 0, rb.m_line, MPI_COMM_WORLD); // Tag = file line
+ }
+ }
+}; // MpiConsoleReporter
+
+// "1" is the priority - used for ordering when multiple reporters/listeners are used
+REGISTER_REPORTER("MpiConsoleReporter", 1, MpiConsoleReporter);
+REGISTER_REPORTER("MpiFileReporter", 1, MpiFileReporter);
+
+} // anonymous
+} // doctest
diff --git a/contrib/doctest/doctest/parts/doctest.cpp b/contrib/doctest/doctest/parts/doctest.cpp
new file mode 100644
index 0000000..5b74420
--- /dev/null
+++ b/contrib/doctest/doctest/parts/doctest.cpp
@@ -0,0 +1,3866 @@
+#if defined(DOCTEST_CONFIG_IMPLEMENT) || !defined(DOCTEST_SINGLE_HEADER)
+
+#ifndef DOCTEST_SINGLE_HEADER
+#include "doctest_fwd.h"
+#endif // DOCTEST_SINGLE_HEADER
+
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-macros")
+
+#ifndef DOCTEST_LIBRARY_IMPLEMENTATION
+#define DOCTEST_LIBRARY_IMPLEMENTATION
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wpadded")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wweak-vtables")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wglobal-constructors")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-prototypes")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-conversion")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wshorten-64-to-32")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-variable-declarations")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wswitch")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wswitch-enum")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wcovered-switch-default")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-noreturn")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-local-typedef")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wdisabled-macro-expansion")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-braces")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-field-initializers")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat-pedantic")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-member-function")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wnonportable-system-include-path")
+
+DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wpragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wconversion")
+DOCTEST_GCC_SUPPRESS_WARNING("-Weffc++")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-conversion")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-overflow")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-aliasing")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-field-initializers")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-braces")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-declarations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch-enum")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch-default")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunsafe-loop-optimizations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wold-style-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-local-typedefs")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wuseless-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-function")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmultiple-inheritance")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsuggest-attribute")
+
+DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+DOCTEST_MSVC_SUPPRESS_WARNING(4616) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4619) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4996) // The compiler encountered a deprecated declaration
+DOCTEST_MSVC_SUPPRESS_WARNING(4267) // 'var' : conversion from 'x' to 'y', possible loss of data
+DOCTEST_MSVC_SUPPRESS_WARNING(4706) // assignment within conditional expression
+DOCTEST_MSVC_SUPPRESS_WARNING(4512) // 'class' : assignment operator could not be generated
+DOCTEST_MSVC_SUPPRESS_WARNING(4127) // conditional expression is constant
+DOCTEST_MSVC_SUPPRESS_WARNING(4530) // C++ exception handler used, but unwind semantics not enabled
+DOCTEST_MSVC_SUPPRESS_WARNING(4577) // 'noexcept' used with no exception handling mode specified
+DOCTEST_MSVC_SUPPRESS_WARNING(4774) // format string expected in argument is not a string literal
+DOCTEST_MSVC_SUPPRESS_WARNING(4365) // conversion from 'int' to 'unsigned', signed/unsigned mismatch
+DOCTEST_MSVC_SUPPRESS_WARNING(4820) // padding in structs
+DOCTEST_MSVC_SUPPRESS_WARNING(4640) // construction of local static object is not thread-safe
+DOCTEST_MSVC_SUPPRESS_WARNING(5039) // pointer to potentially throwing function passed to extern C
+DOCTEST_MSVC_SUPPRESS_WARNING(5045) // Spectre mitigation stuff
+DOCTEST_MSVC_SUPPRESS_WARNING(4626) // assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5027) // move assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5026) // move constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4625) // copy constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4800) // forcing value to bool 'true' or 'false' (performance warning)
+// static analysis
+DOCTEST_MSVC_SUPPRESS_WARNING(26439) // This kind of function may not throw. Declare it 'noexcept'
+DOCTEST_MSVC_SUPPRESS_WARNING(26495) // Always initialize a member variable
+DOCTEST_MSVC_SUPPRESS_WARNING(26451) // Arithmetic overflow ...
+DOCTEST_MSVC_SUPPRESS_WARNING(26444) // Avoid unnamed objects with custom construction and dtor...
+DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum'
+
+DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN
+
+// required includes - will go only in one translation unit!
+#include <ctime>
+#include <cmath>
+#include <climits>
+// borland (Embarcadero) compiler requires math.h and not cmath - https://github.com/onqtam/doctest/pull/37
+#ifdef __BORLANDC__
+#include <math.h>
+#endif // __BORLANDC__
+#include <new>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <utility>
+#include <fstream>
+#include <sstream>
+#include <iostream>
+#include <algorithm>
+#include <iomanip>
+#include <vector>
+#include <atomic>
+#include <mutex>
+#include <set>
+#include <map>
+#include <exception>
+#include <stdexcept>
+#include <csignal>
+#include <cfloat>
+#include <cctype>
+#include <cstdint>
+
+#ifdef DOCTEST_PLATFORM_MAC
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/sysctl.h>
+#endif // DOCTEST_PLATFORM_MAC
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+
+// defines for a leaner windows.h
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif // WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif // NOMINMAX
+
+// not sure what AfxWin.h is for - here I do what Catch does
+#ifdef __AFXDLL
+#include <AfxWin.h>
+#else
+#include <windows.h>
+#endif
+#include <io.h>
+
+#else // DOCTEST_PLATFORM_WINDOWS
+
+#include <sys/time.h>
+#include <unistd.h>
+
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+// this is a fix for https://github.com/onqtam/doctest/issues/348
+// https://mail.gnome.org/archives/xml/2012-January/msg00000.html
+#if !defined(HAVE_UNISTD_H) && !defined(STDOUT_FILENO)
+#define STDOUT_FILENO fileno(stdout)
+#endif // HAVE_UNISTD_H
+
+DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END
+
+// counts the number of elements in a C array
+#define DOCTEST_COUNTOF(x) (sizeof(x) / sizeof(x[0]))
+
+#ifdef DOCTEST_CONFIG_DISABLE
+#define DOCTEST_BRANCH_ON_DISABLED(if_disabled, if_not_disabled) if_disabled
+#else // DOCTEST_CONFIG_DISABLE
+#define DOCTEST_BRANCH_ON_DISABLED(if_disabled, if_not_disabled) if_not_disabled
+#endif // DOCTEST_CONFIG_DISABLE
+
+#ifndef DOCTEST_CONFIG_OPTIONS_PREFIX
+#define DOCTEST_CONFIG_OPTIONS_PREFIX "dt-"
+#endif
+
+#ifndef DOCTEST_THREAD_LOCAL
+#define DOCTEST_THREAD_LOCAL thread_local
+#endif
+
+#ifndef DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES
+#define DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES 32
+#endif
+
+#ifndef DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE
+#define DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE 64
+#endif
+
+#ifdef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+#define DOCTEST_OPTIONS_PREFIX_DISPLAY DOCTEST_CONFIG_OPTIONS_PREFIX
+#else
+#define DOCTEST_OPTIONS_PREFIX_DISPLAY ""
+#endif
+
+#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP)
+#define DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+#endif
+
+namespace doctest {
+
+bool is_running_in_test = false;
+
+namespace {
+ using namespace detail;
+ // case insensitive strcmp
+ int stricmp(const char* a, const char* b) {
+ for(;; a++, b++) {
+ const int d = tolower(*a) - tolower(*b);
+ if(d != 0 || !*a)
+ return d;
+ }
+ }
+
+ template <typename T>
+ String fpToString(T value, int precision) {
+ std::ostringstream oss;
+ oss << std::setprecision(precision) << std::fixed << value;
+ std::string d = oss.str();
+ size_t i = d.find_last_not_of('0');
+ if(i != std::string::npos && i != d.size() - 1) {
+ if(d[i] == '.')
+ i++;
+ d = d.substr(0, i + 1);
+ }
+ return d.c_str();
+ }
+
+ struct Endianness
+ {
+ enum Arch
+ {
+ Big,
+ Little
+ };
+
+ static Arch which() {
+ int x = 1;
+ // casting any data pointer to char* is allowed
+ auto ptr = reinterpret_cast<char*>(&x);
+ if(*ptr)
+ return Little;
+ return Big;
+ }
+ };
+} // namespace
+
+namespace detail {
+ void my_memcpy(void* dest, const void* src, unsigned num) { memcpy(dest, src, num); }
+
+ String rawMemoryToString(const void* object, unsigned size) {
+ // Reverse order for little endian architectures
+ int i = 0, end = static_cast<int>(size), inc = 1;
+ if(Endianness::which() == Endianness::Little) {
+ i = end - 1;
+ end = inc = -1;
+ }
+
+ unsigned const char* bytes = static_cast<unsigned const char*>(object);
+ std::ostringstream oss;
+ oss << "0x" << std::setfill('0') << std::hex;
+ for(; i != end; i += inc)
+ oss << std::setw(2) << static_cast<unsigned>(bytes[i]);
+ return oss.str().c_str();
+ }
+
+ DOCTEST_THREAD_LOCAL std::ostringstream g_oss; // NOLINT(cert-err58-cpp)
+
+ std::ostream* getTlsOss() {
+ g_oss.clear(); // there shouldn't be anything worth clearing in the flags
+ g_oss.str(""); // the slow way of resetting a string stream
+ //g_oss.seekp(0); // optimal reset - as seen here: https://stackoverflow.com/a/624291/3162383
+ return &g_oss;
+ }
+
+ String getTlsOssResult() {
+ //g_oss << std::ends; // needed - as shown here: https://stackoverflow.com/a/624291/3162383
+ return g_oss.str().c_str();
+ }
+
+#ifndef DOCTEST_CONFIG_DISABLE
+
+namespace timer_large_integer
+{
+
+#if defined(DOCTEST_PLATFORM_WINDOWS)
+ typedef ULONGLONG type;
+#else // DOCTEST_PLATFORM_WINDOWS
+ using namespace std;
+ typedef uint64_t type;
+#endif // DOCTEST_PLATFORM_WINDOWS
+}
+
+typedef timer_large_integer::type ticks_t;
+
+#ifdef DOCTEST_CONFIG_GETCURRENTTICKS
+ ticks_t getCurrentTicks() { return DOCTEST_CONFIG_GETCURRENTTICKS(); }
+#elif defined(DOCTEST_PLATFORM_WINDOWS)
+ ticks_t getCurrentTicks() {
+ static LARGE_INTEGER hz = {0}, hzo = {0};
+ if(!hz.QuadPart) {
+ QueryPerformanceFrequency(&hz);
+ QueryPerformanceCounter(&hzo);
+ }
+ LARGE_INTEGER t;
+ QueryPerformanceCounter(&t);
+ return ((t.QuadPart - hzo.QuadPart) * LONGLONG(1000000)) / hz.QuadPart;
+ }
+#else // DOCTEST_PLATFORM_WINDOWS
+ ticks_t getCurrentTicks() {
+ timeval t;
+ gettimeofday(&t, nullptr);
+ return static_cast<ticks_t>(t.tv_sec) * 1000000 + static_cast<ticks_t>(t.tv_usec);
+ }
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ struct Timer
+ {
+ void start() { m_ticks = getCurrentTicks(); }
+ unsigned int getElapsedMicroseconds() const {
+ return static_cast<unsigned int>(getCurrentTicks() - m_ticks);
+ }
+ //unsigned int getElapsedMilliseconds() const {
+ // return static_cast<unsigned int>(getElapsedMicroseconds() / 1000);
+ //}
+ double getElapsedSeconds() const { return static_cast<double>(getCurrentTicks() - m_ticks) / 1000000.0; }
+
+ private:
+ ticks_t m_ticks = 0;
+ };
+
+#ifdef DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+ template <typename T>
+ using AtomicOrMultiLaneAtomic = std::atomic<T>;
+#else // DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+ // Provides a multilane implementation of an atomic variable that supports add, sub, load,
+ // store. Instead of using a single atomic variable, this splits up into multiple ones,
+ // each sitting on a separate cache line. The goal is to provide a speedup when most
+ // operations are modifying. It achieves this with two properties:
+ //
+ // * Multiple atomics are used, so chance of congestion from the same atomic is reduced.
+ // * Each atomic sits on a separate cache line, so false sharing is reduced.
+ //
+ // The disadvantage is that there is a small overhead due to the use of TLS, and load/store
+ // is slower because all atomics have to be accessed.
+ template <typename T>
+ class MultiLaneAtomic
+ {
+ struct CacheLineAlignedAtomic
+ {
+ std::atomic<T> atomic{};
+ char padding[DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE - sizeof(std::atomic<T>)];
+ };
+ CacheLineAlignedAtomic m_atomics[DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES];
+
+ static_assert(sizeof(CacheLineAlignedAtomic) == DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE,
+ "guarantee one atomic takes exactly one cache line");
+
+ public:
+ T operator++() DOCTEST_NOEXCEPT { return fetch_add(1) + 1; }
+
+ T operator++(int) DOCTEST_NOEXCEPT { return fetch_add(1); }
+
+ T fetch_add(T arg, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ return myAtomic().fetch_add(arg, order);
+ }
+
+ T fetch_sub(T arg, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ return myAtomic().fetch_sub(arg, order);
+ }
+
+ operator T() const DOCTEST_NOEXCEPT { return load(); }
+
+ T load(std::memory_order order = std::memory_order_seq_cst) const DOCTEST_NOEXCEPT {
+ auto result = T();
+ for(auto const& c : m_atomics) {
+ result += c.atomic.load(order);
+ }
+ return result;
+ }
+
+ T operator=(T desired) DOCTEST_NOEXCEPT {
+ store(desired);
+ return desired;
+ }
+
+ void store(T desired, std::memory_order order = std::memory_order_seq_cst) DOCTEST_NOEXCEPT {
+ // first value becomes desired", all others become 0.
+ for(auto& c : m_atomics) {
+ c.atomic.store(desired, order);
+ desired = {};
+ }
+ }
+
+ private:
+ // Each thread has a different atomic that it operates on. If more than NumLanes threads
+ // use this, some will use the same atomic. So performance will degrate a bit, but still
+ // everything will work.
+ //
+ // The logic here is a bit tricky. The call should be as fast as possible, so that there
+ // is minimal to no overhead in determining the correct atomic for the current thread.
+ //
+ // 1. A global static counter laneCounter counts continuously up.
+ // 2. Each successive thread will use modulo operation of that counter so it gets an atomic
+ // assigned in a round-robin fashion.
+ // 3. This tlsLaneIdx is stored in the thread local data, so it is directly available with
+ // little overhead.
+ std::atomic<T>& myAtomic() DOCTEST_NOEXCEPT {
+ static std::atomic<size_t> laneCounter;
+ DOCTEST_THREAD_LOCAL size_t tlsLaneIdx =
+ laneCounter++ % DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES;
+
+ return m_atomics[tlsLaneIdx].atomic;
+ }
+ };
+
+ template <typename T>
+ using AtomicOrMultiLaneAtomic = MultiLaneAtomic<T>;
+#endif // DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS
+
+ // this holds both parameters from the command line and runtime data for tests
+ struct ContextState : ContextOptions, TestRunStats, CurrentTestCaseStats
+ {
+ AtomicOrMultiLaneAtomic<int> numAssertsCurrentTest_atomic;
+ AtomicOrMultiLaneAtomic<int> numAssertsFailedCurrentTest_atomic;
+
+ std::vector<std::vector<String>> filters = decltype(filters)(9); // 9 different filters
+
+ std::vector<IReporter*> reporters_currently_used;
+
+ assert_handler ah = nullptr;
+
+ Timer timer;
+
+ std::vector<String> stringifiedContexts; // logging from INFO() due to an exception
+
+ // stuff for subcases
+ std::vector<SubcaseSignature> subcasesStack;
+ std::set<decltype(subcasesStack)> subcasesPassed;
+ int subcasesCurrentMaxLevel;
+ bool should_reenter;
+ std::atomic<bool> shouldLogCurrentException;
+
+ void resetRunData() {
+ numTestCases = 0;
+ numTestCasesPassingFilters = 0;
+ numTestSuitesPassingFilters = 0;
+ numTestCasesFailed = 0;
+ numAsserts = 0;
+ numAssertsFailed = 0;
+ numAssertsCurrentTest = 0;
+ numAssertsFailedCurrentTest = 0;
+ }
+
+ void finalizeTestCaseData() {
+ seconds = timer.getElapsedSeconds();
+
+ // update the non-atomic counters
+ numAsserts += numAssertsCurrentTest_atomic;
+ numAssertsFailed += numAssertsFailedCurrentTest_atomic;
+ numAssertsCurrentTest = numAssertsCurrentTest_atomic;
+ numAssertsFailedCurrentTest = numAssertsFailedCurrentTest_atomic;
+
+ if(numAssertsFailedCurrentTest)
+ failure_flags |= TestCaseFailureReason::AssertFailure;
+
+ if(Approx(currentTest->m_timeout).epsilon(DBL_EPSILON) != 0 &&
+ Approx(seconds).epsilon(DBL_EPSILON) > currentTest->m_timeout)
+ failure_flags |= TestCaseFailureReason::Timeout;
+
+ if(currentTest->m_should_fail) {
+ if(failure_flags) {
+ failure_flags |= TestCaseFailureReason::ShouldHaveFailedAndDid;
+ } else {
+ failure_flags |= TestCaseFailureReason::ShouldHaveFailedButDidnt;
+ }
+ } else if(failure_flags && currentTest->m_may_fail) {
+ failure_flags |= TestCaseFailureReason::CouldHaveFailedAndDid;
+ } else if(currentTest->m_expected_failures > 0) {
+ if(numAssertsFailedCurrentTest == currentTest->m_expected_failures) {
+ failure_flags |= TestCaseFailureReason::FailedExactlyNumTimes;
+ } else {
+ failure_flags |= TestCaseFailureReason::DidntFailExactlyNumTimes;
+ }
+ }
+
+ bool ok_to_fail = (TestCaseFailureReason::ShouldHaveFailedAndDid & failure_flags) ||
+ (TestCaseFailureReason::CouldHaveFailedAndDid & failure_flags) ||
+ (TestCaseFailureReason::FailedExactlyNumTimes & failure_flags);
+
+ // if any subcase has failed - the whole test case has failed
+ if(failure_flags && !ok_to_fail)
+ numTestCasesFailed++;
+ }
+ };
+
+ ContextState* g_cs = nullptr;
+
+ // used to avoid locks for the debug output
+ // TODO: figure out if this is indeed necessary/correct - seems like either there still
+ // could be a race or that there wouldn't be a race even if using the context directly
+ DOCTEST_THREAD_LOCAL bool g_no_colors;
+
+#endif // DOCTEST_CONFIG_DISABLE
+} // namespace detail
+
+void String::setOnHeap() { *reinterpret_cast<unsigned char*>(&buf[last]) = 128; }
+void String::setLast(unsigned in) { buf[last] = char(in); }
+
+void String::copy(const String& other) {
+ using namespace std;
+ if(other.isOnStack()) {
+ memcpy(buf, other.buf, len);
+ } else {
+ setOnHeap();
+ data.size = other.data.size;
+ data.capacity = data.size + 1;
+ data.ptr = new char[data.capacity];
+ memcpy(data.ptr, other.data.ptr, data.size + 1);
+ }
+}
+
+String::String() {
+ buf[0] = '\0';
+ setLast();
+}
+
+String::~String() {
+ if(!isOnStack())
+ delete[] data.ptr;
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+}
+
+String::String(const char* in)
+ : String(in, strlen(in)) {}
+
+String::String(const char* in, unsigned in_size) {
+ using namespace std;
+ if(in_size <= last) {
+ memcpy(buf, in, in_size);
+ buf[in_size] = '\0';
+ setLast(last - in_size);
+ } else {
+ setOnHeap();
+ data.size = in_size;
+ data.capacity = data.size + 1;
+ data.ptr = new char[data.capacity];
+ memcpy(data.ptr, in, in_size);
+ data.ptr[in_size] = '\0';
+ }
+}
+
+String::String(const String& other) { copy(other); }
+
+String& String::operator=(const String& other) {
+ if(this != &other) {
+ if(!isOnStack())
+ delete[] data.ptr;
+
+ copy(other);
+ }
+
+ return *this;
+}
+
+String& String::operator+=(const String& other) {
+ const unsigned my_old_size = size();
+ const unsigned other_size = other.size();
+ const unsigned total_size = my_old_size + other_size;
+ using namespace std;
+ if(isOnStack()) {
+ if(total_size < len) {
+ // append to the current stack space
+ memcpy(buf + my_old_size, other.c_str(), other_size + 1);
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ setLast(last - total_size);
+ } else {
+ // alloc new chunk
+ char* temp = new char[total_size + 1];
+ // copy current data to new location before writing in the union
+ memcpy(temp, buf, my_old_size); // skip the +1 ('\0') for speed
+ // update data in union
+ setOnHeap();
+ data.size = total_size;
+ data.capacity = data.size + 1;
+ data.ptr = temp;
+ // transfer the rest of the data
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ }
+ } else {
+ if(data.capacity > total_size) {
+ // append to the current heap block
+ data.size = total_size;
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ } else {
+ // resize
+ data.capacity *= 2;
+ if(data.capacity <= total_size)
+ data.capacity = total_size + 1;
+ // alloc new chunk
+ char* temp = new char[data.capacity];
+ // copy current data to new location before releasing it
+ memcpy(temp, data.ptr, my_old_size); // skip the +1 ('\0') for speed
+ // release old chunk
+ delete[] data.ptr;
+ // update the rest of the union members
+ data.size = total_size;
+ data.ptr = temp;
+ // transfer the rest of the data
+ memcpy(data.ptr + my_old_size, other.c_str(), other_size + 1);
+ }
+ }
+
+ return *this;
+}
+
+// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+String String::operator+(const String& other) const { return String(*this) += other; }
+
+String::String(String&& other) {
+ using namespace std;
+ memcpy(buf, other.buf, len);
+ other.buf[0] = '\0';
+ other.setLast();
+}
+
+String& String::operator=(String&& other) {
+ using namespace std;
+ if(this != &other) {
+ if(!isOnStack())
+ delete[] data.ptr;
+ memcpy(buf, other.buf, len);
+ other.buf[0] = '\0';
+ other.setLast();
+ }
+ return *this;
+}
+
+char String::operator[](unsigned i) const {
+ return const_cast<String*>(this)->operator[](i); // NOLINT
+}
+
+char& String::operator[](unsigned i) {
+ if(isOnStack())
+ return reinterpret_cast<char*>(buf)[i];
+ return data.ptr[i];
+}
+
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wmaybe-uninitialized")
+unsigned String::size() const {
+ if(isOnStack())
+ return last - (unsigned(buf[last]) & 31); // using "last" would work only if "len" is 32
+ return data.size;
+}
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+unsigned String::capacity() const {
+ if(isOnStack())
+ return len;
+ return data.capacity;
+}
+
+int String::compare(const char* other, bool no_case) const {
+ if(no_case)
+ return doctest::stricmp(c_str(), other);
+ return std::strcmp(c_str(), other);
+}
+
+int String::compare(const String& other, bool no_case) const {
+ return compare(other.c_str(), no_case);
+}
+
+// clang-format off
+bool operator==(const String& lhs, const String& rhs) { return lhs.compare(rhs) == 0; }
+bool operator!=(const String& lhs, const String& rhs) { return lhs.compare(rhs) != 0; }
+bool operator< (const String& lhs, const String& rhs) { return lhs.compare(rhs) < 0; }
+bool operator> (const String& lhs, const String& rhs) { return lhs.compare(rhs) > 0; }
+bool operator<=(const String& lhs, const String& rhs) { return (lhs != rhs) ? lhs.compare(rhs) < 0 : true; }
+bool operator>=(const String& lhs, const String& rhs) { return (lhs != rhs) ? lhs.compare(rhs) > 0 : true; }
+// clang-format on
+
+std::ostream& operator<<(std::ostream& s, const String& in) { return s << in.c_str(); }
+
+namespace {
+ void color_to_stream(std::ostream&, Color::Enum) DOCTEST_BRANCH_ON_DISABLED({}, ;)
+} // namespace
+
+namespace Color {
+ std::ostream& operator<<(std::ostream& s, Color::Enum code) {
+ color_to_stream(s, code);
+ return s;
+ }
+} // namespace Color
+
+// clang-format off
+const char* assertString(assertType::Enum at) {
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4062) // enum 'x' in switch of enum 'y' is not handled
+ switch(at) { //!OCLINT missing default in switch statements
+ case assertType::DT_WARN : return "WARN";
+ case assertType::DT_CHECK : return "CHECK";
+ case assertType::DT_REQUIRE : return "REQUIRE";
+
+ case assertType::DT_WARN_FALSE : return "WARN_FALSE";
+ case assertType::DT_CHECK_FALSE : return "CHECK_FALSE";
+ case assertType::DT_REQUIRE_FALSE : return "REQUIRE_FALSE";
+
+ case assertType::DT_WARN_THROWS : return "WARN_THROWS";
+ case assertType::DT_CHECK_THROWS : return "CHECK_THROWS";
+ case assertType::DT_REQUIRE_THROWS : return "REQUIRE_THROWS";
+
+ case assertType::DT_WARN_THROWS_AS : return "WARN_THROWS_AS";
+ case assertType::DT_CHECK_THROWS_AS : return "CHECK_THROWS_AS";
+ case assertType::DT_REQUIRE_THROWS_AS : return "REQUIRE_THROWS_AS";
+
+ case assertType::DT_WARN_THROWS_WITH : return "WARN_THROWS_WITH";
+ case assertType::DT_CHECK_THROWS_WITH : return "CHECK_THROWS_WITH";
+ case assertType::DT_REQUIRE_THROWS_WITH : return "REQUIRE_THROWS_WITH";
+
+ case assertType::DT_WARN_THROWS_WITH_AS : return "WARN_THROWS_WITH_AS";
+ case assertType::DT_CHECK_THROWS_WITH_AS : return "CHECK_THROWS_WITH_AS";
+ case assertType::DT_REQUIRE_THROWS_WITH_AS : return "REQUIRE_THROWS_WITH_AS";
+
+ case assertType::DT_WARN_NOTHROW : return "WARN_NOTHROW";
+ case assertType::DT_CHECK_NOTHROW : return "CHECK_NOTHROW";
+ case assertType::DT_REQUIRE_NOTHROW : return "REQUIRE_NOTHROW";
+
+ case assertType::DT_WARN_EQ : return "WARN_EQ";
+ case assertType::DT_CHECK_EQ : return "CHECK_EQ";
+ case assertType::DT_REQUIRE_EQ : return "REQUIRE_EQ";
+ case assertType::DT_WARN_NE : return "WARN_NE";
+ case assertType::DT_CHECK_NE : return "CHECK_NE";
+ case assertType::DT_REQUIRE_NE : return "REQUIRE_NE";
+ case assertType::DT_WARN_GT : return "WARN_GT";
+ case assertType::DT_CHECK_GT : return "CHECK_GT";
+ case assertType::DT_REQUIRE_GT : return "REQUIRE_GT";
+ case assertType::DT_WARN_LT : return "WARN_LT";
+ case assertType::DT_CHECK_LT : return "CHECK_LT";
+ case assertType::DT_REQUIRE_LT : return "REQUIRE_LT";
+ case assertType::DT_WARN_GE : return "WARN_GE";
+ case assertType::DT_CHECK_GE : return "CHECK_GE";
+ case assertType::DT_REQUIRE_GE : return "REQUIRE_GE";
+ case assertType::DT_WARN_LE : return "WARN_LE";
+ case assertType::DT_CHECK_LE : return "CHECK_LE";
+ case assertType::DT_REQUIRE_LE : return "REQUIRE_LE";
+
+ case assertType::DT_WARN_UNARY : return "WARN_UNARY";
+ case assertType::DT_CHECK_UNARY : return "CHECK_UNARY";
+ case assertType::DT_REQUIRE_UNARY : return "REQUIRE_UNARY";
+ case assertType::DT_WARN_UNARY_FALSE : return "WARN_UNARY_FALSE";
+ case assertType::DT_CHECK_UNARY_FALSE : return "CHECK_UNARY_FALSE";
+ case assertType::DT_REQUIRE_UNARY_FALSE : return "REQUIRE_UNARY_FALSE";
+ }
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ return "";
+}
+// clang-format on
+
+const char* failureString(assertType::Enum at) {
+ if(at & assertType::is_warn) //!OCLINT bitwise operator in conditional
+ return "WARNING";
+ if(at & assertType::is_check) //!OCLINT bitwise operator in conditional
+ return "ERROR";
+ if(at & assertType::is_require) //!OCLINT bitwise operator in conditional
+ return "FATAL ERROR";
+ return "";
+}
+
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wnull-dereference")
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wnull-dereference")
+// depending on the current options this will remove the path of filenames
+const char* skipPathFromFilename(const char* file) {
+#ifndef DOCTEST_CONFIG_DISABLE
+ if(getContextOptions()->no_path_in_filenames) {
+ auto back = std::strrchr(file, '\\');
+ auto forward = std::strrchr(file, '/');
+ if(back || forward) {
+ if(back > forward)
+ forward = back;
+ return forward + 1;
+ }
+ }
+#endif // DOCTEST_CONFIG_DISABLE
+ return file;
+}
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+bool SubcaseSignature::operator<(const SubcaseSignature& other) const {
+ if(m_line != other.m_line)
+ return m_line < other.m_line;
+ if(std::strcmp(m_file, other.m_file) != 0)
+ return std::strcmp(m_file, other.m_file) < 0;
+ return m_name.compare(other.m_name) < 0;
+}
+
+IContextScope::IContextScope() = default;
+IContextScope::~IContextScope() = default;
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+String toString(char* in) { return toString(static_cast<const char*>(in)); }
+// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+String toString(const char* in) { return String("\"") + (in ? in : "{null string}") + "\""; }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+String toString(bool in) { return in ? "true" : "false"; }
+String toString(float in) { return fpToString(in, 5) + "f"; }
+String toString(double in) { return fpToString(in, 10); }
+String toString(double long in) { return fpToString(in, 15); }
+
+#define DOCTEST_TO_STRING_OVERLOAD(type, fmt) \
+ String toString(type in) { \
+ char buf[64]; \
+ std::sprintf(buf, fmt, in); \
+ return buf; \
+ }
+
+DOCTEST_TO_STRING_OVERLOAD(char, "%d")
+DOCTEST_TO_STRING_OVERLOAD(char signed, "%d")
+DOCTEST_TO_STRING_OVERLOAD(char unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int short, "%d")
+DOCTEST_TO_STRING_OVERLOAD(int short unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int, "%d")
+DOCTEST_TO_STRING_OVERLOAD(unsigned, "%u")
+DOCTEST_TO_STRING_OVERLOAD(int long, "%ld")
+DOCTEST_TO_STRING_OVERLOAD(int long unsigned, "%lu")
+DOCTEST_TO_STRING_OVERLOAD(int long long, "%lld")
+DOCTEST_TO_STRING_OVERLOAD(int long long unsigned, "%llu")
+
+String toString(std::nullptr_t) { return "NULL"; }
+
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+String toString(const std::string& in) { return in.c_str(); }
+#endif // VS 2019
+
+Approx::Approx(double value)
+ : m_epsilon(static_cast<double>(std::numeric_limits<float>::epsilon()) * 100)
+ , m_scale(1.0)
+ , m_value(value) {}
+
+Approx Approx::operator()(double value) const {
+ Approx approx(value);
+ approx.epsilon(m_epsilon);
+ approx.scale(m_scale);
+ return approx;
+}
+
+Approx& Approx::epsilon(double newEpsilon) {
+ m_epsilon = newEpsilon;
+ return *this;
+}
+Approx& Approx::scale(double newScale) {
+ m_scale = newScale;
+ return *this;
+}
+
+bool operator==(double lhs, const Approx& rhs) {
+ // Thanks to Richard Harris for his help refining this formula
+ return std::fabs(lhs - rhs.m_value) <
+ rhs.m_epsilon * (rhs.m_scale + std::max<double>(std::fabs(lhs), std::fabs(rhs.m_value)));
+}
+bool operator==(const Approx& lhs, double rhs) { return operator==(rhs, lhs); }
+bool operator!=(double lhs, const Approx& rhs) { return !operator==(lhs, rhs); }
+bool operator!=(const Approx& lhs, double rhs) { return !operator==(rhs, lhs); }
+bool operator<=(double lhs, const Approx& rhs) { return lhs < rhs.m_value || lhs == rhs; }
+bool operator<=(const Approx& lhs, double rhs) { return lhs.m_value < rhs || lhs == rhs; }
+bool operator>=(double lhs, const Approx& rhs) { return lhs > rhs.m_value || lhs == rhs; }
+bool operator>=(const Approx& lhs, double rhs) { return lhs.m_value > rhs || lhs == rhs; }
+bool operator<(double lhs, const Approx& rhs) { return lhs < rhs.m_value && lhs != rhs; }
+bool operator<(const Approx& lhs, double rhs) { return lhs.m_value < rhs && lhs != rhs; }
+bool operator>(double lhs, const Approx& rhs) { return lhs > rhs.m_value && lhs != rhs; }
+bool operator>(const Approx& lhs, double rhs) { return lhs.m_value > rhs && lhs != rhs; }
+
+String toString(const Approx& in) {
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ return String("Approx( ") + doctest::toString(in.m_value) + " )";
+}
+const ContextOptions* getContextOptions() { return DOCTEST_BRANCH_ON_DISABLED(nullptr, g_cs); }
+
+} // namespace doctest
+
+#ifdef DOCTEST_CONFIG_DISABLE
+namespace doctest {
+Context::Context(int, const char* const*) {}
+Context::~Context() = default;
+void Context::applyCommandLine(int, const char* const*) {}
+void Context::addFilter(const char*, const char*) {}
+void Context::clearFilters() {}
+void Context::setOption(const char*, int) {}
+void Context::setOption(const char*, const char*) {}
+bool Context::shouldExit() { return false; }
+void Context::setAsDefaultForAssertsOutOfTestCases() {}
+void Context::setAssertHandler(detail::assert_handler) {}
+int Context::run() { return 0; }
+
+IReporter::~IReporter() = default;
+
+int IReporter::get_num_active_contexts() { return 0; }
+const IContextScope* const* IReporter::get_active_contexts() { return nullptr; }
+int IReporter::get_num_stringified_contexts() { return 0; }
+const String* IReporter::get_stringified_contexts() { return nullptr; }
+
+int registerReporter(const char*, int, IReporter*) { return 0; }
+
+} // namespace doctest
+#else // DOCTEST_CONFIG_DISABLE
+
+#if !defined(DOCTEST_CONFIG_COLORS_NONE)
+#if !defined(DOCTEST_CONFIG_COLORS_WINDOWS) && !defined(DOCTEST_CONFIG_COLORS_ANSI)
+#ifdef DOCTEST_PLATFORM_WINDOWS
+#define DOCTEST_CONFIG_COLORS_WINDOWS
+#else // linux
+#define DOCTEST_CONFIG_COLORS_ANSI
+#endif // platform
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS && DOCTEST_CONFIG_COLORS_ANSI
+#endif // DOCTEST_CONFIG_COLORS_NONE
+
+namespace doctest_detail_test_suite_ns {
+// holds the current test suite
+doctest::detail::TestSuite& getCurrentTestSuite() {
+ static doctest::detail::TestSuite data{};
+ return data;
+}
+} // namespace doctest_detail_test_suite_ns
+
+namespace doctest {
+namespace {
+ // the int (priority) is part of the key for automatic sorting - sadly one can register a
+ // reporter with a duplicate name and a different priority but hopefully that won't happen often :|
+ typedef std::map<std::pair<int, String>, reporterCreatorFunc> reporterMap;
+
+ reporterMap& getReporters() {
+ static reporterMap data;
+ return data;
+ }
+ reporterMap& getListeners() {
+ static reporterMap data;
+ return data;
+ }
+} // namespace
+namespace detail {
+#define DOCTEST_ITERATE_THROUGH_REPORTERS(function, ...) \
+ for(auto& curr_rep : g_cs->reporters_currently_used) \
+ curr_rep->function(__VA_ARGS__)
+
+ bool checkIfShouldThrow(assertType::Enum at) {
+ if(at & assertType::is_require) //!OCLINT bitwise operator in conditional
+ return true;
+
+ if((at & assertType::is_check) //!OCLINT bitwise operator in conditional
+ && getContextOptions()->abort_after > 0 &&
+ (g_cs->numAssertsFailed + g_cs->numAssertsFailedCurrentTest_atomic) >=
+ getContextOptions()->abort_after)
+ return true;
+
+ return false;
+ }
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_NORETURN void throwException() {
+ g_cs->shouldLogCurrentException = false;
+ throw TestFailureException();
+ } // NOLINT(cert-err60-cpp)
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ void throwException() {}
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+} // namespace detail
+
+namespace {
+ using namespace detail;
+ // matching of a string against a wildcard mask (case sensitivity configurable) taken from
+ // https://www.codeproject.com/Articles/1088/Wildcard-string-compare-globbing
+ int wildcmp(const char* str, const char* wild, bool caseSensitive) {
+ const char* cp = str;
+ const char* mp = wild;
+
+ while((*str) && (*wild != '*')) {
+ if((caseSensitive ? (*wild != *str) : (tolower(*wild) != tolower(*str))) &&
+ (*wild != '?')) {
+ return 0;
+ }
+ wild++;
+ str++;
+ }
+
+ while(*str) {
+ if(*wild == '*') {
+ if(!*++wild) {
+ return 1;
+ }
+ mp = wild;
+ cp = str + 1;
+ } else if((caseSensitive ? (*wild == *str) : (tolower(*wild) == tolower(*str))) ||
+ (*wild == '?')) {
+ wild++;
+ str++;
+ } else {
+ wild = mp; //!OCLINT parameter reassignment
+ str = cp++; //!OCLINT parameter reassignment
+ }
+ }
+
+ while(*wild == '*') {
+ wild++;
+ }
+ return !*wild;
+ }
+
+ //// C string hash function (djb2) - taken from http://www.cse.yorku.ca/~oz/hash.html
+ //unsigned hashStr(unsigned const char* str) {
+ // unsigned long hash = 5381;
+ // char c;
+ // while((c = *str++))
+ // hash = ((hash << 5) + hash) + c; // hash * 33 + c
+ // return hash;
+ //}
+
+ // checks if the name matches any of the filters (and can be configured what to do when empty)
+ bool matchesAny(const char* name, const std::vector<String>& filters, bool matchEmpty,
+ bool caseSensitive) {
+ if(filters.empty() && matchEmpty)
+ return true;
+ for(auto& curr : filters)
+ if(wildcmp(name, curr.c_str(), caseSensitive))
+ return true;
+ return false;
+ }
+} // namespace
+namespace detail {
+
+ Subcase::Subcase(const String& name, const char* file, int line)
+ : m_signature({name, file, line}) {
+ auto* s = g_cs;
+
+ // check subcase filters
+ if(s->subcasesStack.size() < size_t(s->subcase_filter_levels)) {
+ if(!matchesAny(m_signature.m_name.c_str(), s->filters[6], true, s->case_sensitive))
+ return;
+ if(matchesAny(m_signature.m_name.c_str(), s->filters[7], false, s->case_sensitive))
+ return;
+ }
+
+ // if a Subcase on the same level has already been entered
+ if(s->subcasesStack.size() < size_t(s->subcasesCurrentMaxLevel)) {
+ s->should_reenter = true;
+ return;
+ }
+
+ // push the current signature to the stack so we can check if the
+ // current stack + the current new subcase have been traversed
+ s->subcasesStack.push_back(m_signature);
+ if(s->subcasesPassed.count(s->subcasesStack) != 0) {
+ // pop - revert to previous stack since we've already passed this
+ s->subcasesStack.pop_back();
+ return;
+ }
+
+ s->subcasesCurrentMaxLevel = s->subcasesStack.size();
+ m_entered = true;
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_start, m_signature);
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+
+ Subcase::~Subcase() {
+ if(m_entered) {
+ // only mark the subcase stack as passed if no subcases have been skipped
+ if(g_cs->should_reenter == false)
+ g_cs->subcasesPassed.insert(g_cs->subcasesStack);
+ g_cs->subcasesStack.pop_back();
+
+#if defined(__cpp_lib_uncaught_exceptions) && __cpp_lib_uncaught_exceptions >= 201411L && (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200)
+ if(std::uncaught_exceptions() > 0
+#else
+ if(std::uncaught_exception()
+#endif
+ && g_cs->shouldLogCurrentException) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(
+ test_case_exception, {"exception thrown in subcase - will translate later "
+ "when the whole test case has been exited (cannot "
+ "translate while there is an active exception)",
+ false});
+ g_cs->shouldLogCurrentException = false;
+ }
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_end, DOCTEST_EMPTY);
+ }
+ }
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ Subcase::operator bool() const { return m_entered; }
+
+ Result::Result(bool passed, const String& decomposition)
+ : m_passed(passed)
+ , m_decomp(decomposition) {}
+
+ ExpressionDecomposer::ExpressionDecomposer(assertType::Enum at)
+ : m_at(at) {}
+
+ TestSuite& TestSuite::operator*(const char* in) {
+ m_test_suite = in;
+ // clear state
+ m_description = nullptr;
+ m_skip = false;
+ m_no_breaks = false;
+ m_no_output = false;
+ m_may_fail = false;
+ m_should_fail = false;
+ m_expected_failures = 0;
+ m_timeout = 0;
+ return *this;
+ }
+
+ TestCase::TestCase(funcType test, const char* file, unsigned line, const TestSuite& test_suite,
+ const char* type, int template_id) {
+ m_file = file;
+ m_line = line;
+ m_name = nullptr; // will be later overridden in operator*
+ m_test_suite = test_suite.m_test_suite;
+ m_description = test_suite.m_description;
+ m_skip = test_suite.m_skip;
+ m_no_breaks = test_suite.m_no_breaks;
+ m_no_output = test_suite.m_no_output;
+ m_may_fail = test_suite.m_may_fail;
+ m_should_fail = test_suite.m_should_fail;
+ m_expected_failures = test_suite.m_expected_failures;
+ m_timeout = test_suite.m_timeout;
+
+ m_test = test;
+ m_type = type;
+ m_template_id = template_id;
+ }
+
+ TestCase::TestCase(const TestCase& other)
+ : TestCaseData() {
+ *this = other;
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(26434) // hides a non-virtual function
+ DOCTEST_MSVC_SUPPRESS_WARNING(26437) // Do not slice
+ TestCase& TestCase::operator=(const TestCase& other) {
+ static_cast<TestCaseData&>(*this) = static_cast<const TestCaseData&>(other);
+
+ m_test = other.m_test;
+ m_type = other.m_type;
+ m_template_id = other.m_template_id;
+ m_full_name = other.m_full_name;
+
+ if(m_template_id != -1)
+ m_name = m_full_name.c_str();
+ return *this;
+ }
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ TestCase& TestCase::operator*(const char* in) {
+ m_name = in;
+ // make a new name with an appended type for templated test case
+ if(m_template_id != -1) {
+ m_full_name = String(m_name) + m_type;
+ // redirect the name to point to the newly constructed full name
+ m_name = m_full_name.c_str();
+ }
+ return *this;
+ }
+
+ bool TestCase::operator<(const TestCase& other) const {
+ // this will be used only to differentiate between test cases - not relevant for sorting
+ if(m_line != other.m_line)
+ return m_line < other.m_line;
+ const int name_cmp = strcmp(m_name, other.m_name);
+ if(name_cmp != 0)
+ return name_cmp < 0;
+ const int file_cmp = m_file.compare(other.m_file);
+ if(file_cmp != 0)
+ return file_cmp < 0;
+ return m_template_id < other.m_template_id;
+ }
+
+ // all the registered tests
+ std::set<TestCase>& getRegisteredTests() {
+ static std::set<TestCase> data;
+ return data;
+ }
+} // namespace detail
+namespace {
+ using namespace detail;
+ // for sorting tests by file/line
+ bool fileOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ // this is needed because MSVC gives different case for drive letters
+ // for __FILE__ when evaluated in a header and a source file
+ const int res = lhs->m_file.compare(rhs->m_file, bool(DOCTEST_MSVC));
+ if(res != 0)
+ return res < 0;
+ if(lhs->m_line != rhs->m_line)
+ return lhs->m_line < rhs->m_line;
+ return lhs->m_template_id < rhs->m_template_id;
+ }
+
+ // for sorting tests by suite/file/line
+ bool suiteOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ const int res = std::strcmp(lhs->m_test_suite, rhs->m_test_suite);
+ if(res != 0)
+ return res < 0;
+ return fileOrderComparator(lhs, rhs);
+ }
+
+ // for sorting tests by name/suite/file/line
+ bool nameOrderComparator(const TestCase* lhs, const TestCase* rhs) {
+ const int res = std::strcmp(lhs->m_name, rhs->m_name);
+ if(res != 0)
+ return res < 0;
+ return suiteOrderComparator(lhs, rhs);
+ }
+
+#ifdef DOCTEST_CONFIG_COLORS_WINDOWS
+ HANDLE g_stdoutHandle;
+ WORD g_origFgAttrs;
+ WORD g_origBgAttrs;
+ bool g_attrsInitted = false;
+
+ int colors_init() {
+ if(!g_attrsInitted) {
+ g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);
+ g_attrsInitted = true;
+ CONSOLE_SCREEN_BUFFER_INFO csbiInfo;
+ GetConsoleScreenBufferInfo(g_stdoutHandle, &csbiInfo);
+ g_origFgAttrs = csbiInfo.wAttributes & ~(BACKGROUND_GREEN | BACKGROUND_RED |
+ BACKGROUND_BLUE | BACKGROUND_INTENSITY);
+ g_origBgAttrs = csbiInfo.wAttributes & ~(FOREGROUND_GREEN | FOREGROUND_RED |
+ FOREGROUND_BLUE | FOREGROUND_INTENSITY);
+ }
+ return 0;
+ }
+
+ int dumy_init_console_colors = colors_init();
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ void color_to_stream(std::ostream& s, Color::Enum code) {
+ static_cast<void>(s); // for DOCTEST_CONFIG_COLORS_NONE or DOCTEST_CONFIG_COLORS_WINDOWS
+ static_cast<void>(code); // for DOCTEST_CONFIG_COLORS_NONE
+#ifdef DOCTEST_CONFIG_COLORS_ANSI
+ if(g_no_colors ||
+ (isatty(STDOUT_FILENO) == false && getContextOptions()->force_colors == false))
+ return;
+
+ auto col = "";
+ // clang-format off
+ switch(code) { //!OCLINT missing break in switch statement / unnecessary default statement in covered switch statement
+ case Color::Red: col = "[0;31m"; break;
+ case Color::Green: col = "[0;32m"; break;
+ case Color::Blue: col = "[0;34m"; break;
+ case Color::Cyan: col = "[0;36m"; break;
+ case Color::Yellow: col = "[0;33m"; break;
+ case Color::Grey: col = "[1;30m"; break;
+ case Color::LightGrey: col = "[0;37m"; break;
+ case Color::BrightRed: col = "[1;31m"; break;
+ case Color::BrightGreen: col = "[1;32m"; break;
+ case Color::BrightWhite: col = "[1;37m"; break;
+ case Color::Bright: // invalid
+ case Color::None:
+ case Color::White:
+ default: col = "[0m";
+ }
+ // clang-format on
+ s << "\033" << col;
+#endif // DOCTEST_CONFIG_COLORS_ANSI
+
+#ifdef DOCTEST_CONFIG_COLORS_WINDOWS
+ if(g_no_colors ||
+ (isatty(fileno(stdout)) == false && getContextOptions()->force_colors == false))
+ return;
+
+#define DOCTEST_SET_ATTR(x) SetConsoleTextAttribute(g_stdoutHandle, x | g_origBgAttrs)
+
+ // clang-format off
+ switch (code) {
+ case Color::White: DOCTEST_SET_ATTR(FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE); break;
+ case Color::Red: DOCTEST_SET_ATTR(FOREGROUND_RED); break;
+ case Color::Green: DOCTEST_SET_ATTR(FOREGROUND_GREEN); break;
+ case Color::Blue: DOCTEST_SET_ATTR(FOREGROUND_BLUE); break;
+ case Color::Cyan: DOCTEST_SET_ATTR(FOREGROUND_BLUE | FOREGROUND_GREEN); break;
+ case Color::Yellow: DOCTEST_SET_ATTR(FOREGROUND_RED | FOREGROUND_GREEN); break;
+ case Color::Grey: DOCTEST_SET_ATTR(0); break;
+ case Color::LightGrey: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY); break;
+ case Color::BrightRed: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_RED); break;
+ case Color::BrightGreen: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_GREEN); break;
+ case Color::BrightWhite: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE); break;
+ case Color::None:
+ case Color::Bright: // invalid
+ default: DOCTEST_SET_ATTR(g_origFgAttrs);
+ }
+ // clang-format on
+#endif // DOCTEST_CONFIG_COLORS_WINDOWS
+ }
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+ std::vector<const IExceptionTranslator*>& getExceptionTranslators() {
+ static std::vector<const IExceptionTranslator*> data;
+ return data;
+ }
+
+ String translateActiveException() {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ String res;
+ auto& translators = getExceptionTranslators();
+ for(auto& curr : translators)
+ if(curr->translate(res))
+ return res;
+ // clang-format off
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wcatch-value")
+ try {
+ throw;
+ } catch(std::exception& ex) {
+ return ex.what();
+ } catch(std::string& msg) {
+ return msg.c_str();
+ } catch(const char* msg) {
+ return msg;
+ } catch(...) {
+ return "unknown exception";
+ }
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+// clang-format on
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ return "";
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ }
+} // namespace
+
+namespace detail {
+ // used by the macros for registering tests
+ int regTest(const TestCase& tc) {
+ getRegisteredTests().insert(tc);
+ return 0;
+ }
+
+ // sets the current test suite
+ int setTestSuite(const TestSuite& ts) {
+ doctest_detail_test_suite_ns::getCurrentTestSuite() = ts;
+ return 0;
+ }
+
+#ifdef DOCTEST_IS_DEBUGGER_ACTIVE
+ bool isDebuggerActive() { return DOCTEST_IS_DEBUGGER_ACTIVE(); }
+#else // DOCTEST_IS_DEBUGGER_ACTIVE
+#ifdef DOCTEST_PLATFORM_LINUX
+ class ErrnoGuard {
+ public:
+ ErrnoGuard() : m_oldErrno(errno) {}
+ ~ErrnoGuard() { errno = m_oldErrno; }
+ private:
+ int m_oldErrno;
+ };
+ // See the comments in Catch2 for the reasoning behind this implementation:
+ // https://github.com/catchorg/Catch2/blob/v2.13.1/include/internal/catch_debugger.cpp#L79-L102
+ bool isDebuggerActive() {
+ ErrnoGuard guard;
+ std::ifstream in("/proc/self/status");
+ for(std::string line; std::getline(in, line);) {
+ static const int PREFIX_LEN = 11;
+ if(line.compare(0, PREFIX_LEN, "TracerPid:\t") == 0) {
+ return line.length() > PREFIX_LEN && line[PREFIX_LEN] != '0';
+ }
+ }
+ return false;
+ }
+#elif defined(DOCTEST_PLATFORM_MAC)
+ // The following function is taken directly from the following technical note:
+ // https://developer.apple.com/library/archive/qa/qa1361/_index.html
+ // Returns true if the current process is being debugged (either
+ // running under the debugger or has a debugger attached post facto).
+ bool isDebuggerActive() {
+ int mib[4];
+ kinfo_proc info;
+ size_t size;
+ // Initialize the flags so that, if sysctl fails for some bizarre
+ // reason, we get a predictable result.
+ info.kp_proc.p_flag = 0;
+ // Initialize mib, which tells sysctl the info we want, in this case
+ // we're looking for information about a specific process ID.
+ mib[0] = CTL_KERN;
+ mib[1] = KERN_PROC;
+ mib[2] = KERN_PROC_PID;
+ mib[3] = getpid();
+ // Call sysctl.
+ size = sizeof(info);
+ if(sysctl(mib, DOCTEST_COUNTOF(mib), &info, &size, 0, 0) != 0) {
+ std::cerr << "\nCall to sysctl failed - unable to determine if debugger is active **\n";
+ return false;
+ }
+ // We're being debugged if the P_TRACED flag is set.
+ return ((info.kp_proc.p_flag & P_TRACED) != 0);
+ }
+#elif DOCTEST_MSVC || defined(__MINGW32__) || defined(__MINGW64__)
+ bool isDebuggerActive() { return ::IsDebuggerPresent() != 0; }
+#else
+ bool isDebuggerActive() { return false; }
+#endif // Platform
+#endif // DOCTEST_IS_DEBUGGER_ACTIVE
+
+ void registerExceptionTranslatorImpl(const IExceptionTranslator* et) {
+ if(std::find(getExceptionTranslators().begin(), getExceptionTranslators().end(), et) ==
+ getExceptionTranslators().end())
+ getExceptionTranslators().push_back(et);
+ }
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ void toStream(std::ostream* s, char* in) { *s << in; }
+ void toStream(std::ostream* s, const char* in) { *s << in; }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ void toStream(std::ostream* s, bool in) { *s << std::boolalpha << in << std::noboolalpha; }
+ void toStream(std::ostream* s, float in) { *s << in; }
+ void toStream(std::ostream* s, double in) { *s << in; }
+ void toStream(std::ostream* s, double long in) { *s << in; }
+
+ void toStream(std::ostream* s, char in) { *s << in; }
+ void toStream(std::ostream* s, char signed in) { *s << in; }
+ void toStream(std::ostream* s, char unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int short in) { *s << in; }
+ void toStream(std::ostream* s, int short unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int in) { *s << in; }
+ void toStream(std::ostream* s, int unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int long in) { *s << in; }
+ void toStream(std::ostream* s, int long unsigned in) { *s << in; }
+ void toStream(std::ostream* s, int long long in) { *s << in; }
+ void toStream(std::ostream* s, int long long unsigned in) { *s << in; }
+
+ DOCTEST_THREAD_LOCAL std::vector<IContextScope*> g_infoContexts; // for logging with INFO()
+
+ ContextScopeBase::ContextScopeBase() {
+ g_infoContexts.push_back(this);
+ }
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+
+ // destroy cannot be inlined into the destructor because that would mean calling stringify after
+ // ContextScope has been destroyed (base class destructors run after derived class destructors).
+ // Instead, ContextScope calls this method directly from its destructor.
+ void ContextScopeBase::destroy() {
+#if defined(__cpp_lib_uncaught_exceptions) && __cpp_lib_uncaught_exceptions >= 201411L && (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200)
+ if(std::uncaught_exceptions() > 0) {
+#else
+ if(std::uncaught_exception()) {
+#endif
+ std::ostringstream s;
+ this->stringify(&s);
+ g_cs->stringifiedContexts.push_back(s.str().c_str());
+ }
+ g_infoContexts.pop_back();
+ }
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+} // namespace detail
+namespace {
+ using namespace detail;
+
+#if !defined(DOCTEST_CONFIG_POSIX_SIGNALS) && !defined(DOCTEST_CONFIG_WINDOWS_SEH)
+ struct FatalConditionHandler
+ {
+ static void reset() {}
+ static void allocateAltStackMem() {}
+ static void freeAltStackMem() {}
+ };
+#else // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+
+ void reportFatal(const std::string&);
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+
+ struct SignalDefs
+ {
+ DWORD id;
+ const char* name;
+ };
+ // There is no 1-1 mapping between signals and windows exceptions.
+ // Windows can easily distinguish between SO and SigSegV,
+ // but SigInt, SigTerm, etc are handled differently.
+ SignalDefs signalDefs[] = {
+ {static_cast<DWORD>(EXCEPTION_ILLEGAL_INSTRUCTION),
+ "SIGILL - Illegal instruction signal"},
+ {static_cast<DWORD>(EXCEPTION_STACK_OVERFLOW), "SIGSEGV - Stack overflow"},
+ {static_cast<DWORD>(EXCEPTION_ACCESS_VIOLATION),
+ "SIGSEGV - Segmentation violation signal"},
+ {static_cast<DWORD>(EXCEPTION_INT_DIVIDE_BY_ZERO), "Divide by zero error"},
+ };
+
+ struct FatalConditionHandler
+ {
+ static LONG CALLBACK handleException(PEXCEPTION_POINTERS ExceptionInfo) {
+ // Multiple threads may enter this filter/handler at once. We want the error message to be printed on the
+ // console just once no matter how many threads have crashed.
+ static std::mutex mutex;
+ static bool execute = true;
+ {
+ std::lock_guard<std::mutex> lock(mutex);
+ if(execute) {
+ bool reported = false;
+ for(size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ if(ExceptionInfo->ExceptionRecord->ExceptionCode == signalDefs[i].id) {
+ reportFatal(signalDefs[i].name);
+ reported = true;
+ break;
+ }
+ }
+ if(reported == false)
+ reportFatal("Unhandled SEH exception caught");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ }
+ execute = false;
+ }
+ std::exit(EXIT_FAILURE);
+ }
+
+ static void allocateAltStackMem() {}
+ static void freeAltStackMem() {}
+
+ FatalConditionHandler() {
+ isSet = true;
+ // 32k seems enough for doctest to handle stack overflow,
+ // but the value was found experimentally, so there is no strong guarantee
+ guaranteeSize = 32 * 1024;
+ // Register an unhandled exception filter
+ previousTop = SetUnhandledExceptionFilter(handleException);
+ // Pass in guarantee size to be filled
+ SetThreadStackGuarantee(&guaranteeSize);
+
+ // On Windows uncaught exceptions from another thread, exceptions from
+ // destructors, or calls to std::terminate are not a SEH exception
+
+ // The terminal handler gets called when:
+ // - std::terminate is called FROM THE TEST RUNNER THREAD
+ // - an exception is thrown from a destructor FROM THE TEST RUNNER THREAD
+ original_terminate_handler = std::get_terminate();
+ std::set_terminate([]() DOCTEST_NOEXCEPT {
+ reportFatal("Terminate handler called");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ std::exit(EXIT_FAILURE); // explicitly exit - otherwise the SIGABRT handler may be called as well
+ });
+
+ // SIGABRT is raised when:
+ // - std::terminate is called FROM A DIFFERENT THREAD
+ // - an exception is thrown from a destructor FROM A DIFFERENT THREAD
+ // - an uncaught exception is thrown FROM A DIFFERENT THREAD
+ prev_sigabrt_handler = std::signal(SIGABRT, [](int signal) DOCTEST_NOEXCEPT {
+ if(signal == SIGABRT) {
+ reportFatal("SIGABRT - Abort (abnormal termination) signal");
+ if(isDebuggerActive() && !g_cs->no_breaks)
+ DOCTEST_BREAK_INTO_DEBUGGER();
+ std::exit(EXIT_FAILURE);
+ }
+ });
+
+ // The following settings are taken from google test, and more
+ // specifically from UnitTest::Run() inside of gtest.cc
+
+ // the user does not want to see pop-up dialogs about crashes
+ prev_error_mode_1 = SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT |
+ SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX);
+ // This forces the abort message to go to stderr in all circumstances.
+ prev_error_mode_2 = _set_error_mode(_OUT_TO_STDERR);
+ // In the debug version, Visual Studio pops up a separate dialog
+ // offering a choice to debug the aborted program - we want to disable that.
+ prev_abort_behavior = _set_abort_behavior(0x0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+ // In debug mode, the Windows CRT can crash with an assertion over invalid
+ // input (e.g. passing an invalid file descriptor). The default handling
+ // for these assertions is to pop up a dialog and wait for user input.
+ // Instead ask the CRT to dump such assertions to stderr non-interactively.
+ prev_report_mode = _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG);
+ prev_report_file = _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR);
+ }
+
+ static void reset() {
+ if(isSet) {
+ // Unregister handler and restore the old guarantee
+ SetUnhandledExceptionFilter(previousTop);
+ SetThreadStackGuarantee(&guaranteeSize);
+ std::set_terminate(original_terminate_handler);
+ std::signal(SIGABRT, prev_sigabrt_handler);
+ SetErrorMode(prev_error_mode_1);
+ _set_error_mode(prev_error_mode_2);
+ _set_abort_behavior(prev_abort_behavior, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+ static_cast<void>(_CrtSetReportMode(_CRT_ASSERT, prev_report_mode));
+ static_cast<void>(_CrtSetReportFile(_CRT_ASSERT, prev_report_file));
+ isSet = false;
+ }
+ }
+
+ ~FatalConditionHandler() { reset(); }
+
+ private:
+ static UINT prev_error_mode_1;
+ static int prev_error_mode_2;
+ static unsigned int prev_abort_behavior;
+ static int prev_report_mode;
+ static _HFILE prev_report_file;
+ static void (*prev_sigabrt_handler)(int);
+ static std::terminate_handler original_terminate_handler;
+ static bool isSet;
+ static ULONG guaranteeSize;
+ static LPTOP_LEVEL_EXCEPTION_FILTER previousTop;
+ };
+
+ UINT FatalConditionHandler::prev_error_mode_1;
+ int FatalConditionHandler::prev_error_mode_2;
+ unsigned int FatalConditionHandler::prev_abort_behavior;
+ int FatalConditionHandler::prev_report_mode;
+ _HFILE FatalConditionHandler::prev_report_file;
+ void (*FatalConditionHandler::prev_sigabrt_handler)(int);
+ std::terminate_handler FatalConditionHandler::original_terminate_handler;
+ bool FatalConditionHandler::isSet = false;
+ ULONG FatalConditionHandler::guaranteeSize = 0;
+ LPTOP_LEVEL_EXCEPTION_FILTER FatalConditionHandler::previousTop = nullptr;
+
+#else // DOCTEST_PLATFORM_WINDOWS
+
+ struct SignalDefs
+ {
+ int id;
+ const char* name;
+ };
+ SignalDefs signalDefs[] = {{SIGINT, "SIGINT - Terminal interrupt signal"},
+ {SIGILL, "SIGILL - Illegal instruction signal"},
+ {SIGFPE, "SIGFPE - Floating point error signal"},
+ {SIGSEGV, "SIGSEGV - Segmentation violation signal"},
+ {SIGTERM, "SIGTERM - Termination request signal"},
+ {SIGABRT, "SIGABRT - Abort (abnormal termination) signal"}};
+
+ struct FatalConditionHandler
+ {
+ static bool isSet;
+ static struct sigaction oldSigActions[DOCTEST_COUNTOF(signalDefs)];
+ static stack_t oldSigStack;
+ static size_t altStackSize;
+ static char* altStackMem;
+
+ static void handleSignal(int sig) {
+ const char* name = "<unknown signal>";
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ SignalDefs& def = signalDefs[i];
+ if(sig == def.id) {
+ name = def.name;
+ break;
+ }
+ }
+ reset();
+ reportFatal(name);
+ raise(sig);
+ }
+
+ static void allocateAltStackMem() {
+ altStackMem = new char[altStackSize];
+ }
+
+ static void freeAltStackMem() {
+ delete[] altStackMem;
+ }
+
+ FatalConditionHandler() {
+ isSet = true;
+ stack_t sigStack;
+ sigStack.ss_sp = altStackMem;
+ sigStack.ss_size = altStackSize;
+ sigStack.ss_flags = 0;
+ sigaltstack(&sigStack, &oldSigStack);
+ struct sigaction sa = {};
+ sa.sa_handler = handleSignal; // NOLINT
+ sa.sa_flags = SA_ONSTACK;
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ sigaction(signalDefs[i].id, &sa, &oldSigActions[i]);
+ }
+ }
+
+ ~FatalConditionHandler() { reset(); }
+ static void reset() {
+ if(isSet) {
+ // Set signals back to previous values -- hopefully nobody overwrote them in the meantime
+ for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) {
+ sigaction(signalDefs[i].id, &oldSigActions[i], nullptr);
+ }
+ // Return the old stack
+ sigaltstack(&oldSigStack, nullptr);
+ isSet = false;
+ }
+ }
+ };
+
+ bool FatalConditionHandler::isSet = false;
+ struct sigaction FatalConditionHandler::oldSigActions[DOCTEST_COUNTOF(signalDefs)] = {};
+ stack_t FatalConditionHandler::oldSigStack = {};
+ size_t FatalConditionHandler::altStackSize = 4 * SIGSTKSZ;
+ char* FatalConditionHandler::altStackMem = nullptr;
+
+#endif // DOCTEST_PLATFORM_WINDOWS
+#endif // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+
+} // namespace
+
+namespace {
+ using namespace detail;
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+#define DOCTEST_OUTPUT_DEBUG_STRING(text) ::OutputDebugStringA(text)
+#else
+ // TODO: integration with XCode and other IDEs
+#define DOCTEST_OUTPUT_DEBUG_STRING(text) // NOLINT(clang-diagnostic-unused-macros)
+#endif // Platform
+
+ void addAssert(assertType::Enum at) {
+ if((at & assertType::is_warn) == 0) //!OCLINT bitwise operator in conditional
+ g_cs->numAssertsCurrentTest_atomic++;
+ }
+
+ void addFailedAssert(assertType::Enum at) {
+ if((at & assertType::is_warn) == 0) //!OCLINT bitwise operator in conditional
+ g_cs->numAssertsFailedCurrentTest_atomic++;
+ }
+
+#if defined(DOCTEST_CONFIG_POSIX_SIGNALS) || defined(DOCTEST_CONFIG_WINDOWS_SEH)
+ void reportFatal(const std::string& message) {
+ g_cs->failure_flags |= TestCaseFailureReason::Crash;
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_exception, {message.c_str(), true});
+
+ while(g_cs->subcasesStack.size()) {
+ g_cs->subcasesStack.pop_back();
+ DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_end, DOCTEST_EMPTY);
+ }
+
+ g_cs->finalizeTestCaseData();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_end, *g_cs);
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_end, *g_cs);
+ }
+#endif // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH
+} // namespace
+namespace detail {
+
+ ResultBuilder::ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr,
+ const char* exception_type, const char* exception_string) {
+ m_test_case = g_cs->currentTest;
+ m_at = at;
+ m_file = file;
+ m_line = line;
+ m_expr = expr;
+ m_failed = true;
+ m_threw = false;
+ m_threw_as = false;
+ m_exception_type = exception_type;
+ m_exception_string = exception_string;
+#if DOCTEST_MSVC
+ if(m_expr[0] == ' ') // this happens when variadic macros are disabled under MSVC
+ ++m_expr;
+#endif // MSVC
+ }
+
+ void ResultBuilder::setResult(const Result& res) {
+ m_decomp = res.m_decomp;
+ m_failed = !res.m_passed;
+ }
+
+ void ResultBuilder::translateException() {
+ m_threw = true;
+ m_exception = translateActiveException();
+ }
+
+ bool ResultBuilder::log() {
+ if(m_at & assertType::is_throws) { //!OCLINT bitwise operator in conditional
+ m_failed = !m_threw;
+ } else if((m_at & assertType::is_throws_as) && (m_at & assertType::is_throws_with)) { //!OCLINT
+ m_failed = !m_threw_as || (m_exception != m_exception_string);
+ } else if(m_at & assertType::is_throws_as) { //!OCLINT bitwise operator in conditional
+ m_failed = !m_threw_as;
+ } else if(m_at & assertType::is_throws_with) { //!OCLINT bitwise operator in conditional
+ m_failed = m_exception != m_exception_string;
+ } else if(m_at & assertType::is_nothrow) { //!OCLINT bitwise operator in conditional
+ m_failed = m_threw;
+ }
+
+ if(m_exception.size())
+ m_exception = String("\"") + m_exception + "\"";
+
+ if(is_running_in_test) {
+ addAssert(m_at);
+ DOCTEST_ITERATE_THROUGH_REPORTERS(log_assert, *this);
+
+ if(m_failed)
+ addFailedAssert(m_at);
+ } else if(m_failed) {
+ failed_out_of_a_testing_context(*this);
+ }
+
+ return m_failed && isDebuggerActive() && !getContextOptions()->no_breaks &&
+ (g_cs->currentTest == nullptr || !g_cs->currentTest->m_no_breaks); // break into debugger
+ }
+
+ void ResultBuilder::react() const {
+ if(m_failed && checkIfShouldThrow(m_at))
+ throwException();
+ }
+
+ void failed_out_of_a_testing_context(const AssertData& ad) {
+ if(g_cs->ah)
+ g_cs->ah(ad);
+ else
+ std::abort();
+ }
+
+ void decomp_assert(assertType::Enum at, const char* file, int line, const char* expr,
+ Result result) {
+ bool failed = !result.m_passed;
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(result.m_decomp);
+ DOCTEST_ASSERT_IN_TESTS(result.m_decomp);
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ }
+
+ MessageBuilder::MessageBuilder(const char* file, int line, assertType::Enum severity) {
+ m_stream = getTlsOss();
+ m_file = file;
+ m_line = line;
+ m_severity = severity;
+ }
+
+ IExceptionTranslator::IExceptionTranslator() = default;
+ IExceptionTranslator::~IExceptionTranslator() = default;
+
+ bool MessageBuilder::log() {
+ m_string = getTlsOssResult();
+ DOCTEST_ITERATE_THROUGH_REPORTERS(log_message, *this);
+
+ const bool isWarn = m_severity & assertType::is_warn;
+
+ // warn is just a message in this context so we don't treat it as an assert
+ if(!isWarn) {
+ addAssert(m_severity);
+ addFailedAssert(m_severity);
+ }
+
+ return isDebuggerActive() && !getContextOptions()->no_breaks && !isWarn &&
+ (g_cs->currentTest == nullptr || !g_cs->currentTest->m_no_breaks); // break into debugger
+ }
+
+ void MessageBuilder::react() {
+ if(m_severity & assertType::is_require) //!OCLINT bitwise operator in conditional
+ throwException();
+ }
+
+ MessageBuilder::~MessageBuilder() = default;
+} // namespace detail
+namespace {
+ using namespace detail;
+
+ template <typename Ex>
+ DOCTEST_NORETURN void throw_exception(Ex const& e) {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ throw e;
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS
+ std::cerr << "doctest will terminate because it needed to throw an exception.\n"
+ << "The message was: " << e.what() << '\n';
+ std::terminate();
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ }
+
+#ifndef DOCTEST_INTERNAL_ERROR
+#define DOCTEST_INTERNAL_ERROR(msg) \
+ throw_exception(std::logic_error( \
+ __FILE__ ":" DOCTEST_TOSTR(__LINE__) ": Internal doctest error: " msg))
+#endif // DOCTEST_INTERNAL_ERROR
+
+ // clang-format off
+
+// =================================================================================================
+// The following code has been taken verbatim from Catch2/include/internal/catch_xmlwriter.h/cpp
+// This is done so cherry-picking bug fixes is trivial - even the style/formatting is untouched.
+// =================================================================================================
+
+ class XmlEncode {
+ public:
+ enum ForWhat { ForTextNodes, ForAttributes };
+
+ XmlEncode( std::string const& str, ForWhat forWhat = ForTextNodes );
+
+ void encodeTo( std::ostream& os ) const;
+
+ friend std::ostream& operator << ( std::ostream& os, XmlEncode const& xmlEncode );
+
+ private:
+ std::string m_str;
+ ForWhat m_forWhat;
+ };
+
+ class XmlWriter {
+ public:
+
+ class ScopedElement {
+ public:
+ ScopedElement( XmlWriter* writer );
+
+ ScopedElement( ScopedElement&& other ) DOCTEST_NOEXCEPT;
+ ScopedElement& operator=( ScopedElement&& other ) DOCTEST_NOEXCEPT;
+
+ ~ScopedElement();
+
+ ScopedElement& writeText( std::string const& text, bool indent = true );
+
+ template<typename T>
+ ScopedElement& writeAttribute( std::string const& name, T const& attribute ) {
+ m_writer->writeAttribute( name, attribute );
+ return *this;
+ }
+
+ private:
+ mutable XmlWriter* m_writer = nullptr;
+ };
+
+ XmlWriter( std::ostream& os = std::cout );
+ ~XmlWriter();
+
+ XmlWriter( XmlWriter const& ) = delete;
+ XmlWriter& operator=( XmlWriter const& ) = delete;
+
+ XmlWriter& startElement( std::string const& name );
+
+ ScopedElement scopedElement( std::string const& name );
+
+ XmlWriter& endElement();
+
+ XmlWriter& writeAttribute( std::string const& name, std::string const& attribute );
+
+ XmlWriter& writeAttribute( std::string const& name, const char* attribute );
+
+ XmlWriter& writeAttribute( std::string const& name, bool attribute );
+
+ template<typename T>
+ XmlWriter& writeAttribute( std::string const& name, T const& attribute ) {
+ std::stringstream rss;
+ rss << attribute;
+ return writeAttribute( name, rss.str() );
+ }
+
+ XmlWriter& writeText( std::string const& text, bool indent = true );
+
+ //XmlWriter& writeComment( std::string const& text );
+
+ //void writeStylesheetRef( std::string const& url );
+
+ //XmlWriter& writeBlankLine();
+
+ void ensureTagClosed();
+
+ private:
+
+ void writeDeclaration();
+
+ void newlineIfNecessary();
+
+ bool m_tagIsOpen = false;
+ bool m_needsNewline = false;
+ std::vector<std::string> m_tags;
+ std::string m_indent;
+ std::ostream& m_os;
+ };
+
+// =================================================================================================
+// The following code has been taken verbatim from Catch2/include/internal/catch_xmlwriter.h/cpp
+// This is done so cherry-picking bug fixes is trivial - even the style/formatting is untouched.
+// =================================================================================================
+
+using uchar = unsigned char;
+
+namespace {
+
+ size_t trailingBytes(unsigned char c) {
+ if ((c & 0xE0) == 0xC0) {
+ return 2;
+ }
+ if ((c & 0xF0) == 0xE0) {
+ return 3;
+ }
+ if ((c & 0xF8) == 0xF0) {
+ return 4;
+ }
+ DOCTEST_INTERNAL_ERROR("Invalid multibyte utf-8 start byte encountered");
+ }
+
+ uint32_t headerValue(unsigned char c) {
+ if ((c & 0xE0) == 0xC0) {
+ return c & 0x1F;
+ }
+ if ((c & 0xF0) == 0xE0) {
+ return c & 0x0F;
+ }
+ if ((c & 0xF8) == 0xF0) {
+ return c & 0x07;
+ }
+ DOCTEST_INTERNAL_ERROR("Invalid multibyte utf-8 start byte encountered");
+ }
+
+ void hexEscapeChar(std::ostream& os, unsigned char c) {
+ std::ios_base::fmtflags f(os.flags());
+ os << "\\x"
+ << std::uppercase << std::hex << std::setfill('0') << std::setw(2)
+ << static_cast<int>(c);
+ os.flags(f);
+ }
+
+} // anonymous namespace
+
+ XmlEncode::XmlEncode( std::string const& str, ForWhat forWhat )
+ : m_str( str ),
+ m_forWhat( forWhat )
+ {}
+
+ void XmlEncode::encodeTo( std::ostream& os ) const {
+ // Apostrophe escaping not necessary if we always use " to write attributes
+ // (see: https://www.w3.org/TR/xml/#syntax)
+
+ for( std::size_t idx = 0; idx < m_str.size(); ++ idx ) {
+ uchar c = m_str[idx];
+ switch (c) {
+ case '<': os << "&lt;"; break;
+ case '&': os << "&amp;"; break;
+
+ case '>':
+ // See: https://www.w3.org/TR/xml/#syntax
+ if (idx > 2 && m_str[idx - 1] == ']' && m_str[idx - 2] == ']')
+ os << "&gt;";
+ else
+ os << c;
+ break;
+
+ case '\"':
+ if (m_forWhat == ForAttributes)
+ os << "&quot;";
+ else
+ os << c;
+ break;
+
+ default:
+ // Check for control characters and invalid utf-8
+
+ // Escape control characters in standard ascii
+ // see https://stackoverflow.com/questions/404107/why-are-control-characters-illegal-in-xml-1-0
+ if (c < 0x09 || (c > 0x0D && c < 0x20) || c == 0x7F) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ // Plain ASCII: Write it to stream
+ if (c < 0x7F) {
+ os << c;
+ break;
+ }
+
+ // UTF-8 territory
+ // Check if the encoding is valid and if it is not, hex escape bytes.
+ // Important: We do not check the exact decoded values for validity, only the encoding format
+ // First check that this bytes is a valid lead byte:
+ // This means that it is not encoded as 1111 1XXX
+ // Or as 10XX XXXX
+ if (c < 0xC0 ||
+ c >= 0xF8) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ auto encBytes = trailingBytes(c);
+ // Are there enough bytes left to avoid accessing out-of-bounds memory?
+ if (idx + encBytes - 1 >= m_str.size()) {
+ hexEscapeChar(os, c);
+ break;
+ }
+ // The header is valid, check data
+ // The next encBytes bytes must together be a valid utf-8
+ // This means: bitpattern 10XX XXXX and the extracted value is sane (ish)
+ bool valid = true;
+ uint32_t value = headerValue(c);
+ for (std::size_t n = 1; n < encBytes; ++n) {
+ uchar nc = m_str[idx + n];
+ valid &= ((nc & 0xC0) == 0x80);
+ value = (value << 6) | (nc & 0x3F);
+ }
+
+ if (
+ // Wrong bit pattern of following bytes
+ (!valid) ||
+ // Overlong encodings
+ (value < 0x80) ||
+ ( value < 0x800 && encBytes > 2) || // removed "0x80 <= value &&" because redundant
+ (0x800 < value && value < 0x10000 && encBytes > 3) ||
+ // Encoded value out of range
+ (value >= 0x110000)
+ ) {
+ hexEscapeChar(os, c);
+ break;
+ }
+
+ // If we got here, this is in fact a valid(ish) utf-8 sequence
+ for (std::size_t n = 0; n < encBytes; ++n) {
+ os << m_str[idx + n];
+ }
+ idx += encBytes - 1;
+ break;
+ }
+ }
+ }
+
+ std::ostream& operator << ( std::ostream& os, XmlEncode const& xmlEncode ) {
+ xmlEncode.encodeTo( os );
+ return os;
+ }
+
+ XmlWriter::ScopedElement::ScopedElement( XmlWriter* writer )
+ : m_writer( writer )
+ {}
+
+ XmlWriter::ScopedElement::ScopedElement( ScopedElement&& other ) DOCTEST_NOEXCEPT
+ : m_writer( other.m_writer ){
+ other.m_writer = nullptr;
+ }
+ XmlWriter::ScopedElement& XmlWriter::ScopedElement::operator=( ScopedElement&& other ) DOCTEST_NOEXCEPT {
+ if ( m_writer ) {
+ m_writer->endElement();
+ }
+ m_writer = other.m_writer;
+ other.m_writer = nullptr;
+ return *this;
+ }
+
+
+ XmlWriter::ScopedElement::~ScopedElement() {
+ if( m_writer )
+ m_writer->endElement();
+ }
+
+ XmlWriter::ScopedElement& XmlWriter::ScopedElement::writeText( std::string const& text, bool indent ) {
+ m_writer->writeText( text, indent );
+ return *this;
+ }
+
+ XmlWriter::XmlWriter( std::ostream& os ) : m_os( os )
+ {
+ writeDeclaration();
+ }
+
+ XmlWriter::~XmlWriter() {
+ while( !m_tags.empty() )
+ endElement();
+ }
+
+ XmlWriter& XmlWriter::startElement( std::string const& name ) {
+ ensureTagClosed();
+ newlineIfNecessary();
+ m_os << m_indent << '<' << name;
+ m_tags.push_back( name );
+ m_indent += " ";
+ m_tagIsOpen = true;
+ return *this;
+ }
+
+ XmlWriter::ScopedElement XmlWriter::scopedElement( std::string const& name ) {
+ ScopedElement scoped( this );
+ startElement( name );
+ return scoped;
+ }
+
+ XmlWriter& XmlWriter::endElement() {
+ newlineIfNecessary();
+ m_indent = m_indent.substr( 0, m_indent.size()-2 );
+ if( m_tagIsOpen ) {
+ m_os << "/>";
+ m_tagIsOpen = false;
+ }
+ else {
+ m_os << m_indent << "</" << m_tags.back() << ">";
+ }
+ m_os << std::endl;
+ m_tags.pop_back();
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, std::string const& attribute ) {
+ if( !name.empty() && !attribute.empty() )
+ m_os << ' ' << name << "=\"" << XmlEncode( attribute, XmlEncode::ForAttributes ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, const char* attribute ) {
+ if( !name.empty() && attribute && attribute[0] != '\0' )
+ m_os << ' ' << name << "=\"" << XmlEncode( attribute, XmlEncode::ForAttributes ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeAttribute( std::string const& name, bool attribute ) {
+ m_os << ' ' << name << "=\"" << ( attribute ? "true" : "false" ) << '"';
+ return *this;
+ }
+
+ XmlWriter& XmlWriter::writeText( std::string const& text, bool indent ) {
+ if( !text.empty() ){
+ bool tagWasOpen = m_tagIsOpen;
+ ensureTagClosed();
+ if( tagWasOpen && indent )
+ m_os << m_indent;
+ m_os << XmlEncode( text );
+ m_needsNewline = true;
+ }
+ return *this;
+ }
+
+ //XmlWriter& XmlWriter::writeComment( std::string const& text ) {
+ // ensureTagClosed();
+ // m_os << m_indent << "<!--" << text << "-->";
+ // m_needsNewline = true;
+ // return *this;
+ //}
+
+ //void XmlWriter::writeStylesheetRef( std::string const& url ) {
+ // m_os << "<?xml-stylesheet type=\"text/xsl\" href=\"" << url << "\"?>\n";
+ //}
+
+ //XmlWriter& XmlWriter::writeBlankLine() {
+ // ensureTagClosed();
+ // m_os << '\n';
+ // return *this;
+ //}
+
+ void XmlWriter::ensureTagClosed() {
+ if( m_tagIsOpen ) {
+ m_os << ">" << std::endl;
+ m_tagIsOpen = false;
+ }
+ }
+
+ void XmlWriter::writeDeclaration() {
+ m_os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+ }
+
+ void XmlWriter::newlineIfNecessary() {
+ if( m_needsNewline ) {
+ m_os << std::endl;
+ m_needsNewline = false;
+ }
+ }
+
+// =================================================================================================
+// End of copy-pasted code from Catch
+// =================================================================================================
+
+ // clang-format on
+
+ struct XmlReporter : public IReporter
+ {
+ XmlWriter xml;
+ std::mutex mutex;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc = nullptr;
+
+ XmlReporter(const ContextOptions& co)
+ : xml(*co.cout)
+ , opt(co) {}
+
+ void log_contexts() {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+ std::stringstream ss;
+ for(int i = 0; i < num_contexts; ++i) {
+ contexts[i]->stringify(&ss);
+ xml.scopedElement("Info").writeText(ss.str());
+ ss.str("");
+ }
+ }
+ }
+
+ unsigned line(unsigned l) const { return opt.no_line_numbers ? 0 : l; }
+
+ void test_case_start_impl(const TestCaseData& in) {
+ bool open_ts_tag = false;
+ if(tc != nullptr) { // we have already opened a test suite
+ if(std::strcmp(tc->m_test_suite, in.m_test_suite) != 0) {
+ xml.endElement();
+ open_ts_tag = true;
+ }
+ }
+ else {
+ open_ts_tag = true; // first test case ==> first test suite
+ }
+
+ if(open_ts_tag) {
+ xml.startElement("TestSuite");
+ xml.writeAttribute("name", in.m_test_suite);
+ }
+
+ tc = &in;
+ xml.startElement("TestCase")
+ .writeAttribute("name", in.m_name)
+ .writeAttribute("filename", skipPathFromFilename(in.m_file.c_str()))
+ .writeAttribute("line", line(in.m_line))
+ .writeAttribute("description", in.m_description);
+
+ if(Approx(in.m_timeout) != 0)
+ xml.writeAttribute("timeout", in.m_timeout);
+ if(in.m_may_fail)
+ xml.writeAttribute("may_fail", true);
+ if(in.m_should_fail)
+ xml.writeAttribute("should_fail", true);
+ }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData& in) override {
+ test_run_start();
+ if(opt.list_reporters) {
+ for(auto& curr : getListeners())
+ xml.scopedElement("Listener")
+ .writeAttribute("priority", curr.first.first)
+ .writeAttribute("name", curr.first.second);
+ for(auto& curr : getReporters())
+ xml.scopedElement("Reporter")
+ .writeAttribute("priority", curr.first.first)
+ .writeAttribute("name", curr.first.second);
+ } else if(opt.count || opt.list_test_cases) {
+ for(unsigned i = 0; i < in.num_data; ++i) {
+ xml.scopedElement("TestCase").writeAttribute("name", in.data[i]->m_name)
+ .writeAttribute("testsuite", in.data[i]->m_test_suite)
+ .writeAttribute("filename", skipPathFromFilename(in.data[i]->m_file.c_str()))
+ .writeAttribute("line", line(in.data[i]->m_line));
+ }
+ xml.scopedElement("OverallResultsTestCases")
+ .writeAttribute("unskipped", in.run_stats->numTestCasesPassingFilters);
+ } else if(opt.list_test_suites) {
+ for(unsigned i = 0; i < in.num_data; ++i)
+ xml.scopedElement("TestSuite").writeAttribute("name", in.data[i]->m_test_suite);
+ xml.scopedElement("OverallResultsTestCases")
+ .writeAttribute("unskipped", in.run_stats->numTestCasesPassingFilters);
+ xml.scopedElement("OverallResultsTestSuites")
+ .writeAttribute("unskipped", in.run_stats->numTestSuitesPassingFilters);
+ }
+ xml.endElement();
+ }
+
+ void test_run_start() override {
+ // remove .exe extension - mainly to have the same output on UNIX and Windows
+ std::string binary_name = skipPathFromFilename(opt.binary_name.c_str());
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(binary_name.rfind(".exe") != std::string::npos)
+ binary_name = binary_name.substr(0, binary_name.length() - 4);
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ xml.startElement("doctest").writeAttribute("binary", binary_name);
+ if(opt.no_version == false)
+ xml.writeAttribute("version", DOCTEST_VERSION_STR);
+
+ // only the consequential ones (TODO: filters)
+ xml.scopedElement("Options")
+ .writeAttribute("order_by", opt.order_by.c_str())
+ .writeAttribute("rand_seed", opt.rand_seed)
+ .writeAttribute("first", opt.first)
+ .writeAttribute("last", opt.last)
+ .writeAttribute("abort_after", opt.abort_after)
+ .writeAttribute("subcase_filter_levels", opt.subcase_filter_levels)
+ .writeAttribute("case_sensitive", opt.case_sensitive)
+ .writeAttribute("no_throw", opt.no_throw)
+ .writeAttribute("no_skip", opt.no_skip);
+ }
+
+ void test_run_end(const TestRunStats& p) override {
+ if(tc) // the TestSuite tag - only if there has been at least 1 test case
+ xml.endElement();
+
+ xml.scopedElement("OverallResultsAsserts")
+ .writeAttribute("successes", p.numAsserts - p.numAssertsFailed)
+ .writeAttribute("failures", p.numAssertsFailed);
+
+ xml.startElement("OverallResultsTestCases")
+ .writeAttribute("successes",
+ p.numTestCasesPassingFilters - p.numTestCasesFailed)
+ .writeAttribute("failures", p.numTestCasesFailed);
+ if(opt.no_skipped_summary == false)
+ xml.writeAttribute("skipped", p.numTestCases - p.numTestCasesPassingFilters);
+ xml.endElement();
+
+ xml.endElement();
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ test_case_start_impl(in);
+ xml.ensureTagClosed();
+ }
+
+ void test_case_reenter(const TestCaseData&) override {}
+
+ void test_case_end(const CurrentTestCaseStats& st) override {
+ xml.startElement("OverallResultsAsserts")
+ .writeAttribute("successes",
+ st.numAssertsCurrentTest - st.numAssertsFailedCurrentTest)
+ .writeAttribute("failures", st.numAssertsFailedCurrentTest);
+ if(opt.duration)
+ xml.writeAttribute("duration", st.seconds);
+ if(tc->m_expected_failures)
+ xml.writeAttribute("expected_failures", tc->m_expected_failures);
+ xml.endElement();
+
+ xml.endElement();
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.scopedElement("Exception")
+ .writeAttribute("crash", e.is_crash)
+ .writeText(e.error_string.c_str());
+ }
+
+ void subcase_start(const SubcaseSignature& in) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("SubCase")
+ .writeAttribute("name", in.m_name)
+ .writeAttribute("filename", skipPathFromFilename(in.m_file))
+ .writeAttribute("line", line(in.m_line));
+ xml.ensureTagClosed();
+ }
+
+ void subcase_end() override { xml.endElement(); }
+
+ void log_assert(const AssertData& rb) override {
+ if(!rb.m_failed && !opt.success)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("Expression")
+ .writeAttribute("success", !rb.m_failed)
+ .writeAttribute("type", assertString(rb.m_at))
+ .writeAttribute("filename", skipPathFromFilename(rb.m_file))
+ .writeAttribute("line", line(rb.m_line));
+
+ xml.scopedElement("Original").writeText(rb.m_expr);
+
+ if(rb.m_threw)
+ xml.scopedElement("Exception").writeText(rb.m_exception.c_str());
+
+ if(rb.m_at & assertType::is_throws_as)
+ xml.scopedElement("ExpectedException").writeText(rb.m_exception_type);
+ if(rb.m_at & assertType::is_throws_with)
+ xml.scopedElement("ExpectedExceptionString").writeText(rb.m_exception_string);
+ if((rb.m_at & assertType::is_normal) && !rb.m_threw)
+ xml.scopedElement("Expanded").writeText(rb.m_decomp.c_str());
+
+ log_contexts();
+
+ xml.endElement();
+ }
+
+ void log_message(const MessageData& mb) override {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ xml.startElement("Message")
+ .writeAttribute("type", failureString(mb.m_severity))
+ .writeAttribute("filename", skipPathFromFilename(mb.m_file))
+ .writeAttribute("line", line(mb.m_line));
+
+ xml.scopedElement("Text").writeText(mb.m_string.c_str());
+
+ log_contexts();
+
+ xml.endElement();
+ }
+
+ void test_case_skipped(const TestCaseData& in) override {
+ if(opt.no_skipped_summary == false) {
+ test_case_start_impl(in);
+ xml.writeAttribute("skipped", "true");
+ xml.endElement();
+ }
+ }
+ };
+
+ DOCTEST_REGISTER_REPORTER("xml", 0, XmlReporter);
+
+ void fulltext_log_assert_to_stream(std::ostream& s, const AssertData& rb) {
+ if((rb.m_at & (assertType::is_throws_as | assertType::is_throws_with)) ==
+ 0) //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << " ) "
+ << Color::None;
+
+ if(rb.m_at & assertType::is_throws) { //!OCLINT bitwise operator in conditional
+ s << (rb.m_threw ? "threw as expected!" : "did NOT throw at all!") << "\n";
+ } else if((rb.m_at & assertType::is_throws_as) &&
+ (rb.m_at & assertType::is_throws_with)) { //!OCLINT
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", \""
+ << rb.m_exception_string << "\", " << rb.m_exception_type << " ) " << Color::None;
+ if(rb.m_threw) {
+ if(!rb.m_failed) {
+ s << "threw as expected!\n";
+ } else {
+ s << "threw a DIFFERENT exception! (contents: " << rb.m_exception << ")\n";
+ }
+ } else {
+ s << "did NOT throw at all!\n";
+ }
+ } else if(rb.m_at &
+ assertType::is_throws_as) { //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", "
+ << rb.m_exception_type << " ) " << Color::None
+ << (rb.m_threw ? (rb.m_threw_as ? "threw as expected!" :
+ "threw a DIFFERENT exception: ") :
+ "did NOT throw at all!")
+ << Color::Cyan << rb.m_exception << "\n";
+ } else if(rb.m_at &
+ assertType::is_throws_with) { //!OCLINT bitwise operator in conditional
+ s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", \""
+ << rb.m_exception_string << "\" ) " << Color::None
+ << (rb.m_threw ? (!rb.m_failed ? "threw as expected!" :
+ "threw a DIFFERENT exception: ") :
+ "did NOT throw at all!")
+ << Color::Cyan << rb.m_exception << "\n";
+ } else if(rb.m_at & assertType::is_nothrow) { //!OCLINT bitwise operator in conditional
+ s << (rb.m_threw ? "THREW exception: " : "didn't throw!") << Color::Cyan
+ << rb.m_exception << "\n";
+ } else {
+ s << (rb.m_threw ? "THREW exception: " :
+ (!rb.m_failed ? "is correct!\n" : "is NOT correct!\n"));
+ if(rb.m_threw)
+ s << rb.m_exception << "\n";
+ else
+ s << " values: " << assertString(rb.m_at) << "( " << rb.m_decomp << " )\n";
+ }
+ }
+
+ // TODO:
+ // - log_message()
+ // - respond to queries
+ // - honor remaining options
+ // - more attributes in tags
+ struct JUnitReporter : public IReporter
+ {
+ XmlWriter xml;
+ std::mutex mutex;
+ Timer timer;
+ std::vector<String> deepestSubcaseStackNames;
+
+ struct JUnitTestCaseData
+ {
+ static std::string getCurrentTimestamp() {
+ // Beware, this is not reentrant because of backward compatibility issues
+ // Also, UTC only, again because of backward compatibility (%z is C++11)
+ time_t rawtime;
+ std::time(&rawtime);
+ auto const timeStampSize = sizeof("2017-01-16T17:06:45Z");
+
+ std::tm timeInfo;
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ gmtime_s(&timeInfo, &rawtime);
+#else // DOCTEST_PLATFORM_WINDOWS
+ gmtime_r(&rawtime, &timeInfo);
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ char timeStamp[timeStampSize];
+ const char* const fmt = "%Y-%m-%dT%H:%M:%SZ";
+
+ std::strftime(timeStamp, timeStampSize, fmt, &timeInfo);
+ return std::string(timeStamp);
+ }
+
+ struct JUnitTestMessage
+ {
+ JUnitTestMessage(const std::string& _message, const std::string& _type, const std::string& _details)
+ : message(_message), type(_type), details(_details) {}
+
+ JUnitTestMessage(const std::string& _message, const std::string& _details)
+ : message(_message), type(), details(_details) {}
+
+ std::string message, type, details;
+ };
+
+ struct JUnitTestCase
+ {
+ JUnitTestCase(const std::string& _classname, const std::string& _name)
+ : classname(_classname), name(_name), time(0), failures() {}
+
+ std::string classname, name;
+ double time;
+ std::vector<JUnitTestMessage> failures, errors;
+ };
+
+ void add(const std::string& classname, const std::string& name) {
+ testcases.emplace_back(classname, name);
+ }
+
+ void appendSubcaseNamesToLastTestcase(std::vector<String> nameStack) {
+ for(auto& curr: nameStack)
+ if(curr.size())
+ testcases.back().name += std::string("/") + curr.c_str();
+ }
+
+ void addTime(double time) {
+ if(time < 1e-4)
+ time = 0;
+ testcases.back().time = time;
+ totalSeconds += time;
+ }
+
+ void addFailure(const std::string& message, const std::string& type, const std::string& details) {
+ testcases.back().failures.emplace_back(message, type, details);
+ ++totalFailures;
+ }
+
+ void addError(const std::string& message, const std::string& details) {
+ testcases.back().errors.emplace_back(message, details);
+ ++totalErrors;
+ }
+
+ std::vector<JUnitTestCase> testcases;
+ double totalSeconds = 0;
+ int totalErrors = 0, totalFailures = 0;
+ };
+
+ JUnitTestCaseData testCaseData;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc = nullptr;
+
+ JUnitReporter(const ContextOptions& co)
+ : xml(*co.cout)
+ , opt(co) {}
+
+ unsigned line(unsigned l) const { return opt.no_line_numbers ? 0 : l; }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData&) override {}
+
+ void test_run_start() override {}
+
+ void test_run_end(const TestRunStats& p) override {
+ // remove .exe extension - mainly to have the same output on UNIX and Windows
+ std::string binary_name = skipPathFromFilename(opt.binary_name.c_str());
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(binary_name.rfind(".exe") != std::string::npos)
+ binary_name = binary_name.substr(0, binary_name.length() - 4);
+#endif // DOCTEST_PLATFORM_WINDOWS
+ xml.startElement("testsuites");
+ xml.startElement("testsuite").writeAttribute("name", binary_name)
+ .writeAttribute("errors", testCaseData.totalErrors)
+ .writeAttribute("failures", testCaseData.totalFailures)
+ .writeAttribute("tests", p.numAsserts);
+ if(opt.no_time_in_output == false) {
+ xml.writeAttribute("time", testCaseData.totalSeconds);
+ xml.writeAttribute("timestamp", JUnitTestCaseData::getCurrentTimestamp());
+ }
+ if(opt.no_version == false)
+ xml.writeAttribute("doctest_version", DOCTEST_VERSION_STR);
+
+ for(const auto& testCase : testCaseData.testcases) {
+ xml.startElement("testcase")
+ .writeAttribute("classname", testCase.classname)
+ .writeAttribute("name", testCase.name);
+ if(opt.no_time_in_output == false)
+ xml.writeAttribute("time", testCase.time);
+ // This is not ideal, but it should be enough to mimic gtest's junit output.
+ xml.writeAttribute("status", "run");
+
+ for(const auto& failure : testCase.failures) {
+ xml.scopedElement("failure")
+ .writeAttribute("message", failure.message)
+ .writeAttribute("type", failure.type)
+ .writeText(failure.details, false);
+ }
+
+ for(const auto& error : testCase.errors) {
+ xml.scopedElement("error")
+ .writeAttribute("message", error.message)
+ .writeText(error.details);
+ }
+
+ xml.endElement();
+ }
+ xml.endElement();
+ xml.endElement();
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ testCaseData.add(skipPathFromFilename(in.m_file.c_str()), in.m_name);
+ timer.start();
+ }
+
+ void test_case_reenter(const TestCaseData& in) override {
+ testCaseData.addTime(timer.getElapsedSeconds());
+ testCaseData.appendSubcaseNamesToLastTestcase(deepestSubcaseStackNames);
+ deepestSubcaseStackNames.clear();
+
+ timer.start();
+ testCaseData.add(skipPathFromFilename(in.m_file.c_str()), in.m_name);
+ }
+
+ void test_case_end(const CurrentTestCaseStats&) override {
+ testCaseData.addTime(timer.getElapsedSeconds());
+ testCaseData.appendSubcaseNamesToLastTestcase(deepestSubcaseStackNames);
+ deepestSubcaseStackNames.clear();
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ testCaseData.addError("exception", e.error_string.c_str());
+ }
+
+ void subcase_start(const SubcaseSignature& in) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ deepestSubcaseStackNames.push_back(in.m_name);
+ }
+
+ void subcase_end() override {}
+
+ void log_assert(const AssertData& rb) override {
+ if(!rb.m_failed) // report only failures & ignore the `success` option
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ std::ostringstream os;
+ os << skipPathFromFilename(rb.m_file) << (opt.gnu_file_line ? ":" : "(")
+ << line(rb.m_line) << (opt.gnu_file_line ? ":" : "):") << std::endl;
+
+ fulltext_log_assert_to_stream(os, rb);
+ log_contexts(os);
+ testCaseData.addFailure(rb.m_decomp.c_str(), assertString(rb.m_at), os.str());
+ }
+
+ void log_message(const MessageData&) override {}
+
+ void test_case_skipped(const TestCaseData&) override {}
+
+ void log_contexts(std::ostringstream& s) {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+
+ s << " logged: ";
+ for(int i = 0; i < num_contexts; ++i) {
+ s << (i == 0 ? "" : " ");
+ contexts[i]->stringify(&s);
+ s << std::endl;
+ }
+ }
+ }
+ };
+
+ DOCTEST_REGISTER_REPORTER("junit", 0, JUnitReporter);
+
+ struct Whitespace
+ {
+ int nrSpaces;
+ explicit Whitespace(int nr)
+ : nrSpaces(nr) {}
+ };
+
+ std::ostream& operator<<(std::ostream& out, const Whitespace& ws) {
+ if(ws.nrSpaces != 0)
+ out << std::setw(ws.nrSpaces) << ' ';
+ return out;
+ }
+
+ struct ConsoleReporter : public IReporter
+ {
+ std::ostream& s;
+ bool hasLoggedCurrentTestStart;
+ std::vector<SubcaseSignature> subcasesStack;
+ size_t currentSubcaseLevel;
+ std::mutex mutex;
+
+ // caching pointers/references to objects of these types - safe to do
+ const ContextOptions& opt;
+ const TestCaseData* tc;
+
+ ConsoleReporter(const ContextOptions& co)
+ : s(*co.cout)
+ , opt(co) {}
+
+ ConsoleReporter(const ContextOptions& co, std::ostream& ostr)
+ : s(ostr)
+ , opt(co) {}
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE HELPERS USED BY THE OVERRIDES OF THE VIRTUAL METHODS OF THE INTERFACE
+ // =========================================================================================
+
+ void separator_to_stream() {
+ s << Color::Yellow
+ << "==============================================================================="
+ "\n";
+ }
+
+ const char* getSuccessOrFailString(bool success, assertType::Enum at,
+ const char* success_str) {
+ if(success)
+ return success_str;
+ return failureString(at);
+ }
+
+ Color::Enum getSuccessOrFailColor(bool success, assertType::Enum at) {
+ return success ? Color::BrightGreen :
+ (at & assertType::is_warn) ? Color::Yellow : Color::Red;
+ }
+
+ void successOrFailColoredStringToStream(bool success, assertType::Enum at,
+ const char* success_str = "SUCCESS") {
+ s << getSuccessOrFailColor(success, at)
+ << getSuccessOrFailString(success, at, success_str) << ": ";
+ }
+
+ void log_contexts() {
+ int num_contexts = get_num_active_contexts();
+ if(num_contexts) {
+ auto contexts = get_active_contexts();
+
+ s << Color::None << " logged: ";
+ for(int i = 0; i < num_contexts; ++i) {
+ s << (i == 0 ? "" : " ");
+ contexts[i]->stringify(&s);
+ s << "\n";
+ }
+ }
+
+ s << "\n";
+ }
+
+ // this was requested to be made virtual so users could override it
+ virtual void file_line_to_stream(const char* file, int line,
+ const char* tail = "") {
+ s << Color::LightGrey << skipPathFromFilename(file) << (opt.gnu_file_line ? ":" : "(")
+ << (opt.no_line_numbers ? 0 : line) // 0 or the real num depending on the option
+ << (opt.gnu_file_line ? ":" : "):") << tail;
+ }
+
+ void logTestStart() {
+ if(hasLoggedCurrentTestStart)
+ return;
+
+ separator_to_stream();
+ file_line_to_stream(tc->m_file.c_str(), tc->m_line, "\n");
+ if(tc->m_description)
+ s << Color::Yellow << "DESCRIPTION: " << Color::None << tc->m_description << "\n";
+ if(tc->m_test_suite && tc->m_test_suite[0] != '\0')
+ s << Color::Yellow << "TEST SUITE: " << Color::None << tc->m_test_suite << "\n";
+ if(strncmp(tc->m_name, " Scenario:", 11) != 0)
+ s << Color::Yellow << "TEST CASE: ";
+ s << Color::None << tc->m_name << "\n";
+
+ for(size_t i = 0; i < currentSubcaseLevel; ++i) {
+ if(subcasesStack[i].m_name[0] != '\0')
+ s << " " << subcasesStack[i].m_name << "\n";
+ }
+
+ if(currentSubcaseLevel != subcasesStack.size()) {
+ s << Color::Yellow << "\nDEEPEST SUBCASE STACK REACHED (DIFFERENT FROM THE CURRENT ONE):\n" << Color::None;
+ for(size_t i = 0; i < subcasesStack.size(); ++i) {
+ if(subcasesStack[i].m_name[0] != '\0')
+ s << " " << subcasesStack[i].m_name << "\n";
+ }
+ }
+
+ s << "\n";
+
+ hasLoggedCurrentTestStart = true;
+ }
+
+ void printVersion() {
+ if(opt.no_version == false)
+ s << Color::Cyan << "[doctest] " << Color::None << "doctest version is \""
+ << DOCTEST_VERSION_STR << "\"\n";
+ }
+
+ void printIntro() {
+ printVersion();
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "run with \"--" DOCTEST_OPTIONS_PREFIX_DISPLAY "help\" for options\n";
+ }
+
+ void printHelp() {
+ int sizePrefixDisplay = static_cast<int>(strlen(DOCTEST_OPTIONS_PREFIX_DISPLAY));
+ printVersion();
+ // clang-format off
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "boolean values: \"1/on/yes/true\" or \"0/off/no/false\"\n";
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "filter values: \"str1,str2,str3\" (comma separated strings)\n";
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "filters use wildcards for matching strings\n";
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "something passes a filter if any of the strings in a filter matches\n";
+#ifndef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "ALL FLAGS, OPTIONS AND FILTERS ALSO AVAILABLE WITH A \"" DOCTEST_CONFIG_OPTIONS_PREFIX "\" PREFIX!!!\n";
+#endif
+ s << Color::Cyan << "[doctest]\n" << Color::None;
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "Query flags - the program quits after them. Available:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "?, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "help, -" DOCTEST_OPTIONS_PREFIX_DISPLAY "h "
+ << Whitespace(sizePrefixDisplay*0) << "prints this message\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "v, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "version "
+ << Whitespace(sizePrefixDisplay*1) << "prints the version\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "c, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "count "
+ << Whitespace(sizePrefixDisplay*1) << "prints the number of matching tests\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ltc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-test-cases "
+ << Whitespace(sizePrefixDisplay*1) << "lists all matching tests by name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "lts, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-test-suites "
+ << Whitespace(sizePrefixDisplay*1) << "lists all matching test suites\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "lr, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "list-reporters "
+ << Whitespace(sizePrefixDisplay*1) << "lists all registered reporters\n\n";
+ // ================================================================================== << 79
+ s << Color::Cyan << "[doctest] " << Color::None;
+ s << "The available <int>/<string> options/filters are:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-case=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tce, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-case-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sf, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "source-file=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their file\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sfe, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "source-file-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their file\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ts, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-suite=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters tests by their test suite\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "tse, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "test-suite-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT tests by their test suite\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters subcases by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "sce, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase-exclude=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "filters OUT subcases by their name\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "r, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "reporters=<filters> "
+ << Whitespace(sizePrefixDisplay*1) << "reporters to use (console is default)\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "o, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "out=<string> "
+ << Whitespace(sizePrefixDisplay*1) << "output filename\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ob, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "order-by=<string> "
+ << Whitespace(sizePrefixDisplay*1) << "how the tests should be ordered\n";
+ s << Whitespace(sizePrefixDisplay*3) << " <string> - [file/suite/name/rand/none]\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "rs, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "rand-seed=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "seed for random ordering\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "f, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "first=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "the first test passing the filters to\n";
+ s << Whitespace(sizePrefixDisplay*3) << " execute - for range-based execution\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "l, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "last=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "the last test passing the filters to\n";
+ s << Whitespace(sizePrefixDisplay*3) << " execute - for range-based execution\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "aa, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "abort-after=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "stop after <int> failed assertions\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "scfl,--" DOCTEST_OPTIONS_PREFIX_DISPLAY "subcase-filter-levels=<int> "
+ << Whitespace(sizePrefixDisplay*1) << "apply filters for the first <int> levels\n";
+ s << Color::Cyan << "\n[doctest] " << Color::None;
+ s << "Bool options - can be used like flags and true is assumed. Available:\n\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "s, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "success=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "include successful assertions in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "cs, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "case-sensitive=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "filters being treated as case sensitive\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "e, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "exit=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "exits after the tests finish\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "d, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "duration=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "prints the time duration of each test\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nt, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-throw=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "skips exceptions-related assert checks\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ne, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-exitcode=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "returns (or exits) always with success\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nr, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-run=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "skips all runtime doctest operations\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nv, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-version=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "omit the framework version in the output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-colors=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "disables colors in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "fc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "force-colors=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "use colors even when not in a tty\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nb, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-breaks=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "disables breakpoints in debuggers\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ns, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-skip=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "don't skip test cases marked as skip\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "gfl, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "gnu-file-line=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << ":n: vs (n): for line numbers in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "npf, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-path-filenames=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "only filenames and no paths in output\n";
+ s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nln, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-line-numbers=<bool> "
+ << Whitespace(sizePrefixDisplay*1) << "0 instead of real line numbers in output\n";
+ // ================================================================================== << 79
+ // clang-format on
+
+ s << Color::Cyan << "\n[doctest] " << Color::None;
+ s << "for more information visit the project documentation\n\n";
+ }
+
+ void printRegisteredReporters() {
+ printVersion();
+ auto printReporters = [this] (const reporterMap& reporters, const char* type) {
+ if(reporters.size()) {
+ s << Color::Cyan << "[doctest] " << Color::None << "listing all registered " << type << "\n";
+ for(auto& curr : reporters)
+ s << "priority: " << std::setw(5) << curr.first.first
+ << " name: " << curr.first.second << "\n";
+ }
+ };
+ printReporters(getListeners(), "listeners");
+ printReporters(getReporters(), "reporters");
+ }
+
+ void list_query_results() {
+ separator_to_stream();
+ if(opt.count || opt.list_test_cases) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ } else if(opt.list_test_suites) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "test suites with unskipped test cases passing the current filters: "
+ << g_cs->numTestSuitesPassingFilters << "\n";
+ }
+ }
+
+ // =========================================================================================
+ // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE
+ // =========================================================================================
+
+ void report_query(const QueryData& in) override {
+ if(opt.version) {
+ printVersion();
+ } else if(opt.help) {
+ printHelp();
+ } else if(opt.list_reporters) {
+ printRegisteredReporters();
+ } else if(opt.count || opt.list_test_cases) {
+ if(opt.list_test_cases) {
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "listing all test case names\n";
+ separator_to_stream();
+ }
+
+ for(unsigned i = 0; i < in.num_data; ++i)
+ s << Color::None << in.data[i]->m_name << "\n";
+
+ separator_to_stream();
+
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+
+ } else if(opt.list_test_suites) {
+ s << Color::Cyan << "[doctest] " << Color::None << "listing all test suites\n";
+ separator_to_stream();
+
+ for(unsigned i = 0; i < in.num_data; ++i)
+ s << Color::None << in.data[i]->m_test_suite << "\n";
+
+ separator_to_stream();
+
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "unskipped test cases passing the current filters: "
+ << g_cs->numTestCasesPassingFilters << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "test suites with unskipped test cases passing the current filters: "
+ << g_cs->numTestSuitesPassingFilters << "\n";
+ }
+ }
+
+ void test_run_start() override { printIntro(); }
+
+ void test_run_end(const TestRunStats& p) override {
+ separator_to_stream();
+ s << std::dec;
+
+ auto totwidth = int(std::ceil(log10((std::max(p.numTestCasesPassingFilters, static_cast<unsigned>(p.numAsserts))) + 1)));
+ auto passwidth = int(std::ceil(log10((std::max(p.numTestCasesPassingFilters - p.numTestCasesFailed, static_cast<unsigned>(p.numAsserts - p.numAssertsFailed))) + 1)));
+ auto failwidth = int(std::ceil(log10((std::max(p.numTestCasesFailed, static_cast<unsigned>(p.numAssertsFailed))) + 1)));
+ const bool anythingFailed = p.numTestCasesFailed > 0 || p.numAssertsFailed > 0;
+ s << Color::Cyan << "[doctest] " << Color::None << "test cases: " << std::setw(totwidth)
+ << p.numTestCasesPassingFilters << " | "
+ << ((p.numTestCasesPassingFilters == 0 || anythingFailed) ? Color::None :
+ Color::Green)
+ << std::setw(passwidth) << p.numTestCasesPassingFilters - p.numTestCasesFailed << " passed"
+ << Color::None << " | " << (p.numTestCasesFailed > 0 ? Color::Red : Color::None)
+ << std::setw(failwidth) << p.numTestCasesFailed << " failed" << Color::None << " |";
+ if(opt.no_skipped_summary == false) {
+ const int numSkipped = p.numTestCases - p.numTestCasesPassingFilters;
+ s << " " << (numSkipped == 0 ? Color::None : Color::Yellow) << numSkipped
+ << " skipped" << Color::None;
+ }
+ s << "\n";
+ s << Color::Cyan << "[doctest] " << Color::None << "assertions: " << std::setw(totwidth)
+ << p.numAsserts << " | "
+ << ((p.numAsserts == 0 || anythingFailed) ? Color::None : Color::Green)
+ << std::setw(passwidth) << (p.numAsserts - p.numAssertsFailed) << " passed" << Color::None
+ << " | " << (p.numAssertsFailed > 0 ? Color::Red : Color::None) << std::setw(failwidth)
+ << p.numAssertsFailed << " failed" << Color::None << " |\n";
+ s << Color::Cyan << "[doctest] " << Color::None
+ << "Status: " << (p.numTestCasesFailed > 0 ? Color::Red : Color::Green)
+ << ((p.numTestCasesFailed > 0) ? "FAILURE!" : "SUCCESS!") << Color::None << std::endl;
+ }
+
+ void test_case_start(const TestCaseData& in) override {
+ hasLoggedCurrentTestStart = false;
+ tc = &in;
+ subcasesStack.clear();
+ currentSubcaseLevel = 0;
+ }
+
+ void test_case_reenter(const TestCaseData&) override {
+ subcasesStack.clear();
+ }
+
+ void test_case_end(const CurrentTestCaseStats& st) override {
+ if(tc->m_no_output)
+ return;
+
+ // log the preamble of the test case only if there is something
+ // else to print - something other than that an assert has failed
+ if(opt.duration ||
+ (st.failure_flags && st.failure_flags != TestCaseFailureReason::AssertFailure))
+ logTestStart();
+
+ if(opt.duration)
+ s << Color::None << std::setprecision(6) << std::fixed << st.seconds
+ << " s: " << tc->m_name << "\n";
+
+ if(st.failure_flags & TestCaseFailureReason::Timeout)
+ s << Color::Red << "Test case exceeded time limit of " << std::setprecision(6)
+ << std::fixed << tc->m_timeout << "!\n";
+
+ if(st.failure_flags & TestCaseFailureReason::ShouldHaveFailedButDidnt) {
+ s << Color::Red << "Should have failed but didn't! Marking it as failed!\n";
+ } else if(st.failure_flags & TestCaseFailureReason::ShouldHaveFailedAndDid) {
+ s << Color::Yellow << "Failed as expected so marking it as not failed\n";
+ } else if(st.failure_flags & TestCaseFailureReason::CouldHaveFailedAndDid) {
+ s << Color::Yellow << "Allowed to fail so marking it as not failed\n";
+ } else if(st.failure_flags & TestCaseFailureReason::DidntFailExactlyNumTimes) {
+ s << Color::Red << "Didn't fail exactly " << tc->m_expected_failures
+ << " times so marking it as failed!\n";
+ } else if(st.failure_flags & TestCaseFailureReason::FailedExactlyNumTimes) {
+ s << Color::Yellow << "Failed exactly " << tc->m_expected_failures
+ << " times as expected so marking it as not failed!\n";
+ }
+ if(st.failure_flags & TestCaseFailureReason::TooManyFailedAsserts) {
+ s << Color::Red << "Aborting - too many failed asserts!\n";
+ }
+ s << Color::None; // lgtm [cpp/useless-expression]
+ }
+
+ void test_case_exception(const TestCaseException& e) override {
+ if(tc->m_no_output)
+ return;
+
+ logTestStart();
+
+ file_line_to_stream(tc->m_file.c_str(), tc->m_line, " ");
+ successOrFailColoredStringToStream(false, e.is_crash ? assertType::is_require :
+ assertType::is_check);
+ s << Color::Red << (e.is_crash ? "test case CRASHED: " : "test case THREW exception: ")
+ << Color::Cyan << e.error_string << "\n";
+
+ int num_stringified_contexts = get_num_stringified_contexts();
+ if(num_stringified_contexts) {
+ auto stringified_contexts = get_stringified_contexts();
+ s << Color::None << " logged: ";
+ for(int i = num_stringified_contexts; i > 0; --i) {
+ s << (i == num_stringified_contexts ? "" : " ")
+ << stringified_contexts[i - 1] << "\n";
+ }
+ }
+ s << "\n" << Color::None;
+ }
+
+ void subcase_start(const SubcaseSignature& subc) override {
+ std::lock_guard<std::mutex> lock(mutex);
+ subcasesStack.push_back(subc);
+ ++currentSubcaseLevel;
+ hasLoggedCurrentTestStart = false;
+ }
+
+ void subcase_end() override {
+ std::lock_guard<std::mutex> lock(mutex);
+ --currentSubcaseLevel;
+ hasLoggedCurrentTestStart = false;
+ }
+
+ void log_assert(const AssertData& rb) override {
+ if((!rb.m_failed && !opt.success) || tc->m_no_output)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ logTestStart();
+
+ file_line_to_stream(rb.m_file, rb.m_line, " ");
+ successOrFailColoredStringToStream(!rb.m_failed, rb.m_at);
+
+ fulltext_log_assert_to_stream(s, rb);
+
+ log_contexts();
+ }
+
+ void log_message(const MessageData& mb) override {
+ if(tc->m_no_output)
+ return;
+
+ std::lock_guard<std::mutex> lock(mutex);
+
+ logTestStart();
+
+ file_line_to_stream(mb.m_file, mb.m_line, " ");
+ s << getSuccessOrFailColor(false, mb.m_severity)
+ << getSuccessOrFailString(mb.m_severity & assertType::is_warn, mb.m_severity,
+ "MESSAGE") << ": ";
+ s << Color::None << mb.m_string << "\n";
+ log_contexts();
+ }
+
+ void test_case_skipped(const TestCaseData&) override {}
+ };
+
+ DOCTEST_REGISTER_REPORTER("console", 0, ConsoleReporter);
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ struct DebugOutputWindowReporter : public ConsoleReporter
+ {
+ DOCTEST_THREAD_LOCAL static std::ostringstream oss;
+
+ DebugOutputWindowReporter(const ContextOptions& co)
+ : ConsoleReporter(co, oss) {}
+
+#define DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(func, type, arg) \
+ void func(type arg) override { \
+ bool with_col = g_no_colors; \
+ g_no_colors = false; \
+ ConsoleReporter::func(arg); \
+ if(oss.tellp() != std::streampos{}) { \
+ DOCTEST_OUTPUT_DEBUG_STRING(oss.str().c_str()); \
+ oss.str(""); \
+ } \
+ g_no_colors = with_col; \
+ }
+
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_run_start, DOCTEST_EMPTY, DOCTEST_EMPTY)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_run_end, const TestRunStats&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_start, const TestCaseData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_reenter, const TestCaseData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_end, const CurrentTestCaseStats&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_exception, const TestCaseException&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(subcase_start, const SubcaseSignature&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(subcase_end, DOCTEST_EMPTY, DOCTEST_EMPTY)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(log_assert, const AssertData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(log_message, const MessageData&, in)
+ DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_skipped, const TestCaseData&, in)
+ };
+
+ DOCTEST_THREAD_LOCAL std::ostringstream DebugOutputWindowReporter::oss;
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ // the implementation of parseOption()
+ bool parseOptionImpl(int argc, const char* const* argv, const char* pattern, String* value) {
+ // going from the end to the beginning and stopping on the first occurrence from the end
+ for(int i = argc; i > 0; --i) {
+ auto index = i - 1;
+ auto temp = std::strstr(argv[index], pattern);
+ if(temp && (value || strlen(temp) == strlen(pattern))) { //!OCLINT prefer early exits and continue
+ // eliminate matches in which the chars before the option are not '-'
+ bool noBadCharsFound = true;
+ auto curr = argv[index];
+ while(curr != temp) {
+ if(*curr++ != '-') {
+ noBadCharsFound = false;
+ break;
+ }
+ }
+ if(noBadCharsFound && argv[index][0] == '-') {
+ if(value) {
+ // parsing the value of an option
+ temp += strlen(pattern);
+ const unsigned len = strlen(temp);
+ if(len) {
+ *value = temp;
+ return true;
+ }
+ } else {
+ // just a flag - no value
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ // parses an option and returns the string after the '=' character
+ bool parseOption(int argc, const char* const* argv, const char* pattern, String* value = nullptr,
+ const String& defaultVal = String()) {
+ if(value)
+ *value = defaultVal;
+#ifndef DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ // offset (normally 3 for "dt-") to skip prefix
+ if(parseOptionImpl(argc, argv, pattern + strlen(DOCTEST_CONFIG_OPTIONS_PREFIX), value))
+ return true;
+#endif // DOCTEST_CONFIG_NO_UNPREFIXED_OPTIONS
+ return parseOptionImpl(argc, argv, pattern, value);
+ }
+
+ // locates a flag on the command line
+ bool parseFlag(int argc, const char* const* argv, const char* pattern) {
+ return parseOption(argc, argv, pattern);
+ }
+
+ // parses a comma separated list of words after a pattern in one of the arguments in argv
+ bool parseCommaSepArgs(int argc, const char* const* argv, const char* pattern,
+ std::vector<String>& res) {
+ String filtersString;
+ if(parseOption(argc, argv, pattern, &filtersString)) {
+ // tokenize with "," as a separator
+ // cppcheck-suppress strtokCalled
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations")
+ auto pch = std::strtok(filtersString.c_str(), ","); // modifies the string
+ while(pch != nullptr) {
+ if(strlen(pch))
+ res.push_back(pch);
+ // uses the strtok() internal state to go to the next token
+ // cppcheck-suppress strtokCalled
+ pch = std::strtok(nullptr, ",");
+ }
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ return true;
+ }
+ return false;
+ }
+
+ enum optionType
+ {
+ option_bool,
+ option_int
+ };
+
+ // parses an int/bool option from the command line
+ bool parseIntOption(int argc, const char* const* argv, const char* pattern, optionType type,
+ int& res) {
+ String parsedValue;
+ if(!parseOption(argc, argv, pattern, &parsedValue))
+ return false;
+
+ if(type == 0) {
+ // boolean
+ const char positive[][5] = {"1", "true", "on", "yes"}; // 5 - strlen("true") + 1
+ const char negative[][6] = {"0", "false", "off", "no"}; // 6 - strlen("false") + 1
+
+ // if the value matches any of the positive/negative possibilities
+ for(unsigned i = 0; i < 4; i++) {
+ if(parsedValue.compare(positive[i], true) == 0) {
+ res = 1; //!OCLINT parameter reassignment
+ return true;
+ }
+ if(parsedValue.compare(negative[i], true) == 0) {
+ res = 0; //!OCLINT parameter reassignment
+ return true;
+ }
+ }
+ } else {
+ // integer
+ // TODO: change this to use std::stoi or something else! currently it uses undefined behavior - assumes '0' on failed parse...
+ int theInt = std::atoi(parsedValue.c_str()); // NOLINT
+ if(theInt != 0) {
+ res = theInt; //!OCLINT parameter reassignment
+ return true;
+ }
+ }
+ return false;
+ }
+} // namespace
+
+Context::Context(int argc, const char* const* argv)
+ : p(new detail::ContextState) {
+ parseArgs(argc, argv, true);
+ if(argc)
+ p->binary_name = argv[0];
+}
+
+Context::~Context() {
+ if(g_cs == p)
+ g_cs = nullptr;
+ delete p;
+}
+
+void Context::applyCommandLine(int argc, const char* const* argv) {
+ parseArgs(argc, argv);
+ if(argc)
+ p->binary_name = argv[0];
+}
+
+// parses args
+void Context::parseArgs(int argc, const char* const* argv, bool withDefaults) {
+ using namespace detail;
+
+ // clang-format off
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "source-file=", p->filters[0]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sf=", p->filters[0]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "source-file-exclude=",p->filters[1]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sfe=", p->filters[1]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-suite=", p->filters[2]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "ts=", p->filters[2]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-suite-exclude=", p->filters[3]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tse=", p->filters[3]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-case=", p->filters[4]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tc=", p->filters[4]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "test-case-exclude=", p->filters[5]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "tce=", p->filters[5]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "subcase=", p->filters[6]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sc=", p->filters[6]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "subcase-exclude=", p->filters[7]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "sce=", p->filters[7]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "reporters=", p->filters[8]);
+ parseCommaSepArgs(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "r=", p->filters[8]);
+ // clang-format on
+
+ int intRes = 0;
+ String strRes;
+
+#define DOCTEST_PARSE_AS_BOOL_OR_FLAG(name, sname, var, default) \
+ if(parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", option_bool, intRes) || \
+ parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", option_bool, intRes)) \
+ p->var = static_cast<bool>(intRes); \
+ else if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name) || \
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname)) \
+ p->var = true; \
+ else if(withDefaults) \
+ p->var = default
+
+#define DOCTEST_PARSE_INT_OPTION(name, sname, var, default) \
+ if(parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", option_int, intRes) || \
+ parseIntOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", option_int, intRes)) \
+ p->var = intRes; \
+ else if(withDefaults) \
+ p->var = default
+
+#define DOCTEST_PARSE_STR_OPTION(name, sname, var, default) \
+ if(parseOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX name "=", &strRes, default) || \
+ parseOption(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX sname "=", &strRes, default) || \
+ withDefaults) \
+ p->var = strRes
+
+ // clang-format off
+ DOCTEST_PARSE_STR_OPTION("out", "o", out, "");
+ DOCTEST_PARSE_STR_OPTION("order-by", "ob", order_by, "file");
+ DOCTEST_PARSE_INT_OPTION("rand-seed", "rs", rand_seed, 0);
+
+ DOCTEST_PARSE_INT_OPTION("first", "f", first, 0);
+ DOCTEST_PARSE_INT_OPTION("last", "l", last, UINT_MAX);
+
+ DOCTEST_PARSE_INT_OPTION("abort-after", "aa", abort_after, 0);
+ DOCTEST_PARSE_INT_OPTION("subcase-filter-levels", "scfl", subcase_filter_levels, INT_MAX);
+
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("success", "s", success, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("case-sensitive", "cs", case_sensitive, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("exit", "e", exit, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("duration", "d", duration, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-throw", "nt", no_throw, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-exitcode", "ne", no_exitcode, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-run", "nr", no_run, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-version", "nv", no_version, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-colors", "nc", no_colors, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("force-colors", "fc", force_colors, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-breaks", "nb", no_breaks, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-skip", "ns", no_skip, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("gnu-file-line", "gfl", gnu_file_line, !bool(DOCTEST_MSVC));
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-path-filenames", "npf", no_path_in_filenames, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-line-numbers", "nln", no_line_numbers, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-debug-output", "ndo", no_debug_output, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-skipped-summary", "nss", no_skipped_summary, false);
+ DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-time-in-output", "ntio", no_time_in_output, false);
+ // clang-format on
+
+ if(withDefaults) {
+ p->help = false;
+ p->version = false;
+ p->count = false;
+ p->list_test_cases = false;
+ p->list_test_suites = false;
+ p->list_reporters = false;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "help") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "h") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "?")) {
+ p->help = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "version") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "v")) {
+ p->version = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "count") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "c")) {
+ p->count = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-test-cases") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "ltc")) {
+ p->list_test_cases = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-test-suites") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "lts")) {
+ p->list_test_suites = true;
+ p->exit = true;
+ }
+ if(parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "list-reporters") ||
+ parseFlag(argc, argv, DOCTEST_CONFIG_OPTIONS_PREFIX "lr")) {
+ p->list_reporters = true;
+ p->exit = true;
+ }
+}
+
+// allows the user to add procedurally to the filters from the command line
+void Context::addFilter(const char* filter, const char* value) { setOption(filter, value); }
+
+// allows the user to clear all filters from the command line
+void Context::clearFilters() {
+ for(auto& curr : p->filters)
+ curr.clear();
+}
+
+// allows the user to override procedurally the int/bool options from the command line
+void Context::setOption(const char* option, int value) {
+ setOption(option, toString(value).c_str());
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+}
+
+// allows the user to override procedurally the string options from the command line
+void Context::setOption(const char* option, const char* value) {
+ auto argv = String("-") + option + "=" + value;
+ auto lvalue = argv.c_str();
+ parseArgs(1, &lvalue);
+}
+
+// users should query this in their main() and exit the program if true
+bool Context::shouldExit() { return p->exit; }
+
+void Context::setAsDefaultForAssertsOutOfTestCases() { g_cs = p; }
+
+void Context::setAssertHandler(detail::assert_handler ah) { p->ah = ah; }
+
+// the main function that does all the filtering and test running
+int Context::run() {
+ using namespace detail;
+
+ // save the old context state in case such was setup - for using asserts out of a testing context
+ auto old_cs = g_cs;
+ // this is the current contest
+ g_cs = p;
+ is_running_in_test = true;
+
+ g_no_colors = p->no_colors;
+ p->resetRunData();
+
+ // stdout by default
+ p->cout = &std::cout;
+ p->cerr = &std::cerr;
+
+ // or to a file if specified
+ std::fstream fstr;
+ if(p->out.size()) {
+ fstr.open(p->out.c_str(), std::fstream::out);
+ p->cout = &fstr;
+ }
+
+ FatalConditionHandler::allocateAltStackMem();
+
+ auto cleanup_and_return = [&]() {
+ FatalConditionHandler::freeAltStackMem();
+
+ if(fstr.is_open())
+ fstr.close();
+
+ // restore context
+ g_cs = old_cs;
+ is_running_in_test = false;
+
+ // we have to free the reporters which were allocated when the run started
+ for(auto& curr : p->reporters_currently_used)
+ delete curr;
+ p->reporters_currently_used.clear();
+
+ if(p->numTestCasesFailed && !p->no_exitcode)
+ return EXIT_FAILURE;
+ return EXIT_SUCCESS;
+ };
+
+ // setup default reporter if none is given through the command line
+ if(p->filters[8].empty())
+ p->filters[8].push_back("console");
+
+ // check to see if any of the registered reporters has been selected
+ for(auto& curr : getReporters()) {
+ if(matchesAny(curr.first.second.c_str(), p->filters[8], false, p->case_sensitive))
+ p->reporters_currently_used.push_back(curr.second(*g_cs));
+ }
+
+ // TODO: check if there is nothing in reporters_currently_used
+
+ // prepend all listeners
+ for(auto& curr : getListeners())
+ p->reporters_currently_used.insert(p->reporters_currently_used.begin(), curr.second(*g_cs));
+
+#ifdef DOCTEST_PLATFORM_WINDOWS
+ if(isDebuggerActive() && p->no_debug_output == false)
+ p->reporters_currently_used.push_back(new DebugOutputWindowReporter(*g_cs));
+#endif // DOCTEST_PLATFORM_WINDOWS
+
+ // handle version, help and no_run
+ if(p->no_run || p->version || p->help || p->list_reporters) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(report_query, QueryData());
+
+ return cleanup_and_return();
+ }
+
+ std::vector<const TestCase*> testArray;
+ for(auto& curr : getRegisteredTests())
+ testArray.push_back(&curr);
+ p->numTestCases = testArray.size();
+
+ // sort the collected records
+ if(!testArray.empty()) {
+ if(p->order_by.compare("file", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), fileOrderComparator);
+ } else if(p->order_by.compare("suite", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), suiteOrderComparator);
+ } else if(p->order_by.compare("name", true) == 0) {
+ std::sort(testArray.begin(), testArray.end(), nameOrderComparator);
+ } else if(p->order_by.compare("rand", true) == 0) {
+ std::srand(p->rand_seed);
+
+ // random_shuffle implementation
+ const auto first = &testArray[0];
+ for(size_t i = testArray.size() - 1; i > 0; --i) {
+ int idxToSwap = std::rand() % (i + 1); // NOLINT
+
+ const auto temp = first[i];
+
+ first[i] = first[idxToSwap];
+ first[idxToSwap] = temp;
+ }
+ } else if(p->order_by.compare("none", true) == 0) {
+ // means no sorting - beneficial for death tests which call into the executable
+ // with a specific test case in mind - we don't want to slow down the startup times
+ }
+ }
+
+ std::set<String> testSuitesPassingFilt;
+
+ bool query_mode = p->count || p->list_test_cases || p->list_test_suites;
+ std::vector<const TestCaseData*> queryResults;
+
+ if(!query_mode)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_start, DOCTEST_EMPTY);
+
+ // invoke the registered functions if they match the filter criteria (or just count them)
+ for(auto& curr : testArray) {
+ const auto& tc = *curr;
+
+ bool skip_me = false;
+ if(tc.m_skip && !p->no_skip)
+ skip_me = true;
+
+ if(!matchesAny(tc.m_file.c_str(), p->filters[0], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_file.c_str(), p->filters[1], false, p->case_sensitive))
+ skip_me = true;
+ if(!matchesAny(tc.m_test_suite, p->filters[2], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_test_suite, p->filters[3], false, p->case_sensitive))
+ skip_me = true;
+ if(!matchesAny(tc.m_name, p->filters[4], true, p->case_sensitive))
+ skip_me = true;
+ if(matchesAny(tc.m_name, p->filters[5], false, p->case_sensitive))
+ skip_me = true;
+
+ if(!skip_me)
+ p->numTestCasesPassingFilters++;
+
+ // skip the test if it is not in the execution range
+ if((p->last < p->numTestCasesPassingFilters && p->first <= p->last) ||
+ (p->first > p->numTestCasesPassingFilters))
+ skip_me = true;
+
+ if(skip_me) {
+ if(!query_mode)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_skipped, tc);
+ continue;
+ }
+
+ // do not execute the test if we are to only count the number of filter passing tests
+ if(p->count)
+ continue;
+
+ // print the name of the test and don't execute it
+ if(p->list_test_cases) {
+ queryResults.push_back(&tc);
+ continue;
+ }
+
+ // print the name of the test suite if not done already and don't execute it
+ if(p->list_test_suites) {
+ if((testSuitesPassingFilt.count(tc.m_test_suite) == 0) && tc.m_test_suite[0] != '\0') {
+ queryResults.push_back(&tc);
+ testSuitesPassingFilt.insert(tc.m_test_suite);
+ p->numTestSuitesPassingFilters++;
+ }
+ continue;
+ }
+
+ // execute the test if it passes all the filtering
+ {
+ p->currentTest = &tc;
+
+ p->failure_flags = TestCaseFailureReason::None;
+ p->seconds = 0;
+
+ // reset atomic counters
+ p->numAssertsFailedCurrentTest_atomic = 0;
+ p->numAssertsCurrentTest_atomic = 0;
+
+ p->subcasesPassed.clear();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_start, tc);
+
+ p->timer.start();
+
+ bool run_test = true;
+
+ do {
+ // reset some of the fields for subcases (except for the set of fully passed ones)
+ p->should_reenter = false;
+ p->subcasesCurrentMaxLevel = 0;
+ p->subcasesStack.clear();
+
+ p->shouldLogCurrentException = true;
+
+ // reset stuff for logging with INFO()
+ p->stringifiedContexts.clear();
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ try {
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+// MSVC 2015 diagnoses fatalConditionHandler as unused (because reset() is a static method)
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4101) // unreferenced local variable
+ FatalConditionHandler fatalConditionHandler; // Handle signals
+ // execute the test
+ tc.m_test();
+ fatalConditionHandler.reset();
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ } catch(const TestFailureException&) {
+ p->failure_flags |= TestCaseFailureReason::AssertFailure;
+ } catch(...) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_exception,
+ {translateActiveException(), false});
+ p->failure_flags |= TestCaseFailureReason::Exception;
+ }
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+ // exit this loop if enough assertions have failed - even if there are more subcases
+ if(p->abort_after > 0 &&
+ p->numAssertsFailed + p->numAssertsFailedCurrentTest_atomic >= p->abort_after) {
+ run_test = false;
+ p->failure_flags |= TestCaseFailureReason::TooManyFailedAsserts;
+ }
+
+ if(p->should_reenter && run_test)
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_reenter, tc);
+ if(!p->should_reenter)
+ run_test = false;
+ } while(run_test);
+
+ p->finalizeTestCaseData();
+
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_end, *g_cs);
+
+ p->currentTest = nullptr;
+
+ // stop executing tests if enough assertions have failed
+ if(p->abort_after > 0 && p->numAssertsFailed >= p->abort_after)
+ break;
+ }
+ }
+
+ if(!query_mode) {
+ DOCTEST_ITERATE_THROUGH_REPORTERS(test_run_end, *g_cs);
+ } else {
+ QueryData qdata;
+ qdata.run_stats = g_cs;
+ qdata.data = queryResults.data();
+ qdata.num_data = unsigned(queryResults.size());
+ DOCTEST_ITERATE_THROUGH_REPORTERS(report_query, qdata);
+ }
+
+ // see these issues on the reasoning for this:
+ // - https://github.com/onqtam/doctest/issues/143#issuecomment-414418903
+ // - https://github.com/onqtam/doctest/issues/126
+ auto DOCTEST_FIX_FOR_MACOS_LIBCPP_IOSFWD_STRING_LINK_ERRORS = []() DOCTEST_NOINLINE
+ { std::cout << std::string(); };
+ DOCTEST_FIX_FOR_MACOS_LIBCPP_IOSFWD_STRING_LINK_ERRORS();
+
+ return cleanup_and_return();
+}
+
+IReporter::~IReporter() = default;
+
+int IReporter::get_num_active_contexts() { return detail::g_infoContexts.size(); }
+const IContextScope* const* IReporter::get_active_contexts() {
+ return get_num_active_contexts() ? &detail::g_infoContexts[0] : nullptr;
+}
+
+int IReporter::get_num_stringified_contexts() { return detail::g_cs->stringifiedContexts.size(); }
+const String* IReporter::get_stringified_contexts() {
+ return get_num_stringified_contexts() ? &detail::g_cs->stringifiedContexts[0] : nullptr;
+}
+
+namespace detail {
+ void registerReporterImpl(const char* name, int priority, reporterCreatorFunc c, bool isReporter) {
+ if(isReporter)
+ getReporters().insert(reporterMap::value_type(reporterMap::key_type(priority, name), c));
+ else
+ getListeners().insert(reporterMap::value_type(reporterMap::key_type(priority, name), c));
+ }
+} // namespace detail
+
+} // namespace doctest
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+#ifdef DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4007) // 'function' : must be 'attribute' - see issue #182
+int main(int argc, char** argv) { return doctest::Context(argc, argv).run(); }
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#endif // DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_LIBRARY_IMPLEMENTATION
+#endif // DOCTEST_CONFIG_IMPLEMENT
diff --git a/contrib/doctest/doctest/parts/doctest_fwd.h b/contrib/doctest/doctest/parts/doctest_fwd.h
new file mode 100644
index 0000000..e83b0ca
--- /dev/null
+++ b/contrib/doctest/doctest/parts/doctest_fwd.h
@@ -0,0 +1,2706 @@
+//
+// doctest.h - the lightest feature-rich C++ single-header testing framework for unit tests and TDD
+//
+// Copyright (c) 2016-2021 Viktor Kirilov
+//
+// Distributed under the MIT Software License
+// See accompanying file LICENSE.txt or copy at
+// https://opensource.org/licenses/MIT
+//
+// The documentation can be found at the library's page:
+// https://github.com/onqtam/doctest/blob/master/doc/markdown/readme.md
+//
+// =================================================================================================
+// =================================================================================================
+// =================================================================================================
+//
+// The library is heavily influenced by Catch - https://github.com/catchorg/Catch2
+// which uses the Boost Software License - Version 1.0
+// see here - https://github.com/catchorg/Catch2/blob/master/LICENSE.txt
+//
+// The concept of subcases (sections in Catch) and expression decomposition are from there.
+// Some parts of the code are taken directly:
+// - stringification - the detection of "ostream& operator<<(ostream&, const T&)" and StringMaker<>
+// - the Approx() helper class for floating point comparison
+// - colors in the console
+// - breaking into a debugger
+// - signal / SEH handling
+// - timer
+// - XmlWriter class - thanks to Phil Nash for allowing the direct reuse (AKA copy/paste)
+//
+// The expression decomposing templates are taken from lest - https://github.com/martinmoene/lest
+// which uses the Boost Software License - Version 1.0
+// see here - https://github.com/martinmoene/lest/blob/master/LICENSE.txt
+//
+// =================================================================================================
+// =================================================================================================
+// =================================================================================================
+
+#ifndef DOCTEST_LIBRARY_INCLUDED
+#define DOCTEST_LIBRARY_INCLUDED
+
+// =================================================================================================
+// == VERSION ======================================================================================
+// =================================================================================================
+
+#define DOCTEST_VERSION_MAJOR 2
+#define DOCTEST_VERSION_MINOR 4
+#define DOCTEST_VERSION_PATCH 6
+#define DOCTEST_VERSION_STR "2.4.6"
+
+#define DOCTEST_VERSION \
+ (DOCTEST_VERSION_MAJOR * 10000 + DOCTEST_VERSION_MINOR * 100 + DOCTEST_VERSION_PATCH)
+
+// =================================================================================================
+// == COMPILER VERSION =============================================================================
+// =================================================================================================
+
+// ideas for the version stuff are taken from here: https://github.com/cxxstuff/cxx_detect
+
+#define DOCTEST_COMPILER(MAJOR, MINOR, PATCH) ((MAJOR)*10000000 + (MINOR)*100000 + (PATCH))
+
+// GCC/Clang and GCC/MSVC are mutually exclusive, but Clang/MSVC are not because of clang-cl...
+#if defined(_MSC_VER) && defined(_MSC_FULL_VER)
+#if _MSC_VER == _MSC_FULL_VER / 10000
+#define DOCTEST_MSVC DOCTEST_COMPILER(_MSC_VER / 100, _MSC_VER % 100, _MSC_FULL_VER % 10000)
+#else // MSVC
+#define DOCTEST_MSVC \
+ DOCTEST_COMPILER(_MSC_VER / 100, (_MSC_FULL_VER / 100000) % 100, _MSC_FULL_VER % 100000)
+#endif // MSVC
+#endif // MSVC
+#if defined(__clang__) && defined(__clang_minor__)
+#define DOCTEST_CLANG DOCTEST_COMPILER(__clang_major__, __clang_minor__, __clang_patchlevel__)
+#elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && \
+ !defined(__INTEL_COMPILER)
+#define DOCTEST_GCC DOCTEST_COMPILER(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__)
+#endif // GCC
+
+#ifndef DOCTEST_MSVC
+#define DOCTEST_MSVC 0
+#endif // DOCTEST_MSVC
+#ifndef DOCTEST_CLANG
+#define DOCTEST_CLANG 0
+#endif // DOCTEST_CLANG
+#ifndef DOCTEST_GCC
+#define DOCTEST_GCC 0
+#endif // DOCTEST_GCC
+
+// =================================================================================================
+// == COMPILER WARNINGS HELPERS ====================================================================
+// =================================================================================================
+
+#if DOCTEST_CLANG
+#define DOCTEST_PRAGMA_TO_STR(x) _Pragma(#x)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_PUSH _Pragma("clang diagnostic push")
+#define DOCTEST_CLANG_SUPPRESS_WARNING(w) DOCTEST_PRAGMA_TO_STR(clang diagnostic ignored w)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_POP _Pragma("clang diagnostic pop")
+#define DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_PUSH DOCTEST_CLANG_SUPPRESS_WARNING(w)
+#else // DOCTEST_CLANG
+#define DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+#define DOCTEST_CLANG_SUPPRESS_WARNING(w)
+#define DOCTEST_CLANG_SUPPRESS_WARNING_POP
+#define DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_CLANG
+
+#if DOCTEST_GCC
+#define DOCTEST_PRAGMA_TO_STR(x) _Pragma(#x)
+#define DOCTEST_GCC_SUPPRESS_WARNING_PUSH _Pragma("GCC diagnostic push")
+#define DOCTEST_GCC_SUPPRESS_WARNING(w) DOCTEST_PRAGMA_TO_STR(GCC diagnostic ignored w)
+#define DOCTEST_GCC_SUPPRESS_WARNING_POP _Pragma("GCC diagnostic pop")
+#define DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_GCC_SUPPRESS_WARNING_PUSH DOCTEST_GCC_SUPPRESS_WARNING(w)
+#else // DOCTEST_GCC
+#define DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+#define DOCTEST_GCC_SUPPRESS_WARNING(w)
+#define DOCTEST_GCC_SUPPRESS_WARNING_POP
+#define DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_GCC
+
+#if DOCTEST_MSVC
+#define DOCTEST_MSVC_SUPPRESS_WARNING_PUSH __pragma(warning(push))
+#define DOCTEST_MSVC_SUPPRESS_WARNING(w) __pragma(warning(disable : w))
+#define DOCTEST_MSVC_SUPPRESS_WARNING_POP __pragma(warning(pop))
+#define DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(w) \
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH DOCTEST_MSVC_SUPPRESS_WARNING(w)
+#else // DOCTEST_MSVC
+#define DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+#define DOCTEST_MSVC_SUPPRESS_WARNING(w)
+#define DOCTEST_MSVC_SUPPRESS_WARNING_POP
+#define DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(w)
+#endif // DOCTEST_MSVC
+
+// =================================================================================================
+// == COMPILER WARNINGS ============================================================================
+// =================================================================================================
+
+DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wnon-virtual-dtor")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wweak-vtables")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wpadded")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wdeprecated")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-prototypes")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-local-typedef")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat-pedantic")
+
+DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunknown-pragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wpragmas")
+DOCTEST_GCC_SUPPRESS_WARNING("-Weffc++")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-overflow")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-aliasing")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wctor-dtor-privacy")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-declarations")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnon-virtual-dtor")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-local-typedefs")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wuseless-cast")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept")
+DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-promo")
+
+DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+DOCTEST_MSVC_SUPPRESS_WARNING(4616) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4619) // invalid compiler warning
+DOCTEST_MSVC_SUPPRESS_WARNING(4996) // The compiler encountered a deprecated declaration
+DOCTEST_MSVC_SUPPRESS_WARNING(4706) // assignment within conditional expression
+DOCTEST_MSVC_SUPPRESS_WARNING(4512) // 'class' : assignment operator could not be generated
+DOCTEST_MSVC_SUPPRESS_WARNING(4127) // conditional expression is constant
+DOCTEST_MSVC_SUPPRESS_WARNING(4820) // padding
+DOCTEST_MSVC_SUPPRESS_WARNING(4625) // copy constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4626) // assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5027) // move assignment operator was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(5026) // move constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4623) // default constructor was implicitly defined as deleted
+DOCTEST_MSVC_SUPPRESS_WARNING(4640) // construction of local static object is not thread-safe
+// static analysis
+DOCTEST_MSVC_SUPPRESS_WARNING(26439) // This kind of function may not throw. Declare it 'noexcept'
+DOCTEST_MSVC_SUPPRESS_WARNING(26495) // Always initialize a member variable
+DOCTEST_MSVC_SUPPRESS_WARNING(26451) // Arithmetic overflow ...
+DOCTEST_MSVC_SUPPRESS_WARNING(26444) // Avoid unnamed objects with custom construction and dtr...
+DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum'
+
+// 4548 - expression before comma has no effect; expected expression with side - effect
+// 4265 - class has virtual functions, but destructor is not virtual
+// 4986 - exception specification does not match previous declaration
+// 4350 - behavior change: 'member1' called instead of 'member2'
+// 4668 - 'x' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif'
+// 4365 - conversion from 'int' to 'unsigned long', signed/unsigned mismatch
+// 4774 - format string expected in argument 'x' is not a string literal
+// 4820 - padding in structs
+
+// only 4 should be disabled globally:
+// - 4514 # unreferenced inline function has been removed
+// - 4571 # SEH related
+// - 4710 # function not inlined
+// - 4711 # function 'x' selected for automatic inline expansion
+
+#define DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN \
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4548) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4265) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4986) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4350) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4668) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4365) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4774) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4820) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4625) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4626) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5027) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5026) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(4623) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5039) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5045) \
+ DOCTEST_MSVC_SUPPRESS_WARNING(5105)
+
+#define DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+// =================================================================================================
+// == FEATURE DETECTION ============================================================================
+// =================================================================================================
+
+// general compiler feature support table: https://en.cppreference.com/w/cpp/compiler_support
+// MSVC C++11 feature support table: https://msdn.microsoft.com/en-us/library/hh567368.aspx
+// GCC C++11 feature support table: https://gcc.gnu.org/projects/cxx-status.html
+// MSVC version table:
+// https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B#Internal_version_numbering
+// MSVC++ 14.2 (16) _MSC_VER == 1920 (Visual Studio 2019)
+// MSVC++ 14.1 (15) _MSC_VER == 1910 (Visual Studio 2017)
+// MSVC++ 14.0 _MSC_VER == 1900 (Visual Studio 2015)
+// MSVC++ 12.0 _MSC_VER == 1800 (Visual Studio 2013)
+// MSVC++ 11.0 _MSC_VER == 1700 (Visual Studio 2012)
+// MSVC++ 10.0 _MSC_VER == 1600 (Visual Studio 2010)
+// MSVC++ 9.0 _MSC_VER == 1500 (Visual Studio 2008)
+// MSVC++ 8.0 _MSC_VER == 1400 (Visual Studio 2005)
+
+#if DOCTEST_MSVC && !defined(DOCTEST_CONFIG_WINDOWS_SEH)
+#define DOCTEST_CONFIG_WINDOWS_SEH
+#endif // MSVC
+#if defined(DOCTEST_CONFIG_NO_WINDOWS_SEH) && defined(DOCTEST_CONFIG_WINDOWS_SEH)
+#undef DOCTEST_CONFIG_WINDOWS_SEH
+#endif // DOCTEST_CONFIG_NO_WINDOWS_SEH
+
+#if !defined(_WIN32) && !defined(__QNX__) && !defined(DOCTEST_CONFIG_POSIX_SIGNALS) && \
+ !defined(__EMSCRIPTEN__)
+#define DOCTEST_CONFIG_POSIX_SIGNALS
+#endif // _WIN32
+#if defined(DOCTEST_CONFIG_NO_POSIX_SIGNALS) && defined(DOCTEST_CONFIG_POSIX_SIGNALS)
+#undef DOCTEST_CONFIG_POSIX_SIGNALS
+#endif // DOCTEST_CONFIG_NO_POSIX_SIGNALS
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND)
+#define DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // no exceptions
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+#define DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#if defined(DOCTEST_CONFIG_NO_EXCEPTIONS) && !defined(DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS)
+#define DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS && !DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+
+#if defined(DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN) && !defined(DOCTEST_CONFIG_IMPLEMENT)
+#define DOCTEST_CONFIG_IMPLEMENT
+#endif // DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#if DOCTEST_MSVC
+#define DOCTEST_SYMBOL_EXPORT __declspec(dllexport)
+#define DOCTEST_SYMBOL_IMPORT __declspec(dllimport)
+#else // MSVC
+#define DOCTEST_SYMBOL_EXPORT __attribute__((dllexport))
+#define DOCTEST_SYMBOL_IMPORT __attribute__((dllimport))
+#endif // MSVC
+#else // _WIN32
+#define DOCTEST_SYMBOL_EXPORT __attribute__((visibility("default")))
+#define DOCTEST_SYMBOL_IMPORT
+#endif // _WIN32
+
+#ifdef DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#ifdef DOCTEST_CONFIG_IMPLEMENT
+#define DOCTEST_INTERFACE DOCTEST_SYMBOL_EXPORT
+#else // DOCTEST_CONFIG_IMPLEMENT
+#define DOCTEST_INTERFACE DOCTEST_SYMBOL_IMPORT
+#endif // DOCTEST_CONFIG_IMPLEMENT
+#else // DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#define DOCTEST_INTERFACE
+#endif // DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+
+#define DOCTEST_EMPTY
+
+#if DOCTEST_MSVC
+#define DOCTEST_NOINLINE __declspec(noinline)
+#define DOCTEST_UNUSED
+#define DOCTEST_ALIGNMENT(x)
+#elif DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 5, 0)
+#define DOCTEST_NOINLINE
+#define DOCTEST_UNUSED
+#define DOCTEST_ALIGNMENT(x)
+#else
+#define DOCTEST_NOINLINE __attribute__((noinline))
+#define DOCTEST_UNUSED __attribute__((unused))
+#define DOCTEST_ALIGNMENT(x) __attribute__((aligned(x)))
+#endif
+
+#ifndef DOCTEST_NORETURN
+#define DOCTEST_NORETURN [[noreturn]]
+#endif // DOCTEST_NORETURN
+
+#ifndef DOCTEST_NOEXCEPT
+#define DOCTEST_NOEXCEPT noexcept
+#endif // DOCTEST_NOEXCEPT
+
+// =================================================================================================
+// == FEATURE DETECTION END ========================================================================
+// =================================================================================================
+
+// internal macros for string concatenation and anonymous variable name generation
+#define DOCTEST_CAT_IMPL(s1, s2) s1##s2
+#define DOCTEST_CAT(s1, s2) DOCTEST_CAT_IMPL(s1, s2)
+#ifdef __COUNTER__ // not standard and may be missing for some compilers
+#define DOCTEST_ANONYMOUS(x) DOCTEST_CAT(x, __COUNTER__)
+#else // __COUNTER__
+#define DOCTEST_ANONYMOUS(x) DOCTEST_CAT(x, __LINE__)
+#endif // __COUNTER__
+
+#define DOCTEST_TOSTR(x) #x
+
+#ifndef DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+#define DOCTEST_REF_WRAP(x) x&
+#else // DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+#define DOCTEST_REF_WRAP(x) x
+#endif // DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE
+
+// not using __APPLE__ because... this is how Catch does it
+#ifdef __MAC_OS_X_VERSION_MIN_REQUIRED
+#define DOCTEST_PLATFORM_MAC
+#elif defined(__IPHONE_OS_VERSION_MIN_REQUIRED)
+#define DOCTEST_PLATFORM_IPHONE
+#elif defined(_WIN32)
+#define DOCTEST_PLATFORM_WINDOWS
+#else // DOCTEST_PLATFORM
+#define DOCTEST_PLATFORM_LINUX
+#endif // DOCTEST_PLATFORM
+
+#define DOCTEST_GLOBAL_NO_WARNINGS(var) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wglobal-constructors") \
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-variable") \
+ static const int var DOCTEST_UNUSED // NOLINT(fuchsia-statically-constructed-objects,cert-err58-cpp)
+#define DOCTEST_GLOBAL_NO_WARNINGS_END() DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#ifndef DOCTEST_BREAK_INTO_DEBUGGER
+// should probably take a look at https://github.com/scottt/debugbreak
+#ifdef DOCTEST_PLATFORM_LINUX
+#if defined(__GNUC__) && (defined(__i386) || defined(__x86_64))
+// Break at the location of the failing check if possible
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT (hicpp-no-assembler)
+#else
+#include <signal.h>
+#define DOCTEST_BREAK_INTO_DEBUGGER() raise(SIGTRAP)
+#endif
+#elif defined(DOCTEST_PLATFORM_MAC)
+#if defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__i386)
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT (hicpp-no-assembler)
+#else
+#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("brk #0"); // NOLINT (hicpp-no-assembler)
+#endif
+#elif DOCTEST_MSVC
+#define DOCTEST_BREAK_INTO_DEBUGGER() __debugbreak()
+#elif defined(__MINGW32__)
+DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wredundant-decls")
+extern "C" __declspec(dllimport) void __stdcall DebugBreak();
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+#define DOCTEST_BREAK_INTO_DEBUGGER() ::DebugBreak()
+#else // linux
+#define DOCTEST_BREAK_INTO_DEBUGGER() (static_cast<void>(0))
+#endif // linux
+#endif // DOCTEST_BREAK_INTO_DEBUGGER
+
+// this is kept here for backwards compatibility since the config option was changed
+#ifdef DOCTEST_CONFIG_USE_IOSFWD
+#define DOCTEST_CONFIG_USE_STD_HEADERS
+#endif // DOCTEST_CONFIG_USE_IOSFWD
+
+#ifdef DOCTEST_CONFIG_USE_STD_HEADERS
+#ifndef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#define DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#include <iosfwd>
+#include <cstddef>
+#include <ostream>
+#else // DOCTEST_CONFIG_USE_STD_HEADERS
+
+#if DOCTEST_CLANG
+// to detect if libc++ is being used with clang (the _LIBCPP_VERSION identifier)
+#include <ciso646>
+#endif // clang
+
+#ifdef _LIBCPP_VERSION
+#define DOCTEST_STD_NAMESPACE_BEGIN _LIBCPP_BEGIN_NAMESPACE_STD
+#define DOCTEST_STD_NAMESPACE_END _LIBCPP_END_NAMESPACE_STD
+#else // _LIBCPP_VERSION
+#define DOCTEST_STD_NAMESPACE_BEGIN namespace std {
+#define DOCTEST_STD_NAMESPACE_END }
+#endif // _LIBCPP_VERSION
+
+// Forward declaring 'X' in namespace std is not permitted by the C++ Standard.
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4643)
+
+DOCTEST_STD_NAMESPACE_BEGIN // NOLINT (cert-dcl58-cpp)
+typedef decltype(nullptr) nullptr_t;
+template <class charT>
+struct char_traits;
+template <>
+struct char_traits<char>;
+template <class charT, class traits>
+class basic_ostream;
+typedef basic_ostream<char, char_traits<char>> ostream;
+template <class... Types>
+class tuple;
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+template <class _Ty>
+class allocator;
+template <class _Elem, class _Traits, class _Alloc>
+class basic_string;
+using string = basic_string<char, char_traits<char>, allocator<char>>;
+#endif // VS 2019
+DOCTEST_STD_NAMESPACE_END
+
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_USE_STD_HEADERS
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#include <type_traits>
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+namespace doctest {
+
+DOCTEST_INTERFACE extern bool is_running_in_test;
+
+// A 24 byte string class (can be as small as 17 for x64 and 13 for x86) that can hold strings with length
+// of up to 23 chars on the stack before going on the heap - the last byte of the buffer is used for:
+// - "is small" bit - the highest bit - if "0" then it is small - otherwise its "1" (128)
+// - if small - capacity left before going on the heap - using the lowest 5 bits
+// - if small - 2 bits are left unused - the second and third highest ones
+// - if small - acts as a null terminator if strlen() is 23 (24 including the null terminator)
+// and the "is small" bit remains "0" ("as well as the capacity left") so its OK
+// Idea taken from this lecture about the string implementation of facebook/folly - fbstring
+// https://www.youtube.com/watch?v=kPR8h4-qZdk
+// TODO:
+// - optimizations - like not deleting memory unnecessarily in operator= and etc.
+// - resize/reserve/clear
+// - substr
+// - replace
+// - back/front
+// - iterator stuff
+// - find & friends
+// - push_back/pop_back
+// - assign/insert/erase
+// - relational operators as free functions - taking const char* as one of the params
+class DOCTEST_INTERFACE String
+{
+ static const unsigned len = 24; //!OCLINT avoid private static members
+ static const unsigned last = len - 1; //!OCLINT avoid private static members
+
+ struct view // len should be more than sizeof(view) - because of the final byte for flags
+ {
+ char* ptr;
+ unsigned size;
+ unsigned capacity;
+ };
+
+ union
+ {
+ char buf[len];
+ view data;
+ };
+
+ bool isOnStack() const { return (buf[last] & 128) == 0; }
+ void setOnHeap();
+ void setLast(unsigned in = last);
+
+ void copy(const String& other);
+
+public:
+ String();
+ ~String();
+
+ // cppcheck-suppress noExplicitConstructor
+ String(const char* in);
+ String(const char* in, unsigned in_size);
+
+ String(const String& other);
+ String& operator=(const String& other);
+
+ String& operator+=(const String& other);
+ String operator+(const String& other) const;
+
+ String(String&& other);
+ String& operator=(String&& other);
+
+ char operator[](unsigned i) const;
+ char& operator[](unsigned i);
+
+ // the only functions I'm willing to leave in the interface - available for inlining
+ const char* c_str() const { return const_cast<String*>(this)->c_str(); } // NOLINT
+ char* c_str() {
+ if(isOnStack())
+ return reinterpret_cast<char*>(buf);
+ return data.ptr;
+ }
+
+ unsigned size() const;
+ unsigned capacity() const;
+
+ int compare(const char* other, bool no_case = false) const;
+ int compare(const String& other, bool no_case = false) const;
+};
+
+DOCTEST_INTERFACE bool operator==(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator!=(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator<(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator>(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator<=(const String& lhs, const String& rhs);
+DOCTEST_INTERFACE bool operator>=(const String& lhs, const String& rhs);
+
+DOCTEST_INTERFACE std::ostream& operator<<(std::ostream& s, const String& in);
+
+namespace Color {
+ enum Enum
+ {
+ None = 0,
+ White,
+ Red,
+ Green,
+ Blue,
+ Cyan,
+ Yellow,
+ Grey,
+
+ Bright = 0x10,
+
+ BrightRed = Bright | Red,
+ BrightGreen = Bright | Green,
+ LightGrey = Bright | Grey,
+ BrightWhite = Bright | White
+ };
+
+ DOCTEST_INTERFACE std::ostream& operator<<(std::ostream& s, Color::Enum code);
+} // namespace Color
+
+namespace assertType {
+ enum Enum
+ {
+ // macro traits
+
+ is_warn = 1,
+ is_check = 2 * is_warn,
+ is_require = 2 * is_check,
+
+ is_normal = 2 * is_require,
+ is_throws = 2 * is_normal,
+ is_throws_as = 2 * is_throws,
+ is_throws_with = 2 * is_throws_as,
+ is_nothrow = 2 * is_throws_with,
+
+ is_false = 2 * is_nothrow,
+ is_unary = 2 * is_false, // not checked anywhere - used just to distinguish the types
+
+ is_eq = 2 * is_unary,
+ is_ne = 2 * is_eq,
+
+ is_lt = 2 * is_ne,
+ is_gt = 2 * is_lt,
+
+ is_ge = 2 * is_gt,
+ is_le = 2 * is_ge,
+
+ // macro types
+
+ DT_WARN = is_normal | is_warn,
+ DT_CHECK = is_normal | is_check,
+ DT_REQUIRE = is_normal | is_require,
+
+ DT_WARN_FALSE = is_normal | is_false | is_warn,
+ DT_CHECK_FALSE = is_normal | is_false | is_check,
+ DT_REQUIRE_FALSE = is_normal | is_false | is_require,
+
+ DT_WARN_THROWS = is_throws | is_warn,
+ DT_CHECK_THROWS = is_throws | is_check,
+ DT_REQUIRE_THROWS = is_throws | is_require,
+
+ DT_WARN_THROWS_AS = is_throws_as | is_warn,
+ DT_CHECK_THROWS_AS = is_throws_as | is_check,
+ DT_REQUIRE_THROWS_AS = is_throws_as | is_require,
+
+ DT_WARN_THROWS_WITH = is_throws_with | is_warn,
+ DT_CHECK_THROWS_WITH = is_throws_with | is_check,
+ DT_REQUIRE_THROWS_WITH = is_throws_with | is_require,
+
+ DT_WARN_THROWS_WITH_AS = is_throws_with | is_throws_as | is_warn,
+ DT_CHECK_THROWS_WITH_AS = is_throws_with | is_throws_as | is_check,
+ DT_REQUIRE_THROWS_WITH_AS = is_throws_with | is_throws_as | is_require,
+
+ DT_WARN_NOTHROW = is_nothrow | is_warn,
+ DT_CHECK_NOTHROW = is_nothrow | is_check,
+ DT_REQUIRE_NOTHROW = is_nothrow | is_require,
+
+ DT_WARN_EQ = is_normal | is_eq | is_warn,
+ DT_CHECK_EQ = is_normal | is_eq | is_check,
+ DT_REQUIRE_EQ = is_normal | is_eq | is_require,
+
+ DT_WARN_NE = is_normal | is_ne | is_warn,
+ DT_CHECK_NE = is_normal | is_ne | is_check,
+ DT_REQUIRE_NE = is_normal | is_ne | is_require,
+
+ DT_WARN_GT = is_normal | is_gt | is_warn,
+ DT_CHECK_GT = is_normal | is_gt | is_check,
+ DT_REQUIRE_GT = is_normal | is_gt | is_require,
+
+ DT_WARN_LT = is_normal | is_lt | is_warn,
+ DT_CHECK_LT = is_normal | is_lt | is_check,
+ DT_REQUIRE_LT = is_normal | is_lt | is_require,
+
+ DT_WARN_GE = is_normal | is_ge | is_warn,
+ DT_CHECK_GE = is_normal | is_ge | is_check,
+ DT_REQUIRE_GE = is_normal | is_ge | is_require,
+
+ DT_WARN_LE = is_normal | is_le | is_warn,
+ DT_CHECK_LE = is_normal | is_le | is_check,
+ DT_REQUIRE_LE = is_normal | is_le | is_require,
+
+ DT_WARN_UNARY = is_normal | is_unary | is_warn,
+ DT_CHECK_UNARY = is_normal | is_unary | is_check,
+ DT_REQUIRE_UNARY = is_normal | is_unary | is_require,
+
+ DT_WARN_UNARY_FALSE = is_normal | is_false | is_unary | is_warn,
+ DT_CHECK_UNARY_FALSE = is_normal | is_false | is_unary | is_check,
+ DT_REQUIRE_UNARY_FALSE = is_normal | is_false | is_unary | is_require,
+ };
+} // namespace assertType
+
+DOCTEST_INTERFACE const char* assertString(assertType::Enum at);
+DOCTEST_INTERFACE const char* failureString(assertType::Enum at);
+DOCTEST_INTERFACE const char* skipPathFromFilename(const char* file);
+
+struct DOCTEST_INTERFACE TestCaseData
+{
+ String m_file; // the file in which the test was registered (using String - see #350)
+ unsigned m_line; // the line where the test was registered
+ const char* m_name; // name of the test case
+ const char* m_test_suite; // the test suite in which the test was added
+ const char* m_description;
+ bool m_skip;
+ bool m_no_breaks;
+ bool m_no_output;
+ bool m_may_fail;
+ bool m_should_fail;
+ int m_expected_failures;
+ double m_timeout;
+};
+
+struct DOCTEST_INTERFACE AssertData
+{
+ // common - for all asserts
+ const TestCaseData* m_test_case;
+ assertType::Enum m_at;
+ const char* m_file;
+ int m_line;
+ const char* m_expr;
+ bool m_failed;
+
+ // exception-related - for all asserts
+ bool m_threw;
+ String m_exception;
+
+ // for normal asserts
+ String m_decomp;
+
+ // for specific exception-related asserts
+ bool m_threw_as;
+ const char* m_exception_type;
+ const char* m_exception_string;
+};
+
+struct DOCTEST_INTERFACE MessageData
+{
+ String m_string;
+ const char* m_file;
+ int m_line;
+ assertType::Enum m_severity;
+};
+
+struct DOCTEST_INTERFACE SubcaseSignature
+{
+ String m_name;
+ const char* m_file;
+ int m_line;
+
+ bool operator<(const SubcaseSignature& other) const;
+};
+
+struct DOCTEST_INTERFACE IContextScope
+{
+ IContextScope();
+ virtual ~IContextScope();
+ virtual void stringify(std::ostream*) const = 0;
+};
+
+namespace detail {
+ struct DOCTEST_INTERFACE TestCase;
+} // namespace detail
+
+struct ContextOptions //!OCLINT too many fields
+{
+ std::ostream* cout; // stdout stream - std::cout by default
+ std::ostream* cerr; // stderr stream - std::cerr by default
+ String binary_name; // the test binary name
+
+ const detail::TestCase* currentTest = nullptr;
+
+ // == parameters from the command line
+ String out; // output filename
+ String order_by; // how tests should be ordered
+ unsigned rand_seed; // the seed for rand ordering
+
+ unsigned first; // the first (matching) test to be executed
+ unsigned last; // the last (matching) test to be executed
+
+ int abort_after; // stop tests after this many failed assertions
+ int subcase_filter_levels; // apply the subcase filters for the first N levels
+
+ bool success; // include successful assertions in output
+ bool case_sensitive; // if filtering should be case sensitive
+ bool exit; // if the program should be exited after the tests are ran/whatever
+ bool duration; // print the time duration of each test case
+ bool no_throw; // to skip exceptions-related assertion macros
+ bool no_exitcode; // if the framework should return 0 as the exitcode
+ bool no_run; // to not run the tests at all (can be done with an "*" exclude)
+ bool no_version; // to not print the version of the framework
+ bool no_colors; // if output to the console should be colorized
+ bool force_colors; // forces the use of colors even when a tty cannot be detected
+ bool no_breaks; // to not break into the debugger
+ bool no_skip; // don't skip test cases which are marked to be skipped
+ bool gnu_file_line; // if line numbers should be surrounded with :x: and not (x):
+ bool no_path_in_filenames; // if the path to files should be removed from the output
+ bool no_line_numbers; // if source code line numbers should be omitted from the output
+ bool no_debug_output; // no output in the debug console when a debugger is attached
+ bool no_skipped_summary; // don't print "skipped" in the summary !!! UNDOCUMENTED !!!
+ bool no_time_in_output; // omit any time/timestamps from output !!! UNDOCUMENTED !!!
+
+ bool help; // to print the help
+ bool version; // to print the version
+ bool count; // if only the count of matching tests is to be retrieved
+ bool list_test_cases; // to list all tests matching the filters
+ bool list_test_suites; // to list all suites matching the filters
+ bool list_reporters; // lists all registered reporters
+};
+
+namespace detail {
+ template <bool CONDITION, typename TYPE = void>
+ struct enable_if
+ {};
+
+ template <typename TYPE>
+ struct enable_if<true, TYPE>
+ { typedef TYPE type; };
+
+ // clang-format off
+ template<class T> struct remove_reference { typedef T type; };
+ template<class T> struct remove_reference<T&> { typedef T type; };
+ template<class T> struct remove_reference<T&&> { typedef T type; };
+
+ template<typename T, typename U = T&&> U declval(int);
+
+ template<typename T> T declval(long);
+
+ template<typename T> auto declval() DOCTEST_NOEXCEPT -> decltype(declval<T>(0)) ;
+
+ template<class T> struct is_lvalue_reference { const static bool value=false; };
+ template<class T> struct is_lvalue_reference<T&> { const static bool value=true; };
+
+ template <class T>
+ inline T&& forward(typename remove_reference<T>::type& t) DOCTEST_NOEXCEPT
+ {
+ return static_cast<T&&>(t);
+ }
+
+ template <class T>
+ inline T&& forward(typename remove_reference<T>::type&& t) DOCTEST_NOEXCEPT
+ {
+ static_assert(!is_lvalue_reference<T>::value,
+ "Can not forward an rvalue as an lvalue.");
+ return static_cast<T&&>(t);
+ }
+
+ template<class T> struct remove_const { typedef T type; };
+ template<class T> struct remove_const<const T> { typedef T type; };
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template<class T> struct is_enum : public std::is_enum<T> {};
+ template<class T> struct underlying_type : public std::underlying_type<T> {};
+#else
+ // Use compiler intrinsics
+ template<class T> struct is_enum { constexpr static bool value = __is_enum(T); };
+ template<class T> struct underlying_type { typedef __underlying_type(T) type; };
+#endif
+ // clang-format on
+
+ template <typename T>
+ struct deferred_false
+ // cppcheck-suppress unusedStructMember
+ { static const bool value = false; };
+
+ namespace has_insertion_operator_impl {
+ std::ostream &os();
+ template<class T>
+ DOCTEST_REF_WRAP(T) val();
+
+ template<class, class = void>
+ struct check {
+ static constexpr bool value = false;
+ };
+
+ template<class T>
+ struct check<T, decltype(os() << val<T>(), void())> {
+ static constexpr bool value = true;
+ };
+ } // namespace has_insertion_operator_impl
+
+ template<class T>
+ using has_insertion_operator = has_insertion_operator_impl::check<const T>;
+
+ DOCTEST_INTERFACE void my_memcpy(void* dest, const void* src, unsigned num);
+
+ DOCTEST_INTERFACE std::ostream* getTlsOss(); // returns a thread-local ostringstream
+ DOCTEST_INTERFACE String getTlsOssResult();
+
+ template <bool C>
+ struct StringMakerBase
+ {
+ template <typename T>
+ static String convert(const DOCTEST_REF_WRAP(T)) {
+ return "{?}";
+ }
+ };
+
+ template <>
+ struct StringMakerBase<true>
+ {
+ template <typename T>
+ static String convert(const DOCTEST_REF_WRAP(T) in) {
+ *getTlsOss() << in;
+ return getTlsOssResult();
+ }
+ };
+
+ DOCTEST_INTERFACE String rawMemoryToString(const void* object, unsigned size);
+
+ template <typename T>
+ String rawMemoryToString(const DOCTEST_REF_WRAP(T) object) {
+ return rawMemoryToString(&object, sizeof(object));
+ }
+
+ template <typename T>
+ const char* type_to_string() {
+ return "<>";
+ }
+} // namespace detail
+
+template <typename T>
+struct StringMaker : public detail::StringMakerBase<detail::has_insertion_operator<T>::value>
+{};
+
+template <typename T>
+struct StringMaker<T*>
+{
+ template <typename U>
+ static String convert(U* p) {
+ if(p)
+ return detail::rawMemoryToString(p);
+ return "NULL";
+ }
+};
+
+template <typename R, typename C>
+struct StringMaker<R C::*>
+{
+ static String convert(R C::*p) {
+ if(p)
+ return detail::rawMemoryToString(p);
+ return "NULL";
+ }
+};
+
+template <typename T, typename detail::enable_if<!detail::is_enum<T>::value, bool>::type = true>
+String toString(const DOCTEST_REF_WRAP(T) value) {
+ return StringMaker<T>::convert(value);
+}
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+DOCTEST_INTERFACE String toString(char* in);
+DOCTEST_INTERFACE String toString(const char* in);
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+DOCTEST_INTERFACE String toString(bool in);
+DOCTEST_INTERFACE String toString(float in);
+DOCTEST_INTERFACE String toString(double in);
+DOCTEST_INTERFACE String toString(double long in);
+
+DOCTEST_INTERFACE String toString(char in);
+DOCTEST_INTERFACE String toString(char signed in);
+DOCTEST_INTERFACE String toString(char unsigned in);
+DOCTEST_INTERFACE String toString(int short in);
+DOCTEST_INTERFACE String toString(int short unsigned in);
+DOCTEST_INTERFACE String toString(int in);
+DOCTEST_INTERFACE String toString(int unsigned in);
+DOCTEST_INTERFACE String toString(int long in);
+DOCTEST_INTERFACE String toString(int long unsigned in);
+DOCTEST_INTERFACE String toString(int long long in);
+DOCTEST_INTERFACE String toString(int long long unsigned in);
+DOCTEST_INTERFACE String toString(std::nullptr_t in);
+
+template <typename T, typename detail::enable_if<detail::is_enum<T>::value, bool>::type = true>
+String toString(const DOCTEST_REF_WRAP(T) value) {
+ typedef typename detail::underlying_type<T>::type UT;
+ return toString(static_cast<UT>(value));
+}
+
+#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0)
+// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183
+DOCTEST_INTERFACE String toString(const std::string& in);
+#endif // VS 2019
+
+class DOCTEST_INTERFACE Approx
+{
+public:
+ explicit Approx(double value);
+
+ Approx operator()(double value) const;
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ explicit Approx(const T& value,
+ typename detail::enable_if<std::is_constructible<double, T>::value>::type* =
+ static_cast<T*>(nullptr)) {
+ *this = Approx(static_cast<double>(value));
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ Approx& epsilon(double newEpsilon);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ typename detail::enable_if<std::is_constructible<double, T>::value, Approx&>::type epsilon(
+ const T& newEpsilon) {
+ m_epsilon = static_cast<double>(newEpsilon);
+ return *this;
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ Approx& scale(double newScale);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+ template <typename T>
+ typename detail::enable_if<std::is_constructible<double, T>::value, Approx&>::type scale(
+ const T& newScale) {
+ m_scale = static_cast<double>(newScale);
+ return *this;
+ }
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ // clang-format off
+ DOCTEST_INTERFACE friend bool operator==(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator==(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator!=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator!=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator<=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator<=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator>=(double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator>=(const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator< (double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator< (const Approx & lhs, double rhs);
+ DOCTEST_INTERFACE friend bool operator> (double lhs, const Approx & rhs);
+ DOCTEST_INTERFACE friend bool operator> (const Approx & lhs, double rhs);
+
+ DOCTEST_INTERFACE friend String toString(const Approx& in);
+
+#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+#define DOCTEST_APPROX_PREFIX \
+ template <typename T> friend typename detail::enable_if<std::is_constructible<double, T>::value, bool>::type
+
+ DOCTEST_APPROX_PREFIX operator==(const T& lhs, const Approx& rhs) { return operator==(double(lhs), rhs); }
+ DOCTEST_APPROX_PREFIX operator==(const Approx& lhs, const T& rhs) { return operator==(rhs, lhs); }
+ DOCTEST_APPROX_PREFIX operator!=(const T& lhs, const Approx& rhs) { return !operator==(lhs, rhs); }
+ DOCTEST_APPROX_PREFIX operator!=(const Approx& lhs, const T& rhs) { return !operator==(rhs, lhs); }
+ DOCTEST_APPROX_PREFIX operator<=(const T& lhs, const Approx& rhs) { return double(lhs) < rhs.m_value || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator<=(const Approx& lhs, const T& rhs) { return lhs.m_value < double(rhs) || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator>=(const T& lhs, const Approx& rhs) { return double(lhs) > rhs.m_value || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator>=(const Approx& lhs, const T& rhs) { return lhs.m_value > double(rhs) || lhs == rhs; }
+ DOCTEST_APPROX_PREFIX operator< (const T& lhs, const Approx& rhs) { return double(lhs) < rhs.m_value && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator< (const Approx& lhs, const T& rhs) { return lhs.m_value < double(rhs) && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator> (const T& lhs, const Approx& rhs) { return double(lhs) > rhs.m_value && lhs != rhs; }
+ DOCTEST_APPROX_PREFIX operator> (const Approx& lhs, const T& rhs) { return lhs.m_value > double(rhs) && lhs != rhs; }
+#undef DOCTEST_APPROX_PREFIX
+#endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS
+
+ // clang-format on
+
+private:
+ double m_epsilon;
+ double m_scale;
+ double m_value;
+};
+
+DOCTEST_INTERFACE String toString(const Approx& in);
+
+DOCTEST_INTERFACE const ContextOptions* getContextOptions();
+
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+namespace detail {
+ // clang-format off
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ template<class T> struct decay_array { typedef T type; };
+ template<class T, unsigned N> struct decay_array<T[N]> { typedef T* type; };
+ template<class T> struct decay_array<T[]> { typedef T* type; };
+
+ template<class T> struct not_char_pointer { enum { value = 1 }; };
+ template<> struct not_char_pointer<char*> { enum { value = 0 }; };
+ template<> struct not_char_pointer<const char*> { enum { value = 0 }; };
+
+ template<class T> struct can_use_op : public not_char_pointer<typename decay_array<T>::type> {};
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ // clang-format on
+
+ struct DOCTEST_INTERFACE TestFailureException
+ {
+ };
+
+ DOCTEST_INTERFACE bool checkIfShouldThrow(assertType::Enum at);
+
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_NORETURN
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ DOCTEST_INTERFACE void throwException();
+
+ struct DOCTEST_INTERFACE Subcase
+ {
+ SubcaseSignature m_signature;
+ bool m_entered = false;
+
+ Subcase(const String& name, const char* file, int line);
+ ~Subcase();
+
+ operator bool() const;
+ };
+
+ template <typename L, typename R>
+ String stringifyBinaryExpr(const DOCTEST_REF_WRAP(L) lhs, const char* op,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ return toString(lhs) + op + toString(rhs);
+ }
+
+#if DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 6, 0)
+DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-comparison")
+#endif
+
+// This will check if there is any way it could find a operator like member or friend and uses it.
+// If not it doesn't find the operator or if the operator at global scope is defined after
+// this template, the template won't be instantiated due to SFINAE. Once the template is not
+// instantiated it can look for global operator using normal conversions.
+#define SFINAE_OP(ret,op) decltype(doctest::detail::declval<L>() op doctest::detail::declval<R>(),static_cast<ret>(0))
+
+#define DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(op, op_str, op_macro) \
+ template <typename R> \
+ DOCTEST_NOINLINE SFINAE_OP(Result,op) operator op(R&& rhs) { \
+ bool res = op_macro(doctest::detail::forward<L>(lhs), doctest::detail::forward<R>(rhs)); \
+ if(m_at & assertType::is_false) \
+ res = !res; \
+ if(!res || doctest::getContextOptions()->success) \
+ return Result(res, stringifyBinaryExpr(lhs, op_str, rhs)); \
+ return Result(res); \
+ }
+
+ // more checks could be added - like in Catch:
+ // https://github.com/catchorg/Catch2/pull/1480/files
+ // https://github.com/catchorg/Catch2/pull/1481/files
+#define DOCTEST_FORBIT_EXPRESSION(rt, op) \
+ template <typename R> \
+ rt& operator op(const R&) { \
+ static_assert(deferred_false<R>::value, \
+ "Expression Too Complex Please Rewrite As Binary Comparison!"); \
+ return *this; \
+ }
+
+ struct DOCTEST_INTERFACE Result
+ {
+ bool m_passed;
+ String m_decomp;
+
+ Result(bool passed, const String& decomposition = String());
+
+ // forbidding some expressions based on this table: https://en.cppreference.com/w/cpp/language/operator_precedence
+ DOCTEST_FORBIT_EXPRESSION(Result, &)
+ DOCTEST_FORBIT_EXPRESSION(Result, ^)
+ DOCTEST_FORBIT_EXPRESSION(Result, |)
+ DOCTEST_FORBIT_EXPRESSION(Result, &&)
+ DOCTEST_FORBIT_EXPRESSION(Result, ||)
+ DOCTEST_FORBIT_EXPRESSION(Result, ==)
+ DOCTEST_FORBIT_EXPRESSION(Result, !=)
+ DOCTEST_FORBIT_EXPRESSION(Result, <)
+ DOCTEST_FORBIT_EXPRESSION(Result, >)
+ DOCTEST_FORBIT_EXPRESSION(Result, <=)
+ DOCTEST_FORBIT_EXPRESSION(Result, >=)
+ DOCTEST_FORBIT_EXPRESSION(Result, =)
+ DOCTEST_FORBIT_EXPRESSION(Result, +=)
+ DOCTEST_FORBIT_EXPRESSION(Result, -=)
+ DOCTEST_FORBIT_EXPRESSION(Result, *=)
+ DOCTEST_FORBIT_EXPRESSION(Result, /=)
+ DOCTEST_FORBIT_EXPRESSION(Result, %=)
+ DOCTEST_FORBIT_EXPRESSION(Result, <<=)
+ DOCTEST_FORBIT_EXPRESSION(Result, >>=)
+ DOCTEST_FORBIT_EXPRESSION(Result, &=)
+ DOCTEST_FORBIT_EXPRESSION(Result, ^=)
+ DOCTEST_FORBIT_EXPRESSION(Result, |=)
+ };
+
+#ifndef DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-conversion")
+ DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-compare")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wdouble-promotion")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wconversion")
+ //DOCTEST_CLANG_SUPPRESS_WARNING("-Wfloat-equal")
+
+ DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+ DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-conversion")
+ DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-compare")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wdouble-promotion")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wconversion")
+ //DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal")
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH
+ // https://stackoverflow.com/questions/39479163 what's the difference between 4018 and 4389
+ DOCTEST_MSVC_SUPPRESS_WARNING(4388) // signed/unsigned mismatch
+ DOCTEST_MSVC_SUPPRESS_WARNING(4389) // 'operator' : signed/unsigned mismatch
+ DOCTEST_MSVC_SUPPRESS_WARNING(4018) // 'expression' : signed/unsigned mismatch
+ //DOCTEST_MSVC_SUPPRESS_WARNING(4805) // 'operation' : unsafe mix of type 'type' and type 'type' in operation
+
+#endif // DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ // clang-format off
+#ifndef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_COMPARISON_RETURN_TYPE bool
+#else // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_COMPARISON_RETURN_TYPE typename enable_if<can_use_op<L>::value || can_use_op<R>::value, bool>::type
+ // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+ inline bool eq(const char* lhs, const char* rhs) { return String(lhs) == String(rhs); }
+ inline bool ne(const char* lhs, const char* rhs) { return String(lhs) != String(rhs); }
+ inline bool lt(const char* lhs, const char* rhs) { return String(lhs) < String(rhs); }
+ inline bool gt(const char* lhs, const char* rhs) { return String(lhs) > String(rhs); }
+ inline bool le(const char* lhs, const char* rhs) { return String(lhs) <= String(rhs); }
+ inline bool ge(const char* lhs, const char* rhs) { return String(lhs) >= String(rhs); }
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ // clang-format on
+
+#define DOCTEST_RELATIONAL_OP(name, op) \
+ template <typename L, typename R> \
+ DOCTEST_COMPARISON_RETURN_TYPE name(const DOCTEST_REF_WRAP(L) lhs, \
+ const DOCTEST_REF_WRAP(R) rhs) { \
+ return lhs op rhs; \
+ }
+
+ DOCTEST_RELATIONAL_OP(eq, ==)
+ DOCTEST_RELATIONAL_OP(ne, !=)
+ DOCTEST_RELATIONAL_OP(lt, <)
+ DOCTEST_RELATIONAL_OP(gt, >)
+ DOCTEST_RELATIONAL_OP(le, <=)
+ DOCTEST_RELATIONAL_OP(ge, >=)
+
+#ifndef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_CMP_EQ(l, r) l == r
+#define DOCTEST_CMP_NE(l, r) l != r
+#define DOCTEST_CMP_GT(l, r) l > r
+#define DOCTEST_CMP_LT(l, r) l < r
+#define DOCTEST_CMP_GE(l, r) l >= r
+#define DOCTEST_CMP_LE(l, r) l <= r
+#else // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+#define DOCTEST_CMP_EQ(l, r) eq(l, r)
+#define DOCTEST_CMP_NE(l, r) ne(l, r)
+#define DOCTEST_CMP_GT(l, r) gt(l, r)
+#define DOCTEST_CMP_LT(l, r) lt(l, r)
+#define DOCTEST_CMP_GE(l, r) ge(l, r)
+#define DOCTEST_CMP_LE(l, r) le(l, r)
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+
+ template <typename L>
+ // cppcheck-suppress copyCtorAndEqOperator
+ struct Expression_lhs
+ {
+ L lhs;
+ assertType::Enum m_at;
+
+ explicit Expression_lhs(L&& in, assertType::Enum at)
+ : lhs(doctest::detail::forward<L>(in))
+ , m_at(at) {}
+
+ DOCTEST_NOINLINE operator Result() {
+// this is needed only foc MSVC 2015:
+// https://ci.appveyor.com/project/onqtam/doctest/builds/38181202
+DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4800) // 'int': forcing value to bool
+ bool res = static_cast<bool>(lhs);
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ if(m_at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ res = !res;
+
+ if(!res || getContextOptions()->success)
+ return Result(res, toString(lhs));
+ return Result(res);
+ }
+
+ /* This is required for user-defined conversions from Expression_lhs to L */
+ //operator L() const { return lhs; }
+ operator L() const { return lhs; }
+
+ // clang-format off
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(==, " == ", DOCTEST_CMP_EQ) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(!=, " != ", DOCTEST_CMP_NE) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(>, " > ", DOCTEST_CMP_GT) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(<, " < ", DOCTEST_CMP_LT) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(>=, " >= ", DOCTEST_CMP_GE) //!OCLINT bitwise operator in conditional
+ DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(<=, " <= ", DOCTEST_CMP_LE) //!OCLINT bitwise operator in conditional
+ // clang-format on
+
+ // forbidding some expressions based on this table: https://en.cppreference.com/w/cpp/language/operator_precedence
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ^)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, |)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &&)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ||)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, =)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, +=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, -=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, *=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, /=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, %=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, <<=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, >>=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, &=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, ^=)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, |=)
+ // these 2 are unfortunate because they should be allowed - they have higher precedence over the comparisons, but the
+ // ExpressionDecomposer class uses the left shift operator to capture the left operand of the binary expression...
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, <<)
+ DOCTEST_FORBIT_EXPRESSION(Expression_lhs, >>)
+ };
+
+#ifndef DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_NO_COMPARISON_WARNING_SUPPRESSION
+
+#if DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 6, 0)
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+#endif
+
+ struct DOCTEST_INTERFACE ExpressionDecomposer
+ {
+ assertType::Enum m_at;
+
+ ExpressionDecomposer(assertType::Enum at);
+
+ // The right operator for capturing expressions is "<=" instead of "<<" (based on the operator precedence table)
+ // but then there will be warnings from GCC about "-Wparentheses" and since "_Pragma()" is problematic this will stay for now...
+ // https://github.com/catchorg/Catch2/issues/870
+ // https://github.com/catchorg/Catch2/issues/565
+ template <typename L>
+ Expression_lhs<L> operator<<(L &&operand) {
+ return Expression_lhs<L>(doctest::detail::forward<L>(operand), m_at);
+ }
+ };
+
+ struct DOCTEST_INTERFACE TestSuite
+ {
+ const char* m_test_suite;
+ const char* m_description;
+ bool m_skip;
+ bool m_no_breaks;
+ bool m_no_output;
+ bool m_may_fail;
+ bool m_should_fail;
+ int m_expected_failures;
+ double m_timeout;
+
+ TestSuite& operator*(const char* in);
+
+ template <typename T>
+ TestSuite& operator*(const T& in) {
+ in.fill(*this);
+ return *this;
+ }
+ };
+
+ typedef void (*funcType)();
+
+ struct DOCTEST_INTERFACE TestCase : public TestCaseData
+ {
+ funcType m_test; // a function pointer to the test case
+
+ const char* m_type; // for templated test cases - gets appended to the real name
+ int m_template_id; // an ID used to distinguish between the different versions of a templated test case
+ String m_full_name; // contains the name (only for templated test cases!) + the template type
+
+ TestCase(funcType test, const char* file, unsigned line, const TestSuite& test_suite,
+ const char* type = "", int template_id = -1);
+
+ TestCase(const TestCase& other);
+
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(26434) // hides a non-virtual function
+ TestCase& operator=(const TestCase& other);
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP
+
+ TestCase& operator*(const char* in);
+
+ template <typename T>
+ TestCase& operator*(const T& in) {
+ in.fill(*this);
+ return *this;
+ }
+
+ bool operator<(const TestCase& other) const;
+ };
+
+ // forward declarations of functions used by the macros
+ DOCTEST_INTERFACE int regTest(const TestCase& tc);
+ DOCTEST_INTERFACE int setTestSuite(const TestSuite& ts);
+ DOCTEST_INTERFACE bool isDebuggerActive();
+
+ template<typename T>
+ int instantiationHelper(const T&) { return 0; }
+
+ namespace binaryAssertComparison {
+ enum Enum
+ {
+ eq = 0,
+ ne,
+ gt,
+ lt,
+ ge,
+ le
+ };
+ } // namespace binaryAssertComparison
+
+ // clang-format off
+ template <int, class L, class R> struct RelationalComparator { bool operator()(const DOCTEST_REF_WRAP(L), const DOCTEST_REF_WRAP(R) ) const { return false; } };
+
+#define DOCTEST_BINARY_RELATIONAL_OP(n, op) \
+ template <class L, class R> struct RelationalComparator<n, L, R> { bool operator()(const DOCTEST_REF_WRAP(L) lhs, const DOCTEST_REF_WRAP(R) rhs) const { return op(lhs, rhs); } };
+ // clang-format on
+
+ DOCTEST_BINARY_RELATIONAL_OP(0, doctest::detail::eq)
+ DOCTEST_BINARY_RELATIONAL_OP(1, doctest::detail::ne)
+ DOCTEST_BINARY_RELATIONAL_OP(2, doctest::detail::gt)
+ DOCTEST_BINARY_RELATIONAL_OP(3, doctest::detail::lt)
+ DOCTEST_BINARY_RELATIONAL_OP(4, doctest::detail::ge)
+ DOCTEST_BINARY_RELATIONAL_OP(5, doctest::detail::le)
+
+ struct DOCTEST_INTERFACE ResultBuilder : public AssertData
+ {
+ ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr,
+ const char* exception_type = "", const char* exception_string = "");
+
+ void setResult(const Result& res);
+
+ template <int comparison, typename L, typename R>
+ DOCTEST_NOINLINE void binary_assert(const DOCTEST_REF_WRAP(L) lhs,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ m_failed = !RelationalComparator<comparison, L, R>()(lhs, rhs);
+ if(m_failed || getContextOptions()->success)
+ m_decomp = stringifyBinaryExpr(lhs, ", ", rhs);
+ }
+
+ template <typename L>
+ DOCTEST_NOINLINE void unary_assert(const DOCTEST_REF_WRAP(L) val) {
+ m_failed = !val;
+
+ if(m_at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ m_failed = !m_failed;
+
+ if(m_failed || getContextOptions()->success)
+ m_decomp = toString(val);
+ }
+
+ void translateException();
+
+ bool log();
+ void react() const;
+ };
+
+ namespace assertAction {
+ enum Enum
+ {
+ nothing = 0,
+ dbgbreak = 1,
+ shouldthrow = 2
+ };
+ } // namespace assertAction
+
+ DOCTEST_INTERFACE void failed_out_of_a_testing_context(const AssertData& ad);
+
+ DOCTEST_INTERFACE void decomp_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, Result result);
+
+#define DOCTEST_ASSERT_OUT_OF_TESTS(decomp) \
+ do { \
+ if(!is_running_in_test) { \
+ if(failed) { \
+ ResultBuilder rb(at, file, line, expr); \
+ rb.m_failed = failed; \
+ rb.m_decomp = decomp; \
+ failed_out_of_a_testing_context(rb); \
+ if(isDebuggerActive() && !getContextOptions()->no_breaks) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ if(checkIfShouldThrow(at)) \
+ throwException(); \
+ } \
+ return; \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_IN_TESTS(decomp) \
+ ResultBuilder rb(at, file, line, expr); \
+ rb.m_failed = failed; \
+ if(rb.m_failed || getContextOptions()->success) \
+ rb.m_decomp = decomp; \
+ if(rb.log()) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ if(rb.m_failed && checkIfShouldThrow(at)) \
+ throwException()
+
+ template <int comparison, typename L, typename R>
+ DOCTEST_NOINLINE void binary_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, const DOCTEST_REF_WRAP(L) lhs,
+ const DOCTEST_REF_WRAP(R) rhs) {
+ bool failed = !RelationalComparator<comparison, L, R>()(lhs, rhs);
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(stringifyBinaryExpr(lhs, ", ", rhs));
+ DOCTEST_ASSERT_IN_TESTS(stringifyBinaryExpr(lhs, ", ", rhs));
+ }
+
+ template <typename L>
+ DOCTEST_NOINLINE void unary_assert(assertType::Enum at, const char* file, int line,
+ const char* expr, const DOCTEST_REF_WRAP(L) val) {
+ bool failed = !val;
+
+ if(at & assertType::is_false) //!OCLINT bitwise operator in conditional
+ failed = !failed;
+
+ // ###################################################################################
+ // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT
+ // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED
+ // ###################################################################################
+ DOCTEST_ASSERT_OUT_OF_TESTS(toString(val));
+ DOCTEST_ASSERT_IN_TESTS(toString(val));
+ }
+
+ struct DOCTEST_INTERFACE IExceptionTranslator
+ {
+ IExceptionTranslator();
+ virtual ~IExceptionTranslator();
+ virtual bool translate(String&) const = 0;
+ };
+
+ template <typename T>
+ class ExceptionTranslator : public IExceptionTranslator //!OCLINT destructor of virtual class
+ {
+ public:
+ explicit ExceptionTranslator(String (*translateFunction)(T))
+ : m_translateFunction(translateFunction) {}
+
+ bool translate(String& res) const override {
+#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS
+ try {
+ throw; // lgtm [cpp/rethrow-no-exception]
+ // cppcheck-suppress catchExceptionByValue
+ } catch(T ex) { // NOLINT
+ res = m_translateFunction(ex); //!OCLINT parameter reassignment
+ return true;
+ } catch(...) {} //!OCLINT - empty catch statement
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+ static_cast<void>(res); // to silence -Wunused-parameter
+ return false;
+ }
+
+ private:
+ String (*m_translateFunction)(T);
+ };
+
+ DOCTEST_INTERFACE void registerExceptionTranslatorImpl(const IExceptionTranslator* et);
+
+ template <bool C>
+ struct StringStreamBase
+ {
+ template <typename T>
+ static void convert(std::ostream* s, const T& in) {
+ *s << toString(in);
+ }
+
+ // always treat char* as a string in this context - no matter
+ // if DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING is defined
+ static void convert(std::ostream* s, const char* in) { *s << String(in); }
+ };
+
+ template <>
+ struct StringStreamBase<true>
+ {
+ template <typename T>
+ static void convert(std::ostream* s, const T& in) {
+ *s << in;
+ }
+ };
+
+ template <typename T>
+ struct StringStream : public StringStreamBase<has_insertion_operator<T>::value>
+ {};
+
+ template <typename T>
+ void toStream(std::ostream* s, const T& value) {
+ StringStream<T>::convert(s, value);
+ }
+
+#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char* in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, const char* in);
+#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
+ DOCTEST_INTERFACE void toStream(std::ostream* s, bool in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, float in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, double in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, double long in);
+
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char signed in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, char unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int short in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int short unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long unsigned in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long long in);
+ DOCTEST_INTERFACE void toStream(std::ostream* s, int long long unsigned in);
+
+ // ContextScope base class used to allow implementing methods of ContextScope
+ // that don't depend on the template parameter in doctest.cpp.
+ class DOCTEST_INTERFACE ContextScopeBase : public IContextScope {
+ protected:
+ ContextScopeBase();
+
+ void destroy();
+ };
+
+ template <typename L> class ContextScope : public ContextScopeBase
+ {
+ const L lambda_;
+
+ public:
+ explicit ContextScope(const L &lambda) : lambda_(lambda) {}
+
+ ContextScope(ContextScope &&other) : lambda_(other.lambda_) {}
+
+ void stringify(std::ostream* s) const override { lambda_(s); }
+
+ ~ContextScope() override { destroy(); }
+ };
+
+ struct DOCTEST_INTERFACE MessageBuilder : public MessageData
+ {
+ std::ostream* m_stream;
+
+ MessageBuilder(const char* file, int line, assertType::Enum severity);
+ MessageBuilder() = delete;
+ ~MessageBuilder();
+
+ // the preferred way of chaining parameters for stringification
+ template <typename T>
+ MessageBuilder& operator,(const T& in) {
+ toStream(m_stream, in);
+ return *this;
+ }
+
+ // kept here just for backwards-compatibility - the comma operator should be preferred now
+ template <typename T>
+ MessageBuilder& operator<<(const T& in) { return this->operator,(in); }
+
+ // the `,` operator has the lowest operator precedence - if `<<` is used by the user then
+ // the `,` operator will be called last which is not what we want and thus the `*` operator
+ // is used first (has higher operator precedence compared to `<<`) so that we guarantee that
+ // an operator of the MessageBuilder class is called first before the rest of the parameters
+ template <typename T>
+ MessageBuilder& operator*(const T& in) { return this->operator,(in); }
+
+ bool log();
+ void react();
+ };
+
+ template <typename L>
+ ContextScope<L> MakeContextScope(const L &lambda) {
+ return ContextScope<L>(lambda);
+ }
+} // namespace detail
+
+#define DOCTEST_DEFINE_DECORATOR(name, type, def) \
+ struct name \
+ { \
+ type data; \
+ name(type in = def) \
+ : data(in) {} \
+ void fill(detail::TestCase& state) const { state.DOCTEST_CAT(m_, name) = data; } \
+ void fill(detail::TestSuite& state) const { state.DOCTEST_CAT(m_, name) = data; } \
+ }
+
+DOCTEST_DEFINE_DECORATOR(test_suite, const char*, "");
+DOCTEST_DEFINE_DECORATOR(description, const char*, "");
+DOCTEST_DEFINE_DECORATOR(skip, bool, true);
+DOCTEST_DEFINE_DECORATOR(no_breaks, bool, true);
+DOCTEST_DEFINE_DECORATOR(no_output, bool, true);
+DOCTEST_DEFINE_DECORATOR(timeout, double, 0);
+DOCTEST_DEFINE_DECORATOR(may_fail, bool, true);
+DOCTEST_DEFINE_DECORATOR(should_fail, bool, true);
+DOCTEST_DEFINE_DECORATOR(expected_failures, int, 0);
+
+template <typename T>
+int registerExceptionTranslator(String (*translateFunction)(T)) {
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wexit-time-destructors")
+ static detail::ExceptionTranslator<T> exceptionTranslator(translateFunction);
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+ detail::registerExceptionTranslatorImpl(&exceptionTranslator);
+ return 0;
+}
+
+} // namespace doctest
+
+// in a separate namespace outside of doctest because the DOCTEST_TEST_SUITE macro
+// introduces an anonymous namespace in which getCurrentTestSuite gets overridden
+namespace doctest_detail_test_suite_ns {
+DOCTEST_INTERFACE doctest::detail::TestSuite& getCurrentTestSuite();
+} // namespace doctest_detail_test_suite_ns
+
+namespace doctest {
+#else // DOCTEST_CONFIG_DISABLE
+template <typename T>
+int registerExceptionTranslator(String (*)(T)) {
+ return 0;
+}
+#endif // DOCTEST_CONFIG_DISABLE
+
+namespace detail {
+ typedef void (*assert_handler)(const AssertData&);
+ struct ContextState;
+} // namespace detail
+
+class DOCTEST_INTERFACE Context
+{
+ detail::ContextState* p;
+
+ void parseArgs(int argc, const char* const* argv, bool withDefaults = false);
+
+public:
+ explicit Context(int argc = 0, const char* const* argv = nullptr);
+
+ ~Context();
+
+ void applyCommandLine(int argc, const char* const* argv);
+
+ void addFilter(const char* filter, const char* value);
+ void clearFilters();
+ void setOption(const char* option, int value);
+ void setOption(const char* option, const char* value);
+
+ bool shouldExit();
+
+ void setAsDefaultForAssertsOutOfTestCases();
+
+ void setAssertHandler(detail::assert_handler ah);
+
+ int run();
+};
+
+namespace TestCaseFailureReason {
+ enum Enum
+ {
+ None = 0,
+ AssertFailure = 1, // an assertion has failed in the test case
+ Exception = 2, // test case threw an exception
+ Crash = 4, // a crash...
+ TooManyFailedAsserts = 8, // the abort-after option
+ Timeout = 16, // see the timeout decorator
+ ShouldHaveFailedButDidnt = 32, // see the should_fail decorator
+ ShouldHaveFailedAndDid = 64, // see the should_fail decorator
+ DidntFailExactlyNumTimes = 128, // see the expected_failures decorator
+ FailedExactlyNumTimes = 256, // see the expected_failures decorator
+ CouldHaveFailedAndDid = 512 // see the may_fail decorator
+ };
+} // namespace TestCaseFailureReason
+
+struct DOCTEST_INTERFACE CurrentTestCaseStats
+{
+ int numAssertsCurrentTest;
+ int numAssertsFailedCurrentTest;
+ double seconds;
+ int failure_flags; // use TestCaseFailureReason::Enum
+};
+
+struct DOCTEST_INTERFACE TestCaseException
+{
+ String error_string;
+ bool is_crash;
+};
+
+struct DOCTEST_INTERFACE TestRunStats
+{
+ unsigned numTestCases;
+ unsigned numTestCasesPassingFilters;
+ unsigned numTestSuitesPassingFilters;
+ unsigned numTestCasesFailed;
+ int numAsserts;
+ int numAssertsFailed;
+};
+
+struct QueryData
+{
+ const TestRunStats* run_stats = nullptr;
+ const TestCaseData** data = nullptr;
+ unsigned num_data = 0;
+};
+
+struct DOCTEST_INTERFACE IReporter
+{
+ // The constructor has to accept "const ContextOptions&" as a single argument
+ // which has most of the options for the run + a pointer to the stdout stream
+ // Reporter(const ContextOptions& in)
+
+ // called when a query should be reported (listing test cases, printing the version, etc.)
+ virtual void report_query(const QueryData&) = 0;
+
+ // called when the whole test run starts
+ virtual void test_run_start() = 0;
+ // called when the whole test run ends (caching a pointer to the input doesn't make sense here)
+ virtual void test_run_end(const TestRunStats&) = 0;
+
+ // called when a test case is started (safe to cache a pointer to the input)
+ virtual void test_case_start(const TestCaseData&) = 0;
+ // called when a test case is reentered because of unfinished subcases (safe to cache a pointer to the input)
+ virtual void test_case_reenter(const TestCaseData&) = 0;
+ // called when a test case has ended
+ virtual void test_case_end(const CurrentTestCaseStats&) = 0;
+
+ // called when an exception is thrown from the test case (or it crashes)
+ virtual void test_case_exception(const TestCaseException&) = 0;
+
+ // called whenever a subcase is entered (don't cache pointers to the input)
+ virtual void subcase_start(const SubcaseSignature&) = 0;
+ // called whenever a subcase is exited (don't cache pointers to the input)
+ virtual void subcase_end() = 0;
+
+ // called for each assert (don't cache pointers to the input)
+ virtual void log_assert(const AssertData&) = 0;
+ // called for each message (don't cache pointers to the input)
+ virtual void log_message(const MessageData&) = 0;
+
+ // called when a test case is skipped either because it doesn't pass the filters, has a skip decorator
+ // or isn't in the execution range (between first and last) (safe to cache a pointer to the input)
+ virtual void test_case_skipped(const TestCaseData&) = 0;
+
+ // doctest will not be managing the lifetimes of reporters given to it but this would still be nice to have
+ virtual ~IReporter();
+
+ // can obtain all currently active contexts and stringify them if one wishes to do so
+ static int get_num_active_contexts();
+ static const IContextScope* const* get_active_contexts();
+
+ // can iterate through contexts which have been stringified automatically in their destructors when an exception has been thrown
+ static int get_num_stringified_contexts();
+ static const String* get_stringified_contexts();
+};
+
+namespace detail {
+ typedef IReporter* (*reporterCreatorFunc)(const ContextOptions&);
+
+ DOCTEST_INTERFACE void registerReporterImpl(const char* name, int prio, reporterCreatorFunc c, bool isReporter);
+
+ template <typename Reporter>
+ IReporter* reporterCreator(const ContextOptions& o) {
+ return new Reporter(o);
+ }
+} // namespace detail
+
+template <typename Reporter>
+int registerReporter(const char* name, int priority, bool isReporter) {
+ detail::registerReporterImpl(name, priority, detail::reporterCreator<Reporter>, isReporter);
+ return 0;
+}
+} // namespace doctest
+
+// if registering is not disabled
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+// common code in asserts - for convenience
+#define DOCTEST_ASSERT_LOG_AND_REACT(b) \
+ if(b.log()) \
+ DOCTEST_BREAK_INTO_DEBUGGER(); \
+ b.react()
+
+#ifdef DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#define DOCTEST_WRAP_IN_TRY(x) x;
+#else // DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+#define DOCTEST_WRAP_IN_TRY(x) \
+ try { \
+ x; \
+ } catch(...) { _DOCTEST_RB.translateException(); }
+#endif // DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
+
+#ifdef DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+#define DOCTEST_CAST_TO_VOID(...) \
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wuseless-cast") \
+ static_cast<void>(__VA_ARGS__); \
+ DOCTEST_GCC_SUPPRESS_WARNING_POP
+#else // DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+#define DOCTEST_CAST_TO_VOID(...) __VA_ARGS__;
+#endif // DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS
+
+// registers the test by initializing a dummy var with a function
+#define DOCTEST_REGISTER_FUNCTION(global_prefix, f, decorators) \
+ global_prefix DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::regTest( \
+ doctest::detail::TestCase( \
+ f, __FILE__, __LINE__, \
+ doctest_detail_test_suite_ns::getCurrentTestSuite()) * \
+ decorators); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END()
+
+#define DOCTEST_IMPLEMENT_FIXTURE(der, base, func, decorators) \
+ namespace { \
+ struct der : public base \
+ { \
+ void f(); \
+ }; \
+ static void func() { \
+ der v; \
+ v.f(); \
+ } \
+ DOCTEST_REGISTER_FUNCTION(DOCTEST_EMPTY, func, decorators) \
+ } \
+ inline DOCTEST_NOINLINE void der::f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION(f, decorators) \
+ static void f(); \
+ DOCTEST_REGISTER_FUNCTION(DOCTEST_EMPTY, f, decorators) \
+ static void f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION_IN_CLASS(f, proxy, decorators) \
+ static doctest::detail::funcType proxy() { return f; } \
+ DOCTEST_REGISTER_FUNCTION(inline const, proxy(), decorators) \
+ static void f()
+
+// for registering tests
+#define DOCTEST_TEST_CASE(decorators) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), decorators)
+
+// for registering tests in classes - requires C++17 for inline variables!
+#if __cplusplus >= 201703L || (DOCTEST_MSVC >= DOCTEST_COMPILER(19, 12, 0) && _MSVC_LANG >= 201703L)
+#define DOCTEST_TEST_CASE_CLASS(decorators) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION_IN_CLASS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_PROXY_), \
+ decorators)
+#else // DOCTEST_TEST_CASE_CLASS
+#define DOCTEST_TEST_CASE_CLASS(...) \
+ TEST_CASES_CAN_BE_REGISTERED_IN_CLASSES_ONLY_IN_CPP17_MODE_OR_WITH_VS_2017_OR_NEWER
+#endif // DOCTEST_TEST_CASE_CLASS
+
+// for registering tests with a fixture
+#define DOCTEST_TEST_CASE_FIXTURE(c, decorators) \
+ DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(_DOCTEST_ANON_CLASS_), c, \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), decorators)
+
+// for converting types to strings without the <typeinfo> header and demangling
+#define DOCTEST_TYPE_TO_STRING_IMPL(...) \
+ template <> \
+ inline const char* type_to_string<__VA_ARGS__>() { \
+ return "<" #__VA_ARGS__ ">"; \
+ }
+#define DOCTEST_TYPE_TO_STRING(...) \
+ namespace doctest { namespace detail { \
+ DOCTEST_TYPE_TO_STRING_IMPL(__VA_ARGS__) \
+ } \
+ } \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, iter, func) \
+ template <typename T> \
+ static void func(); \
+ namespace { \
+ template <typename Tuple> \
+ struct iter; \
+ template <typename Type, typename... Rest> \
+ struct iter<std::tuple<Type, Rest...>> \
+ { \
+ iter(const char* file, unsigned line, int index) { \
+ doctest::detail::regTest(doctest::detail::TestCase(func<Type>, file, line, \
+ doctest_detail_test_suite_ns::getCurrentTestSuite(), \
+ doctest::detail::type_to_string<Type>(), \
+ int(line) * 1000 + index) \
+ * dec); \
+ iter<std::tuple<Rest...>>(file, line, index + 1); \
+ } \
+ }; \
+ template <> \
+ struct iter<std::tuple<>> \
+ { \
+ iter(const char*, unsigned, int) {} \
+ }; \
+ } \
+ template <typename T> \
+ static void func()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE(dec, T, id) \
+ DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, DOCTEST_CAT(id, ITERATOR), \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_))
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, anon, ...) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_CAT(anon, DUMMY)) = \
+ doctest::detail::instantiationHelper(DOCTEST_CAT(id, ITERATOR)<__VA_ARGS__>(__FILE__, __LINE__, 0));\
+ DOCTEST_GLOBAL_NO_WARNINGS_END()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), std::tuple<__VA_ARGS__>) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), __VA_ARGS__) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_IMPL(dec, T, anon, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, DOCTEST_CAT(anon, ITERATOR), anon); \
+ DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(anon, anon, std::tuple<__VA_ARGS__>) \
+ template <typename T> \
+ static void anon()
+
+#define DOCTEST_TEST_CASE_TEMPLATE(dec, T, ...) \
+ DOCTEST_TEST_CASE_TEMPLATE_IMPL(dec, T, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), __VA_ARGS__)
+
+// for subcases
+#define DOCTEST_SUBCASE(name) \
+ if(const doctest::detail::Subcase & DOCTEST_ANONYMOUS(_DOCTEST_ANON_SUBCASE_) DOCTEST_UNUSED = \
+ doctest::detail::Subcase(name, __FILE__, __LINE__))
+
+// for grouping tests in test suites by using code blocks
+#define DOCTEST_TEST_SUITE_IMPL(decorators, ns_name) \
+ namespace ns_name { namespace doctest_detail_test_suite_ns { \
+ static DOCTEST_NOINLINE doctest::detail::TestSuite& getCurrentTestSuite() { \
+ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4640) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wexit-time-destructors") \
+ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wmissing-field-initializers") \
+ static doctest::detail::TestSuite data{}; \
+ static bool inited = false; \
+ DOCTEST_MSVC_SUPPRESS_WARNING_POP \
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP \
+ DOCTEST_GCC_SUPPRESS_WARNING_POP \
+ if(!inited) { \
+ data* decorators; \
+ inited = true; \
+ } \
+ return data; \
+ } \
+ } \
+ } \
+ namespace ns_name
+
+#define DOCTEST_TEST_SUITE(decorators) \
+ DOCTEST_TEST_SUITE_IMPL(decorators, DOCTEST_ANONYMOUS(_DOCTEST_ANON_SUITE_))
+
+// for starting a testsuite block
+#define DOCTEST_TEST_SUITE_BEGIN(decorators) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::setTestSuite(doctest::detail::TestSuite() * decorators); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for ending a testsuite block
+#define DOCTEST_TEST_SUITE_END \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \
+ doctest::detail::setTestSuite(doctest::detail::TestSuite() * ""); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for registering exception translators
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR_IMPL(translatorName, signature) \
+ inline doctest::String translatorName(signature); \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_)) = \
+ doctest::registerExceptionTranslator(translatorName); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() \
+ doctest::String translatorName(signature)
+
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature) \
+ DOCTEST_REGISTER_EXCEPTION_TRANSLATOR_IMPL(DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_), \
+ signature)
+
+// for registering reporters
+#define DOCTEST_REGISTER_REPORTER(name, priority, reporter) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_REPORTER_)) = \
+ doctest::registerReporter<reporter>(name, priority, true); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for registering listeners
+#define DOCTEST_REGISTER_LISTENER(name, priority, reporter) \
+ DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_REPORTER_)) = \
+ doctest::registerReporter<reporter>(name, priority, false); \
+ DOCTEST_GLOBAL_NO_WARNINGS_END() typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for logging
+#define DOCTEST_INFO(...) \
+ DOCTEST_INFO_IMPL(DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), \
+ __VA_ARGS__)
+
+#define DOCTEST_INFO_IMPL(mb_name, s_name, ...) \
+ auto DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_) = doctest::detail::MakeContextScope( \
+ [&](std::ostream* s_name) { \
+ doctest::detail::MessageBuilder mb_name(__FILE__, __LINE__, doctest::assertType::is_warn); \
+ mb_name.m_stream = s_name; \
+ mb_name * __VA_ARGS__; \
+ })
+
+#define DOCTEST_CAPTURE(x) DOCTEST_INFO(#x " := ", x)
+
+#define DOCTEST_ADD_AT_IMPL(type, file, line, mb, ...) \
+ do { \
+ doctest::detail::MessageBuilder mb(file, line, doctest::assertType::type); \
+ mb * __VA_ARGS__; \
+ DOCTEST_ASSERT_LOG_AND_REACT(mb); \
+ } while(false)
+
+// clang-format off
+#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_warn, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_check, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+#define DOCTEST_ADD_FAIL_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_require, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__)
+// clang-format on
+
+#define DOCTEST_MESSAGE(...) DOCTEST_ADD_MESSAGE_AT(__FILE__, __LINE__, __VA_ARGS__)
+#define DOCTEST_FAIL_CHECK(...) DOCTEST_ADD_FAIL_CHECK_AT(__FILE__, __LINE__, __VA_ARGS__)
+#define DOCTEST_FAIL(...) DOCTEST_ADD_FAIL_AT(__FILE__, __LINE__, __VA_ARGS__)
+
+#define DOCTEST_TO_LVALUE(...) __VA_ARGS__ // Not removed to keep backwards compatibility.
+
+#ifndef DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_ASSERT_IMPLEMENT_2(assert_type, ...) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Woverloaded-shift-op-parentheses") \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY(_DOCTEST_RB.setResult( \
+ doctest::detail::ExpressionDecomposer(doctest::assertType::assert_type) \
+ << __VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#define DOCTEST_ASSERT_IMPLEMENT_1(assert_type, ...) \
+ do { \
+ DOCTEST_ASSERT_IMPLEMENT_2(assert_type, __VA_ARGS__); \
+ } while(false)
+
+#else // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+// necessary for <ASSERT>_MESSAGE
+#define DOCTEST_ASSERT_IMPLEMENT_2 DOCTEST_ASSERT_IMPLEMENT_1
+
+#define DOCTEST_ASSERT_IMPLEMENT_1(assert_type, ...) \
+ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Woverloaded-shift-op-parentheses") \
+ doctest::detail::decomp_assert( \
+ doctest::assertType::assert_type, __FILE__, __LINE__, #__VA_ARGS__, \
+ doctest::detail::ExpressionDecomposer(doctest::assertType::assert_type) \
+ << __VA_ARGS__) DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_WARN(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_WARN, __VA_ARGS__)
+#define DOCTEST_CHECK(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_CHECK, __VA_ARGS__)
+#define DOCTEST_REQUIRE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_REQUIRE, __VA_ARGS__)
+#define DOCTEST_WARN_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_WARN_FALSE, __VA_ARGS__)
+#define DOCTEST_CHECK_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_CHECK_FALSE, __VA_ARGS__)
+#define DOCTEST_REQUIRE_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_REQUIRE_FALSE, __VA_ARGS__)
+
+// clang-format off
+#define DOCTEST_WARN_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN, cond); } while(false)
+#define DOCTEST_CHECK_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK, cond); } while(false)
+#define DOCTEST_REQUIRE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE, cond); } while(false)
+#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN_FALSE, cond); } while(false)
+#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK_FALSE, cond); } while(false)
+#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE_FALSE, cond); } while(false)
+// clang-format on
+
+#define DOCTEST_ASSERT_THROWS_AS(expr, assert_type, message, ...) \
+ do { \
+ if(!doctest::getContextOptions()->no_throw) { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #expr, #__VA_ARGS__, message); \
+ try { \
+ DOCTEST_CAST_TO_VOID(expr) \
+ } catch(const typename doctest::detail::remove_const< \
+ typename doctest::detail::remove_reference<__VA_ARGS__>::type>::type&) { \
+ _DOCTEST_RB.translateException(); \
+ _DOCTEST_RB.m_threw_as = true; \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_THROWS_WITH(expr, expr_str, assert_type, ...) \
+ do { \
+ if(!doctest::getContextOptions()->no_throw) { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, expr_str, "", __VA_ARGS__); \
+ try { \
+ DOCTEST_CAST_TO_VOID(expr) \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } \
+ } while(false)
+
+#define DOCTEST_ASSERT_NOTHROW(assert_type, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ try { \
+ DOCTEST_CAST_TO_VOID(__VA_ARGS__) \
+ } catch(...) { _DOCTEST_RB.translateException(); } \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+// clang-format off
+#define DOCTEST_WARN_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_WARN_THROWS, "")
+#define DOCTEST_CHECK_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_CHECK_THROWS, "")
+#define DOCTEST_REQUIRE_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_REQUIRE_THROWS, "")
+
+#define DOCTEST_WARN_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_WARN_THROWS_AS, "", __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_CHECK_THROWS_AS, "", __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_REQUIRE_THROWS_AS, "", __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_WARN_THROWS_WITH, __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_CHECK_THROWS_WITH, __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) DOCTEST_ASSERT_THROWS_WITH(expr, #expr, DT_REQUIRE_THROWS_WITH, __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_WARN_THROWS_WITH_AS, message, __VA_ARGS__)
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_CHECK_THROWS_WITH_AS, message, __VA_ARGS__)
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, message, ...) DOCTEST_ASSERT_THROWS_AS(expr, DT_REQUIRE_THROWS_WITH_AS, message, __VA_ARGS__)
+
+#define DOCTEST_WARN_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_WARN_NOTHROW, __VA_ARGS__)
+#define DOCTEST_CHECK_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_CHECK_NOTHROW, __VA_ARGS__)
+#define DOCTEST_REQUIRE_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_REQUIRE_NOTHROW, __VA_ARGS__)
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS(expr); } while(false)
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS(expr); } while(false)
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS(expr); } while(false)
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_AS(expr, ex); } while(false)
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH(expr, with); } while(false)
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ex); } while(false)
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_NOTHROW(expr); } while(false)
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_NOTHROW(expr); } while(false)
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_NOTHROW(expr); } while(false)
+// clang-format on
+
+#ifndef DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_BINARY_ASSERT(assert_type, comp, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY( \
+ _DOCTEST_RB.binary_assert<doctest::detail::binaryAssertComparison::comp>( \
+ __VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+#define DOCTEST_UNARY_ASSERT(assert_type, ...) \
+ do { \
+ doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \
+ __LINE__, #__VA_ARGS__); \
+ DOCTEST_WRAP_IN_TRY(_DOCTEST_RB.unary_assert(__VA_ARGS__)) \
+ DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \
+ } while(false)
+
+#else // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_BINARY_ASSERT(assert_type, comparison, ...) \
+ doctest::detail::binary_assert<doctest::detail::binaryAssertComparison::comparison>( \
+ doctest::assertType::assert_type, __FILE__, __LINE__, #__VA_ARGS__, __VA_ARGS__)
+
+#define DOCTEST_UNARY_ASSERT(assert_type, ...) \
+ doctest::detail::unary_assert(doctest::assertType::assert_type, __FILE__, __LINE__, \
+ #__VA_ARGS__, __VA_ARGS__)
+
+#endif // DOCTEST_CONFIG_SUPER_FAST_ASSERTS
+
+#define DOCTEST_WARN_EQ(...) DOCTEST_BINARY_ASSERT(DT_WARN_EQ, eq, __VA_ARGS__)
+#define DOCTEST_CHECK_EQ(...) DOCTEST_BINARY_ASSERT(DT_CHECK_EQ, eq, __VA_ARGS__)
+#define DOCTEST_REQUIRE_EQ(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_EQ, eq, __VA_ARGS__)
+#define DOCTEST_WARN_NE(...) DOCTEST_BINARY_ASSERT(DT_WARN_NE, ne, __VA_ARGS__)
+#define DOCTEST_CHECK_NE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_NE, ne, __VA_ARGS__)
+#define DOCTEST_REQUIRE_NE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_NE, ne, __VA_ARGS__)
+#define DOCTEST_WARN_GT(...) DOCTEST_BINARY_ASSERT(DT_WARN_GT, gt, __VA_ARGS__)
+#define DOCTEST_CHECK_GT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GT, gt, __VA_ARGS__)
+#define DOCTEST_REQUIRE_GT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GT, gt, __VA_ARGS__)
+#define DOCTEST_WARN_LT(...) DOCTEST_BINARY_ASSERT(DT_WARN_LT, lt, __VA_ARGS__)
+#define DOCTEST_CHECK_LT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LT, lt, __VA_ARGS__)
+#define DOCTEST_REQUIRE_LT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LT, lt, __VA_ARGS__)
+#define DOCTEST_WARN_GE(...) DOCTEST_BINARY_ASSERT(DT_WARN_GE, ge, __VA_ARGS__)
+#define DOCTEST_CHECK_GE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GE, ge, __VA_ARGS__)
+#define DOCTEST_REQUIRE_GE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GE, ge, __VA_ARGS__)
+#define DOCTEST_WARN_LE(...) DOCTEST_BINARY_ASSERT(DT_WARN_LE, le, __VA_ARGS__)
+#define DOCTEST_CHECK_LE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LE, le, __VA_ARGS__)
+#define DOCTEST_REQUIRE_LE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LE, le, __VA_ARGS__)
+
+#define DOCTEST_WARN_UNARY(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY, __VA_ARGS__)
+#define DOCTEST_CHECK_UNARY(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY, __VA_ARGS__)
+#define DOCTEST_REQUIRE_UNARY(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY, __VA_ARGS__)
+#define DOCTEST_WARN_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY_FALSE, __VA_ARGS__)
+#define DOCTEST_CHECK_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY_FALSE, __VA_ARGS__)
+#define DOCTEST_REQUIRE_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY_FALSE, __VA_ARGS__)
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS
+
+#undef DOCTEST_WARN_THROWS
+#undef DOCTEST_CHECK_THROWS
+#undef DOCTEST_REQUIRE_THROWS
+#undef DOCTEST_WARN_THROWS_AS
+#undef DOCTEST_CHECK_THROWS_AS
+#undef DOCTEST_REQUIRE_THROWS_AS
+#undef DOCTEST_WARN_THROWS_WITH
+#undef DOCTEST_CHECK_THROWS_WITH
+#undef DOCTEST_REQUIRE_THROWS_WITH
+#undef DOCTEST_WARN_THROWS_WITH_AS
+#undef DOCTEST_CHECK_THROWS_WITH_AS
+#undef DOCTEST_REQUIRE_THROWS_WITH_AS
+#undef DOCTEST_WARN_NOTHROW
+#undef DOCTEST_CHECK_NOTHROW
+#undef DOCTEST_REQUIRE_NOTHROW
+
+#undef DOCTEST_WARN_THROWS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_MESSAGE
+#undef DOCTEST_WARN_THROWS_AS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_AS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_AS_MESSAGE
+#undef DOCTEST_WARN_THROWS_WITH_MESSAGE
+#undef DOCTEST_CHECK_THROWS_WITH_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_WITH_MESSAGE
+#undef DOCTEST_WARN_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE
+#undef DOCTEST_WARN_NOTHROW_MESSAGE
+#undef DOCTEST_CHECK_NOTHROW_MESSAGE
+#undef DOCTEST_REQUIRE_NOTHROW_MESSAGE
+
+#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#define DOCTEST_WARN_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+
+#else // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#undef DOCTEST_REQUIRE
+#undef DOCTEST_REQUIRE_FALSE
+#undef DOCTEST_REQUIRE_MESSAGE
+#undef DOCTEST_REQUIRE_FALSE_MESSAGE
+#undef DOCTEST_REQUIRE_EQ
+#undef DOCTEST_REQUIRE_NE
+#undef DOCTEST_REQUIRE_GT
+#undef DOCTEST_REQUIRE_LT
+#undef DOCTEST_REQUIRE_GE
+#undef DOCTEST_REQUIRE_LE
+#undef DOCTEST_REQUIRE_UNARY
+#undef DOCTEST_REQUIRE_UNARY_FALSE
+
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS
+
+#endif // DOCTEST_CONFIG_NO_EXCEPTIONS
+
+// =================================================================================================
+// == WHAT FOLLOWS IS VERSIONS OF THE MACROS THAT DO NOT DO ANY REGISTERING! ==
+// == THIS CAN BE ENABLED BY DEFINING DOCTEST_CONFIG_DISABLE GLOBALLY! ==
+// =================================================================================================
+#else // DOCTEST_CONFIG_DISABLE
+
+#define DOCTEST_IMPLEMENT_FIXTURE(der, base, func, name) \
+ namespace { \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ struct der : public base \
+ { void f(); }; \
+ } \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ inline void der<DOCTEST_UNUSED_TEMPLATE_TYPE>::f()
+
+#define DOCTEST_CREATE_AND_REGISTER_FUNCTION(f, name) \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ static inline void f()
+
+// for registering tests
+#define DOCTEST_TEST_CASE(name) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for registering tests in classes
+#define DOCTEST_TEST_CASE_CLASS(name) \
+ DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for registering tests with a fixture
+#define DOCTEST_TEST_CASE_FIXTURE(x, name) \
+ DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(_DOCTEST_ANON_CLASS_), x, \
+ DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name)
+
+// for converting types to strings without the <typeinfo> header and demangling
+#define DOCTEST_TYPE_TO_STRING(...) typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+#define DOCTEST_TYPE_TO_STRING_IMPL(...)
+
+// for typed tests
+#define DOCTEST_TEST_CASE_TEMPLATE(name, type, ...) \
+ template <typename type> \
+ inline void DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_)()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_DEFINE(name, type, id) \
+ template <typename type> \
+ inline void DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_)()
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, ...) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, ...) \
+ typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for subcases
+#define DOCTEST_SUBCASE(name)
+
+// for a testsuite block
+#define DOCTEST_TEST_SUITE(name) namespace
+
+// for starting a testsuite block
+#define DOCTEST_TEST_SUITE_BEGIN(name) typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+// for ending a testsuite block
+#define DOCTEST_TEST_SUITE_END typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_)
+
+#define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature) \
+ template <typename DOCTEST_UNUSED_TEMPLATE_TYPE> \
+ static inline doctest::String DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_)(signature)
+
+#define DOCTEST_REGISTER_REPORTER(name, priority, reporter)
+#define DOCTEST_REGISTER_LISTENER(name, priority, reporter)
+
+#define DOCTEST_INFO(...) (static_cast<void>(0))
+#define DOCTEST_CAPTURE(x) (static_cast<void>(0))
+#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_ADD_FAIL_AT(file, line, ...) (static_cast<void>(0))
+#define DOCTEST_MESSAGE(...) (static_cast<void>(0))
+#define DOCTEST_FAIL_CHECK(...) (static_cast<void>(0))
+#define DOCTEST_FAIL(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN(...) (static_cast<void>(0))
+#define DOCTEST_CHECK(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_FALSE(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS(...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast<void>(0))
+#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_EQ(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_EQ(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_EQ(...) (static_cast<void>(0))
+#define DOCTEST_WARN_NE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_NE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_NE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_GT(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_GT(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_GT(...) (static_cast<void>(0))
+#define DOCTEST_WARN_LT(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_LT(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_LT(...) (static_cast<void>(0))
+#define DOCTEST_WARN_GE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_GE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_GE(...) (static_cast<void>(0))
+#define DOCTEST_WARN_LE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_LE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_LE(...) (static_cast<void>(0))
+
+#define DOCTEST_WARN_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_UNARY(...) (static_cast<void>(0))
+#define DOCTEST_WARN_UNARY_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_CHECK_UNARY_FALSE(...) (static_cast<void>(0))
+#define DOCTEST_REQUIRE_UNARY_FALSE(...) (static_cast<void>(0))
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+// clang-format off
+// KEPT FOR BACKWARDS COMPATIBILITY - FORWARDING TO THE RIGHT MACROS
+#define DOCTEST_FAST_WARN_EQ DOCTEST_WARN_EQ
+#define DOCTEST_FAST_CHECK_EQ DOCTEST_CHECK_EQ
+#define DOCTEST_FAST_REQUIRE_EQ DOCTEST_REQUIRE_EQ
+#define DOCTEST_FAST_WARN_NE DOCTEST_WARN_NE
+#define DOCTEST_FAST_CHECK_NE DOCTEST_CHECK_NE
+#define DOCTEST_FAST_REQUIRE_NE DOCTEST_REQUIRE_NE
+#define DOCTEST_FAST_WARN_GT DOCTEST_WARN_GT
+#define DOCTEST_FAST_CHECK_GT DOCTEST_CHECK_GT
+#define DOCTEST_FAST_REQUIRE_GT DOCTEST_REQUIRE_GT
+#define DOCTEST_FAST_WARN_LT DOCTEST_WARN_LT
+#define DOCTEST_FAST_CHECK_LT DOCTEST_CHECK_LT
+#define DOCTEST_FAST_REQUIRE_LT DOCTEST_REQUIRE_LT
+#define DOCTEST_FAST_WARN_GE DOCTEST_WARN_GE
+#define DOCTEST_FAST_CHECK_GE DOCTEST_CHECK_GE
+#define DOCTEST_FAST_REQUIRE_GE DOCTEST_REQUIRE_GE
+#define DOCTEST_FAST_WARN_LE DOCTEST_WARN_LE
+#define DOCTEST_FAST_CHECK_LE DOCTEST_CHECK_LE
+#define DOCTEST_FAST_REQUIRE_LE DOCTEST_REQUIRE_LE
+
+#define DOCTEST_FAST_WARN_UNARY DOCTEST_WARN_UNARY
+#define DOCTEST_FAST_CHECK_UNARY DOCTEST_CHECK_UNARY
+#define DOCTEST_FAST_REQUIRE_UNARY DOCTEST_REQUIRE_UNARY
+#define DOCTEST_FAST_WARN_UNARY_FALSE DOCTEST_WARN_UNARY_FALSE
+#define DOCTEST_FAST_CHECK_UNARY_FALSE DOCTEST_CHECK_UNARY_FALSE
+#define DOCTEST_FAST_REQUIRE_UNARY_FALSE DOCTEST_REQUIRE_UNARY_FALSE
+
+#define DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id,__VA_ARGS__)
+// clang-format on
+
+// BDD style macros
+// clang-format off
+#define DOCTEST_SCENARIO(name) DOCTEST_TEST_CASE(" Scenario: " name)
+#define DOCTEST_SCENARIO_CLASS(name) DOCTEST_TEST_CASE_CLASS(" Scenario: " name)
+#define DOCTEST_SCENARIO_TEMPLATE(name, T, ...) DOCTEST_TEST_CASE_TEMPLATE(" Scenario: " name, T, __VA_ARGS__)
+#define DOCTEST_SCENARIO_TEMPLATE_DEFINE(name, T, id) DOCTEST_TEST_CASE_TEMPLATE_DEFINE(" Scenario: " name, T, id)
+
+#define DOCTEST_GIVEN(name) DOCTEST_SUBCASE(" Given: " name)
+#define DOCTEST_WHEN(name) DOCTEST_SUBCASE(" When: " name)
+#define DOCTEST_AND_WHEN(name) DOCTEST_SUBCASE("And when: " name)
+#define DOCTEST_THEN(name) DOCTEST_SUBCASE(" Then: " name)
+#define DOCTEST_AND_THEN(name) DOCTEST_SUBCASE(" And: " name)
+// clang-format on
+
+// == SHORT VERSIONS OF THE MACROS
+#if !defined(DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES)
+
+#define TEST_CASE(name) DOCTEST_TEST_CASE(name)
+#define TEST_CASE_CLASS(name) DOCTEST_TEST_CASE_CLASS(name)
+#define TEST_CASE_FIXTURE(x, name) DOCTEST_TEST_CASE_FIXTURE(x, name)
+#define TYPE_TO_STRING(...) DOCTEST_TYPE_TO_STRING(__VA_ARGS__)
+#define TEST_CASE_TEMPLATE(name, T, ...) DOCTEST_TEST_CASE_TEMPLATE(name, T, __VA_ARGS__)
+#define TEST_CASE_TEMPLATE_DEFINE(name, T, id) DOCTEST_TEST_CASE_TEMPLATE_DEFINE(name, T, id)
+#define TEST_CASE_TEMPLATE_INVOKE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, __VA_ARGS__)
+#define TEST_CASE_TEMPLATE_APPLY(id, ...) DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, __VA_ARGS__)
+#define SUBCASE(name) DOCTEST_SUBCASE(name)
+#define TEST_SUITE(decorators) DOCTEST_TEST_SUITE(decorators)
+#define TEST_SUITE_BEGIN(name) DOCTEST_TEST_SUITE_BEGIN(name)
+#define TEST_SUITE_END DOCTEST_TEST_SUITE_END
+#define REGISTER_EXCEPTION_TRANSLATOR(signature) DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature)
+#define REGISTER_REPORTER(name, priority, reporter) DOCTEST_REGISTER_REPORTER(name, priority, reporter)
+#define REGISTER_LISTENER(name, priority, reporter) DOCTEST_REGISTER_LISTENER(name, priority, reporter)
+#define INFO(...) DOCTEST_INFO(__VA_ARGS__)
+#define CAPTURE(x) DOCTEST_CAPTURE(x)
+#define ADD_MESSAGE_AT(file, line, ...) DOCTEST_ADD_MESSAGE_AT(file, line, __VA_ARGS__)
+#define ADD_FAIL_CHECK_AT(file, line, ...) DOCTEST_ADD_FAIL_CHECK_AT(file, line, __VA_ARGS__)
+#define ADD_FAIL_AT(file, line, ...) DOCTEST_ADD_FAIL_AT(file, line, __VA_ARGS__)
+#define MESSAGE(...) DOCTEST_MESSAGE(__VA_ARGS__)
+#define FAIL_CHECK(...) DOCTEST_FAIL_CHECK(__VA_ARGS__)
+#define FAIL(...) DOCTEST_FAIL(__VA_ARGS__)
+#define TO_LVALUE(...) DOCTEST_TO_LVALUE(__VA_ARGS__)
+
+#define WARN(...) DOCTEST_WARN(__VA_ARGS__)
+#define WARN_FALSE(...) DOCTEST_WARN_FALSE(__VA_ARGS__)
+#define WARN_THROWS(...) DOCTEST_WARN_THROWS(__VA_ARGS__)
+#define WARN_THROWS_AS(expr, ...) DOCTEST_WARN_THROWS_AS(expr, __VA_ARGS__)
+#define WARN_THROWS_WITH(expr, ...) DOCTEST_WARN_THROWS_WITH(expr, __VA_ARGS__)
+#define WARN_THROWS_WITH_AS(expr, with, ...) DOCTEST_WARN_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define WARN_NOTHROW(...) DOCTEST_WARN_NOTHROW(__VA_ARGS__)
+#define CHECK(...) DOCTEST_CHECK(__VA_ARGS__)
+#define CHECK_FALSE(...) DOCTEST_CHECK_FALSE(__VA_ARGS__)
+#define CHECK_THROWS(...) DOCTEST_CHECK_THROWS(__VA_ARGS__)
+#define CHECK_THROWS_AS(expr, ...) DOCTEST_CHECK_THROWS_AS(expr, __VA_ARGS__)
+#define CHECK_THROWS_WITH(expr, ...) DOCTEST_CHECK_THROWS_WITH(expr, __VA_ARGS__)
+#define CHECK_THROWS_WITH_AS(expr, with, ...) DOCTEST_CHECK_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define CHECK_NOTHROW(...) DOCTEST_CHECK_NOTHROW(__VA_ARGS__)
+#define REQUIRE(...) DOCTEST_REQUIRE(__VA_ARGS__)
+#define REQUIRE_FALSE(...) DOCTEST_REQUIRE_FALSE(__VA_ARGS__)
+#define REQUIRE_THROWS(...) DOCTEST_REQUIRE_THROWS(__VA_ARGS__)
+#define REQUIRE_THROWS_AS(expr, ...) DOCTEST_REQUIRE_THROWS_AS(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH(expr, ...) DOCTEST_REQUIRE_THROWS_WITH(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_AS(expr, with, ...) DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, __VA_ARGS__)
+#define REQUIRE_NOTHROW(...) DOCTEST_REQUIRE_NOTHROW(__VA_ARGS__)
+
+#define WARN_MESSAGE(cond, ...) DOCTEST_WARN_MESSAGE(cond, __VA_ARGS__)
+#define WARN_FALSE_MESSAGE(cond, ...) DOCTEST_WARN_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define WARN_THROWS_MESSAGE(expr, ...) DOCTEST_WARN_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define WARN_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define WARN_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define WARN_NOTHROW_MESSAGE(expr, ...) DOCTEST_WARN_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+#define CHECK_MESSAGE(cond, ...) DOCTEST_CHECK_MESSAGE(cond, __VA_ARGS__)
+#define CHECK_FALSE_MESSAGE(cond, ...) DOCTEST_CHECK_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define CHECK_THROWS_MESSAGE(expr, ...) DOCTEST_CHECK_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define CHECK_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define CHECK_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define CHECK_NOTHROW_MESSAGE(expr, ...) DOCTEST_CHECK_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+#define REQUIRE_MESSAGE(cond, ...) DOCTEST_REQUIRE_MESSAGE(cond, __VA_ARGS__)
+#define REQUIRE_FALSE_MESSAGE(cond, ...) DOCTEST_REQUIRE_FALSE_MESSAGE(cond, __VA_ARGS__)
+#define REQUIRE_THROWS_MESSAGE(expr, ...) DOCTEST_REQUIRE_THROWS_MESSAGE(expr, __VA_ARGS__)
+#define REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, __VA_ARGS__)
+#define REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, __VA_ARGS__)
+#define REQUIRE_NOTHROW_MESSAGE(expr, ...) DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, __VA_ARGS__)
+
+#define SCENARIO(name) DOCTEST_SCENARIO(name)
+#define SCENARIO_CLASS(name) DOCTEST_SCENARIO_CLASS(name)
+#define SCENARIO_TEMPLATE(name, T, ...) DOCTEST_SCENARIO_TEMPLATE(name, T, __VA_ARGS__)
+#define SCENARIO_TEMPLATE_DEFINE(name, T, id) DOCTEST_SCENARIO_TEMPLATE_DEFINE(name, T, id)
+#define GIVEN(name) DOCTEST_GIVEN(name)
+#define WHEN(name) DOCTEST_WHEN(name)
+#define AND_WHEN(name) DOCTEST_AND_WHEN(name)
+#define THEN(name) DOCTEST_THEN(name)
+#define AND_THEN(name) DOCTEST_AND_THEN(name)
+
+#define WARN_EQ(...) DOCTEST_WARN_EQ(__VA_ARGS__)
+#define CHECK_EQ(...) DOCTEST_CHECK_EQ(__VA_ARGS__)
+#define REQUIRE_EQ(...) DOCTEST_REQUIRE_EQ(__VA_ARGS__)
+#define WARN_NE(...) DOCTEST_WARN_NE(__VA_ARGS__)
+#define CHECK_NE(...) DOCTEST_CHECK_NE(__VA_ARGS__)
+#define REQUIRE_NE(...) DOCTEST_REQUIRE_NE(__VA_ARGS__)
+#define WARN_GT(...) DOCTEST_WARN_GT(__VA_ARGS__)
+#define CHECK_GT(...) DOCTEST_CHECK_GT(__VA_ARGS__)
+#define REQUIRE_GT(...) DOCTEST_REQUIRE_GT(__VA_ARGS__)
+#define WARN_LT(...) DOCTEST_WARN_LT(__VA_ARGS__)
+#define CHECK_LT(...) DOCTEST_CHECK_LT(__VA_ARGS__)
+#define REQUIRE_LT(...) DOCTEST_REQUIRE_LT(__VA_ARGS__)
+#define WARN_GE(...) DOCTEST_WARN_GE(__VA_ARGS__)
+#define CHECK_GE(...) DOCTEST_CHECK_GE(__VA_ARGS__)
+#define REQUIRE_GE(...) DOCTEST_REQUIRE_GE(__VA_ARGS__)
+#define WARN_LE(...) DOCTEST_WARN_LE(__VA_ARGS__)
+#define CHECK_LE(...) DOCTEST_CHECK_LE(__VA_ARGS__)
+#define REQUIRE_LE(...) DOCTEST_REQUIRE_LE(__VA_ARGS__)
+#define WARN_UNARY(...) DOCTEST_WARN_UNARY(__VA_ARGS__)
+#define CHECK_UNARY(...) DOCTEST_CHECK_UNARY(__VA_ARGS__)
+#define REQUIRE_UNARY(...) DOCTEST_REQUIRE_UNARY(__VA_ARGS__)
+#define WARN_UNARY_FALSE(...) DOCTEST_WARN_UNARY_FALSE(__VA_ARGS__)
+#define CHECK_UNARY_FALSE(...) DOCTEST_CHECK_UNARY_FALSE(__VA_ARGS__)
+#define REQUIRE_UNARY_FALSE(...) DOCTEST_REQUIRE_UNARY_FALSE(__VA_ARGS__)
+
+// KEPT FOR BACKWARDS COMPATIBILITY
+#define FAST_WARN_EQ(...) DOCTEST_FAST_WARN_EQ(__VA_ARGS__)
+#define FAST_CHECK_EQ(...) DOCTEST_FAST_CHECK_EQ(__VA_ARGS__)
+#define FAST_REQUIRE_EQ(...) DOCTEST_FAST_REQUIRE_EQ(__VA_ARGS__)
+#define FAST_WARN_NE(...) DOCTEST_FAST_WARN_NE(__VA_ARGS__)
+#define FAST_CHECK_NE(...) DOCTEST_FAST_CHECK_NE(__VA_ARGS__)
+#define FAST_REQUIRE_NE(...) DOCTEST_FAST_REQUIRE_NE(__VA_ARGS__)
+#define FAST_WARN_GT(...) DOCTEST_FAST_WARN_GT(__VA_ARGS__)
+#define FAST_CHECK_GT(...) DOCTEST_FAST_CHECK_GT(__VA_ARGS__)
+#define FAST_REQUIRE_GT(...) DOCTEST_FAST_REQUIRE_GT(__VA_ARGS__)
+#define FAST_WARN_LT(...) DOCTEST_FAST_WARN_LT(__VA_ARGS__)
+#define FAST_CHECK_LT(...) DOCTEST_FAST_CHECK_LT(__VA_ARGS__)
+#define FAST_REQUIRE_LT(...) DOCTEST_FAST_REQUIRE_LT(__VA_ARGS__)
+#define FAST_WARN_GE(...) DOCTEST_FAST_WARN_GE(__VA_ARGS__)
+#define FAST_CHECK_GE(...) DOCTEST_FAST_CHECK_GE(__VA_ARGS__)
+#define FAST_REQUIRE_GE(...) DOCTEST_FAST_REQUIRE_GE(__VA_ARGS__)
+#define FAST_WARN_LE(...) DOCTEST_FAST_WARN_LE(__VA_ARGS__)
+#define FAST_CHECK_LE(...) DOCTEST_FAST_CHECK_LE(__VA_ARGS__)
+#define FAST_REQUIRE_LE(...) DOCTEST_FAST_REQUIRE_LE(__VA_ARGS__)
+
+#define FAST_WARN_UNARY(...) DOCTEST_FAST_WARN_UNARY(__VA_ARGS__)
+#define FAST_CHECK_UNARY(...) DOCTEST_FAST_CHECK_UNARY(__VA_ARGS__)
+#define FAST_REQUIRE_UNARY(...) DOCTEST_FAST_REQUIRE_UNARY(__VA_ARGS__)
+#define FAST_WARN_UNARY_FALSE(...) DOCTEST_FAST_WARN_UNARY_FALSE(__VA_ARGS__)
+#define FAST_CHECK_UNARY_FALSE(...) DOCTEST_FAST_CHECK_UNARY_FALSE(__VA_ARGS__)
+#define FAST_REQUIRE_UNARY_FALSE(...) DOCTEST_FAST_REQUIRE_UNARY_FALSE(__VA_ARGS__)
+
+#define TEST_CASE_TEMPLATE_INSTANTIATE(id, ...) DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE(id, __VA_ARGS__)
+
+#endif // DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES
+
+#if !defined(DOCTEST_CONFIG_DISABLE)
+
+// this is here to clear the 'current test suite' for the current translation unit - at the top
+DOCTEST_TEST_SUITE_END();
+
+// add stringification for primitive/fundamental types
+namespace doctest { namespace detail {
+ DOCTEST_TYPE_TO_STRING_IMPL(bool)
+ DOCTEST_TYPE_TO_STRING_IMPL(float)
+ DOCTEST_TYPE_TO_STRING_IMPL(double)
+ DOCTEST_TYPE_TO_STRING_IMPL(long double)
+ DOCTEST_TYPE_TO_STRING_IMPL(char)
+ DOCTEST_TYPE_TO_STRING_IMPL(signed char)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned char)
+#if !DOCTEST_MSVC || defined(_NATIVE_WCHAR_T_DEFINED)
+ DOCTEST_TYPE_TO_STRING_IMPL(wchar_t)
+#endif // not MSVC or wchar_t support enabled
+ DOCTEST_TYPE_TO_STRING_IMPL(short int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned short int)
+ DOCTEST_TYPE_TO_STRING_IMPL(int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned int)
+ DOCTEST_TYPE_TO_STRING_IMPL(long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(long long int)
+ DOCTEST_TYPE_TO_STRING_IMPL(unsigned long long int)
+}} // namespace doctest::detail
+
+#endif // DOCTEST_CONFIG_DISABLE
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_MSVC_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP
+
+#endif // DOCTEST_LIBRARY_INCLUDED
diff --git a/contrib/elastic/kibana.json b/contrib/elastic/kibana.json
new file mode 100644
index 0000000..17b68b6
--- /dev/null
+++ b/contrib/elastic/kibana.json
@@ -0,0 +1,138 @@
+[
+ {
+ "_type": "index-pattern",
+ "_id": "eb48a1c0-23a2-11e8-b222-e710267d9b66",
+ "_score": 1,
+ "_source": {
+ "type": "index-pattern",
+ "index-pattern": {
+ "title": "rspamd-*",
+ "timeFieldName": "@timestamp",
+ "fields": "[{\"name\":\"rspamd_meta.action\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.asn.asn\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.asn.country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.asn.ipnet\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.direction\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.city_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.continent_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.country_iso_code\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.location.lat\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.location.lon\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.geoip.region_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.header_date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.header_from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.header_subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.header_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.is_local\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.message_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.qid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.rcpt\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.symbols.group\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.symbols.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.symbols.options\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.symbols.score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.user\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rspamd_meta.webmail\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
+ "sourceFilters": "[{\"value\":\"rspamd_meta*\"}]"
+ }
+ }
+ },
+ {
+ "_id": "6c6a2ed0-8660-11e7-85ae-fbc80f1b7844",
+ "_type": "dashboard",
+ "_source": {
+ "type": "dashboard",
+ "dashboard": {
+ "title": "Rspamd Dashboard",
+ "hits": 0,
+ "description": "",
+ "panelsJSON": "[{\"size_x\":6,\"size_y\":3,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"6413f870-80f6-11e7-91e6-0986b0b459e7\",\"col\":1,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":2,\"type\":\"visualization\",\"id\":\"927debf0-8649-11e7-967f-798bfd7ac13a\",\"col\":7,\"row\":1},{\"size_x\":12,\"size_y\":3,\"panelIndex\":3,\"type\":\"visualization\",\"id\":\"efa3f7a0-80f6-11e7-91e6-0986b0b459e7\",\"col\":1,\"row\":7},{\"size_x\":12,\"size_y\":3,\"panelIndex\":4,\"type\":\"visualization\",\"id\":\"1f7d9210-80f7-11e7-91e6-0986b0b459e7\",\"col\":1,\"row\":10},{\"size_x\":6,\"size_y\":3,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"2be7b6f0-8649-11e7-967f-798bfd7ac13a\",\"col\":7,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":6,\"type\":\"visualization\",\"id\":\"680b6480-826e-11e7-8a20-b7bc68c2e9e7\",\"col\":7,\"row\":13},{\"size_x\":6,\"size_y\":3,\"panelIndex\":7,\"type\":\"visualization\",\"id\":\"158dfc80-864d-11e7-bce7-4532b9d239a0\",\"col\":1,\"row\":4}]",
+ "optionsJSON": "{\"darkTheme\":false}",
+ "uiStateJSON": "{\"P-3\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-4\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-1\":{\"mapZoom\":2,\"mapCenter\":[40.58058466412761,1.7578125]},\"P-6\":{\"vis\":{\"defaultColors\":{\"0 - 0.25\":\"rgb(247,252,245)\",\"0.25 - 0.5\":\"rgb(199,233,192)\",\"0.5 - 0.75\":\"rgb(116,196,118)\",\"0.75 - 1\":\"rgb(35,139,69)\"}}}}",
+ "version": 1,
+ "timeRestore": false,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}],\"highlightAll\":true,\"version\":true}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "927debf0-8649-11e7-967f-798bfd7ac13a",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Actions",
+ "visState": "{\"title\":\"Rspamd Actions\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"rspamd_meta.action\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+ "uiStateJSON": "{}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"index\":\"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "6413f870-80f6-11e7-91e6-0986b0b459e7",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Geo Map",
+ "visState": "{\n \"title\": \"Rspamd Geo Map\",\n \"type\": \"tile_map\",\n \"params\": {\n \"mapType\": \"Scaled Circle Markers\",\n \"isDesaturated\": true,\n \"addTooltip\": true,\n \"heatMaxZoom\": 0,\n \"heatMinOpacity\": 0.1,\n \"heatRadius\": 25,\n \"heatBlur\": 15,\n \"legendPosition\": \"bottomright\",\n \"mapZoom\": 2,\n \"mapCenter\": [\n 0,\n 0\n ],\n \"wms\": {\n \"enabled\": false,\n \"url\": \"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\",\n \"options\": {\n \"version\": \"1.3.0\",\n \"layers\": \"0\",\n \"format\": \"image/png\",\n \"transparent\": true,\n \"attribution\": \"Maps provided by USGS\",\n \"styles\": \"\"\n }\n }\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"geohash_grid\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"rspamd_meta.geoip.location\",\n \"autoPrecision\": true,\n \"useGeocentroid\": true,\n \"precision\": 2\n }\n }\n ],\n \"listeners\": {}\n}",
+ "uiStateJSON": "{}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\n \"index\": \"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "92a92c00-80f6-11e7-91e6-0986b0b459e7",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Spam Map",
+ "visState": "{\n \"title\": \"Rspamd Spam Map\",\n \"type\": \"tile_map\",\n \"params\": {\n \"mapType\": \"Scaled Circle Markers\",\n \"isDesaturated\": true,\n \"addTooltip\": true,\n \"heatMaxZoom\": 0,\n \"heatMinOpacity\": 0.1,\n \"heatRadius\": 25,\n \"heatBlur\": 15,\n \"legendPosition\": \"bottomright\",\n \"mapZoom\": 2,\n \"mapCenter\": [\n 0,\n 0\n ],\n \"wms\": {\n \"enabled\": false,\n \"url\": \"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\",\n \"options\": {\n \"version\": \"1.3.0\",\n \"layers\": \"0\",\n \"format\": \"image/png\",\n \"transparent\": true,\n \"attribution\": \"Maps provided by USGS\",\n \"styles\": \"\"\n }\n }\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"geohash_grid\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"rspamd_meta.geoip.location\",\n \"autoPrecision\": true,\n \"useGeocentroid\": true,\n \"precision\": 2\n }\n }\n ],\n \"listeners\": {}\n}",
+ "uiStateJSON": "{}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\n \"index\": \"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": [\n {\n \"meta\": {\n \"index\": \"rspamd_beat-*\",\n \"negate\": true,\n \"disabled\": false,\n \"alias\": null,\n \"type\": \"phrase\",\n \"key\": \"rspamd_meta.action\",\n \"value\": \"no action\"\n },\n \"query\": {\n \"match\": {\n \"rspamd_meta.action\": {\n \"query\": \"no action\",\n \"type\": \"phrase\"\n }\n }\n },\n \"$state\": {\n \"store\": \"appState\"\n }\n }\n ]\n}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "2be7b6f0-8649-11e7-967f-798bfd7ac13a",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Symbols Cloud",
+ "visState": "{\"title\":\"Rspamd Symbols Cloud\",\"type\":\"tagcloud\",\"params\":{\"scale\":\"linear\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"rspamd_meta.symbols.name\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+ "uiStateJSON": "{}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"index\":\"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "1f7d9210-80f7-11e7-91e6-0986b0b459e7",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Top recipients",
+ "visState": "{\n \"title\": \"Rspamd Top recipients\",\n \"type\": \"metric\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": false,\n \"type\": \"gauge\",\n \"gauge\": {\n \"verticalSplit\": false,\n \"autoExtend\": false,\n \"percentageMode\": false,\n \"gaugeType\": \"Metric\",\n \"gaugeStyle\": \"Full\",\n \"backStyle\": \"Full\",\n \"orientation\": \"vertical\",\n \"colorSchema\": \"Green to Red\",\n \"gaugeColorMode\": \"None\",\n \"useRange\": false,\n \"colorsRange\": [\n {\n \"from\": 0,\n \"to\": 100\n }\n ],\n \"invertColors\": false,\n \"labels\": {\n \"show\": true,\n \"color\": \"black\"\n },\n \"scale\": {\n \"show\": false,\n \"labels\": false,\n \"color\": \"#333\",\n \"width\": 2\n },\n \"type\": \"simple\",\n \"style\": {\n \"fontSize\": 60,\n \"bgFill\": \"#000\",\n \"bgColor\": false,\n \"labelColor\": false,\n \"subText\": \"\"\n }\n }\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"rspamd_meta.rcpt\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}",
+ "uiStateJSON": "{\n \"vis\": {\n \"defaultColors\": {\n \"0 - 100\": \"rgb(0,104,55)\"\n }\n }\n}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\n \"index\": \"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}"
+ }
+ }
+ }
+ },
+ {
+ "_id": "efa3f7a0-80f6-11e7-91e6-0986b0b459e7",
+ "_type": "visualization",
+ "_source": {
+ "type": "visualization",
+ "visualization": {
+ "title": "Rspamd Top Senders",
+ "visState": "{\n \"title\": \"Rspamd Top Senders\",\n \"type\": \"metric\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": false,\n \"type\": \"gauge\",\n \"gauge\": {\n \"verticalSplit\": false,\n \"autoExtend\": false,\n \"percentageMode\": false,\n \"gaugeType\": \"Metric\",\n \"gaugeStyle\": \"Full\",\n \"backStyle\": \"Full\",\n \"orientation\": \"vertical\",\n \"colorSchema\": \"Green to Red\",\n \"gaugeColorMode\": \"None\",\n \"useRange\": false,\n \"colorsRange\": [\n {\n \"from\": 0,\n \"to\": 100\n }\n ],\n \"invertColors\": false,\n \"labels\": {\n \"show\": true,\n \"color\": \"black\"\n },\n \"scale\": {\n \"show\": false,\n \"labels\": false,\n \"color\": \"#333\",\n \"width\": 2\n },\n \"type\": \"simple\",\n \"style\": {\n \"fontSize\": 60,\n \"bgFill\": \"#000\",\n \"bgColor\": false,\n \"labelColor\": false,\n \"subText\": \"\"\n }\n }\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"rspamd_meta.user\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}",
+ "uiStateJSON": "{\n \"vis\": {\n \"defaultColors\": {\n \"0 - 100\": \"rgb(0,104,55)\"\n }\n }\n}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\n \"index\": \"eb48a1c0-23a2-11e8-b222-e710267d9b66\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": [\n {\n \"meta\": {\n \"index\": \"rspamd_beat-*\",\n \"negate\": true,\n \"disabled\": false,\n \"alias\": null,\n \"type\": \"phrase\",\n \"key\": \"rspamd_meta.user\",\n \"value\": \"unknown\"\n },\n \"query\": {\n \"match\": {\n \"rspamd_meta.user\": {\n \"query\": \"unknown\",\n \"type\": \"phrase\"\n }\n }\n },\n \"$state\": {\n \"store\": \"appState\"\n }\n }\n ]\n}"
+ }
+ }
+ }
+ }
+]
diff --git a/contrib/elastic/rspamd_template.json b/contrib/elastic/rspamd_template.json
new file mode 100644
index 0000000..ebd87fa
--- /dev/null
+++ b/contrib/elastic/rspamd_template.json
@@ -0,0 +1,149 @@
+{
+ "mappings": {
+ "_meta": {
+ "version": "5.5.3"
+ },
+ "date_detection": false,
+ "dynamic_templates": [
+ {
+ "strings_as_keyword": {
+ "mapping": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "match_mapping_type": "string"
+ }
+ }
+ ],
+ "properties": {
+ "@timestamp": {
+ "type": "date"
+ },
+ "meta": {
+ "properties": {
+ "cloud": {
+ "properties": {
+ "availability_zone": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "instance_id": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "machine_type": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "project_id": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "provider": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "region": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "rspamd_meta": {
+ "properties": {
+ "action": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "direction": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "asn": {
+ "properties": {
+ "asn": {
+ "type": "long"
+ },
+ "country_code": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "ipnet": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "registrant": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "from": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "is_local": {
+ "type": "boolean"
+ },
+ "webmail": {
+ "type": "boolean"
+ },
+ "sender_ip": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "geoip": {
+ "properties": {
+ "city_name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "continent_name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "country_iso_code": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "location": {
+ "type": "geo_point"
+ }
+ }
+ },
+ "ip": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "qid": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "hostname": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "score": {
+ "type": "float"
+ },
+ "user": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "tags": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "order": 0,
+ "settings": {
+ "index.mapping.total_fields.limit": 10000,
+ "index.refresh_interval": "5s"
+ },
+ "index_patterns" : ["rspamd-*", "*-rspamd-*"]
+}
diff --git a/contrib/exim/dlfunc-json/README b/contrib/exim/dlfunc-json/README
new file mode 100644
index 0000000..0e52270
--- /dev/null
+++ b/contrib/exim/dlfunc-json/README
@@ -0,0 +1,10 @@
+FEATURES
+1) Support new http-protocol (/checkv2)
+2) Return action, symbols, symbol options, messages, scan time
+
+INSTALL
+
+1) Get cJSON.c, cJSON.h, put to dlfunc src dir (https://github.com/DaveGamble/cJSON)
+2) Compile dlfunc library:
+ cc rspamd.c -fPIC -fpic -shared -I/root/rpmbuild/BUILD/exim-4.89/build-Linux-x86_64/ -o exim-rspamd-http-dlfunc.so
+3) See exim-example.txt for exim configure
diff --git a/contrib/exim/dlfunc-json/exim-example.txt b/contrib/exim/dlfunc-json/exim-example.txt
new file mode 100644
index 0000000..e6da334
--- /dev/null
+++ b/contrib/exim/dlfunc-json/exim-example.txt
@@ -0,0 +1,75 @@
+acl_smtp_data = acl_check_data
+
+.....
+
+acl_check_data:
+
+.....
+
+# RSPAMD: START
+ warn
+ !authenticated = *
+ add_header = X-Spam-Checker-Version: Rspamd
+ add_header = :at_start:Authentication-Results: ip=$sender_host_address:$sender_host_port, host=$sender_host_name, helo=$sender_helo_name, mailfrom=$sender_address
+ warn
+ #spam = nobody:true
+ #set acl_m0_rspamd = $spam_report
+ set acl_m0_rspamd = ${dlfunc{/usr/local/libexec/exim/exim-rspamd-http-dlfunc.so}{rspamd}{/var/run/rspamd/rspamd.sock}{defer_ok}}
+ accept
+ authenticated = *
+ warn
+ condition = ${if eq{$acl_m0_rspamd}{}}
+ logwrite = RSPAMD check failed
+ add_header = X-Spam-Info: Check failed
+ warn
+ condition = ${if match{$acl_m0_rspamd}{\N^rspamd dlfunc:\s*\N}{yes}{no}}
+ logwrite = RSPAMD check defer: ${sg{$acl_m0_rspamd}{\N^rspamd dlfunc:\s*\N}{}}
+ add_header = X-Spam-Info: Check deffered
+
+ warn
+ remove_header = X-Spam-Checker-Version:X-Spam-Status:X-Spam-Info:X-Spam-Result
+ set acl_m1 = No
+ warn
+ condition = ${if !eq{$acl_m0_rspamd}{}}
+ set acl_m1_yesno = ${if match{$acl_m0_rspamd}{\NAction: (.+?)\n\N}{$1}{}}
+ set acl_m2_status = ${if eq{$acl_m1_yesno}{reject}{REJECT}{\
+ ${if eq{$acl_m1_yesno}{add header}{PROBABLY}{\
+ ${if eq{$acl_m1_yesno}{rewrite subject}{PROBABLY}{\
+ ${if eq{$acl_m1_yesno}{soft reject}{SOFT}{\
+ ${if eq{$acl_m1_yesno}{greylist}{GREYLIST}{NO}}\
+ }}\
+ }}\
+ }}\
+ }}
+ set acl_m1_yesno = ${if eq{$acl_m1_yesno}{}{unknown}{\
+ ${if eq{$acl_m1_yesno}{reject}{Yes}{\
+ ${if eq{$acl_m1_yesno}{add header}{Yes}{\
+ ${if eq{$acl_m1_yesno}{rewrite subject}{Yes}{\
+ ${if eq{$acl_m1_yesno}{soft reject}{Probably}{\
+ ${if eq{$acl_m1_yesno}{greylist}{Probably}{No}}\
+ }}\
+ }}\
+ }}\
+ }}\
+ }}
+ #logwrite = RSPAMD: status: $acl_m2_status
+ #logwrite = RSPAMD DEBUG: $acl_m0_rspamd
+ set acl_m0_rspamd = ${sg{$acl_m0_rspamd}{ Action:.+\n}{}}
+ warn
+ condition = ${if !eq{$acl_m0_rspamd}{}}
+ logwrite = RSPAMD: $acl_m2_status, $acl_m0_rspamd
+ add_header = X-Spam-Result: $acl_m0_rspamd
+ add_header = X-Spam-Status: $acl_m1_yesno
+ defer
+ condition = ${if eq{$acl_m2_status}{GREYLIST}}
+ log_message = Rspamd $acl_m2_status
+ message = Try again later. Message greylisted
+ defer
+ condition = ${if eq{$acl_m2_status}{SOFT}}
+ log_message = Rspamd $acl_m2_status
+ message = Try again later. Message previously greylisted
+ deny
+ condition = ${if eq{$acl_m2_status}{REJECT}}
+ log_message = Rspamd $acl_m2_status
+ message = This message detected as SPAM and rejected
+# RSPAMD: END
diff --git a/contrib/exim/dlfunc-json/rspamd.c b/contrib/exim/dlfunc-json/rspamd.c
new file mode 100644
index 0000000..2623c7d
--- /dev/null
+++ b/contrib/exim/dlfunc-json/rspamd.c
@@ -0,0 +1,576 @@
+/*-
+ * Copyright (c) 2013-2015, Alexey Savelyev <info@homeweb.ru>
+ * Copyright 2017 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+
+ * Based on source code from http://www.ols.es/exim/dlext/ by David Saez <david@ols.es>,
+ * source code of exim by Philip Hazel <ph10@cam.ac.uk>
+ * and source code of exiscan by Tom Kistner <tom@duncanthrax.net>
+ * and source code of Victor Ustugov http://mta.org.ua/
+*/
+
+#include "exim.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdarg.h>
+#include <errno.h>
+#include "cJSON.h"
+#include "cJSON.c"
+
+#define RSPAMD_TIMEOUT 120
+
+extern uschar *tod_stamp (int);
+//extern BOOL split_spool_directory; /* TRUE to use multiple subdirs */
+//extern uschar *spool_directory; /* Name of spool directory */
+//extern uschar message_subdir[]; /* Subdirectory for messages */
+
+//-------------------------------------------------------------------------
+
+int
+rspamd (uschar **yield, int argc, uschar *argv[]) {
+ char *arg_socket_addr;
+ char *arg_defer_ok;
+ int defer_ok;
+ int rspamd_command;
+ char tcp_addr[15];
+ int tcp_port;
+ FILE *mbox_file = NULL;
+// unsigned long mbox_size;
+ off_t mbox_size;
+ uschar *s, *p;
+ header_line *my_header, *header_new, *header_last, *tmp_headerlist;
+ header_line *last_received = NULL;
+ uschar *address;
+ uschar *helo;
+ uschar *sender_host_name;
+ uschar *authenticated_id;
+ char mbox_path[8192];
+ int max_len, len;
+ int rspamd_sock = 0;
+ struct hostent *he;
+ struct in_addr in;
+ struct sockaddr_un server;
+#ifndef NO_POLL_H
+ int result;
+ struct pollfd pollfd;
+#endif
+ int offset;
+ uschar spamd_buffer[32600];
+ uschar spamd_buffer2[32600];
+ time_t start;
+ size_t read, wrote;
+ int i, j, c;
+
+ arg_socket_addr = argv[0];
+ arg_defer_ok = argv[1];
+
+ if (argc < 2) {
+ defer_ok = 0;
+ } else if (strcmpic (arg_defer_ok, US"1") == 0)
+ || (strcmpic (arg_defer_ok, US
+ "yes") == 0)
+ || (strcmpic (arg_defer_ok, US
+ "true") == 0)
+ || (strcmpic (arg_defer_ok, US
+ "defer_ok") == 0) {
+ defer_ok = 1;
+ } else {
+ defer_ok = 0;
+ }
+
+ if ((arg_socket_addr == NULL) || (arg_socket_addr[0] == 0)) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: Socket address expected");
+ *yield = string_sprintf ("rspamd dlfunc: Socket address expected");
+ goto RETURN_DEFER;
+ }
+
+
+ if (split_spool_directory == 0) {
+ snprintf (mbox_path, sizeof (mbox_path), "%s/input/%s-D",
+ spool_directory,
+ message_id);
+ } else {
+ snprintf (mbox_path, sizeof (mbox_path), "%s/input/%s/%s-D",
+ spool_directory, message_subdir, message_id);
+ }
+
+ mbox_file = fopen (mbox_path, "rb");
+
+ if (!mbox_file) {
+ *yield = string_sprintf ("rspamd dlfunc: Unable to spool message '%s'",
+ mbox_path);
+ return (defer_ok ? OK : ERROR);
+ }
+
+ (void) fseek (mbox_file, 0, SEEK_END);
+ mbox_size = ftell (mbox_file);
+//debug_printf(" Total spool file size: %d\n", mbox_size);
+ mbox_size -= SPOOL_DATA_START_OFFSET;
+//debug_printf(" Spool file size: %d\n", mbox_size);
+//debug_printf(" fseek %d, %d\n", SPOOL_DATA_START_OFFSET, SEEK_SET);
+ (void) fseek (mbox_file, SPOOL_DATA_START_OFFSET, SEEK_SET);
+
+ start = time (NULL);
+ /* socket does not start with '/' -> network socket */
+ if (arg_socket_addr[0] != '/') {
+ if (sscanf (CS arg_socket_addr, "%s %u", tcp_addr, &tcp_port) != 2 ) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: Invalid rspamd address: '%s'",
+ arg_socket_addr);
+ *yield = string_sprintf (
+ "rspamd dlfunc: Invalid rspamd address: '%s'",
+ arg_socket_addr);
+ goto RETURN_DEFER;
+ }
+
+ /* Lookup the host */
+ if ((he = gethostbyname (CS tcp_addr)) == 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: failed to lookup host '%s'", tcp_addr);
+ *yield = string_sprintf (
+ "rspamd dlfunc: failed to lookup host '%s'", tcp_addr);
+ goto RETURN_DEFER;
+ }
+
+ in = *(struct in_addr *) he->h_addr_list[0];
+
+/* contact a rspamd */
+
+ if ((rspamd_sock = ip_socket (SOCK_STREAM, AF_INET)) < 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: TCP socket creation failed: %s",
+ strerror (errno));
+ *yield = string_sprintf (
+ "rspamd dlfunc: TCP socket creation failed: %s",
+ strerror (errno));
+ goto RETURN_DEFER;
+ };
+
+ if (ip_connect (rspamd_sock, AF_INET, (uschar *) inet_ntoa (in),
+ tcp_port, 5, FALSE) < 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: connection to %s, port %u failed: %s",
+ tcp_addr, tcp_port, strerror (errno));
+ *yield = string_sprintf (
+ "rspamd dlfunc: connection to %s, port %u failed: %s",
+ tcp_addr, tcp_port, strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+//debug_printf(" Use TCP socket %s:%d\n", tcp_addr, tcp_port);
+ } else {
+ if ((rspamd_sock = socket (AF_UNIX, SOCK_STREAM, 0)) < 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: Unable to acquire socket (%s)",
+ strerror (errno));
+ *yield = string_sprintf (
+ "rspamd dlfunc: Unable to acquire socket (%s)",
+ strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+ server.sun_family = AF_UNIX;
+ Ustrcpy (server.sun_path, arg_socket_addr);
+
+ if (connect (rspamd_sock, (struct sockaddr *) &server,
+ sizeof (struct sockaddr_un)) < 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: Unable to connect to UNIX socket %s (%s)",
+ socket, strerror (errno));
+ *yield = string_sprintf (
+ "rspamd dlfunc: Unable to connect to UNIX socket %s (%s)",
+ socket, strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+//debug_printf(" Use UNIX Domain socket %s\n", arg_socket_addr);
+ }
+
+// now we are connected to rspamd on rspamd_sock
+
+ memset (spamd_buffer2, 0, sizeof (spamd_buffer2));
+ offset = 0;
+
+ // headers list
+ tmp_headerlist = NULL;
+ header_last = NULL;
+ for (my_header = header_list; my_header; my_header = my_header->next) {
+ if ((my_header->type != '*') && (my_header->type != htype_old)) {
+ header_new = store_get (sizeof (header_line));
+ header_new->text = my_header->text;
+
+ header_new->slen = my_header->slen;
+ header_new->type = my_header->type;
+ header_new->next = NULL;
+ //debug_printf(" copy header item: '%s'\n", my_header->text);
+
+ max_len = sizeof (spamd_buffer2) - offset - 1;
+ len = my_header->slen;
+ if (len > max_len) len = max_len;
+ Ustrncpy (spamd_buffer2 + offset, my_header->text, len);
+ offset += len;
+
+ if (header_last != NULL) header_last->next = header_new;
+ header_last = header_new;
+ }
+ }
+
+ s = string_sprintf ("\n");
+ max_len = sizeof (spamd_buffer2) - offset - 1;
+ len = Ustrlen (s);
+ if (len > max_len) len = max_len;
+ Ustrncpy (spamd_buffer2 + offset, s, len);
+ offset += len;
+
+//debug_printf(" Headers size: %d\n", offset);
+ mbox_size += offset;
+//debug_printf(" Total message size: %d\n", mbox_size);
+
+// copy request to buffer
+ memset (spamd_buffer, 0, sizeof (spamd_buffer));
+ string_format (spamd_buffer,
+ sizeof (spamd_buffer),
+ "POST /checkv2 HTTP/1.0\r\nContent-length: "OFF_T_FMT
+ "\r\nQueue-Id: %s\r\nFrom: %s\r\n",
+ mbox_size, message_id, sender_address);
+
+ for (i = 0; i < recipients_count; i++) {
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer), "Rcpt: %s\r\n",
+ recipients_list[i].address);
+ }
+
+ if ((helo = expand_string (US"$sender_helo_name")) != NULL && *helo != '\0') {
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer), "Helo: %s\r\n",
+ helo);
+ }
+
+ if ((sender_host_name = expand_string (US"$sender_host_name")) != NULL &&
+ *sender_host_name != '\0') {
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer),
+ "Hostname: %s\r\n",
+ sender_host_name);
+ }
+ //else
+ //string_format(spamd_buffer+Ustrlen(spamd_buffer), sizeof(spamd_buffer)-Ustrlen(spamd_buffer), "Hostname: unknown\r\n");
+
+ if (sender_host_address != NULL) {
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer), "IP: %s\r\n",
+ sender_host_address);
+ }
+
+ //authenticated_id
+ if ((authenticated_id = expand_string (US"$authenticated_id")) != NULL &&
+ *authenticated_id != '\0') {
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer), "User: %s\r\n",
+ authenticated_id);
+ }
+
+ string_format (spamd_buffer + Ustrlen (spamd_buffer),
+ sizeof (spamd_buffer) - Ustrlen (spamd_buffer), "\r\n");
+
+ if (send (rspamd_sock, spamd_buffer, Ustrlen (spamd_buffer), 0) < 0) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: rspamd send failed: %s", strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+ /*
+ * now send the data buffer and spool file
+ */
+ Ustrcpy (big_buffer, "sending data block");
+
+ wrote = send (rspamd_sock, spamd_buffer2, strlen (spamd_buffer2), 0);
+ if (wrote == -1) {
+ goto WRITE_FAILED;
+ }
+
+ /*
+ * Note: poll() is not supported in OSX 10.2.
+ */
+
+#ifndef NO_POLL_H
+ pollfd.fd = rspamd_sock;
+ pollfd.events = POLLOUT;
+#endif
+// (void)fcntl(rspamd_sock, F_SETFL, O_NONBLOCK);
+ do {
+ read = fread (spamd_buffer, 1, sizeof (spamd_buffer) - 1, mbox_file);
+
+ if (read < sizeof (spamd_buffer)) {
+ spamd_buffer[read] = 0;
+ }
+//debug_printf(" Read from spool file: %s", spamd_buffer);
+ if (read > 0) {
+ offset = 0;
+ again:
+
+#ifndef NO_POLL_H
+ result = poll (&pollfd, 1, 1000);
+ if (result == -1 && errno == EINTR) {
+ continue;
+ }
+ else if (result < 1) {
+ if (result == -1)
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: %s on rspamd socket",
+ strerror (errno));
+ else {
+ if (time (NULL) - start < RSPAMD_TIMEOUT)
+ goto again;
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: timed out writing rspamd socket");
+ *yield = string_sprintf (
+ "rspamd dlfunc: timed out writing rspamd socket");
+ }
+ goto RETURN_DEFER;
+ }
+#endif
+ wrote = send (rspamd_sock, spamd_buffer + offset, read - offset, 0);
+
+ if (wrote == -1) {
+ goto WRITE_FAILED;
+ }
+
+ if (offset + wrote != read) {
+ offset += wrote;
+ goto again;
+ }
+ }
+ } while (!feof (mbox_file) && !ferror (mbox_file));
+
+ if (ferror (mbox_file)) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: error reading spool file: %s",
+ strerror (errno));
+ *yield = string_sprintf ("rspamd dlfunc: error reading spool file: %s",
+ strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+ /*
+ read rspamd response using what's left of the timeout.
+ */
+
+ memset (spamd_buffer, 0, sizeof (spamd_buffer));
+ offset = 0;
+ while ((i = ip_recv (rspamd_sock,
+ spamd_buffer + offset,
+ sizeof (spamd_buffer) - offset - 1,
+ RSPAMD_TIMEOUT - time (NULL) + start)) > 0
+ ) {
+ //debug_printf(" read %d bytes from socket\n", i);
+ offset += i;
+ }
+//debug_printf(" total read %d bytes from socket\n", offset);
+
+/* error handling */
+ if ((i <= 0) && (errno != 0)) {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: error reading from rspamd socket: %s",
+ strerror (errno));
+ *yield = string_sprintf (
+ "rspamd dlfunc: error reading from rspamd socket: %s",
+ strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+//debug_printf("read from socket: %s\n", spamd_buffer);
+
+ if (rspamd_sock > 0) {
+ (void) close (rspamd_sock);
+ rspamd_sock = 0;
+ }
+ if (mbox_file != NULL) {
+ (void) fclose (mbox_file);
+ mbox_file = NULL;
+ }
+
+ //Parse http response code
+ if (strstr (spamd_buffer, "HTTP/1.1 200 OK") == NULL &&
+ strstr (spamd_buffer, "HTTP/1.0 200 OK") == NULL) {
+ *yield = string_sprintf ("rspamd dlfunc: HTTP return code != 200: %s",
+ spamd_buffer);
+ goto RETURN_DEFER;
+ }
+
+ //Parse http response
+ const char *crlf_pos = strstr (spamd_buffer, "\r\n\r\n");
+ if (crlf_pos == NULL) {
+ *yield = string_sprintf ("rspamd dlfunc: HTTP response error: %s",
+ spamd_buffer);
+ goto RETURN_DEFER;
+ }
+
+ char *json_answer = string_sprintf ("%s", crlf_pos + 4);
+
+ //Parse json
+ cJSON *json = NULL;
+ json = cJSON_Parse (json_answer);
+
+ if (!json) {
+ *yield = string_sprintf ("rspamd dlfunc: Json parse error, json: %s",
+ spamd_buffer);
+ goto RETURN_DEFER;
+ }
+
+ //Score
+ cJSON *score = cJSON_GetObjectItem (json, "score");
+ if (!cJSON_IsNumber (score)) {
+ *yield = string_sprintf (
+ "rspamd dlfunc: Json parse error, no found 'score'");
+ goto RETURN_DEFER;
+ }
+ //required_score
+ cJSON *required_score = cJSON_GetObjectItem (json, "required_score");
+ if (!cJSON_IsNumber (required_score)) {
+ *yield = string_sprintf (
+ "rspamd dlfunc: Json parse error, no found 'required_score'");
+ goto RETURN_DEFER;
+ }
+ //Action
+ cJSON *action = cJSON_GetObjectItem (json, "action");
+ if (!cJSON_IsString (action)) {
+ *yield = string_sprintf (
+ "rspamd dlfunc: Json parse error, no found 'action'");
+ goto RETURN_DEFER;
+ }
+ *yield = string_sprintf ("[%.2f / %.2f]", score->valuedouble,
+ required_score->valuedouble);
+
+ //Parse scan time
+ cJSON *time_real = cJSON_GetObjectItem (json, "time_real");
+ cJSON *time_virtual = cJSON_GetObjectItem (json, "time_virtual");
+ if (cJSON_IsNumber (time_real) && cJSON_IsNumber (time_virtual))
+ *yield = string_sprintf ("%s [time: %.6f, %.6f]", *yield,
+ time_real->valuedouble, time_virtual->valuedouble);
+
+ *yield = string_sprintf ("%s\n Action: %s\n", *yield, action->valuestring);
+
+ cJSON *symbol = NULL;
+ cJSON *symbol_name = NULL;
+ cJSON *symbol_score = NULL;
+ cJSON *symbol_options = NULL;
+ cJSON *option = NULL;
+
+ //parse symbols
+ cJSON *symbols = cJSON_GetObjectItem (json, "symbols");
+ for (i = 0; i < cJSON_GetArraySize (symbols); i++) {
+ symbol = cJSON_GetArrayItem (symbols, i);
+ symbol_name = cJSON_GetObjectItem (symbol, "name");
+ symbol_score = cJSON_GetObjectItem (symbol, "score");
+ symbol_options = cJSON_GetObjectItem (symbol, "options");
+
+ if (cJSON_IsString (symbol_name)) {
+ *yield = string_sprintf ("%s %s", *yield, symbol_name->valuestring);
+ }
+ if (cJSON_IsNumber (symbol_score)) {
+ *yield = string_sprintf ("%s(%.2f)", *yield,
+ symbol_score->valuedouble);
+ }
+
+ //parse options
+ c = cJSON_GetArraySize (symbol_options);
+ if (c > 0) {
+ *yield = string_sprintf ("%s[", *yield);
+ }
+
+ for (j = 0; j < c; j++) {
+ option = cJSON_GetArrayItem (symbol_options, j);
+
+ if (cJSON_IsString (option)) {
+ *yield = string_sprintf ("%s%s", *yield, option->valuestring);
+ if (j < c - 1) *yield = string_sprintf ("%s, ", *yield);
+ }
+ }
+ if (c > 0) {
+ *yield = string_sprintf ("%s]", *yield);
+ }
+
+ *yield = string_sprintf ("%s\n", *yield);
+ }
+
+ //Parse messages
+ cJSON *mess = NULL;
+ cJSON *messages = cJSON_GetObjectItem (json, "messages");
+ c = cJSON_GetArraySize (messages);
+
+ for (i = 0; i < c; i++) {
+ mess = cJSON_GetArrayItem (messages, i);
+ if (cJSON_IsString (mess)) {
+ *yield = string_sprintf ("%s %s", *yield, mess->valuestring);
+ }
+
+ if (i < c - 1) {
+ *yield = string_sprintf ("%s\n", *yield);
+ }
+ }
+
+ return OK;
+
+/* Come here if any call to read_response, other than a response after the data
+phase, failed. Analyse the error, and if isn't too bad, send a QUIT
+command. Wait for the response with a short timeout, so we don't wind up this
+process before the far end has had time to read the QUIT. */
+
+ WRITE_FAILED:
+ {
+ log_write (0, LOG_MAIN | LOG_PANIC,
+ "rspamd dlfunc: %s on rspamd socket", strerror (errno));
+ *yield = string_sprintf ("rspamd dlfunc: %s on rspamd socket",
+ strerror (errno));
+ goto RETURN_DEFER;
+ }
+
+ RESPONSE_FAILED:
+ {
+ int code;
+ int save_errno;
+ int more_errno;
+ uschar message_buffer[256];
+ uschar *message;
+
+ save_errno = errno;
+
+ message = &message_buffer[0];
+
+ log_write (0, LOG_MAIN | LOG_PANIC, "rspamd dlfunc: %s", message);
+ *yield = string_sprintf ("rspamd dlfunc: %s", message);
+
+ goto RETURN_DEFER;
+ }
+
+ RETURN_DEFER:
+ {
+ if (rspamd_sock > 0) {
+ (void) close (rspamd_sock);
+ rspamd_sock = 0;
+ }
+ if (mbox_file != NULL) {
+ (void) fclose (mbox_file);
+ mbox_file = NULL;
+ }
+
+ return (defer_ok ? OK : ERROR);
+ }
+
+ return OK;
+}
diff --git a/contrib/exim/local_scan.c b/contrib/exim/local_scan.c
new file mode 100644
index 0000000..42cd5ad
--- /dev/null
+++ b/contrib/exim/local_scan.c
@@ -0,0 +1,700 @@
+/*
+ This program is RSPAMD agent for use with
+ exim (http://www.exim.org) MTA by its local_scan feature.
+
+ To enable exim local scan please copy this file to exim source tree
+ Local/local_scan.c, edit Local/Makefile to add
+
+ LOCAL_SCAN_SOURCE=Local/local_scan.c
+ LOCAL_SCAN_HAS_OPTIONS=yes
+
+ and compile exim.
+
+ Comment out RSPAM_UNIXSOCKET definition below if you have remote RSPAMD
+ daemon
+
+ AND
+
+ use Exim parameters daemonIP and daemonPort to configure remote
+ RSPAMD daemon.
+
+ For exim compilation with local scan feature details please visit
+ http://www.exim.org/exim-html-4.50/doc/html/spec_toc.html#TOC333
+
+ For RSPAMD details please visit
+ http://rspamd.sourceforge.net
+*/
+
+/* Comment out the row below to use socket type AF_INET
+ to connect RSPAMD daemon */
+//#define RSPAM_UNIXSOCKET
+
+#include <stdio.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <netinet/in.h>
+#include <stdlib.h>
+
+#include "local_scan.h"
+
+extern uschar *sender_helo_name;
+extern int message_size;
+
+#define READ_FAIL(x) ((x) < 0)
+#define RSPAMD_FAILURE_HDR "X-Spam-Flag"
+#define RSPAMD_SCORE_HDR "X-Spam-Status"
+#define REJECT_ON_ERROR 0
+
+static int _OK = 0;
+static int ERR_WRITE = 53;
+static int ERR_READ = 54;
+static int MAX_FAILS_C = 256;
+static int MAX_PATH = 256;
+static int MAX_SIZE_FILE = 64*1024;
+
+static uschar *daemonIP = US"127.0.0.1";
+static int daemonPort = 11333;
+static uschar *temp_dir = US"/var/tmp";
+static uschar *socket_name = US"/var/run/rspamd.sock";
+static int strange = 0;
+
+optionlist local_scan_options[] =
+{
+ {"rspam_ip", opt_stringptr, &daemonIP},
+ {"rspam_port", opt_int, &daemonPort},
+ {"rspam_tmp", opt_stringptr, &temp_dir},
+ {"rspam_sock", opt_stringptr, &socket_name},
+
+};
+
+int local_scan_options_count = sizeof (local_scan_options) / sizeof (optionlist);
+
+typedef int socket_t;
+static socket_t sock = -1;
+
+int iFdInp;
+struct sockaddr_un ssun;
+struct sockaddr_in ssin;
+
+static int mOpenTmp (char *pszDir, char *pszPrefix, char *pszPath)
+{
+ int iLen;
+ int iFd = -1;
+ char *pszSep = "";
+
+ iLen = (int)strlen(pszDir);
+ if (iLen > MAX_PATH)
+ return -1;
+
+ if (pszDir[iLen - 1] != '/')
+ pszSep = "/";
+
+ sprintf (pszPath, "%s%s%sXXXXXX", pszDir, pszSep, pszPrefix);
+ iFd = mkstemp (pszPath);
+
+ if (iFd < 0)
+ log_write (0, LOG_MAIN, "rspam-exim: Temp file create error %d", errno);
+
+ return iFd;
+}
+
+static int ReadFd (int iFdMsg, int fd)
+{
+ char psMsg [MAX_SIZE_FILE]; /* max size SO can swallow */
+ int iLen, result = _OK;
+
+ if ((iLen = read (fd, psMsg, sizeof (psMsg))) > 0)
+ {
+ if (write (iFdMsg, psMsg, (unsigned int) iLen) != iLen)
+ result = ERR_WRITE;
+ }
+ else
+ result = ERR_READ;
+
+ close (iFdMsg);
+
+ return result;
+}
+
+
+void CleanupInp (char *sName)
+{
+ if (sName) unlink (sName);
+
+ close (iFdInp);
+ return;
+}
+
+
+int FakeSMTPCommand (socket_t sock,
+ char *command,
+ char *value,
+ char *sName,
+ int Cleanup,
+ int wa)
+{
+ char sCommand[1024];
+ char answ [3];
+ int Len;
+
+ sprintf (sCommand, "%s %s\r\n", command, value);
+
+ if (send (sock, sCommand, strlen (sCommand), 0) != (int) strlen (sCommand))
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: socket sending '%s' error %d", sCommand, errno);
+ if (Cleanup)
+ CleanupInp (sName);
+ return ERR_WRITE;
+ }
+
+ if(wa) {
+ memset (answ, '\0', sizeof (answ));
+ Len = read (sock, answ, sizeof (answ));
+ if (READ_FAIL (Len))
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: read() error %d, len=%d", errno, Len);
+ if (Cleanup)
+ CleanupInp (sName);
+ return ERR_WRITE;
+ }
+
+ if (strncmp (answ, "OK", 2) != 0)
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: server did not confirm, answ=%s", answ);
+ if (Cleanup)
+ CleanupInp (sName);
+ return ERR_WRITE; /* Cannot read message error code */
+ }
+ }
+
+ return OK;
+}
+
+
+static int written (socket_t fd, const char *vptr, int n)
+{
+ size_t nleft;
+ int nwritten;
+ const char *ptr;
+
+ ptr = vptr;
+ nleft = n;
+ while (nleft > 0)
+ {
+ if ((nwritten = send (fd, ptr, nleft, 0)) <= 0)
+ {
+ if (errno == EINTR)
+ nwritten = 0;
+ else
+ return (-1);
+ }
+
+ nleft -= nwritten;
+ ptr += nwritten;
+ }
+
+ return (n);
+}
+
+
+static int SendEnvelope (char *sFile)
+{
+ int i;
+ char str [256], *rh;
+ void *psBuf;
+ int fd, bytesRead;
+
+ if(message_size > MAX_SIZE_FILE) {
+ log_write (0, LOG_MAIN, "rspam-exim: file %s is great %d bytes", sFile, MAX_SIZE_FILE);
+ return ERR_WRITE;
+ }
+
+ /* send greeting */
+// if(FakeSMTPCommand(sock, "PROCESS", "RSPAMC/1.0", sFile, 1, 0) != _OK)
+// return ERR_WRITE;
+ if(FakeSMTPCommand(sock, "SYMBOLS", "RSPAMC/1.1", sFile, 1, 0) != _OK)
+// if(FakeSMTPCommand(sock, "CHECK", "RSPAMC/1.0", sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+
+
+ /* sender IP */
+ if (FakeSMTPCommand (sock, "IP:", sender_host_address, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ /* mail from */
+ if (FakeSMTPCommand (sock, "From:",
+ strlen (sender_address) == 0 ? "MAILER-DAEMON" : (char*) sender_address, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ /* send helo */
+ if (FakeSMTPCommand (sock, "Helo:", sender_helo_name, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ /* send helo */
+ sprintf(str, "%d", message_size);
+ if (FakeSMTPCommand (sock, "Content-Length:", str, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ /* number of recipient */
+ sprintf(str, "%d", recipients_count);
+ if (FakeSMTPCommand (sock, "Recipient-Number:", str, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ /* envelope rcpto */
+ for (i = 0; i < recipients_count; i ++)
+ {
+ if (FakeSMTPCommand (sock, "Rcpt:", recipients_list[i].address, sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+ }
+
+ psBuf = store_get (MAX_SIZE_FILE);
+
+ fd = open (sFile, O_RDONLY);
+ if (fd > 0)
+ {
+ bytesRead = read (fd, psBuf, MAX_SIZE_FILE);
+ close (fd);
+
+ if (FakeSMTPCommand (sock, "\r\n", "", sFile, 1, 0) != _OK)
+ return ERR_WRITE;
+
+ if (written (sock, psBuf, bytesRead) != bytesRead)
+ return ERR_WRITE;
+ }
+ else
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: file %s open error %d", sFile, errno);
+ return ERR_WRITE;
+ }
+
+ return _OK;
+}
+
+
+int GetFiles (char *pInpFile, int local_scan_fd)
+{
+ /*
+ Returns OK if no errors, else error code.
+ On successful return, pEnvFile points to Envelope file name and
+ pInpFile points to Message filename
+ */
+ int iStatus;
+ struct header_line *h_line;
+
+ iFdInp = mOpenTmp ((char *)temp_dir, "sp-inp", pInpFile);
+ if (iFdInp == -1)
+ {
+ return ERR_WRITE;
+ }
+
+ /* Emit headers */
+ h_line = header_list;
+ while (h_line != NULL)
+ {
+ if (h_line->type == '*') /* internal header */
+ {
+ h_line = h_line->next;
+ continue;
+ }
+
+ if (write (iFdInp, h_line->text, strlen (h_line->text)) != strlen (h_line->text))
+ {
+ CleanupInp ("");
+ return ERR_WRITE;
+ }
+ h_line = h_line->next;
+ }
+ if (write (iFdInp, "\n", 1) != 1)
+ {
+ CleanupInp ("");
+ return ERR_WRITE;
+ }
+
+ /* Read msg */
+ if ((iStatus = ReadFd (iFdInp, local_scan_fd)))
+ {
+ return iStatus;
+ }
+
+ /* Return success */
+ return _OK;
+}
+
+
+int GetAndTransferMessage (int fd, char *sFile)
+{
+ char answ [4];
+ int iStatus;
+ int Len, ccnt;
+ int test;
+
+ iStatus = GetFiles ((char *)sFile, fd);
+
+ if (iStatus != _OK)
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: Error %d getting message", iStatus);
+ close (sock);
+ return iStatus;
+ }
+
+ for (ccnt = 0; ccnt <= MAX_FAILS_C; ccnt ++)
+ {
+#ifdef RSPAM_UNIXSOCKET
+ test = connect (sock, (struct sockaddr *) &ssun, sizeof (struct sockaddr_un)) < 0;
+#else
+ test = connect (sock, (struct sockaddr *) &ssin, sizeof (struct sockaddr_in)) < 0;
+#endif
+ if (test)
+ {
+ if (ccnt < MAX_FAILS_C)
+ usleep (1000);
+ else
+ {
+ close (sock);
+#ifdef RSPAM_UNIXSOCKET
+ log_write (0, LOG_MAIN, "rspam-exim: socket connect to %s failed", (char *)socket_name);
+#else
+ log_write (0, LOG_MAIN, "rspam-exim: socket connect to %s:%u failed", daemonIP, daemonPort);
+#endif
+ return REJECT_ON_ERROR ? LOCAL_SCAN_TEMPREJECT:LOCAL_SCAN_ACCEPT;
+ }
+ }
+ else
+ break;
+ }
+
+ iStatus = SendEnvelope (sFile);
+ if (iStatus != _OK)
+ {
+ log_write (0, LOG_MAIN, "rspam-exim: error %d sending envelope data", iStatus);
+ close (sock);
+ return iStatus;
+ }
+
+ /* fprintf (stderr, "Transmit OK\n"); */
+ return _OK;
+}
+
+void header_del (uschar *hdr)
+{
+ struct header_line *h_line;
+
+ h_line = header_list;
+ while (h_line != NULL)
+ {
+ if (h_line->type == '*') /* internal header */
+ {
+ h_line = h_line->next;
+ continue;
+ }
+
+ if (strncasecmp (h_line->text, hdr, strlen(hdr)) == 0)
+ {
+ h_line->type = '*';
+ while (h_line->next &&
+ (*h_line->next->text == ' ' || *h_line->next->text == '\t'))
+ {
+ h_line = h_line->next;
+ h_line->type = '*';
+ }
+ }
+ h_line = h_line->next;
+ }
+}
+
+void AlterSubject (char *label)
+{
+ struct header_line *h_line;
+ char *subject, *strP;
+
+ h_line = header_list;
+
+ while (h_line != NULL)
+ {
+ if (h_line->type == '*') /* internal header */
+ {
+ h_line = h_line->next;
+ continue;
+ }
+
+ if (strncasecmp (h_line->text, "Subject", strlen("Subject")) == 0)
+ {
+ strP = strchr (h_line->text, ':');
+ subject = string_copy (++strP);
+ while (h_line->next &&
+ (*h_line->next->text == ' ' || *h_line->next->text == '\t'))
+ {
+ h_line = h_line->next;
+ subject = string_sprintf ("%s\n%s", subject, h_line->text);
+ }
+ header_del (US "Subject");
+ break;
+ }
+
+ h_line = h_line->next;
+ }
+ header_add (' ', "Subject: %s%s", label, subject ? subject : "");
+}
+
+int
+io_read(int fd, char *buf, size_t size)
+{
+ int nfd, next = 0, rcount = 15;
+ size_t len = 0;
+ fd_set fds;
+ struct timeval tv;
+
+ if((sock < 0) || (buf == NULL))
+ return -1;
+
+ FD_ZERO(&fds);
+
+repeat_read:
+
+ tv.tv_sec = 5;
+ tv.tv_usec = 0;
+ FD_SET(fd, &fds);
+
+// log_write(0, LOG_MAIN, "rspam-exim: before select");
+
+ if((nfd=select(fd+1, &fds, NULL, NULL, &tv)) == -1) {
+// log_write(0, LOG_MAIN, "rspam-exim: select error: %s", strerror(errno));
+ return -1;
+ }
+
+// log_write(0, LOG_MAIN, "rspam-exim: select return %d fds, rcount %d, next %d", nfd, rcount, next);
+
+ if((nfd>0) && (FD_ISSET(fd, &fds))) {
+ next += len = read(fd, buf + next, size - next);
+// log_write(0, LOG_MAIN, "rspam-exim: read %d bytes", len);
+// if(next<size)
+// goto repeat_read;
+ }
+ rcount--;
+ if(rcount>0)
+ goto repeat_read;
+
+ return next;
+}
+
+int WaitForScanResult (uschar **resStr)
+{
+ int Len, i;
+ int rej = 0, result = LOCAL_SCAN_ACCEPT, answer_size, spm = 0, code = 0, ns = 0, smb = 0, urf = 0;
+ char *strP, *tok, *tmp;
+ char *hdr = NULL, *hdrv = NULL, *spmStr = NULL, *symbols=NULL, *urls=NULL;
+ char answ [4096], state[6], metric[128], back;
+ float sm=0, smd=0, smr=0;
+
+ memset (answ, '\0', sizeof (answ));
+// log_write(0, LOG_MAIN, "rspam-exim: before read from %d", sock);
+// Len = read (sock, answ, sizeof (answ) - 1);
+ Len = io_read(sock, answ, sizeof (answ) - 1);
+ log_write(0, LOG_MAIN, "rspam-exim: read %d bytes", Len);
+
+ if (strncmp (answ, "RSPAMD/1.1 ", 11) == 0)
+ {
+ strP = (char *)answ;
+ for (tok = strtok (strP, "\n"); tok; tok = strtok (NULL, "\n"))
+ {
+// log_write(0, LOG_MAIN, "rspam-exim: process line '%s'", tok);
+
+ if (strncmp (tok, "RSPAMD/1.1 ", 11) == 0)
+ {
+ if (sscanf (tok, "%*s %d %s", &code, state) == 2)
+ {
+// log_write(0, LOG_MAIN, "rspam-exim: daemon reports code %d %s", code, state);
+ if ((code == 0) && (strcmp(state,"OK")==0)) {
+ header_del ((uschar *) RSPAMD_FAILURE_HDR);
+ header_add (' ', "%s: SKIP\n", RSPAMD_FAILURE_HDR);
+ strange = 1;
+ continue;
+ } else {
+ header_del ((uschar *) RSPAMD_FAILURE_HDR);
+ header_add (' ', "%s: SKIP\n", RSPAMD_FAILURE_HDR);
+ log_write(0, LOG_MAIN, "rspam-exim: daemon reports code %d %s", code, state);
+ return LOCAL_SCAN_ACCEPT;
+ }
+ }
+ continue;
+ }
+
+ /* Metric: default; False; 6.00 / 10.00 */
+ /* Process metric */
+ if (strncmp (tok, "Metric:", 7) == 0)
+ {
+ tmp = tok;
+ while( (*tmp++) &&
+ ((*tmp!='\r') || (*tmp!='\n'))
+ );
+ back = *tmp;
+ *tmp = '\0';
+ if (sscanf (tok, "Metric: %[^';']; %[^';']; %f / %f / %f", metric, state, &sm, &smd, &smr) == 5) {
+ log_write(0, LOG_MAIN, "rspam-exim: metric: %s; %s; %f / %f / %f", metric, state, sm, smd, smr );
+ if(strcasecmp(state,"true")==0) {
+ header_del ((uschar *) RSPAMD_FAILURE_HDR);
+ header_add (' ', "%s: %s\n", RSPAMD_FAILURE_HDR, "Yes");
+ } else if(strcasecmp(state,"skip")==0) {
+ header_del ((uschar *) RSPAMD_FAILURE_HDR);
+ header_add (' ', "%s: %s\n", RSPAMD_FAILURE_HDR, "Skip");
+ } else {
+ header_del ((uschar *) RSPAMD_FAILURE_HDR);
+ header_add (' ', "%s: %s\n", RSPAMD_FAILURE_HDR, "No");
+ }
+ header_del ((uschar *) RSPAMD_SCORE_HDR);
+ header_add (' ', "%s: %.2f / %.2f / %.2f\n", RSPAMD_SCORE_HDR, sm, smd, smr);
+ strange = 0;
+ }
+ *tmp = back;
+ continue;
+ }
+
+ if (strncmp (tok, "Symbol:", 7) == 0)
+ {
+ tmp = tok;
+ while( (*tmp++) &&
+ ((*tmp!='\r') || (*tmp!='\n'))
+ );
+ back = *tmp;
+ *tmp = '\0';
+ if(smb>0) {
+ tok += 7;
+ while(*tok && isspace(*tok)) tok++;
+ if(strlen(tok)>0) {
+ symbols = string_sprintf ("%s\n %s", symbols, tok);
+ }
+ } else {
+ tok += 7;
+ while(*tok && isspace(*tok)) tok++;
+ symbols = string_copy (tok);
+ }
+ smb = 1;
+ *tmp = back;
+ continue;
+ }
+
+ if (strncmp (tok, "Urls:", 5) == 0)
+ {
+ tmp = tok;
+ while( (*tmp++) &&
+ ((*tmp!='\r') || (*tmp!='\n'))
+ );
+ back = *tmp;
+ *tmp = '\0';
+ if(urf>0) {
+ tok[0] = tok[1]= tok[2]= tok[3]= tok[4] = ' ';
+ urls = string_sprintf ("%s\n%s", urls, tok+3);
+ } else {
+ tok += 5;
+ while(*tok && isspace(*tok)) tok++;
+ urls = string_copy (tok);
+ }
+ urf = 1;
+ *tmp = back;
+ continue;
+ }
+ }
+
+
+ /* do not forget the symbols */
+ if (symbols != NULL && strlen(symbols))
+ {
+ i = 0;
+ tmp = tok = string_copy(symbols);
+ header_del ((uschar *) "X-Spam-Sybmols");
+ header_add (' ', "%s: %s\n", "X-Spam-Sybmols", symbols);
+ while(*tmp!='\0') {
+ if(*tmp == '\r')
+ *tmp = ' ';
+ if(*tmp == '\n')
+ *tmp = ',';
+ tmp++;
+ }
+ *tmp = '\0';
+ log_write(0, LOG_MAIN, "rspam-exim: symbols: %s", tok);
+ }
+
+ /* do not forget the urls */
+ if (urls != NULL && strlen(urls))
+ {
+ log_write(0, LOG_MAIN, "rspam-exim: urls: %s", urls);
+ header_del ((uschar *) "X-Spam-Urls");
+ header_add (' ', "%s: %s\n", "X-Spam-Urls", urls);
+ }
+
+ log_write (0, LOG_MAIN, "rspam-exim: For message from %s will return %s, mailfrom: <%s>, rcpto: <%s>", sender_host_address, rej == 2 ? "DISCARD" : rej == 1 ? "REJECT" : "ACCEPT", sender_address, recipients_list[0].address);
+
+ }
+ else
+ {
+ result = LOCAL_SCAN_ACCEPT;
+ log_write(0, LOG_MAIN, "rspam-exim: wrong signature in answer: %s", answ);
+ }
+
+ if((sm>0) && (smr>0) && (sm>=smr)) {
+ result = LOCAL_SCAN_REJECT;
+ }
+ return result;
+}
+
+
+int
+local_scan(int fd, uschar **return_text)
+{
+ int retval = _OK;
+ char sFileInp [MAX_PATH + 81];
+
+ /* Socket stuff */
+
+ strange = 0;
+#ifdef RSPAM_UNIXSOCKET
+ if ((sock = socket (AF_UNIX, SOCK_STREAM, 0)) < 0)
+ {
+ log_write(0, LOG_MAIN, "rspam-exim: socket() failed");
+ exit (EXIT_FAILURE);
+ }
+ memset (&ssun, '\0', sizeof (struct sockaddr_un));
+ ssun.sun_family = AF_UNIX;
+ if (sizeof (socket_name) > sizeof (ssun.sun_path))
+ {
+ close (sock);
+ log_write(0, LOG_MAIN, "rspam-exim: UNIX socket name %s too long", socket_name);
+ exit (EXIT_FAILURE);
+ }
+ strcpy (ssun.sun_path, socket_name);
+#else
+ if ((sock = socket (AF_INET, SOCK_STREAM, 0)) < 0)
+ {
+ log_write(0, LOG_MAIN, "rspam-exim: socket() failed");
+ exit (EXIT_FAILURE);
+ }
+ memset (&ssin, '\0', sizeof (struct sockaddr_in));
+ ssin.sin_family = AF_INET;
+ ssin.sin_addr.s_addr = inet_addr (daemonIP);
+ ssin.sin_port = htons (daemonPort);
+#endif
+
+ if (GetAndTransferMessage (fd, (char *)sFileInp) != _OK)
+ {
+ close (sock);
+ unlink (sFileInp);
+ SPOOL_DATA_START_OFFSET;
+ return REJECT_ON_ERROR ? LOCAL_SCAN_TEMPREJECT:LOCAL_SCAN_ACCEPT;
+ }
+
+ retval = WaitForScanResult (return_text);
+
+ if(!strange)
+ unlink (sFileInp);
+ close (sock);
+ SPOOL_DATA_START_OFFSET;
+
+ return retval;
+}
+
+/* End of local_scan.c */
diff --git a/contrib/exim/local_scan.c.in b/contrib/exim/local_scan.c.in
new file mode 100644
index 0000000..72d2f79
--- /dev/null
+++ b/contrib/exim/local_scan.c.in
@@ -0,0 +1,377 @@
+/*
+ * This program is RSPAMD agent for use with
+ * exim (http://www.exim.org) MTA by its local_scan feature.
+ *
+ * To enable exim local scan please copy this file to exim source tree
+ * Local/local_scan.c, edit Local/Makefile to add
+ *
+ * LOCAL_SCAN_SOURCE=Local/local_scan.c
+ * LOCAL_SCAN_HAS_OPTIONS=yes
+ *
+ * and compile exim.
+ *
+ * For exim compilation with local scan feature details please visit
+ * http://www.exim.org/exim-html-current/doc/html/spec_html/ch42.html
+ *
+ * For RSPAMD details please visit
+ * https://bitbucket.org/vstakhov/rspamd/
+ *
+ * Example configuration:
+ * **********************
+ *
+ * local_scan_timeout = 50s
+ *
+ * begin local_scan
+ * rspam_ip = 127.0.0.1
+ * rspam_port = 11333
+ * rspam_skip_sasl_authenticated = true
+ * # don't reject message if on of recipients from this list
+ * rspam_skip_rcpt = postmaster@example.com : some_user@example.com
+ * rspam_message = "Spam rejected; If this is not spam, please contact <postmaster@example.com>"
+ *
+ *
+ * $Id: local_scan.c 646 2010-08-11 11:49:36Z ayuzhaninov $
+ */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/uio.h>
+
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include <errno.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "local_scan.h"
+
+#define REQUEST_LINES 64
+#define REPLY_BUF_SIZE 16384
+#define HEADER_STATUS "X-Rspam-Status"
+#define HEADER_METRIC "X-Rspam-Metric"
+#define HEADER_SCORE "X-Rspam-Score"
+
+/* configuration options */
+static uschar *daemon_ip = US"127.0.0.1";
+static int max_scan_size = 4 * 1024 * 1024;
+static uschar *reject_message = US"Spam message rejected";
+static int daemon_port = 11333;
+static BOOL skip_authenticated = TRUE;
+static uschar *want_spam_rcpt_list = US"";
+
+/* the entries must appear in alphabetical order */
+optionlist local_scan_options[] = {
+ { "rspam_ip", opt_stringptr, &daemon_ip },
+ { "rspam_max_scan_size", opt_mkint, &max_scan_size },
+ { "rspam_message", opt_stringptr, &reject_message },
+ { "rspam_port", opt_int, &daemon_port },
+ { "rspam_skip_rcpt", opt_stringptr, &want_spam_rcpt_list },
+ { "rspam_skip_sasl_authenticated", opt_bool, &skip_authenticated },
+};
+
+int local_scan_options_count = sizeof(local_scan_options) / sizeof(optionlist);
+
+/* push formatted line into vector */
+int push_line(struct iovec *iov, int i, const char *fmt, ...);
+
+int
+local_scan(int fd, uschar **return_text)
+{
+ struct stat sb;
+ struct sockaddr_in server_in;
+ int s, i, r, request_p = 0, headers_count = 0, is_spam = 0, is_reject = 0;
+ off_t message_size;
+ struct iovec request_v[REQUEST_LINES], *headers_v;
+#if "@CMAKE_SYSTEM_NAME@" == "FreeBSD"
+ struct sf_hdtr headers_sf;
+#endif
+ uschar *helo, *log_buf;
+ header_line *header_p;
+ char reply_buf[REPLY_BUF_SIZE], io_buf[BUFSIZ];
+ ssize_t size;
+ char *tok_ptr, *str;
+ char mteric[128], result[8];
+ float score, required_score;
+
+ *return_text = reject_message;
+
+ /*
+ * one msaage can be send via exim+rspamd twice
+ * remove header from previous pass
+ */
+ header_remove(0, US HEADER_STATUS);
+ header_remove(0, US HEADER_METRIC);
+
+ /* check message size */
+ fstat(fd,&sb); /* XXX shuld check error */
+ message_size = sb.st_size - SPOOL_DATA_START_OFFSET;
+ if (message_size > max_scan_size) {
+ header_add(' ', HEADER_STATUS ": skip_big\n");
+ log_write (0, LOG_MAIN, "rspam: message larger than rspam_max_scan_size, accept");
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ /* don't scan mail from authenticated hosts */
+ if (skip_authenticated && sender_host_authenticated != NULL) {
+ header_add(' ', HEADER_STATUS ": skip_authenticated\n");
+ log_write(0, LOG_MAIN, "rspam: from=<%s> ip=%s authenticated (%s), skip check\n",
+ sender_address,
+ sender_host_address == NULL ? US"localhost" : sender_host_address,
+ sender_host_authenticated);
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ /*
+ * add status header, which mean, that message was not scanned
+ * if message will be scanned, this header will be replaced
+ */
+ header_add(' ', HEADER_STATUS ": check_error\n");
+
+ /* create socket */
+ memset(&server_in, 0, sizeof(server_in));
+ server_in.sin_family = AF_INET;
+ server_in.sin_port = htons(daemon_port);
+ server_in.sin_addr.s_addr = inet_addr(daemon_ip);
+ if ((s = socket(PF_INET, SOCK_STREAM, 0)) < 0) {
+ log_write(0, LOG_MAIN, "rspam: socket (%d: %s)", errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+ if (connect(s, (struct sockaddr *) &server_in, sizeof(server_in)) < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't connect to %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ /* count message headers */
+ for (header_p = header_list; header_p != NULL; header_p = header_p->next) {
+ /* header type '*' is used for replaced or deleted header */
+ if (header_p->type == '*')
+ continue;
+ headers_count++;
+ }
+
+ /* write message headers to vector */
+#if "@CMAKE_SYSTEM_NAME@" == "FreeBSD"
+ memset(&headers_sf, 0, sizeof(headers_sf));
+ if (headers_count > 0) {
+ headers_v = store_get((headers_count + 1)* sizeof(*headers_v));
+ i = 0;
+ for (header_p = header_list; header_p != NULL; header_p = header_p->next) {
+ if (header_p->type == '*')
+ continue;
+ headers_v[i].iov_base = header_p->text;
+ headers_v[i].iov_len = header_p->slen;
+ i++;
+ message_size += header_p->slen;
+ }
+ headers_v[i].iov_base = "\n";
+ headers_v[i].iov_len = strlen("\n");
+ message_size += strlen("\n");
+
+ headers_sf.headers = headers_v;
+ headers_sf.hdr_cnt = headers_count + 1;
+ }
+#else
+ if (headers_count > 0) {
+ headers_v = store_get((headers_count + 1)* sizeof(*headers_v));
+ i = 0;
+ for (header_p = header_list; header_p != NULL; header_p = header_p->next) {
+ if (header_p->type == '*')
+ continue;
+ headers_v[i].iov_base = header_p->text;
+ headers_v[i].iov_len = header_p->slen;
+ i++;
+ message_size += header_p->slen;
+ }
+ headers_v[i].iov_base = "\n";
+ headers_v[i].iov_len = strlen("\n");
+ message_size += strlen("\n");
+#endif
+
+ /* write request to vector */
+ r = 0;
+ r += push_line(request_v, request_p++, "SYMBOLS RSPAMC/1.1\r\n");
+ r += push_line(request_v, request_p++, "Content-length: " OFF_T_FMT "\r\n", message_size);
+ r += push_line(request_v, request_p++, "Queue-Id: %s\r\n", message_id);
+ r += push_line(request_v, request_p++, "From: %s\r\n", sender_address);
+ r += push_line(request_v, request_p++, "Recipient-Number: %d\r\n", recipients_count);
+ for (i = 0; i < recipients_count; i ++)
+ r += push_line(request_v, request_p++, "Rcpt: %s\r\n", recipients_list[i].address);
+ if ((helo = expand_string(US"$sender_helo_name")) != NULL && *helo != '\0')
+ r += push_line(request_v, request_p++, "Helo: %s\r\n", helo);
+ if (sender_host_address != NULL)
+ r += push_line(request_v, request_p++, "IP: %s\r\n", sender_host_address);
+ r += push_line(request_v, request_p++, "\r\n");
+
+ if (r < 0) {
+ close(s);
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ /* send request */
+ if (writev(s, request_v, request_p) < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't send request to %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+#if "@CMAKE_SYSTEM_NAME@" == "FreeBSD"
+ /* send headers (from iovec) and message body (from file) */
+ if (sendfile(fd, s, SPOOL_DATA_START_OFFSET, 0, &headers_sf, NULL, 0) < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't send message to %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+#else
+ /* send headers */
+ if (writev(s, headers_v, headers_count) < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't send headers to %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ /* Send message */
+ while ((r = read (fd, io_buf, sizeof (io_buf))) > 0) {
+ if (write (s, io_buf, r) < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't send message to %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+ }
+#endif
+
+ /* read reply from rspamd */
+ reply_buf[0] = '\0';
+ size = 0;
+ while ((r = read(s, reply_buf + size, sizeof(reply_buf) - size - 1)) > 0 && size < sizeof(reply_buf) - 1) {
+ size += r;
+ }
+
+ if (r < 0) {
+ close(s);
+ log_write(0, LOG_MAIN, "rspam: can't read from %s:%d (%d: %s)", daemon_ip, daemon_port, errno, strerror(errno));
+ return LOCAL_SCAN_ACCEPT;
+ }
+ reply_buf[size] = '\0';
+ close(s);
+
+ if (size >= REPLY_BUF_SIZE - 1) {
+ log_write(0, LOG_MAIN, "rspam: buffer is full, reply may be truncated");
+ }
+
+ /* parse reply */
+ tok_ptr = reply_buf;
+
+ /*
+ * rspamd can use several metrics, logic implemented here:
+ * if any metric more than reject_score - will reject
+ * if any metric true - message will be marked as spam
+ */
+
+ /* First line is: <PROTOCOL>/<VERSION> <ERROR_CODE> <ERROR_REPLY> */
+ str = strsep(&tok_ptr, "\r\n");
+ if (str != NULL && sscanf(str, "%*s %d %*s", &i) == 1) {
+ if (i != 0) {
+ log_write(0, LOG_MAIN, "rspam: server error: %s", str);
+ return LOCAL_SCAN_ACCEPT;
+ }
+ } else {
+ log_write(0, LOG_MAIN, "rspam: bad reply from server: %s", str);
+ return LOCAL_SCAN_ACCEPT;
+ }
+
+ while ((str = strsep(&tok_ptr, "\r\n")) != NULL) {
+ /* skip empty tockens */
+ if (*str == '\0')
+ continue;
+ if (strncmp(str, "Metric:", strlen("Metric:")) == 0) {
+ /*
+ * parse line like
+ * Metric: default; False; 27.00 / 30.00
+ */
+ if (sscanf(str, "Metric: %s %s %f / %f",
+ mteric, result, &score, &required_score) == 4) {
+ log_write(0, LOG_MAIN, "rspam: metric %s %s %.2f / %.2f",
+ mteric, result, score, required_score);
+ header_add(' ', HEADER_METRIC ": %s %s %.2f / %.2f\n",
+ mteric, result, score, required_score);
+ /* integers score for use in sieve ascii-numeric comparator */
+ if (strcmp(mteric, "default;") == 0)
+ header_add(' ', HEADER_SCORE ": %d\n",
+ (int)round(score));
+ } else {
+ log_write(0, LOG_MAIN, "rspam: can't parse: %s", str);
+ return LOCAL_SCAN_ACCEPT;
+ }
+ } else if (strncmp(str, "Action:", strlen("Action:")) == 0) {
+ /* line like Action: add header */
+ str += strlen("Action: ");
+ if (strncmp(str, "reject", strlen("reject")) == 0) {
+ is_reject = 1;
+ is_spam = 1;
+ } else if (strncmp(str, "add header", strlen("add header")) == 0) {
+ is_spam = 1;
+ }
+ }
+ }
+
+ /* XXX many allocs by string_sprintf()
+ * better to sprintf() to single buffer allocated by store_get()
+ */
+ log_buf = string_sprintf("message to");
+ for (i = 0; i < recipients_count; i ++) {
+ log_buf = string_sprintf("%s %s", log_buf, recipients_list[i].address);
+ if (is_reject && lss_match_address(recipients_list[i].address, want_spam_rcpt_list, TRUE) == OK) {
+ is_reject = 0;
+ log_write(0, LOG_MAIN, "rspam: %s want spam, don't reject this message", recipients_list[i].address);
+ }
+ }
+
+ if (is_reject) {
+ log_write(0, LOG_MAIN, "rspam: reject %s", log_buf);
+ return LOCAL_SCAN_REJECT;
+ }
+
+ header_remove(0, US HEADER_STATUS);
+ if (is_spam) {
+ header_add(' ', HEADER_STATUS ": spam\n");
+ log_write(0, LOG_MAIN, "rspam: message marked as spam");
+ } else {
+ header_add(' ', HEADER_STATUS ": ham\n");
+ log_write(0, LOG_MAIN, "rspam: message marked as ham");
+ }
+
+ return LOCAL_SCAN_ACCEPT;
+}
+
+int
+push_line(struct iovec *iov, const int i, const char *fmt, ...)
+{
+ va_list ap;
+ size_t len;
+ char buf[512];
+
+ if (i >= REQUEST_LINES) {
+ log_write(0, LOG_MAIN, "rspam: %s: index out of bounds", __FUNCTION__);
+ return (-1);
+ }
+
+ va_start(ap, fmt);
+ len = vsnprintf(buf, sizeof(buf), fmt, ap);
+ va_end(ap);
+
+ iov[i].iov_base = string_copy(US buf);
+ iov[i].iov_len = len;
+
+ if (len >= sizeof(buf)) {
+ log_write(0, LOG_MAIN, "rspam: %s: error, string was longer than %d", __FUNCTION__, sizeof(buf));
+ return (-1);
+ }
+
+ return 0;
+}
+
diff --git a/contrib/exim/patch-exim-src_spam.c.diff b/contrib/exim/patch-exim-src_spam.c.diff
new file mode 100644
index 0000000..f3ac788
--- /dev/null
+++ b/contrib/exim/patch-exim-src_spam.c.diff
@@ -0,0 +1,338 @@
+diff -ru exim-4.70.orig/src/expand.c exim-4.70/src/expand.c
+--- exim-4.70.orig/src/expand.c 2016-04-09 13:42:00.227625074 +0200
++++ exim-4.70/src/expand.c 2016-04-09 13:42:37.183633096 +0200
+@@ -578,6 +578,7 @@
+ { "sn8", vtype_filter_int, &filter_sn[8] },
+ { "sn9", vtype_filter_int, &filter_sn[9] },
+ #ifdef WITH_CONTENT_SCAN
++ { "spam_action", vtype_stringptr, &spam_action },
+ { "spam_bar", vtype_stringptr, &spam_bar },
+ { "spam_report", vtype_stringptr, &spam_report },
+ { "spam_score", vtype_stringptr, &spam_score },
+diff -ru exim-4.70.orig/src/globals.c exim-4.70/src/globals.c
+--- exim-4.70.orig/src/globals.c 2016-04-09 13:42:00.219625073 +0200
++++ exim-4.70/src/globals.c 2016-04-09 13:42:37.187633096 +0200
+@@ -1136,6 +1136,7 @@
+ uschar *spamd_address = US"127.0.0.1 783";
+ uschar *spam_bar = NULL;
+ uschar *spam_report = NULL;
++uschar *spam_action = NULL;
+ uschar *spam_score = NULL;
+ uschar *spam_score_int = NULL;
+ #endif
+diff -ru exim-4.70.orig/src/globals.h exim-4.70/src/globals.h
+--- exim-4.70.orig/src/globals.h 2016-04-09 13:42:00.219625073 +0200
++++ exim-4.70/src/globals.h 2016-04-09 13:42:37.187633096 +0200
+@@ -703,6 +703,7 @@
+ extern uschar *spamd_address; /* address for the spamassassin daemon */
+ extern uschar *spam_bar; /* the spam "bar" (textual representation of spam_score) */
+ extern uschar *spam_report; /* the spamd report (multiline) */
++extern uschar *spam_action; /* the spamd action */
+ extern uschar *spam_score; /* the spam score (float) */
+ extern uschar *spam_score_int; /* spam_score * 10 (int) */
+ #endif
+diff -ru exim-4.70.orig/src/spam.c exim-4.70/src/spam.c
+--- exim-4.70.orig/src/spam.c 2016-04-09 13:42:00.231625075 +0200
++++ exim-4.70/src/spam.c 2016-04-09 13:43:43.927647677 +0200
+@@ -16,6 +16,7 @@
+ uschar spam_score_buffer[16];
+ uschar spam_score_int_buffer[16];
+ uschar spam_bar_buffer[128];
++uschar spam_action_buffer[32];
+ uschar spam_report_buffer[32600];
+ uschar prev_user_name[128] = "";
+ int spam_ok = 0;
+@@ -31,9 +32,11 @@
+ int spamd_sock;
+ uschar spamd_buffer[32600];
+ int i, j, offset, result;
++ BOOL is_rspamd;
+ uschar spamd_version[8];
++ uschar spamd_short_result[8];
+ uschar spamd_score_char;
+- double spamd_threshold, spamd_score;
++ double spamd_threshold, spamd_score, spamd_reject_score;
+ int spamd_report_offset;
+ uschar *p,*q;
+ int override = 0;
+@@ -122,8 +125,15 @@
+ spamd_address_container *this_spamd =
+ (spamd_address_container *)store_get(sizeof(spamd_address_container));
+
++ /* Check for spamd variant */
++ if( Ustrstr(address, "variant=rspamd") != NULL ) {
++ this_spamd->is_rspamd = TRUE;
++ }
++ else {
++ this_spamd->is_rspamd = FALSE;
++ }
+ /* grok spamd address and port */
+- if( sscanf(CS address, "%s %u", this_spamd->tcp_addr, &(this_spamd->tcp_port)) != 2 ) {
++ if( sscanf(CS address, "%23s %hu", this_spamd->tcp_addr, &(this_spamd->tcp_port)) != 2 ) {
+ log_write(0, LOG_MAIN,
+ "spam acl condition: warning - invalid spamd address: '%s'", address);
+ continue;
+@@ -165,6 +175,7 @@
+ spamd_address_vector[current_server]->tcp_port,
+ 5 ) > -1) {
+ /* connection OK */
++ is_rspamd = spamd_address_vector[current_server]->is_rspamd;
+ break;
+ };
+
+@@ -197,12 +208,28 @@
+ }
+
+ server.sun_family = AF_UNIX;
+- Ustrcpy(server.sun_path, spamd_address_work);
++ p = Ustrstr(spamd_address_work, "variant=rspamd");
++ if( p != NULL ) {
++ is_rspamd = TRUE;
++ /* strip spaces */
++ p --;
++ while (p > spamd_address_work && isspace (*p)) {
++ p --;
++ }
++ Ustrncpy(server.sun_path, spamd_address_work, p - spamd_address_work + 1);
++ /* zero terminate */
++ server.sun_path[p - spamd_address_work + 1] = 0;
++ }
++ else {
++ is_rspamd = FALSE;
++ Ustrcpy(server.sun_path, spamd_address_work);
++ }
++
+
+ if (connect(spamd_sock, (struct sockaddr *) &server, sizeof(struct sockaddr_un)) < 0) {
+ log_write(0, LOG_MAIN|LOG_PANIC,
+ "malware acl condition: spamd: unable to connect to UNIX socket %s (%s)",
+- spamd_address_work, strerror(errno) );
++ server.sun_path, strerror(errno) );
+ (void)fclose(mbox_file);
+ (void)close(spamd_sock);
+ return DEFER;
+@@ -210,22 +237,50 @@
+
+ }
+
++ (void)fcntl(spamd_sock, F_SETFL, O_NONBLOCK);
+ /* now we are connected to spamd on spamd_sock */
+- (void)string_format(spamd_buffer,
+- sizeof(spamd_buffer),
+- "REPORT SPAMC/1.2\r\nUser: %s\r\nContent-length: %ld\r\n\r\n",
+- user_name,
+- mbox_size);
+-
+- /* send our request */
+- if (send(spamd_sock, spamd_buffer, Ustrlen(spamd_buffer), 0) < 0) {
++ if (is_rspamd) {
++ /* rspamd variant */
++ const char *helo;
++ const char *fcrdns;
++ const char *authid;
++ uschar *req_str;
++
++ req_str = string_sprintf("CHECK RSPAMC/1.3\r\nContent-length: %lu\r\n"
++ "Queue-Id: %s\r\nFrom: <%s>\r\nRecipient-Number: %d\r\n", mbox_size,
++ message_id, sender_address, recipients_count);
++ for (i = 0; i < recipients_count; i ++)
++ req_str = string_sprintf("%sRcpt: <%s>\r\n", req_str, recipients_list[i].address);
++ if ((helo = expand_string(US"$sender_helo_name")) != NULL && *helo != '\0')
++ req_str = string_sprintf("%sHelo: %s\r\n", req_str, helo);
++ if ((fcrdns = expand_string(US"$sender_host_name")) != NULL && *fcrdns != '\0')
++ req_str = string_sprintf("%sHostname: %s\r\n", req_str, fcrdns);
++ if (sender_host_address != NULL)
++ req_str = string_sprintf("%sIP: %s\r\n", req_str, sender_host_address);
++ if ((authid = expand_string(US"$authenticated_id")) != NULL && *authid != '\0')
++ req_str = string_sprintf("%sUser: %s\r\n", req_str, authid);
++ req_str = string_sprintf("%s\r\n", req_str);
++ wrote = send(spamd_sock, req_str, Ustrlen(req_str), 0);
++ }
++ else {
++ /* spamassassin variant */
++ (void)string_format(spamd_buffer,
++ sizeof(spamd_buffer),
++ "REPORT SPAMC/1.2\r\nUser: %s\r\nContent-length: %ld\r\n\r\n",
++ user_name,
++ mbox_size);
++ /* send our request */
++ wrote = send(spamd_sock, spamd_buffer, Ustrlen(spamd_buffer), 0);
++ }
++ if(wrote == -1)
++ {
+ (void)close(spamd_sock);
+ log_write(0, LOG_MAIN|LOG_PANIC,
+- "spam acl condition: spamd send failed: %s", strerror(errno));
++ "spam acl condition: spamd send failed: %s", strerror(errno));
+ (void)fclose(mbox_file);
+ (void)close(spamd_sock);
+ return DEFER;
+- };
++ }
+
+ /* now send the file */
+ /* spamd sometimes accepts connections but doesn't read data off
+@@ -304,7 +359,9 @@
+ (void)fclose(mbox_file);
+
+ /* we're done sending, close socket for writing */
+- shutdown(spamd_sock,SHUT_WR);
++ if (!is_rspamd) {
++ shutdown(spamd_sock,SHUT_WR);
++ }
+
+ /* read spamd response using what's left of the timeout.
+ */
+@@ -328,60 +385,93 @@
+ /* reading done */
+ (void)close(spamd_sock);
+
+- /* dig in the spamd output and put the report in a multiline header, if requested */
+- if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nContent-length: %*u\r\n\r\n%lf/%lf\r\n%n",
+- spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
+-
+- /* try to fall back to pre-2.50 spamd output */
+- if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nSpam: %*s ; %lf / %lf\r\n\r\n%n",
+- spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++ if (!is_rspamd) {
++ /* dig in the spamd output and put the report in a multiline header, if requested */
++ if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nContent-length: %*u\r\n\r\n%lf/%lf\r\n%n",
++ spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++
++ /* try to fall back to pre-2.50 spamd output */
++ if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nSpam: %*s ; %lf / %lf\r\n\r\n%n",
++ spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++ log_write(0, LOG_MAIN|LOG_PANIC,
++ "spam acl condition: cannot parse spamd output");
++ return DEFER;
++ };
++ };
++
++ if( spamd_score >= spamd_threshold ) {
++ Ustrcpy(spam_action_buffer, "reject");
++ }
++ else {
++ Ustrcpy(spam_action_buffer, "no action");
++ }
++ }
++ else {
++ /* rspamd variant of reply */
++ int r;
++ if( (r = sscanf(CS spamd_buffer,"RSPAMD/%7s 0 EX_OK\r\nMetric: default; %7s %lf / %lf / %lf\r\n%n",
++ spamd_version,spamd_short_result,&spamd_score,&spamd_threshold,&spamd_reject_score,&spamd_report_offset)) != 5 ) {
+ log_write(0, LOG_MAIN|LOG_PANIC,
+- "spam acl condition: cannot parse spamd output");
++ "spam acl condition: cannot parse spamd output: %d", r);
+ return DEFER;
+ };
+- };
++ /* now parse action */
++ p = &spamd_buffer[spamd_report_offset];
++
++ if( Ustrncmp(p, "Action: ", sizeof("Action: ") - 1) == 0 ) {
++ p += sizeof("Action: ") - 1;
++ q = &spam_action_buffer[0];
++ while (*p && *p != '\r' && (q - spam_action_buffer) < sizeof(spam_action_buffer) - 1) {
++ *q++ = *p++;
++ }
++ *q = '\0';
++ }
++ }
+
+ /* Create report. Since this is a multiline string,
+ we must hack it into shape first */
+ p = &spamd_buffer[spamd_report_offset];
+ q = spam_report_buffer;
+ while (*p != '\0') {
+- /* skip \r */
+- if (*p == '\r') {
+- p++;
+- continue;
+- };
+- *q = *p;
+- q++;
+- if (*p == '\n') {
+- /* add an extra space after the newline to ensure
+- that it is treated as a header continuation line */
+- *q = ' ';
+- q++;
+- };
+- p++;
++ /* skip \r */
++ if (*p == '\r') {
++ p++;
++ continue;
++ };
++ *q = *p;
++ q++;
++ if (*p == '\n') {
++ /* add an extra space after the newline to ensure
++ that it is treated as a header continuation line */
++ *q = ' ';
++ q++;
++ };
++ p++;
+ };
+ /* NULL-terminate */
+ *q = '\0';
+ q--;
+ /* cut off trailing leftovers */
+ while (*q <= ' ') {
+- *q = '\0';
+- q--;
++ *q = '\0';
++ q--;
+ };
++
++ /* common spamd actions */
+ spam_report = spam_report_buffer;
++ spam_action = spam_action_buffer;
+
+ /* create spam bar */
+ spamd_score_char = spamd_score > 0 ? '+' : '-';
+ j = abs((int)(spamd_score));
+ i = 0;
+ if( j != 0 ) {
+- while((i < j) && (i <= MAX_SPAM_BAR_CHARS))
+- spam_bar_buffer[i++] = spamd_score_char;
++ while((i < j) && (i <= MAX_SPAM_BAR_CHARS))
++ spam_bar_buffer[i++] = spamd_score_char;
+ }
+ else{
+- spam_bar_buffer[0] = '/';
+- i = 1;
++ spam_bar_buffer[0] = '/';
++ i = 1;
+ }
+ spam_bar_buffer[i] = '\0';
+ spam_bar = spam_bar_buffer;
+@@ -397,12 +487,12 @@
+
+ /* compare threshold against score */
+ if (spamd_score >= spamd_threshold) {
+- /* spam as determined by user's threshold */
+- spam_rc = OK;
++ /* spam as determined by user's threshold */
++ spam_rc = OK;
+ }
+ else {
+- /* not spam */
+- spam_rc = FAIL;
++ /* not spam */
++ spam_rc = FAIL;
+ };
+
+ /* remember user name and "been here" for it unless spamd_socket was expanded */
+diff -ru exim-4.70.orig/src/spam.h exim-4.70/src/spam.h
+--- exim-4.70.orig/src/spam.h 2016-04-09 13:42:00.235625076 +0200
++++ exim-4.70/src/spam.h 2016-04-09 13:42:37.187633096 +0200
+@@ -24,7 +24,8 @@
+
+ typedef struct spamd_address_container {
+ uschar tcp_addr[24];
+- unsigned int tcp_port;
++ unsigned short int tcp_port;
++ int is_rspamd:1;
+ } spamd_address_container;
+
+ #endif
diff --git a/contrib/exim/patch-exim-src_spam.c.diff.exim-4.85.diff b/contrib/exim/patch-exim-src_spam.c.diff.exim-4.85.diff
new file mode 100644
index 0000000..0389aea
--- /dev/null
+++ b/contrib/exim/patch-exim-src_spam.c.diff.exim-4.85.diff
@@ -0,0 +1,332 @@
+diff -ru exim-4.85.orig/src/expand.c exim-4.85/src/expand.c
+--- exim-4.85.orig/src/expand.c 2016-04-09 13:47:01.707691638 +0200
++++ exim-4.85/src/expand.c 2016-04-09 13:47:29.771697969 +0200
+@@ -652,6 +652,7 @@
+ { "sn8", vtype_filter_int, &filter_sn[8] },
+ { "sn9", vtype_filter_int, &filter_sn[9] },
+ #ifdef WITH_CONTENT_SCAN
++ { "spam_action", vtype_stringptr, &spam_action },
+ { "spam_bar", vtype_stringptr, &spam_bar },
+ { "spam_report", vtype_stringptr, &spam_report },
+ { "spam_score", vtype_stringptr, &spam_score },
+Only in exim-4.85/src: expand.c.orig
+diff -ru exim-4.85.orig/src/globals.c exim-4.85/src/globals.c
+--- exim-4.85.orig/src/globals.c 2016-04-09 13:47:01.695691635 +0200
++++ exim-4.85/src/globals.c 2016-04-09 13:47:29.771697969 +0200
+@@ -1276,6 +1276,7 @@
+ uschar *spamd_address = US"127.0.0.1 783";
+ uschar *spam_bar = NULL;
+ uschar *spam_report = NULL;
++uschar *spam_action = NULL;
+ uschar *spam_score = NULL;
+ uschar *spam_score_int = NULL;
+ #endif
+Only in exim-4.85/src: globals.c.orig
+diff -ru exim-4.85.orig/src/globals.h exim-4.85/src/globals.h
+--- exim-4.85.orig/src/globals.h 2016-04-09 13:47:01.695691635 +0200
++++ exim-4.85/src/globals.h 2016-04-09 13:47:29.771697969 +0200
+@@ -819,6 +819,7 @@
+ extern uschar *spamd_address; /* address for the spamassassin daemon */
+ extern uschar *spam_bar; /* the spam "bar" (textual representation of spam_score) */
+ extern uschar *spam_report; /* the spamd report (multiline) */
++extern uschar *spam_action; /* the spamd action */
+ extern uschar *spam_score; /* the spam score (float) */
+ extern uschar *spam_score_int; /* spam_score * 10 (int) */
+ #endif
+Only in exim-4.85/src: globals.h.orig
+diff -ru exim-4.85.orig/src/spam.c exim-4.85/src/spam.c
+--- exim-4.85.orig/src/spam.c 2016-04-09 13:47:01.711691638 +0200
++++ exim-4.85/src/spam.c 2016-04-09 13:52:12.611762892 +0200
+@@ -14,6 +14,7 @@
+ uschar spam_score_buffer[16];
+ uschar spam_score_int_buffer[16];
+ uschar spam_bar_buffer[128];
++uschar spam_action_buffer[32];
+ uschar spam_report_buffer[32600];
+ uschar prev_user_name[128] = "";
+ int spam_ok = 0;
+@@ -32,9 +33,11 @@
+ int spamd_sock = -1;
+ uschar spamd_buffer[32600];
+ int i, j, offset, result;
++ BOOL is_rspamd;
+ uschar spamd_version[8];
++ uschar spamd_short_result[8];
+ uschar spamd_score_char;
+- double spamd_threshold, spamd_score;
++ double spamd_threshold, spamd_score, spamd_reject_score;
+ int spamd_report_offset;
+ uschar *p,*q;
+ int override = 0;
+@@ -128,8 +131,15 @@
+ spamd_address_container *this_spamd =
+ (spamd_address_container *)store_get(sizeof(spamd_address_container));
+
++ /* Check for spamd variant */
++ if( Ustrstr(address, "variant=rspamd") != NULL ) {
++ this_spamd->is_rspamd = TRUE;
++ }
++ else {
++ this_spamd->is_rspamd = FALSE;
++ }
+ /* grok spamd address and port */
+- if (sscanf(CS address, "%23s %u", this_spamd->tcp_addr, &(this_spamd->tcp_port)) != 2)
++ if (sscanf(CS address, "%23s %hu", this_spamd->tcp_addr, &(this_spamd->tcp_port)) != 2)
+ {
+ log_write(0, LOG_MAIN,
+ "spam acl condition: warning - invalid spamd address: '%s'", address);
+@@ -174,6 +184,7 @@
+ spamd_address_vector[current_server]->tcp_port,
+ 5 ) > -1) {
+ /* connection OK */
++ is_rspamd = spamd_address_vector[current_server]->is_rspamd;
+ break;
+ };
+
+@@ -210,12 +221,28 @@
+ }
+
+ server.sun_family = AF_UNIX;
+- Ustrcpy(server.sun_path, spamd_address_work);
++ p = Ustrstr(spamd_address_work, "variant=rspamd");
++ if( p != NULL ) {
++ is_rspamd = TRUE;
++ /* strip spaces */
++ p --;
++ while (p > spamd_address_work && isspace (*p)) {
++ p --;
++ }
++ Ustrncpy(server.sun_path, spamd_address_work, p - spamd_address_work + 1);
++ /* zero terminate */
++ server.sun_path[p - spamd_address_work + 1] = 0;
++ }
++ else {
++ is_rspamd = FALSE;
++ Ustrcpy(server.sun_path, spamd_address_work);
++ }
++
+
+ if (connect(spamd_sock, (struct sockaddr *) &server, sizeof(struct sockaddr_un)) < 0) {
+ log_write(0, LOG_MAIN|LOG_PANIC,
+ "malware acl condition: spamd: unable to connect to UNIX socket %s (%s)",
+- spamd_address_work, strerror(errno) );
++ server.sun_path, strerror(errno) );
+ (void)fclose(mbox_file);
+ (void)close(spamd_sock);
+ return DEFER;
+@@ -231,22 +258,50 @@
+ return DEFER;
+ }
+
++ (void)fcntl(spamd_sock, F_SETFL, O_NONBLOCK);
+ /* now we are connected to spamd on spamd_sock */
+- (void)string_format(spamd_buffer,
+- sizeof(spamd_buffer),
+- "REPORT SPAMC/1.2\r\nUser: %s\r\nContent-length: %ld\r\n\r\n",
+- user_name,
+- mbox_size);
+-
+- /* send our request */
+- if (send(spamd_sock, spamd_buffer, Ustrlen(spamd_buffer), 0) < 0) {
++ if (is_rspamd) {
++ /* rspamd variant */
++ const char *helo;
++ const char *fcrdns;
++ const char *authid;
++ uschar *req_str;
++
++ req_str = string_sprintf("CHECK RSPAMC/1.3\r\nContent-length: %lu\r\n"
++ "Queue-Id: %s\r\nFrom: <%s>\r\nRecipient-Number: %d\r\n", mbox_size,
++ message_id, sender_address, recipients_count);
++ for (i = 0; i < recipients_count; i ++)
++ req_str = string_sprintf("%sRcpt: <%s>\r\n", req_str, recipients_list[i].address);
++ if ((helo = expand_string(US"$sender_helo_name")) != NULL && *helo != '\0')
++ req_str = string_sprintf("%sHelo: %s\r\n", req_str, helo);
++ if ((fcrdns = expand_string(US"$sender_host_name")) != NULL && *fcrdns != '\0')
++ req_str = string_sprintf("%sHostname: %s\r\n", req_str, fcrdns);
++ if (sender_host_address != NULL)
++ req_str = string_sprintf("%sIP: %s\r\n", req_str, sender_host_address);
++ if ((authid = expand_string(US"$authenticated_id")) != NULL && *authid != '\0')
++ req_str = string_sprintf("%sUser: %s\r\n", req_str, authid);
++ req_str = string_sprintf("%s\r\n", req_str);
++ wrote = send(spamd_sock, req_str, Ustrlen(req_str), 0);
++ }
++ else {
++ /* spamassassin variant */
++ (void)string_format(spamd_buffer,
++ sizeof(spamd_buffer),
++ "REPORT SPAMC/1.2\r\nUser: %s\r\nContent-length: %ld\r\n\r\n",
++ user_name,
++ mbox_size);
++ /* send our request */
++ wrote = send(spamd_sock, spamd_buffer, Ustrlen(spamd_buffer), 0);
++ }
++ if(wrote == -1)
++ {
+ (void)close(spamd_sock);
+ log_write(0, LOG_MAIN|LOG_PANIC,
+- "spam acl condition: spamd send failed: %s", strerror(errno));
++ "spam acl condition: spamd send failed: %s", strerror(errno));
+ (void)fclose(mbox_file);
+ (void)close(spamd_sock);
+ return DEFER;
+- };
++ }
+
+ /* now send the file */
+ /* spamd sometimes accepts connections but doesn't read data off
+@@ -349,60 +404,93 @@
+ /* reading done */
+ (void)close(spamd_sock);
+
+- /* dig in the spamd output and put the report in a multiline header, if requested */
+- if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nContent-length: %*u\r\n\r\n%lf/%lf\r\n%n",
+- spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
+-
+- /* try to fall back to pre-2.50 spamd output */
+- if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nSpam: %*s ; %lf / %lf\r\n\r\n%n",
+- spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++ if (!is_rspamd) {
++ /* dig in the spamd output and put the report in a multiline header, if requested */
++ if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nContent-length: %*u\r\n\r\n%lf/%lf\r\n%n",
++ spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++
++ /* try to fall back to pre-2.50 spamd output */
++ if( sscanf(CS spamd_buffer,"SPAMD/%7s 0 EX_OK\r\nSpam: %*s ; %lf / %lf\r\n\r\n%n",
++ spamd_version,&spamd_score,&spamd_threshold,&spamd_report_offset) != 3 ) {
++ log_write(0, LOG_MAIN|LOG_PANIC,
++ "spam acl condition: cannot parse spamd output");
++ return DEFER;
++ };
++ };
++
++ if( spamd_score >= spamd_threshold ) {
++ Ustrcpy(spam_action_buffer, "reject");
++ }
++ else {
++ Ustrcpy(spam_action_buffer, "no action");
++ }
++ }
++ else {
++ /* rspamd variant of reply */
++ int r;
++ if( (r = sscanf(CS spamd_buffer,"RSPAMD/%7s 0 EX_OK\r\nMetric: default; %7s %lf / %lf / %lf\r\n%n",
++ spamd_version,spamd_short_result,&spamd_score,&spamd_threshold,&spamd_reject_score,&spamd_report_offset)) != 5 ) {
+ log_write(0, LOG_MAIN|LOG_PANIC,
+- "spam acl condition: cannot parse spamd output");
++ "spam acl condition: cannot parse spamd output: %d", r);
+ return DEFER;
+ };
+- };
++ /* now parse action */
++ p = &spamd_buffer[spamd_report_offset];
++
++ if( Ustrncmp(p, "Action: ", sizeof("Action: ") - 1) == 0 ) {
++ p += sizeof("Action: ") - 1;
++ q = &spam_action_buffer[0];
++ while (*p && *p != '\r' && (q - spam_action_buffer) < sizeof(spam_action_buffer) - 1) {
++ *q++ = *p++;
++ }
++ *q = '\0';
++ }
++ }
+
+ /* Create report. Since this is a multiline string,
+ we must hack it into shape first */
+ p = &spamd_buffer[spamd_report_offset];
+ q = spam_report_buffer;
+ while (*p != '\0') {
+- /* skip \r */
+- if (*p == '\r') {
+- p++;
+- continue;
+- };
+- *q = *p;
+- q++;
+- if (*p == '\n') {
+- /* add an extra space after the newline to ensure
+- that it is treated as a header continuation line */
+- *q = ' ';
+- q++;
+- };
+- p++;
++ /* skip \r */
++ if (*p == '\r') {
++ p++;
++ continue;
++ };
++ *q = *p;
++ q++;
++ if (*p == '\n') {
++ /* add an extra space after the newline to ensure
++ that it is treated as a header continuation line */
++ *q = ' ';
++ q++;
++ };
++ p++;
+ };
+ /* NULL-terminate */
+ *q = '\0';
+ q--;
+ /* cut off trailing leftovers */
+ while (*q <= ' ') {
+- *q = '\0';
+- q--;
++ *q = '\0';
++ q--;
+ };
++
++ /* common spamd actions */
+ spam_report = spam_report_buffer;
++ spam_action = spam_action_buffer;
+
+ /* create spam bar */
+ spamd_score_char = spamd_score > 0 ? '+' : '-';
+ j = abs((int)(spamd_score));
+ i = 0;
+ if( j != 0 ) {
+- while((i < j) && (i <= MAX_SPAM_BAR_CHARS))
+- spam_bar_buffer[i++] = spamd_score_char;
++ while((i < j) && (i <= MAX_SPAM_BAR_CHARS))
++ spam_bar_buffer[i++] = spamd_score_char;
+ }
+ else{
+- spam_bar_buffer[0] = '/';
+- i = 1;
++ spam_bar_buffer[0] = '/';
++ i = 1;
+ }
+ spam_bar_buffer[i] = '\0';
+ spam_bar = spam_bar_buffer;
+@@ -418,12 +506,12 @@
+
+ /* compare threshold against score */
+ if (spamd_score >= spamd_threshold) {
+- /* spam as determined by user's threshold */
+- spam_rc = OK;
++ /* spam as determined by user's threshold */
++ spam_rc = OK;
+ }
+ else {
+- /* not spam */
+- spam_rc = FAIL;
++ /* not spam */
++ spam_rc = FAIL;
+ };
+
+ /* remember expanded spamd_address if needed */
+Only in exim-4.85/src: spam.c.orig
+Only in exim-4.85/src: .spam.c.rej.swp
+diff -ru exim-4.85.orig/src/spam.h exim-4.85/src/spam.h
+--- exim-4.85.orig/src/spam.h 2016-04-09 13:47:01.715691640 +0200
++++ exim-4.85/src/spam.h 2016-04-09 13:47:29.775697969 +0200
+@@ -22,7 +22,8 @@
+
+ typedef struct spamd_address_container {
+ uschar tcp_addr[24];
+- unsigned int tcp_port;
++ unsigned short int tcp_port;
++ int is_rspamd:1;
+ } spamd_address_container;
+
+ #endif
diff --git a/contrib/exim/shutdown.patch b/contrib/exim/shutdown.patch
new file mode 100644
index 0000000..e8bf8a0
--- /dev/null
+++ b/contrib/exim/shutdown.patch
@@ -0,0 +1,14 @@
+--- exim4-4.86.2.orig/src/spam.c
++++ exim4-4.86.2/src/spam.c
+@@ -499,7 +499,10 @@ if (ferror(mbox_file))
+ (void)fclose(mbox_file);
+
+ /* we're done sending, close socket for writing */
+-shutdown(spamd_sock,SHUT_WR);
++if (!sd->is_rspamd)
++ {
++ shutdown(spamd_sock,SHUT_WR);
++ }
+
+ /* read spamd response using what's left of the timeout. */
+ memset(spamd_buffer, 0, sizeof(spamd_buffer));
diff --git a/contrib/expected/COPYING b/contrib/expected/COPYING
new file mode 100644
index 0000000..0e259d4
--- /dev/null
+++ b/contrib/expected/COPYING
@@ -0,0 +1,121 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+ CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+ LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+ ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+ INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+ REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+ PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+ THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+ HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+ i. the right to reproduce, adapt, distribute, perform, display,
+ communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+ likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+ subject to the limitations in paragraph 4(a), below;
+ v. rights protecting the extraction, dissemination, use and reuse of data
+ in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+ European Parliament and of the Council of 11 March 1996 on the legal
+ protection of databases, and under any national implementation
+ thereof, including any amended or successor version of such
+ directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+ world based on applicable law or treaty, and any national
+ implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+ surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+ warranties of any kind concerning the Work, express, implied,
+ statutory or otherwise, including without limitation warranties of
+ title, merchantability, fitness for a particular purpose, non
+ infringement, or the absence of latent or other defects, accuracy, or
+ the present or absence of errors, whether or not discoverable, all to
+ the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+ that may apply to the Work or any use thereof, including without
+ limitation any person's Copyright and Related Rights in the Work.
+ Further, Affirmer disclaims responsibility for obtaining any necessary
+ consents, permissions or other rights required for any use of the
+ Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+ party to this document and has no duty or obligation with respect to
+ this CC0 or use of the Work.
diff --git a/contrib/expected/expected.hpp b/contrib/expected/expected.hpp
new file mode 100644
index 0000000..31a5193
--- /dev/null
+++ b/contrib/expected/expected.hpp
@@ -0,0 +1,2326 @@
+///
+// expected - An implementation of std::expected with extensions
+// Written in 2017 by Simon Brand (simonrbrand@gmail.com, @TartanLlama)
+//
+// Documentation available at http://tl.tartanllama.xyz/
+//
+// To the extent possible under law, the author(s) have dedicated all
+// copyright and related and neighboring rights to this software to the
+// public domain worldwide. This software is distributed without any warranty.
+//
+// You should have received a copy of the CC0 Public Domain Dedication
+// along with this software. If not, see
+// <http://creativecommons.org/publicdomain/zero/1.0/>.
+///
+
+#ifndef TL_EXPECTED_HPP
+#define TL_EXPECTED_HPP
+
+#define TL_EXPECTED_VERSION_MAJOR 1
+#define TL_EXPECTED_VERSION_MINOR 0
+#define TL_EXPECTED_VERSION_PATCH 1
+
+#include <exception>
+#include <functional>
+#include <type_traits>
+#include <utility>
+
+#if defined(__EXCEPTIONS) || defined(_CPPUNWIND)
+#define TL_EXPECTED_EXCEPTIONS_ENABLED
+#endif
+
+#if (defined(_MSC_VER) && _MSC_VER == 1900)
+#define TL_EXPECTED_MSVC2015
+#define TL_EXPECTED_MSVC2015_CONSTEXPR
+#else
+#define TL_EXPECTED_MSVC2015_CONSTEXPR constexpr
+#endif
+
+#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ <= 9 && \
+ !defined(__clang__))
+#define TL_EXPECTED_GCC49
+#endif
+
+#if (defined(__GNUC__) && __GNUC__ == 5 && __GNUC_MINOR__ <= 4 && \
+ !defined(__clang__))
+#define TL_EXPECTED_GCC54
+#endif
+
+#if (defined(__GNUC__) && __GNUC__ == 5 && __GNUC_MINOR__ <= 5 && \
+ !defined(__clang__))
+#define TL_EXPECTED_GCC55
+#endif
+
+#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ <= 9 && \
+ !defined(__clang__))
+// GCC < 5 doesn't support overloading on const&& for member functions
+
+#define TL_EXPECTED_NO_CONSTRR
+// GCC < 5 doesn't support some standard C++11 type traits
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \
+ std::has_trivial_copy_constructor<T>
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \
+ std::has_trivial_copy_assign<T>
+
+// This one will be different for GCC 5.7 if it's ever supported
+#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) \
+ std::is_trivially_destructible<T>
+
+// GCC 5 < v < 8 has a bug in is_trivially_copy_constructible which breaks std::vector
+// for non-copyable types
+#elif (defined(__GNUC__) && __GNUC__ < 8 && \
+ !defined(__clang__))
+#ifndef TL_GCC_LESS_8_TRIVIALLY_COPY_CONSTRUCTIBLE_MUTEX
+#define TL_GCC_LESS_8_TRIVIALLY_COPY_CONSTRUCTIBLE_MUTEX
+namespace tl {
+ namespace detail {
+ template<class T>
+ struct is_trivially_copy_constructible : std::is_trivially_copy_constructible<T>{};
+#ifdef _GLIBCXX_VECTOR
+ template<class T, class A>
+ struct is_trivially_copy_constructible<std::vector<T,A>>
+ : std::false_type{};
+#endif
+ }
+}
+#endif
+
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \
+ tl::detail::is_trivially_copy_constructible<T>
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \
+ std::is_trivially_copy_assignable<T>
+#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) std::is_trivially_destructible<T>
+#else
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \
+ std::is_trivially_copy_constructible<T>
+#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \
+ std::is_trivially_copy_assignable<T>
+#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) \
+ std::is_trivially_destructible<T>
+#endif
+
+#if __cplusplus > 201103L
+#define TL_EXPECTED_CXX14
+#endif
+
+#ifdef TL_EXPECTED_GCC49
+#define TL_EXPECTED_GCC49_CONSTEXPR
+#else
+#define TL_EXPECTED_GCC49_CONSTEXPR constexpr
+#endif
+
+#if (__cplusplus == 201103L || defined(TL_EXPECTED_MSVC2015) || \
+ defined(TL_EXPECTED_GCC49))
+#define TL_EXPECTED_11_CONSTEXPR
+#else
+#define TL_EXPECTED_11_CONSTEXPR constexpr
+#endif
+
+namespace tl {
+template <class T, class E> class expected;
+
+#ifndef TL_MONOSTATE_INPLACE_MUTEX
+#define TL_MONOSTATE_INPLACE_MUTEX
+class monostate {};
+
+struct in_place_t {
+ explicit in_place_t() = default;
+};
+static constexpr in_place_t in_place{};
+#endif
+
+template <class E> class unexpected {
+public:
+ static_assert(!std::is_same<E, void>::value, "E must not be void");
+
+ unexpected() = delete;
+ constexpr explicit unexpected(const E &e) : m_val(e) {}
+
+ constexpr explicit unexpected(E &&e) : m_val(std::move(e)) {}
+
+ constexpr const E &value() const & { return m_val; }
+ TL_EXPECTED_11_CONSTEXPR E &value() & { return m_val; }
+ TL_EXPECTED_11_CONSTEXPR E &&value() && { return std::move(m_val); }
+ constexpr const E &&value() const && { return std::move(m_val); }
+
+private:
+ E m_val;
+};
+
+template <class E>
+constexpr bool operator==(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() == rhs.value();
+}
+template <class E>
+constexpr bool operator!=(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() != rhs.value();
+}
+template <class E>
+constexpr bool operator<(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() < rhs.value();
+}
+template <class E>
+constexpr bool operator<=(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() <= rhs.value();
+}
+template <class E>
+constexpr bool operator>(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() > rhs.value();
+}
+template <class E>
+constexpr bool operator>=(const unexpected<E> &lhs, const unexpected<E> &rhs) {
+ return lhs.value() >= rhs.value();
+}
+
+template <class E>
+unexpected<typename std::decay<E>::type> make_unexpected(E &&e) {
+ return unexpected<typename std::decay<E>::type>(std::forward<E>(e));
+}
+
+struct unexpect_t {
+ unexpect_t() = default;
+};
+static constexpr unexpect_t unexpect{};
+
+namespace detail {
+template<typename E>
+[[noreturn]] TL_EXPECTED_11_CONSTEXPR void throw_exception(E &&e) {
+#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ throw std::forward<E>(e);
+#else
+ #ifdef _MSC_VER
+ __assume(0);
+ #else
+ __builtin_unreachable();
+ #endif
+#endif
+}
+
+#ifndef TL_TRAITS_MUTEX
+#define TL_TRAITS_MUTEX
+// C++14-style aliases for brevity
+template <class T> using remove_const_t = typename std::remove_const<T>::type;
+template <class T>
+using remove_reference_t = typename std::remove_reference<T>::type;
+template <class T> using decay_t = typename std::decay<T>::type;
+template <bool E, class T = void>
+using enable_if_t = typename std::enable_if<E, T>::type;
+template <bool B, class T, class F>
+using conditional_t = typename std::conditional<B, T, F>::type;
+
+// std::conjunction from C++17
+template <class...> struct conjunction : std::true_type {};
+template <class B> struct conjunction<B> : B {};
+template <class B, class... Bs>
+struct conjunction<B, Bs...>
+ : std::conditional<bool(B::value), conjunction<Bs...>, B>::type {};
+
+#if defined(_LIBCPP_VERSION) && __cplusplus == 201103L
+#define TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND
+#endif
+
+// In C++11 mode, there's an issue in libc++'s std::mem_fn
+// which results in a hard-error when using it in a noexcept expression
+// in some cases. This is a check to workaround the common failing case.
+#ifdef TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND
+template <class T> struct is_pointer_to_non_const_member_func : std::false_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...)> : std::true_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...)&> : std::true_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...) &&> : std::true_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...) volatile> : std::true_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...) volatile &> : std::true_type {};
+template <class T, class Ret, class... Args>
+struct is_pointer_to_non_const_member_func<Ret(T::*) (Args...) volatile &&> : std::true_type {};
+
+template <class T> struct is_const_or_const_ref : std::false_type {};
+template <class T> struct is_const_or_const_ref<T const&> : std::true_type {};
+template <class T> struct is_const_or_const_ref<T const> : std::true_type {};
+#endif
+
+// std::invoke from C++17
+// https://stackoverflow.com/questions/38288042/c11-14-invoke-workaround
+template <typename Fn, typename... Args,
+#ifdef TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND
+ typename = enable_if_t<!(is_pointer_to_non_const_member_func<Fn>::value
+ && is_const_or_const_ref<Args...>::value)>,
+#endif
+ typename = enable_if_t<std::is_member_pointer<decay_t<Fn>>::value>,
+ int = 0>
+ constexpr auto invoke(Fn && f, Args && ... args) noexcept(
+ noexcept(std::mem_fn(f)(std::forward<Args>(args)...)))
+ -> decltype(std::mem_fn(f)(std::forward<Args>(args)...)) {
+ return std::mem_fn(f)(std::forward<Args>(args)...);
+}
+
+template <typename Fn, typename... Args,
+ typename = enable_if_t<!std::is_member_pointer<decay_t<Fn>>::value>>
+ constexpr auto invoke(Fn && f, Args && ... args) noexcept(
+ noexcept(std::forward<Fn>(f)(std::forward<Args>(args)...)))
+ -> decltype(std::forward<Fn>(f)(std::forward<Args>(args)...)) {
+ return std::forward<Fn>(f)(std::forward<Args>(args)...);
+}
+
+// std::invoke_result from C++17
+template <class F, class, class... Us> struct invoke_result_impl;
+
+template <class F, class... Us>
+struct invoke_result_impl<
+ F, decltype(detail::invoke(std::declval<F>(), std::declval<Us>()...), void()),
+ Us...> {
+ using type = decltype(detail::invoke(std::declval<F>(), std::declval<Us>()...));
+};
+
+template <class F, class... Us>
+using invoke_result = invoke_result_impl<F, void, Us...>;
+
+template <class F, class... Us>
+using invoke_result_t = typename invoke_result<F, Us...>::type;
+
+#if defined(_MSC_VER) && _MSC_VER <= 1900
+// TODO make a version which works with MSVC 2015
+template <class T, class U = T> struct is_swappable : std::true_type {};
+
+template <class T, class U = T> struct is_nothrow_swappable : std::true_type {};
+#else
+// https://stackoverflow.com/questions/26744589/what-is-a-proper-way-to-implement-is-swappable-to-test-for-the-swappable-concept
+namespace swap_adl_tests {
+ // if swap ADL finds this then it would call std::swap otherwise (same
+ // signature)
+ struct tag {};
+
+ template <class T> tag swap(T&, T&);
+ template <class T, std::size_t N> tag swap(T(&a)[N], T(&b)[N]);
+
+ // helper functions to test if an unqualified swap is possible, and if it
+ // becomes std::swap
+ template <class, class> std::false_type can_swap(...) noexcept(false);
+ template <class T, class U,
+ class = decltype(swap(std::declval<T&>(), std::declval<U&>()))>
+ std::true_type can_swap(int) noexcept(noexcept(swap(std::declval<T&>(),
+ std::declval<U&>())));
+
+ template <class, class> std::false_type uses_std(...);
+ template <class T, class U>
+ std::is_same<decltype(swap(std::declval<T&>(), std::declval<U&>())), tag>
+ uses_std(int);
+
+ template <class T>
+ struct is_std_swap_noexcept
+ : std::integral_constant<bool,
+ std::is_nothrow_move_constructible<T>::value&&
+ std::is_nothrow_move_assignable<T>::value> {};
+
+ template <class T, std::size_t N>
+ struct is_std_swap_noexcept<T[N]> : is_std_swap_noexcept<T> {};
+
+ template <class T, class U>
+ struct is_adl_swap_noexcept
+ : std::integral_constant<bool, noexcept(can_swap<T, U>(0))> {};
+} // namespace swap_adl_tests
+
+template <class T, class U = T>
+struct is_swappable
+ : std::integral_constant<
+ bool,
+ decltype(detail::swap_adl_tests::can_swap<T, U>(0))::value &&
+ (!decltype(detail::swap_adl_tests::uses_std<T, U>(0))::value ||
+ (std::is_move_assignable<T>::value &&
+ std::is_move_constructible<T>::value))> {};
+
+template <class T, std::size_t N>
+struct is_swappable<T[N], T[N]>
+ : std::integral_constant<
+ bool,
+ decltype(detail::swap_adl_tests::can_swap<T[N], T[N]>(0))::value &&
+ (!decltype(
+ detail::swap_adl_tests::uses_std<T[N], T[N]>(0))::value ||
+ is_swappable<T, T>::value)> {};
+
+template <class T, class U = T>
+struct is_nothrow_swappable
+ : std::integral_constant<
+ bool,
+ is_swappable<T, U>::value &&
+ ((decltype(detail::swap_adl_tests::uses_std<T, U>(0))::value
+ && detail::swap_adl_tests::is_std_swap_noexcept<T>::value) ||
+ (!decltype(detail::swap_adl_tests::uses_std<T, U>(0))::value &&
+ detail::swap_adl_tests::is_adl_swap_noexcept<T,
+ U>::value))> {
+};
+#endif
+#endif
+
+// Trait for checking if a type is a tl::expected
+template <class T> struct is_expected_impl : std::false_type {};
+template <class T, class E>
+struct is_expected_impl<expected<T, E>> : std::true_type {};
+template <class T> using is_expected = is_expected_impl<decay_t<T>>;
+
+template <class T, class E, class U>
+using expected_enable_forward_value = detail::enable_if_t<
+ std::is_constructible<T, U &&>::value &&
+ !std::is_same<detail::decay_t<U>, in_place_t>::value &&
+ !std::is_same<expected<T, E>, detail::decay_t<U>>::value &&
+ !std::is_same<unexpected<E>, detail::decay_t<U>>::value>;
+
+template <class T, class E, class U, class G, class UR, class GR>
+using expected_enable_from_other = detail::enable_if_t<
+ std::is_constructible<T, UR>::value &&
+ std::is_constructible<E, GR>::value &&
+ !std::is_constructible<T, expected<U, G> &>::value &&
+ !std::is_constructible<T, expected<U, G> &&>::value &&
+ !std::is_constructible<T, const expected<U, G> &>::value &&
+ !std::is_constructible<T, const expected<U, G> &&>::value &&
+ !std::is_convertible<expected<U, G> &, T>::value &&
+ !std::is_convertible<expected<U, G> &&, T>::value &&
+ !std::is_convertible<const expected<U, G> &, T>::value &&
+ !std::is_convertible<const expected<U, G> &&, T>::value>;
+
+template <class T, class U>
+using is_void_or = conditional_t<std::is_void<T>::value, std::true_type, U>;
+
+template <class T>
+using is_copy_constructible_or_void =
+ is_void_or<T, std::is_copy_constructible<T>>;
+
+template <class T>
+using is_move_constructible_or_void =
+ is_void_or<T, std::is_move_constructible<T>>;
+
+template <class T>
+using is_copy_assignable_or_void =
+ is_void_or<T, std::is_copy_assignable<T>>;
+
+
+template <class T>
+using is_move_assignable_or_void =
+ is_void_or<T, std::is_move_assignable<T>>;
+
+
+} // namespace detail
+
+namespace detail {
+struct no_init_t {};
+static constexpr no_init_t no_init{};
+
+// Implements the storage of the values, and ensures that the destructor is
+// trivial if it can be.
+//
+// This specialization is for where neither `T` or `E` is trivially
+// destructible, so the destructors must be called on destruction of the
+// `expected`
+template <class T, class E, bool = std::is_trivially_destructible<T>::value,
+ bool = std::is_trivially_destructible<E>::value>
+struct expected_storage_base {
+ constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
+ constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
+ nullptr>
+ constexpr expected_storage_base(in_place_t, Args &&... args)
+ : m_val(std::forward<Args>(args)...), m_has_val(true) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
+ Args &&... args)
+ : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() {
+ if (m_has_val) {
+ m_val.~T();
+ } else {
+ m_unexpect.~unexpected<E>();
+ }
+ }
+ union {
+ T m_val;
+ unexpected<E> m_unexpect;
+ char m_no_init;
+ };
+ bool m_has_val;
+};
+
+// This specialization is for when both `T` and `E` are trivially-destructible,
+// so the destructor of the `expected` can be trivial.
+template <class T, class E> struct expected_storage_base<T, E, true, true> {
+ constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
+ constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
+ nullptr>
+ constexpr expected_storage_base(in_place_t, Args &&... args)
+ : m_val(std::forward<Args>(args)...), m_has_val(true) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
+ Args &&... args)
+ : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() = default;
+ union {
+ T m_val;
+ unexpected<E> m_unexpect;
+ char m_no_init;
+ };
+ bool m_has_val;
+};
+
+// T is trivial, E is not.
+template <class T, class E> struct expected_storage_base<T, E, true, false> {
+ constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
+ TL_EXPECTED_MSVC2015_CONSTEXPR expected_storage_base(no_init_t)
+ : m_no_init(), m_has_val(false) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
+ nullptr>
+ constexpr expected_storage_base(in_place_t, Args &&... args)
+ : m_val(std::forward<Args>(args)...), m_has_val(true) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
+ Args &&... args)
+ : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() {
+ if (!m_has_val) {
+ m_unexpect.~unexpected<E>();
+ }
+ }
+
+ union {
+ T m_val;
+ unexpected<E> m_unexpect;
+ char m_no_init;
+ };
+ bool m_has_val;
+};
+
+// E is trivial, T is not.
+template <class T, class E> struct expected_storage_base<T, E, false, true> {
+ constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
+ constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
+ nullptr>
+ constexpr expected_storage_base(in_place_t, Args &&... args)
+ : m_val(std::forward<Args>(args)...), m_has_val(true) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
+ Args &&... args)
+ : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() {
+ if (m_has_val) {
+ m_val.~T();
+ }
+ }
+ union {
+ T m_val;
+ unexpected<E> m_unexpect;
+ char m_no_init;
+ };
+ bool m_has_val;
+};
+
+// `T` is `void`, `E` is trivially-destructible
+template <class E> struct expected_storage_base<void, E, false, true> {
+ TL_EXPECTED_MSVC2015_CONSTEXPR expected_storage_base() : m_has_val(true) {}
+ constexpr expected_storage_base(no_init_t) : m_val(), m_has_val(false) {}
+
+ constexpr expected_storage_base(in_place_t) : m_has_val(true) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() = default;
+ struct dummy {};
+ union {
+ unexpected<E> m_unexpect;
+ dummy m_val;
+ };
+ bool m_has_val;
+};
+
+// `T` is `void`, `E` is not trivially-destructible
+template <class E> struct expected_storage_base<void, E, false, false> {
+ constexpr expected_storage_base() : m_dummy(), m_has_val(true) {}
+ constexpr expected_storage_base(no_init_t) : m_dummy(), m_has_val(false) {}
+
+ constexpr expected_storage_base(in_place_t) : m_dummy(), m_has_val(true) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
+ : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected_storage_base(unexpect_t,
+ std::initializer_list<U> il,
+ Args &&... args)
+ : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
+
+ ~expected_storage_base() {
+ if (!m_has_val) {
+ m_unexpect.~unexpected<E>();
+ }
+ }
+
+ union {
+ unexpected<E> m_unexpect;
+ char m_dummy;
+ };
+ bool m_has_val;
+};
+
+// This base class provides some handy member functions which can be used in
+// further derived classes
+template <class T, class E>
+struct expected_operations_base : expected_storage_base<T, E> {
+ using expected_storage_base<T, E>::expected_storage_base;
+
+ template <class... Args> void construct(Args &&... args) noexcept {
+ new (std::addressof(this->m_val)) T(std::forward<Args>(args)...);
+ this->m_has_val = true;
+ }
+
+ template <class Rhs> void construct_with(Rhs &&rhs) noexcept {
+ new (std::addressof(this->m_val)) T(std::forward<Rhs>(rhs).get());
+ this->m_has_val = true;
+ }
+
+ template <class... Args> void construct_error(Args &&... args) noexcept {
+ new (std::addressof(this->m_unexpect))
+ unexpected<E>(std::forward<Args>(args)...);
+ this->m_has_val = false;
+ }
+
+ #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+
+ // These assign overloads ensure that the most efficient assignment
+ // implementation is used while maintaining the strong exception guarantee.
+ // The problematic case is where rhs has a value, but *this does not.
+ //
+ // This overload handles the case where we can just copy-construct `T`
+ // directly into place without throwing.
+ template <class U = T,
+ detail::enable_if_t<std::is_nothrow_copy_constructible<U>::value>
+ * = nullptr>
+ void assign(const expected_operations_base &rhs) noexcept {
+ if (!this->m_has_val && rhs.m_has_val) {
+ geterr().~unexpected<E>();
+ construct(rhs.get());
+ } else {
+ assign_common(rhs);
+ }
+ }
+
+ // This overload handles the case where we can attempt to create a copy of
+ // `T`, then no-throw move it into place if the copy was successful.
+ template <class U = T,
+ detail::enable_if_t<!std::is_nothrow_copy_constructible<U>::value &&
+ std::is_nothrow_move_constructible<U>::value>
+ * = nullptr>
+ void assign(const expected_operations_base &rhs) noexcept {
+ if (!this->m_has_val && rhs.m_has_val) {
+ T tmp = rhs.get();
+ geterr().~unexpected<E>();
+ construct(std::move(tmp));
+ } else {
+ assign_common(rhs);
+ }
+ }
+
+ // This overload is the worst-case, where we have to move-construct the
+ // unexpected value into temporary storage, then try to copy the T into place.
+ // If the construction succeeds, then everything is fine, but if it throws,
+ // then we move the old unexpected value back into place before rethrowing the
+ // exception.
+ template <class U = T,
+ detail::enable_if_t<!std::is_nothrow_copy_constructible<U>::value &&
+ !std::is_nothrow_move_constructible<U>::value>
+ * = nullptr>
+ void assign(const expected_operations_base &rhs) {
+ if (!this->m_has_val && rhs.m_has_val) {
+ auto tmp = std::move(geterr());
+ geterr().~unexpected<E>();
+
+#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ construct(rhs.get());
+ } catch (...) {
+ geterr() = std::move(tmp);
+ throw;
+ }
+#else
+ construct(rhs.get());
+#endif
+ } else {
+ assign_common(rhs);
+ }
+ }
+
+ // These overloads do the same as above, but for rvalues
+ template <class U = T,
+ detail::enable_if_t<std::is_nothrow_move_constructible<U>::value>
+ * = nullptr>
+ void assign(expected_operations_base &&rhs) noexcept {
+ if (!this->m_has_val && rhs.m_has_val) {
+ geterr().~unexpected<E>();
+ construct(std::move(rhs).get());
+ } else {
+ assign_common(std::move(rhs));
+ }
+ }
+
+ template <class U = T,
+ detail::enable_if_t<!std::is_nothrow_move_constructible<U>::value>
+ * = nullptr>
+ void assign(expected_operations_base &&rhs) {
+ if (!this->m_has_val && rhs.m_has_val) {
+ auto tmp = std::move(geterr());
+ geterr().~unexpected<E>();
+#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ construct(std::move(rhs).get());
+ } catch (...) {
+ geterr() = std::move(tmp);
+ throw;
+ }
+#else
+ construct(std::move(rhs).get());
+#endif
+ } else {
+ assign_common(std::move(rhs));
+ }
+ }
+
+ #else
+
+ // If exceptions are disabled then we can just copy-construct
+ void assign(const expected_operations_base &rhs) noexcept {
+ if (!this->m_has_val && rhs.m_has_val) {
+ geterr().~unexpected<E>();
+ construct(rhs.get());
+ } else {
+ assign_common(rhs);
+ }
+ }
+
+ void assign(expected_operations_base &&rhs) noexcept {
+ if (!this->m_has_val && rhs.m_has_val) {
+ geterr().~unexpected<E>();
+ construct(std::move(rhs).get());
+ } else {
+ assign_common(rhs);
+ }
+ }
+
+ #endif
+
+ // The common part of move/copy assigning
+ template <class Rhs> void assign_common(Rhs &&rhs) {
+ if (this->m_has_val) {
+ if (rhs.m_has_val) {
+ get() = std::forward<Rhs>(rhs).get();
+ } else {
+ destroy_val();
+ construct_error(std::forward<Rhs>(rhs).geterr());
+ }
+ } else {
+ if (!rhs.m_has_val) {
+ geterr() = std::forward<Rhs>(rhs).geterr();
+ }
+ }
+ }
+
+ bool has_value() const { return this->m_has_val; }
+
+ TL_EXPECTED_11_CONSTEXPR T &get() & { return this->m_val; }
+ constexpr const T &get() const & { return this->m_val; }
+ TL_EXPECTED_11_CONSTEXPR T &&get() && { return std::move(this->m_val); }
+#ifndef TL_EXPECTED_NO_CONSTRR
+ constexpr const T &&get() const && { return std::move(this->m_val); }
+#endif
+
+ TL_EXPECTED_11_CONSTEXPR unexpected<E> &geterr() & {
+ return this->m_unexpect;
+ }
+ constexpr const unexpected<E> &geterr() const & { return this->m_unexpect; }
+ TL_EXPECTED_11_CONSTEXPR unexpected<E> &&geterr() && {
+ return std::move(this->m_unexpect);
+ }
+#ifndef TL_EXPECTED_NO_CONSTRR
+ constexpr const unexpected<E> &&geterr() const && {
+ return std::move(this->m_unexpect);
+ }
+#endif
+
+ TL_EXPECTED_11_CONSTEXPR void destroy_val() {
+ get().~T();
+ }
+};
+
+// This base class provides some handy member functions which can be used in
+// further derived classes
+template <class E>
+struct expected_operations_base<void, E> : expected_storage_base<void, E> {
+ using expected_storage_base<void, E>::expected_storage_base;
+
+ template <class... Args> void construct() noexcept { this->m_has_val = true; }
+
+ // This function doesn't use its argument, but needs it so that code in
+ // levels above this can work independently of whether T is void
+ template <class Rhs> void construct_with(Rhs &&) noexcept {
+ this->m_has_val = true;
+ }
+
+ template <class... Args> void construct_error(Args &&... args) noexcept {
+ new (std::addressof(this->m_unexpect))
+ unexpected<E>(std::forward<Args>(args)...);
+ this->m_has_val = false;
+ }
+
+ template <class Rhs> void assign(Rhs &&rhs) noexcept {
+ if (!this->m_has_val) {
+ if (rhs.m_has_val) {
+ geterr().~unexpected<E>();
+ construct();
+ } else {
+ geterr() = std::forward<Rhs>(rhs).geterr();
+ }
+ } else {
+ if (!rhs.m_has_val) {
+ construct_error(std::forward<Rhs>(rhs).geterr());
+ }
+ }
+ }
+
+ bool has_value() const { return this->m_has_val; }
+
+ TL_EXPECTED_11_CONSTEXPR unexpected<E> &geterr() & {
+ return this->m_unexpect;
+ }
+ constexpr const unexpected<E> &geterr() const & { return this->m_unexpect; }
+ TL_EXPECTED_11_CONSTEXPR unexpected<E> &&geterr() && {
+ return std::move(this->m_unexpect);
+ }
+#ifndef TL_EXPECTED_NO_CONSTRR
+ constexpr const unexpected<E> &&geterr() const && {
+ return std::move(this->m_unexpect);
+ }
+#endif
+
+ TL_EXPECTED_11_CONSTEXPR void destroy_val() {
+ //no-op
+ }
+};
+
+// This class manages conditionally having a trivial copy constructor
+// This specialization is for when T and E are trivially copy constructible
+template <class T, class E,
+ bool = is_void_or<T, TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T)>::
+ value &&TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(E)::value>
+struct expected_copy_base : expected_operations_base<T, E> {
+ using expected_operations_base<T, E>::expected_operations_base;
+};
+
+// This specialization is for when T or E are not trivially copy constructible
+template <class T, class E>
+struct expected_copy_base<T, E, false> : expected_operations_base<T, E> {
+ using expected_operations_base<T, E>::expected_operations_base;
+
+ expected_copy_base() = default;
+ expected_copy_base(const expected_copy_base &rhs)
+ : expected_operations_base<T, E>(no_init) {
+ if (rhs.has_value()) {
+ this->construct_with(rhs);
+ } else {
+ this->construct_error(rhs.geterr());
+ }
+ }
+
+ expected_copy_base(expected_copy_base &&rhs) = default;
+ expected_copy_base &operator=(const expected_copy_base &rhs) = default;
+ expected_copy_base &operator=(expected_copy_base &&rhs) = default;
+};
+
+// This class manages conditionally having a trivial move constructor
+// Unfortunately there's no way to achieve this in GCC < 5 AFAIK, since it
+// doesn't implement an analogue to std::is_trivially_move_constructible. We
+// have to make do with a non-trivial move constructor even if T is trivially
+// move constructible
+#ifndef TL_EXPECTED_GCC49
+template <class T, class E,
+ bool = is_void_or<T, std::is_trivially_move_constructible<T>>::value
+ &&std::is_trivially_move_constructible<E>::value>
+struct expected_move_base : expected_copy_base<T, E> {
+ using expected_copy_base<T, E>::expected_copy_base;
+};
+#else
+template <class T, class E, bool = false> struct expected_move_base;
+#endif
+template <class T, class E>
+struct expected_move_base<T, E, false> : expected_copy_base<T, E> {
+ using expected_copy_base<T, E>::expected_copy_base;
+
+ expected_move_base() = default;
+ expected_move_base(const expected_move_base &rhs) = default;
+
+ expected_move_base(expected_move_base &&rhs) noexcept(
+ std::is_nothrow_move_constructible<T>::value)
+ : expected_copy_base<T, E>(no_init) {
+ if (rhs.has_value()) {
+ this->construct_with(std::move(rhs));
+ } else {
+ this->construct_error(std::move(rhs.geterr()));
+ }
+ }
+ expected_move_base &operator=(const expected_move_base &rhs) = default;
+ expected_move_base &operator=(expected_move_base &&rhs) = default;
+};
+
+// This class manages conditionally having a trivial copy assignment operator
+template <class T, class E,
+ bool = is_void_or<
+ T, conjunction<TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T),
+ TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T),
+ TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T)>>::value
+ &&TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(E)::value
+ &&TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(E)::value
+ &&TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(E)::value>
+struct expected_copy_assign_base : expected_move_base<T, E> {
+ using expected_move_base<T, E>::expected_move_base;
+};
+
+template <class T, class E>
+struct expected_copy_assign_base<T, E, false> : expected_move_base<T, E> {
+ using expected_move_base<T, E>::expected_move_base;
+
+ expected_copy_assign_base() = default;
+ expected_copy_assign_base(const expected_copy_assign_base &rhs) = default;
+
+ expected_copy_assign_base(expected_copy_assign_base &&rhs) = default;
+ expected_copy_assign_base &operator=(const expected_copy_assign_base &rhs) {
+ this->assign(rhs);
+ return *this;
+ }
+ expected_copy_assign_base &
+ operator=(expected_copy_assign_base &&rhs) = default;
+};
+
+// This class manages conditionally having a trivial move assignment operator
+// Unfortunately there's no way to achieve this in GCC < 5 AFAIK, since it
+// doesn't implement an analogue to std::is_trivially_move_assignable. We have
+// to make do with a non-trivial move assignment operator even if T is trivially
+// move assignable
+#ifndef TL_EXPECTED_GCC49
+template <class T, class E,
+ bool =
+ is_void_or<T, conjunction<std::is_trivially_destructible<T>,
+ std::is_trivially_move_constructible<T>,
+ std::is_trivially_move_assignable<T>>>::
+ value &&std::is_trivially_destructible<E>::value
+ &&std::is_trivially_move_constructible<E>::value
+ &&std::is_trivially_move_assignable<E>::value>
+struct expected_move_assign_base : expected_copy_assign_base<T, E> {
+ using expected_copy_assign_base<T, E>::expected_copy_assign_base;
+};
+#else
+template <class T, class E, bool = false> struct expected_move_assign_base;
+#endif
+
+template <class T, class E>
+struct expected_move_assign_base<T, E, false>
+ : expected_copy_assign_base<T, E> {
+ using expected_copy_assign_base<T, E>::expected_copy_assign_base;
+
+ expected_move_assign_base() = default;
+ expected_move_assign_base(const expected_move_assign_base &rhs) = default;
+
+ expected_move_assign_base(expected_move_assign_base &&rhs) = default;
+
+ expected_move_assign_base &
+ operator=(const expected_move_assign_base &rhs) = default;
+
+ expected_move_assign_base &
+ operator=(expected_move_assign_base &&rhs) noexcept(
+ std::is_nothrow_move_constructible<T>::value
+ &&std::is_nothrow_move_assignable<T>::value) {
+ this->assign(std::move(rhs));
+ return *this;
+ }
+};
+
+// expected_delete_ctor_base will conditionally delete copy and move
+// constructors depending on whether T is copy/move constructible
+template <class T, class E,
+ bool EnableCopy = (is_copy_constructible_or_void<T>::value &&
+ std::is_copy_constructible<E>::value),
+ bool EnableMove = (is_move_constructible_or_void<T>::value &&
+ std::is_move_constructible<E>::value)>
+struct expected_delete_ctor_base {
+ expected_delete_ctor_base() = default;
+ expected_delete_ctor_base(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = default;
+ expected_delete_ctor_base &
+ operator=(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base &
+ operator=(expected_delete_ctor_base &&) noexcept = default;
+};
+
+template <class T, class E>
+struct expected_delete_ctor_base<T, E, true, false> {
+ expected_delete_ctor_base() = default;
+ expected_delete_ctor_base(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = delete;
+ expected_delete_ctor_base &
+ operator=(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base &
+ operator=(expected_delete_ctor_base &&) noexcept = default;
+};
+
+template <class T, class E>
+struct expected_delete_ctor_base<T, E, false, true> {
+ expected_delete_ctor_base() = default;
+ expected_delete_ctor_base(const expected_delete_ctor_base &) = delete;
+ expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = default;
+ expected_delete_ctor_base &
+ operator=(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base &
+ operator=(expected_delete_ctor_base &&) noexcept = default;
+};
+
+template <class T, class E>
+struct expected_delete_ctor_base<T, E, false, false> {
+ expected_delete_ctor_base() = default;
+ expected_delete_ctor_base(const expected_delete_ctor_base &) = delete;
+ expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = delete;
+ expected_delete_ctor_base &
+ operator=(const expected_delete_ctor_base &) = default;
+ expected_delete_ctor_base &
+ operator=(expected_delete_ctor_base &&) noexcept = default;
+};
+
+// expected_delete_assign_base will conditionally delete copy and move
+// constructors depending on whether T and E are copy/move constructible +
+// assignable
+template <class T, class E,
+ bool EnableCopy = (is_copy_constructible_or_void<T>::value &&
+ std::is_copy_constructible<E>::value &&
+ is_copy_assignable_or_void<T>::value &&
+ std::is_copy_assignable<E>::value),
+ bool EnableMove = (is_move_constructible_or_void<T>::value &&
+ std::is_move_constructible<E>::value &&
+ is_move_assignable_or_void<T>::value &&
+ std::is_move_assignable<E>::value)>
+struct expected_delete_assign_base {
+ expected_delete_assign_base() = default;
+ expected_delete_assign_base(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
+ default;
+ expected_delete_assign_base &
+ operator=(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base &
+ operator=(expected_delete_assign_base &&) noexcept = default;
+};
+
+template <class T, class E>
+struct expected_delete_assign_base<T, E, true, false> {
+ expected_delete_assign_base() = default;
+ expected_delete_assign_base(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
+ default;
+ expected_delete_assign_base &
+ operator=(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base &
+ operator=(expected_delete_assign_base &&) noexcept = delete;
+};
+
+template <class T, class E>
+struct expected_delete_assign_base<T, E, false, true> {
+ expected_delete_assign_base() = default;
+ expected_delete_assign_base(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
+ default;
+ expected_delete_assign_base &
+ operator=(const expected_delete_assign_base &) = delete;
+ expected_delete_assign_base &
+ operator=(expected_delete_assign_base &&) noexcept = default;
+};
+
+template <class T, class E>
+struct expected_delete_assign_base<T, E, false, false> {
+ expected_delete_assign_base() = default;
+ expected_delete_assign_base(const expected_delete_assign_base &) = default;
+ expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
+ default;
+ expected_delete_assign_base &
+ operator=(const expected_delete_assign_base &) = delete;
+ expected_delete_assign_base &
+ operator=(expected_delete_assign_base &&) noexcept = delete;
+};
+
+// This is needed to be able to construct the expected_default_ctor_base which
+// follows, while still conditionally deleting the default constructor.
+struct default_constructor_tag {
+ explicit constexpr default_constructor_tag() = default;
+};
+
+// expected_default_ctor_base will ensure that expected has a deleted default
+// consturctor if T is not default constructible.
+// This specialization is for when T is default constructible
+template <class T, class E,
+ bool Enable =
+ std::is_default_constructible<T>::value || std::is_void<T>::value>
+struct expected_default_ctor_base {
+ constexpr expected_default_ctor_base() noexcept = default;
+ constexpr expected_default_ctor_base(
+ expected_default_ctor_base const &) noexcept = default;
+ constexpr expected_default_ctor_base(expected_default_ctor_base &&) noexcept =
+ default;
+ expected_default_ctor_base &
+ operator=(expected_default_ctor_base const &) noexcept = default;
+ expected_default_ctor_base &
+ operator=(expected_default_ctor_base &&) noexcept = default;
+
+ constexpr explicit expected_default_ctor_base(default_constructor_tag) {}
+};
+
+// This specialization is for when T is not default constructible
+template <class T, class E> struct expected_default_ctor_base<T, E, false> {
+ constexpr expected_default_ctor_base() noexcept = delete;
+ constexpr expected_default_ctor_base(
+ expected_default_ctor_base const &) noexcept = default;
+ constexpr expected_default_ctor_base(expected_default_ctor_base &&) noexcept =
+ default;
+ expected_default_ctor_base &
+ operator=(expected_default_ctor_base const &) noexcept = default;
+ expected_default_ctor_base &
+ operator=(expected_default_ctor_base &&) noexcept = default;
+
+ constexpr explicit expected_default_ctor_base(default_constructor_tag) {}
+};
+} // namespace detail
+
+template <class E> class bad_expected_access : public std::exception {
+public:
+ explicit bad_expected_access(E e) : m_val(std::move(e)) {}
+
+ virtual const char *what() const noexcept override {
+ return "Bad expected access";
+ }
+
+ const E &error() const & { return m_val; }
+ E &error() & { return m_val; }
+ const E &&error() const && { return std::move(m_val); }
+ E &&error() && { return std::move(m_val); }
+
+private:
+ E m_val;
+};
+
+/// An `expected<T, E>` object is an object that contains the storage for
+/// another object and manages the lifetime of this contained object `T`.
+/// Alternatively it could contain the storage for another unexpected object
+/// `E`. The contained object may not be initialized after the expected object
+/// has been initialized, and may not be destroyed before the expected object
+/// has been destroyed. The initialization state of the contained object is
+/// tracked by the expected object.
+template <class T, class E>
+class expected : private detail::expected_move_assign_base<T, E>,
+ private detail::expected_delete_ctor_base<T, E>,
+ private detail::expected_delete_assign_base<T, E>,
+ private detail::expected_default_ctor_base<T, E> {
+ static_assert(!std::is_reference<T>::value, "T must not be a reference");
+ static_assert(!std::is_same<T, std::remove_cv<in_place_t>>::value,
+ "T must not be in_place_t");
+ static_assert(!std::is_same<T, std::remove_cv<unexpect_t>>::value,
+ "T must not be unexpect_t");
+ static_assert(!std::is_same<T, std::remove_cv<unexpected<E>>>::value,
+ "T must not be unexpected<E>");
+ static_assert(!std::is_reference<E>::value, "E must not be a reference");
+
+ T *valptr() { return std::addressof(this->m_val); }
+ const T *valptr() const { return std::addressof(this->m_val); }
+ unexpected<E> *errptr() { return std::addressof(this->m_unexpect); }
+ const unexpected<E> *errptr() const { return std::addressof(this->m_unexpect); }
+
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR U &val() {
+ return this->m_val;
+ }
+ TL_EXPECTED_11_CONSTEXPR unexpected<E> &err() { return this->m_unexpect; }
+
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ constexpr const U &val() const {
+ return this->m_val;
+ }
+ constexpr const unexpected<E> &err() const { return this->m_unexpect; }
+
+ using impl_base = detail::expected_move_assign_base<T, E>;
+ using ctor_base = detail::expected_default_ctor_base<T, E>;
+
+public:
+ typedef T value_type;
+ typedef E error_type;
+ typedef unexpected<E> unexpected_type;
+
+#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \
+ !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) & {
+ return and_then_impl(*this, std::forward<F>(f));
+ }
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) && {
+ return and_then_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F> constexpr auto and_then(F &&f) const & {
+ return and_then_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F> constexpr auto and_then(F &&f) const && {
+ return and_then_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+
+#else
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR auto
+ and_then(F &&f) & -> decltype(and_then_impl(std::declval<expected&>(), std::forward<F>(f))) {
+ return and_then_impl(*this, std::forward<F>(f));
+ }
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) && -> decltype(
+ and_then_impl(std::declval<expected&&>(), std::forward<F>(f))) {
+ return and_then_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F>
+ constexpr auto and_then(F &&f) const & -> decltype(
+ and_then_impl(std::declval<expected const&>(), std::forward<F>(f))) {
+ return and_then_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F>
+ constexpr auto and_then(F &&f) const && -> decltype(
+ and_then_impl(std::declval<expected const&&>(), std::forward<F>(f))) {
+ return and_then_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+#endif
+
+#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \
+ !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto map(F &&f) & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto map(F &&f) && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F> constexpr auto map(F &&f) const & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F> constexpr auto map(F &&f) const && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+#else
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(
+ expected_map_impl(std::declval<expected &>(), std::declval<F &&>()))
+ map(F &&f) & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(
+ expected_map_impl(std::declval<expected>(), std::declval<F &&>()))
+ map(F &&f) && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F>
+ constexpr decltype(expected_map_impl(std::declval<const expected &>(),
+ std::declval<F &&>()))
+ map(F &&f) const & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F>
+ constexpr decltype(expected_map_impl(std::declval<const expected &&>(),
+ std::declval<F &&>()))
+ map(F &&f) const && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+#endif
+
+#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \
+ !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto transform(F &&f) & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto transform(F &&f) && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F> constexpr auto transform(F &&f) const & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F> constexpr auto transform(F &&f) const && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+#else
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(
+ expected_map_impl(std::declval<expected &>(), std::declval<F &&>()))
+ transform(F &&f) & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(
+ expected_map_impl(std::declval<expected>(), std::declval<F &&>()))
+ transform(F &&f) && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F>
+ constexpr decltype(expected_map_impl(std::declval<const expected &>(),
+ std::declval<F &&>()))
+ transform(F &&f) const & {
+ return expected_map_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F>
+ constexpr decltype(expected_map_impl(std::declval<const expected &&>(),
+ std::declval<F &&>()))
+ transform(F &&f) const && {
+ return expected_map_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+#endif
+
+#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \
+ !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto map_error(F &&f) & {
+ return map_error_impl(*this, std::forward<F>(f));
+ }
+ template <class F> TL_EXPECTED_11_CONSTEXPR auto map_error(F &&f) && {
+ return map_error_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F> constexpr auto map_error(F &&f) const & {
+ return map_error_impl(*this, std::forward<F>(f));
+ }
+ template <class F> constexpr auto map_error(F &&f) const && {
+ return map_error_impl(std::move(*this), std::forward<F>(f));
+ }
+#else
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval<expected &>(),
+ std::declval<F &&>()))
+ map_error(F &&f) & {
+ return map_error_impl(*this, std::forward<F>(f));
+ }
+ template <class F>
+ TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval<expected &&>(),
+ std::declval<F &&>()))
+ map_error(F &&f) && {
+ return map_error_impl(std::move(*this), std::forward<F>(f));
+ }
+ template <class F>
+ constexpr decltype(map_error_impl(std::declval<const expected &>(),
+ std::declval<F &&>()))
+ map_error(F &&f) const & {
+ return map_error_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F>
+ constexpr decltype(map_error_impl(std::declval<const expected &&>(),
+ std::declval<F &&>()))
+ map_error(F &&f) const && {
+ return map_error_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+#endif
+ template <class F> expected TL_EXPECTED_11_CONSTEXPR or_else(F &&f) & {
+ return or_else_impl(*this, std::forward<F>(f));
+ }
+
+ template <class F> expected TL_EXPECTED_11_CONSTEXPR or_else(F &&f) && {
+ return or_else_impl(std::move(*this), std::forward<F>(f));
+ }
+
+ template <class F> expected constexpr or_else(F &&f) const & {
+ return or_else_impl(*this, std::forward<F>(f));
+ }
+
+#ifndef TL_EXPECTED_NO_CONSTRR
+ template <class F> expected constexpr or_else(F &&f) const && {
+ return or_else_impl(std::move(*this), std::forward<F>(f));
+ }
+#endif
+ constexpr expected() = default;
+ constexpr expected(const expected &rhs) = default;
+ constexpr expected(expected &&rhs) = default;
+ expected &operator=(const expected &rhs) = default;
+ expected &operator=(expected &&rhs) = default;
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
+ nullptr>
+ constexpr expected(in_place_t, Args &&... args)
+ : impl_base(in_place, std::forward<Args>(args)...),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr expected(in_place_t, std::initializer_list<U> il, Args &&... args)
+ : impl_base(in_place, il, std::forward<Args>(args)...),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <class G = E,
+ detail::enable_if_t<std::is_constructible<E, const G &>::value> * =
+ nullptr,
+ detail::enable_if_t<!std::is_convertible<const G &, E>::value> * =
+ nullptr>
+ explicit constexpr expected(const unexpected<G> &e)
+ : impl_base(unexpect, e.value()),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <
+ class G = E,
+ detail::enable_if_t<std::is_constructible<E, const G &>::value> * =
+ nullptr,
+ detail::enable_if_t<std::is_convertible<const G &, E>::value> * = nullptr>
+ constexpr expected(unexpected<G> const &e)
+ : impl_base(unexpect, e.value()),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <
+ class G = E,
+ detail::enable_if_t<std::is_constructible<E, G &&>::value> * = nullptr,
+ detail::enable_if_t<!std::is_convertible<G &&, E>::value> * = nullptr>
+ explicit constexpr expected(unexpected<G> &&e) noexcept(
+ std::is_nothrow_constructible<E, G &&>::value)
+ : impl_base(unexpect, std::move(e.value())),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <
+ class G = E,
+ detail::enable_if_t<std::is_constructible<E, G &&>::value> * = nullptr,
+ detail::enable_if_t<std::is_convertible<G &&, E>::value> * = nullptr>
+ constexpr expected(unexpected<G> &&e) noexcept(
+ std::is_nothrow_constructible<E, G &&>::value)
+ : impl_base(unexpect, std::move(e.value())),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <class... Args,
+ detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
+ nullptr>
+ constexpr explicit expected(unexpect_t, Args &&... args)
+ : impl_base(unexpect, std::forward<Args>(args)...),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_constructible<
+ E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ constexpr explicit expected(unexpect_t, std::initializer_list<U> il,
+ Args &&... args)
+ : impl_base(unexpect, il, std::forward<Args>(args)...),
+ ctor_base(detail::default_constructor_tag{}) {}
+
+ template <class U, class G,
+ detail::enable_if_t<!(std::is_convertible<U const &, T>::value &&
+ std::is_convertible<G const &, E>::value)> * =
+ nullptr,
+ detail::expected_enable_from_other<T, E, U, G, const U &, const G &>
+ * = nullptr>
+ explicit TL_EXPECTED_11_CONSTEXPR expected(const expected<U, G> &rhs)
+ : ctor_base(detail::default_constructor_tag{}) {
+ if (rhs.has_value()) {
+ this->construct(*rhs);
+ } else {
+ this->construct_error(rhs.error());
+ }
+ }
+
+ template <class U, class G,
+ detail::enable_if_t<(std::is_convertible<U const &, T>::value &&
+ std::is_convertible<G const &, E>::value)> * =
+ nullptr,
+ detail::expected_enable_from_other<T, E, U, G, const U &, const G &>
+ * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR expected(const expected<U, G> &rhs)
+ : ctor_base(detail::default_constructor_tag{}) {
+ if (rhs.has_value()) {
+ this->construct(*rhs);
+ } else {
+ this->construct_error(rhs.error());
+ }
+ }
+
+ template <
+ class U, class G,
+ detail::enable_if_t<!(std::is_convertible<U &&, T>::value &&
+ std::is_convertible<G &&, E>::value)> * = nullptr,
+ detail::expected_enable_from_other<T, E, U, G, U &&, G &&> * = nullptr>
+ explicit TL_EXPECTED_11_CONSTEXPR expected(expected<U, G> &&rhs)
+ : ctor_base(detail::default_constructor_tag{}) {
+ if (rhs.has_value()) {
+ this->construct(std::move(*rhs));
+ } else {
+ this->construct_error(std::move(rhs.error()));
+ }
+ }
+
+ template <
+ class U, class G,
+ detail::enable_if_t<(std::is_convertible<U &&, T>::value &&
+ std::is_convertible<G &&, E>::value)> * = nullptr,
+ detail::expected_enable_from_other<T, E, U, G, U &&, G &&> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR expected(expected<U, G> &&rhs)
+ : ctor_base(detail::default_constructor_tag{}) {
+ if (rhs.has_value()) {
+ this->construct(std::move(*rhs));
+ } else {
+ this->construct_error(std::move(rhs.error()));
+ }
+ }
+
+ template <
+ class U = T,
+ detail::enable_if_t<!std::is_convertible<U &&, T>::value> * = nullptr,
+ detail::expected_enable_forward_value<T, E, U> * = nullptr>
+ explicit TL_EXPECTED_MSVC2015_CONSTEXPR expected(U &&v)
+ : expected(in_place, std::forward<U>(v)) {}
+
+ template <
+ class U = T,
+ detail::enable_if_t<std::is_convertible<U &&, T>::value> * = nullptr,
+ detail::expected_enable_forward_value<T, E, U> * = nullptr>
+ TL_EXPECTED_MSVC2015_CONSTEXPR expected(U &&v)
+ : expected(in_place, std::forward<U>(v)) {}
+
+ template <
+ class U = T, class G = T,
+ detail::enable_if_t<std::is_nothrow_constructible<T, U &&>::value> * =
+ nullptr,
+ detail::enable_if_t<!std::is_void<G>::value> * = nullptr,
+ detail::enable_if_t<
+ (!std::is_same<expected<T, E>, detail::decay_t<U>>::value &&
+ !detail::conjunction<std::is_scalar<T>,
+ std::is_same<T, detail::decay_t<U>>>::value &&
+ std::is_constructible<T, U>::value &&
+ std::is_assignable<G &, U>::value &&
+ std::is_nothrow_move_constructible<E>::value)> * = nullptr>
+ expected &operator=(U &&v) {
+ if (has_value()) {
+ val() = std::forward<U>(v);
+ } else {
+ err().~unexpected<E>();
+ ::new (valptr()) T(std::forward<U>(v));
+ this->m_has_val = true;
+ }
+
+ return *this;
+ }
+
+ template <
+ class U = T, class G = T,
+ detail::enable_if_t<!std::is_nothrow_constructible<T, U &&>::value> * =
+ nullptr,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr,
+ detail::enable_if_t<
+ (!std::is_same<expected<T, E>, detail::decay_t<U>>::value &&
+ !detail::conjunction<std::is_scalar<T>,
+ std::is_same<T, detail::decay_t<U>>>::value &&
+ std::is_constructible<T, U>::value &&
+ std::is_assignable<G &, U>::value &&
+ std::is_nothrow_move_constructible<E>::value)> * = nullptr>
+ expected &operator=(U &&v) {
+ if (has_value()) {
+ val() = std::forward<U>(v);
+ } else {
+ auto tmp = std::move(err());
+ err().~unexpected<E>();
+
+ #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ ::new (valptr()) T(std::forward<U>(v));
+ this->m_has_val = true;
+ } catch (...) {
+ err() = std::move(tmp);
+ throw;
+ }
+ #else
+ ::new (valptr()) T(std::forward<U>(v));
+ this->m_has_val = true;
+ #endif
+ }
+
+ return *this;
+ }
+
+ template <class G = E,
+ detail::enable_if_t<std::is_nothrow_copy_constructible<G>::value &&
+ std::is_assignable<G &, G>::value> * = nullptr>
+ expected &operator=(const unexpected<G> &rhs) {
+ if (!has_value()) {
+ err() = rhs;
+ } else {
+ this->destroy_val();
+ ::new (errptr()) unexpected<E>(rhs);
+ this->m_has_val = false;
+ }
+
+ return *this;
+ }
+
+ template <class G = E,
+ detail::enable_if_t<std::is_nothrow_move_constructible<G>::value &&
+ std::is_move_assignable<G>::value> * = nullptr>
+ expected &operator=(unexpected<G> &&rhs) noexcept {
+ if (!has_value()) {
+ err() = std::move(rhs);
+ } else {
+ this->destroy_val();
+ ::new (errptr()) unexpected<E>(std::move(rhs));
+ this->m_has_val = false;
+ }
+
+ return *this;
+ }
+
+ template <class... Args, detail::enable_if_t<std::is_nothrow_constructible<
+ T, Args &&...>::value> * = nullptr>
+ void emplace(Args &&... args) {
+ if (has_value()) {
+ val() = T(std::forward<Args>(args)...);
+ } else {
+ err().~unexpected<E>();
+ ::new (valptr()) T(std::forward<Args>(args)...);
+ this->m_has_val = true;
+ }
+ }
+
+ template <class... Args, detail::enable_if_t<!std::is_nothrow_constructible<
+ T, Args &&...>::value> * = nullptr>
+ void emplace(Args &&... args) {
+ if (has_value()) {
+ val() = T(std::forward<Args>(args)...);
+ } else {
+ auto tmp = std::move(err());
+ err().~unexpected<E>();
+
+ #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ ::new (valptr()) T(std::forward<Args>(args)...);
+ this->m_has_val = true;
+ } catch (...) {
+ err() = std::move(tmp);
+ throw;
+ }
+ #else
+ ::new (valptr()) T(std::forward<Args>(args)...);
+ this->m_has_val = true;
+ #endif
+ }
+ }
+
+ template <class U, class... Args,
+ detail::enable_if_t<std::is_nothrow_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ void emplace(std::initializer_list<U> il, Args &&... args) {
+ if (has_value()) {
+ T t(il, std::forward<Args>(args)...);
+ val() = std::move(t);
+ } else {
+ err().~unexpected<E>();
+ ::new (valptr()) T(il, std::forward<Args>(args)...);
+ this->m_has_val = true;
+ }
+ }
+
+ template <class U, class... Args,
+ detail::enable_if_t<!std::is_nothrow_constructible<
+ T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
+ void emplace(std::initializer_list<U> il, Args &&... args) {
+ if (has_value()) {
+ T t(il, std::forward<Args>(args)...);
+ val() = std::move(t);
+ } else {
+ auto tmp = std::move(err());
+ err().~unexpected<E>();
+
+ #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ ::new (valptr()) T(il, std::forward<Args>(args)...);
+ this->m_has_val = true;
+ } catch (...) {
+ err() = std::move(tmp);
+ throw;
+ }
+ #else
+ ::new (valptr()) T(il, std::forward<Args>(args)...);
+ this->m_has_val = true;
+ #endif
+ }
+ }
+
+private:
+ using t_is_void = std::true_type;
+ using t_is_not_void = std::false_type;
+ using t_is_nothrow_move_constructible = std::true_type;
+ using move_constructing_t_can_throw = std::false_type;
+ using e_is_nothrow_move_constructible = std::true_type;
+ using move_constructing_e_can_throw = std::false_type;
+
+ void swap_where_both_have_value(expected &/*rhs*/ , t_is_void) noexcept {
+ // swapping void is a no-op
+ }
+
+ void swap_where_both_have_value(expected &rhs, t_is_not_void) {
+ using std::swap;
+ swap(val(), rhs.val());
+ }
+
+ void swap_where_only_one_has_value(expected &rhs, t_is_void) noexcept(
+ std::is_nothrow_move_constructible<E>::value) {
+ ::new (errptr()) unexpected_type(std::move(rhs.err()));
+ rhs.err().~unexpected_type();
+ std::swap(this->m_has_val, rhs.m_has_val);
+ }
+
+ void swap_where_only_one_has_value(expected &rhs, t_is_not_void) {
+ swap_where_only_one_has_value_and_t_is_not_void(
+ rhs, typename std::is_nothrow_move_constructible<T>::type{},
+ typename std::is_nothrow_move_constructible<E>::type{});
+ }
+
+ void swap_where_only_one_has_value_and_t_is_not_void(
+ expected &rhs, t_is_nothrow_move_constructible,
+ e_is_nothrow_move_constructible) noexcept {
+ auto temp = std::move(val());
+ val().~T();
+ ::new (errptr()) unexpected_type(std::move(rhs.err()));
+ rhs.err().~unexpected_type();
+ ::new (rhs.valptr()) T(std::move(temp));
+ std::swap(this->m_has_val, rhs.m_has_val);
+ }
+
+ void swap_where_only_one_has_value_and_t_is_not_void(
+ expected &rhs, t_is_nothrow_move_constructible,
+ move_constructing_e_can_throw) {
+ auto temp = std::move(val());
+ val().~T();
+#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ ::new (errptr()) unexpected_type(std::move(rhs.err()));
+ rhs.err().~unexpected_type();
+ ::new (rhs.valptr()) T(std::move(temp));
+ std::swap(this->m_has_val, rhs.m_has_val);
+ } catch (...) {
+ val() = std::move(temp);
+ throw;
+ }
+#else
+ ::new (errptr()) unexpected_type(std::move(rhs.err()));
+ rhs.err().~unexpected_type();
+ ::new (rhs.valptr()) T(std::move(temp));
+ std::swap(this->m_has_val, rhs.m_has_val);
+#endif
+ }
+
+ void swap_where_only_one_has_value_and_t_is_not_void(
+ expected &rhs, move_constructing_t_can_throw,
+ t_is_nothrow_move_constructible) {
+ auto temp = std::move(rhs.err());
+ rhs.err().~unexpected_type();
+#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
+ try {
+ ::new (rhs.valptr()) T(val());
+ val().~T();
+ ::new (errptr()) unexpected_type(std::move(temp));
+ std::swap(this->m_has_val, rhs.m_has_val);
+ } catch (...) {
+ rhs.err() = std::move(temp);
+ throw;
+ }
+#else
+ ::new (rhs.valptr()) T(val());
+ val().~T();
+ ::new (errptr()) unexpected_type(std::move(temp));
+ std::swap(this->m_has_val, rhs.m_has_val);
+#endif
+ }
+
+public:
+ template <class OT = T, class OE = E>
+ detail::enable_if_t<detail::is_swappable<OT>::value &&
+ detail::is_swappable<OE>::value &&
+ (std::is_nothrow_move_constructible<OT>::value ||
+ std::is_nothrow_move_constructible<OE>::value)>
+ swap(expected &rhs) noexcept(
+ std::is_nothrow_move_constructible<T>::value
+ &&detail::is_nothrow_swappable<T>::value
+ &&std::is_nothrow_move_constructible<E>::value
+ &&detail::is_nothrow_swappable<E>::value) {
+ if (has_value() && rhs.has_value()) {
+ swap_where_both_have_value(rhs, typename std::is_void<T>::type{});
+ } else if (!has_value() && rhs.has_value()) {
+ rhs.swap(*this);
+ } else if (has_value()) {
+ swap_where_only_one_has_value(rhs, typename std::is_void<T>::type{});
+ } else {
+ using std::swap;
+ swap(err(), rhs.err());
+ }
+ }
+
+ constexpr const T *operator->() const { return valptr(); }
+ TL_EXPECTED_11_CONSTEXPR T *operator->() { return valptr(); }
+
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ constexpr const U &operator*() const & {
+ return val();
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR U &operator*() & {
+ return val();
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ constexpr const U &&operator*() const && {
+ return std::move(val());
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR U &&operator*() && {
+ return std::move(val());
+ }
+
+ constexpr bool has_value() const noexcept { return this->m_has_val; }
+ constexpr explicit operator bool() const noexcept { return this->m_has_val; }
+
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR const U &value() const & {
+ if (!has_value())
+ detail::throw_exception(bad_expected_access<E>(err().value()));
+ return val();
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR U &value() & {
+ if (!has_value())
+ detail::throw_exception(bad_expected_access<E>(err().value()));
+ return val();
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR const U &&value() const && {
+ if (!has_value())
+ detail::throw_exception(bad_expected_access<E>(std::move(err()).value()));
+ return std::move(val());
+ }
+ template <class U = T,
+ detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
+ TL_EXPECTED_11_CONSTEXPR U &&value() && {
+ if (!has_value())
+ detail::throw_exception(bad_expected_access<E>(std::move(err()).value()));
+ return std::move(val());
+ }
+
+ constexpr const E &error() const & { return err().value(); }
+ TL_EXPECTED_11_CONSTEXPR E &error() & { return err().value(); }
+ constexpr const E &&error() const && { return std::move(err().value()); }
+ TL_EXPECTED_11_CONSTEXPR E &&error() && { return std::move(err().value()); }
+
+ template <class U> constexpr T value_or(U &&v) const & {
+ static_assert(std::is_copy_constructible<T>::value &&
+ std::is_convertible<U &&, T>::value,
+ "T must be copy-constructible and convertible to from U&&");
+ return bool(*this) ? **this : static_cast<T>(std::forward<U>(v));
+ }
+ template <class U> TL_EXPECTED_11_CONSTEXPR T value_or(U &&v) && {
+ static_assert(std::is_move_constructible<T>::value &&
+ std::is_convertible<U &&, T>::value,
+ "T must be move-constructible and convertible to from U&&");
+ return bool(*this) ? std::move(**this) : static_cast<T>(std::forward<U>(v));
+ }
+};
+
+namespace detail {
+template <class Exp> using exp_t = typename detail::decay_t<Exp>::value_type;
+template <class Exp> using err_t = typename detail::decay_t<Exp>::error_type;
+template <class Exp, class Ret> using ret_t = expected<Ret, err_t<Exp>>;
+
+#ifdef TL_EXPECTED_CXX14
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>()))>
+constexpr auto and_then_impl(Exp &&exp, F &&f) {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+
+ return exp.has_value()
+ ? detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp))
+ : Ret(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>()))>
+constexpr auto and_then_impl(Exp &&exp, F &&f) {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+
+ return exp.has_value() ? detail::invoke(std::forward<F>(f))
+ : Ret(unexpect, std::forward<Exp>(exp).error());
+}
+#else
+template <class> struct TC;
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>())),
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr>
+auto and_then_impl(Exp &&exp, F &&f) -> Ret {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+
+ return exp.has_value()
+ ? detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp))
+ : Ret(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>())),
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr>
+constexpr auto and_then_impl(Exp &&exp, F &&f) -> Ret {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+
+ return exp.has_value() ? detail::invoke(std::forward<F>(f))
+ : Ret(unexpect, std::forward<Exp>(exp).error());
+}
+#endif
+
+#ifdef TL_EXPECTED_CXX14
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto expected_map_impl(Exp &&exp, F &&f) {
+ using result = ret_t<Exp, detail::decay_t<Ret>>;
+ return exp.has_value() ? result(detail::invoke(std::forward<F>(f),
+ *std::forward<Exp>(exp)))
+ : result(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto expected_map_impl(Exp &&exp, F &&f) {
+ using result = expected<void, err_t<Exp>>;
+ if (exp.has_value()) {
+ detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp));
+ return result();
+ }
+
+ return result(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto expected_map_impl(Exp &&exp, F &&f) {
+ using result = ret_t<Exp, detail::decay_t<Ret>>;
+ return exp.has_value() ? result(detail::invoke(std::forward<F>(f)))
+ : result(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto expected_map_impl(Exp &&exp, F &&f) {
+ using result = expected<void, err_t<Exp>>;
+ if (exp.has_value()) {
+ detail::invoke(std::forward<F>(f));
+ return result();
+ }
+
+ return result(unexpect, std::forward<Exp>(exp).error());
+}
+#else
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+
+constexpr auto expected_map_impl(Exp &&exp, F &&f)
+ -> ret_t<Exp, detail::decay_t<Ret>> {
+ using result = ret_t<Exp, detail::decay_t<Ret>>;
+
+ return exp.has_value() ? result(detail::invoke(std::forward<F>(f),
+ *std::forward<Exp>(exp)))
+ : result(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ *std::declval<Exp>())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+
+auto expected_map_impl(Exp &&exp, F &&f) -> expected<void, err_t<Exp>> {
+ if (exp.has_value()) {
+ detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp));
+ return {};
+ }
+
+ return unexpected<err_t<Exp>>(std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+
+constexpr auto expected_map_impl(Exp &&exp, F &&f)
+ -> ret_t<Exp, detail::decay_t<Ret>> {
+ using result = ret_t<Exp, detail::decay_t<Ret>>;
+
+ return exp.has_value() ? result(detail::invoke(std::forward<F>(f)))
+ : result(unexpect, std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+
+auto expected_map_impl(Exp &&exp, F &&f) -> expected<void, err_t<Exp>> {
+ if (exp.has_value()) {
+ detail::invoke(std::forward<F>(f));
+ return {};
+ }
+
+ return unexpected<err_t<Exp>>(std::forward<Exp>(exp).error());
+}
+#endif
+
+#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \
+ !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto map_error_impl(Exp &&exp, F &&f) {
+ using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
+ return exp.has_value()
+ ? result(*std::forward<Exp>(exp))
+ : result(unexpect, detail::invoke(std::forward<F>(f),
+ std::forward<Exp>(exp).error()));
+}
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto map_error_impl(Exp &&exp, F &&f) {
+ using result = expected<exp_t<Exp>, monostate>;
+ if (exp.has_value()) {
+ return result(*std::forward<Exp>(exp));
+ }
+
+ detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+ return result(unexpect, monostate{});
+}
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto map_error_impl(Exp &&exp, F &&f) {
+ using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
+ return exp.has_value()
+ ? result()
+ : result(unexpect, detail::invoke(std::forward<F>(f),
+ std::forward<Exp>(exp).error()));
+}
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto map_error_impl(Exp &&exp, F &&f) {
+ using result = expected<exp_t<Exp>, monostate>;
+ if (exp.has_value()) {
+ return result();
+ }
+
+ detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+ return result(unexpect, monostate{});
+}
+#else
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto map_error_impl(Exp &&exp, F &&f)
+ -> expected<exp_t<Exp>, detail::decay_t<Ret>> {
+ using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
+
+ return exp.has_value()
+ ? result(*std::forward<Exp>(exp))
+ : result(unexpect, detail::invoke(std::forward<F>(f),
+ std::forward<Exp>(exp).error()));
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto map_error_impl(Exp &&exp, F &&f) -> expected<exp_t<Exp>, monostate> {
+ using result = expected<exp_t<Exp>, monostate>;
+ if (exp.has_value()) {
+ return result(*std::forward<Exp>(exp));
+ }
+
+ detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+ return result(unexpect, monostate{});
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto map_error_impl(Exp &&exp, F &&f)
+ -> expected<exp_t<Exp>, detail::decay_t<Ret>> {
+ using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
+
+ return exp.has_value()
+ ? result()
+ : result(unexpect, detail::invoke(std::forward<F>(f),
+ std::forward<Exp>(exp).error()));
+}
+
+template <class Exp, class F,
+ detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+auto map_error_impl(Exp &&exp, F &&f) -> expected<exp_t<Exp>, monostate> {
+ using result = expected<exp_t<Exp>, monostate>;
+ if (exp.has_value()) {
+ return result();
+ }
+
+ detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+ return result(unexpect, monostate{});
+}
+#endif
+
+#ifdef TL_EXPECTED_CXX14
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+constexpr auto or_else_impl(Exp &&exp, F &&f) {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+ return exp.has_value()
+ ? std::forward<Exp>(exp)
+ : detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+detail::decay_t<Exp> or_else_impl(Exp &&exp, F &&f) {
+ return exp.has_value()
+ ? std::forward<Exp>(exp)
+ : (detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error()),
+ std::forward<Exp>(exp));
+}
+#else
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
+auto or_else_impl(Exp &&exp, F &&f) -> Ret {
+ static_assert(detail::is_expected<Ret>::value, "F must return an expected");
+ return exp.has_value()
+ ? std::forward<Exp>(exp)
+ : detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
+}
+
+template <class Exp, class F,
+ class Ret = decltype(detail::invoke(std::declval<F>(),
+ std::declval<Exp>().error())),
+ detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
+detail::decay_t<Exp> or_else_impl(Exp &&exp, F &&f) {
+ return exp.has_value()
+ ? std::forward<Exp>(exp)
+ : (detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error()),
+ std::forward<Exp>(exp));
+}
+#endif
+} // namespace detail
+
+template <class T, class E, class U, class F>
+constexpr bool operator==(const expected<T, E> &lhs,
+ const expected<U, F> &rhs) {
+ return (lhs.has_value() != rhs.has_value())
+ ? false
+ : (!lhs.has_value() ? lhs.error() == rhs.error() : *lhs == *rhs);
+}
+template <class T, class E, class U, class F>
+constexpr bool operator!=(const expected<T, E> &lhs,
+ const expected<U, F> &rhs) {
+ return (lhs.has_value() != rhs.has_value())
+ ? true
+ : (!lhs.has_value() ? lhs.error() != rhs.error() : *lhs != *rhs);
+}
+
+template <class T, class E, class U>
+constexpr bool operator==(const expected<T, E> &x, const U &v) {
+ return x.has_value() ? *x == v : false;
+}
+template <class T, class E, class U>
+constexpr bool operator==(const U &v, const expected<T, E> &x) {
+ return x.has_value() ? *x == v : false;
+}
+template <class T, class E, class U>
+constexpr bool operator!=(const expected<T, E> &x, const U &v) {
+ return x.has_value() ? *x != v : true;
+}
+template <class T, class E, class U>
+constexpr bool operator!=(const U &v, const expected<T, E> &x) {
+ return x.has_value() ? *x != v : true;
+}
+
+template <class T, class E>
+constexpr bool operator==(const expected<T, E> &x, const unexpected<E> &e) {
+ return x.has_value() ? false : x.error() == e.value();
+}
+template <class T, class E>
+constexpr bool operator==(const unexpected<E> &e, const expected<T, E> &x) {
+ return x.has_value() ? false : x.error() == e.value();
+}
+template <class T, class E>
+constexpr bool operator!=(const expected<T, E> &x, const unexpected<E> &e) {
+ return x.has_value() ? true : x.error() != e.value();
+}
+template <class T, class E>
+constexpr bool operator!=(const unexpected<E> &e, const expected<T, E> &x) {
+ return x.has_value() ? true : x.error() != e.value();
+}
+
+template <class T, class E,
+ detail::enable_if_t<(std::is_void<T>::value ||
+ std::is_move_constructible<T>::value) &&
+ detail::is_swappable<T>::value &&
+ std::is_move_constructible<E>::value &&
+ detail::is_swappable<E>::value> * = nullptr>
+void swap(expected<T, E> &lhs,
+ expected<T, E> &rhs) noexcept(noexcept(lhs.swap(rhs))) {
+ lhs.swap(rhs);
+}
+} // namespace tl
+
+#endif
diff --git a/contrib/fastutf8/CMakeLists.txt b/contrib/fastutf8/CMakeLists.txt
new file mode 100644
index 0000000..2a98ed8
--- /dev/null
+++ b/contrib/fastutf8/CMakeLists.txt
@@ -0,0 +1,11 @@
+SET(UTFSRC ${CMAKE_CURRENT_SOURCE_DIR}/fastutf8.c)
+IF(HAVE_AVX2 AND "${ARCH}" STREQUAL "x86_64")
+ SET(UTFSRC ${UTFSRC} ${CMAKE_CURRENT_SOURCE_DIR}/avx2.c)
+ MESSAGE(STATUS "UTF8: AVX2 support is added")
+ENDIF()
+IF(HAVE_SSE41 AND "${ARCH}" STREQUAL "x86_64")
+ SET(UTFSRC ${UTFSRC} ${CMAKE_CURRENT_SOURCE_DIR}/sse41.c)
+ MESSAGE(STATUS "UTF8: SSE41 support is added")
+ENDIF()
+
+ADD_LIBRARY(rspamd-fastutf8 STATIC ${UTFSRC}) \ No newline at end of file
diff --git a/contrib/fastutf8/LICENSE b/contrib/fastutf8/LICENSE
new file mode 100644
index 0000000..9b5471b
--- /dev/null
+++ b/contrib/fastutf8/LICENSE
@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2019 Yibo Cai
+Copyright (c) 2019 Vsevolod Stakhov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE. \ No newline at end of file
diff --git a/contrib/fastutf8/avx2.c b/contrib/fastutf8/avx2.c
new file mode 100644
index 0000000..765c62f
--- /dev/null
+++ b/contrib/fastutf8/avx2.c
@@ -0,0 +1,314 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2019 Yibo Cai
+ * Copyright (c) 2019 Vsevolod Stakhov
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "config.h"
+#include "fastutf8.h"
+#include "platform_config.h"
+
+
+#ifndef __clang__
+#pragma GCC push_options
+#pragma GCC target("avx2")
+#endif
+
+#ifndef __SSE2__
+#define __SSE2__
+#endif
+#ifndef __SSE__
+#define __SSE__
+#endif
+#ifndef __SSE4_2__
+#define __SSE4_2__
+#endif
+#ifndef __SSE4_1__
+#define __SSE4_1__
+#endif
+#ifndef __SSEE3__
+#define __SSEE3__
+#endif
+#ifndef __AVX__
+#define __AVX__
+#endif
+#ifndef __AVX2__
+#define __AVX2__
+#endif
+
+#include <immintrin.h>
+
+/*
+ * Map high nibble of "First Byte" to legal character length minus 1
+ * 0x00 ~ 0xBF --> 0
+ * 0xC0 ~ 0xDF --> 1
+ * 0xE0 ~ 0xEF --> 2
+ * 0xF0 ~ 0xFF --> 3
+ */
+static const int8_t _first_len_tbl[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
+};
+
+/* Map "First Byte" to 8-th item of range table (0xC2 ~ 0xF4) */
+static const int8_t _first_range_tbl[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
+};
+
+/*
+ * Range table, map range index to min and max values
+ * Index 0 : 00 ~ 7F (First Byte, ascii)
+ * Index 1,2,3: 80 ~ BF (Second, Third, Fourth Byte)
+ * Index 4 : A0 ~ BF (Second Byte after E0)
+ * Index 5 : 80 ~ 9F (Second Byte after ED)
+ * Index 6 : 90 ~ BF (Second Byte after F0)
+ * Index 7 : 80 ~ 8F (Second Byte after F4)
+ * Index 8 : C2 ~ F4 (First Byte, non ascii)
+ * Index 9~15 : illegal: i >= 127 && i <= -128
+ */
+static const int8_t _range_min_tbl[] = {
+ 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
+ 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F,
+ 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
+ 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F,
+};
+static const int8_t _range_max_tbl[] = {
+ 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
+ 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
+ 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+};
+
+/*
+ * Tables for fast handling of four special First Bytes(E0,ED,F0,F4), after
+ * which the Second Byte are not 80~BF. It contains "range index adjustment".
+ * +------------+---------------+------------------+----------------+
+ * | First Byte | original range| range adjustment | adjusted range |
+ * +------------+---------------+------------------+----------------+
+ * | E0 | 2 | 2 | 4 |
+ * +------------+---------------+------------------+----------------+
+ * | ED | 2 | 3 | 5 |
+ * +------------+---------------+------------------+----------------+
+ * | F0 | 3 | 3 | 6 |
+ * +------------+---------------+------------------+----------------+
+ * | F4 | 4 | 4 | 8 |
+ * +------------+---------------+------------------+----------------+
+ */
+/* index1 -> E0, index14 -> ED */
+static const int8_t _df_ee_tbl[] = {
+ 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0,
+ 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0,
+};
+/* index1 -> F0, index5 -> F4 */
+static const int8_t _ef_fe_tbl[] = {
+ 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+
+static inline __m256i push_last_byte_of_a_to_b(__m256i a, __m256i b)
+ __attribute__((__target__("avx2")));
+static inline __m256i push_last_byte_of_a_to_b(__m256i a, __m256i b)
+{
+ return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 15);
+}
+
+static inline __m256i push_last_2bytes_of_a_to_b(__m256i a, __m256i b)
+ __attribute__((__target__("avx2")));
+static inline __m256i push_last_2bytes_of_a_to_b(__m256i a, __m256i b)
+{
+ return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 14);
+}
+
+static inline __m256i push_last_3bytes_of_a_to_b(__m256i a, __m256i b)
+ __attribute__((__target__("avx2")));
+static inline __m256i push_last_3bytes_of_a_to_b(__m256i a, __m256i b)
+{
+ return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 13);
+}
+
+off_t rspamd_fast_utf8_validate_avx2 (const unsigned char *data, size_t len)
+ __attribute__((__target__("avx2")));
+
+/* 5x faster than naive method */
+/* Return 0 - success, -1 - error, >0 - first error char(if RET_ERR_IDX = 1) */
+off_t rspamd_fast_utf8_validate_avx2 (const unsigned char *data, size_t len)
+{
+ off_t err_pos = 1;
+
+ if (len >= 32) {
+ __m256i prev_input = _mm256_set1_epi8 (0);
+ __m256i prev_first_len = _mm256_set1_epi8 (0);
+
+ /* Cached tables */
+ const __m256i first_len_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _first_len_tbl);
+ const __m256i first_range_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _first_range_tbl);
+ const __m256i range_min_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _range_min_tbl);
+ const __m256i range_max_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _range_max_tbl);
+ const __m256i df_ee_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _df_ee_tbl);
+ const __m256i ef_fe_tbl =
+ _mm256_lddqu_si256 ((const __m256i *) _ef_fe_tbl);
+
+ __m256i error = _mm256_set1_epi8 (0);
+
+ while (len >= 32) {
+ const __m256i input = _mm256_lddqu_si256 ((const __m256i *) data);
+
+ /* high_nibbles = input >> 4 */
+ const __m256i high_nibbles =
+ _mm256_and_si256 (_mm256_srli_epi16 (input, 4), _mm256_set1_epi8 (0x0F));
+
+ /* first_len = legal character length minus 1 */
+ /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
+ /* first_len = first_len_tbl[high_nibbles] */
+ __m256i first_len = _mm256_shuffle_epi8 (first_len_tbl, high_nibbles);
+
+ /* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */
+ /* range = first_range_tbl[high_nibbles] */
+ __m256i range = _mm256_shuffle_epi8 (first_range_tbl, high_nibbles);
+
+ /* Second Byte: set range index to first_len */
+ /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
+ /* range |= (first_len, prev_first_len) << 1 byte */
+ range = _mm256_or_si256 (
+ range, push_last_byte_of_a_to_b (prev_first_len, first_len));
+
+ /* Third Byte: set range index to saturate_sub(first_len, 1) */
+ /* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */
+ __m256i tmp1, tmp2;
+
+ /* tmp1 = saturate_sub(first_len, 1) */
+ tmp1 = _mm256_subs_epu8 (first_len, _mm256_set1_epi8 (1));
+ /* tmp2 = saturate_sub(prev_first_len, 1) */
+ tmp2 = _mm256_subs_epu8 (prev_first_len, _mm256_set1_epi8 (1));
+
+ /* range |= (tmp1, tmp2) << 2 bytes */
+ range = _mm256_or_si256 (range, push_last_2bytes_of_a_to_b (tmp2, tmp1));
+
+ /* Fourth Byte: set range index to saturate_sub(first_len, 2) */
+ /* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */
+ /* tmp1 = saturate_sub(first_len, 2) */
+ tmp1 = _mm256_subs_epu8 (first_len, _mm256_set1_epi8 (2));
+ /* tmp2 = saturate_sub(prev_first_len, 2) */
+ tmp2 = _mm256_subs_epu8 (prev_first_len, _mm256_set1_epi8 (2));
+ /* range |= (tmp1, tmp2) << 3 bytes */
+ range = _mm256_or_si256 (range, push_last_3bytes_of_a_to_b (tmp2, tmp1));
+
+ /*
+ * Now we have below range indices caluclated
+ * Correct cases:
+ * - 8 for C0~FF
+ * - 3 for 1st byte after F0~FF
+ * - 2 for 1st byte after E0~EF or 2nd byte after F0~FF
+ * - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or
+ * 3rd byte after F0~FF
+ * - 0 for others
+ * Error cases:
+ * 9,10,11 if non ascii First Byte overlaps
+ * E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error
+ */
+
+ /* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */
+ /* Overlaps lead to index 9~15, which are illegal in range table */
+ __m256i shift1, pos, range2;
+ /* shift1 = (input, prev_input) << 1 byte */
+ shift1 = push_last_byte_of_a_to_b (prev_input, input);
+ pos = _mm256_sub_epi8 (shift1, _mm256_set1_epi8 (0xEF));
+ /*
+ * shift1: | EF F0 ... FE | FF 00 ... ... DE | DF E0 ... EE |
+ * pos: | 0 1 15 | 16 17 239| 240 241 255|
+ * pos-240: | 0 0 0 | 0 0 0 | 0 1 15 |
+ * pos+112: | 112 113 127| >= 128 | >= 128 |
+ */
+ tmp1 = _mm256_subs_epu8 (pos, _mm256_set1_epi8 ((char)240));
+ range2 = _mm256_shuffle_epi8 (df_ee_tbl, tmp1);
+ tmp2 = _mm256_adds_epu8 (pos, _mm256_set1_epi8 (112));
+ range2 = _mm256_add_epi8 (range2, _mm256_shuffle_epi8 (ef_fe_tbl, tmp2));
+
+ range = _mm256_add_epi8 (range, range2);
+
+ /* Load min and max values per calculated range index */
+ __m256i minv = _mm256_shuffle_epi8 (range_min_tbl, range);
+ __m256i maxv = _mm256_shuffle_epi8 (range_max_tbl, range);
+
+ /* Check value range */
+ error = _mm256_cmpgt_epi8(minv, input);
+ error = _mm256_or_si256(error, _mm256_cmpgt_epi8(input, maxv));
+ /* 5% performance drop from this conditional branch */
+ if (!_mm256_testz_si256(error, error)) {
+ break;
+ }
+
+ prev_input = input;
+ prev_first_len = first_len;
+
+ data += 32;
+ len -= 32;
+ err_pos += 32;
+ }
+
+ /* Error in first 16 bytes */
+ if (err_pos == 1) {
+ goto do_naive;
+ }
+
+ /* Find previous token (not 80~BF) */
+ int32_t token4 = _mm256_extract_epi32 (prev_input, 7);
+ const int8_t *token = (const int8_t *) &token4;
+ int lookahead = 0;
+
+ if (token[3] > (int8_t) 0xBF) {
+ lookahead = 1;
+ }
+ else if (token[2] > (int8_t) 0xBF) {
+ lookahead = 2;
+ }
+ else if (token[1] > (int8_t) 0xBF) {
+ lookahead = 3;
+ }
+
+ data -= lookahead;
+ len += lookahead;
+ err_pos -= lookahead;
+ }
+
+ /* Check remaining bytes with naive method */
+do_naive:
+ if (len > 0) {
+ off_t err_pos2 = rspamd_fast_utf8_validate_ref (data, len);
+
+ if (err_pos2) {
+ return err_pos + err_pos2 - 1;
+ }
+ }
+
+ return 0;
+}
+
+#ifndef __clang__
+#pragma GCC pop_options
+#endif
+
diff --git a/contrib/fastutf8/fastutf8.c b/contrib/fastutf8/fastutf8.c
new file mode 100644
index 0000000..89becaf
--- /dev/null
+++ b/contrib/fastutf8/fastutf8.c
@@ -0,0 +1,160 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2019 Yibo Cai
+ * Copyright (c) 2019 Vsevolod Stakhov
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "fastutf8.h"
+#include "libcryptobox/platform_config.h"
+
+
+/*
+ * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
+ *
+ * Table 3-7. Well-Formed UTF-8 Byte Sequences
+ *
+ * +--------------------+------------+-------------+------------+-------------+
+ * | Code Points | First Byte | Second Byte | Third Byte | Fourth Byte |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+0000..U+007F | 00..7F | | | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+0080..U+07FF | C2..DF | 80..BF | | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+0800..U+0FFF | E0 | A0..BF | 80..BF | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+1000..U+CFFF | E1..EC | 80..BF | 80..BF | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+D000..U+D7FF | ED | 80..9F | 80..BF | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+E000..U+FFFF | EE..EF | 80..BF | 80..BF | |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF |
+ * +--------------------+------------+-------------+------------+-------------+
+ * | U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF |
+ * +--------------------+------------+-------------+------------+-------------+
+ */
+
+/* Return 0 - success, >0 - index (1 based) of first error char */
+off_t
+rspamd_fast_utf8_validate_ref (const unsigned char *data, size_t len)
+{
+ off_t err_pos = 1;
+
+ while (len) {
+ int bytes;
+ const unsigned char byte1 = data[0];
+
+ /* 00..7F */
+ if (byte1 <= 0x7F) {
+ bytes = 1;
+ /* C2..DF, 80..BF */
+ }
+ else if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF &&
+ (signed char) data[1] <= (signed char) 0xBF) {
+ bytes = 2;
+ }
+ else if (len >= 3) {
+ const unsigned char byte2 = data[1];
+
+ /* Is byte2, byte3 between 0x80 ~ 0xBF */
+ const int byte2_ok = (signed char) byte2 <= (signed char) 0xBF;
+ const int byte3_ok = (signed char) data[2] <= (signed char) 0xBF;
+
+ if (byte2_ok && byte3_ok &&
+ /* E0, A0..BF, 80..BF */
+ ((byte1 == 0xE0 && byte2 >= 0xA0) ||
+ /* E1..EC, 80..BF, 80..BF */
+ (byte1 >= 0xE1 && byte1 <= 0xEC) ||
+ /* ED, 80..9F, 80..BF */
+ (byte1 == 0xED && byte2 <= 0x9F) ||
+ /* EE..EF, 80..BF, 80..BF */
+ (byte1 >= 0xEE && byte1 <= 0xEF))) {
+ bytes = 3;
+ }
+ else if (len >= 4) {
+ /* Is byte4 between 0x80 ~ 0xBF */
+ const int byte4_ok = (signed char) data[3] <= (signed char) 0xBF;
+
+ if (byte2_ok && byte3_ok && byte4_ok &&
+ /* F0, 90..BF, 80..BF, 80..BF */
+ ((byte1 == 0xF0 && byte2 >= 0x90) ||
+ /* F1..F3, 80..BF, 80..BF, 80..BF */
+ (byte1 >= 0xF1 && byte1 <= 0xF3) ||
+ /* F4, 80..8F, 80..BF, 80..BF */
+ (byte1 == 0xF4 && byte2 <= 0x8F))) {
+ bytes = 4;
+ }
+ else {
+ return err_pos;
+ }
+ }
+ else {
+ return err_pos;
+ }
+ }
+ else {
+ return err_pos;
+ }
+
+ len -= bytes;
+ err_pos += bytes;
+ data += bytes;
+ }
+
+ return 0;
+}
+
+/* Prototypes */
+#if defined(HAVE_SSE41) && defined(__x86_64__)
+extern off_t rspamd_fast_utf8_validate_sse41 (const unsigned char *data, size_t len);
+#endif
+#if defined(HAVE_AVX2) && defined(__x86_64__)
+extern off_t rspamd_fast_utf8_validate_avx2 (const unsigned char *data, size_t len);
+#endif
+
+static off_t (*validate_func) (const unsigned char *data, size_t len) =
+ rspamd_fast_utf8_validate_ref;
+
+
+void
+rspamd_fast_utf8_library_init (unsigned flags)
+{
+#if defined(HAVE_SSE41) && defined(__x86_64__)
+ if (flags & RSPAMD_FAST_UTF8_FLAG_SSE41) {
+ validate_func = rspamd_fast_utf8_validate_sse41;
+ }
+#endif
+#if defined(HAVE_AVX2) && defined(__x86_64__)
+ if (flags & RSPAMD_FAST_UTF8_FLAG_AVX2) {
+ validate_func = rspamd_fast_utf8_validate_avx2;
+ }
+#endif
+}
+
+off_t
+rspamd_fast_utf8_validate (const unsigned char *data, size_t len)
+{
+ return len >= 64 ?
+ validate_func (data, len) :
+ rspamd_fast_utf8_validate_ref (data, len);
+} \ No newline at end of file
diff --git a/contrib/fastutf8/fastutf8.h b/contrib/fastutf8/fastutf8.h
new file mode 100644
index 0000000..a1e9cbf
--- /dev/null
+++ b/contrib/fastutf8/fastutf8.h
@@ -0,0 +1,65 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2019 Yibo Cai
+ * Copyright (c) 2019 Vsevolod Stakhov
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RSPAMD_FASTUTF8_H
+#define RSPAMD_FASTUTF8_H
+
+#include <sys/types.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+enum rspamd_fast_utf8_cpu_flags {
+ RSPAMD_FAST_UTF8_FLAG_SSE41 = 1u << 0u,
+ RSPAMD_FAST_UTF8_FLAG_AVX2 = 1u << 1u,
+};
+
+/**
+ * Called to init codecs
+ * @param flags
+ */
+void rspamd_fast_utf8_library_init(unsigned flags);
+
+/**
+ * Called to validate input using fast codec
+ * @param data
+ * @param len
+ * @return
+ */
+off_t rspamd_fast_utf8_validate(const unsigned char *data, size_t len);
+
+/**
+ * Use plain C implementation
+ * @param data
+ * @param len
+ * @return
+ */
+off_t rspamd_fast_utf8_validate_ref(const unsigned char *data, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/contrib/fastutf8/sse41.c b/contrib/fastutf8/sse41.c
new file mode 100644
index 0000000..df338cf
--- /dev/null
+++ b/contrib/fastutf8/sse41.c
@@ -0,0 +1,272 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2019 Yibo Cai
+ * Copyright (c) 2019 Vsevolod Stakhov
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "config.h"
+#include "fastutf8.h"
+#include "platform_config.h"
+
+#ifndef __clang__
+#pragma GCC push_options
+#pragma GCC target("sse4.1")
+#endif
+
+#ifndef __SSE2__
+#define __SSE2__
+#endif
+#ifndef __SSE__
+#define __SSE__
+#endif
+#ifndef __SSEE3__
+#define __SSEE3__
+#endif
+#ifndef __SSE4_1__
+#define __SSE4_1__
+#endif
+
+#include <smmintrin.h>
+
+/*
+ * Map high nibble of "First Byte" to legal character length minus 1
+ * 0x00 ~ 0xBF --> 0
+ * 0xC0 ~ 0xDF --> 1
+ * 0xE0 ~ 0xEF --> 2
+ * 0xF0 ~ 0xFF --> 3
+ */
+static const int8_t _first_len_tbl[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
+};
+
+/* Map "First Byte" to 8-th item of range table (0xC2 ~ 0xF4) */
+static const int8_t _first_range_tbl[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
+};
+
+/*
+ * Range table, map range index to min and max values
+ * Index 0 : 00 ~ 7F (First Byte, ascii)
+ * Index 1,2,3: 80 ~ BF (Second, Third, Fourth Byte)
+ * Index 4 : A0 ~ BF (Second Byte after E0)
+ * Index 5 : 80 ~ 9F (Second Byte after ED)
+ * Index 6 : 90 ~ BF (Second Byte after F0)
+ * Index 7 : 80 ~ 8F (Second Byte after F4)
+ * Index 8 : C2 ~ F4 (First Byte, non ascii)
+ * Index 9~15 : illegal: i >= 127 && i <= -128
+ */
+static const int8_t _range_min_tbl[] = {
+ 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
+ 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F,
+};
+static const int8_t _range_max_tbl[] = {
+ 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
+ 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+};
+
+/*
+ * Tables for fast handling of four special First Bytes(E0,ED,F0,F4), after
+ * which the Second Byte are not 80~BF. It contains "range index adjustment".
+ * +------------+---------------+------------------+----------------+
+ * | First Byte | original range| range adjustment | adjusted range |
+ * +------------+---------------+------------------+----------------+
+ * | E0 | 2 | 2 | 4 |
+ * +------------+---------------+------------------+----------------+
+ * | ED | 2 | 3 | 5 |
+ * +------------+---------------+------------------+----------------+
+ * | F0 | 3 | 3 | 6 |
+ * +------------+---------------+------------------+----------------+
+ * | F4 | 4 | 4 | 8 |
+ * +------------+---------------+------------------+----------------+
+ */
+/* index1 -> E0, index14 -> ED */
+static const int8_t _df_ee_tbl[] = {
+ 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0,
+};
+/* index1 -> F0, index5 -> F4 */
+static const int8_t _ef_fe_tbl[] = {
+ 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+
+off_t
+rspamd_fast_utf8_validate_sse41 (const unsigned char *data, size_t len)
+ __attribute__((__target__("sse4.1")));
+
+/* Return 0 - success, >0 - first error char(if RET_ERR_IDX = 1) */
+off_t
+rspamd_fast_utf8_validate_sse41 (const unsigned char *data, size_t len)
+{
+ off_t err_pos = 1;
+
+ if (len >= 16) {
+ __m128i prev_input = _mm_set1_epi8 (0);
+ __m128i prev_first_len = _mm_set1_epi8 (0);
+
+ /* Cached tables */
+ const __m128i first_len_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _first_len_tbl);
+ const __m128i first_range_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _first_range_tbl);
+ const __m128i range_min_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _range_min_tbl);
+ const __m128i range_max_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _range_max_tbl);
+ const __m128i df_ee_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _df_ee_tbl);
+ const __m128i ef_fe_tbl =
+ _mm_lddqu_si128 ((const __m128i *) _ef_fe_tbl);
+
+ __m128i error = _mm_set1_epi8 (0);
+
+ while (len >= 16) {
+ const __m128i input = _mm_lddqu_si128 ((const __m128i *) data);
+
+ /* high_nibbles = input >> 4 */
+ const __m128i high_nibbles =
+ _mm_and_si128 (_mm_srli_epi16 (input, 4), _mm_set1_epi8 (0x0F));
+
+ /* first_len = legal character length minus 1 */
+ /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
+ /* first_len = first_len_tbl[high_nibbles] */
+ __m128i first_len = _mm_shuffle_epi8 (first_len_tbl, high_nibbles);
+
+ /* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */
+ /* range = first_range_tbl[high_nibbles] */
+ __m128i range = _mm_shuffle_epi8 (first_range_tbl, high_nibbles);
+
+ /* Second Byte: set range index to first_len */
+ /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
+ /* range |= (first_len, prev_first_len) << 1 byte */
+ range = _mm_or_si128 (
+ range, _mm_alignr_epi8(first_len, prev_first_len, 15));
+
+ /* Third Byte: set range index to saturate_sub(first_len, 1) */
+ /* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */
+ __m128i tmp1, tmp2;
+ /* tmp1 = saturate_sub(first_len, 1) */
+ tmp1 = _mm_subs_epu8 (first_len, _mm_set1_epi8 (1));
+ /* tmp2 = saturate_sub(prev_first_len, 1) */
+ tmp2 = _mm_subs_epu8 (prev_first_len, _mm_set1_epi8 (1));
+ /* range |= (tmp1, tmp2) << 2 bytes */
+ range = _mm_or_si128 (range, _mm_alignr_epi8(tmp1, tmp2, 14));
+
+ /* Fourth Byte: set range index to saturate_sub(first_len, 2) */
+ /* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */
+ /* tmp1 = saturate_sub(first_len, 2) */
+ tmp1 = _mm_subs_epu8 (first_len, _mm_set1_epi8 (2));
+ /* tmp2 = saturate_sub(prev_first_len, 2) */
+ tmp2 = _mm_subs_epu8 (prev_first_len, _mm_set1_epi8 (2));
+ /* range |= (tmp1, tmp2) << 3 bytes */
+ range = _mm_or_si128 (range, _mm_alignr_epi8(tmp1, tmp2, 13));
+
+ /*
+ * Now we have below range indices caluclated
+ * Correct cases:
+ * - 8 for C0~FF
+ * - 3 for 1st byte after F0~FF
+ * - 2 for 1st byte after E0~EF or 2nd byte after F0~FF
+ * - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or
+ * 3rd byte after F0~FF
+ * - 0 for others
+ * Error cases:
+ * 9,10,11 if non ascii First Byte overlaps
+ * E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error
+ */
+
+ /* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */
+ /* Overlaps lead to index 9~15, which are illegal in range table */
+ __m128i shift1, pos, range2;
+ /* shift1 = (input, prev_input) << 1 byte */
+ shift1 = _mm_alignr_epi8(input, prev_input, 15);
+ pos = _mm_sub_epi8 (shift1, _mm_set1_epi8 (0xEF));
+ /*
+ * shift1: | EF F0 ... FE | FF 00 ... ... DE | DF E0 ... EE |
+ * pos: | 0 1 15 | 16 17 239| 240 241 255|
+ * pos-240: | 0 0 0 | 0 0 0 | 0 1 15 |
+ * pos+112: | 112 113 127| >= 128 | >= 128 |
+ */
+ tmp1 = _mm_subs_epu8 (pos, _mm_set1_epi8 ((char)240));
+ range2 = _mm_shuffle_epi8 (df_ee_tbl, tmp1);
+ tmp2 = _mm_adds_epu8 (pos, _mm_set1_epi8 (112));
+ range2 = _mm_add_epi8 (range2, _mm_shuffle_epi8 (ef_fe_tbl, tmp2));
+
+ range = _mm_add_epi8 (range, range2);
+
+ /* Load min and max values per calculated range index */
+ __m128i minv = _mm_shuffle_epi8 (range_min_tbl, range);
+ __m128i maxv = _mm_shuffle_epi8 (range_max_tbl, range);
+
+ /* Check value range */
+ error = _mm_cmplt_epi8(input, minv);
+ error = _mm_or_si128(error, _mm_cmpgt_epi8(input, maxv));
+ /* 5% performance drop from this conditional branch */
+ if (!_mm_testz_si128(error, error)) {
+ break;
+ }
+
+ prev_input = input;
+ prev_first_len = first_len;
+
+ data += 16;
+ len -= 16;
+ err_pos += 16;
+ }
+
+ /* Error in first 16 bytes */
+ if (err_pos == 1) {
+ goto do_naive;
+ }
+
+ /* Find previous token (not 80~BF) */
+ int32_t token4 = _mm_extract_epi32 (prev_input, 3);
+ const int8_t *token = (const int8_t *) &token4;
+ int lookahead = 0;
+
+ if (token[3] > (int8_t) 0xBF) {
+ lookahead = 1;
+ }
+ else if (token[2] > (int8_t) 0xBF) {
+ lookahead = 2;
+ }
+ else if (token[1] > (int8_t) 0xBF) {
+ lookahead = 3;
+ }
+
+ data -= lookahead;
+ len += lookahead;
+ err_pos -= lookahead;
+ }
+
+ do_naive:
+ if (len > 0) {
+ off_t err_pos2 = rspamd_fast_utf8_validate_ref (data, len);
+
+ if (err_pos2) {
+ return err_pos + err_pos2 - 1;
+ }
+ }
+
+ return 0;
+}
+
+#ifndef __clang__
+#pragma GCC pop_options
+#endif \ No newline at end of file
diff --git a/contrib/fmt/LICENSE.rst b/contrib/fmt/LICENSE.rst
new file mode 100644
index 0000000..f0ec3db
--- /dev/null
+++ b/contrib/fmt/LICENSE.rst
@@ -0,0 +1,27 @@
+Copyright (c) 2012 - present, Victor Zverovich
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+--- Optional exception to the license ---
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into a machine-executable object form of such
+source code, you may redistribute such embedded portions in such object form
+without including the above copyright and permission notices.
diff --git a/contrib/fmt/README.rst b/contrib/fmt/README.rst
new file mode 100644
index 0000000..acddc70
--- /dev/null
+++ b/contrib/fmt/README.rst
@@ -0,0 +1,506 @@
+{fmt}
+=====
+
+.. image:: https://travis-ci.org/fmtlib/fmt.png?branch=master
+ :target: https://travis-ci.org/fmtlib/fmt
+
+.. image:: https://ci.appveyor.com/api/projects/status/ehjkiefde6gucy1v
+ :target: https://ci.appveyor.com/project/vitaut/fmt
+
+.. image:: https://oss-fuzz-build-logs.storage.googleapis.com/badges/fmt.svg
+ :alt: fmt is continuously fuzzed at oss-fuzz
+ :target: https://bugs.chromium.org/p/oss-fuzz/issues/list?\
+ colspec=ID%20Type%20Component%20Status%20Proj%20Reported%20Owner%20\
+ Summary&q=proj%3Dfmt&can=1
+
+.. image:: https://img.shields.io/badge/stackoverflow-fmt-blue.svg
+ :alt: Ask questions at StackOverflow with the tag fmt
+ :target: https://stackoverflow.com/questions/tagged/fmt
+
+**{fmt}** is an open-source formatting library providing a fast and safe
+alternative to C stdio and C++ iostreams.
+
+If you like this project, please consider donating to BYSOL,
+an initiative to help victims of political repressions in Belarus:
+https://www.facebook.com/donate/759400044849707/108388587646909/.
+
+`Documentation <https://fmt.dev>`__
+
+Q&A: ask questions on `StackOverflow with the tag fmt
+<https://stackoverflow.com/questions/tagged/fmt>`_.
+
+Try {fmt} in `Compiler Explorer <https://godbolt.org/z/Eq5763>`_.
+
+Features
+--------
+
+* Simple `format API <https://fmt.dev/latest/api.html>`_ with positional arguments
+ for localization
+* Implementation of `C++20 std::format
+ <https://en.cppreference.com/w/cpp/utility/format>`__
+* `Format string syntax <https://fmt.dev/latest/syntax.html>`_ similar to Python's
+ `format <https://docs.python.org/3/library/stdtypes.html#str.format>`_
+* Fast IEEE 754 floating-point formatter with correct rounding, shortness and
+ round-trip guarantees
+* Safe `printf implementation
+ <https://fmt.dev/latest/api.html#printf-formatting>`_ including the POSIX
+ extension for positional arguments
+* Extensibility: `support for user-defined types
+ <https://fmt.dev/latest/api.html#formatting-user-defined-types>`_
+* High performance: faster than common standard library implementations of
+ ``(s)printf``, iostreams, ``to_string`` and ``to_chars``, see `Speed tests`_
+ and `Converting a hundred million integers to strings per second
+ <http://www.zverovich.net/2020/06/13/fast-int-to-string-revisited.html>`_
+* Small code size both in terms of source code with the minimum configuration
+ consisting of just three files, ``core.h``, ``format.h`` and ``format-inl.h``,
+ and compiled code; see `Compile time and code bloat`_
+* Reliability: the library has an extensive set of `tests
+ <https://github.com/fmtlib/fmt/tree/master/test>`_ and is `continuously fuzzed
+ <https://bugs.chromium.org/p/oss-fuzz/issues/list?colspec=ID%20Type%20
+ Component%20Status%20Proj%20Reported%20Owner%20Summary&q=proj%3Dfmt&can=1>`_
+* Safety: the library is fully type safe, errors in format strings can be
+ reported at compile time, automatic memory management prevents buffer overflow
+ errors
+* Ease of use: small self-contained code base, no external dependencies,
+ permissive MIT `license
+ <https://github.com/fmtlib/fmt/blob/master/LICENSE.rst>`_
+* `Portability <https://fmt.dev/latest/index.html#portability>`_ with
+ consistent output across platforms and support for older compilers
+* Clean warning-free codebase even on high warning levels such as
+ ``-Wall -Wextra -pedantic``
+* Locale-independence by default
+* Optional header-only configuration enabled with the ``FMT_HEADER_ONLY`` macro
+
+See the `documentation <https://fmt.dev>`_ for more details.
+
+Examples
+--------
+
+**Print to stdout** (`run <https://godbolt.org/z/Tevcjh>`_)
+
+.. code:: c++
+
+ #include <fmt/core.h>
+
+ int main() {
+ fmt::print("Hello, world!\n");
+ }
+
+**Format a string** (`run <https://godbolt.org/z/oK8h33>`_)
+
+.. code:: c++
+
+ std::string s = fmt::format("The answer is {}.", 42);
+ // s == "The answer is 42."
+
+**Format a string using positional arguments** (`run <https://godbolt.org/z/Yn7Txe>`_)
+
+.. code:: c++
+
+ std::string s = fmt::format("I'd rather be {1} than {0}.", "right", "happy");
+ // s == "I'd rather be happy than right."
+
+**Print chrono durations** (`run <https://godbolt.org/z/K8s4Mc>`_)
+
+.. code:: c++
+
+ #include <fmt/chrono.h>
+
+ int main() {
+ using namespace std::literals::chrono_literals;
+ fmt::print("Default format: {} {}\n", 42s, 100ms);
+ fmt::print("strftime-like format: {:%H:%M:%S}\n", 3h + 15min + 30s);
+ }
+
+Output::
+
+ Default format: 42s 100ms
+ strftime-like format: 03:15:30
+
+**Print a container** (`run <https://godbolt.org/z/MjsY7c>`_)
+
+.. code:: c++
+
+ #include <vector>
+ #include <fmt/ranges.h>
+
+ int main() {
+ std::vector<int> v = {1, 2, 3};
+ fmt::print("{}\n", v);
+ }
+
+Output::
+
+ {1, 2, 3}
+
+**Check a format string at compile time**
+
+.. code:: c++
+
+ std::string s = fmt::format(FMT_STRING("{:d}"), "don't panic");
+
+This gives a compile-time error because ``d`` is an invalid format specifier for
+a string.
+
+**Write a file from a single thread**
+
+.. code:: c++
+
+ #include <fmt/os.h>
+
+ int main() {
+ auto out = fmt::output_file("guide.txt");
+ out.print("Don't {}", "Panic");
+ }
+
+This can be `5 to 9 times faster than fprintf
+<http://www.zverovich.net/2020/08/04/optimal-file-buffer-size.html>`_.
+
+**Print with colors and text styles**
+
+.. code:: c++
+
+ #include <fmt/color.h>
+
+ int main() {
+ fmt::print(fg(fmt::color::crimson) | fmt::emphasis::bold,
+ "Hello, {}!\n", "world");
+ fmt::print(fg(fmt::color::floral_white) | bg(fmt::color::slate_gray) |
+ fmt::emphasis::underline, "Hello, {}!\n", "мир");
+ fmt::print(fg(fmt::color::steel_blue) | fmt::emphasis::italic,
+ "Hello, {}!\n", "世界");
+ }
+
+Output on a modern terminal:
+
+.. image:: https://user-images.githubusercontent.com/
+ 576385/88485597-d312f600-cf2b-11ea-9cbe-61f535a86e28.png
+
+Benchmarks
+----------
+
+Speed tests
+~~~~~~~~~~~
+
+================= ============= ===========
+Library Method Run Time, s
+================= ============= ===========
+libc printf 1.04
+libc++ std::ostream 3.05
+{fmt} 6.1.1 fmt::print 0.75
+Boost Format 1.67 boost::format 7.24
+Folly Format folly::format 2.23
+================= ============= ===========
+
+{fmt} is the fastest of the benchmarked methods, ~35% faster than ``printf``.
+
+The above results were generated by building ``tinyformat_test.cpp`` on macOS
+10.14.6 with ``clang++ -O3 -DNDEBUG -DSPEED_TEST -DHAVE_FORMAT``, and taking the
+best of three runs. In the test, the format string ``"%0.10f:%04d:%+g:%s:%p:%c:%%\n"``
+or equivalent is filled 2,000,000 times with output sent to ``/dev/null``; for
+further details refer to the `source
+<https://github.com/fmtlib/format-benchmark/blob/master/tinyformat_test.cpp>`_.
+
+{fmt} is up to 20-30x faster than ``std::ostringstream`` and ``sprintf`` on
+floating-point formatting (`dtoa-benchmark <https://github.com/fmtlib/dtoa-benchmark>`_)
+and faster than `double-conversion <https://github.com/google/double-conversion>`_ and
+`ryu <https://github.com/ulfjack/ryu>`_:
+
+.. image:: https://user-images.githubusercontent.com/576385/
+ 95684665-11719600-0ba8-11eb-8e5b-972ff4e49428.png
+ :target: https://fmt.dev/unknown_mac64_clang12.0.html
+
+Compile time and code bloat
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The script `bloat-test.py
+<https://github.com/fmtlib/format-benchmark/blob/master/bloat-test.py>`_
+from `format-benchmark <https://github.com/fmtlib/format-benchmark>`_
+tests compile time and code bloat for nontrivial projects.
+It generates 100 translation units and uses ``printf()`` or its alternative
+five times in each to simulate a medium sized project. The resulting
+executable size and compile time (Apple LLVM version 8.1.0 (clang-802.0.42),
+macOS Sierra, best of three) is shown in the following tables.
+
+**Optimized build (-O3)**
+
+============= =============== ==================== ==================
+Method Compile Time, s Executable size, KiB Stripped size, KiB
+============= =============== ==================== ==================
+printf 2.6 29 26
+printf+string 16.4 29 26
+iostreams 31.1 59 55
+{fmt} 19.0 37 34
+Boost Format 91.9 226 203
+Folly Format 115.7 101 88
+============= =============== ==================== ==================
+
+As you can see, {fmt} has 60% less overhead in terms of resulting binary code
+size compared to iostreams and comes pretty close to ``printf``. Boost Format
+and Folly Format have the largest overheads.
+
+``printf+string`` is the same as ``printf`` but with extra ``<string>``
+include to measure the overhead of the latter.
+
+**Non-optimized build**
+
+============= =============== ==================== ==================
+Method Compile Time, s Executable size, KiB Stripped size, KiB
+============= =============== ==================== ==================
+printf 2.2 33 30
+printf+string 16.0 33 30
+iostreams 28.3 56 52
+{fmt} 18.2 59 50
+Boost Format 54.1 365 303
+Folly Format 79.9 445 430
+============= =============== ==================== ==================
+
+``libc``, ``lib(std)c++`` and ``libfmt`` are all linked as shared libraries to
+compare formatting function overhead only. Boost Format is a
+header-only library so it doesn't provide any linkage options.
+
+Running the tests
+~~~~~~~~~~~~~~~~~
+
+Please refer to `Building the library`__ for the instructions on how to build
+the library and run the unit tests.
+
+__ https://fmt.dev/latest/usage.html#building-the-library
+
+Benchmarks reside in a separate repository,
+`format-benchmarks <https://github.com/fmtlib/format-benchmark>`_,
+so to run the benchmarks you first need to clone this repository and
+generate Makefiles with CMake::
+
+ $ git clone --recursive https://github.com/fmtlib/format-benchmark.git
+ $ cd format-benchmark
+ $ cmake .
+
+Then you can run the speed test::
+
+ $ make speed-test
+
+or the bloat test::
+
+ $ make bloat-test
+
+Projects using this library
+---------------------------
+
+* `0 A.D. <https://play0ad.com/>`_: a free, open-source, cross-platform
+ real-time strategy game
+
+* `AMPL/MP <https://github.com/ampl/mp>`_:
+ an open-source library for mathematical programming
+
+* `Aseprite <https://github.com/aseprite/aseprite>`_:
+ animated sprite editor & pixel art tool
+
+* `AvioBook <https://www.aviobook.aero/en>`_: a comprehensive aircraft
+ operations suite
+
+* `Blizzard Battle.net <https://battle.net/>`_: an online gaming platform
+
+* `Celestia <https://celestia.space/>`_: real-time 3D visualization of space
+
+* `Ceph <https://ceph.com/>`_: a scalable distributed storage system
+
+* `ccache <https://ccache.dev/>`_: a compiler cache
+
+* `ClickHouse <https://github.com/ClickHouse/ClickHouse>`_: analytical database
+ management system
+
+* `CUAUV <http://cuauv.org/>`_: Cornell University's autonomous underwater
+ vehicle
+
+* `Drake <https://drake.mit.edu/>`_: a planning, control, and analysis toolbox
+ for nonlinear dynamical systems (MIT)
+
+* `Envoy <https://lyft.github.io/envoy/>`_: C++ L7 proxy and communication bus
+ (Lyft)
+
+* `FiveM <https://fivem.net/>`_: a modification framework for GTA V
+
+* `Folly <https://github.com/facebook/folly>`_: Facebook open-source library
+
+* `HarpyWar/pvpgn <https://github.com/pvpgn/pvpgn-server>`_:
+ Player vs Player Gaming Network with tweaks
+
+* `KBEngine <https://github.com/kbengine/kbengine>`_: an open-source MMOG server
+ engine
+
+* `Keypirinha <https://keypirinha.com/>`_: a semantic launcher for Windows
+
+* `Kodi <https://kodi.tv/>`_ (formerly xbmc): home theater software
+
+* `Knuth <https://kth.cash/>`_: high-performance Bitcoin full-node
+
+* `Microsoft Verona <https://github.com/microsoft/verona>`_:
+ research programming language for concurrent ownership
+
+* `MongoDB <https://mongodb.com/>`_: distributed document database
+
+* `MongoDB Smasher <https://github.com/duckie/mongo_smasher>`_: a small tool to
+ generate randomized datasets
+
+* `OpenSpace <https://openspaceproject.com/>`_: an open-source
+ astrovisualization framework
+
+* `PenUltima Online (POL) <https://www.polserver.com/>`_:
+ an MMO server, compatible with most Ultima Online clients
+
+* `PyTorch <https://github.com/pytorch/pytorch>`_: an open-source machine
+ learning library
+
+* `quasardb <https://www.quasardb.net/>`_: a distributed, high-performance,
+ associative database
+
+* `Quill <https://github.com/odygrd/quill>`_: asynchronous low-latency logging library
+
+* `QKW <https://github.com/ravijanjam/qkw>`_: generalizing aliasing to simplify
+ navigation, and executing complex multi-line terminal command sequences
+
+* `redis-cerberus <https://github.com/HunanTV/redis-cerberus>`_: a Redis cluster
+ proxy
+
+* `redpanda <https://vectorized.io/redpanda>`_: a 10x faster Kafka® replacement
+ for mission critical systems written in C++
+
+* `rpclib <http://rpclib.net/>`_: a modern C++ msgpack-RPC server and client
+ library
+
+* `Salesforce Analytics Cloud
+ <https://www.salesforce.com/analytics-cloud/overview/>`_:
+ business intelligence software
+
+* `Scylla <https://www.scylladb.com/>`_: a Cassandra-compatible NoSQL data store
+ that can handle 1 million transactions per second on a single server
+
+* `Seastar <http://www.seastar-project.org/>`_: an advanced, open-source C++
+ framework for high-performance server applications on modern hardware
+
+* `spdlog <https://github.com/gabime/spdlog>`_: super fast C++ logging library
+
+* `Stellar <https://www.stellar.org/>`_: financial platform
+
+* `Touch Surgery <https://www.touchsurgery.com/>`_: surgery simulator
+
+* `TrinityCore <https://github.com/TrinityCore/TrinityCore>`_: open-source
+ MMORPG framework
+
+* `Windows Terminal <https://github.com/microsoft/terminal>`_: the new Windows
+ terminal
+
+`More... <https://github.com/search?q=fmtlib&type=Code>`_
+
+If you are aware of other projects using this library, please let me know
+by `email <mailto:victor.zverovich@gmail.com>`_ or by submitting an
+`issue <https://github.com/fmtlib/fmt/issues>`_.
+
+Motivation
+----------
+
+So why yet another formatting library?
+
+There are plenty of methods for doing this task, from standard ones like
+the printf family of function and iostreams to Boost Format and FastFormat
+libraries. The reason for creating a new library is that every existing
+solution that I found either had serious issues or didn't provide
+all the features I needed.
+
+printf
+~~~~~~
+
+The good thing about ``printf`` is that it is pretty fast and readily available
+being a part of the C standard library. The main drawback is that it
+doesn't support user-defined types. ``printf`` also has safety issues although
+they are somewhat mitigated with `__attribute__ ((format (printf, ...))
+<https://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html>`_ in GCC.
+There is a POSIX extension that adds positional arguments required for
+`i18n <https://en.wikipedia.org/wiki/Internationalization_and_localization>`_
+to ``printf`` but it is not a part of C99 and may not be available on some
+platforms.
+
+iostreams
+~~~~~~~~~
+
+The main issue with iostreams is best illustrated with an example:
+
+.. code:: c++
+
+ std::cout << std::setprecision(2) << std::fixed << 1.23456 << "\n";
+
+which is a lot of typing compared to printf:
+
+.. code:: c++
+
+ printf("%.2f\n", 1.23456);
+
+Matthew Wilson, the author of FastFormat, called this "chevron hell". iostreams
+don't support positional arguments by design.
+
+The good part is that iostreams support user-defined types and are safe although
+error handling is awkward.
+
+Boost Format
+~~~~~~~~~~~~
+
+This is a very powerful library which supports both ``printf``-like format
+strings and positional arguments. Its main drawback is performance. According to
+various, benchmarks it is much slower than other methods considered here. Boost
+Format also has excessive build times and severe code bloat issues (see
+`Benchmarks`_).
+
+FastFormat
+~~~~~~~~~~
+
+This is an interesting library which is fast, safe and has positional arguments.
+However, it has significant limitations, citing its author:
+
+ Three features that have no hope of being accommodated within the
+ current design are:
+
+ * Leading zeros (or any other non-space padding)
+ * Octal/hexadecimal encoding
+ * Runtime width/alignment specification
+
+It is also quite big and has a heavy dependency, STLSoft, which might be too
+restrictive for using it in some projects.
+
+Boost Spirit.Karma
+~~~~~~~~~~~~~~~~~~
+
+This is not really a formatting library but I decided to include it here for
+completeness. As iostreams, it suffers from the problem of mixing verbatim text
+with arguments. The library is pretty fast, but slower on integer formatting
+than ``fmt::format_to`` with format string compilation on Karma's own benchmark,
+see `Converting a hundred million integers to strings per second
+<http://www.zverovich.net/2020/06/13/fast-int-to-string-revisited.html>`_.
+
+License
+-------
+
+{fmt} is distributed under the MIT `license
+<https://github.com/fmtlib/fmt/blob/master/LICENSE.rst>`_.
+
+Documentation License
+---------------------
+
+The `Format String Syntax <https://fmt.dev/latest/syntax.html>`_
+section in the documentation is based on the one from Python `string module
+documentation <https://docs.python.org/3/library/string.html#module-string>`_.
+For this reason the documentation is distributed under the Python Software
+Foundation license available in `doc/python-license.txt
+<https://raw.github.com/fmtlib/fmt/master/doc/python-license.txt>`_.
+It only applies if you distribute the documentation of {fmt}.
+
+Maintainers
+-----------
+
+The {fmt} library is maintained by Victor Zverovich (`vitaut
+<https://github.com/vitaut>`_) and Jonathan Müller (`foonathan
+<https://github.com/foonathan>`_) with contributions from many other people.
+See `Contributors <https://github.com/fmtlib/fmt/graphs/contributors>`_ and
+`Releases <https://github.com/fmtlib/fmt/releases>`_ for some of the names.
+Let us know if your contribution is not listed or mentioned incorrectly and
+we'll make it right.
diff --git a/contrib/fmt/include/fmt/args.h b/contrib/fmt/include/fmt/args.h
new file mode 100644
index 0000000..a3966d1
--- /dev/null
+++ b/contrib/fmt/include/fmt/args.h
@@ -0,0 +1,234 @@
+// Formatting library for C++ - dynamic format arguments
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_ARGS_H_
+#define FMT_ARGS_H_
+
+#include <functional> // std::reference_wrapper
+#include <memory> // std::unique_ptr
+#include <vector>
+
+#include "core.h"
+
+FMT_BEGIN_NAMESPACE
+
+namespace detail {
+
+template <typename T> struct is_reference_wrapper : std::false_type {};
+template <typename T>
+struct is_reference_wrapper<std::reference_wrapper<T>> : std::true_type {};
+
+template <typename T> const T& unwrap(const T& v) { return v; }
+template <typename T> const T& unwrap(const std::reference_wrapper<T>& v) {
+ return static_cast<const T&>(v);
+}
+
+class dynamic_arg_list {
+ // Workaround for clang's -Wweak-vtables. Unlike for regular classes, for
+ // templates it doesn't complain about inability to deduce single translation
+ // unit for placing vtable. So storage_node_base is made a fake template.
+ template <typename = void> struct node {
+ virtual ~node() = default;
+ std::unique_ptr<node<>> next;
+ };
+
+ template <typename T> struct typed_node : node<> {
+ T value;
+
+ template <typename Arg>
+ FMT_CONSTEXPR typed_node(const Arg& arg) : value(arg) {}
+
+ template <typename Char>
+ FMT_CONSTEXPR typed_node(const basic_string_view<Char>& arg)
+ : value(arg.data(), arg.size()) {}
+ };
+
+ std::unique_ptr<node<>> head_;
+
+ public:
+ template <typename T, typename Arg> const T& push(const Arg& arg) {
+ auto new_node = std::unique_ptr<typed_node<T>>(new typed_node<T>(arg));
+ auto& value = new_node->value;
+ new_node->next = std::move(head_);
+ head_ = std::move(new_node);
+ return value;
+ }
+};
+} // namespace detail
+
+/**
+ \rst
+ A dynamic version of `fmt::format_arg_store`.
+ It's equipped with a storage to potentially temporary objects which lifetimes
+ could be shorter than the format arguments object.
+
+ It can be implicitly converted into `~fmt::basic_format_args` for passing
+ into type-erased formatting functions such as `~fmt::vformat`.
+ \endrst
+ */
+template <typename Context>
+class dynamic_format_arg_store
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+ // Workaround a GCC template argument substitution bug.
+ : public basic_format_args<Context>
+#endif
+{
+ private:
+ using char_type = typename Context::char_type;
+
+ template <typename T> struct need_copy {
+ static constexpr detail::type mapped_type =
+ detail::mapped_type_constant<T, Context>::value;
+
+ enum {
+ value = !(detail::is_reference_wrapper<T>::value ||
+ std::is_same<T, basic_string_view<char_type>>::value ||
+ std::is_same<T, detail::std_string_view<char_type>>::value ||
+ (mapped_type != detail::type::cstring_type &&
+ mapped_type != detail::type::string_type &&
+ mapped_type != detail::type::custom_type))
+ };
+ };
+
+ template <typename T>
+ using stored_type = conditional_t<
+ std::is_convertible<T, std::basic_string<char_type>>::value &&
+ !detail::is_reference_wrapper<T>::value,
+ std::basic_string<char_type>, T>;
+
+ // Storage of basic_format_arg must be contiguous.
+ std::vector<basic_format_arg<Context>> data_;
+ std::vector<detail::named_arg_info<char_type>> named_info_;
+
+ // Storage of arguments not fitting into basic_format_arg must grow
+ // without relocation because items in data_ refer to it.
+ detail::dynamic_arg_list dynamic_args_;
+
+ friend class basic_format_args<Context>;
+
+ unsigned long long get_types() const {
+ return detail::is_unpacked_bit | data_.size() |
+ (named_info_.empty()
+ ? 0ULL
+ : static_cast<unsigned long long>(detail::has_named_args_bit));
+ }
+
+ const basic_format_arg<Context>* data() const {
+ return named_info_.empty() ? data_.data() : data_.data() + 1;
+ }
+
+ template <typename T> void emplace_arg(const T& arg) {
+ data_.emplace_back(detail::make_arg<Context>(arg));
+ }
+
+ template <typename T>
+ void emplace_arg(const detail::named_arg<char_type, T>& arg) {
+ if (named_info_.empty()) {
+ constexpr const detail::named_arg_info<char_type>* zero_ptr{nullptr};
+ data_.insert(data_.begin(), {zero_ptr, 0});
+ }
+ data_.emplace_back(detail::make_arg<Context>(detail::unwrap(arg.value)));
+ auto pop_one = [](std::vector<basic_format_arg<Context>>* data) {
+ data->pop_back();
+ };
+ std::unique_ptr<std::vector<basic_format_arg<Context>>, decltype(pop_one)>
+ guard{&data_, pop_one};
+ named_info_.push_back({arg.name, static_cast<int>(data_.size() - 2u)});
+ data_[0].value_.named_args = {named_info_.data(), named_info_.size()};
+ guard.release();
+ }
+
+ public:
+ constexpr dynamic_format_arg_store() = default;
+
+ /**
+ \rst
+ Adds an argument into the dynamic store for later passing to a formatting
+ function.
+
+ Note that custom types and string types (but not string views) are copied
+ into the store dynamically allocating memory if necessary.
+
+ **Example**::
+
+ fmt::dynamic_format_arg_store<fmt::format_context> store;
+ store.push_back(42);
+ store.push_back("abc");
+ store.push_back(1.5f);
+ std::string result = fmt::vformat("{} and {} and {}", store);
+ \endrst
+ */
+ template <typename T> void push_back(const T& arg) {
+ if (detail::const_check(need_copy<T>::value))
+ emplace_arg(dynamic_args_.push<stored_type<T>>(arg));
+ else
+ emplace_arg(detail::unwrap(arg));
+ }
+
+ /**
+ \rst
+ Adds a reference to the argument into the dynamic store for later passing to
+ a formatting function.
+
+ **Example**::
+
+ fmt::dynamic_format_arg_store<fmt::format_context> store;
+ char band[] = "Rolling Stones";
+ store.push_back(std::cref(band));
+ band[9] = 'c'; // Changing str affects the output.
+ std::string result = fmt::vformat("{}", store);
+ // result == "Rolling Scones"
+ \endrst
+ */
+ template <typename T> void push_back(std::reference_wrapper<T> arg) {
+ static_assert(
+ need_copy<T>::value,
+ "objects of built-in types and string views are always copied");
+ emplace_arg(arg.get());
+ }
+
+ /**
+ Adds named argument into the dynamic store for later passing to a formatting
+ function. ``std::reference_wrapper`` is supported to avoid copying of the
+ argument. The name is always copied into the store.
+ */
+ template <typename T>
+ void push_back(const detail::named_arg<char_type, T>& arg) {
+ const char_type* arg_name =
+ dynamic_args_.push<std::basic_string<char_type>>(arg.name).c_str();
+ if (detail::const_check(need_copy<T>::value)) {
+ emplace_arg(
+ fmt::arg(arg_name, dynamic_args_.push<stored_type<T>>(arg.value)));
+ } else {
+ emplace_arg(fmt::arg(arg_name, arg.value));
+ }
+ }
+
+ /** Erase all elements from the store */
+ void clear() {
+ data_.clear();
+ named_info_.clear();
+ dynamic_args_ = detail::dynamic_arg_list();
+ }
+
+ /**
+ \rst
+ Reserves space to store at least *new_cap* arguments including
+ *new_cap_named* named arguments.
+ \endrst
+ */
+ void reserve(size_t new_cap, size_t new_cap_named) {
+ FMT_ASSERT(new_cap >= new_cap_named,
+ "Set of arguments includes set of named arguments");
+ data_.reserve(new_cap);
+ named_info_.reserve(new_cap_named);
+ }
+};
+
+FMT_END_NAMESPACE
+
+#endif // FMT_ARGS_H_
diff --git a/contrib/fmt/include/fmt/chrono.h b/contrib/fmt/include/fmt/chrono.h
new file mode 100644
index 0000000..55e8a50
--- /dev/null
+++ b/contrib/fmt/include/fmt/chrono.h
@@ -0,0 +1,2267 @@
+// Formatting library for C++ - chrono support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_CHRONO_H_
+#define FMT_CHRONO_H_
+
+#include <algorithm>
+#include <chrono>
+#include <cmath> // std::isfinite
+#include <cstring> // std::memcpy
+#include <ctime>
+#include <iterator>
+#include <locale>
+#include <ostream>
+#include <type_traits>
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+
+// Check if std::chrono::local_t is available.
+#ifndef FMT_USE_LOCAL_TIME
+# ifdef __cpp_lib_chrono
+# define FMT_USE_LOCAL_TIME (__cpp_lib_chrono >= 201907L)
+# else
+# define FMT_USE_LOCAL_TIME 0
+# endif
+#endif
+
+// Check if std::chrono::utc_timestamp is available.
+#ifndef FMT_USE_UTC_TIME
+# ifdef __cpp_lib_chrono
+# define FMT_USE_UTC_TIME (__cpp_lib_chrono >= 201907L)
+# else
+# define FMT_USE_UTC_TIME 0
+# endif
+#endif
+
+// Enable tzset.
+#ifndef FMT_USE_TZSET
+// UWP doesn't provide _tzset.
+# if FMT_HAS_INCLUDE("winapifamily.h")
+# include <winapifamily.h>
+# endif
+# if defined(_WIN32) && (!defined(WINAPI_FAMILY) || \
+ (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP))
+# define FMT_USE_TZSET 1
+# else
+# define FMT_USE_TZSET 0
+# endif
+#endif
+
+// Enable safe chrono durations, unless explicitly disabled.
+#ifndef FMT_SAFE_DURATION_CAST
+# define FMT_SAFE_DURATION_CAST 1
+#endif
+#if FMT_SAFE_DURATION_CAST
+
+// For conversion between std::chrono::durations without undefined
+// behaviour or erroneous results.
+// This is a stripped down version of duration_cast, for inclusion in fmt.
+// See https://github.com/pauldreik/safe_duration_cast
+//
+// Copyright Paul Dreik 2019
+namespace safe_duration_cast {
+
+template <typename To, typename From,
+ FMT_ENABLE_IF(!std::is_same<From, To>::value &&
+ std::numeric_limits<From>::is_signed ==
+ std::numeric_limits<To>::is_signed)>
+FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) {
+ ec = 0;
+ using F = std::numeric_limits<From>;
+ using T = std::numeric_limits<To>;
+ static_assert(F::is_integer, "From must be integral");
+ static_assert(T::is_integer, "To must be integral");
+
+ // A and B are both signed, or both unsigned.
+ if (detail::const_check(F::digits <= T::digits)) {
+ // From fits in To without any problem.
+ } else {
+ // From does not always fit in To, resort to a dynamic check.
+ if (from < (T::min)() || from > (T::max)()) {
+ // outside range.
+ ec = 1;
+ return {};
+ }
+ }
+ return static_cast<To>(from);
+}
+
+/**
+ * converts From to To, without loss. If the dynamic value of from
+ * can't be converted to To without loss, ec is set.
+ */
+template <typename To, typename From,
+ FMT_ENABLE_IF(!std::is_same<From, To>::value &&
+ std::numeric_limits<From>::is_signed !=
+ std::numeric_limits<To>::is_signed)>
+FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) {
+ ec = 0;
+ using F = std::numeric_limits<From>;
+ using T = std::numeric_limits<To>;
+ static_assert(F::is_integer, "From must be integral");
+ static_assert(T::is_integer, "To must be integral");
+
+ if (detail::const_check(F::is_signed && !T::is_signed)) {
+ // From may be negative, not allowed!
+ if (fmt::detail::is_negative(from)) {
+ ec = 1;
+ return {};
+ }
+ // From is positive. Can it always fit in To?
+ if (detail::const_check(F::digits > T::digits) &&
+ from > static_cast<From>(detail::max_value<To>())) {
+ ec = 1;
+ return {};
+ }
+ }
+
+ if (detail::const_check(!F::is_signed && T::is_signed &&
+ F::digits >= T::digits) &&
+ from > static_cast<From>(detail::max_value<To>())) {
+ ec = 1;
+ return {};
+ }
+ return static_cast<To>(from); // Lossless conversion.
+}
+
+template <typename To, typename From,
+ FMT_ENABLE_IF(std::is_same<From, To>::value)>
+FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) {
+ ec = 0;
+ return from;
+} // function
+
+// clang-format off
+/**
+ * converts From to To if possible, otherwise ec is set.
+ *
+ * input | output
+ * ---------------------------------|---------------
+ * NaN | NaN
+ * Inf | Inf
+ * normal, fits in output | converted (possibly lossy)
+ * normal, does not fit in output | ec is set
+ * subnormal | best effort
+ * -Inf | -Inf
+ */
+// clang-format on
+template <typename To, typename From,
+ FMT_ENABLE_IF(!std::is_same<From, To>::value)>
+FMT_CONSTEXPR To safe_float_conversion(const From from, int& ec) {
+ ec = 0;
+ using T = std::numeric_limits<To>;
+ static_assert(std::is_floating_point<From>::value, "From must be floating");
+ static_assert(std::is_floating_point<To>::value, "To must be floating");
+
+ // catch the only happy case
+ if (std::isfinite(from)) {
+ if (from >= T::lowest() && from <= (T::max)()) {
+ return static_cast<To>(from);
+ }
+ // not within range.
+ ec = 1;
+ return {};
+ }
+
+ // nan and inf will be preserved
+ return static_cast<To>(from);
+} // function
+
+template <typename To, typename From,
+ FMT_ENABLE_IF(std::is_same<From, To>::value)>
+FMT_CONSTEXPR To safe_float_conversion(const From from, int& ec) {
+ ec = 0;
+ static_assert(std::is_floating_point<From>::value, "From must be floating");
+ return from;
+}
+
+/**
+ * safe duration cast between integral durations
+ */
+template <typename To, typename FromRep, typename FromPeriod,
+ FMT_ENABLE_IF(std::is_integral<FromRep>::value),
+ FMT_ENABLE_IF(std::is_integral<typename To::rep>::value)>
+To safe_duration_cast(std::chrono::duration<FromRep, FromPeriod> from,
+ int& ec) {
+ using From = std::chrono::duration<FromRep, FromPeriod>;
+ ec = 0;
+ // the basic idea is that we need to convert from count() in the from type
+ // to count() in the To type, by multiplying it with this:
+ struct Factor
+ : std::ratio_divide<typename From::period, typename To::period> {};
+
+ static_assert(Factor::num > 0, "num must be positive");
+ static_assert(Factor::den > 0, "den must be positive");
+
+ // the conversion is like this: multiply from.count() with Factor::num
+ // /Factor::den and convert it to To::rep, all this without
+ // overflow/underflow. let's start by finding a suitable type that can hold
+ // both To, From and Factor::num
+ using IntermediateRep =
+ typename std::common_type<typename From::rep, typename To::rep,
+ decltype(Factor::num)>::type;
+
+ // safe conversion to IntermediateRep
+ IntermediateRep count =
+ lossless_integral_conversion<IntermediateRep>(from.count(), ec);
+ if (ec) return {};
+ // multiply with Factor::num without overflow or underflow
+ if (detail::const_check(Factor::num != 1)) {
+ const auto max1 = detail::max_value<IntermediateRep>() / Factor::num;
+ if (count > max1) {
+ ec = 1;
+ return {};
+ }
+ const auto min1 =
+ (std::numeric_limits<IntermediateRep>::min)() / Factor::num;
+ if (detail::const_check(!std::is_unsigned<IntermediateRep>::value) &&
+ count < min1) {
+ ec = 1;
+ return {};
+ }
+ count *= Factor::num;
+ }
+
+ if (detail::const_check(Factor::den != 1)) count /= Factor::den;
+ auto tocount = lossless_integral_conversion<typename To::rep>(count, ec);
+ return ec ? To() : To(tocount);
+}
+
+/**
+ * safe duration_cast between floating point durations
+ */
+template <typename To, typename FromRep, typename FromPeriod,
+ FMT_ENABLE_IF(std::is_floating_point<FromRep>::value),
+ FMT_ENABLE_IF(std::is_floating_point<typename To::rep>::value)>
+To safe_duration_cast(std::chrono::duration<FromRep, FromPeriod> from,
+ int& ec) {
+ using From = std::chrono::duration<FromRep, FromPeriod>;
+ ec = 0;
+ if (std::isnan(from.count())) {
+ // nan in, gives nan out. easy.
+ return To{std::numeric_limits<typename To::rep>::quiet_NaN()};
+ }
+ // maybe we should also check if from is denormal, and decide what to do about
+ // it.
+
+ // +-inf should be preserved.
+ if (std::isinf(from.count())) {
+ return To{from.count()};
+ }
+
+ // the basic idea is that we need to convert from count() in the from type
+ // to count() in the To type, by multiplying it with this:
+ struct Factor
+ : std::ratio_divide<typename From::period, typename To::period> {};
+
+ static_assert(Factor::num > 0, "num must be positive");
+ static_assert(Factor::den > 0, "den must be positive");
+
+ // the conversion is like this: multiply from.count() with Factor::num
+ // /Factor::den and convert it to To::rep, all this without
+ // overflow/underflow. let's start by finding a suitable type that can hold
+ // both To, From and Factor::num
+ using IntermediateRep =
+ typename std::common_type<typename From::rep, typename To::rep,
+ decltype(Factor::num)>::type;
+
+ // force conversion of From::rep -> IntermediateRep to be safe,
+ // even if it will never happen be narrowing in this context.
+ IntermediateRep count =
+ safe_float_conversion<IntermediateRep>(from.count(), ec);
+ if (ec) {
+ return {};
+ }
+
+ // multiply with Factor::num without overflow or underflow
+ if (detail::const_check(Factor::num != 1)) {
+ constexpr auto max1 = detail::max_value<IntermediateRep>() /
+ static_cast<IntermediateRep>(Factor::num);
+ if (count > max1) {
+ ec = 1;
+ return {};
+ }
+ constexpr auto min1 = std::numeric_limits<IntermediateRep>::lowest() /
+ static_cast<IntermediateRep>(Factor::num);
+ if (count < min1) {
+ ec = 1;
+ return {};
+ }
+ count *= static_cast<IntermediateRep>(Factor::num);
+ }
+
+ // this can't go wrong, right? den>0 is checked earlier.
+ if (detail::const_check(Factor::den != 1)) {
+ using common_t = typename std::common_type<IntermediateRep, intmax_t>::type;
+ count /= static_cast<common_t>(Factor::den);
+ }
+
+ // convert to the to type, safely
+ using ToRep = typename To::rep;
+
+ const ToRep tocount = safe_float_conversion<ToRep>(count, ec);
+ if (ec) {
+ return {};
+ }
+ return To{tocount};
+}
+} // namespace safe_duration_cast
+#endif
+
+// Prevents expansion of a preceding token as a function-style macro.
+// Usage: f FMT_NOMACRO()
+#define FMT_NOMACRO
+
+namespace detail {
+template <typename T = void> struct null {};
+inline null<> localtime_r FMT_NOMACRO(...) { return null<>(); }
+inline null<> localtime_s(...) { return null<>(); }
+inline null<> gmtime_r(...) { return null<>(); }
+inline null<> gmtime_s(...) { return null<>(); }
+
+inline const std::locale& get_classic_locale() {
+ static const auto& locale = std::locale::classic();
+ return locale;
+}
+
+template <typename CodeUnit> struct codecvt_result {
+ static constexpr const size_t max_size = 32;
+ CodeUnit buf[max_size];
+ CodeUnit* end;
+};
+template <typename CodeUnit>
+constexpr const size_t codecvt_result<CodeUnit>::max_size;
+
+template <typename CodeUnit>
+void write_codecvt(codecvt_result<CodeUnit>& out, string_view in_buf,
+ const std::locale& loc) {
+#if FMT_CLANG_VERSION
+# pragma clang diagnostic push
+# pragma clang diagnostic ignored "-Wdeprecated"
+ auto& f = std::use_facet<std::codecvt<CodeUnit, char, std::mbstate_t>>(loc);
+# pragma clang diagnostic pop
+#else
+ auto& f = std::use_facet<std::codecvt<CodeUnit, char, std::mbstate_t>>(loc);
+#endif
+ auto mb = std::mbstate_t();
+ const char* from_next = nullptr;
+ auto result = f.in(mb, in_buf.begin(), in_buf.end(), from_next,
+ std::begin(out.buf), std::end(out.buf), out.end);
+ if (result != std::codecvt_base::ok)
+ FMT_THROW(format_error("failed to format time"));
+}
+
+template <typename OutputIt>
+auto write_encoded_tm_str(OutputIt out, string_view in, const std::locale& loc)
+ -> OutputIt {
+ if (detail::is_utf8() && loc != get_classic_locale()) {
+ // char16_t and char32_t codecvts are broken in MSVC (linkage errors) and
+ // gcc-4.
+#if FMT_MSC_VERSION != 0 || \
+ (defined(__GLIBCXX__) && !defined(_GLIBCXX_USE_DUAL_ABI))
+ // The _GLIBCXX_USE_DUAL_ABI macro is always defined in libstdc++ from gcc-5
+ // and newer.
+ using code_unit = wchar_t;
+#else
+ using code_unit = char32_t;
+#endif
+
+ using unit_t = codecvt_result<code_unit>;
+ unit_t unit;
+ write_codecvt(unit, in, loc);
+ // In UTF-8 is used one to four one-byte code units.
+ unicode_to_utf8<code_unit, basic_memory_buffer<char, unit_t::max_size * 4>>
+ u;
+ if (!u.convert({unit.buf, to_unsigned(unit.end - unit.buf)}))
+ FMT_THROW(format_error("failed to format time"));
+ return copy_str<char>(u.c_str(), u.c_str() + u.size(), out);
+ }
+ return copy_str<char>(in.data(), in.data() + in.size(), out);
+}
+
+template <typename Char, typename OutputIt,
+ FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
+auto write_tm_str(OutputIt out, string_view sv, const std::locale& loc)
+ -> OutputIt {
+ codecvt_result<Char> unit;
+ write_codecvt(unit, sv, loc);
+ return copy_str<Char>(unit.buf, unit.end, out);
+}
+
+template <typename Char, typename OutputIt,
+ FMT_ENABLE_IF(std::is_same<Char, char>::value)>
+auto write_tm_str(OutputIt out, string_view sv, const std::locale& loc)
+ -> OutputIt {
+ return write_encoded_tm_str(out, sv, loc);
+}
+
+template <typename Char>
+inline void do_write(buffer<Char>& buf, const std::tm& time,
+ const std::locale& loc, char format, char modifier) {
+ auto&& format_buf = formatbuf<std::basic_streambuf<Char>>(buf);
+ auto&& os = std::basic_ostream<Char>(&format_buf);
+ os.imbue(loc);
+ using iterator = std::ostreambuf_iterator<Char>;
+ const auto& facet = std::use_facet<std::time_put<Char, iterator>>(loc);
+ auto end = facet.put(os, os, Char(' '), &time, format, modifier);
+ if (end.failed()) FMT_THROW(format_error("failed to format time"));
+}
+
+template <typename Char, typename OutputIt,
+ FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
+auto write(OutputIt out, const std::tm& time, const std::locale& loc,
+ char format, char modifier = 0) -> OutputIt {
+ auto&& buf = get_buffer<Char>(out);
+ do_write<Char>(buf, time, loc, format, modifier);
+ return get_iterator(buf, out);
+}
+
+template <typename Char, typename OutputIt,
+ FMT_ENABLE_IF(std::is_same<Char, char>::value)>
+auto write(OutputIt out, const std::tm& time, const std::locale& loc,
+ char format, char modifier = 0) -> OutputIt {
+ auto&& buf = basic_memory_buffer<Char>();
+ do_write<char>(buf, time, loc, format, modifier);
+ return write_encoded_tm_str(out, string_view(buf.data(), buf.size()), loc);
+}
+
+} // namespace detail
+
+FMT_BEGIN_EXPORT
+
+/**
+ Converts given time since epoch as ``std::time_t`` value into calendar time,
+ expressed in local time. Unlike ``std::localtime``, this function is
+ thread-safe on most platforms.
+ */
+inline std::tm localtime(std::time_t time) {
+ struct dispatcher {
+ std::time_t time_;
+ std::tm tm_;
+
+ dispatcher(std::time_t t) : time_(t) {}
+
+ bool run() {
+ using namespace fmt::detail;
+ return handle(localtime_r(&time_, &tm_));
+ }
+
+ bool handle(std::tm* tm) { return tm != nullptr; }
+
+ bool handle(detail::null<>) {
+ using namespace fmt::detail;
+ return fallback(localtime_s(&tm_, &time_));
+ }
+
+ bool fallback(int res) { return res == 0; }
+
+#if !FMT_MSC_VERSION
+ bool fallback(detail::null<>) {
+ using namespace fmt::detail;
+ std::tm* tm = std::localtime(&time_);
+ if (tm) tm_ = *tm;
+ return tm != nullptr;
+ }
+#endif
+ };
+ dispatcher lt(time);
+ // Too big time values may be unsupported.
+ if (!lt.run()) FMT_THROW(format_error("time_t value out of range"));
+ return lt.tm_;
+}
+
+#if FMT_USE_LOCAL_TIME
+template <typename Duration>
+inline auto localtime(std::chrono::local_time<Duration> time) -> std::tm {
+ return localtime(std::chrono::system_clock::to_time_t(
+ std::chrono::current_zone()->to_sys(time)));
+}
+#endif
+
+/**
+ Converts given time since epoch as ``std::time_t`` value into calendar time,
+ expressed in Coordinated Universal Time (UTC). Unlike ``std::gmtime``, this
+ function is thread-safe on most platforms.
+ */
+inline std::tm gmtime(std::time_t time) {
+ struct dispatcher {
+ std::time_t time_;
+ std::tm tm_;
+
+ dispatcher(std::time_t t) : time_(t) {}
+
+ bool run() {
+ using namespace fmt::detail;
+ return handle(gmtime_r(&time_, &tm_));
+ }
+
+ bool handle(std::tm* tm) { return tm != nullptr; }
+
+ bool handle(detail::null<>) {
+ using namespace fmt::detail;
+ return fallback(gmtime_s(&tm_, &time_));
+ }
+
+ bool fallback(int res) { return res == 0; }
+
+#if !FMT_MSC_VERSION
+ bool fallback(detail::null<>) {
+ std::tm* tm = std::gmtime(&time_);
+ if (tm) tm_ = *tm;
+ return tm != nullptr;
+ }
+#endif
+ };
+ dispatcher gt(time);
+ // Too big time values may be unsupported.
+ if (!gt.run()) FMT_THROW(format_error("time_t value out of range"));
+ return gt.tm_;
+}
+
+inline std::tm gmtime(
+ std::chrono::time_point<std::chrono::system_clock> time_point) {
+ return gmtime(std::chrono::system_clock::to_time_t(time_point));
+}
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+// DEPRECATED!
+template <typename Char>
+FMT_CONSTEXPR auto parse_align(const Char* begin, const Char* end,
+ format_specs<Char>& specs) -> const Char* {
+ FMT_ASSERT(begin != end, "");
+ auto align = align::none;
+ auto p = begin + code_point_length(begin);
+ if (end - p <= 0) p = begin;
+ for (;;) {
+ switch (to_ascii(*p)) {
+ case '<':
+ align = align::left;
+ break;
+ case '>':
+ align = align::right;
+ break;
+ case '^':
+ align = align::center;
+ break;
+ }
+ if (align != align::none) {
+ if (p != begin) {
+ auto c = *begin;
+ if (c == '}') return begin;
+ if (c == '{') {
+ throw_format_error("invalid fill character '{'");
+ return begin;
+ }
+ specs.fill = {begin, to_unsigned(p - begin)};
+ begin = p + 1;
+ } else {
+ ++begin;
+ }
+ break;
+ } else if (p == begin) {
+ break;
+ }
+ p = begin;
+ }
+ specs.align = align;
+ return begin;
+}
+
+// Writes two-digit numbers a, b and c separated by sep to buf.
+// The method by Pavel Novikov based on
+// https://johnnylee-sde.github.io/Fast-unsigned-integer-to-time-string/.
+inline void write_digit2_separated(char* buf, unsigned a, unsigned b,
+ unsigned c, char sep) {
+ unsigned long long digits =
+ a | (b << 24) | (static_cast<unsigned long long>(c) << 48);
+ // Convert each value to BCD.
+ // We have x = a * 10 + b and we want to convert it to BCD y = a * 16 + b.
+ // The difference is
+ // y - x = a * 6
+ // a can be found from x:
+ // a = floor(x / 10)
+ // then
+ // y = x + a * 6 = x + floor(x / 10) * 6
+ // floor(x / 10) is (x * 205) >> 11 (needs 16 bits).
+ digits += (((digits * 205) >> 11) & 0x000f00000f00000f) * 6;
+ // Put low nibbles to high bytes and high nibbles to low bytes.
+ digits = ((digits & 0x00f00000f00000f0) >> 4) |
+ ((digits & 0x000f00000f00000f) << 8);
+ auto usep = static_cast<unsigned long long>(sep);
+ // Add ASCII '0' to each digit byte and insert separators.
+ digits |= 0x3030003030003030 | (usep << 16) | (usep << 40);
+
+ constexpr const size_t len = 8;
+ if (const_check(is_big_endian())) {
+ char tmp[len];
+ std::memcpy(tmp, &digits, len);
+ std::reverse_copy(tmp, tmp + len, buf);
+ } else {
+ std::memcpy(buf, &digits, len);
+ }
+}
+
+template <typename Period> FMT_CONSTEXPR inline const char* get_units() {
+ if (std::is_same<Period, std::atto>::value) return "as";
+ if (std::is_same<Period, std::femto>::value) return "fs";
+ if (std::is_same<Period, std::pico>::value) return "ps";
+ if (std::is_same<Period, std::nano>::value) return "ns";
+ if (std::is_same<Period, std::micro>::value) return "µs";
+ if (std::is_same<Period, std::milli>::value) return "ms";
+ if (std::is_same<Period, std::centi>::value) return "cs";
+ if (std::is_same<Period, std::deci>::value) return "ds";
+ if (std::is_same<Period, std::ratio<1>>::value) return "s";
+ if (std::is_same<Period, std::deca>::value) return "das";
+ if (std::is_same<Period, std::hecto>::value) return "hs";
+ if (std::is_same<Period, std::kilo>::value) return "ks";
+ if (std::is_same<Period, std::mega>::value) return "Ms";
+ if (std::is_same<Period, std::giga>::value) return "Gs";
+ if (std::is_same<Period, std::tera>::value) return "Ts";
+ if (std::is_same<Period, std::peta>::value) return "Ps";
+ if (std::is_same<Period, std::exa>::value) return "Es";
+ if (std::is_same<Period, std::ratio<60>>::value) return "m";
+ if (std::is_same<Period, std::ratio<3600>>::value) return "h";
+ return nullptr;
+}
+
+enum class numeric_system {
+ standard,
+ // Alternative numeric system, e.g. å二 instead of 12 in ja_JP locale.
+ alternative
+};
+
+// Glibc extensions for formatting numeric values.
+enum class pad_type {
+ unspecified,
+ // Do not pad a numeric result string.
+ none,
+ // Pad a numeric result string with zeros even if the conversion specifier
+ // character uses space-padding by default.
+ zero,
+ // Pad a numeric result string with spaces.
+ space,
+};
+
+template <typename OutputIt>
+auto write_padding(OutputIt out, pad_type pad, int width) -> OutputIt {
+ if (pad == pad_type::none) return out;
+ return std::fill_n(out, width, pad == pad_type::space ? ' ' : '0');
+}
+
+template <typename OutputIt>
+auto write_padding(OutputIt out, pad_type pad) -> OutputIt {
+ if (pad != pad_type::none) *out++ = pad == pad_type::space ? ' ' : '0';
+ return out;
+}
+
+// Parses a put_time-like format string and invokes handler actions.
+template <typename Char, typename Handler>
+FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin,
+ const Char* end,
+ Handler&& handler) {
+ if (begin == end || *begin == '}') return begin;
+ if (*begin != '%') FMT_THROW(format_error("invalid format"));
+ auto ptr = begin;
+ pad_type pad = pad_type::unspecified;
+ while (ptr != end) {
+ auto c = *ptr;
+ if (c == '}') break;
+ if (c != '%') {
+ ++ptr;
+ continue;
+ }
+ if (begin != ptr) handler.on_text(begin, ptr);
+ ++ptr; // consume '%'
+ if (ptr == end) FMT_THROW(format_error("invalid format"));
+ c = *ptr;
+ switch (c) {
+ case '_':
+ pad = pad_type::space;
+ ++ptr;
+ break;
+ case '-':
+ pad = pad_type::none;
+ ++ptr;
+ break;
+ case '0':
+ pad = pad_type::zero;
+ ++ptr;
+ break;
+ }
+ if (ptr == end) FMT_THROW(format_error("invalid format"));
+ c = *ptr++;
+ switch (c) {
+ case '%':
+ handler.on_text(ptr - 1, ptr);
+ break;
+ case 'n': {
+ const Char newline[] = {'\n'};
+ handler.on_text(newline, newline + 1);
+ break;
+ }
+ case 't': {
+ const Char tab[] = {'\t'};
+ handler.on_text(tab, tab + 1);
+ break;
+ }
+ // Year:
+ case 'Y':
+ handler.on_year(numeric_system::standard);
+ break;
+ case 'y':
+ handler.on_short_year(numeric_system::standard);
+ break;
+ case 'C':
+ handler.on_century(numeric_system::standard);
+ break;
+ case 'G':
+ handler.on_iso_week_based_year();
+ break;
+ case 'g':
+ handler.on_iso_week_based_short_year();
+ break;
+ // Day of the week:
+ case 'a':
+ handler.on_abbr_weekday();
+ break;
+ case 'A':
+ handler.on_full_weekday();
+ break;
+ case 'w':
+ handler.on_dec0_weekday(numeric_system::standard);
+ break;
+ case 'u':
+ handler.on_dec1_weekday(numeric_system::standard);
+ break;
+ // Month:
+ case 'b':
+ case 'h':
+ handler.on_abbr_month();
+ break;
+ case 'B':
+ handler.on_full_month();
+ break;
+ case 'm':
+ handler.on_dec_month(numeric_system::standard);
+ break;
+ // Day of the year/month:
+ case 'U':
+ handler.on_dec0_week_of_year(numeric_system::standard);
+ break;
+ case 'W':
+ handler.on_dec1_week_of_year(numeric_system::standard);
+ break;
+ case 'V':
+ handler.on_iso_week_of_year(numeric_system::standard);
+ break;
+ case 'j':
+ handler.on_day_of_year();
+ break;
+ case 'd':
+ handler.on_day_of_month(numeric_system::standard);
+ break;
+ case 'e':
+ handler.on_day_of_month_space(numeric_system::standard);
+ break;
+ // Hour, minute, second:
+ case 'H':
+ handler.on_24_hour(numeric_system::standard, pad);
+ break;
+ case 'I':
+ handler.on_12_hour(numeric_system::standard, pad);
+ break;
+ case 'M':
+ handler.on_minute(numeric_system::standard, pad);
+ break;
+ case 'S':
+ handler.on_second(numeric_system::standard, pad);
+ break;
+ // Other:
+ case 'c':
+ handler.on_datetime(numeric_system::standard);
+ break;
+ case 'x':
+ handler.on_loc_date(numeric_system::standard);
+ break;
+ case 'X':
+ handler.on_loc_time(numeric_system::standard);
+ break;
+ case 'D':
+ handler.on_us_date();
+ break;
+ case 'F':
+ handler.on_iso_date();
+ break;
+ case 'r':
+ handler.on_12_hour_time();
+ break;
+ case 'R':
+ handler.on_24_hour_time();
+ break;
+ case 'T':
+ handler.on_iso_time();
+ break;
+ case 'p':
+ handler.on_am_pm();
+ break;
+ case 'Q':
+ handler.on_duration_value();
+ break;
+ case 'q':
+ handler.on_duration_unit();
+ break;
+ case 'z':
+ handler.on_utc_offset(numeric_system::standard);
+ break;
+ case 'Z':
+ handler.on_tz_name();
+ break;
+ // Alternative representation:
+ case 'E': {
+ if (ptr == end) FMT_THROW(format_error("invalid format"));
+ c = *ptr++;
+ switch (c) {
+ case 'Y':
+ handler.on_year(numeric_system::alternative);
+ break;
+ case 'y':
+ handler.on_offset_year();
+ break;
+ case 'C':
+ handler.on_century(numeric_system::alternative);
+ break;
+ case 'c':
+ handler.on_datetime(numeric_system::alternative);
+ break;
+ case 'x':
+ handler.on_loc_date(numeric_system::alternative);
+ break;
+ case 'X':
+ handler.on_loc_time(numeric_system::alternative);
+ break;
+ case 'z':
+ handler.on_utc_offset(numeric_system::alternative);
+ break;
+ default:
+ FMT_THROW(format_error("invalid format"));
+ }
+ break;
+ }
+ case 'O':
+ if (ptr == end) FMT_THROW(format_error("invalid format"));
+ c = *ptr++;
+ switch (c) {
+ case 'y':
+ handler.on_short_year(numeric_system::alternative);
+ break;
+ case 'm':
+ handler.on_dec_month(numeric_system::alternative);
+ break;
+ case 'U':
+ handler.on_dec0_week_of_year(numeric_system::alternative);
+ break;
+ case 'W':
+ handler.on_dec1_week_of_year(numeric_system::alternative);
+ break;
+ case 'V':
+ handler.on_iso_week_of_year(numeric_system::alternative);
+ break;
+ case 'd':
+ handler.on_day_of_month(numeric_system::alternative);
+ break;
+ case 'e':
+ handler.on_day_of_month_space(numeric_system::alternative);
+ break;
+ case 'w':
+ handler.on_dec0_weekday(numeric_system::alternative);
+ break;
+ case 'u':
+ handler.on_dec1_weekday(numeric_system::alternative);
+ break;
+ case 'H':
+ handler.on_24_hour(numeric_system::alternative, pad);
+ break;
+ case 'I':
+ handler.on_12_hour(numeric_system::alternative, pad);
+ break;
+ case 'M':
+ handler.on_minute(numeric_system::alternative, pad);
+ break;
+ case 'S':
+ handler.on_second(numeric_system::alternative, pad);
+ break;
+ case 'z':
+ handler.on_utc_offset(numeric_system::alternative);
+ break;
+ default:
+ FMT_THROW(format_error("invalid format"));
+ }
+ break;
+ default:
+ FMT_THROW(format_error("invalid format"));
+ }
+ begin = ptr;
+ }
+ if (begin != ptr) handler.on_text(begin, ptr);
+ return ptr;
+}
+
+template <typename Derived> struct null_chrono_spec_handler {
+ FMT_CONSTEXPR void unsupported() {
+ static_cast<Derived*>(this)->unsupported();
+ }
+ FMT_CONSTEXPR void on_year(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_short_year(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_offset_year() { unsupported(); }
+ FMT_CONSTEXPR void on_century(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_iso_week_based_year() { unsupported(); }
+ FMT_CONSTEXPR void on_iso_week_based_short_year() { unsupported(); }
+ FMT_CONSTEXPR void on_abbr_weekday() { unsupported(); }
+ FMT_CONSTEXPR void on_full_weekday() { unsupported(); }
+ FMT_CONSTEXPR void on_dec0_weekday(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_dec1_weekday(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_abbr_month() { unsupported(); }
+ FMT_CONSTEXPR void on_full_month() { unsupported(); }
+ FMT_CONSTEXPR void on_dec_month(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_dec0_week_of_year(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_dec1_week_of_year(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_iso_week_of_year(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_day_of_year() { unsupported(); }
+ FMT_CONSTEXPR void on_day_of_month(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_day_of_month_space(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_24_hour(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_12_hour(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_minute(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_second(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_datetime(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_loc_date(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_loc_time(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_us_date() { unsupported(); }
+ FMT_CONSTEXPR void on_iso_date() { unsupported(); }
+ FMT_CONSTEXPR void on_12_hour_time() { unsupported(); }
+ FMT_CONSTEXPR void on_24_hour_time() { unsupported(); }
+ FMT_CONSTEXPR void on_iso_time() { unsupported(); }
+ FMT_CONSTEXPR void on_am_pm() { unsupported(); }
+ FMT_CONSTEXPR void on_duration_value() { unsupported(); }
+ FMT_CONSTEXPR void on_duration_unit() { unsupported(); }
+ FMT_CONSTEXPR void on_utc_offset(numeric_system) { unsupported(); }
+ FMT_CONSTEXPR void on_tz_name() { unsupported(); }
+};
+
+struct tm_format_checker : null_chrono_spec_handler<tm_format_checker> {
+ FMT_NORETURN void unsupported() { FMT_THROW(format_error("no format")); }
+
+ template <typename Char>
+ FMT_CONSTEXPR void on_text(const Char*, const Char*) {}
+ FMT_CONSTEXPR void on_year(numeric_system) {}
+ FMT_CONSTEXPR void on_short_year(numeric_system) {}
+ FMT_CONSTEXPR void on_offset_year() {}
+ FMT_CONSTEXPR void on_century(numeric_system) {}
+ FMT_CONSTEXPR void on_iso_week_based_year() {}
+ FMT_CONSTEXPR void on_iso_week_based_short_year() {}
+ FMT_CONSTEXPR void on_abbr_weekday() {}
+ FMT_CONSTEXPR void on_full_weekday() {}
+ FMT_CONSTEXPR void on_dec0_weekday(numeric_system) {}
+ FMT_CONSTEXPR void on_dec1_weekday(numeric_system) {}
+ FMT_CONSTEXPR void on_abbr_month() {}
+ FMT_CONSTEXPR void on_full_month() {}
+ FMT_CONSTEXPR void on_dec_month(numeric_system) {}
+ FMT_CONSTEXPR void on_dec0_week_of_year(numeric_system) {}
+ FMT_CONSTEXPR void on_dec1_week_of_year(numeric_system) {}
+ FMT_CONSTEXPR void on_iso_week_of_year(numeric_system) {}
+ FMT_CONSTEXPR void on_day_of_year() {}
+ FMT_CONSTEXPR void on_day_of_month(numeric_system) {}
+ FMT_CONSTEXPR void on_day_of_month_space(numeric_system) {}
+ FMT_CONSTEXPR void on_24_hour(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_12_hour(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_minute(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_second(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_datetime(numeric_system) {}
+ FMT_CONSTEXPR void on_loc_date(numeric_system) {}
+ FMT_CONSTEXPR void on_loc_time(numeric_system) {}
+ FMT_CONSTEXPR void on_us_date() {}
+ FMT_CONSTEXPR void on_iso_date() {}
+ FMT_CONSTEXPR void on_12_hour_time() {}
+ FMT_CONSTEXPR void on_24_hour_time() {}
+ FMT_CONSTEXPR void on_iso_time() {}
+ FMT_CONSTEXPR void on_am_pm() {}
+ FMT_CONSTEXPR void on_utc_offset(numeric_system) {}
+ FMT_CONSTEXPR void on_tz_name() {}
+};
+
+inline const char* tm_wday_full_name(int wday) {
+ static constexpr const char* full_name_list[] = {
+ "Sunday", "Monday", "Tuesday", "Wednesday",
+ "Thursday", "Friday", "Saturday"};
+ return wday >= 0 && wday <= 6 ? full_name_list[wday] : "?";
+}
+inline const char* tm_wday_short_name(int wday) {
+ static constexpr const char* short_name_list[] = {"Sun", "Mon", "Tue", "Wed",
+ "Thu", "Fri", "Sat"};
+ return wday >= 0 && wday <= 6 ? short_name_list[wday] : "???";
+}
+
+inline const char* tm_mon_full_name(int mon) {
+ static constexpr const char* full_name_list[] = {
+ "January", "February", "March", "April", "May", "June",
+ "July", "August", "September", "October", "November", "December"};
+ return mon >= 0 && mon <= 11 ? full_name_list[mon] : "?";
+}
+inline const char* tm_mon_short_name(int mon) {
+ static constexpr const char* short_name_list[] = {
+ "Jan", "Feb", "Mar", "Apr", "May", "Jun",
+ "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
+ };
+ return mon >= 0 && mon <= 11 ? short_name_list[mon] : "???";
+}
+
+template <typename T, typename = void>
+struct has_member_data_tm_gmtoff : std::false_type {};
+template <typename T>
+struct has_member_data_tm_gmtoff<T, void_t<decltype(T::tm_gmtoff)>>
+ : std::true_type {};
+
+template <typename T, typename = void>
+struct has_member_data_tm_zone : std::false_type {};
+template <typename T>
+struct has_member_data_tm_zone<T, void_t<decltype(T::tm_zone)>>
+ : std::true_type {};
+
+#if FMT_USE_TZSET
+inline void tzset_once() {
+ static bool init = []() -> bool {
+ _tzset();
+ return true;
+ }();
+ ignore_unused(init);
+}
+#endif
+
+// Converts value to Int and checks that it's in the range [0, upper).
+template <typename T, typename Int, FMT_ENABLE_IF(std::is_integral<T>::value)>
+inline Int to_nonnegative_int(T value, Int upper) {
+ FMT_ASSERT(std::is_unsigned<Int>::value ||
+ (value >= 0 && to_unsigned(value) <= to_unsigned(upper)),
+ "invalid value");
+ (void)upper;
+ return static_cast<Int>(value);
+}
+template <typename T, typename Int, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+inline Int to_nonnegative_int(T value, Int upper) {
+ if (value < 0 || value > static_cast<T>(upper))
+ FMT_THROW(format_error("invalid value"));
+ return static_cast<Int>(value);
+}
+
+constexpr long long pow10(std::uint32_t n) {
+ return n == 0 ? 1 : 10 * pow10(n - 1);
+}
+
+// Counts the number of fractional digits in the range [0, 18] according to the
+// C++20 spec. If more than 18 fractional digits are required then returns 6 for
+// microseconds precision.
+template <long long Num, long long Den, int N = 0,
+ bool Enabled = (N < 19) && (Num <= max_value<long long>() / 10)>
+struct count_fractional_digits {
+ static constexpr int value =
+ Num % Den == 0 ? N : count_fractional_digits<Num * 10, Den, N + 1>::value;
+};
+
+// Base case that doesn't instantiate any more templates
+// in order to avoid overflow.
+template <long long Num, long long Den, int N>
+struct count_fractional_digits<Num, Den, N, false> {
+ static constexpr int value = (Num % Den == 0) ? N : 6;
+};
+
+// Format subseconds which are given as an integer type with an appropriate
+// number of digits.
+template <typename Char, typename OutputIt, typename Duration>
+void write_fractional_seconds(OutputIt& out, Duration d, int precision = -1) {
+ constexpr auto num_fractional_digits =
+ count_fractional_digits<Duration::period::num,
+ Duration::period::den>::value;
+
+ using subsecond_precision = std::chrono::duration<
+ typename std::common_type<typename Duration::rep,
+ std::chrono::seconds::rep>::type,
+ std::ratio<1, detail::pow10(num_fractional_digits)>>;
+
+ const auto fractional =
+ d - std::chrono::duration_cast<std::chrono::seconds>(d);
+ const auto subseconds =
+ std::chrono::treat_as_floating_point<
+ typename subsecond_precision::rep>::value
+ ? fractional.count()
+ : std::chrono::duration_cast<subsecond_precision>(fractional).count();
+ auto n = static_cast<uint32_or_64_or_128_t<long long>>(subseconds);
+ const int num_digits = detail::count_digits(n);
+
+ int leading_zeroes = (std::max)(0, num_fractional_digits - num_digits);
+ if (precision < 0) {
+ FMT_ASSERT(!std::is_floating_point<typename Duration::rep>::value, "");
+ if (std::ratio_less<typename subsecond_precision::period,
+ std::chrono::seconds::period>::value) {
+ *out++ = '.';
+ out = std::fill_n(out, leading_zeroes, '0');
+ out = format_decimal<Char>(out, n, num_digits).end;
+ }
+ } else {
+ *out++ = '.';
+ leading_zeroes = (std::min)(leading_zeroes, precision);
+ out = std::fill_n(out, leading_zeroes, '0');
+ int remaining = precision - leading_zeroes;
+ if (remaining != 0 && remaining < num_digits) {
+ n /= to_unsigned(detail::pow10(to_unsigned(num_digits - remaining)));
+ out = format_decimal<Char>(out, n, remaining).end;
+ return;
+ }
+ out = format_decimal<Char>(out, n, num_digits).end;
+ remaining -= num_digits;
+ out = std::fill_n(out, remaining, '0');
+ }
+}
+
+// Format subseconds which are given as a floating point type with an
+// appropriate number of digits. We cannot pass the Duration here, as we
+// explicitly need to pass the Rep value in the chrono_formatter.
+template <typename Duration>
+void write_floating_seconds(memory_buffer& buf, Duration duration,
+ int num_fractional_digits = -1) {
+ using rep = typename Duration::rep;
+ FMT_ASSERT(std::is_floating_point<rep>::value, "");
+
+ auto val = duration.count();
+
+ if (num_fractional_digits < 0) {
+ // For `std::round` with fallback to `round`:
+ // On some toolchains `std::round` is not available (e.g. GCC 6).
+ using namespace std;
+ num_fractional_digits =
+ count_fractional_digits<Duration::period::num,
+ Duration::period::den>::value;
+ if (num_fractional_digits < 6 && static_cast<rep>(round(val)) != val)
+ num_fractional_digits = 6;
+ }
+
+ format_to(std::back_inserter(buf), FMT_STRING("{:.{}f}"),
+ std::fmod(val * static_cast<rep>(Duration::period::num) /
+ static_cast<rep>(Duration::period::den),
+ static_cast<rep>(60)),
+ num_fractional_digits);
+}
+
+template <typename OutputIt, typename Char,
+ typename Duration = std::chrono::seconds>
+class tm_writer {
+ private:
+ static constexpr int days_per_week = 7;
+
+ const std::locale& loc_;
+ const bool is_classic_;
+ OutputIt out_;
+ const Duration* subsecs_;
+ const std::tm& tm_;
+
+ auto tm_sec() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_sec >= 0 && tm_.tm_sec <= 61, "");
+ return tm_.tm_sec;
+ }
+ auto tm_min() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_min >= 0 && tm_.tm_min <= 59, "");
+ return tm_.tm_min;
+ }
+ auto tm_hour() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_hour >= 0 && tm_.tm_hour <= 23, "");
+ return tm_.tm_hour;
+ }
+ auto tm_mday() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_mday >= 1 && tm_.tm_mday <= 31, "");
+ return tm_.tm_mday;
+ }
+ auto tm_mon() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_mon >= 0 && tm_.tm_mon <= 11, "");
+ return tm_.tm_mon;
+ }
+ auto tm_year() const noexcept -> long long { return 1900ll + tm_.tm_year; }
+ auto tm_wday() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_wday >= 0 && tm_.tm_wday <= 6, "");
+ return tm_.tm_wday;
+ }
+ auto tm_yday() const noexcept -> int {
+ FMT_ASSERT(tm_.tm_yday >= 0 && tm_.tm_yday <= 365, "");
+ return tm_.tm_yday;
+ }
+
+ auto tm_hour12() const noexcept -> int {
+ const auto h = tm_hour();
+ const auto z = h < 12 ? h : h - 12;
+ return z == 0 ? 12 : z;
+ }
+
+ // POSIX and the C Standard are unclear or inconsistent about what %C and %y
+ // do if the year is negative or exceeds 9999. Use the convention that %C
+ // concatenated with %y yields the same output as %Y, and that %Y contains at
+ // least 4 characters, with more only if necessary.
+ auto split_year_lower(long long year) const noexcept -> int {
+ auto l = year % 100;
+ if (l < 0) l = -l; // l in [0, 99]
+ return static_cast<int>(l);
+ }
+
+ // Algorithm:
+ // https://en.wikipedia.org/wiki/ISO_week_date#Calculating_the_week_number_from_a_month_and_day_of_the_month_or_ordinal_date
+ auto iso_year_weeks(long long curr_year) const noexcept -> int {
+ const auto prev_year = curr_year - 1;
+ const auto curr_p =
+ (curr_year + curr_year / 4 - curr_year / 100 + curr_year / 400) %
+ days_per_week;
+ const auto prev_p =
+ (prev_year + prev_year / 4 - prev_year / 100 + prev_year / 400) %
+ days_per_week;
+ return 52 + ((curr_p == 4 || prev_p == 3) ? 1 : 0);
+ }
+ auto iso_week_num(int tm_yday, int tm_wday) const noexcept -> int {
+ return (tm_yday + 11 - (tm_wday == 0 ? days_per_week : tm_wday)) /
+ days_per_week;
+ }
+ auto tm_iso_week_year() const noexcept -> long long {
+ const auto year = tm_year();
+ const auto w = iso_week_num(tm_yday(), tm_wday());
+ if (w < 1) return year - 1;
+ if (w > iso_year_weeks(year)) return year + 1;
+ return year;
+ }
+ auto tm_iso_week_of_year() const noexcept -> int {
+ const auto year = tm_year();
+ const auto w = iso_week_num(tm_yday(), tm_wday());
+ if (w < 1) return iso_year_weeks(year - 1);
+ if (w > iso_year_weeks(year)) return 1;
+ return w;
+ }
+
+ void write1(int value) {
+ *out_++ = static_cast<char>('0' + to_unsigned(value) % 10);
+ }
+ void write2(int value) {
+ const char* d = digits2(to_unsigned(value) % 100);
+ *out_++ = *d++;
+ *out_++ = *d;
+ }
+ void write2(int value, pad_type pad) {
+ unsigned int v = to_unsigned(value) % 100;
+ if (v >= 10) {
+ const char* d = digits2(v);
+ *out_++ = *d++;
+ *out_++ = *d;
+ } else {
+ out_ = detail::write_padding(out_, pad);
+ *out_++ = static_cast<char>('0' + v);
+ }
+ }
+
+ void write_year_extended(long long year) {
+ // At least 4 characters.
+ int width = 4;
+ if (year < 0) {
+ *out_++ = '-';
+ year = 0 - year;
+ --width;
+ }
+ uint32_or_64_or_128_t<long long> n = to_unsigned(year);
+ const int num_digits = count_digits(n);
+ if (width > num_digits) out_ = std::fill_n(out_, width - num_digits, '0');
+ out_ = format_decimal<Char>(out_, n, num_digits).end;
+ }
+ void write_year(long long year) {
+ if (year >= 0 && year < 10000) {
+ write2(static_cast<int>(year / 100));
+ write2(static_cast<int>(year % 100));
+ } else {
+ write_year_extended(year);
+ }
+ }
+
+ void write_utc_offset(long offset, numeric_system ns) {
+ if (offset < 0) {
+ *out_++ = '-';
+ offset = -offset;
+ } else {
+ *out_++ = '+';
+ }
+ offset /= 60;
+ write2(static_cast<int>(offset / 60));
+ if (ns != numeric_system::standard) *out_++ = ':';
+ write2(static_cast<int>(offset % 60));
+ }
+ template <typename T, FMT_ENABLE_IF(has_member_data_tm_gmtoff<T>::value)>
+ void format_utc_offset_impl(const T& tm, numeric_system ns) {
+ write_utc_offset(tm.tm_gmtoff, ns);
+ }
+ template <typename T, FMT_ENABLE_IF(!has_member_data_tm_gmtoff<T>::value)>
+ void format_utc_offset_impl(const T& tm, numeric_system ns) {
+#if defined(_WIN32) && defined(_UCRT)
+# if FMT_USE_TZSET
+ tzset_once();
+# endif
+ long offset = 0;
+ _get_timezone(&offset);
+ if (tm.tm_isdst) {
+ long dstbias = 0;
+ _get_dstbias(&dstbias);
+ offset += dstbias;
+ }
+ write_utc_offset(-offset, ns);
+#else
+ if (ns == numeric_system::standard) return format_localized('z');
+
+ // Extract timezone offset from timezone conversion functions.
+ std::tm gtm = tm;
+ std::time_t gt = std::mktime(&gtm);
+ std::tm ltm = gmtime(gt);
+ std::time_t lt = std::mktime(&ltm);
+ long offset = gt - lt;
+ write_utc_offset(offset, ns);
+#endif
+ }
+
+ template <typename T, FMT_ENABLE_IF(has_member_data_tm_zone<T>::value)>
+ void format_tz_name_impl(const T& tm) {
+ if (is_classic_)
+ out_ = write_tm_str<Char>(out_, tm.tm_zone, loc_);
+ else
+ format_localized('Z');
+ }
+ template <typename T, FMT_ENABLE_IF(!has_member_data_tm_zone<T>::value)>
+ void format_tz_name_impl(const T&) {
+ format_localized('Z');
+ }
+
+ void format_localized(char format, char modifier = 0) {
+ out_ = write<Char>(out_, tm_, loc_, format, modifier);
+ }
+
+ public:
+ tm_writer(const std::locale& loc, OutputIt out, const std::tm& tm,
+ const Duration* subsecs = nullptr)
+ : loc_(loc),
+ is_classic_(loc_ == get_classic_locale()),
+ out_(out),
+ subsecs_(subsecs),
+ tm_(tm) {}
+
+ OutputIt out() const { return out_; }
+
+ FMT_CONSTEXPR void on_text(const Char* begin, const Char* end) {
+ out_ = copy_str<Char>(begin, end, out_);
+ }
+
+ void on_abbr_weekday() {
+ if (is_classic_)
+ out_ = write(out_, tm_wday_short_name(tm_wday()));
+ else
+ format_localized('a');
+ }
+ void on_full_weekday() {
+ if (is_classic_)
+ out_ = write(out_, tm_wday_full_name(tm_wday()));
+ else
+ format_localized('A');
+ }
+ void on_dec0_weekday(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) return write1(tm_wday());
+ format_localized('w', 'O');
+ }
+ void on_dec1_weekday(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) {
+ auto wday = tm_wday();
+ write1(wday == 0 ? days_per_week : wday);
+ } else {
+ format_localized('u', 'O');
+ }
+ }
+
+ void on_abbr_month() {
+ if (is_classic_)
+ out_ = write(out_, tm_mon_short_name(tm_mon()));
+ else
+ format_localized('b');
+ }
+ void on_full_month() {
+ if (is_classic_)
+ out_ = write(out_, tm_mon_full_name(tm_mon()));
+ else
+ format_localized('B');
+ }
+
+ void on_datetime(numeric_system ns) {
+ if (is_classic_) {
+ on_abbr_weekday();
+ *out_++ = ' ';
+ on_abbr_month();
+ *out_++ = ' ';
+ on_day_of_month_space(numeric_system::standard);
+ *out_++ = ' ';
+ on_iso_time();
+ *out_++ = ' ';
+ on_year(numeric_system::standard);
+ } else {
+ format_localized('c', ns == numeric_system::standard ? '\0' : 'E');
+ }
+ }
+ void on_loc_date(numeric_system ns) {
+ if (is_classic_)
+ on_us_date();
+ else
+ format_localized('x', ns == numeric_system::standard ? '\0' : 'E');
+ }
+ void on_loc_time(numeric_system ns) {
+ if (is_classic_)
+ on_iso_time();
+ else
+ format_localized('X', ns == numeric_system::standard ? '\0' : 'E');
+ }
+ void on_us_date() {
+ char buf[8];
+ write_digit2_separated(buf, to_unsigned(tm_mon() + 1),
+ to_unsigned(tm_mday()),
+ to_unsigned(split_year_lower(tm_year())), '/');
+ out_ = copy_str<Char>(std::begin(buf), std::end(buf), out_);
+ }
+ void on_iso_date() {
+ auto year = tm_year();
+ char buf[10];
+ size_t offset = 0;
+ if (year >= 0 && year < 10000) {
+ copy2(buf, digits2(static_cast<size_t>(year / 100)));
+ } else {
+ offset = 4;
+ write_year_extended(year);
+ year = 0;
+ }
+ write_digit2_separated(buf + 2, static_cast<unsigned>(year % 100),
+ to_unsigned(tm_mon() + 1), to_unsigned(tm_mday()),
+ '-');
+ out_ = copy_str<Char>(std::begin(buf) + offset, std::end(buf), out_);
+ }
+
+ void on_utc_offset(numeric_system ns) { format_utc_offset_impl(tm_, ns); }
+ void on_tz_name() { format_tz_name_impl(tm_); }
+
+ void on_year(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write_year(tm_year());
+ format_localized('Y', 'E');
+ }
+ void on_short_year(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(split_year_lower(tm_year()));
+ format_localized('y', 'O');
+ }
+ void on_offset_year() {
+ if (is_classic_) return write2(split_year_lower(tm_year()));
+ format_localized('y', 'E');
+ }
+
+ void on_century(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) {
+ auto year = tm_year();
+ auto upper = year / 100;
+ if (year >= -99 && year < 0) {
+ // Zero upper on negative year.
+ *out_++ = '-';
+ *out_++ = '0';
+ } else if (upper >= 0 && upper < 100) {
+ write2(static_cast<int>(upper));
+ } else {
+ out_ = write<Char>(out_, upper);
+ }
+ } else {
+ format_localized('C', 'E');
+ }
+ }
+
+ void on_dec_month(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(tm_mon() + 1);
+ format_localized('m', 'O');
+ }
+
+ void on_dec0_week_of_year(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2((tm_yday() + days_per_week - tm_wday()) / days_per_week);
+ format_localized('U', 'O');
+ }
+ void on_dec1_week_of_year(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) {
+ auto wday = tm_wday();
+ write2((tm_yday() + days_per_week -
+ (wday == 0 ? (days_per_week - 1) : (wday - 1))) /
+ days_per_week);
+ } else {
+ format_localized('W', 'O');
+ }
+ }
+ void on_iso_week_of_year(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(tm_iso_week_of_year());
+ format_localized('V', 'O');
+ }
+
+ void on_iso_week_based_year() { write_year(tm_iso_week_year()); }
+ void on_iso_week_based_short_year() {
+ write2(split_year_lower(tm_iso_week_year()));
+ }
+
+ void on_day_of_year() {
+ auto yday = tm_yday() + 1;
+ write1(yday / 100);
+ write2(yday % 100);
+ }
+ void on_day_of_month(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) return write2(tm_mday());
+ format_localized('d', 'O');
+ }
+ void on_day_of_month_space(numeric_system ns) {
+ if (is_classic_ || ns == numeric_system::standard) {
+ auto mday = to_unsigned(tm_mday()) % 100;
+ const char* d2 = digits2(mday);
+ *out_++ = mday < 10 ? ' ' : d2[0];
+ *out_++ = d2[1];
+ } else {
+ format_localized('e', 'O');
+ }
+ }
+
+ void on_24_hour(numeric_system ns, pad_type pad) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(tm_hour(), pad);
+ format_localized('H', 'O');
+ }
+ void on_12_hour(numeric_system ns, pad_type pad) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(tm_hour12(), pad);
+ format_localized('I', 'O');
+ }
+ void on_minute(numeric_system ns, pad_type pad) {
+ if (is_classic_ || ns == numeric_system::standard)
+ return write2(tm_min(), pad);
+ format_localized('M', 'O');
+ }
+
+ void on_second(numeric_system ns, pad_type pad) {
+ if (is_classic_ || ns == numeric_system::standard) {
+ write2(tm_sec(), pad);
+ if (subsecs_) {
+ if (std::is_floating_point<typename Duration::rep>::value) {
+ auto buf = memory_buffer();
+ write_floating_seconds(buf, *subsecs_);
+ if (buf.size() > 1) {
+ // Remove the leading "0", write something like ".123".
+ out_ = std::copy(buf.begin() + 1, buf.end(), out_);
+ }
+ } else {
+ write_fractional_seconds<Char>(out_, *subsecs_);
+ }
+ }
+ } else {
+ // Currently no formatting of subseconds when a locale is set.
+ format_localized('S', 'O');
+ }
+ }
+
+ void on_12_hour_time() {
+ if (is_classic_) {
+ char buf[8];
+ write_digit2_separated(buf, to_unsigned(tm_hour12()),
+ to_unsigned(tm_min()), to_unsigned(tm_sec()), ':');
+ out_ = copy_str<Char>(std::begin(buf), std::end(buf), out_);
+ *out_++ = ' ';
+ on_am_pm();
+ } else {
+ format_localized('r');
+ }
+ }
+ void on_24_hour_time() {
+ write2(tm_hour());
+ *out_++ = ':';
+ write2(tm_min());
+ }
+ void on_iso_time() {
+ on_24_hour_time();
+ *out_++ = ':';
+ on_second(numeric_system::standard, pad_type::unspecified);
+ }
+
+ void on_am_pm() {
+ if (is_classic_) {
+ *out_++ = tm_hour() < 12 ? 'A' : 'P';
+ *out_++ = 'M';
+ } else {
+ format_localized('p');
+ }
+ }
+
+ // These apply to chrono durations but not tm.
+ void on_duration_value() {}
+ void on_duration_unit() {}
+};
+
+struct chrono_format_checker : null_chrono_spec_handler<chrono_format_checker> {
+ bool has_precision_integral = false;
+
+ FMT_NORETURN void unsupported() { FMT_THROW(format_error("no date")); }
+
+ template <typename Char>
+ FMT_CONSTEXPR void on_text(const Char*, const Char*) {}
+ FMT_CONSTEXPR void on_24_hour(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_12_hour(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_minute(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_second(numeric_system, pad_type) {}
+ FMT_CONSTEXPR void on_12_hour_time() {}
+ FMT_CONSTEXPR void on_24_hour_time() {}
+ FMT_CONSTEXPR void on_iso_time() {}
+ FMT_CONSTEXPR void on_am_pm() {}
+ FMT_CONSTEXPR void on_duration_value() const {
+ if (has_precision_integral) {
+ FMT_THROW(format_error("precision not allowed for this argument type"));
+ }
+ }
+ FMT_CONSTEXPR void on_duration_unit() {}
+};
+
+template <typename T,
+ FMT_ENABLE_IF(std::is_integral<T>::value&& has_isfinite<T>::value)>
+inline bool isfinite(T) {
+ return true;
+}
+
+template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+inline T mod(T x, int y) {
+ return x % static_cast<T>(y);
+}
+template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value)>
+inline T mod(T x, int y) {
+ return std::fmod(x, static_cast<T>(y));
+}
+
+// If T is an integral type, maps T to its unsigned counterpart, otherwise
+// leaves it unchanged (unlike std::make_unsigned).
+template <typename T, bool INTEGRAL = std::is_integral<T>::value>
+struct make_unsigned_or_unchanged {
+ using type = T;
+};
+
+template <typename T> struct make_unsigned_or_unchanged<T, true> {
+ using type = typename std::make_unsigned<T>::type;
+};
+
+#if FMT_SAFE_DURATION_CAST
+// throwing version of safe_duration_cast
+template <typename To, typename FromRep, typename FromPeriod>
+To fmt_safe_duration_cast(std::chrono::duration<FromRep, FromPeriod> from) {
+ int ec;
+ To to = safe_duration_cast::safe_duration_cast<To>(from, ec);
+ if (ec) FMT_THROW(format_error("cannot format duration"));
+ return to;
+}
+#endif
+
+template <typename Rep, typename Period,
+ FMT_ENABLE_IF(std::is_integral<Rep>::value)>
+inline std::chrono::duration<Rep, std::milli> get_milliseconds(
+ std::chrono::duration<Rep, Period> d) {
+ // this may overflow and/or the result may not fit in the
+ // target type.
+#if FMT_SAFE_DURATION_CAST
+ using CommonSecondsType =
+ typename std::common_type<decltype(d), std::chrono::seconds>::type;
+ const auto d_as_common = fmt_safe_duration_cast<CommonSecondsType>(d);
+ const auto d_as_whole_seconds =
+ fmt_safe_duration_cast<std::chrono::seconds>(d_as_common);
+ // this conversion should be nonproblematic
+ const auto diff = d_as_common - d_as_whole_seconds;
+ const auto ms =
+ fmt_safe_duration_cast<std::chrono::duration<Rep, std::milli>>(diff);
+ return ms;
+#else
+ auto s = std::chrono::duration_cast<std::chrono::seconds>(d);
+ return std::chrono::duration_cast<std::chrono::milliseconds>(d - s);
+#endif
+}
+
+template <typename Char, typename Rep, typename OutputIt,
+ FMT_ENABLE_IF(std::is_integral<Rep>::value)>
+OutputIt format_duration_value(OutputIt out, Rep val, int) {
+ return write<Char>(out, val);
+}
+
+template <typename Char, typename Rep, typename OutputIt,
+ FMT_ENABLE_IF(std::is_floating_point<Rep>::value)>
+OutputIt format_duration_value(OutputIt out, Rep val, int precision) {
+ auto specs = format_specs<Char>();
+ specs.precision = precision;
+ specs.type = precision >= 0 ? presentation_type::fixed_lower
+ : presentation_type::general_lower;
+ return write<Char>(out, val, specs);
+}
+
+template <typename Char, typename OutputIt>
+OutputIt copy_unit(string_view unit, OutputIt out, Char) {
+ return std::copy(unit.begin(), unit.end(), out);
+}
+
+template <typename OutputIt>
+OutputIt copy_unit(string_view unit, OutputIt out, wchar_t) {
+ // This works when wchar_t is UTF-32 because units only contain characters
+ // that have the same representation in UTF-16 and UTF-32.
+ utf8_to_utf16 u(unit);
+ return std::copy(u.c_str(), u.c_str() + u.size(), out);
+}
+
+template <typename Char, typename Period, typename OutputIt>
+OutputIt format_duration_unit(OutputIt out) {
+ if (const char* unit = get_units<Period>())
+ return copy_unit(string_view(unit), out, Char());
+ *out++ = '[';
+ out = write<Char>(out, Period::num);
+ if (const_check(Period::den != 1)) {
+ *out++ = '/';
+ out = write<Char>(out, Period::den);
+ }
+ *out++ = ']';
+ *out++ = 's';
+ return out;
+}
+
+class get_locale {
+ private:
+ union {
+ std::locale locale_;
+ };
+ bool has_locale_ = false;
+
+ public:
+ get_locale(bool localized, locale_ref loc) : has_locale_(localized) {
+ if (localized)
+ ::new (&locale_) std::locale(loc.template get<std::locale>());
+ }
+ ~get_locale() {
+ if (has_locale_) locale_.~locale();
+ }
+ operator const std::locale&() const {
+ return has_locale_ ? locale_ : get_classic_locale();
+ }
+};
+
+template <typename FormatContext, typename OutputIt, typename Rep,
+ typename Period>
+struct chrono_formatter {
+ FormatContext& context;
+ OutputIt out;
+ int precision;
+ bool localized = false;
+ // rep is unsigned to avoid overflow.
+ using rep =
+ conditional_t<std::is_integral<Rep>::value && sizeof(Rep) < sizeof(int),
+ unsigned, typename make_unsigned_or_unchanged<Rep>::type>;
+ rep val;
+ using seconds = std::chrono::duration<rep>;
+ seconds s;
+ using milliseconds = std::chrono::duration<rep, std::milli>;
+ bool negative;
+
+ using char_type = typename FormatContext::char_type;
+ using tm_writer_type = tm_writer<OutputIt, char_type>;
+
+ chrono_formatter(FormatContext& ctx, OutputIt o,
+ std::chrono::duration<Rep, Period> d)
+ : context(ctx),
+ out(o),
+ val(static_cast<rep>(d.count())),
+ negative(false) {
+ if (d.count() < 0) {
+ val = 0 - val;
+ negative = true;
+ }
+
+ // this may overflow and/or the result may not fit in the
+ // target type.
+#if FMT_SAFE_DURATION_CAST
+ // might need checked conversion (rep!=Rep)
+ auto tmpval = std::chrono::duration<rep, Period>(val);
+ s = fmt_safe_duration_cast<seconds>(tmpval);
+#else
+ s = std::chrono::duration_cast<seconds>(
+ std::chrono::duration<rep, Period>(val));
+#endif
+ }
+
+ // returns true if nan or inf, writes to out.
+ bool handle_nan_inf() {
+ if (isfinite(val)) {
+ return false;
+ }
+ if (isnan(val)) {
+ write_nan();
+ return true;
+ }
+ // must be +-inf
+ if (val > 0) {
+ write_pinf();
+ } else {
+ write_ninf();
+ }
+ return true;
+ }
+
+ Rep hour() const { return static_cast<Rep>(mod((s.count() / 3600), 24)); }
+
+ Rep hour12() const {
+ Rep hour = static_cast<Rep>(mod((s.count() / 3600), 12));
+ return hour <= 0 ? 12 : hour;
+ }
+
+ Rep minute() const { return static_cast<Rep>(mod((s.count() / 60), 60)); }
+ Rep second() const { return static_cast<Rep>(mod(s.count(), 60)); }
+
+ std::tm time() const {
+ auto time = std::tm();
+ time.tm_hour = to_nonnegative_int(hour(), 24);
+ time.tm_min = to_nonnegative_int(minute(), 60);
+ time.tm_sec = to_nonnegative_int(second(), 60);
+ return time;
+ }
+
+ void write_sign() {
+ if (negative) {
+ *out++ = '-';
+ negative = false;
+ }
+ }
+
+ void write(Rep value, int width, pad_type pad = pad_type::unspecified) {
+ write_sign();
+ if (isnan(value)) return write_nan();
+ uint32_or_64_or_128_t<int> n =
+ to_unsigned(to_nonnegative_int(value, max_value<int>()));
+ int num_digits = detail::count_digits(n);
+ if (width > num_digits) {
+ out = detail::write_padding(out, pad, width - num_digits);
+ }
+ out = format_decimal<char_type>(out, n, num_digits).end;
+ }
+
+ void write_nan() { std::copy_n("nan", 3, out); }
+ void write_pinf() { std::copy_n("inf", 3, out); }
+ void write_ninf() { std::copy_n("-inf", 4, out); }
+
+ template <typename Callback, typename... Args>
+ void format_tm(const tm& time, Callback cb, Args... args) {
+ if (isnan(val)) return write_nan();
+ get_locale loc(localized, context.locale());
+ auto w = tm_writer_type(loc, out, time);
+ (w.*cb)(args...);
+ out = w.out();
+ }
+
+ void on_text(const char_type* begin, const char_type* end) {
+ std::copy(begin, end, out);
+ }
+
+ // These are not implemented because durations don't have date information.
+ void on_abbr_weekday() {}
+ void on_full_weekday() {}
+ void on_dec0_weekday(numeric_system) {}
+ void on_dec1_weekday(numeric_system) {}
+ void on_abbr_month() {}
+ void on_full_month() {}
+ void on_datetime(numeric_system) {}
+ void on_loc_date(numeric_system) {}
+ void on_loc_time(numeric_system) {}
+ void on_us_date() {}
+ void on_iso_date() {}
+ void on_utc_offset(numeric_system) {}
+ void on_tz_name() {}
+ void on_year(numeric_system) {}
+ void on_short_year(numeric_system) {}
+ void on_offset_year() {}
+ void on_century(numeric_system) {}
+ void on_iso_week_based_year() {}
+ void on_iso_week_based_short_year() {}
+ void on_dec_month(numeric_system) {}
+ void on_dec0_week_of_year(numeric_system) {}
+ void on_dec1_week_of_year(numeric_system) {}
+ void on_iso_week_of_year(numeric_system) {}
+ void on_day_of_year() {}
+ void on_day_of_month(numeric_system) {}
+ void on_day_of_month_space(numeric_system) {}
+
+ void on_24_hour(numeric_system ns, pad_type pad) {
+ if (handle_nan_inf()) return;
+
+ if (ns == numeric_system::standard) return write(hour(), 2, pad);
+ auto time = tm();
+ time.tm_hour = to_nonnegative_int(hour(), 24);
+ format_tm(time, &tm_writer_type::on_24_hour, ns, pad);
+ }
+
+ void on_12_hour(numeric_system ns, pad_type pad) {
+ if (handle_nan_inf()) return;
+
+ if (ns == numeric_system::standard) return write(hour12(), 2, pad);
+ auto time = tm();
+ time.tm_hour = to_nonnegative_int(hour12(), 12);
+ format_tm(time, &tm_writer_type::on_12_hour, ns, pad);
+ }
+
+ void on_minute(numeric_system ns, pad_type pad) {
+ if (handle_nan_inf()) return;
+
+ if (ns == numeric_system::standard) return write(minute(), 2, pad);
+ auto time = tm();
+ time.tm_min = to_nonnegative_int(minute(), 60);
+ format_tm(time, &tm_writer_type::on_minute, ns, pad);
+ }
+
+ void on_second(numeric_system ns, pad_type pad) {
+ if (handle_nan_inf()) return;
+
+ if (ns == numeric_system::standard) {
+ if (std::is_floating_point<rep>::value) {
+ auto buf = memory_buffer();
+ write_floating_seconds(buf, std::chrono::duration<rep, Period>(val),
+ precision);
+ if (negative) *out++ = '-';
+ if (buf.size() < 2 || buf[1] == '.') {
+ out = detail::write_padding(out, pad);
+ }
+ out = std::copy(buf.begin(), buf.end(), out);
+ } else {
+ write(second(), 2, pad);
+ write_fractional_seconds<char_type>(
+ out, std::chrono::duration<rep, Period>(val), precision);
+ }
+ return;
+ }
+ auto time = tm();
+ time.tm_sec = to_nonnegative_int(second(), 60);
+ format_tm(time, &tm_writer_type::on_second, ns, pad);
+ }
+
+ void on_12_hour_time() {
+ if (handle_nan_inf()) return;
+ format_tm(time(), &tm_writer_type::on_12_hour_time);
+ }
+
+ void on_24_hour_time() {
+ if (handle_nan_inf()) {
+ *out++ = ':';
+ handle_nan_inf();
+ return;
+ }
+
+ write(hour(), 2);
+ *out++ = ':';
+ write(minute(), 2);
+ }
+
+ void on_iso_time() {
+ on_24_hour_time();
+ *out++ = ':';
+ if (handle_nan_inf()) return;
+ on_second(numeric_system::standard, pad_type::unspecified);
+ }
+
+ void on_am_pm() {
+ if (handle_nan_inf()) return;
+ format_tm(time(), &tm_writer_type::on_am_pm);
+ }
+
+ void on_duration_value() {
+ if (handle_nan_inf()) return;
+ write_sign();
+ out = format_duration_value<char_type>(out, val, precision);
+ }
+
+ void on_duration_unit() {
+ out = format_duration_unit<char_type, Period>(out);
+ }
+};
+
+FMT_END_DETAIL_NAMESPACE
+
+#if defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201907
+using weekday = std::chrono::weekday;
+#else
+// A fallback version of weekday.
+class weekday {
+ private:
+ unsigned char value;
+
+ public:
+ weekday() = default;
+ explicit constexpr weekday(unsigned wd) noexcept
+ : value(static_cast<unsigned char>(wd != 7 ? wd : 0)) {}
+ constexpr unsigned c_encoding() const noexcept { return value; }
+};
+
+class year_month_day {};
+#endif
+
+// A rudimentary weekday formatter.
+template <typename Char> struct formatter<weekday, Char> {
+ private:
+ bool localized = false;
+
+ public:
+ FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+ -> decltype(ctx.begin()) {
+ auto begin = ctx.begin(), end = ctx.end();
+ if (begin != end && *begin == 'L') {
+ ++begin;
+ localized = true;
+ }
+ return begin;
+ }
+
+ template <typename FormatContext>
+ auto format(weekday wd, FormatContext& ctx) const -> decltype(ctx.out()) {
+ auto time = std::tm();
+ time.tm_wday = static_cast<int>(wd.c_encoding());
+ detail::get_locale loc(localized, ctx.locale());
+ auto w = detail::tm_writer<decltype(ctx.out()), Char>(loc, ctx.out(), time);
+ w.on_abbr_weekday();
+ return w.out();
+ }
+};
+
+template <typename Rep, typename Period, typename Char>
+struct formatter<std::chrono::duration<Rep, Period>, Char> {
+ private:
+ format_specs<Char> specs;
+ int precision = -1;
+ using arg_ref_type = detail::arg_ref<Char>;
+ arg_ref_type width_ref;
+ arg_ref_type precision_ref;
+ bool localized = false;
+ basic_string_view<Char> format_str;
+ using duration = std::chrono::duration<Rep, Period>;
+
+ using iterator = typename basic_format_parse_context<Char>::iterator;
+ struct parse_range {
+ iterator begin;
+ iterator end;
+ };
+
+ FMT_CONSTEXPR parse_range do_parse(basic_format_parse_context<Char>& ctx) {
+ auto begin = ctx.begin(), end = ctx.end();
+ if (begin == end || *begin == '}') return {begin, begin};
+
+ begin = detail::parse_align(begin, end, specs);
+ if (begin == end) return {begin, begin};
+
+ begin = detail::parse_dynamic_spec(begin, end, specs.width, width_ref, ctx);
+ if (begin == end) return {begin, begin};
+
+ auto checker = detail::chrono_format_checker();
+ if (*begin == '.') {
+ checker.has_precision_integral = !std::is_floating_point<Rep>::value;
+ begin =
+ detail::parse_precision(begin, end, precision, precision_ref, ctx);
+ }
+ if (begin != end && *begin == 'L') {
+ ++begin;
+ localized = true;
+ }
+ end = detail::parse_chrono_format(begin, end, checker);
+ return {begin, end};
+ }
+
+ public:
+ FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+ -> decltype(ctx.begin()) {
+ auto range = do_parse(ctx);
+ format_str = basic_string_view<Char>(
+ &*range.begin, detail::to_unsigned(range.end - range.begin));
+ return range.end;
+ }
+
+ template <typename FormatContext>
+ auto format(const duration& d, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ auto specs_copy = specs;
+ auto precision_copy = precision;
+ auto begin = format_str.begin(), end = format_str.end();
+ // As a possible future optimization, we could avoid extra copying if width
+ // is not specified.
+ basic_memory_buffer<Char> buf;
+ auto out = std::back_inserter(buf);
+ detail::handle_dynamic_spec<detail::width_checker>(specs_copy.width,
+ width_ref, ctx);
+ detail::handle_dynamic_spec<detail::precision_checker>(precision_copy,
+ precision_ref, ctx);
+ if (begin == end || *begin == '}') {
+ out = detail::format_duration_value<Char>(out, d.count(), precision_copy);
+ detail::format_duration_unit<Char, Period>(out);
+ } else {
+ detail::chrono_formatter<FormatContext, decltype(out), Rep, Period> f(
+ ctx, out, d);
+ f.precision = precision_copy;
+ f.localized = localized;
+ detail::parse_chrono_format(begin, end, f);
+ }
+ return detail::write(
+ ctx.out(), basic_string_view<Char>(buf.data(), buf.size()), specs_copy);
+ }
+};
+
+template <typename Char, typename Duration>
+struct formatter<std::chrono::time_point<std::chrono::system_clock, Duration>,
+ Char> : formatter<std::tm, Char> {
+ FMT_CONSTEXPR formatter() {
+ this->format_str = detail::string_literal<Char, '%', 'F', ' ', '%', 'T'>{};
+ }
+
+ template <typename FormatContext>
+ auto format(std::chrono::time_point<std::chrono::system_clock, Duration> val,
+ FormatContext& ctx) const -> decltype(ctx.out()) {
+ using period = typename Duration::period;
+ if (period::num != 1 || period::den != 1 ||
+ std::is_floating_point<typename Duration::rep>::value) {
+ const auto epoch = val.time_since_epoch();
+ auto subsecs = std::chrono::duration_cast<Duration>(
+ epoch - std::chrono::duration_cast<std::chrono::seconds>(epoch));
+
+ if (subsecs.count() < 0) {
+ auto second = std::chrono::seconds(1);
+ if (epoch.count() < ((Duration::min)() + second).count())
+ FMT_THROW(format_error("duration is too small"));
+ subsecs += second;
+ val -= second;
+ }
+
+ return formatter<std::tm, Char>::do_format(
+ gmtime(std::chrono::time_point_cast<std::chrono::seconds>(val)), ctx,
+ &subsecs);
+ }
+
+ return formatter<std::tm, Char>::format(
+ gmtime(std::chrono::time_point_cast<std::chrono::seconds>(val)), ctx);
+ }
+};
+
+#if FMT_USE_LOCAL_TIME
+template <typename Char, typename Duration>
+struct formatter<std::chrono::local_time<Duration>, Char>
+ : formatter<std::tm, Char> {
+ FMT_CONSTEXPR formatter() {
+ this->format_str = detail::string_literal<Char, '%', 'F', ' ', '%', 'T'>{};
+ }
+
+ template <typename FormatContext>
+ auto format(std::chrono::local_time<Duration> val, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ using period = typename Duration::period;
+ if (period::num != 1 || period::den != 1 ||
+ std::is_floating_point<typename Duration::rep>::value) {
+ const auto epoch = val.time_since_epoch();
+ const auto subsecs = std::chrono::duration_cast<Duration>(
+ epoch - std::chrono::duration_cast<std::chrono::seconds>(epoch));
+
+ return formatter<std::tm, Char>::do_format(
+ localtime(std::chrono::time_point_cast<std::chrono::seconds>(val)),
+ ctx, &subsecs);
+ }
+
+ return formatter<std::tm, Char>::format(
+ localtime(std::chrono::time_point_cast<std::chrono::seconds>(val)),
+ ctx);
+ }
+};
+#endif
+
+#if FMT_USE_UTC_TIME
+template <typename Char, typename Duration>
+struct formatter<std::chrono::time_point<std::chrono::utc_clock, Duration>,
+ Char>
+ : formatter<std::chrono::time_point<std::chrono::system_clock, Duration>,
+ Char> {
+ template <typename FormatContext>
+ auto format(std::chrono::time_point<std::chrono::utc_clock, Duration> val,
+ FormatContext& ctx) const -> decltype(ctx.out()) {
+ return formatter<
+ std::chrono::time_point<std::chrono::system_clock, Duration>,
+ Char>::format(std::chrono::utc_clock::to_sys(val), ctx);
+ }
+};
+#endif
+
+template <typename Char> struct formatter<std::tm, Char> {
+ private:
+ format_specs<Char> specs;
+ detail::arg_ref<Char> width_ref;
+
+ protected:
+ basic_string_view<Char> format_str;
+
+ FMT_CONSTEXPR auto do_parse(basic_format_parse_context<Char>& ctx)
+ -> decltype(ctx.begin()) {
+ auto begin = ctx.begin(), end = ctx.end();
+ if (begin == end || *begin == '}') return begin;
+
+ begin = detail::parse_align(begin, end, specs);
+ if (begin == end) return end;
+
+ begin = detail::parse_dynamic_spec(begin, end, specs.width, width_ref, ctx);
+ if (begin == end) return end;
+
+ end = detail::parse_chrono_format(begin, end, detail::tm_format_checker());
+ // Replace default format_str only if the new spec is not empty.
+ if (end != begin) format_str = {begin, detail::to_unsigned(end - begin)};
+ return end;
+ }
+
+ template <typename FormatContext, typename Duration>
+ auto do_format(const std::tm& tm, FormatContext& ctx,
+ const Duration* subsecs) const -> decltype(ctx.out()) {
+ auto specs_copy = specs;
+ basic_memory_buffer<Char> buf;
+ auto out = std::back_inserter(buf);
+ detail::handle_dynamic_spec<detail::width_checker>(specs_copy.width,
+ width_ref, ctx);
+
+ const auto loc_ref = ctx.locale();
+ detail::get_locale loc(static_cast<bool>(loc_ref), loc_ref);
+ auto w =
+ detail::tm_writer<decltype(out), Char, Duration>(loc, out, tm, subsecs);
+ detail::parse_chrono_format(format_str.begin(), format_str.end(), w);
+ return detail::write(
+ ctx.out(), basic_string_view<Char>(buf.data(), buf.size()), specs_copy);
+ }
+
+ public:
+ FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+ -> decltype(ctx.begin()) {
+ return this->do_parse(ctx);
+ }
+
+ template <typename FormatContext>
+ auto format(const std::tm& tm, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ return do_format<FormatContext, std::chrono::seconds>(tm, ctx, nullptr);
+ }
+};
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_CHRONO_H_
diff --git a/contrib/fmt/include/fmt/color.h b/contrib/fmt/include/fmt/color.h
new file mode 100644
index 0000000..d175448
--- /dev/null
+++ b/contrib/fmt/include/fmt/color.h
@@ -0,0 +1,633 @@
+// Formatting library for C++ - color support
+//
+// Copyright (c) 2018 - present, Victor Zverovich and fmt contributors
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_COLOR_H_
+#define FMT_COLOR_H_
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+FMT_BEGIN_EXPORT
+
+enum class color : uint32_t {
+ alice_blue = 0xF0F8FF, // rgb(240,248,255)
+ antique_white = 0xFAEBD7, // rgb(250,235,215)
+ aqua = 0x00FFFF, // rgb(0,255,255)
+ aquamarine = 0x7FFFD4, // rgb(127,255,212)
+ azure = 0xF0FFFF, // rgb(240,255,255)
+ beige = 0xF5F5DC, // rgb(245,245,220)
+ bisque = 0xFFE4C4, // rgb(255,228,196)
+ black = 0x000000, // rgb(0,0,0)
+ blanched_almond = 0xFFEBCD, // rgb(255,235,205)
+ blue = 0x0000FF, // rgb(0,0,255)
+ blue_violet = 0x8A2BE2, // rgb(138,43,226)
+ brown = 0xA52A2A, // rgb(165,42,42)
+ burly_wood = 0xDEB887, // rgb(222,184,135)
+ cadet_blue = 0x5F9EA0, // rgb(95,158,160)
+ chartreuse = 0x7FFF00, // rgb(127,255,0)
+ chocolate = 0xD2691E, // rgb(210,105,30)
+ coral = 0xFF7F50, // rgb(255,127,80)
+ cornflower_blue = 0x6495ED, // rgb(100,149,237)
+ cornsilk = 0xFFF8DC, // rgb(255,248,220)
+ crimson = 0xDC143C, // rgb(220,20,60)
+ cyan = 0x00FFFF, // rgb(0,255,255)
+ dark_blue = 0x00008B, // rgb(0,0,139)
+ dark_cyan = 0x008B8B, // rgb(0,139,139)
+ dark_golden_rod = 0xB8860B, // rgb(184,134,11)
+ dark_gray = 0xA9A9A9, // rgb(169,169,169)
+ dark_green = 0x006400, // rgb(0,100,0)
+ dark_khaki = 0xBDB76B, // rgb(189,183,107)
+ dark_magenta = 0x8B008B, // rgb(139,0,139)
+ dark_olive_green = 0x556B2F, // rgb(85,107,47)
+ dark_orange = 0xFF8C00, // rgb(255,140,0)
+ dark_orchid = 0x9932CC, // rgb(153,50,204)
+ dark_red = 0x8B0000, // rgb(139,0,0)
+ dark_salmon = 0xE9967A, // rgb(233,150,122)
+ dark_sea_green = 0x8FBC8F, // rgb(143,188,143)
+ dark_slate_blue = 0x483D8B, // rgb(72,61,139)
+ dark_slate_gray = 0x2F4F4F, // rgb(47,79,79)
+ dark_turquoise = 0x00CED1, // rgb(0,206,209)
+ dark_violet = 0x9400D3, // rgb(148,0,211)
+ deep_pink = 0xFF1493, // rgb(255,20,147)
+ deep_sky_blue = 0x00BFFF, // rgb(0,191,255)
+ dim_gray = 0x696969, // rgb(105,105,105)
+ dodger_blue = 0x1E90FF, // rgb(30,144,255)
+ fire_brick = 0xB22222, // rgb(178,34,34)
+ floral_white = 0xFFFAF0, // rgb(255,250,240)
+ forest_green = 0x228B22, // rgb(34,139,34)
+ fuchsia = 0xFF00FF, // rgb(255,0,255)
+ gainsboro = 0xDCDCDC, // rgb(220,220,220)
+ ghost_white = 0xF8F8FF, // rgb(248,248,255)
+ gold = 0xFFD700, // rgb(255,215,0)
+ golden_rod = 0xDAA520, // rgb(218,165,32)
+ gray = 0x808080, // rgb(128,128,128)
+ green = 0x008000, // rgb(0,128,0)
+ green_yellow = 0xADFF2F, // rgb(173,255,47)
+ honey_dew = 0xF0FFF0, // rgb(240,255,240)
+ hot_pink = 0xFF69B4, // rgb(255,105,180)
+ indian_red = 0xCD5C5C, // rgb(205,92,92)
+ indigo = 0x4B0082, // rgb(75,0,130)
+ ivory = 0xFFFFF0, // rgb(255,255,240)
+ khaki = 0xF0E68C, // rgb(240,230,140)
+ lavender = 0xE6E6FA, // rgb(230,230,250)
+ lavender_blush = 0xFFF0F5, // rgb(255,240,245)
+ lawn_green = 0x7CFC00, // rgb(124,252,0)
+ lemon_chiffon = 0xFFFACD, // rgb(255,250,205)
+ light_blue = 0xADD8E6, // rgb(173,216,230)
+ light_coral = 0xF08080, // rgb(240,128,128)
+ light_cyan = 0xE0FFFF, // rgb(224,255,255)
+ light_golden_rod_yellow = 0xFAFAD2, // rgb(250,250,210)
+ light_gray = 0xD3D3D3, // rgb(211,211,211)
+ light_green = 0x90EE90, // rgb(144,238,144)
+ light_pink = 0xFFB6C1, // rgb(255,182,193)
+ light_salmon = 0xFFA07A, // rgb(255,160,122)
+ light_sea_green = 0x20B2AA, // rgb(32,178,170)
+ light_sky_blue = 0x87CEFA, // rgb(135,206,250)
+ light_slate_gray = 0x778899, // rgb(119,136,153)
+ light_steel_blue = 0xB0C4DE, // rgb(176,196,222)
+ light_yellow = 0xFFFFE0, // rgb(255,255,224)
+ lime = 0x00FF00, // rgb(0,255,0)
+ lime_green = 0x32CD32, // rgb(50,205,50)
+ linen = 0xFAF0E6, // rgb(250,240,230)
+ magenta = 0xFF00FF, // rgb(255,0,255)
+ maroon = 0x800000, // rgb(128,0,0)
+ medium_aquamarine = 0x66CDAA, // rgb(102,205,170)
+ medium_blue = 0x0000CD, // rgb(0,0,205)
+ medium_orchid = 0xBA55D3, // rgb(186,85,211)
+ medium_purple = 0x9370DB, // rgb(147,112,219)
+ medium_sea_green = 0x3CB371, // rgb(60,179,113)
+ medium_slate_blue = 0x7B68EE, // rgb(123,104,238)
+ medium_spring_green = 0x00FA9A, // rgb(0,250,154)
+ medium_turquoise = 0x48D1CC, // rgb(72,209,204)
+ medium_violet_red = 0xC71585, // rgb(199,21,133)
+ midnight_blue = 0x191970, // rgb(25,25,112)
+ mint_cream = 0xF5FFFA, // rgb(245,255,250)
+ misty_rose = 0xFFE4E1, // rgb(255,228,225)
+ moccasin = 0xFFE4B5, // rgb(255,228,181)
+ navajo_white = 0xFFDEAD, // rgb(255,222,173)
+ navy = 0x000080, // rgb(0,0,128)
+ old_lace = 0xFDF5E6, // rgb(253,245,230)
+ olive = 0x808000, // rgb(128,128,0)
+ olive_drab = 0x6B8E23, // rgb(107,142,35)
+ orange = 0xFFA500, // rgb(255,165,0)
+ orange_red = 0xFF4500, // rgb(255,69,0)
+ orchid = 0xDA70D6, // rgb(218,112,214)
+ pale_golden_rod = 0xEEE8AA, // rgb(238,232,170)
+ pale_green = 0x98FB98, // rgb(152,251,152)
+ pale_turquoise = 0xAFEEEE, // rgb(175,238,238)
+ pale_violet_red = 0xDB7093, // rgb(219,112,147)
+ papaya_whip = 0xFFEFD5, // rgb(255,239,213)
+ peach_puff = 0xFFDAB9, // rgb(255,218,185)
+ peru = 0xCD853F, // rgb(205,133,63)
+ pink = 0xFFC0CB, // rgb(255,192,203)
+ plum = 0xDDA0DD, // rgb(221,160,221)
+ powder_blue = 0xB0E0E6, // rgb(176,224,230)
+ purple = 0x800080, // rgb(128,0,128)
+ rebecca_purple = 0x663399, // rgb(102,51,153)
+ red = 0xFF0000, // rgb(255,0,0)
+ rosy_brown = 0xBC8F8F, // rgb(188,143,143)
+ royal_blue = 0x4169E1, // rgb(65,105,225)
+ saddle_brown = 0x8B4513, // rgb(139,69,19)
+ salmon = 0xFA8072, // rgb(250,128,114)
+ sandy_brown = 0xF4A460, // rgb(244,164,96)
+ sea_green = 0x2E8B57, // rgb(46,139,87)
+ sea_shell = 0xFFF5EE, // rgb(255,245,238)
+ sienna = 0xA0522D, // rgb(160,82,45)
+ silver = 0xC0C0C0, // rgb(192,192,192)
+ sky_blue = 0x87CEEB, // rgb(135,206,235)
+ slate_blue = 0x6A5ACD, // rgb(106,90,205)
+ slate_gray = 0x708090, // rgb(112,128,144)
+ snow = 0xFFFAFA, // rgb(255,250,250)
+ spring_green = 0x00FF7F, // rgb(0,255,127)
+ steel_blue = 0x4682B4, // rgb(70,130,180)
+ tan = 0xD2B48C, // rgb(210,180,140)
+ teal = 0x008080, // rgb(0,128,128)
+ thistle = 0xD8BFD8, // rgb(216,191,216)
+ tomato = 0xFF6347, // rgb(255,99,71)
+ turquoise = 0x40E0D0, // rgb(64,224,208)
+ violet = 0xEE82EE, // rgb(238,130,238)
+ wheat = 0xF5DEB3, // rgb(245,222,179)
+ white = 0xFFFFFF, // rgb(255,255,255)
+ white_smoke = 0xF5F5F5, // rgb(245,245,245)
+ yellow = 0xFFFF00, // rgb(255,255,0)
+ yellow_green = 0x9ACD32 // rgb(154,205,50)
+}; // enum class color
+
+enum class terminal_color : uint8_t {
+ black = 30,
+ red,
+ green,
+ yellow,
+ blue,
+ magenta,
+ cyan,
+ white,
+ bright_black = 90,
+ bright_red,
+ bright_green,
+ bright_yellow,
+ bright_blue,
+ bright_magenta,
+ bright_cyan,
+ bright_white
+};
+
+enum class emphasis : uint8_t {
+ bold = 1,
+ faint = 1 << 1,
+ italic = 1 << 2,
+ underline = 1 << 3,
+ blink = 1 << 4,
+ reverse = 1 << 5,
+ conceal = 1 << 6,
+ strikethrough = 1 << 7,
+};
+
+// rgb is a struct for red, green and blue colors.
+// Using the name "rgb" makes some editors show the color in a tooltip.
+struct rgb {
+ FMT_CONSTEXPR rgb() : r(0), g(0), b(0) {}
+ FMT_CONSTEXPR rgb(uint8_t r_, uint8_t g_, uint8_t b_) : r(r_), g(g_), b(b_) {}
+ FMT_CONSTEXPR rgb(uint32_t hex)
+ : r((hex >> 16) & 0xFF), g((hex >> 8) & 0xFF), b(hex & 0xFF) {}
+ FMT_CONSTEXPR rgb(color hex)
+ : r((uint32_t(hex) >> 16) & 0xFF),
+ g((uint32_t(hex) >> 8) & 0xFF),
+ b(uint32_t(hex) & 0xFF) {}
+ uint8_t r;
+ uint8_t g;
+ uint8_t b;
+};
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+// color is a struct of either a rgb color or a terminal color.
+struct color_type {
+ FMT_CONSTEXPR color_type() noexcept : is_rgb(), value{} {}
+ FMT_CONSTEXPR color_type(color rgb_color) noexcept : is_rgb(true), value{} {
+ value.rgb_color = static_cast<uint32_t>(rgb_color);
+ }
+ FMT_CONSTEXPR color_type(rgb rgb_color) noexcept : is_rgb(true), value{} {
+ value.rgb_color = (static_cast<uint32_t>(rgb_color.r) << 16) |
+ (static_cast<uint32_t>(rgb_color.g) << 8) | rgb_color.b;
+ }
+ FMT_CONSTEXPR color_type(terminal_color term_color) noexcept
+ : is_rgb(), value{} {
+ value.term_color = static_cast<uint8_t>(term_color);
+ }
+ bool is_rgb;
+ union color_union {
+ uint8_t term_color;
+ uint32_t rgb_color;
+ } value;
+};
+
+FMT_END_DETAIL_NAMESPACE
+
+/** A text style consisting of foreground and background colors and emphasis. */
+class text_style {
+ public:
+ FMT_CONSTEXPR text_style(emphasis em = emphasis()) noexcept
+ : set_foreground_color(), set_background_color(), ems(em) {}
+
+ FMT_CONSTEXPR text_style& operator|=(const text_style& rhs) {
+ if (!set_foreground_color) {
+ set_foreground_color = rhs.set_foreground_color;
+ foreground_color = rhs.foreground_color;
+ } else if (rhs.set_foreground_color) {
+ if (!foreground_color.is_rgb || !rhs.foreground_color.is_rgb)
+ FMT_THROW(format_error("can't OR a terminal color"));
+ foreground_color.value.rgb_color |= rhs.foreground_color.value.rgb_color;
+ }
+
+ if (!set_background_color) {
+ set_background_color = rhs.set_background_color;
+ background_color = rhs.background_color;
+ } else if (rhs.set_background_color) {
+ if (!background_color.is_rgb || !rhs.background_color.is_rgb)
+ FMT_THROW(format_error("can't OR a terminal color"));
+ background_color.value.rgb_color |= rhs.background_color.value.rgb_color;
+ }
+
+ ems = static_cast<emphasis>(static_cast<uint8_t>(ems) |
+ static_cast<uint8_t>(rhs.ems));
+ return *this;
+ }
+
+ friend FMT_CONSTEXPR text_style operator|(text_style lhs,
+ const text_style& rhs) {
+ return lhs |= rhs;
+ }
+
+ FMT_CONSTEXPR bool has_foreground() const noexcept {
+ return set_foreground_color;
+ }
+ FMT_CONSTEXPR bool has_background() const noexcept {
+ return set_background_color;
+ }
+ FMT_CONSTEXPR bool has_emphasis() const noexcept {
+ return static_cast<uint8_t>(ems) != 0;
+ }
+ FMT_CONSTEXPR detail::color_type get_foreground() const noexcept {
+ FMT_ASSERT(has_foreground(), "no foreground specified for this style");
+ return foreground_color;
+ }
+ FMT_CONSTEXPR detail::color_type get_background() const noexcept {
+ FMT_ASSERT(has_background(), "no background specified for this style");
+ return background_color;
+ }
+ FMT_CONSTEXPR emphasis get_emphasis() const noexcept {
+ FMT_ASSERT(has_emphasis(), "no emphasis specified for this style");
+ return ems;
+ }
+
+ private:
+ FMT_CONSTEXPR text_style(bool is_foreground,
+ detail::color_type text_color) noexcept
+ : set_foreground_color(), set_background_color(), ems() {
+ if (is_foreground) {
+ foreground_color = text_color;
+ set_foreground_color = true;
+ } else {
+ background_color = text_color;
+ set_background_color = true;
+ }
+ }
+
+ friend FMT_CONSTEXPR text_style fg(detail::color_type foreground) noexcept;
+
+ friend FMT_CONSTEXPR text_style bg(detail::color_type background) noexcept;
+
+ detail::color_type foreground_color;
+ detail::color_type background_color;
+ bool set_foreground_color;
+ bool set_background_color;
+ emphasis ems;
+};
+
+/** Creates a text style from the foreground (text) color. */
+FMT_CONSTEXPR inline text_style fg(detail::color_type foreground) noexcept {
+ return text_style(true, foreground);
+}
+
+/** Creates a text style from the background color. */
+FMT_CONSTEXPR inline text_style bg(detail::color_type background) noexcept {
+ return text_style(false, background);
+}
+
+FMT_CONSTEXPR inline text_style operator|(emphasis lhs, emphasis rhs) noexcept {
+ return text_style(lhs) | rhs;
+}
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+template <typename Char> struct ansi_color_escape {
+ FMT_CONSTEXPR ansi_color_escape(detail::color_type text_color,
+ const char* esc) noexcept {
+ // If we have a terminal color, we need to output another escape code
+ // sequence.
+ if (!text_color.is_rgb) {
+ bool is_background = esc == string_view("\x1b[48;2;");
+ uint32_t value = text_color.value.term_color;
+ // Background ASCII codes are the same as the foreground ones but with
+ // 10 more.
+ if (is_background) value += 10u;
+
+ size_t index = 0;
+ buffer[index++] = static_cast<Char>('\x1b');
+ buffer[index++] = static_cast<Char>('[');
+
+ if (value >= 100u) {
+ buffer[index++] = static_cast<Char>('1');
+ value %= 100u;
+ }
+ buffer[index++] = static_cast<Char>('0' + value / 10u);
+ buffer[index++] = static_cast<Char>('0' + value % 10u);
+
+ buffer[index++] = static_cast<Char>('m');
+ buffer[index++] = static_cast<Char>('\0');
+ return;
+ }
+
+ for (int i = 0; i < 7; i++) {
+ buffer[i] = static_cast<Char>(esc[i]);
+ }
+ rgb color(text_color.value.rgb_color);
+ to_esc(color.r, buffer + 7, ';');
+ to_esc(color.g, buffer + 11, ';');
+ to_esc(color.b, buffer + 15, 'm');
+ buffer[19] = static_cast<Char>(0);
+ }
+ FMT_CONSTEXPR ansi_color_escape(emphasis em) noexcept {
+ uint8_t em_codes[num_emphases] = {};
+ if (has_emphasis(em, emphasis::bold)) em_codes[0] = 1;
+ if (has_emphasis(em, emphasis::faint)) em_codes[1] = 2;
+ if (has_emphasis(em, emphasis::italic)) em_codes[2] = 3;
+ if (has_emphasis(em, emphasis::underline)) em_codes[3] = 4;
+ if (has_emphasis(em, emphasis::blink)) em_codes[4] = 5;
+ if (has_emphasis(em, emphasis::reverse)) em_codes[5] = 7;
+ if (has_emphasis(em, emphasis::conceal)) em_codes[6] = 8;
+ if (has_emphasis(em, emphasis::strikethrough)) em_codes[7] = 9;
+
+ size_t index = 0;
+ for (size_t i = 0; i < num_emphases; ++i) {
+ if (!em_codes[i]) continue;
+ buffer[index++] = static_cast<Char>('\x1b');
+ buffer[index++] = static_cast<Char>('[');
+ buffer[index++] = static_cast<Char>('0' + em_codes[i]);
+ buffer[index++] = static_cast<Char>('m');
+ }
+ buffer[index++] = static_cast<Char>(0);
+ }
+ FMT_CONSTEXPR operator const Char*() const noexcept { return buffer; }
+
+ FMT_CONSTEXPR const Char* begin() const noexcept { return buffer; }
+ FMT_CONSTEXPR_CHAR_TRAITS const Char* end() const noexcept {
+ return buffer + std::char_traits<Char>::length(buffer);
+ }
+
+ private:
+ static constexpr size_t num_emphases = 8;
+ Char buffer[7u + 3u * num_emphases + 1u];
+
+ static FMT_CONSTEXPR void to_esc(uint8_t c, Char* out,
+ char delimiter) noexcept {
+ out[0] = static_cast<Char>('0' + c / 100);
+ out[1] = static_cast<Char>('0' + c / 10 % 10);
+ out[2] = static_cast<Char>('0' + c % 10);
+ out[3] = static_cast<Char>(delimiter);
+ }
+ static FMT_CONSTEXPR bool has_emphasis(emphasis em, emphasis mask) noexcept {
+ return static_cast<uint8_t>(em) & static_cast<uint8_t>(mask);
+ }
+};
+
+template <typename Char>
+FMT_CONSTEXPR ansi_color_escape<Char> make_foreground_color(
+ detail::color_type foreground) noexcept {
+ return ansi_color_escape<Char>(foreground, "\x1b[38;2;");
+}
+
+template <typename Char>
+FMT_CONSTEXPR ansi_color_escape<Char> make_background_color(
+ detail::color_type background) noexcept {
+ return ansi_color_escape<Char>(background, "\x1b[48;2;");
+}
+
+template <typename Char>
+FMT_CONSTEXPR ansi_color_escape<Char> make_emphasis(emphasis em) noexcept {
+ return ansi_color_escape<Char>(em);
+}
+
+template <typename Char> inline void reset_color(buffer<Char>& buffer) {
+ auto reset_color = string_view("\x1b[0m");
+ buffer.append(reset_color.begin(), reset_color.end());
+}
+
+template <typename T> struct styled_arg {
+ const T& value;
+ text_style style;
+};
+
+template <typename Char>
+void vformat_to(buffer<Char>& buf, const text_style& ts,
+ basic_string_view<Char> format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args) {
+ bool has_style = false;
+ if (ts.has_emphasis()) {
+ has_style = true;
+ auto emphasis = detail::make_emphasis<Char>(ts.get_emphasis());
+ buf.append(emphasis.begin(), emphasis.end());
+ }
+ if (ts.has_foreground()) {
+ has_style = true;
+ auto foreground = detail::make_foreground_color<Char>(ts.get_foreground());
+ buf.append(foreground.begin(), foreground.end());
+ }
+ if (ts.has_background()) {
+ has_style = true;
+ auto background = detail::make_background_color<Char>(ts.get_background());
+ buf.append(background.begin(), background.end());
+ }
+ detail::vformat_to(buf, format_str, args, {});
+ if (has_style) detail::reset_color<Char>(buf);
+}
+
+FMT_END_DETAIL_NAMESPACE
+
+inline void vprint(std::FILE* f, const text_style& ts, string_view fmt,
+ format_args args) {
+ // Legacy wide streams are not supported.
+ auto buf = memory_buffer();
+ detail::vformat_to(buf, ts, fmt, args);
+ if (detail::is_utf8()) {
+ detail::print(f, string_view(buf.begin(), buf.size()));
+ return;
+ }
+ buf.push_back('\0');
+ int result = std::fputs(buf.data(), f);
+ if (result < 0)
+ FMT_THROW(system_error(errno, FMT_STRING("cannot write to file")));
+}
+
+/**
+ \rst
+ Formats a string and prints it to the specified file stream using ANSI
+ escape sequences to specify text formatting.
+
+ **Example**::
+
+ fmt::print(fmt::emphasis::bold | fg(fmt::color::red),
+ "Elapsed time: {0:.2f} seconds", 1.23);
+ \endrst
+ */
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_string<S>::value)>
+void print(std::FILE* f, const text_style& ts, const S& format_str,
+ const Args&... args) {
+ vprint(f, ts, format_str,
+ fmt::make_format_args<buffer_context<char_t<S>>>(args...));
+}
+
+/**
+ \rst
+ Formats a string and prints it to stdout using ANSI escape sequences to
+ specify text formatting.
+
+ **Example**::
+
+ fmt::print(fmt::emphasis::bold | fg(fmt::color::red),
+ "Elapsed time: {0:.2f} seconds", 1.23);
+ \endrst
+ */
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_string<S>::value)>
+void print(const text_style& ts, const S& format_str, const Args&... args) {
+ return print(stdout, ts, format_str, args...);
+}
+
+template <typename S, typename Char = char_t<S>>
+inline std::basic_string<Char> vformat(
+ const text_style& ts, const S& format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args) {
+ basic_memory_buffer<Char> buf;
+ detail::vformat_to(buf, ts, detail::to_string_view(format_str), args);
+ return fmt::to_string(buf);
+}
+
+/**
+ \rst
+ Formats arguments and returns the result as a string using ANSI
+ escape sequences to specify text formatting.
+
+ **Example**::
+
+ #include <fmt/color.h>
+ std::string message = fmt::format(fmt::emphasis::bold | fg(fmt::color::red),
+ "The answer is {}", 42);
+ \endrst
+*/
+template <typename S, typename... Args, typename Char = char_t<S>>
+inline std::basic_string<Char> format(const text_style& ts, const S& format_str,
+ const Args&... args) {
+ return fmt::vformat(ts, detail::to_string_view(format_str),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+/**
+ Formats a string with the given text_style and writes the output to ``out``.
+ */
+template <typename OutputIt, typename Char,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value)>
+OutputIt vformat_to(
+ OutputIt out, const text_style& ts, basic_string_view<Char> format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args) {
+ auto&& buf = detail::get_buffer<Char>(out);
+ detail::vformat_to(buf, ts, format_str, args);
+ return detail::get_iterator(buf, out);
+}
+
+/**
+ \rst
+ Formats arguments with the given text_style, writes the result to the output
+ iterator ``out`` and returns the iterator past the end of the output range.
+
+ **Example**::
+
+ std::vector<char> out;
+ fmt::format_to(std::back_inserter(out),
+ fmt::emphasis::bold | fg(fmt::color::red), "{}", 42);
+ \endrst
+*/
+template <typename OutputIt, typename S, typename... Args,
+ bool enable = detail::is_output_iterator<OutputIt, char_t<S>>::value&&
+ detail::is_string<S>::value>
+inline auto format_to(OutputIt out, const text_style& ts, const S& format_str,
+ Args&&... args) ->
+ typename std::enable_if<enable, OutputIt>::type {
+ return vformat_to(out, ts, detail::to_string_view(format_str),
+ fmt::make_format_args<buffer_context<char_t<S>>>(args...));
+}
+
+template <typename T, typename Char>
+struct formatter<detail::styled_arg<T>, Char> : formatter<T, Char> {
+ template <typename FormatContext>
+ auto format(const detail::styled_arg<T>& arg, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ const auto& ts = arg.style;
+ const auto& value = arg.value;
+ auto out = ctx.out();
+
+ bool has_style = false;
+ if (ts.has_emphasis()) {
+ has_style = true;
+ auto emphasis = detail::make_emphasis<Char>(ts.get_emphasis());
+ out = std::copy(emphasis.begin(), emphasis.end(), out);
+ }
+ if (ts.has_foreground()) {
+ has_style = true;
+ auto foreground =
+ detail::make_foreground_color<Char>(ts.get_foreground());
+ out = std::copy(foreground.begin(), foreground.end(), out);
+ }
+ if (ts.has_background()) {
+ has_style = true;
+ auto background =
+ detail::make_background_color<Char>(ts.get_background());
+ out = std::copy(background.begin(), background.end(), out);
+ }
+ out = formatter<T, Char>::format(value, ctx);
+ if (has_style) {
+ auto reset_color = string_view("\x1b[0m");
+ out = std::copy(reset_color.begin(), reset_color.end(), out);
+ }
+ return out;
+ }
+};
+
+/**
+ \rst
+ Returns an argument that will be formatted using ANSI escape sequences,
+ to be used in a formatting function.
+
+ **Example**::
+
+ fmt::print("Elapsed time: {0:.2f} seconds",
+ fmt::styled(1.23, fmt::fg(fmt::color::green) |
+ fmt::bg(fmt::color::blue)));
+ \endrst
+ */
+template <typename T>
+FMT_CONSTEXPR auto styled(const T& value, text_style ts)
+ -> detail::styled_arg<remove_cvref_t<T>> {
+ return detail::styled_arg<remove_cvref_t<T>>{value, ts};
+}
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_COLOR_H_
diff --git a/contrib/fmt/include/fmt/compile.h b/contrib/fmt/include/fmt/compile.h
new file mode 100644
index 0000000..94e13c0
--- /dev/null
+++ b/contrib/fmt/include/fmt/compile.h
@@ -0,0 +1,607 @@
+// Formatting library for C++ - experimental format string compilation
+//
+// Copyright (c) 2012 - present, Victor Zverovich and fmt contributors
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_COMPILE_H_
+#define FMT_COMPILE_H_
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+template <typename Char, typename InputIt>
+FMT_CONSTEXPR inline counting_iterator copy_str(InputIt begin, InputIt end,
+ counting_iterator it) {
+ return it + (end - begin);
+}
+
+template <typename OutputIt> class truncating_iterator_base {
+ protected:
+ OutputIt out_;
+ size_t limit_;
+ size_t count_ = 0;
+
+ truncating_iterator_base() : out_(), limit_(0) {}
+
+ truncating_iterator_base(OutputIt out, size_t limit)
+ : out_(out), limit_(limit) {}
+
+ public:
+ using iterator_category = std::output_iterator_tag;
+ using value_type = typename std::iterator_traits<OutputIt>::value_type;
+ using difference_type = std::ptrdiff_t;
+ using pointer = void;
+ using reference = void;
+ FMT_UNCHECKED_ITERATOR(truncating_iterator_base);
+
+ OutputIt base() const { return out_; }
+ size_t count() const { return count_; }
+};
+
+// An output iterator that truncates the output and counts the number of objects
+// written to it.
+template <typename OutputIt,
+ typename Enable = typename std::is_void<
+ typename std::iterator_traits<OutputIt>::value_type>::type>
+class truncating_iterator;
+
+template <typename OutputIt>
+class truncating_iterator<OutputIt, std::false_type>
+ : public truncating_iterator_base<OutputIt> {
+ mutable typename truncating_iterator_base<OutputIt>::value_type blackhole_;
+
+ public:
+ using value_type = typename truncating_iterator_base<OutputIt>::value_type;
+
+ truncating_iterator() = default;
+
+ truncating_iterator(OutputIt out, size_t limit)
+ : truncating_iterator_base<OutputIt>(out, limit) {}
+
+ truncating_iterator& operator++() {
+ if (this->count_++ < this->limit_) ++this->out_;
+ return *this;
+ }
+
+ truncating_iterator operator++(int) {
+ auto it = *this;
+ ++*this;
+ return it;
+ }
+
+ value_type& operator*() const {
+ return this->count_ < this->limit_ ? *this->out_ : blackhole_;
+ }
+};
+
+template <typename OutputIt>
+class truncating_iterator<OutputIt, std::true_type>
+ : public truncating_iterator_base<OutputIt> {
+ public:
+ truncating_iterator() = default;
+
+ truncating_iterator(OutputIt out, size_t limit)
+ : truncating_iterator_base<OutputIt>(out, limit) {}
+
+ template <typename T> truncating_iterator& operator=(T val) {
+ if (this->count_++ < this->limit_) *this->out_++ = val;
+ return *this;
+ }
+
+ truncating_iterator& operator++() { return *this; }
+ truncating_iterator& operator++(int) { return *this; }
+ truncating_iterator& operator*() { return *this; }
+};
+
+// A compile-time string which is compiled into fast formatting code.
+class compiled_string {};
+
+template <typename S>
+struct is_compiled_string : std::is_base_of<compiled_string, S> {};
+
+/**
+ \rst
+ Converts a string literal *s* into a format string that will be parsed at
+ compile time and converted into efficient formatting code. Requires C++17
+ ``constexpr if`` compiler support.
+
+ **Example**::
+
+ // Converts 42 into std::string using the most efficient method and no
+ // runtime format string processing.
+ std::string s = fmt::format(FMT_COMPILE("{}"), 42);
+ \endrst
+ */
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+# define FMT_COMPILE(s) \
+ FMT_STRING_IMPL(s, fmt::detail::compiled_string, explicit)
+#else
+# define FMT_COMPILE(s) FMT_STRING(s)
+#endif
+
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <typename Char, size_t N,
+ fmt::detail_exported::fixed_string<Char, N> Str>
+struct udl_compiled_string : compiled_string {
+ using char_type = Char;
+ explicit constexpr operator basic_string_view<char_type>() const {
+ return {Str.data, N - 1};
+ }
+};
+#endif
+
+template <typename T, typename... Tail>
+const T& first(const T& value, const Tail&...) {
+ return value;
+}
+
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+template <typename... Args> struct type_list {};
+
+// Returns a reference to the argument at index N from [first, rest...].
+template <int N, typename T, typename... Args>
+constexpr const auto& get([[maybe_unused]] const T& first,
+ [[maybe_unused]] const Args&... rest) {
+ static_assert(N < 1 + sizeof...(Args), "index is out of bounds");
+ if constexpr (N == 0)
+ return first;
+ else
+ return detail::get<N - 1>(rest...);
+}
+
+template <typename Char, typename... Args>
+constexpr int get_arg_index_by_name(basic_string_view<Char> name,
+ type_list<Args...>) {
+ return get_arg_index_by_name<Args...>(name);
+}
+
+template <int N, typename> struct get_type_impl;
+
+template <int N, typename... Args> struct get_type_impl<N, type_list<Args...>> {
+ using type =
+ remove_cvref_t<decltype(detail::get<N>(std::declval<Args>()...))>;
+};
+
+template <int N, typename T>
+using get_type = typename get_type_impl<N, T>::type;
+
+template <typename T> struct is_compiled_format : std::false_type {};
+
+template <typename Char> struct text {
+ basic_string_view<Char> data;
+ using char_type = Char;
+
+ template <typename OutputIt, typename... Args>
+ constexpr OutputIt format(OutputIt out, const Args&...) const {
+ return write<Char>(out, data);
+ }
+};
+
+template <typename Char>
+struct is_compiled_format<text<Char>> : std::true_type {};
+
+template <typename Char>
+constexpr text<Char> make_text(basic_string_view<Char> s, size_t pos,
+ size_t size) {
+ return {{&s[pos], size}};
+}
+
+template <typename Char> struct code_unit {
+ Char value;
+ using char_type = Char;
+
+ template <typename OutputIt, typename... Args>
+ constexpr OutputIt format(OutputIt out, const Args&...) const {
+ return write<Char>(out, value);
+ }
+};
+
+// This ensures that the argument type is convertible to `const T&`.
+template <typename T, int N, typename... Args>
+constexpr const T& get_arg_checked(const Args&... args) {
+ const auto& arg = detail::get<N>(args...);
+ if constexpr (detail::is_named_arg<remove_cvref_t<decltype(arg)>>()) {
+ return arg.value;
+ } else {
+ return arg;
+ }
+}
+
+template <typename Char>
+struct is_compiled_format<code_unit<Char>> : std::true_type {};
+
+// A replacement field that refers to argument N.
+template <typename Char, typename T, int N> struct field {
+ using char_type = Char;
+
+ template <typename OutputIt, typename... Args>
+ constexpr OutputIt format(OutputIt out, const Args&... args) const {
+ return write<Char>(out, get_arg_checked<T, N>(args...));
+ }
+};
+
+template <typename Char, typename T, int N>
+struct is_compiled_format<field<Char, T, N>> : std::true_type {};
+
+// A replacement field that refers to argument with name.
+template <typename Char> struct runtime_named_field {
+ using char_type = Char;
+ basic_string_view<Char> name;
+
+ template <typename OutputIt, typename T>
+ constexpr static bool try_format_argument(
+ OutputIt& out,
+ // [[maybe_unused]] due to unused-but-set-parameter warning in GCC 7,8,9
+ [[maybe_unused]] basic_string_view<Char> arg_name, const T& arg) {
+ if constexpr (is_named_arg<typename std::remove_cv<T>::type>::value) {
+ if (arg_name == arg.name) {
+ out = write<Char>(out, arg.value);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ template <typename OutputIt, typename... Args>
+ constexpr OutputIt format(OutputIt out, const Args&... args) const {
+ bool found = (try_format_argument(out, name, args) || ...);
+ if (!found) {
+ FMT_THROW(format_error("argument with specified name is not found"));
+ }
+ return out;
+ }
+};
+
+template <typename Char>
+struct is_compiled_format<runtime_named_field<Char>> : std::true_type {};
+
+// A replacement field that refers to argument N and has format specifiers.
+template <typename Char, typename T, int N> struct spec_field {
+ using char_type = Char;
+ formatter<T, Char> fmt;
+
+ template <typename OutputIt, typename... Args>
+ constexpr FMT_INLINE OutputIt format(OutputIt out,
+ const Args&... args) const {
+ const auto& vargs =
+ fmt::make_format_args<basic_format_context<OutputIt, Char>>(args...);
+ basic_format_context<OutputIt, Char> ctx(out, vargs);
+ return fmt.format(get_arg_checked<T, N>(args...), ctx);
+ }
+};
+
+template <typename Char, typename T, int N>
+struct is_compiled_format<spec_field<Char, T, N>> : std::true_type {};
+
+template <typename L, typename R> struct concat {
+ L lhs;
+ R rhs;
+ using char_type = typename L::char_type;
+
+ template <typename OutputIt, typename... Args>
+ constexpr OutputIt format(OutputIt out, const Args&... args) const {
+ out = lhs.format(out, args...);
+ return rhs.format(out, args...);
+ }
+};
+
+template <typename L, typename R>
+struct is_compiled_format<concat<L, R>> : std::true_type {};
+
+template <typename L, typename R>
+constexpr concat<L, R> make_concat(L lhs, R rhs) {
+ return {lhs, rhs};
+}
+
+struct unknown_format {};
+
+template <typename Char>
+constexpr size_t parse_text(basic_string_view<Char> str, size_t pos) {
+ for (size_t size = str.size(); pos != size; ++pos) {
+ if (str[pos] == '{' || str[pos] == '}') break;
+ }
+ return pos;
+}
+
+template <typename Args, size_t POS, int ID, typename S>
+constexpr auto compile_format_string(S format_str);
+
+template <typename Args, size_t POS, int ID, typename T, typename S>
+constexpr auto parse_tail(T head, S format_str) {
+ if constexpr (POS !=
+ basic_string_view<typename S::char_type>(format_str).size()) {
+ constexpr auto tail = compile_format_string<Args, POS, ID>(format_str);
+ if constexpr (std::is_same<remove_cvref_t<decltype(tail)>,
+ unknown_format>())
+ return tail;
+ else
+ return make_concat(head, tail);
+ } else {
+ return head;
+ }
+}
+
+template <typename T, typename Char> struct parse_specs_result {
+ formatter<T, Char> fmt;
+ size_t end;
+ int next_arg_id;
+};
+
+enum { manual_indexing_id = -1 };
+
+template <typename T, typename Char>
+constexpr parse_specs_result<T, Char> parse_specs(basic_string_view<Char> str,
+ size_t pos, int next_arg_id) {
+ str.remove_prefix(pos);
+ auto ctx =
+ compile_parse_context<Char>(str, max_value<int>(), nullptr, next_arg_id);
+ auto f = formatter<T, Char>();
+ auto end = f.parse(ctx);
+ return {f, pos + fmt::detail::to_unsigned(end - str.data()),
+ next_arg_id == 0 ? manual_indexing_id : ctx.next_arg_id()};
+}
+
+template <typename Char> struct arg_id_handler {
+ arg_ref<Char> arg_id;
+
+ constexpr int on_auto() {
+ FMT_ASSERT(false, "handler cannot be used with automatic indexing");
+ return 0;
+ }
+ constexpr int on_index(int id) {
+ arg_id = arg_ref<Char>(id);
+ return 0;
+ }
+ constexpr int on_name(basic_string_view<Char> id) {
+ arg_id = arg_ref<Char>(id);
+ return 0;
+ }
+};
+
+template <typename Char> struct parse_arg_id_result {
+ arg_ref<Char> arg_id;
+ const Char* arg_id_end;
+};
+
+template <int ID, typename Char>
+constexpr auto parse_arg_id(const Char* begin, const Char* end) {
+ auto handler = arg_id_handler<Char>{arg_ref<Char>{}};
+ auto arg_id_end = parse_arg_id(begin, end, handler);
+ return parse_arg_id_result<Char>{handler.arg_id, arg_id_end};
+}
+
+template <typename T, typename Enable = void> struct field_type {
+ using type = remove_cvref_t<T>;
+};
+
+template <typename T>
+struct field_type<T, enable_if_t<detail::is_named_arg<T>::value>> {
+ using type = remove_cvref_t<decltype(T::value)>;
+};
+
+template <typename T, typename Args, size_t END_POS, int ARG_INDEX, int NEXT_ID,
+ typename S>
+constexpr auto parse_replacement_field_then_tail(S format_str) {
+ using char_type = typename S::char_type;
+ constexpr auto str = basic_string_view<char_type>(format_str);
+ constexpr char_type c = END_POS != str.size() ? str[END_POS] : char_type();
+ if constexpr (c == '}') {
+ return parse_tail<Args, END_POS + 1, NEXT_ID>(
+ field<char_type, typename field_type<T>::type, ARG_INDEX>(),
+ format_str);
+ } else if constexpr (c != ':') {
+ FMT_THROW(format_error("expected ':'"));
+ } else {
+ constexpr auto result = parse_specs<typename field_type<T>::type>(
+ str, END_POS + 1, NEXT_ID == manual_indexing_id ? 0 : NEXT_ID);
+ if constexpr (result.end >= str.size() || str[result.end] != '}') {
+ FMT_THROW(format_error("expected '}'"));
+ return 0;
+ } else {
+ return parse_tail<Args, result.end + 1, result.next_arg_id>(
+ spec_field<char_type, typename field_type<T>::type, ARG_INDEX>{
+ result.fmt},
+ format_str);
+ }
+ }
+}
+
+// Compiles a non-empty format string and returns the compiled representation
+// or unknown_format() on unrecognized input.
+template <typename Args, size_t POS, int ID, typename S>
+constexpr auto compile_format_string(S format_str) {
+ using char_type = typename S::char_type;
+ constexpr auto str = basic_string_view<char_type>(format_str);
+ if constexpr (str[POS] == '{') {
+ if constexpr (POS + 1 == str.size())
+ FMT_THROW(format_error("unmatched '{' in format string"));
+ if constexpr (str[POS + 1] == '{') {
+ return parse_tail<Args, POS + 2, ID>(make_text(str, POS, 1), format_str);
+ } else if constexpr (str[POS + 1] == '}' || str[POS + 1] == ':') {
+ static_assert(ID != manual_indexing_id,
+ "cannot switch from manual to automatic argument indexing");
+ constexpr auto next_id =
+ ID != manual_indexing_id ? ID + 1 : manual_indexing_id;
+ return parse_replacement_field_then_tail<get_type<ID, Args>, Args,
+ POS + 1, ID, next_id>(
+ format_str);
+ } else {
+ constexpr auto arg_id_result =
+ parse_arg_id<ID>(str.data() + POS + 1, str.data() + str.size());
+ constexpr auto arg_id_end_pos = arg_id_result.arg_id_end - str.data();
+ constexpr char_type c =
+ arg_id_end_pos != str.size() ? str[arg_id_end_pos] : char_type();
+ static_assert(c == '}' || c == ':', "missing '}' in format string");
+ if constexpr (arg_id_result.arg_id.kind == arg_id_kind::index) {
+ static_assert(
+ ID == manual_indexing_id || ID == 0,
+ "cannot switch from automatic to manual argument indexing");
+ constexpr auto arg_index = arg_id_result.arg_id.val.index;
+ return parse_replacement_field_then_tail<get_type<arg_index, Args>,
+ Args, arg_id_end_pos,
+ arg_index, manual_indexing_id>(
+ format_str);
+ } else if constexpr (arg_id_result.arg_id.kind == arg_id_kind::name) {
+ constexpr auto arg_index =
+ get_arg_index_by_name(arg_id_result.arg_id.val.name, Args{});
+ if constexpr (arg_index != invalid_arg_index) {
+ constexpr auto next_id =
+ ID != manual_indexing_id ? ID + 1 : manual_indexing_id;
+ return parse_replacement_field_then_tail<
+ decltype(get_type<arg_index, Args>::value), Args, arg_id_end_pos,
+ arg_index, next_id>(format_str);
+ } else {
+ if constexpr (c == '}') {
+ return parse_tail<Args, arg_id_end_pos + 1, ID>(
+ runtime_named_field<char_type>{arg_id_result.arg_id.val.name},
+ format_str);
+ } else if constexpr (c == ':') {
+ return unknown_format(); // no type info for specs parsing
+ }
+ }
+ }
+ }
+ } else if constexpr (str[POS] == '}') {
+ if constexpr (POS + 1 == str.size())
+ FMT_THROW(format_error("unmatched '}' in format string"));
+ return parse_tail<Args, POS + 2, ID>(make_text(str, POS, 1), format_str);
+ } else {
+ constexpr auto end = parse_text(str, POS + 1);
+ if constexpr (end - POS > 1) {
+ return parse_tail<Args, end, ID>(make_text(str, POS, end - POS),
+ format_str);
+ } else {
+ return parse_tail<Args, end, ID>(code_unit<char_type>{str[POS]},
+ format_str);
+ }
+ }
+}
+
+template <typename... Args, typename S,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+constexpr auto compile(S format_str) {
+ constexpr auto str = basic_string_view<typename S::char_type>(format_str);
+ if constexpr (str.size() == 0) {
+ return detail::make_text(str, 0, 0);
+ } else {
+ constexpr auto result =
+ detail::compile_format_string<detail::type_list<Args...>, 0, 0>(
+ format_str);
+ return result;
+ }
+}
+#endif // defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+} // namespace detail
+
+FMT_BEGIN_EXPORT
+
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+
+template <typename CompiledFormat, typename... Args,
+ typename Char = typename CompiledFormat::char_type,
+ FMT_ENABLE_IF(detail::is_compiled_format<CompiledFormat>::value)>
+FMT_INLINE std::basic_string<Char> format(const CompiledFormat& cf,
+ const Args&... args) {
+ auto s = std::basic_string<Char>();
+ cf.format(std::back_inserter(s), args...);
+ return s;
+}
+
+template <typename OutputIt, typename CompiledFormat, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_format<CompiledFormat>::value)>
+constexpr FMT_INLINE OutputIt format_to(OutputIt out, const CompiledFormat& cf,
+ const Args&... args) {
+ return cf.format(out, args...);
+}
+
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+FMT_INLINE std::basic_string<typename S::char_type> format(const S&,
+ Args&&... args) {
+ if constexpr (std::is_same<typename S::char_type, char>::value) {
+ constexpr auto str = basic_string_view<typename S::char_type>(S());
+ if constexpr (str.size() == 2 && str[0] == '{' && str[1] == '}') {
+ const auto& first = detail::first(args...);
+ if constexpr (detail::is_named_arg<
+ remove_cvref_t<decltype(first)>>::value) {
+ return fmt::to_string(first.value);
+ } else {
+ return fmt::to_string(first);
+ }
+ }
+ }
+ constexpr auto compiled = detail::compile<Args...>(S());
+ if constexpr (std::is_same<remove_cvref_t<decltype(compiled)>,
+ detail::unknown_format>()) {
+ return fmt::format(
+ static_cast<basic_string_view<typename S::char_type>>(S()),
+ std::forward<Args>(args)...);
+ } else {
+ return fmt::format(compiled, std::forward<Args>(args)...);
+ }
+}
+
+template <typename OutputIt, typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+FMT_CONSTEXPR OutputIt format_to(OutputIt out, const S&, Args&&... args) {
+ constexpr auto compiled = detail::compile<Args...>(S());
+ if constexpr (std::is_same<remove_cvref_t<decltype(compiled)>,
+ detail::unknown_format>()) {
+ return fmt::format_to(
+ out, static_cast<basic_string_view<typename S::char_type>>(S()),
+ std::forward<Args>(args)...);
+ } else {
+ return fmt::format_to(out, compiled, std::forward<Args>(args)...);
+ }
+}
+#endif
+
+template <typename OutputIt, typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+format_to_n_result<OutputIt> format_to_n(OutputIt out, size_t n,
+ const S& format_str, Args&&... args) {
+ auto it = fmt::format_to(detail::truncating_iterator<OutputIt>(out, n),
+ format_str, std::forward<Args>(args)...);
+ return {it.base(), it.count()};
+}
+
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+FMT_CONSTEXPR20 size_t formatted_size(const S& format_str,
+ const Args&... args) {
+ return fmt::format_to(detail::counting_iterator(), format_str, args...)
+ .count();
+}
+
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+void print(std::FILE* f, const S& format_str, const Args&... args) {
+ memory_buffer buffer;
+ fmt::format_to(std::back_inserter(buffer), format_str, args...);
+ detail::print(f, {buffer.data(), buffer.size()});
+}
+
+template <typename S, typename... Args,
+ FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+void print(const S& format_str, const Args&... args) {
+ print(stdout, format_str, args...);
+}
+
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+inline namespace literals {
+template <detail_exported::fixed_string Str> constexpr auto operator""_cf() {
+ using char_t = remove_cvref_t<decltype(Str.data[0])>;
+ return detail::udl_compiled_string<char_t, sizeof(Str.data) / sizeof(char_t),
+ Str>();
+}
+} // namespace literals
+#endif
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_COMPILE_H_
diff --git a/contrib/fmt/include/fmt/core.h b/contrib/fmt/include/fmt/core.h
new file mode 100644
index 0000000..46723d5
--- /dev/null
+++ b/contrib/fmt/include/fmt/core.h
@@ -0,0 +1,2951 @@
+// Formatting library for C++ - the core API for char/UTF-8
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_CORE_H_
+#define FMT_CORE_H_
+
+#include <cstddef> // std::byte
+#include <cstdio> // std::FILE
+#include <cstring> // std::strlen
+#include <iterator>
+#include <limits>
+#include <string>
+#include <type_traits>
+
+// The fmt library version in the form major * 10000 + minor * 100 + patch.
+#define FMT_VERSION 100000
+
+#if defined(__clang__) && !defined(__ibmxl__)
+# define FMT_CLANG_VERSION (__clang_major__ * 100 + __clang_minor__)
+#else
+# define FMT_CLANG_VERSION 0
+#endif
+
+#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) && \
+ !defined(__NVCOMPILER)
+# define FMT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#else
+# define FMT_GCC_VERSION 0
+#endif
+
+#ifndef FMT_GCC_PRAGMA
+// Workaround _Pragma bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59884.
+# if FMT_GCC_VERSION >= 504
+# define FMT_GCC_PRAGMA(arg) _Pragma(arg)
+# else
+# define FMT_GCC_PRAGMA(arg)
+# endif
+#endif
+
+#ifdef __ICL
+# define FMT_ICC_VERSION __ICL
+#elif defined(__INTEL_COMPILER)
+# define FMT_ICC_VERSION __INTEL_COMPILER
+#else
+# define FMT_ICC_VERSION 0
+#endif
+
+#ifdef _MSC_VER
+# define FMT_MSC_VERSION _MSC_VER
+# define FMT_MSC_WARNING(...) __pragma(warning(__VA_ARGS__))
+#else
+# define FMT_MSC_VERSION 0
+# define FMT_MSC_WARNING(...)
+#endif
+
+#ifdef _MSVC_LANG
+# define FMT_CPLUSPLUS _MSVC_LANG
+#else
+# define FMT_CPLUSPLUS __cplusplus
+#endif
+
+#ifdef __has_feature
+# define FMT_HAS_FEATURE(x) __has_feature(x)
+#else
+# define FMT_HAS_FEATURE(x) 0
+#endif
+
+#if defined(__has_include) || FMT_ICC_VERSION >= 1600 || FMT_MSC_VERSION > 1900
+# define FMT_HAS_INCLUDE(x) __has_include(x)
+#else
+# define FMT_HAS_INCLUDE(x) 0
+#endif
+
+#ifdef __has_cpp_attribute
+# define FMT_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+# define FMT_HAS_CPP_ATTRIBUTE(x) 0
+#endif
+
+#define FMT_HAS_CPP14_ATTRIBUTE(attribute) \
+ (FMT_CPLUSPLUS >= 201402L && FMT_HAS_CPP_ATTRIBUTE(attribute))
+
+#define FMT_HAS_CPP17_ATTRIBUTE(attribute) \
+ (FMT_CPLUSPLUS >= 201703L && FMT_HAS_CPP_ATTRIBUTE(attribute))
+
+// Check if relaxed C++14 constexpr is supported.
+// GCC doesn't allow throw in constexpr until version 6 (bug 67371).
+#ifndef FMT_USE_CONSTEXPR
+# if (FMT_HAS_FEATURE(cxx_relaxed_constexpr) || FMT_MSC_VERSION >= 1912 || \
+ (FMT_GCC_VERSION >= 600 && FMT_CPLUSPLUS >= 201402L)) && \
+ !FMT_ICC_VERSION && !defined(__NVCC__)
+# define FMT_USE_CONSTEXPR 1
+# else
+# define FMT_USE_CONSTEXPR 0
+# endif
+#endif
+#if FMT_USE_CONSTEXPR
+# define FMT_CONSTEXPR constexpr
+#else
+# define FMT_CONSTEXPR
+#endif
+
+#if ((FMT_CPLUSPLUS >= 202002L) && \
+ (!defined(_GLIBCXX_RELEASE) || _GLIBCXX_RELEASE > 9)) || \
+ (FMT_CPLUSPLUS >= 201709L && FMT_GCC_VERSION >= 1002)
+# define FMT_CONSTEXPR20 constexpr
+#else
+# define FMT_CONSTEXPR20
+#endif
+
+// Check if constexpr std::char_traits<>::{compare,length} are supported.
+#if defined(__GLIBCXX__)
+# if FMT_CPLUSPLUS >= 201703L && defined(_GLIBCXX_RELEASE) && \
+ _GLIBCXX_RELEASE >= 7 // GCC 7+ libstdc++ has _GLIBCXX_RELEASE.
+# define FMT_CONSTEXPR_CHAR_TRAITS constexpr
+# endif
+#elif defined(_LIBCPP_VERSION) && FMT_CPLUSPLUS >= 201703L && \
+ _LIBCPP_VERSION >= 4000
+# define FMT_CONSTEXPR_CHAR_TRAITS constexpr
+#elif FMT_MSC_VERSION >= 1914 && FMT_CPLUSPLUS >= 201703L
+# define FMT_CONSTEXPR_CHAR_TRAITS constexpr
+#endif
+#ifndef FMT_CONSTEXPR_CHAR_TRAITS
+# define FMT_CONSTEXPR_CHAR_TRAITS
+#endif
+
+// Check if exceptions are disabled.
+#ifndef FMT_EXCEPTIONS
+# if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || \
+ (FMT_MSC_VERSION && !_HAS_EXCEPTIONS)
+# define FMT_EXCEPTIONS 0
+# else
+# define FMT_EXCEPTIONS 1
+# endif
+#endif
+
+// Disable [[noreturn]] on MSVC/NVCC because of bogus unreachable code warnings.
+#if FMT_EXCEPTIONS && FMT_HAS_CPP_ATTRIBUTE(noreturn) && !FMT_MSC_VERSION && \
+ !defined(__NVCC__)
+# define FMT_NORETURN [[noreturn]]
+#else
+# define FMT_NORETURN
+#endif
+
+#ifndef FMT_NODISCARD
+# if FMT_HAS_CPP17_ATTRIBUTE(nodiscard)
+# define FMT_NODISCARD [[nodiscard]]
+# else
+# define FMT_NODISCARD
+# endif
+#endif
+
+#ifndef FMT_INLINE
+# if FMT_GCC_VERSION || FMT_CLANG_VERSION
+# define FMT_INLINE inline __attribute__((always_inline))
+# else
+# define FMT_INLINE inline
+# endif
+#endif
+
+// An inline std::forward replacement.
+#define FMT_FORWARD(...) static_cast<decltype(__VA_ARGS__)&&>(__VA_ARGS__)
+
+#ifdef _MSC_VER
+# define FMT_UNCHECKED_ITERATOR(It) \
+ using _Unchecked_type = It // Mark iterator as checked.
+#else
+# define FMT_UNCHECKED_ITERATOR(It) using unchecked_type = It
+#endif
+
+#ifndef FMT_BEGIN_NAMESPACE
+# define FMT_BEGIN_NAMESPACE \
+ namespace fmt { \
+ inline namespace v10 {
+# define FMT_END_NAMESPACE \
+ } \
+ }
+#endif
+
+#ifndef FMT_MODULE_EXPORT
+# define FMT_MODULE_EXPORT
+# define FMT_BEGIN_EXPORT
+# define FMT_END_EXPORT
+#endif
+
+#if !defined(FMT_HEADER_ONLY) && defined(_WIN32)
+# ifdef FMT_LIB_EXPORT
+# define FMT_API __declspec(dllexport)
+# elif defined(FMT_SHARED)
+# define FMT_API __declspec(dllimport)
+# endif
+#else
+# if defined(FMT_LIB_EXPORT) || defined(FMT_SHARED)
+# if defined(__GNUC__) || defined(__clang__)
+# define FMT_API __attribute__((visibility("default")))
+# endif
+# endif
+#endif
+#ifndef FMT_API
+# define FMT_API
+#endif
+
+// libc++ supports string_view in pre-c++17.
+#if FMT_HAS_INCLUDE(<string_view>) && \
+ (FMT_CPLUSPLUS >= 201703L || defined(_LIBCPP_VERSION))
+# include <string_view>
+# define FMT_USE_STRING_VIEW
+#elif FMT_HAS_INCLUDE("experimental/string_view") && FMT_CPLUSPLUS >= 201402L
+# include <experimental/string_view>
+# define FMT_USE_EXPERIMENTAL_STRING_VIEW
+#endif
+
+#ifndef FMT_UNICODE
+# define FMT_UNICODE !FMT_MSC_VERSION
+#endif
+
+#ifndef FMT_CONSTEVAL
+# if ((FMT_GCC_VERSION >= 1000 || FMT_CLANG_VERSION >= 1101) && \
+ (!defined(__apple_build_version__) || \
+ __apple_build_version__ >= 14000029L) && \
+ FMT_CPLUSPLUS >= 202002L) || \
+ (defined(__cpp_consteval) && \
+ (!FMT_MSC_VERSION || _MSC_FULL_VER >= 193030704))
+// consteval is broken in MSVC before VS2022 and Apple clang before 14.
+# define FMT_CONSTEVAL consteval
+# define FMT_HAS_CONSTEVAL
+# else
+# define FMT_CONSTEVAL
+# endif
+#endif
+
+#ifndef FMT_USE_NONTYPE_TEMPLATE_ARGS
+# if defined(__cpp_nontype_template_args) && \
+ ((FMT_GCC_VERSION >= 903 && FMT_CPLUSPLUS >= 201709L) || \
+ __cpp_nontype_template_args >= 201911L) && \
+ !defined(__NVCOMPILER) && !defined(__LCC__)
+# define FMT_USE_NONTYPE_TEMPLATE_ARGS 1
+# else
+# define FMT_USE_NONTYPE_TEMPLATE_ARGS 0
+# endif
+#endif
+
+#if defined __cpp_inline_variables && __cpp_inline_variables >= 201606L
+# define FMT_INLINE_VARIABLE inline
+#else
+# define FMT_INLINE_VARIABLE
+#endif
+
+// Enable minimal optimizations for more compact code in debug mode.
+FMT_GCC_PRAGMA("GCC push_options")
+#if !defined(__OPTIMIZE__) && !defined(__NVCOMPILER) && !defined(__LCC__) && \
+ !defined(__CUDACC__)
+FMT_GCC_PRAGMA("GCC optimize(\"Og\")")
+#endif
+
+FMT_BEGIN_NAMESPACE
+
+// Implementations of enable_if_t and other metafunctions for older systems.
+template <bool B, typename T = void>
+using enable_if_t = typename std::enable_if<B, T>::type;
+template <bool B, typename T, typename F>
+using conditional_t = typename std::conditional<B, T, F>::type;
+template <bool B> using bool_constant = std::integral_constant<bool, B>;
+template <typename T>
+using remove_reference_t = typename std::remove_reference<T>::type;
+template <typename T>
+using remove_const_t = typename std::remove_const<T>::type;
+template <typename T>
+using remove_cvref_t = typename std::remove_cv<remove_reference_t<T>>::type;
+template <typename T> struct type_identity { using type = T; };
+template <typename T> using type_identity_t = typename type_identity<T>::type;
+template <typename T>
+using underlying_t = typename std::underlying_type<T>::type;
+
+struct monostate {
+ constexpr monostate() {}
+};
+
+// An enable_if helper to be used in template parameters which results in much
+// shorter symbols: https://godbolt.org/z/sWw4vP. Extra parentheses are needed
+// to workaround a bug in MSVC 2019 (see #1140 and #1186).
+#ifdef FMT_DOC
+# define FMT_ENABLE_IF(...)
+#else
+# define FMT_ENABLE_IF(...) fmt::enable_if_t<(__VA_ARGS__), int> = 0
+#endif
+
+#ifdef __cpp_lib_byte
+inline auto format_as(std::byte b) -> unsigned char {
+ return static_cast<unsigned char>(b);
+}
+#endif
+
+namespace detail {
+// Suppresses "unused variable" warnings with the method described in
+// https://herbsutter.com/2009/10/18/mailbag-shutting-up-compiler-warnings/.
+// (void)var does not work on many Intel compilers.
+template <typename... T> FMT_CONSTEXPR void ignore_unused(const T&...) {}
+
+constexpr FMT_INLINE auto is_constant_evaluated(
+ bool default_value = false) noexcept -> bool {
+// Workaround for incompatibility between libstdc++ consteval-based
+// std::is_constant_evaluated() implementation and clang-14.
+// https://github.com/fmtlib/fmt/issues/3247
+#if FMT_CPLUSPLUS >= 202002L && defined(_GLIBCXX_RELEASE) && \
+ _GLIBCXX_RELEASE >= 12 && \
+ (FMT_CLANG_VERSION >= 1400 && FMT_CLANG_VERSION < 1500)
+ ignore_unused(default_value);
+ return __builtin_is_constant_evaluated();
+#elif defined(__cpp_lib_is_constant_evaluated)
+ ignore_unused(default_value);
+ return std::is_constant_evaluated();
+#else
+ return default_value;
+#endif
+}
+
+// Suppresses "conditional expression is constant" warnings.
+template <typename T> constexpr FMT_INLINE auto const_check(T value) -> T {
+ return value;
+}
+
+FMT_NORETURN FMT_API void assert_fail(const char* file, int line,
+ const char* message);
+
+#ifndef FMT_ASSERT
+# ifdef NDEBUG
+// FMT_ASSERT is not empty to avoid -Wempty-body.
+# define FMT_ASSERT(condition, message) \
+ fmt::detail::ignore_unused((condition), (message))
+# else
+# define FMT_ASSERT(condition, message) \
+ ((condition) /* void() fails with -Winvalid-constexpr on clang 4.0.1 */ \
+ ? (void)0 \
+ : fmt::detail::assert_fail(__FILE__, __LINE__, (message)))
+# endif
+#endif
+
+#if defined(FMT_USE_STRING_VIEW)
+template <typename Char> using std_string_view = std::basic_string_view<Char>;
+#elif defined(FMT_USE_EXPERIMENTAL_STRING_VIEW)
+template <typename Char>
+using std_string_view = std::experimental::basic_string_view<Char>;
+#else
+template <typename T> struct std_string_view {};
+#endif
+
+#ifdef FMT_USE_INT128
+// Do nothing.
+#elif defined(__SIZEOF_INT128__) && !defined(__NVCC__) && \
+ !(FMT_CLANG_VERSION && FMT_MSC_VERSION)
+# define FMT_USE_INT128 1
+using int128_opt = __int128_t; // An optional native 128-bit integer.
+using uint128_opt = __uint128_t;
+template <typename T> inline auto convert_for_visit(T value) -> T {
+ return value;
+}
+#else
+# define FMT_USE_INT128 0
+#endif
+#if !FMT_USE_INT128
+enum class int128_opt {};
+enum class uint128_opt {};
+// Reduce template instantiations.
+template <typename T> auto convert_for_visit(T) -> monostate { return {}; }
+#endif
+
+// Casts a nonnegative integer to unsigned.
+template <typename Int>
+FMT_CONSTEXPR auto to_unsigned(Int value) ->
+ typename std::make_unsigned<Int>::type {
+ FMT_ASSERT(std::is_unsigned<Int>::value || value >= 0, "negative value");
+ return static_cast<typename std::make_unsigned<Int>::type>(value);
+}
+
+FMT_CONSTEXPR inline auto is_utf8() -> bool {
+ FMT_MSC_WARNING(suppress : 4566) constexpr unsigned char section[] = "\u00A7";
+
+ // Avoid buggy sign extensions in MSVC's constant evaluation mode (#2297).
+ using uchar = unsigned char;
+ return FMT_UNICODE || (sizeof(section) == 3 && uchar(section[0]) == 0xC2 &&
+ uchar(section[1]) == 0xA7);
+}
+} // namespace detail
+
+/**
+ An implementation of ``std::basic_string_view`` for pre-C++17. It provides a
+ subset of the API. ``fmt::basic_string_view`` is used for format strings even
+ if ``std::string_view`` is available to prevent issues when a library is
+ compiled with a different ``-std`` option than the client code (which is not
+ recommended).
+ */
+FMT_MODULE_EXPORT
+template <typename Char> class basic_string_view {
+ private:
+ const Char* data_;
+ size_t size_;
+
+ public:
+ using value_type = Char;
+ using iterator = const Char*;
+
+ constexpr basic_string_view() noexcept : data_(nullptr), size_(0) {}
+
+ /** Constructs a string reference object from a C string and a size. */
+ constexpr basic_string_view(const Char* s, size_t count) noexcept
+ : data_(s), size_(count) {}
+
+ /**
+ \rst
+ Constructs a string reference object from a C string computing
+ the size with ``std::char_traits<Char>::length``.
+ \endrst
+ */
+ FMT_CONSTEXPR_CHAR_TRAITS
+ FMT_INLINE
+ basic_string_view(const Char* s)
+ : data_(s),
+ size_(detail::const_check(std::is_same<Char, char>::value &&
+ !detail::is_constant_evaluated(true))
+ ? std::strlen(reinterpret_cast<const char*>(s))
+ : std::char_traits<Char>::length(s)) {}
+
+ /** Constructs a string reference from a ``std::basic_string`` object. */
+ template <typename Traits, typename Alloc>
+ FMT_CONSTEXPR basic_string_view(
+ const std::basic_string<Char, Traits, Alloc>& s) noexcept
+ : data_(s.data()), size_(s.size()) {}
+
+ template <typename S, FMT_ENABLE_IF(std::is_same<
+ S, detail::std_string_view<Char>>::value)>
+ FMT_CONSTEXPR basic_string_view(S s) noexcept
+ : data_(s.data()), size_(s.size()) {}
+
+ /** Returns a pointer to the string data. */
+ constexpr auto data() const noexcept -> const Char* { return data_; }
+
+ /** Returns the string size. */
+ constexpr auto size() const noexcept -> size_t { return size_; }
+
+ constexpr auto begin() const noexcept -> iterator { return data_; }
+ constexpr auto end() const noexcept -> iterator { return data_ + size_; }
+
+ constexpr auto operator[](size_t pos) const noexcept -> const Char& {
+ return data_[pos];
+ }
+
+ FMT_CONSTEXPR void remove_prefix(size_t n) noexcept {
+ data_ += n;
+ size_ -= n;
+ }
+
+ FMT_CONSTEXPR_CHAR_TRAITS bool starts_with(
+ basic_string_view<Char> sv) const noexcept {
+ return size_ >= sv.size_ &&
+ std::char_traits<Char>::compare(data_, sv.data_, sv.size_) == 0;
+ }
+ FMT_CONSTEXPR_CHAR_TRAITS bool starts_with(Char c) const noexcept {
+ return size_ >= 1 && std::char_traits<Char>::eq(*data_, c);
+ }
+ FMT_CONSTEXPR_CHAR_TRAITS bool starts_with(const Char* s) const {
+ return starts_with(basic_string_view<Char>(s));
+ }
+
+ // Lexicographically compare this string reference to other.
+ FMT_CONSTEXPR_CHAR_TRAITS auto compare(basic_string_view other) const -> int {
+ size_t str_size = size_ < other.size_ ? size_ : other.size_;
+ int result = std::char_traits<Char>::compare(data_, other.data_, str_size);
+ if (result == 0)
+ result = size_ == other.size_ ? 0 : (size_ < other.size_ ? -1 : 1);
+ return result;
+ }
+
+ FMT_CONSTEXPR_CHAR_TRAITS friend auto operator==(basic_string_view lhs,
+ basic_string_view rhs)
+ -> bool {
+ return lhs.compare(rhs) == 0;
+ }
+ friend auto operator!=(basic_string_view lhs, basic_string_view rhs) -> bool {
+ return lhs.compare(rhs) != 0;
+ }
+ friend auto operator<(basic_string_view lhs, basic_string_view rhs) -> bool {
+ return lhs.compare(rhs) < 0;
+ }
+ friend auto operator<=(basic_string_view lhs, basic_string_view rhs) -> bool {
+ return lhs.compare(rhs) <= 0;
+ }
+ friend auto operator>(basic_string_view lhs, basic_string_view rhs) -> bool {
+ return lhs.compare(rhs) > 0;
+ }
+ friend auto operator>=(basic_string_view lhs, basic_string_view rhs) -> bool {
+ return lhs.compare(rhs) >= 0;
+ }
+};
+
+FMT_MODULE_EXPORT
+using string_view = basic_string_view<char>;
+
+/** Specifies if ``T`` is a character type. Can be specialized by users. */
+FMT_MODULE_EXPORT
+template <typename T> struct is_char : std::false_type {};
+template <> struct is_char<char> : std::true_type {};
+
+namespace detail {
+
+// A base class for compile-time strings.
+struct compile_string {};
+
+template <typename S>
+struct is_compile_string : std::is_base_of<compile_string, S> {};
+
+template <typename Char, FMT_ENABLE_IF(is_char<Char>::value)>
+FMT_INLINE auto to_string_view(const Char* s) -> basic_string_view<Char> {
+ return s;
+}
+template <typename Char, typename Traits, typename Alloc>
+inline auto to_string_view(const std::basic_string<Char, Traits, Alloc>& s)
+ -> basic_string_view<Char> {
+ return s;
+}
+template <typename Char>
+constexpr auto to_string_view(basic_string_view<Char> s)
+ -> basic_string_view<Char> {
+ return s;
+}
+template <typename Char,
+ FMT_ENABLE_IF(!std::is_empty<std_string_view<Char>>::value)>
+inline auto to_string_view(std_string_view<Char> s) -> basic_string_view<Char> {
+ return s;
+}
+template <typename S, FMT_ENABLE_IF(is_compile_string<S>::value)>
+constexpr auto to_string_view(const S& s)
+ -> basic_string_view<typename S::char_type> {
+ return basic_string_view<typename S::char_type>(s);
+}
+void to_string_view(...);
+
+// Specifies whether S is a string type convertible to fmt::basic_string_view.
+// It should be a constexpr function but MSVC 2017 fails to compile it in
+// enable_if and MSVC 2015 fails to compile it as an alias template.
+// ADL is intentionally disabled as to_string_view is not an extension point.
+template <typename S>
+struct is_string
+ : std::is_class<decltype(detail::to_string_view(std::declval<S>()))> {};
+
+template <typename S, typename = void> struct char_t_impl {};
+template <typename S> struct char_t_impl<S, enable_if_t<is_string<S>::value>> {
+ using result = decltype(to_string_view(std::declval<S>()));
+ using type = typename result::value_type;
+};
+
+enum class type {
+ none_type,
+ // Integer types should go first,
+ int_type,
+ uint_type,
+ long_long_type,
+ ulong_long_type,
+ int128_type,
+ uint128_type,
+ bool_type,
+ char_type,
+ last_integer_type = char_type,
+ // followed by floating-point types.
+ float_type,
+ double_type,
+ long_double_type,
+ last_numeric_type = long_double_type,
+ cstring_type,
+ string_type,
+ pointer_type,
+ custom_type
+};
+
+// Maps core type T to the corresponding type enum constant.
+template <typename T, typename Char>
+struct type_constant : std::integral_constant<type, type::custom_type> {};
+
+#define FMT_TYPE_CONSTANT(Type, constant) \
+ template <typename Char> \
+ struct type_constant<Type, Char> \
+ : std::integral_constant<type, type::constant> {}
+
+FMT_TYPE_CONSTANT(int, int_type);
+FMT_TYPE_CONSTANT(unsigned, uint_type);
+FMT_TYPE_CONSTANT(long long, long_long_type);
+FMT_TYPE_CONSTANT(unsigned long long, ulong_long_type);
+FMT_TYPE_CONSTANT(int128_opt, int128_type);
+FMT_TYPE_CONSTANT(uint128_opt, uint128_type);
+FMT_TYPE_CONSTANT(bool, bool_type);
+FMT_TYPE_CONSTANT(Char, char_type);
+FMT_TYPE_CONSTANT(float, float_type);
+FMT_TYPE_CONSTANT(double, double_type);
+FMT_TYPE_CONSTANT(long double, long_double_type);
+FMT_TYPE_CONSTANT(const Char*, cstring_type);
+FMT_TYPE_CONSTANT(basic_string_view<Char>, string_type);
+FMT_TYPE_CONSTANT(const void*, pointer_type);
+
+constexpr bool is_integral_type(type t) {
+ return t > type::none_type && t <= type::last_integer_type;
+}
+constexpr bool is_arithmetic_type(type t) {
+ return t > type::none_type && t <= type::last_numeric_type;
+}
+
+constexpr auto set(type rhs) -> int { return 1 << static_cast<int>(rhs); }
+constexpr auto in(type t, int set) -> bool {
+ return ((set >> static_cast<int>(t)) & 1) != 0;
+}
+
+// Bitsets of types.
+enum {
+ sint_set =
+ set(type::int_type) | set(type::long_long_type) | set(type::int128_type),
+ uint_set = set(type::uint_type) | set(type::ulong_long_type) |
+ set(type::uint128_type),
+ bool_set = set(type::bool_type),
+ char_set = set(type::char_type),
+ float_set = set(type::float_type) | set(type::double_type) |
+ set(type::long_double_type),
+ string_set = set(type::string_type),
+ cstring_set = set(type::cstring_type),
+ pointer_set = set(type::pointer_type)
+};
+
+FMT_NORETURN FMT_API void throw_format_error(const char* message);
+
+struct error_handler {
+ constexpr error_handler() = default;
+
+ // This function is intentionally not constexpr to give a compile-time error.
+ FMT_NORETURN void on_error(const char* message) {
+ throw_format_error(message);
+ }
+};
+} // namespace detail
+
+/** String's character type. */
+template <typename S> using char_t = typename detail::char_t_impl<S>::type;
+
+/**
+ \rst
+ Parsing context consisting of a format string range being parsed and an
+ argument counter for automatic indexing.
+ You can use the ``format_parse_context`` type alias for ``char`` instead.
+ \endrst
+ */
+FMT_MODULE_EXPORT
+template <typename Char> class basic_format_parse_context {
+ private:
+ basic_string_view<Char> format_str_;
+ int next_arg_id_;
+
+ FMT_CONSTEXPR void do_check_arg_id(int id);
+
+ public:
+ using char_type = Char;
+ using iterator = const Char*;
+
+ explicit constexpr basic_format_parse_context(
+ basic_string_view<Char> format_str, int next_arg_id = 0)
+ : format_str_(format_str), next_arg_id_(next_arg_id) {}
+
+ /**
+ Returns an iterator to the beginning of the format string range being
+ parsed.
+ */
+ constexpr auto begin() const noexcept -> iterator {
+ return format_str_.begin();
+ }
+
+ /**
+ Returns an iterator past the end of the format string range being parsed.
+ */
+ constexpr auto end() const noexcept -> iterator { return format_str_.end(); }
+
+ /** Advances the begin iterator to ``it``. */
+ FMT_CONSTEXPR void advance_to(iterator it) {
+ format_str_.remove_prefix(detail::to_unsigned(it - begin()));
+ }
+
+ /**
+ Reports an error if using the manual argument indexing; otherwise returns
+ the next argument index and switches to the automatic indexing.
+ */
+ FMT_CONSTEXPR auto next_arg_id() -> int {
+ if (next_arg_id_ < 0) {
+ detail::throw_format_error(
+ "cannot switch from manual to automatic argument indexing");
+ return 0;
+ }
+ int id = next_arg_id_++;
+ do_check_arg_id(id);
+ return id;
+ }
+
+ /**
+ Reports an error if using the automatic argument indexing; otherwise
+ switches to the manual indexing.
+ */
+ FMT_CONSTEXPR void check_arg_id(int id) {
+ if (next_arg_id_ > 0) {
+ detail::throw_format_error(
+ "cannot switch from automatic to manual argument indexing");
+ return;
+ }
+ next_arg_id_ = -1;
+ do_check_arg_id(id);
+ }
+ FMT_CONSTEXPR void check_arg_id(basic_string_view<Char>) {}
+ FMT_CONSTEXPR void check_dynamic_spec(int arg_id);
+};
+
+FMT_MODULE_EXPORT
+using format_parse_context = basic_format_parse_context<char>;
+
+namespace detail {
+// A parse context with extra data used only in compile-time checks.
+template <typename Char>
+class compile_parse_context : public basic_format_parse_context<Char> {
+ private:
+ int num_args_;
+ const type* types_;
+ using base = basic_format_parse_context<Char>;
+
+ public:
+ explicit FMT_CONSTEXPR compile_parse_context(
+ basic_string_view<Char> format_str, int num_args, const type* types,
+ int next_arg_id = 0)
+ : base(format_str, next_arg_id), num_args_(num_args), types_(types) {}
+
+ constexpr auto num_args() const -> int { return num_args_; }
+ constexpr auto arg_type(int id) const -> type { return types_[id]; }
+
+ FMT_CONSTEXPR auto next_arg_id() -> int {
+ int id = base::next_arg_id();
+ if (id >= num_args_) throw_format_error("argument not found");
+ return id;
+ }
+
+ FMT_CONSTEXPR void check_arg_id(int id) {
+ base::check_arg_id(id);
+ if (id >= num_args_) throw_format_error("argument not found");
+ }
+ using base::check_arg_id;
+
+ FMT_CONSTEXPR void check_dynamic_spec(int arg_id) {
+ detail::ignore_unused(arg_id);
+#if !defined(__LCC__)
+ if (arg_id < num_args_ && types_ && !is_integral_type(types_[arg_id]))
+ throw_format_error("width/precision is not integer");
+#endif
+ }
+};
+} // namespace detail
+
+template <typename Char>
+FMT_CONSTEXPR void basic_format_parse_context<Char>::do_check_arg_id(int id) {
+ // Argument id is only checked at compile-time during parsing because
+ // formatting has its own validation.
+ if (detail::is_constant_evaluated() &&
+ (!FMT_GCC_VERSION || FMT_GCC_VERSION >= 1200)) {
+ using context = detail::compile_parse_context<Char>;
+ if (id >= static_cast<context*>(this)->num_args())
+ detail::throw_format_error("argument not found");
+ }
+}
+
+template <typename Char>
+FMT_CONSTEXPR void basic_format_parse_context<Char>::check_dynamic_spec(
+ int arg_id) {
+ if (detail::is_constant_evaluated() &&
+ (!FMT_GCC_VERSION || FMT_GCC_VERSION >= 1200)) {
+ using context = detail::compile_parse_context<Char>;
+ static_cast<context*>(this)->check_dynamic_spec(arg_id);
+ }
+}
+
+FMT_MODULE_EXPORT template <typename Context> class basic_format_arg;
+FMT_MODULE_EXPORT template <typename Context> class basic_format_args;
+FMT_MODULE_EXPORT template <typename Context> class dynamic_format_arg_store;
+
+// A formatter for objects of type T.
+FMT_MODULE_EXPORT
+template <typename T, typename Char = char, typename Enable = void>
+struct formatter {
+ // A deleted default constructor indicates a disabled formatter.
+ formatter() = delete;
+};
+
+// Specifies if T has an enabled formatter specialization. A type can be
+// formattable even if it doesn't have a formatter e.g. via a conversion.
+template <typename T, typename Context>
+using has_formatter =
+ std::is_constructible<typename Context::template formatter_type<T>>;
+
+// Checks whether T is a container with contiguous storage.
+template <typename T> struct is_contiguous : std::false_type {};
+template <typename Char>
+struct is_contiguous<std::basic_string<Char>> : std::true_type {};
+
+class appender;
+
+namespace detail {
+
+template <typename Context, typename T>
+constexpr auto has_const_formatter_impl(T*)
+ -> decltype(typename Context::template formatter_type<T>().format(
+ std::declval<const T&>(), std::declval<Context&>()),
+ true) {
+ return true;
+}
+template <typename Context>
+constexpr auto has_const_formatter_impl(...) -> bool {
+ return false;
+}
+template <typename T, typename Context>
+constexpr auto has_const_formatter() -> bool {
+ return has_const_formatter_impl<Context>(static_cast<T*>(nullptr));
+}
+
+// Extracts a reference to the container from back_insert_iterator.
+template <typename Container>
+inline auto get_container(std::back_insert_iterator<Container> it)
+ -> Container& {
+ using base = std::back_insert_iterator<Container>;
+ struct accessor : base {
+ accessor(base b) : base(b) {}
+ using base::container;
+ };
+ return *accessor(it).container;
+}
+
+template <typename Char, typename InputIt, typename OutputIt>
+FMT_CONSTEXPR auto copy_str(InputIt begin, InputIt end, OutputIt out)
+ -> OutputIt {
+ while (begin != end) *out++ = static_cast<Char>(*begin++);
+ return out;
+}
+
+template <typename Char, typename T, typename U,
+ FMT_ENABLE_IF(
+ std::is_same<remove_const_t<T>, U>::value&& is_char<U>::value)>
+FMT_CONSTEXPR auto copy_str(T* begin, T* end, U* out) -> U* {
+ if (is_constant_evaluated()) return copy_str<Char, T*, U*>(begin, end, out);
+ auto size = to_unsigned(end - begin);
+ if (size > 0) memcpy(out, begin, size * sizeof(U));
+ return out + size;
+}
+
+/**
+ \rst
+ A contiguous memory buffer with an optional growing ability. It is an internal
+ class and shouldn't be used directly, only via `~fmt::basic_memory_buffer`.
+ \endrst
+ */
+template <typename T> class buffer {
+ private:
+ T* ptr_;
+ size_t size_;
+ size_t capacity_;
+
+ protected:
+ // Don't initialize ptr_ since it is not accessed to save a few cycles.
+ FMT_MSC_WARNING(suppress : 26495)
+ buffer(size_t sz) noexcept : size_(sz), capacity_(sz) {}
+
+ FMT_CONSTEXPR20 buffer(T* p = nullptr, size_t sz = 0, size_t cap = 0) noexcept
+ : ptr_(p), size_(sz), capacity_(cap) {}
+
+ FMT_CONSTEXPR20 ~buffer() = default;
+ buffer(buffer&&) = default;
+
+ /** Sets the buffer data and capacity. */
+ FMT_CONSTEXPR void set(T* buf_data, size_t buf_capacity) noexcept {
+ ptr_ = buf_data;
+ capacity_ = buf_capacity;
+ }
+
+ /** Increases the buffer capacity to hold at least *capacity* elements. */
+ virtual FMT_CONSTEXPR20 void grow(size_t capacity) = 0;
+
+ public:
+ using value_type = T;
+ using const_reference = const T&;
+
+ buffer(const buffer&) = delete;
+ void operator=(const buffer&) = delete;
+
+ FMT_INLINE auto begin() noexcept -> T* { return ptr_; }
+ FMT_INLINE auto end() noexcept -> T* { return ptr_ + size_; }
+
+ FMT_INLINE auto begin() const noexcept -> const T* { return ptr_; }
+ FMT_INLINE auto end() const noexcept -> const T* { return ptr_ + size_; }
+
+ /** Returns the size of this buffer. */
+ constexpr auto size() const noexcept -> size_t { return size_; }
+
+ /** Returns the capacity of this buffer. */
+ constexpr auto capacity() const noexcept -> size_t { return capacity_; }
+
+ /** Returns a pointer to the buffer data. */
+ FMT_CONSTEXPR auto data() noexcept -> T* { return ptr_; }
+
+ /** Returns a pointer to the buffer data. */
+ FMT_CONSTEXPR auto data() const noexcept -> const T* { return ptr_; }
+
+ /** Clears this buffer. */
+ void clear() { size_ = 0; }
+
+ // Tries resizing the buffer to contain *count* elements. If T is a POD type
+ // the new elements may not be initialized.
+ FMT_CONSTEXPR20 void try_resize(size_t count) {
+ try_reserve(count);
+ size_ = count <= capacity_ ? count : capacity_;
+ }
+
+ // Tries increasing the buffer capacity to *new_capacity*. It can increase the
+ // capacity by a smaller amount than requested but guarantees there is space
+ // for at least one additional element either by increasing the capacity or by
+ // flushing the buffer if it is full.
+ FMT_CONSTEXPR20 void try_reserve(size_t new_capacity) {
+ if (new_capacity > capacity_) grow(new_capacity);
+ }
+
+ FMT_CONSTEXPR20 void push_back(const T& value) {
+ try_reserve(size_ + 1);
+ ptr_[size_++] = value;
+ }
+
+ /** Appends data to the end of the buffer. */
+ template <typename U> void append(const U* begin, const U* end);
+
+ template <typename Idx> FMT_CONSTEXPR auto operator[](Idx index) -> T& {
+ return ptr_[index];
+ }
+ template <typename Idx>
+ FMT_CONSTEXPR auto operator[](Idx index) const -> const T& {
+ return ptr_[index];
+ }
+};
+
+struct buffer_traits {
+ explicit buffer_traits(size_t) {}
+ auto count() const -> size_t { return 0; }
+ auto limit(size_t size) -> size_t { return size; }
+};
+
+class fixed_buffer_traits {
+ private:
+ size_t count_ = 0;
+ size_t limit_;
+
+ public:
+ explicit fixed_buffer_traits(size_t limit) : limit_(limit) {}
+ auto count() const -> size_t { return count_; }
+ auto limit(size_t size) -> size_t {
+ size_t n = limit_ > count_ ? limit_ - count_ : 0;
+ count_ += size;
+ return size < n ? size : n;
+ }
+};
+
+// A buffer that writes to an output iterator when flushed.
+template <typename OutputIt, typename T, typename Traits = buffer_traits>
+class iterator_buffer final : public Traits, public buffer<T> {
+ private:
+ OutputIt out_;
+ enum { buffer_size = 256 };
+ T data_[buffer_size];
+
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t) override {
+ if (this->size() == buffer_size) flush();
+ }
+
+ void flush() {
+ auto size = this->size();
+ this->clear();
+ out_ = copy_str<T>(data_, data_ + this->limit(size), out_);
+ }
+
+ public:
+ explicit iterator_buffer(OutputIt out, size_t n = buffer_size)
+ : Traits(n), buffer<T>(data_, 0, buffer_size), out_(out) {}
+ iterator_buffer(iterator_buffer&& other)
+ : Traits(other), buffer<T>(data_, 0, buffer_size), out_(other.out_) {}
+ ~iterator_buffer() { flush(); }
+
+ auto out() -> OutputIt {
+ flush();
+ return out_;
+ }
+ auto count() const -> size_t { return Traits::count() + this->size(); }
+};
+
+template <typename T>
+class iterator_buffer<T*, T, fixed_buffer_traits> final
+ : public fixed_buffer_traits,
+ public buffer<T> {
+ private:
+ T* out_;
+ enum { buffer_size = 256 };
+ T data_[buffer_size];
+
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t) override {
+ if (this->size() == this->capacity()) flush();
+ }
+
+ void flush() {
+ size_t n = this->limit(this->size());
+ if (this->data() == out_) {
+ out_ += n;
+ this->set(data_, buffer_size);
+ }
+ this->clear();
+ }
+
+ public:
+ explicit iterator_buffer(T* out, size_t n = buffer_size)
+ : fixed_buffer_traits(n), buffer<T>(out, 0, n), out_(out) {}
+ iterator_buffer(iterator_buffer&& other)
+ : fixed_buffer_traits(other),
+ buffer<T>(std::move(other)),
+ out_(other.out_) {
+ if (this->data() != out_) {
+ this->set(data_, buffer_size);
+ this->clear();
+ }
+ }
+ ~iterator_buffer() { flush(); }
+
+ auto out() -> T* {
+ flush();
+ return out_;
+ }
+ auto count() const -> size_t {
+ return fixed_buffer_traits::count() + this->size();
+ }
+};
+
+template <typename T> class iterator_buffer<T*, T> final : public buffer<T> {
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t) override {}
+
+ public:
+ explicit iterator_buffer(T* out, size_t = 0) : buffer<T>(out, 0, ~size_t()) {}
+
+ auto out() -> T* { return &*this->end(); }
+};
+
+// A buffer that writes to a container with the contiguous storage.
+template <typename Container>
+class iterator_buffer<std::back_insert_iterator<Container>,
+ enable_if_t<is_contiguous<Container>::value,
+ typename Container::value_type>>
+ final : public buffer<typename Container::value_type> {
+ private:
+ Container& container_;
+
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t capacity) override {
+ container_.resize(capacity);
+ this->set(&container_[0], capacity);
+ }
+
+ public:
+ explicit iterator_buffer(Container& c)
+ : buffer<typename Container::value_type>(c.size()), container_(c) {}
+ explicit iterator_buffer(std::back_insert_iterator<Container> out, size_t = 0)
+ : iterator_buffer(get_container(out)) {}
+
+ auto out() -> std::back_insert_iterator<Container> {
+ return std::back_inserter(container_);
+ }
+};
+
+// A buffer that counts the number of code units written discarding the output.
+template <typename T = char> class counting_buffer final : public buffer<T> {
+ private:
+ enum { buffer_size = 256 };
+ T data_[buffer_size];
+ size_t count_ = 0;
+
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t) override {
+ if (this->size() != buffer_size) return;
+ count_ += this->size();
+ this->clear();
+ }
+
+ public:
+ counting_buffer() : buffer<T>(data_, 0, buffer_size) {}
+
+ auto count() -> size_t { return count_ + this->size(); }
+};
+
+template <typename T>
+using buffer_appender = conditional_t<std::is_same<T, char>::value, appender,
+ std::back_insert_iterator<buffer<T>>>;
+
+// Maps an output iterator to a buffer.
+template <typename T, typename OutputIt>
+auto get_buffer(OutputIt out) -> iterator_buffer<OutputIt, T> {
+ return iterator_buffer<OutputIt, T>(out);
+}
+template <typename T, typename Buf,
+ FMT_ENABLE_IF(std::is_base_of<buffer<char>, Buf>::value)>
+auto get_buffer(std::back_insert_iterator<Buf> out) -> buffer<char>& {
+ return get_container(out);
+}
+
+template <typename Buf, typename OutputIt>
+FMT_INLINE auto get_iterator(Buf& buf, OutputIt) -> decltype(buf.out()) {
+ return buf.out();
+}
+template <typename T, typename OutputIt>
+auto get_iterator(buffer<T>&, OutputIt out) -> OutputIt {
+ return out;
+}
+
+struct view {};
+
+template <typename Char, typename T> struct named_arg : view {
+ const Char* name;
+ const T& value;
+ named_arg(const Char* n, const T& v) : name(n), value(v) {}
+};
+
+template <typename Char> struct named_arg_info {
+ const Char* name;
+ int id;
+};
+
+template <typename T, typename Char, size_t NUM_ARGS, size_t NUM_NAMED_ARGS>
+struct arg_data {
+ // args_[0].named_args points to named_args_ to avoid bloating format_args.
+ // +1 to workaround a bug in gcc 7.5 that causes duplicated-branches warning.
+ T args_[1 + (NUM_ARGS != 0 ? NUM_ARGS : +1)];
+ named_arg_info<Char> named_args_[NUM_NAMED_ARGS];
+
+ template <typename... U>
+ arg_data(const U&... init) : args_{T(named_args_, NUM_NAMED_ARGS), init...} {}
+ arg_data(const arg_data& other) = delete;
+ auto args() const -> const T* { return args_ + 1; }
+ auto named_args() -> named_arg_info<Char>* { return named_args_; }
+};
+
+template <typename T, typename Char, size_t NUM_ARGS>
+struct arg_data<T, Char, NUM_ARGS, 0> {
+ // +1 to workaround a bug in gcc 7.5 that causes duplicated-branches warning.
+ T args_[NUM_ARGS != 0 ? NUM_ARGS : +1];
+
+ template <typename... U>
+ FMT_CONSTEXPR FMT_INLINE arg_data(const U&... init) : args_{init...} {}
+ FMT_CONSTEXPR FMT_INLINE auto args() const -> const T* { return args_; }
+ FMT_CONSTEXPR FMT_INLINE auto named_args() -> std::nullptr_t {
+ return nullptr;
+ }
+};
+
+template <typename Char>
+inline void init_named_args(named_arg_info<Char>*, int, int) {}
+
+template <typename T> struct is_named_arg : std::false_type {};
+template <typename T> struct is_statically_named_arg : std::false_type {};
+
+template <typename T, typename Char>
+struct is_named_arg<named_arg<Char, T>> : std::true_type {};
+
+template <typename Char, typename T, typename... Tail,
+ FMT_ENABLE_IF(!is_named_arg<T>::value)>
+void init_named_args(named_arg_info<Char>* named_args, int arg_count,
+ int named_arg_count, const T&, const Tail&... args) {
+ init_named_args(named_args, arg_count + 1, named_arg_count, args...);
+}
+
+template <typename Char, typename T, typename... Tail,
+ FMT_ENABLE_IF(is_named_arg<T>::value)>
+void init_named_args(named_arg_info<Char>* named_args, int arg_count,
+ int named_arg_count, const T& arg, const Tail&... args) {
+ named_args[named_arg_count++] = {arg.name, arg_count};
+ init_named_args(named_args, arg_count + 1, named_arg_count, args...);
+}
+
+template <typename... Args>
+FMT_CONSTEXPR FMT_INLINE void init_named_args(std::nullptr_t, int, int,
+ const Args&...) {}
+
+template <bool B = false> constexpr auto count() -> size_t { return B ? 1 : 0; }
+template <bool B1, bool B2, bool... Tail> constexpr auto count() -> size_t {
+ return (B1 ? 1 : 0) + count<B2, Tail...>();
+}
+
+template <typename... Args> constexpr auto count_named_args() -> size_t {
+ return count<is_named_arg<Args>::value...>();
+}
+
+template <typename... Args>
+constexpr auto count_statically_named_args() -> size_t {
+ return count<is_statically_named_arg<Args>::value...>();
+}
+
+struct unformattable {};
+struct unformattable_char : unformattable {};
+struct unformattable_pointer : unformattable {};
+
+template <typename Char> struct string_value {
+ const Char* data;
+ size_t size;
+};
+
+template <typename Char> struct named_arg_value {
+ const named_arg_info<Char>* data;
+ size_t size;
+};
+
+template <typename Context> struct custom_value {
+ using parse_context = typename Context::parse_context_type;
+ void* value;
+ void (*format)(void* arg, parse_context& parse_ctx, Context& ctx);
+};
+
+// A formatting argument value.
+template <typename Context> class value {
+ public:
+ using char_type = typename Context::char_type;
+
+ union {
+ monostate no_value;
+ int int_value;
+ unsigned uint_value;
+ long long long_long_value;
+ unsigned long long ulong_long_value;
+ int128_opt int128_value;
+ uint128_opt uint128_value;
+ bool bool_value;
+ char_type char_value;
+ float float_value;
+ double double_value;
+ long double long_double_value;
+ const void* pointer;
+ string_value<char_type> string;
+ custom_value<Context> custom;
+ named_arg_value<char_type> named_args;
+ };
+
+ constexpr FMT_INLINE value() : no_value() {}
+ constexpr FMT_INLINE value(int val) : int_value(val) {}
+ constexpr FMT_INLINE value(unsigned val) : uint_value(val) {}
+ constexpr FMT_INLINE value(long long val) : long_long_value(val) {}
+ constexpr FMT_INLINE value(unsigned long long val) : ulong_long_value(val) {}
+ FMT_INLINE value(int128_opt val) : int128_value(val) {}
+ FMT_INLINE value(uint128_opt val) : uint128_value(val) {}
+ constexpr FMT_INLINE value(float val) : float_value(val) {}
+ constexpr FMT_INLINE value(double val) : double_value(val) {}
+ FMT_INLINE value(long double val) : long_double_value(val) {}
+ constexpr FMT_INLINE value(bool val) : bool_value(val) {}
+ constexpr FMT_INLINE value(char_type val) : char_value(val) {}
+ FMT_CONSTEXPR FMT_INLINE value(const char_type* val) {
+ string.data = val;
+ if (is_constant_evaluated()) string.size = {};
+ }
+ FMT_CONSTEXPR FMT_INLINE value(basic_string_view<char_type> val) {
+ string.data = val.data();
+ string.size = val.size();
+ }
+ FMT_INLINE value(const void* val) : pointer(val) {}
+ FMT_INLINE value(const named_arg_info<char_type>* args, size_t size)
+ : named_args{args, size} {}
+
+ template <typename T> FMT_CONSTEXPR FMT_INLINE value(T& val) {
+ using value_type = remove_cvref_t<T>;
+ custom.value = const_cast<value_type*>(&val);
+ // Get the formatter type through the context to allow different contexts
+ // have different extension points, e.g. `formatter<T>` for `format` and
+ // `printf_formatter<T>` for `printf`.
+ custom.format = format_custom_arg<
+ value_type, typename Context::template formatter_type<value_type>>;
+ }
+ value(unformattable);
+ value(unformattable_char);
+ value(unformattable_pointer);
+
+ private:
+ // Formats an argument of a custom type, such as a user-defined class.
+ template <typename T, typename Formatter>
+ static void format_custom_arg(void* arg,
+ typename Context::parse_context_type& parse_ctx,
+ Context& ctx) {
+ auto f = Formatter();
+ parse_ctx.advance_to(f.parse(parse_ctx));
+ using qualified_type =
+ conditional_t<has_const_formatter<T, Context>(), const T, T>;
+ ctx.advance_to(f.format(*static_cast<qualified_type*>(arg), ctx));
+ }
+};
+
+template <typename Context, typename T>
+FMT_CONSTEXPR auto make_arg(T&& value) -> basic_format_arg<Context>;
+
+// To minimize the number of types we need to deal with, long is translated
+// either to int or to long long depending on its size.
+enum { long_short = sizeof(long) == sizeof(int) };
+using long_type = conditional_t<long_short, int, long long>;
+using ulong_type = conditional_t<long_short, unsigned, unsigned long long>;
+
+template <typename T> struct format_as_result {
+ template <typename U,
+ FMT_ENABLE_IF(std::is_enum<U>::value || std::is_class<U>::value)>
+ static auto map(U*) -> decltype(format_as(std::declval<U>()));
+ static auto map(...) -> void;
+
+ using type = decltype(map(static_cast<T*>(nullptr)));
+};
+template <typename T> using format_as_t = typename format_as_result<T>::type;
+
+template <typename T>
+struct has_format_as
+ : bool_constant<!std::is_same<format_as_t<T>, void>::value> {};
+
+// Maps formatting arguments to core types.
+// arg_mapper reports errors by returning unformattable instead of using
+// static_assert because it's used in the is_formattable trait.
+template <typename Context> struct arg_mapper {
+ using char_type = typename Context::char_type;
+
+ FMT_CONSTEXPR FMT_INLINE auto map(signed char val) -> int { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(unsigned char val) -> unsigned {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(short val) -> int { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(unsigned short val) -> unsigned {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(int val) -> int { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(unsigned val) -> unsigned { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(long val) -> long_type { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(unsigned long val) -> ulong_type {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(long long val) -> long long { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(unsigned long long val)
+ -> unsigned long long {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(int128_opt val) -> int128_opt {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(uint128_opt val) -> uint128_opt {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(bool val) -> bool { return val; }
+
+ template <typename T, FMT_ENABLE_IF(std::is_same<T, char>::value ||
+ std::is_same<T, char_type>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(T val) -> char_type {
+ return val;
+ }
+ template <typename T, enable_if_t<(std::is_same<T, wchar_t>::value ||
+#ifdef __cpp_char8_t
+ std::is_same<T, char8_t>::value ||
+#endif
+ std::is_same<T, char16_t>::value ||
+ std::is_same<T, char32_t>::value) &&
+ !std::is_same<T, char_type>::value,
+ int> = 0>
+ FMT_CONSTEXPR FMT_INLINE auto map(T) -> unformattable_char {
+ return {};
+ }
+
+ FMT_CONSTEXPR FMT_INLINE auto map(float val) -> float { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(double val) -> double { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(long double val) -> long double {
+ return val;
+ }
+
+ FMT_CONSTEXPR FMT_INLINE auto map(char_type* val) -> const char_type* {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(const char_type* val) -> const char_type* {
+ return val;
+ }
+ template <typename T,
+ FMT_ENABLE_IF(is_string<T>::value && !std::is_pointer<T>::value &&
+ std::is_same<char_type, char_t<T>>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(const T& val)
+ -> basic_string_view<char_type> {
+ return to_string_view(val);
+ }
+ template <typename T,
+ FMT_ENABLE_IF(is_string<T>::value && !std::is_pointer<T>::value &&
+ !std::is_same<char_type, char_t<T>>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(const T&) -> unformattable_char {
+ return {};
+ }
+
+ FMT_CONSTEXPR FMT_INLINE auto map(void* val) -> const void* { return val; }
+ FMT_CONSTEXPR FMT_INLINE auto map(const void* val) -> const void* {
+ return val;
+ }
+ FMT_CONSTEXPR FMT_INLINE auto map(std::nullptr_t val) -> const void* {
+ return val;
+ }
+
+ // Use SFINAE instead of a const T* parameter to avoid a conflict with the
+ // array overload.
+ template <
+ typename T,
+ FMT_ENABLE_IF(
+ std::is_pointer<T>::value || std::is_member_pointer<T>::value ||
+ std::is_function<typename std::remove_pointer<T>::type>::value ||
+ (std::is_convertible<const T&, const void*>::value &&
+ !std::is_convertible<const T&, const char_type*>::value &&
+ !has_formatter<T, Context>::value))>
+ FMT_CONSTEXPR auto map(const T&) -> unformattable_pointer {
+ return {};
+ }
+
+ template <typename T, std::size_t N,
+ FMT_ENABLE_IF(!std::is_same<T, wchar_t>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(const T (&values)[N]) -> const T (&)[N] {
+ return values;
+ }
+
+ // Only map owning types because mapping views can be unsafe.
+ template <typename T, typename U = format_as_t<T>,
+ FMT_ENABLE_IF(std::is_arithmetic<U>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(const T& val) -> decltype(this->map(U())) {
+ return map(format_as(val));
+ }
+
+ template <typename T, typename U = remove_cvref_t<T>>
+ struct formattable
+ : bool_constant<has_const_formatter<U, Context>() ||
+ (has_formatter<U, Context>::value &&
+ !std::is_const<remove_reference_t<T>>::value)> {};
+
+ template <typename T, FMT_ENABLE_IF(formattable<T>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto do_map(T&& val) -> T& {
+ return val;
+ }
+ template <typename T, FMT_ENABLE_IF(!formattable<T>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto do_map(T&&) -> unformattable {
+ return {};
+ }
+
+ template <typename T, typename U = remove_cvref_t<T>,
+ FMT_ENABLE_IF((std::is_class<U>::value || std::is_enum<U>::value ||
+ std::is_union<U>::value) &&
+ !is_string<U>::value && !is_char<U>::value &&
+ !is_named_arg<U>::value &&
+ !std::is_arithmetic<format_as_t<U>>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(T&& val)
+ -> decltype(this->do_map(std::forward<T>(val))) {
+ return do_map(std::forward<T>(val));
+ }
+
+ template <typename T, FMT_ENABLE_IF(is_named_arg<T>::value)>
+ FMT_CONSTEXPR FMT_INLINE auto map(const T& named_arg)
+ -> decltype(this->map(named_arg.value)) {
+ return map(named_arg.value);
+ }
+
+ auto map(...) -> unformattable { return {}; }
+};
+
+// A type constant after applying arg_mapper<Context>.
+template <typename T, typename Context>
+using mapped_type_constant =
+ type_constant<decltype(arg_mapper<Context>().map(std::declval<const T&>())),
+ typename Context::char_type>;
+
+enum { packed_arg_bits = 4 };
+// Maximum number of arguments with packed types.
+enum { max_packed_args = 62 / packed_arg_bits };
+enum : unsigned long long { is_unpacked_bit = 1ULL << 63 };
+enum : unsigned long long { has_named_args_bit = 1ULL << 62 };
+} // namespace detail
+
+// An output iterator that appends to a buffer.
+// It is used to reduce symbol sizes for the common case.
+class appender : public std::back_insert_iterator<detail::buffer<char>> {
+ using base = std::back_insert_iterator<detail::buffer<char>>;
+
+ public:
+ using std::back_insert_iterator<detail::buffer<char>>::back_insert_iterator;
+ appender(base it) noexcept : base(it) {}
+ FMT_UNCHECKED_ITERATOR(appender);
+
+ auto operator++() noexcept -> appender& { return *this; }
+ auto operator++(int) noexcept -> appender { return *this; }
+};
+
+// A formatting argument. It is a trivially copyable/constructible type to
+// allow storage in basic_memory_buffer.
+template <typename Context> class basic_format_arg {
+ private:
+ detail::value<Context> value_;
+ detail::type type_;
+
+ template <typename ContextType, typename T>
+ friend FMT_CONSTEXPR auto detail::make_arg(T&& value)
+ -> basic_format_arg<ContextType>;
+
+ template <typename Visitor, typename Ctx>
+ friend FMT_CONSTEXPR auto visit_format_arg(Visitor&& vis,
+ const basic_format_arg<Ctx>& arg)
+ -> decltype(vis(0));
+
+ friend class basic_format_args<Context>;
+ friend class dynamic_format_arg_store<Context>;
+
+ using char_type = typename Context::char_type;
+
+ template <typename T, typename Char, size_t NUM_ARGS, size_t NUM_NAMED_ARGS>
+ friend struct detail::arg_data;
+
+ basic_format_arg(const detail::named_arg_info<char_type>* args, size_t size)
+ : value_(args, size) {}
+
+ public:
+ class handle {
+ public:
+ explicit handle(detail::custom_value<Context> custom) : custom_(custom) {}
+
+ void format(typename Context::parse_context_type& parse_ctx,
+ Context& ctx) const {
+ custom_.format(custom_.value, parse_ctx, ctx);
+ }
+
+ private:
+ detail::custom_value<Context> custom_;
+ };
+
+ constexpr basic_format_arg() : type_(detail::type::none_type) {}
+
+ constexpr explicit operator bool() const noexcept {
+ return type_ != detail::type::none_type;
+ }
+
+ auto type() const -> detail::type { return type_; }
+
+ auto is_integral() const -> bool { return detail::is_integral_type(type_); }
+ auto is_arithmetic() const -> bool {
+ return detail::is_arithmetic_type(type_);
+ }
+};
+
+/**
+ \rst
+ Visits an argument dispatching to the appropriate visit method based on
+ the argument type. For example, if the argument type is ``double`` then
+ ``vis(value)`` will be called with the value of type ``double``.
+ \endrst
+ */
+FMT_MODULE_EXPORT
+template <typename Visitor, typename Context>
+FMT_CONSTEXPR FMT_INLINE auto visit_format_arg(
+ Visitor&& vis, const basic_format_arg<Context>& arg) -> decltype(vis(0)) {
+ switch (arg.type_) {
+ case detail::type::none_type:
+ break;
+ case detail::type::int_type:
+ return vis(arg.value_.int_value);
+ case detail::type::uint_type:
+ return vis(arg.value_.uint_value);
+ case detail::type::long_long_type:
+ return vis(arg.value_.long_long_value);
+ case detail::type::ulong_long_type:
+ return vis(arg.value_.ulong_long_value);
+ case detail::type::int128_type:
+ return vis(detail::convert_for_visit(arg.value_.int128_value));
+ case detail::type::uint128_type:
+ return vis(detail::convert_for_visit(arg.value_.uint128_value));
+ case detail::type::bool_type:
+ return vis(arg.value_.bool_value);
+ case detail::type::char_type:
+ return vis(arg.value_.char_value);
+ case detail::type::float_type:
+ return vis(arg.value_.float_value);
+ case detail::type::double_type:
+ return vis(arg.value_.double_value);
+ case detail::type::long_double_type:
+ return vis(arg.value_.long_double_value);
+ case detail::type::cstring_type:
+ return vis(arg.value_.string.data);
+ case detail::type::string_type:
+ using sv = basic_string_view<typename Context::char_type>;
+ return vis(sv(arg.value_.string.data, arg.value_.string.size));
+ case detail::type::pointer_type:
+ return vis(arg.value_.pointer);
+ case detail::type::custom_type:
+ return vis(typename basic_format_arg<Context>::handle(arg.value_.custom));
+ }
+ return vis(monostate());
+}
+
+namespace detail {
+
+template <typename Char, typename InputIt>
+auto copy_str(InputIt begin, InputIt end, appender out) -> appender {
+ get_container(out).append(begin, end);
+ return out;
+}
+
+template <typename Char, typename R, typename OutputIt>
+FMT_CONSTEXPR auto copy_str(R&& rng, OutputIt out) -> OutputIt {
+ return detail::copy_str<Char>(rng.begin(), rng.end(), out);
+}
+
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 500
+// A workaround for gcc 4.8 to make void_t work in a SFINAE context.
+template <typename...> struct void_t_impl { using type = void; };
+template <typename... T> using void_t = typename void_t_impl<T...>::type;
+#else
+template <typename...> using void_t = void;
+#endif
+
+template <typename It, typename T, typename Enable = void>
+struct is_output_iterator : std::false_type {};
+
+template <typename It, typename T>
+struct is_output_iterator<
+ It, T,
+ void_t<typename std::iterator_traits<It>::iterator_category,
+ decltype(*std::declval<It>() = std::declval<T>())>>
+ : std::true_type {};
+
+template <typename It> struct is_back_insert_iterator : std::false_type {};
+template <typename Container>
+struct is_back_insert_iterator<std::back_insert_iterator<Container>>
+ : std::true_type {};
+
+template <typename It>
+struct is_contiguous_back_insert_iterator : std::false_type {};
+template <typename Container>
+struct is_contiguous_back_insert_iterator<std::back_insert_iterator<Container>>
+ : is_contiguous<Container> {};
+template <>
+struct is_contiguous_back_insert_iterator<appender> : std::true_type {};
+
+// A type-erased reference to an std::locale to avoid a heavy <locale> include.
+class locale_ref {
+ private:
+ const void* locale_; // A type-erased pointer to std::locale.
+
+ public:
+ constexpr FMT_INLINE locale_ref() : locale_(nullptr) {}
+ template <typename Locale> explicit locale_ref(const Locale& loc);
+
+ explicit operator bool() const noexcept { return locale_ != nullptr; }
+
+ template <typename Locale> auto get() const -> Locale;
+};
+
+template <typename> constexpr auto encode_types() -> unsigned long long {
+ return 0;
+}
+
+template <typename Context, typename Arg, typename... Args>
+constexpr auto encode_types() -> unsigned long long {
+ return static_cast<unsigned>(mapped_type_constant<Arg, Context>::value) |
+ (encode_types<Context, Args...>() << packed_arg_bits);
+}
+
+template <typename Context, typename T>
+FMT_CONSTEXPR FMT_INLINE auto make_value(T&& val) -> value<Context> {
+ auto&& arg = arg_mapper<Context>().map(FMT_FORWARD(val));
+ using arg_type = remove_cvref_t<decltype(arg)>;
+
+ constexpr bool formattable_char =
+ !std::is_same<arg_type, unformattable_char>::value;
+ static_assert(formattable_char, "Mixing character types is disallowed.");
+
+ // Formatting of arbitrary pointers is disallowed. If you want to format a
+ // pointer cast it to `void*` or `const void*`. In particular, this forbids
+ // formatting of `[const] volatile char*` printed as bool by iostreams.
+ constexpr bool formattable_pointer =
+ !std::is_same<arg_type, unformattable_pointer>::value;
+ static_assert(formattable_pointer,
+ "Formatting of non-void pointers is disallowed.");
+
+ constexpr bool formattable = !std::is_same<arg_type, unformattable>::value;
+ static_assert(
+ formattable,
+ "Cannot format an argument. To make type T formattable provide a "
+ "formatter<T> specialization: https://fmt.dev/latest/api.html#udt");
+ return {arg};
+}
+
+template <typename Context, typename T>
+FMT_CONSTEXPR auto make_arg(T&& value) -> basic_format_arg<Context> {
+ auto arg = basic_format_arg<Context>();
+ arg.type_ = mapped_type_constant<T, Context>::value;
+ arg.value_ = make_value<Context>(value);
+ return arg;
+}
+
+// The DEPRECATED type template parameter is there to avoid an ODR violation
+// when using a fallback formatter in one translation unit and an implicit
+// conversion in another (not recommended).
+template <bool IS_PACKED, typename Context, type, typename T,
+ FMT_ENABLE_IF(IS_PACKED)>
+FMT_CONSTEXPR FMT_INLINE auto make_arg(T&& val) -> value<Context> {
+ return make_value<Context>(val);
+}
+
+template <bool IS_PACKED, typename Context, type, typename T,
+ FMT_ENABLE_IF(!IS_PACKED)>
+FMT_CONSTEXPR inline auto make_arg(T&& value) -> basic_format_arg<Context> {
+ return make_arg<Context>(value);
+}
+} // namespace detail
+FMT_BEGIN_EXPORT
+
+// Formatting context.
+template <typename OutputIt, typename Char> class basic_format_context {
+ private:
+ OutputIt out_;
+ basic_format_args<basic_format_context> args_;
+ detail::locale_ref loc_;
+
+ public:
+ using iterator = OutputIt;
+ using format_arg = basic_format_arg<basic_format_context>;
+ using format_args = basic_format_args<basic_format_context>;
+ using parse_context_type = basic_format_parse_context<Char>;
+ template <typename T> using formatter_type = formatter<T, Char>;
+
+ /** The character type for the output. */
+ using char_type = Char;
+
+ basic_format_context(basic_format_context&&) = default;
+ basic_format_context(const basic_format_context&) = delete;
+ void operator=(const basic_format_context&) = delete;
+ /**
+ Constructs a ``basic_format_context`` object. References to the arguments
+ are stored in the object so make sure they have appropriate lifetimes.
+ */
+ constexpr basic_format_context(OutputIt out, format_args ctx_args,
+ detail::locale_ref loc = {})
+ : out_(out), args_(ctx_args), loc_(loc) {}
+
+ constexpr auto arg(int id) const -> format_arg { return args_.get(id); }
+ FMT_CONSTEXPR auto arg(basic_string_view<Char> name) -> format_arg {
+ return args_.get(name);
+ }
+ FMT_CONSTEXPR auto arg_id(basic_string_view<Char> name) -> int {
+ return args_.get_id(name);
+ }
+ auto args() const -> const format_args& { return args_; }
+
+ FMT_CONSTEXPR auto error_handler() -> detail::error_handler { return {}; }
+ void on_error(const char* message) { error_handler().on_error(message); }
+
+ // Returns an iterator to the beginning of the output range.
+ FMT_CONSTEXPR auto out() -> iterator { return out_; }
+
+ // Advances the begin iterator to ``it``.
+ void advance_to(iterator it) {
+ if (!detail::is_back_insert_iterator<iterator>()) out_ = it;
+ }
+
+ FMT_CONSTEXPR auto locale() -> detail::locale_ref { return loc_; }
+};
+
+template <typename Char>
+using buffer_context =
+ basic_format_context<detail::buffer_appender<Char>, Char>;
+using format_context = buffer_context<char>;
+
+template <typename T, typename Char = char>
+using is_formattable = bool_constant<!std::is_base_of<
+ detail::unformattable, decltype(detail::arg_mapper<buffer_context<Char>>()
+ .map(std::declval<T>()))>::value>;
+
+/**
+ \rst
+ An array of references to arguments. It can be implicitly converted into
+ `~fmt::basic_format_args` for passing into type-erased formatting functions
+ such as `~fmt::vformat`.
+ \endrst
+ */
+template <typename Context, typename... Args>
+class format_arg_store
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+ // Workaround a GCC template argument substitution bug.
+ : public basic_format_args<Context>
+#endif
+{
+ private:
+ static const size_t num_args = sizeof...(Args);
+ static const size_t num_named_args = detail::count_named_args<Args...>();
+ static const bool is_packed = num_args <= detail::max_packed_args;
+
+ using value_type = conditional_t<is_packed, detail::value<Context>,
+ basic_format_arg<Context>>;
+
+ detail::arg_data<value_type, typename Context::char_type, num_args,
+ num_named_args>
+ data_;
+
+ friend class basic_format_args<Context>;
+
+ static constexpr unsigned long long desc =
+ (is_packed ? detail::encode_types<Context, Args...>()
+ : detail::is_unpacked_bit | num_args) |
+ (num_named_args != 0
+ ? static_cast<unsigned long long>(detail::has_named_args_bit)
+ : 0);
+
+ public:
+ template <typename... T>
+ FMT_CONSTEXPR FMT_INLINE format_arg_store(T&&... args)
+ :
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+ basic_format_args<Context>(*this),
+#endif
+ data_{detail::make_arg<
+ is_packed, Context,
+ detail::mapped_type_constant<remove_cvref_t<T>, Context>::value>(
+ FMT_FORWARD(args))...} {
+ detail::init_named_args(data_.named_args(), 0, 0, args...);
+ }
+};
+
+/**
+ \rst
+ Constructs a `~fmt::format_arg_store` object that contains references to
+ arguments and can be implicitly converted to `~fmt::format_args`. `Context`
+ can be omitted in which case it defaults to `~fmt::context`.
+ See `~fmt::arg` for lifetime considerations.
+ \endrst
+ */
+template <typename Context = format_context, typename... T>
+constexpr auto make_format_args(T&&... args)
+ -> format_arg_store<Context, remove_cvref_t<T>...> {
+ return {FMT_FORWARD(args)...};
+}
+
+/**
+ \rst
+ Returns a named argument to be used in a formatting function.
+ It should only be used in a call to a formatting function or
+ `dynamic_format_arg_store::push_back`.
+
+ **Example**::
+
+ fmt::print("Elapsed time: {s:.2f} seconds", fmt::arg("s", 1.23));
+ \endrst
+ */
+template <typename Char, typename T>
+inline auto arg(const Char* name, const T& arg) -> detail::named_arg<Char, T> {
+ static_assert(!detail::is_named_arg<T>(), "nested named arguments");
+ return {name, arg};
+}
+FMT_END_EXPORT
+
+/**
+ \rst
+ A view of a collection of formatting arguments. To avoid lifetime issues it
+ should only be used as a parameter type in type-erased functions such as
+ ``vformat``::
+
+ void vlog(string_view format_str, format_args args); // OK
+ format_args args = make_format_args(42); // Error: dangling reference
+ \endrst
+ */
+template <typename Context> class basic_format_args {
+ public:
+ using size_type = int;
+ using format_arg = basic_format_arg<Context>;
+
+ private:
+ // A descriptor that contains information about formatting arguments.
+ // If the number of arguments is less or equal to max_packed_args then
+ // argument types are passed in the descriptor. This reduces binary code size
+ // per formatting function call.
+ unsigned long long desc_;
+ union {
+ // If is_packed() returns true then argument values are stored in values_;
+ // otherwise they are stored in args_. This is done to improve cache
+ // locality and reduce compiled code size since storing larger objects
+ // may require more code (at least on x86-64) even if the same amount of
+ // data is actually copied to stack. It saves ~10% on the bloat test.
+ const detail::value<Context>* values_;
+ const format_arg* args_;
+ };
+
+ constexpr auto is_packed() const -> bool {
+ return (desc_ & detail::is_unpacked_bit) == 0;
+ }
+ auto has_named_args() const -> bool {
+ return (desc_ & detail::has_named_args_bit) != 0;
+ }
+
+ FMT_CONSTEXPR auto type(int index) const -> detail::type {
+ int shift = index * detail::packed_arg_bits;
+ unsigned int mask = (1 << detail::packed_arg_bits) - 1;
+ return static_cast<detail::type>((desc_ >> shift) & mask);
+ }
+
+ constexpr FMT_INLINE basic_format_args(unsigned long long desc,
+ const detail::value<Context>* values)
+ : desc_(desc), values_(values) {}
+ constexpr basic_format_args(unsigned long long desc, const format_arg* args)
+ : desc_(desc), args_(args) {}
+
+ public:
+ constexpr basic_format_args() : desc_(0), args_(nullptr) {}
+
+ /**
+ \rst
+ Constructs a `basic_format_args` object from `~fmt::format_arg_store`.
+ \endrst
+ */
+ template <typename... Args>
+ constexpr FMT_INLINE basic_format_args(
+ const format_arg_store<Context, Args...>& store)
+ : basic_format_args(format_arg_store<Context, Args...>::desc,
+ store.data_.args()) {}
+
+ /**
+ \rst
+ Constructs a `basic_format_args` object from
+ `~fmt::dynamic_format_arg_store`.
+ \endrst
+ */
+ constexpr FMT_INLINE basic_format_args(
+ const dynamic_format_arg_store<Context>& store)
+ : basic_format_args(store.get_types(), store.data()) {}
+
+ /**
+ \rst
+ Constructs a `basic_format_args` object from a dynamic set of arguments.
+ \endrst
+ */
+ constexpr basic_format_args(const format_arg* args, int count)
+ : basic_format_args(detail::is_unpacked_bit | detail::to_unsigned(count),
+ args) {}
+
+ /** Returns the argument with the specified id. */
+ FMT_CONSTEXPR auto get(int id) const -> format_arg {
+ format_arg arg;
+ if (!is_packed()) {
+ if (id < max_size()) arg = args_[id];
+ return arg;
+ }
+ if (id >= detail::max_packed_args) return arg;
+ arg.type_ = type(id);
+ if (arg.type_ == detail::type::none_type) return arg;
+ arg.value_ = values_[id];
+ return arg;
+ }
+
+ template <typename Char>
+ auto get(basic_string_view<Char> name) const -> format_arg {
+ int id = get_id(name);
+ return id >= 0 ? get(id) : format_arg();
+ }
+
+ template <typename Char>
+ auto get_id(basic_string_view<Char> name) const -> int {
+ if (!has_named_args()) return -1;
+ const auto& named_args =
+ (is_packed() ? values_[-1] : args_[-1].value_).named_args;
+ for (size_t i = 0; i < named_args.size; ++i) {
+ if (named_args.data[i].name == name) return named_args.data[i].id;
+ }
+ return -1;
+ }
+
+ auto max_size() const -> int {
+ unsigned long long max_packed = detail::max_packed_args;
+ return static_cast<int>(is_packed() ? max_packed
+ : desc_ & ~detail::is_unpacked_bit);
+ }
+};
+
+/** An alias to ``basic_format_args<format_context>``. */
+// A separate type would result in shorter symbols but break ABI compatibility
+// between clang and gcc on ARM (#1919).
+FMT_MODULE_EXPORT using format_args = basic_format_args<format_context>;
+
+// We cannot use enum classes as bit fields because of a gcc bug, so we put them
+// in namespaces instead (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414).
+// Additionally, if an underlying type is specified, older gcc incorrectly warns
+// that the type is too small. Both bugs are fixed in gcc 9.3.
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 903
+# define FMT_ENUM_UNDERLYING_TYPE(type)
+#else
+# define FMT_ENUM_UNDERLYING_TYPE(type) : type
+#endif
+namespace align {
+enum type FMT_ENUM_UNDERLYING_TYPE(unsigned char){none, left, right, center,
+ numeric};
+}
+using align_t = align::type;
+namespace sign {
+enum type FMT_ENUM_UNDERLYING_TYPE(unsigned char){none, minus, plus, space};
+}
+using sign_t = sign::type;
+
+namespace detail {
+
+// Workaround an array initialization issue in gcc 4.8.
+template <typename Char> struct fill_t {
+ private:
+ enum { max_size = 4 };
+ Char data_[max_size] = {Char(' '), Char(0), Char(0), Char(0)};
+ unsigned char size_ = 1;
+
+ public:
+ FMT_CONSTEXPR void operator=(basic_string_view<Char> s) {
+ auto size = s.size();
+ FMT_ASSERT(size <= max_size, "invalid fill");
+ for (size_t i = 0; i < size; ++i) data_[i] = s[i];
+ size_ = static_cast<unsigned char>(size);
+ }
+
+ constexpr auto size() const -> size_t { return size_; }
+ constexpr auto data() const -> const Char* { return data_; }
+
+ FMT_CONSTEXPR auto operator[](size_t index) -> Char& { return data_[index]; }
+ FMT_CONSTEXPR auto operator[](size_t index) const -> const Char& {
+ return data_[index];
+ }
+};
+} // namespace detail
+
+enum class presentation_type : unsigned char {
+ none,
+ dec, // 'd'
+ oct, // 'o'
+ hex_lower, // 'x'
+ hex_upper, // 'X'
+ bin_lower, // 'b'
+ bin_upper, // 'B'
+ hexfloat_lower, // 'a'
+ hexfloat_upper, // 'A'
+ exp_lower, // 'e'
+ exp_upper, // 'E'
+ fixed_lower, // 'f'
+ fixed_upper, // 'F'
+ general_lower, // 'g'
+ general_upper, // 'G'
+ chr, // 'c'
+ string, // 's'
+ pointer, // 'p'
+ debug // '?'
+};
+
+// Format specifiers for built-in and string types.
+template <typename Char = char> struct format_specs {
+ int width;
+ int precision;
+ presentation_type type;
+ align_t align : 4;
+ sign_t sign : 3;
+ bool alt : 1; // Alternate form ('#').
+ bool localized : 1;
+ detail::fill_t<Char> fill;
+
+ constexpr format_specs()
+ : width(0),
+ precision(-1),
+ type(presentation_type::none),
+ align(align::none),
+ sign(sign::none),
+ alt(false),
+ localized(false) {}
+};
+
+namespace detail {
+
+enum class arg_id_kind { none, index, name };
+
+// An argument reference.
+template <typename Char> struct arg_ref {
+ FMT_CONSTEXPR arg_ref() : kind(arg_id_kind::none), val() {}
+
+ FMT_CONSTEXPR explicit arg_ref(int index)
+ : kind(arg_id_kind::index), val(index) {}
+ FMT_CONSTEXPR explicit arg_ref(basic_string_view<Char> name)
+ : kind(arg_id_kind::name), val(name) {}
+
+ FMT_CONSTEXPR auto operator=(int idx) -> arg_ref& {
+ kind = arg_id_kind::index;
+ val.index = idx;
+ return *this;
+ }
+
+ arg_id_kind kind;
+ union value {
+ FMT_CONSTEXPR value(int idx = 0) : index(idx) {}
+ FMT_CONSTEXPR value(basic_string_view<Char> n) : name(n) {}
+
+ int index;
+ basic_string_view<Char> name;
+ } val;
+};
+
+// Format specifiers with width and precision resolved at formatting rather
+// than parsing time to allow reusing the same parsed specifiers with
+// different sets of arguments (precompilation of format strings).
+template <typename Char = char>
+struct dynamic_format_specs : format_specs<Char> {
+ arg_ref<Char> width_ref;
+ arg_ref<Char> precision_ref;
+};
+
+// Converts a character to ASCII. Returns '\0' on conversion failure.
+template <typename Char, FMT_ENABLE_IF(std::is_integral<Char>::value)>
+constexpr auto to_ascii(Char c) -> char {
+ return c <= 0xff ? static_cast<char>(c) : '\0';
+}
+template <typename Char, FMT_ENABLE_IF(std::is_enum<Char>::value)>
+constexpr auto to_ascii(Char c) -> char {
+ return c <= 0xff ? static_cast<char>(c) : '\0';
+}
+
+// Returns the number of code units in a code point or 1 on error.
+template <typename Char>
+FMT_CONSTEXPR auto code_point_length(const Char* begin) -> int {
+ if (const_check(sizeof(Char) != 1)) return 1;
+ auto c = static_cast<unsigned char>(*begin);
+ return static_cast<int>((0x3a55000000000000ull >> (2 * (c >> 3))) & 0x3) + 1;
+}
+
+// Return the result via the out param to workaround gcc bug 77539.
+template <bool IS_CONSTEXPR, typename T, typename Ptr = const T*>
+FMT_CONSTEXPR auto find(Ptr first, Ptr last, T value, Ptr& out) -> bool {
+ for (out = first; out != last; ++out) {
+ if (*out == value) return true;
+ }
+ return false;
+}
+
+template <>
+inline auto find<false, char>(const char* first, const char* last, char value,
+ const char*& out) -> bool {
+ out = static_cast<const char*>(
+ std::memchr(first, value, to_unsigned(last - first)));
+ return out != nullptr;
+}
+
+// Parses the range [begin, end) as an unsigned integer. This function assumes
+// that the range is non-empty and the first character is a digit.
+template <typename Char>
+FMT_CONSTEXPR auto parse_nonnegative_int(const Char*& begin, const Char* end,
+ int error_value) noexcept -> int {
+ FMT_ASSERT(begin != end && '0' <= *begin && *begin <= '9', "");
+ unsigned value = 0, prev = 0;
+ auto p = begin;
+ do {
+ prev = value;
+ value = value * 10 + unsigned(*p - '0');
+ ++p;
+ } while (p != end && '0' <= *p && *p <= '9');
+ auto num_digits = p - begin;
+ begin = p;
+ if (num_digits <= std::numeric_limits<int>::digits10)
+ return static_cast<int>(value);
+ // Check for overflow.
+ const unsigned max = to_unsigned((std::numeric_limits<int>::max)());
+ return num_digits == std::numeric_limits<int>::digits10 + 1 &&
+ prev * 10ull + unsigned(p[-1] - '0') <= max
+ ? static_cast<int>(value)
+ : error_value;
+}
+
+FMT_CONSTEXPR inline auto parse_align(char c) -> align_t {
+ switch (c) {
+ case '<':
+ return align::left;
+ case '>':
+ return align::right;
+ case '^':
+ return align::center;
+ }
+ return align::none;
+}
+
+template <typename Char> constexpr auto is_name_start(Char c) -> bool {
+ return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '_';
+}
+
+template <typename Char, typename Handler>
+FMT_CONSTEXPR auto do_parse_arg_id(const Char* begin, const Char* end,
+ Handler&& handler) -> const Char* {
+ Char c = *begin;
+ if (c >= '0' && c <= '9') {
+ int index = 0;
+ constexpr int max = (std::numeric_limits<int>::max)();
+ if (c != '0')
+ index = parse_nonnegative_int(begin, end, max);
+ else
+ ++begin;
+ if (begin == end || (*begin != '}' && *begin != ':'))
+ throw_format_error("invalid format string");
+ else
+ handler.on_index(index);
+ return begin;
+ }
+ if (!is_name_start(c)) {
+ throw_format_error("invalid format string");
+ return begin;
+ }
+ auto it = begin;
+ do {
+ ++it;
+ } while (it != end && (is_name_start(*it) || ('0' <= *it && *it <= '9')));
+ handler.on_name({begin, to_unsigned(it - begin)});
+ return it;
+}
+
+template <typename Char, typename Handler>
+FMT_CONSTEXPR FMT_INLINE auto parse_arg_id(const Char* begin, const Char* end,
+ Handler&& handler) -> const Char* {
+ FMT_ASSERT(begin != end, "");
+ Char c = *begin;
+ if (c != '}' && c != ':') return do_parse_arg_id(begin, end, handler);
+ handler.on_auto();
+ return begin;
+}
+
+template <typename Char> struct dynamic_spec_id_handler {
+ basic_format_parse_context<Char>& ctx;
+ arg_ref<Char>& ref;
+
+ FMT_CONSTEXPR void on_auto() {
+ int id = ctx.next_arg_id();
+ ref = arg_ref<Char>(id);
+ ctx.check_dynamic_spec(id);
+ }
+ FMT_CONSTEXPR void on_index(int id) {
+ ref = arg_ref<Char>(id);
+ ctx.check_arg_id(id);
+ ctx.check_dynamic_spec(id);
+ }
+ FMT_CONSTEXPR void on_name(basic_string_view<Char> id) {
+ ref = arg_ref<Char>(id);
+ ctx.check_arg_id(id);
+ }
+};
+
+// Parses [integer | "{" [arg_id] "}"].
+template <typename Char>
+FMT_CONSTEXPR auto parse_dynamic_spec(const Char* begin, const Char* end,
+ int& value, arg_ref<Char>& ref,
+ basic_format_parse_context<Char>& ctx)
+ -> const Char* {
+ FMT_ASSERT(begin != end, "");
+ if ('0' <= *begin && *begin <= '9') {
+ int val = parse_nonnegative_int(begin, end, -1);
+ if (val != -1)
+ value = val;
+ else
+ throw_format_error("number is too big");
+ } else if (*begin == '{') {
+ ++begin;
+ auto handler = dynamic_spec_id_handler<Char>{ctx, ref};
+ if (begin != end) begin = parse_arg_id(begin, end, handler);
+ if (begin != end && *begin == '}') return ++begin;
+ throw_format_error("invalid format string");
+ }
+ return begin;
+}
+
+template <typename Char>
+FMT_CONSTEXPR auto parse_precision(const Char* begin, const Char* end,
+ int& value, arg_ref<Char>& ref,
+ basic_format_parse_context<Char>& ctx)
+ -> const Char* {
+ ++begin;
+ if (begin == end || *begin == '}') {
+ throw_format_error("invalid precision");
+ return begin;
+ }
+ return parse_dynamic_spec(begin, end, value, ref, ctx);
+}
+
+enum class state { start, align, sign, hash, zero, width, precision, locale };
+
+// Parses standard format specifiers.
+template <typename Char>
+FMT_CONSTEXPR FMT_INLINE auto parse_format_specs(
+ const Char* begin, const Char* end, dynamic_format_specs<Char>& specs,
+ basic_format_parse_context<Char>& ctx, type arg_type) -> const Char* {
+ auto c = '\0';
+ if (end - begin > 1) {
+ auto next = to_ascii(begin[1]);
+ c = parse_align(next) == align::none ? to_ascii(*begin) : '\0';
+ } else {
+ if (begin == end) return begin;
+ c = to_ascii(*begin);
+ }
+
+ struct {
+ state current_state = state::start;
+ FMT_CONSTEXPR void operator()(state s, bool valid = true) {
+ if (current_state >= s || !valid)
+ throw_format_error("invalid format specifier");
+ current_state = s;
+ }
+ } enter_state;
+
+ using pres = presentation_type;
+ constexpr auto integral_set = sint_set | uint_set | bool_set | char_set;
+ struct {
+ const Char*& begin;
+ dynamic_format_specs<Char>& specs;
+ type arg_type;
+
+ FMT_CONSTEXPR auto operator()(pres type, int set) -> const Char* {
+ if (!in(arg_type, set)) throw_format_error("invalid format specifier");
+ specs.type = type;
+ return begin + 1;
+ }
+ } parse_presentation_type{begin, specs, arg_type};
+
+ for (;;) {
+ switch (c) {
+ case '<':
+ case '>':
+ case '^':
+ enter_state(state::align);
+ specs.align = parse_align(c);
+ ++begin;
+ break;
+ case '+':
+ case '-':
+ case ' ':
+ enter_state(state::sign, in(arg_type, sint_set | float_set));
+ switch (c) {
+ case '+':
+ specs.sign = sign::plus;
+ break;
+ case '-':
+ specs.sign = sign::minus;
+ break;
+ case ' ':
+ specs.sign = sign::space;
+ break;
+ }
+ ++begin;
+ break;
+ case '#':
+ enter_state(state::hash, is_arithmetic_type(arg_type));
+ specs.alt = true;
+ ++begin;
+ break;
+ case '0':
+ enter_state(state::zero);
+ if (!is_arithmetic_type(arg_type))
+ throw_format_error("format specifier requires numeric argument");
+ if (specs.align == align::none) {
+ // Ignore 0 if align is specified for compatibility with std::format.
+ specs.align = align::numeric;
+ specs.fill[0] = Char('0');
+ }
+ ++begin;
+ break;
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ case '{':
+ enter_state(state::width);
+ begin = parse_dynamic_spec(begin, end, specs.width, specs.width_ref, ctx);
+ break;
+ case '.':
+ enter_state(state::precision,
+ in(arg_type, float_set | string_set | cstring_set));
+ begin = parse_precision(begin, end, specs.precision, specs.precision_ref,
+ ctx);
+ break;
+ case 'L':
+ enter_state(state::locale, is_arithmetic_type(arg_type));
+ specs.localized = true;
+ ++begin;
+ break;
+ case 'd':
+ return parse_presentation_type(pres::dec, integral_set);
+ case 'o':
+ return parse_presentation_type(pres::oct, integral_set);
+ case 'x':
+ return parse_presentation_type(pres::hex_lower, integral_set);
+ case 'X':
+ return parse_presentation_type(pres::hex_upper, integral_set);
+ case 'b':
+ return parse_presentation_type(pres::bin_lower, integral_set);
+ case 'B':
+ return parse_presentation_type(pres::bin_upper, integral_set);
+ case 'a':
+ return parse_presentation_type(pres::hexfloat_lower, float_set);
+ case 'A':
+ return parse_presentation_type(pres::hexfloat_upper, float_set);
+ case 'e':
+ return parse_presentation_type(pres::exp_lower, float_set);
+ case 'E':
+ return parse_presentation_type(pres::exp_upper, float_set);
+ case 'f':
+ return parse_presentation_type(pres::fixed_lower, float_set);
+ case 'F':
+ return parse_presentation_type(pres::fixed_upper, float_set);
+ case 'g':
+ return parse_presentation_type(pres::general_lower, float_set);
+ case 'G':
+ return parse_presentation_type(pres::general_upper, float_set);
+ case 'c':
+ return parse_presentation_type(pres::chr, integral_set);
+ case 's':
+ return parse_presentation_type(pres::string,
+ bool_set | string_set | cstring_set);
+ case 'p':
+ return parse_presentation_type(pres::pointer, pointer_set | cstring_set);
+ case '?':
+ return parse_presentation_type(pres::debug,
+ char_set | string_set | cstring_set);
+ case '}':
+ return begin;
+ default: {
+ if (*begin == '}') return begin;
+ // Parse fill and alignment.
+ auto fill_end = begin + code_point_length(begin);
+ if (end - fill_end <= 0) {
+ throw_format_error("invalid format specifier");
+ return begin;
+ }
+ if (*begin == '{') {
+ throw_format_error("invalid fill character '{'");
+ return begin;
+ }
+ auto align = parse_align(to_ascii(*fill_end));
+ enter_state(state::align, align != align::none);
+ specs.fill = {begin, to_unsigned(fill_end - begin)};
+ specs.align = align;
+ begin = fill_end + 1;
+ }
+ }
+ if (begin == end) return begin;
+ c = to_ascii(*begin);
+ }
+}
+
+template <typename Char, typename Handler>
+FMT_CONSTEXPR auto parse_replacement_field(const Char* begin, const Char* end,
+ Handler&& handler) -> const Char* {
+ struct id_adapter {
+ Handler& handler;
+ int arg_id;
+
+ FMT_CONSTEXPR void on_auto() { arg_id = handler.on_arg_id(); }
+ FMT_CONSTEXPR void on_index(int id) { arg_id = handler.on_arg_id(id); }
+ FMT_CONSTEXPR void on_name(basic_string_view<Char> id) {
+ arg_id = handler.on_arg_id(id);
+ }
+ };
+
+ ++begin;
+ if (begin == end) return handler.on_error("invalid format string"), end;
+ if (*begin == '}') {
+ handler.on_replacement_field(handler.on_arg_id(), begin);
+ } else if (*begin == '{') {
+ handler.on_text(begin, begin + 1);
+ } else {
+ auto adapter = id_adapter{handler, 0};
+ begin = parse_arg_id(begin, end, adapter);
+ Char c = begin != end ? *begin : Char();
+ if (c == '}') {
+ handler.on_replacement_field(adapter.arg_id, begin);
+ } else if (c == ':') {
+ begin = handler.on_format_specs(adapter.arg_id, begin + 1, end);
+ if (begin == end || *begin != '}')
+ return handler.on_error("unknown format specifier"), end;
+ } else {
+ return handler.on_error("missing '}' in format string"), end;
+ }
+ }
+ return begin + 1;
+}
+
+template <bool IS_CONSTEXPR, typename Char, typename Handler>
+FMT_CONSTEXPR FMT_INLINE void parse_format_string(
+ basic_string_view<Char> format_str, Handler&& handler) {
+ auto begin = format_str.data();
+ auto end = begin + format_str.size();
+ if (end - begin < 32) {
+ // Use a simple loop instead of memchr for small strings.
+ const Char* p = begin;
+ while (p != end) {
+ auto c = *p++;
+ if (c == '{') {
+ handler.on_text(begin, p - 1);
+ begin = p = parse_replacement_field(p - 1, end, handler);
+ } else if (c == '}') {
+ if (p == end || *p != '}')
+ return handler.on_error("unmatched '}' in format string");
+ handler.on_text(begin, p);
+ begin = ++p;
+ }
+ }
+ handler.on_text(begin, end);
+ return;
+ }
+ struct writer {
+ FMT_CONSTEXPR void operator()(const Char* from, const Char* to) {
+ if (from == to) return;
+ for (;;) {
+ const Char* p = nullptr;
+ if (!find<IS_CONSTEXPR>(from, to, Char('}'), p))
+ return handler_.on_text(from, to);
+ ++p;
+ if (p == to || *p != '}')
+ return handler_.on_error("unmatched '}' in format string");
+ handler_.on_text(from, p);
+ from = p + 1;
+ }
+ }
+ Handler& handler_;
+ } write = {handler};
+ while (begin != end) {
+ // Doing two passes with memchr (one for '{' and another for '}') is up to
+ // 2.5x faster than the naive one-pass implementation on big format strings.
+ const Char* p = begin;
+ if (*begin != '{' && !find<IS_CONSTEXPR>(begin + 1, end, Char('{'), p))
+ return write(begin, end);
+ write(begin, p);
+ begin = parse_replacement_field(p, end, handler);
+ }
+}
+
+template <typename T, bool = is_named_arg<T>::value> struct strip_named_arg {
+ using type = T;
+};
+template <typename T> struct strip_named_arg<T, true> {
+ using type = remove_cvref_t<decltype(T::value)>;
+};
+
+template <typename T, typename ParseContext>
+FMT_CONSTEXPR auto parse_format_specs(ParseContext& ctx)
+ -> decltype(ctx.begin()) {
+ using char_type = typename ParseContext::char_type;
+ using context = buffer_context<char_type>;
+ using mapped_type = conditional_t<
+ mapped_type_constant<T, context>::value != type::custom_type,
+ decltype(arg_mapper<context>().map(std::declval<const T&>())),
+ typename strip_named_arg<T>::type>;
+ return formatter<mapped_type, char_type>().parse(ctx);
+}
+
+// Checks char specs and returns true iff the presentation type is char-like.
+template <typename Char>
+FMT_CONSTEXPR auto check_char_specs(const format_specs<Char>& specs) -> bool {
+ if (specs.type != presentation_type::none &&
+ specs.type != presentation_type::chr &&
+ specs.type != presentation_type::debug) {
+ return false;
+ }
+ if (specs.align == align::numeric || specs.sign != sign::none || specs.alt)
+ throw_format_error("invalid format specifier for char");
+ return true;
+}
+
+constexpr FMT_INLINE_VARIABLE int invalid_arg_index = -1;
+
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <int N, typename T, typename... Args, typename Char>
+constexpr auto get_arg_index_by_name(basic_string_view<Char> name) -> int {
+ if constexpr (is_statically_named_arg<T>()) {
+ if (name == T::name) return N;
+ }
+ if constexpr (sizeof...(Args) > 0)
+ return get_arg_index_by_name<N + 1, Args...>(name);
+ (void)name; // Workaround an MSVC bug about "unused" parameter.
+ return invalid_arg_index;
+}
+#endif
+
+template <typename... Args, typename Char>
+FMT_CONSTEXPR auto get_arg_index_by_name(basic_string_view<Char> name) -> int {
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+ if constexpr (sizeof...(Args) > 0)
+ return get_arg_index_by_name<0, Args...>(name);
+#endif
+ (void)name;
+ return invalid_arg_index;
+}
+
+template <typename Char, typename... Args> class format_string_checker {
+ private:
+ using parse_context_type = compile_parse_context<Char>;
+ static constexpr int num_args = sizeof...(Args);
+
+ // Format specifier parsing function.
+ // In the future basic_format_parse_context will replace compile_parse_context
+ // here and will use is_constant_evaluated and downcasting to access the data
+ // needed for compile-time checks: https://godbolt.org/z/GvWzcTjh1.
+ using parse_func = const Char* (*)(parse_context_type&);
+
+ parse_context_type context_;
+ parse_func parse_funcs_[num_args > 0 ? static_cast<size_t>(num_args) : 1];
+ type types_[num_args > 0 ? static_cast<size_t>(num_args) : 1];
+
+ public:
+ explicit FMT_CONSTEXPR format_string_checker(basic_string_view<Char> fmt)
+ : context_(fmt, num_args, types_),
+ parse_funcs_{&parse_format_specs<Args, parse_context_type>...},
+ types_{mapped_type_constant<Args, buffer_context<Char>>::value...} {}
+
+ FMT_CONSTEXPR void on_text(const Char*, const Char*) {}
+
+ FMT_CONSTEXPR auto on_arg_id() -> int { return context_.next_arg_id(); }
+ FMT_CONSTEXPR auto on_arg_id(int id) -> int {
+ return context_.check_arg_id(id), id;
+ }
+ FMT_CONSTEXPR auto on_arg_id(basic_string_view<Char> id) -> int {
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+ auto index = get_arg_index_by_name<Args...>(id);
+ if (index == invalid_arg_index) on_error("named argument is not found");
+ return index;
+#else
+ (void)id;
+ on_error("compile-time checks for named arguments require C++20 support");
+ return 0;
+#endif
+ }
+
+ FMT_CONSTEXPR void on_replacement_field(int, const Char*) {}
+
+ FMT_CONSTEXPR auto on_format_specs(int id, const Char* begin, const Char*)
+ -> const Char* {
+ context_.advance_to(begin);
+ // id >= 0 check is a workaround for gcc 10 bug (#2065).
+ return id >= 0 && id < num_args ? parse_funcs_[id](context_) : begin;
+ }
+
+ FMT_CONSTEXPR void on_error(const char* message) {
+ throw_format_error(message);
+ }
+};
+
+// Reports a compile-time error if S is not a valid format string.
+template <typename..., typename S, FMT_ENABLE_IF(!is_compile_string<S>::value)>
+FMT_INLINE void check_format_string(const S&) {
+#ifdef FMT_ENFORCE_COMPILE_STRING
+ static_assert(is_compile_string<S>::value,
+ "FMT_ENFORCE_COMPILE_STRING requires all format strings to use "
+ "FMT_STRING.");
+#endif
+}
+template <typename... Args, typename S,
+ FMT_ENABLE_IF(is_compile_string<S>::value)>
+void check_format_string(S format_str) {
+ using char_t = typename S::char_type;
+ FMT_CONSTEXPR auto s = basic_string_view<char_t>(format_str);
+ using checker = format_string_checker<char_t, remove_cvref_t<Args>...>;
+ FMT_CONSTEXPR bool error = (parse_format_string<true>(s, checker(s)), true);
+ ignore_unused(error);
+}
+
+template <typename Char = char> struct vformat_args {
+ using type = basic_format_args<
+ basic_format_context<std::back_insert_iterator<buffer<Char>>, Char>>;
+};
+template <> struct vformat_args<char> { using type = format_args; };
+
+// Use vformat_args and avoid type_identity to keep symbols short.
+template <typename Char>
+void vformat_to(buffer<Char>& buf, basic_string_view<Char> fmt,
+ typename vformat_args<Char>::type args, locale_ref loc = {});
+
+FMT_API void vprint_mojibake(std::FILE*, string_view, format_args);
+#ifndef _WIN32
+inline void vprint_mojibake(std::FILE*, string_view, format_args) {}
+#endif
+} // namespace detail
+
+FMT_BEGIN_EXPORT
+
+// A formatter specialization for natively supported types.
+template <typename T, typename Char>
+struct formatter<T, Char,
+ enable_if_t<detail::type_constant<T, Char>::value !=
+ detail::type::custom_type>> {
+ private:
+ detail::dynamic_format_specs<Char> specs_;
+
+ public:
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const Char* {
+ auto type = detail::type_constant<T, Char>::value;
+ auto end =
+ detail::parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx, type);
+ if (type == detail::type::char_type) detail::check_char_specs(specs_);
+ return end;
+ }
+
+ template <detail::type U = detail::type_constant<T, Char>::value,
+ FMT_ENABLE_IF(U == detail::type::string_type ||
+ U == detail::type::cstring_type ||
+ U == detail::type::char_type)>
+ FMT_CONSTEXPR void set_debug_format(bool set = true) {
+ specs_.type = set ? presentation_type::debug : presentation_type::none;
+ }
+
+ template <typename FormatContext>
+ FMT_CONSTEXPR auto format(const T& val, FormatContext& ctx) const
+ -> decltype(ctx.out());
+};
+
+#define FMT_FORMAT_AS(Type, Base) \
+ template <typename Char> \
+ struct formatter<Type, Char> : formatter<Base, Char> { \
+ template <typename FormatContext> \
+ auto format(const Type& val, FormatContext& ctx) const \
+ -> decltype(ctx.out()) { \
+ return formatter<Base, Char>::format(static_cast<Base>(val), ctx); \
+ } \
+ }
+
+FMT_FORMAT_AS(signed char, int);
+FMT_FORMAT_AS(unsigned char, unsigned);
+FMT_FORMAT_AS(short, int);
+FMT_FORMAT_AS(unsigned short, unsigned);
+FMT_FORMAT_AS(long, long long);
+FMT_FORMAT_AS(unsigned long, unsigned long long);
+FMT_FORMAT_AS(Char*, const Char*);
+FMT_FORMAT_AS(std::basic_string<Char>, basic_string_view<Char>);
+FMT_FORMAT_AS(std::nullptr_t, const void*);
+FMT_FORMAT_AS(detail::std_string_view<Char>, basic_string_view<Char>);
+
+template <typename Char = char> struct runtime_format_string {
+ basic_string_view<Char> str;
+};
+
+/** A compile-time format string. */
+template <typename Char, typename... Args> class basic_format_string {
+ private:
+ basic_string_view<Char> str_;
+
+ public:
+ template <typename S,
+ FMT_ENABLE_IF(
+ std::is_convertible<const S&, basic_string_view<Char>>::value)>
+ FMT_CONSTEVAL FMT_INLINE basic_format_string(const S& s) : str_(s) {
+ static_assert(
+ detail::count<
+ (std::is_base_of<detail::view, remove_reference_t<Args>>::value &&
+ std::is_reference<Args>::value)...>() == 0,
+ "passing views as lvalues is disallowed");
+#ifdef FMT_HAS_CONSTEVAL
+ if constexpr (detail::count_named_args<Args...>() ==
+ detail::count_statically_named_args<Args...>()) {
+ using checker =
+ detail::format_string_checker<Char, remove_cvref_t<Args>...>;
+ detail::parse_format_string<true>(str_, checker(s));
+ }
+#else
+ detail::check_format_string<Args...>(s);
+#endif
+ }
+ basic_format_string(runtime_format_string<Char> fmt) : str_(fmt.str) {}
+
+ FMT_INLINE operator basic_string_view<Char>() const { return str_; }
+ FMT_INLINE auto get() const -> basic_string_view<Char> { return str_; }
+};
+
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+// Workaround broken conversion on older gcc.
+template <typename...> using format_string = string_view;
+inline auto runtime(string_view s) -> string_view { return s; }
+#else
+template <typename... Args>
+using format_string = basic_format_string<char, type_identity_t<Args>...>;
+/**
+ \rst
+ Creates a runtime format string.
+
+ **Example**::
+
+ // Check format string at runtime instead of compile-time.
+ fmt::print(fmt::runtime("{:d}"), "I am not a number");
+ \endrst
+ */
+inline auto runtime(string_view s) -> runtime_format_string<> { return {{s}}; }
+#endif
+
+FMT_API auto vformat(string_view fmt, format_args args) -> std::string;
+
+/**
+ \rst
+ Formats ``args`` according to specifications in ``fmt`` and returns the result
+ as a string.
+
+ **Example**::
+
+ #include <fmt/core.h>
+ std::string message = fmt::format("The answer is {}.", 42);
+ \endrst
+*/
+template <typename... T>
+FMT_NODISCARD FMT_INLINE auto format(format_string<T...> fmt, T&&... args)
+ -> std::string {
+ return vformat(fmt, fmt::make_format_args(args...));
+}
+
+/** Formats a string and writes the output to ``out``. */
+template <typename OutputIt,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+auto vformat_to(OutputIt out, string_view fmt, format_args args) -> OutputIt {
+ auto&& buf = detail::get_buffer<char>(out);
+ detail::vformat_to(buf, fmt, args, {});
+ return detail::get_iterator(buf, out);
+}
+
+/**
+ \rst
+ Formats ``args`` according to specifications in ``fmt``, writes the result to
+ the output iterator ``out`` and returns the iterator past the end of the output
+ range. `format_to` does not append a terminating null character.
+
+ **Example**::
+
+ auto out = std::vector<char>();
+ fmt::format_to(std::back_inserter(out), "{}", 42);
+ \endrst
+ */
+template <typename OutputIt, typename... T,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+FMT_INLINE auto format_to(OutputIt out, format_string<T...> fmt, T&&... args)
+ -> OutputIt {
+ return vformat_to(out, fmt, fmt::make_format_args(args...));
+}
+
+template <typename OutputIt> struct format_to_n_result {
+ /** Iterator past the end of the output range. */
+ OutputIt out;
+ /** Total (not truncated) output size. */
+ size_t size;
+};
+
+template <typename OutputIt, typename... T,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+auto vformat_to_n(OutputIt out, size_t n, string_view fmt, format_args args)
+ -> format_to_n_result<OutputIt> {
+ using traits = detail::fixed_buffer_traits;
+ auto buf = detail::iterator_buffer<OutputIt, char, traits>(out, n);
+ detail::vformat_to(buf, fmt, args, {});
+ return {buf.out(), buf.count()};
+}
+
+/**
+ \rst
+ Formats ``args`` according to specifications in ``fmt``, writes up to ``n``
+ characters of the result to the output iterator ``out`` and returns the total
+ (not truncated) output size and the iterator past the end of the output range.
+ `format_to_n` does not append a terminating null character.
+ \endrst
+ */
+template <typename OutputIt, typename... T,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+FMT_INLINE auto format_to_n(OutputIt out, size_t n, format_string<T...> fmt,
+ T&&... args) -> format_to_n_result<OutputIt> {
+ return vformat_to_n(out, n, fmt, fmt::make_format_args(args...));
+}
+
+/** Returns the number of chars in the output of ``format(fmt, args...)``. */
+template <typename... T>
+FMT_NODISCARD FMT_INLINE auto formatted_size(format_string<T...> fmt,
+ T&&... args) -> size_t {
+ auto buf = detail::counting_buffer<>();
+ detail::vformat_to<char>(buf, fmt, fmt::make_format_args(args...), {});
+ return buf.count();
+}
+
+FMT_API void vprint(string_view fmt, format_args args);
+FMT_API void vprint(std::FILE* f, string_view fmt, format_args args);
+
+/**
+ \rst
+ Formats ``args`` according to specifications in ``fmt`` and writes the output
+ to ``stdout``.
+
+ **Example**::
+
+ fmt::print("Elapsed time: {0:.2f} seconds", 1.23);
+ \endrst
+ */
+template <typename... T>
+FMT_INLINE void print(format_string<T...> fmt, T&&... args) {
+ const auto& vargs = fmt::make_format_args(args...);
+ return detail::is_utf8() ? vprint(fmt, vargs)
+ : detail::vprint_mojibake(stdout, fmt, vargs);
+}
+
+/**
+ \rst
+ Formats ``args`` according to specifications in ``fmt`` and writes the
+ output to the file ``f``.
+
+ **Example**::
+
+ fmt::print(stderr, "Don't {}!", "panic");
+ \endrst
+ */
+template <typename... T>
+FMT_INLINE void print(std::FILE* f, format_string<T...> fmt, T&&... args) {
+ const auto& vargs = fmt::make_format_args(args...);
+ return detail::is_utf8() ? vprint(f, fmt, vargs)
+ : detail::vprint_mojibake(f, fmt, vargs);
+}
+
+/**
+ Formats ``args`` according to specifications in ``fmt`` and writes the
+ output to the file ``f`` followed by a newline.
+ */
+template <typename... T>
+FMT_INLINE void println(std::FILE* f, format_string<T...> fmt, T&&... args) {
+ return fmt::print(f, "{}\n", fmt::format(fmt, std::forward<T>(args)...));
+}
+
+/**
+ Formats ``args`` according to specifications in ``fmt`` and writes the output
+ to ``stdout`` followed by a newline.
+ */
+template <typename... T>
+FMT_INLINE void println(format_string<T...> fmt, T&&... args) {
+ return fmt::println(stdout, fmt, std::forward<T>(args)...);
+}
+
+FMT_END_EXPORT
+FMT_GCC_PRAGMA("GCC pop_options")
+FMT_END_NAMESPACE
+
+#ifdef FMT_HEADER_ONLY
+# include "format.h"
+#endif
+#endif // FMT_CORE_H_
diff --git a/contrib/fmt/include/fmt/format-inl.h b/contrib/fmt/include/fmt/format-inl.h
new file mode 100644
index 0000000..5bae3c7
--- /dev/null
+++ b/contrib/fmt/include/fmt/format-inl.h
@@ -0,0 +1,1681 @@
+// Formatting library for C++ - implementation
+//
+// Copyright (c) 2012 - 2016, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_FORMAT_INL_H_
+#define FMT_FORMAT_INL_H_
+
+#include <algorithm>
+#include <cerrno> // errno
+#include <climits>
+#include <cmath>
+#include <exception>
+
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+# include <locale>
+#endif
+
+#ifdef _WIN32
+# include <io.h> // _isatty
+#endif
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+FMT_FUNC void assert_fail(const char* file, int line, const char* message) {
+ // Use unchecked std::fprintf to avoid triggering another assertion when
+ // writing to stderr fails
+ std::fprintf(stderr, "%s:%d: assertion failed: %s", file, line, message);
+ // Chosen instead of std::abort to satisfy Clang in CUDA mode during device
+ // code pass.
+ std::terminate();
+}
+
+FMT_FUNC void throw_format_error(const char* message) {
+ FMT_THROW(format_error(message));
+}
+
+FMT_FUNC void format_error_code(detail::buffer<char>& out, int error_code,
+ string_view message) noexcept {
+ // Report error code making sure that the output fits into
+ // inline_buffer_size to avoid dynamic memory allocation and potential
+ // bad_alloc.
+ out.try_resize(0);
+ static const char SEP[] = ": ";
+ static const char ERROR_STR[] = "error ";
+ // Subtract 2 to account for terminating null characters in SEP and ERROR_STR.
+ size_t error_code_size = sizeof(SEP) + sizeof(ERROR_STR) - 2;
+ auto abs_value = static_cast<uint32_or_64_or_128_t<int>>(error_code);
+ if (detail::is_negative(error_code)) {
+ abs_value = 0 - abs_value;
+ ++error_code_size;
+ }
+ error_code_size += detail::to_unsigned(detail::count_digits(abs_value));
+ auto it = buffer_appender<char>(out);
+ if (message.size() <= inline_buffer_size - error_code_size)
+ format_to(it, FMT_STRING("{}{}"), message, SEP);
+ format_to(it, FMT_STRING("{}{}"), ERROR_STR, error_code);
+ FMT_ASSERT(out.size() <= inline_buffer_size, "");
+}
+
+FMT_FUNC void report_error(format_func func, int error_code,
+ const char* message) noexcept {
+ memory_buffer full_message;
+ func(full_message, error_code, message);
+ // Don't use fwrite_fully because the latter may throw.
+ if (std::fwrite(full_message.data(), full_message.size(), 1, stderr) > 0)
+ std::fputc('\n', stderr);
+}
+
+// A wrapper around fwrite that throws on error.
+inline void fwrite_fully(const void* ptr, size_t size, size_t count,
+ FILE* stream) {
+ size_t written = std::fwrite(ptr, size, count, stream);
+ if (written < count)
+ FMT_THROW(system_error(errno, FMT_STRING("cannot write to file")));
+}
+
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+template <typename Locale>
+locale_ref::locale_ref(const Locale& loc) : locale_(&loc) {
+ static_assert(std::is_same<Locale, std::locale>::value, "");
+}
+
+template <typename Locale> Locale locale_ref::get() const {
+ static_assert(std::is_same<Locale, std::locale>::value, "");
+ return locale_ ? *static_cast<const std::locale*>(locale_) : std::locale();
+}
+
+template <typename Char>
+FMT_FUNC auto thousands_sep_impl(locale_ref loc) -> thousands_sep_result<Char> {
+ auto& facet = std::use_facet<std::numpunct<Char>>(loc.get<std::locale>());
+ auto grouping = facet.grouping();
+ auto thousands_sep = grouping.empty() ? Char() : facet.thousands_sep();
+ return {std::move(grouping), thousands_sep};
+}
+template <typename Char> FMT_FUNC Char decimal_point_impl(locale_ref loc) {
+ return std::use_facet<std::numpunct<Char>>(loc.get<std::locale>())
+ .decimal_point();
+}
+#else
+template <typename Char>
+FMT_FUNC auto thousands_sep_impl(locale_ref) -> thousands_sep_result<Char> {
+ return {"\03", FMT_STATIC_THOUSANDS_SEPARATOR};
+}
+template <typename Char> FMT_FUNC Char decimal_point_impl(locale_ref) {
+ return '.';
+}
+#endif
+
+FMT_FUNC auto write_loc(appender out, loc_value value,
+ const format_specs<>& specs, locale_ref loc) -> bool {
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+ auto locale = loc.get<std::locale>();
+ // We cannot use the num_put<char> facet because it may produce output in
+ // a wrong encoding.
+ using facet = format_facet<std::locale>;
+ if (std::has_facet<facet>(locale))
+ return std::use_facet<facet>(locale).put(out, value, specs);
+ return facet(locale).put(out, value, specs);
+#endif
+ return false;
+}
+} // namespace detail
+
+template <typename Locale> typename Locale::id format_facet<Locale>::id;
+
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+template <typename Locale> format_facet<Locale>::format_facet(Locale& loc) {
+ auto& numpunct = std::use_facet<std::numpunct<char>>(loc);
+ grouping_ = numpunct.grouping();
+ if (!grouping_.empty()) separator_ = std::string(1, numpunct.thousands_sep());
+}
+
+template <>
+FMT_API FMT_FUNC auto format_facet<std::locale>::do_put(
+ appender out, loc_value val, const format_specs<>& specs) const -> bool {
+ return val.visit(
+ detail::loc_writer<>{out, specs, separator_, grouping_, decimal_point_});
+}
+#endif
+
+FMT_FUNC std::system_error vsystem_error(int error_code, string_view fmt,
+ format_args args) {
+ auto ec = std::error_code(error_code, std::generic_category());
+ return std::system_error(ec, vformat(fmt, args));
+}
+
+namespace detail {
+
+template <typename F> inline bool operator==(basic_fp<F> x, basic_fp<F> y) {
+ return x.f == y.f && x.e == y.e;
+}
+
+// Compilers should be able to optimize this into the ror instruction.
+FMT_CONSTEXPR inline uint32_t rotr(uint32_t n, uint32_t r) noexcept {
+ r &= 31;
+ return (n >> r) | (n << (32 - r));
+}
+FMT_CONSTEXPR inline uint64_t rotr(uint64_t n, uint32_t r) noexcept {
+ r &= 63;
+ return (n >> r) | (n << (64 - r));
+}
+
+// Implementation of Dragonbox algorithm: https://github.com/jk-jeon/dragonbox.
+namespace dragonbox {
+// Computes upper 64 bits of multiplication of a 32-bit unsigned integer and a
+// 64-bit unsigned integer.
+inline uint64_t umul96_upper64(uint32_t x, uint64_t y) noexcept {
+ return umul128_upper64(static_cast<uint64_t>(x) << 32, y);
+}
+
+// Computes lower 128 bits of multiplication of a 64-bit unsigned integer and a
+// 128-bit unsigned integer.
+inline uint128_fallback umul192_lower128(uint64_t x,
+ uint128_fallback y) noexcept {
+ uint64_t high = x * y.high();
+ uint128_fallback high_low = umul128(x, y.low());
+ return {high + high_low.high(), high_low.low()};
+}
+
+// Computes lower 64 bits of multiplication of a 32-bit unsigned integer and a
+// 64-bit unsigned integer.
+inline uint64_t umul96_lower64(uint32_t x, uint64_t y) noexcept {
+ return x * y;
+}
+
+// Various fast log computations.
+inline int floor_log10_pow2_minus_log10_4_over_3(int e) noexcept {
+ FMT_ASSERT(e <= 2936 && e >= -2985, "too large exponent");
+ return (e * 631305 - 261663) >> 21;
+}
+
+FMT_INLINE_VARIABLE constexpr struct {
+ uint32_t divisor;
+ int shift_amount;
+} div_small_pow10_infos[] = {{10, 16}, {100, 16}};
+
+// Replaces n by floor(n / pow(10, N)) returning true if and only if n is
+// divisible by pow(10, N).
+// Precondition: n <= pow(10, N + 1).
+template <int N>
+bool check_divisibility_and_divide_by_pow10(uint32_t& n) noexcept {
+ // The numbers below are chosen such that:
+ // 1. floor(n/d) = floor(nm / 2^k) where d=10 or d=100,
+ // 2. nm mod 2^k < m if and only if n is divisible by d,
+ // where m is magic_number, k is shift_amount
+ // and d is divisor.
+ //
+ // Item 1 is a common technique of replacing division by a constant with
+ // multiplication, see e.g. "Division by Invariant Integers Using
+ // Multiplication" by Granlund and Montgomery (1994). magic_number (m) is set
+ // to ceil(2^k/d) for large enough k.
+ // The idea for item 2 originates from Schubfach.
+ constexpr auto info = div_small_pow10_infos[N - 1];
+ FMT_ASSERT(n <= info.divisor * 10, "n is too large");
+ constexpr uint32_t magic_number =
+ (1u << info.shift_amount) / info.divisor + 1;
+ n *= magic_number;
+ const uint32_t comparison_mask = (1u << info.shift_amount) - 1;
+ bool result = (n & comparison_mask) < magic_number;
+ n >>= info.shift_amount;
+ return result;
+}
+
+// Computes floor(n / pow(10, N)) for small n and N.
+// Precondition: n <= pow(10, N + 1).
+template <int N> uint32_t small_division_by_pow10(uint32_t n) noexcept {
+ constexpr auto info = div_small_pow10_infos[N - 1];
+ FMT_ASSERT(n <= info.divisor * 10, "n is too large");
+ constexpr uint32_t magic_number =
+ (1u << info.shift_amount) / info.divisor + 1;
+ return (n * magic_number) >> info.shift_amount;
+}
+
+// Computes floor(n / 10^(kappa + 1)) (float)
+inline uint32_t divide_by_10_to_kappa_plus_1(uint32_t n) noexcept {
+ // 1374389535 = ceil(2^37/100)
+ return static_cast<uint32_t>((static_cast<uint64_t>(n) * 1374389535) >> 37);
+}
+// Computes floor(n / 10^(kappa + 1)) (double)
+inline uint64_t divide_by_10_to_kappa_plus_1(uint64_t n) noexcept {
+ // 2361183241434822607 = ceil(2^(64+7)/1000)
+ return umul128_upper64(n, 2361183241434822607ull) >> 7;
+}
+
+// Various subroutines using pow10 cache
+template <typename T> struct cache_accessor;
+
+template <> struct cache_accessor<float> {
+ using carrier_uint = float_info<float>::carrier_uint;
+ using cache_entry_type = uint64_t;
+
+ static uint64_t get_cached_power(int k) noexcept {
+ FMT_ASSERT(k >= float_info<float>::min_k && k <= float_info<float>::max_k,
+ "k is out of range");
+ static constexpr const uint64_t pow10_significands[] = {
+ 0x81ceb32c4b43fcf5, 0xa2425ff75e14fc32, 0xcad2f7f5359a3b3f,
+ 0xfd87b5f28300ca0e, 0x9e74d1b791e07e49, 0xc612062576589ddb,
+ 0xf79687aed3eec552, 0x9abe14cd44753b53, 0xc16d9a0095928a28,
+ 0xf1c90080baf72cb2, 0x971da05074da7bef, 0xbce5086492111aeb,
+ 0xec1e4a7db69561a6, 0x9392ee8e921d5d08, 0xb877aa3236a4b44a,
+ 0xe69594bec44de15c, 0x901d7cf73ab0acda, 0xb424dc35095cd810,
+ 0xe12e13424bb40e14, 0x8cbccc096f5088cc, 0xafebff0bcb24aaff,
+ 0xdbe6fecebdedd5bf, 0x89705f4136b4a598, 0xabcc77118461cefd,
+ 0xd6bf94d5e57a42bd, 0x8637bd05af6c69b6, 0xa7c5ac471b478424,
+ 0xd1b71758e219652c, 0x83126e978d4fdf3c, 0xa3d70a3d70a3d70b,
+ 0xcccccccccccccccd, 0x8000000000000000, 0xa000000000000000,
+ 0xc800000000000000, 0xfa00000000000000, 0x9c40000000000000,
+ 0xc350000000000000, 0xf424000000000000, 0x9896800000000000,
+ 0xbebc200000000000, 0xee6b280000000000, 0x9502f90000000000,
+ 0xba43b74000000000, 0xe8d4a51000000000, 0x9184e72a00000000,
+ 0xb5e620f480000000, 0xe35fa931a0000000, 0x8e1bc9bf04000000,
+ 0xb1a2bc2ec5000000, 0xde0b6b3a76400000, 0x8ac7230489e80000,
+ 0xad78ebc5ac620000, 0xd8d726b7177a8000, 0x878678326eac9000,
+ 0xa968163f0a57b400, 0xd3c21bcecceda100, 0x84595161401484a0,
+ 0xa56fa5b99019a5c8, 0xcecb8f27f4200f3a, 0x813f3978f8940985,
+ 0xa18f07d736b90be6, 0xc9f2c9cd04674edf, 0xfc6f7c4045812297,
+ 0x9dc5ada82b70b59e, 0xc5371912364ce306, 0xf684df56c3e01bc7,
+ 0x9a130b963a6c115d, 0xc097ce7bc90715b4, 0xf0bdc21abb48db21,
+ 0x96769950b50d88f5, 0xbc143fa4e250eb32, 0xeb194f8e1ae525fe,
+ 0x92efd1b8d0cf37bf, 0xb7abc627050305ae, 0xe596b7b0c643c71a,
+ 0x8f7e32ce7bea5c70, 0xb35dbf821ae4f38c, 0xe0352f62a19e306f};
+ return pow10_significands[k - float_info<float>::min_k];
+ }
+
+ struct compute_mul_result {
+ carrier_uint result;
+ bool is_integer;
+ };
+ struct compute_mul_parity_result {
+ bool parity;
+ bool is_integer;
+ };
+
+ static compute_mul_result compute_mul(
+ carrier_uint u, const cache_entry_type& cache) noexcept {
+ auto r = umul96_upper64(u, cache);
+ return {static_cast<carrier_uint>(r >> 32),
+ static_cast<carrier_uint>(r) == 0};
+ }
+
+ static uint32_t compute_delta(const cache_entry_type& cache,
+ int beta) noexcept {
+ return static_cast<uint32_t>(cache >> (64 - 1 - beta));
+ }
+
+ static compute_mul_parity_result compute_mul_parity(
+ carrier_uint two_f, const cache_entry_type& cache, int beta) noexcept {
+ FMT_ASSERT(beta >= 1, "");
+ FMT_ASSERT(beta < 64, "");
+
+ auto r = umul96_lower64(two_f, cache);
+ return {((r >> (64 - beta)) & 1) != 0,
+ static_cast<uint32_t>(r >> (32 - beta)) == 0};
+ }
+
+ static carrier_uint compute_left_endpoint_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return static_cast<carrier_uint>(
+ (cache - (cache >> (num_significand_bits<float>() + 2))) >>
+ (64 - num_significand_bits<float>() - 1 - beta));
+ }
+
+ static carrier_uint compute_right_endpoint_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return static_cast<carrier_uint>(
+ (cache + (cache >> (num_significand_bits<float>() + 1))) >>
+ (64 - num_significand_bits<float>() - 1 - beta));
+ }
+
+ static carrier_uint compute_round_up_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return (static_cast<carrier_uint>(
+ cache >> (64 - num_significand_bits<float>() - 2 - beta)) +
+ 1) /
+ 2;
+ }
+};
+
+template <> struct cache_accessor<double> {
+ using carrier_uint = float_info<double>::carrier_uint;
+ using cache_entry_type = uint128_fallback;
+
+ static uint128_fallback get_cached_power(int k) noexcept {
+ FMT_ASSERT(k >= float_info<double>::min_k && k <= float_info<double>::max_k,
+ "k is out of range");
+
+ static constexpr const uint128_fallback pow10_significands[] = {
+#if FMT_USE_FULL_CACHE_DRAGONBOX
+ {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b},
+ {0x9faacf3df73609b1, 0x77b191618c54e9ad},
+ {0xc795830d75038c1d, 0xd59df5b9ef6a2418},
+ {0xf97ae3d0d2446f25, 0x4b0573286b44ad1e},
+ {0x9becce62836ac577, 0x4ee367f9430aec33},
+ {0xc2e801fb244576d5, 0x229c41f793cda740},
+ {0xf3a20279ed56d48a, 0x6b43527578c11110},
+ {0x9845418c345644d6, 0x830a13896b78aaaa},
+ {0xbe5691ef416bd60c, 0x23cc986bc656d554},
+ {0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa9},
+ {0x94b3a202eb1c3f39, 0x7bf7d71432f3d6aa},
+ {0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc54},
+ {0xe858ad248f5c22c9, 0xd1b3400f8f9cff69},
+ {0x91376c36d99995be, 0x23100809b9c21fa2},
+ {0xb58547448ffffb2d, 0xabd40a0c2832a78b},
+ {0xe2e69915b3fff9f9, 0x16c90c8f323f516d},
+ {0x8dd01fad907ffc3b, 0xae3da7d97f6792e4},
+ {0xb1442798f49ffb4a, 0x99cd11cfdf41779d},
+ {0xdd95317f31c7fa1d, 0x40405643d711d584},
+ {0x8a7d3eef7f1cfc52, 0x482835ea666b2573},
+ {0xad1c8eab5ee43b66, 0xda3243650005eed0},
+ {0xd863b256369d4a40, 0x90bed43e40076a83},
+ {0x873e4f75e2224e68, 0x5a7744a6e804a292},
+ {0xa90de3535aaae202, 0x711515d0a205cb37},
+ {0xd3515c2831559a83, 0x0d5a5b44ca873e04},
+ {0x8412d9991ed58091, 0xe858790afe9486c3},
+ {0xa5178fff668ae0b6, 0x626e974dbe39a873},
+ {0xce5d73ff402d98e3, 0xfb0a3d212dc81290},
+ {0x80fa687f881c7f8e, 0x7ce66634bc9d0b9a},
+ {0xa139029f6a239f72, 0x1c1fffc1ebc44e81},
+ {0xc987434744ac874e, 0xa327ffb266b56221},
+ {0xfbe9141915d7a922, 0x4bf1ff9f0062baa9},
+ {0x9d71ac8fada6c9b5, 0x6f773fc3603db4aa},
+ {0xc4ce17b399107c22, 0xcb550fb4384d21d4},
+ {0xf6019da07f549b2b, 0x7e2a53a146606a49},
+ {0x99c102844f94e0fb, 0x2eda7444cbfc426e},
+ {0xc0314325637a1939, 0xfa911155fefb5309},
+ {0xf03d93eebc589f88, 0x793555ab7eba27cb},
+ {0x96267c7535b763b5, 0x4bc1558b2f3458df},
+ {0xbbb01b9283253ca2, 0x9eb1aaedfb016f17},
+ {0xea9c227723ee8bcb, 0x465e15a979c1cadd},
+ {0x92a1958a7675175f, 0x0bfacd89ec191eca},
+ {0xb749faed14125d36, 0xcef980ec671f667c},
+ {0xe51c79a85916f484, 0x82b7e12780e7401b},
+ {0x8f31cc0937ae58d2, 0xd1b2ecb8b0908811},
+ {0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa16},
+ {0xdfbdcece67006ac9, 0x67a791e093e1d49b},
+ {0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e1},
+ {0xaecc49914078536d, 0x58fae9f773886e19},
+ {0xda7f5bf590966848, 0xaf39a475506a899f},
+ {0x888f99797a5e012d, 0x6d8406c952429604},
+ {0xaab37fd7d8f58178, 0xc8e5087ba6d33b84},
+ {0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a65},
+ {0x855c3be0a17fcd26, 0x5cf2eea09a550680},
+ {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f},
+ {0xd0601d8efc57b08b, 0xf13b94daf124da27},
+ {0x823c12795db6ce57, 0x76c53d08d6b70859},
+ {0xa2cb1717b52481ed, 0x54768c4b0c64ca6f},
+ {0xcb7ddcdda26da268, 0xa9942f5dcf7dfd0a},
+ {0xfe5d54150b090b02, 0xd3f93b35435d7c4d},
+ {0x9efa548d26e5a6e1, 0xc47bc5014a1a6db0},
+ {0xc6b8e9b0709f109a, 0x359ab6419ca1091c},
+ {0xf867241c8cc6d4c0, 0xc30163d203c94b63},
+ {0x9b407691d7fc44f8, 0x79e0de63425dcf1e},
+ {0xc21094364dfb5636, 0x985915fc12f542e5},
+ {0xf294b943e17a2bc4, 0x3e6f5b7b17b2939e},
+ {0x979cf3ca6cec5b5a, 0xa705992ceecf9c43},
+ {0xbd8430bd08277231, 0x50c6ff782a838354},
+ {0xece53cec4a314ebd, 0xa4f8bf5635246429},
+ {0x940f4613ae5ed136, 0x871b7795e136be9a},
+ {0xb913179899f68584, 0x28e2557b59846e40},
+ {0xe757dd7ec07426e5, 0x331aeada2fe589d0},
+ {0x9096ea6f3848984f, 0x3ff0d2c85def7622},
+ {0xb4bca50b065abe63, 0x0fed077a756b53aa},
+ {0xe1ebce4dc7f16dfb, 0xd3e8495912c62895},
+ {0x8d3360f09cf6e4bd, 0x64712dd7abbbd95d},
+ {0xb080392cc4349dec, 0xbd8d794d96aacfb4},
+ {0xdca04777f541c567, 0xecf0d7a0fc5583a1},
+ {0x89e42caaf9491b60, 0xf41686c49db57245},
+ {0xac5d37d5b79b6239, 0x311c2875c522ced6},
+ {0xd77485cb25823ac7, 0x7d633293366b828c},
+ {0x86a8d39ef77164bc, 0xae5dff9c02033198},
+ {0xa8530886b54dbdeb, 0xd9f57f830283fdfd},
+ {0xd267caa862a12d66, 0xd072df63c324fd7c},
+ {0x8380dea93da4bc60, 0x4247cb9e59f71e6e},
+ {0xa46116538d0deb78, 0x52d9be85f074e609},
+ {0xcd795be870516656, 0x67902e276c921f8c},
+ {0x806bd9714632dff6, 0x00ba1cd8a3db53b7},
+ {0xa086cfcd97bf97f3, 0x80e8a40eccd228a5},
+ {0xc8a883c0fdaf7df0, 0x6122cd128006b2ce},
+ {0xfad2a4b13d1b5d6c, 0x796b805720085f82},
+ {0x9cc3a6eec6311a63, 0xcbe3303674053bb1},
+ {0xc3f490aa77bd60fc, 0xbedbfc4411068a9d},
+ {0xf4f1b4d515acb93b, 0xee92fb5515482d45},
+ {0x991711052d8bf3c5, 0x751bdd152d4d1c4b},
+ {0xbf5cd54678eef0b6, 0xd262d45a78a0635e},
+ {0xef340a98172aace4, 0x86fb897116c87c35},
+ {0x9580869f0e7aac0e, 0xd45d35e6ae3d4da1},
+ {0xbae0a846d2195712, 0x8974836059cca10a},
+ {0xe998d258869facd7, 0x2bd1a438703fc94c},
+ {0x91ff83775423cc06, 0x7b6306a34627ddd0},
+ {0xb67f6455292cbf08, 0x1a3bc84c17b1d543},
+ {0xe41f3d6a7377eeca, 0x20caba5f1d9e4a94},
+ {0x8e938662882af53e, 0x547eb47b7282ee9d},
+ {0xb23867fb2a35b28d, 0xe99e619a4f23aa44},
+ {0xdec681f9f4c31f31, 0x6405fa00e2ec94d5},
+ {0x8b3c113c38f9f37e, 0xde83bc408dd3dd05},
+ {0xae0b158b4738705e, 0x9624ab50b148d446},
+ {0xd98ddaee19068c76, 0x3badd624dd9b0958},
+ {0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d7},
+ {0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4d},
+ {0xd47487cc8470652b, 0x7647c32000696720},
+ {0x84c8d4dfd2c63f3b, 0x29ecd9f40041e074},
+ {0xa5fb0a17c777cf09, 0xf468107100525891},
+ {0xcf79cc9db955c2cc, 0x7182148d4066eeb5},
+ {0x81ac1fe293d599bf, 0xc6f14cd848405531},
+ {0xa21727db38cb002f, 0xb8ada00e5a506a7d},
+ {0xca9cf1d206fdc03b, 0xa6d90811f0e4851d},
+ {0xfd442e4688bd304a, 0x908f4a166d1da664},
+ {0x9e4a9cec15763e2e, 0x9a598e4e043287ff},
+ {0xc5dd44271ad3cdba, 0x40eff1e1853f29fe},
+ {0xf7549530e188c128, 0xd12bee59e68ef47d},
+ {0x9a94dd3e8cf578b9, 0x82bb74f8301958cf},
+ {0xc13a148e3032d6e7, 0xe36a52363c1faf02},
+ {0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac2},
+ {0x96f5600f15a7b7e5, 0x29ab103a5ef8c0ba},
+ {0xbcb2b812db11a5de, 0x7415d448f6b6f0e8},
+ {0xebdf661791d60f56, 0x111b495b3464ad22},
+ {0x936b9fcebb25c995, 0xcab10dd900beec35},
+ {0xb84687c269ef3bfb, 0x3d5d514f40eea743},
+ {0xe65829b3046b0afa, 0x0cb4a5a3112a5113},
+ {0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ac},
+ {0xb3f4e093db73a093, 0x59ed216765690f57},
+ {0xe0f218b8d25088b8, 0x306869c13ec3532d},
+ {0x8c974f7383725573, 0x1e414218c73a13fc},
+ {0xafbd2350644eeacf, 0xe5d1929ef90898fb},
+ {0xdbac6c247d62a583, 0xdf45f746b74abf3a},
+ {0x894bc396ce5da772, 0x6b8bba8c328eb784},
+ {0xab9eb47c81f5114f, 0x066ea92f3f326565},
+ {0xd686619ba27255a2, 0xc80a537b0efefebe},
+ {0x8613fd0145877585, 0xbd06742ce95f5f37},
+ {0xa798fc4196e952e7, 0x2c48113823b73705},
+ {0xd17f3b51fca3a7a0, 0xf75a15862ca504c6},
+ {0x82ef85133de648c4, 0x9a984d73dbe722fc},
+ {0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebbb},
+ {0xcc963fee10b7d1b3, 0x318df905079926a9},
+ {0xffbbcfe994e5c61f, 0xfdf17746497f7053},
+ {0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa634},
+ {0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc1},
+ {0xf9bd690a1b68637b, 0x3dfdce7aa3c673b1},
+ {0x9c1661a651213e2d, 0x06bea10ca65c084f},
+ {0xc31bfa0fe5698db8, 0x486e494fcff30a63},
+ {0xf3e2f893dec3f126, 0x5a89dba3c3efccfb},
+ {0x986ddb5c6b3a76b7, 0xf89629465a75e01d},
+ {0xbe89523386091465, 0xf6bbb397f1135824},
+ {0xee2ba6c0678b597f, 0x746aa07ded582e2d},
+ {0x94db483840b717ef, 0xa8c2a44eb4571cdd},
+ {0xba121a4650e4ddeb, 0x92f34d62616ce414},
+ {0xe896a0d7e51e1566, 0x77b020baf9c81d18},
+ {0x915e2486ef32cd60, 0x0ace1474dc1d122f},
+ {0xb5b5ada8aaff80b8, 0x0d819992132456bb},
+ {0xe3231912d5bf60e6, 0x10e1fff697ed6c6a},
+ {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2},
+ {0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb3},
+ {0xddd0467c64bce4a0, 0xac7cb3f6d05ddbdf},
+ {0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96c},
+ {0xad4ab7112eb3929d, 0x86c16c98d2c953c7},
+ {0xd89d64d57a607744, 0xe871c7bf077ba8b8},
+ {0x87625f056c7c4a8b, 0x11471cd764ad4973},
+ {0xa93af6c6c79b5d2d, 0xd598e40d3dd89bd0},
+ {0xd389b47879823479, 0x4aff1d108d4ec2c4},
+ {0x843610cb4bf160cb, 0xcedf722a585139bb},
+ {0xa54394fe1eedb8fe, 0xc2974eb4ee658829},
+ {0xce947a3da6a9273e, 0x733d226229feea33},
+ {0x811ccc668829b887, 0x0806357d5a3f5260},
+ {0xa163ff802a3426a8, 0xca07c2dcb0cf26f8},
+ {0xc9bcff6034c13052, 0xfc89b393dd02f0b6},
+ {0xfc2c3f3841f17c67, 0xbbac2078d443ace3},
+ {0x9d9ba7832936edc0, 0xd54b944b84aa4c0e},
+ {0xc5029163f384a931, 0x0a9e795e65d4df12},
+ {0xf64335bcf065d37d, 0x4d4617b5ff4a16d6},
+ {0x99ea0196163fa42e, 0x504bced1bf8e4e46},
+ {0xc06481fb9bcf8d39, 0xe45ec2862f71e1d7},
+ {0xf07da27a82c37088, 0x5d767327bb4e5a4d},
+ {0x964e858c91ba2655, 0x3a6a07f8d510f870},
+ {0xbbe226efb628afea, 0x890489f70a55368c},
+ {0xeadab0aba3b2dbe5, 0x2b45ac74ccea842f},
+ {0x92c8ae6b464fc96f, 0x3b0b8bc90012929e},
+ {0xb77ada0617e3bbcb, 0x09ce6ebb40173745},
+ {0xe55990879ddcaabd, 0xcc420a6a101d0516},
+ {0x8f57fa54c2a9eab6, 0x9fa946824a12232e},
+ {0xb32df8e9f3546564, 0x47939822dc96abfa},
+ {0xdff9772470297ebd, 0x59787e2b93bc56f8},
+ {0x8bfbea76c619ef36, 0x57eb4edb3c55b65b},
+ {0xaefae51477a06b03, 0xede622920b6b23f2},
+ {0xdab99e59958885c4, 0xe95fab368e45ecee},
+ {0x88b402f7fd75539b, 0x11dbcb0218ebb415},
+ {0xaae103b5fcd2a881, 0xd652bdc29f26a11a},
+ {0xd59944a37c0752a2, 0x4be76d3346f04960},
+ {0x857fcae62d8493a5, 0x6f70a4400c562ddc},
+ {0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb953},
+ {0xd097ad07a71f26b2, 0x7e2000a41346a7a8},
+ {0x825ecc24c873782f, 0x8ed400668c0c28c9},
+ {0xa2f67f2dfa90563b, 0x728900802f0f32fb},
+ {0xcbb41ef979346bca, 0x4f2b40a03ad2ffba},
+ {0xfea126b7d78186bc, 0xe2f610c84987bfa9},
+ {0x9f24b832e6b0f436, 0x0dd9ca7d2df4d7ca},
+ {0xc6ede63fa05d3143, 0x91503d1c79720dbc},
+ {0xf8a95fcf88747d94, 0x75a44c6397ce912b},
+ {0x9b69dbe1b548ce7c, 0xc986afbe3ee11abb},
+ {0xc24452da229b021b, 0xfbe85badce996169},
+ {0xf2d56790ab41c2a2, 0xfae27299423fb9c4},
+ {0x97c560ba6b0919a5, 0xdccd879fc967d41b},
+ {0xbdb6b8e905cb600f, 0x5400e987bbc1c921},
+ {0xed246723473e3813, 0x290123e9aab23b69},
+ {0x9436c0760c86e30b, 0xf9a0b6720aaf6522},
+ {0xb94470938fa89bce, 0xf808e40e8d5b3e6a},
+ {0xe7958cb87392c2c2, 0xb60b1d1230b20e05},
+ {0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c3},
+ {0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af4},
+ {0xe2280b6c20dd5232, 0x25c6da63c38de1b1},
+ {0x8d590723948a535f, 0x579c487e5a38ad0f},
+ {0xb0af48ec79ace837, 0x2d835a9df0c6d852},
+ {0xdcdb1b2798182244, 0xf8e431456cf88e66},
+ {0x8a08f0f8bf0f156b, 0x1b8e9ecb641b5900},
+ {0xac8b2d36eed2dac5, 0xe272467e3d222f40},
+ {0xd7adf884aa879177, 0x5b0ed81dcc6abb10},
+ {0x86ccbb52ea94baea, 0x98e947129fc2b4ea},
+ {0xa87fea27a539e9a5, 0x3f2398d747b36225},
+ {0xd29fe4b18e88640e, 0x8eec7f0d19a03aae},
+ {0x83a3eeeef9153e89, 0x1953cf68300424ad},
+ {0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd8},
+ {0xcdb02555653131b6, 0x3792f412cb06794e},
+ {0x808e17555f3ebf11, 0xe2bbd88bbee40bd1},
+ {0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec5},
+ {0xc8de047564d20a8b, 0xf245825a5a445276},
+ {0xfb158592be068d2e, 0xeed6e2f0f0d56713},
+ {0x9ced737bb6c4183d, 0x55464dd69685606c},
+ {0xc428d05aa4751e4c, 0xaa97e14c3c26b887},
+ {0xf53304714d9265df, 0xd53dd99f4b3066a9},
+ {0x993fe2c6d07b7fab, 0xe546a8038efe402a},
+ {0xbf8fdb78849a5f96, 0xde98520472bdd034},
+ {0xef73d256a5c0f77c, 0x963e66858f6d4441},
+ {0x95a8637627989aad, 0xdde7001379a44aa9},
+ {0xbb127c53b17ec159, 0x5560c018580d5d53},
+ {0xe9d71b689dde71af, 0xaab8f01e6e10b4a7},
+ {0x9226712162ab070d, 0xcab3961304ca70e9},
+ {0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d23},
+ {0xe45c10c42a2b3b05, 0x8cb89a7db77c506b},
+ {0x8eb98a7a9a5b04e3, 0x77f3608e92adb243},
+ {0xb267ed1940f1c61c, 0x55f038b237591ed4},
+ {0xdf01e85f912e37a3, 0x6b6c46dec52f6689},
+ {0x8b61313bbabce2c6, 0x2323ac4b3b3da016},
+ {0xae397d8aa96c1b77, 0xabec975e0a0d081b},
+ {0xd9c7dced53c72255, 0x96e7bd358c904a22},
+ {0x881cea14545c7575, 0x7e50d64177da2e55},
+ {0xaa242499697392d2, 0xdde50bd1d5d0b9ea},
+ {0xd4ad2dbfc3d07787, 0x955e4ec64b44e865},
+ {0x84ec3c97da624ab4, 0xbd5af13bef0b113f},
+ {0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58f},
+ {0xcfb11ead453994ba, 0x67de18eda5814af3},
+ {0x81ceb32c4b43fcf4, 0x80eacf948770ced8},
+ {0xa2425ff75e14fc31, 0xa1258379a94d028e},
+ {0xcad2f7f5359a3b3e, 0x096ee45813a04331},
+ {0xfd87b5f28300ca0d, 0x8bca9d6e188853fd},
+ {0x9e74d1b791e07e48, 0x775ea264cf55347e},
+ {0xc612062576589dda, 0x95364afe032a819e},
+ {0xf79687aed3eec551, 0x3a83ddbd83f52205},
+ {0x9abe14cd44753b52, 0xc4926a9672793543},
+ {0xc16d9a0095928a27, 0x75b7053c0f178294},
+ {0xf1c90080baf72cb1, 0x5324c68b12dd6339},
+ {0x971da05074da7bee, 0xd3f6fc16ebca5e04},
+ {0xbce5086492111aea, 0x88f4bb1ca6bcf585},
+ {0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6},
+ {0x9392ee8e921d5d07, 0x3aff322e62439fd0},
+ {0xb877aa3236a4b449, 0x09befeb9fad487c3},
+ {0xe69594bec44de15b, 0x4c2ebe687989a9b4},
+ {0x901d7cf73ab0acd9, 0x0f9d37014bf60a11},
+ {0xb424dc35095cd80f, 0x538484c19ef38c95},
+ {0xe12e13424bb40e13, 0x2865a5f206b06fba},
+ {0x8cbccc096f5088cb, 0xf93f87b7442e45d4},
+ {0xafebff0bcb24aafe, 0xf78f69a51539d749},
+ {0xdbe6fecebdedd5be, 0xb573440e5a884d1c},
+ {0x89705f4136b4a597, 0x31680a88f8953031},
+ {0xabcc77118461cefc, 0xfdc20d2b36ba7c3e},
+ {0xd6bf94d5e57a42bc, 0x3d32907604691b4d},
+ {0x8637bd05af6c69b5, 0xa63f9a49c2c1b110},
+ {0xa7c5ac471b478423, 0x0fcf80dc33721d54},
+ {0xd1b71758e219652b, 0xd3c36113404ea4a9},
+ {0x83126e978d4fdf3b, 0x645a1cac083126ea},
+ {0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4},
+ {0xcccccccccccccccc, 0xcccccccccccccccd},
+ {0x8000000000000000, 0x0000000000000000},
+ {0xa000000000000000, 0x0000000000000000},
+ {0xc800000000000000, 0x0000000000000000},
+ {0xfa00000000000000, 0x0000000000000000},
+ {0x9c40000000000000, 0x0000000000000000},
+ {0xc350000000000000, 0x0000000000000000},
+ {0xf424000000000000, 0x0000000000000000},
+ {0x9896800000000000, 0x0000000000000000},
+ {0xbebc200000000000, 0x0000000000000000},
+ {0xee6b280000000000, 0x0000000000000000},
+ {0x9502f90000000000, 0x0000000000000000},
+ {0xba43b74000000000, 0x0000000000000000},
+ {0xe8d4a51000000000, 0x0000000000000000},
+ {0x9184e72a00000000, 0x0000000000000000},
+ {0xb5e620f480000000, 0x0000000000000000},
+ {0xe35fa931a0000000, 0x0000000000000000},
+ {0x8e1bc9bf04000000, 0x0000000000000000},
+ {0xb1a2bc2ec5000000, 0x0000000000000000},
+ {0xde0b6b3a76400000, 0x0000000000000000},
+ {0x8ac7230489e80000, 0x0000000000000000},
+ {0xad78ebc5ac620000, 0x0000000000000000},
+ {0xd8d726b7177a8000, 0x0000000000000000},
+ {0x878678326eac9000, 0x0000000000000000},
+ {0xa968163f0a57b400, 0x0000000000000000},
+ {0xd3c21bcecceda100, 0x0000000000000000},
+ {0x84595161401484a0, 0x0000000000000000},
+ {0xa56fa5b99019a5c8, 0x0000000000000000},
+ {0xcecb8f27f4200f3a, 0x0000000000000000},
+ {0x813f3978f8940984, 0x4000000000000000},
+ {0xa18f07d736b90be5, 0x5000000000000000},
+ {0xc9f2c9cd04674ede, 0xa400000000000000},
+ {0xfc6f7c4045812296, 0x4d00000000000000},
+ {0x9dc5ada82b70b59d, 0xf020000000000000},
+ {0xc5371912364ce305, 0x6c28000000000000},
+ {0xf684df56c3e01bc6, 0xc732000000000000},
+ {0x9a130b963a6c115c, 0x3c7f400000000000},
+ {0xc097ce7bc90715b3, 0x4b9f100000000000},
+ {0xf0bdc21abb48db20, 0x1e86d40000000000},
+ {0x96769950b50d88f4, 0x1314448000000000},
+ {0xbc143fa4e250eb31, 0x17d955a000000000},
+ {0xeb194f8e1ae525fd, 0x5dcfab0800000000},
+ {0x92efd1b8d0cf37be, 0x5aa1cae500000000},
+ {0xb7abc627050305ad, 0xf14a3d9e40000000},
+ {0xe596b7b0c643c719, 0x6d9ccd05d0000000},
+ {0x8f7e32ce7bea5c6f, 0xe4820023a2000000},
+ {0xb35dbf821ae4f38b, 0xdda2802c8a800000},
+ {0xe0352f62a19e306e, 0xd50b2037ad200000},
+ {0x8c213d9da502de45, 0x4526f422cc340000},
+ {0xaf298d050e4395d6, 0x9670b12b7f410000},
+ {0xdaf3f04651d47b4c, 0x3c0cdd765f114000},
+ {0x88d8762bf324cd0f, 0xa5880a69fb6ac800},
+ {0xab0e93b6efee0053, 0x8eea0d047a457a00},
+ {0xd5d238a4abe98068, 0x72a4904598d6d880},
+ {0x85a36366eb71f041, 0x47a6da2b7f864750},
+ {0xa70c3c40a64e6c51, 0x999090b65f67d924},
+ {0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d},
+ {0x82818f1281ed449f, 0xbff8f10e7a8921a5},
+ {0xa321f2d7226895c7, 0xaff72d52192b6a0e},
+ {0xcbea6f8ceb02bb39, 0x9bf4f8a69f764491},
+ {0xfee50b7025c36a08, 0x02f236d04753d5b5},
+ {0x9f4f2726179a2245, 0x01d762422c946591},
+ {0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef6},
+ {0xf8ebad2b84e0d58b, 0xd2e0898765a7deb3},
+ {0x9b934c3b330c8577, 0x63cc55f49f88eb30},
+ {0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fc},
+ {0xf316271c7fc3908a, 0x8bef464e3945ef7b},
+ {0x97edd871cfda3a56, 0x97758bf0e3cbb5ad},
+ {0xbde94e8e43d0c8ec, 0x3d52eeed1cbea318},
+ {0xed63a231d4c4fb27, 0x4ca7aaa863ee4bde},
+ {0x945e455f24fb1cf8, 0x8fe8caa93e74ef6b},
+ {0xb975d6b6ee39e436, 0xb3e2fd538e122b45},
+ {0xe7d34c64a9c85d44, 0x60dbbca87196b617},
+ {0x90e40fbeea1d3a4a, 0xbc8955e946fe31ce},
+ {0xb51d13aea4a488dd, 0x6babab6398bdbe42},
+ {0xe264589a4dcdab14, 0xc696963c7eed2dd2},
+ {0x8d7eb76070a08aec, 0xfc1e1de5cf543ca3},
+ {0xb0de65388cc8ada8, 0x3b25a55f43294bcc},
+ {0xdd15fe86affad912, 0x49ef0eb713f39ebf},
+ {0x8a2dbf142dfcc7ab, 0x6e3569326c784338},
+ {0xacb92ed9397bf996, 0x49c2c37f07965405},
+ {0xd7e77a8f87daf7fb, 0xdc33745ec97be907},
+ {0x86f0ac99b4e8dafd, 0x69a028bb3ded71a4},
+ {0xa8acd7c0222311bc, 0xc40832ea0d68ce0d},
+ {0xd2d80db02aabd62b, 0xf50a3fa490c30191},
+ {0x83c7088e1aab65db, 0x792667c6da79e0fb},
+ {0xa4b8cab1a1563f52, 0x577001b891185939},
+ {0xcde6fd5e09abcf26, 0xed4c0226b55e6f87},
+ {0x80b05e5ac60b6178, 0x544f8158315b05b5},
+ {0xa0dc75f1778e39d6, 0x696361ae3db1c722},
+ {0xc913936dd571c84c, 0x03bc3a19cd1e38ea},
+ {0xfb5878494ace3a5f, 0x04ab48a04065c724},
+ {0x9d174b2dcec0e47b, 0x62eb0d64283f9c77},
+ {0xc45d1df942711d9a, 0x3ba5d0bd324f8395},
+ {0xf5746577930d6500, 0xca8f44ec7ee3647a},
+ {0x9968bf6abbe85f20, 0x7e998b13cf4e1ecc},
+ {0xbfc2ef456ae276e8, 0x9e3fedd8c321a67f},
+ {0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101f},
+ {0x95d04aee3b80ece5, 0xbba1f1d158724a13},
+ {0xbb445da9ca61281f, 0x2a8a6e45ae8edc98},
+ {0xea1575143cf97226, 0xf52d09d71a3293be},
+ {0x924d692ca61be758, 0x593c2626705f9c57},
+ {0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836d},
+ {0xe498f455c38b997a, 0x0b6dfb9c0f956448},
+ {0x8edf98b59a373fec, 0x4724bd4189bd5ead},
+ {0xb2977ee300c50fe7, 0x58edec91ec2cb658},
+ {0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ee},
+ {0x8b865b215899f46c, 0xbd79e0d20082ee75},
+ {0xae67f1e9aec07187, 0xecd8590680a3aa12},
+ {0xda01ee641a708de9, 0xe80e6f4820cc9496},
+ {0x884134fe908658b2, 0x3109058d147fdcde},
+ {0xaa51823e34a7eede, 0xbd4b46f0599fd416},
+ {0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91b},
+ {0x850fadc09923329e, 0x03e2cf6bc604ddb1},
+ {0xa6539930bf6bff45, 0x84db8346b786151d},
+ {0xcfe87f7cef46ff16, 0xe612641865679a64},
+ {0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07f},
+ {0xa26da3999aef7749, 0xe3be5e330f38f09e},
+ {0xcb090c8001ab551c, 0x5cadf5bfd3072cc6},
+ {0xfdcb4fa002162a63, 0x73d9732fc7c8f7f7},
+ {0x9e9f11c4014dda7e, 0x2867e7fddcdd9afb},
+ {0xc646d63501a1511d, 0xb281e1fd541501b9},
+ {0xf7d88bc24209a565, 0x1f225a7ca91a4227},
+ {0x9ae757596946075f, 0x3375788de9b06959},
+ {0xc1a12d2fc3978937, 0x0052d6b1641c83af},
+ {0xf209787bb47d6b84, 0xc0678c5dbd23a49b},
+ {0x9745eb4d50ce6332, 0xf840b7ba963646e1},
+ {0xbd176620a501fbff, 0xb650e5a93bc3d899},
+ {0xec5d3fa8ce427aff, 0xa3e51f138ab4cebf},
+ {0x93ba47c980e98cdf, 0xc66f336c36b10138},
+ {0xb8a8d9bbe123f017, 0xb80b0047445d4185},
+ {0xe6d3102ad96cec1d, 0xa60dc059157491e6},
+ {0x9043ea1ac7e41392, 0x87c89837ad68db30},
+ {0xb454e4a179dd1877, 0x29babe4598c311fc},
+ {0xe16a1dc9d8545e94, 0xf4296dd6fef3d67b},
+ {0x8ce2529e2734bb1d, 0x1899e4a65f58660d},
+ {0xb01ae745b101e9e4, 0x5ec05dcff72e7f90},
+ {0xdc21a1171d42645d, 0x76707543f4fa1f74},
+ {0x899504ae72497eba, 0x6a06494a791c53a9},
+ {0xabfa45da0edbde69, 0x0487db9d17636893},
+ {0xd6f8d7509292d603, 0x45a9d2845d3c42b7},
+ {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b3},
+ {0xa7f26836f282b732, 0x8e6cac7768d7141f},
+ {0xd1ef0244af2364ff, 0x3207d795430cd927},
+ {0x8335616aed761f1f, 0x7f44e6bd49e807b9},
+ {0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a7},
+ {0xcd036837130890a1, 0x36dba887c37a8c10},
+ {0x802221226be55a64, 0xc2494954da2c978a},
+ {0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6d},
+ {0xc83553c5c8965d3d, 0x6f92829494e5acc8},
+ {0xfa42a8b73abbf48c, 0xcb772339ba1f17fa},
+ {0x9c69a97284b578d7, 0xff2a760414536efc},
+ {0xc38413cf25e2d70d, 0xfef5138519684abb},
+ {0xf46518c2ef5b8cd1, 0x7eb258665fc25d6a},
+ {0x98bf2f79d5993802, 0xef2f773ffbd97a62},
+ {0xbeeefb584aff8603, 0xaafb550ffacfd8fb},
+ {0xeeaaba2e5dbf6784, 0x95ba2a53f983cf39},
+ {0x952ab45cfa97a0b2, 0xdd945a747bf26184},
+ {0xba756174393d88df, 0x94f971119aeef9e5},
+ {0xe912b9d1478ceb17, 0x7a37cd5601aab85e},
+ {0x91abb422ccb812ee, 0xac62e055c10ab33b},
+ {0xb616a12b7fe617aa, 0x577b986b314d600a},
+ {0xe39c49765fdf9d94, 0xed5a7e85fda0b80c},
+ {0x8e41ade9fbebc27d, 0x14588f13be847308},
+ {0xb1d219647ae6b31c, 0x596eb2d8ae258fc9},
+ {0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bc},
+ {0x8aec23d680043bee, 0x25de7bb9480d5855},
+ {0xada72ccc20054ae9, 0xaf561aa79a10ae6b},
+ {0xd910f7ff28069da4, 0x1b2ba1518094da05},
+ {0x87aa9aff79042286, 0x90fb44d2f05d0843},
+ {0xa99541bf57452b28, 0x353a1607ac744a54},
+ {0xd3fa922f2d1675f2, 0x42889b8997915ce9},
+ {0x847c9b5d7c2e09b7, 0x69956135febada12},
+ {0xa59bc234db398c25, 0x43fab9837e699096},
+ {0xcf02b2c21207ef2e, 0x94f967e45e03f4bc},
+ {0x8161afb94b44f57d, 0x1d1be0eebac278f6},
+ {0xa1ba1ba79e1632dc, 0x6462d92a69731733},
+ {0xca28a291859bbf93, 0x7d7b8f7503cfdcff},
+ {0xfcb2cb35e702af78, 0x5cda735244c3d43f},
+ {0x9defbf01b061adab, 0x3a0888136afa64a8},
+ {0xc56baec21c7a1916, 0x088aaa1845b8fdd1},
+ {0xf6c69a72a3989f5b, 0x8aad549e57273d46},
+ {0x9a3c2087a63f6399, 0x36ac54e2f678864c},
+ {0xc0cb28a98fcf3c7f, 0x84576a1bb416a7de},
+ {0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d6},
+ {0x969eb7c47859e743, 0x9f644ae5a4b1b326},
+ {0xbc4665b596706114, 0x873d5d9f0dde1fef},
+ {0xeb57ff22fc0c7959, 0xa90cb506d155a7eb},
+ {0x9316ff75dd87cbd8, 0x09a7f12442d588f3},
+ {0xb7dcbf5354e9bece, 0x0c11ed6d538aeb30},
+ {0xe5d3ef282a242e81, 0x8f1668c8a86da5fb},
+ {0x8fa475791a569d10, 0xf96e017d694487bd},
+ {0xb38d92d760ec4455, 0x37c981dcc395a9ad},
+ {0xe070f78d3927556a, 0x85bbe253f47b1418},
+ {0x8c469ab843b89562, 0x93956d7478ccec8f},
+ {0xaf58416654a6babb, 0x387ac8d1970027b3},
+ {0xdb2e51bfe9d0696a, 0x06997b05fcc0319f},
+ {0x88fcf317f22241e2, 0x441fece3bdf81f04},
+ {0xab3c2fddeeaad25a, 0xd527e81cad7626c4},
+ {0xd60b3bd56a5586f1, 0x8a71e223d8d3b075},
+ {0x85c7056562757456, 0xf6872d5667844e4a},
+ {0xa738c6bebb12d16c, 0xb428f8ac016561dc},
+ {0xd106f86e69d785c7, 0xe13336d701beba53},
+ {0x82a45b450226b39c, 0xecc0024661173474},
+ {0xa34d721642b06084, 0x27f002d7f95d0191},
+ {0xcc20ce9bd35c78a5, 0x31ec038df7b441f5},
+ {0xff290242c83396ce, 0x7e67047175a15272},
+ {0x9f79a169bd203e41, 0x0f0062c6e984d387},
+ {0xc75809c42c684dd1, 0x52c07b78a3e60869},
+ {0xf92e0c3537826145, 0xa7709a56ccdf8a83},
+ {0x9bbcc7a142b17ccb, 0x88a66076400bb692},
+ {0xc2abf989935ddbfe, 0x6acff893d00ea436},
+ {0xf356f7ebf83552fe, 0x0583f6b8c4124d44},
+ {0x98165af37b2153de, 0xc3727a337a8b704b},
+ {0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5d},
+ {0xeda2ee1c7064130c, 0x1162def06f79df74},
+ {0x9485d4d1c63e8be7, 0x8addcb5645ac2ba9},
+ {0xb9a74a0637ce2ee1, 0x6d953e2bd7173693},
+ {0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0438},
+ {0x910ab1d4db9914a0, 0x1d9c9892400a22a3},
+ {0xb54d5e4a127f59c8, 0x2503beb6d00cab4c},
+ {0xe2a0b5dc971f303a, 0x2e44ae64840fd61e},
+ {0x8da471a9de737e24, 0x5ceaecfed289e5d3},
+ {0xb10d8e1456105dad, 0x7425a83e872c5f48},
+ {0xdd50f1996b947518, 0xd12f124e28f7771a},
+ {0x8a5296ffe33cc92f, 0x82bd6b70d99aaa70},
+ {0xace73cbfdc0bfb7b, 0x636cc64d1001550c},
+ {0xd8210befd30efa5a, 0x3c47f7e05401aa4f},
+ {0x8714a775e3e95c78, 0x65acfaec34810a72},
+ {0xa8d9d1535ce3b396, 0x7f1839a741a14d0e},
+ {0xd31045a8341ca07c, 0x1ede48111209a051},
+ {0x83ea2b892091e44d, 0x934aed0aab460433},
+ {0xa4e4b66b68b65d60, 0xf81da84d56178540},
+ {0xce1de40642e3f4b9, 0x36251260ab9d668f},
+ {0x80d2ae83e9ce78f3, 0xc1d72b7c6b42601a},
+ {0xa1075a24e4421730, 0xb24cf65b8612f820},
+ {0xc94930ae1d529cfc, 0xdee033f26797b628},
+ {0xfb9b7cd9a4a7443c, 0x169840ef017da3b2},
+ {0x9d412e0806e88aa5, 0x8e1f289560ee864f},
+ {0xc491798a08a2ad4e, 0xf1a6f2bab92a27e3},
+ {0xf5b5d7ec8acb58a2, 0xae10af696774b1dc},
+ {0x9991a6f3d6bf1765, 0xacca6da1e0a8ef2a},
+ {0xbff610b0cc6edd3f, 0x17fd090a58d32af4},
+ {0xeff394dcff8a948e, 0xddfc4b4cef07f5b1},
+ {0x95f83d0a1fb69cd9, 0x4abdaf101564f98f},
+ {0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f2},
+ {0xea53df5fd18d5513, 0x84c86189216dc5ee},
+ {0x92746b9be2f8552c, 0x32fd3cf5b4e49bb5},
+ {0xb7118682dbb66a77, 0x3fbc8c33221dc2a2},
+ {0xe4d5e82392a40515, 0x0fabaf3feaa5334b},
+ {0x8f05b1163ba6832d, 0x29cb4d87f2a7400f},
+ {0xb2c71d5bca9023f8, 0x743e20e9ef511013},
+ {0xdf78e4b2bd342cf6, 0x914da9246b255417},
+ {0x8bab8eefb6409c1a, 0x1ad089b6c2f7548f},
+ {0xae9672aba3d0c320, 0xa184ac2473b529b2},
+ {0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741f},
+ {0x8865899617fb1871, 0x7e2fa67c7a658893},
+ {0xaa7eebfb9df9de8d, 0xddbb901b98feeab8},
+ {0xd51ea6fa85785631, 0x552a74227f3ea566},
+ {0x8533285c936b35de, 0xd53a88958f872760},
+ {0xa67ff273b8460356, 0x8a892abaf368f138},
+ {0xd01fef10a657842c, 0x2d2b7569b0432d86},
+ {0x8213f56a67f6b29b, 0x9c3b29620e29fc74},
+ {0xa298f2c501f45f42, 0x8349f3ba91b47b90},
+ {0xcb3f2f7642717713, 0x241c70a936219a74},
+ {0xfe0efb53d30dd4d7, 0xed238cd383aa0111},
+ {0x9ec95d1463e8a506, 0xf4363804324a40ab},
+ {0xc67bb4597ce2ce48, 0xb143c6053edcd0d6},
+ {0xf81aa16fdc1b81da, 0xdd94b7868e94050b},
+ {0x9b10a4e5e9913128, 0xca7cf2b4191c8327},
+ {0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f1},
+ {0xf24a01a73cf2dccf, 0xbc633b39673c8ced},
+ {0x976e41088617ca01, 0xd5be0503e085d814},
+ {0xbd49d14aa79dbc82, 0x4b2d8644d8a74e19},
+ {0xec9c459d51852ba2, 0xddf8e7d60ed1219f},
+ {0x93e1ab8252f33b45, 0xcabb90e5c942b504},
+ {0xb8da1662e7b00a17, 0x3d6a751f3b936244},
+ {0xe7109bfba19c0c9d, 0x0cc512670a783ad5},
+ {0x906a617d450187e2, 0x27fb2b80668b24c6},
+ {0xb484f9dc9641e9da, 0xb1f9f660802dedf7},
+ {0xe1a63853bbd26451, 0x5e7873f8a0396974},
+ {0x8d07e33455637eb2, 0xdb0b487b6423e1e9},
+ {0xb049dc016abc5e5f, 0x91ce1a9a3d2cda63},
+ {0xdc5c5301c56b75f7, 0x7641a140cc7810fc},
+ {0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9e},
+ {0xac2820d9623bf429, 0x546345fa9fbdcd45},
+ {0xd732290fbacaf133, 0xa97c177947ad4096},
+ {0x867f59a9d4bed6c0, 0x49ed8eabcccc485e},
+ {0xa81f301449ee8c70, 0x5c68f256bfff5a75},
+ {0xd226fc195c6a2f8c, 0x73832eec6fff3112},
+ {0x83585d8fd9c25db7, 0xc831fd53c5ff7eac},
+ {0xa42e74f3d032f525, 0xba3e7ca8b77f5e56},
+ {0xcd3a1230c43fb26f, 0x28ce1bd2e55f35ec},
+ {0x80444b5e7aa7cf85, 0x7980d163cf5b81b4},
+ {0xa0555e361951c366, 0xd7e105bcc3326220},
+ {0xc86ab5c39fa63440, 0x8dd9472bf3fefaa8},
+ {0xfa856334878fc150, 0xb14f98f6f0feb952},
+ {0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d4},
+ {0xc3b8358109e84f07, 0x0a862f80ec4700c9},
+ {0xf4a642e14c6262c8, 0xcd27bb612758c0fb},
+ {0x98e7e9cccfbd7dbd, 0x8038d51cb897789d},
+ {0xbf21e44003acdd2c, 0xe0470a63e6bd56c4},
+ {0xeeea5d5004981478, 0x1858ccfce06cac75},
+ {0x95527a5202df0ccb, 0x0f37801e0c43ebc9},
+ {0xbaa718e68396cffd, 0xd30560258f54e6bb},
+ {0xe950df20247c83fd, 0x47c6b82ef32a206a},
+ {0x91d28b7416cdd27e, 0x4cdc331d57fa5442},
+ {0xb6472e511c81471d, 0xe0133fe4adf8e953},
+ {0xe3d8f9e563a198e5, 0x58180fddd97723a7},
+ {0x8e679c2f5e44ff8f, 0x570f09eaa7ea7649},
+ {0xb201833b35d63f73, 0x2cd2cc6551e513db},
+ {0xde81e40a034bcf4f, 0xf8077f7ea65e58d2},
+ {0x8b112e86420f6191, 0xfb04afaf27faf783},
+ {0xadd57a27d29339f6, 0x79c5db9af1f9b564},
+ {0xd94ad8b1c7380874, 0x18375281ae7822bd},
+ {0x87cec76f1c830548, 0x8f2293910d0b15b6},
+ {0xa9c2794ae3a3c69a, 0xb2eb3875504ddb23},
+ {0xd433179d9c8cb841, 0x5fa60692a46151ec},
+ {0x849feec281d7f328, 0xdbc7c41ba6bcd334},
+ {0xa5c7ea73224deff3, 0x12b9b522906c0801},
+ {0xcf39e50feae16bef, 0xd768226b34870a01},
+ {0x81842f29f2cce375, 0xe6a1158300d46641},
+ {0xa1e53af46f801c53, 0x60495ae3c1097fd1},
+ {0xca5e89b18b602368, 0x385bb19cb14bdfc5},
+ {0xfcf62c1dee382c42, 0x46729e03dd9ed7b6},
+ {0x9e19db92b4e31ba9, 0x6c07a2c26a8346d2},
+ {0xc5a05277621be293, 0xc7098b7305241886},
+ {0xf70867153aa2db38, 0xb8cbee4fc66d1ea8},
+ {0x9a65406d44a5c903, 0x737f74f1dc043329},
+ {0xc0fe908895cf3b44, 0x505f522e53053ff3},
+ {0xf13e34aabb430a15, 0x647726b9e7c68ff0},
+ {0x96c6e0eab509e64d, 0x5eca783430dc19f6},
+ {0xbc789925624c5fe0, 0xb67d16413d132073},
+ {0xeb96bf6ebadf77d8, 0xe41c5bd18c57e890},
+ {0x933e37a534cbaae7, 0x8e91b962f7b6f15a},
+ {0xb80dc58e81fe95a1, 0x723627bbb5a4adb1},
+ {0xe61136f2227e3b09, 0xcec3b1aaa30dd91d},
+ {0x8fcac257558ee4e6, 0x213a4f0aa5e8a7b2},
+ {0xb3bd72ed2af29e1f, 0xa988e2cd4f62d19e},
+ {0xe0accfa875af45a7, 0x93eb1b80a33b8606},
+ {0x8c6c01c9498d8b88, 0xbc72f130660533c4},
+ {0xaf87023b9bf0ee6a, 0xeb8fad7c7f8680b5},
+ { 0xdb68c2ca82ed2a05,
+ 0xa67398db9f6820e2 }
+#else
+ {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b},
+ {0xce5d73ff402d98e3, 0xfb0a3d212dc81290},
+ {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f},
+ {0x86a8d39ef77164bc, 0xae5dff9c02033198},
+ {0xd98ddaee19068c76, 0x3badd624dd9b0958},
+ {0xafbd2350644eeacf, 0xe5d1929ef90898fb},
+ {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2},
+ {0xe55990879ddcaabd, 0xcc420a6a101d0516},
+ {0xb94470938fa89bce, 0xf808e40e8d5b3e6a},
+ {0x95a8637627989aad, 0xdde7001379a44aa9},
+ {0xf1c90080baf72cb1, 0x5324c68b12dd6339},
+ {0xc350000000000000, 0x0000000000000000},
+ {0x9dc5ada82b70b59d, 0xf020000000000000},
+ {0xfee50b7025c36a08, 0x02f236d04753d5b5},
+ {0xcde6fd5e09abcf26, 0xed4c0226b55e6f87},
+ {0xa6539930bf6bff45, 0x84db8346b786151d},
+ {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b3},
+ {0xd910f7ff28069da4, 0x1b2ba1518094da05},
+ {0xaf58416654a6babb, 0x387ac8d1970027b3},
+ {0x8da471a9de737e24, 0x5ceaecfed289e5d3},
+ {0xe4d5e82392a40515, 0x0fabaf3feaa5334b},
+ {0xb8da1662e7b00a17, 0x3d6a751f3b936244},
+ {0x95527a5202df0ccb, 0x0f37801e0c43ebc9},
+ {0xf13e34aabb430a15, 0x647726b9e7c68ff0}
+#endif
+ };
+
+#if FMT_USE_FULL_CACHE_DRAGONBOX
+ return pow10_significands[k - float_info<double>::min_k];
+#else
+ static constexpr const uint64_t powers_of_5_64[] = {
+ 0x0000000000000001, 0x0000000000000005, 0x0000000000000019,
+ 0x000000000000007d, 0x0000000000000271, 0x0000000000000c35,
+ 0x0000000000003d09, 0x000000000001312d, 0x000000000005f5e1,
+ 0x00000000001dcd65, 0x00000000009502f9, 0x0000000002e90edd,
+ 0x000000000e8d4a51, 0x0000000048c27395, 0x000000016bcc41e9,
+ 0x000000071afd498d, 0x0000002386f26fc1, 0x000000b1a2bc2ec5,
+ 0x000003782dace9d9, 0x00001158e460913d, 0x000056bc75e2d631,
+ 0x0001b1ae4d6e2ef5, 0x000878678326eac9, 0x002a5a058fc295ed,
+ 0x00d3c21bcecceda1, 0x0422ca8b0a00a425, 0x14adf4b7320334b9};
+
+ static const int compression_ratio = 27;
+
+ // Compute base index.
+ int cache_index = (k - float_info<double>::min_k) / compression_ratio;
+ int kb = cache_index * compression_ratio + float_info<double>::min_k;
+ int offset = k - kb;
+
+ // Get base cache.
+ uint128_fallback base_cache = pow10_significands[cache_index];
+ if (offset == 0) return base_cache;
+
+ // Compute the required amount of bit-shift.
+ int alpha = floor_log2_pow10(kb + offset) - floor_log2_pow10(kb) - offset;
+ FMT_ASSERT(alpha > 0 && alpha < 64, "shifting error detected");
+
+ // Try to recover the real cache.
+ uint64_t pow5 = powers_of_5_64[offset];
+ uint128_fallback recovered_cache = umul128(base_cache.high(), pow5);
+ uint128_fallback middle_low = umul128(base_cache.low(), pow5);
+
+ recovered_cache += middle_low.high();
+
+ uint64_t high_to_middle = recovered_cache.high() << (64 - alpha);
+ uint64_t middle_to_low = recovered_cache.low() << (64 - alpha);
+
+ recovered_cache =
+ uint128_fallback{(recovered_cache.low() >> alpha) | high_to_middle,
+ ((middle_low.low() >> alpha) | middle_to_low)};
+ FMT_ASSERT(recovered_cache.low() + 1 != 0, "");
+ return {recovered_cache.high(), recovered_cache.low() + 1};
+#endif
+ }
+
+ struct compute_mul_result {
+ carrier_uint result;
+ bool is_integer;
+ };
+ struct compute_mul_parity_result {
+ bool parity;
+ bool is_integer;
+ };
+
+ static compute_mul_result compute_mul(
+ carrier_uint u, const cache_entry_type& cache) noexcept {
+ auto r = umul192_upper128(u, cache);
+ return {r.high(), r.low() == 0};
+ }
+
+ static uint32_t compute_delta(cache_entry_type const& cache,
+ int beta) noexcept {
+ return static_cast<uint32_t>(cache.high() >> (64 - 1 - beta));
+ }
+
+ static compute_mul_parity_result compute_mul_parity(
+ carrier_uint two_f, const cache_entry_type& cache, int beta) noexcept {
+ FMT_ASSERT(beta >= 1, "");
+ FMT_ASSERT(beta < 64, "");
+
+ auto r = umul192_lower128(two_f, cache);
+ return {((r.high() >> (64 - beta)) & 1) != 0,
+ ((r.high() << beta) | (r.low() >> (64 - beta))) == 0};
+ }
+
+ static carrier_uint compute_left_endpoint_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return (cache.high() -
+ (cache.high() >> (num_significand_bits<double>() + 2))) >>
+ (64 - num_significand_bits<double>() - 1 - beta);
+ }
+
+ static carrier_uint compute_right_endpoint_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return (cache.high() +
+ (cache.high() >> (num_significand_bits<double>() + 1))) >>
+ (64 - num_significand_bits<double>() - 1 - beta);
+ }
+
+ static carrier_uint compute_round_up_for_shorter_interval_case(
+ const cache_entry_type& cache, int beta) noexcept {
+ return ((cache.high() >> (64 - num_significand_bits<double>() - 2 - beta)) +
+ 1) /
+ 2;
+ }
+};
+
+FMT_FUNC uint128_fallback get_cached_power(int k) noexcept {
+ return cache_accessor<double>::get_cached_power(k);
+}
+
+// Various integer checks
+template <typename T>
+bool is_left_endpoint_integer_shorter_interval(int exponent) noexcept {
+ const int case_shorter_interval_left_endpoint_lower_threshold = 2;
+ const int case_shorter_interval_left_endpoint_upper_threshold = 3;
+ return exponent >= case_shorter_interval_left_endpoint_lower_threshold &&
+ exponent <= case_shorter_interval_left_endpoint_upper_threshold;
+}
+
+// Remove trailing zeros from n and return the number of zeros removed (float)
+FMT_INLINE int remove_trailing_zeros(uint32_t& n) noexcept {
+ FMT_ASSERT(n != 0, "");
+ // Modular inverse of 5 (mod 2^32): (mod_inv_5 * 5) mod 2^32 = 1.
+ // See https://github.com/fmtlib/fmt/issues/3163 for more details.
+ const uint32_t mod_inv_5 = 0xcccccccd;
+ // Casts are needed to workaround a bug in MSVC 19.22 and older.
+ const uint32_t mod_inv_25 =
+ static_cast<uint32_t>(uint64_t(mod_inv_5) * mod_inv_5);
+
+ int s = 0;
+ while (true) {
+ auto q = rotr(n * mod_inv_25, 2);
+ if (q > max_value<uint32_t>() / 100) break;
+ n = q;
+ s += 2;
+ }
+ auto q = rotr(n * mod_inv_5, 1);
+ if (q <= max_value<uint32_t>() / 10) {
+ n = q;
+ s |= 1;
+ }
+ return s;
+}
+
+// Removes trailing zeros and returns the number of zeros removed (double)
+FMT_INLINE int remove_trailing_zeros(uint64_t& n) noexcept {
+ FMT_ASSERT(n != 0, "");
+
+ // This magic number is ceil(2^90 / 10^8).
+ constexpr uint64_t magic_number = 12379400392853802749ull;
+ auto nm = umul128(n, magic_number);
+
+ // Is n is divisible by 10^8?
+ if ((nm.high() & ((1ull << (90 - 64)) - 1)) == 0 && nm.low() < magic_number) {
+ // If yes, work with the quotient.
+ auto n32 = static_cast<uint32_t>(nm.high() >> (90 - 64));
+
+ const uint32_t mod_inv_5 = 0xcccccccd;
+ const uint32_t mod_inv_25 = mod_inv_5 * mod_inv_5;
+
+ int s = 8;
+ while (true) {
+ auto q = rotr(n32 * mod_inv_25, 2);
+ if (q > max_value<uint32_t>() / 100) break;
+ n32 = q;
+ s += 2;
+ }
+ auto q = rotr(n32 * mod_inv_5, 1);
+ if (q <= max_value<uint32_t>() / 10) {
+ n32 = q;
+ s |= 1;
+ }
+
+ n = n32;
+ return s;
+ }
+
+ // If n is not divisible by 10^8, work with n itself.
+ const uint64_t mod_inv_5 = 0xcccccccccccccccd;
+ const uint64_t mod_inv_25 = mod_inv_5 * mod_inv_5;
+
+ int s = 0;
+ while (true) {
+ auto q = rotr(n * mod_inv_25, 2);
+ if (q > max_value<uint64_t>() / 100) break;
+ n = q;
+ s += 2;
+ }
+ auto q = rotr(n * mod_inv_5, 1);
+ if (q <= max_value<uint64_t>() / 10) {
+ n = q;
+ s |= 1;
+ }
+
+ return s;
+}
+
+// The main algorithm for shorter interval case
+template <typename T>
+FMT_INLINE decimal_fp<T> shorter_interval_case(int exponent) noexcept {
+ decimal_fp<T> ret_value;
+ // Compute k and beta
+ const int minus_k = floor_log10_pow2_minus_log10_4_over_3(exponent);
+ const int beta = exponent + floor_log2_pow10(-minus_k);
+
+ // Compute xi and zi
+ using cache_entry_type = typename cache_accessor<T>::cache_entry_type;
+ const cache_entry_type cache = cache_accessor<T>::get_cached_power(-minus_k);
+
+ auto xi = cache_accessor<T>::compute_left_endpoint_for_shorter_interval_case(
+ cache, beta);
+ auto zi = cache_accessor<T>::compute_right_endpoint_for_shorter_interval_case(
+ cache, beta);
+
+ // If the left endpoint is not an integer, increase it
+ if (!is_left_endpoint_integer_shorter_interval<T>(exponent)) ++xi;
+
+ // Try bigger divisor
+ ret_value.significand = zi / 10;
+
+ // If succeed, remove trailing zeros if necessary and return
+ if (ret_value.significand * 10 >= xi) {
+ ret_value.exponent = minus_k + 1;
+ ret_value.exponent += remove_trailing_zeros(ret_value.significand);
+ return ret_value;
+ }
+
+ // Otherwise, compute the round-up of y
+ ret_value.significand =
+ cache_accessor<T>::compute_round_up_for_shorter_interval_case(cache,
+ beta);
+ ret_value.exponent = minus_k;
+
+ // When tie occurs, choose one of them according to the rule
+ if (exponent >= float_info<T>::shorter_interval_tie_lower_threshold &&
+ exponent <= float_info<T>::shorter_interval_tie_upper_threshold) {
+ ret_value.significand = ret_value.significand % 2 == 0
+ ? ret_value.significand
+ : ret_value.significand - 1;
+ } else if (ret_value.significand < xi) {
+ ++ret_value.significand;
+ }
+ return ret_value;
+}
+
+template <typename T> decimal_fp<T> to_decimal(T x) noexcept {
+ // Step 1: integer promotion & Schubfach multiplier calculation.
+
+ using carrier_uint = typename float_info<T>::carrier_uint;
+ using cache_entry_type = typename cache_accessor<T>::cache_entry_type;
+ auto br = bit_cast<carrier_uint>(x);
+
+ // Extract significand bits and exponent bits.
+ const carrier_uint significand_mask =
+ (static_cast<carrier_uint>(1) << num_significand_bits<T>()) - 1;
+ carrier_uint significand = (br & significand_mask);
+ int exponent =
+ static_cast<int>((br & exponent_mask<T>()) >> num_significand_bits<T>());
+
+ if (exponent != 0) { // Check if normal.
+ exponent -= exponent_bias<T>() + num_significand_bits<T>();
+
+ // Shorter interval case; proceed like Schubfach.
+ // In fact, when exponent == 1 and significand == 0, the interval is
+ // regular. However, it can be shown that the end-results are anyway same.
+ if (significand == 0) return shorter_interval_case<T>(exponent);
+
+ significand |= (static_cast<carrier_uint>(1) << num_significand_bits<T>());
+ } else {
+ // Subnormal case; the interval is always regular.
+ if (significand == 0) return {0, 0};
+ exponent =
+ std::numeric_limits<T>::min_exponent - num_significand_bits<T>() - 1;
+ }
+
+ const bool include_left_endpoint = (significand % 2 == 0);
+ const bool include_right_endpoint = include_left_endpoint;
+
+ // Compute k and beta.
+ const int minus_k = floor_log10_pow2(exponent) - float_info<T>::kappa;
+ const cache_entry_type cache = cache_accessor<T>::get_cached_power(-minus_k);
+ const int beta = exponent + floor_log2_pow10(-minus_k);
+
+ // Compute zi and deltai.
+ // 10^kappa <= deltai < 10^(kappa + 1)
+ const uint32_t deltai = cache_accessor<T>::compute_delta(cache, beta);
+ const carrier_uint two_fc = significand << 1;
+
+ // For the case of binary32, the result of integer check is not correct for
+ // 29711844 * 2^-82
+ // = 6.1442653300000000008655037797566933477355632930994033813476... * 10^-18
+ // and 29711844 * 2^-81
+ // = 1.2288530660000000001731007559513386695471126586198806762695... * 10^-17,
+ // and they are the unique counterexamples. However, since 29711844 is even,
+ // this does not cause any problem for the endpoints calculations; it can only
+ // cause a problem when we need to perform integer check for the center.
+ // Fortunately, with these inputs, that branch is never executed, so we are
+ // fine.
+ const typename cache_accessor<T>::compute_mul_result z_mul =
+ cache_accessor<T>::compute_mul((two_fc | 1) << beta, cache);
+
+ // Step 2: Try larger divisor; remove trailing zeros if necessary.
+
+ // Using an upper bound on zi, we might be able to optimize the division
+ // better than the compiler; we are computing zi / big_divisor here.
+ decimal_fp<T> ret_value;
+ ret_value.significand = divide_by_10_to_kappa_plus_1(z_mul.result);
+ uint32_t r = static_cast<uint32_t>(z_mul.result - float_info<T>::big_divisor *
+ ret_value.significand);
+
+ if (r < deltai) {
+ // Exclude the right endpoint if necessary.
+ if (r == 0 && (z_mul.is_integer & !include_right_endpoint)) {
+ --ret_value.significand;
+ r = float_info<T>::big_divisor;
+ goto small_divisor_case_label;
+ }
+ } else if (r > deltai) {
+ goto small_divisor_case_label;
+ } else {
+ // r == deltai; compare fractional parts.
+ const typename cache_accessor<T>::compute_mul_parity_result x_mul =
+ cache_accessor<T>::compute_mul_parity(two_fc - 1, cache, beta);
+
+ if (!(x_mul.parity | (x_mul.is_integer & include_left_endpoint)))
+ goto small_divisor_case_label;
+ }
+ ret_value.exponent = minus_k + float_info<T>::kappa + 1;
+
+ // We may need to remove trailing zeros.
+ ret_value.exponent += remove_trailing_zeros(ret_value.significand);
+ return ret_value;
+
+ // Step 3: Find the significand with the smaller divisor.
+
+small_divisor_case_label:
+ ret_value.significand *= 10;
+ ret_value.exponent = minus_k + float_info<T>::kappa;
+
+ uint32_t dist = r - (deltai / 2) + (float_info<T>::small_divisor / 2);
+ const bool approx_y_parity =
+ ((dist ^ (float_info<T>::small_divisor / 2)) & 1) != 0;
+
+ // Is dist divisible by 10^kappa?
+ const bool divisible_by_small_divisor =
+ check_divisibility_and_divide_by_pow10<float_info<T>::kappa>(dist);
+
+ // Add dist / 10^kappa to the significand.
+ ret_value.significand += dist;
+
+ if (!divisible_by_small_divisor) return ret_value;
+
+ // Check z^(f) >= epsilon^(f).
+ // We have either yi == zi - epsiloni or yi == (zi - epsiloni) - 1,
+ // where yi == zi - epsiloni if and only if z^(f) >= epsilon^(f).
+ // Since there are only 2 possibilities, we only need to care about the
+ // parity. Also, zi and r should have the same parity since the divisor
+ // is an even number.
+ const auto y_mul = cache_accessor<T>::compute_mul_parity(two_fc, cache, beta);
+
+ // If z^(f) >= epsilon^(f), we might have a tie when z^(f) == epsilon^(f),
+ // or equivalently, when y is an integer.
+ if (y_mul.parity != approx_y_parity)
+ --ret_value.significand;
+ else if (y_mul.is_integer & (ret_value.significand % 2 != 0))
+ --ret_value.significand;
+ return ret_value;
+}
+} // namespace dragonbox
+} // namespace detail
+
+template <> struct formatter<detail::bigint> {
+ FMT_CONSTEXPR auto parse(format_parse_context& ctx)
+ -> format_parse_context::iterator {
+ return ctx.begin();
+ }
+
+ auto format(const detail::bigint& n, format_context& ctx) const
+ -> format_context::iterator {
+ auto out = ctx.out();
+ bool first = true;
+ for (auto i = n.bigits_.size(); i > 0; --i) {
+ auto value = n.bigits_[i - 1u];
+ if (first) {
+ out = format_to(out, FMT_STRING("{:x}"), value);
+ first = false;
+ continue;
+ }
+ out = format_to(out, FMT_STRING("{:08x}"), value);
+ }
+ if (n.exp_ > 0)
+ out = format_to(out, FMT_STRING("p{}"),
+ n.exp_ * detail::bigint::bigit_bits);
+ return out;
+ }
+};
+
+FMT_FUNC detail::utf8_to_utf16::utf8_to_utf16(string_view s) {
+ for_each_codepoint(s, [this](uint32_t cp, string_view) {
+ if (cp == invalid_code_point) FMT_THROW(std::runtime_error("invalid utf8"));
+ if (cp <= 0xFFFF) {
+ buffer_.push_back(static_cast<wchar_t>(cp));
+ } else {
+ cp -= 0x10000;
+ buffer_.push_back(static_cast<wchar_t>(0xD800 + (cp >> 10)));
+ buffer_.push_back(static_cast<wchar_t>(0xDC00 + (cp & 0x3FF)));
+ }
+ return true;
+ });
+ buffer_.push_back(0);
+}
+
+FMT_FUNC void format_system_error(detail::buffer<char>& out, int error_code,
+ const char* message) noexcept {
+ FMT_TRY {
+ auto ec = std::error_code(error_code, std::generic_category());
+ write(std::back_inserter(out), std::system_error(ec, message).what());
+ return;
+ }
+ FMT_CATCH(...) {}
+ format_error_code(out, error_code, message);
+}
+
+FMT_FUNC void report_system_error(int error_code,
+ const char* message) noexcept {
+ report_error(format_system_error, error_code, message);
+}
+
+FMT_FUNC std::string vformat(string_view fmt, format_args args) {
+ // Don't optimize the "{}" case to keep the binary size small and because it
+ // can be better optimized in fmt::format anyway.
+ auto buffer = memory_buffer();
+ detail::vformat_to(buffer, fmt, args);
+ return to_string(buffer);
+}
+
+namespace detail {
+#ifndef _WIN32
+FMT_FUNC bool write_console(std::FILE*, string_view) { return false; }
+#else
+using dword = conditional_t<sizeof(long) == 4, unsigned long, unsigned>;
+extern "C" __declspec(dllimport) int __stdcall WriteConsoleW( //
+ void*, const void*, dword, dword*, void*);
+
+FMT_FUNC bool write_console(std::FILE* f, string_view text) {
+ auto fd = _fileno(f);
+ if (!_isatty(fd)) return false;
+ auto u16 = utf8_to_utf16(text);
+ auto written = dword();
+ return WriteConsoleW(reinterpret_cast<void*>(_get_osfhandle(fd)), u16.c_str(),
+ static_cast<uint32_t>(u16.size()), &written, nullptr);
+}
+
+// Print assuming legacy (non-Unicode) encoding.
+FMT_FUNC void vprint_mojibake(std::FILE* f, string_view fmt, format_args args) {
+ auto buffer = memory_buffer();
+ detail::vformat_to(buffer, fmt,
+ basic_format_args<buffer_context<char>>(args));
+ fwrite_fully(buffer.data(), 1, buffer.size(), f);
+}
+#endif
+
+FMT_FUNC void print(std::FILE* f, string_view text) {
+ if (!write_console(f, text)) fwrite_fully(text.data(), 1, text.size(), f);
+}
+} // namespace detail
+
+FMT_FUNC void vprint(std::FILE* f, string_view fmt, format_args args) {
+ auto buffer = memory_buffer();
+ detail::vformat_to(buffer, fmt, args);
+ detail::print(f, {buffer.data(), buffer.size()});
+}
+
+FMT_FUNC void vprint(string_view fmt, format_args args) {
+ vprint(stdout, fmt, args);
+}
+
+namespace detail {
+
+struct singleton {
+ unsigned char upper;
+ unsigned char lower_count;
+};
+
+inline auto is_printable(uint16_t x, const singleton* singletons,
+ size_t singletons_size,
+ const unsigned char* singleton_lowers,
+ const unsigned char* normal, size_t normal_size)
+ -> bool {
+ auto upper = x >> 8;
+ auto lower_start = 0;
+ for (size_t i = 0; i < singletons_size; ++i) {
+ auto s = singletons[i];
+ auto lower_end = lower_start + s.lower_count;
+ if (upper < s.upper) break;
+ if (upper == s.upper) {
+ for (auto j = lower_start; j < lower_end; ++j) {
+ if (singleton_lowers[j] == (x & 0xff)) return false;
+ }
+ }
+ lower_start = lower_end;
+ }
+
+ auto xsigned = static_cast<int>(x);
+ auto current = true;
+ for (size_t i = 0; i < normal_size; ++i) {
+ auto v = static_cast<int>(normal[i]);
+ auto len = (v & 0x80) != 0 ? (v & 0x7f) << 8 | normal[++i] : v;
+ xsigned -= len;
+ if (xsigned < 0) break;
+ current = !current;
+ }
+ return current;
+}
+
+// This code is generated by support/printable.py.
+FMT_FUNC auto is_printable(uint32_t cp) -> bool {
+ static constexpr singleton singletons0[] = {
+ {0x00, 1}, {0x03, 5}, {0x05, 6}, {0x06, 3}, {0x07, 6}, {0x08, 8},
+ {0x09, 17}, {0x0a, 28}, {0x0b, 25}, {0x0c, 20}, {0x0d, 16}, {0x0e, 13},
+ {0x0f, 4}, {0x10, 3}, {0x12, 18}, {0x13, 9}, {0x16, 1}, {0x17, 5},
+ {0x18, 2}, {0x19, 3}, {0x1a, 7}, {0x1c, 2}, {0x1d, 1}, {0x1f, 22},
+ {0x20, 3}, {0x2b, 3}, {0x2c, 2}, {0x2d, 11}, {0x2e, 1}, {0x30, 3},
+ {0x31, 2}, {0x32, 1}, {0xa7, 2}, {0xa9, 2}, {0xaa, 4}, {0xab, 8},
+ {0xfa, 2}, {0xfb, 5}, {0xfd, 4}, {0xfe, 3}, {0xff, 9},
+ };
+ static constexpr unsigned char singletons0_lower[] = {
+ 0xad, 0x78, 0x79, 0x8b, 0x8d, 0xa2, 0x30, 0x57, 0x58, 0x8b, 0x8c, 0x90,
+ 0x1c, 0x1d, 0xdd, 0x0e, 0x0f, 0x4b, 0x4c, 0xfb, 0xfc, 0x2e, 0x2f, 0x3f,
+ 0x5c, 0x5d, 0x5f, 0xb5, 0xe2, 0x84, 0x8d, 0x8e, 0x91, 0x92, 0xa9, 0xb1,
+ 0xba, 0xbb, 0xc5, 0xc6, 0xc9, 0xca, 0xde, 0xe4, 0xe5, 0xff, 0x00, 0x04,
+ 0x11, 0x12, 0x29, 0x31, 0x34, 0x37, 0x3a, 0x3b, 0x3d, 0x49, 0x4a, 0x5d,
+ 0x84, 0x8e, 0x92, 0xa9, 0xb1, 0xb4, 0xba, 0xbb, 0xc6, 0xca, 0xce, 0xcf,
+ 0xe4, 0xe5, 0x00, 0x04, 0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, 0x34, 0x3a,
+ 0x3b, 0x45, 0x46, 0x49, 0x4a, 0x5e, 0x64, 0x65, 0x84, 0x91, 0x9b, 0x9d,
+ 0xc9, 0xce, 0xcf, 0x0d, 0x11, 0x29, 0x45, 0x49, 0x57, 0x64, 0x65, 0x8d,
+ 0x91, 0xa9, 0xb4, 0xba, 0xbb, 0xc5, 0xc9, 0xdf, 0xe4, 0xe5, 0xf0, 0x0d,
+ 0x11, 0x45, 0x49, 0x64, 0x65, 0x80, 0x84, 0xb2, 0xbc, 0xbe, 0xbf, 0xd5,
+ 0xd7, 0xf0, 0xf1, 0x83, 0x85, 0x8b, 0xa4, 0xa6, 0xbe, 0xbf, 0xc5, 0xc7,
+ 0xce, 0xcf, 0xda, 0xdb, 0x48, 0x98, 0xbd, 0xcd, 0xc6, 0xce, 0xcf, 0x49,
+ 0x4e, 0x4f, 0x57, 0x59, 0x5e, 0x5f, 0x89, 0x8e, 0x8f, 0xb1, 0xb6, 0xb7,
+ 0xbf, 0xc1, 0xc6, 0xc7, 0xd7, 0x11, 0x16, 0x17, 0x5b, 0x5c, 0xf6, 0xf7,
+ 0xfe, 0xff, 0x80, 0x0d, 0x6d, 0x71, 0xde, 0xdf, 0x0e, 0x0f, 0x1f, 0x6e,
+ 0x6f, 0x1c, 0x1d, 0x5f, 0x7d, 0x7e, 0xae, 0xaf, 0xbb, 0xbc, 0xfa, 0x16,
+ 0x17, 0x1e, 0x1f, 0x46, 0x47, 0x4e, 0x4f, 0x58, 0x5a, 0x5c, 0x5e, 0x7e,
+ 0x7f, 0xb5, 0xc5, 0xd4, 0xd5, 0xdc, 0xf0, 0xf1, 0xf5, 0x72, 0x73, 0x8f,
+ 0x74, 0x75, 0x96, 0x2f, 0x5f, 0x26, 0x2e, 0x2f, 0xa7, 0xaf, 0xb7, 0xbf,
+ 0xc7, 0xcf, 0xd7, 0xdf, 0x9a, 0x40, 0x97, 0x98, 0x30, 0x8f, 0x1f, 0xc0,
+ 0xc1, 0xce, 0xff, 0x4e, 0x4f, 0x5a, 0x5b, 0x07, 0x08, 0x0f, 0x10, 0x27,
+ 0x2f, 0xee, 0xef, 0x6e, 0x6f, 0x37, 0x3d, 0x3f, 0x42, 0x45, 0x90, 0x91,
+ 0xfe, 0xff, 0x53, 0x67, 0x75, 0xc8, 0xc9, 0xd0, 0xd1, 0xd8, 0xd9, 0xe7,
+ 0xfe, 0xff,
+ };
+ static constexpr singleton singletons1[] = {
+ {0x00, 6}, {0x01, 1}, {0x03, 1}, {0x04, 2}, {0x08, 8}, {0x09, 2},
+ {0x0a, 5}, {0x0b, 2}, {0x0e, 4}, {0x10, 1}, {0x11, 2}, {0x12, 5},
+ {0x13, 17}, {0x14, 1}, {0x15, 2}, {0x17, 2}, {0x19, 13}, {0x1c, 5},
+ {0x1d, 8}, {0x24, 1}, {0x6a, 3}, {0x6b, 2}, {0xbc, 2}, {0xd1, 2},
+ {0xd4, 12}, {0xd5, 9}, {0xd6, 2}, {0xd7, 2}, {0xda, 1}, {0xe0, 5},
+ {0xe1, 2}, {0xe8, 2}, {0xee, 32}, {0xf0, 4}, {0xf8, 2}, {0xf9, 2},
+ {0xfa, 2}, {0xfb, 1},
+ };
+ static constexpr unsigned char singletons1_lower[] = {
+ 0x0c, 0x27, 0x3b, 0x3e, 0x4e, 0x4f, 0x8f, 0x9e, 0x9e, 0x9f, 0x06, 0x07,
+ 0x09, 0x36, 0x3d, 0x3e, 0x56, 0xf3, 0xd0, 0xd1, 0x04, 0x14, 0x18, 0x36,
+ 0x37, 0x56, 0x57, 0x7f, 0xaa, 0xae, 0xaf, 0xbd, 0x35, 0xe0, 0x12, 0x87,
+ 0x89, 0x8e, 0x9e, 0x04, 0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, 0x34, 0x3a,
+ 0x45, 0x46, 0x49, 0x4a, 0x4e, 0x4f, 0x64, 0x65, 0x5c, 0xb6, 0xb7, 0x1b,
+ 0x1c, 0x07, 0x08, 0x0a, 0x0b, 0x14, 0x17, 0x36, 0x39, 0x3a, 0xa8, 0xa9,
+ 0xd8, 0xd9, 0x09, 0x37, 0x90, 0x91, 0xa8, 0x07, 0x0a, 0x3b, 0x3e, 0x66,
+ 0x69, 0x8f, 0x92, 0x6f, 0x5f, 0xee, 0xef, 0x5a, 0x62, 0x9a, 0x9b, 0x27,
+ 0x28, 0x55, 0x9d, 0xa0, 0xa1, 0xa3, 0xa4, 0xa7, 0xa8, 0xad, 0xba, 0xbc,
+ 0xc4, 0x06, 0x0b, 0x0c, 0x15, 0x1d, 0x3a, 0x3f, 0x45, 0x51, 0xa6, 0xa7,
+ 0xcc, 0xcd, 0xa0, 0x07, 0x19, 0x1a, 0x22, 0x25, 0x3e, 0x3f, 0xc5, 0xc6,
+ 0x04, 0x20, 0x23, 0x25, 0x26, 0x28, 0x33, 0x38, 0x3a, 0x48, 0x4a, 0x4c,
+ 0x50, 0x53, 0x55, 0x56, 0x58, 0x5a, 0x5c, 0x5e, 0x60, 0x63, 0x65, 0x66,
+ 0x6b, 0x73, 0x78, 0x7d, 0x7f, 0x8a, 0xa4, 0xaa, 0xaf, 0xb0, 0xc0, 0xd0,
+ 0xae, 0xaf, 0x79, 0xcc, 0x6e, 0x6f, 0x93,
+ };
+ static constexpr unsigned char normal0[] = {
+ 0x00, 0x20, 0x5f, 0x22, 0x82, 0xdf, 0x04, 0x82, 0x44, 0x08, 0x1b, 0x04,
+ 0x06, 0x11, 0x81, 0xac, 0x0e, 0x80, 0xab, 0x35, 0x28, 0x0b, 0x80, 0xe0,
+ 0x03, 0x19, 0x08, 0x01, 0x04, 0x2f, 0x04, 0x34, 0x04, 0x07, 0x03, 0x01,
+ 0x07, 0x06, 0x07, 0x11, 0x0a, 0x50, 0x0f, 0x12, 0x07, 0x55, 0x07, 0x03,
+ 0x04, 0x1c, 0x0a, 0x09, 0x03, 0x08, 0x03, 0x07, 0x03, 0x02, 0x03, 0x03,
+ 0x03, 0x0c, 0x04, 0x05, 0x03, 0x0b, 0x06, 0x01, 0x0e, 0x15, 0x05, 0x3a,
+ 0x03, 0x11, 0x07, 0x06, 0x05, 0x10, 0x07, 0x57, 0x07, 0x02, 0x07, 0x15,
+ 0x0d, 0x50, 0x04, 0x43, 0x03, 0x2d, 0x03, 0x01, 0x04, 0x11, 0x06, 0x0f,
+ 0x0c, 0x3a, 0x04, 0x1d, 0x25, 0x5f, 0x20, 0x6d, 0x04, 0x6a, 0x25, 0x80,
+ 0xc8, 0x05, 0x82, 0xb0, 0x03, 0x1a, 0x06, 0x82, 0xfd, 0x03, 0x59, 0x07,
+ 0x15, 0x0b, 0x17, 0x09, 0x14, 0x0c, 0x14, 0x0c, 0x6a, 0x06, 0x0a, 0x06,
+ 0x1a, 0x06, 0x59, 0x07, 0x2b, 0x05, 0x46, 0x0a, 0x2c, 0x04, 0x0c, 0x04,
+ 0x01, 0x03, 0x31, 0x0b, 0x2c, 0x04, 0x1a, 0x06, 0x0b, 0x03, 0x80, 0xac,
+ 0x06, 0x0a, 0x06, 0x21, 0x3f, 0x4c, 0x04, 0x2d, 0x03, 0x74, 0x08, 0x3c,
+ 0x03, 0x0f, 0x03, 0x3c, 0x07, 0x38, 0x08, 0x2b, 0x05, 0x82, 0xff, 0x11,
+ 0x18, 0x08, 0x2f, 0x11, 0x2d, 0x03, 0x20, 0x10, 0x21, 0x0f, 0x80, 0x8c,
+ 0x04, 0x82, 0x97, 0x19, 0x0b, 0x15, 0x88, 0x94, 0x05, 0x2f, 0x05, 0x3b,
+ 0x07, 0x02, 0x0e, 0x18, 0x09, 0x80, 0xb3, 0x2d, 0x74, 0x0c, 0x80, 0xd6,
+ 0x1a, 0x0c, 0x05, 0x80, 0xff, 0x05, 0x80, 0xdf, 0x0c, 0xee, 0x0d, 0x03,
+ 0x84, 0x8d, 0x03, 0x37, 0x09, 0x81, 0x5c, 0x14, 0x80, 0xb8, 0x08, 0x80,
+ 0xcb, 0x2a, 0x38, 0x03, 0x0a, 0x06, 0x38, 0x08, 0x46, 0x08, 0x0c, 0x06,
+ 0x74, 0x0b, 0x1e, 0x03, 0x5a, 0x04, 0x59, 0x09, 0x80, 0x83, 0x18, 0x1c,
+ 0x0a, 0x16, 0x09, 0x4c, 0x04, 0x80, 0x8a, 0x06, 0xab, 0xa4, 0x0c, 0x17,
+ 0x04, 0x31, 0xa1, 0x04, 0x81, 0xda, 0x26, 0x07, 0x0c, 0x05, 0x05, 0x80,
+ 0xa5, 0x11, 0x81, 0x6d, 0x10, 0x78, 0x28, 0x2a, 0x06, 0x4c, 0x04, 0x80,
+ 0x8d, 0x04, 0x80, 0xbe, 0x03, 0x1b, 0x03, 0x0f, 0x0d,
+ };
+ static constexpr unsigned char normal1[] = {
+ 0x5e, 0x22, 0x7b, 0x05, 0x03, 0x04, 0x2d, 0x03, 0x66, 0x03, 0x01, 0x2f,
+ 0x2e, 0x80, 0x82, 0x1d, 0x03, 0x31, 0x0f, 0x1c, 0x04, 0x24, 0x09, 0x1e,
+ 0x05, 0x2b, 0x05, 0x44, 0x04, 0x0e, 0x2a, 0x80, 0xaa, 0x06, 0x24, 0x04,
+ 0x24, 0x04, 0x28, 0x08, 0x34, 0x0b, 0x01, 0x80, 0x90, 0x81, 0x37, 0x09,
+ 0x16, 0x0a, 0x08, 0x80, 0x98, 0x39, 0x03, 0x63, 0x08, 0x09, 0x30, 0x16,
+ 0x05, 0x21, 0x03, 0x1b, 0x05, 0x01, 0x40, 0x38, 0x04, 0x4b, 0x05, 0x2f,
+ 0x04, 0x0a, 0x07, 0x09, 0x07, 0x40, 0x20, 0x27, 0x04, 0x0c, 0x09, 0x36,
+ 0x03, 0x3a, 0x05, 0x1a, 0x07, 0x04, 0x0c, 0x07, 0x50, 0x49, 0x37, 0x33,
+ 0x0d, 0x33, 0x07, 0x2e, 0x08, 0x0a, 0x81, 0x26, 0x52, 0x4e, 0x28, 0x08,
+ 0x2a, 0x56, 0x1c, 0x14, 0x17, 0x09, 0x4e, 0x04, 0x1e, 0x0f, 0x43, 0x0e,
+ 0x19, 0x07, 0x0a, 0x06, 0x48, 0x08, 0x27, 0x09, 0x75, 0x0b, 0x3f, 0x41,
+ 0x2a, 0x06, 0x3b, 0x05, 0x0a, 0x06, 0x51, 0x06, 0x01, 0x05, 0x10, 0x03,
+ 0x05, 0x80, 0x8b, 0x62, 0x1e, 0x48, 0x08, 0x0a, 0x80, 0xa6, 0x5e, 0x22,
+ 0x45, 0x0b, 0x0a, 0x06, 0x0d, 0x13, 0x39, 0x07, 0x0a, 0x36, 0x2c, 0x04,
+ 0x10, 0x80, 0xc0, 0x3c, 0x64, 0x53, 0x0c, 0x48, 0x09, 0x0a, 0x46, 0x45,
+ 0x1b, 0x48, 0x08, 0x53, 0x1d, 0x39, 0x81, 0x07, 0x46, 0x0a, 0x1d, 0x03,
+ 0x47, 0x49, 0x37, 0x03, 0x0e, 0x08, 0x0a, 0x06, 0x39, 0x07, 0x0a, 0x81,
+ 0x36, 0x19, 0x80, 0xb7, 0x01, 0x0f, 0x32, 0x0d, 0x83, 0x9b, 0x66, 0x75,
+ 0x0b, 0x80, 0xc4, 0x8a, 0xbc, 0x84, 0x2f, 0x8f, 0xd1, 0x82, 0x47, 0xa1,
+ 0xb9, 0x82, 0x39, 0x07, 0x2a, 0x04, 0x02, 0x60, 0x26, 0x0a, 0x46, 0x0a,
+ 0x28, 0x05, 0x13, 0x82, 0xb0, 0x5b, 0x65, 0x4b, 0x04, 0x39, 0x07, 0x11,
+ 0x40, 0x05, 0x0b, 0x02, 0x0e, 0x97, 0xf8, 0x08, 0x84, 0xd6, 0x2a, 0x09,
+ 0xa2, 0xf7, 0x81, 0x1f, 0x31, 0x03, 0x11, 0x04, 0x08, 0x81, 0x8c, 0x89,
+ 0x04, 0x6b, 0x05, 0x0d, 0x03, 0x09, 0x07, 0x10, 0x93, 0x60, 0x80, 0xf6,
+ 0x0a, 0x73, 0x08, 0x6e, 0x17, 0x46, 0x80, 0x9a, 0x14, 0x0c, 0x57, 0x09,
+ 0x19, 0x80, 0x87, 0x81, 0x47, 0x03, 0x85, 0x42, 0x0f, 0x15, 0x85, 0x50,
+ 0x2b, 0x80, 0xd5, 0x2d, 0x03, 0x1a, 0x04, 0x02, 0x81, 0x70, 0x3a, 0x05,
+ 0x01, 0x85, 0x00, 0x80, 0xd7, 0x29, 0x4c, 0x04, 0x0a, 0x04, 0x02, 0x83,
+ 0x11, 0x44, 0x4c, 0x3d, 0x80, 0xc2, 0x3c, 0x06, 0x01, 0x04, 0x55, 0x05,
+ 0x1b, 0x34, 0x02, 0x81, 0x0e, 0x2c, 0x04, 0x64, 0x0c, 0x56, 0x0a, 0x80,
+ 0xae, 0x38, 0x1d, 0x0d, 0x2c, 0x04, 0x09, 0x07, 0x02, 0x0e, 0x06, 0x80,
+ 0x9a, 0x83, 0xd8, 0x08, 0x0d, 0x03, 0x0d, 0x03, 0x74, 0x0c, 0x59, 0x07,
+ 0x0c, 0x14, 0x0c, 0x04, 0x38, 0x08, 0x0a, 0x06, 0x28, 0x08, 0x22, 0x4e,
+ 0x81, 0x54, 0x0c, 0x15, 0x03, 0x03, 0x05, 0x07, 0x09, 0x19, 0x07, 0x07,
+ 0x09, 0x03, 0x0d, 0x07, 0x29, 0x80, 0xcb, 0x25, 0x0a, 0x84, 0x06,
+ };
+ auto lower = static_cast<uint16_t>(cp);
+ if (cp < 0x10000) {
+ return is_printable(lower, singletons0,
+ sizeof(singletons0) / sizeof(*singletons0),
+ singletons0_lower, normal0, sizeof(normal0));
+ }
+ if (cp < 0x20000) {
+ return is_printable(lower, singletons1,
+ sizeof(singletons1) / sizeof(*singletons1),
+ singletons1_lower, normal1, sizeof(normal1));
+ }
+ if (0x2a6de <= cp && cp < 0x2a700) return false;
+ if (0x2b735 <= cp && cp < 0x2b740) return false;
+ if (0x2b81e <= cp && cp < 0x2b820) return false;
+ if (0x2cea2 <= cp && cp < 0x2ceb0) return false;
+ if (0x2ebe1 <= cp && cp < 0x2f800) return false;
+ if (0x2fa1e <= cp && cp < 0x30000) return false;
+ if (0x3134b <= cp && cp < 0xe0100) return false;
+ if (0xe01f0 <= cp && cp < 0x110000) return false;
+ return cp < 0x110000;
+}
+
+} // namespace detail
+
+FMT_END_NAMESPACE
+
+#endif // FMT_FORMAT_INL_H_
diff --git a/contrib/fmt/include/fmt/format.h b/contrib/fmt/include/fmt/format.h
new file mode 100644
index 0000000..ed8b29e
--- /dev/null
+++ b/contrib/fmt/include/fmt/format.h
@@ -0,0 +1,4735 @@
+/*
+ Formatting library for C++
+
+ Copyright (c) 2012 - present, Victor Zverovich
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ --- Optional exception to the license ---
+
+ As an exception, if, as a result of your compiling your source code, portions
+ of this Software are embedded into a machine-executable object form of such
+ source code, you may redistribute such embedded portions in such object form
+ without including the above copyright and permission notices.
+ */
+
+#ifndef FMT_FORMAT_H_
+#define FMT_FORMAT_H_
+
+#include <cmath> // std::signbit
+#include <cstdint> // uint32_t
+#include <cstring> // std::memcpy
+#include <initializer_list> // std::initializer_list
+#include <limits> // std::numeric_limits
+#include <memory> // std::uninitialized_copy
+#include <stdexcept> // std::runtime_error
+#include <system_error> // std::system_error
+
+#ifdef __cpp_lib_bit_cast
+# include <bit> // std::bitcast
+#endif
+
+#include "core.h"
+
+#ifndef FMT_BEGIN_DETAIL_NAMESPACE
+# define FMT_BEGIN_DETAIL_NAMESPACE namespace detail {
+# define FMT_END_DETAIL_NAMESPACE }
+#endif
+
+#if FMT_HAS_CPP17_ATTRIBUTE(fallthrough)
+# define FMT_FALLTHROUGH [[fallthrough]]
+#elif defined(__clang__)
+# define FMT_FALLTHROUGH [[clang::fallthrough]]
+#elif FMT_GCC_VERSION >= 700 && \
+ (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 520)
+# define FMT_FALLTHROUGH [[gnu::fallthrough]]
+#else
+# define FMT_FALLTHROUGH
+#endif
+
+#ifndef FMT_DEPRECATED
+# if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VERSION >= 1900
+# define FMT_DEPRECATED [[deprecated]]
+# else
+# if (defined(__GNUC__) && !defined(__LCC__)) || defined(__clang__)
+# define FMT_DEPRECATED __attribute__((deprecated))
+# elif FMT_MSC_VERSION
+# define FMT_DEPRECATED __declspec(deprecated)
+# else
+# define FMT_DEPRECATED /* deprecated */
+# endif
+# endif
+#endif
+
+#if FMT_GCC_VERSION
+# define FMT_GCC_VISIBILITY_HIDDEN __attribute__((visibility("hidden")))
+#else
+# define FMT_GCC_VISIBILITY_HIDDEN
+#endif
+
+#ifdef __NVCC__
+# define FMT_CUDA_VERSION (__CUDACC_VER_MAJOR__ * 100 + __CUDACC_VER_MINOR__)
+#else
+# define FMT_CUDA_VERSION 0
+#endif
+
+#ifdef __has_builtin
+# define FMT_HAS_BUILTIN(x) __has_builtin(x)
+#else
+# define FMT_HAS_BUILTIN(x) 0
+#endif
+
+#if FMT_GCC_VERSION || FMT_CLANG_VERSION
+# define FMT_NOINLINE __attribute__((noinline))
+#else
+# define FMT_NOINLINE
+#endif
+
+#ifndef FMT_THROW
+# if FMT_EXCEPTIONS
+# if FMT_MSC_VERSION || defined(__NVCC__)
+FMT_BEGIN_NAMESPACE
+namespace detail {
+template <typename Exception> inline void do_throw(const Exception& x) {
+ // Silence unreachable code warnings in MSVC and NVCC because these
+ // are nearly impossible to fix in a generic code.
+ volatile bool b = true;
+ if (b) throw x;
+}
+} // namespace detail
+FMT_END_NAMESPACE
+# define FMT_THROW(x) detail::do_throw(x)
+# else
+# define FMT_THROW(x) throw x
+# endif
+# else
+# define FMT_THROW(x) \
+ do { \
+ FMT_ASSERT(false, (x).what()); \
+ } while (false)
+# endif
+#endif
+
+#if FMT_EXCEPTIONS
+# define FMT_TRY try
+# define FMT_CATCH(x) catch (x)
+#else
+# define FMT_TRY if (true)
+# define FMT_CATCH(x) if (false)
+#endif
+
+#ifndef FMT_MAYBE_UNUSED
+# if FMT_HAS_CPP17_ATTRIBUTE(maybe_unused)
+# define FMT_MAYBE_UNUSED [[maybe_unused]]
+# else
+# define FMT_MAYBE_UNUSED
+# endif
+#endif
+
+#ifndef FMT_USE_USER_DEFINED_LITERALS
+// EDG based compilers (Intel, NVIDIA, Elbrus, etc), GCC and MSVC support UDLs.
+# if (FMT_HAS_FEATURE(cxx_user_literals) || FMT_GCC_VERSION >= 407 || \
+ FMT_MSC_VERSION >= 1900) && \
+ (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= /* UDL feature */ 480)
+# define FMT_USE_USER_DEFINED_LITERALS 1
+# else
+# define FMT_USE_USER_DEFINED_LITERALS 0
+# endif
+#endif
+
+// Defining FMT_REDUCE_INT_INSTANTIATIONS to 1, will reduce the number of
+// integer formatter template instantiations to just one by only using the
+// largest integer type. This results in a reduction in binary size but will
+// cause a decrease in integer formatting performance.
+#if !defined(FMT_REDUCE_INT_INSTANTIATIONS)
+# define FMT_REDUCE_INT_INSTANTIATIONS 0
+#endif
+
+// __builtin_clz is broken in clang with Microsoft CodeGen:
+// https://github.com/fmtlib/fmt/issues/519.
+#if !FMT_MSC_VERSION
+# if FMT_HAS_BUILTIN(__builtin_clz) || FMT_GCC_VERSION || FMT_ICC_VERSION
+# define FMT_BUILTIN_CLZ(n) __builtin_clz(n)
+# endif
+# if FMT_HAS_BUILTIN(__builtin_clzll) || FMT_GCC_VERSION || FMT_ICC_VERSION
+# define FMT_BUILTIN_CLZLL(n) __builtin_clzll(n)
+# endif
+#endif
+
+// __builtin_ctz is broken in Intel Compiler Classic on Windows:
+// https://github.com/fmtlib/fmt/issues/2510.
+#ifndef __ICL
+# if FMT_HAS_BUILTIN(__builtin_ctz) || FMT_GCC_VERSION || FMT_ICC_VERSION || \
+ defined(__NVCOMPILER)
+# define FMT_BUILTIN_CTZ(n) __builtin_ctz(n)
+# endif
+# if FMT_HAS_BUILTIN(__builtin_ctzll) || FMT_GCC_VERSION || \
+ FMT_ICC_VERSION || defined(__NVCOMPILER)
+# define FMT_BUILTIN_CTZLL(n) __builtin_ctzll(n)
+# endif
+#endif
+
+#if FMT_MSC_VERSION
+# include <intrin.h> // _BitScanReverse[64], _BitScanForward[64], _umul128
+#endif
+
+// Some compilers masquerade as both MSVC and GCC-likes or otherwise support
+// __builtin_clz and __builtin_clzll, so only define FMT_BUILTIN_CLZ using the
+// MSVC intrinsics if the clz and clzll builtins are not available.
+#if FMT_MSC_VERSION && !defined(FMT_BUILTIN_CLZLL) && \
+ !defined(FMT_BUILTIN_CTZLL)
+FMT_BEGIN_NAMESPACE
+namespace detail {
+// Avoid Clang with Microsoft CodeGen's -Wunknown-pragmas warning.
+# if !defined(__clang__)
+# pragma intrinsic(_BitScanForward)
+# pragma intrinsic(_BitScanReverse)
+# if defined(_WIN64)
+# pragma intrinsic(_BitScanForward64)
+# pragma intrinsic(_BitScanReverse64)
+# endif
+# endif
+
+inline auto clz(uint32_t x) -> int {
+ unsigned long r = 0;
+ _BitScanReverse(&r, x);
+ FMT_ASSERT(x != 0, "");
+ // Static analysis complains about using uninitialized data
+ // "r", but the only way that can happen is if "x" is 0,
+ // which the callers guarantee to not happen.
+ FMT_MSC_WARNING(suppress : 6102)
+ return 31 ^ static_cast<int>(r);
+}
+# define FMT_BUILTIN_CLZ(n) detail::clz(n)
+
+inline auto clzll(uint64_t x) -> int {
+ unsigned long r = 0;
+# ifdef _WIN64
+ _BitScanReverse64(&r, x);
+# else
+ // Scan the high 32 bits.
+ if (_BitScanReverse(&r, static_cast<uint32_t>(x >> 32)))
+ return 63 ^ static_cast<int>(r + 32);
+ // Scan the low 32 bits.
+ _BitScanReverse(&r, static_cast<uint32_t>(x));
+# endif
+ FMT_ASSERT(x != 0, "");
+ FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning.
+ return 63 ^ static_cast<int>(r);
+}
+# define FMT_BUILTIN_CLZLL(n) detail::clzll(n)
+
+inline auto ctz(uint32_t x) -> int {
+ unsigned long r = 0;
+ _BitScanForward(&r, x);
+ FMT_ASSERT(x != 0, "");
+ FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning.
+ return static_cast<int>(r);
+}
+# define FMT_BUILTIN_CTZ(n) detail::ctz(n)
+
+inline auto ctzll(uint64_t x) -> int {
+ unsigned long r = 0;
+ FMT_ASSERT(x != 0, "");
+ FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning.
+# ifdef _WIN64
+ _BitScanForward64(&r, x);
+# else
+ // Scan the low 32 bits.
+ if (_BitScanForward(&r, static_cast<uint32_t>(x))) return static_cast<int>(r);
+ // Scan the high 32 bits.
+ _BitScanForward(&r, static_cast<uint32_t>(x >> 32));
+ r += 32;
+# endif
+ return static_cast<int>(r);
+}
+# define FMT_BUILTIN_CTZLL(n) detail::ctzll(n)
+} // namespace detail
+FMT_END_NAMESPACE
+#endif
+
+FMT_BEGIN_NAMESPACE
+
+template <typename...> struct disjunction : std::false_type {};
+template <typename P> struct disjunction<P> : P {};
+template <typename P1, typename... Pn>
+struct disjunction<P1, Pn...>
+ : conditional_t<bool(P1::value), P1, disjunction<Pn...>> {};
+
+template <typename...> struct conjunction : std::true_type {};
+template <typename P> struct conjunction<P> : P {};
+template <typename P1, typename... Pn>
+struct conjunction<P1, Pn...>
+ : conditional_t<bool(P1::value), conjunction<Pn...>, P1> {};
+
+namespace detail {
+
+FMT_CONSTEXPR inline void abort_fuzzing_if(bool condition) {
+ ignore_unused(condition);
+#ifdef FMT_FUZZ
+ if (condition) throw std::runtime_error("fuzzing limit reached");
+#endif
+}
+
+template <typename CharT, CharT... C> struct string_literal {
+ static constexpr CharT value[sizeof...(C)] = {C...};
+ constexpr operator basic_string_view<CharT>() const {
+ return {value, sizeof...(C)};
+ }
+};
+
+#if FMT_CPLUSPLUS < 201703L
+template <typename CharT, CharT... C>
+constexpr CharT string_literal<CharT, C...>::value[sizeof...(C)];
+#endif
+
+template <typename Streambuf> class formatbuf : public Streambuf {
+ private:
+ using char_type = typename Streambuf::char_type;
+ using streamsize = decltype(std::declval<Streambuf>().sputn(nullptr, 0));
+ using int_type = typename Streambuf::int_type;
+ using traits_type = typename Streambuf::traits_type;
+
+ buffer<char_type>& buffer_;
+
+ public:
+ explicit formatbuf(buffer<char_type>& buf) : buffer_(buf) {}
+
+ protected:
+ // The put area is always empty. This makes the implementation simpler and has
+ // the advantage that the streambuf and the buffer are always in sync and
+ // sputc never writes into uninitialized memory. A disadvantage is that each
+ // call to sputc always results in a (virtual) call to overflow. There is no
+ // disadvantage here for sputn since this always results in a call to xsputn.
+
+ auto overflow(int_type ch) -> int_type override {
+ if (!traits_type::eq_int_type(ch, traits_type::eof()))
+ buffer_.push_back(static_cast<char_type>(ch));
+ return ch;
+ }
+
+ auto xsputn(const char_type* s, streamsize count) -> streamsize override {
+ buffer_.append(s, s + count);
+ return count;
+ }
+};
+
+// Implementation of std::bit_cast for pre-C++20.
+template <typename To, typename From, FMT_ENABLE_IF(sizeof(To) == sizeof(From))>
+FMT_CONSTEXPR20 auto bit_cast(const From& from) -> To {
+#ifdef __cpp_lib_bit_cast
+ if (is_constant_evaluated()) return std::bit_cast<To>(from);
+#endif
+ auto to = To();
+ // The cast suppresses a bogus -Wclass-memaccess on GCC.
+ std::memcpy(static_cast<void*>(&to), &from, sizeof(to));
+ return to;
+}
+
+inline auto is_big_endian() -> bool {
+#ifdef _WIN32
+ return false;
+#elif defined(__BIG_ENDIAN__)
+ return true;
+#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__)
+ return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__;
+#else
+ struct bytes {
+ char data[sizeof(int)];
+ };
+ return bit_cast<bytes>(1).data[0] == 0;
+#endif
+}
+
+class uint128_fallback {
+ private:
+ uint64_t lo_, hi_;
+
+ friend uint128_fallback umul128(uint64_t x, uint64_t y) noexcept;
+
+ public:
+ constexpr uint128_fallback(uint64_t hi, uint64_t lo) : lo_(lo), hi_(hi) {}
+ constexpr uint128_fallback(uint64_t value = 0) : lo_(value), hi_(0) {}
+
+ constexpr uint64_t high() const noexcept { return hi_; }
+ constexpr uint64_t low() const noexcept { return lo_; }
+
+ template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+ constexpr explicit operator T() const {
+ return static_cast<T>(lo_);
+ }
+
+ friend constexpr auto operator==(const uint128_fallback& lhs,
+ const uint128_fallback& rhs) -> bool {
+ return lhs.hi_ == rhs.hi_ && lhs.lo_ == rhs.lo_;
+ }
+ friend constexpr auto operator!=(const uint128_fallback& lhs,
+ const uint128_fallback& rhs) -> bool {
+ return !(lhs == rhs);
+ }
+ friend constexpr auto operator>(const uint128_fallback& lhs,
+ const uint128_fallback& rhs) -> bool {
+ return lhs.hi_ != rhs.hi_ ? lhs.hi_ > rhs.hi_ : lhs.lo_ > rhs.lo_;
+ }
+ friend constexpr auto operator|(const uint128_fallback& lhs,
+ const uint128_fallback& rhs)
+ -> uint128_fallback {
+ return {lhs.hi_ | rhs.hi_, lhs.lo_ | rhs.lo_};
+ }
+ friend constexpr auto operator&(const uint128_fallback& lhs,
+ const uint128_fallback& rhs)
+ -> uint128_fallback {
+ return {lhs.hi_ & rhs.hi_, lhs.lo_ & rhs.lo_};
+ }
+ friend constexpr auto operator~(const uint128_fallback& n)
+ -> uint128_fallback {
+ return {~n.hi_, ~n.lo_};
+ }
+ friend auto operator+(const uint128_fallback& lhs,
+ const uint128_fallback& rhs) -> uint128_fallback {
+ auto result = uint128_fallback(lhs);
+ result += rhs;
+ return result;
+ }
+ friend auto operator*(const uint128_fallback& lhs, uint32_t rhs)
+ -> uint128_fallback {
+ FMT_ASSERT(lhs.hi_ == 0, "");
+ uint64_t hi = (lhs.lo_ >> 32) * rhs;
+ uint64_t lo = (lhs.lo_ & ~uint32_t()) * rhs;
+ uint64_t new_lo = (hi << 32) + lo;
+ return {(hi >> 32) + (new_lo < lo ? 1 : 0), new_lo};
+ }
+ friend auto operator-(const uint128_fallback& lhs, uint64_t rhs)
+ -> uint128_fallback {
+ return {lhs.hi_ - (lhs.lo_ < rhs ? 1 : 0), lhs.lo_ - rhs};
+ }
+ FMT_CONSTEXPR auto operator>>(int shift) const -> uint128_fallback {
+ if (shift == 64) return {0, hi_};
+ if (shift > 64) return uint128_fallback(0, hi_) >> (shift - 64);
+ return {hi_ >> shift, (hi_ << (64 - shift)) | (lo_ >> shift)};
+ }
+ FMT_CONSTEXPR auto operator<<(int shift) const -> uint128_fallback {
+ if (shift == 64) return {lo_, 0};
+ if (shift > 64) return uint128_fallback(lo_, 0) << (shift - 64);
+ return {hi_ << shift | (lo_ >> (64 - shift)), (lo_ << shift)};
+ }
+ FMT_CONSTEXPR auto operator>>=(int shift) -> uint128_fallback& {
+ return *this = *this >> shift;
+ }
+ FMT_CONSTEXPR void operator+=(uint128_fallback n) {
+ uint64_t new_lo = lo_ + n.lo_;
+ uint64_t new_hi = hi_ + n.hi_ + (new_lo < lo_ ? 1 : 0);
+ FMT_ASSERT(new_hi >= hi_, "");
+ lo_ = new_lo;
+ hi_ = new_hi;
+ }
+ FMT_CONSTEXPR void operator&=(uint128_fallback n) {
+ lo_ &= n.lo_;
+ hi_ &= n.hi_;
+ }
+
+ FMT_CONSTEXPR20 uint128_fallback& operator+=(uint64_t n) noexcept {
+ if (is_constant_evaluated()) {
+ lo_ += n;
+ hi_ += (lo_ < n ? 1 : 0);
+ return *this;
+ }
+#if FMT_HAS_BUILTIN(__builtin_addcll) && !defined(__ibmxl__)
+ unsigned long long carry;
+ lo_ = __builtin_addcll(lo_, n, 0, &carry);
+ hi_ += carry;
+#elif FMT_HAS_BUILTIN(__builtin_ia32_addcarryx_u64) && !defined(__ibmxl__)
+ unsigned long long result;
+ auto carry = __builtin_ia32_addcarryx_u64(0, lo_, n, &result);
+ lo_ = result;
+ hi_ += carry;
+#elif defined(_MSC_VER) && defined(_M_X64)
+ auto carry = _addcarry_u64(0, lo_, n, &lo_);
+ _addcarry_u64(carry, hi_, 0, &hi_);
+#else
+ lo_ += n;
+ hi_ += (lo_ < n ? 1 : 0);
+#endif
+ return *this;
+ }
+};
+
+using uint128_t = conditional_t<FMT_USE_INT128, uint128_opt, uint128_fallback>;
+
+#ifdef UINTPTR_MAX
+using uintptr_t = ::uintptr_t;
+#else
+using uintptr_t = uint128_t;
+#endif
+
+// Returns the largest possible value for type T. Same as
+// std::numeric_limits<T>::max() but shorter and not affected by the max macro.
+template <typename T> constexpr auto max_value() -> T {
+ return (std::numeric_limits<T>::max)();
+}
+template <typename T> constexpr auto num_bits() -> int {
+ return std::numeric_limits<T>::digits;
+}
+// std::numeric_limits<T>::digits may return 0 for 128-bit ints.
+template <> constexpr auto num_bits<int128_opt>() -> int { return 128; }
+template <> constexpr auto num_bits<uint128_t>() -> int { return 128; }
+
+// A heterogeneous bit_cast used for converting 96-bit long double to uint128_t
+// and 128-bit pointers to uint128_fallback.
+template <typename To, typename From, FMT_ENABLE_IF(sizeof(To) > sizeof(From))>
+inline auto bit_cast(const From& from) -> To {
+ constexpr auto size = static_cast<int>(sizeof(From) / sizeof(unsigned));
+ struct data_t {
+ unsigned value[static_cast<unsigned>(size)];
+ } data = bit_cast<data_t>(from);
+ auto result = To();
+ if (const_check(is_big_endian())) {
+ for (int i = 0; i < size; ++i)
+ result = (result << num_bits<unsigned>()) | data.value[i];
+ } else {
+ for (int i = size - 1; i >= 0; --i)
+ result = (result << num_bits<unsigned>()) | data.value[i];
+ }
+ return result;
+}
+
+template <typename UInt>
+FMT_CONSTEXPR20 inline auto countl_zero_fallback(UInt n) -> int {
+ int lz = 0;
+ constexpr UInt msb_mask = static_cast<UInt>(1) << (num_bits<UInt>() - 1);
+ for (; (n & msb_mask) == 0; n <<= 1) lz++;
+ return lz;
+}
+
+FMT_CONSTEXPR20 inline auto countl_zero(uint32_t n) -> int {
+#ifdef FMT_BUILTIN_CLZ
+ if (!is_constant_evaluated()) return FMT_BUILTIN_CLZ(n);
+#endif
+ return countl_zero_fallback(n);
+}
+
+FMT_CONSTEXPR20 inline auto countl_zero(uint64_t n) -> int {
+#ifdef FMT_BUILTIN_CLZLL
+ if (!is_constant_evaluated()) return FMT_BUILTIN_CLZLL(n);
+#endif
+ return countl_zero_fallback(n);
+}
+
+FMT_INLINE void assume(bool condition) {
+ (void)condition;
+#if FMT_HAS_BUILTIN(__builtin_assume) && !FMT_ICC_VERSION
+ __builtin_assume(condition);
+#endif
+}
+
+// An approximation of iterator_t for pre-C++20 systems.
+template <typename T>
+using iterator_t = decltype(std::begin(std::declval<T&>()));
+template <typename T> using sentinel_t = decltype(std::end(std::declval<T&>()));
+
+// A workaround for std::string not having mutable data() until C++17.
+template <typename Char>
+inline auto get_data(std::basic_string<Char>& s) -> Char* {
+ return &s[0];
+}
+template <typename Container>
+inline auto get_data(Container& c) -> typename Container::value_type* {
+ return c.data();
+}
+
+#if defined(_SECURE_SCL) && _SECURE_SCL
+// Make a checked iterator to avoid MSVC warnings.
+template <typename T> using checked_ptr = stdext::checked_array_iterator<T*>;
+template <typename T>
+constexpr auto make_checked(T* p, size_t size) -> checked_ptr<T> {
+ return {p, size};
+}
+#else
+template <typename T> using checked_ptr = T*;
+template <typename T> constexpr auto make_checked(T* p, size_t) -> T* {
+ return p;
+}
+#endif
+
+// Attempts to reserve space for n extra characters in the output range.
+// Returns a pointer to the reserved range or a reference to it.
+template <typename Container, FMT_ENABLE_IF(is_contiguous<Container>::value)>
+#if FMT_CLANG_VERSION >= 307 && !FMT_ICC_VERSION
+__attribute__((no_sanitize("undefined")))
+#endif
+inline auto
+reserve(std::back_insert_iterator<Container> it, size_t n)
+ -> checked_ptr<typename Container::value_type> {
+ Container& c = get_container(it);
+ size_t size = c.size();
+ c.resize(size + n);
+ return make_checked(get_data(c) + size, n);
+}
+
+template <typename T>
+inline auto reserve(buffer_appender<T> it, size_t n) -> buffer_appender<T> {
+ buffer<T>& buf = get_container(it);
+ buf.try_reserve(buf.size() + n);
+ return it;
+}
+
+template <typename Iterator>
+constexpr auto reserve(Iterator& it, size_t) -> Iterator& {
+ return it;
+}
+
+template <typename OutputIt>
+using reserve_iterator =
+ remove_reference_t<decltype(reserve(std::declval<OutputIt&>(), 0))>;
+
+template <typename T, typename OutputIt>
+constexpr auto to_pointer(OutputIt, size_t) -> T* {
+ return nullptr;
+}
+template <typename T> auto to_pointer(buffer_appender<T> it, size_t n) -> T* {
+ buffer<T>& buf = get_container(it);
+ auto size = buf.size();
+ if (buf.capacity() < size + n) return nullptr;
+ buf.try_resize(size + n);
+ return buf.data() + size;
+}
+
+template <typename Container, FMT_ENABLE_IF(is_contiguous<Container>::value)>
+inline auto base_iterator(std::back_insert_iterator<Container>& it,
+ checked_ptr<typename Container::value_type>)
+ -> std::back_insert_iterator<Container> {
+ return it;
+}
+
+template <typename Iterator>
+constexpr auto base_iterator(Iterator, Iterator it) -> Iterator {
+ return it;
+}
+
+// <algorithm> is spectacularly slow to compile in C++20 so use a simple fill_n
+// instead (#1998).
+template <typename OutputIt, typename Size, typename T>
+FMT_CONSTEXPR auto fill_n(OutputIt out, Size count, const T& value)
+ -> OutputIt {
+ for (Size i = 0; i < count; ++i) *out++ = value;
+ return out;
+}
+template <typename T, typename Size>
+FMT_CONSTEXPR20 auto fill_n(T* out, Size count, char value) -> T* {
+ if (is_constant_evaluated()) {
+ return fill_n<T*, Size, T>(out, count, value);
+ }
+ std::memset(out, value, to_unsigned(count));
+ return out + count;
+}
+
+#ifdef __cpp_char8_t
+using char8_type = char8_t;
+#else
+enum char8_type : unsigned char {};
+#endif
+
+template <typename OutChar, typename InputIt, typename OutputIt>
+FMT_CONSTEXPR FMT_NOINLINE auto copy_str_noinline(InputIt begin, InputIt end,
+ OutputIt out) -> OutputIt {
+ return copy_str<OutChar>(begin, end, out);
+}
+
+// A public domain branchless UTF-8 decoder by Christopher Wellons:
+// https://github.com/skeeto/branchless-utf8
+/* Decode the next character, c, from s, reporting errors in e.
+ *
+ * Since this is a branchless decoder, four bytes will be read from the
+ * buffer regardless of the actual length of the next character. This
+ * means the buffer _must_ have at least three bytes of zero padding
+ * following the end of the data stream.
+ *
+ * Errors are reported in e, which will be non-zero if the parsed
+ * character was somehow invalid: invalid byte sequence, non-canonical
+ * encoding, or a surrogate half.
+ *
+ * The function returns a pointer to the next character. When an error
+ * occurs, this pointer will be a guess that depends on the particular
+ * error, but it will always advance at least one byte.
+ */
+FMT_CONSTEXPR inline auto utf8_decode(const char* s, uint32_t* c, int* e)
+ -> const char* {
+ constexpr const int masks[] = {0x00, 0x7f, 0x1f, 0x0f, 0x07};
+ constexpr const uint32_t mins[] = {4194304, 0, 128, 2048, 65536};
+ constexpr const int shiftc[] = {0, 18, 12, 6, 0};
+ constexpr const int shifte[] = {0, 6, 4, 2, 0};
+
+ int len = "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0\0\0\2\2\2\2\3\3\4"
+ [static_cast<unsigned char>(*s) >> 3];
+ // Compute the pointer to the next character early so that the next
+ // iteration can start working on the next character. Neither Clang
+ // nor GCC figure out this reordering on their own.
+ const char* next = s + len + !len;
+
+ using uchar = unsigned char;
+
+ // Assume a four-byte character and load four bytes. Unused bits are
+ // shifted out.
+ *c = uint32_t(uchar(s[0]) & masks[len]) << 18;
+ *c |= uint32_t(uchar(s[1]) & 0x3f) << 12;
+ *c |= uint32_t(uchar(s[2]) & 0x3f) << 6;
+ *c |= uint32_t(uchar(s[3]) & 0x3f) << 0;
+ *c >>= shiftc[len];
+
+ // Accumulate the various error conditions.
+ *e = (*c < mins[len]) << 6; // non-canonical encoding
+ *e |= ((*c >> 11) == 0x1b) << 7; // surrogate half?
+ *e |= (*c > 0x10FFFF) << 8; // out of range?
+ *e |= (uchar(s[1]) & 0xc0) >> 2;
+ *e |= (uchar(s[2]) & 0xc0) >> 4;
+ *e |= uchar(s[3]) >> 6;
+ *e ^= 0x2a; // top two bits of each tail byte correct?
+ *e >>= shifte[len];
+
+ return next;
+}
+
+constexpr FMT_INLINE_VARIABLE uint32_t invalid_code_point = ~uint32_t();
+
+// Invokes f(cp, sv) for every code point cp in s with sv being the string view
+// corresponding to the code point. cp is invalid_code_point on error.
+template <typename F>
+FMT_CONSTEXPR void for_each_codepoint(string_view s, F f) {
+ auto decode = [f](const char* buf_ptr, const char* ptr) {
+ auto cp = uint32_t();
+ auto error = 0;
+ auto end = utf8_decode(buf_ptr, &cp, &error);
+ bool result = f(error ? invalid_code_point : cp,
+ string_view(ptr, error ? 1 : to_unsigned(end - buf_ptr)));
+ return result ? (error ? buf_ptr + 1 : end) : nullptr;
+ };
+ auto p = s.data();
+ const size_t block_size = 4; // utf8_decode always reads blocks of 4 chars.
+ if (s.size() >= block_size) {
+ for (auto end = p + s.size() - block_size + 1; p < end;) {
+ p = decode(p, p);
+ if (!p) return;
+ }
+ }
+ if (auto num_chars_left = s.data() + s.size() - p) {
+ char buf[2 * block_size - 1] = {};
+ copy_str<char>(p, p + num_chars_left, buf);
+ const char* buf_ptr = buf;
+ do {
+ auto end = decode(buf_ptr, p);
+ if (!end) return;
+ p += end - buf_ptr;
+ buf_ptr = end;
+ } while (buf_ptr - buf < num_chars_left);
+ }
+}
+
+template <typename Char>
+inline auto compute_width(basic_string_view<Char> s) -> size_t {
+ return s.size();
+}
+
+// Computes approximate display width of a UTF-8 string.
+FMT_CONSTEXPR inline size_t compute_width(string_view s) {
+ size_t num_code_points = 0;
+ // It is not a lambda for compatibility with C++14.
+ struct count_code_points {
+ size_t* count;
+ FMT_CONSTEXPR auto operator()(uint32_t cp, string_view) const -> bool {
+ *count += detail::to_unsigned(
+ 1 +
+ (cp >= 0x1100 &&
+ (cp <= 0x115f || // Hangul Jamo init. consonants
+ cp == 0x2329 || // LEFT-POINTING ANGLE BRACKET
+ cp == 0x232a || // RIGHT-POINTING ANGLE BRACKET
+ // CJK ... Yi except IDEOGRAPHIC HALF FILL SPACE:
+ (cp >= 0x2e80 && cp <= 0xa4cf && cp != 0x303f) ||
+ (cp >= 0xac00 && cp <= 0xd7a3) || // Hangul Syllables
+ (cp >= 0xf900 && cp <= 0xfaff) || // CJK Compatibility Ideographs
+ (cp >= 0xfe10 && cp <= 0xfe19) || // Vertical Forms
+ (cp >= 0xfe30 && cp <= 0xfe6f) || // CJK Compatibility Forms
+ (cp >= 0xff00 && cp <= 0xff60) || // Fullwidth Forms
+ (cp >= 0xffe0 && cp <= 0xffe6) || // Fullwidth Forms
+ (cp >= 0x20000 && cp <= 0x2fffd) || // CJK
+ (cp >= 0x30000 && cp <= 0x3fffd) ||
+ // Miscellaneous Symbols and Pictographs + Emoticons:
+ (cp >= 0x1f300 && cp <= 0x1f64f) ||
+ // Supplemental Symbols and Pictographs:
+ (cp >= 0x1f900 && cp <= 0x1f9ff))));
+ return true;
+ }
+ };
+ // We could avoid branches by using utf8_decode directly.
+ for_each_codepoint(s, count_code_points{&num_code_points});
+ return num_code_points;
+}
+
+inline auto compute_width(basic_string_view<char8_type> s) -> size_t {
+ return compute_width(
+ string_view(reinterpret_cast<const char*>(s.data()), s.size()));
+}
+
+template <typename Char>
+inline auto code_point_index(basic_string_view<Char> s, size_t n) -> size_t {
+ size_t size = s.size();
+ return n < size ? n : size;
+}
+
+// Calculates the index of the nth code point in a UTF-8 string.
+inline auto code_point_index(string_view s, size_t n) -> size_t {
+ const char* data = s.data();
+ size_t num_code_points = 0;
+ for (size_t i = 0, size = s.size(); i != size; ++i) {
+ if ((data[i] & 0xc0) != 0x80 && ++num_code_points > n) return i;
+ }
+ return s.size();
+}
+
+inline auto code_point_index(basic_string_view<char8_type> s, size_t n)
+ -> size_t {
+ return code_point_index(
+ string_view(reinterpret_cast<const char*>(s.data()), s.size()), n);
+}
+
+template <typename T> struct is_integral : std::is_integral<T> {};
+template <> struct is_integral<int128_opt> : std::true_type {};
+template <> struct is_integral<uint128_t> : std::true_type {};
+
+template <typename T>
+using is_signed =
+ std::integral_constant<bool, std::numeric_limits<T>::is_signed ||
+ std::is_same<T, int128_opt>::value>;
+
+template <typename T>
+using is_integer =
+ bool_constant<is_integral<T>::value && !std::is_same<T, bool>::value &&
+ !std::is_same<T, char>::value &&
+ !std::is_same<T, wchar_t>::value>;
+
+#ifndef FMT_USE_FLOAT
+# define FMT_USE_FLOAT 1
+#endif
+#ifndef FMT_USE_DOUBLE
+# define FMT_USE_DOUBLE 1
+#endif
+#ifndef FMT_USE_LONG_DOUBLE
+# define FMT_USE_LONG_DOUBLE 1
+#endif
+
+#ifndef FMT_USE_FLOAT128
+# ifdef __clang__
+// Clang emulates GCC, so it has to appear early.
+# if FMT_HAS_INCLUDE(<quadmath.h>)
+# define FMT_USE_FLOAT128 1
+# endif
+# elif defined(__GNUC__)
+// GNU C++:
+# if defined(_GLIBCXX_USE_FLOAT128) && !defined(__STRICT_ANSI__)
+# define FMT_USE_FLOAT128 1
+# endif
+# endif
+# ifndef FMT_USE_FLOAT128
+# define FMT_USE_FLOAT128 0
+# endif
+#endif
+
+#if FMT_USE_FLOAT128
+using float128 = __float128;
+#else
+using float128 = void;
+#endif
+template <typename T> using is_float128 = std::is_same<T, float128>;
+
+template <typename T>
+using is_floating_point =
+ bool_constant<std::is_floating_point<T>::value || is_float128<T>::value>;
+
+template <typename T, bool = std::is_floating_point<T>::value>
+struct is_fast_float : bool_constant<std::numeric_limits<T>::is_iec559 &&
+ sizeof(T) <= sizeof(double)> {};
+template <typename T> struct is_fast_float<T, false> : std::false_type {};
+
+template <typename T>
+using is_double_double = bool_constant<std::numeric_limits<T>::digits == 106>;
+
+#ifndef FMT_USE_FULL_CACHE_DRAGONBOX
+# define FMT_USE_FULL_CACHE_DRAGONBOX 0
+#endif
+
+template <typename T>
+template <typename U>
+void buffer<T>::append(const U* begin, const U* end) {
+ while (begin != end) {
+ auto count = to_unsigned(end - begin);
+ try_reserve(size_ + count);
+ auto free_cap = capacity_ - size_;
+ if (free_cap < count) count = free_cap;
+ std::uninitialized_copy_n(begin, count, make_checked(ptr_ + size_, count));
+ size_ += count;
+ begin += count;
+ }
+}
+
+template <typename T, typename Enable = void>
+struct is_locale : std::false_type {};
+template <typename T>
+struct is_locale<T, void_t<decltype(T::classic())>> : std::true_type {};
+} // namespace detail
+
+FMT_BEGIN_EXPORT
+
+// The number of characters to store in the basic_memory_buffer object itself
+// to avoid dynamic memory allocation.
+enum { inline_buffer_size = 500 };
+
+/**
+ \rst
+ A dynamically growing memory buffer for trivially copyable/constructible types
+ with the first ``SIZE`` elements stored in the object itself.
+
+ You can use the ``memory_buffer`` type alias for ``char`` instead.
+
+ **Example**::
+
+ auto out = fmt::memory_buffer();
+ format_to(std::back_inserter(out), "The answer is {}.", 42);
+
+ This will append the following output to the ``out`` object:
+
+ .. code-block:: none
+
+ The answer is 42.
+
+ The output can be converted to an ``std::string`` with ``to_string(out)``.
+ \endrst
+ */
+template <typename T, size_t SIZE = inline_buffer_size,
+ typename Allocator = std::allocator<T>>
+class basic_memory_buffer final : public detail::buffer<T> {
+ private:
+ T store_[SIZE];
+
+ // Don't inherit from Allocator avoid generating type_info for it.
+ Allocator alloc_;
+
+ // Deallocate memory allocated by the buffer.
+ FMT_CONSTEXPR20 void deallocate() {
+ T* data = this->data();
+ if (data != store_) alloc_.deallocate(data, this->capacity());
+ }
+
+ protected:
+ FMT_CONSTEXPR20 void grow(size_t size) override {
+ detail::abort_fuzzing_if(size > 5000);
+ const size_t max_size = std::allocator_traits<Allocator>::max_size(alloc_);
+ size_t old_capacity = this->capacity();
+ size_t new_capacity = old_capacity + old_capacity / 2;
+ if (size > new_capacity)
+ new_capacity = size;
+ else if (new_capacity > max_size)
+ new_capacity = size > max_size ? size : max_size;
+ T* old_data = this->data();
+ T* new_data =
+ std::allocator_traits<Allocator>::allocate(alloc_, new_capacity);
+ // The following code doesn't throw, so the raw pointer above doesn't leak.
+ std::uninitialized_copy(old_data, old_data + this->size(),
+ detail::make_checked(new_data, new_capacity));
+ this->set(new_data, new_capacity);
+ // deallocate must not throw according to the standard, but even if it does,
+ // the buffer already uses the new storage and will deallocate it in
+ // destructor.
+ if (old_data != store_) alloc_.deallocate(old_data, old_capacity);
+ }
+
+ public:
+ using value_type = T;
+ using const_reference = const T&;
+
+ FMT_CONSTEXPR20 explicit basic_memory_buffer(
+ const Allocator& alloc = Allocator())
+ : alloc_(alloc) {
+ this->set(store_, SIZE);
+ if (detail::is_constant_evaluated()) detail::fill_n(store_, SIZE, T());
+ }
+ FMT_CONSTEXPR20 ~basic_memory_buffer() { deallocate(); }
+
+ private:
+ // Move data from other to this buffer.
+ FMT_CONSTEXPR20 void move(basic_memory_buffer& other) {
+ alloc_ = std::move(other.alloc_);
+ T* data = other.data();
+ size_t size = other.size(), capacity = other.capacity();
+ if (data == other.store_) {
+ this->set(store_, capacity);
+ detail::copy_str<T>(other.store_, other.store_ + size,
+ detail::make_checked(store_, capacity));
+ } else {
+ this->set(data, capacity);
+ // Set pointer to the inline array so that delete is not called
+ // when deallocating.
+ other.set(other.store_, 0);
+ other.clear();
+ }
+ this->resize(size);
+ }
+
+ public:
+ /**
+ \rst
+ Constructs a :class:`fmt::basic_memory_buffer` object moving the content
+ of the other object to it.
+ \endrst
+ */
+ FMT_CONSTEXPR20 basic_memory_buffer(basic_memory_buffer&& other) noexcept {
+ move(other);
+ }
+
+ /**
+ \rst
+ Moves the content of the other ``basic_memory_buffer`` object to this one.
+ \endrst
+ */
+ auto operator=(basic_memory_buffer&& other) noexcept -> basic_memory_buffer& {
+ FMT_ASSERT(this != &other, "");
+ deallocate();
+ move(other);
+ return *this;
+ }
+
+ // Returns a copy of the allocator associated with this buffer.
+ auto get_allocator() const -> Allocator { return alloc_; }
+
+ /**
+ Resizes the buffer to contain *count* elements. If T is a POD type new
+ elements may not be initialized.
+ */
+ FMT_CONSTEXPR20 void resize(size_t count) { this->try_resize(count); }
+
+ /** Increases the buffer capacity to *new_capacity*. */
+ void reserve(size_t new_capacity) { this->try_reserve(new_capacity); }
+
+ // Directly append data into the buffer
+ using detail::buffer<T>::append;
+ template <typename ContiguousRange>
+ void append(const ContiguousRange& range) {
+ append(range.data(), range.data() + range.size());
+ }
+};
+
+using memory_buffer = basic_memory_buffer<char>;
+
+template <typename T, size_t SIZE, typename Allocator>
+struct is_contiguous<basic_memory_buffer<T, SIZE, Allocator>> : std::true_type {
+};
+
+FMT_END_EXPORT
+namespace detail {
+FMT_API bool write_console(std::FILE* f, string_view text);
+FMT_API void print(std::FILE*, string_view);
+} // namespace detail
+FMT_BEGIN_EXPORT
+
+// Suppress a misleading warning in older versions of clang.
+#if FMT_CLANG_VERSION
+# pragma clang diagnostic ignored "-Wweak-vtables"
+#endif
+
+/** An error reported from a formatting function. */
+class FMT_API format_error : public std::runtime_error {
+ public:
+ using std::runtime_error::runtime_error;
+};
+
+namespace detail_exported {
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <typename Char, size_t N> struct fixed_string {
+ constexpr fixed_string(const Char (&str)[N]) {
+ detail::copy_str<Char, const Char*, Char*>(static_cast<const Char*>(str),
+ str + N, data);
+ }
+ Char data[N] = {};
+};
+#endif
+
+// Converts a compile-time string to basic_string_view.
+template <typename Char, size_t N>
+constexpr auto compile_string_to_view(const Char (&s)[N])
+ -> basic_string_view<Char> {
+ // Remove trailing NUL character if needed. Won't be present if this is used
+ // with a raw character array (i.e. not defined as a string).
+ return {s, N - (std::char_traits<Char>::to_int_type(s[N - 1]) == 0 ? 1 : 0)};
+}
+template <typename Char>
+constexpr auto compile_string_to_view(detail::std_string_view<Char> s)
+ -> basic_string_view<Char> {
+ return {s.data(), s.size()};
+}
+} // namespace detail_exported
+
+class loc_value {
+ private:
+ basic_format_arg<format_context> value_;
+
+ public:
+ template <typename T, FMT_ENABLE_IF(!detail::is_float128<T>::value)>
+ loc_value(T value) : value_(detail::make_arg<format_context>(value)) {}
+
+ template <typename T, FMT_ENABLE_IF(detail::is_float128<T>::value)>
+ loc_value(T) {}
+
+ template <typename Visitor> auto visit(Visitor&& vis) -> decltype(vis(0)) {
+ return visit_format_arg(vis, value_);
+ }
+};
+
+// A locale facet that formats values in UTF-8.
+// It is parameterized on the locale to avoid the heavy <locale> include.
+template <typename Locale> class format_facet : public Locale::facet {
+ private:
+ std::string separator_;
+ std::string grouping_;
+ std::string decimal_point_;
+
+ protected:
+ virtual auto do_put(appender out, loc_value val,
+ const format_specs<>& specs) const -> bool;
+
+ public:
+ static FMT_API typename Locale::id id;
+
+ explicit format_facet(Locale& loc);
+ explicit format_facet(string_view sep = "",
+ std::initializer_list<unsigned char> g = {3},
+ std::string decimal_point = ".")
+ : separator_(sep.data(), sep.size()),
+ grouping_(g.begin(), g.end()),
+ decimal_point_(decimal_point) {}
+
+ auto put(appender out, loc_value val, const format_specs<>& specs) const
+ -> bool {
+ return do_put(out, val, specs);
+ }
+};
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+// Returns true if value is negative, false otherwise.
+// Same as `value < 0` but doesn't produce warnings if T is an unsigned type.
+template <typename T, FMT_ENABLE_IF(is_signed<T>::value)>
+constexpr auto is_negative(T value) -> bool {
+ return value < 0;
+}
+template <typename T, FMT_ENABLE_IF(!is_signed<T>::value)>
+constexpr auto is_negative(T) -> bool {
+ return false;
+}
+
+template <typename T>
+FMT_CONSTEXPR auto is_supported_floating_point(T) -> bool {
+ if (std::is_same<T, float>()) return FMT_USE_FLOAT;
+ if (std::is_same<T, double>()) return FMT_USE_DOUBLE;
+ if (std::is_same<T, long double>()) return FMT_USE_LONG_DOUBLE;
+ return true;
+}
+
+// Smallest of uint32_t, uint64_t, uint128_t that is large enough to
+// represent all values of an integral type T.
+template <typename T>
+using uint32_or_64_or_128_t =
+ conditional_t<num_bits<T>() <= 32 && !FMT_REDUCE_INT_INSTANTIATIONS,
+ uint32_t,
+ conditional_t<num_bits<T>() <= 64, uint64_t, uint128_t>>;
+template <typename T>
+using uint64_or_128_t = conditional_t<num_bits<T>() <= 64, uint64_t, uint128_t>;
+
+#define FMT_POWERS_OF_10(factor) \
+ factor * 10, (factor)*100, (factor)*1000, (factor)*10000, (factor)*100000, \
+ (factor)*1000000, (factor)*10000000, (factor)*100000000, \
+ (factor)*1000000000
+
+// Converts value in the range [0, 100) to a string.
+constexpr const char* digits2(size_t value) {
+ // GCC generates slightly better code when value is pointer-size.
+ return &"0001020304050607080910111213141516171819"
+ "2021222324252627282930313233343536373839"
+ "4041424344454647484950515253545556575859"
+ "6061626364656667686970717273747576777879"
+ "8081828384858687888990919293949596979899"[value * 2];
+}
+
+// Sign is a template parameter to workaround a bug in gcc 4.8.
+template <typename Char, typename Sign> constexpr Char sign(Sign s) {
+#if !FMT_GCC_VERSION || FMT_GCC_VERSION >= 604
+ static_assert(std::is_same<Sign, sign_t>::value, "");
+#endif
+ return static_cast<Char>("\0-+ "[s]);
+}
+
+template <typename T> FMT_CONSTEXPR auto count_digits_fallback(T n) -> int {
+ int count = 1;
+ for (;;) {
+ // Integer division is slow so do it for a group of four digits instead
+ // of for every digit. The idea comes from the talk by Alexandrescu
+ // "Three Optimization Tips for C++". See speed-test for a comparison.
+ if (n < 10) return count;
+ if (n < 100) return count + 1;
+ if (n < 1000) return count + 2;
+ if (n < 10000) return count + 3;
+ n /= 10000u;
+ count += 4;
+ }
+}
+#if FMT_USE_INT128
+FMT_CONSTEXPR inline auto count_digits(uint128_opt n) -> int {
+ return count_digits_fallback(n);
+}
+#endif
+
+#ifdef FMT_BUILTIN_CLZLL
+// It is a separate function rather than a part of count_digits to workaround
+// the lack of static constexpr in constexpr functions.
+inline auto do_count_digits(uint64_t n) -> int {
+ // This has comparable performance to the version by Kendall Willets
+ // (https://github.com/fmtlib/format-benchmark/blob/master/digits10)
+ // but uses smaller tables.
+ // Maps bsr(n) to ceil(log10(pow(2, bsr(n) + 1) - 1)).
+ static constexpr uint8_t bsr2log10[] = {
+ 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5,
+ 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10,
+ 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 15, 15,
+ 15, 16, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 19, 20};
+ auto t = bsr2log10[FMT_BUILTIN_CLZLL(n | 1) ^ 63];
+ static constexpr const uint64_t zero_or_powers_of_10[] = {
+ 0, 0, FMT_POWERS_OF_10(1U), FMT_POWERS_OF_10(1000000000ULL),
+ 10000000000000000000ULL};
+ return t - (n < zero_or_powers_of_10[t]);
+}
+#endif
+
+// Returns the number of decimal digits in n. Leading zeros are not counted
+// except for n == 0 in which case count_digits returns 1.
+FMT_CONSTEXPR20 inline auto count_digits(uint64_t n) -> int {
+#ifdef FMT_BUILTIN_CLZLL
+ if (!is_constant_evaluated()) {
+ return do_count_digits(n);
+ }
+#endif
+ return count_digits_fallback(n);
+}
+
+// Counts the number of digits in n. BITS = log2(radix).
+template <int BITS, typename UInt>
+FMT_CONSTEXPR auto count_digits(UInt n) -> int {
+#ifdef FMT_BUILTIN_CLZ
+ if (!is_constant_evaluated() && num_bits<UInt>() == 32)
+ return (FMT_BUILTIN_CLZ(static_cast<uint32_t>(n) | 1) ^ 31) / BITS + 1;
+#endif
+ // Lambda avoids unreachable code warnings from NVHPC.
+ return [](UInt m) {
+ int num_digits = 0;
+ do {
+ ++num_digits;
+ } while ((m >>= BITS) != 0);
+ return num_digits;
+ }(n);
+}
+
+#ifdef FMT_BUILTIN_CLZ
+// It is a separate function rather than a part of count_digits to workaround
+// the lack of static constexpr in constexpr functions.
+FMT_INLINE auto do_count_digits(uint32_t n) -> int {
+// An optimization by Kendall Willets from https://bit.ly/3uOIQrB.
+// This increments the upper 32 bits (log10(T) - 1) when >= T is added.
+# define FMT_INC(T) (((sizeof(# T) - 1ull) << 32) - T)
+ static constexpr uint64_t table[] = {
+ FMT_INC(0), FMT_INC(0), FMT_INC(0), // 8
+ FMT_INC(10), FMT_INC(10), FMT_INC(10), // 64
+ FMT_INC(100), FMT_INC(100), FMT_INC(100), // 512
+ FMT_INC(1000), FMT_INC(1000), FMT_INC(1000), // 4096
+ FMT_INC(10000), FMT_INC(10000), FMT_INC(10000), // 32k
+ FMT_INC(100000), FMT_INC(100000), FMT_INC(100000), // 256k
+ FMT_INC(1000000), FMT_INC(1000000), FMT_INC(1000000), // 2048k
+ FMT_INC(10000000), FMT_INC(10000000), FMT_INC(10000000), // 16M
+ FMT_INC(100000000), FMT_INC(100000000), FMT_INC(100000000), // 128M
+ FMT_INC(1000000000), FMT_INC(1000000000), FMT_INC(1000000000), // 1024M
+ FMT_INC(1000000000), FMT_INC(1000000000) // 4B
+ };
+ auto inc = table[FMT_BUILTIN_CLZ(n | 1) ^ 31];
+ return static_cast<int>((n + inc) >> 32);
+}
+#endif
+
+// Optional version of count_digits for better performance on 32-bit platforms.
+FMT_CONSTEXPR20 inline auto count_digits(uint32_t n) -> int {
+#ifdef FMT_BUILTIN_CLZ
+ if (!is_constant_evaluated()) {
+ return do_count_digits(n);
+ }
+#endif
+ return count_digits_fallback(n);
+}
+
+template <typename Int> constexpr auto digits10() noexcept -> int {
+ return std::numeric_limits<Int>::digits10;
+}
+template <> constexpr auto digits10<int128_opt>() noexcept -> int { return 38; }
+template <> constexpr auto digits10<uint128_t>() noexcept -> int { return 38; }
+
+template <typename Char> struct thousands_sep_result {
+ std::string grouping;
+ Char thousands_sep;
+};
+
+template <typename Char>
+FMT_API auto thousands_sep_impl(locale_ref loc) -> thousands_sep_result<Char>;
+template <typename Char>
+inline auto thousands_sep(locale_ref loc) -> thousands_sep_result<Char> {
+ auto result = thousands_sep_impl<char>(loc);
+ return {result.grouping, Char(result.thousands_sep)};
+}
+template <>
+inline auto thousands_sep(locale_ref loc) -> thousands_sep_result<wchar_t> {
+ return thousands_sep_impl<wchar_t>(loc);
+}
+
+template <typename Char>
+FMT_API auto decimal_point_impl(locale_ref loc) -> Char;
+template <typename Char> inline auto decimal_point(locale_ref loc) -> Char {
+ return Char(decimal_point_impl<char>(loc));
+}
+template <> inline auto decimal_point(locale_ref loc) -> wchar_t {
+ return decimal_point_impl<wchar_t>(loc);
+}
+
+// Compares two characters for equality.
+template <typename Char> auto equal2(const Char* lhs, const char* rhs) -> bool {
+ return lhs[0] == Char(rhs[0]) && lhs[1] == Char(rhs[1]);
+}
+inline auto equal2(const char* lhs, const char* rhs) -> bool {
+ return memcmp(lhs, rhs, 2) == 0;
+}
+
+// Copies two characters from src to dst.
+template <typename Char>
+FMT_CONSTEXPR20 FMT_INLINE void copy2(Char* dst, const char* src) {
+ if (!is_constant_evaluated() && sizeof(Char) == sizeof(char)) {
+ memcpy(dst, src, 2);
+ return;
+ }
+ *dst++ = static_cast<Char>(*src++);
+ *dst = static_cast<Char>(*src);
+}
+
+template <typename Iterator> struct format_decimal_result {
+ Iterator begin;
+ Iterator end;
+};
+
+// Formats a decimal unsigned integer value writing into out pointing to a
+// buffer of specified size. The caller must ensure that the buffer is large
+// enough.
+template <typename Char, typename UInt>
+FMT_CONSTEXPR20 auto format_decimal(Char* out, UInt value, int size)
+ -> format_decimal_result<Char*> {
+ FMT_ASSERT(size >= count_digits(value), "invalid digit count");
+ out += size;
+ Char* end = out;
+ while (value >= 100) {
+ // Integer division is slow so do it for a group of two digits instead
+ // of for every digit. The idea comes from the talk by Alexandrescu
+ // "Three Optimization Tips for C++". See speed-test for a comparison.
+ out -= 2;
+ copy2(out, digits2(static_cast<size_t>(value % 100)));
+ value /= 100;
+ }
+ if (value < 10) {
+ *--out = static_cast<Char>('0' + value);
+ return {out, end};
+ }
+ out -= 2;
+ copy2(out, digits2(static_cast<size_t>(value)));
+ return {out, end};
+}
+
+template <typename Char, typename UInt, typename Iterator,
+ FMT_ENABLE_IF(!std::is_pointer<remove_cvref_t<Iterator>>::value)>
+FMT_CONSTEXPR inline auto format_decimal(Iterator out, UInt value, int size)
+ -> format_decimal_result<Iterator> {
+ // Buffer is large enough to hold all digits (digits10 + 1).
+ Char buffer[digits10<UInt>() + 1] = {};
+ auto end = format_decimal(buffer, value, size).end;
+ return {out, detail::copy_str_noinline<Char>(buffer, end, out)};
+}
+
+template <unsigned BASE_BITS, typename Char, typename UInt>
+FMT_CONSTEXPR auto format_uint(Char* buffer, UInt value, int num_digits,
+ bool upper = false) -> Char* {
+ buffer += num_digits;
+ Char* end = buffer;
+ do {
+ const char* digits = upper ? "0123456789ABCDEF" : "0123456789abcdef";
+ unsigned digit = static_cast<unsigned>(value & ((1 << BASE_BITS) - 1));
+ *--buffer = static_cast<Char>(BASE_BITS < 4 ? static_cast<char>('0' + digit)
+ : digits[digit]);
+ } while ((value >>= BASE_BITS) != 0);
+ return end;
+}
+
+template <unsigned BASE_BITS, typename Char, typename It, typename UInt>
+inline auto format_uint(It out, UInt value, int num_digits, bool upper = false)
+ -> It {
+ if (auto ptr = to_pointer<Char>(out, to_unsigned(num_digits))) {
+ format_uint<BASE_BITS>(ptr, value, num_digits, upper);
+ return out;
+ }
+ // Buffer should be large enough to hold all digits (digits / BASE_BITS + 1).
+ char buffer[num_bits<UInt>() / BASE_BITS + 1];
+ format_uint<BASE_BITS>(buffer, value, num_digits, upper);
+ return detail::copy_str_noinline<Char>(buffer, buffer + num_digits, out);
+}
+
+// A converter from UTF-8 to UTF-16.
+class utf8_to_utf16 {
+ private:
+ basic_memory_buffer<wchar_t> buffer_;
+
+ public:
+ FMT_API explicit utf8_to_utf16(string_view s);
+ operator basic_string_view<wchar_t>() const { return {&buffer_[0], size()}; }
+ auto size() const -> size_t { return buffer_.size() - 1; }
+ auto c_str() const -> const wchar_t* { return &buffer_[0]; }
+ auto str() const -> std::wstring { return {&buffer_[0], size()}; }
+};
+
+// A converter from UTF-16/UTF-32 (host endian) to UTF-8.
+template <typename WChar, typename Buffer = memory_buffer>
+class unicode_to_utf8 {
+ private:
+ Buffer buffer_;
+
+ public:
+ unicode_to_utf8() {}
+ explicit unicode_to_utf8(basic_string_view<WChar> s) {
+ static_assert(sizeof(WChar) == 2 || sizeof(WChar) == 4,
+ "Expect utf16 or utf32");
+
+ if (!convert(s))
+ FMT_THROW(std::runtime_error(sizeof(WChar) == 2 ? "invalid utf16"
+ : "invalid utf32"));
+ }
+ operator string_view() const { return string_view(&buffer_[0], size()); }
+ size_t size() const { return buffer_.size() - 1; }
+ const char* c_str() const { return &buffer_[0]; }
+ std::string str() const { return std::string(&buffer_[0], size()); }
+
+ // Performs conversion returning a bool instead of throwing exception on
+ // conversion error. This method may still throw in case of memory allocation
+ // error.
+ bool convert(basic_string_view<WChar> s) {
+ if (!convert(buffer_, s)) return false;
+ buffer_.push_back(0);
+ return true;
+ }
+ static bool convert(Buffer& buf, basic_string_view<WChar> s) {
+ for (auto p = s.begin(); p != s.end(); ++p) {
+ uint32_t c = static_cast<uint32_t>(*p);
+ if (sizeof(WChar) == 2 && c >= 0xd800 && c <= 0xdfff) {
+ // surrogate pair
+ ++p;
+ if (p == s.end() || (c & 0xfc00) != 0xd800 || (*p & 0xfc00) != 0xdc00) {
+ return false;
+ }
+ c = (c << 10) + static_cast<uint32_t>(*p) - 0x35fdc00;
+ }
+ if (c < 0x80) {
+ buf.push_back(static_cast<char>(c));
+ } else if (c < 0x800) {
+ buf.push_back(static_cast<char>(0xc0 | (c >> 6)));
+ buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+ } else if ((c >= 0x800 && c <= 0xd7ff) || (c >= 0xe000 && c <= 0xffff)) {
+ buf.push_back(static_cast<char>(0xe0 | (c >> 12)));
+ buf.push_back(static_cast<char>(0x80 | ((c & 0xfff) >> 6)));
+ buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+ } else if (c >= 0x10000 && c <= 0x10ffff) {
+ buf.push_back(static_cast<char>(0xf0 | (c >> 18)));
+ buf.push_back(static_cast<char>(0x80 | ((c & 0x3ffff) >> 12)));
+ buf.push_back(static_cast<char>(0x80 | ((c & 0xfff) >> 6)));
+ buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+ } else {
+ return false;
+ }
+ }
+ return true;
+ }
+};
+
+// Computes 128-bit result of multiplication of two 64-bit unsigned integers.
+inline uint128_fallback umul128(uint64_t x, uint64_t y) noexcept {
+#if FMT_USE_INT128
+ auto p = static_cast<uint128_opt>(x) * static_cast<uint128_opt>(y);
+ return {static_cast<uint64_t>(p >> 64), static_cast<uint64_t>(p)};
+#elif defined(_MSC_VER) && defined(_M_X64)
+ auto result = uint128_fallback();
+ result.lo_ = _umul128(x, y, &result.hi_);
+ return result;
+#else
+ const uint64_t mask = static_cast<uint64_t>(max_value<uint32_t>());
+
+ uint64_t a = x >> 32;
+ uint64_t b = x & mask;
+ uint64_t c = y >> 32;
+ uint64_t d = y & mask;
+
+ uint64_t ac = a * c;
+ uint64_t bc = b * c;
+ uint64_t ad = a * d;
+ uint64_t bd = b * d;
+
+ uint64_t intermediate = (bd >> 32) + (ad & mask) + (bc & mask);
+
+ return {ac + (intermediate >> 32) + (ad >> 32) + (bc >> 32),
+ (intermediate << 32) + (bd & mask)};
+#endif
+}
+
+namespace dragonbox {
+// Computes floor(log10(pow(2, e))) for e in [-2620, 2620] using the method from
+// https://fmt.dev/papers/Dragonbox.pdf#page=28, section 6.1.
+inline int floor_log10_pow2(int e) noexcept {
+ FMT_ASSERT(e <= 2620 && e >= -2620, "too large exponent");
+ static_assert((-1 >> 1) == -1, "right shift is not arithmetic");
+ return (e * 315653) >> 20;
+}
+
+inline int floor_log2_pow10(int e) noexcept {
+ FMT_ASSERT(e <= 1233 && e >= -1233, "too large exponent");
+ return (e * 1741647) >> 19;
+}
+
+// Computes upper 64 bits of multiplication of two 64-bit unsigned integers.
+inline uint64_t umul128_upper64(uint64_t x, uint64_t y) noexcept {
+#if FMT_USE_INT128
+ auto p = static_cast<uint128_opt>(x) * static_cast<uint128_opt>(y);
+ return static_cast<uint64_t>(p >> 64);
+#elif defined(_MSC_VER) && defined(_M_X64)
+ return __umulh(x, y);
+#else
+ return umul128(x, y).high();
+#endif
+}
+
+// Computes upper 128 bits of multiplication of a 64-bit unsigned integer and a
+// 128-bit unsigned integer.
+inline uint128_fallback umul192_upper128(uint64_t x,
+ uint128_fallback y) noexcept {
+ uint128_fallback r = umul128(x, y.high());
+ r += umul128_upper64(x, y.low());
+ return r;
+}
+
+FMT_API uint128_fallback get_cached_power(int k) noexcept;
+
+// Type-specific information that Dragonbox uses.
+template <typename T, typename Enable = void> struct float_info;
+
+template <> struct float_info<float> {
+ using carrier_uint = uint32_t;
+ static const int exponent_bits = 8;
+ static const int kappa = 1;
+ static const int big_divisor = 100;
+ static const int small_divisor = 10;
+ static const int min_k = -31;
+ static const int max_k = 46;
+ static const int shorter_interval_tie_lower_threshold = -35;
+ static const int shorter_interval_tie_upper_threshold = -35;
+};
+
+template <> struct float_info<double> {
+ using carrier_uint = uint64_t;
+ static const int exponent_bits = 11;
+ static const int kappa = 2;
+ static const int big_divisor = 1000;
+ static const int small_divisor = 100;
+ static const int min_k = -292;
+ static const int max_k = 341;
+ static const int shorter_interval_tie_lower_threshold = -77;
+ static const int shorter_interval_tie_upper_threshold = -77;
+};
+
+// An 80- or 128-bit floating point number.
+template <typename T>
+struct float_info<T, enable_if_t<std::numeric_limits<T>::digits == 64 ||
+ std::numeric_limits<T>::digits == 113 ||
+ is_float128<T>::value>> {
+ using carrier_uint = detail::uint128_t;
+ static const int exponent_bits = 15;
+};
+
+// A double-double floating point number.
+template <typename T>
+struct float_info<T, enable_if_t<is_double_double<T>::value>> {
+ using carrier_uint = detail::uint128_t;
+};
+
+template <typename T> struct decimal_fp {
+ using significand_type = typename float_info<T>::carrier_uint;
+ significand_type significand;
+ int exponent;
+};
+
+template <typename T> FMT_API auto to_decimal(T x) noexcept -> decimal_fp<T>;
+} // namespace dragonbox
+
+// Returns true iff Float has the implicit bit which is not stored.
+template <typename Float> constexpr bool has_implicit_bit() {
+ // An 80-bit FP number has a 64-bit significand an no implicit bit.
+ return std::numeric_limits<Float>::digits != 64;
+}
+
+// Returns the number of significand bits stored in Float. The implicit bit is
+// not counted since it is not stored.
+template <typename Float> constexpr int num_significand_bits() {
+ // std::numeric_limits may not support __float128.
+ return is_float128<Float>() ? 112
+ : (std::numeric_limits<Float>::digits -
+ (has_implicit_bit<Float>() ? 1 : 0));
+}
+
+template <typename Float>
+constexpr auto exponent_mask() ->
+ typename dragonbox::float_info<Float>::carrier_uint {
+ using float_uint = typename dragonbox::float_info<Float>::carrier_uint;
+ return ((float_uint(1) << dragonbox::float_info<Float>::exponent_bits) - 1)
+ << num_significand_bits<Float>();
+}
+template <typename Float> constexpr auto exponent_bias() -> int {
+ // std::numeric_limits may not support __float128.
+ return is_float128<Float>() ? 16383
+ : std::numeric_limits<Float>::max_exponent - 1;
+}
+
+// Writes the exponent exp in the form "[+-]d{2,3}" to buffer.
+template <typename Char, typename It>
+FMT_CONSTEXPR auto write_exponent(int exp, It it) -> It {
+ FMT_ASSERT(-10000 < exp && exp < 10000, "exponent out of range");
+ if (exp < 0) {
+ *it++ = static_cast<Char>('-');
+ exp = -exp;
+ } else {
+ *it++ = static_cast<Char>('+');
+ }
+ if (exp >= 100) {
+ const char* top = digits2(to_unsigned(exp / 100));
+ if (exp >= 1000) *it++ = static_cast<Char>(top[0]);
+ *it++ = static_cast<Char>(top[1]);
+ exp %= 100;
+ }
+ const char* d = digits2(to_unsigned(exp));
+ *it++ = static_cast<Char>(d[0]);
+ *it++ = static_cast<Char>(d[1]);
+ return it;
+}
+
+// A floating-point number f * pow(2, e) where F is an unsigned type.
+template <typename F> struct basic_fp {
+ F f;
+ int e;
+
+ static constexpr const int num_significand_bits =
+ static_cast<int>(sizeof(F) * num_bits<unsigned char>());
+
+ constexpr basic_fp() : f(0), e(0) {}
+ constexpr basic_fp(uint64_t f_val, int e_val) : f(f_val), e(e_val) {}
+
+ // Constructs fp from an IEEE754 floating-point number.
+ template <typename Float> FMT_CONSTEXPR basic_fp(Float n) { assign(n); }
+
+ // Assigns n to this and return true iff predecessor is closer than successor.
+ template <typename Float, FMT_ENABLE_IF(!is_double_double<Float>::value)>
+ FMT_CONSTEXPR auto assign(Float n) -> bool {
+ static_assert(std::numeric_limits<Float>::digits <= 113, "unsupported FP");
+ // Assume Float is in the format [sign][exponent][significand].
+ using carrier_uint = typename dragonbox::float_info<Float>::carrier_uint;
+ const auto num_float_significand_bits =
+ detail::num_significand_bits<Float>();
+ const auto implicit_bit = carrier_uint(1) << num_float_significand_bits;
+ const auto significand_mask = implicit_bit - 1;
+ auto u = bit_cast<carrier_uint>(n);
+ f = static_cast<F>(u & significand_mask);
+ auto biased_e = static_cast<int>((u & exponent_mask<Float>()) >>
+ num_float_significand_bits);
+ // The predecessor is closer if n is a normalized power of 2 (f == 0)
+ // other than the smallest normalized number (biased_e > 1).
+ auto is_predecessor_closer = f == 0 && biased_e > 1;
+ if (biased_e == 0)
+ biased_e = 1; // Subnormals use biased exponent 1 (min exponent).
+ else if (has_implicit_bit<Float>())
+ f += static_cast<F>(implicit_bit);
+ e = biased_e - exponent_bias<Float>() - num_float_significand_bits;
+ if (!has_implicit_bit<Float>()) ++e;
+ return is_predecessor_closer;
+ }
+
+ template <typename Float, FMT_ENABLE_IF(is_double_double<Float>::value)>
+ FMT_CONSTEXPR auto assign(Float n) -> bool {
+ static_assert(std::numeric_limits<double>::is_iec559, "unsupported FP");
+ return assign(static_cast<double>(n));
+ }
+};
+
+using fp = basic_fp<unsigned long long>;
+
+// Normalizes the value converted from double and multiplied by (1 << SHIFT).
+template <int SHIFT = 0, typename F>
+FMT_CONSTEXPR basic_fp<F> normalize(basic_fp<F> value) {
+ // Handle subnormals.
+ const auto implicit_bit = F(1) << num_significand_bits<double>();
+ const auto shifted_implicit_bit = implicit_bit << SHIFT;
+ while ((value.f & shifted_implicit_bit) == 0) {
+ value.f <<= 1;
+ --value.e;
+ }
+ // Subtract 1 to account for hidden bit.
+ const auto offset = basic_fp<F>::num_significand_bits -
+ num_significand_bits<double>() - SHIFT - 1;
+ value.f <<= offset;
+ value.e -= offset;
+ return value;
+}
+
+// Computes lhs * rhs / pow(2, 64) rounded to nearest with half-up tie breaking.
+FMT_CONSTEXPR inline uint64_t multiply(uint64_t lhs, uint64_t rhs) {
+#if FMT_USE_INT128
+ auto product = static_cast<__uint128_t>(lhs) * rhs;
+ auto f = static_cast<uint64_t>(product >> 64);
+ return (static_cast<uint64_t>(product) & (1ULL << 63)) != 0 ? f + 1 : f;
+#else
+ // Multiply 32-bit parts of significands.
+ uint64_t mask = (1ULL << 32) - 1;
+ uint64_t a = lhs >> 32, b = lhs & mask;
+ uint64_t c = rhs >> 32, d = rhs & mask;
+ uint64_t ac = a * c, bc = b * c, ad = a * d, bd = b * d;
+ // Compute mid 64-bit of result and round.
+ uint64_t mid = (bd >> 32) + (ad & mask) + (bc & mask) + (1U << 31);
+ return ac + (ad >> 32) + (bc >> 32) + (mid >> 32);
+#endif
+}
+
+FMT_CONSTEXPR inline fp operator*(fp x, fp y) {
+ return {multiply(x.f, y.f), x.e + y.e + 64};
+}
+
+template <typename T = void> struct basic_data {
+ // Normalized 64-bit significands of pow(10, k), for k = -348, -340, ..., 340.
+ // These are generated by support/compute-powers.py.
+ static constexpr uint64_t pow10_significands[87] = {
+ 0xfa8fd5a0081c0288, 0xbaaee17fa23ebf76, 0x8b16fb203055ac76,
+ 0xcf42894a5dce35ea, 0x9a6bb0aa55653b2d, 0xe61acf033d1a45df,
+ 0xab70fe17c79ac6ca, 0xff77b1fcbebcdc4f, 0xbe5691ef416bd60c,
+ 0x8dd01fad907ffc3c, 0xd3515c2831559a83, 0x9d71ac8fada6c9b5,
+ 0xea9c227723ee8bcb, 0xaecc49914078536d, 0x823c12795db6ce57,
+ 0xc21094364dfb5637, 0x9096ea6f3848984f, 0xd77485cb25823ac7,
+ 0xa086cfcd97bf97f4, 0xef340a98172aace5, 0xb23867fb2a35b28e,
+ 0x84c8d4dfd2c63f3b, 0xc5dd44271ad3cdba, 0x936b9fcebb25c996,
+ 0xdbac6c247d62a584, 0xa3ab66580d5fdaf6, 0xf3e2f893dec3f126,
+ 0xb5b5ada8aaff80b8, 0x87625f056c7c4a8b, 0xc9bcff6034c13053,
+ 0x964e858c91ba2655, 0xdff9772470297ebd, 0xa6dfbd9fb8e5b88f,
+ 0xf8a95fcf88747d94, 0xb94470938fa89bcf, 0x8a08f0f8bf0f156b,
+ 0xcdb02555653131b6, 0x993fe2c6d07b7fac, 0xe45c10c42a2b3b06,
+ 0xaa242499697392d3, 0xfd87b5f28300ca0e, 0xbce5086492111aeb,
+ 0x8cbccc096f5088cc, 0xd1b71758e219652c, 0x9c40000000000000,
+ 0xe8d4a51000000000, 0xad78ebc5ac620000, 0x813f3978f8940984,
+ 0xc097ce7bc90715b3, 0x8f7e32ce7bea5c70, 0xd5d238a4abe98068,
+ 0x9f4f2726179a2245, 0xed63a231d4c4fb27, 0xb0de65388cc8ada8,
+ 0x83c7088e1aab65db, 0xc45d1df942711d9a, 0x924d692ca61be758,
+ 0xda01ee641a708dea, 0xa26da3999aef774a, 0xf209787bb47d6b85,
+ 0xb454e4a179dd1877, 0x865b86925b9bc5c2, 0xc83553c5c8965d3d,
+ 0x952ab45cfa97a0b3, 0xde469fbd99a05fe3, 0xa59bc234db398c25,
+ 0xf6c69a72a3989f5c, 0xb7dcbf5354e9bece, 0x88fcf317f22241e2,
+ 0xcc20ce9bd35c78a5, 0x98165af37b2153df, 0xe2a0b5dc971f303a,
+ 0xa8d9d1535ce3b396, 0xfb9b7cd9a4a7443c, 0xbb764c4ca7a44410,
+ 0x8bab8eefb6409c1a, 0xd01fef10a657842c, 0x9b10a4e5e9913129,
+ 0xe7109bfba19c0c9d, 0xac2820d9623bf429, 0x80444b5e7aa7cf85,
+ 0xbf21e44003acdd2d, 0x8e679c2f5e44ff8f, 0xd433179d9c8cb841,
+ 0x9e19db92b4e31ba9, 0xeb96bf6ebadf77d9, 0xaf87023b9bf0ee6b,
+ };
+
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wnarrowing"
+#endif
+ // Binary exponents of pow(10, k), for k = -348, -340, ..., 340, corresponding
+ // to significands above.
+ static constexpr int16_t pow10_exponents[87] = {
+ -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, -954,
+ -927, -901, -874, -847, -821, -794, -768, -741, -715, -688, -661,
+ -635, -608, -582, -555, -529, -502, -475, -449, -422, -396, -369,
+ -343, -316, -289, -263, -236, -210, -183, -157, -130, -103, -77,
+ -50, -24, 3, 30, 56, 83, 109, 136, 162, 189, 216,
+ 242, 269, 295, 322, 348, 375, 402, 428, 455, 481, 508,
+ 534, 561, 588, 614, 641, 667, 694, 720, 747, 774, 800,
+ 827, 853, 880, 907, 933, 960, 986, 1013, 1039, 1066};
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+# pragma GCC diagnostic pop
+#endif
+
+ static constexpr uint64_t power_of_10_64[20] = {
+ 1, FMT_POWERS_OF_10(1ULL), FMT_POWERS_OF_10(1000000000ULL),
+ 10000000000000000000ULL};
+
+ // For checking rounding thresholds.
+ // The kth entry is chosen to be the smallest integer such that the
+ // upper 32-bits of 10^(k+1) times it is strictly bigger than 5 * 10^k.
+ static constexpr uint32_t fractional_part_rounding_thresholds[8] = {
+ 2576980378, // ceil(2^31 + 2^32/10^1)
+ 2190433321, // ceil(2^31 + 2^32/10^2)
+ 2151778616, // ceil(2^31 + 2^32/10^3)
+ 2147913145, // ceil(2^31 + 2^32/10^4)
+ 2147526598, // ceil(2^31 + 2^32/10^5)
+ 2147487943, // ceil(2^31 + 2^32/10^6)
+ 2147484078, // ceil(2^31 + 2^32/10^7)
+ 2147483691 // ceil(2^31 + 2^32/10^8)
+ };
+};
+
+#if FMT_CPLUSPLUS < 201703L
+template <typename T> constexpr uint64_t basic_data<T>::pow10_significands[];
+template <typename T> constexpr int16_t basic_data<T>::pow10_exponents[];
+template <typename T> constexpr uint64_t basic_data<T>::power_of_10_64[];
+template <typename T>
+constexpr uint32_t basic_data<T>::fractional_part_rounding_thresholds[];
+#endif
+
+// This is a struct rather than an alias to avoid shadowing warnings in gcc.
+struct data : basic_data<> {};
+
+// Returns a cached power of 10 `c_k = c_k.f * pow(2, c_k.e)` such that its
+// (binary) exponent satisfies `min_exponent <= c_k.e <= min_exponent + 28`.
+FMT_CONSTEXPR inline fp get_cached_power(int min_exponent,
+ int& pow10_exponent) {
+ const int shift = 32;
+ // log10(2) = 0x0.4d104d427de7fbcc...
+ const int64_t significand = 0x4d104d427de7fbcc;
+ int index = static_cast<int>(
+ ((min_exponent + fp::num_significand_bits - 1) * (significand >> shift) +
+ ((int64_t(1) << shift) - 1)) // ceil
+ >> 32 // arithmetic shift
+ );
+ // Decimal exponent of the first (smallest) cached power of 10.
+ const int first_dec_exp = -348;
+ // Difference between 2 consecutive decimal exponents in cached powers of 10.
+ const int dec_exp_step = 8;
+ index = (index - first_dec_exp - 1) / dec_exp_step + 1;
+ pow10_exponent = first_dec_exp + index * dec_exp_step;
+ // Using *(x + index) instead of x[index] avoids an issue with some compilers
+ // using the EDG frontend (e.g. nvhpc/22.3 in C++17 mode).
+ return {*(data::pow10_significands + index),
+ *(data::pow10_exponents + index)};
+}
+
+template <typename T>
+using convert_float_result =
+ conditional_t<std::is_same<T, float>::value ||
+ std::numeric_limits<T>::digits ==
+ std::numeric_limits<double>::digits,
+ double, T>;
+
+template <typename T>
+constexpr auto convert_float(T value) -> convert_float_result<T> {
+ return static_cast<convert_float_result<T>>(value);
+}
+
+template <typename OutputIt, typename Char>
+FMT_NOINLINE FMT_CONSTEXPR auto fill(OutputIt it, size_t n,
+ const fill_t<Char>& fill) -> OutputIt {
+ auto fill_size = fill.size();
+ if (fill_size == 1) return detail::fill_n(it, n, fill[0]);
+ auto data = fill.data();
+ for (size_t i = 0; i < n; ++i)
+ it = copy_str<Char>(data, data + fill_size, it);
+ return it;
+}
+
+// Writes the output of f, padded according to format specifications in specs.
+// size: output size in code units.
+// width: output display width in (terminal) column positions.
+template <align::type align = align::left, typename OutputIt, typename Char,
+ typename F>
+FMT_CONSTEXPR auto write_padded(OutputIt out, const format_specs<Char>& specs,
+ size_t size, size_t width, F&& f) -> OutputIt {
+ static_assert(align == align::left || align == align::right, "");
+ unsigned spec_width = to_unsigned(specs.width);
+ size_t padding = spec_width > width ? spec_width - width : 0;
+ // Shifts are encoded as string literals because static constexpr is not
+ // supported in constexpr functions.
+ auto* shifts = align == align::left ? "\x1f\x1f\x00\x01" : "\x00\x1f\x00\x01";
+ size_t left_padding = padding >> shifts[specs.align];
+ size_t right_padding = padding - left_padding;
+ auto it = reserve(out, size + padding * specs.fill.size());
+ if (left_padding != 0) it = fill(it, left_padding, specs.fill);
+ it = f(it);
+ if (right_padding != 0) it = fill(it, right_padding, specs.fill);
+ return base_iterator(out, it);
+}
+
+template <align::type align = align::left, typename OutputIt, typename Char,
+ typename F>
+constexpr auto write_padded(OutputIt out, const format_specs<Char>& specs,
+ size_t size, F&& f) -> OutputIt {
+ return write_padded<align>(out, specs, size, size, f);
+}
+
+template <align::type align = align::left, typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write_bytes(OutputIt out, string_view bytes,
+ const format_specs<Char>& specs) -> OutputIt {
+ return write_padded<align>(
+ out, specs, bytes.size(), [bytes](reserve_iterator<OutputIt> it) {
+ const char* data = bytes.data();
+ return copy_str<Char>(data, data + bytes.size(), it);
+ });
+}
+
+template <typename Char, typename OutputIt, typename UIntPtr>
+auto write_ptr(OutputIt out, UIntPtr value, const format_specs<Char>* specs)
+ -> OutputIt {
+ int num_digits = count_digits<4>(value);
+ auto size = to_unsigned(num_digits) + size_t(2);
+ auto write = [=](reserve_iterator<OutputIt> it) {
+ *it++ = static_cast<Char>('0');
+ *it++ = static_cast<Char>('x');
+ return format_uint<4, Char>(it, value, num_digits);
+ };
+ return specs ? write_padded<align::right>(out, *specs, size, write)
+ : base_iterator(out, write(reserve(out, size)));
+}
+
+// Returns true iff the code point cp is printable.
+FMT_API auto is_printable(uint32_t cp) -> bool;
+
+inline auto needs_escape(uint32_t cp) -> bool {
+ return cp < 0x20 || cp == 0x7f || cp == '"' || cp == '\\' ||
+ !is_printable(cp);
+}
+
+template <typename Char> struct find_escape_result {
+ const Char* begin;
+ const Char* end;
+ uint32_t cp;
+};
+
+template <typename Char>
+using make_unsigned_char =
+ typename conditional_t<std::is_integral<Char>::value,
+ std::make_unsigned<Char>,
+ type_identity<uint32_t>>::type;
+
+template <typename Char>
+auto find_escape(const Char* begin, const Char* end)
+ -> find_escape_result<Char> {
+ for (; begin != end; ++begin) {
+ uint32_t cp = static_cast<make_unsigned_char<Char>>(*begin);
+ if (const_check(sizeof(Char) == 1) && cp >= 0x80) continue;
+ if (needs_escape(cp)) return {begin, begin + 1, cp};
+ }
+ return {begin, nullptr, 0};
+}
+
+inline auto find_escape(const char* begin, const char* end)
+ -> find_escape_result<char> {
+ if (!is_utf8()) return find_escape<char>(begin, end);
+ auto result = find_escape_result<char>{end, nullptr, 0};
+ for_each_codepoint(string_view(begin, to_unsigned(end - begin)),
+ [&](uint32_t cp, string_view sv) {
+ if (needs_escape(cp)) {
+ result = {sv.begin(), sv.end(), cp};
+ return false;
+ }
+ return true;
+ });
+ return result;
+}
+
+#define FMT_STRING_IMPL(s, base, explicit) \
+ [] { \
+ /* Use the hidden visibility as a workaround for a GCC bug (#1973). */ \
+ /* Use a macro-like name to avoid shadowing warnings. */ \
+ struct FMT_GCC_VISIBILITY_HIDDEN FMT_COMPILE_STRING : base { \
+ using char_type FMT_MAYBE_UNUSED = fmt::remove_cvref_t<decltype(s[0])>; \
+ FMT_MAYBE_UNUSED FMT_CONSTEXPR explicit \
+ operator fmt::basic_string_view<char_type>() const { \
+ return fmt::detail_exported::compile_string_to_view<char_type>(s); \
+ } \
+ }; \
+ return FMT_COMPILE_STRING(); \
+ }()
+
+/**
+ \rst
+ Constructs a compile-time format string from a string literal *s*.
+
+ **Example**::
+
+ // A compile-time error because 'd' is an invalid specifier for strings.
+ std::string s = fmt::format(FMT_STRING("{:d}"), "foo");
+ \endrst
+ */
+#define FMT_STRING(s) FMT_STRING_IMPL(s, fmt::detail::compile_string, )
+
+template <size_t width, typename Char, typename OutputIt>
+auto write_codepoint(OutputIt out, char prefix, uint32_t cp) -> OutputIt {
+ *out++ = static_cast<Char>('\\');
+ *out++ = static_cast<Char>(prefix);
+ Char buf[width];
+ fill_n(buf, width, static_cast<Char>('0'));
+ format_uint<4>(buf, cp, width);
+ return copy_str<Char>(buf, buf + width, out);
+}
+
+template <typename OutputIt, typename Char>
+auto write_escaped_cp(OutputIt out, const find_escape_result<Char>& escape)
+ -> OutputIt {
+ auto c = static_cast<Char>(escape.cp);
+ switch (escape.cp) {
+ case '\n':
+ *out++ = static_cast<Char>('\\');
+ c = static_cast<Char>('n');
+ break;
+ case '\r':
+ *out++ = static_cast<Char>('\\');
+ c = static_cast<Char>('r');
+ break;
+ case '\t':
+ *out++ = static_cast<Char>('\\');
+ c = static_cast<Char>('t');
+ break;
+ case '"':
+ FMT_FALLTHROUGH;
+ case '\'':
+ FMT_FALLTHROUGH;
+ case '\\':
+ *out++ = static_cast<Char>('\\');
+ break;
+ default:
+ if (escape.cp < 0x100) {
+ return write_codepoint<2, Char>(out, 'x', escape.cp);
+ }
+ if (escape.cp < 0x10000) {
+ return write_codepoint<4, Char>(out, 'u', escape.cp);
+ }
+ if (escape.cp < 0x110000) {
+ return write_codepoint<8, Char>(out, 'U', escape.cp);
+ }
+ for (Char escape_char : basic_string_view<Char>(
+ escape.begin, to_unsigned(escape.end - escape.begin))) {
+ out = write_codepoint<2, Char>(out, 'x',
+ static_cast<uint32_t>(escape_char) & 0xFF);
+ }
+ return out;
+ }
+ *out++ = c;
+ return out;
+}
+
+template <typename Char, typename OutputIt>
+auto write_escaped_string(OutputIt out, basic_string_view<Char> str)
+ -> OutputIt {
+ *out++ = static_cast<Char>('"');
+ auto begin = str.begin(), end = str.end();
+ do {
+ auto escape = find_escape(begin, end);
+ out = copy_str<Char>(begin, escape.begin, out);
+ begin = escape.end;
+ if (!begin) break;
+ out = write_escaped_cp<OutputIt, Char>(out, escape);
+ } while (begin != end);
+ *out++ = static_cast<Char>('"');
+ return out;
+}
+
+template <typename Char, typename OutputIt>
+auto write_escaped_char(OutputIt out, Char v) -> OutputIt {
+ *out++ = static_cast<Char>('\'');
+ if ((needs_escape(static_cast<uint32_t>(v)) && v != static_cast<Char>('"')) ||
+ v == static_cast<Char>('\'')) {
+ out = write_escaped_cp(
+ out, find_escape_result<Char>{&v, &v + 1, static_cast<uint32_t>(v)});
+ } else {
+ *out++ = v;
+ }
+ *out++ = static_cast<Char>('\'');
+ return out;
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write_char(OutputIt out, Char value,
+ const format_specs<Char>& specs) -> OutputIt {
+ bool is_debug = specs.type == presentation_type::debug;
+ return write_padded(out, specs, 1, [=](reserve_iterator<OutputIt> it) {
+ if (is_debug) return write_escaped_char(it, value);
+ *it++ = value;
+ return it;
+ });
+}
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, Char value,
+ const format_specs<Char>& specs, locale_ref loc = {})
+ -> OutputIt {
+ // char is formatted as unsigned char for consistency across platforms.
+ using unsigned_type =
+ conditional_t<std::is_same<Char, char>::value, unsigned char, unsigned>;
+ return check_char_specs(specs)
+ ? write_char(out, value, specs)
+ : write(out, static_cast<unsigned_type>(value), specs, loc);
+}
+
+// Data for write_int that doesn't depend on output iterator type. It is used to
+// avoid template code bloat.
+template <typename Char> struct write_int_data {
+ size_t size;
+ size_t padding;
+
+ FMT_CONSTEXPR write_int_data(int num_digits, unsigned prefix,
+ const format_specs<Char>& specs)
+ : size((prefix >> 24) + to_unsigned(num_digits)), padding(0) {
+ if (specs.align == align::numeric) {
+ auto width = to_unsigned(specs.width);
+ if (width > size) {
+ padding = width - size;
+ size = width;
+ }
+ } else if (specs.precision > num_digits) {
+ size = (prefix >> 24) + to_unsigned(specs.precision);
+ padding = to_unsigned(specs.precision - num_digits);
+ }
+ }
+};
+
+// Writes an integer in the format
+// <left-padding><prefix><numeric-padding><digits><right-padding>
+// where <digits> are written by write_digits(it).
+// prefix contains chars in three lower bytes and the size in the fourth byte.
+template <typename OutputIt, typename Char, typename W>
+FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, int num_digits,
+ unsigned prefix,
+ const format_specs<Char>& specs,
+ W write_digits) -> OutputIt {
+ // Slightly faster check for specs.width == 0 && specs.precision == -1.
+ if ((specs.width | (specs.precision + 1)) == 0) {
+ auto it = reserve(out, to_unsigned(num_digits) + (prefix >> 24));
+ if (prefix != 0) {
+ for (unsigned p = prefix & 0xffffff; p != 0; p >>= 8)
+ *it++ = static_cast<Char>(p & 0xff);
+ }
+ return base_iterator(out, write_digits(it));
+ }
+ auto data = write_int_data<Char>(num_digits, prefix, specs);
+ return write_padded<align::right>(
+ out, specs, data.size, [=](reserve_iterator<OutputIt> it) {
+ for (unsigned p = prefix & 0xffffff; p != 0; p >>= 8)
+ *it++ = static_cast<Char>(p & 0xff);
+ it = detail::fill_n(it, data.padding, static_cast<Char>('0'));
+ return write_digits(it);
+ });
+}
+
+template <typename Char> class digit_grouping {
+ private:
+ std::string grouping_;
+ std::basic_string<Char> thousands_sep_;
+
+ struct next_state {
+ std::string::const_iterator group;
+ int pos;
+ };
+ next_state initial_state() const { return {grouping_.begin(), 0}; }
+
+ // Returns the next digit group separator position.
+ int next(next_state& state) const {
+ if (thousands_sep_.empty()) return max_value<int>();
+ if (state.group == grouping_.end()) return state.pos += grouping_.back();
+ if (*state.group <= 0 || *state.group == max_value<char>())
+ return max_value<int>();
+ state.pos += *state.group++;
+ return state.pos;
+ }
+
+ public:
+ explicit digit_grouping(locale_ref loc, bool localized = true) {
+ if (!localized) return;
+ auto sep = thousands_sep<Char>(loc);
+ grouping_ = sep.grouping;
+ if (sep.thousands_sep) thousands_sep_.assign(1, sep.thousands_sep);
+ }
+ digit_grouping(std::string grouping, std::basic_string<Char> sep)
+ : grouping_(std::move(grouping)), thousands_sep_(std::move(sep)) {}
+
+ bool has_separator() const { return !thousands_sep_.empty(); }
+
+ int count_separators(int num_digits) const {
+ int count = 0;
+ auto state = initial_state();
+ while (num_digits > next(state)) ++count;
+ return count;
+ }
+
+ // Applies grouping to digits and write the output to out.
+ template <typename Out, typename C>
+ Out apply(Out out, basic_string_view<C> digits) const {
+ auto num_digits = static_cast<int>(digits.size());
+ auto separators = basic_memory_buffer<int>();
+ separators.push_back(0);
+ auto state = initial_state();
+ while (int i = next(state)) {
+ if (i >= num_digits) break;
+ separators.push_back(i);
+ }
+ for (int i = 0, sep_index = static_cast<int>(separators.size() - 1);
+ i < num_digits; ++i) {
+ if (num_digits - i == separators[sep_index]) {
+ out =
+ copy_str<Char>(thousands_sep_.data(),
+ thousands_sep_.data() + thousands_sep_.size(), out);
+ --sep_index;
+ }
+ *out++ = static_cast<Char>(digits[to_unsigned(i)]);
+ }
+ return out;
+ }
+};
+
+// Writes a decimal integer with digit grouping.
+template <typename OutputIt, typename UInt, typename Char>
+auto write_int(OutputIt out, UInt value, unsigned prefix,
+ const format_specs<Char>& specs,
+ const digit_grouping<Char>& grouping) -> OutputIt {
+ static_assert(std::is_same<uint64_or_128_t<UInt>, UInt>::value, "");
+ int num_digits = count_digits(value);
+ char digits[40];
+ format_decimal(digits, value, num_digits);
+ unsigned size = to_unsigned((prefix != 0 ? 1 : 0) + num_digits +
+ grouping.count_separators(num_digits));
+ return write_padded<align::right>(
+ out, specs, size, size, [&](reserve_iterator<OutputIt> it) {
+ if (prefix != 0) {
+ char sign = static_cast<char>(prefix);
+ *it++ = static_cast<Char>(sign);
+ }
+ return grouping.apply(it, string_view(digits, to_unsigned(num_digits)));
+ });
+}
+
+// Writes a localized value.
+FMT_API auto write_loc(appender out, loc_value value,
+ const format_specs<>& specs, locale_ref loc) -> bool;
+template <typename OutputIt, typename Char>
+inline auto write_loc(OutputIt, loc_value, const format_specs<Char>&,
+ locale_ref) -> bool {
+ return false;
+}
+
+FMT_CONSTEXPR inline void prefix_append(unsigned& prefix, unsigned value) {
+ prefix |= prefix != 0 ? value << 8 : value;
+ prefix += (1u + (value > 0xff ? 1 : 0)) << 24;
+}
+
+template <typename UInt> struct write_int_arg {
+ UInt abs_value;
+ unsigned prefix;
+};
+
+template <typename T>
+FMT_CONSTEXPR auto make_write_int_arg(T value, sign_t sign)
+ -> write_int_arg<uint32_or_64_or_128_t<T>> {
+ auto prefix = 0u;
+ auto abs_value = static_cast<uint32_or_64_or_128_t<T>>(value);
+ if (is_negative(value)) {
+ prefix = 0x01000000 | '-';
+ abs_value = 0 - abs_value;
+ } else {
+ constexpr const unsigned prefixes[4] = {0, 0, 0x1000000u | '+',
+ 0x1000000u | ' '};
+ prefix = prefixes[sign];
+ }
+ return {abs_value, prefix};
+}
+
+template <typename Char = char> struct loc_writer {
+ buffer_appender<Char> out;
+ const format_specs<Char>& specs;
+ std::basic_string<Char> sep;
+ std::string grouping;
+ std::basic_string<Char> decimal_point;
+
+ template <typename T, FMT_ENABLE_IF(is_integer<T>::value)>
+ auto operator()(T value) -> bool {
+ auto arg = make_write_int_arg(value, specs.sign);
+ write_int(out, static_cast<uint64_or_128_t<T>>(arg.abs_value), arg.prefix,
+ specs, digit_grouping<Char>(grouping, sep));
+ return true;
+ }
+
+ template <typename T, FMT_ENABLE_IF(!is_integer<T>::value)>
+ auto operator()(T) -> bool {
+ return false;
+ }
+};
+
+template <typename Char, typename OutputIt, typename T>
+FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, write_int_arg<T> arg,
+ const format_specs<Char>& specs,
+ locale_ref) -> OutputIt {
+ static_assert(std::is_same<T, uint32_or_64_or_128_t<T>>::value, "");
+ auto abs_value = arg.abs_value;
+ auto prefix = arg.prefix;
+ switch (specs.type) {
+ case presentation_type::none:
+ case presentation_type::dec: {
+ auto num_digits = count_digits(abs_value);
+ return write_int(
+ out, num_digits, prefix, specs, [=](reserve_iterator<OutputIt> it) {
+ return format_decimal<Char>(it, abs_value, num_digits).end;
+ });
+ }
+ case presentation_type::hex_lower:
+ case presentation_type::hex_upper: {
+ bool upper = specs.type == presentation_type::hex_upper;
+ if (specs.alt)
+ prefix_append(prefix, unsigned(upper ? 'X' : 'x') << 8 | '0');
+ int num_digits = count_digits<4>(abs_value);
+ return write_int(
+ out, num_digits, prefix, specs, [=](reserve_iterator<OutputIt> it) {
+ return format_uint<4, Char>(it, abs_value, num_digits, upper);
+ });
+ }
+ case presentation_type::bin_lower:
+ case presentation_type::bin_upper: {
+ bool upper = specs.type == presentation_type::bin_upper;
+ if (specs.alt)
+ prefix_append(prefix, unsigned(upper ? 'B' : 'b') << 8 | '0');
+ int num_digits = count_digits<1>(abs_value);
+ return write_int(out, num_digits, prefix, specs,
+ [=](reserve_iterator<OutputIt> it) {
+ return format_uint<1, Char>(it, abs_value, num_digits);
+ });
+ }
+ case presentation_type::oct: {
+ int num_digits = count_digits<3>(abs_value);
+ // Octal prefix '0' is counted as a digit, so only add it if precision
+ // is not greater than the number of digits.
+ if (specs.alt && specs.precision <= num_digits && abs_value != 0)
+ prefix_append(prefix, '0');
+ return write_int(out, num_digits, prefix, specs,
+ [=](reserve_iterator<OutputIt> it) {
+ return format_uint<3, Char>(it, abs_value, num_digits);
+ });
+ }
+ case presentation_type::chr:
+ return write_char(out, static_cast<Char>(abs_value), specs);
+ default:
+ throw_format_error("invalid format specifier");
+ }
+ return out;
+}
+template <typename Char, typename OutputIt, typename T>
+FMT_CONSTEXPR FMT_NOINLINE auto write_int_noinline(
+ OutputIt out, write_int_arg<T> arg, const format_specs<Char>& specs,
+ locale_ref loc) -> OutputIt {
+ return write_int(out, arg, specs, loc);
+}
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_integral<T>::value &&
+ !std::is_same<T, bool>::value &&
+ std::is_same<OutputIt, buffer_appender<Char>>::value)>
+FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value,
+ const format_specs<Char>& specs,
+ locale_ref loc) -> OutputIt {
+ if (specs.localized && write_loc(out, value, specs, loc)) return out;
+ return write_int_noinline(out, make_write_int_arg(value, specs.sign), specs,
+ loc);
+}
+// An inlined version of write used in format string compilation.
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_integral<T>::value &&
+ !std::is_same<T, bool>::value &&
+ !std::is_same<OutputIt, buffer_appender<Char>>::value)>
+FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value,
+ const format_specs<Char>& specs,
+ locale_ref loc) -> OutputIt {
+ if (specs.localized && write_loc(out, value, specs, loc)) return out;
+ return write_int(out, make_write_int_arg(value, specs.sign), specs, loc);
+}
+
+// An output iterator that counts the number of objects written to it and
+// discards them.
+class counting_iterator {
+ private:
+ size_t count_;
+
+ public:
+ using iterator_category = std::output_iterator_tag;
+ using difference_type = std::ptrdiff_t;
+ using pointer = void;
+ using reference = void;
+ FMT_UNCHECKED_ITERATOR(counting_iterator);
+
+ struct value_type {
+ template <typename T> FMT_CONSTEXPR void operator=(const T&) {}
+ };
+
+ FMT_CONSTEXPR counting_iterator() : count_(0) {}
+
+ FMT_CONSTEXPR size_t count() const { return count_; }
+
+ FMT_CONSTEXPR counting_iterator& operator++() {
+ ++count_;
+ return *this;
+ }
+ FMT_CONSTEXPR counting_iterator operator++(int) {
+ auto it = *this;
+ ++*this;
+ return it;
+ }
+
+ FMT_CONSTEXPR friend counting_iterator operator+(counting_iterator it,
+ difference_type n) {
+ it.count_ += static_cast<size_t>(n);
+ return it;
+ }
+
+ FMT_CONSTEXPR value_type operator*() const { return {}; }
+};
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, basic_string_view<Char> s,
+ const format_specs<Char>& specs) -> OutputIt {
+ auto data = s.data();
+ auto size = s.size();
+ if (specs.precision >= 0 && to_unsigned(specs.precision) < size)
+ size = code_point_index(s, to_unsigned(specs.precision));
+ bool is_debug = specs.type == presentation_type::debug;
+ size_t width = 0;
+ if (specs.width != 0) {
+ if (is_debug)
+ width = write_escaped_string(counting_iterator{}, s).count();
+ else
+ width = compute_width(basic_string_view<Char>(data, size));
+ }
+ return write_padded(out, specs, size, width,
+ [=](reserve_iterator<OutputIt> it) {
+ if (is_debug) return write_escaped_string(it, s);
+ return copy_str<Char>(data, data + size, it);
+ });
+}
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out,
+ basic_string_view<type_identity_t<Char>> s,
+ const format_specs<Char>& specs, locale_ref)
+ -> OutputIt {
+ return write(out, s, specs);
+}
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, const Char* s,
+ const format_specs<Char>& specs, locale_ref)
+ -> OutputIt {
+ return specs.type != presentation_type::pointer
+ ? write(out, basic_string_view<Char>(s), specs, {})
+ : write_ptr<Char>(out, bit_cast<uintptr_t>(s), &specs);
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_integral<T>::value &&
+ !std::is_same<T, bool>::value &&
+ !std::is_same<T, Char>::value)>
+FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt {
+ auto abs_value = static_cast<uint32_or_64_or_128_t<T>>(value);
+ bool negative = is_negative(value);
+ // Don't do -abs_value since it trips unsigned-integer-overflow sanitizer.
+ if (negative) abs_value = ~abs_value + 1;
+ int num_digits = count_digits(abs_value);
+ auto size = (negative ? 1 : 0) + static_cast<size_t>(num_digits);
+ auto it = reserve(out, size);
+ if (auto ptr = to_pointer<Char>(it, size)) {
+ if (negative) *ptr++ = static_cast<Char>('-');
+ format_decimal<Char>(ptr, abs_value, num_digits);
+ return out;
+ }
+ if (negative) *it++ = static_cast<Char>('-');
+ it = format_decimal<Char>(it, abs_value, num_digits).end;
+ return base_iterator(out, it);
+}
+
+// A floating-point presentation format.
+enum class float_format : unsigned char {
+ general, // General: exponent notation or fixed point based on magnitude.
+ exp, // Exponent notation with the default precision of 6, e.g. 1.2e-3.
+ fixed, // Fixed point with the default precision of 6, e.g. 0.0012.
+ hex
+};
+
+struct float_specs {
+ int precision;
+ float_format format : 8;
+ sign_t sign : 8;
+ bool upper : 1;
+ bool locale : 1;
+ bool binary32 : 1;
+ bool showpoint : 1;
+};
+
+template <typename ErrorHandler = error_handler, typename Char>
+FMT_CONSTEXPR auto parse_float_type_spec(const format_specs<Char>& specs,
+ ErrorHandler&& eh = {})
+ -> float_specs {
+ auto result = float_specs();
+ result.showpoint = specs.alt;
+ result.locale = specs.localized;
+ switch (specs.type) {
+ case presentation_type::none:
+ result.format = float_format::general;
+ break;
+ case presentation_type::general_upper:
+ result.upper = true;
+ FMT_FALLTHROUGH;
+ case presentation_type::general_lower:
+ result.format = float_format::general;
+ break;
+ case presentation_type::exp_upper:
+ result.upper = true;
+ FMT_FALLTHROUGH;
+ case presentation_type::exp_lower:
+ result.format = float_format::exp;
+ result.showpoint |= specs.precision != 0;
+ break;
+ case presentation_type::fixed_upper:
+ result.upper = true;
+ FMT_FALLTHROUGH;
+ case presentation_type::fixed_lower:
+ result.format = float_format::fixed;
+ result.showpoint |= specs.precision != 0;
+ break;
+ case presentation_type::hexfloat_upper:
+ result.upper = true;
+ FMT_FALLTHROUGH;
+ case presentation_type::hexfloat_lower:
+ result.format = float_format::hex;
+ break;
+ default:
+ eh.on_error("invalid format specifier");
+ break;
+ }
+ return result;
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR20 auto write_nonfinite(OutputIt out, bool isnan,
+ format_specs<Char> specs,
+ const float_specs& fspecs) -> OutputIt {
+ auto str =
+ isnan ? (fspecs.upper ? "NAN" : "nan") : (fspecs.upper ? "INF" : "inf");
+ constexpr size_t str_size = 3;
+ auto sign = fspecs.sign;
+ auto size = str_size + (sign ? 1 : 0);
+ // Replace '0'-padding with space for non-finite values.
+ const bool is_zero_fill =
+ specs.fill.size() == 1 && *specs.fill.data() == static_cast<Char>('0');
+ if (is_zero_fill) specs.fill[0] = static_cast<Char>(' ');
+ return write_padded(out, specs, size, [=](reserve_iterator<OutputIt> it) {
+ if (sign) *it++ = detail::sign<Char>(sign);
+ return copy_str<Char>(str, str + str_size, it);
+ });
+}
+
+// A decimal floating-point number significand * pow(10, exp).
+struct big_decimal_fp {
+ const char* significand;
+ int significand_size;
+ int exponent;
+};
+
+constexpr auto get_significand_size(const big_decimal_fp& f) -> int {
+ return f.significand_size;
+}
+template <typename T>
+inline auto get_significand_size(const dragonbox::decimal_fp<T>& f) -> int {
+ return count_digits(f.significand);
+}
+
+template <typename Char, typename OutputIt>
+constexpr auto write_significand(OutputIt out, const char* significand,
+ int significand_size) -> OutputIt {
+ return copy_str<Char>(significand, significand + significand_size, out);
+}
+template <typename Char, typename OutputIt, typename UInt>
+inline auto write_significand(OutputIt out, UInt significand,
+ int significand_size) -> OutputIt {
+ return format_decimal<Char>(out, significand, significand_size).end;
+}
+template <typename Char, typename OutputIt, typename T, typename Grouping>
+FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand,
+ int significand_size, int exponent,
+ const Grouping& grouping) -> OutputIt {
+ if (!grouping.has_separator()) {
+ out = write_significand<Char>(out, significand, significand_size);
+ return detail::fill_n(out, exponent, static_cast<Char>('0'));
+ }
+ auto buffer = memory_buffer();
+ write_significand<char>(appender(buffer), significand, significand_size);
+ detail::fill_n(appender(buffer), exponent, '0');
+ return grouping.apply(out, string_view(buffer.data(), buffer.size()));
+}
+
+template <typename Char, typename UInt,
+ FMT_ENABLE_IF(std::is_integral<UInt>::value)>
+inline auto write_significand(Char* out, UInt significand, int significand_size,
+ int integral_size, Char decimal_point) -> Char* {
+ if (!decimal_point)
+ return format_decimal(out, significand, significand_size).end;
+ out += significand_size + 1;
+ Char* end = out;
+ int floating_size = significand_size - integral_size;
+ for (int i = floating_size / 2; i > 0; --i) {
+ out -= 2;
+ copy2(out, digits2(static_cast<std::size_t>(significand % 100)));
+ significand /= 100;
+ }
+ if (floating_size % 2 != 0) {
+ *--out = static_cast<Char>('0' + significand % 10);
+ significand /= 10;
+ }
+ *--out = decimal_point;
+ format_decimal(out - integral_size, significand, integral_size);
+ return end;
+}
+
+template <typename OutputIt, typename UInt, typename Char,
+ FMT_ENABLE_IF(!std::is_pointer<remove_cvref_t<OutputIt>>::value)>
+inline auto write_significand(OutputIt out, UInt significand,
+ int significand_size, int integral_size,
+ Char decimal_point) -> OutputIt {
+ // Buffer is large enough to hold digits (digits10 + 1) and a decimal point.
+ Char buffer[digits10<UInt>() + 2];
+ auto end = write_significand(buffer, significand, significand_size,
+ integral_size, decimal_point);
+ return detail::copy_str_noinline<Char>(buffer, end, out);
+}
+
+template <typename OutputIt, typename Char>
+FMT_CONSTEXPR auto write_significand(OutputIt out, const char* significand,
+ int significand_size, int integral_size,
+ Char decimal_point) -> OutputIt {
+ out = detail::copy_str_noinline<Char>(significand,
+ significand + integral_size, out);
+ if (!decimal_point) return out;
+ *out++ = decimal_point;
+ return detail::copy_str_noinline<Char>(significand + integral_size,
+ significand + significand_size, out);
+}
+
+template <typename OutputIt, typename Char, typename T, typename Grouping>
+FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand,
+ int significand_size, int integral_size,
+ Char decimal_point,
+ const Grouping& grouping) -> OutputIt {
+ if (!grouping.has_separator()) {
+ return write_significand(out, significand, significand_size, integral_size,
+ decimal_point);
+ }
+ auto buffer = basic_memory_buffer<Char>();
+ write_significand(buffer_appender<Char>(buffer), significand,
+ significand_size, integral_size, decimal_point);
+ grouping.apply(
+ out, basic_string_view<Char>(buffer.data(), to_unsigned(integral_size)));
+ return detail::copy_str_noinline<Char>(buffer.data() + integral_size,
+ buffer.end(), out);
+}
+
+template <typename OutputIt, typename DecimalFP, typename Char,
+ typename Grouping = digit_grouping<Char>>
+FMT_CONSTEXPR20 auto do_write_float(OutputIt out, const DecimalFP& f,
+ const format_specs<Char>& specs,
+ float_specs fspecs, locale_ref loc)
+ -> OutputIt {
+ auto significand = f.significand;
+ int significand_size = get_significand_size(f);
+ const Char zero = static_cast<Char>('0');
+ auto sign = fspecs.sign;
+ size_t size = to_unsigned(significand_size) + (sign ? 1 : 0);
+ using iterator = reserve_iterator<OutputIt>;
+
+ Char decimal_point =
+ fspecs.locale ? detail::decimal_point<Char>(loc) : static_cast<Char>('.');
+
+ int output_exp = f.exponent + significand_size - 1;
+ auto use_exp_format = [=]() {
+ if (fspecs.format == float_format::exp) return true;
+ if (fspecs.format != float_format::general) return false;
+ // Use the fixed notation if the exponent is in [exp_lower, exp_upper),
+ // e.g. 0.0001 instead of 1e-04. Otherwise use the exponent notation.
+ const int exp_lower = -4, exp_upper = 16;
+ return output_exp < exp_lower ||
+ output_exp >= (fspecs.precision > 0 ? fspecs.precision : exp_upper);
+ };
+ if (use_exp_format()) {
+ int num_zeros = 0;
+ if (fspecs.showpoint) {
+ num_zeros = fspecs.precision - significand_size;
+ if (num_zeros < 0) num_zeros = 0;
+ size += to_unsigned(num_zeros);
+ } else if (significand_size == 1) {
+ decimal_point = Char();
+ }
+ auto abs_output_exp = output_exp >= 0 ? output_exp : -output_exp;
+ int exp_digits = 2;
+ if (abs_output_exp >= 100) exp_digits = abs_output_exp >= 1000 ? 4 : 3;
+
+ size += to_unsigned((decimal_point ? 1 : 0) + 2 + exp_digits);
+ char exp_char = fspecs.upper ? 'E' : 'e';
+ auto write = [=](iterator it) {
+ if (sign) *it++ = detail::sign<Char>(sign);
+ // Insert a decimal point after the first digit and add an exponent.
+ it = write_significand(it, significand, significand_size, 1,
+ decimal_point);
+ if (num_zeros > 0) it = detail::fill_n(it, num_zeros, zero);
+ *it++ = static_cast<Char>(exp_char);
+ return write_exponent<Char>(output_exp, it);
+ };
+ return specs.width > 0 ? write_padded<align::right>(out, specs, size, write)
+ : base_iterator(out, write(reserve(out, size)));
+ }
+
+ int exp = f.exponent + significand_size;
+ if (f.exponent >= 0) {
+ // 1234e5 -> 123400000[.0+]
+ size += to_unsigned(f.exponent);
+ int num_zeros = fspecs.precision - exp;
+ abort_fuzzing_if(num_zeros > 5000);
+ if (fspecs.showpoint) {
+ ++size;
+ if (num_zeros <= 0 && fspecs.format != float_format::fixed) num_zeros = 0;
+ if (num_zeros > 0) size += to_unsigned(num_zeros);
+ }
+ auto grouping = Grouping(loc, fspecs.locale);
+ size += to_unsigned(grouping.count_separators(exp));
+ return write_padded<align::right>(out, specs, size, [&](iterator it) {
+ if (sign) *it++ = detail::sign<Char>(sign);
+ it = write_significand<Char>(it, significand, significand_size,
+ f.exponent, grouping);
+ if (!fspecs.showpoint) return it;
+ *it++ = decimal_point;
+ return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it;
+ });
+ } else if (exp > 0) {
+ // 1234e-2 -> 12.34[0+]
+ int num_zeros = fspecs.showpoint ? fspecs.precision - significand_size : 0;
+ size += 1 + to_unsigned(num_zeros > 0 ? num_zeros : 0);
+ auto grouping = Grouping(loc, fspecs.locale);
+ size += to_unsigned(grouping.count_separators(exp));
+ return write_padded<align::right>(out, specs, size, [&](iterator it) {
+ if (sign) *it++ = detail::sign<Char>(sign);
+ it = write_significand(it, significand, significand_size, exp,
+ decimal_point, grouping);
+ return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it;
+ });
+ }
+ // 1234e-6 -> 0.001234
+ int num_zeros = -exp;
+ if (significand_size == 0 && fspecs.precision >= 0 &&
+ fspecs.precision < num_zeros) {
+ num_zeros = fspecs.precision;
+ }
+ bool pointy = num_zeros != 0 || significand_size != 0 || fspecs.showpoint;
+ size += 1 + (pointy ? 1 : 0) + to_unsigned(num_zeros);
+ return write_padded<align::right>(out, specs, size, [&](iterator it) {
+ if (sign) *it++ = detail::sign<Char>(sign);
+ *it++ = zero;
+ if (!pointy) return it;
+ *it++ = decimal_point;
+ it = detail::fill_n(it, num_zeros, zero);
+ return write_significand<Char>(it, significand, significand_size);
+ });
+}
+
+template <typename Char> class fallback_digit_grouping {
+ public:
+ constexpr fallback_digit_grouping(locale_ref, bool) {}
+
+ constexpr bool has_separator() const { return false; }
+
+ constexpr int count_separators(int) const { return 0; }
+
+ template <typename Out, typename C>
+ constexpr Out apply(Out out, basic_string_view<C>) const {
+ return out;
+ }
+};
+
+template <typename OutputIt, typename DecimalFP, typename Char>
+FMT_CONSTEXPR20 auto write_float(OutputIt out, const DecimalFP& f,
+ const format_specs<Char>& specs,
+ float_specs fspecs, locale_ref loc)
+ -> OutputIt {
+ if (is_constant_evaluated()) {
+ return do_write_float<OutputIt, DecimalFP, Char,
+ fallback_digit_grouping<Char>>(out, f, specs, fspecs,
+ loc);
+ } else {
+ return do_write_float(out, f, specs, fspecs, loc);
+ }
+}
+
+template <typename T> constexpr bool isnan(T value) {
+ return !(value >= value); // std::isnan doesn't support __float128.
+}
+
+template <typename T, typename Enable = void>
+struct has_isfinite : std::false_type {};
+
+template <typename T>
+struct has_isfinite<T, enable_if_t<sizeof(std::isfinite(T())) != 0>>
+ : std::true_type {};
+
+template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value&&
+ has_isfinite<T>::value)>
+FMT_CONSTEXPR20 bool isfinite(T value) {
+ constexpr T inf = T(std::numeric_limits<double>::infinity());
+ if (is_constant_evaluated())
+ return !detail::isnan(value) && value < inf && value > -inf;
+ return std::isfinite(value);
+}
+template <typename T, FMT_ENABLE_IF(!has_isfinite<T>::value)>
+FMT_CONSTEXPR bool isfinite(T value) {
+ T inf = T(std::numeric_limits<double>::infinity());
+ // std::isfinite doesn't support __float128.
+ return !detail::isnan(value) && value < inf && value > -inf;
+}
+
+template <typename T, FMT_ENABLE_IF(is_floating_point<T>::value)>
+FMT_INLINE FMT_CONSTEXPR bool signbit(T value) {
+ if (is_constant_evaluated()) {
+#ifdef __cpp_if_constexpr
+ if constexpr (std::numeric_limits<double>::is_iec559) {
+ auto bits = detail::bit_cast<uint64_t>(static_cast<double>(value));
+ return (bits >> (num_bits<uint64_t>() - 1)) != 0;
+ }
+#endif
+ }
+ return std::signbit(static_cast<double>(value));
+}
+
+enum class round_direction { unknown, up, down };
+
+// Given the divisor (normally a power of 10), the remainder = v % divisor for
+// some number v and the error, returns whether v should be rounded up, down, or
+// whether the rounding direction can't be determined due to error.
+// error should be less than divisor / 2.
+FMT_CONSTEXPR inline round_direction get_round_direction(uint64_t divisor,
+ uint64_t remainder,
+ uint64_t error) {
+ FMT_ASSERT(remainder < divisor, ""); // divisor - remainder won't overflow.
+ FMT_ASSERT(error < divisor, ""); // divisor - error won't overflow.
+ FMT_ASSERT(error < divisor - error, ""); // error * 2 won't overflow.
+ // Round down if (remainder + error) * 2 <= divisor.
+ if (remainder <= divisor - remainder && error * 2 <= divisor - remainder * 2)
+ return round_direction::down;
+ // Round up if (remainder - error) * 2 >= divisor.
+ if (remainder >= error &&
+ remainder - error >= divisor - (remainder - error)) {
+ return round_direction::up;
+ }
+ return round_direction::unknown;
+}
+
+namespace digits {
+enum result {
+ more, // Generate more digits.
+ done, // Done generating digits.
+ error // Digit generation cancelled due to an error.
+};
+}
+
+struct gen_digits_handler {
+ char* buf;
+ int size;
+ int precision;
+ int exp10;
+ bool fixed;
+
+ FMT_CONSTEXPR digits::result on_digit(char digit, uint64_t divisor,
+ uint64_t remainder, uint64_t error,
+ bool integral) {
+ FMT_ASSERT(remainder < divisor, "");
+ buf[size++] = digit;
+ if (!integral && error >= remainder) return digits::error;
+ if (size < precision) return digits::more;
+ if (!integral) {
+ // Check if error * 2 < divisor with overflow prevention.
+ // The check is not needed for the integral part because error = 1
+ // and divisor > (1 << 32) there.
+ if (error >= divisor || error >= divisor - error) return digits::error;
+ } else {
+ FMT_ASSERT(error == 1 && divisor > 2, "");
+ }
+ auto dir = get_round_direction(divisor, remainder, error);
+ if (dir != round_direction::up)
+ return dir == round_direction::down ? digits::done : digits::error;
+ ++buf[size - 1];
+ for (int i = size - 1; i > 0 && buf[i] > '9'; --i) {
+ buf[i] = '0';
+ ++buf[i - 1];
+ }
+ if (buf[0] > '9') {
+ buf[0] = '1';
+ if (fixed)
+ buf[size++] = '0';
+ else
+ ++exp10;
+ }
+ return digits::done;
+ }
+};
+
+inline FMT_CONSTEXPR20 void adjust_precision(int& precision, int exp10) {
+ // Adjust fixed precision by exponent because it is relative to decimal
+ // point.
+ if (exp10 > 0 && precision > max_value<int>() - exp10)
+ FMT_THROW(format_error("number is too big"));
+ precision += exp10;
+}
+
+// Generates output using the Grisu digit-gen algorithm.
+// error: the size of the region (lower, upper) outside of which numbers
+// definitely do not round to value (Delta in Grisu3).
+FMT_INLINE FMT_CONSTEXPR20 auto grisu_gen_digits(fp value, uint64_t error,
+ int& exp,
+ gen_digits_handler& handler)
+ -> digits::result {
+ const fp one(1ULL << -value.e, value.e);
+ // The integral part of scaled value (p1 in Grisu) = value / one. It cannot be
+ // zero because it contains a product of two 64-bit numbers with MSB set (due
+ // to normalization) - 1, shifted right by at most 60 bits.
+ auto integral = static_cast<uint32_t>(value.f >> -one.e);
+ FMT_ASSERT(integral != 0, "");
+ FMT_ASSERT(integral == value.f >> -one.e, "");
+ // The fractional part of scaled value (p2 in Grisu) c = value % one.
+ uint64_t fractional = value.f & (one.f - 1);
+ exp = count_digits(integral); // kappa in Grisu.
+ // Non-fixed formats require at least one digit and no precision adjustment.
+ if (handler.fixed) {
+ adjust_precision(handler.precision, exp + handler.exp10);
+ // Check if precision is satisfied just by leading zeros, e.g.
+ // format("{:.2f}", 0.001) gives "0.00" without generating any digits.
+ if (handler.precision <= 0) {
+ if (handler.precision < 0) return digits::done;
+ // Divide by 10 to prevent overflow.
+ uint64_t divisor = data::power_of_10_64[exp - 1] << -one.e;
+ auto dir = get_round_direction(divisor, value.f / 10, error * 10);
+ if (dir == round_direction::unknown) return digits::error;
+ handler.buf[handler.size++] = dir == round_direction::up ? '1' : '0';
+ return digits::done;
+ }
+ }
+ // Generate digits for the integral part. This can produce up to 10 digits.
+ do {
+ uint32_t digit = 0;
+ auto divmod_integral = [&](uint32_t divisor) {
+ digit = integral / divisor;
+ integral %= divisor;
+ };
+ // This optimization by Milo Yip reduces the number of integer divisions by
+ // one per iteration.
+ switch (exp) {
+ case 10:
+ divmod_integral(1000000000);
+ break;
+ case 9:
+ divmod_integral(100000000);
+ break;
+ case 8:
+ divmod_integral(10000000);
+ break;
+ case 7:
+ divmod_integral(1000000);
+ break;
+ case 6:
+ divmod_integral(100000);
+ break;
+ case 5:
+ divmod_integral(10000);
+ break;
+ case 4:
+ divmod_integral(1000);
+ break;
+ case 3:
+ divmod_integral(100);
+ break;
+ case 2:
+ divmod_integral(10);
+ break;
+ case 1:
+ digit = integral;
+ integral = 0;
+ break;
+ default:
+ FMT_ASSERT(false, "invalid number of digits");
+ }
+ --exp;
+ auto remainder = (static_cast<uint64_t>(integral) << -one.e) + fractional;
+ auto result = handler.on_digit(static_cast<char>('0' + digit),
+ data::power_of_10_64[exp] << -one.e,
+ remainder, error, true);
+ if (result != digits::more) return result;
+ } while (exp > 0);
+ // Generate digits for the fractional part.
+ for (;;) {
+ fractional *= 10;
+ error *= 10;
+ char digit = static_cast<char>('0' + (fractional >> -one.e));
+ fractional &= one.f - 1;
+ --exp;
+ auto result = handler.on_digit(digit, one.f, fractional, error, false);
+ if (result != digits::more) return result;
+ }
+}
+
+class bigint {
+ private:
+ // A bigint is stored as an array of bigits (big digits), with bigit at index
+ // 0 being the least significant one.
+ using bigit = uint32_t;
+ using double_bigit = uint64_t;
+ enum { bigits_capacity = 32 };
+ basic_memory_buffer<bigit, bigits_capacity> bigits_;
+ int exp_;
+
+ FMT_CONSTEXPR20 bigit operator[](int index) const {
+ return bigits_[to_unsigned(index)];
+ }
+ FMT_CONSTEXPR20 bigit& operator[](int index) {
+ return bigits_[to_unsigned(index)];
+ }
+
+ static constexpr const int bigit_bits = num_bits<bigit>();
+
+ friend struct formatter<bigint>;
+
+ FMT_CONSTEXPR20 void subtract_bigits(int index, bigit other, bigit& borrow) {
+ auto result = static_cast<double_bigit>((*this)[index]) - other - borrow;
+ (*this)[index] = static_cast<bigit>(result);
+ borrow = static_cast<bigit>(result >> (bigit_bits * 2 - 1));
+ }
+
+ FMT_CONSTEXPR20 void remove_leading_zeros() {
+ int num_bigits = static_cast<int>(bigits_.size()) - 1;
+ while (num_bigits > 0 && (*this)[num_bigits] == 0) --num_bigits;
+ bigits_.resize(to_unsigned(num_bigits + 1));
+ }
+
+ // Computes *this -= other assuming aligned bigints and *this >= other.
+ FMT_CONSTEXPR20 void subtract_aligned(const bigint& other) {
+ FMT_ASSERT(other.exp_ >= exp_, "unaligned bigints");
+ FMT_ASSERT(compare(*this, other) >= 0, "");
+ bigit borrow = 0;
+ int i = other.exp_ - exp_;
+ for (size_t j = 0, n = other.bigits_.size(); j != n; ++i, ++j)
+ subtract_bigits(i, other.bigits_[j], borrow);
+ while (borrow > 0) subtract_bigits(i, 0, borrow);
+ remove_leading_zeros();
+ }
+
+ FMT_CONSTEXPR20 void multiply(uint32_t value) {
+ const double_bigit wide_value = value;
+ bigit carry = 0;
+ for (size_t i = 0, n = bigits_.size(); i < n; ++i) {
+ double_bigit result = bigits_[i] * wide_value + carry;
+ bigits_[i] = static_cast<bigit>(result);
+ carry = static_cast<bigit>(result >> bigit_bits);
+ }
+ if (carry != 0) bigits_.push_back(carry);
+ }
+
+ template <typename UInt, FMT_ENABLE_IF(std::is_same<UInt, uint64_t>::value ||
+ std::is_same<UInt, uint128_t>::value)>
+ FMT_CONSTEXPR20 void multiply(UInt value) {
+ using half_uint =
+ conditional_t<std::is_same<UInt, uint128_t>::value, uint64_t, uint32_t>;
+ const int shift = num_bits<half_uint>() - bigit_bits;
+ const UInt lower = static_cast<half_uint>(value);
+ const UInt upper = value >> num_bits<half_uint>();
+ UInt carry = 0;
+ for (size_t i = 0, n = bigits_.size(); i < n; ++i) {
+ UInt result = lower * bigits_[i] + static_cast<bigit>(carry);
+ carry = (upper * bigits_[i] << shift) + (result >> bigit_bits) +
+ (carry >> bigit_bits);
+ bigits_[i] = static_cast<bigit>(result);
+ }
+ while (carry != 0) {
+ bigits_.push_back(static_cast<bigit>(carry));
+ carry >>= bigit_bits;
+ }
+ }
+
+ template <typename UInt, FMT_ENABLE_IF(std::is_same<UInt, uint64_t>::value ||
+ std::is_same<UInt, uint128_t>::value)>
+ FMT_CONSTEXPR20 void assign(UInt n) {
+ size_t num_bigits = 0;
+ do {
+ bigits_[num_bigits++] = static_cast<bigit>(n);
+ n >>= bigit_bits;
+ } while (n != 0);
+ bigits_.resize(num_bigits);
+ exp_ = 0;
+ }
+
+ public:
+ FMT_CONSTEXPR20 bigint() : exp_(0) {}
+ explicit bigint(uint64_t n) { assign(n); }
+
+ bigint(const bigint&) = delete;
+ void operator=(const bigint&) = delete;
+
+ FMT_CONSTEXPR20 void assign(const bigint& other) {
+ auto size = other.bigits_.size();
+ bigits_.resize(size);
+ auto data = other.bigits_.data();
+ std::copy(data, data + size, make_checked(bigits_.data(), size));
+ exp_ = other.exp_;
+ }
+
+ template <typename Int> FMT_CONSTEXPR20 void operator=(Int n) {
+ FMT_ASSERT(n > 0, "");
+ assign(uint64_or_128_t<Int>(n));
+ }
+
+ FMT_CONSTEXPR20 int num_bigits() const {
+ return static_cast<int>(bigits_.size()) + exp_;
+ }
+
+ FMT_NOINLINE FMT_CONSTEXPR20 bigint& operator<<=(int shift) {
+ FMT_ASSERT(shift >= 0, "");
+ exp_ += shift / bigit_bits;
+ shift %= bigit_bits;
+ if (shift == 0) return *this;
+ bigit carry = 0;
+ for (size_t i = 0, n = bigits_.size(); i < n; ++i) {
+ bigit c = bigits_[i] >> (bigit_bits - shift);
+ bigits_[i] = (bigits_[i] << shift) + carry;
+ carry = c;
+ }
+ if (carry != 0) bigits_.push_back(carry);
+ return *this;
+ }
+
+ template <typename Int> FMT_CONSTEXPR20 bigint& operator*=(Int value) {
+ FMT_ASSERT(value > 0, "");
+ multiply(uint32_or_64_or_128_t<Int>(value));
+ return *this;
+ }
+
+ friend FMT_CONSTEXPR20 int compare(const bigint& lhs, const bigint& rhs) {
+ int num_lhs_bigits = lhs.num_bigits(), num_rhs_bigits = rhs.num_bigits();
+ if (num_lhs_bigits != num_rhs_bigits)
+ return num_lhs_bigits > num_rhs_bigits ? 1 : -1;
+ int i = static_cast<int>(lhs.bigits_.size()) - 1;
+ int j = static_cast<int>(rhs.bigits_.size()) - 1;
+ int end = i - j;
+ if (end < 0) end = 0;
+ for (; i >= end; --i, --j) {
+ bigit lhs_bigit = lhs[i], rhs_bigit = rhs[j];
+ if (lhs_bigit != rhs_bigit) return lhs_bigit > rhs_bigit ? 1 : -1;
+ }
+ if (i != j) return i > j ? 1 : -1;
+ return 0;
+ }
+
+ // Returns compare(lhs1 + lhs2, rhs).
+ friend FMT_CONSTEXPR20 int add_compare(const bigint& lhs1, const bigint& lhs2,
+ const bigint& rhs) {
+ auto minimum = [](int a, int b) { return a < b ? a : b; };
+ auto maximum = [](int a, int b) { return a > b ? a : b; };
+ int max_lhs_bigits = maximum(lhs1.num_bigits(), lhs2.num_bigits());
+ int num_rhs_bigits = rhs.num_bigits();
+ if (max_lhs_bigits + 1 < num_rhs_bigits) return -1;
+ if (max_lhs_bigits > num_rhs_bigits) return 1;
+ auto get_bigit = [](const bigint& n, int i) -> bigit {
+ return i >= n.exp_ && i < n.num_bigits() ? n[i - n.exp_] : 0;
+ };
+ double_bigit borrow = 0;
+ int min_exp = minimum(minimum(lhs1.exp_, lhs2.exp_), rhs.exp_);
+ for (int i = num_rhs_bigits - 1; i >= min_exp; --i) {
+ double_bigit sum =
+ static_cast<double_bigit>(get_bigit(lhs1, i)) + get_bigit(lhs2, i);
+ bigit rhs_bigit = get_bigit(rhs, i);
+ if (sum > rhs_bigit + borrow) return 1;
+ borrow = rhs_bigit + borrow - sum;
+ if (borrow > 1) return -1;
+ borrow <<= bigit_bits;
+ }
+ return borrow != 0 ? -1 : 0;
+ }
+
+ // Assigns pow(10, exp) to this bigint.
+ FMT_CONSTEXPR20 void assign_pow10(int exp) {
+ FMT_ASSERT(exp >= 0, "");
+ if (exp == 0) return *this = 1;
+ // Find the top bit.
+ int bitmask = 1;
+ while (exp >= bitmask) bitmask <<= 1;
+ bitmask >>= 1;
+ // pow(10, exp) = pow(5, exp) * pow(2, exp). First compute pow(5, exp) by
+ // repeated squaring and multiplication.
+ *this = 5;
+ bitmask >>= 1;
+ while (bitmask != 0) {
+ square();
+ if ((exp & bitmask) != 0) *this *= 5;
+ bitmask >>= 1;
+ }
+ *this <<= exp; // Multiply by pow(2, exp) by shifting.
+ }
+
+ FMT_CONSTEXPR20 void square() {
+ int num_bigits = static_cast<int>(bigits_.size());
+ int num_result_bigits = 2 * num_bigits;
+ basic_memory_buffer<bigit, bigits_capacity> n(std::move(bigits_));
+ bigits_.resize(to_unsigned(num_result_bigits));
+ auto sum = uint128_t();
+ for (int bigit_index = 0; bigit_index < num_bigits; ++bigit_index) {
+ // Compute bigit at position bigit_index of the result by adding
+ // cross-product terms n[i] * n[j] such that i + j == bigit_index.
+ for (int i = 0, j = bigit_index; j >= 0; ++i, --j) {
+ // Most terms are multiplied twice which can be optimized in the future.
+ sum += static_cast<double_bigit>(n[i]) * n[j];
+ }
+ (*this)[bigit_index] = static_cast<bigit>(sum);
+ sum >>= num_bits<bigit>(); // Compute the carry.
+ }
+ // Do the same for the top half.
+ for (int bigit_index = num_bigits; bigit_index < num_result_bigits;
+ ++bigit_index) {
+ for (int j = num_bigits - 1, i = bigit_index - j; i < num_bigits;)
+ sum += static_cast<double_bigit>(n[i++]) * n[j--];
+ (*this)[bigit_index] = static_cast<bigit>(sum);
+ sum >>= num_bits<bigit>();
+ }
+ remove_leading_zeros();
+ exp_ *= 2;
+ }
+
+ // If this bigint has a bigger exponent than other, adds trailing zero to make
+ // exponents equal. This simplifies some operations such as subtraction.
+ FMT_CONSTEXPR20 void align(const bigint& other) {
+ int exp_difference = exp_ - other.exp_;
+ if (exp_difference <= 0) return;
+ int num_bigits = static_cast<int>(bigits_.size());
+ bigits_.resize(to_unsigned(num_bigits + exp_difference));
+ for (int i = num_bigits - 1, j = i + exp_difference; i >= 0; --i, --j)
+ bigits_[j] = bigits_[i];
+ std::uninitialized_fill_n(bigits_.data(), exp_difference, 0);
+ exp_ -= exp_difference;
+ }
+
+ // Divides this bignum by divisor, assigning the remainder to this and
+ // returning the quotient.
+ FMT_CONSTEXPR20 int divmod_assign(const bigint& divisor) {
+ FMT_ASSERT(this != &divisor, "");
+ if (compare(*this, divisor) < 0) return 0;
+ FMT_ASSERT(divisor.bigits_[divisor.bigits_.size() - 1u] != 0, "");
+ align(divisor);
+ int quotient = 0;
+ do {
+ subtract_aligned(divisor);
+ ++quotient;
+ } while (compare(*this, divisor) >= 0);
+ return quotient;
+ }
+};
+
+// format_dragon flags.
+enum dragon {
+ predecessor_closer = 1,
+ fixup = 2, // Run fixup to correct exp10 which can be off by one.
+ fixed = 4,
+};
+
+// Formats a floating-point number using a variation of the Fixed-Precision
+// Positive Floating-Point Printout ((FPP)^2) algorithm by Steele & White:
+// https://fmt.dev/papers/p372-steele.pdf.
+FMT_CONSTEXPR20 inline void format_dragon(basic_fp<uint128_t> value,
+ unsigned flags, int num_digits,
+ buffer<char>& buf, int& exp10) {
+ bigint numerator; // 2 * R in (FPP)^2.
+ bigint denominator; // 2 * S in (FPP)^2.
+ // lower and upper are differences between value and corresponding boundaries.
+ bigint lower; // (M^- in (FPP)^2).
+ bigint upper_store; // upper's value if different from lower.
+ bigint* upper = nullptr; // (M^+ in (FPP)^2).
+ // Shift numerator and denominator by an extra bit or two (if lower boundary
+ // is closer) to make lower and upper integers. This eliminates multiplication
+ // by 2 during later computations.
+ bool is_predecessor_closer = (flags & dragon::predecessor_closer) != 0;
+ int shift = is_predecessor_closer ? 2 : 1;
+ if (value.e >= 0) {
+ numerator = value.f;
+ numerator <<= value.e + shift;
+ lower = 1;
+ lower <<= value.e;
+ if (is_predecessor_closer) {
+ upper_store = 1;
+ upper_store <<= value.e + 1;
+ upper = &upper_store;
+ }
+ denominator.assign_pow10(exp10);
+ denominator <<= shift;
+ } else if (exp10 < 0) {
+ numerator.assign_pow10(-exp10);
+ lower.assign(numerator);
+ if (is_predecessor_closer) {
+ upper_store.assign(numerator);
+ upper_store <<= 1;
+ upper = &upper_store;
+ }
+ numerator *= value.f;
+ numerator <<= shift;
+ denominator = 1;
+ denominator <<= shift - value.e;
+ } else {
+ numerator = value.f;
+ numerator <<= shift;
+ denominator.assign_pow10(exp10);
+ denominator <<= shift - value.e;
+ lower = 1;
+ if (is_predecessor_closer) {
+ upper_store = 1ULL << 1;
+ upper = &upper_store;
+ }
+ }
+ int even = static_cast<int>((value.f & 1) == 0);
+ if (!upper) upper = &lower;
+ if ((flags & dragon::fixup) != 0) {
+ if (add_compare(numerator, *upper, denominator) + even <= 0) {
+ --exp10;
+ numerator *= 10;
+ if (num_digits < 0) {
+ lower *= 10;
+ if (upper != &lower) *upper *= 10;
+ }
+ }
+ if ((flags & dragon::fixed) != 0) adjust_precision(num_digits, exp10 + 1);
+ }
+ // Invariant: value == (numerator / denominator) * pow(10, exp10).
+ if (num_digits < 0) {
+ // Generate the shortest representation.
+ num_digits = 0;
+ char* data = buf.data();
+ for (;;) {
+ int digit = numerator.divmod_assign(denominator);
+ bool low = compare(numerator, lower) - even < 0; // numerator <[=] lower.
+ // numerator + upper >[=] pow10:
+ bool high = add_compare(numerator, *upper, denominator) + even > 0;
+ data[num_digits++] = static_cast<char>('0' + digit);
+ if (low || high) {
+ if (!low) {
+ ++data[num_digits - 1];
+ } else if (high) {
+ int result = add_compare(numerator, numerator, denominator);
+ // Round half to even.
+ if (result > 0 || (result == 0 && (digit % 2) != 0))
+ ++data[num_digits - 1];
+ }
+ buf.try_resize(to_unsigned(num_digits));
+ exp10 -= num_digits - 1;
+ return;
+ }
+ numerator *= 10;
+ lower *= 10;
+ if (upper != &lower) *upper *= 10;
+ }
+ }
+ // Generate the given number of digits.
+ exp10 -= num_digits - 1;
+ if (num_digits == 0) {
+ denominator *= 10;
+ auto digit = add_compare(numerator, numerator, denominator) > 0 ? '1' : '0';
+ buf.push_back(digit);
+ return;
+ }
+ buf.try_resize(to_unsigned(num_digits));
+ for (int i = 0; i < num_digits - 1; ++i) {
+ int digit = numerator.divmod_assign(denominator);
+ buf[i] = static_cast<char>('0' + digit);
+ numerator *= 10;
+ }
+ int digit = numerator.divmod_assign(denominator);
+ auto result = add_compare(numerator, numerator, denominator);
+ if (result > 0 || (result == 0 && (digit % 2) != 0)) {
+ if (digit == 9) {
+ const auto overflow = '0' + 10;
+ buf[num_digits - 1] = overflow;
+ // Propagate the carry.
+ for (int i = num_digits - 1; i > 0 && buf[i] == overflow; --i) {
+ buf[i] = '0';
+ ++buf[i - 1];
+ }
+ if (buf[0] == overflow) {
+ buf[0] = '1';
+ ++exp10;
+ }
+ return;
+ }
+ ++digit;
+ }
+ buf[num_digits - 1] = static_cast<char>('0' + digit);
+}
+
+// Formats a floating-point number using the hexfloat format.
+template <typename Float, FMT_ENABLE_IF(!is_double_double<Float>::value)>
+FMT_CONSTEXPR20 void format_hexfloat(Float value, int precision,
+ float_specs specs, buffer<char>& buf) {
+ // float is passed as double to reduce the number of instantiations and to
+ // simplify implementation.
+ static_assert(!std::is_same<Float, float>::value, "");
+
+ using info = dragonbox::float_info<Float>;
+
+ // Assume Float is in the format [sign][exponent][significand].
+ using carrier_uint = typename info::carrier_uint;
+
+ constexpr auto num_float_significand_bits =
+ detail::num_significand_bits<Float>();
+
+ basic_fp<carrier_uint> f(value);
+ f.e += num_float_significand_bits;
+ if (!has_implicit_bit<Float>()) --f.e;
+
+ constexpr auto num_fraction_bits =
+ num_float_significand_bits + (has_implicit_bit<Float>() ? 1 : 0);
+ constexpr auto num_xdigits = (num_fraction_bits + 3) / 4;
+
+ constexpr auto leading_shift = ((num_xdigits - 1) * 4);
+ const auto leading_mask = carrier_uint(0xF) << leading_shift;
+ const auto leading_xdigit =
+ static_cast<uint32_t>((f.f & leading_mask) >> leading_shift);
+ if (leading_xdigit > 1) f.e -= (32 - countl_zero(leading_xdigit) - 1);
+
+ int print_xdigits = num_xdigits - 1;
+ if (precision >= 0 && print_xdigits > precision) {
+ const int shift = ((print_xdigits - precision - 1) * 4);
+ const auto mask = carrier_uint(0xF) << shift;
+ const auto v = static_cast<uint32_t>((f.f & mask) >> shift);
+
+ if (v >= 8) {
+ const auto inc = carrier_uint(1) << (shift + 4);
+ f.f += inc;
+ f.f &= ~(inc - 1);
+ }
+
+ // Check long double overflow
+ if (!has_implicit_bit<Float>()) {
+ const auto implicit_bit = carrier_uint(1) << num_float_significand_bits;
+ if ((f.f & implicit_bit) == implicit_bit) {
+ f.f >>= 4;
+ f.e += 4;
+ }
+ }
+
+ print_xdigits = precision;
+ }
+
+ char xdigits[num_bits<carrier_uint>() / 4];
+ detail::fill_n(xdigits, sizeof(xdigits), '0');
+ format_uint<4>(xdigits, f.f, num_xdigits, specs.upper);
+
+ // Remove zero tail
+ while (print_xdigits > 0 && xdigits[print_xdigits] == '0') --print_xdigits;
+
+ buf.push_back('0');
+ buf.push_back(specs.upper ? 'X' : 'x');
+ buf.push_back(xdigits[0]);
+ if (specs.showpoint || print_xdigits > 0 || print_xdigits < precision)
+ buf.push_back('.');
+ buf.append(xdigits + 1, xdigits + 1 + print_xdigits);
+ for (; print_xdigits < precision; ++print_xdigits) buf.push_back('0');
+
+ buf.push_back(specs.upper ? 'P' : 'p');
+
+ uint32_t abs_e;
+ if (f.e < 0) {
+ buf.push_back('-');
+ abs_e = static_cast<uint32_t>(-f.e);
+ } else {
+ buf.push_back('+');
+ abs_e = static_cast<uint32_t>(f.e);
+ }
+ format_decimal<char>(appender(buf), abs_e, detail::count_digits(abs_e));
+}
+
+template <typename Float, FMT_ENABLE_IF(is_double_double<Float>::value)>
+FMT_CONSTEXPR20 void format_hexfloat(Float value, int precision,
+ float_specs specs, buffer<char>& buf) {
+ format_hexfloat(static_cast<double>(value), precision, specs, buf);
+}
+
+template <typename Float>
+FMT_CONSTEXPR20 auto format_float(Float value, int precision, float_specs specs,
+ buffer<char>& buf) -> int {
+ // float is passed as double to reduce the number of instantiations.
+ static_assert(!std::is_same<Float, float>::value, "");
+ FMT_ASSERT(value >= 0, "value is negative");
+ auto converted_value = convert_float(value);
+
+ const bool fixed = specs.format == float_format::fixed;
+ if (value <= 0) { // <= instead of == to silence a warning.
+ if (precision <= 0 || !fixed) {
+ buf.push_back('0');
+ return 0;
+ }
+ buf.try_resize(to_unsigned(precision));
+ fill_n(buf.data(), precision, '0');
+ return -precision;
+ }
+
+ int exp = 0;
+ bool use_dragon = true;
+ unsigned dragon_flags = 0;
+ if (!is_fast_float<Float>()) {
+ const auto inv_log2_10 = 0.3010299956639812; // 1 / log2(10)
+ using info = dragonbox::float_info<decltype(converted_value)>;
+ const auto f = basic_fp<typename info::carrier_uint>(converted_value);
+ // Compute exp, an approximate power of 10, such that
+ // 10^(exp - 1) <= value < 10^exp or 10^exp <= value < 10^(exp + 1).
+ // This is based on log10(value) == log2(value) / log2(10) and approximation
+ // of log2(value) by e + num_fraction_bits idea from double-conversion.
+ exp = static_cast<int>(
+ std::ceil((f.e + count_digits<1>(f.f) - 1) * inv_log2_10 - 1e-10));
+ dragon_flags = dragon::fixup;
+ } else if (!is_constant_evaluated() && precision < 0) {
+ // Use Dragonbox for the shortest format.
+ if (specs.binary32) {
+ auto dec = dragonbox::to_decimal(static_cast<float>(value));
+ write<char>(buffer_appender<char>(buf), dec.significand);
+ return dec.exponent;
+ }
+ auto dec = dragonbox::to_decimal(static_cast<double>(value));
+ write<char>(buffer_appender<char>(buf), dec.significand);
+ return dec.exponent;
+ } else if (is_constant_evaluated()) {
+ // Use Grisu + Dragon4 for the given precision:
+ // https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf.
+ const int min_exp = -60; // alpha in Grisu.
+ int cached_exp10 = 0; // K in Grisu.
+ fp normalized = normalize(fp(converted_value));
+ const auto cached_pow = get_cached_power(
+ min_exp - (normalized.e + fp::num_significand_bits), cached_exp10);
+ normalized = normalized * cached_pow;
+ gen_digits_handler handler{buf.data(), 0, precision, -cached_exp10, fixed};
+ if (grisu_gen_digits(normalized, 1, exp, handler) != digits::error &&
+ !is_constant_evaluated()) {
+ exp += handler.exp10;
+ buf.try_resize(to_unsigned(handler.size));
+ use_dragon = false;
+ } else {
+ exp += handler.size - cached_exp10 - 1;
+ precision = handler.precision;
+ }
+ } else {
+ // Extract significand bits and exponent bits.
+ using info = dragonbox::float_info<double>;
+ auto br = bit_cast<uint64_t>(static_cast<double>(value));
+
+ const uint64_t significand_mask =
+ (static_cast<uint64_t>(1) << num_significand_bits<double>()) - 1;
+ uint64_t significand = (br & significand_mask);
+ int exponent = static_cast<int>((br & exponent_mask<double>()) >>
+ num_significand_bits<double>());
+
+ if (exponent != 0) { // Check if normal.
+ exponent -= exponent_bias<double>() + num_significand_bits<double>();
+ significand |=
+ (static_cast<uint64_t>(1) << num_significand_bits<double>());
+ significand <<= 1;
+ } else {
+ // Normalize subnormal inputs.
+ FMT_ASSERT(significand != 0, "zeros should not appear hear");
+ int shift = countl_zero(significand);
+ FMT_ASSERT(shift >= num_bits<uint64_t>() - num_significand_bits<double>(),
+ "");
+ shift -= (num_bits<uint64_t>() - num_significand_bits<double>() - 2);
+ exponent = (std::numeric_limits<double>::min_exponent -
+ num_significand_bits<double>()) -
+ shift;
+ significand <<= shift;
+ }
+
+ // Compute the first several nonzero decimal significand digits.
+ // We call the number we get the first segment.
+ const int k = info::kappa - dragonbox::floor_log10_pow2(exponent);
+ exp = -k;
+ const int beta = exponent + dragonbox::floor_log2_pow10(k);
+ uint64_t first_segment;
+ bool has_more_segments;
+ int digits_in_the_first_segment;
+ {
+ const auto r = dragonbox::umul192_upper128(
+ significand << beta, dragonbox::get_cached_power(k));
+ first_segment = r.high();
+ has_more_segments = r.low() != 0;
+
+ // The first segment can have 18 ~ 19 digits.
+ if (first_segment >= 1000000000000000000ULL) {
+ digits_in_the_first_segment = 19;
+ } else {
+ // When it is of 18-digits, we align it to 19-digits by adding a bogus
+ // zero at the end.
+ digits_in_the_first_segment = 18;
+ first_segment *= 10;
+ }
+ }
+
+ // Compute the actual number of decimal digits to print.
+ if (fixed) {
+ adjust_precision(precision, exp + digits_in_the_first_segment);
+ }
+
+ // Use Dragon4 only when there might be not enough digits in the first
+ // segment.
+ if (digits_in_the_first_segment > precision) {
+ use_dragon = false;
+
+ if (precision <= 0) {
+ exp += digits_in_the_first_segment;
+
+ if (precision < 0) {
+ // Nothing to do, since all we have are just leading zeros.
+ buf.try_resize(0);
+ } else {
+ // We may need to round-up.
+ buf.try_resize(1);
+ if ((first_segment | static_cast<uint64_t>(has_more_segments)) >
+ 5000000000000000000ULL) {
+ buf[0] = '1';
+ } else {
+ buf[0] = '0';
+ }
+ }
+ } // precision <= 0
+ else {
+ exp += digits_in_the_first_segment - precision;
+
+ // When precision > 0, we divide the first segment into three
+ // subsegments, each with 9, 9, and 0 ~ 1 digits so that each fits
+ // in 32-bits which usually allows faster calculation than in
+ // 64-bits. Since some compiler (e.g. MSVC) doesn't know how to optimize
+ // division-by-constant for large 64-bit divisors, we do it here
+ // manually. The magic number 7922816251426433760 below is equal to
+ // ceil(2^(64+32) / 10^10).
+ const uint32_t first_subsegment = static_cast<uint32_t>(
+ dragonbox::umul128_upper64(first_segment, 7922816251426433760ULL) >>
+ 32);
+ const uint64_t second_third_subsegments =
+ first_segment - first_subsegment * 10000000000ULL;
+
+ uint64_t prod;
+ uint32_t digits;
+ bool should_round_up;
+ int number_of_digits_to_print = precision > 9 ? 9 : precision;
+
+ // Print a 9-digits subsegment, either the first or the second.
+ auto print_subsegment = [&](uint32_t subsegment, char* buffer) {
+ int number_of_digits_printed = 0;
+
+ // If we want to print an odd number of digits from the subsegment,
+ if ((number_of_digits_to_print & 1) != 0) {
+ // Convert to 64-bit fixed-point fractional form with 1-digit
+ // integer part. The magic number 720575941 is a good enough
+ // approximation of 2^(32 + 24) / 10^8; see
+ // https://jk-jeon.github.io/posts/2022/12/fixed-precision-formatting/#fixed-length-case
+ // for details.
+ prod = ((subsegment * static_cast<uint64_t>(720575941)) >> 24) + 1;
+ digits = static_cast<uint32_t>(prod >> 32);
+ *buffer = static_cast<char>('0' + digits);
+ number_of_digits_printed++;
+ }
+ // If we want to print an even number of digits from the
+ // first_subsegment,
+ else {
+ // Convert to 64-bit fixed-point fractional form with 2-digits
+ // integer part. The magic number 450359963 is a good enough
+ // approximation of 2^(32 + 20) / 10^7; see
+ // https://jk-jeon.github.io/posts/2022/12/fixed-precision-formatting/#fixed-length-case
+ // for details.
+ prod = ((subsegment * static_cast<uint64_t>(450359963)) >> 20) + 1;
+ digits = static_cast<uint32_t>(prod >> 32);
+ copy2(buffer, digits2(digits));
+ number_of_digits_printed += 2;
+ }
+
+ // Print all digit pairs.
+ while (number_of_digits_printed < number_of_digits_to_print) {
+ prod = static_cast<uint32_t>(prod) * static_cast<uint64_t>(100);
+ digits = static_cast<uint32_t>(prod >> 32);
+ copy2(buffer + number_of_digits_printed, digits2(digits));
+ number_of_digits_printed += 2;
+ }
+ };
+
+ // Print first subsegment.
+ print_subsegment(first_subsegment, buf.data());
+
+ // Perform rounding if the first subsegment is the last subsegment to
+ // print.
+ if (precision <= 9) {
+ // Rounding inside the subsegment.
+ // We round-up if:
+ // - either the fractional part is strictly larger than 1/2, or
+ // - the fractional part is exactly 1/2 and the last digit is odd.
+ // We rely on the following observations:
+ // - If fractional_part >= threshold, then the fractional part is
+ // strictly larger than 1/2.
+ // - If the MSB of fractional_part is set, then the fractional part
+ // must be at least 1/2.
+ // - When the MSB of fractional_part is set, either
+ // second_third_subsegments being nonzero or has_more_segments
+ // being true means there are further digits not printed, so the
+ // fractional part is strictly larger than 1/2.
+ if (precision < 9) {
+ uint32_t fractional_part = static_cast<uint32_t>(prod);
+ should_round_up = fractional_part >=
+ data::fractional_part_rounding_thresholds
+ [8 - number_of_digits_to_print] ||
+ ((fractional_part >> 31) &
+ ((digits & 1) | (second_third_subsegments != 0) |
+ has_more_segments)) != 0;
+ }
+ // Rounding at the subsegment boundary.
+ // In this case, the fractional part is at least 1/2 if and only if
+ // second_third_subsegments >= 5000000000ULL, and is strictly larger
+ // than 1/2 if we further have either second_third_subsegments >
+ // 5000000000ULL or has_more_segments == true.
+ else {
+ should_round_up = second_third_subsegments > 5000000000ULL ||
+ (second_third_subsegments == 5000000000ULL &&
+ ((digits & 1) != 0 || has_more_segments));
+ }
+ }
+ // Otherwise, print the second subsegment.
+ else {
+ // Compilers are not aware of how to leverage the maximum value of
+ // second_third_subsegments to find out a better magic number which
+ // allows us to eliminate an additional shift. 1844674407370955162 =
+ // ceil(2^64/10) < ceil(2^64*(10^9/(10^10 - 1))).
+ const uint32_t second_subsegment =
+ static_cast<uint32_t>(dragonbox::umul128_upper64(
+ second_third_subsegments, 1844674407370955162ULL));
+ const uint32_t third_subsegment =
+ static_cast<uint32_t>(second_third_subsegments) -
+ second_subsegment * 10;
+
+ number_of_digits_to_print = precision - 9;
+ print_subsegment(second_subsegment, buf.data() + 9);
+
+ // Rounding inside the subsegment.
+ if (precision < 18) {
+ // The condition third_subsegment != 0 implies that the segment was
+ // of 19 digits, so in this case the third segment should be
+ // consisting of a genuine digit from the input.
+ uint32_t fractional_part = static_cast<uint32_t>(prod);
+ should_round_up = fractional_part >=
+ data::fractional_part_rounding_thresholds
+ [8 - number_of_digits_to_print] ||
+ ((fractional_part >> 31) &
+ ((digits & 1) | (third_subsegment != 0) |
+ has_more_segments)) != 0;
+ }
+ // Rounding at the subsegment boundary.
+ else {
+ // In this case, the segment must be of 19 digits, thus
+ // the third subsegment should be consisting of a genuine digit from
+ // the input.
+ should_round_up = third_subsegment > 5 ||
+ (third_subsegment == 5 &&
+ ((digits & 1) != 0 || has_more_segments));
+ }
+ }
+
+ // Round-up if necessary.
+ if (should_round_up) {
+ ++buf[precision - 1];
+ for (int i = precision - 1; i > 0 && buf[i] > '9'; --i) {
+ buf[i] = '0';
+ ++buf[i - 1];
+ }
+ if (buf[0] > '9') {
+ buf[0] = '1';
+ if (fixed)
+ buf[precision++] = '0';
+ else
+ ++exp;
+ }
+ }
+ buf.try_resize(to_unsigned(precision));
+ }
+ } // if (digits_in_the_first_segment > precision)
+ else {
+ // Adjust the exponent for its use in Dragon4.
+ exp += digits_in_the_first_segment - 1;
+ }
+ }
+ if (use_dragon) {
+ auto f = basic_fp<uint128_t>();
+ bool is_predecessor_closer = specs.binary32
+ ? f.assign(static_cast<float>(value))
+ : f.assign(converted_value);
+ if (is_predecessor_closer) dragon_flags |= dragon::predecessor_closer;
+ if (fixed) dragon_flags |= dragon::fixed;
+ // Limit precision to the maximum possible number of significant digits in
+ // an IEEE754 double because we don't need to generate zeros.
+ const int max_double_digits = 767;
+ if (precision > max_double_digits) precision = max_double_digits;
+ format_dragon(f, dragon_flags, precision, buf, exp);
+ }
+ if (!fixed && !specs.showpoint) {
+ // Remove trailing zeros.
+ auto num_digits = buf.size();
+ while (num_digits > 0 && buf[num_digits - 1] == '0') {
+ --num_digits;
+ ++exp;
+ }
+ buf.try_resize(num_digits);
+ }
+ return exp;
+}
+template <typename Char, typename OutputIt, typename T>
+FMT_CONSTEXPR20 auto write_float(OutputIt out, T value,
+ format_specs<Char> specs, locale_ref loc)
+ -> OutputIt {
+ float_specs fspecs = parse_float_type_spec(specs);
+ fspecs.sign = specs.sign;
+ if (detail::signbit(value)) { // value < 0 is false for NaN so use signbit.
+ fspecs.sign = sign::minus;
+ value = -value;
+ } else if (fspecs.sign == sign::minus) {
+ fspecs.sign = sign::none;
+ }
+
+ if (!detail::isfinite(value))
+ return write_nonfinite(out, detail::isnan(value), specs, fspecs);
+
+ if (specs.align == align::numeric && fspecs.sign) {
+ auto it = reserve(out, 1);
+ *it++ = detail::sign<Char>(fspecs.sign);
+ out = base_iterator(out, it);
+ fspecs.sign = sign::none;
+ if (specs.width != 0) --specs.width;
+ }
+
+ memory_buffer buffer;
+ if (fspecs.format == float_format::hex) {
+ if (fspecs.sign) buffer.push_back(detail::sign<char>(fspecs.sign));
+ format_hexfloat(convert_float(value), specs.precision, fspecs, buffer);
+ return write_bytes<align::right>(out, {buffer.data(), buffer.size()},
+ specs);
+ }
+ int precision = specs.precision >= 0 || specs.type == presentation_type::none
+ ? specs.precision
+ : 6;
+ if (fspecs.format == float_format::exp) {
+ if (precision == max_value<int>())
+ throw_format_error("number is too big");
+ else
+ ++precision;
+ } else if (fspecs.format != float_format::fixed && precision == 0) {
+ precision = 1;
+ }
+ if (const_check(std::is_same<T, float>())) fspecs.binary32 = true;
+ int exp = format_float(convert_float(value), precision, fspecs, buffer);
+ fspecs.precision = precision;
+ auto f = big_decimal_fp{buffer.data(), static_cast<int>(buffer.size()), exp};
+ return write_float(out, f, specs, fspecs, loc);
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_floating_point<T>::value)>
+FMT_CONSTEXPR20 auto write(OutputIt out, T value, format_specs<Char> specs,
+ locale_ref loc = {}) -> OutputIt {
+ if (const_check(!is_supported_floating_point(value))) return out;
+ return specs.localized && write_loc(out, value, specs, loc)
+ ? out
+ : write_float(out, value, specs, loc);
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_fast_float<T>::value)>
+FMT_CONSTEXPR20 auto write(OutputIt out, T value) -> OutputIt {
+ if (is_constant_evaluated()) return write(out, value, format_specs<Char>());
+ if (const_check(!is_supported_floating_point(value))) return out;
+
+ auto fspecs = float_specs();
+ if (detail::signbit(value)) {
+ fspecs.sign = sign::minus;
+ value = -value;
+ }
+
+ constexpr auto specs = format_specs<Char>();
+ using floaty = conditional_t<std::is_same<T, long double>::value, double, T>;
+ using floaty_uint = typename dragonbox::float_info<floaty>::carrier_uint;
+ floaty_uint mask = exponent_mask<floaty>();
+ if ((bit_cast<floaty_uint>(value) & mask) == mask)
+ return write_nonfinite(out, std::isnan(value), specs, fspecs);
+
+ auto dec = dragonbox::to_decimal(static_cast<floaty>(value));
+ return write_float(out, dec, specs, fspecs, {});
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_floating_point<T>::value &&
+ !is_fast_float<T>::value)>
+inline auto write(OutputIt out, T value) -> OutputIt {
+ return write(out, value, format_specs<Char>());
+}
+
+template <typename Char, typename OutputIt>
+auto write(OutputIt out, monostate, format_specs<Char> = {}, locale_ref = {})
+ -> OutputIt {
+ FMT_ASSERT(false, "");
+ return out;
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, basic_string_view<Char> value)
+ -> OutputIt {
+ auto it = reserve(out, value.size());
+ it = copy_str_noinline<Char>(value.begin(), value.end(), it);
+ return base_iterator(out, it);
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(is_string<T>::value)>
+constexpr auto write(OutputIt out, const T& value) -> OutputIt {
+ return write<Char>(out, to_string_view(value));
+}
+
+// FMT_ENABLE_IF() condition separated to workaround an MSVC bug.
+template <
+ typename Char, typename OutputIt, typename T,
+ bool check =
+ std::is_enum<T>::value && !std::is_same<T, Char>::value &&
+ mapped_type_constant<T, basic_format_context<OutputIt, Char>>::value !=
+ type::custom_type,
+ FMT_ENABLE_IF(check)>
+FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt {
+ return write<Char>(out, static_cast<underlying_t<T>>(value));
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(std::is_same<T, bool>::value)>
+FMT_CONSTEXPR auto write(OutputIt out, T value,
+ const format_specs<Char>& specs = {}, locale_ref = {})
+ -> OutputIt {
+ return specs.type != presentation_type::none &&
+ specs.type != presentation_type::string
+ ? write(out, value ? 1 : 0, specs, {})
+ : write_bytes(out, value ? "true" : "false", specs);
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, Char value) -> OutputIt {
+ auto it = reserve(out, 1);
+ *it++ = value;
+ return base_iterator(out, it);
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR_CHAR_TRAITS auto write(OutputIt out, const Char* value)
+ -> OutputIt {
+ if (value) return write(out, basic_string_view<Char>(value));
+ throw_format_error("string pointer is null");
+ return out;
+}
+
+template <typename Char, typename OutputIt, typename T,
+ FMT_ENABLE_IF(std::is_same<T, void>::value)>
+auto write(OutputIt out, const T* value, const format_specs<Char>& specs = {},
+ locale_ref = {}) -> OutputIt {
+ return write_ptr<Char>(out, bit_cast<uintptr_t>(value), &specs);
+}
+
+// A write overload that handles implicit conversions.
+template <typename Char, typename OutputIt, typename T,
+ typename Context = basic_format_context<OutputIt, Char>>
+FMT_CONSTEXPR auto write(OutputIt out, const T& value) -> enable_if_t<
+ std::is_class<T>::value && !is_string<T>::value &&
+ !is_floating_point<T>::value && !std::is_same<T, Char>::value &&
+ !std::is_same<T, remove_cvref_t<decltype(arg_mapper<Context>().map(
+ value))>>::value,
+ OutputIt> {
+ return write<Char>(out, arg_mapper<Context>().map(value));
+}
+
+template <typename Char, typename OutputIt, typename T,
+ typename Context = basic_format_context<OutputIt, Char>>
+FMT_CONSTEXPR auto write(OutputIt out, const T& value)
+ -> enable_if_t<mapped_type_constant<T, Context>::value == type::custom_type,
+ OutputIt> {
+ auto ctx = Context(out, {}, {});
+ return typename Context::template formatter_type<T>().format(value, ctx);
+}
+
+// An argument visitor that formats the argument and writes it via the output
+// iterator. It's a class and not a generic lambda for compatibility with C++11.
+template <typename Char> struct default_arg_formatter {
+ using iterator = buffer_appender<Char>;
+ using context = buffer_context<Char>;
+
+ iterator out;
+ basic_format_args<context> args;
+ locale_ref loc;
+
+ template <typename T> auto operator()(T value) -> iterator {
+ return write<Char>(out, value);
+ }
+ auto operator()(typename basic_format_arg<context>::handle h) -> iterator {
+ basic_format_parse_context<Char> parse_ctx({});
+ context format_ctx(out, args, loc);
+ h.format(parse_ctx, format_ctx);
+ return format_ctx.out();
+ }
+};
+
+template <typename Char> struct arg_formatter {
+ using iterator = buffer_appender<Char>;
+ using context = buffer_context<Char>;
+
+ iterator out;
+ const format_specs<Char>& specs;
+ locale_ref locale;
+
+ template <typename T>
+ FMT_CONSTEXPR FMT_INLINE auto operator()(T value) -> iterator {
+ return detail::write(out, value, specs, locale);
+ }
+ auto operator()(typename basic_format_arg<context>::handle) -> iterator {
+ // User-defined types are handled separately because they require access
+ // to the parse context.
+ return out;
+ }
+};
+
+template <typename Char> struct custom_formatter {
+ basic_format_parse_context<Char>& parse_ctx;
+ buffer_context<Char>& ctx;
+
+ void operator()(
+ typename basic_format_arg<buffer_context<Char>>::handle h) const {
+ h.format(parse_ctx, ctx);
+ }
+ template <typename T> void operator()(T) const {}
+};
+
+template <typename ErrorHandler> class width_checker {
+ public:
+ explicit FMT_CONSTEXPR width_checker(ErrorHandler& eh) : handler_(eh) {}
+
+ template <typename T, FMT_ENABLE_IF(is_integer<T>::value)>
+ FMT_CONSTEXPR auto operator()(T value) -> unsigned long long {
+ if (is_negative(value)) handler_.on_error("negative width");
+ return static_cast<unsigned long long>(value);
+ }
+
+ template <typename T, FMT_ENABLE_IF(!is_integer<T>::value)>
+ FMT_CONSTEXPR auto operator()(T) -> unsigned long long {
+ handler_.on_error("width is not integer");
+ return 0;
+ }
+
+ private:
+ ErrorHandler& handler_;
+};
+
+template <typename ErrorHandler> class precision_checker {
+ public:
+ explicit FMT_CONSTEXPR precision_checker(ErrorHandler& eh) : handler_(eh) {}
+
+ template <typename T, FMT_ENABLE_IF(is_integer<T>::value)>
+ FMT_CONSTEXPR auto operator()(T value) -> unsigned long long {
+ if (is_negative(value)) handler_.on_error("negative precision");
+ return static_cast<unsigned long long>(value);
+ }
+
+ template <typename T, FMT_ENABLE_IF(!is_integer<T>::value)>
+ FMT_CONSTEXPR auto operator()(T) -> unsigned long long {
+ handler_.on_error("precision is not integer");
+ return 0;
+ }
+
+ private:
+ ErrorHandler& handler_;
+};
+
+template <template <typename> class Handler, typename FormatArg,
+ typename ErrorHandler>
+FMT_CONSTEXPR auto get_dynamic_spec(FormatArg arg, ErrorHandler eh) -> int {
+ unsigned long long value = visit_format_arg(Handler<ErrorHandler>(eh), arg);
+ if (value > to_unsigned(max_value<int>())) eh.on_error("number is too big");
+ return static_cast<int>(value);
+}
+
+template <typename Context, typename ID>
+FMT_CONSTEXPR auto get_arg(Context& ctx, ID id) ->
+ typename Context::format_arg {
+ auto arg = ctx.arg(id);
+ if (!arg) ctx.on_error("argument not found");
+ return arg;
+}
+
+template <template <typename> class Handler, typename Context>
+FMT_CONSTEXPR void handle_dynamic_spec(int& value,
+ arg_ref<typename Context::char_type> ref,
+ Context& ctx) {
+ switch (ref.kind) {
+ case arg_id_kind::none:
+ break;
+ case arg_id_kind::index:
+ value = detail::get_dynamic_spec<Handler>(get_arg(ctx, ref.val.index),
+ ctx.error_handler());
+ break;
+ case arg_id_kind::name:
+ value = detail::get_dynamic_spec<Handler>(get_arg(ctx, ref.val.name),
+ ctx.error_handler());
+ break;
+ }
+}
+
+#if FMT_USE_USER_DEFINED_LITERALS
+template <typename Char> struct udl_formatter {
+ basic_string_view<Char> str;
+
+ template <typename... T>
+ auto operator()(T&&... args) const -> std::basic_string<Char> {
+ return vformat(str, fmt::make_format_args<buffer_context<Char>>(args...));
+ }
+};
+
+# if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <typename T, typename Char, size_t N,
+ fmt::detail_exported::fixed_string<Char, N> Str>
+struct statically_named_arg : view {
+ static constexpr auto name = Str.data;
+
+ const T& value;
+ statically_named_arg(const T& v) : value(v) {}
+};
+
+template <typename T, typename Char, size_t N,
+ fmt::detail_exported::fixed_string<Char, N> Str>
+struct is_named_arg<statically_named_arg<T, Char, N, Str>> : std::true_type {};
+
+template <typename T, typename Char, size_t N,
+ fmt::detail_exported::fixed_string<Char, N> Str>
+struct is_statically_named_arg<statically_named_arg<T, Char, N, Str>>
+ : std::true_type {};
+
+template <typename Char, size_t N,
+ fmt::detail_exported::fixed_string<Char, N> Str>
+struct udl_arg {
+ template <typename T> auto operator=(T&& value) const {
+ return statically_named_arg<T, Char, N, Str>(std::forward<T>(value));
+ }
+};
+# else
+template <typename Char> struct udl_arg {
+ const Char* str;
+
+ template <typename T> auto operator=(T&& value) const -> named_arg<Char, T> {
+ return {str, std::forward<T>(value)};
+ }
+};
+# endif
+#endif // FMT_USE_USER_DEFINED_LITERALS
+
+template <typename Locale, typename Char>
+auto vformat(const Locale& loc, basic_string_view<Char> fmt,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args)
+ -> std::basic_string<Char> {
+ auto buf = basic_memory_buffer<Char>();
+ detail::vformat_to(buf, fmt, args, detail::locale_ref(loc));
+ return {buf.data(), buf.size()};
+}
+
+using format_func = void (*)(detail::buffer<char>&, int, const char*);
+
+FMT_API void format_error_code(buffer<char>& out, int error_code,
+ string_view message) noexcept;
+
+FMT_API void report_error(format_func func, int error_code,
+ const char* message) noexcept;
+FMT_END_DETAIL_NAMESPACE
+
+FMT_API auto vsystem_error(int error_code, string_view format_str,
+ format_args args) -> std::system_error;
+
+/**
+ \rst
+ Constructs :class:`std::system_error` with a message formatted with
+ ``fmt::format(fmt, args...)``.
+ *error_code* is a system error code as given by ``errno``.
+
+ **Example**::
+
+ // This throws std::system_error with the description
+ // cannot open file 'madeup': No such file or directory
+ // or similar (system message may vary).
+ const char* filename = "madeup";
+ std::FILE* file = std::fopen(filename, "r");
+ if (!file)
+ throw fmt::system_error(errno, "cannot open file '{}'", filename);
+ \endrst
+*/
+template <typename... T>
+auto system_error(int error_code, format_string<T...> fmt, T&&... args)
+ -> std::system_error {
+ return vsystem_error(error_code, fmt, fmt::make_format_args(args...));
+}
+
+/**
+ \rst
+ Formats an error message for an error returned by an operating system or a
+ language runtime, for example a file opening error, and writes it to *out*.
+ The format is the same as the one used by ``std::system_error(ec, message)``
+ where ``ec`` is ``std::error_code(error_code, std::generic_category()})``.
+ It is implementation-defined but normally looks like:
+
+ .. parsed-literal::
+ *<message>*: *<system-message>*
+
+ where *<message>* is the passed message and *<system-message>* is the system
+ message corresponding to the error code.
+ *error_code* is a system error code as given by ``errno``.
+ \endrst
+ */
+FMT_API void format_system_error(detail::buffer<char>& out, int error_code,
+ const char* message) noexcept;
+
+// Reports a system error without throwing an exception.
+// Can be used to report errors from destructors.
+FMT_API void report_system_error(int error_code, const char* message) noexcept;
+
+/** Fast integer formatter. */
+class format_int {
+ private:
+ // Buffer should be large enough to hold all digits (digits10 + 1),
+ // a sign and a null character.
+ enum { buffer_size = std::numeric_limits<unsigned long long>::digits10 + 3 };
+ mutable char buffer_[buffer_size];
+ char* str_;
+
+ template <typename UInt> auto format_unsigned(UInt value) -> char* {
+ auto n = static_cast<detail::uint32_or_64_or_128_t<UInt>>(value);
+ return detail::format_decimal(buffer_, n, buffer_size - 1).begin;
+ }
+
+ template <typename Int> auto format_signed(Int value) -> char* {
+ auto abs_value = static_cast<detail::uint32_or_64_or_128_t<Int>>(value);
+ bool negative = value < 0;
+ if (negative) abs_value = 0 - abs_value;
+ auto begin = format_unsigned(abs_value);
+ if (negative) *--begin = '-';
+ return begin;
+ }
+
+ public:
+ explicit format_int(int value) : str_(format_signed(value)) {}
+ explicit format_int(long value) : str_(format_signed(value)) {}
+ explicit format_int(long long value) : str_(format_signed(value)) {}
+ explicit format_int(unsigned value) : str_(format_unsigned(value)) {}
+ explicit format_int(unsigned long value) : str_(format_unsigned(value)) {}
+ explicit format_int(unsigned long long value)
+ : str_(format_unsigned(value)) {}
+
+ /** Returns the number of characters written to the output buffer. */
+ auto size() const -> size_t {
+ return detail::to_unsigned(buffer_ - str_ + buffer_size - 1);
+ }
+
+ /**
+ Returns a pointer to the output buffer content. No terminating null
+ character is appended.
+ */
+ auto data() const -> const char* { return str_; }
+
+ /**
+ Returns a pointer to the output buffer content with terminating null
+ character appended.
+ */
+ auto c_str() const -> const char* {
+ buffer_[buffer_size - 1] = '\0';
+ return str_;
+ }
+
+ /**
+ \rst
+ Returns the content of the output buffer as an ``std::string``.
+ \endrst
+ */
+ auto str() const -> std::string { return std::string(str_, size()); }
+};
+
+template <typename T, typename Char>
+struct formatter<T, Char, enable_if_t<detail::has_format_as<T>::value>>
+ : private formatter<detail::format_as_t<T>> {
+ using base = formatter<detail::format_as_t<T>>;
+ using base::parse;
+
+ template <typename FormatContext>
+ auto format(const T& value, FormatContext& ctx) const -> decltype(ctx.out()) {
+ return base::format(format_as(value), ctx);
+ }
+};
+
+template <typename Char>
+struct formatter<void*, Char> : formatter<const void*, Char> {
+ template <typename FormatContext>
+ auto format(void* val, FormatContext& ctx) const -> decltype(ctx.out()) {
+ return formatter<const void*, Char>::format(val, ctx);
+ }
+};
+
+template <typename Char, size_t N>
+struct formatter<Char[N], Char> : formatter<basic_string_view<Char>, Char> {
+ template <typename FormatContext>
+ FMT_CONSTEXPR auto format(const Char* val, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ return formatter<basic_string_view<Char>, Char>::format(val, ctx);
+ }
+};
+
+/**
+ \rst
+ Converts ``p`` to ``const void*`` for pointer formatting.
+
+ **Example**::
+
+ auto s = fmt::format("{}", fmt::ptr(p));
+ \endrst
+ */
+template <typename T> auto ptr(T p) -> const void* {
+ static_assert(std::is_pointer<T>::value, "");
+ return detail::bit_cast<const void*>(p);
+}
+template <typename T, typename Deleter>
+auto ptr(const std::unique_ptr<T, Deleter>& p) -> const void* {
+ return p.get();
+}
+template <typename T> auto ptr(const std::shared_ptr<T>& p) -> const void* {
+ return p.get();
+}
+
+/**
+ \rst
+ Converts ``e`` to the underlying type.
+
+ **Example**::
+
+ enum class color { red, green, blue };
+ auto s = fmt::format("{}", fmt::underlying(color::red));
+ \endrst
+ */
+template <typename Enum>
+constexpr auto underlying(Enum e) noexcept -> underlying_t<Enum> {
+ return static_cast<underlying_t<Enum>>(e);
+}
+
+namespace enums {
+template <typename Enum, FMT_ENABLE_IF(std::is_enum<Enum>::value)>
+constexpr auto format_as(Enum e) noexcept -> underlying_t<Enum> {
+ return static_cast<underlying_t<Enum>>(e);
+}
+} // namespace enums
+
+class bytes {
+ private:
+ string_view data_;
+ friend struct formatter<bytes>;
+
+ public:
+ explicit bytes(string_view data) : data_(data) {}
+};
+
+template <> struct formatter<bytes> {
+ private:
+ detail::dynamic_format_specs<> specs_;
+
+ public:
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const char* {
+ return parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx,
+ detail::type::string_type);
+ }
+
+ template <typename FormatContext>
+ auto format(bytes b, FormatContext& ctx) -> decltype(ctx.out()) {
+ detail::handle_dynamic_spec<detail::width_checker>(specs_.width,
+ specs_.width_ref, ctx);
+ detail::handle_dynamic_spec<detail::precision_checker>(
+ specs_.precision, specs_.precision_ref, ctx);
+ return detail::write_bytes(ctx.out(), b.data_, specs_);
+ }
+};
+
+// group_digits_view is not derived from view because it copies the argument.
+template <typename T> struct group_digits_view { T value; };
+
+/**
+ \rst
+ Returns a view that formats an integer value using ',' as a locale-independent
+ thousands separator.
+
+ **Example**::
+
+ fmt::print("{}", fmt::group_digits(12345));
+ // Output: "12,345"
+ \endrst
+ */
+template <typename T> auto group_digits(T value) -> group_digits_view<T> {
+ return {value};
+}
+
+template <typename T> struct formatter<group_digits_view<T>> : formatter<T> {
+ private:
+ detail::dynamic_format_specs<> specs_;
+
+ public:
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const char* {
+ return parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx,
+ detail::type::int_type);
+ }
+
+ template <typename FormatContext>
+ auto format(group_digits_view<T> t, FormatContext& ctx)
+ -> decltype(ctx.out()) {
+ detail::handle_dynamic_spec<detail::width_checker>(specs_.width,
+ specs_.width_ref, ctx);
+ detail::handle_dynamic_spec<detail::precision_checker>(
+ specs_.precision, specs_.precision_ref, ctx);
+ return detail::write_int(
+ ctx.out(), static_cast<detail::uint64_or_128_t<T>>(t.value), 0, specs_,
+ detail::digit_grouping<char>("\3", ","));
+ }
+};
+
+// DEPRECATED! join_view will be moved to ranges.h.
+template <typename It, typename Sentinel, typename Char = char>
+struct join_view : detail::view {
+ It begin;
+ Sentinel end;
+ basic_string_view<Char> sep;
+
+ join_view(It b, Sentinel e, basic_string_view<Char> s)
+ : begin(b), end(e), sep(s) {}
+};
+
+template <typename It, typename Sentinel, typename Char>
+struct formatter<join_view<It, Sentinel, Char>, Char> {
+ private:
+ using value_type =
+#ifdef __cpp_lib_ranges
+ std::iter_value_t<It>;
+#else
+ typename std::iterator_traits<It>::value_type;
+#endif
+ formatter<remove_cvref_t<value_type>, Char> value_formatter_;
+
+ public:
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const Char* {
+ return value_formatter_.parse(ctx);
+ }
+
+ template <typename FormatContext>
+ auto format(const join_view<It, Sentinel, Char>& value,
+ FormatContext& ctx) const -> decltype(ctx.out()) {
+ auto it = value.begin;
+ auto out = ctx.out();
+ if (it != value.end) {
+ out = value_formatter_.format(*it, ctx);
+ ++it;
+ while (it != value.end) {
+ out = detail::copy_str<Char>(value.sep.begin(), value.sep.end(), out);
+ ctx.advance_to(out);
+ out = value_formatter_.format(*it, ctx);
+ ++it;
+ }
+ }
+ return out;
+ }
+};
+
+/**
+ Returns a view that formats the iterator range `[begin, end)` with elements
+ separated by `sep`.
+ */
+template <typename It, typename Sentinel>
+auto join(It begin, Sentinel end, string_view sep) -> join_view<It, Sentinel> {
+ return {begin, end, sep};
+}
+
+/**
+ \rst
+ Returns a view that formats `range` with elements separated by `sep`.
+
+ **Example**::
+
+ std::vector<int> v = {1, 2, 3};
+ fmt::print("{}", fmt::join(v, ", "));
+ // Output: "1, 2, 3"
+
+ ``fmt::join`` applies passed format specifiers to the range elements::
+
+ fmt::print("{:02}", fmt::join(v, ", "));
+ // Output: "01, 02, 03"
+ \endrst
+ */
+template <typename Range>
+auto join(Range&& range, string_view sep)
+ -> join_view<detail::iterator_t<Range>, detail::sentinel_t<Range>> {
+ return join(std::begin(range), std::end(range), sep);
+}
+
+/**
+ \rst
+ Converts *value* to ``std::string`` using the default format for type *T*.
+
+ **Example**::
+
+ #include <fmt/format.h>
+
+ std::string answer = fmt::to_string(42);
+ \endrst
+ */
+template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+inline auto to_string(const T& value) -> std::string {
+ auto buffer = memory_buffer();
+ detail::write<char>(appender(buffer), value);
+ return {buffer.data(), buffer.size()};
+}
+
+template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+FMT_NODISCARD inline auto to_string(T value) -> std::string {
+ // The buffer should be large enough to store the number including the sign
+ // or "false" for bool.
+ constexpr int max_size = detail::digits10<T>() + 2;
+ char buffer[max_size > 5 ? static_cast<unsigned>(max_size) : 5];
+ char* begin = buffer;
+ return std::string(begin, detail::write<char>(begin, value));
+}
+
+template <typename Char, size_t SIZE>
+FMT_NODISCARD auto to_string(const basic_memory_buffer<Char, SIZE>& buf)
+ -> std::basic_string<Char> {
+ auto size = buf.size();
+ detail::assume(size < std::basic_string<Char>().max_size());
+ return std::basic_string<Char>(buf.data(), size);
+}
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+template <typename Char>
+void vformat_to(buffer<Char>& buf, basic_string_view<Char> fmt,
+ typename vformat_args<Char>::type args, locale_ref loc) {
+ auto out = buffer_appender<Char>(buf);
+ if (fmt.size() == 2 && equal2(fmt.data(), "{}")) {
+ auto arg = args.get(0);
+ if (!arg) error_handler().on_error("argument not found");
+ visit_format_arg(default_arg_formatter<Char>{out, args, loc}, arg);
+ return;
+ }
+
+ struct format_handler : error_handler {
+ basic_format_parse_context<Char> parse_context;
+ buffer_context<Char> context;
+
+ format_handler(buffer_appender<Char> p_out, basic_string_view<Char> str,
+ basic_format_args<buffer_context<Char>> p_args,
+ locale_ref p_loc)
+ : parse_context(str), context(p_out, p_args, p_loc) {}
+
+ void on_text(const Char* begin, const Char* end) {
+ auto text = basic_string_view<Char>(begin, to_unsigned(end - begin));
+ context.advance_to(write<Char>(context.out(), text));
+ }
+
+ FMT_CONSTEXPR auto on_arg_id() -> int {
+ return parse_context.next_arg_id();
+ }
+ FMT_CONSTEXPR auto on_arg_id(int id) -> int {
+ return parse_context.check_arg_id(id), id;
+ }
+ FMT_CONSTEXPR auto on_arg_id(basic_string_view<Char> id) -> int {
+ int arg_id = context.arg_id(id);
+ if (arg_id < 0) on_error("argument not found");
+ return arg_id;
+ }
+
+ FMT_INLINE void on_replacement_field(int id, const Char*) {
+ auto arg = get_arg(context, id);
+ context.advance_to(visit_format_arg(
+ default_arg_formatter<Char>{context.out(), context.args(),
+ context.locale()},
+ arg));
+ }
+
+ auto on_format_specs(int id, const Char* begin, const Char* end)
+ -> const Char* {
+ auto arg = get_arg(context, id);
+ if (arg.type() == type::custom_type) {
+ parse_context.advance_to(begin);
+ visit_format_arg(custom_formatter<Char>{parse_context, context}, arg);
+ return parse_context.begin();
+ }
+ auto specs = detail::dynamic_format_specs<Char>();
+ begin = parse_format_specs(begin, end, specs, parse_context, arg.type());
+ detail::handle_dynamic_spec<detail::width_checker>(
+ specs.width, specs.width_ref, context);
+ detail::handle_dynamic_spec<detail::precision_checker>(
+ specs.precision, specs.precision_ref, context);
+ if (begin == end || *begin != '}')
+ on_error("missing '}' in format string");
+ auto f = arg_formatter<Char>{context.out(), specs, context.locale()};
+ context.advance_to(visit_format_arg(f, arg));
+ return begin;
+ }
+ };
+ detail::parse_format_string<false>(fmt, format_handler(out, fmt, args, loc));
+}
+
+#ifndef FMT_HEADER_ONLY
+extern template FMT_API void vformat_to(buffer<char>&, string_view,
+ typename vformat_args<>::type,
+ locale_ref);
+extern template FMT_API auto thousands_sep_impl<char>(locale_ref)
+ -> thousands_sep_result<char>;
+extern template FMT_API auto thousands_sep_impl<wchar_t>(locale_ref)
+ -> thousands_sep_result<wchar_t>;
+extern template FMT_API auto decimal_point_impl(locale_ref) -> char;
+extern template FMT_API auto decimal_point_impl(locale_ref) -> wchar_t;
+#endif // FMT_HEADER_ONLY
+
+FMT_END_DETAIL_NAMESPACE
+
+#if FMT_USE_USER_DEFINED_LITERALS
+inline namespace literals {
+/**
+ \rst
+ User-defined literal equivalent of :func:`fmt::arg`.
+
+ **Example**::
+
+ using namespace fmt::literals;
+ fmt::print("Elapsed time: {s:.2f} seconds", "s"_a=1.23);
+ \endrst
+ */
+# if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <detail_exported::fixed_string Str> constexpr auto operator""_a() {
+ using char_t = remove_cvref_t<decltype(Str.data[0])>;
+ return detail::udl_arg<char_t, sizeof(Str.data) / sizeof(char_t), Str>();
+}
+# else
+constexpr auto operator"" _a(const char* s, size_t) -> detail::udl_arg<char> {
+ return {s};
+}
+# endif
+} // namespace literals
+#endif // FMT_USE_USER_DEFINED_LITERALS
+
+template <typename Locale, FMT_ENABLE_IF(detail::is_locale<Locale>::value)>
+inline auto vformat(const Locale& loc, string_view fmt, format_args args)
+ -> std::string {
+ return detail::vformat(loc, fmt, args);
+}
+
+template <typename Locale, typename... T,
+ FMT_ENABLE_IF(detail::is_locale<Locale>::value)>
+inline auto format(const Locale& loc, format_string<T...> fmt, T&&... args)
+ -> std::string {
+ return fmt::vformat(loc, string_view(fmt), fmt::make_format_args(args...));
+}
+
+template <typename OutputIt, typename Locale,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value&&
+ detail::is_locale<Locale>::value)>
+auto vformat_to(OutputIt out, const Locale& loc, string_view fmt,
+ format_args args) -> OutputIt {
+ using detail::get_buffer;
+ auto&& buf = get_buffer<char>(out);
+ detail::vformat_to(buf, fmt, args, detail::locale_ref(loc));
+ return detail::get_iterator(buf, out);
+}
+
+template <typename OutputIt, typename Locale, typename... T,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value&&
+ detail::is_locale<Locale>::value)>
+FMT_INLINE auto format_to(OutputIt out, const Locale& loc,
+ format_string<T...> fmt, T&&... args) -> OutputIt {
+ return vformat_to(out, loc, fmt, fmt::make_format_args(args...));
+}
+
+template <typename Locale, typename... T,
+ FMT_ENABLE_IF(detail::is_locale<Locale>::value)>
+FMT_NODISCARD FMT_INLINE auto formatted_size(const Locale& loc,
+ format_string<T...> fmt,
+ T&&... args) -> size_t {
+ auto buf = detail::counting_buffer<>();
+ detail::vformat_to<char>(buf, fmt, fmt::make_format_args(args...),
+ detail::locale_ref(loc));
+ return buf.count();
+}
+
+FMT_END_EXPORT
+
+template <typename T, typename Char>
+template <typename FormatContext>
+FMT_CONSTEXPR FMT_INLINE auto
+formatter<T, Char,
+ enable_if_t<detail::type_constant<T, Char>::value !=
+ detail::type::custom_type>>::format(const T& val,
+ FormatContext& ctx)
+ const -> decltype(ctx.out()) {
+ if (specs_.width_ref.kind != detail::arg_id_kind::none ||
+ specs_.precision_ref.kind != detail::arg_id_kind::none) {
+ auto specs = specs_;
+ detail::handle_dynamic_spec<detail::width_checker>(specs.width,
+ specs.width_ref, ctx);
+ detail::handle_dynamic_spec<detail::precision_checker>(
+ specs.precision, specs.precision_ref, ctx);
+ return detail::write<Char>(ctx.out(), val, specs, ctx.locale());
+ }
+ return detail::write<Char>(ctx.out(), val, specs_, ctx.locale());
+}
+
+FMT_END_NAMESPACE
+
+#ifdef FMT_HEADER_ONLY
+# define FMT_FUNC inline
+# include "format-inl.h"
+#else
+# define FMT_FUNC
+#endif
+
+#endif // FMT_FORMAT_H_
diff --git a/contrib/fmt/include/fmt/locale.h b/contrib/fmt/include/fmt/locale.h
new file mode 100644
index 0000000..7571b52
--- /dev/null
+++ b/contrib/fmt/include/fmt/locale.h
@@ -0,0 +1,2 @@
+#include "xchar.h"
+#warning fmt/locale.h is deprecated, include fmt/format.h or fmt/xchar.h instead
diff --git a/contrib/fmt/include/fmt/os.h b/contrib/fmt/include/fmt/os.h
new file mode 100644
index 0000000..ec29040
--- /dev/null
+++ b/contrib/fmt/include/fmt/os.h
@@ -0,0 +1,451 @@
+// Formatting library for C++ - optional OS-specific functionality
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_OS_H_
+#define FMT_OS_H_
+
+#include <cerrno>
+#include <cstddef>
+#include <cstdio>
+#include <system_error> // std::system_error
+
+#if defined __APPLE__ || defined(__FreeBSD__)
+# include <xlocale.h> // for LC_NUMERIC_MASK on OS X
+#endif
+
+#include "format.h"
+
+#ifndef FMT_USE_FCNTL
+// UWP doesn't provide _pipe.
+# if FMT_HAS_INCLUDE("winapifamily.h")
+# include <winapifamily.h>
+# endif
+# if (FMT_HAS_INCLUDE(<fcntl.h>) || defined(__APPLE__) || \
+ defined(__linux__)) && \
+ (!defined(WINAPI_FAMILY) || \
+ (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP))
+# include <fcntl.h> // for O_RDONLY
+# define FMT_USE_FCNTL 1
+# else
+# define FMT_USE_FCNTL 0
+# endif
+#endif
+
+#ifndef FMT_POSIX
+# if defined(_WIN32) && !defined(__MINGW32__)
+// Fix warnings about deprecated symbols.
+# define FMT_POSIX(call) _##call
+# else
+# define FMT_POSIX(call) call
+# endif
+#endif
+
+// Calls to system functions are wrapped in FMT_SYSTEM for testability.
+#ifdef FMT_SYSTEM
+# define FMT_POSIX_CALL(call) FMT_SYSTEM(call)
+#else
+# define FMT_SYSTEM(call) ::call
+# ifdef _WIN32
+// Fix warnings about deprecated symbols.
+# define FMT_POSIX_CALL(call) ::_##call
+# else
+# define FMT_POSIX_CALL(call) ::call
+# endif
+#endif
+
+// Retries the expression while it evaluates to error_result and errno
+// equals to EINTR.
+#ifndef _WIN32
+# define FMT_RETRY_VAL(result, expression, error_result) \
+ do { \
+ (result) = (expression); \
+ } while ((result) == (error_result) && errno == EINTR)
+#else
+# define FMT_RETRY_VAL(result, expression, error_result) result = (expression)
+#endif
+
+#define FMT_RETRY(result, expression) FMT_RETRY_VAL(result, expression, -1)
+
+FMT_BEGIN_NAMESPACE
+FMT_BEGIN_EXPORT
+
+/**
+ \rst
+ A reference to a null-terminated string. It can be constructed from a C
+ string or ``std::string``.
+
+ You can use one of the following type aliases for common character types:
+
+ +---------------+-----------------------------+
+ | Type | Definition |
+ +===============+=============================+
+ | cstring_view | basic_cstring_view<char> |
+ +---------------+-----------------------------+
+ | wcstring_view | basic_cstring_view<wchar_t> |
+ +---------------+-----------------------------+
+
+ This class is most useful as a parameter type to allow passing
+ different types of strings to a function, for example::
+
+ template <typename... Args>
+ std::string format(cstring_view format_str, const Args & ... args);
+
+ format("{}", 42);
+ format(std::string("{}"), 42);
+ \endrst
+ */
+template <typename Char> class basic_cstring_view {
+ private:
+ const Char* data_;
+
+ public:
+ /** Constructs a string reference object from a C string. */
+ basic_cstring_view(const Char* s) : data_(s) {}
+
+ /**
+ \rst
+ Constructs a string reference from an ``std::string`` object.
+ \endrst
+ */
+ basic_cstring_view(const std::basic_string<Char>& s) : data_(s.c_str()) {}
+
+ /** Returns the pointer to a C string. */
+ const Char* c_str() const { return data_; }
+};
+
+using cstring_view = basic_cstring_view<char>;
+using wcstring_view = basic_cstring_view<wchar_t>;
+
+#ifdef _WIN32
+FMT_API const std::error_category& system_category() noexcept;
+
+FMT_BEGIN_DETAIL_NAMESPACE
+FMT_API void format_windows_error(buffer<char>& out, int error_code,
+ const char* message) noexcept;
+FMT_END_DETAIL_NAMESPACE
+
+FMT_API std::system_error vwindows_error(int error_code, string_view format_str,
+ format_args args);
+
+/**
+ \rst
+ Constructs a :class:`std::system_error` object with the description
+ of the form
+
+ .. parsed-literal::
+ *<message>*: *<system-message>*
+
+ where *<message>* is the formatted message and *<system-message>* is the
+ system message corresponding to the error code.
+ *error_code* is a Windows error code as given by ``GetLastError``.
+ If *error_code* is not a valid error code such as -1, the system message
+ will look like "error -1".
+
+ **Example**::
+
+ // This throws a system_error with the description
+ // cannot open file 'madeup': The system cannot find the file specified.
+ // or similar (system message may vary).
+ const char *filename = "madeup";
+ LPOFSTRUCT of = LPOFSTRUCT();
+ HFILE file = OpenFile(filename, &of, OF_READ);
+ if (file == HFILE_ERROR) {
+ throw fmt::windows_error(GetLastError(),
+ "cannot open file '{}'", filename);
+ }
+ \endrst
+*/
+template <typename... Args>
+std::system_error windows_error(int error_code, string_view message,
+ const Args&... args) {
+ return vwindows_error(error_code, message, fmt::make_format_args(args...));
+}
+
+// Reports a Windows error without throwing an exception.
+// Can be used to report errors from destructors.
+FMT_API void report_windows_error(int error_code, const char* message) noexcept;
+#else
+inline const std::error_category& system_category() noexcept {
+ return std::system_category();
+}
+#endif // _WIN32
+
+// std::system is not available on some platforms such as iOS (#2248).
+#ifdef __OSX__
+template <typename S, typename... Args, typename Char = char_t<S>>
+void say(const S& format_str, Args&&... args) {
+ std::system(format("say \"{}\"", format(format_str, args...)).c_str());
+}
+#endif
+
+// A buffered file.
+class buffered_file {
+ private:
+ FILE* file_;
+
+ friend class file;
+
+ explicit buffered_file(FILE* f) : file_(f) {}
+
+ public:
+ buffered_file(const buffered_file&) = delete;
+ void operator=(const buffered_file&) = delete;
+
+ // Constructs a buffered_file object which doesn't represent any file.
+ buffered_file() noexcept : file_(nullptr) {}
+
+ // Destroys the object closing the file it represents if any.
+ FMT_API ~buffered_file() noexcept;
+
+ public:
+ buffered_file(buffered_file&& other) noexcept : file_(other.file_) {
+ other.file_ = nullptr;
+ }
+
+ buffered_file& operator=(buffered_file&& other) {
+ close();
+ file_ = other.file_;
+ other.file_ = nullptr;
+ return *this;
+ }
+
+ // Opens a file.
+ FMT_API buffered_file(cstring_view filename, cstring_view mode);
+
+ // Closes the file.
+ FMT_API void close();
+
+ // Returns the pointer to a FILE object representing this file.
+ FILE* get() const noexcept { return file_; }
+
+ FMT_API int descriptor() const;
+
+ void vprint(string_view format_str, format_args args) {
+ fmt::vprint(file_, format_str, args);
+ }
+
+ template <typename... Args>
+ inline void print(string_view format_str, const Args&... args) {
+ vprint(format_str, fmt::make_format_args(args...));
+ }
+};
+
+#if FMT_USE_FCNTL
+// A file. Closed file is represented by a file object with descriptor -1.
+// Methods that are not declared with noexcept may throw
+// fmt::system_error in case of failure. Note that some errors such as
+// closing the file multiple times will cause a crash on Windows rather
+// than an exception. You can get standard behavior by overriding the
+// invalid parameter handler with _set_invalid_parameter_handler.
+class FMT_API file {
+ private:
+ int fd_; // File descriptor.
+
+ // Constructs a file object with a given descriptor.
+ explicit file(int fd) : fd_(fd) {}
+
+ public:
+ // Possible values for the oflag argument to the constructor.
+ enum {
+ RDONLY = FMT_POSIX(O_RDONLY), // Open for reading only.
+ WRONLY = FMT_POSIX(O_WRONLY), // Open for writing only.
+ RDWR = FMT_POSIX(O_RDWR), // Open for reading and writing.
+ CREATE = FMT_POSIX(O_CREAT), // Create if the file doesn't exist.
+ APPEND = FMT_POSIX(O_APPEND), // Open in append mode.
+ TRUNC = FMT_POSIX(O_TRUNC) // Truncate the content of the file.
+ };
+
+ // Constructs a file object which doesn't represent any file.
+ file() noexcept : fd_(-1) {}
+
+ // Opens a file and constructs a file object representing this file.
+ file(cstring_view path, int oflag);
+
+ public:
+ file(const file&) = delete;
+ void operator=(const file&) = delete;
+
+ file(file&& other) noexcept : fd_(other.fd_) { other.fd_ = -1; }
+
+ // Move assignment is not noexcept because close may throw.
+ file& operator=(file&& other) {
+ close();
+ fd_ = other.fd_;
+ other.fd_ = -1;
+ return *this;
+ }
+
+ // Destroys the object closing the file it represents if any.
+ ~file() noexcept;
+
+ // Returns the file descriptor.
+ int descriptor() const noexcept { return fd_; }
+
+ // Closes the file.
+ void close();
+
+ // Returns the file size. The size has signed type for consistency with
+ // stat::st_size.
+ long long size() const;
+
+ // Attempts to read count bytes from the file into the specified buffer.
+ size_t read(void* buffer, size_t count);
+
+ // Attempts to write count bytes from the specified buffer to the file.
+ size_t write(const void* buffer, size_t count);
+
+ // Duplicates a file descriptor with the dup function and returns
+ // the duplicate as a file object.
+ static file dup(int fd);
+
+ // Makes fd be the copy of this file descriptor, closing fd first if
+ // necessary.
+ void dup2(int fd);
+
+ // Makes fd be the copy of this file descriptor, closing fd first if
+ // necessary.
+ void dup2(int fd, std::error_code& ec) noexcept;
+
+ // Creates a pipe setting up read_end and write_end file objects for reading
+ // and writing respectively.
+ static void pipe(file& read_end, file& write_end);
+
+ // Creates a buffered_file object associated with this file and detaches
+ // this file object from the file.
+ buffered_file fdopen(const char* mode);
+
+# if defined(_WIN32) && !defined(__MINGW32__)
+ // Opens a file and constructs a file object representing this file by
+ // wcstring_view filename. Windows only.
+ static file open_windows_file(wcstring_view path, int oflag);
+# endif
+};
+
+// Returns the memory page size.
+long getpagesize();
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+struct buffer_size {
+ buffer_size() = default;
+ size_t value = 0;
+ buffer_size operator=(size_t val) const {
+ auto bs = buffer_size();
+ bs.value = val;
+ return bs;
+ }
+};
+
+struct ostream_params {
+ int oflag = file::WRONLY | file::CREATE | file::TRUNC;
+ size_t buffer_size = BUFSIZ > 32768 ? BUFSIZ : 32768;
+
+ ostream_params() {}
+
+ template <typename... T>
+ ostream_params(T... params, int new_oflag) : ostream_params(params...) {
+ oflag = new_oflag;
+ }
+
+ template <typename... T>
+ ostream_params(T... params, detail::buffer_size bs)
+ : ostream_params(params...) {
+ this->buffer_size = bs.value;
+ }
+
+// Intel has a bug that results in failure to deduce a constructor
+// for empty parameter packs.
+# if defined(__INTEL_COMPILER) && __INTEL_COMPILER < 2000
+ ostream_params(int new_oflag) : oflag(new_oflag) {}
+ ostream_params(detail::buffer_size bs) : buffer_size(bs.value) {}
+# endif
+};
+
+class file_buffer final : public buffer<char> {
+ file file_;
+
+ FMT_API void grow(size_t) override;
+
+ public:
+ FMT_API file_buffer(cstring_view path, const ostream_params& params);
+ FMT_API file_buffer(file_buffer&& other);
+ FMT_API ~file_buffer();
+
+ void flush() {
+ if (size() == 0) return;
+ file_.write(data(), size() * sizeof(data()[0]));
+ clear();
+ }
+
+ void close() {
+ flush();
+ file_.close();
+ }
+};
+
+FMT_END_DETAIL_NAMESPACE
+
+// Added {} below to work around default constructor error known to
+// occur in Xcode versions 7.2.1 and 8.2.1.
+constexpr detail::buffer_size buffer_size{};
+
+/** A fast output stream which is not thread-safe. */
+class FMT_API ostream {
+ private:
+ FMT_MSC_WARNING(suppress : 4251)
+ detail::file_buffer buffer_;
+
+ ostream(cstring_view path, const detail::ostream_params& params)
+ : buffer_(path, params) {}
+
+ public:
+ ostream(ostream&& other) : buffer_(std::move(other.buffer_)) {}
+
+ ~ostream();
+
+ void flush() { buffer_.flush(); }
+
+ template <typename... T>
+ friend ostream output_file(cstring_view path, T... params);
+
+ void close() { buffer_.close(); }
+
+ /**
+ Formats ``args`` according to specifications in ``fmt`` and writes the
+ output to the file.
+ */
+ template <typename... T> void print(format_string<T...> fmt, T&&... args) {
+ vformat_to(detail::buffer_appender<char>(buffer_), fmt,
+ fmt::make_format_args(args...));
+ }
+};
+
+/**
+ \rst
+ Opens a file for writing. Supported parameters passed in *params*:
+
+ * ``<integer>``: Flags passed to `open
+ <https://pubs.opengroup.org/onlinepubs/007904875/functions/open.html>`_
+ (``file::WRONLY | file::CREATE | file::TRUNC`` by default)
+ * ``buffer_size=<integer>``: Output buffer size
+
+ **Example**::
+
+ auto out = fmt::output_file("guide.txt");
+ out.print("Don't {}", "Panic");
+ \endrst
+ */
+template <typename... T>
+inline ostream output_file(cstring_view path, T... params) {
+ return {path, detail::ostream_params(params...)};
+}
+#endif // FMT_USE_FCNTL
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_OS_H_
diff --git a/contrib/fmt/include/fmt/ostream.h b/contrib/fmt/include/fmt/ostream.h
new file mode 100644
index 0000000..ce65909
--- /dev/null
+++ b/contrib/fmt/include/fmt/ostream.h
@@ -0,0 +1,209 @@
+// Formatting library for C++ - std::ostream support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_OSTREAM_H_
+#define FMT_OSTREAM_H_
+
+#include <fstream> // std::filebuf
+
+#if defined(_WIN32) && defined(__GLIBCXX__)
+# include <ext/stdio_filebuf.h>
+# include <ext/stdio_sync_filebuf.h>
+#elif defined(_WIN32) && defined(_LIBCPP_VERSION)
+# include <__std_stream>
+#endif
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+
+namespace detail {
+
+// Generate a unique explicit instantion in every translation unit using a tag
+// type in an anonymous namespace.
+namespace {
+struct file_access_tag {};
+} // namespace
+template <typename Tag, typename BufType, FILE* BufType::*FileMemberPtr>
+class file_access {
+ friend auto get_file(BufType& obj) -> FILE* { return obj.*FileMemberPtr; }
+};
+
+#if FMT_MSC_VERSION
+template class file_access<file_access_tag, std::filebuf,
+ &std::filebuf::_Myfile>;
+auto get_file(std::filebuf&) -> FILE*;
+#elif defined(_WIN32) && defined(_LIBCPP_VERSION)
+template class file_access<file_access_tag, std::__stdoutbuf<char>,
+ &std::__stdoutbuf<char>::__file_>;
+auto get_file(std::__stdoutbuf<char>&) -> FILE*;
+#endif
+
+inline bool write_ostream_unicode(std::ostream& os, fmt::string_view data) {
+#if FMT_MSC_VERSION
+ if (auto* buf = dynamic_cast<std::filebuf*>(os.rdbuf()))
+ if (FILE* f = get_file(*buf)) return write_console(f, data);
+#elif defined(_WIN32) && defined(__GLIBCXX__)
+ auto* rdbuf = os.rdbuf();
+ FILE* c_file;
+ if (auto* sfbuf = dynamic_cast<__gnu_cxx::stdio_sync_filebuf<char>*>(rdbuf))
+ c_file = sfbuf->file();
+ else if (auto* fbuf = dynamic_cast<__gnu_cxx::stdio_filebuf<char>*>(rdbuf))
+ c_file = fbuf->file();
+ else
+ return false;
+ if (c_file) return write_console(c_file, data);
+#elif defined(_WIN32) && defined(_LIBCPP_VERSION)
+ if (auto* buf = dynamic_cast<std::__stdoutbuf<char>*>(os.rdbuf()))
+ if (FILE* f = get_file(*buf)) return write_console(f, data);
+#else
+ ignore_unused(os, data);
+#endif
+ return false;
+}
+inline bool write_ostream_unicode(std::wostream&,
+ fmt::basic_string_view<wchar_t>) {
+ return false;
+}
+
+// Write the content of buf to os.
+// It is a separate function rather than a part of vprint to simplify testing.
+template <typename Char>
+void write_buffer(std::basic_ostream<Char>& os, buffer<Char>& buf) {
+ const Char* buf_data = buf.data();
+ using unsigned_streamsize = std::make_unsigned<std::streamsize>::type;
+ unsigned_streamsize size = buf.size();
+ unsigned_streamsize max_size = to_unsigned(max_value<std::streamsize>());
+ do {
+ unsigned_streamsize n = size <= max_size ? size : max_size;
+ os.write(buf_data, static_cast<std::streamsize>(n));
+ buf_data += n;
+ size -= n;
+ } while (size != 0);
+}
+
+template <typename Char, typename T>
+void format_value(buffer<Char>& buf, const T& value,
+ locale_ref loc = locale_ref()) {
+ auto&& format_buf = formatbuf<std::basic_streambuf<Char>>(buf);
+ auto&& output = std::basic_ostream<Char>(&format_buf);
+#if !defined(FMT_STATIC_THOUSANDS_SEPARATOR)
+ if (loc) output.imbue(loc.get<std::locale>());
+#endif
+ output << value;
+ output.exceptions(std::ios_base::failbit | std::ios_base::badbit);
+}
+
+template <typename T> struct streamed_view { const T& value; };
+
+} // namespace detail
+
+// Formats an object of type T that has an overloaded ostream operator<<.
+template <typename Char>
+struct basic_ostream_formatter : formatter<basic_string_view<Char>, Char> {
+ void set_debug_format() = delete;
+
+ template <typename T, typename OutputIt>
+ auto format(const T& value, basic_format_context<OutputIt, Char>& ctx) const
+ -> OutputIt {
+ auto buffer = basic_memory_buffer<Char>();
+ detail::format_value(buffer, value, ctx.locale());
+ return formatter<basic_string_view<Char>, Char>::format(
+ {buffer.data(), buffer.size()}, ctx);
+ }
+};
+
+using ostream_formatter = basic_ostream_formatter<char>;
+
+template <typename T, typename Char>
+struct formatter<detail::streamed_view<T>, Char>
+ : basic_ostream_formatter<Char> {
+ template <typename OutputIt>
+ auto format(detail::streamed_view<T> view,
+ basic_format_context<OutputIt, Char>& ctx) const -> OutputIt {
+ return basic_ostream_formatter<Char>::format(view.value, ctx);
+ }
+};
+
+/**
+ \rst
+ Returns a view that formats `value` via an ostream ``operator<<``.
+
+ **Example**::
+
+ fmt::print("Current thread id: {}\n",
+ fmt::streamed(std::this_thread::get_id()));
+ \endrst
+ */
+template <typename T>
+auto streamed(const T& value) -> detail::streamed_view<T> {
+ return {value};
+}
+
+namespace detail {
+
+inline void vprint_directly(std::ostream& os, string_view format_str,
+ format_args args) {
+ auto buffer = memory_buffer();
+ detail::vformat_to(buffer, format_str, args);
+ detail::write_buffer(os, buffer);
+}
+
+} // namespace detail
+
+FMT_MODULE_EXPORT template <typename Char>
+void vprint(std::basic_ostream<Char>& os,
+ basic_string_view<type_identity_t<Char>> format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args) {
+ auto buffer = basic_memory_buffer<Char>();
+ detail::vformat_to(buffer, format_str, args);
+ if (detail::write_ostream_unicode(os, {buffer.data(), buffer.size()})) return;
+ detail::write_buffer(os, buffer);
+}
+
+/**
+ \rst
+ Prints formatted data to the stream *os*.
+
+ **Example**::
+
+ fmt::print(cerr, "Don't {}!", "panic");
+ \endrst
+ */
+FMT_MODULE_EXPORT template <typename... T>
+void print(std::ostream& os, format_string<T...> fmt, T&&... args) {
+ const auto& vargs = fmt::make_format_args(args...);
+ if (detail::is_utf8())
+ vprint(os, fmt, vargs);
+ else
+ detail::vprint_directly(os, fmt, vargs);
+}
+
+FMT_MODULE_EXPORT
+template <typename... Args>
+void print(std::wostream& os,
+ basic_format_string<wchar_t, type_identity_t<Args>...> fmt,
+ Args&&... args) {
+ vprint(os, fmt, fmt::make_format_args<buffer_context<wchar_t>>(args...));
+}
+
+FMT_MODULE_EXPORT template <typename... T>
+void println(std::ostream& os, format_string<T...> fmt, T&&... args) {
+ fmt::print(os, "{}\n", fmt::format(fmt, std::forward<T>(args)...));
+}
+
+FMT_MODULE_EXPORT
+template <typename... Args>
+void println(std::wostream& os,
+ basic_format_string<wchar_t, type_identity_t<Args>...> fmt,
+ Args&&... args) {
+ print(os, L"{}\n", fmt::format(fmt, std::forward<Args>(args)...));
+}
+
+FMT_END_NAMESPACE
+
+#endif // FMT_OSTREAM_H_
diff --git a/contrib/fmt/include/fmt/posix.h b/contrib/fmt/include/fmt/posix.h
new file mode 100644
index 0000000..da19e9d
--- /dev/null
+++ b/contrib/fmt/include/fmt/posix.h
@@ -0,0 +1,2 @@
+#include "os.h"
+#warning "fmt/posix.h is deprecated; use fmt/os.h instead"
diff --git a/contrib/fmt/include/fmt/printf.h b/contrib/fmt/include/fmt/printf.h
new file mode 100644
index 0000000..554715e
--- /dev/null
+++ b/contrib/fmt/include/fmt/printf.h
@@ -0,0 +1,679 @@
+// Formatting library for C++ - legacy printf implementation
+//
+// Copyright (c) 2012 - 2016, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_PRINTF_H_
+#define FMT_PRINTF_H_
+
+#include <algorithm> // std::max
+#include <limits> // std::numeric_limits
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+FMT_BEGIN_EXPORT
+
+template <typename T> struct printf_formatter { printf_formatter() = delete; };
+
+template <typename Char>
+class basic_printf_parse_context : public basic_format_parse_context<Char> {
+ using basic_format_parse_context<Char>::basic_format_parse_context;
+};
+
+template <typename OutputIt, typename Char> class basic_printf_context {
+ private:
+ OutputIt out_;
+ basic_format_args<basic_printf_context> args_;
+
+ public:
+ using char_type = Char;
+ using format_arg = basic_format_arg<basic_printf_context>;
+ using parse_context_type = basic_printf_parse_context<Char>;
+ template <typename T> using formatter_type = printf_formatter<T>;
+
+ /**
+ \rst
+ Constructs a ``printf_context`` object. References to the arguments are
+ stored in the context object so make sure they have appropriate lifetimes.
+ \endrst
+ */
+ basic_printf_context(OutputIt out,
+ basic_format_args<basic_printf_context> args)
+ : out_(out), args_(args) {}
+
+ OutputIt out() { return out_; }
+ void advance_to(OutputIt it) { out_ = it; }
+
+ detail::locale_ref locale() { return {}; }
+
+ format_arg arg(int id) const { return args_.get(id); }
+
+ FMT_CONSTEXPR void on_error(const char* message) {
+ detail::error_handler().on_error(message);
+ }
+};
+
+FMT_BEGIN_DETAIL_NAMESPACE
+
+// Checks if a value fits in int - used to avoid warnings about comparing
+// signed and unsigned integers.
+template <bool IsSigned> struct int_checker {
+ template <typename T> static bool fits_in_int(T value) {
+ unsigned max = max_value<int>();
+ return value <= max;
+ }
+ static bool fits_in_int(bool) { return true; }
+};
+
+template <> struct int_checker<true> {
+ template <typename T> static bool fits_in_int(T value) {
+ return value >= (std::numeric_limits<int>::min)() &&
+ value <= max_value<int>();
+ }
+ static bool fits_in_int(int) { return true; }
+};
+
+class printf_precision_handler {
+ public:
+ template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+ int operator()(T value) {
+ if (!int_checker<std::numeric_limits<T>::is_signed>::fits_in_int(value))
+ throw_format_error("number is too big");
+ return (std::max)(static_cast<int>(value), 0);
+ }
+
+ template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+ int operator()(T) {
+ throw_format_error("precision is not integer");
+ return 0;
+ }
+};
+
+// An argument visitor that returns true iff arg is a zero integer.
+class is_zero_int {
+ public:
+ template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+ bool operator()(T value) {
+ return value == 0;
+ }
+
+ template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+ bool operator()(T) {
+ return false;
+ }
+};
+
+template <typename T> struct make_unsigned_or_bool : std::make_unsigned<T> {};
+
+template <> struct make_unsigned_or_bool<bool> { using type = bool; };
+
+template <typename T, typename Context> class arg_converter {
+ private:
+ using char_type = typename Context::char_type;
+
+ basic_format_arg<Context>& arg_;
+ char_type type_;
+
+ public:
+ arg_converter(basic_format_arg<Context>& arg, char_type type)
+ : arg_(arg), type_(type) {}
+
+ void operator()(bool value) {
+ if (type_ != 's') operator()<bool>(value);
+ }
+
+ template <typename U, FMT_ENABLE_IF(std::is_integral<U>::value)>
+ void operator()(U value) {
+ bool is_signed = type_ == 'd' || type_ == 'i';
+ using target_type = conditional_t<std::is_same<T, void>::value, U, T>;
+ if (const_check(sizeof(target_type) <= sizeof(int))) {
+ // Extra casts are used to silence warnings.
+ if (is_signed) {
+ arg_ = detail::make_arg<Context>(
+ static_cast<int>(static_cast<target_type>(value)));
+ } else {
+ using unsigned_type = typename make_unsigned_or_bool<target_type>::type;
+ arg_ = detail::make_arg<Context>(
+ static_cast<unsigned>(static_cast<unsigned_type>(value)));
+ }
+ } else {
+ if (is_signed) {
+ // glibc's printf doesn't sign extend arguments of smaller types:
+ // std::printf("%lld", -42); // prints "4294967254"
+ // but we don't have to do the same because it's a UB.
+ arg_ = detail::make_arg<Context>(static_cast<long long>(value));
+ } else {
+ arg_ = detail::make_arg<Context>(
+ static_cast<typename make_unsigned_or_bool<U>::type>(value));
+ }
+ }
+ }
+
+ template <typename U, FMT_ENABLE_IF(!std::is_integral<U>::value)>
+ void operator()(U) {} // No conversion needed for non-integral types.
+};
+
+// Converts an integer argument to T for printf, if T is an integral type.
+// If T is void, the argument is converted to corresponding signed or unsigned
+// type depending on the type specifier: 'd' and 'i' - signed, other -
+// unsigned).
+template <typename T, typename Context, typename Char>
+void convert_arg(basic_format_arg<Context>& arg, Char type) {
+ visit_format_arg(arg_converter<T, Context>(arg, type), arg);
+}
+
+// Converts an integer argument to char for printf.
+template <typename Context> class char_converter {
+ private:
+ basic_format_arg<Context>& arg_;
+
+ public:
+ explicit char_converter(basic_format_arg<Context>& arg) : arg_(arg) {}
+
+ template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+ void operator()(T value) {
+ arg_ = detail::make_arg<Context>(
+ static_cast<typename Context::char_type>(value));
+ }
+
+ template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+ void operator()(T) {} // No conversion needed for non-integral types.
+};
+
+// An argument visitor that return a pointer to a C string if argument is a
+// string or null otherwise.
+template <typename Char> struct get_cstring {
+ template <typename T> const Char* operator()(T) { return nullptr; }
+ const Char* operator()(const Char* s) { return s; }
+};
+
+// Checks if an argument is a valid printf width specifier and sets
+// left alignment if it is negative.
+template <typename Char> class printf_width_handler {
+ private:
+ format_specs<Char>& specs_;
+
+ public:
+ explicit printf_width_handler(format_specs<Char>& specs) : specs_(specs) {}
+
+ template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+ unsigned operator()(T value) {
+ auto width = static_cast<uint32_or_64_or_128_t<T>>(value);
+ if (detail::is_negative(value)) {
+ specs_.align = align::left;
+ width = 0 - width;
+ }
+ unsigned int_max = max_value<int>();
+ if (width > int_max) throw_format_error("number is too big");
+ return static_cast<unsigned>(width);
+ }
+
+ template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+ unsigned operator()(T) {
+ throw_format_error("width is not integer");
+ return 0;
+ }
+};
+
+// Workaround for a bug with the XL compiler when initializing
+// printf_arg_formatter's base class.
+template <typename Char>
+auto make_arg_formatter(buffer_appender<Char> iter, format_specs<Char>& s)
+ -> arg_formatter<Char> {
+ return {iter, s, locale_ref()};
+}
+
+// The ``printf`` argument formatter.
+template <typename OutputIt, typename Char>
+class printf_arg_formatter : public arg_formatter<Char> {
+ private:
+ using base = arg_formatter<Char>;
+ using context_type = basic_printf_context<OutputIt, Char>;
+
+ context_type& context_;
+
+ OutputIt write_null_pointer(bool is_string = false) {
+ auto s = this->specs;
+ s.type = presentation_type::none;
+ return write_bytes(this->out, is_string ? "(null)" : "(nil)", s);
+ }
+
+ public:
+ printf_arg_formatter(OutputIt iter, format_specs<Char>& s, context_type& ctx)
+ : base(make_arg_formatter(iter, s)), context_(ctx) {}
+
+ OutputIt operator()(monostate value) { return base::operator()(value); }
+
+ template <typename T, FMT_ENABLE_IF(detail::is_integral<T>::value)>
+ OutputIt operator()(T value) {
+ // MSVC2013 fails to compile separate overloads for bool and Char so use
+ // std::is_same instead.
+ if (std::is_same<T, Char>::value) {
+ format_specs<Char> fmt_specs = this->specs;
+ if (fmt_specs.type != presentation_type::none &&
+ fmt_specs.type != presentation_type::chr) {
+ return (*this)(static_cast<int>(value));
+ }
+ fmt_specs.sign = sign::none;
+ fmt_specs.alt = false;
+ fmt_specs.fill[0] = ' '; // Ignore '0' flag for char types.
+ // align::numeric needs to be overwritten here since the '0' flag is
+ // ignored for non-numeric types
+ if (fmt_specs.align == align::none || fmt_specs.align == align::numeric)
+ fmt_specs.align = align::right;
+ return write<Char>(this->out, static_cast<Char>(value), fmt_specs);
+ }
+ return base::operator()(value);
+ }
+
+ template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value)>
+ OutputIt operator()(T value) {
+ return base::operator()(value);
+ }
+
+ /** Formats a null-terminated C string. */
+ OutputIt operator()(const char* value) {
+ if (value) return base::operator()(value);
+ return write_null_pointer(this->specs.type != presentation_type::pointer);
+ }
+
+ /** Formats a null-terminated wide C string. */
+ OutputIt operator()(const wchar_t* value) {
+ if (value) return base::operator()(value);
+ return write_null_pointer(this->specs.type != presentation_type::pointer);
+ }
+
+ OutputIt operator()(basic_string_view<Char> value) {
+ return base::operator()(value);
+ }
+
+ /** Formats a pointer. */
+ OutputIt operator()(const void* value) {
+ return value ? base::operator()(value) : write_null_pointer();
+ }
+
+ /** Formats an argument of a custom (user-defined) type. */
+ OutputIt operator()(typename basic_format_arg<context_type>::handle handle) {
+ auto parse_ctx =
+ basic_printf_parse_context<Char>(basic_string_view<Char>());
+ handle.format(parse_ctx, context_);
+ return this->out;
+ }
+};
+
+template <typename Char>
+void parse_flags(format_specs<Char>& specs, const Char*& it, const Char* end) {
+ for (; it != end; ++it) {
+ switch (*it) {
+ case '-':
+ specs.align = align::left;
+ break;
+ case '+':
+ specs.sign = sign::plus;
+ break;
+ case '0':
+ specs.fill[0] = '0';
+ break;
+ case ' ':
+ if (specs.sign != sign::plus) {
+ specs.sign = sign::space;
+ }
+ break;
+ case '#':
+ specs.alt = true;
+ break;
+ default:
+ return;
+ }
+ }
+}
+
+template <typename Char, typename GetArg>
+int parse_header(const Char*& it, const Char* end, format_specs<Char>& specs,
+ GetArg get_arg) {
+ int arg_index = -1;
+ Char c = *it;
+ if (c >= '0' && c <= '9') {
+ // Parse an argument index (if followed by '$') or a width possibly
+ // preceded with '0' flag(s).
+ int value = parse_nonnegative_int(it, end, -1);
+ if (it != end && *it == '$') { // value is an argument index
+ ++it;
+ arg_index = value != -1 ? value : max_value<int>();
+ } else {
+ if (c == '0') specs.fill[0] = '0';
+ if (value != 0) {
+ // Nonzero value means that we parsed width and don't need to
+ // parse it or flags again, so return now.
+ if (value == -1) throw_format_error("number is too big");
+ specs.width = value;
+ return arg_index;
+ }
+ }
+ }
+ parse_flags(specs, it, end);
+ // Parse width.
+ if (it != end) {
+ if (*it >= '0' && *it <= '9') {
+ specs.width = parse_nonnegative_int(it, end, -1);
+ if (specs.width == -1) throw_format_error("number is too big");
+ } else if (*it == '*') {
+ ++it;
+ specs.width = static_cast<int>(visit_format_arg(
+ detail::printf_width_handler<Char>(specs), get_arg(-1)));
+ }
+ }
+ return arg_index;
+}
+
+inline auto parse_printf_presentation_type(char c, type t)
+ -> presentation_type {
+ using pt = presentation_type;
+ constexpr auto integral_set = sint_set | uint_set | bool_set | char_set;
+ switch (c) {
+ case 'd':
+ return in(t, integral_set) ? pt::dec : pt::none;
+ case 'o':
+ return in(t, integral_set) ? pt::oct : pt::none;
+ case 'x':
+ return in(t, integral_set) ? pt::hex_lower : pt::none;
+ case 'X':
+ return in(t, integral_set) ? pt::hex_upper : pt::none;
+ case 'a':
+ return in(t, float_set) ? pt::hexfloat_lower : pt::none;
+ case 'A':
+ return in(t, float_set) ? pt::hexfloat_upper : pt::none;
+ case 'e':
+ return in(t, float_set) ? pt::exp_lower : pt::none;
+ case 'E':
+ return in(t, float_set) ? pt::exp_upper : pt::none;
+ case 'f':
+ return in(t, float_set) ? pt::fixed_lower : pt::none;
+ case 'F':
+ return in(t, float_set) ? pt::fixed_upper : pt::none;
+ case 'g':
+ return in(t, float_set) ? pt::general_lower : pt::none;
+ case 'G':
+ return in(t, float_set) ? pt::general_upper : pt::none;
+ case 'c':
+ return in(t, integral_set) ? pt::chr : pt::none;
+ case 's':
+ return in(t, string_set | cstring_set) ? pt::string : pt::none;
+ case 'p':
+ return in(t, pointer_set | cstring_set) ? pt::pointer : pt::none;
+ default:
+ return pt::none;
+ }
+}
+
+template <typename Char, typename Context>
+void vprintf(buffer<Char>& buf, basic_string_view<Char> format,
+ basic_format_args<Context> args) {
+ using iterator = buffer_appender<Char>;
+ auto out = iterator(buf);
+ auto context = basic_printf_context<iterator, Char>(out, args);
+ auto parse_ctx = basic_printf_parse_context<Char>(format);
+
+ // Returns the argument with specified index or, if arg_index is -1, the next
+ // argument.
+ auto get_arg = [&](int arg_index) {
+ if (arg_index < 0)
+ arg_index = parse_ctx.next_arg_id();
+ else
+ parse_ctx.check_arg_id(--arg_index);
+ return detail::get_arg(context, arg_index);
+ };
+
+ const Char* start = parse_ctx.begin();
+ const Char* end = parse_ctx.end();
+ auto it = start;
+ while (it != end) {
+ if (!find<false, Char>(it, end, '%', it)) {
+ it = end; // find leaves it == nullptr if it doesn't find '%'.
+ break;
+ }
+ Char c = *it++;
+ if (it != end && *it == c) {
+ out = write(out, basic_string_view<Char>(start, to_unsigned(it - start)));
+ start = ++it;
+ continue;
+ }
+ out =
+ write(out, basic_string_view<Char>(start, to_unsigned(it - 1 - start)));
+
+ auto specs = format_specs<Char>();
+ specs.align = align::right;
+
+ // Parse argument index, flags and width.
+ int arg_index = parse_header(it, end, specs, get_arg);
+ if (arg_index == 0) throw_format_error("argument not found");
+
+ // Parse precision.
+ if (it != end && *it == '.') {
+ ++it;
+ c = it != end ? *it : 0;
+ if ('0' <= c && c <= '9') {
+ specs.precision = parse_nonnegative_int(it, end, 0);
+ } else if (c == '*') {
+ ++it;
+ specs.precision = static_cast<int>(
+ visit_format_arg(printf_precision_handler(), get_arg(-1)));
+ } else {
+ specs.precision = 0;
+ }
+ }
+
+ auto arg = get_arg(arg_index);
+ // For d, i, o, u, x, and X conversion specifiers, if a precision is
+ // specified, the '0' flag is ignored
+ if (specs.precision >= 0 && arg.is_integral())
+ specs.fill[0] =
+ ' '; // Ignore '0' flag for non-numeric types or if '-' present.
+ if (specs.precision >= 0 && arg.type() == type::cstring_type) {
+ auto str = visit_format_arg(get_cstring<Char>(), arg);
+ auto str_end = str + specs.precision;
+ auto nul = std::find(str, str_end, Char());
+ arg = make_arg<basic_printf_context<iterator, Char>>(
+ basic_string_view<Char>(
+ str, to_unsigned(nul != str_end ? nul - str : specs.precision)));
+ }
+ if (specs.alt && visit_format_arg(is_zero_int(), arg)) specs.alt = false;
+ if (specs.fill[0] == '0') {
+ if (arg.is_arithmetic() && specs.align != align::left)
+ specs.align = align::numeric;
+ else
+ specs.fill[0] = ' '; // Ignore '0' flag for non-numeric types or if '-'
+ // flag is also present.
+ }
+
+ // Parse length and convert the argument to the required type.
+ c = it != end ? *it++ : 0;
+ Char t = it != end ? *it : 0;
+ switch (c) {
+ case 'h':
+ if (t == 'h') {
+ ++it;
+ t = it != end ? *it : 0;
+ convert_arg<signed char>(arg, t);
+ } else {
+ convert_arg<short>(arg, t);
+ }
+ break;
+ case 'l':
+ if (t == 'l') {
+ ++it;
+ t = it != end ? *it : 0;
+ convert_arg<long long>(arg, t);
+ } else {
+ convert_arg<long>(arg, t);
+ }
+ break;
+ case 'j':
+ convert_arg<intmax_t>(arg, t);
+ break;
+ case 'z':
+ convert_arg<size_t>(arg, t);
+ break;
+ case 't':
+ convert_arg<std::ptrdiff_t>(arg, t);
+ break;
+ case 'L':
+ // printf produces garbage when 'L' is omitted for long double, no
+ // need to do the same.
+ break;
+ default:
+ --it;
+ convert_arg<void>(arg, c);
+ }
+
+ // Parse type.
+ if (it == end) throw_format_error("invalid format string");
+ char type = static_cast<char>(*it++);
+ if (arg.is_integral()) {
+ // Normalize type.
+ switch (type) {
+ case 'i':
+ case 'u':
+ type = 'd';
+ break;
+ case 'c':
+ visit_format_arg(
+ char_converter<basic_printf_context<iterator, Char>>(arg), arg);
+ break;
+ }
+ }
+ specs.type = parse_printf_presentation_type(type, arg.type());
+ if (specs.type == presentation_type::none)
+ throw_format_error("invalid format specifier");
+
+ start = it;
+
+ // Format argument.
+ out = visit_format_arg(
+ printf_arg_formatter<iterator, Char>(out, specs, context), arg);
+ }
+ write(out, basic_string_view<Char>(start, to_unsigned(it - start)));
+}
+FMT_END_DETAIL_NAMESPACE
+
+template <typename Char>
+using basic_printf_context_t =
+ basic_printf_context<detail::buffer_appender<Char>, Char>;
+
+using printf_context = basic_printf_context_t<char>;
+using wprintf_context = basic_printf_context_t<wchar_t>;
+
+using printf_args = basic_format_args<printf_context>;
+using wprintf_args = basic_format_args<wprintf_context>;
+
+/**
+ \rst
+ Constructs an `~fmt::format_arg_store` object that contains references to
+ arguments and can be implicitly converted to `~fmt::printf_args`.
+ \endrst
+ */
+template <typename... T>
+inline auto make_printf_args(const T&... args)
+ -> format_arg_store<printf_context, T...> {
+ return {args...};
+}
+
+/**
+ \rst
+ Constructs an `~fmt::format_arg_store` object that contains references to
+ arguments and can be implicitly converted to `~fmt::wprintf_args`.
+ \endrst
+ */
+template <typename... T>
+inline auto make_wprintf_args(const T&... args)
+ -> format_arg_store<wprintf_context, T...> {
+ return {args...};
+}
+
+template <typename S, typename Char = char_t<S>>
+inline auto vsprintf(
+ const S& fmt,
+ basic_format_args<basic_printf_context_t<type_identity_t<Char>>> args)
+ -> std::basic_string<Char> {
+ auto buf = basic_memory_buffer<Char>();
+ detail::vprintf(buf, detail::to_string_view(fmt), args);
+ return to_string(buf);
+}
+
+/**
+ \rst
+ Formats arguments and returns the result as a string.
+
+ **Example**::
+
+ std::string message = fmt::sprintf("The answer is %d", 42);
+ \endrst
+*/
+template <typename S, typename... T,
+ typename Char = enable_if_t<detail::is_string<S>::value, char_t<S>>>
+inline auto sprintf(const S& fmt, const T&... args) -> std::basic_string<Char> {
+ using context = basic_printf_context_t<Char>;
+ return vsprintf(detail::to_string_view(fmt),
+ fmt::make_format_args<context>(args...));
+}
+
+template <typename S, typename Char = char_t<S>>
+inline auto vfprintf(
+ std::FILE* f, const S& fmt,
+ basic_format_args<basic_printf_context_t<type_identity_t<Char>>> args)
+ -> int {
+ auto buf = basic_memory_buffer<Char>();
+ detail::vprintf(buf, detail::to_string_view(fmt), args);
+ size_t size = buf.size();
+ return std::fwrite(buf.data(), sizeof(Char), size, f) < size
+ ? -1
+ : static_cast<int>(size);
+}
+
+/**
+ \rst
+ Prints formatted data to the file *f*.
+
+ **Example**::
+
+ fmt::fprintf(stderr, "Don't %s!", "panic");
+ \endrst
+ */
+template <typename S, typename... T, typename Char = char_t<S>>
+inline auto fprintf(std::FILE* f, const S& fmt, const T&... args) -> int {
+ using context = basic_printf_context_t<Char>;
+ return vfprintf(f, detail::to_string_view(fmt),
+ fmt::make_format_args<context>(args...));
+}
+
+template <typename S, typename Char = char_t<S>>
+inline auto vprintf(
+ const S& fmt,
+ basic_format_args<basic_printf_context_t<type_identity_t<Char>>> args)
+ -> int {
+ return vfprintf(stdout, detail::to_string_view(fmt), args);
+}
+
+/**
+ \rst
+ Prints formatted data to ``stdout``.
+
+ **Example**::
+
+ fmt::printf("Elapsed time: %.2f seconds", 1.23);
+ \endrst
+ */
+template <typename S, typename... T, FMT_ENABLE_IF(detail::is_string<S>::value)>
+inline auto printf(const S& fmt, const T&... args) -> int {
+ return vprintf(
+ detail::to_string_view(fmt),
+ fmt::make_format_args<basic_printf_context_t<char_t<S>>>(args...));
+}
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_PRINTF_H_
diff --git a/contrib/fmt/include/fmt/ranges.h b/contrib/fmt/include/fmt/ranges.h
new file mode 100644
index 0000000..266b9e1
--- /dev/null
+++ b/contrib/fmt/include/fmt/ranges.h
@@ -0,0 +1,732 @@
+// Formatting library for C++ - experimental range support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+//
+// Copyright (c) 2018 - present, Remotion (Igor Schulz)
+// All Rights Reserved
+// {fmt} support for ranges, containers and types tuple interface.
+
+#ifndef FMT_RANGES_H_
+#define FMT_RANGES_H_
+
+#include <initializer_list>
+#include <tuple>
+#include <type_traits>
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+
+namespace detail {
+
+template <typename Range, typename OutputIt>
+auto copy(const Range& range, OutputIt out) -> OutputIt {
+ for (auto it = range.begin(), end = range.end(); it != end; ++it)
+ *out++ = *it;
+ return out;
+}
+
+template <typename OutputIt>
+auto copy(const char* str, OutputIt out) -> OutputIt {
+ while (*str) *out++ = *str++;
+ return out;
+}
+
+template <typename OutputIt> auto copy(char ch, OutputIt out) -> OutputIt {
+ *out++ = ch;
+ return out;
+}
+
+template <typename OutputIt> auto copy(wchar_t ch, OutputIt out) -> OutputIt {
+ *out++ = ch;
+ return out;
+}
+
+// Returns true if T has a std::string-like interface, like std::string_view.
+template <typename T> class is_std_string_like {
+ template <typename U>
+ static auto check(U* p)
+ -> decltype((void)p->find('a'), p->length(), (void)p->data(), int());
+ template <typename> static void check(...);
+
+ public:
+ static constexpr const bool value =
+ is_string<T>::value ||
+ std::is_convertible<T, std_string_view<char>>::value ||
+ !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
+template <typename Char>
+struct is_std_string_like<fmt::basic_string_view<Char>> : std::true_type {};
+
+template <typename T> class is_map {
+ template <typename U> static auto check(U*) -> typename U::mapped_type;
+ template <typename> static void check(...);
+
+ public:
+#ifdef FMT_FORMAT_MAP_AS_LIST // DEPRECATED!
+ static constexpr const bool value = false;
+#else
+ static constexpr const bool value =
+ !std::is_void<decltype(check<T>(nullptr))>::value;
+#endif
+};
+
+template <typename T> class is_set {
+ template <typename U> static auto check(U*) -> typename U::key_type;
+ template <typename> static void check(...);
+
+ public:
+#ifdef FMT_FORMAT_SET_AS_LIST // DEPRECATED!
+ static constexpr const bool value = false;
+#else
+ static constexpr const bool value =
+ !std::is_void<decltype(check<T>(nullptr))>::value && !is_map<T>::value;
+#endif
+};
+
+template <typename... Ts> struct conditional_helper {};
+
+template <typename T, typename _ = void> struct is_range_ : std::false_type {};
+
+#if !FMT_MSC_VERSION || FMT_MSC_VERSION > 1800
+
+# define FMT_DECLTYPE_RETURN(val) \
+ ->decltype(val) { return val; } \
+ static_assert( \
+ true, "") // This makes it so that a semicolon is required after the
+ // macro, which helps clang-format handle the formatting.
+
+// C array overload
+template <typename T, std::size_t N>
+auto range_begin(const T (&arr)[N]) -> const T* {
+ return arr;
+}
+template <typename T, std::size_t N>
+auto range_end(const T (&arr)[N]) -> const T* {
+ return arr + N;
+}
+
+template <typename T, typename Enable = void>
+struct has_member_fn_begin_end_t : std::false_type {};
+
+template <typename T>
+struct has_member_fn_begin_end_t<T, void_t<decltype(std::declval<T>().begin()),
+ decltype(std::declval<T>().end())>>
+ : std::true_type {};
+
+// Member function overload
+template <typename T>
+auto range_begin(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).begin());
+template <typename T>
+auto range_end(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).end());
+
+// ADL overload. Only participates in overload resolution if member functions
+// are not found.
+template <typename T>
+auto range_begin(T&& rng)
+ -> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
+ decltype(begin(static_cast<T&&>(rng)))> {
+ return begin(static_cast<T&&>(rng));
+}
+template <typename T>
+auto range_end(T&& rng) -> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
+ decltype(end(static_cast<T&&>(rng)))> {
+ return end(static_cast<T&&>(rng));
+}
+
+template <typename T, typename Enable = void>
+struct has_const_begin_end : std::false_type {};
+template <typename T, typename Enable = void>
+struct has_mutable_begin_end : std::false_type {};
+
+template <typename T>
+struct has_const_begin_end<
+ T,
+ void_t<
+ decltype(detail::range_begin(std::declval<const remove_cvref_t<T>&>())),
+ decltype(detail::range_end(std::declval<const remove_cvref_t<T>&>()))>>
+ : std::true_type {};
+
+template <typename T>
+struct has_mutable_begin_end<
+ T, void_t<decltype(detail::range_begin(std::declval<T>())),
+ decltype(detail::range_end(std::declval<T>())),
+ // the extra int here is because older versions of MSVC don't
+ // SFINAE properly unless there are distinct types
+ int>> : std::true_type {};
+
+template <typename T>
+struct is_range_<T, void>
+ : std::integral_constant<bool, (has_const_begin_end<T>::value ||
+ has_mutable_begin_end<T>::value)> {};
+# undef FMT_DECLTYPE_RETURN
+#endif
+
+// tuple_size and tuple_element check.
+template <typename T> class is_tuple_like_ {
+ template <typename U>
+ static auto check(U* p) -> decltype(std::tuple_size<U>::value, int());
+ template <typename> static void check(...);
+
+ public:
+ static constexpr const bool value =
+ !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
+// Check for integer_sequence
+#if defined(__cpp_lib_integer_sequence) || FMT_MSC_VERSION >= 1900
+template <typename T, T... N>
+using integer_sequence = std::integer_sequence<T, N...>;
+template <size_t... N> using index_sequence = std::index_sequence<N...>;
+template <size_t N> using make_index_sequence = std::make_index_sequence<N>;
+#else
+template <typename T, T... N> struct integer_sequence {
+ using value_type = T;
+
+ static FMT_CONSTEXPR size_t size() { return sizeof...(N); }
+};
+
+template <size_t... N> using index_sequence = integer_sequence<size_t, N...>;
+
+template <typename T, size_t N, T... Ns>
+struct make_integer_sequence : make_integer_sequence<T, N - 1, N - 1, Ns...> {};
+template <typename T, T... Ns>
+struct make_integer_sequence<T, 0, Ns...> : integer_sequence<T, Ns...> {};
+
+template <size_t N>
+using make_index_sequence = make_integer_sequence<size_t, N>;
+#endif
+
+template <typename T>
+using tuple_index_sequence = make_index_sequence<std::tuple_size<T>::value>;
+
+template <typename T, typename C, bool = is_tuple_like_<T>::value>
+class is_tuple_formattable_ {
+ public:
+ static constexpr const bool value = false;
+};
+template <typename T, typename C> class is_tuple_formattable_<T, C, true> {
+ template <std::size_t... Is>
+ static std::true_type check2(index_sequence<Is...>,
+ integer_sequence<bool, (Is == Is)...>);
+ static std::false_type check2(...);
+ template <std::size_t... Is>
+ static decltype(check2(
+ index_sequence<Is...>{},
+ integer_sequence<
+ bool, (is_formattable<typename std::tuple_element<Is, T>::type,
+ C>::value)...>{})) check(index_sequence<Is...>);
+
+ public:
+ static constexpr const bool value =
+ decltype(check(tuple_index_sequence<T>{}))::value;
+};
+
+template <typename Tuple, typename F, size_t... Is>
+FMT_CONSTEXPR void for_each(index_sequence<Is...>, Tuple&& t, F&& f) {
+ using std::get;
+ // Using a free function get<Is>(Tuple) now.
+ const int unused[] = {0, ((void)f(get<Is>(t)), 0)...};
+ ignore_unused(unused);
+}
+
+template <typename Tuple, typename F>
+FMT_CONSTEXPR void for_each(Tuple&& t, F&& f) {
+ for_each(tuple_index_sequence<remove_cvref_t<Tuple>>(),
+ std::forward<Tuple>(t), std::forward<F>(f));
+}
+
+template <typename Tuple1, typename Tuple2, typename F, size_t... Is>
+void for_each2(index_sequence<Is...>, Tuple1&& t1, Tuple2&& t2, F&& f) {
+ using std::get;
+ const int unused[] = {0, ((void)f(get<Is>(t1), get<Is>(t2)), 0)...};
+ ignore_unused(unused);
+}
+
+template <typename Tuple1, typename Tuple2, typename F>
+void for_each2(Tuple1&& t1, Tuple2&& t2, F&& f) {
+ for_each2(tuple_index_sequence<remove_cvref_t<Tuple1>>(),
+ std::forward<Tuple1>(t1), std::forward<Tuple2>(t2),
+ std::forward<F>(f));
+}
+
+namespace tuple {
+// Workaround a bug in MSVC 2019 (v140).
+template <typename Char, typename... T>
+using result_t = std::tuple<formatter<remove_cvref_t<T>, Char>...>;
+
+using std::get;
+template <typename Tuple, typename Char, std::size_t... Is>
+auto get_formatters(index_sequence<Is...>)
+ -> result_t<Char, decltype(get<Is>(std::declval<Tuple>()))...>;
+} // namespace tuple
+
+#if FMT_MSC_VERSION && FMT_MSC_VERSION < 1920
+// Older MSVC doesn't get the reference type correctly for arrays.
+template <typename R> struct range_reference_type_impl {
+ using type = decltype(*detail::range_begin(std::declval<R&>()));
+};
+
+template <typename T, std::size_t N> struct range_reference_type_impl<T[N]> {
+ using type = T&;
+};
+
+template <typename T>
+using range_reference_type = typename range_reference_type_impl<T>::type;
+#else
+template <typename Range>
+using range_reference_type =
+ decltype(*detail::range_begin(std::declval<Range&>()));
+#endif
+
+// We don't use the Range's value_type for anything, but we do need the Range's
+// reference type, with cv-ref stripped.
+template <typename Range>
+using uncvref_type = remove_cvref_t<range_reference_type<Range>>;
+
+template <typename Formatter>
+FMT_CONSTEXPR auto maybe_set_debug_format(Formatter& f, bool set)
+ -> decltype(f.set_debug_format(set)) {
+ f.set_debug_format(set);
+}
+template <typename Formatter>
+FMT_CONSTEXPR void maybe_set_debug_format(Formatter&, ...) {}
+
+// These are not generic lambdas for compatibility with C++11.
+template <typename ParseContext> struct parse_empty_specs {
+ template <typename Formatter> FMT_CONSTEXPR void operator()(Formatter& f) {
+ f.parse(ctx);
+ detail::maybe_set_debug_format(f, true);
+ }
+ ParseContext& ctx;
+};
+template <typename FormatContext> struct format_tuple_element {
+ using char_type = typename FormatContext::char_type;
+
+ template <typename T>
+ void operator()(const formatter<T, char_type>& f, const T& v) {
+ if (i > 0)
+ ctx.advance_to(detail::copy_str<char_type>(separator, ctx.out()));
+ ctx.advance_to(f.format(v, ctx));
+ ++i;
+ }
+
+ int i;
+ FormatContext& ctx;
+ basic_string_view<char_type> separator;
+};
+
+} // namespace detail
+
+template <typename T> struct is_tuple_like {
+ static constexpr const bool value =
+ detail::is_tuple_like_<T>::value && !detail::is_range_<T>::value;
+};
+
+template <typename T, typename C> struct is_tuple_formattable {
+ static constexpr const bool value =
+ detail::is_tuple_formattable_<T, C>::value;
+};
+
+template <typename Tuple, typename Char>
+struct formatter<Tuple, Char,
+ enable_if_t<fmt::is_tuple_like<Tuple>::value &&
+ fmt::is_tuple_formattable<Tuple, Char>::value>> {
+ private:
+ decltype(detail::tuple::get_formatters<Tuple, Char>(
+ detail::tuple_index_sequence<Tuple>())) formatters_;
+
+ basic_string_view<Char> separator_ = detail::string_literal<Char, ',', ' '>{};
+ basic_string_view<Char> opening_bracket_ =
+ detail::string_literal<Char, '('>{};
+ basic_string_view<Char> closing_bracket_ =
+ detail::string_literal<Char, ')'>{};
+
+ public:
+ FMT_CONSTEXPR formatter() {}
+
+ FMT_CONSTEXPR void set_separator(basic_string_view<Char> sep) {
+ separator_ = sep;
+ }
+
+ FMT_CONSTEXPR void set_brackets(basic_string_view<Char> open,
+ basic_string_view<Char> close) {
+ opening_bracket_ = open;
+ closing_bracket_ = close;
+ }
+
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ auto it = ctx.begin();
+ if (it != ctx.end() && *it != '}')
+ FMT_THROW(format_error("invalid format specifier"));
+ detail::for_each(formatters_, detail::parse_empty_specs<ParseContext>{ctx});
+ return it;
+ }
+
+ template <typename FormatContext>
+ auto format(const Tuple& value, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ ctx.advance_to(detail::copy_str<Char>(opening_bracket_, ctx.out()));
+ detail::for_each2(
+ formatters_, value,
+ detail::format_tuple_element<FormatContext>{0, ctx, separator_});
+ return detail::copy_str<Char>(closing_bracket_, ctx.out());
+ }
+};
+
+template <typename T, typename Char> struct is_range {
+ static constexpr const bool value =
+ detail::is_range_<T>::value && !detail::is_std_string_like<T>::value &&
+ !std::is_convertible<T, std::basic_string<Char>>::value &&
+ !std::is_convertible<T, detail::std_string_view<Char>>::value;
+};
+
+namespace detail {
+template <typename Context> struct range_mapper {
+ using mapper = arg_mapper<Context>;
+
+ template <typename T,
+ FMT_ENABLE_IF(has_formatter<remove_cvref_t<T>, Context>::value)>
+ static auto map(T&& value) -> T&& {
+ return static_cast<T&&>(value);
+ }
+ template <typename T,
+ FMT_ENABLE_IF(!has_formatter<remove_cvref_t<T>, Context>::value)>
+ static auto map(T&& value)
+ -> decltype(mapper().map(static_cast<T&&>(value))) {
+ return mapper().map(static_cast<T&&>(value));
+ }
+};
+
+template <typename Char, typename Element>
+using range_formatter_type =
+ formatter<remove_cvref_t<decltype(range_mapper<buffer_context<Char>>{}.map(
+ std::declval<Element>()))>,
+ Char>;
+
+template <typename R>
+using maybe_const_range =
+ conditional_t<has_const_begin_end<R>::value, const R, R>;
+
+// Workaround a bug in MSVC 2015 and earlier.
+#if !FMT_MSC_VERSION || FMT_MSC_VERSION >= 1910
+template <typename R, typename Char>
+struct is_formattable_delayed
+ : is_formattable<uncvref_type<maybe_const_range<R>>, Char> {};
+#endif
+} // namespace detail
+
+template <typename T, typename Char, typename Enable = void>
+struct range_formatter;
+
+template <typename T, typename Char>
+struct range_formatter<
+ T, Char,
+ enable_if_t<conjunction<std::is_same<T, remove_cvref_t<T>>,
+ is_formattable<T, Char>>::value>> {
+ private:
+ detail::range_formatter_type<Char, T> underlying_;
+ basic_string_view<Char> separator_ = detail::string_literal<Char, ',', ' '>{};
+ basic_string_view<Char> opening_bracket_ =
+ detail::string_literal<Char, '['>{};
+ basic_string_view<Char> closing_bracket_ =
+ detail::string_literal<Char, ']'>{};
+
+ public:
+ FMT_CONSTEXPR range_formatter() {}
+
+ FMT_CONSTEXPR auto underlying() -> detail::range_formatter_type<Char, T>& {
+ return underlying_;
+ }
+
+ FMT_CONSTEXPR void set_separator(basic_string_view<Char> sep) {
+ separator_ = sep;
+ }
+
+ FMT_CONSTEXPR void set_brackets(basic_string_view<Char> open,
+ basic_string_view<Char> close) {
+ opening_bracket_ = open;
+ closing_bracket_ = close;
+ }
+
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ auto it = ctx.begin();
+ auto end = ctx.end();
+
+ if (it != end && *it == 'n') {
+ set_brackets({}, {});
+ ++it;
+ }
+
+ if (it != end && *it != '}') {
+ if (*it != ':') FMT_THROW(format_error("invalid format specifier"));
+ ++it;
+ } else {
+ detail::maybe_set_debug_format(underlying_, true);
+ }
+
+ ctx.advance_to(it);
+ return underlying_.parse(ctx);
+ }
+
+ template <typename R, typename FormatContext>
+ auto format(R&& range, FormatContext& ctx) const -> decltype(ctx.out()) {
+ detail::range_mapper<buffer_context<Char>> mapper;
+ auto out = ctx.out();
+ out = detail::copy_str<Char>(opening_bracket_, out);
+ int i = 0;
+ auto it = detail::range_begin(range);
+ auto end = detail::range_end(range);
+ for (; it != end; ++it) {
+ if (i > 0) out = detail::copy_str<Char>(separator_, out);
+ ctx.advance_to(out);
+ out = underlying_.format(mapper.map(*it), ctx);
+ ++i;
+ }
+ out = detail::copy_str<Char>(closing_bracket_, out);
+ return out;
+ }
+};
+
+enum class range_format { disabled, map, set, sequence, string, debug_string };
+
+namespace detail {
+template <typename T>
+struct range_format_kind_
+ : std::integral_constant<range_format,
+ std::is_same<uncvref_type<T>, T>::value
+ ? range_format::disabled
+ : is_map<T>::value ? range_format::map
+ : is_set<T>::value ? range_format::set
+ : range_format::sequence> {};
+
+template <range_format K, typename R, typename Char, typename Enable = void>
+struct range_default_formatter;
+
+template <range_format K>
+using range_format_constant = std::integral_constant<range_format, K>;
+
+template <range_format K, typename R, typename Char>
+struct range_default_formatter<
+ K, R, Char,
+ enable_if_t<(K == range_format::sequence || K == range_format::map ||
+ K == range_format::set)>> {
+ using range_type = detail::maybe_const_range<R>;
+ range_formatter<detail::uncvref_type<range_type>, Char> underlying_;
+
+ FMT_CONSTEXPR range_default_formatter() { init(range_format_constant<K>()); }
+
+ FMT_CONSTEXPR void init(range_format_constant<range_format::set>) {
+ underlying_.set_brackets(detail::string_literal<Char, '{'>{},
+ detail::string_literal<Char, '}'>{});
+ }
+
+ FMT_CONSTEXPR void init(range_format_constant<range_format::map>) {
+ underlying_.set_brackets(detail::string_literal<Char, '{'>{},
+ detail::string_literal<Char, '}'>{});
+ underlying_.underlying().set_brackets({}, {});
+ underlying_.underlying().set_separator(
+ detail::string_literal<Char, ':', ' '>{});
+ }
+
+ FMT_CONSTEXPR void init(range_format_constant<range_format::sequence>) {}
+
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ return underlying_.parse(ctx);
+ }
+
+ template <typename FormatContext>
+ auto format(range_type& range, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ return underlying_.format(range, ctx);
+ }
+};
+} // namespace detail
+
+template <typename T, typename Char, typename Enable = void>
+struct range_format_kind
+ : conditional_t<
+ is_range<T, Char>::value, detail::range_format_kind_<T>,
+ std::integral_constant<range_format, range_format::disabled>> {};
+
+template <typename R, typename Char>
+struct formatter<
+ R, Char,
+ enable_if_t<conjunction<bool_constant<range_format_kind<R, Char>::value !=
+ range_format::disabled>
+// Workaround a bug in MSVC 2015 and earlier.
+#if !FMT_MSC_VERSION || FMT_MSC_VERSION >= 1910
+ ,
+ detail::is_formattable_delayed<R, Char>
+#endif
+ >::value>>
+ : detail::range_default_formatter<range_format_kind<R, Char>::value, R,
+ Char> {
+};
+
+template <typename Char, typename... T> struct tuple_join_view : detail::view {
+ const std::tuple<T...>& tuple;
+ basic_string_view<Char> sep;
+
+ tuple_join_view(const std::tuple<T...>& t, basic_string_view<Char> s)
+ : tuple(t), sep{s} {}
+};
+
+// Define FMT_TUPLE_JOIN_SPECIFIERS to enable experimental format specifiers
+// support in tuple_join. It is disabled by default because of issues with
+// the dynamic width and precision.
+#ifndef FMT_TUPLE_JOIN_SPECIFIERS
+# define FMT_TUPLE_JOIN_SPECIFIERS 0
+#endif
+
+template <typename Char, typename... T>
+struct formatter<tuple_join_view<Char, T...>, Char> {
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ return do_parse(ctx, std::integral_constant<size_t, sizeof...(T)>());
+ }
+
+ template <typename FormatContext>
+ auto format(const tuple_join_view<Char, T...>& value,
+ FormatContext& ctx) const -> typename FormatContext::iterator {
+ return do_format(value, ctx,
+ std::integral_constant<size_t, sizeof...(T)>());
+ }
+
+ private:
+ std::tuple<formatter<typename std::decay<T>::type, Char>...> formatters_;
+
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto do_parse(ParseContext& ctx,
+ std::integral_constant<size_t, 0>)
+ -> decltype(ctx.begin()) {
+ return ctx.begin();
+ }
+
+ template <typename ParseContext, size_t N>
+ FMT_CONSTEXPR auto do_parse(ParseContext& ctx,
+ std::integral_constant<size_t, N>)
+ -> decltype(ctx.begin()) {
+ auto end = ctx.begin();
+#if FMT_TUPLE_JOIN_SPECIFIERS
+ end = std::get<sizeof...(T) - N>(formatters_).parse(ctx);
+ if (N > 1) {
+ auto end1 = do_parse(ctx, std::integral_constant<size_t, N - 1>());
+ if (end != end1)
+ FMT_THROW(format_error("incompatible format specs for tuple elements"));
+ }
+#endif
+ return end;
+ }
+
+ template <typename FormatContext>
+ auto do_format(const tuple_join_view<Char, T...>&, FormatContext& ctx,
+ std::integral_constant<size_t, 0>) const ->
+ typename FormatContext::iterator {
+ return ctx.out();
+ }
+
+ template <typename FormatContext, size_t N>
+ auto do_format(const tuple_join_view<Char, T...>& value, FormatContext& ctx,
+ std::integral_constant<size_t, N>) const ->
+ typename FormatContext::iterator {
+ auto out = std::get<sizeof...(T) - N>(formatters_)
+ .format(std::get<sizeof...(T) - N>(value.tuple), ctx);
+ if (N > 1) {
+ out = std::copy(value.sep.begin(), value.sep.end(), out);
+ ctx.advance_to(out);
+ return do_format(value, ctx, std::integral_constant<size_t, N - 1>());
+ }
+ return out;
+ }
+};
+
+namespace detail {
+// Check if T has an interface like a container adaptor (e.g. std::stack,
+// std::queue, std::priority_queue).
+template <typename T> class is_container_adaptor_like {
+ template <typename U> static auto check(U* p) -> typename U::container_type;
+ template <typename> static void check(...);
+
+ public:
+ static constexpr const bool value =
+ !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
+template <typename Container> struct all {
+ const Container& c;
+ auto begin() const -> typename Container::const_iterator { return c.begin(); }
+ auto end() const -> typename Container::const_iterator { return c.end(); }
+};
+} // namespace detail
+
+template <typename T, typename Char>
+struct formatter<T, Char,
+ enable_if_t<detail::is_container_adaptor_like<T>::value>>
+ : formatter<detail::all<typename T::container_type>, Char> {
+ using all = detail::all<typename T::container_type>;
+ template <typename FormatContext>
+ auto format(const T& t, FormatContext& ctx) const -> decltype(ctx.out()) {
+ struct getter : T {
+ static auto get(const T& t) -> all {
+ return {t.*(&getter::c)}; // Access c through the derived class.
+ }
+ };
+ return formatter<all>::format(getter::get(t), ctx);
+ }
+};
+
+FMT_BEGIN_EXPORT
+
+/**
+ \rst
+ Returns an object that formats `tuple` with elements separated by `sep`.
+
+ **Example**::
+
+ std::tuple<int, char> t = {1, 'a'};
+ fmt::print("{}", fmt::join(t, ", "));
+ // Output: "1, a"
+ \endrst
+ */
+template <typename... T>
+FMT_CONSTEXPR auto join(const std::tuple<T...>& tuple, string_view sep)
+ -> tuple_join_view<char, T...> {
+ return {tuple, sep};
+}
+
+template <typename... T>
+FMT_CONSTEXPR auto join(const std::tuple<T...>& tuple,
+ basic_string_view<wchar_t> sep)
+ -> tuple_join_view<wchar_t, T...> {
+ return {tuple, sep};
+}
+
+/**
+ \rst
+ Returns an object that formats `initializer_list` with elements separated by
+ `sep`.
+
+ **Example**::
+
+ fmt::print("{}", fmt::join({1, 2, 3}, ", "));
+ // Output: "1, 2, 3"
+ \endrst
+ */
+template <typename T>
+auto join(std::initializer_list<T> list, string_view sep)
+ -> join_view<const T*, const T*> {
+ return join(std::begin(list), std::end(list), sep);
+}
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_RANGES_H_
diff --git a/contrib/fmt/include/fmt/std.h b/contrib/fmt/include/fmt/std.h
new file mode 100644
index 0000000..4c2a28c
--- /dev/null
+++ b/contrib/fmt/include/fmt/std.h
@@ -0,0 +1,349 @@
+// Formatting library for C++ - formatters for standard library types
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_STD_H_
+#define FMT_STD_H_
+
+#include <cstdlib>
+#include <exception>
+#include <memory>
+#include <thread>
+#include <type_traits>
+#include <typeinfo>
+#include <utility>
+
+#include "ostream.h"
+
+#if FMT_HAS_INCLUDE(<version>)
+# include <version>
+#endif
+// Checking FMT_CPLUSPLUS for warning suppression in MSVC.
+#if FMT_CPLUSPLUS >= 201703L
+# if FMT_HAS_INCLUDE(<filesystem>)
+# include <filesystem>
+# endif
+# if FMT_HAS_INCLUDE(<variant>)
+# include <variant>
+# endif
+# if FMT_HAS_INCLUDE(<optional>)
+# include <optional>
+# endif
+#endif
+
+// GCC 4 does not support FMT_HAS_INCLUDE.
+#if FMT_HAS_INCLUDE(<cxxabi.h>) || defined(__GLIBCXX__)
+# include <cxxabi.h>
+// Android NDK with gabi++ library on some architectures does not implement
+// abi::__cxa_demangle().
+# ifndef __GABIXX_CXXABI_H__
+# define FMT_HAS_ABI_CXA_DEMANGLE
+# endif
+#endif
+
+#ifdef __cpp_lib_filesystem
+FMT_BEGIN_NAMESPACE
+
+namespace detail {
+
+template <typename Char>
+void write_escaped_path(basic_memory_buffer<Char>& quoted,
+ const std::filesystem::path& p) {
+ write_escaped_string<Char>(std::back_inserter(quoted), p.string<Char>());
+}
+# ifdef _WIN32
+template <>
+inline void write_escaped_path<char>(memory_buffer& quoted,
+ const std::filesystem::path& p) {
+ auto buf = basic_memory_buffer<wchar_t>();
+ write_escaped_string<wchar_t>(std::back_inserter(buf), p.native());
+ // Convert UTF-16 to UTF-8.
+ if (!unicode_to_utf8<wchar_t>::convert(quoted, {buf.data(), buf.size()}))
+ FMT_THROW(std::runtime_error("invalid utf16"));
+}
+# endif
+template <>
+inline void write_escaped_path<std::filesystem::path::value_type>(
+ basic_memory_buffer<std::filesystem::path::value_type>& quoted,
+ const std::filesystem::path& p) {
+ write_escaped_string<std::filesystem::path::value_type>(
+ std::back_inserter(quoted), p.native());
+}
+
+} // namespace detail
+
+FMT_MODULE_EXPORT
+template <typename Char>
+struct formatter<std::filesystem::path, Char>
+ : formatter<basic_string_view<Char>> {
+ template <typename ParseContext> FMT_CONSTEXPR auto parse(ParseContext& ctx) {
+ auto out = formatter<basic_string_view<Char>>::parse(ctx);
+ this->set_debug_format(false);
+ return out;
+ }
+ template <typename FormatContext>
+ auto format(const std::filesystem::path& p, FormatContext& ctx) const ->
+ typename FormatContext::iterator {
+ auto quoted = basic_memory_buffer<Char>();
+ detail::write_escaped_path(quoted, p);
+ return formatter<basic_string_view<Char>>::format(
+ basic_string_view<Char>(quoted.data(), quoted.size()), ctx);
+ }
+};
+FMT_END_NAMESPACE
+#endif
+
+FMT_BEGIN_NAMESPACE
+FMT_MODULE_EXPORT
+template <typename Char>
+struct formatter<std::thread::id, Char> : basic_ostream_formatter<Char> {};
+FMT_END_NAMESPACE
+
+#ifdef __cpp_lib_optional
+FMT_BEGIN_NAMESPACE
+FMT_MODULE_EXPORT
+template <typename T, typename Char>
+struct formatter<std::optional<T>, Char,
+ std::enable_if_t<is_formattable<T, Char>::value>> {
+ private:
+ formatter<T, Char> underlying_;
+ static constexpr basic_string_view<Char> optional =
+ detail::string_literal<Char, 'o', 'p', 't', 'i', 'o', 'n', 'a', 'l',
+ '('>{};
+ static constexpr basic_string_view<Char> none =
+ detail::string_literal<Char, 'n', 'o', 'n', 'e'>{};
+
+ template <class U>
+ FMT_CONSTEXPR static auto maybe_set_debug_format(U& u, bool set)
+ -> decltype(u.set_debug_format(set)) {
+ u.set_debug_format(set);
+ }
+
+ template <class U>
+ FMT_CONSTEXPR static void maybe_set_debug_format(U&, ...) {}
+
+ public:
+ template <typename ParseContext> FMT_CONSTEXPR auto parse(ParseContext& ctx) {
+ maybe_set_debug_format(underlying_, true);
+ return underlying_.parse(ctx);
+ }
+
+ template <typename FormatContext>
+ auto format(std::optional<T> const& opt, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ if (!opt) return detail::write<Char>(ctx.out(), none);
+
+ auto out = ctx.out();
+ out = detail::write<Char>(out, optional);
+ ctx.advance_to(out);
+ out = underlying_.format(*opt, ctx);
+ return detail::write(out, ')');
+ }
+};
+FMT_END_NAMESPACE
+#endif // __cpp_lib_optional
+
+#ifdef __cpp_lib_variant
+FMT_BEGIN_NAMESPACE
+FMT_MODULE_EXPORT
+template <typename Char> struct formatter<std::monostate, Char> {
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ return ctx.begin();
+ }
+
+ template <typename FormatContext>
+ auto format(const std::monostate&, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ auto out = ctx.out();
+ out = detail::write<Char>(out, "monostate");
+ return out;
+ }
+};
+
+namespace detail {
+
+template <typename T>
+using variant_index_sequence =
+ std::make_index_sequence<std::variant_size<T>::value>;
+
+template <typename> struct is_variant_like_ : std::false_type {};
+template <typename... Types>
+struct is_variant_like_<std::variant<Types...>> : std::true_type {};
+
+// formattable element check.
+template <typename T, typename C> class is_variant_formattable_ {
+ template <std::size_t... Is>
+ static std::conjunction<
+ is_formattable<std::variant_alternative_t<Is, T>, C>...>
+ check(std::index_sequence<Is...>);
+
+ public:
+ static constexpr const bool value =
+ decltype(check(variant_index_sequence<T>{}))::value;
+};
+
+template <typename Char, typename OutputIt, typename T>
+auto write_variant_alternative(OutputIt out, const T& v) -> OutputIt {
+ if constexpr (is_string<T>::value)
+ return write_escaped_string<Char>(out, detail::to_string_view(v));
+ else if constexpr (std::is_same_v<T, Char>)
+ return write_escaped_char(out, v);
+ else
+ return write<Char>(out, v);
+}
+
+} // namespace detail
+template <typename T> struct is_variant_like {
+ static constexpr const bool value = detail::is_variant_like_<T>::value;
+};
+
+template <typename T, typename C> struct is_variant_formattable {
+ static constexpr const bool value =
+ detail::is_variant_formattable_<T, C>::value;
+};
+
+FMT_MODULE_EXPORT
+template <typename Variant, typename Char>
+struct formatter<
+ Variant, Char,
+ std::enable_if_t<std::conjunction_v<
+ is_variant_like<Variant>, is_variant_formattable<Variant, Char>>>> {
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ return ctx.begin();
+ }
+
+ template <typename FormatContext>
+ auto format(const Variant& value, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ auto out = ctx.out();
+
+ out = detail::write<Char>(out, "variant(");
+ try {
+ std::visit(
+ [&](const auto& v) {
+ out = detail::write_variant_alternative<Char>(out, v);
+ },
+ value);
+ } catch (const std::bad_variant_access&) {
+ detail::write<Char>(out, "valueless by exception");
+ }
+ *out++ = ')';
+ return out;
+ }
+};
+FMT_END_NAMESPACE
+#endif // __cpp_lib_variant
+
+FMT_BEGIN_NAMESPACE
+FMT_MODULE_EXPORT
+template <typename Char> struct formatter<std::error_code, Char> {
+ template <typename ParseContext>
+ FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+ return ctx.begin();
+ }
+
+ template <typename FormatContext>
+ FMT_CONSTEXPR auto format(const std::error_code& ec, FormatContext& ctx) const
+ -> decltype(ctx.out()) {
+ auto out = ctx.out();
+ out = detail::write_bytes(out, ec.category().name(), format_specs<Char>());
+ out = detail::write<Char>(out, Char(':'));
+ out = detail::write<Char>(out, ec.value());
+ return out;
+ }
+};
+
+FMT_MODULE_EXPORT
+template <typename T, typename Char>
+struct formatter<
+ T, Char,
+ typename std::enable_if<std::is_base_of<std::exception, T>::value>::type> {
+ private:
+ bool with_typename_ = false;
+
+ public:
+ FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+ -> decltype(ctx.begin()) {
+ auto it = ctx.begin();
+ auto end = ctx.end();
+ if (it == end || *it == '}') return it;
+ if (*it == 't') {
+ ++it;
+ with_typename_ = true;
+ }
+ return it;
+ }
+
+ template <typename OutputIt>
+ auto format(const std::exception& ex,
+ basic_format_context<OutputIt, Char>& ctx) const -> OutputIt {
+ format_specs<Char> spec;
+ auto out = ctx.out();
+ if (!with_typename_)
+ return detail::write_bytes(out, string_view(ex.what()), spec);
+
+ const std::type_info& ti = typeid(ex);
+#ifdef FMT_HAS_ABI_CXA_DEMANGLE
+ int status = 0;
+ std::size_t size = 0;
+ std::unique_ptr<char, decltype(&std::free)> demangled_name_ptr(
+ abi::__cxa_demangle(ti.name(), nullptr, &size, &status), &std::free);
+
+ string_view demangled_name_view;
+ if (demangled_name_ptr) {
+ demangled_name_view = demangled_name_ptr.get();
+
+ // Normalization of stdlib inline namespace names.
+ // libc++ inline namespaces.
+ // std::__1::* -> std::*
+ // std::__1::__fs::* -> std::*
+ // libstdc++ inline namespaces.
+ // std::__cxx11::* -> std::*
+ // std::filesystem::__cxx11::* -> std::filesystem::*
+ if (demangled_name_view.starts_with("std::")) {
+ char* begin = demangled_name_ptr.get();
+ char* to = begin + 5; // std::
+ for (char *from = to, *end = begin + demangled_name_view.size();
+ from < end;) {
+ // This is safe, because demangled_name is NUL-terminated.
+ if (from[0] == '_' && from[1] == '_') {
+ char* next = from + 1;
+ while (next < end && *next != ':') next++;
+ if (next[0] == ':' && next[1] == ':') {
+ from = next + 2;
+ continue;
+ }
+ }
+ *to++ = *from++;
+ }
+ demangled_name_view = {begin, detail::to_unsigned(to - begin)};
+ }
+ } else {
+ demangled_name_view = string_view(ti.name());
+ }
+ out = detail::write_bytes(out, demangled_name_view, spec);
+#elif FMT_MSC_VERSION
+ string_view demangled_name_view(ti.name());
+ if (demangled_name_view.starts_with("class "))
+ demangled_name_view.remove_prefix(6);
+ else if (demangled_name_view.starts_with("struct "))
+ demangled_name_view.remove_prefix(7);
+ out = detail::write_bytes(out, demangled_name_view, spec);
+#else
+ out = detail::write_bytes(out, string_view(ti.name()), spec);
+#endif
+ out = detail::write<Char>(out, Char(':'));
+ out = detail::write<Char>(out, Char(' '));
+ out = detail::write_bytes(out, string_view(ex.what()), spec);
+
+ return out;
+ }
+};
+FMT_END_NAMESPACE
+
+#endif // FMT_STD_H_
diff --git a/contrib/fmt/include/fmt/xchar.h b/contrib/fmt/include/fmt/xchar.h
new file mode 100644
index 0000000..4b87f8d
--- /dev/null
+++ b/contrib/fmt/include/fmt/xchar.h
@@ -0,0 +1,259 @@
+// Formatting library for C++ - optional wchar_t and exotic character support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_XCHAR_H_
+#define FMT_XCHAR_H_
+
+#include <cwchar>
+
+#include "format.h"
+
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+# include <locale>
+#endif
+
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+template <typename T>
+using is_exotic_char = bool_constant<!std::is_same<T, char>::value>;
+
+inline auto write_loc(std::back_insert_iterator<detail::buffer<wchar_t>> out,
+ loc_value value, const format_specs<wchar_t>& specs,
+ locale_ref loc) -> bool {
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+ auto& numpunct =
+ std::use_facet<std::numpunct<wchar_t>>(loc.get<std::locale>());
+ auto separator = std::wstring();
+ auto grouping = numpunct.grouping();
+ if (!grouping.empty()) separator = std::wstring(1, numpunct.thousands_sep());
+ return value.visit(loc_writer<wchar_t>{out, specs, separator, grouping, {}});
+#endif
+ return false;
+}
+} // namespace detail
+
+FMT_BEGIN_EXPORT
+
+using wstring_view = basic_string_view<wchar_t>;
+using wformat_parse_context = basic_format_parse_context<wchar_t>;
+using wformat_context = buffer_context<wchar_t>;
+using wformat_args = basic_format_args<wformat_context>;
+using wmemory_buffer = basic_memory_buffer<wchar_t>;
+
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+// Workaround broken conversion on older gcc.
+template <typename... Args> using wformat_string = wstring_view;
+inline auto runtime(wstring_view s) -> wstring_view { return s; }
+#else
+template <typename... Args>
+using wformat_string = basic_format_string<wchar_t, type_identity_t<Args>...>;
+inline auto runtime(wstring_view s) -> runtime_format_string<wchar_t> {
+ return {{s}};
+}
+#endif
+
+template <> struct is_char<wchar_t> : std::true_type {};
+template <> struct is_char<detail::char8_type> : std::true_type {};
+template <> struct is_char<char16_t> : std::true_type {};
+template <> struct is_char<char32_t> : std::true_type {};
+
+template <typename... Args>
+constexpr format_arg_store<wformat_context, Args...> make_wformat_args(
+ const Args&... args) {
+ return {args...};
+}
+
+inline namespace literals {
+#if FMT_USE_USER_DEFINED_LITERALS && !FMT_USE_NONTYPE_TEMPLATE_ARGS
+constexpr detail::udl_arg<wchar_t> operator"" _a(const wchar_t* s, size_t) {
+ return {s};
+}
+#endif
+} // namespace literals
+
+template <typename It, typename Sentinel>
+auto join(It begin, Sentinel end, wstring_view sep)
+ -> join_view<It, Sentinel, wchar_t> {
+ return {begin, end, sep};
+}
+
+template <typename Range>
+auto join(Range&& range, wstring_view sep)
+ -> join_view<detail::iterator_t<Range>, detail::sentinel_t<Range>,
+ wchar_t> {
+ return join(std::begin(range), std::end(range), sep);
+}
+
+template <typename T>
+auto join(std::initializer_list<T> list, wstring_view sep)
+ -> join_view<const T*, const T*, wchar_t> {
+ return join(std::begin(list), std::end(list), sep);
+}
+
+template <typename Char, FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
+auto vformat(basic_string_view<Char> format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args)
+ -> std::basic_string<Char> {
+ basic_memory_buffer<Char> buffer;
+ detail::vformat_to(buffer, format_str, args);
+ return to_string(buffer);
+}
+
+template <typename... T>
+auto format(wformat_string<T...> fmt, T&&... args) -> std::wstring {
+ return vformat(fmt::wstring_view(fmt), fmt::make_wformat_args(args...));
+}
+
+// Pass char_t as a default template parameter instead of using
+// std::basic_string<char_t<S>> to reduce the symbol size.
+template <typename S, typename... Args, typename Char = char_t<S>,
+ FMT_ENABLE_IF(!std::is_same<Char, char>::value &&
+ !std::is_same<Char, wchar_t>::value)>
+auto format(const S& format_str, Args&&... args) -> std::basic_string<Char> {
+ return vformat(detail::to_string_view(format_str),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+template <typename Locale, typename S, typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_locale<Locale>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto vformat(
+ const Locale& loc, const S& format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args)
+ -> std::basic_string<Char> {
+ return detail::vformat(loc, detail::to_string_view(format_str), args);
+}
+
+template <typename Locale, typename S, typename... Args,
+ typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_locale<Locale>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto format(const Locale& loc, const S& format_str, Args&&... args)
+ -> std::basic_string<Char> {
+ return detail::vformat(loc, detail::to_string_view(format_str),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+template <typename OutputIt, typename S, typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_exotic_char<Char>::value)>
+auto vformat_to(OutputIt out, const S& format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args)
+ -> OutputIt {
+ auto&& buf = detail::get_buffer<Char>(out);
+ detail::vformat_to(buf, detail::to_string_view(format_str), args);
+ return detail::get_iterator(buf, out);
+}
+
+template <typename OutputIt, typename S, typename... Args,
+ typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto format_to(OutputIt out, const S& fmt, Args&&... args) -> OutputIt {
+ return vformat_to(out, detail::to_string_view(fmt),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+template <typename Locale, typename S, typename OutputIt, typename... Args,
+ typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_locale<Locale>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto vformat_to(
+ OutputIt out, const Locale& loc, const S& format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args) -> OutputIt {
+ auto&& buf = detail::get_buffer<Char>(out);
+ vformat_to(buf, detail::to_string_view(format_str), args,
+ detail::locale_ref(loc));
+ return detail::get_iterator(buf, out);
+}
+
+template <
+ typename OutputIt, typename Locale, typename S, typename... Args,
+ typename Char = char_t<S>,
+ bool enable = detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_locale<Locale>::value&& detail::is_exotic_char<Char>::value>
+inline auto format_to(OutputIt out, const Locale& loc, const S& format_str,
+ Args&&... args) ->
+ typename std::enable_if<enable, OutputIt>::type {
+ return vformat_to(out, loc, detail::to_string_view(format_str),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+template <typename OutputIt, typename Char, typename... Args,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto vformat_to_n(
+ OutputIt out, size_t n, basic_string_view<Char> format_str,
+ basic_format_args<buffer_context<type_identity_t<Char>>> args)
+ -> format_to_n_result<OutputIt> {
+ detail::iterator_buffer<OutputIt, Char, detail::fixed_buffer_traits> buf(out,
+ n);
+ detail::vformat_to(buf, format_str, args);
+ return {buf.out(), buf.count()};
+}
+
+template <typename OutputIt, typename S, typename... Args,
+ typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+ detail::is_exotic_char<Char>::value)>
+inline auto format_to_n(OutputIt out, size_t n, const S& fmt,
+ const Args&... args) -> format_to_n_result<OutputIt> {
+ return vformat_to_n(out, n, detail::to_string_view(fmt),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+}
+
+template <typename S, typename... Args, typename Char = char_t<S>,
+ FMT_ENABLE_IF(detail::is_exotic_char<Char>::value)>
+inline auto formatted_size(const S& fmt, Args&&... args) -> size_t {
+ detail::counting_buffer<Char> buf;
+ detail::vformat_to(buf, detail::to_string_view(fmt),
+ fmt::make_format_args<buffer_context<Char>>(args...));
+ return buf.count();
+}
+
+inline void vprint(std::FILE* f, wstring_view fmt, wformat_args args) {
+ wmemory_buffer buffer;
+ detail::vformat_to(buffer, fmt, args);
+ buffer.push_back(L'\0');
+ if (std::fputws(buffer.data(), f) == -1)
+ FMT_THROW(system_error(errno, FMT_STRING("cannot write to file")));
+}
+
+inline void vprint(wstring_view fmt, wformat_args args) {
+ vprint(stdout, fmt, args);
+}
+
+template <typename... T>
+void print(std::FILE* f, wformat_string<T...> fmt, T&&... args) {
+ return vprint(f, wstring_view(fmt), fmt::make_wformat_args(args...));
+}
+
+template <typename... T> void print(wformat_string<T...> fmt, T&&... args) {
+ return vprint(wstring_view(fmt), fmt::make_wformat_args(args...));
+}
+
+template <typename... T>
+void println(std::FILE* f, wformat_string<T...> fmt, T&&... args) {
+ return print(f, L"{}\n", fmt::format(fmt, std::forward<T>(args)...));
+}
+
+template <typename... T> void println(wformat_string<T...> fmt, T&&... args) {
+ return print(L"{}\n", fmt::format(fmt, std::forward<T>(args)...));
+}
+
+/**
+ Converts *value* to ``std::wstring`` using the default format for type *T*.
+ */
+template <typename T> inline auto to_wstring(const T& value) -> std::wstring {
+ return format(FMT_STRING(L"{}"), value);
+}
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif // FMT_XCHAR_H_
diff --git a/contrib/fpconv/CMakeLists.txt b/contrib/fpconv/CMakeLists.txt
new file mode 100644
index 0000000..b330525
--- /dev/null
+++ b/contrib/fpconv/CMakeLists.txt
@@ -0,0 +1,7 @@
+SET(FPCONVSRC fpconv.c)
+
+SET(FTPCONV_COMPILE_FLAGS "-DRSPAMD_LIB")
+
+ADD_LIBRARY(rspamd-fpconv STATIC ${FPCONVSRC})
+SET_TARGET_PROPERTIES(rspamd-fpconv PROPERTIES VERSION ${RSPAMD_VERSION})
+SET_TARGET_PROPERTIES(rspamd-fpconv PROPERTIES COMPILE_FLAGS "${FTPCONV_COMPILE_FLAGS}") \ No newline at end of file
diff --git a/contrib/fpconv/LICENSE b/contrib/fpconv/LICENSE
new file mode 100644
index 0000000..36b7cd9
--- /dev/null
+++ b/contrib/fpconv/LICENSE
@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/contrib/fpconv/fpconv.c b/contrib/fpconv/fpconv.c
new file mode 100644
index 0000000..f8b601a
--- /dev/null
+++ b/contrib/fpconv/fpconv.c
@@ -0,0 +1,480 @@
+#include <stdbool.h>
+#include <string.h>
+#include <sys/param.h>
+
+#include "fpconv.h"
+#include "powers.h"
+
+#define fracmask 0x000FFFFFFFFFFFFFU
+#define expmask 0x7FF0000000000000U
+#define hiddenbit 0x0010000000000000U
+#define signmask 0x8000000000000000U
+#define expbias (1023 + 52)
+
+#define absv(n) ((n) < 0 ? -(n) : (n))
+#define minv(a, b) ((a) < (b) ? (a) : (b))
+
+static uint64_t tens[] = {
+ 10000000000000000000U, 1000000000000000000U, 100000000000000000U,
+ 10000000000000000U, 1000000000000000U, 100000000000000U,
+ 10000000000000U, 1000000000000U, 100000000000U,
+ 10000000000U, 1000000000U, 100000000U,
+ 10000000U, 1000000U, 100000U,
+ 10000U, 1000U, 100U,
+ 10U, 1U
+};
+
+static inline uint64_t get_dbits (double d) {
+ union {
+ double dbl;
+ uint64_t i;
+ } dbl_bits = {d};
+
+ return dbl_bits.i;
+}
+
+static Fp build_fp (double d) {
+ uint64_t bits = get_dbits (d);
+
+ Fp fp;
+ fp.frac = bits & fracmask;
+ fp.exp = (bits & expmask) >> 52u;
+
+ if (fp.exp) {
+ fp.frac += hiddenbit;
+ fp.exp -= expbias;
+
+ }
+ else {
+ fp.exp = -expbias + 1;
+ }
+
+ return fp;
+}
+
+static void normalize (Fp *fp) {
+ while ((fp->frac & hiddenbit) == 0) {
+ fp->frac <<= 1u;
+ fp->exp--;
+ }
+
+ const unsigned int shift = 64 - 52 - 1;
+ fp->frac <<= shift;
+ fp->exp -= shift;
+}
+
+static void get_normalized_boundaries (Fp *fp, Fp *lower, Fp *upper) {
+ upper->frac = (fp->frac << 1u) + 1u;
+ upper->exp = fp->exp - 1u;
+
+ while ((upper->frac & (hiddenbit << 1u)) == 0) {
+ upper->frac <<= 1u;
+ upper->exp--;
+ }
+
+ const unsigned int u_shift = 64 - 52 - 2;
+
+ upper->frac <<= u_shift;
+ upper->exp = upper->exp - u_shift;
+
+
+ unsigned int l_shift = fp->frac == hiddenbit ? 2u : 1u;
+
+ lower->frac = (fp->frac << l_shift) - 1;
+ lower->exp = fp->exp - l_shift;
+
+
+ lower->frac <<= lower->exp - upper->exp;
+ lower->exp = upper->exp;
+}
+
+static Fp multiply (Fp *a, Fp *b) {
+ const uint64_t lomask = 0x00000000FFFFFFFFu;
+
+ uint64_t ah_bl = (a->frac >> 32u) * (b->frac & lomask);
+ uint64_t al_bh = (a->frac & lomask) * (b->frac >> 32u);
+ uint64_t al_bl = (a->frac & lomask) * (b->frac & lomask);
+ uint64_t ah_bh = (a->frac >> 32u) * (b->frac >> 32u);
+
+ uint64_t tmp = (ah_bl & lomask) + (al_bh & lomask) + (al_bl >> 32u);
+ /* round up */
+ tmp += 1U << 31u;
+
+ Fp fp = {
+ ah_bh + (ah_bl >> 32u) + (al_bh >> 32u) + (tmp >> 32u),
+ a->exp + b->exp + 64u
+ };
+
+ return fp;
+}
+
+static void round_digit (char *digits, int ndigits, uint64_t delta, uint64_t rem, uint64_t kappa, uint64_t frac) {
+ while (rem < frac && delta - rem >= kappa &&
+ (rem + kappa < frac || frac - rem > rem + kappa - frac)) {
+
+ digits[ndigits - 1]--;
+ rem += kappa;
+ }
+}
+
+static int generate_digits (Fp *fp, Fp *upper, Fp *lower, char *digits, int *K) {
+ uint64_t wfrac = upper->frac - fp->frac;
+ uint64_t delta = upper->frac - lower->frac;
+
+ Fp one;
+ one.frac = 1ULL << -upper->exp;
+ one.exp = upper->exp;
+
+ uint64_t part1 = upper->frac >> -one.exp;
+ uint64_t part2 = upper->frac & (one.frac - 1);
+
+ int idx = 0, kappa = 10;
+ uint64_t *divp;
+ /* 1000000000 */
+ for (divp = tens + 10; kappa > 0; divp++) {
+
+ uint64_t div = *divp;
+ unsigned digit = part1 / div;
+
+ if (digit || idx) {
+ digits[idx++] = digit + '0';
+ }
+
+ part1 -= digit * div;
+ kappa--;
+
+ uint64_t tmp = (part1 << -one.exp) + part2;
+ if (tmp <= delta) {
+ *K += kappa;
+ round_digit (digits, idx, delta, tmp, div << -one.exp, wfrac);
+
+ return idx;
+ }
+ }
+
+ /* 10 */
+ uint64_t *unit = tens + 18;
+
+ while (true) {
+ part2 *= 10;
+ delta *= 10;
+ kappa--;
+
+ unsigned digit = part2 >> -one.exp;
+ if (digit || idx) {
+ digits[idx++] = digit + '0';
+ }
+
+ part2 &= one.frac - 1;
+ if (part2 < delta) {
+ *K += kappa;
+ round_digit (digits, idx, delta, part2, one.frac, wfrac * *unit);
+
+ return idx;
+ }
+
+ unit--;
+ }
+}
+
+static int grisu2 (double d, char *digits, int *K) {
+ Fp w = build_fp (d);
+
+ Fp lower, upper;
+ get_normalized_boundaries (&w, &lower, &upper);
+
+ normalize (&w);
+
+ int k;
+ Fp cp = find_cachedpow10 (upper.exp, &k);
+
+ w = multiply (&w, &cp);
+ upper = multiply (&upper, &cp);
+ lower = multiply (&lower, &cp);
+
+ lower.frac++;
+ upper.frac--;
+
+ *K = -k;
+
+ return generate_digits (&w, &upper, &lower, digits, K);
+}
+
+static inline int emit_integer (char *digits, int ndigits,
+ char *dest, int K, bool neg,
+ unsigned precision)
+{
+ char *d = dest;
+
+ memcpy (d, digits, ndigits);
+ d += ndigits;
+ memset (d, '0', K);
+ d += K;
+
+ precision = MIN(precision, FPCONV_BUFLEN - (ndigits + K + 1));
+
+ if (precision) {
+ *d++ = '.';
+ memset (d, '0', precision);
+ d += precision;
+ }
+
+ return d - dest;
+}
+
+static inline int emit_scientific_digits (char *digits, int ndigits,
+ char *dest, int K, bool neg,
+ unsigned precision, int exp)
+{
+ /* write decimal w/ scientific notation */
+ ndigits = minv(ndigits, 18 - neg);
+
+ int idx = 0;
+ dest[idx++] = digits[0];
+
+ if (ndigits > 1) {
+ dest[idx++] = '.';
+ memcpy(dest + idx, digits + 1, ndigits - 1);
+ idx += ndigits - 1;
+ }
+
+ dest[idx++] = 'e';
+
+ char sign = K + ndigits - 1 < 0 ? '-' : '+';
+ dest[idx++] = sign;
+
+ int cent = 0;
+
+ if (exp > 99) {
+ cent = exp / 100;
+ dest[idx++] = cent + '0';
+ exp -= cent * 100;
+ }
+ if (exp > 9) {
+ int dec = exp / 10;
+ dest[idx++] = dec + '0';
+ exp -= dec * 10;
+
+ }
+ else if (cent) {
+ dest[idx++] = '0';
+ }
+
+ dest[idx++] = exp % 10 + '0';
+
+ return idx;
+}
+
+static inline int emit_fixed_digits (char *digits, int ndigits,
+ char *dest, int K, bool neg,
+ unsigned precision, int exp)
+{
+ int offset = ndigits - absv(K), to_print;
+ /* fp < 1.0 -> write leading zero */
+ if (K < 0) {
+ if (offset <= 0) {
+ if (precision) {
+ if (-offset >= precision) {
+ /* Just print 0.[0]{precision} */
+ dest[0] = '0';
+ dest[1] = '.';
+ memset(dest + 2, '0', precision);
+
+ return precision + 2;
+ }
+
+ to_print = MAX(ndigits - offset, precision);
+ }
+ else {
+ to_print = ndigits - offset;
+ }
+
+ if (to_print <= FPCONV_BUFLEN - 3) {
+ offset = -offset;
+ dest[0] = '0';
+ dest[1] = '.';
+ memset(dest + 2, '0', offset);
+
+ if (precision) {
+ /* The case where offset > precision is covered previously */
+ precision -= offset;
+
+ if (precision <= ndigits) {
+ /* Truncate or leave as is */
+ memcpy(dest + offset + 2, digits, precision);
+
+ return precision + 2 + offset;
+ }
+ else {
+ /* Expand */
+ memcpy(dest + offset + 2, digits, ndigits);
+ precision -= ndigits;
+ memset(dest + offset + 2 + ndigits, '0', precision);
+
+ return ndigits + 2 + offset + precision;
+ }
+ }
+ else {
+ memcpy(dest + offset + 2, digits, ndigits);
+ }
+
+ return ndigits + 2 + offset;
+ }
+ else {
+ return emit_scientific_digits (digits, ndigits, dest, K, neg, precision, exp);
+ }
+ }
+ else {
+ /*
+ * fp > 1.0, if offset > 0 then we have less digits than
+ * fp exponent, so we need to switch to scientific notation to
+ * display number at least more or less precisely
+ */
+ if (offset > 0 && ndigits <= FPCONV_BUFLEN - 3) {
+ char *d = dest;
+ memcpy(d, digits, offset);
+ d += offset;
+ *d++ = '.';
+
+ ndigits -= offset;
+
+ if (precision) {
+ if (ndigits >= precision) {
+ /* Truncate or leave as is */
+ memcpy(d, digits + offset, precision);
+ d += precision;
+ }
+ else {
+ /* Expand */
+ memcpy(d, digits + offset, ndigits);
+ precision -= ndigits;
+ d += ndigits;
+
+ /* Check if we have enough bufspace */
+ if ((d - dest) + precision <= FPCONV_BUFLEN) {
+ memset (d, '0', precision);
+ d += precision;
+ }
+ else {
+ memset (d, '0', FPCONV_BUFLEN - (d - dest));
+ d += FPCONV_BUFLEN - (d - dest);
+ }
+ }
+ }
+ else {
+ memcpy(d, digits + offset, ndigits);
+ d += ndigits;
+ }
+
+ return d - dest;
+ }
+ }
+ }
+
+ return emit_scientific_digits (digits, ndigits, dest, K, neg, precision, exp);
+}
+
+static int emit_digits (char *digits, int ndigits, char *dest, int K, bool neg,
+ unsigned precision, bool scientific)
+{
+ int exp = absv(K + ndigits - 1);
+
+ /* write plain integer */
+ if (K >= 0 && (exp < (ndigits + 7))) {
+ return emit_integer (digits, ndigits, dest, K, neg, precision);
+ }
+
+ /* write decimal w/o scientific notation */
+ if (!scientific || (K < 0 && (K > -7 || exp < 4))) {
+ return emit_fixed_digits (digits, ndigits, dest, K, neg, precision, exp);
+ }
+
+ return emit_scientific_digits (digits, ndigits, dest, K, neg, precision, exp);
+}
+
+static int filter_special (double fp, char *dest, unsigned precision)
+{
+ int nchars = 3;
+ char *d = dest;
+
+ if (fp == 0.0) {
+ if (get_dbits (fp) & signmask) {
+ *d++ = '-';
+ *d++ = '0';
+ }
+ else {
+ *d++ = '0';
+ }
+
+ if (precision) {
+ *d ++ = '.';
+ memset (d, '0', precision);
+ }
+
+ return d - dest + precision;
+ }
+
+ uint64_t bits = get_dbits (fp);
+
+ bool nan = (bits & expmask) == expmask;
+
+ if (!nan) {
+ return 0;
+ }
+
+ if (bits & fracmask) {
+ dest[0] = 'n';
+ dest[1] = 'a';
+ dest[2] = 'n';
+ }
+ else {
+ if (get_dbits (fp) & signmask) {
+ dest[0] = '-';
+ dest[1] = 'i';
+ dest[2] = 'n';
+ dest[3] = 'f';
+ nchars = 4;
+ }
+ else {
+ dest[0] = 'i';
+ dest[1] = 'n';
+ dest[2] = 'f';
+ }
+ }
+
+ return nchars;
+}
+
+int
+fpconv_dtoa (double d, char dest[FPCONV_BUFLEN],
+ unsigned precision, bool scientific)
+{
+ char digits[18];
+
+ int str_len = 0;
+ bool neg = false;
+
+ if (precision > FPCONV_BUFLEN - 5) {
+ precision = FPCONV_BUFLEN - 5;
+ }
+
+ int spec = filter_special (d, dest, precision);
+
+ if (spec) {
+ return spec;
+ }
+
+ if (get_dbits (d) & signmask) {
+ dest[0] = '-';
+ str_len++;
+ neg = true;
+ }
+
+ int K = 0;
+ int ndigits = grisu2 (d, digits, &K);
+
+ str_len += emit_digits (digits, ndigits, dest + str_len, K, neg, precision,
+ scientific);
+
+ return str_len;
+}
diff --git a/contrib/fpconv/fpconv.h b/contrib/fpconv/fpconv.h
new file mode 100644
index 0000000..8c07c13
--- /dev/null
+++ b/contrib/fpconv/fpconv.h
@@ -0,0 +1,35 @@
+#ifndef FPCONV_H
+#define FPCONV_H
+
+#define FPCONV_BUFLEN 32
+/* Fast and accurate double to string conversion based on Florian Loitsch's
+ * Grisu-algorithm[1].
+ *
+ * Input:
+ * fp -> the double to convert, dest -> destination buffer.
+ * The generated string will never be longer than 24 characters.
+ * Make sure to pass a pointer to at least 24 bytes of memory.
+ * The emitted string will not be null terminated.
+ *
+ * Output:
+ * The number of written characters.
+ *
+ * Exemplary usage:
+ *
+ * void print(double d)
+ * {
+ * char buf[24 + 1] // plus null terminator
+ * int str_len = fpconv_dtoa(d, buf);
+ *
+ * buf[str_len] = '\0';
+ * printf("%s", buf);
+ * }
+ *
+ */
+
+int fpconv_dtoa(double fp, char dest[FPCONV_BUFLEN], unsigned precision,
+ bool scientific);
+
+#endif
+
+/* [1] http://florian.loitsch.com/publications/dtoa-pldi2010.pdf */
diff --git a/contrib/fpconv/powers.h b/contrib/fpconv/powers.h
new file mode 100644
index 0000000..c707eed
--- /dev/null
+++ b/contrib/fpconv/powers.h
@@ -0,0 +1,87 @@
+#include <stdint.h>
+
+#define npowers 87
+#define steppowers 8
+#define firstpower -348 /* 10 ^ -348 */
+
+#define expmax -32
+#define expmin -60
+
+
+typedef struct Fp {
+ uint64_t frac;
+ int exp;
+} Fp;
+
+static Fp powers_ten[] = {
+ { 18054884314459144840U, -1220 }, { 13451937075301367670U, -1193 },
+ { 10022474136428063862U, -1166 }, { 14934650266808366570U, -1140 },
+ { 11127181549972568877U, -1113 }, { 16580792590934885855U, -1087 },
+ { 12353653155963782858U, -1060 }, { 18408377700990114895U, -1034 },
+ { 13715310171984221708U, -1007 }, { 10218702384817765436U, -980 },
+ { 15227053142812498563U, -954 }, { 11345038669416679861U, -927 },
+ { 16905424996341287883U, -901 }, { 12595523146049147757U, -874 },
+ { 9384396036005875287U, -847 }, { 13983839803942852151U, -821 },
+ { 10418772551374772303U, -794 }, { 15525180923007089351U, -768 },
+ { 11567161174868858868U, -741 }, { 17236413322193710309U, -715 },
+ { 12842128665889583758U, -688 }, { 9568131466127621947U, -661 },
+ { 14257626930069360058U, -635 }, { 10622759856335341974U, -608 },
+ { 15829145694278690180U, -582 }, { 11793632577567316726U, -555 },
+ { 17573882009934360870U, -529 }, { 13093562431584567480U, -502 },
+ { 9755464219737475723U, -475 }, { 14536774485912137811U, -449 },
+ { 10830740992659433045U, -422 }, { 16139061738043178685U, -396 },
+ { 12024538023802026127U, -369 }, { 17917957937422433684U, -343 },
+ { 13349918974505688015U, -316 }, { 9946464728195732843U, -289 },
+ { 14821387422376473014U, -263 }, { 11042794154864902060U, -236 },
+ { 16455045573212060422U, -210 }, { 12259964326927110867U, -183 },
+ { 18268770466636286478U, -157 }, { 13611294676837538539U, -130 },
+ { 10141204801825835212U, -103 }, { 15111572745182864684U, -77 },
+ { 11258999068426240000U, -50 }, { 16777216000000000000U, -24 },
+ { 12500000000000000000U, 3 }, { 9313225746154785156U, 30 },
+ { 13877787807814456755U, 56 }, { 10339757656912845936U, 83 },
+ { 15407439555097886824U, 109 }, { 11479437019748901445U, 136 },
+ { 17105694144590052135U, 162 }, { 12744735289059618216U, 189 },
+ { 9495567745759798747U, 216 }, { 14149498560666738074U, 242 },
+ { 10542197943230523224U, 269 }, { 15709099088952724970U, 295 },
+ { 11704190886730495818U, 322 }, { 17440603504673385349U, 348 },
+ { 12994262207056124023U, 375 }, { 9681479787123295682U, 402 },
+ { 14426529090290212157U, 428 }, { 10748601772107342003U, 455 },
+ { 16016664761464807395U, 481 }, { 11933345169920330789U, 508 },
+ { 17782069995880619868U, 534 }, { 13248674568444952270U, 561 },
+ { 9871031767461413346U, 588 }, { 14708983551653345445U, 614 },
+ { 10959046745042015199U, 641 }, { 16330252207878254650U, 667 },
+ { 12166986024289022870U, 694 }, { 18130221999122236476U, 720 },
+ { 13508068024458167312U, 747 }, { 10064294952495520794U, 774 },
+ { 14996968138956309548U, 800 }, { 11173611982879273257U, 827 },
+ { 16649979327439178909U, 853 }, { 12405201291620119593U, 880 },
+ { 9242595204427927429U, 907 }, { 13772540099066387757U, 933 },
+ { 10261342003245940623U, 960 }, { 15290591125556738113U, 986 },
+ { 11392378155556871081U, 1013 }, { 16975966327722178521U, 1039 },
+ { 12648080533535911531U, 1066 }
+};
+
+static Fp find_cachedpow10(int exp, int* k)
+{
+ const double one_log_ten = 0.30102999566398114;
+
+ int approx = -(exp + npowers) * one_log_ten;
+ int idx = (approx - firstpower) / steppowers;
+
+ while(1) {
+ int current = exp + powers_ten[idx].exp + 64;
+
+ if(current < expmin) {
+ idx++;
+ continue;
+ }
+
+ if(current > expmax) {
+ idx--;
+ continue;
+ }
+
+ *k = (firstpower + idx * steppowers);
+
+ return powers_ten[idx];
+ }
+}
diff --git a/contrib/frozen/AUTHORS b/contrib/frozen/AUTHORS
new file mode 100644
index 0000000..d83d0f8
--- /dev/null
+++ b/contrib/frozen/AUTHORS
@@ -0,0 +1,3 @@
+serge-sans-paille <sguelton@quarkslab.com>
+Jérôme Dumesnil <jerome.dumesnil@gmail.com>
+Chris Beck <chbeck@tesla.com>
diff --git a/contrib/frozen/CMakeLists.txt b/contrib/frozen/CMakeLists.txt
new file mode 100644
index 0000000..185378d
--- /dev/null
+++ b/contrib/frozen/CMakeLists.txt
@@ -0,0 +1,12 @@
+target_sources(frozen-headers INTERFACE
+ "${prefix}/frozen/algorithm.h"
+ "${prefix}/frozen/map.h"
+ "${prefix}/frozen/random.h"
+ "${prefix}/frozen/set.h"
+ "${prefix}/frozen/string.h"
+ "${prefix}/frozen/unordered_map.h"
+ "${prefix}/frozen/unordered_set.h"
+ "${prefix}/frozen/bits/algorithms.h"
+ "${prefix}/frozen/bits/basic_types.h"
+ "${prefix}/frozen/bits/elsa.h"
+ "${prefix}/frozen/bits/pmh.h")
diff --git a/contrib/frozen/LICENSE b/contrib/frozen/LICENSE
new file mode 100644
index 0000000..5b4b9bd
--- /dev/null
+++ b/contrib/frozen/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2017 Quarkslab
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/contrib/frozen/include/frozen/algorithm.h b/contrib/frozen/include/frozen/algorithm.h
new file mode 100644
index 0000000..a543eb3
--- /dev/null
+++ b/contrib/frozen/include/frozen/algorithm.h
@@ -0,0 +1,197 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_ALGORITHM_H
+#define FROZEN_LETITGO_ALGORITHM_H
+
+#include "frozen/bits/basic_types.h"
+#include "frozen/bits/version.h"
+#include "frozen/string.h"
+
+namespace frozen {
+
+// 'search' implementation if C++17 is not available
+// https://en.cppreference.com/w/cpp/algorithm/search
+template<class ForwardIterator, class Searcher>
+ForwardIterator search(ForwardIterator first, ForwardIterator last, const Searcher & searcher)
+{
+ return searcher(first, last).first;
+}
+
+// text book implementation from
+// https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm
+
+template <std::size_t size> class knuth_morris_pratt_searcher {
+ bits::carray<std::ptrdiff_t, size> step_;
+ bits::carray<char, size> needle_;
+
+ static constexpr bits::carray<std::ptrdiff_t, size>
+ build_kmp_cache(char const (&needle)[size + 1]) {
+ std::ptrdiff_t cnd = 0;
+ bits::carray<std::ptrdiff_t, size> cache;
+
+ cache.fill(-1);
+ for (std::size_t pos = 1; pos < size; ++pos) {
+ if (needle[pos] == needle[cnd]) {
+ cache[pos] = cache[cnd];
+ cnd += 1;
+ } else {
+ cache[pos] = cnd;
+ cnd = cache[cnd];
+ while (cnd >= 0 && needle[pos] != needle[cnd])
+ cnd = cache[cnd];
+ cnd += 1;
+ }
+ }
+ return cache;
+ }
+
+public:
+ constexpr knuth_morris_pratt_searcher(char const (&needle)[size + 1])
+ : step_{build_kmp_cache(needle)}, needle_(needle) {}
+
+ template <class ForwardIterator>
+ constexpr std::pair<ForwardIterator, ForwardIterator> operator()(ForwardIterator first, ForwardIterator last) const {
+ std::size_t i = 0;
+ ForwardIterator iter = first;
+ while (iter != last) {
+ if (needle_[i] == *iter) {
+ if (i == (size - 1))
+ return { iter - i, iter - i + size };
+ ++i;
+ ++iter;
+ } else {
+ if (step_[i] > -1) {
+ i = step_[i];
+ } else {
+ ++iter;
+ i = 0;
+ }
+ }
+ }
+ return { last, last };
+ }
+};
+
+template <std::size_t N>
+constexpr knuth_morris_pratt_searcher<N - 1> make_knuth_morris_pratt_searcher(char const (&needle)[N]) {
+ return {needle};
+}
+
+// text book implementation from
+// https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore%E2%80%93Horspool_algorithm
+
+template <std::size_t size> class boyer_moore_searcher {
+ using skip_table_type = bits::carray<std::ptrdiff_t, sizeof(char) << 8>;
+ using suffix_table_type = bits::carray<std::ptrdiff_t, size>;
+
+ skip_table_type skip_table_;
+ suffix_table_type suffix_table_;
+ bits::carray<char, size> needle_;
+
+ constexpr auto build_skip_table(char const (&needle)[size + 1]) {
+ skip_table_type skip_table;
+
+ skip_table.fill(size);
+ for (std::size_t i = 0; i < size - 1; ++i)
+ skip_table[needle[i]] -= i + 1;
+ return skip_table;
+ }
+
+ constexpr bool is_prefix(char const (&needle)[size + 1], std::size_t pos) {
+ std::size_t suffixlen = size - pos;
+
+ for (std::size_t i = 0; i < suffixlen; i++) {
+ if (needle[i] != needle[pos + i])
+ return false;
+ }
+ return true;
+ }
+
+ constexpr std::size_t suffix_length(char const (&needle)[size + 1],
+ std::size_t pos) {
+ // increment suffix length slen to the first mismatch or beginning
+ // of the word
+ for (std::size_t slen = 0; slen < pos ; slen++)
+ if (needle[pos - slen] != needle[size - 1 - slen])
+ return slen;
+
+ return pos;
+ }
+
+ constexpr auto build_suffix_table(char const (&needle)[size + 1]) {
+ suffix_table_type suffix;
+ std::ptrdiff_t last_prefix_index = size - 1;
+
+ // first loop
+ for (std::ptrdiff_t p = size - 1; p >= 0; p--) {
+ if (is_prefix(needle, p + 1))
+ last_prefix_index = p + 1;
+
+ suffix[p] = last_prefix_index + (size - 1 - p);
+ }
+
+ // second loop
+ for (std::size_t p = 0; p < size - 1; p++) {
+ auto slen = suffix_length(needle, p);
+ if (needle[p - slen] != needle[size - 1 - slen])
+ suffix[size - 1 - slen] = size - 1 - p + slen;
+
+ }
+ return suffix;
+ }
+
+public:
+ constexpr boyer_moore_searcher(char const (&needle)[size + 1])
+ : skip_table_{build_skip_table(needle)},
+ suffix_table_{build_suffix_table(needle)},
+ needle_(needle) {}
+
+ template <class ForwardIterator>
+ constexpr std::pair<ForwardIterator, ForwardIterator> operator()(ForwardIterator first, ForwardIterator last) const {
+ if (size == 0)
+ return { first, first + size };
+
+ ForwardIterator iter = first + size - 1;
+ while (iter < last) {
+ std::ptrdiff_t j = size - 1;
+ while (j > 0 && (*iter == needle_[j])) {
+ --iter;
+ --j;
+ }
+ if (*iter == needle_[0])
+ return { iter, iter + size};
+
+ iter += std::max(skip_table_[*iter], suffix_table_[j]);
+ }
+ return { last, last + size};
+ }
+};
+
+template <std::size_t N>
+constexpr boyer_moore_searcher<N - 1> make_boyer_moore_searcher(char const (&needle)[N]) {
+ return {needle};
+}
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/algorithms.h b/contrib/frozen/include/frozen/bits/algorithms.h
new file mode 100644
index 0000000..8d1ffbc
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/algorithms.h
@@ -0,0 +1,229 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_BITS_ALGORITHMS_H
+#define FROZEN_LETITGO_BITS_ALGORITHMS_H
+
+#include "frozen/bits/basic_types.h"
+
+#include <limits>
+#include <tuple>
+
+namespace frozen {
+
+namespace bits {
+
+auto constexpr next_highest_power_of_two(std::size_t v) {
+ // https://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
+ constexpr auto trip_count = std::numeric_limits<decltype(v)>::digits;
+ v--;
+ for(std::size_t i = 1; i < trip_count; i <<= 1)
+ v |= v >> i;
+ v++;
+ return v;
+}
+
+template<class T>
+auto constexpr log(T v) {
+ std::size_t n = 0;
+ while (v > 1) {
+ n += 1;
+ v >>= 1;
+ }
+ return n;
+}
+
+constexpr std::size_t bit_weight(std::size_t n) {
+ return (n <= 8*sizeof(unsigned int))
+ + (n <= 8*sizeof(unsigned long))
+ + (n <= 8*sizeof(unsigned long long))
+ + (n <= 128);
+}
+
+unsigned int select_uint_least(std::integral_constant<std::size_t, 4>);
+unsigned long select_uint_least(std::integral_constant<std::size_t, 3>);
+unsigned long long select_uint_least(std::integral_constant<std::size_t, 2>);
+template<std::size_t N>
+unsigned long long select_uint_least(std::integral_constant<std::size_t, N>) {
+ static_assert(N < 2, "unsupported type size");
+ return {};
+}
+
+
+template<std::size_t N>
+using select_uint_least_t = decltype(select_uint_least(std::integral_constant<std::size_t, bit_weight(N)>()));
+
+template <typename Iter, typename Compare>
+constexpr auto min_element(Iter begin, const Iter end,
+ Compare const &compare) {
+ auto result = begin;
+ while (begin != end) {
+ if (compare(*begin, *result)) {
+ result = begin;
+ }
+ ++begin;
+ }
+ return result;
+}
+
+template <class T>
+constexpr void cswap(T &a, T &b) {
+ auto tmp = a;
+ a = b;
+ b = tmp;
+}
+
+template <class T, class U>
+constexpr void cswap(std::pair<T, U> & a, std::pair<T, U> & b) {
+ cswap(a.first, b.first);
+ cswap(a.second, b.second);
+}
+
+template <class... Tys, std::size_t... Is>
+constexpr void cswap(std::tuple<Tys...> &a, std::tuple<Tys...> &b, std::index_sequence<Is...>) {
+ using swallow = int[];
+ (void) swallow{(cswap(std::get<Is>(a), std::get<Is>(b)), 0)...};
+}
+
+template <class... Tys>
+constexpr void cswap(std::tuple<Tys...> &a, std::tuple<Tys...> &b) {
+ cswap(a, b, std::make_index_sequence<sizeof...(Tys)>());
+}
+
+template <typename Iterator, class Compare>
+constexpr Iterator partition(Iterator left, Iterator right, Compare const &compare) {
+ auto pivot = left + (right - left) / 2;
+ auto value = *pivot;
+ cswap(*right, *pivot);
+ for (auto it = left; 0 < right - it; ++it) {
+ if (compare(*it, value)) {
+ cswap(*it, *left);
+ left++;
+ }
+ }
+ cswap(*right, *left);
+ return left;
+}
+
+template <typename Iterator, class Compare>
+constexpr void quicksort(Iterator left, Iterator right, Compare const &compare) {
+ while (0 < right - left) {
+ auto new_pivot = bits::partition(left, right, compare);
+ quicksort(left, new_pivot, compare);
+ left = new_pivot + 1;
+ }
+}
+
+template <typename T, std::size_t N, class Compare>
+constexpr bits::carray<T, N> quicksort(bits::carray<T, N> const &array,
+ Compare const &compare) {
+ bits::carray<T, N> res = array;
+ quicksort(res.begin(), res.end() - 1, compare);
+ return res;
+}
+
+template <class T, class Compare> struct LowerBound {
+ T const &value_;
+ Compare const &compare_;
+ constexpr LowerBound(T const &value, Compare const &compare)
+ : value_(value), compare_(compare) {}
+
+ template <class ForwardIt>
+ inline constexpr ForwardIt doit_fast(ForwardIt first,
+ std::integral_constant<std::size_t, 0>) {
+ return first;
+ }
+
+ template <class ForwardIt, std::size_t N>
+ inline constexpr ForwardIt doit_fast(ForwardIt first,
+ std::integral_constant<std::size_t, N>) {
+ auto constexpr step = N / 2;
+ static_assert(N/2 == N - N / 2 - 1, "power of two minus 1");
+ auto it = first + step;
+ auto next_it = compare_(*it, value_) ? it + 1 : first;
+ return doit_fast(next_it, std::integral_constant<std::size_t, N / 2>{});
+ }
+
+ template <class ForwardIt, std::size_t N>
+ inline constexpr ForwardIt doitfirst(ForwardIt first, std::integral_constant<std::size_t, N>, std::integral_constant<bool, true>) {
+ return doit_fast(first, std::integral_constant<std::size_t, N>{});
+ }
+
+ template <class ForwardIt, std::size_t N>
+ inline constexpr ForwardIt doitfirst(ForwardIt first, std::integral_constant<std::size_t, N>, std::integral_constant<bool, false>) {
+ auto constexpr next_power = next_highest_power_of_two(N);
+ auto constexpr next_start = next_power / 2 - 1;
+ auto it = first + next_start;
+ if (compare_(*it, value_)) {
+ auto constexpr next = N - next_start - 1;
+ return doitfirst(it + 1, std::integral_constant<std::size_t, next>{}, std::integral_constant<bool, next_highest_power_of_two(next) - 1 == next>{});
+ }
+ else
+ return doit_fast(first, std::integral_constant<std::size_t, next_start>{});
+ }
+
+ template <class ForwardIt>
+ inline constexpr ForwardIt doitfirst(ForwardIt first, std::integral_constant<std::size_t, 1>, std::integral_constant<bool, false>) {
+ return doit_fast(first, std::integral_constant<std::size_t, 1>{});
+ }
+};
+
+template <std::size_t N, class ForwardIt, class T, class Compare>
+constexpr ForwardIt lower_bound(ForwardIt first, const T &value, Compare const &compare) {
+ return LowerBound<T, Compare>{value, compare}.doitfirst(first, std::integral_constant<std::size_t, N>{}, std::integral_constant<bool, next_highest_power_of_two(N) - 1 == N>{});
+}
+
+template <std::size_t N, class Compare, class ForwardIt, class T>
+constexpr bool binary_search(ForwardIt first, const T &value,
+ Compare const &compare) {
+ ForwardIt where = lower_bound<N>(first, value, compare);
+ return (!(where == first + N) && !(compare(value, *where)));
+}
+
+
+template<class InputIt1, class InputIt2>
+constexpr bool equal(InputIt1 first1, InputIt1 last1, InputIt2 first2)
+{
+ for (; first1 != last1; ++first1, ++first2) {
+ if (!(*first1 == *first2)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+template<class InputIt1, class InputIt2>
+constexpr bool lexicographical_compare(InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2)
+{
+ for (; (first1 != last1) && (first2 != last2); ++first1, ++first2) {
+ if (*first1 < *first2)
+ return true;
+ if (*first2 < *first1)
+ return false;
+ }
+ return (first1 == last1) && (first2 != last2);
+}
+
+} // namespace bits
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/basic_types.h b/contrib/frozen/include/frozen/bits/basic_types.h
new file mode 100644
index 0000000..9814bac
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/basic_types.h
@@ -0,0 +1,200 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_BASIC_TYPES_H
+#define FROZEN_LETITGO_BASIC_TYPES_H
+
+#include "frozen/bits/exceptions.h"
+
+#include <utility>
+#include <iterator>
+#include <string>
+
+namespace frozen {
+
+namespace bits {
+
+// used as a fake argument for frozen::make_set and frozen::make_map in the case of N=0
+struct ignored_arg {};
+
+template <class T, std::size_t N>
+class cvector {
+ T data [N] = {}; // zero-initialization for scalar type T, default-initialized otherwise
+ std::size_t dsize = 0;
+
+public:
+ // Container typdefs
+ using value_type = T;
+ using reference = value_type &;
+ using const_reference = const value_type &;
+ using pointer = value_type *;
+ using const_pointer = const value_type *;
+ using iterator = pointer;
+ using const_iterator = const_pointer;
+ using size_type = std::size_t;
+ using difference_type = std::ptrdiff_t;
+
+ // Constructors
+ constexpr cvector(void) = default;
+ constexpr cvector(size_type count, const T& value) : dsize(count) {
+ for (std::size_t i = 0; i < N; ++i)
+ data[i] = value;
+ }
+
+ // Iterators
+ constexpr iterator begin() noexcept { return data; }
+ constexpr iterator end() noexcept { return data + dsize; }
+
+ // Capacity
+ constexpr size_type size() const { return dsize; }
+
+ // Element access
+ constexpr reference operator[](std::size_t index) { return data[index]; }
+ constexpr const_reference operator[](std::size_t index) const { return data[index]; }
+
+ constexpr reference back() { return data[dsize - 1]; }
+ constexpr const_reference back() const { return data[dsize - 1]; }
+
+ // Modifiers
+ constexpr void push_back(const T & a) { data[dsize++] = a; }
+ constexpr void push_back(T && a) { data[dsize++] = std::move(a); }
+ constexpr void pop_back() { --dsize; }
+
+ constexpr void clear() { dsize = 0; }
+};
+
+template <class T, std::size_t N>
+class carray {
+ T data_ [N] = {}; // zero-initialization for scalar type T, default-initialized otherwise
+
+ template <std::size_t M, std::size_t... I>
+ constexpr carray(T const (&init)[M], std::index_sequence<I...>)
+ : data_{init[I]...} {}
+ template <class Iter, std::size_t... I>
+ constexpr carray(Iter iter, std::index_sequence<I...>)
+ : data_{((void)I, *iter++)...} {}
+
+public:
+ // Container typdefs
+ using value_type = T;
+ using reference = value_type &;
+ using const_reference = const value_type &;
+ using pointer = value_type *;
+ using const_pointer = const value_type *;
+ using iterator = pointer;
+ using const_iterator = const_pointer;
+ using reverse_iterator = std::reverse_iterator<iterator>;
+ using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+ using size_type = std::size_t;
+ using difference_type = std::ptrdiff_t;
+
+ // Constructors
+ constexpr carray(void) = default;
+ template <std::size_t M>
+ constexpr carray(T const (&init)[M])
+ : carray(init, std::make_index_sequence<N>())
+ {
+ static_assert(M >= N, "Cannot initialize a carray with an smaller array");
+ }
+ constexpr carray(std::initializer_list<T> init)
+ : carray(init.begin(), std::make_index_sequence<N>())
+ {
+ // clang & gcc doesn't recognize init.size() as a constexpr
+ // static_assert(init.size() >= N, "Cannot initialize a carray with an smaller initializer list");
+ }
+
+ // Iterators
+ constexpr iterator begin() noexcept { return data_; }
+ constexpr const_iterator begin() const noexcept { return data_; }
+ constexpr const_iterator cbegin() const noexcept { return data_; }
+ constexpr iterator end() noexcept { return data_ + N; }
+ constexpr const_iterator end() const noexcept { return data_ + N; }
+ constexpr const_iterator cend() const noexcept { return data_ + N; }
+
+ constexpr reverse_iterator rbegin() noexcept { return reverse_iterator(end()); }
+ constexpr const_reverse_iterator rbegin() const noexcept { return const_reverse_iterator(end()); }
+ constexpr const_reverse_iterator crbegin() const noexcept { return const_reverse_iterator(end()); }
+ constexpr reverse_iterator rend() noexcept { return reverse_iterator(begin()); }
+ constexpr const_reverse_iterator rend() const noexcept { return const_reverse_iterator(begin()); }
+ constexpr const_reverse_iterator crend() const noexcept { return const_reverse_iterator(begin()); }
+
+ // Capacity
+ constexpr size_type size() const { return N; }
+ constexpr size_type max_size() const { return N; }
+
+ // Element access
+ constexpr reference operator[](std::size_t index) { return data_[index]; }
+ constexpr const_reference operator[](std::size_t index) const { return data_[index]; }
+
+ constexpr reference at(std::size_t index) {
+ if (index > N)
+ FROZEN_THROW_OR_ABORT(std::out_of_range("Index (" + std::to_string(index) + ") out of bound (" + std::to_string(N) + ')'));
+ return data_[index];
+ }
+ constexpr const_reference at(std::size_t index) const {
+ if (index > N)
+ FROZEN_THROW_OR_ABORT(std::out_of_range("Index (" + std::to_string(index) + ") out of bound (" + std::to_string(N) + ')'));
+ return data_[index];
+ }
+
+ constexpr reference front() { return data_[0]; }
+ constexpr const_reference front() const { return data_[0]; }
+
+ constexpr reference back() { return data_[N - 1]; }
+ constexpr const_reference back() const { return data_[N - 1]; }
+
+ constexpr value_type* data() noexcept { return data_; }
+ constexpr const value_type* data() const noexcept { return data_; }
+
+ // Modifiers
+ constexpr void fill(const value_type& val) {
+ for (std::size_t i = 0; i < N; ++i)
+ data_[i] = val;
+ }
+};
+template <class T>
+class carray<T, 0> {
+
+public:
+ // Container typdefs
+ using value_type = T;
+ using reference = value_type &;
+ using const_reference = const value_type &;
+ using pointer = value_type *;
+ using const_pointer = const value_type *;
+ using iterator = pointer;
+ using const_iterator = const_pointer;
+ using reverse_iterator = std::reverse_iterator<iterator>;
+ using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+ using size_type = std::size_t;
+ using difference_type = std::ptrdiff_t;
+
+ // Constructors
+ constexpr carray(void) = default;
+
+};
+
+} // namespace bits
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/constexpr_assert.h b/contrib/frozen/include/frozen/bits/constexpr_assert.h
new file mode 100644
index 0000000..912210d
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/constexpr_assert.h
@@ -0,0 +1,40 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_CONSTEXPR_ASSERT_H
+#define FROZEN_LETITGO_CONSTEXPR_ASSERT_H
+
+#include <cassert>
+
+#ifdef _MSC_VER
+
+// FIXME: find a way to implement that correctly for msvc
+#define constexpr_assert(cond, msg)
+
+#else
+
+#define constexpr_assert(cond, msg)\
+ assert(cond && msg);
+#endif
+
+#endif
+
diff --git a/contrib/frozen/include/frozen/bits/defines.h b/contrib/frozen/include/frozen/bits/defines.h
new file mode 100644
index 0000000..0a1663d
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/defines.h
@@ -0,0 +1,58 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_DEFINES_H
+#define FROZEN_LETITGO_DEFINES_H
+
+#if defined(_MSVC_LANG) && !(defined(__EDG__) && defined(__clang__)) // TRANSITION, VSO#273681
+ #define FROZEN_LETITGO_IS_MSVC
+#endif
+
+// Code taken from https://stackoverflow.com/questions/43639122/which-values-can-msvc-lang-have
+#if defined(FROZEN_LETITGO_IS_MSVC)
+ #if _MSVC_LANG > 201402
+ #define FROZEN_LETITGO_HAS_CXX17 1
+ #else /* _MSVC_LANG > 201402 */
+ #define FROZEN_LETITGO_HAS_CXX17 0
+ #endif /* _MSVC_LANG > 201402 */
+#else /* _MSVC_LANG etc. */
+ #if __cplusplus > 201402
+ #define FROZEN_LETITGO_HAS_CXX17 1
+ #else /* __cplusplus > 201402 */
+ #define FROZEN_LETITGO_HAS_CXX17 0
+ #endif /* __cplusplus > 201402 */
+#endif /* _MSVC_LANG etc. */
+// End if taken code
+
+#if FROZEN_LETITGO_HAS_CXX17 == 1 && defined(FROZEN_LETITGO_IS_MSVC)
+ #define FROZEN_LETITGO_HAS_STRING_VIEW // We assume Visual Studio always has string_view in C++17
+#else
+ #if FROZEN_LETITGO_HAS_CXX17 == 1 && __has_include(<string_view>)
+ #define FROZEN_LETITGO_HAS_STRING_VIEW
+ #endif
+#endif
+
+#ifdef __cpp_char8_t
+ #define FROZEN_LETITGO_HAS_CHAR8T
+#endif
+
+#endif // FROZEN_LETITGO_DEFINES_H
diff --git a/contrib/frozen/include/frozen/bits/elsa.h b/contrib/frozen/include/frozen/bits/elsa.h
new file mode 100644
index 0000000..d7388be
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/elsa.h
@@ -0,0 +1,50 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_ELSA_H
+#define FROZEN_LETITGO_ELSA_H
+
+#include <type_traits>
+
+namespace frozen {
+
+template <class T> struct elsa {
+ static_assert(std::is_integral<T>::value || std::is_enum<T>::value,
+ "only supports integral types, specialize for other types");
+
+ constexpr std::size_t operator()(T const &value, std::size_t seed) const {
+ std::size_t key = seed ^ static_cast<std::size_t>(value);
+ key = (~key) + (key << 21); // key = (key << 21) - key - 1;
+ key = key ^ (key >> 24);
+ key = (key + (key << 3)) + (key << 8); // key * 265
+ key = key ^ (key >> 14);
+ key = (key + (key << 2)) + (key << 4); // key * 21
+ key = key ^ (key >> 28);
+ key = key + (key << 31);
+ return key;
+ }
+};
+
+template <class T> using anna = elsa<T>;
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/exceptions.h b/contrib/frozen/include/frozen/bits/exceptions.h
new file mode 100644
index 0000000..b43e3e6
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/exceptions.h
@@ -0,0 +1,39 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_EXCEPTIONS_H
+#define FROZEN_LETITGO_EXCEPTIONS_H
+
+#if defined(FROZEN_NO_EXCEPTIONS) || (defined(_MSC_VER) && !defined(_CPPUNWIND)) || (!defined(_MSC_VER) && !defined(__cpp_exceptions))
+
+#include <cstdlib>
+#define FROZEN_THROW_OR_ABORT(_) std::abort()
+
+#else
+
+#include <stdexcept>
+#define FROZEN_THROW_OR_ABORT(err) throw err
+
+
+#endif
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/pmh.h b/contrib/frozen/include/frozen/bits/pmh.h
new file mode 100644
index 0000000..76e7ebe
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/pmh.h
@@ -0,0 +1,240 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// inspired from http://stevehanov.ca/blog/index.php?id=119
+#ifndef FROZEN_LETITGO_PMH_H
+#define FROZEN_LETITGO_PMH_H
+
+#include "frozen/bits/algorithms.h"
+#include "frozen/bits/basic_types.h"
+
+#include <array>
+#include <limits>
+
+namespace frozen {
+
+namespace bits {
+
+// Function object for sorting buckets in decreasing order of size
+struct bucket_size_compare {
+ template <typename B>
+ bool constexpr operator()(B const &b0,
+ B const &b1) const {
+ return b0.size() > b1.size();
+ }
+};
+
+// Step One in pmh routine is to take all items and hash them into buckets,
+// with some collisions. Then process those buckets further to build a perfect
+// hash function.
+// pmh_buckets represents the initial placement into buckets.
+
+template <size_t M>
+struct pmh_buckets {
+ // Step 0: Bucket max is 2 * sqrt M
+ // TODO: Come up with justification for this, should it not be O(log M)?
+ static constexpr auto bucket_max = 2 * (1u << (log(M) / 2));
+
+ using bucket_t = cvector<std::size_t, bucket_max>;
+ carray<bucket_t, M> buckets;
+ uint64_t seed;
+
+ // Represents a reference to a bucket. This is used because the buckets
+ // have to be sorted, but buckets are big, making it slower than sorting refs
+ struct bucket_ref {
+ unsigned hash;
+ const bucket_t * ptr;
+
+ // Forward some interface of bucket
+ using value_type = typename bucket_t::value_type;
+ using const_iterator = typename bucket_t::const_iterator;
+
+ constexpr auto size() const { return ptr->size(); }
+ constexpr const auto & operator[](std::size_t idx) const { return (*ptr)[idx]; }
+ constexpr auto begin() const { return ptr->begin(); }
+ constexpr auto end() const { return ptr->end(); }
+ };
+
+ // Make a bucket_ref for each bucket
+ template <std::size_t... Is>
+ carray<bucket_ref, M> constexpr make_bucket_refs(std::index_sequence<Is...>) const {
+ return {{ bucket_ref{Is, &buckets[Is]}... }};
+ }
+
+ // Makes a bucket_ref for each bucket and sorts them by size
+ carray<bucket_ref, M> constexpr get_sorted_buckets() const {
+ carray<bucket_ref, M> result{this->make_bucket_refs(std::make_index_sequence<M>())};
+ bits::quicksort(result.begin(), result.end() - 1, bucket_size_compare{});
+ return result;
+ }
+};
+
+template <size_t M, class Item, size_t N, class Hash, class Key, class PRG>
+pmh_buckets<M> constexpr make_pmh_buckets(const carray<Item, N> & items,
+ Hash const & hash,
+ Key const & key,
+ PRG & prg) {
+ using result_t = pmh_buckets<M>;
+ result_t result{};
+ bool rejected = false;
+ // Continue until all items are placed without exceeding bucket_max
+ while (1) {
+ for (auto & b : result.buckets) {
+ b.clear();
+ }
+ result.seed = prg();
+ rejected = false;
+ for (std::size_t i = 0; i < N; ++i) {
+ auto & bucket = result.buckets[hash(key(items[i]), static_cast<size_t>(result.seed)) % M];
+ if (bucket.size() >= result_t::bucket_max) {
+ rejected = true;
+ break;
+ }
+ bucket.push_back(i);
+ }
+ if (!rejected) { return result; }
+ }
+}
+
+// Check if an item appears in a cvector
+template<class T, size_t N>
+constexpr bool all_different_from(cvector<T, N> & data, T & a) {
+ for (std::size_t i = 0; i < data.size(); ++i)
+ if (data[i] == a)
+ return false;
+
+ return true;
+}
+
+// Represents either an index to a data item array, or a seed to be used with
+// a hasher. Seed must have high bit of 1, value has high bit of zero.
+struct seed_or_index {
+ using value_type = uint64_t;
+
+private:
+ static constexpr value_type MINUS_ONE = std::numeric_limits<value_type>::max();
+ static constexpr value_type HIGH_BIT = ~(MINUS_ONE >> 1);
+
+ value_type value_ = 0;
+
+public:
+ constexpr value_type value() const { return value_; }
+ constexpr bool is_seed() const { return value_ & HIGH_BIT; }
+
+ constexpr seed_or_index(bool is_seed, value_type value)
+ : value_(is_seed ? (value | HIGH_BIT) : (value & ~HIGH_BIT)) {}
+
+ constexpr seed_or_index() = default;
+ constexpr seed_or_index(const seed_or_index &) = default;
+ constexpr seed_or_index & operator =(const seed_or_index &) = default;
+};
+
+// Represents the perfect hash function created by pmh algorithm
+template <std::size_t M, class Hasher>
+struct pmh_tables {
+ uint64_t first_seed_;
+ carray<seed_or_index, M> first_table_;
+ carray<std::size_t, M> second_table_;
+ Hasher hash_;
+
+ // Looks up a given key, to find its expected index in carray<Item, N>
+ // Always returns a valid index, must use KeyEqual test after to confirm.
+ template <typename KeyType>
+ constexpr std::size_t lookup(const KeyType & key) const {
+ auto const d = first_table_[hash_(key, static_cast<size_t>(first_seed_)) % M];
+ if (!d.is_seed()) { return static_cast<std::size_t>(d.value()); } // this is narrowing uint64 -> size_t but should be fine
+ else { return second_table_[hash_(key, static_cast<std::size_t>(d.value())) % M]; }
+ }
+};
+
+// Make pmh tables for given items, hash function, prg, etc.
+template <std::size_t M, class Item, std::size_t N, class Hash, class Key, class PRG>
+pmh_tables<M, Hash> constexpr make_pmh_tables(const carray<Item, N> &
+ items,
+ Hash const &hash,
+ Key const &key,
+ PRG prg) {
+ // Step 1: Place all of the keys into buckets
+ auto step_one = make_pmh_buckets<M>(items, hash, key, prg);
+
+ // Step 2: Sort the buckets to process the ones with the most items first.
+ auto buckets = step_one.get_sorted_buckets();
+
+ // G becomes the first hash table in the resulting pmh function
+ carray<seed_or_index, M> G; // Default constructed to "index 0"
+
+ // H becomes the second hash table in the resulting pmh function
+ constexpr std::size_t UNUSED = std::numeric_limits<std::size_t>::max();
+ carray<std::size_t, M> H;
+ H.fill(UNUSED);
+
+ // Step 3: Map the items in buckets into hash tables.
+ for (const auto & bucket : buckets) {
+ auto const bsize = bucket.size();
+
+ if (bsize == 1) {
+ // Store index to the (single) item in G
+ // assert(bucket.hash == hash(key(items[bucket[0]]), step_one.seed) % M);
+ G[bucket.hash] = {false, static_cast<uint64_t>(bucket[0])};
+ } else if (bsize > 1) {
+
+ // Repeatedly try different H of d until we find a hash function
+ // that places all items in the bucket into free slots
+ seed_or_index d{true, prg()};
+ cvector<std::size_t, decltype(step_one)::bucket_max> bucket_slots;
+
+ while (bucket_slots.size() < bsize) {
+ auto slot = hash(key(items[bucket[bucket_slots.size()]]), static_cast<size_t>(d.value())) % M;
+
+ if (H[slot] != UNUSED || !all_different_from(bucket_slots, slot)) {
+ bucket_slots.clear();
+ d = {true, prg()};
+ continue;
+ }
+
+ bucket_slots.push_back(slot);
+ }
+
+ // Put successful seed in G, and put indices to items in their slots
+ // assert(bucket.hash == hash(key(items[bucket[0]]), step_one.seed) % M);
+ G[bucket.hash] = d;
+ for (std::size_t i = 0; i < bsize; ++i)
+ H[bucket_slots[i]] = bucket[i];
+ }
+ }
+
+ // Any unused entries in the H table have to get changed to zero.
+ // This is because hashing should not fail or return an out-of-bounds entry.
+ // A lookup fails after we apply user-supplied KeyEqual to the query and the
+ // key found by hashing. Sending such queries to zero cannot hurt.
+ for (std::size_t i = 0; i < M; ++i)
+ if (H[i] == UNUSED)
+ H[i] = 0;
+
+ return {step_one.seed, G, H, hash};
+}
+
+} // namespace bits
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/bits/version.h b/contrib/frozen/include/frozen/bits/version.h
new file mode 100644
index 0000000..51804d2
--- /dev/null
+++ b/contrib/frozen/include/frozen/bits/version.h
@@ -0,0 +1,30 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_VERSION_H
+#define FROZEN_LETITGO_VERSION_H
+
+#define FROZEN_MAJOR_VERSION 1
+#define FROZEN_MINOR_VERSION 0
+#define FROZEN_PATCH_VERSION 1
+
+#endif
diff --git a/contrib/frozen/include/frozen/map.h b/contrib/frozen/include/frozen/map.h
new file mode 100644
index 0000000..107179c
--- /dev/null
+++ b/contrib/frozen/include/frozen/map.h
@@ -0,0 +1,323 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_MAP_H
+#define FROZEN_LETITGO_MAP_H
+
+#include "frozen/bits/algorithms.h"
+#include "frozen/bits/basic_types.h"
+#include "frozen/bits/constexpr_assert.h"
+#include "frozen/bits/exceptions.h"
+#include "frozen/bits/version.h"
+
+#include <utility>
+
+namespace frozen {
+
+namespace impl {
+
+template <class Comparator> class CompareKey {
+
+ Comparator const comparator_;
+
+public:
+ constexpr CompareKey(Comparator const &comparator)
+ : comparator_(comparator) {}
+
+ template <class Key, class Value>
+ constexpr int operator()(std::pair<Key, Value> const &self,
+ std::pair<Key, Value> const &other) const {
+ return comparator_(std::get<0>(self), std::get<0>(other));
+ }
+
+ template <class Key, class Value>
+ constexpr int operator()(Key const &self_key,
+ std::pair<Key, Value> const &other) const {
+ return comparator_(self_key, std::get<0>(other));
+ }
+
+ template <class Key, class Value>
+ constexpr int operator()(std::pair<Key, Value> const &self,
+ Key const &other_key) const {
+ return comparator_(std::get<0>(self), other_key);
+ }
+
+ template <class Key>
+ constexpr int operator()(Key const &self_key, Key const &other_key) const {
+ return comparator_(self_key, other_key);
+ }
+};
+
+} // namespace impl
+
+template <class Key, class Value, std::size_t N, class Compare = std::less<Key>>
+class map {
+ using container_type = bits::carray<std::pair<Key, Value>, N>;
+ impl::CompareKey<Compare> less_than_;
+ container_type items_;
+
+public:
+ using key_type = Key;
+ using mapped_type = Value;
+ using value_type = typename container_type::value_type;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::difference_type;
+ using key_compare = decltype(less_than_);
+ using reference = typename container_type::reference;
+ using const_reference = typename container_type::const_reference;
+ using pointer = typename container_type::pointer;
+ using const_pointer = typename container_type::const_pointer;
+ using iterator = typename container_type::iterator;
+ using const_iterator = typename container_type::const_iterator;
+ using reverse_iterator = typename container_type::reverse_iterator;
+ using const_reverse_iterator =
+ typename container_type::const_reverse_iterator;
+
+public:
+ /* constructors */
+ constexpr map(container_type items, Compare const &compare)
+ : less_than_{compare}
+ , items_{bits::quicksort(items, less_than_)} {}
+
+ explicit constexpr map(container_type items)
+ : map{items, Compare{}} {}
+
+ constexpr map(std::initializer_list<value_type> items, Compare const &compare)
+ : map{container_type {items}, compare} {
+ constexpr_assert(items.size() == N, "Inconsistent initializer_list size and type size argument");
+ }
+
+ constexpr map(std::initializer_list<value_type> items)
+ : map{items, Compare{}} {}
+
+ /* element access */
+ constexpr Value const& at(Key const &key) const {
+ return at_impl(*this, key);
+ }
+ constexpr Value& at(Key const &key) {
+ return at_impl(*this, key);
+ }
+
+ /* iterators */
+ constexpr iterator begin() { return items_.begin(); }
+ constexpr const_iterator begin() const { return items_.begin(); }
+ constexpr const_iterator cbegin() const { return items_.cbegin(); }
+ constexpr iterator end() { return items_.end(); }
+ constexpr const_iterator end() const { return items_.end(); }
+ constexpr const_iterator cend() const { return items_.cend(); }
+
+ constexpr reverse_iterator rbegin() { return items_.rbegin(); }
+ constexpr const_reverse_iterator rbegin() const { return items_.rbegin(); }
+ constexpr const_reverse_iterator crbegin() const { return items_.crbegin(); }
+ constexpr reverse_iterator rend() { return items_.rend(); }
+ constexpr const_reverse_iterator rend() const { return items_.rend(); }
+ constexpr const_reverse_iterator crend() const { return items_.crend(); }
+
+ /* capacity */
+ constexpr bool empty() const { return !N; }
+ constexpr size_type size() const { return N; }
+ constexpr size_type max_size() const { return N; }
+
+ /* lookup */
+
+ constexpr std::size_t count(Key const &key) const {
+ return bits::binary_search<N>(items_.begin(), key, less_than_);
+ }
+
+ constexpr const_iterator find(Key const &key) const {
+ return find_impl(*this, key);
+ }
+ constexpr iterator find(Key const &key) {
+ return find_impl(*this, key);
+ }
+
+ constexpr std::pair<const_iterator, const_iterator>
+ equal_range(Key const &key) const {
+ return equal_range_impl(*this, key);
+ }
+ constexpr std::pair<iterator, iterator> equal_range(Key const &key) {
+ return equal_range_impl(*this, key);
+ }
+
+ constexpr const_iterator lower_bound(Key const &key) const {
+ return lower_bound_impl(*this, key);
+ }
+ constexpr iterator lower_bound(Key const &key) {
+ return lower_bound_impl(*this, key);
+ }
+
+ constexpr const_iterator upper_bound(Key const &key) const {
+ return upper_bound_impl(*this, key);
+ }
+ constexpr iterator upper_bound(Key const &key) {
+ return upper_bound_impl(*this, key);
+ }
+
+ /* observers */
+ constexpr key_compare key_comp() const { return less_than_; }
+ constexpr key_compare value_comp() const { return less_than_; }
+
+ private:
+ template <class This>
+ static inline constexpr auto& at_impl(This&& self, Key const &key) {
+ auto where = self.lower_bound(key);
+ if (where != self.end())
+ return where->second;
+ else
+ FROZEN_THROW_OR_ABORT(std::out_of_range("unknown key"));
+ }
+
+ template <class This>
+ static inline constexpr auto find_impl(This&& self, Key const &key) {
+ auto where = self.lower_bound(key);
+ if ((where != self.end()) && !self.less_than_(key, *where))
+ return where;
+ else
+ return self.end();
+ }
+
+ template <class This>
+ static inline constexpr auto equal_range_impl(This&& self, Key const &key) {
+ auto lower = self.lower_bound(key);
+ using lower_t = decltype(lower);
+ if (lower == self.end())
+ return std::pair<lower_t, lower_t>{lower, lower};
+ else
+ return std::pair<lower_t, lower_t>{lower, lower + 1};
+ }
+
+ template <class This>
+ static inline constexpr auto lower_bound_impl(This&& self, Key const &key) -> decltype(self.end()) {
+ auto where = bits::lower_bound<N>(self.items_.begin(), key, self.less_than_);
+ if ((where != self.end()) && !self.less_than_(key, *where))
+ return where;
+ else
+ return self.end();
+ }
+
+ template <class This>
+ static inline constexpr auto upper_bound_impl(This&& self, Key const &key) -> decltype(self.end()) {
+ auto where = bits::lower_bound<N>(self.items_.begin(), key, self.less_than_);
+ if ((where != self.end()) && !self.less_than_(key, *where))
+ return where + 1;
+ else
+ return self.end();
+ }
+};
+
+template <class Key, class Value, class Compare>
+class map<Key, Value, 0, Compare> {
+ using container_type = bits::carray<std::pair<Key, Value>, 0>;
+ impl::CompareKey<Compare> less_than_;
+
+public:
+ using key_type = Key;
+ using mapped_type = Value;
+ using value_type = typename container_type::value_type;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::difference_type;
+ using key_compare = decltype(less_than_);
+ using reference = typename container_type::reference;
+ using const_reference = typename container_type::const_reference;
+ using pointer = typename container_type::pointer;
+ using const_pointer = typename container_type::const_pointer;
+ using iterator = pointer;
+ using const_iterator = const_pointer;
+ using reverse_iterator = pointer;
+ using const_reverse_iterator = const_pointer;
+
+public:
+ /* constructors */
+ constexpr map(const map &other) = default;
+ constexpr map(std::initializer_list<value_type>, Compare const &compare)
+ : less_than_{compare} {}
+ constexpr map(std::initializer_list<value_type> items)
+ : map{items, Compare{}} {}
+
+ /* element access */
+ constexpr mapped_type at(Key const &) const {
+ FROZEN_THROW_OR_ABORT(std::out_of_range("invalid key"));
+ }
+ constexpr mapped_type at(Key const &) {
+ FROZEN_THROW_OR_ABORT(std::out_of_range("invalid key"));
+ }
+
+ /* iterators */
+ constexpr iterator begin() { return nullptr; }
+ constexpr const_iterator begin() const { return nullptr; }
+ constexpr const_iterator cbegin() const { return nullptr; }
+ constexpr iterator end() { return nullptr; }
+ constexpr const_iterator end() const { return nullptr; }
+ constexpr const_iterator cend() const { return nullptr; }
+
+ constexpr reverse_iterator rbegin() { return nullptr; }
+ constexpr const_reverse_iterator rbegin() const { return nullptr; }
+ constexpr const_reverse_iterator crbegin() const { return nullptr; }
+ constexpr reverse_iterator rend() { return nullptr; }
+ constexpr const_reverse_iterator rend() const { return nullptr; }
+ constexpr const_reverse_iterator crend() const { return nullptr; }
+
+ /* capacity */
+ constexpr bool empty() const { return true; }
+ constexpr size_type size() const { return 0; }
+ constexpr size_type max_size() const { return 0; }
+
+ /* lookup */
+
+ constexpr std::size_t count(Key const &) const { return 0; }
+
+ constexpr const_iterator find(Key const &) const { return end(); }
+ constexpr iterator find(Key const &) { return end(); }
+
+ constexpr std::pair<const_iterator, const_iterator>
+ equal_range(Key const &) const {
+ return {end(), end()};
+ }
+ constexpr std::pair<iterator, iterator>
+ equal_range(Key const &) {
+ return {end(), end()};
+ }
+
+ constexpr const_iterator lower_bound(Key const &) const { return end(); }
+ constexpr iterator lower_bound(Key const &) { return end(); }
+
+ constexpr const_iterator upper_bound(Key const &) const { return end(); }
+ constexpr iterator upper_bound(Key const &) { return end(); }
+
+ /* observers */
+ constexpr key_compare key_comp() const { return less_than_; }
+ constexpr key_compare value_comp() const { return less_than_; }
+};
+
+template <typename T, typename U>
+constexpr auto make_map(bits::ignored_arg = {}/* for consistency with the initializer below for N = 0*/) {
+ return map<T, U, 0>{};
+}
+
+template <typename T, typename U, std::size_t N>
+constexpr auto make_map(std::pair<T, U> const (&items)[N]) {
+ return map<T, U, N>{items};
+}
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/random.h b/contrib/frozen/include/frozen/random.h
new file mode 100644
index 0000000..ea494dc
--- /dev/null
+++ b/contrib/frozen/include/frozen/random.h
@@ -0,0 +1,90 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_RANDOM_H
+#define FROZEN_LETITGO_RANDOM_H
+
+#include "frozen/bits/algorithms.h"
+#include "frozen/bits/version.h"
+
+#include <cstdint>
+#include <type_traits>
+
+namespace frozen {
+template <class UIntType, UIntType a, UIntType c, UIntType m>
+class linear_congruential_engine {
+
+ static_assert(std::is_unsigned<UIntType>::value,
+ "UIntType must be an unsigned integral type");
+
+public:
+ using result_type = UIntType;
+ static constexpr result_type multiplier = a;
+ static constexpr result_type increment = c;
+ static constexpr result_type modulus = m;
+ static constexpr result_type default_seed = 1u;
+
+ linear_congruential_engine() = default;
+ constexpr linear_congruential_engine(result_type s) { seed(s); }
+
+ void seed(result_type s = default_seed) { state_ = s; }
+ constexpr result_type operator()() {
+ using uint_least_t = bits::select_uint_least_t<bits::log(a) + bits::log(m) + 4>;
+ uint_least_t tmp = static_cast<uint_least_t>(multiplier) * state_ + increment;
+
+ // the static cast below may end up doing a truncation
+ if(modulus != 0)
+ state_ = static_cast<result_type>(tmp % modulus);
+ else
+ state_ = static_cast<result_type>(tmp);
+ return state_;
+ }
+ constexpr void discard(unsigned long long n) {
+ while (n--)
+ operator()();
+ }
+ static constexpr result_type min() { return increment == 0u ? 1u : 0u; };
+ static constexpr result_type max() { return modulus - 1u; };
+ friend constexpr bool operator==(linear_congruential_engine const &self,
+ linear_congruential_engine const &other) {
+ return self.state_ == other.state_;
+ }
+ friend constexpr bool operator!=(linear_congruential_engine const &self,
+ linear_congruential_engine const &other) {
+ return !(self == other);
+ }
+
+private:
+ result_type state_ = default_seed;
+};
+
+using minstd_rand0 =
+ linear_congruential_engine<std::uint_fast32_t, 16807, 0, 2147483647>;
+using minstd_rand =
+ linear_congruential_engine<std::uint_fast32_t, 48271, 0, 2147483647>;
+
+// This generator is used by default in unordered frozen containers
+using default_prg_t = minstd_rand;
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/set.h b/contrib/frozen/include/frozen/set.h
new file mode 100644
index 0000000..d86d814
--- /dev/null
+++ b/contrib/frozen/include/frozen/set.h
@@ -0,0 +1,220 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_SET_H
+#define FROZEN_SET_H
+
+#include "frozen/bits/algorithms.h"
+#include "frozen/bits/basic_types.h"
+#include "frozen/bits/constexpr_assert.h"
+#include "frozen/bits/version.h"
+
+#include <utility>
+
+namespace frozen {
+
+template <class Key, std::size_t N, class Compare = std::less<Key>> class set {
+ using container_type = bits::carray<Key, N>;
+ Compare less_than_;
+ container_type keys_;
+
+public:
+ /* container typedefs*/
+ using key_type = Key;
+ using value_type = Key;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::size_type;
+ using key_compare = Compare;
+ using value_compare = Compare;
+ using reference = typename container_type::const_reference;
+ using const_reference = reference;
+ using pointer = typename container_type::const_pointer;
+ using const_pointer = pointer;
+ using iterator = typename container_type::const_iterator;
+ using reverse_iterator = typename container_type::const_reverse_iterator;
+ using const_iterator = iterator;
+ using const_reverse_iterator = reverse_iterator;
+
+public:
+ /* constructors */
+ constexpr set(const set &other) = default;
+
+ constexpr set(container_type keys, Compare const & comp)
+ : less_than_{comp}
+ , keys_(bits::quicksort(keys, less_than_)) {
+ }
+
+ explicit constexpr set(container_type keys)
+ : set{keys, Compare{}} {}
+
+ constexpr set(std::initializer_list<Key> keys, Compare const & comp)
+ : set{container_type{keys}, comp} {
+ constexpr_assert(keys.size() == N, "Inconsistent initializer_list size and type size argument");
+ }
+
+ constexpr set(std::initializer_list<Key> keys)
+ : set{keys, Compare{}} {}
+
+ /* capacity */
+ constexpr bool empty() const { return !N; }
+ constexpr size_type size() const { return N; }
+ constexpr size_type max_size() const { return N; }
+
+ /* lookup */
+ constexpr std::size_t count(Key const &key) const {
+ return bits::binary_search<N>(keys_.begin(), key, less_than_);
+ }
+
+ constexpr const_iterator find(Key const &key) const {
+ const_iterator where = lower_bound(key);
+ if ((where != end()) && !less_than_(key, *where))
+ return where;
+ else
+ return end();
+ }
+
+ constexpr std::pair<const_iterator, const_iterator> equal_range(Key const &key) const {
+ auto const lower = lower_bound(key);
+ if (lower == end())
+ return {lower, lower};
+ else
+ return {lower, lower + 1};
+ }
+
+ constexpr const_iterator lower_bound(Key const &key) const {
+ auto const where = bits::lower_bound<N>(keys_.begin(), key, less_than_);
+ if ((where != end()) && !less_than_(key, *where))
+ return where;
+ else
+ return end();
+ }
+
+ constexpr const_iterator upper_bound(Key const &key) const {
+ auto const where = bits::lower_bound<N>(keys_.begin(), key, less_than_);
+ if ((where != end()) && !less_than_(key, *where))
+ return where + 1;
+ else
+ return end();
+ }
+
+ /* observers */
+ constexpr key_compare key_comp() const { return less_than_; }
+ constexpr key_compare value_comp() const { return less_than_; }
+
+ /* iterators */
+ constexpr const_iterator begin() const { return keys_.begin(); }
+ constexpr const_iterator cbegin() const { return keys_.cbegin(); }
+ constexpr const_iterator end() const { return keys_.end(); }
+ constexpr const_iterator cend() const { return keys_.cend(); }
+
+ constexpr const_reverse_iterator rbegin() const { return keys_.rbegin(); }
+ constexpr const_reverse_iterator crbegin() const { return keys_.crbegin(); }
+ constexpr const_reverse_iterator rend() const { return keys_.rend(); }
+ constexpr const_reverse_iterator crend() const { return keys_.crend(); }
+
+ /* comparison */
+ constexpr bool operator==(set const& rhs) const { return bits::equal(begin(), end(), rhs.begin()); }
+ constexpr bool operator!=(set const& rhs) const { return !(*this == rhs); }
+ constexpr bool operator<(set const& rhs) const { return bits::lexicographical_compare(begin(), end(), rhs.begin(), rhs.end()); }
+ constexpr bool operator<=(set const& rhs) const { return (*this < rhs) || (*this == rhs); }
+ constexpr bool operator>(set const& rhs) const { return bits::lexicographical_compare(rhs.begin(), rhs.end(), begin(), end()); }
+ constexpr bool operator>=(set const& rhs) const { return (*this > rhs) || (*this == rhs); }
+};
+
+template <class Key, class Compare> class set<Key, 0, Compare> {
+ using container_type = bits::carray<Key, 0>; // just for the type definitions
+ Compare less_than_;
+
+public:
+ /* container typedefs*/
+ using key_type = Key;
+ using value_type = Key;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::size_type;
+ using key_compare = Compare;
+ using value_compare = Compare;
+ using reference = typename container_type::const_reference;
+ using const_reference = reference;
+ using pointer = typename container_type::const_pointer;
+ using const_pointer = pointer;
+ using iterator = pointer;
+ using reverse_iterator = pointer;
+ using const_iterator = const_pointer;
+ using const_reverse_iterator = const_pointer;
+
+public:
+ /* constructors */
+ constexpr set(const set &other) = default;
+ constexpr set(bits::carray<Key, 0>, Compare const &) {}
+ explicit constexpr set(bits::carray<Key, 0>) {}
+
+ constexpr set(std::initializer_list<Key>, Compare const &comp)
+ : less_than_{comp} {}
+ constexpr set(std::initializer_list<Key> keys) : set{keys, Compare{}} {}
+
+ /* capacity */
+ constexpr bool empty() const { return true; }
+ constexpr size_type size() const { return 0; }
+ constexpr size_type max_size() const { return 0; }
+
+ /* lookup */
+ constexpr std::size_t count(Key const &) const { return 0; }
+
+ constexpr const_iterator find(Key const &) const { return end(); }
+
+ constexpr std::pair<const_iterator, const_iterator>
+ equal_range(Key const &) const { return {end(), end()}; }
+
+ constexpr const_iterator lower_bound(Key const &) const { return end(); }
+
+ constexpr const_iterator upper_bound(Key const &) const { return end(); }
+
+ /* observers */
+ constexpr key_compare key_comp() const { return less_than_; }
+ constexpr key_compare value_comp() const { return less_than_; }
+
+ /* iterators */
+ constexpr const_iterator begin() const { return nullptr; }
+ constexpr const_iterator cbegin() const { return nullptr; }
+ constexpr const_iterator end() const { return nullptr; }
+ constexpr const_iterator cend() const { return nullptr; }
+
+ constexpr const_reverse_iterator rbegin() const { return nullptr; }
+ constexpr const_reverse_iterator crbegin() const { return nullptr; }
+ constexpr const_reverse_iterator rend() const { return nullptr; }
+ constexpr const_reverse_iterator crend() const { return nullptr; }
+};
+
+template <typename T>
+constexpr auto make_set(bits::ignored_arg = {}/* for consistency with the initializer below for N = 0*/) {
+ return set<T, 0>{};
+}
+
+template <typename T, std::size_t N>
+constexpr auto make_set(const T (&args)[N]) {
+ return set<T, N>(args);
+}
+
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/string.h b/contrib/frozen/include/frozen/string.h
new file mode 100644
index 0000000..4505bbf
--- /dev/null
+++ b/contrib/frozen/include/frozen/string.h
@@ -0,0 +1,152 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_STRING_H
+#define FROZEN_LETITGO_STRING_H
+
+#include "frozen/bits/elsa.h"
+#include "frozen/bits/version.h"
+#include "frozen/bits/defines.h"
+
+#include <functional>
+
+#ifdef FROZEN_LETITGO_HAS_STRING_VIEW
+#include <string_view>
+#endif
+
+namespace frozen {
+
+template <typename _CharT>
+class basic_string {
+ using chr_t = _CharT;
+
+ chr_t const *data_;
+ std::size_t size_;
+
+public:
+ template <std::size_t N>
+ constexpr basic_string(chr_t const (&data)[N])
+ : data_(data), size_(N - 1) {}
+ constexpr basic_string(chr_t const *data, std::size_t size)
+ : data_(data), size_(size) {}
+
+#ifdef FROZEN_LETITGO_HAS_STRING_VIEW
+ constexpr basic_string(std::basic_string_view<chr_t> data)
+ : data_(data.data()), size_(data.size()) {}
+#endif
+
+ constexpr basic_string(const basic_string &) noexcept = default;
+ constexpr basic_string &operator=(const basic_string &) noexcept = default;
+
+ constexpr std::size_t size() const { return size_; }
+
+ constexpr chr_t operator[](std::size_t i) const { return data_[i]; }
+
+ constexpr bool operator==(basic_string other) const {
+ if (size_ != other.size_)
+ return false;
+ for (std::size_t i = 0; i < size_; ++i)
+ if (data_[i] != other.data_[i])
+ return false;
+ return true;
+ }
+
+ constexpr bool operator<(const basic_string &other) const {
+ unsigned i = 0;
+ while (i < size() && i < other.size()) {
+ if ((*this)[i] < other[i]) {
+ return true;
+ }
+ if ((*this)[i] > other[i]) {
+ return false;
+ }
+ ++i;
+ }
+ return size() < other.size();
+ }
+
+ constexpr const chr_t *data() const { return data_; }
+};
+
+template <typename _CharT> struct elsa<basic_string<_CharT>> {
+ constexpr std::size_t operator()(basic_string<_CharT> value) const {
+ std::size_t d = 5381;
+ for (std::size_t i = 0; i < value.size(); ++i)
+ d = d * 33 + value[i];
+ return d;
+ }
+ // https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
+ // With the lowest bits removed, based on experimental setup.
+ constexpr std::size_t operator()(basic_string<_CharT> value, std::size_t seed) const {
+ std::size_t d = (0x811c9dc5 ^ seed) * 0x01000193;
+ for (std::size_t i = 0; i < value.size(); ++i)
+ d = (d ^ value[i]) * 0x01000193;
+ return d >> 8 ;
+ }
+};
+
+using string = basic_string<char>;
+using wstring = basic_string<wchar_t>;
+using u16string = basic_string<char16_t>;
+using u32string = basic_string<char32_t>;
+
+#ifdef FROZEN_LETITGO_HAS_CHAR8T
+using u8string = basic_string<char8_t>;
+#endif
+
+namespace string_literals {
+
+constexpr string operator"" _s(const char *data, std::size_t size) {
+ return {data, size};
+}
+
+constexpr wstring operator"" _s(const wchar_t *data, std::size_t size) {
+ return {data, size};
+}
+
+constexpr u16string operator"" _s(const char16_t *data, std::size_t size) {
+ return {data, size};
+}
+
+constexpr u32string operator"" _s(const char32_t *data, std::size_t size) {
+ return {data, size};
+}
+
+#ifdef FROZEN_LETITGO_HAS_CHAR8T
+constexpr u8string operator"" _s(const char8_t *data, std::size_t size) {
+ return {data, size};
+}
+#endif
+
+} // namespace string_literals
+
+} // namespace frozen
+
+namespace std {
+template <typename _CharT> struct hash<frozen::basic_string<_CharT>> {
+ size_t operator()(frozen::basic_string<_CharT> s) const {
+ return frozen::elsa<frozen::basic_string<_CharT>>{}(s);
+ }
+};
+} // namespace std
+
+#endif
diff --git a/contrib/frozen/include/frozen/unordered_map.h b/contrib/frozen/include/frozen/unordered_map.h
new file mode 100644
index 0000000..5e1e399
--- /dev/null
+++ b/contrib/frozen/include/frozen/unordered_map.h
@@ -0,0 +1,197 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_UNORDERED_MAP_H
+#define FROZEN_LETITGO_UNORDERED_MAP_H
+
+#include "frozen/bits/basic_types.h"
+#include "frozen/bits/constexpr_assert.h"
+#include "frozen/bits/elsa.h"
+#include "frozen/bits/exceptions.h"
+#include "frozen/bits/pmh.h"
+#include "frozen/bits/version.h"
+#include "frozen/random.h"
+
+#include <tuple>
+#include <functional>
+
+namespace frozen {
+
+namespace bits {
+
+struct GetKey {
+ template <class KV> constexpr auto const &operator()(KV const &kv) const {
+ return kv.first;
+ }
+};
+
+} // namespace bits
+
+template <class Key, class Value, std::size_t N, typename Hash = anna<Key>,
+ class KeyEqual = std::equal_to<Key>>
+class unordered_map {
+ static constexpr std::size_t storage_size =
+ bits::next_highest_power_of_two(N) * (N < 32 ? 2 : 1); // size adjustment to prevent high collision rate for small sets
+ using container_type = bits::carray<std::pair<Key, Value>, N>;
+ using tables_type = bits::pmh_tables<storage_size, Hash>;
+
+ KeyEqual const equal_;
+ container_type items_;
+ tables_type tables_;
+
+public:
+ /* typedefs */
+ using Self = unordered_map<Key, Value, N, Hash, KeyEqual>;
+ using key_type = Key;
+ using mapped_type = Value;
+ using value_type = typename container_type::value_type;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::difference_type;
+ using hasher = Hash;
+ using key_equal = KeyEqual;
+ using reference = typename container_type::reference;
+ using const_reference = typename container_type::const_reference;
+ using pointer = typename container_type::pointer;
+ using const_pointer = typename container_type::const_pointer;
+ using iterator = typename container_type::iterator;
+ using const_iterator = typename container_type::const_iterator;
+
+public:
+ /* constructors */
+ unordered_map(unordered_map const &) = default;
+ constexpr unordered_map(container_type items,
+ Hash const &hash, KeyEqual const &equal)
+ : equal_{equal}
+ , items_{items}
+ , tables_{
+ bits::make_pmh_tables<storage_size>(
+ items_, hash, bits::GetKey{}, default_prg_t{})} {}
+ explicit constexpr unordered_map(container_type items)
+ : unordered_map{items, Hash{}, KeyEqual{}} {}
+
+ constexpr unordered_map(std::initializer_list<value_type> items,
+ Hash const & hash, KeyEqual const & equal)
+ : unordered_map{container_type{items}, hash, equal} {
+ constexpr_assert(items.size() == N, "Inconsistent initializer_list size and type size argument");
+ }
+
+ constexpr unordered_map(std::initializer_list<value_type> items)
+ : unordered_map{items, Hash{}, KeyEqual{}} {}
+
+ /* iterators */
+ constexpr iterator begin() { return items_.begin(); }
+ constexpr iterator end() { return items_.end(); }
+ constexpr const_iterator begin() const { return items_.begin(); }
+ constexpr const_iterator end() const { return items_.end(); }
+ constexpr const_iterator cbegin() const { return items_.cbegin(); }
+ constexpr const_iterator cend() const { return items_.cend(); }
+
+ /* capacity */
+ constexpr bool empty() const { return !N; }
+ constexpr size_type size() const { return N; }
+ constexpr size_type max_size() const { return N; }
+
+ /* lookup */
+ constexpr std::size_t count(Key const &key) const {
+ auto const &kv = lookup(key);
+ return equal_(kv.first, key);
+ }
+
+ constexpr Value const &at(Key const &key) const {
+ return at_impl(*this, key);
+ }
+ constexpr Value &at(Key const &key) {
+ return at_impl(*this, key);
+ }
+
+ constexpr const_iterator find(Key const &key) const {
+ return find_impl(*this, key);
+ }
+ constexpr iterator find(Key const &key) {
+ return find_impl(*this, key);
+ }
+
+ constexpr std::pair<const_iterator, const_iterator> equal_range(Key const &key) const {
+ return equal_range_impl(*this, key);
+ }
+ constexpr std::pair<iterator, iterator> equal_range(Key const &key) {
+ return equal_range_impl(*this, key);
+ }
+
+ /* bucket interface */
+ constexpr std::size_t bucket_count() const { return storage_size; }
+ constexpr std::size_t max_bucket_count() const { return storage_size; }
+
+ /* observers*/
+ constexpr hasher hash_function() const { return tables_.hash_; }
+ constexpr key_equal key_eq() const { return equal_; }
+
+private:
+ template <class This>
+ static inline constexpr auto& at_impl(This&& self, Key const &key) {
+ auto& kv = self.lookup(key);
+ if (self.equal_(kv.first, key))
+ return kv.second;
+ else
+ FROZEN_THROW_OR_ABORT(std::out_of_range("unknown key"));
+ }
+
+ template <class This>
+ static inline constexpr auto find_impl(This&& self, Key const &key) {
+ auto& kv = self.lookup(key);
+ if (self.equal_(kv.first, key))
+ return &kv;
+ else
+ return self.items_.end();
+ }
+
+ template <class This>
+ static inline constexpr auto equal_range_impl(This&& self, Key const &key) {
+ auto& kv = self.lookup(key);
+ using kv_ptr = decltype(&kv);
+ if (self.equal_(kv.first, key))
+ return std::pair<kv_ptr, kv_ptr>{&kv, &kv + 1};
+ else
+ return std::pair<kv_ptr, kv_ptr>{self.items_.end(), self.items_.end()};
+ }
+
+ template <class This>
+ static inline constexpr auto& lookup_impl(This&& self, Key const &key) {
+ return self.items_[self.tables_.lookup(key)];
+ }
+
+ constexpr auto const& lookup(Key const &key) const {
+ return lookup_impl(*this, key);
+ }
+ constexpr auto& lookup(Key const &key) {
+ return lookup_impl(*this, key);
+ }
+};
+
+template <typename T, typename U, std::size_t N>
+constexpr auto make_unordered_map(std::pair<T, U> const (&items)[N]) {
+ return unordered_map<T, U, N>{items};
+}
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/frozen/include/frozen/unordered_set.h b/contrib/frozen/include/frozen/unordered_set.h
new file mode 100644
index 0000000..0fca292
--- /dev/null
+++ b/contrib/frozen/include/frozen/unordered_set.h
@@ -0,0 +1,147 @@
+/*
+ * Frozen
+ * Copyright 2016 QuarksLab
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef FROZEN_LETITGO_UNORDERED_SET_H
+#define FROZEN_LETITGO_UNORDERED_SET_H
+
+#include "frozen/bits/basic_types.h"
+#include "frozen/bits/constexpr_assert.h"
+#include "frozen/bits/elsa.h"
+#include "frozen/bits/pmh.h"
+#include "frozen/bits/version.h"
+#include "frozen/random.h"
+
+#include <utility>
+
+namespace frozen {
+
+namespace bits {
+
+struct Get {
+ template <class T> constexpr T const &operator()(T const &key) const {
+ return key;
+ }
+};
+
+} // namespace bits
+
+template <class Key, std::size_t N, typename Hash = elsa<Key>,
+ class KeyEqual = std::equal_to<Key>>
+class unordered_set {
+ static constexpr std::size_t storage_size =
+ bits::next_highest_power_of_two(N) * (N < 32 ? 2 : 1); // size adjustment to prevent high collision rate for small sets
+ using container_type = bits::carray<Key, N>;
+ using tables_type = bits::pmh_tables<storage_size, Hash>;
+
+ KeyEqual const equal_;
+ container_type keys_;
+ tables_type tables_;
+
+public:
+ /* typedefs */
+ using key_type = Key;
+ using value_type = Key;
+ using size_type = typename container_type::size_type;
+ using difference_type = typename container_type::difference_type;
+ using hasher = Hash;
+ using key_equal = KeyEqual;
+ using const_reference = typename container_type::const_reference;
+ using reference = const_reference;
+ using const_pointer = typename container_type::const_pointer;
+ using pointer = const_pointer;
+ using const_iterator = const_pointer;
+ using iterator = const_iterator;
+
+public:
+ /* constructors */
+ unordered_set(unordered_set const &) = default;
+ constexpr unordered_set(container_type keys, Hash const &hash,
+ KeyEqual const &equal)
+ : equal_{equal}
+ , keys_{keys}
+ , tables_{bits::make_pmh_tables<storage_size>(
+ keys_, hash, bits::Get{}, default_prg_t{})} {}
+ explicit constexpr unordered_set(container_type keys)
+ : unordered_set{keys, Hash{}, KeyEqual{}} {}
+
+ constexpr unordered_set(std::initializer_list<Key> keys)
+ : unordered_set{keys, Hash{}, KeyEqual{}} {}
+
+ constexpr unordered_set(std::initializer_list<Key> keys, Hash const & hash, KeyEqual const & equal)
+ : unordered_set{container_type{keys}, hash, equal} {
+ constexpr_assert(keys.size() == N, "Inconsistent initializer_list size and type size argument");
+ }
+
+ /* iterators */
+ constexpr const_iterator begin() const { return keys_.begin(); }
+ constexpr const_iterator end() const { return keys_.end(); }
+ constexpr const_iterator cbegin() const { return keys_.cbegin(); }
+ constexpr const_iterator cend() const { return keys_.cend(); }
+
+ /* capacity */
+ constexpr bool empty() const { return !N; }
+ constexpr size_type size() const { return N; }
+ constexpr size_type max_size() const { return N; }
+
+ /* lookup */
+ constexpr std::size_t count(Key const &key) const {
+ auto const k = lookup(key);
+ return equal_(k, key);
+ }
+ constexpr const_iterator find(Key const &key) const {
+ auto const &k = lookup(key);
+ if (equal_(k, key))
+ return &k;
+ else
+ return keys_.end();
+ }
+
+ constexpr std::pair<const_iterator, const_iterator> equal_range(Key const &key) const {
+ auto const &k = lookup(key);
+ if (equal_(k, key))
+ return {&k, &k + 1};
+ else
+ return {keys_.end(), keys_.end()};
+ }
+
+ /* bucket interface */
+ constexpr std::size_t bucket_count() const { return storage_size; }
+ constexpr std::size_t max_bucket_count() const { return storage_size; }
+
+ /* observers*/
+ constexpr hasher hash_function() const { return tables_.hash_; }
+ constexpr key_equal key_eq() const { return equal_; }
+
+private:
+ constexpr auto const &lookup(Key const &key) const {
+ return keys_[tables_.lookup(key)];
+ }
+};
+
+template <typename T, std::size_t N>
+constexpr auto make_unordered_set(T const (&keys)[N]) {
+ return unordered_set<T, N>{keys};
+}
+
+} // namespace frozen
+
+#endif
diff --git a/contrib/fu2/LICENSE.txt b/contrib/fu2/LICENSE.txt
new file mode 100644
index 0000000..36b7cd9
--- /dev/null
+++ b/contrib/fu2/LICENSE.txt
@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/contrib/fu2/include/function2/function2.hpp b/contrib/fu2/include/function2/function2.hpp
new file mode 100644
index 0000000..a45cb58
--- /dev/null
+++ b/contrib/fu2/include/function2/function2.hpp
@@ -0,0 +1,1792 @@
+
+// Copyright 2015-2020 Denis Blank <denis.blank at outlook dot com>
+// Distributed under the Boost Software License, Version 1.0
+// (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef FU2_INCLUDED_FUNCTION2_HPP_
+#define FU2_INCLUDED_FUNCTION2_HPP_
+
+#include <cassert>
+#include <cstddef>
+#include <cstdlib>
+#include <memory>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+// Defines:
+// - FU2_HAS_DISABLED_EXCEPTIONS
+#if defined(FU2_WITH_DISABLED_EXCEPTIONS) || \
+ defined(FU2_MACRO_DISABLE_EXCEPTIONS)
+#define FU2_HAS_DISABLED_EXCEPTIONS
+#else // FU2_WITH_DISABLED_EXCEPTIONS
+#if defined(_MSC_VER)
+#if !defined(_HAS_EXCEPTIONS) || (_HAS_EXCEPTIONS == 0)
+#define FU2_HAS_DISABLED_EXCEPTIONS
+#endif
+#elif defined(__clang__)
+#if !(__EXCEPTIONS && __has_feature(cxx_exceptions))
+#define FU2_HAS_DISABLED_EXCEPTIONS
+#endif
+#elif defined(__GNUC__)
+#if !__EXCEPTIONS
+#define FU2_HAS_DISABLED_EXCEPTIONS
+#endif
+#endif
+#endif // FU2_WITH_DISABLED_EXCEPTIONS
+// - FU2_HAS_NO_FUNCTIONAL_HEADER
+#if !defined(FU2_WITH_NO_FUNCTIONAL_HEADER) && \
+ !defined(FU2_NO_FUNCTIONAL_HEADER) && \
+ !defined(FU2_HAS_DISABLED_EXCEPTIONS)
+#include <functional>
+#else
+#define FU2_HAS_NO_FUNCTIONAL_HEADER
+#endif
+// - FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#if defined(FU2_WITH_CXX17_NOEXCEPT_FUNCTION_TYPE)
+#define FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#else // FU2_WITH_CXX17_NOEXCEPT_FUNCTION_TYPE
+#if defined(_MSC_VER)
+#if defined(_HAS_CXX17) && _HAS_CXX17
+#define FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#endif
+#elif defined(__cpp_noexcept_function_type)
+#define FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#elif defined(__cplusplus) && (__cplusplus >= 201703L)
+#define FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#endif
+#endif // FU2_WITH_CXX17_NOEXCEPT_FUNCTION_TYPE
+
+// - FU2_HAS_NO_EMPTY_PROPAGATION
+#if defined(FU2_WITH_NO_EMPTY_PROPAGATION)
+#define FU2_HAS_NO_EMPTY_PROPAGATION
+#endif // FU2_WITH_NO_EMPTY_PROPAGATION
+
+#if !defined(FU2_HAS_DISABLED_EXCEPTIONS)
+#include <exception>
+#endif
+
+#if defined(__cpp_constexpr) && (__cpp_constexpr >= 201304)
+#define FU2_DETAIL_CXX14_CONSTEXPR constexpr
+#elif defined(__clang__) && defined(__has_feature)
+#if __has_feature(__cxx_generic_lambdas__) && \
+ __has_feature(__cxx_relaxed_constexpr__)
+#define FU2_DETAIL_CXX14_CONSTEXPR constexpr
+#endif
+#elif defined(_MSC_VER) && (_MSC_VER >= 1915) && (_MSVC_LANG >= 201402)
+#define FU2_DETAIL_CXX14_CONSTEXPR constexpr
+#endif
+#ifndef FU2_DETAIL_CXX14_CONSTEXPR
+#define FU2_DETAIL_CXX14_CONSTEXPR
+#endif
+
+/// Hint for the compiler that this point should be unreachable
+#if defined(_MSC_VER)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE_INTRINSIC() __assume(false)
+#elif defined(__GNUC__)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE_INTRINSIC() __builtin_unreachable()
+#elif defined(__has_builtin)
+#if __has_builtin(__builtin_unreachable)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE_INTRINSIC() __builtin_unreachable()
+#endif
+#endif
+#ifndef FU2_DETAIL_UNREACHABLE_INTRINSIC
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE_INTRINSIC() abort()
+#endif
+
+/// Causes the application to exit abnormally
+#if defined(_MSC_VER)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_TRAP() __debugbreak()
+#elif defined(__GNUC__)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_TRAP() __builtin_trap()
+#elif defined(__has_builtin)
+#if __has_builtin(__builtin_trap)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_TRAP() __builtin_trap()
+#endif
+#endif
+#ifndef FU2_DETAIL_TRAP
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_TRAP() *(volatile int*)0x11 = 0
+#endif
+
+#ifndef NDEBUG
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE() ::fu2::detail::unreachable_debug()
+#else
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define FU2_DETAIL_UNREACHABLE() FU2_DETAIL_UNREACHABLE_INTRINSIC()
+#endif
+
+namespace fu2 {
+inline namespace abi_400 {
+namespace detail {
+template <typename Config, typename Property>
+class function;
+
+template <typename...>
+struct identity {};
+
+// Equivalent to C++17's std::void_t which targets a bug in GCC,
+// that prevents correct SFINAE behavior.
+// See http://stackoverflow.com/questions/35753920 for details.
+template <typename...>
+struct deduce_to_void : std::common_type<void> {};
+
+template <typename... T>
+using void_t = typename deduce_to_void<T...>::type;
+
+template <typename T>
+using unrefcv_t = std::remove_cv_t<std::remove_reference_t<T>>;
+
+// Copy enabler helper class
+template <bool /*Copyable*/>
+struct copyable {};
+template <>
+struct copyable<false> {
+ copyable() = default;
+ ~copyable() = default;
+ copyable(copyable const&) = delete;
+ copyable(copyable&&) = default;
+ copyable& operator=(copyable const&) = delete;
+ copyable& operator=(copyable&&) = default;
+};
+
+/// Configuration trait to configure the function_base class.
+template <bool Owning, bool Copyable, typename Capacity>
+struct config {
+ // Is true if the function is owning.
+ static constexpr auto const is_owning = Owning;
+
+ // Is true if the function is copyable.
+ static constexpr auto const is_copyable = Copyable;
+
+ // The internal capacity of the function
+ // used in small functor optimization.
+ // The object shall expose the real capacity through Capacity::capacity
+ // and the intended alignment through Capacity::alignment.
+ using capacity = Capacity;
+};
+
+/// A config which isn't compatible to other configs
+template <bool Throws, bool HasStrongExceptGuarantee, typename... Args>
+struct property {
+ // Is true when the function throws an exception on empty invocation.
+ static constexpr auto const is_throwing = Throws;
+
+ // Is true when the function throws an exception on empty invocation.
+ static constexpr auto const is_strong_exception_guaranteed =
+ HasStrongExceptGuarantee;
+};
+
+#ifndef NDEBUG
+[[noreturn]] inline void unreachable_debug() {
+ FU2_DETAIL_TRAP();
+ std::abort();
+}
+#endif
+
+/// Provides utilities for invocing callable objects
+namespace invocation {
+/// Invokes the given callable object with the given arguments
+template <typename Callable, typename... Args>
+constexpr auto invoke(Callable&& callable, Args&&... args) noexcept(
+ noexcept(std::forward<Callable>(callable)(std::forward<Args>(args)...)))
+ -> decltype(std::forward<Callable>(callable)(std::forward<Args>(args)...)) {
+
+ return std::forward<Callable>(callable)(std::forward<Args>(args)...);
+}
+/// Invokes the given member function pointer by reference
+template <typename T, typename Type, typename Self, typename... Args>
+constexpr auto invoke(Type T::*member, Self&& self, Args&&... args) noexcept(
+ noexcept((std::forward<Self>(self).*member)(std::forward<Args>(args)...)))
+ -> decltype((std::forward<Self>(self).*
+ member)(std::forward<Args>(args)...)) {
+ return (std::forward<Self>(self).*member)(std::forward<Args>(args)...);
+}
+/// Invokes the given member function pointer by pointer
+template <typename T, typename Type, typename Self, typename... Args>
+constexpr auto invoke(Type T::*member, Self&& self, Args&&... args) noexcept(
+ noexcept((std::forward<Self>(self)->*member)(std::forward<Args>(args)...)))
+ -> decltype(
+ (std::forward<Self>(self)->*member)(std::forward<Args>(args)...)) {
+ return (std::forward<Self>(self)->*member)(std::forward<Args>(args)...);
+}
+/// Invokes the given pointer to a scalar member by reference
+template <typename T, typename Type, typename Self>
+constexpr auto
+invoke(Type T::*member,
+ Self&& self) noexcept(noexcept(std::forward<Self>(self).*member))
+ -> decltype(std::forward<Self>(self).*member) {
+ return (std::forward<Self>(self).*member);
+}
+/// Invokes the given pointer to a scalar member by pointer
+template <typename T, typename Type, typename Self>
+constexpr auto
+invoke(Type T::*member,
+ Self&& self) noexcept(noexcept(std::forward<Self>(self)->*member))
+ -> decltype(std::forward<Self>(self)->*member) {
+ return std::forward<Self>(self)->*member;
+}
+
+/// Deduces to a true type if the callable object can be invoked with
+/// the given arguments.
+/// We don't use invoke here because MSVC can't evaluate the nested expression
+/// SFINAE here.
+template <typename T, typename Args, typename = void>
+struct can_invoke : std::false_type {};
+template <typename T, typename... Args>
+struct can_invoke<T, identity<Args...>,
+ decltype((void)std::declval<T>()(std::declval<Args>()...))>
+ : std::true_type {};
+template <typename Pointer, typename T, typename... Args>
+struct can_invoke<Pointer, identity<T&, Args...>,
+ decltype((void)((std::declval<T&>().*std::declval<Pointer>())(
+ std::declval<Args>()...)))> : std::true_type {};
+template <typename Pointer, typename T, typename... Args>
+struct can_invoke<Pointer, identity<T&&, Args...>,
+ decltype(
+ (void)((std::declval<T&&>().*std::declval<Pointer>())(
+ std::declval<Args>()...)))> : std::true_type {};
+template <typename Pointer, typename T, typename... Args>
+struct can_invoke<Pointer, identity<T*, Args...>,
+ decltype(
+ (void)((std::declval<T*>()->*std::declval<Pointer>())(
+ std::declval<Args>()...)))> : std::true_type {};
+template <typename Pointer, typename T>
+struct can_invoke<Pointer, identity<T&>,
+ decltype((void)(std::declval<T&>().*std::declval<Pointer>()))>
+ : std::true_type {};
+template <typename Pointer, typename T>
+struct can_invoke<Pointer, identity<T&&>,
+ decltype(
+ (void)(std::declval<T&&>().*std::declval<Pointer>()))>
+ : std::true_type {};
+template <typename Pointer, typename T>
+struct can_invoke<Pointer, identity<T*>,
+ decltype(
+ (void)(std::declval<T*>()->*std::declval<Pointer>()))>
+ : std::true_type {};
+
+template <bool RequiresNoexcept, typename T, typename Args>
+struct is_noexcept_correct : std::true_type {};
+template <typename T, typename... Args>
+struct is_noexcept_correct<true, T, identity<Args...>>
+ : std::integral_constant<bool,
+ noexcept(::fu2::detail::invocation::invoke(
+ std::declval<T>(), std::declval<Args>()...))> {
+};
+} // end namespace invocation
+
+namespace overloading {
+template <typename... Args>
+struct overload_impl;
+template <typename Current, typename Next, typename... Rest>
+struct overload_impl<Current, Next, Rest...> : Current,
+ overload_impl<Next, Rest...> {
+ explicit overload_impl(Current current, Next next, Rest... rest)
+ : Current(std::move(current)), overload_impl<Next, Rest...>(
+ std::move(next), std::move(rest)...) {
+ }
+
+ using Current::operator();
+ using overload_impl<Next, Rest...>::operator();
+};
+template <typename Current>
+struct overload_impl<Current> : Current {
+ explicit overload_impl(Current current) : Current(std::move(current)) {
+ }
+
+ using Current::operator();
+};
+
+template <typename... T>
+constexpr auto overload(T&&... callables) {
+ return overload_impl<std::decay_t<T>...>{std::forward<T>(callables)...};
+}
+} // namespace overloading
+
+/// Declares the namespace which provides the functionality to work with a
+/// type-erased object.
+namespace type_erasure {
+/// Specialization to work with addresses of callable objects
+template <typename T, typename = void>
+struct address_taker {
+ template <typename O>
+ static void* take(O&& obj) {
+ return std::addressof(obj);
+ }
+ static T& restore(void* ptr) {
+ return *static_cast<T*>(ptr);
+ }
+ static T const& restore(void const* ptr) {
+ return *static_cast<T const*>(ptr);
+ }
+ static T volatile& restore(void volatile* ptr) {
+ return *static_cast<T volatile*>(ptr);
+ }
+ static T const volatile& restore(void const volatile* ptr) {
+ return *static_cast<T const volatile*>(ptr);
+ }
+};
+/// Specialization to work with addresses of raw function pointers
+template <typename T>
+struct address_taker<T, std::enable_if_t<std::is_pointer<T>::value>> {
+ template <typename O>
+ static void* take(O&& obj) {
+ return reinterpret_cast<void*>(obj);
+ }
+ template <typename O>
+ static T restore(O ptr) {
+ return reinterpret_cast<T>(const_cast<void*>(ptr));
+ }
+};
+
+template <typename Box>
+struct box_factory;
+/// Store the allocator inside the box
+template <bool IsCopyable, typename T, typename Allocator>
+struct box : private Allocator {
+ friend box_factory<box>;
+
+ T value_;
+
+ explicit box(T value, Allocator allocator)
+ : Allocator(std::move(allocator)), value_(std::move(value)) {
+ }
+
+ box(box&&) = default;
+ box(box const&) = default;
+ box& operator=(box&&) = default;
+ box& operator=(box const&) = default;
+ ~box() = default;
+};
+template <typename T, typename Allocator>
+struct box<false, T, Allocator> : private Allocator {
+ friend box_factory<box>;
+
+ T value_;
+
+ explicit box(T value, Allocator allocator)
+ : Allocator(std::move(allocator)), value_(std::move(value)) {
+ }
+
+ box(box&&) = default;
+ box(box const&) = delete;
+ box& operator=(box&&) = default;
+ box& operator=(box const&) = delete;
+ ~box() = default;
+};
+
+template <bool IsCopyable, typename T, typename Allocator>
+struct box_factory<box<IsCopyable, T, Allocator>> {
+ using real_allocator =
+ typename std::allocator_traits<std::decay_t<Allocator>>::
+ template rebind_alloc<box<IsCopyable, T, Allocator>>;
+
+ /// Allocates space through the boxed allocator
+ static box<IsCopyable, T, Allocator>*
+ box_allocate(box<IsCopyable, T, Allocator> const* me) {
+ real_allocator allocator(*static_cast<Allocator const*>(me));
+
+ return static_cast<box<IsCopyable, T, Allocator>*>(
+ std::allocator_traits<real_allocator>::allocate(allocator, 1U));
+ }
+
+ /// Destroys the box through the given allocator
+ static void box_deallocate(box<IsCopyable, T, Allocator>* me) {
+ real_allocator allocator(*static_cast<Allocator const*>(me));
+
+ me->~box();
+ std::allocator_traits<real_allocator>::deallocate(allocator, me, 1U);
+ }
+};
+
+/// Creates a box containing the given value and allocator
+template <bool IsCopyable, typename T, typename Allocator>
+auto make_box(std::integral_constant<bool, IsCopyable>, T&& value,
+ Allocator&& allocator) {
+ return box<IsCopyable, std::decay_t<T>, std::decay_t<Allocator>>(
+ std::forward<T>(value), std::forward<Allocator>(allocator));
+}
+
+template <typename T>
+struct is_box : std::false_type {};
+template <bool IsCopyable, typename T, typename Allocator>
+struct is_box<box<IsCopyable, T, Allocator>> : std::true_type {};
+
+/// Provides access to the pointer to a heal allocated erased object
+/// as well to the inplace storage.
+union data_accessor {
+ data_accessor() = default;
+ explicit constexpr data_accessor(std::nullptr_t) noexcept : ptr_(nullptr) {
+ }
+ explicit constexpr data_accessor(void* ptr) noexcept : ptr_(ptr) {
+ }
+
+ /// The pointer we use if the object is on the heap
+ void* ptr_;
+ /// The first field of the inplace storage
+ std::size_t inplace_storage_;
+};
+
+/// See opcode::op_fetch_empty
+static FU2_DETAIL_CXX14_CONSTEXPR void write_empty(data_accessor* accessor,
+ bool empty) noexcept {
+ accessor->inplace_storage_ = std::size_t(empty);
+}
+
+template <typename From, typename To>
+using transfer_const_t =
+ std::conditional_t<std::is_const<std::remove_pointer_t<From>>::value,
+ std::add_const_t<To>, To>;
+template <typename From, typename To>
+using transfer_volatile_t =
+ std::conditional_t<std::is_volatile<std::remove_pointer_t<From>>::value,
+ std::add_volatile_t<To>, To>;
+
+/// The retriever when the object is allocated inplace
+template <typename T, typename Accessor>
+FU2_DETAIL_CXX14_CONSTEXPR auto retrieve(std::true_type /*is_inplace*/,
+ Accessor from,
+ std::size_t from_capacity) {
+ using type = transfer_const_t<Accessor, transfer_volatile_t<Accessor, void>>*;
+
+ /// Process the command by using the data inside the internal capacity
+ auto storage = &(from->inplace_storage_);
+ auto inplace = const_cast<void*>(static_cast<type>(storage));
+ return type(std::align(alignof(T), sizeof(T), inplace, from_capacity));
+}
+
+/// The retriever which is used when the object is allocated
+/// through the allocator
+template <typename T, typename Accessor>
+constexpr auto retrieve(std::false_type /*is_inplace*/, Accessor from,
+ std::size_t /*from_capacity*/) {
+
+ return from->ptr_;
+}
+
+namespace invocation_table {
+#if !defined(FU2_HAS_DISABLED_EXCEPTIONS)
+#if defined(FU2_HAS_NO_FUNCTIONAL_HEADER)
+struct bad_function_call : std::exception {
+ bad_function_call() noexcept {
+ }
+
+ char const* what() const noexcept override {
+ return "bad function call";
+ }
+};
+#else
+using std::bad_function_call;
+#endif
+#endif
+
+#ifdef FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#define FU2_DETAIL_EXPAND_QUALIFIERS_NOEXCEPT(F) \
+ F(, , noexcept, , &) \
+ F(const, , noexcept, , &) \
+ F(, volatile, noexcept, , &) \
+ F(const, volatile, noexcept, , &) \
+ F(, , noexcept, &, &) \
+ F(const, , noexcept, &, &) \
+ F(, volatile, noexcept, &, &) \
+ F(const, volatile, noexcept, &, &) \
+ F(, , noexcept, &&, &&) \
+ F(const, , noexcept, &&, &&) \
+ F(, volatile, noexcept, &&, &&) \
+ F(const, volatile, noexcept, &&, &&)
+#define FU2_DETAIL_EXPAND_CV_NOEXCEPT(F) \
+ F(, , noexcept) \
+ F(const, , noexcept) \
+ F(, volatile, noexcept) \
+ F(const, volatile, noexcept)
+#else // FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+#define FU2_DETAIL_EXPAND_QUALIFIERS_NOEXCEPT(F)
+#define FU2_DETAIL_EXPAND_CV_NOEXCEPT(F)
+#endif // FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE
+
+#define FU2_DETAIL_EXPAND_QUALIFIERS(F) \
+ F(, , , , &) \
+ F(const, , , , &) \
+ F(, volatile, , , &) \
+ F(const, volatile, , , &) \
+ F(, , , &, &) \
+ F(const, , , &, &) \
+ F(, volatile, , &, &) \
+ F(const, volatile, , &, &) \
+ F(, , , &&, &&) \
+ F(const, , , &&, &&) \
+ F(, volatile, , &&, &&) \
+ F(const, volatile, , &&, &&) \
+ FU2_DETAIL_EXPAND_QUALIFIERS_NOEXCEPT(F)
+#define FU2_DETAIL_EXPAND_CV(F) \
+ F(, , ) \
+ F(const, , ) \
+ F(, volatile, ) \
+ F(const, volatile, ) \
+ FU2_DETAIL_EXPAND_CV_NOEXCEPT(F)
+
+/// If the function is qualified as noexcept, the call will never throw
+template <bool IsNoexcept>
+[[noreturn]] void throw_or_abortnoexcept(
+ std::integral_constant<bool, IsNoexcept> /*is_throwing*/) noexcept {
+ std::abort();
+}
+/// Calls std::abort on empty function calls
+[[noreturn]] inline void
+throw_or_abort(std::false_type /*is_throwing*/) noexcept {
+ std::abort();
+}
+/// Throws bad_function_call on empty funciton calls
+[[noreturn]] inline void throw_or_abort(std::true_type /*is_throwing*/) {
+#ifdef FU2_HAS_DISABLED_EXCEPTIONS
+ throw_or_abort(std::false_type{});
+#else
+ throw bad_function_call{};
+#endif
+}
+
+template <typename T>
+struct function_trait;
+
+using is_noexcept_ = std::false_type;
+using is_noexcept_noexcept = std::true_type;
+
+#define FU2_DEFINE_FUNCTION_TRAIT(CONST, VOLATILE, NOEXCEPT, OVL_REF, REF) \
+ template <typename Ret, typename... Args> \
+ struct function_trait<Ret(Args...) CONST VOLATILE OVL_REF NOEXCEPT> { \
+ using pointer_type = Ret (*)(data_accessor CONST VOLATILE*, \
+ std::size_t capacity, Args...); \
+ template <typename T, bool IsInplace> \
+ struct internal_invoker { \
+ static Ret invoke(data_accessor CONST VOLATILE* data, \
+ std::size_t capacity, Args... args) NOEXCEPT { \
+ auto obj = retrieve<T>(std::integral_constant<bool, IsInplace>{}, \
+ data, capacity); \
+ auto box = static_cast<T CONST VOLATILE*>(obj); \
+ return invocation::invoke( \
+ static_cast<std::decay_t<decltype(box->value_)> CONST VOLATILE \
+ REF>(box->value_), \
+ std::forward<Args>(args)...); \
+ } \
+ }; \
+ \
+ template <typename T> \
+ struct view_invoker { \
+ static Ret invoke(data_accessor CONST VOLATILE* data, std::size_t, \
+ Args... args) NOEXCEPT { \
+ \
+ auto ptr = static_cast<void CONST VOLATILE*>(data->ptr_); \
+ return invocation::invoke(address_taker<T>::restore(ptr), \
+ std::forward<Args>(args)...); \
+ } \
+ }; \
+ \
+ template <typename T> \
+ using callable = T CONST VOLATILE REF; \
+ \
+ using arguments = identity<Args...>; \
+ \
+ using is_noexcept = is_noexcept_##NOEXCEPT; \
+ \
+ template <bool Throws> \
+ struct empty_invoker { \
+ static Ret invoke(data_accessor CONST VOLATILE* /*data*/, \
+ std::size_t /*capacity*/, Args... /*args*/) NOEXCEPT { \
+ throw_or_abort##NOEXCEPT(std::integral_constant<bool, Throws>{}); \
+ } \
+ }; \
+ };
+
+FU2_DETAIL_EXPAND_QUALIFIERS(FU2_DEFINE_FUNCTION_TRAIT)
+#undef FU2_DEFINE_FUNCTION_TRAIT
+
+/// Deduces to the function pointer to the given signature
+template <typename Signature>
+using function_pointer_of = typename function_trait<Signature>::pointer_type;
+
+template <typename... Args>
+struct invoke_table;
+
+/// We optimize the vtable_t in case there is a single function overload
+template <typename First>
+struct invoke_table<First> {
+ using type = function_pointer_of<First>;
+
+ /// Return the function pointer itself
+ template <std::size_t Index>
+ static constexpr auto fetch(type pointer) noexcept {
+ static_assert(Index == 0U, "The index should be 0 here!");
+ return pointer;
+ }
+
+ /// Returns the thunk of an single overloaded callable
+ template <typename T, bool IsInplace>
+ static constexpr type get_invocation_table_of() noexcept {
+ return &function_trait<First>::template internal_invoker<T,
+ IsInplace>::invoke;
+ }
+ /// Returns the thunk of an single overloaded callable
+ template <typename T>
+ static constexpr type get_invocation_view_table_of() noexcept {
+ return &function_trait<First>::template view_invoker<T>::invoke;
+ }
+ /// Returns the thunk of an empty single overloaded callable
+ template <bool IsThrowing>
+ static constexpr type get_empty_invocation_table() noexcept {
+ return &function_trait<First>::template empty_invoker<IsThrowing>::invoke;
+ }
+};
+/// We generate a table in case of multiple function overloads
+template <typename First, typename Second, typename... Args>
+struct invoke_table<First, Second, Args...> {
+ using type =
+ std::tuple<function_pointer_of<First>, function_pointer_of<Second>,
+ function_pointer_of<Args>...> const*;
+
+ /// Return the function pointer at the particular index
+ template <std::size_t Index>
+ static constexpr auto fetch(type table) noexcept {
+ return std::get<Index>(*table);
+ }
+
+ /// The invocation vtable for a present object
+ template <typename T, bool IsInplace>
+ struct invocation_vtable : public std::tuple<function_pointer_of<First>,
+ function_pointer_of<Second>,
+ function_pointer_of<Args>...> {
+ constexpr invocation_vtable() noexcept
+ : std::tuple<function_pointer_of<First>, function_pointer_of<Second>,
+ function_pointer_of<Args>...>(std::make_tuple(
+ &function_trait<First>::template internal_invoker<
+ T, IsInplace>::invoke,
+ &function_trait<Second>::template internal_invoker<
+ T, IsInplace>::invoke,
+ &function_trait<Args>::template internal_invoker<
+ T, IsInplace>::invoke...)) {
+ }
+ };
+
+ /// Returns the thunk of an multi overloaded callable
+ template <typename T, bool IsInplace>
+ static type get_invocation_table_of() noexcept {
+ static invocation_vtable<T, IsInplace> const table;
+ return &table;
+ }
+
+ /// The invocation vtable for a present object
+ template <typename T>
+ struct invocation_view_vtable
+ : public std::tuple<function_pointer_of<First>,
+ function_pointer_of<Second>,
+ function_pointer_of<Args>...> {
+ constexpr invocation_view_vtable() noexcept
+ : std::tuple<function_pointer_of<First>, function_pointer_of<Second>,
+ function_pointer_of<Args>...>(std::make_tuple(
+ &function_trait<First>::template view_invoker<T>::invoke,
+ &function_trait<Second>::template view_invoker<T>::invoke,
+ &function_trait<Args>::template view_invoker<T>::invoke...)) {
+ }
+ };
+
+ /// Returns the thunk of an multi overloaded callable
+ template <typename T>
+ static type get_invocation_view_table_of() noexcept {
+ static invocation_view_vtable<T> const table;
+ return &table;
+ }
+
+ /// The invocation table for an empty wrapper
+ template <bool IsThrowing>
+ struct empty_vtable : public std::tuple<function_pointer_of<First>,
+ function_pointer_of<Second>,
+ function_pointer_of<Args>...> {
+ constexpr empty_vtable() noexcept
+ : std::tuple<function_pointer_of<First>, function_pointer_of<Second>,
+ function_pointer_of<Args>...>(
+ std::make_tuple(&function_trait<First>::template empty_invoker<
+ IsThrowing>::invoke,
+ &function_trait<Second>::template empty_invoker<
+ IsThrowing>::invoke,
+ &function_trait<Args>::template empty_invoker<
+ IsThrowing>::invoke...)) {
+ }
+ };
+
+ /// Returns the thunk of an multi single overloaded callable
+ template <bool IsThrowing>
+ static type get_empty_invocation_table() noexcept {
+ static empty_vtable<IsThrowing> const table;
+ return &table;
+ }
+};
+
+template <std::size_t Index, typename Function, typename... Signatures>
+class operator_impl;
+
+#define FU2_DEFINE_FUNCTION_TRAIT(CONST, VOLATILE, NOEXCEPT, OVL_REF, REF) \
+ template <std::size_t Index, typename Function, typename Ret, \
+ typename... Args, typename Next, typename... Signatures> \
+ class operator_impl<Index, Function, \
+ Ret(Args...) CONST VOLATILE OVL_REF NOEXCEPT, Next, \
+ Signatures...> \
+ : operator_impl<Index + 1, Function, Next, Signatures...> { \
+ \
+ template <std::size_t, typename, typename...> \
+ friend class operator_impl; \
+ \
+ protected: \
+ operator_impl() = default; \
+ ~operator_impl() = default; \
+ operator_impl(operator_impl const&) = default; \
+ operator_impl(operator_impl&&) = default; \
+ operator_impl& operator=(operator_impl const&) = default; \
+ operator_impl& operator=(operator_impl&&) = default; \
+ \
+ using operator_impl<Index + 1, Function, Next, Signatures...>::operator(); \
+ \
+ Ret operator()(Args... args) CONST VOLATILE OVL_REF NOEXCEPT { \
+ auto parent = static_cast<Function CONST VOLATILE*>(this); \
+ using erasure_t = std::decay_t<decltype(parent->erasure_)>; \
+ \
+ /* `std::decay_t<decltype(parent->erasure_)>` is a workaround for a */ \
+ /* compiler regression of MSVC 16.3.1, see #29 for details. */ \
+ return std::decay_t<decltype(parent->erasure_)>::template invoke<Index>( \
+ static_cast<erasure_t CONST VOLATILE REF>(parent->erasure_), \
+ std::forward<Args>(args)...); \
+ } \
+ }; \
+ template <std::size_t Index, typename Config, typename Property, \
+ typename Ret, typename... Args> \
+ class operator_impl<Index, function<Config, Property>, \
+ Ret(Args...) CONST VOLATILE OVL_REF NOEXCEPT> \
+ : copyable<!Config::is_owning || Config::is_copyable> { \
+ \
+ template <std::size_t, typename, typename...> \
+ friend class operator_impl; \
+ \
+ protected: \
+ operator_impl() = default; \
+ ~operator_impl() = default; \
+ operator_impl(operator_impl const&) = default; \
+ operator_impl(operator_impl&&) = default; \
+ operator_impl& operator=(operator_impl const&) = default; \
+ operator_impl& operator=(operator_impl&&) = default; \
+ \
+ Ret operator()(Args... args) CONST VOLATILE OVL_REF NOEXCEPT { \
+ auto parent = \
+ static_cast<function<Config, Property> CONST VOLATILE*>(this); \
+ using erasure_t = std::decay_t<decltype(parent->erasure_)>; \
+ \
+ /* `std::decay_t<decltype(parent->erasure_)>` is a workaround for a */ \
+ /* compiler regression of MSVC 16.3.1, see #29 for details. */ \
+ return std::decay_t<decltype(parent->erasure_)>::template invoke<Index>( \
+ static_cast<erasure_t CONST VOLATILE REF>(parent->erasure_), \
+ std::forward<Args>(args)...); \
+ } \
+ };
+
+FU2_DETAIL_EXPAND_QUALIFIERS(FU2_DEFINE_FUNCTION_TRAIT)
+#undef FU2_DEFINE_FUNCTION_TRAIT
+} // namespace invocation_table
+
+namespace tables {
+/// Identifies the action which is dispatched on the erased object
+enum class opcode {
+ op_move, ///< Move the object and set the vtable
+ op_copy, ///< Copy the object and set the vtable
+ op_destroy, ///< Destroy the object and reset the vtable
+ op_weak_destroy, ///< Destroy the object without resetting the vtable
+ op_fetch_empty, ///< Stores true or false into the to storage
+ ///< to indicate emptiness
+};
+
+/// Abstraction for a vtable together with a command table
+/// TODO Add optimization for a single formal argument
+/// TODO Add optimization to merge both tables if the function is size
+/// optimized
+template <typename Property>
+class vtable;
+template <bool IsThrowing, bool HasStrongExceptGuarantee,
+ typename... FormalArgs>
+class vtable<property<IsThrowing, HasStrongExceptGuarantee, FormalArgs...>> {
+ using command_function_t = void (*)(vtable* /*this*/, opcode /*op*/,
+ data_accessor* /*from*/,
+ std::size_t /*from_capacity*/,
+ data_accessor* /*to*/,
+ std::size_t /*to_capacity*/);
+
+ using invoke_table_t = invocation_table::invoke_table<FormalArgs...>;
+
+ command_function_t cmd_;
+ typename invoke_table_t::type vtable_;
+
+ template <typename T>
+ struct trait {
+ static_assert(is_box<T>::value,
+ "The trait must be specialized with a box!");
+
+ /// The command table
+ template <bool IsInplace>
+ static void process_cmd(vtable* to_table, opcode op, data_accessor* from,
+ std::size_t from_capacity, data_accessor* to,
+ std::size_t to_capacity) {
+
+ switch (op) {
+ case opcode::op_move: {
+ /// Retrieve the pointer to the object
+ auto box = static_cast<T*>(retrieve<T>(
+ std::integral_constant<bool, IsInplace>{}, from, from_capacity));
+ assert(box && "The object must not be over aligned or null!");
+
+ if (!IsInplace) {
+ // Just swap both pointers if we allocated on the heap
+ to->ptr_ = from->ptr_;
+
+#ifndef NDEBUG
+ // We don't need to null the pointer since we know that
+ // we don't own the data anymore through the vtable
+ // which is set to empty.
+ from->ptr_ = nullptr;
+#endif
+
+ to_table->template set_allocated<T>();
+
+ }
+ // The object is allocated inplace
+ else {
+ construct(std::true_type{}, std::move(*box), to_table, to,
+ to_capacity);
+ box->~T();
+ }
+ return;
+ }
+ case opcode::op_copy: {
+ auto box = static_cast<T const*>(retrieve<T>(
+ std::integral_constant<bool, IsInplace>{}, from, from_capacity));
+ assert(box && "The object must not be over aligned or null!");
+
+ assert(std::is_copy_constructible<T>::value &&
+ "The box is required to be copyable here!");
+
+ // Try to allocate the object inplace
+ construct(std::is_copy_constructible<T>{}, *box, to_table, to,
+ to_capacity);
+ return;
+ }
+ case opcode::op_destroy:
+ case opcode::op_weak_destroy: {
+
+ assert(!to && !to_capacity && "Arg overflow!");
+ auto box = static_cast<T*>(retrieve<T>(
+ std::integral_constant<bool, IsInplace>{}, from, from_capacity));
+
+ if (IsInplace) {
+ box->~T();
+ } else {
+ box_factory<T>::box_deallocate(box);
+ }
+
+ if (op == opcode::op_destroy) {
+ to_table->set_empty();
+ }
+ return;
+ }
+ case opcode::op_fetch_empty: {
+ write_empty(to, false);
+ return;
+ }
+ }
+
+ FU2_DETAIL_UNREACHABLE();
+ }
+
+ template <typename Box>
+ static void
+ construct(std::true_type /*apply*/, Box&& box, vtable* to_table,
+ data_accessor* to,
+ std::size_t to_capacity) noexcept(HasStrongExceptGuarantee) {
+ // Try to allocate the object inplace
+ void* storage = retrieve<T>(std::true_type{}, to, to_capacity);
+ if (storage) {
+ to_table->template set_inplace<T>();
+ } else {
+ // Allocate the object through the allocator
+ to->ptr_ = storage =
+ box_factory<std::decay_t<Box>>::box_allocate(std::addressof(box));
+ to_table->template set_allocated<T>();
+ }
+ new (storage) T(std::forward<Box>(box));
+ }
+
+ template <typename Box>
+ static void
+ construct(std::false_type /*apply*/, Box&& /*box*/, vtable* /*to_table*/,
+ data_accessor* /*to*/,
+ std::size_t /*to_capacity*/) noexcept(HasStrongExceptGuarantee) {
+ }
+ };
+
+ /// The command table
+ static void empty_cmd(vtable* to_table, opcode op, data_accessor* /*from*/,
+ std::size_t /*from_capacity*/, data_accessor* to,
+ std::size_t /*to_capacity*/) {
+
+ switch (op) {
+ case opcode::op_move:
+ case opcode::op_copy: {
+ to_table->set_empty();
+ break;
+ }
+ case opcode::op_destroy:
+ case opcode::op_weak_destroy: {
+ // Do nothing
+ break;
+ }
+ case opcode::op_fetch_empty: {
+ write_empty(to, true);
+ break;
+ }
+ default: {
+ FU2_DETAIL_UNREACHABLE();
+ }
+ }
+ }
+
+public:
+ vtable() noexcept = default;
+
+ /// Initialize an object at the given position
+ template <typename T>
+ static void init(vtable& table, T&& object, data_accessor* to,
+ std::size_t to_capacity) {
+
+ trait<std::decay_t<T>>::construct(std::true_type{}, std::forward<T>(object),
+ &table, to, to_capacity);
+ }
+
+ /// Moves the object at the given position
+ void move(vtable& to_table, data_accessor* from, std::size_t from_capacity,
+ data_accessor* to,
+ std::size_t to_capacity) noexcept(HasStrongExceptGuarantee) {
+ cmd_(&to_table, opcode::op_move, from, from_capacity, to, to_capacity);
+ set_empty();
+ }
+
+ /// Destroys the object at the given position
+ void copy(vtable& to_table, data_accessor const* from,
+ std::size_t from_capacity, data_accessor* to,
+ std::size_t to_capacity) const {
+ cmd_(&to_table, opcode::op_copy, const_cast<data_accessor*>(from),
+ from_capacity, to, to_capacity);
+ }
+
+ /// Destroys the object at the given position
+ void destroy(data_accessor* from,
+ std::size_t from_capacity) noexcept(HasStrongExceptGuarantee) {
+ cmd_(this, opcode::op_destroy, from, from_capacity, nullptr, 0U);
+ }
+
+ /// Destroys the object at the given position without invalidating the
+ /// vtable
+ void
+ weak_destroy(data_accessor* from,
+ std::size_t from_capacity) noexcept(HasStrongExceptGuarantee) {
+ cmd_(this, opcode::op_weak_destroy, from, from_capacity, nullptr, 0U);
+ }
+
+ /// Returns true when the vtable doesn't hold any erased object
+ bool empty() const noexcept {
+ data_accessor data;
+ cmd_(nullptr, opcode::op_fetch_empty, nullptr, 0U, &data, 0U);
+ return bool(data.inplace_storage_);
+ }
+
+ /// Invoke the function at the given index
+ template <std::size_t Index, typename... Args>
+ constexpr decltype(auto) invoke(Args&&... args) const {
+ auto thunk = invoke_table_t::template fetch<Index>(vtable_);
+ return thunk(std::forward<Args>(args)...);
+ }
+ /// Invoke the function at the given index
+ template <std::size_t Index, typename... Args>
+ constexpr decltype(auto) invoke(Args&&... args) const volatile {
+ auto thunk = invoke_table_t::template fetch<Index>(vtable_);
+ return thunk(std::forward<Args>(args)...);
+ }
+
+ template <typename T>
+ void set_inplace() noexcept {
+ using type = std::decay_t<T>;
+ vtable_ = invoke_table_t::template get_invocation_table_of<type, true>();
+ cmd_ = &trait<type>::template process_cmd<true>;
+ }
+
+ template <typename T>
+ void set_allocated() noexcept {
+ using type = std::decay_t<T>;
+ vtable_ = invoke_table_t::template get_invocation_table_of<type, false>();
+ cmd_ = &trait<type>::template process_cmd<false>;
+ }
+
+ void set_empty() noexcept {
+ vtable_ = invoke_table_t::template get_empty_invocation_table<IsThrowing>();
+ cmd_ = &empty_cmd;
+ }
+};
+} // namespace tables
+
+/// A union which makes the pointer to the heap object share the
+/// same space with the internal capacity.
+/// The storage type is distinguished by multiple versions of the
+/// control and vtable.
+template <typename Capacity, typename = void>
+struct internal_capacity {
+ /// We extend the union through a technique similar to the tail object hack
+ typedef union {
+ /// Tag to access the structure in a type-safe way
+ data_accessor accessor_;
+ /// The internal capacity we use to allocate in-place
+ std::aligned_storage_t<Capacity::capacity, Capacity::alignment> capacity_;
+ } type;
+};
+template <typename Capacity>
+struct internal_capacity<
+ Capacity, std::enable_if_t<(Capacity::capacity < sizeof(void*))>> {
+ typedef struct {
+ /// Tag to access the structure in a type-safe way
+ data_accessor accessor_;
+ } type;
+};
+
+template <typename Capacity>
+class internal_capacity_holder {
+ // Tag to access the structure in a type-safe way
+ typename internal_capacity<Capacity>::type storage_;
+
+public:
+ constexpr internal_capacity_holder() = default;
+
+ FU2_DETAIL_CXX14_CONSTEXPR data_accessor* opaque_ptr() noexcept {
+ return &storage_.accessor_;
+ }
+ constexpr data_accessor const* opaque_ptr() const noexcept {
+ return &storage_.accessor_;
+ }
+ FU2_DETAIL_CXX14_CONSTEXPR data_accessor volatile*
+ opaque_ptr() volatile noexcept {
+ return &storage_.accessor_;
+ }
+ constexpr data_accessor const volatile* opaque_ptr() const volatile noexcept {
+ return &storage_.accessor_;
+ }
+
+ static constexpr std::size_t capacity() noexcept {
+ return sizeof(storage_);
+ }
+};
+
+/// An owning erasure
+template <bool IsOwning /* = true*/, typename Config, typename Property>
+class erasure : internal_capacity_holder<typename Config::capacity> {
+ template <bool, typename, typename>
+ friend class erasure;
+ template <std::size_t, typename, typename...>
+ friend class operator_impl;
+
+ using vtable_t = tables::vtable<Property>;
+
+ vtable_t vtable_;
+
+public:
+ /// Returns the capacity of this erasure
+ static constexpr std::size_t capacity() noexcept {
+ return internal_capacity_holder<typename Config::capacity>::capacity();
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure() noexcept {
+ vtable_.set_empty();
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure(std::nullptr_t) noexcept {
+ vtable_.set_empty();
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR
+ erasure(erasure&& right) noexcept(Property::is_strong_exception_guaranteed) {
+ right.vtable_.move(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure(erasure const& right) {
+ right.vtable_.copy(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ }
+
+ template <typename OtherConfig>
+ FU2_DETAIL_CXX14_CONSTEXPR
+ erasure(erasure<true, OtherConfig, Property> right) noexcept(
+ Property::is_strong_exception_guaranteed) {
+ right.vtable_.move(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ }
+
+ template <typename T, typename Allocator = std::allocator<std::decay_t<T>>>
+ FU2_DETAIL_CXX14_CONSTEXPR erasure(std::false_type /*use_bool_op*/,
+ T&& callable,
+ Allocator&& allocator = Allocator{}) {
+ vtable_t::init(vtable_,
+ type_erasure::make_box(
+ std::integral_constant<bool, Config::is_copyable>{},
+ std::forward<T>(callable),
+ std::forward<Allocator>(allocator)),
+ this->opaque_ptr(), capacity());
+ }
+ template <typename T, typename Allocator = std::allocator<std::decay_t<T>>>
+ FU2_DETAIL_CXX14_CONSTEXPR erasure(std::true_type /*use_bool_op*/,
+ T&& callable,
+ Allocator&& allocator = Allocator{}) {
+ if (bool(callable)) {
+ vtable_t::init(vtable_,
+ type_erasure::make_box(
+ std::integral_constant<bool, Config::is_copyable>{},
+ std::forward<T>(callable),
+ std::forward<Allocator>(allocator)),
+ this->opaque_ptr(), capacity());
+ } else {
+ vtable_.set_empty();
+ }
+ }
+
+ ~erasure() {
+ vtable_.weak_destroy(this->opaque_ptr(), capacity());
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure&
+ operator=(std::nullptr_t) noexcept(Property::is_strong_exception_guaranteed) {
+ vtable_.destroy(this->opaque_ptr(), capacity());
+ return *this;
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure& operator=(erasure&& right) noexcept(
+ Property::is_strong_exception_guaranteed) {
+ vtable_.weak_destroy(this->opaque_ptr(), capacity());
+ right.vtable_.move(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ return *this;
+ }
+
+ FU2_DETAIL_CXX14_CONSTEXPR erasure& operator=(erasure const& right) {
+ vtable_.weak_destroy(this->opaque_ptr(), capacity());
+ right.vtable_.copy(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ return *this;
+ }
+
+ template <typename OtherConfig>
+ FU2_DETAIL_CXX14_CONSTEXPR erasure&
+ operator=(erasure<true, OtherConfig, Property> right) noexcept(
+ Property::is_strong_exception_guaranteed) {
+ vtable_.weak_destroy(this->opaque_ptr(), capacity());
+ right.vtable_.move(vtable_, right.opaque_ptr(), right.capacity(),
+ this->opaque_ptr(), capacity());
+ return *this;
+ }
+
+ template <typename T, typename Allocator = std::allocator<std::decay_t<T>>>
+ void assign(std::false_type /*use_bool_op*/, T&& callable,
+ Allocator&& allocator = {}) {
+ vtable_.weak_destroy(this->opaque_ptr(), capacity());
+ vtable_t::init(vtable_,
+ type_erasure::make_box(
+ std::integral_constant<bool, Config::is_copyable>{},
+ std::forward<T>(callable),
+ std::forward<Allocator>(allocator)),
+ this->opaque_ptr(), capacity());
+ }
+
+ template <typename T, typename Allocator = std::allocator<std::decay_t<T>>>
+ void assign(std::true_type /*use_bool_op*/, T&& callable,
+ Allocator&& allocator = {}) {
+ if (bool(callable)) {
+ assign(std::false_type{}, std::forward<T>(callable),
+ std::forward<Allocator>(allocator));
+ } else {
+ operator=(nullptr);
+ }
+ }
+
+ /// Returns true when the erasure doesn't hold any erased object
+ constexpr bool empty() const noexcept {
+ return vtable_.empty();
+ }
+
+ /// Invoke the function of the erasure at the given index
+ ///
+ /// We define this out of class to be able to forward the qualified
+ /// erasure correctly.
+ template <std::size_t Index, typename Erasure, typename... Args>
+ static constexpr decltype(auto) invoke(Erasure&& erasure, Args&&... args) {
+ auto const capacity = erasure.capacity();
+ return erasure.vtable_.template invoke<Index>(
+ std::forward<Erasure>(erasure).opaque_ptr(), capacity,
+ std::forward<Args>(args)...);
+ }
+};
+
+// A non owning erasure
+template </*bool IsOwning = false, */ typename Config, bool IsThrowing,
+ bool HasStrongExceptGuarantee, typename... Args>
+class erasure<false, Config,
+ property<IsThrowing, HasStrongExceptGuarantee, Args...>> {
+ template <bool, typename, typename>
+ friend class erasure;
+ template <std::size_t, typename, typename...>
+ friend class operator_impl;
+
+ using property_t = property<IsThrowing, HasStrongExceptGuarantee, Args...>;
+
+ using invoke_table_t = invocation_table::invoke_table<Args...>;
+ typename invoke_table_t::type invoke_table_;
+
+ /// The internal pointer to the non owned object
+ data_accessor view_;
+
+public:
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ constexpr erasure() noexcept
+ : invoke_table_(
+ invoke_table_t::template get_empty_invocation_table<IsThrowing>()),
+ view_(nullptr) {
+ }
+
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ constexpr erasure(std::nullptr_t) noexcept
+ : invoke_table_(
+ invoke_table_t::template get_empty_invocation_table<IsThrowing>()),
+ view_(nullptr) {
+ }
+
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ constexpr erasure(erasure&& right) noexcept
+ : invoke_table_(right.invoke_table_), view_(right.view_) {
+ }
+
+ constexpr erasure(erasure const& /*right*/) = default;
+
+ template <typename OtherConfig>
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ constexpr erasure(erasure<false, OtherConfig, property_t> right) noexcept
+ : invoke_table_(right.invoke_table_), view_(right.view_) {
+ }
+
+ template <typename T>
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ constexpr erasure(std::false_type /*use_bool_op*/, T&& object)
+ : invoke_table_(invoke_table_t::template get_invocation_view_table_of<
+ std::decay_t<T>>()),
+ view_(address_taker<std::decay_t<T>>::take(std::forward<T>(object))) {
+ }
+ template <typename T>
+ // NOLINTNEXTLINE(cppcoreguidlines-pro-type-member-init)
+ FU2_DETAIL_CXX14_CONSTEXPR erasure(std::true_type use_bool_op, T&& object) {
+ this->assign(use_bool_op, std::forward<T>(object));
+ }
+
+ ~erasure() = default;
+
+ constexpr erasure&
+ operator=(std::nullptr_t) noexcept(HasStrongExceptGuarantee) {
+ invoke_table_ =
+ invoke_table_t::template get_empty_invocation_table<IsThrowing>();
+ view_.ptr_ = nullptr;
+ return *this;
+ }
+
+ constexpr erasure& operator=(erasure&& right) noexcept {
+ invoke_table_ = right.invoke_table_;
+ view_ = right.view_;
+ right = nullptr;
+ return *this;
+ }
+
+ constexpr erasure& operator=(erasure const& /*right*/) = default;
+
+ template <typename OtherConfig>
+ constexpr erasure&
+ operator=(erasure<true, OtherConfig, property_t> right) noexcept {
+ invoke_table_ = right.invoke_table_;
+ view_ = right.view_;
+ return *this;
+ }
+
+ template <typename T>
+ constexpr void assign(std::false_type /*use_bool_op*/, T&& callable) {
+ invoke_table_ = invoke_table_t::template get_invocation_view_table_of<
+ std::decay_t<T>>();
+ view_.ptr_ =
+ address_taker<std::decay_t<T>>::take(std::forward<T>(callable));
+ }
+ template <typename T>
+ constexpr void assign(std::true_type /*use_bool_op*/, T&& callable) {
+ if (bool(callable)) {
+ assign(std::false_type{}, std::forward<T>(callable));
+ } else {
+ operator=(nullptr);
+ }
+ }
+
+ /// Returns true when the erasure doesn't hold any erased object
+ constexpr bool empty() const noexcept {
+ return view_.ptr_ == nullptr;
+ }
+
+ template <std::size_t Index, typename Erasure, typename... T>
+ static constexpr decltype(auto) invoke(Erasure&& erasure, T&&... args) {
+ auto thunk = invoke_table_t::template fetch<Index>(erasure.invoke_table_);
+ return thunk(&(erasure.view_), 0UL, std::forward<T>(args)...);
+ }
+};
+} // namespace type_erasure
+
+/// Deduces to a true_type if the type T provides the given signature and the
+/// signature is noexcept correct callable.
+template <typename T, typename Signature,
+ typename Trait =
+ type_erasure::invocation_table::function_trait<Signature>>
+struct accepts_one
+ : std::integral_constant<
+ bool, invocation::can_invoke<typename Trait::template callable<T>,
+ typename Trait::arguments>::value &&
+ invocation::is_noexcept_correct<
+ Trait::is_noexcept::value,
+ typename Trait::template callable<T>,
+ typename Trait::arguments>::value> {};
+
+/// Deduces to a true_type if the type T provides all signatures
+template <typename T, typename Signatures, typename = void>
+struct accepts_all : std::false_type {};
+template <typename T, typename... Signatures>
+struct accepts_all<
+ T, identity<Signatures...>,
+ void_t<std::enable_if_t<accepts_one<T, Signatures>::value>...>>
+ : std::true_type {};
+
+/// Deduces to a true_type if the type T is implementing operator bool()
+/// or if the type is convertible to bool directly, this also implements an
+/// optimizations for function references `void(&)()` which are can never
+/// be null and for such a conversion to bool would never return false.
+#if defined(FU2_HAS_NO_EMPTY_PROPAGATION)
+template <typename T>
+struct use_bool_op : std::false_type {};
+#else
+template <typename T, typename = void>
+struct has_bool_op : std::false_type {};
+template <typename T>
+struct has_bool_op<T, void_t<decltype(bool(std::declval<T>()))>>
+ : std::true_type {
+#ifndef NDEBUG
+ static_assert(!std::is_pointer<T>::value,
+ "Missing deduction for function pointer!");
+#endif
+};
+
+template <typename T>
+struct use_bool_op : has_bool_op<T> {};
+
+#define FU2_DEFINE_USE_OP_TRAIT(CONST, VOLATILE, NOEXCEPT) \
+ template <typename Ret, typename... Args> \
+ struct use_bool_op<Ret (*CONST VOLATILE)(Args...) NOEXCEPT> \
+ : std::true_type {};
+
+FU2_DETAIL_EXPAND_CV(FU2_DEFINE_USE_OP_TRAIT)
+#undef FU2_DEFINE_USE_OP_TRAIT
+
+template <typename Ret, typename... Args>
+struct use_bool_op<Ret(Args...)> : std::false_type {};
+
+#if defined(FU2_HAS_CXX17_NOEXCEPT_FUNCTION_TYPE)
+template <typename Ret, typename... Args>
+struct use_bool_op<Ret(Args...) noexcept> : std::false_type {};
+#endif
+#endif // FU2_HAS_NO_EMPTY_PROPAGATION
+
+template <typename Config, typename T>
+struct assert_wrong_copy_assign {
+ static_assert(!Config::is_owning || !Config::is_copyable ||
+ std::is_copy_constructible<std::decay_t<T>>::value,
+ "Can't wrap a non copyable object into a unique function!");
+
+ using type = void;
+};
+
+template <bool IsStrongExceptGuaranteed, typename T>
+struct assert_no_strong_except_guarantee {
+ static_assert(
+ !IsStrongExceptGuaranteed ||
+ (std::is_nothrow_move_constructible<T>::value &&
+ std::is_nothrow_destructible<T>::value),
+ "Can't wrap a object an object that has no strong exception guarantees "
+ "if this is required by the wrapper!");
+
+ using type = void;
+};
+
+/// SFINAES out if the given callable is not copyable correct to the left one.
+template <typename LeftConfig, typename RightConfig>
+using enable_if_copyable_correct_t =
+ std::enable_if_t<(!LeftConfig::is_copyable || RightConfig::is_copyable)>;
+
+template <typename LeftConfig, typename RightConfig>
+using is_owning_correct =
+ std::integral_constant<bool,
+ (LeftConfig::is_owning == RightConfig::is_owning)>;
+
+/// SFINAES out if the given function2 is not owning correct to this one
+template <typename LeftConfig, typename RightConfig>
+using enable_if_owning_correct_t =
+ std::enable_if_t<is_owning_correct<LeftConfig, RightConfig>::value>;
+
+template <typename Config, bool IsThrowing, bool HasStrongExceptGuarantee,
+ typename... Args>
+class function<Config, property<IsThrowing, HasStrongExceptGuarantee, Args...>>
+ : type_erasure::invocation_table::operator_impl<
+ 0U,
+ function<Config,
+ property<IsThrowing, HasStrongExceptGuarantee, Args...>>,
+ Args...> {
+
+ template <typename, typename>
+ friend class function;
+
+ template <std::size_t, typename, typename...>
+ friend class type_erasure::invocation_table::operator_impl;
+
+ using property_t = property<IsThrowing, HasStrongExceptGuarantee, Args...>;
+ using erasure_t =
+ type_erasure::erasure<Config::is_owning, Config, property_t>;
+
+ template <typename T>
+ using enable_if_can_accept_all_t =
+ std::enable_if_t<accepts_all<std::decay_t<T>, identity<Args...>>::value>;
+
+ template <typename Function, typename = void>
+ struct is_convertible_to_this : std::false_type {};
+ template <typename RightConfig>
+ struct is_convertible_to_this<
+ function<RightConfig, property_t>,
+ void_t<enable_if_copyable_correct_t<Config, RightConfig>,
+ enable_if_owning_correct_t<Config, RightConfig>>>
+ : std::true_type {};
+
+ template <typename T>
+ using enable_if_not_convertible_to_this =
+ std::enable_if_t<!is_convertible_to_this<std::decay_t<T>>::value>;
+
+ template <typename T>
+ using enable_if_owning_t =
+ std::enable_if_t<std::is_same<T, T>::value && Config::is_owning>;
+
+ template <typename T>
+ using assert_wrong_copy_assign_t =
+ typename assert_wrong_copy_assign<Config, std::decay_t<T>>::type;
+
+ template <typename T>
+ using assert_no_strong_except_guarantee_t =
+ typename assert_no_strong_except_guarantee<HasStrongExceptGuarantee,
+ std::decay_t<T>>::type;
+
+ erasure_t erasure_;
+
+public:
+ /// Default constructor which empty constructs the function
+ function() = default;
+ ~function() = default;
+
+ explicit FU2_DETAIL_CXX14_CONSTEXPR
+ function(function const& /*right*/) = default;
+ explicit FU2_DETAIL_CXX14_CONSTEXPR function(function&& /*right*/) = default;
+
+ /// Copy construction from another copyable function
+ template <typename RightConfig,
+ std::enable_if_t<RightConfig::is_copyable>* = nullptr,
+ enable_if_copyable_correct_t<Config, RightConfig>* = nullptr,
+ enable_if_owning_correct_t<Config, RightConfig>* = nullptr>
+ FU2_DETAIL_CXX14_CONSTEXPR
+ function(function<RightConfig, property_t> const& right)
+ : erasure_(right.erasure_) {
+ }
+
+ /// Move construction from another function
+ template <typename RightConfig,
+ enable_if_copyable_correct_t<Config, RightConfig>* = nullptr,
+ enable_if_owning_correct_t<Config, RightConfig>* = nullptr>
+ FU2_DETAIL_CXX14_CONSTEXPR function(function<RightConfig, property_t>&& right)
+ : erasure_(std::move(right.erasure_)) {
+ }
+
+ /// Construction from a callable object which overloads the `()` operator
+ template <typename T, //
+ enable_if_not_convertible_to_this<T>* = nullptr,
+ enable_if_can_accept_all_t<T>* = nullptr,
+ assert_wrong_copy_assign_t<T>* = nullptr,
+ assert_no_strong_except_guarantee_t<T>* = nullptr>
+ FU2_DETAIL_CXX14_CONSTEXPR function(T&& callable)
+ : erasure_(use_bool_op<unrefcv_t<T>>{}, std::forward<T>(callable)) {
+ }
+ template <typename T, typename Allocator, //
+ enable_if_not_convertible_to_this<T>* = nullptr,
+ enable_if_can_accept_all_t<T>* = nullptr,
+ enable_if_owning_t<T>* = nullptr,
+ assert_wrong_copy_assign_t<T>* = nullptr,
+ assert_no_strong_except_guarantee_t<T>* = nullptr>
+ FU2_DETAIL_CXX14_CONSTEXPR function(T&& callable, Allocator&& allocator)
+ : erasure_(use_bool_op<unrefcv_t<T>>{}, std::forward<T>(callable),
+ std::forward<Allocator>(allocator)) {
+ }
+
+ /// Empty constructs the function
+ FU2_DETAIL_CXX14_CONSTEXPR function(std::nullptr_t np) : erasure_(np) {
+ }
+
+ function& operator=(function const& /*right*/) = default;
+ function& operator=(function&& /*right*/) = default;
+
+ /// Copy assigning from another copyable function
+ template <typename RightConfig,
+ std::enable_if_t<RightConfig::is_copyable>* = nullptr,
+ enable_if_copyable_correct_t<Config, RightConfig>* = nullptr,
+ enable_if_owning_correct_t<Config, RightConfig>* = nullptr>
+ function& operator=(function<RightConfig, property_t> const& right) {
+ erasure_ = right.erasure_;
+ return *this;
+ }
+
+ /// Move assigning from another function
+ template <typename RightConfig,
+ enable_if_copyable_correct_t<Config, RightConfig>* = nullptr,
+ enable_if_owning_correct_t<Config, RightConfig>* = nullptr>
+ function& operator=(function<RightConfig, property_t>&& right) {
+ erasure_ = std::move(right.erasure_);
+ return *this;
+ }
+
+ /// Move assigning from a callable object
+ template <typename T, // ...
+ enable_if_not_convertible_to_this<T>* = nullptr,
+ enable_if_can_accept_all_t<T>* = nullptr,
+ assert_wrong_copy_assign_t<T>* = nullptr,
+ assert_no_strong_except_guarantee_t<T>* = nullptr>
+ function& operator=(T&& callable) {
+ erasure_.assign(use_bool_op<unrefcv_t<T>>{}, std::forward<T>(callable));
+ return *this;
+ }
+
+ /// Clears the function
+ function& operator=(std::nullptr_t np) {
+ erasure_ = np;
+ return *this;
+ }
+
+ /// Returns true when the function is empty
+ bool empty() const noexcept {
+ return erasure_.empty();
+ }
+
+ /// Returns true when the function isn't empty
+ explicit operator bool() const noexcept {
+ return !empty();
+ }
+
+ /// Assigns a new target with an optional allocator
+ template <typename T, typename Allocator = std::allocator<std::decay_t<T>>,
+ enable_if_not_convertible_to_this<T>* = nullptr,
+ enable_if_can_accept_all_t<T>* = nullptr,
+ assert_wrong_copy_assign_t<T>* = nullptr,
+ assert_no_strong_except_guarantee_t<T>* = nullptr>
+ void assign(T&& callable, Allocator&& allocator = Allocator{}) {
+ erasure_.assign(use_bool_op<unrefcv_t<T>>{}, std::forward<T>(callable),
+ std::forward<Allocator>(allocator));
+ }
+
+ /// Swaps this function with the given function
+ void swap(function& other) noexcept(HasStrongExceptGuarantee) {
+ if (&other == this) {
+ return;
+ }
+
+ function cache = std::move(other);
+ other = std::move(*this);
+ *this = std::move(cache);
+ }
+
+ /// Swaps the left function with the right one
+ friend void swap(function& left,
+ function& right) noexcept(HasStrongExceptGuarantee) {
+ left.swap(right);
+ }
+
+ /// Calls the wrapped callable object
+ using type_erasure::invocation_table::operator_impl<
+ 0U, function<Config, property_t>, Args...>::operator();
+};
+
+template <typename Config, typename Property>
+bool operator==(function<Config, Property> const& f, std::nullptr_t) {
+ return !bool(f);
+}
+
+template <typename Config, typename Property>
+bool operator!=(function<Config, Property> const& f, std::nullptr_t) {
+ return bool(f);
+}
+
+template <typename Config, typename Property>
+bool operator==(std::nullptr_t, function<Config, Property> const& f) {
+ return !bool(f);
+}
+
+template <typename Config, typename Property>
+bool operator!=(std::nullptr_t, function<Config, Property> const& f) {
+ return bool(f);
+}
+
+// Default intended object size of the function
+using object_size = std::integral_constant<std::size_t, 32U>;
+} // namespace detail
+} // namespace abi_400
+
+/// Can be passed to function_base as template argument which causes
+/// the internal small buffer to be sized according to the given size,
+/// and aligned with the given alignment.
+template <std::size_t Capacity,
+ std::size_t Alignment = alignof(std::max_align_t)>
+struct capacity_fixed {
+ static constexpr std::size_t capacity = Capacity;
+ static constexpr std::size_t alignment = Alignment;
+};
+
+/// Default capacity for small functor optimization
+struct capacity_default
+ : capacity_fixed<detail::object_size::value - (2 * sizeof(void*))> {};
+
+/// Can be passed to function_base as template argument which causes
+/// the internal small buffer to be removed from the callable wrapper.
+/// The owning function_base will then allocate memory for every object
+/// it applies a type erasure on.
+struct capacity_none : capacity_fixed<0UL> {};
+
+/// Can be passed to function_base as template argument which causes
+/// the internal small buffer to be sized such that it can hold
+/// the given object without allocating memory for an applied type erasure.
+template <typename T>
+struct capacity_can_hold {
+ static constexpr std::size_t capacity = sizeof(T);
+ static constexpr std::size_t alignment = alignof(T);
+};
+
+/// An adaptable function wrapper base for arbitrary functional types.
+///
+/// \tparam IsOwning Is true when the type erasure shall be owning the object.
+///
+/// \tparam IsCopyable Defines whether the function is copyable or not
+///
+/// \tparam Capacity Defines the internal capacity of the function
+/// for small functor optimization.
+/// The size of the whole function object will be the capacity
+/// plus the size of two pointers. If the capacity is zero,
+/// the size will increase through one additional pointer
+/// so the whole object has the size of 3 * sizeof(void*).
+/// The type which is passed to the Capacity template parameter
+/// shall provide a capacity and alignment member which
+/// looks like the following example:
+/// ```cpp
+/// struct my_capacity {
+/// static constexpr std::size_t capacity = sizeof(my_type);
+/// static constexpr std::size_t alignment = alignof(my_type);
+/// };
+/// ```
+///
+/// \tparam IsThrowing Defines whether the function throws an exception on
+/// empty function call, `std::abort` is called otherwise.
+///
+/// \tparam HasStrongExceptGuarantee Defines whether all objects satisfy the
+/// strong exception guarantees,
+/// which means the function type will satisfy
+/// the strong exception guarantees too.
+///
+/// \tparam Signatures Defines the signature of the callable wrapper
+///
+template <bool IsOwning, bool IsCopyable, typename Capacity, bool IsThrowing,
+ bool HasStrongExceptGuarantee, typename... Signatures>
+using function_base = detail::function<
+ detail::config<IsOwning, IsCopyable, Capacity>,
+ detail::property<IsThrowing, HasStrongExceptGuarantee, Signatures...>>;
+
+/// An owning copyable function wrapper for arbitrary callable types.
+template <typename... Signatures>
+using function = function_base<true, true, capacity_default, //
+ true, false, Signatures...>;
+
+/// An owning non copyable function wrapper for arbitrary callable types.
+template <typename... Signatures>
+using unique_function = function_base<true, false, capacity_default, //
+ true, false, Signatures...>;
+
+/// A non owning copyable function wrapper for arbitrary callable types.
+template <typename... Signatures>
+using function_view = function_base<false, true, capacity_default, //
+ true, false, Signatures...>;
+
+#if !defined(FU2_HAS_DISABLED_EXCEPTIONS)
+/// Exception type that is thrown when invoking empty function objects
+/// and exception support isn't disabled.
+///
+/// Exception support is enabled if
+/// the template parameter 'Throwing' is set to true (default).
+///
+/// This type will default to std::bad_function_call if the
+/// functional header is used, otherwise the library provides its own type.
+///
+/// You may disable the inclusion of the functional header
+/// through defining `FU2_WITH_NO_FUNCTIONAL_HEADER`.
+///
+using detail::type_erasure::invocation_table::bad_function_call;
+#endif
+
+/// Returns a callable object, which unifies all callable objects
+/// that were passed to this function.
+///
+/// ```cpp
+/// auto overloaded = fu2::overload([](std::true_type) { return true; },
+/// [](std::false_type) { return false; });
+/// ```
+///
+/// \param callables A pack of callable objects with arbitrary signatures.
+///
+/// \returns A callable object which exposes the
+///
+template <typename... T>
+constexpr auto overload(T&&... callables) {
+ return detail::overloading::overload(std::forward<T>(callables)...);
+}
+} // namespace fu2
+
+#undef FU2_DETAIL_EXPAND_QUALIFIERS
+#undef FU2_DETAIL_EXPAND_QUALIFIERS_NOEXCEPT
+#undef FU2_DETAIL_EXPAND_CV
+#undef FU2_DETAIL_EXPAND_CV_NOEXCEPT
+#undef FU2_DETAIL_UNREACHABLE_INTRINSIC
+#undef FU2_DETAIL_TRAP
+#undef FU2_DETAIL_CXX14_CONSTEXPR
+
+#endif // FU2_INCLUDED_FUNCTION2_HPP_
diff --git a/contrib/google-ced/CMakeLists.txt b/contrib/google-ced/CMakeLists.txt
new file mode 100644
index 0000000..668c635
--- /dev/null
+++ b/contrib/google-ced/CMakeLists.txt
@@ -0,0 +1,26 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+#
+# This library is derived from https://github.com/google/compact_enc_det
+# git id: 37529e628fbac2e4c0d4d8520be9db789f316c9e
+
+project(CED CXX)
+set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS 1 CACHE INTERNAL "No dev warnings")
+
+option(BUILD_SHARED_LIBS "Build shared libraries" OFF)
+
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wno-narrowing")
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+set(CED_LIBRARY_SOURCES
+ compact_enc_det.cc
+ compact_enc_det_hint_code.cc
+ util/encodings/encodings.cc
+ util/languages/languages.cc
+ ced_c.cc
+ )
+
+add_library(rspamd-ced STATIC ${CED_LIBRARY_SOURCES})
diff --git a/contrib/google-ced/LICENSE b/contrib/google-ced/LICENSE
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/contrib/google-ced/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/contrib/google-ced/ced_c.cc b/contrib/google-ced/ced_c.cc
new file mode 100644
index 0000000..5167121
--- /dev/null
+++ b/contrib/google-ced/ced_c.cc
@@ -0,0 +1,25 @@
+#include "ced_c.h"
+#include "compact_enc_det.h"
+
+const char* ced_encoding_detect(const char* text, int text_length,
+ const char* url_hint,
+ const char* http_charset_hint,
+ const char* meta_charset_hint,
+ const int encoding_hint,
+ CedTextCorpusType corpus_type, bool ignore_7bit_mail_encodings,
+ int* bytes_consumed, bool* is_reliable)
+{
+ CompactEncDet::TextCorpusType ct = CompactEncDet::NUM_CORPA;
+
+ ct = static_cast<CompactEncDet::TextCorpusType>(corpus_type);
+
+ auto enc = CompactEncDet::DetectEncoding(text, text_length, url_hint,
+ http_charset_hint, meta_charset_hint, encoding_hint, default_language(),
+ ct, ignore_7bit_mail_encodings, bytes_consumed, is_reliable);
+
+ if (IsValidEncoding(enc)) {
+ return MimeEncodingName(enc);
+ }
+
+ return nullptr;
+}
diff --git a/contrib/google-ced/ced_c.h b/contrib/google-ced/ced_c.h
new file mode 100644
index 0000000..c086f0c
--- /dev/null
+++ b/contrib/google-ced/ced_c.h
@@ -0,0 +1,32 @@
+#ifndef RSPAMD_CED_C_H
+#define RSPAMD_CED_C_H
+
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+enum CedTextCorpusType {
+ CED_WEB_CORPUS,
+ CED_XML_CORPUS,
+ CED_QUERY_CORPUS,
+ CED_EMAIL_CORPUS,
+ CED_NUM_CORPA,
+};
+
+/*
+ * XXX: Rspamd addition: it actually returns Mime format of the encoding
+ */
+const char *ced_encoding_detect (const char *text, int text_length,
+ const char *url_hint,
+ const char *http_charset_hint,
+ const char *meta_charset_hint,
+ const int encoding_hint,
+ enum CedTextCorpusType corpus_type,
+ bool ignore_7bit_mail_encodings,
+ int *bytes_consumed, bool *is_reliable);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/contrib/google-ced/compact_enc_det.cc b/contrib/google-ced/compact_enc_det.cc
new file mode 100644
index 0000000..c962b43
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det.cc
@@ -0,0 +1,5719 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "compact_enc_det.h"
+
+#include <math.h> // for sqrt
+#include <stddef.h> // for size_t
+#include <stdio.h> // for printf, fprintf, NULL, etc
+#include <stdlib.h> // for qsort
+#include <string.h> // for memset, memcpy, memcmp, etc
+#include <memory>
+#include <string> // for string, operator==, etc
+
+#include "compact_enc_det_hint_code.h"
+#include "util/string_util.h"
+#include "util/basictypes.h"
+#include "util/commandlineflags.h"
+#include "util/logging.h"
+
+using std::string;
+
+// TODO as of 2007.10.09:
+//
+// Consider font=TT-BHxxx as user-defined => binary
+// Demote GB18030 if no 8x3x pair
+// Map byte2 ascii punct to 0x60, digits to 0x7e, gets them into hires
+// Consider removing/ignoring bytes 01-1F to avoid crap pollution
+// Possibly boost declared encoding in robust scan
+// googlebot tiny files
+// look for ranges of encodings
+// consider tags just as > < within aligned block of 32
+// flag too few characters in postproc (Latin 6 problem)
+// Remove slow scan beyond 16KB
+// Consider removing kMostLikelyEncoding or cut it in half
+
+
+// A note on mixed encodings
+//
+// The most common encoding error on the web is a page containing a mixture of
+// CP-1252 and UTF-8. A less common encoding error is a third-party feed that
+// has been converted from CP-1252 to UTF-8 and then those bytes converted a
+// second time to UTF-8. CED originally attempted to detect these error cases
+// by using two synthetic encodings, UTF8CP1252 and UTF8UTF8. The intended
+// implementation was to start these just below CP1252 and UTF8 respectively in
+// overall liklihood, and allow 1252 and UTF8 to fall behind if mixtures are
+// found.
+//
+// The UTF8UTF8 encoding is a possible outcome from CED, but unfortunately the
+// UTF8CP1252 internal encoding was added late and not put into encodings.proto,
+// so at the final step it is mapped to UTF8UTF8 also. This was a bad idea and
+// is removed in this November 2011 CL.
+//
+// Mixed encoding detection never worked out as well as envisioned, so the
+// ced_allow_utf8utf8 flag normally disables all this.
+//
+// The effect is that CP-1252 and UTF-8 mixtures will usually be detected as
+// UTF8, and the inputconverter code for UTF8 normally will convert bare
+// CP-1252 bytes to UTF-8, instead of the less-helpful FFFD substitution. UTF-8
+// and double-UTF-8 mixtures will be detected as UTF-8, and the double
+// conversion will stand.
+//
+// However, it is occasionally useful to use CED to detect double-converted
+// UTF-8 coming from third-party data feeds, so they can be fixed at the source.
+// For this purpose, the UTF8UTF8 encoding remains available under the
+// ced_allow_utf8utf8 flag.
+//
+// When UTF8UTF8 is detected, the inputconverter code will undo the double
+// conversion, giving good text.
+
+// Norbert Runge has noted these words in CP1252 that are mistakenly identified
+// as UTF-8 because of the last pair of characters:
+// NESTLÉ® 0xC9 0xAE U+00C9 U+00AE C9AE = U+026E;SMALL LEZH
+// drauß\u2019 0xDF 0x92 U+00DF U+2019 DF92 = U+07D2;NKO LETTER N
+// Mutterschoß\u201c 0xDF 0x93 U+00DF U+201C DF93 = U+07D3;NKO LETTER BA
+// Schoß\u201c 0xDF 0x93 U+00DF U+201C
+// weiß\u201c 0xDF 0x93 U+00DF U+00AB
+// Schnellfuß\u201c 0xDF 0x93 U+00DF U+201C
+// süß« 0xDF 0xAB U+00DF U+00AB DFAB = U+07EB;NKO HIGH TONE
+// These four byte combinations now explicitly boost Latin1/CP1252.
+
+// And for reference, here are a couple of Portuguese spellings
+// that may be mistaken as double-byte encodings.
+// informações 0xE7 0xF5
+// traição 0xE7 0xE3
+
+
+static const char* kVersion = "2.2";
+
+DEFINE_bool(ced_allow_utf8utf8, false, "Allow the UTF8UTF8 encoding, "
+ "to handle mixtures of CP1252 "
+ "converted to UTF-8 zero, one, "
+ "or two times");
+DEFINE_int32(enc_detect_slow_max_kb, 16,
+ "Maximum number of Kbytes to examine for "
+ "7-bit-only (2022, Hz, UTF7) encoding detect. "
+ "You are unlikely to want to change this.");
+DEFINE_int32(enc_detect_fast_max_kb, 256,
+ "Maximum number of Kbytes to examine for encoding detect. "
+ "You are unlikely to want to change this.");
+
+DEFINE_int32(ced_reliable_difference, 300, "30 * Bits of minimum probablility "
+ "difference 1st - 2nd to be considered reliable \n"
+ " 2 corresponds to min 4x difference\n"
+ " 4 corresponds to min 16x difference\n"
+ " 8 corresponds to min 256x difference\n"
+ " 10 corresponds to min 1024x difference\n"
+ " 20 corresponds to min 1Mx difference.");
+
+// Text debug output options
+DEFINE_bool(enc_detect_summary, false,
+ "Print first 16 interesting pairs at exit.");
+DEFINE_bool(counts, false, "Count major-section usage");
+
+// PostScript debug output options
+DEFINE_bool(enc_detect_detail, false,
+ "Print PostScript of every update, to stderr.");
+DEFINE_bool(enc_detect_detail2, false,
+ "More PostScript detail of every update, to stderr.");
+DEFINE_bool(enc_detect_source, false, "Include source text in detail");
+// Encoding name must exactly match FIRST column of kI18NInfoByEncoding in
+// lang_enc.cc
+
+// Following flags are not in use. Replace them with constants to
+// avoid static initialization.
+
+//DEFINE_string(enc_detect_watch1, "", "Do detail2 about this encoding name.");
+//DEFINE_string(enc_detect_watch2, "", "Do detail2 about this encoding name.");
+
+static const char* const FLAGS_enc_detect_watch1 = "";
+static const char* const FLAGS_enc_detect_watch2 = "";
+
+// Only for experiments. Delete soon.
+DEFINE_bool(force127, false, "Force Latin1, Latin2, Latin7 based on trigrams");
+
+// Demo-mode/debugging experiment
+DEFINE_bool(demo_nodefault, false,
+ "Default to all equal; no boost for declared encoding.");
+DEFINE_bool(dirtsimple, false, "Just scan and count for all encodings");
+DEFINE_bool(ced_echo_input, false, "Echo ced input to stderr");
+
+
+static const int XDECILOG2 = 3; // Multiplier for log base 2 ** n/10
+static const int XLOG2 = 30; // Multiplier for log base 2 ** n
+
+static const int kFinalPruneDifference = 10 * XLOG2;
+ // Final bits of minimum
+ // probability difference 1st-nth
+ // to be pruned
+
+static const int kInititalPruneDifference = kFinalPruneDifference * 4;
+ // Initial bits of minimum
+ // probability difference 1st-nth
+ // to be pruned
+ //
+static const int kPruneDiffDecrement = kFinalPruneDifference;
+ // Decrements bits of minimum
+ // probability difference 1st-nth
+ // to be pruned
+
+static const int kSmallInitDiff = 2 * XLOG2; // bits of minimum
+ // probability difference, base to
+ // superset encodings
+
+static const int kBoostInitial = 20 * XLOG2; // bits of boost for
+ // initial byte patterns (BOM, 00)
+
+static const int kBadPairWhack = 20 * XLOG2; // bits of whack for
+ // one bad pair
+
+static const int kBoostOnePair = 20 * XLOG2; // bits of boost for
+ // one good pair in Hz, etc.
+
+static const int kGentleOnePair = 4 * XLOG2; // bits of boost for
+ // one good sequence
+ //
+static const int kGentlePairWhack = 2 * XLOG2; // bits of whack
+ // for ill-formed sequence
+
+static const int kGentlePairBoost = 2 * XLOG2; // bits of boost
+ // for well-formed sequence
+
+static const int kDeclaredEncBoost = 5 * XDECILOG2; // bits/10 of boost for
+ // best declared encoding per bigram
+
+static const int kBestEncBoost = 5 * XDECILOG2; // bits/10 of boost for
+ // best encoding per bigram
+
+static const int kTrigramBoost = 2 * XLOG2; // bits of boost for Latin127 tri
+
+static const int kMaxPairs = 48; // Max interesting pairs to look at
+ // If you change this,
+ // adjust *PruneDiff*
+
+static const int kPruneMask = 0x07; // Prune every 8 interesting pairs
+
+
+static const int kBestPairsCount = 16; // For first N pairs, do extra boost
+ // based on most likely encoding
+ // of pair over entire web
+
+static const int kDerateHintsBelow = 12; // If we have fewer than N bigrams,
+ // weaken the hints enough that
+ // unhinted encodings have a hope of
+ // rising to the top
+
+static const int kMinRescanLength = 800; // Don't bother rescanning for
+ // unreliable encoding if fewer
+ // than this many bytes unscanned.
+ // We will rescan at most last half
+ // of this.
+
+static const int kStrongBinary = 12; // Make F_BINARY the only encoding
+static const int kWeakerBinary = 4; // Make F_BINARY likely encoding
+
+// These are byte counts from front of file
+static const int kBinaryHardAsciiLimit = 6 * 1024; // Not binary if all ASCII
+static const int kBinarySoftAsciiLimit = 8 * 1024; // " if mostly ASCII
+
+// We try here to avoid having title text dominate the encoding detection,
+// for the not-infrequent error case of title in encoding1, body in encoding2:
+// we want to bias toward encoding2 winning.
+//
+// kMaxBigramsTagTitleText should be a multiple of 2, 3, and 4, so that we
+// rarely cut off mid-character in the original (not-yet-detected) encoding.
+// This matters most for UTF-8 two- and three-byte codes and for
+// Shift-JIS three-byte codes.
+static const int kMaxBigramsTagTitleText = 12; // Keep only some tag text
+static const int kWeightshiftForTagTitleText = 4; // Give text in tags, etc.
+ // 1/16 normal weight
+
+static const int kStrongPairs = 6; // Let reliable enc with this many
+ // pairs overcome missing hint
+
+enum CEDInternalFlags {
+ kCEDNone = 0, // The empty flag
+ kCEDRescanning = 1, // Do not further recurse
+ kCEDSlowscore = 2, // Do extra scoring
+ kCEDForceTags = 4, // Always examine text inside tags
+};
+
+// Forward declaration
+Encoding InternalDetectEncoding(
+ CEDInternalFlags flags, const char* text, int text_length,
+ const char* url_hint, const char* http_charset_hint,
+ const char* meta_charset_hint, const int encoding_hint,
+ const Language language_hint, // User interface lang
+ const CompactEncDet::TextCorpusType corpus_type,
+ bool ignore_7bit_mail_encodings, int* bytes_consumed, bool* is_reliable,
+ Encoding* second_best_enc);
+
+typedef struct {
+ const uint8* hires[4]; // Pointers to possible high-resolution bigram deltas
+ uint8 x_bar; // Average byte2 value
+ uint8 y_bar; // Average byte1 value
+ uint8 x_stddev; // Standard deviation of byte2 value
+ uint8 y_stddev; // Standard deviation of byte1 value
+ int so; // Scaling offset -- add to probabilities below
+ uint8 b1[256]; // Unigram probability for first byte of aligned bigram
+ uint8 b2[256]; // Unigram probability for second byte of aligned bigram
+ uint8 b12[256]; // Unigram probability for cross bytes of aligned bigram
+} UnigramEntry;
+
+//typedef struct {
+// uint8 b12[256*256]; // Bigram probability for aligned bigram
+//} FullBigramEntry;
+
+
+// Include all the postproc-generated tables here:
+// RankedEncoding
+// kMapToEncoding
+// unigram_table
+// kMostLIkelyEncoding
+// kTLDHintProbs
+// kCharsetHintProbs
+// HintEntry, kMaxTldKey kMaxTldVector, etc.
+// =============================================================================
+
+#include "compact_enc_det_generated_tables.h"
+
+
+#define F_ASCII F_Latin1 // "ASCII" is a misnomer, so this code uses "Latin1"
+
+#define F_BINARY F_X_BINARYENC // We are mid-update for name change
+#define F_UTF8UTF8 F_X_UTF8UTF8 // We are mid-update for name change
+#define F_BIG5_CP950 F_BIG5 // We are mid-update for name change
+#define F_Unicode F_UTF_16LE // We are mid-update for name change
+// =============================================================================
+
+// 7-bit encodings have at least one "interesting" byte value < 0x80
+// (00 0E 1B + ~)
+// JIS 2022-cn 2022-kr hz utf7
+// Unicode UTF-16 UTF-32
+// 8-bit encodings have no interesting byte values < 0x80
+static const uint32 kSevenBitActive = 0x00000001; // needs <80 to detect
+static const uint32 kUTF7Active = 0x00000002; // <80 and +
+static const uint32 kHzActive = 0x00000004; // <80 and ~
+static const uint32 kIso2022Active = 0x00000008; // <80 and 1B 0E 0F
+static const uint32 kUTF8Active = 0x00000010;
+static const uint32 kUTF8UTF8Active = 0x00000020;
+static const uint32 kUTF1632Active = 0x00000040; // <80 and 00
+static const uint32 kBinaryActive = 0x00000080; // <80 and 00
+static const uint32 kTwobyteCode = 0x00000100; // Needs 8xxx
+static const uint32 kIsIndicCode = 0x00000200; //
+static const uint32 kHighAlphaCode = 0x00000400; // full alphabet in 8x-Fx
+static const uint32 kHighAccentCode = 0x00000800; // accents in 8x-Fx
+static const uint32 kEUCJPActive = 0x00001000; // Have to mess with phase
+
+
+// Debug only. not thread safe
+static int encdet_used = 0;
+static int rescore_used = 0;
+static int rescan_used = 0;
+static int robust_used = 0;
+static int looking_used = 0;
+static int doing_used = 0;
+
+
+// For debugging only -- about 256B/entry times about 500 = 128KB
+// TODO: only allocate this if being used
+typedef struct {
+ int offset;
+ int best_enc; // Best ranked encoding for this bigram, or
+ // -1 for overhead entries
+ string label;
+ int detail_enc_prob[NUM_RANKEDENCODING];
+} DetailEntry;
+
+static int watch1_rankedenc = -1; // Debug. not threadsafe
+static int watch2_rankedenc = -1; // Debug. not threadsafe
+////static int next_detail_entry = 0; // Debug. not threadsafe
+////static DetailEntry details[kMaxPairs * 10]; // Allow 10 details per bigram
+// End For debugging only
+
+// Must match kTestPrintableAsciiTildePlus exit codes, minus one
+enum PairSet {AsciiPair = 0, OtherPair = 1, NUM_PAIR_SETS = 2};
+
+// The reasons for pruning
+enum PruneReason {PRUNE_NORMAL, PRUNE_SLOWEND, PRUNE_FINAL};
+
+static const char* kWhatSetName[] = {"Ascii", "Other"};
+
+
+// State for encodings that do shift-out/shift-in between one- and two-byte
+// regions (ISO-2022-xx, HZ)
+enum StateSoSi {SOSI_NONE, SOSI_ERROR, SOSI_ONEBYTE, SOSI_TWOBYTE};
+
+typedef struct {
+ const uint8* initial_src; // For calculating byte offsets
+ const uint8* limit_src; // Range of input source
+ const uint8* prior_src; // Source consumed by prior call to BoostPrune
+ const uint8* last_pair; // Last pair inserted into interesting_pairs
+
+ DetailEntry* debug_data; // Normally NULL. Ptr to debug data for
+ // FLAGS_enc_detect_detail PostScript data
+ int next_detail_entry; // Debug
+
+ bool done;
+ bool reliable;
+ bool hints_derated;
+ int declared_enc_1; // From http/meta hint
+ int declared_enc_2; // from http/meta hint
+ int prune_count; // Number of times we have pruned
+
+ int trigram_highwater_mark; // Byte offset of last trigram processing
+ bool looking_for_latin_trigrams; // True if we should test for doing
+ // Latin1/2/7 trigram processing
+ bool do_latin_trigrams; // True if we actually are scoring trigrams
+
+ // Miscellaneous state variables for difficult encodings
+ int binary_quadrants_count; // Number of four bigram quadrants seen:
+ // 0xxxxxxx0xxxxxxx 0xxxxxxx1xxxxxx
+ // 1xxxxxxx0xxxxxxx 1xxxxxxx1xxxxxx
+ int binary_8x4_count; // Number of 8x4 buckets seen:
+ uint32 binary_quadrants_seen; // Bit[i] set if bigram i.......i....... seen
+ uint32 binary_8x4_seen; // Bit[i] set if bigram iii.....ii...... seen
+ int utf7_starts; // Count of possible UTF-7 beginnings seen
+ int prior_utf7_offset; // Source consumed by prior UTF-7 string
+ int next_utf8_ministate; // Mini state for UTF-8 sequences
+ int utf8_minicount[6]; // Number of correct 2- 3- 4-byte seq, errors
+ int next_utf8utf8_ministate; // Mini state for UTF8UTF8 sequences
+ int utf8utf8_odd_byte; // UTF8UTF8 seq has odd number of bytes
+ int utf8utf8_minicount[6]; // Number of correct 2- 3- 4-byte seq, errors
+ StateSoSi next_2022_state; // Mini state for 2022 sequences
+ StateSoSi next_hz_state; // Mini state for HZ sequences
+ bool next_eucjp_oddphase; // Mini state for EUC-JP sequences
+ int byte32_count[8]; // Count of top 3 bits of byte1 of bigram
+ // 0x1x 2x3x 4x5x 6x7x 8x9x AxBx CxDx ExFx
+ uint32 active_special; // Bits showing which special cases are active
+
+ Encoding tld_hint; // Top TLD encoding or UNKNOWN
+ Encoding http_hint; // What the document says about itself or
+ Encoding meta_hint; // UNKNOWN_ENCODING. BOM is initial byte
+ Encoding bom_hint; // order mark for UTF-xx
+
+ // small cache of previous interesting bigrams
+ int next_prior_bigram;
+ int prior_bigram[4];
+ int prior_binary[1];
+
+ int top_rankedencoding; // Top two probabilities and families
+ int second_top_rankedencoding;
+ int top_prob;
+ int second_top_prob;
+ int prune_difference; // Prune things this much below the top prob
+ int rankedencoding_list_len; // Number of active encodings
+ int rankedencoding_list[NUM_RANKEDENCODING]; // List of active encodings
+ //
+ int enc_prob[NUM_RANKEDENCODING]; // Cumulative probability per enc
+ // This is where all the action is
+ int hint_prob[NUM_RANKEDENCODING]; // Initial hint probabilities
+ int hint_weight[NUM_RANKEDENCODING]; // Number of hints for this enc
+
+ // Two sets -- one for printable ASCII, one for the rest
+ int prior_interesting_pair[NUM_PAIR_SETS]; // Pairs consumed by prior call
+ int next_interesting_pair[NUM_PAIR_SETS]; // Next pair to write
+ char interesting_pairs[NUM_PAIR_SETS][kMaxPairs * 2]; // Two bytes per pair
+ int interesting_offsets[NUM_PAIR_SETS][kMaxPairs]; // Src offset of pair
+ int interesting_weightshift[NUM_PAIR_SETS][kMaxPairs]; // weightshift of pair
+} DetectEncodingState;
+
+
+// Record a debug event that changes probabilities
+void SetDetailsEncProb(DetectEncodingState* destatep,
+ int offset, int best_enc, const char* label) {
+ int next = destatep->next_detail_entry;
+ destatep->debug_data[next].offset = offset;
+ destatep->debug_data[next].best_enc = best_enc;
+ destatep->debug_data[next].label = label;
+ memcpy(&destatep->debug_data[next].detail_enc_prob,
+ &destatep->enc_prob,
+ sizeof(destatep->enc_prob));
+ ++destatep->next_detail_entry;
+}
+
+// Record a debug event that changes probabilities, copy offset
+void SetDetailsEncProbCopyOffset(DetectEncodingState* destatep,
+ int best_enc, const char* label) {
+ int next = destatep->next_detail_entry;
+ destatep->debug_data[next].offset = destatep->debug_data[next - 1].offset;
+ destatep->debug_data[next].best_enc = best_enc;
+ destatep->debug_data[next].label = label;
+ memcpy(&destatep->debug_data[next].detail_enc_prob,
+ &destatep->enc_prob,
+ sizeof(destatep->enc_prob));
+ ++destatep->next_detail_entry;
+}
+
+// Record a debug event that changes probs and has simple text label
+void SetDetailsEncLabel(DetectEncodingState* destatep, const char* label) {
+ int next = destatep->next_detail_entry;
+ destatep->debug_data[next].offset = destatep->debug_data[next - 1].offset;
+ destatep->debug_data[next].best_enc = -1;
+ destatep->debug_data[next].label = label;
+ memcpy(&destatep->debug_data[next].detail_enc_prob,
+ &destatep->enc_prob,
+ sizeof(destatep->enc_prob));
+ ++destatep->next_detail_entry;
+}
+
+// Record a debug event that is just a text label, no change in probs
+void SetDetailsLabel(DetectEncodingState* destatep, const char* label) {
+ int next = destatep->next_detail_entry;
+ destatep->debug_data[next].offset = destatep->debug_data[next - 1].offset;
+ destatep->debug_data[next].best_enc = -1;
+ destatep->debug_data[next].label = label;
+ memcpy(&destatep->debug_data[next].detail_enc_prob,
+ &destatep->debug_data[next - 1].detail_enc_prob,
+ sizeof(destatep->enc_prob));
+ ++destatep->next_detail_entry;
+}
+
+
+// Maps superset encodings to base, to see if 2 encodings are compatible
+// (Non-identity mappings are marked "-->" below.)
+static const Encoding kMapEncToBaseEncoding[] = {
+ ISO_8859_1, // 0: Teragram ASCII
+ ISO_8859_2, // 1: Teragram Latin2
+ ISO_8859_3, // 2: in BasisTech but not in Teragram
+ ISO_8859_4, // 3: Teragram Latin4
+ ISO_8859_5, // 4: Teragram ISO-8859-5
+ ISO_8859_6, // 5: Teragram Arabic
+ ISO_8859_7, // 6: Teragram Greek
+ MSFT_CP1255, // 7: Teragram Hebrew --> 36
+ ISO_8859_9, // 8: in BasisTech but not in Teragram
+ ISO_8859_10, // 9: in BasisTech but not in Teragram
+ JAPANESE_EUC_JP, // 10: Teragram EUC_JP
+ JAPANESE_SHIFT_JIS, // 11: Teragram SJS
+ JAPANESE_JIS, // 12: Teragram JIS
+ CHINESE_BIG5, // 13: Teragram BIG5
+ CHINESE_GB, // 14: Teragram GB
+ CHINESE_EUC_CN, // 15: Teragram EUC-CN
+ KOREAN_EUC_KR, // 16: Teragram KSC
+ UNICODE, // 17: Teragram Unicode
+ CHINESE_EUC_CN, // 18: Teragram EUC --> 15
+ CHINESE_EUC_CN, // 19: Teragram CNS --> 15
+ CHINESE_BIG5, // 20: Teragram BIG5_CP950 --> 13
+ JAPANESE_SHIFT_JIS, // 21: Teragram CP932 --> 11
+ UTF8, // 22
+ UNKNOWN_ENCODING, // 23
+ ISO_8859_1, // 24: ISO_8859_1 with all characters <= 127 --> 0
+ RUSSIAN_KOI8_R, // 25: Teragram KOI8R
+ RUSSIAN_CP1251, // 26: Teragram CP1251
+ ISO_8859_1, // 27: CP1252 aka MSFT euro ascii --> 0
+ RUSSIAN_KOI8_RU, // 28: CP21866 aka KOI8_RU, used for Ukrainian
+ MSFT_CP1250, // 29: CP1250 aka MSFT eastern european
+ ISO_8859_1, // 30: aka ISO_8859_0 aka ISO_8859_1 euroized --> 0
+ ISO_8859_9, // 31: used for Turkish
+ ISO_8859_13, // 32: used in Baltic countries --> 43
+ ISO_8859_11, // 33: aka TIS-620, used for Thai
+ ISO_8859_11, // 34: used for Thai --> 33
+ MSFT_CP1256, // 35: used for Arabic
+ MSFT_CP1255, // 36: Logical Hebrew Microsoft
+ MSFT_CP1255, // 37: Iso Hebrew Logical --> 36
+ MSFT_CP1255, // 38: Iso Hebrew Visual --> 36
+ CZECH_CP852, // 39
+ ISO_8859_2, // 40: aka ISO_IR_139 aka KOI8_CS --> 1
+ MSFT_CP1253, // 41: used for Greek, but NOT a superset of 8859-7
+ RUSSIAN_CP866, // 42
+ ISO_8859_13, // 43
+ ISO_2022_KR, // 44
+ CHINESE_GB, // 45 GBK --> 14
+ CHINESE_GB, // 46 GB18030 --> 14
+ CHINESE_BIG5, // 47 BIG5_HKSCS --> 13
+ ISO_2022_KR, // 48 ISO_2022_CN --> 44
+ TSCII, // 49 Indic encoding
+ TAMIL_MONO, // 50 Indic encoding - Tamil
+ TAMIL_BI, // 51 Indic encoding - Tamil
+ JAGRAN, // 52 Indic encoding - Devanagari
+ MACINTOSH_ROMAN, // 53
+ UTF7, // 54
+ BHASKAR, // 55 Indic encoding - Devanagari
+ HTCHANAKYA, // 56 Indic encoding - Devanagari
+ UTF16BE, // 57
+ UTF16LE, // 58
+ UTF32BE, // 59
+ UTF32LE, // 60
+ BINARYENC, // 61
+ HZ_GB_2312, // 62
+ UTF8UTF8, // 63
+ TAM_ELANGO, // 64 Elango - Tamil
+ TAM_LTTMBARANI, // 65 Barani - Tamil
+ TAM_SHREE, // 66 Shree - Tamil
+ TAM_TBOOMIS, // 67 TBoomis - Tamil
+ TAM_TMNEWS, // 68 TMNews - Tamil
+ TAM_WEBTAMIL, // 69 Webtamil - Tamil
+ KDDI_SHIFT_JIS, // 70 KDDI Shift_JIS
+ DOCOMO_SHIFT_JIS, // 71 DoCoMo Shift_JIS
+ SOFTBANK_SHIFT_JIS, // 72 SoftBank Shift_JIS
+ KDDI_ISO_2022_JP, // 73 KDDI ISO-2022-JP
+ SOFTBANK_ISO_2022_JP, // 74 SOFTBANK ISO-2022-JP
+};
+
+COMPILE_ASSERT(arraysize(kMapEncToBaseEncoding) == NUM_ENCODINGS,
+ kMapEncToBaseEncoding_has_incorrect_size);
+
+// Maps base encodings to 0, supersets to 1+, undesired to -1
+// (Non-identity mappings are marked "-->" below.)
+static const int kMapEncToSuperLevel[] = {
+ 0, // 0: Teragram ASCII
+ 0, // 1: Teragram Latin2
+ 0, // 2: in BasisTech but not in Teragram
+ 0, // 3: Teragram Latin4
+ 0, // 4: Teragram ISO-8859-5
+ 0, // 5: Teragram Arabic
+ 0, // 6: Teragram Greek
+ 0, // 7: Teragram Hebrew
+ 0, // 8: in BasisTech but not in Teragram
+ 0, // 9: in BasisTech but not in Teragram
+ 0, // 10: Teragram EUC_JP
+ 0, // 11: Teragram SJS
+ 0, // 12: Teragram JIS
+ 0, // 13: Teragram BIG5
+ 0, // 14: Teragram GB
+ 0, // 15: Teragram EUC-CN
+ 0, // 16: Teragram KSC
+ 0, // 17: Teragram Unicode
+ -1, // 18: Teragram EUC --> 15
+ -1, // 19: Teragram CNS --> 15
+ 1, // 20: Teragram BIG5_CP950 --> 13
+ 1, // 21: Teragram CP932 --> 11
+ 0, // 22
+ -1, // 23
+ -1, // 24: ISO_8859_1 with all characters <= 127 --> 0
+ 0, // 25: Teragram KOI8R
+ 0, // 26: Teragram CP1251
+ 1, // 27: CP1252 aka MSFT euro ascii --> 0
+ 0, // 28: CP21866 aka KOI8_RU, used for Ukrainian
+ 0, // 29: CP1250 aka MSFT eastern european
+ 1, // 30: aka ISO_8859_0 aka ISO_8859_1 euroized --> 0
+ 0, // 31: used for Turkish
+ 1, // 32: used in Baltic countries --> 43
+ 0, // 33: aka TIS-620, used for Thai
+ 1, // 34: used for Thai --> 33
+ 0, // 35: used for Arabic
+ 0, // 36: Logical Hebrew Microsoft
+ -1, // 37: Iso Hebrew Logical --> 36
+ -1, // 38: Iso Hebrew Visual --> 7
+ 0, // 39
+ 1, // 40: aka ISO_IR_139 aka KOI8_CS --> 1
+ 0, // 41: used for Greek, NOT superset of 8859-7
+ 0, // 42
+ 0, // 43
+ 0, // 44
+ 1, // 45 GBK --> 14
+ 1, // 46 GB18030 --> 14
+ 1, // 47 BIG5_HKSCS --> 13
+ 1, // 48 ISO_2022_CN --> 44
+ 0, // 49 Indic encoding
+ 0, // 50 Indic encoding - Tamil
+ 0, // 51 Indic encoding - Tamil
+ 0, // 52 Indic encoding - Devanagari
+ 0, // 53
+ 0, // 54
+ 0, // 55 Indic encoding - Devanagari
+ 0, // 56 Indic encoding - Devanagari
+ 0, // 57
+ 0, // 58
+ 0, // 59
+ 0, // 60
+ 0, // 61
+ 0, // 62
+ 2, // 63
+ 0, 0, 0, 0, 0, 0, // add six more Tamil
+ 0, 0, 0, 0, 0, // add five encodings with emoji
+};
+
+COMPILE_ASSERT(arraysize(kMapEncToSuperLevel) == NUM_ENCODINGS,
+ kMapEncToSuperLevel_has_incorrect_size);
+
+
+
+// Subscripted by Encoding enum value
+static const uint32 kSpecialMask[] = {
+ kHighAccentCode, // 0
+ kHighAccentCode,
+ kHighAccentCode,
+ kHighAccentCode,
+ kHighAlphaCode, // 4
+ kHighAlphaCode,
+ kHighAlphaCode,
+ kHighAlphaCode,
+ kHighAccentCode,
+ kHighAccentCode,
+
+ kTwobyteCode + kEUCJPActive, // 10 euc-jp
+ kTwobyteCode,
+ kSevenBitActive + kIso2022Active, // jis
+ kTwobyteCode,
+ kTwobyteCode,
+ kTwobyteCode,
+ kTwobyteCode,
+ kSevenBitActive + kUTF1632Active, // Unicode
+ kTwobyteCode,
+ kTwobyteCode,
+
+ kTwobyteCode, // 20
+ kTwobyteCode,
+ kUTF8Active, // UTF-8
+ 0,
+ 0,
+ kHighAlphaCode, // 25
+ kHighAlphaCode,
+ kHighAccentCode,
+ kHighAlphaCode,
+ kHighAccentCode,
+
+ kHighAccentCode, // 30
+ kHighAccentCode,
+ kHighAccentCode,
+ kHighAlphaCode,
+ kHighAlphaCode,
+ kHighAlphaCode, // 35
+ kHighAlphaCode,
+ kHighAlphaCode,
+ kHighAlphaCode,
+ 0,
+
+ 0, // 40
+ kHighAlphaCode,
+ kHighAlphaCode,
+ kHighAccentCode,
+ kSevenBitActive + kIso2022Active, // 2022-kr
+ kTwobyteCode,
+ kTwobyteCode,
+ kTwobyteCode,
+ kSevenBitActive + kIso2022Active, // 2022-cn
+ kHighAlphaCode + kIsIndicCode, // 49 TSCII
+
+ kHighAlphaCode + kIsIndicCode, // 50 TAMIL_MONO
+ kHighAlphaCode + kIsIndicCode, // 51 TAMIL_BI
+ kHighAlphaCode + kIsIndicCode, // 52 JAGRAN
+ kHighAccentCode, // 53 MACINTOSH_ROMAN
+ kSevenBitActive + kUTF7Active, // 54 UTF-7
+ kHighAlphaCode + kIsIndicCode, // 55 BHASKAR Indic encoding - Devanagari
+ kHighAlphaCode + kIsIndicCode, // 56 HTCHANAKYA Indic encoding - Devanagari
+ kSevenBitActive + kUTF1632Active, // 57 UTF16BE
+ kSevenBitActive + kUTF1632Active, // 58 UTF16LE
+ kSevenBitActive + kUTF1632Active, // 59 UTF32BE
+ kSevenBitActive + kUTF1632Active, // 60 UTF32LE
+
+ kSevenBitActive + kBinaryActive, // 61 BINARYENC
+ kSevenBitActive + kHzActive, // 62 HZ_GB_2312
+ kHighAccentCode + kUTF8Active + kUTF8UTF8Active, // 63 UTF8UTF8
+ kHighAlphaCode + kIsIndicCode, // 64 Elango - Tamil
+ kHighAlphaCode + kIsIndicCode, // 65 Barani - Tamil
+ kHighAlphaCode + kIsIndicCode, // 66 Shree - Tamil
+ kHighAlphaCode + kIsIndicCode, // 67 TBoomis - Tamil
+ kHighAlphaCode + kIsIndicCode, // 68 TMNews - Tamil
+ kHighAlphaCode + kIsIndicCode, // 69 Webtamil - Tamil
+ kTwobyteCode, // 70 KDDI Shift_JIS
+ kTwobyteCode, // 71 DoCoMo Shift_JIS
+ kTwobyteCode, // 72 SoftBank Shift_JIS
+ kSevenBitActive + kIso2022Active, // 73 KDDI-ISO-2022-JP
+ kSevenBitActive + kIso2022Active, // 74 SOFTBANK-ISO-2022-JP
+};
+
+COMPILE_ASSERT(arraysize(kSpecialMask) == NUM_ENCODINGS,
+ kSpecialMask_has_incorrect_size);
+
+
+/***
+ kHighAlphaCode -- full alphabet in 8x-Fx range, not just accents
+
+ ISO_8859_5, // 4: Teragram ISO-8859-5 Cyrl UL bd
+ RUSSIAN_CP1251, // 26: Teragram CP1251 UL cdef
+ RUSSIAN_KOI8_R, // 25: Teragram KOI8R LU cdef
+ RUSSIAN_KOI8_RU, // 28: CP21866 aka KOI8_RU, LU cdef
+ RUSSIAN_CP866, // 42 89ae
+
+ ISO_8859_6, // 5: Teragram Arabic nocase cde
+ MSFT_CP1256, // 35: used for Arabic nocase cde
+
+ ISO_8859_7, // 6: Teragram Greek UL cdef
+ MSFT_CP1253, // 41: used for Greek UL cdef
+
+ ISO_8859_8, // 7: Teragram Hebrew nocase ef
+ MSFT_CP1255, // 36: Logical Hebrew Microsoft nocase ef
+ ISO_8859_8_I, // 37: Iso Hebrew Logical nocase ef
+ HEBREW_VISUAL, // 38: Iso Hebrew Visual nocase ef
+
+ ISO_8859_11, // 33: aka TIS-620, used for Thai nocase abcde
+ MSFT_CP874, // 34: used for Thai nocase abcde
+
+ TSCII, // 49 8-f
+ TAMIL_MONO, // 50
+ TAMIL_BI, // 51
+ JAGRAN, // 52
+ BHASKAR, // 55 Indic encoding - Devanagari
+ HTCHANAKYA, // 56 Indic encoding - Devanagari
+***/
+
+// We can scan bytes using this at about 500 MB/sec 2.8GHz P4
+// Slow scan uses this, stopping on NUL ESC SO SI bad C0 and + ~
+// We allow FF, 0x0C, here because it gives a better result for old
+// Ascii text formatted for a TTY
+// non-zero exits scan loop -- 1 for printable ASCII, 2 otherwise
+static const char kTestPrintableAsciiTildePlus[256] = {
+ 2,2,2,2,2,2,2,2, 2,0,0,2,0,0,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,1,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,1,2,
+
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+};
+
+// We can scan bytes using this at about 550 MB/sec 2.8GHz P4
+// Slow scan uses this, stopping on NUL ESC SO SI and bad C0
+// after Hz and UTF7 are pruned away
+// We allow Form Feed, 0x0C, here
+static const char kTestPrintableAscii[256] = {
+ 2,2,2,2,2,2,2,2, 2,0,0,2,0,0,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,2,
+
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+};
+
+// Used in first-four-byte testing
+static const char kIsPrintableAscii[256] = {
+ 0,0,0,0,0,0,0,0, 0,1,1,0,0,1,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,0,
+
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+
+static const signed char kBase64Value[256] = {
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,62,-1,-1,-1,63,
+ 52,53,54,55,56,57,58,59, 60,61,-1,-1,-1,-1,-1,-1,
+
+ -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,
+ 15,16,17,18,19,20,21,22, 23,24,25,-1,-1,-1,-1,-1,
+ -1,26,27,28,29,30,31,32, 33,34,35,36,37,38,39,40,
+ 41,42,43,44,45,46,47,48, 49,50,51,-1,-1,-1,-1,-1,
+
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
+};
+
+
+// Subscripted by <state, byte/16>
+// Accepts Cx->8x Dx->8x Ex->8x->8x Fx->8x->8x->8x
+//
+// Fixed Problem: GB has sequences like B2DB B8D6 BDE1 B9B9
+// which we can mis-parse as an error byte followed by good UTF-8:
+// B2 DBB8 D6BD E1B9B9
+// To counteract this, we now require an ASCII7 byte to resync out
+// of the error state
+// Next problem: good UTF-8 with bad byte
+// efbc a012 eea4 bee7 b280 c2b7
+// efbca0 12 eea4be e7b280 c2b7
+// ^^ bad byte
+// fix: change state0 byte 1x to be don't-care
+//
+// Short UTF-8 ending in ASCII7 byte should resync immediately:
+// E0 20 E0 A6 AA should give one error and resync at 2nd E0
+//
+static const char kMiniUTF8State[8][16] = {
+ {0,0,0,0,0,0,0,0, 7,7,7,7,1,1,2,4,}, // [0] start char (allow cr/lf/ht)
+ {0,7,0,0,0,0,0,0, 0,0,0,0,7,7,7,7,}, // [1] continue 1 of 2
+ {0,7,0,0,0,0,0,0, 3,3,3,3,7,7,7,7,}, // [2] continue 1 of 3
+ {0,7,0,0,0,0,0,0, 0,0,0,0,7,7,7,7,}, // [3] continue 2 of 3
+ {0,7,0,0,0,0,0,0, 5,5,5,5,7,7,7,7,}, // [4] continue 1 of 4
+ {0,7,0,0,0,0,0,0, 6,6,6,6,7,7,7,7,}, // [5] continue 2 of 4
+ {0,7,0,0,0,0,0,0, 0,0,0,0,7,7,7,7,}, // [6] continue 3 of 4
+ {0,7,0,0,0,0,0,0, 7,7,7,7,7,7,7,7,}, // [7] error, soak up continues,
+ // ONLY resync after Ascii char
+ // then restart
+};
+// Counter to increment: 0-don'tcare 1-error 2-good_2B 3-good_3B 4-good_4B
+static const char kMiniUTF8Count[8][16] = {
+ {0,0,0,0,0,0,0,0, 1,1,1,1,0,0,0,0,}, // [0] start char (allow cr/lf/ht)
+ {1,1,1,1,1,1,1,1, 2,2,2,2,1,1,1,1,}, // [1] continue 1 of 2
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [2] continue 1 of 3
+ {1,1,1,1,1,1,1,1, 3,3,3,3,1,1,1,1,}, // [3] continue 2 of 3
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [4] continue 1 of 4
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [5] continue 2 of 4
+ {1,1,1,1,1,1,1,1, 4,4,4,4,1,1,1,1,}, // [6] continue 3 of 4
+ {0,1,0,0,0,0,0,0, 1,1,1,1,1,1,1,1,}, // [7] error, soak up continues,
+ // then restart
+};
+
+// Subscripted by <state, f(byte1) + g(byte2)>
+// where f(x)= E2->4, Cx->8 and C3->12 and 0 otherwise
+// and g(x) = (x >> 4) & 3 8x->0 9x->1 Ax->2 Bx->3 Cx->0, etc.
+// (no checking for illegal bytes)
+// Here are example patterns of CP1252 converted to UTF-8 0/1/2 times. We want
+// to detect two, so we can back-convert to one.
+// zero one two pattern
+// ---- ------ ---------------- -----------------
+// 81 C281 C382C281 C3->8x->C2->xx
+// 98 CB9C C38BC593 C3->8x->C5->xx
+// C3 C383 C383C692 C3->8x->C6->xx
+// C8 C388 C383CB86 C3->8x->CB->xx
+// 83 C692 C386E28099 C3->8x->E2->xx->8x
+// 80 E282AC C3A2E2809AC2AC C3->A2->E2->xx->xx->Cx->xx
+// 92 E28099 C3A2E282ACE284A2 C3->A2->E2->xx->xx->E2->xx->xx
+//
+// We also want to detect bare-byte extra UTF-8 conversions:
+// zero one two pattern
+// ---- ------ ---------------- -----------------
+// C3 C3 C383 C3->8x->C2->xx
+// D3 D3 C393 C3->9x->C2->xx->C2->xx
+// E3 E3 C3A3 C3->Ax->C2->xx->C2->xx->C2->xx
+// F3 F3 C3B2 C3->Bx->C2->xx->C2->xx->C2->xx->C2->xx
+//
+
+/**
+CP1252 => UTF8 => UTF8UTF8
+80 => E282AC => C3A2E2809AC2AC
+81 => C281 => C382C281
+82 => E2809A => C3A2E282ACC5A1
+83 => C692 => C386E28099
+84 => E2809E => C3A2E282ACC5BE
+85 => E280A6 => C3A2E282ACC2A6
+86 => E280A0 => C3A2E282ACC2A0
+87 => E280A1 => C3A2E282ACC2A1
+88 => CB86 => C38BE280A0
+89 => E280B0 => C3A2E282ACC2B0
+8A => C5A0 => C385C2A0
+8B => E280B9 => C3A2E282ACC2B9
+8C => C592 => C385E28099
+8D => C28D => C382C28D
+8E => C5BD => C385C2BD
+8F => C28F => C382C28F
+90 => C290 => C382C290
+91 => E28098 => C3A2E282ACCB9C
+92 => E28099 => C3A2E282ACE284A2
+93 => E2809C => C3A2E282ACC593
+94 => E2809D => C3A2E282ACC29D
+95 => E280A2 => C3A2E282ACC2A2
+96 => E28093 => C3A2E282ACE2809C
+97 => E28094 => C3A2E282ACE2809D
+98 => CB9C => C38BC593
+99 => E284A2 => C3A2E2809EC2A2
+9A => C5A1 => C385C2A1
+9B => E280BA => C3A2E282ACC2BA
+9C => C593 => C385E2809C
+9D => C29D => C382C29D
+9E => C5BE => C385C2BE
+9F => C5B8 => C385C2B8
+A0 => C2A0 => C382C2A0
+A1 => C2A1 => C382C2A1
+A2 => C2A2 => C382C2A2
+A3 => C2A3 => C382C2A3
+A4 => C2A4 => C382C2A4
+A5 => C2A5 => C382C2A5
+A6 => C2A6 => C382C2A6
+A7 => C2A7 => C382C2A7
+A8 => C2A8 => C382C2A8
+A9 => C2A9 => C382C2A9
+AA => C2AA => C382C2AA
+AB => C2AB => C382C2AB
+AC => C2AC => C382C2AC
+AD => C2AD => C382C2AD
+AE => C2AE => C382C2AE
+AF => C2AF => C382C2AF
+B0 => C2B0 => C382C2B0
+B1 => C2B1 => C382C2B1
+B2 => C2B2 => C382C2B2
+B3 => C2B3 => C382C2B3
+B4 => C2B4 => C382C2B4
+B5 => C2B5 => C382C2B5
+B6 => C2B6 => C382C2B6
+B7 => C2B7 => C382C2B7
+B8 => C2B8 => C382C2B8
+B9 => C2B9 => C382C2B9
+BA => C2BA => C382C2BA
+BB => C2BB => C382C2BB
+BC => C2BC => C382C2BC
+BD => C2BD => C382C2BD
+BE => C2BE => C382C2BE
+BF => C2BF => C382C2BF
+C0 => C380 => C383E282AC
+C1 => C381 => C383C281
+C2 => C382 => C383E2809A
+C3 => C383 => C383C692
+C4 => C384 => C383E2809E
+C5 => C385 => C383E280A6
+C6 => C386 => C383E280A0
+C7 => C387 => C383E280A1
+C8 => C388 => C383CB86
+C9 => C389 => C383E280B0
+CA => C38A => C383C5A0
+CB => C38B => C383E280B9
+CC => C38C => C383C592
+CD => C38D => C383C28D
+CE => C38E => C383C5BD
+CF => C38F => C383C28F
+D0 => C390 => C383C290
+D1 => C391 => C383E28098
+D2 => C392 => C383E28099
+D3 => C393 => C383E2809C
+D4 => C394 => C383E2809D
+D5 => C395 => C383E280A2
+D6 => C396 => C383E28093
+D7 => C397 => C383E28094
+D8 => C398 => C383CB9C
+D9 => C399 => C383E284A2
+DA => C39A => C383C5A1
+DB => C39B => C383E280BA
+DC => C39C => C383C593
+DD => C39D => C383C29D
+DE => C39E => C383C5BE
+DF => C39F => C383C5B8
+E0 => C3A0 => C383C2A0
+E1 => C3A1 => C383C2A1
+E2 => C3A2 => C383C2A2
+E3 => C3A3 => C383C2A3
+E4 => C3A4 => C383C2A4
+E5 => C3A5 => C383C2A5
+E6 => C3A6 => C383C2A6
+E7 => C3A7 => C383C2A7
+E8 => C3A8 => C383C2A8
+E9 => C3A9 => C383C2A9
+EA => C3AA => C383C2AA
+EB => C3AB => C383C2AB
+EC => C3AC => C383C2AC
+ED => C3AD => C383C2AD
+EE => C3AE => C383C2AE
+EF => C3AF => C383C2AF
+F0 => C3B0 => C383C2B0
+F1 => C3B1 => C383C2B1
+F2 => C3B2 => C383C2B2
+F3 => C3B3 => C383C2B3
+F4 => C3B4 => C383C2B4
+F5 => C3B5 => C383C2B5
+F6 => C3B6 => C383C2B6
+F7 => C3B7 => C383C2B7
+F8 => C3B8 => C383C2B8
+F9 => C3B9 => C383C2B9
+FA => C3BA => C383C2BA
+FB => C3BB => C383C2BB
+FC => C3BC => C383C2BC
+FD => C3BD => C383C2BD
+FE => C3BE => C383C2BE
+FF => C3BF => C383C2BF
+**/
+
+// Subscripted by <state, f(byte1) + g(byte2)>
+// where f(x)= E2->4, C2/5/6/B->8 and C3->12 and 0 otherwise
+// and g(x) = (x >> 4) & 3 8x->0 9x->1 Ax->2 Bx->3 Cx->0, etc.
+
+// 81 C281 C382C281 C3->8x->C2->xx
+// 98 CB9C C38BC593 C3->8x->C5->xx
+// C3 C383 C383C692 C3->8x->C6->xx
+// C8 C388 C383CB86 C3->8x->CB->xx
+// [0] [2] [0]
+// 83 C692 C386E28099 C3->8x->E2->xx->xx
+// odd_byte=0 [0] [2] [0+] odd_byte flipped
+// odd_byte=1 [0+] [2] [0] [0] odd_byte unflipped
+// 80 E282AC C3A2E2809AC2AC C3->A2->E2->xx->xx->Cx->xx
+// odd_byte=0 [0] [3] [4] [0+]
+// odd_byte=1 [0+] [3] [4] [4] [0]
+// 92 E28099 C3A2E282ACE284A2 C3->A2->E2->xx->xx->E2->xx->xx
+// odd_byte=0 [0] [3] [4] [0] [0]
+// odd_byte=1 [0+] [3] [4] [4] [0+]
+//
+// When an E2xxxx sequence is encountered, we absorb the two bytes E2xx and flip
+// the odd_byte state. If that goes from 0 to 1, the next pair is offset up
+// by one byte, picking up the two bytes just after E2xxxx. If odd_byte goes
+// from 1 to 0, the next two bytes picked up are the two bytes xxxx of E2xxxx.
+// These are absorbed with no error in state 0 or state 4
+//
+// C3 C3 C383 C3->8x->C2->xx
+// D3 D3 C393 C3->9x->C2->xx->C2->xx
+// E3 E3 C3A3 C3->Ax->C2->xx->C2->xx->C2->xx
+// F3 F3 C3B2 C3->Bx->C2->xx->C2->xx->C2->xx->C2->xx
+// Counter3 for Fx Ex sequences is incremented at last C2
+
+static const char kMiniUTF8UTF8State[8][16] = {
+ // xxxx E2xx CXxx C3xx
+ // 8 9 a b 8 9 a b 8 9 a b
+ {0,0,0,0,1,1,1,1, 1,1,1,1,2,2,3,5,}, // [0] looking for C38x/C3Ax/2020/8x8x, or err
+ {0,0,0,0,1,1,1,1, 1,1,1,1,2,2,3,5,}, // [1] error, back to looking
+ {1,1,1,1,0,0,0,0, 0,0,0,0,1,1,1,1,}, // [2] C38x looking for CXxx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {1,1,1,1,4,4,4,4, 7,7,7,7,1,1,1,1,}, // [3] C3Ax looking for E2xx or C2xxC2xx
+ // + + + + // E2xxxx flips odd_byte
+ {4,4,4,4,0,0,0,0, 0,0,0,0,1,1,1,1,}, // [4] C3AxE2xx-- looking for C2xx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {1,1,1,1,1,1,1,1, 6,6,6,6,1,1,1,1,}, // [5] C3Bx -- looking for C2xxC2xxC2xx
+ {1,1,1,1,1,1,1,1, 7,7,7,7,1,1,1,1,}, // [6] C3Bx -- looking for C2xxC2xx
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [7] C3Bx -- looking for C2xx
+};
+// Counter to increment: 0-don'tcare 1-error 2-good_2B 3-good_3B 4-good_4B
+static const char kMiniUTF8UTF8Count[8][16] = {
+ // xxxx E2xx C2Xx C3xx
+ // 8 9 a b 8 9 a b 8 9 a b
+ {0,0,0,0,1,1,1,1, 1,1,1,1,0,0,0,0,}, // [0] looking for C38x/C3Ax/2020/8x8x, or err
+ {0,0,0,0,1,1,1,1, 1,1,1,1,0,0,0,0,}, // [1] error, back to looking
+ {1,1,1,1,3,3,3,3, 2,2,2,2,1,1,1,1,}, // [2] C38x looking for CXxx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {1,1,1,1,0,0,0,0, 0,0,0,0,1,1,1,1,}, // [3] C3Ax looking for E2xx
+ // + + + + // E2xxxx flips odd_byte
+ {1,1,1,1,4,4,4,4, 4,4,4,4,1,1,1,1,}, // [4] C3AxE2xx-- looking for C2xx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [5] C3Bx -- looking for C2xxC2xxC2xx
+ {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [6] C3Bx -- looking for C2xxC2xx
+ {1,1,1,1,1,1,1,1, 3,3,3,3,1,1,1,1,}, // [7] C3Bx -- looking for C2xx
+};
+
+static const char kMiniUTF8UTF8Odd[8][16] = {
+ // xxxx E2xx C2Xx C3xx
+ // 8 9 a b 8 9 a b 8 9 a b
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [0] looking for C38x/C3Ax/2020/8x8x, or err
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [1] error, back to looking
+ {0,0,0,0,1,1,1,1, 0,0,0,0,0,0,0,0,}, // [2] C38x looking for CXxx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {0,0,0,0,1,1,1,1, 0,0,0,0,0,0,0,0,}, // [3] C3Ax looking for E2xx
+ // + + + + // E2xxxx flips odd_byte
+ {0,0,0,0,1,1,1,1, 0,0,0,0,0,0,0,0,}, // [4] C3AxE2xx-- looking for C2xx/E2xxxx
+ // + + + + // E2xxxx flips odd_byte
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [5] C3Bx -- looking for C2xxC2xxC2xx
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [6] C3Bx -- looking for C2xxC2xx
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [7] C3Bx -- looking for C2xx
+};
+
+// Turn a pair of bytes into the subscript for UTF8UTF8 tables above
+int UTF88Sub(char s0, char s1) {
+ int sub = (s1 >> 4) & 0x03;
+ uint8 u0 = static_cast<uint8>(s0);
+ if (u0 == 0xc3) {
+ sub += 12;
+ } else if ((u0 & 0xf0) == 0xc0) {
+ if ((u0 == 0xc2) || (u0 == 0xc5) || (u0 == 0xc6) || (u0 == 0xcb)) {
+ sub += 8;
+ }
+ } else if (u0 == 0xe2) {
+ sub += 4;
+ }
+ return sub;
+}
+
+
+
+
+
+// Default probability for an encoding rankedencoding
+// Based on a scan of 55M web pages
+// These values are 255 - log base 2**1/10 (occurrences / total)
+// Large values are most likely. This the reverse of some Google code
+// 255 = 1.0, 245 = 1/2, 235 = 1/4, 15 = 1/2**24, 0 = 0 (< 1/50M)
+//
+// TODO change this to be per encoding, not permuted
+//
+
+
+// Support function for unit test program
+// Return ranked encoding corresponding to enc
+// (also exported to compact_enc_det_text.cc)
+int CompactEncDet::BackmapEncodingToRankedEncoding(Encoding enc) {
+ for (int i = 0; i < NUM_RANKEDENCODING; ++i) {
+ if (kMapToEncoding[i] == enc) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+
+string DecodeActive(uint32 active) {
+ string temp("");
+ if (active & kBinaryActive) {
+ temp.append("Binary ");
+ }
+ if (active & kUTF1632Active) {
+ temp.append("UTF1632 ");
+ }
+ if (active & kUTF8UTF8Active) {
+ temp.append("UTF8UTF8 ");
+ }
+ if (active & kUTF8Active) {
+ temp.append("UTF8 ");
+ }
+ if (active & kIso2022Active) {
+ temp.append("Iso2022 ");
+ }
+ if (active & kHzActive) {
+ temp.append("Hz ");
+ }
+ if (active & kUTF7Active) {
+ temp.append("UTF7A ");
+ }
+ if (active & kSevenBitActive) {
+ temp.append("SevenBit ");
+ }
+ if (active & kIsIndicCode) {
+ temp.append("Indic ");
+ }
+ if (active & kHighAlphaCode) {
+ temp.append("HighAlpha ");
+ }
+ if (active & kHighAccentCode) {
+ temp.append("HighAccent ");
+ }
+ if (active & kEUCJPActive) {
+ temp.append("EUCJP ");
+ }
+ return temp;
+}
+
+static inline bool SevenBitEncoding(int enc) {
+ return ((kSpecialMask[enc] & kSevenBitActive) != 0);
+}
+static inline bool TwoByteEncoding(int enc) {
+ return ((kSpecialMask[enc] & kTwobyteCode) != 0);
+}
+static inline bool IndicEncoding(int enc) {
+ return ((kSpecialMask[enc] & kIsIndicCode) != 0);
+}
+static inline bool HighAlphaEncoding(int enc) {
+ return ((kSpecialMask[enc] & kHighAlphaCode) != 0);
+}
+static inline bool HighAccentEncoding(int enc) {
+ return ((kSpecialMask[enc] & kHighAccentCode) != 0);
+}
+
+
+static inline bool AnyActive(DetectEncodingState* destatep) {
+ return (destatep->active_special != 0);
+}
+static inline bool SevenBitActive(DetectEncodingState* destatep) {
+ return (destatep->active_special & kSevenBitActive) != 0;
+}
+static inline bool HzActive(DetectEncodingState* destatep) {
+ return (destatep->active_special & kHzActive) != 0;
+}
+static inline bool Iso2022Active(DetectEncodingState* destatep) {
+ return (destatep->active_special & kIso2022Active) != 0;
+}
+static inline bool UTF8Active(DetectEncodingState* destatep) {
+ return (destatep->active_special & kUTF8Active) != 0;
+}
+static inline bool UTF8UTF8Active(DetectEncodingState* destatep) {
+ return (destatep->active_special & kUTF8UTF8Active) != 0;
+}
+static inline bool UTF1632Active(DetectEncodingState* destatep) {
+ return (destatep->active_special & kUTF1632Active) != 0;
+}
+static inline bool BinaryActive(DetectEncodingState* destatep) {
+ return (destatep->active_special & kBinaryActive) != 0;
+}
+static inline bool UTF7OrHzActive(DetectEncodingState* destatep) {
+ return (destatep->active_special & (kHzActive + kUTF7Active)) != 0;
+}
+static inline bool EUCJPActive(DetectEncodingState* destatep) {
+ return ((destatep->active_special & kEUCJPActive) != 0);
+}
+static inline bool OtherActive(DetectEncodingState* destatep) {
+ return (destatep->active_special & (kIso2022Active + kBinaryActive +
+ kUTF8Active + kUTF8UTF8Active +
+ kUTF1632Active + kEUCJPActive)) != 0;
+}
+
+
+static inline bool CEDFlagRescanning(CEDInternalFlags flags) {
+ return (flags & kCEDRescanning) != 0;
+}
+
+static inline bool CEDFlagForceTags(CEDInternalFlags flags) {
+ return (flags & kCEDForceTags) != 0;
+}
+
+
+static inline int maxint(int a, int b) {return (a > b) ? a : b;}
+static inline int minint(int a, int b) {return (a < b) ? a : b;}
+
+static inline const char* MyRankedEncName(int r_enc) {
+ return MyEncodingName(kMapToEncoding[r_enc]);
+}
+
+
+// Only for debugging. not thread safe
+static const int kPsSourceWidth = 32;
+static int pssourcenext = 0; // debug only. not threadsafe. dump only >= this
+static int pssourcewidth = 0; // debug only.
+static char* pssource_mark_buffer = NULL;
+int next_do_src_line;
+int do_src_offset[16];
+
+
+void PsSourceInit(int len) {
+ pssourcenext = 0;
+ pssourcewidth = len;
+ delete[] pssource_mark_buffer;
+ // Allocate 2 Ascii characters per input byte
+ pssource_mark_buffer = new char[(pssourcewidth * 2) + 8]; // 8 = overscan
+ memset(pssource_mark_buffer, ' ', pssourcewidth * 2);
+ memset(pssource_mark_buffer + (pssourcewidth * 2), '\0', 8);
+
+ next_do_src_line = 0;
+ memset(do_src_offset, 0, sizeof(do_src_offset));
+}
+
+void PsSourceFinish() {
+ // Print preceding mark buffer
+ int j = (pssourcewidth * 2) - 1;
+ while ((0 <= j) && (pssource_mark_buffer[j] == ' ')) {--j;} // trim
+ pssource_mark_buffer[j + 1] = '\0';
+ fprintf(stderr, "( %s) do-src\n", pssource_mark_buffer);
+ memset(pssource_mark_buffer, ' ', pssourcewidth * 2);
+ memset(pssource_mark_buffer + (pssourcewidth * 2), '\0', 8);
+
+ delete[] pssource_mark_buffer;
+ pssource_mark_buffer = NULL;
+}
+
+// Dump aligned len bytes src... if not already dumped
+void PsSource(const uint8* src, const uint8* isrc, const uint8* srclimit) {
+ int offset = src - isrc;
+ offset -= (offset % pssourcewidth); // round down to multiple of len bytes
+ if (offset < pssourcenext) {
+ return;
+ }
+ pssourcenext = offset + pssourcewidth; // Min offset for next dump
+
+ // Print preceding mark buffer
+ int j = (pssourcewidth * 2) - 1;
+ while ((0 <= j) && (pssource_mark_buffer[j] == ' ')) {--j;} // trim
+ pssource_mark_buffer[j + 1] = '\0';
+ fprintf(stderr, "( %s) do-src\n", pssource_mark_buffer);
+ memset(pssource_mark_buffer, ' ', pssourcewidth * 2);
+ memset(pssource_mark_buffer + (pssourcewidth * 2), '\0', 8);
+
+ // Print source bytes
+ const uint8* src_aligned = isrc + offset;
+ int length = srclimit - src_aligned;
+ length = minint(pssourcewidth, length);
+
+ fprintf(stderr, "(%05x ", offset);
+ for (int i = 0; i < length; ++i) {
+ char c = src_aligned[i];
+ if (c == '\n') {c = ' ';}
+ if (c == '\r') {c = ' ';}
+ if (c == '\t') {c = ' ';}
+ if (c == '(') {
+ fprintf(stderr, "%s", "\\( ");
+ } else if (c == ')') {
+ fprintf(stderr, "%s", "\\) ");
+ } else if (c == '\\') {
+ fprintf(stderr, "%s", "\\\\ ");
+ } else if ((0x20 <= c) && (c <= 0x7e)) {
+ fprintf(stderr, "%c ", c);
+ } else {
+ fprintf(stderr, "%02x", c);
+ }
+ }
+ fprintf(stderr, ") do-src\n");
+ // Remember which source offsets are where, mod 16
+ do_src_offset[next_do_src_line & 0x0f] = offset;
+ ++next_do_src_line;
+}
+
+// Mark bytes in just-previous source bytes
+void PsMark(const uint8* src, int len, const uint8* isrc, int weightshift) {
+ int offset = src - isrc;
+ offset = (offset % pssourcewidth); // mod len bytes
+ char mark = (weightshift == 0) ? '-' : 'x';
+
+ pssource_mark_buffer[(offset * 2)] = '=';
+ pssource_mark_buffer[(offset * 2) + 1] = '=';
+ for (int i = 1; i < len; ++i) {
+ pssource_mark_buffer[(offset + i) * 2] = mark;
+ pssource_mark_buffer[((offset + i) * 2) + 1] = mark;
+ }
+}
+
+
+// Highlight trigram bytes in just-previous source bytes
+// Unfortunately, we have to skip back N lines since source was printed for
+// up to 8 bigrams before we get here. Match on src+1 to handle 0/31 better
+void PsHighlight(const uint8* src, const uint8* isrc, int trigram_val, int n) {
+ int offset = (src + 1) - isrc;
+ int offset32 = (offset % pssourcewidth); // mod len bytes
+ offset -= offset32; // round down to multiple of len bytes
+
+ for (int i = 1; i <= 16; ++i) {
+ if (do_src_offset[(next_do_src_line - i) & 0x0f] == offset) {
+ fprintf(stderr, "%d %d %d do-highlight%d\n",
+ i, offset32 - 1, trigram_val, n);
+ break;
+ }
+ }
+}
+
+
+void InitDetectEncodingState(DetectEncodingState* destatep) {
+ destatep->initial_src = NULL; // Filled in by caller
+ destatep->limit_src = NULL;
+ destatep->prior_src = NULL;
+ destatep->last_pair = NULL;
+
+ destatep->debug_data = NULL;
+ destatep->next_detail_entry = 0;
+
+ destatep->done = false;
+ destatep->reliable = false;
+ destatep->hints_derated = false;
+ //destatep->declared_enc_1 init in ApplyHints
+ //destatep->declared_enc_2 init in ApplyHints
+ destatep->prune_count = 0;
+
+ destatep->trigram_highwater_mark = 0;
+ destatep->looking_for_latin_trigrams = false;
+ destatep->do_latin_trigrams = false;
+
+ // Miscellaneous state variables for difficult encodings
+ destatep->binary_quadrants_count = 0;
+ destatep->binary_8x4_count = 0;
+ destatep->binary_quadrants_seen = 0;
+ destatep->binary_8x4_seen = 0;
+ destatep->utf7_starts = 0;
+ destatep->prior_utf7_offset = 0;
+ destatep->next_utf8_ministate = 0;
+ for (int i = 0; i < 6; i++) {destatep->utf8_minicount[i] = 0;}
+ destatep->next_utf8utf8_ministate = 0;
+ destatep->utf8utf8_odd_byte = 0;
+ for (int i = 0; i < 6; i++) {destatep->utf8utf8_minicount[i] = 0;}
+ destatep->next_2022_state = SOSI_NONE;
+ destatep->next_hz_state = SOSI_NONE;
+ destatep->next_eucjp_oddphase = false;
+ for (int i = 0; i < 8; i++) {destatep->byte32_count[i] = 0;}
+ destatep->active_special = 0xffffffff;
+ destatep->tld_hint = UNKNOWN_ENCODING;
+ destatep->http_hint = UNKNOWN_ENCODING;
+ destatep->meta_hint = UNKNOWN_ENCODING;
+ destatep->bom_hint = UNKNOWN_ENCODING;
+ destatep->top_rankedencoding = 0; // ASCII [seven-bit] is the default
+ destatep->second_top_rankedencoding = 0; // ASCII [seven-bit] is the default
+ destatep->top_prob = -1;
+ destatep->second_top_prob = -1;
+ // This is wide for first pruning, shrinks for 2nd and later
+ destatep->prune_difference = kInititalPruneDifference;
+
+ destatep->next_prior_bigram = 0;
+ destatep->prior_bigram[0] = -1;
+ destatep->prior_bigram[1] = -1;
+ destatep->prior_bigram[2] = -1;
+ destatep->prior_bigram[3] = -1;
+
+ destatep->prior_binary[0] = -1;
+
+ // Initialize with all but Indic encodings, which we never detect
+ int k = 0;
+ for (int rankedencoding = 0;
+ rankedencoding < NUM_RANKEDENCODING;
+ rankedencoding++) {
+ Encoding enc = kMapToEncoding[rankedencoding];
+ if (!IndicEncoding(enc)) {
+ destatep->rankedencoding_list[k++] = rankedencoding;
+ }
+ }
+ destatep->rankedencoding_list_len = k;
+
+ // This is where all the action is
+ memset(destatep->enc_prob, 0, sizeof(destatep->enc_prob));
+
+ memset(destatep->hint_prob, 0, sizeof(destatep->hint_prob));
+ memset(destatep->hint_weight, 0, sizeof(destatep->hint_weight));
+
+ destatep->prior_interesting_pair[AsciiPair] = 0;
+ destatep->prior_interesting_pair[OtherPair] = 0;
+ destatep->next_interesting_pair[AsciiPair] = 0;
+ destatep->next_interesting_pair[OtherPair] = 0;
+ // interesting_pairs/offsets/weightshifts not initialized; no need
+}
+
+// Probability strings are uint8, with zeros removed via simple run-length:
+// (<skip-take byte> <data bytes>)*
+// skip-take:
+// 00 end
+// x0 skip 16 x locations, take 0 data values
+// xy skip x locations, take y data values
+// Multiply all the incoming values by 3 to account for 3x unigram sums
+//
+// {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x35,
+// 0x01,0xc2,0x10,0x41,0xfe,0x71,0xba,0x00,}}, // "wind1255"
+//
+// Weight is 0..100 percent
+//
+// Returns subscript of largest (most probable) value
+//
+
+
+// {{0x6e,0x6c,0x5f,0x5f, 0x05,0xb2,0xae,0xa0,0x32,0xa1,0x36,0x31,0x42,0x39,0x3b,0x33,0x45,0x11,0x6f,0x00,}}, // "nl__"
+// // ASCII-7-bit=178 Latin1=174 UTF8=160 GB=50 CP1252=161 BIG5=49 Latin2=66 CP1251=57 CP1256=59 CP1250=51 Latin5=69 ISO-8859-15=111 [top ASCII-7-bit]
+int ApplyCompressedProb(const char* iprob, int len,
+ int weight, DetectEncodingState* destatep) {
+ int* dst = &destatep->enc_prob[0];
+ int* dst2 = &destatep->hint_weight[0];
+ const uint8* prob = reinterpret_cast<const uint8*>(iprob);
+ const uint8* problimit = prob + len;
+
+ int largest = -1;
+ int subscript_of_largest = 0;
+
+ // Continue with first byte and subsequent ones
+ while (prob < problimit) {
+ int skiptake = *prob++;
+ int skip = (skiptake & 0xf0) >> 4;
+ int take = skiptake & 0x0f;
+ if (skiptake == 00) {
+ break;
+ } else if (take == 0) {
+ dst += (skip << 4);
+ dst2 += (skip << 4);
+ } else {
+ dst += skip; // Normal case
+ dst2 += skip; // Normal case
+ for (int i = 0; i < take; i++) {
+ int enc = static_cast<int>(dst - &destatep->enc_prob[0]) + i;
+ if (largest < prob[i]) {
+ largest = prob[i];
+ subscript_of_largest = enc;
+ }
+
+ int increment = prob[i] * 3; // The actual increment
+
+ // Do maximum of previous hints plus this new one
+ if (weight > 0) {
+ increment = (increment * weight) / 100;
+ dst[i] = maxint(dst[i], increment);
+ dst2[i] = 1; // New total weight
+ }
+ }
+ prob += take;
+ dst += take;
+ dst2 += take;
+ }
+ }
+ return subscript_of_largest;
+}
+
+
+// Returns subscript of largest (most probable) value [for unit test]
+int TopCompressedProb(const char* iprob, int len) {
+ const uint8* prob = reinterpret_cast<const uint8*>(iprob);
+ const uint8* problimit = prob + len;
+ int next_prob_sub = 0;
+ int topprob = 0;
+ int toprankenc = 0;
+
+ while (prob < problimit) {
+ int skiptake = *prob++;
+ int skip = (skiptake & 0xf0) >> 4;
+ int take = skiptake & 0x0f;
+ if (skiptake == 0) {
+ break;
+ } else if (take == 0) {
+ next_prob_sub += (skip << 4);
+ } else {
+ next_prob_sub += skip; // Normal case
+ for (int i = 0; i < take; i++) {
+ if (topprob < prob[i]) {
+ topprob = prob[i];
+ toprankenc = next_prob_sub + i;
+ }
+ }
+ prob += take;
+ next_prob_sub += take;
+ }
+ }
+ return toprankenc;
+}
+
+
+// Find subscript of matching key in first 8 bytes of sorted hint array, or -1
+int HintBinaryLookup8(const HintEntry* hintprobs, int hintprobssize,
+ const char* norm_key) {
+ // Key is always in range [lo..hi)
+ int lo = 0;
+ int hi = hintprobssize;
+ while (lo < hi) {
+ int mid = (lo + hi) >> 1;
+ int comp = memcmp(&hintprobs[mid].key_prob[0], norm_key, 8);
+ if (comp < 0) {
+ lo = mid + 1;
+ } else if (comp > 0) {
+ hi = mid;
+ } else {
+ return mid;
+ }
+ }
+ return -1;
+}
+
+// Find subscript of matching key in first 4 bytes of sorted hint array, or -1
+int HintBinaryLookup4(const HintEntry* hintprobs, int hintprobssize,
+ const char* norm_key) {
+ // Key is always in range [lo..hi)
+ int lo = 0;
+ int hi = hintprobssize;
+ while (lo < hi) {
+ int mid = (lo + hi) >> 1;
+ int comp = memcmp(&hintprobs[mid].key_prob[0], norm_key, 4);
+ if (comp < 0) {
+ lo = mid + 1;
+ } else if (comp > 0) {
+ hi = mid;
+ } else {
+ return mid;
+ }
+ }
+ return -1;
+}
+
+static inline void Boost(DetectEncodingState* destatep, int r_enc, int boost) {
+ destatep->enc_prob[r_enc] += boost;
+}
+
+static inline void Whack(DetectEncodingState* destatep, int r_enc, int whack) {
+ destatep->enc_prob[r_enc] -= whack;
+}
+
+// Apply initial probability hint based on top level domain name
+// Weight is 0..100 percent
+// Return 1 if name match found
+int ApplyTldHint(const char* url_tld_hint, int weight,
+ DetectEncodingState* destatep) {
+ if (url_tld_hint[0] == '~') {
+ return 0;
+ }
+ string normalized_tld = MakeChar4(string(url_tld_hint));
+ int n = HintBinaryLookup4(kTLDHintProbs, kTLDHintProbsSize,
+ normalized_tld.c_str());
+ if (n >= 0) {
+ // TLD is four bytes, probability table is ~12 bytes
+ int best_sub = ApplyCompressedProb((const char *)&kTLDHintProbs[n].key_prob[kMaxTldKey],
+ kMaxTldVector, weight, destatep);
+ // Never boost ASCII7; do CP1252 instead
+ if (best_sub == F_ASCII_7_bit) {best_sub = F_CP1252;}
+ destatep->declared_enc_1 = best_sub;
+ if (destatep->debug_data != NULL) {
+ // Show TLD hint
+ SetDetailsEncProb(destatep, 0, best_sub, url_tld_hint);
+ }
+ return 1;
+ }
+ return 0;
+}
+
+// Apply initial probability hint based on charset= name
+// Weight is 0..100 percent
+// Return 1 if name match found
+int ApplyCharsetHint(const char* charset_hint, int weight,
+ DetectEncodingState* destatep) {
+ if (charset_hint[0] == '~') {
+ return 0;
+ }
+ string normalized_charset = MakeChar44(string(charset_hint));
+ int n = HintBinaryLookup8(kCharsetHintProbs, kCharsetHintProbsSize,
+ normalized_charset.c_str());
+ if (n >= 0) {
+ // Charset is eight bytes, probability table is ~eight bytes
+ int best_sub = ApplyCompressedProb((const char *)&kCharsetHintProbs[n].key_prob[kMaxCharsetKey],
+ kMaxCharsetVector, weight, destatep);
+ // Never boost ASCII7; do CP1252 instead
+ if (best_sub == F_ASCII_7_bit) {best_sub = F_CP1252;}
+ destatep->declared_enc_1 = best_sub;
+
+ // If first explicitly declared charset is confusable with Latin1/1252, put
+ // both declared forms in declared_enc_*, displacing Latin1/1252.
+ // This avoids a bit of Latin1 creep.
+ // Also boost the declared encoding and its pair
+ // TODO: This should all be folded into postproc-enc-detect.cc
+ if ((destatep->http_hint == UNKNOWN_ENCODING) &&
+ (destatep->meta_hint == UNKNOWN_ENCODING)) {
+ // This is the first charset=hint
+ switch (best_sub) {
+ case F_Latin2: // 8859-2 Latin2, east euro
+ destatep->declared_enc_2 = F_CP1250;
+ Boost(destatep, F_Latin2, kGentleOnePair);
+ Boost(destatep, F_CP1250, kGentleOnePair);
+ break;
+ case F_CP1250:
+ destatep->declared_enc_2 = F_Latin2;
+ Boost(destatep, F_Latin2, kGentleOnePair);
+ Boost(destatep, F_CP1250, kGentleOnePair);
+ break;
+
+ case F_Latin3: // 8859-3 Latin3, south euro, Esperanto
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+ Boost(destatep, F_Latin3, kGentleOnePair);
+ break;
+
+ case F_Latin4: // 8859-4 Latin4, north euro
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+ Boost(destatep, F_Latin4, kGentleOnePair);
+ break;
+
+ case F_ISO_8859_5: // 8859-5 Cyrillic
+ destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost 1251
+ Boost(destatep, F_ISO_8859_5, kGentleOnePair); // (too different)
+ break;
+ case F_CP1251:
+ destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost -5
+ Boost(destatep, F_CP1251, kGentleOnePair); // (too different)
+ break;
+
+ case F_Arabic: // 8859-6 Arabic
+ destatep->declared_enc_2 = F_CP1256;
+ Boost(destatep, F_Arabic, kGentleOnePair);
+ Boost(destatep, F_CP1256, kGentleOnePair);
+ break;
+ case F_CP1256:
+ destatep->declared_enc_2 = F_Arabic;
+ Boost(destatep, F_Arabic, kGentleOnePair);
+ Boost(destatep, F_CP1256, kGentleOnePair);
+ break;
+
+ case F_Greek: // 8859-7 Greek
+ destatep->declared_enc_2 = F_CP1253;
+ Boost(destatep, F_Greek, kGentleOnePair);
+ Boost(destatep, F_CP1253, kGentleOnePair);
+ break;
+ case F_CP1253:
+ destatep->declared_enc_2 = F_Greek;
+ Boost(destatep, F_Greek, kGentleOnePair);
+ Boost(destatep, F_CP1253, kGentleOnePair);
+ break;
+
+ case F_Hebrew: // 8859-8 Hebrew
+ destatep->declared_enc_2 = F_CP1255;
+ Boost(destatep, F_Hebrew, kGentleOnePair);
+ Boost(destatep, F_CP1255, kGentleOnePair);
+ break;
+ case F_CP1255:
+ destatep->declared_enc_2 = F_Hebrew;
+ Boost(destatep, F_Hebrew, kGentleOnePair);
+ Boost(destatep, F_CP1255, kGentleOnePair);
+ break;
+
+ case F_Latin5: // 8859-9 Latin5, Turkish
+ destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost 1254
+ Boost(destatep, F_Latin5, kGentleOnePair); // (too different)
+ break;
+ case F_CP1254:
+ destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost Latin5
+ Boost(destatep, F_CP1254, kGentleOnePair); // (too different)
+ break;
+
+ case F_Latin6: // 8859-10 Latin6, Nordic
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+ Boost(destatep, F_Latin6, kGentleOnePair);
+ break;
+
+ case F_ISO_8859_11: // 8859-11 Thai,
+ destatep->declared_enc_2 = F_CP874;
+ Boost(destatep, F_ISO_8859_11, kGentleOnePair);
+ Boost(destatep, F_CP874, kGentleOnePair);
+ break;
+ case F_CP874:
+ destatep->declared_enc_2 = F_ISO_8859_11;
+ Boost(destatep, F_ISO_8859_11, kGentleOnePair);
+ Boost(destatep, F_CP874, kGentleOnePair);
+ break;
+
+ case F_ISO_8859_13: // 8859-13 Latin7, Baltic
+ destatep->declared_enc_2 = F_CP1257;
+ Boost(destatep, F_ISO_8859_13, kGentleOnePair);
+ Boost(destatep, F_CP1257, kGentleOnePair);
+ break;
+ case F_CP1257:
+ destatep->declared_enc_2 = F_ISO_8859_13;
+ Boost(destatep, F_ISO_8859_13, kGentleOnePair);
+ Boost(destatep, F_CP1257, kGentleOnePair);
+ break;
+
+ case F_ISO_8859_15: // 8859-15 Latin9, Latin0, Euro-ized Latin1
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+ Boost(destatep, F_ISO_8859_15, kGentleOnePair);
+ break;
+
+
+ // Greek all-caps is confusable with KOI8x all-lower and Hebrew.
+ // This turns some Greek documents into Cyrillic, etc. by mistake.
+ // Greek and Hebrew are boosted explicitly above; do KOI8x here.
+ // Boosting the declared encodingmakes it harder for the wrong one to
+ // creep up.
+ case F_KOI8R:
+ Boost(destatep, F_KOI8R, kGentleOnePair);
+ break;
+ case F_KOI8U:
+ Boost(destatep, F_KOI8U, kGentleOnePair);
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ if (destatep->debug_data != NULL) {
+ // Show charset hint
+ SetDetailsEncProb(destatep, 0, best_sub, charset_hint);
+ }
+
+ //
+ // Some fix-ups for the declared encodings
+ //
+
+ // If non-UTF8, non-Latin1/1252 encoding declared, disable UTF8 combos
+ // TODO: This should all be folded into postproc-enc-detect.cc
+ if ((best_sub != F_UTF8) &&
+ (best_sub != F_Latin1) &&
+ (best_sub != F_CP1252)) {
+ Whack(destatep, F_UTF8UTF8, kBadPairWhack * 4); // demote
+ }
+
+ // Latin2 and CP1250 differ in the overlap part, such as B1 or B9
+ // The initial probabilites for charset=Latin2 explicitly put CP1250
+ // down twice as far as normal, and vice versa. This is done in
+ // postproc-enc-detect.cc
+
+ // If charset=user-defined, treat as Binary --
+ // we can safely only do low ASCII, might be Indic
+ if (normalized_charset.substr(0,4) == "user") {
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ }
+
+ return 1;
+ }
+ return 0;
+}
+
+// Apply initial probability hint based on caller-supplied encoding
+// Negative hint whacks ~encoding, non-negative boosts encoding
+//
+// Negative hints are an experiment to see if they might be useful.
+// Not operator used instead of unary minus to allow specifying not-zero
+int ApplyEncodingHint(const int encoding_hint, int weight,
+ DetectEncodingState* destatep) {
+ Encoding enc_hint = static_cast<Encoding>((encoding_hint < 0) ?
+ ~encoding_hint : encoding_hint);
+ // Map to the right internal subscript
+ int rankedenc_hint = CompactEncDet::BackmapEncodingToRankedEncoding(enc_hint);
+
+ // I'm not sure how strong this hint should be. Weight 100% = 1 bigram
+ int increment = (kBoostOnePair * weight) / 100;
+
+ if (encoding_hint < 0) {
+ destatep->enc_prob[rankedenc_hint] -= increment;
+ } else {
+ destatep->enc_prob[rankedenc_hint] += increment;
+ }
+
+ if (destatep->debug_data != NULL) {
+ // Show encoding hint
+ SetDetailsEncProb(destatep, 0, -1, MyEncodingName(enc_hint));
+ }
+ return 1;
+}
+
+// Apply initial probability hint based on user interface language
+// Weight is 0..100 percent
+// Return 1 if name match found
+int ApplyUILanguageHint(const Language language_hint,
+ int weight, DetectEncodingState* destatep) {
+ if (language_hint == UNKNOWN_LANGUAGE) {
+ return 0;
+ }
+ string normalized_lang = MakeChar8(LanguageName(language_hint));
+ int n = HintBinaryLookup8(kLangHintProbs, kLangHintProbsSize,
+ normalized_lang.c_str());
+ if (n >= 0) {
+ // Language is eight bytes, probability table is ~eight bytes
+ int best_sub = ApplyCompressedProb((const char *)&kLangHintProbs[n].key_prob[kMaxLangKey],
+ kMaxLangVector, weight, destatep);
+ // Never boost ASCII7; do CP1252 instead
+ if (best_sub == F_ASCII_7_bit) {best_sub = F_CP1252;}
+ destatep->declared_enc_1 = best_sub;
+ if (destatep->debug_data != NULL) {
+ // Show language hint
+ SetDetailsEncProb(destatep, 0, best_sub, normalized_lang.c_str());
+ }
+ return 1;
+ }
+ return 0;
+}
+
+// Apply initial probability hint based on corpus type (web, email, etc)
+// Return 1 if name match found
+int ApplyDefaultHint(const CompactEncDet::TextCorpusType corpus_type,
+ DetectEncodingState* destatep) {
+
+ for (int i = 0; i < NUM_RANKEDENCODING; i++) {
+ // Set the default probability
+ destatep->enc_prob[i] = kDefaultProb[i] * 3;
+ // Deliberately set 2022 seven-bit encodings to zero,
+ // so we can look for actual use
+ // TODO: This should all be folded into postproc-enc-detect.cc
+ if (SevenBitEncoding(kMapToEncoding[i])) {
+ destatep->enc_prob[i] = 0;
+ }
+ }
+
+ // A little corpus distinction
+ switch (corpus_type) {
+ case CompactEncDet::WEB_CORPUS:
+ case CompactEncDet::XML_CORPUS:
+ // Allow double-converted UTF-8 to start nearly equal to normal UTF-8
+ destatep->enc_prob[F_UTF8UTF8] =
+ destatep->enc_prob[F_UTF8] - kSmallInitDiff;
+ break;
+ case CompactEncDet::QUERY_CORPUS:
+ case CompactEncDet::EMAIL_CORPUS:
+ default:
+ break;
+ }
+
+ if (FLAGS_demo_nodefault) {
+ // Demo, make initial probs all zero
+ for (int i = 0; i < NUM_RANKEDENCODING; i++) {
+ destatep->enc_prob[i] = 0;
+ }
+ }
+
+ if (destatep->debug_data != NULL) {
+ // Show default hint
+ SetDetailsEncProb(destatep, 0, -1, "Default");
+ }
+ return 1;
+}
+
+
+
+// Do reverse search for c in [str..str+len)
+// Note: initial pointer is to FRONT of string, not back
+const char* MyMemrchr(const char* str, char c, size_t len) {
+ const char* ret = str + len;
+ while (str <= --ret) {
+ if (*ret == c) {return ret;}
+ }
+ return NULL;
+}
+
+
+// Minimum real URL is 11 bytes: "http://a.bc" -- shorter is assumed to be TLD
+// Now that we are no longer trying to do Indic font-based encodigns, we
+// don't need the full URL and can go back to simple TLD. This test remains for
+// backwards compatility with any caller using full URL.
+static const int kMinURLLength = 11;
+
+// Extract TLD from a full URL or just a TLD
+// Return hostname and length if a full URL
+void ExtractTLD(const char* url_hint, char* tld_hint, int tld_hint_len,
+ const char** ret_host_start, int* ret_host_len) {
+ // url_hint can either be a full URL (preferred) or just top-level domain name
+ // Extract the TLD from a full URL and use it for
+ // a normal TLD hint
+
+ strncpy(tld_hint, "~", tld_hint_len);
+ tld_hint[tld_hint_len - 1] = '\0';
+ *ret_host_start = NULL;
+ *ret_host_len = 0;
+
+ int url_len = (url_hint != NULL) ? strlen(url_hint) : 0;
+ if (url_len == 0) {
+ // Empty TLD
+ return;
+ }
+
+ // Minimum real URL is 11 bytes: "http://a.bc" -- shorter is assumed to be TLD
+ if (kMinURLLength <= url_len) {
+ // See if it really is a URL
+ const char* first_slash = strchr(url_hint, '/');
+ if ((first_slash != NULL) && (first_slash != url_hint) &&
+ (first_slash[-1] == ':') && (first_slash[1] == '/') &&
+ (memrchr(url_hint, '.', first_slash - url_hint) == NULL)) {
+ // We found :// and no dot in front of it, so declare a real URL
+
+ const char* hostname_start = first_slash + 2;
+ const char* hostname_end = strchr(hostname_start, '/');
+ if (hostname_end == NULL) {
+ // No slash; end is first byte off end of the URL string
+ hostname_end = url_hint + url_len;
+ }
+ size_t hostname_len = hostname_end - hostname_start;
+ const char* port_start =
+ (const char*)memchr(hostname_start, ':', hostname_len);
+ if (port_start != NULL) {
+ // Port; shorten hostname
+ hostname_end = port_start;
+ hostname_len = hostname_end - hostname_start;
+ }
+
+ const char* tld_start = MyMemrchr(hostname_start, '.', hostname_len);
+ if (tld_start != NULL) {
+ // Remember the TLD we just found
+ int tld_len = hostname_start + hostname_len - tld_start - 1;
+ if (tld_len > (tld_hint_len - 1)) {
+ tld_len = tld_hint_len - 1;
+ }
+ memcpy(tld_hint, tld_start + 1, tld_len);
+ tld_hint[tld_len] = '\0';
+ }
+ *ret_host_start = hostname_start;
+ *ret_host_len = hostname_len;
+ return;
+ }
+ } else {
+ strncpy(tld_hint, url_hint, tld_hint_len);
+ tld_hint[tld_hint_len - 1] = '\0';
+ }
+}
+
+// Apply hints, if any, to probabilities
+// NOTE: Encoding probabilites are all zero at this point
+void ApplyHints(const char* url_hint,
+ const char* http_charset_hint,
+ const char* meta_charset_hint,
+ const int encoding_hint,
+ const Language language_hint,
+ const CompactEncDet::TextCorpusType corpus_type,
+ DetectEncodingState* destatep) {
+ int hint_count = 0;
+ // url_hint can either be a full URL (preferred) or just top-level domain name
+ // Extract the TLD from a full URL and use it for
+ // a normal TLD hint
+
+ char tld_hint[16];
+ const char* hostname_start = NULL;
+ int hostname_len = 0;
+ ExtractTLD(url_hint, tld_hint, sizeof(tld_hint),
+ &hostname_start, &hostname_len);
+
+
+ // Initial hints give slight boost to Ascii-7-bit and code page 1252
+ // ApplyXxx routines copy enc_1 to enc_2 then update declared_enc_1
+ // This gives a boost to 1252 if one of HTTP/META is specified,
+ // but this could be the wrong thing to do if Latin2/3/4/etc. is specified
+ destatep->declared_enc_1 = F_CP1252;
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+
+ // Applying various hints takes max of new hint and any old hint.
+ // This does better on multiple hints that a weighted average
+
+ // Weight is 0..100 percent
+ if ((http_charset_hint != NULL) && (http_charset_hint[0] != '~')) {
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyCharsetHint(http_charset_hint, 100, destatep);
+ destatep->http_hint = kMapToEncoding[destatep->declared_enc_1];
+ if ((destatep->declared_enc_1 == F_CP1252) ||
+ (destatep->declared_enc_1 == F_Latin1)) {
+ destatep->looking_for_latin_trigrams = true;
+ }
+ }
+ if ((meta_charset_hint != NULL) && (meta_charset_hint[0] != '~')) {
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyCharsetHint(meta_charset_hint, 100, destatep);
+ destatep->meta_hint = kMapToEncoding[destatep->declared_enc_1];
+ if ((destatep->declared_enc_1 == F_CP1252) ||
+ (destatep->declared_enc_1 == F_Latin1)) {
+ destatep->looking_for_latin_trigrams = true;
+ }
+ }
+ if (encoding_hint != UNKNOWN_ENCODING) {
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyEncodingHint(encoding_hint, 50, destatep);
+ }
+ if (language_hint != UNKNOWN_LANGUAGE) {
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyUILanguageHint(language_hint, 50, destatep);
+ }
+ // Use top level domain if not .com and <=1 other hint was available
+ if (url_hint != NULL) {
+ destatep->tld_hint = CompactEncDet::TopEncodingOfTLDHint(tld_hint);
+ if (hint_count == 0) {
+ // Apply with weight 100%
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyTldHint(tld_hint, 100, destatep);
+ if ((destatep->declared_enc_1 == F_CP1252) ||
+ (destatep->declared_enc_1 == F_Latin1)) {
+ destatep->looking_for_latin_trigrams = true;
+ }
+ if (strcmp("hu", tld_hint) == 0) {
+ // Hungarian is particularly difficult to separate Latin2 from Latin1,
+ // so always look for trigram scanning if bare TLD=hu hint
+ destatep->looking_for_latin_trigrams = true;
+ }
+ // Treat .com as no TLD hint at all
+ } else if ((hint_count == 1) && (strcmp("com", tld_hint) != 0)) {
+ // Either shift weighting or consider doing no TLD here -- seems to
+ // distract from correct charset= hints. Or perhaps apply only if
+ // charset = Latin1/1252...
+ // Apply with weight 50%
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyTldHint(tld_hint, 50, destatep);
+ if ((destatep->declared_enc_1 == F_CP1252) ||
+ (destatep->declared_enc_1 == F_Latin1)) {
+ destatep->looking_for_latin_trigrams = true; // These need trigrams
+ }
+ }
+ // Else ignore TLD hint entirely
+ }
+
+ // Use all-web default distribution if not even a TLD hint
+ if (hint_count == 0) {
+ destatep->looking_for_latin_trigrams = true; // Default needs trigrams
+ destatep->declared_enc_2 = destatep->declared_enc_1;
+ hint_count += ApplyDefaultHint(corpus_type, destatep);
+ }
+
+
+// ISO-Microsoft Pairs
+// F_Latin1, F_CP1252,
+// F_Latin2, F_CP1250, NOT really strict subset/superset pairs
+// F_Latin3,
+// F_Latin4,
+// F_ISO_8859_5, F_CP1251,
+// F_Arabic, F_CP1256, NOT
+// F_Greek, F_CP1253, NOT really pairs
+// (or upgrade incvt to make Greek use CP)
+// F_Hebrew, F_CP1255, NOT really pairs
+// F_Latin5, F_CP1254,
+// F_Latin6,
+// F_ISO_8859_11,
+// F_ISO_8859_13, F_CP1257,
+// F_ISO_8859_15,
+// ISO-Microsoft Pairs
+
+ // Get important families started together
+ // // This should fall out of the initializatoin vectors for charset,
+ // but we need to get rid of families alltogetrher
+ //
+ // TODO make this more graceful
+
+ // Add small bias for subsets
+
+ // Subtract small bias for supersets
+ destatep->enc_prob[F_CP932] = destatep->enc_prob[F_SJS] - kSmallInitDiff;
+
+ destatep->enc_prob[F_GBK] = destatep->enc_prob[F_GB] - kSmallInitDiff;
+ destatep->enc_prob[F_GB18030] = destatep->enc_prob[F_GB] - kSmallInitDiff;
+
+ destatep->enc_prob[F_BIG5_CP950] = destatep->enc_prob[F_BIG5] -
+ kSmallInitDiff;
+ destatep->enc_prob[F_BIG5_HKSCS] = destatep->enc_prob[F_BIG5] -
+ kSmallInitDiff;
+
+ // Deliberate over-bias Ascii7 and underbias Binary [unneeded]
+ // destatep->enc_prob[F_ASCII_7_bit] = destatep->enc_prob[F_ASCII_7_bit] + kSmallInitDiff;
+ // destatep->enc_prob[F_BINARY] = destatep->enc_prob[F_BINARY] - (kBoostInitial / 2);
+
+ if (destatep->debug_data != NULL) {
+ // Show state at end of hints
+ SetDetailsEncProb(destatep, 0, -1, "Endhints");
+ if(FLAGS_enc_detect_detail2) {
+ // Add a line showing the watched encoding(s)
+ if (watch1_rankedenc >= 0) {
+ SetDetailsEncProb(destatep, 0,
+ watch1_rankedenc, FLAGS_enc_detect_watch1);
+ }
+ if (watch2_rankedenc >= 0) {
+ SetDetailsEncProb(destatep, 0,
+ watch2_rankedenc, FLAGS_enc_detect_watch2);
+ }
+ } // End detail2
+ }
+
+ // If duplicate hints, set second one to ASCII_7BIT to prevent double-boost
+ if (destatep->declared_enc_1 == destatep->declared_enc_2) {
+ destatep->declared_enc_2 = F_ASCII_7_bit;
+ }
+
+ if (FLAGS_force127) {
+ destatep->do_latin_trigrams = true;
+ if (FLAGS_enc_detect_source) {
+ PsHighlight(0, destatep->initial_src, 0, 2);
+ }
+ }
+
+
+ if (FLAGS_counts && destatep->looking_for_latin_trigrams) {++looking_used;}
+ if (FLAGS_counts && destatep->do_latin_trigrams) {++doing_used;}
+
+ //
+ // At this point, destatep->enc_prob[] is an initial probability vector based
+ // on the given hints/default. In general, it spreads out least-likely
+ // encodings to be about 2**-25 below the most-likely encoding.
+ // For input text with lots of bigrams, an unlikely encoding can rise to
+ // the top at a rate of about 2**6 per bigram, and more commonly 2**2 per
+ // bigram. So more than 4 bigrams and commonly more than 12 are
+ // needed to overcome the initial hints when the least-likely encoding
+ // is in fact the correct answer. So if the entire text has very few bigrams
+ // (as a two-word query might), it can be impossible for the correct
+ // encoding to win.
+ //
+ // To compensate for this, we take the initial hint vector and effectively
+ // apply it at the rate of 1/16 every bigram for the first 16 bigrams. The
+ // actual mechanism is done just before the last prune.
+ //
+
+ // Remember Initial hint probabilities
+ memcpy(destatep->hint_prob, destatep->enc_prob, sizeof(destatep->enc_prob));
+}
+
+// Look for specific high-value patterns in the first 4 bytes
+// Byte order marks (BOM)
+// EFBBBF UTF-8
+// FEFF UTF-16 BE
+// FFFE UTF-16 LE
+// FFFE0000 UTF-32 BE
+// 0000FEFF UTF-32 LE
+//
+// Likely UTF-x of seven-bit ASCII
+// 00xx UTF-16 BE xx printable ASCII
+// xx00 UTF-16 LE
+// 000000xx UTF-32 BE
+// xx000000 UTF-32 LE
+//
+void InitialBytesBoost(const uint8* src,
+ int text_length,
+ DetectEncodingState* destatep) {
+ if (text_length < 4) {return;}
+
+ uint32 pair01 = (src[0] << 8) | src[1];
+ uint32 pair23 = (src[2] << 8) | src[3];
+ uint32 quad0123 = (pair01 << 16) | pair23;
+
+ bool utf_16_indication = false;
+ bool utf_32_indication = false;
+ int best_enc = -1;
+
+ // Byte order marks
+ // UTF-8
+ if ((quad0123 & 0xffffff00) == 0xEFBBBF00) {
+ destatep->bom_hint = UTF8;
+ Boost(destatep, F_UTF8, kBoostInitial * 2);
+ Boost(destatep, F_UTF8UTF8, kBoostInitial * 2);
+ best_enc = F_UTF8;
+ // UTF-32 (test before UTF-16)
+ } else if (quad0123 == 0x0000FEFF) {
+ destatep->bom_hint = UTF32BE;
+ Boost(destatep, F_UTF_32BE, kBoostInitial * 2);
+ best_enc = F_UTF_32BE;
+ } else if (quad0123 == 0xFFFE0000) {
+ destatep->bom_hint = UTF32LE;
+ Boost(destatep, F_UTF_32LE, kBoostInitial * 2);
+ best_enc = F_UTF_32LE;
+ // UTF-16
+ } else if (pair01 == 0xFEFF) {
+ destatep->bom_hint = UTF16BE;
+ Boost(destatep, F_UTF_16BE, kBoostInitial * 3);
+ best_enc = F_UTF_16BE;
+ } else if (pair01 == 0xFFFE) {
+ destatep->bom_hint = UTF16LE;
+ Boost(destatep, F_UTF_16LE, kBoostInitial * 3);
+ best_enc = F_UTF_16LE;
+
+ // Possible seven-bit ASCII encoded as UTF-16/32
+ // UTF-32 (test before UTF-16)
+ } else if (((quad0123 & 0xffffff00) == 0) &&
+ (kIsPrintableAscii[src[3]] != 0)) {
+ Boost(destatep, F_UTF_32BE, kBoostInitial);
+ Whack(destatep, F_UTF_32LE, kBadPairWhack); // Illegal char
+ best_enc = F_UTF_32BE;
+ } else if (((quad0123 & 0x00ffffff) == 0) &&
+ (kIsPrintableAscii[src[0]] != 0)) {
+ Boost(destatep, F_UTF_32LE, kBoostInitial);
+ Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal char
+ best_enc = F_UTF_32LE;
+ } else if ((src[0] == 0x00) && (kIsPrintableAscii[src[1]] != 0)) {
+ Boost(destatep, F_UTF_16BE, kBoostInitial);
+ best_enc = F_UTF_16BE;
+ } else if ((src[1] == 0x00) && (kIsPrintableAscii[src[0]] != 0)) {
+ Boost(destatep, F_UTF_16LE, kBoostInitial);
+ best_enc = F_UTF_16LE;
+
+ // Whack if 0000 or FFFF
+ // UTF-32 (test before UTF-16)
+ } else if (quad0123 == 0x00000000) {
+ Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal char
+ Whack(destatep, F_UTF_32LE, kBadPairWhack);
+ Whack(destatep, F_UTF_16BE, kBadPairWhack);
+ Whack(destatep, F_UTF_16LE, kBadPairWhack);
+ best_enc = -1;
+ } else if (quad0123 == 0xffffffff) {
+ Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal char
+ Whack(destatep, F_UTF_32LE, kBadPairWhack);
+ Whack(destatep, F_UTF_16BE, kBadPairWhack);
+ Whack(destatep, F_UTF_16LE, kBadPairWhack);
+ best_enc = -1;
+ } else if (pair01 == 0x0000) {
+ Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal char
+ Whack(destatep, F_UTF_16LE, kBadPairWhack);
+ best_enc = -1;
+ } else if (pair01 == 0xffff) {
+ Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal char
+ Whack(destatep, F_UTF_16LE, kBadPairWhack);
+ best_enc = -1;
+
+
+ // These are the first four bytes of some known binary file formats
+
+ // Boost BINARY bigtime if JPEG FFD8FFxx
+ // Boost BINARY bigtime if png 89504E47 (.PNG)
+ // Boost BINARY bigtime if gif 47494638 (GIF8)
+ // Boost BINARY bigtime if zip 504B0304 (PK..)
+ // Boost BINARY bigtime if gzip 1F8B08xx
+ // Boost BINARY bigtime if gzip 78DAxxxx
+ // Boost BINARY if PDF 25504446 (%PDF)
+ // Boost BINARY if SWF (FWSx or CWSx where x <= 0x1f)
+ } else if ((quad0123 & 0xffffff00) == 0xFFD8FF00) { // JPEG FFD8FFxx
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x89504E47) { // Hex 89 P N G
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x47494638) { // Hex GIF8
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x504B0304) { // Hex P K 03 04
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if ((quad0123 & 0xffffff00) == 0x1F8B0800) { // gzip 1F8B08xx
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (pair01 == 0x78DA) { // gzip 78DAxxxx
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x25504446) { // Hex %PDF
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if ((quad0123 & 0xffffff1f) == 0x66535700) { // Hex FWSx
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if ((quad0123 & 0xffffff1f) == 0x63535700) { // Hex CWSx
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+
+ // More binary detect prefixes
+ // 7F E L F Executable and linking format
+ // M M 00 * TIFF (little-endian)
+ // * 00 M M TIFF (big-endian)
+ // 01 f c p Final cut pro
+ } else if (quad0123 == 0x7F454C46) { // Hex 7F E L F
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x4D4D002A) { // Hex M M 00 *
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x2A004D4D) { // Hex * 00 M M
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x01666370) { // Hex 01 f c p
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+
+ // More binary detect prefixes; all-ASCII names; heavy weight to avoid ASCII
+ // prefix overcoming binary
+ // C C S D USGS ISIS 3-D cube files
+ // S I M P FITS image header "SIMPLE "
+ } else if (quad0123 == 0x43435344) { // Hex C C S D
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x53494D50) { // Hex S I M P
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+
+ // More binary detect prefixes; all-ASCII names; lighter weight
+ // H W P Hangul word processor
+ // 8 B P S Photoshop
+ // P D S _ xx "PDS_VERSION_ID "
+ } else if (quad0123 == 0x48575020) { // Hex H W P
+ if ((19 <= text_length) &&
+ (memcmp(src, "HWP.Document.File.V", 19) == 0)) {
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if ((19 <= text_length) &&
+ (memcmp(src, "HWP Document File V", 19) == 0)) {
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else {
+ Boost(destatep, F_BINARY, kBoostInitial * kWeakerBinary);
+ }
+ } else if (quad0123 == 0x38425053) { // Hex 8 B P S
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else if (quad0123 == 0x5044535F) { // Hex P D S _
+ if ((14 <= text_length) && (memcmp(src, "PDS_VERSION_ID", 14) == 0)) {
+ Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
+ } else {
+ Boost(destatep, F_BINARY, kBoostInitial * kWeakerBinary);
+ }
+ }
+
+ // There are several main Windows EXE file formats.
+ // Not examined here (prefix too short; never see them in Google pipeline)
+ // M Z DOS .exe Mark Zbikowski
+ // N E DOS 4.0 16-bit
+ // L E OS/2 VxD drivers
+ // L X OS/2
+ // P E Windows NT
+
+
+ // More user-defined
+ // http://www.freenet.am/armscii/ Armenian
+
+ // If any hints or BOM, etc. keep UTF 16/32 around
+ if ((destatep->enc_prob[F_UTF_16BE] > 0) ||
+ (destatep->enc_prob[F_UTF_16LE] > 0)) {
+ utf_16_indication = true;
+ }
+ if ((destatep->enc_prob[F_UTF_32BE] > 0) ||
+ (destatep->enc_prob[F_UTF_32LE] > 0)) {
+ utf_32_indication = true;
+ }
+
+
+ // Kill UTF16/32 right now if no positive indication of them
+ // Otherwise, they tend to rise to the top in 7-bit files with an
+ // occasional 0x02 byte in some comment or javascript
+ if (!utf_16_indication) {
+ Whack(destatep, F_UTF_16BE, kBadPairWhack * 8);
+ Whack(destatep, F_UTF_16LE, kBadPairWhack * 8);
+ Whack(destatep, F_Unicode, kBadPairWhack * 8);
+ }
+ if (!utf_32_indication) {
+ Whack(destatep, F_UTF_32BE, kBadPairWhack * 8);
+ Whack(destatep, F_UTF_32LE, kBadPairWhack * 8);
+ }
+
+ // Usually kill mixed encodings
+ if (!FLAGS_ced_allow_utf8utf8) {
+ Whack(destatep, F_UTF8UTF8, kBadPairWhack * 8);
+ }
+ // 2011.11.07 never use UTF8CP1252 -- answer will be UTF8 instead
+ Whack(destatep, F_UTF8CP1252, kBadPairWhack * 8);
+
+ if (destatep->debug_data != NULL) {
+ // Show first four bytes of the input
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%04x%04x", pair01, pair23);
+ SetDetailsEncProb(destatep, 0, best_enc, buff);
+ }
+}
+
+
+
+// Descending order
+int IntCompare(const void* v1, const void* v2) {
+ const int* p1 = reinterpret_cast<const int*>(v1);
+ const int* p2 = reinterpret_cast<const int*>(v2);
+ if (*p1 < *p2) {return 1;}
+ if (*p1 > *p2) {return -1;}
+ return 0;
+}
+
+bool Base64Char(uint8 c) {
+ if (('A' <= c) && (c <= 'Z')) {return true;}
+ if (('a' <= c) && (c <= 'z')) {return true;}
+ if (('0' <= c) && (c <= '9')) {return true;}
+ if ('+' == c) {return true;}
+ if ('/' == c) {return true;}
+ return false;
+}
+
+int Base64ScanLen(const uint8* start, const uint8* limit) {
+ // We have a plausible beginning; scan entire base64 string
+ const uint8* ib64str = start;
+ const uint8* b64str = ib64str;
+ const uint8* b64strlimit = limit;
+ // if starts with + +++, assume it is drawing, so bogus
+ if (((limit - start) > 3) && (start[0] == '+') &&
+ (start[1] == '+') && (start[2] == '+')) {
+ return 81;
+ }
+ // Scan over base64
+ while ((b64str < b64strlimit) && (kBase64Value[*b64str++] >= 0)) {
+ }
+ b64str--; // We overshot by 1
+ return b64str - ib64str;
+}
+
+// Input is at least 8-character legal base64 string after +.
+// But might be say + "Presse+Termine"
+bool GoodUnicodeFromBase64(const uint8* start, const uint8* limit) {
+ // Reject base64 string len N if density of '+' is > 1 + N/16 (expect 1/64)
+ // Reject base64 string len N if density of A-Z is < 1 + N/16 (expect 26/64)
+ // Reject base64 string len N if density of a-z is < 1 + N/16 (expect 26/64)
+ // Reject base64 string len N if density of 0-9 is < 1 + N/32 (expect 10/64)
+ // NOTE: this requires at least one lower AND one upper AND one digit to pass
+ //
+ int plus_count = 0;
+ int lower_count = 0;
+ int upper_count = 0;
+ int digit_count = 0;
+ int len = limit - start;
+ for (const uint8* src = start; src < limit; ++src) {
+ uint8 c = *src;
+ if (('a' <= c) && (c <= 'z')) {
+ ++lower_count;
+ } else if (('A' <= c) && (c <= 'Z')) {
+ ++upper_count;
+ } else if (('0' <= c) && (c <= '0')) {
+ ++digit_count;
+ } else if (*src == '+') {
+ ++plus_count;
+ }
+ }
+
+ if (plus_count > (1 + (len >> 4))) {return false;}
+ if (lower_count < (1 + (len >> 4))) {return false;}
+ if (upper_count < (1 + (len >> 4))) {return false;}
+ if (digit_count < (1 + (len >> 5))) {return false;}
+
+ // checking the last character to reduce false positive
+ // since the last character may be padded to 0 bits at the end.
+ // refer to http://en.wikipedia.org/wiki/UTF-7
+ int nmod8 = len & 7;
+ const uint8 last = *(start+len-1);
+ // When UTF-7 string length%8=3, the last two bits must be padded as 0
+ if ((nmod8 == 3) && (kBase64Value[last] & 3)) {return false;}
+ // When UTF-7 string length%8=6, the last four bits must be padded as 0
+ if ((nmod8 == 6) && (kBase64Value[last] & 15)) {return false;}
+ return true;
+}
+
+// Prune here after N bytes
+// Boost here for seven-bit sequences (at every prune)
+// if (sevenbitrankedencoding)
+// + UTF7 scan and boost/demote len mod 8 = 0 3 6
+// ~ Hz scan and boost/demote len mod 8 = 0 2 4 6
+// 1B 2022 scan and boost/demote len mod 8 = 0 2 4 6
+// 0E 2022 scan and boost/demote len mod 8 = 0 2 4 6
+// [0F 2022 boost/demote]
+// 00 UTF16/32 scan and boost/demote offset = even/odd
+//
+// If still some seven-bit possibilities > pure ASCII,
+// scan each possibility for clearer prob, s.t. about
+// two good sequences is a clear win
+// A-Z 00-19 00xx-64xx (B = 04xx)
+// a-z 1A-33 68xx-CCxx (f = 7Cxx)
+// 0-9 34-3D D0xx-F4xx (1 = D4xx)
+// + 3E F8xx
+// / 3F FCxx
+// do another chunk with slow scan
+
+
+// Boost, whack, or leave alone UTF-7 probablilty
+void UTF7BoostWhack(DetectEncodingState* destatep, int next_pair, uint8 byte2) {
+ int off = destatep->interesting_offsets[AsciiPair][next_pair];
+ if (off >= destatep->prior_utf7_offset) {
+ // Not part of a previous successful UTF-7 string
+ ++destatep->utf7_starts;
+
+ if (byte2 == '-') {
+ // +- encoding for '+' neutral
+ } else if (!Base64Char(byte2)) {
+ // Not base64 -- not UTF-7, whack
+ Whack(destatep, F_UTF7, kBadPairWhack); // Illegal pair
+ } else {
+ // Starts with base64 byte, might be a good UTF7 sequence
+ const uint8* start = destatep->initial_src + off + 1; // over the +
+ int n = Base64ScanLen(start, destatep->limit_src);
+ int nmod8 = n & 7;
+ if ((n == 3) || (n == 6)) {
+ // short but legal -- treat as neutral
+ } else if ((nmod8 == 0) | (nmod8 == 3) | (nmod8 == 6)) {
+ // Good length. Check for good Unicode.
+ if (GoodUnicodeFromBase64(start, start + n)) {
+ // Good length and Unicode, boost
+ Boost(destatep, F_UTF7, kBoostOnePair); // Found good
+ destatep->prior_utf7_offset = off + n + 1;
+ } else {
+ // Bad Unicode. Whack
+ Whack(destatep, F_UTF7, kBadPairWhack); // Illegal length
+ }
+ } else {
+ // Bad length. Whack
+ Whack(destatep, F_UTF7, kBadPairWhack); // Illegal length
+ }
+ }
+ }
+}
+
+// Boost, whack, or leave alone HZ probablilty
+void HzBoostWhack(DetectEncodingState* destatep, uint8 byte2) {
+ if ((byte2 == '{') || (byte2 == '}')) {
+ Boost(destatep, F_HZ_GB_2312, kBoostOnePair); // Found ~{ or ~}
+ } else if ((byte2 == '~') || (byte2 == '\n')) {
+ destatep->enc_prob[F_HZ_GB_2312] += 0; // neutral
+ } else {
+ Whack(destatep, F_HZ_GB_2312, kBadPairWhack); // Illegal pair
+ }
+}
+
+// Boost, whack, or leave alone BINARY probablilty
+void BinaryBoostWhack(DetectEncodingState* destatep, uint8 byte1, uint8 byte2) {
+ int quadrant = ((byte1 & 0x80) >> 6) | ((byte2 & 0x80) >> 7);
+ int bucket8x4 = ((byte1 & 0xe0) >> 3) | ((byte2 & 0xc0) >> 6);
+ uint32 quad_mask = 1 << quadrant;
+ uint32 bucket8x4_mask = 1 << bucket8x4;
+ if ((destatep->binary_quadrants_seen & quad_mask) == 0) {
+ destatep->binary_quadrants_seen |= quad_mask;
+ destatep->binary_quadrants_count += 1;
+ if (destatep->binary_quadrants_count == 4) {
+ Boost(destatep, F_BINARY, kBoostOnePair * 2); // Found all 4 quadrants,
+ // boost 2 pairs
+ }
+ }
+ if ((destatep->binary_8x4_seen & bucket8x4_mask) == 0) {
+ destatep->binary_8x4_seen |= bucket8x4_mask;
+ destatep->binary_8x4_count += 1;
+ if (destatep->binary_8x4_count >= 11) {
+ Boost(destatep, F_BINARY, kBoostOnePair * 4); // Found 11+/20 buckets,
+ // boost 4 pairs each time
+ }
+ }
+}
+
+
+// Demote UTF-16/32 on 0000 or FFFF, favoring Binary
+void UTF1632BoostWhack(DetectEncodingState* destatep, int offset, uint8 byte1) {
+ if (byte1 == 0) { // We have 0000
+ Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal pair
+ Whack(destatep, F_UTF_16LE, kBadPairWhack); // Illegal pair
+ switch (offset & 3) {
+ case 0: // We get called with 0 4 8, etc. for ASCII/BMP as UTF-32BE
+ Whack(destatep, F_UTF_32LE, kBadPairWhack); // Illegal pair
+ Boost(destatep, F_UTF_32BE, kSmallInitDiff); // Good pair
+ break;
+ case 1: // We get called with 1 5 9, etc. for ASCII as UTF-32LE
+ case 2: // We get called with 2 6 10, etc. for BMP as UTF-32LE
+ Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal pair
+ Boost(destatep, F_UTF_32LE, kSmallInitDiff); // Good pair
+ break;
+ case 3: // ambiguous
+ break;
+ }
+ } else { // We have ffff
+ Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal pair
+ Whack(destatep, F_UTF_32LE, kBadPairWhack); // Illegal pair
+ Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal pair
+ Whack(destatep, F_UTF_16LE, kBadPairWhack); // Illegal pair
+ }
+}
+
+// Make even offset
+void UTF16MakeEven(DetectEncodingState* destatep, int next_pair) {
+ destatep->interesting_offsets[OtherPair][next_pair] &= ~1;
+}
+
+bool ConsecutivePair(DetectEncodingState* destatep, int i) {
+ if (i <= 0) {
+ return false;
+ }
+ return destatep->interesting_offsets[OtherPair][i] ==
+ (destatep->interesting_offsets[OtherPair][i - 1] + 2);
+}
+
+// boost, whack, or leave alone UTF-8 probablilty
+// Any whacks are also applied to UTF8UTF8; CheckUTF8UTF8Seq assumes good UTF8
+// Returns total boost
+int CheckUTF8Seq(DetectEncodingState* destatep, int weightshift) {
+ int startcount = destatep->prior_interesting_pair[OtherPair];
+ int endcount = destatep->next_interesting_pair[OtherPair];
+
+ int demotion_count = 0;
+ for (int i = startcount; i < endcount; ++i) {
+ int sub;
+ char* s = &destatep->interesting_pairs[OtherPair][i * 2];
+ // Demote four byte patterns that are more likely Latin1 than UTF-8
+ // C9AE, DF92, DF93, DFAB. See note at top.
+ // Demotion also boosts Latin1 and CP1252
+ uint8 s0 = static_cast<uint8>(s[0]);
+ uint8 s1 = static_cast<uint8>(s[1]);
+ if ((s0 == 0xc9) && (s1 == 0xae)) {++demotion_count;}
+ if ((s0 == 0xdf) && (s1 == 0x92)) {++demotion_count;}
+ if ((s0 == 0xdf) && (s1 == 0x93)) {++demotion_count;}
+ if ((s0 == 0xdf) && (s1 == 0xab)) {++demotion_count;}
+
+ if (!ConsecutivePair(destatep, i)) {
+ // Insert a blank into the sequence; avoid wrong splices
+ sub = (' ' >> 4) & 0x0f;
+ ++destatep->utf8_minicount[
+ static_cast<int>(kMiniUTF8Count[static_cast<int>(destatep->next_utf8_ministate)][sub])];
+ destatep->next_utf8_ministate =
+ kMiniUTF8State[destatep->next_utf8_ministate][sub];
+ }
+ // Byte 0
+ sub = (s0 >> 4) & 0x0f;
+ ++destatep->utf8_minicount[
+ static_cast<int>(kMiniUTF8Count[static_cast<int>(destatep->next_utf8_ministate)][sub])];
+ destatep->next_utf8_ministate =
+ kMiniUTF8State[destatep->next_utf8_ministate][sub];
+ // Byte 1
+ sub = (s1 >> 4) & 0x0f;
+ ++destatep->utf8_minicount[
+ static_cast<int>(kMiniUTF8Count[static_cast<int>(destatep->next_utf8_ministate)][sub])];
+ destatep->next_utf8_ministate =
+ kMiniUTF8State[destatep->next_utf8_ministate][sub];
+ DCHECK((0 <= destatep->next_utf8_ministate) &&
+ (destatep->next_utf8_ministate < 8));
+ }
+
+
+ // For the four specific byte combinations above, Latin1/CP1252 is more likely
+ if (demotion_count > 0) {
+ Boost(destatep, F_Latin1, kGentleOnePair * demotion_count);
+ Boost(destatep, F_CP1252, kGentleOnePair * demotion_count);
+ }
+
+ // Boost UTF8 for completed good sequences
+ int total_boost = 2 * destatep->utf8_minicount[2] +
+ 3 * destatep->utf8_minicount[3] +
+ 4 * destatep->utf8_minicount[4];
+ // But not so much for demoted bytes
+ total_boost -= (3 * demotion_count);
+
+ total_boost *= kGentleOnePair;
+ total_boost >>= weightshift;
+ // Design: boost both UTF8 and UTF8UTF8 for each good sequence
+ Boost(destatep, F_UTF8, total_boost);
+ Boost(destatep, F_UTF8UTF8, total_boost);
+
+ destatep->utf8_minicount[5] += destatep->utf8_minicount[2]; // total chars
+ destatep->utf8_minicount[5] += destatep->utf8_minicount[3]; // total chars
+ destatep->utf8_minicount[5] += destatep->utf8_minicount[4]; // total chars
+ destatep->utf8_minicount[2] = 0;
+ destatep->utf8_minicount[3] = 0;
+ destatep->utf8_minicount[4] = 0;
+
+ // Whack (2 bytes) for errors
+ int error_whack = 2 * destatep->utf8_minicount[1];
+ error_whack *= kGentlePairWhack;
+ error_whack >>= weightshift;
+ Whack(destatep, F_UTF8, error_whack);
+ Whack(destatep, F_UTF8UTF8, error_whack);
+ destatep->utf8_minicount[1] = 0;
+
+ return total_boost - error_whack;
+}
+
+
+// Boost, whack, or leave alone UTF8UTF8 probablilty
+//
+// We are looking for
+// (1) chars ONLY in set UTF8(0080)..UTF8(00FF), including for 80..9F the
+// MS CP1252 mappings, and
+// (2) sequences of 2 or more such characters
+//
+// If so, we could be looking at some non-7-bit encoding extra-converted
+// to UTF-8. The most common observed is CP1252->UTF8 twice,
+// 1252=>UTF8 : 1252=>UTF8
+// where the colon means "take those bytes and pretend that they are 1252".
+// We have a couple of examples of BIG5 bytes converted as though
+// they were 1252,
+// BIG5 : 1252=>UTF8
+//
+// Of course, we don't want correctly converted 1252 to be flagged here
+// 1252=>UTF8
+// So we want the input high bytes to be in pairs or longer, hence the
+// output UTF8 in groups of four bytes or more
+//
+// Good chars: C2xx, C3xx,
+// Good chars: C592, C593, C5A0, C5A1, C5B8, C5BD, C5BE, C692, CB86, CB9C
+// Good chars: E280xx E282AC E284A2
+// C2xx 1100001x 10xxxxxx (128/128)
+// C5xx 11000101 10xx00xx (16/4)
+// C5xx 11000101 10111xxx (8/3)
+// C692 11000110 10010010 (1/1)
+// CBxx 11001011 100xx1x0 (8/2)
+// E28x 11100010 10000xx0 (4/3)
+//
+// Returns total boost
+int CheckUTF8UTF8Seq(DetectEncodingState* destatep, int weightshift) {
+ int this_pair = destatep->prior_interesting_pair[OtherPair];
+ int startbyteoffset = this_pair * 2;
+ int endbyteoffset = destatep->next_interesting_pair[OtherPair] * 2;
+ char* startbyte = &destatep->interesting_pairs[OtherPair][startbyteoffset];
+ char* endbyte = &destatep->interesting_pairs[OtherPair][endbyteoffset];
+
+ int pair_number = this_pair;
+ for (char* s = startbyte; s < endbyte; s += 2) {
+ int next = destatep->next_utf8utf8_ministate;
+ if (!ConsecutivePair(destatep, pair_number)) {
+ // Insert two blanks into the sequence to avoid wrong splices
+ // go back to no odd-byte offset
+ destatep->utf8utf8_odd_byte = 0;
+ int sub = UTF88Sub(' ', ' ');
+ ++destatep->utf8utf8_minicount[static_cast<int>(kMiniUTF8UTF8Count[next][sub])];
+ next = kMiniUTF8UTF8State[next][sub];
+ }
+
+ int odd = destatep->utf8utf8_odd_byte;
+ if (s + 1 + odd >= endbyte) continue;
+ int sub = UTF88Sub(s[0 + odd], s[1 + odd]);
+ destatep->utf8utf8_odd_byte ^= kMiniUTF8UTF8Odd[next][sub];
+ ++destatep->utf8utf8_minicount[
+ static_cast<int>(kMiniUTF8UTF8Count[next][sub])];
+ destatep->next_utf8utf8_ministate = kMiniUTF8UTF8State[next][sub];
+ ++pair_number;
+ }
+
+ // Boost for completed good sequences; each count covers two chars.
+ // Design: boost UTF8UTF8 above UTF8 for each good sequence
+ int total_boost = (2) * destatep->utf8utf8_minicount[2] +
+ (2) * destatep->utf8utf8_minicount[3] +
+ (2) * destatep->utf8utf8_minicount[4];
+ total_boost *= kGentleOnePair;
+ total_boost >>= weightshift;
+ Boost(destatep, F_UTF8UTF8, total_boost);
+
+ // Track total characters
+ destatep->utf8utf8_minicount[5] += destatep->utf8utf8_minicount[2];
+ destatep->utf8utf8_minicount[5] += destatep->utf8utf8_minicount[3];
+ destatep->utf8utf8_minicount[5] += destatep->utf8utf8_minicount[4];
+ destatep->utf8utf8_minicount[2] = 0;
+ destatep->utf8utf8_minicount[3] = 0;
+ destatep->utf8utf8_minicount[4] = 0;
+
+ // Design: Do not whack UTF8UTF8 below UTF8 for each bad sequence
+
+ destatep->utf8utf8_minicount[1] = 0;
+ return total_boost;
+}
+
+
+// We give a gentle boost for each paired SO ... SI, whack others
+void CheckIso2022ActiveSeq(DetectEncodingState* destatep) {
+ int this_pair = destatep->prior_interesting_pair[OtherPair];
+ int startbyteoffset = this_pair * 2;
+ int endbyteoffset = destatep->next_interesting_pair[OtherPair] * 2;
+ char* startbyte = &destatep->interesting_pairs[OtherPair][startbyteoffset];
+ char* endbyte = &destatep->interesting_pairs[OtherPair][endbyteoffset];
+
+ // Initial <esc> char must precede SO/SI
+ // HZ_GB_2312 has no alternation constraint on 1- and 2-byte segments
+ // ISO-2022-JP (JIS) has no alternation constraint on 1- and 2-byte segments
+ // ISO-2022-CN has no alternation constraint on 1- and 2-byte segments
+ // ISO-2022-KR requires alternation between 1- and 2-byte segments
+ // JIS:
+ // <esc> ( B ISO-2022-JP [1b 28 42] SI to ASCII
+ // <esc> ( J ISO-2022-JP [1b 28 4a] SI to X0201
+ // <esc> $ @ ISO-2022-JP [1b 24 40] SO to X0208-78 twobyte
+ // <esc> $ B ISO-2022-JP [1b 24 42] SO to X0208-83 twobyte
+ for (char* s = startbyte; s < endbyte; s += 2) {
+ if (s[0] == 0x1b) {
+ if (s[1] == 0x24) {
+ // <esc> $ is SO
+ destatep->next_2022_state = SOSI_TWOBYTE; // SO to two-byte
+ } else if (s[1] == 0x28) {
+ if (destatep->next_2022_state == SOSI_TWOBYTE) {
+ Boost(destatep, F_JIS, kGentlePairBoost);
+ } else if (destatep->next_2022_state == SOSI_ONEBYTE) {
+ Whack(destatep, F_JIS, kGentlePairWhack);
+ }
+ destatep->next_2022_state = SOSI_ONEBYTE; // JIS SI to one-byte
+ } else {
+ Whack(destatep, F_JIS, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
+ destatep->next_2022_state = SOSI_ERROR; // not 2022
+ }
+ } else if (s[0] == 0x0e) {
+ // <so>
+ Whack(destatep, F_JIS, kBadPairWhack);
+ if (destatep->next_2022_state != SOSI_NONE) {
+ destatep->next_2022_state = SOSI_TWOBYTE; // SO to two-byte
+ } else {
+ // ESC required before SO/SI
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack * 4);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack * 4);
+ destatep->next_2022_state = SOSI_ERROR; // SO not after SI
+ }
+ } else if (s[0] == 0x0f) {
+ // <si>
+ Whack(destatep, F_JIS, kBadPairWhack);
+ if (destatep->next_2022_state != SOSI_NONE) {
+ if (destatep->next_2022_state == SOSI_TWOBYTE) {
+ Boost(destatep, F_ISO_2022_CN, kGentlePairBoost);
+ Boost(destatep, F_ISO_2022_KR, kGentlePairBoost);
+ } else if (destatep->next_2022_state == SOSI_ONEBYTE) {
+ Whack(destatep, F_ISO_2022_CN, kGentlePairWhack);
+ Whack(destatep, F_ISO_2022_KR, kGentlePairWhack);
+ }
+ destatep->next_2022_state = SOSI_ONEBYTE; // SI to one-byte
+ } else {
+ // ESC required before SO/SI
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack * 4);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack * 4);
+ destatep->next_2022_state = SOSI_ERROR; // SI not after SO
+ }
+ } else if (s[0] <= 0x1f) {
+ // Some other control code. Allow ht lf [ff] cr
+ if ((s[0] != 0x09) && (s[0] != 0x0a) &&
+ (s[0] != 0x0c) && (s[0] != 0x0d)) {
+ // Otherwise these can float to the top on bad bytes
+ Whack(destatep, F_JIS, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
+ }
+ }
+ }
+
+ // If no start, keep the probability pinned at zero (or below)
+ if (destatep->next_2022_state == SOSI_NONE) {
+ destatep->enc_prob[F_ISO_2022_CN] =
+ minint(0, destatep->enc_prob[F_ISO_2022_CN]);
+ destatep->enc_prob[F_ISO_2022_KR] =
+ minint(0, destatep->enc_prob[F_ISO_2022_KR]);
+ destatep->enc_prob[F_JIS] =
+ minint(0, destatep->enc_prob[F_JIS]);
+ }
+}
+
+// We give a gentle boost for each paired ~{ ... ~}, whack others
+void CheckHzActiveSeq(DetectEncodingState* destatep) {
+ int this_pair = destatep->prior_interesting_pair[AsciiPair];
+ int startbyteoffset = this_pair * 2;
+ int endbyteoffset = destatep->next_interesting_pair[AsciiPair] * 2;
+ char* startbyte = &destatep->interesting_pairs[AsciiPair][startbyteoffset];
+ char* endbyte = &destatep->interesting_pairs[AsciiPair][endbyteoffset];
+
+ for (char* s = startbyte; s < endbyte; s += 2) {
+ // Look for initial ~{ pair
+ if ((s[0] == '~') && (s[1] == '{')) {
+ destatep->next_hz_state = SOSI_TWOBYTE; // SO to two-byte
+ }
+ // Also look for closing ~} pair
+ if ((s[0] == '~') && (s[1] == '}')) {
+ if (destatep->next_hz_state == SOSI_TWOBYTE) {
+ Boost(destatep, F_HZ_GB_2312, kGentlePairBoost);
+ } else if (destatep->next_hz_state == SOSI_ONEBYTE) {
+ Whack(destatep, F_HZ_GB_2312, kGentlePairWhack);
+ }
+ destatep->next_hz_state = SOSI_ONEBYTE; // SI to one-byte
+ }
+ }
+
+ // If no start, keep the probability pinned at zero (or below)
+ if (destatep->next_hz_state == SOSI_NONE) {
+ destatep->enc_prob[F_HZ_GB_2312] =
+ minint(0, destatep->enc_prob[F_HZ_GB_2312]);
+ }
+}
+
+// We give a gentle boost after an odd number of 8Fxxxx triples, which
+// put subsequent bigrams out of phase until a low byte or another 8Fxxxx
+void CheckEucJpSeq(DetectEncodingState* destatep) {
+ int this_pair = destatep->prior_interesting_pair[OtherPair];
+ int startbyteoffset = this_pair * 2;
+ int endbyteoffset = destatep->next_interesting_pair[OtherPair] * 2;
+ char* startbyte = &destatep->interesting_pairs[OtherPair][startbyteoffset];
+ char* endbyte = &destatep->interesting_pairs[OtherPair][endbyteoffset];
+
+ for (char* s = startbyte; s < endbyte; s += 2) {
+ // Boost if out of phase (otherwise, EUC-JP will score badly after 8Fxxxx)
+ if (destatep->next_eucjp_oddphase) {
+ //printf(" EucJp boost[%02x%02x]\n", s[0], s[1]); // TEMP
+ Boost(destatep, F_EUC_JP, kGentlePairBoost * 2);
+ }
+
+ uint8 s0 = static_cast<uint8>(s[0]);
+ uint8 s1 = static_cast<uint8>(s[1]);
+ // Look for phase flip at 8F
+ if ((s0 & 0x80) == 0x00) {
+ destatep->next_eucjp_oddphase = false;
+ } else if (s0 == 0x8f) {
+ destatep->next_eucjp_oddphase = !destatep->next_eucjp_oddphase;
+ }
+ if ((s1 & 0x80) == 0x00) {
+ destatep->next_eucjp_oddphase = false;
+ } else if (s1 == 0x8f) {
+ destatep->next_eucjp_oddphase = !destatep->next_eucjp_oddphase;
+ }
+ }
+}
+
+// Boost, whack, or leave alone BINARY probablilty
+// Also called if UTF 16/32 active
+void CheckBinaryDensity(const uint8* src, DetectEncodingState* destatep,
+ int delta_otherpairs) {
+ // No change if not much gathered information
+ if (delta_otherpairs == 0) {
+ // Only ASCII pairs this call
+ return;
+ }
+ int next_pair = destatep->next_interesting_pair[OtherPair];
+
+ // Look at density of interesting pairs [0..src)
+ int delta_offset = static_cast<int>(src - destatep->initial_src); // actual
+
+ // Look at density of interesting pairs [0..next_interesting)
+ int low_byte = destatep->interesting_offsets[OtherPair][0];
+ //int high_byte = destatep->interesting_offsets[OtherPair][next_pair - 1] + 2;
+ //int byte_span = high_byte - low_byte;
+ int byte_span = delta_offset - low_byte;
+
+ // If all ASCII for the first 4KB, reject
+ // If mostly ASCII in the first 5KB, reject
+ if ((low_byte >= kBinaryHardAsciiLimit) || (delta_offset >= kBinarySoftAsciiLimit)) {
+ // Not binary early enough in text
+ Whack(destatep, F_BINARY, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_32BE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_32LE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_16BE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_16LE, kBadPairWhack * 4);
+ return;
+ }
+
+ // Density 1.0 for N pairs takes 2*N bytes
+ // Whack if < 1/16 after first non_ASCII pair
+ if ((next_pair * 2 * 16) < byte_span) {
+ // Not dense enough
+ Whack(destatep, F_BINARY, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_32BE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_32LE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_16BE, kBadPairWhack * 4);
+ Whack(destatep, F_UTF_16LE, kBadPairWhack * 4);
+ }
+
+ if (next_pair < 8) {
+ // Fewer than 8 non-ASCII total; too soon to boost
+ return;
+ }
+
+ // Density 1.0 for N pairs takes 2*N bytes
+ // Boost if density >= 1/4, whack if < 1/16
+ if ((next_pair * 2 * 4) >= byte_span) {
+ // Very dense
+ // Only boost if at least 2 quadrants seen
+ if (destatep->binary_quadrants_count >= 2) {
+ Boost(destatep, F_BINARY, kSmallInitDiff);
+ Boost(destatep, F_UTF_32BE, kSmallInitDiff);
+ Boost(destatep, F_UTF_32LE, kSmallInitDiff);
+ Boost(destatep, F_UTF_16BE, kSmallInitDiff);
+ Boost(destatep, F_UTF_16LE, kSmallInitDiff);
+ }
+ }
+}
+
+
+// Look at a number of special-case encodings whose reliable detection depends
+// on sequencing or other properties
+// AsciiPair probibilities (UTF7 and HZ) are all done here
+void ActiveSpecialBoostWhack(const uint8* src, DetectEncodingState* destatep) {
+ int delta_asciipairs = destatep->next_interesting_pair[AsciiPair] -
+ destatep->prior_interesting_pair[AsciiPair];
+ int delta_otherpairs = destatep->next_interesting_pair[OtherPair] -
+ destatep->prior_interesting_pair[OtherPair];
+
+ // The two pure ASCII encodings
+ if (UTF7OrHzActive(destatep) && (delta_asciipairs > 0)) {
+ // Adjust per pair
+ for (int i = 0; i < delta_asciipairs; ++i) {
+ int next_pair = destatep->prior_interesting_pair[AsciiPair] + i;
+ uint8 byte1 = destatep->interesting_pairs[AsciiPair][next_pair * 2 + 0];
+ uint8 byte2 = destatep->interesting_pairs[AsciiPair][next_pair * 2 + 1];
+ if (byte1 == '+') {
+ // Boost, whack, or leave alone UTF-7 probablilty
+ UTF7BoostWhack(destatep, next_pair, byte2);
+ if (destatep->debug_data != NULL) {
+ // Show UTF7 entry
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%02x%02x+", byte1, byte2);
+ SetDetailsEncProb(destatep,
+ destatep->interesting_offsets[AsciiPair][next_pair],
+ kMostLikelyEncoding[(byte1 << 8) + byte2],
+ buff);
+ }
+ } else if (byte1 == '~') {
+ // Boost, whack, or leave alone HZ probablilty
+ HzBoostWhack(destatep, byte2);
+ if (destatep->debug_data != NULL) {
+ // Show Hz entry
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%02x%02x~", byte1, byte2);
+ SetDetailsEncProb(destatep,
+ destatep->interesting_offsets[AsciiPair][next_pair],
+ kMostLikelyEncoding[(byte1 << 8) + byte2],
+ buff);
+ }
+ }
+ }
+
+ // Kill UTF-7 now if at least 8 + pairs and not confirmed valid UTF-7
+ if ((destatep->utf7_starts >= 8) && (destatep->prior_utf7_offset == 0)) {
+ Whack(destatep, F_UTF7, kBadPairWhack * 8); // flush
+ }
+ }
+
+
+
+ // All the other encodings
+ if (OtherActive(destatep) && (delta_otherpairs > 0)) {
+ // Adjust per pair
+ int biggest_weightshift = 0;
+ for (int i = 0; i < delta_otherpairs; ++i) {
+ int next_pair = destatep->prior_interesting_pair[OtherPair] + i;
+ uint8 byte1 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 0];
+ uint8 byte2 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 1];
+ int off = destatep->interesting_offsets[OtherPair][next_pair];
+ int weightshift = destatep->interesting_weightshift[OtherPair][next_pair];
+ biggest_weightshift = maxint(biggest_weightshift, weightshift);
+
+ if (byte1 == 0x00) {
+ if (byte2 == 0x00) {
+ UTF1632BoostWhack(destatep, off, byte1);
+ } else if ((kIsPrintableAscii[byte2] != 0) && ((off & 1) != 0)) {
+ // We have 00xx at an odd offset. Turn into preceding even offset
+ // for possible Ascii text in UTF-16LE or UTF-32LE (vs BE)
+ // This will cascade into caller's probability update
+ // 00 is illegal for all other encodings, so it doesn't matter to them
+ UTF16MakeEven(destatep, next_pair);
+ }
+ if (destatep->debug_data != NULL) {
+ // Show 0000 detail entry for this bigram
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%02x%02xZ", byte1, byte2);
+ SetDetailsEncProb(destatep,
+ destatep->interesting_offsets[OtherPair][next_pair],
+ kMostLikelyEncoding[(byte1 << 8) + byte2],
+ buff);
+ }
+ }
+ if (byte1 == 0xff) {
+ if (byte2 == 0xff) {
+ UTF1632BoostWhack(destatep, off, byte1);
+ }
+ if (destatep->debug_data != NULL) {
+ // Show FFFF detail entry for this bigram
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%02x%02xF", byte1, byte2);
+ SetDetailsEncProb(destatep,
+ destatep->interesting_offsets[OtherPair][next_pair],
+ kMostLikelyEncoding[(byte1 << 8) + byte2],
+ buff);
+ }
+ }
+ if (BinaryActive(destatep)) {
+ BinaryBoostWhack(destatep, byte1, byte2);
+ }
+ } // End for i
+
+ // Adjust per entire-pair-span
+ if (UTF8Active(destatep)) {
+ CheckUTF8Seq(destatep, biggest_weightshift);
+ }
+
+ if (UTF8UTF8Active(destatep)) {
+ CheckUTF8UTF8Seq(destatep, biggest_weightshift);
+ }
+
+ if (Iso2022Active(destatep)) {
+ CheckIso2022ActiveSeq(destatep);
+ }
+
+ if (HzActive(destatep)) {
+ CheckHzActiveSeq(destatep);
+ }
+
+ if (EUCJPActive(destatep)) {
+ CheckEucJpSeq(destatep);
+ }
+
+ if (BinaryActive(destatep) || UTF1632Active(destatep)) {
+ CheckBinaryDensity(src, destatep, delta_otherpairs);
+ }
+ }
+ // ISO-2022 do OK on their own, using stright probabilities? Not on bad bytes
+
+ if (destatep->debug_data != NULL) {
+ // Show sequencing result
+ SetDetailsEncLabel(destatep, "seq");
+ }
+}
+
+
+void PrintTopEnc(DetectEncodingState* destatep, int n) {
+ // Print top n or fewer
+ int temp_sort[NUM_RANKEDENCODING];
+ for (int j = 0; j < destatep->rankedencoding_list_len; ++j) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ temp_sort[j] = destatep->enc_prob[rankedencoding];
+ }
+
+ qsort(temp_sort, destatep->rankedencoding_list_len,
+ sizeof(temp_sort[0]), IntCompare);
+
+ int top_n = minint(n, destatep->rankedencoding_list_len);
+ int showme = temp_sort[top_n - 1]; // Print this value and above
+
+ printf("rankedencodingList top %d: ", top_n);
+ for (int j = 0; j < destatep->rankedencoding_list_len; ++j) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if (showme <= destatep->enc_prob[rankedencoding]) {
+ printf("%s=%d ",
+ MyEncodingName(kMapToEncoding[rankedencoding]),
+ destatep->enc_prob[rankedencoding]);
+ }
+ }
+ printf("\n\n");
+}
+
+// If the same bigram repeats, don't boost its best encoding too much
+bool RepeatedBigram(DetectEncodingState* destatep, uint8 byte1, uint8 byte2) {
+ int this_bigram = (byte1 << 8) | byte2;
+ // If 00xx 01xx 02xx ... 1fxx, take out bottom 4 bits of xx.
+ // This ignores parts of Yahoo 0255 0254 0243 0247 0245 0243 0250 0255 ...
+ // It may screw up UTF-16BE
+ // It may screw up ISO-2022 (1b24 suppresses 1b28)
+ if (byte1 < 0x20) {
+ this_bigram &= 0xfff0;
+ }
+ if (this_bigram == destatep->prior_bigram[0]) {return true;}
+ if (this_bigram == destatep->prior_bigram[1]) {return true;}
+ if (this_bigram == destatep->prior_bigram[2]) {return true;}
+ if (this_bigram == destatep->prior_bigram[3]) {return true;}
+ // Round-robin replacement
+ destatep->prior_bigram[destatep->next_prior_bigram] = this_bigram;
+ destatep->next_prior_bigram = (destatep->next_prior_bigram + 1) & 3;
+ return false;
+}
+
+// Sometimes illegal bytes are used as markers between text that Javascript
+// is going to decode. Don't overboost the Binary encoding for markers 01-FF.
+// Just count first pair per 8x4 bucket
+bool RepeatedBinary(DetectEncodingState* destatep, uint8 byte1, uint8 byte2) {
+ int bucket8x4 = ((byte1 & 0xe0) >> 3) | ((byte2 & 0xc0) >> 6);
+ uint32 bucket8x4_mask = 1 << bucket8x4;
+ if ((destatep->binary_8x4_seen & bucket8x4_mask) == 0) {
+ destatep->binary_8x4_seen |= bucket8x4_mask;
+ destatep->binary_8x4_count += 1;
+ return false;
+ }
+ return true;
+}
+
+
+
+
+// Find current top two rankedencoding probabilities
+void ReRank(DetectEncodingState* destatep) {
+ destatep->top_prob = -1;
+ destatep->second_top_prob = -1;
+ // Leave unchanged
+ //destatep->top_rankedencoding =
+ // destatep->rankedencoding_list[0]; // Just to make well-defined
+ //destatep->second_top_rankedencoding =
+ // destatep->rankedencoding_list[1]; // Just to make well-defined
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if (destatep->top_prob < destatep->enc_prob[rankedencoding]) {
+ // Make sure top 2 are in different superset groups
+ if (kMapEncToBaseEncoding[kMapToEncoding[destatep->top_rankedencoding]] !=
+ kMapEncToBaseEncoding[kMapToEncoding[rankedencoding]]) {
+ destatep->second_top_prob =
+ destatep->top_prob; // old top to second
+ destatep->second_top_rankedencoding =
+ destatep->top_rankedencoding; // old top to second
+ }
+ destatep->top_prob = destatep->enc_prob[rankedencoding];
+ destatep->top_rankedencoding = rankedencoding;
+ } else if (destatep->second_top_prob < destatep->enc_prob[rankedencoding]) {
+ if (kMapEncToBaseEncoding[kMapToEncoding[destatep->top_rankedencoding]] !=
+ kMapEncToBaseEncoding[kMapToEncoding[rankedencoding]]) {
+ destatep->second_top_prob = destatep->enc_prob[rankedencoding];
+ destatep->second_top_rankedencoding = rankedencoding;
+ }
+ }
+ }
+}
+
+void SimplePrune(DetectEncodingState* destatep, int prune_diff) {
+ // Prune the list of active encoding families
+ int keep_prob = destatep->top_prob - prune_diff;
+
+ destatep->active_special = 0;
+ int k = 0;
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ bool keep = true;
+ int rankedencoding = destatep->rankedencoding_list[j];
+
+ // If count is too low, ditch it
+ if (destatep->enc_prob[rankedencoding] < keep_prob) {keep = false;}
+
+ // Keep it. This will always keep at least top_prob rankedencoding
+ if (keep) {
+ destatep->active_special |= kSpecialMask[kMapToEncoding[rankedencoding]];
+ destatep->rankedencoding_list[k++] = rankedencoding;
+ }
+ }
+
+ destatep->rankedencoding_list_len = k;
+}
+
+// Recalculate reliable
+void CalcReliable(DetectEncodingState* destatep) {
+ // Encoding result is reliable if big difference in top two, or if
+ // only Ascii7 ever encountered
+ // Also reliable if exactly one OtherPair and it's best encoding matches top
+ destatep->reliable = false;
+ if (destatep->next_interesting_pair[OtherPair] == 0) {
+ // Only 7-bit ASCII
+ destatep->reliable = true;
+ return;
+ }
+ if ((destatep->top_prob - destatep->second_top_prob) >=
+ FLAGS_ced_reliable_difference) {
+ destatep->reliable = true;
+ return;
+ }
+ if (destatep->next_interesting_pair[OtherPair] == 1) {
+ uint8 byte1 = destatep->interesting_pairs[OtherPair][0];
+ uint8 byte2 = destatep->interesting_pairs[OtherPair][1];
+ int best_enc = kMostLikelyEncoding[(byte1 << 8) + byte2];
+ if (best_enc == destatep->top_rankedencoding) {
+ destatep->reliable = true;
+ return;
+ }
+ }
+
+ // If we pruned to one encoding, we are done
+ if (destatep->rankedencoding_list_len == 1) {
+ destatep->reliable = true;
+ destatep->done = true;
+ return;
+ }
+
+ // If we pruned to two or three encodings in the same *superset/subset
+ // rankedencoding* and enough pairs, we are done. Else keep going
+ if (destatep->rankedencoding_list_len == 2) {
+ Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
+ Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
+ if (kMapEncToBaseEncoding[enc0] == kMapEncToBaseEncoding[enc1]) {
+ if (destatep->prune_count >= 3) {
+ destatep->reliable = true;
+ destatep->done = true;
+ return;
+ }
+ }
+ } else if (destatep->rankedencoding_list_len == 3) {
+ Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
+ Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
+ Encoding enc2 = kMapToEncoding[destatep->rankedencoding_list[2]];
+ Encoding base0 = kMapEncToBaseEncoding[enc0];
+ Encoding base1 = kMapEncToBaseEncoding[enc1];
+ Encoding base2 = kMapEncToBaseEncoding[enc2];
+
+ if ((base0 == base1) && (base0 == base2)) {
+ if (destatep->prune_count >= 3) {
+ destatep->reliable = true;
+ destatep->done = true;
+ return;
+ }
+ }
+ }
+
+}
+
+
+// Find current top two rankedencoding probabilities
+void FindTop2(DetectEncodingState* destatep,
+ int* first_renc, int* second_renc,
+ int* first_prob, int* second_prob) {
+ *first_prob = -1;
+ *second_prob = -1;
+ *first_renc = 0;
+ *second_renc = 0;
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if (*first_prob < destatep->enc_prob[rankedencoding]) {
+ *second_prob = *first_prob; // old top to second
+ *second_renc = *first_renc; // old top to second
+ *first_prob = destatep->enc_prob[rankedencoding];
+ *first_renc = rankedencoding;
+ } else if (*second_prob < destatep->enc_prob[rankedencoding]) {
+ *second_prob = destatep->enc_prob[rankedencoding];
+ *second_renc = rankedencoding;
+ }
+ }
+}
+
+
+void PrintRankedEncodingList(DetectEncodingState* destatep, const char* str) {
+ printf("Current ranked encoding list %s\n", str);
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if ((rankedencoding < 0) || (rankedencoding > NUM_RANKEDENCODING)) {
+ printf(" [%d] BOGUS rankedencoding = %d\n", j, rankedencoding);
+ } else {
+ printf(" [%d] rankedencoding = %d %-12.12s enc_prob = %d\n",
+ j, rankedencoding, MyRankedEncName(rankedencoding),
+ destatep->enc_prob[rankedencoding]);
+ }
+ }
+ printf("End current ranked encoding list\n\n");
+}
+
+
+
+
+// Map unencoded bytes down to five bits, largely preserving letters
+// This design struggles to put 33 values into 5 bits.
+#define XX 0 // Punctuation (00-7F range)
+#define HA 27 // High vowel a in Latin1/2/sometimes7
+#define HE 28 // High vowel e
+#define HI 29 // High vowel i
+#define HO 30 // High vowel o
+#define HU 30 // High vowel u on top of HO
+#define Hc 31 // High consonant (80-FF range)
+static const char kMapToFiveBits[256] = {
+ XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
+ XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
+ XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
+ XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
+
+ XX, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,
+ 16,17,18,19,20,21,22,23, 24,25,26,XX,XX,XX,XX,XX,
+ XX, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,
+ 16,17,18,19,20,21,22,23, 24,25,26,XX,XX,XX,XX,XX,
+
+ Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
+ Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
+ Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
+ Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
+
+ Hc,HA,HA,HA,HA,Hc,Hc,Hc, Hc,HE,HE,HE,HI,HI,HI,Hc,
+ Hc,Hc,Hc,HO,HO,HO,HO,Hc, Hc,HU,HU,HU,HU,Hc,Hc,Hc,
+ Hc,HA,HA,HA,HA,Hc,Hc,Hc, Hc,HE,HE,HE,HI,HI,HI,Hc,
+ Hc,Hc,Hc,HO,HO,HO,HO,Hc, Hc,HU,HU,HU,HU,Hc,Hc,Hc,
+
+};
+#undef XX
+#undef HA
+#undef HE
+#undef HI
+#undef HO
+#undef HU
+#undef Hc
+
+static const int kTriLatin1Likely = 1;
+static const int kTriLatin2Likely = 2;
+static const int kTriLatin7Likely = 3;
+
+// Each table entry has 32 times two bits, selected by byte[2]
+// Entry subscript is selected by byte[0] and byte[1]
+// Latin1/2/7 boost vector, generated 2007.09.26 by postproc-enc-detect-short.cc
+static const uint64 kLatin127Trigrams[1024] = {
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x304080c0402c3330ULL, 0x0008400004000000ULL, 0x082800000c200000ULL,
+0x23a0000420800030ULL, 0x00000000000ccc00ULL, 0x0500100100100000ULL, 0x0388400000200010ULL,
+0x0000000000000c00ULL, 0xd0f0300740f0cf00ULL, 0x2aa0a2a22882a2acULL, 0x081d800000000080ULL,
+0x0c82000020000000ULL, 0x200a03c000a00000ULL, 0x0008400400290000ULL, 0x0400870000000000ULL,
+0x00f040c00000c080ULL, 0x0008004000000410ULL, 0x0020300000000030ULL, 0x00a030002c300000ULL,
+0x0c8030c020a00000ULL, 0x15410030f0f4c000ULL, 0x3000000300a00000ULL, 0xa2880980a0880a88ULL,
+0x0900300000000000ULL, 0x0000040100300000ULL, 0x0888820020a00000ULL, 0xc044002242010000ULL,
+0x000000121d300040ULL, 0x40100040440c0d54ULL, 0x00008423102f8144ULL, 0x0b40808400000280ULL,
+0x0000000000000000ULL, 0x0680a000000c0000ULL, 0x0880008020aa0000ULL, 0x2aaa0141010a4940ULL,
+0xcb80000000010000ULL, 0x2280000000000000ULL, 0x5248000001800000ULL, 0x8000401004040010ULL,
+0x1540010201001010ULL, 0x0080080400000000ULL, 0x5a00044040000108ULL, 0x0288000282080008ULL,
+0x4800008002200000ULL, 0x4a00000000010100ULL, 0x8a88040080000800ULL, 0x0140800000000400ULL,
+0x40010050000c0000ULL, 0x0000008000000000ULL, 0x0028000020140040ULL, 0x8620401401005308ULL,
+0xc082000000000400ULL, 0x05c0b004c0240600ULL, 0x0288000080000000ULL, 0x0000014000000000ULL,
+0x00000000040000c0ULL, 0x8001861008004280ULL, 0x0200000000000300ULL, 0x0000240242288620ULL,
+0x801000c05434c200ULL, 0x9020162040a2d2b4ULL, 0x0021840000240704ULL, 0x2a80280080084908ULL,
+0x0000000000000000ULL, 0x0500004000000040ULL, 0x0080000000040000ULL, 0x0108058104440000ULL,
+0x0900000000040000ULL, 0x00c0000000208008ULL, 0x2000005000000000ULL, 0x0080000000050000ULL,
+0x0808000000001080ULL, 0x9880810100308000ULL, 0x2285480080081a08ULL, 0x8a80000080080000ULL,
+0x1450000000600010ULL, 0x2210000100000000ULL, 0x8a88000100011000ULL, 0x1541804000000010ULL,
+0xc084011140040100ULL, 0x0000000000000800ULL, 0x0400000000000030ULL, 0x2a800000a0890128ULL,
+0x1140a00054000104ULL, 0x1440000101200404ULL, 0x028800400400d800ULL, 0x0000000000000000ULL,
+0x0000000000002330ULL, 0x0020820228a02280ULL, 0xa2888a02aa8008a8ULL, 0xd0040a0044202500ULL,
+0x8000044104a29424ULL, 0xc000100178b2c5b4ULL, 0x0000810100241504ULL, 0xd040030000380008ULL,
+0x0000000000000000ULL, 0x26c08c0000200130ULL, 0x4a08000110080000ULL, 0x2aa0004001080800ULL,
+0x0aac000000004000ULL, 0x2000000000200000ULL, 0x4240000100020000ULL, 0x4100000080000000ULL,
+0x4900040000000000ULL, 0x0800000400300040ULL, 0x6a80000000040800ULL, 0x2a08182000588008ULL,
+0x0a00000c81000008ULL, 0x0a000c0010000000ULL, 0x8a88001080280808ULL, 0x0020000200300600ULL,
+0xaac00000900a0000ULL, 0x0000100004000000ULL, 0x0020081020000000ULL, 0x8220105010084110ULL,
+0x4a80800000004000ULL, 0x050000c0c0200000ULL, 0x288c000084000000ULL, 0xa048082280000000ULL,
+0x0000000000000000ULL, 0x8000900000032080ULL, 0xee889e81b8880820ULL, 0xc2200a8142800424ULL,
+0xc020141543361010ULL, 0x10a000204a801634ULL, 0x3a808800802a00a0ULL, 0x28808b00803d0800ULL,
+0x0000000000000000ULL, 0x0020000000000030ULL, 0x0808400121010040ULL, 0x0c28240100200040ULL,
+0x2008200028800000ULL, 0xc10004c80f30c030ULL, 0x0400440114100000ULL, 0x2208200280a22220ULL,
+0x0600000030c01000ULL, 0x1201001040c00000ULL, 0x0aa02ea22aa22aa0ULL, 0x30008000000200a0ULL,
+0x20c8400400800000ULL, 0x08280b0420800000ULL, 0x0800100000210000ULL, 0x10000300c0100400ULL,
+0xc8c0000420000000ULL, 0x1000000010000000ULL, 0x0420000400000000ULL, 0x0220000500204000ULL,
+0x2200000420000000ULL, 0x0000540400000000ULL, 0x0000000020000000ULL, 0x00080c00a0810080ULL,
+0x1540000000043000ULL, 0x0000000000100000ULL, 0x2e88a22220200a20ULL, 0xc06030e34ea503a0ULL,
+0x0001100204048500ULL, 0x000000e0000c0d54ULL, 0x3000820310a31400ULL, 0x13088c0320e00280ULL,
+0x0000000000000000ULL, 0x0480000000200000ULL, 0x4000200100000000ULL, 0x0000300040040000ULL,
+0x4400000000000000ULL, 0x0401000002240000ULL, 0x0540000000040000ULL, 0x4004010000000000ULL,
+0x4001111001100000ULL, 0x2880000000300040ULL, 0x4040004040002404ULL, 0x0200000000000000ULL,
+0x0140040000100000ULL, 0x4040010040040080ULL, 0x0a00140000041004ULL, 0x0000a00400808000ULL,
+0x1010200000430040ULL, 0x0010000000000000ULL, 0x0540000000104000ULL, 0x1400114005000000ULL,
+0x0000204000440010ULL, 0x0500000000004400ULL, 0x4500000018000400ULL, 0x0000400000000000ULL,
+0x000000300000cc00ULL, 0x0100001011300000ULL, 0x0040000000000000ULL, 0xc0e0000248a00444ULL,
+0x0000040020340144ULL, 0x0000046445105454ULL, 0x32a0a80280880128ULL, 0x0880040000100100ULL,
+0x0000000000000000ULL, 0x14003000030c0004ULL, 0x4a04001100000000ULL, 0x0a00108010000000ULL,
+0x28a8004000200248ULL, 0x0100040000b00000ULL, 0x42000000000008c0ULL, 0x6008044010550010ULL,
+0x0800401000010400ULL, 0x080080040cf80000ULL, 0x5080000001001010ULL, 0x2a80100000000000ULL,
+0xcc8010010d401100ULL, 0x0200000001001000ULL, 0x0480001004001000ULL, 0x8d00800040b40210ULL,
+0x6200800000300000ULL, 0x0000010000000000ULL, 0x0428004100010000ULL, 0x4320105141501100ULL,
+0xe28c0000000c1000ULL, 0xd5c000c3c0e00300ULL, 0x0001000000100200ULL, 0x1004010202400008ULL,
+0x0000000000003000ULL, 0x2aa038a0800aab08ULL, 0x2a88038000000000ULL, 0xc220040242f09720ULL,
+0x8020200200ba0420ULL, 0x0020106105101004ULL, 0x0480800000220400ULL, 0x2280100080000008ULL,
+0x0000000000000000ULL, 0x9000000000200000ULL, 0x0001000000100000ULL, 0x2aa40c0000080800ULL,
+0x0040000040010000ULL, 0x0040000000c01000ULL, 0x4000000040000400ULL, 0x0000001000200000ULL,
+0x0000010000000000ULL, 0x05808004000c0000ULL, 0x50400c0000000400ULL, 0x020040008f000040ULL,
+0x0800000000100000ULL, 0x0000000000000000ULL, 0x0a08440000004000ULL, 0x0064000400008200ULL,
+0x0010010010034170ULL, 0x0000000010000000ULL, 0x0100204021000000ULL, 0x022000d000010100ULL,
+0x0840300000c00000ULL, 0x1400000040204400ULL, 0x09800c0040000000ULL, 0x0209708000000000ULL,
+0x000000000000c040ULL, 0x90000c50204040a0ULL, 0x0000000000000000ULL, 0x00e1500040200004ULL,
+0x8020260540204494ULL, 0x0020026150201054ULL, 0x0281800380105634ULL, 0x0884900481105000ULL,
+0x0000000000000000ULL, 0x84203c00002c0200ULL, 0xc089040000000000ULL, 0xc2a8100040200004ULL,
+0xe00c1c0000000000ULL, 0x0ce1330080200080ULL, 0x0000000000200000ULL, 0xc400110000404010ULL,
+0x0088400000000000ULL, 0x00083cc00c00c00cULL, 0xcac01c00c000580cULL, 0xe300b0f000100000ULL,
+0x0300000000000000ULL, 0xc0000f0000000000ULL, 0xc3c01c0400000000ULL, 0x81008004c0f40000ULL,
+0xc3d8003000000440ULL, 0x0000000000000000ULL, 0xc430000000000000ULL, 0x0060000000001000ULL,
+0x0800000000000000ULL, 0x00c03300f0fc0008ULL, 0x3000000400200010ULL, 0xa2a80892a0880a28ULL,
+0x0500000040000004ULL, 0x0000000000000000ULL, 0xc80032070c200020ULL, 0x0220820060a296a0ULL,
+0x802084021db486a0ULL, 0x00000d60080c0080ULL, 0xb281803313a32428ULL, 0x1808300320300000ULL,
+0x0000000000000000ULL, 0x85208cc0ccac1f20ULL, 0x2081000186100808ULL, 0x22a80880000a0808ULL,
+0xaaa8086880000000ULL, 0x802084800a2e9200ULL, 0xa280000000002008ULL, 0xa000000080080400ULL,
+0x2080010000000008ULL, 0x802020c00c028c80ULL, 0x2080000000140810ULL, 0x2a80086080080008ULL,
+0x2a800000a8000800ULL, 0xaa881800a2080800ULL, 0xaa98004080280808ULL, 0x004483d0c0300000ULL,
+0xa280002080080000ULL, 0x0000000000300000ULL, 0x22a1030000000008ULL, 0xa8a0301088880880ULL,
+0xaa80002080222808ULL, 0x85400c03fc030400ULL, 0x8a88000000000008ULL, 0xa008008010080008ULL,
+0x0000000000010000ULL, 0x0040100000301040ULL, 0x28800000a0002008ULL, 0x122482306cbc0eacULL,
+0x8020224222b8c6a0ULL, 0x802002004a82c284ULL, 0x0aa08fc440a41c80ULL, 0x888080d181385098ULL,
+0x0000000000000000ULL, 0x00c0b000000c0080ULL, 0x2208001000000800ULL, 0x0a28000000200000ULL,
+0x0000000300000000ULL, 0x00c1040000200000ULL, 0x0203020000000000ULL, 0x0248000000020000ULL,
+0x0000840000100000ULL, 0x0a808c00c000008cULL, 0x5200040040000004ULL, 0x02000c00000080a0ULL,
+0x0b0c000020000000ULL, 0x0b04000001000000ULL, 0x088c0010002000c0ULL, 0x80e08b00c0030c20ULL,
+0x0280000200014040ULL, 0x0000000000000000ULL, 0x0e20a0a008000020ULL, 0x0e280fd03f00111cULL,
+0x200080c020001000ULL, 0x8cc00c02c02f0400ULL, 0x480c0001000c404cULL, 0x0208014281080808ULL,
+0x000000000000fcfcULL, 0x004403300cf00030ULL, 0x2200000000004400ULL, 0x02202000c08c0c20ULL,
+0x02202022683a80a0ULL, 0x4020228028008c00ULL, 0x32208cc0002c0200ULL, 0x3ec00c0080304008ULL,
+0x0000000000000000ULL, 0x34000c00002c0000ULL, 0x0b00000100100030ULL, 0x0823018000000000ULL,
+0x0e8c001c01e00000ULL, 0x1200800600330000ULL, 0x4000110000000000ULL, 0x0080000300000000ULL,
+0x0800000000000000ULL, 0x08c08c04000c0000ULL, 0x0080400000880000ULL, 0x0a08000080c00008ULL,
+0x0800000304400000ULL, 0x0208000000c00000ULL, 0x2888300080400800ULL, 0x8dc0204400000000ULL,
+0xc0000000c0800000ULL, 0x0000c10000000000ULL, 0x24000c4010c00000ULL, 0x272000541d811000ULL,
+0x0200400000001000ULL, 0x0400000400001004ULL, 0xc08c007004001000ULL, 0x2048004000000000ULL,
+0x000000000003fcfcULL, 0x2aa030000cf8c800ULL, 0xe280000000000000ULL, 0x0a21008142000340ULL,
+0x0021002000b61040ULL, 0x800004064006d444ULL, 0x3aa0800300230008ULL, 0x0b00030000300000ULL,
+0x0000000000000000ULL, 0x01c080000000040cULL, 0x0100000000004000ULL, 0x0aa8018010001000ULL,
+0x0800000000100000ULL, 0x3000000000008c00ULL, 0x5400000013000000ULL, 0x02c0c00004004010ULL,
+0x5241100010000c00ULL, 0x0e00080000000808ULL, 0x5281000000000800ULL, 0x0a08108020000800ULL,
+0x0a80000000005210ULL, 0x0100000041000000ULL, 0x2a88000002080110ULL, 0x8520800000c00080ULL,
+0x01000010108c0100ULL, 0x0000000000000000ULL, 0x42a0420080000000ULL, 0x0020001004010010ULL,
+0xc4000000000c0000ULL, 0x01000c00c0200400ULL, 0x4600000100000000ULL, 0x0000000000000000ULL,
+0x0010001000000010ULL, 0x910400900820d030ULL, 0x2280000000000000ULL, 0xc2212004400040e4ULL,
+0x8001000000b61420ULL, 0xa00002a248e810b4ULL, 0x32008000002c0008ULL, 0x0c010034803c5010ULL,
+0x0000000000000000ULL, 0x85008002002c0000ULL, 0x0204001000004010ULL, 0x0120008000200000ULL,
+0x000010000c2000c0ULL, 0xccc0000000200000ULL, 0x0400000c00100040ULL, 0x0003300100004100ULL,
+0x4000551040000004ULL, 0x0e0080000c820808ULL, 0xc000000000080800ULL, 0xc803000000000000ULL,
+0x0a4000c000200000ULL, 0x0040000000c00000ULL, 0x0918145000405000ULL, 0x81400000c0300400ULL,
+0x0050000000000000ULL, 0xd000045000000000ULL, 0x0400004000400000ULL, 0x0420104010000110ULL,
+0x0700000000203000ULL, 0x34800300c0e00704ULL, 0x4440100044000400ULL, 0x0040000040000000ULL,
+0x0030000044000000ULL, 0xeaaca0008808c880ULL, 0x0a01000000200000ULL, 0x1220a300403ccf20ULL,
+0x002024c200b61044ULL, 0x802014346aa2d434ULL, 0x30008c00c0820c44ULL, 0x0a000000000c4800ULL,
+0x0000000000000000ULL, 0x0000404000340c90ULL, 0x08a8a10820800280ULL, 0x8128009022201000ULL,
+0x0020808228a000a0ULL, 0x0020400100410000ULL, 0x0400000110000000ULL, 0xa609000000200000ULL,
+0x8008330000d00000ULL, 0x8060100040404010ULL, 0xeaa00ea0ea00808cULL, 0x200c8020a0000020ULL,
+0x0408800020200000ULL, 0x0189001403200000ULL, 0xc00800000000c000ULL, 0x200430c00c300000ULL,
+0x0100300100004000ULL, 0x0000040000000000ULL, 0x2420000400001000ULL, 0x89a1200400000000ULL,
+0x20c8a000208c0000ULL, 0x8080000000000000ULL, 0x28a0108020210080ULL, 0xa2a84800a0880988ULL,
+0x258008000400c000ULL, 0x0140000000100000ULL, 0xa028a222a0aa0228ULL, 0xc060012054044040ULL,
+0x0010010400000000ULL, 0x00000050150c0114ULL, 0x0000008010c20010ULL, 0xaa088000a0200880ULL,
+0x0000000000000000ULL, 0x0700b0c0000c0000ULL, 0x2200040000080030ULL, 0x2aa8808040240800ULL,
+0x08b0500000000100ULL, 0x1000830400200000ULL, 0x4204000010000000ULL, 0x40c2200050040050ULL,
+0x0104404001010000ULL, 0x1a808c8103c00030ULL, 0x30900010c0000b00ULL, 0x200812b283000008ULL,
+0x000c000020e00000ULL, 0x2140000000400000ULL, 0x0288000080200000ULL, 0x8060a200c8a20280ULL,
+0x0400114010215000ULL, 0x0000000000000000ULL, 0x082b200002000010ULL, 0x22a0030000031000ULL,
+0x008100001000000cULL, 0x05400c00c0230400ULL, 0xca3000003c080100ULL, 0x0000000020000004ULL,
+0x0000000100000000ULL, 0x8004320813f5c000ULL, 0xa280080200000800ULL, 0xc22000044e334c20ULL,
+0x000004146e361024ULL, 0x800126806aa0d584ULL, 0xb000a0040023c41cULL, 0x0a083000803053d8ULL,
+0x0000000000000000ULL, 0x0000100000020000ULL, 0x0000000010000010ULL, 0x0000000045040004ULL,
+0x0000000000100000ULL, 0x0000020400000010ULL, 0x0003015000000000ULL, 0x0400000000000000ULL,
+0x0000000400000000ULL, 0x0100000000000800ULL, 0x0000001000000000ULL, 0x0000000000000000ULL,
+0x0000000040000000ULL, 0x0000000000000000ULL, 0x0004001000000000ULL, 0x0008001000000000ULL,
+0x0010000000000004ULL, 0x0000010100001000ULL, 0x0004000000000004ULL, 0x0000014040050014ULL,
+0x0014000000000040ULL, 0x5540000000041000ULL, 0x0000000000000000ULL, 0x0000040000000d00ULL,
+0x0000000000000000ULL, 0x0000000000100000ULL, 0x0001000000000000ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x4500000000040400ULL, 0x0000800000000400ULL,
+0x0000000000000000ULL, 0x13e080000020000cULL, 0xcf00001005100000ULL, 0x04a8008000200300ULL,
+0x00280100100000c0ULL, 0x1c8c000040200000ULL, 0x0600005000100000ULL, 0x050800000c104000ULL,
+0x4c10101000110000ULL, 0x0c00000000300000ULL, 0x22040c00100000c0ULL, 0x0800700010100000ULL,
+0x0000000000001000ULL, 0x0a08000010000040ULL, 0x0800034004210010ULL, 0x04e0000400000000ULL,
+0x0800030020000000ULL, 0x0000005000000000ULL, 0x0400110101304110ULL, 0x0428000010a01000ULL,
+0x060b000000800010ULL, 0x35810c00c020c000ULL, 0x00800c4321800000ULL, 0x4208088020000080ULL,
+0x040000111003ff00ULL, 0x0020900020202080ULL, 0x22888180a8000888ULL, 0x0225200542005420ULL,
+0x2020040400340020ULL, 0x10300424500cc444ULL, 0x3081a00400e00200ULL, 0x33001300c0300000ULL,
+0x0000000000000000ULL, 0x04003c0000000000ULL, 0x0a04001000100100ULL, 0x1408000001000000ULL,
+0x1800000044100000ULL, 0x3400040400000300ULL, 0x5000040801000040ULL, 0x4088401040000040ULL,
+0x1010110130100000ULL, 0xca800c3000300000ULL, 0x5a01000000080100ULL, 0x020280000cd01300ULL,
+0x0302000410200010ULL, 0x0000102000300000ULL, 0x0b09000000000000ULL, 0x20008004c4800004ULL,
+0x28c0410010000000ULL, 0x0004015041000050ULL, 0x0a01006000200200ULL, 0x0020d00000100040ULL,
+0x0010a00100900000ULL, 0x3500bf00c0030300ULL, 0x080c010000200d00ULL, 0x2248000004020010ULL,
+0x0000c00000000000ULL, 0x8044b00200e08000ULL, 0xaaa82aa2aa8a2aa8ULL, 0x0220002241c08604ULL,
+0x4200260440328444ULL, 0x68001226103008b4ULL, 0x3a0080c0b0000400ULL, 0x2a804804803c4008ULL,
+0x0000000000000000ULL, 0x04008c0300000400ULL, 0x008000c0000c0000ULL, 0x088001000000001cULL,
+0x0840000001000010ULL, 0x0400000000200c00ULL, 0x4244000101040000ULL, 0x4238007011100000ULL,
+0x1000d00100000010ULL, 0x1d00800400300000ULL, 0x4204080c00000000ULL, 0x2a88080080000008ULL,
+0x08001c0200001000ULL, 0x0a00000400000000ULL, 0x8a88003080080000ULL, 0x0521800400300000ULL,
+0x3200051000201000ULL, 0x0000000000000000ULL, 0x0020801404000000ULL, 0x322010401c0c101cULL,
+0x0c01100013000000ULL, 0x04003000c0204000ULL, 0x088c0020a0cc0000ULL, 0x2200000080000018ULL,
+0x0404000044000000ULL, 0x82a0b000008820b0ULL, 0x0000040020440000ULL, 0xc2650004403f1420ULL,
+0x0021340241b64464ULL, 0x8020040242c2d474ULL, 0x32018c0480288000ULL, 0x00800b0080300000ULL,
+0x0000000000000000ULL, 0x05008c0000040130ULL, 0xc0d8000000800000ULL, 0x0020000020200200ULL,
+0x23a2000120204000ULL, 0x5052100550104150ULL, 0x1000101100040000ULL, 0xc40001c301000000ULL,
+0x8288000000c00000ULL, 0x5150040144d01404ULL, 0xea8c0ea028ae088cULL, 0xc31010c000000c80ULL,
+0x0002000060000000ULL, 0xc80800f030000000ULL, 0x0000000400300000ULL, 0xc00080c00ff0c344ULL,
+0x00080001200c0000ULL, 0x0000050080000000ULL, 0x0328000300300000ULL, 0x082030000cc01040ULL,
+0xeb08800100004000ULL, 0x8030003300c80f00ULL, 0xfb0d0000e4ac0000ULL, 0x0020006080000008ULL,
+0x0500100100040000ULL, 0x1140000000000000ULL, 0xcb883330a0e00000ULL, 0xc000010050000080ULL,
+0x0010104005b54150ULL, 0x40111d5155001554ULL, 0x80000070140f0004ULL, 0x0b0830c3a0003380ULL,
+0x0000000000000000ULL, 0x04c13000000f830cULL, 0x2808000000000000ULL, 0x2810000000000800ULL,
+0x08c0080004400000ULL, 0x04c0240300801c20ULL, 0x4040000080000004ULL, 0x0000400100100010ULL,
+0x020001008000c0c0ULL, 0x1d008c000c3c0000ULL, 0x0080003000000800ULL, 0x2288080080000008ULL,
+0x0a84004020220000ULL, 0x0800080000100000ULL, 0xaa80004080400008ULL, 0x8024000400c01660ULL,
+0x80841c2001000104ULL, 0x0001000000000000ULL, 0x0020028020020280ULL, 0x0860404011900100ULL,
+0xec80080200000000ULL, 0x010103c100200400ULL, 0x0200004000000000ULL, 0x0000000000400400ULL,
+0x000010000003fcfcULL, 0x8040083238c20000ULL, 0x08800220a0920a00ULL, 0x08210004483c0c24ULL,
+0xc020240740b0a200ULL, 0x802006014a201494ULL, 0x3201233070ac0e00ULL, 0x08002806033a48a0ULL,
+0x0000000000000000ULL, 0x8020820028a00680ULL, 0x2000002000000104ULL, 0x22a80801100a0808ULL,
+0xa2a8002080000000ULL, 0xa000800008a08000ULL, 0x0000100000400000ULL, 0x8000002100000000ULL,
+0x0000010000004404ULL, 0xa2a0088080000888ULL, 0x0000000010400800ULL, 0xa280082080080008ULL,
+0x2280000080010008ULL, 0x2000000000000000ULL, 0x228800008c080808ULL, 0x8021828002a98200ULL,
+0xa200002000080000ULL, 0x0000040000000000ULL, 0x22a0000080000000ULL, 0x202882c200800080ULL,
+0xa000000001004000ULL, 0x000000c808a00600ULL, 0x0000000010000000ULL, 0x000001000000040cULL,
+0x0000000000000000ULL, 0x802002a2a8aa82a0ULL, 0x20000024a8088228ULL, 0x8020820001000000ULL,
+0x8020000000808280ULL, 0x8000000000000000ULL, 0x0020800000200280ULL, 0x2080082280a00888ULL,
+0x0000000000000000ULL, 0x0000015000000040ULL, 0x0000040000040000ULL, 0x0100010010001000ULL,
+0x0000003210008000ULL, 0x0000000404000000ULL, 0x0000000000000400ULL, 0x0200000000000000ULL,
+0x0000000000000100ULL, 0x5180014400004050ULL, 0x1000000014000000ULL, 0x4200000000000000ULL,
+0x0040200000000000ULL, 0x0201004000000000ULL, 0x0a00000000000010ULL, 0x0040200000800000ULL,
+0x0040051000000500ULL, 0x0000000100800400ULL, 0x6000000000000000ULL, 0x0000000000000000ULL,
+0x280000c1400040ccULL, 0x4180001000000000ULL, 0x00000000c1000104ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x0000000000000000ULL, 0x0080000000c00000ULL, 0x0004006066004000ULL,
+0x0000005000040440ULL, 0x0000106005804044ULL, 0x0000a10511004440ULL, 0x0000000000000110ULL,
+0x0000000000000000ULL, 0x0000000000080000ULL, 0xeb0808a020800080ULL, 0x29a80081002a1800ULL,
+0x0b2c000202100100ULL, 0x0001000000888000ULL, 0x2280102010000000ULL, 0x020000602a004110ULL,
+0x8a800160a6108100ULL, 0x0280000000000020ULL, 0x8a8000a0a8808208ULL, 0x0280882080500308ULL,
+0x0b18010020804100ULL, 0xeb080000c0080080ULL, 0x2b08000000810130ULL, 0x0000000008040020ULL,
+0xaa0a08e082894140ULL, 0x0000000000000000ULL, 0x202081409010001cULL, 0x8aa8805082806000ULL,
+0xeb082900289c0000ULL, 0x0000000000008000ULL, 0xf80c2e20002e0000ULL, 0xa288080420880888ULL,
+0x0000010000000000ULL, 0x0000000000102000ULL, 0x22880000a8a80808ULL, 0x022022a22aa880a0ULL,
+0x0000222222aa0620ULL, 0x0000022002800000ULL, 0x208080004028a000ULL, 0x2b888800801c0828ULL,
+0x0000000000000000ULL, 0x22e0828280a08028ULL, 0xaa88002082080308ULL, 0x0ea80080410a0040ULL,
+0x2a28222000a00000ULL, 0x8aa2808028a0a2a0ULL, 0x0200001000000000ULL, 0x82080000a0000000ULL,
+0x8800000082000808ULL, 0x2a008a0000300888ULL, 0x0a80080080080808ULL, 0xaa882800840b0808ULL,
+0x0a80000080000040ULL, 0xea080820a0000000ULL, 0xaa88080080080808ULL, 0x8040a2800a8024a0ULL,
+0xaa800020a0080808ULL, 0x0000040000000000ULL, 0x2a280a0080080880ULL, 0x2a20081080008a00ULL,
+0x2a88882088aa0008ULL, 0x81800202c0a01480ULL, 0xea88082082200000ULL, 0xaa88002080080008ULL,
+0x0000100000000000ULL, 0x802082a22aa0a2a0ULL, 0x2e80000000000000ULL, 0x0220a2a26aa0a2a8ULL,
+0x800022a2228a22a0ULL, 0x880002212e82c0b0ULL, 0x02a0aa0002a82228ULL, 0x2d808b0080380008ULL,
+0x0000000000000000ULL, 0x000407551c154244ULL, 0x2a00208088a02228ULL, 0x12a82182a2402a88ULL,
+0xe32821e020826d00ULL, 0x801130100ccc1330ULL, 0x028010c000841008ULL, 0x88a08002a0a664a0ULL,
+0x0048270080000100ULL, 0x00001f010cd10f30ULL, 0xe2242ce22aaea2a0ULL, 0xc2c00cc20ae22460ULL,
+0xe208003128021c10ULL, 0x2a2021c010821080ULL, 0x2a88202082202020ULL, 0x4010111104941410ULL,
+0xc80c02c182b00080ULL, 0x0000040000000000ULL, 0xe28030068002c300ULL, 0x2aa02024a2a22228ULL,
+0xe20889328aa22080ULL, 0x0000000000210100ULL, 0xaa0028e0a9b221a0ULL, 0x2000008080400000ULL,
+0x0000010041150404ULL, 0x0000105114410100ULL, 0xeaa82aa6aaaaaaa8ULL, 0x000000f44300c434ULL,
+0x0000222222b00020ULL, 0x0000002000000000ULL, 0x0000004014000000ULL, 0x0039b3f73fbcd3fcULL,
+0x0000000000000000ULL, 0x0000104015045040ULL, 0x20a80490a08800a0ULL, 0x40a8258410a909a0ULL,
+0xe0a8a2022aa2e2a0ULL, 0xc111010014000500ULL, 0x2080044041840004ULL, 0x28a8200220a2aba0ULL,
+0x008400a0a2840800ULL, 0x0101015451009464ULL, 0x20000ea0e02c2c2cULL, 0xe2a828a2aca2aaa8ULL,
+0x682020a228a222a0ULL, 0xe8882ae22aa2a2a0ULL, 0xe9a80e6022a24140ULL, 0x0011055005001040ULL,
+0x2aa8208229a0aaa4ULL, 0x0000040000000000ULL, 0x28a0228026a62260ULL, 0xe2a020a422a2a020ULL,
+0xe808a0022aa1a220ULL, 0x0000010014000100ULL, 0x28ac22802aa2a020ULL, 0x0020000000000000ULL,
+0x0100010100040000ULL, 0x0000000000000000ULL, 0x22a822a22a8aaaa0ULL, 0x0000000000000000ULL,
+0x0000102410800100ULL, 0x0000000000000000ULL, 0x0000000002000000ULL, 0x00000fb2a08c0aa8ULL,
+0x0000000000000000ULL, 0x4010005015440140ULL, 0x18c81c00b180001cULL, 0x2800048021820800ULL,
+0x8ab820c06a802580ULL, 0x00100170f4040000ULL, 0x4000144041041404ULL, 0x0ac800d0002e440cULL,
+0x20880820a2000808ULL, 0x400000f03f300c00ULL, 0xaa000ea22aa22aa0ULL, 0xa2880ac0a8942a20ULL,
+0xaa880a81a1804188ULL, 0xeea022a0aaa02080ULL, 0xaaa820a2aaa66120ULL, 0x0000005115800150ULL,
+0x2a880920a0840040ULL, 0x0000040000000000ULL, 0xaea82222aaa22a28ULL, 0x8a28041260055150ULL,
+0xa28824008aa28880ULL, 0x0000025014019000ULL, 0xea882ae02aa200a0ULL, 0x0000000000000000ULL,
+0x0000000040000400ULL, 0x0000000000000000ULL, 0xaaa82aa22aaaaaa0ULL, 0x0000000000000000ULL,
+0x0000000000000000ULL, 0x002003003c80c000ULL, 0x0000020014000000ULL, 0x00200010a0980a20ULL,
+0x0000000000000000ULL, 0x0020001200801240ULL, 0x0a88000089800020ULL, 0xcaa00080a1000000ULL,
+0x0a200c0020a04080ULL, 0x4002034003840880ULL, 0x4690500190000050ULL, 0x2228004000601000ULL,
+0x0a803f00803f400cULL, 0x400033e24dd0cf34ULL, 0xaa80a2a229a220a0ULL, 0x0a224000002c0000ULL,
+0x028000202000008cULL, 0x0a08000070000030ULL, 0x00800c040020000cULL, 0x0000000002850000ULL,
+0x02881cc310200000ULL, 0x0000040004000000ULL, 0xcba8000400000080ULL, 0xcaa02c0680000000ULL,
+0xcc880002008c4080ULL, 0x300000f007f0cf0cULL, 0x0a80001080a00000ULL, 0x820880802a880a80ULL,
+0x0000050001040004ULL, 0x0000011000000000ULL, 0x0a8020a2a0202000ULL, 0x0000022202008000ULL,
+0x0000222212808000ULL, 0x0020226010000000ULL, 0x000033f33ff3c33cULL, 0x00288002a08c02a8ULL,
+0x0000000000000000ULL, 0x04408e0000008200ULL, 0x0808004000900000ULL, 0x0aa8200010ca00c0ULL,
+0x0ba80101005d4010ULL, 0x00018604802c8288ULL, 0x00049400101c0000ULL, 0x000c101110505010ULL,
+0x0000000000100000ULL, 0x30000c00c022000cULL, 0xd0c00dd0d51d431cULL, 0x0008000010100000ULL,
+0x000c1001a0280000ULL, 0x0bc80000c0000000ULL, 0x0a00000080280000ULL, 0x8000a00220308420ULL,
+0x0808000010301000ULL, 0x0000040000000000ULL, 0x0d00031480100000ULL, 0x07200000108c0300ULL,
+0x0bc0a0c000004000ULL, 0x8000b002c0208480ULL, 0x340c0100118c111cULL, 0x8008008020890000ULL,
+0x0000000000040010ULL, 0x0020b00320c1d0b0ULL, 0x00002000000c0000ULL, 0x0020be226e2008a0ULL,
+0x002010c03fb0a6a0ULL, 0x00202e222aaec284ULL, 0x00008f0000208400ULL, 0x0000000000300000ULL,
+};
+// Latin1 6%, Latin2 11%, Latin7 3%
+
+
+
+// Just for debugging. not thread-safe
+static char tri_string[4];
+char* Latin127Str(int trisub) {
+ tri_string[0] = "_abcdefghijklmnopqrstuvwxyzAEIOC"[(trisub >> 10) & 0x1f];
+ tri_string[1] = "_abcdefghijklmnopqrstuvwxyzAEIOC"[(trisub >> 5) & 0x1f];
+ tri_string[2] = "_abcdefghijklmnopqrstuvwxyzAEIOC"[(trisub >> 0) & 0x1f];
+ tri_string[3] = '\0';
+ return tri_string;
+}
+
+// Returns two bits per three-byte trigram, indicating
+// dont-care, Latin1 likely, Latin2 likely, and Latin7 (ISO-8859-13) likely
+int TrigramValue(const uint8* trisrc) {
+ int byte0_p = kMapToFiveBits[trisrc[0]];
+ int byte1_p = kMapToFiveBits[trisrc[1]];
+ int byte2_p = kMapToFiveBits[trisrc[2]];
+ int subscr = ((byte0_p) << 5) | byte1_p;
+ int temp = static_cast<int>((kLatin127Trigrams[subscr] >> (byte2_p * 2)));
+ //printf("%s=%d ", Latin127Str((subscr << 5) | byte2_p), temp & 3);
+ return temp & 3;
+}
+
+
+// Put out trigrams for surrounding 32 bytes for Latin encodings
+// Return true if more Latin2 & 7 than Latin1
+bool BoostLatin127Trigrams(int tri_block_offset,
+ DetectEncodingState* destatep) {
+ //printf("BoostLatin127Trigrams[%06x]\n", tri_block_offset);
+ int excess_latin27 = 0;
+ int srclen = destatep->limit_src - destatep->initial_src;
+ int hi_limit = minint(tri_block_offset + 32, srclen - 2);
+ const uint8* trisrc = &destatep->initial_src[tri_block_offset];
+ const uint8* trisrclimit = &destatep->initial_src[hi_limit];
+ while (trisrc < trisrclimit) {
+ // Selectively boost Latin1, Latin2, or Latin7 and friends
+ int trigram_val = TrigramValue(trisrc);
+ if (trigram_val != 0) {
+ if (FLAGS_enc_detect_source) {
+ PsHighlight(trisrc, destatep->initial_src, trigram_val, 1);
+ }
+ if (trigram_val == kTriLatin1Likely) {
+ Boost(destatep, F_Latin1, kTrigramBoost);
+ Boost(destatep, F_CP1252, kTrigramBoost);
+ // We don't want to upset the relative rank of a declared 8859-15
+ Boost(destatep, F_ISO_8859_15, kTrigramBoost);
+ --excess_latin27;
+ } else if (trigram_val == kTriLatin2Likely) {
+ Boost(destatep, F_Latin2, kTrigramBoost);
+ Boost(destatep, F_CP1250, kTrigramBoost);
+ ++excess_latin27;
+ } else if (trigram_val == kTriLatin7Likely) {
+ Boost(destatep, F_ISO_8859_13, kTrigramBoost);
+ Boost(destatep, F_CP1257, kTrigramBoost);
+ // We don't want to upset the relative rank of a declared 8859-4 or -6
+ // for Estonian
+ Boost(destatep, F_Latin4, kTrigramBoost);
+ Boost(destatep, F_Latin6, kTrigramBoost);
+ ++excess_latin27;
+ }
+ }
+
+ ++trisrc;
+ }
+ //printf("\n");
+
+ return (0 < excess_latin27);
+}
+
+
+
+// Boost any encodings that need extra detection help, then prune
+// src is first unscanned byte
+// slowend means extra pruning when dropping out of initial slow scan
+// final means last call -- no bigram at src
+void BoostPrune(const uint8* src, DetectEncodingState* destatep,
+ int prunereason) {
+ int delta_asciipairs = destatep->next_interesting_pair[AsciiPair] -
+ destatep->prior_interesting_pair[AsciiPair];
+ int delta_otherpairs = destatep->next_interesting_pair[OtherPair] -
+ destatep->prior_interesting_pair[OtherPair];
+
+ if (prunereason == PRUNE_FINAL) {
+ // We are about done
+ // If we get here with very little accumulated data, the initial hints
+ // were too strong, so we derate them to n+1 / 12 for n bigrams
+ if (!destatep->hints_derated &&
+ (destatep->next_interesting_pair[OtherPair] < kDerateHintsBelow)) {
+ int n = destatep->next_interesting_pair[OtherPair];
+
+ // Map N pairs to (N+1)/12 portions of the initial hints, etc.
+ // Floor of 3/12 -- 1/12 and 2/12 are too easy to overcome
+ int m = maxint(3, (n + 1));
+ for (int i = 0; i < NUM_RANKEDENCODING; ++i) {
+ int original_delta = destatep->hint_prob[i];
+ int scaled_delta = (original_delta * m) / kDerateHintsBelow;
+ destatep->enc_prob[i] -= original_delta;
+ destatep->enc_prob[i] += scaled_delta;
+ }
+ destatep->hints_derated = true;
+ if (destatep->debug_data != NULL) {
+ // Show derated-hint result
+ char buff[32];
+ snprintf(buff, sizeof(buff), "Hints %d/%d", m, kDerateHintsBelow);
+ SetDetailsEncLabel(destatep, buff);
+ }
+ }
+ }
+
+
+ ++destatep->prune_count;
+
+ if (prunereason != PRUNE_FINAL) {
+ // Early outs
+ if (destatep->rankedencoding_list_len <= 1) { // nothing to prune
+ destatep->done = true;
+ return;
+ }
+
+ if ((destatep->prune_count > 0) &&
+ (delta_asciipairs + delta_otherpairs) == 0) {
+ // Nothing to do; must have just been called earlier
+ return;
+ }
+ }
+
+
+
+ // INCREMENT
+ // ====================
+ // Accumulate OtherPair probibilities over all active families
+ // AsciiPair probibilities are all done in ActiveSpecialBoostWhack
+ uint8 prior_bad_byte1 = ' '; // won't match first bad pair
+ uint8 prior_bad_byte2 = ' '; // won't match first bad pair
+ uint8 or_byte1 = 0; // Track if any current pair has a high bit
+ int counted_otherpairs = 0;
+ uint8 prior_byte1x2x = 0;
+ for (int i = 0; i < delta_otherpairs; ++i) {
+ int watch1_incr = 0;
+ int watch2_incr = 0;
+ int next_pair = destatep->prior_interesting_pair[OtherPair] + i;
+
+ uint8 byte1 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 0];
+ uint8 byte2 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 1];
+ uint8 byte1x2x = (byte1 & 0xf0) | ((byte2 >> 4) & 0x0f);
+ int weightshift = destatep->interesting_weightshift[OtherPair][next_pair];
+
+ int offset_byte12 = destatep->interesting_offsets[OtherPair][next_pair];
+
+ // To help distinguish some Cyrillic, Arabic, Greek, Hebrew, Thai
+ // Remember if this is a CDEF pair immediately following the previous pair
+ // 8xxx CxCx or CxCx 8xxx
+ bool next_pair_consec_hi = false;
+ if (ConsecutivePair(destatep, next_pair)) {
+ if ((byte1x2x & 0xcc) == 0xcc) { // 8xxx CxCx
+ next_pair_consec_hi = true;
+ } else if ((prior_byte1x2x & 0xcc) == 0xcc) { // CxCx 8xxx
+ next_pair_consec_hi = true;
+ }
+ }
+ //printf("prior/cur/consec %02x %02x %d\n",
+ // prior_byte1x2x, byte1x2x, next_pair_consec_hi);
+ prior_byte1x2x = byte1x2x;
+
+ or_byte1 |= byte1;
+ uint8 byte1f = byte1;
+ // Flip top bit of subscript to better separate quadrant 4 (esp. for Hebrew)
+ byte1f ^= (byte2 & 0x80);
+
+ // If the same bigram occurred recently, don't increment again
+ bool pair_used = false;
+ if (!RepeatedBigram(destatep, byte1, byte2)) {
+ ++counted_otherpairs;
+ pair_used = true;
+ // Boost both charset= declared encodings, so
+ // Nearly-same probability nearby encoding doesn't drift to the top
+ if (!FLAGS_demo_nodefault) {
+ destatep->enc_prob[destatep->declared_enc_1] += kDeclaredEncBoost >> weightshift;
+ destatep->enc_prob[destatep->declared_enc_2] += kDeclaredEncBoost >> weightshift;
+ }
+ bool was_bad_pair = false;
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int incr_shift = 0;
+ int rankedencoding = destatep->rankedencoding_list[j];
+ Encoding enc = kMapToEncoding[rankedencoding];
+
+ // For binary, Skip over repeated marker bytes, such as 02, FF, etc.
+ if ((rankedencoding == F_BINARY) &&
+ RepeatedBinary(destatep, byte1, byte2)) {
+ incr_shift = 2; // count 1/4 as much if repeated
+ }
+
+ // If byte 1x2x for this encoding is exactly zero, illegal byte pair
+ // Don't increment, but instead penalize
+ const UnigramEntry* ue = &unigram_table[rankedencoding];
+ if (ue->b12[byte1x2x] == 0) {
+ // Don't whack consecutive duplicate bad pairs -- overkill
+ if ((byte1 != prior_bad_byte1) || (byte2 != prior_bad_byte2)) {
+ // Extra whack for illegal pair in this encoding
+ Whack(destatep, rankedencoding, kBadPairWhack >> weightshift);
+ was_bad_pair = true;
+ }
+ } else {
+ // OK to do the real increment
+ int incr = ue->b1[byte1f] + ue->b2[byte2] + ue->b12[byte1x2x];
+ if ((ue->b12[byte1x2x] & 0x01) != 0) {
+ // Use a more-precise table
+ int byte32x32 = ((byte1 & 0x1f) << 5) | (byte2 & 0x1f);
+ int hiressub = (byte2 & 0x60) >> 5; // select w/bits 5&6 of byte 2
+ DCHECK(ue->hires[hiressub] != NULL);
+ incr += ue->hires[hiressub][byte32x32];
+ } else {
+ // Default final offset
+ incr += ue->so;
+ }
+ incr >>= incr_shift;
+
+ incr >>= weightshift;
+ destatep->enc_prob[rankedencoding] += incr; // The actual increment
+
+ if (FLAGS_enc_detect_detail2) {
+ if (watch1_rankedenc == rankedencoding) {watch1_incr = incr;}
+ if (watch2_rankedenc == rankedencoding) {watch2_incr = incr;}
+ }
+ }
+
+
+ // If consecutive pair of high bytes, give slight boost to one-byte
+ // encodings that have a full alphabet in the high bytes
+ if (next_pair_consec_hi && HighAlphaEncoding(enc)) {
+ Boost(destatep, rankedencoding, kDeclaredEncBoost >> weightshift);
+ }
+ } // End for j < rankedencoding_list_len
+
+ if (was_bad_pair) {
+ prior_bad_byte1 = byte1;
+ prior_bad_byte2 = byte2;
+ }
+
+ // Fold in per-bigram most likely encoding for first N bigrams
+ if (next_pair < kBestPairsCount) {
+ int best_enc = kMostLikelyEncoding[(byte1 << 8) + byte2];
+ Boost(destatep, best_enc, kBestEncBoost >> weightshift);
+ }
+
+ // Possibly score 32 trigrams around a bigram to better separate
+ // Latin1 from Latin2 and Latin7. Especially helpful for detecting
+ // mis-labelled Hungarian latin2.
+ // If looking and at bigram 0,8,16,... do full scoring, else just 1 tri
+ if (destatep->do_latin_trigrams ||
+ destatep->looking_for_latin_trigrams) {
+ // If just looking, do full scan every 8 times
+ // Just look up one trigram the other 7 and do full scan if Latin2,7
+ bool scan32 = false;
+ const uint8* trisrc = &destatep->initial_src[offset_byte12 - 1];
+ if (!destatep->do_latin_trigrams) {
+ if ((i & 7) == 0 || trisrc + 3 > destatep->limit_src) {
+ scan32 = true;
+ } else {
+ scan32 = (kTriLatin1Likely < TrigramValue(trisrc));
+ }
+ }
+ if (destatep->do_latin_trigrams || scan32) {
+ // Just score each block of 32 bytes once
+ int tri_block_offset = offset_byte12 & ~0x1f;
+ if (destatep->trigram_highwater_mark <= tri_block_offset) {
+ bool turnon = BoostLatin127Trigrams(tri_block_offset, destatep);
+ if (FLAGS_counts && !destatep->do_latin_trigrams && turnon) {
+ ++doing_used; // First time
+ }
+ if (FLAGS_enc_detect_source) {
+ if (!destatep->do_latin_trigrams && turnon) {
+ // First time
+ PsHighlight(trisrc, destatep->initial_src, 0, 2);
+ }
+ }
+ destatep->do_latin_trigrams |= turnon;
+ destatep->trigram_highwater_mark = tri_block_offset + 32;
+ }
+ }
+ }
+
+ } // end if RepeatedBigram()
+
+ // Keep track of initial byte high 3 bits
+ ++destatep->byte32_count[byte1 >> 5];
+
+
+ // TODO: boost subset/superset also
+ // Boost(destatep, kRelatedEncoding[best_enc], kBestEncBoost);
+
+ if (destatep->debug_data != NULL) {
+ // Show detail entry for this bigram
+ char buff[16];
+ snprintf(buff, sizeof(buff), "%c%02x%02x%c%c",
+ pair_used ? ' ' : '[',
+ byte1,
+ byte2,
+ pair_used ? ' ' : ']',
+ (weightshift == 0) ? ' ' : '-');
+
+ SetDetailsEncProb(destatep,
+ destatep->interesting_offsets[OtherPair][next_pair],
+ kMostLikelyEncoding[(byte1 << 8) + byte2],
+ buff);
+ }
+ if (FLAGS_enc_detect_detail2) {
+ if ((watch1_incr != 0) || (watch2_incr != 0)) {
+ // Show increment detail for this encoding
+ char buff[32];
+ snprintf(buff, sizeof(buff), "%c%d %c%d",
+ (watch1_incr < 0) ? '-' : '+', watch1_incr,
+ (watch2_incr < 0) ? '-' : '+', watch2_incr);
+ SetDetailsEncLabel(destatep, buff);
+ }
+ }
+ } // End for i
+
+
+ // If no high bit on, demote all the two-byte codes
+ // WAS BUG. This was inside the loop above and should be outside
+ if ((counted_otherpairs > 0) && ((or_byte1 & 0x80) == 0)) {
+ // No high bit in this group (just 02xx, etc.). Whack 2-byte codes
+ // This keeps SJS from creeping past Latin1 on illegal C0 bytes
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ Encoding enc = kMapToEncoding[rankedencoding];
+ if (TwoByteEncoding(enc)) {
+ Whack(destatep, rankedencoding, kGentlePairWhack * counted_otherpairs);
+ }
+ }
+ }
+
+
+ // BOOST
+ // ====================
+ if (AnyActive(destatep)) {
+ ActiveSpecialBoostWhack(src, destatep);
+ }
+
+ // Update for next time
+ destatep->prior_src = src;
+ destatep->prior_interesting_pair[AsciiPair] =
+ destatep->next_interesting_pair[AsciiPair];
+ destatep->prior_interesting_pair[OtherPair] =
+ destatep->next_interesting_pair[OtherPair];
+
+
+ // Do any pre-prune final adjustments
+ // ====================
+ if (prunereason == PRUNE_FINAL) {
+ // If UTF8 not in base state, whack
+ if (destatep->next_utf8_ministate != 0) {
+ Whack(destatep, F_UTF8, kGentlePairWhack * 2 * 1);
+ }
+ // If UTF8UTF8 not in base state, whack
+ if (destatep->next_utf8utf8_ministate != 0) {
+ Whack(destatep, F_UTF8UTF8, kGentlePairWhack * 2 * 1);
+ }
+
+ // If no valid UTF-8 char ever seen, whack
+ if (destatep->utf8_minicount[5] == 0) {
+ Whack(destatep, F_UTF8, kBadPairWhack * 8); // No sequence
+ Whack(destatep, F_UTF8UTF8, kBadPairWhack * 8); // No sequence
+ }
+
+ // If no valid UTF8UTF8 char ever seen, whack
+ if (destatep->utf8utf8_minicount[5] == 0) {
+ Whack(destatep, F_UTF8UTF8, kBadPairWhack * 8); // No sequence
+ }
+
+ // If not all four binary quadrants, whack BINARY;
+ // worth 2 pair if 3 quads, 4 pair if 1 or 2 quads
+ if (destatep->binary_quadrants_count < 4) {
+ if (destatep->binary_quadrants_count == 3) {
+ Whack(destatep, F_BINARY, kBadPairWhack * 2);
+ } else {
+ Whack(destatep, F_BINARY, kBadPairWhack * 4);
+ }
+ }
+
+ // If 1st pair is 1b24, choose between ISO-2022-xx
+ // <esc> $ ) C ISO-2022-KR [1b 24 29 43]
+ // <esc> $ ) A ISO-2022-CN [1b 24 29 41]
+ // <esc> $ ) G ISO-2022-CN [1b 24 29 47]
+ // <esc> $ * H ISO-2022-CN [1b 24 2a 48]
+ // <esc> ( B ISO-2022-JP [1b 28 42] to ASCII
+ // <esc> ( J ISO-2022-JP [1b 28 4a] to X0201
+ // <esc> $ @ ISO-2022-JP [1b 24 40] to X0208-78 twobyte
+ // <esc> $ B ISO-2022-JP [1b 24 42] to X0208-83 twobyte
+ if ((destatep->next_interesting_pair[OtherPair] >= 1) &&
+ Iso2022Active(destatep)) {
+ if ((destatep->interesting_pairs[OtherPair][0] == 0x1b) &&
+ (destatep->interesting_pairs[OtherPair][1] == 0x24)) {
+ int offset = destatep->interesting_offsets[OtherPair][0];
+ const uint8* esc_src = destatep->initial_src + offset;
+ if ((destatep->initial_src + offset) < (destatep->limit_src - 3)) {
+ if ((esc_src[2] == ')') && (esc_src[3] == 'C')) {
+ Boost(destatep, F_ISO_2022_KR, kBoostOnePair);
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
+ Whack(destatep, F_JIS, kBadPairWhack);
+ } else if ((esc_src[2] == ')') && ((esc_src[3] == 'A') ||
+ (esc_src[3] == 'G'))) {
+ Boost(destatep, F_ISO_2022_CN, kBoostOnePair);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
+ Whack(destatep, F_JIS, kBadPairWhack);
+ } else if ((esc_src[2] == '@') || (esc_src[2] == 'B')) {
+ Boost(destatep, F_JIS, kBoostOnePair);
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
+ }
+ } else {
+ // Incomplete escape sequence. Whack them all
+ Whack(destatep, F_JIS, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
+ Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
+ }
+ }
+ }
+ if (destatep->debug_data != NULL) {
+ SetDetailsEncLabel(destatep, "pre-final");
+ }
+ }
+
+ // PRUNE
+ // ====================
+ // Find current top two rankedencoding probabilities
+ ReRank(destatep);
+
+ if (prunereason == PRUNE_SLOWEND) {
+ if (destatep->debug_data != NULL) {
+ SetDetailsEncLabel(destatep, "slow-end");
+ }
+ }
+
+ // Keep every rankedencoding with probablity >= top_prob - prune_difference
+ int prune_diff = destatep->prune_difference;
+ // If the top encoding is BINARY, it might be overstated, and we might
+ // therefore prune away the real encoding. Make the pruning delta
+ // twice as big.
+ if (destatep->top_rankedencoding == F_BINARY) {
+ prune_diff *= 2;
+ }
+ int keep_prob = destatep->top_prob - prune_diff;
+
+ // Tighten pruning difference (we start wide) for next time
+ if (destatep->prune_difference > kFinalPruneDifference) {
+ int decrement = kPruneDiffDecrement;
+ // If only ASCII pairs, small tighten; if some non-ASCII, full tighten
+ if (counted_otherpairs == 0) {
+ decrement >>= 1;
+ }
+ destatep->prune_difference -= decrement;
+ }
+
+ // Prune the list of active encoding families
+ destatep->active_special = 0;
+ int k = 0;
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ bool keep = true;
+ int rankedencoding = destatep->rankedencoding_list[j];
+
+ // If count is too low, ditch it
+ if (destatep->enc_prob[rankedencoding] < keep_prob) {
+ keep = false;
+ }
+
+ // If at end of slow section, ditch any 7-bit with zero evidence so far
+ if ((prunereason == PRUNE_SLOWEND) &&
+ SevenBitEncoding(kMapToEncoding[rankedencoding]) &&
+ (destatep->enc_prob[rankedencoding] <= 0) &&
+ (rankedencoding != destatep->top_rankedencoding)) {
+ keep = false;
+ }
+
+ // Keep it. This will always keep at least top_prob rankedencoding
+ if (keep) {
+ destatep->active_special |= kSpecialMask[kMapToEncoding[rankedencoding]];
+ destatep->rankedencoding_list[k++] = rankedencoding;
+ }
+ }
+
+ if (destatep->debug_data != NULL) {
+ char buff[32];
+ snprintf(buff, sizeof(buff), "%d prune", prune_diff / XLOG2);
+ SetDetailsEncLabel(destatep, buff);
+ }
+ destatep->rankedencoding_list_len = k;
+
+
+
+ // Force final result in some cases
+ // Do any post-prune final adjustments
+ if (prunereason == PRUNE_FINAL) {
+ // If no high-byte pairs, result is ASCII7, BINARY, UTF7, 2022, or HZ
+ if (destatep->next_interesting_pair[OtherPair] == 0) {
+ if ((destatep->top_rankedencoding != F_BINARY) &&
+ (destatep->top_rankedencoding != F_UTF7) &&
+ (destatep->top_rankedencoding != F_ISO_2022_CN) &&
+ (destatep->top_rankedencoding != F_ISO_2022_KR) &&
+ (destatep->top_rankedencoding != F_JIS) &&
+ (destatep->top_rankedencoding != F_HZ_GB_2312)) {
+ destatep->top_rankedencoding = F_ASCII_7_bit;
+ Boost(destatep, F_ASCII_7_bit, kBoostOnePair * 2);
+ }
+ }
+
+ // If some 89 pairs, not ISO_8859_x and vice versa
+ if (destatep->byte32_count[4] > 0) {
+ switch (destatep->top_rankedencoding) {
+ case F_ASCII: // ISO-8859-1
+ destatep->top_rankedencoding = F_CP1252;
+ // Better: destatep->enc_prob[F_ASCII] <==> destatep->enc_prob[F_CP1252]
+ Boost(destatep, F_CP1252, kBoostOnePair * 2);
+ break;
+ case F_Latin2: // ISO-8859-2
+ // Don't swap back; not superset
+ //destatep->top_rankedencoding = F_CP1250;
+ //Boost(destatep, F_CP1250, kBoostOnePair * 2);
+ break;
+ case F_Arabic: // ISO-8859-6
+ destatep->top_rankedencoding = F_CP1256;
+ Boost(destatep, F_CP1256, kBoostOnePair * 2);
+ break;
+ case F_Greek: // ISO-8859-7
+ // Don't swap -- not proper superset
+ // Capital Alpha tonos at 0xB6 in ISO-8859-7, 0xA2 in CP1253
+ //destatep->top_rankedencoding = F_CP1253;
+ //Boost(destatep, F_CP1253, kBoostOnePair * 2);
+ break;
+ case F_Hebrew: // ISO-8859-8
+ // Don't swap -- visual vs. logical
+ //destatep->top_rankedencoding = F_CP1255;
+ //Boost(destatep, F_CP1255, kBoostOnePair * 2);
+ break;
+ case F_Latin5: // ISO-8859-9
+ destatep->top_rankedencoding = F_CP1254;
+ Boost(destatep, F_CP1254, kBoostOnePair * 2);
+ break;
+ case F_ISO_8859_11: // ISO-8859-11
+ destatep->top_rankedencoding = F_CP874;
+ Boost(destatep, F_CP874, kBoostOnePair * 2);
+ break;
+ }
+ } else {
+ switch (destatep->top_rankedencoding) {
+ case F_CP1252: // ISO-8859-1
+ destatep->top_rankedencoding = F_ASCII;
+ Boost(destatep, F_ASCII, kBoostOnePair * 2);
+ break;
+ case F_CP1250: // ISO-8859-2
+ // Don't swap back; not superset
+ //destatep->top_rankedencoding = F_Latin2;
+ //Boost(destatep, F_Latin2, kBoostOnePair * 2);
+ break;
+ case F_CP1256: // ISO-8859-6
+ // Don't swap back -- not proper superset
+ //destatep->top_rankedencoding = F_Arabic;
+ //Boost(destatep, F_Arabic, kBoostOnePair * 2);
+ break;
+ case F_CP1253: // ISO-8859-7
+ // Don't swap back -- not proper superset
+ //destatep->top_rankedencoding = F_Greek;
+ //Boost(destatep, F_Greek, kBoostOnePair * 2);
+ break;
+ case F_CP1255: // ISO-8859-8
+ // Don't swap back -- not proper superset
+ //destatep->top_rankedencoding = F_Hebrew;
+ //Boost(destatep, F_Hebrew, kBoostOnePair * 2);
+ break;
+ case F_CP1254: // ISO-8859-9
+ destatep->top_rankedencoding = F_Latin5;
+ Boost(destatep, F_Latin5, kBoostOnePair * 2);
+ break;
+ case F_CP874: // ISO-8859-11
+ destatep->top_rankedencoding = F_ISO_8859_11;
+ Boost(destatep, F_ISO_8859_11, kBoostOnePair * 2);
+ break;
+ }
+ }
+
+ if (destatep->debug_data != NULL) {
+ char buff[32];
+ snprintf(buff, sizeof(buff), "final %d",
+ static_cast<int>(src - destatep->initial_src));
+ SetDetailsEncLabel(destatep, buff);
+
+ // Show winning encoding and its delta log base2 from 2nd-best
+ // Divide delta by XLOG2 to get log base 2
+ int delta = destatep->top_prob - destatep->second_top_prob;
+ if (delta < (2 * XLOG2)) {
+ delta /= XDECILOG2;
+ snprintf(buff, sizeof(buff), "+%d.%d %s ",
+ delta / 10, delta % 10,
+ MyEncodingName(kMapToEncoding[destatep->top_rankedencoding]));
+ } else if (delta < (50 * XLOG2)) {
+ delta /= XLOG2;
+ snprintf(buff, sizeof(buff), "+%d %s",
+ delta,
+ MyEncodingName(kMapToEncoding[destatep->top_rankedencoding]));
+ } else {
+ snprintf(buff, sizeof(buff), "%s",
+ MyEncodingName(kMapToEncoding[destatep->top_rankedencoding]));
+ }
+ SetDetailsEncProbCopyOffset(destatep, destatep->top_rankedencoding, buff);
+ }
+ }
+
+
+ // FINISH
+ // ====================
+ // Eventual encoding result is reliable if big difference in top two, or if
+ // only Ascii7 ever encountered
+ // Also reliable if exactly one OtherPair and it's best encoding matches top
+ destatep->reliable = false;
+ if (destatep->next_interesting_pair[OtherPair] == 0) {
+ // Only 7-bit ASCII
+ destatep->reliable = true;
+ }
+ if ((destatep->top_prob - destatep->second_top_prob) >=
+ FLAGS_ced_reliable_difference) {
+ destatep->reliable = true;
+ }
+ if (destatep->next_interesting_pair[OtherPair] == 1) {
+ uint8 byte1 = destatep->interesting_pairs[OtherPair][0];
+ uint8 byte2 = destatep->interesting_pairs[OtherPair][1];
+ int best_enc = kMostLikelyEncoding[(byte1 << 8) + byte2];
+ if (best_enc == destatep->top_rankedencoding) {
+ destatep->reliable = true;
+ }
+ }
+
+ // If we pruned to one encoding, we are done
+ if (destatep->rankedencoding_list_len == 1) {
+ destatep->reliable = true;
+ destatep->done = true;
+ }
+
+ // If we pruned to two or three encodings in the same *superset/subset
+ // rankedencoding* and enough pairs, we are done. Else keep going
+ if (destatep->rankedencoding_list_len == 2) {
+ Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
+ Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
+ if (kMapEncToBaseEncoding[enc0] == kMapEncToBaseEncoding[enc1]) {
+ if (destatep->prune_count >= 3) {
+ destatep->reliable = true;
+ destatep->done = true;
+ }
+ }
+ } else if (destatep->rankedencoding_list_len == 3) {
+ Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
+ Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
+ Encoding enc2 = kMapToEncoding[destatep->rankedencoding_list[2]];
+ Encoding base0 = kMapEncToBaseEncoding[enc0];
+ Encoding base1 = kMapEncToBaseEncoding[enc1];
+ Encoding base2 = kMapEncToBaseEncoding[enc2];
+
+ if ((base0 == base1) && (base0 == base2)) {
+ if (destatep->prune_count >= 3) {
+ destatep->reliable = true;
+ destatep->done = true;
+ }
+ }
+ }
+}
+
+
+// Accumulate aligned byte-pair at src
+// Occasionally, calc boost for some encodings and then prune the active list
+// weightshift is used to give low weight some text, such as inside tags
+// Returns true if pruning occurred
+bool IncrementAndBoostPrune(const uint8* src,
+ int remaining_length,
+ DetectEncodingState* destatep,
+ int weightshift,
+ int exit_reason) {
+ destatep->last_pair = src;
+ // Pick up byte pair, or very last byte plus 0x20
+ uint8 byte1 = src[0];
+ uint8 byte2 = 0x20;
+ if (1 < remaining_length) {byte2 = src[1];}
+
+ // whatset=0 for Ascii + ~, 1 for all others; see kTestPrintableAsciiTildePlus
+ int whatset = exit_reason - 1;
+ int next_pair = destatep->next_interesting_pair[whatset];
+
+ if (next_pair > 16) {
+ // If not clear by 16 bigrams, stop accumulating + ~ 00
+ if (byte1 == '+') {return false;}
+ if (byte1 == '~') {return false;}
+ if (byte1 == 0x00) {return false;}
+ }
+
+ // Remember pair in appropriate list
+ if (next_pair >= kMaxPairs) {
+ // We have filled up our alloted space for interesting pairs with no
+ // decision. If ASCII pairs full, just skip until end of slow loop; if
+ // non-Ascii pairs full, force done
+ if (whatset == OtherPair) {
+ destatep->done = true;
+ }
+ } else {
+ int offset = static_cast<int>(src - destatep->initial_src);
+ destatep->interesting_pairs[whatset][next_pair * 2 + 0] = byte1;
+ destatep->interesting_pairs[whatset][next_pair * 2 + 1] = byte2;
+ destatep->interesting_offsets[whatset][next_pair] = offset;
+ destatep->interesting_weightshift[whatset][next_pair] = weightshift;
+ ++destatep->next_interesting_pair[whatset];
+ ++next_pair;
+ }
+
+ // Prune now and then , but always if forced to be done
+ if (destatep->done || ((next_pair & kPruneMask) == 0)) { // Prune every M
+ BoostPrune(src + 2, destatep, PRUNE_NORMAL); // src+2 first unscanned byte
+ // may be off end of input
+ return true;
+ }
+ return false;
+}
+
+void DumpSummary(DetectEncodingState* destatep, int whatset, int n) {
+ printf(" %sSummary[%2d]: ", kWhatSetName[whatset],
+ destatep->next_interesting_pair[whatset]);
+ int limit = minint(n, destatep->next_interesting_pair[whatset]);
+ for (int i = 0; i < limit; ++i) {
+ printf("%02x%02x ",
+ destatep->interesting_pairs[whatset][i * 2 + 0],
+ destatep->interesting_pairs[whatset][i * 2 + 1]);
+ if ((i & 7) == 7) {printf(" ");}
+ }
+ printf("\n");
+}
+
+void BeginDetail(DetectEncodingState* destatep) {
+ fprintf(stderr, "%d [", NUM_RANKEDENCODING);
+ for (int e = 0; e < NUM_RANKEDENCODING; ++e) {
+ fprintf(stderr, "(%s)", MyRankedEncName(e));
+ if ((e % 10) == 9) {fprintf(stderr, "\n ");}
+ }
+ fprintf(stderr, "] size-detail\n");
+ destatep->next_detail_entry = 0;
+}
+
+// Single character to represent (printable ASCII) gap between bigrams
+char DetailOffsetChar(int delta) {
+ if (delta == 0) {return ' ';}
+ if (delta <= 2) {return '=';}
+ if (delta <= 15) {return '_';}
+ if (delta <= 31) {return '+';}
+ {return ' ';}
+}
+
+void DumpDetail(DetectEncodingState* destatep) {
+ // Turn all counts into delta from previous entry
+ fprintf(stderr, "%d count-detail\n", destatep->next_detail_entry);
+ // Rewrite, recording deltas
+ for (int z = destatep->next_detail_entry - 1; z > 0; --z) {
+ destatep->debug_data[z].offset -= destatep->debug_data[z - 1].offset;
+ for (int e = 0; e < NUM_RANKEDENCODING; ++e) {
+ destatep->debug_data[z].detail_enc_prob[e] -=
+ destatep->debug_data[z - 1].detail_enc_prob[e];
+ }
+ }
+ // Now print
+ for (int z = 0; z < destatep->next_detail_entry; ++z) {
+ // Highlight some entries ending in '!' with light red underbar
+ int len = destatep->debug_data[z].label.size();
+ if (destatep->debug_data[z].label[len - 1] == '!') {
+ fprintf(stderr, "1 0.9 0.9 do-flag\n");
+ }
+ fprintf(stderr, "(%c%s) %d [",
+ DetailOffsetChar(destatep->debug_data[z].offset),
+ destatep->debug_data[z].label.c_str(),
+ destatep->debug_data[z].best_enc);
+ for (int e = 0; e < NUM_RANKEDENCODING; ++e) {
+ fprintf(stderr, "%d ", destatep->debug_data[z].detail_enc_prob[e]);
+ if ((e % 10) == 9) {fprintf(stderr, " ");}
+ }
+ fprintf(stderr, "] do-detail-e\n");
+ }
+ // Get ready for next time,if any
+ destatep->next_detail_entry = 0;
+}
+
+void PsRecurse(const char* buff) {
+ fprintf(stderr, "() end-detail (%s) start-detail\n\n", buff);
+}
+
+void DumpReliable(DetectEncodingState* destatep) {
+ printf("Not reliable: ");
+
+ // Find center of gravity of OtherPair list
+ int x_sum = 0;
+ int y_sum = 0;
+ int count = destatep->next_interesting_pair[OtherPair];
+ for (int i = 0; i < count; ++i) {
+ uint8 byte1 = destatep->interesting_pairs[OtherPair][i * 2 + 0];
+ uint8 byte2 = destatep->interesting_pairs[OtherPair][i * 2 + 1];
+ x_sum += byte2;
+ y_sum += byte1;
+ }
+ if (count == 0) {count = 1;} // adoid zdiv
+ int x_bar = x_sum / count;
+ int y_bar = y_sum / count;
+ printf("center %02X,%02X\n", x_bar, y_bar);
+
+ double closest_dist = 999.0;
+ int closest = 0;
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ const UnigramEntry* ue = &unigram_table[rankedencoding];
+ printf(" %8s = %4d at %02x,%02x +/- %02X,%02X ",
+ MyEncodingName(kMapToEncoding[rankedencoding]),
+ destatep->enc_prob[rankedencoding],
+ ue->x_bar, ue->y_bar,
+ ue->x_stddev, ue->y_stddev);
+ double x_diff = x_bar - ue->x_bar;
+ double y_diff = y_bar - ue->y_bar;
+ double dist = sqrt((x_diff * x_diff) + (y_diff * y_diff));
+ printf("(%3.1f)\n", dist);
+
+ if (closest_dist > dist) {
+ closest_dist = dist;
+ closest = rankedencoding;
+ }
+ }
+ printf("Closest=%s (%3.1f)\n",
+ MyEncodingName(kMapToEncoding[closest]), closest_dist);
+
+ for (int i = 0; i < 8; ++i) {
+ // Demote by distance to CG and see if that helps, or just quit
+ }
+}
+
+// Scan short single lines quickly for all printable ASCII
+// Return true if all bytes are in [20..7F], false otherwise
+bool QuickPrintableAsciiScan(const char* text, int text_length) {
+ const uint8* src = reinterpret_cast<const uint8*>(text);
+ const uint8* srclimit = src + text_length;
+ const uint8* srclimit8 = srclimit - 7;
+ while (src < srclimit8) {
+ // Exits on any byte outside [0x20..0x7E] range (HT LF CR exit)
+ uint8 mask = 0;
+ for (int i = 0; i < 8; ++i) mask |= (src[i]-0x20)|(src[i]+0x01);
+ if ((mask & 0x80) != 0) break;
+ src += 8;
+ }
+ while (src < srclimit) {
+ uint8 uc = *src++;
+ if (kIsPrintableAscii[uc] == 0) {return false;}
+ }
+ return true;
+}
+
+static const int kMaxScanBack = 192;
+
+// Return true if text is inside a tag or JS comment
+bool TextInsideTag(const uint8* isrc, const uint8* src, const uint8* srclimit) {
+ const uint8* srcbacklimit = src - kMaxScanBack;
+ if (srcbacklimit < isrc) {
+ srcbacklimit = isrc;
+ }
+ const uint8* ss = src - 1;
+ while (srcbacklimit <= ss) {
+ uint8 c = *ss--;
+ if ((c & ~0x02) == '<') {
+ // We found preceding < 3C or > 3E nearby
+ // Even cheaper: if inside a tag, we don't care what tag; return true
+ if (c == '<') {
+ return true;
+ }
+ // See if we are just after <title>...
+ if ((c == '>') && (isrc <= (ss - 5)) &&
+ (ss[-5] == '<') &&
+ ((ss[-4] | 0x20) == 't') &&
+ ((ss[-3] | 0x20) == 'i') &&
+ ((ss[-2] | 0x20) == 't') &&
+ ((ss[-1] | 0x20) == 'l') &&
+ ((ss[-0] | 0x20) == 'e')) {
+ return true;
+ }
+ // See if we are just after <SCRIPT language=javascript>...
+ if ((c == '>') && (isrc <= (ss - 5)) &&
+ (ss[-5] == 's') &&
+ ((ss[-4] | 0x20) == 'c') &&
+ ((ss[-3] | 0x20) == 'r') &&
+ ((ss[-2] | 0x20) == 'i') &&
+ ((ss[-1] | 0x20) == 'p') &&
+ ((ss[-0] | 0x20) == 't')) {
+ return true;
+ }
+ // Not in a tag
+ return false;
+ // See if we are just after JavaScript comment /* ...
+ } else if (c == '/') {
+ if (((ss + 2) < srclimit) && (ss[2] == '*')) {
+ // We backscanned to /*
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
+const uint8* SkipToTagEnd(const uint8* src, const uint8* srclimit) {
+ const uint8* ss = src + 1;
+ while (ss <= srclimit) {
+ uint8 c = *ss++;
+ if ((c == '<') || (c == '>')) {
+ return ss;
+ }
+ }
+ return src + 2; // Always make progress, Otherwise we get an infinite loop
+}
+
+
+// Take a watch string and map to a ranked encoding. If no match, return -1
+int LookupWatchEnc(const string& watch_str) {
+ int watchval = -1;
+ // Mixed encoding maps to enc=UTF8UTF8
+ if (watch_str == "UTF8UTF8") {
+ watchval = F_UTF8UTF8;
+ } else {
+ Encoding enc;
+ if (EncodingFromName(watch_str.c_str(), &enc)) {
+ watchval = CompactEncDet::BackmapEncodingToRankedEncoding(enc);
+ }
+ }
+ return watchval;
+}
+
+// Return true if enc and enc2 are equal or one is a subset of the other
+// or either is UNKNOWN
+// also UTF8UTF8 is compatible with both Latin1 and UTF8
+bool CompatibleEnc(Encoding enc, Encoding enc2) {
+ if (enc < 0) {return false;}
+ if (NUM_ENCODINGS <= enc) {return false;}
+ if (enc2 < 0) {return false;}
+ if (NUM_ENCODINGS <= enc2) {return false;}
+ if (enc == enc2) {return true;}
+ if (kMapEncToBaseEncoding[enc] == kMapEncToBaseEncoding[enc2]) {return true;}
+
+ if (enc == ASCII_7BIT) {return true;}
+ if (enc2 == ASCII_7BIT) {return true;}
+ if (enc == UNKNOWN_ENCODING) {return true;}
+ if (enc2 == UNKNOWN_ENCODING) {return true;}
+ if (enc == UTF8UTF8) {
+ if (enc2 == UTF8) {return true;}
+ if (kMapEncToBaseEncoding[enc2] == ISO_8859_1) {return true;}
+ }
+ if (enc2 == UTF8UTF8) {
+ if (enc == UTF8) {return true;}
+ if (kMapEncToBaseEncoding[enc] == ISO_8859_1) {return true;}
+ }
+
+ return false;
+}
+
+// Return superset of enc and enc2, which must be compatible
+Encoding SupersetEnc(Encoding enc, Encoding enc2) {
+ //printf(" SupersetEnc (%s, ", MyEncodingName(enc)); // TEMP
+ //printf("%s) ", MyEncodingName(enc2));
+ //printf("= %s\n",
+ // MyEncodingName(kMapEncToSuperLevel[enc] >= kMapEncToSuperLevel[enc2] ?
+ // enc :enc2));
+ if (kMapEncToSuperLevel[enc] >= kMapEncToSuperLevel[enc2]) {
+ return enc;
+ }
+ return enc2;
+}
+
+
+// If unreliable, try rescoring to separate some encodings
+Encoding Rescore(Encoding enc, const uint8* isrc,
+ const uint8* srctextlimit, DetectEncodingState* destatep) {
+ if (FLAGS_counts) {++rescore_used;}
+ Encoding new_enc = enc;
+
+ bool rescore_change = false;
+
+ int count = destatep->next_interesting_pair[OtherPair];
+ int text_length = srctextlimit - isrc;
+ for (int i = 0; i < count; ++i) {
+ int bigram_offset = destatep->interesting_offsets[OtherPair][i];
+ uint8 byte0 = (0 < bigram_offset) ?
+ isrc[bigram_offset - 1] : 0x20;
+ uint8 byte1 = isrc[bigram_offset + 0]; // Known to have high bit on
+ uint8 byte2 = ((bigram_offset + 1) < text_length) ?
+ isrc[bigram_offset + 1] : 0x20;
+ uint8 byte3 = ((bigram_offset + 2) < text_length) ?
+ isrc[bigram_offset + 2] : 0x20;
+ int high_hash = ((byte0 & 0xc0) >> 0) |
+ ((byte1 & 0xc0) >> 1) |
+ ((byte2 & 0xc0) >> 4) |
+ ((byte3 & 0xc0) >> 6); // 00112233
+
+ // Boost HighAccent encodings for Ascii bit patterns
+ // 0x1x 0x0x
+ // 1010 1010
+ // 0010 0000
+ //
+ if ((high_hash & 0xaa) == 0x20) {
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if (HighAccentEncoding(kMapToEncoding[rankedencoding])) {
+ // TODO: also want to boost Shift-JIS here if byte1 is Ax..Dx
+ // TEMP
+ //printf(" Rescore[%02x] %s +%d\n",
+ // high_hash, MyRankedEncName(rankedencoding), kGentlePairBoost);
+ Boost(destatep, rankedencoding, kGentlePairBoost);
+ rescore_change = true;
+ }
+ }
+ }
+
+ // Whack HighAccent encodings for high bit patterns
+ // 1x1x 1x1x
+ // 1010 1010
+ // 1010 1010
+ //
+ if ((high_hash & 0xaa) == 0xaa) {
+ for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
+ int rankedencoding = destatep->rankedencoding_list[j];
+ if (HighAccentEncoding(kMapToEncoding[rankedencoding])) {
+ // TEMP
+ //printf(" Rescore[%02x] %s -%d\n",
+ // high_hash, MyRankedEncName(rankedencoding), kGentlePairBoost);
+ Whack(destatep, rankedencoding, kGentlePairBoost);
+ rescore_change = true;
+ }
+ }
+ }
+
+ }
+
+ if (rescore_change) {
+ ReRank(destatep);
+ new_enc = kMapToEncoding[destatep->top_rankedencoding];
+
+ if (destatep->debug_data != NULL) {
+ char buff[32];
+ snprintf(buff, sizeof(buff), "=Rescore %s", MyEncodingName(new_enc));
+ SetDetailsEncProb(destatep,
+ 0,
+ CompactEncDet::BackmapEncodingToRankedEncoding(new_enc),
+ buff);
+ //// DumpDetail(destatep);
+ }
+
+ SimplePrune(destatep, kFinalPruneDifference);
+ CalcReliable(destatep);
+ }
+
+ //if (new_enc != enc) {
+ // // TEMP
+ // printf(" Rescore new top encoding = %s\n",
+ // MyRankedEncName(destatep->top_rankedencoding));
+ //}
+
+ return new_enc;
+}
+
+
+// Given an encoding, add its corresponding ranked encoding to the set
+void AddToSet(Encoding enc, int* list_len, int* list) {
+ // TEMP print
+ int item = CompactEncDet::BackmapEncodingToRankedEncoding(enc);
+ for (int i = 0; i < *list_len; ++i) {
+ if (list[i] == item) {
+ return; // Already in the set; don't add again
+ }
+ }
+ list[(*list_len)++] = item;
+}
+
+
+static const int kMinRobustBigramCount = 1000;
+static const int kMinKBToRobustScan = 64;
+static const int kMaxKBToRobustScan = 256;
+
+// Scan the first 64K or so, just doing raw bigram increments on given
+// probability list.
+// No fancy duplicate filtering or anything else here.
+// Returns number of bigrams counted
+int RobustScan(const char* text,
+ int text_length,
+ int robust_renc_list_len,
+ int* robust_renc_list,
+ int* robust_renc_probs) {
+ if (FLAGS_counts) {++robust_used;}
+ // Zero all the result probabilities
+ for (int i = 0; i < robust_renc_list_len; ++i) {
+ robust_renc_probs[i] = 0;
+ }
+ int max_fast_len = minint(text_length, (kMaxKBToRobustScan << 10));
+ const uint8* isrc = reinterpret_cast<const uint8*>(text);
+ const uint8* src = isrc;
+ const uint8* srclimitfast2 = isrc + max_fast_len - 1;
+ const uint8* srclimitfast4 = isrc + max_fast_len - 3;
+
+ int min_fast_len = minint(text_length, (kMinKBToRobustScan << 10));
+ const uint8* srclimitmin = isrc + min_fast_len - 1;
+
+ int bigram_count = 0;
+
+ if (FLAGS_enc_detect_source) {
+ PsSourceInit(kPsSourceWidth);
+ fprintf(stderr, "(RobustScan) do-src\n");
+ }
+
+ // Sum over a big chunk of the input
+ // Faster loop, no 7-bit-encodings possible, approx 3000 GB/sec
+ //====================================
+ while (src < srclimitfast2) {
+ // Skip to next interesting bigram
+
+ while (src < srclimitfast4) {
+ if (((src[0] | src[1] | src[2] | src[3]) & 0x80) != 0) break;
+ src += 4;
+ }
+
+ while (src < srclimitfast2) {
+ if ((src[0] & 0x80) != 0) break;
+ src++;
+ }
+
+ if (src < srclimitfast2) {
+ // We found a bigram with high bit on
+ // Next 5 lines commented out so we don't show all the source.
+ //const uint8* srctextlimit = isrc + text_length;
+ //if (FLAGS_enc_detect_source) {
+ // PsSource(src, isrc, srctextlimit);
+ // PsMark(src, 2, isrc, 0);
+ //}
+
+ uint8 byte1 = src[0];
+ uint8 byte2 = src[1];
+ uint8 byte1x2x = (byte1 & 0xf0) | ((byte2 >> 4) & 0x0f);
+ uint8 byte1f = byte1;
+ // Flip top bit of subscript to better separate quadrant 4 (esp. for Hebrew)
+ byte1f ^= (byte2 & 0x80);
+
+ // The real increments
+ for (int j = 0; j < robust_renc_list_len; ++j) {
+ int rankedencoding = robust_renc_list[j];
+ const UnigramEntry* ue = &unigram_table[rankedencoding];
+ int incr = ue->b1[byte1f] + ue->b2[byte2] + ue->b12[byte1x2x];
+ if ((ue->b12[byte1x2x] & 0x01) != 0) {
+ // Use a more-precise table
+ int byte32x32 = ((byte1 & 0x1f) << 5) | (byte2 & 0x1f);
+ int hiressub = (byte2 & 0x60) >> 5; // select w/bits 5&6 of byte 2
+ DCHECK(ue->hires[hiressub] != NULL);
+ incr += ue->hires[hiressub][byte32x32];
+ } else {
+ // Default final offset
+ incr += ue->so;
+ }
+ robust_renc_probs[j] += incr;
+ }
+
+ src += 2; // Continue after this bigram
+ ++bigram_count;
+
+ // Stop after 1000 bigrams reached, if at least 64KB scanned
+ if ((bigram_count > kMinRobustBigramCount) && (src > srclimitmin)) {
+ break;
+ }
+
+ }
+ }
+
+ if (FLAGS_enc_detect_source) {
+ fprintf(stderr, "( bigram_count = %d) do-src\n", bigram_count);
+ if (bigram_count == 0) {bigram_count = 1;} // zdiv
+ for (int i = 0; i < robust_renc_list_len; ++i) {
+ fprintf(stderr, "( enc[%-12.12s] = %7d (avg %d)) do-src\n",
+ MyRankedEncName(robust_renc_list[i]), robust_renc_probs[i],
+ robust_renc_probs[i] / bigram_count);
+ }
+ PsSourceFinish();
+ }
+
+ return bigram_count;
+}
+
+// If unreliable, rescan middle of document to see if we can get a better
+// answer. Rescan is only worthwhile if there are ~200 bytes or more left,
+// since the detector takes as much as 96 bytes of bigrams to decide.
+Encoding Rescan(Encoding enc,
+ const uint8* isrc,
+ const uint8* src,
+ const uint8* srctextlimit,
+ const char* url_hint,
+ const char* http_charset_hint,
+ const char* meta_charset_hint,
+ const int encoding_hint,
+ const Language language_hint,
+ const CompactEncDet::TextCorpusType corpus_type,
+ bool ignore_7bit_mail_encodings,
+ DetectEncodingState* destatep) {
+ bool enc_is_reliable = destatep->reliable;
+ Encoding new_enc = enc;
+ Encoding second_best_enc =
+ kMapToEncoding[destatep->second_top_rankedencoding];
+
+ if (FLAGS_counts) {++rescan_used;}
+
+ int scanned_bytes = src - isrc;
+ int unscanned_bytes = srctextlimit - src;
+ int text_length = srctextlimit - isrc;
+ bool empty_rescan = true;
+
+ // See if enough bytes left to bother doing rescan
+ if (kMinRescanLength < unscanned_bytes) {
+ const char* text = reinterpret_cast<const char*>(isrc);
+
+ Encoding one_hint = destatep->http_hint;
+ if ((one_hint == UNKNOWN_ENCODING) &&
+ (destatep->meta_hint != UNKNOWN_ENCODING)) {
+ one_hint = destatep->meta_hint;
+ }
+ if ((one_hint == UNKNOWN_ENCODING) &&
+ (destatep->bom_hint != UNKNOWN_ENCODING)) {
+ one_hint = destatep->bom_hint;
+ }
+
+ // Go to an even offset to keep UTF-16 in synch
+ int middle_offset = (scanned_bytes + (unscanned_bytes / 2)) & ~1;
+ CHECK(middle_offset <= text_length);
+
+ // Look back a bit for a low byte to synchronize, else hope for the best.
+ const uint8* srcbacklimit = isrc + middle_offset - kMaxScanBack;
+ if (srcbacklimit < src) {
+ srcbacklimit = src;
+ }
+ const uint8* ss = isrc + middle_offset - 1;
+ while (srcbacklimit <= ss) {
+ if ((*ss & 0x80) == 0) {break;}
+ --ss;
+ }
+ // Leave middle offset unchanged unless we found a low byte
+ if (srcbacklimit <= ss) {
+ // Align to low byte or high byte just after it, whichever is even
+ middle_offset = (ss - isrc + 1) & ~1; // Even to keep UTF-16 in sync
+ }
+ CHECK(middle_offset <= text_length);
+
+ if (destatep->debug_data != NULL) {
+ SetDetailsEncLabel(destatep, ">> Rescan");
+ // Print the current chart before recursive call
+ DumpDetail(destatep);
+
+ char buff[32];
+ snprintf(buff, sizeof(buff), ">> Rescan[%d..%d]",
+ middle_offset, text_length);
+ PsRecurse(buff);
+ }
+
+ int mid_bytes_consumed;
+ bool mid_is_reliable;
+ Encoding mid_second_best_enc;
+ CEDInternalFlags newflags = static_cast<CEDInternalFlags>(
+ kCEDRescanning + kCEDForceTags);
+ // Recursive call for rescan of half of remaining
+ Encoding mid_enc = InternalDetectEncoding(
+ newflags,
+ text + middle_offset,
+ text_length - middle_offset,
+ url_hint,
+ http_charset_hint,
+ meta_charset_hint,
+ encoding_hint,
+ language_hint, // User interface lang
+ corpus_type,
+ ignore_7bit_mail_encodings,
+ &mid_bytes_consumed,
+ &mid_is_reliable,
+ &mid_second_best_enc);
+ destatep->reliable = mid_is_reliable;
+
+ empty_rescan = (mid_enc == ASCII_7BIT);
+
+ // Not the right decision if, e.g. enc=Greek, mid=ASCII7, one=KSC
+ // hence the !empty_rescan term
+ if (!empty_rescan && CompatibleEnc(one_hint, mid_enc)) {
+ // Encoding we just found is compatible with the
+ // single hint (if any); return superset
+ new_enc = SupersetEnc(one_hint, mid_enc);
+ }
+
+ // If original and mid are compatible, and both reliable,
+ // return new_enc = SupersetEnc(enc, mid_enc)
+ //
+ // This avoids too much weight on a bogus hint causing a RobustScan
+ // that gets the wrong answer
+ if (!empty_rescan && mid_is_reliable && enc_is_reliable &&
+ CompatibleEnc(enc, mid_enc)) {
+ new_enc = SupersetEnc(enc, mid_enc);
+ return new_enc;
+ }
+
+ // if mid unreliable, robustscan
+ // if mid empty, robustscan
+ // if original and mid not compatible, robustscan
+ // if mid and one_hint not compatible, robustscan
+
+ // If we found conflicting data, drop back and do a robust scan of a big
+ // chunk of the input over a set of candidate encodings
+ //
+ if (!mid_is_reliable ||
+ empty_rescan ||
+ !CompatibleEnc(enc, mid_enc) ||
+ !CompatibleEnc(one_hint, mid_enc)) {
+ int robust_renc_list_len; // Number of active encodings
+ int robust_renc_list[NUM_RANKEDENCODING]; // List of ranked encodings
+ int robust_renc_probs[NUM_RANKEDENCODING]; // List of matching probs
+
+ robust_renc_list_len = 0;
+ AddToSet(enc, &robust_renc_list_len, robust_renc_list);
+ AddToSet(second_best_enc, &robust_renc_list_len, robust_renc_list);
+ AddToSet(mid_enc, &robust_renc_list_len, robust_renc_list);
+ AddToSet(mid_second_best_enc, &robust_renc_list_len, robust_renc_list);
+ if (destatep->http_hint != UNKNOWN_ENCODING) {
+ AddToSet(destatep->http_hint, &robust_renc_list_len, robust_renc_list);
+ }
+ if (destatep->meta_hint != UNKNOWN_ENCODING) {
+ AddToSet(destatep->meta_hint, &robust_renc_list_len, robust_renc_list);
+ }
+ if (destatep->bom_hint != UNKNOWN_ENCODING) {
+ AddToSet(destatep->bom_hint, &robust_renc_list_len, robust_renc_list);
+ }
+ if (destatep->tld_hint != UNKNOWN_ENCODING) {
+ AddToSet(destatep->tld_hint, &robust_renc_list_len, robust_renc_list);
+ }
+
+ // Separate simple scan
+ // =====================
+ if (destatep->debug_data != NULL) {
+ SetDetailsEncLabel(destatep, ">> RobustScan");
+ // Print the current chart before recursive call
+ DumpDetail(destatep);
+
+ char buff[32];
+ snprintf(buff, sizeof(buff), ">> RobustScan[0..%d]", text_length);
+ PsRecurse(buff);
+ }
+
+ int bigram_count = RobustScan(text, text_length,
+ robust_renc_list_len, robust_renc_list, robust_renc_probs);
+
+ // Default to new_enc and update if something better was found
+ int best_prob = -1;
+ // TEMP print
+ for (int i = 0; i < robust_renc_list_len; ++i) {
+ if (best_prob < robust_renc_probs[i]) {
+ best_prob = robust_renc_probs[i];
+ new_enc = kMapToEncoding[robust_renc_list[i]];
+ }
+ }
+
+ if (destatep->debug_data != NULL) {
+ char buff[32];
+ snprintf(buff, sizeof(buff), "=Robust[%d] %s",
+ bigram_count, MyEncodingName(new_enc));
+ SetDetailsEncProb(destatep,
+ 0,
+ CompactEncDet::BackmapEncodingToRankedEncoding(new_enc),
+ buff);
+ }
+ }
+ } // End if enough bytes
+
+ return new_enc;
+}
+
+// With no hints at all, and perhaps on rescan, we relax our pickiness
+// and go ahead and accept the top multibyte encodings, even though
+// strictly their web pages should have declared an explicit encoding to
+// avoid the HTML standard's default ISO-8859-1.
+bool NoHintsCloseEnoughCompatible(Encoding top_enc) {
+ // First test accepts degenerate cases plus UTF8 and UTF8UTF8
+ if (CompatibleEnc(UTF8, top_enc)) {return true;}
+
+ // The rest look for exact match of base encoding
+ Encoding base_enc = kMapEncToBaseEncoding[top_enc];
+ if (base_enc == JAPANESE_EUC_JP) {return true;}
+ if (base_enc == JAPANESE_SHIFT_JIS) {return true;}
+ if (base_enc == CHINESE_BIG5) {return true;}
+ if (base_enc == CHINESE_GB) {return true;}
+ if (base_enc == KOREAN_EUC_KR) {return true;}
+ return false;
+}
+
+
+
+// Scan raw bytes and detect most likely encoding
+// Design goals:
+// Skip over big initial stretches of seven-bit ASCII bytes very quickly
+// Thread safe
+// Works equally well on
+// 50-byte queries,
+// 5000-byte email and
+// 50000-byte web pages
+// Length 0 input returns ISO_8859_1 (ASCII) encoding
+// Setting ignore_7bit_mail_encodings effectively turns off detection of
+// UTF-7, HZ, and ISO-2022-xx
+Encoding InternalDetectEncoding(
+ CEDInternalFlags flags, const char* text, int text_length,
+ const char* url_hint, const char* http_charset_hint,
+ const char* meta_charset_hint, const int encoding_hint,
+ const Language language_hint, // User interface lang
+ const CompactEncDet::TextCorpusType corpus_type,
+ bool ignore_7bit_mail_encodings, int* bytes_consumed, bool* is_reliable,
+ Encoding* second_best_enc) {
+ *bytes_consumed = 0;
+ *is_reliable = false;
+ *second_best_enc = ASCII_7BIT;
+
+ if (text_length == 0) {
+ // Follow the spec. Text might be NULL.
+ *is_reliable = true;
+ return ISO_8859_1;
+ }
+
+ // For very short (20-50 byte) input strings that are highly likely to be
+ // all printable ASCII, our startup overhead might dominate. We have to do the
+ // full detection if the ISO-2022-xx, HZ, or UTF-7 encodings are possible.
+ // Otherwise, we can do a quick scan for printable ASCII.
+ if ((text_length <= 500) && ignore_7bit_mail_encodings &&
+ QuickPrintableAsciiScan(text, text_length)) {
+ *is_reliable = true;
+ return ASCII_7BIT;
+ }
+
+ // Go for the full boat detection
+ DetectEncodingState destate;
+ InitDetectEncodingState(&destate);
+
+ std::unique_ptr<DetailEntry[]> scoped_debug_data;
+ if (FLAGS_enc_detect_detail) {
+ // Allocate max 10 details per bigram
+ scoped_debug_data.reset(new DetailEntry[kMaxPairs * 10]);
+ destate.debug_data = scoped_debug_data.get();
+ // NOTE: destate and scoped_debug_data have exactly the same scope
+ // All other FLAGS_enc_detect_detail tests use destate.debug_data != NULL
+ }
+
+ // Get text length limits
+ // Typically, we scan the first 16KB looking for all encodings, then
+ // scan the rest (up to 256KB) a bit faster by no longer looking for
+ // interesting bytes below 0x80. This allows us to skip over runs of
+ // 7-bit-ASCII much more quickly.
+ int slow_len = minint(text_length, (FLAGS_enc_detect_slow_max_kb << 10));
+ int fast_len = minint(text_length, (FLAGS_enc_detect_fast_max_kb << 10));
+
+ // Initialize pointers.
+ // In general, we do not look at last 3 bytes of input in the fast scan
+ // We do, however want to look at the last byte or so in the slow scan,
+ // especilly in the case of a very short text whose only interesting
+ // information is a 3-byte UTF-8 character in the last three bytes.
+ // If necessary, we fake a last bigram with 0x20 space as a pad byte.
+ const uint8* isrc = reinterpret_cast<const uint8*>(text);
+ const uint8* src = isrc;
+ const uint8* srctextlimit = isrc + text_length;
+ const uint8* srclimitslow2 = isrc + slow_len - 1;
+ const uint8* srclimitfast2 = isrc + fast_len - 1;
+ const uint8* srclimitfast4 = isrc + fast_len - 3;
+ if (srclimitslow2 > srclimitfast2) {
+ srclimitslow2 = srclimitfast2;
+ }
+ destate.initial_src = isrc;
+ destate.limit_src = srclimitfast2 + 1; // May include last byte
+ destate.prior_src = isrc;
+ destate.last_pair = isrc - 2;
+
+ const char* scan_table = kTestPrintableAsciiTildePlus;
+ if (ignore_7bit_mail_encodings) {
+ // Caller wants to ignore UTF-7, HZ, ISO-2022-xx
+ // Don't stop on + (for UTF-7), nor on ~ (for HZ)
+ scan_table = kTestPrintableAscii;
+ }
+ int exit_reason = 0;
+
+ if (destate.debug_data != NULL) {
+ BeginDetail(&destate);
+ // Take any incoming watch encoding name and backmap to the corresponding
+ // ranked enum value
+ watch1_rankedenc = LookupWatchEnc(FLAGS_enc_detect_watch1);
+ if (watch1_rankedenc >= 0) {
+ fprintf(stderr, "/track-me %d def\n", watch1_rankedenc);
+ }
+
+ watch2_rankedenc = LookupWatchEnc(FLAGS_enc_detect_watch2);
+ if (watch2_rankedenc >= 0) {
+ fprintf(stderr, "/track-me2 %d def\n", watch2_rankedenc);
+ }
+
+ fprintf(stderr, "%% kDerateHintsBelow = %d\n", kDerateHintsBelow);
+ }
+ if (FLAGS_enc_detect_source) {
+ PsSourceInit(kPsSourceWidth);
+ PsSource(src, isrc, srctextlimit);
+ PsMark(src, 4, isrc, 0);
+ }
+
+ // Apply hints, if any, to probabilities
+ // NOTE: Encoding probabilites are all zero at this point
+ ApplyHints(url_hint,
+ http_charset_hint,
+ meta_charset_hint,
+ encoding_hint,
+ language_hint,
+ corpus_type,
+ &destate);
+
+ // NOTE: probabilities up to this point are subject to derating for
+ // small numbers of bigrams.
+ // Probability changes after this point are not derated.
+
+ // Do first 4 bytes to pick off strong markers
+ InitialBytesBoost(isrc, text_length, &destate);
+
+ bool ignored_some_tag_text = false;
+ int tag_text_bigram_count = 0;
+
+ // Slower loop, approx 500 MB/sec (2.8 GHz P4)
+ // ASSERT(srclimitslow2 <= srclimitfast2);
+ //====================================
+ DoMoreSlowLoop:
+ while (src < srclimitslow2) {
+ // Skip to next interesting byte (this is the slower part)
+ while (src < srclimitslow2) {
+ uint8 uc = *src++;
+ if (scan_table[uc] != 0) {exit_reason = scan_table[uc]; src--; break;}
+ }
+
+ if (src < srclimitslow2) {
+ if (FLAGS_enc_detect_source) {
+ PsSource(src, isrc, srctextlimit); // don't mark yet
+ }
+
+ int weightshift = 0;
+ // In the first 16KB, derate new text run inside <title>...</title> and
+ // inside <!-- ... -->
+ if (////((destate.last_pair + 6) <= src) && // if beyond last one
+ ////(tag_text_bigram_count < kMaxBigramsTagTitleText) &&
+ (corpus_type == CompactEncDet::WEB_CORPUS) && // and web page
+ !CEDFlagForceTags(flags)) { // and OK to skip
+ ////if (TextInsideTag(destate.last_pair + 2, src, srclimitslow2)) {
+ if (TextInsideTag(isrc, src, srclimitslow2)) {
+ if (tag_text_bigram_count >= kMaxBigramsTagTitleText) {
+ ignored_some_tag_text = true;
+ src = SkipToTagEnd(src, srclimitslow2);
+ continue;
+ } else {
+ weightshift = kWeightshiftForTagTitleText;
+ ++tag_text_bigram_count;
+ }
+ }
+ }
+ if (FLAGS_enc_detect_source) {
+ PsMark(src, 2, isrc, weightshift);
+ }
+ // Saves byte pair and offset
+ bool pruned = IncrementAndBoostPrune(src, srctextlimit - src,
+ &destate, weightshift, exit_reason);
+ // Advance; if inside tag, advance to end of tag
+ if (weightshift == 0) {
+ src += exit_reason; // 1 Ascii, 2 other
+ } else {
+ src += exit_reason; // 1 Ascii, 2 other
+ //// src = SkipToTagEnd(src, srclimitslow2);
+ }
+
+ if (pruned) {
+ // Scoring and active encodings have been updated
+ if (destate.done) {break;}
+ // Check if all the reasons for the slow loop have been pruned
+ // If so, go to fast loop
+ if (!SevenBitActive(&destate)) {break;}
+ }
+ }
+ }
+ //====================================
+
+ // We reached the end of a slow scan, possibly because no more SevenBitActive,
+ // or possibly are at end of source.
+ // If we are exactly at the end of the source, make sure we look at the very
+ // last byte.
+ bool very_last_byte_incremented = false;
+ if (src == (srctextlimit - 1)) {
+ exit_reason = scan_table[*src];
+ if (exit_reason != 0) {
+ // The very last byte is an interesting byte
+ // Saves byte pair and offset
+ //printf("Interesting very last slow byte = 0x%02x\n", *src);
+ IncrementAndBoostPrune(src, srctextlimit - src, &destate, 0, exit_reason);
+ very_last_byte_incremented = true;
+ }
+ }
+
+ if (FLAGS_enc_detect_source) {
+ PsSource(src, isrc, srctextlimit);
+ PsMark(src, 2, isrc, 0);
+ }
+ // Force a pruning based on whatever we have
+ // Delete the seven-bit encodings if there is no evidence of them so far
+ BoostPrune(src, &destate, PRUNE_SLOWEND);
+
+ if (!destate.done) {
+ // If not clear yet on 7-bit-encodings and more bytes, do more slow
+ if (SevenBitActive(&destate) && (src < srclimitfast2)) {
+ // Increment limit by another xxxK
+ slow_len += (FLAGS_enc_detect_slow_max_kb << 10);
+ srclimitslow2 = isrc + slow_len - 1;
+ if (srclimitslow2 > srclimitfast2) {
+ srclimitslow2 = srclimitfast2;
+ }
+ if (!UTF7OrHzActive(&destate)) {
+ // We can switch to table that does not stop on + ~
+ scan_table = kTestPrintableAscii;
+ }
+ goto DoMoreSlowLoop;
+ }
+
+
+ exit_reason = 2;
+ // Faster loop, no 7-bit-encodings possible, approx 3000 GB/sec
+ //====================================
+ while (src < srclimitfast2) {
+ // Skip to next interesting byte (this is the faster part)
+ while (src < srclimitfast4) {
+ if (((src[0] | src[1] | src[2] | src[3]) & 0x80) != 0) break;
+ src += 4;
+ }
+
+ while (src < srclimitfast2) {
+ if ((src[0] & 0x80) != 0) break;
+ src++;
+ }
+
+ if (src < srclimitfast2) {
+ if (FLAGS_enc_detect_source) {
+ PsSource(src, isrc, srctextlimit);
+ PsMark(src, 2, isrc, 0);
+ }
+ // saves byte pair and offset
+ bool pruned = IncrementAndBoostPrune(src, srctextlimit - src,
+ &destate, 0, exit_reason);
+ src += exit_reason; // 1 Ascii, 2 other
+ if (pruned) {
+ // Scoring and active encodings have been updated
+ if (destate.done) {break;}
+ }
+ }
+ }
+ //====================================
+ // We reached the end of fast scan
+
+ // If we are exactly at the end of the source, make sure we look at the very
+ // last byte.
+ if (src == (srctextlimit - 1) && !very_last_byte_incremented) {
+ exit_reason = scan_table[*src];
+ if (exit_reason != 0) {
+ // The very last byte is an interesting byte
+ // Saves byte pair and offset
+ //printf("Interesting very last fast byte = 0x%02x\n", *src);
+ IncrementAndBoostPrune(src, srctextlimit - src, &destate, 0, exit_reason);
+ very_last_byte_incremented = true;
+ }
+ }
+
+ } // End if !done
+
+ if (FLAGS_enc_detect_source) {
+ PsSource(src, isrc, srctextlimit);
+ PsMark(src, 2, isrc, 0);
+ }
+ // Force a pruning based on whatever we have
+ BoostPrune(src, &destate, PRUNE_FINAL);
+
+ if (FLAGS_enc_detect_summary) {
+ DumpSummary(&destate, AsciiPair, 32);
+ DumpSummary(&destate, OtherPair, 32);
+ }
+ if (FLAGS_enc_detect_source) {
+ PsSourceFinish();
+ }
+ if (destate.debug_data != NULL) {
+ //// DumpDetail(&destate);
+ }
+
+
+ if (ignored_some_tag_text &&
+ (kMapToEncoding[destate.top_rankedencoding] == ASCII_7BIT)) {
+ // There were some interesting bytes, but only in tag text.
+ // Recursive call to reprocess looking at the tags this time.
+
+ if (destate.debug_data != NULL) {
+ SetDetailsEncLabel(&destate, ">> Recurse/tags");
+ // Print the current chart before recursive call
+ DumpDetail(&destate);
+
+ char buff[32];
+ snprintf(buff, sizeof(buff), ">> Recurse for tags");
+ PsRecurse(buff);
+ }
+
+ // Recursive call for high bytes in tags [no longer used, 1/16 tag score]
+ Encoding enc2 = InternalDetectEncoding(
+ kCEDForceTags, // force
+ text,
+ text_length,
+ url_hint,
+ http_charset_hint,
+ meta_charset_hint,
+ encoding_hint,
+ language_hint,
+ corpus_type,
+ ignore_7bit_mail_encodings,
+ bytes_consumed,
+ is_reliable,
+ second_best_enc);
+
+ if (destate.debug_data != NULL) {
+ // Show winning encoding and dump PostScript
+ char buff[32];
+ snprintf(buff, sizeof(buff), "=2 %s", MyEncodingName(enc2));
+ SetDetailsEncProb(&destate,
+ 0,
+ CompactEncDet::BackmapEncodingToRankedEncoding(enc2),
+ buff);
+ DumpDetail(&destate);
+ }
+
+ return enc2;
+ }
+
+
+ // If the detected encoding does not match default/hints, or if the hints
+ // conflict with each other, mark as unreliable. This can be used to trigger
+ // further scoring.
+ // Three buckets of input documents;
+ // ~19% of the web no hints, and top == 7bit, Latin1, or CP1252
+ // ~79% of the web one or more hints, all same encoding X and top == X
+ // ~ 2% of the web one or more hints that are inconsistent
+
+ Encoding top_enc = kMapToEncoding[destate.top_rankedencoding];
+ Encoding one_hint = destate.http_hint;
+ if ((one_hint == UNKNOWN_ENCODING) &&
+ (destate.meta_hint != UNKNOWN_ENCODING)) {
+ one_hint = destate.meta_hint;
+ }
+ if ((one_hint == UNKNOWN_ENCODING) &&
+ (destate.bom_hint != UNKNOWN_ENCODING)) {
+ one_hint = destate.bom_hint;
+ }
+
+ bool found_compatible_encoding = true;
+ if (one_hint == UNKNOWN_ENCODING) {
+ // [~14% of the web] No hints, and top == 7bit, Latin1, or CP1252
+ if (!CompatibleEnc(ISO_8859_1, top_enc)) {
+ found_compatible_encoding = false;
+ // If there is nothing but a TLD hint and its top encoding matches, OK
+ if ((destate.tld_hint != UNKNOWN_ENCODING) &&
+ CompatibleEnc(destate.tld_hint, top_enc)) {
+ found_compatible_encoding = true;
+ }
+ }
+ } else if (CompatibleEnc(one_hint, destate.http_hint) &&
+ CompatibleEnc(one_hint, destate.meta_hint) &&
+ CompatibleEnc(one_hint, destate.bom_hint)) {
+ // [~83% of the web] One or more hints, all same encoding X and top == X
+ if (!CompatibleEnc(one_hint, top_enc)) {
+ // [~ 2% of the web] Oops, not the declared encoding
+ found_compatible_encoding = false;
+ }
+ } else {
+ // [~ 3% of the web] Two or more hints that are inconsistent
+ one_hint = UNKNOWN_ENCODING;
+ found_compatible_encoding = false;
+ }
+
+ // If we turned Latin1 into Latin2 or 7 via trigrams, don't fail it here
+ if (destate.do_latin_trigrams) {
+ if (CompatibleEnc(kMapToEncoding[F_Latin1], top_enc) ||
+ CompatibleEnc(kMapToEncoding[F_Latin2], top_enc) ||
+ CompatibleEnc(kMapToEncoding[F_CP1250], top_enc) ||
+ CompatibleEnc(kMapToEncoding[F_ISO_8859_13], top_enc)) {
+ found_compatible_encoding = true;
+ destate.reliable = true;
+ }
+ }
+
+ // If top encoding is not compatible with the hints, but it is reliably
+ // UTF-8, accept it anyway.
+ // This will perform badly with mixed UTF-8 prefix plus another encoding in
+ // the body if done too early, so we want to be rescanning.
+ if (!found_compatible_encoding &&
+ destate.reliable &&
+ NoHintsCloseEnoughCompatible(top_enc) &&
+ (destate.next_interesting_pair[OtherPair] >= kStrongPairs) &&
+ CEDFlagRescanning(flags)) {
+ found_compatible_encoding = true;
+ }
+
+ // Hold off on this so Rescan() can see if the original encoding was reliable
+ //if (!found_compatible_encoding) {
+ // destate.reliable = false;
+ //}
+
+ // If unreliable, try rescoring to separate some encodings
+ if (!destate.reliable || !found_compatible_encoding) {
+ top_enc = Rescore(top_enc, isrc, srctextlimit, &destate);
+ }
+
+ *second_best_enc = kMapToEncoding[destate.second_top_rankedencoding];
+
+ // If unreliable, and not already rescanning,
+ // rescan middle of document to see if we can get a better
+ // answer. Rescan is only worthwhile if there are ~200 bytes or more left,
+ // since the detector takes as much as 96 bytes of bigrams to decide.
+ //
+ // CANNOT retry ISO-2022-xx HZ etc. because no declaration escape at the front
+ // or we may land in the middle of some partial state. Skip them all.
+ //
+ if ((!destate.reliable || !found_compatible_encoding) &&
+ !CEDFlagRescanning(flags) &&
+ !SevenBitEncoding(top_enc)) {
+ top_enc = Rescan(top_enc,
+ isrc,
+ src,
+ srctextlimit,
+ url_hint,
+ http_charset_hint,
+ meta_charset_hint,
+ encoding_hint,
+ language_hint,
+ corpus_type,
+ ignore_7bit_mail_encodings,
+ &destate);
+ } else {
+ if (!found_compatible_encoding) {
+ destate.reliable = false;
+ }
+ }
+
+ if (destate.debug_data != NULL) {
+ // Dump PostScript
+ DumpDetail(&destate);
+ }
+
+ *bytes_consumed = src - isrc + 1; // We looked 1 byte beyond src
+ *is_reliable = destate.reliable;
+ return top_enc;
+}
+
+Encoding CompactEncDet::DetectEncoding(
+ const char* text, int text_length, const char* url_hint,
+ const char* http_charset_hint, const char* meta_charset_hint,
+ const int encoding_hint,
+ const Language language_hint, // User interface lang
+ const TextCorpusType corpus_type, bool ignore_7bit_mail_encodings,
+ int* bytes_consumed, bool* is_reliable) {
+ if (FLAGS_ced_echo_input) {
+ string temp(text, text_length);
+ fprintf(stderr, "CompactEncDet::DetectEncoding()\n%s\n\n", temp.c_str());
+ }
+
+ if (FLAGS_counts) {
+ encdet_used = 0;
+ rescore_used = 0;
+ rescan_used = 0;
+ robust_used = 0;
+ looking_used = 0;
+ doing_used = 0;
+ ++encdet_used;
+ }
+ if (FLAGS_dirtsimple) {
+ // Just count first 64KB bigram encoding probabilities for each encoding
+ int robust_renc_list_len; // Number of active encodings
+ int robust_renc_list[NUM_RANKEDENCODING]; // List of ranked encodings
+ int robust_renc_probs[NUM_RANKEDENCODING]; // List of matching probs
+
+ for (int i = 0; i < NUM_RANKEDENCODING; ++i) {
+ robust_renc_list[i] = i;
+ }
+ robust_renc_list_len = NUM_RANKEDENCODING;
+
+ RobustScan(text, text_length,
+ robust_renc_list_len, robust_renc_list, robust_renc_probs);
+
+ // Pick off best encoding
+ int best_prob = -1;
+ Encoding enc = UNKNOWN_ENCODING;
+ for (int i = 0; i < robust_renc_list_len; ++i) {
+ if (best_prob < robust_renc_probs[i]) {
+ best_prob = robust_renc_probs[i];
+ enc = kMapToEncoding[robust_renc_list[i]];
+ }
+ }
+
+ *bytes_consumed = minint(text_length, (kMaxKBToRobustScan << 10));
+ *is_reliable = true;
+ if (FLAGS_counts) {
+ printf("CEDcounts ");
+ while (encdet_used--) {printf("encdet ");}
+ while (rescore_used--) {printf("rescore ");}
+ while (rescan_used--) {printf("rescan ");}
+ while (robust_used--) {printf("robust ");}
+ while (looking_used--) {printf("looking ");}
+ while (doing_used--) {printf("doing ");}
+ printf("\n");
+ }
+
+ return enc;
+ }
+
+ Encoding second_best_enc;
+ Encoding enc = InternalDetectEncoding(kCEDNone,
+ text,
+ text_length,
+ url_hint,
+ http_charset_hint,
+ meta_charset_hint,
+ encoding_hint,
+ language_hint, // User interface lang
+ corpus_type,
+ ignore_7bit_mail_encodings,
+ bytes_consumed,
+ is_reliable,
+ &second_best_enc);
+ if (FLAGS_counts) {
+ printf("CEDcounts ");
+ while (encdet_used--) {printf("encdet ");}
+ while (rescore_used--) {printf("rescore ");}
+ while (rescan_used--) {printf("rescan ");}
+ while (robust_used--) {printf("robust ");}
+ while (looking_used--) {printf("looking ");}
+ while (doing_used--) {printf("doing ");}
+ printf("\n");
+ }
+
+#if defined(HTML5_MODE)
+ // Map all the Shift-JIS variants to Shift-JIS when used in Japanese locale.
+ if (language_hint == JAPANESE && IsShiftJisOrVariant(enc)) {
+ enc = JAPANESE_SHIFT_JIS;
+ }
+
+ // 7-bit encodings (except ISO-2022-JP), and some obscure encodings not
+ // supported in WHATWG encoding standard are marked as ASCII to keep the raw
+ // bytes intact.
+ switch (enc) {
+ case ISO_2022_KR:
+ case ISO_2022_CN:
+ case HZ_GB_2312:
+ case UTF7:
+ case UTF16LE:
+ case UTF16BE:
+
+ case CHINESE_EUC_DEC:
+ case CHINESE_CNS:
+ case CHINESE_BIG5_CP950:
+ case JAPANESE_CP932:
+ case MSFT_CP874:
+ case TSCII:
+ case TAMIL_MONO:
+ case TAMIL_BI:
+ case JAGRAN:
+ case BHASKAR:
+ case HTCHANAKYA:
+ case BINARYENC:
+ case UTF8UTF8:
+ case TAM_ELANGO:
+ case TAM_LTTMBARANI:
+ case TAM_SHREE:
+ case TAM_TBOOMIS:
+ case TAM_TMNEWS:
+ case TAM_WEBTAMIL:
+ case KDDI_SHIFT_JIS:
+ case DOCOMO_SHIFT_JIS:
+ case SOFTBANK_SHIFT_JIS:
+ case KDDI_ISO_2022_JP:
+ case SOFTBANK_ISO_2022_JP:
+ enc = ASCII_7BIT;
+ break;
+ default:
+ break;
+ }
+#endif
+
+ return enc;
+}
+
+
+// Return top encoding hint for given string
+Encoding CompactEncDet::TopEncodingOfLangHint(const char* name) {
+ string normalized_lang = MakeChar8(string(name));
+ int n = HintBinaryLookup8(kLangHintProbs, kLangHintProbsSize,
+ normalized_lang.c_str());
+ if (n < 0) {return UNKNOWN_ENCODING;}
+
+ // Charset is eight bytes, probability table is eight bytes
+ int toprankenc =
+ TopCompressedProb((const char *)&kLangHintProbs[n].key_prob[kMaxLangKey],
+ kMaxLangVector);
+ return kMapToEncoding[toprankenc];
+}
+
+// Return top encoding hint for given string
+Encoding CompactEncDet::TopEncodingOfTLDHint(const char* name) {
+ string normalized_tld = MakeChar4(string(name));
+ int n = HintBinaryLookup4(kTLDHintProbs, kTLDHintProbsSize,
+ normalized_tld.c_str());
+ if (n < 0) {return UNKNOWN_ENCODING;}
+
+ // TLD is four bytes, probability table is 12 bytes
+ int toprankenc =
+ TopCompressedProb((const char *)&kTLDHintProbs[n].key_prob[kMaxTldKey],
+ kMaxTldVector);
+ return kMapToEncoding[toprankenc];
+}
+
+// Return top encoding hint for given string
+Encoding CompactEncDet::TopEncodingOfCharsetHint(const char* name) {
+ string normalized_charset = MakeChar44(string(name));
+ int n = HintBinaryLookup8(kCharsetHintProbs, kCharsetHintProbsSize,
+ normalized_charset.c_str());
+ if (n < 0) {return UNKNOWN_ENCODING;}
+
+ // Charset is eight bytes, probability table is eight bytes
+ int toprankenc =
+ TopCompressedProb((const char *)&kCharsetHintProbs[n].key_prob[kMaxCharsetKey],
+ kMaxCharsetVector);
+ return kMapToEncoding[toprankenc];
+}
+
+const char* CompactEncDet::Version(void) {
+ return kVersion;
+}
diff --git a/contrib/google-ced/compact_enc_det.h b/contrib/google-ced/compact_enc_det.h
new file mode 100644
index 0000000..01adf45
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det.h
@@ -0,0 +1,83 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_H_
+#define COMPACT_ENC_DET_COMPACT_ENC_DET_H_
+
+#include "util/encodings/encodings.h" // for Encoding
+#include "util/languages/languages.h" // for Language
+
+#include <string.h>
+
+namespace CompactEncDet {
+ // We may want different statistics, depending on whether the text being
+ // identfied is from the web, from email, etc. This is currently ignored,
+ // except WEB_CORPUS enables ignoring chars inside tags.
+ enum TextCorpusType {
+ WEB_CORPUS,
+ XML_CORPUS,
+ QUERY_CORPUS, // Use this for vanilla plaintext
+ EMAIL_CORPUS,
+ NUM_CORPA, // always last
+ };
+
+ // Scan raw bytes and detect most likely encoding
+ // Design goals:
+ // Skip over big initial stretches of seven-bit ASCII bytes very quickly
+ // Thread safe
+ // Works equally well on
+ // 50-byte queries,
+ // 5000-byte email and
+ // 50000-byte web pages
+ // Length 0 input returns ASCII (aka ISO-8859-1 or Latin1)
+ //
+ // Inputs: text and text_length
+ // web page's url (preferred) or just
+ // top-level domain name (e.g. "com") or NULL as a hint
+ // web page's HTTPheader charset= string (e.g. "Latin1") or NULL as a hint
+ // web page's <meta> tag charset= string (e.g. "utf-8") or NULL as a hint
+ // an Encoding or UNKNOWN_ENCODING as a hint
+ // a Language or UNKNOWN_LANGUAGE as a hint
+ // corpus type from the list above. Currently ignored; may select
+ // different probability tables in the future
+ // ignore_7bit if true says to NOT return the pure seven-bit encodings
+ // ISO-2022-JP (aka JIS), ISO-2022-CN, ISO-2022-KR, HZ, and UTF-7.
+ // This may save a little scoring time on pure printable ASCII input text
+ // Outputs: bytes_consumed says how much of text_length was actually examined
+ // is_reliable set true if the returned encoding is at least 2**10 time more
+ // probable then the second-best encoding
+ // Return value: the most likely encoding for the input text
+ //
+ // Setting ignore_7bit_mail_encodings effectively turns off detection of
+ // UTF-7, HZ, and ISO-2022-xx. It is recommended that this flag be true
+ // when corpus_type is QUERY_CORPUS.
+ Encoding DetectEncoding(
+ const char* text, int text_length, const char* url_hint,
+ const char* http_charset_hint, const char* meta_charset_hint,
+ const int encoding_hint,
+ const Language language_hint, // User interface lang
+ const TextCorpusType corpus_type, bool ignore_7bit_mail_encodings,
+ int* bytes_consumed, bool* is_reliable);
+
+ // Support functions for unit test program
+ int BackmapEncodingToRankedEncoding(Encoding enc);
+ Encoding TopEncodingOfLangHint(const char* name);
+ Encoding TopEncodingOfTLDHint(const char* name);
+ Encoding TopEncodingOfCharsetHint(const char* name);
+ const char* Version(void);
+} // End namespace CompactEncDet
+
+#endif // COMPACT_ENC_DET_COMPACT_ENC_DET_H_
diff --git a/contrib/google-ced/compact_enc_det_generated_tables.h b/contrib/google-ced/compact_enc_det_generated_tables.h
new file mode 100644
index 0000000..d2174a1
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det_generated_tables.h
@@ -0,0 +1,6326 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_GENERATED_TABLES_H_
+#define COMPACT_ENC_DET_COMPACT_ENC_DET_GENERATED_TABLES_H_
+
+#include "compact_enc_det.h"
+#include "compact_enc_det_generated_tables2.h"
+#include "util/basictypes.h"
+#include "util/encodings/encodings.pb.h"
+
+enum RankedEncoding {
+ F_ASCII_7_bit, // [0] encoding 24
+ F_Latin1, // [1] encoding 0
+ F_UTF8, // [2] encoding 22
+ F_GB, // [3] encoding 14
+ F_CP1252, // [4] encoding 27
+ F_KSC, // [5] encoding 16
+ F_SJS, // [6] encoding 11
+ F_EUC_JP, // [7] encoding 10
+ F_BIG5, // [8] encoding 13
+ F_Latin2, // [9] encoding 1
+ F_CP1251, // [10] encoding 26
+ F_CP1256, // [11] encoding 35
+ F_CP1250, // [12] encoding 29
+ F_Latin5, // [13] encoding 8
+ F_ISO_8859_11, // [14] encoding 33
+ F_ISO_8859_15, // [15] encoding 30
+ F_CP1257, // [16] encoding 32
+ F_CP1255, // [17] encoding 36
+ F_KOI8R, // [18] encoding 25
+ F_GBK, // [19] encoding 45
+ F_Greek, // [20] encoding 6
+ F_JIS, // [21] encoding 12
+ F_CP1254, // [22] encoding 31
+ F_CP1253, // [23] encoding 41
+ F_CP932, // [24] encoding 21
+ F_Hebrew, // [25] encoding 7
+ F_KOI8U, // [26] encoding 28
+ F_ISO_8859_5, // [27] encoding 4
+ F_CP874, // [28] encoding 34
+ F_ISO_8859_13, // [29] encoding 43
+ F_Latin4, // [30] encoding 3
+ F_MACINTOSH, // [31] encoding 53
+ F_GB18030, // [32] encoding 46
+ F_CP852, // [33] encoding 39
+ F_Arabic, // [34] encoding 5
+ F_BIG5_HKSCS, // [35] encoding 47
+ F_CP866, // [36] encoding 42
+ F_UTF_16BE, // [37] encoding 57
+ F_Latin3, // [38] encoding 2
+ F_UTF_16LE, // [39] encoding 58
+ F_HZ_GB_2312, // [40] encoding 62
+ F_CSN_369103, // [41] encoding 40
+ F_ISO_2022_KR, // [42] encoding 44
+ F_Latin6, // [43] encoding 9
+ F_UTF7, // [44] encoding 54
+ F_ISO_2022_CN, // [45] encoding 48
+ F_BIG5_CP950, // [46] encoding 20
+ F_JAGRAN, // [47] encoding 52
+ F_BHASKAR, // [48] encoding 55
+ F_HTCHANAKYA, // [49] encoding 56
+ F_TSCII, // [50] encoding 49
+ F_TAM, // [51] encoding 50
+ F_TAB, // [52] encoding 51
+ F_EUC_CN, // [53] encoding 15
+ F_EUC, // [54] encoding 18
+ F_CNS, // [55] encoding 19
+ F_UTF_32BE, // [56] encoding 59
+ F_UTF_32LE, // [57] encoding 60
+ F_X_BINARYENC, // [58] encoding 61
+ F_X_UTF8UTF8, // [59] encoding 63
+ F_X_TAM_ELANGO, // [60] encoding 64
+ F_X_TAM_LTTMBARANI, // [61] encoding 65
+ F_X_TAM_SHREE, // [62] encoding 66
+ F_X_TAM_TBOOMIS, // [63] encoding 67
+ F_X_TAM_TMNEWS, // [64] encoding 68
+ F_X_TAM_WEBTAMIL, // [65] encoding 69
+ F_UTF8CP1252, // [66] encoding 63
+ NUM_RANKEDENCODING
+};
+
+static const Encoding kMapToEncoding[NUM_RANKEDENCODING] = {
+ ASCII_7BIT, // encoding 24
+ ISO_8859_1, // encoding 0
+ UTF8, // encoding 22
+ CHINESE_GB, // encoding 14
+ MSFT_CP1252, // encoding 27
+ KOREAN_EUC_KR, // encoding 16
+ JAPANESE_SHIFT_JIS, // encoding 11
+ JAPANESE_EUC_JP, // encoding 10
+ CHINESE_BIG5, // encoding 13
+ ISO_8859_2, // encoding 1
+ RUSSIAN_CP1251, // encoding 26
+ MSFT_CP1256, // encoding 35
+ MSFT_CP1250, // encoding 29
+ ISO_8859_9, // encoding 8
+ ISO_8859_11, // encoding 33
+ ISO_8859_15, // encoding 30
+ MSFT_CP1257, // encoding 32
+ MSFT_CP1255, // encoding 36
+ RUSSIAN_KOI8_R, // encoding 25
+ GBK, // encoding 45
+ ISO_8859_7, // encoding 6
+ JAPANESE_JIS, // encoding 12
+ MSFT_CP1254, // encoding 31
+ MSFT_CP1253, // encoding 41
+ JAPANESE_CP932, // encoding 21
+ ISO_8859_8, // encoding 7
+ RUSSIAN_KOI8_RU, // encoding 28
+ ISO_8859_5, // encoding 4
+ MSFT_CP874, // encoding 34
+ ISO_8859_13, // encoding 43
+ ISO_8859_4, // encoding 3
+ MACINTOSH_ROMAN, // encoding 53
+ GB18030, // encoding 46
+ CZECH_CP852, // encoding 39
+ ISO_8859_6, // encoding 5
+ BIG5_HKSCS, // encoding 47
+ RUSSIAN_CP866, // encoding 42
+ UTF16BE, // encoding 57
+ ISO_8859_3, // encoding 2
+ UTF16LE, // encoding 58
+ HZ_GB_2312, // encoding 62
+ CZECH_CSN_369103, // encoding 40
+ ISO_2022_KR, // encoding 44
+ ISO_8859_10, // encoding 9
+ UTF7, // encoding 54
+ ISO_2022_CN, // encoding 48
+ CHINESE_BIG5_CP950, // encoding 20
+ JAGRAN, // encoding 52
+ BHASKAR, // encoding 55
+ HTCHANAKYA, // encoding 56
+ TSCII, // encoding 49
+ TAMIL_MONO, // encoding 50
+ TAMIL_BI, // encoding 51
+ CHINESE_EUC_CN, // encoding 15
+ CHINESE_EUC_DEC, // encoding 18
+ CHINESE_CNS, // encoding 19
+ UTF32BE, // encoding 59
+ UTF32LE, // encoding 60
+ BINARYENC, // encoding 61
+ UTF8UTF8, // encoding 63
+ TAM_ELANGO, // encoding 64
+ TAM_LTTMBARANI, // encoding 65
+ TAM_SHREE, // encoding 66
+ TAM_TBOOMIS, // encoding 67
+ TAM_TMNEWS, // encoding 68
+ TAM_WEBTAMIL, // encoding 69
+ UTF8UTF8, // encoding 63
+};
+
+// Massaged TLD or charset, followed by packed encoding probs
+typedef struct {
+ unsigned char key_prob[20];
+} HintEntry;
+
+static const HintEntry kLangHintProbs[] = { // MaxRange 192
+ {{0x61,0x62,0x6b,0x68,0x61,0x7a,0x69,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "abkhazia"
+ // UTF8=191 [top UTF8]
+ {{0x61,0x66,0x61,0x72,0x5f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "afar____"
+ // UTF8=191 [top UTF8]
+ {{0x61,0x66,0x72,0x69,0x6b,0x61,0x61,0x6e, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "afrikaan"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x61,0x6c,0x62,0x61,0x6e,0x69,0x61,0x6e, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "albanian"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x61,0x6d,0x68,0x61,0x72,0x69,0x63,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "amharic_"
+ // UTF8=191 [top UTF8]
+ {{0x61,0x72,0x61,0x62,0x69,0x63,0x5f,0x5f, 0x03,0x84,0x53,0xa2,0x11,0x3b,0x62,0xbc,0x34,0x10,0x51,0x83,}}, // "arabic__"
+ // ASCII-7-bit=132 Latin1=83 UTF8=162 CP1252=59 CP1256=188 CP1250=52 Arabic=131 [top CP1256]
+ {{0x61,0x72,0x6d,0x65,0x6e,0x69,0x61,0x6e, 0x01,0x5f,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "armenian"
+ // ASCII-7-bit=95 UTF8=190 [top UTF8]
+ {{0x61,0x73,0x73,0x61,0x6d,0x65,0x73,0x65, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "assamese"
+ // UTF8=191 [top UTF8]
+ {{0x61,0x79,0x6d,0x61,0x72,0x61,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "aymara__"
+ // UTF8=191 [top UTF8]
+ {{0x61,0x7a,0x65,0x72,0x62,0x61,0x69,0x6a, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "azerbaij"
+ // UTF8=191 [top UTF8]
+ {{0x62,0x61,0x73,0x68,0x6b,0x69,0x72,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bashkir_"
+ // UTF8=191 [top UTF8]
+ {{0x62,0x61,0x73,0x71,0x75,0x65,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "basque__"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x62,0x65,0x6c,0x61,0x72,0x75,0x73,0x69, 0xa1,0xb5,0x71,0xa1,0x72,0x97,0xab,0x81,0x8d,0x00,0x00,0x00,}}, // "belarusi"
+ // CP1251=181 KOI8R=161 KOI8U=151 ISO-8859-5=171 CP866=141 [top CP1251]
+ {{0x62,0x65,0x6e,0x67,0x61,0x6c,0x69,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bengali_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x62,0x69,0x68,0x61,0x72,0x69,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bihari__"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x62,0x69,0x73,0x6c,0x61,0x6d,0x61,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bislama_"
+ // UTF8=191 [top UTF8]
+ {{0x62,0x6f,0x73,0x6e,0x69,0x61,0x6e,0x5f, 0x91,0xaf,0x21,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bosnian_"
+ // Latin2=175 CP1250=185 [top CP1250]
+ {{0x62,0x72,0x65,0x74,0x6f,0x6e,0x5f,0x5f, 0x11,0xb5,0x21,0x97,0x81,0xab,0x11,0xa1,0x00,0x00,0x00,0x00,}}, // "breton__"
+ // Latin1=181 CP1252=151 Latin5=171 ISO-8859-15=161 [top Latin1]
+ {{0x62,0x75,0x6c,0x67,0x61,0x72,0x69,0x61, 0x03,0x70,0x47,0xad,0x11,0x45,0x51,0xb5,0x71,0x95,0x81,0x9f,}}, // "bulgaria"
+ // ASCII-7-bit=112 Latin1=71 UTF8=173 CP1252=69 CP1251=181 KOI8R=149 ISO-8859-5=159 [top CP1251]
+ {{0x62,0x75,0x72,0x6d,0x65,0x73,0x65,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "burmese_"
+ // UTF8=191 [top UTF8]
+ {{0x63,0x61,0x74,0x61,0x6c,0x61,0x6e,0x5f, 0x03,0x8b,0xb8,0xa0,0x11,0xa4,0xa1,0x96,0x10,0x61,0x31,0x00,}}, // "catalan_"
+ // ASCII-7-bit=139 Latin1=184 UTF8=160 CP1252=164 ISO-8859-15=150 Latin3=49 [top Latin1]
+ {{0x63,0x68,0x65,0x72,0x6f,0x6b,0x65,0x65, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cherokee"
+ // UTF8=191 [top UTF8]
+ {{0x63,0x68,0x69,0x6e,0x65,0x73,0x65,0x5f, 0x01,0x5c,0x12,0xa8,0xbb,0x11,0x74,0x21,0x6d,0xa1,0x7d,0x00,}}, // "chinese_"
+ // ASCII-7-bit=92 UTF8=168 GB=187 KSC=116 BIG5=109 GBK=125 [top GB]
+ {{0x63,0x68,0x69,0x6e,0x65,0x73,0x65,0x74, 0x06,0x73,0x5f,0xad,0x59,0x43,0x36,0x21,0xb9,0x10,0xa1,0x38,}}, // "chineset"
+ // ASCII-7-bit=115 Latin1=95 UTF8=173 GB=89 CP1252=67 KSC=54 BIG5=185 BIG5_HKSCS=56 [top BIG5]
+ {{0x63,0x6f,0x72,0x73,0x69,0x63,0x61,0x6e, 0x12,0xaf,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "corsican"
+ // Latin1=175 UTF8=185 [top UTF8]
+ {{0x63,0x72,0x65,0x6f,0x6c,0x65,0x73,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "creolesa"
+ // UTF8=191 [top UTF8]
+ {{0x63,0x72,0x6f,0x61,0x74,0x69,0x61,0x6e, 0x03,0x91,0x7b,0xa6,0x11,0x86,0x41,0xac,0x21,0xb4,0x31,0x4d,}}, // "croatian"
+ // ASCII-7-bit=145 Latin1=123 UTF8=166 CP1252=134 Latin2=172 CP1250=180 CP1257=77 [top CP1250]
+ {{0x63,0x7a,0x65,0x63,0x68,0x5f,0x5f,0x5f, 0x01,0x89,0x11,0xb1,0x61,0x98,0x21,0xb5,0x10,0x41,0x7d,0x00,}}, // "czech___"
+ // ASCII-7-bit=137 UTF8=177 Latin2=152 CP1250=181 CP852=125 [top CP1250]
+ {{0x64,0x61,0x6e,0x69,0x73,0x68,0x5f,0x5f, 0x03,0x99,0xb8,0xa6,0x11,0x9a,0x41,0x38,0x21,0x32,0x21,0x84,}}, // "danish__"
+ // ASCII-7-bit=153 Latin1=184 UTF8=166 CP1252=154 Latin2=56 CP1250=50 ISO-8859-15=132 [top Latin1]
+ {{0x64,0x68,0x69,0x76,0x65,0x68,0x69,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dhivehi_"
+ // UTF8=191 [top UTF8]
+ {{0x64,0x75,0x74,0x63,0x68,0x5f,0x5f,0x5f, 0x03,0xb1,0xae,0xa3,0x11,0xa1,0x41,0x41,0x21,0x44,0x21,0x7f,}}, // "dutch___"
+ // ASCII-7-bit=177 Latin1=174 UTF8=163 CP1252=161 Latin2=65 CP1250=68 ISO-8859-15=127 [top ASCII-7-bit]
+ {{0x64,0x7a,0x6f,0x6e,0x67,0x6b,0x68,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dzongkha"
+ // UTF8=191 [top UTF8]
+ {{0x65,0x6e,0x67,0x6c,0x69,0x73,0x68,0x5f, 0x06,0xb9,0xa0,0xa2,0x5d,0x94,0x55,0x21,0x56,0x61,0x69,0x00,}}, // "english_"
+ // ASCII-7-bit=185 Latin1=160 UTF8=162 GB=93 CP1252=148 KSC=85 BIG5=86 ISO-8859-15=105 [top ASCII-7-bit]
+ {{0x65,0x73,0x70,0x65,0x72,0x61,0x6e,0x74, 0x03,0x89,0xb4,0xa2,0x12,0xaa,0x45,0x61,0x4c,0x21,0xa0,0x00,}}, // "esperant"
+ // ASCII-7-bit=137 Latin1=180 UTF8=162 CP1252=170 KSC=69 CP1250=76 ISO-8859-15=160 [top Latin1]
+ {{0x65,0x73,0x74,0x6f,0x6e,0x69,0x61,0x6e, 0x03,0x90,0xab,0xb1,0x11,0x91,0xa2,0x7e,0xa3,0xc2,0x8e,0x98,}}, // "estonian"
+ // ASCII-7-bit=144 Latin1=171 UTF8=177 CP1252=145 ISO-8859-15=126 CP1257=163 ISO-8859-13=142 Latin4=152 [top UTF8]
+ {{0x66,0x61,0x72,0x6f,0x65,0x73,0x65,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "faroese_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x66,0x69,0x6a,0x69,0x61,0x6e,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fijian__"
+ // UTF8=191 [top UTF8]
+ {{0x66,0x69,0x6e,0x6e,0x69,0x73,0x68,0x5f, 0x03,0x96,0xb7,0xa9,0x11,0x9c,0x71,0x42,0x22,0x8b,0x39,0x00,}}, // "finnish_"
+ // ASCII-7-bit=150 Latin1=183 UTF8=169 CP1252=156 CP1250=66 ISO-8859-15=139 CP1257=57 [top Latin1]
+ {{0x66,0x72,0x65,0x6e,0x63,0x68,0x5f,0x5f, 0x03,0x99,0xb6,0xaa,0x11,0xa0,0x62,0x4f,0x46,0x21,0x86,0x00,}}, // "french__"
+ // ASCII-7-bit=153 Latin1=182 UTF8=170 CP1252=160 CP1256=79 CP1250=70 ISO-8859-15=134 [top Latin1]
+ {{0x66,0x72,0x69,0x73,0x69,0x61,0x6e,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "frisian_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x67,0x61,0x6c,0x69,0x63,0x69,0x61,0x6e, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "galician"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x67,0x61,0x6e,0x64,0x61,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ganda___"
+ // UTF8=191 [top UTF8]
+ {{0x67,0x65,0x6f,0x72,0x67,0x69,0x61,0x6e, 0x01,0x6c,0x11,0xbe,0x11,0x1c,0x10,0x21,0x1c,0x00,0x00,0x00,}}, // "georgian"
+ // ASCII-7-bit=108 UTF8=190 CP1252=28 CP1253=28 [top UTF8]
+ {{0x67,0x65,0x72,0x6d,0x61,0x6e,0x5f,0x5f, 0x03,0xa2,0xb7,0xa6,0x11,0x9b,0x41,0x56,0x21,0x5d,0x21,0x7c,}}, // "german__"
+ // ASCII-7-bit=162 Latin1=183 UTF8=166 CP1252=155 Latin2=86 CP1250=93 ISO-8859-15=124 [top Latin1]
+ {{0x67,0x72,0x65,0x65,0x6b,0x5f,0x5f,0x5f, 0x03,0x81,0x54,0xad,0x11,0x52,0xd1,0x31,0x11,0xb4,0x21,0xa6,}}, // "greek___"
+ // ASCII-7-bit=129 Latin1=84 UTF8=173 CP1252=82 KOI8R=49 Greek=180 CP1253=166 [top Greek]
+ {{0x67,0x72,0x65,0x65,0x6e,0x6c,0x61,0x6e, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "greenlan"
+ // UTF8=191 [top UTF8]
+ {{0x67,0x75,0x61,0x72,0x61,0x6e,0x69,0x5f, 0x11,0xb9,0x20,0x91,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "guarani_"
+ // Latin1=185 Latin6=175 [top Latin1]
+ {{0x67,0x75,0x6a,0x61,0x72,0x61,0x74,0x69, 0x03,0x79,0xb6,0x76,0x11,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,}}, // "gujarati"
+ // ASCII-7-bit=121 Latin1=182 UTF8=118 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x68,0x61,0x69,0x74,0x69,0x61,0x6e,0x63, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "haitianc"
+ // UTF8=191 [top UTF8]
+ {{0x68,0x61,0x75,0x73,0x61,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "hausa___"
+ // UTF8=191 [top UTF8]
+ {{0x68,0x65,0x62,0x72,0x65,0x77,0x5f,0x5f, 0x03,0x76,0x46,0xab,0x11,0x3b,0x51,0x32,0x61,0xb8,0x71,0x9f,}}, // "hebrew__"
+ // ASCII-7-bit=118 Latin1=70 UTF8=171 CP1252=59 CP1251=50 CP1255=184 Hebrew=159 [top CP1255]
+ {{0x68,0x69,0x6e,0x64,0x69,0x5f,0x5f,0x5f, 0x11,0xb5,0x21,0xab,0xa1,0xa1,0x10,0xf3,0x97,0x8d,0x83,0x00,}}, // "hindi___"
+ // Latin1=181 CP1252=171 ISO-8859-15=161 JAGRAN=151 BHASKAR=141 HTCHANAKYA=131 [top Latin1]
+ {{0x68,0x75,0x6e,0x67,0x61,0x72,0x69,0x61, 0x03,0x93,0x9f,0xad,0x11,0x6f,0x41,0xae,0x21,0xa9,0x21,0x40,}}, // "hungaria"
+ // ASCII-7-bit=147 Latin1=159 UTF8=173 CP1252=111 Latin2=174 CP1250=169 ISO-8859-15=64 [top Latin2]
+ {{0x69,0x63,0x65,0x6c,0x61,0x6e,0x64,0x69, 0x03,0x7f,0xb8,0x9c,0x11,0xa4,0x11,0x1d,0x51,0x2f,0x21,0x99,}}, // "icelandi"
+ // ASCII-7-bit=127 Latin1=184 UTF8=156 CP1252=164 SJS=29 CP1250=47 ISO-8859-15=153 [top Latin1]
+ {{0x69,0x6e,0x64,0x6f,0x6e,0x65,0x73,0x69, 0x03,0xb2,0xae,0x99,0x11,0xa2,0x11,0x5b,0x41,0x70,0x31,0x91,}}, // "indonesi"
+ // ASCII-7-bit=178 Latin1=174 UTF8=153 CP1252=162 SJS=91 CP1256=112 ISO-8859-15=145 [top ASCII-7-bit]
+ {{0x69,0x6e,0x74,0x65,0x72,0x6c,0x69,0x6e, 0x12,0xb0,0xb0,0x11,0xa6,0xa1,0x9c,0x00,0x00,0x00,0x00,0x00,}}, // "interlin"
+ // Latin1=176 UTF8=176 CP1252=166 ISO-8859-15=156 [top Latin1]
+ {{0x69,0x6e,0x75,0x6b,0x74,0x69,0x74,0x75, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "inuktitu"
+ // UTF8=191 [top UTF8]
+ {{0x69,0x6e,0x75,0x70,0x69,0x61,0x6b,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "inupiak_"
+ // UTF8=191 [top UTF8]
+ {{0x69,0x72,0x69,0x73,0x68,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "irish___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x69,0x74,0x61,0x6c,0x69,0x61,0x6e,0x5f, 0x03,0xa7,0xb4,0xa4,0x11,0xa4,0x41,0x4d,0x21,0x55,0x21,0x78,}}, // "italian_"
+ // ASCII-7-bit=167 Latin1=180 UTF8=164 CP1252=164 Latin2=77 CP1250=85 ISO-8859-15=120 [top Latin1]
+ {{0x6a,0x61,0x70,0x61,0x6e,0x65,0x73,0x65, 0x01,0x68,0x11,0xa7,0x32,0xb4,0xad,0xd1,0x78,0x21,0x62,0x00,}}, // "japanese"
+ // ASCII-7-bit=104 UTF8=167 SJS=180 EUC-JP=173 JIS=120 CP932=98 [top SJS]
+ {{0x6a,0x61,0x76,0x61,0x6e,0x65,0x73,0x65, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "javanese"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6b,0x61,0x6e,0x6e,0x61,0x64,0x61,0x5f, 0x03,0x65,0xb6,0x81,0x11,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,}}, // "kannada_"
+ // ASCII-7-bit=101 Latin1=182 UTF8=129 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6b,0x61,0x73,0x68,0x6d,0x69,0x72,0x69, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kashmiri"
+ // UTF8=191 [top UTF8]
+ {{0x6b,0x61,0x7a,0x61,0x6b,0x68,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kazakh__"
+ // UTF8=191 [top UTF8]
+ {{0x6b,0x68,0x61,0x73,0x69,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "khasi___"
+ // UTF8=191 [top UTF8]
+ {{0x6b,0x68,0x6d,0x65,0x72,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "khmer___"
+ // UTF8=191 [top UTF8]
+ {{0x6b,0x69,0x6e,0x79,0x61,0x72,0x77,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kinyarwa"
+ // UTF8=191 [top UTF8]
+ {{0x6b,0x6f,0x72,0x65,0x61,0x6e,0x5f,0x5f, 0x06,0x5d,0x34,0x9d,0x20,0x1a,0xbd,0x11,0x0c,0x20,0x21,0x76,}}, // "korean__"
+ // ASCII-7-bit=93 Latin1=52 UTF8=157 GB=32 CP1252=26 KSC=189 EUC-JP=12 ISO-2022-KR=118 [top KSC]
+ {{0x6b,0x75,0x72,0x64,0x69,0x73,0x68,0x5f, 0xb1,0xb9,0x10,0x61,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kurdish_"
+ // CP1256=185 Arabic=175 [top CP1256]
+ {{0x6b,0x79,0x72,0x67,0x79,0x7a,0x5f,0x5f, 0x10,0x61,0xaf,0x41,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kyrgyz__"
+ // CP1254=175 ISO-8859-5=185 [top ISO-8859-5]
+ {{0x6c,0x61,0x6f,0x74,0x68,0x69,0x61,0x6e, 0x01,0x40,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "laothian"
+ // ASCII-7-bit=64 UTF8=190 [top UTF8]
+ {{0x6c,0x61,0x74,0x69,0x6e,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "latin___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6c,0x61,0x74,0x76,0x69,0x61,0x6e,0x5f, 0x03,0x80,0x55,0xac,0x11,0x64,0xb1,0xb4,0xc2,0x99,0xa3,0x00,}}, // "latvian_"
+ // ASCII-7-bit=128 Latin1=85 UTF8=172 CP1252=100 CP1257=180 ISO-8859-13=153 Latin4=163 [top CP1257]
+ {{0x6c,0x69,0x6d,0x62,0x75,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "limbu___"
+ // UTF8=191 [top UTF8]
+ {{0x6c,0x69,0x6e,0x67,0x61,0x6c,0x61,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lingala_"
+ // UTF8=191 [top UTF8]
+ {{0x6c,0x69,0x74,0x68,0x75,0x61,0x6e,0x69, 0x03,0x7c,0x5d,0xaa,0x11,0x73,0xb1,0xb7,0xc2,0x94,0x9d,0x00,}}, // "lithuani"
+ // ASCII-7-bit=124 Latin1=93 UTF8=170 CP1252=115 CP1257=183 ISO-8859-13=148 Latin4=157 [top CP1257]
+ {{0x6c,0x75,0x78,0x65,0x6d,0x62,0x6f,0x75, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "luxembou"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6d,0x61,0x63,0x65,0x64,0x6f,0x6e,0x69, 0x03,0x7a,0x54,0xa9,0x11,0x4b,0x51,0xb3,0x71,0x9e,0x81,0xa8,}}, // "macedoni"
+ // ASCII-7-bit=122 Latin1=84 UTF8=169 CP1252=75 CP1251=179 KOI8R=158 ISO-8859-5=168 [top CP1251]
+ {{0x6d,0x61,0x6c,0x61,0x67,0x61,0x73,0x79, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "malagasy"
+ // UTF8=191 [top UTF8]
+ {{0x6d,0x61,0x6c,0x61,0x79,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "malay___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6d,0x61,0x6c,0x61,0x79,0x61,0x6c,0x61, 0x03,0x48,0xb6,0x81,0x11,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,}}, // "malayala"
+ // ASCII-7-bit=72 Latin1=182 UTF8=129 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6d,0x61,0x6c,0x74,0x65,0x73,0x65,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "maltese_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6d,0x61,0x6e,0x78,0x5f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "manx____"
+ // UTF8=191 [top UTF8]
+ {{0x6d,0x61,0x6f,0x72,0x69,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "maori___"
+ // UTF8=191 [top UTF8]
+ {{0x6d,0x61,0x72,0x61,0x74,0x68,0x69,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "marathi_"
+ // UTF8=191 [top UTF8]
+ {{0x6d,0x6f,0x6c,0x64,0x61,0x76,0x69,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "moldavia"
+ // UTF8=191 [top UTF8]
+ {{0x6d,0x6f,0x6e,0x67,0x6f,0x6c,0x69,0x61, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mongolia"
+ // CP1251=191 [top CP1251]
+ {{0x6e,0x61,0x75,0x72,0x75,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nauru___"
+ // UTF8=191 [top UTF8]
+ {{0x6e,0x65,0x70,0x61,0x6c,0x69,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nepali__"
+ // UTF8=191 [top UTF8]
+ {{0x6e,0x6f,0x72,0x77,0x65,0x67,0x69,0x61, 0x03,0x92,0xb8,0xa8,0x11,0x9c,0x41,0x30,0x31,0x24,0x11,0x8e,}}, // "norwegia"
+ // ASCII-7-bit=146 Latin1=184 UTF8=168 CP1252=156 Latin2=48 Latin5=36 ISO-8859-15=142 [top Latin1]
+ {{0x6f,0x63,0x63,0x69,0x74,0x61,0x6e,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "occitan_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6f,0x72,0x69,0x79,0x61,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "oriya___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x6f,0x72,0x6f,0x6d,0x6f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "oromo___"
+ // UTF8=191 [top UTF8]
+ {{0x70,0x61,0x73,0x68,0x74,0x6f,0x5f,0x5f, 0xb1,0xb9,0x10,0x61,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pashto__"
+ // CP1256=185 Arabic=175 [top CP1256]
+ {{0x70,0x65,0x72,0x73,0x69,0x61,0x6e,0x5f, 0x12,0x44,0xb6,0x11,0x33,0x62,0xae,0x19,0x10,0x51,0x9f,0x00,}}, // "persian_"
+ // Latin1=68 UTF8=182 CP1252=51 CP1256=174 CP1250=25 Arabic=159 [top UTF8]
+ {{0x70,0x6f,0x6c,0x69,0x73,0x68,0x5f,0x5f, 0x05,0x85,0x6c,0xa8,0x26,0x57,0x41,0xb9,0x21,0x99,0x31,0x23,}}, // "polish__"
+ // ASCII-7-bit=133 Latin1=108 UTF8=168 GB=38 CP1252=87 Latin2=185 CP1250=153 CP1257=35 [top Latin2]
+ {{0x70,0x6f,0x72,0x74,0x75,0x67,0x75,0x65, 0x03,0x96,0xb9,0xa6,0x11,0x9a,0x11,0x30,0x51,0x36,0x21,0x86,}}, // "portugue"
+ // ASCII-7-bit=150 Latin1=185 UTF8=166 CP1252=154 SJS=48 CP1250=54 ISO-8859-15=134 [top Latin1]
+ {{0x70,0x75,0x6e,0x6a,0x61,0x62,0x69,0x5f, 0x03,0x42,0xb6,0x7b,0x11,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,}}, // "punjabi_"
+ // ASCII-7-bit=66 Latin1=182 UTF8=123 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x71,0x75,0x65,0x63,0x68,0x75,0x61,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "quechua_"
+ // UTF8=191 [top UTF8]
+ {{0x72,0x68,0x61,0x65,0x74,0x6f,0x72,0x6f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "rhaetoro"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x72,0x6f,0x6d,0x61,0x6e,0x69,0x61,0x6e, 0x03,0xb2,0x9d,0xa5,0x11,0x92,0x42,0xa7,0x51,0x11,0x99,0x00,}}, // "romanian"
+ // ASCII-7-bit=178 Latin1=157 UTF8=165 CP1252=146 Latin2=167 CP1251=81 CP1250=153 [top ASCII-7-bit]
+ {{0x72,0x75,0x6e,0x64,0x69,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "rundi___"
+ // UTF8=191 [top UTF8]
+ {{0x72,0x75,0x73,0x73,0x69,0x61,0x6e,0x5f, 0x01,0x74,0x11,0xa9,0x71,0xb9,0x71,0x99,0x81,0x82,0x81,0x6d,}}, // "russian_"
+ // ASCII-7-bit=116 UTF8=169 CP1251=185 KOI8R=153 ISO-8859-5=130 CP866=109 [top CP1251]
+ {{0x73,0x61,0x6d,0x6f,0x61,0x6e,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "samoan__"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x61,0x6e,0x67,0x6f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sango___"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x61,0x6e,0x73,0x6b,0x72,0x69,0x74, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sanskrit"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x63,0x6f,0x74,0x73,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "scots___"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x63,0x6f,0x74,0x73,0x67,0x61,0x65, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "scotsgae"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x73,0x65,0x72,0x62,0x69,0x61,0x6e,0x5f, 0x03,0x93,0x77,0xad,0x11,0x85,0x42,0xad,0x52,0x12,0xae,0x4a,}}, // "serbian_"
+ // ASCII-7-bit=147 Latin1=119 UTF8=173 CP1252=133 Latin2=173 CP1251=82 CP1250=174 Latin5=74 [top CP1250]
+ {{0x73,0x65,0x72,0x62,0x6f,0x63,0x72,0x6f, 0x91,0xaf,0x21,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "serbocro"
+ // Latin2=175 CP1250=185 [top CP1250]
+ {{0x73,0x65,0x73,0x6f,0x74,0x68,0x6f,0x5f, 0x11,0xb9,0x21,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sesotho_"
+ // Latin1=185 CP1252=175 [top Latin1]
+ {{0x73,0x68,0x6f,0x6e,0x61,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "shona___"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x69,0x6e,0x64,0x68,0x69,0x5f,0x5f, 0xb1,0xb9,0x10,0x61,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sindhi__"
+ // CP1256=185 Arabic=175 [top CP1256]
+ {{0x73,0x69,0x6e,0x68,0x61,0x6c,0x65,0x73, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sinhales"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x73,0x69,0x73,0x77,0x61,0x6e,0x74,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "siswant_"
+ // UTF8=191 [top UTF8]
+ {{0x73,0x6c,0x6f,0x76,0x61,0x6b,0x5f,0x5f, 0x03,0x88,0x6e,0xaf,0x11,0x67,0x41,0xa5,0x21,0xb3,0x00,0x00,}}, // "slovak__"
+ // ASCII-7-bit=136 Latin1=110 UTF8=175 CP1252=103 Latin2=165 CP1250=179 [top CP1250]
+ {{0x73,0x6c,0x6f,0x76,0x65,0x6e,0x69,0x61, 0x03,0x8e,0x71,0xb2,0x11,0x80,0x42,0xaa,0x39,0x11,0xad,0x00,}}, // "slovenia"
+ // ASCII-7-bit=142 Latin1=113 UTF8=178 CP1252=128 Latin2=170 CP1251=57 CP1250=173 [top UTF8]
+ {{0x73,0x6f,0x6d,0x61,0x6c,0x69,0x5f,0x5f, 0x11,0xb9,0x21,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "somali__"
+ // Latin1=185 CP1252=175 [top Latin1]
+ {{0x73,0x70,0x61,0x6e,0x69,0x73,0x68,0x5f, 0x03,0x9b,0xb8,0xa7,0x11,0x98,0x41,0x45,0x21,0x45,0x21,0x77,}}, // "spanish_"
+ // ASCII-7-bit=155 Latin1=184 UTF8=167 CP1252=152 Latin2=69 CP1250=69 ISO-8859-15=119 [top Latin1]
+ {{0x73,0x75,0x6e,0x64,0x61,0x6e,0x65,0x73, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sundanes"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x73,0x77,0x61,0x68,0x69,0x6c,0x69,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "swahili_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x73,0x77,0x65,0x64,0x69,0x73,0x68,0x5f, 0x03,0x90,0xba,0xa4,0x11,0x8d,0x41,0x2c,0x21,0x2c,0x21,0x7a,}}, // "swedish_"
+ // ASCII-7-bit=144 Latin1=186 UTF8=164 CP1252=141 Latin2=44 CP1250=44 ISO-8859-15=122 [top Latin1]
+ {{0x73,0x79,0x72,0x69,0x61,0x63,0x5f,0x5f, 0x01,0x6a,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "syriac__"
+ // ASCII-7-bit=106 UTF8=190 [top UTF8]
+ {{0x74,0x61,0x67,0x61,0x6c,0x6f,0x67,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tagalog_"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x74,0x61,0x6a,0x69,0x6b,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tajik___"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x61,0x6d,0x69,0x6c,0x5f,0x5f,0x5f, 0x12,0xb4,0x8e,0x11,0xaa,0xa1,0xa0,0x20,0x23,0x96,0x8c,0x82,}}, // "tamil___"
+ // Latin1=180 UTF8=142 CP1252=170 ISO-8859-15=160 TSCII=150 TAM=140 TAB=130 [top Latin1]
+ {{0x74,0x61,0x74,0x61,0x72,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tatar___"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x65,0x6c,0x75,0x67,0x75,0x5f,0x5f, 0x03,0x66,0xb6,0x90,0x11,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,}}, // "telugu__"
+ // ASCII-7-bit=102 Latin1=182 UTF8=144 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x74,0x68,0x61,0x69,0x5f,0x5f,0x5f,0x5f, 0x05,0x7a,0x53,0xa2,0x24,0x46,0x91,0xba,0xd1,0x9e,0x21,0x29,}}, // "thai____"
+ // ASCII-7-bit=122 Latin1=83 UTF8=162 GB=36 CP1252=70 ISO-8859-11=186 CP874=158 MACINTOSH=41 [top ISO-8859-11]
+ {{0x74,0x69,0x62,0x65,0x74,0x61,0x6e,0x5f, 0x01,0x42,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tibetan_"
+ // ASCII-7-bit=66 UTF8=190 [top UTF8]
+ {{0x74,0x69,0x67,0x72,0x69,0x6e,0x79,0x61, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tigrinya"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x6f,0x6e,0x67,0x61,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tonga___"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x73,0x6f,0x6e,0x67,0x61,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tsonga__"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x73,0x77,0x61,0x6e,0x61,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tswana__"
+ // UTF8=191 [top UTF8]
+ {{0x74,0x75,0x72,0x6b,0x69,0x73,0x68,0x5f, 0x03,0x81,0x7f,0xa5,0x11,0x6e,0x81,0xba,0x11,0x3d,0x61,0x95,}}, // "turkish_"
+ // ASCII-7-bit=129 Latin1=127 UTF8=165 CP1252=110 Latin5=186 ISO-8859-15=61 CP1254=149 [top Latin5]
+ {{0x74,0x75,0x72,0x6b,0x6d,0x65,0x6e,0x5f, 0x91,0xb9,0x21,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "turkmen_"
+ // Latin2=185 CP1250=175 [top Latin2]
+ {{0x74,0x77,0x69,0x5f,0x5f,0x5f,0x5f,0x5f, 0x11,0xac,0x21,0xb6,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "twi_____"
+ // Latin1=172 CP1252=182 ISO-8859-15=162 [top CP1252]
+ {{0x75,0x69,0x67,0x68,0x75,0x72,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "uighur__"
+ // UTF8=191 [top UTF8]
+ {{0x75,0x6b,0x72,0x61,0x69,0x6e,0x69,0x61, 0x21,0xa0,0x71,0xb7,0x71,0x91,0x72,0x98,0xa2,0x81,0x84,0x00,}}, // "ukrainia"
+ // UTF8=160 CP1251=183 KOI8R=145 KOI8U=152 ISO-8859-5=162 CP866=132 [top CP1251]
+ {{0x75,0x72,0x64,0x75,0x5f,0x5f,0x5f,0x5f, 0xb1,0xb9,0x10,0x61,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "urdu____"
+ // CP1256=185 Arabic=175 [top CP1256]
+ {{0x75,0x7a,0x62,0x65,0x6b,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "uzbek___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x76,0x69,0x65,0x74,0x6e,0x61,0x6d,0x65, 0x03,0x81,0xa8,0xb7,0x11,0x9e,0xa1,0x94,0x00,0x00,0x00,0x00,}}, // "vietname"
+ // ASCII-7-bit=129 Latin1=168 UTF8=183 CP1252=158 ISO-8859-15=148 [top UTF8]
+ {{0x76,0x6f,0x6c,0x61,0x70,0x75,0x6b,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "volapuk_"
+ // UTF8=191 [top UTF8]
+ {{0x77,0x65,0x6c,0x73,0x68,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "welsh___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x77,0x6f,0x6c,0x6f,0x66,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wolof___"
+ // UTF8=191 [top UTF8]
+ {{0x78,0x68,0x6f,0x73,0x61,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "xhosa___"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+ {{0x79,0x69,0x64,0x64,0x69,0x73,0x68,0x5f, 0x10,0x11,0xb9,0x71,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "yiddish_"
+ // CP1255=185 Hebrew=175 [top CP1255]
+ {{0x79,0x6f,0x72,0x75,0x62,0x61,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "yoruba__"
+ // UTF8=191 [top UTF8]
+ {{0x7a,0x68,0x75,0x61,0x6e,0x67,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "zhuang__"
+ // UTF8=191 [top UTF8]
+ {{0x7a,0x75,0x6c,0x75,0x5f,0x5f,0x5f,0x5f, 0x11,0xb6,0x21,0xac,0xa1,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "zulu____"
+ // Latin1=182 CP1252=172 ISO-8859-15=162 [top Latin1]
+};
+
+static const int kLangHintProbsSize = 151;
+
+static const HintEntry kTLDHintProbs[] = { // MaxRange 192
+ {{0x5f,0x5f,0x5f,0x5f, 0x0f,0xa8,0xa1,0xa3,0xa0,0x8e,0x8e,0x8a,0x7e,0xa8,0x77,0x7b,0x8b,0x75,0x79,0x7e,}}, // "____"
+ // ASCII-7-bit=168 Latin1=161 UTF8=163 GB=160 CP1252=142 KSC=142 SJS=138 EUC-JP=126 BIG5=168 Latin2=119 CP1251=123 CP1256=139 CP1250=117 Latin5=121 ISO-8859-11=126 [top ASCII-7-bit]
+ {{0x61,0x63,0x5f,0x5f, 0x08,0xa0,0x9a,0xa1,0x65,0x92,0x8f,0xb1,0xa2,0x22,0x56,0x8a,0x21,0x56,0x61,0x87,}}, // "ac__"
+ // ASCII-7-bit=160 Latin1=154 UTF8=161 GB=101 CP1252=146 KSC=143 SJS=177 EUC-JP=162 CP1251=86 CP1256=138 ISO-8859-11=86 JIS=135 [top SJS]
+ {{0x61,0x64,0x5f,0x5f, 0x03,0xa6,0xb6,0x93,0x11,0xa8,0x11,0x74,0x81,0x5d,0x81,0x5d,0x00,0x00,0x00,0x00,}}, // "ad__"
+ // ASCII-7-bit=166 Latin1=182 UTF8=147 CP1252=168 SJS=116 ISO-8859-15=93 CP932=93 [top Latin1]
+ {{0x61,0x65,0x5f,0x5f, 0x05,0xa4,0x81,0xac,0x42,0x86,0x11,0x5b,0x25,0x4f,0x4a,0xb5,0x3b,0x52,0x00,0x00,}}, // "ae__"
+ // ASCII-7-bit=164 Latin1=129 UTF8=172 GB=66 CP1252=134 SJS=91 Latin2=79 CP1251=74 CP1256=181 CP1250=59 Latin5=82 [top CP1256]
+ {{0x61,0x65,0x72,0x6f, 0x03,0xaf,0xab,0xab,0x12,0x98,0x6a,0x11,0x6a,0x21,0x96,0x21,0x6a,0x00,0x00,0x00,}}, // "aero"
+ // ASCII-7-bit=175 Latin1=171 UTF8=171 CP1252=152 KSC=106 EUC-JP=106 CP1251=150 Latin5=106 [top ASCII-7-bit]
+ {{0x61,0x66,0x5f,0x5f, 0x03,0xb6,0x95,0xaf,0x11,0x8c,0x61,0x80,0x11,0x62,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "af__"
+ // ASCII-7-bit=182 Latin1=149 UTF8=175 CP1252=140 CP1256=128 Latin5=98 [top ASCII-7-bit]
+ {{0x61,0x67,0x5f,0x5f, 0x03,0xa8,0xb4,0xa2,0x11,0x9a,0x12,0x95,0x86,0x21,0x60,0x41,0x7a,0x00,0x00,0x00,}}, // "ag__"
+ // ASCII-7-bit=168 Latin1=180 UTF8=162 CP1252=154 SJS=149 EUC-JP=134 CP1251=96 ISO-8859-15=122 [top Latin1]
+ {{0x61,0x69,0x5f,0x5f, 0x03,0xb8,0x8f,0x9b,0x11,0x9d,0x12,0x8c,0x97,0x11,0x90,0xb1,0x67,0x00,0x00,0x00,}}, // "ai__"
+ // ASCII-7-bit=184 Latin1=143 UTF8=155 CP1252=157 SJS=140 EUC-JP=151 Latin2=144 JIS=103 [top ASCII-7-bit]
+ {{0x61,0x6c,0x5f,0x5f, 0x03,0xac,0x99,0xae,0x11,0xa1,0x31,0x57,0x41,0x57,0x21,0xa7,0x31,0x57,0x00,0x00,}}, // "al__"
+ // ASCII-7-bit=172 Latin1=153 UTF8=174 CP1252=161 BIG5=87 Latin5=87 CP1257=167 Greek=87 [top UTF8]
+ {{0x61,0x6d,0x5f,0x5f, 0x08,0xac,0x9a,0xab,0x68,0x9d,0x58,0x82,0x56,0x22,0xac,0x5a,0x00,0x00,0x00,0x00,}}, // "am__"
+ // ASCII-7-bit=172 Latin1=154 UTF8=171 GB=104 CP1252=157 KSC=88 SJS=130 EUC-JP=86 CP1251=172 CP1256=90 [top ASCII-7-bit]
+ {{0x61,0x6e,0x5f,0x5f, 0x03,0xb6,0xad,0x94,0x11,0x99,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "an__"
+ // ASCII-7-bit=182 Latin1=173 UTF8=148 CP1252=153 [top ASCII-7-bit]
+ {{0x61,0x6f,0x5f,0x5f, 0x03,0x9f,0xb5,0xab,0x11,0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ao__"
+ // ASCII-7-bit=159 Latin1=181 UTF8=171 CP1252=159 [top Latin1]
+ {{0x61,0x71,0x5f,0x5f, 0x03,0xb7,0xa9,0x9c,0x11,0x8a,0x51,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "aq__"
+ // ASCII-7-bit=183 Latin1=169 UTF8=156 CP1252=138 CP1251=151 [top ASCII-7-bit]
+ {{0x61,0x72,0x5f,0x5f, 0x03,0xa0,0xb9,0x9e,0x13,0x98,0x55,0x2c,0x13,0x28,0x26,0x27,0x11,0x2e,0x21,0x42,}}, // "ar__"
+ // ASCII-7-bit=160 Latin1=185 UTF8=158 CP1252=152 KSC=85 SJS=44 BIG5=40 Latin2=38 CP1251=39 CP1250=46 ISO-8859-15=66 [top Latin1]
+ {{0x61,0x73,0x5f,0x5f, 0x03,0xa9,0xb7,0x9f,0x11,0x94,0x11,0x52,0x22,0x64,0x52,0x12,0x7d,0x74,0x21,0x52,}}, // "as__"
+ // ASCII-7-bit=169 Latin1=183 UTF8=159 CP1252=148 SJS=82 Latin2=100 CP1251=82 CP1250=125 Latin5=116 CP1257=82 [top Latin1]
+ {{0x61,0x74,0x5f,0x5f, 0x03,0xa1,0xb8,0xa5,0x11,0x9a,0x11,0x48,0x21,0x51,0x13,0x45,0x53,0x4a,0x11,0x62,}}, // "at__"
+ // ASCII-7-bit=161 Latin1=184 UTF8=165 CP1252=154 SJS=72 Latin2=81 CP1256=69 CP1250=83 Latin5=74 ISO-8859-15=98 [top Latin1]
+ {{0x61,0x75,0x5f,0x5f, 0x09,0xb8,0xa3,0x9f,0x4e,0x9a,0x55,0x54,0x3e,0x5e,0x22,0x30,0x3d,0x21,0x36,0x00,}}, // "au__"
+ // ASCII-7-bit=184 Latin1=163 UTF8=159 GB=78 CP1252=154 KSC=85 SJS=84 EUC-JP=62 BIG5=94 CP1256=48 CP1250=61 ISO-8859-15=54 [top ASCII-7-bit]
+ {{0x61,0x77,0x5f,0x5f, 0x03,0xb6,0xa2,0xaa,0x11,0x99,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "aw__"
+ // ASCII-7-bit=182 Latin1=162 UTF8=170 CP1252=153 [top ASCII-7-bit]
+ {{0x61,0x78,0x5f,0x5f, 0x03,0x9d,0xba,0xa2,0x11,0x90,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ax__"
+ // ASCII-7-bit=157 Latin1=186 UTF8=162 CP1252=144 [top Latin1]
+ {{0x61,0x7a,0x5f,0x5f, 0x03,0x9a,0x7d,0xb8,0x11,0x86,0x54,0xa8,0x54,0x54,0x91,0x41,0x4c,0x31,0x6c,0x00,}}, // "az__"
+ // ASCII-7-bit=154 Latin1=125 UTF8=184 CP1252=134 CP1251=168 CP1256=84 CP1250=84 Latin5=145 KOI8R=76 CP1254=108 [top UTF8]
+ {{0x62,0x61,0x5f,0x5f, 0x03,0xa0,0x7e,0xb2,0x11,0x78,0x44,0x89,0x66,0x49,0xb1,0x00,0x00,0x00,0x00,0x00,}}, // "ba__"
+ // ASCII-7-bit=160 Latin1=126 UTF8=178 CP1252=120 Latin2=137 CP1251=102 CP1256=73 CP1250=177 [top UTF8]
+ {{0x62,0x62,0x5f,0x5f, 0x03,0xba,0xa0,0x7f,0x11,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bb__"
+ // ASCII-7-bit=186 Latin1=160 UTF8=127 CP1252=160 [top ASCII-7-bit]
+ {{0x62,0x64,0x5f,0x5f, 0x03,0xbd,0x94,0x8c,0x11,0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bd__"
+ // ASCII-7-bit=189 Latin1=148 UTF8=140 CP1252=138 [top ASCII-7-bit]
+ {{0x62,0x65,0x5f,0x5f, 0x03,0xb1,0xb0,0xa1,0x11,0x9d,0x11,0x5f,0x22,0x4e,0x50,0x12,0x4d,0x59,0x11,0x5f,}}, // "be__"
+ // ASCII-7-bit=177 Latin1=176 UTF8=161 CP1252=157 SJS=95 Latin2=78 CP1251=80 CP1250=77 Latin5=89 ISO-8859-15=95 [top ASCII-7-bit]
+ {{0x62,0x66,0x5f,0x5f, 0x03,0x9f,0xb9,0x63,0x11,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bf__"
+ // ASCII-7-bit=159 Latin1=185 UTF8=99 CP1252=166 [top Latin1]
+ {{0x62,0x67,0x5f,0x5f, 0x05,0x96,0x70,0xab,0x4a,0x74,0x51,0xb9,0x11,0x4f,0x51,0x44,0x31,0x45,0x41,0x54,}}, // "bg__"
+ // ASCII-7-bit=150 Latin1=112 UTF8=171 GB=74 CP1252=116 CP1251=185 CP1250=79 KOI8R=68 CP1254=69 ISO-8859-5=84 [top CP1251]
+ {{0x62,0x68,0x5f,0x5f, 0x03,0x9f,0x94,0xa5,0x11,0x84,0x11,0x53,0x41,0xb8,0x10,0x61,0x70,0x00,0x00,0x00,}}, // "bh__"
+ // ASCII-7-bit=159 Latin1=148 UTF8=165 CP1252=132 SJS=83 CP1256=184 Arabic=112 [top CP1256]
+ {{0x62,0x69,0x5f,0x5f, 0x03,0xa4,0xa5,0xb8,0x12,0x82,0x65,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bi__"
+ // ASCII-7-bit=164 Latin1=165 UTF8=184 CP1252=130 KSC=101 [top UTF8]
+ {{0x62,0x69,0x7a,0x5f, 0x0e,0xae,0xa5,0xa1,0x77,0x96,0x7f,0x95,0x9c,0x7a,0x8e,0x8b,0x80,0x80,0x92,0x00,}}, // "biz_"
+ // ASCII-7-bit=174 Latin1=165 UTF8=161 GB=119 CP1252=150 KSC=127 SJS=149 EUC-JP=156 BIG5=122 Latin2=142 CP1251=139 CP1256=128 CP1250=128 Latin5=146 [top ASCII-7-bit]
+ {{0x62,0x6a,0x5f,0x5f, 0x03,0x9b,0xb6,0x8a,0x11,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bj__"
+ // ASCII-7-bit=155 Latin1=182 UTF8=138 CP1252=175 [top Latin1]
+ {{0x62,0x6d,0x5f,0x5f, 0x05,0xbb,0x95,0xa0,0x5a,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bm__"
+ // ASCII-7-bit=187 Latin1=149 UTF8=160 GB=90 CP1252=149 [top ASCII-7-bit]
+ {{0x62,0x6e,0x5f,0x5f, 0x05,0xb8,0x98,0xa6,0x6d,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bn__"
+ // ASCII-7-bit=184 Latin1=152 UTF8=166 GB=109 CP1252=160 [top ASCII-7-bit]
+ {{0x62,0x6f,0x5f,0x5f, 0x03,0x9a,0xba,0x9f,0x11,0x9c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bo__"
+ // ASCII-7-bit=154 Latin1=186 UTF8=159 CP1252=156 [top Latin1]
+ {{0x62,0x72,0x5f,0x5f, 0x07,0x9c,0xba,0x9c,0x1f,0x95,0x21,0x43,0x15,0x1c,0x20,0x17,0x0e,0x2b,0x21,0x5a,}}, // "br__"
+ // ASCII-7-bit=156 Latin1=186 UTF8=156 GB=31 CP1252=149 KSC=33 SJS=67 BIG5=28 Latin2=32 CP1251=23 CP1256=14 CP1250=43 ISO-8859-15=90 [top Latin1]
+ {{0x62,0x73,0x5f,0x5f, 0x03,0xb2,0xb4,0x9c,0x11,0x76,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bs__"
+ // ASCII-7-bit=178 Latin1=180 UTF8=156 CP1252=118 [top Latin1]
+ {{0x62,0x74,0x5f,0x5f, 0x03,0xb9,0x96,0xa7,0x11,0x94,0x11,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bt__"
+ // ASCII-7-bit=185 Latin1=150 UTF8=167 CP1252=148 SJS=111 [top ASCII-7-bit]
+ {{0x62,0x77,0x5f,0x5f, 0x03,0xbb,0x9b,0x88,0x11,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bw__"
+ // ASCII-7-bit=187 Latin1=155 UTF8=136 CP1252=157 [top ASCII-7-bit]
+ {{0x62,0x79,0x5f,0x5f, 0x03,0x8a,0x7b,0xa4,0x11,0x74,0x42,0x5d,0xb6,0x11,0x4b,0x21,0x5f,0x21,0xa9,0x00,}}, // "by__"
+ // ASCII-7-bit=138 Latin1=123 UTF8=164 CP1252=116 Latin2=93 CP1251=182 CP1250=75 ISO-8859-15=95 KOI8R=169 [top CP1251]
+ {{0x62,0x7a,0x5f,0x5f, 0x03,0xaf,0x9f,0xa1,0x19,0x90,0x89,0xa4,0x9e,0x8c,0x65,0x8d,0x64,0x70,0x51,0x7e,}}, // "bz__"
+ // ASCII-7-bit=175 Latin1=159 UTF8=161 CP1252=144 KSC=137 SJS=164 EUC-JP=158 BIG5=140 Latin2=101 CP1251=141 CP1256=100 CP1250=112 KOI8R=126 [top ASCII-7-bit]
+ {{0x63,0x61,0x5f,0x5f, 0x07,0xb3,0xac,0xa0,0x5b,0x9b,0x5f,0x49,0x15,0x56,0x3c,0x5d,0x48,0x42,0x21,0x94,}}, // "ca__"
+ // ASCII-7-bit=179 Latin1=172 UTF8=160 GB=91 CP1252=155 KSC=95 SJS=73 BIG5=86 Latin2=60 CP1251=93 CP1256=72 CP1250=66 ISO-8859-15=148 [top ASCII-7-bit]
+ {{0x63,0x61,0x74,0x5f, 0x03,0x9a,0xb4,0xad,0x11,0x9f,0x11,0x30,0x31,0x30,0x32,0x30,0x6e,0x00,0x00,0x00,}}, // "cat_"
+ // ASCII-7-bit=154 Latin1=180 UTF8=173 CP1252=159 SJS=48 CP1251=48 ISO-8859-11=48 ISO-8859-15=110 [top Latin1]
+ {{0x63,0x63,0x5f,0x5f, 0x09,0x9d,0xab,0xad,0x9b,0x86,0x80,0x90,0x9e,0x92,0x21,0x8a,0x11,0x7a,0x51,0x75,}}, // "cc__"
+ // ASCII-7-bit=157 Latin1=171 UTF8=173 GB=155 CP1252=134 KSC=128 SJS=144 EUC-JP=158 BIG5=146 CP1256=138 Latin5=122 GBK=117 [top UTF8]
+ {{0x63,0x64,0x5f,0x5f, 0x09,0xae,0xa2,0xb2,0x5a,0x95,0x5a,0x8f,0x64,0x5a,0x11,0x7d,0x11,0x74,0x11,0x5a,}}, // "cd__"
+ // ASCII-7-bit=174 Latin1=162 UTF8=178 GB=90 CP1252=149 KSC=90 SJS=143 EUC-JP=100 BIG5=90 CP1251=125 CP1250=116 ISO-8859-11=90 [top UTF8]
+ {{0x63,0x67,0x5f,0x5f, 0x03,0x83,0x8d,0xbe,0x11,0x83,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cg__"
+ // ASCII-7-bit=131 Latin1=141 UTF8=190 CP1252=131 [top UTF8]
+ {{0x63,0x68,0x5f,0x5f, 0x05,0xaa,0xb6,0xa1,0x4c,0x9a,0x11,0x46,0x25,0x49,0x3e,0x41,0x44,0x43,0x11,0x66,}}, // "ch__"
+ // ASCII-7-bit=170 Latin1=182 UTF8=161 GB=76 CP1252=154 SJS=70 Latin2=73 CP1251=62 CP1256=65 CP1250=68 Latin5=67 ISO-8859-15=102 [top Latin1]
+ {{0x63,0x69,0x5f,0x5f, 0x03,0x9c,0xae,0xb3,0x11,0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ci__"
+ // ASCII-7-bit=156 Latin1=174 UTF8=179 CP1252=161 [top UTF8]
+ {{0x63,0x6b,0x5f,0x5f, 0x03,0xba,0x9c,0x9e,0x11,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ck__"
+ // ASCII-7-bit=186 Latin1=156 UTF8=158 CP1252=154 [top ASCII-7-bit]
+ {{0x63,0x6c,0x5f,0x5f, 0x03,0xa4,0xb9,0x9c,0x11,0x97,0x11,0x3e,0x27,0x1b,0x1b,0x2d,0x34,0x2b,0x21,0x3b,}}, // "cl__"
+ // ASCII-7-bit=164 Latin1=185 UTF8=156 CP1252=151 SJS=62 Latin2=27 CP1251=27 CP1256=45 CP1250=52 Latin5=43 ISO-8859-11=33 ISO-8859-15=59 [top Latin1]
+ {{0x63,0x6d,0x5f,0x5f, 0x03,0x93,0xbd,0x64,0x11,0x97,0xa1,0x6c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cm__"
+ // ASCII-7-bit=147 Latin1=189 UTF8=100 CP1252=151 ISO-8859-15=108 [top Latin1]
+ {{0x63,0x6e,0x5f,0x5f, 0x09,0x8c,0x5c,0xa7,0xba,0x4f,0x48,0x57,0x3c,0x8d,0x12,0x4e,0x4f,0x71,0x64,0x00,}}, // "cn__"
+ // ASCII-7-bit=140 Latin1=92 UTF8=167 GB=186 CP1252=79 KSC=72 SJS=87 EUC-JP=60 BIG5=141 CP1251=78 CP1256=79 GBK=100 [top GB]
+ {{0x63,0x6f,0x5f,0x5f, 0x03,0xa8,0xb7,0xa3,0x12,0x91,0x27,0x31,0x3f,0x13,0x2f,0x2a,0x2a,0x61,0x27,0x00,}}, // "co__"
+ // ASCII-7-bit=168 Latin1=183 UTF8=163 CP1252=145 KSC=39 Latin2=63 CP1256=47 CP1250=42 Latin5=42 Greek=39 [top Latin1]
+ {{0x63,0x6f,0x6d,0x5f, 0x09,0xb2,0xa5,0xa7,0x94,0x94,0x87,0x87,0x7d,0x82,0x12,0x6e,0x89,0x12,0x7f,0x70,}}, // "com_"
+ // ASCII-7-bit=178 Latin1=165 UTF8=167 GB=148 CP1252=148 KSC=135 SJS=135 EUC-JP=125 BIG5=130 CP1251=110 CP1256=137 Latin5=127 ISO-8859-11=112 [top ASCII-7-bit]
+ {{0x63,0x6f,0x6f,0x70, 0x03,0xaf,0xa8,0xa0,0x14,0x9c,0x75,0xa7,0x86,0x71,0x78,0x00,0x00,0x00,0x00,0x00,}}, // "coop"
+ // ASCII-7-bit=175 Latin1=168 UTF8=160 CP1252=156 KSC=117 SJS=167 EUC-JP=134 ISO-8859-15=120 [top ASCII-7-bit]
+ {{0x63,0x72,0x5f,0x5f, 0x03,0x99,0xb7,0xad,0x11,0x84,0x81,0x28,0x11,0x28,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cr__"
+ // ASCII-7-bit=153 Latin1=183 UTF8=173 CP1252=132 Latin5=40 ISO-8859-15=40 [top Latin1]
+ {{0x63,0x75,0x5f,0x5f, 0x03,0xa0,0xb7,0x9f,0x11,0xa6,0x53,0x31,0x45,0x45,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cu__"
+ // ASCII-7-bit=160 Latin1=183 UTF8=159 CP1252=166 CP1251=49 CP1256=69 CP1250=69 [top Latin1]
+ {{0x63,0x76,0x5f,0x5f, 0x03,0x90,0xbc,0x8f,0x11,0x98,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cv__"
+ // ASCII-7-bit=144 Latin1=188 UTF8=143 CP1252=152 [top Latin1]
+ {{0x63,0x78,0x5f,0x5f, 0x03,0xae,0xa2,0xa5,0x15,0x87,0x76,0x9f,0x9a,0x83,0x41,0x85,0x11,0x7e,0x51,0x99,}}, // "cx__"
+ // ASCII-7-bit=174 Latin1=162 UTF8=165 CP1252=135 KSC=118 SJS=159 EUC-JP=154 BIG5=131 Latin5=133 ISO-8859-15=126 JIS=153 [top ASCII-7-bit]
+ {{0x63,0x79,0x5f,0x5f, 0x03,0xaa,0x88,0xac,0x11,0x86,0x51,0x63,0x21,0x5c,0x61,0x9e,0x12,0x52,0xae,0x00,}}, // "cy__"
+ // ASCII-7-bit=170 Latin1=136 UTF8=172 CP1252=134 CP1251=99 Latin5=92 Greek=158 CP1254=82 CP1253=174 [top CP1253]
+ {{0x63,0x7a,0x5f,0x5f, 0x03,0x8f,0x74,0xb2,0x11,0x56,0x42,0x8c,0x4a,0x11,0xb5,0x10,0x21,0x3f,0x11,0x41,}}, // "cz__"
+ // ASCII-7-bit=143 Latin1=116 UTF8=178 CP1252=86 Latin2=140 CP1251=74 CP1250=181 MACINTOSH=63 CP852=65 [top CP1250]
+ {{0x64,0x65,0x5f,0x5f, 0x06,0xa4,0xb7,0xa4,0x40,0x9a,0x36,0x35,0x4b,0x4d,0x43,0x4d,0x4f,0x11,0x79,0x00,}}, // "de__"
+ // ASCII-7-bit=164 Latin1=183 UTF8=164 GB=64 CP1252=154 KSC=54 Latin2=75 CP1251=77 CP1256=67 CP1250=77 Latin5=79 ISO-8859-15=121 [top Latin1]
+ {{0x64,0x6a,0x5f,0x5f, 0x08,0xa3,0xad,0xa9,0x90,0xa2,0x7d,0x7a,0x68,0x21,0xa0,0x11,0x5e,0xb1,0x5e,0x00,}}, // "dj__"
+ // ASCII-7-bit=163 Latin1=173 UTF8=169 GB=144 CP1252=162 KSC=125 SJS=122 EUC-JP=104 CP1251=160 CP1250=94 CP932=94 [top Latin1]
+ {{0x64,0x6b,0x5f,0x5f, 0x03,0x9d,0xb8,0xa7,0x11,0x93,0x11,0x39,0x25,0x38,0x34,0x57,0x43,0x3d,0x11,0x54,}}, // "dk__"
+ // ASCII-7-bit=157 Latin1=184 UTF8=167 CP1252=147 SJS=57 Latin2=56 CP1251=52 CP1256=87 CP1250=67 Latin5=61 ISO-8859-15=84 [top Latin1]
+ {{0x64,0x6d,0x5f,0x5f, 0x03,0xbc,0x76,0xa3,0x11,0x83,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dm__"
+ // ASCII-7-bit=188 Latin1=118 UTF8=163 CP1252=131 [top ASCII-7-bit]
+ {{0x64,0x6f,0x5f,0x5f, 0x05,0xa4,0xb6,0xa9,0x6b,0x93,0x31,0x43,0x61,0x57,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "do__"
+ // ASCII-7-bit=164 Latin1=182 UTF8=169 GB=107 CP1252=147 BIG5=67 ISO-8859-15=87 [top Latin1]
+ {{0x64,0x7a,0x5f,0x5f, 0x03,0x9e,0xb6,0x8d,0x11,0xa1,0x62,0xa6,0x4e,0x10,0x51,0x58,0x00,0x00,0x00,0x00,}}, // "dz__"
+ // ASCII-7-bit=158 Latin1=182 UTF8=141 CP1252=161 CP1256=166 CP1250=78 Arabic=88 [top Latin1]
+ {{0x65,0x63,0x5f,0x5f, 0x03,0xa2,0xba,0x9c,0x11,0x96,0x35,0x3c,0x32,0x5a,0x32,0x3c,0x00,0x00,0x00,0x00,}}, // "ec__"
+ // ASCII-7-bit=162 Latin1=186 UTF8=156 CP1252=150 BIG5=60 Latin2=50 CP1251=90 CP1256=50 CP1250=60 [top Latin1]
+ {{0x65,0x64,0x75,0x5f, 0x07,0xbb,0x97,0x99,0x51,0x94,0x6b,0x49,0x11,0x4e,0x21,0x4f,0x13,0x4c,0x41,0x44,}}, // "edu_"
+ // ASCII-7-bit=187 Latin1=151 UTF8=153 GB=81 CP1252=148 KSC=107 SJS=73 BIG5=78 CP1256=79 Latin5=76 ISO-8859-11=65 ISO-8859-15=68 [top ASCII-7-bit]
+ {{0x65,0x65,0x5f,0x5f, 0x03,0x97,0xaf,0xb4,0x11,0x95,0x42,0x6f,0x78,0x42,0x82,0x87,0xc2,0x5e,0x65,0x00,}}, // "ee__"
+ // ASCII-7-bit=151 Latin1=175 UTF8=180 CP1252=149 Latin2=111 CP1251=120 ISO-8859-15=130 CP1257=135 ISO-8859-13=94 Latin4=101 [top UTF8]
+ {{0x65,0x67,0x5f,0x5f, 0x05,0x9f,0x7a,0xa7,0x55,0x7d,0x61,0xb9,0x32,0x28,0x28,0x61,0x28,0x00,0x00,0x00,}}, // "eg__"
+ // ASCII-7-bit=159 Latin1=122 UTF8=167 GB=85 CP1252=125 CP1256=185 ISO-8859-15=40 CP1257=40 CP1253=40 [top CP1256]
+ {{0x65,0x73,0x5f,0x5f, 0x05,0x9f,0xb8,0xa8,0x22,0x91,0x11,0x2b,0x15,0x18,0x33,0x4d,0x31,0x23,0x21,0x6c,}}, // "es__"
+ // ASCII-7-bit=159 Latin1=184 UTF8=168 GB=34 CP1252=145 SJS=43 BIG5=24 Latin2=51 CP1251=77 CP1256=49 CP1250=35 ISO-8859-15=108 [top Latin1]
+ {{0x65,0x74,0x5f,0x5f, 0x03,0xb5,0x9a,0xa8,0x11,0xa6,0x11,0x7e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "et__"
+ // ASCII-7-bit=181 Latin1=154 UTF8=168 CP1252=166 SJS=126 [top ASCII-7-bit]
+ {{0x65,0x75,0x5f,0x5f, 0x03,0xa8,0xa4,0xb5,0x11,0x91,0x42,0x8d,0x69,0x12,0x8b,0x6a,0x11,0x78,0x41,0x78,}}, // "eu__"
+ // ASCII-7-bit=168 Latin1=164 UTF8=181 CP1252=145 Latin2=141 CP1251=105 CP1250=139 Latin5=106 ISO-8859-15=120 Greek=120 [top UTF8]
+ {{0x66,0x69,0x5f,0x5f, 0x03,0x9e,0xb7,0xaa,0x11,0x96,0x51,0x46,0x11,0x31,0x22,0x69,0x31,0x00,0x00,0x00,}}, // "fi__"
+ // ASCII-7-bit=158 Latin1=183 UTF8=170 CP1252=150 CP1251=70 CP1250=49 ISO-8859-15=105 CP1257=49 [top Latin1]
+ {{0x66,0x6a,0x5f,0x5f, 0x05,0xba,0x8b,0x9f,0x59,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fj__"
+ // ASCII-7-bit=186 Latin1=139 UTF8=159 GB=89 CP1252=160 [top ASCII-7-bit]
+ {{0x66,0x6b,0x5f,0x5f, 0x02,0xba,0xa6,0x21,0x96,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fk__"
+ // ASCII-7-bit=186 Latin1=166 CP1252=150 [top ASCII-7-bit]
+ {{0x66,0x6d,0x5f,0x5f, 0x03,0x91,0x92,0xbc,0x11,0x82,0x12,0x7d,0x6d,0x12,0x86,0x6a,0x51,0x67,0x00,0x00,}}, // "fm__"
+ // ASCII-7-bit=145 Latin1=146 UTF8=188 CP1252=130 SJS=125 EUC-JP=109 Latin2=134 CP1251=106 CP1257=103 [top UTF8]
+ {{0x66,0x6f,0x5f,0x5f, 0x03,0x93,0xbc,0x9c,0x11,0x8c,0x71,0x57,0x21,0x51,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fo__"
+ // ASCII-7-bit=147 Latin1=188 UTF8=156 CP1252=140 CP1250=87 ISO-8859-15=81 [top Latin1]
+ {{0x66,0x72,0x5f,0x5f, 0x03,0x9f,0xb3,0xaf,0x11,0x99,0x17,0x37,0x32,0x32,0x35,0x41,0x4f,0x35,0x21,0x77,}}, // "fr__"
+ // ASCII-7-bit=159 Latin1=179 UTF8=175 CP1252=153 SJS=55 EUC-JP=50 BIG5=50 Latin2=53 CP1251=65 CP1256=79 CP1250=53 ISO-8859-15=119 [top Latin1]
+ {{0x67,0x61,0x5f,0x5f, 0x03,0xa8,0xb6,0xa5,0x11,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ga__"
+ // ASCII-7-bit=168 Latin1=182 UTF8=165 CP1252=149 [top Latin1]
+ {{0x67,0x64,0x5f,0x5f, 0x05,0xb5,0xac,0x9a,0x80,0x8a,0x81,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gd__"
+ // ASCII-7-bit=181 Latin1=172 UTF8=154 GB=128 CP1252=138 Latin5=151 [top ASCII-7-bit]
+ {{0x67,0x65,0x5f,0x5f, 0x03,0xa5,0x7d,0xba,0x11,0x8d,0x21,0x58,0x21,0x8b,0x11,0x71,0x21,0x4f,0xb1,0x4c,}}, // "ge__"
+ // ASCII-7-bit=165 Latin1=125 UTF8=186 CP1252=141 EUC-JP=88 CP1251=139 CP1250=113 ISO-8859-15=79 ISO-8859-5=76 [top UTF8]
+ {{0x67,0x67,0x5f,0x5f, 0x03,0xad,0xa9,0xb0,0x11,0x95,0x11,0x60,0x81,0x93,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gg__"
+ // ASCII-7-bit=173 Latin1=169 UTF8=176 CP1252=149 SJS=96 ISO-8859-15=147 [top UTF8]
+ {{0x67,0x68,0x5f,0x5f, 0x03,0xac,0xb3,0x99,0x11,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gh__"
+ // ASCII-7-bit=172 Latin1=179 UTF8=153 CP1252=166 [top Latin1]
+ {{0x67,0x69,0x5f,0x5f, 0x03,0xb3,0xa1,0xa1,0x11,0x9c,0xa1,0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gi__"
+ // ASCII-7-bit=179 Latin1=161 UTF8=161 CP1252=156 ISO-8859-15=168 [top ASCII-7-bit]
+ {{0x67,0x6c,0x5f,0x5f, 0x03,0xa7,0xb2,0xaa,0x11,0xa1,0x11,0x43,0x11,0x4d,0x61,0x70,0x00,0x00,0x00,0x00,}}, // "gl__"
+ // ASCII-7-bit=167 Latin1=178 UTF8=170 CP1252=161 SJS=67 BIG5=77 ISO-8859-15=112 [top Latin1]
+ {{0x67,0x6d,0x5f,0x5f, 0x03,0xb8,0x89,0xa0,0x11,0xa7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gm__"
+ // ASCII-7-bit=184 Latin1=137 UTF8=160 CP1252=167 [top ASCII-7-bit]
+ {{0x67,0x6e,0x5f,0x5f, 0x03,0xaa,0xb8,0x89,0x11,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gn__"
+ // ASCII-7-bit=170 Latin1=184 UTF8=137 CP1252=157 [top Latin1]
+ {{0x67,0x6f,0x76,0x5f, 0x05,0xbd,0x92,0x94,0x22,0x87,0x11,0x2e,0x33,0x36,0x15,0x2e,0x13,0x14,0x1f,0x0e,}}, // "gov_"
+ // ASCII-7-bit=189 Latin1=146 UTF8=148 GB=34 CP1252=135 SJS=46 CP1251=54 CP1256=21 CP1250=46 ISO-8859-11=20 ISO-8859-15=31 CP1257=14 [top ASCII-7-bit]
+ {{0x67,0x70,0x5f,0x5f, 0x03,0x98,0x9f,0xbb,0x11,0x83,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gp__"
+ // ASCII-7-bit=152 Latin1=159 UTF8=187 CP1252=131 [top UTF8]
+ {{0x67,0x72,0x5f,0x5f, 0x05,0xa1,0x7f,0xa8,0x45,0x80,0x42,0x49,0x54,0x12,0x44,0x45,0x61,0xb5,0x21,0x9e,}}, // "gr__"
+ // ASCII-7-bit=161 Latin1=127 UTF8=168 GB=69 CP1252=128 Latin2=73 CP1251=84 CP1250=68 Latin5=69 Greek=181 CP1253=158 [top Greek]
+ {{0x67,0x73,0x5f,0x5f, 0x05,0xb0,0x93,0x98,0x68,0x88,0x13,0xa2,0xa7,0x75,0x42,0x99,0x98,0x91,0x66,0x00,}}, // "gs__"
+ // ASCII-7-bit=176 Latin1=147 UTF8=152 GB=104 CP1252=136 SJS=162 EUC-JP=167 BIG5=117 Latin5=153 ISO-8859-11=152 CP932=102 [top ASCII-7-bit]
+ {{0x67,0x74,0x5f,0x5f, 0x03,0xa3,0xb6,0xa9,0x12,0x95,0x3d,0x91,0x3d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gt__"
+ // ASCII-7-bit=163 Latin1=182 UTF8=169 CP1252=149 KSC=61 ISO-8859-15=61 [top Latin1]
+ {{0x67,0x75,0x5f,0x5f, 0x03,0xb9,0xa0,0x7d,0x11,0xa5,0x11,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gu__"
+ // ASCII-7-bit=185 Latin1=160 UTF8=125 CP1252=165 SJS=125 [top ASCII-7-bit]
+ {{0x67,0x79,0x5f,0x5f, 0x03,0xbc,0x82,0xa2,0x11,0x87,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gy__"
+ // ASCII-7-bit=188 Latin1=130 UTF8=162 CP1252=135 [top ASCII-7-bit]
+ {{0x68,0x6b,0x5f,0x5f, 0x07,0xa6,0x6c,0xa3,0xa0,0x6b,0x38,0x53,0x11,0xb5,0xa1,0x73,0xc1,0x3d,0x21,0x49,}}, // "hk__"
+ // ASCII-7-bit=166 Latin1=108 UTF8=163 GB=160 CP1252=107 KSC=56 SJS=83 BIG5=181 GBK=115 GB18030=61 BIG5_HKSCS=73 [top BIG5]
+ {{0x68,0x6d,0x5f,0x5f, 0x05,0xa8,0x9e,0xaa,0x67,0x67,0x13,0xac,0x9f,0x95,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "hm__"
+ // ASCII-7-bit=168 Latin1=158 UTF8=170 GB=103 CP1252=103 SJS=172 EUC-JP=159 BIG5=149 [top SJS]
+ {{0x68,0x6e,0x5f,0x5f, 0x03,0xa1,0xb7,0xa6,0x11,0x9c,0x51,0x4d,0x31,0x4d,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "hn__"
+ // ASCII-7-bit=161 Latin1=183 UTF8=166 CP1252=156 CP1251=77 ISO-8859-11=77 [top Latin1]
+ {{0x68,0x72,0x5f,0x5f, 0x03,0x99,0x67,0xa7,0x11,0x67,0x42,0x9d,0x20,0x11,0xb8,0x10,0x21,0x26,0x11,0x32,}}, // "hr__"
+ // ASCII-7-bit=153 Latin1=103 UTF8=167 CP1252=103 Latin2=157 CP1251=32 CP1250=184 MACINTOSH=38 CP852=50 [top CP1250]
+ {{0x68,0x74,0x5f,0x5f, 0x03,0x95,0x97,0xbc,0x11,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ht__"
+ // ASCII-7-bit=149 Latin1=151 UTF8=188 CP1252=146 [top UTF8]
+ {{0x68,0x75,0x5f,0x5f, 0x05,0xa2,0xa4,0xad,0x33,0x76,0x11,0x3d,0x25,0xa7,0x4a,0x40,0xa7,0x35,0x11,0x3a,}}, // "hu__"
+ // ASCII-7-bit=162 Latin1=164 UTF8=173 GB=51 CP1252=118 SJS=61 Latin2=167 CP1251=74 CP1256=64 CP1250=167 Latin5=53 ISO-8859-15=58 [top UTF8]
+ {{0x69,0x64,0x5f,0x5f, 0x07,0xb8,0xab,0x8c,0x24,0x97,0x33,0x46,0x14,0x2c,0x1f,0x1f,0x7b,0x22,0x29,0x4b,}}, // "id__"
+ // ASCII-7-bit=184 Latin1=171 UTF8=140 GB=36 CP1252=151 KSC=51 SJS=70 BIG5=44 Latin2=31 CP1251=31 CP1256=123 ISO-8859-11=41 ISO-8859-15=75 [top ASCII-7-bit]
+ {{0x69,0x65,0x5f,0x5f, 0x05,0xb3,0xa6,0xa0,0x45,0x9d,0x11,0x46,0x21,0x67,0x12,0x32,0x4a,0x22,0xa3,0x36,}}, // "ie__"
+ // ASCII-7-bit=179 Latin1=166 UTF8=160 GB=69 CP1252=157 SJS=70 Latin2=103 CP1256=50 CP1250=74 ISO-8859-15=163 CP1257=54 [top ASCII-7-bit]
+ {{0x69,0x6c,0x5f,0x5f, 0x03,0x94,0x74,0xa9,0x11,0x6e,0x21,0x3b,0x23,0x79,0x83,0x3d,0x41,0xb8,0x71,0x90,}}, // "il__"
+ // ASCII-7-bit=148 Latin1=116 UTF8=169 CP1252=110 EUC-JP=59 CP1251=121 CP1256=131 CP1250=61 CP1255=184 Hebrew=144 [top CP1255]
+ {{0x69,0x6d,0x5f,0x5f, 0x05,0xb7,0x8e,0xaf,0x89,0x7e,0x51,0x48,0x21,0x71,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "im__"
+ // ASCII-7-bit=183 Latin1=142 UTF8=175 GB=137 CP1252=126 CP1251=72 Latin5=113 [top ASCII-7-bit]
+ {{0x69,0x6e,0x5f,0x5f, 0x08,0xbb,0x95,0x9c,0x5e,0x99,0x61,0x6f,0x67,0x21,0x60,0x22,0x69,0x4d,0x11,0x4c,}}, // "in__"
+ // ASCII-7-bit=187 Latin1=149 UTF8=156 GB=94 CP1252=153 KSC=97 SJS=111 EUC-JP=103 CP1251=96 Latin5=105 ISO-8859-11=77 CP1257=76 [top ASCII-7-bit]
+ {{0x69,0x6e,0x66,0x6f, 0x05,0xac,0xa7,0xa7,0x76,0x94,0x18,0x93,0x82,0x70,0x90,0x91,0x82,0x9c,0x78,0x00,}}, // "info"
+ // ASCII-7-bit=172 Latin1=167 UTF8=167 GB=118 CP1252=148 SJS=147 EUC-JP=130 BIG5=112 Latin2=144 CP1251=145 CP1256=130 CP1250=156 Latin5=120 [top ASCII-7-bit]
+ {{0x69,0x6e,0x74,0x5f, 0x05,0xb2,0xa7,0xb0,0x3a,0x84,0x53,0x62,0x55,0x54,0x71,0x45,0x21,0x39,0x31,0x50,}}, // "int_"
+ // ASCII-7-bit=178 Latin1=167 UTF8=176 GB=58 CP1252=132 CP1251=98 CP1256=85 CP1250=84 Greek=69 CP1253=57 ISO-8859-5=80 [top ASCII-7-bit]
+ {{0x69,0x6f,0x5f,0x5f, 0x09,0xaf,0x98,0xac,0xa8,0x85,0x77,0x83,0x7b,0x67,0x12,0x88,0x71,0x22,0x77,0x67,}}, // "io__"
+ // ASCII-7-bit=175 Latin1=152 UTF8=172 GB=168 CP1252=133 KSC=119 SJS=131 EUC-JP=123 BIG5=103 CP1251=136 CP1256=113 ISO-8859-11=119 ISO-8859-15=103 [top ASCII-7-bit]
+ {{0x69,0x72,0x5f,0x5f, 0x07,0x9e,0x8c,0xb5,0x55,0x7e,0x22,0x3c,0x25,0x32,0x32,0xae,0x36,0x3e,0x81,0x52,}}, // "ir__"
+ // ASCII-7-bit=158 Latin1=140 UTF8=181 GB=85 CP1252=126 KSC=34 SJS=60 Latin2=50 CP1251=50 CP1256=174 CP1250=54 Latin5=62 CP1254=82 [top UTF8]
+ {{0x69,0x73,0x5f,0x5f, 0x05,0x8f,0xbc,0xa0,0x1b,0x84,0x15,0x3b,0x1b,0x2f,0x1b,0x25,0x11,0x32,0x21,0x1b,}}, // "is__"
+ // ASCII-7-bit=143 Latin1=188 UTF8=160 GB=27 CP1252=132 SJS=59 EUC-JP=27 BIG5=47 Latin2=27 CP1251=37 CP1250=50 ISO-8859-15=27 [top Latin1]
+ {{0x69,0x74,0x5f,0x5f, 0x03,0xac,0xb4,0x9e,0x11,0xa2,0x11,0x35,0x21,0x44,0x21,0x3e,0x21,0x60,0x51,0x55,}}, // "it__"
+ // ASCII-7-bit=172 Latin1=180 UTF8=158 CP1252=162 SJS=53 Latin2=68 CP1250=62 ISO-8859-15=96 JIS=85 [top Latin1]
+ {{0x6a,0x65,0x5f,0x5f, 0x03,0xb7,0x8e,0xab,0x11,0x9e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "je__"
+ // ASCII-7-bit=183 Latin1=142 UTF8=171 CP1252=158 [top ASCII-7-bit]
+ {{0x6a,0x6d,0x5f,0x5f, 0x03,0xb8,0xa6,0x93,0x11,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "jm__"
+ // ASCII-7-bit=184 Latin1=166 UTF8=147 CP1252=160 [top ASCII-7-bit]
+ {{0x6a,0x6f,0x5f,0x5f, 0x03,0xa5,0x73,0xb0,0x11,0x82,0x52,0x3c,0xb2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "jo__"
+ // ASCII-7-bit=165 Latin1=115 UTF8=176 CP1252=130 CP1251=60 CP1256=178 [top CP1256]
+ {{0x6a,0x6f,0x62,0x73, 0x03,0xb7,0xa1,0xa5,0x11,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "jobs"
+ // ASCII-7-bit=183 Latin1=161 UTF8=165 CP1252=160 [top ASCII-7-bit]
+ {{0x6a,0x70,0x5f,0x5f, 0x09,0x9b,0x48,0xa4,0x45,0x42,0x4b,0xb3,0xad,0x48,0x51,0x1e,0x61,0x73,0x21,0x63,}}, // "jp__"
+ // ASCII-7-bit=155 Latin1=72 UTF8=164 GB=69 CP1252=66 KSC=75 SJS=179 EUC-JP=173 BIG5=72 ISO-8859-11=30 JIS=115 CP932=99 [top SJS]
+ {{0x6b,0x65,0x5f,0x5f, 0x03,0xb9,0x9a,0xa1,0x13,0x9e,0x4b,0x5b,0x11,0x4b,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ke__"
+ // ASCII-7-bit=185 Latin1=154 UTF8=161 CP1252=158 KSC=75 SJS=91 BIG5=75 [top ASCII-7-bit]
+ {{0x6b,0x67,0x5f,0x5f, 0x05,0x94,0x71,0x9f,0x67,0x83,0x11,0xa8,0x31,0xb7,0x12,0x57,0x57,0x41,0x6f,0x00,}}, // "kg__"
+ // ASCII-7-bit=148 Latin1=113 UTF8=159 GB=103 CP1252=131 SJS=168 CP1251=183 CP1250=87 Latin5=87 KOI8R=111 [top CP1251]
+ {{0x6b,0x68,0x5f,0x5f, 0x05,0xb6,0xa1,0xa1,0x53,0xa5,0x12,0x74,0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kh__"
+ // ASCII-7-bit=182 Latin1=161 UTF8=161 GB=83 CP1252=165 SJS=116 EUC-JP=115 [top ASCII-7-bit]
+ {{0x6b,0x69,0x5f,0x5f, 0x03,0xb8,0xac,0x98,0x11,0x82,0x81,0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ki__"
+ // ASCII-7-bit=184 Latin1=172 UTF8=152 CP1252=130 Latin5=97 [top ASCII-7-bit]
+ {{0x6b,0x6e,0x5f,0x5f, 0x01,0xba,0x11,0x89,0x11,0xaa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kn__"
+ // ASCII-7-bit=186 UTF8=137 CP1252=170 [top ASCII-7-bit]
+ {{0x6b,0x72,0x5f,0x5f, 0x09,0x80,0x39,0x92,0x43,0x2e,0xbe,0x5f,0x3a,0x3d,0x31,0x0c,0x00,0x00,0x00,0x00,}}, // "kr__"
+ // ASCII-7-bit=128 Latin1=57 UTF8=146 GB=67 CP1252=46 KSC=190 SJS=95 EUC-JP=58 BIG5=61 CP1250=12 [top KSC]
+ {{0x6b,0x77,0x5f,0x5f, 0x03,0x91,0x69,0xb2,0x11,0x71,0x61,0xb5,0x11,0x40,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kw__"
+ // ASCII-7-bit=145 Latin1=105 UTF8=178 CP1252=113 CP1256=181 Latin5=64 [top CP1256]
+ {{0x6b,0x79,0x5f,0x5f, 0x03,0xab,0x9d,0xb6,0x11,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ky__"
+ // ASCII-7-bit=171 Latin1=157 UTF8=182 CP1252=154 [top UTF8]
+ {{0x6b,0x7a,0x5f,0x5f, 0x03,0x8d,0x7b,0xb1,0x12,0x74,0x3d,0x41,0xb5,0x21,0x4a,0x41,0x8b,0x81,0x4a,0x00,}}, // "kz__"
+ // ASCII-7-bit=141 Latin1=123 UTF8=177 CP1252=116 KSC=61 CP1251=181 Latin5=74 KOI8R=139 ISO-8859-5=74 [top CP1251]
+ {{0x6c,0x61,0x5f,0x5f, 0x05,0xa8,0x8e,0xab,0xab,0x71,0x13,0x9f,0x6a,0x99,0x41,0x77,0x51,0x64,0x41,0x7e,}}, // "la__"
+ // ASCII-7-bit=168 Latin1=142 UTF8=171 GB=171 CP1252=113 SJS=159 EUC-JP=106 BIG5=153 Latin5=119 GBK=100 CP932=126 [top UTF8]
+ {{0x6c,0x62,0x5f,0x5f, 0x03,0xb3,0x95,0xaf,0x11,0x90,0x11,0x35,0x41,0x9f,0x10,0x61,0x3f,0x00,0x00,0x00,}}, // "lb__"
+ // ASCII-7-bit=179 Latin1=149 UTF8=175 CP1252=144 SJS=53 CP1256=159 Arabic=63 [top ASCII-7-bit]
+ {{0x6c,0x63,0x5f,0x5f, 0x02,0xab,0xa8,0x21,0xb5,0x11,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lc__"
+ // ASCII-7-bit=171 Latin1=168 CP1252=181 SJS=127 [top CP1252]
+ {{0x6c,0x69,0x5f,0x5f, 0x05,0xad,0xb5,0x9d,0x72,0x93,0x12,0x5f,0x53,0x22,0x89,0x64,0x11,0x5f,0x51,0x53,}}, // "li__"
+ // ASCII-7-bit=173 Latin1=181 UTF8=157 GB=114 CP1252=147 SJS=95 EUC-JP=83 CP1251=137 CP1256=100 Latin5=95 GBK=83 [top Latin1]
+ {{0x6c,0x6b,0x5f,0x5f, 0x03,0xb9,0x9e,0x9d,0x11,0xa1,0x12,0x47,0x47,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lk__"
+ // ASCII-7-bit=185 Latin1=158 UTF8=157 CP1252=161 SJS=71 EUC-JP=71 [top ASCII-7-bit]
+ {{0x6c,0x72,0x5f,0x5f, 0x03,0xad,0xac,0x8a,0x11,0xb1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lr__"
+ // ASCII-7-bit=173 Latin1=172 UTF8=138 CP1252=177 [top CP1252]
+ {{0x6c,0x73,0x5f,0x5f, 0x03,0xb7,0xa3,0x97,0x11,0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ls__"
+ // ASCII-7-bit=183 Latin1=163 UTF8=151 CP1252=168 [top ASCII-7-bit]
+ {{0x6c,0x74,0x5f,0x5f, 0x06,0x8c,0x64,0xae,0x3b,0x6d,0x5d,0x32,0x48,0x71,0x11,0x55,0x31,0xb8,0xc1,0x6e,}}, // "lt__"
+ // ASCII-7-bit=140 Latin1=100 UTF8=174 GB=59 CP1252=109 KSC=93 Latin2=72 CP1251=113 CP1250=85 CP1257=184 ISO-8859-13=110 [top CP1257]
+ {{0x6c,0x75,0x5f,0x5f, 0x05,0xa6,0xb2,0xad,0x47,0x9f,0x41,0x42,0x22,0x54,0x3f,0x11,0x53,0x10,0x51,0x3b,}}, // "lu__"
+ // ASCII-7-bit=166 Latin1=178 UTF8=173 GB=71 CP1252=159 Latin2=66 CP1250=84 Latin5=63 ISO-8859-15=83 UTF-16BE=59 [top Latin1]
+ {{0x6c,0x76,0x5f,0x5f, 0x03,0x97,0x6d,0xb5,0x11,0x6b,0x51,0x92,0x51,0xb0,0x11,0x4e,0xa2,0x5f,0x52,0x00,}}, // "lv__"
+ // ASCII-7-bit=151 Latin1=109 UTF8=181 CP1252=107 CP1251=146 CP1257=176 KOI8R=78 ISO-8859-13=95 Latin4=82 [top UTF8]
+ {{0x6c,0x79,0x5f,0x5f, 0x07,0xa4,0x77,0xb0,0x59,0x8c,0x92,0x59,0x32,0x59,0xb1,0x00,0x00,0x00,0x00,0x00,}}, // "ly__"
+ // ASCII-7-bit=164 Latin1=119 UTF8=176 GB=89 CP1252=140 KSC=146 SJS=89 CP1251=89 CP1256=177 [top CP1256]
+ {{0x6d,0x61,0x5f,0x5f, 0x03,0xa2,0xb3,0xa6,0x11,0xa0,0x62,0x9e,0x5d,0x22,0x7d,0x4b,0x10,0x11,0x47,0x00,}}, // "ma__"
+ // ASCII-7-bit=162 Latin1=179 UTF8=166 CP1252=160 CP1256=158 CP1250=93 ISO-8859-15=125 CP1257=75 Arabic=71 [top Latin1]
+ {{0x6d,0x63,0x5f,0x5f, 0x03,0x87,0xbe,0x73,0x11,0x89,0xa1,0x47,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mc__"
+ // ASCII-7-bit=135 Latin1=190 UTF8=115 CP1252=137 ISO-8859-15=71 [top Latin1]
+ {{0x6d,0x64,0x5f,0x5f, 0x03,0xa4,0x82,0xad,0x11,0x86,0x11,0x61,0x15,0x61,0x74,0xae,0x6e,0x99,0x51,0x9d,}}, // "md__"
+ // ASCII-7-bit=164 Latin1=130 UTF8=173 CP1252=134 SJS=97 BIG5=97 Latin2=116 CP1251=174 CP1256=110 CP1250=153 KOI8R=157 [top CP1251]
+ {{0x6d,0x67,0x5f,0x5f, 0x03,0x99,0xba,0x85,0x11,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mg__"
+ // ASCII-7-bit=153 Latin1=186 UTF8=133 CP1252=166 [top Latin1]
+ {{0x6d,0x69,0x6c,0x5f, 0x03,0xba,0x9a,0x9a,0x13,0x9a,0x44,0x50,0x11,0x24,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mil_"
+ // ASCII-7-bit=186 Latin1=154 UTF8=154 CP1252=154 KSC=68 SJS=80 BIG5=36 [top ASCII-7-bit]
+ {{0x6d,0x6b,0x5f,0x5f, 0x03,0x95,0x72,0xab,0x11,0x73,0x42,0x3f,0xa0,0x11,0x66,0x51,0xb6,0x81,0x56,0x00,}}, // "mk__"
+ // ASCII-7-bit=149 Latin1=114 UTF8=171 CP1252=115 Latin2=63 CP1251=160 CP1250=102 KOI8R=182 ISO-8859-5=86 [top KOI8R]
+ {{0x6d,0x6c,0x5f,0x5f, 0x03,0x85,0xbd,0x85,0x11,0x96,0x81,0x59,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ml__"
+ // ASCII-7-bit=133 Latin1=189 UTF8=133 CP1252=150 Latin5=89 [top Latin1]
+ {{0x6d,0x6d,0x5f,0x5f, 0x03,0xb5,0xa1,0x7d,0x11,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mm__"
+ // ASCII-7-bit=181 Latin1=161 UTF8=125 CP1252=175 [top ASCII-7-bit]
+ {{0x6d,0x6e,0x5f,0x5f, 0x07,0x9f,0x7f,0xb8,0x79,0x7d,0x4f,0x5f,0x11,0x56,0x11,0xa7,0x00,0x00,0x00,0x00,}}, // "mn__"
+ // ASCII-7-bit=159 Latin1=127 UTF8=184 GB=121 CP1252=125 KSC=79 SJS=95 BIG5=86 CP1251=167 [top UTF8]
+ {{0x6d,0x6f,0x5f,0x5f, 0x05,0xa0,0x9e,0xaa,0x9e,0x86,0x11,0x4a,0x11,0xb2,0xa1,0x59,0xc1,0x36,0x00,0x00,}}, // "mo__"
+ // ASCII-7-bit=160 Latin1=158 UTF8=170 GB=158 CP1252=134 SJS=74 BIG5=178 GBK=89 GB18030=54 [top BIG5]
+ {{0x6d,0x6f,0x62,0x69, 0x08,0xb6,0x95,0xaa,0x7f,0x7d,0x53,0xa0,0x71,0x13,0x65,0x6d,0x78,0xc1,0x73,0x00,}}, // "mobi"
+ // ASCII-7-bit=182 Latin1=149 UTF8=170 GB=127 CP1252=125 KSC=83 SJS=160 EUC-JP=113 Latin2=101 CP1251=109 CP1256=120 CP932=115 [top ASCII-7-bit]
+ {{0x6d,0x70,0x5f,0x5f, 0x03,0xbe,0x5a,0x54,0x11,0x5c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mp__"
+ // ASCII-7-bit=190 Latin1=90 UTF8=84 CP1252=92 [top ASCII-7-bit]
+ {{0x6d,0x71,0x5f,0x5f, 0x03,0xa3,0xb8,0x95,0x11,0xa5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mq__"
+ // ASCII-7-bit=163 Latin1=184 UTF8=149 CP1252=165 [top Latin1]
+ {{0x6d,0x72,0x5f,0x5f, 0x03,0x9b,0xab,0x8d,0x11,0x96,0x61,0xb6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mr__"
+ // ASCII-7-bit=155 Latin1=171 UTF8=141 CP1252=150 CP1256=182 [top CP1256]
+ {{0x6d,0x73,0x5f,0x5f, 0x03,0xb4,0xae,0x9c,0x18,0x8a,0x73,0x86,0x6f,0x53,0x56,0x6c,0x75,0x11,0x85,0x00,}}, // "ms__"
+ // ASCII-7-bit=180 Latin1=174 UTF8=156 CP1252=138 KSC=115 SJS=134 EUC-JP=111 BIG5=83 Latin2=86 CP1251=108 CP1256=117 Latin5=133 [top ASCII-7-bit]
+ {{0x6d,0x74,0x5f,0x5f, 0x05,0xbc,0x87,0x9b,0x5c,0x8b,0x42,0x2c,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mt__"
+ // ASCII-7-bit=188 Latin1=135 UTF8=155 GB=92 CP1252=139 Latin2=44 CP1251=64 [top ASCII-7-bit]
+ {{0x6d,0x75,0x5f,0x5f, 0x09,0xb4,0xa4,0xa9,0x57,0x9f,0x4d,0x7e,0x70,0x4d,0x11,0x4d,0x21,0x57,0x11,0x4d,}}, // "mu__"
+ // ASCII-7-bit=180 Latin1=164 UTF8=169 GB=87 CP1252=159 KSC=77 SJS=126 EUC-JP=112 BIG5=77 CP1251=77 Latin5=87 ISO-8859-15=77 [top ASCII-7-bit]
+ {{0x6d,0x75,0x73,0x65, 0x07,0xb3,0xa9,0xa6,0x76,0x90,0x56,0x88,0x13,0x8e,0x6a,0x7a,0x00,0x00,0x00,0x00,}}, // "muse"
+ // ASCII-7-bit=179 Latin1=169 UTF8=166 GB=118 CP1252=144 KSC=86 SJS=136 BIG5=142 Latin2=106 CP1251=122 [top ASCII-7-bit]
+ {{0x6d,0x76,0x5f,0x5f, 0x03,0xb6,0x98,0xab,0x11,0x9f,0x61,0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mv__"
+ // ASCII-7-bit=182 Latin1=152 UTF8=171 CP1252=159 CP1256=83 [top ASCII-7-bit]
+ {{0x6d,0x77,0x5f,0x5f, 0x05,0xb7,0xa3,0xa8,0x7f,0x8d,0x11,0x7f,0xc1,0x6b,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mw__"
+ // ASCII-7-bit=183 Latin1=163 UTF8=168 GB=127 CP1252=141 SJS=127 GBK=107 [top ASCII-7-bit]
+ {{0x6d,0x78,0x5f,0x5f, 0x05,0xa7,0xba,0x94,0x45,0x93,0x11,0x1d,0x12,0x25,0x29,0x21,0x27,0x21,0x3c,0x00,}}, // "mx__"
+ // ASCII-7-bit=167 Latin1=186 UTF8=148 GB=69 CP1252=147 SJS=29 BIG5=37 Latin2=41 CP1250=39 ISO-8859-15=60 [top Latin1]
+ {{0x6d,0x79,0x5f,0x5f, 0x05,0xb3,0xb3,0x8d,0x6b,0x8f,0x13,0x4d,0x50,0x7f,0x12,0x3f,0x6b,0x12,0x4b,0x3e,}}, // "my__"
+ // ASCII-7-bit=179 Latin1=179 UTF8=141 GB=107 CP1252=143 SJS=77 EUC-JP=80 BIG5=127 CP1251=63 CP1256=107 Latin5=75 ISO-8859-11=62 [top ASCII-7-bit]
+ {{0x6d,0x7a,0x5f,0x5f, 0x03,0x8e,0xaf,0xb7,0x11,0x8f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mz__"
+ // ASCII-7-bit=142 Latin1=175 UTF8=183 CP1252=143 [top UTF8]
+ {{0x6e,0x61,0x5f,0x5f, 0x03,0xba,0xa1,0x97,0x11,0x9c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "na__"
+ // ASCII-7-bit=186 Latin1=161 UTF8=151 CP1252=156 [top ASCII-7-bit]
+ {{0x6e,0x61,0x6d,0x65, 0x08,0xb2,0xa4,0xa9,0x8a,0x90,0x7d,0x7c,0x6b,0x15,0x80,0x94,0x6b,0x8b,0x77,0x00,}}, // "name"
+ // ASCII-7-bit=178 Latin1=164 UTF8=169 GB=138 CP1252=144 KSC=125 SJS=124 EUC-JP=107 Latin2=128 CP1251=148 CP1256=107 CP1250=139 Latin5=119 [top ASCII-7-bit]
+ {{0x6e,0x63,0x5f,0x5f, 0x03,0xa7,0xb9,0x8c,0x11,0x9e,0x11,0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nc__"
+ // ASCII-7-bit=167 Latin1=185 UTF8=140 CP1252=158 SJS=114 [top Latin1]
+ {{0x6e,0x65,0x5f,0x5f, 0x03,0xad,0xb9,0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ne__"
+ // ASCII-7-bit=173 Latin1=185 UTF8=128 [top Latin1]
+ {{0x6e,0x65,0x74,0x5f, 0x0f,0xac,0xa6,0xa4,0x93,0x93,0x94,0x97,0x83,0x86,0x7e,0x80,0x96,0x7a,0x89,0x73,}}, // "net_"
+ // ASCII-7-bit=172 Latin1=166 UTF8=164 GB=147 CP1252=147 KSC=148 SJS=151 EUC-JP=131 BIG5=134 Latin2=126 CP1251=128 CP1256=150 CP1250=122 Latin5=137 ISO-8859-11=115 [top ASCII-7-bit]
+ {{0x6e,0x66,0x5f,0x5f, 0x03,0xb5,0xa9,0xa8,0x11,0x87,0xa1,0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nf__"
+ // ASCII-7-bit=181 Latin1=169 UTF8=168 CP1252=135 ISO-8859-15=119 [top ASCII-7-bit]
+ {{0x6e,0x67,0x5f,0x5f, 0x03,0xbd,0x95,0x89,0x11,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ng__"
+ // ASCII-7-bit=189 Latin1=149 UTF8=137 CP1252=146 [top ASCII-7-bit]
+ {{0x6e,0x69,0x5f,0x5f, 0x03,0x9d,0xb3,0xa9,0x11,0xa8,0x81,0x40,0x11,0x36,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ni__"
+ // ASCII-7-bit=157 Latin1=179 UTF8=169 CP1252=168 Latin5=64 ISO-8859-15=54 [top Latin1]
+ {{0x6e,0x6c,0x5f,0x5f, 0x05,0xb2,0xae,0xa0,0x32,0xa1,0x36,0x31,0x42,0x39,0x3b,0x33,0x45,0x11,0x6f,0x00,}}, // "nl__"
+ // ASCII-7-bit=178 Latin1=174 UTF8=160 GB=50 CP1252=161 BIG5=49 Latin2=66 CP1251=57 CP1256=59 CP1250=51 Latin5=69 ISO-8859-15=111 [top ASCII-7-bit]
+ {{0x6e,0x6f,0x5f,0x5f, 0x05,0x99,0xb8,0xaa,0x47,0x8d,0x11,0x31,0x22,0x34,0x3e,0x42,0x70,0x33,0x00,0x00,}}, // "no__"
+ // ASCII-7-bit=153 Latin1=184 UTF8=170 GB=71 CP1252=141 SJS=49 Latin2=52 CP1251=62 ISO-8859-15=112 CP1257=51 [top Latin1]
+ {{0x6e,0x70,0x5f,0x5f, 0x03,0xb2,0x9f,0xa5,0x11,0xac,0x11,0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "np__"
+ // ASCII-7-bit=178 Latin1=159 UTF8=165 CP1252=172 SJS=97 [top ASCII-7-bit]
+ {{0x6e,0x72,0x5f,0x5f, 0x03,0xbe,0x77,0x7a,0x11,0x62,0x71,0x44,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nr__"
+ // ASCII-7-bit=190 Latin1=119 UTF8=122 CP1252=98 CP1250=68 [top ASCII-7-bit]
+ {{0x6e,0x75,0x5f,0x5f, 0x03,0xae,0xb4,0x9e,0x11,0x96,0x12,0x7c,0x76,0x22,0x75,0x62,0xa1,0x65,0x00,0x00,}}, // "nu__"
+ // ASCII-7-bit=174 Latin1=180 UTF8=158 CP1252=150 SJS=124 EUC-JP=118 CP1251=117 CP1256=98 CP1254=101 [top Latin1]
+ {{0x6e,0x7a,0x5f,0x5f, 0x0d,0xba,0x97,0xa2,0x5f,0x97,0x5c,0x5a,0x4b,0x4a,0x30,0x47,0x36,0x34,0x21,0x2b,}}, // "nz__"
+ // ASCII-7-bit=186 Latin1=151 UTF8=162 GB=95 CP1252=151 KSC=92 SJS=90 EUC-JP=75 BIG5=74 Latin2=48 CP1251=71 CP1256=54 CP1250=52 ISO-8859-15=43 [top ASCII-7-bit]
+ {{0x6f,0x6d,0x5f,0x5f, 0x03,0x9a,0x8a,0x89,0x11,0x87,0x61,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "om__"
+ // ASCII-7-bit=154 Latin1=138 UTF8=137 CP1252=135 CP1256=188 [top CP1256]
+ {{0x6f,0x72,0x67,0x5f, 0x0e,0xb5,0x9f,0xac,0x76,0x90,0x7e,0x6e,0x69,0x71,0x6d,0x6a,0x78,0x60,0x73,0x00,}}, // "org_"
+ // ASCII-7-bit=181 Latin1=159 UTF8=172 GB=118 CP1252=144 KSC=126 SJS=110 EUC-JP=105 BIG5=113 Latin2=109 CP1251=106 CP1256=120 CP1250=96 Latin5=115 [top ASCII-7-bit]
+ {{0x70,0x61,0x5f,0x5f, 0x03,0xa3,0xb8,0xa7,0x11,0x87,0x71,0x31,0x21,0x41,0x21,0x31,0x00,0x00,0x00,0x00,}}, // "pa__"
+ // ASCII-7-bit=163 Latin1=184 UTF8=167 CP1252=135 CP1250=49 ISO-8859-15=65 KOI8R=49 [top Latin1]
+ {{0x70,0x65,0x5f,0x5f, 0x05,0xa8,0xb8,0x9d,0x3a,0x99,0x11,0x2e,0x13,0x2e,0x1e,0x1e,0x11,0x32,0x00,0x00,}}, // "pe__"
+ // ASCII-7-bit=168 Latin1=184 UTF8=157 GB=58 CP1252=153 SJS=46 BIG5=46 Latin2=30 CP1251=30 CP1250=50 [top Latin1]
+ {{0x70,0x66,0x5f,0x5f, 0x03,0xa2,0xb5,0xa3,0x11,0xa7,0x11,0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pf__"
+ // ASCII-7-bit=162 Latin1=181 UTF8=163 CP1252=167 SJS=99 [top Latin1]
+ {{0x70,0x67,0x5f,0x5f, 0x05,0xb8,0xa6,0x81,0x6a,0xa5,0x11,0x70,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pg__"
+ // ASCII-7-bit=184 Latin1=166 UTF8=129 GB=106 CP1252=165 SJS=112 [top ASCII-7-bit]
+ {{0x70,0x68,0x5f,0x5f, 0x03,0xb5,0x9e,0x99,0x15,0xac,0x57,0x65,0x4f,0x51,0x31,0x31,0x11,0x4b,0xd1,0x29,}}, // "ph__"
+ // ASCII-7-bit=181 Latin1=158 UTF8=153 CP1252=172 KSC=87 SJS=101 EUC-JP=79 BIG5=81 CP1250=49 ISO-8859-11=75 CP874=41 [top ASCII-7-bit]
+ {{0x70,0x6b,0x5f,0x5f, 0x03,0xb9,0x9b,0x8c,0x11,0xa3,0x11,0x33,0x11,0x99,0x21,0x29,0x00,0x00,0x00,0x00,}}, // "pk__"
+ // ASCII-7-bit=185 Latin1=155 UTF8=140 CP1252=163 SJS=51 BIG5=153 CP1256=41 [top ASCII-7-bit]
+ {{0x70,0x6c,0x5f,0x5f, 0x03,0x89,0x62,0xa8,0x11,0x4b,0x42,0xba,0x40,0x11,0x92,0xe1,0x3e,0x00,0x00,0x00,}}, // "pl__"
+ // ASCII-7-bit=137 Latin1=98 UTF8=168 CP1252=75 Latin2=186 CP1251=64 CP1250=146 ISO-8859-5=62 [top Latin2]
+ {{0x70,0x6e,0x5f,0x5f, 0x06,0xbb,0xa1,0x99,0x66,0x66,0x66,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pn__"
+ // ASCII-7-bit=187 Latin1=161 UTF8=153 GB=102 CP1252=102 KSC=102 [top ASCII-7-bit]
+ {{0x70,0x72,0x5f,0x5f, 0x03,0x9b,0xb5,0xaf,0x11,0x94,0x41,0x65,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pr__"
+ // ASCII-7-bit=155 Latin1=181 UTF8=175 CP1252=148 Latin2=101 [top Latin1]
+ {{0x70,0x72,0x6f,0x5f, 0x03,0xb3,0x9e,0xad,0x11,0x87,0x13,0x65,0x6f,0x65,0x11,0x9f,0x21,0x79,0x00,0x00,}}, // "pro_"
+ // ASCII-7-bit=179 Latin1=158 UTF8=173 CP1252=135 SJS=101 EUC-JP=111 BIG5=101 CP1251=159 Latin5=121 [top ASCII-7-bit]
+ {{0x70,0x73,0x5f,0x5f, 0x03,0x99,0x8f,0x9d,0x11,0x9f,0x61,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ps__"
+ // ASCII-7-bit=153 Latin1=143 UTF8=157 CP1252=159 CP1256=185 [top CP1256]
+ {{0x70,0x74,0x5f,0x5f, 0x05,0x99,0xb5,0xad,0x1d,0x95,0x11,0x21,0x52,0x36,0x40,0x12,0x90,0x23,0x51,0x1d,}}, // "pt__"
+ // ASCII-7-bit=153 Latin1=181 UTF8=173 GB=29 CP1252=149 SJS=33 CP1250=54 Latin5=64 ISO-8859-15=144 CP1257=35 CP1254=29 [top Latin1]
+ {{0x70,0x79,0x5f,0x5f, 0x03,0x9f,0xbb,0x90,0x13,0x97,0x49,0x35,0x21,0x35,0x51,0x5a,0x00,0x00,0x00,0x00,}}, // "py__"
+ // ASCII-7-bit=159 Latin1=187 UTF8=144 CP1252=151 KSC=73 SJS=53 Latin2=53 ISO-8859-15=90 [top Latin1]
+ {{0x71,0x61,0x5f,0x5f, 0x03,0x9a,0x89,0xb0,0x11,0x82,0x61,0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "qa__"
+ // ASCII-7-bit=154 Latin1=137 UTF8=176 CP1252=130 CP1256=181 [top CP1256]
+ {{0x72,0x65,0x5f,0x5f, 0x03,0x8a,0xb4,0xb1,0x11,0x97,0xa1,0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "re__"
+ // ASCII-7-bit=138 Latin1=180 UTF8=177 CP1252=151 ISO-8859-15=138 [top Latin1]
+ {{0x72,0x6f,0x5f,0x5f, 0x08,0xb4,0xa0,0xa4,0x3c,0x94,0x57,0x2e,0x29,0x14,0xa2,0x3a,0x33,0x8b,0x21,0x37,}}, // "ro__"
+ // ASCII-7-bit=180 Latin1=160 UTF8=164 GB=60 CP1252=148 KSC=87 SJS=46 EUC-JP=41 Latin2=162 CP1251=58 CP1256=51 CP1250=139 ISO-8859-15=55 [top ASCII-7-bit]
+ {{0x72,0x75,0x5f,0x5f, 0x05,0x8d,0x6d,0xa1,0x56,0x67,0x31,0x43,0x12,0xba,0x46,0x61,0x9b,0x72,0x46,0x48,}}, // "ru__"
+ // ASCII-7-bit=141 Latin1=109 UTF8=161 GB=86 CP1252=103 BIG5=67 CP1251=186 CP1256=70 KOI8R=155 KOI8U=70 ISO-8859-5=72 [top CP1251]
+ {{0x72,0x77,0x5f,0x5f, 0x03,0xb7,0xa2,0xa4,0x11,0xa0,0x61,0x5e,0x31,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "rw__"
+ // ASCII-7-bit=183 Latin1=162 UTF8=164 CP1252=160 CP1256=94 ISO-8859-15=110 [top ASCII-7-bit]
+ {{0x73,0x61,0x5f,0x5f, 0x05,0x91,0x5b,0xac,0x3f,0x69,0x11,0x1f,0x41,0xb9,0x11,0x1f,0x00,0x00,0x00,0x00,}}, // "sa__"
+ // ASCII-7-bit=145 Latin1=91 UTF8=172 GB=63 CP1252=105 SJS=31 CP1256=185 Latin5=31 [top CP1256]
+ {{0x73,0x62,0x5f,0x5f, 0x03,0xb8,0x8a,0xad,0x11,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sb__"
+ // ASCII-7-bit=184 Latin1=138 UTF8=173 CP1252=139 [top ASCII-7-bit]
+ {{0x73,0x63,0x5f,0x5f, 0x03,0xb5,0x9e,0xad,0x11,0x85,0x12,0x8d,0x88,0x23,0x57,0x4d,0x5c,0x00,0x00,0x00,}}, // "sc__"
+ // ASCII-7-bit=181 Latin1=158 UTF8=173 CP1252=133 SJS=141 EUC-JP=136 CP1251=87 CP1256=77 CP1250=92 [top ASCII-7-bit]
+ {{0x73,0x64,0x5f,0x5f, 0x03,0x9b,0x77,0x8a,0x11,0x6d,0x61,0xbd,0x10,0x61,0x73,0x00,0x00,0x00,0x00,0x00,}}, // "sd__"
+ // ASCII-7-bit=155 Latin1=119 UTF8=138 CP1252=109 CP1256=189 Arabic=115 [top CP1256]
+ {{0x73,0x65,0x5f,0x5f, 0x05,0x96,0xbb,0xa0,0x23,0x82,0x36,0x23,0x2e,0x25,0x3b,0x43,0x22,0x11,0x4c,0x00,}}, // "se__"
+ // ASCII-7-bit=150 Latin1=187 UTF8=160 GB=35 CP1252=130 BIG5=35 Latin2=46 CP1251=37 CP1256=59 CP1250=67 Latin5=34 ISO-8859-15=76 [top Latin1]
+ {{0x73,0x67,0x5f,0x5f, 0x09,0xb8,0x9c,0xa6,0x84,0x94,0x4c,0x6e,0x50,0x71,0x31,0x41,0x11,0x48,0xd1,0x3a,}}, // "sg__"
+ // ASCII-7-bit=184 Latin1=156 UTF8=166 GB=132 CP1252=148 KSC=76 SJS=110 EUC-JP=80 BIG5=113 CP1250=65 ISO-8859-11=72 CP874=58 [top ASCII-7-bit]
+ {{0x73,0x68,0x5f,0x5f, 0x0a,0xaa,0x9b,0xa1,0xa9,0x84,0x77,0xa1,0x98,0x9b,0x5d,0x51,0x6d,0x31,0x6f,0x00,}}, // "sh__"
+ // ASCII-7-bit=170 Latin1=155 UTF8=161 GB=169 CP1252=132 KSC=119 SJS=161 EUC-JP=152 BIG5=155 Latin2=93 ISO-8859-15=109 GBK=111 [top ASCII-7-bit]
+ {{0x73,0x69,0x5f,0x5f, 0x03,0x95,0x6b,0xb7,0x11,0x6e,0x42,0x9f,0x3d,0x11,0xa9,0x21,0x17,0x10,0x11,0x24,}}, // "si__"
+ // ASCII-7-bit=149 Latin1=107 UTF8=183 CP1252=110 Latin2=159 CP1251=61 CP1250=169 ISO-8859-15=23 CP852=36 [top UTF8]
+ {{0x73,0x6b,0x5f,0x5f, 0x03,0x95,0x74,0xb0,0x11,0x60,0x36,0x53,0x92,0x55,0x47,0xb5,0x3f,0x10,0x11,0x3a,}}, // "sk__"
+ // ASCII-7-bit=149 Latin1=116 UTF8=176 CP1252=96 BIG5=83 Latin2=146 CP1251=85 CP1256=71 CP1250=181 Latin5=63 MACINTOSH=58 [top CP1250]
+ {{0x73,0x6c,0x5f,0x5f, 0x03,0xac,0x85,0x8f,0x11,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sl__"
+ // ASCII-7-bit=172 Latin1=133 UTF8=143 CP1252=185 [top CP1252]
+ {{0x73,0x6d,0x5f,0x5f, 0x03,0xa8,0xa7,0xb1,0x11,0xa8,0x91,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sm__"
+ // ASCII-7-bit=168 Latin1=167 UTF8=177 CP1252=168 ISO-8859-11=111 [top UTF8]
+ {{0x73,0x6e,0x5f,0x5f, 0x03,0x9d,0xb8,0x9f,0x11,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sn__"
+ // ASCII-7-bit=157 Latin1=184 UTF8=159 CP1252=162 [top Latin1]
+ {{0x73,0x72,0x5f,0x5f, 0x03,0xa6,0xad,0xb2,0x12,0x9d,0x6a,0x61,0x84,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sr__"
+ // ASCII-7-bit=166 Latin1=173 UTF8=178 CP1252=157 KSC=106 CP1250=132 [top UTF8]
+ {{0x73,0x74,0x5f,0x5f, 0x03,0xb4,0xab,0x9d,0x15,0x88,0x79,0x99,0x94,0x7e,0x31,0x61,0x81,0x4e,0x21,0x6d,}}, // "st__"
+ // ASCII-7-bit=180 Latin1=171 UTF8=157 CP1252=136 KSC=121 SJS=153 EUC-JP=148 BIG5=126 CP1250=97 JIS=78 CP932=109 [top ASCII-7-bit]
+ {{0x73,0x75,0x5f,0x5f, 0x03,0xa4,0x6f,0xa0,0x11,0x70,0x51,0xb9,0x71,0x94,0x71,0x44,0x00,0x00,0x00,0x00,}}, // "su__"
+ // ASCII-7-bit=164 Latin1=111 UTF8=160 CP1252=112 CP1251=185 KOI8R=148 KOI8U=68 [top CP1251]
+ {{0x73,0x76,0x5f,0x5f, 0x03,0x9d,0xb9,0xa5,0x11,0x8f,0x41,0x3c,0x51,0x3c,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sv__"
+ // ASCII-7-bit=157 Latin1=185 UTF8=165 CP1252=143 Latin2=60 ISO-8859-15=60 [top Latin1]
+ {{0x73,0x79,0x5f,0x5f, 0x03,0x82,0x5e,0x90,0x11,0x5d,0x61,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sy__"
+ // ASCII-7-bit=130 Latin1=94 UTF8=144 CP1252=93 CP1256=190 [top CP1256]
+ {{0x73,0x7a,0x5f,0x5f, 0x03,0xb7,0xac,0x6c,0x11,0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sz__"
+ // ASCII-7-bit=183 Latin1=172 UTF8=108 CP1252=161 [top ASCII-7-bit]
+ {{0x74,0x63,0x5f,0x5f, 0x08,0xa9,0xaa,0x9b,0x62,0x74,0x8d,0x8f,0x7a,0x21,0x67,0x22,0xab,0x9b,0x41,0x70,}}, // "tc__"
+ // ASCII-7-bit=169 Latin1=170 UTF8=155 GB=98 CP1252=116 KSC=141 SJS=143 EUC-JP=122 CP1251=103 Latin5=171 ISO-8859-11=155 GBK=112 [top Latin5]
+ {{0x74,0x66,0x5f,0x5f, 0x02,0xa3,0xbc,0x21,0x71,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tf__"
+ // ASCII-7-bit=163 Latin1=188 CP1252=113 [top Latin1]
+ {{0x74,0x67,0x5f,0x5f, 0x03,0xb0,0xb3,0x7e,0x11,0xa5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tg__"
+ // ASCII-7-bit=176 Latin1=179 UTF8=126 CP1252=165 [top Latin1]
+ {{0x74,0x68,0x5f,0x5f, 0x09,0x96,0x61,0x93,0x3e,0x67,0x2f,0x52,0x38,0x23,0x11,0x35,0x31,0xbd,0xd1,0x76,}}, // "th__"
+ // ASCII-7-bit=150 Latin1=97 UTF8=147 GB=62 CP1252=103 KSC=47 SJS=82 EUC-JP=56 BIG5=35 CP1251=53 ISO-8859-11=189 CP874=118 [top ISO-8859-11]
+ {{0x74,0x6a,0x5f,0x5f, 0x03,0xab,0x74,0xb1,0x11,0x67,0x51,0xae,0x71,0x84,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tj__"
+ // ASCII-7-bit=171 Latin1=116 UTF8=177 CP1252=103 CP1251=174 KOI8R=132 [top UTF8]
+ {{0x74,0x6b,0x5f,0x5f, 0x03,0xbc,0x94,0x9d,0x11,0x6b,0x12,0x74,0x6b,0x12,0x53,0x46,0x12,0x74,0x6d,0x00,}}, // "tk__"
+ // ASCII-7-bit=188 Latin1=148 UTF8=157 CP1252=107 SJS=116 EUC-JP=107 Latin2=83 CP1251=70 CP1250=116 Latin5=109 [top ASCII-7-bit]
+ {{0x74,0x6c,0x5f,0x5f, 0x05,0xb1,0xb0,0x88,0x61,0x83,0xa1,0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tl__"
+ // ASCII-7-bit=177 Latin1=176 UTF8=136 GB=97 CP1252=131 ISO-8859-15=168 [top ASCII-7-bit]
+ {{0x74,0x6d,0x5f,0x5f, 0x03,0xb4,0x84,0xad,0x11,0x82,0x42,0x5a,0xa4,0x12,0x5a,0x5a,0x00,0x00,0x00,0x00,}}, // "tm__"
+ // ASCII-7-bit=180 Latin1=132 UTF8=173 CP1252=130 Latin2=90 CP1251=164 CP1250=90 Latin5=90 [top ASCII-7-bit]
+ {{0x74,0x6e,0x5f,0x5f, 0x03,0x9f,0xa2,0xac,0x11,0x9d,0x11,0x4b,0x21,0x3b,0x11,0xb0,0x10,0x61,0x3b,0x00,}}, // "tn__"
+ // ASCII-7-bit=159 Latin1=162 UTF8=172 CP1252=157 SJS=75 Latin2=59 CP1256=176 Arabic=59 [top CP1256]
+ {{0x74,0x6f,0x5f,0x5f, 0x03,0xa5,0xa2,0xa5,0x15,0x89,0x8a,0xac,0x99,0x9b,0x31,0x70,0x21,0x87,0x51,0x6c,}}, // "to__"
+ // ASCII-7-bit=165 Latin1=162 UTF8=165 CP1252=137 KSC=138 SJS=172 EUC-JP=153 BIG5=155 CP1250=112 ISO-8859-15=135 JIS=108 [top SJS]
+ {{0x74,0x70,0x5f,0x5f, 0x03,0x95,0x9e,0xad,0x11,0x67,0x12,0xb3,0x99,0xd1,0x67,0x21,0x67,0x00,0x00,0x00,}}, // "tp__"
+ // ASCII-7-bit=149 Latin1=158 UTF8=173 CP1252=103 SJS=179 EUC-JP=153 JIS=103 CP932=103 [top SJS]
+ {{0x74,0x72,0x5f,0x5f, 0x03,0x8d,0x6c,0xa6,0x11,0x61,0x11,0x3c,0x32,0x3c,0x4a,0x11,0xba,0x81,0x91,0x00,}}, // "tr__"
+ // ASCII-7-bit=141 Latin1=108 UTF8=166 CP1252=97 SJS=60 CP1251=60 CP1256=74 Latin5=186 CP1254=145 [top Latin5]
+ {{0x74,0x72,0x61,0x76, 0x05,0xa9,0xa3,0xa7,0x97,0x95,0x11,0xac,0x13,0x74,0x7f,0x6b,0x11,0x63,0x00,0x00,}}, // "trav"
+ // ASCII-7-bit=169 Latin1=163 UTF8=167 GB=151 CP1252=149 SJS=172 BIG5=116 Latin2=127 CP1251=107 CP1250=99 [top SJS]
+ {{0x74,0x74,0x5f,0x5f, 0x07,0xb5,0xaf,0x9a,0x49,0x92,0x59,0x49,0x61,0x75,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tt__"
+ // ASCII-7-bit=181 Latin1=175 UTF8=154 GB=73 CP1252=146 KSC=89 SJS=73 Latin5=117 [top ASCII-7-bit]
+ {{0x74,0x76,0x5f,0x5f, 0x0e,0xa7,0xa6,0xad,0x89,0x94,0x85,0x9e,0x8d,0x7b,0x74,0x7c,0x88,0x7b,0x81,0x00,}}, // "tv__"
+ // ASCII-7-bit=167 Latin1=166 UTF8=173 GB=137 CP1252=148 KSC=133 SJS=158 EUC-JP=141 BIG5=123 Latin2=116 CP1251=124 CP1256=136 CP1250=123 Latin5=129 [top UTF8]
+ {{0x74,0x77,0x5f,0x5f, 0x05,0x85,0x52,0xab,0x5d,0x57,0x13,0x50,0x2e,0xba,0x31,0x23,0x61,0x2e,0xf1,0x21,}}, // "tw__"
+ // ASCII-7-bit=133 Latin1=82 UTF8=171 GB=93 CP1252=87 SJS=80 EUC-JP=46 BIG5=186 CP1250=35 GBK=46 BIG5_HKSCS=33 [top BIG5]
+ {{0x74,0x7a,0x5f,0x5f, 0x03,0xae,0xb5,0x8b,0x13,0xa0,0x4c,0x4c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tz__"
+ // ASCII-7-bit=174 Latin1=181 UTF8=139 CP1252=160 KSC=76 SJS=76 [top Latin1]
+ {{0x75,0x61,0x5f,0x5f, 0x03,0x87,0x61,0xa2,0x11,0x57,0x44,0x4d,0xbb,0x35,0x48,0x51,0x80,0x72,0x8c,0x3d,}}, // "ua__"
+ // ASCII-7-bit=135 Latin1=97 UTF8=162 CP1252=87 Latin2=77 CP1251=187 CP1256=53 CP1250=72 KOI8R=128 KOI8U=140 ISO-8859-5=61 [top CP1251]
+ {{0x75,0x67,0x5f,0x5f, 0x03,0xb8,0x9a,0xa0,0x11,0xa6,0x11,0x68,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ug__"
+ // ASCII-7-bit=184 Latin1=154 UTF8=160 CP1252=166 SJS=104 [top ASCII-7-bit]
+ {{0x75,0x6b,0x5f,0x5f, 0x05,0xb7,0xa7,0x9e,0x62,0x97,0x27,0x58,0x62,0x51,0x53,0x62,0x64,0x54,0x11,0x6f,}}, // "uk__"
+ // ASCII-7-bit=183 Latin1=167 UTF8=158 GB=98 CP1252=151 EUC-JP=88 BIG5=98 Latin2=81 CP1251=83 CP1256=98 CP1250=100 Latin5=84 ISO-8859-15=111 [top ASCII-7-bit]
+ {{0x75,0x73,0x5f,0x5f, 0x06,0xba,0x94,0xa5,0x45,0x92,0x48,0x24,0x54,0x47,0x5a,0x56,0x11,0x66,0x11,0x40,}}, // "us__"
+ // ASCII-7-bit=186 Latin1=148 UTF8=165 GB=69 CP1252=146 KSC=72 BIG5=84 Latin2=71 CP1251=90 CP1256=86 Latin5=102 ISO-8859-15=64 [top ASCII-7-bit]
+ {{0x75,0x79,0x5f,0x5f, 0x03,0xa2,0xba,0x96,0x11,0x9a,0x72,0x2c,0x2c,0x11,0x52,0x00,0x00,0x00,0x00,0x00,}}, // "uy__"
+ // ASCII-7-bit=162 Latin1=186 UTF8=150 CP1252=154 CP1250=44 Latin5=44 ISO-8859-15=82 [top Latin1]
+ {{0x75,0x7a,0x5f,0x5f, 0x05,0x91,0x68,0xb1,0x39,0x88,0x54,0xb5,0x43,0x39,0x6e,0x41,0x7c,0x00,0x00,0x00,}}, // "uz__"
+ // ASCII-7-bit=145 Latin1=104 UTF8=177 GB=57 CP1252=136 CP1251=181 CP1256=67 CP1250=57 Latin5=110 KOI8R=124 [top CP1251]
+ {{0x76,0x61,0x5f,0x5f, 0x03,0xae,0xb3,0x8f,0x11,0xa7,0xb1,0x4a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "va__"
+ // ASCII-7-bit=174 Latin1=179 UTF8=143 CP1252=167 CP1257=74 [top Latin1]
+ {{0x76,0x63,0x5f,0x5f, 0x08,0xba,0x7a,0x9a,0x5e,0x79,0x4a,0x9a,0x95,0x12,0x54,0x7f,0x81,0x4a,0x00,0x00,}}, // "vc__"
+ // ASCII-7-bit=186 Latin1=122 UTF8=154 GB=94 CP1252=121 KSC=74 SJS=154 EUC-JP=149 Latin2=84 CP1251=127 GBK=74 [top ASCII-7-bit]
+ {{0x76,0x65,0x5f,0x5f, 0x05,0x97,0xbc,0x8c,0x14,0x9b,0x11,0x14,0x11,0x1e,0x31,0x42,0x21,0x55,0x31,0x24,}}, // "ve__"
+ // ASCII-7-bit=151 Latin1=188 UTF8=140 GB=20 CP1252=155 SJS=20 BIG5=30 CP1250=66 ISO-8859-15=85 GBK=36 [top Latin1]
+ {{0x76,0x67,0x5f,0x5f, 0x05,0xac,0xb2,0x9e,0x6d,0x97,0x13,0x95,0x81,0x95,0x11,0x69,0x21,0x63,0x61,0x59,}}, // "vg__"
+ // ASCII-7-bit=172 Latin1=178 UTF8=158 GB=109 CP1252=151 SJS=149 EUC-JP=129 BIG5=149 CP1251=105 Latin5=99 Greek=89 [top Latin1]
+ {{0x76,0x69,0x5f,0x5f, 0x03,0xb9,0x90,0xaa,0x11,0x93,0x11,0x68,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "vi__"
+ // ASCII-7-bit=185 Latin1=144 UTF8=170 CP1252=147 SJS=104 [top ASCII-7-bit]
+ {{0x76,0x6e,0x5f,0x5f, 0x03,0x94,0x92,0xbd,0x12,0x83,0x22,0x21,0x2c,0x32,0x26,0x2e,0x00,0x00,0x00,0x00,}}, // "vn__"
+ // ASCII-7-bit=148 Latin1=146 UTF8=189 CP1252=131 KSC=34 BIG5=44 CP1250=38 Latin5=46 [top UTF8]
+ {{0x76,0x75,0x5f,0x5f, 0x03,0xae,0xb6,0x97,0x11,0x7b,0x11,0x5b,0x31,0x6a,0x41,0x4e,0x71,0x90,0x00,0x00,}}, // "vu__"
+ // ASCII-7-bit=174 Latin1=182 UTF8=151 CP1252=123 SJS=91 CP1251=106 ISO-8859-15=78 CP1253=144 [top Latin1]
+ {{0x77,0x73,0x5f,0x5f, 0x05,0xae,0xaa,0xa6,0x6c,0x91,0x18,0x76,0x69,0x7c,0x78,0x99,0x9d,0x7d,0x70,0x00,}}, // "ws__"
+ // ASCII-7-bit=174 Latin1=170 UTF8=166 GB=108 CP1252=145 SJS=118 EUC-JP=105 BIG5=124 Latin2=120 CP1251=153 CP1256=157 CP1250=125 Latin5=112 [top ASCII-7-bit]
+ {{0x79,0x65,0x5f,0x5f, 0x03,0x9e,0x94,0xab,0x11,0x8b,0x61,0xb6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ye__"
+ // ASCII-7-bit=158 Latin1=148 UTF8=171 CP1252=139 CP1256=182 [top CP1256]
+ {{0x79,0x75,0x5f,0x5f, 0x03,0xa4,0x7f,0xaf,0x11,0x78,0x42,0x87,0x80,0x12,0xb3,0x2c,0xd1,0x60,0x00,0x00,}}, // "yu__"
+ // ASCII-7-bit=164 Latin1=127 UTF8=175 CP1252=120 Latin2=135 CP1251=128 CP1250=179 Latin5=44 ISO-8859-5=96 [top CP1250]
+ {{0x7a,0x61,0x5f,0x5f, 0x05,0xb8,0xa3,0x97,0x42,0xa1,0x11,0x30,0x11,0x4e,0x13,0x3a,0x4f,0x41,0x21,0x42,}}, // "za__"
+ // ASCII-7-bit=184 Latin1=163 UTF8=151 GB=66 CP1252=161 SJS=48 BIG5=78 CP1251=58 CP1256=79 CP1250=65 ISO-8859-15=66 [top ASCII-7-bit]
+ {{0x7a,0x6d,0x5f,0x5f, 0x03,0xb8,0x8e,0x9c,0x11,0xa9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "zm__"
+ // ASCII-7-bit=184 Latin1=142 UTF8=156 CP1252=169 [top ASCII-7-bit]
+ {{0x7a,0x77,0x5f,0x5f, 0x05,0xbb,0x95,0x9b,0x59,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "zw__"
+ // ASCII-7-bit=187 Latin1=149 UTF8=155 GB=89 CP1252=154 [top ASCII-7-bit]
+};
+
+static const int kTLDHintProbsSize = 247;
+
+static const HintEntry kCharsetHintProbs[] = { // MaxRange 192
+ {{0x5f,0x5f,0x5f,0x5f,0x30,0x36,0x34,0x36, 0x02,0xbd,0x7f,0x21,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____0646"
+ // ASCII-7-bit=189 Latin1=127 CP1252=149 [top ASCII-7-bit]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x30, 0x01,0x96,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1250"
+ // ASCII-7-bit=150 CP1250=190 [top CP1250]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x31, 0x01,0x7a,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1251"
+ // ASCII-7-bit=122 CP1251=190 [top CP1251]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x32, 0x02,0x99,0x9d,0x21,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1252"
+ // ASCII-7-bit=153 Latin1=157 CP1252=188 [top CP1252]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x33, 0x01,0x79,0x10,0x61,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1253"
+ // ASCII-7-bit=121 CP1253=190 [top CP1253]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x34, 0x01,0x71,0xc1,0xaf,0x81,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1254"
+ // ASCII-7-bit=113 Latin5=175 CP1254=185 [top CP1254]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x35, 0x01,0x86,0x10,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1255"
+ // ASCII-7-bit=134 CP1255=190 [top CP1255]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x36, 0x01,0x78,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1256"
+ // ASCII-7-bit=120 CP1256=190 [top CP1256]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x32,0x35,0x37, 0x01,0x79,0xf1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1257"
+ // ASCII-7-bit=121 CP1257=190 [top CP1257]
+ {{0x5f,0x5f,0x5f,0x5f,0x31,0x38,0x30,0x30, 0x10,0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____1800"
+ // KOI8R=191 [top KOI8R]
+ {{0x5f,0x5f,0x5f,0x5f,0x33,0x36,0x30,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____3600"
+ // CP1250=191 [top CP1250]
+ {{0x5f,0x5f,0x5f,0x5f,0x33,0x36,0x39,0x39, 0x01,0xad,0x11,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____3699"
+ // ASCII-7-bit=173 UTF8=185 [top UTF8]
+ {{0x5f,0x5f,0x5f,0x5f,0x34,0x34,0x30,0x30, 0x02,0xbc,0x87,0x21,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____4400"
+ // ASCII-7-bit=188 Latin1=135 CP1252=164 [top ASCII-7-bit]
+ {{0x5f,0x5f,0x5f,0x5f,0x35,0x30,0x30,0x31, 0x01,0x9a,0x11,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____5001"
+ // ASCII-7-bit=154 UTF8=189 [top UTF8]
+ {{0x5f,0x5f,0x5f,0x5f,0x35,0x39,0x31,0x35, 0x02,0xa8,0xa6,0x21,0xa7,0xa1,0xb2,0x00,0x00,0x00,0x00,0x00,}}, // "____5915"
+ // ASCII-7-bit=168 Latin1=166 CP1252=167 ISO-8859-15=178 [top ISO-8859-15]
+ {{0x5f,0x5f,0x5f,0x5f,0x36,0x34,0x36,0x5f, 0x02,0xbe,0x8e,0x21,0x81,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____646_"
+ // ASCII-7-bit=190 Latin1=142 CP1252=129 [top ASCII-7-bit]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0xae,0xb8,0x21,0x94,0xa1,0x2f,0x00,0x00,0x00,0x00,0x00,}}, // "____8591"
+ // ASCII-7-bit=174 Latin1=184 CP1252=148 ISO-8859-15=47 [top Latin1]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x32, 0x01,0x8c,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8592"
+ // ASCII-7-bit=140 Latin2=190 [top Latin2]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x34, 0x10,0xe1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8594"
+ // Latin4=191 [top Latin4]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x35, 0x01,0x6f,0x10,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8595"
+ // ASCII-7-bit=111 ISO-8859-5=190 [top ISO-8859-5]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x37, 0x10,0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8597"
+ // Greek=191 [top Greek]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x39, 0x01,0x72,0xc1,0xbe,0x81,0x7b,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8599"
+ // ASCII-7-bit=114 Latin5=190 CP1254=123 [top Latin5]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x38,0x36,0x31, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8861"
+ // Latin2=191 [top Latin2]
+ {{0x5f,0x5f,0x5f,0x5f,0x38,0x5f,0x5f,0x5f, 0x03,0x98,0x5d,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "____8___"
+ // ASCII-7-bit=152 Latin1=93 UTF8=189 [top UTF8]
+ {{0x5f,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x07,0xb1,0xaa,0x9c,0x95,0x96,0x8e,0x8c,0x11,0x82,0x00,0x00,}}, // "________"
+ // ASCII-7-bit=177 Latin1=170 UTF8=156 GB=149 CP1252=150 KSC=142 SJS=140 BIG5=130 [top ASCII-7-bit]
+ {{0x61,0x6e,0x73,0x69,0x33,0x34,0x5f,0x5f, 0x02,0xbe,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ansi34__"
+ // ASCII-7-bit=190 Latin1=110 [top ASCII-7-bit]
+ {{0x61,0x6e,0x73,0x69,0x5f,0x5f,0x5f,0x5f, 0x02,0xa2,0xb9,0x21,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ansi____"
+ // ASCII-7-bit=162 Latin1=185 CP1252=164 [top Latin1]
+ {{0x61,0x72,0x72,0x61,0x5f,0x5f,0x5f,0x5f, 0x01,0x90,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "arra____"
+ // ASCII-7-bit=144 CP1256=190 [top CP1256]
+ {{0x61,0x73,0x63,0x69,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x72,0x21,0x71,0xa1,0x53,0x00,0x00,0x00,0x00,0x00,}}, // "asci____"
+ // ASCII-7-bit=190 Latin1=114 CP1252=113 ISO-8859-15=83 [top ASCII-7-bit]
+ {{0x61,0x75,0x74,0x6f,0x5f,0x5f,0x5f,0x5f, 0x01,0x9b,0x51,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "auto____"
+ // ASCII-7-bit=155 SJS=189 [top SJS]
+ {{0x62,0x67,0x5f,0x5f,0x32,0x33,0x31,0x32, 0x01,0x93,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bg__2312"
+ // ASCII-7-bit=147 GB=190 [top GB]
+ {{0x62,0x68,0x61,0x73,0x5f,0x5f,0x5f,0x5f, 0x30,0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bhas____"
+ // BHASKAR=191 [top BHASKAR]
+ {{0x62,0x69,0x67,0x5f,0x35,0x5f,0x5f,0x5f, 0x01,0x84,0x71,0xbe,0x10,0xa1,0x2f,0x00,0x00,0x00,0x00,0x00,}}, // "big_5___"
+ // ASCII-7-bit=132 BIG5=190 BIG5_HKSCS=47 [top BIG5]
+ {{0x62,0x69,0x67,0x5f,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "big_8591"
+ // Latin1=191 [top Latin1]
+ {{0x62,0x69,0x67,0x68,0x35,0x5f,0x5f,0x5f, 0x01,0x88,0x71,0xae,0x10,0xa1,0xb8,0x00,0x00,0x00,0x00,0x00,}}, // "bigh5___"
+ // ASCII-7-bit=136 BIG5=174 BIG5_HKSCS=184 [top BIG5_HKSCS]
+ {{0x62,0x69,0x6e,0x61,0x5f,0x5f,0x5f,0x5f, 0x30,0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bina____"
+ // X-BINARYENC=191 [top X-BINARYENC]
+ {{0x62,0x6f,0x74,0x5f,0x5f,0x5f,0x5f,0x5f, 0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bot_____"
+ // Latin5=191 [top Latin5]
+ {{0x62,0x73,0x5f,0x5f,0x34,0x37,0x33,0x30, 0x02,0xb8,0xa8,0x21,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "bs__4730"
+ // ASCII-7-bit=184 Latin1=168 CP1252=163 [top ASCII-7-bit]
+ {{0x63,0x68,0x61,0x72,0x5f,0x5f,0x5f,0x5f, 0x02,0xa5,0xbb,0x21,0x91,0xa1,0x28,0x00,0x00,0x00,0x00,0x00,}}, // "char____"
+ // ASCII-7-bit=165 Latin1=187 CP1252=145 ISO-8859-15=40 [top Latin1]
+ {{0x63,0x6e,0x73,0x5f,0x5f,0x5f,0x5f,0x5f, 0x30,0x71,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cns_____"
+ // CNS=191 [top CNS]
+ {{0x63,0x6f,0x6e,0x66,0x5f,0x5f,0x5f,0x5f, 0x01,0x9f,0x11,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "conf____"
+ // ASCII-7-bit=159 UTF8=189 [top UTF8]
+ {{0x63,0x6f,0x6e,0x74,0x5f,0x5f,0x5f,0x5f, 0x01,0xa4,0x11,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cont____"
+ // ASCII-7-bit=164 UTF8=188 [top UTF8]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x30, 0x01,0x97,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1250"
+ // ASCII-7-bit=151 CP1250=190 [top CP1250]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x31, 0x01,0x7c,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1251"
+ // ASCII-7-bit=124 CP1251=190 [top CP1251]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x32, 0x02,0xab,0xa9,0x21,0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1252"
+ // ASCII-7-bit=171 Latin1=169 CP1252=181 [top CP1252]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x33, 0x01,0x79,0x10,0x31,0x7f,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1253"
+ // ASCII-7-bit=121 Greek=127 CP1253=190 [top CP1253]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x34, 0x01,0x5b,0xc1,0xaf,0x81,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1254"
+ // ASCII-7-bit=91 Latin5=175 CP1254=185 [top CP1254]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x35, 0x01,0x86,0x10,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1255"
+ // ASCII-7-bit=134 CP1255=190 [top CP1255]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x36, 0x01,0x5e,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1256"
+ // ASCII-7-bit=94 CP1256=190 [top CP1256]
+ {{0x63,0x70,0x5f,0x5f,0x31,0x32,0x35,0x37, 0x01,0xa8,0xf1,0xbb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__1257"
+ // ASCII-7-bit=168 CP1257=187 [top CP1257]
+ {{0x63,0x70,0x5f,0x5f,0x38,0x35,0x30,0x5f, 0x02,0x97,0x98,0x21,0x8c,0xa1,0xbc,0x00,0x00,0x00,0x00,0x00,}}, // "cp__850_"
+ // ASCII-7-bit=151 Latin1=152 CP1252=140 ISO-8859-15=188 [top ISO-8859-15]
+ {{0x63,0x70,0x5f,0x5f,0x38,0x35,0x32,0x5f, 0x01,0x8f,0x20,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__852_"
+ // ASCII-7-bit=143 CP852=190 [top CP852]
+ {{0x63,0x70,0x5f,0x5f,0x38,0x36,0x36,0x5f, 0x01,0xa2,0x20,0x31,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cp__866_"
+ // ASCII-7-bit=162 CP866=188 [top CP866]
+ {{0x63,0x70,0x63,0x5f,0x39,0x34,0x33,0x5f, 0x01,0x26,0x51,0xbe,0x10,0x11,0x68,0x00,0x00,0x00,0x00,0x00,}}, // "cpc_943_"
+ // ASCII-7-bit=38 SJS=190 CP932=104 [top SJS]
+ {{0x63,0x70,0x63,0x7a,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cpcz1250"
+ // CP1250=191 [top CP1250]
+ {{0x63,0x73,0x69,0x73,0x5f,0x5f,0x5f,0x5f, 0x01,0x9c,0x10,0x01,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "csis____"
+ // ASCII-7-bit=156 CP1255=189 [top CP1255]
+ {{0x63,0x73,0x6e,0x5f,0x39,0x31,0x30,0x33, 0x20,0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "csn_9103"
+ // CSN_369103=191 [top CSN_369103]
+ {{0x63,0x73,0x73,0x68,0x5f,0x5f,0x5f,0x5f, 0x01,0x7f,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cssh____"
+ // ASCII-7-bit=127 SJS=190 [top SJS]
+ {{0x63,0x73,0x77,0x69,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cswi1250"
+ // CP1250=191 [top CP1250]
+ {{0x63,0x73,0x77,0x69,0x33,0x31,0x5f,0x5f, 0x61,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "cswi31__"
+ // SJS=191 [top SJS]
+ {{0x63,0x7a,0x77,0x69,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "czwi1250"
+ // CP1250=191 [top CP1250]
+ {{0x64,0x61,0x64,0x6b,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dadk8591"
+ // Latin1=190 CP1252=125 [top Latin1]
+ {{0x64,0x61,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dais8591"
+ // ASCII-7-bit=111 Latin1=190 CP1252=111 [top Latin1]
+ {{0x64,0x65,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0x9d,0xbc,0x21,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "de______"
+ // ASCII-7-bit=157 Latin1=188 CP1252=149 [top Latin1]
+ {{0x64,0x65,0x61,0x73,0x5f,0x5f,0x5f,0x5f, 0x02,0x8f,0xbd,0x21,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "deas____"
+ // ASCII-7-bit=143 Latin1=189 CP1252=146 [top Latin1]
+ {{0x64,0x65,0x64,0x65,0x38,0x35,0x39,0x31, 0x02,0x92,0xbe,0x21,0x87,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dede8591"
+ // ASCII-7-bit=146 Latin1=190 CP1252=135 [top Latin1]
+ {{0x64,0x65,0x66,0x61,0x5f,0x5f,0x5f,0x5f, 0x02,0xbc,0x9f,0x21,0x89,0xa1,0x6b,0x00,0x00,0x00,0x00,0x00,}}, // "defa____"
+ // ASCII-7-bit=188 Latin1=159 CP1252=137 ISO-8859-15=107 [top ASCII-7-bit]
+ {{0x64,0x65,0x69,0x73,0x35,0x39,0x31,0x35, 0x11,0x83,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "deis5915"
+ // Latin1=131 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x64,0x65,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x92,0xbd,0x21,0x89,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "deis8591"
+ // ASCII-7-bit=146 Latin1=189 CP1252=137 [top Latin1]
+ {{0x64,0x65,0x6c,0x65,0x5f,0x5f,0x5f,0x5f, 0x02,0xa9,0xba,0x21,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "dele____"
+ // ASCII-7-bit=169 Latin1=186 CP1252=146 [top Latin1]
+ {{0x64,0x65,0x75,0x74,0x5f,0x5f,0x5f,0x5f, 0x02,0x74,0xb8,0x21,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "deut____"
+ // ASCII-7-bit=116 Latin1=184 CP1252=175 [top Latin1]
+ {{0x64,0x6f,0x6f,0x72,0x31,0x32,0x35,0x32, 0x11,0x79,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "door1252"
+ // Latin1=121 CP1252=190 [top CP1252]
+ {{0x65,0x63,0x75,0x6a,0x5f,0x5f,0x5f,0x5f, 0x71,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ecuj____"
+ // EUC-JP=191 [top EUC-JP]
+ {{0x65,0x63,0x75,0x6b,0x5f,0x5f,0x5f,0x5f, 0x01,0x71,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ecuk____"
+ // ASCII-7-bit=113 KSC=190 [top KSC]
+ {{0x65,0x65,0x6d,0x73,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eems1250"
+ // CP1250=191 [top CP1250]
+ {{0x65,0x6e,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "en__8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x65,0x6e,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x92,0x21,0x82,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "en______"
+ // ASCII-7-bit=190 Latin1=146 CP1252=130 [top ASCII-7-bit]
+ {{0x65,0x6e,0x63,0x6f,0x5f,0x5f,0x5f,0x5f, 0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enco____"
+ // ASCII-7-bit=191 [top ASCII-7-bit]
+ {{0x65,0x6e,0x67,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x8b,0x71,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eng_____"
+ // ASCII-7-bit=139 BIG5=190 [top BIG5]
+ {{0x65,0x6e,0x67,0x62,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "engb____"
+ // ASCII-7-bit=190 Latin1=125 [top ASCII-7-bit]
+ {{0x65,0x6e,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x96,0xbc,0x21,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enis8591"
+ // ASCII-7-bit=150 Latin1=188 CP1252=154 [top Latin1]
+ {{0x65,0x6e,0x75,0x6b,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enuk8591"
+ // Latin1=191 [top Latin1]
+ {{0x65,0x6e,0x75,0x6b,0x5f,0x5f,0x5f,0x5f, 0x51,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enuk____"
+ // KSC=191 [top KSC]
+ {{0x65,0x6e,0x75,0x73,0x35,0x39,0x31,0x35, 0x02,0x6f,0x7f,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enus5915"
+ // ASCII-7-bit=111 Latin1=127 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x65,0x6e,0x75,0x73,0x38,0x35,0x39,0x31, 0x02,0x9c,0xbc,0x21,0x9b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enus8591"
+ // ASCII-7-bit=156 Latin1=188 CP1252=155 [top Latin1]
+ {{0x65,0x6e,0x75,0x73,0x5f,0x5f,0x5f,0x5f, 0x02,0xbb,0xa1,0x21,0x9e,0xa1,0x68,0x00,0x00,0x00,0x00,0x00,}}, // "enus____"
+ // ASCII-7-bit=187 Latin1=161 CP1252=158 ISO-8859-15=104 [top ASCII-7-bit]
+ {{0x65,0x6e,0x75,0x74,0x38,0x5f,0x5f,0x5f, 0x01,0x81,0xf1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "enut8___"
+ // ASCII-7-bit=129 CP1257=190 [top CP1257]
+ {{0x65,0x73,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xb4,0xb3,0x21,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "es______"
+ // ASCII-7-bit=180 Latin1=179 CP1252=157 [top ASCII-7-bit]
+ {{0x65,0x73,0x65,0x73,0x38,0x35,0x39,0x31, 0x02,0x82,0xbe,0x21,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eses8591"
+ // ASCII-7-bit=130 Latin1=190 CP1252=110 [top Latin1]
+ {{0x65,0x73,0x65,0x73,0x5f,0x5f,0x5f,0x5f, 0x02,0xa6,0xba,0x21,0x96,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eses____"
+ // ASCII-7-bit=166 Latin1=186 CP1252=150 [top Latin1]
+ {{0x65,0x73,0x69,0x73,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x87,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "esis8591"
+ // Latin1=190 CP1252=135 [top Latin1]
+ {{0x65,0x74,0x65,0x65,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "etee8591"
+ // Latin1=191 [top Latin1]
+ {{0x65,0x74,0x69,0x73,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "etis8591"
+ // Latin1=191 [top Latin1]
+ {{0x65,0x75,0x63,0x5f,0x32,0x5f,0x5f,0x5f, 0x01,0xbe,0x31,0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "euc_2___"
+ // ASCII-7-bit=190 CP1252=114 [top ASCII-7-bit]
+ {{0x65,0x75,0x63,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x7d,0x61,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "euc_____"
+ // ASCII-7-bit=125 EUC-JP=190 [top EUC-JP]
+ {{0x65,0x75,0x63,0x63,0x5f,0x5f,0x5f,0x5f, 0x01,0x6f,0x30,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eucc____"
+ // ASCII-7-bit=111 EUC-CN=190 [top EUC-CN]
+ {{0x65,0x75,0x63,0x64,0x5f,0x5f,0x5f,0x5f, 0x30,0x61,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eucd____"
+ // EUC=191 [top EUC]
+ {{0x65,0x75,0x63,0x6a,0x5f,0x5f,0x5f,0x5f, 0x01,0x68,0x61,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eucj____"
+ // ASCII-7-bit=104 EUC-JP=190 [top EUC-JP]
+ {{0x65,0x75,0x63,0x6b,0x5f,0x5f,0x5f,0x5f, 0x01,0x6d,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "euck____"
+ // ASCII-7-bit=109 KSC=190 [top KSC]
+ {{0x65,0x75,0x63,0x75,0x5f,0x5f,0x5f,0x5f, 0x01,0x6d,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eucu____"
+ // ASCII-7-bit=109 KSC=190 [top KSC]
+ {{0x65,0x75,0x6b,0x6b,0x5f,0x5f,0x5f,0x5f, 0x51,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eukk____"
+ // KSC=191 [top KSC]
+ {{0x65,0x75,0x72,0x6b,0x5f,0x5f,0x5f,0x5f, 0x01,0x71,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "eurk____"
+ // ASCII-7-bit=113 KSC=190 [top KSC]
+ {{0x66,0x65,0x61,0x74,0x5f,0x5f,0x5f,0x5f, 0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "feat____"
+ // CP1252=191 [top CP1252]
+ {{0x66,0x66,0x5f,0x5f,0x30,0x5f,0x5f,0x5f, 0x02,0x9e,0xba,0x21,0xa5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ff__0___"
+ // ASCII-7-bit=158 Latin1=186 CP1252=165 [top Latin1]
+ {{0x66,0x69,0x66,0x69,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fifi8591"
+ // Latin1=191 [top Latin1]
+ {{0x66,0x72,0x66,0x72,0x38,0x35,0x39,0x31, 0x02,0x79,0xbc,0x21,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "frfr8591"
+ // ASCII-7-bit=121 Latin1=188 CP1252=163 [top Latin1]
+ {{0x66,0x72,0x66,0x72,0x38,0x5f,0x5f,0x5f, 0x02,0xa6,0xad,0x21,0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "frfr8___"
+ // ASCII-7-bit=166 Latin1=173 CP1252=181 [top CP1252]
+ {{0x66,0x72,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x80,0xbd,0x21,0x9e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "fris8591"
+ // ASCII-7-bit=128 Latin1=189 CP1252=158 [top Latin1]
+ {{0x66,0x72,0x75,0x74,0x38,0x5f,0x5f,0x5f, 0x02,0x8c,0xb3,0x21,0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "frut8___"
+ // ASCII-7-bit=140 Latin1=179 CP1252=181 [top CP1252]
+ {{0x67,0x62,0x5f,0x5f,0x31,0x32,0x35,0x31, 0x01,0x6f,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb__1251"
+ // ASCII-7-bit=111 CP1251=190 [top CP1251]
+ {{0x67,0x62,0x5f,0x5f,0x32,0x31,0x33,0x32, 0x01,0x91,0x21,0xbe,0xf1,0x70,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb__2132"
+ // ASCII-7-bit=145 GB=190 GBK=112 [top GB]
+ {{0x67,0x62,0x5f,0x5f,0x32,0x33,0x31,0x32, 0x01,0x7a,0x21,0xbe,0xf1,0x5c,0xc1,0x37,0x00,0x00,0x00,0x00,}}, // "gb__2312"
+ // ASCII-7-bit=122 GB=190 GBK=92 GB18030=55 [top GB]
+ {{0x67,0x62,0x5f,0x5f,0x32,0x33,0x32,0x31, 0x01,0x7d,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb__2321"
+ // ASCII-7-bit=125 GB=190 [top GB]
+ {{0x67,0x62,0x5f,0x5f,0x33,0x32,0x31,0x32, 0x01,0x92,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb__3212"
+ // ASCII-7-bit=146 GB=190 [top GB]
+ {{0x67,0x62,0x5f,0x5f,0x38,0x30,0x33,0x30, 0x01,0x73,0x21,0xaf,0xf1,0x59,0xc1,0xb9,0x00,0x00,0x00,0x00,}}, // "gb__8030"
+ // ASCII-7-bit=115 GB=175 GBK=89 GB18030=185 [top GB18030]
+ {{0x67,0x62,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x7f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb__8591"
+ // ASCII-7-bit=127 Latin1=190 [top Latin1]
+ {{0x67,0x62,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x71,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gb______"
+ // ASCII-7-bit=113 GB=190 [top GB]
+ {{0x67,0x62,0x6b,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x76,0x21,0xaf,0xf1,0xb9,0xc1,0x13,0x00,0x00,0x00,0x00,}}, // "gbk_____"
+ // ASCII-7-bit=118 GB=175 GBK=185 GB18030=19 [top GBK]
+ {{0x67,0x64,0x5f,0x5f,0x32,0x33,0x31,0x32, 0x01,0x56,0x21,0xbe,0xf1,0x72,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gd__2312"
+ // ASCII-7-bit=86 GB=190 GBK=114 [top GB]
+ {{0x67,0x65,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "geis8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x67,0x65,0x6e,0x65,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "gene1251"
+ // CP1251=191 [top CP1251]
+ {{0x67,0x69,0x73,0x6f,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "giso8591"
+ // Latin1=190 CP1252=111 [top Latin1]
+ {{0x67,0x72,0x65,0x65,0x5f,0x5f,0x5f,0x5f, 0x01,0x90,0x10,0x31,0xbe,0x21,0x86,0x00,0x00,0x00,0x00,0x00,}}, // "gree____"
+ // ASCII-7-bit=144 Greek=190 CP1253=134 [top Greek]
+ {{0x68,0x72,0x77,0x69,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "hrwi1250"
+ // CP1250=191 [top CP1250]
+ {{0x68,0x74,0x63,0x68,0x5f,0x5f,0x5f,0x5f, 0x30,0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "htch____"
+ // HTCHANAKYA=191 [top HTCHANAKYA]
+ {{0x68,0x74,0x6d,0x6c,0x5f,0x5f,0x5f,0x5f, 0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "html____"
+ // ASCII-7-bit=191 [top ASCII-7-bit]
+ {{0x68,0x74,0x74,0x70,0x5f,0x5f,0x5f,0x5f, 0x02,0xbb,0xa4,0x21,0x8d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "http____"
+ // ASCII-7-bit=187 Latin1=164 CP1252=141 [top ASCII-7-bit]
+ {{0x68,0x7a,0x67,0x62,0x32,0x33,0x31,0x32, 0x01,0x85,0x20,0x71,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "hzgb2312"
+ // ASCII-7-bit=133 HZ-GB-2312=190 [top HZ-GB-2312]
+ {{0x69,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "i___8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x61,0x6e,0x6f,0x35,0x5f,0x5f,0x5f, 0x02,0xbe,0x61,0x21,0x54,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iano5___"
+ // ASCII-7-bit=190 Latin1=97 CP1252=84 [top ASCII-7-bit]
+ {{0x69,0x62,0x6d,0x5f,0x38,0x35,0x32,0x5f, 0x01,0xac,0x20,0x01,0xba,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ibm_852_"
+ // ASCII-7-bit=172 CP852=186 [top CP852]
+ {{0x69,0x62,0x6d,0x5f,0x38,0x36,0x36,0x5f, 0x01,0x84,0x20,0x31,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ibm_866_"
+ // ASCII-7-bit=132 CP866=190 [top CP866]
+ {{0x69,0x62,0x6d,0x5f,0x39,0x34,0x32,0x5f, 0x61,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ibm_942_"
+ // SJS=191 [top SJS]
+ {{0x69,0x63,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbb,0x21,0xa9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ico_8591"
+ // ASCII-7-bit=121 Latin1=187 CP1252=169 [top Latin1]
+ {{0x69,0x6e,0x64,0x6f,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "indo1251"
+ // CP1251=191 [top CP1251]
+ {{0x69,0x6e,0x73,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "inso8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x6f,0x73,0x5f,0x38,0x35,0x39,0x31, 0x02,0x97,0xbd,0x21,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ios_8591"
+ // ASCII-7-bit=151 Latin1=189 CP1252=110 [top Latin1]
+ {{0x69,0x6f,0x73,0x6f,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ioso8591"
+ // Latin1=190 CP1252=121 [top Latin1]
+ {{0x69,0x73,0x5f,0x5f,0x35,0x39,0x31,0x35, 0x11,0x7f,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__5915"
+ // Latin1=127 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x69,0x73,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0xad,0xb7,0x21,0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__8591"
+ // ASCII-7-bit=173 Latin1=183 CP1252=159 [top Latin1]
+ {{0x69,0x73,0x5f,0x5f,0x38,0x35,0x39,0x32, 0x01,0x78,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__8592"
+ // ASCII-7-bit=120 Latin2=190 [top Latin2]
+ {{0x69,0x73,0x5f,0x5f,0x38,0x35,0x39,0x37, 0x10,0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__8597"
+ // Greek=191 [top Greek]
+ {{0x69,0x73,0x5f,0x5f,0x38,0x35,0x39,0x38, 0x01,0x6f,0x10,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__8598"
+ // ASCII-7-bit=111 Hebrew=190 [top Hebrew]
+ {{0x69,0x73,0x5f,0x5f,0x38,0x35,0x39,0x39, 0xd1,0xbe,0x81,0x88,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "is__8599"
+ // Latin5=190 CP1254=136 [top Latin5]
+ {{0x69,0x73,0x61,0x5f,0x35,0x39,0x31,0x35, 0x02,0x86,0x89,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isa_5915"
+ // ASCII-7-bit=134 Latin1=137 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x69,0x73,0x64,0x5f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isd_8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x64,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isdo8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6e,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isn_8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x30,0x36,0x34,0x36, 0x02,0xb8,0xaa,0x21,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_0646"
+ // ASCII-7-bit=184 Latin1=170 CP1252=163 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x30,0x34,0x30, 0x02,0x98,0xb2,0x21,0xb4,0xa1,0x5e,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1040"
+ // ASCII-7-bit=152 Latin1=178 CP1252=180 ISO-8859-15=94 [top CP1252]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x30, 0x01,0x90,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1250"
+ // ASCII-7-bit=144 CP1250=190 [top CP1250]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x31, 0x01,0x78,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1251"
+ // ASCII-7-bit=120 CP1251=190 [top CP1251]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x32, 0x02,0xad,0x9e,0x21,0xb7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1252"
+ // ASCII-7-bit=173 Latin1=158 CP1252=183 [top CP1252]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x33, 0x10,0x41,0x83,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1253"
+ // Greek=131 CP1253=190 [top CP1253]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x34, 0xd1,0x9b,0x81,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1254"
+ // Latin5=155 CP1254=189 [top CP1254]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x35, 0x01,0x79,0x10,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1255"
+ // ASCII-7-bit=121 CP1255=190 [top CP1255]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x36, 0x01,0x6f,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1256"
+ // ASCII-7-bit=111 CP1256=190 [top CP1256]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x32,0x35,0x37, 0x01,0x7f,0xf1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1257"
+ // ASCII-7-bit=127 CP1257=190 [top CP1257]
+ {{0x69,0x73,0x6f,0x5f,0x31,0x5f,0x5f,0x5f, 0x02,0x85,0xb5,0x21,0xb3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_1___"
+ // ASCII-7-bit=133 Latin1=181 CP1252=179 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x32,0x30,0x32,0x32, 0x01,0x97,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_2022"
+ // ASCII-7-bit=151 SJS=190 [top SJS]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x35,0x39,0x31, 0x02,0xa9,0xb8,0x21,0xa4,0xa1,0x3c,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5591"
+ // ASCII-7-bit=169 Latin1=184 CP1252=164 ISO-8859-15=60 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x35,0x39,0x32, 0x02,0x9a,0xbd,0x21,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5592"
+ // ASCII-7-bit=154 Latin1=189 CP1252=146 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x38,0x39,0x31, 0x02,0xa1,0xbc,0x21,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5891"
+ // ASCII-7-bit=161 Latin1=188 CP1252=139 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x30, 0x01,0xaa,0x20,0xa1,0xba,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5910"
+ // ASCII-7-bit=170 Latin6=186 [top Latin6]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x31, 0x01,0x86,0xd1,0xbe,0xd1,0x66,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5911"
+ // ASCII-7-bit=134 ISO-8859-11=190 CP874=102 [top ISO-8859-11]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x33, 0x01,0x9c,0xf1,0xa1,0xc1,0xbb,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5913"
+ // ASCII-7-bit=156 CP1257=161 ISO-8859-13=187 [top ISO-8859-13]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x34, 0x02,0x93,0xbd,0x21,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5914"
+ // ASCII-7-bit=147 Latin1=189 CP1252=149 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x35, 0x02,0x98,0xad,0x21,0x81,0xa1,0xb7,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5915"
+ // ASCII-7-bit=152 Latin1=173 CP1252=129 ISO-8859-15=183 [top ISO-8859-15]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x31,0x36, 0x01,0xae,0xb1,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5916"
+ // ASCII-7-bit=174 CP1250=185 [top CP1250]
+ {{0x69,0x73,0x6f,0x5f,0x35,0x39,0x32,0x32, 0x01,0xa7,0x81,0xbb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_5922"
+ // ASCII-7-bit=167 Latin2=187 [top Latin2]
+ {{0x69,0x73,0x6f,0x5f,0x36,0x33,0x39,0x32, 0x02,0x7e,0xbe,0x21,0x82,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_6392"
+ // ASCII-7-bit=126 Latin1=190 CP1252=130 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x36,0x33,0x39,0x5f, 0x01,0xa6,0xa1,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_639_"
+ // ASCII-7-bit=166 CP1256=188 [top CP1256]
+ {{0x69,0x73,0x6f,0x5f,0x36,0x34,0x36,0x31, 0x02,0x7d,0xbe,0x21,0x68,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_6461"
+ // ASCII-7-bit=125 Latin1=190 CP1252=104 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x31,0x31, 0x02,0xb0,0xb7,0x21,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8511"
+ // ASCII-7-bit=176 Latin1=183 CP1252=146 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x36,0x31, 0x02,0x9f,0xba,0x21,0xa5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8561"
+ // ASCII-7-bit=159 Latin1=186 CP1252=165 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x38,0x31, 0x01,0x8d,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8581"
+ // ASCII-7-bit=141 SJS=190 [top SJS]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x30, 0x02,0x99,0xbc,0x21,0x9c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8590"
+ // ASCII-7-bit=153 Latin1=188 CP1252=156 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0xae,0xb8,0x21,0x99,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8591"
+ // ASCII-7-bit=174 Latin1=184 CP1252=153 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x32, 0x01,0x95,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8592"
+ // ASCII-7-bit=149 Latin2=190 [top Latin2]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x33, 0x01,0x9f,0x20,0x51,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8593"
+ // ASCII-7-bit=159 Latin3=189 [top Latin3]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x34, 0x01,0xac,0x10,0xd1,0xba,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8594"
+ // ASCII-7-bit=172 Latin4=186 [top Latin4]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x35, 0x01,0xa6,0x10,0xa1,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8595"
+ // ASCII-7-bit=166 ISO-8859-5=188 [top ISO-8859-5]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x36, 0x01,0xae,0x20,0x11,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8596"
+ // ASCII-7-bit=174 Arabic=185 [top Arabic]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x37, 0x01,0x96,0x10,0x31,0xbd,0x21,0x8f,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8597"
+ // ASCII-7-bit=150 Greek=189 CP1253=143 [top Greek]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x38, 0x01,0x9b,0x10,0x81,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8598"
+ // ASCII-7-bit=155 Hebrew=189 [top Hebrew]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x35,0x39,0x39, 0x01,0x7a,0xc1,0xbe,0x81,0x7e,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8599"
+ // ASCII-7-bit=122 Latin5=190 CP1254=126 [top Latin5]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x36,0x30,0x31, 0x02,0xba,0x94,0x21,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8601"
+ // ASCII-7-bit=186 Latin1=148 CP1252=166 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x36,0x39,0x31, 0x02,0xad,0xb9,0x21,0x83,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8691"
+ // ASCII-7-bit=173 Latin1=185 CP1252=131 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x36,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8692"
+ // Latin2=191 [top Latin2]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x38,0x35,0x31, 0x02,0xac,0xb7,0x21,0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8851"
+ // ASCII-7-bit=172 Latin1=183 CP1252=159 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x38,0x35,0x39, 0x02,0xaa,0xba,0x21,0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8859"
+ // ASCII-7-bit=170 Latin1=186 CP1252=128 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x38,0x39,0x39, 0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8899"
+ // Latin5=191 [top Latin5]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x39,0x31,0x31, 0x02,0x8c,0xbd,0x21,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8911"
+ // ASCII-7-bit=140 Latin1=189 CP1252=154 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x39,0x31,0x5f, 0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_891_"
+ // ASCII-7-bit=191 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x5f,0x38,0x39,0x35,0x31, 0x02,0xa3,0xbc,0x21,0x91,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_8951"
+ // ASCII-7-bit=163 Latin1=188 CP1252=145 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x39,0x30,0x30,0x31, 0x02,0x75,0xa7,0x21,0x85,0xa1,0xbb,0x00,0x00,0x00,0x00,0x00,}}, // "iso_9001"
+ // ASCII-7-bit=117 Latin1=167 CP1252=133 ISO-8859-15=187 [top ISO-8859-15]
+ {{0x69,0x73,0x6f,0x5f,0x39,0x35,0x35,0x31, 0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_9551"
+ // ASCII-7-bit=191 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x5f,0x39,0x35,0x39,0x31, 0x02,0x72,0xbe,0x21,0x7b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_9591"
+ // ASCII-7-bit=114 Latin1=190 CP1252=123 [top Latin1]
+ {{0x69,0x73,0x6f,0x5f,0x39,0x35,0x39,0x32, 0x01,0x7f,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_9592"
+ // ASCII-7-bit=127 Latin2=190 [top Latin2]
+ {{0x69,0x73,0x6f,0x5f,0x39,0x35,0x39,0x39, 0x01,0x84,0xc1,0xbb,0x81,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iso_9599"
+ // ASCII-7-bit=132 Latin5=187 CP1254=166 [top Latin5]
+ {{0x69,0x73,0x6f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0x99,0xbc,0x21,0x96,0xa1,0x2e,0x00,0x00,0x00,0x00,0x00,}}, // "iso_____"
+ // ASCII-7-bit=153 Latin1=188 CP1252=150 ISO-8859-15=46 [top Latin1]
+ {{0x69,0x73,0x6f,0x61,0x38,0x35,0x39,0x31, 0x02,0x9a,0xbd,0x21,0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoa8591"
+ // ASCII-7-bit=154 Latin1=189 CP1252=138 [top Latin1]
+ {{0x69,0x73,0x6f,0x62,0x38,0x35,0x39,0x31, 0x02,0x86,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isob8591"
+ // ASCII-7-bit=134 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x63,0x32,0x30,0x32,0x32, 0x20,0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoc2022"
+ // ISO_2022_CN=191 [top ISO_2022_CN]
+ {{0x69,0x73,0x6f,0x63,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoc8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x63,0x38,0x35,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoc8592"
+ // Latin2=191 [top Latin2]
+ {{0x69,0x73,0x6f,0x64,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isod8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x65,0x38,0x35,0x39,0x31, 0x02,0x93,0xbd,0x21,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoe8591"
+ // ASCII-7-bit=147 Latin1=189 CP1252=139 [top Latin1]
+ {{0x69,0x73,0x6f,0x66,0x35,0x39,0x31,0x35, 0x11,0x6f,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isof5915"
+ // Latin1=111 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x69,0x73,0x6f,0x68,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoh8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x69,0x36,0x5f,0x5f,0x5f, 0x02,0xbe,0x92,0x21,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi6___"
+ // ASCII-7-bit=190 Latin1=146 CP1252=127 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x31, 0x02,0xa2,0xbc,0x21,0x8c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8591"
+ // ASCII-7-bit=162 Latin1=188 CP1252=140 [top Latin1]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8592"
+ // Latin2=191 [top Latin2]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x35, 0x01,0xa4,0x10,0xa1,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8595"
+ // ASCII-7-bit=164 ISO-8859-5=188 [top ISO-8859-5]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x36, 0x01,0x79,0x20,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8596"
+ // ASCII-7-bit=121 Arabic=190 [top Arabic]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x38, 0x01,0x83,0x10,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8598"
+ // ASCII-7-bit=131 CP1255=190 [top CP1255]
+ {{0x69,0x73,0x6f,0x69,0x38,0x35,0x39,0x39, 0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8599"
+ // Latin5=191 [top Latin5]
+ {{0x69,0x73,0x6f,0x69,0x38,0x38,0x35,0x39, 0x02,0xae,0xb7,0x21,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi8859"
+ // ASCII-7-bit=174 Latin1=183 CP1252=154 [top Latin1]
+ {{0x69,0x73,0x6f,0x69,0x38,0x39,0x5f,0x5f, 0xb1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoi89__"
+ // CP1256=191 [top CP1256]
+ {{0x69,0x73,0x6f,0x6a,0x32,0x30,0x30,0x32, 0x71,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoj2002"
+ // EUC-JP=191 [top EUC-JP]
+ {{0x69,0x73,0x6f,0x6a,0x32,0x30,0x32,0x32, 0x01,0x44,0x10,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoj2022"
+ // ASCII-7-bit=68 JIS=190 [top JIS]
+ {{0x69,0x73,0x6f,0x6a,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoj8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x6b,0x32,0x30,0x30,0x32, 0x01,0x7a,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isok2002"
+ // ASCII-7-bit=122 KSC=190 [top KSC]
+ {{0x69,0x73,0x6f,0x6b,0x32,0x30,0x32,0x32, 0x20,0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isok2022"
+ // ISO-2022-KR=191 [top ISO-2022-KR]
+ {{0x69,0x73,0x6f,0x6c,0x31,0x5f,0x5f,0x5f, 0x01,0xa6,0x11,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isol1___"
+ // ASCII-7-bit=166 UTF8=188 [top UTF8]
+ {{0x69,0x73,0x6f,0x6c,0x35,0x39,0x31,0x31, 0x01,0x83,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isol5911"
+ // ASCII-7-bit=131 ISO-8859-11=190 [top ISO-8859-11]
+ {{0x69,0x73,0x6f,0x6c,0x37,0x5f,0x5f,0x5f, 0x02,0xa4,0xb8,0x21,0xa7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isol7___"
+ // ASCII-7-bit=164 Latin1=184 CP1252=167 [top Latin1]
+ {{0x69,0x73,0x6f,0x6c,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isol8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x6d,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isom8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x6e,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ison8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6e,0xbe,0x21,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoo8591"
+ // ASCII-7-bit=110 Latin1=190 CP1252=110 [top Latin1]
+ {{0x69,0x73,0x6f,0x70,0x35,0x39,0x31,0x35, 0xf1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isop5915"
+ // ISO-8859-15=191 [top ISO-8859-15]
+ {{0x69,0x73,0x6f,0x70,0x38,0x35,0x39,0x31, 0x02,0x91,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isop8591"
+ // ASCII-7-bit=145 Latin1=190 CP1252=111 [top Latin1]
+ {{0x69,0x73,0x6f,0x73,0x38,0x35,0x39,0x31, 0x02,0x84,0xbe,0x21,0x8d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isos8591"
+ // ASCII-7-bit=132 Latin1=190 CP1252=141 [top Latin1]
+ {{0x69,0x73,0x6f,0x75,0x36,0x34,0x36,0x31, 0x02,0xa6,0xb9,0x21,0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isou6461"
+ // ASCII-7-bit=166 Latin1=185 CP1252=161 [top Latin1]
+ {{0x69,0x73,0x6f,0x75,0x36,0x34,0x36,0x5f, 0x01,0xbe,0x31,0x8e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isou646_"
+ // ASCII-7-bit=190 CP1252=142 [top ASCII-7-bit]
+ {{0x69,0x73,0x6f,0x75,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isou8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x75,0x38,0x5f,0x5f,0x5f, 0x02,0xa2,0xbc,0x21,0x8c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isou8___"
+ // ASCII-7-bit=162 Latin1=188 CP1252=140 [top Latin1]
+ {{0x69,0x73,0x6f,0x77,0x31,0x32,0x35,0x30, 0x01,0x6e,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isow1250"
+ // ASCII-7-bit=110 CP1250=190 [top CP1250]
+ {{0x69,0x73,0x6f,0x77,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isow1251"
+ // CP1251=191 [top CP1251]
+ {{0x69,0x73,0x6f,0x77,0x31,0x32,0x35,0x33, 0x01,0x6f,0x10,0x61,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isow1253"
+ // ASCII-7-bit=111 CP1253=190 [top CP1253]
+ {{0x69,0x73,0x6f,0x77,0x38,0x35,0x39,0x31, 0x02,0x89,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isow8591"
+ // ASCII-7-bit=137 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x78,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isox8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x6f,0x7a,0x38,0x35,0x39,0x31, 0x02,0x8b,0xbe,0x21,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isoz8591"
+ // ASCII-7-bit=139 Latin1=190 CP1252=121 [top Latin1]
+ {{0x69,0x73,0x70,0x5f,0x38,0x35,0x39,0x31, 0x02,0x86,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isp_8591"
+ // ASCII-7-bit=134 Latin1=190 CP1252=111 [top Latin1]
+ {{0x69,0x73,0x73,0x5f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iss_8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x73,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isso8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x73,0x74,0x5f,0x35,0x39,0x31,0x35, 0x01,0x79,0xe1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ist_5915"
+ // ASCII-7-bit=121 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x69,0x73,0x74,0x6f,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "isto8591"
+ // Latin1=190 CP1252=111 [top Latin1]
+ {{0x69,0x74,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x21,0x86,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "itis8591"
+ // ASCII-7-bit=111 Latin1=190 CP1252=134 [top Latin1]
+ {{0x69,0x74,0x69,0x74,0x35,0x39,0x31,0x35, 0x41,0x79,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "itit5915"
+ // CP1252=121 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x69,0x74,0x69,0x74,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x8f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "itit8591"
+ // Latin1=190 CP1252=143 [top Latin1]
+ {{0x69,0x74,0x69,0x74,0x5f,0x5f,0x5f,0x5f, 0x02,0xb7,0xab,0x21,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "itit____"
+ // ASCII-7-bit=183 Latin1=171 CP1252=164 [top ASCII-7-bit]
+ {{0x69,0x75,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iu__8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x69,0x77,0x69,0x6e,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iwin1250"
+ // CP1250=191 [top CP1250]
+ {{0x69,0x77,0x69,0x6e,0x31,0x32,0x35,0x37, 0x10,0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iwin1257"
+ // CP1257=191 [top CP1257]
+ {{0x69,0x79,0x73,0x6f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "iyso8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x6a,0x61,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x78,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ja______"
+ // ASCII-7-bit=120 SJS=190 [top SJS]
+ {{0x6a,0x61,0x67,0x72,0x5f,0x5f,0x5f,0x5f, 0x20,0xf1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "jagr____"
+ // JAGRAN=191 [top JAGRAN]
+ {{0x6a,0x69,0x73,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x81,0x10,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "jis_____"
+ // ASCII-7-bit=129 JIS=190 [top JIS]
+ {{0x6b,0x61,0x6d,0x63,0x5f,0x5f,0x5f,0x5f, 0x20,0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kamc____"
+ // CP852=191 [top CP852]
+ {{0x6b,0x6f,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x7c,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ko______"
+ // ASCII-7-bit=124 KSC=190 [top KSC]
+ {{0x6b,0x6f,0x69,0x5f,0x37,0x5f,0x5f,0x5f, 0x01,0xbe,0x31,0x6b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "koi_7___"
+ // ASCII-7-bit=190 CP1252=107 [top ASCII-7-bit]
+ {{0x6b,0x6f,0x69,0x72,0x38,0x5f,0x5f,0x5f, 0x01,0x8b,0x10,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "koir8___"
+ // ASCII-7-bit=139 KOI8R=190 [top KOI8R]
+ {{0x6b,0x6f,0x69,0x75,0x38,0x5f,0x5f,0x5f, 0x01,0x77,0x10,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "koiu8___"
+ // ASCII-7-bit=119 KOI8U=190 [top KOI8U]
+ {{0x6b,0x6f,0x6b,0x72,0x5f,0x5f,0x5f,0x5f, 0x01,0x4b,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kokr____"
+ // ASCII-7-bit=75 KSC=190 [top KSC]
+ {{0x6b,0x6f,0x6b,0x73,0x35,0x36,0x30,0x31, 0x01,0x75,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "koks5601"
+ // ASCII-7-bit=117 KSC=190 [top KSC]
+ {{0x6b,0x6f,0x72,0x65,0x5f,0x5f,0x5f,0x5f, 0x01,0x4e,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kore____"
+ // ASCII-7-bit=78 KSC=190 [top KSC]
+ {{0x6b,0x72,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x74,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "kr______"
+ // ASCII-7-bit=116 KSC=190 [top KSC]
+ {{0x6b,0x72,0x63,0x5f,0x35,0x36,0x30,0x31, 0x01,0x74,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "krc_5601"
+ // ASCII-7-bit=116 KSC=190 [top KSC]
+ {{0x6b,0x73,0x63,0x5f,0x35,0x35,0x30,0x31, 0x51,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ksc_5501"
+ // KSC=191 [top KSC]
+ {{0x6b,0x73,0x63,0x5f,0x35,0x36,0x30,0x31, 0x01,0x62,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ksc_5601"
+ // ASCII-7-bit=98 KSC=190 [top KSC]
+ {{0x6b,0x73,0x63,0x5f,0x36,0x30,0x30,0x31, 0x51,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ksc_6001"
+ // KSC=191 [top KSC]
+ {{0x6c,0x61,0x73,0x74,0x5f,0x5f,0x5f,0x5f, 0x02,0xb7,0xaf,0x21,0x90,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "last____"
+ // ASCII-7-bit=183 Latin1=175 CP1252=144 [top ASCII-7-bit]
+ {{0x6c,0x61,0x74,0x69,0x31,0x5f,0x5f,0x5f, 0x02,0xa3,0xbb,0x21,0x9b,0xa1,0x73,0x00,0x00,0x00,0x00,0x00,}}, // "lati1___"
+ // ASCII-7-bit=163 Latin1=187 CP1252=155 ISO-8859-15=115 [top Latin1]
+ {{0x6c,0x61,0x74,0x69,0x32,0x5f,0x5f,0x5f, 0x01,0x94,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lati2___"
+ // ASCII-7-bit=148 Latin2=190 [top Latin2]
+ {{0x6c,0x61,0x74,0x69,0x35,0x5f,0x5f,0x5f, 0x01,0x7c,0xc1,0xbe,0x81,0x87,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lati5___"
+ // ASCII-7-bit=124 Latin5=190 CP1254=135 [top Latin5]
+ {{0x6c,0x61,0x74,0x69,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lati8591"
+ // Latin1=191 [top Latin1]
+ {{0x6c,0x61,0x74,0x69,0x38,0x38,0x35,0x39, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lati8859"
+ // Latin2=191 [top Latin2]
+ {{0x6c,0x69,0x6e,0x75,0x31,0x32,0x35,0x32, 0x11,0x79,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "linu1252"
+ // Latin1=121 CP1252=190 [top CP1252]
+ {{0x6c,0x6f,0x67,0x69,0x5f,0x5f,0x5f,0x5f, 0x01,0x88,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "logi____"
+ // ASCII-7-bit=136 UTF8=190 [top UTF8]
+ {{0x6c,0x73,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lso_8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x6c,0x74,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "lto_8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x6c,0x74,0x77,0x69,0x31,0x32,0x35,0x37, 0x10,0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ltwi1257"
+ // CP1257=191 [top CP1257]
+ {{0x6d,0x61,0x63,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x82,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mac_____"
+ // ASCII-7-bit=130 CP1251=190 [top CP1251]
+ {{0x6d,0x61,0x63,0x63,0x5f,0x5f,0x5f,0x5f, 0x01,0x94,0x10,0xe1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "macc____"
+ // ASCII-7-bit=148 MACINTOSH=190 [top MACINTOSH]
+ {{0x6d,0x61,0x63,0x69,0x5f,0x5f,0x5f,0x5f, 0x02,0xbd,0x93,0x21,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "maci____"
+ // ASCII-7-bit=189 Latin1=147 CP1252=139 [top ASCII-7-bit]
+ {{0x6d,0x61,0x63,0x72,0x5f,0x5f,0x5f,0x5f, 0x01,0xaf,0x10,0xe1,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "macr____"
+ // ASCII-7-bit=175 MACINTOSH=185 [top MACINTOSH]
+ {{0x6d,0x73,0x5f,0x5f,0x38,0x37,0x34,0x5f, 0x01,0x80,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ms__874_"
+ // ASCII-7-bit=128 ISO-8859-11=190 [top ISO-8859-11]
+ {{0x6d,0x73,0x5f,0x5f,0x39,0x33,0x32,0x5f, 0x01,0x91,0x51,0xbe,0x10,0x11,0x82,0x00,0x00,0x00,0x00,0x00,}}, // "ms__932_"
+ // ASCII-7-bit=145 SJS=190 CP932=130 [top SJS]
+ {{0x6d,0x73,0x5f,0x5f,0x39,0x34,0x39,0x5f, 0x01,0x49,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ms__949_"
+ // ASCII-7-bit=73 KSC=190 [top KSC]
+ {{0x6d,0x73,0x5f,0x5f,0x39,0x35,0x30,0x5f, 0x01,0x75,0x71,0xbe,0x10,0xa1,0x43,0x00,0x00,0x00,0x00,0x00,}}, // "ms__950_"
+ // ASCII-7-bit=117 BIG5=190 BIG5_HKSCS=67 [top BIG5]
+ {{0x6d,0x73,0x63,0x70,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mscp1250"
+ // CP1250=191 [top CP1250]
+ {{0x6d,0x73,0x68,0x6b,0x39,0x35,0x30,0x5f, 0x01,0x82,0x71,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mshk950_"
+ // ASCII-7-bit=130 BIG5=190 [top BIG5]
+ {{0x6d,0x73,0x77,0x69,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mswi1250"
+ // CP1250=191 [top CP1250]
+ {{0x6d,0x73,0x77,0x69,0x31,0x32,0x35,0x33, 0x10,0x41,0x8f,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mswi1253"
+ // Greek=143 CP1253=190 [top CP1253]
+ {{0x6d,0x78,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "mx______"
+ // UTF8=191 [top UTF8]
+ {{0x6e,0x65,0x77,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xab,0xb2,0x21,0xaf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "new_____"
+ // ASCII-7-bit=171 Latin1=178 CP1252=175 [top Latin1]
+ {{0x6e,0x66,0x7a,0x5f,0x32,0x30,0x31,0x30, 0x02,0x80,0xbc,0x21,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nfz_2010"
+ // ASCII-7-bit=128 Latin1=188 CP1252=163 [top Latin1]
+ {{0x6e,0x69,0x73,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "niso8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x6e,0x6c,0x61,0x69,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nlai8591"
+ // Latin1=191 [top Latin1]
+ {{0x6e,0x6c,0x6e,0x6c,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nlnl8591"
+ // Latin1=190 CP1252=111 [top Latin1]
+ {{0x6e,0x6f,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0xa4,0x71,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "no______"
+ // ASCII-7-bit=164 BIG5=188 [top BIG5]
+ {{0x6e,0x6f,0x69,0x73,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "nois8591"
+ // Latin1=191 [top Latin1]
+ {{0x6e,0x6f,0x6e,0x65,0x5f,0x5f,0x5f,0x5f, 0x01,0x9b,0x51,0xbd,0x10,0x11,0x70,0x00,0x00,0x00,0x00,0x00,}}, // "none____"
+ // ASCII-7-bit=155 SJS=189 CP932=112 [top SJS]
+ {{0x6e,0x75,0x6c,0x6c,0x5f,0x5f,0x5f,0x5f, 0x01,0x92,0x71,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "null____"
+ // ASCII-7-bit=146 BIG5=190 [top BIG5]
+ {{0x6f,0x5f,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "o___8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x6f,0x6e,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "on______"
+ // UTF8=191 [top UTF8]
+ {{0x6f,0x73,0x69,0x5f,0x35,0x39,0x31,0x35, 0x01,0x6f,0xe1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "osi_5915"
+ // ASCII-7-bit=111 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x6f,0x73,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "oso_8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x6f,0x73,0x70,0x5f,0x38,0x35,0x39,0x38, 0x01,0x6f,0x10,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "osp_8598"
+ // ASCII-7-bit=111 Hebrew=190 [top Hebrew]
+ {{0x6f,0x77,0x69,0x6e,0x31,0x32,0x35,0x36, 0xb1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "owin1256"
+ // CP1256=191 [top CP1256]
+ {{0x70,0x61,0x72,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0x6e,0xb8,0x21,0xaf,0xa1,0x64,0x00,0x00,0x00,0x00,0x00,}}, // "par_____"
+ // ASCII-7-bit=110 Latin1=184 CP1252=175 ISO-8859-15=100 [top Latin1]
+ {{0x70,0x63,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pc______"
+ // UTF8=191 [top UTF8]
+ {{0x70,0x6c,0x69,0x73,0x38,0x35,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "plis8592"
+ // Latin2=191 [top Latin2]
+ {{0x70,0x6c,0x70,0x6c,0x38,0x35,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "plpl8592"
+ // Latin2=191 [top Latin2]
+ {{0x70,0x72,0x65,0x64,0x5f,0x5f,0x5f,0x5f, 0x02,0xb4,0xa3,0x21,0xb1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "pred____"
+ // ASCII-7-bit=180 Latin1=163 CP1252=177 [top ASCII-7-bit]
+ {{0x70,0x74,0x62,0x72,0x38,0x35,0x39,0x31, 0x02,0x6e,0xbd,0x21,0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ptbr8591"
+ // ASCII-7-bit=110 Latin1=189 CP1252=154 [top Latin1]
+ {{0x70,0x74,0x62,0x72,0x5f,0x5f,0x5f,0x5f, 0x01,0x79,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ptbr____"
+ // ASCII-7-bit=121 UTF8=190 [top UTF8]
+ {{0x70,0x74,0x69,0x73,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x7e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ptis8591"
+ // Latin1=190 CP1252=126 [top Latin1]
+ {{0x70,0x74,0x70,0x74,0x35,0x39,0x31,0x35, 0x11,0x89,0x21,0x6f,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ptpt5915"
+ // Latin1=137 CP1252=111 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x72,0x66,0x63,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x87,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "rfc_____"
+ // ASCII-7-bit=135 UTF8=190 [top UTF8]
+ {{0x72,0x6f,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x83,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "rois8591"
+ // ASCII-7-bit=131 Latin1=190 [top Latin1]
+ {{0x72,0x6f,0x72,0x6f,0x38,0x35,0x39,0x32, 0x01,0x99,0x81,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "roro8592"
+ // ASCII-7-bit=153 Latin2=189 [top Latin2]
+ {{0x72,0x75,0x72,0x75,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ruru1251"
+ // CP1251=191 [top CP1251]
+ {{0x72,0x75,0x77,0x69,0x31,0x32,0x35,0x31, 0x01,0x6f,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ruwi1251"
+ // ASCII-7-bit=111 CP1251=190 [top CP1251]
+ {{0x73,0x65,0x65,0x6d,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "seem8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x73,0x65,0x74,0x63,0x5f,0x5f,0x5f,0x5f, 0x01,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "setc____"
+ // ASCII-7-bit=191 [top ASCII-7-bit]
+ {{0x73,0x68,0x69,0x66,0x31,0x32,0x35,0x32, 0x02,0x86,0x6f,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "shif1252"
+ // ASCII-7-bit=134 Latin1=111 CP1252=190 [top CP1252]
+ {{0x73,0x68,0x69,0x66,0x5f,0x5f,0x5f,0x5f, 0x01,0x6e,0x51,0xbe,0x10,0x11,0x6b,0x00,0x00,0x00,0x00,0x00,}}, // "shif____"
+ // ASCII-7-bit=110 SJS=190 CP932=107 [top SJS]
+ {{0x73,0x69,0x66,0x74,0x5f,0x5f,0x5f,0x5f, 0x01,0x72,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "sift____"
+ // ASCII-7-bit=114 SJS=190 [top SJS]
+ {{0x73,0x6a,0x69,0x73,0x5f,0x5f,0x5f,0x5f, 0x01,0x79,0x51,0xbe,0x10,0x11,0x5d,0x00,0x00,0x00,0x00,0x00,}}, // "sjis____"
+ // ASCII-7-bit=121 SJS=190 CP932=93 [top SJS]
+ {{0x73,0x6b,0x77,0x69,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "skwi1250"
+ // CP1250=191 [top CP1250]
+ {{0x73,0x6f,0x5f,0x5f,0x35,0x39,0x31,0x35, 0x02,0x86,0x6f,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "so__5915"
+ // ASCII-7-bit=134 Latin1=111 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x73,0x6f,0x5f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x9a,0xbd,0x21,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "so__8591"
+ // ASCII-7-bit=154 Latin1=189 CP1252=139 [top Latin1]
+ {{0x73,0x6f,0x5f,0x5f,0x38,0x35,0x39,0x32, 0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "so__8592"
+ // Latin2=191 [top Latin2]
+ {{0x73,0x76,0x73,0x65,0x38,0x35,0x39,0x31, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "svse8591"
+ // Latin1=191 [top Latin1]
+ {{0x74,0x61,0x62,0x5f,0x5f,0x5f,0x5f,0x5f, 0x30,0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tab_____"
+ // TAB=191 [top TAB]
+ {{0x74,0x61,0x6d,0x5f,0x5f,0x5f,0x5f,0x5f, 0x30,0x31,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tam_____"
+ // TAM=191 [top TAM]
+ {{0x74,0x65,0x78,0x74,0x5f,0x5f,0x5f,0x5f, 0x02,0xac,0xb7,0x21,0xa0,0xa1,0x49,0x00,0x00,0x00,0x00,0x00,}}, // "text____"
+ // ASCII-7-bit=172 Latin1=183 CP1252=160 ISO-8859-15=73 [top Latin1]
+ {{0x74,0x69,0x73,0x5f,0x36,0x31,0x38,0x5f, 0x01,0x75,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tis_618_"
+ // ASCII-7-bit=117 ISO-8859-11=190 [top ISO-8859-11]
+ {{0x74,0x69,0x73,0x5f,0x36,0x32,0x30,0x5f, 0x01,0x82,0xd1,0xbe,0xd1,0x7b,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tis_620_"
+ // ASCII-7-bit=130 ISO-8859-11=190 CP874=123 [top ISO-8859-11]
+ {{0x74,0x72,0x5f,0x5f,0x38,0x35,0x39,0x39, 0xd1,0xbe,0x81,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tr__8599"
+ // Latin5=190 CP1254=111 [top Latin5]
+ {{0x74,0x72,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0xd1,0xbe,0x81,0x5f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tr______"
+ // Latin5=190 CP1254=95 [top Latin5]
+ {{0x74,0x72,0x69,0x73,0x38,0x35,0x39,0x39, 0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tris8599"
+ // Latin5=191 [top Latin5]
+ {{0x74,0x73,0x63,0x69,0x5f,0x5f,0x5f,0x5f, 0x30,0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "tsci____"
+ // TSCII=191 [top TSCII]
+ {{0x75,0x63,0x73,0x5f,0x32,0x5f,0x5f,0x5f, 0x02,0xb8,0xa7,0x21,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "ucs_2___"
+ // ASCII-7-bit=184 Latin1=167 CP1252=163 [top ASCII-7-bit]
+ {{0x75,0x66,0x74,0x5f,0x38,0x5f,0x5f,0x5f, 0x01,0xb0,0x11,0xb8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "uft_8___"
+ // ASCII-7-bit=176 UTF8=184 [top UTF8]
+ {{0x75,0x69,0x73,0x6f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "uiso8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+ {{0x75,0x6e,0x69,0x63,0x31,0x31,0x5f,0x5f, 0x01,0xa7,0x11,0xbb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "unic11__"
+ // ASCII-7-bit=167 UTF8=187 [top UTF8]
+ {{0x75,0x6e,0x69,0x63,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x90,0x21,0x85,0xa1,0x45,0x00,0x00,0x00,0x00,0x00,}}, // "unic____"
+ // ASCII-7-bit=190 Latin1=144 CP1252=133 ISO-8859-15=69 [top ASCII-7-bit]
+ {{0x75,0x6e,0x6b,0x6e,0x38,0x5f,0x5f,0x5f, 0x02,0xa2,0xbb,0x21,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "unkn8___"
+ // ASCII-7-bit=162 Latin1=187 CP1252=149 [top Latin1]
+ {{0x75,0x6e,0x6b,0x6e,0x5f,0x5f,0x5f,0x5f, 0x01,0x9c,0x51,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "unkn____"
+ // ASCII-7-bit=156 SJS=189 [top SJS]
+ {{0x75,0x70,0x66,0x5f,0x38,0x5f,0x5f,0x5f, 0x21,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "upf_8___"
+ // UTF8=191 [top UTF8]
+ {{0x75,0x73,0x5f,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x7d,0x21,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "us______"
+ // ASCII-7-bit=190 Latin1=125 CP1252=125 [top ASCII-7-bit]
+ {{0x75,0x73,0x61,0x73,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x83,0x21,0x6a,0xa1,0x38,0x00,0x00,0x00,0x00,0x00,}}, // "usas____"
+ // ASCII-7-bit=190 Latin1=131 CP1252=106 ISO-8859-15=56 [top ASCII-7-bit]
+ {{0x75,0x73,0x65,0x6e,0x5f,0x5f,0x5f,0x5f, 0x02,0xb8,0x94,0x21,0xad,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "usen____"
+ // ASCII-7-bit=184 Latin1=148 CP1252=173 [top ASCII-7-bit]
+ {{0x75,0x73,0x65,0x72,0x5f,0x5f,0x5f,0x5f, 0x02,0xb9,0x9e,0x21,0xa7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "user____"
+ // ASCII-7-bit=185 Latin1=158 CP1252=167 [top ASCII-7-bit]
+ {{0x75,0x73,0x69,0x73,0x38,0x35,0x39,0x31, 0x02,0x78,0xbe,0x21,0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "usis8591"
+ // ASCII-7-bit=120 Latin1=190 CP1252=120 [top Latin1]
+ {{0x75,0x73,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x79,0xbc,0x21,0xa5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "uso_8591"
+ // ASCII-7-bit=121 Latin1=188 CP1252=165 [top Latin1]
+ {{0x75,0x74,0x66,0x5f,0x31,0x36,0x5f,0x5f, 0x01,0xb0,0x11,0xb8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_16__"
+ // ASCII-7-bit=176 UTF8=184 [top UTF8]
+ {{0x75,0x74,0x66,0x5f,0x33,0x32,0x5f,0x5f, 0x02,0xb5,0xa9,0x21,0x9f,0xa1,0xa1,0x00,0x00,0x00,0x00,0x00,}}, // "utf_32__"
+ // ASCII-7-bit=181 Latin1=169 CP1252=159 ISO-8859-15=161 [top ASCII-7-bit]
+ {{0x75,0x74,0x66,0x5f,0x35,0x39,0x31,0x35, 0x11,0x90,0xd1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_5915"
+ // Latin1=144 ISO-8859-15=190 [top ISO-8859-15]
+ {{0x75,0x74,0x66,0x5f,0x37,0x5f,0x5f,0x5f, 0x01,0x88,0x20,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_7___"
+ // ASCII-7-bit=136 UTF7=190 [top UTF7]
+ {{0x75,0x74,0x66,0x5f,0x38,0x35,0x39,0x31, 0x02,0x95,0xbd,0x21,0x8c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_8591"
+ // ASCII-7-bit=149 Latin1=189 CP1252=140 [top Latin1]
+ {{0x75,0x74,0x66,0x5f,0x38,0x35,0x39,0x39, 0xd1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_8599"
+ // Latin5=191 [top Latin5]
+ {{0x75,0x74,0x66,0x5f,0x38,0x5f,0x5f,0x5f, 0x01,0xae,0x11,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_8___"
+ // ASCII-7-bit=174 UTF8=185 [top UTF8]
+ {{0x75,0x74,0x66,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x8a,0x21,0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utf_____"
+ // ASCII-7-bit=190 Latin1=138 CP1252=116 [top ASCII-7-bit]
+ {{0x75,0x74,0x66,0x62,0x31,0x36,0x5f,0x5f, 0x01,0xa5,0x20,0x41,0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfb16__"
+ // ASCII-7-bit=165 UTF-16BE=188 [top UTF-16BE]
+ {{0x75,0x74,0x66,0x62,0x33,0x32,0x5f,0x5f, 0x30,0x81,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfb32__"
+ // UTF-32BE=191 [top UTF-32BE]
+ {{0x75,0x74,0x66,0x69,0x38,0x35,0x39,0x31, 0x02,0x99,0xbd,0x21,0x87,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfi8591"
+ // ASCII-7-bit=153 Latin1=189 CP1252=135 [top Latin1]
+ {{0x75,0x74,0x66,0x6c,0x31,0x36,0x5f,0x5f, 0x20,0x71,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfl16__"
+ // UTF-16LE=191 [top UTF-16LE]
+ {{0x75,0x74,0x66,0x6c,0x33,0x32,0x5f,0x5f, 0x30,0x91,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfl32__"
+ // UTF-32LE=191 [top UTF-32LE]
+ {{0x75,0x74,0x66,0x75,0x38,0x38,0x5f,0x5f, 0x30,0xb1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "utfu88__"
+ // X-UTF8UTF8=191 [top X-UTF8UTF8]
+ {{0x76,0x61,0x6c,0x75,0x5f,0x5f,0x5f,0x5f, 0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "valu____"
+ // Latin1=191 [top Latin1]
+ {{0x76,0x69,0x73,0x75,0x5f,0x5f,0x5f,0x5f, 0x01,0x84,0x10,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "visu____"
+ // ASCII-7-bit=132 Hebrew=190 [top Hebrew]
+ {{0x77,0x61,0x69,0x6e,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wain1250"
+ // CP1250=191 [top CP1250]
+ {{0x77,0x65,0x69,0x73,0x35,0x39,0x31,0x35, 0x02,0x9f,0x7d,0x21,0x84,0xa1,0xbc,0x00,0x00,0x00,0x00,0x00,}}, // "weis5915"
+ // ASCII-7-bit=159 Latin1=125 CP1252=132 ISO-8859-15=188 [top ISO-8859-15]
+ {{0x77,0x65,0x69,0x73,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x7e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "weis8591"
+ // Latin1=190 CP1252=126 [top Latin1]
+ {{0x77,0x65,0x73,0x74,0x31,0x32,0x35,0x32, 0x01,0x6f,0x31,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "west1252"
+ // ASCII-7-bit=111 CP1252=190 [top CP1252]
+ {{0x77,0x65,0x73,0x74,0x38,0x35,0x39,0x31, 0x02,0x79,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "west8591"
+ // ASCII-7-bit=121 Latin1=190 [top Latin1]
+ {{0x77,0x65,0x73,0x74,0x5f,0x5f,0x5f,0x5f, 0x02,0xa9,0x9d,0x21,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "west____"
+ // ASCII-7-bit=169 Latin1=157 CP1252=185 [top CP1252]
+ {{0x77,0x69,0x64,0x6e,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "widn1250"
+ // CP1250=191 [top CP1250]
+ {{0x77,0x69,0x64,0x6f,0x31,0x32,0x35,0x30, 0x01,0x7c,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wido1250"
+ // ASCII-7-bit=124 CP1250=190 [top CP1250]
+ {{0x77,0x69,0x64,0x6f,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wido1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x69,0x64,0x6f,0x31,0x32,0x35,0x32, 0x11,0xa9,0x21,0xbb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wido1252"
+ // Latin1=169 CP1252=187 [top CP1252]
+ {{0x77,0x69,0x64,0x6f,0x31,0x32,0x35,0x36, 0xb1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wido1256"
+ // CP1256=191 [top CP1256]
+ {{0x77,0x69,0x6d,0x64,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wimd1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x30, 0x01,0x8d,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1250"
+ // ASCII-7-bit=141 CP1250=190 [top CP1250]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x31, 0x01,0x8f,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1251"
+ // ASCII-7-bit=143 CP1251=190 [top CP1251]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x32, 0x02,0xac,0xa4,0x21,0xb6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1252"
+ // ASCII-7-bit=172 Latin1=164 CP1252=182 [top CP1252]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x33, 0x10,0x41,0x85,0x21,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1253"
+ // Greek=133 CP1253=190 [top CP1253]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x34, 0x01,0x6f,0xc1,0xaf,0x81,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1254"
+ // ASCII-7-bit=111 Latin5=175 CP1254=185 [top CP1254]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x35, 0x10,0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1255"
+ // CP1255=191 [top CP1255]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x36, 0x01,0x7f,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1256"
+ // ASCII-7-bit=127 CP1256=190 [top CP1256]
+ {{0x77,0x69,0x6e,0x5f,0x31,0x32,0x35,0x37, 0x01,0x8c,0xf1,0xbe,0xc1,0x77,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_1257"
+ // ASCII-7-bit=140 CP1257=190 ISO-8859-13=119 [top CP1257]
+ {{0x77,0x69,0x6e,0x5f,0x38,0x37,0x34,0x5f, 0x01,0x56,0xd1,0xaf,0xd1,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_874_"
+ // ASCII-7-bit=86 ISO-8859-11=175 CP874=185 [top CP874]
+ {{0x77,0x69,0x6e,0x5f,0x5f,0x5f,0x5f,0x5f, 0x01,0x9a,0x91,0xbd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "win_____"
+ // ASCII-7-bit=154 CP1251=189 [top CP1251]
+ {{0x77,0x69,0x6e,0x63,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "winc1250"
+ // CP1250=191 [top CP1250]
+ {{0x77,0x69,0x6e,0x63,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "winc1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x33,0x34, 0xb1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1234"
+ // CP1256=191 [top CP1256]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x30, 0x01,0x88,0xb1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1250"
+ // ASCII-7-bit=136 CP1250=190 [top CP1250]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x31, 0x01,0x8b,0x91,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1251"
+ // ASCII-7-bit=139 CP1251=190 [top CP1251]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x32, 0x02,0xa5,0xac,0x21,0xb6,0xa1,0x4f,0x00,0x00,0x00,0x00,0x00,}}, // "wind1252"
+ // ASCII-7-bit=165 Latin1=172 CP1252=182 ISO-8859-15=79 [top CP1252]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x33, 0x01,0x94,0x10,0x31,0xae,0x21,0xb8,0x00,0x00,0x00,0x00,0x00,}}, // "wind1253"
+ // ASCII-7-bit=148 Greek=174 CP1253=184 [top CP1253]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x34, 0x01,0x73,0xc1,0xaf,0x81,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1254"
+ // ASCII-7-bit=115 Latin5=175 CP1254=185 [top CP1254]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x35, 0x01,0x86,0x10,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1255"
+ // ASCII-7-bit=134 CP1255=190 [top CP1255]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x36, 0x01,0x74,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1256"
+ // ASCII-7-bit=116 CP1256=190 [top CP1256]
+ {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x37, 0x01,0x87,0xf1,0xbe,0xc1,0x52,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind1257"
+ // ASCII-7-bit=135 CP1257=190 ISO-8859-13=82 [top CP1257]
+ {{0x77,0x69,0x6e,0x64,0x33,0x31,0x5f,0x5f, 0x01,0x62,0x51,0xbe,0x10,0x11,0x5e,0x00,0x00,0x00,0x00,0x00,}}, // "wind31__"
+ // ASCII-7-bit=98 SJS=190 CP932=94 [top SJS]
+ {{0x77,0x69,0x6e,0x64,0x38,0x34,0x37,0x5f, 0xe1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind847_"
+ // ISO-8859-11=191 [top ISO-8859-11]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x32,0x5f, 0x01,0x79,0x20,0x01,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind852_"
+ // ASCII-7-bit=121 CP852=190 [top CP852]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x39,0x31, 0x02,0x9a,0xbd,0x21,0x89,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8591"
+ // ASCII-7-bit=154 Latin1=189 CP1252=137 [top Latin1]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x39,0x32, 0x01,0x83,0x81,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8592"
+ // ASCII-7-bit=131 Latin2=190 [top Latin2]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x39,0x36, 0x01,0x6f,0x20,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8596"
+ // ASCII-7-bit=111 Arabic=190 [top Arabic]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x39,0x37, 0x01,0x6f,0x10,0x31,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8597"
+ // ASCII-7-bit=111 Greek=190 [top Greek]
+ {{0x77,0x69,0x6e,0x64,0x38,0x35,0x39,0x39, 0x01,0x6c,0xc1,0xbe,0x81,0x6c,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8599"
+ // ASCII-7-bit=108 Latin5=190 CP1254=108 [top Latin5]
+ {{0x77,0x69,0x6e,0x64,0x38,0x36,0x36,0x5f, 0x20,0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind866_"
+ // CP866=191 [top CP866]
+ {{0x77,0x69,0x6e,0x64,0x38,0x37,0x34,0x5f, 0x01,0x8a,0xd1,0xbe,0xd1,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind874_"
+ // ASCII-7-bit=138 ISO-8859-11=190 CP874=125 [top ISO-8859-11]
+ {{0x77,0x69,0x6e,0x64,0x38,0x38,0x35,0x39, 0x02,0x97,0xb6,0x21,0xb1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8859"
+ // ASCII-7-bit=151 Latin1=182 CP1252=177 [top Latin1]
+ {{0x77,0x69,0x6e,0x64,0x38,0x5f,0x5f,0x5f, 0x01,0x93,0x11,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind8___"
+ // ASCII-7-bit=147 UTF8=190 [top UTF8]
+ {{0x77,0x69,0x6e,0x64,0x39,0x33,0x32,0x5f, 0x01,0x7d,0x51,0xa4,0x10,0x11,0xbc,0x00,0x00,0x00,0x00,0x00,}}, // "wind932_"
+ // ASCII-7-bit=125 SJS=164 CP932=188 [top CP932]
+ {{0x77,0x69,0x6e,0x64,0x39,0x34,0x39,0x5f, 0x01,0x7b,0x41,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind949_"
+ // ASCII-7-bit=123 KSC=190 [top KSC]
+ {{0x77,0x69,0x6e,0x64,0x39,0x35,0x30,0x5f, 0x01,0x6f,0x71,0x7f,0x20,0x51,0xbe,0x00,0x00,0x00,0x00,0x00,}}, // "wind950_"
+ // ASCII-7-bit=111 BIG5=127 BIG5-CP950=190 [top BIG5-CP950]
+ {{0x77,0x69,0x6e,0x64,0x5f,0x5f,0x5f,0x5f, 0x01,0xb5,0x11,0xb4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wind____"
+ // ASCII-7-bit=181 UTF8=180 [top ASCII-7-bit]
+ {{0x77,0x69,0x6e,0x65,0x31,0x32,0x35,0x32, 0x01,0x6f,0x31,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wine1252"
+ // ASCII-7-bit=111 CP1252=190 [top CP1252]
+ {{0x77,0x69,0x6e,0x6f,0x31,0x32,0x35,0x30, 0xc1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wino1250"
+ // CP1250=191 [top CP1250]
+ {{0x77,0x69,0x6e,0x6f,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wino1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x69,0x6e,0x73,0x31,0x32,0x35,0x35, 0x10,0x11,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wins1255"
+ // CP1255=191 [top CP1255]
+ {{0x77,0x69,0x72,0x64,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wird1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x69,0x73,0x6f,0x38,0x35,0x39,0x31, 0x11,0xbe,0x21,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wiso8591"
+ // Latin1=190 CP1252=127 [top Latin1]
+ {{0x77,0x6e,0x64,0x6f,0x31,0x32,0x35,0x31, 0xa1,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wndo1251"
+ // CP1251=191 [top CP1251]
+ {{0x77,0x6e,0x64,0x6f,0x31,0x32,0x35,0x36, 0x01,0x6e,0xa1,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wndo1256"
+ // ASCII-7-bit=110 CP1256=190 [top CP1256]
+ {{0x77,0x6f,0x6e,0x64,0x31,0x32,0x35,0x32, 0x41,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "wond1252"
+ // CP1252=191 [top CP1252]
+ {{0x77,0x6f,0x72,0x67,0x31,0x32,0x35,0x32, 0x01,0x83,0x31,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "worg1252"
+ // ASCII-7-bit=131 CP1252=190 [top CP1252]
+ {{0x79,0x65,0x73,0x5f,0x5f,0x5f,0x5f,0x5f, 0x02,0xbe,0x81,0x21,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "yes_____"
+ // ASCII-7-bit=190 Latin1=129 CP1252=139 [top ASCII-7-bit]
+ {{0x79,0x6b,0x74,0x63,0x5f,0x5f,0x5f,0x5f, 0x51,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "yktc____"
+ // KSC=191 [top KSC]
+ {{0x7a,0x73,0x6f,0x5f,0x38,0x35,0x39,0x31, 0x02,0x6f,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,}}, // "zso_8591"
+ // ASCII-7-bit=111 Latin1=190 [top Latin1]
+};
+
+static const int kCharsetHintProbsSize = 438;
+
+static const uint8 kDefaultProb[NUM_RANKEDENCODING] = { // MaxRange 192
+177, 170, 156, 149, 150, 142, 140, 124, 130, 127, 124, 118, 127, 118, 109, 104, 98, 93, 96, 82, 84, 81, 80, 64, 61, 57, 53, 47, 42, 28, 24, 22,
+ 17, 0, 5, 1, 5, 12, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, };
+
+static const int kMaxTldKey = 4;
+static const int kMaxTldVector = 16;
+static const int kMaxCharsetKey = 8;
+static const int kMaxCharsetVector = 12;
+static const int kMaxLangKey = 8;
+static const int kMaxLangVector = 12;
+// Smoothing percentage across encodings with same UTF-8 result: 100%
+
+static const UnigramEntry unigram_table[NUM_RANKEDENCODING] = {
+{ // ASCII-7-bit (788.373M chars) [0]
+ {NULL, NULL, NULL, NULL},
+ 77, 207, 29, 27, 255,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,189,189,0,0,189,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // Latin1 (1792.786M chars) [1]
+ {NULL, NULL, ced_hires_13, ced_hires_13, },
+ 87, 217, 37, 20, 128,
+ {1,0,1,1,0,0,1,0, 0,9,9,0,1,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 186,137,105,112,140,106,126,145, 132,113,128,124,123,101,126,119, 141,113,107,116,129,113,113,143, 130,105,127,128,103,116,106,128,
+ 122,138,132,155,161,129,133,190, 124,152,124,119,117,144,120,121, 127,136,122,132,112,139,144,116, 113,101,117,117,145,120,135,114,
+ 121,138,107,125,163,129,131,190, 120,141,112,110,114,143,115,133, 124,135,126,131,118,141,160,110, 114,104,119,99,148,119,129,117,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 195,159,115,166,155,125,133,156, 130,168,156,159,121,135,150,133, 169,125,135,118,150,113,114,173, 126,107,176,178,97,131,103,170,
+ 165,209,166,183,211,198,182,194, 193,219,182,153,130,205,161,148, 160,187,142,208,169,167,210,131, 192,122,189,152,212,181,158,185,
+ 195,214,155,184,212,208,182,181, 183,213,171,154,161,215,150,137, 175,187,162,210,158,168,208,131, 192,156,189,142,199,193,155,120,
+ },
+ {43,13,19,71,0,0,0,0, 0,145,153,0,0,165,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,142,175,92,109,120,168,161, 150,151,135,110,183,159,177,142, 146,163,159,149,146,144,137,136, 133,134,166,128,200,102,117,143,
+ 102,197,188,202,195,190,180,196, 185,179,172,189,202,198,215,195, 185,164,216,206,204,172,190,182, 158,170,181,119,107,136,114,132,
+ 134,194,189,198,192,189,178,194, 182,176,171,189,201,194,213,194, 181,151,213,203,201,171,189,182, 155,176,182,106,136,85,122,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 187,138,112,113,139,107,128,136, 133,137,118,120,118,129,135,123, 139,125,101,126,128,108,123,140, 131,107,125,133,123,119,104,138,
+ 133,128,130,187,160,126,123,145, 123,153,122,122,114,136,125,117, 153,141,127,129,107,167,144,114, 113,105,118,118,134,117,122,164,
+ 130,132,122,187,161,131,122,139, 122,137,119,120,113,138,123,110, 153,141,127,130,133,167,146,104, 117,105,130,109,131,117,133,119,
+ },
+ {128,0,110,142,142,142,142,140, 0,0,142,142,142,142,142,142, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 22,0,2,2,2,2,2,2, 0,0,2,14,2,2,2,2, 46,0,2,14,2,2,2,2, 0,0,2,18,2,4,2,6,
+ 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 42,0,2,10,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 18,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 38,0,2,6,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 164,0,136,164,110,108,108,106, 0,0,190,164,136,134,122,152, 158,0,144,156,112,116,106,106, 0,0,166,212,158,164,146,162,
+ 90,0,92,96,139,139,94,92, 0,0,148,166,184,176,134,146, 98,0,96,104,139,139,110,110, 0,0,138,172,174,190,154,164,
+ 130,0,138,126,38,48,135,135, 0,0,130,156,128,148,186,180, 120,0,126,120,48,56,139,141, 0,0,142,156,140,186,166,188,
+ },
+},
+
+{ // UTF8 (16713.069M chars) [2]
+ {NULL, NULL, NULL, NULL},
+ 169, 203, 42, 24, 131,
+ {197,207,201,202,188,181,180,180, 188,183,184,187,188,182,178,183, 181,179,172,178,182,182,183,183, 181,181,183,180,189,181,175,180,
+ 182,183,175,177,184,183,183,184, 187,176,181,181,178,186,187,184, 187,177,175,183,184,176,174,177, 196,183,188,186,194,184,180,181,
+ 0,0,183,211,189,187,147,104, 65,123,87,102,121,13,187,176, 216,203,118,108,88,134,124,193, 206,202,155,161,58,0,119,0,
+ 181,176,180,213,192,206,202,197, 192,187,165,177,183,170,90,182, 71,0,0,13,1,0,0,0, 0,0,0,0,0,0,0,0,
+ 119,115,120,96,114,98,104,106, 121,111,96,110,113,105,109,100, 109,107,92,112,112,97,91,102, 110,119,116,99,117,110,104,106,
+ 133,132,135,130,132,139,129,131, 133,127,121,126,132,128,121,138, 136,128,120,126,145,114,120,123, 132,124,128,122,138,120,117,132,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,64,
+ },
+ {0,0,0,0,0,0,0,0, 0,109,143,0,0,144,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 189,125,169,73,68,107,151,137, 146,147,111,102,152,138,154,129, 142,152,148,137,131,132,127,127, 123,121,139,118,189,98,96,127,
+ 87,124,119,124,117,115,112,111, 114,127,109,102,114,120,120,114, 120,94,118,122,122,113,106,113, 95,99,88,117,109,125,97,107,
+ 80,139,113,146,112,126,103,99, 115,147,96,97,118,131,156,123, 132,80,111,140,146,136,107,102, 86,116,85,83,110,84,118,0,
+ 204,210,207,204,196,192,190,188, 194,187,191,189,194,188,179,187, 184,183,175,181,186,187,183,187, 184,189,185,186,191,184,178,186,
+ 192,196,179,183,192,185,183,198, 192,196,189,182,182,193,189,192, 199,194,188,195,190,194,185,185, 203,191,196,195,200,196,197,188,
+ 0,0,121,113,72,69,49,4, 0,0,0,71,0,0,113,107, 113,102,0,0,0,0,19,86, 134,132,91,105,0,0,0,0,
+ 174,140,150,207,185,199,194,190, 187,181,160,174,177,157,75,176, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 6,0,2,2,2,6,2,2, 0,0,0,0,50,38,2,128, 4,0,2,2,2,4,2,2, 0,0,0,0,54,40,2,128,
+ 4,0,2,2,2,6,2,2, 0,0,0,0,128,128,128,128, 2,0,2,2,2,2,2,2, 0,0,0,0,128,128,128,128,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,128,128,128,128, 128,0,118,118,128,128,128,128, 0,0,0,0,128,128,64,128,
+ 176,0,168,172,172,172,162,164, 120,126,126,116,126,140,134,128, 168,0,170,166,172,170,174,176, 118,122,118,116,142,116,138,128,
+ 168,0,170,168,168,172,176,174, 110,120,114,112,136,142,140,128, 168,0,172,170,160,166,160,164, 120,126,118,116,140,134,136,128,
+ 0,0,0,0,0,0,0,0, 116,122,140,128,0,0,0,0, 0,0,0,0,0,0,0,0, 128,120,126,134,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 134,132,120,126,0,0,0,0, 0,0,0,0,0,0,0,0, 116,152,138,116,0,0,0,0,
+ },
+},
+
+{ // GB (9061.562M chars) [3]
+ {NULL, ced_hires_3, ced_hires_4, ced_hires_5, },
+ 204, 189, 27, 16, 128,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,66,73, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 70,204,125,202,219,216,120,114, 118,156,113,85,96,78,81,79, 190,196,199,195,195,208,196,199, 196,203,195,200,201,197,193,195,
+ 193,195,190,195,195,183,196,191, 201,197,207,197,197,198,201,199, 201,191,203,201,198,196,202,200, 134,116,114,132,122,124,118,115,
+ 115,120,116,120,141,121,124,123, 121,120,121,113,125,116,117,110, 117,114,112,113,119,112,124,123, 84,74,83,78,69,83,77,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,93,66,68,84,3,51,0, 116,102,41,61,83,57,66,12, 70,85,73,66,54,64,44,68, 0,3,3,3,54,73,70,54,
+ 95,66,54,44,66,78,26,37, 57,41,3,57,13,2,0,0, 54,51,0,41,68,70,70,59, 41,70,41,13,0,64,59,80,
+ 12,3,51,12,13,26,41,3, 0,79,47,61,2,19,66,64, 37,37,13,93,54,13,54,51, 51,12,59,73,89,54,54,94,
+ },
+ {101,14,0,0,0,0,0,0, 16,16,0,0,68,49,44,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 81,56,73,0,0,0,64,71, 49,49,0,84,53,69,60,32, 84,60,49,35,15,19,18,1, 2,0,15,0,92,29,0,0,
+ 83,69,66,85,103,49,69,51, 56,71,53,83,70,73,65,91, 71,59,61,71,80,77,72,79, 61,69,67,64,66,73,77,73,
+ 70,75,71,70,86,62,64,69, 71,66,39,69,73,64,74,75, 79,69,71,70,83,80,53,92, 69,66,59,69,69,85,90,3,
+ 94,72,77,73,64,67,62,71, 80,76,76,72,65,74,80,73, 77,93,79,99,84,69,81,94, 51,35,67,74,49,67,84,78,
+ 126,196,197,192,196,176,185,184, 195,185,196,194,198,187,184,191, 190,190,180,189,189,194,181,191, 182,191,188,196,189,191,187,192,
+ 186,174,192,193,203,190,193,194, 190,188,189,195,181,187,196,191, 200,178,184,186,189,186,192,180, 193,176,195,182,185,179,182,186,
+ 185,187,190,185,183,187,185,186, 177,190,189,192,187,187,181,180, 178,185,188,196,172,177,186,192, 192,182,197,186,178,187,185,0,
+ },
+ {128,0,128,128,128,128,128,128, 128,128,104,128,114,124,128,118, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 46,0,46,50,32,40,40,34, 128,128,26,28,26,30,32,32, 42,0,42,44,26,34,34,28, 128,128,46,50,46,52,54,54,
+ 44,0,44,46,28,36,36,30, 128,128,48,50,46,52,56,54, 50,0,50,54,34,42,42,38, 128,128,50,52,50,56,58,58,
+ 128,0,128,128,128,128,128,128, 128,128,58,62,58,64,70,66, 128,0,128,128,128,128,128,128, 128,128,40,42,40,44,48,46,
+ 0,0,0,0,154,180,164,168, 188,176,128,106,122,124,134,92, 0,0,0,0,128,128,128,136, 128,128,132,130,130,132,130,130,
+ 0,0,0,0,242,218,218,232, 138,134,137,131,129,117,127,113, 0,0,0,0,202,206,218,216, 110,112,129,127,129,127,127,131,
+ 0,0,0,0,202,218,218,206, 114,106,122,126,126,128,130,132, 0,0,0,0,210,214,206,206, 120,114,124,128,126,136,122,126,
+ 0,0,0,0,206,208,200,202, 218,236,136,124,124,122,126,126, 230,194,200,214,190,198,194,216, 192,186,128,122,132,128,130,120,
+ },
+},
+
+{ // CP1252 (408.280M chars) [4]
+ {NULL, NULL, ced_hires_13, ced_hires_13, },
+ 89, 209, 40, 30, 128,
+ {116,114,130,121,123,133,113,118, 108,96,180,101,111,113,172,102, 63,68,105,82,59,104,73,70, 61,79,141,65,74,59,133,92,
+ 184,136,106,111,139,104,125,143, 132,111,126,123,121,100,125,118, 143,111,106,114,127,111,112,142, 128,104,127,127,102,114,106,126,
+ 120,172,132,158,159,131,136,188, 123,153,123,119,120,159,118,120, 127,136,121,131,136,137,143,117, 112,109,131,116,143,151,134,114,
+ 120,172,130,128,161,130,131,188, 120,143,112,109,119,159,114,131, 122,133,125,130,137,139,158,109, 114,108,131,98,146,151,128,116,
+ 169,126,98,99,143,148,118,105, 94,83,203,88,113,99,199,98, 95,134,176,160,158,170,169,143, 114,141,201,118,111,110,201,130,
+ 193,157,113,163,153,123,131,154, 127,166,153,157,119,132,148,131, 167,123,133,115,148,111,112,171, 124,105,174,176,96,128,101,168,
+ 163,207,164,180,208,196,180,192, 191,217,180,151,128,203,159,146, 158,185,140,206,167,165,207,128, 190,119,187,150,210,179,156,183,
+ 193,212,152,182,209,206,180,179, 181,211,168,151,159,213,148,135, 173,185,160,208,156,166,206,129, 190,153,187,139,197,191,153,118,
+ },
+ {128,43,4,69,19,0,0,0, 0,146,153,0,0,165,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,141,174,92,108,119,170,160, 148,152,134,110,182,158,177,142, 147,162,157,147,145,143,136,135, 132,133,165,129,200,100,121,143,
+ 118,197,187,200,193,200,178,194, 183,191,170,191,201,196,214,194, 184,162,214,205,205,174,188,180, 157,168,179,122,110,135,115,131,
+ 133,196,189,196,191,201,176,192, 180,192,169,191,199,192,212,193, 182,149,211,202,203,175,187,180, 153,174,180,105,136,86,122,30,
+ 113,107,124,132,114,134,112,118, 113,93,171,97,104,105,166,96, 99,108,124,118,117,133,112,109, 102,129,171,117,112,107,166,135,
+ 183,137,111,110,137,105,126,133, 132,134,125,119,116,126,133,122, 144,123,99,123,127,108,120,139, 128,105,124,131,120,117,103,138,
+ 130,152,132,184,157,126,125,142, 124,153,120,120,118,183,123,115, 150,138,125,127,107,164,145,111, 111,103,118,117,132,116,120,161,
+ 130,157,121,184,158,129,123,136, 124,145,121,118,111,183,121,110, 150,139,124,127,130,164,143,101, 116,102,132,106,129,115,130,117,
+ },
+ {32,0,2,24,2,2,2,2, 22,20,16,36,6,24,6,26, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 24,0,2,2,2,2,2,2, 2,2,2,16,2,2,2,4, 46,0,2,16,2,2,2,2, 6,6,2,20,2,6,2,10,
+ 14,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 38,0,2,6,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 14,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 34,0,2,4,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 130,0,106,124,140,126,112,112, 160,134,134,140,180,136,142,134, 134,0,132,126,86,84,140,128, 118,162,132,166,130,138,180,136,
+ 162,0,136,166,110,110,108,106, 118,138,192,162,130,136,116,152, 158,0,144,154,112,118,106,106, 148,164,166,210,152,164,138,164,
+ 88,0,92,96,139,141,94,92, 180,136,146,162,174,174,124,144, 98,0,96,104,139,139,110,112, 184,150,134,164,162,186,142,158,
+ 128,0,138,128,38,48,135,137, 130,178,128,152,118,146,176,178, 120,0,126,120,48,56,139,141, 136,180,140,150,130,184,156,184,
+ },
+},
+
+{ // KSC (5258.976M chars) [5]
+ {NULL, ced_hires_6, ced_hires_7, ced_hires_8, },
+ 203, 186, 27, 9, 128,
+ {71,109,117,106,108,109,104,107, 110,108,108,112,103,104,106,105, 102,108,101,103,104,107,99,103, 104,99,107,103,102,103,107,98,
+ 106,206,164,204,164,121,141,122, 146,119,136,130,120,90,91,81, 216,212,164,201,215,207,187,206, 213,208,207,204,210,209,205,216,
+ 225,214,178,201,188,198,199,213, 199,93,107,104,116,113,115,113, 108,107,109,113,110,97,99,104, 101,113,115,107,110,108,111,117,
+ 110,108,103,116,115,142,146,136, 118,134,125,131,144,112,112,126, 112,117,107,103,107,103,102,104, 101,114,109,104,105,93,0,0,
+ 52,0,0,60,0,1,0,0, 0,12,0,0,28,0,18,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,78,41,83,69,52,75,65, 1,50,60,76,0,62,88,58, 79,72,60,68,0,65,54,44, 72,58,78,86,46,82,73,48,
+ 57,67,58,67,72,57,60,86, 18,31,65,50,69,65,85,79, 52,57,54,34,54,0,41,38, 0,78,82,58,50,78,1,46,
+ 0,57,68,0,72,67,48,65, 48,41,31,107,18,67,63,0, 12,48,0,52,0,28,85,46, 82,67,44,48,48,1,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,44,71,0,0,61,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 117,80,77,0,0,0,66,63, 78,84,0,97,84,72,82,66, 4,74,65,78,51,54,50,9, 24,6,52,5,117,40,0,44,
+ 63,9,0,52,36,0,9,0, 0,0,0,24,48,0,51,51, 60,1,55,58,40,0,7,6, 59,8,0,81,54,73,75,73,
+ 57,2,57,35,48,28,1,2, 0,42,0,6,50,1,20,8, 38,0,0,32,57,0,4,2, 35,32,36,56,57,50,89,38,
+ 66,99,111,102,98,106,96,97, 95,99,98,96,98,104,96,95, 98,97,92,103,92,97,93,96, 95,96,104,97,94,93,92,91,
+ 157,208,188,195,195,184,194,180, 174,186,184,181,197,195,200,187, 193,181,186,192,167,192,186,169, 205,187,203,199,192,172,174,185,
+ 186,179,196,199,181,187,182,201, 179,175,184,177,204,183,204,207, 190,189,185,176,191,179,192,189, 182,200,194,190,178,181,182,187,
+ 177,176,199,166,189,187,186,198, 184,182,174,196,175,194,193,169, 178,186,174,191,161,176,204,191, 198,180,187,178,193,181,149,9,
+ },
+ {128,0,128,128,128,128,128,128, 128,128,80,86,78,94,92,92, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 92,0,42,48,90,68,96,74, 128,128,44,44,42,48,48,50, 58,0,10,16,56,36,58,40, 128,128,42,44,40,46,48,48,
+ 66,0,18,26,66,44,66,50, 128,128,42,44,42,48,48,50, 128,0,128,128,128,128,128,128, 128,128,50,52,48,58,58,58,
+ 128,0,108,108,128,128,128,128, 128,128,40,40,38,44,44,46, 128,0,128,128,128,128,128,128, 128,128,54,56,50,60,58,60,
+ 0,0,0,0,186,204,190,182, 244,244,124,124,72,82,142,64, 0,0,0,0,164,206,174,184, 240,238,122,124,78,80,144,52,
+ 0,0,0,0,186,200,180,188, 142,140,147,131,99,107,101,91, 0,0,0,0,174,196,184,194, 112,114,125,127,127,125,133,129,
+ 0,0,0,0,196,204,194,204, 120,108,124,126,130,132,116,128, 0,0,0,0,0,0,0,0, 0,0,124,130,122,130,126,126,
+ 0,0,0,0,0,0,0,0, 0,0,144,130,102,108,110,110, 0,0,0,0,0,0,0,0, 0,0,126,130,124,124,130,126,
+ },
+},
+
+{ // SJS (6339.756M chars) [6]
+ {NULL, NULL, NULL, NULL},
+ 151, 136, 55, 16, 129,
+ {50,188,238,215,160,59,66,108, 192,200,192,194,200,199,202,200, 190,190,186,192,182,192,188,186, 101,96,109,95,94,95,102,113,
+ 84,100,117,108,117,135,81,77, 99,97,99,87,94,104,96,118, 120,115,111,101,96,105,108,108, 111,98,126,107,129,116,102,97,
+ 121,104,84,106,118,104,105,80, 97,98,110,102,121,108,114,99, 91,90,109,89,86,81,89,103, 106,103,118,100,101,114,121,115,
+ 114,112,106,111,106,104,109,112, 108,105,167,179,185,80,87,184, 83,79,20,45,22,55,39,0, 21,0,65,55,62,0,0,0,
+ 0,223,195,229,105,32,59,131, 61,180,189,185,180,185,191,185, 177,167,169,174,171,174,169,172, 163,80,84,78,90,80,80,85,
+ 59,108,81,89,100,121,105,17, 50,24,63,0,25,17,1,53, 95,61,66,50,0,31,31,81, 122,2,32,1,95,88,0,56,
+ 61,50,56,0,94,59,44,0, 38,84,50,0,106,63,0,18, 44,63,28,0,24,0,80,50, 56,88,38,24,1,93,98,91,
+ 113,91,95,105,97,102,91,93, 93,99,91,0,23,0,0,44, 0,76,0,0,7,42,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,15,79,0,0,65,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 113,69,98,19,0,8,69,67, 80,86,51,58,73,87,70,67, 54,73,78,62,54,54,44,46, 38,34,67,70,119,41,36,55,
+ 200,198,192,189,158,190,180,168, 171,181,178,171,176,161,191,184, 172,169,180,178,178,160,185,181, 191,169,175,200,180,158,184,172,
+ 177,159,190,176,163,179,174,191, 178,187,183,159,184,168,174,182, 179,167,178,179,178,180,188,176, 172,175,172,178,169,173,171,37,
+ 175,177,171,172,170,176,176,180, 168,182,188,188,179,179,172,172, 155,178,155,196,173,164,176,173, 162,162,164,156,168,174,177,175,
+ 179,175,195,159,185,163,180,172, 181,189,191,180,175,184,173,177, 166,185,174,185,165,192,178,192, 167,176,168,178,189,191,180,179,
+ 167,183,176,162,191,193,192,176, 191,193,177,160,203,189,170,166, 173,161,169,158,174,146,173,170, 162,160,175,166,189,175,165,174,
+ 186,163,175,177,160,176,185,183, 187,191,188,159,174,173,175,176, 193,187,161,164,151,172,158,144, 152,140,181,168,175,37,20,2,
+ },
+ {46,0,6,6,2,2,2,2, 2,2,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 128,0,126,120,8,12,14,18, 34,38,30,30,26,42,32,42, 128,0,128,126,6,10,10,14, 38,42,34,34,30,46,36,46,
+ 128,0,128,128,10,14,16,20, 52,56,48,46,42,60,50,60, 128,0,128,128,12,16,18,22, 52,56,48,46,42,60,50,60,
+ 94,0,52,54,2,2,2,2, 32,38,28,28,24,40,30,40, 128,0,128,128,46,50,52,58, 82,88,78,78,72,94,80,90,
+ 0,0,0,0,140,140,138,138, 132,130,132,130,134,132,132,128, 0,0,0,0,134,142,142,142, 130,136,134,134,124,134,132,142,
+ 214,0,248,242,110,114,102,98, 152,140,134,122,120,120,88,72, 182,0,226,234,144,134,80,86, 136,114,134,134,126,154,96,86,
+ 178,0,234,228,102,112,100,108, 140,126,126,128,112,160,72,74, 180,0,232,230,100,114,108,110, 138,130,120,146,128,142,50,56,
+ 0,0,0,0,134,146,130,142, 138,144,128,148,64,80,66,78, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // EUC-JP (4368.914M chars) [7]
+ {NULL, ced_hires_0, ced_hires_1, ced_hires_2, },
+ 202, 178, 27, 15, 128,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,173,100, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,228,186,180,229,225,122,123, 168,7,2,28,62,133,86,74, 195,190,202,195,193,195,191,198, 201,199,199,204,200,197,198,200,
+ 200,193,197,196,192,199,202,195, 189,196,201,190,197,197,191,183, 120,107,102,109,112,111,101,113, 107,110,96,107,105,115,111,124,
+ 116,109,107,105,105,141,147,137, 110,140,100,120,146,102,103,102, 105,112,105,133,84,0,0,0, 0,54,7,43,73,0,9,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {146,59,0,0,61,22,13,0, 60,71,0,0,80,26,61,28, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 83,27,41,0,46,51,35,0, 49,37,24,68,35,38,32,13, 131,42,28,35,34,12,0,0, 63,21,33,0,74,25,11,68,
+ 61,0,0,0,68,11,0,0, 31,1,1,0,39,0,101,78, 61,83,87,94,79,39,85,70, 76,64,19,86,75,87,77,75,
+ 72,46,73,78,49,96,75,69, 60,49,27,69,85,65,60,49, 74,36,61,72,76,65,80,67, 67,77,74,64,73,63,88,36,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 164,202,199,199,199,177,202,190, 185,187,195,193,192,191,183,196, 192,181,174,190,182,189,177,195, 190,197,177,183,211,189,184,194,
+ 186,192,177,195,187,176,192,194, 198,186,201,196,186,188,198,191, 190,186,182,176,189,182,193,190, 188,181,187,183,189,177,189,189,
+ 180,185,188,180,184,187,184,191, 191,188,197,192,194,179,192,186, 182,187,188,200,180,176,184,190, 174,170,179,160,193,182,193,31,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,44,8,34,20,28,28, 0,0,0,0,0,0,0,0, 2,0,56,14,40,28,34,34, 0,0,0,0,0,0,0,0,
+ 2,0,60,16,42,30,36,38, 0,0,0,0,0,0,0,0, 92,0,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 70,0,128,88,128,128,128,128, 0,0,0,0,0,0,0,0, 102,0,128,120,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 0,0,146,186,190,162,182,180, 178,180,134,130,124,136,18,30, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 246,0,174,232,204,218,212,204, 0,0,131,131,129,123,123,117, 250,0,190,230,210,212,216,214, 0,0,125,121,125,131,131,133,
+ 250,0,192,232,204,216,202,214, 0,0,120,126,124,128,128,132, 244,0,172,236,206,224,218,216, 0,0,126,128,124,130,124,128,
+ 246,0,202,232,208,214,212,210, 0,0,144,110,104,110,108,108, 238,0,210,224,196,224,210,208, 0,0,118,114,104,116,104,152,
+ },
+},
+
+{ // BIG5 (2431.102M chars) [8]
+ {NULL, NULL, NULL, NULL},
+ 157, 174, 62, 10, 129,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,70,70, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,182,164,130,214,207,204,195, 202,198,207,192,194,201,196,187, 202,189,195,196,188,194,190,181, 194,188,192,185,188,188,191,185,
+ 184,179,185,186,180,189,110,108, 94,123,111,121,108,113,105,112, 116,103,101,106,110,102,114,99, 88,91,120,98,95,107,108,94,
+ 113,93,95,93,94,131,139,129, 121,130,93,111,138,87,98,87, 90,89,97,92,97,92,92,96, 102,147,84,82,84,87,88,107,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,216,162,127,212,204,205,192, 184,187,189,191,193,185,187,179, 188,190,183,192,177,189,189,198, 181,189,172,184,183,180,173,165,
+ 177,175,172,174,167,176,167,100, 42,127,129,92,89,80,107,94, 90,84,93,108,107,102,93,84, 105,100,83,90,94,79,92,95,
+ 84,84,90,92,77,89,103,113, 89,87,81,74,82,87,92,56, 77,96,70,88,72,107,102,81, 90,70,90,80,91,56,42,19,
+ },
+ {90,0,0,0,0,0,0,0, 0,0,0,0,0,4,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 77,44,72,0,0,0,44,0, 53,54,4,69,21,39,51,1, 78,36,31,5,0,4,0,0, 0,0,5,0,85,0,0,0,
+ 200,201,188,193,189,168,184,189, 192,178,176,174,183,184,182,191, 186,180,173,170,180,184,162,188, 182,171,166,172,168,185,178,176,
+ 173,183,178,161,180,180,169,177, 177,186,185,178,177,170,187,186, 184,188,165,193,177,186,185,182, 179,178,178,175,185,176,192,2,
+ 51,0,0,0,0,0,0,0, 0,0,0,19,0,0,0,14, 52,0,0,0,3,17,19,1, 0,0,0,0,0,0,0,0,
+ 157,190,169,191,185,175,165,184, 179,175,174,184,175,175,167,166, 181,177,178,189,170,179,183,169, 182,186,200,181,167,183,169,167,
+ 184,179,171,169,182,173,187,181, 182,184,182,175,183,177,188,186, 186,187,182,194,163,176,175,182, 180,172,184,181,179,176,173,180,
+ 175,175,178,172,178,181,187,170, 177,191,189,178,186,179,167,176, 165,169,173,182,190,175,181,181, 187,183,185,177,183,178,177,107,
+ },
+ {128,0,128,128,72,86,86,80, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 48,0,42,44,2,2,2,2, 80,76,2,2,2,2,2,2, 54,0,52,52,2,2,2,2, 96,96,2,2,2,2,2,2,
+ 74,0,76,72,2,2,2,2, 128,128,2,2,2,2,2,2, 128,0,128,128,8,18,18,14, 128,128,28,26,26,26,26,28,
+ 128,0,128,128,2,4,4,2, 128,128,34,30,30,30,32,32, 128,0,128,128,4,14,12,10, 128,128,38,34,34,34,36,36,
+ 0,0,0,0,160,160,160,166, 0,0,160,122,152,116,140,114, 0,0,0,0,150,150,150,148, 0,0,148,150,150,150,150,148,
+ 0,0,0,0,142,138,138,134, 0,0,136,138,134,134,136,134, 0,0,0,0,132,138,142,146, 0,0,136,130,138,138,134,136,
+ 0,0,0,0,136,146,134,142, 0,0,126,134,134,130,144,140, 0,0,0,0,136,138,138,144, 0,0,142,144,128,136,130,124,
+ 0,0,0,0,144,140,134,134, 0,0,160,126,108,102,108,106, 224,128,202,216,130,148,132,136, 204,204,126,118,114,144,122,150,
+ },
+},
+
+{ // Latin2 (315.882M chars) [9]
+ {NULL, NULL, ced_hires_14, ced_hires_14, },
+ 90, 204, 45, 27, 127,
+ {0,0,0,0,0,0,0,0, 0,13,13,0,0,13,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 176,178,79,184,121,96,188,129, 119,153,115,86,132,84,143,152, 125,178,73,182,114,95,188,79, 117,152,115,86,133,93,141,152,
+ 89,170,119,130,151,109,85,137, 171,147,170,100,153,155,102,118, 87,92,116,171,110,99,128,101, 181,128,158,80,133,126,117,98,
+ 90,170,94,124,153,103,95,127, 169,141,170,94,155,155,99,117, 81,92,116,171,113,99,148,82, 180,129,154,79,137,126,117,81,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 185,202,72,207,144,140,210,145, 119,178,153,122,170,124,172,196, 158,217,79,211,140,141,210,120, 115,176,151,152,187,124,173,200,
+ 97,198,155,156,200,127,165,183, 202,208,195,143,194,195,151,143, 149,174,157,197,159,157,199,120, 192,171,178,136,201,170,155,174,
+ 102,203,144,173,201,101,197,171, 203,203,208,143,202,204,139,150, 148,188,166,199,147,166,197,121, 192,191,178,144,189,182,155,121,
+ },
+ {65,0,0,0,0,0,0,0, 0,129,154,0,0,162,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 228,154,181,85,87,110,164,158, 144,155,125,102,191,153,189,148, 131,145,140,130,127,125,117,115, 113,114,177,135,201,88,123,155,
+ 81,200,180,210,197,203,168,188, 175,191,170,190,200,190,208,194, 182,141,204,199,200,181,182,186, 146,188,181,102,112,126,100,120,
+ 110,200,181,209,196,205,167,186, 172,191,169,190,200,189,207,194, 180,135,203,198,199,182,181,191, 143,189,181,91,125,72,102,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 176,176,79,167,122,89,163,118, 119,144,100,124,140,114,142,179, 121,178,71,168,109,88,163,67, 117,144,103,124,140,102,142,179,
+ 91,160,116,126,149,84,190,132, 166,138,143,91,83,185,105,126, 81,118,127,168,91,58,130,99, 171,124,126,82,122,95,120,153,
+ 93,161,107,127,150,87,190,124, 167,128,145,91,82,185,102,129, 83,119,127,170,121,59,134,85, 172,125,112,80,118,93,120,85,
+ },
+ {132,0,128,156,156,156,156,154, 0,0,156,156,156,156,156,156, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 10,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 14,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 132,0,98,130,142,136,102,114, 0,0,172,130,168,162,108,142, 130,0,136,132,52,54,136,130, 0,0,108,174,82,90,170,166,
+ 98,0,90,98,141,143,100,90, 0,0,164,118,170,178,130,128, 100,0,92,100,139,143,108,110, 0,0,166,92,174,156,128,142,
+ 134,0,136,134,36,54,135,135, 0,0,114,168,90,144,168,180, 130,0,130,134,40,56,137,141, 0,0,102,170,80,158,172,156,
+ },
+},
+
+{ // CP1251 (2609.249M chars) [10]
+ {NULL, NULL, ced_hires_10, ced_hires_10, },
+ 190, 219, 69, 19, 128,
+ {61,54,71,50,88,0,0,0, 0,0,73,0,84,68,72,51, 56,82,97,103,74,0,0,0, 0,0,69,0,81,63,70,32,
+ 175,85,84,108,0,54,0,0, 121,94,119,0,0,89,99,124, 0,0,150,148,47,70,0,0, 121,0,117,0,105,2,0,124,
+ 203,190,201,189,197,200,175,191, 205,172,201,198,196,207,207,203, 205,207,204,187,179,172,176,183, 172,169,147,176,170,168,160,171,
+ 201,185,197,186,194,200,174,188, 203,172,197,197,192,205,205,197, 203,204,203,185,172,167,174,181, 170,169,147,176,170,161,157,169,
+ 31,0,88,0,134,0,0,0, 85,0,28,0,0,0,5,0, 0,100,141,125,124,0,0,0, 1,0,16,0,11,11,33,0,
+ 184,2,95,21,0,6,0,0, 87,157,63,0,0,123,138,72, 0,0,102,143,7,102,0,0, 126,0,110,0,84,0,0,130,
+ 157,125,161,130,131,135,111,123, 148,122,140,123,137,137,153,133, 131,147,134,135,118,122,118,114, 110,103,83,118,114,116,116,143,
+ 193,147,189,166,165,195,131,156, 199,187,178,167,181,169,190,141, 170,175,180,173,120,175,145,137, 140,111,88,180,182,106,164,193,
+ },
+ {0,0,0,0,0,0,0,0, 0,123,152,0,0,158,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 223,147,180,67,83,115,163,151, 135,154,99,111,188,166,187,138, 128,140,138,123,122,123,115,111, 108,109,171,133,198,101,111,143,
+ 134,133,114,124,118,126,111,112, 113,129,111,108,118,121,114,117, 124,83,116,126,121,108,110,114, 76,103,91,97,102,121,55,113,
+ 81,143,106,111,119,137,102,102, 116,138,96,102,116,118,119,128, 122,71,118,150,131,126,116,96, 72,98,99,47,123,70,85,0,
+ 48,44,53,47,0,85,0,0, 0,0,66,0,81,17,64,0, 50,75,98,66,98,71,62,54, 0,60,67,96,81,29,65,16,
+ 175,84,89,98,74,0,54,89, 125,68,109,58,57,85,81,121, 74,34,155,158,0,74,29,81, 128,29,110,103,98,0,0,121,
+ 210,181,193,178,187,209,169,174, 206,177,192,194,187,202,211,177, 199,195,203,188,160,171,169,173, 162,156,142,188,185,132,168,185,
+ 213,184,195,179,188,212,171,176, 208,178,193,197,188,203,215,181, 203,197,205,192,162,172,170,174, 162,157,150,190,185,141,170,186,
+ },
+ {128,0,52,78,128,128,128,128, 128,128,74,82,12,26,10,22, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 42,0,2,6,48,58,46,48, 106,84,22,28,2,2,2,2, 56,0,2,20,64,76,62,64, 128,128,56,64,2,8,2,6,
+ 2,0,2,2,2,2,2,2, 128,110,26,34,2,2,2,2, 2,0,2,2,2,6,2,2, 128,128,42,50,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 66,44,2,2,2,2,2,2, 2,0,2,2,2,8,2,2, 80,56,2,4,2,2,2,2,
+ 130,0,112,138,218,228,196,188, 184,166,182,184,138,132,110,112, 144,0,134,138,190,188,202,204, 196,226,158,170,124,124,124,126,
+ 174,0,136,170,182,184,172,174, 168,194,204,146,106,102,84,84, 154,0,152,148,150,166,162,138, 158,178,156,166,124,128,130,130,
+ 146,0,152,154,166,162,144,134, 134,86,98,126,139,139,112,110, 148,0,150,162,176,170,154,134, 130,104,96,128,139,141,112,108,
+ 150,0,154,148,132,128,120,110, 122,130,92,130,6,4,139,139, 150,0,154,152,96,132,122,106, 124,124,82,130,6,2,139,141,
+ },
+},
+
+{ // CP1256 (4291.965M chars) [11]
+ {NULL, NULL, NULL, NULL},
+ 175, 213, 75, 16, 129,
+ {89,120,82,85,70,119,81,87, 78,57,0,61,52,107,87,0, 120,88,122,83,73,125,94,89, 114,96,1,85,34,116,73,0,
+ 173,142,76,85,128,71,110,132, 117,107,30,127,106,73,102,99, 133,84,73,90,108,82,96,130, 115,70,110,116,69,78,76,146,
+ 37,143,167,196,136,188,168,229, 204,148,208,168,191,193,183,197, 170,203,175,195,189,190,178,101, 181,157,204,175,195,197,193,197,
+ 97,209,116,216,198,191,209,110, 85,125,90,85,128,210,82,85, 126,99,116,154,99,137,142,85, 146,71,143,75,117,89,124,44,
+ 157,89,85,86,132,136,106,93, 83,69,0,77,102,72,53,0, 66,103,144,128,127,138,137,111, 67,109,0,87,78,86,35,10,
+ 182,172,102,152,141,111,120,142, 116,155,25,145,107,121,136,120, 155,111,121,104,137,100,101,159, 112,93,117,165,83,117,88,152,
+ 26,167,114,129,77,84,119,189, 178,200,192,163,167,160,159,182, 136,190,152,168,143,156,162,117, 154,133,178,121,152,172,165,172,
+ 181,188,141,190,186,184,180,168, 169,200,157,140,190,187,136,123, 149,117,121,139,145,135,137,118, 122,142,125,128,186,100,134,77,
+ },
+ {116,3,1,0,0,0,0,0, 0,134,143,0,0,157,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 228,134,177,78,100,111,160,151, 140,149,130,106,178,144,169,141, 137,152,147,136,134,132,126,124, 121,123,169,107,200,102,108,114,
+ 99,132,117,131,120,127,115,113, 116,121,112,109,121,123,118,119, 123,94,122,129,129,117,115,119, 115,108,101,110,97,133,114,122,
+ 120,160,152,167,156,164,144,156, 150,149,124,140,162,163,170,153, 151,130,177,173,168,136,148,104, 122,130,139,80,126,96,114,0,
+ 97,97,88,86,83,119,83,85, 98,64,79,60,61,80,88,47, 115,91,101,80,93,122,92,91, 105,117,46,102,80,115,80,37,
+ 173,163,81,85,125,72,114,122, 119,118,106,104,102,81,117,107, 133,87,73,88,113,82,96,129, 115,78,124,125,69,76,76,152,
+ 36,168,139,171,153,157,172,218, 198,200,201,168,187,192,186,204, 174,209,179,200,190,181,185,101, 182,157,199,169,196,190,195,192,
+ 104,226,105,204,209,198,208,118, 101,130,101,80,174,214,92,83, 163,120,130,171,118,155,158,79, 156,70,154,80,108,87,119,81,
+ },
+ {90,0,18,44,128,128,54,56, 102,84,40,60,2,2,2,34, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 42,0,2,2,48,48,10,14, 62,48,10,28,2,2,2,2, 62,0,2,22,72,74,30,34, 80,62,22,42,2,2,2,16,
+ 2,0,2,2,2,2,2,2, 36,24,2,6,2,2,2,2, 2,0,2,2,2,2,2,2, 48,34,2,16,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 30,18,2,2,2,2,2,2, 48,0,2,8,54,56,16,20, 64,52,12,30,2,2,2,4,
+ 178,0,138,160,188,202,138,136, 246,218,168,188,116,124,126,114, 160,0,142,144,184,174,160,162, 212,248,162,204,118,126,126,120,
+ 168,0,140,164,178,172,140,138, 162,176,200,170,106,100,92,136, 172,0,144,162,174,166,138,138, 212,212,170,220,92,90,90,122,
+ 142,0,150,150,92,128,44,72, 114,114,114,108,128,130,134,130, 142,0,150,148,114,142,74,80, 126,116,112,110,134,132,130,132,
+ 140,0,148,142,112,124,154,154, 120,132,116,116,134,132,128,132, 140,0,132,132,126,136,172,174, 158,152,148,140,132,130,128,164,
+ },
+},
+
+{ // CP1250 (456.295M chars) [12]
+ {NULL, NULL, ced_hires_15, ced_hires_15, },
+ 90, 207, 44, 30, 128,
+ {106,94,109,3,114,124,102,101, 0,71,177,95,143,118,167,98, 48,59,96,73,40,99,64,61, 0,67,141,59,108,83,130,60,
+ 178,82,81,175,131,134,115,136, 125,101,117,114,114,90,115,134, 133,104,75,173,117,99,105,135, 120,134,117,119,129,95,129,131,
+ 94,177,121,140,156,127,86,139, 173,151,161,106,170,163,105,122, 92,104,118,172,129,101,131,105, 183,159,162,83,135,145,120,103,
+ 95,177,123,127,157,116,103,130, 171,145,162,96,170,163,102,121, 83,93,119,172,131,103,150,86, 182,159,158,82,139,145,119,83,
+ 162,120,91,35,137,141,111,98, 13,75,196,81,168,151,193,131, 88,128,169,153,152,163,162,136, 10,134,194,112,166,184,194,139,
+ 187,92,75,209,146,163,125,147, 121,160,155,150,112,126,141,198, 160,116,81,213,142,104,106,164, 117,176,153,170,170,126,171,202,
+ 99,200,157,158,202,129,167,185, 204,210,197,145,196,197,153,145, 151,176,159,199,161,159,201,122, 194,173,180,138,203,172,157,176,
+ 104,205,146,175,203,103,199,173, 205,205,211,145,204,206,141,152, 150,191,168,201,150,168,199,123, 194,193,180,146,191,184,157,123,
+ },
+ {122,0,0,0,0,0,0,0, 0,143,154,0,0,165,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 227,153,180,86,97,112,169,159, 147,157,128,106,190,155,189,149, 138,150,147,134,133,129,124,123, 120,121,178,134,203,92,128,154,
+ 110,204,183,200,193,208,171,187, 178,196,172,193,197,190,210,197, 183,143,205,201,201,184,184,176, 150,190,178,114,114,132,107,122,
+ 115,204,184,198,191,209,169,185, 175,197,172,193,196,187,209,197, 181,137,202,200,201,186,184,176, 145,192,178,94,130,77,114,0,
+ 105,89,105,77,114,127,87,104, 65,77,167,87,121,154,174,103, 85,93,106,115,112,126,100,99, 71,123,167,110,121,155,174,104,
+ 177,116,81,166,127,131,117,124, 123,122,102,105,107,115,124,164, 136,109,72,166,119,93,103,132, 119,133,104,122,116,103,122,164,
+ 93,165,121,130,150,102,158,134, 169,143,144,96,85,190,109,129, 88,98,136,169,98,62,135,101, 172,130,135,83,124,100,121,154,
+ 95,167,111,128,151,90,158,126, 170,139,146,91,84,191,105,135, 89,99,137,172,123,63,135,88, 173,130,131,81,120,97,121,86,
+ },
+ {28,0,2,18,2,2,2,2, 14,14,14,22,4,16,2,16, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 22,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 32,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 16,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 18,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 16,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 18,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 130,0,104,120,140,130,110,116, 162,132,114,116,180,146,140,112, 134,0,136,128,84,84,138,128, 108,162,124,148,120,118,180,146,
+ 140,0,112,138,140,134,98,104, 108,128,184,142,144,178,120,158, 136,0,132,138,94,84,138,132, 116,142,142,178,124,144,154,188,
+ 94,0,90,96,141,143,100,90, 180,114,154,138,170,172,130,124, 96,0,94,98,139,143,108,112, 170,106,170,116,174,152,128,136,
+ 130,0,138,132,36,52,135,135, 114,180,122,162,92,140,170,174, 128,0,130,132,40,56,137,141, 94,170,112,180,80,154,172,150,
+ },
+},
+
+{ // Latin5 (322.539M chars) [13]
+ {NULL, NULL, ced_hires_18, ced_hires_18, },
+ 96, 232, 51, 21, 128,
+ {20,0,20,20,20,20,20,20, 20,37,37,20,20,16,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 174,125,94,100,128,93,114,133, 120,100,115,113,111,97,114,106, 129,100,95,102,119,100,101,130, 117,91,117,117,87,95,93,116,
+ 109,114,123,142,149,117,104,183, 111,140,112,106,104,116,108,107, 147,123,109,109,99,127,169,104, 100,88,100,105,175,189,165,102,
+ 108,113,104,109,151,116,97,192, 108,128,102,97,101,120,103,120, 178,123,114,107,105,128,165,98, 101,92,102,87,175,201,186,105,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 183,147,103,153,143,113,121,144, 117,156,143,147,109,122,138,121, 157,113,123,105,138,101,102,161, 114,95,164,166,85,118,91,158,
+ 153,197,154,170,198,186,170,182, 181,207,170,141,118,193,149,136, 201,175,130,196,157,155,197,118, 180,109,177,140,200,218,211,173,
+ 183,202,142,172,199,196,170,169, 171,201,158,141,149,203,138,125, 202,175,150,198,146,156,196,119, 180,143,177,129,187,238,211,108,
+ },
+ {50,6,11,57,0,0,0,0, 0,144,149,0,0,164,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,146,185,81,100,113,168,166, 142,154,126,112,190,161,183,145, 134,151,147,137,134,132,126,124, 121,122,176,137,206,105,128,149,
+ 92,194,178,190,185,190,170,185, 176,197,160,189,202,193,206,184, 174,152,207,197,198,181,179,170, 146,175,180,111,108,128,102,138,
+ 127,196,180,195,187,192,172,183, 173,199,158,205,212,204,220,183, 177,140,211,202,199,186,178,170, 143,189,200,94,140,80,111,5,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 175,126,100,100,127,94,116,124, 121,125,106,108,106,120,123,111, 127,113,88,113,118,96,111,128, 119,95,113,121,111,107,92,126,
+ 118,110,119,175,148,113,108,168, 110,141,110,109,101,124,114,104, 171,129,114,115,94,154,141,101, 100,92,102,106,132,171,177,152,
+ 114,115,112,174,149,118,107,171, 109,124,108,108,100,125,114,97, 190,129,114,113,120,155,143,91, 105,92,113,100,137,200,199,107,
+ },
+ {130,0,102,144,144,142,108,138, 0,0,154,154,154,154,154,154, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 36,0,2,2,2,2,2,2, 0,0,14,38,8,2,8,2, 62,0,2,22,2,10,2,4, 0,0,20,44,12,6,12,2,
+ 26,0,2,2,2,2,2,2, 0,0,2,4,2,2,2,2, 14,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 20,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 4,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 164,0,136,160,116,116,104,106, 0,0,202,178,144,126,130,118, 160,0,142,150,118,124,102,106, 0,0,180,246,168,156,154,124,
+ 90,0,92,92,145,147,90,92, 0,0,156,176,188,168,138,134, 112,0,112,116,141,141,114,116, 0,0,110,138,166,180,166,148,
+ 132,0,138,122,44,54,131,135, 0,0,130,156,124,128,182,162, 128,0,132,132,50,60,135,133, 0,0,112,120,100,130,142,168,
+ },
+},
+
+{ // ISO-8859-11 (489.481M chars) [14]
+ {NULL, NULL, NULL, NULL},
+ 184, 198, 48, 20, 127,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 175,209,195,88,202,83,128,203, 192,152,191,169,88,166,132,138, 156,134,149,172,200,200,176,200, 176,211,199,194,179,150,192,164,
+ 175,205,201,213,136,199,75,202, 176,173,203,200,140,209,144,123, 197,203,215,179,198,198,169,189, 190,187,93,39,18,43,4,72,
+ 210,196,186,186,192,106,151,183, 207,206,151,138,179,120,113,126, 122,127,132,111,120,111,102,99, 96,94,33,88,83,68,64,77,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 184,175,143,82,159,80,91,182, 155,114,141,116,87,135,98,96, 126,88,93,153,172,160,139,156, 134,188,176,156,101,92,163,122,
+ 113,183,179,173,84,159,74,169, 155,135,151,108,92,170,98,140, 166,90,180,144,149,166,80,79, 133,137,84,1,4,52,40,119,
+ 116,99,97,96,100,92,161,101, 174,168,108,110,179,87,92,109, 117,117,114,109,105,106,99,96, 94,95,20,89,93,21,54,98,
+ },
+ {70,0,0,46,0,0,0,0, 0,136,148,0,0,160,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 207,129,181,82,82,93,164,157, 152,152,114,111,166,150,188,146, 128,141,136,126,124,123,115,111, 109,109,165,100,202,96,118,126,
+ 79,118,108,118,109,116,106,106, 104,104,108,101,111,116,105,100, 115,76,111,119,110,92,99,104, 73,98,81,125,114,120,90,106,
+ 61,105,100,99,109,106,92,82, 92,104,93,95,95,106,105,107, 114,58,104,111,104,93,100,89, 80,71,96,55,132,63,99,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 175,206,190,91,199,78,140,206, 190,148,188,166,92,168,152,144, 148,139,143,179,199,196,177,200, 172,213,199,192,172,143,191,162,
+ 175,208,201,213,153,202,62,203, 176,170,200,196,121,209,145,145, 194,206,218,188,201,202,172,189, 193,191,94,64,59,63,57,74,
+ 203,186,178,183,187,119,161,181, 208,206,150,139,187,118,121,123, 127,115,118,111,108,125,108,109, 106,112,71,99,90,69,77,91,
+ },
+ {128,0,148,148,148,148,148,148, 0,0,148,148,148,148,148,148, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,8,8,16,14, 0,0,2,2,2,2,2,36, 2,0,2,2,4,4,14,10, 0,0,2,2,2,2,2,34,
+ 2,0,2,2,2,2,8,4, 0,0,2,2,2,2,2,32, 2,0,2,2,4,4,14,10, 0,0,2,2,2,2,2,44,
+ 2,0,2,2,6,6,14,10, 0,0,2,2,2,2,2,42, 68,0,20,30,128,126,128,112, 0,0,16,12,6,8,14,128,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 166,0,158,164,182,174,180,178, 0,0,126,118,128,132,130,78, 158,0,162,160,132,152,142,144, 0,0,120,122,126,134,130,92,
+ 158,0,162,160,128,160,132,146, 0,0,126,126,124,134,128,90, 166,0,162,160,142,146,146,156, 0,0,134,134,128,68,132,84,
+ 162,0,162,160,138,154,148,150, 0,0,130,134,132,114,112,128, 158,0,164,148,196,208,204,216, 0,0,84,76,64,52,134,254,
+ },
+},
+
+{ // ISO-8859-15 (27.581M chars) [15]
+ {NULL, NULL, ced_hires_21, ced_hires_21, },
+ 86, 217, 37, 21, 127,
+ {0,0,0,0,0,0,0,0, 0,85,85,0,0,85,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 187,137,107,112,127,105,104,142, 106,111,127,124,123,101,126,118, 133,112,107,116,117,113,113,142, 111,105,127,127,92,89,91,126,
+ 121,138,130,159,161,129,133,190, 123,152,123,118,116,144,119,121, 126,135,122,132,111,139,144,115, 113,103,117,117,145,120,135,114,
+ 121,137,113,131,163,128,131,190, 121,141,112,111,114,143,115,133, 123,135,125,131,110,140,160,110, 114,105,118,101,148,119,129,118,
+ 0,0,0,0,0,0,0,0, 0,34,33,0,0,33,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 195,159,116,166,180,125,125,156, 145,168,155,159,121,135,150,134, 169,125,135,118,145,114,115,173, 125,108,176,178,138,141,110,170,
+ 165,209,166,183,211,198,182,194, 193,219,182,153,130,205,161,148, 160,187,142,208,169,167,209,131, 192,122,189,152,212,181,158,185,
+ 195,214,155,184,212,208,182,181, 183,213,170,154,161,215,150,137, 175,187,162,210,158,168,208,131, 192,156,189,142,199,193,155,120,
+ },
+ {23,0,0,70,0,0,0,0, 0,145,155,0,0,165,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,142,175,96,115,116,169,162, 150,154,125,112,184,159,177,144, 147,165,159,150,148,149,139,138, 134,136,166,128,201,103,116,143,
+ 104,197,188,202,195,190,180,196, 185,179,172,189,202,198,215,195, 185,164,216,206,204,173,190,182, 158,170,181,116,107,135,114,133,
+ 130,194,189,197,192,189,178,194, 182,176,171,189,201,194,213,194, 181,151,213,203,201,172,189,182, 155,176,182,107,136,90,117,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 187,138,113,113,149,107,108,134, 123,135,117,120,118,129,131,122, 135,125,101,126,120,109,123,140, 113,107,124,133,118,119,101,138,
+ 132,128,129,187,160,126,122,145, 123,152,121,121,113,136,125,116, 152,141,127,129,106,167,144,114, 113,104,118,117,134,117,122,164,
+ 130,132,121,187,161,130,119,138, 122,137,119,120,112,137,123,109, 153,141,126,130,124,167,145,104, 116,104,130,109,131,116,130,120,
+ },
+ {178,0,128,156,122,126,124,128, 0,0,168,202,168,202,168,202, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 22,0,2,2,2,2,2,2, 0,0,2,14,2,2,2,2, 50,0,2,18,2,2,2,2, 0,0,2,20,2,2,2,6,
+ 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 42,0,2,8,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 38,0,2,4,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 160,0,138,164,108,108,102,104, 0,0,190,160,132,134,120,146, 158,0,144,154,112,118,106,110, 0,0,160,218,160,164,148,160,
+ 88,0,92,94,139,139,94,92, 0,0,152,166,184,176,134,148, 98,0,96,102,139,139,110,110, 0,0,138,172,174,190,154,164,
+ 128,0,138,126,40,48,135,135, 0,0,130,160,128,148,186,180, 120,0,126,120,48,54,139,141, 0,0,136,160,142,188,166,188,
+ },
+},
+
+{ // CP1257 (41.264M chars) [16]
+ {NULL, NULL, ced_hires_20, ced_hires_20, },
+ 84, 222, 39, 20, 128,
+ {112,108,100,0,132,119,103,89, 0,78,64,75,0,77,68,67, 11,35,79,71,21,82,53,35, 0,31,45,33,0,0,49,0,
+ 173,72,85,94,121,69,110,128, 74,94,81,110,109,101,104,75, 124,97,89,88,110,94,97,129, 74,80,81,113,82,88,85,80,
+ 132,139,163,137,149,121,106,164, 124,141,114,153,127,139,154,153, 181,121,165,109,89,101,125,103, 97,82,78,145,112,85,162,98,
+ 138,140,163,106,150,118,115,164, 124,135,112,155,125,133,155,153, 179,115,165,109,88,104,145,100, 130,82,79,145,129,84,161,94,
+ 157,114,85,13,131,136,105,92, 1,81,72,77,10,83,100,70, 81,122,164,148,146,158,157,131, 0,129,85,106,48,81,114,1,
+ 181,85,101,152,141,118,119,142, 101,154,98,145,107,120,136,92, 155,111,121,103,136,99,100,159, 111,92,101,164,84,116,89,89,
+ 171,189,211,72,196,184,158,199, 199,205,85,210,168,152,203,176, 211,170,179,194,91,153,195,116, 172,97,91,199,198,89,204,171,
+ 205,199,217,92,198,194,183,201, 200,199,104,216,166,147,205,177, 211,185,178,196,93,154,194,117, 220,101,91,199,185,90,202,118,
+ },
+ {116,0,0,0,0,0,0,0, 0,139,155,0,0,164,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 228,153,188,77,92,107,206,155, 138,165,122,102,192,146,189,135, 135,150,145,135,132,131,124,123, 120,121,178,143,202,85,105,154,
+ 107,193,189,191,196,191,164,188, 169,204,198,197,197,192,203,181, 177,134,209,211,206,182,187,168, 142,176,171,109,99,129,101,112,
+ 100,195,190,187,196,195,162,186, 165,207,198,197,197,192,202,181, 176,126,208,211,206,184,192,168, 136,178,172,83,124,77,109,0,
+ 117,90,127,119,115,124,85,99, 57,71,0,89,81,86,68,85, 80,86,101,138,134,121,102,96, 36,92,81,109,21,74,91,19,
+ 173,101,111,88,123,70,112,120, 89,109,84,101,109,115,119,115, 133,107,98,98,116,107,100,128, 113,93,107,121,96,107,110,117,
+ 136,143,168,99,146,122,112,156, 159,136,103,157,139,159,155,146, 175,101,144,111,83,103,128,97, 149,82,75,140,115,76,159,150,
+ 138,144,169,93,147,125,112,157, 159,131,104,158,139,160,159,147, 175,102,144,111,84,103,122,91, 149,83,76,143,108,75,158,101,
+ },
+ {100,0,60,88,62,66,62,66, 128,122,100,126,86,90,84,92, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 38,0,2,2,2,2,2,2, 52,46,20,44,6,12,6,14, 70,0,6,34,6,12,6,12, 64,56,32,54,18,22,16,24,
+ 26,0,2,2,2,2,2,2, 4,2,2,2,2,2,2,2, 22,0,2,2,2,2,2,2, 8,4,2,2,2,2,2,2,
+ 26,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 24,0,2,2,2,2,2,2, 4,2,2,2,2,2,2,2,
+ 168,0,132,156,116,128,102,104, 234,200,166,204,168,184,164,158, 152,0,138,142,110,104,124,126, 210,216,166,230,168,168,172,154,
+ 164,0,130,166,110,108,106,106, 148,164,204,170,130,130,130,128, 164,0,140,158,102,98,106,104, 204,214,178,244,158,162,146,152,
+ 102,0,88,106,141,141,94,98, 190,154,154,162,178,188,140,126, 102,0,90,98,141,139,114,106, 152,140,118,176,188,174,148,132,
+ 132,0,134,134,26,40,135,135, 176,186,144,164,96,138,176,190, 126,0,136,122,28,38,135,133, 140,172,130,130,114,154,186,176,
+ },
+},
+
+{ // CP1255 (313.575M chars) [17]
+ {NULL, NULL, NULL, ced_hires_12, },
+ 192, 233, 81, 15, 127,
+ {98,94,106,111,72,121,83,91, 81,65,26,68,0,71,0,69, 81,85,102,93,99,127,102,95, 75,97,17,87,61,82,71,67,
+ 175,124,97,90,58,73,104,127, 119,93,86,110,111,82,101,112, 129,87,94,110,112,86,97,132, 116,73,60,114,71,123,75,120,
+ 119,65,141,119,118,109,112,113, 121,112,61,74,117,67,73,62, 61,98,78,56,68,54,124,148, 38,49,42,32,35,40,36,27,
+ 211,211,193,200,214,221,187,200, 193,220,109,200,214,125,214,131, 205,198,202,108,201,91,192,199, 210,210,202,32,36,81,108,32,
+ 159,117,88,89,134,138,108,95, 85,73,57,78,26,89,0,88, 63,104,145,129,128,139,138,112, 83,110,64,88,33,78,35,6,
+ 184,147,104,154,150,113,122,144, 118,156,77,147,109,123,138,122, 157,113,123,106,139,101,103,161, 114,95,71,167,86,119,91,159,
+ 81,40,33,27,57,50,53,76, 89,84,40,25,97,25,54,51, 36,79,53,49,25,25,27,56, 26,26,27,24,26,39,25,39,
+ 181,180,170,178,203,186,155,176, 171,197,173,144,189,203,159,191, 133,169,170,166,142,163,146,172, 192,178,205,35,59,102,136,39,
+ },
+ {118,0,0,0,0,0,0,0, 0,137,146,0,0,165,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 227,155,187,100,97,112,172,178, 136,158,128,132,188,170,183,153, 137,155,149,140,137,135,129,128, 124,127,173,136,203,112,110,154,
+ 108,138,122,146,128,136,120,118, 124,127,114,112,126,128,125,124, 133,137,129,135,139,118,124,121, 117,118,103,111,143,128,106,131,
+ 142,144,112,120,123,140,110,110, 119,135,106,107,129,123,126,129, 124,108,122,153,134,129,118,103, 84,104,101,80,145,91,116,0,
+ 106,72,108,107,84,120,79,80, 98,59,0,68,0,76,0,66, 101,102,115,98,109,128,101,107, 92,118,68,108,103,102,97,114,
+ 175,126,108,86,95,75,110,121, 124,112,102,101,106,84,113,116, 133,82,132,131,119,84,97,130, 116,74,64,119,72,123,75,135,
+ 128,83,129,119,124,114,117,127, 127,114,43,82,134,59,75,65, 44,112,90,59,50,58,80,109, 50,44,39,43,41,44,40,43,
+ 205,205,192,202,209,223,186,199, 192,223,176,195,211,204,206,190, 204,195,200,171,199,162,190,198, 213,203,213,43,41,82,94,46,
+ },
+ {86,0,44,70,130,132,130,122, 110,94,68,88,90,132,2,10, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 36,0,2,2,40,40,40,40, 60,46,16,34,34,72,2,2, 62,0,2,26,74,72,72,70, 70,54,24,42,44,82,2,2,
+ 60,0,2,22,70,70,68,68, 128,128,100,128,128,128,26,36, 62,0,2,26,66,66,66,68, 128,128,128,128,128,128,48,62,
+ 2,0,2,2,2,2,2,2, 24,10,2,2,2,34,2,2, 2,0,2,2,2,2,2,2, 34,20,2,10,10,46,2,2,
+ 170,0,136,156,182,190,170,166, 248,224,180,222,232,172,106,66, 154,0,140,142,176,164,188,188, 214,246,176,224,176,218,104,106,
+ 166,0,132,164,178,170,168,166, 160,170,204,170,154,182,76,76, 162,0,140,154,182,180,172,168, 224,230,176,228,214,218,94,74,
+ 172,0,148,146,180,182,188,180, 236,188,168,220,190,202,120,120, 158,0,146,158,180,180,180,178, 162,232,182,208,174,184,68,66,
+ 146,0,150,148,106,132,122,126, 110,90,74,98,128,90,131,133, 142,0,150,144,92,132,120,128, 78,90,74,72,124,138,133,127,
+ },
+},
+
+{ // KOI8R (315.553M chars) [18]
+ {NULL, NULL, ced_hires_9, ced_hires_9, },
+ 189, 220, 69, 17, 128,
+ {17,17,17,17,17,17,17,17, 17,39,39,17,17,39,17,17, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 17,17,17,117,17,17,17,17, 17,17,17,17,17,17,17,17, 17,17,17,118,17,25,17,17, 17,17,17,25,17,17,17,47,
+ 155,199,183,172,193,200,169,185, 168,203,166,197,197,191,205,205, 196,169,201,203,201,186,173,196, 170,177,187,171,168,166,181,137,
+ 158,201,188,174,195,200,175,189, 171,204,166,200,198,196,207,207, 201,170,204,206,202,187,175,199, 170,177,189,174,172,166,183,137,
+ 7,7,7,7,7,7,7,7, 7,39,39,7,7,39,7,7, 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,
+ 7,7,7,123,7,7,7,7, 7,7,7,7,7,7,7,7, 7,7,7,74,7,7,7,7, 7,7,7,7,7,7,7,100,
+ 164,191,135,141,166,194,118,163, 175,198,187,177,169,182,165,191, 138,192,164,174,180,171,125,189, 179,178,152,139,98,108,139,73,
+ 131,162,130,110,132,145,129,147, 126,158,123,145,139,148,148,152, 142,148,128,154,139,149,108,167, 114,119,123,105,122,90,109,85,
+ },
+ {0,0,0,0,0,0,0,0, 0,111,171,0,0,149,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 223,146,172,30,25,90,164,138, 116,148,90,87,187,167,194,124, 72,106,101,90,90,85,93,77, 75,75,165,131,188,82,114,147,
+ 125,92,36,71,49,93,30,0, 64,104,0,30,49,47,61,97, 110,0,62,58,56,11,54,71, 45,83,11,70,85,112,54,106,
+ 92,94,57,74,51,94,36,30, 66,106,0,34,57,53,67,98, 113,0,67,61,51,49,56,17, 49,84,41,0,111,57,57,0,
+ 47,49,56,30,0,36,0,0, 0,0,0,25,17,0,0,11, 0,36,17,0,0,0,0,0, 0,0,49,0,0,11,0,0,
+ 25,0,0,123,0,0,0,0, 0,0,0,0,0,0,0,0, 57,17,45,121,30,57,11,36, 66,11,36,41,36,49,58,53,
+ 169,212,182,169,188,212,160,181, 172,207,177,191,196,188,203,214, 181,185,201,197,203,191,170,194, 184,189,174,162,139,157,176,118,
+ 167,209,180,168,186,208,159,179, 171,206,177,190,193,187,201,211, 178,185,198,195,202,188,168,192, 184,187,173,162,133,156,176,117,
+ },
+ {130,0,102,154,154,154,154,154, 154,154,154,154,112,154,116,154, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 92,0,38,74,128,128,128,128, 128,128,128,128,22,34,24,36, 92,0,38,74,128,128,128,128, 128,128,128,128,44,54,46,56,
+ 2,0,2,2,18,22,26,26, 110,128,40,40,2,2,2,2, 2,0,2,2,22,28,32,30, 128,128,48,50,2,2,2,2,
+ 2,0,2,2,16,20,24,24, 128,128,76,80,2,2,2,2, 2,0,2,2,20,24,30,28, 128,128,82,84,2,2,2,2,
+ 154,0,128,178,178,178,178,178, 178,178,178,178,128,178,140,178, 154,0,132,178,178,178,178,178, 178,178,178,178,138,178,178,178,
+ 146,0,154,136,178,182,180,178, 178,178,180,178,134,146,82,92, 148,0,152,168,180,184,180,178, 178,180,180,180,90,106,136,148,
+ 154,0,154,152,154,146,152,150, 134,154,138,86,141,141,6,4, 154,0,154,156,114,150,158,160, 166,172,142,122,139,141,2,12,
+ 136,0,154,156,180,184,184,158, 124,140,110,136,112,106,141,139, 140,0,152,158,180,186,178,172, 130,144,112,138,114,110,139,141,
+ },
+},
+
+{ // GBK (106.219M chars) [19]
+ {NULL, NULL, NULL, NULL},
+ 203, 189, 27, 17, 128,
+ {80,136,138,128,141,121,130,133, 125,121,114,108,141,130,70,75, 125,127,117,120,119,135,127,109, 133,129,119,130,122,125,128,114,
+ 74,204,125,202,219,215,119,114, 118,156,113,86,97,79,82,81, 189,196,199,195,195,208,196,199, 196,203,194,200,201,197,193,195,
+ 192,195,189,194,195,183,196,191, 201,197,207,197,197,197,201,199, 201,191,203,201,198,196,202,200, 134,116,114,132,122,124,118,115,
+ 115,120,116,120,141,121,124,123, 121,120,121,113,125,116,117,110, 117,114,112,113,119,112,124,123, 85,77,84,80,72,84,79,0,
+ 135,112,99,113,112,124,117,103, 124,95,92,89,122,95,0,51, 0,0,0,0,0,0,0,0, 25,1,0,0,0,0,0,0,
+ 1,113,69,80,84,69,59,13, 116,104,61,70,84,69,73,47, 74,88,74,72,72,73,61,73, 51,54,54,59,70,77,75,75,
+ 96,68,68,66,84,83,47,68, 64,64,44,78,54,61,26,26, 64,64,47,59,70,74,70,66, 64,80,61,44,44,69,66,82,
+ 51,57,66,57,64,47,64,54, 18,82,68,66,40,66,70,69, 51,70,40,88,37,37,40,59, 59,40,102,70,73,70,68,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,1,60,0,0,64,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 94,78,82,0,0,0,93,107, 64,60,46,107,76,56,82,44, 29,64,73,39,27,27,18,14, 14,15,18,0,111,60,0,46,
+ 104,95,102,121,118,73,105,75, 96,131,74,109,89,100,103,111, 93,108,98,107,126,103,101,125, 113,99,98,90,89,109,120,88,
+ 77,85,85,92,91,104,101,109, 101,83,74,90,111,98,98,117, 96,93,127,97,106,100,91,99, 91,102,105,84,114,93,91,17,
+ 124,109,99,116,90,100,105,89, 96,95,103,108,93,117,111,110, 98,115,123,116,94,82,112,111, 83,97,106,96,74,105,115,121,
+ 126,196,197,192,196,176,185,184, 195,185,196,194,198,187,184,191, 190,190,180,189,189,194,181,191, 181,191,188,196,189,191,187,192,
+ 186,174,192,193,202,189,193,194, 190,188,189,195,181,187,196,191, 200,178,184,186,189,186,192,180, 193,176,195,182,185,179,182,186,
+ 185,187,190,185,183,187,185,186, 177,190,189,191,187,187,181,180, 178,185,188,196,173,177,186,192, 192,182,197,186,177,187,185,0,
+ },
+ {144,0,168,168,120,122,152,128, 124,116,2,2,2,2,4,4, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 78,0,26,38,2,2,12,8, 128,128,22,22,20,24,26,26, 72,0,20,32,2,2,6,2, 128,128,40,42,40,44,46,46,
+ 74,0,22,34,2,2,6,4, 128,128,40,42,38,44,46,46, 80,0,28,40,6,4,14,10, 128,128,44,46,42,50,52,50,
+ 128,0,128,128,98,102,124,110, 128,128,46,50,46,54,56,54, 128,0,128,128,128,128,128,128, 128,128,38,40,38,44,46,44,
+ 0,0,0,0,236,252,250,242, 230,226,122,106,114,124,130,128, 0,0,0,0,252,238,254,254, 240,234,112,116,106,116,144,130,
+ 0,0,0,0,242,220,220,232, 106,108,136,130,128,116,126,114, 0,0,0,0,206,208,218,216, 82,88,128,126,128,126,126,132,
+ 0,0,0,0,204,218,220,208, 84,82,122,128,128,128,132,132, 0,0,0,0,212,214,210,210, 92,88,124,128,126,136,120,126,
+ 0,0,0,0,208,210,204,206, 200,208,136,124,124,122,126,126, 0,0,0,0,198,202,200,218, 200,196,128,122,132,128,130,120,
+ },
+},
+
+{ // Greek (109.816M chars) [20]
+ {NULL, NULL, ced_hires_11, ced_hires_11, },
+ 186, 219, 72, 19, 128,
+ {0,0,0,0,0,0,0,0, 0,24,24,0,0,24,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 174,64,123,80,13,49,101,123, 114,89,0,152,107,83,76,53, 122,81,71,65,105,84,180,126, 183,174,186,111,183,56,165,168,
+ 117,206,178,191,194,206,163,186, 183,198,202,195,198,199,165,203, 201,196,56,205,210,185,180,183, 154,177,136,87,180,183,177,186,
+ 157,202,171,187,188,204,165,188, 183,199,199,194,197,198,161,202, 194,196,154,201,208,185,175,180, 147,179,136,79,183,166,168,0,
+ 0,0,0,0,0,0,0,0, 0,34,34,0,0,34,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 183,58,94,153,91,63,120,143, 117,155,71,146,108,122,78,71, 156,112,122,105,105,84,83,160, 96,97,84,166,98,118,76,75,
+ 70,162,129,130,139,152,103,165, 133,144,142,123,130,144,98,158, 139,113,63,159,134,140,122,126, 98,117,81,46,175,143,180,165,
+ 64,199,125,122,122,176,111,189, 115,196,146,142,141,197,113,188, 137,142,206,133,145,191,115,124, 78,148,115,74,181,161,152,93,
+ },
+ {98,0,0,0,0,0,0,0, 2,135,160,0,67,161,42,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 225,142,174,72,92,107,173,153, 133,152,109,98,183,147,185,142, 132,150,145,135,133,131,124,123, 119,121,167,138,195,117,108,120,
+ 78,122,108,129,112,120,111,107, 109,113,108,105,115,117,111,121, 118,81,119,121,112,95,108,105, 80,100,85,102,97,121,98,109,
+ 109,116,101,104,112,116,102,95, 104,115,100,100,121,112,116,124, 116,72,112,117,112,104,110,87, 82,90,98,64,123,66,101,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 174,42,84,78,172,159,104,116, 116,102,153,94,101,82,171,60, 122,76,68,71,118,78,185,127, 180,182,189,143,179,67,177,172,
+ 111,207,161,179,175,201,159,197, 167,203,190,188,186,197,160,205, 189,198,0,199,200,190,174,173, 145,183,137,110,188,186,184,191,
+ 66,211,163,182,177,206,160,199, 172,206,192,193,189,200,165,208, 196,203,194,200,201,194,176,176, 146,185,138,110,182,181,174,0,
+ },
+ {146,0,130,170,170,170,170,168, 0,0,166,170,136,168,130,154, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 34,0,2,6,52,58,56,62, 0,0,4,2,2,2,2,2, 4,0,2,2,22,28,24,32, 0,0,14,2,2,2,2,2,
+ 2,0,2,2,2,2,2,4, 0,0,6,2,2,2,2,2, 2,0,2,2,2,6,2,8, 0,0,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,6, 0,0,2,2,2,2,2,2, 2,0,2,2,2,10,6,12, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 166,0,136,172,188,188,186,188, 0,0,186,116,110,106,106,106, 164,0,146,164,180,178,180,174, 0,0,98,100,140,144,98,104,
+ 146,0,152,154,164,164,162,156, 0,0,52,142,139,137,118,112, 148,0,152,154,150,144,142,134, 0,0,48,136,139,131,120,126,
+ 150,0,152,146,92,128,132,126, 0,0,144,72,30,124,139,139, 150,0,152,150,88,124,136,128, 0,0,66,76,4,118,141,139,
+ },
+},
+
+{ // JIS (138.804M chars) [21]
+ {NULL, NULL, NULL, NULL},
+ 37, 154, 2, 1, 129,
+ {0,0,0,0,0,0,0,0, 0,99,99,0,0,99,107,197, 0,0,0,0,0,0,0,0, 0,0,0,254,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {119,0,0,0,0,0,0,0, 0,53,49,0,0,49,0,0, 0,0,0,0,0,0,0,0, 0,0,0,197,0,0,0,0,
+ 73,56,65,75,246,65,49,49, 243,61,49,49,49,49,49,49, 49,56,61,49,49,49,49,49, 49,49,56,56,49,49,49,49,
+ 49,49,49,49,49,49,49,49, 49,49,49,49,49,56,143,49, 49,49,97,49,75,49,59,49, 49,49,61,49,49,53,56,53,
+ 49,98,49,53,49,49,49,94, 49,49,49,49,63,69,74,49, 49,49,53,65,49,49,49,49, 49,49,49,49,49,56,53,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {130,184,30,216,132,172,164,208, 0,0,0,0,0,0,0,0, 128,2,128,38,128,122,124,104, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // CP1254 (20.130M chars) [22]
+ {NULL, NULL, ced_hires_18, ced_hires_18, },
+ 97, 228, 51, 26, 128,
+ {105,102,117,110,111,122,102,106, 97,93,168,92,91,102,30,86, 53,83,126,96,72,96,70,66, 56,74,134,61,61,50,82,84,
+ 173,124,96,100,127,93,113,131, 120,100,114,112,110,97,113,106, 131,100,95,101,117,100,100,131, 116,92,116,116,90,96,94,114,
+ 108,154,124,147,148,125,114,181, 110,139,111,106,104,133,107,106, 146,124,109,108,100,126,167,105, 99,95,111,104,173,187,163,102,
+ 108,154,121,114,150,117,103,190, 108,128,103,98,103,136,103,119, 177,122,113,108,105,127,163,98, 101,94,113,91,173,200,184,105,
+ 157,114,89,88,132,136,106,94, 88,90,191,83,103,92,21,83, 80,123,164,148,146,158,157,131, 101,129,189,106,99,97,60,118,
+ 182,145,102,152,141,111,120,142, 116,154,142,145,107,121,136,120, 155,111,121,104,137,100,101,159, 112,95,162,165,88,117,91,157,
+ 152,195,152,169,197,184,168,180, 179,205,169,140,116,192,148,134, 200,173,128,194,156,153,196,117, 179,108,175,138,198,216,209,171,
+ 181,200,141,171,198,195,168,167, 169,200,157,140,147,201,136,123, 200,173,148,196,144,154,194,118, 178,142,175,128,185,237,209,106,
+ },
+ {116,4,4,52,4,0,0,0, 0,144,148,0,0,163,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 225,145,184,86,101,112,168,165, 141,153,125,111,189,160,182,143, 136,151,146,136,133,132,125,124, 121,122,175,136,205,105,128,148,
+ 108,194,176,188,184,192,169,184, 174,197,159,189,200,191,205,183, 174,151,205,195,198,180,178,169, 146,174,178,113,109,127,105,136,
+ 126,195,178,194,186,194,171,181, 172,199,157,204,211,202,219,182, 177,139,210,201,200,185,177,168, 141,187,198,94,139,85,111,0,
+ 103,94,114,122,104,131,103,108, 103,87,157,89,94,91,51,79, 84,99,153,107,139,122,104,101, 94,118,157,108,103,94,118,122,
+ 173,126,101,99,126,94,115,123, 121,124,114,109,106,119,122,111, 133,112,90,112,118,97,110,129, 118,94,113,120,110,107,93,127,
+ 117,118,123,173,147,114,112,166, 112,142,110,109,107,167,114,104, 170,127,113,114,95,153,141,101, 99,92,103,107,131,170,176,150,
+ 116,123,113,173,147,118,110,169, 112,133,111,107,99,168,113,98, 188,128,113,112,119,153,142,91, 104,92,117,99,135,199,197,107,
+ },
+ {84,0,50,70,50,58,42,52, 90,82,80,100,68,64,68,44, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 36,0,2,2,2,2,2,2, 32,24,16,38,6,2,6,2, 62,0,2,22,2,10,2,4, 36,28,22,44,12,8,10,2,
+ 26,0,2,2,2,2,2,2, 2,2,2,4,2,2,2,2, 16,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 20,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 4,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 140,0,112,126,144,140,106,118, 180,152,152,160,192,130,148,104, 142,0,134,130,100,98,134,132, 138,168,142,178,148,152,184,152,
+ 164,0,136,162,116,118,104,106, 140,152,204,176,140,126,126,118, 160,0,144,152,118,124,102,106, 174,182,180,238,162,154,150,126,
+ 92,0,94,92,145,149,90,92, 190,154,156,172,184,168,132,134, 112,0,114,118,143,143,116,118, 140,146,112,138,166,182,162,148,
+ 132,0,138,124,44,56,131,135, 142,176,130,156,120,128,176,162, 128,0,132,132,50,60,137,135, 130,160,114,122,98,130,138,168,
+ },
+},
+
+{ // CP1253 (37.682M chars) [23]
+ {NULL, NULL, ced_hires_11, ced_hires_11, },
+ 186, 218, 73, 21, 128,
+ {98,79,82,91,78,122,85,90, 0,82,68,76,0,75,0,65, 79,105,101,115,101,127,107,94, 0,98,0,91,0,78,48,0,
+ 174,70,182,88,122,76,110,130, 119,95,70,153,108,88,101,71, 130,87,79,77,106,197,107,129, 184,175,187,114,184,75,166,169,
+ 118,206,179,192,195,207,164,187, 184,199,203,195,198,200,165,203, 202,197,70,205,210,185,180,184, 155,178,137,90,181,184,178,187,
+ 86,203,172,188,189,203,161,188, 181,199,199,194,198,198,162,203, 195,196,156,202,208,186,176,181, 148,179,137,85,184,167,169,70,
+ 158,116,88,89,133,138,107,95, 0,84,72,80,9,91,0,87, 58,101,143,127,125,137,136,110, 0,108,60,84,28,75,32,6,
+ 183,68,95,153,143,112,121,144, 117,156,66,147,109,122,138,78, 157,112,123,105,106,101,102,161, 98,99,87,166,100,118,81,80,
+ 77,163,130,131,140,152,105,166, 133,145,142,124,131,145,100,159, 140,114,66,160,135,141,123,127, 100,118,86,68,176,144,181,166,
+ 74,200,125,123,122,177,112,190, 115,197,147,142,142,198,114,189, 137,143,206,134,146,192,116,125, 82,149,116,80,182,162,153,66,
+ },
+ {118,0,0,0,0,0,0,0, 0,138,161,0,0,163,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,143,175,80,96,111,174,154, 134,154,123,101,184,149,186,143, 136,152,147,137,135,133,126,125, 122,123,168,139,197,118,114,121,
+ 107,133,118,132,121,129,117,114, 119,125,113,112,122,124,119,128, 125,97,124,130,131,115,120,120, 116,111,103,108,101,126,102,112,
+ 111,143,112,125,126,141,110,108, 120,135,105,107,128,122,124,134, 122,86,123,151,133,128,120,103, 90,103,102,77,126,78,111,0,
+ 91,79,78,82,79,127,85,84, 0,77,0,78,58,76,0,63, 76,87,132,84,113,123,101,97, 0,101,76,105,55,86,59,12,
+ 175,74,185,85,123,84,113,121, 119,111,74,101,104,87,118,74, 133,87,79,85,119,144,95,130, 181,183,190,144,180,79,178,172,
+ 112,208,162,180,176,202,160,198, 168,204,190,189,187,198,160,206, 190,199,70,200,200,190,175,174, 145,183,138,111,190,189,185,193,
+ 80,212,167,183,177,208,161,200, 173,208,193,194,190,201,165,209, 197,204,194,200,202,195,177,177, 147,186,139,111,184,182,175,72,
+ },
+ {120,0,80,110,184,184,160,166, 142,128,86,68,44,50,40,48, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 20,0,2,2,32,36,28,32, 64,50,2,2,2,2,2,2, 2,0,2,2,10,14,6,10, 78,62,14,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 70,56,6,2,2,2,2,2, 2,0,2,2,2,2,2,2, 58,46,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 32,20,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 38,26,2,2,2,2,2,2,
+ 168,0,138,162,188,202,172,168, 248,222,164,132,100,104,104,108, 150,0,142,148,180,174,188,192, 212,242,158,156,116,114,118,120,
+ 162,0,136,170,180,178,174,172, 148,160,176,108,132,134,106,104, 162,0,146,164,172,168,166,156, 118,142,100,98,134,138,126,114,
+ 144,0,152,152,156,152,144,136, 116,120,132,142,139,135,116,112, 146,0,150,152,142,134,126,118, 106,104,132,134,139,131,120,126,
+ 148,0,152,144,90,118,116,110, 122,122,68,86,30,122,137,139, 150,0,152,148,90,116,122,112, 122,132,66,88,12,118,139,137,
+ },
+},
+
+{ // CP932 (5.390M chars) [24]
+ {NULL, NULL, NULL, NULL},
+ 151, 140, 55, 26, 129,
+ {73,188,237,214,159,83,84,109, 191,199,191,194,200,199,202,199, 190,190,186,192,182,191,188,186, 105,102,112,102,102,102,107,115,
+ 101,107,119,112,119,135,100,99, 106,106,107,101,103,109,105,120, 121,117,114,106,104,110,111,112, 114,105,127,110,129,118,108,105,
+ 123,109,100,111,120,109,109,99, 105,106,114,108,122,112,116,106, 102,102,113,101,100,99,102,108, 110,108,119,107,108,116,122,117,
+ 117,116,111,115,112,109,113,115, 113,111,167,178,184,104,106,183, 160,142,115,156,149,130,163,163, 184,191,154,133,75,97,94,93,
+ 0,223,195,229,107,12,47,131, 44,180,189,184,179,185,191,185, 177,166,168,174,170,173,169,172, 162,86,89,86,93,87,87,91,
+ 99,113,100,104,106,122,111,98, 99,99,99,99,99,99,99,99, 105,99,100,99,98,99,99,102, 124,99,99,99,106,103,99,99,
+ 100,99,99,98,105,99,99,99, 99,101,99,99,112,99,98,99, 99,100,99,99,99,98,102,99, 99,103,99,99,99,104,106,104,
+ 115,99,101,108,103,106,99,99, 99,103,97,0,23,91,96,0, 105,107,95,165,148,101,154,130, 96,175,132,116,100,109,98,101,
+ },
+ {87,0,0,0,0,0,0,0, 0,84,91,0,0,87,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 114,86,99,84,84,84,86,86, 90,93,85,89,87,94,87,86, 91,88,89,86,85,85,84,84, 84,84,85,87,119,84,84,84,
+ 199,198,192,189,158,189,180,168, 171,180,178,171,176,161,191,183, 172,169,179,177,178,160,185,181, 190,169,175,200,180,158,185,172,
+ 177,159,190,175,162,179,173,191, 177,187,182,158,184,167,174,182, 179,167,177,179,178,179,187,176, 174,176,173,178,169,174,176,0,
+ 175,177,175,172,170,176,176,180, 169,182,188,188,178,179,172,172, 159,180,158,195,174,169,176,176, 162,162,166,158,168,174,176,177,
+ 179,175,195,161,185,163,180,172, 181,189,190,180,175,184,173,177, 166,185,175,184,166,192,178,192, 166,176,169,178,190,190,180,180,
+ 168,183,176,163,192,193,192,176, 191,192,177,162,203,189,171,167, 173,162,169,159,174,148,173,170, 163,160,175,166,188,175,165,174,
+ 186,163,176,177,162,176,185,182, 187,190,188,161,175,174,179,176, 192,187,162,164,156,172,160,145, 155,145,181,167,175,93,93,93,
+ },
+ {84,0,82,82,2,2,4,6, 2,2,8,8,4,18,2,2, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 128,0,126,120,6,10,10,14, 22,24,18,18,14,28,20,30, 128,0,128,126,2,8,8,12, 22,26,18,18,14,30,20,30,
+ 128,0,128,128,6,10,12,14, 26,28,22,22,18,32,24,34, 128,0,128,128,8,12,14,16, 26,30,22,22,18,32,24,34,
+ 68,0,44,44,2,2,2,2, 26,30,24,22,18,34,24,34, 66,0,40,42,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 0,0,0,0,140,138,140,138, 132,130,132,132,136,132,134,128, 0,0,0,0,134,140,142,142, 130,134,132,134,124,134,132,142,
+ 222,128,252,248,124,128,128,130, 148,136,134,126,124,132,86,120, 214,128,240,244,136,132,130,132, 136,118,136,132,126,152,84,118,
+ 212,128,244,242,126,130,132,134, 136,128,130,128,118,156,82,122, 212,128,242,242,126,130,132,134, 136,130,128,144,130,142,66,122,
+ 0,0,0,0,134,144,136,142, 138,144,128,150,72,88,76,86, 184,180,164,164,126,140,126,154, 134,144,126,132,124,128,136,134,
+ },
+},
+
+{ // Hebrew (10.753M chars) [25]
+ {NULL, NULL, NULL, NULL},
+ 196, 235, 78, 9, 128,
+ {0,0,0,0,0,0,0,0, 0,63,63,0,0,63,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 165,0,80,80,80,80,80,80, 80,82,80,80,82,89,80,80, 85,80,80,80,86,80,80,82, 80,80,83,80,80,80,80,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,80,
+ 205,203,189,201,216,218,185,196, 190,220,185,190,212,212,202,200, 199,194,199,175,193,168,187,196, 214,202,217,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,77,77,0,0,77,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 145,0,87,87,105,87,87,87, 88,138,88,88,88,89,105,87, 89,88,88,87,91,87,87,118, 87,87,87,88,88,89,87,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,87,
+ 194,196,170,172,206,189,164,179, 163,185,130,182,196,152,197,145, 174,174,179,130,174,124,162,174, 179,192,183,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,136,147,0,0,157,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 227,138,183,80,80,95,170,157, 104,163,103,95,150,163,151,133, 86,103,112,90,86,90,87,88, 85,86,126,96,204,95,106,111,
+ 82,84,81,84,82,82,81,81, 82,84,81,81,81,81,82,82, 82,80,85,94,81,81,80,82, 81,80,82,83,104,130,83,93,
+ 114,87,82,84,83,83,82,81, 82,88,80,81,82,82,82,86, 83,81,84,85,88,82,81,85, 80,82,80,81,92,108,97,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 165,0,81,80,80,80,80,81, 80,82,80,80,81,81,80,80, 86,80,81,80,85,80,80,83, 80,80,81,80,80,80,80,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,80,
+ 209,210,192,200,210,223,187,198, 189,223,131,200,211,161,212,142, 207,193,203,135,198,112,191,196, 208,208,204,0,0,0,0,0,
+ },
+ {180,0,128,152,204,204,204,202, 0,0,202,202,0,164,118,124, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 54,0,2,14,128,84,104,102, 0,0,66,128,0,128,2,2, 128,0,36,58,128,128,128,128, 0,0,98,128,0,128,4,16,
+ 128,0,128,128,128,128,128,128, 0,0,128,128,0,128,128,128, 128,0,76,98,128,128,128,128, 0,0,128,128,0,128,50,62,
+ 2,0,2,2,16,2,8,10, 0,0,2,38,0,78,2,2, 2,0,2,2,26,10,18,20, 0,0,6,56,0,98,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 154,0,146,158,230,204,230,232, 0,0,186,226,0,186,128,130, 186,0,128,164,228,226,226,226, 0,0,226,226,0,186,118,124,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 164,0,128,152,188,188,188,186, 0,0,186,186,0,148,116,124,
+ 150,0,150,150,136,142,142,136, 0,0,130,118,0,118,130,132, 154,0,150,154,146,158,148,158, 0,0,124,124,0,122,132,128,
+ },
+},
+
+{ // KOI8U (14.358M chars) [26]
+ {NULL, NULL, ced_hires_9, ced_hires_9, },
+ 189, 220, 69, 18, 129,
+ {87,87,87,87,87,87,87,87, 87,95,95,87,87,95,87,87, 84,84,84,84,84,84,84,84, 84,84,84,84,84,84,84,84,
+ 87,87,87,119,129,87,162,137, 87,87,87,87,87,90,87,87, 87,87,87,119,131,87,163,137, 87,87,87,87,87,92,87,87,
+ 156,198,183,173,193,200,169,185, 168,203,166,197,197,191,205,205, 196,169,201,203,201,185,173,196, 170,177,187,171,168,165,181,137,
+ 158,200,188,175,195,200,175,189, 171,204,166,200,198,196,207,207, 201,170,204,206,202,187,175,199, 170,177,189,174,172,166,183,137,
+ 82,82,82,82,82,82,82,82, 82,95,95,82,82,95,82,82, 56,56,56,56,56,56,56,56, 56,56,56,56,56,56,56,56,
+ 82,82,82,124,131,82,157,142, 82,82,82,82,82,83,82,82, 82,82,82,87,89,82,115,97, 82,82,82,82,82,82,82,103,
+ 164,191,135,141,166,194,119,163, 175,198,187,177,169,181,165,191, 138,192,163,174,180,171,125,189, 179,178,152,139,101,110,139,87,
+ 131,161,130,111,132,145,129,147, 126,157,123,145,139,148,148,152, 142,148,128,154,139,149,110,167, 115,120,124,107,123,95,110,92,
+ },
+ {0,0,0,0,0,0,0,0, 0,112,170,0,0,149,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 223,146,172,86,86,96,164,138, 117,148,96,95,187,167,194,124, 88,108,104,96,97,94,99,91, 91,91,165,131,188,92,116,147,
+ 126,98,86,90,87,99,86,86, 87,107,86,86,86,87,87,101, 112,86,87,87,86,86,87,91, 86,93,86,87,94,113,86,107,
+ 97,99,87,90,86,99,86,86, 87,108,86,86,87,87,88,102, 114,86,88,87,86,86,87,86, 86,94,86,86,113,86,87,0,
+ 87,87,87,86,86,86,86,86, 86,86,86,86,86,86,86,86, 86,86,86,86,86,86,86,86, 86,86,86,86,86,86,86,86,
+ 86,86,86,123,128,86,170,136, 86,86,86,86,86,87,86,86, 88,86,87,121,127,88,168,135, 88,86,86,87,87,87,86,87,
+ 169,212,182,169,188,211,160,181, 172,207,177,191,196,188,203,214, 181,185,201,197,203,191,170,194, 184,189,174,162,139,157,176,119,
+ 167,209,179,168,186,208,159,179, 171,205,177,190,193,187,202,210, 178,185,198,195,202,188,168,192, 184,187,173,162,133,156,176,118,
+ },
+ {132,0,98,138,200,200,200,198, 200,200,174,176,92,102,94,104, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 44,0,2,24,84,92,96,96, 128,128,46,48,2,2,2,2, 42,0,2,24,84,90,96,94, 128,128,88,90,14,24,16,26,
+ 2,0,2,2,10,14,16,16, 36,36,2,2,2,2,2,2, 2,0,2,2,16,20,22,22, 46,46,2,2,2,2,2,2,
+ 2,0,2,2,8,12,14,14, 128,128,26,28,2,2,2,2, 2,0,2,2,14,16,20,18, 112,128,30,32,2,2,2,2,
+ 174,0,126,164,224,224,224,222, 224,224,180,182,112,124,116,126, 200,0,128,168,224,224,224,222, 224,224,182,184,114,126,118,128,
+ 146,0,154,150,184,194,200,194, 224,224,170,142,134,140,78,88, 158,0,142,170,224,224,224,222, 224,224,140,172,104,106,136,142,
+ 156,0,154,152,148,140,142,142, 118,120,138,74,139,139,16,24, 154,0,154,156,130,146,148,152, 126,124,138,84,139,141,18,30,
+ 136,0,154,156,180,186,186,174, 116,118,108,138,112,106,141,139, 140,0,152,158,182,184,182,178, 122,122,112,138,114,110,139,141,
+ },
+},
+
+{ // ISO-8859-5 (4.566M chars) [27]
+ {NULL, NULL, NULL, NULL},
+ 178, 203, 63, 18, 128,
+ {0,0,0,0,0,0,0,0, 0,73,73,0,0,73,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 174,139,153,105,121,99,146,124, 153,123,130,127,103,100,117,115, 203,190,202,193,196,200,173,190, 205,165,202,200,198,206,204,202,
+ 204,206,200,185,176,172,175,186, 177,166,129,174,167,168,175,170, 198,179,195,186,191,199,170,186, 202,165,197,197,190,203,202,194,
+ 202,202,199,184,167,167,172,183, 170,164,142,174,169,160,162,170, 145,139,152,104,117,99,146,124, 151,121,129,126,103,100,117,115,
+ 0,0,0,0,0,0,0,0, 0,96,96,0,0,96,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 183,107,100,119,101,102,114,101, 108,107,102,105,110,123,100,105, 155,132,162,147,135,132,126,125, 144,118,136,128,135,134,143,136,
+ 129,148,131,134,127,123,119,121, 119,113,118,126,116,118,119,138, 196,133,191,156,166,192,126,148, 197,189,175,160,180,171,185,142,
+ 170,174,180,171,119,174,145,137, 136,107,110,179,181,109,162,193, 112,129,101,118,119,103,147,123, 126,105,102,131,111,99,125,102,
+ },
+ {0,0,0,0,0,0,0,0, 0,114,146,0,0,157,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 222,138,169,99,103,106,165,146, 132,152,105,105,185,162,182,129, 128,138,133,123,122,122,115,115, 112,113,166,133,204,103,107,138,
+ 107,127,114,123,114,133,112,117, 112,112,116,112,118,118,113,127, 126,101,118,122,116,107,107,118, 101,108,104,104,106,112,100,115,
+ 103,127,109,118,116,128,106,107, 105,117,110,111,115,113,118,136, 126,100,117,119,115,110,109,117, 100,113,108,99,124,100,99,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 174,147,138,136,115,100,153,120, 139,118,125,128,109,100,117,138, 209,174,192,178,187,205,163,176, 205,178,190,191,185,200,208,172,
+ 198,193,199,188,157,168,168,173, 166,150,130,185,182,133,166,183, 215,178,196,181,189,211,165,178, 208,178,192,200,187,202,213,176,
+ 204,195,202,197,156,173,167,173, 161,154,133,189,185,135,171,185, 146,148,138,136,116,99,155,119, 139,120,128,128,109,103,118,137,
+ },
+ {176,0,130,152,216,216,216,214, 0,0,182,122,134,118,130,216, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 36,0,2,2,42,50,44,50, 0,0,16,2,2,2,2,24, 2,0,2,2,2,2,2,2, 0,0,18,2,2,2,2,26,
+ 2,0,2,2,2,4,2,4, 0,0,32,2,2,2,2,42, 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 2,0,2,2,2,8,2,8, 0,0,2,2,2,2,2,2, 46,0,2,4,54,64,54,64, 0,0,38,2,2,2,2,48,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 178,0,132,168,188,188,180,186, 0,0,192,126,130,94,98,138, 140,0,150,154,182,178,186,182, 0,0,122,138,138,120,116,102,
+ 146,0,146,160,198,200,184,190, 0,0,126,138,140,118,114,104, 148,0,154,154,116,130,140,130, 0,0,78,28,40,138,138,134,
+ 156,0,154,148,128,138,132,134, 0,0,88,38,50,138,140,144, 150,0,152,140,186,198,198,204, 0,0,142,88,100,130,142,182,
+ },
+},
+
+{ // CP874 (90.889M chars) [28]
+ {NULL, NULL, NULL, NULL},
+ 183, 197, 49, 21, 127,
+ {105,102,97,58,82,121,0,91, 87,94,89,61,69,81,0,58, 84,99,96,131,98,112,94,97, 77,90,89,80,56,80,84,0,
+ 175,209,195,89,202,83,128,203, 192,151,191,169,87,166,132,138, 156,134,149,172,200,200,176,200, 176,211,199,193,179,149,192,164,
+ 175,205,201,213,136,199,77,202, 175,173,203,200,140,209,144,122, 197,203,215,179,198,198,169,189, 190,187,93,58,56,58,53,74,
+ 209,196,186,186,191,106,151,183, 207,205,151,138,179,120,113,126, 122,127,132,111,120,111,102,99, 97,95,58,89,84,72,69,79,
+ 159,116,43,30,27,138,0,65, 1,58,62,0,19,89,0,87, 48,92,134,118,116,128,127,100, 0,39,50,0,17,66,20,0,
+ 184,175,143,83,159,81,91,181, 155,114,141,116,87,135,98,96, 126,89,93,153,172,160,139,156, 133,188,176,156,101,93,162,122,
+ 113,182,178,173,85,158,76,169, 155,135,151,108,93,170,98,140, 166,90,179,144,149,166,81,80, 132,137,85,52,53,62,58,119,
+ 116,99,98,96,101,93,161,101, 173,168,107,110,179,86,93,109, 117,116,114,109,105,106,99,96, 94,95,54,89,94,56,64,98,
+ },
+ {119,0,0,44,0,0,0,0, 0,138,149,0,0,161,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 209,129,181,84,86,94,166,157, 152,153,115,111,167,151,188,147, 133,142,138,128,126,125,117,113, 111,112,165,104,202,96,121,126,
+ 106,131,115,123,116,126,111,111, 114,120,111,105,118,121,113,118, 121,94,117,127,130,114,114,118, 116,109,100,126,115,121,92,108,
+ 78,142,110,112,119,136,104,98, 116,133,97,101,116,118,116,127, 119,79,119,150,131,126,114,102, 84,97,98,64,134,70,101,0,
+ 99,94,75,63,80,121,0,92, 94,84,92,59,78,81,49,49, 77,82,103,84,123,111,96,101, 80,100,87,78,55,86,81,0,
+ 175,206,190,92,199,79,139,206, 190,148,188,166,94,168,152,144, 148,139,143,179,199,195,177,200, 172,213,199,192,172,143,191,161,
+ 174,207,201,213,153,202,68,203, 176,170,200,196,121,209,144,145, 194,206,217,188,201,202,172,189, 193,191,94,68,66,68,64,77,
+ 203,186,177,183,187,119,161,181, 208,206,149,139,187,118,121,123, 127,115,118,111,108,125,108,109, 106,112,74,99,91,72,79,91,
+ },
+ {110,0,78,92,172,172,158,156, 114,122,40,36,30,34,38,118, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 54,48,2,2,2,2,2,36, 2,0,2,2,2,2,2,2, 54,46,2,2,2,2,2,34,
+ 2,0,2,2,2,2,2,2, 54,46,2,2,2,2,2,34, 2,0,2,2,2,2,2,2, 64,58,2,2,2,2,2,44,
+ 2,0,2,2,2,2,2,2, 62,56,2,2,2,2,2,42, 66,0,18,30,108,104,86,80, 128,128,16,12,6,8,14,128,
+ 174,0,152,160,182,202,168,162, 246,218,114,92,92,80,144,174, 156,0,156,144,186,176,196,194, 204,246,128,124,120,72,138,184,
+ 164,0,158,164,172,166,158,160, 106,118,126,118,128,132,130,82, 156,0,162,160,124,144,118,126, 140,136,120,122,126,134,130,94,
+ 156,0,162,160,120,152,110,128, 92,114,126,124,126,134,128,90, 164,0,160,162,132,138,124,138, 88,116,134,134,128,66,132,86,
+ 160,0,160,160,130,146,126,132, 98,116,132,134,134,114,112,128, 156,0,164,148,202,212,190,188, 168,192,88,80,70,70,134,254,
+ },
+},
+
+{ // ISO-8859-13 (0.207M chars) [29]
+ {NULL, NULL, ced_hires_20, ced_hires_20, },
+ 87, 221, 44, 20, 128,
+ {0,0,0,0,0,0,0,0, 0,140,140,0,0,140,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 172,141,141,141,142,142,141,144, 141,141,141,141,141,141,141,141, 142,141,141,141,141,141,141,144, 141,141,141,142,141,141,141,141,
+ 145,148,162,142,153,142,141,165, 141,149,141,156,144,147,157,156, 180,141,166,141,141,141,143,141, 141,141,141,151,142,141,162,141,
+ 145,149,162,141,154,141,141,165, 142,147,141,156,144,146,157,156, 178,141,165,141,141,141,151,141, 142,141,141,151,145,141,162,141,
+ 0,0,0,0,0,0,0,0, 0,103,103,0,0,103,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 179,147,141,155,148,163,142,148, 141,157,141,149,141,141,147,141, 156,141,143,141,164,141,141,160, 142,141,141,164,141,141,141,141,
+ 171,188,209,141,195,182,160,197, 197,203,141,208,168,155,202,175, 209,170,178,192,141,156,194,142, 172,141,141,197,196,141,202,170,
+ 203,197,216,141,196,193,181,199, 198,198,141,215,167,153,203,176, 209,184,177,194,141,156,192,141, 218,141,141,197,184,141,201,147,
+ },
+ {0,0,0,0,0,0,0,0, 0,147,156,0,0,163,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 226,155,187,141,141,141,205,156, 147,165,143,141,191,150,188,145, 145,153,151,146,146,145,143,143, 142,143,177,149,200,141,141,157,
+ 141,192,188,189,194,190,165,186, 169,203,196,195,196,191,201,180, 177,147,207,210,204,181,186,168, 149,175,171,141,141,143,141,141,
+ 141,193,189,186,194,193,163,185, 166,205,196,195,195,191,200,180, 175,144,206,209,204,182,190,168, 147,177,171,141,143,141,141,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 172,142,142,141,143,141,142,142, 141,142,141,141,142,141,142,143, 144,141,141,141,144,142,141,144, 142,141,142,143,141,142,142,143,
+ 146,151,168,141,152,142,142,159, 160,146,141,160,148,161,157,152, 174,141,150,141,141,141,143,141, 154,141,141,149,142,141,160,154,
+ 147,151,168,141,152,142,142,159, 161,143,141,160,148,161,160,152, 174,141,150,141,141,141,143,141, 154,141,141,150,142,141,160,141,
+ },
+ {158,0,124,150,126,130,126,128, 0,0,170,174,162,166,162,166, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 24,0,2,2,2,2,2,2, 0,0,2,6,2,2,2,2, 30,0,2,2,2,2,2,2, 0,0,6,12,2,2,2,4,
+ 18,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 18,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 162,0,130,160,124,122,116,118, 0,0,184,168,156,160,156,160, 160,0,136,152,118,120,118,120, 0,0,168,176,162,164,162,166,
+ 112,0,92,110,143,143,98,102, 0,0,160,164,168,178,152,154, 116,0,94,110,143,141,116,110, 0,0,156,162,178,166,152,152,
+ 130,0,134,134,68,72,135,137, 0,0,158,164,152,156,168,178, 126,0,136,124,74,78,137,135, 0,0,156,162,150,158,178,166,
+ },
+},
+
+{ // Latin4 (1.274M chars) [30]
+ {NULL, NULL, ced_hires_17, ced_hires_17, },
+ 82, 215, 39, 25, 128,
+ {0,0,0,0,0,0,0,0, 0,115,115,0,0,115,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 173,117,116,117,126,117,148,132, 125,177,162,132,116,116,164,120, 128,118,116,117,124,117,148,116, 124,175,162,130,116,116,163,116,
+ 157,149,123,151,148,122,117,138, 167,143,123,118,151,148,118,153, 121,162,122,139,120,119,129,118, 117,116,148,118,124,121,142,117,
+ 157,149,118,127,151,122,117,137, 166,138,123,117,150,149,118,153, 120,162,122,131,121,119,146,117, 118,128,146,116,134,116,142,117,
+ 0,0,0,0,0,0,0,0, 0,70,70,0,0,70,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 181,199,116,118,142,116,174,142, 124,175,196,171,116,126,169,127, 155,213,117,118,137,116,173,125, 122,173,197,168,117,142,170,118,
+ 207,195,152,169,196,183,168,186, 199,205,192,140,210,191,148,201, 147,178,118,155,156,154,195,123, 178,174,175,140,198,117,198,171,
+ 213,200,142,170,197,194,168,195, 200,199,205,141,216,201,138,202, 146,177,121,146,145,154,194,125, 178,220,175,131,185,119,198,126,
+ },
+ {57,0,0,38,0,0,0,0, 0,131,151,0,0,166,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 228,152,180,117,117,121,211,153, 140,167,128,117,189,148,187,138, 132,143,139,132,130,129,124,123, 122,123,176,138,202,117,118,154,
+ 118,186,190,201,200,189,167,191, 171,184,194,192,197,190,201,180, 181,140,208,211,206,171,184,129, 140,156,180,118,122,126,119,124,
+ 127,186,190,200,200,192,165,190, 168,185,194,192,197,189,200,179, 179,135,207,211,206,172,186,130, 136,162,180,118,128,117,119,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 173,117,117,117,128,123,146,126, 126,173,155,140,117,124,156,121, 126,119,117,117,121,134,147,117, 125,173,155,141,136,134,156,135,
+ 164,151,124,120,147,123,118,143, 164,136,118,120,153,166,122,151, 121,140,124,152,118,120,129,119, 117,154,119,117,123,117,140,151,
+ 164,153,120,117,148,124,119,144, 165,128,118,120,153,167,121,156, 125,142,122,153,127,119,128,121, 118,154,121,117,120,117,144,118,
+ },
+ {162,0,122,152,126,130,126,128, 0,0,168,174,170,184,168,186, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 20,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 26,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 36,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 26,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 36,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 142,0,104,138,144,136,100,96, 0,0,180,140,178,180,148,160, 132,0,136,126,68,68,138,130, 0,0,140,176,140,152,182,186,
+ 108,0,90,112,143,143,98,98, 0,0,176,164,182,164,150,154, 118,0,100,112,139,145,112,108, 0,0,180,154,176,174,158,170,
+ 134,0,132,134,46,52,137,137, 0,0,142,182,140,160,182,166, 120,0,140,126,56,60,127,133, 0,0,152,184,148,180,174,174,
+ },
+},
+
+{ // MACINTOSH (7.890M chars) [31]
+ {NULL, NULL, NULL, NULL},
+ 93, 176, 57, 40, 128,
+ {157,149,165,157,149,138,123,160, 152,165,162,148,156,168,162,155, 132,145,140,141,135,131,120,139, 145,132,114,87,138,116,126,107,
+ 122,121,109,120,124,153,124,137, 115,116,118,128,124,104,106,113, 104,117,104,104,109,107,139,107, 126,104,109,117,116,104,105,110,
+ 111,159,167,120,108,142,122,128, 139,154,202,125,151,121,112,145, 122,135,125,128,127,169,111,105, 105,105,106,131,106,115,157,151,
+ 178,165,177,166,176,175,150,161, 175,138,169,172,167,178,180,169, 181,177,176,161,145,151,149,162, 150,150,134,154,149,146,136,108,
+ 187,160,176,222,183,180,186,200, 209,171,190,175,171,176,227,198, 178,155,199,157,162,155,181,198, 159,170,176,133,176,169,177,184,
+ 153,174,115,162,149,204,131,171, 181,164,183,160,143,100,144,155, 100,123,132,132,125,113,123,102, 131,114,100,143,166,100,142,153,
+ 167,156,133,122,124,144,123,171, 183,183,202,158,173,141,144,148, 197,182,196,197,174,218,115,99, 113,111,102,198,116,150,173,165,
+ 163,179,162,158,169,174,179,196, 171,195,195,169,176,142,198,171, 137,149,180,180,122,144,118,130, 126,177,118,147,152,115,134,107,
+ },
+ {42,0,13,0,0,0,0,0, 0,162,166,0,0,180,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 232,145,182,109,125,120,185,162, 148,164,123,123,190,165,187,146, 152,167,161,152,152,149,142,140, 139,140,164,140,212,106,144,148,
+ 114,197,179,197,187,193,175,187, 174,172,164,164,193,190,204,189, 182,167,202,202,200,165,181,159, 152,152,154,137,122,135,112,138,
+ 124,201,182,197,188,200,175,187, 178,184,163,166,195,192,204,192, 182,163,203,213,204,179,184,151, 152,150,156,105,146,106,124,0,
+ 154,118,136,147,125,135,127,149, 150,171,149,158,123,138,165,150, 145,167,173,164,158,153,119,174, 161,141,125,139,166,124,149,128,
+ 121,135,106,117,127,149,123,154, 119,135,121,131,129,103,106,107, 103,121,103,103,108,106,141,104, 125,103,103,109,119,103,106,107,
+ 121,146,166,124,112,147,123,119, 142,156,201,137,158,141,109,143, 126,135,114,134,124,136,109,103, 105,103,103,132,103,152,150,164,
+ 189,156,172,160,169,185,143,151, 179,154,167,169,166,174,187,145, 178,169,174,164,137,148,147,154, 144,138,124,165,164,121,147,108,
+ },
+ {64,0,40,60,48,54,44,48, 40,32,58,74,24,52,16,30, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 30,0,2,2,2,2,2,2, 2,2,2,14,2,2,2,2, 40,0,2,14,2,8,2,2, 14,8,34,52,2,28,2,6,
+ 2,0,2,2,2,2,2,2, 2,2,2,14,2,2,2,2, 16,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,4,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,14,30,2,8,2,2,
+ 110,0,128,112,130,132,130,130, 172,160,146,144,140,156,152,144, 116,0,126,118,72,74,142,134, 166,176,152,142,114,142,138,132,
+ 150,0,140,144,110,108,104,102, 150,144,198,192,158,160,124,142, 118,0,138,130,110,122,126,128, 152,146,184,206,166,166,128,142,
+ 154,0,132,150,124,122,106,110, 110,102,160,166,174,156,114,104, 126,0,130,136,116,114,128,136, 178,160,156,162,138,174,134,142,
+ 118,0,122,114,142,144,108,96, 120,128,142,140,104,154,162,162, 124,0,132,118,132,136,124,118, 114,100,130,140,112,150,162,164,
+ },
+},
+
+{ // GB18030 (0.640M chars) [32]
+ {NULL, NULL, NULL, NULL},
+ 202, 189, 30, 18, 127,
+ {107,143,144,138,146,135,139,141, 137,141,140,132,146,144,130,130, 137,138,134,135,134,142,138,131, 141,139,135,139,136,137,138,133,
+ 130,205,136,202,219,216,134,132, 133,158,133,130,130,130,130,130, 190,196,199,196,196,209,197,199, 197,204,195,200,202,198,193,195,
+ 193,196,190,195,196,183,196,192, 202,197,207,198,197,198,202,200, 202,191,204,201,199,197,203,200, 140,132,132,140,134,135,133,131,
+ 132,133,132,133,146,134,134,134, 133,133,134,132,135,132,132,131, 133,132,131,131,133,132,135,135, 130,130,130,130,130,130,130,0,
+ 150,126,124,127,127,133,129,124, 132,124,123,123,132,122,120,121, 35,63,39,52,35,68,64,64, 75,69,63,67,52,52,39,65,
+ 121,121,121,121,122,120,121,121, 126,122,121,120,121,120,121,121, 121,121,121,120,121,121,120,121, 121,121,120,121,122,121,121,121,
+ 121,121,121,121,121,120,120,120, 121,120,120,121,121,121,121,120, 120,120,121,120,120,120,120,121, 121,121,121,121,121,121,121,121,
+ 121,120,121,120,121,120,120,120, 121,121,121,121,120,121,120,120, 120,120,120,122,120,120,120,121, 121,121,120,120,121,121,120,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,79,79,0,0,79,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 79,85,79,79,79,79,95,79, 83,83,79,93,93,79,87,79, 126,115,111,106,110,106,103,106, 108,108,79,79,140,83,79,110,
+ 132,131,132,136,135,131,132,131, 131,141,131,133,131,132,132,133, 131,133,131,132,138,132,132,137, 134,131,131,131,131,133,136,131,
+ 131,131,131,131,131,132,132,133, 131,131,131,131,134,131,131,135, 131,131,139,131,132,131,131,132, 131,132,132,131,134,131,131,0,
+ 138,132,131,134,130,131,132,130, 130,130,131,132,131,134,133,133, 131,134,137,134,131,130,133,133, 130,131,132,131,130,132,134,136,
+ 138,196,198,193,197,176,186,185, 196,185,197,195,199,187,185,192, 190,191,180,190,190,195,182,192, 182,191,188,196,190,191,188,193,
+ 187,175,193,194,203,190,193,195, 191,189,190,196,182,188,196,192, 200,179,185,187,190,187,193,181, 193,177,196,183,186,180,183,186,
+ 186,188,190,186,184,188,186,187, 178,191,190,192,188,188,182,181, 179,186,189,196,174,178,187,193, 193,183,197,187,178,188,186,0,
+ },
+ {218,0,240,172,146,146,148,146, 120,44,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 56,0,26,2,2,2,2,2, 74,70,2,2,2,2,2,2, 50,0,20,2,2,2,2,2, 84,76,2,2,2,2,2,2,
+ 50,0,22,2,2,2,2,2, 86,76,2,2,2,2,2,2, 58,0,28,2,2,2,2,2, 86,76,2,2,2,2,2,2,
+ 128,0,128,64,36,36,38,38, 86,76,2,2,2,2,2,2, 128,0,128,68,40,40,42,42, 84,76,2,2,2,2,2,2,
+ 0,0,0,212,188,194,188,186, 180,180,120,114,116,122,126,126, 0,0,0,192,194,188,194,194, 176,176,116,120,116,122,136,128,
+ 0,0,0,194,206,204,216,214, 118,118,136,130,128,116,126,112, 0,0,0,198,208,208,228,220, 112,112,126,126,128,126,126,130,
+ 0,0,0,188,210,208,232,222, 112,112,120,126,126,126,130,130, 0,0,0,192,210,208,232,222, 120,120,124,128,126,134,120,124,
+ 0,0,0,188,210,208,236,222, 176,176,128,122,120,122,124,124, 0,0,0,180,210,208,236,222, 178,178,122,122,122,124,126,124,
+ },
+},
+
+{ // CP852 (9.112M chars) [33]
+ {NULL, NULL, NULL, NULL},
+ 85, 183, 47, 36, 128,
+ {147,99,153,98,113,162,97,141, 139,113,105,98,99,105,112,106, 117,111,78,82,68,70,92,81, 93,63,63,89,87,89,89,145,
+ 187,171,114,163,117,105,174,173, 163,149,96,97,180,97,98,99, 96,96,96,96,96,186,101,174, 97,96,96,96,96,99,100,96,
+ 96,96,96,96,96,96,112,102, 96,96,96,96,96,96,96,96, 97,99,112,109,110,120,170,99, 174,96,96,96,96,98,162,96,
+ 115,98,115,99,99,120,184,183, 99,167,99,97,153,153,98,101, 97,97,96,97,96,158,99,96, 100,97,97,100,191,191,96,158,
+ 178,142,216,118,134,202,143,174, 177,140,130,157,125,109,134,147, 194,166,135,125,142,148,164,161, 172,142,141,143,163,165,129,208,
+ 222,229,177,175,143,129,196,201, 175,165,92,102,208,119,112,129, 92,92,92,92,92,217,114,205, 120,92,92,92,92,119,130,92,
+ 92,92,92,92,92,92,117,134, 95,95,92,92,92,92,92,99, 100,101,140,134,151,163,210,124, 213,92,92,92,92,116,181,94,
+ 176,147,127,120,122,171,198,197, 98,182,97,109,208,197,117,134, 100,102,93,126,93,173,129,92, 111,100,112,145,203,203,92,168,
+ },
+ {0,0,0,0,0,0,0,0, 0,124,152,0,0,168,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 234,148,183,99,97,113,164,166, 117,160,105,108,196,160,193,147, 123,151,151,145,144,134,140,137, 123,116,177,135,205,109,140,149,
+ 99,185,176,199,192,207,123,147, 188,196,177,198,197,199,208,171, 173,98,190,193,202,160,196,129, 101,116,185,101,102,119,97,112,
+ 97,188,176,199,192,210,124,151, 188,197,178,199,198,199,208,173, 176,98,191,194,203,165,197,129, 100,117,186,97,109,97,97,0,
+ 127,96,131,106,97,136,102,129, 109,96,96,129,100,96,97,102, 122,147,109,96,96,114,120,101, 101,96,96,149,150,109,105,174,
+ 176,201,108,134,125,115,179,179, 153,141,96,96,173,96,98,99, 96,96,96,96,96,174,106,99, 96,96,96,96,96,110,110,96,
+ 96,96,96,96,96,96,98,98, 96,96,96,96,96,96,96,98, 97,97,137,96,141,138,199,98, 99,96,96,96,96,100,136,96,
+ 108,105,96,97,97,138,174,174, 99,131,100,96,100,100,100,102, 116,100,96,107,96,115,97,96, 100,97,97,99,180,181,96,164,
+ },
+ {92,0,50,80,56,70,54,68, 82,58,30,64,104,40,54,46, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 8,0,2,2,2,2,2,2, 2,2,2,2,6,2,2,2, 20,0,2,2,2,2,2,2, 4,2,2,2,24,2,2,2,
+ 78,0,10,38,14,28,12,26, 96,60,28,64,128,38,54,46, 26,0,2,2,2,2,2,2, 2,2,2,2,24,2,2,2,
+ 14,0,2,2,2,2,2,2, 2,2,2,2,24,2,2,2, 8,0,2,2,2,2,2,2, 10,2,2,2,32,2,2,2,
+ 138,0,138,136,100,100,126,124, 180,178,162,150,190,144,146,142, 122,0,120,134,128,124,134,128, 182,146,170,148,172,110,140,132,
+ 132,0,132,130,120,106,130,132, 158,170,152,172,148,158,164,162, 86,0,86,90,142,152,50,60, 150,144,154,132,162,132,174,174,
+ 148,0,132,140,116,136,116,134, 232,176,140,174,232,152,168,164, 128,0,124,124,134,132,122,132, 156,160,164,142,166,136,162,164,
+ 124,0,124,120,130,132,128,132, 156,156,160,132,156,166,160,146, 118,0,96,134,136,98,136,108, 136,130,160,158,150,168,118,150,
+ },
+},
+
+{ // Arabic (0.205M chars) [34]
+ {NULL, NULL, NULL, NULL},
+ 180, 214, 71, 14, 128,
+ {0,0,0,0,0,0,0,0, 0,121,121,0,0,121,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 136,0,0,0,136,0,0,0, 0,0,0,0,138,136,0,0, 0,0,0,0,0,0,0,0, 0,0,0,136,0,0,0,152,
+ 0,151,167,198,147,188,171,230, 205,155,210,171,192,196,187,199, 176,204,173,195,191,195,175,181, 170,203,174,0,0,0,0,0,
+ 182,201,194,197,213,218,204,199, 211,169,210,171,161,136,141,183, 141,152,136,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,133,133,0,0,133,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 145,0,0,0,145,0,0,0, 0,0,0,0,184,145,0,0, 0,0,0,0,0,0,0,0, 0,0,0,148,0,0,0,159,
+ 0,171,146,149,145,145,147,191, 180,203,194,168,171,165,164,185, 152,192,159,171,155,163,167,168, 152,175,156,0,0,0,0,0,
+ 152,177,173,172,190,189,191,184, 173,192,184,156,145,145,146,145, 145,149,145,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,137,171,0,0,150,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 227,142,169,137,137,137,163,144, 137,152,138,137,171,141,178,147, 138,139,137,137,137,138,137,137, 137,137,167,137,194,137,137,137,
+ 137,137,137,137,137,137,137,137, 137,137,137,137,137,137,137,137, 137,137,137,137,137,137,137,137, 137,137,137,137,138,137,137,142,
+ 137,137,137,137,137,137,137,137, 137,137,137,137,137,137,137,137, 137,137,137,137,137,137,137,137, 137,137,137,137,137,137,137,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 137,164,151,154,137,159,165,165, 160,155,154,156,173,137,172,165, 170,158,160,163,159,161,154,161, 189,173,179,140,185,172,164,157,
+ 0,171,144,171,160,163,176,218, 199,202,201,175,192,195,189,204, 182,210,177,202,184,184,183,185, 163,200,179,0,0,0,0,0,
+ 160,196,197,191,226,205,211,195, 209,178,216,158,137,138,147,151, 146,154,138,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {158,0,126,158,180,178,180,180, 0,0,144,124,130,132,124,178, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 52,0,2,28,50,50,50,52, 0,0,2,2,2,2,2,34, 52,0,2,28,50,50,50,52, 0,0,20,10,2,2,2,56,
+ 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,16,
+ 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 48,0,2,24,46,46,46,48, 0,0,22,12,2,2,2,58,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 166,0,150,140,158,156,158,158, 0,0,146,124,130,132,122,178, 154,0,138,166,170,170,170,170, 0,0,134,162,120,124,114,168,
+ 134,0,150,152,144,144,144,144, 0,0,106,68,128,132,132,124, 144,0,150,152,154,154,154,154, 0,0,110,76,134,124,130,130,
+ 156,0,150,144,146,146,146,146, 0,0,140,142,130,128,126,130, 156,0,134,156,178,178,178,178, 0,0,138,116,132,130,128,174,
+ },
+},
+
+{ // BIG5_HKSCS (0.271M chars) [35]
+ {NULL, NULL, NULL, NULL},
+ 157, 175, 62, 14, 128,
+ {0,0,0,0,0,0,0,0, 108,149,150,140,0,150,152,140, 132,132,131,132,132,132,132,132, 132,132,132,131,131,132,131,130,
+ 139,181,164,132,214,207,204,195, 203,198,208,192,195,201,196,187, 202,189,195,196,188,194,190,182, 195,188,192,186,188,188,191,185,
+ 184,180,185,187,180,189,169,181, 135,142,140,142,141,141,140,141, 141,140,140,140,141,140,141,140, 140,140,142,140,140,141,141,140,
+ 140,140,140,140,140,146,149,145, 143,146,140,141,149,140,140,140, 140,140,140,140,140,140,140,140, 140,151,141,140,141,139,140,0,
+ 0,0,0,0,0,0,0,0, 134,133,133,134,0,124,134,133, 122,122,122,122,122,122,122,122, 122,122,122,121,121,122,122,121,
+ 132,216,163,137,212,204,205,192, 184,187,190,192,193,185,187,179, 188,190,183,193,178,189,189,198, 181,189,172,184,183,181,173,166,
+ 178,176,172,174,168,176,168,176, 163,140,141,134,134,134,135,134, 134,134,135,135,135,135,134,134, 135,135,134,134,134,134,134,134,
+ 134,134,134,134,134,134,135,136, 134,134,134,134,134,134,135,134, 134,134,134,134,134,136,135,134, 134,134,144,134,134,134,134,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,81,81,0,0,81,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,81,81,81,81,81,81,81, 81,81,81,81,81,81,81,81, 81,81,81,81,81,81,81,81, 81,81,81,81,85,81,81,81,
+ 200,202,189,193,189,173,185,189, 193,179,177,176,184,185,182,192, 186,181,175,172,181,185,166,189, 183,172,168,173,170,186,179,178,
+ 174,184,179,166,182,181,171,178, 178,187,186,179,178,172,187,186, 185,189,169,194,178,186,186,183, 180,180,179,177,186,177,193,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 157,191,170,191,186,176,168,184, 181,177,175,185,177,176,169,168, 182,178,179,190,172,180,184,171, 183,186,201,182,169,184,171,170,
+ 185,180,173,171,183,175,187,182, 183,185,183,177,184,178,189,186, 187,188,183,195,166,177,177,182, 181,174,185,182,180,178,175,181,
+ 176,176,179,175,179,183,187,172, 179,192,190,179,187,180,169,178, 168,171,175,183,191,176,182,182, 188,183,186,178,183,179,177,0,
+ },
+ {232,0,254,254,110,120,118,114, 128,128,4,2,2,2,2,4, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 40,0,16,16,2,2,2,2, 128,128,2,2,2,2,2,2, 46,0,24,22,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 66,0,42,40,2,2,2,2, 128,128,2,2,2,2,2,2, 128,0,128,128,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 128,0,128,128,2,2,2,2, 128,128,2,2,2,2,2,2, 128,0,128,128,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 0,0,0,0,132,142,144,140, 0,0,140,124,126,126,126,126, 0,0,0,0,134,142,142,138, 0,0,136,134,134,134,134,134,
+ 0,0,0,0,142,138,138,134, 0,0,136,138,132,132,134,134, 0,0,0,0,132,138,140,144, 0,0,134,128,138,138,132,136,
+ 0,0,0,0,134,144,136,140, 0,0,126,134,132,132,142,138, 0,0,0,0,134,144,142,138, 0,0,136,134,134,134,136,136,
+ 0,0,0,0,134,144,142,138, 0,0,144,132,132,132,132,132, 0,0,0,0,136,142,140,138, 0,0,136,132,134,136,134,138,
+ },
+},
+
+{ // CP866 (75.238M chars) [36]
+ {NULL, NULL, NULL, NULL},
+ 144, 168, 54, 34, 129,
+ {202,189,201,189,198,201,176,191, 205,171,201,198,196,207,207,203, 202,204,202,184,177,168,173,180, 170,166,142,175,168,167,156,168,
+ 200,185,197,186,194,201,174,188, 204,171,197,197,192,205,205,197, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60,
+ 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60,
+ 203,204,203,185,172,167,174,181, 171,168,145,178,171,163,156,169, 122,121,61,61,64,61,61,61, 82,60,60,60,60,60,60,149,
+ 155,129,160,132,130,136,112,122, 155,123,142,126,140,137,156,133, 109,124,109,108,94,99,94,88, 84,75,58,94,87,91,94,119,
+ 193,150,189,167,164,195,129,155, 199,188,180,165,183,167,190,143, 55,55,55,55,55,55,55,55, 55,55,55,55,55,55,55,55,
+ 55,55,55,55,55,55,55,55, 55,55,55,55,55,55,55,55, 55,55,55,55,55,55,55,55, 55,55,55,55,55,55,55,55,
+ 172,176,181,172,118,175,146,138, 137,99,105,181,182,104,163,194, 90,129,56,55,55,62,55,57, 106,70,89,55,55,55,55,167,
+ },
+ {0,0,0,0,0,0,0,0, 0,118,150,0,0,157,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 221,147,180,74,65,105,160,149, 120,156,94,110,186,165,186,136, 108,128,124,115,112,121,107,104, 100,97,169,132,195,113,104,144,
+ 134,102,74,101,82,91,78,71, 88,102,80,94,92,80,75,94, 112,60,73,125,77,72,89,82, 78,91,81,80,102,111,73,107,
+ 93,109,104,99,117,99,90,77, 101,105,109,107,97,101,119,121, 134,59,107,112,113,105,117,82, 72,94,116,60,90,75,74,0,
+ 210,180,193,179,186,209,168,174, 206,179,191,195,187,202,211,177, 198,196,203,188,161,170,168,174, 161,156,124,189,187,127,169,186,
+ 213,183,195,180,188,212,170,177, 208,179,193,198,188,204,215,181, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60,
+ 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60, 60,60,60,60,60,60,60,60,
+ 202,198,205,193,164,172,168,175, 162,157,125,191,187,137,171,187, 127,130,61,61,67,67,60,61, 82,60,61,60,66,61,60,147,
+ },
+ {2,0,2,2,50,52,44,42, 12,26,10,176,176,176,22,96, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,4,6,2,2, 2,2,2,68,68,68,2,8, 128,0,58,84,128,128,128,128, 52,68,48,128,128,128,66,128,
+ 128,0,58,84,128,128,128,128, 52,68,48,128,128,128,66,128, 128,0,58,84,128,128,128,128, 52,68,48,128,128,128,66,128,
+ 2,0,2,2,16,18,10,10, 2,2,2,86,86,86,2,18, 64,0,4,30,96,98,92,86, 2,2,2,128,128,128,2,62,
+ 146,0,152,154,174,180,156,152, 138,138,114,116,116,116,110,112, 148,0,150,162,184,184,176,158, 138,140,112,130,130,130,110,118,
+ 154,0,152,150,150,138,126,128, 10,12,138,118,118,118,138,116, 176,0,132,158,200,200,200,198, 118,136,116,200,200,200,132,200,
+ 176,0,132,158,200,200,200,198, 118,136,116,200,200,200,132,200, 176,0,132,158,200,200,200,198, 118,136,116,200,200,200,132,200,
+ 152,0,152,154,122,140,132,122, 10,14,136,132,132,132,138,120, 166,0,126,164,192,204,210,212, 118,118,110,200,200,200,116,224,
+ },
+},
+
+{ // UTF-16BE (921.761M chars) [37]
+ {NULL, NULL, NULL, NULL},
+ 58, 128, 46, 5, 128,
+ {255,148,128,107,171,104,157,100, 118,108,101,120,124,90,133,117, 110,115,104,101,109,109,114,104, 109,116,118,96,107,107,124,118,
+ 70,74,71,58,71,66,66,63, 77,54,49,59,103,87,65,64, 97,64,107,90,85,62,70,84, 68,102,96,78,93,84,65,62,
+ 98,86,102,72,68,106,98,98, 86,95,76,75,78,83,75,75, 86,85,82,77,76,85,76,77, 73,80,63,66,87,93,89,101,
+ 71,70,54,77,57,100,56,43, 81,50,69,75,59,63,75,80, 86,62,64,76,70,35,84,71, 72,68,83,83,81,83,96,112,
+ 158,116,117,132,124,138,121,108, 116,113,119,126,121,103,105,108, 120,97,117,112,89,96,106,109, 108,102,107,100,99,84,118,104,
+ 97,103,109,97,115,85,102,101, 103,94,96,102,108,105,104,111, 112,95,96,109,94,101,100,108, 109,120,113,105,98,89,95,106,
+ 95,111,93,114,92,107,109,118, 118,105,105,108,86,103,97,102, 101,110,88,98,94,117,109,105, 78,77,73,73,66,78,86,78,
+ 104,107,89,105,83,121,89,89, 97,107,111,111,101,103,99,100, 88,81,89,122,99,70,98,85, 99,87,106,114,125,103,97,136,
+ },
+ {236,123,89,99,100,109,89,100, 108,198,193,73,119,181,104,83, 99,103,97,99,99,97,82,94, 101,109,110,107,104,93,93,106,
+ 216,151,203,161,118,166,169,173, 164,164,144,152,168,182,186,198, 189,182,179,173,169,172,169,168, 170,168,177,180,198,196,198,153,
+ 139,171,167,169,169,168,165,158, 160,167,150,145,164,164,166,162, 165,146,165,168,170,153,155,156, 147,151,149,145,145,143,117,176,
+ 111,206,186,196,198,208,187,191, 193,205,165,175,201,192,202,201, 195,149,202,203,208,188,183,182, 175,180,162,150,135,150,118,103,
+ 112,108,111,92,98,93,88,76, 111,111,111,115,106,106,97,112, 115,111,81,95,101,109,110,98, 111,93,99,76,83,70,93,76,
+ 118,92,84,86,91,83,65,85, 86,99,71,87,101,87,83,95, 96,99,90,82,97,95,62,97, 88,93,93,96,90,93,79,99,
+ 100,101,86,92,104,90,86,96, 95,94,77,82,107,97,85,85, 105,105,86,90,102,108,87,87, 97,112,82,93,109,99,100,96,
+ 112,124,97,99,120,103,92,104, 114,131,110,75,88,121,83,84, 84,109,72,116,98,89,110,72, 99,78,96,85,114,106,69,129,
+ },
+ {126,120,126,128,126,126,126,126, 206,210,222,240,224,222,220,216, 122,164,100,126,150,150,112,122, 232,224,222,208,220,232,222,228,
+ 28,128,42,46,76,82,30,42, 128,128,128,128,128,128,128,128, 18,128,28,34,58,64,18,28, 128,128,128,128,128,128,128,128,
+ 16,128,26,32,54,62,16,26, 128,128,128,128,128,128,128,128, 22,128,32,38,70,74,22,32, 128,128,128,128,128,128,128,128,
+ 30,128,42,48,82,88,32,42, 128,128,128,128,128,128,128,128, 90,190,74,128,122,156,100,110, 212,182,194,168,162,186,118,178,
+ 96,180,78,128,126,140,100,104, 98,96,94,100,98,90,80,92, 108,222,102,132,140,156,114,122, 236,234,220,202,226,226,226,230,
+ 118,206,78,150,134,154,108,118, 228,218,192,196,192,176,210,224, 112,218,96,148,132,158,110,122, 232,232,220,208,226,212,224,228,
+ 112,224,102,146,142,156,108,124, 234,232,226,222,226,226,216,222, 114,196,84,148,140,164,110,118, 242,232,196,208,200,198,192,206,
+ 110,160,74,150,140,156,114,118, 230,224,160,160,162,160,160,220, 124,240,102,144,132,152,106,116, 230,226,200,198,162,174,190,230,
+ },
+},
+
+{ // Latin3 (0.294M chars) [38]
+ {NULL, NULL, ced_hires_16, ced_hires_16, },
+ 99, 202, 43, 23, 128,
+ {0,0,0,0,0,0,0,0, 0,142,142,0,0,142,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 180,143,143,143,145,0,143,148, 145,143,143,143,143,143,0,146, 147,143,143,143,144,143,143,148, 145,206,195,143,143,143,0,145,
+ 143,143,143,0,158,143,143,150, 143,151,143,143,143,143,143,143, 0,145,143,162,143,143,147,143, 143,143,143,143,172,143,143,143,
+ 154,143,143,0,165,177,154,182, 165,150,143,148,143,144,143,145, 0,145,143,162,143,143,157,143, 143,143,143,143,172,144,143,143,
+ 0,0,0,0,0,0,0,0, 0,112,112,0,0,112,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 188,148,144,162,154,0,145,154, 146,209,160,192,146,145,0,200, 163,148,147,145,151,144,144,167, 145,236,159,192,146,145,0,204,
+ 161,202,162,87,203,144,144,187, 186,211,176,153,145,198,159,150, 76,180,149,201,165,145,202,146, 144,144,182,153,204,144,144,178,
+ 188,206,154,0,204,144,144,175, 177,206,166,153,158,208,152,146, 0,180,159,203,157,145,200,146, 144,155,182,149,192,144,144,147,
+ },
+ {86,0,0,0,0,0,0,0, 0,149,151,0,0,159,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 223,146,167,143,143,144,158,154, 152,149,146,143,193,154,169,146, 150,160,164,151,150,149,147,146, 146,146,160,144,200,143,144,146,
+ 143,194,180,194,186,193,173,187, 178,180,164,179,196,190,207,184, 177,157,206,202,196,184,182,176, 156,184,174,143,143,145,144,143,
+ 144,192,181,189,184,193,172,184, 175,179,163,204,214,197,221,182, 173,152,217,199,214,184,180,176, 154,186,193,144,147,143,144,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 180,143,143,143,146,164,143,146, 146,145,143,171,143,144,177,162, 146,143,143,143,145,143,143,148, 146,199,202,192,143,143,169,162,
+ 145,144,145,0,158,143,143,149, 143,151,144,143,143,146,144,143, 0,144,144,146,143,143,148,144, 143,143,143,143,146,143,143,161,
+ 144,144,144,0,159,143,143,146, 143,147,144,143,143,146,143,143, 0,144,144,147,146,143,150,143, 143,143,144,143,145,143,143,143,
+ },
+ {162,0,130,150,128,132,120,124, 0,0,156,140,170,170,172,172, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 30,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 26,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 16,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 144,0,124,142,134,134,124,120, 0,0,174,132,162,162,162,166, 118,0,132,130,74,76,134,134, 0,0,126,164,140,140,140,142,
+ 118,0,100,110,147,145,94,90, 0,0,152,136,178,170,170,170, 124,0,102,116,143,145,108,108, 0,0,170,132,166,166,164,166,
+ 134,0,140,130,82,86,131,131, 0,0,168,152,154,156,164,156, 130,0,124,124,92,96,137,137, 0,0,148,152,164,170,166,166,
+ },
+},
+
+{ // UTF-16LE (912.138M chars) [39]
+ {NULL, NULL, NULL, NULL},
+ 58, 128, 46, 4, 127,
+ {255,146,126,120,177,123,159,99, 110,78,94,116,115,99,130,112, 110,112,82,97,100,94,113,102, 100,94,112,92,103,102,120,113,
+ 60,84,76,72,67,43,63,72, 75,64,74,96,95,88,83,52, 78,81,96,84,76,57,31,82, 83,85,86,82,92,78,88,68,
+ 84,91,96,52,84,96,99,105, 89,83,72,65,83,88,84,86, 81,94,75,70,97,95,78,57, 109,82,33,91,80,77,77,75,
+ 25,59,64,81,47,69,60,19, 96,31,92,30,20,68,72,81, 58,66,54,80,88,55,60,66, 55,40,45,78,55,99,113,116,
+ 161,117,113,130,127,131,118,108, 119,110,115,123,129,107,105,108, 116,99,114,110,93,99,108,111, 109,103,113,103,97,84,101,103,
+ 91,100,101,94,110,85,98,97, 98,88,86,93,108,101,99,107, 111,91,107,105,91,95,97,104, 105,118,111,98,97,92,87,101,
+ 101,106,99,111,87,110,107,116, 114,105,99,102,87,99,93,98, 100,103,88,98,89,111,105,101, 111,92,89,98,104,108,110,105,
+ 102,102,83,99,77,115,78,86, 85,100,104,107,98,100,92,97, 86,77,86,117,94,64,98,84, 97,89,103,109,121,97,98,126,
+ },
+ {236,123,100,94,96,100,91,97, 108,196,193,90,112,182,101,92, 101,105,95,98,102,96,89,96, 98,112,104,103,108,101,95,106,
+ 216,149,203,169,119,166,174,176, 165,165,142,152,168,181,187,198, 189,182,180,175,173,177,172,171, 173,176,177,184,198,197,198,156,
+ 139,172,168,169,171,168,166,159, 159,167,150,145,165,165,166,163, 165,145,164,168,170,153,155,156, 147,150,149,144,147,141,118,176,
+ 125,206,186,196,197,208,187,190, 192,205,164,175,200,191,201,201, 195,151,202,202,207,188,182,182, 175,179,161,149,134,149,121,100,
+ 115,103,100,89,101,94,90,87, 101,114,108,116,108,98,92,113, 113,107,90,81,101,110,105,94, 107,95,107,74,84,72,92,81,
+ 118,95,89,86,95,86,81,84, 92,99,79,88,102,91,89,97, 104,100,92,91,99,99,79,96, 89,102,98,99,90,95,85,101,
+ 101,103,87,94,104,94,85,94, 97,94,83,90,106,96,82,86, 105,103,85,93,102,108,92,91, 91,103,79,87,101,91,92,87,
+ 112,124,97,99,119,112,90,105, 108,128,107,81,91,121,89,85, 88,97,79,113,98,90,111,79, 102,83,100,94,112,102,86,130,
+ },
+ {128,124,128,126,126,126,126,126, 204,202,224,222,224,224,220,218, 120,164,116,98,140,156,124,122, 220,224,168,162,168,220,206,238,
+ 82,158,96,72,110,162,88,92, 170,160,158,158,158,176,158,164, 54,158,66,68,104,108,56,66, 158,158,158,158,158,148,158,158,
+ 48,158,58,62,88,96,50,60, 158,158,158,158,158,148,158,158, 50,158,60,64,90,98,52,62, 158,158,158,158,158,148,158,158,
+ 62,158,74,80,110,116,66,76, 158,158,158,158,158,148,158,158, 112,192,96,138,138,132,104,106, 206,210,184,198,188,196,186,186,
+ 106,174,86,120,124,134,92,112, 96,94,82,82,82,78,66,76, 122,214,110,132,136,160,112,112, 228,230,210,212,214,210,216,212,
+ 90,200,118,146,140,162,114,120, 234,222,202,200,206,188,196,196, 96,206,128,142,144,156,116,126, 238,216,210,206,214,206,218,208,
+ 94,216,130,142,136,160,108,126, 238,236,218,214,222,212,214,212, 94,214,134,138,140,158,114,124, 238,238,212,208,212,204,214,208,
+ 62,162,54,148,148,164,116,124, 232,226,164,164,162,150,164,174, 94,210,110,146,134,162,112,118, 232,228,192,198,194,188,198,208,
+ },
+},
+
+{ // HZ-GB-2312 (87.400M chars) [40]
+ {NULL, NULL, NULL, NULL},
+ 92, 0, 37, 0, 255,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {126,0,164,0,0,0,126,0, 178,139,148,0,0,132,154,146, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 161,195,176,211,162,158,148,144, 213,209,179,164,154,148,168,174, 182,184,172,182,184,188,187,175, 180,183,176,176,182,177,179,172,
+ 170,193,196,199,167,165,195,168, 183,178,178,180,168,178,186,182, 183,158,180,182,181,183,186,180, 155,132,136,132,146,136,132,132,
+ 132,132,136,139,173,183,178,171, 162,161,132,142,144,132,139,139, 132,136,139,144,132,132,149,139, 132,132,136,152,139,132,245,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // CSN_369103 (8.850M chars) [41]
+ {NULL, NULL, NULL, NULL},
+ 90, 204, 46, 27, 127,
+ {0,0,0,0,0,0,0,0, 0,90,90,0,0,90,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 176,178,96,184,92,102,188,127, 120,153,117,98,133,96,143,153, 122,178,93,182,115,102,188,96, 119,152,117,98,133,101,141,152,
+ 98,169,119,130,151,111,96,137, 171,147,170,103,153,155,105,120, 96,98,118,171,112,104,128,105, 180,129,158,96,133,127,119,102,
+ 99,169,99,125,153,107,102,128, 169,142,170,99,155,155,103,119, 95,99,118,171,112,105,148,95, 180,130,154,95,137,127,119,96,
+ 0,0,0,0,0,0,0,0, 0,55,55,0,0,55,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 185,202,93,207,98,140,210,145, 119,178,153,123,170,124,172,196, 158,217,94,211,139,141,210,121, 116,176,151,152,187,124,173,200,
+ 103,198,155,156,200,128,165,183, 202,208,195,143,194,194,151,144, 149,174,157,197,159,157,199,120, 192,171,178,137,201,170,155,174,
+ 106,203,144,173,201,105,197,170, 203,203,208,143,202,204,139,150, 148,188,166,199,147,166,197,121, 192,191,178,144,188,182,155,122,
+ },
+ {62,0,0,0,0,0,0,0, 0,129,153,0,0,162,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 228,154,181,95,91,108,164,158, 144,155,114,105,191,153,189,148, 132,145,139,130,128,126,119,117, 115,116,177,135,201,95,124,155,
+ 93,200,180,210,197,203,168,188, 175,191,170,190,200,190,208,194, 182,141,204,199,200,181,182,186, 146,188,181,105,114,126,105,120,
+ 112,201,181,209,196,205,167,186, 172,191,169,190,200,189,207,194, 180,135,203,198,199,182,181,191, 143,189,181,100,125,92,99,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 176,176,95,167,93,99,163,117, 120,144,106,125,140,115,142,179, 121,178,93,168,111,97,163,92, 119,144,107,125,141,107,143,179,
+ 99,160,117,127,149,96,190,132, 166,138,143,97,95,185,108,127, 94,120,128,168,99,91,130,104, 171,126,127,95,123,101,122,153,
+ 100,161,110,128,150,96,190,125, 167,128,145,96,95,185,105,130, 94,120,128,170,114,91,134,95, 172,126,113,95,119,100,122,97,
+ },
+ {182,0,124,152,122,132,122,132, 0,0,162,164,156,176,156,176, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 12,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 14,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 26,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 24,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 26,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 132,0,96,130,142,136,100,114, 0,0,172,130,168,162,110,142, 130,0,136,132,52,54,136,130, 0,0,108,174,92,106,170,166,
+ 98,0,90,100,140,142,100,90, 0,0,164,122,170,178,132,132, 100,0,92,102,138,142,108,110, 0,0,166,106,174,156,130,144,
+ 134,0,136,134,38,54,134,134, 0,0,118,168,102,146,168,180, 132,0,130,134,44,58,136,140, 0,0,108,170,98,158,172,156,
+ },
+},
+
+{ // ISO-2022-KR (85.145M chars) [42]
+ {NULL, NULL, NULL, NULL},
+ 44, 144, 15, 3, 129,
+ {0,0,130,0,0,0,0,0, 0,66,66,0,0,66,213,252, 0,0,0,0,0,0,0,0, 0,0,0,224,0,0,91,91,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {80,0,0,0,0,0,0,0, 0,155,178,0,0,191,94,16, 0,0,0,0,0,0,0,0, 0,0,0,159,0,0,115,80,
+ 237,178,211,150,215,142,197,197, 215,186,147,141,202,191,204,184, 182,193,189,179,176,177,168,168, 173,174,186,191,233,164,175,184,
+ 177,172,153,166,162,158,159,169, 164,164,152,159,147,160,152,147, 160,143,152,154,156,148,153,153, 128,113,118,160,143,179,134,168,
+ 120,133,133,128,134,139,123,121, 142,133,113,107,120,125,168,114, 123,102,122,140,136,109,112,132, 116,115,107,101,157,124,155,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {130,130,126,130,130,130,118,128, 0,0,0,0,0,0,0,0, 42,2,134,2,62,40,70,60, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // Latin6 (0.061M chars) [43]
+ {NULL, NULL, ced_hires_19, ced_hires_19, },
+ 93, 214, 54, 26, 129,
+ {0,0,0,0,0,0,0,0, 0,156,156,0,0,156,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 175,158,158,158,158,158,158,159, 158,158,161,158,159,158,158,158, 158,158,158,158,159,158,158,159, 158,158,161,158,159,158,158,158,
+ 159,164,158,159,162,158,158,158, 171,161,158,158,159,162,158,158, 158,160,158,158,158,158,159,158, 158,158,162,158,158,158,158,158,
+ 159,164,158,158,163,158,158,158, 171,160,158,158,159,162,158,158, 158,160,158,158,158,158,162,158, 158,159,162,158,159,158,158,158,
+ 0,0,0,0,0,0,0,0, 0,130,130,0,0,130,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 180,198,158,158,160,158,158,160, 158,163,190,158,168,158,159,158, 163,212,158,158,162,158,158,166, 159,158,180,158,168,158,158,158,
+ 206,194,163,172,195,183,171,185, 198,203,191,160,209,190,162,159, 161,179,158,193,165,163,194,158, 179,176,175,160,196,171,160,172,
+ 211,198,160,173,196,193,171,194, 199,198,204,160,214,199,160,158, 167,178,159,195,161,163,193,158, 178,219,176,159,185,179,160,158,
+ },
+ {0,0,0,0,0,0,0,0, 0,160,164,0,0,168,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 227,163,177,159,159,159,210,162, 160,170,159,159,189,161,185,159, 162,161,160,159,159,159,159,159, 159,159,178,159,200,159,159,164,
+ 159,184,177,200,197,187,171,187, 174,180,189,189,194,188,202,180, 180,160,204,208,200,171,181,173, 162,166,178,159,159,159,159,159,
+ 159,184,179,199,197,189,170,185, 172,181,189,189,194,186,201,179, 179,159,203,208,200,171,185,173, 162,169,178,159,159,159,159,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 176,159,159,159,159,159,159,159, 159,159,163,159,161,159,159,159, 159,159,159,159,159,160,159,159, 159,159,163,159,161,159,159,159,
+ 161,164,159,159,163,159,159,159, 168,160,159,159,159,168,159,159, 160,159,159,159,159,159,159,159, 159,159,159,159,159,159,159,164,
+ 161,164,159,159,163,159,159,159, 169,159,159,159,159,169,159,159, 160,159,159,159,159,159,159,159, 159,159,159,159,159,159,159,159,
+ },
+ {152,0,122,146,128,132,128,130, 0,0,154,156,154,156,154,158, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 4,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 4,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2, 4,0,2,2,2,2,2,2, 0,0,2,2,2,2,2,2,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 144,0,114,140,138,132,120,128, 0,0,162,154,152,154,152,154, 142,0,138,134,108,112,134,126, 0,0,154,158,154,156,154,156,
+ 118,0,98,118,143,145,106,106, 0,0,154,154,162,156,154,154, 132,0,108,126,143,143,118,118, 0,0,154,156,158,158,154,156,
+ 126,0,132,132,90,92,139,139, 0,0,152,156,152,156,162,156, 126,0,140,126,98,102,131,131, 0,0,154,156,154,158,156,158,
+ },
+},
+
+{ // UTF7 (0.037M chars) [44]
+ {NULL, NULL, NULL, NULL},
+ 77, 207, 29, 27, 255,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,184,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,189,189,0,0,189,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,189,189,189,189,189,189,189, 189,189,189,0,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189,
+ 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,189, 189,189,189,189,189,189,189,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // ISO_2022_CN (63.392M chars) [45]
+ {NULL, NULL, NULL, NULL},
+ 43, 144, 14, 3, 129,
+ {0,0,0,0,0,0,0,0, 0,70,70,0,0,70,213,252, 0,0,0,0,0,0,0,0, 0,0,0,222,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {79,0,0,0,0,0,0,0, 0,155,177,0,0,190,19,19, 0,0,0,0,0,0,0,0, 0,0,0,158,0,0,114,79,
+ 240,177,210,148,212,141,197,195, 214,185,147,140,202,190,203,183, 182,192,188,179,175,177,167,168, 173,174,186,190,231,164,175,183,
+ 177,172,152,166,161,157,158,168, 164,163,151,158,146,160,151,146, 159,143,151,153,155,147,152,152, 127,112,117,160,142,178,133,168,
+ 119,132,130,126,124,138,122,120, 141,131,112,106,119,124,122,112, 122,101,108,135,135,101,111,130, 112,115,106,96,156,123,154,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {130,130,126,128,130,128,128,130, 0,0,0,0,0,0,0,0, 44,2,134,2,62,26,62,10, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+},
+
+{ // BIG5-CP950 (0.029M chars) [46]
+ {NULL, NULL, NULL, NULL},
+ 158, 182, 61, 26, 128,
+ {0,166,166,166,166,166,166,166, 166,177,177,166,166,177,166,166, 158,158,158,158,158,158,158,158, 158,158,158,158,158,158,158,158,
+ 166,179,169,151,208,201,198,190, 197,193,202,187,190,196,191,184, 197,185,191,191,184,189,185,179, 190,185,188,182,184,184,187,182,
+ 181,178,182,183,178,186,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166,
+ 166,166,166,166,166,167,168,167, 166,167,166,166,168,166,166,166, 166,166,166,166,166,166,166,166, 166,168,166,166,166,166,166,101,
+ 0,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,161, 149,149,149,149,149,149,149,149, 149,149,149,149,149,149,149,149,
+ 161,211,166,161,206,199,200,187, 180,182,185,186,188,181,182,176, 183,186,179,188,175,185,185,193, 178,185,170,180,179,177,170,167,
+ 175,174,171,172,168,174,169,161, 161,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,161,
+ 161,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,161, 161,161,161,161,161,161,161,167,
+ },
+ {0,0,0,0,0,0,0,0, 0,113,113,0,0,113,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,113,113,113,113,113,113,113, 113,113,113,113,113,113,113,113, 113,113,113,113,113,113,113,113, 113,113,113,113,113,113,113,113,
+ 197,198,187,191,188,176,184,188, 190,180,179,179,184,184,183,189, 185,182,178,177,182,184,174,187, 183,177,175,178,176,185,181,180,
+ 178,183,181,174,182,182,176,180, 180,186,185,181,180,177,186,185, 184,187,175,191,180,185,185,183, 181,180,181,179,185,179,190,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 150,188,176,189,185,179,174,184, 181,179,178,184,179,179,175,174, 182,179,180,187,176,180,183,176, 183,185,197,182,175,183,176,175,
+ 184,181,177,176,183,178,186,182, 182,184,183,179,184,180,187,185, 186,186,182,191,174,179,179,182, 182,177,184,182,180,179,178,181,
+ 179,179,180,177,180,182,186,176, 180,190,188,180,185,181,175,179, 174,176,178,183,188,178,182,182, 186,183,185,180,183,180,180,121,
+ },
+ {170,0,170,170,104,110,110,108, 128,128,2,2,2,2,2,78, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 14,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 20,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 34,0,10,10,2,2,2,2, 128,128,2,2,2,2,2,2, 42,0,18,18,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 40,0,18,16,2,2,2,2, 128,128,2,2,2,2,2,2, 42,0,18,18,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 0,0,0,0,134,142,140,138, 0,0,132,132,132,132,132,132, 0,0,0,0,134,142,140,138, 0,0,136,134,134,136,136,136,
+ 0,0,0,0,142,136,138,134, 0,0,134,138,132,134,134,134, 0,0,0,0,134,136,140,144, 0,0,134,130,138,136,134,136,
+ 0,0,0,0,136,142,138,140, 0,0,130,134,134,132,140,138, 0,0,0,0,134,142,140,138, 0,0,136,134,134,136,136,136,
+ 0,0,0,0,134,142,140,138, 0,0,138,134,134,134,134,134, 160,128,160,160,134,140,140,138, 128,128,136,134,134,134,136,136,
+ },
+},
+
+{ // JAGRAN (0.046M chars) [47]
+ {NULL, NULL, NULL, NULL},
+ 142, 199, 66, 34, 133,
+ {174,174,174,174,174,174,174,174, 174,182,182,174,174,182,174,174, 174,174,174,174,174,174,174,174, 174,174,174,0,0,0,174,174,
+ 0,182,182,182,182,182,182,182, 182,182,182,182,182,0,182,182, 182,182,0,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,0,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,0,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 166,166,166,166,166,166,166,166, 166,178,178,166,166,178,166,166, 166,166,166,166,166,166,166,166, 166,166,166,0,0,0,166,166,
+ 0,178,178,178,178,178,178,178, 178,178,178,178,178,0,178,178, 166,166,0,166,166,166,166,166, 166,166,166,166,166,166,166,166,
+ 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,0,166,
+ 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,0,166, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ },
+ {0,0,0,0,0,0,0,0, 0,180,180,0,0,180,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,180,127,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,127,180, 180,180,180,180,180,180,180,180, 180,180,127,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,127,180,180,0,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,0,0,0,176,176,
+ 0,176,176,176,176,176,176,176, 176,176,176,176,176,0,176,176, 176,176,0,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,0,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,0,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 132,0,132,132,132,132,132,132, 130,130,130,130,130,130,130,130, 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130, 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 136,0,134,136,134,134,136,134, 130,130,130,130,130,130,130,130, 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130, 136,0,134,136,134,134,136,134, 130,130,130,130,130,130,130,130,
+ },
+},
+
+{ // BHASKAR (0.047M chars) [48]
+ {NULL, NULL, NULL, NULL},
+ 141, 199, 66, 34, 132,
+ {174,174,174,174,174,174,174,174, 174,182,182,174,174,182,174,174, 174,174,174,174,174,174,174,174, 174,174,174,0,0,0,174,174,
+ 0,182,182,182,181,182,182,182, 182,182,182,182,182,0,182,182, 182,182,0,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,0,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,0,181, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 166,166,166,166,166,166,166,166, 166,178,178,166,166,178,166,166, 166,166,166,166,166,166,166,166, 166,166,166,0,0,0,166,166,
+ 0,178,178,178,178,178,178,178, 178,178,178,178,178,0,178,178, 166,166,0,166,166,166,166,166, 166,166,166,166,166,166,166,166,
+ 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,0,166,
+ 166,166,166,166,166,166,166,166, 166,166,166,166,166,166,0,166, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ },
+ {0,0,0,0,0,0,0,0, 0,180,180,0,0,180,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,126,180, 180,180,180,180,180,180,180,180, 180,180,126,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,0,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,0,0,0,176,176,
+ 0,176,176,176,176,176,176,176, 176,176,176,176,176,0,176,176, 176,176,0,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,0,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,0,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 132,0,132,132,132,132,132,132, 130,130,130,130,130,130,130,130, 134,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130, 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 134,0,134,136,134,134,136,134, 130,130,130,130,130,130,130,130, 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130,
+ 136,0,134,134,134,134,134,134, 130,130,130,130,130,130,130,130, 134,0,134,136,134,134,136,134, 130,130,130,130,130,130,130,130,
+ },
+},
+
+{ // HTCHANAKYA (0.041M chars) [49]
+ {NULL, NULL, NULL, NULL},
+ 142, 202, 68, 32, 133,
+ {173,0,0,0,173,173,173,173, 173,182,171,173,105,171,0,0, 0,173,173,0,173,173,173,173, 0,0,173,0,0,0,0,0,
+ 181,182,182,182,181,182,182,182, 182,182,182,182,182,0,182,182, 182,182,182,182,182,182,182,0, 182,182,182,182,182,182,182,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,0,181, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 169,0,0,0,169,169,169,169, 169,180,170,169,0,170,0,0, 0,169,169,0,169,169,169,169, 0,0,169,0,0,0,0,0,
+ 0,180,180,180,179,180,180,180, 180,180,180,180,180,0,180,180, 169,169,169,169,169,169,169,0, 169,169,169,169,169,169,169,169,
+ 169,169,169,169,169,169,169,169, 169,169,169,169,169,169,169,169, 169,169,169,169,169,169,169,169, 169,169,169,169,169,169,169,169,
+ 169,169,169,169,169,169,169,169, 169,169,169,169,169,169,0,168, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ },
+ {0,0,0,0,0,0,0,0, 0,181,181,0,0,181,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,181,129,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ 181,181,181,181,181,181,181,181, 181,181,181,129,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,0,
+ 176,0,0,0,176,176,176,176, 176,176,0,176,0,0,0,0, 0,176,176,0,176,176,176,176, 0,0,176,0,0,0,0,0,
+ 175,176,176,176,176,176,176,176, 176,176,176,176,176,0,176,176, 176,176,176,176,176,176,176,0, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,0,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 130,0,128,130,128,130,130,128, 128,128,128,128,128,128,128,128, 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132, 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132, 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132, 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132,
+ },
+},
+
+{ // TSCII (0.047M chars) [50]
+ {NULL, NULL, NULL, NULL},
+ 141, 199, 66, 33, 134,
+ {173,173,173,173,173,173,173,173, 173,182,182,173,173,182,173,173, 173,173,173,173,173,173,173,173, 173,173,173,173,173,173,0,173,
+ 0,181,181,181,181,181,157,157, 157,181,0,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,0,
+ 167,167,167,167,167,167,167,167, 167,179,179,167,167,167,167,167, 167,167,175,168,170,167,167,167, 167,167,167,167,167,167,0,167,
+ 0,178,178,178,178,178,0,0, 0,183,0,178,178,178,178,178, 167,167,167,167,167,167,167,167, 167,167,167,167,167,167,167,167,
+ 167,167,167,167,167,167,167,167, 167,167,167,167,167,167,167,167, 167,167,167,167,167,167,167,167, 167,167,167,167,167,167,167,167,
+ 167,167,167,167,167,167,167,167, 167,167,167,167,167,167,167,167, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,179,179,0,0,125,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 170,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,180,179,179,179,180,179,179, 179,180,179,179,179,179,179,179, 179,179,179,182,179,179,179,179, 179,179,179,179,179,179,179,0,
+ 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,0,175,
+ 0,175,175,175,175,175,0,0, 0,175,0,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175,
+ 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175,
+ 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,175, 175,175,175,175,175,175,175,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 132,0,132,132,132,132,132,132, 128,128,130,128,128,128,128,128, 132,0,134,132,132,132,134,136, 130,130,132,130,130,130,130,130,
+ 134,0,136,134,134,134,134,134, 132,130,130,132,132,130,130,130, 134,0,134,134,134,134,134,134, 130,130,132,130,130,130,130,130,
+ 134,0,134,134,134,134,134,134, 130,130,132,130,130,130,130,130, 134,0,134,134,134,134,134,134, 130,130,132,130,130,130,130,130,
+ 134,0,134,134,134,134,134,134, 130,130,132,130,130,130,130,130, 134,0,134,134,134,134,134,134, 130,130,132,130,130,130,130,130,
+ },
+},
+
+{ // TAM (0.036M chars) [51]
+ {NULL, NULL, NULL, NULL},
+ 140, 203, 70, 33, 133,
+ {0,0,174,174,174,174,174,174, 174,184,184,174,174,174,0,0, 0,0,0,0,0,0,0,0, 174,174,174,174,174,0,0,174,
+ 0,183,183,183,183,0,183,183, 183,0,161,161,161,0,183,183, 183,183,183,183,183,183,183,0, 183,183,183,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,0,183,183,183,183,183, 0,0,0,0,0,0,183,183, 183,183,183,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183,
+ 0,0,170,170,170,170,170,170, 170,181,181,170,170,0,0,0, 0,0,0,0,0,0,0,0, 170,170,170,170,170,0,0,170,
+ 0,181,181,181,181,0,181,181, 181,0,0,0,0,0,181,181, 170,170,170,170,170,170,170,0, 170,170,170,170,170,170,170,170,
+ 170,170,170,170,170,170,170,170, 170,170,0,170,170,170,170,170, 0,0,0,0,0,0,170,170, 170,170,170,170,170,170,170,170,
+ 170,170,170,170,170,170,170,170, 170,170,170,170,170,170,170,170, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ },
+ {0,0,0,0,0,0,0,0, 0,182,182,0,0,132,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182,
+ 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,132,132,132,132,132,
+ 132,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,182,182,182,182,182, 182,182,182,132,132,132,132,0,
+ 0,0,177,177,177,177,177,177, 177,177,177,177,177,0,0,0, 0,0,0,0,0,0,0,0, 177,177,177,177,177,0,0,177,
+ 0,177,177,177,177,0,177,177, 177,0,0,0,0,0,177,177, 177,177,177,177,177,177,177,0, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,0,177,177,177,177,177, 0,0,0,0,0,0,177,177, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,177,177,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 132,0,132,132,132,132,132,132, 130,130,130,128,128,128,128,128, 134,0,136,136,136,134,136,134, 132,132,132,134,134,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,134,134, 134,0,134,134,134,134,134,134, 132,134,132,132,132,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,134,132,132,132,132,132,132, 134,0,134,134,134,134,134,134, 132,132,134,132,132,132,132,132,
+ 134,0,134,134,134,134,134,134, 132,134,132,132,132,132,132,132, 134,0,134,134,134,134,134,134, 132,134,132,132,132,132,132,132,
+ },
+},
+
+{ // TAB (0.030M chars) [52]
+ {NULL, NULL, NULL, NULL},
+ 137, 210, 72, 28, 132,
+ {0,0,0,0,0,0,0,0, 0,174,174,0,0,174,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,183,183,183,0,183,183, 183,0,165,165,165,0,183,183, 183,183,183,183,183,183,183,0, 183,183,183,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,0,183,183,183,183,183, 0,0,0,0,0,0,183,0, 0,0,0,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183,
+ 0,0,0,0,0,0,0,0, 0,174,174,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,186,186,186,0,186,186, 186,0,0,0,0,0,186,186, 177,177,177,177,177,177,177,0, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,0,177,177,177,177,177, 0,0,0,0,0,0,177,0, 0,0,0,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 186,186,186,186,186,186,186,186, 186,186,186,186,186,186,186,186,
+ },
+ {0,0,0,0,0,0,0,0, 0,183,183,0,0,136,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183,
+ 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,183, 183,183,183,183,183,183,183,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,176,176,176,0,176,176, 176,0,0,0,0,0,176,176, 176,176,176,176,176,176,176,0, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,0,176,176,176,176,176, 0,0,0,0,0,0,176,0, 0,0,0,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,176,176,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,0,0,0,0, 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 132,0,132,134,134,134,134,132, 128,128,134,134,134,134,138,138, 132,0,132,132,132,132,132,132, 128,128,136,136,136,136,136,136,
+ 132,0,132,132,132,132,132,132, 128,128,136,136,136,136,136,136, 132,0,134,134,134,134,134,134, 128,128,136,136,136,136,136,136,
+ 132,0,132,134,134,134,134,132, 128,128,136,136,136,136,136,136, 132,0,132,134,134,134,134,132, 128,128,136,136,136,136,136,136,
+ },
+},
+
+{ // EUC-CN (0.035M chars) [53]
+ {NULL, NULL, NULL, NULL},
+ 197, 192, 37, 32, 128,
+ {0,0,0,0,0,0,0,0, 0,169,169,0,0,169,119,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,240,216,176,191,186,168,185, 184,166,0,0,118,0,0,0, 0,103,0,0,0,0,29,0, 0,0,0,0,0,0,0,0,
+ 0,0,169,0,197,197,190,192, 190,212,188,187,188,186,190,188, 187,186,186,185,184,186,187,187, 185,185,186,185,193,186,186,190,
+ 185,185,185,187,189,185,186,191, 185,185,185,184,186,185,184,184, 185,184,184,184,184,184,184,184, 186,184,184,184,184,173,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,147,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,134,134,0,0,134,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,134,134,134,134,134,134,134, 134,134,134,134,134,134,134,134, 134,134,134,134,134,134,134,134, 134,134,134,134,134,134,134,134,
+ 134,134,134,134,138,134,138,134, 134,138,134,134,138,134,134,134, 134,134,134,134,138,138,134,134, 134,134,134,134,134,134,134,134,
+ 134,134,134,134,134,134,134,134, 134,134,134,134,138,134,134,134, 134,134,134,134,134,134,134,134, 134,134,134,134,134,134,134,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 170,230,193,197,184,184,184,184, 210,184,185,182,188,181,181,185, 183,207,189,182,181,182,181,180, 181,181,182,185,181,179,187,187,
+ 183,180,180,181,182,180,186,180, 180,185,195,191,183,180,182,180, 179,182,189,183,182,180,192,195, 179,179,181,182,181,179,183,183,
+ 180,180,178,184,182,198,195,192, 180,183,186,180,180,183,181,181, 183,179,180,189,179,179,178,178, 182,179,212,180,186,180,211,0,
+ },
+ {182,0,182,182,180,182,182,182, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 128,128,2,10,14,14,12,8, 76,0,52,52,50,52,52,52, 128,128,128,128,128,128,128,128,
+ 4,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 8,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 8,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 12,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 0,0,0,0,0,0,0,0, 0,0,76,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,134,124,114,118,122,124, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,116,126,130,128,126,136, 0,0,0,0,0,0,0,0, 0,0,116,128,136,132,134,124,
+ 0,0,0,0,0,0,0,0, 0,0,118,130,132,134,130,124, 0,0,0,0,0,0,0,0, 0,0,118,130,132,134,130,126,
+ },
+},
+
+{ // EUC (15478 chars) [54]
+ {NULL, NULL, NULL, NULL},
+ 197, 200, 38, 33, 129,
+ {0,0,0,0,0,0,0,0, 0,173,173,0,0,173,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,234,211,180,190,188,173,189, 189,170,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,174,0,193,195,190,191, 191,208,190,189,190,189,191,190, 190,189,190,189,189,190,189,190, 189,189,190,189,193,189,190,191,
+ 189,189,189,190,191,190,190,193, 189,190,189,189,190,189,189,189, 189,189,189,189,189,189,189,189, 190,189,189,189,189,178,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,140,140,0,0,140,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140,
+ 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140,
+ 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 162,226,192,195,186,186,185,186, 205,186,187,185,188,185,185,186, 184,202,187,184,184,184,183,183, 184,184,184,185,183,183,187,187,
+ 183,183,183,183,184,183,186,183, 183,185,192,189,184,183,184,183, 182,184,187,184,183,182,190,192, 182,182,182,183,184,182,184,184,
+ 182,182,182,185,184,194,191,190, 183,184,185,183,183,184,183,183, 184,182,182,188,182,182,182,182, 184,182,206,183,185,182,206,0,
+ },
+ {176,0,176,176,176,176,176,176, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 0,0,0,0,0,0,0,0, 0,0,2,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,136,124,116,118,122,124, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,118,126,128,128,128,136, 0,0,0,0,0,0,0,0, 0,0,118,128,132,130,130,126,
+ 0,0,0,0,0,0,0,0, 0,0,120,128,130,132,130,126, 0,0,0,0,0,0,0,0, 0,0,120,130,130,130,130,126,
+ },
+},
+
+{ // CNS (15478 chars) [55]
+ {NULL, NULL, NULL, NULL},
+ 197, 200, 38, 33, 129,
+ {0,0,0,0,0,0,0,0, 0,173,173,0,0,173,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,234,211,180,190,188,173,189, 189,170,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,174,0,193,195,190,191, 191,208,190,189,190,189,191,190, 190,189,190,189,189,190,189,190, 189,189,190,189,193,189,190,191,
+ 189,189,189,190,191,190,190,193, 189,190,189,189,190,189,189,189, 189,189,189,189,189,189,189,189, 190,189,189,189,189,178,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,140,140,0,0,140,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140,
+ 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140,
+ 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,140, 140,140,140,140,140,140,140,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 162,226,192,195,186,186,185,186, 205,186,187,185,188,185,185,186, 184,202,187,184,184,184,183,183, 184,184,184,185,183,183,187,187,
+ 183,183,183,183,184,183,186,183, 183,185,192,189,184,183,184,183, 182,184,187,184,183,182,190,192, 182,182,182,183,184,182,184,184,
+ 182,182,182,185,184,194,191,190, 183,184,185,183,183,184,183,183, 184,182,182,188,182,182,182,182, 184,182,206,183,185,182,206,0,
+ },
+ {176,0,176,176,176,176,176,176, 128,128,128,128,128,128,128,128, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128, 2,0,2,2,2,2,2,2, 128,128,128,128,128,128,128,128,
+ 0,0,0,0,0,0,0,0, 0,0,2,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,136,124,116,118,122,124, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,118,126,128,128,128,136, 0,0,0,0,0,0,0,0, 0,0,118,128,132,130,130,126,
+ 0,0,0,0,0,0,0,0, 0,0,120,128,130,132,130,126, 0,0,0,0,0,0,0,0, 0,0,120,130,130,130,130,126,
+ },
+},
+
+{ // UTF-32BE (1032.458M chars) [56]
+ {NULL, NULL, NULL, NULL},
+ 77, 151, 56, 41, 127,
+ {250,137,119,116,158,116,151,92, 103,89,82,111,111,75,204,151, 102,106,71,86,94,92,104,91, 88,92,106,215,97,93,99,106,
+ 79,97,116,106,115,126,78,66, 89,96,94,80,79,92,90,90, 93,101,90,90,72,96,79,91, 82,89,78,88,80,99,84,86,
+ 86,92,83,59,101,100,101,79, 89,93,71,46,102,89,64,68, 78,74,72,65,71,56,71,64, 76,84,65,68,92,98,94,106,
+ 182,177,181,215,194,207,203,198, 193,188,166,179,184,171,90,183, 67,2,0,27,20,10,16,20, 7,0,3,46,24,71,43,104,
+ 160,104,104,120,120,123,106,99, 106,101,107,114,119,100,91,96, 110,88,107,102,79,91,100,101, 99,92,100,94,88,67,104,92,
+ 68,80,83,73,92,62,75,75, 79,64,69,73,88,82,78,88, 92,73,82,84,71,76,74,82, 84,98,91,76,78,66,71,80,
+ 78,89,78,91,66,88,86,94, 97,83,76,79,62,78,71,75, 80,86,62,78,68,93,85,80, 83,80,76,75,69,83,90,83,
+ 82,82,61,78,58,98,56,64, 68,80,85,85,74,85,73,74, 60,46,58,83,68,31,72,64, 71,62,78,78,88,72,62,103,
+ },
+ {231,96,83,86,82,95,79,87, 91,190,189,71,101,175,87,75, 88,93,82,83,89,84,75,84, 89,98,94,150,95,84,82,97,
+ 211,164,198,158,206,163,163,167, 204,160,139,149,163,176,181,194, 186,180,173,172,169,172,164,166, 171,171,175,177,193,193,193,164,
+ 169,173,164,169,169,167,165,166, 164,164,153,153,160,162,163,160, 163,145,162,165,167,153,156,157, 144,147,146,140,138,138,110,171,
+ 103,201,181,192,192,203,181,185, 187,200,161,171,196,187,196,196, 190,145,197,197,203,183,178,178, 170,174,157,147,130,147,113,85,
+ 194,202,196,197,161,172,160,164, 174,168,169,165,163,162,161,172, 167,152,151,154,172,173,173,172, 170,167,172,172,181,172,158,158,
+ 158,165,152,155,174,160,165,166, 161,156,155,157,154,167,173,166, 170,159,161,164,160,162,155,162, 190,174,180,181,186,173,165,168,
+ 96,98,83,90,99,87,83,90, 91,88,75,80,102,90,78,81, 99,97,81,86,97,102,86,85, 95,107,84,92,106,96,96,92,
+ 105,120,91,98,114,104,86,102, 104,125,102,72,80,118,81,79, 79,91,70,110,91,83,108,69, 93,74,94,86,107,97,65,124,
+ },
+ {132,132,126,132,132,132,132,132, 122,128,140,130,230,224,222,224, 60,24,152,14,68,74,40,40, 104,122,122,98,204,218,206,224,
+ 8,96,12,22,38,48,10,20, 44,64,80,56,128,128,128,128, 20,128,26,34,56,70,22,32, 40,58,72,50,128,128,128,128,
+ 20,128,24,34,54,66,22,32, 38,56,70,50,128,128,128,128, 26,128,30,40,62,74,28,38, 40,60,76,52,128,128,128,128,
+ 2,2,2,2,2,2,2,2, 42,62,78,54,128,128,128,128, 134,180,74,128,144,158,114,92, 116,128,122,78,150,192,132,164,
+ 118,142,64,114,106,120,82,106, 14,12,26,10,92,80,68,72, 138,192,94,128,132,154,112,108, 34,52,40,14,124,118,102,106,
+ 142,188,72,144,148,176,108,110, 156,152,90,112,202,162,200,174, 142,200,86,138,146,162,110,118, 156,146,132,106,208,200,212,192,
+ 142,208,88,138,138,162,96,118, 154,152,138,122,208,212,182,178, 138,180,68,144,148,178,114,122, 154,162,112,112,184,192,158,168,
+ 140,128,102,142,150,172,108,116, 148,148,148,148,20,14,4,12, 136,196,112,138,136,176,110,118, 132,148,136,112,148,160,162,150,
+ },
+},
+
+{ // UTF-32LE (1032.461M chars) [57]
+ {NULL, NULL, NULL, NULL},
+ 77, 152, 56, 41, 127,
+ {250,137,119,116,158,116,151,92, 103,89,82,111,111,75,204,150, 102,106,71,86,94,92,104,91, 88,92,106,215,97,93,99,106,
+ 79,97,116,106,114,126,78,66, 89,96,94,144,79,92,90,90, 93,101,90,90,72,96,79,91, 80,89,78,88,80,99,84,86,
+ 86,92,83,58,101,100,101,79, 89,93,71,46,102,89,64,68, 78,74,72,65,71,54,71,64, 76,84,65,68,92,98,94,106,
+ 182,177,181,215,193,207,203,198, 193,188,166,178,184,171,90,183, 67,3,0,27,20,11,17,20, 8,0,4,46,24,71,162,104,
+ 160,104,104,120,120,123,106,99, 106,101,107,114,118,100,91,94, 110,88,107,102,79,91,100,101, 99,92,100,94,88,67,104,92,
+ 68,80,83,73,92,62,75,75, 79,64,71,124,88,82,78,88, 92,73,82,84,71,76,74,82, 84,98,91,76,78,66,71,80,
+ 78,89,78,91,66,88,86,94, 97,83,76,79,62,78,71,75, 80,86,62,78,68,93,85,80, 83,80,76,75,69,83,90,83,
+ 82,82,61,78,57,98,57,64, 68,80,85,85,74,85,73,74, 60,46,57,83,68,31,72,64, 71,62,78,78,88,72,122,103,
+ },
+ {231,96,83,86,82,95,79,87, 91,190,189,71,101,175,87,75, 88,93,82,83,89,84,75,84, 88,98,94,158,95,84,82,97,
+ 211,164,198,159,206,163,163,167, 204,160,139,149,163,176,181,194, 186,180,174,172,169,172,165,166, 171,171,175,177,193,193,193,164,
+ 169,173,164,169,169,167,165,166, 164,164,155,154,161,163,165,160, 163,145,162,165,167,154,157,157, 144,147,146,140,138,138,110,171,
+ 103,201,181,192,192,203,181,185, 187,200,161,171,196,187,196,196, 190,145,197,197,203,183,178,177, 170,174,157,147,130,147,131,85,
+ 193,202,196,197,161,172,160,164, 174,168,169,165,163,162,161,172, 167,152,151,154,172,173,173,172, 170,167,172,172,181,172,158,158,
+ 158,165,152,155,174,160,165,166, 161,155,155,157,154,167,173,166, 170,159,161,164,160,162,155,162, 190,174,180,181,186,173,165,168,
+ 99,99,87,92,100,90,85,93, 93,89,79,83,102,91,81,82, 101,97,82,87,97,102,87,86, 95,107,84,92,106,96,96,92,
+ 105,120,93,102,115,107,92,103, 105,125,103,77,82,118,81,84, 79,91,70,110,91,83,107,69, 93,74,94,86,107,97,65,124,
+ },
+ {132,124,126,132,132,132,132,132, 122,128,140,130,228,224,222,224, 60,14,152,16,66,74,40,40, 104,122,122,98,204,218,206,224,
+ 44,212,2,12,34,104,44,134, 116,112,144,150,220,208,220,130, 20,116,26,34,56,70,22,32, 40,58,72,50,128,128,128,128,
+ 20,104,24,34,54,66,22,32, 38,56,70,50,128,128,128,128, 26,114,30,40,60,74,28,38, 42,60,76,52,128,128,128,128,
+ 2,2,2,2,2,2,2,2, 44,62,78,54,128,128,128,128, 84,190,114,148,160,156,70,62, 116,118,160,144,212,200,222,166,
+ 118,134,64,114,106,120,82,106, 14,12,26,8,90,80,66,72, 138,168,94,128,130,154,112,108, 34,52,40,14,122,116,100,106,
+ 130,162,60,130,122,146,98,100, 146,140,78,102,202,162,172,162, 142,200,86,138,144,162,110,118, 158,146,132,106,208,200,212,192,
+ 142,208,88,138,136,162,96,118, 154,152,138,122,208,212,182,176, 138,180,68,144,148,178,114,122, 156,162,112,112,182,192,156,168,
+ 140,128,102,142,150,172,108,116, 148,148,148,148,18,14,4,12, 124,168,100,124,114,154,98,106, 76,90,78,54,122,120,108,108,
+ },
+},
+
+{ // X-BINARYENC (0 chars) [58]
+ {NULL, NULL, NULL, NULL},
+ 0, 0, 0, 0, 255,
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ },
+},
+
+{ // X-UTF8UTF8 (43.271M chars) [59]
+ {NULL, NULL, NULL, NULL},
+ 150, 194, 32, 13, 129,
+ {191,104,167,173,203,123,113,0, 0,69,69,80,0,107,0,105, 95,0,125,146,116,0,0,0, 104,129,184,0,125,122,159,0,
+ 139,144,162,111,112,105,148,117, 115,101,122,104,217,112,130,114, 151,116,107,120,116,116,98,89, 124,119,136,113,115,116,126,105,
+ 0,0,239,243,62,181,145,62, 62,62,62,147,62,62,62,62, 62,62,62,62,62,62,62,62, 62,62,62,62,62,62,62,62,
+ 0,0,220,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,58,0,0,0,0,5,0, 0,0,36,0,102,23,86,28, 9,0,75,142,74,0,0,0, 89,85,64,0,119,139,99,0,
+ 121,119,159,71,88,75,139,88, 67,69,78,82,156,71,75,61, 134,70,55,54,61,63,56,60, 80,102,119,63,76,71,134,73,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,24,140,0,0,147,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 195,107,138,80,100,83,158,123, 107,139,106,91,157,127,159,122, 111,135,127,120,115,117,108,109, 106,109,120,113,182,78,121,111,
+ 84,149,138,143,143,145,133,136, 138,145,127,129,143,142,149,143, 141,113,143,148,151,137,137,140, 124,119,115,112,96,117,97,106,
+ 108,137,132,140,136,131,138,130, 131,133,119,128,140,133,137,128, 135,113,137,152,143,136,130,123, 116,108,106,112,84,77,91,0,
+ 196,150,226,235,119,165,141,52, 52,52,52,115,52,142,52,124, 121,52,147,165,107,52,52,52, 97,124,126,52,187,166,117,52,
+ 216,191,222,181,200,177,168,187, 191,218,177,170,169,189,169,153, 169,175,160,194,171,165,189,171, 182,163,176,175,198,160,155,150,
+ 0,0,205,182,0,189,133,0, 0,0,0,163,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,209,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 12,0,2,2,2,2,2,2, 0,0,0,0,2,128,2,128, 66,0,20,36,40,48,50,54, 0,0,0,0,2,128,2,128,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,128,128,128,128, 128,0,84,128,128,128,128,128, 0,0,0,0,128,128,128,128,
+ 10,0,2,2,2,2,2,2, 0,0,0,0,128,128,128,128, 128,0,128,128,128,128,128,128, 0,0,0,0,128,128,128,128,
+ 120,0,130,112,192,198,136,140, 2,168,134,98,134,128,124,128, 160,0,168,162,178,176,178,178, 2,2,2,2,170,128,2,128,
+ 178,0,174,178,150,154,158,156, 2,2,2,2,158,128,164,128, 156,0,162,152,182,182,182,184, 2,26,2,2,170,128,10,128,
+ 0,0,128,128,128,128,128,128, 128,108,130,130,2,128,2,128, 0,0,128,128,128,128,128,128, 120,210,124,154,74,128,70,128,
+ 0,0,128,128,128,128,128,128, 140,2,2,2,2,128,2,128, 0,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ },
+},
+
+{ // X-TAM-ELANGO (0.036M chars) [60]
+ {NULL, NULL, NULL, NULL},
+ 126, 180, 58, 30, 129,
+ {0,180,180,180,180,180,180,180, 180,191,191,180,180,191,0,180, 170,170,170,170,170,170,170,170, 170,170,170,170,170,170,0,170,
+ 0,180,180,180,180,180,180,180, 180,180,180,180,0,0,180,0, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,0,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,0,0,180,180,
+ 180,0,180,180,0,0,0,0, 0,0,0,0,180,0,180,180, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,181,181,181,181,181,181,181, 181,191,191,181,181,191,0,181, 171,171,171,171,171,171,171,171, 171,171,171,171,171,171,0,171,
+ 0,181,181,181,181,181,181,181, 181,181,181,181,0,0,181,0, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181,
+ 181,181,181,181,181,181,0,181, 181,181,181,181,181,181,181,181, 181,181,181,181,181,181,181,181, 181,181,181,181,0,0,181,181,
+ 181,0,181,181,0,0,0,0, 0,0,0,0,181,0,181,181, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {0,0,0,0,0,0,0,0, 0,180,180,0,0,180,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,0,
+ 0,180,180,180,180,180,180,180, 180,180,180,180,180,180,0,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,0,180,
+ 0,180,180,180,180,180,180,180, 180,180,180,180,0,0,180,0, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,0,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,0,0,180,180,
+ 180,0,180,180,0,0,0,0, 0,0,0,0,180,0,180,180, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ },
+ {110,0,110,110,110,110,110,110, 110,110,110,110,110,110,110,128, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,128, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,128,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,128, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,128,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,128, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 132,0,132,132,132,132,132,132, 132,132,132,132,132,132,132,128, 136,0,134,134,134,134,134,134, 136,136,136,136,136,136,136,128,
+ 136,0,136,136,136,136,136,136, 136,136,136,136,136,136,136,128, 136,0,134,134,134,134,134,134, 136,136,136,136,136,136,136,128,
+ 136,0,134,134,134,134,134,134, 136,136,136,136,136,136,136,128, 136,0,134,134,134,134,134,134, 136,136,136,136,136,136,136,128,
+ 136,0,136,136,136,136,136,136, 136,136,136,136,136,136,136,128, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ },
+},
+
+{ // X-TAM-LTTMBARANI (0.043M chars) [61]
+ {NULL, NULL, NULL, NULL},
+ 141, 199, 69, 34, 128,
+ {0,178,178,0,178,0,178,178, 0,187,187,178,178,176,0,0, 0,0,0,0,0,168,0,0, 0,0,168,168,168,0,0,168,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 0,176,176,0,176,0,176,176, 0,187,187,176,176,178,0,0, 0,0,0,0,0,165,0,0, 0,0,165,165,165,0,0,165,
+ 0,176,176,176,176,176,177,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,182, 176,176,176,176,176,176,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 176,176,176,176,176,177,176,176,
+ 176,176,176,176,176,176,176,176, 176,176,176,176,176,176,176,176, 181,176,176,176,176,176,176,176, 176,176,176,176,176,189,176,176,
+ },
+ {0,0,0,0,0,0,0,0, 0,178,177,0,0,177,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 185,177,177,177,177,177,178,177, 177,177,177,177,178,177,178,177, 177,177,177,177,177,177,177,177, 177,177,177,177,180,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,0,
+ 0,178,178,0,178,0,178,178, 0,178,178,178,178,0,0,0, 0,0,0,0,0,178,0,0, 0,0,178,178,178,0,0,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,179, 179,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ },
+ {116,0,114,116,116,116,116,116, 118,118,116,118,118,118,118,118, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 134,0,132,134,134,134,134,134, 132,132,132,132,132,132,132,132, 138,0,136,138,138,138,138,138, 136,136,136,136,136,136,136,136,
+ 138,0,138,138,138,138,138,138, 136,136,136,136,136,136,136,136, 138,0,138,138,138,138,138,138, 136,136,136,136,136,136,136,136,
+ 138,0,136,138,138,138,138,138, 136,136,136,136,136,136,136,136, 138,0,138,138,138,138,138,138, 136,136,136,136,136,136,136,136,
+ 138,0,136,138,138,138,138,138, 136,136,136,136,136,136,136,136, 136,0,142,138,136,136,136,136, 136,136,136,136,136,136,136,136,
+ },
+},
+
+{ // X-TAM-SHREE (0.037M chars) [62]
+ {NULL, NULL, NULL, NULL},
+ 140, 204, 70, 30, 129,
+ {0,0,0,0,0,0,0,0, 0,188,179,0,0,179,0,0, 0,168,0,0,0,168,0,0, 0,0,0,168,0,0,0,168,
+ 0,178,178,178,178,178,178,178, 178,178,178,178,0,0,178,178, 178,178,178,178,178,178,0,0, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,0,178,
+ 0,0,0,0,0,0,0,0, 0,188,178,0,0,178,0,0, 0,169,0,0,0,169,0,0, 0,0,0,169,0,0,0,169,
+ 0,179,179,179,179,179,179,179, 179,179,179,179,0,0,179,179, 179,179,179,179,179,179,0,0, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,0,179,
+ },
+ {0,0,0,0,0,0,0,0, 0,179,179,0,0,179,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,0,
+ 0,0,0,0,0,0,0,0, 0,179,0,0,0,0,0,0, 0,179,0,0,0,179,0,0, 0,0,0,179,0,0,0,179,
+ 0,179,179,179,179,179,179,179, 179,179,179,179,0,0,179,179, 179,179,179,179,179,179,0,0, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,0,179,
+ },
+ {132,0,132,132,132,132,132,132, 134,134,132,132,132,132,132,132, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 118,0,118,118,118,118,118,118, 118,118,118,118,118,118,118,118, 136,0,136,138,138,138,138,136, 138,138,138,138,138,138,138,138,
+ 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138, 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138,
+ 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138, 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138,
+ 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138, 136,0,136,136,136,136,136,136, 138,138,138,138,138,138,138,138,
+ },
+},
+
+{ // X-TAM-TBOOMIS (0.038M chars) [63]
+ {NULL, NULL, NULL, NULL},
+ 139, 205, 71, 31, 129,
+ {178,0,0,0,0,0,0,0, 0,178,178,0,0,178,0,0, 0,0,0,168,168,0,0,0, 0,0,0,0,0,0,0,0,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,0,0,0,0,0,0,0, 0,178,178,0,0,178,0,0, 0,0,0,168,168,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,200,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 179,178,178,178,181,190,178,188,
+ },
+ {0,0,0,0,0,0,0,0, 0,178,178,0,0,178,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 200,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,180,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,177,
+ 178,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,178,178,0,0,0, 0,0,0,0,0,0,0,0,
+ 177,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ },
+ {134,0,130,132,132,132,132,132, 132,132,132,132,132,132,132,132, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 94,94,94,94,94,94,94,94,
+ 118,0,114,116,116,116,116,118, 116,116,118,116,116,116,116,116, 138,0,134,136,136,136,136,138, 136,136,138,136,136,136,136,136,
+ 138,0,134,136,136,136,136,136, 136,136,138,136,136,136,136,136, 138,0,134,136,136,136,136,138, 136,136,138,136,136,136,136,136,
+ 138,0,134,136,136,136,136,138, 136,136,138,136,136,136,136,136, 138,0,134,136,136,136,136,138, 136,136,138,136,136,136,136,136,
+ 134,0,144,134,134,134,134,134, 136,136,138,136,136,136,136,136, 134,0,138,136,134,134,134,134, 136,136,138,136,136,136,136,136,
+ },
+},
+
+{ // X-TAM-TMNEWS (0.037M chars) [64]
+ {NULL, NULL, NULL, NULL},
+ 141, 205, 71, 29, 128,
+ {0,0,0,0,0,0,0,0, 0,179,179,0,0,179,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 0,0,0,0,0,0,0,0, 0,179,179,0,0,179,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,190,179,180,181,179, 180,179,179,179,179,179,179,179,
+ },
+ {0,0,0,0,0,0,0,0, 0,179,179,0,0,179,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 180,179,180,179,179,179,179,179, 179,179,179,179,180,179,180,179, 179,179,179,179,179,179,179,179, 179,179,180,179,181,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179,
+ 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,179, 179,179,179,179,179,179,179,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 179,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180, 180,180,180,180,180,180,180,180,
+ },
+ {138,0,136,136,138,138,138,138, 128,128,136,136,136,136,136,136, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 128,128,2,2,2,2,2,2, 128,0,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 138,0,136,136,136,136,136,136, 128,128,136,136,136,136,136,136, 138,0,136,136,138,138,138,136, 128,128,136,136,136,136,136,136,
+ 138,0,136,136,138,138,138,136, 128,128,136,136,136,136,136,136, 138,0,136,136,138,138,138,136, 128,128,136,136,136,136,136,136,
+ 138,0,136,136,138,138,138,136, 128,128,136,136,136,136,136,136, 136,0,140,136,136,136,136,136, 128,128,136,136,136,136,136,136,
+ },
+},
+
+{ // X-TAM-WEBTAMIL (0.050M chars) [65]
+ {NULL, NULL, NULL, NULL},
+ 142, 193, 66, 37, 129,
+ {178,178,178,178,178,178,178,178, 178,186,186,178,178,186,178,178, 169,169,169,169,169,169,0,169, 169,169,169,169,169,169,169,169,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,179, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178, 178,178,178,178,178,178,178,178,
+ 174,174,174,174,174,174,174,174, 174,186,186,174,174,186,174,174, 162,162,162,162,162,162,0,162, 162,162,162,162,162,162,162,162,
+ 0,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,181, 174,174,174,174,174,174,174,174,
+ 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174,
+ 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174, 174,174,174,174,174,174,174,174,
+ },
+ {0,0,0,0,0,0,0,0, 0,178,177,0,0,177,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 158,177,177,177,177,177,178,177, 177,177,177,177,177,177,178,177, 177,177,177,177,177,177,177,177, 177,177,177,177,179,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,178,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,0,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,133,177, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,178, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177,
+ 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177, 177,177,177,177,177,177,177,177,
+ },
+ {108,0,108,108,108,108,108,108, 110,110,110,110,110,110,110,110, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,0,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 134,0,134,134,134,134,134,134, 132,132,132,132,132,132,132,132, 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134,
+ 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134, 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134,
+ 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134, 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134,
+ 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134, 138,0,138,138,138,138,138,138, 134,134,134,134,134,134,134,134,
+ },
+},
+
+{ // UTF8CP1252 (178.156M chars) [66]
+ {NULL, NULL, NULL, NULL},
+ 127, 200, 59, 31, 133,
+ {181,189,183,184,176,167,163,162, 171,165,167,170,170,164,161,165, 163,161,165,161,164,166,165,166, 163,164,168,163,171,163,158,162,
+ 206,174,166,168,175,173,174,175, 177,166,173,171,187,176,178,175, 178,168,166,174,175,166,165,168, 186,173,179,177,184,174,171,171,
+ 119,121,206,216,183,183,147,141, 116,157,118,125,127,125,181,171, 210,198,125,120,113,136,145,187, 200,196,150,157,110,121,122,116,
+ 177,173,191,209,190,201,198,198, 188,185,166,178,183,167,123,183, 120,134,134,122,116,132,155,115, 122,120,125,97,146,127,124,122,
+ 186,143,146,128,162,163,140,136, 145,136,144,136,144,130,143,126, 134,159,199,183,181,182,183,161, 136,161,149,139,147,147,140,133,
+ 194,145,152,160,146,142,140,152, 139,166,140,162,150,144,159,142, 165,141,142,133,158,130,129,165, 138,130,156,172,141,139,132,152,
+ 146,177,150,157,188,159,153,165, 178,205,165,144,125,177,148,136, 125,164,126,181,156,148,180,118, 161,109,162,137,189,137,127,170,
+ 196,183,151,159,189,172,153,166, 185,210,166,152,154,180,147,138, 133,164,153,186,156,147,178,152, 160,162,164,140,188,144,126,109,
+ },
+ {148,50,56,78,0,0,0,0, 0,146,153,0,0,166,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 225,137,174,103,110,121,177,155, 149,159,126,115,183,160,179,142, 162,172,165,154,151,151,144,143, 141,143,157,138,205,108,138,142,
+ 124,187,176,189,179,184,170,182, 175,169,154,158,186,182,198,181, 175,156,199,194,193,169,173,155, 158,150,147,134,124,138,112,134,
+ 122,193,181,189,179,191,172,183, 177,178,152,159,187,182,199,184, 175,153,200,203,195,176,174,143, 147,148,143,122,135,97,118,45,
+ 189,193,197,200,179,177,173,171, 177,169,174,172,176,171,162,170, 167,165,159,165,169,171,167,170, 167,172,168,170,176,168,162,169,
+ 201,180,183,168,179,169,167,182, 176,187,173,166,167,178,173,174, 182,177,171,180,173,177,171,169, 186,175,179,179,186,179,179,171,
+ 129,129,169,152,124,152,130,133, 107,158,114,130,116,114,119,120, 120,118,126,120,107,124,147,116, 137,132,118,122,118,119,119,156,
+ 174,145,176,207,186,198,193,190, 186,184,161,173,176,161,128,175, 133,123,125,123,107,155,137,103, 119,110,131,110,136,122,127,123,
+ },
+ {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,2,4,2,6, 2,0,2,2,2,2,2,2, 0,0,0,0,4,14,2,16,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,2,2,2,2, 2,0,2,2,2,2,2,2, 0,0,0,0,2,2,2,2,
+ 2,0,2,2,2,2,2,2, 0,0,0,0,2,2,2,2, 32,0,2,4,2,2,2,2, 0,0,0,0,2,2,2,2,
+ 156,0,136,152,124,138,112,108, 126,134,130,124,120,126,140,96, 138,0,138,136,122,116,134,138, 124,130,118,122,140,122,146,112,
+ 150,0,140,156,114,112,112,112, 108,122,140,112,150,128,142,122, 144,0,142,152,118,120,114,114, 124,134,120,126,132,136,144,128,
+ 0,0,90,80,150,152,98,98, 134,120,142,130,128,128,70,112, 0,0,102,98,150,148,124,110, 134,128,128,144,104,128,66,114,
+ 0,0,138,118,64,72,142,142, 138,138,122,134,110,120,110,140, 0,0,126,122,70,82,148,144, 88,108,108,110,162,194,128,196,
+ },
+},
+
+}; // End unigram_table
+
+static const uint8 kMostLikelyEncoding[] = {
+// 00xx
+ 37,39,39,39,39,39,39,39, 39,37,37,39,39,39,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 37,37,37,39,39,37,39,39, 39,39,37,39,37,37,39,37, 39,39,39,39,39,39,39,39, 39,39,37,39,37,39,37,39,
+ 37,39,39,39,39,37,39,39, 37,37,39,37,39,39,39,39, 37,39,37,37,37,37,37,37, 39,39,39,37,39,37,56,39,
+ 39,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37, 39,37,37,37,39,37,39,56,
+ 56,39,56,56,39,39,39,39, 56,56,56,56,56,56,39,56, 56,56,39,39,57,56,56,39, 56,56,56,57,39,39,56,39,
+ 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,57, 39,56,39,39,39,56,39,39, 39,39,39,39,39,57,39,39,
+ 39,39,57,39,39,39,39,39, 39,39,39,39,56,39,39,39, 39,39,39,39,39,57,39,39, 57,57,57,57,56,57,56,57,
+ 39,37,39,39,37,39,39,39, 39,37,39,39,39,37,39,39, 39,37,39,37,39,39,39,39, 39,39,39,39,37,39,39,57,
+ // 01xx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37,
+ 39,39,39,37,37,37,39,39, 39,39,37,37,39,39,39,37, 39,37,39,37,37,37,37,37, 37,37,39,39,39,37,37,39,
+ 39,37,37,37,37,39,37,39, 37,39,39,39,39,39,37,39, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,39,39,39,39,37,37, 37,39,39,37,39,39,39,39, 39,37,39,39,39,39,39,39, 39,39,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37,
+ 39,37,37,39,39,39,39,39, 37,39,37,39,39,39,37,37, 37,39,37,39,39,37,39,39, 37,39,37,37,37,37,37,39,
+ 37,39,39,39,37,39,39,39, 39,39,39,39,39,37,37,37, 37,37,37,39,37,39,37,39, 37,56,56,56,37,56,39,56,
+ 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,37, 39,39,39,39,39,37,39,37, 39,39,39,37,39,39,37,39,
+ // 02xx
+ 37,37,37,39,37,37,37,39, 39,39,39,37,37,39,37,37, 37,8,37,37,37,37,37,37, 37,37,37,37,39,39,37,39,
+ 37,37,39,40,37,39,37,37, 39,39,39,39,39,39,39,39, 39,37,37,37,37,37,40,37, 40,40,39,39,39,39,37,37,
+ 6,39,39,40,39,37,39,39, 39,39,37,37,39,39,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,39,37,
+ 37,37,37,37,39,37,37,37, 37,37,39,37,37,37,37,39, 37,37,39,37,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,39,37,37,37,39, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37,
+ 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,39,39,37,37,39, 39,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,39,37,37,37,37,37,39, 37,39,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,39,37,37,37,37,
+ // 03xx
+ 56,37,37,37,56,37,37,37, 37,37,39,56,37,39,39,37, 39,37,37,39,37,39,37,37, 37,39,56,39,37,37,39,39,
+ 39,39,39,37,37,37,39,39, 39,39,39,39,39,39,39,39, 37,39,39,39,39,39,39,39, 39,56,39,39,39,39,39,56,
+ 39,39,39,56,56,39,39,39, 39,39,39,39,39,39,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,39,37,
+ 37,37,37,39,37,37,37,39, 37,39,37,37,37,39,39,39, 39,37,37,37,39,37,37,37, 37,37,37,37,37,37,40,37,
+ 39,37,37,39,39,37,37,37, 37,37,37,37,39,37,37,39, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37,
+ 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 37,37,37,39,37,39,37,37, 37,39,39,37,39,37,37,37, 39,37,39,37,39,37,37,37, 39,39,39,39,39,39,37,37,
+ // 04xx
+ 56,39,37,39,37,39,37,37, 37,39,39,37,39,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,39,37,37,37,39,39, 37,39,39,37,39,39,39,37, 39,39,37,39,39,39,39,37, 39,39,39,39,39,39,39,37,
+ 39,39,39,39,39,37,39,39, 39,39,39,39,39,37,39,39, 37,39,37,37,39,37,37,37, 37,39,37,37,39,39,37,39,
+ 37,39,37,39,39,37,37,37, 39,39,37,39,37,37,39,39, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,39,37,
+ 37,37,37,39,37,37,37,37, 37,37,37,39,37,37,37,37, 37,37,37,39,37,37,37,39, 37,37,37,37,39,37,37,39,
+ 39,37,39,39,37,37,39,39, 37,39,37,37,39,39,37,37, 37,39,37,37,39,39,37,39, 37,39,37,39,39,39,39,37,
+ 37,37,37,39,37,37,39,37, 39,37,39,37,37,39,39,37, 39,39,37,37,37,39,39,39, 37,37,37,37,37,37,37,37,
+ 39,37,39,39,39,39,39,39, 37,37,39,37,39,37,37,37, 37,37,37,37,39,39,39,37, 39,39,39,39,37,39,37,37,
+ // 05xx
+ 56,37,37,37,37,37,37,37, 37,37,39,37,37,39,37,37, 37,37,37,39,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,39,39,37,37,37,39,39, 39,39,39,39,39,39,39,39, 37,39,39,39,37,37,40,37, 37,39,39,39,39,39,39,39,
+ 37,39,40,39,37,37,37,39, 37,37,37,37,40,37,37,37, 37,37,37,39,37,37,39,37, 37,37,37,37,37,37,39,39,
+ 39,37,37,37,39,37,37,37, 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,39,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,39,37, 39,39,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,37,39,37,37,37,
+ // 06xx
+ 56,37,37,37,37,37,37,37, 37,37,39,37,39,39,39,39, 37,37,37,39,37,37,37,37, 39,39,37,37,37,37,37,37,
+ 39,39,39,37,37,37,39,37, 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,39,39,39,39,39,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37, 37,37,37,37,39,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,39,37,39,37,37,37, 37,37,37,37,37,37,37,39,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,39,37,37,37,37,37, 37,37,37,39,37,37,37,37,
+ 39,37,37,39,37,37,37,39, 37,37,37,39,39,39,39,37, 37,37,37,37,37,37,37,39, 37,37,37,39,37,37,37,39,
+ 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,39, 37,37,37,39,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 39,39,39,39,39,37,39,37, 39,37,39,37,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 07xx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37,
+ 39,37,39,40,37,37,37,37, 37,39,37,37,39,37,39,37, 37,39,37,40,37,40,37,37, 37,37,40,37,39,37,37,37,
+ 37,37,39,37,37,37,40,40, 37,37,37,40,37,37,39,37, 37,37,39,37,37,39,37,37, 37,39,39,37,39,39,37,37,
+ 37,37,37,39,37,37,37,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,39,37,37,37,39,37, 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 08xx
+ 37,37,37,37,37,37,37,37, 3,39,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,39,40,40,37,37,37, 40,40,37,37,37,37,37,37, 37,39,40,40,40,40,40,40, 40,40,40,40,40,40,40,37,
+ 40,40,40,40,37,37,37,37, 40,40,37,37,40,37,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,39,37,39,37,
+ 37,39,37,37,37,37,37,39, 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,40,37,
+ 37,37,39,37,37,37,37,37, 37,37,39,37,39,37,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,39,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,37, 37,39,37,37,37,37,37,37, 39,37,37,37,37,37,37,37,
+ 37,37,37,39,37,37,39,37, 39,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,39,
+ // 09xx
+ 37,37,37,37,37,37,37,37, 37,0,0,37,37,0,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 0,10,0,0,0,0,0,0, 0,0,0,0,0,10,0,10, 10,0,0,0,0,0,0,0, 0,0,0,0,0,10,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ // 0Axx
+ 37,37,37,37,37,37,37,37, 37,0,0,37,37,0,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 0,10,0,0,0,0,0,0, 0,0,0,0,0,10,0,10, 10,0,0,0,0,0,0,0, 0,0,0,0,0,10,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ // 0Bxx
+ 56,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,40, 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37,
+ 39,40,39,40,37,37,37,37, 37,39,37,37,39,39,39,40, 37,39,39,39,39,39,39,40, 39,39,39,37,39,37,37,37,
+ 37,37,40,40,37,37,40,37, 39,37,37,37,39,37,39,37, 39,37,37,37,37,37,37,39, 37,37,37,37,39,37,37,37,
+ 37,37,37,37,37,37,37,39, 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,39, 37,37,39,37,37,37,40,37,
+ 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 0Cxx
+ 57,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,39,37,37,37,37,37, 37,39,37,37,39,37,37,37, 37,39,37,37,37,37,37,37, 37,37,39,37,39,37,37,39,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,39,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 0Dxx
+ 37,37,37,37,37,37,37,37, 37,0,0,37,37,0,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 0,10,0,0,0,0,0,0, 0,0,0,0,0,10,0,10, 10,0,0,0,0,0,0,0, 0,0,0,0,0,10,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ // 0Exx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,57,56,57,56,37,56,37, 56,56,37,56,56,37,39,39, 57,57,57,57,57,57,57,57, 57,57,57,57,57,57,57,57,
+ 57,57,56,57,57,57,57,57, 57,57,56,57,57,57,56,56, 57,56,57,56,56,57,56,57, 56,56,56,56,57,56,56,56,
+ 56,56,56,56,56,56,39,56, 56,56,56,56,56,56,56,56, 56,56,56,56,56,37,56,56, 37,56,39,56,56,37,37,37,
+ 37,37,37,39,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,39,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 6,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,37,
+ 37,37,37,37,37,37,37,37, 37,39,37,39,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 0Fxx
+ 37,37,40,37,37,37,37,37, 40,42,42,37,37,42,40,37, 37,37,37,37,37,37,37,37, 37,40,37,57,37,37,37,37,
+ 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42,
+ 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42,
+ 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,42, 42,42,42,42,42,42,42,37,
+ 37,39,42,37,37,37,37,37, 37,37,37,37,37,37,39,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 10xx
+ 56,39,37,37,39,37,39,37, 37,37,39,37,37,37,37,37, 37,37,37,39,37,37,37,37, 37,37,37,37,39,39,37,39,
+ 39,40,39,37,37,37,37,37, 40,39,40,37,39,39,39,39, 37,39,37,39,40,40,37,37, 40,40,39,39,39,37,37,39,
+ 37,40,40,40,37,40,37,37, 37,40,40,37,40,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,39,37, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,39,37,39,37, 39,37,37,39,37,37,37,37, 39,39,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 3,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37,
+ // 11xx
+ 56,39,37,37,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,40,39,40,37,37,37,37, 37,37,37,37,37,37,39,37, 37,40,40,40,40,40,40,37, 40,40,40,40,40,40,40,40,
+ 37,40,40,40,40,37,40,40, 40,40,40,40,37,40,37,39, 39,37,37,37,39,37,37,37, 37,37,37,37,39,37,37,37,
+ 37,37,39,39,37,37,37,37, 39,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,39,37,37,40,37,
+ 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,3,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37,
+ // 12xx
+ 37,37,37,37,39,39,37,37, 37,37,37,39,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,40,39,39,37,37,37,37, 37,39,37,39,37,37,39,39, 37,39,37,40,37,37,37,37, 39,40,37,39,39,37,37,37,
+ 40,37,37,37,37,39,37,37, 37,37,39,37,37,37,37,37, 39,37,39,37,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 39,37,39,37,37,37,37,37, 37,39,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,40,37,
+ 37,37,39,37,37,37,37,39, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 13xx
+ 37,39,37,37,37,37,37,37, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37,
+ 39,37,37,40,37,37,37,37, 37,37,39,37,39,39,37,37, 37,39,40,37,40,37,37,37, 37,37,40,37,39,37,37,37,
+ 37,37,37,40,37,37,37,37, 37,37,37,37,40,37,39,37, 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,39,37,39,37, 39,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,40,39,
+ 37,37,37,37,37,39,37,37, 37,37,39,37,39,37,37,37, 37,37,37,37,37,37,37,39, 37,39,39,37,39,37,37,39,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,6,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 39,37,37,39,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,39,37,39,37,37, 37,37,37,37,37,37,37,39,
+ // 14xx
+ 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,39,39,39,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,40,37,39,37,37, 39,37,37,39,39,39,39,37, 37,37,37,39,40,40,40,40, 40,37,40,40,39,37,40,40,
+ 40,40,40,40,40,37,39,39, 40,40,40,40,37,40,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,39, 37,37,37,37,37,37,37,37, 39,37,37,39,37,39,40,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,37,39,37,39,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 15xx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,40,39,40,39,37,39,37, 37,37,37,37,39,37,37,37, 37,37,39,37,40,40,37,37, 37,40,37,37,39,37,40,40,
+ 37,40,40,37,37,37,37,37, 37,37,40,37,40,37,37,37, 37,37,37,37,37,37,39,37, 39,39,37,37,37,37,37,39,
+ 39,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,40,37,
+ 39,37,37,37,37,37,37,39, 39,37,37,37,37,37,37,39, 37,39,39,37,37,37,37,37, 37,37,37,37,37,39,37,37,
+ 37,39,37,37,37,39,37,39, 37,37,37,39,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37,
+ 37,37,39,37,37,37,37,37, 39,37,37,37,37,37,37,37, 39,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 16xx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,39,37,37,37,37,37,37, 37,40,37,37,37,37,37,40, 37,39,37,40,37,37,40,37, 40,40,37,40,37,37,40,37,
+ 37,40,37,37,40,37,40,37, 40,37,37,40,37,37,39,37, 37,37,37,39,37,37,37,39, 37,39,37,37,37,37,37,37,
+ 39,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 17xx
+ 56,37,37,37,37,37,37,37, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,39,37,37, 40,37,37,37,37,37,40,37, 37,39,40,37,37,37,37,37, 37,37,37,37,37,40,40,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,39,39,39,37,39,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 39,37,37,37,39,37,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,40,37,
+ 39,37,39,37,37,39,37,39, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 18xx
+ 37,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,39,37,40,37,37,37,37, 37,37,37,37,40,37,37,40,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 39,37,37,37,37,37,37,39, 37,37,39,37,39,37,37,37,
+ 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ 37,37,37,37,37,37,37,39, 37,37,39,37,37,37,37,37, 37,37,37,37,39,37,37,37, 39,39,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 19xx
+ 37,37,37,37,37,37,37,37, 37,39,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,39,37,40,37,37,37, 37,37,37,37,37,37,40,37, 37,37,37,37,37,37,37,37, 37,37,40,37,37,40,37,37,
+ 37,37,37,40,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,39,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,39,37,37,37,37, 37,37,39,37,37,37,37,37, 37,39,37,37,37,37,37,37, 37,39,37,37,37,39,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 1Axx
+ 37,37,37,37,37,37,37,37, 37,37,4,37,37,37,37,40, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,40,39,40,37,37,37,37, 37,37,37,37,37,37,37,37, 37,39,37,37,40,40,37,37, 37,37,37,37,37,37,40,6,
+ 40,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,39, 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37,
+ 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 1Bxx
+ 56,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,57,37,37,37, 57,37,37,37,37,37,37,37, 37,39,37,37,40,40,37,37, 40,37,37,37,37,40,37,37,
+ 37,37,37,37,37,40,37,37, 37,37,37,37,37,37,56,37, 37,37,39,37,39,37,37,37, 37,37,39,37,37,39,37,37,
+ 37,39,37,37,37,37,37,39, 37,37,37,37,39,39,39,37, 37,37,39,39,37,37,37,37, 37,37,37,37,37,37,40,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,39,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 1Cxx
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,37,40,37,39,37,37, 37,37,40,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,40,37,37,37,37, 37,40,40,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,39,39,37,37,37,37,37,
+ 37,37,37,37,39,37,37,37, 37,39,37,37,37,37,37,37, 37,37,37,39,37,39,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,39, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,39, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,39,37,37,37,
+ 37,39,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 1Dxx
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 39,37,40,40,37,39,37,37, 37,37,37,37,37,37,37,37, 37,39,37,37,37,37,37,37, 40,37,37,37,37,40,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39, 37,37,39,37,37,37,37,37, 37,37,37,37,37,37,39,37,
+ 39,37,37,37,37,37,37,39, 37,37,37,37,37,37,39,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,39,37,37,37, 37,37,37,39,37,39,39,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 1Exx
+ 57,37,37,37,37,37,37,37, 37,39,39,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,37,
+ 39,40,39,40,37,37,39,39, 37,39,37,37,39,39,39,39, 37,40,42,40,40,40,40,37, 40,40,39,39,39,40,37,39,
+ 37,39,42,37,37,37,42,37, 42,39,40,40,40,39,39,39, 39,37,37,37,39,39,37,37, 37,37,42,37,37,37,37,37,
+ 37,39,37,39,39,39,37,37, 39,39,42,39,37,37,39,39, 39,37,37,37,39,39,37,39, 37,39,37,37,37,37,40,37,
+ 37,37,39,37,37,37,37,37, 37,37,37,37,37,39,37,37, 39,37,39,37,37,37,37,39, 37,37,37,37,37,37,39,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 1Fxx
+ 37,37,37,37,39,37,39,37, 37,39,37,37,37,39,40,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,40,42,40,37,37,37,37, 42,37,37,42,39,37,37,42, 37,39,40,40,40,40,40,40, 40,40,40,40,40,40,42,40,
+ 40,40,42,40,40,40,40,42, 40,40,40,40,42,42,37,37, 39,37,37,37,37,37,37,39, 37,37,39,37,37,37,37,37,
+ 37,39,37,37,37,37,39,39, 37,37,37,37,37,37,37,37, 37,37,37,37,37,39,37,39, 37,37,37,37,37,37,40,37,
+ 37,37,39,37,37,37,37,37, 37,37,37,37,37,39,37,37, 39,37,37,37,37,37,39,37, 37,37,37,37,37,37,39,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,39,37,37,37,37, 37,37,37,39,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,39,
+ // 20xx
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ // 21xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 22xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 23xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 24xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 25xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 26xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 27xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 28xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 29xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 2Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 2Bxx
+ 2,39,1,37,37,37,37,5, 3,2,2,37,3,2,42,42, 37,40,37,37,37,40,37,37, 40,37,40,57,37,37,37,21,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,3,3,3,11,11,11,11, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,11,12,11,12,36,12,4,
+ 11,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3,
+ 2,2,2,2,2,2,2,11, 2,1,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,11,2,1,11,13,6,
+ 14,10,2,2,2,2,2,2, 2,2,2,2,2,10,17,2, 10,10,10,10,10,10,1,17, 17,17,17,2,11,10,13,37,
+ // 2Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 2Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 2Exx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 2Fxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 30xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 31xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 32xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 33xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 34xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 35xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 36xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 37xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 38xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 39xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Bxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Exx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 3Fxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 40xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 41xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 42xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 43xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 44xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 45xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 46xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 47xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 48xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 49xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Bxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Exx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 4Fxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 50xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 51xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 52xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 53xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 54xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 55xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 56xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 57xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 58xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 59xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Bxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Exx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 5Fxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 60xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 61xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 62xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 63xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 64xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 65xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 66xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 67xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 68xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 69xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Bxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Exx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 6Fxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 70xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 71xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 72xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 73xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,31, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,33,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 74xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 75xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 76xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 77xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 78xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 79xx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 7Axx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 7Bxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 7Cxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 7Dxx
+ 37,37,37,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,37,
+ 3,10,11,11,11,11,11,11, 11,11,0,11,11,10,0,10, 10,11,11,11,11,11,11,11, 0,11,0,11,11,10,0,0,
+ 11,1,11,11,11,11,11,11, 11,11,1,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,1,11,11,11,11,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,11, 1,1,1,1,1,1,1,1,
+ 11,1,11,1,1,1,1,11, 11,11,11,11,1,1,11,11, 1,1,1,1,11,1,1,11, 1,11,1,11,11,1,1,1,
+ // 7Exx
+ 2,39,39,39,39,39,39,39, 39,2,2,39,39,2,42,42, 39,39,39,39,39,39,39,39, 39,39,39,56,39,39,39,39,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,3,
+ 3,3,3,3,39,39,11,11, 11,3,3,3,3,3,3,3, 3,3,3,3,3,11,3,3, 39,11,39,3,39,39,39,37,
+ 11,3,3,3,11,3,3,3, 3,3,3,3,3,3,11,3, 11,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3,
+ 2,2,2,2,2,2,2,11, 2,2,2,2,11,11,2,2, 2,11,2,11,11,2,2,2, 2,37,11,11,11,11,11,11,
+ 17,11,2,2,2,2,2,2, 2,2,2,2,2,2,17,2, 14,17,17,11,37,1,11,17, 16,17,11,39,37,11,39,37,
+ // 7Fxx
+ 56,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39, 39,39,39,37,39,37,37,39, 37,39,39,39,39,37,39,39,
+ 39,39,39,39,39,39,39,39, 39,37,39,39,39,39,39,39, 37,39,39,39,39,39,37,39, 39,39,37,39,39,39,39,39,
+ 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,37,39, 37,37,37,37,37,39,37,37, 39,37,39,37,37,37,39,37,
+ 37,39,39,37,37,37,37,39, 39,37,37,37,37,37,37,39, 37,37,37,39,37,37,37,39, 39,37,39,37,39,37,37,39,
+ 37,39,37,37,39,37,39,39, 37,39,37,37,39,37,37,37, 39,39,39,39,37,39,37,39, 39,37,37,39,39,39,39,39,
+ 39,39,39,39,37,39,39,39, 37,37,39,39,39,39,39,39, 39,39,39,39,37,39,39,39, 39,39,39,39,39,37,39,39,
+ 39,37,39,39,39,37,39,39, 39,39,39,39,37,39,39,39, 39,39,37,39,37,39,39,39, 56,56,39,56,56,56,56,56,
+ 37,39,39,39,39,39,39,39, 39,37,39,39,39,39,39,39, 37,37,39,39,39,39,39,39, 39,39,39,37,37,39,39,37,
+ // 80xx
+ 11,37,39,37,39,37,39,39, 39,2,2,39,37,2,39,39, 37,39,39,39,39,37,39,37, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,11, 2,11,11,2,11,11,11,11, 11,11,11,2,11,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,11,2,2, 2,2,2,2,2,2,2,39,
+ 2,2,2,2,36,2,2,36, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 59,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 10,39,2,2,2,10,37,2, 10,2,39,2,37,39,2,2, 2,2,2,10,39,2,33,11, 2,2,2,2,37,5,37,5,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 36,31,39,39,39,31,37,39, 37,39,37,39,11,37,4,37,
+ // 81xx
+ 56,39,37,39,39,37,39,37, 37,2,2,39,39,2,39,39, 37,37,39,39,39,39,39,39, 39,37,37,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,6,6,6,6, 6,6,2,6,6,6,2,6, 6,6,6,2,2,6,6,2, 6,2,6,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,37,
+ 36,6,2,6,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,6,2, 2,2,6,2,6,2,2,2,
+ 2,6,6,2,2,6,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 10,39,2,2,2,10,4,11, 6,6,6,6,6,6,2,2, 2,11,11,11,11,39,12,19, 2,2,6,2,6,6,6,6,
+ 2,6,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 6,6,6,6,6,6,6,6, 4,4,37,39,6,19,39,37,
+ // 82xx
+ 56,39,39,39,39,39,39,37, 37,2,2,39,39,2,42,39, 39,39,37,39,39,39,39,39, 39,39,39,39,39,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,6, 6,6,6,6,6,6,6,6, 6,2,2,2,2,2,2,2,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,2,2,2,2,2,37,
+ 2,2,2,2,2,6,2,2, 2,2,2,2,2,2,6,2, 6,36,2,2,6,6,6,6, 2,2,2,36,2,2,2,6,
+ 6,2,6,2,6,2,6,2, 6,6,6,6,2,6,6,2, 2,6,6,6,2,6,6,6, 2,2,2,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,2,2,2,6,6, 6,6,6,6,6,6,6,2, 6,6,31,12,11,31,39,19, 5,19,39,19,11,33,39,37,
+ // 83xx
+ 56,39,37,37,37,37,39,39, 39,2,2,39,39,2,37,39, 39,39,39,39,39,39,39,37, 39,39,39,39,37,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,37,
+ 6,6,6,2,6,6,2,6, 2,6,6,6,6,6,2,6, 2,2,2,6,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,6,
+ 6,6,59,6,6,59,59,6, 19,6,6,6,6,6,2,2, 2,2,6,6,19,6,6,11, 2,2,2,2,37,6,4,4,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 31,4,11,10,39,19,19,11, 37,39,39,39,39,39,39,37,
+ // 84xx
+ 57,39,39,39,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 37,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,11,2,2,2, 2,2,2,11,2,2,2,2, 2,2,2,11,2,2,11,11, 2,2,11,2,2,2,2,2,
+ 37,11,2,2,11,2,2,11, 2,2,2,11,2,2,11,2, 2,6,2,2,6,6,39,2, 6,6,11,6,2,6,2,37,
+ 36,2,2,2,2,36,2,2, 2,2,2,36,2,36,36,2, 2,2,2,36,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,2,59,6,2,6,2,2, 36,6,6,6,6,2,36,6, 2,2,2,2,2,6,2,2, 2,2,6,5,2,5,2,2,
+ 10,10,2,2,2,10,10,10, 12,12,10,2,10,10,2,2, 2,2,10,19,10,19,12,2, 2,2,2,2,12,19,16,10,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 36,10,10,10,11,12,12,10, 12,12,12,39,11,39,16,37,
+ // 85xx
+ 56,11,39,37,39,39,39,39, 39,2,2,39,39,2,39,11, 39,39,39,37,37,39,39,37, 37,37,37,39,39,39,39,39,
+ 2,2,2,2,11,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,39,2,2,2,2,11,11, 2,2,2,2,2,2,19,2,
+ 2,11,11,2,11,2,2,2, 2,11,33,2,2,2,2,11, 11,11,2,11,11,2,33,11, 11,11,33,2,2,2,2,39,
+ 2,2,36,2,2,2,2,2, 2,2,36,2,2,2,2,36, 2,36,2,2,2,2,36,36, 2,2,2,5,2,2,2,36,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 39,5,2,2,19,11,5,11, 11,12,5,2,2,37,2,2, 2,2,6,11,11,19,12,39, 2,2,2,2,11,11,11,11,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 36,36,4,23,23,4,19,11, 19,11,12,37,11,33,11,37,
+ // 86xx
+ 56,39,39,11,39,39,39,37, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,37,
+ 2,6,2,2,36,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,19,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,36,2, 2,2,2,2,2,5,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 39,19,2,2,2,2,2,2, 11,2,19,2,19,37,2,2, 2,2,4,12,12,19,12,11, 2,2,2,2,11,12,4,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 36,36,39,31,39,4,39,11, 4,39,39,39,11,12,19,37,
+ // 87xx
+ 56,39,37,39,39,39,39,39, 39,2,2,39,39,2,39,37, 39,37,37,37,39,37,39,37, 37,39,39,39,37,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,6,6,6,6, 6,2,6,6,2,2,2,2, 2,11,2,2,6,6,6,6, 6,6,6,2,2,2,2,2,
+ 6,2,2,2,6,39,2,31, 2,11,11,2,2,2,2,6, 2,6,31,31,2,2,31,2, 11,2,37,2,2,2,2,37,
+ 36,6,2,2,2,36,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 2,2,2,36,2,2,5,2,
+ 2,2,36,2,2,2,2,2, 2,5,2,36,2,2,11,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 19,12,2,2,2,2,4,11, 4,12,4,2,19,39,2,2, 2,2,4,12,12,4,19,11, 2,2,2,2,11,11,31,19,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 36,4,37,11,11,11,4,11, 19,37,19,11,19,4,4,37,
+ // 88xx
+ 57,39,39,39,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,39,
+ 2,36,36,36,2,36,2,2, 2,2,2,36,36,36,2,2, 2,2,2,36,2,2,2,2, 2,2,2,2,2,2,2,36,
+ 2,2,6,6,6,2,2,36, 2,2,2,2,2,2,6,2, 2,2,2,6,2,6,2,2, 2,2,2,2,6,2,2,2,
+ 6,6,2,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6,
+ 2,6,6,2,2,2,2,2, 2,2,6,2,2,2,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,37,39,37,
+ // 89xx
+ 56,39,39,39,39,39,39,37, 39,2,2,39,39,2,39,39, 37,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,2,2,6,6,6, 6,2,6,6,6,2,6,2, 6,6,6,6,2,2,6,2, 6,6,6,2,6,2,6,6,
+ 6,6,6,6,2,6,6,6, 6,6,6,6,6,2,2,2, 6,6,6,6,6,6,6,6, 6,37,6,6,6,6,6,37,
+ 2,2,2,6,6,6,2,2, 2,6,2,2,2,2,2,6, 6,6,36,2,6,2,6,6, 6,2,6,6,6,6,6,6,
+ 6,6,6,2,6,2,2,2, 6,2,6,6,2,6,6,2, 2,2,2,6,2,2,2,6, 6,6,6,6,6,6,2,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,2, 2,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,6,2,2,2,2,6,2, 2,2,6,2,2,2,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,39,39,37,
+ // 8Axx
+ 56,39,37,39,39,39,39,39, 37,2,2,39,39,2,37,39, 39,39,39,37,39,39,39,39, 37,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,12,6,6,6,6,6,6, 2,12,6,6,12,2,12,6, 12,6,2,2,6,6,12,6, 6,6,6,6,2,2,6,6,
+ 6,12,2,2,6,6,37,6, 2,6,6,12,6,6,6,6, 6,37,6,6,6,6,6,6, 6,6,6,6,6,6,6,39,
+ 2,6,2,6,6,36,36,6, 6,2,2,6,6,12,36,6, 2,2,36,2,2,2,2,2, 2,6,6,2,2,2,2,2,
+ 2,2,2,6,2,2,6,6, 2,2,6,2,2,2,6,6, 6,2,6,6,6,2,2,6, 2,2,6,6,2,6,6,2,
+ 10,12,6,6,6,6,12,6, 6,6,6,6,6,12,6,6, 2,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,6,6,2,2,2,2,2, 2,2,2,6,2,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,39,4,37,
+ // 8Bxx
+ 56,39,37,39,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,37,39,37,39,37,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,2,6,6,6, 6,6,2,6,6,6,6,6, 6,6,6,6,6,6,6,2, 6,6,6,2,6,2,6,6,
+ 6,2,2,6,2,6,2,6, 2,6,6,6,6,2,2,31, 6,6,6,6,6,6,6,2, 6,6,6,6,6,6,6,37,
+ 2,6,2,6,6,6,6,2, 2,2,36,6,6,2,6,6, 6,2,6,6,2,2,6,2, 2,2,5,6,2,2,6,6,
+ 2,2,2,6,2,2,6,2, 2,2,2,6,2,6,2,2, 2,2,6,6,6,6,6,6, 6,2,6,6,2,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,2,2,2,2,2, 2,2,6,2,2,2,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,39,37,37,
+ // 8Cxx
+ 56,39,39,37,39,39,39,39, 37,2,2,37,39,2,39,39, 37,39,39,37,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,2,2,12,2,2,6,2, 2,6,6,6,12,2,6,2, 6,6,6,6,2,11,2,6, 6,6,6,6,6,2,6,6,
+ 6,6,6,6,2,6,6,6, 6,6,6,6,2,2,6,6, 6,6,2,2,2,6,6,12, 6,6,2,6,6,6,2,37,
+ 2,2,6,2,2,2,6,2, 6,2,6,6,6,36,6,6, 2,2,6,6,6,6,2,2, 6,6,6,6,6,6,6,6,
+ 6,2,6,6,6,2,2,6, 2,6,2,2,6,6,2,6, 6,6,2,6,6,6,2,6, 6,6,2,6,6,2,6,2,
+ 6,6,6,6,6,6,12,6, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6, 2,2,2,2,6,6,6,6,
+ 2,6,2,2,2,2,2,2, 2,2,6,2,2,2,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,4,4,37,
+ // 8Dxx
+ 57,39,39,39,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,37,37,
+ 2,2,2,37,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,12,2,6,6,6,6,6, 6,6,6,6,6,2,6,6, 6,6,6,6,6,6,2,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,2,6,6,6,6, 6,6,6,6,6,6,2,6, 2,6,6,6,2,6,6,6, 6,6,6,2,6,6,6,39,
+ 6,2,6,2,6,36,6,6, 2,2,2,6,6,36,2,6, 6,6,2,2,2,2,2,2, 2,36,2,36,6,6,6,2,
+ 6,6,6,2,2,6,2,2, 2,6,6,2,6,6,2,2, 2,2,6,2,2,2,6,2, 6,6,2,6,2,6,2,2,
+ 6,12,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,11,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,6,6,6,2, 6,6,6,6,6,6,6,6, 6,6,4,6,6,39,39,37,
+ // 8Exx
+ 56,39,39,39,39,39,39,39, 39,2,2,39,39,2,39,37, 39,39,39,37,37,37,39,39, 37,39,39,39,39,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,12,6,2,6,12,6,6, 2,12,2,12,2,6,12,6, 2,6,6,12,2,6,2,2, 6,6,6,2,6,6,6,6,
+ 6,12,6,6,6,12,6,6, 6,6,6,6,6,6,6,6, 6,6,31,6,6,6,6,6, 6,2,2,6,6,6,6,39,
+ 6,6,2,6,6,2,6,2, 2,2,6,2,6,6,6,6, 36,6,2,2,6,6,6,6, 2,6,6,6,6,6,6,6,
+ 6,6,2,7,7,2,6,2, 2,6,2,6,7,6,6,6, 2,7,2,7,7,6,7,2, 6,7,6,2,7,7,6,6,
+ 6,12,6,7,7,6,7,11, 7,6,6,6,7,12,6,6, 6,7,6,6,6,6,6,7, 6,7,7,7,7,7,7,7,
+ 2,6,2,2,2,2,2,2, 6,2,6,6,2,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,37,8,37,
+ // 8Fxx
+ 57,39,39,37,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,37,37,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,2,6,6,6, 6,6,6,6,6,6,12,6, 6,6,6,2,6,2,6,6, 6,6,6,6,6,6,2,6,
+ 6,6,6,6,6,6,6,2, 6,2,6,6,2,6,6,6, 6,6,6,2,6,6,6,2, 6,6,6,6,2,6,6,39,
+ 6,6,2,6,2,2,6,6, 6,6,6,2,2,2,36,6, 2,6,2,2,6,6,2,6, 2,2,2,2,6,6,6,6,
+ 2,2,6,2,6,2,2,6, 36,2,2,6,6,6,36,2, 2,2,2,6,2,6,2,2, 2,6,6,6,6,2,2,5,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,2,6,6,6,6,6,6,
+ 2,6,2,2,2,2,2,2, 2,2,6,6,6,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,37,8,37,
+ // 90xx
+ 57,37,37,37,39,39,37,39, 39,2,2,39,39,2,39,37, 39,39,37,39,37,39,39,39, 39,37,37,39,39,39,39,37,
+ 2,2,2,37,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,2,6,6,6,6, 6,2,6,6,6,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,2,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,37,
+ 36,6,6,2,2,6,2,6, 2,2,2,6,2,2,2,6, 2,2,36,36,6,2,2,2, 2,6,6,2,2,6,2,6,
+ 36,6,6,6,6,2,2,6, 6,6,6,6,6,6,6,6, 6,2,6,6,6,2,6,6, 6,6,6,6,6,2,2,6,
+ 6,6,6,6,6,6,6,11, 6,6,6,6,6,6,6,6, 6,6,11,6,6,6,6,2, 6,6,6,6,6,6,6,6,
+ 6,2,6,2,2,2,2,2, 2,2,6,2,2,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,37,39,37,
+ // 91xx
+ 57,39,39,39,39,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,37,39,39, 39,39,39,39,39,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,2,6,2,6,6,6,2, 6,6,6,6,2,2,6,6, 6,6,6,6,6,6,6,11, 6,6,11,6,6,6,6,6,
+ 6,6,6,2,6,6,6,6, 6,2,6,6,2,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,2,39,
+ 6,6,2,6,2,36,2,6, 6,2,2,6,36,6,36,36, 6,6,36,36,6,6,6,6, 2,2,2,6,6,6,2,6,
+ 6,6,6,6,6,6,6,6, 2,2,6,6,6,6,6,36, 6,6,6,2,6,6,6,6, 6,6,6,2,6,6,6,6,
+ 10,6,2,6,6,6,2,6, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,2,2,2,2,6,2,2, 2,2,2,2,2,2,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,22,22,37,
+ // 92xx
+ 56,39,39,37,39,39,37,39, 39,2,2,39,39,2,39,39, 37,39,37,39,37,37,39,37, 37,39,39,39,37,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,11,2,2,2, 39,2,2,2,2,2,2,2, 2,2,2,11,2,2,2,2,
+ 6,11,6,6,6,6,2,2, 11,6,6,6,2,6,6,6, 6,6,2,6,6,6,2,6, 2,6,6,6,2,6,6,6,
+ 6,11,6,6,11,11,6,6, 11,6,6,6,6,6,6,11, 6,6,6,11,11,6,6,6, 6,11,6,6,6,6,6,37,
+ 36,6,36,6,2,6,6,6, 2,6,6,6,2,6,36,2, 36,6,2,6,2,2,2,2, 6,2,6,36,36,6,2,2,
+ 6,6,2,6,6,6,6,6, 2,6,6,2,6,2,6,2, 2,2,6,2,6,6,6,6, 6,6,6,2,6,2,6,6,
+ 6,6,6,6,6,6,2,6, 6,6,6,6,6,6,6,6, 6,2,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,2,2,2,2,2, 6,2,6,6,2,6,11,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,22,10,10,
+ // 93xx
+ 56,39,39,39,37,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,37, 37,39,39,39,39,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,11,11,11,11,6,11,6, 11,6,6,6,11,11,6,6, 11,11,11,6,6,11,6,6, 6,6,6,2,6,6,6,6,
+ 6,6,2,6,6,11,6,11, 6,2,11,6,6,2,2,6, 11,6,6,6,11,2,11,6, 6,6,6,6,6,6,6,37,
+ 6,2,6,6,2,2,36,6, 2,2,6,6,6,2,2,6, 6,36,6,2,6,6,6,36, 36,6,6,6,2,6,6,6,
+ 2,2,6,2,2,2,2,6, 2,6,6,6,6,6,6,6, 6,6,2,5,6,6,6,2, 2,6,2,6,2,6,6,6,
+ 10,6,6,59,6,6,6,6, 6,12,6,6,10,6,2,6, 6,2,6,10,6,6,6,2, 6,2,6,6,6,6,6,6,
+ 6,2,2,2,2,2,2,2, 2,2,2,2,6,6,6,2, 6,6,6,6,19,6,6,6, 6,6,6,6,6,10,19,37,
+ // 94xx
+ 56,39,39,39,39,39,39,39, 39,2,2,11,39,2,39,39, 39,39,39,39,39,39,39,37, 37,39,39,39,37,39,39,39,
+ 2,2,2,11,2,2,2,2, 2,2,2,2,11,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,11,2,
+ 6,6,2,6,6,6,6,6, 2,2,6,6,6,6,6,6, 2,2,6,6,6,2,6,2, 6,6,6,6,6,6,2,6,
+ 6,2,2,6,6,6,2,6, 6,2,6,2,2,2,6,6, 6,6,6,6,6,2,6,6, 6,6,6,6,6,6,6,39,
+ 2,2,2,6,6,2,2,2, 2,2,2,2,2,6,6,6, 2,6,6,6,2,6,6,6, 2,2,6,6,2,2,6,2,
+ 6,2,2,2,2,36,6,6, 2,2,6,6,2,6,2,2, 2,2,6,2,2,2,2,2, 2,2,2,2,6,6,2,2,
+ 6,6,6,2,6,6,6,6, 6,6,6,6,6,6,2,6, 2,6,6,6,6,6,6,2, 6,2,6,6,6,6,6,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,22,37,37,
+ // 95xx
+ 56,39,39,39,39,37,39,39, 37,11,2,39,39,11,39,39, 37,39,39,39,39,39,39,39, 39,37,39,39,39,39,39,39,
+ 11,2,2,39,11,2,11,2, 2,2,2,2,2,2,11,2, 2,2,2,2,2,2,2,2, 2,2,11,2,2,2,2,2,
+ 6,2,2,6,6,6,6,6, 6,6,2,6,11,6,11,6, 6,6,6,6,11,6,11,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,37,
+ 2,6,6,6,2,2,2,2, 2,6,2,6,2,6,6,2, 6,6,6,2,6,6,2,6, 2,2,6,6,2,6,6,6,
+ 6,6,2,2,2,6,2,6, 6,2,6,2,2,2,2,2, 2,6,6,6,2,6,6,6, 2,6,6,6,2,6,6,2,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,39,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,2,6,2,2,2,2,2, 2,2,6,2,2,2,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,37,19,37,
+ // 96xx
+ 57,39,39,39,39,37,39,39, 39,2,2,39,39,11,39,39, 37,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,2,2,2,11,11,2,2, 2,2,2,2,2,2,2,2, 11,11,11,2,11,2,11,11, 11,11,2,2,2,2,11,2,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,2,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,11,6,6, 6,6,6,11,11,6,6,6, 6,6,6,6,6,6,6,39,
+ 2,6,6,6,2,6,2,2, 6,2,2,2,6,6,2,2, 2,2,2,2,6,6,6,2, 6,2,2,6,6,6,6,6,
+ 2,6,6,6,2,2,2,6, 6,2,2,2,2,2,2,6, 2,6,2,6,2,2,2,2, 2,2,6,2,6,2,6,6,
+ 6,6,6,6,6,6,6,11, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,2,6,2,2,2,2,2, 2,2,2,6,6,6,6,2, 6,6,6,6,6,6,6,6, 6,6,17,6,6,39,37,37,
+ // 97xx
+ 56,39,39,39,39,39,39,37, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,37,39,37,39,
+ 2,2,2,11,11,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,11,2,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,2,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 6,6,11,6,6,6,6,6, 6,2,6,6,6,6,6,11, 6,6,6,11,6,6,6,6, 6,6,6,6,2,6,6,39,
+ 36,6,6,6,2,2,2,2, 6,2,6,2,2,6,6,2, 2,6,6,6,2,2,6,6, 6,6,6,2,2,6,6,6,
+ 2,2,6,6,6,2,6,6, 2,2,6,6,6,2,2,6, 2,6,6,6,2,6,2,6, 2,6,2,2,6,2,6,6,
+ 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,
+ 2,6,6,2,2,2,2,2, 2,2,6,6,6,6,6,2, 6,6,6,6,6,6,6,6, 6,6,6,6,6,39,39,37,
+ // 98xx
+ 57,37,37,37,39,37,37,39, 37,2,2,39,39,2,39,39, 37,37,37,37,39,39,39,39, 37,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,6,6,2,2,6,2, 6,6,6,2,6,2,6,6, 2,6,6,2,2,6,6,2, 6,6,6,6,6,2,6,6,
+ 6,6,6,2,2,6,6,6, 6,2,6,2,6,2,2,2, 6,6,6,2,2,2,2,2, 2,2,2,2,2,2,2,39,
+ 2,2,2,2,2,2,2,2, 2,2,36,2,2,2,2,2, 2,2,36,2,2,2,2,2, 2,5,2,2,2,11,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,5,2,2,
+ 11,39,2,2,2,6,39,11, 6,11,11,19,6,39,2,11, 2,11,6,11,11,39,6,2, 2,2,2,2,11,11,5,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 6,6,37,6,6,4,39,6, 6,5,6,39,6,37,37,37,
+ // 99xx
+ 56,39,39,37,39,37,39,39, 39,2,2,39,39,2,39,39, 37,39,39,37,39,39,39,39, 37,37,39,39,39,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,6,2,2,2,11,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,6,6,2,2,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,37,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,5,2,2,2,2,2, 2,2,2,2,2,2,2,5,
+ 6,6,2,2,2,2,2,11, 6,6,6,2,6,6,2,2, 2,2,6,6,6,6,6,2, 2,2,19,2,11,6,11,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 6,36,6,6,6,6,6,6, 6,6,6,6,6,39,4,37,
+ // 9Axx
+ 56,39,39,39,39,39,39,39, 39,2,2,39,39,2,37,39, 39,39,37,37,39,39,39,39, 39,37,39,37,39,39,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,12,2,2,2,12,2,2, 2,12,2,12,12,2,12,12, 12,6,2,2,12,12,12,2, 2,2,2,2,2,2,2,37,
+ 2,2,2,5,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,6,2,2,2,2,2, 2,2,12,2,2,12,6,36,
+ 6,6,2,2,2,5,2,2, 2,2,2,2,6,6,2,5, 2,2,5,5,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 5,6,59,2,2,2,6,12, 6,2,6,2,4,6,2,2, 2,2,6,6,37,39,6,2, 37,2,6,6,6,19,6,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,5,2,12,6,2, 36,6,12,10,12,6,6,6, 6,12,6,6,12,12,39,37,
+ // 9Bxx
+ 56,39,39,39,39,39,37,39, 39,2,2,39,39,2,39,39, 39,39,39,37,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,6,2,2,2,2,2,2, 2,2,39,2,2,2,2,2, 5,2,6,2,2,2,2,2,
+ 2,2,2,2,2,2,2,6, 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,6,6,2,2,2,37,
+ 2,2,36,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,6,2,36,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,5,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,6,2,2,6,2,6,2, 4,6,6,2,2,12,2,2, 2,2,6,6,6,39,2,2, 2,2,6,6,6,6,6,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,6,2, 6,6,6,6,6,6,6,11, 4,6,6,6,11,11,22,37,
+ // 9Cxx
+ 56,39,39,11,37,37,39,39, 39,2,2,39,39,2,39,39, 37,37,39,37,39,37,39,39, 39,39,39,39,39,39,39,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,6,2,2,2,
+ 2,2,2,12,2,2,2,2, 2,2,2,2,12,2,2,2, 2,2,12,2,2,11,2,12, 2,2,2,2,2,2,2,37,
+ 2,2,2,2,2,36,2,2, 2,2,2,2,2,2,2,2, 2,36,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,4,2,2,2,2,6,2, 6,6,19,2,6,4,2,2, 2,2,2,2,6,2,2,2, 2,2,2,6,6,6,6,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 6,12,6,10,37,4,6,6, 6,6,6,6,11,4,19,37,
+ // 9Dxx
+ 56,39,12,4,37,39,37,39, 37,2,2,39,39,2,39,39, 37,39,37,39,37,37,39,37, 37,37,37,39,37,37,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,6, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 6,2,2,2,2,2,2,2,
+ 2,12,2,2,2,2,6,2, 2,2,2,2,2,2,2,12, 2,2,2,2,12,12,2,2, 2,2,2,2,6,2,2,37,
+ 2,2,2,2,2,5,2,2, 2,2,2,36,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,5, 2,2,2,36,2,2,2,2, 2,2,5,2,2,2,2,2, 2,2,6,2,2,2,2,2,
+ 5,6,2,2,2,2,11,11, 11,6,11,2,11,11,11,11, 2,11,11,11,11,11,4,6, 2,2,11,6,6,11,11,11,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,19,2, 6,19,6,6,6,6,6,19, 6,6,12,10,4,19,11,37,
+ // 9Exx
+ 56,39,39,39,37,39,39,39, 39,2,2,39,39,2,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,37,37,
+ 12,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,12,12,2,12,12,2,2, 2,12,2,12,2,2,12,12, 2,2,2,12,2,12,2,2, 2,2,2,2,2,2,2,37,
+ 2,2,2,2,2,2,6,2, 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,11,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,5,6,2,2,2,2,2, 2,6,2,2,2,6,2,2,
+ 6,12,59,2,6,2,2,2, 6,6,6,6,2,5,2,6, 2,2,6,11,6,6,6,2, 11,6,6,2,6,19,37,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,12,5,2, 12,4,12,10,6,19,39,6, 6,12,12,6,6,12,11,37,
+ // 9Fxx
+ 56,39,39,39,4,39,39,39, 39,2,2,39,39,2,37,39, 39,39,39,37,39,37,39,39, 37,39,39,39,39,39,39,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,6,6, 2,2,2,2,6,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,33,2,31,12,33,2,31, 31,2,2,33,33,2,2,2, 33,6,31,2,33,33,2,12, 2,2,2,6,2,2,2,37,
+ 2,2,36,2,2,36,6,36, 2,2,36,2,36,36,2,2, 2,2,2,2,2,6,36,2, 2,36,2,2,2,2,36,2,
+ 2,33,2,2,2,2,2,2, 6,2,36,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 6,4,2,2,6,2,6,6, 6,6,6,2,6,6,2,2, 2,2,1,6,6,6,4,2, 2,2,6,6,4,6,6,6,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 6,12,6,6,19,6,6,6, 6,6,6,6,6,19,4,37,
+ // A0xx
+ 7,11,37,11,37,1,37,1, 1,11,11,37,1,11,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,2,2,11,11,11,2,2, 11,2,2,11,2,11,2,2, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,2,
+ 2,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,2,2,2,
+ 2,11,11,11,11,11,11,11, 11,11,11,11,33,11,33,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,2,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,11,2,2,12,2,
+ 11,2,36,36,2,2,36,36, 2,36,2,36,36,2,36,2, 2,2,2,5,2,2,2,2, 2,2,2,2,2,2,2,12,
+ 10,10,2,2,10,11,10,11, 11,1,10,14,10,10,10,10, 2,11,11,11,11,11,1,2, 2,2,11,11,1,10,11,11,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,36,2, 10,10,10,10,10,10,10,10, 12,17,1,10,11,33,1,10,
+ // A1xx
+ 7,37,37,1,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,3,37,37,37,37,
+ 11,2,2,2,2,2,11,2, 2,2,2,2,2,2,14,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,14,2,
+ 8,8,8,8,9,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,9,9,8,8,8, 8,1,9,2,8,8,8,2,
+ 8,8,33,2,33,8,8,1, 33,8,8,33,33,8,8,2, 2,8,8,33,2,8,8,8, 8,8,8,39,2,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,39,2,2,2,2, 2,2,2,2,2,2,6,2, 2,2,2,2,2,2,2,33,
+ 2,3,3,3,3,7,7,7, 2,7,7,3,2,3,3,3, 3,3,7,8,8,2,3,3, 8,7,3,3,7,7,3,3,
+ 3,7,7,7,7,14,9,14, 7,7,7,7,7,14,7,7, 8,14,14,14,14,14,7,7, 7,7,7,7,7,7,5,7,
+ 2,5,5,2,2,2,2,2, 2,2,2,7,2,2,3,3, 3,3,3,7,3,7,7,7, 3,7,7,7,7,7,7,37,
+ // A2xx
+ 7,37,37,1,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,2,2,8,2,2,8, 8,8,8,2,2,2,2,2, 2,2,2,2,2,2,2,2, 8,2,2,2,2,2,2,2,
+ 2,2,8,8,33,8,8,2, 8,8,2,8,2,2,2,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,6,2,2,2,2,2, 2,6,2,2,2,2,2,6, 6,2,2,2,6,6,2,2, 2,2,2,2,2,2,2,2,
+ 36,7,7,7,7,7,7,7, 7,7,7,2,7,7,36,8, 2,8,8,8,8,8,8,14, 5,5,5,14,2,5,5,5,
+ 5,5,5,5,5,14,5,14, 5,5,14,23,5,14,5,5, 8,5,14,8,8,14,14,8, 14,3,3,3,3,5,8,5,
+ 2,2,2,2,2,2,2,2, 2,2,8,36,8,8,8,2, 8,3,3,3,8,8,7,8, 8,3,8,8,8,8,8,37,
+ // A3xx
+ 7,37,37,39,37,37,37,37, 37,2,2,37,4,2,9,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,11,11,2,2, 2,2,2,2,2,2,2,2, 11,11,11,11,11,11,11,11, 11,11,2,2,2,2,2,2,
+ 36,12,12,12,12,12,2,12, 2,2,2,12,2,12,12,12, 12,2,2,12,12,12,11,12, 8,12,12,2,8,2,11,2,
+ 8,12,2,2,33,12,2,8, 11,2,8,12,2,2,2,2, 2,11,2,8,2,12,8,2, 8,2,8,8,8,8,8,37,
+ 2,2,2,6,2,2,2,2, 3,2,2,2,2,2,2,2, 2,2,2,2,3,2,5,2, 2,2,2,2,2,2,2,2,
+ 3,3,3,3,3,3,3,3, 3,3,3,2,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,2,2,2,3,
+ 10,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,5,3,3,3,
+ 2,2,2,2,2,2,2,2, 2,2,2,3,3,3,3,2, 3,3,3,12,3,3,3,3, 3,3,3,3,3,3,3,37,
+ // A4xx
+ 7,37,37,11,1,37,11,39, 11,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,7,37,37,37,37,37,
+ 2,2,2,11,2,11,2,2, 2,2,11,2,2,2,2,2, 37,2,11,2,2,2,2,17, 2,17,2,2,2,2,2,2,
+ 8,8,8,8,8,8,8,8, 8,2,8,8,8,8,2,8, 2,8,8,8,8,8,8,8, 8,8,8,8,8,8,2,8,
+ 8,8,8,11,8,11,8,8, 8,8,8,8,8,2,2,8, 8,8,8,8,8,8,8,8, 8,8,8,2,2,8,8,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,6,2,2,2,2, 2,2,2,6,6,2,2,2, 2,2,2,2,2,2,2,2,
+ 36,3,3,8,3,36,3,2, 3,2,3,3,3,3,3,3, 3,3,3,3,3,3,8,3, 3,3,8,3,3,3,2,3,
+ 3,3,14,3,3,14,3,3, 3,3,3,3,3,3,3,3, 3,8,3,3,14,3,8,3, 3,3,8,3,3,3,3,3,
+ 3,3,3,2,3,8,2,2, 3,3,3,3,3,3,8,3, 8,8,3,3,8,8,8,8, 11,8,8,8,8,5,8,1,
+ // A5xx
+ 7,6,37,39,37,37,6,37, 1,2,2,37,1,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,11, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,9,8,12,8,2,8,8, 8,8,8,8,8,2,8,8, 8,2,8,8,12,8,8,2, 8,8,8,8,8,8,8,8,
+ 8,8,8,2,8,8,8,2, 8,8,8,8,8,2,2,2, 2,8,2,8,2,8,8,2, 8,2,8,8,8,8,8,37,
+ 2,2,2,6,2,2,2,2, 2,2,2,2,2,2,6,2, 2,2,6,6,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,3,3,3,6,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,8,2,2,3,3,
+ 3,3,2,3,3,8,3,3, 3,3,3,3,3,8,8,3, 3,3,3,3,3,3,3,3, 8,3,3,3,3,3,3,3,
+ 3,3,3,2,2,2,2,2, 2,3,3,3,3,3,8,3, 8,8,8,3,8,8,3,8, 8,8,8,8,8,8,8,37,
+ // A6xx
+ 7,37,37,39,37,37,37,1, 12,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,6,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,8,9,8,33,2,2, 8,33,8,9,9,9,9,8, 8,8,9,8,8,8,8,8, 8,8,8,2,2,8,8,8,
+ 8,8,8,9,2,2,2,2, 8,8,8,8,9,9,8,8, 8,2,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 36,8,5,8,2,36,5,2, 8,8,2,8,2,8,2,2, 2,8,2,8,2,8,8,8, 8,8,8,2,2,2,2,8,
+ 8,3,3,8,8,3,9,8, 8,14,8,8,8,3,2,8, 8,8,8,8,3,8,33,8, 3,8,8,8,8,8,8,8,
+ 2,2,8,2,2,2,8,2, 2,2,8,8,8,8,8,2, 3,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // A7xx
+ 7,1,1,37,37,37,37,37, 37,14,2,14,37,14,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,11,2,2,14, 2,2,11,2,2,2,2,2, 11,11,11,11,11,11,11,11, 11,11,2,2,2,2,11,2,
+ 8,8,8,8,8,8,2,8, 8,8,8,8,8,8,8,8, 8,8,8,2,8,8,8,2, 8,8,8,11,2,11,8,8,
+ 8,2,8,8,8,8,8,8, 8,8,8,8,8,2,2,8, 8,8,8,8,8,2,8,8, 8,2,2,8,8,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,5, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 36,14,14,2,14,3,2,11, 8,8,14,2,2,36,8,2, 2,2,8,5,14,14,14,14, 2,8,8,2,14,8,14,2,
+ 8,14,14,14,14,14,8,8, 14,8,14,14,8,14,8,8, 8,8,14,8,14,3,8,14, 2,2,8,2,8,8,8,8,
+ 2,14,8,2,2,2,2,2, 2,2,2,8,8,8,3,2, 8,3,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // A8xx
+ 7,11,1,39,1,37,37,37, 4,2,2,14,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,11,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,2,8,8,8,2,8,2, 8,8,2,8,8,8,8,3, 2,2,8,8,8,8,2,2, 2,2,8,2,2,2,8,2,
+ 11,8,8,8,8,8,8,8, 2,2,8,8,8,8,8,8, 8,8,3,8,8,8,8,3, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,5,2,2,2,2,2,3, 3,3,3,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,7,8,8,8,7,7, 14,8,2,8,7,8,8,8, 11,2,7,2,2,8,2,14, 2,2,8,2,2,8,8,2,
+ 8,14,2,8,8,8,8,8, 8,8,8,14,8,14,8,8, 14,8,14,8,14,5,8,8, 14,2,8,5,5,6,5,8,
+ 2,2,8,2,2,2,2,2, 2,2,2,2,8,2,8,2, 5,5,8,8,8,8,5,8, 8,5,8,8,8,8,8,37,
+ // A9xx
+ 7,1,37,39,37,37,39,37, 37,2,2,37,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,2,2,2,11,2,11,2, 2,2,2,11,2,2,2,2, 37,11,11,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,9,8,8,9,9,11,8, 8,8,8,9,9,8,9,8, 8,8,8,8,8,8,9,8, 2,8,8,8,2,8,8,8,
+ 8,2,8,2,8,8,8,8, 8,8,8,9,8,8,2,3, 8,2,2,8,2,8,8,8, 8,8,8,8,8,8,8,39,
+ 2,3,6,3,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,6,2,2,3,2, 2,2,2,2,2,2,2,2,
+ 2,8,2,8,3,3,3,3, 8,2,3,8,2,3,3,8, 2,8,8,8,2,8,2,2, 3,8,2,2,8,3,8,2,
+ 3,3,2,2,3,14,8,8, 8,8,8,2,3,9,8,3, 8,14,8,8,8,14,8,2, 2,2,8,8,8,8,8,8,
+ 2,2,8,2,2,2,2,2, 2,2,2,2,2,2,8,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // AAxx
+ 7,37,37,37,37,1,37,37, 4,2,2,1,37,2,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,8,8,8,8,8,8, 2,12,2,8,8,8,2,8, 8,8,8,2,12,2,2,2, 2,8,8,8,2,2,8,8,
+ 8,8,2,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,2,2,8,8,8, 8,8,8,8,2,2,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,5,2,2,2,2,2,2,
+ 36,14,8,8,2,3,8,8, 2,8,2,8,8,2,36,8, 2,8,2,5,8,5,2,5, 14,14,8,2,8,8,8,2,
+ 8,14,14,14,10,8,5,14, 8,1,5,5,8,14,8,8, 14,8,14,8,14,14,8,14, 14,2,8,2,8,8,8,5,
+ 2,8,2,2,2,2,2,2, 2,2,8,5,8,8,8,2, 8,8,8,5,8,10,8,8, 8,8,8,8,8,8,8,37,
+ // ABxx
+ 7,1,1,37,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,6,37,37,37, 37,37,37,37,37,37,37,37,
+ 2,2,2,2,2,2,11,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,11,8,8,8,11,8,8, 8,8,8,8,8,11,8,8, 8,8,2,11,8,8,11,8, 8,8,2,2,8,2,8,8,
+ 8,8,8,11,8,8,11,8, 8,8,8,8,8,2,8,8, 8,8,8,8,8,2,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,5,2, 2,2,2,2,2,2,39,2, 5,2,2,2,2,11,2,2, 2,2,5,2,2,5,2,2,
+ 2,8,8,8,5,2,2,8, 36,2,2,8,8,2,36,2, 8,8,3,8,8,8,2,5, 2,2,8,2,8,8,2,2,
+ 8,8,8,8,8,8,5,8, 8,8,8,8,8,14,8,11, 14,8,8,8,14,14,14,8, 8,14,11,8,8,8,11,8,
+ 2,8,2,2,2,2,2,2, 2,2,2,2,36,8,36,2, 20,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // ACxx
+ 7,37,37,39,37,39,39,37, 37,2,2,37,39,2,37,39, 37,37,37,37,37,39,37,37, 39,37,37,39,37,37,37,37,
+ 2,2,2,11,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,8,9,8,33,8,8, 2,8,8,8,9,8,9,8, 8,8,9,2,2,8,8,9, 8,8,8,8,8,2,2,8,
+ 8,8,2,8,8,33,8,11, 8,2,2,2,2,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,2,8,8,8,37,
+ 2,6,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,6,5,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 36,8,2,8,2,36,2,2, 36,6,8,36,2,36,36,8, 8,2,2,8,2,8,8,3, 2,8,2,2,2,8,2,3,
+ 8,8,59,8,8,59,8,3, 8,8,3,59,8,8,3,2, 8,5,8,8,8,8,8,8, 2,8,5,8,8,8,8,8,
+ 2,2,59,2,2,2,2,2, 2,2,2,2,2,2,8,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // ADxx
+ 7,39,37,37,39,39,37,39, 39,2,2,37,39,2,37,39, 39,37,39,39,39,39,39,37, 39,39,37,39,39,39,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,2,8,8,2,8,8, 2,8,8,2,8,8,2,2, 8,2,2,8,8,8,8,8, 8,8,8,8,2,8,8,2,
+ 8,2,8,2,11,8,8,11, 8,2,11,11,8,8,8,11, 8,8,11,8,2,8,11,8, 8,2,8,8,2,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,39,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2,
+ 36,2,2,2,2,2,2,8, 36,2,14,8,2,8,36,2, 2,8,8,2,2,8,8,8, 2,8,8,8,8,2,2,8,
+ 8,14,8,14,8,8,8,8, 8,8,8,8,8,8,2,2, 7,8,14,8,8,14,7,8, 2,8,8,8,8,8,8,8,
+ 2,8,2,2,2,2,2,2, 2,2,2,2,8,2,36,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // AExx
+ 7,39,1,39,39,39,37,37, 37,2,2,37,39,2,37,39, 39,39,39,39,39,39,37,37, 39,39,37,39,39,39,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,5,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,9,8,2,2,9,2,2, 8,9,8,9,8,8,9,9, 2,8,2,8,8,2,8,2, 2,2,8,2,2,8,2,8,
+ 8,8,8,8,8,8,2,8, 8,8,2,8,8,8,2,8, 8,8,8,2,8,8,8,8, 8,8,8,8,8,8,11,37,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,36,8,8, 2,2,2,2,36,2,2,2, 2,2,2,2,2,2,2,8, 2,2,2,8,8,2,2,2,
+ 8,8,8,14,8,6,8,8, 8,8,8,14,8,9,2,2, 8,8,8,8,8,8,8,8, 8,2,8,8,8,6,8,8,
+ 2,36,36,2,2,2,2,2, 2,2,2,2,2,8,8,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // AFxx
+ 7,9,1,1,39,39,39,37, 37,2,2,37,39,2,37,37, 37,37,37,37,37,39,37,37, 39,39,37,37,39,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,11,2,2,2,2,2,2, 39,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,12,8,2,12,12,8,2, 2,8,2,12,12,8,12,12, 8,8,8,8,8,2,2,2, 2,12,8,8,2,8,2,2,
+ 11,2,8,2,8,8,8,8, 2,2,2,8,8,2,2,12, 2,8,2,2,2,8,8,8, 8,12,8,8,2,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 36,9,2,6,2,2,2,2, 2,2,8,8,8,2,36,2, 11,2,8,8,2,8,2,2, 8,2,8,2,2,8,2,2,
+ 8,8,8,8,6,8,8,8, 8,8,8,6,6,10,8,8, 8,8,8,8,14,10,1,8, 2,2,2,2,8,8,8,8,
+ 8,2,2,2,2,2,2,2, 2,2,2,8,2,2,8,2, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ // B0xx
+ 7,39,37,37,39,4,37,37, 39,11,2,37,37,2,37,37, 39,39,37,37,39,39,37,37, 39,39,37,39,39,39,37,37,
+ 2,2,2,2,11,2,2,2, 2,2,11,2,2,2,2,2, 11,11,11,11,11,11,11,11, 11,11,2,2,2,2,2,2,
+ 8,8,2,11,8,11,11,2, 2,8,8,8,2,2,2,8, 2,8,2,2,8,8,8,2, 2,2,8,2,8,8,8,8,
+ 11,8,2,2,8,8,8,8, 8,2,8,8,8,11,2,11, 8,8,2,8,8,8,8,8, 8,8,2,8,8,8,11,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 11,5,5,5,2,5,7,2, 5,5,8,8,3,5,3,2, 5,2,3,5,3,8,5,8, 3,5,3,8,8,2,2,2,
+ 6,3,7,2,3,5,7,5, 8,3,8,5,7,5,3,8, 2,3,14,8,5,7,3,3, 3,3,5,7,5,5,7,5,
+ 2,5,8,2,2,2,5,2, 5,2,8,2,3,5,5,3, 5,5,8,8,3,8,8,5, 5,3,5,7,5,5,3,37,
+ // B1xx
+ 7,11,11,39,39,37,39,37, 37,2,2,37,37,9,39,39, 37,39,37,37,39,39,37,37, 37,39,37,37,39,39,37,37,
+ 9,9,2,2,37,2,2,2, 2,2,2,5,9,2,9,2, 2,2,2,2,2,2,2,2, 2,2,9,9,2,2,2,9,
+ 8,2,8,8,2,8,8,8, 8,8,8,8,8,8,8,8, 2,2,11,2,8,8,8,8, 8,8,8,8,2,2,8,8,
+ 8,8,8,9,9,8,8,9, 8,8,8,9,8,8,2,8, 9,8,8,9,9,2,8,8, 8,8,8,8,2,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,5,2, 2,2,2,2,2,2,2,2,
+ 2,8,7,3,5,3,3,3, 3,3,2,7,3,3,3,3, 2,3,3,5,3,8,3,8, 5,5,5,3,5,2,3,7,
+ 8,5,8,5,7,5,8,5, 3,7,3,3,3,5,3,3, 8,7,3,8,5,3,5,5, 3,5,8,5,3,5,5,7,
+ 3,8,5,2,3,2,2,2, 5,2,3,2,2,3,5,2, 3,8,5,7,5,3,3,7, 8,3,5,7,7,8,7,37,
+ // B2xx
+ 7,39,37,39,37,37,39,1, 37,2,2,39,37,2,39,39, 39,39,37,37,39,39,37,39, 39,39,37,37,39,39,37,39,
+ 2,2,2,2,2,39,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,2,8,2,8,8,2,8, 8,2,8,8,8,8,8,8, 8,8,8,2,2,8,8,8, 8,8,8,8,8,8,8,8,
+ 8,8,8,2,8,2,2,8, 2,8,8,8,8,8,8,8, 8,8,8,2,3,2,8,8, 8,8,8,8,2,39,8,37,
+ 2,2,2,2,2,2,5,5, 2,2,5,2,2,2,5,5, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,3,8,8,3,7,8, 3,3,3,7,7,7,3,7, 7,8,5,8,2,3,3,2, 2,3,3,3,3,3,2,3,
+ 5,7,3,7,8,3,3,7, 7,3,3,3,7,3,3,7, 7,10,3,8,7,8,7,8, 3,3,7,7,3,3,5,3,
+ 2,3,3,2,2,2,2,2, 7,3,7,5,3,10,3,2, 7,7,7,7,5,7,5,7, 8,7,3,5,8,3,7,37,
+ // B3xx
+ 7,39,1,39,39,37,37,37, 39,12,2,37,39,2,37,39, 39,39,37,37,39,39,39,37, 39,39,37,39,39,39,37,37,
+ 12,12,2,2,11,39,2,2, 2,2,2,2,12,2,12,12, 37,2,2,2,2,2,2,2, 2,2,12,12,2,2,2,12,
+ 8,2,8,8,8,2,2,2, 2,2,8,2,2,2,8,12, 2,8,2,8,8,8,8,8, 8,8,8,2,8,8,8,2,
+ 2,12,12,12,8,12,8,8, 8,2,8,12,2,12,8,8, 8,8,8,8,8,12,8,8, 8,12,8,8,2,8,8,37,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,6, 2,2,2,2,2,2,2,2, 5,2,2,2,2,2,2,2,
+ 2,3,2,3,3,3,7,3, 2,3,5,7,3,5,8,5, 7,9,5,5,2,3,2,3, 2,8,8,5,2,8,3,3,
+ 7,3,3,5,5,3,3,3, 7,3,5,3,3,7,7,3, 3,5,8,7,3,8,3,5, 7,3,7,7,8,5,3,8,
+ 5,8,5,2,2,2,2,2, 2,2,3,5,5,5,5,2, 5,10,8,5,7,8,3,3, 8,8,5,3,8,3,3,10,
+ // B4xx
+ 7,39,1,39,39,37,37,4, 39,14,2,14,39,2,37,39, 37,39,37,37,39,39,37,37, 37,39,37,39,37,39,39,37,
+ 2,2,2,11,11,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,11,8,8,8,11,2,2, 8,8,8,8,8,8,8,8, 8,8,11,11,8,8,8,2, 8,8,8,8,2,2,8,8,
+ 8,11,8,8,8,11,8,8, 11,11,8,8,2,11,11,8, 11,8,8,11,11,11,11,8, 8,8,8,2,2,8,2,37,
+ 2,2,2,6,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,2,8,8,3,3,14, 3,5,2,3,3,2,3,2, 2,7,3,7,3,8,7,14, 2,2,5,2,8,3,2,3,
+ 5,8,5,5,7,3,7,3, 3,5,3,3,3,14,3,5, 5,7,14,3,5,14,7,5, 7,5,8,7,5,5,5,3,
+ 2,14,2,2,5,2,2,5, 2,2,2,5,2,2,3,3, 3,5,3,3,7,5,5,3, 3,7,3,3,7,3,8,37,
+ // B5xx
+ 7,39,37,39,39,37,37,37, 39,2,2,37,39,2,37,37, 37,39,39,37,37,37,37,37, 37,39,37,37,37,37,37,37,
+ 2,2,2,2,39,2,2,2, 2,2,2,2,2,2,14,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,2,33,33,33,2,11,8, 8,8,8,33,8,8,33,8, 8,39,33,8,33,8,33,8, 8,8,33,8,8,2,8,2,
+ 8,9,8,8,8,8,8,11, 8,8,8,8,8,11,8,8, 8,8,11,8,3,8,8,8, 8,8,8,8,8,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,5, 2,2,2,2,2,2,2,5,
+ 2,7,7,3,7,3,8,8, 5,3,8,3,7,7,3,3, 2,3,8,3,8,5,5,2, 5,8,3,7,3,3,3,5,
+ 3,3,3,3,3,5,3,5, 3,5,5,5,3,3,5,3, 3,14,7,14,14,7,3,3, 3,7,3,8,7,3,7,7,
+ 2,7,5,3,3,5,2,3, 2,5,3,2,2,2,5,2, 5,5,5,8,3,3,7,3, 8,8,3,8,8,8,7,37,
+ // B6xx
+ 7,1,4,39,37,37,37,37, 37,2,2,37,37,2,37,37, 37,37,37,37,39,37,37,37, 37,39,37,37,37,37,37,37,
+ 9,2,2,2,11,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,9,2,2,2,9,
+ 8,2,8,2,8,8,8,2, 8,2,8,8,8,2,2,8, 8,8,8,8,8,8,8,8, 8,3,8,2,2,8,8,2,
+ 8,30,8,9,8,8,2,8, 8,8,2,9,9,8,9,8, 9,8,9,8,8,8,8,9, 8,2,8,8,8,8,8,37,
+ 2,2,2,5,2,2,5,5, 2,2,2,2,2,5,2,2, 2,11,5,2,2,2,5,5, 2,2,2,2,2,2,2,2,
+ 2,8,8,2,8,3,7,5, 3,3,3,3,3,3,8,3, 8,5,2,2,3,7,7,3, 7,3,8,5,3,3,3,7,
+ 3,3,8,8,3,3,3,5, 3,7,7,3,3,3,3,3, 7,7,14,3,3,8,3,8, 7,3,7,7,8,7,6,5,
+ 3,7,7,2,2,2,2,2, 2,2,8,8,7,3,3,2, 8,7,5,5,5,5,5,5, 3,3,3,3,3,8,3,37,
+ // B7xx
+ 7,39,37,14,39,37,39,37, 39,11,11,37,39,2,37,39, 39,39,37,39,39,39,39,37, 39,39,37,39,39,39,37,37,
+ 11,2,2,11,11,2,11,2, 2,2,11,2,2,2,11,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,11,2,
+ 8,8,8,33,33,11,8,8, 8,2,8,33,8,8,8,8, 8,8,8,8,33,8,8,8, 8,8,33,2,8,2,2,2,
+ 11,11,8,2,8,11,8,11, 8,8,8,8,11,8,8,8, 2,8,8,8,2,8,8,8, 8,8,8,2,8,8,8,37,
+ 2,6,6,2,2,2,2,2, 2,6,2,2,2,2,2,2, 2,2,2,2,2,11,2,2, 2,2,2,5,2,2,2,2,
+ 2,5,3,5,7,2,5,8, 3,5,7,5,3,3,5,5, 5,5,2,5,3,3,3,11, 2,5,3,5,2,3,3,3,
+ 3,5,5,5,5,3,3,3, 7,3,7,8,3,14,5,5, 5,3,14,8,5,14,3,7, 14,14,7,3,3,3,7,3,
+ 2,5,3,2,2,2,2,3, 2,2,7,7,5,8,7,7, 3,3,3,7,3,7,3,7, 3,5,7,3,5,5,3,37,
+ // B8xx
+ 7,39,37,39,39,39,37,37, 37,2,2,37,37,2,37,1, 39,39,37,39,39,39,39,37, 39,37,37,37,39,39,39,37,
+ 2,2,2,2,37,2,2,2, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,8,2,8,2,6,2,8, 8,8,8,8,8,6,8,8, 2,2,8,8,8,8,8,8, 8,8,8,2,37,2,2,8,
+ 8,8,2,11,8,8,8,8, 8,8,8,8,2,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,7,2,5,2,5,5,5, 8,7,2,7,2,2,5,2, 2,2,5,5,3,5,5,5, 5,8,2,5,5,3,3,5,
+ 7,5,5,3,3,5,5,5, 8,3,3,8,8,7,3,3, 3,8,3,5,5,5,3,3, 14,3,3,3,8,7,5,3,
+ 2,7,3,2,2,2,3,5, 3,5,8,2,2,5,7,3, 5,5,7,5,8,8,3,3, 3,3,3,8,3,7,7,37,
+ // B9xx
+ 7,39,4,37,39,39,39,37, 39,14,2,14,39,14,4,37, 39,39,39,39,39,39,37,37, 39,39,37,39,39,39,39,37,
+ 14,2,2,2,2,2,2,14, 2,2,2,2,2,2,14,2, 37,2,2,2,2,2,2,2, 2,2,2,2,2,2,14,2,
+ 8,8,8,8,8,8,8,8, 8,2,8,8,8,8,2,8, 2,8,2,8,8,2,8,2, 8,8,2,2,8,8,2,8,
+ 8,8,12,12,12,9,2,12, 8,9,8,9,9,8,9,9, 8,8,38,8,9,8,8,8, 8,8,12,8,8,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,14,8,2,3,7,3,14, 14,3,8,3,3,7,5,7, 5,3,3,2,2,14,2,14, 2,3,3,3,2,7,2,2,
+ 3,7,5,14,3,3,7,3, 3,3,3,3,5,7,5,8, 5,14,14,14,7,14,7,5, 3,5,5,3,3,5,5,5,
+ 7,7,3,3,2,2,5,7, 5,5,8,2,2,3,8,8, 7,7,3,3,8,7,5,5, 5,3,3,3,5,3,7,37,
+ // BAxx
+ 7,39,37,39,39,39,37,37, 39,2,2,37,39,2,37,39, 39,39,37,39,39,39,39,37, 39,39,37,39,39,39,37,37,
+ 1,2,2,2,2,1,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,14,2,2,2,2,2,
+ 8,8,2,8,8,2,2,8, 2,8,8,8,8,8,8,8, 8,2,8,2,2,8,8,1, 8,2,8,8,2,8,8,1,
+ 2,8,8,8,2,12,2,2, 8,12,8,8,2,2,8,12, 8,8,2,2,8,8,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,8,3,3,2,5,2,2, 14,8,2,2,2,2,5,5, 5,3,2,2,5,3,2,2, 5,5,2,5,3,3,5,2,
+ 5,3,7,3,3,3,7,7, 3,3,8,3,7,3,5,3, 5,7,5,3,14,3,8,7, 8,7,3,3,3,7,8,7,
+ 7,3,8,2,2,2,2,2, 2,2,5,8,3,5,7,2, 3,5,3,3,8,3,7,7, 3,5,3,8,5,3,3,37,
+ // BBxx
+ 7,39,1,39,39,37,14,39, 39,2,2,1,39,11,37,11, 39,39,37,37,39,39,39,37, 39,39,37,37,39,39,37,37,
+ 11,2,2,2,2,11,11,11, 2,2,2,2,11,2,11,2, 37,2,2,11,11,2,11,11, 11,11,2,11,11,2,14,2,
+ 8,8,8,2,8,8,8,8, 2,8,2,11,8,2,2,8, 8,8,8,2,8,2,11,8, 8,8,8,8,8,8,2,8,
+ 8,8,2,2,11,8,11,2, 8,2,11,8,2,2,8,2, 8,8,8,8,2,8,37,8, 8,8,8,8,8,8,8,37,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,8,3,2,3,2,3,3, 3,5,3,8,2,3,3,3, 3,7,7,7,2,3,2,3, 2,3,2,3,3,2,2,2,
+ 7,14,7,14,7,7,3,5, 7,7,3,7,5,7,7,7, 7,14,7,3,7,3,3,7, 3,7,8,3,7,3,7,7,
+ 7,3,8,2,2,2,2,5, 2,2,5,7,5,2,3,7, 3,3,3,5,8,5,5,3, 3,3,3,7,3,5,7,37,
+ // BCxx
+ 7,39,39,39,39,39,37,37, 39,2,2,37,39,2,37,37, 39,39,39,39,37,37,37,37, 37,39,37,37,37,37,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,12,8,8,2,2,8,12, 8,2,8,12,8,12,12,12, 8,8,2,12,2,12,12,8, 8,8,8,8,2,2,8,2,
+ 2,8,8,9,9,8,8,8, 8,8,2,9,9,9,9,8, 8,8,9,8,8,8,8,8, 8,8,8,8,2,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,7,2,7,3,5,3,2, 7,2,3,7,2,5,5,3, 3,5,3,2,3,2,3,2, 3,2,5,2,5,8,5,5,
+ 3,7,7,3,3,5,3,3, 3,8,3,8,7,3,3,8, 8,3,3,3,7,5,8,3, 3,3,8,3,3,3,3,8,
+ 3,3,5,2,3,2,2,2, 7,2,7,2,3,2,5,7, 7,7,3,7,3,3,5,5, 5,3,5,3,3,7,3,37,
+ // BDxx
+ 7,39,37,11,37,37,37,37, 39,2,2,37,39,2,6,37, 37,39,37,37,39,39,37,37, 39,39,37,39,39,39,37,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 8,2,2,2,2,6,8,8, 8,2,2,8,8,8,2,2, 2,8,8,8,8,8,8,8, 8,8,8,2,8,8,2,8,
+ 8,8,8,8,8,2,8,8, 2,2,2,8,8,8,8,8, 2,8,2,8,8,8,8,8, 2,3,2,8,8,8,8,39,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,2,3,7,2,2,8, 3,2,7,3,2,3,2,2, 2,2,3,2,5,3,3,3, 7,3,5,3,5,2,7,2,
+ 5,5,5,5,5,5,8,5, 5,5,5,7,3,5,5,3, 7,7,8,3,3,8,3,8, 3,3,3,3,3,3,8,8,
+ 3,3,3,2,2,2,2,3, 2,3,7,2,3,2,3,2, 3,7,3,3,3,3,3,7, 3,7,3,3,3,7,3,37,
+ // BExx
+ 7,39,37,14,39,1,37,37, 39,2,2,37,37,2,37,39, 37,39,39,37,37,39,37,37, 39,39,37,39,39,37,37,39,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,14,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,14,2,
+ 8,8,8,2,8,8,8,8, 8,2,8,8,8,2,2,2, 8,8,8,59,8,8,2,8, 8,8,8,2,8,2,8,8,
+ 8,12,9,8,8,9,2,12, 8,8,8,12,8,12,8,12, 8,8,8,12,12,12,8,8, 8,8,8,8,8,2,8,37,
+ 2,2,6,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,3,3,2,8,7,3, 8,3,3,3,3,3,7,3, 3,3,5,3,5,3,3,2, 5,8,3,3,2,2,5,3,
+ 3,14,8,3,14,3,5,8, 5,8,5,5,8,3,7,5, 7,14,5,3,8,5,3,5, 5,3,7,3,7,3,3,5,
+ 5,8,2,2,2,2,2,5, 2,2,2,7,7,3,5,5, 7,3,7,5,8,3,3,5, 5,3,5,5,3,3,7,37,
+ // BFxx
+ 7,11,37,37,37,12,37,1, 37,2,2,37,39,2,37,37, 37,37,37,37,39,37,37,37, 37,37,37,37,39,37,37,37,
+ 12,11,2,2,2,2,2,11, 2,2,2,2,2,2,2,2, 37,2,2,2,2,2,2,2, 2,2,12,2,2,2,2,1,
+ 8,8,1,1,8,8,1,2, 1,1,2,8,1,1,8,8, 1,1,1,1,1,8,1,8, 8,1,8,2,8,2,8,2,
+ 8,12,12,8,12,12,8,2, 1,8,8,12,12,2,2,12, 2,8,1,12,2,2,8,12, 2,12,8,2,8,8,2,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,6,2,2,2,2, 2,2,2,5,2,2,2,2, 2,2,5,2,2,2,2,2,
+ 2,5,5,5,8,5,7,7, 3,5,3,2,5,5,7,3, 5,5,7,8,3,5,7,7, 3,5,2,2,3,2,5,7,
+ 5,5,5,5,8,3,3,7, 7,3,7,3,3,7,3,5, 5,14,3,3,5,3,3,3, 3,3,3,3,5,3,7,8,
+ 2,3,3,2,5,2,2,2, 2,8,3,5,5,3,5,5, 8,3,5,3,7,5,3,3, 5,5,8,8,5,8,5,37,
+ // C0xx
+ 7,39,37,1,39,39,37,1, 39,10,18,1,39,10,37,39, 39,39,37,37,39,39,39,37, 39,39,7,37,39,39,37,39,
+ 10,10,10,10,1,10,10,10, 10,10,10,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10,
+ 8,8,1,1,1,8,1,1, 8,8,8,30,1,1,1,1, 1,1,8,1,1,3,8,8, 1,8,8,8,8,8,3,3,
+ 8,3,8,1,8,8,8,8, 39,3,8,3,3,8,8,39, 8,8,1,8,8,8,8,8, 8,8,8,8,8,37,8,37,
+ 10,6,6,6,6,39,3,3, 39,6,12,6,10,6,6,6, 6,6,6,6,6,6,6,6, 6,3,37,3,3,5,37,1,
+ 10,3,3,8,7,5,8,5, 3,3,7,3,3,3,7,5, 5,5,5,8,3,7,3,14, 5,7,5,5,3,5,7,3,
+ 3,10,10,10,10,10,10,5, 3,8,10,10,5,10,5,5, 10,10,10,5,5,10,5,3, 10,8,5,5,7,10,10,7,
+ 3,5,10,7,3,5,3,5, 7,7,10,3,7,3,3,3, 10,3,8,10,8,7,3,3, 3,8,5,5,5,3,7,10,
+ // C1xx
+ 7,39,1,39,39,39,37,37, 39,14,18,37,39,11,37,39, 39,39,37,39,39,39,37,39, 39,39,39,39,37,37,37,37,
+ 11,18,14,14,37,11,18,14, 14,11,11,14,18,18,14,11, 37,20,11,14,14,14,14,11, 37,14,11,18,11,10,18,18,
+ 8,8,1,1,1,1,1,1, 1,1,1,1,1,1,1,8, 1,1,1,1,1,1,1,8, 1,8,1,14,8,11,8,8,
+ 8,8,1,8,16,16,1,1, 1,10,8,1,1,16,8,8, 8,8,1,16,1,8,16,8, 8,8,1,8,8,8,8,37,
+ 39,6,6,4,39,11,3,6, 3,39,12,39,37,12,12,6, 37,6,6,6,39,3,6,6, 37,3,3,3,4,11,12,1,
+ 14,5,3,3,5,3,5,3, 5,5,3,3,3,3,5,5, 7,7,10,7,3,3,5,5, 5,5,3,5,3,3,5,3,
+ 10,5,5,14,3,10,3,7, 10,11,8,3,18,20,10,3, 3,14,14,10,18,14,5,14, 5,3,3,10,3,5,3,5,
+ 10,7,3,3,8,3,3,8, 14,3,7,10,3,3,5,5, 5,5,5,10,3,5,5,3, 5,3,5,10,7,5,5,10,
+ // C2xx
+ 7,39,37,37,39,39,39,37, 39,14,14,1,39,14,37,1, 39,37,39,39,39,39,39,37, 37,37,37,39,37,39,37,39,
+ 10,14,14,14,39,10,14,14, 14,14,14,10,14,10,10,14, 37,10,10,14,14,10,10,10, 39,14,14,10,14,10,14,14,
+ 8,8,1,16,16,8,8,1, 8,8,16,16,16,1,1,8, 16,8,16,16,1,1,16,8, 8,1,8,8,8,14,8,8,
+ 8,8,8,8,8,8,8,1, 1,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,1, 8,8,8,8,14,8,8,37,
+ 37,59,2,11,3,39,11,11, 2,39,10,2,2,59,11,59, 11,11,12,3,2,3,2,2, 11,11,3,12,4,59,17,11,
+ 2,2,7,2,2,3,7,2, 59,2,2,2,3,7,2,3, 2,2,8,7,2,2,3,3, 7,14,2,2,3,3,7,2,
+ 10,3,8,3,3,10,3,3, 10,3,3,10,3,10,10,18, 7,10,14,7,3,18,3,3, 7,14,10,3,8,3,3,7,
+ 8,11,10,11,3,7,7,7, 7,3,7,3,14,3,10,5, 3,10,3,3,3,10,3,5, 5,5,3,5,5,3,3,10,
+ // C3xx
+ 7,1,4,39,39,37,37,1, 37,14,12,3,39,14,1,1, 39,37,39,37,37,39,37,37, 39,39,37,37,39,37,37,37,
+ 14,14,14,1,1,1,14,14, 14,11,11,5,14,11,14,11, 37,14,10,14,14,10,14,14, 10,10,11,14,14,11,14,1,
+ 8,10,8,8,8,1,8,12, 8,12,8,8,12,8,12,1, 8,8,12,1,12,12,10,10, 8,8,8,8,8,8,8,39,
+ 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 1,8,1,8,8,8,8,8, 8,8,8,8,14,8,8,37,
+ 2,2,59,59,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,5,2,5, 2,2,2,5,2,2,2,3,
+ 3,14,7,14,10,3,7,3, 11,3,5,3,11,11,10,7, 14,11,14,11,11,11,5,3, 3,14,11,11,3,11,11,5,
+ 5,11,5,11,11,7,3,3, 3,14,7,5,14,11,10,3, 10,3,5,11,3,11,8,3, 5,3,7,3,3,8,8,10,
+ // C4xx
+ 7,6,1,39,37,37,37,37, 39,10,18,1,39,18,1,37, 37,37,37,37,39,37,37,37, 39,37,37,37,39,37,37,37,
+ 18,18,10,1,1,1,10,1, 6,10,6,3,18,10,18,10, 37,10,10,10,10,5,5,5, 5,5,18,18,10,6,10,1,
+ 8,8,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,8,1,1,1,1,1,8, 1,8,1,8,8,8,8,8,
+ 3,8,8,8,8,1,8,1, 1,8,8,1,8,8,1,8, 1,8,1,1,1,1,8,8, 8,8,8,8,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,5,7,3,3,7,3,5, 5,3,3,5,3,5,7,2, 2,2,8,8,7,8,3,3, 3,7,7,3,2,3,3,3,
+ 10,5,10,3,5,10,10,3, 10,3,10,10,7,10,10,3, 3,3,8,10,3,18,3,18, 3,18,3,10,3,8,5,10,
+ 10,3,3,3,3,10,8,20, 10,20,3,10,7,5,10,3, 10,3,8,7,3,8,5,5, 7,7,3,5,5,8,3,10,
+ // C5xx
+ 7,39,37,14,39,39,37,37, 39,14,18,39,39,18,39,39, 39,39,37,37,39,39,37,39, 39,39,39,39,39,39,37,37,
+ 18,18,18,14,39,18,18,1, 14,18,14,3,18,18,18,14, 37,10,10,10,10,6,10,10, 39,20,18,18,18,1,18,18,
+ 8,8,1,3,1,1,8,1, 1,1,8,1,1,1,1,1, 1,8,1,1,1,8,1,8, 8,8,1,8,8,8,8,8,
+ 39,8,1,3,1,8,8,1, 1,8,12,1,1,1,8,8, 1,8,1,1,1,8,8,8, 8,8,12,3,8,8,8,37,
+ 2,2,2,2,2,2,2,2, 2,37,5,2,2,2,12,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,3,2,3,14, 3,5,7,2,5,5,3,2, 5,2,5,2,7,3,3,7, 5,7,2,5,2,2,2,5,
+ 7,5,5,5,10,3,3,10, 11,10,10,10,10,5,18,11, 10,10,10,11,7,10,11,5, 5,7,11,5,8,3,11,7,
+ 3,11,3,8,5,8,5,7, 14,8,7,5,7,11,3,3, 5,10,7,3,3,5,7,5, 5,7,3,3,7,8,7,37,
+ // C6xx
+ 7,37,39,1,39,39,37,37, 37,10,12,37,39,11,37,37, 39,39,37,39,39,39,39,37, 39,39,37,39,39,39,37,39,
+ 11,12,10,1,37,39,11,1, 37,11,11,10,12,18,10,10, 37,3,11,10,3,3,3,3, 39,3,12,39,11,6,10,12,
+ 37,12,1,8,1,12,8,1, 1,12,8,1,1,1,1,8, 8,8,1,1,1,8,1,8, 8,8,8,8,8,8,8,8,
+ 8,8,1,1,1,8,8,8, 8,12,8,8,8,12,1,12, 8,8,8,8,1,12,8,12, 8,1,8,8,8,8,8,37,
+ 37,6,6,6,4,3,2,4, 6,39,6,39,6,6,6,2, 6,6,2,6,37,6,39,6, 39,2,4,2,39,3,37,4,
+ 2,2,3,7,3,3,7,7, 7,7,3,3,3,3,5,5, 2,3,5,7,3,3,3,3, 3,3,3,7,5,3,3,3,
+ 3,3,3,7,5,10,3,5, 10,7,10,5,3,10,10,7, 5,11,5,10,5,3,3,3, 3,3,3,5,11,5,3,3,
+ 7,11,7,11,3,10,3,3, 10,7,5,3,3,5,7,3, 3,7,5,3,3,3,11,5, 3,5,5,3,7,7,7,37,
+ // C7xx
+ 7,39,39,39,37,39,39,39, 39,11,11,14,39,11,37,39, 39,39,37,39,39,39,39,37, 39,39,37,39,39,39,39,39,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 37,11,11,11,11,14,11,11, 11,11,11,11,11,11,1,11,
+ 18,1,1,16,16,1,8,16, 1,1,16,16,1,1,16,1, 16,8,16,16,16,1,16,1, 37,1,16,11,39,11,11,11,
+ 39,1,1,31,16,1,8,1, 1,1,8,1,1,1,1,1, 8,8,1,1,37,1,1,3, 37,8,8,39,37,11,8,37,
+ 2,11,6,2,37,37,39,12, 37,6,39,3,39,2,2,39, 11,2,2,2,2,11,2,6, 11,11,2,2,2,2,11,2,
+ 11,11,3,3,7,5,3,3, 3,3,5,3,3,7,3,7, 5,3,5,5,14,7,3,3, 3,14,7,5,7,7,7,3,
+ 10,11,10,1,10,3,11,11, 11,11,11,11,11,11,11,5, 5,5,5,11,11,5,11,5, 5,5,11,11,7,11,11,11,
+ 5,11,5,11,11,11,11,3, 5,3,3,3,3,11,7,3, 11,7,3,3,5,5,5,3, 3,5,3,5,5,5,3,10,
+ // C8xx
+ 7,37,37,14,37,37,39,39, 37,11,11,37,39,11,37,39, 37,37,37,37,39,37,37,39, 39,39,37,39,37,37,37,37,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 37,11,11,11,11,11,18,11, 18,11,11,18,11,11,10,18,
+ 11,12,1,1,1,12,1,1, 12,12,12,12,12,1,12,12, 12,1,1,1,1,12,1,12, 1,1,1,11,11,11,11,11,
+ 11,12,1,1,1,12,3,1, 12,12,12,12,12,1,1,12, 10,37,12,12,12,12,12,11, 39,12,12,11,11,11,3,37,
+ 39,11,6,6,3,37,2,3, 39,6,12,37,39,11,10,6, 11,6,6,6,11,11,6,6, 11,2,2,2,2,11,11,1,
+ 11,3,7,5,3,3,3,3, 3,7,3,3,7,5,5,5, 5,3,5,7,3,3,3,3, 5,5,3,3,3,7,3,5,
+ 10,10,10,3,5,10,5,11, 3,11,10,3,10,11,3,3, 10,11,10,11,11,3,7,10, 11,3,11,11,11,3,11,10,
+ 7,11,3,11,11,11,11,3, 7,3,5,3,10,11,10,5, 10,5,7,7,3,3,11,5, 3,5,11,5,3,3,7,37,
+ // C9xx
+ 7,39,11,39,37,39,37,10, 39,11,18,39,37,11,37,39, 39,37,39,37,39,39,37,37, 39,39,37,39,39,39,37,39,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 37,11,11,11,11,11,11,11, 11,11,11,18,11,11,11,18,
+ 8,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,11,11,11,11,11,
+ 18,8,1,1,1,8,1,1, 1,1,1,1,1,1,1,8, 1,1,1,1,1,8,1,3, 1,1,1,1,11,11,11,37,
+ 39,6,6,6,6,11,37,12, 6,6,12,6,6,6,12,6, 2,2,2,6,2,11,6,2, 6,2,2,2,2,11,11,2,
+ 11,11,3,3,7,3,7,7, 3,7,3,3,7,3,7,3, 3,3,7,3,7,3,3,3, 7,3,3,11,7,3,7,11,
+ 14,3,7,7,10,18,3,3, 18,18,7,3,3,10,18,3, 3,10,10,18,7,7,18,7, 7,3,18,7,3,7,18,7,
+ 3,7,7,3,3,7,3,3, 3,7,3,7,3,3,3,20, 7,3,3,3,7,3,3,7, 3,3,3,7,7,3,7,37,
+ // CAxx
+ 7,6,39,37,39,37,37,37, 37,11,11,37,37,11,37,37, 37,37,37,37,39,37,37,37, 39,37,37,37,39,37,37,4,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 37,11,11,11,11,11,11,11, 14,11,11,18,11,11,10,18,
+ 11,10,12,12,12,1,1,12, 1,8,1,12,1,1,1,10, 12,1,1,1,1,1,1,12, 1,37,12,11,8,11,8,11,
+ 37,10,1,8,12,8,37,37, 1,1,12,12,12,12,1,8, 10,1,1,12,1,1,39,1, 1,8,12,1,11,11,11,37,
+ 2,2,6,2,2,11,2,2, 2,2,2,2,12,2,39,6, 11,39,2,6,2,11,6,6, 11,11,37,2,10,11,2,2,
+ 11,3,3,7,3,3,3,3, 3,3,7,3,7,3,3,3, 3,3,3,3,14,3,3,3, 7,3,14,3,3,3,3,3,
+ 10,14,3,11,7,10,3,3, 11,7,3,10,7,11,10,11, 3,11,3,11,3,3,3,3, 11,3,11,3,3,11,11,11,
+ 10,11,7,11,3,11,11,7, 14,3,3,10,3,11,10,3, 10,7,10,7,3,3,3,3, 3,20,7,3,7,3,7,11,
+ // CBxx
+ 7,1,39,37,37,39,37,37, 1,11,18,37,37,11,37,37, 37,37,37,37,37,37,37,37, 37,37,7,37,37,37,37,37,
+ 11,18,11,10,1,39,18,11, 11,11,11,3,18,18,10,11, 37,10,10,3,10,3,10,10, 3,11,11,18,11,3,11,18,
+ 8,10,1,1,16,1,1,16, 1,1,16,16,1,16,1,10, 16,1,1,16,16,1,16,1, 1,39,1,37,11,10,11,11,
+ 37,1,1,1,1,37,1,1, 1,10,1,1,1,8,1,1, 1,1,1,1,1,37,37,37, 8,39,39,3,8,37,3,37,
+ 2,6,6,4,39,2,2,2, 2,2,2,2,2,2,2,37, 2,2,6,16,6,11,37,6, 2,2,2,2,2,2,2,1,
+ 11,7,3,3,3,3,7,3, 7,3,3,3,7,3,3,3, 3,7,10,3,7,3,3,7, 3,3,7,7,3,3,3,3,
+ 10,14,3,14,3,10,3,11, 10,3,10,10,7,3,3,18, 3,3,14,10,18,3,3,3, 3,3,7,10,7,8,10,3,
+ 3,11,7,3,3,10,3,14, 7,14,3,3,11,11,10,3, 3,3,8,10,7,3,7,3, 3,3,3,3,3,3,7,10,
+ // CCxx
+ 7,39,39,37,39,39,39,37, 39,11,18,37,37,11,37,39, 39,39,39,37,39,39,39,37, 39,39,37,37,39,39,37,37,
+ 11,10,11,11,11,11,11,11, 18,11,11,10,11,10,10,11, 37,11,11,11,11,11,11,11, 11,11,11,6,11,37,6,18,
+ 8,1,12,12,12,16,1,1, 12,16,12,12,12,12,12,31, 12,1,12,12,12,8,12,6, 8,10,12,1,11,11,1,11,
+ 10,1,1,1,8,16,1,1, 1,10,1,1,1,3,1,1, 1,8,18,1,1,10,37,11, 8,10,1,8,8,11,3,39,
+ 2,2,6,2,2,2,2,2, 2,2,12,6,6,12,12,6, 6,6,6,6,6,6,3,6, 6,37,2,3,37,11,2,1,
+ 11,7,7,7,7,3,3,3, 3,3,7,3,3,3,7,3, 3,7,3,3,7,7,3,3, 3,3,3,3,3,3,7,7,
+ 10,18,7,3,7,10,3,11, 10,18,11,10,7,10,10,11, 11,11,11,10,3,3,3,3, 3,3,7,10,7,3,11,10,
+ 10,3,3,11,11,3,11,7, 10,7,3,7,3,11,10,3, 3,7,7,7,3,3,7,7, 3,3,3,3,3,3,7,10,
+ // CDxx
+ 7,39,37,14,39,37,37,37, 37,14,18,14,37,11,14,11, 39,39,37,39,39,39,37,37, 39,37,37,37,39,37,37,37,
+ 11,18,11,11,11,11,14,14, 14,11,11,11,18,18,14,11, 37,14,11,14,14,14,14,14, 14,11,14,18,11,11,10,18,
+ 18,1,1,1,1,1,1,1, 1,10,1,1,1,1,1,1, 1,1,1,1,1,1,1,8, 1,10,1,37,37,11,37,11,
+ 6,1,1,1,1,16,37,1, 8,10,8,39,1,1,1,1, 37,8,1,1,1,37,10,8, 1,39,8,37,14,11,11,37,
+ 37,6,6,6,37,39,37,2, 6,37,12,6,6,12,12,6, 6,39,6,6,37,6,6,6, 11,37,37,3,37,11,11,1,
+ 1,14,7,3,14,3,3,14, 3,3,3,7,3,7,3,3, 3,3,10,3,3,3,3,3, 3,14,14,3,3,7,3,3,
+ 10,3,14,14,10,10,3,11, 10,11,11,3,11,10,10,11, 3,7,10,11,14,7,11,7, 3,11,3,10,11,11,11,11,
+ 10,11,3,11,11,10,11,3, 7,14,3,3,11,11,10,20, 7,7,3,11,3,3,3,7, 3,3,3,3,3,7,3,10,
+ // CExx
+ 7,39,37,37,39,39,37,37, 39,11,18,37,39,11,37,37, 39,39,37,37,39,39,37,37, 39,39,3,37,39,39,37,37,
+ 11,10,11,11,39,10,10,10, 11,10,11,5,11,18,10,11, 37,11,10,10,11,10,10,11, 11,11,11,18,11,11,10,18,
+ 5,8,16,16,16,10,1,16, 1,1,16,16,1,16,1,37, 16,37,16,16,1,31,16,39, 37,1,16,37,37,8,37,10,
+ 8,18,37,1,8,37,10,8, 37,18,1,39,1,1,1,1, 16,8,16,16,37,37,16,37, 8,18,1,37,39,8,3,37,
+ 37,6,6,6,2,2,2,2, 2,2,2,6,2,6,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,3,2,2,3,2,3, 7,7,3,3,3,2,2,3, 3,2,2,2,3,2,3,2, 2,2,2,3,2,2,2,2,
+ 3,10,10,10,3,3,10,11, 3,10,3,10,10,10,11,7, 10,10,3,7,10,3,10,10, 11,18,3,3,11,3,3,10,
+ 7,11,3,7,3,3,11,3, 3,3,10,10,10,11,7,3, 10,3,10,3,10,3,3,3, 7,10,3,3,3,3,3,37,
+ // CFxx
+ 7,11,37,37,37,39,37,37, 37,11,18,11,39,11,37,37, 39,37,37,37,39,39,37,37, 39,39,37,39,37,39,37,37,
+ 11,11,11,6,11,11,11,11, 11,11,11,11,11,18,11,11, 37,11,11,11,11,11,11,11, 11,11,11,18,11,11,11,18,
+ 11,12,1,1,1,1,1,30, 1,16,30,1,1,1,1,16, 10,1,1,1,1,16,1,1, 1,1,1,11,11,11,11,11,
+ 8,12,37,39,12,39,8,37, 8,8,1,39,1,8,1,10, 10,8,12,8,1,12,8,8, 8,8,37,37,18,11,39,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,37, 11,39,11,6,11,2,2,6, 11,11,37,2,2,11,2,1,
+ 11,11,3,3,3,3,3,3, 3,7,3,7,3,7,3,3, 3,3,3,7,3,3,3,3, 3,3,7,7,3,3,20,7,
+ 10,3,3,7,3,10,3,11, 3,11,3,10,18,18,10,11, 10,11,18,10,3,3,3,3, 3,3,11,10,3,11,3,3,
+ 3,11,3,11,3,11,11,3, 3,3,3,3,3,11,10,3, 10,3,3,3,3,11,11,3, 11,3,3,3,3,3,3,10,
+ // D0xx
+ 7,39,1,37,39,1,39,9, 39,14,14,1,39,14,1,39, 39,39,37,37,39,39,37,37, 39,39,37,39,39,37,37,37,
+ 11,14,14,11,1,10,14,14, 14,14,14,14,10,10,10,11, 37,10,10,10,10,14,10,31, 10,10,11,10,14,10,10,14,
+ 1,13,1,13,13,13,1,1, 1,13,1,16,13,13,16,16, 16,37,13,1,16,13,16,1, 39,16,13,14,3,14,14,10,
+ 8,16,1,39,1,16,31,3, 37,16,1,16,16,16,16,16, 18,8,16,8,16,16,16,8, 39,16,8,8,14,8,3,37,
+ 10,2,6,2,2,2,2,2, 2,2,2,2,2,3,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,3,2,3,2,2,3,3, 2,3,14,2,3,3,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 10,3,3,10,3,3,10,3, 10,11,10,3,10,3,10,18, 3,10,10,10,3,3,3,3, 3,18,10,10,10,3,3,11,
+ 10,11,3,3,11,10,11,3, 3,3,3,3,3,3,10,20, 3,20,3,10,3,3,11,3, 3,3,11,3,3,3,3,10,
+ // D1xx
+ 7,37,37,37,39,39,11,37, 39,11,18,37,39,11,37,39, 39,39,37,37,39,39,37,37, 39,39,37,39,39,37,37,37,
+ 11,11,11,10,11,11,11,11, 11,11,11,11,11,11,11,11, 37,11,11,11,11,11,11,11, 11,11,11,18,11,11,11,18,
+ 11,1,12,12,10,1,10,1, 10,1,1,12,8,12,37,1, 10,37,37,12,10,1,10,31, 11,3,1,37,11,11,11,11,
+ 39,1,3,10,31,1,37,37, 10,10,1,11,11,37,7,1, 31,11,10,37,31,1,37,31, 11,39,37,8,11,11,11,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 11,2,2,2,2,2,2,2, 2,2,2,2,2,11,2,2,
+ 11,3,3,3,14,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,14,14,3,3,14, 3,3,14,3,3,3,14,3,
+ 10,20,10,11,11,10,11,11, 11,11,10,10,11,11,10,11, 3,10,10,11,3,3,11,3, 3,20,11,3,10,11,11,11,
+ 10,10,10,11,11,11,11,3, 10,3,10,3,11,11,10,10, 10,10,10,3,3,3,11,10, 3,3,11,3,3,3,3,10,
+ // D2xx
+ 7,39,39,37,39,39,37,12, 39,14,14,39,39,14,37,37, 39,39,37,39,39,39,39,37, 39,39,37,37,39,37,37,37,
+ 11,10,14,14,11,11,14,14, 14,14,11,11,11,10,10,14, 37,10,14,14,10,14,11,11, 14,11,14,18,14,14,14,18,
+ 11,16,1,1,1,16,8,1, 31,16,31,12,1,1,1,12, 1,1,1,12,12,12,1,31, 1,31,1,11,8,14,11,11,
+ 10,31,31,31,31,12,31,31, 31,12,8,39,31,31,31,1, 31,31,1,31,31,16,10,31, 1,31,8,1,14,39,14,39,
+ 37,6,6,2,2,39,3,4, 39,39,3,6,6,12,3,6, 11,2,2,2,28,3,2,2, 11,2,2,2,2,2,39,1,
+ 14,14,14,3,14,3,3,14, 14,3,3,3,3,14,3,2, 3,3,3,3,14,3,3,14, 14,14,3,3,14,3,14,14,
+ 10,14,10,14,3,10,3,11, 10,3,14,3,11,10,10,18, 10,3,3,10,3,3,3,3, 18,18,3,10,10,3,11,3,
+ 10,11,3,11,3,3,3,3, 10,3,3,3,3,11,10,3, 10,3,3,10,3,3,3,3, 3,3,10,3,3,3,3,10,
+ // D3xx
+ 7,37,37,1,39,37,37,37, 39,11,18,39,37,11,37,18, 37,39,37,39,37,39,37,37, 39,39,37,39,39,37,37,37,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,18,18,11, 11,11,11,11,11,11,11,11, 11,11,11,18,11,11,31,18,
+ 11,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,3,8,11,3,11,
+ 37,1,1,1,1,1,1,37, 1,8,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,8,1,11,11,8,39,37,
+ 10,11,6,6,2,11,3,12, 39,6,6,3,6,3,6,12, 11,2,11,3,11,6,6,2, 2,2,3,3,37,11,3,2,
+ 11,3,3,12,3,3,3,14, 3,2,3,3,9,3,3,3, 3,3,3,3,3,3,3,14, 20,14,14,14,14,3,3,3,
+ 3,10,3,3,10,3,10,11, 11,3,3,18,11,3,3,10, 3,3,3,11,18,3,3,10, 11,10,3,18,11,11,10,11,
+ 3,11,10,11,11,11,11,10, 3,3,3,3,11,11,3,3, 10,3,3,11,3,20,3,10, 11,3,3,3,3,3,3,10,
+ // D4xx
+ 7,39,37,1,39,39,39,37, 39,14,18,37,39,11,37,37, 39,39,39,37,39,39,37,37, 39,39,37,37,39,37,37,37,
+ 11,18,18,1,11,11,11,1, 18,11,11,3,18,18,18,14, 37,11,11,10,11,11,10,11, 11,11,18,18,11,11,11,18,
+ 1,1,1,1,1,8,1,8, 1,1,1,1,1,1,1,1, 1,1,1,1,1,8,1,11, 1,39,1,37,37,3,11,18,
+ 18,33,14,31,39,20,31,1, 11,10,1,39,1,1,1,20, 18,8,8,31,1,3,31,31, 1,18,1,8,11,11,3,39,
+ 37,6,6,3,3,11,12,3, 37,6,3,6,6,6,12,37, 11,6,3,3,3,6,6,6, 11,3,39,3,3,11,3,4,
+ 11,14,3,3,3,3,3,14, 14,3,3,3,3,3,3,3, 3,3,3,5,3,3,3,14, 3,14,3,14,3,3,3,3,
+ 10,18,3,3,3,18,3,11, 11,18,3,3,11,11,11,18, 3,11,18,3,3,18,3,18, 3,3,3,11,11,3,11,11,
+ 10,11,3,11,11,11,11,3, 10,14,14,3,3,11,10,20, 3,3,3,3,3,3,3,3, 3,3,3,3,3,10,3,37,
+ // D5xx
+ 7,39,37,39,37,39,37,37, 39,14,18,14,37,11,37,37, 39,39,37,39,39,39,37,37, 39,39,37,39,39,39,39,39,
+ 11,14,11,14,39,11,14,14, 14,11,11,14,18,10,18,11, 37,11,11,11,11,11,10,11, 11,11,11,18,11,11,10,18,
+ 1,31,1,12,12,1,12,1, 1,1,1,12,1,1,1,12, 1,37,1,12,1,1,12,37, 37,37,12,37,37,18,8,11,
+ 37,31,1,1,31,31,3,37, 1,1,1,1,39,31,1,31, 1,39,31,31,31,8,31,1, 1,31,37,3,14,37,8,37,
+ 2,2,2,2,2,2,2,2, 2,37,2,39,2,2,2,2, 2,2,2,3,2,2,2,6, 39,11,2,2,2,2,2,4,
+ 14,2,3,2,14,2,3,2, 3,3,3,2,3,3,3,3, 3,3,2,3,2,2,2,14, 2,3,2,14,3,3,3,3,
+ 10,14,3,3,3,3,3,11, 11,11,3,3,18,11,10,11, 3,11,3,18,3,3,3,3, 3,3,11,3,11,11,18,3,
+ 10,11,3,3,11,11,11,3, 14,14,3,3,3,11,10,3, 3,3,3,3,3,3,11,3, 11,3,11,3,3,3,3,37,
+ // D6xx
+ 7,39,37,37,37,39,37,37, 37,11,11,37,39,11,39,39, 39,37,37,37,37,37,37,37, 39,39,39,39,39,39,37,37,
+ 11,11,11,1,39,11,11,1, 11,6,11,3,11,1,11,1, 37,1,11,1,11,11,11,11, 37,1,11,1,11,3,1,1,
+ 39,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,37,37,11,3,11,
+ 8,1,1,1,1,37,1,1, 1,1,1,1,1,1,1,1, 1,8,1,1,1,1,1,1, 1,1,1,1,39,8,3,37,
+ 2,2,2,2,2,2,2,2, 6,2,39,12,6,4,6,37, 6,2,2,6,2,12,2,2, 6,2,2,2,2,2,2,4,
+ 2,14,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,2,3, 3,3,20,3,3,3,3,3,
+ 10,3,3,3,18,10,3,11, 10,11,3,18,3,11,3,11, 3,11,3,3,3,3,3,3, 3,3,3,10,3,3,3,8,
+ 3,11,10,11,11,10,11,3, 10,3,3,3,3,11,3,3, 3,3,3,3,3,3,11,3, 3,3,3,3,3,3,3,10,
+ // D7xx
+ 7,39,37,37,39,37,37,6, 39,18,18,37,39,18,37,4, 37,37,37,39,39,39,37,37, 37,39,37,37,39,39,37,37,
+ 18,18,18,11,11,11,18,10, 11,18,11,3,18,10,18,10, 37,11,11,11,11,11,11,11, 11,11,18,18,18,11,10,11,
+ 11,10,11,3,11,10,11,11, 8,10,11,11,11,11,11,37, 11,8,3,39,11,8,11,11, 11,10,11,11,11,11,8,39,
+ 11,11,11,11,11,37,11,3, 11,18,11,11,11,11,11,3, 18,11,11,11,11,3,37,11, 3,18,11,11,11,11,11,37,
+ 2,2,2,2,2,3,11,6, 2,6,37,6,3,4,6,6, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,3,3,2,3,2,2, 2,2,2,3,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3,
+ 10,18,10,3,7,3,3,3, 10,3,3,3,3,14,18,18, 3,20,10,3,3,18,3,11, 10,18,3,3,3,3,3,3,
+ 10,20,3,3,3,3,3,3, 14,3,3,10,3,3,3,3, 3,3,3,3,3,3,3,3, 3,3,11,10,7,8,20,37,
+ // D8xx
+ 7,39,39,39,39,39,39,39, 39,1,18,39,39,11,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,11,11,39,39,39,11,11, 11,11,1,3,11,1,11,1, 7,1,1,1,1,1,1,1, 1,1,11,18,11,1,18,18,
+ 1,12,1,12,1,12,1,1, 1,12,1,1,1,1,1,12, 1,39,1,1,1,1,1,1, 1,1,1,11,11,11,39,11,
+ 39,12,1,33,33,12,1,1, 33,1,33,1,33,33,1,1, 8,39,1,1,33,1,1,1, 1,1,33,8,18,11,3,39,
+ 39,6,6,6,39,39,39,39, 3,39,12,6,2,6,6,39, 3,6,39,3,3,6,6,6, 39,39,39,2,33,11,39,2,
+ 11,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,14,14,3,14,5,
+ 10,14,14,14,3,10,11,11, 11,11,10,10,10,12,18,10, 10,11,10,10,3,27,10,27, 11,27,11,11,11,11,11,11,
+ 10,11,10,11,11,11,11,20, 14,14,10,10,11,11,10,10, 10,11,10,11,3,11,11,3, 11,3,11,10,3,3,3,3,
+ // D9xx
+ 7,12,39,1,4,39,4,39, 39,18,18,39,39,11,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,18,18,14,1,18,18,11, 6,18,14,5,18,18,18,18, 7,6,6,6,6,6,6,6, 6,6,18,18,18,18,14,18,
+ 8,1,12,12,12,1,3,1, 12,1,12,12,12,12,12,1, 12,3,12,12,12,1,12,7, 1,3,12,7,7,6,8,39,
+ 18,8,12,12,12,1,7,8, 12,1,1,12,1,12,1,8, 3,8,12,12,12,3,12,6, 1,8,39,7,3,14,5,3,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,3,6,6, 6,6,3,3,3,3,39,3,
+ 2,14,2,5,3,5,2,14, 14,3,14,2,2,14,9,3, 6,2,3,3,14,14,3,14, 3,2,14,14,2,5,2,3,
+ 10,14,7,14,18,10,3,11, 10,11,18,18,18,20,10,6, 20,11,18,20,20,5,20,18, 12,3,11,18,10,11,18,11,
+ 14,11,3,11,11,11,11,8, 14,14,14,5,11,11,10,7, 3,3,3,11,3,11,11,3, 11,5,11,7,5,5,5,10,
+ // DAxx
+ 7,1,39,1,39,39,39,39, 11,11,11,39,39,11,39,11, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,11,11,11,11,11,11,11, 11,11,11,5,11,11,11,11, 7,11,11,5,5,3,5,11, 11,11,11,18,11,11,11,18,
+ 11,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,3, 1,1,1,11,11,11,11,11,
+ 8,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,8,1,1,1,1,1,3, 1,1,1,8,8,39,11,3,
+ 2,2,6,6,2,2,2,2, 2,2,12,6,2,12,12,2, 6,2,6,2,11,2,2,6, 2,2,2,3,2,11,12,1,
+ 11,11,5,3,3,8,5,3, 10,2,5,3,8,2,3,2, 6,5,6,2,3,2,3,5, 5,3,2,8,3,6,2,11,
+ 18,18,11,3,18,10,10,11, 11,11,11,11,11,18,18,11, 11,11,11,11,11,11,11,18, 11,11,11,3,11,11,11,10,
+ 27,11,27,11,11,11,11,7, 12,3,3,5,11,11,3,3, 11,11,11,11,3,11,11,3, 12,3,11,3,3,5,3,11,
+ // DBxx
+ 7,39,39,39,39,39,39,39, 39,10,10,39,39,10,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,10,10,10,39,10,10,1, 3,11,11,5,11,10,10,31, 7,31,31,31,31,31,31,31, 31,31,10,18,10,10,10,10,
+ 18,12,1,16,16,12,12,16, 12,1,12,16,16,16,16,39, 16,7,16,16,1,1,12,7, 10,1,16,7,7,8,7,8,
+ 8,1,12,39,16,1,39,16, 31,8,8,16,8,8,1,10, 8,39,1,1,1,8,39,1, 1,8,7,39,3,8,5,39,
+ 2,2,6,2,39,39,2,2, 2,6,6,2,2,2,2,39, 2,6,2,2,2,2,6,6, 39,3,2,39,2,2,2,1,
+ 11,3,3,8,3,8,8,3, 10,2,3,3,3,3,5,3, 2,2,2,2,2,2,2,2, 2,2,3,3,3,6,5,3,
+ 3,18,10,10,10,10,10,11, 11,10,10,10,10,10,18,11, 11,11,10,11,18,10,11,10, 11,7,3,1,11,11,27,10,
+ 3,11,3,11,11,11,11,3, 3,1,7,3,11,11,27,27, 7,3,11,11,5,11,5,3, 11,7,11,3,3,3,3,3,
+ // DCxx
+ 7,39,1,11,4,39,39,39, 39,1,11,39,39,11,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,11,11,11,11,11,11,1, 11,1,11,11,11,11,11,11, 7,11,11,11,11,11,11,11, 11,11,11,20,11,11,1,1,
+ 11,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,11,1,1,1,11,1,1, 1,1,1,11,11,11,11,11,
+ 8,11,1,1,1,1,1,8, 1,11,11,1,1,1,1,11, 11,11,1,1,1,11,1,11, 1,1,1,11,11,3,11,3,
+ 39,6,6,39,6,11,11,3, 39,3,3,6,6,6,3,39, 2,11,11,6,3,11,6,6, 2,11,2,3,3,2,3,2,
+ 2,11,3,3,3,7,11,11, 5,11,2,3,5,3,11,6, 3,5,3,3,7,3,6,3, 3,5,11,5,3,3,3,3,
+ 3,10,11,11,11,10,18,11, 11,11,11,18,11,10,11,11, 13,11,11,11,18,11,11,11, 11,11,11,11,11,11,13,11,
+ 3,11,5,11,11,11,11,3, 20,20,20,20,11,11,3,3, 11,20,11,20,20,11,3,20, 11,3,11,3,1,5,3,11,
+ // DDxx
+ 7,1,4,39,39,39,39,39, 39,13,11,13,39,11,4,39, 39,3,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,13,11,11,11,11,13,13, 13,11,13,5,11,13,13,11, 7,11,11,11,11,11,11,13, 11,11,11,13,11,11,13,13,
+ 11,13,1,1,13,13,13,13, 13,1,1,13,13,13,13,13, 1,39,13,13,13,1,1,13, 13,13,13,7,13,11,3,11,
+ 13,13,13,13,13,13,13,13, 13,13,13,13,13,13,13,13, 13,13,13,13,13,7,13,7, 13,13,13,11,13,11,5,3,
+ 56,6,6,6,3,3,3,6, 39,6,12,6,6,12,12,6, 6,6,6,6,6,6,6,6, 11,6,6,3,3,11,3,3,
+ 11,11,8,5,5,3,7,8, 7,9,3,3,3,3,3,8, 3,3,6,1,3,7,3,3, 3,3,3,5,3,6,5,11,
+ 6,5,10,11,11,11,11,11, 11,11,11,10,11,11,11,11, 11,11,10,11,11,11,11,5, 11,11,11,11,11,11,11,11,
+ 3,11,10,11,11,11,11,13, 20,10,10,10,11,11,20,10, 10,20,10,11,20,11,11,20, 11,20,11,8,3,10,13,3,
+ // DExx
+ 7,1,39,39,4,39,4,39, 39,11,11,39,6,11,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39,
+ 11,11,11,11,11,11,11,11, 6,11,11,11,11,11,13,11, 7,11,11,11,11,11,11,6, 11,11,11,6,11,11,13,18,
+ 11,13,13,13,16,13,13,13, 13,13,1,13,13,13,16,16, 16,8,1,16,13,13,13,13, 1,13,33,11,11,11,11,1,
+ 13,13,13,1,1,13,13,1, 13,13,1,3,13,16,13,13, 1,3,1,8,13,13,16,1, 7,16,39,7,6,11,11,3,
+ 2,6,2,2,2,2,2,2, 2,2,2,2,2,2,2,6, 2,2,31,2,2,2,2,2, 6,2,2,4,2,2,2,4,
+ 11,11,3,3,3,3,2,2, 2,2,2,2,2,2,2,7, 2,3,6,8,5,6,6,3, 6,3,3,3,7,6,6,11,
+ 5,18,10,10,10,18,10,11, 11,11,11,18,10,11,18,11, 10,11,10,11,18,11,11,10, 11,10,11,18,11,11,11,11,
+ 27,11,10,11,11,11,11,10, 20,5,20,10,11,11,13,10, 10,20,20,11,20,11,11,7, 11,10,11,7,13,13,11,11,
+ // DFxx
+ 7,1,39,39,7,10,39,39, 39,11,11,7,39,11,39,11, 39,39,39,39,39,39,39,39, 39,39,37,37,39,39,39,39,
+ 11,11,11,11,11,11,11,11, 11,11,11,6,11,1,11,11, 7,11,11,11,11,14,14,11, 14,14,11,6,11,11,11,1,
+ 3,11,1,1,1,1,1,1, 1,1,1,8,1,3,8,1, 1,1,1,1,1,1,1,39, 11,8,1,3,3,11,3,11,
+ 3,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,39,1,1,1,1,1,1, 3,1,1,7,11,11,39,3,
+ 3,6,6,6,6,3,12,6, 6,6,6,6,6,6,6,6, 6,6,6,12,6,6,6,3, 39,6,6,6,6,11,3,4,
+ 11,11,3,5,5,6,3,5, 11,5,10,5,3,6,3,6, 6,3,5,3,3,3,3,7, 3,3,3,3,6,3,5,11,
+ 7,10,10,11,10,10,10,11, 11,11,11,11,10,10,11,11, 11,11,10,11,11,10,10,10, 10,10,11,11,11,11,10,11,
+ 27,11,10,11,11,11,11,10, 20,3,10,20,11,11,20,20, 10,10,3,11,20,11,11,3, 11,20,11,3,1,7,11,11,
+ // E0xx
+ 7,37,37,14,1,37,37,37, 37,10,10,37,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,10,10,10,11,11,10,10, 10,10,10,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10,
+ 10,3,11,11,33,11,6,33, 8,11,11,6,33,11,33,3, 11,39,33,8,8,11,6,33, 11,6,8,10,10,10,10,6,
+ 17,11,11,11,11,39,11,11, 6,11,16,30,11,11,11,11, 11,11,11,11,11,6,11,11, 11,11,39,6,17,37,3,37,
+ 6,6,6,6,6,11,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,10,6,6,6, 37,6,12,3,10,6,10,10,
+ 36,14,14,36,2,2,2,2, 14,14,14,14,36,36,2,2, 2,2,2,2,2,14,2,14, 2,2,14,14,14,2,14,14,
+ 14,14,14,14,17,14,17,14, 14,1,14,14,18,14,14,8, 27,3,18,27,5,27,3,3, 27,3,3,6,27,7,3,6,
+ 3,10,10,10,10,17,10,10, 17,17,10,10,10,10,17,10, 10,10,10,10,10,10,10,10, 17,17,17,18,5,10,10,10,
+ // E1xx
+ 7,1,1,37,37,39,4,37, 12,1,11,20,37,11,3,11, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,1,10,11, 37,11,11,11,17,11,11,11, 11,11,11,1,11,11,11,1,
+ 11,11,1,1,1,17,1,17, 17,8,1,1,1,8,1,39, 6,39,1,6,39,1,8,11, 6,8,1,11,17,11,11,11,
+ 37,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,6, 1,6,1,6,11,11,11,37,
+ 2,2,2,2,2,2,2,6, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,6, 11,2,12,6,12,12,12,2,
+ 36,14,14,3,14,36,5,14, 14,6,36,36,36,36,36,36, 3,7,3,10,14,14,14,14, 2,14,2,2,2,2,14,14,
+ 3,14,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,18, 11,11,11,11,11,11,11,11,
+ 10,11,36,11,11,11,11,17, 10,17,20,10,11,11,10,17, 10,18,17,10,18,11,11,18, 17,17,17,10,10,18,18,10,
+ // E2xx
+ 7,7,37,37,37,39,37,37, 37,10,10,37,4,10,1,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,11,10,10,17, 10,10,10,3,10,10,10,10, 37,10,17,10,10,10,10,10, 10,10,10,10,10,10,10,10,
+ 10,8,8,11,3,6,11,6, 6,6,6,6,6,6,11,6, 6,6,11,6,11,6,6,6, 6,37,6,10,17,10,37,10,
+ 17,10,11,16,16,10,16,11, 11,10,16,16,16,11,11,10, 16,8,16,16,11,11,16,6, 8,11,16,3,6,6,3,37,
+ 2,2,2,6,2,2,2,2, 2,2,2,6,2,6,2,6, 6,2,2,2,2,2,2,2, 2,2,10,6,2,2,6,6,
+ 36,14,36,3,14,36,14,3, 36,5,14,14,3,36,36,3, 3,3,3,10,14,14,3,14, 10,14,14,14,14,3,14,3,
+ 14,18,14,14,6,18,3,14, 3,18,14,14,18,14,14,18, 27,3,18,3,7,18,3,3, 27,18,27,7,20,20,27,3,
+ 10,18,10,17,10,10,17,10, 10,17,10,10,17,10,10,18, 10,10,10,10,17,18,10,10, 17,18,10,10,10,3,16,10,
+ // E3xx
+ 7,37,37,37,37,37,37,1, 3,11,11,37,37,11,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,39,37,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,10,11, 37,11,11,11,11,11,11,11, 11,11,11,10,11,11,11,17,
+ 10,12,1,6,6,12,6,1, 31,18,6,6,1,31,39,6, 6,37,6,39,12,1,6,6, 1,6,8,11,11,10,11,11,
+ 6,10,12,12,12,1,37,12, 11,12,6,6,12,12,8,1, 12,8,12,1,12,12,10,37, 37,3,12,11,17,11,11,37,
+ 2,2,2,2,2,2,2,6, 2,2,2,2,2,2,2,2, 11,11,6,6,6,6,6,6, 11,11,6,4,6,11,6,4,
+ 36,11,36,36,14,36,36,36, 14,6,14,36,36,36,7,36, 3,3,3,10,14,14,3,7, 8,14,14,11,5,3,3,11,
+ 5,5,11,11,11,18,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,18, 11,11,11,11,11,11,11,11,
+ 10,11,17,11,11,11,11,36, 10,17,10,10,11,11,10,20, 10,20,17,11,17,11,11,17, 17,17,11,3,3,5,3,1,
+ // E4xx
+ 7,11,1,11,37,37,37,37, 1,11,11,4,4,11,3,11, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,17,11,11,11,11,11,17, 11,11,11,17,11,17,11,11, 11,11,11,11,11,11,11,11, 11,11,11,17,11,11,11,17,
+ 11,1,6,6,6,1,6,1, 1,17,1,1,1,1,6,11, 6,37,1,6,1,1,1,8, 6,1,1,11,11,11,11,11,
+ 17,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,17,11,11,1,
+ 2,11,6,6,12,11,2,6, 6,6,6,6,6,6,11,6, 11,6,31,3,11,6,6,6, 11,11,6,3,12,12,2,6,
+ 36,11,14,7,14,36,8,14, 36,14,14,14,5,3,36,3, 3,3,5,10,14,14,6,14, 2,2,2,2,2,2,2,2,
+ 3,14,3,14,11,18,11,11, 11,11,11,14,11,11,11,11, 11,11,11,11,11,11,11,18, 11,11,11,11,11,11,11,11,
+ 10,17,10,11,1,10,11,17, 10,17,10,10,11,11,10,18, 10,17,17,10,17,18,17,17, 17,17,17,10,10,20,11,10,
+ // E5xx
+ 7,10,37,37,37,39,37,4, 39,10,10,1,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,11,11,10,10,10, 11,11,11,10,10,10,10,10, 37,11,11,11,36,11,10,11, 11,11,10,10,11,11,10,10,
+ 11,6,31,31,6,31,8,1, 6,6,6,6,1,6,6,37, 8,6,1,37,31,6,1,37, 3,6,6,11,10,11,11,11,
+ 17,1,1,10,1,1,1,1, 1,1,6,1,1,1,1,33, 1,39,1,1,1,33,1,39, 8,6,1,11,11,11,11,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 17,7,11,11,11,18,18,11, 11,11,11,3,11,18,11,11, 11,11,11,18,11,5,11,18, 11,11,11,3,11,11,11,20,
+ 17,11,10,11,10,11,11,10, 17,10,10,10,10,10,18,17, 10,10,10,18,17,11,10,17, 17,17,17,18,5,20,10,10,
+ // E6xx
+ 7,11,1,9,1,39,1,12, 1,12,12,11,37,11,37,11, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,12,11,11,11,11,11,11, 11,11,11,5,12,11,12,11, 37,11,11,11,11,11,11,11, 11,11,11,12,11,11,11,12,
+ 6,33,11,8,11,33,6,11, 11,33,11,33,33,31,33,33, 33,6,1,6,33,33,6,11, 8,37,11,39,17,6,11,11,
+ 17,12,1,1,1,12,1,1, 1,12,1,1,1,1,1,12, 1,1,1,1,1,12,1,12, 6,8,1,11,6,11,6,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 5,11,11,11,3,11,7,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,3, 11,11,11,11,11,11,11,11,
+ 10,11,10,11,11,11,11,20, 10,17,10,17,11,11,17,18, 1,10,11,11,18,11,11,17, 11,20,11,10,10,3,11,10,
+ // E7xx
+ 7,10,37,37,4,39,37,37, 37,11,10,20,4,10,10,37, 37,37,37,37,37,37,37,37, 37,37,37,3,37,37,37,37,
+ 10,10,10,11,11,10,11,11, 10,10,10,11,10,10,10,17, 37,10,10,11,11,11,10,17, 11,10,11,20,10,11,11,10,
+ 11,11,11,11,31,8,31,31, 6,11,11,11,31,8,31,6, 31,3,31,31,31,11,31,6, 11,11,31,11,17,11,6,11,
+ 11,11,11,16,16,11,11,16, 11,11,16,16,11,11,16,11, 16,6,16,16,16,11,16,11, 8,11,16,11,17,11,8,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,14,2,2,2,2,2,2,
+ 1,14,14,14,18,18,1,14, 5,18,14,14,18,3,6,18, 27,3,18,7,3,18,6,3, 27,18,27,3,8,5,3,3,
+ 10,17,10,1,10,17,17,17, 10,17,10,10,10,10,10,18, 10,17,18,10,17,1,1,17, 17,17,17,10,11,13,10,10,
+ // E8xx
+ 7,1,37,1,12,39,1,37, 37,10,10,10,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,17,11,10,10,10, 10,10,10,10,10,10,10,10, 37,14,14,14,14,14,14,14, 14,14,10,10,10,10,10,10,
+ 10,6,6,11,6,12,3,11, 14,6,6,12,31,6,12,6, 18,37,11,11,11,12,6,6, 8,37,6,10,10,10,10,10,
+ 10,12,11,11,11,12,11,11, 12,12,12,12,12,11,12,12, 12,11,11,11,12,12,11,6, 11,11,11,6,10,39,10,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 10,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,14,2,2,2,2,2,2,
+ 14,14,14,14,7,14,3,14, 14,3,14,14,18,14,14,18, 14,3,14,14,6,18,3,3, 5,3,27,3,20,20,20,7,
+ 10,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,18, 10,10,10,10,10,10,10,10, 17,10,17,8,20,20,10,10,
+ // E9xx
+ 7,1,11,4,1,37,39,37, 37,11,11,11,37,11,37,1, 37,37,37,37,37,37,37,37, 37,37,37,3,37,37,39,37,
+ 11,10,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 37,17,11,17,11,11,11,11, 11,11,10,10,11,10,11,11,
+ 11,11,11,11,11,11,3,8, 33,11,6,33,6,31,11,11, 33,6,31,31,3,11,11,11, 11,11,33,6,6,11,11,11,
+ 6,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,8, 11,11,11,6,17,6,17,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,14, 2,2,2,2,2,2,2,2, 2,2,2,2,14,14,2,14, 2,14,2,2,2,2,2,3,
+ 18,14,14,14,3,14,3,14, 18,3,14,14,18,14,18,3, 14,3,14,14,18,27,18,18, 3,3,18,18,20,20,20,3,
+ 14,17,17,17,17,17,17,17, 17,17,20,17,17,17,18,17, 17,10,10,18,18,17,17,17, 17,17,17,18,20,18,18,10,
+ // EAxx
+ 7,37,9,37,37,39,37,9, 37,10,10,37,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,12,10,12,11,10,10,10, 10,10,10,10,10,10,10,10, 37,10,6,10,10,10,10,10, 10,10,10,10,10,10,10,12,
+ 10,31,31,31,31,31,6,12, 31,6,6,31,31,8,31,31, 3,31,37,31,31,37,31,8, 11,11,31,10,12,10,12,10,
+ 10,10,12,12,12,11,37,12, 11,10,11,12,11,11,11,10, 12,11,11,11,11,11,11,12, 11,6,12,8,17,6,3,37,
+ 6,39,6,6,2,3,2,3, 6,6,6,2,39,6,6,6, 6,3,6,6,6,6,6,6, 6,11,6,11,12,6,6,6,
+ 10,14,6,5,14,36,8,14, 3,3,5,5,5,5,5,3, 2,2,2,2,2,2,9,2, 2,2,2,2,2,2,2,12,
+ 3,3,14,3,10,5,3,3, 3,3,3,3,3,3,5,3, 14,3,14,3,3,3,5,3, 3,34,3,7,20,20,20,20,
+ 10,20,10,10,18,10,10,20, 10,20,10,10,10,10,10,20, 10,10,10,10,18,20,10,10, 10,20,10,18,20,20,18,10,
+ // EBxx
+ 7,10,37,37,4,37,37,37, 37,10,10,10,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,11,10,16,10, 10,11,10,5,10,10,10,11, 37,10,17,17,17,10,10,17, 17,10,10,10,10,36,10,10,
+ 10,3,8,8,8,3,11,11, 3,18,14,11,8,3,31,18, 18,37,8,3,31,3,8,37, 39,18,37,3,10,10,37,39,
+ 17,10,11,11,16,11,11,16, 11,11,16,16,11,16,11,10, 16,11,11,16,16,11,16,11, 3,3,11,37,8,37,5,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,36,2,5,2,2,2, 2,2,2,36,2,2,2,2, 2,2,2,2,2,5,2,2, 2,2,5,2,5,2,7,2,
+ 5,18,14,5,3,18,3,14, 3,18,3,3,18,14,18,18, 3,3,18,18,18,18,3,3, 3,3,3,8,20,20,20,20,
+ 10,18,10,10,10,10,10,20, 10,18,10,10,17,10,10,18, 17,10,18,10,18,18,18,10, 17,17,17,10,10,20,10,10,
+ // ECxx
+ 7,11,37,1,37,37,37,37, 37,14,10,14,37,11,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,10,11,11,11,10,10,17, 14,10,11,11,10,10,10,10, 37,11,10,10,17,11,17,11, 17,17,11,10,11,10,14,10,
+ 11,1,17,8,8,18,8,8, 8,8,1,8,31,17,39,11, 17,31,17,17,1,1,17,17, 17,18,3,14,17,11,11,11,
+ 3,1,12,12,12,16,1,1, 12,16,12,12,12,12,12,1, 12,1,12,12,12,16,12,1, 11,10,12,11,17,11,11,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,10,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 2,2,2,2,2,2,2,2, 14,2,2,3,36,36,3,2, 2,2,2,10,2,2,2,14, 2,2,2,2,2,2,14,11,
+ 18,14,14,14,17,3,17,14, 14,18,14,14,3,14,14,18, 3,5,3,11,7,18,3,5, 18,3,12,5,20,20,20,20,
+ 10,18,17,17,17,10,17,17, 10,17,10,17,10,10,10,10, 17,17,17,10,17,18,17,17, 18,17,17,10,10,10,11,10,
+ // EDxx
+ 7,11,37,37,37,37,37,37, 4,1,1,1,37,1,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,1,11,17,11,11,1,1, 11,11,11,17,1,10,1,1, 37,11,11,11,11,11,11,11, 11,11,10,1,11,11,10,1,
+ 11,1,33,33,33,18,1,1, 33,18,1,33,8,33,1,1, 33,39,33,33,33,33,33,37, 11,37,33,11,10,11,11,11,
+ 17,1,1,1,1,1,1,1, 1,10,1,1,1,1,1,1, 1,1,1,1,1,1,1,8, 1,10,1,11,17,10,11,37,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 11,2,2,3,2,2,2,2, 2,2,2,2,2,2,2,10,
+ 1,11,36,3,11,3,3,3, 11,7,36,36,3,5,5,7, 7,5,3,10,7,3,3,3, 10,9,5,5,3,3,9,11,
+ 5,18,11,11,11,18,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11,
+ 10,11,36,11,11,10,11,10, 10,18,10,10,11,10,10,18, 18,10,10,10,10,11,10,10, 11,18,11,10,10,20,10,10,
+ // EExx
+ 7,10,37,37,37,37,4,37, 37,10,10,1,37,10,10,37, 39,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,11,10,10,10, 10,10,10,10,10,10,10,10, 37,17,17,17,10,17,10,10, 10,17,10,10,10,10,10,10,
+ 10,3,3,31,31,18,31,31, 3,3,31,8,31,31,31,39, 8,37,31,31,31,37,31,31, 31,18,8,10,10,10,3,10,
+ 17,11,16,16,16,37,11,16, 8,11,16,16,11,16,11,11, 16,11,16,16,11,8,16,3, 11,11,16,37,10,3,3,37,
+ 2,2,6,2,2,10,3,2, 2,2,3,2,6,2,6,2, 3,6,11,2,10,6,2,2, 37,2,2,2,2,3,2,2,
+ 10,3,5,3,5,3,8,3, 3,3,3,3,36,10,3,5, 5,7,3,5,3,7,3,3, 10,2,10,5,10,3,3,10,
+ 17,18,3,10,17,18,3,8, 3,18,5,3,17,5,3,18, 3,3,3,3,3,18,3,27, 18,3,3,3,5,20,20,20,
+ 17,10,10,10,10,10,10,17, 10,10,10,10,10,10,18,10, 10,10,10,18,10,10,10,10, 17,17,17,18,18,10,10,10,
+ // EFxx
+ 7,37,37,37,37,37,37,10, 37,17,20,17,37,17,37,37, 37,37,37,37,37,37,37,37, 37,37,37,3,37,37,37,37,
+ 17,17,17,17,17,10,18,17, 17,17,17,17,17,10,10,17, 37,10,10,17,10,10,10,17, 17,10,17,17,17,17,17,17,
+ 3,11,11,11,11,11,3,3, 8,18,3,11,3,31,37,37, 18,37,37,39,31,11,37,8, 8,37,11,37,17,17,37,17,
+ 17,12,11,11,11,11,11,30, 3,16,30,11,11,11,11,16, 10,11,11,11,11,16,11,11, 11,39,39,8,17,37,3,37,
+ 2,2,2,2,2,2,2,2, 2,37,2,2,2,39,2,2, 2,2,2,2,2,2,2,3, 39,3,2,2,2,2,12,2,
+ 17,36,36,3,2,5,5,5, 2,36,7,7,36,2,3,2, 5,3,3,10,2,3,3,5, 2,2,2,2,2,2,2,2,
+ 3,5,18,18,18,7,18,18, 18,3,18,18,18,18,18,3, 18,3,18,18,18,7,18,5, 3,5,3,3,20,20,18,20,
+ 10,5,18,20,18,10,18,18, 10,20,18,10,18,18,10,10, 10,20,18,10,18,20,18,18, 10,20,18,10,10,20,18,10,
+ // F0xx
+ 3,37,1,37,10,37,39,39, 37,10,10,37,37,11,4,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,11,10,10,10, 10,10,11,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,1,
+ 10,12,3,14,1,1,1,13, 1,13,8,8,13,13,37,18, 18,37,13,39,37,13,39,11, 39,37,1,11,10,10,3,10,
+ 17,13,1,13,13,13,1,1, 1,13,1,16,13,13,16,16, 16,37,13,1,16,13,16,1, 1,16,13,11,10,11,14,39,
+ 39,6,6,6,36,11,4,36, 6,6,6,6,6,6,6,6, 2,6,6,6,6,6,6,6, 6,4,3,3,10,2,10,10,
+ 2,11,3,3,3,3,2,3, 5,3,3,5,5,5,3,5, 3,3,5,10,5,3,5,5, 7,5,11,11,5,3,5,11,
+ 3,18,5,8,3,18,3,11, 5,18,3,3,18,3,18,18, 3,18,18,18,18,18,3,7, 3,18,5,3,20,20,20,20,
+ 10,10,10,10,17,10,10,10, 10,17,10,17,10,10,10,18, 10,10,10,10,10,10,10,17, 10,17,17,10,10,13,10,10,
+ // F1xx
+ 3,9,37,9,1,10,39,37, 1,12,10,37,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,10,10,10,10, 10,10,17,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,12,10,10,10,10,
+ 1,1,1,12,8,1,8,1, 1,1,1,8,1,1,37,1, 3,8,3,12,8,1,37,12, 8,37,8,10,1,12,8,10,
+ 17,1,12,12,10,1,10,1, 10,1,8,12,10,12,1,1, 10,1,10,12,10,1,37,1, 10,10,1,3,17,11,5,37,
+ 39,6,6,6,4,10,4,4, 6,6,37,6,6,6,6,6, 6,6,6,6,10,6,6,6, 6,37,10,3,4,3,4,4,
+ 10,11,36,3,3,7,5,5, 3,8,5,3,5,7,3,8, 3,3,3,10,3,3,5,3, 10,5,3,3,3,7,3,3,
+ 17,5,11,3,3,3,3,3, 11,3,3,18,3,18,18,11, 18,3,18,5,3,5,3,18, 3,7,18,18,20,20,20,20,
+ 10,20,10,17,10,10,10,20, 10,17,10,10,10,10,10,10, 10,10,10,10,17,10,10,10, 10,20,10,10,10,18,10,10,
+ // F2xx
+ 3,1,37,1,37,37,39,37, 37,10,10,10,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,1,10,10,10, 10,10,10,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,10,10,20,10,10,
+ 10,31,31,17,31,12,10,1, 1,18,31,1,31,31,31,1, 31,37,8,31,31,8,39,37, 39,3,31,10,10,10,8,10,
+ 17,16,1,1,1,16,39,1, 12,16,1,12,1,1,1,12, 1,1,1,12,12,12,1,1, 1,10,1,1,17,11,37,37,
+ 37,6,6,39,4,10,3,4, 37,6,6,6,6,6,6,37, 6,6,6,6,10,6,39,6, 39,4,12,11,3,12,3,4,
+ 10,11,5,3,5,5,3,3, 5,5,5,3,5,8,1,3, 3,5,7,10,7,5,26,5, 10,3,3,20,3,3,3,7,
+ 3,18,11,3,5,18,3,11, 11,18,11,3,3,11,3,18, 3,3,1,11,3,18,11,3, 5,18,3,3,11,11,11,3,
+ 10,18,10,17,17,10,17,18, 10,17,10,10,17,10,10,18, 10,10,10,10,18,18,17,10, 17,18,17,10,10,10,10,10,
+ // F3xx
+ 3,1,1,37,4,4,39,37, 1,10,10,9,37,10,37,1, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,10,17,10,10,10, 10,10,11,17,10,10,10,1, 37,18,1,10,10,18,10,18, 10,18,1,10,10,10,1,10,
+ 3,11,1,1,1,1,1,1, 18,18,31,31,31,1,1,8, 1,11,1,39,31,37,39,1, 37,37,31,37,1,10,37,1,
+ 39,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,11,17,11,24,37,
+ 37,39,6,6,37,10,3,12, 37,37,6,6,6,6,6,6, 6,11,23,6,12,6,6,39, 37,24,10,3,12,3,10,12,
+ 10,11,5,3,8,8,3,3, 5,3,8,3,3,5,5,3, 3,3,3,12,3,3,9,5, 3,3,10,3,9,2,3,12,
+ 5,18,18,11,18,18,3,11, 11,11,11,18,18,11,11,18, 18,11,18,11,18,18,11,18, 11,11,11,11,11,11,11,11,
+ 10,10,10,10,10,18,10,10, 10,18,10,18,18,10,18,10, 10,10,10,18,18,10,10,10, 10,10,18,18,20,20,10,10,
+ // F4xx
+ 3,4,37,37,37,37,39,37, 37,11,10,37,37,11,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,11,17,11,11,10,20,11, 11,10,11,3,10,10,18,20, 37,10,25,17,10,10,10,11, 10,10,10,11,11,17,10,10,
+ 17,18,11,8,11,18,11,11, 11,18,24,11,11,11,37,18, 18,37,11,37,11,11,39,11, 11,37,11,11,17,11,37,11,
+ 17,11,11,11,11,37,39,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,39,3,37,
+ 2,39,6,6,3,24,3,3, 11,37,37,39,11,6,6,6, 37,6,23,6,3,39,6,37, 37,11,12,3,39,12,12,4,
+ 25,5,3,7,11,3,11,11, 8,3,8,3,3,8,3,8, 8,8,3,10,3,3,5,3, 5,5,3,5,3,3,9,8,
+ 18,18,3,3,3,18,3,3, 5,3,3,3,3,3,8,18, 3,5,18,3,3,18,3,18, 3,18,1,3,20,20,20,3,
+ 10,18,17,17,17,17,3,20, 10,17,10,10,17,10,10,18, 10,17,18,10,10,18,17,18, 17,17,17,10,20,20,20,37,
+ // F5xx
+ 3,37,1,4,37,39,39,37, 37,10,10,37,37,10,37,39, 37,39,37,37,37,37,37,37, 39,37,37,37,37,37,37,37,
+ 10,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10, 10,10,10,10,10,20,10,10,
+ 10,10,11,1,3,8,8,1, 10,1,1,1,1,1,37,8, 1,1,1,1,1,1,1,8, 8,1,1,1,17,10,37,39,
+ 17,12,1,12,12,1,12,1, 1,1,1,12,1,1,1,12, 1,3,1,12,1,1,12,1, 1,1,12,1,8,11,3,37,
+ 37,6,6,6,4,10,4,4, 6,6,6,6,6,39,6,6, 6,11,6,6,10,6,6,6, 4,4,4,4,3,3,3,4,
+ 10,11,5,3,3,3,8,8, 5,1,3,3,8,8,5,1, 8,5,3,10,3,3,5,3, 3,3,10,3,3,5,3,3,
+ 3,5,18,5,18,5,18,1, 11,18,11,18,3,11,3,11, 18,11,18,18,11,11,11,18, 11,11,11,11,11,11,18,11,
+ 10,11,18,11,18,11,11,18, 10,25,20,18,18,10,10,10, 10,1,18,18,18,10,18,18, 20,3,18,18,3,18,18,33,
+ // F6xx
+ 3,1,37,37,37,37,39,37, 37,1,1,1,37,1,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 11,10,1,1,1,10,1,17, 3,10,11,5,10,10,10,1, 37,5,1,1,5,10,10,3, 1,1,10,10,1,11,1,1,
+ 5,11,1,1,1,1,1,1, 1,24,1,1,1,1,39,37, 11,37,1,1,1,37,1,37, 1,1,1,37,17,1,39,1,
+ 17,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,8,1,11,5,37,
+ 39,6,6,4,24,24,3,3, 37,6,6,6,37,37,6,37, 6,17,17,6,39,37,39,6, 37,37,3,11,4,3,3,4,
+ 11,11,5,3,3,8,3,3, 3,3,3,3,3,3,3,8, 3,3,3,10,3,3,3,5, 3,5,3,3,3,5,5,5,
+ 3,3,3,3,3,18,11,11, 11,18,11,11,11,11,3,11, 11,11,11,11,11,18,11,3, 11,11,11,11,11,20,11,1,
+ 10,17,10,11,17,10,11,17, 10,17,10,18,17,11,10,20, 13,20,17,10,17,18,1,17, 17,20,17,10,10,20,13,10,
+ // F7xx
+ 3,1,37,37,37,37,39,37, 37,10,18,6,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,7,37,37,37,37,39,
+ 17,17,17,17,11,10,18,17, 11,10,17,3,17,10,18,10, 11,11,11,11,11,11,11,11, 11,11,17,17,10,17,10,17,
+ 8,18,11,8,3,18,8,8, 3,18,11,3,11,11,39,18, 11,37,11,39,37,3,11,39, 11,37,3,11,11,8,17,17,
+ 17,10,37,11,11,10,11,11, 37,8,11,11,11,39,11,20, 11,11,11,11,11,11,11,11, 39,10,11,37,17,17,8,37,
+ 3,6,6,3,3,3,3,3, 37,6,6,39,6,37,6,39, 6,37,11,6,10,6,6,6, 4,37,3,3,10,4,12,4,
+ 17,3,3,3,11,3,26,3, 3,3,8,8,3,3,3,3, 3,3,3,10,8,3,26,3, 10,5,3,3,3,5,3,3,
+ 3,18,3,3,3,18,3,5, 3,18,3,18,18,18,18,18, 18,3,18,18,18,18,3,18, 18,18,18,3,20,20,20,20,
+ 10,18,10,17,17,10,17,17, 10,17,10,10,17,10,18,18, 17,17,10,18,17,18,17,18, 17,18,17,18,10,20,20,10,
+ // F8xx
+ 3,1,37,37,37,37,39,37, 1,17,17,37,37,17,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 17,17,17,17,1,17,16,17, 17,16,17,5,17,17,17,17, 37,5,1,17,1,1,1,1, 1,10,17,17,17,17,17,17,
+ 10,8,3,1,1,1,12,1, 1,1,8,1,1,1,37,37, 1,37,1,39,37,8,1,8, 8,37,1,37,17,17,37,1,
+ 17,12,1,12,1,12,1,1, 1,12,1,1,1,1,1,12, 1,8,1,1,1,1,1,1, 1,1,1,3,37,37,5,37,
+ 39,6,6,39,4,16,3,17, 39,6,37,39,6,37,6,37, 37,11,6,16,16,37,16,6, 37,37,12,3,10,3,12,24,
+ 17,11,5,3,11,8,8,8, 5,1,8,5,8,8,1,8, 8,5,3,10,3,5,3,3, 10,5,5,3,5,5,3,5,
+ 5,5,3,11,24,17,17,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,3, 11,3,11,11,11,11,11,11,
+ 17,17,17,17,17,17,17,17, 10,17,10,17,17,12,18,17, 17,17,10,11,17,17,11,17, 17,17,17,18,10,18,10,37,
+ // F9xx
+ 3,1,37,37,37,37,39,39, 37,12,12,1,37,12,37,39, 37,37,37,37,39,37,37,37, 37,37,37,37,37,37,37,37,
+ 12,17,12,17,11,11,30,12, 12,12,17,5,12,12,12,17, 37,17,17,17,17,17,10,17, 3,17,12,12,12,17,12,12,
+ 8,8,11,8,12,11,24,24, 24,24,11,17,11,11,11,8, 39,37,11,8,11,11,11,39, 11,11,3,39,17,12,24,11,
+ 17,11,12,12,12,11,12,11, 12,11,12,12,12,12,12,11, 12,37,12,12,12,11,12,24, 11,11,12,37,17,24,24,37,
+ 37,24,6,11,11,12,37,24, 24,24,24,37,37,24,24,24, 24,24,24,12,24,24,37,24, 24,24,12,24,24,24,12,24,
+ 12,5,24,3,8,5,5,24, 5,24,1,5,5,5,8,8, 5,5,3,8,3,3,5,8, 10,5,8,5,24,3,9,24,
+ 1,5,24,5,8,5,3,5, 3,1,5,24,17,5,5,5, 8,17,17,5,3,8,8,8, 8,8,8,5,8,8,8,8,
+ 10,17,17,17,17,10,17,17, 10,17,18,17,17,17,17,17, 17,20,17,20,17,7,17,17, 17,17,17,18,10,8,8,10,
+ // FAxx
+ 3,10,1,37,37,37,37,37, 37,1,17,1,37,17,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 17,17,17,17,1,17,17,17, 17,17,17,17,17,17,17,17, 37,17,17,17,17,17,17,17, 1,17,17,17,17,17,1,17,
+ 8,8,1,8,8,1,1,8, 8,1,1,24,1,1,37,37, 8,39,1,37,1,1,39,39, 11,1,1,37,17,11,39,17,
+ 17,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,37, 1,37,1,1,17,11,11,37,
+ 39,6,6,6,12,17,37,12, 37,6,37,37,6,6,6,6, 6,6,6,6,12,6,6,24, 6,37,12,12,24,12,12,4,
+ 17,5,3,5,5,6,8,8, 8,5,5,11,8,5,1,5, 5,3,3,3,5,3,3,5, 10,5,3,9,5,3,5,11,
+ 5,18,11,11,18,18,3,11, 11,18,11,11,11,11,11,11, 11,11,11,11,11,11,11,18, 11,11,11,11,11,11,11,11,
+ 17,18,17,11,17,17,11,17, 12,17,17,17,17,18,17,18, 17,18,17,17,17,18,17,17, 17,17,17,1,5,5,5,10,
+ // FBxx
+ 3,37,37,37,37,37,37,14, 37,10,10,10,39,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,3,37,37,37,37,
+ 10,10,10,10,11,10,10,10, 10,10,10,10,10,10,10,10, 39,10,10,10,10,10,10,11, 10,10,10,10,10,10,10,10,
+ 10,8,11,11,8,18,8,3, 11,18,8,11,11,10,39,37, 11,39,37,37,37,37,37,37, 11,37,11,37,10,10,3,10,
+ 10,12,11,16,16,12,12,16, 12,8,12,16,16,16,16,11, 16,3,16,16,11,37,12,37, 10,39,16,37,39,37,37,37,
+ 37,6,6,6,37,10,11,39, 37,6,37,39,6,6,6,6, 3,37,37,6,6,39,6,6, 6,37,37,11,3,10,37,3,
+ 10,5,10,3,8,8,5,8, 8,3,5,5,5,16,11,8, 3,8,8,5,5,3,3,3, 5,3,5,3,5,3,5,5,
+ 5,18,5,3,3,18,3,5, 3,18,3,18,18,18,18,18, 18,3,18,3,18,18,3,18, 3,18,7,5,5,5,5,5,
+ 5,18,10,10,10,10,10,10, 16,10,10,10,10,10,18,18, 10,10,10,5,18,10,10,10, 10,5,5,11,11,5,5,10,
+ // FCxx
+ 3,10,37,37,37,37,37,37, 1,11,10,37,4,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,11,11,10,10,11, 10,10,10,10,10,10,10,10, 37,10,10,10,10,10,10,10, 10,10,10,10,10,11,10,10,
+ 10,33,11,11,11,33,11,11, 11,33,11,11,11,11,11,33, 11,37,11,11,11,33,11,11, 11,11,11,39,10,10,37,11,
+ 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11, 11,11,11,37,11,37,39,39,
+ 37,6,6,6,37,11,11,11, 6,6,6,6,39,6,6,6, 6,6,11,6,10,6,6,6, 39,11,3,11,11,4,39,4,
+ 10,5,5,5,5,5,8,5, 8,8,8,8,5,11,8,5, 5,5,3,5,5,33,3,5, 5,5,5,5,3,5,5,5,
+ 5,5,18,5,18,3,18,18, 18,3,18,18,18,18,18,5, 18,3,18,18,18,5,33,18, 5,3,18,18,5,1,5,1,
+ 10,10,20,10,10,10,18,11, 10,18,10,18,20,10,10,10, 13,10,10,20,18,10,10,20, 10,10,18,18,11,1,13,10,
+ // FDxx
+ 3,37,1,37,37,37,37,37, 37,13,13,37,37,13,4,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 13,13,13,13,13,13,13,13, 13,13,13,13,13,13,13,13, 11,11,11,11,11,11,11,11, 11,11,13,13,13,13,13,13,
+ 13,13,13,13,13,13,13,13, 13,13,1,13,13,13,13,37, 13,37,13,13,13,13,39,37, 1,13,13,13,13,13,37,13,
+ 13,33,1,13,13,33,13,13, 1,33,1,13,13,13,13,13, 13,13,13,13,13,13,1,13, 13,13,13,37,13,13,3,37,
+ 39,37,37,37,37,22,3,3, 3,37,37,37,39,37,39,39, 39,37,22,12,22,11,37,37, 3,37,12,3,3,12,12,4,
+ 33,33,8,3,8,8,5,5, 5,8,8,8,8,5,8,1, 3,5,3,5,13,3,3,3, 3,9,5,3,3,3,9,3,
+ 5,3,3,3,3,18,3,11, 3,18,3,3,5,3,3,3, 13,3,3,3,3,3,3,3, 5,5,11,5,11,1,13,11,
+ 10,18,10,11,10,18,11,13, 12,18,10,10,10,10,18,10, 13,10,10,20,10,10,1,20, 18,20,5,5,5,13,13,10,
+ // FExx
+ 3,39,39,39,37,37,39,39, 37,13,10,37,39,11,39,39, 39,39,39,39,39,37,39,37, 39,39,37,39,37,39,37,39,
+ 10,10,10,1,1,10,13,13, 11,10,11,13,10,10,10,13, 11,11,11,11,11,11,11,11, 11,11,13,13,13,13,13,10,
+ 13,13,1,11,13,13,13,13, 11,13,11,13,13,11,37,37, 11,37,37,11,13,13,39,37, 37,13,11,11,13,11,3,13,
+ 13,13,13,13,16,13,13,13, 13,13,1,13,13,13,16,16, 16,13,1,16,13,13,13,11, 11,13,13,11,13,11,3,37,
+ 3,11,37,37,4,4,4,4, 37,37,39,39,37,11,37,37, 11,37,22,16,39,11,39,37, 39,4,3,37,4,11,37,3,
+ 3,11,1,3,8,8,8,8, 8,8,8,11,8,13,1,1, 8,1,3,18,3,3,3,8, 3,3,10,3,3,3,3,11,
+ 5,18,11,11,3,18,3,11, 11,18,11,11,11,11,11,11, 11,11,11,11,18,18,18,3, 11,11,11,11,11,11,11,11,
+ 10,18,10,10,10,18,10,10, 10,18,10,18,10,20,18,18, 10,10,10,20,18,18,13,10, 18,10,1,18,13,13,11,37,
+ // FFxx
+ 3,37,39,39,39,37,37,39, 37,37,10,37,37,10,37,37, 37,37,37,37,37,37,37,37, 37,37,37,37,37,37,37,37,
+ 10,10,10,37,37,10,10,10, 10,10,10,10,10,10,10,10, 37,36,36,37,36,36,39,39, 39,39,10,10,10,1,10,10,
+ 10,39,39,39,39,37,39,39, 1,39,39,36,39,39,37,39, 39,39,37,36,39,39,36,37, 36,39,36,10,10,10,37,10,
+ 10,2,2,2,36,2,2,2, 2,2,2,36,2,2,36,36, 36,2,2,2,2,2,36,2, 2,2,36,39,10,39,39,39,
+ 36,36,39,37,39,10,39,39, 4,37,36,39,36,36,36,36, 36,37,39,39,10,11,39,39, 37,39,39,39,39,37,39,33,
+ 10,39,10,33,39,39,36,33, 36,1,36,39,33,10,36,36, 39,39,39,18,39,39,39,39, 39,39,10,11,39,39,39,1,
+ 10,1,1,11,39,39,39,39, 39,39,39,39,39,39,39,39, 39,39,39,39,39,39,39,39, 3,3,3,3,11,3,3,1,
+ 10,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10, 10,10,10,10,10,10,10,10, 10,10,39,39,10,37,10,10,
+
+}; // End kMostLikelyEncoding
+
+// End of generated tables
+#endif // COMPACT_ENC_DET_COMPACT_ENC_DET_GENERATED_TABLES_H_
diff --git a/contrib/google-ced/compact_enc_det_generated_tables2.h b/contrib/google-ced/compact_enc_det_generated_tables2.h
new file mode 100644
index 0000000..4ecf966
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det_generated_tables2.h
@@ -0,0 +1,856 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/basictypes.h"
+
+static const uint8 ced_hires_0[1024] = {
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 0,135,134,135,89,111,136,129, 81,121,130,40,64,74,78,13, 90,87,103,56,51,37,63,44, 0,97,60,55,143,97,70,100,
+ 29,126,116,149,131,156,108,153, 147,119,135,111,133,135,115,0, 2,13,20,4,12,5,17,5, 4,0,58,50,2,30,80,64,
+ 35,0,0,0,0,22,0,9, 14,12,4,6,7,8,16,3, 147,160,163,141,144,139,146,127, 131,124,23,17,0,11,16,6,
+ 0,68,108,52,129,66,109,65, 113,58,116,124,126,117,88,114, 85,122,102,126,116,122,96,131, 102,126,107,117,49,113,80,128,
+ 0,85,119,107,127,75,102,105, 118,101,108,118,101,117,98,128, 125,114,115,125,115,121,116,121, 125,130,122,119,64,102,78,124,
+ 93,87,64,93,105,92,76,80, 113,82,75,88,89,68,88,97, 87,102,145,90,90,130,106,116, 129,61,81,75,47,69,74,64,
+ 92,98,81,95,81,161,75,81, 101,95,93,75,100,87,108,91, 113,110,118,114,105,105,123,94, 89,92,104,90,62,85,81,84,
+
+ 47,141,155,88,85,108,121,141, 94,73,82,81,163,125,111,89, 106,133,156,95,80,96,88,61, 65,47,74,55,33,54,59,19,
+ 128,129,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,131,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,135,128,128,128, 128,128,161,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 141,115,118,118,118,128,115,127, 136,136,122,124,125,126,128,121, 147,128,128,128,128,128,128,123, 128,121,128,128,107,154,128,124,
+ 82,141,145,136,128,144,115,123, 124,117,107,104,103,99,107,90, 92,98,102,82,88,134,161,126, 115,104,114,106,73,91,111,84,
+ 128,93,94,94,94,116,91,103, 108,106,98,100,101,102,110,97, 137,133,120,132,146,114,136,131, 144,145,155,131,107,151,144,133,
+ 128,103,106,106,106,128,103,115, 120,118,110,112,113,114,122,109, 157,138,128,117,136,123,136,128, 137,123,152,126,104,136,137,134,
+
+ 32,94,46,28,98,102,130,97, 59,91,81,73,26,125,102,48, 100,66,105,57,87,121,71,120, 86,64,64,83,68,105,91,70,
+ 32,130,98,103,64,30,123,122, 97,123,58,127,70,74,55,59, 53,99,95,99,105,52,48,73, 52,68,138,89,29,104,97,142,
+ 20,99,54,114,87,86,108,55, 46,27,42,99,130,118,83,95, 138,115,69,67,96,95,125,86, 82,121,92,145,131,142,115,134,
+ 27,53,57,104,139,102,124,124, 135,81,79,148,118,90,71,25, 151,83,146,71,74,129,109,35, 84,115,121,77,48,24,58,63,
+ 29,55,85,51,70,123,21,95, 116,128,90,107,124,74,98,80, 132,125,112,114,118,109,167,103, 57,127,95,84,13,65,84,109,
+ 27,137,123,69,151,81,84,95, 124,80,115,38,140,164,121,122, 86,116,83,102,117,110,111,92, 72,94,114,144,62,90,84,109,
+ 31,128,63,37,14,138,126,79, 132,63,54,95,60,132,89,132, 25,77,134,94,93,154,147,125, 104,98,113,119,70,130,98,111,
+ 24,78,61,30,100,41,65,115, 80,28,93,97,86,38,74,112, 85,105,135,113,132,45,65,30, 126,91,113,114,73,89,65,130,
+
+ 21,150,114,47,106,125,121,37, 79,158,90,155,73,92,94,107, 87,137,97,135,91,139,152,102, 98,85,126,121,80,140,42,25,
+ 23,47,75,42,64,150,67,93, 100,140,70,62,117,140,54,122, 100,98,95,105,88,100,132,41, 81,137,76,144,60,131,128,66,
+ 23,59,107,144,98,44,20,119, 70,80,58,85,119,93,113,59, 83,28,117,67,135,65,77,60, 130,114,138,111,31,109,92,87,
+ 18,93,79,109,57,128,94,69, 135,54,72,38,53,55,94,56, 133,98,153,147,82,58,124,55, 48,50,153,128,24,54,80,89,
+ 29,130,95,117,49,47,46,91, 164,58,99,148,64,113,86,110, 144,134,49,60,102,62,103,107, 69,89,149,71,113,31,107,71,
+ 25,90,106,113,119,82,75,91, 111,128,139,79,130,101,90,31, 97,103,87,84,53,138,43,87, 155,57,75,150,93,128,120,43,
+ 24,118,82,52,79,56,143,92, 47,103,22,46,70,106,155,129, 93,104,125,42,96,124,137,93, 73,43,134,98,100,111,139,56,
+ 22,65,103,76,38,122,121,155, 111,147,47,76,96,103,153,78, 71,103,137,104,150,89,139,159, 77,120,71,98,98,125,78,143,
+};
+
+static const uint8 ced_hires_1[1024] = {
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 62,124,71,116,117,77,46,71, 78,98,127,132,69,67,72,79, 34,37,88,95,81,88,135,138, 116,123,132,136,97,128,45,101,
+ 68,71,36,0,24,19,3,1, 0,9,60,40,64,146,70,113, 20,16,20,26,13,20,9,12, 14,21,15,19,55,50,99,61,
+ 17,127,133,119,128,134,113,109, 106,124,100,107,119,131,111,122, 138,113,136,154,132,125,120,114, 115,119,101,25,19,31,19,19,
+ 122,112,48,123,115,89,133,132, 124,114,123,130,73,103,139,130, 106,90,103,114,71,97,92,74, 115,112,59,105,83,90,140,121,
+ 118,113,51,133,112,43,123,112, 133,130,102,109,71,110,89,102, 128,125,104,139,115,139,129,134, 112,120,123,117,111,132,129,118,
+ 73,151,165,130,127,146,100,107, 108,121,103,114,156,112,107,111, 125,129,150,135,107,155,118,116, 168,85,79,83,77,89,77,77,
+ 98,101,81,63,71,82,66,64, 60,72,57,62,72,70,60,67, 75,153,138,154,129,161,146,125, 125,139,151,134,139,151,135,146,
+
+ 59,21,36,18,26,37,21,19, 15,27,12,17,27,25,15,22, 30,34,38,44,31,38,27,30, 32,39,33,37,31,43,31,31,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,127,128,124,144,128,127,126, 121,128,118,123,128,128,121,128, 128,128,128,128,128,128,128,128, 128,128,128,128,150,128,128,128,
+ 90,112,109,86,83,124,57,54, 84,70,53,84,62,78,53,75, 124,134,116,95,118,85,129,65, 67,74,68,72,66,78,66,66,
+ 141,134,118,107,135,127,130,134, 118,109,94,99,109,107,97,104, 112,116,120,126,113,120,109,112, 114,121,115,119,113,125,113,113,
+ 161,134,128,112,120,139,115,128, 122,121,106,111,121,119,109,116, 124,128,128,128,125,128,121,124, 126,128,127,128,125,128,125,125,
+
+ 76,29,158,87,47,122,133,89, 63,101,139,116,147,111,79,122, 60,112,93,68,42,145,73,100, 89,123,46,122,124,112,99,90,
+ 118,56,99,61,145,53,131,144, 66,122,110,97,113,82,19,27, 33,134,120,84,79,124,78,108, 138,87,26,133,114,84,47,165,
+ 62,148,104,137,139,97,125,86, 134,47,127,93,141,85,119,111, 126,56,42,26,102,89,122,48, 59,116,110,106,46,115,64,113,
+ 107,81,49,29,28,64,130,22, 111,53,142,89,89,127,131,86, 122,122,37,143,66,96,82,105, 155,96,154,124,63,121,90,60,
+ 115,107,42,30,134,130,119,108, 66,145,122,85,105,19,88,92, 36,134,34,101,97,112,152,73, 150,89,116,144,71,130,131,103,
+ 88,123,77,30,132,53,98,51, 116,61,95,57,84,109,31,74, 20,98,135,114,78,114,95,123, 45,136,116,111,130,94,120,99,
+ 114,121,64,107,94,104,29,77, 152,133,124,119,135,98,0,78, 118,115,88,102,57,61,51,89, 120,51,115,109,64,111,61,66,
+ 110,135,106,110,98,74,86,137, 124,103,134,80,86,43,76,131, 134,105,91,84,73,34,71,133, 60,111,117,53,90,133,106,61,
+
+ 147,38,145,28,137,149,110,112, 52,83,99,128,56,129,105,74, 93,60,45,62,73,86,30,82, 80,44,19,97,85,87,108,103,
+ 99,114,94,104,97,104,140,51, 98,62,119,74,85,141,51,23, 71,58,119,110,151,65,116,74, 130,81,86,90,7,99,38,116,
+ 61,59,140,66,91,133,123,152, 80,82,98,67,112,110,110,89, 121,132,120,62,43,74,45,110, 88,146,113,101,134,135,104,131,
+ 110,47,96,67,123,147,69,70, 131,106,98,109,71,118,116,128, 92,100,149,53,144,117,99,136, 127,128,3,43,114,100,85,117,
+ 92,134,156,8,88,62,89,98, 6,19,46,96,145,108,88,69, 78,94,152,96,146,108,133,66, 75,102,101,10,79,44,57,61,
+ 102,96,123,91,109,149,84,63, 39,134,53,110,111,64,97,103, 155,133,105,105,34,125,93,56, 28,84,65,95,103,58,41,83,
+ 48,25,116,120,91,104,124,84, 111,119,105,64,63,97,116,114, 128,103,136,74,59,33,47,93, 54,80,130,39,139,130,119,83,
+ 140,68,94,85,77,79,118,109, 120,106,114,103,115,165,94,79, 60,52,84,77,82,78,49,74, 88,49,48,76,102,99,138,71,
+};
+
+static const uint8 ced_hires_2[1024] = {
+ 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 44,106,18,108,117,63,65,73, 40,57,43,81,77,63,68,90, 54,16,20,95,85,131,112,100, 58,144,140,135,114,96,110,128,
+ 111,93,68,129,135,62,95,32, 82,42,46,10,8,23,10,16, 26,21,70,26,93,84,168,105, 79,71,29,48,15,26,68,128,
+ 28,126,105,123,109,126,102,109, 105,122,72,104,108,132,113,126, 118,82,122,110,129,125,114,134, 113,133,103,54,21,32,21,128,
+ 113,122,134,118,123,77,97,101, 121,135,128,138,125,120,35,120, 31,24,148,129,0,0,0,0, 0,0,0,5,0,0,0,128,
+ 137,137,112,130,105,127,110,121, 86,140,134,139,128,145,17,121, 19,27,60,150,106,62,95,0, 0,0,0,9,0,0,0,128,
+ 86,81,78,87,82,79,82,75, 75,78,69,74,72,87,74,80, 90,85,84,72,92,96,88,82, 98,102,93,112,79,90,79,128,
+ 162,135,144,153,151,137,127,118, 113,125,105,106,91,139,129,109, 124,140,83,71,91,95,87,81, 97,101,92,111,78,89,78,128,
+
+ 40,35,34,43,36,33,36,29, 29,32,23,28,26,41,28,34, 44,39,38,26,46,50,42,36, 52,56,47,66,33,44,33,128,
+ 128,128,128,131,128,128,130,128, 147,152,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,138,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,143,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 166,128,128,130,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,146,128,128, 128,128,128,128,148,128,128,128,
+ 139,134,133,75,121,105,114,103, 64,68,129,100,62,76,63,69, 79,74,73,114,137,85,77,71, 174,140,82,101,68,79,68,128,
+ 122,117,114,122,118,115,118,113, 112,117,105,110,108,123,110,116, 126,121,120,108,128,128,124,118, 128,128,128,128,115,126,115,128,
+ 128,128,126,154,128,127,132,123, 137,126,117,122,120,128,122,128, 128,128,128,120,128,128,128,128, 128,128,128,128,127,128,127,128,
+
+ 67,115,51,135,110,136,135,45, 132,133,58,84,156,90,77,88, 106,108,86,29,51,132,78,133, 122,67,145,151,77,36,57,128,
+ 142,57,81,110,113,64,18,123, 101,136,98,45,102,79,83,102, 85,86,54,106,94,38,117,74, 115,80,89,151,109,98,137,128,
+ 57,126,70,74,47,54,112,60, 146,46,72,36,108,101,48,40, 135,151,135,128,78,111,70,101, 107,100,25,118,65,57,106,128,
+ 67,74,25,123,123,65,42,95, 135,62,82,86,45,70,104,59, 92,30,63,55,143,71,26,83, 97,107,87,111,25,86,45,128,
+ 65,102,90,116,124,16,45,51, 110,129,118,128,73,120,108,139, 139,102,109,110,120,134,85,66, 104,100,96,129,134,94,84,128,
+ 68,135,60,114,78,126,135,56, 22,121,35,117,101,118,119,120, 109,82,99,96,29,91,118,96, 63,104,18,139,47,66,138,128,
+ 78,148,152,94,129,120,107,96, 141,71,52,46,111,50,97,36, 101,133,87,23,91,150,92,38, 110,103,97,84,79,68,85,128,
+ 118,16,110,124,126,74,99,94, 132,79,88,135,103,84,161,153, 35,41,125,78,122,107,74,90, 93,122,134,117,97,91,38,128,
+
+ 56,121,75,93,68,139,117,78, 23,63,43,75,128,101,116,59, 57,65,121,13,48,108,74,121, 144,120,128,78,95,135,119,128,
+ 141,128,143,68,91,46,137,151, 29,50,32,92,55,73,81,107, 139,145,72,67,54,129,86,35, 105,40,40,99,98,61,142,128,
+ 136,98,109,69,113,126,85,56, 56,103,117,104,17,70,148,128, 55,61,117,55,58,53,121,155, 86,126,75,104,27,105,100,128,
+ 117,110,66,101,135,87,114,88, 69,93,78,106,98,98,117,132, 43,125,54,39,97,119,157,98, 79,131,147,134,59,130,143,128,
+ 60,89,40,119,87,99,73,123, 138,107,135,72,78,97,83,134, 85,110,119,93,43,141,77,98, 119,124,28,104,55,131,118,128,
+ 130,92,71,117,97,53,50,134, 114,136,136,98,51,71,62,115, 96,155,36,1,104,124,59,137, 117,93,26,47,129,107,82,128,
+ 25,44,71,115,56,152,109,40, 117,70,60,119,146,80,83,130, 162,18,126,52,102,137,91,39, 105,113,91,105,37,46,95,128,
+ 68,104,78,51,112,141,87,82, 79,68,59,77,84,49,21,91, 81,68,66,30,149,62,30,70, 81,114,25,107,6,89,62,128,
+};
+
+static const uint8 ced_hires_3[1024] = {
+ 145,108,107,112,108,128,119,120, 123,130,108,110,133,138,120,117, 120,120,128,121,121,116,128,119, 128,119,122,114,121,119,123,124,
+ 77,160,155,161,143,37,29,56, 47,35,127,103,75,130,115,110, 149,149,88,80,89,85,145,135, 102,93,107,99,91,89,140,135,
+ 123,90,70,105,58,80,64,65, 54,64,75,114,51,62,65,58, 91,121,130,110,109,105,111,102, 107,100,93,91,94,94,90,92,
+ 173,130,70,57,104,116,65,65, 135,145,70,76,167,129,117,91, 127,129,135,120,115,112,120,109, 117,108,170,131,84,86,91,132,
+ 62,79,115,64,137,72,131,76, 108,65,120,128,125,126,92,124, 97,123,106,137,119,127,102,145, 120,143,106,114,81,121,87,140,
+ 32,95,125,118,134,80,123,115, 112,107,111,122,99,125,101,137, 136,114,118,135,117,125,121,134, 142,145,120,115,95,109,84,135,
+ 149,90,62,97,105,90,90,83, 100,81,70,84,80,69,84,99, 91,95,141,93,85,127,104,122, 139,91,91,84,114,85,87,82,
+ 128,108,87,106,88,166,96,91, 95,101,96,78,98,95,111,100, 124,110,121,124,107,109,128,107, 106,107,102,86,93,92,87,95,
+
+ 128,128,121,112,126,121,150,106, 125,124,120,96,136,130,135,110, 131,109,148,108,124,113,113,94, 107,139,145,102,114,94,96,95,
+ 92,96,25,73,155,144,172,124, 82,104,144,100,98,115,154,121, 106,60,79,85,106,68,75,81, 150,81,74,87,105,46,52,81,
+ 128,115,75,109,150,190,102,84, 108,126,118,117,98,102,82,70, 101,160,145,120,104,121,112,110, 124,103,124,102,101,99,97,98,
+ 128,121,92,125,93,113,104,105, 94,104,93,99,93,102,105,98, 129,127,166,129,127,135,139,136, 142,143,142,131,131,127,123,126,
+ 128,115,81,140,82,102,107,94, 83,93,83,84,80,91,94,87, 131,132,152,131,139,139,150,144, 142,131,134,133,139,135,137,136,
+ 128,144,99,131,100,120,124,112, 101,111,100,102,98,109,112,105, 126,133,137,137,132,146,148,131, 144,153,139,142,147,123,115,120,
+ 128,132,96,135,97,117,108,109, 98,108,97,99,95,106,109,108, 133,126,134,126,133,142,142,127, 140,133,144,121,134,129,135,117,
+ 128,137,98,138,99,119,110,111, 100,110,99,102,97,112,111,111, 133,122,147,135,132,142,126,127, 144,133,138,140,129,124,135,131,
+
+ 66,120,131,103,77,97,82,96, 49,122,50,79,108,111,157,61, 103,104,175,93,150,114,129,97, 163,62,107,74,85,83,86,65,
+ 60,104,55,155,93,106,145,111, 154,124,81,79,114,119,104,98, 78,156,105,125,122,58,126,51, 158,67,68,140,110,97,170,83,
+ 57,133,137,102,68,149,108,61, 123,146,85,82,59,65,109,45, 93,71,73,77,93,78,111,93, 84,127,90,164,148,138,91,161,
+ 61,151,94,146,151,131,111,141, 77,115,111,98,134,96,86,120, 72,120,108,84,108,154,91,107, 59,119,71,81,97,99,110,111,
+ 61,123,117,54,44,130,154,70, 125,120,23,146,110,91,116,88, 138,83,134,93,150,98,76,53, 90,99,140,60,102,90,84,118,
+ 48,50,50,114,97,154,78,31, 88,101,72,126,31,104,99,105, 108,142,96,135,96,117,112,62, 90,103,116,59,144,151,100,63,
+ 60,111,72,74,81,149,104,72, 154,140,90,149,106,117,107,164, 94,68,40,103,113,75,95,119, 78,119,86,84,153,133,135,61,
+ 57,55,162,113,44,109,106,106, 156,45,51,86,91,122,91,49, 59,115,122,101,140,123,148,88, 128,112,113,99,106,160,94,146,
+
+ 60,103,65,134,55,108,82,107, 109,74,62,66,51,60,154,112, 107,132,118,98,153,90,145,96, 127,103,138,131,32,128,125,71,
+ 83,37,62,74,156,128,140,85, 67,149,49,158,98,78,96,58, 74,105,154,86,85,104,70,106, 71,139,144,112,70,87,62,57,
+ 61,52,121,154,38,91,135,69, 50,69,82,123,119,105,113,111, 94,87,99,91,73,100,97,82, 110,100,136,69,126,142,55,51,
+ 74,69,108,46,132,141,110,157, 133,69,143,44,100,133,144,154, 148,71,78,118,106,110,142,141, 77,145,112,124,118,87,69,36,
+ 55,94,65,106,116,57,117,82, 111,82,114,127,58,134,115,141, 151,122,124,60,133,57,158,91, 139,85,137,75,157,84,128,56,
+ 59,132,91,110,40,123,69,72, 156,84,95,148,88,151,121,95, 59,132,143,80,88,128,94,95, 76,120,121,149,98,86,94,91,
+ 63,124,105,95,61,107,107,129, 65,159,114,147,62,170,118,134, 140,104,135,145,113,123,129,67, 107,117,135,114,64,66,85,147,
+ 65,101,84,63,70,103,113,106, 138,80,159,36,114,114,86,133, 93,102,95,90,159,137,73,65, 75,126,75,50,154,87,101,119,
+};
+
+static const uint8 ced_hires_4[1024] = {
+ 125,128,119,118,134,121,118,117, 121,123,122,116,128,124,115,120, 123,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,133,
+ 89,125,49,76,12,21,28,18, 21,36,39,85,85,24,34,27, 45,91,39,57,98,41,58,61, 34,65,57,69,88,91,63,51,
+ 88,107,83,89,76,130,128,123, 123,120,114,104,121,106,102,107, 111,122,121,124,104,124,105,123, 75,193,173,183,166,161,151,140,
+ 57,123,96,99,90,98,90,87, 92,100,90,86,102,110,91,100, 111,104,117,127,115,104,104,107, 93,107,76,126,75,129,66,99,
+ 132,140,43,135,109,85,142,143, 142,123,145,142,88,114,151,140, 111,113,116,119,86,108,108,99, 125,132,66,121,102,103,162,139,
+ 127,140,45,144,105,38,131,121, 150,137,124,119,85,120,100,111, 132,147,116,143,129,149,144,158, 121,139,129,132,129,144,150,135,
+ 118,171,152,134,113,134,101,109, 118,121,117,117,163,115,111,113, 122,144,155,132,114,158,126,133, 170,108,78,105,109,94,91,87,
+ 107,128,102,85,100,86,92,85, 99,107,95,79,99,102,94,112, 103,175,150,158,143,171,161,149, 134,158,157,149,157,163,156,163,
+
+ 109,134,92,99,79,88,99,96, 113,116,109,96,117,109,94,107, 109,117,140,123,112,109,118,129, 82,99,110,93,90,120,93,105,
+ 174,130,49,65,96,52,55,98, 100,54,66,43,94,53,51,88, 117,70,89,96,67,74,69,96, 111,61,42,98,52,58,55,82,
+ 104,127,95,100,95,101,95,103, 111,102,126,112,114,120,101,120, 111,128,128,120,116,125,119,128, 87,104,85,98,95,101,98,94,
+ 127,145,120,137,126,116,135,123, 128,134,149,145,134,137,137,135, 138,172,148,150,144,142,146,155, 116,128,113,126,123,128,126,122,
+ 127,149,116,132,127,126,132,129, 142,132,146,137,136,137,141,133, 138,152,166,158,154,155,150,158, 104,121,102,120,112,119,115,111,
+ 147,147,113,127,121,115,115,109, 131,129,132,124,135,125,120,129, 126,150,153,163,145,131,145,157, 122,128,120,128,128,128,128,128,
+ 132,146,128,118,123,118,136,128, 140,142,131,133,128,133,123,134, 135,144,155,144,129,141,146,152, 119,128,117,129,127,128,128,126,
+ 136,161,124,119,112,127,127,117, 125,135,136,116,159,139,127,131, 135,144,151,151,142,140,146,158, 121,128,119,128,128,128,128,128,
+
+ 56,117,131,52,109,89,49,75, 81,137,58,134,87,143,104,64, 67,158,68,113,116,101,108,155, 107,161,107,92,127,123,72,117,
+ 86,78,40,98,54,51,93,94, 146,77,132,88,105,47,86,121, 62,41,132,68,70,140,41,104, 135,103,110,99,123,62,88,138,
+ 101,108,91,108,121,139,137,42, 81,139,135,123,107,124,135,85, 101,65,99,90,94,85,106,115, 139,139,64,101,103,134,95,143,
+ 70,126,127,75,75,101,140,147, 87,164,107,104,165,99,81,128, 127,71,73,92,129,93,149,99, 116,117,63,116,99,109,92,116,
+ 76,80,61,56,75,111,73,103, 101,107,139,145,119,80,146,97, 80,85,92,153,99,90,106,97, 63,145,46,83,77,87,100,104,
+ 146,115,124,137,168,42,104,127, 143,56,58,87,91,127,81,104, 84,80,65,78,61,37,98,132, 153,100,149,107,103,115,86,77,
+ 130,158,89,75,83,106,85,85, 154,103,67,119,146,96,127,133, 57,111,108,142,164,78,117,83, 96,129,56,73,114,66,45,67,
+ 144,99,131,129,98,141,102,131, 94,130,123,75,85,40,91,109, 73,161,102,81,53,102,161,124, 69,88,72,127,111,144,48,90,
+
+ 63,71,74,140,135,125,89,112, 80,139,117,97,84,70,104,105, 134,91,113,99,91,128,126,100, 80,119,115,145,82,39,81,166,
+ 114,92,92,84,94,122,49,104, 105,137,124,110,127,85,72,101, 21,83,117,52,78,79,102,69, 159,137,104,143,160,134,95,104,
+ 121,128,79,153,100,154,100,103, 107,108,78,130,91,172,136,158, 103,65,53,145,51,100,86,107, 108,103,136,103,149,108,105,77,
+ 79,64,42,108,78,77,129,65, 55,77,109,88,67,93,68,52, 53,94,114,121,109,121,106,60, 146,113,83,116,96,136,58,100,
+ 84,132,50,139,86,86,148,141, 107,91,140,68,130,131,107,42, 102,135,166,160,54,62,91,126, 79,136,52,161,124,123,100,71,
+ 70,99,52,78,44,116,54,121, 67,112,82,59,164,83,92,133, 108,81,120,156,91,84,129,131, 107,110,139,91,122,128,82,92,
+ 118,77,72,119,66,133,131,40, 125,123,54,65,72,159,54,65, 84,86,68,142,68,101,155,79, 102,155,56,134,117,163,131,155,
+ 66,82,99,78,48,100,154,96, 81,163,89,132,131,155,126,110, 54,98,82,94,51,152,112,120, 138,77,149,127,74,99,109,80,
+};
+
+static const uint8 ced_hires_5[1024] = {
+ 128,127,124,128,128,147,138,139, 162,124,125,126,127,127,152,184, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+ 59,56,52,84,74,62,105,50, 75,46,54,59,89,56,113,130, 119,132,110,115,136,127,123,76, 85,121,121,94,112,104,106,128,
+ 126,117,109,74,77,123,124,119, 122,105,96,86,86,78,87,79, 94,169,180,158,166,149,129,116, 113,117,109,100,100,85,105,128,
+ 59,98,77,92,84,100,75,88, 93,94,54,78,89,98,98,106, 103,65,103,95,118,105,93,113, 76,102,66,54,135,52,60,128,
+ 114,126,138,119,130,83,102,112, 141,139,142,144,138,118,52,132, 48,39,161,146,26,1,0,0, 0,6,0,43,34,8,25,128,
+ 137,140,115,130,111,132,114,131, 105,143,147,144,140,142,33,132, 35,41,72,166,126,73,105,0, 0,0,0,53,3,50,23,128,
+ 112,128,104,79,120,147,178,100, 105,85,90,74,80,77,120,87, 130,112,180,83,120,100,91,85, 85,95,84,102,99,90,92,128,
+ 162,138,147,153,157,142,131,128, 132,128,118,111,103,136,145,120, 140,154,95,87,111,106,97,91, 91,101,86,97,105,96,98,128,
+
+ 96,79,91,123,95,99,109,128, 124,80,77,74,81,79,85,86, 101,94,91,83,107,102,93,87, 87,97,91,93,101,92,94,128,
+ 130,76,66,68,64,65,77,111, 81,61,51,63,45,76,57,136, 63,56,53,46,77,64,62,49, 49,59,123,97,63,54,56,128,
+ 86,86,81,86,91,131,96,85, 96,81,82,79,84,117,90,91, 106,99,96,88,112,107,98,92, 95,102,87,98,113,97,99,128,
+ 114,112,109,114,116,119,125,113, 127,111,110,121,112,112,118,119, 128,127,124,116,128,128,126,120, 120,128,115,126,140,125,127,128,
+ 103,101,98,103,105,118,105,102, 113,98,99,96,101,101,107,108, 147,116,113,105,128,124,115,109, 109,119,104,115,136,114,116,128,
+ 121,119,116,121,123,142,128,128, 128,116,117,114,119,119,125,126, 128,128,128,123,128,128,128,127, 127,128,122,128,128,128,128,128,
+ 118,116,113,118,120,127,150,122, 139,118,114,111,116,116,122,123, 128,128,128,120,128,128,128,124, 124,128,119,128,128,128,128,128,
+ 127,118,115,120,139,136,123,124, 134,127,116,113,118,118,124,125, 128,128,128,122,128,128,128,126, 126,128,121,128,128,128,128,128,
+
+ 144,113,79,134,119,150,168,110, 109,123,87,135,164,70,123,153, 60,128,94,93,119,96,90,82, 80,91,52,120,157,65,95,128,
+ 152,77,95,138,152,63,122,108, 72,111,161,80,53,163,71,71, 154,47,85,88,86,124,118,33, 115,126,72,98,91,95,82,128,
+ 128,146,140,137,59,123,98,75, 133,154,41,39,133,67,133,64, 115,98,30,37,84,76,58,26, 78,73,153,84,82,120,63,128,
+ 106,93,89,71,142,126,122,109, 127,113,101,74,47,71,102,125, 99,87,54,87,115,144,165,81, 99,49,47,81,86,139,114,128,
+ 95,47,91,59,105,149,149,123, 89,57,69,124,88,133,119,156, 141,64,149,165,107,86,69,108, 135,57,146,122,119,128,82,128,
+ 44,75,63,157,128,38,92,158, 41,77,129,36,81,96,88,54, 44,93,58,46,115,99,80,131, 96,65,76,87,70,72,83,128,
+ 165,111,57,90,104,46,83,56, 72,75,68,75,96,118,139,55, 81,118,70,56,99,104,83,106, 143,142,105,139,81,83,155,128,
+ 86,127,119,93,103,124,116,150, 118,69,88,93,98,72,109,119, 122,130,119,64,126,86,111,57, 87,131,74,125,103,69,156,128,
+
+ 98,68,93,126,62,132,161,123, 149,72,93,84,65,65,112,144, 101,154,60,90,129,83,167,144, 140,140,113,105,164,79,61,128,
+ 112,113,139,157,96,91,153,68, 106,113,83,60,100,103,74,71, 111,102,68,110,41,73,115,67, 93,107,158,143,83,149,120,128,
+ 95,117,116,117,99,81,87,114, 115,118,121,91,142,90,110,101, 82,114,128,148,135,125,105,70, 80,91,110,89,129,99,140,128,
+ 85,169,69,133,73,49,110,60, 96,123,99,77,114,72,157,121, 140,133,145,94,106,142,90,139, 96,154,153,76,95,136,38,128,
+ 144,125,98,54,162,77,113,95, 108,88,42,67,143,69,97,30, 68,71,136,64,127,135,138,72, 107,115,59,137,148,97,158,128,
+ 122,156,149,122,108,71,79,147, 128,144,38,68,131,98,99,121, 164,143,123,52,140,126,130,91, 154,67,95,124,146,43,91,128,
+ 124,62,79,123,127,91,91,135, 121,69,95,76,76,128,74,47, 52,71,100,47,105,141,145,88, 123,140,106,86,152,110,91,128,
+ 123,121,138,108,94,76,62,115, 57,128,70,49,149,129,153,78, 71,120,117,115,74,80,143,95, 64,35,73,80,92,79,49,128,
+};
+
+static const uint8 ced_hires_6[1024] = {
+ 131,89,116,77,83,97,102,136, 133,105,111,105,87,87,87,105, 108,118,117,124,140,105,110,104, 68,96,93,87,81,101,115,102,
+ 17,136,152,147,132,68,100,48, 56,48,81,53,20,72,87,102, 143,155,79,74,108,84,137,154, 76,94,89,93,83,102,92,87,
+ 66,79,73,41,49,115,104,62, 46,89,89,70,100,49,89,121, 91,76,85,75,125,76,65,130, 119,156,144,96,123,158,156,118,
+ 20,106,67,42,24,96,44,57, 144,132,70,77,156,109,89,83, 121,135,126,114,134,111,112,128, 91,109,152,125,78,102,101,136,
+ 43,96,88,68,104,16,25,125, 72,107,35,59,22,13,0,14, 89,145,128,73,85,131,89,165, 108,108,98,155,106,124,167,120,
+ 128,81,99,82,73,82,85,63, 69,94,110,62,46,48,43,56, 155,167,150,139,160,127,125,138, 97,126,85,77,73,94,89,89,
+ 67,136,152,100,83,91,124,114, 106,89,92,124,146,131,103,110, 120,137,118,86,122,92,116,123, 90,103,95,79,104,103,104,97,
+ 122,74,119,104,124,104,96,98, 90,118,93,81,76,100,118,130, 143,123,120,167,152,126,138,151, 125,120,83,101,102,118,116,108,
+
+ 79,88,103,83,60,34,70,38, 92,77,94,49,116,95,82,88, 43,144,139,131,133,98,98,121, 86,94,95,101,96,80,109,97,
+ 88,98,105,102,82,110,51,104, 115,100,128,93,119,80,46,69, 110,117,74,90,120,121,93,128, 119,85,83,79,94,103,104,110,
+ 89,75,115,84,121,92,114,102, 122,90,122,128,106,108,84,118, 107,133,113,126,141,123,115,158, 103,135,99,112,107,138,121,141,
+ 94,79,115,97,115,104,113,108, 122,102,110,119,95,109,90,121, 119,127,118,122,154,126,117,158, 112,137,104,112,113,135,125,129,
+ 121,83,109,77,84,83,62,98, 104,84,113,116,93,81,73,85, 118,113,99,96,123,96,108,131, 95,111,79,75,101,113,113,103,
+ 140,66,86,79,79,90,80,94, 100,88,90,93,77,79,74,87, 118,152,103,106,142,108,110,143, 111,131,122,106,115,147,126,135,
+ 166,67,85,78,78,89,79,93, 99,87,89,92,76,78,73,86, 130,127,102,124,155,105,121,151, 123,149,123,109,120,162,148,136,
+ 154,75,95,88,88,99,89,103, 109,97,99,102,86,88,83,96, 149,130,112,108,140,112,120,145, 114,125,119,102,115,145,139,135,
+
+ 31,150,134,141,48,120,36,27, 151,116,122,82,89,136,91,28, 119,68,44,150,145,40,107,55, 36,92,23,66,46,41,24,59,
+ 41,45,12,0,141,58,83,30, 9,9,61,106,0,45,9,16, 14,23,66,152,39,5,42,56, 144,155,123,64,151,84,75,33,
+ 107,62,98,81,89,126,87,70, 167,111,120,112,104,100,108,129, 72,124,176,74,96,72,84,77, 115,63,29,72,108,97,111,154,
+ 85,144,123,65,80,95,99,66, 65,55,176,121,104,135,27,146, 62,44,158,114,101,106,105,126, 60,80,72,160,55,126,113,101,
+ 36,0,87,20,72,44,8,0, 67,127,87,121,10,84,77,51, 34,21,68,32,57,64,38,24, 9,14,132,0,17,87,36,41,
+ 67,104,95,43,73,154,82,100, 126,72,76,68,26,71,58,32, 24,45,36,21,55,166,135,135, 134,126,7,36,81,101,84,170,
+ 75,82,59,49,43,129,38,169, 67,116,63,94,69,64,52,101, 133,117,129,120,66,52,28,69, 66,82,50,119,104,83,74,71,
+ 47,131,113,126,57,142,116,109, 99,107,76,126,9,6,123,152, 96,136,103,126,142,67,90,109, 102,148,97,101,76,87,83,95,
+
+ 49,8,75,129,53,138,158,150, 98,83,101,27,3,66,159,125, 128,114,158,134,119,126,153,139, 137,128,35,126,84,61,108,84,
+ 34,0,18,21,24,22,93,33, 38,1,18,168,88,79,156,105, 149,47,24,56,50,69,76,64, 20,102,71,68,46,62,48,58,
+ 30,53,84,128,83,135,48,120, 32,36,94,29,44,92,100,149, 138,79,46,24,153,61,30,49, 158,143,56,138,115,118,110,93,
+ 74,93,20,31,73,44,42,111, 52,105,68,74,46,43,29,66, 45,67,37,7,76,88,82,101, 52,70,30,45,51,78,66,100,
+ 54,32,43,64,72,117,82,71, 69,29,37,29,4,167,126,75, 26,161,51,143,53,30,104,121, 74,79,144,27,157,129,153,117,
+ 55,53,53,87,24,19,80,75, 68,63,40,42,111,20,59,68, 72,114,50,26,138,29,70,86, 43,51,156,34,99,130,66,103,
+ 53,0,46,31,71,79,87,57, 59,51,34,39,24,0,25,66, 1,47,140,45,144,92,34,50, 90,75,50,2,24,40,147,114,
+ 63,150,105,113,95,108,52,72, 62,157,153,73,141,125,32,38, 104,104,25,64,137,147,31,105, 16,139,27,46,1,90,101,23,
+};
+
+static const uint8 ced_hires_7[1024] = {
+ 120,127,110,107,125,119,124,105, 127,128,122,128,102,123,102,99, 129,154,113,122,107,119,106,109, 116,98,104,108,120,117,116,111,
+ 91,83,50,46,97,86,122,78, 99,143,61,80,78,80,62,52, 17,15,56,62,51,91,64,93, 138,118,150,133,136,119,114,121,
+ 173,162,166,159,148,141,205,123, 204,166,143,146,120,144,133,153, 134,173,138,140,103,106,108,117, 108,108,71,103,148,151,114,164,
+ 86,147,121,122,141,130,130,109, 132,142,124,133,108,143,112,113, 130,102,125,146,122,120,113,107, 113,92,86,127,123,136,75,107,
+ 137,141,85,124,139,117,115,121, 96,81,80,124,153,77,61,54, 173,171,86,199,83,61,48,51, 58,40,46,50,62,59,58,53,
+ 145,149,144,119,151,127,147,125, 167,161,116,130,99,120,128,128, 128,103,131,112,106,147,108,130, 151,83,89,93,105,102,101,96,
+ 129,134,99,103,101,147,132,118, 111,96,87,94,100,88,67,64, 93,74,78,87,72,84,86,74, 81,63,69,73,85,82,81,76,
+ 125,123,130,99,124,109,144,109, 111,115,106,113,93,107,93,83, 92,101,134,122,117,122,132,121, 100,122,88,92,104,101,112,95,
+
+ 117,129,112,84,103,101,127,90, 87,91,82,100,75,179,112,185, 110,132,107,115,106,185,103,103, 136,103,135,134,129,86,137,125,
+ 139,147,101,95,125,132,138,134, 114,118,109,116,89,113,94,123, 98,96,100,109,94,106,93,96, 103,85,91,95,107,104,103,98,
+ 159,165,129,148,166,141,172,150, 179,163,171,181,122,149,165,145, 138,128,145,149,124,146,132,127, 143,122,121,133,144,135,162,148,
+ 171,165,133,151,163,146,162,148, 172,168,156,165,126,154,137,132, 143,140,143,155,135,156,140,142, 142,127,130,137,149,145,153,146,
+ 132,133,96,93,127,105,110,92, 113,117,108,115,88,109,88,85, 94,169,171,176,140,166,165,125, 134,143,158,156,158,166,150,158,
+ 169,154,126,147,134,128,175,148, 138,128,128,128,118,128,118,115, 124,125,128,128,123,128,122,125, 128,114,120,124,128,128,128,127,
+ 160,159,125,129,155,136,156,153, 149,128,128,128,117,128,117,114, 123,124,128,128,122,128,121,124, 128,113,119,123,128,128,128,126,
+ 168,147,128,128,128,139,128,143, 141,128,128,128,127,128,127,124, 128,128,128,128,128,128,128,128, 128,123,128,128,128,128,128,128,
+
+ 26,71,23,0,18,145,93,129, 88,129,0,160,78,145,20,29, 21,64,41,28,155,90,68,73, 46,65,128,9,130,142,83,131,
+ 59,84,81,102,49,55,8,132, 69,48,0,80,0,123,0,36, 37,33,0,62,129,121,64,162, 123,122,0,154,69,160,143,89,
+ 155,85,24,119,133,127,86,100, 126,180,98,119,16,37,81,98, 82,71,113,46,31,50,30,33, 70,148,98,135,141,102,162,99,
+ 94,74,90,98,122,40,42,20, 72,125,152,93,42,97,112,5, 103,130,75,82,21,65,120,149, 107,58,80,48,61,150,41,63,
+ 118,82,161,115,70,0,55,0, 13,151,100,56,59,0,25,151, 102,102,94,0,133,95,68,98, 35,166,95,80,163,108,138,85,
+ 47,27,46,3,18,108,113,138, 156,140,109,142,22,49,123,83, 98,107,78,74,32,109,58,60, 83,0,109,0,1,0,105,55,
+ 71,49,44,31,49,30,38,141, 120,37,96,114,14,14,14,21, 22,111,109,31,70,103,7,100, 17,109,62,65,72,49,18,128,
+ 40,154,151,137,126,105,104,61, 119,144,127,42,0,0,164,141, 135,126,95,119,107,112,29,0, 0,0,99,11,15,13,9,0,
+
+ 108,133,116,95,53,149,107,100, 56,66,115,79,15,100,65,0, 1,0,7,140,111,132,101,0, 95,4,92,92,46,22,161,72,
+ 58,38,113,14,88,36,11,97, 67,63,27,39,143,108,133,82, 135,0,52,60,72,93,90,149, 109,147,143,103,45,159,148,153,
+ 132,115,1,88,45,51,72,56, 44,62,5,83,23,28,153,121, 167,37,147,101,0,82,26,76, 97,99,17,1,1,0,40,86,
+ 83,46,30,15,41,21,26,83, 78,66,41,84,84,78,0,41, 50,103,63,122,73,69,0,61, 48,0,97,105,70,96,62,90,
+ 85,84,93,6,54,125,46,123, 102,62,7,71,96,44,54,0, 54,4,166,155,31,144,107,0, 87,55,65,143,79,47,0,5,
+ 163,95,121,171,161,163,63,142, 100,152,141,64,95,125,105,56, 0,89,99,83,64,43,95,89, 83,50,23,0,68,0,0,76,
+ 92,65,42,71,88,88,176,120, 171,105,154,154,18,44,54,106, 112,92,134,139,0,134,133,130, 116,101,84,51,64,46,90,155,
+ 152,122,117,113,19,86,30,51, 85,122,99,90,18,146,45,102, 58,13,43,75,104,119,101,0, 77,0,0,0,146,0,119,0,
+};
+
+static const uint8 ced_hires_8[1024] = {
+ 130,127,155,168,146,193,173,178, 138,176,189,169,199,152,146,172, 128,127,128,122,128,128,109,122, 115,128,126,128,120,128,128,128,
+ 142,160,113,159,112,138,145,87, 133,125,100,110,109,101,70,134, 52,81,111,42,63,48,9,22, 34,44,34,64,45,60,106,128,
+ 98,91,146,105,104,134,59,53, 74,82,71,49,70,51,52,76, 77,69,81,64,94,79,51,64, 57,75,68,77,62,74,106,128,
+ 90,132,91,134,101,123,97,99, 109,125,92,97,124,114,109,140, 123,84,137,120,149,126,95,134, 90,124,96,82,140,78,116,128,
+ 68,69,88,133,99,128,124,111, 101,105,137,125,134,111,84,138, 137,75,81,64,111,89,60,64, 57,75,68,77,62,215,110,128,
+ 111,181,158,179,148,178,161,155, 155,149,184,172,200,153,139,187, 156,138,159,137,153,154,108,115, 129,118,111,120,105,117,128,128,
+ 91,101,111,146,104,81,82,70, 84,86,94,72,93,74,75,99, 100,92,104,87,117,102,74,87, 80,98,91,100,85,97,128,128,
+ 137,111,137,155,136,166,170,156, 138,140,168,155,174,149,139,169, 119,111,123,106,128,121,93,106, 99,117,110,119,104,116,128,128,
+
+ 127,121,128,160,166,147,149,173, 185,181,184,160,174,149,143,166, 164,138,146,126,151,136,178,115, 117,182,167,95,80,92,124,128,
+ 113,114,134,159,120,170,172,178, 160,151,180,174,185,158,134,167, 122,114,126,109,128,124,181,188, 193,130,119,145,158,155,152,128,
+ 155,164,146,164,148,177,155,147, 159,166,174,167,174,147,135,174, 105,97,186,168,122,107,79,92, 85,103,96,105,90,102,128,128,
+ 161,164,146,176,146,180,170,159, 156,165,177,172,179,155,145,171, 111,103,162,180,176,113,104,98, 91,109,102,111,96,108,128,128,
+ 187,159,158,190,172,180,181,159, 155,156,183,160,177,165,170,171, 148,172,125,108,128,141,95,108, 101,119,112,121,126,118,128,128,
+ 128,128,120,133,128,128,128,121, 128,128,128,123,128,125,126,128, 128,128,128,128,128,128,125,128, 166,128,128,128,131,128,128,128,
+ 128,128,119,128,128,128,128,122, 129,131,128,122,128,124,125,128, 128,128,128,128,128,128,124,128, 128,128,128,128,128,128,128,128,
+ 128,128,128,154,128,128,133,128, 142,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
+
+ 20,141,64,92,31,78,151,53, 145,20,53,48,65,154,102,114, 93,115,27,0,49,93,58,115, 144,65,153,88,149,88,17,128,
+ 84,40,161,60,108,42,125,33, 139,68,83,54,43,81,132,82, 69,84,108,0,122,85,6,2, 53,70,102,0,36,34,41,128,
+ 100,104,66,145,69,110,103,89, 76,65,98,98,84,63,121,133, 106,95,152,44,168,45,122,127, 129,38,117,149,53,137,128,128,
+ 134,104,149,56,114,62,74,104, 71,45,50,137,132,122,110,44, 112,56,54,136,150,128,59,0, 0,81,103,3,0,0,33,128,
+ 81,17,22,151,126,101,49,126, 22,36,76,149,81,76,40,83, 64,111,53,68,65,145,96,0, 110,0,87,0,39,98,99,128,
+ 104,20,78,59,26,158,114,116, 105,158,62,77,97,99,149,60, 158,104,111,51,111,77,41,70, 35,124,61,140,87,83,139,128,
+ 89,103,71,128,99,85,122,108, 62,50,67,61,120,64,44,79, 43,9,107,179,176,163,107,154, 141,87,110,163,2,37,115,128,
+ 0,162,20,49,20,47,97,121, 95,83,96,88,115,58,47,97, 61,11,73,19,29,17,0,0, 0,148,82,94,111,89,40,128,
+
+ 120,92,69,58,56,38,49,120, 38,153,95,56,69,140,92,44, 157,155,88,91,145,0,89,50, 109,97,0,66,0,18,97,128,
+ 114,13,57,120,93,77,153,77, 152,135,99,80,88,24,69,60, 107,53,43,2,13,0,131,78, 141,49,121,0,119,153,135,128,
+ 0,4,6,55,124,56,65,29, 21,40,149,41,100,137,17,39, 3,163,111,111,140,0,52,59, 72,113,76,118,116,81,92,128,
+ 47,32,26,62,0,51,45,166, 142,38,168,49,140,20,96,145, 124,74,90,169,73,151,143,89, 64,116,26,75,20,162,143,128,
+ 0,0,91,31,9,39,39,26, 90,29,42,35,53,27,112,56, 58,67,4,76,79,51,153,115, 122,50,137,120,19,114,123,128,
+ 78,120,61,100,65,59,85,52, 72,57,93,54,61,10,72,78, 56,75,77,0,33,34,43,38, 0,0,14,44,0,44,24,128,
+ 151,95,67,85,59,72,70,135, 53,60,105,54,54,25,160,139, 142,61,116,126,90,1,104,159, 139,115,141,100,41,62,100,128,
+ 0,0,16,46,152,103,55,43, 56,47,60,148,157,92,133,148, 0,0,158,50,138,103,111,67, 150,157,47,12,78,21,146,128,
+};
+
+static const uint8 ced_hires_9[1024] = {
+ 130,71,129,141,137,60,133,127, 113,78,80,127,94,127,117,51, 103,62,119,129,148,68,133,101, 38,42,158,125,112,190,163,124,
+ 139,45,127,155,132,105,145,125, 136,78,127,128,145,136,141,61, 122,142,128,129,137,97,140,143, 1,3,145,120,131,129,135,62,
+ 124,125,104,88,80,128,69,72, 68,127,36,103,135,92,101,131, 77,126,132,112,51,143,88,87, 94,151,72,95,120,110,80,161,
+ 81,123,101,98,60,133,79,54, 52,155,28,125,46,72,67,101, 90,70,86,76,45,128,37,119, 86,129,81,86,102,49,41,88,
+ 96,132,92,99,114,136,90,108, 121,130,15,114,132,93,122,129, 135,112,129,111,82,133,131,124, 119,129,105,108,112,39,105,147,
+ 115,68,124,132,149,103,109,139, 127,72,144,126,137,143,146,76, 122,99,138,142,133,69,134,126, 32,11,131,145,107,148,126,61,
+ 77,126,72,78,49,124,127,85, 75,135,43,82,121,79,83,143, 74,43,129,98,111,130,44,45, 87,106,101,92,138,51,52,89,
+ 95,126,67,51,135,116,72,112, 51,131,24,106,133,97,111,143, 71,33,136,82,68,135,62,78, 22,49,63,84,114,36,104,82,
+
+ 46,129,63,71,69,93,73,121, 119,140,58,72,114,107,136,142, 96,57,126,112,97,126,48,117, 55,54,76,101,134,53,81,140,
+ 139,93,114,144,122,123,128,130, 146,120,146,137,129,138,134,101, 109,151,120,136,133,78,129,131, 0,6,154,131,126,122,151,58,
+ 103,57,111,129,142,74,122,99, 101,56,40,136,131,124,138,107, 93,87,91,150,153,48,74,96, 29,52,125,155,90,119,156,95,
+ 71,137,72,129,58,106,76,87, 65,135,10,79,127,82,98,139, 95,29,127,121,126,138,126,104, 58,30,84,95,111,27,55,91,
+ 148,128,84,57,100,136,84,113, 74,138,28,114,116,81,111,129, 86,138,17,107,80,134,130,64, 160,116,80,94,104,93,101,84,
+ 76,131,112,98,55,135,114,120, 76,132,40,106,110,135,121,131, 146,114,77,99,59,136,66,90, 106,140,89,78,129,75,105,92,
+ 119,136,75,128,125,129,141,113, 81,139,14,124,57,64,124,132, 74,124,75,119,117,122,73,70, 119,145,101,93,142,120,105,136,
+ 111,62,156,130,141,107,139,152, 124,103,148,125,129,141,131,71, 138,116,130,137,129,82,139,148, 0,0,125,133,142,126,134,55,
+
+ 72,116,48,77,48,122,93,65, 57,115,22,108,127,51,83,142, 87,100,151,86,87,131,39,43, 86,113,70,90,98,53,85,72,
+ 149,20,138,146,143,112,62,118, 158,65,125,130,127,150,129,33, 114,116,113,124,140,58,141,141, 24,20,165,129,84,163,122,98,
+ 106,140,115,104,111,132,132,124, 123,125,23,112,87,134,120,128, 118,121,93,124,114,141,143,116, 117,135,106,137,107,90,103,86,
+ 104,114,115,105,103,116,131,80, 117,119,30,147,132,120,108,122, 147,136,106,132,147,127,81,123, 134,114,71,127,123,66,120,147,
+ 106,131,95,77,65,135,100,63, 69,133,8,113,86,81,116,133, 74,114,131,122,67,128,53,135, 151,132,60,61,111,92,110,88,
+ 161,88,152,116,143,105,115,146, 136,61,114,137,128,126,120,44, 151,90,132,138,125,54,155,126, 8,28,138,144,121,160,160,75,
+ 87,124,98,82,154,145,88,92, 46,138,32,132,81,107,138,90, 81,34,84,103,27,129,104,81, 109,61,73,49,116,47,121,91,
+ 72,136,72,95,97,132,49,95, 111,125,35,117,129,103,124,133, 112,95,117,134,95,115,67,97, 102,148,126,135,108,97,99,123,
+
+ 165,12,134,132,98,115,126,113, 92,122,36,138,34,110,157,63, 107,139,40,148,113,22,79,71, 31,22,127,133,78,127,95,90,
+ 38,12,112,64,133,138,43,108, 175,16,168,125,147,148,91,8, 100,54,87,106,121,17,106,130, 17,47,101,161,65,95,131,83,
+ 119,144,110,98,143,110,64,120, 43,121,32,102,111,138,133,123, 86,119,120,80,52,126,109,143, 98,130,90,84,85,38,91,148,
+ 95,124,58,99,38,144,76,50, 77,139,32,143,125,91,120,110, 106,28,108,77,132,126,36,92, 136,57,38,62,101,62,40,90,
+ 47,23,65,73,87,20,162,94, 103,127,100,162,133,118,116,37, 117,60,94,98,151,51,49,98, 24,20,116,121,117,51,42,90,
+ 54,123,33,45,31,151,56,33, 42,145,36,29,25,35,119,81, 32,43,48,22,12,118,44,43, 149,32,38,50,76,65,37,95,
+ 46,130,41,34,22,145,59,49, 59,134,20,125,111,79,130,70, 69,37,81,47,140,127,73,75, 123,43,61,147,80,46,72,78,
+ 115,32,81,74,58,149,102,67, 71,46,65,60,49,57,61,30, 68,174,54,51,45,52,72,53, 57,53,71,79,105,84,67,123,
+};
+
+static const uint8 ced_hires_10[1024] = {
+ 58,122,143,128,136,100,142,138, 75,129,136,141,139,140,49,124, 126,129,133,99,142,134,152,137, 120,138,45,14,25,143,136,140,
+ 125,108,87,90,85,127,88,79, 128,11,95,136,93,104,132,81, 132,106,62,140,86,72,90,83, 121,114,149,146,89,121,124,118,
+ 133,75,100,96,92,129,79,127, 127,44,117,130,100,119,132,110, 114,131,93,115,52,110,92,107, 134,108,137,144,103,119,70,100,
+ 129,70,93,99,140,110,67,56, 133,40,103,132,85,100,142,72, 134,84,77,132,70,53,62,94, 79,31,106,60,43,131,68,32,
+ 132,100,125,106,118,135,136,106, 130,30,113,134,97,123,128,124, 125,111,84,130,80,106,89,109, 111,56,133,127,111,123,95,111,
+ 67,126,128,140,149,99,132,133, 69,144,128,139,143,147,68,121, 136,137,133,67,119,131,136,128, 136,149,79,10,23,96,114,96,
+ 125,102,73,98,152,143,106,80, 137,19,128,84,119,136,89,83, 74,107,89,132,74,58,81,126, 52,49,110,73,108,116,88,42,
+ 142,113,137,119,142,109,104,86, 122,38,105,107,137,130,124,108, 116,84,69,126,79,79,97,100, 101,51,133,133,97,103,115,109,
+
+ 96,116,125,136,124,122,129,150, 119,148,137,131,138,134,97,111, 120,136,131,74,132,148,147,147, 126,125,57,11,16,101,139,150,
+ 64,113,99,105,134,74,84,122, 61,63,130,131,122,139,112,93, 90,147,154,53,127,109,126,145, 147,109,83,59,31,99,68,83,
+ 137,90,110,94,59,103,126,88, 132,21,84,127,93,105,137,97, 124,122,120,138,85,68,128,65, 88,44,132,42,58,106,79,34,
+ 124,85,81,119,108,133,128,91, 138,36,114,116,86,113,128,87, 26,102,87,132,98,77,76,98, 95,93,108,117,161,118,149,141,
+ 131,125,92,102,68,135,80,85, 131,41,104,110,136,120,132,145, 74,100,62,135,119,73,95,106, 84,72,119,134,102,130,74,113,
+ 136,71,72,117,124,128,73,106, 138,23,122,64,52,124,130,76, 75,120,120,120,127,85,126,105, 103,120,93,146,119,110,119,124,
+ 59,156,145,150,141,110,137,123, 104,147,127,129,138,125,77,138, 130,135,130,87,148,123,129,136, 129,131,48,15,26,124,110,114,
+ 115,51,73,57,54,120,36,57, 115,11,101,125,70,76,140,93, 150,94,84,126,85,49,83,103, 85,56,128,111,78,103,71,102,
+
+ 137,112,116,124,112,131,132,116, 126,33,113,86,134,122,127,115, 94,124,117,137,127,116,110,91, 128,86,117,135,114,119,99,120,
+ 116,104,122,83,102,118,81,68, 114,12,145,131,124,108,121,145, 103,127,145,127,115,112,108,119, 140,58,140,108,123,116,98,134,
+ 131,98,133,63,58,136,35,61, 131,29,113,81,78,117,131,74, 128,121,76,127,101,71,79,105, 70,83,126,131,150,127,103,111,
+ 95,145,124,149,145,102,158,144, 73,115,137,128,126,118,39,154, 127,136,124,85,112,136,102,157, 149,155,63,39,21,131,157,85,
+ 121,75,41,91,52,123,59,122, 133,55,87,113,80,75,144,80, 121,95,110,126,123,72,70,79, 101,39,94,99,84,136,68,51,
+ 127,76,124,126,75,95,60,87, 130,36,76,111,110,138,142,109, 125,113,92,127,77,120,87,88, 92,44,116,59,67,146,52,55,
+ 124,94,127,65,59,135,31,83, 154,42,119,47,70,56,99,85, 70,79,69,127,80,57,107,42, 77,38,108,135,113,116,94,95,
+ 134,58,90,56,42,142,76,51, 134,21,125,102,78,135,80,81, 78,60,137,122,61,53,43,84, 147,45,82,70,120,107,56,52,
+
+ 125,58,101,70,50,142,31,51, 142,44,142,128,93,121,116,115, 115,88,119,127,79,67,100,81, 78,68,109,74,131,115,109,34,
+ 128,20,41,24,44,151,46,58, 143,31,53,37,38,115,104,35, 49,13,50,110,40,29,34,32, 52,77,75,38,140,67,53,62,
+ 30,102,98,113,102,138,130,139, 39,47,119,112,103,104,37,128, 128,103,110,33,80,102,106,125, 86,137,98,43,38,91,95,175,
+ 21,105,132,102,123,141,110,109, 31,168,125,139,150,92,18,94, 86,105,122,38,63,172,77,132, 155,93,48,68,35,60,65,79,
+ 28,139,76,117,98,118,79,120, 117,45,140,50,110,154,88,110, 45,147,116,36,127,99,132,98, 137,124,54,34,56,82,164,143,
+ 35,74,103,109,98,33,86,125, 55,113,149,141,121,121,50,122, 113,105,152,60,154,113,84,61, 128,49,72,16,20,144,58,61,
+ 82,126,110,148,135,61,143,155, 65,78,116,108,135,109,46,91, 122,125,145,67,103,116,124,162, 127,185,84,36,46,110,131,52,
+ 42,137,146,121,136,110,144,167, 67,120,128,125,146,130,34,118, 111,122,138,47,72,152,133,119, 127,160,56,29,36,97,143,120,
+};
+
+static const uint8 ced_hires_11[1024] = {
+ 129,46,102,93,87,61,93,55, 96,48,77,76,65,67,91,46, 64,58,113,75,57,62,82,91, 106,70,112,123,68,70,75,108,
+ 113,55,138,144,134,87,155,40, 138,123,135,133,127,142,137,53, 147,126,126,136,132,133,154,124, 123,43,142,128,28,86,28,104,
+ 122,134,145,124,106,123,46,119, 48,128,47,146,45,45,47,127, 41,140,89,43,38,100,46,40, 62,104,61,104,126,110,126,124,
+ 91,125,26,136,96,129,23,115, 19,144,133,118,113,115,63,132, 59,137,94,51,29,117,36,127, 39,136,46,74,118,107,102,110,
+ 89,128,72,40,74,135,21,135, 34,144,57,55,66,55,51,129, 53,127,93,50,32,126,63,29, 37,128,44,72,113,117,128,118,
+ 77,69,127,126,130,77,123,31, 130,136,133,130,123,130,153,82, 138,132,127,134,127,137,125,128, 123,115,114,116,75,59,43,117,
+ 113,116,62,47,39,142,82,137, 41,119,39,88,88,58,47,136, 28,38,72,11,27,108,35,44, 65,155,73,100,130,105,132,111,
+ 93,34,108,139,100,55,67,50, 146,50,129,131,153,145,111,31, 105,138,137,151,137,51,116,127, 146,30,74,76,15,37,10,59,
+
+ 98,133,33,28,17,143,30,136, 41,115,34,100,131,73,38,123, 12,134,69,24,4,134,39,29, 52,118,53,81,110,140,124,88,
+ 82,128,133,117,145,106,126,85, 134,68,159,123,131,130,125,127, 111,117,130,146,125,44,111,141, 127,118,87,70,96,9,66,50,
+ 92,145,86,40,68,123,37,120, 71,114,90,126,74,74,28,128, 73,120,89,54,119,118,68,72, 51,112,35,63,137,116,114,104,
+ 98,123,104,91,72,135,54,139, 112,130,98,141,114,67,63,136, 86,42,87,61,112,117,101,58, 106,134,41,69,123,123,123,117,
+ 85,137,150,37,54,142,42,126, 36,124,60,30,130,98,12,132, 135,24,89,53,49,98,125,57, 119,105,38,66,134,116,118,107,
+ 98,141,79,89,141,133,98,129, 133,130,72,83,48,92,48,131, 38,31,95,107,139,113,59,69, 40,143,37,65,112,132,107,111,
+ 112,118,69,50,46,149,49,152, 39,130,45,42,40,30,66,118, 54,26,75,14,107,112,49,54, 65,116,73,100,115,143,78,91,
+ 91,69,145,144,140,88,104,98, 127,131,122,136,136,132,115,76, 134,134,127,139,121,152,149,142, 144,83,166,167,31,77,39,99,
+
+ 82,130,53,59,64,129,54,110, 44,118,48,133,87,91,27,140, 84,148,91,51,122,101,61,86, 50,134,46,84,126,106,112,112,
+ 84,135,121,143,124,127,87,121, 113,142,112,75,123,109,90,133, 90,98,88,84,123,123,125,129, 92,141,44,70,118,108,114,104,
+ 134,108,127,101,121,121,98,125, 121,115,125,94,128,88,98,112, 124,82,126,112,130,134,126,138, 101,122,105,119,92,85,86,126,
+ 77,117,116,48,78,138,27,142, 122,120,126,63,134,48,25,112, 112,39,86,120,154,143,120,143, 28,123,36,63,109,110,108,101,
+ 71,131,58,45,30,128,122,146, 62,130,49,95,90,25,18,143, 48,127,97,99,74,119,33,43, 22,146,30,57,118,112,106,106,
+ 95,104,131,129,128,72,131,81, 132,75,120,142,143,138,133,92, 155,132,138,149,139,63,128,144, 160,88,119,81,75,7,47,66,
+ 102,128,48,47,57,119,70,107, 140,130,80,106,65,90,32,136, 116,123,82,61,117,132,66,43, 55,154,61,89,127,128,105,122,
+ 110,129,58,13,38,138,30,112, 136,126,30,98,89,135,39,126, 55,147,70,32,117,122,68,87, 58,145,57,84,130,114,103,117,
+
+ 118,115,58,40,45,141,61,156, 51,109,30,30,37,33,59,110, 34,25,81,42,99,147,71,54, 97,128,88,115,120,107,128,103,
+ 102,62,107,133,115,63,64,42, 135,82,88,103,135,156,108,60, 147,137,139,149,144,30,125,110, 68,80,128,101,53,14,38,71,
+ 127,100,145,84,145,70,134,37, 72,39,175,114,120,112,75,120, 76,99,111,119,117,48,91,106, 110,125,102,128,84,56,63,90,
+ 129,77,130,151,138,88,126,88, 117,82,96,136,127,108,124,80, 155,101,115,124,94,99,175,159, 129,107,129,129,106,107,112,114,
+ 98,9,84,76,73,44,76,33, 73,22,61,73,76,66,62,27, 84,63,137,71,74,28,80,84, 106,47,83,88,29,18,15,66,
+ 102,23,80,68,70,48,77,40, 82,24,57,72,77,67,80,46, 70,66,135,70,62,10,75,87, 108,64,76,91,16,23,22,74,
+ 105,19,61,52,70,26,63,47, 80,27,40,55,73,79,73,33, 71,69,127,73,72,12,72,69, 65,55,77,93,18,43,34,72,
+ 104,44,77,65,69,48,79,52, 74,47,58,58,60,52,78,46, 61,55,123,55,55,61,74,74, 89,67,93,106,71,67,69,73,
+};
+
+static const uint8 ced_hires_12[1024] = {
+ 79,129,118,114,118,130,135,136, 122,129,124,105,133,117,122,111, 134,119,65,123,122,86,117,114, 121,123,136,124,126,111,73,121,
+ 134,116,126,128,129,122,119,131, 131,128,98,130,123,67,131,120, 126,125,137,42,120,129,124,126, 132,124,121,124,126,120,89,121,
+ 112,133,109,133,111,132,130,71, 99,133,71,58,135,135,120,135, 124,96,123,110,95,72,54,65, 130,119,105,127,127,103,105,127,
+ 118,122,128,106,129,130,78,100, 80,135,92,115,113,114,117,119, 117,117,148,147,127,77,100,118, 129,121,108,127,127,101,91,127,
+ 135,132,136,125,118,124,139,134, 122,120,34,135,116,109,141,104, 122,131,129,5,132,82,135,130, 120,130,119,128,123,119,103,118,
+ 126,133,130,135,119,105,128,126, 122,109,135,125,129,124,128,144, 130,124,127,128,125,138,129,123, 129,123,141,114,116,100,75,111,
+ 126,110,99,100,152,132,105,74, 73,128,77,140,108,97,135,114, 115,76,102,86,74,55,93,128, 128,65,82,127,127,109,97,127,
+ 83,137,114,142,118,131,144,118, 104,132,68,106,127,101,119,118, 124,123,62,94,123,122,117,132, 123,130,118,127,145,96,99,127,
+
+ 115,129,121,73,117,134,107,125, 107,139,57,129,133,75,111,122, 115,108,124,112,115,65,89,120, 136,89,100,127,127,107,91,162,
+ 116,124,116,129,125,122,126,120, 126,115,136,128,123,153,120,132, 126,124,119,124,121,125,124,127, 124,129,124,115,117,98,87,112,
+ 110,93,120,88,101,100,92,100, 101,103,193,110,109,83,98,97, 81,85,93,129,84,120,83,96, 93,94,86,127,127,127,127,127,
+ 130,129,102,125,118,128,139,116, 89,126,144,112,143,114,130,139, 137,125,99,129,122,105,77,70, 122,122,127,127,144,123,84,127,
+ 140,124,123,126,138,124,114,136, 125,129,136,130,115,106,135,110, 119,120,124,115,130,112,127,128, 111,122,116,121,123,113,80,118,
+ 109,106,107,107,110,111,111,99, 109,127,112,107,106,162,102,138, 105,111,114,99,121,103,86,90, 93,107,100,127,127,170,152,127,
+ 129,122,126,129,130,127,131,138, 125,126,108,134,116,77,116,126, 127,138,131,43,123,110,137,129, 121,138,123,121,123,110,70,118,
+ 108,114,111,100,106,124,104,100, 108,110,114,106,106,96,119,162, 90,141,96,108,93,105,112,107, 100,92,93,127,127,144,144,127,
+
+ 119,102,134,125,132,132,111,116, 139,135,118,123,93,104,109,107, 109,137,126,109,126,113,120,127, 111,127,122,127,127,96,84,127,
+ 108,121,133,121,111,125,73,107, 148,128,119,114,117,108,124,100, 119,106,116,170,155,38,84,144, 123,58,113,127,127,105,99,154,
+ 57,128,109,135,124,126,134,42, 110,124,72,117,140,133,125,114, 124,133,85,97,99,143,140,113, 129,122,120,127,127,99,87,127,
+ 93,88,77,107,97,101,91,104, 98,96,155,103,106,69,77,79, 81,137,97,206,81,184,134,112, 92,116,64,127,127,127,127,127,
+ 118,85,115,94,120,133,99,119, 127,125,107,92,120,53,74,109, 133,143,137,92,89,114,118,130, 145,133,124,127,127,100,88,127,
+ 93,97,96,121,91,124,100,109, 94,125,112,118,104,82,100,96, 123,97,112,138,108,200,107,142, 135,117,94,127,127,127,127,127,
+ 130,129,119,115,125,130,53,116, 116,133,51,40,120,97,122,101, 108,62,141,108,137,124,103,105, 138,40,104,127,127,109,97,127,
+ 111,131,74,122,129,132,85,105, 144,128,63,62,121,83,101,114, 133,139,115,123,125,105,139,99, 136,139,110,127,127,106,92,127,
+
+ 132,119,131,119,128,131,116,120, 132,131,135,136,87,90,112,113, 116,132,115,124,125,154,127,132, 94,130,122,127,127,108,106,127,
+ 127,129,109,98,125,122,112,115, 115,128,120,126,142,114,126,103, 132,96,127,105,124,69,102,122, 128,103,120,127,127,98,112,127,
+ 121,120,145,97,119,126,115,132, 89,127,134,127,112,113,133,129, 130,102,128,127,135,64,118,140, 138,124,104,127,127,129,118,127,
+ 127,127,127,127,127,122,127,127, 127,122,127,127,127,127,127,127, 127,127,127,127,127,127,127,127, 127,127,127,127,127,127,127,127,
+ 127,127,127,127,127,118,127,127, 127,118,127,127,127,127,127,127, 145,127,127,127,127,127,127,127, 127,127,127,127,127,127,127,127,
+ 121,124,104,109,109,81,110,97, 104,77,120,119,111,92,119,106, 100,107,107,127,117,127,112,104, 89,124,89,127,127,197,184,127,
+ 115,99,101,103,110,78,92,110, 84,142,93,113,91,65,122,79, 105,120,109,104,120,113,97,90, 86,115,97,127,127,182,217,127,
+ 127,127,127,127,127,122,127,127, 127,124,127,127,127,127,127,127, 127,127,127,127,127,127,127,127, 127,127,127,127,127,127,127,127,
+};
+
+static const uint8 ced_hires_13[1024] = {
+ 79,36,84,88,70,26,101,101, 57,93,76,44,82,95,88,77, 79,102,93,112,76,86,62,31, 126,102,50,102,97,110,99,131,
+ 52,43,113,117,112,51,119,123, 107,86,124,118,123,119,119,49, 114,113,112,125,118,105,132,18, 125,60,129,75,77,71,51,90,
+ 99,65,124,123,98,65,96,121, 107,100,76,93,112,140,131,73, 92,127,90,86,132,109,81,54, 57,124,94,100,98,115,106,112,
+ 81,31,42,63,61,107,26,43, 32,76,44,28,57,75,56,165, 61,40,75,79,77,82,47,21, 39,68,76,80,81,88,78,107,
+ 53,55,83,115,118,88,124,129, 135,116,109,113,126,117,121,71, 103,69,120,116,124,140,113,25, 116,121,78,69,55,61,52,95,
+ 75,63,96,27,126,105,92,121, 78,64,55,123,122,101,117,34, 107,33,121,105,112,47,100,18, 60,36,113,61,64,87,64,99,
+ 70,75,100,78,116,77,119,135, 81,90,103,129,128,106,113,57, 91,57,132,124,117,87,117,49, 60,46,37,98,75,96,77,107,
+ 104,148,95,50,41,136,45,51, 72,150,86,83,112,103,30,141, 59,39,43,55,90,129,56,44, 57,84,26,104,80,110,75,126,
+
+ 72,54,96,117,102,69,79,123, 44,79,60,58,105,135,103,47, 69,152,129,129,112,72,124,24, 118,59,89,72,68,75,69,97,
+ 49,104,110,125,122,126,125,119, 116,94,112,92,115,120,104,110, 129,129,113,124,117,103,117,22, 126,38,90,48,80,82,62,97,
+ 76,70,64,110,64,64,46,62, 48,63,80,67,84,137,129,30, 93,115,59,121,142,110,109,61, 115,61,70,84,90,95,95,95,
+ 100,63,73,65,82,99,97,66, 73,125,80,83,138,86,127,52, 78,109,102,98,105,78,103,62, 112,76,108,111,109,108,113,111,
+ 111,114,86,93,91,76,81,78, 85,82,114,98,92,112,111,94, 98,94,75,90,95,93,90,72, 91,82,103,151,113,118,141,119,
+ 49,138,104,130,121,72,113,92, 114,38,101,119,105,122,105,109, 107,112,87,116,119,68,115,8, 83,24,113,61,68,71,56,94,
+ 114,52,76,101,85,55,85,54, 85,94,75,85,117,114,135,41, 81,88,76,85,140,64,70,51, 76,101,82,105,98,101,102,103,
+ 114,111,126,117,134,117,113,90, 68,92,84,110,117,110,122,91, 91,164,90,122,124,100,108,120, 102,94,88,118,111,108,113,111,
+
+ 101,137,103,32,71,95,111,96, 97,145,123,85,103,96,79,61, 52,54,86,108,68,157,102,41, 69,71,61,101,99,107,96,119,
+ 73,150,20,28,25,114,26,26, 37,112,46,33,31,22,13,156, 23,49,1,30,13,101,28,20, 32,47,30,71,90,77,68,96,
+ 105,79,95,84,96,64,96,113, 63,97,73,68,104,114,113,65, 121,104,111,119,80,88,85,63, 121,61,65,125,108,129,103,127,
+ 42,53,109,91,120,40,90,107, 68,92,130,92,111,108,134,29, 112,80,111,94,96,54,105,150, 132,48,101,55,61,61,49,94,
+ 106,68,93,88,93,69,69,61, 73,98,117,78,128,133,126,94, 114,78,82,97,141,98,102,65, 75,66,104,102,89,122,99,112,
+ 107,83,114,50,97,149,84,90, 128,149,119,100,113,91,105,79, 132,52,98,96,108,142,94,37, 91,66,96,111,86,94,85,107,
+ 46,66,107,104,113,66,126,117, 117,92,123,126,115,109,116,65, 119,42,129,113,104,53,118,79, 78,127,128,50,61,72,51,99,
+ 121,71,63,62,64,61,69,65, 74,70,74,66,65,54,63,58, 71,84,44,54,50,87,74,66, 113,81,60,152,130,138,134,138,
+
+ 84,83,129,39,123,95,106,128, 82,84,142,138,113,108,107,57, 118,38,127,106,107,90,120,60, 59,142,45,75,98,85,68,107,
+ 105,90,94,88,71,76,71,67, 72,101,88,64,81,91,95,66, 85,76,67,85,79,82,82,58, 109,96,81,127,113,121,114,123,
+ 73,100,141,107,128,71,80,84, 100,85,121,103,123,126,115,69, 119,72,94,128,109,59,112,26, 83,59,116,115,83,89,73,94,
+ 104,73,90,91,87,88,89,78, 90,76,102,101,111,102,89,43, 82,81,118,96,146,75,100,50, 87,96,96,113,113,106,109,114,
+ 66,58,131,124,106,86,118,114, 132,74,68,119,117,117,117,35, 102,36,125,106,106,24,98,35, 71,128,128,81,81,80,47,96,
+ 73,41,118,138,109,28,78,48, 115,45,110,113,95,133,88,35, 121,38,106,109,103,103,117,18, 44,86,119,71,77,86,76,105,
+ 94,137,71,54,52,142,72,61, 72,154,145,98,92,97,38,93, 58,58,95,69,85,132,127,42, 91,123,55,108,102,106,97,140,
+ 113,65,76,58,62,92,85,62, 80,69,57,64,67,66,51,59, 80,72,50,69,62,68,58,73, 71,60,58,101,115,108,109,118,
+};
+
+static const uint8 ced_hires_14[1024] = {
+ 127,73,147,96,119,88,105,109, 120,113,103,118,88,140,101,79, 110,127,68,112,151,94,90,83, 125,83,125,127,127,127,127,127,
+ 88,49,130,116,118,52,139,140, 125,89,134,128,134,134,134,59, 124,140,131,139,131,105,148,19, 145,52,137,100,89,91,77,99,
+ 131,71,141,122,104,66,116,138, 125,103,87,103,122,155,146,83, 102,154,109,100,144,109,97,55, 77,116,102,118,110,122,130,121,
+ 134,25,93,100,106,94,68,101, 54,113,93,18,111,141,103,68, 142,68,136,117,127,146,99,18, 62,18,121,110,88,106,105,109,
+ 87,60,100,113,123,88,143,145, 153,119,119,122,136,131,136,80, 113,95,138,130,136,140,128,26, 136,113,85,93,67,79,77,103,
+ 127,99,154,57,97,71,91,93, 125,69,89,116,117,76,110,81, 149,113,85,77,84,90,75,68, 110,72,120,127,127,120,127,123,
+ 107,127,106,67,34,132,96,32, 46,138,56,96,38,110,100,88, 74,63,26,39,49,130,29,120, 45,34,49,93,90,113,96,118,
+ 140,153,112,49,46,136,65,67, 90,152,96,92,122,118,45,150, 69,64,61,68,102,129,71,44, 77,75,33,128,92,127,99,134,
+
+ 74,133,89,64,55,143,71,52, 72,150,122,145,125,84,137,124, 106,53,105,97,125,131,90,27, 24,49,71,72,81,88,67,109,
+ 83,109,127,124,128,126,144,135, 134,97,122,101,125,134,119,119, 139,155,131,138,129,102,132,22, 146,29,97,71,91,102,87,105,
+ 88,4,133,137,142,3,28,140, 12,31,52,141,80,31,31,20, 149,46,21,114,133,20,2,90, 69,0,136,73,62,95,79,102,
+ 126,68,90,64,87,99,117,82, 91,127,90,92,147,101,142,60, 88,136,120,111,116,77,118,62, 130,67,116,128,121,114,137,119,
+ 79,15,89,109,132,0,34,27, 136,25,159,134,135,122,132,40, 105,35,121,133,141,23,146,2, 28,0,125,76,68,108,72,91,
+ 84,143,121,128,126,72,133,108, 132,41,111,128,115,136,120,118, 117,138,105,130,131,68,130,9, 103,16,120,86,80,91,80,101,
+ 144,57,93,100,91,55,105,70, 103,96,85,94,127,129,150,50, 91,114,94,98,152,63,85,51, 96,92,89,120,109,108,123,111,
+ 121,154,45,60,63,64,68,39, 59,85,56,120,31,120,40,125, 43,86,21,83,136,120,56,36, 80,36,43,124,106,113,123,115,
+
+ 121,150,74,14,27,151,56,36, 50,150,69,55,34,102,58,106, 39,82,28,56,34,160,40,31, 75,45,39,131,101,109,119,112,
+ 104,54,77,132,55,34,31,18, 17,42,46,106,0,82,44,80, 36,49,0,155,40,40,6,33, 43,19,25,91,101,99,89,114,
+ 112,141,38,57,37,72,43,25, 101,67,64,150,25,96,40,152, 26,69,15,132,116,149,27,18, 62,32,26,112,97,105,106,108,
+ 78,60,127,91,126,42,111,125, 87,96,141,103,122,124,150,40, 122,106,129,108,108,54,120,151, 152,40,108,78,73,81,74,102,
+ 140,74,110,87,99,70,89,78, 91,102,128,88,139,149,142,104, 123,104,100,110,153,97,117,66, 94,57,111,124,100,129,123,119,
+ 112,104,136,78,126,105,142,117, 127,127,143,142,137,115,105,119, 116,69,120,135,123,76,130,29, 62,18,144,114,88,105,106,108,
+ 82,72,125,103,119,67,146,134, 136,96,134,136,126,124,132,75, 129,68,147,127,116,53,133,80, 97,119,135,74,72,91,76,106,
+ 131,76,81,61,69,61,88,82, 92,73,84,76,75,69,78,66, 81,110,61,67,61,85,87,66, 132,71,67,154,139,145,139,146,
+
+ 84,133,98,52,42,158,37,23, 89,160,53,107,63,86,87,104, 70,38,30,103,87,98,78,0, 32,26,51,80,58,80,75,91,
+ 95,14,130,98,109,14,63,29, 116,13,156,109,122,149,89,39, 70,49,106,128,105,19,149,25, 43,4,137,91,82,95,86,110,
+ 109,107,159,107,134,73,101,102, 119,89,132,114,134,142,131,80, 129,98,112,142,121,59,127,27, 103,51,123,138,94,108,99,102,
+ 123,108,129,80,86,116,133,102, 135,85,156,146,124,112,121,59, 80,90,122,135,124,64,156,40, 93,40,143,127,110,116,125,119,
+ 102,64,148,123,112,87,139,131, 152,77,79,129,128,132,133,45, 111,62,143,119,118,22,113,35, 90,119,135,104,92,99,72,104,
+ 109,48,136,138,115,29,99,66, 134,49,121,124,106,149,104,46, 131,63,124,123,115,103,132,19, 63,78,126,96,88,105,101,112,
+ 117,136,38,70,20,133,49,30, 43,174,47,36,24,27,9,70, 34,75,10,15,28,140,32,25, 69,25,32,117,95,106,112,109,
+ 136,54,77,34,51,77,76,61, 86,49,60,57,54,66,64,51, 69,93,44,77,50,54,57,55, 81,42,72,112,121,115,118,126,
+};
+
+static const uint8 ced_hires_15[1024] = {
+ 128,67,142,105,122,81,101,108, 115,105,99,113,90,139,98,75, 108,127,69,108,148,90,86,94, 122,79,127,128,128,128,128,128,
+ 69,43,125,125,121,45,135,139, 120,81,130,123,135,133,130,54, 122,137,129,135,128,100,144,30, 140,48,139,91,84,84,66,98,
+ 115,65,136,131,106,59,112,137, 120,95,82,98,123,154,142,78, 100,151,107,96,141,104,93,66, 72,112,103,112,105,119,121,120,
+ 117,20,88,109,109,87,64,100, 48,105,89,13,112,140,99,63, 140,65,134,113,124,141,95,29, 57,14,123,101,83,100,94,108,
+ 68,54,95,122,126,81,139,144, 148,111,115,117,137,130,132,75, 111,92,136,126,133,135,124,37, 131,109,87,85,62,73,66,102,
+ 128,93,149,66,99,65,86,92, 121,64,84,111,119,76,106,76, 146,112,85,74,81,85,71,79, 108,70,121,126,126,117,128,122,
+ 88,121,101,75,36,126,91,31, 42,131,51,91,40,109,96,83, 71,59,25,36,46,125,25,131, 41,30,50,84,85,105,85,117,
+ 121,147,107,57,49,130,60,66, 85,145,91,87,124,117,41,145, 66,61,60,65,99,124,67,55, 72,71,35,119,87,122,89,133,
+
+ 57,127,84,73,58,136,67,51, 67,142,118,140,126,83,133,119, 104,49,103,93,122,126,86,38, 19,45,73,63,77,81,56,108,
+ 65,103,122,132,130,120,140,134, 129,89,118,96,126,133,115,114, 137,152,130,134,126,97,128,34, 141,25,99,63,86,95,76,104,
+ 69,0,128,145,144,0,23,138, 6,23,47,135,81,30,27,14, 146,42,19,110,129,14,0,100, 64,0,137,63,57,87,67,101,
+ 115,62,85,72,89,93,112,81, 86,120,86,87,149,100,138,56, 85,132,119,108,113,72,114,73, 126,63,117,122,116,111,127,118,
+ 60,9,84,117,134,0,29,27, 131,18,154,129,137,121,128,35, 102,31,120,130,138,19,142,14, 24,0,126,67,63,101,61,91,
+ 65,137,116,137,129,65,129,107, 127,33,107,123,116,135,116,113, 115,135,103,126,128,63,126,20, 98,12,122,77,75,84,70,100,
+ 130,51,88,108,93,49,100,69, 98,89,80,89,129,128,146,44, 88,110,92,95,149,58,81,63, 91,88,91,114,104,105,116,110,
+ 111,148,40,69,66,57,65,38, 55,77,52,115,34,119,36,120, 41,83,19,79,133,115,53,47, 75,32,45,118,101,111,112,114,
+
+ 109,144,70,22,29,145,51,35, 45,143,64,50,37,101,54,101, 36,78,28,54,31,155,37,42, 71,41,40,124,96,106,108,111,
+ 84,47,72,141,57,27,25,15, 12,34,40,100,1,81,40,74, 33,45,0,151,36,34,1,42, 38,14,26,81,95,91,77,113,
+ 96,135,34,65,40,66,38,23, 96,60,59,145,27,95,36,147, 23,65,15,129,113,144,24,29, 58,28,27,104,92,101,95,107,
+ 59,54,122,100,129,35,107,124, 82,88,137,98,123,123,146,35, 120,103,127,104,105,49,116,162, 147,36,110,70,68,74,63,101,
+ 123,67,105,95,101,63,84,76, 86,94,123,82,140,147,138,98, 120,100,98,106,149,91,112,76, 89,52,112,117,95,126,112,118,
+ 96,98,131,86,128,99,137,116, 122,120,138,137,139,114,101,114, 113,65,119,132,120,71,126,41, 58,14,145,105,83,101,95,107,
+ 63,66,120,112,122,60,142,133, 131,88,130,131,127,123,128,70, 127,65,145,123,113,48,129,91, 92,115,137,65,68,84,65,105,
+ 132,70,76,69,72,54,84,81, 87,66,80,71,77,68,74,62, 78,106,61,63,58,81,84,77, 127,67,68,155,136,142,140,145,
+
+ 65,127,93,61,45,151,33,22, 84,152,49,102,64,85,83,99, 68,35,28,99,84,93,74,2, 27,22,53,71,53,74,64,90,
+ 76,8,125,107,112,8,59,28, 111,6,152,104,123,148,85,34, 68,46,104,124,102,13,145,36, 38,2,139,82,77,88,75,110,
+ 90,101,154,116,137,66,97,101, 114,81,128,109,135,141,127,75, 127,95,110,138,118,54,123,38, 98,47,125,130,89,101,88,102,
+ 117,102,124,88,89,109,129,101, 130,77,152,141,125,111,117,55, 79,87,120,131,121,58,152,51, 90,36,145,122,105,113,116,118,
+ 83,58,143,132,115,81,134,130, 147,70,74,124,129,131,129,40, 109,59,141,116,115,18,109,46, 85,115,137,95,87,92,61,103,
+ 90,42,131,147,118,23,95,65, 129,41,117,119,107,148,100,41, 129,60,122,119,112,98,128,30, 59,74,128,87,83,99,90,112,
+ 102,130,34,79,23,126,45,29, 38,166,43,32,26,27,5,65, 30,72,8,11,24,135,28,36, 64,21,34,108,90,103,101,108,
+ 121,48,73,42,53,70,71,61, 81,41,56,52,55,65,60,46, 67,89,42,73,48,49,53,67, 76,38,74,106,116,112,115,125,
+};
+
+static const uint8 ced_hires_16[1024] = {
+ 126,77,94,99,85,77,105,111, 93,98,106,78,81,99,92,92, 100,116,99,119,82,86,89,94, 132,90,87,127,125,126,126,131,
+ 97,47,126,130,125,63,130,137, 117,95,136,122,124,126,123,57, 128,120,120,134,123,99,146,65, 132,56,132,97,96,97,97,110,
+ 143,93,132,131,106,93,114,132, 110,112,123,95,107,141,130,103, 112,132,92,86,131,103,106,111, 132,107,103,143,142,143,143,130,
+ 128,105,112,105,109,106,116,109, 113,112,120,113,104,107,99,110, 114,124,100,102,105,111,112,115, 125,111,116,128,128,128,128,128,
+ 97,58,96,127,131,95,134,142, 146,126,121,116,127,123,126,86, 117,87,128,126,128,135,126,65, 123,114,81,98,96,97,97,110,
+ 157,107,120,109,115,107,128,115, 124,121,137,109,95,107,86,117, 126,146,89,100,96,117,120,125, 146,116,117,157,156,157,157,139,
+ 157,107,120,109,115,107,128,115, 124,121,137,109,95,107,86,117, 126,146,89,100,96,117,120,125, 146,116,117,157,156,157,157,139,
+ 120,155,109,72,78,142,91,78, 87,159,100,85,112,108,49,156, 89,109,52,63,93,123,83,88, 109,79,80,120,119,120,120,118,
+
+ 119,70,107,128,115,75,90,135, 86,88,99,71,105,140,107,80, 93,159,136,138,115,79,136,87, 118,78,88,120,118,119,119,118,
+ 92,112,123,137,135,133,135,132, 127,104,124,95,116,126,108,126, 143,137,121,134,121,97,130,60, 133,51,92,93,91,92,92,106,
+ 130,80,93,121,88,80,101,88, 97,94,110,82,83,142,131,90, 104,119,62,128,144,100,121,98, 119,89,90,130,129,130,130,123,
+ 148,98,111,100,106,98,119,106, 115,127,128,100,131,98,124,108, 117,137,102,96,97,108,111,116, 137,107,108,148,147,148,148,135,
+ 149,100,112,101,108,100,120,107, 116,113,129,101,88,99,99,110, 118,139,82,93,88,109,112,117, 138,108,110,150,148,149,149,139,
+ 98,145,117,142,133,78,123,105, 125,62,112,122,105,128,109,124, 120,119,94,125,123,58,128,66, 97,57,115,98,97,98,98,112,
+ 145,96,108,110,104,96,116,103, 112,109,125,97,110,112,132,106, 114,135,78,89,137,105,108,113, 134,104,106,146,144,145,145,132,
+ 153,111,124,118,134,108,124,111, 120,117,133,105,101,103,113,113, 122,159,85,115,115,113,116,121, 142,112,113,153,152,153,153,136,
+
+ 128,112,119,112,116,113,123,116, 120,119,127,120,111,114,106,117, 120,128,106,108,111,117,118,121, 128,117,122,128,128,128,128,128,
+ 120,158,83,72,78,121,91,78, 87,121,100,72,58,70,49,172, 88,108,51,62,58,95,82,87, 108,78,79,119,118,119,119,122,
+ 146,96,109,98,104,96,117,120, 113,110,126,98,94,116,108,106, 130,134,109,118,84,105,108,113, 134,104,105,145,144,145,145,137,
+ 98,58,122,104,132,48,101,121, 75,101,142,96,111,115,138,58, 126,86,118,103,100,57,118,161, 139,56,102,97,96,97,97,111,
+ 139,89,102,96,102,89,110,97, 106,103,124,91,124,135,126,104, 122,127,80,99,141,98,106,106, 127,97,98,138,137,138,138,129,
+ 155,105,128,107,113,105,126,113, 122,119,135,107,93,105,84,115, 133,143,86,97,93,114,117,122, 143,113,114,154,153,154,154,139,
+ 99,72,121,117,126,72,137,131, 129,103,136,130,116,116,121,79, 134,87,137,123,109,58,132,89, 87,121,131,98,97,98,98,111,
+ 154,104,117,106,112,104,125,112, 121,118,134,106,92,104,83,114, 122,142,85,96,92,113,116,121, 142,112,113,153,152,153,153,139,
+
+ 156,106,119,108,114,106,127,114, 123,120,136,108,94,106,85,116, 124,144,87,98,94,115,118,123, 144,114,115,155,154,155,155,140,
+ 150,101,113,102,109,101,121,108, 117,114,130,102,89,100,80,111, 118,139,82,93,88,109,112,117, 138,108,110,150,148,149,149,140,
+ 118,107,154,120,140,68,89,96, 111,92,132,106,123,133,119,78, 133,106,101,137,113,77,125,85, 106,76,118,122,116,117,117,121,
+ 149,99,112,101,107,99,120,107, 116,113,129,101,102,99,83,109, 117,137,115,96,139,108,111,116, 137,107,108,148,147,148,148,135,
+ 102,62,144,137,119,94,129,128, 144,82,82,123,118,124,122,62, 116,90,133,116,111,61,112,69, 90,121,131,101,100,101,101,110,
+ 156,106,119,108,114,106,127,114, 123,120,136,108,94,106,85,116, 124,144,87,98,94,115,118,123, 144,114,115,155,154,155,155,140,
+ 156,106,119,108,114,106,127,114, 123,120,136,108,94,106,85,116, 124,144,87,98,94,115,118,123, 144,114,115,155,154,155,155,140,
+ 137,88,100,89,96,96,108,95, 104,101,117,89,76,87,67,98, 105,126,69,80,75,96,99,104, 125,95,97,137,135,136,136,123,
+};
+
+static const uint8 ced_hires_17[1024] = {
+ 89,91,92,133,130,37,125,74, 42,95,144,140,137,137,121,42, 123,75,132,124,133,87,128,83, 74,53,119,94,87,91,93,108,
+ 102,61,121,126,115,65,141,137, 130,96,110,126,137,135,142,73, 128,143,129,129,127,116,147,95, 154,82,141,107,100,103,106,114,
+ 152,89,132,131,102,84,114,134, 128,107,81,104,125,155,153,95, 103,154,108,87,139,120,95,146, 137,146,100,157,150,154,156,136,
+ 130,66,62,72,62,122,86,62, 83,84,58,60,65,91,78,190, 73,115,92,83,85,91,68,123, 115,94,83,135,128,131,134,127,
+ 103,75,93,124,121,103,146,143, 159,127,96,121,140,133,145,96, 118,98,137,121,133,153,128,96, 146,146,90,108,101,104,107,114,
+ 111,83,105,33,129,120,114,135, 102,75,39,131,136,117,141,54, 122,96,138,110,121,62,115,104, 96,75,125,116,109,112,115,120,
+ 131,94,108,84,119,91,141,148, 104,99,90,137,142,121,136,74, 104,117,149,128,125,99,131,125, 116,95,74,136,129,133,135,128,
+ 109,71,69,85,106,85,106,131, 62,57,93,110,101,126,73,52, 116,94,127,130,102,60,143,102, 94,73,52,114,107,118,113,119,
+
+ 100,147,79,74,50,157,72,48, 76,157,98,143,128,85,145,139, 110,85,103,87,121,143,89,93, 85,84,74,105,98,101,104,112,
+ 97,124,119,134,125,141,147,132, 139,104,99,100,129,135,127,134, 143,159,130,128,125,115,131,91, 155,61,101,102,95,99,101,109,
+ 101,37,124,147,139,33,57,137, 54,39,29,139,83,34,39,44, 153,86,16,104,129,52,39,151, 86,65,140,106,99,102,105,116,
+ 159,95,91,81,81,113,115,91, 112,133,87,89,150,97,149,102, 102,144,118,100,111,110,113,152, 144,123,117,164,157,160,163,142,
+ 86,23,75,47,132,18,43,105, 39,54,141,105,128,124,110,29, 89,72,105,147,131,38,115,80, 71,50,78,91,84,88,90,107,
+ 103,158,113,138,124,86,135,105, 137,41,88,127,119,137,128,133, 120,142,104,120,127,81,129,97, 111,67,124,108,101,105,107,116,
+ 156,93,94,108,84,88,113,88, 109,99,85,92,130,128,156,99, 99,142,88,85,147,108,94,150, 141,120,99,161,154,158,160,138,
+ 98,50,162,115,131,30,54,149, 51,46,135,121,106,128,119,41, 135,83,109,116,138,49,147,91, 83,62,136,103,96,99,102,111,
+
+ 160,171,92,82,82,172,116,92, 113,164,88,90,85,109,82,129, 97,139,69,66,71,175,92,147, 139,118,97,159,152,155,158,137,
+ 129,176,77,84,94,159,85,102, 82,160,57,105,70,85,71,176, 89,108,48,73,73,173,81,116, 108,87,76,128,121,124,127,122,
+ 180,139,119,109,109,119,143,119, 140,135,125,117,112,120,109,130, 124,166,112,93,98,160,119,174, 166,145,124,180,179,177,180,152,
+ 156,139,88,78,116,166,138,88, 109,177,94,86,81,89,111,115, 93,135,95,88,77,162,98,171, 135,114,93,155,148,151,154,133,
+ 156,92,106,103,101,88,112,88, 114,119,110,91,149,156,157,125, 129,135,101,102,152,114,117,143, 135,114,116,155,148,151,154,133,
+ 152,109,131,74,109,171,109,112, 159,167,115,117,135,115,136,115, 148,132,118,103,119,157,112,140, 131,110,109,151,144,148,150,134,
+ 112,94,124,121,124,90,156,139, 149,111,118,142,137,133,148,98, 137,91,149,121,116,67,136,145, 107,155,143,111,104,107,110,113,
+ 180,119,115,104,105,114,139,114, 135,120,111,113,108,115,104,125, 119,162,92,88,93,128,114,170, 161,140,119,178,174,174,178,149,
+
+ 128,111,146,50,134,118,137,150, 112,103,137,155,135,132,138,87, 135,108,147,113,118,105,137,126, 107,169,65,127,120,124,126,122,
+ 109,46,52,31,32,41,66,41, 62,47,127,60,65,52,41,52, 46,89,19,130,30,55,41,97, 88,67,66,108,101,105,107,132,
+ 131,129,158,124,139,93,111,106, 131,102,116,120,145,150,146,102, 136,111,114,135,120,77,129,119, 110,89,130,146,123,127,129,123,
+ 171,107,103,101,93,108,127,103, 124,109,99,114,130,122,116,114, 108,150,135,102,155,116,111,158, 150,129,108,170,163,166,169,141,
+ 115,87,147,141,117,109,149,136, 164,92,64,135,139,141,148,71, 119,94,145,113,118,60,116,102, 99,155,143,114,107,110,113,112,
+ 180,125,121,120,111,120,145,120, 141,126,117,119,130,121,126,131, 125,168,98,94,99,134,120,176, 167,146,125,180,179,177,180,152,
+ 108,45,99,103,139,40,96,117, 61,87,85,145,139,127,138,51, 127,88,152,133,136,64,131,96, 87,66,135,107,100,104,106,112,
+ 157,94,95,80,80,109,114,90, 119,96,86,88,83,91,80,101, 95,137,67,64,69,103,90,145, 137,116,95,154,150,150,154,125,
+};
+
+static const uint8 ced_hires_18[1024] = {
+ 100,47,105,106,88,37,119,122, 76,90,99,47,87,103,100,98, 95,121,106,126,89,84,80,50, 146,101,49,119,100,126,121,130,
+ 74,55,134,135,131,65,138,145, 126,83,147,122,129,128,131,71, 130,133,126,138,131,103,151,36, 145,59,129,97,84,83,73,90,
+ 123,78,145,142,117,80,115,143, 127,97,99,98,117,149,143,95, 108,146,104,100,145,107,101,72, 83,123,94,119,107,123,126,111,
+ 102,42,63,81,78,121,44,64, 50,73,66,32,62,84,68,187, 77,62,89,92,89,80,66,40, 63,66,76,102,89,103,100,106,
+ 72,67,105,133,136,102,142,150, 155,114,132,117,131,125,133,93, 119,88,133,130,137,139,132,46, 137,120,78,89,64,76,73,94,
+ 94,75,117,44,144,118,110,142, 98,61,78,127,127,109,129,55, 123,48,134,119,125,45,118,38, 80,35,112,83,72,101,83,96,
+ 95,87,122,95,134,91,137,156, 101,88,126,133,133,114,125,79, 107,76,145,138,130,85,136,69, 79,46,38,117,85,112,100,104,
+ 124,160,116,69,60,150,64,73, 92,147,109,87,118,112,42,163, 75,62,57,68,103,128,75,63, 80,83,28,120,89,124,96,125,
+
+ 94,66,118,135,120,83,97,144, 63,76,82,62,110,143,115,69, 85,171,142,143,125,71,143,43, 138,58,89,97,78,95,93,94,
+ 67,116,131,143,140,140,143,140, 136,91,135,96,120,128,115,131, 145,148,127,138,130,101,135,42, 146,37,89,70,86,94,79,96,
+ 101,82,86,128,82,78,65,84, 68,60,103,72,89,145,141,49, 109,135,73,134,155,108,128,81, 135,61,70,108,97,107,117,96,
+ 123,75,94,83,100,113,116,87, 92,122,102,87,143,94,139,71, 93,129,116,111,117,75,121,80, 131,72,108,126,118,118,133,111,
+ 125,126,107,111,110,90,98,100, 104,78,136,102,98,121,122,116, 113,113,88,103,108,91,109,90, 113,82,103,166,118,129,154,121,
+ 71,150,125,148,139,85,131,113, 134,35,124,123,110,130,117,130, 123,131,100,130,132,66,133,26, 103,24,112,82,78,81,75,92,
+ 134,66,98,119,103,70,104,77, 104,91,98,89,123,122,146,64, 97,108,89,98,153,65,88,72, 98,100,82,121,109,113,126,105,
+ 134,123,148,135,153,131,131,112, 86,89,109,114,122,118,134,113, 106,183,104,135,137,97,127,140, 120,93,88,129,122,120,132,113,
+
+ 95,138,100,94,128,146,84,59, 64,158,26,72,140,108,64,96, 48,37,129,66,66,162,79,20, 36,60,93,75,70,90,70,98,
+ 95,163,41,47,45,129,41,47, 56,110,67,38,37,31,25,179, 45,69,19,50,32,105,52,43, 62,51,33,101,100,101,98,98,
+ 122,92,117,103,114,78,115,136, 82,95,96,73,110,123,125,87, 143,128,130,139,99,92,109,87, 147,65,68,148,119,143,129,127,
+ 68,66,131,110,138,55,109,130, 88,90,153,97,117,117,146,52, 134,105,130,114,115,58,129,176, 158,53,106,82,71,82,78,96,
+ 129,80,114,106,112,83,88,82, 92,96,140,82,134,142,139,116, 135,103,101,116,160,101,127,91, 102,70,110,124,105,134,125,113,
+ 123,95,135,69,116,163,102,112, 149,147,143,105,119,100,118,101, 153,80,118,115,127,146,119,62, 116,70,102,133,102,113,117,109,
+ 70,78,129,122,132,80,145,139, 137,90,147,130,121,118,129,87, 141,67,148,133,123,58,143,105, 104,132,134,80,74,95,79,100,
+ 136,82,86,79,83,76,90,87, 92,67,102,69,70,65,74,80, 92,115,66,71,68,91,96,94, 140,85,71,156,136,150,142,140,
+
+ 104,96,151,58,141,110,125,150, 102,82,165,143,118,117,119,80, 140,61,146,125,125,94,144,85, 85,147,50,100,110,107,95,109,
+ 122,103,115,106,89,90,88,87, 92,99,110,70,86,100,108,86, 107,107,86,104,98,85,106,87, 134,101,84,138,122,134,133,128,
+ 93,113,163,126,146,86,99,107, 120,83,144,108,129,135,127,92, 141,96,113,148,128,63,136,50, 109,64,121,140,95,106,100,97,
+ 128,86,112,109,106,102,108,101, 111,75,126,105,117,111,101,66, 104,108,137,116,165,81,125,82, 114,102,102,129,126,121,134,116,
+ 89,71,152,143,125,101,137,136, 153,72,92,124,123,126,129,57, 123,62,145,126,125,29,123,61, 97,133,134,107,94,103,79,98,
+ 99,71,108,117,119,66,122,99, 108,25,83,132,126,131,130,88, 120,83,124,123,119,71,105,40, 88,140,139,89,119,109,90,120,
+ 109,146,119,71,82,143,99,110, 107,147,49,135,131,127,53,105, 80,80,69,92,140,135,125,42, 60,114,37,69,92,91,80,109,
+ 129,78,95,77,81,106,104,83, 99,66,82,66,73,73,63,80, 101,97,69,88,80,74,80,98, 98,65,64,114,130,119,121,120,
+};
+
+static const uint8 ced_hires_19[1024] = {
+ 98,93,104,135,134,69,120,71, 84,99,150,144,141,141,121,77, 123,96,135,127,139,85,130,83, 94,88,120,97,97,97,97,113,
+ 110,85,134,128,119,81,138,142, 127,104,115,130,141,138,142,90, 128,119,133,131,132,117,148,95, 129,101,142,109,109,109,109,119,
+ 145,120,126,112,107,116,133,123, 131,123,115,115,110,142,136,124, 123,143,99,95,128,132,120,130, 141,135,125,144,144,144,144,134,
+ 134,109,115,93,96,120,122,107, 120,112,104,104,99,106,91,186, 112,132,88,84,92,121,109,119, 130,124,114,133,133,133,133,130,
+ 111,86,102,124,125,107,142,148, 156,131,104,126,144,136,144,90, 119,109,140,123,138,153,129,96, 122,136,91,110,110,110,110,118,
+ 118,93,119,78,133,123,107,139, 104,97,88,134,139,118,140,98, 122,117,141,111,125,105,113,103, 114,109,123,117,117,117,117,124,
+ 135,110,116,95,117,106,124,148, 121,114,105,136,141,117,132,115, 114,134,148,125,126,122,125,120, 131,126,115,134,134,134,134,130,
+ 117,92,98,86,109,98,105,136, 103,95,97,112,105,129,74,96, 110,115,130,131,106,104,144,102, 113,107,97,116,116,116,116,123,
+
+ 108,150,89,67,70,160,96,81, 94,161,103,147,132,80,144,139, 109,106,105,88,126,143,83,93, 104,98,88,107,107,107,107,117,
+ 106,127,132,135,129,144,143,138, 136,107,104,103,133,139,126,134, 143,136,133,131,131,114,133,91, 130,96,102,105,105,105,105,114,
+ 109,84,137,148,143,80,97,142, 95,87,79,143,84,81,66,88, 152,107,63,106,134,96,84,104, 105,99,141,108,108,108,108,120,
+ 146,121,127,106,108,117,135,119, 132,125,116,116,133,118,126,126, 125,145,101,96,104,133,121,131, 142,137,126,145,145,145,145,136,
+ 95,70,86,54,136,66,83,111, 81,73,147,109,132,128,110,74, 88,93,108,151,137,82,117,80, 91,85,75,94,94,94,94,111,
+ 112,161,124,140,127,83,131,110, 135,90,92,132,123,141,128,133, 120,110,106,123,133,99,130,97, 108,102,125,111,111,111,111,121,
+ 145,120,126,105,107,116,134,118, 131,124,115,115,115,117,137,125, 124,144,100,95,132,132,120,130, 141,136,125,144,144,144,144,135,
+ 148,123,129,107,115,119,136,121, 134,126,118,118,113,120,115,127, 126,146,102,103,111,135,123,133, 144,138,128,147,147,147,147,136,
+
+ 147,153,128,106,109,118,135,120, 133,153,117,117,112,119,104,126, 126,146,102,108,106,165,123,133, 144,138,128,147,147,147,147,136,
+ 132,174,113,92,94,156,121,105, 118,160,102,102,97,104,90,172, 112,132,88,83,91,175,108,118, 129,124,113,132,132,132,132,127,
+ 152,127,133,112,114,123,141,125, 138,131,122,122,117,124,110,132, 132,152,108,103,111,140,128,138, 149,144,133,152,152,152,152,138,
+ 117,92,136,107,132,88,105,132, 103,110,127,110,134,133,162,96, 133,116,138,108,116,105,127,176, 144,108,118,117,117,117,117,120,
+ 148,123,129,107,110,119,136,121, 134,126,118,118,135,141,139,127, 127,147,103,99,147,136,124,134, 145,139,129,148,148,148,148,134,
+ 148,123,129,107,110,162,136,121, 144,159,118,118,123,120,120,127, 142,147,113,99,117,151,124,134, 145,139,129,148,148,148,148,135,
+ 117,92,133,120,125,98,149,141, 144,116,122,143,138,132,145,97, 142,117,157,127,125,105,141,103, 114,150,148,117,117,117,117,120,
+ 153,128,134,112,115,124,141,126, 139,131,123,123,118,125,110,132, 132,152,108,104,112,141,129,139, 150,144,134,153,153,153,153,138,
+
+ 132,107,153,92,134,113,121,151, 118,111,137,154,135,129,133,112, 140,132,153,117,125,120,141,118, 129,163,113,132,132,132,132,127,
+ 113,88,94,73,75,84,102,86, 99,92,127,83,78,85,71,93, 93,113,69,135,72,101,89,99, 110,105,94,113,113,113,113,129,
+ 135,125,166,120,137,106,124,108, 121,114,115,115,143,149,142,115, 138,135,119,140,128,123,131,121, 132,127,131,135,135,135,135,129,
+ 151,126,132,111,113,122,140,124, 137,130,121,121,116,123,109,131, 131,151,114,102,138,139,127,137, 148,143,132,151,151,151,151,137,
+ 120,95,157,140,118,108,142,138, 158,99,90,136,140,141,145,100, 121,120,153,120,127,108,119,106, 117,149,148,120,120,120,120,119,
+ 136,111,140,151,118,107,124,109, 137,114,106,126,111,155,113,115, 138,135,129,118,120,124,137,122, 133,127,132,136,136,136,136,131,
+ 151,146,132,110,113,145,139,124, 137,154,121,121,116,123,108,130, 130,150,106,102,110,139,127,137, 148,142,132,151,151,151,151,137,
+ 146,121,127,105,108,117,134,119, 132,124,116,116,111,118,103,125, 125,145,101,97,105,134,122,132, 143,137,127,146,146,146,146,131,
+};
+
+static const uint8 ced_hires_20[1024] = {
+ 111,51,50,66,67,37,61,60, 62,52,151,76,117,107,89,44, 64,85,123,112,43,48,109,55, 80,47,73,105,103,120,107,126,
+ 99,78,66,40,113,81,103,136, 51,33,85,110,79,133,68,53, 121,80,125,133,107,49,144,49, 79,46,58,104,94,114,94,119,
+ 82,89,103,144,141,45,125,85, 60,80,142,139,139,140,118,46, 133,65,134,127,134,77,126,29, 65,28,130,79,73,84,74,99,
+ 148,112,111,112,105,115,135,114, 133,109,103,112,104,109,99,132, 123,145,97,101,94,117,110,138, 144,123,128,148,147,147,148,137,
+ 97,70,96,138,128,103,151,149, 163,109,95,119,143,133,146,96, 122,107,138,122,134,143,125,67, 141,129,98,104,95,92,94,114,
+ 112,73,108,62,136,120,118,141, 105,57,53,129,139,116,142,70, 126,92,139,111,122,64,111,67, 100,63,134,104,102,110,102,119,
+ 126,53,74,82,77,64,78,76, 74,51,58,72,68,60,68,61, 85,102,50,148,65,58,59,77, 97,64,90,122,116,120,117,129,
+ 94,50,95,141,134,56,69,127, 60,37,151,133,141,146,122,58, 115,74,138,121,145,52,105,44, 78,35,128,96,85,89,89,105,
+
+ 93,143,83,89,63,158,83,60, 83,139,98,142,132,86,147,141, 115,84,105,89,123,134,85,67, 69,69,82,94,100,99,89,116,
+ 92,119,122,149,133,142,152,139, 144,87,98,98,132,136,128,136, 148,160,131,130,127,105,129,63, 151,58,111,91,104,103,95,113,
+ 148,106,116,121,106,102,130,108, 127,89,97,117,103,117,112,113, 133,145,97,94,107,111,109,132, 144,121,140,148,147,147,148,137,
+ 96,26,70,54,136,23,56,118, 57,39,143,109,136,129,114,24, 102,75,104,149,132,26,115,40, 62,32,99,95,91,85,89,104,
+ 122,116,63,61,63,156,88,58, 77,162,52,66,68,77,80,112, 84,106,43,53,46,123,55,75, 100,67,77,125,117,120,120,120,
+ 137,128,86,79,104,158,111,95, 93,154,68,90,84,84,82,113, 113,121,63,81,93,155,94,91, 115,82,99,136,132,131,136,128,
+ 91,40,166,130,139,40,74,156, 53,33,137,119,108,132,120,47, 139,70,109,117,140,49,148,47, 74,31,142,89,83,85,84,103,
+ 113,165,84,84,91,133,107,85, 101,142,80,94,94,82,76,158, 92,97,86,89,67,171,112,67, 91,63,89,116,108,115,112,117,
+
+ 84,151,102,50,102,126,72,103, 59,149,74,148,115,116,104,148, 125,78,106,107,133,146,137,43, 60,158,87,85,79,92,80,104,
+ 115,72,78,158,69,65,71,53, 67,50,47,103,38,82,62,99, 72,98,29,152,56,69,47,84, 93,67,72,118,125,119,113,127,
+ 116,167,96,89,95,156,80,88, 85,135,64,95,66,79,70,161, 97,98,65,84,85,166,84,67, 92,59,94,117,117,113,112,116,
+ 95,66,122,116,132,60,119,129, 93,85,117,101,129,126,160,66, 136,118,134,105,111,69,122,180, 161,71,127,101,101,101,96,115,
+ 148,107,121,119,120,108,132,120, 135,112,126,122,123,123,121,115, 140,147,122,115,120,131,127,129, 146,120,138,149,149,149,149,138,
+ 145,100,125,89,108,164,120,114, 156,141,106,107,130,110,130,109, 154,126,119,105,123,147,113,96, 136,98,123,146,132,131,136,131,
+ 96,81,121,129,125,83,154,139, 147,86,111,134,134,126,142,92, 143,98,152,124,120,68,135,108, 111,140,154,101,101,106,99,117,
+ 148,101,97,103,91,93,119,102, 119,82,86,90,96,95,97,105, 112,145,87,80,83,109,104,119, 153,110,113,160,153,158,154,146,
+
+ 96,28,42,33,46,35,53,45, 49,31,126,40,50,53,52,43, 68,80,39,131,35,47,41,50, 74,51,47,99,104,105,97,134,
+ 148,137,108,111,112,125,128,118, 139,87,95,116,95,112,110,149, 117,147,86,88,103,140,105,130, 146,146,127,149,149,149,149,138,
+ 148,129,121,142,101,121,132,114, 128,114,99,114,126,128,120,128, 126,147,104,99,97,132,116,156, 146,121,126,149,149,149,149,138,
+ 94,39,97,117,138,45,88,117, 67,38,75,139,136,117,133,55, 139,78,153,138,143,54,128,55, 79,53,151,97,89,100,92,107,
+ 108,73,144,149,117,103,147,136, 162,65,59,127,136,135,143,70, 126,97,148,117,122,55,115,79, 109,140,154,117,114,112,100,115,
+ 148,128,120,108,115,137,133,110, 129,119,99,112,110,115,116,137, 127,147,98,107,93,131,122,130, 146,149,127,149,149,149,149,138,
+ 90,141,92,62,123,145,98,94, 71,157,54,115,84,131,112,151, 121,73,99,126,103,146,129,51, 68,144,73,93,87,100,88,107,
+ 137,80,87,78,75,97,105,85, 110,75,63,75,82,82,81,90, 99,126,66,86,72,87,81,105, 118,85,106,131,142,134,137,133,
+};
+
+static const uint8 ced_hires_21[1024] = {
+ 83,35,83,86,68,31,99,99, 55,99,74,45,80,93,87,75, 79,100,91,111,76,86,61,33, 125,101,49,100,98,111,105,129,
+ 59,43,112,116,111,56,117,121, 105,92,122,118,122,117,117,48, 113,112,111,124,119,105,131,14, 124,58,128,80,75,64,53,89,
+ 102,65,123,122,96,71,94,119, 105,106,73,93,110,138,129,71, 91,126,89,85,132,109,80,55, 63,122,92,108,98,117,104,108,
+ 87,29,40,62,58,112,27,40, 29,82,41,27,55,73,54,164, 60,39,74,78,77,82,45,14, 40,65,75,85,83,89,81,104,
+ 56,54,82,113,116,92,121,126, 133,122,107,112,124,114,119,69, 102,67,118,115,124,140,111,23, 115,119,76,70,57,61,58,92,
+ 78,62,95,25,124,109,90,119, 76,70,53,123,120,99,116,31, 107,29,119,104,112,46,98,15, 58,34,111,65,65,82,61,97,
+ 81,74,99,75,114,81,117,133, 79,96,101,129,126,104,112,55, 91,50,130,123,117,86,115,48, 57,41,32,98,76,95,82,105,
+ 107,147,94,49,39,141,43,49, 70,156,84,83,111,101,28,140, 58,35,42,54,90,129,55,41, 53,82,24,102,79,105,77,124,
+
+ 75,53,95,115,100,73,77,121, 43,85,59,58,103,133,102,46, 69,150,127,128,112,72,122,25, 117,58,87,80,75,79,76,97,
+ 52,103,109,123,121,131,123,117, 114,100,110,92,113,118,102,108, 129,127,111,123,117,103,115,21, 125,36,88,52,79,81,65,95,
+ 87,70,63,109,62,69,46,61, 46,69,77,67,83,135,128,27, 93,114,58,120,143,110,108,59, 114,61,69,92,90,96,100,95,
+ 109,61,71,65,81,104,96,64, 70,131,79,83,136,84,125,49, 77,107,101,97,105,80,101,61, 111,75,106,115,115,112,116,109,
+ 112,114,84,92,90,82,79,75, 83,89,111,97,91,110,108,93, 97,93,73,90,96,92,89,68, 88,77,101,154,114,122,143,121,
+ 53,137,103,128,119,76,111,90, 112,44,99,119,103,120,104,107, 107,110,85,115,119,68,113,4, 82,21,111,63,72,62,54,91,
+ 115,54,75,100,84,61,84,54, 83,100,70,85,116,112,133,43, 80,88,74,84,141,60,68,52, 81,99,81,113,103,108,109,105,
+ 120,111,126,115,133,122,111,89, 67,99,81,110,116,108,120,90, 90,163,89,121,125,100,107,119, 103,94,88,124,116,114,122,112,
+
+ 102,136,101,31,69,99,109,94, 95,151,120,85,100,94,78,59, 51,53,85,107,69,157,101,38, 64,67,59,99,103,108,95,116,
+ 79,150,19,27,24,119,22,24, 34,118,44,32,28,19,11,155, 18,43,0,29,14,101,25,9, 34,44,25,79,93,78,75,98,
+ 109,79,94,83,94,67,94,111, 63,103,71,67,102,112,111,62, 121,102,109,118,80,89,83,60, 119,63,60,126,109,130,110,127,
+ 52,53,108,90,118,45,88,105, 66,98,128,92,109,106,132,29, 112,78,109,93,96,54,103,149, 131,47,99,57,55,61,53,92,
+ 113,67,91,86,91,74,66,59, 71,104,115,78,126,131,125,92, 113,78,81,96,142,98,101,64, 76,66,103,108,98,123,99,112,
+ 107,81,112,47,95,153,81,88, 126,155,117,100,111,89,104,77, 131,53,97,95,109,142,93,38, 90,63,95,109,89,98,95,108,
+ 53,65,106,102,111,70,124,115, 115,98,121,126,113,107,115,63, 119,40,128,113,105,54,117,78, 77,126,127,58,62,74,59,98,
+ 130,70,67,61,62,63,68,65, 77,75,76,63,59,56,60,58, 74,89,44,52,52,89,72,70, 112,74,65,152,122,141,139,137,
+
+ 87,83,128,38,121,100,104,126, 80,90,140,138,111,106,105,56, 118,38,125,105,107,90,118,59, 57,141,42,74,97,86,75,106,
+ 113,89,93,86,69,80,73,66, 73,107,89,65,79,88,93,67, 86,81,65,83,79,83,81,62, 108,93,80,129,114,126,119,125,
+ 72,100,140,106,126,76,78,82, 98,91,119,103,121,124,113,68, 119,69,92,127,109,58,110,23, 82,57,114,116,85,81,78,92,
+ 114,71,89,89,85,92,85,77, 88,84,100,100,109,99,87,42, 81,78,116,95,146,76,98,54, 89,93,94,119,114,113,115,110,
+ 73,57,129,122,104,91,116,112, 130,80,66,119,115,115,115,33, 101,33,124,105,107,25,97,34, 70,127,127,84,81,81,57,94,
+ 74,41,117,137,107,32,76,46, 113,50,108,113,93,131,86,33, 121,33,104,108,103,103,115,9, 34,85,117,79,74,86,80,101,
+ 105,136,69,50,52,146,69,59, 68,160,143,98,90,95,36,91, 54,64,94,68,86,132,126,39, 89,121,50,110,105,109,106,138,
+ 119,64,72,56,60,96,83,58, 74,75,59,61,65,63,48,59, 78,73,49,68,62,65,54,71, 69,57,57,111,119,111,115,117,
+};
+
+
+
diff --git a/contrib/google-ced/compact_enc_det_hint_code.cc b/contrib/google-ced/compact_enc_det_hint_code.cc
new file mode 100644
index 0000000..7e7b77a
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det_hint_code.cc
@@ -0,0 +1,169 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "compact_enc_det_hint_code.h"
+
+#include <ctype.h> // for isalpha
+#include <string.h> // for NULL, memchr, strlen, etc
+
+#include "util/basictypes.h" // for uint8, uint32
+#include "util/string_util.h"
+
+// Upper to lower, keep digits, everything else to minus '-' (2d)
+static const char kCharsetToLowerTbl[256] = {
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37, 0x38,0x39,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+ 0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
+ 0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
+ 0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+ 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+};
+
+
+static const char kIsAlpha[256] = {
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
+ 0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
+
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+static const char kIsDigit[256] = {
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 1,1,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+static const char* kFakeEncodingName[] = {
+ "FakeEnc100", "FakeEnc101", "FakeEnc102", "FakeEnc103", "FakeEnc104",
+ "FakeEnc105", "FakeEnc106", "FakeEnc107", "FakeEnc108", "FakeEnc109",
+ "FakeEnc110", "FakeEnc111", "FakeEnc112", "FakeEnc113", "FakeEnc114",
+ "FakeEnc115", "FakeEnc116", "FakeEnc117", "FakeEnc118", "FakeEnc119",
+};
+static const char* kFakeEncodingName2[] = {
+ "FakeEnc_0", "FakeEnc_1", "FakeEnc_2", "FakeEnc_3", "FakeEnc_4",
+};
+
+// Return name for extended encoding
+const char* MyEncodingName(Encoding enc) {
+ if (enc < 0) {
+ return "~";
+ }
+ if (enc == ISO_8859_1) {
+ return "Latin1"; // I can't stand "ASCII" for this
+ }
+ if (enc < NUM_ENCODINGS) {
+ return EncodingName(enc);
+ }
+ // allow fake names, for exploration
+ if ((NUM_ENCODINGS <= enc) && (enc < (NUM_ENCODINGS + 4))) {
+ return kFakeEncodingName2[enc - NUM_ENCODINGS];
+ }
+ if ((100 <= enc) && (enc < 120)) {
+ return kFakeEncodingName[enc - 100];
+ }
+ return "~";
+}
+
+
+// Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize charset= values
+string MakeChar44(const string& str) {
+ string res("________"); // eight underscores
+ int l_ptr = 0;
+ int d_ptr = 0;
+ for (uint32 i = 0; i < str.size(); ++i) {
+ uint8 uc = static_cast<uint8>(str[i]);
+ if (kIsAlpha[uc]) {
+ if (l_ptr < 4) { // Else ignore
+ res[l_ptr] = kCharsetToLowerTbl[uc];
+ l_ptr++;
+ }
+ } else if (kIsDigit[uc]) {
+ if (d_ptr < 4) {
+ res[4 + d_ptr] = kCharsetToLowerTbl[uc];
+ } else {
+ // Keep last 4 digits by shifting left
+ res[4] = res[5];
+ res[5] = res[6];
+ res[6] = res[7];
+ res[7] = kCharsetToLowerTbl[uc];
+ }
+ d_ptr++;
+ } // If neither letter nor digit, drop entirely
+ }
+ return res;
+}
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar4(const string& str) {
+ string res("____"); // four underscores
+ int l_ptr = 0;
+ for (uint32 i = 0; i < str.size(); ++i) {
+ uint8 uc = static_cast<uint8>(str[i]);
+ if (kIsAlpha[uc] | kIsDigit[uc]) {
+ if (l_ptr < 4) { // Else ignore
+ res[l_ptr] = kCharsetToLowerTbl[uc];
+ l_ptr++;
+ }
+ }
+ }
+ return res;
+}
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar8(const string& str) {
+ string res("________"); // eight dots
+ int l_ptr = 0;
+ for (uint32 i = 0; i < str.size(); ++i) {
+ uint8 uc = static_cast<uint8>(str[i]);
+ if (kIsAlpha[uc] | kIsDigit[uc]) {
+ if (l_ptr < 8) { // Else ignore
+ res[l_ptr] = kCharsetToLowerTbl[uc];
+ l_ptr++;
+ }
+ }
+ }
+ return res;
+}
diff --git a/contrib/google-ced/compact_enc_det_hint_code.h b/contrib/google-ced/compact_enc_det_hint_code.h
new file mode 100644
index 0000000..53fe67a
--- /dev/null
+++ b/contrib/google-ced/compact_enc_det_hint_code.h
@@ -0,0 +1,45 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
+#define COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
+
+#include <string> // for string
+
+#include "util/basictypes.h" // for uint32
+#include "util/encodings/encodings.h" // for Encoding
+
+using std::string;
+
+// Return name for extended encoding
+const char* MyEncodingName(Encoding enc);
+
+// Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize charset= values
+string MakeChar44(const string& str);
+
+// Normalize ASCII string to first 4 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar4(const string& str);
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize other values
+string MakeChar8(const string& str);
+
+#endif // COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
diff --git a/contrib/google-ced/detail_head_string.inc b/contrib/google-ced/detail_head_string.inc
new file mode 100644
index 0000000..e70e5ff
--- /dev/null
+++ b/contrib/google-ced/detail_head_string.inc
@@ -0,0 +1,152 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+// Produced by stringify.cc on 2007-09-28 09:13 from file i18n/encodings/compact_enc_det/tools/detail_head.ps
+ "%!PS-Adobe-2.0\n\n/inch {72 mul} def\n/cshow {dup stringwidth pop -2 div"
+ " 0 rmoveto show} def\n\n/lmargin 0.5 inch def\n/rmargin 8.5 inch def\n/t"
+ "margin 10.5 inch def\n/bmargin 0.5 inch def\n\n\n% set to N>=0 to track "
+ "ranked encoding N\n/track-me -1 def\n/track-me2 -1 def\n\n/columns 2 def"
+ "\n\n/lpi 18 def % lines per inch\n/lpc lpi 10 mul def "
+ " % lines per column\n/lpp lpc columns mul def % lines per page\n"
+ "/probw 3.0 inch def % probability width\n/probr 50 def "
+ " % probability range\n/widowlines lpi 2 idiv def % 1/2 inch widow a"
+ "t bottom\n/widowlines lpi def % 1 inch widow at bottom\n\n/lpg l"
+ "pi 2 idiv def % 1/2 inch spacing between groups\n\n/delc 4 inch "
+ "def\n/dell -1 inch lpi div def\n\n/next-line 0 def % 24 lines per i"
+ "nch, 240 per column\n\n/Cfont /Courier findfont 6 scalefont def\n/Hfont "
+ "/Helvetica findfont 6 scalefont def\nHfont setfont\n\n\n% simple string "
+ "hash -- sum the characters\n/strhash {\n /hstr exch def\n /h 0 def\n "
+ "0 1 hstr length 1 sub {/i exch def /h h hstr i get add def} for\n h\n}"
+ " def\n\n% convert pro at 30 per 2x to 0-2.5 inches spanning -50 to 0\n/p"
+ "rob2x {\n 30 idiv probr div probw mul neg probw add\n}def\n\n\n/cliptoc"
+ "olumn {\n % ====== MUST MATCH ME ======\n gsave\n lmargin tmargin mov"
+ "eto\n next-line lpc idiv delc mul 0 rmoveto\n -1 18 rmoveto 0 -10.5 in"
+ "ch rlineto delc 2 add 0 rlineto 0 10.5 inch rlineto closepath\n clip\n"
+ " newpath\n} def\n\n/endcliptocolumn {\n grestore\n % ====== MUST MATC"
+ "H ME ======\n} def\n\n\n\n/column-box {\n lmargin tmargin moveto\n nex"
+ "t-line 1 sub lpc idiv delc mul next-line 1 sub lpc mod 1 add dell mul r"
+ "moveto\n % box\n gsave -1.5 0 rmoveto 0 detail-count dell mul neg rmov"
+ "eto probw 3 add 0 rlineto 0.25 setlinewidth stroke grestore\n gsave -1"
+ ".5 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlinewidth strok"
+ "e grestore\n gsave probw .8 mul 0 rmoveto 0 detail-count dell mul neg r"
+ "lineto 0.25 setlinewidth 0.8 setgray stroke grestore\n gsave probw .6 m"
+ "ul 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlinewidth 0.8 s"
+ "etgray stroke grestore\n gsave probw .4 mul 0 rmoveto 0 detail-count de"
+ "ll mul neg rlineto 0.25 setlinewidth 0.8 setgray stroke grestore\n gsav"
+ "e probw .2 mul 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlin"
+ "ewidth 0.8 setgray stroke grestore\n gsave probw 1.5 add 0 rmoveto 0 de"
+ "tail-count dell mul neg rlineto 0.25 setlinewidth stroke grestore\n} def"
+ "\n\n\n/IncrementLine {\n /incr exch def\n /next-line next-line incr ad"
+ "d def\n next-line lpc ge next-line incr sub lpc lt and {\n % We "
+ "just went to the top of column 2; redo clip\n endcliptocolumn % M"
+ "UST match\n column-box\n /next-line lpc def\n cliptocolumn "
+ " % MUST match\n } if\n next-line lpp ge {\n % We just went to the"
+ " top of column 3; start new page column 1\n endcliptocolumn % MUS"
+ "T match\n column-box\n showpage\n Hfont setfont\n /next-line"
+ " 0 def\n show-pageno\n cliptocolumn % MUST match\n } if\n}"
+ " def\n\n/IncrementLineOutside {\n /incr exch def\n /next-line next-lin"
+ "e incr add def\n next-line lpc ge next-line incr sub lpc lt and {\n"
+ " % We just went to the top of column 2\n /next-line lpc def\n } i"
+ "f\n next-line lpp ge {\n % We just went to the top of column 3; star"
+ "t new page column 1\n showpage\n Hfont setfont\n /next-line 0 d"
+ "ef\n show-pageno\n } if\n} def\n\n/NextColumn {\n lpc 1 sub Increme"
+ "ntLine\n} def\n\n/NextPage {\n lpp 1 sub IncrementLine\n} def\n\n% Up"
+ "on entry, we are OUTSIDE the clip\n/start-detail {\n /d-title exch def\n"
+ "\n % align >= 1 inch at bottom of column, and/or start new column\n lp"
+ "c next-line lpc mod sub widowlines lt {\n % Start at top of a column\n"
+ " next-line lpc ge {\n % Start new page\n showpage\n Hf"
+ "ont setfont\n /next-line 0 def\n show-pageno\n } {\n %"
+ " Start new column\n /next-line lpc def\n } ifelse\n } if\n\n l"
+ "margin tmargin moveto\n next-line lpc idiv delc mul next-line lpc mod "
+ "dell mul rmoveto\n gsave d-title show grestore\n 0 dell rmoveto\n 1 1"
+ " 4 {/j exch def gsave probw j mul 5 div -2 rmoveto 50 j 10 mul sub 20 st"
+ "ring cvs cshow grestore} for\n 2 IncrementLineOutside\n /detail-count "
+ "1 def\n cliptocolumn % MUST match\n /d-array [] def\n} def\n"
+ "\n/size-detail {\n /d-names exch def\n /d-size exch def\n % zero sums"
+ "\n /sums d-size array def\n 0 1 d-size 1 sub {/i exch def sums i 0 pu"
+ "t} for\n /old-d-max 0 def\n /colors d-size array def\n 0 1 d-size 1 s"
+ "ub {/i exch def colors i i 3 mul 17 mod 17 div put} for\n %0 1 d-size "
+ "1 sub {/i exch def colors i d-names i get strhash 3 mul 17 mod 17 div p"
+ "ut} for\n %0 1 d-size 1 sub {/i exch def ( ) show colors i get 20 stri"
+ "ng cvs show} for\n} def\n\n/count-detail {\n /detail-total-count exch d"
+ "ef\n % if total-count >= one column, start at top of a column\n detail"
+ "-total-count lpp ge {\n % Start new page\n NextPage\n } {\n de"
+ "tail-total-count lpc ge {\n % Start new column\n NextColumn\n "
+ " } if\n } ifelse\n} def\n\n% highlight next entry with underbar\n/do-"
+ "flag {\ngsave\n setrgbcolor\n lmargin tmargin moveto\n next-line lpc "
+ "idiv delc mul next-line lpc mod dell mul rmoveto\n 0 -2 rmoveto\n pro"
+ "bw 0 rlineto\n 0 dell neg rlineto\n probw neg 0 rlineto\n closepath\n"
+ " fill\ngrestore\n} def\n\n/do-detail-e {\n /d-array exch def\n /d-enc"
+ " exch def\n /d-label exch def\n /d-max -999999 def\n\n lmargin tmargi"
+ "n moveto\n next-line lpc idiv delc mul next-line lpc mod dell mul rmov"
+ "eto\n 0.25 setlinewidth\n\n % show label, using encoding color\n gsav"
+ "e\n probw 2 add -2 rmoveto\n detail-count 1 sub 2 mod 0.25 inch mu"
+ "l 0 rmoveto\n % ([) show detail-count 20 string cvs show (] ) show\n "
+ " d-enc 0 lt {\n 0 setgray\n }{\n colors d-enc get 1 .8 se"
+ "thsbcolor\n } ifelse\n d-label show\n grestore\n % For -prune- d"
+ "raw horizontal line\n d-label length 8 gt {d-label 4 get (p) 0 get eq d"
+ "-label 5 get (r) 0 get eq and {\n /prune-val d-label cvi def\n /ne"
+ "wx prune-val 30 mul prob2x def\n gsave newx 6 rmoveto 0 -12 rlineto 1"
+ ".5 setlinewidth 0.8 setgray stroke grestore\n gsave probw 0 add 0 rli"
+ "neto 0.25 setlinewidth 0.8 setgray stroke grestore\n } if } if\n\n % t"
+ "rack max per new row\n 0 1 d-array length 1 sub {\n /i exch def\n "
+ " /sum sums i get d-array i get add def\n d-max sum lt {/d-max sum def"
+ "} if\n } for\n\n % draw line increments\n 0 1 d-array length 1 sub {\n"
+ " /i exch def\n detail-count 1 gt {\n /oldx old-d-max sums i g"
+ "et sub prob2x def\n } {\n /oldx 600 prob2x def\n } ifelse\n "
+ " /oldy dell neg def\n /newx d-max sums i get d-array i get add sub"
+ " prob2x def\n /newy 0 def\n gsave\n oldx oldy rmoveto\n newx"
+ " oldx sub newy oldy sub rlineto\n % if encoding is being tracked, ma"
+ "ke bold line\n i track-me eq\n {2 setlinewidth}\n {i track-"
+ "me2 eq {1.25 setlinewidth} {0.25 setlinewidth} ifelse}\n ifelse\n "
+ " colors i get 1 .8 sethsbcolor stroke\n grestore\n } for\n /detail"
+ "-count detail-count 1 add def\n\n % increment running total in sums, tr"
+ "ack max per row\n 0 1 d-array length 1 sub {\n /i exch def\n sums"
+ " i sums i get d-array i get add put\n } for\n /old-d-max d-max def\n"
+ "\n 1 IncrementLine\n} def\n\n\n/do-detail {\n /d-array exch def\n /d-"
+ "label exch def\n d-label -1 d-array do-detail-e\n} def\n\n% Upon exit, "
+ "we are outside the clip\n/end-detail {\n pop\n endcliptocolumn "
+ " % MUST match\n column-box\n\n % text labels\n 0 1 d-array length 1 s"
+ "ub {\n /i exch def\n gsave\n /newx old-d-max sums i get sub pro"
+ "b2x def\n newx 0 ge {\n newx 0 rmoveto\n currentpoint trans"
+ "late\n colors i get 1 .8 sethsbcolor\n gsave 0 dell neg rline"
+ "to 0.25 setlinewidth stroke grestore\n -60 rotate\n 0 -2 movet"
+ "o d-names i get show\n } if\n grestore\n } for\n d-array length "
+ "0 gt {\n lpg IncrementLineOutside\n } {\n lpg 4 idiv IncrementLin"
+ "eOutside\n } ifelse\n} def\n\n/do-src {\n/src exch def\n lmargin tmarg"
+ "in moveto\n next-line lpc idiv delc mul next-line lpc mod dell mul rmo"
+ "veto\n Cfont setfont\n src show\n Hfont setfont\n 1 IncrementLine\n}"
+ " def\n\n% Underline trigram in source text\n/do-highlight1 {\n /hl-colo"
+ "r exch def\n /hl-offset exch def\n /hl-line exch 1 sub 2 mul def\n gs"
+ "ave\n lmargin tmargin moveto\n next-line hl-line sub lpc idiv delc mul"
+ "\n next-line hl-line sub lpc mod dell mul rmoveto\n % Assume text is 6"
+ " chars in and 3.6 pts per char, but 2 chars per offset\n hl-offset 2 mu"
+ "l 6 add 3.6 mul 4 rmoveto\n\n 0 setgray 0.5 setlinewidth\n hl-color 1"
+ " eq {0 0 1 setrgbcolor} if % Latin1 blue\n hl-color 2 eq {1 0 1 setrgb"
+ "color} if % Latin2 magenta\n hl-color 3 eq {1 0.67 0 setrgbcolor} if "
+ "% Latin7 orange\n 18 -2 rlineto stroke\n grestore\n} def\n\n% Box trig"
+ "ram in source text\n/do-highlight2 {\n /hl-color exch def\n /hl-offset"
+ " exch def\n /hl-line exch 1 sub 2 mul def\n gsave\n lmargin tmargin m"
+ "oveto\n next-line hl-line sub lpc idiv delc mul\n next-line hl-line su"
+ "b lpc mod dell mul rmoveto\n % Assume text is 6 chars in and 3.6 pts pe"
+ "r char, but 2 chars per offset\n hl-offset 2 mul 6 add 3.6 mul 4 rmove"
+ "to\n\n 0 setgray 0.25 setlinewidth\n hl-color 1 eq {0 0 1 setrgbcolor}"
+ " if % Latin1 blue\n hl-color 2 eq {1 0 1 setrgbcolor} if % Latin2 mag"
+ "enta\n hl-color 3 eq {1 0.67 0 setrgbcolor} if % Latin7 orange\n -0.5"
+ " -0.5 rmoveto\n 22 0 rlineto\n 0 4 rlineto\n -11 2 rlineto\n -11 -2 "
+ "rlineto\n closepath\n stroke\n grestore\n} def\n\n/show-pageno {\ngsa"
+ "ve\nlmargin bmargin moveto 0 -12 rmoveto\n(Page ) show pageno 20 string "
+ "cvs show\ngrestore\n/pageno pageno 1 add def\n} def\n\n/pageno 1 def\nsh"
+ "ow-pageno\n%=============================\n\n\n"
diff --git a/contrib/google-ced/util/basictypes.h b/contrib/google-ced/util/basictypes.h
new file mode 100644
index 0000000..af391c7
--- /dev/null
+++ b/contrib/google-ced/util/basictypes.h
@@ -0,0 +1,331 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_BASICTYPES_H_
+#define UTIL_BASICTYPES_H_
+
+#include <limits.h> // So we can set the bounds of our types
+#include <stddef.h> // For size_t
+#include <string.h> // for memcpy
+
+#include "util/port.h" // Types that only need exist on certain systems
+
+#ifndef COMPILER_MSVC
+// stdint.h is part of C99 but MSVC doesn't have it.
+#include <stdint.h> // For intptr_t.
+#endif
+
+typedef signed char schar;
+typedef signed char int8;
+typedef short int16;
+// TODO(mbelshe) Remove these type guards. These are
+// temporary to avoid conflicts with npapi.h.
+#ifndef _INT32
+#define _INT32
+typedef int int32;
+#endif
+
+// The NSPR system headers define 64-bit as |long| when possible. In order to
+// not have typedef mismatches, we do the same on LP64.
+#if __LP64__
+typedef long int64;
+#else
+typedef long long int64;
+#endif
+
+// NOTE: unsigned types are DANGEROUS in loops and other arithmetical
+// places. Use the signed types unless your variable represents a bit
+// pattern (eg a hash value) or you really need the extra bit. Do NOT
+// use 'unsigned' to express "this value should always be positive";
+// use assertions for this.
+
+typedef unsigned char uint8;
+typedef unsigned short uint16;
+// TODO(mbelshe) Remove these type guards. These are
+// temporary to avoid conflicts with npapi.h.
+#ifndef _UINT32
+#define _UINT32
+typedef unsigned int uint32;
+#endif
+
+// See the comment above about NSPR and 64-bit.
+#if __LP64__
+typedef unsigned long uint64;
+#else
+typedef unsigned long long uint64;
+#endif
+
+// A type to represent a Unicode code-point value. As of Unicode 4.0,
+// such values require up to 21 bits.
+// (For type-checking on pointers, make this explicitly signed,
+// and it should always be the signed version of whatever int32 is.)
+typedef signed int char32;
+
+const uint8 kuint8max = (( uint8) 0xFF);
+const uint16 kuint16max = ((uint16) 0xFFFF);
+const uint32 kuint32max = ((uint32) 0xFFFFFFFF);
+const uint64 kuint64max = ((uint64) GG_LONGLONG(0xFFFFFFFFFFFFFFFF));
+const int8 kint8min = (( int8) 0x80);
+const int8 kint8max = (( int8) 0x7F);
+const int16 kint16min = (( int16) 0x8000);
+const int16 kint16max = (( int16) 0x7FFF);
+const int32 kint32min = (( int32) 0x80000000);
+const int32 kint32max = (( int32) 0x7FFFFFFF);
+const int64 kint64min = (( int64) GG_LONGLONG(0x8000000000000000));
+const int64 kint64max = (( int64) GG_LONGLONG(0x7FFFFFFFFFFFFFFF));
+
+// A macro to disallow the copy constructor and operator= functions
+// This should be used in the private: declarations for a class
+#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
+ TypeName(const TypeName&); \
+ void operator=(const TypeName&)
+
+// An older, deprecated, politically incorrect name for the above.
+#define DISALLOW_EVIL_CONSTRUCTORS(TypeName) DISALLOW_COPY_AND_ASSIGN(TypeName)
+
+// A macro to disallow all the implicit constructors, namely the
+// default constructor, copy constructor and operator= functions.
+//
+// This should be used in the private: declarations for a class
+// that wants to prevent anyone from instantiating it. This is
+// especially useful for classes containing only static methods.
+#define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \
+ TypeName(); \
+ DISALLOW_COPY_AND_ASSIGN(TypeName)
+
+// The arraysize(arr) macro returns the # of elements in an array arr.
+// The expression is a compile-time constant, and therefore can be
+// used in defining new arrays, for example. If you use arraysize on
+// a pointer by mistake, you will get a compile-time error.
+
+// This template function declaration is used in defining arraysize.
+// Note that the function doesn't need an implementation, as we only
+// use its type.
+template <typename T, size_t N>
+char (&ArraySizeHelper(T (&array)[N]))[N];
+
+// That gcc wants both of these prototypes seems mysterious. VC, for
+// its part, can't decide which to use (another mystery). Matching of
+// template overloads: the final frontier.
+#ifndef _MSC_VER
+template <typename T, size_t N>
+char (&ArraySizeHelper(const T (&array)[N]))[N];
+#endif
+
+#define arraysize(array) (sizeof(ArraySizeHelper(array)))
+
+
+// Use implicit_cast as a safe version of static_cast or const_cast
+// for upcasting in the type hierarchy (i.e. casting a pointer to Foo
+// to a pointer to SuperclassOfFoo or casting a pointer to Foo to
+// a const pointer to Foo).
+// When you use implicit_cast, the compiler checks that the cast is safe.
+// Such explicit implicit_casts are necessary in surprisingly many
+// situations where C++ demands an exact type match instead of an
+// argument type convertable to a target type.
+//
+// The From type can be inferred, so the preferred syntax for using
+// implicit_cast is the same as for static_cast etc.:
+//
+// implicit_cast<ToType>(expr)
+//
+// implicit_cast would have been part of the C++ standard library,
+// but the proposal was submitted too late. It will probably make
+// its way into the language in the future.
+template<typename To, typename From>
+inline To implicit_cast(From const &f) {
+ return f;
+}
+
+// The COMPILE_ASSERT macro can be used to verify that a compile time
+// expression is true. For example, you could use it to verify the
+// size of a static array:
+//
+// COMPILE_ASSERT(arraysize(content_type_names) == CONTENT_NUM_TYPES,
+// content_type_names_incorrect_size);
+//
+// or to make sure a struct is smaller than a certain size:
+//
+// COMPILE_ASSERT(sizeof(foo) < 128, foo_too_large);
+//
+// The second argument to the macro is the name of the variable. If
+// the expression is false, most compilers will issue a warning/error
+// containing the name of the variable.
+
+template <bool>
+struct CompileAssert {
+};
+
+#undef COMPILE_ASSERT
+#define COMPILE_ASSERT(expr, msg) \
+ typedef CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1]
+
+// Implementation details of COMPILE_ASSERT:
+//
+// - COMPILE_ASSERT works by defining an array type that has -1
+// elements (and thus is invalid) when the expression is false.
+//
+// - The simpler definition
+//
+// #define COMPILE_ASSERT(expr, msg) typedef char msg[(expr) ? 1 : -1]
+//
+// does not work, as gcc supports variable-length arrays whose sizes
+// are determined at run-time (this is gcc's extension and not part
+// of the C++ standard). As a result, gcc fails to reject the
+// following code with the simple definition:
+//
+// int foo;
+// COMPILE_ASSERT(foo, msg); // not supposed to compile as foo is
+// // not a compile-time constant.
+//
+// - By using the type CompileAssert<(bool(expr))>, we ensures that
+// expr is a compile-time constant. (Template arguments must be
+// determined at compile-time.)
+//
+// - The outter parentheses in CompileAssert<(bool(expr))> are necessary
+// to work around a bug in gcc 3.4.4 and 4.0.1. If we had written
+//
+// CompileAssert<bool(expr)>
+//
+// instead, these compilers will refuse to compile
+//
+// COMPILE_ASSERT(5 > 0, some_message);
+//
+// (They seem to think the ">" in "5 > 0" marks the end of the
+// template argument list.)
+//
+// - The array size is (bool(expr) ? 1 : -1), instead of simply
+//
+// ((expr) ? 1 : -1).
+//
+// This is to avoid running into a bug in MS VC 7.1, which
+// causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1.
+
+
+// MetatagId refers to metatag-id that we assign to
+// each metatag <name, value> pair..
+typedef uint32 MetatagId;
+
+// Argument type used in interfaces that can optionally take ownership
+// of a passed in argument. If TAKE_OWNERSHIP is passed, the called
+// object takes ownership of the argument. Otherwise it does not.
+enum Ownership {
+ DO_NOT_TAKE_OWNERSHIP,
+ TAKE_OWNERSHIP
+};
+
+// bit_cast<Dest,Source> is a template function that implements the
+// equivalent of "*reinterpret_cast<Dest*>(&source)". We need this in
+// very low-level functions like the protobuf library and fast math
+// support.
+//
+// float f = 3.14159265358979;
+// int i = bit_cast<int32>(f);
+// // i = 0x40490fdb
+//
+// The classical address-casting method is:
+//
+// // WRONG
+// float f = 3.14159265358979; // WRONG
+// int i = * reinterpret_cast<int*>(&f); // WRONG
+//
+// The address-casting method actually produces undefined behavior
+// according to ISO C++ specification section 3.10 -15 -. Roughly, this
+// section says: if an object in memory has one type, and a program
+// accesses it with a different type, then the result is undefined
+// behavior for most values of "different type".
+//
+// This is true for any cast syntax, either *(int*)&f or
+// *reinterpret_cast<int*>(&f). And it is particularly true for
+// conversions betweeen integral lvalues and floating-point lvalues.
+//
+// The purpose of 3.10 -15- is to allow optimizing compilers to assume
+// that expressions with different types refer to different memory. gcc
+// 4.0.1 has an optimizer that takes advantage of this. So a
+// non-conforming program quietly produces wildly incorrect output.
+//
+// The problem is not the use of reinterpret_cast. The problem is type
+// punning: holding an object in memory of one type and reading its bits
+// back using a different type.
+//
+// The C++ standard is more subtle and complex than this, but that
+// is the basic idea.
+//
+// Anyways ...
+//
+// bit_cast<> calls memcpy() which is blessed by the standard,
+// especially by the example in section 3.9 . Also, of course,
+// bit_cast<> wraps up the nasty logic in one place.
+//
+// Fortunately memcpy() is very fast. In optimized mode, with a
+// constant size, gcc 2.95.3, gcc 4.0.1, and msvc 7.1 produce inline
+// code with the minimal amount of data movement. On a 32-bit system,
+// memcpy(d,s,4) compiles to one load and one store, and memcpy(d,s,8)
+// compiles to two loads and two stores.
+//
+// I tested this code with gcc 2.95.3, gcc 4.0.1, icc 8.1, and msvc 7.1.
+//
+// WARNING: if Dest or Source is a non-POD type, the result of the memcpy
+// is likely to surprise you.
+
+template <class Dest, class Source>
+inline Dest bit_cast(const Source& source) {
+ // Compile time assertion: sizeof(Dest) == sizeof(Source)
+ // A compile error here means your Dest and Source have different sizes.
+ // typedef char VerifySizesAreEqual [sizeof(Dest) == sizeof(Source) ? 1 : -1];
+
+ Dest dest;
+ memcpy(&dest, &source, sizeof(dest));
+ return dest;
+}
+
+// The following enum should be used only as a constructor argument to indicate
+// that the variable has static storage class, and that the constructor should
+// do nothing to its state. It indicates to the reader that it is legal to
+// declare a static instance of the class, provided the constructor is given
+// the base::LINKER_INITIALIZED argument. Normally, it is unsafe to declare a
+// static variable that has a constructor or a destructor because invocation
+// order is undefined. However, IF the type can be initialized by filling with
+// zeroes (which the loader does for static variables), AND the destructor also
+// does nothing to the storage, AND there are no virtual methods, then a
+// constructor declared as
+// explicit MyClass(base::LinkerInitialized x) {}
+// and invoked as
+// static MyClass my_variable_name(base::LINKER_INITIALIZED);
+namespace base {
+enum LinkerInitialized { LINKER_INITIALIZED };
+} // base
+
+// UnaligndLoad32 is put here instead of util/port.h to
+// avoid the circular dependency between port.h and basictypes.h
+// ARM does not support unaligned memory access.
+#if defined(ARCH_CPU_X86_FAMILY)
+// x86 and x86-64 can perform unaligned loads/stores directly;
+inline uint32 UnalignedLoad32(const void* p) {
+ return *reinterpret_cast<const uint32*>(p);
+}
+#else
+#define NEED_ALIGNED_LOADS
+// If target architecture does not support unaligned loads and stores,
+// use memcpy version of UNALIGNED_LOAD32.
+inline uint32 UnalignedLoad32(const void* p) {
+ uint32 t;
+ memcpy(&t, reinterpret_cast<const uint8*>(p), sizeof(t));
+ return t;
+}
+
+#endif
+#endif // UTIL_BASICTYPES_H_
diff --git a/contrib/google-ced/util/case_insensitive_hash.h b/contrib/google-ced/util/case_insensitive_hash.h
new file mode 100644
index 0000000..7b0c9db
--- /dev/null
+++ b/contrib/google-ced/util/case_insensitive_hash.h
@@ -0,0 +1,88 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_CASE_INSENSITIVE_HASH_H_
+#define UTIL_CASE_INSENSITIVE_HASH_H_
+
+#include <ctype.h>
+#include <stddef.h>
+#ifndef _MSC_VER
+#include <strings.h>
+#endif
+
+#include <string>
+
+#include "util/basictypes.h"
+#include "util/string_util.h"
+
+// Functors for hashing c-strings with case-insensitive semantics.
+struct CStringCaseHash {
+ size_t operator()(const char *str) const {
+ unsigned long hash_val = 0;
+ while (*str) {
+ hash_val = 5*hash_val + tolower(*str);
+ str++;
+ }
+ return (size_t)hash_val;
+ }
+};
+
+struct CStringCaseEqual {
+ bool operator()(const char *str1, const char *str2) const {
+ return !base::strcasecmp(str1, str2);
+ }
+};
+
+// These functors, in addition to being case-insensitive, ignore all
+// non-alphanumeric characters. This is useful when we want all variants of
+// a string -- where variants can differ in puncutation and whitespace -- to
+// map to the same value.
+struct CStringAlnumCaseHash {
+ size_t operator()(const char *str) const {
+ unsigned long hash_val = 0;
+ while (*str) {
+ if (isalnum(*str)) {
+ hash_val = 5*hash_val + tolower(*str);
+ }
+ str++;
+ }
+ return (size_t)hash_val;
+ }
+};
+
+struct CStringAlnumCaseEqual {
+ bool operator()(const char *str1, const char *str2) const {
+ while (true) {
+ // Skip until each pointer is pointing to an alphanumeric char or '\0'
+ while (!isalnum(*str1) && (*str1 != '\0')) {
+ str1++;
+ }
+ while (!isalnum(*str2) && (*str2 != '\0')) {
+ str2++;
+ }
+ if (tolower(*str1) != tolower(*str2)) {
+ return false; // mismatch on alphanumeric char or '\0'
+ }
+ if (*str1 == '\0') { // in which case *str2 must be '\0' as well
+ return true; // reached '\0' in both strings without mismatch
+ }
+ str1++;
+ str2++;
+ }
+ }
+};
+
+#endif // UTIL_CASE_INSENSITIVE_HASH_H_
diff --git a/contrib/google-ced/util/commandlineflags.h b/contrib/google-ced/util/commandlineflags.h
new file mode 100644
index 0000000..341a659
--- /dev/null
+++ b/contrib/google-ced/util/commandlineflags.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_COMMANDLINEFLAGS_H_
+#define UTIL_COMMANDLINEFLAGS_H_
+
+
+#undef DEFINE_bool
+#define DEFINE_bool(name, default_value, comment) \
+ bool FLAGS_##name = default_value
+#undef DEFINE_int32
+#define DEFINE_int32(name, default_value, comment) \
+ int32 FLAGS_##name = default_value
+#undef DEFINE_string
+#define DEFINE_string(name, default_value, comment) \
+ string FLAGS_##name = default_value
+
+#undef DECLARE_bool
+#define DECLARE_bool(name) extern bool FLAGS_##name
+#undef DECLARE_int32
+#define DECLARE_int32(name) extern int32 FLAGS_##name
+#undef DECLARE_string
+#define DECLARE_string(name) extern string FLAGS_##name
+
+
+#endif // UTIL_COMMANDLINEFLAGS_H_
diff --git a/contrib/google-ced/util/encodings/encodings.cc b/contrib/google-ced/util/encodings/encodings.cc
new file mode 100644
index 0000000..b5f8dc5
--- /dev/null
+++ b/contrib/google-ced/util/encodings/encodings.cc
@@ -0,0 +1,891 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/encodings/encodings.h"
+
+#include <string.h> // for strcasecmp
+#include <unordered_map>
+#include <utility> // for pair
+
+#include "util/basictypes.h"
+#include "util/string_util.h"
+#include "util/case_insensitive_hash.h"
+
+struct EncodingInfo {
+ // The standard name for this encoding.
+ //
+ const char* encoding_name_;
+
+ // The "preferred MIME name" of an encoding as specified by the IANA at:
+ // http://www.iana.org/assignments/character-sets
+ //
+ // Note that the preferred MIME name may differ slightly from the
+ // official IANA name: i.e. ISO-8859-1 vs. ISO_8859-1:1987
+ //
+ const char* mime_encoding_name_;
+
+ // It is an internal policy that if an encoding has an IANA name,
+ // then encoding_name_ and mime_encoding_name_ must be the same string.
+ //
+ // However, there can be exceptions if there are compelling reasons.
+ // For example, Japanese mobile handsets require the name
+ // "Shift_JIS" in charset=... parameter in Content-Type headers to
+ // process emoji (emoticons) in their private encodings. In that
+ // case, mime_encoding_name_ should be "Shift_JIS", despite
+ // encoding_name_ actually is "X-KDDI-Shift_JIS".
+
+ // Some multi-byte encodings use byte values that coincide with the
+ // ASCII codes for HTML syntax characters <>"&' and browsers like MSIE
+ // can misinterpret these, as indicated in an external XSS report from
+ // 2007-02-15. Here, we map these dangerous encodings to safer ones. We
+ // also use UTF8 instead of encodings that we don't support in our
+ // output, and we generally try to be conservative in what we send out.
+ // Where the client asks for single- or double-byte encodings that are
+ // not as common, we substitute a more common single- or double-byte
+ // encoding, if there is one, thereby preserving the client's intent
+ // to use less space than UTF-8. This also means that characters
+ // outside the destination set will be converted to HTML NCRs (&#NNN;)
+ // if requested.
+
+ Encoding preferred_web_output_encoding_;
+};
+
+static const EncodingInfo kEncodingInfoTable[] = {
+ { "ASCII", "ISO-8859-1", ISO_8859_1},
+ { "Latin2", "ISO-8859-2", ISO_8859_2},
+ { "Latin3", "ISO-8859-3", UTF8},
+ // MSIE 6 does not support ISO-8859-3 (XSS issue)
+ { "Latin4", "ISO-8859-4", ISO_8859_4},
+ { "ISO-8859-5", "ISO-8859-5", ISO_8859_5},
+ { "Arabic", "ISO-8859-6", ISO_8859_6},
+ { "Greek", "ISO-8859-7", ISO_8859_7},
+ { "Hebrew", "ISO-8859-8", MSFT_CP1255},
+ // we do not endorse the visual order
+ { "Latin5", "ISO-8859-9", ISO_8859_9},
+ { "Latin6", "ISO-8859-10", UTF8},
+ // MSIE does not support ISO-8859-10 (XSS issue)
+ { "EUC-JP", "EUC-JP", JAPANESE_EUC_JP},
+ { "SJS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+ { "JIS", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
+ // due to potential confusion with HTML syntax chars
+ { "BIG5", "Big5", CHINESE_BIG5},
+ { "GB", "GB2312", CHINESE_GB},
+ { "EUC-CN",
+ "EUC-CN",
+ // Misnamed. Should be EUC-TW.
+ CHINESE_BIG5},
+ // MSIE treats "EUC-CN" like GB2312, which is not EUC-TW,
+ // and EUC-TW is rare, so we prefer Big5 for output.
+ { "KSC", "EUC-KR", KOREAN_EUC_KR},
+ { "Unicode",
+ "UTF-16LE",
+ // Internet Explorer doesn't recognize "ISO-10646-UCS-2"
+ UTF8
+ // due to potential confusion with HTML syntax chars
+ },
+ { "EUC",
+ "EUC", // Misnamed. Should be EUC-TW.
+ CHINESE_BIG5
+ // MSIE does not recognize "EUC" (XSS issue),
+ // and EUC-TW is rare, so we prefer Big5 for output.
+ },
+ { "CNS",
+ "CNS", // Misnamed. Should be EUC-TW.
+ CHINESE_BIG5},
+ // MSIE does not recognize "CNS" (XSS issue),
+ // and EUC-TW is rare, so we prefer Big5 for output.
+ { "BIG5-CP950",
+ "BIG5-CP950", // Not an IANA name
+ CHINESE_BIG5
+ // MSIE does not recognize "BIG5-CP950" (XSS issue)
+ },
+ { "CP932", "CP932", // Not an IANA name
+ JAPANESE_SHIFT_JIS}, // MSIE does not recognize "CP932" (XSS issue)
+ { "UTF8", "UTF-8", UTF8},
+ { "Unknown",
+ "x-unknown", // Not an IANA name
+ UTF8}, // UTF-8 is our default output encoding
+ { "ASCII-7-bit", "US-ASCII", ASCII_7BIT},
+ { "KOI8R", "KOI8-R", RUSSIAN_KOI8_R},
+ { "CP1251", "windows-1251", RUSSIAN_CP1251},
+ { "CP1252", "windows-1252", MSFT_CP1252},
+ { "KOI8U",
+ "KOI8-U",
+ ISO_8859_5}, // because koi8-u is not as common
+ { "CP1250", "windows-1250", MSFT_CP1250},
+ { "ISO-8859-15", "ISO-8859-15", ISO_8859_15},
+ { "CP1254", "windows-1254", MSFT_CP1254},
+ { "CP1257", "windows-1257", MSFT_CP1257},
+ { "ISO-8859-11", "ISO-8859-11", ISO_8859_11},
+ { "CP874", "windows-874", MSFT_CP874},
+ { "CP1256", "windows-1256", MSFT_CP1256},
+ { "CP1255", "windows-1255", MSFT_CP1255},
+ { "ISO-8859-8-I", "ISO-8859-8-I", MSFT_CP1255},
+ // Java does not support iso-8859-8-i
+ { "VISUAL", "ISO-8859-8", MSFT_CP1255},
+ // we do not endorse the visual order
+ { "CP852", "cp852", MSFT_CP1250},
+ // because cp852 is not as common
+ { "CSN_369103", "csn_369103", MSFT_CP1250},
+ // MSIE does not recognize "csn_369103" (XSS issue)
+ { "CP1253", "windows-1253", MSFT_CP1253},
+ { "CP866", "IBM866", RUSSIAN_CP1251},
+ // because cp866 is not as common
+ { "ISO-8859-13", "ISO-8859-13", UTF8},
+ // because iso-8859-13 is not widely supported
+ { "ISO-2022-KR", "ISO-2022-KR", KOREAN_EUC_KR},
+ // due to potential confusion with HTML syntax chars
+ { "GBK", "GBK", GBK},
+ { "GB18030", "GB18030", GBK},
+ // because gb18030 is not widely supported
+ { "BIG5_HKSCS", "BIG5-HKSCS", CHINESE_BIG5},
+ // because Big5-HKSCS is not widely supported
+ { "ISO_2022_CN", "ISO-2022-CN", CHINESE_GB},
+ // due to potential confusion with HTML syntax chars
+ { "TSCII", "tscii", UTF8},
+ // we do not have an output converter for this font encoding
+ { "TAM", "tam", UTF8},
+ // we do not have an output converter for this font encoding
+ { "TAB", "tab", UTF8},
+ // we do not have an output converter for this font encoding
+ { "JAGRAN", "jagran", UTF8},
+ // we do not have an output converter for this font encoding
+ { "MACINTOSH", "MACINTOSH", ISO_8859_1},
+ // because macintosh is relatively uncommon
+ { "UTF7", "UTF-7",
+ UTF8}, // UTF-7 has been the subject of XSS attacks and is deprecated
+ { "BHASKAR", "bhaskar",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "HTCHANAKYA", "htchanakya", // not an IANA charset name.
+ UTF8}, // we do not have an output converter for this font encoding
+ { "UTF-16BE", "UTF-16BE",
+ UTF8}, // due to potential confusion with HTML syntax chars
+ { "UTF-16LE", "UTF-16LE",
+ UTF8}, // due to potential confusion with HTML syntax chars
+ { "UTF-32BE", "UTF-32BE",
+ UTF8}, // unlikely to cause XSS bugs, but very uncommon on Web
+ { "UTF-32LE", "UTF-32LE",
+ UTF8}, // unlikely to cause XSS bugs, but very uncommon on Web
+ { "X-BINARYENC", "x-binaryenc", // Not an IANA name
+ UTF8}, // because this one is not intended for output (just input)
+ { "HZ-GB-2312", "HZ-GB-2312",
+ CHINESE_GB}, // due to potential confusion with HTML syntax chars
+ { "X-UTF8UTF8", "x-utf8utf8", // Not an IANA name
+ UTF8}, // because this one is not intended for output (just input)
+ { "X-TAM-ELANGO", "x-tam-elango",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "X-TAM-LTTMBARANI", "x-tam-lttmbarani",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "X-TAM-SHREE", "x-tam-shree",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "X-TAM-TBOOMIS", "x-tam-tboomis",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "X-TAM-TMNEWS", "x-tam-tmnews",
+ UTF8}, // we do not have an output converter for this font encoding
+ { "X-TAM-WEBTAMIL", "x-tam-webtamil",
+ UTF8}, // we do not have an output converter for this font encoding
+
+ { "X-KDDI-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+ // KDDI version of Shift_JIS with Google Emoji PUA mappings.
+ // Note that MimeEncodingName() returns "Shift_JIS", since KDDI uses
+ // "Shift_JIS" in HTTP headers and email messages.
+
+ { "X-DoCoMo-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+ // DoCoMo version of Shift_JIS with Google Emoji PUA mappings.
+ // See the comment at KDDI_SHIFT_JIS for other issues.
+
+ { "X-SoftBank-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+ // SoftBank version of Shift_JIS with Google Emoji PUA mappings.
+ // See the comment at KDDI_SHIFT_JIS for other issues.
+
+ { "X-KDDI-ISO-2022-JP", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
+ // KDDI version of ISO-2022-JP with Google Emoji PUA mappings.
+ // See the comment at KDDI_SHIFT_JIS for other issues.
+ // The preferred Web encoding is due to potential confusion with
+ // HTML syntax chars.
+
+ { "X-SoftBank-ISO-2022-JP", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
+ // SoftBank version of ISO-2022-JP with Google Emoji PUA mappings.
+ // See the comment at KDDI_SHIFT_JIS for other issues.
+ // The preferred Web encoding is due to potential confusion with
+ // HTML syntax chars.
+
+ // Please refer to NOTE: section in the comments in the definition
+ // of "struct I18NInfoByEncoding", before adding new encodings.
+
+};
+
+
+
+COMPILE_ASSERT(arraysize(kEncodingInfoTable) == NUM_ENCODINGS,
+ kEncodingInfoTable_has_incorrect_size);
+
+Encoding default_encoding() {return LATIN1;}
+
+// *************************************************************
+// Encoding predicates
+// IsValidEncoding()
+// IsEncEncCompatible
+// IsEncodingWithSupportedLanguage
+// IsSupersetOfAscii7Bit
+// Is8BitEncoding
+// IsCJKEncoding
+// IsHebrewEncoding
+// IsRightToLeftEncoding
+// IsLogicalRightToLeftEncoding
+// IsVisualRightToLeftEncoding
+// IsIso2022Encoding
+// IsIso2022JpOrVariant
+// IsShiftJisOrVariant
+// IsJapaneseCellPhoneCarrierSpecificEncoding
+// *************************************************************
+
+bool IsValidEncoding(Encoding enc) {
+ return ((enc >= 0) && (enc < kNumEncodings));
+}
+
+bool IsEncEncCompatible(const Encoding from, const Encoding to) {
+ // Tests compatibility between the "from" and "to" encodings; in
+ // the typical case -- when both are valid known encodings -- this
+ // returns true iff converting from first to second is a no-op.
+ if (!IsValidEncoding(from) || !IsValidEncoding(to)) {
+ return false; // we only work with valid encodings...
+ } else if (to == from) {
+ return true; // the trivial common case
+ }
+
+ if (to == UNKNOWN_ENCODING) {
+ return true; // all valid encodings are compatible with the unknown
+ }
+
+ if (from == UNKNOWN_ENCODING) {
+ return false; // no unknown encoding is compatible with one that is
+ }
+
+ if (from == ASCII_7BIT) {
+ return IsSupersetOfAscii7Bit(to);
+ }
+
+ return (from == ISO_8859_1 && to == MSFT_CP1252) ||
+ (from == ISO_8859_8 && to == HEBREW_VISUAL) ||
+ (from == HEBREW_VISUAL && to == ISO_8859_8) ||
+ (from == ISO_8859_9 && to == MSFT_CP1254) ||
+ (from == ISO_8859_11 && to == MSFT_CP874) ||
+ (from == JAPANESE_SHIFT_JIS && to == JAPANESE_CP932) ||
+ (from == CHINESE_BIG5 && to == CHINESE_BIG5_CP950) ||
+ (from == CHINESE_GB && to == GBK) ||
+ (from == CHINESE_GB && to == GB18030) ||
+ (from == CHINESE_EUC_CN && to == CHINESE_EUC_DEC) ||
+ (from == CHINESE_EUC_CN && to == CHINESE_CNS) ||
+ (from == CHINESE_EUC_DEC && to == CHINESE_EUC_CN) ||
+ (from == CHINESE_EUC_DEC && to == CHINESE_CNS) ||
+ (from == CHINESE_CNS && to == CHINESE_EUC_CN) ||
+ (from == CHINESE_CNS && to == CHINESE_EUC_DEC);
+}
+
+// To be a superset of 7-bit Ascii means that bytes 0...127 in the given
+// encoding represent the same characters as they do in ISO_8859_1.
+
+// TODO: This list could be expanded. Many other encodings are supersets
+// of 7-bit Ascii. In fact, Japanese JIS and Unicode are the only two
+// encodings that I know for a fact should *not* be in this list.
+bool IsSupersetOfAscii7Bit(Encoding e) {
+ switch (e) {
+ case ISO_8859_1:
+ case ISO_8859_2:
+ case ISO_8859_3:
+ case ISO_8859_4:
+ case ISO_8859_5:
+ case ISO_8859_6:
+ case ISO_8859_7:
+ case ISO_8859_8:
+ case ISO_8859_9:
+ case ISO_8859_10:
+ case JAPANESE_EUC_JP:
+ case JAPANESE_SHIFT_JIS:
+ case CHINESE_BIG5:
+ case CHINESE_GB:
+ case CHINESE_EUC_CN:
+ case KOREAN_EUC_KR:
+ case CHINESE_EUC_DEC:
+ case CHINESE_CNS:
+ case CHINESE_BIG5_CP950:
+ case JAPANESE_CP932:
+ case UTF8:
+ case UNKNOWN_ENCODING:
+ case ASCII_7BIT:
+ case RUSSIAN_KOI8_R:
+ case RUSSIAN_CP1251:
+ case MSFT_CP1252:
+ case RUSSIAN_KOI8_RU:
+ case MSFT_CP1250:
+ case ISO_8859_15:
+ case MSFT_CP1254:
+ case MSFT_CP1257:
+ case ISO_8859_11:
+ case MSFT_CP874:
+ case MSFT_CP1256:
+ case MSFT_CP1255:
+ case ISO_8859_8_I:
+ case HEBREW_VISUAL:
+ case CZECH_CP852:
+ case MSFT_CP1253:
+ case RUSSIAN_CP866:
+ case ISO_8859_13:
+ case GBK:
+ case GB18030:
+ case BIG5_HKSCS:
+ case MACINTOSH_ROMAN:
+ return true;
+ default:
+ return false;
+ }
+}
+
+// To be an 8-bit encoding means that there are fewer than 256 symbols.
+// Each byte determines a new character; there are no multi-byte sequences.
+
+// TODO: This list could maybe be expanded. Other encodings may be 8-bit.
+bool Is8BitEncoding(Encoding e) {
+ switch (e) {
+ case ASCII_7BIT:
+ case ISO_8859_1:
+ case ISO_8859_2:
+ case ISO_8859_3:
+ case ISO_8859_4:
+ case ISO_8859_5:
+ case ISO_8859_6:
+ case ISO_8859_7:
+ case ISO_8859_8:
+ case ISO_8859_8_I:
+ case ISO_8859_9:
+ case ISO_8859_10:
+ case ISO_8859_11:
+ case ISO_8859_13:
+ case ISO_8859_15:
+ case MSFT_CP1252:
+ case MSFT_CP1253:
+ case MSFT_CP1254:
+ case MSFT_CP1255:
+ case MSFT_CP1256:
+ case MSFT_CP1257:
+ case RUSSIAN_KOI8_R:
+ case RUSSIAN_KOI8_RU:
+ case RUSSIAN_CP866:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool IsCJKEncoding(Encoding e) {
+ switch (e) {
+ case JAPANESE_EUC_JP:
+ case JAPANESE_SHIFT_JIS:
+ case JAPANESE_JIS:
+ case CHINESE_BIG5:
+ case CHINESE_GB:
+ case CHINESE_EUC_CN:
+ case KOREAN_EUC_KR:
+ case CHINESE_EUC_DEC:
+ case CHINESE_CNS:
+ case CHINESE_BIG5_CP950:
+ case JAPANESE_CP932:
+ case ISO_2022_KR:
+ case GBK:
+ case GB18030:
+ case BIG5_HKSCS:
+ case ISO_2022_CN:
+ case HZ_GB_2312:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool IsHebrewEncoding(Encoding e) {
+ return (e == ISO_8859_8 ||
+ e == ISO_8859_8_I ||
+ e == MSFT_CP1255 ||
+ e == HEBREW_VISUAL);
+}
+
+
+
+bool IsRightToLeftEncoding(Encoding enc) {
+ switch (enc) {
+ case MSFT_CP1255:
+ case MSFT_CP1256:
+ case ARABIC_ENCODING:
+ case HEBREW_ENCODING:
+ case ISO_8859_8_I:
+ case HEBREW_VISUAL:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool IsLogicalRightToLeftEncoding(Encoding enc) {
+ return IsRightToLeftEncoding(enc) && !IsVisualRightToLeftEncoding(enc);
+}
+
+// Note that despite an RFC to the contrary, ARABIC_ENCODING (ISO-8859-6)
+// is NOT visual.
+bool IsVisualRightToLeftEncoding(Encoding enc) {
+ switch (enc) {
+ case HEBREW_ENCODING:
+ case HEBREW_VISUAL:
+ return true;
+ default:
+ return false;
+ }
+}
+
+
+
+
+
+bool IsIso2022Encoding(Encoding enc) {
+ return (IsIso2022JpOrVariant(enc) ||
+ enc == ISO_2022_KR ||
+ enc == ISO_2022_CN);
+}
+
+bool IsIso2022JpOrVariant(Encoding enc) {
+ return (enc == JAPANESE_JIS ||
+ enc == KDDI_ISO_2022_JP ||
+ enc == SOFTBANK_ISO_2022_JP);
+}
+
+bool IsShiftJisOrVariant(Encoding enc) {
+ return (enc == JAPANESE_SHIFT_JIS ||
+ enc == JAPANESE_CP932 ||
+ enc == KDDI_SHIFT_JIS ||
+ enc == DOCOMO_SHIFT_JIS ||
+ enc == SOFTBANK_SHIFT_JIS);
+}
+
+bool IsJapaneseCellPhoneCarrierSpecificEncoding(Encoding enc) {
+ return (enc == KDDI_ISO_2022_JP ||
+ enc == KDDI_SHIFT_JIS ||
+ enc == DOCOMO_SHIFT_JIS ||
+ enc == SOFTBANK_SHIFT_JIS ||
+ enc == SOFTBANK_ISO_2022_JP);
+}
+
+
+// *************************************************************
+// ENCODING NAMES
+// EncodingName() [Encoding to name]
+// MimeEncodingName() [Encoding to name]
+// EncodingFromName() [name to Encoding]
+// EncodingNameAliasToEncoding() [name to Encoding]
+// default_encoding_name()
+// invalid_encoding_name()
+// *************************************************************
+
+const char * EncodingName(const Encoding enc) {
+ if ( (enc < 0) || (enc >= kNumEncodings) )
+ return invalid_encoding_name();
+ return kEncodingInfoTable[enc].encoding_name_;
+}
+
+// TODO: Unify MimeEncodingName and EncodingName, or determine why
+// such a unification is not possible.
+
+const char * MimeEncodingName(Encoding enc) {
+ if ( (enc < 0) || (enc >= kNumEncodings) )
+ return ""; // TODO: Should this be invalid_encoding_name()?
+ return kEncodingInfoTable[enc].mime_encoding_name_;
+}
+
+bool EncodingFromName(const char* enc_name, Encoding *encoding) {
+ *encoding = UNKNOWN_ENCODING;
+ if ( enc_name == NULL ) return false;
+
+ for ( int i = 0; i < kNumEncodings; i++ ) {
+ if (!base::strcasecmp(enc_name, kEncodingInfoTable[i].encoding_name_) ) {
+ *encoding = static_cast<Encoding>(i);
+ return true;
+ }
+ }
+ return false;
+}
+
+// The encoding_map maps standard and non-standard encoding-names
+// (strings) to Encoding enums. It is used only by
+// EncodingNameAliasToEncoding. Note that the map uses
+// case-insensitive hash and comparison functions.
+
+typedef std::unordered_map<const char *, Encoding,
+ CStringAlnumCaseHash,
+ CStringAlnumCaseEqual> EncodingMap;
+
+static const EncodingMap& GetEncodingMap() {
+ static EncodingMap encoding_map;
+ if (!encoding_map.empty()) {
+ // Already initialized
+ return encoding_map;
+ }
+
+ // Initialize the map with all the "standard" encoding names,
+ // i.e., the ones returned by EncodingName and MimeEncodingName.
+ //
+ // First, add internal encoding names returned by EncodingName().
+ for (int i = 0; i < NUM_ENCODINGS; ++i) {
+ Encoding e = static_cast<Encoding>(i);
+ // Internal encoding names must be unique.
+ // The internal names are guaranteed to be unique by the CHECK_EQ.
+ const char *encoding_name = EncodingName(e);
+ // CHECK_EQ(0, encoding_map.count(encoding_name))
+ // << "Duplicate found for " << encoding_name;
+ encoding_map[encoding_name] = e;
+ }
+ // Then, add mime encoding names returned by MimeEncodingName().
+ // We don't override existing entries, to give precedence to entries
+ // added earlier.
+ for (int i = 0; i < NUM_ENCODINGS; ++i) {
+ Encoding e = static_cast<Encoding>(i);
+ // Note that MimeEncodingName() can return the same mime encoding
+ // name for different encoding enums like JAPANESE_SHIFT_JIS and
+ // KDDI_SHIFT_JIS. In that case, the encoding enum first seen
+ // will be the value for the encoding name in the map.
+ const char *mime_encoding_name = MimeEncodingName(e);
+ if (encoding_map.count(mime_encoding_name) == 0) {
+ encoding_map[mime_encoding_name] = e;
+ }
+ }
+
+ // Add some non-standard names: alternate spellings, common typos,
+ // etc. (It does no harm to add names already in the map.) Note
+ // that although the map is case-insensitive, by convention the
+ // keys are written here in lower case. For ease of maintenance,
+ // they are listed in alphabetical order.
+ encoding_map["5601"] = KOREAN_EUC_KR;
+ encoding_map["646"] = ASCII_7BIT;
+ encoding_map["852"] = CZECH_CP852;
+ encoding_map["866"] = RUSSIAN_CP866;
+ encoding_map["8859-1"] = ISO_8859_1;
+ encoding_map["ansi-1251"] = RUSSIAN_CP1251;
+ encoding_map["ansi_x3.4-1968"] = ASCII_7BIT;
+ encoding_map["arabic"] = ISO_8859_6;
+ encoding_map["ascii"] = ISO_8859_1;
+ encoding_map["ascii-7-bit"] = ASCII_7BIT; // not iana standard
+ encoding_map["asmo-708"] = ISO_8859_6;
+ encoding_map["bhaskar"] = BHASKAR;
+ encoding_map["big5"] = CHINESE_BIG5;
+ encoding_map["big5-cp950"] = CHINESE_BIG5_CP950; // not iana standard
+ encoding_map["big5-hkscs"] = BIG5_HKSCS;
+ encoding_map["chinese"] = CHINESE_GB;
+ encoding_map["cns"] = CHINESE_CNS; // not iana standard
+ encoding_map["cns11643"] = CHINESE_CNS;
+ encoding_map["cp1250"] = MSFT_CP1250; // not iana standard
+ encoding_map["cp1251"] = RUSSIAN_CP1251; // not iana standard
+ encoding_map["cp1252"] = MSFT_CP1252; // not iana standard
+ encoding_map["cp1253"] = MSFT_CP1253; // not iana standard
+ encoding_map["cp1254"] = MSFT_CP1254; // not iana standard
+ encoding_map["cp1255"] = MSFT_CP1255;
+ encoding_map["cp1256"] = MSFT_CP1256;
+ encoding_map["cp1257"] = MSFT_CP1257; // not iana standard
+ encoding_map["cp819"] = ISO_8859_1;
+ encoding_map["cp852"] = CZECH_CP852;
+ encoding_map["cp866"] = RUSSIAN_CP866;
+ encoding_map["cp-866"] = RUSSIAN_CP866;
+ encoding_map["cp874"] = MSFT_CP874;
+ encoding_map["cp932"] = JAPANESE_CP932; // not iana standard
+ encoding_map["cp950"] = CHINESE_BIG5_CP950; // not iana standard
+ encoding_map["csbig5"] = CHINESE_BIG5;
+ encoding_map["cseucjpkdfmtjapanese"] = JAPANESE_EUC_JP;
+ encoding_map["cseuckr"] = KOREAN_EUC_KR;
+ encoding_map["csgb2312"] = CHINESE_GB;
+ encoding_map["csibm852"] = CZECH_CP852;
+ encoding_map["csibm866"] = RUSSIAN_CP866;
+ encoding_map["csiso2022jp"] = JAPANESE_JIS;
+ encoding_map["csiso2022kr"] = ISO_2022_KR;
+ encoding_map["csiso58gb231280"] = CHINESE_GB;
+ encoding_map["csiso88598i"] = ISO_8859_8_I;
+ encoding_map["csisolatin1"] = ISO_8859_1;
+ encoding_map["csisolatin2"] = ISO_8859_2;
+ encoding_map["csisolatin3"] = ISO_8859_3;
+ encoding_map["csisolatin4"] = ISO_8859_4;
+ encoding_map["csisolatin5"] = ISO_8859_9;
+ encoding_map["csisolatin6"] = ISO_8859_10;
+ encoding_map["csisolatinarabic"] = ISO_8859_6;
+ encoding_map["csisolatincyrillic"] = ISO_8859_5;
+ encoding_map["csisolatingreek"] = ISO_8859_7;
+ encoding_map["csisolatinhebrew"] = ISO_8859_8;
+ encoding_map["csksc56011987"] = KOREAN_EUC_KR;
+ encoding_map["csmacintosh"] = MACINTOSH_ROMAN;
+ encoding_map["csn-369103"] = CZECH_CSN_369103;
+ encoding_map["csshiftjis"] = JAPANESE_SHIFT_JIS;
+ encoding_map["csunicode"] = UTF16BE;
+ encoding_map["csunicode11"] = UTF16BE;
+ encoding_map["csunicode11utf7"] = UTF7;
+ encoding_map["csunicodeascii"] = UTF16BE;
+ encoding_map["csunicodelatin1"] = UTF16BE;
+ encoding_map["cyrillic"] = ISO_8859_5;
+ encoding_map["ecma-114"] = ISO_8859_6;
+ encoding_map["ecma-118"] = ISO_8859_7;
+ encoding_map["elot_928"] = ISO_8859_7;
+ encoding_map["euc"] = CHINESE_EUC_DEC; // not iana standard
+ encoding_map["euc-cn"] = CHINESE_EUC_CN; // not iana standard
+ encoding_map["euc-dec"] = CHINESE_EUC_DEC; // not iana standard
+ encoding_map["euc-jp"] = JAPANESE_EUC_JP;
+ encoding_map["euc-kr"] = KOREAN_EUC_KR;
+ encoding_map["eucgb2312_cn"] = CHINESE_GB;
+ encoding_map["gb"] = CHINESE_GB; // not iana standard
+ encoding_map["gb18030"] = GB18030;
+ encoding_map["gb2132"] = CHINESE_GB; // common typo
+ encoding_map["gb2312"] = CHINESE_GB;
+ encoding_map["gb_2312-80"] = CHINESE_GB;
+ encoding_map["gbk"] = GBK;
+ encoding_map["greek"] = ISO_8859_7;
+ encoding_map["greek8"] = ISO_8859_7;
+ encoding_map["hebrew"] = ISO_8859_8;
+ encoding_map["htchanakya"] = HTCHANAKYA;
+ encoding_map["hz-gb-2312"] = HZ_GB_2312;
+ encoding_map["ibm819"] = ISO_8859_1;
+ encoding_map["ibm852"] = CZECH_CP852;
+ encoding_map["ibm874"] = MSFT_CP874;
+ encoding_map["iso-10646"] = UTF16BE;
+ encoding_map["iso-10646-j-1"] = UTF16BE;
+ encoding_map["iso-10646-ucs-2"] = UNICODE;
+ encoding_map["iso-10646-ucs-4"] = UTF32BE;
+ encoding_map["iso-10646-ucs-basic"] = UTF16BE;
+ encoding_map["iso-10646-unicode-latin1"] = UTF16BE;
+ encoding_map["iso-2022-cn"] = ISO_2022_CN;
+ encoding_map["iso-2022-jp"] = JAPANESE_JIS;
+ encoding_map["iso-2022-kr"] = ISO_2022_KR;
+ encoding_map["iso-8559-1"] = ISO_8859_1; // common typo
+ encoding_map["iso-874"] = MSFT_CP874;
+ encoding_map["iso-8858-1"] = ISO_8859_1; // common typo
+ // iso-8859-0 was a temporary name, eventually renamed iso-8859-15
+ encoding_map["iso-8859-0"] = ISO_8859_15;
+ encoding_map["iso-8859-1"] = ISO_8859_1;
+ encoding_map["iso-8859-10"] = ISO_8859_10;
+ encoding_map["iso-8859-11"] = ISO_8859_11;
+ encoding_map["iso-8859-13"] = ISO_8859_13;
+ encoding_map["iso-8859-15"] = ISO_8859_15;
+ encoding_map["iso-8859-2"] = ISO_8859_2;
+ encoding_map["iso-8859-3"] = ISO_8859_3;
+ encoding_map["iso-8859-4"] = ISO_8859_4;
+ encoding_map["iso-8859-5"] = ISO_8859_5;
+ encoding_map["iso-8859-6"] = ISO_8859_6;
+ encoding_map["iso-8859-7"] = ISO_8859_7;
+ encoding_map["iso-8859-8"] = ISO_8859_8;
+ encoding_map["iso-8859-8-i"] = ISO_8859_8_I;
+ encoding_map["iso-8859-9"] = ISO_8859_9;
+ encoding_map["iso-9959-1"] = ISO_8859_1; // common typo
+ encoding_map["iso-ir-100"] = ISO_8859_1;
+ encoding_map["iso-ir-101"] = ISO_8859_2;
+ encoding_map["iso-ir-109"] = ISO_8859_3;
+ encoding_map["iso-ir-110"] = ISO_8859_4;
+ encoding_map["iso-ir-126"] = ISO_8859_7;
+ encoding_map["iso-ir-127"] = ISO_8859_6;
+ encoding_map["iso-ir-138"] = ISO_8859_8;
+ encoding_map["iso-ir-144"] = ISO_8859_5;
+ encoding_map["iso-ir-148"] = ISO_8859_9;
+ encoding_map["iso-ir-149"] = KOREAN_EUC_KR;
+ encoding_map["iso-ir-157"] = ISO_8859_10;
+ encoding_map["iso-ir-58"] = CHINESE_GB;
+ encoding_map["iso-latin-1"] = ISO_8859_1;
+ encoding_map["iso_2022-cn"] = ISO_2022_CN;
+ encoding_map["iso_2022-kr"] = ISO_2022_KR;
+ encoding_map["iso_8859-1"] = ISO_8859_1;
+ encoding_map["iso_8859-10:1992"] = ISO_8859_10;
+ encoding_map["iso_8859-11"] = ISO_8859_11;
+ encoding_map["iso_8859-13"] = ISO_8859_13;
+ encoding_map["iso_8859-15"] = ISO_8859_15;
+ encoding_map["iso_8859-1:1987"] = ISO_8859_1;
+ encoding_map["iso_8859-2"] = ISO_8859_2;
+ encoding_map["iso_8859-2:1987"] = ISO_8859_2;
+ encoding_map["iso_8859-3"] = ISO_8859_3;
+ encoding_map["iso_8859-3:1988"] = ISO_8859_3;
+ encoding_map["iso_8859-4"] = ISO_8859_4;
+ encoding_map["iso_8859-4:1988"] = ISO_8859_4;
+ encoding_map["iso_8859-5"] = ISO_8859_5;
+ encoding_map["iso_8859-5:1988"] = ISO_8859_5;
+ encoding_map["iso_8859-6"] = ISO_8859_6;
+ encoding_map["iso_8859-6:1987"] = ISO_8859_6;
+ encoding_map["iso_8859-7"] = ISO_8859_7;
+ encoding_map["iso_8859-7:1987"] = ISO_8859_7;
+ encoding_map["iso_8859-8"] = ISO_8859_8;
+ encoding_map["iso_8859-8:1988:"] = ISO_8859_8;
+ encoding_map["iso_8859-9"] = ISO_8859_9;
+ encoding_map["iso_8859-9:1989"] = ISO_8859_9;
+ encoding_map["jagran"] = JAGRAN;
+ encoding_map["jis"] = JAPANESE_JIS; // not iana standard
+ encoding_map["koi8-cs"] = CZECH_CSN_369103;
+ encoding_map["koi8-r"] = RUSSIAN_KOI8_R;
+ encoding_map["koi8-ru"] = RUSSIAN_KOI8_RU; // not iana standard
+ encoding_map["koi8-u"] = RUSSIAN_KOI8_RU;
+ encoding_map["koi8r"] = RUSSIAN_KOI8_R; // not iana standard
+ encoding_map["koi8u"] = RUSSIAN_KOI8_RU; // not iana standard
+ encoding_map["korean"] = KOREAN_EUC_KR; // i assume this is what is meant
+ encoding_map["ks-c-5601"] = KOREAN_EUC_KR; // not iana standard
+ encoding_map["ks-c-5601-1987"] = KOREAN_EUC_KR; // not iana standard
+ encoding_map["ks_c_5601-1989"] = KOREAN_EUC_KR;
+ encoding_map["ksc"] = KOREAN_EUC_KR; // not iana standard
+ encoding_map["l1"] = ISO_8859_1;
+ encoding_map["l2"] = ISO_8859_2;
+ encoding_map["l3"] = ISO_8859_3;
+ encoding_map["l4"] = ISO_8859_4;
+ encoding_map["l5"] = ISO_8859_9;
+ encoding_map["l6"] = ISO_8859_10;
+ encoding_map["latin-1"] = ISO_8859_1; // not iana standard
+ encoding_map["latin1"] = ISO_8859_1;
+ encoding_map["latin2"] = ISO_8859_2;
+ encoding_map["latin3"] = ISO_8859_3;
+ encoding_map["latin4"] = ISO_8859_4;
+ encoding_map["latin5"] = ISO_8859_9;
+ encoding_map["latin6"] = ISO_8859_10;
+ encoding_map["mac"] = MACINTOSH_ROMAN;
+ encoding_map["macintosh"] = MACINTOSH_ROMAN;
+ encoding_map["macintosh-roman"] = MACINTOSH_ROMAN;
+ encoding_map["ms932"] = JAPANESE_CP932; // not iana standard
+ encoding_map["ms_kanji"] = JAPANESE_CP932;
+ encoding_map["shift-jis"] = JAPANESE_SHIFT_JIS;
+ encoding_map["shift_jis"] = JAPANESE_SHIFT_JIS;
+ encoding_map["sjis"] = JAPANESE_SHIFT_JIS; // not iana standard
+ encoding_map["sjs"] = JAPANESE_SHIFT_JIS; // not iana standard
+ encoding_map["sun_eu_greek"] = ISO_8859_7;
+ encoding_map["tab"] = TAMIL_BI;
+ encoding_map["tam"] = TAMIL_MONO;
+ encoding_map["tis-620"] = ISO_8859_11;
+ encoding_map["tscii"] = TSCII;
+ encoding_map["un"] = UNKNOWN_ENCODING; // not iana standard
+ encoding_map["unicode"] = UNICODE; // not iana standard
+ encoding_map["unicode-1-1-utf-7"] = UTF7;
+ encoding_map["unicode-1-1-utf-8"] = UTF8;
+ encoding_map["unicode-2-0-utf-7"] = UTF7;
+ encoding_map["unknown"] = UNKNOWN_ENCODING; // not iana standard
+ encoding_map["us"] = ISO_8859_1;
+ encoding_map["us-ascii"] = ISO_8859_1;
+ encoding_map["utf-16be"] = UTF16BE;
+ encoding_map["utf-16le"] = UTF16LE;
+ encoding_map["utf-32be"] = UTF32BE;
+ encoding_map["utf-32le"] = UTF32LE;
+ encoding_map["utf-7"] = UTF7;
+ encoding_map["utf-8"] = UTF8;
+ encoding_map["utf7"] = UTF7;
+ encoding_map["utf8"] = UTF8; // not iana standard
+ encoding_map["visual"] = HEBREW_VISUAL;
+ encoding_map["win-1250"] = MSFT_CP1250; // not iana standard
+ encoding_map["win-1251"] = RUSSIAN_CP1251; // not iana standard
+ encoding_map["window-874"] = MSFT_CP874;
+ encoding_map["windows-1250"] = MSFT_CP1250;
+ encoding_map["windows-1251"] = RUSSIAN_CP1251;
+ encoding_map["windows-1252"] = MSFT_CP1252;
+ encoding_map["windows-1253"] = MSFT_CP1253;
+ encoding_map["windows-1254"] = MSFT_CP1254;
+ encoding_map["windows-1255"] = MSFT_CP1255;
+ encoding_map["windows-1256"] = MSFT_CP1256;
+ encoding_map["windows-1257"] = MSFT_CP1257;
+ encoding_map["windows-31j"] = JAPANESE_CP932;
+ encoding_map["windows-874"] = MSFT_CP874;
+ encoding_map["windows-936"] = GBK;
+ encoding_map["x-big5"] = CHINESE_BIG5;
+ encoding_map["x-binaryenc"] = BINARYENC; // not iana standard
+ encoding_map["x-cp1250"] = MSFT_CP1250;
+ encoding_map["x-cp1251"] = RUSSIAN_CP1251;
+ encoding_map["x-cp1252"] = MSFT_CP1252;
+ encoding_map["x-cp1253"] = MSFT_CP1253;
+ encoding_map["x-cp1254"] = MSFT_CP1254;
+ encoding_map["x-cp1255"] = MSFT_CP1255;
+ encoding_map["x-cp1256"] = MSFT_CP1256;
+ encoding_map["x-cp1257"] = MSFT_CP1257;
+ encoding_map["x-euc-jp"] = JAPANESE_EUC_JP;
+ encoding_map["x-euc-tw"] = CHINESE_CNS;
+ encoding_map["x-gbk"] = GBK;
+ encoding_map["x-iso-10646-ucs-2-be"] = UTF16BE;
+ encoding_map["x-iso-10646-ucs-2-le"] = UTF16LE;
+ encoding_map["x-iso-10646-ucs-4-be"] = UTF32BE;
+ encoding_map["x-iso-10646-ucs-4-le"] = UTF32LE;
+ encoding_map["x-jis"] = JAPANESE_JIS; // not iana standard
+ encoding_map["x-mac-roman"] = MACINTOSH_ROMAN;
+ encoding_map["x-shift_jis"] = JAPANESE_SHIFT_JIS; // not iana standard
+ encoding_map["x-sjis"] = JAPANESE_SHIFT_JIS;
+ encoding_map["x-unicode-2-0-utf-7"] = UTF7;
+ encoding_map["x-utf8utf8"] = UTF8UTF8; // not iana standard
+ encoding_map["x-x-big5"] = CHINESE_BIG5;
+ encoding_map["zh_cn.euc"] = CHINESE_GB;
+ encoding_map["zh_tw-big5"] = CHINESE_BIG5;
+ encoding_map["zh_tw-euc"] = CHINESE_CNS;
+
+ // Remove they entry for the empty string, if any.
+ encoding_map.erase("");
+
+ return encoding_map;
+}
+
+// ----------------------------------------------------------------------
+// EncodingNameAliasToEncoding()
+//
+// This function takes an encoding name/alias and returns the Encoding
+// enum. The input is case insensitive. It is the union of the common
+// IANA standard names, the charset names used in Netscape Navigator,
+// and some common names we have been using.
+// See: http://www.iana.org/assignments/character-sets
+// http://physics.hallym.ac.kr/resource/relnotes/windows-2.0.html
+//
+// UNKNOWN_ENCODING is returned if none matches.
+//
+// TODO: Check if it is possible to remove the non-standard,
+// non-netscape-use names. It is because this routine is used for
+// encoding detections from html meta info. Non-standard names may
+// introduce noise on encoding detection.
+//
+// TODO: Unify EncodingNameAliasToEncoding and EncodingFromName,
+// or determine why such a unification is not possible.
+// ----------------------------------------------------------------------
+Encoding EncodingNameAliasToEncoding(const char *encoding_name) {
+ if (!encoding_name) {
+ return UNKNOWN_ENCODING;
+ }
+
+ const EncodingMap& encoding_map = GetEncodingMap();
+
+ EncodingMap::const_iterator emi = encoding_map.find(encoding_name);
+ if (emi != encoding_map.end()) {
+ return emi->second;
+ } else {
+ return UNKNOWN_ENCODING;
+ }
+}
+
+const char* default_encoding_name() {
+ return kEncodingInfoTable[LATIN1].encoding_name_;
+}
+
+static const char* const kInvalidEncodingName = "invalid_encoding";
+
+const char *invalid_encoding_name() {
+ return kInvalidEncodingName;
+}
+
+
+
+// *************************************************************
+// Miscellany
+// *************************************************************
+
+
+Encoding PreferredWebOutputEncoding(Encoding enc) {
+ return IsValidEncoding(enc)
+ ? kEncodingInfoTable[enc].preferred_web_output_encoding_
+ : UTF8;
+}
diff --git a/contrib/google-ced/util/encodings/encodings.h b/contrib/google-ced/util/encodings/encodings.h
new file mode 100644
index 0000000..6477974
--- /dev/null
+++ b/contrib/google-ced/util/encodings/encodings.h
@@ -0,0 +1,299 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_ENCODINGS_ENCODINGS_H_
+#define UTIL_ENCODINGS_ENCODINGS_H_
+
+// This interface defines the Encoding enum and various functions that
+// depend only on Encoding values.
+
+// A hash-function for Encoding, hash<Encoding>, is defined in
+// i18n/encodings/public/encodings-hash.h
+
+// On some Windows projects, UNICODE may be defined, which would prevent the
+// Encoding enum below from compiling. Note that this is a quick fix that does
+// not break any existing projects. The UNICODE enum may someday be changed
+// to something more specific and non-colliding, but this involves careful
+// testing of changes in many other projects.
+#undef UNICODE
+
+// NOTE: The Encoding enum must always start at 0. This assumption has
+// been made and used.
+
+#ifndef SWIG
+
+#include "util/encodings/encodings.pb.h"
+
+#else
+
+// TODO: Include a SWIG workaround header file.
+
+#endif
+
+const int kNumEncodings = NUM_ENCODINGS;
+
+// some of the popular encoding aliases
+// TODO: Make these static const Encoding values instead of macros.
+#define LATIN1 ISO_8859_1
+#define LATIN2 ISO_8859_2
+#define LATIN3 ISO_8859_3
+#define LATIN4 ISO_8859_4
+#define CYRILLIC ISO_8859_5
+#define ARABIC_ENCODING ISO_8859_6 // avoiding the same name as language
+#define GREEK_ENCODING ISO_8859_7 // avoiding the same name as language
+#define HEBREW_ENCODING ISO_8859_8 // avoiding the same name as language
+#define LATIN5 ISO_8859_9
+#define LATIN6 ISO_8859_10
+#define KOREAN_HANGUL KOREAN_EUC_KR
+
+// The default Encoding (LATIN1).
+Encoding default_encoding();
+
+
+
+// *************************************************************
+// Encoding predicates
+// IsValidEncoding()
+// IsEncEncCompatible
+// IsSupersetOfAscii7Bit
+// Is8BitEncoding
+// IsCJKEncoding
+// IsHebrewEncoding
+// IsRightToLeftEncoding
+// IsLogicalRightToLeftEncoding
+// IsVisualRightToLeftEncoding
+// IsIso2022Encoding
+// IsIso2022JpOrVariant
+// IsShiftJisOrVariant
+// IsJapaneseCellPhoneCarrierSpecificEncoding
+// *************************************************************
+
+// IsValidEncoding
+// ===================================
+//
+// Function to check if the input language enum is within range.
+//
+
+bool IsValidEncoding(Encoding enc);
+
+//
+// IsEncEncCompatible
+// ------------------
+//
+// This function is to determine whether or not converting from the
+// first encoding to the second requires any changes to the underlying
+// text (e.g. ASCII_7BIT is a subset of UTF8).
+//
+// TODO: the current implementation is likely incomplete. It would be
+// good to consider the full matrix of all pairs of encodings and to fish out
+// all compatible pairs.
+//
+bool IsEncEncCompatible(const Encoding from, const Encoding to);
+
+// To be a superset of 7-bit Ascii means that bytes 0...127 in the given
+// encoding represent the same characters as they do in ISO_8859_1.
+
+// WARNING: This function does not currently return true for all encodings that
+// are supersets of Ascii 7-bit.
+bool IsSupersetOfAscii7Bit(Encoding e);
+
+// To be an 8-bit encoding means that there are fewer than 256 symbols.
+// Each byte determines a new character; there are no multi-byte sequences.
+
+// WARNING: This function does not currently return true for all encodings that
+// are 8-bit encodings.
+bool Is8BitEncoding(Encoding e);
+
+// IsCJKEncoding
+// -------------
+//
+// This function returns true if the encoding is either Chinese
+// (simplified or traditional), Japanese, or Korean. Note: UTF8 is not
+// considered a CJK encoding.
+bool IsCJKEncoding(Encoding e);
+
+// IsHebrewEncoding
+// -------------
+//
+// This function returns true if the encoding is a Hebrew specific
+// encoding (not UTF8, etc).
+bool IsHebrewEncoding(Encoding e);
+
+// IsRightToLeftEncoding
+// ---------------------
+//
+// Returns true if the encoding is a right-to-left encoding.
+//
+// Note that the name of this function is somewhat misleading. There is nothing
+// "right to left" about these encodings. They merely contain code points for
+// characters in RTL languages such as Hebrew and Arabic. But this is also
+// true for UTF-8.
+//
+// TODO: Get rid of this function. The only special-case we
+// should need to worry about are visual encodings. Anything we
+// need to do for all 'RTL' encodings we need to do for UTF-8 as well.
+bool IsRightToLeftEncoding(Encoding enc);
+
+// IsLogicalRightToLeftEncoding
+// ----------------------------
+//
+// Returns true if the encoding is a logical right-to-left encoding.
+// Logical right-to-left encodings are those that the browser renders
+// right-to-left and applies the BiDi algorithm to. Therefore the characters
+// appear in reading order in the file, and indexing, snippet generation etc.
+// should all just work with no special processing.
+//
+// TODO: Get rid of this function. The only special-case we
+// should need to worry about are visual encodings.
+bool IsLogicalRightToLeftEncoding(Encoding enc);
+
+// IsVisualRightToLeftEncoding
+// ---------------------------
+//
+// Returns true if the encoding is a visual right-to-left encoding.
+// Visual right-to-left encodings are those that the browser renders
+// left-to-right and does not apply the BiDi algorithm to. Therefore each
+// line appears in reverse order in the file, lines are manually wrapped
+// by abusing <br> or <p> tags, etc. Visual RTL encoding is a relic of
+// the prehistoric days when browsers couldn't render right-to-left, but
+// unfortunately some visual pages persist to this day. These documents require
+// special processing so that we don't index or snippet them with each line
+// reversed.
+bool IsVisualRightToLeftEncoding(Encoding enc);
+
+// IsIso2022Encoding
+// -----------------
+//
+// Returns true if the encoding is a kind of ISO 2022 such as
+// ISO-2022-JP.
+bool IsIso2022Encoding(Encoding enc);
+
+// IsIso2022JpOrVariant
+// --------------------
+//
+// Returns true if the encoding is ISO-2022-JP or a variant such as
+// KDDI's ISO-2022-JP.
+bool IsIso2022JpOrVariant(Encoding enc);
+
+// IsShiftJisOrVariant
+// --------------------
+//
+// Returns true if the encoding is Shift_JIS or a variant such as
+// KDDI's Shift_JIS.
+bool IsShiftJisOrVariant(Encoding enc);
+
+// IsJapanesCellPhoneCarrierSpecificEncoding
+// -----------------------------------------
+//
+// Returns true if it's Japanese cell phone carrier specific encoding
+// such as KDDI_SHIFT_JIS.
+bool IsJapaneseCellPhoneCarrierSpecificEncoding(Encoding enc);
+
+
+
+// *************************************************************
+// ENCODING NAMES
+//
+// This interface defines a standard name for each valid encoding, and
+// a standard name for invalid encodings. (Some names use all upper
+// case, but others use mixed case.)
+//
+// EncodingName() [Encoding to name]
+// MimeEncodingName() [Encoding to name]
+// EncodingFromName() [name to Encoding]
+// EncodingNameAliasToEncoding() [name to Encoding]
+// default_encoding_name()
+// invalid_encoding_name()
+// *************************************************************
+
+// EncodingName
+// ------------
+//
+// Given the encoding, returns its standard name.
+// Return invalid_encoding_name() if the encoding is invalid.
+//
+const char* EncodingName(Encoding enc);
+
+//
+// MimeEncodingName
+// ----------------
+//
+// Return the "preferred MIME name" of an encoding.
+//
+// This name is suitable for using in HTTP headers, HTML tags,
+// and as the "charset" parameter of a MIME Content-Type.
+const char* MimeEncodingName(Encoding enc);
+
+
+// The maximum length of an encoding name
+const int kMaxEncodingNameSize = 50;
+
+// The standard name of the default encoding.
+const char* default_encoding_name();
+
+// The name used for an invalid encoding.
+const char* invalid_encoding_name();
+
+// EncodingFromName
+// ----------------
+//
+// If enc_name matches the standard name of an Encoding, using a
+// case-insensitive comparison, set *encoding to that Encoding and
+// return true. Otherwise set *encoding to UNKNOWN_ENCODING and
+// return false.
+//
+// REQUIRES: encoding must not be NULL.
+//
+bool EncodingFromName(const char* enc_name, Encoding *encoding);
+
+//
+// EncodingNameAliasToEncoding
+// ---------------------------
+//
+// If enc_name matches the standard name or an alias of an Encoding,
+// using a case-insensitive comparison, return that
+// Encoding. Otherwise, return UNKNOWN_ENCODING.
+//
+// Aliases include most mime-encoding names (e.g., "ISO-8859-7" for
+// GREEK), alternate names (e.g., "cyrillic" for ISO_8859_5) and
+// common variations with hyphens and underscores (e.g., "koi8-u" and
+// "koi8u" for RUSSIAN_KOI8_R).
+
+Encoding EncodingNameAliasToEncoding(const char *enc_name);
+
+// *************************************************************
+// Miscellany
+// *************************************************************
+
+// PreferredWebOutputEncoding
+// --------------------------
+//
+// Some multi-byte encodings use byte values that coincide with the
+// ASCII codes for HTML syntax characters <>"&' and browsers like MSIE
+// can misinterpret these, as indicated in an external XSS report from
+// 2007-02-15. Here, we map these dangerous encodings to safer ones. We
+// also use UTF8 instead of encodings that we don't support in our
+// output, and we generally try to be conservative in what we send out.
+// Where the client asks for single- or double-byte encodings that are
+// not as common, we substitute a more common single- or double-byte
+// encoding, if there is one, thereby preserving the client's intent
+// to use less space than UTF-8. This also means that characters
+// outside the destination set will be converted to HTML NCRs (&#NNN;)
+// if requested.
+Encoding PreferredWebOutputEncoding(Encoding enc);
+
+
+#endif // UTIL_ENCODINGS_ENCODINGS_H_
diff --git a/contrib/google-ced/util/encodings/encodings.pb.h b/contrib/google-ced/util/encodings/encodings.pb.h
new file mode 100644
index 0000000..ffbd716
--- /dev/null
+++ b/contrib/google-ced/util/encodings/encodings.pb.h
@@ -0,0 +1,181 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_ENCODINGS_ENCODINGS_PB_H_
+#define UTIL_ENCODINGS_ENCODINGS_PB_H_
+
+enum Encoding {
+ ISO_8859_1 = 0, // Teragram ASCII
+ ISO_8859_2 = 1, // Teragram Latin2
+ ISO_8859_3 = 2, // in BasisTech but not in Teragram
+ ISO_8859_4 = 3, // Teragram Latin4
+ ISO_8859_5 = 4, // Teragram ISO-8859-5
+ ISO_8859_6 = 5, // Teragram Arabic
+ ISO_8859_7 = 6, // Teragram Greek
+ ISO_8859_8 = 7, // Teragram Hebrew
+ ISO_8859_9 = 8, // in BasisTech but not in Teragram
+ ISO_8859_10 = 9, // in BasisTech but not in Teragram
+ JAPANESE_EUC_JP = 10, // Teragram EUC_JP
+ JAPANESE_SHIFT_JIS = 11, // Teragram SJS
+ JAPANESE_JIS = 12, // Teragram JIS
+ CHINESE_BIG5 = 13, // Teragram BIG5
+ CHINESE_GB = 14, // Teragram GB
+ CHINESE_EUC_CN = 15, // Misnamed. Should be EUC_TW. Was Basis Tech
+ // CNS11643EUC, before that Teragram EUC-CN(!)
+ // See //i18n/basistech/basistech_encodings.h
+ KOREAN_EUC_KR = 16, // Teragram KSC
+ UNICODE = 17, // Teragram Unicode
+ CHINESE_EUC_DEC = 18, // Misnamed. Should be EUC_TW. Was Basis Tech
+ // CNS11643EUC, before that Teragram EUC.
+ CHINESE_CNS = 19, // Misnamed. Should be EUC_TW. Was Basis Tech
+ // CNS11643EUC, before that Teragram CNS.
+ CHINESE_BIG5_CP950 = 20, // Teragram BIG5_CP950
+ JAPANESE_CP932 = 21, // Teragram CP932
+ UTF8 = 22,
+ UNKNOWN_ENCODING = 23,
+ ASCII_7BIT = 24, // ISO_8859_1 with all characters <= 127.
+ // Should be present only in the crawler
+ // and in the repository,
+ // *never* as a result of Document::encoding().
+ RUSSIAN_KOI8_R = 25, // Teragram KOI8R
+ RUSSIAN_CP1251 = 26, // Teragram CP1251
+
+ //----------------------------------------------------------
+ // These are _not_ output from teragram. Instead, they are as
+ // detected in the headers of usenet articles.
+ MSFT_CP1252 = 27, // 27: CP1252 aka MSFT euro ascii
+ RUSSIAN_KOI8_RU = 28, // CP21866 aka KOI8-U, used for Ukrainian.
+ // Misnamed, this is _not_ KOI8-RU but KOI8-U.
+ // KOI8-U is used much more often than KOI8-RU.
+ MSFT_CP1250 = 29, // CP1250 aka MSFT eastern european
+ ISO_8859_15 = 30, // aka ISO_8859_0 aka ISO_8859_1 euroized
+ //----------------------------------------------------------
+
+ //----------------------------------------------------------
+ // These are in BasisTech but not in Teragram. They are
+ // needed for new interface languages. Now detected by
+ // research langid
+ MSFT_CP1254 = 31, // used for Turkish
+ MSFT_CP1257 = 32, // used in Baltic countries
+ //----------------------------------------------------------
+
+ //----------------------------------------------------------
+ //----------------------------------------------------------
+ // New encodings detected by Teragram
+ ISO_8859_11 = 33, // aka TIS-620, used for Thai
+ MSFT_CP874 = 34, // used for Thai
+ MSFT_CP1256 = 35, // used for Arabic
+
+ //----------------------------------------------------------
+ // Detected as ISO_8859_8 by Teragram, but can be found in META tags
+ MSFT_CP1255 = 36, // Logical Hebrew Microsoft
+ ISO_8859_8_I = 37, // Iso Hebrew Logical
+ HEBREW_VISUAL = 38, // Iso Hebrew Visual
+ //----------------------------------------------------------
+
+ //----------------------------------------------------------
+ // Detected by research langid
+ CZECH_CP852 = 39,
+ CZECH_CSN_369103 = 40, // aka ISO_IR_139 aka KOI8_CS
+ MSFT_CP1253 = 41, // used for Greek
+ RUSSIAN_CP866 = 42,
+ //----------------------------------------------------------
+
+ //----------------------------------------------------------
+ // Handled by iconv in glibc
+ ISO_8859_13 = 43,
+ ISO_2022_KR = 44,
+ GBK = 45,
+ GB18030 = 46,
+ BIG5_HKSCS = 47,
+ ISO_2022_CN = 48,
+
+ //-----------------------------------------------------------
+ // Detected by xin liu's detector
+ // Handled by transcoder
+ // (Indic encodings)
+
+ TSCII = 49,
+ TAMIL_MONO = 50,
+ TAMIL_BI = 51,
+ JAGRAN = 52,
+
+
+ MACINTOSH_ROMAN = 53,
+ UTF7 = 54,
+ BHASKAR = 55, // Indic encoding - Devanagari
+ HTCHANAKYA = 56, // 56 Indic encoding - Devanagari
+
+ //-----------------------------------------------------------
+ // These allow a single place (inputconverter and outputconverter)
+ // to do UTF-16 <==> UTF-8 bulk conversions and UTF-32 <==> UTF-8
+ // bulk conversions, with interchange-valid checking on input and
+ // fallback if needed on ouput.
+ UTF16BE = 57, // big-endian UTF-16
+ UTF16LE = 58, // little-endian UTF-16
+ UTF32BE = 59, // big-endian UTF-32
+ UTF32LE = 60, // little-endian UTF-32
+ //-----------------------------------------------------------
+
+ //-----------------------------------------------------------
+ // An encoding that means "This is not text, but it may have some
+ // simple ASCII text embedded". Intended input conversion (not yet
+ // implemented) is to keep strings of >=4 seven-bit ASCII characters
+ // (follow each kept string with an ASCII space), delete the rest of
+ // the bytes. This will pick up and allow indexing of e.g. captions
+ // in JPEGs. No output conversion needed.
+ BINARYENC = 61,
+ //-----------------------------------------------------------
+
+ //-----------------------------------------------------------
+ // Some Web pages allow a mixture of HZ-GB and GB-2312 by using
+ // ~{ ... ~} for 2-byte pairs, and the browsers support this.
+ HZ_GB_2312 = 62,
+ //-----------------------------------------------------------
+
+ //-----------------------------------------------------------
+ // Some external vendors make the common input error of
+ // converting MSFT_CP1252 to UTF8 *twice*. No output conversion needed.
+ UTF8UTF8 = 63,
+ //-----------------------------------------------------------
+
+ //-----------------------------------------------------------
+ // Handled by transcoder for tamil language specific font
+ // encodings without the support for detection at present.
+ TAM_ELANGO = 64, // Elango - Tamil
+ TAM_LTTMBARANI = 65, // Barani - Tamil
+ TAM_SHREE = 66, // Shree - Tamil
+ TAM_TBOOMIS = 67, // TBoomis - Tamil
+ TAM_TMNEWS = 68, // TMNews - Tamil
+ TAM_WEBTAMIL = 69, // Webtamil - Tamil
+ //-----------------------------------------------------------
+
+ //-----------------------------------------------------------
+ // Shift_JIS variants used by Japanese cell phone carriers.
+ KDDI_SHIFT_JIS = 70,
+ DOCOMO_SHIFT_JIS = 71,
+ SOFTBANK_SHIFT_JIS = 72,
+ // ISO-2022-JP variants used by KDDI and SoftBank.
+ KDDI_ISO_2022_JP = 73,
+ SOFTBANK_ISO_2022_JP = 74,
+ //-----------------------------------------------------------
+
+ NUM_ENCODINGS = 75, // Always keep this at the end. It is not a
+ // valid Encoding enum, it is only used to
+ // indicate the total number of Encodings.
+};
+
+#endif // UTIL_ENCODINGS_ENCODINGS_PB_H_
diff --git a/contrib/google-ced/util/encodings/encodings_unittest.cc b/contrib/google-ced/util/encodings/encodings_unittest.cc
new file mode 100644
index 0000000..223e3e4
--- /dev/null
+++ b/contrib/google-ced/util/encodings/encodings_unittest.cc
@@ -0,0 +1,34 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/encodings/encodings.h"
+
+#include "gtest/gtest.h"
+
+TEST(EncodingsTest, EncodingNameAliasToEncoding) {
+ // Test that cases, non-alpha-numeric chars are ignored.
+ EXPECT_EQ(ISO_8859_1, EncodingNameAliasToEncoding("iso_8859_1"));
+ EXPECT_EQ(ISO_8859_1, EncodingNameAliasToEncoding("iso-8859-1"));
+
+ // Test that spaces are ignored.
+ EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF8"));
+ EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF 8"));
+ EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF-8"));
+
+ // Test alphanumeric differences are counted.
+ EXPECT_NE(UTF8, EncodingNameAliasToEncoding("UTF-7"));
+ EXPECT_NE(KOREAN_EUC_KR, EncodingNameAliasToEncoding("euc-jp"));
+}
diff --git a/contrib/google-ced/util/languages/languages.cc b/contrib/google-ced/util/languages/languages.cc
new file mode 100644
index 0000000..852351f
--- /dev/null
+++ b/contrib/google-ced/util/languages/languages.cc
@@ -0,0 +1,349 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/languages/languages.h"
+
+#include "util/basictypes.h"
+#include "util/string_util.h"
+
+
+Language default_language() {return ENGLISH;}
+
+
+// Language names and codes
+
+struct LanguageInfo {
+ const char * language_name_;
+ const char * language_code_639_1_; // the ISO-639-1 code for the language
+ const char * language_code_639_2_; // the ISO-639-2 code for the language
+ const char * language_code_other_; // some nonstandard code for the language
+};
+
+static const LanguageInfo kLanguageInfoTable[] = {
+ { "ENGLISH", "en", "eng", NULL},
+ { "DANISH", "da", "dan", NULL},
+ { "DUTCH", "nl", "dut", NULL},
+ { "FINNISH", "fi", "fin", NULL},
+ { "FRENCH", "fr", "fre", NULL},
+ { "GERMAN", "de", "ger", NULL},
+ { "HEBREW", "he", "heb", NULL},
+ { "ITALIAN", "it", "ita", NULL},
+ { "Japanese", "ja", "jpn", NULL},
+ { "Korean", "ko", "kor", NULL},
+ { "NORWEGIAN", "nb", "nor", NULL},
+ { "POLISH", "pl", "pol", NULL},
+ { "PORTUGUESE", "pt", "por", NULL},
+ { "RUSSIAN", "ru", "rus", NULL},
+ { "SPANISH", "es", "spa", NULL},
+ { "SWEDISH", "sv", "swe", NULL},
+ { "Chinese", "zh", "chi", "zh-CN"},
+ { "CZECH", "cs", "cze", NULL},
+ { "GREEK", "el", "gre", NULL},
+ { "ICELANDIC", "is", "ice", NULL},
+ { "LATVIAN", "lv", "lav", NULL},
+ { "LITHUANIAN", "lt", "lit", NULL},
+ { "ROMANIAN", "ro", "rum", NULL},
+ { "HUNGARIAN", "hu", "hun", NULL},
+ { "ESTONIAN", "et", "est", NULL},
+ // TODO: Although Teragram has two output names "TG_UNKNOWN_LANGUAGE"
+ // and "Unknown", they are essentially the same. Need to unify them.
+ // "un" and "ut" are invented by us, not from ISO-639.
+ //
+ { "TG_UNKNOWN_LANGUAGE", NULL, NULL, "ut"},
+ { "Unknown", NULL, NULL, "un"},
+ { "BULGARIAN", "bg", "bul", NULL},
+ { "CROATIAN", "hr", "scr", NULL},
+ { "SERBIAN", "sr", "scc", NULL},
+ { "IRISH", "ga", "gle", NULL},
+ { "GALICIAN", "gl", "glg", NULL},
+ // Impossible to tell Tagalog from Filipino at the moment.
+ // Use ISO 639-2 code for Filipino here.
+ { "TAGALOG", NULL, "fil", NULL},
+ { "TURKISH", "tr", "tur", NULL},
+ { "UKRAINIAN", "uk", "ukr", NULL},
+ { "HINDI", "hi", "hin", NULL},
+ { "MACEDONIAN", "mk", "mac", NULL},
+ { "BENGALI", "bn", "ben", NULL},
+ { "INDONESIAN", "id", "ind", NULL},
+ { "LATIN", "la", "lat", NULL},
+ { "MALAY", "ms", "may", NULL},
+ { "MALAYALAM", "ml", "mal", NULL},
+ { "WELSH", "cy", "wel", NULL},
+ { "NEPALI", "ne", "nep", NULL},
+ { "TELUGU", "te", "tel", NULL},
+ { "ALBANIAN", "sq", "alb", NULL},
+ { "TAMIL", "ta", "tam", NULL},
+ { "BELARUSIAN", "be", "bel", NULL},
+ { "JAVANESE", "jw", "jav", NULL},
+ { "OCCITAN", "oc", "oci", NULL},
+ { "URDU", "ur", "urd", NULL},
+ { "BIHARI", "bh", "bih", NULL},
+ { "GUJARATI", "gu", "guj", NULL},
+ { "THAI", "th", "tha", NULL},
+ { "ARABIC", "ar", "ara", NULL},
+ { "CATALAN", "ca", "cat", NULL},
+ { "ESPERANTO", "eo", "epo", NULL},
+ { "BASQUE", "eu", "baq", NULL},
+ { "INTERLINGUA", "ia", "ina", NULL},
+ { "KANNADA", "kn", "kan", NULL},
+ { "PUNJABI", "pa", "pan", NULL},
+ { "SCOTS_GAELIC", "gd", "gla", NULL},
+ { "SWAHILI", "sw", "swa", NULL},
+ { "SLOVENIAN", "sl", "slv", NULL},
+ { "MARATHI", "mr", "mar", NULL},
+ { "MALTESE", "mt", "mlt", NULL},
+ { "VIETNAMESE", "vi", "vie", NULL},
+ { "FRISIAN", "fy", "fry", NULL},
+ { "SLOVAK", "sk", "slo", NULL},
+ { "ChineseT",
+ NULL, NULL, // We intentionally set these 2 fields to NULL to avoid
+ // confusion between CHINESE_T and CHINESE.
+ "zh-TW"},
+ { "FAROESE", "fo", "fao", NULL},
+ { "SUNDANESE", "su", "sun", NULL},
+ { "UZBEK", "uz", "uzb", NULL},
+ { "AMHARIC", "am", "amh", NULL},
+ { "AZERBAIJANI", "az", "aze", NULL},
+ { "GEORGIAN", "ka", "geo", NULL},
+ { "TIGRINYA", "ti", "tir", NULL},
+ { "PERSIAN", "fa", "per", NULL},
+ { "BOSNIAN", "bs", "bos", NULL},
+ { "SINHALESE", "si", "sin", NULL},
+ { "NORWEGIAN_N", "nn", "nno", NULL},
+ { "PORTUGUESE_P", NULL, NULL, "pt-PT"},
+ { "PORTUGUESE_B", NULL, NULL, "pt-BR"},
+ { "XHOSA", "xh", "xho", NULL},
+ { "ZULU", "zu", "zul", NULL},
+ { "GUARANI", "gn", "grn", NULL},
+ { "SESOTHO", "st", "sot", NULL},
+ { "TURKMEN", "tk", "tuk", NULL},
+ { "KYRGYZ", "ky", "kir", NULL},
+ { "BRETON", "br", "bre", NULL},
+ { "TWI", "tw", "twi", NULL},
+ { "YIDDISH", "yi", "yid", NULL},
+ { "SERBO_CROATIAN", "sh", NULL, NULL},
+ { "SOMALI", "so", "som", NULL},
+ { "UIGHUR", "ug", "uig", NULL},
+ { "KURDISH", "ku", "kur", NULL},
+ { "MONGOLIAN", "mn", "mon", NULL},
+ { "ARMENIAN", "hy", "arm", NULL},
+ { "LAOTHIAN", "lo", "lao", NULL},
+ { "SINDHI", "sd", "snd", NULL},
+ { "RHAETO_ROMANCE", "rm", "roh", NULL},
+ { "AFRIKAANS", "af", "afr", NULL},
+ { "LUXEMBOURGISH", "lb", "ltz", NULL},
+ { "BURMESE", "my", "bur", NULL},
+ // KHMER is known as Cambodian for Google user interfaces.
+ { "KHMER", "km", "khm", NULL},
+ { "TIBETAN", "bo", "tib", NULL},
+ { "DHIVEHI", "dv", "div", NULL},
+ { "CHEROKEE", NULL, "chr", NULL},
+ { "SYRIAC", NULL, "syr", NULL},
+ { "LIMBU", NULL, NULL, "sit-NP"},
+ { "ORIYA", "or", "ori", NULL},
+ { "ASSAMESE", "as", "asm", NULL},
+ { "CORSICAN", "co", "cos", NULL},
+ { "INTERLINGUE", "ie", "ine", NULL},
+ { "KAZAKH", "kk", "kaz", NULL},
+ { "LINGALA", "ln", "lin", NULL},
+ { "MOLDAVIAN", "mo", "mol", NULL},
+ { "PASHTO", "ps", "pus", NULL},
+ { "QUECHUA", "qu", "que", NULL},
+ { "SHONA", "sn", "sna", NULL},
+ { "TAJIK", "tg", "tgk", NULL},
+ { "TATAR", "tt", "tat", NULL},
+ { "TONGA", "to", "tog", NULL},
+ { "YORUBA", "yo", "yor", NULL},
+ { "CREOLES_AND_PIDGINS_ENGLISH_BASED", NULL, "cpe", NULL},
+ { "CREOLES_AND_PIDGINS_FRENCH_BASED", NULL, "cpf", NULL},
+ { "CREOLES_AND_PIDGINS_PORTUGUESE_BASED", NULL, "cpp", NULL},
+ { "CREOLES_AND_PIDGINS_OTHER", NULL, "crp", NULL},
+ { "MAORI", "mi", "mao", NULL},
+ { "WOLOF", "wo", "wol", NULL},
+ { "ABKHAZIAN", "ab", "abk", NULL},
+ { "AFAR", "aa", "aar", NULL},
+ { "AYMARA", "ay", "aym", NULL},
+ { "BASHKIR", "ba", "bak", NULL},
+ { "BISLAMA", "bi", "bis", NULL},
+ { "DZONGKHA", "dz", "dzo", NULL},
+ { "FIJIAN", "fj", "fij", NULL},
+ { "GREENLANDIC", "kl", "kal", NULL},
+ { "HAUSA", "ha", "hau", NULL},
+ { "HAITIAN_CREOLE", "ht", NULL, NULL},
+ { "INUPIAK", "ik", "ipk", NULL},
+ { "INUKTITUT", "iu", "iku", NULL},
+ { "KASHMIRI", "ks", "kas", NULL},
+ { "KINYARWANDA", "rw", "kin", NULL},
+ { "MALAGASY", "mg", "mlg", NULL},
+ { "NAURU", "na", "nau", NULL},
+ { "OROMO", "om", "orm", NULL},
+ { "RUNDI", "rn", "run", NULL},
+ { "SAMOAN", "sm", "smo", NULL},
+ { "SANGO", "sg", "sag", NULL},
+ { "SANSKRIT", "sa", "san", NULL},
+ { "SISWANT", "ss", "ssw", NULL},
+ { "TSONGA", "ts", "tso", NULL},
+ { "TSWANA", "tn", "tsn", NULL},
+ { "VOLAPUK", "vo", "vol", NULL},
+ { "ZHUANG", "za", "zha", NULL},
+ { "KHASI", NULL, "kha", NULL},
+ { "SCOTS", NULL, "sco", NULL},
+ { "GANDA", "lg", "lug", NULL},
+ { "MANX", "gv", "glv", NULL},
+ { "MONTENEGRIN", NULL, NULL, "sr-ME"},
+ { "XX", NULL, NULL, "XX"},
+};
+
+COMPILE_ASSERT(arraysize(kLanguageInfoTable) == NUM_LANGUAGES + 1,
+ kLanguageInfoTable_has_incorrect_length);
+
+
+// LANGUAGE NAMES
+
+const char* default_language_name() {
+ return kLanguageInfoTable[ENGLISH].language_name_;
+}
+
+static const char* const kInvalidLanguageName = "invalid_language";
+
+const char *invalid_language_name() {
+ return kInvalidLanguageName;
+}
+
+const char* LanguageName(Language lang) {
+ return IsValidLanguage(lang)
+ ? kLanguageInfoTable[lang].language_name_
+ : kInvalidLanguageName;
+}
+
+
+
+// LANGUAGE CODES
+
+
+// The space before invalid_language_code is intentional. It is used
+// to prevent it matching any two letter language code.
+//
+static const char* const kInvalidLanguageCode = " invalid_language_code";
+
+const char *invalid_language_code() {
+ return kInvalidLanguageCode;
+}
+
+const char * LanguageCode(Language lang) {
+ if (! IsValidLanguage(lang))
+ return kInvalidLanguageCode;
+ const LanguageInfo& info = kLanguageInfoTable[lang];
+ if (info.language_code_639_1_) {
+ return info.language_code_639_1_;
+ } else if (info.language_code_639_2_) {
+ return info.language_code_639_2_;
+ } else if (info.language_code_other_) {
+ return info.language_code_other_;
+ } else {
+ return kInvalidLanguageCode;
+ }
+}
+
+const char* default_language_code() {
+ return kLanguageInfoTable[ENGLISH].language_code_639_1_;
+}
+
+const char* LanguageCodeISO639_1(Language lang) {
+ if (! IsValidLanguage(lang))
+ return kInvalidLanguageCode;
+ if (const char* code = kLanguageInfoTable[lang].language_code_639_1_)
+ return code;
+ return kInvalidLanguageCode;
+}
+
+const char* LanguageCodeISO639_2(Language lang) {
+ if (! IsValidLanguage(lang))
+ return kInvalidLanguageCode;
+ if (const char* code = kLanguageInfoTable[lang].language_code_639_2_)
+ return code;
+ return kInvalidLanguageCode;
+}
+
+const char* LanguageCodeWithDialects(Language lang) {
+ if (lang == CHINESE)
+ return "zh-CN";
+ return LanguageCode(lang);
+}
+
+
+
+bool LanguageFromCode(const char* lang_code, Language *language) {
+ *language = UNKNOWN_LANGUAGE;
+ if ( lang_code == NULL ) return false;
+
+ for ( int i = 0 ; i < kNumLanguages ; i++ ) {
+ const LanguageInfo& info = kLanguageInfoTable[i];
+ if ((info.language_code_639_1_ &&
+ !base::strcasecmp(lang_code, info.language_code_639_1_)) ||
+ (info.language_code_639_2_ &&
+ !base::strcasecmp(lang_code, info.language_code_639_2_)) ||
+ (info.language_code_other_ &&
+ !base::strcasecmp(lang_code, info.language_code_other_))) {
+ *language = static_cast<Language>(i);
+ return true;
+ }
+ }
+
+ // For convenience, this function can also parse the non-standard
+ // five-letter language codes "zh-cn" and "zh-tw" which are used by
+ // front-ends such as GWS to distinguish Simplified from Traditional
+ // Chinese.
+ if (!base::strcasecmp(lang_code, "zh-cn") ||
+ !base::strcasecmp(lang_code, "zh_cn")) {
+ *language = CHINESE;
+ return true;
+ }
+ if (!base::strcasecmp(lang_code, "zh-tw") ||
+ !base::strcasecmp(lang_code, "zh_tw")) {
+ *language = CHINESE_T;
+ return true;
+ }
+ if (!base::strcasecmp(lang_code, "sr-me") ||
+ !base::strcasecmp(lang_code, "sr_me")) {
+ *language = MONTENEGRIN;
+ return true;
+ }
+
+ // Process language-code synonyms.
+ if (!base::strcasecmp(lang_code, "he")) {
+ *language = HEBREW; // Use "iw".
+ return true;
+ }
+ if (!base::strcasecmp(lang_code, "in")) {
+ *language = INDONESIAN; // Use "id".
+ return true;
+ }
+ if (!base::strcasecmp(lang_code, "ji")) {
+ *language = YIDDISH; // Use "yi".
+ return true;
+ }
+
+ // Process language-detection synonyms.
+ // These distinct languages cannot be differentiated by our current
+ // language-detection algorithms.
+ if (!base::strcasecmp(lang_code, "fil")) {
+ *language = TAGALOG;
+ return true;
+ }
+
+ return false;
+}
diff --git a/contrib/google-ced/util/languages/languages.h b/contrib/google-ced/util/languages/languages.h
new file mode 100644
index 0000000..4237961
--- /dev/null
+++ b/contrib/google-ced/util/languages/languages.h
@@ -0,0 +1,381 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_LANGUAGES_LANGUAGES_H_
+#define UTIL_LANGUAGES_LANGUAGES_H_
+
+// This interface defines the Language enum and functions that depend
+// only on Language values.
+
+// A hash-function for Language, hash<Language>, is defined in
+// i18n/languages/public/languages-hash.h
+
+#ifndef SWIG
+// Language enum defined in languages.proto
+// Also description on how to add languages.
+#include "util/languages/languages.pb.h"
+
+#else
+
+// TODO: Include a header containing swig-compatible enum.
+
+#endif
+
+const int kNumLanguages = NUM_LANGUAGES;
+
+// Return the default language (ENGLISH).
+Language default_language();
+
+
+// *******************************************
+// Language predicates
+// IsValidLanguage()
+// IS_LANGUAGE_UNKNOWN()
+// IsCJKLanguage()
+// IsChineseLanguage()
+// IsNorwegianLanguage()
+// IsPortugueseLanguage()
+// IsRightToLeftLanguage()
+// IsMaybeRightToLeftLanguage()
+// IsSameLanguage()
+// IsScriptRequiringLongerSnippets()
+// *******************************************
+
+// IsValidLanguage
+// ===============
+//
+// Function to check if the input is within range of the Language enum. If
+// IsValidLanguage(lang) returns true, it is safe to call
+// static_cast<Language>(lang).
+//
+inline bool IsValidLanguage(int lang) {
+ return ((lang >= 0) && (lang < kNumLanguages));
+}
+
+// Return true if the language is "unknown". (This function was
+// previously a macro, hence the spelling in all caps.)
+//
+inline bool IS_LANGUAGE_UNKNOWN(Language lang) {
+ return lang == TG_UNKNOWN_LANGUAGE || lang == UNKNOWN_LANGUAGE;
+}
+
+// IsCJKLanguage
+// -------------
+//
+// This function returns true if the language is either Chinese
+// (simplified or traditional), Japanese, or Korean.
+bool IsCJKLanguage(Language lang);
+
+// IsChineseLanguage
+// -----------------
+//
+// This function returns true if the language is either Chinese
+// (simplified or traditional)
+bool IsChineseLanguage(Language lang);
+
+// IsNorwegianLanguage
+// --------------------
+//
+// This function returns true if the language is any of the Norwegian
+// (regular or Nynorsk).
+bool IsNorwegianLanguage(Language lang);
+
+// IsPortugueseLanguage
+// --------------------
+//
+// This function returns true if the language is any of the Portuguese
+// languages (regular, Portugal or Brazil)
+bool IsPortugueseLanguage(Language lang);
+
+// IsSameLanguage
+// --------------
+//
+// WARNING: This function provides only a simple test on the values of
+// the two Language arguments. It returns false if either language is
+// invalid. It returns true if the language arguments are equal, or
+// if they are both Chinese languages, both Norwegian languages, or
+// both Portuguese languages, as defined by IsChineseLanguage,
+// IsNorwegianLanguage, and IsPortugueseLanguage. Otherwise it returns
+// false.
+bool IsSameLanguage(Language lang1, Language lang2);
+
+
+// IsRightToLeftLanguage
+// ---------------------
+//
+// This function returns true if the language is only written right-to-left
+// (E.g., Hebrew, Arabic, Persian etc.)
+//
+// IMPORTANT NOTE: Technically we're talking about scripts, not languages.
+// There are languages that can be written in more than one script.
+// Examples:
+// - Kurdish and Azeri ('AZERBAIJANI') can be written left-to-right in
+// Latin or Cyrillic script, and right-to-left in Arabic script.
+// - Sindhi and Punjabi are written in different scripts, depending on
+// region and dialect.
+// - Turkmen used an Arabic script historically, but not any more.
+// - Pashto and Uyghur can use Arabic script, but use a Roman script
+// on the Internet.
+// - Kashmiri and Urdu are written either with Arabic or Devanagari script.
+//
+// This function only returns true for languages that are always, unequivocally
+// written in right-to-left script.
+//
+// TODO: If we want to do anything special with multi-script languages
+// we should create new 'languages' for each language+script, as we do for
+// traditional vs. simplified Chinese. However most such languages are rare in
+// use and even rarer on the web, so this is unlikely to be something we'll
+// be concerned with for a while.
+bool IsRightToLeftLanguage(Language lang);
+
+// IsMaybeRightToLeftLanguage
+// --------------------------
+//
+// This function returns true if the language may appear on the web in a
+// right-to-left script (E.g., Hebrew, Arabic, Persian, Urdu, Kurdish, etc.)
+//
+// NOTE: See important notes under IsRightToLeftLanguage(...).
+//
+// This function returns true for languages that *may* appear on the web in a
+// right-to-left script, even if they may also appear in a left-to-right
+// script.
+//
+// This function should typically be used in cases where doing some work on
+// left-to-right text would be OK (usually a no-op), and this function is used
+// just to cut down on unnecessary work on regular, LTR text.
+bool IsMaybeRightToLeftLanguage(Language lang);
+
+// IsScriptRequiringLongerSnippets
+// --------------------
+//
+// This function returns true if the script chracteristics require longer
+// snippet length (Devanagari, Bengali, Gurmukhi,
+// Gujarati, Oriya, Tamil, Telugu, Kannada, Malayalam).
+// COMMENTED OUT TO REDUCE DEPENDENCIES ON GOOGLE3 CODE
+// bool IsScriptRequiringLongerSnippets(UnicodeScript script);
+
+
+// *******************************************
+// LANGUAGE NAMES
+//
+// This interface defines a standard name for each valid Language,
+// and a standard name for invalid languages. Some language names use all
+// uppercase letters, but others use mixed case.
+// LanguageName() [Language to name]
+// LanguageEnumName() [language to enum name]
+// LanguageFromName() [name to Language]
+// default_language_name()
+// invalid_language_name()
+// *******************************************
+
+// Given a Language, returns its standard name.
+// Return invalid_language_name() if the language is invalid.
+const char* LanguageName(Language lang);
+
+// Given a Language, return the name of the enum constant for that
+// language. In all but a few cases, this is the same as its standard
+// name. For example, LanguageName(CHINESE) returns "Chinese", but
+// LanguageEnumName(CHINESE) returns "CHINESE". This is intended for
+// code that is generating C++ code, where the enum constant is more
+// useful than its integer value. Return "NUM_LANGUAGES" if
+// the language is invalid.
+const char* LanguageEnumName(Language lang);
+
+// The maximum length of a standard language name.
+const int kMaxLanguageNameSize = 50;
+
+// The standard name for the default language.
+const char* default_language_name();
+
+// The standard name for all invalid languages.
+const char* invalid_language_name();
+
+// If lang_name matches the standard name of a Language, using a
+// case-insensitive comparison, set *language to that Language and
+// return true.
+// Otherwise, set *language to UNKNOWN_LANGUAGE and return false.
+//
+// For backwards compatibility, "HATIAN_CREOLE" is allowed as a name
+// for HAITIAN_CREOLE, and "QUECHAU" is allowed as a name for QUECHUA.
+// For compatibility with LanguageEnumName, "UNKNOWN_LANGUAGE" is allowed
+// as a name for UNKNOWN_LANGUAGE (the return value is true in this case,
+// as it is for "Unknown"), and "CHINESE_T" is allowed as a name for
+// CHINESE_T (i.e., a synonym for "ChineseT").
+//
+// REQUIRES: language must not be NULL.
+//
+bool LanguageFromName(const char* lang_name, Language *language);
+
+
+
+// *******************************************
+// LANGUAGE CODES
+//
+// This interface defines a standard code for each valid language, and
+// a standard code for invalid languages. These are derived from ISO codes,
+// with some Google additions.
+// LanguageCode()
+// default_language_code()
+// invalid_language_code()
+// LanguageCodeWithDialects()
+// LanguageCodeISO639_1()
+// LanguageCodeISO639_2()
+// *******************************************
+
+// Given a Language, return its standard code. There are Google-specific codes:
+// For CHINESE_T, return "zh-TW".
+// For TG_UNKNOWN_LANGUAGE, return "ut".
+// For UNKNOWN_LANGUAGE, return "un".
+// For PORTUGUESE_P, return "pt-PT".
+// For PORTUGUESE_B, return "pt-BR".
+// For LIMBU, return "sit-NP".
+// For CHEROKEE, return "chr".
+// For SYRIAC, return "syr".
+// Otherwise return the ISO 639-1 two-letter language code for lang.
+// If lang is invalid, return invalid_language_code().
+//
+// NOTE: See the note below about the codes for Chinese languages.
+//
+const char* LanguageCode(Language lang);
+
+// The maximum length of a language code.
+const int kMaxLanguageCodeSize = 50;
+
+// The standard code for the default language.
+const char* default_language_code();
+
+// The standard code for all invalid languages.
+const char* invalid_language_code();
+
+
+// --------------------------------------------
+// NOTE: CHINESE LANGUAGE CODES
+//
+// There are three functions that return codes for Chinese languages.
+// LanguageCode(lang) and LanguageCodeWithDialects(lang) are defined here.
+// LanguageCode(lang, encoding) is defined in i18n/encodings.lang_enc.h.
+// The following list shows the different results.
+//
+// LanguageCode(CHINESE) returns "zh"
+// LanguageCode(CHINESE_T) returns "zh-TW".
+//
+// LanguageCodeWithDialects(CHINESE) returns "zh-CN".
+// LanguageCodeWithDialects(CHINESE_T) returns "zh-TW".
+//
+// LanguageCode(CHINESE_T, <any encoding>) returns "zh-TW".
+// LanguageCode(CHINESE, CHINESE_BIG5) returns "zh-TW".
+// LanguageCode(CHINESE, <any other encoding>) returns "zh-CN".
+//
+// --------------------------------------------
+
+// LanguageCodeWithDialects
+// ------------------------
+//
+// If lang is CHINESE, return "zh-CN". Otherwise return LanguageCode(lang).
+const char* LanguageCodeWithDialects(Language lang);
+
+// LanguageCodeISO639_1
+// --------------------
+//
+// Return the ISO 639-1 two-letter language code for lang.
+// Return invalid_language_code() if lang is invalid or does not have
+// an ISO 639-1 two-letter language code.
+const char* LanguageCodeISO639_1(Language lang);
+
+// LanguageCodeISO639_2
+// --------------------
+//
+// Return the ISO 639-2 three-letter language for lang.
+// Return invalid_language_code() if lang is invalid or does not have
+// an ISO 639-2 three-letter language code.
+const char* LanguageCodeISO639_2(Language lang);
+
+// LanguageFromCode
+// ----------------
+//
+// If lang_code matches the code for a Language, using a case-insensitive
+// comparison, set *lang to that Language and return true.
+// Otherwise, set *lang to UNKNOWN_LANGUAGE and return false.
+//
+// lang_code can be an ISO 639-1 (two-letter) code, an ISO 639-2
+// (three-letter) code, or a Google-specific code (see LanguageCode).
+//
+// Certain language-code aliases are also allowed:
+// For "zh-cn" and "zh_cn", set *lang to CHINESE.
+// For "zh-tw" and "zh_tw", set *lang to CHINESE_T.
+// For "he", set *lang to HEBREW.
+// For "in", set *lang to INDONESIAN.
+// For "ji", set *lang to YIDDISH.
+// For "fil", set *lang to TAGALOG.
+//
+// REQUIRES: 'lang' must not be NULL.
+bool LanguageFromCode(const char* lang_code, Language *language);
+
+
+// LanguageFromCodeOrName
+// ----------------------
+//
+// If lang_code_or_name is a language code or a language name.
+// set *language to the corresponding Language and return true.
+// Otherwise set *language to UNKNOWN_LANGUAGE and return false.
+//
+bool LanguageFromCodeOrName(const char* lang_code_or_name,
+ Language* language);
+
+// LanguageNameFromCode
+// --------------------
+//
+// If language_code is the code for a Language (see LanguageFromCode),
+// return the standard name of that language (see LanguageName).
+// Otherwise return invalid_language_name().
+//
+const char* LanguageNameFromCode(const char* language_code);
+
+
+// Miscellany
+
+// LanguageCodeToUnderscoreForm
+// ----------------------------
+//
+// Given a language code, convert the dash "-" to underscore "_".
+//
+// Specifically, if result_length <= strlen(lang_code), set result[0]
+// to '\0' and return false. Otherwise, copy lang_code to result,
+// converting every dash to an underscore, converting every character
+// before the first dash or underscore to lower case, and converting
+// every character after the first dash or underscore to upper
+// case. If there is no dash or underscore, convert the entire string
+// to lower case.
+//
+// REQUIRES: 'lang_code' must not be NULL. 'result' must not be NULL.
+
+bool LanguageCodeToUnderscoreForm(const char* lang_code,
+ char* result,
+ int result_length);
+
+//
+// AlwaysPutInExpectedRestrict
+// ---------------------------
+//
+// For Web pages in certain top-level domains, Web Search always
+// applies a "country restrict". If 'tld' matches one of those, using
+// a case-SENSITIVE comparison, set *expected_language to the Language
+// most commonly found in that top-level domain and return true.
+// Otherwise, set *expected_language to UNKNOWN_LANGUAGE and return false.
+bool AlwaysPutInExpectedRestrict(const char *tld, Language *expected_language);
+
+
+#endif // UTIL_LANGUAGES_LANGUAGES_H_
diff --git a/contrib/google-ced/util/languages/languages.pb.h b/contrib/google-ced/util/languages/languages.pb.h
new file mode 100644
index 0000000..84f1d6a
--- /dev/null
+++ b/contrib/google-ced/util/languages/languages.pb.h
@@ -0,0 +1,191 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_LANGUAGES_LANGUAGES_PB_H_
+#define UTIL_LANGUAGES_LANGUAGES_PB_H_
+
+enum Language {
+ ENGLISH = 0,
+ DANISH = 1,
+ DUTCH = 2,
+ FINNISH = 3,
+ FRENCH = 4,
+ GERMAN = 5,
+ HEBREW = 6,
+ ITALIAN = 7,
+ JAPANESE = 8,
+ KOREAN = 9,
+ NORWEGIAN = 10,
+ POLISH = 11,
+ PORTUGUESE = 12,
+ RUSSIAN = 13,
+ SPANISH = 14,
+ SWEDISH = 15,
+ CHINESE = 16,
+ CZECH = 17,
+ GREEK = 18,
+ ICELANDIC = 19,
+ LATVIAN = 20,
+ LITHUANIAN = 21,
+ ROMANIAN = 22,
+ HUNGARIAN = 23,
+ ESTONIAN = 24,
+ TG_UNKNOWN_LANGUAGE = 25,
+ UNKNOWN_LANGUAGE = 26,
+ BULGARIAN = 27,
+ CROATIAN = 28,
+ SERBIAN = 29,
+ IRISH = 30, // UI only.
+ GALICIAN = 31,
+ TAGALOG = 32, // Tagalog (tl) + Filipino (fil),
+ TURKISH = 33,
+ UKRAINIAN = 34,
+ HINDI = 35,
+ MACEDONIAN = 36,
+ BENGALI = 37,
+ INDONESIAN = 38,
+ LATIN = 39, // UI only.
+ MALAY = 40,
+ MALAYALAM = 41,
+ WELSH = 42, // UI only.
+ NEPALI = 43,
+ TELUGU = 44,
+ ALBANIAN = 45,
+ TAMIL = 46,
+ BELARUSIAN = 47,
+ JAVANESE = 48, // UI only.
+ OCCITAN = 49, // UI only.
+ URDU = 50,
+ BIHARI = 51,
+ GUJARATI = 52,
+ THAI = 53,
+ ARABIC = 54,
+ CATALAN = 55,
+ ESPERANTO = 56,
+ BASQUE = 57,
+ INTERLINGUA = 58, // UI only.
+ KANNADA = 59,
+ PUNJABI = 60,
+ SCOTS_GAELIC = 61, // UI only.
+ SWAHILI = 62,
+ SLOVENIAN = 63,
+ MARATHI = 64,
+ MALTESE = 65,
+ VIETNAMESE = 66,
+ FRISIAN = 67, // UI only.
+ SLOVAK = 68,
+ CHINESE_T = 69, // This is added to solve the problem of
+ // distinguishing Traditional and Simplified
+ // Chinese when the encoding is UTF8.
+ FAROESE = 70, // UI only.
+ SUNDANESE = 71, // UI only.
+ UZBEK = 72,
+ AMHARIC = 73,
+ AZERBAIJANI = 74,
+ GEORGIAN = 75,
+ TIGRINYA = 76, // UI only.
+ PERSIAN = 77,
+ BOSNIAN = 78, // UI only. LangId language: CROATIAN (28)
+ SINHALESE = 79,
+ NORWEGIAN_N = 80, // UI only. LangId language: NORWEGIAN (10)
+ PORTUGUESE_P = 81, // UI only. LangId language: PORTUGUESE (12)
+ PORTUGUESE_B = 82, // UI only. LangId language: PORTUGUESE (12)
+ XHOSA = 83, // UI only.
+ ZULU = 84, // UI only.
+ GUARANI = 85,
+ SESOTHO = 86, // UI only.
+ TURKMEN = 87, // UI only.
+ KYRGYZ = 88,
+ BRETON = 89, // UI only.
+ TWI = 90, // UI only.
+ YIDDISH = 91, // UI only.
+ SERBO_CROATIAN= 92, // UI only. LangId language: SERBIAN (29)
+ SOMALI = 93, // UI only.
+ UIGHUR = 94,
+ KURDISH = 95,
+ MONGOLIAN = 96,
+ ARMENIAN = 97,
+ LAOTHIAN = 98,
+ SINDHI = 99,
+ RHAETO_ROMANCE= 100, // UI only.
+ AFRIKAANS = 101,
+ LUXEMBOURGISH = 102, // UI only.
+ BURMESE = 103,
+ KHMER = 104,
+ TIBETAN = 105,
+ DHIVEHI = 106, // sometimes spelled Divehi, lang of Maldives
+ CHEROKEE = 107,
+ SYRIAC = 108, // UI only.
+ LIMBU = 109, // UI only.
+ ORIYA = 110,
+ ASSAMESE = 111, // UI only.
+ CORSICAN = 112, // UI only.
+ INTERLINGUE = 113, // UI only.
+ KAZAKH = 114,
+ LINGALA = 115, // UI only.
+ MOLDAVIAN = 116, // UI only. LangId language: ROMANIAN (22)
+ PASHTO = 117,
+ QUECHUA = 118, // UI only.
+ SHONA = 119, // UI only.
+ TAJIK = 120,
+ TATAR = 121, // UI only.
+ TONGA = 122, // UI only.
+ YORUBA = 123, // UI only.
+ CREOLES_AND_PIDGINS_ENGLISH_BASED = 124, // UI only.
+ CREOLES_AND_PIDGINS_FRENCH_BASED = 125, // UI only.
+ CREOLES_AND_PIDGINS_PORTUGUESE_BASED = 126, // UI only.
+ CREOLES_AND_PIDGINS_OTHER = 127, // UI only.
+ MAORI = 128, // UI only.
+ WOLOF = 129, // UI only.
+ ABKHAZIAN = 130, // UI only.
+ AFAR = 131, // UI only.
+ AYMARA = 132, // UI only.
+ BASHKIR = 133, // UI only.
+ BISLAMA = 134, // UI only.
+ DZONGKHA = 135, // UI only.
+ FIJIAN = 136, // UI only.
+ GREENLANDIC = 137, // UI only.
+ HAUSA = 138, // UI only.
+ HAITIAN_CREOLE= 139, // UI only.
+ INUPIAK = 140, // UI only.
+ INUKTITUT = 141,
+ KASHMIRI = 142, // UI only.
+ KINYARWANDA = 143, // UI only.
+ MALAGASY = 144, // UI only.
+ NAURU = 145, // UI only.
+ OROMO = 146, // UI only.
+ RUNDI = 147, // UI only.
+ SAMOAN = 148, // UI only.
+ SANGO = 149, // UI only.
+ SANSKRIT = 150,
+ SISWANT = 151, // UI only.
+ TSONGA = 152, // UI only.
+ TSWANA = 153, // UI only.
+ VOLAPUK = 154, // UI only.
+ ZHUANG = 155, // UI only.
+ KHASI = 156, // UI only.
+ SCOTS = 157, // UI only.
+ GANDA = 158, // UI only.
+ MANX = 159, // UI only.
+ MONTENEGRIN = 160, // UI only. LangId language: SERBIAN (29)
+ NUM_LANGUAGES = 161, // Always keep this at the end. It is not a
+ // valid Language enum. It is only used to
+ // indicate the total number of Languages.
+ // NOTE: If you add a language, you will break a unittest. See the note
+ // at the top of this enum.
+};
+
+#endif // UTIL_LANGUAGES_LANGUAGES_PB_H_
diff --git a/contrib/google-ced/util/logging.h b/contrib/google-ced/util/logging.h
new file mode 100644
index 0000000..16e50f2
--- /dev/null
+++ b/contrib/google-ced/util/logging.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_LOGGING_H_
+#define UTIL_LOGGING_H_
+
+#undef CHECK
+#define CHECK(expr)
+#undef DCHECK
+#define DCHECK(expr)
+
+#endif // UTIL_LOGGING_H_
diff --git a/contrib/google-ced/util/port.h b/contrib/google-ced/util/port.h
new file mode 100644
index 0000000..3799b16
--- /dev/null
+++ b/contrib/google-ced/util/port.h
@@ -0,0 +1,53 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_PORT_H_
+#define UTIL_PORT_H_
+
+#include <stdarg.h>
+
+#if defined(_MSC_VER)
+#define GG_LONGLONG(x) x##I64
+#define GG_ULONGLONG(x) x##UI64
+#else
+#define GG_LONGLONG(x) x##LL
+#define GG_ULONGLONG(x) x##ULL
+#endif
+
+// Per C99 7.8.14, define __STDC_CONSTANT_MACROS before including <stdint.h>
+// to get the INTn_C and UINTn_C macros for integer constants. It's difficult
+// to guarantee any specific ordering of header includes, so it's difficult to
+// guarantee that the INTn_C macros can be defined by including <stdint.h> at
+// any specific point. Provide GG_INTn_C macros instead.
+
+#define GG_INT8_C(x) (x)
+#define GG_INT16_C(x) (x)
+#define GG_INT32_C(x) (x)
+#define GG_INT64_C(x) GG_LONGLONG(x)
+
+#define GG_UINT8_C(x) (x ## U)
+#define GG_UINT16_C(x) (x ## U)
+#define GG_UINT32_C(x) (x ## U)
+#define GG_UINT64_C(x) GG_ULONGLONG(x)
+
+// Define an OS-neutral wrapper for shared library entry points
+#if defined(_WIN32)
+#define API_CALL __stdcall
+#else
+#define API_CALL
+#endif
+
+#endif // UTIL_PORT_H_
diff --git a/contrib/google-ced/util/string_util.h b/contrib/google-ced/util/string_util.h
new file mode 100644
index 0000000..5977f4f
--- /dev/null
+++ b/contrib/google-ced/util/string_util.h
@@ -0,0 +1,61 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_STRING_UTIL_H_
+#define UTIL_STRING_UTIL_H_
+
+#include <string.h>
+
+namespace base {
+
+#if defined(_WIN32)
+// Compare the two strings s1 and s2 without regard to case using
+// the current locale; returns 0 if they are equal, 1 if s1 > s2, and -1 if
+// s2 > s1 according to a lexicographic comparison.
+inline int strcasecmp(const char* s1, const char* s2) {
+ return _stricmp(s1, s2);
+}
+inline int strncasecmp(const char* s1, const char* s2, size_t n) {
+ return _strnicmp(s1, s2, n);
+}
+#else
+inline int strcasecmp(const char* s1, const char* s2) {
+ return ::strcasecmp(s1, s2);
+}
+inline int strncasecmp(const char* s1, const char* s2, size_t n) {
+ return ::strncasecmp(s1, s2, n);
+}
+#endif
+}
+
+#ifndef HAVE_MEMRCHR
+#if defined(__GLIBC__) && ((__GLIBC__ > 2) || ((__GLIBC__ == 2) && (__GLIBC_MINOR__ >= 2)))
+#define HAVE_MEMRCHR
+#endif
+#endif
+
+#ifndef HAVE_MEMRCHR
+inline void* memrchr(const void* s, int c, size_t n) {
+ const unsigned char* p = (const unsigned char*) s;
+ for (p += n; n > 0; n--) {
+ if (*--p == c)
+ return (void*) p;
+ }
+ return NULL;
+}
+#endif
+
+#endif // UTIL_STRING_UTIL_H_
diff --git a/contrib/google-ced/util/varsetter.h b/contrib/google-ced/util/varsetter.h
new file mode 100644
index 0000000..8e8cbf2
--- /dev/null
+++ b/contrib/google-ced/util/varsetter.h
@@ -0,0 +1,66 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_VARSETTER_H_
+#define UTIL_VARSETTER_H_
+
+//
+// Use a VarSetter object to temporarily set an object of some sort to
+// a particular value. When the VarSetter object is destructed, the
+// underlying object will revert to its former value.
+//
+// Sample code:
+//
+#if 0
+{
+ bool b = true;
+ {
+ VarSetter<bool> bool_setter(&b, false);
+ // Now b == false.
+ }
+ // Now b == true again.
+}
+#endif
+
+template <class C>
+class VarSetter {
+public:
+
+ // Constructor that just sets the object to a fixed value
+ VarSetter(C* object, const C& value) : object_(object), old_value_(*object) {
+ *object = value;
+ }
+
+ ~VarSetter() { *object_ = old_value_; }
+
+private:
+
+ C*const object_;
+ C old_value_;
+
+ // Disallow
+ VarSetter(const VarSetter&);
+ VarSetter& operator=(const VarSetter&);
+
+ // VarSetters always live on the stack
+ static void* operator new (size_t);
+ static void* operator new[](size_t); // Redundant, no default ctor
+
+ static void operator delete (void*);
+ static void operator delete[](void*);
+};
+
+#endif // UTIL_VARSETTER_H_
diff --git a/contrib/hiredis/CMakeLists.txt b/contrib/hiredis/CMakeLists.txt
new file mode 100644
index 0000000..1e05631
--- /dev/null
+++ b/contrib/hiredis/CMakeLists.txt
@@ -0,0 +1,11 @@
+SET(HIREDISSRC async.c
+ dict.c
+ hiredis.c
+ net.c
+ read.c
+ sds.c)
+
+SET(HIREDIS_CFLAGS "")
+ADD_LIBRARY(rspamd-hiredis STATIC ${HIREDISSRC})
+
+SET_TARGET_PROPERTIES(rspamd-hiredis PROPERTIES COMPILE_FLAGS "${HIREDIS_CFLAGS}") \ No newline at end of file
diff --git a/contrib/hiredis/COPYING b/contrib/hiredis/COPYING
new file mode 100644
index 0000000..a5fc973
--- /dev/null
+++ b/contrib/hiredis/COPYING
@@ -0,0 +1,29 @@
+Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+* Neither the name of Redis nor the names of its contributors may be used
+ to endorse or promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/contrib/hiredis/README.md b/contrib/hiredis/README.md
new file mode 100644
index 0000000..4f1a58d
--- /dev/null
+++ b/contrib/hiredis/README.md
@@ -0,0 +1,392 @@
+[![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis)
+
+# HIREDIS
+
+Hiredis is a minimalistic C client library for the [Redis](http://redis.io/) database.
+
+It is minimalistic because it just adds minimal support for the protocol, but
+at the same time it uses a high level printf-alike API in order to make it
+much higher level than otherwise suggested by its minimal code base and the
+lack of explicit bindings for every Redis command.
+
+Apart from supporting sending commands and receiving replies, it comes with
+a reply parser that is decoupled from the I/O layer. It
+is a stream parser designed for easy reusability, which can for instance be used
+in higher level language bindings for efficient reply parsing.
+
+Hiredis only supports the binary-safe Redis protocol, so you can use it with any
+Redis version >= 1.2.0.
+
+The library comes with multiple APIs. There is the
+*synchronous API*, the *asynchronous API* and the *reply parsing API*.
+
+## UPGRADING
+
+Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing
+code using hiredis should not be a big pain. The key thing to keep in mind when
+upgrading is that hiredis >= 0.9.0 uses a `redisContext*` to keep state, in contrast to
+the stateless 0.0.1 that only has a file descriptor to work with.
+
+## Synchronous API
+
+To consume the synchronous API, there are only a few function calls that need to be introduced:
+
+```c
+redisContext *redisConnect(const char *ip, int port);
+void *redisCommand(redisContext *c, const char *format, ...);
+void freeReplyObject(void *reply);
+```
+
+### Connecting
+
+The function `redisConnect` is used to create a so-called `redisContext`. The
+context is where Hiredis holds state for a connection. The `redisContext`
+struct has an integer `err` field that is non-zero when the connection is in
+an error state. The field `errstr` will contain a string with a description of
+the error. More information on errors can be found in the **Errors** section.
+After trying to connect to Redis using `redisConnect` you should
+check the `err` field to see if establishing the connection was successful:
+```c
+redisContext *c = redisConnect("127.0.0.1", 6379);
+if (c != NULL && c->err) {
+ printf("Error: %s\n", c->errstr);
+ // handle error
+}
+```
+
+### Sending commands
+
+There are several ways to issue commands to Redis. The first that will be introduced is
+`redisCommand`. This function takes a format similar to printf. In the simplest form,
+it is used like this:
+```c
+reply = redisCommand(context, "SET foo bar");
+```
+
+The specifier `%s` interpolates a string in the command, and uses `strlen` to
+determine the length of the string:
+```c
+reply = redisCommand(context, "SET foo %s", value);
+```
+When you need to pass binary safe strings in a command, the `%b` specifier can be
+used. Together with a pointer to the string, it requires a `size_t` length argument
+of the string:
+```c
+reply = redisCommand(context, "SET foo %b", value, (size_t) valuelen);
+```
+Internally, Hiredis splits the command in different arguments and will
+convert it to the protocol used to communicate with Redis.
+One or more spaces separates arguments, so you can use the specifiers
+anywhere in an argument:
+```c
+reply = redisCommand(context, "SET key:%s %s", myid, value);
+```
+
+### Using replies
+
+The return value of `redisCommand` holds a reply when the command was
+successfully executed. When an error occurs, the return value is `NULL` and
+the `err` field in the context will be set (see section on **Errors**).
+Once an error is returned the context cannot be reused and you should set up
+a new connection.
+
+The standard replies that `redisCommand` are of the type `redisReply`. The
+`type` field in the `redisReply` should be used to test what kind of reply
+was received:
+
+* **`REDIS_REPLY_STATUS`**:
+ * The command replied with a status reply. The status string can be accessed using `reply->str`.
+ The length of this string can be accessed using `reply->len`.
+
+* **`REDIS_REPLY_ERROR`**:
+ * The command replied with an error. The error string can be accessed identical to `REDIS_REPLY_STATUS`.
+
+* **`REDIS_REPLY_INTEGER`**:
+ * The command replied with an integer. The integer value can be accessed using the
+ `reply->integer` field of type `long long`.
+
+* **`REDIS_REPLY_NIL`**:
+ * The command replied with a **nil** object. There is no data to access.
+
+* **`REDIS_REPLY_STRING`**:
+ * A bulk (string) reply. The value of the reply can be accessed using `reply->str`.
+ The length of this string can be accessed using `reply->len`.
+
+* **`REDIS_REPLY_ARRAY`**:
+ * A multi bulk reply. The number of elements in the multi bulk reply is stored in
+ `reply->elements`. Every element in the multi bulk reply is a `redisReply` object as well
+ and can be accessed via `reply->element[..index..]`.
+ Redis may reply with nested arrays but this is fully supported.
+
+Replies should be freed using the `freeReplyObject()` function.
+Note that this function will take care of freeing sub-reply objects
+contained in arrays and nested arrays, so there is no need for the user to
+free the sub replies (it is actually harmful and will corrupt the memory).
+
+**Important:** the current version of hiredis (0.10.0) frees replies when the
+asynchronous API is used. This means you should not call `freeReplyObject` when
+you use this API. The reply is cleaned up by hiredis _after_ the callback
+returns. This behavior will probably change in future releases, so make sure to
+keep an eye on the changelog when upgrading (see issue #39).
+
+### Cleaning up
+
+To disconnect and free the context the following function can be used:
+```c
+void redisFree(redisContext *c);
+```
+This function immediately closes the socket and then frees the allocations done in
+creating the context.
+
+### Sending commands (cont'd)
+
+Together with `redisCommand`, the function `redisCommandArgv` can be used to issue commands.
+It has the following prototype:
+```c
+void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+```
+It takes the number of arguments `argc`, an array of strings `argv` and the lengths of the
+arguments `argvlen`. For convenience, `argvlen` may be set to `NULL` and the function will
+use `strlen(3)` on every argument to determine its length. Obviously, when any of the arguments
+need to be binary safe, the entire array of lengths `argvlen` should be provided.
+
+The return value has the same semantic as `redisCommand`.
+
+### Pipelining
+
+To explain how Hiredis supports pipelining in a blocking connection, there needs to be
+understanding of the internal execution flow.
+
+When any of the functions in the `redisCommand` family is called, Hiredis first formats the
+command according to the Redis protocol. The formatted command is then put in the output buffer
+of the context. This output buffer is dynamic, so it can hold any number of commands.
+After the command is put in the output buffer, `redisGetReply` is called. This function has the
+following two execution paths:
+
+1. The input buffer is non-empty:
+ * Try to parse a single reply from the input buffer and return it
+ * If no reply could be parsed, continue at *2*
+2. The input buffer is empty:
+ * Write the **entire** output buffer to the socket
+ * Read from the socket until a single reply could be parsed
+
+The function `redisGetReply` is exported as part of the Hiredis API and can be used when a reply
+is expected on the socket. To pipeline commands, the only things that needs to be done is
+filling up the output buffer. For this cause, two commands can be used that are identical
+to the `redisCommand` family, apart from not returning a reply:
+```c
+void redisAppendCommand(redisContext *c, const char *format, ...);
+void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+```
+After calling either function one or more times, `redisGetReply` can be used to receive the
+subsequent replies. The return value for this function is either `REDIS_OK` or `REDIS_ERR`, where
+the latter means an error occurred while reading a reply. Just as with the other commands,
+the `err` field in the context can be used to find out what the cause of this error is.
+
+The following examples shows a simple pipeline (resulting in only a single call to `write(2)` and
+a single call to `read(2)`):
+```c
+redisReply *reply;
+redisAppendCommand(context,"SET foo bar");
+redisAppendCommand(context,"GET foo");
+redisGetReply(context,&reply); // reply for SET
+freeReplyObject(reply);
+redisGetReply(context,&reply); // reply for GET
+freeReplyObject(reply);
+```
+This API can also be used to implement a blocking subscriber:
+```c
+reply = redisCommand(context,"SUBSCRIBE foo");
+freeReplyObject(reply);
+while(redisGetReply(context,&reply) == REDIS_OK) {
+ // consume message
+ freeReplyObject(reply);
+}
+```
+### Errors
+
+When a function call is not successful, depending on the function either `NULL` or `REDIS_ERR` is
+returned. The `err` field inside the context will be non-zero and set to one of the
+following constants:
+
+* **`REDIS_ERR_IO`**:
+ There was an I/O error while creating the connection, trying to write
+ to the socket or read from the socket. If you included `errno.h` in your
+ application, you can use the global `errno` variable to find out what is
+ wrong.
+
+* **`REDIS_ERR_EOF`**:
+ The server closed the connection which resulted in an empty read.
+
+* **`REDIS_ERR_PROTOCOL`**:
+ There was an error while parsing the protocol.
+
+* **`REDIS_ERR_OTHER`**:
+ Any other error. Currently, it is only used when a specified hostname to connect
+ to cannot be resolved.
+
+In every case, the `errstr` field in the context will be set to hold a string representation
+of the error.
+
+## Asynchronous API
+
+Hiredis comes with an asynchronous API that works easily with any event library.
+Examples are bundled that show using Hiredis with [libev](http://software.schmorp.de/pkg/libev.html)
+and [libevent](http://monkey.org/~provos/libevent/).
+
+### Connecting
+
+The function `redisAsyncConnect` can be used to establish a non-blocking connection to
+Redis. It returns a pointer to the newly created `redisAsyncContext` struct. The `err` field
+should be checked after creation to see if there were errors creating the connection.
+Because the connection that will be created is non-blocking, the kernel is not able to
+instantly return if the specified host and port is able to accept a connection.
+```c
+redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
+if (c->err) {
+ printf("Error: %s\n", c->errstr);
+ // handle error
+}
+```
+
+The asynchronous context can hold a disconnect callback function that is called when the
+connection is disconnected (either because of an error or per user request). This function should
+have the following prototype:
+```c
+void(const redisAsyncContext *c, int status);
+```
+On a disconnect, the `status` argument is set to `REDIS_OK` when disconnection was initiated by the
+user, or `REDIS_ERR` when the disconnection was caused by an error. When it is `REDIS_ERR`, the `err`
+field in the context can be accessed to find out the cause of the error.
+
+The context object is always freed after the disconnect callback fired. When a reconnect is needed,
+the disconnect callback is a good point to do so.
+
+Setting the disconnect callback can only be done once per context. For subsequent calls it will
+return `REDIS_ERR`. The function to set the disconnect callback has the following prototype:
+```c
+int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
+```
+### Sending commands and their callbacks
+
+In an asynchronous context, commands are automatically pipelined due to the nature of an event loop.
+Therefore, unlike the synchronous API, there is only a single way to send commands.
+Because commands are sent to Redis asynchronously, issuing a command requires a callback function
+that is called when the reply is received. Reply callbacks should have the following prototype:
+```c
+void(redisAsyncContext *c, void *reply, void *privdata);
+```
+The `privdata` argument can be used to curry arbitrary data to the callback from the point where
+the command is initially queued for execution.
+
+The functions that can be used to issue commands in an asynchronous context are:
+```c
+int redisAsyncCommand(
+ redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
+ const char *format, ...);
+int redisAsyncCommandArgv(
+ redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
+ int argc, const char **argv, const size_t *argvlen);
+```
+Both functions work like their blocking counterparts. The return value is `REDIS_OK` when the command
+was successfully added to the output buffer and `REDIS_ERR` otherwise. Example: when the connection
+is being disconnected per user-request, no new commands may be added to the output buffer and `REDIS_ERR` is
+returned on calls to the `redisAsyncCommand` family.
+
+If the reply for a command with a `NULL` callback is read, it is immediately freed. When the callback
+for a command is non-`NULL`, the memory is freed immediately following the callback: the reply is only
+valid for the duration of the callback.
+
+All pending callbacks are called with a `NULL` reply when the context encountered an error.
+
+### Disconnecting
+
+An asynchronous connection can be terminated using:
+```c
+void redisAsyncDisconnect(redisAsyncContext *ac);
+```
+When this function is called, the connection is **not** immediately terminated. Instead, new
+commands are no longer accepted and the connection is only terminated when all pending commands
+have been written to the socket, their respective replies have been read and their respective
+callbacks have been executed. After this, the disconnection callback is executed with the
+`REDIS_OK` status and the context object is freed.
+
+### Hooking it up to event library *X*
+
+There are a few hooks that need to be set on the context object after it is created.
+See the `adapters/` directory for bindings to *libev* and *libevent*.
+
+## Reply parsing API
+
+Hiredis comes with a reply parsing API that makes it easy for writing higher
+level language bindings.
+
+The reply parsing API consists of the following functions:
+```c
+redisReader *redisReaderCreate(void);
+void redisReaderFree(redisReader *reader);
+int redisReaderFeed(redisReader *reader, const char *buf, size_t len);
+int redisReaderGetReply(redisReader *reader, void **reply);
+```
+The same set of functions are used internally by hiredis when creating a
+normal Redis context, the above API just exposes it to the user for a direct
+usage.
+
+### Usage
+
+The function `redisReaderCreate` creates a `redisReader` structure that holds a
+buffer with unparsed data and state for the protocol parser.
+
+Incoming data -- most likely from a socket -- can be placed in the internal
+buffer of the `redisReader` using `redisReaderFeed`. This function will make a
+copy of the buffer pointed to by `buf` for `len` bytes. This data is parsed
+when `redisReaderGetReply` is called. This function returns an integer status
+and a reply object (as described above) via `void **reply`. The returned status
+can be either `REDIS_OK` or `REDIS_ERR`, where the latter means something went
+wrong (either a protocol error, or an out of memory error).
+
+The parser limits the level of nesting for multi bulk payloads to 7. If the
+multi bulk nesting level is higher than this, the parser returns an error.
+
+### Customizing replies
+
+The function `redisReaderGetReply` creates `redisReply` and makes the function
+argument `reply` point to the created `redisReply` variable. For instance, if
+the response of type `REDIS_REPLY_STATUS` then the `str` field of `redisReply`
+will hold the status as a vanilla C string. However, the functions that are
+responsible for creating instances of the `redisReply` can be customized by
+setting the `fn` field on the `redisReader` struct. This should be done
+immediately after creating the `redisReader`.
+
+For example, [hiredis-rb](https://github.com/pietern/hiredis-rb/blob/master/ext/hiredis_ext/reader.c)
+uses customized reply object functions to create Ruby objects.
+
+### Reader max buffer
+
+Both when using the Reader API directly or when using it indirectly via a
+normal Redis context, the redisReader structure uses a buffer in order to
+accumulate data from the server.
+Usually this buffer is destroyed when it is empty and is larger than 16
+KiB in order to avoid wasting memory in unused buffers
+
+However when working with very big payloads destroying the buffer may slow
+down performances considerably, so it is possible to modify the max size of
+an idle buffer changing the value of the `maxbuf` field of the reader structure
+to the desired value. The special value of 0 means that there is no maximum
+value for an idle buffer, so the buffer will never get freed.
+
+For instance if you have a normal Redis context you can set the maximum idle
+buffer to zero (unlimited) just with:
+```c
+context->reader->maxbuf = 0;
+```
+This should be done only in order to maximize performances when working with
+large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again
+as soon as possible in order to prevent allocation of useless memory.
+
+## AUTHORS
+
+Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and
+Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license.
+Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and
+Jan-Erik Rediger (janerik at fnordig dot com)
diff --git a/contrib/hiredis/adapters/libev.h b/contrib/hiredis/adapters/libev.h
new file mode 100644
index 0000000..9cf00df
--- /dev/null
+++ b/contrib/hiredis/adapters/libev.h
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_LIBEV_H__
+#define __HIREDIS_LIBEV_H__
+#include <stdlib.h>
+#include <sys/types.h>
+#include "contrib/libev/ev.h"
+#include "../hiredis.h"
+#include "../async.h"
+
+typedef struct redisLibevEvents {
+ redisAsyncContext *context;
+ struct ev_loop *loop;
+ int reading, writing;
+ ev_io rev, wev;
+} redisLibevEvents;
+
+static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) {
+#if EV_MULTIPLICITY
+ ((void)loop);
+#endif
+ ((void)revents);
+
+ redisLibevEvents *e = (redisLibevEvents*)watcher->data;
+ redisAsyncHandleRead(e->context);
+}
+
+static void redisLibevWriteEvent(EV_P_ ev_io *watcher, int revents) {
+#if EV_MULTIPLICITY
+ ((void)loop);
+#endif
+ ((void)revents);
+
+ redisLibevEvents *e = (redisLibevEvents*)watcher->data;
+ redisAsyncHandleWrite(e->context);
+}
+
+static void redisLibevAddRead(void *privdata) {
+ redisLibevEvents *e = (redisLibevEvents*)privdata;
+ struct ev_loop *loop = e->loop;
+ ((void)loop);
+ if (!e->reading) {
+ e->reading = 1;
+ ev_io_start(EV_A_ &e->rev);
+ }
+}
+
+static void redisLibevDelRead(void *privdata) {
+ redisLibevEvents *e = (redisLibevEvents*)privdata;
+ struct ev_loop *loop = e->loop;
+ ((void)loop);
+ if (e->reading) {
+ e->reading = 0;
+ ev_io_stop(EV_A_ &e->rev);
+ }
+}
+
+static void redisLibevAddWrite(void *privdata) {
+ redisLibevEvents *e = (redisLibevEvents*)privdata;
+ struct ev_loop *loop = e->loop;
+ ((void)loop);
+ if (!e->writing) {
+ e->writing = 1;
+ ev_io_start(EV_A_ &e->wev);
+ }
+}
+
+static void redisLibevDelWrite(void *privdata) {
+ redisLibevEvents *e = (redisLibevEvents*)privdata;
+ struct ev_loop *loop = e->loop;
+ ((void)loop);
+ if (e->writing) {
+ e->writing = 0;
+ ev_io_stop(EV_A_ &e->wev);
+ }
+}
+
+static void redisLibevCleanup(void *privdata) {
+ redisLibevEvents *e = (redisLibevEvents*)privdata;
+ redisLibevDelRead(privdata);
+ redisLibevDelWrite(privdata);
+ free(e);
+}
+
+static int redisLibevAttach(EV_P_ redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ redisLibevEvents *e;
+
+ /* Nothing should be attached when something is already attached */
+ if (ac->ev.data != NULL)
+ return REDIS_ERR;
+
+ /* Create container for context and r/w events */
+ e = (redisLibevEvents*)malloc(sizeof(*e));
+ e->context = ac;
+#if EV_MULTIPLICITY
+ e->loop = loop;
+#else
+ e->loop = NULL;
+#endif
+ e->reading = e->writing = 0;
+ e->rev.data = e;
+ e->wev.data = e;
+
+ /* Register functions to start/stop listening for events */
+ ac->ev.addRead = redisLibevAddRead;
+ ac->ev.delRead = redisLibevDelRead;
+ ac->ev.addWrite = redisLibevAddWrite;
+ ac->ev.delWrite = redisLibevDelWrite;
+ ac->ev.cleanup = redisLibevCleanup;
+ ac->ev.data = e;
+
+ /* Initialize read/write events */
+ ev_io_init(&e->rev,redisLibevReadEvent,c->fd,EV_READ);
+ ev_io_init(&e->wev,redisLibevWriteEvent,c->fd,EV_WRITE);
+ return REDIS_OK;
+}
+
+#endif
diff --git a/contrib/hiredis/adapters/libevent.h b/contrib/hiredis/adapters/libevent.h
new file mode 100644
index 0000000..1c2b271
--- /dev/null
+++ b/contrib/hiredis/adapters/libevent.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_LIBEVENT_H__
+#define __HIREDIS_LIBEVENT_H__
+#include <event.h>
+#include "../hiredis.h"
+#include "../async.h"
+
+typedef struct redisLibeventEvents {
+ redisAsyncContext *context;
+ struct event rev, wev;
+} redisLibeventEvents;
+
+static void redisLibeventReadEvent(int fd, short event, void *arg) {
+ ((void)fd); ((void)event);
+ redisLibeventEvents *e = (redisLibeventEvents*)arg;
+ redisAsyncHandleRead(e->context);
+}
+
+static void redisLibeventWriteEvent(int fd, short event, void *arg) {
+ ((void)fd); ((void)event);
+ redisLibeventEvents *e = (redisLibeventEvents*)arg;
+ redisAsyncHandleWrite(e->context);
+}
+
+static void redisLibeventAddRead(void *privdata) {
+ redisLibeventEvents *e = (redisLibeventEvents*)privdata;
+ event_add(&e->rev,NULL);
+}
+
+static void redisLibeventDelRead(void *privdata) {
+ redisLibeventEvents *e = (redisLibeventEvents*)privdata;
+ event_del(&e->rev);
+}
+
+static void redisLibeventAddWrite(void *privdata) {
+ redisLibeventEvents *e = (redisLibeventEvents*)privdata;
+ event_add(&e->wev,NULL);
+}
+
+static void redisLibeventDelWrite(void *privdata) {
+ redisLibeventEvents *e = (redisLibeventEvents*)privdata;
+ event_del(&e->wev);
+}
+
+static void redisLibeventCleanup(void *privdata) {
+ redisLibeventEvents *e = (redisLibeventEvents*)privdata;
+ event_del(&e->rev);
+ event_del(&e->wev);
+ free(e);
+}
+
+static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) {
+ redisContext *c = &(ac->c);
+ redisLibeventEvents *e;
+
+ /* Nothing should be attached when something is already attached */
+ if (ac->ev.data != NULL)
+ return REDIS_ERR;
+
+ /* Create container for context and r/w events */
+ e = (redisLibeventEvents*)malloc(sizeof(*e));
+ e->context = ac;
+
+ /* Register functions to start/stop listening for events */
+ ac->ev.addRead = redisLibeventAddRead;
+ ac->ev.delRead = redisLibeventDelRead;
+ ac->ev.addWrite = redisLibeventAddWrite;
+ ac->ev.delWrite = redisLibeventDelWrite;
+ ac->ev.cleanup = redisLibeventCleanup;
+ ac->ev.data = e;
+
+ /* Initialize and install read/write events */
+ event_set(&e->rev,c->fd,EV_READ,redisLibeventReadEvent,e);
+ event_set(&e->wev,c->fd,EV_WRITE,redisLibeventWriteEvent,e);
+ event_base_set(base,&e->rev);
+ event_base_set(base,&e->wev);
+ return REDIS_OK;
+}
+#endif
diff --git a/contrib/hiredis/async.c b/contrib/hiredis/async.c
new file mode 100644
index 0000000..afa5413
--- /dev/null
+++ b/contrib/hiredis/async.c
@@ -0,0 +1,693 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <assert.h>
+#include <ctype.h>
+#include <errno.h>
+#include "async.h"
+#include "net.h"
+#include "dict.c"
+#include "sds.h"
+
+#define _EL_ADD_READ(ctx) do { \
+ if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \
+ } while(0)
+#define _EL_DEL_READ(ctx) do { \
+ if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \
+ } while(0)
+#define _EL_ADD_WRITE(ctx) do { \
+ if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \
+ } while(0)
+#define _EL_DEL_WRITE(ctx) do { \
+ if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \
+ } while(0)
+#define _EL_CLEANUP(ctx) do { \
+ if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \
+ } while(0);
+
+/* Forward declaration of function in hiredis.c */
+int __redisAppendCommand(redisContext *c, const char *cmd, size_t len);
+
+/* Functions managing dictionary of callbacks for pub/sub. */
+static unsigned int callbackHash(const void *key) {
+ return dictGenHashFunction((const unsigned char *)key,
+ sdslen((const sds)key));
+}
+
+static void *callbackValDup(void *privdata, const void *src) {
+ ((void) privdata);
+ redisCallback *dup = malloc(sizeof(*dup));
+ memcpy(dup,src,sizeof(*dup));
+ return dup;
+}
+
+static int callbackKeyCompare(void *privdata, const void *key1, const void *key2) {
+ int l1, l2;
+ ((void) privdata);
+
+ l1 = sdslen((const sds)key1);
+ l2 = sdslen((const sds)key2);
+ if (l1 != l2) return 0;
+ return memcmp(key1,key2,l1) == 0;
+}
+
+static void callbackKeyDestructor(void *privdata, void *key) {
+ ((void) privdata);
+ sdsfree((sds)key);
+}
+
+static void callbackValDestructor(void *privdata, void *val) {
+ ((void) privdata);
+ free(val);
+}
+
+static dictType callbackDict = {
+ callbackHash,
+ NULL,
+ callbackValDup,
+ callbackKeyCompare,
+ callbackKeyDestructor,
+ callbackValDestructor
+};
+
+static redisAsyncContext *redisAsyncInitialize(redisContext *c) {
+ redisAsyncContext *ac;
+
+ ac = realloc(c,sizeof(redisAsyncContext));
+ if (ac == NULL)
+ return NULL;
+
+ c = &(ac->c);
+
+ /* The regular connect functions will always set the flag REDIS_CONNECTED.
+ * For the async API, we want to wait until the first write event is
+ * received up before setting this flag, so reset it here. */
+ c->flags &= ~REDIS_CONNECTED;
+
+ ac->err = 0;
+ ac->errstr = NULL;
+ ac->data = NULL;
+
+ ac->ev.data = NULL;
+ ac->ev.addRead = NULL;
+ ac->ev.delRead = NULL;
+ ac->ev.addWrite = NULL;
+ ac->ev.delWrite = NULL;
+ ac->ev.cleanup = NULL;
+
+ ac->onConnect = NULL;
+ ac->onDisconnect = NULL;
+
+ ac->replies.head = NULL;
+ ac->replies.tail = NULL;
+ ac->sub.invalid.head = NULL;
+ ac->sub.invalid.tail = NULL;
+ ac->sub.channels = dictCreate(&callbackDict,NULL);
+ ac->sub.patterns = dictCreate(&callbackDict,NULL);
+ return ac;
+}
+
+/* We want the error field to be accessible directly instead of requiring
+ * an indirection to the redisContext struct. */
+static void __redisAsyncCopyError(redisAsyncContext *ac) {
+ if (!ac)
+ return;
+
+ redisContext *c = &(ac->c);
+ ac->err = c->err;
+ ac->errstr = c->errstr;
+}
+
+redisAsyncContext *redisAsyncConnect(const char *ip, int port) {
+ redisContext *c;
+ redisAsyncContext *ac;
+
+ c = redisConnectNonBlock(ip,port);
+ if (c == NULL)
+ return NULL;
+
+ ac = redisAsyncInitialize(c);
+ if (ac == NULL) {
+ redisFree(c);
+ return NULL;
+ }
+
+ __redisAsyncCopyError(ac);
+ return ac;
+}
+
+redisAsyncContext *redisAsyncConnectBind(const char *ip, int port,
+ const char *source_addr) {
+ redisContext *c = redisConnectBindNonBlock(ip,port,source_addr);
+ redisAsyncContext *ac = redisAsyncInitialize(c);
+ __redisAsyncCopyError(ac);
+ return ac;
+}
+
+redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
+ const char *source_addr) {
+ redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr);
+ redisAsyncContext *ac = redisAsyncInitialize(c);
+ __redisAsyncCopyError(ac);
+ return ac;
+}
+
+redisAsyncContext *redisAsyncConnectUnix(const char *path) {
+ redisContext *c;
+ redisAsyncContext *ac;
+
+ c = redisConnectUnixNonBlock(path);
+ if (c == NULL)
+ return NULL;
+
+ ac = redisAsyncInitialize(c);
+ if (ac == NULL) {
+ redisFree(c);
+ return NULL;
+ }
+
+ __redisAsyncCopyError(ac);
+ return ac;
+}
+
+int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) {
+ if (ac->onConnect == NULL) {
+ ac->onConnect = fn;
+
+ /* The common way to detect an established connection is to wait for
+ * the first write event to be fired. This assumes the related event
+ * library functions are already set. */
+ _EL_ADD_WRITE(ac);
+ return REDIS_OK;
+ }
+ return REDIS_ERR;
+}
+
+int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn) {
+ if (ac->onDisconnect == NULL) {
+ ac->onDisconnect = fn;
+ return REDIS_OK;
+ }
+ return REDIS_ERR;
+}
+
+/* Helper functions to push/shift callbacks */
+static int __redisPushCallback(redisCallbackList *list, redisCallback *source) {
+ redisCallback *cb;
+
+ /* Copy callback from stack to heap */
+ cb = malloc(sizeof(*cb));
+ if (cb == NULL)
+ return REDIS_ERR_OOM;
+
+ if (source != NULL) {
+ memcpy(cb,source,sizeof(*cb));
+ cb->next = NULL;
+ }
+
+ /* Store callback in list */
+ if (list->head == NULL)
+ list->head = cb;
+ if (list->tail != NULL)
+ list->tail->next = cb;
+ list->tail = cb;
+ return REDIS_OK;
+}
+
+static int __redisShiftCallback(redisCallbackList *list, redisCallback *target) {
+ redisCallback *cb = list->head;
+ if (cb != NULL) {
+ list->head = cb->next;
+ if (cb == list->tail)
+ list->tail = NULL;
+
+ /* Copy callback from heap to stack */
+ if (target != NULL)
+ memcpy(target,cb,sizeof(*cb));
+ free(cb);
+ return REDIS_OK;
+ }
+ return REDIS_ERR;
+}
+
+static void __redisRunCallback(redisAsyncContext *ac, redisCallback *cb, redisReply *reply) {
+ redisContext *c = &(ac->c);
+ if (cb->fn != NULL) {
+ c->flags |= REDIS_IN_CALLBACK;
+ cb->fn(ac,reply,cb->privdata);
+ c->flags &= ~REDIS_IN_CALLBACK;
+ }
+}
+
+/* Helper function to free the context. */
+static void __redisAsyncFree(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ redisCallback cb;
+ dictIterator *it;
+ dictEntry *de;
+
+ /* Execute pending callbacks with NULL reply. */
+ while (__redisShiftCallback(&ac->replies,&cb) == REDIS_OK)
+ __redisRunCallback(ac,&cb,NULL);
+
+ /* Execute callbacks for invalid commands */
+ while (__redisShiftCallback(&ac->sub.invalid,&cb) == REDIS_OK)
+ __redisRunCallback(ac,&cb,NULL);
+
+ /* Run subscription callbacks callbacks with NULL reply */
+ it = dictGetIterator(ac->sub.channels);
+ while ((de = dictNext(it)) != NULL)
+ __redisRunCallback(ac,dictGetEntryVal(de),NULL);
+ dictReleaseIterator(it);
+ dictRelease(ac->sub.channels);
+
+ it = dictGetIterator(ac->sub.patterns);
+ while ((de = dictNext(it)) != NULL)
+ __redisRunCallback(ac,dictGetEntryVal(de),NULL);
+ dictReleaseIterator(it);
+ dictRelease(ac->sub.patterns);
+
+ /* Signal event lib to clean up */
+ _EL_CLEANUP(ac);
+
+ /* Execute disconnect callback. When redisAsyncFree() initiated destroying
+ * this context, the status will always be REDIS_OK. */
+ if (ac->onDisconnect && (c->flags & REDIS_CONNECTED)) {
+ if (c->flags & REDIS_FREEING) {
+ ac->onDisconnect(ac,REDIS_OK);
+ } else {
+ c->flags |= REDIS_FREEING;
+ ac->onDisconnect(ac,(ac->err == 0) ? REDIS_OK : REDIS_ERR);
+ }
+ }
+
+ /* Cleanup self */
+ redisFree(c);
+}
+
+/* Free the async context. When this function is called from a callback,
+ * control needs to be returned to redisProcessCallbacks() before actual
+ * free'ing. To do so, a flag is set on the context which is picked up by
+ * redisProcessCallbacks(). Otherwise, the context is immediately free'd. */
+void redisAsyncFree(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ c->flags |= REDIS_FREEING;
+ if (!(c->flags & REDIS_IN_CALLBACK))
+ __redisAsyncFree(ac);
+}
+
+/* Helper function to make the disconnect happen and clean up. */
+static void __redisAsyncDisconnect(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+
+ /* Make sure error is accessible if there is any */
+ __redisAsyncCopyError(ac);
+
+ if (ac->err == 0) {
+ /* For clean disconnects, there should be no pending callbacks. */
+ assert(__redisShiftCallback(&ac->replies,NULL) == REDIS_ERR);
+ } else {
+ /* Disconnection is caused by an error, make sure that pending
+ * callbacks cannot call new commands. */
+ c->flags |= REDIS_DISCONNECTING;
+ }
+
+ /* For non-clean disconnects, __redisAsyncFree() will execute pending
+ * callbacks with a NULL-reply. */
+ __redisAsyncFree(ac);
+}
+
+/* Tries to do a clean disconnect from Redis, meaning it stops new commands
+ * from being issued, but tries to flush the output buffer and execute
+ * callbacks for all remaining replies. When this function is called from a
+ * callback, there might be more replies and we can safely defer disconnecting
+ * to redisProcessCallbacks(). Otherwise, we can only disconnect immediately
+ * when there are no pending callbacks. */
+void redisAsyncDisconnect(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ c->flags |= REDIS_DISCONNECTING;
+ if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL)
+ __redisAsyncDisconnect(ac);
+}
+
+static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, redisCallback *dstcb) {
+ redisContext *c = &(ac->c);
+ dict *callbacks;
+ dictEntry *de;
+ int pvariant;
+ char *stype;
+ sds sname;
+
+ /* Custom reply functions are not supported for pub/sub. This will fail
+ * very hard when they are used... */
+ if (reply->type == REDIS_REPLY_ARRAY) {
+ assert(reply->elements >= 2);
+ assert(reply->element[0]->type == REDIS_REPLY_STRING);
+ stype = reply->element[0]->str;
+ pvariant = (tolower(stype[0]) == 'p') ? 1 : 0;
+
+ if (pvariant)
+ callbacks = ac->sub.patterns;
+ else
+ callbacks = ac->sub.channels;
+
+ /* Locate the right callback */
+ assert(reply->element[1]->type == REDIS_REPLY_STRING);
+ sname = sdsnewlen(reply->element[1]->str,reply->element[1]->len);
+ de = dictFind(callbacks,sname);
+ if (de != NULL) {
+ memcpy(dstcb,dictGetEntryVal(de),sizeof(*dstcb));
+
+ /* If this is an unsubscribe message, remove it. */
+ if (strcasecmp(stype+pvariant,"unsubscribe") == 0) {
+ dictDelete(callbacks,sname);
+
+ /* If this was the last unsubscribe message, revert to
+ * non-subscribe mode. */
+ assert(reply->element[2]->type == REDIS_REPLY_INTEGER);
+ if (reply->element[2]->integer == 0)
+ c->flags &= ~REDIS_SUBSCRIBED;
+ }
+ }
+ sdsfree(sname);
+ } else {
+ /* Shift callback for invalid commands. */
+ __redisShiftCallback(&ac->sub.invalid,dstcb);
+ }
+ return REDIS_OK;
+}
+
+void redisProcessCallbacks(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ redisCallback cb = {NULL, NULL, NULL};
+ void *reply = NULL;
+ int status;
+
+ while((status = redisGetReply(c,&reply)) == REDIS_OK) {
+ if (reply == NULL) {
+ /* When the connection is being disconnected and there are
+ * no more replies, this is the cue to really disconnect. */
+ if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0
+ && ac->replies.head == NULL) {
+ __redisAsyncDisconnect(ac);
+ return;
+ }
+
+ /* If monitor mode, repush callback */
+ if(c->flags & REDIS_MONITORING) {
+ __redisPushCallback(&ac->replies,&cb);
+ }
+
+ /* When the connection is not being disconnected, simply stop
+ * trying to get replies and wait for the next loop tick. */
+ break;
+ }
+
+ /* Even if the context is subscribed, pending regular callbacks will
+ * get a reply before pub/sub messages arrive. */
+ if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) {
+ /*
+ * A spontaneous reply in a not-subscribed context can be the error
+ * reply that is sent when a new connection exceeds the maximum
+ * number of allowed connections on the server side.
+ *
+ * This is seen as an error instead of a regular reply because the
+ * server closes the connection after sending it.
+ *
+ * To prevent the error from being overwritten by an EOF error the
+ * connection is closed here. See issue #43.
+ *
+ * Another possibility is that the server is loading its dataset.
+ * In this case we also want to close the connection, and have the
+ * user wait until the server is ready to take our request.
+ */
+ if (((redisReply*)reply)->type == REDIS_REPLY_ERROR) {
+ c->err = REDIS_ERR_OTHER;
+ snprintf(c->errstr,sizeof(c->errstr),"%s",((redisReply*)reply)->str);
+ c->reader->fn->freeObject(reply);
+ __redisAsyncDisconnect(ac);
+ return;
+ }
+ /* No more regular callbacks and no errors, the context *must* be subscribed or monitoring. */
+ assert((c->flags & REDIS_SUBSCRIBED || c->flags & REDIS_MONITORING));
+ if(c->flags & REDIS_SUBSCRIBED)
+ __redisGetSubscribeCallback(ac,reply,&cb);
+ }
+
+ if (cb.fn != NULL) {
+ __redisRunCallback(ac,&cb,reply);
+ c->reader->fn->freeObject(reply);
+
+ /* Proceed with free'ing when redisAsyncFree() was called. */
+ if (c->flags & REDIS_FREEING) {
+ __redisAsyncFree(ac);
+ return;
+ }
+ } else {
+ /* No callback for this reply. This can either be a NULL callback,
+ * or there were no callbacks to begin with. Either way, don't
+ * abort with an error, but simply ignore it because the client
+ * doesn't know what the server will spit out over the wire. */
+ c->reader->fn->freeObject(reply);
+ /* Proceed with free'ing when redisAsyncFree() was called. */
+ if (c->flags & REDIS_FREEING) {
+ __redisAsyncFree(ac);
+ return;
+ }
+ }
+ }
+
+ /* Disconnect when there was an error reading the reply */
+ if (status != REDIS_OK)
+ __redisAsyncDisconnect(ac);
+}
+
+/* Internal helper function to detect socket status the first time a read or
+ * write event fires. When connecting was not successful, the connect callback
+ * is called with a REDIS_ERR status and the context is free'd. */
+static int __redisAsyncHandleConnect(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+
+ if (redisCheckSocketError(c) == REDIS_ERR) {
+ /* Try again later when connect(2) is still in progress. */
+ if (errno == EINPROGRESS)
+ return REDIS_OK;
+
+ if (ac->onConnect) ac->onConnect(ac,REDIS_ERR);
+ __redisAsyncDisconnect(ac);
+ return REDIS_ERR;
+ }
+
+ /* Mark context as connected. */
+ c->flags |= REDIS_CONNECTED;
+ if (ac->onConnect) ac->onConnect(ac,REDIS_OK);
+ return REDIS_OK;
+}
+
+/* This function should be called when the socket is readable.
+ * It processes all replies that can be read and executes their callbacks.
+ */
+void redisAsyncHandleRead(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+
+ if (!(c->flags & REDIS_CONNECTED)) {
+ /* Abort connect was not successful. */
+ if (__redisAsyncHandleConnect(ac) != REDIS_OK)
+ return;
+ /* Try again later when the context is still not connected. */
+ if (!(c->flags & REDIS_CONNECTED))
+ return;
+ }
+
+ if (redisBufferRead(c) == REDIS_ERR) {
+ __redisAsyncDisconnect(ac);
+ } else {
+ /* Always re-schedule reads */
+ _EL_ADD_READ(ac);
+ redisProcessCallbacks(ac);
+ }
+}
+
+void redisAsyncHandleWrite(redisAsyncContext *ac) {
+ redisContext *c = &(ac->c);
+ int done = 0;
+
+ if (!(c->flags & REDIS_CONNECTED)) {
+ /* Abort connect was not successful. */
+ if (__redisAsyncHandleConnect(ac) != REDIS_OK)
+ return;
+ /* Try again later when the context is still not connected. */
+ if (!(c->flags & REDIS_CONNECTED))
+ return;
+ }
+
+ if (redisBufferWrite(c,&done) == REDIS_ERR) {
+ __redisAsyncDisconnect(ac);
+ } else {
+ /* Continue writing when not done, stop writing otherwise */
+ if (!done)
+ _EL_ADD_WRITE(ac);
+ else
+ _EL_DEL_WRITE(ac);
+
+ /* Always schedule reads after writes */
+ _EL_ADD_READ(ac);
+ }
+}
+
+/* Sets a pointer to the first argument and its length starting at p. Returns
+ * the number of bytes to skip to get to the following argument. */
+static const char *nextArgument(const char *start, const char **str, size_t *len) {
+ const char *p = start;
+ if (p[0] != '$') {
+ p = strchr(p,'$');
+ if (p == NULL) return NULL;
+ }
+
+ *len = (int)strtol(p+1,NULL,10);
+ p = strchr(p,'\r');
+ assert(p);
+ *str = p+2;
+ return p+2+(*len)+2;
+}
+
+/* Helper function for the redisAsyncCommand* family of functions. Writes a
+ * formatted command to the output buffer and registers the provided callback
+ * function with the context. */
+static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) {
+ redisContext *c = &(ac->c);
+ redisCallback cb;
+ int pvariant, hasnext;
+ const char *cstr, *astr;
+ size_t clen, alen;
+ const char *p;
+ sds sname;
+ int ret;
+
+ /* Don't accept new commands when the connection is about to be closed. */
+ if (c->flags & (REDIS_DISCONNECTING | REDIS_FREEING)) return REDIS_ERR;
+
+ /* Setup callback */
+ cb.fn = fn;
+ cb.privdata = privdata;
+
+ /* Find out which command will be appended. */
+ p = nextArgument(cmd,&cstr,&clen);
+ assert(p != NULL);
+ hasnext = (p[0] == '$');
+ pvariant = (tolower(cstr[0]) == 'p') ? 1 : 0;
+ cstr += pvariant;
+ clen -= pvariant;
+
+ if (hasnext && clen >= 9 && strncasecmp(cstr,"subscribe\r\n",9) == 0) {
+ c->flags |= REDIS_SUBSCRIBED;
+
+ /* Add every channel/pattern to the list of subscription callbacks. */
+ while ((p = nextArgument(p,&astr,&alen)) != NULL) {
+ sname = sdsnewlen(astr,alen);
+ if (pvariant)
+ ret = dictReplace(ac->sub.patterns,sname,&cb);
+ else
+ ret = dictReplace(ac->sub.channels,sname,&cb);
+
+ if (ret == 0) sdsfree(sname);
+ }
+ } else if (clen >= 11 && strncasecmp(cstr,"unsubscribe\r\n",11) == 0) {
+ /* It is only useful to call (P)UNSUBSCRIBE when the context is
+ * subscribed to one or more channels or patterns. */
+ if (!(c->flags & REDIS_SUBSCRIBED)) return REDIS_ERR;
+
+ /* (P)UNSUBSCRIBE does not have its own response: every channel or
+ * pattern that is unsubscribed will receive a message. This means we
+ * should not append a callback function for this command. */
+ } else if(clen >= 7 && strncasecmp(cstr,"monitor\r\n",7) == 0) {
+ /* Set monitor flag and push callback */
+ c->flags |= REDIS_MONITORING;
+ __redisPushCallback(&ac->replies,&cb);
+ } else {
+ if (c->flags & REDIS_SUBSCRIBED)
+ /* This will likely result in an error reply, but it needs to be
+ * received and passed to the callback. */
+ __redisPushCallback(&ac->sub.invalid,&cb);
+ else
+ __redisPushCallback(&ac->replies,&cb);
+ }
+
+ __redisAppendCommand(c,cmd,len);
+
+ /* Always schedule a write when the write buffer is non-empty */
+ _EL_ADD_WRITE(ac);
+
+ return REDIS_OK;
+}
+
+int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap) {
+ char *cmd;
+ int len;
+ int status;
+ len = redisvFormatCommand(&cmd,format,ap);
+
+ /* We don't want to pass -1 or -2 to future functions as a length. */
+ if (len < 0)
+ return REDIS_ERR;
+
+ status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
+ free(cmd);
+ return status;
+}
+
+int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...) {
+ va_list ap;
+ int status;
+ va_start(ap,format);
+ status = redisvAsyncCommand(ac,fn,privdata,format,ap);
+ va_end(ap);
+ return status;
+}
+
+int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) {
+ sds cmd;
+ int len;
+ int status;
+ len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
+ status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
+ sdsfree(cmd);
+ return status;
+}
+
+int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) {
+ int status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
+ return status;
+}
diff --git a/contrib/hiredis/async.h b/contrib/hiredis/async.h
new file mode 100644
index 0000000..f19139c
--- /dev/null
+++ b/contrib/hiredis/async.h
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_ASYNC_H
+#define __HIREDIS_ASYNC_H
+#include "hiredis.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct redisAsyncContext; /* need forward declaration of redisAsyncContext */
+struct dict; /* dictionary header is included in async.c */
+
+/* Reply callback prototype and container */
+typedef void (redisCallbackFn)(struct redisAsyncContext*, void*, void*);
+typedef struct redisCallback {
+ struct redisCallback *next; /* simple singly linked list */
+ redisCallbackFn *fn;
+ void *privdata;
+} redisCallback;
+
+/* List of callbacks for either regular replies or pub/sub */
+typedef struct redisCallbackList {
+ redisCallback *head, *tail;
+} redisCallbackList;
+
+/* Connection callback prototypes */
+typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status);
+typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status);
+
+/* Context for an async connection to Redis */
+typedef struct redisAsyncContext {
+ /* Hold the regular context, so it can be realloc'ed. */
+ redisContext c;
+
+ /* Setup error flags so they can be used directly. */
+ int err;
+ char *errstr;
+
+ /* Not used by hiredis */
+ void *data;
+
+ /* Event library data and hooks */
+ struct {
+ void *data;
+
+ /* Hooks that are called when the library expects to start
+ * reading/writing. These functions should be idempotent. */
+ void (*addRead)(void *privdata);
+ void (*delRead)(void *privdata);
+ void (*addWrite)(void *privdata);
+ void (*delWrite)(void *privdata);
+ void (*cleanup)(void *privdata);
+ } ev;
+
+ /* Called when either the connection is terminated due to an error or per
+ * user request. The status is set accordingly (REDIS_OK, REDIS_ERR). */
+ redisDisconnectCallback *onDisconnect;
+
+ /* Called when the first write event was received. */
+ redisConnectCallback *onConnect;
+
+ /* Regular command callbacks */
+ redisCallbackList replies;
+
+ /* Subscription callbacks */
+ struct {
+ redisCallbackList invalid;
+ struct dict *channels;
+ struct dict *patterns;
+ } sub;
+} redisAsyncContext;
+
+/* Functions that proxy to hiredis */
+redisAsyncContext *redisAsyncConnect(const char *ip, int port);
+redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr);
+redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
+ const char *source_addr);
+redisAsyncContext *redisAsyncConnectUnix(const char *path);
+int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn);
+int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
+void redisAsyncDisconnect(redisAsyncContext *ac);
+void redisAsyncFree(redisAsyncContext *ac);
+
+/* Handle read/write events */
+void redisAsyncHandleRead(redisAsyncContext *ac);
+void redisAsyncHandleWrite(redisAsyncContext *ac);
+
+/* Command functions for an async context. Write the command to the
+ * output buffer and register the provided callback. */
+#ifdef __GNUC__
+__attribute__((format(printf, 4, 0)))
+#endif
+int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap);
+#ifdef __GNUC__
+__attribute__((format(printf, 4, 5)))
+#endif
+int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...);
+int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen);
+int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/hiredis/dict.c b/contrib/hiredis/dict.c
new file mode 100644
index 0000000..0fbc1b4
--- /dev/null
+++ b/contrib/hiredis/dict.c
@@ -0,0 +1,340 @@
+/* Hash table implementation.
+ *
+ * This file implements in memory hash tables with insert/del/replace/find/
+ * get-random-element operations. Hash tables will auto resize if needed
+ * tables of power of two in size are used, collisions are handled by
+ * chaining. See the source code for more information... :)
+ *
+ * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <stdlib.h>
+#include <assert.h>
+#include <limits.h>
+#include "dict.h"
+
+/* -------------------------- private prototypes ---------------------------- */
+
+static int _dictExpandIfNeeded(dict *ht);
+static unsigned long _dictNextPower(unsigned long size);
+static int _dictKeyIndex(dict *ht, const void *key);
+static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
+
+/* -------------------------- hash functions -------------------------------- */
+
+/* Generic hash function (a popular one from Bernstein).
+ * I tested a few and this was the best. */
+static unsigned int dictGenHashFunction(const unsigned char *buf, int len) {
+ unsigned int hash = 5381;
+
+ while (len--)
+ hash = ((hash << 5) + hash) + (*buf++); /* hash * 33 + c */
+ return hash;
+}
+
+/* ----------------------------- API implementation ------------------------- */
+
+/* Reset an hashtable already initialized with ht_init().
+ * NOTE: This function should only called by ht_destroy(). */
+static void _dictReset(dict *ht) {
+ ht->table = NULL;
+ ht->size = 0;
+ ht->sizemask = 0;
+ ht->used = 0;
+}
+
+/* Create a new hash table */
+static dict *dictCreate(dictType *type, void *privDataPtr) {
+ dict *ht = malloc(sizeof(*ht));
+ _dictInit(ht,type,privDataPtr);
+ return ht;
+}
+
+/* Initialize the hash table */
+static int _dictInit(dict *ht, dictType *type, void *privDataPtr) {
+ _dictReset(ht);
+ ht->type = type;
+ ht->privdata = privDataPtr;
+ return DICT_OK;
+}
+
+/* Expand or create the hashtable */
+static int dictExpand(dict *ht, unsigned long size) {
+ dict n; /* the new hashtable */
+ unsigned long realsize = _dictNextPower(size), i;
+
+ /* the size is invalid if it is smaller than the number of
+ * elements already inside the hashtable */
+ if (ht->used > size)
+ return DICT_ERR;
+
+ _dictInit(&n, ht->type, ht->privdata);
+ n.size = realsize;
+ n.sizemask = realsize-1;
+ n.table = calloc(realsize,sizeof(dictEntry*));
+
+ /* Copy all the elements from the old to the new table:
+ * note that if the old hash table is empty ht->size is zero,
+ * so dictExpand just creates an hash table. */
+ n.used = ht->used;
+ for (i = 0; i < ht->size && ht->used > 0; i++) {
+ dictEntry *he, *nextHe;
+
+ if (ht->table[i] == NULL) continue;
+
+ /* For each hash entry on this slot... */
+ he = ht->table[i];
+ while(he) {
+ unsigned int h;
+
+ nextHe = he->next;
+ /* Get the new element index */
+ h = dictHashKey(ht, he->key) & n.sizemask;
+ he->next = n.table[h];
+ n.table[h] = he;
+ ht->used--;
+ /* Pass to the next element */
+ he = nextHe;
+ }
+ }
+ assert(ht->used == 0);
+ free(ht->table);
+
+ /* Remap the new hashtable in the old */
+ *ht = n;
+ return DICT_OK;
+}
+
+/* Add an element to the target hash table */
+static int dictAdd(dict *ht, void *key, void *val) {
+ int index;
+ dictEntry *entry;
+
+ /* Get the index of the new element, or -1 if
+ * the element already exists. */
+ if ((index = _dictKeyIndex(ht, key)) == -1)
+ return DICT_ERR;
+
+ /* Allocates the memory and stores key */
+ entry = malloc(sizeof(*entry));
+ entry->next = ht->table[index];
+ ht->table[index] = entry;
+
+ /* Set the hash entry fields. */
+ dictSetHashKey(ht, entry, key);
+ dictSetHashVal(ht, entry, val);
+ ht->used++;
+ return DICT_OK;
+}
+
+/* Add an element, discarding the old if the key already exists.
+ * Return 1 if the key was added from scratch, 0 if there was already an
+ * element with such key and dictReplace() just performed a value update
+ * operation. */
+static int dictReplace(dict *ht, void *key, void *val) {
+ dictEntry *entry, auxentry;
+
+ /* Try to add the element. If the key
+ * does not exists dictAdd will succeed. */
+ if (dictAdd(ht, key, val) == DICT_OK)
+ return 1;
+ /* It already exists, get the entry */
+ entry = dictFind(ht, key);
+ /* Free the old value and set the new one */
+ /* Set the new value and free the old one. Note that it is important
+ * to do that in this order, as the value may just be exactly the same
+ * as the previous one. In this context, think to reference counting,
+ * you want to increment (set), and then decrement (free), and not the
+ * reverse. */
+ if (entry) {
+ auxentry = *entry;
+ dictSetHashVal(ht, entry, val);
+ dictFreeEntryVal(ht, &auxentry);
+ }
+ return 0;
+}
+
+/* Search and remove an element */
+static int dictDelete(dict *ht, const void *key) {
+ unsigned int h;
+ dictEntry *de, *prevde;
+
+ if (ht->size == 0)
+ return DICT_ERR;
+ h = dictHashKey(ht, key) & ht->sizemask;
+ de = ht->table[h];
+
+ prevde = NULL;
+ while(de) {
+ if (dictCompareHashKeys(ht,key,de->key)) {
+ /* Unlink the element from the list */
+ if (prevde)
+ prevde->next = de->next;
+ else
+ ht->table[h] = de->next;
+
+ dictFreeEntryKey(ht,de);
+ dictFreeEntryVal(ht,de);
+ free(de);
+ ht->used--;
+ return DICT_OK;
+ }
+ prevde = de;
+ de = de->next;
+ }
+ return DICT_ERR; /* not found */
+}
+
+/* Destroy an entire hash table */
+static int _dictClear(dict *ht) {
+ unsigned long i;
+
+ /* Free all the elements */
+ for (i = 0; i < ht->size && ht->used > 0; i++) {
+ dictEntry *he, *nextHe;
+
+ if ((he = ht->table[i]) == NULL) continue;
+ while(he) {
+ nextHe = he->next;
+ dictFreeEntryKey(ht, he);
+ dictFreeEntryVal(ht, he);
+ free(he);
+ ht->used--;
+ he = nextHe;
+ }
+ }
+ /* Free the table and the allocated cache structure */
+ free(ht->table);
+ /* Re-initialize the table */
+ _dictReset(ht);
+ return DICT_OK; /* never fails */
+}
+
+/* Clear & Release the hash table */
+static void dictRelease(dict *ht) {
+ _dictClear(ht);
+ free(ht);
+}
+
+static dictEntry *dictFind(dict *ht, const void *key) {
+ dictEntry *he;
+ unsigned int h;
+
+ if (ht->size == 0) return NULL;
+ h = dictHashKey(ht, key) & ht->sizemask;
+ he = ht->table[h];
+ while(he) {
+ if (dictCompareHashKeys(ht, key, he->key))
+ return he;
+ he = he->next;
+ }
+ return NULL;
+}
+
+static dictIterator *dictGetIterator(dict *ht) {
+ dictIterator *iter = malloc(sizeof(*iter));
+
+ iter->ht = ht;
+ iter->index = -1;
+ iter->entry = NULL;
+ iter->nextEntry = NULL;
+ return iter;
+}
+
+static dictEntry *dictNext(dictIterator *iter) {
+ while (1) {
+ if (iter->entry == NULL) {
+ iter->index++;
+ if (iter->index >=
+ (signed)iter->ht->size) break;
+ iter->entry = iter->ht->table[iter->index];
+ } else {
+ iter->entry = iter->nextEntry;
+ }
+ if (iter->entry) {
+ /* We need to save the 'next' here, the iterator user
+ * may delete the entry we are returning. */
+ iter->nextEntry = iter->entry->next;
+ return iter->entry;
+ }
+ }
+ return NULL;
+}
+
+static void dictReleaseIterator(dictIterator *iter) {
+ free(iter);
+}
+
+/* ------------------------- private functions ------------------------------ */
+
+/* Expand the hash table if needed */
+static int _dictExpandIfNeeded(dict *ht) {
+ /* If the hash table is empty expand it to the initial size,
+ * if the table is "full" dobule its size. */
+ if (ht->size == 0)
+ return dictExpand(ht, DICT_HT_INITIAL_SIZE);
+ if (ht->used == ht->size)
+ return dictExpand(ht, ht->size*2);
+ return DICT_OK;
+}
+
+/* Our hash table capability is a power of two */
+static unsigned long _dictNextPower(unsigned long size) {
+ unsigned long i = DICT_HT_INITIAL_SIZE;
+
+ if (size >= LONG_MAX) return LONG_MAX;
+ while(1) {
+ if (i >= size)
+ return i;
+ i *= 2;
+ }
+}
+
+/* Returns the index of a free slot that can be populated with
+ * an hash entry for the given 'key'.
+ * If the key already exists, -1 is returned. */
+static int _dictKeyIndex(dict *ht, const void *key) {
+ unsigned int h;
+ dictEntry *he;
+
+ /* Expand the hashtable if needed */
+ if (_dictExpandIfNeeded(ht) == DICT_ERR)
+ return -1;
+ /* Compute the key hash value */
+ h = dictHashKey(ht, key) & ht->sizemask;
+ /* Search if this slot does not already contain the given key */
+ he = ht->table[h];
+ while(he) {
+ if (dictCompareHashKeys(ht, key, he->key))
+ return -1;
+ he = he->next;
+ }
+ return h;
+}
+
diff --git a/contrib/hiredis/dict.h b/contrib/hiredis/dict.h
new file mode 100644
index 0000000..95fcd28
--- /dev/null
+++ b/contrib/hiredis/dict.h
@@ -0,0 +1,126 @@
+/* Hash table implementation.
+ *
+ * This file implements in memory hash tables with insert/del/replace/find/
+ * get-random-element operations. Hash tables will auto resize if needed
+ * tables of power of two in size are used, collisions are handled by
+ * chaining. See the source code for more information... :)
+ *
+ * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __DICT_H
+#define __DICT_H
+
+#define DICT_OK 0
+#define DICT_ERR 1
+
+/* Unused arguments generate annoying warnings... */
+#define DICT_NOTUSED(V) ((void) V)
+
+typedef struct dictEntry {
+ void *key;
+ void *val;
+ struct dictEntry *next;
+} dictEntry;
+
+typedef struct dictType {
+ unsigned int (*hashFunction)(const void *key);
+ void *(*keyDup)(void *privdata, const void *key);
+ void *(*valDup)(void *privdata, const void *obj);
+ int (*keyCompare)(void *privdata, const void *key1, const void *key2);
+ void (*keyDestructor)(void *privdata, void *key);
+ void (*valDestructor)(void *privdata, void *obj);
+} dictType;
+
+typedef struct dict {
+ dictEntry **table;
+ dictType *type;
+ unsigned long size;
+ unsigned long sizemask;
+ unsigned long used;
+ void *privdata;
+} dict;
+
+typedef struct dictIterator {
+ dict *ht;
+ int index;
+ dictEntry *entry, *nextEntry;
+} dictIterator;
+
+/* This is the initial size of every hash table */
+#define DICT_HT_INITIAL_SIZE 4
+
+/* ------------------------------- Macros ------------------------------------*/
+#define dictFreeEntryVal(ht, entry) \
+ if ((ht)->type->valDestructor) \
+ (ht)->type->valDestructor((ht)->privdata, (entry)->val)
+
+#define dictSetHashVal(ht, entry, _val_) do { \
+ if ((ht)->type->valDup) \
+ entry->val = (ht)->type->valDup((ht)->privdata, _val_); \
+ else \
+ entry->val = (_val_); \
+} while(0)
+
+#define dictFreeEntryKey(ht, entry) \
+ if ((ht)->type->keyDestructor) \
+ (ht)->type->keyDestructor((ht)->privdata, (entry)->key)
+
+#define dictSetHashKey(ht, entry, _key_) do { \
+ if ((ht)->type->keyDup) \
+ entry->key = (ht)->type->keyDup((ht)->privdata, _key_); \
+ else \
+ entry->key = (_key_); \
+} while(0)
+
+#define dictCompareHashKeys(ht, key1, key2) \
+ (((ht)->type->keyCompare) ? \
+ (ht)->type->keyCompare((ht)->privdata, key1, key2) : \
+ (key1) == (key2))
+
+#define dictHashKey(ht, key) (ht)->type->hashFunction(key)
+
+#define dictGetEntryKey(he) ((he)->key)
+#define dictGetEntryVal(he) ((he)->val)
+#define dictSlots(ht) ((ht)->size)
+#define dictSize(ht) ((ht)->used)
+
+/* API */
+static unsigned int dictGenHashFunction(const unsigned char *buf, int len);
+static dict *dictCreate(dictType *type, void *privDataPtr);
+static int dictExpand(dict *ht, unsigned long size);
+static int dictAdd(dict *ht, void *key, void *val);
+static int dictReplace(dict *ht, void *key, void *val);
+static int dictDelete(dict *ht, const void *key);
+static void dictRelease(dict *ht);
+static dictEntry * dictFind(dict *ht, const void *key);
+static dictIterator *dictGetIterator(dict *ht);
+static dictEntry *dictNext(dictIterator *iter);
+static void dictReleaseIterator(dictIterator *iter);
+
+#endif /* __DICT_H */
diff --git a/contrib/hiredis/fmacros.h b/contrib/hiredis/fmacros.h
new file mode 100644
index 0000000..19d7b21
--- /dev/null
+++ b/contrib/hiredis/fmacros.h
@@ -0,0 +1,21 @@
+#ifndef __HIREDIS_FMACRO_H
+#define __HIREDIS_FMACRO_H
+
+#if defined(__linux__)
+#define _BSD_SOURCE
+#define _DEFAULT_SOURCE
+#endif
+
+#if defined(__sun__)
+#define _POSIX_C_SOURCE 200112L
+#elif defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__)
+#define _XOPEN_SOURCE 600
+#else
+#define _XOPEN_SOURCE
+#endif
+
+#if __APPLE__ && __MACH__
+#define _OSX
+#endif
+
+#endif
diff --git a/contrib/hiredis/hiredis.c b/contrib/hiredis/hiredis.c
new file mode 100644
index 0000000..0f87bc3
--- /dev/null
+++ b/contrib/hiredis/hiredis.c
@@ -0,0 +1,1025 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ * Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+#include <errno.h>
+#include <ctype.h>
+
+#include "hiredis.h"
+#include "net.h"
+#include "sds.h"
+
+static redisReply *createReplyObject(int type);
+static void *createStringObject(const redisReadTask *task, char *str, size_t len);
+static void *createArrayObject(const redisReadTask *task, int elements);
+static void *createIntegerObject(const redisReadTask *task, long long value);
+static void *createNilObject(const redisReadTask *task);
+
+/* Default set of functions to build the reply. Keep in mind that such a
+ * function returning NULL is interpreted as OOM. */
+static redisReplyObjectFunctions defaultFunctions = {
+ createStringObject,
+ createArrayObject,
+ createIntegerObject,
+ createNilObject,
+ freeReplyObject
+};
+
+/* Create a reply object */
+static redisReply *createReplyObject(int type) {
+ redisReply *r = calloc(1,sizeof(*r));
+
+ if (r == NULL)
+ return NULL;
+
+ r->type = type;
+ return r;
+}
+
+/* Free a reply object */
+void freeReplyObject(void *reply) {
+ redisReply *r = reply;
+ size_t j;
+
+ if (r == NULL)
+ return;
+
+ switch(r->type) {
+ case REDIS_REPLY_INTEGER:
+ break; /* Nothing to free */
+ case REDIS_REPLY_ARRAY:
+ if (r->element != NULL) {
+ for (j = 0; j < r->elements; j++)
+ if (r->element[j] != NULL)
+ freeReplyObject(r->element[j]);
+ free(r->element);
+ }
+ break;
+ case REDIS_REPLY_ERROR:
+ case REDIS_REPLY_STATUS:
+ case REDIS_REPLY_STRING:
+ if (r->str != NULL)
+ free(r->str);
+ break;
+ }
+ free(r);
+}
+
+static void *createStringObject(const redisReadTask *task, char *str, size_t len) {
+ redisReply *r, *parent;
+ char *buf;
+
+ r = createReplyObject(task->type);
+ if (r == NULL)
+ return NULL;
+
+ buf = malloc(len+1);
+ if (buf == NULL) {
+ freeReplyObject(r);
+ return NULL;
+ }
+
+ assert(task->type == REDIS_REPLY_ERROR ||
+ task->type == REDIS_REPLY_STATUS ||
+ task->type == REDIS_REPLY_STRING);
+
+ /* Copy string value */
+ memcpy(buf,str,len);
+ buf[len] = '\0';
+ r->str = buf;
+ r->len = len;
+
+ if (task->parent) {
+ parent = task->parent->obj;
+ assert(parent->type == REDIS_REPLY_ARRAY);
+ parent->element[task->idx] = r;
+ }
+ return r;
+}
+
+static void *createArrayObject(const redisReadTask *task, int elements) {
+ redisReply *r, *parent;
+
+ r = createReplyObject(REDIS_REPLY_ARRAY);
+ if (r == NULL)
+ return NULL;
+
+ if (elements > 0) {
+ r->element = calloc(elements,sizeof(redisReply*));
+ if (r->element == NULL) {
+ freeReplyObject(r);
+ return NULL;
+ }
+ }
+
+ r->elements = elements;
+
+ if (task->parent) {
+ parent = task->parent->obj;
+ assert(parent->type == REDIS_REPLY_ARRAY);
+ parent->element[task->idx] = r;
+ }
+ return r;
+}
+
+static void *createIntegerObject(const redisReadTask *task, long long value) {
+ redisReply *r, *parent;
+
+ r = createReplyObject(REDIS_REPLY_INTEGER);
+ if (r == NULL)
+ return NULL;
+
+ r->integer = value;
+
+ if (task->parent) {
+ parent = task->parent->obj;
+ assert(parent->type == REDIS_REPLY_ARRAY);
+ parent->element[task->idx] = r;
+ }
+ return r;
+}
+
+static void *createNilObject(const redisReadTask *task) {
+ redisReply *r, *parent;
+
+ r = createReplyObject(REDIS_REPLY_NIL);
+ if (r == NULL)
+ return NULL;
+
+ if (task->parent) {
+ parent = task->parent->obj;
+ assert(parent->type == REDIS_REPLY_ARRAY);
+ parent->element[task->idx] = r;
+ }
+ return r;
+}
+
+/* Return the number of digits of 'v' when converted to string in radix 10.
+ * Implementation borrowed from link in redis/src/util.c:string2ll(). */
+static uint32_t countDigits(uint64_t v) {
+ uint32_t result = 1;
+ for (;;) {
+ if (v < 10) return result;
+ if (v < 100) return result + 1;
+ if (v < 1000) return result + 2;
+ if (v < 10000) return result + 3;
+ v /= 10000U;
+ result += 4;
+ }
+}
+
+/* Helper that calculates the bulk length given a certain string length. */
+static size_t bulklen(size_t len) {
+ return 1+countDigits(len)+2+len+2;
+}
+
+int redisvFormatCommand(char **target, const char *format, va_list ap) {
+ const char *c = format;
+ char *cmd = NULL; /* final command */
+ int pos; /* position in final command */
+ sds curarg, newarg; /* current argument */
+ int touched = 0; /* was the current argument touched? */
+ char **curargv = NULL, **newargv = NULL;
+ int argc = 0;
+ int totlen = 0;
+ int error_type = 0; /* 0 = no error; -1 = memory error; -2 = format error */
+ int j;
+
+ /* Abort if there is not target to set */
+ if (target == NULL)
+ return -1;
+
+ /* Build the command string accordingly to protocol */
+ curarg = sdsempty();
+ if (curarg == NULL)
+ return -1;
+
+ while(*c != '\0') {
+ if (*c != '%' || c[1] == '\0') {
+ if (*c == ' ') {
+ if (touched) {
+ newargv = realloc(curargv,sizeof(char*)*(argc+1));
+ if (newargv == NULL) goto memory_err;
+ curargv = newargv;
+ curargv[argc++] = curarg;
+ totlen += bulklen(sdslen(curarg));
+
+ /* curarg is put in argv so it can be overwritten. */
+ curarg = sdsempty();
+ if (curarg == NULL) goto memory_err;
+ touched = 0;
+ }
+ } else {
+ newarg = sdscatlen(curarg,c,1);
+ if (newarg == NULL) goto memory_err;
+ curarg = newarg;
+ touched = 1;
+ }
+ } else {
+ char *arg;
+ size_t size;
+
+ /* Set newarg so it can be checked even if it is not touched. */
+ newarg = curarg;
+
+ switch(c[1]) {
+ case 's':
+ arg = va_arg(ap,char*);
+ size = strlen(arg);
+ if (size > 0)
+ newarg = sdscatlen(curarg,arg,size);
+ break;
+ case 'b':
+ arg = va_arg(ap,char*);
+ size = va_arg(ap,size_t);
+ if (size > 0)
+ newarg = sdscatlen(curarg,arg,size);
+ break;
+ case '%':
+ newarg = sdscat(curarg,"%");
+ break;
+ default:
+ /* Try to detect printf format */
+ {
+ static const char intfmts[] = "diouxX";
+ static const char flags[] = "#0-+ ";
+ char _format[16];
+ const char *_p = c+1;
+ size_t _l = 0;
+ va_list _cpy;
+
+ /* Flags */
+ while (*_p != '\0' && strchr(flags,*_p) != NULL) _p++;
+
+ /* Field width */
+ while (*_p != '\0' && isdigit(*_p)) _p++;
+
+ /* Precision */
+ if (*_p == '.') {
+ _p++;
+ while (*_p != '\0' && isdigit(*_p)) _p++;
+ }
+
+ /* Copy va_list before consuming with va_arg */
+ va_copy(_cpy,ap);
+
+ /* Integer conversion (without modifiers) */
+ if (strchr(intfmts,*_p) != NULL) {
+ va_arg(ap,int);
+ goto fmt_valid;
+ }
+
+ /* Double conversion (without modifiers) */
+ if (strchr("eEfFgGaA",*_p) != NULL) {
+ va_arg(ap,double);
+ goto fmt_valid;
+ }
+
+ /* Size: char */
+ if (_p[0] == 'h' && _p[1] == 'h') {
+ _p += 2;
+ if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+ va_arg(ap,int); /* char gets promoted to int */
+ goto fmt_valid;
+ }
+ goto fmt_invalid;
+ }
+
+ /* Size: short */
+ if (_p[0] == 'h') {
+ _p += 1;
+ if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+ va_arg(ap,int); /* short gets promoted to int */
+ goto fmt_valid;
+ }
+ goto fmt_invalid;
+ }
+
+ /* Size: long long */
+ if (_p[0] == 'l' && _p[1] == 'l') {
+ _p += 2;
+ if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+ va_arg(ap,long long);
+ goto fmt_valid;
+ }
+ goto fmt_invalid;
+ }
+
+ /* Size: long */
+ if (_p[0] == 'l') {
+ _p += 1;
+ if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+ va_arg(ap,long);
+ goto fmt_valid;
+ }
+ goto fmt_invalid;
+ }
+
+ fmt_invalid:
+ va_end(_cpy);
+ goto format_err;
+
+ fmt_valid:
+ _l = (_p+1)-c;
+ if (_l < sizeof(_format)-2) {
+ memcpy(_format,c,_l);
+ _format[_l] = '\0';
+ newarg = sdscatvprintf(curarg,_format,_cpy);
+
+ /* Update current position (note: outer blocks
+ * increment c twice so compensate here) */
+ c = _p-1;
+ }
+
+ va_end(_cpy);
+ break;
+ }
+ }
+
+ if (newarg == NULL) goto memory_err;
+ curarg = newarg;
+
+ touched = 1;
+ c++;
+ }
+ c++;
+ }
+
+ /* Add the last argument if needed */
+ if (touched) {
+ newargv = realloc(curargv,sizeof(char*)*(argc+1));
+ if (newargv == NULL) goto memory_err;
+ curargv = newargv;
+ curargv[argc++] = curarg;
+ totlen += bulklen(sdslen(curarg));
+ } else {
+ sdsfree(curarg);
+ }
+
+ /* Clear curarg because it was put in curargv or was free'd. */
+ curarg = NULL;
+
+ /* Add bytes needed to hold multi bulk count */
+ totlen += 1+countDigits(argc)+2;
+
+ /* Build the command at protocol level */
+ cmd = malloc(totlen+1);
+ if (cmd == NULL) goto memory_err;
+
+ pos = sprintf(cmd,"*%d\r\n",argc);
+ for (j = 0; j < argc; j++) {
+ pos += sprintf(cmd+pos,"$%zu\r\n",sdslen(curargv[j]));
+ memcpy(cmd+pos,curargv[j],sdslen(curargv[j]));
+ pos += sdslen(curargv[j]);
+ sdsfree(curargv[j]);
+ cmd[pos++] = '\r';
+ cmd[pos++] = '\n';
+ }
+ assert(pos == totlen);
+ cmd[pos] = '\0';
+
+ free(curargv);
+ *target = cmd;
+ return totlen;
+
+format_err:
+ error_type = -2;
+ goto cleanup;
+
+memory_err:
+ error_type = -1;
+ goto cleanup;
+
+cleanup:
+ if (curargv) {
+ while(argc--)
+ sdsfree(curargv[argc]);
+ free(curargv);
+ }
+
+ sdsfree(curarg);
+
+ /* No need to check cmd since it is the last statement that can fail,
+ * but do it anyway to be as defensive as possible. */
+ if (cmd != NULL)
+ free(cmd);
+
+ return error_type;
+}
+
+/* Format a command according to the Redis protocol. This function
+ * takes a format similar to printf:
+ *
+ * %s represents a C null terminated string you want to interpolate
+ * %b represents a binary safe string
+ *
+ * When using %b you need to provide both the pointer to the string
+ * and the length in bytes as a size_t. Examples:
+ *
+ * len = redisFormatCommand(target, "GET %s", mykey);
+ * len = redisFormatCommand(target, "SET %s %b", mykey, myval, myvallen);
+ */
+int redisFormatCommand(char **target, const char *format, ...) {
+ va_list ap;
+ int len;
+ va_start(ap,format);
+ len = redisvFormatCommand(target,format,ap);
+ va_end(ap);
+
+ /* The API says "-1" means bad result, but we now also return "-2" in some
+ * cases. Force the return value to always be -1. */
+ if (len < 0)
+ len = -1;
+
+ return len;
+}
+
+/* Format a command according to the Redis protocol using an sds string and
+ * sdscatfmt for the processing of arguments. This function takes the
+ * number of arguments, an array with arguments and an array with their
+ * lengths. If the latter is set to NULL, strlen will be used to compute the
+ * argument lengths.
+ */
+int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv,
+ const size_t *argvlen)
+{
+ sds cmd;
+ unsigned long long totlen;
+ int j;
+ size_t len;
+
+ /* Abort on a NULL target */
+ if (target == NULL)
+ return -1;
+
+ /* Calculate our total size */
+ totlen = 1+countDigits(argc)+2;
+ for (j = 0; j < argc; j++) {
+ len = argvlen ? argvlen[j] : strlen(argv[j]);
+ totlen += bulklen(len);
+ }
+
+ /* Use an SDS string for command construction */
+ cmd = sdsempty();
+ if (cmd == NULL)
+ return -1;
+
+ /* We already know how much storage we need */
+ cmd = sdsMakeRoomFor(cmd, totlen);
+ if (cmd == NULL)
+ return -1;
+
+ /* Construct command */
+ cmd = sdscatfmt(cmd, "*%i\r\n", argc);
+ for (j=0; j < argc; j++) {
+ len = argvlen ? argvlen[j] : strlen(argv[j]);
+ cmd = sdscatfmt(cmd, "$%T\r\n", len);
+ cmd = sdscatlen(cmd, argv[j], len);
+ cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1);
+ }
+
+ assert(sdslen(cmd)==totlen);
+
+ *target = cmd;
+ return totlen;
+}
+
+void redisFreeSdsCommand(sds cmd) {
+ sdsfree(cmd);
+}
+
+/* Format a command according to the Redis protocol. This function takes the
+ * number of arguments, an array with arguments and an array with their
+ * lengths. If the latter is set to NULL, strlen will be used to compute the
+ * argument lengths.
+ */
+int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen) {
+ char *cmd = NULL; /* final command */
+ int pos; /* position in final command */
+ size_t len;
+ int totlen, j;
+
+ /* Abort on a NULL target */
+ if (target == NULL)
+ return -1;
+
+ /* Calculate number of bytes needed for the command */
+ totlen = 1+countDigits(argc)+2;
+ for (j = 0; j < argc; j++) {
+ len = argvlen ? argvlen[j] : strlen(argv[j]);
+ totlen += bulklen(len);
+ }
+
+ /* Build the command at protocol level */
+ cmd = malloc(totlen+1);
+ if (cmd == NULL)
+ return -1;
+
+ pos = sprintf(cmd,"*%d\r\n",argc);
+ for (j = 0; j < argc; j++) {
+ len = argvlen ? argvlen[j] : strlen(argv[j]);
+ pos += sprintf(cmd+pos,"$%zu\r\n",len);
+ memcpy(cmd+pos,argv[j],len);
+ pos += len;
+ cmd[pos++] = '\r';
+ cmd[pos++] = '\n';
+ }
+ assert(pos == totlen);
+ cmd[pos] = '\0';
+
+ *target = cmd;
+ return totlen;
+}
+
+void redisFreeCommand(char *cmd) {
+ free(cmd);
+}
+
+void __redisSetError(redisContext *c, int type, const char *str) {
+ size_t len;
+
+ c->err = type;
+ if (str != NULL) {
+ len = strlen(str);
+ len = len < (sizeof(c->errstr)-1) ? len : (sizeof(c->errstr)-1);
+ memcpy(c->errstr,str,len);
+ c->errstr[len] = '\0';
+ } else {
+ /* Only REDIS_ERR_IO may lack a description! */
+ assert(type == REDIS_ERR_IO);
+ __redis_strerror_r(errno, c->errstr, sizeof(c->errstr));
+ }
+}
+
+redisReader *redisReaderCreate(void) {
+ return redisReaderCreateWithFunctions(&defaultFunctions);
+}
+
+static redisContext *redisContextInit(void) {
+ redisContext *c;
+
+ c = calloc(1,sizeof(redisContext));
+ if (c == NULL)
+ return NULL;
+
+ c->err = 0;
+ c->errstr[0] = '\0';
+ c->obuf = sdsempty();
+ c->reader = redisReaderCreate();
+ c->tcp.host = NULL;
+ c->tcp.source_addr = NULL;
+ c->unix_sock.path = NULL;
+ c->timeout = NULL;
+
+ if (c->obuf == NULL || c->reader == NULL) {
+ redisFree(c);
+ return NULL;
+ }
+
+ return c;
+}
+
+void redisFree(redisContext *c) {
+ if (c == NULL)
+ return;
+ if (c->fd > 0)
+ close(c->fd);
+ if (c->obuf != NULL)
+ sdsfree(c->obuf);
+ if (c->reader != NULL)
+ redisReaderFree(c->reader);
+ if (c->tcp.host)
+ free(c->tcp.host);
+ if (c->tcp.source_addr)
+ free(c->tcp.source_addr);
+ if (c->unix_sock.path)
+ free(c->unix_sock.path);
+ if (c->timeout)
+ free(c->timeout);
+ free(c);
+}
+
+int redisFreeKeepFd(redisContext *c) {
+ int fd = c->fd;
+ c->fd = -1;
+ redisFree(c);
+ return fd;
+}
+
+int redisReconnect(redisContext *c) {
+ c->err = 0;
+ memset(c->errstr, '\0', strlen(c->errstr));
+
+ if (c->fd > 0) {
+ close(c->fd);
+ }
+
+ sdsfree(c->obuf);
+ redisReaderFree(c->reader);
+
+ c->obuf = sdsempty();
+ c->reader = redisReaderCreate();
+
+ if (c->connection_type == REDIS_CONN_TCP) {
+ return redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port,
+ c->timeout, c->tcp.source_addr);
+ } else if (c->connection_type == REDIS_CONN_UNIX) {
+ return redisContextConnectUnix(c, c->unix_sock.path, c->timeout);
+ } else {
+ /* Something bad happened here and shouldn't have. There isn't
+ enough information in the context to reconnect. */
+ __redisSetError(c,REDIS_ERR_OTHER,"Not enough information to reconnect");
+ }
+
+ return REDIS_ERR;
+}
+
+/* Connect to a Redis instance. On error the field error in the returned
+ * context will be set to the return value of the error function.
+ * When no set of reply functions is given, the default set will be used. */
+redisContext *redisConnect(const char *ip, int port) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags |= REDIS_BLOCK;
+ redisContextConnectTcp(c,ip,port,NULL);
+ return c;
+}
+
+redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags |= REDIS_BLOCK;
+ redisContextConnectTcp(c,ip,port,&tv);
+ return c;
+}
+
+redisContext *redisConnectNonBlock(const char *ip, int port) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags &= ~REDIS_BLOCK;
+ redisContextConnectTcp(c,ip,port,NULL);
+ return c;
+}
+
+redisContext *redisConnectBindNonBlock(const char *ip, int port,
+ const char *source_addr) {
+ redisContext *c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+ c->flags &= ~REDIS_BLOCK;
+ redisContextConnectBindTcp(c,ip,port,NULL,source_addr);
+ return c;
+}
+
+redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port,
+ const char *source_addr) {
+ redisContext *c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+ c->flags &= ~REDIS_BLOCK;
+ c->flags |= REDIS_REUSEADDR;
+ redisContextConnectBindTcp(c,ip,port,NULL,source_addr);
+ return c;
+}
+
+redisContext *redisConnectUnix(const char *path) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags |= REDIS_BLOCK;
+ redisContextConnectUnix(c,path,NULL);
+ return c;
+}
+
+redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags |= REDIS_BLOCK;
+ redisContextConnectUnix(c,path,&tv);
+ return c;
+}
+
+redisContext *redisConnectUnixNonBlock(const char *path) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->flags &= ~REDIS_BLOCK;
+ redisContextConnectUnix(c,path,NULL);
+ return c;
+}
+
+redisContext *redisConnectFd(int fd) {
+ redisContext *c;
+
+ c = redisContextInit();
+ if (c == NULL)
+ return NULL;
+
+ c->fd = fd;
+ c->flags |= REDIS_BLOCK | REDIS_CONNECTED;
+ return c;
+}
+
+/* Set read/write timeout on a blocking socket. */
+int redisSetTimeout(redisContext *c, const struct timeval tv) {
+ if (c->flags & REDIS_BLOCK)
+ return redisContextSetTimeout(c,tv);
+ return REDIS_ERR;
+}
+
+/* Enable connection KeepAlive. */
+int redisEnableKeepAlive(redisContext *c) {
+ if (redisKeepAlive(c, REDIS_KEEPALIVE_INTERVAL) != REDIS_OK)
+ return REDIS_ERR;
+ return REDIS_OK;
+}
+
+/* Use this function to handle a read event on the descriptor. It will try
+ * and read some bytes from the socket and feed them to the reply parser.
+ *
+ * After this function is called, you may use redisContextReadReply to
+ * see if there is a reply available. */
+int redisBufferRead(redisContext *c) {
+ char buf[1024*16];
+ int nread;
+
+ /* Return early when the context has seen an error. */
+ if (c->err)
+ return REDIS_ERR;
+
+ nread = read(c->fd,buf,sizeof(buf));
+ if (nread == -1) {
+ if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) {
+ /* Try again later */
+ } else {
+ __redisSetError(c,REDIS_ERR_IO,NULL);
+ return REDIS_ERR;
+ }
+ } else if (nread == 0) {
+ __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection");
+ return REDIS_ERR;
+ } else {
+ if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) {
+ __redisSetError(c,c->reader->err,c->reader->errstr);
+ return REDIS_ERR;
+ }
+ }
+ return REDIS_OK;
+}
+
+/* Write the output buffer to the socket.
+ *
+ * Returns REDIS_OK when the buffer is empty, or (a part of) the buffer was
+ * successfully written to the socket. When the buffer is empty after the
+ * write operation, "done" is set to 1 (if given).
+ *
+ * Returns REDIS_ERR if an error occurred trying to write and sets
+ * c->errstr to hold the appropriate error string.
+ */
+int redisBufferWrite(redisContext *c, int *done) {
+ int nwritten;
+
+ /* Return early when the context has seen an error. */
+ if (c->err)
+ return REDIS_ERR;
+
+ if (sdslen(c->obuf) > 0) {
+ nwritten = write(c->fd,c->obuf,sdslen(c->obuf));
+ if (nwritten == -1) {
+ if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) {
+ /* Try again later */
+ } else {
+ __redisSetError(c,REDIS_ERR_IO,NULL);
+ return REDIS_ERR;
+ }
+ } else if (nwritten > 0) {
+ if (nwritten == (signed)sdslen(c->obuf)) {
+ sdsfree(c->obuf);
+ c->obuf = sdsempty();
+ } else {
+ sdsrange(c->obuf,nwritten,-1);
+ }
+ }
+ }
+ if (done != NULL) *done = (sdslen(c->obuf) == 0);
+ return REDIS_OK;
+}
+
+/* Internal helper function to try and get a reply from the reader,
+ * or set an error in the context otherwise. */
+int redisGetReplyFromReader(redisContext *c, void **reply) {
+ if (redisReaderGetReply(c->reader,reply) == REDIS_ERR) {
+ __redisSetError(c,c->reader->err,c->reader->errstr);
+ return REDIS_ERR;
+ }
+ return REDIS_OK;
+}
+
+int redisGetReply(redisContext *c, void **reply) {
+ int wdone = 0;
+ void *aux = NULL;
+
+ /* Try to read pending replies */
+ if (redisGetReplyFromReader(c,&aux) == REDIS_ERR)
+ return REDIS_ERR;
+
+ /* For the blocking context, flush output buffer and read reply */
+ if (aux == NULL && c->flags & REDIS_BLOCK) {
+ /* Write until done */
+ do {
+ if (redisBufferWrite(c,&wdone) == REDIS_ERR)
+ return REDIS_ERR;
+ } while (!wdone);
+
+ /* Read until there is a reply */
+ do {
+ if (redisBufferRead(c) == REDIS_ERR)
+ return REDIS_ERR;
+ if (redisGetReplyFromReader(c,&aux) == REDIS_ERR)
+ return REDIS_ERR;
+ } while (aux == NULL);
+ }
+
+ /* Set reply object */
+ if (reply != NULL) *reply = aux;
+ return REDIS_OK;
+}
+
+
+/* Helper function for the redisAppendCommand* family of functions.
+ *
+ * Write a formatted command to the output buffer. When this family
+ * is used, you need to call redisGetReply yourself to retrieve
+ * the reply (or replies in pub/sub).
+ */
+int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) {
+ sds newbuf;
+
+ newbuf = sdscatlen(c->obuf,cmd,len);
+ if (newbuf == NULL) {
+ __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
+ return REDIS_ERR;
+ }
+
+ c->obuf = newbuf;
+ return REDIS_OK;
+}
+
+int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len) {
+
+ if (__redisAppendCommand(c, cmd, len) != REDIS_OK) {
+ return REDIS_ERR;
+ }
+
+ return REDIS_OK;
+}
+
+int redisvAppendCommand(redisContext *c, const char *format, va_list ap) {
+ char *cmd;
+ int len;
+
+ len = redisvFormatCommand(&cmd,format,ap);
+ if (len == -1) {
+ __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
+ return REDIS_ERR;
+ } else if (len == -2) {
+ __redisSetError(c,REDIS_ERR_OTHER,"Invalid format string");
+ return REDIS_ERR;
+ }
+
+ if (__redisAppendCommand(c,cmd,len) != REDIS_OK) {
+ free(cmd);
+ return REDIS_ERR;
+ }
+
+ free(cmd);
+ return REDIS_OK;
+}
+
+int redisAppendCommand(redisContext *c, const char *format, ...) {
+ va_list ap;
+ int ret;
+
+ va_start(ap,format);
+ ret = redisvAppendCommand(c,format,ap);
+ va_end(ap);
+ return ret;
+}
+
+int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) {
+ sds cmd;
+ int len;
+
+ len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
+ if (len == -1) {
+ __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
+ return REDIS_ERR;
+ }
+
+ if (__redisAppendCommand(c,cmd,len) != REDIS_OK) {
+ sdsfree(cmd);
+ return REDIS_ERR;
+ }
+
+ sdsfree(cmd);
+ return REDIS_OK;
+}
+
+/* Helper function for the redisCommand* family of functions.
+ *
+ * Write a formatted command to the output buffer. If the given context is
+ * blocking, immediately read the reply into the "reply" pointer. When the
+ * context is non-blocking, the "reply" pointer will not be used and the
+ * command is simply appended to the write buffer.
+ *
+ * Returns the reply when a reply was successfully retrieved. Returns NULL
+ * otherwise. When NULL is returned in a blocking context, the error field
+ * in the context will be set.
+ */
+static void *__redisBlockForReply(redisContext *c) {
+ void *reply;
+
+ if (c->flags & REDIS_BLOCK) {
+ if (redisGetReply(c,&reply) != REDIS_OK)
+ return NULL;
+ return reply;
+ }
+ return NULL;
+}
+
+void *redisvCommand(redisContext *c, const char *format, va_list ap) {
+ if (redisvAppendCommand(c,format,ap) != REDIS_OK)
+ return NULL;
+ return __redisBlockForReply(c);
+}
+
+void *redisCommand(redisContext *c, const char *format, ...) {
+ va_list ap;
+ void *reply = NULL;
+ va_start(ap,format);
+ reply = redisvCommand(c,format,ap);
+ va_end(ap);
+ return reply;
+}
+
+void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) {
+ if (redisAppendCommandArgv(c,argc,argv,argvlen) != REDIS_OK)
+ return NULL;
+ return __redisBlockForReply(c);
+}
diff --git a/contrib/hiredis/hiredis.h b/contrib/hiredis/hiredis.h
new file mode 100644
index 0000000..6b531b9
--- /dev/null
+++ b/contrib/hiredis/hiredis.h
@@ -0,0 +1,242 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ * Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_H
+#define __HIREDIS_H
+#include "read.h"
+#include <stdarg.h> /* for va_list */
+#include <sys/time.h> /* for struct timeval */
+#include <stdint.h> /* uintXX_t, etc */
+#include <string.h> /* strerror_r, etc */
+#include "sds.h" /* for sds */
+
+#define HIREDIS_MAJOR 0
+#define HIREDIS_MINOR 13
+#define HIREDIS_PATCH 3
+#define HIREDIS_SONAME 0.13
+
+/* Connection type can be blocking or non-blocking and is set in the
+ * least significant bit of the flags field in redisContext. */
+#define REDIS_BLOCK 0x1
+
+/* Connection may be disconnected before being free'd. The second bit
+ * in the flags field is set when the context is connected. */
+#define REDIS_CONNECTED 0x2
+
+/* The async API might try to disconnect cleanly and flush the output
+ * buffer and read all subsequent replies before disconnecting.
+ * This flag means no new commands can come in and the connection
+ * should be terminated once all replies have been read. */
+#define REDIS_DISCONNECTING 0x4
+
+/* Flag specific to the async API which means that the context should be clean
+ * up as soon as possible. */
+#define REDIS_FREEING 0x8
+
+/* Flag that is set when an async callback is executed. */
+#define REDIS_IN_CALLBACK 0x10
+
+/* Flag that is set when the async context has one or more subscriptions. */
+#define REDIS_SUBSCRIBED 0x20
+
+/* Flag that is set when monitor mode is active */
+#define REDIS_MONITORING 0x40
+
+/* Flag that is set when we should set SO_REUSEADDR before calling bind() */
+#define REDIS_REUSEADDR 0x80
+
+#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */
+
+/* number of times we retry to connect in the case of EADDRNOTAVAIL and
+ * SO_REUSEADDR is being used. */
+#define REDIS_CONNECT_RETRIES 10
+
+/* strerror_r has two completely different prototypes and behaviors
+ * depending on system issues, so we need to operate on the error buffer
+ * differently depending on which strerror_r we're using. */
+#ifndef _GNU_SOURCE
+/* "regular" POSIX strerror_r that does the right thing. */
+#define __redis_strerror_r(errno, buf, len) \
+ do { \
+ strerror_r((errno), (buf), (len)); \
+ } while (0)
+#else
+/* "bad" GNU strerror_r we need to clean up after. */
+#define __redis_strerror_r(errno, buf, len) \
+ do { \
+ char *err_str = strerror((errno)); \
+ /* If return value _isn't_ the start of the buffer we passed in, \
+ * then GNU strerror_r returned an internal static buffer and we \
+ * need to copy the result into our private buffer. */ \
+ if (err_str != (buf)) { \
+ buf[(len)-1] = '\0'; \
+ strncat((buf), err_str, ((len) - 1)); \
+ } \
+ } while (0)
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* This is the reply object returned by redisCommand() */
+typedef struct redisReply {
+ int type; /* REDIS_REPLY_* */
+ long long integer; /* The integer when type is REDIS_REPLY_INTEGER */
+ int len; /* Length of string */
+ char *str; /* Used for both REDIS_REPLY_ERROR and REDIS_REPLY_STRING */
+ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */
+ struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */
+} redisReply;
+
+redisReader *redisReaderCreate(void);
+
+/* Function to free the reply objects hiredis returns by default. */
+void freeReplyObject(void *reply);
+
+/* Functions to format a command according to the protocol. */
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 0)))
+#endif
+int redisvFormatCommand(char **target, const char *format, va_list ap);
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 3)))
+#endif
+int redisFormatCommand(char **target, const char *format, ...);
+int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen);
+int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen);
+void redisFreeCommand(char *cmd);
+void redisFreeSdsCommand(sds cmd);
+
+enum redisConnectionType {
+ REDIS_CONN_TCP,
+ REDIS_CONN_UNIX,
+};
+
+/* Context for a connection to Redis */
+typedef struct redisContext {
+ int err; /* Error flags, 0 when there is no error */
+ char errstr[128]; /* String representation of error when applicable */
+ int fd;
+ int flags;
+ char *obuf; /* Write buffer */
+ redisReader *reader; /* Protocol reader */
+
+ enum redisConnectionType connection_type;
+ struct timeval *timeout;
+
+ struct {
+ char *host;
+ char *source_addr;
+ int port;
+ } tcp;
+
+ struct {
+ char *path;
+ } unix_sock;
+
+} redisContext;
+
+redisContext *redisConnect(const char *ip, int port);
+redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv);
+redisContext *redisConnectNonBlock(const char *ip, int port);
+redisContext *redisConnectBindNonBlock(const char *ip, int port,
+ const char *source_addr);
+redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port,
+ const char *source_addr);
+redisContext *redisConnectUnix(const char *path);
+redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv);
+redisContext *redisConnectUnixNonBlock(const char *path);
+redisContext *redisConnectFd(int fd);
+
+/**
+ * Reconnect the given context using the saved information.
+ *
+ * This re-uses the exact same connect options as in the initial connection.
+ * host, ip (or path), timeout and bind address are reused,
+ * flags are used unmodified from the existing context.
+ *
+ * Returns REDIS_OK on successful connect or REDIS_ERR otherwise.
+ */
+int redisReconnect(redisContext *c);
+
+int redisSetTimeout(redisContext *c, const struct timeval tv);
+int redisEnableKeepAlive(redisContext *c);
+void redisFree(redisContext *c);
+int redisFreeKeepFd(redisContext *c);
+int redisBufferRead(redisContext *c);
+int redisBufferWrite(redisContext *c, int *done);
+
+/* In a blocking context, this function first checks if there are unconsumed
+ * replies to return and returns one if so. Otherwise, it flushes the output
+ * buffer to the socket and reads until it has a reply. In a non-blocking
+ * context, it will return unconsumed replies until there are no more. */
+int redisGetReply(redisContext *c, void **reply);
+int redisGetReplyFromReader(redisContext *c, void **reply);
+
+/* Write a formatted command to the output buffer. Use these functions in blocking mode
+ * to get a pipeline of commands. */
+int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len);
+
+/* Write a command to the output buffer. Use these functions in blocking mode
+ * to get a pipeline of commands. */
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 0)))
+#endif
+int redisvAppendCommand(redisContext *c, const char *format, va_list ap);
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 3)))
+#endif
+int redisAppendCommand(redisContext *c, const char *format, ...);
+int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+
+/* Issue a command to Redis. In a blocking context, it is identical to calling
+ * redisAppendCommand, followed by redisGetReply. The function will return
+ * NULL if there was an error in performing the request, otherwise it will
+ * return the reply. In a non-blocking context, it is identical to calling
+ * only redisAppendCommand and will always return NULL. */
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 0)))
+#endif
+void *redisvCommand(redisContext *c, const char *format, va_list ap);
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 3)))
+#endif
+void *redisCommand(redisContext *c, const char *format, ...);
+void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/hiredis/net.c b/contrib/hiredis/net.c
new file mode 100644
index 0000000..97fd42c
--- /dev/null
+++ b/contrib/hiredis/net.c
@@ -0,0 +1,464 @@
+/* Extracted from anet.c to work properly with Hiredis error reporting.
+ *
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ * Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/select.h>
+#include <sys/un.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <netdb.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <poll.h>
+#include <limits.h>
+#include <stdlib.h>
+
+#include "net.h"
+#include "sds.h"
+
+/* Defined in hiredis.c */
+void __redisSetError(redisContext *c, int type, const char *str);
+
+static void redisContextCloseFd(redisContext *c) {
+ if (c && c->fd >= 0) {
+ close(c->fd);
+ c->fd = -1;
+ }
+}
+
+static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) {
+ char buf[128] = { 0 };
+ char *p;
+ size_t len = 0;
+
+ if (prefix != NULL)
+ len = snprintf(buf,sizeof(buf),"%s: ",prefix);
+ p = buf + len;
+ __redis_strerror_r(errno, p, sizeof(buf) - len);
+ __redisSetError(c,type,buf);
+}
+
+static int redisSetReuseAddr(redisContext *c) {
+ int on = 1;
+ if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+ return REDIS_OK;
+}
+
+static int redisCreateSocket(redisContext *c, int type) {
+ int s;
+ if ((s = socket(type, SOCK_STREAM, 0)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+ return REDIS_ERR;
+ }
+ c->fd = s;
+ if (type == AF_INET) {
+ if (redisSetReuseAddr(c) == REDIS_ERR) {
+ return REDIS_ERR;
+ }
+ }
+ return REDIS_OK;
+}
+
+static int redisSetBlocking(redisContext *c, int blocking) {
+ int flags;
+
+ /* Set the socket nonblocking.
+ * Note that fcntl(2) for F_GETFL and F_SETFL can't be
+ * interrupted by a signal. */
+ if ((flags = fcntl(c->fd, F_GETFL)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)");
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+
+ if (blocking)
+ flags &= ~O_NONBLOCK;
+ else
+ flags |= O_NONBLOCK;
+
+ if (fcntl(c->fd, F_SETFL, flags) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)");
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+ return REDIS_OK;
+}
+
+int redisKeepAlive(redisContext *c, int interval) {
+ int val = 1;
+ int fd = c->fd;
+
+ if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){
+ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+ return REDIS_ERR;
+ }
+
+ val = interval;
+
+#ifdef _OSX
+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) {
+ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+ return REDIS_ERR;
+ }
+#else
+#if defined(__GLIBC__) && !defined(__FreeBSD_kernel__)
+ val = interval;
+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) {
+ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+ return REDIS_ERR;
+ }
+
+ val = interval/3;
+ if (val == 0) val = 1;
+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) {
+ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+ return REDIS_ERR;
+ }
+
+ val = 3;
+ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) {
+ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+ return REDIS_ERR;
+ }
+#endif
+#endif
+
+ return REDIS_OK;
+}
+
+static int redisSetTcpNoDelay(redisContext *c) {
+ int yes = 1;
+ if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)");
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+ return REDIS_OK;
+}
+
+#define __MAX_MSEC (((LONG_MAX) - 999) / 1000)
+
+static int redisContextWaitReady(redisContext *c, const struct timeval *timeout) {
+ struct pollfd wfd[1];
+ long msec;
+
+ msec = -1;
+ wfd[0].fd = c->fd;
+ wfd[0].events = POLLOUT;
+
+ /* Only use timeout when not NULL. */
+ if (timeout != NULL) {
+ if (timeout->tv_usec > 1000000 || timeout->tv_sec > __MAX_MSEC) {
+ __redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL);
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+
+ msec = (timeout->tv_sec * 1000) + ((timeout->tv_usec + 999) / 1000);
+
+ if (msec < 0 || msec > INT_MAX) {
+ msec = INT_MAX;
+ }
+ }
+
+ if (errno == EINPROGRESS) {
+ int res;
+
+ if ((res = poll(wfd, 1, msec)) == -1) {
+ __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)");
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ } else if (res == 0) {
+ errno = ETIMEDOUT;
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+ }
+
+ if (redisCheckSocketError(c) != REDIS_OK)
+ return REDIS_ERR;
+
+ return REDIS_OK;
+ }
+
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+ redisContextCloseFd(c);
+ return REDIS_ERR;
+}
+
+int redisCheckSocketError(redisContext *c) {
+ int err = 0;
+ socklen_t errlen = sizeof(err);
+
+ if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"getsockopt(SO_ERROR)");
+ return REDIS_ERR;
+ }
+
+ if (err) {
+ errno = err;
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+ return REDIS_ERR;
+ }
+
+ return REDIS_OK;
+}
+
+int redisContextSetTimeout(redisContext *c, const struct timeval tv) {
+ if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)");
+ return REDIS_ERR;
+ }
+ if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) {
+ __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)");
+ return REDIS_ERR;
+ }
+ return REDIS_OK;
+}
+
+static int _redisContextConnectTcp(redisContext *c, const char *addr, int port,
+ const struct timeval *timeout,
+ const char *source_addr) {
+ int s, rv, n;
+ char _port[6]; /* strlen("65535"); */
+ struct addrinfo hints, *servinfo, *bservinfo, *p, *b;
+ int blocking = (c->flags & REDIS_BLOCK);
+ int reuseaddr = (c->flags & REDIS_REUSEADDR);
+ int reuses = 0;
+
+ c->connection_type = REDIS_CONN_TCP;
+ c->tcp.port = port;
+
+ /* We need to take possession of the passed parameters
+ * to make them reusable for a reconnect.
+ * We also carefully check we don't free data we already own,
+ * as in the case of the reconnect method.
+ *
+ * This is a bit ugly, but atleast it works and doesn't leak memory.
+ **/
+ if (c->tcp.host != addr) {
+ if (c->tcp.host)
+ free(c->tcp.host);
+
+ c->tcp.host = strdup(addr);
+ }
+
+ if (timeout) {
+ if (c->timeout != timeout) {
+ if (c->timeout == NULL)
+ c->timeout = malloc(sizeof(struct timeval));
+
+ memcpy(c->timeout, timeout, sizeof(struct timeval));
+ }
+ } else {
+ if (c->timeout)
+ free(c->timeout);
+ c->timeout = NULL;
+ }
+
+ if (source_addr == NULL) {
+ free(c->tcp.source_addr);
+ c->tcp.source_addr = NULL;
+ } else if (c->tcp.source_addr != source_addr) {
+ free(c->tcp.source_addr);
+ c->tcp.source_addr = strdup(source_addr);
+ }
+
+ snprintf(_port, 6, "%d", port);
+ memset(&hints,0,sizeof(hints));
+ hints.ai_family = AF_INET;
+ hints.ai_socktype = SOCK_STREAM;
+ if (!blocking) {
+ /* Rspamd specific: never try to resolve on non-blocking conn requests */
+ hints.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV;
+ }
+
+ /* Try with IPv6 if no IPv4 address was found. We do it in this order since
+ * in a Redis client you can't afford to test if you have IPv6 connectivity
+ * as this would add latency to every connect. Otherwise a more sensible
+ * route could be: Use IPv6 if both addresses are available and there is IPv6
+ * connectivity. */
+ if ((rv = getaddrinfo(c->tcp.host,_port,&hints,&servinfo)) != 0) {
+ hints.ai_family = AF_INET6;
+ if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) {
+ __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv));
+ return REDIS_ERR;
+ }
+ }
+ for (p = servinfo; p != NULL; p = p->ai_next) {
+addrretry:
+ if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1)
+ continue;
+
+ c->fd = s;
+ if (redisSetBlocking(c,0) != REDIS_OK)
+ goto error;
+ if (c->tcp.source_addr) {
+ int bound = 0;
+ /* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */
+ if ((rv = getaddrinfo(c->tcp.source_addr, NULL, &hints, &bservinfo)) != 0) {
+ char buf[128];
+ snprintf(buf,sizeof(buf),"Can't get addr: %s",gai_strerror(rv));
+ __redisSetError(c,REDIS_ERR_OTHER,buf);
+ goto error;
+ }
+
+ if (reuseaddr) {
+ n = 1;
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char*) &n,
+ sizeof(n)) < 0) {
+ goto error;
+ }
+ }
+
+ for (b = bservinfo; b != NULL; b = b->ai_next) {
+ if (bind(s,b->ai_addr,b->ai_addrlen) != -1) {
+ bound = 1;
+ break;
+ }
+ }
+ freeaddrinfo(bservinfo);
+ if (!bound) {
+ char buf[128];
+ snprintf(buf,sizeof(buf),"Can't bind socket: %s",strerror(errno));
+ __redisSetError(c,REDIS_ERR_OTHER,buf);
+ goto error;
+ }
+ }
+ if (redisSetTcpNoDelay(c) != REDIS_OK)
+ goto error;
+ if (connect(s,p->ai_addr,p->ai_addrlen) == -1) {
+ if (errno == EHOSTUNREACH) {
+ redisContextCloseFd(c);
+ continue;
+ } else if (errno == EINPROGRESS && !blocking) {
+ /* This is ok. */
+ } else if (errno == EADDRNOTAVAIL && reuseaddr) {
+ if (++reuses >= REDIS_CONNECT_RETRIES) {
+ goto error;
+ } else {
+ goto addrretry;
+ }
+ } else {
+ if (redisContextWaitReady(c,c->timeout) != REDIS_OK)
+ goto error;
+ }
+ }
+
+ c->flags |= REDIS_CONNECTED;
+ rv = REDIS_OK;
+ goto end;
+ }
+ if (blocking && redisSetBlocking(c,1) != REDIS_OK)
+ goto error;
+ if (p == NULL) {
+ char buf[128];
+ snprintf(buf,sizeof(buf),"Can't create socket: %s",strerror(errno));
+ __redisSetError(c,REDIS_ERR_OTHER,buf);
+ goto error;
+ }
+
+error:
+ rv = REDIS_ERR;
+end:
+ freeaddrinfo(servinfo);
+ return rv; // Need to return REDIS_OK if alright
+}
+
+int redisContextConnectTcp(redisContext *c, const char *addr, int port,
+ const struct timeval *timeout) {
+ return _redisContextConnectTcp(c, addr, port, timeout, NULL);
+}
+
+int redisContextConnectBindTcp(redisContext *c, const char *addr, int port,
+ const struct timeval *timeout,
+ const char *source_addr) {
+ return _redisContextConnectTcp(c, addr, port, timeout, source_addr);
+}
+
+int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) {
+ int blocking = (c->flags & REDIS_BLOCK);
+ struct sockaddr_un sa;
+
+ if (redisCreateSocket(c,AF_LOCAL) < 0)
+ return REDIS_ERR;
+ if (redisSetBlocking(c,0) != REDIS_OK)
+ return REDIS_ERR;
+
+ c->connection_type = REDIS_CONN_UNIX;
+ if (c->unix_sock.path != path)
+ c->unix_sock.path = strdup(path);
+
+ if (timeout) {
+ if (c->timeout != timeout) {
+ if (c->timeout == NULL)
+ c->timeout = malloc(sizeof(struct timeval));
+
+ memcpy(c->timeout, timeout, sizeof(struct timeval));
+ }
+ } else {
+ if (c->timeout)
+ free(c->timeout);
+ c->timeout = NULL;
+ }
+
+ sa.sun_family = AF_LOCAL;
+ strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1);
+ if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) {
+ if (errno == EINPROGRESS && !blocking) {
+ /* This is ok. */
+ } else {
+ if (redisContextWaitReady(c,c->timeout) != REDIS_OK)
+ return REDIS_ERR;
+ }
+ }
+
+ /* Reset socket to be blocking after connect(2). */
+ if (blocking && redisSetBlocking(c,1) != REDIS_OK)
+ return REDIS_ERR;
+
+ c->flags |= REDIS_CONNECTED;
+ return REDIS_OK;
+}
diff --git a/contrib/hiredis/net.h b/contrib/hiredis/net.h
new file mode 100644
index 0000000..2f1a0bf
--- /dev/null
+++ b/contrib/hiredis/net.h
@@ -0,0 +1,53 @@
+/* Extracted from anet.c to work properly with Hiredis error reporting.
+ *
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ * Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __NET_H
+#define __NET_H
+
+#include "hiredis.h"
+
+#if defined(__sun)
+#define AF_LOCAL AF_UNIX
+#endif
+
+int redisCheckSocketError(redisContext *c);
+int redisContextSetTimeout(redisContext *c, const struct timeval tv);
+int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout);
+int redisContextConnectBindTcp(redisContext *c, const char *addr, int port,
+ const struct timeval *timeout,
+ const char *source_addr);
+int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout);
+int redisKeepAlive(redisContext *c, int interval);
+
+#endif
diff --git a/contrib/hiredis/read.c b/contrib/hiredis/read.c
new file mode 100644
index 0000000..df1a467
--- /dev/null
+++ b/contrib/hiredis/read.c
@@ -0,0 +1,525 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "fmacros.h"
+#include <string.h>
+#include <stdlib.h>
+#ifndef _MSC_VER
+#include <unistd.h>
+#endif
+#include <assert.h>
+#include <errno.h>
+#include <ctype.h>
+
+#include "read.h"
+#include "sds.h"
+
+static void __redisReaderSetError(redisReader *r, int type, const char *str) {
+ size_t len;
+
+ if (r->reply != NULL && r->fn && r->fn->freeObject) {
+ r->fn->freeObject(r->reply);
+ r->reply = NULL;
+ }
+
+ /* Clear input buffer on errors. */
+ if (r->buf != NULL) {
+ sdsfree(r->buf);
+ r->buf = NULL;
+ r->pos = r->len = 0;
+ }
+
+ /* Reset task stack. */
+ r->ridx = -1;
+
+ /* Set error. */
+ r->err = type;
+ len = strlen(str);
+ len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1);
+ memcpy(r->errstr,str,len);
+ r->errstr[len] = '\0';
+}
+
+static size_t chrtos(char *buf, size_t size, char byte) {
+ size_t len = 0;
+
+ switch(byte) {
+ case '\\':
+ case '"':
+ len = snprintf(buf,size,"\"\\%c\"",byte);
+ break;
+ case '\n': len = snprintf(buf,size,"\"\\n\""); break;
+ case '\r': len = snprintf(buf,size,"\"\\r\""); break;
+ case '\t': len = snprintf(buf,size,"\"\\t\""); break;
+ case '\a': len = snprintf(buf,size,"\"\\a\""); break;
+ case '\b': len = snprintf(buf,size,"\"\\b\""); break;
+ default:
+ if (isprint(byte))
+ len = snprintf(buf,size,"\"%c\"",byte);
+ else
+ len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte);
+ break;
+ }
+
+ return len;
+}
+
+static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) {
+ char cbuf[8], sbuf[128];
+
+ chrtos(cbuf,sizeof(cbuf),byte);
+ snprintf(sbuf,sizeof(sbuf),
+ "Protocol error, got %s as reply type byte", cbuf);
+ __redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf);
+}
+
+static void __redisReaderSetErrorOOM(redisReader *r) {
+ __redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory");
+}
+
+static char *readBytes(redisReader *r, unsigned int bytes) {
+ char *p;
+ if (r->len-r->pos >= bytes) {
+ p = r->buf+r->pos;
+ r->pos += bytes;
+ return p;
+ }
+ return NULL;
+}
+
+/* Find pointer to \r\n. */
+static char *seekNewline(char *s, size_t len) {
+ int pos = 0;
+ int _len = len-1;
+
+ /* Position should be < len-1 because the character at "pos" should be
+ * followed by a \n. Note that strchr cannot be used because it doesn't
+ * allow to search a limited length and the buffer that is being searched
+ * might not have a trailing NULL character. */
+ while (pos < _len) {
+ while(pos < _len && s[pos] != '\r') pos++;
+ if (s[pos] != '\r') {
+ /* Not found. */
+ return NULL;
+ } else {
+ if (s[pos+1] == '\n') {
+ /* Found. */
+ return s+pos;
+ } else {
+ /* Continue searching. */
+ pos++;
+ }
+ }
+ }
+ return NULL;
+}
+
+/* Read a long long value starting at *s, under the assumption that it will be
+ * terminated by \r\n. Ambiguously returns -1 for unexpected input. */
+static long long readLongLong(char *s) {
+ long long v = 0;
+ int dec, mult = 1;
+ char c;
+
+ if (*s == '-') {
+ mult = -1;
+ s++;
+ } else if (*s == '+') {
+ mult = 1;
+ s++;
+ }
+
+ while ((c = *(s++)) != '\r') {
+ dec = c - '0';
+ if (dec >= 0 && dec < 10) {
+ v *= 10;
+ v += dec;
+ } else {
+ /* Should not happen... */
+ return -1;
+ }
+ }
+
+ return mult*v;
+}
+
+static char *readLine(redisReader *r, int *_len) {
+ char *p, *s;
+ int len;
+
+ p = r->buf+r->pos;
+ s = seekNewline(p,(r->len-r->pos));
+ if (s != NULL) {
+ len = s-(r->buf+r->pos);
+ r->pos += len+2; /* skip \r\n */
+ if (_len) *_len = len;
+ return p;
+ }
+ return NULL;
+}
+
+static void moveToNextTask(redisReader *r) {
+ redisReadTask *cur, *prv;
+ while (r->ridx >= 0) {
+ /* Return a.s.a.p. when the stack is now empty. */
+ if (r->ridx == 0) {
+ r->ridx--;
+ return;
+ }
+
+ cur = &(r->rstack[r->ridx]);
+ prv = &(r->rstack[r->ridx-1]);
+ assert(prv->type == REDIS_REPLY_ARRAY);
+ if (cur->idx == prv->elements-1) {
+ r->ridx--;
+ } else {
+ /* Reset the type because the next item can be anything */
+ assert(cur->idx < prv->elements);
+ cur->type = -1;
+ cur->elements = -1;
+ cur->idx++;
+ return;
+ }
+ }
+}
+
+static int processLineItem(redisReader *r) {
+ redisReadTask *cur = &(r->rstack[r->ridx]);
+ void *obj;
+ char *p;
+ int len;
+
+ if ((p = readLine(r,&len)) != NULL) {
+ if (cur->type == REDIS_REPLY_INTEGER) {
+ if (r->fn && r->fn->createInteger)
+ obj = r->fn->createInteger(cur,readLongLong(p));
+ else
+ obj = (void*)REDIS_REPLY_INTEGER;
+ } else {
+ /* Type will be error or status. */
+ if (r->fn && r->fn->createString)
+ obj = r->fn->createString(cur,p,len);
+ else
+ obj = (void*)(size_t)(cur->type);
+ }
+
+ if (obj == NULL) {
+ __redisReaderSetErrorOOM(r);
+ return REDIS_ERR;
+ }
+
+ /* Set reply if this is the root object. */
+ if (r->ridx == 0) r->reply = obj;
+ moveToNextTask(r);
+ return REDIS_OK;
+ }
+
+ return REDIS_ERR;
+}
+
+static int processBulkItem(redisReader *r) {
+ redisReadTask *cur = &(r->rstack[r->ridx]);
+ void *obj = NULL;
+ char *p, *s;
+ long len;
+ unsigned long bytelen;
+ int success = 0;
+
+ p = r->buf+r->pos;
+ s = seekNewline(p,r->len-r->pos);
+ if (s != NULL) {
+ p = r->buf+r->pos;
+ bytelen = s-(r->buf+r->pos)+2; /* include \r\n */
+ len = readLongLong(p);
+
+ if (len < 0) {
+ /* The nil object can always be created. */
+ if (r->fn && r->fn->createNil)
+ obj = r->fn->createNil(cur);
+ else
+ obj = (void*)REDIS_REPLY_NIL;
+ success = 1;
+ } else {
+ /* Only continue when the buffer contains the entire bulk item. */
+ bytelen += len+2; /* include \r\n */
+ if (r->pos+bytelen <= r->len) {
+ if (r->fn && r->fn->createString)
+ obj = r->fn->createString(cur,s+2,len);
+ else
+ obj = (void*)REDIS_REPLY_STRING;
+ success = 1;
+ }
+ }
+
+ /* Proceed when obj was created. */
+ if (success) {
+ if (obj == NULL) {
+ __redisReaderSetErrorOOM(r);
+ return REDIS_ERR;
+ }
+
+ r->pos += bytelen;
+
+ /* Set reply if this is the root object. */
+ if (r->ridx == 0) r->reply = obj;
+ moveToNextTask(r);
+ return REDIS_OK;
+ }
+ }
+
+ return REDIS_ERR;
+}
+
+static int processMultiBulkItem(redisReader *r) {
+ redisReadTask *cur = &(r->rstack[r->ridx]);
+ void *obj;
+ char *p;
+ long elements;
+ int root = 0;
+
+ /* Set error for nested multi bulks with depth > 7 */
+ if (r->ridx == 8) {
+ __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
+ "No support for nested multi bulk replies with depth > 7");
+ return REDIS_ERR;
+ }
+
+ if ((p = readLine(r,NULL)) != NULL) {
+ elements = readLongLong(p);
+ root = (r->ridx == 0);
+
+ if (elements == -1) {
+ if (r->fn && r->fn->createNil)
+ obj = r->fn->createNil(cur);
+ else
+ obj = (void*)REDIS_REPLY_NIL;
+
+ if (obj == NULL) {
+ __redisReaderSetErrorOOM(r);
+ return REDIS_ERR;
+ }
+
+ moveToNextTask(r);
+ } else {
+ if (r->fn && r->fn->createArray)
+ obj = r->fn->createArray(cur,elements);
+ else
+ obj = (void*)REDIS_REPLY_ARRAY;
+
+ if (obj == NULL) {
+ __redisReaderSetErrorOOM(r);
+ return REDIS_ERR;
+ }
+
+ /* Modify task stack when there are more than 0 elements. */
+ if (elements > 0) {
+ cur->elements = elements;
+ cur->obj = obj;
+ r->ridx++;
+ r->rstack[r->ridx].type = -1;
+ r->rstack[r->ridx].elements = -1;
+ r->rstack[r->ridx].idx = 0;
+ r->rstack[r->ridx].obj = NULL;
+ r->rstack[r->ridx].parent = cur;
+ r->rstack[r->ridx].privdata = r->privdata;
+ } else {
+ moveToNextTask(r);
+ }
+ }
+
+ /* Set reply if this is the root object. */
+ if (root) r->reply = obj;
+ return REDIS_OK;
+ }
+
+ return REDIS_ERR;
+}
+
+static int processItem(redisReader *r) {
+ redisReadTask *cur = &(r->rstack[r->ridx]);
+ char *p;
+
+ /* check if we need to read type */
+ if (cur->type < 0) {
+ if ((p = readBytes(r,1)) != NULL) {
+ switch (p[0]) {
+ case '-':
+ cur->type = REDIS_REPLY_ERROR;
+ break;
+ case '+':
+ cur->type = REDIS_REPLY_STATUS;
+ break;
+ case ':':
+ cur->type = REDIS_REPLY_INTEGER;
+ break;
+ case '$':
+ cur->type = REDIS_REPLY_STRING;
+ break;
+ case '*':
+ cur->type = REDIS_REPLY_ARRAY;
+ break;
+ default:
+ __redisReaderSetErrorProtocolByte(r,*p);
+ return REDIS_ERR;
+ }
+ } else {
+ /* could not consume 1 byte */
+ return REDIS_ERR;
+ }
+ }
+
+ /* process typed item */
+ switch(cur->type) {
+ case REDIS_REPLY_ERROR:
+ case REDIS_REPLY_STATUS:
+ case REDIS_REPLY_INTEGER:
+ return processLineItem(r);
+ case REDIS_REPLY_STRING:
+ return processBulkItem(r);
+ case REDIS_REPLY_ARRAY:
+ return processMultiBulkItem(r);
+ default:
+ assert(NULL);
+ return REDIS_ERR; /* Avoid warning. */
+ }
+}
+
+redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) {
+ redisReader *r;
+
+ r = calloc(sizeof(redisReader),1);
+ if (r == NULL)
+ return NULL;
+
+ r->err = 0;
+ r->errstr[0] = '\0';
+ r->fn = fn;
+ r->buf = sdsempty();
+ r->maxbuf = REDIS_READER_MAX_BUF;
+ if (r->buf == NULL) {
+ free(r);
+ return NULL;
+ }
+
+ r->ridx = -1;
+ return r;
+}
+
+void redisReaderFree(redisReader *r) {
+ if (r->reply != NULL && r->fn && r->fn->freeObject)
+ r->fn->freeObject(r->reply);
+ if (r->buf != NULL)
+ sdsfree(r->buf);
+ free(r);
+}
+
+int redisReaderFeed(redisReader *r, const char *buf, size_t len) {
+ sds newbuf;
+
+ /* Return early when this reader is in an erroneous state. */
+ if (r->err)
+ return REDIS_ERR;
+
+ /* Copy the provided buffer. */
+ if (buf != NULL && len >= 1) {
+ /* Destroy internal buffer when it is empty and is quite large. */
+ if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) {
+ sdsfree(r->buf);
+ r->buf = sdsempty();
+ r->pos = 0;
+
+ /* r->buf should not be NULL since we just free'd a larger one. */
+ assert(r->buf != NULL);
+ }
+
+ newbuf = sdscatlen(r->buf,buf,len);
+ if (newbuf == NULL) {
+ __redisReaderSetErrorOOM(r);
+ return REDIS_ERR;
+ }
+
+ r->buf = newbuf;
+ r->len = sdslen(r->buf);
+ }
+
+ return REDIS_OK;
+}
+
+int redisReaderGetReply(redisReader *r, void **reply) {
+ /* Default target pointer to NULL. */
+ if (reply != NULL)
+ *reply = NULL;
+
+ /* Return early when this reader is in an erroneous state. */
+ if (r->err)
+ return REDIS_ERR;
+
+ /* When the buffer is empty, there will never be a reply. */
+ if (r->len == 0)
+ return REDIS_OK;
+
+ /* Set first item to process when the stack is empty. */
+ if (r->ridx == -1) {
+ r->rstack[0].type = -1;
+ r->rstack[0].elements = -1;
+ r->rstack[0].idx = -1;
+ r->rstack[0].obj = NULL;
+ r->rstack[0].parent = NULL;
+ r->rstack[0].privdata = r->privdata;
+ r->ridx = 0;
+ }
+
+ /* Process items in reply. */
+ while (r->ridx >= 0)
+ if (processItem(r) != REDIS_OK)
+ break;
+
+ /* Return ASAP when an error occurred. */
+ if (r->err)
+ return REDIS_ERR;
+
+ /* Discard part of the buffer when we've consumed at least 1k, to avoid
+ * doing unnecessary calls to memmove() in sds.c. */
+ if (r->pos >= 1024) {
+ sdsrange(r->buf,r->pos,-1);
+ r->pos = 0;
+ r->len = sdslen(r->buf);
+ }
+
+ /* Emit a reply when there is one. */
+ if (r->ridx == -1) {
+ if (reply != NULL)
+ *reply = r->reply;
+ r->reply = NULL;
+ }
+ return REDIS_OK;
+}
diff --git a/contrib/hiredis/read.h b/contrib/hiredis/read.h
new file mode 100644
index 0000000..180e6c6
--- /dev/null
+++ b/contrib/hiredis/read.h
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifndef __HIREDIS_READ_H
+#define __HIREDIS_READ_H
+#include <stdio.h> /* for size_t */
+
+#define REDIS_ERR -1
+#define REDIS_OK 0
+
+/* When an error occurs, the err flag in a context is set to hold the type of
+ * error that occurred. REDIS_ERR_IO means there was an I/O error and you
+ * should use the "errno" variable to find out what is wrong.
+ * For other values, the "errstr" field will hold a description. */
+#define REDIS_ERR_IO 1 /* Error in read or write */
+#define REDIS_ERR_EOF 3 /* End of file */
+#define REDIS_ERR_PROTOCOL 4 /* Protocol error */
+#define REDIS_ERR_OOM 5 /* Out of memory */
+#define REDIS_ERR_OTHER 2 /* Everything else... */
+
+#define REDIS_REPLY_STRING 1
+#define REDIS_REPLY_ARRAY 2
+#define REDIS_REPLY_INTEGER 3
+#define REDIS_REPLY_NIL 4
+#define REDIS_REPLY_STATUS 5
+#define REDIS_REPLY_ERROR 6
+
+#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct redisReadTask {
+ int type;
+ int elements; /* number of elements in multibulk container */
+ int idx; /* index in parent (array) object */
+ void *obj; /* holds user-generated value for a read task */
+ struct redisReadTask *parent; /* parent task */
+ void *privdata; /* user-settable arbitrary field */
+} redisReadTask;
+
+typedef struct redisReplyObjectFunctions {
+ void *(*createString)(const redisReadTask*, char*, size_t);
+ void *(*createArray)(const redisReadTask*, int);
+ void *(*createInteger)(const redisReadTask*, long long);
+ void *(*createNil)(const redisReadTask*);
+ void (*freeObject)(void*);
+} redisReplyObjectFunctions;
+
+typedef struct redisReader {
+ int err; /* Error flags, 0 when there is no error */
+ char errstr[128]; /* String representation of error when applicable */
+
+ char *buf; /* Read buffer */
+ size_t pos; /* Buffer cursor */
+ size_t len; /* Buffer length */
+ size_t maxbuf; /* Max length of unused buffer */
+
+ redisReadTask rstack[9];
+ int ridx; /* Index of current read task */
+ void *reply; /* Temporary reply pointer */
+
+ redisReplyObjectFunctions *fn;
+ void *privdata;
+} redisReader;
+
+/* Public API for the protocol parser. */
+redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn);
+void redisReaderFree(redisReader *r);
+int redisReaderFeed(redisReader *r, const char *buf, size_t len);
+int redisReaderGetReply(redisReader *r, void **reply);
+
+/* Backwards compatibility, can be removed on big version bump. */
+#define redisReplyReaderCreate redisReaderCreate
+#define redisReplyReaderFree redisReaderFree
+#define redisReplyReaderFeed redisReaderFeed
+#define redisReplyReaderGetReply redisReaderGetReply
+#define redisReplyReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p))
+#define redisReplyReaderGetObject(_r) (((redisReader*)(_r))->reply)
+#define redisReplyReaderGetError(_r) (((redisReader*)(_r))->errstr)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/hiredis/sds.c b/contrib/hiredis/sds.c
new file mode 100644
index 0000000..5e75516
--- /dev/null
+++ b/contrib/hiredis/sds.c
@@ -0,0 +1,1095 @@
+/* SDS (Simple Dynamic Strings), A C dynamic strings library.
+ *
+ * Copyright (c) 2006-2014, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <assert.h>
+
+#include "sds.h"
+
+/* Create a new sds string with the content specified by the 'init' pointer
+ * and 'initlen'.
+ * If NULL is used for 'init' the string is initialized with zero bytes.
+ *
+ * The string is always null-termined (all the sds strings are, always) so
+ * even if you create an sds string with:
+ *
+ * mystring = sdsnewlen("abc",3");
+ *
+ * You can print the string with printf() as there is an implicit \0 at the
+ * end of the string. However the string is binary safe and can contain
+ * \0 characters in the middle, as the length is stored in the sds header. */
+sds sdsnewlen(const void *init, size_t initlen) {
+ struct sdshdr *sh;
+
+ if (init) {
+ sh = malloc(sizeof *sh+initlen+1);
+ } else {
+ sh = calloc(sizeof *sh+initlen+1,1);
+ }
+ if (sh == NULL) return NULL;
+ sh->len = initlen;
+ sh->free = 0;
+ if (initlen && init)
+ memcpy(sh->buf, init, initlen);
+ sh->buf[initlen] = '\0';
+ return (char*)sh->buf;
+}
+
+/* Create an empty (zero length) sds string. Even in this case the string
+ * always has an implicit null term. */
+sds sdsempty(void) {
+ return sdsnewlen("",0);
+}
+
+/* Create a new sds string starting from a null termined C string. */
+sds sdsnew(const char *init) {
+ size_t initlen = (init == NULL) ? 0 : strlen(init);
+ return sdsnewlen(init, initlen);
+}
+
+/* Duplicate an sds string. */
+sds sdsdup(const sds s) {
+ return sdsnewlen(s, sdslen(s));
+}
+
+/* Free an sds string. No operation is performed if 's' is NULL. */
+void sdsfree(sds s) {
+ if (s == NULL) return;
+ free(s-sizeof(struct sdshdr));
+}
+
+/* Set the sds string length to the length as obtained with strlen(), so
+ * considering as content only up to the first null term character.
+ *
+ * This function is useful when the sds string is hacked manually in some
+ * way, like in the following example:
+ *
+ * s = sdsnew("foobar");
+ * s[2] = '\0';
+ * sdsupdatelen(s);
+ * printf("%d\n", sdslen(s));
+ *
+ * The output will be "2", but if we comment out the call to sdsupdatelen()
+ * the output will be "6" as the string was modified but the logical length
+ * remains 6 bytes. */
+void sdsupdatelen(sds s) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ int reallen = strlen(s);
+ sh->free += (sh->len-reallen);
+ sh->len = reallen;
+}
+
+/* Modify an sds string on-place to make it empty (zero length).
+ * However all the existing buffer is not discarded but set as free space
+ * so that next append operations will not require allocations up to the
+ * number of bytes previously available. */
+void sdsclear(sds s) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ sh->free += sh->len;
+ sh->len = 0;
+ sh->buf[0] = '\0';
+}
+
+/* Enlarge the free space at the end of the sds string so that the caller
+ * is sure that after calling this function can overwrite up to addlen
+ * bytes after the end of the string, plus one more byte for nul term.
+ *
+ * Note: this does not change the *length* of the sds string as returned
+ * by sdslen(), but only the free buffer space we have. */
+sds sdsMakeRoomFor(sds s, size_t addlen) {
+ struct sdshdr *sh, *newsh;
+ size_t free = sdsavail(s);
+ size_t len, newlen;
+
+ if (free >= addlen) return s;
+ len = sdslen(s);
+ sh = (void*) (s-sizeof *sh);
+ newlen = (len+addlen);
+ if (newlen < SDS_MAX_PREALLOC)
+ newlen *= 2;
+ else
+ newlen += SDS_MAX_PREALLOC;
+ newsh = realloc(sh, sizeof *newsh+newlen+1);
+ if (newsh == NULL) return NULL;
+
+ newsh->free = newlen - len;
+ return newsh->buf;
+}
+
+/* Reallocate the sds string so that it has no free space at the end. The
+ * contained string remains not altered, but next concatenation operations
+ * will require a reallocation.
+ *
+ * After the call, the passed sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdsRemoveFreeSpace(sds s) {
+ struct sdshdr *sh;
+
+ sh = (void*) (s-sizeof *sh);
+ sh = realloc(sh, sizeof *sh+sh->len+1);
+ sh->free = 0;
+ return sh->buf;
+}
+
+/* Return the total size of the allocation of the specified sds string,
+ * including:
+ * 1) The sds header before the pointer.
+ * 2) The string.
+ * 3) The free buffer at the end if any.
+ * 4) The implicit null term.
+ */
+size_t sdsAllocSize(sds s) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+
+ return sizeof(*sh)+sh->len+sh->free+1;
+}
+
+/* Increment the sds length and decrements the left free space at the
+ * end of the string according to 'incr'. Also set the null term
+ * in the new end of the string.
+ *
+ * This function is used in order to fix the string length after the
+ * user calls sdsMakeRoomFor(), writes something after the end of
+ * the current string, and finally needs to set the new length.
+ *
+ * Note: it is possible to use a negative increment in order to
+ * right-trim the string.
+ *
+ * Usage example:
+ *
+ * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the
+ * following schema, to cat bytes coming from the kernel to the end of an
+ * sds string without copying into an intermediate buffer:
+ *
+ * oldlen = sdslen(s);
+ * s = sdsMakeRoomFor(s, BUFFER_SIZE);
+ * nread = read(fd, s+oldlen, BUFFER_SIZE);
+ * ... check for nread <= 0 and handle it ...
+ * sdsIncrLen(s, nread);
+ */
+void sdsIncrLen(sds s, int incr) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+
+ assert(sh->free >= incr);
+ sh->len += incr;
+ sh->free -= incr;
+ assert(sh->free >= 0);
+ s[sh->len] = '\0';
+}
+
+/* Grow the sds to have the specified length. Bytes that were not part of
+ * the original length of the sds will be set to zero.
+ *
+ * if the specified length is smaller than the current length, no operation
+ * is performed. */
+sds sdsgrowzero(sds s, size_t len) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ size_t totlen, curlen = sh->len;
+
+ if (len <= curlen) return s;
+ s = sdsMakeRoomFor(s,len-curlen);
+ if (s == NULL) return NULL;
+
+ /* Make sure added region doesn't contain garbage */
+ sh = (void*)(s-sizeof *sh);
+ memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */
+ totlen = sh->len+sh->free;
+ sh->len = len;
+ sh->free = totlen-sh->len;
+ return s;
+}
+
+/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the
+ * end of the specified sds string 's'.
+ *
+ * After the call, the passed sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscatlen(sds s, const void *t, size_t len) {
+ struct sdshdr *sh;
+ size_t curlen = sdslen(s);
+
+ s = sdsMakeRoomFor(s,len);
+ if (s == NULL) return NULL;
+ sh = (void*) (s-sizeof *sh);
+ memcpy(s+curlen, t, len);
+ sh->len = curlen+len;
+ sh->free = sh->free-len;
+ s[curlen+len] = '\0';
+ return s;
+}
+
+/* Append the specified null termianted C string to the sds string 's'.
+ *
+ * After the call, the passed sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscat(sds s, const char *t) {
+ return sdscatlen(s, t, strlen(t));
+}
+
+/* Append the specified sds 't' to the existing sds 's'.
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscatsds(sds s, const sds t) {
+ return sdscatlen(s, t, sdslen(t));
+}
+
+/* Destructively modify the sds string 's' to hold the specified binary
+ * safe string pointed by 't' of length 'len' bytes. */
+sds sdscpylen(sds s, const char *t, size_t len) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ size_t totlen = sh->free+sh->len;
+
+ if (totlen < len) {
+ s = sdsMakeRoomFor(s,len-sh->len);
+ if (s == NULL) return NULL;
+ sh = (void*) (s-sizeof *sh);
+ totlen = sh->free+sh->len;
+ }
+ memcpy(s, t, len);
+ s[len] = '\0';
+ sh->len = len;
+ sh->free = totlen-len;
+ return s;
+}
+
+/* Like sdscpylen() but 't' must be a null-termined string so that the length
+ * of the string is obtained with strlen(). */
+sds sdscpy(sds s, const char *t) {
+ return sdscpylen(s, t, strlen(t));
+}
+
+/* Helper for sdscatlonglong() doing the actual number -> string
+ * conversion. 's' must point to a string with room for at least
+ * SDS_LLSTR_SIZE bytes.
+ *
+ * The function returns the length of the null-terminated string
+ * representation stored at 's'. */
+#define SDS_LLSTR_SIZE 21
+int sdsll2str(char *s, long long value) {
+ char *p, aux;
+ unsigned long long v;
+ size_t l;
+
+ /* Generate the string representation, this method produces
+ * an reversed string. */
+ v = (value < 0) ? -value : value;
+ p = s;
+ do {
+ *p++ = '0'+(v%10);
+ v /= 10;
+ } while(v);
+ if (value < 0) *p++ = '-';
+
+ /* Compute length and add null term. */
+ l = p-s;
+ *p = '\0';
+
+ /* Reverse the string. */
+ p--;
+ while(s < p) {
+ aux = *s;
+ *s = *p;
+ *p = aux;
+ s++;
+ p--;
+ }
+ return l;
+}
+
+/* Identical sdsll2str(), but for unsigned long long type. */
+int sdsull2str(char *s, unsigned long long v) {
+ char *p, aux;
+ size_t l;
+
+ /* Generate the string representation, this method produces
+ * an reversed string. */
+ p = s;
+ do {
+ *p++ = '0'+(v%10);
+ v /= 10;
+ } while(v);
+
+ /* Compute length and add null term. */
+ l = p-s;
+ *p = '\0';
+
+ /* Reverse the string. */
+ p--;
+ while(s < p) {
+ aux = *s;
+ *s = *p;
+ *p = aux;
+ s++;
+ p--;
+ }
+ return l;
+}
+
+/* Like sdscatpritf() but gets va_list instead of being variadic. */
+sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
+ va_list cpy;
+ char *buf, *t;
+ size_t buflen = 16;
+
+ while(1) {
+ buf = malloc(buflen);
+ if (buf == NULL) return NULL;
+ buf[buflen-2] = '\0';
+ va_copy(cpy,ap);
+ vsnprintf(buf, buflen, fmt, cpy);
+ if (buf[buflen-2] != '\0') {
+ free(buf);
+ buflen *= 2;
+ continue;
+ }
+ break;
+ }
+ t = sdscat(s, buf);
+ free(buf);
+ return t;
+}
+
+/* Append to the sds string 's' a string obtained using printf-alike format
+ * specifier.
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call.
+ *
+ * Example:
+ *
+ * s = sdsnew("Sum is: ");
+ * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b);
+ *
+ * Often you need to create a string from scratch with the printf-alike
+ * format. When this is the need, just use sdsempty() as the target string:
+ *
+ * s = sdscatprintf(sdsempty(), "... your format ...", args);
+ */
+sds sdscatprintf(sds s, const char *fmt, ...) {
+ va_list ap;
+ char *t;
+ va_start(ap, fmt);
+ t = sdscatvprintf(s,fmt,ap);
+ va_end(ap);
+ return t;
+}
+
+/* This function is similar to sdscatprintf, but much faster as it does
+ * not rely on sprintf() family functions implemented by the libc that
+ * are often very slow. Moreover directly handling the sds string as
+ * new data is concatenated provides a performance improvement.
+ *
+ * However this function only handles an incompatible subset of printf-alike
+ * format specifiers:
+ *
+ * %s - C String
+ * %S - SDS string
+ * %i - signed int
+ * %I - 64 bit signed integer (long long, int64_t)
+ * %u - unsigned int
+ * %U - 64 bit unsigned integer (unsigned long long, uint64_t)
+ * %T - A size_t variable.
+ * %% - Verbatim "%" character.
+ */
+sds sdscatfmt(sds s, char const *fmt, ...) {
+ struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr)));
+ size_t initlen = sdslen(s);
+ const char *f = fmt;
+ int i;
+ va_list ap;
+
+ va_start(ap,fmt);
+ f = fmt; /* Next format specifier byte to process. */
+ i = initlen; /* Position of the next byte to write to dest str. */
+ while(*f) {
+ char next, *str;
+ int l;
+ long long num;
+ unsigned long long unum;
+
+ /* Make sure there is always space for at least 1 char. */
+ if (sh->free == 0) {
+ s = sdsMakeRoomFor(s,1);
+ sh = (void*) (s-(sizeof(struct sdshdr)));
+ }
+
+ switch(*f) {
+ case '%':
+ next = *(f+1);
+ f++;
+ switch(next) {
+ case 's':
+ case 'S':
+ str = va_arg(ap,char*);
+ l = (next == 's') ? strlen(str) : sdslen(str);
+ if (sh->free < l) {
+ s = sdsMakeRoomFor(s,l);
+ sh = (void*) (s-(sizeof(struct sdshdr)));
+ }
+ memcpy(s+i,str,l);
+ sh->len += l;
+ sh->free -= l;
+ i += l;
+ break;
+ case 'i':
+ case 'I':
+ if (next == 'i')
+ num = va_arg(ap,int);
+ else
+ num = va_arg(ap,long long);
+ {
+ char buf[SDS_LLSTR_SIZE];
+ l = sdsll2str(buf,num);
+ if (sh->free < l) {
+ s = sdsMakeRoomFor(s,l);
+ sh = (void*) (s-(sizeof(struct sdshdr)));
+ }
+ memcpy(s+i,buf,l);
+ sh->len += l;
+ sh->free -= l;
+ i += l;
+ }
+ break;
+ case 'u':
+ case 'U':
+ case 'T':
+ if (next == 'u')
+ unum = va_arg(ap,unsigned int);
+ else if(next == 'U')
+ unum = va_arg(ap,unsigned long long);
+ else
+ unum = (unsigned long long)va_arg(ap,size_t);
+ {
+ char buf[SDS_LLSTR_SIZE];
+ l = sdsull2str(buf,unum);
+ if (sh->free < l) {
+ s = sdsMakeRoomFor(s,l);
+ sh = (void*) (s-(sizeof(struct sdshdr)));
+ }
+ memcpy(s+i,buf,l);
+ sh->len += l;
+ sh->free -= l;
+ i += l;
+ }
+ break;
+ default: /* Handle %% and generally %<unknown>. */
+ s[i++] = next;
+ sh->len += 1;
+ sh->free -= 1;
+ break;
+ }
+ break;
+ default:
+ s[i++] = *f;
+ sh->len += 1;
+ sh->free -= 1;
+ break;
+ }
+ f++;
+ }
+ va_end(ap);
+
+ /* Add null-term */
+ s[i] = '\0';
+ return s;
+}
+
+
+/* Remove the part of the string from left and from right composed just of
+ * contiguous characters found in 'cset', that is a null terminted C string.
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call.
+ *
+ * Example:
+ *
+ * s = sdsnew("AA...AA.a.aa.aHelloWorld :::");
+ * s = sdstrim(s,"A. :");
+ * printf("%s\n", s);
+ *
+ * Output will be just "Hello World".
+ */
+void sdstrim(sds s, const char *cset) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ char *start, *end, *sp, *ep;
+ size_t len;
+
+ sp = start = s;
+ ep = end = s+sdslen(s)-1;
+ while(sp <= end && strchr(cset, *sp)) sp++;
+ while(ep > start && strchr(cset, *ep)) ep--;
+ len = (sp > ep) ? 0 : ((ep-sp)+1);
+ if (sh->buf != sp) memmove(sh->buf, sp, len);
+ sh->buf[len] = '\0';
+ sh->free = sh->free+(sh->len-len);
+ sh->len = len;
+}
+
+/* Turn the string into a smaller (or equal) string containing only the
+ * substring specified by the 'start' and 'end' indexes.
+ *
+ * start and end can be negative, where -1 means the last character of the
+ * string, -2 the penultimate character, and so forth.
+ *
+ * The interval is inclusive, so the start and end characters will be part
+ * of the resulting string.
+ *
+ * The string is modified in-place.
+ *
+ * Example:
+ *
+ * s = sdsnew("Hello World");
+ * sdsrange(s,1,-1); => "ello World"
+ */
+void sdsrange(sds s, int start, int end) {
+ struct sdshdr *sh = (void*) (s-sizeof *sh);
+ size_t newlen, len = sdslen(s);
+
+ if (len == 0) return;
+ if (start < 0) {
+ start = len+start;
+ if (start < 0) start = 0;
+ }
+ if (end < 0) {
+ end = len+end;
+ if (end < 0) end = 0;
+ }
+ newlen = (start > end) ? 0 : (end-start)+1;
+ if (newlen != 0) {
+ if (start >= (signed)len) {
+ newlen = 0;
+ } else if (end >= (signed)len) {
+ end = len-1;
+ newlen = (start > end) ? 0 : (end-start)+1;
+ }
+ } else {
+ start = 0;
+ }
+ if (start && newlen) memmove(sh->buf, sh->buf+start, newlen);
+ sh->buf[newlen] = 0;
+ sh->free = sh->free+(sh->len-newlen);
+ sh->len = newlen;
+}
+
+/* Apply tolower() to every character of the sds string 's'. */
+void sdstolower(sds s) {
+ int len = sdslen(s), j;
+
+ for (j = 0; j < len; j++) s[j] = tolower(s[j]);
+}
+
+/* Apply toupper() to every character of the sds string 's'. */
+void sdstoupper(sds s) {
+ int len = sdslen(s), j;
+
+ for (j = 0; j < len; j++) s[j] = toupper(s[j]);
+}
+
+/* Compare two sds strings s1 and s2 with memcmp().
+ *
+ * Return value:
+ *
+ * 1 if s1 > s2.
+ * -1 if s1 < s2.
+ * 0 if s1 and s2 are exactly the same binary string.
+ *
+ * If two strings share exactly the same prefix, but one of the two has
+ * additional characters, the longer string is considered to be greater than
+ * the smaller one. */
+int sdscmp(const sds s1, const sds s2) {
+ size_t l1, l2, minlen;
+ int cmp;
+
+ l1 = sdslen(s1);
+ l2 = sdslen(s2);
+ minlen = (l1 < l2) ? l1 : l2;
+ cmp = memcmp(s1,s2,minlen);
+ if (cmp == 0) return l1-l2;
+ return cmp;
+}
+
+/* Split 's' with separator in 'sep'. An array
+ * of sds strings is returned. *count will be set
+ * by reference to the number of tokens returned.
+ *
+ * On out of memory, zero length string, zero length
+ * separator, NULL is returned.
+ *
+ * Note that 'sep' is able to split a string using
+ * a multi-character separator. For example
+ * sdssplit("foo_-_bar","_-_"); will return two
+ * elements "foo" and "bar".
+ *
+ * This version of the function is binary-safe but
+ * requires length arguments. sdssplit() is just the
+ * same function but for zero-terminated strings.
+ */
+sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) {
+ int elements = 0, slots = 5, start = 0, j;
+ sds *tokens;
+
+ if (seplen < 1 || len < 0) return NULL;
+
+ tokens = malloc(sizeof(sds)*slots);
+ if (tokens == NULL) return NULL;
+
+ if (len == 0) {
+ *count = 0;
+ return tokens;
+ }
+ for (j = 0; j < (len-(seplen-1)); j++) {
+ /* make sure there is room for the next element and the final one */
+ if (slots < elements+2) {
+ sds *newtokens;
+
+ slots *= 2;
+ newtokens = realloc(tokens,sizeof(sds)*slots);
+ if (newtokens == NULL) goto cleanup;
+ tokens = newtokens;
+ }
+ /* search the separator */
+ if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) {
+ tokens[elements] = sdsnewlen(s+start,j-start);
+ if (tokens[elements] == NULL) goto cleanup;
+ elements++;
+ start = j+seplen;
+ j = j+seplen-1; /* skip the separator */
+ }
+ }
+ /* Add the final element. We are sure there is room in the tokens array. */
+ tokens[elements] = sdsnewlen(s+start,len-start);
+ if (tokens[elements] == NULL) goto cleanup;
+ elements++;
+ *count = elements;
+ return tokens;
+
+cleanup:
+ {
+ int i;
+ for (i = 0; i < elements; i++) sdsfree(tokens[i]);
+ free(tokens);
+ *count = 0;
+ return NULL;
+ }
+}
+
+/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */
+void sdsfreesplitres(sds *tokens, int count) {
+ if (!tokens) return;
+ while(count--)
+ sdsfree(tokens[count]);
+ free(tokens);
+}
+
+/* Create an sds string from a long long value. It is much faster than:
+ *
+ * sdscatprintf(sdsempty(),"%lld\n", value);
+ */
+sds sdsfromlonglong(long long value) {
+ char buf[32], *p;
+ unsigned long long v;
+
+ v = (value < 0) ? -value : value;
+ p = buf+31; /* point to the last character */
+ do {
+ *p-- = '0'+(v%10);
+ v /= 10;
+ } while(v);
+ if (value < 0) *p-- = '-';
+ p++;
+ return sdsnewlen(p,32-(p-buf));
+}
+
+/* Append to the sds string "s" an escaped string representation where
+ * all the non-printable characters (tested with isprint()) are turned into
+ * escapes in the form "\n\r\a...." or "\x<hex-number>".
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscatrepr(sds s, const char *p, size_t len) {
+ s = sdscatlen(s,"\"",1);
+ while(len--) {
+ switch(*p) {
+ case '\\':
+ case '"':
+ s = sdscatprintf(s,"\\%c",*p);
+ break;
+ case '\n': s = sdscatlen(s,"\\n",2); break;
+ case '\r': s = sdscatlen(s,"\\r",2); break;
+ case '\t': s = sdscatlen(s,"\\t",2); break;
+ case '\a': s = sdscatlen(s,"\\a",2); break;
+ case '\b': s = sdscatlen(s,"\\b",2); break;
+ default:
+ if (isprint(*p))
+ s = sdscatprintf(s,"%c",*p);
+ else
+ s = sdscatprintf(s,"\\x%02x",(unsigned char)*p);
+ break;
+ }
+ p++;
+ }
+ return sdscatlen(s,"\"",1);
+}
+
+/* Helper function for sdssplitargs() that returns non zero if 'c'
+ * is a valid hex digit. */
+int is_hex_digit(char c) {
+ return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
+ (c >= 'A' && c <= 'F');
+}
+
+/* Helper function for sdssplitargs() that converts a hex digit into an
+ * integer from 0 to 15 */
+int hex_digit_to_int(char c) {
+ switch(c) {
+ case '0': return 0;
+ case '1': return 1;
+ case '2': return 2;
+ case '3': return 3;
+ case '4': return 4;
+ case '5': return 5;
+ case '6': return 6;
+ case '7': return 7;
+ case '8': return 8;
+ case '9': return 9;
+ case 'a': case 'A': return 10;
+ case 'b': case 'B': return 11;
+ case 'c': case 'C': return 12;
+ case 'd': case 'D': return 13;
+ case 'e': case 'E': return 14;
+ case 'f': case 'F': return 15;
+ default: return 0;
+ }
+}
+
+/* Split a line into arguments, where every argument can be in the
+ * following programming-language REPL-alike form:
+ *
+ * foo bar "newline are supported\n" and "\xff\x00otherstuff"
+ *
+ * The number of arguments is stored into *argc, and an array
+ * of sds is returned.
+ *
+ * The caller should free the resulting array of sds strings with
+ * sdsfreesplitres().
+ *
+ * Note that sdscatrepr() is able to convert back a string into
+ * a quoted string in the same format sdssplitargs() is able to parse.
+ *
+ * The function returns the allocated tokens on success, even when the
+ * input string is empty, or NULL if the input contains unbalanced
+ * quotes or closed quotes followed by non space characters
+ * as in: "foo"bar or "foo'
+ */
+sds *sdssplitargs(const char *line, int *argc) {
+ const char *p = line;
+ char *current = NULL;
+ char **vector = NULL;
+
+ *argc = 0;
+ while(1) {
+ /* skip blanks */
+ while(*p && isspace(*p)) p++;
+ if (*p) {
+ /* get a token */
+ int inq=0; /* set to 1 if we are in "quotes" */
+ int insq=0; /* set to 1 if we are in 'single quotes' */
+ int done=0;
+
+ if (current == NULL) current = sdsempty();
+ while(!done) {
+ if (inq) {
+ if (*p == '\\' && *(p+1) == 'x' &&
+ is_hex_digit(*(p+2)) &&
+ is_hex_digit(*(p+3)))
+ {
+ unsigned char byte;
+
+ byte = (hex_digit_to_int(*(p+2))*16)+
+ hex_digit_to_int(*(p+3));
+ current = sdscatlen(current,(char*)&byte,1);
+ p += 3;
+ } else if (*p == '\\' && *(p+1)) {
+ char c;
+
+ p++;
+ switch(*p) {
+ case 'n': c = '\n'; break;
+ case 'r': c = '\r'; break;
+ case 't': c = '\t'; break;
+ case 'b': c = '\b'; break;
+ case 'a': c = '\a'; break;
+ default: c = *p; break;
+ }
+ current = sdscatlen(current,&c,1);
+ } else if (*p == '"') {
+ /* closing quote must be followed by a space or
+ * nothing at all. */
+ if (*(p+1) && !isspace(*(p+1))) goto err;
+ done=1;
+ } else if (!*p) {
+ /* unterminated quotes */
+ goto err;
+ } else {
+ current = sdscatlen(current,p,1);
+ }
+ } else if (insq) {
+ if (*p == '\\' && *(p+1) == '\'') {
+ p++;
+ current = sdscatlen(current,"'",1);
+ } else if (*p == '\'') {
+ /* closing quote must be followed by a space or
+ * nothing at all. */
+ if (*(p+1) && !isspace(*(p+1))) goto err;
+ done=1;
+ } else if (!*p) {
+ /* unterminated quotes */
+ goto err;
+ } else {
+ current = sdscatlen(current,p,1);
+ }
+ } else {
+ switch(*p) {
+ case ' ':
+ case '\n':
+ case '\r':
+ case '\t':
+ case '\0':
+ done=1;
+ break;
+ case '"':
+ inq=1;
+ break;
+ case '\'':
+ insq=1;
+ break;
+ default:
+ current = sdscatlen(current,p,1);
+ break;
+ }
+ }
+ if (*p) p++;
+ }
+ /* add the token to the vector */
+ vector = realloc(vector,((*argc)+1)*sizeof(char*));
+ vector[*argc] = current;
+ (*argc)++;
+ current = NULL;
+ } else {
+ /* Even on empty input string return something not NULL. */
+ if (vector == NULL) vector = malloc(sizeof(void*));
+ return vector;
+ }
+ }
+
+err:
+ while((*argc)--)
+ sdsfree(vector[*argc]);
+ free(vector);
+ if (current) sdsfree(current);
+ *argc = 0;
+ return NULL;
+}
+
+/* Modify the string substituting all the occurrences of the set of
+ * characters specified in the 'from' string to the corresponding character
+ * in the 'to' array.
+ *
+ * For instance: sdsmapchars(mystring, "ho", "01", 2)
+ * will have the effect of turning the string "hello" into "0ell1".
+ *
+ * The function returns the sds string pointer, that is always the same
+ * as the input pointer since no resize is needed. */
+sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) {
+ size_t j, i, l = sdslen(s);
+
+ for (j = 0; j < l; j++) {
+ for (i = 0; i < setlen; i++) {
+ if (s[j] == from[i]) {
+ s[j] = to[i];
+ break;
+ }
+ }
+ }
+ return s;
+}
+
+/* Join an array of C strings using the specified separator (also a C string).
+ * Returns the result as an sds string. */
+sds sdsjoin(char **argv, int argc, char *sep, size_t seplen) {
+ sds join = sdsempty();
+ int j;
+
+ for (j = 0; j < argc; j++) {
+ join = sdscat(join, argv[j]);
+ if (j != argc-1) join = sdscatlen(join,sep,seplen);
+ }
+ return join;
+}
+
+/* Like sdsjoin, but joins an array of SDS strings. */
+sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) {
+ sds join = sdsempty();
+ int j;
+
+ for (j = 0; j < argc; j++) {
+ join = sdscatsds(join, argv[j]);
+ if (j != argc-1) join = sdscatlen(join,sep,seplen);
+ }
+ return join;
+}
+
+#ifdef SDS_TEST_MAIN
+#include <stdio.h>
+#include "testhelp.h"
+
+int main(void) {
+ {
+ struct sdshdr *sh;
+ sds x = sdsnew("foo"), y;
+
+ test_cond("Create a string and obtain the length",
+ sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0)
+
+ sdsfree(x);
+ x = sdsnewlen("foo",2);
+ test_cond("Create a string with specified length",
+ sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0)
+
+ x = sdscat(x,"bar");
+ test_cond("Strings concatenation",
+ sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0);
+
+ x = sdscpy(x,"a");
+ test_cond("sdscpy() against an originally longer string",
+ sdslen(x) == 1 && memcmp(x,"a\0",2) == 0)
+
+ x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk");
+ test_cond("sdscpy() against an originally shorter string",
+ sdslen(x) == 33 &&
+ memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0)
+
+ sdsfree(x);
+ x = sdscatprintf(sdsempty(),"%d",123);
+ test_cond("sdscatprintf() seems working in the base case",
+ sdslen(x) == 3 && memcmp(x,"123\0",4) ==0)
+
+ sdsfree(x);
+ x = sdsnew("xxciaoyyy");
+ sdstrim(x,"xy");
+ test_cond("sdstrim() correctly trims characters",
+ sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0)
+
+ y = sdsdup(x);
+ sdsrange(y,1,1);
+ test_cond("sdsrange(...,1,1)",
+ sdslen(y) == 1 && memcmp(y,"i\0",2) == 0)
+
+ sdsfree(y);
+ y = sdsdup(x);
+ sdsrange(y,1,-1);
+ test_cond("sdsrange(...,1,-1)",
+ sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
+
+ sdsfree(y);
+ y = sdsdup(x);
+ sdsrange(y,-2,-1);
+ test_cond("sdsrange(...,-2,-1)",
+ sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0)
+
+ sdsfree(y);
+ y = sdsdup(x);
+ sdsrange(y,2,1);
+ test_cond("sdsrange(...,2,1)",
+ sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
+
+ sdsfree(y);
+ y = sdsdup(x);
+ sdsrange(y,1,100);
+ test_cond("sdsrange(...,1,100)",
+ sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
+
+ sdsfree(y);
+ y = sdsdup(x);
+ sdsrange(y,100,100);
+ test_cond("sdsrange(...,100,100)",
+ sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
+
+ sdsfree(y);
+ sdsfree(x);
+ x = sdsnew("foo");
+ y = sdsnew("foa");
+ test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0)
+
+ sdsfree(y);
+ sdsfree(x);
+ x = sdsnew("bar");
+ y = sdsnew("bar");
+ test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0)
+
+ sdsfree(y);
+ sdsfree(x);
+ x = sdsnew("aar");
+ y = sdsnew("bar");
+ test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0)
+
+ sdsfree(y);
+ sdsfree(x);
+ x = sdsnewlen("\a\n\0foo\r",7);
+ y = sdscatrepr(sdsempty(),x,sdslen(x));
+ test_cond("sdscatrepr(...data...)",
+ memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0)
+
+ {
+ int oldfree;
+
+ sdsfree(x);
+ x = sdsnew("0");
+ sh = (void*) (x-(sizeof(struct sdshdr)));
+ test_cond("sdsnew() free/len buffers", sh->len == 1 && sh->free == 0);
+ x = sdsMakeRoomFor(x,1);
+ sh = (void*) (x-(sizeof(struct sdshdr)));
+ test_cond("sdsMakeRoomFor()", sh->len == 1 && sh->free > 0);
+ oldfree = sh->free;
+ x[1] = '1';
+ sdsIncrLen(x,1);
+ test_cond("sdsIncrLen() -- content", x[0] == '0' && x[1] == '1');
+ test_cond("sdsIncrLen() -- len", sh->len == 2);
+ test_cond("sdsIncrLen() -- free", sh->free == oldfree-1);
+ }
+ }
+ test_report()
+ return 0;
+}
+#endif
diff --git a/contrib/hiredis/sds.h b/contrib/hiredis/sds.h
new file mode 100644
index 0000000..a494f2e
--- /dev/null
+++ b/contrib/hiredis/sds.h
@@ -0,0 +1,108 @@
+/* SDS (Simple Dynamic Strings), A C dynamic strings library.
+ *
+ * Copyright (c) 2006-2014, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __SDS_H
+#define __SDS_H
+
+#define SDS_MAX_PREALLOC (1024*1024)
+
+#include <sys/types.h>
+#include <stdarg.h>
+#ifdef _MSC_VER
+#include "win32.h"
+#endif
+
+typedef char *sds;
+
+struct sdshdr {
+ int len;
+ int free;
+ char buf[];
+};
+
+static inline size_t sdslen(const sds s) {
+ struct sdshdr *sh = (struct sdshdr *)(s-sizeof *sh);
+ return sh->len;
+}
+
+static inline size_t sdsavail(const sds s) {
+ struct sdshdr *sh = (struct sdshdr *)(s-sizeof *sh);
+ return sh->free;
+}
+
+sds sdsnewlen(const void *init, size_t initlen);
+sds sdsnew(const char *init);
+sds sdsempty(void);
+size_t sdslen(const sds s);
+sds sdsdup(const sds s);
+void sdsfree(sds s);
+size_t sdsavail(const sds s);
+sds sdsgrowzero(sds s, size_t len);
+sds sdscatlen(sds s, const void *t, size_t len);
+sds sdscat(sds s, const char *t);
+sds sdscatsds(sds s, const sds t);
+sds sdscpylen(sds s, const char *t, size_t len);
+sds sdscpy(sds s, const char *t);
+
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 0)))
+#endif
+sds sdscatvprintf(sds s, const char *fmt, va_list ap);
+#ifdef __GNUC__
+sds sdscatprintf(sds s, const char *fmt, ...)
+ __attribute__((format(printf, 2, 3)));
+#else
+sds sdscatprintf(sds s, const char *fmt, ...);
+#endif
+
+sds sdscatfmt(sds s, char const *fmt, ...);
+void sdstrim(sds s, const char *cset);
+void sdsrange(sds s, int start, int end);
+void sdsupdatelen(sds s);
+void sdsclear(sds s);
+int sdscmp(const sds s1, const sds s2);
+sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count);
+void sdsfreesplitres(sds *tokens, int count);
+void sdstolower(sds s);
+void sdstoupper(sds s);
+sds sdsfromlonglong(long long value);
+sds sdscatrepr(sds s, const char *p, size_t len);
+sds *sdssplitargs(const char *line, int *argc);
+sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen);
+sds sdsjoin(char **argv, int argc, char *sep, size_t seplen);
+sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
+
+/* Low level functions exposed to the user API */
+sds sdsMakeRoomFor(sds s, size_t addlen);
+void sdsIncrLen(sds s, int incr);
+sds sdsRemoveFreeSpace(sds s);
+size_t sdsAllocSize(sds s);
+
+#endif
diff --git a/contrib/http-parser/CMakeLists.txt b/contrib/http-parser/CMakeLists.txt
new file mode 100644
index 0000000..a5da701
--- /dev/null
+++ b/contrib/http-parser/CMakeLists.txt
@@ -0,0 +1,8 @@
+SET(HTTPSRC http_parser.c)
+
+
+SET(HTTP_COMPILE_FLAGS "-DRSPAMD_LIB")
+
+ADD_LIBRARY(rspamd-http-parser STATIC ${HTTPSRC})
+SET_TARGET_PROPERTIES(rspamd-http-parser PROPERTIES VERSION ${RSPAMD_VERSION})
+SET_TARGET_PROPERTIES(rspamd-http-parser PROPERTIES COMPILE_FLAGS "${HTTP_COMPILE_FLAGS}") \ No newline at end of file
diff --git a/contrib/http-parser/LICENSE-MIT b/contrib/http-parser/LICENSE-MIT
new file mode 100644
index 0000000..58010b3
--- /dev/null
+++ b/contrib/http-parser/LICENSE-MIT
@@ -0,0 +1,23 @@
+http_parser.c is based on src/http/ngx_http_parse.c from NGINX copyright
+Igor Sysoev.
+
+Additional changes are licensed under the same terms as NGINX and
+copyright Joyent, Inc. and other Node contributors. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.
diff --git a/contrib/http-parser/http_parser.c b/contrib/http-parser/http_parser.c
new file mode 100644
index 0000000..c14ecc0
--- /dev/null
+++ b/contrib/http-parser/http_parser.c
@@ -0,0 +1,2268 @@
+/* Based on src/http/ngx_http_parse.c from NGINX copyright Igor Sysoev
+ *
+ * Additional changes are licensed under the same terms as NGINX and
+ * copyright Joyent, Inc. and other Node contributors. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#include "config.h"
+#include "http_parser.h"
+#include <assert.h>
+
+#ifndef ULLONG_MAX
+# define ULLONG_MAX ((uint64_t) -1) /* 2^64-1 */
+#endif
+
+#ifndef MIN
+# define MIN(a,b) ((a) < (b) ? (a) : (b))
+#endif
+
+#ifndef ARRAY_SIZE
+# define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
+#endif
+
+#ifndef BIT_AT
+# define BIT_AT(a, i) \
+ (!!((unsigned int) (a)[(unsigned int) (i) >> 3] & \
+ (1 << ((unsigned int) (i) & 7))))
+#endif
+
+#ifndef ELEM_AT
+# define ELEM_AT(a, i, v) ((unsigned int) (i) < ARRAY_SIZE(a) ? (a)[(i)] : (v))
+#endif
+
+#define SET_ERRNO(e) \
+do { \
+ parser->http_errno = (e); \
+} while(0)
+
+
+/* Run the notify callback FOR, returning ER if it fails */
+#define CALLBACK_NOTIFY_(FOR, ER) \
+do { \
+ assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \
+ \
+ if (settings->on_##FOR) { \
+ if (0 != settings->on_##FOR(parser)) { \
+ SET_ERRNO(HPE_CB_##FOR); \
+ } \
+ \
+ /* We either errored above or got paused; get out */ \
+ if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { \
+ return (ER); \
+ } \
+ } \
+} while (0)
+
+/* Run the notify callback FOR and consume the current byte */
+#define CALLBACK_NOTIFY(FOR) CALLBACK_NOTIFY_(FOR, p - data + 1)
+
+/* Run the notify callback FOR and don't consume the current byte */
+#define CALLBACK_NOTIFY_NOADVANCE(FOR) CALLBACK_NOTIFY_(FOR, p - data)
+
+/* Run data callback FOR with LEN bytes, returning ER if it fails */
+#define CALLBACK_DATA_(FOR, LEN, ER) \
+do { \
+ assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \
+ \
+ if (FOR##_mark) { \
+ if (settings->on_##FOR) { \
+ if (0 != settings->on_##FOR(parser, FOR##_mark, (LEN))) { \
+ SET_ERRNO(HPE_CB_##FOR); \
+ } \
+ \
+ /* We either errored above or got paused; get out */ \
+ if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { \
+ return (ER); \
+ } \
+ } \
+ FOR##_mark = NULL; \
+ } \
+} while (0)
+
+/* Run the data callback FOR and consume the current byte */
+#define CALLBACK_DATA(FOR) \
+ CALLBACK_DATA_(FOR, p - FOR##_mark, p - data + 1)
+
+/* Run the data callback FOR and don't consume the current byte */
+#define CALLBACK_DATA_NOADVANCE(FOR) \
+ CALLBACK_DATA_(FOR, p - FOR##_mark, p - data)
+
+/* Set the mark FOR; non-destructive if mark is already set */
+#define MARK(FOR) \
+do { \
+ if (!FOR##_mark) { \
+ FOR##_mark = p; \
+ } \
+} while (0)
+
+
+#define PROXY_CONNECTION "proxy-connection"
+#define CONNECTION "connection"
+#define CONTENT_LENGTH "content-length"
+#define TRANSFER_ENCODING "transfer-encoding"
+#define UPGRADE "upgrade"
+#define CHUNKED "chunked"
+#define KEEP_ALIVE "keep-alive"
+#define CLOSE "close"
+
+enum rspamd_http_message_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH };
+
+static const char *method_strings[] =
+ {
+#define XX(num, name, string) #string,
+ HTTP_METHOD_MAP(XX)
+#undef XX
+ };
+
+
+/* Tokens as defined by rfc 2616. Also lowercases them.
+ * token = 1*<any CHAR except CTLs or separators>
+ * separators = "(" | ")" | "<" | ">" | "@"
+ * | "," | ";" | ":" | "\" | <">
+ * | "/" | "[" | "]" | "?" | "="
+ * | "{" | "}" | SP | HT
+ */
+static const char tokens[256] = {
+/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */
+ 0, '!', 0, '#', '$', '%', '&', '\'',
+/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */
+ 0, 0, '*', '+', 0, '-', '.', 0,
+/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */
+ '0', '1', '2', '3', '4', '5', '6', '7',
+/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */
+ '8', '9', 0, 0, 0, 0, 0, 0,
+/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */
+ 0, 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */
+ 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */
+ 'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
+/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */
+ 'x', 'y', 'z', 0, 0, 0, '^', '_',
+/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */
+ '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */
+ 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */
+ 'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
+/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */
+ 'x', 'y', 'z', 0, '|', 0, '~', 0 };
+
+
+static const int8_t unhex[256] =
+ {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1
+ ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ };
+
+
+#if HTTP_PARSER_STRICT
+# define T(v) 0
+#else
+# define T(v) v
+#endif
+
+
+static const uint8_t normal_url_char[32] = {
+/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */
+ 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
+/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */
+ 0 | T(2) | 0 | 0 | T(16) | 0 | 0 | 0,
+/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */
+ 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
+/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */
+ 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
+/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */
+ 0 | 2 | 4 | 0 | 16 | 32 | 64 | 128,
+/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0,
+/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0, };
+
+#undef T
+
+enum state
+ { s_dead = 1 /* important that this is > 0 */
+
+ , s_start_req_or_res
+ , s_res_or_resp_H
+ , s_start_res
+ , s_res_H
+ , s_res_HT
+ , s_res_HTT
+ , s_res_HTTP
+ , s_res_first_http_major
+ , s_res_http_major
+ , s_res_first_http_minor
+ , s_res_http_minor
+ , s_res_first_status_code
+ , s_res_status_code
+ , s_res_status_start
+ , s_res_status
+ , s_res_line_almost_done
+
+ , s_start_req
+
+ , s_req_method
+ , s_req_spaces_before_url
+ , s_req_schema
+ , s_req_schema_slash
+ , s_req_schema_slash_slash
+ , s_req_server_start
+ , s_req_server
+ , s_req_server_with_at
+ , s_req_path
+ , s_req_query_string_start
+ , s_req_query_string
+ , s_req_fragment_start
+ , s_req_fragment
+ , s_req_http_start
+ , s_req_http_H
+ , s_req_http_HT
+ , s_req_http_HTT
+ , s_req_http_HTTP
+ , s_req_first_http_major
+ , s_req_http_major
+ , s_req_first_http_minor
+ , s_req_http_minor
+ , s_req_spamc_start
+ , s_req_spamc
+ , s_req_line_almost_done
+
+ , s_header_field_start
+ , s_header_field
+ , s_header_value_start
+ , s_header_value
+ , s_header_value_lws
+
+ , s_header_almost_done
+
+ , s_chunk_size_start
+ , s_chunk_size
+ , s_chunk_parameters
+ , s_chunk_size_almost_done
+
+ , s_headers_almost_done
+ , s_headers_done
+
+ /* Important: 's_headers_done' must be the last 'header' state. All
+ * states beyond this must be 'body' states. It is used for overflow
+ * checking. See the PARSING_HEADER() macro.
+ */
+
+ , s_chunk_data
+ , s_chunk_data_almost_done
+ , s_chunk_data_done
+
+ , s_body_identity
+ , s_body_identity_eof
+
+ , s_message_done
+ };
+
+
+#define PARSING_HEADER(state) (state <= s_headers_done)
+
+
+enum header_states
+ { h_general = 0
+ , h_C
+ , h_CO
+ , h_CON
+
+ , h_matching_connection
+ , h_matching_proxy_connection
+ , h_matching_content_length
+ , h_matching_transfer_encoding
+ , h_matching_upgrade
+
+ , h_connection
+ , h_content_length
+ , h_transfer_encoding
+ , h_upgrade
+
+ , h_matching_transfer_encoding_chunked
+ , h_matching_connection_keep_alive
+ , h_matching_connection_close
+
+ , h_transfer_encoding_chunked
+ , h_connection_keep_alive
+ , h_connection_close
+ };
+
+enum http_host_state
+ {
+ s_http_host_dead = 1
+ , s_http_userinfo_start
+ , s_http_userinfo
+ , s_http_host_start
+ , s_http_host_v6_start
+ , s_http_host
+ , s_http_host_v6
+ , s_http_host_v6_end
+ , s_http_host_port_start
+ , s_http_host_port
+};
+
+/* Macros for character classes; depends on strict-mode */
+#define CR '\r'
+#define LF '\n'
+#define LOWER(c) (unsigned char)(c | 0x20)
+#define IS_ALPHA(c) (LOWER(c) >= 'a' && LOWER(c) <= 'z')
+#define IS_NUM(c) ((c) >= '0' && (c) <= '9')
+#define IS_ALPHANUM(c) (IS_ALPHA(c) || IS_NUM(c))
+#define IS_HEX(c) (IS_NUM(c) || (LOWER(c) >= 'a' && LOWER(c) <= 'f'))
+#define IS_MARK(c) ((c) == '-' || (c) == '_' || (c) == '.' || \
+ (c) == '!' || (c) == '~' || (c) == '*' || (c) == '\'' || (c) == '(' || \
+ (c) == ')')
+#define IS_USERINFO_CHAR(c) (IS_ALPHANUM(c) || IS_MARK(c) || (c) == '%' || \
+ (c) == ';' || (c) == ':' || (c) == '&' || (c) == '=' || (c) == '+' || \
+ (c) == '$' || (c) == ',')
+
+#if HTTP_PARSER_STRICT
+#define TOKEN(c) (tokens[(unsigned char)c])
+#define IS_URL_CHAR(c) (BIT_AT(normal_url_char, (unsigned char)c))
+#define IS_HOST_CHAR(c) (IS_ALPHANUM(c) || (c) == '.' || (c) == '-')
+#else
+#define TOKEN(c) ((c == ' ') ? ' ' : tokens[(unsigned char)c])
+#define IS_URL_CHAR(c) \
+ (BIT_AT(normal_url_char, (unsigned char)c) || ((c) & 0x80))
+#define IS_HOST_CHAR(c) \
+ (IS_ALPHANUM(c) || (c) == '.' || (c) == '-' || (c) == '_')
+#endif
+
+
+#define start_state (parser->type == HTTP_REQUEST ? s_start_req : s_start_res)
+
+
+#if HTTP_PARSER_STRICT
+# define STRICT_CHECK(cond) \
+do { \
+ if (cond) { \
+ SET_ERRNO(HPE_STRICT); \
+ goto error; \
+ } \
+} while (0)
+# define NEW_MESSAGE() (http_should_keep_alive(parser) ? start_state : s_dead)
+#else
+# define STRICT_CHECK(cond)
+# define NEW_MESSAGE() start_state
+#endif
+
+
+/* Map errno values to strings for human-readable output */
+#define HTTP_STRERROR_GEN(n, s) { "HPE_" #n, s },
+static struct {
+ const char *name;
+ const char *description;
+} http_strerror_tab[] = {
+ HTTP_ERRNO_MAP(HTTP_STRERROR_GEN)
+};
+#undef HTTP_STRERROR_GEN
+
+int http_message_needs_eof(const http_parser *parser);
+
+/* Our URL parser.
+ *
+ * This is designed to be shared by http_parser_execute() for URL validation,
+ * hence it has a state transition + byte-for-byte interface. In addition, it
+ * is meant to be embedded in http_parser_parse_url(), which does the dirty
+ * work of turning state transitions URL components for its API.
+ *
+ * This function should only be invoked with non-space characters. It is
+ * assumed that the caller cares about (and can detect) the transition between
+ * URL and non-URL states by looking for these.
+ */
+static enum state
+parse_url_char(enum state s, const char ch)
+{
+ if (ch == ' ' || ch == '\r' || ch == '\n') {
+ return s_dead;
+ }
+
+#if HTTP_PARSER_STRICT
+ if (ch == '\t' || ch == '\f') {
+ return s_dead;
+ }
+#endif
+
+ switch (s) {
+ case s_req_spaces_before_url:
+ /* Proxied requests are followed by scheme of an absolute URI (alpha).
+ * All methods except CONNECT are followed by '/' or '*'.
+ */
+
+ if (ch == '/' || ch == '*') {
+ return s_req_path;
+ }
+
+ if (IS_ALPHA(ch)) {
+ return s_req_schema;
+ }
+
+ break;
+
+ case s_req_schema:
+ if (IS_ALPHA(ch)) {
+ return s;
+ }
+
+ if (ch == ':') {
+ return s_req_schema_slash;
+ }
+
+ break;
+
+ case s_req_schema_slash:
+ if (ch == '/') {
+ return s_req_schema_slash_slash;
+ }
+
+ break;
+
+ case s_req_schema_slash_slash:
+ if (ch == '/') {
+ return s_req_server_start;
+ }
+
+ break;
+
+ case s_req_server_with_at:
+ if (ch == '@') {
+ return s_dead;
+ }
+
+ /* FALLTHROUGH */
+ case s_req_server_start:
+ case s_req_server:
+ if (ch == '/') {
+ return s_req_path;
+ }
+
+ if (ch == '?') {
+ return s_req_query_string_start;
+ }
+
+ if (ch == '@') {
+ return s_req_server_with_at;
+ }
+
+ if (IS_USERINFO_CHAR(ch) || ch == '[' || ch == ']') {
+ return s_req_server;
+ }
+
+ break;
+
+ case s_req_path:
+ if (IS_URL_CHAR(ch)) {
+ return s;
+ }
+
+ switch (ch) {
+ case '?':
+ return s_req_query_string_start;
+
+ case '#':
+ return s_req_fragment_start;
+ }
+
+ break;
+
+ case s_req_query_string_start:
+ case s_req_query_string:
+ if (IS_URL_CHAR(ch)) {
+ return s_req_query_string;
+ }
+
+ switch (ch) {
+ case '?':
+ /* allow extra '?' in query string */
+ return s_req_query_string;
+
+ case '#':
+ return s_req_fragment_start;
+ }
+
+ break;
+
+ case s_req_fragment_start:
+ if (IS_URL_CHAR(ch)) {
+ return s_req_fragment;
+ }
+
+ switch (ch) {
+ case '?':
+ return s_req_fragment;
+
+ case '#':
+ return s;
+ }
+
+ break;
+
+ case s_req_fragment:
+ if (IS_URL_CHAR(ch)) {
+ return s;
+ }
+
+ switch (ch) {
+ case '?':
+ case '#':
+ return s;
+ }
+
+ break;
+
+ default:
+ break;
+ }
+
+ /* We should never fall out of the switch above unless there's an error */
+ return s_dead;
+}
+
+size_t http_parser_execute (http_parser *parser,
+ const http_parser_settings *settings,
+ const char *data,
+ size_t len)
+{
+ char c, ch;
+ int8_t unhex_val;
+ const char *p = data;
+ const char *header_field_mark = 0;
+ const char *header_value_mark = 0;
+ const char *url_mark = 0;
+ const char *body_mark = 0;
+ const char *status_mark = 0;
+
+ /* We're in an error state. Don't bother doing anything. */
+ if (HTTP_PARSER_ERRNO(parser) != HPE_OK) {
+ return 0;
+ }
+
+ if (len == 0) {
+ switch (parser->state) {
+ case s_body_identity_eof:
+ /* Use of CALLBACK_NOTIFY() here would erroneously return 1 byte read if
+ * we got paused.
+ */
+ CALLBACK_NOTIFY_NOADVANCE(message_complete);
+ return 0;
+
+ case s_dead:
+ case s_start_req_or_res:
+ case s_start_res:
+ case s_start_req:
+ return 0;
+
+ default:
+ SET_ERRNO(HPE_INVALID_EOF_STATE);
+ return 1;
+ }
+ }
+
+
+ if (parser->state == s_header_field)
+ header_field_mark = data;
+ if (parser->state == s_header_value)
+ header_value_mark = data;
+ switch (parser->state) {
+ case s_req_path:
+ case s_req_schema:
+ case s_req_schema_slash:
+ case s_req_schema_slash_slash:
+ case s_req_server_start:
+ case s_req_server:
+ case s_req_server_with_at:
+ case s_req_query_string_start:
+ case s_req_query_string:
+ case s_req_fragment_start:
+ case s_req_fragment:
+ url_mark = data;
+ break;
+ case s_res_status:
+ status_mark = data;
+ break;
+ }
+
+ for (p=data; p != data + len; p++) {
+ ch = *p;
+
+ if (PARSING_HEADER(parser->state)) {
+ ++parser->nread;
+ /* Don't allow the total size of the HTTP headers (including the status
+ * line) to exceed HTTP_MAX_HEADER_SIZE. This check is here to protect
+ * embedders against denial-of-service attacks where the attacker feeds
+ * us a never-ending header that the embedder keeps buffering.
+ *
+ * This check is arguably the responsibility of embedders but we're doing
+ * it on the embedder's behalf because most won't bother and this way we
+ * make the web a little safer. HTTP_MAX_HEADER_SIZE is still far bigger
+ * than any reasonable request or response so this should never affect
+ * day-to-day operation.
+ */
+ if (parser->nread > HTTP_MAX_HEADER_SIZE) {
+ SET_ERRNO(HPE_HEADER_OVERFLOW);
+ goto error;
+ }
+ }
+
+ reexecute_byte:
+ switch (parser->state) {
+
+ case s_dead:
+ /* this state is used after a 'Connection: close' message
+ * the parser will error out if it reads another message
+ */
+ if (ch == CR || ch == LF)
+ break;
+
+ SET_ERRNO(HPE_CLOSED_CONNECTION);
+ goto error;
+
+ case s_start_req_or_res:
+ {
+ if (ch == CR || ch == LF)
+ break;
+ parser->flags = 0;
+ parser->content_length = ULLONG_MAX;
+
+ if (ch == 'H') {
+ parser->state = s_res_or_resp_H;
+
+ CALLBACK_NOTIFY(message_begin);
+ } else {
+ parser->type = HTTP_REQUEST;
+ parser->state = s_start_req;
+ goto reexecute_byte;
+ }
+
+ break;
+ }
+
+ case s_res_or_resp_H:
+ if (ch == 'T') {
+ parser->type = HTTP_RESPONSE;
+ parser->state = s_res_HT;
+ } else {
+ if (ch != 'E') {
+ SET_ERRNO(HPE_INVALID_CONSTANT);
+ goto error;
+ }
+
+ parser->type = HTTP_REQUEST;
+ parser->method = HTTP_HEAD;
+ parser->index = 2;
+ parser->state = s_req_method;
+ }
+ break;
+
+ case s_start_res:
+ {
+ parser->flags = 0;
+ parser->content_length = ULLONG_MAX;
+
+ switch (ch) {
+ case 'H':
+ parser->state = s_res_H;
+ break;
+
+ case CR:
+ case LF:
+ break;
+
+ default:
+ SET_ERRNO(HPE_INVALID_CONSTANT);
+ goto error;
+ }
+
+ CALLBACK_NOTIFY(message_begin);
+ break;
+ }
+
+ case s_res_H:
+ STRICT_CHECK(ch != 'T');
+ parser->state = s_res_HT;
+ break;
+
+ case s_res_HT:
+ STRICT_CHECK(ch != 'T');
+ parser->state = s_res_HTT;
+ break;
+
+ case s_res_HTT:
+ STRICT_CHECK(ch != 'P');
+ parser->state = s_res_HTTP;
+ break;
+
+ case s_res_HTTP:
+ STRICT_CHECK(ch != '/');
+ parser->state = s_res_first_http_major;
+ break;
+
+ case s_res_first_http_major:
+ if (ch < '0' || ch > '9') {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_major = ch - '0';
+ parser->state = s_res_http_major;
+ break;
+
+ /* major HTTP version or dot */
+ case s_res_http_major:
+ {
+ if (ch == '.') {
+ parser->state = s_res_first_http_minor;
+ break;
+ }
+
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_major *= 10;
+ parser->http_major += ch - '0';
+
+ if (parser->http_major > 999) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ break;
+ }
+
+ /* first digit of minor HTTP version */
+ case s_res_first_http_minor:
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_minor = ch - '0';
+ parser->state = s_res_http_minor;
+ break;
+
+ /* minor HTTP version or end of request line */
+ case s_res_http_minor:
+ {
+ if (ch == ' ') {
+ parser->state = s_res_first_status_code;
+ break;
+ }
+
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_minor *= 10;
+ parser->http_minor += ch - '0';
+
+ if (parser->http_minor > 999) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ break;
+ }
+
+ case s_res_first_status_code:
+ {
+ if (!IS_NUM(ch)) {
+ if (ch == ' ') {
+ break;
+ }
+
+ SET_ERRNO(HPE_INVALID_STATUS);
+ goto error;
+ }
+ parser->status_code = ch - '0';
+ parser->state = s_res_status_code;
+ break;
+ }
+
+ case s_res_status_code:
+ {
+ if (!IS_NUM(ch)) {
+ switch (ch) {
+ case ' ':
+ parser->state = s_res_status_start;
+ break;
+ case CR:
+ parser->state = s_res_line_almost_done;
+ break;
+ case LF:
+ parser->state = s_header_field_start;
+ break;
+ default:
+ SET_ERRNO(HPE_INVALID_STATUS);
+ goto error;
+ }
+ break;
+ }
+
+ parser->status_code *= 10;
+ parser->status_code += ch - '0';
+
+ if (parser->status_code > 999) {
+ SET_ERRNO(HPE_INVALID_STATUS);
+ goto error;
+ }
+
+ break;
+ }
+
+ case s_res_status_start:
+ {
+ if (ch == CR) {
+ parser->state = s_res_line_almost_done;
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_field_start;
+ break;
+ }
+
+ MARK(status);
+ parser->state = s_res_status;
+ parser->index = 0;
+ break;
+ }
+
+ case s_res_status:
+ if (ch == CR) {
+ parser->state = s_res_line_almost_done;
+ CALLBACK_DATA(status);
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_field_start;
+ CALLBACK_DATA(status);
+ break;
+ }
+
+ break;
+
+ case s_res_line_almost_done:
+ STRICT_CHECK(ch != LF);
+ parser->state = s_header_field_start;
+ break;
+
+ case s_start_req:
+ {
+ if (ch == CR || ch == LF)
+ break;
+ parser->flags = 0;
+ parser->content_length = ULLONG_MAX;
+
+ if (!IS_ALPHA(ch)) {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+
+ parser->method = (enum http_method) 0;
+ parser->index = 1;
+ switch (ch) {
+ case 'C': parser->method = HTTP_CONNECT; /* or COPY, CHECKOUT */ break;
+ case 'D': parser->method = HTTP_DELETE; break;
+ case 'G': parser->method = HTTP_GET; break;
+ case 'H': parser->method = HTTP_HEAD; break;
+ case 'L': parser->method = HTTP_LOCK; break;
+ case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH */ break;
+ case 'N': parser->method = HTTP_NOTIFY; break;
+ case 'O': parser->method = HTTP_OPTIONS; break;
+ case 'P': parser->method = HTTP_POST;
+ /* or PROPFIND|PROPPATCH|PUT|PATCH|PURGE */
+ break;
+ case 'R': parser->method = HTTP_REPORT; break;
+ case 'S': parser->method = HTTP_SUBSCRIBE; /* or SEARCH or SYMBOLS */ break;
+ case 'T': parser->method = HTTP_TRACE; break;
+ case 'U': parser->method = HTTP_UNLOCK; /* or UNSUBSCRIBE */ break;
+ default:
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ parser->state = s_req_method;
+
+ CALLBACK_NOTIFY(message_begin);
+
+ break;
+ }
+
+ case s_req_method:
+ {
+ const char *matcher;
+ if (ch == '\0') {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+
+ matcher = method_strings[parser->method];
+ if (ch == ' ' && matcher[parser->index] == '\0') {
+ if (parser->method != HTTP_SYMBOLS && parser->method != HTTP_CHECK) {
+ parser->state = s_req_spaces_before_url;
+ }
+ else {
+ parser->state = s_req_spamc_start;
+ }
+ } else if (ch == matcher[parser->index]) {
+ ; /* nada */
+ } else if (parser->method == HTTP_CONNECT) {
+ if (parser->index == 1 && ch == 'H') {
+ /* XXX: CHECKOUT has been removed */
+ parser->method = HTTP_CHECK;
+ } else if (parser->index == 2 && ch == 'P') {
+ parser->method = HTTP_COPY;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->method == HTTP_MKCOL) {
+ if (parser->index == 1 && ch == 'O') {
+ parser->method = HTTP_MOVE;
+ } else if (parser->index == 1 && ch == 'E') {
+ parser->method = HTTP_MERGE;
+ } else if (parser->index == 1 && ch == '-') {
+ parser->method = HTTP_MSEARCH;
+ } else if (parser->index == 2 && ch == 'A') {
+ parser->method = HTTP_MKACTIVITY;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->method == HTTP_SUBSCRIBE) {
+ if (parser->index == 1 && ch == 'E') {
+ parser->method = HTTP_SEARCH;
+ } else if (ch == 'Y') {
+ parser->method = HTTP_SYMBOLS;
+ }
+ else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->index == 1 && parser->method == HTTP_POST) {
+ if (ch == 'R') {
+ parser->method = HTTP_PROPFIND; /* or HTTP_PROPPATCH */
+ } else if (ch == 'U') {
+ parser->method = HTTP_PUT; /* or HTTP_PURGE */
+ } else if (ch == 'A') {
+ parser->method = HTTP_PATCH;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->index == 2) {
+ if (parser->method == HTTP_PUT) {
+ if (ch == 'R') {
+ parser->method = HTTP_PURGE;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->method == HTTP_UNLOCK) {
+ if (ch == 'S') {
+ parser->method = HTTP_UNSUBSCRIBE;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else if (parser->index == 4 && parser->method == HTTP_PROPFIND && ch == 'P') {
+ parser->method = HTTP_PROPPATCH;
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+
+ ++parser->index;
+ break;
+ }
+
+ case s_req_spaces_before_url:
+ {
+ if (ch == ' ') break;
+
+ MARK(url);
+ if (parser->method == HTTP_CONNECT) {
+ parser->state = s_req_server_start;
+ }
+
+ parser->state = parse_url_char((enum state)parser->state, ch);
+ if (parser->state == s_dead) {
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ }
+
+ break;
+ }
+
+ case s_req_schema:
+ case s_req_schema_slash:
+ case s_req_schema_slash_slash:
+ case s_req_server_start:
+ {
+ switch (ch) {
+ /* No whitespace allowed here */
+ case ' ':
+ case CR:
+ case LF:
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ default:
+ parser->state = parse_url_char((enum state)parser->state, ch);
+ if (parser->state == s_dead) {
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ }
+ }
+
+ break;
+ }
+
+ case s_req_server:
+ case s_req_server_with_at:
+ case s_req_path:
+ case s_req_query_string_start:
+ case s_req_query_string:
+ case s_req_fragment_start:
+ case s_req_fragment:
+ {
+ switch (ch) {
+ case ' ':
+ parser->state = s_req_http_start;
+ CALLBACK_DATA(url);
+ break;
+ case CR:
+ case LF:
+ parser->http_major = 0;
+ parser->http_minor = 9;
+ parser->state = (ch == CR) ?
+ s_req_line_almost_done :
+ s_header_field_start;
+ CALLBACK_DATA(url);
+ break;
+ default:
+ parser->state = parse_url_char((enum state)parser->state, ch);
+ if (parser->state == s_dead) {
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ }
+ }
+ break;
+ }
+
+ case s_req_http_start:
+ switch (ch) {
+ case 'H':
+ parser->state = s_req_http_H;
+ break;
+ case ' ':
+ break;
+ default:
+ SET_ERRNO(HPE_INVALID_CONSTANT);
+ goto error;
+ }
+ break;
+
+ case s_req_http_H:
+ STRICT_CHECK(ch != 'T');
+ parser->state = s_req_http_HT;
+ break;
+
+ case s_req_http_HT:
+ STRICT_CHECK(ch != 'T');
+ parser->state = s_req_http_HTT;
+ break;
+
+ case s_req_http_HTT:
+ STRICT_CHECK(ch != 'P');
+ parser->state = s_req_http_HTTP;
+ break;
+
+ case s_req_http_HTTP:
+ STRICT_CHECK(ch != '/');
+ parser->state = s_req_first_http_major;
+ break;
+
+ /* first digit of major HTTP version */
+ case s_req_first_http_major:
+ if (ch < '1' || ch > '9') {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_major = ch - '0';
+ parser->state = s_req_http_major;
+ break;
+
+ /* major HTTP version or dot */
+ case s_req_http_major:
+ {
+ if (ch == '.') {
+ parser->state = s_req_first_http_minor;
+ break;
+ }
+
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_major *= 10;
+ parser->http_major += ch - '0';
+
+ if (parser->http_major > 999) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ break;
+ }
+
+ /* first digit of minor HTTP version */
+ case s_req_first_http_minor:
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_minor = ch - '0';
+ parser->state = s_req_http_minor;
+ break;
+
+ /* minor HTTP version or end of request line */
+ case s_req_http_minor:
+ {
+ if (ch == CR) {
+ parser->state = s_req_line_almost_done;
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_field_start;
+ break;
+ }
+
+ /* XXX allow spaces after digit? */
+
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_minor *= 10;
+ parser->http_minor += ch - '0';
+
+ if (parser->http_minor > 999) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ break;
+ }
+ case s_req_spamc_start: {
+ if (ch == 'S') {
+ parser->flags |= F_SPAMC;
+ parser->state = s_req_spamc;
+ }
+ else if (ch == 'R') {
+ parser->state = s_req_spamc;
+ }
+ else if (ch != ' ') {
+ SET_ERRNO(HPE_INVALID_CONSTANT);
+ goto error;
+ }
+
+ break;
+ }
+
+ case s_req_spamc:
+ {
+ if (ch == CR) {
+ parser->state = s_req_line_almost_done;
+ }
+ else if (ch == LF) {
+ parser->state = s_header_field_start;
+ }
+ break;
+ }
+
+ /* end of request line */
+ case s_req_line_almost_done:
+ {
+ if (ch != LF) {
+ SET_ERRNO(HPE_LF_EXPECTED);
+ goto error;
+ }
+
+ parser->state = s_header_field_start;
+ break;
+ }
+
+ case s_header_field_start:
+ {
+ if (ch == CR) {
+ parser->state = s_headers_almost_done;
+ break;
+ }
+
+ if (ch == LF) {
+ /* they might be just sending \n instead of \r\n so this would be
+ * the second \n to denote the end of headers*/
+ parser->state = s_headers_almost_done;
+ goto reexecute_byte;
+ }
+
+ c = TOKEN(ch);
+
+ if (!c) {
+ SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
+ goto error;
+ }
+
+ MARK(header_field);
+
+ parser->index = 0;
+ parser->state = s_header_field;
+
+ switch (c) {
+ case 'c':
+ parser->header_state = h_C;
+ break;
+
+ case 'p':
+ parser->header_state = h_matching_proxy_connection;
+ break;
+
+ case 't':
+ parser->header_state = h_matching_transfer_encoding;
+ break;
+
+ case 'u':
+ parser->header_state = h_matching_upgrade;
+ break;
+
+ default:
+ parser->header_state = h_general;
+ break;
+ }
+ break;
+ }
+
+ case s_header_field:
+ {
+ c = TOKEN(ch);
+
+ if (c) {
+ switch (parser->header_state) {
+ case h_general:
+ break;
+
+ case h_C:
+ parser->index++;
+ parser->header_state = (c == 'o' ? h_CO : h_general);
+ break;
+
+ case h_CO:
+ parser->index++;
+ parser->header_state = (c == 'n' ? h_CON : h_general);
+ break;
+
+ case h_CON:
+ parser->index++;
+ switch (c) {
+ case 'n':
+ parser->header_state = h_matching_connection;
+ break;
+ case 't':
+ parser->header_state = h_matching_content_length;
+ break;
+ default:
+ parser->header_state = h_general;
+ break;
+ }
+ break;
+
+ /* connection */
+
+ case h_matching_connection:
+ parser->index++;
+ if (parser->index > sizeof(CONNECTION)-1
+ || c != CONNECTION[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(CONNECTION)-2) {
+ parser->header_state = h_connection;
+ }
+ break;
+
+ /* proxy-connection */
+
+ case h_matching_proxy_connection:
+ parser->index++;
+ if (parser->index > sizeof(PROXY_CONNECTION)-1
+ || c != PROXY_CONNECTION[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(PROXY_CONNECTION)-2) {
+ parser->header_state = h_connection;
+ }
+ break;
+
+ /* content-length */
+
+ case h_matching_content_length:
+ parser->index++;
+ if (parser->index > sizeof(CONTENT_LENGTH)-1
+ || c != CONTENT_LENGTH[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(CONTENT_LENGTH)-2) {
+ parser->header_state = h_content_length;
+ }
+ break;
+
+ /* transfer-encoding */
+
+ case h_matching_transfer_encoding:
+ parser->index++;
+ if (parser->index > sizeof(TRANSFER_ENCODING)-1
+ || c != TRANSFER_ENCODING[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(TRANSFER_ENCODING)-2) {
+ parser->header_state = h_transfer_encoding;
+ }
+ break;
+
+ /* upgrade */
+
+ case h_matching_upgrade:
+ parser->index++;
+ if (parser->index > sizeof(UPGRADE)-1
+ || c != UPGRADE[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(UPGRADE)-2) {
+ parser->header_state = h_upgrade;
+ }
+ break;
+
+ case h_connection:
+ case h_content_length:
+ case h_transfer_encoding:
+ case h_upgrade:
+ if (ch != ' ') parser->header_state = h_general;
+ break;
+
+ default:
+ assert(0 && "Unknown header_state");
+ break;
+ }
+ break;
+ }
+
+ if (ch == ':') {
+ parser->state = s_header_value_start;
+ CALLBACK_DATA(header_field);
+ break;
+ }
+
+ if (ch == CR) {
+ parser->state = s_header_almost_done;
+ CALLBACK_DATA(header_field);
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_field_start;
+ CALLBACK_DATA(header_field);
+ break;
+ }
+
+ SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
+ goto error;
+ }
+
+ case s_header_value_start:
+ {
+ if (ch == ' ' || ch == '\t') break;
+
+ MARK(header_value);
+
+ parser->state = s_header_value;
+ parser->index = 0;
+
+ if (ch == CR) {
+ parser->header_state = h_general;
+ parser->state = s_header_almost_done;
+ CALLBACK_DATA(header_value);
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_field_start;
+ CALLBACK_DATA(header_value);
+ break;
+ }
+
+ c = LOWER(ch);
+
+ switch (parser->header_state) {
+ case h_upgrade:
+ parser->flags |= F_UPGRADE;
+ parser->header_state = h_general;
+ break;
+
+ case h_transfer_encoding:
+ /* looking for 'Transfer-Encoding: chunked' */
+ if ('c' == c) {
+ parser->header_state = h_matching_transfer_encoding_chunked;
+ } else {
+ parser->header_state = h_general;
+ }
+ break;
+
+ case h_content_length:
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ goto error;
+ }
+
+ parser->content_length = ch - '0';
+ break;
+
+ case h_connection:
+ /* looking for 'Connection: keep-alive' */
+ if (c == 'k') {
+ parser->header_state = h_matching_connection_keep_alive;
+ /* looking for 'Connection: close' */
+ } else if (c == 'c') {
+ parser->header_state = h_matching_connection_close;
+ } else {
+ parser->header_state = h_general;
+ }
+ break;
+
+ default:
+ parser->header_state = h_general;
+ break;
+ }
+ break;
+ }
+
+ case s_header_value:
+ {
+
+ if (ch == CR) {
+ parser->state = s_header_almost_done;
+ CALLBACK_DATA(header_value);
+ break;
+ }
+
+ if (ch == LF) {
+ parser->state = s_header_almost_done;
+ CALLBACK_DATA_NOADVANCE(header_value);
+ goto reexecute_byte;
+ }
+
+ c = LOWER(ch);
+
+ switch (parser->header_state) {
+ case h_general:
+ break;
+
+ case h_connection:
+ case h_transfer_encoding:
+ assert(0 && "Shouldn't get here.");
+ break;
+
+ case h_content_length:
+ {
+ uint64_t t;
+
+ if (ch == ' ') break;
+
+ if (!IS_NUM(ch)) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ goto error;
+ }
+
+ t = parser->content_length;
+ t *= 10;
+ t += ch - '0';
+
+ /* Overflow? */
+ if (t < parser->content_length || t == ULLONG_MAX) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ goto error;
+ }
+
+ parser->content_length = t;
+ break;
+ }
+
+ /* Transfer-Encoding: chunked */
+ case h_matching_transfer_encoding_chunked:
+ parser->index++;
+ if (parser->index > sizeof(CHUNKED)-1
+ || c != CHUNKED[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(CHUNKED)-2) {
+ parser->header_state = h_transfer_encoding_chunked;
+ }
+ break;
+
+ /* looking for 'Connection: keep-alive' */
+ case h_matching_connection_keep_alive:
+ parser->index++;
+ if (parser->index > sizeof(KEEP_ALIVE)-1
+ || c != KEEP_ALIVE[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(KEEP_ALIVE)-2) {
+ parser->header_state = h_connection_keep_alive;
+ }
+ break;
+
+ /* looking for 'Connection: close' */
+ case h_matching_connection_close:
+ parser->index++;
+ if (parser->index > sizeof(CLOSE)-1 || c != CLOSE[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(CLOSE)-2) {
+ parser->header_state = h_connection_close;
+ }
+ break;
+
+ case h_transfer_encoding_chunked:
+ case h_connection_keep_alive:
+ case h_connection_close:
+ if (ch != ' ') parser->header_state = h_general;
+ break;
+
+ default:
+ parser->state = s_header_value;
+ parser->header_state = h_general;
+ break;
+ }
+ break;
+ }
+
+ case s_header_almost_done:
+ {
+ STRICT_CHECK(ch != LF);
+
+ parser->state = s_header_value_lws;
+
+ switch (parser->header_state) {
+ case h_connection_keep_alive:
+ parser->flags |= F_CONNECTION_KEEP_ALIVE;
+ break;
+ case h_connection_close:
+ /* XXX: not needed for rspamd parser->flags |= F_CONNECTION_CLOSE; */
+ break;
+ case h_transfer_encoding_chunked:
+ parser->flags |= F_CHUNKED;
+ break;
+ default:
+ break;
+ }
+
+ break;
+ }
+
+ case s_header_value_lws:
+ {
+ if (ch == ' ' || ch == '\t')
+ parser->state = s_header_value_start;
+ else
+ {
+ parser->state = s_header_field_start;
+ goto reexecute_byte;
+ }
+ break;
+ }
+
+ case s_headers_almost_done:
+ {
+ STRICT_CHECK(ch != LF);
+
+ if (parser->flags & F_TRAILING) {
+ /* End of a chunked request */
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ break;
+ }
+
+ parser->state = s_headers_done;
+
+ /* Set this here so that on_headers_complete() callbacks can see it */
+ parser->upgrade =
+ (parser->flags & F_UPGRADE || parser->method == HTTP_CONNECT);
+
+ /* Here we call the headers_complete callback. This is somewhat
+ * different than other callbacks because if the user returns 1, we
+ * will interpret that as saying that this message has no body. This
+ * is needed for the annoying case of receiving a response to a HEAD
+ * request.
+ *
+ * We'd like to use CALLBACK_NOTIFY_NOADVANCE() here but we cannot, so
+ * we have to simulate it by handling a change in errno below.
+ */
+ if (settings->on_headers_complete) {
+ switch (settings->on_headers_complete(parser)) {
+ case 0:
+ break;
+
+ case 1:
+ parser->flags |= F_SKIPBODY;
+ break;
+
+ default:
+ SET_ERRNO(HPE_CB_headers_complete);
+ return p - data; /* Error */
+ }
+ }
+
+ if (HTTP_PARSER_ERRNO(parser) != HPE_OK) {
+ return p - data;
+ }
+
+ goto reexecute_byte;
+ }
+
+ case s_headers_done:
+ {
+ STRICT_CHECK(ch != LF);
+
+ parser->nread = 0;
+
+ /* Exit, the rest of the connect is in a different protocol. */
+ if (parser->upgrade) {
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ return (p - data) + 1;
+ }
+
+ if (parser->flags & F_SKIPBODY) {
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ } else if (parser->flags & F_CHUNKED) {
+ /* chunked encoding - ignore Content-Length header */
+ parser->state = s_chunk_size_start;
+ } else {
+ if (parser->content_length == 0) {
+ /* Content-Length header given but zero: Content-Length: 0\r\n */
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ } else if (parser->content_length != ULLONG_MAX) {
+ /* Content-Length header given and non-zero */
+ parser->state = s_body_identity;
+ } else {
+ if (parser->type == HTTP_REQUEST ||
+ !http_message_needs_eof(parser)) {
+ /* Assume content-length 0 - read the next */
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ } else {
+ /* Read body until EOF */
+ parser->state = s_body_identity_eof;
+ }
+ }
+ }
+
+ break;
+ }
+
+ case s_body_identity:
+ {
+ uint64_t to_read = MIN(parser->content_length,
+ (uint64_t) ((data + len) - p));
+
+ assert(parser->content_length != 0
+ && parser->content_length != ULLONG_MAX);
+
+ /* The difference between advancing content_length and p is because
+ * the latter will automatically advance on the next loop iteration.
+ * Further, if content_length ends up at 0, we want to see the last
+ * byte again for our message complete callback.
+ */
+ MARK(body);
+ parser->content_length -= to_read;
+ p += to_read - 1;
+
+ if (parser->content_length == 0) {
+ parser->state = s_message_done;
+
+ /* Mimic CALLBACK_DATA_NOADVANCE() but with one extra byte.
+ *
+ * The alternative to doing this is to wait for the next byte to
+ * trigger the data callback, just as in every other case. The
+ * problem with this is that this makes it difficult for the test
+ * harness to distinguish between complete-on-EOF and
+ * complete-on-length. It's not clear that this distinction is
+ * important for applications, but let's keep it for now.
+ */
+ CALLBACK_DATA_(body, p - body_mark + 1, p - data);
+ goto reexecute_byte;
+ }
+
+ break;
+ }
+
+ /* read until EOF */
+ case s_body_identity_eof:
+ MARK(body);
+ p = data + len - 1;
+
+ break;
+
+ case s_message_done:
+ parser->state = NEW_MESSAGE();
+ CALLBACK_NOTIFY(message_complete);
+ break;
+
+ case s_chunk_size_start:
+ {
+ assert(parser->nread == 1);
+ assert(parser->flags & F_CHUNKED);
+
+ unhex_val = unhex[(unsigned char)ch];
+ if (unhex_val == -1) {
+ SET_ERRNO(HPE_INVALID_CHUNK_SIZE);
+ goto error;
+ }
+
+ parser->content_length = unhex_val;
+ parser->state = s_chunk_size;
+ break;
+ }
+
+ case s_chunk_size:
+ {
+ uint64_t t;
+
+ assert(parser->flags & F_CHUNKED);
+
+ if (ch == CR) {
+ parser->state = s_chunk_size_almost_done;
+ break;
+ }
+
+ unhex_val = unhex[(unsigned char)ch];
+
+ if (unhex_val == -1) {
+ if (ch == ';' || ch == ' ') {
+ parser->state = s_chunk_parameters;
+ break;
+ }
+
+ SET_ERRNO(HPE_INVALID_CHUNK_SIZE);
+ goto error;
+ }
+
+ t = parser->content_length;
+ t *= 16;
+ t += unhex_val;
+
+ /* Overflow? */
+ if (t < parser->content_length || t == ULLONG_MAX) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ goto error;
+ }
+
+ parser->content_length = t;
+ break;
+ }
+
+ case s_chunk_parameters:
+ {
+ assert(parser->flags & F_CHUNKED);
+ /* just ignore this shit. TODO check for overflow */
+ if (ch == CR) {
+ parser->state = s_chunk_size_almost_done;
+ break;
+ }
+ break;
+ }
+
+ case s_chunk_size_almost_done:
+ {
+ assert(parser->flags & F_CHUNKED);
+ STRICT_CHECK(ch != LF);
+
+ parser->nread = 0;
+
+ if (parser->content_length == 0) {
+ parser->flags |= F_TRAILING;
+ parser->state = s_header_field_start;
+ } else {
+ parser->state = s_chunk_data;
+ }
+ break;
+ }
+
+ case s_chunk_data:
+ {
+ uint64_t to_read = MIN(parser->content_length,
+ (uint64_t) ((data + len) - p));
+
+ assert(parser->flags & F_CHUNKED);
+ assert(parser->content_length != 0
+ && parser->content_length != ULLONG_MAX);
+
+ /* See the explanation in s_body_identity for why the content
+ * length and data pointers are managed this way.
+ */
+ MARK(body);
+ parser->content_length -= to_read;
+ p += to_read - 1;
+
+ if (parser->content_length == 0) {
+ parser->state = s_chunk_data_almost_done;
+ }
+
+ break;
+ }
+
+ case s_chunk_data_almost_done:
+ assert(parser->flags & F_CHUNKED);
+ assert(parser->content_length == 0);
+ STRICT_CHECK(ch != CR);
+ parser->state = s_chunk_data_done;
+ CALLBACK_DATA(body);
+ break;
+
+ case s_chunk_data_done:
+ assert(parser->flags & F_CHUNKED);
+ STRICT_CHECK(ch != LF);
+ parser->nread = 0;
+ parser->state = s_chunk_size_start;
+ break;
+
+ default:
+ assert(0 && "unhandled state");
+ SET_ERRNO(HPE_INVALID_INTERNAL_STATE);
+ goto error;
+ }
+ }
+
+ /* Run callbacks for any marks that we have leftover after we ran our of
+ * bytes. There should be at most one of these set, so it's OK to invoke
+ * them in series (unset marks will not result in callbacks).
+ *
+ * We use the NOADVANCE() variety of callbacks here because 'p' has already
+ * overflowed 'data' and this allows us to correct for the off-by-one that
+ * we'd otherwise have (since CALLBACK_DATA() is meant to be run with a 'p'
+ * value that's in-bounds).
+ */
+
+ assert(((header_field_mark ? 1 : 0) +
+ (header_value_mark ? 1 : 0) +
+ (url_mark ? 1 : 0) +
+ (body_mark ? 1 : 0) +
+ (status_mark ? 1 : 0)) <= 1);
+
+ CALLBACK_DATA_NOADVANCE(header_field);
+ CALLBACK_DATA_NOADVANCE(header_value);
+ CALLBACK_DATA_NOADVANCE(url);
+ CALLBACK_DATA_NOADVANCE(body);
+ CALLBACK_DATA_NOADVANCE(status);
+
+ return len;
+
+error:
+ if (HTTP_PARSER_ERRNO(parser) == HPE_OK) {
+ SET_ERRNO(HPE_UNKNOWN);
+ }
+
+ return (p - data);
+}
+
+
+/* Does the parser need to see an EOF to find the end of the message? */
+int
+http_message_needs_eof (const http_parser *parser)
+{
+ if (parser->type == HTTP_REQUEST) {
+ return 0;
+ }
+
+ /* See RFC 2616 section 4.4 */
+ if (parser->status_code / 100 == 1 || /* 1xx e.g. Continue */
+ parser->status_code == 204 || /* No Content */
+ parser->status_code == 304 || /* Not Modified */
+ parser->flags & F_SKIPBODY) { /* response to a HEAD request */
+ return 0;
+ }
+
+ if ((parser->flags & F_CHUNKED) || parser->content_length != ULLONG_MAX) {
+ return 0;
+ }
+
+ return 1;
+}
+
+
+int
+http_should_keep_alive (const http_parser *parser)
+{
+ if (parser->http_major > 0 && parser->http_minor > 0) {
+ /* HTTP/1.1 */
+ if (!(parser->flags & F_CONNECTION_KEEP_ALIVE)) {
+ return 0;
+ }
+ } else {
+ /* HTTP/1.0 or earlier */
+ if (!(parser->flags & F_CONNECTION_KEEP_ALIVE)) {
+ return 0;
+ }
+ }
+
+ return !http_message_needs_eof(parser);
+}
+
+
+const char *
+http_method_str (enum http_method m)
+{
+ return ELEM_AT(method_strings, m, "<unknown>");
+}
+
+
+void
+http_parser_init (http_parser *parser, int t)
+{
+ void *data = parser->data; /* preserve application data */
+ memset(parser, 0, sizeof(*parser));
+ parser->data = data;
+ parser->type = t;
+ parser->state = (t == HTTP_REQUEST ? s_start_req : (t == HTTP_RESPONSE ? s_start_res : s_start_req_or_res));
+ parser->http_errno = HPE_OK;
+}
+
+const char *
+http_errno_name(enum http_errno err) {
+ assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
+ return http_strerror_tab[err].name;
+}
+
+const char *
+http_errno_description(enum http_errno err) {
+ assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
+ return http_strerror_tab[err].description;
+}
+
+static enum http_host_state
+http_parse_host_char(enum http_host_state s, const char ch) {
+ switch(s) {
+ case s_http_userinfo:
+ case s_http_userinfo_start:
+ if (ch == '@') {
+ return s_http_host_start;
+ }
+
+ if (IS_USERINFO_CHAR(ch)) {
+ return s_http_userinfo;
+ }
+ break;
+
+ case s_http_host_start:
+ if (ch == '[') {
+ return s_http_host_v6_start;
+ }
+
+ if (IS_HOST_CHAR(ch)) {
+ return s_http_host;
+ }
+
+ break;
+
+ case s_http_host:
+ if (IS_HOST_CHAR(ch)) {
+ return s_http_host;
+ }
+
+ /* FALLTHROUGH */
+ case s_http_host_v6_end:
+ if (ch == ':') {
+ return s_http_host_port_start;
+ }
+
+ break;
+
+ case s_http_host_v6:
+ if (ch == ']') {
+ return s_http_host_v6_end;
+ }
+
+ /* FALLTHROUGH */
+ case s_http_host_v6_start:
+ if (IS_HEX(ch) || ch == ':' || ch == '.') {
+ return s_http_host_v6;
+ }
+
+ break;
+
+ case s_http_host_port:
+ case s_http_host_port_start:
+ if (IS_NUM(ch)) {
+ return s_http_host_port;
+ }
+
+ break;
+
+ default:
+ break;
+ }
+ return s_http_host_dead;
+}
+
+static int
+http_parse_host(const char * buf, struct http_parser_url *u, int found_at) {
+ enum http_host_state s;
+
+ const char *p;
+ size_t buflen = u->field_data[UF_HOST].off + u->field_data[UF_HOST].len;
+
+ u->field_data[UF_HOST].len = 0;
+
+ s = found_at ? s_http_userinfo_start : s_http_host_start;
+
+ for (p = buf + u->field_data[UF_HOST].off; p < buf + buflen; p++) {
+ enum http_host_state new_s = http_parse_host_char(s, *p);
+
+ if (new_s == s_http_host_dead) {
+ return 1;
+ }
+
+ switch(new_s) {
+ case s_http_host:
+ if (s != s_http_host) {
+ u->field_data[UF_HOST].off = p - buf;
+ }
+ u->field_data[UF_HOST].len++;
+ break;
+
+ case s_http_host_v6:
+ if (s != s_http_host_v6) {
+ u->field_data[UF_HOST].off = p - buf;
+ }
+ u->field_data[UF_HOST].len++;
+ break;
+
+ case s_http_host_port:
+ if (s != s_http_host_port) {
+ u->field_data[UF_PORT].off = p - buf;
+ u->field_data[UF_PORT].len = 0;
+ u->field_set |= (1 << UF_PORT);
+ }
+ u->field_data[UF_PORT].len++;
+ break;
+
+ case s_http_userinfo:
+ if (s != s_http_userinfo) {
+ u->field_data[UF_USERINFO].off = p - buf ;
+ u->field_data[UF_USERINFO].len = 0;
+ u->field_set |= (1 << UF_USERINFO);
+ }
+ u->field_data[UF_USERINFO].len++;
+ break;
+
+ default:
+ break;
+ }
+ s = new_s;
+ }
+
+ /* Make sure we don't end somewhere unexpected */
+ switch (s) {
+ case s_http_host_start:
+ case s_http_host_v6_start:
+ case s_http_host_v6:
+ case s_http_host_port_start:
+ case s_http_userinfo:
+ case s_http_userinfo_start:
+ return 1;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+int
+http_parser_parse_url(const char *buf, size_t buflen, int is_connect,
+ struct http_parser_url *u)
+{
+ enum state s;
+ const char *p;
+ enum http_parser_url_fields uf, old_uf;
+ int found_at = 0;
+
+ u->port = u->field_set = 0;
+ s = is_connect ? s_req_server_start : s_req_spaces_before_url;
+ uf = old_uf = UF_MAX;
+
+ for (p = buf; p < buf + buflen; p++) {
+ s = parse_url_char(s, *p);
+
+ /* Figure out the next field that we're operating on */
+ switch (s) {
+ case s_dead:
+ return 1;
+
+ /* Skip delimiters */
+ case s_req_schema_slash:
+ case s_req_schema_slash_slash:
+ case s_req_server_start:
+ case s_req_query_string_start:
+ case s_req_fragment_start:
+ continue;
+
+ case s_req_schema:
+ uf = UF_SCHEMA;
+ break;
+
+ case s_req_server_with_at:
+ found_at = 1;
+
+ /* FALLTROUGH */
+ case s_req_server:
+ uf = UF_HOST;
+ break;
+
+ case s_req_path:
+ uf = UF_PATH;
+ break;
+
+ case s_req_query_string:
+ uf = UF_QUERY;
+ break;
+
+ case s_req_fragment:
+ uf = UF_FRAGMENT;
+ break;
+
+ default:
+ assert(!"Unexpected state");
+ return 1;
+ }
+
+ /* Nothing's changed; soldier on */
+ if (uf == old_uf) {
+ u->field_data[uf].len++;
+ continue;
+ }
+
+ u->field_data[uf].off = p - buf;
+ u->field_data[uf].len = 1;
+
+ u->field_set |= (1 << uf);
+ old_uf = uf;
+ }
+
+ /* host must be present if there is a schema */
+ /* parsing http:///toto will fail */
+ if ((u->field_set & ((1 << UF_SCHEMA) | (1 << UF_HOST))) != 0) {
+ if (http_parse_host(buf, u, found_at) != 0) {
+ return 1;
+ }
+ }
+
+ /* CONNECT requests can only contain "hostname:port" */
+ if (is_connect && u->field_set != ((1 << UF_HOST)|(1 << UF_PORT))) {
+ return 1;
+ }
+
+ if (u->field_set & (1 << UF_PORT)) {
+ /* Don't bother with endp; we've already validated the string */
+ unsigned long v = strtoul(buf + u->field_data[UF_PORT].off, NULL, 10);
+
+ /* Ports have a max value of 2^16 */
+ if (v > 0xffff) {
+ return 1;
+ }
+
+ u->port = (uint16_t) v;
+ }
+
+ return 0;
+}
+
+void
+http_parser_pause(http_parser *parser, int paused) {
+ /* Users should only be pausing/unpausing a parser that is not in an error
+ * state. In non-debug builds, there's not much that we can do about this
+ * other than ignore it.
+ */
+ if (HTTP_PARSER_ERRNO(parser) == HPE_OK ||
+ HTTP_PARSER_ERRNO(parser) == HPE_PAUSED) {
+ SET_ERRNO((paused) ? HPE_PAUSED : HPE_OK);
+ } else {
+ assert(0 && "Attempting to pause parser in error state");
+ }
+}
+
+int
+http_body_is_final(const struct http_parser *parser) {
+ return parser->state == s_message_done;
+}
+
+unsigned long
+http_parser_version(void) {
+ return HTTP_PARSER_VERSION_MAJOR * 0x10000 |
+ HTTP_PARSER_VERSION_MINOR * 0x00100 |
+ HTTP_PARSER_VERSION_PATCH * 0x00001;
+}
diff --git a/contrib/http-parser/http_parser.h b/contrib/http-parser/http_parser.h
new file mode 100644
index 0000000..7b4ac49
--- /dev/null
+++ b/contrib/http-parser/http_parser.h
@@ -0,0 +1,321 @@
+/* Copyright Joyent, Inc. and other Node contributors. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifndef http_parser_h
+#define http_parser_h
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Also update SONAME in the Makefile whenever you change these. */
+#define HTTP_PARSER_VERSION_MAJOR 2
+#define HTTP_PARSER_VERSION_MINOR 2
+#define HTTP_PARSER_VERSION_PATCH 0
+
+#include <sys/types.h>
+#if defined(_WIN32) && !defined(__MINGW32__) && (!defined(_MSC_VER) || _MSC_VER<1600)
+#include <BaseTsd.h>
+#include <stddef.h>
+typedef __int8 int8_t;
+typedef unsigned __int8 uint8_t;
+typedef __int16 int16_t;
+typedef unsigned __int16 uint16_t;
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
+typedef __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+#else
+#include <stdint.h>
+#endif
+
+/* Compile with -DHTTP_PARSER_STRICT=0 to make less checks, but run
+ * faster
+ */
+#ifndef HTTP_PARSER_STRICT
+# define HTTP_PARSER_STRICT 1
+#endif
+
+/* Maximium header size allowed */
+#define HTTP_MAX_HEADER_SIZE (80*1024)
+
+
+typedef struct http_parser http_parser;
+typedef struct http_parser_settings http_parser_settings;
+
+
+/* Callbacks should return non-zero to indicate an error. The parser will
+ * then halt execution.
+ *
+ * The one exception is on_headers_complete. In a HTTP_RESPONSE parser
+ * returning '1' from on_headers_complete will tell the parser that it
+ * should not expect a body. This is used when receiving a response to a
+ * HEAD request which may contain 'Content-Length' or 'Transfer-Encoding:
+ * chunked' headers that indicate the presence of a body.
+ *
+ * http_data_cb does not return data chunks. It will be call arbitrarally
+ * many times for each string. E.G. you might get 10 callbacks for "on_url"
+ * each providing just a few characters more data.
+ */
+typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);
+typedef int (*http_cb) (http_parser*);
+
+
+/* Request Methods */
+#define HTTP_METHOD_MAP(XX) \
+ XX(0, DELETE, DELETE) \
+ XX(1, GET, GET) \
+ XX(2, HEAD, HEAD) \
+ XX(3, POST, POST) \
+ XX(4, PUT, PUT) \
+ /* pathological */ \
+ XX(5, CONNECT, CONNECT) \
+ XX(6, OPTIONS, OPTIONS) \
+ XX(7, TRACE, TRACE) \
+ /* webdav */ \
+ XX(8, COPY, COPY) \
+ XX(9, LOCK, LOCK) \
+ XX(10, MKCOL, MKCOL) \
+ XX(11, MOVE, MOVE) \
+ XX(12, PROPFIND, PROPFIND) \
+ XX(13, PROPPATCH, PROPPATCH) \
+ XX(14, SEARCH, SEARCH) \
+ XX(15, UNLOCK, UNLOCK) \
+ /* subversion */ \
+ XX(16, REPORT, REPORT) \
+ XX(17, MKACTIVITY, MKACTIVITY) \
+ XX(18, CHECKOUT, CHECKOUT) \
+ XX(19, MERGE, MERGE) \
+ /* upnp */ \
+ XX(20, MSEARCH, M-SEARCH) \
+ XX(21, NOTIFY, NOTIFY) \
+ XX(22, SUBSCRIBE, SUBSCRIBE) \
+ XX(23, UNSUBSCRIBE, UNSUBSCRIBE) \
+ /* RFC-5789 */ \
+ XX(24, PATCH, PATCH) \
+ XX(25, PURGE, PURGE) \
+ /* SPAMC compatibility */ \
+ XX(26, SYMBOLS, SYMBOLS) \
+ XX(27, CHECK, CHECK) \
+ XX(28, METHOD_MAX, METHOD_MAX) \
+ XX(-1, INVALID, INVALID) \
+
+enum http_method
+ {
+#define XX(num, name, string) HTTP_##name = num,
+ HTTP_METHOD_MAP(XX)
+#undef XX
+ };
+
+
+
+/* Flag values for http_parser.flags field */
+enum flags
+ { F_CHUNKED = 1 << 0
+ , F_CONNECTION_KEEP_ALIVE = 1 << 1
+ , F_SPAMC = 1 << 2
+ , F_TRAILING = 1 << 3
+ , F_UPGRADE = 1 << 4
+ , F_SKIPBODY = 1 << 5
+ };
+
+
+/* Map for errno-related constants
+ *
+ * The provided argument should be a macro that takes 2 arguments.
+ */
+#define HTTP_ERRNO_MAP(XX) \
+ /* No error */ \
+ XX(OK, "success") \
+ \
+ /* Callback-related errors */ \
+ XX(CB_message_begin, "the on_message_begin callback failed") \
+ XX(CB_url, "the on_url callback failed") \
+ XX(CB_header_field, "the on_header_field callback failed") \
+ XX(CB_header_value, "the on_header_value callback failed") \
+ XX(CB_headers_complete, "the on_headers_complete callback failed") \
+ XX(CB_body, "the on_body callback failed") \
+ XX(CB_message_complete, "the on_message_complete callback failed") \
+ XX(CB_status, "the on_status callback failed") \
+ \
+ /* Parsing-related errors */ \
+ XX(INVALID_EOF_STATE, "stream ended at an unexpected time") \
+ XX(HEADER_OVERFLOW, \
+ "too many header bytes seen; overflow detected") \
+ XX(CLOSED_CONNECTION, \
+ "data received after completed connection: close message") \
+ XX(INVALID_VERSION, "invalid HTTP version") \
+ XX(INVALID_STATUS, "invalid HTTP status code") \
+ XX(INVALID_METHOD, "invalid HTTP method") \
+ XX(INVALID_URL, "invalid URL") \
+ XX(INVALID_HOST, "invalid host") \
+ XX(INVALID_PORT, "invalid port") \
+ XX(INVALID_PATH, "invalid path") \
+ XX(INVALID_QUERY_STRING, "invalid query string") \
+ XX(INVALID_FRAGMENT, "invalid fragment") \
+ XX(LF_EXPECTED, "LF character expected") \
+ XX(INVALID_HEADER_TOKEN, "invalid character in header") \
+ XX(INVALID_CONTENT_LENGTH, \
+ "invalid character in content-length header") \
+ XX(INVALID_CHUNK_SIZE, \
+ "invalid character in chunk size header") \
+ XX(INVALID_CONSTANT, "invalid constant string") \
+ XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\
+ XX(STRICT, "strict mode assertion failed") \
+ XX(PAUSED, "parser is paused") \
+ XX(UNKNOWN, "an unknown error occurred")
+
+
+/* Define HPE_* values for each errno value above */
+#define HTTP_ERRNO_GEN(n, s) HPE_##n,
+enum http_errno {
+ HTTP_ERRNO_MAP(HTTP_ERRNO_GEN)
+};
+#undef HTTP_ERRNO_GEN
+
+
+/* Get an http_errno value from an http_parser */
+#define HTTP_PARSER_ERRNO(p) ((enum http_errno) (p)->http_errno)
+
+
+struct http_parser {
+ /** PRIVATE **/
+ unsigned int type : 2; /* enum http_parser_type */
+ unsigned int flags : 6; /* F_* values from 'flags' enum; semi-public */
+ unsigned int state : 8; /* enum state from http_parser.c */
+ unsigned int header_state : 8; /* enum header_state from http_parser.c */
+ unsigned int index : 8; /* index into current matcher */
+
+ uint32_t nread; /* # bytes read in various scenarios */
+ uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */
+
+ /** READ-ONLY **/
+ unsigned short http_major;
+ unsigned short http_minor;
+ unsigned int status_code : 16; /* responses only */
+ unsigned int method : 8; /* requests only */
+ unsigned int http_errno : 7;
+
+ /* 1 = Upgrade header was present and the parser has exited because of that.
+ * 0 = No upgrade header present.
+ * Should be checked when http_parser_execute() returns in addition to
+ * error checking.
+ */
+ unsigned int upgrade : 1;
+
+ /** PUBLIC **/
+ void *data; /* A pointer to get hook to the "connection" or "socket" object */
+};
+
+
+struct http_parser_settings {
+ http_cb on_message_begin;
+ http_data_cb on_url;
+ http_data_cb on_status;
+ http_data_cb on_header_field;
+ http_data_cb on_header_value;
+ http_cb on_headers_complete;
+ http_data_cb on_body;
+ http_cb on_message_complete;
+};
+
+
+enum http_parser_url_fields
+ { UF_SCHEMA = 0
+ , UF_HOST = 1
+ , UF_PORT = 2
+ , UF_PATH = 3
+ , UF_QUERY = 4
+ , UF_FRAGMENT = 5
+ , UF_USERINFO = 6
+ , UF_MAX = 7
+ };
+
+
+/* Result structure for http_parser_parse_url().
+ *
+ * Callers should index into field_data[] with UF_* values iff field_set
+ * has the relevant (1 << UF_*) bit set. As a courtesy to clients (and
+ * because we probably have padding left over), we convert any port to
+ * a uint16_t.
+ */
+struct http_parser_url {
+ uint16_t field_set; /* Bitmask of (1 << UF_*) values */
+ uint16_t port; /* Converted UF_PORT string */
+
+ struct {
+ uint16_t off; /* Offset into buffer in which field starts */
+ uint16_t len; /* Length of run in buffer */
+ } field_data[UF_MAX];
+};
+
+
+/* Returns the library version. Bits 16-23 contain the major version number,
+ * bits 8-15 the minor version number and bits 0-7 the patch level.
+ * Usage example:
+ *
+ * unsigned long version = http_parser_version();
+ * unsigned major = (version >> 16) & 255;
+ * unsigned minor = (version >> 8) & 255;
+ * unsigned patch = version & 255;
+ * printf("http_parser v%u.%u.%u\n", major, minor, version);
+ */
+unsigned long http_parser_version(void);
+
+void http_parser_init(http_parser *parser, int type);
+
+
+size_t http_parser_execute(http_parser *parser,
+ const http_parser_settings *settings,
+ const char *data,
+ size_t len);
+
+
+/* If http_should_keep_alive() in the on_headers_complete or
+ * on_message_complete callback returns 0, then this should be
+ * the last message on the connection.
+ * If you are the server, respond with the "Connection: close" header.
+ * If you are the client, close the connection.
+ */
+int http_should_keep_alive(const http_parser *parser);
+
+/* Returns a string version of the HTTP method. */
+const char *http_method_str(enum http_method m);
+
+/* Return a string name of the given error */
+const char *http_errno_name(enum http_errno err);
+
+/* Return a string description of the given error */
+const char *http_errno_description(enum http_errno err);
+
+/* Parse a URL; return nonzero on failure */
+int http_parser_parse_url(const char *buf, size_t buflen,
+ int is_connect,
+ struct http_parser_url *u);
+
+/* Pause or un-pause the parser; a nonzero value pauses */
+void http_parser_pause(http_parser *parser, int paused);
+
+/* Checks if this is the final chunk of the body. */
+int http_body_is_final(const http_parser *parser);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/contrib/kann/CMakeLists.txt b/contrib/kann/CMakeLists.txt
new file mode 100644
index 0000000..5f1b17a
--- /dev/null
+++ b/contrib/kann/CMakeLists.txt
@@ -0,0 +1,16 @@
+SET(LIBKANNSRC kautodiff.c kann.c)
+
+IF(ENABLE_STATIC MATCHES "ON")
+ ADD_LIBRARY(rspamd-kann STATIC ${LIBKANNSRC})
+ELSE()
+ ADD_LIBRARY(rspamd-kann SHARED ${LIBKANNSRC})
+ENDIF()
+
+target_link_libraries(rspamd-kann "${RSPAMD_REQUIRED_LIBRARIES}")
+target_link_libraries(rspamd-kann "m")
+IF(WITH_BLAS)
+ MESSAGE(STATUS "Use openblas to accelerate kann")
+ TARGET_LINK_LIBRARIES(rspamd-kann ${BLAS_REQUIRED_LIBRARIES})
+ENDIF(WITH_BLAS)
+
+INSTALL(TARGETS rspamd-kann LIBRARY DESTINATION ${RSPAMD_LIBDIR}) \ No newline at end of file
diff --git a/contrib/kann/LICENSE.txt b/contrib/kann/LICENSE.txt
new file mode 100644
index 0000000..8b2cf11
--- /dev/null
+++ b/contrib/kann/LICENSE.txt
@@ -0,0 +1,24 @@
+The MIT License
+
+Copyright (c) 2018-2019 Dana-Farber Cancer Institute
+ 2016-2018 Broad Institute
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/contrib/kann/kann.c b/contrib/kann/kann.c
new file mode 100644
index 0000000..70d1f02
--- /dev/null
+++ b/contrib/kann/kann.c
@@ -0,0 +1,992 @@
+#include "config.h"
+
+#include <math.h>
+#include <float.h>
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <stdarg.h>
+#include "kann.h"
+
+int kann_verbose = 3;
+
+/******************************************
+ *** @@BASIC: fundamental KANN routines ***
+ ******************************************/
+
+static void kad_ext_collate(int n, kad_node_t **a, float **_x, float **_g, float **_c)
+{
+ int i, j, k, l, n_var;
+ float *x, *g, *c;
+ n_var = kad_size_var(n, a);
+ x = *_x = (float*)realloc(*_x, n_var * sizeof(float));
+ g = *_g = (float*)realloc(*_g, n_var * sizeof(float));
+ c = *_c = (float*)realloc(*_c, kad_size_const(n, a) * sizeof(float));
+ memset(g, 0, n_var * sizeof(float));
+ for (i = j = k = 0; i < n; ++i) {
+ kad_node_t *v = a[i];
+ if (kad_is_var(v)) {
+ l = kad_len(v);
+ memcpy(&x[j], v->x, l * sizeof(float));
+ free(v->x);
+ v->x = &x[j];
+ v->g = &g[j];
+ j += l;
+ } else if (kad_is_const(v)) {
+ l = kad_len(v);
+ memcpy(&c[k], v->x, l * sizeof(float));
+ free(v->x);
+ v->x = &c[k];
+ k += l;
+ }
+ }
+}
+
+static void kad_ext_sync(int n, kad_node_t **a, float *x, float *g, float *c)
+{
+ int i, j, k;
+ for (i = j = k = 0; i < n; ++i) {
+ kad_node_t *v = a[i];
+ if (kad_is_var(v)) {
+ v->x = &x[j];
+ v->g = &g[j];
+ j += kad_len(v);
+ } else if (kad_is_const(v)) {
+ v->x = &c[k];
+ k += kad_len(v);
+ }
+ }
+}
+
+kann_t *kann_new(kad_node_t *cost, int n_rest, ...)
+{
+ kann_t *a;
+ int i, n_roots = 1 + n_rest, has_pivot = 0, has_recur = 0;
+ kad_node_t **roots;
+ va_list ap;
+
+ if (cost->n_d != 0) return 0;
+
+ va_start(ap, n_rest);
+ roots = (kad_node_t**)malloc((n_roots + 1) * sizeof(kad_node_t*));
+ for (i = 0; i < n_rest; ++i)
+ roots[i] = va_arg(ap, kad_node_t*);
+ roots[i++] = cost;
+ va_end(ap);
+
+ cost->ext_flag |= KANN_F_COST;
+ a = (kann_t*)calloc(1, sizeof(kann_t));
+ a->v = kad_compile_array(&a->n, n_roots, roots);
+
+ for (i = 0; i < a->n; ++i) {
+ if (a->v[i]->pre) has_recur = 1;
+ if (kad_is_pivot(a->v[i])) has_pivot = 1;
+ }
+ if (has_recur && !has_pivot) { /* an RNN that doesn't have a pivot; then add a pivot on top of cost and recompile */
+ cost->ext_flag &= ~KANN_F_COST;
+ roots[n_roots-1] = cost = kad_avg(1, &cost), cost->ext_flag |= KANN_F_COST;
+ free(a->v);
+ a->v = kad_compile_array(&a->n, n_roots, roots);
+ }
+ kad_ext_collate(a->n, a->v, &a->x, &a->g, &a->c);
+ free(roots);
+ return a;
+}
+
+kann_t *kann_clone(kann_t *a, int batch_size)
+{
+ kann_t *b;
+ b = (kann_t*)calloc(1, sizeof(kann_t));
+ b->n = a->n;
+ b->v = kad_clone(a->n, a->v, batch_size);
+ kad_ext_collate(b->n, b->v, &b->x, &b->g, &b->c);
+ return b;
+}
+
+kann_t *kann_unroll_array(kann_t *a, int *len)
+{
+ kann_t *b;
+ b = (kann_t*)calloc(1, sizeof(kann_t));
+ b->x = a->x, b->g = a->g, b->c = a->c; /* these arrays are shared */
+ b->v = kad_unroll(a->n, a->v, &b->n, len);
+ return b;
+}
+
+kann_t *kann_unroll(kann_t *a, ...)
+{
+ kann_t *b;
+ va_list ap;
+ int i, n_pivots, *len;
+ n_pivots = kad_n_pivots(a->n, a->v);
+ len = (int*)calloc(n_pivots, sizeof(int));
+ va_start(ap, a);
+ for (i = 0; i < n_pivots; ++i) len[i] = va_arg(ap, int);
+ va_end(ap);
+ b = kann_unroll_array(a, len);
+ free(len);
+ return b;
+}
+
+void kann_delete_unrolled(kann_t *a)
+{
+ if (a && a->mt) kann_mt(a, 0, 0);
+ if (a && a->v) kad_delete(a->n, a->v);
+ free(a);
+}
+
+void kann_delete(kann_t *a)
+{
+ if (a == 0) return;
+ free(a->x); free(a->g); free(a->c);
+ kann_delete_unrolled(a);
+}
+
+static void kann_switch_core(kann_t *a, int is_train)
+{
+ int i;
+ for (i = 0; i < a->n; ++i)
+ if (a->v[i]->op == 12 && a->v[i]->n_child == 2)
+ *(int32_t*)a->v[i]->ptr = !!is_train;
+}
+
+#define chk_flg(flag, mask) ((mask) == 0 || ((flag) & (mask)))
+#define chk_lbl(label, query) ((query) == 0 || (label) == (query))
+
+int kann_find(const kann_t *a, uint32_t ext_flag, int32_t ext_label)
+{
+ int i, k, r = -1;
+ for (i = k = 0; i < a->n; ++i)
+ if (chk_flg(a->v[i]->ext_flag, ext_flag) && chk_lbl(a->v[i]->ext_label, ext_label))
+ ++k, r = i;
+ return k == 1? r : k == 0? -1 : -2;
+}
+
+int kann_feed_bind(kann_t *a, uint32_t ext_flag, int32_t ext_label, float **x)
+{
+ int i, k;
+ if (x == 0) return 0;
+ for (i = k = 0; i < a->n; ++i)
+ if (kad_is_feed(a->v[i]) && chk_flg(a->v[i]->ext_flag, ext_flag) && chk_lbl(a->v[i]->ext_label, ext_label))
+ a->v[i]->x = x[k++];
+ return k;
+}
+
+int kann_feed_dim(const kann_t *a, uint32_t ext_flag, int32_t ext_label)
+{
+ int i, k, n = 0;
+ for (i = k = 0; i < a->n; ++i)
+ if (kad_is_feed(a->v[i]) && chk_flg(a->v[i]->ext_flag, ext_flag) && chk_lbl(a->v[i]->ext_label, ext_label))
+ ++k, n = a->v[i]->n_d > 1? kad_len(a->v[i]) / a->v[i]->d[0] : a->v[i]->n_d == 1? a->v[i]->d[0] : 1;
+ return k == 1? n : k == 0? -1 : -2;
+}
+
+static float kann_cost_core(kann_t *a, int cost_label, int cal_grad)
+{
+ int i_cost;
+ float cost;
+ i_cost = kann_find(a, KANN_F_COST, cost_label);
+ assert(i_cost >= 0);
+ cost = *kad_eval_at(a->n, a->v, i_cost);
+ if (cal_grad) kad_grad(a->n, a->v, i_cost);
+ return cost;
+}
+
+int kann_eval(kann_t *a, uint32_t ext_flag, int ext_label)
+{
+ int i, k;
+ for (i = k = 0; i < a->n; ++i)
+ if (chk_flg(a->v[i]->ext_flag, ext_flag) && chk_lbl(a->v[i]->ext_label, ext_label))
+ ++k, a->v[i]->tmp = 1;
+ kad_eval_marked(a->n, a->v);
+ return k;
+}
+
+void kann_rnn_start(kann_t *a)
+{
+ int i;
+ kann_set_batch_size(a, 1);
+ for (i = 0; i < a->n; ++i) {
+ kad_node_t *p = a->v[i];
+ if (p->pre) { /* NB: BE CAREFUL of the interaction between kann_rnn_start() and kann_set_batch_size() */
+ kad_node_t *q = p->pre;
+ if (q->x) memcpy(p->x, q->x, kad_len(p) * sizeof(float));
+ else memset(p->x, 0, kad_len(p) * sizeof(float));
+ if (q->n_child > 0) free(q->x);
+ q->x = p->x;
+ }
+ }
+}
+
+void kann_rnn_end(kann_t *a)
+{
+ int i;
+ kad_ext_sync(a->n, a->v, a->x, a->g, a->c);
+ for (i = 0; i < a->n; ++i)
+ if (a->v[i]->pre && a->v[i]->pre->n_child > 0)
+ a->v[i]->pre->x = (float*)calloc(kad_len(a->v[i]->pre), sizeof(float));
+}
+
+static int kann_class_error_core(const kann_t *ann, int *base)
+{
+ int i, j, k, m, n, off, n_err = 0;
+ for (i = 0, *base = 0; i < ann->n; ++i) {
+ kad_node_t *p = ann->v[i];
+ if (((p->op == 13 && (p->n_child == 2 || p->n_child == 3)) || (p->op == 22 && p->n_child == 2)) && p->n_d == 0) { /* ce_bin or ce_multi */
+ kad_node_t *x = p->child[0], *t = p->child[1];
+ n = t->d[t->n_d - 1], m = kad_len(t) / n;
+ for (j = off = 0; j < m; ++j, off += n) {
+ float t_sum = 0.0f, t_min = 1.0f, t_max = 0.0f, x_max = 0.0f, x_min = 1.0f;
+ int x_max_k = -1, t_max_k = -1;
+ for (k = 0; k < n; ++k) {
+ float xk = x->x[off+k], tk = t->x[off+k];
+ t_sum += tk;
+ t_min = t_min < tk? t_min : tk;
+ x_min = x_min < xk? x_min : xk;
+ if (t_max < tk) t_max = tk, t_max_k = k;
+ if (x_max < xk) x_max = xk, x_max_k = k;
+ }
+ if (t_sum - 1.0f == 0 && t_min >= 0.0f && x_min >= 0.0f && x_max <= 1.0f) {
+ ++(*base);
+ n_err += (x_max_k != t_max_k);
+ }
+ }
+ }
+ }
+ return n_err;
+}
+
+/*************************
+ * @@MT: multi-threading *
+ *************************/
+
+#ifdef HAVE_PTHREAD
+#include <pthread.h>
+
+struct mtaux_t;
+
+typedef struct { /* per-worker data */
+ kann_t *a;
+ float cost;
+ int action;
+ pthread_t tid;
+ struct mtaux_t *g;
+} mtaux1_t;
+
+typedef struct mtaux_t { /* cross-worker data */
+ int n_threads, max_batch_size;
+ int cal_grad, cost_label, eval_out;
+ volatile int n_idle; /* we will be busy waiting on this, so volatile necessary */
+ pthread_mutex_t mtx;
+ pthread_cond_t cv;
+ mtaux1_t *mt;
+} mtaux_t;
+
+static void *mt_worker(void *data) /* pthread worker */
+{
+ mtaux1_t *mt1 = (mtaux1_t*)data;
+ mtaux_t *mt = mt1->g;
+ for (;;) {
+ int action;
+ pthread_mutex_lock(&mt->mtx);
+ mt1->action = 0;
+ ++mt->n_idle;
+ while (mt1->action == 0)
+ pthread_cond_wait(&mt->cv, &mt->mtx);
+ action = mt1->action;
+ pthread_mutex_unlock(&mt->mtx);
+ if (action == -1) break;
+
+ if (mt->eval_out) kann_eval(mt1->a, KANN_F_OUT, 0);
+ else mt1->cost = kann_cost_core(mt1->a, mt->cost_label, mt->cal_grad);
+ }
+ pthread_exit(0);
+}
+
+static void mt_destroy(mtaux_t *mt) /* de-allocate an entire mtaux_t struct */
+{
+ int i;
+ pthread_mutex_lock(&mt->mtx);
+ mt->n_idle = 0;
+ for (i = 1; i < mt->n_threads; ++i) mt->mt[i].action = -1;
+ pthread_cond_broadcast(&mt->cv);
+ pthread_mutex_unlock(&mt->mtx);
+ for (i = 1; i < mt->n_threads; ++i) pthread_join(mt->mt[i].tid, 0);
+ for (i = 0; i < mt->n_threads; ++i) kann_delete(mt->mt[i].a);
+ free(mt->mt);
+ pthread_cond_destroy(&mt->cv);
+ pthread_mutex_destroy(&mt->mtx);
+ free(mt);
+}
+
+void kann_mt(kann_t *ann, int n_threads, int max_batch_size)
+{
+ mtaux_t *mt;
+ int i, k;
+
+ if (n_threads <= 1) {
+ if (ann->mt) mt_destroy((mtaux_t*)ann->mt);
+ ann->mt = 0;
+ return;
+ }
+ if (n_threads > max_batch_size) n_threads = max_batch_size;
+ if (n_threads <= 1) return;
+
+ mt = (mtaux_t*)calloc(1, sizeof(mtaux_t));
+ mt->n_threads = n_threads, mt->max_batch_size = max_batch_size;
+ pthread_mutex_init(&mt->mtx, 0);
+ pthread_cond_init(&mt->cv, 0);
+ mt->mt = (mtaux1_t*)calloc(n_threads, sizeof(mtaux1_t));
+ for (i = k = 0; i < n_threads; ++i) {
+ int size = (max_batch_size - k) / (n_threads - i);
+ mt->mt[i].a = kann_clone(ann, size);
+ mt->mt[i].g = mt;
+ k += size;
+ }
+ for (i = 1; i < n_threads; ++i)
+ pthread_create(&mt->mt[i].tid, 0, mt_worker, &mt->mt[i]);
+ while (mt->n_idle < n_threads - 1); /* busy waiting until all threads in sync */
+ ann->mt = mt;
+}
+
+static void mt_kickoff(kann_t *a, int cost_label, int cal_grad, int eval_out)
+{
+ mtaux_t *mt = (mtaux_t*)a->mt;
+ int i, j, k, B, n_var;
+
+ B = kad_sync_dim(a->n, a->v, -1); /* get the current batch size */
+ assert(B <= mt->max_batch_size); /* TODO: can be relaxed */
+ n_var = kann_size_var(a);
+
+ pthread_mutex_lock(&mt->mtx);
+ mt->cost_label = cost_label, mt->cal_grad = cal_grad, mt->eval_out = eval_out;
+ for (i = k = 0; i < mt->n_threads; ++i) {
+ int size = (B - k) / (mt->n_threads - i);
+ for (j = 0; j < a->n; ++j)
+ if (kad_is_feed(a->v[j]))
+ mt->mt[i].a->v[j]->x = &a->v[j]->x[k * kad_len(a->v[j]) / a->v[j]->d[0]];
+ kad_sync_dim(mt->mt[i].a->n, mt->mt[i].a->v, size); /* TODO: we can point ->x to internal nodes, too */
+ k += size;
+ memcpy(mt->mt[i].a->x, a->x, n_var * sizeof(float));
+ mt->mt[i].action = 1;
+ }
+ mt->n_idle = 0;
+ pthread_cond_broadcast(&mt->cv);
+ pthread_mutex_unlock(&mt->mtx);
+}
+
+float kann_cost(kann_t *a, int cost_label, int cal_grad)
+{
+ mtaux_t *mt = (mtaux_t*)a->mt;
+ int i, j, B, k, n_var;
+ float cost;
+
+ if (mt == 0) return kann_cost_core(a, cost_label, cal_grad);
+ B = kad_sync_dim(a->n, a->v, -1); /* get the current batch size */
+ n_var = kann_size_var(a);
+
+ mt_kickoff(a, cost_label, cal_grad, 0);
+ mt->mt[0].cost = kann_cost_core(mt->mt[0].a, cost_label, cal_grad);
+ while (mt->n_idle < mt->n_threads - 1); /* busy waiting until all threads in sync */
+
+ memset(a->g, 0, n_var * sizeof(float)); /* TODO: check if this is necessary when cal_grad is false */
+ for (i = k = 0, cost = 0.0f; i < mt->n_threads; ++i) {
+ int size = (B - k) / (mt->n_threads - i);
+ cost += mt->mt[i].cost * size / B;
+ kad_saxpy(n_var, (float)size / B, mt->mt[i].a->g, a->g);
+ k += size;
+ }
+ for (j = 0; j < a->n; ++j) { /* copy values back at recurrent nodes (needed by textgen; TODO: temporary solution) */
+ kad_node_t *p = a->v[j];
+ if (p->pre && p->n_d >= 2 && p->d[0] == B) {
+ for (i = k = 0; i < mt->n_threads; ++i) {
+ kad_node_t *q = mt->mt[i].a->v[j];
+ memcpy(&p->x[k], q->x, kad_len(q) * sizeof(float));
+ k += kad_len(q);
+ }
+ }
+ }
+ return cost;
+}
+
+int kann_eval_out(kann_t *a)
+{
+ mtaux_t *mt = (mtaux_t*)a->mt;
+ int j, B, n_eval;
+ if (mt == 0) return kann_eval(a, KANN_F_OUT, 0);
+ B = kad_sync_dim(a->n, a->v, -1); /* get the current batch size */
+ mt_kickoff(a, 0, 0, 1);
+ n_eval = kann_eval(mt->mt[0].a, KANN_F_OUT, 0);
+ while (mt->n_idle < mt->n_threads - 1); /* busy waiting until all threads in sync */
+ for (j = 0; j < a->n; ++j) { /* copy output values back */
+ kad_node_t *p = a->v[j];
+ if (p->ext_flag & KANN_F_OUT) {
+ int i, t, k, d0 = p->d[0] / B, d1 = 1; /* for RNN, p->d[0] may equal unroll_len * batch_size */
+ assert(p->d[0] % B == 0);
+ for (i = 1; i < p->n_d; ++i) d1 *= p->d[i];
+ for (i = 0; i < d0; ++i) {
+ for (t = k = 0; t < mt->n_threads; ++t) { /* similar to the forward pass of kad_op_concat() */
+ kad_node_t *q = mt->mt[t].a->v[j];
+ int size = q->d[0] / d0;
+ memcpy(&p->x[(i * B + k) * d1], &q->x[i * size * d1], size * d1 * sizeof(float));
+ k += size;
+ }
+ }
+ }
+ }
+ return n_eval;
+}
+
+int kann_class_error(const kann_t *ann, int *base)
+{
+ mtaux_t *mt = (mtaux_t*)ann->mt;
+ int i, n_err = 0, b = 0;
+ if (mt == 0) return kann_class_error_core(ann, base);
+ for (i = 0; i < mt->n_threads; ++i) {
+ n_err += kann_class_error_core(mt->mt[i].a, &b);
+ *base += b;
+ }
+ return n_err;
+}
+
+void kann_switch(kann_t *ann, int is_train)
+{
+ mtaux_t *mt = (mtaux_t*)ann->mt;
+ int i;
+ if (mt == 0) {
+ kann_switch_core(ann, is_train);
+ return;
+ }
+ for (i = 0; i < mt->n_threads; ++i)
+ kann_switch_core(mt->mt[i].a, is_train);
+}
+#else
+void kann_mt(kann_t *ann, int n_threads, int max_batch_size) {}
+float kann_cost(kann_t *a, int cost_label, int cal_grad) { return kann_cost_core(a, cost_label, cal_grad); }
+int kann_eval_out(kann_t *a) { return kann_eval(a, KANN_F_OUT, 0); }
+int kann_class_error(const kann_t *a, int *base) { return kann_class_error_core(a, base); }
+void kann_switch(kann_t *ann, int is_train) { return kann_switch_core(ann, is_train); }
+#endif
+
+/***********************
+ *** @@IO: model I/O ***
+ ***********************/
+
+#define KANN_MAGIC "KAN\1"
+
+void kann_save_fp(FILE *fp, kann_t *ann)
+{
+ kann_set_batch_size(ann, 1);
+ fwrite(KANN_MAGIC, 1, 4, fp);
+ kad_save(fp, ann->n, ann->v);
+ fwrite(ann->x, sizeof(float), kann_size_var(ann), fp);
+ fwrite(ann->c, sizeof(float), kann_size_const(ann), fp);
+}
+
+void kann_save(const char *fn, kann_t *ann)
+{
+ FILE *fp;
+ fp = fn && strcmp(fn, "-")? fopen(fn, "wb") : stdout;
+ kann_save_fp(fp, ann);
+ fclose(fp);
+}
+
+kann_t *kann_load_fp(FILE *fp)
+{
+ char magic[4];
+ kann_t *ann;
+ int n_var, n_const;
+
+ (void) !fread(magic, 1, 4, fp);
+ if (strncmp(magic, KANN_MAGIC, 4) != 0) {
+ return 0;
+ }
+ ann = (kann_t*)calloc(1, sizeof(kann_t));
+ ann->v = kad_load(fp, &ann->n);
+ n_var = kad_size_var(ann->n, ann->v);
+ n_const = kad_size_const(ann->n, ann->v);
+ ann->x = (float*)malloc(n_var * sizeof(float));
+ ann->g = (float*)calloc(n_var, sizeof(float));
+ ann->c = (float*)malloc(n_const * sizeof(float));
+ (void) !fread(ann->x, sizeof(float), n_var, fp);
+ (void) !fread(ann->c, sizeof(float), n_const, fp);
+ kad_ext_sync(ann->n, ann->v, ann->x, ann->g, ann->c);
+ return ann;
+}
+
+kann_t *kann_load(const char *fn)
+{
+ FILE *fp;
+ kann_t *ann;
+ fp = fn && strcmp(fn, "-")? fopen(fn, "rb") : stdin;
+ ann = kann_load_fp(fp);
+ fclose(fp);
+ return ann;
+}
+
+/**********************************************
+ *** @@LAYER: layers and model generation ***
+ **********************************************/
+
+/********** General but more complex APIs **********/
+
+kad_node_t *kann_new_leaf_array(int *offset, kad_node_p *par, uint8_t flag, float x0_01, int n_d, int32_t d[KAD_MAX_DIM])
+{
+ int i, len, off = offset && par? *offset : -1;
+ kad_node_t *p;
+
+ if (off >= 0 && par[off]) return par[(*offset)++];
+ p = (kad_node_t*)calloc(1, sizeof(kad_node_t));
+ p->n_d = n_d, p->flag = flag;
+ memcpy(p->d, d, n_d * sizeof(int32_t));
+ len = kad_len(p);
+ p->x = (float*)calloc(len, sizeof(float));
+ if (p->n_d <= 1) {
+ for (i = 0; i < len; ++i)
+ p->x[i] = x0_01;
+ } else {
+ double sdev_inv;
+ sdev_inv = 1.0 / sqrt((double)len / p->d[0]);
+ for (i = 0; i < len; ++i)
+ p->x[i] = (float)(kad_drand_normal(0) * sdev_inv);
+ }
+ if (off >= 0) par[off] = p, ++(*offset);
+ return p;
+}
+
+kad_node_t *kann_new_leaf2(int *offset, kad_node_p *par, uint8_t flag, float x0_01, int n_d, ...)
+{
+ int32_t i, d[KAD_MAX_DIM];
+ va_list ap;
+ va_start(ap, n_d); for (i = 0; i < n_d; ++i) d[i] = va_arg(ap, int); va_end(ap);
+ return kann_new_leaf_array(offset, par, flag, x0_01, n_d, d);
+}
+
+kad_node_t *kann_layer_dense2(int *offset, kad_node_p *par, kad_node_t *in, int n1)
+{
+ int n0;
+ kad_node_t *w, *b;
+ n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in);
+ w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0);
+ b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1);
+ return kad_add(kad_cmul(in, w), b);
+}
+
+kad_node_t *kann_layer_dropout2(int *offset, kad_node_p *par, kad_node_t *t, float r)
+{
+ kad_node_t *x[2], *cr;
+ cr = kann_new_leaf2(offset, par, KAD_CONST, r, 0);
+ x[0] = t, x[1] = kad_dropout(t, cr);
+ return kad_switch(2, x);
+}
+
+kad_node_t *kann_layer_layernorm2(int *offset, kad_node_t **par, kad_node_t *in)
+{
+ int n0;
+ kad_node_t *alpha, *beta;
+ n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in);
+ alpha = kann_new_leaf2(offset, par, KAD_VAR, 1.0f, 1, n0);
+ beta = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n0);
+ return kad_add(kad_mul(kad_stdnorm(in), alpha), beta);
+}
+
+static inline kad_node_t *cmul_norm2(int *offset, kad_node_t **par, kad_node_t *x, kad_node_t *w, int use_norm)
+{
+ return use_norm? kann_layer_layernorm2(offset, par, kad_cmul(x, w)) : kad_cmul(x, w);
+}
+
+kad_node_t *kann_layer_rnn2(int *offset, kad_node_t **par, kad_node_t *in, kad_node_t *h0, int rnn_flag)
+{
+ int n0, n1 = h0->d[h0->n_d-1], use_norm = !!(rnn_flag & KANN_RNN_NORM);
+ kad_node_t *t, *w, *u, *b, *out;
+
+ u = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n1);
+ b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1);
+ t = cmul_norm2(offset, par, h0, u, use_norm);
+ if (in) {
+ n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in);
+ w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0);
+ t = kad_add(cmul_norm2(offset, par, in, w, use_norm), t);
+ }
+ out = kad_tanh(kad_add(t, b));
+ out->pre = h0;
+ return out;
+}
+
+kad_node_t *kann_layer_gru2(int *offset, kad_node_t **par, kad_node_t *in, kad_node_t *h0, int rnn_flag)
+{
+ int n0 = 0, n1 = h0->d[h0->n_d-1], use_norm = !!(rnn_flag & KANN_RNN_NORM);
+ kad_node_t *t, *r, *z, *w, *u, *b, *s, *out;
+
+ if (in) n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in);
+ /* z = sigm(x_t * W_z + h_{t-1} * U_z + b_z) */
+ u = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n1);
+ b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1);
+ t = cmul_norm2(offset, par, h0, u, use_norm);
+ if (in) {
+ w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0);
+ t = kad_add(cmul_norm2(offset, par, in, w, use_norm), t);
+ }
+ z = kad_sigm(kad_add(t, b));
+ /* r = sigm(x_t * W_r + h_{t-1} * U_r + b_r) */
+ u = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n1);
+ b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1);
+ t = cmul_norm2(offset, par, h0, u, use_norm);
+ if (in) {
+ w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0);
+ t = kad_add(cmul_norm2(offset, par, in, w, use_norm), t);
+ }
+ r = kad_sigm(kad_add(t, b));
+ /* s = tanh(x_t * W_s + (h_{t-1} # r) * U_s + b_s) */
+ u = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n1);
+ b = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 1, n1);
+ t = cmul_norm2(offset, par, kad_mul(r, h0), u, use_norm);
+ if (in) {
+ w = kann_new_leaf2(offset, par, KAD_VAR, 0.0f, 2, n1, n0);
+ t = kad_add(cmul_norm2(offset, par, in, w, use_norm), t);
+ }
+ s = kad_tanh(kad_add(t, b));
+ /* h_t = z # h_{t-1} + (1 - z) # s */
+ out = kad_add(kad_mul(kad_1minus(z), s), kad_mul(z, h0));
+ out->pre = h0;
+ return out;
+}
+
+/********** APIs without offset & par **********/
+
+kad_node_t *kann_new_leaf(uint8_t flag, float x0_01, int n_d, ...)
+{
+ int32_t i, d[KAD_MAX_DIM];
+ va_list ap;
+ va_start(ap, n_d); for (i = 0; i < n_d; ++i) d[i] = va_arg(ap, int); va_end(ap);
+ return kann_new_leaf_array(0, 0, flag, x0_01, n_d, d);
+}
+
+kad_node_t *kann_new_scalar(uint8_t flag, float x) { return kann_new_leaf(flag, x, 0); }
+kad_node_t *kann_new_weight(int n_row, int n_col) { return kann_new_leaf(KAD_VAR, 0.0f, 2, n_row, n_col); }
+kad_node_t *kann_new_vec(int n, float x) { return kann_new_leaf(KAD_VAR, x, 1, n); }
+kad_node_t *kann_new_bias(int n) { return kann_new_vec(n, 0.0f); }
+kad_node_t *kann_new_weight_conv2d(int n_out, int n_in, int k_row, int k_col) { return kann_new_leaf(KAD_VAR, 0.0f, 4, n_out, n_in, k_row, k_col); }
+kad_node_t *kann_new_weight_conv1d(int n_out, int n_in, int kernel_len) { return kann_new_leaf(KAD_VAR, 0.0f, 3, n_out, n_in, kernel_len); }
+
+kad_node_t *kann_layer_input(int n1)
+{
+ kad_node_t *t;
+ t = kad_feed(2, 1, n1);
+ t->ext_flag |= KANN_F_IN;
+ return t;
+}
+
+kad_node_t *kann_layer_dense(kad_node_t *in, int n1) { return kann_layer_dense2(0, 0, in, n1); }
+kad_node_t *kann_layer_dropout(kad_node_t *t, float r) { return kann_layer_dropout2(0, 0, t, r); }
+kad_node_t *kann_layer_layernorm(kad_node_t *in) { return kann_layer_layernorm2(0, 0, in); }
+
+kad_node_t *kann_layer_rnn(kad_node_t *in, int n1, int rnn_flag)
+{
+ kad_node_t *h0;
+ h0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1);
+ h0->x = (float*)calloc(n1, sizeof(float));
+ return kann_layer_rnn2(0, 0, in, h0, rnn_flag);
+}
+
+kad_node_t *kann_layer_gru(kad_node_t *in, int n1, int rnn_flag)
+{
+ kad_node_t *h0;
+ h0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1);
+ h0->x = (float*)calloc(n1, sizeof(float));
+ return kann_layer_gru2(0, 0, in, h0, rnn_flag);
+}
+
+static kad_node_t *kann_cmul_norm(kad_node_t *x, kad_node_t *w)
+{
+ return kann_layer_layernorm(kad_cmul(x, w));
+}
+
+kad_node_t *kann_layer_lstm(kad_node_t *in, int n1, int rnn_flag)
+{
+ int n0;
+ kad_node_t *i, *f, *o, *g, *w, *u, *b, *h0, *c0, *c, *out;
+ kad_node_t *(*cmul)(kad_node_t*, kad_node_t*) = (rnn_flag & KANN_RNN_NORM)? kann_cmul_norm : kad_cmul;
+
+ n0 = in->n_d >= 2? kad_len(in) / in->d[0] : kad_len(in);
+ h0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1);
+ h0->x = (float*)calloc(n1, sizeof(float));
+ c0 = (rnn_flag & KANN_RNN_VAR_H0)? kad_var(0, 0, 2, 1, n1) : kad_const(0, 2, 1, n1);
+ c0->x = (float*)calloc(n1, sizeof(float));
+
+ /* i = sigm(x_t * W_i + h_{t-1} * U_i + b_i) */
+ w = kann_new_weight(n1, n0);
+ u = kann_new_weight(n1, n1);
+ b = kann_new_bias(n1);
+ i = kad_sigm(kad_add(kad_add(cmul(in, w), cmul(h0, u)), b));
+ /* f = sigm(x_t * W_f + h_{t-1} * U_f + b_f) */
+ w = kann_new_weight(n1, n0);
+ u = kann_new_weight(n1, n1);
+ b = kann_new_vec(n1, 1.0f); /* see Jozefowicz et al on using a large bias */
+ f = kad_sigm(kad_add(kad_add(cmul(in, w), cmul(h0, u)), b));
+ /* o = sigm(x_t * W_o + h_{t-1} * U_o + b_o) */
+ w = kann_new_weight(n1, n0);
+ u = kann_new_weight(n1, n1);
+ b = kann_new_bias(n1);
+ o = kad_sigm(kad_add(kad_add(cmul(in, w), cmul(h0, u)), b));
+ /* g = tanh(x_t * W_g + h_{t-1} * U_g + b_g) */
+ w = kann_new_weight(n1, n0);
+ u = kann_new_weight(n1, n1);
+ b = kann_new_bias(n1);
+ g = kad_tanh(kad_add(kad_add(cmul(in, w), cmul(h0, u)), b));
+ /* c_t = c_{t-1} # f + g # i */
+ c = kad_add(kad_mul(f, c0), kad_mul(g, i)); /* can't be kad_mul(c0, f)!!! */
+ c->pre = c0;
+ /* h_t = tanh(c_t) # o */
+ if (rnn_flag & KANN_RNN_NORM) c = kann_layer_layernorm(c); /* see Ba et al (2016) about how to apply layer normalization to LSTM */
+ out = kad_mul(kad_tanh(c), o);
+ out->pre = h0;
+ return out;
+}
+
+kad_node_t *kann_layer_conv2d(kad_node_t *in, int n_flt, int k_rows, int k_cols, int stride_r, int stride_c, int pad_r, int pad_c)
+{
+ kad_node_t *w;
+ w = kann_new_weight_conv2d(n_flt, in->d[1], k_rows, k_cols);
+ return kad_conv2d(in, w, stride_r, stride_c, pad_r, pad_c);
+}
+
+kad_node_t *kann_layer_conv1d(kad_node_t *in, int n_flt, int k_size, int stride, int pad)
+{
+ kad_node_t *w;
+ w = kann_new_weight_conv1d(n_flt, in->d[1], k_size);
+ return kad_conv1d(in, w, stride, pad);
+}
+
+kad_node_t *kann_layer_cost(kad_node_t *t, int n_out, int cost_type)
+{
+ kad_node_t *cost = 0, *truth = 0;
+ assert(cost_type == KANN_C_CEB || cost_type == KANN_C_CEM || cost_type == KANN_C_CEB_NEG || cost_type == KANN_C_MSE);
+ t = kann_layer_dense(t, n_out);
+ truth = kad_feed(2, 1, n_out), truth->ext_flag |= KANN_F_TRUTH;
+
+ if (cost_type == KANN_C_MSE) {
+ cost = kad_mse(t, truth);
+ } else if (cost_type == KANN_C_CEB) {
+ t = kad_sigm(t);
+ cost = kad_ce_bin(t, truth);
+ } else if (cost_type == KANN_C_CEB_NEG) {
+ t = kad_tanh(t);
+ cost = kad_ce_bin_neg(t, truth);
+ } else if (cost_type == KANN_C_CEM) {
+ t = kad_softmax(t);
+ cost = kad_ce_multi(t, truth);
+ }
+ else {
+ assert (0);
+ }
+
+ t->ext_flag |= KANN_F_OUT;
+ cost->ext_flag |= KANN_F_COST;
+
+ return cost;
+}
+
+void kann_shuffle(int n, int *s)
+{
+ int i, j, t;
+ for (i = 0; i < n; ++i) s[i] = i;
+ for (i = n; i > 0; --i) {
+ j = (int)(i * kad_drand(0));
+ t = s[j], s[j] = s[i-1], s[i-1] = t;
+ }
+}
+
+/***************************
+ *** @@MIN: minimization ***
+ ***************************/
+
+#ifdef __SSE__
+#include <xmmintrin.h>
+
+void kann_RMSprop(int n, float h0, const float *h, float decay, const float *g, float *t, float *r)
+{
+ int i, n4 = n>>2<<2;
+ __m128 vh, vg, vr, vt, vd, vd1, tmp, vtiny;
+ vh = _mm_set1_ps(h0);
+ vd = _mm_set1_ps(decay);
+ vd1 = _mm_set1_ps(1.0f - decay);
+ vtiny = _mm_set1_ps(1e-6f);
+ for (i = 0; i < n4; i += 4) {
+ vt = _mm_loadu_ps(&t[i]);
+ vr = _mm_loadu_ps(&r[i]);
+ vg = _mm_loadu_ps(&g[i]);
+ if (h) vh = _mm_loadu_ps(&h[i]);
+ vr = _mm_add_ps(_mm_mul_ps(vd1, _mm_mul_ps(vg, vg)), _mm_mul_ps(vd, vr));
+ _mm_storeu_ps(&r[i], vr);
+ tmp = _mm_sub_ps(vt, _mm_mul_ps(_mm_mul_ps(vh, _mm_rsqrt_ps(_mm_add_ps(vtiny, vr))), vg));
+ _mm_storeu_ps(&t[i], tmp);
+ }
+ for (; i < n; ++i) {
+ r[i] = (1. - decay) * g[i] * g[i] + decay * r[i];
+ t[i] -= (h? h[i] : h0) / sqrtf(1e-6f + r[i]) * g[i];
+ }
+}
+#else
+void kann_RMSprop(int n, float h0, const float *h, float decay, const float *g, float *t, float *r)
+{
+ int i;
+ for (i = 0; i < n; ++i) {
+ float lr = h? h[i] : h0;
+ r[i] = (1.0f - decay) * g[i] * g[i] + decay * r[i];
+ t[i] -= lr / sqrtf(1e-6f + r[i]) * g[i];
+ }
+}
+#endif
+
+float kann_grad_clip(float thres, int n, float *g)
+{
+ int i;
+ double s2 = 0.0;
+ for (i = 0; i < n; ++i)
+ s2 += g[i] * g[i];
+ s2 = sqrt(s2);
+ if (s2 > thres)
+ for (i = 0, s2 = 1.0 / s2; i < n; ++i)
+ g[i] *= (float)s2;
+ return (float)s2 / thres;
+}
+
+/****************************************************************
+ *** @@XY: simpler API for network with a single input/output ***
+ ****************************************************************/
+
+int kann_train_fnn1(kann_t *ann, float lr, int mini_size, int max_epoch,
+ int max_drop_streak, float frac_val, int n,
+ float **_x, float **_y,
+ kann_train_cb cb, void *ud)
+{
+ int i, j, *shuf, n_train, n_val, n_in, n_out, n_var, n_const, drop_streak = 0, min_set = 0;
+ float **x, **y, *x1, *y1, *r, min_val_cost = FLT_MAX, *min_x, *min_c;
+
+ n_in = kann_dim_in(ann);
+ n_out = kann_dim_out(ann);
+ if (n_in < 0 || n_out < 0) return -1;
+ n_var = kann_size_var(ann);
+ n_const = kann_size_const(ann);
+ r = (float*)calloc(n_var, sizeof(float));
+ shuf = (int*)malloc(n * sizeof(int));
+ x = (float**)malloc(n * sizeof(float*));
+ y = (float**)malloc(n * sizeof(float*));
+ kann_shuffle(n, shuf);
+ for (j = 0; j < n; ++j)
+ x[j] = _x[shuf[j]], y[j] = _y[shuf[j]];
+ n_val = (int)(n * frac_val);
+ n_train = n - n_val;
+ min_x = (float*)malloc(n_var * sizeof(float));
+ min_c = (float*)malloc(n_const * sizeof(float));
+
+ x1 = (float*)malloc(n_in * mini_size * sizeof(float));
+ y1 = (float*)malloc(n_out * mini_size * sizeof(float));
+ kann_feed_bind(ann, KANN_F_IN, 0, &x1);
+ kann_feed_bind(ann, KANN_F_TRUTH, 0, &y1);
+
+ for (i = 0; i < max_epoch; ++i) {
+ int n_proc = 0, n_train_err = 0, n_val_err = 0, n_train_base = 0, n_val_base = 0;
+ double train_cost = 0.0, val_cost = 0.0;
+ kann_shuffle(n_train, shuf);
+ kann_switch(ann, 1);
+ while (n_proc < n_train) {
+ int b, c, ms = n_train - n_proc < mini_size? n_train - n_proc : mini_size;
+ for (b = 0; b < ms; ++b) {
+ memcpy(&x1[b*n_in], x[shuf[n_proc+b]], n_in * sizeof(float));
+ memcpy(&y1[b*n_out], y[shuf[n_proc+b]], n_out * sizeof(float));
+ }
+ kann_set_batch_size(ann, ms);
+ train_cost += kann_cost(ann, 0, 1) * ms;
+ c = kann_class_error(ann, &b);
+ n_train_err += c, n_train_base += b;
+ kann_RMSprop(n_var, lr, 0, 0.9f, ann->g, ann->x, r);
+ n_proc += ms;
+ }
+ train_cost /= n_train;
+ kann_switch(ann, 0);
+ n_proc = 0;
+ while (n_proc < n_val) {
+ int b, c, ms = n_val - n_proc < mini_size? n_val - n_proc : mini_size;
+ for (b = 0; b < ms; ++b) {
+ memcpy(&x1[b*n_in], x[n_train+n_proc+b], n_in * sizeof(float));
+ memcpy(&y1[b*n_out], y[n_train+n_proc+b], n_out * sizeof(float));
+ }
+ kann_set_batch_size(ann, ms);
+ val_cost += kann_cost(ann, 0, 0) * ms;
+ c = kann_class_error(ann, &b);
+ n_val_err += c, n_val_base += b;
+ n_proc += ms;
+ }
+ if (n_val > 0) val_cost /= n_val;
+ if (cb) {
+ cb(i + 1, train_cost, val_cost, ud);
+#if 0
+ fprintf(stderr, "epoch: %d; training cost: %g", i+1, train_cost);
+ if (n_train_base) fprintf(stderr, " (class error: %.2f%%)", 100.0f * n_train_err / n_train);
+ if (n_val > 0) {
+ fprintf(stderr, "; validation cost: %g", val_cost);
+ if (n_val_base) fprintf(stderr, " (class error: %.2f%%)", 100.0f * n_val_err / n_val);
+ }
+ fputc('\n', stderr);
+#endif
+ }
+ if (i >= max_drop_streak && n_val > 0) {
+ if (val_cost < min_val_cost) {
+ min_set = 1;
+ memcpy(min_x, ann->x, n_var * sizeof(float));
+ memcpy(min_c, ann->c, n_const * sizeof(float));
+ drop_streak = 0;
+ min_val_cost = (float)val_cost;
+ } else if (++drop_streak >= max_drop_streak)
+ break;
+ }
+ }
+ if (min_set) {
+ memcpy(ann->x, min_x, n_var * sizeof(float));
+ memcpy(ann->c, min_c, n_const * sizeof(float));
+ }
+
+ free(min_c); free(min_x); free(y1); free(x1); free(y); free(x); free(shuf); free(r);
+ return i;
+}
+
+float kann_cost_fnn1(kann_t *ann, int n, float **x, float **y)
+{
+ int n_in, n_out, n_proc = 0, mini_size = 64 < n? 64 : n;
+ float *x1, *y1;
+ double cost = 0.0;
+
+ n_in = kann_dim_in(ann);
+ n_out = kann_dim_out(ann);
+ if (n <= 0 || n_in < 0 || n_out < 0) return 0.0;
+
+ x1 = (float*)malloc(n_in * mini_size * sizeof(float));
+ y1 = (float*)malloc(n_out * mini_size * sizeof(float));
+ kann_feed_bind(ann, KANN_F_IN, 0, &x1);
+ kann_feed_bind(ann, KANN_F_TRUTH, 0, &y1);
+ kann_switch(ann, 0);
+ while (n_proc < n) {
+ int b, ms = n - n_proc < mini_size? n - n_proc : mini_size;
+ for (b = 0; b < ms; ++b) {
+ memcpy(&x1[b*n_in], x[n_proc+b], n_in * sizeof(float));
+ memcpy(&y1[b*n_out], y[n_proc+b], n_out * sizeof(float));
+ }
+ kann_set_batch_size(ann, ms);
+ cost += kann_cost(ann, 0, 0) * ms;
+ n_proc += ms;
+ }
+ free(y1); free(x1);
+ return (float)(cost / n);
+}
+
+const float *kann_apply1(kann_t *a, float *x)
+{
+ int i_out;
+ i_out = kann_find(a, KANN_F_OUT, 0);
+ if (i_out < 0) return 0;
+ kann_set_batch_size(a, 1);
+ kann_feed_bind(a, KANN_F_IN, 0, &x);
+ kad_eval_at(a->n, a->v, i_out);
+ return a->v[i_out]->x;
+}
diff --git a/contrib/kann/kann.h b/contrib/kann/kann.h
new file mode 100644
index 0000000..af0de5f
--- /dev/null
+++ b/contrib/kann/kann.h
@@ -0,0 +1,240 @@
+/*
+ The MIT License
+
+ Copyright (c) 2018-2019 Dana-Farber Cancer Institute
+ 2016-2018 Broad Institute
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+#ifndef KANN_H
+#define KANN_H
+
+#define KANN_VERSION "r536"
+
+#define KANN_F_IN 0x1 /* input */
+#define KANN_F_OUT 0x2 /* output */
+#define KANN_F_TRUTH 0x4 /* truth output */
+#define KANN_F_COST 0x8 /* final cost */
+
+#define KANN_C_CEB 1 /* binary cross-entropy cost, used with sigmoid */
+#define KANN_C_CEM 2 /* multi-class cross-entropy cost, used with softmax */
+#define KANN_C_CEB_NEG 3 /* binary cross-enytopy-like cost, used with tanh */
+#define KANN_C_MSE 4 /* mean square error */
+
+#define KANN_RNN_VAR_H0 0x1 /* take the initial hidden values as variables */
+#define KANN_RNN_NORM 0x2 /* apply layer normalization */
+
+#include "kautodiff.h"
+
+typedef struct {
+ int n; /* number of nodes in the computational graph */
+ kad_node_t **v; /* list of nodes */
+ float *x, *g, *c; /* collated variable values, gradients and constant values */
+ void *mt; /* auxiliary data for multi-threading; NULL if multi-threading disabled */
+} kann_t;
+
+extern int kann_verbose;
+
+#define kann_size_var(a) kad_size_var((a)->n, (a)->v)
+#define kann_size_const(a) kad_size_const((a)->n, (a)->v)
+#define kann_dim_in(a) kann_feed_dim((a), KANN_F_IN, 0)
+#define kann_dim_out(a) kann_feed_dim((a), KANN_F_TRUTH, 0)
+#define kann_srand(seed) kad_srand(0, (seed))
+#define kann_drand() kad_drand(0)
+#define kann_set_batch_size(ann, B) kad_sync_dim((ann)->n, (ann)->v, (B))
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Generate a network from a computational graph
+ *
+ * A network must have at least one scalar cost node (i.e. whose n_d==0). It
+ * may optionally contain other cost nodes or output nodes not leading to the
+ * primary cost node.
+ *
+ * @param cost cost node (must be a scalar, i.e. cost->n_d==0)
+ * @param n_rest number of other nodes without predecessors
+ * @param ... other nodes (of type kad_node_t*) without predecessors
+ *
+ * @return network on success, or NULL otherwise
+ */
+kann_t *kann_new(kad_node_t *cost, int n_rest, ...);
+
+/**
+ * Unroll an RNN
+ *
+ * @param a network
+ * @param len number of unrolls
+ *
+ * @return an unrolled network, or NULL if the network is not an RNN
+ */
+kann_t *kann_unroll(kann_t *a, ...);
+
+kann_t *kann_unroll_array(kann_t *a, int *len);
+kann_t *kann_clone(kann_t *a, int batch_size);
+void kann_delete(kann_t *a); /* delete a network generated by kann_new() or kann_layer_final() */
+void kann_delete_unrolled(kann_t *a); /* delete a network generated by kann_unroll() */
+
+/**
+ * Enable/disable multi-threading (requiring pthread)
+ *
+ * KANN splits a mini-batch to $n_threads mini-mini-batches and puts each of
+ * them on one thread. So far, only kann_cost() takes the advantage of
+ * multi-threading.
+ *
+ * @param ann network
+ * @param n_threads number of threads; <=1 to completely disable multi-threading
+ * @param max_batch_size max mini-batch size; shall no smaller than n_threads
+ */
+void kann_mt(kann_t *ann, int n_threads, int max_batch_size);
+
+/**
+ * Bind float arrays to feed nodes
+ *
+ * @param a network
+ * @param ext_flag required external flags
+ * @param ext_label required external label
+ * @param x pointers (size equal to the number of matching feed nodes)
+ *
+ * @return number of matching feed nodes
+ */
+int kann_feed_bind(kann_t *a, uint32_t ext_flag, int32_t ext_label, float **x);
+
+/**
+ * Compute the cost and optionally gradients
+ *
+ * @param a network
+ * @param cost_label required external label
+ * @param cal_grad whether to compute gradients
+ *
+ * @return cost
+ */
+float kann_cost(kann_t *a, int cost_label, int cal_grad);
+
+int kann_eval(kann_t *a, uint32_t ext_flag, int ext_label);
+int kann_eval_out(kann_t *a);
+int kann_class_error(const kann_t *ann, int *base);
+
+/**
+ * Find a node
+ *
+ * @param a network
+ * @param ext_flag required external flags; set to 0 to match all flags
+ * @param ext_label required external label
+ *
+ * @return >=0 if found; -1 if not found; -2 if found multiple
+ */
+int kann_find(const kann_t *a, uint32_t ext_flag, int32_t ext_label);
+
+/**
+ * Get the size of a feed node, assuming mini-batch size 1
+ *
+ * @param a network
+ * @param ext_flag required external flags
+ * @param ext_label required external label
+ *
+ * @return size>=0; -1 if not found; -2 if found multiple
+ */
+int kann_feed_dim(const kann_t *a, uint32_t ext_flag, int32_t ext_label);
+
+/**
+ * Get an RNN ready for continuous feeding
+ *
+ * @param a network
+ */
+void kann_rnn_start(kann_t *a);
+
+void kann_rnn_end(kann_t *a);
+
+/**
+ * Switch between training and prediction networks (effective only when there are switch nodes)
+ *
+ * @param a network
+ * @param is_train 0 for prediction network and non-zero for training net
+ */
+void kann_switch(kann_t *a, int is_train);
+
+/**
+ * RMSprop update
+ *
+ * @param n number of variables
+ * @param h0 learning rate
+ * @param h per-variable learning rate; NULL if not applicable
+ * @param decay RMSprop decay; use 0.9 if unsure
+ * @param g gradient, of size n
+ * @param t variables to change
+ * @param r memory, of size n
+ */
+void kann_RMSprop(int n, float h0, const float *h, float decay, const float *g, float *t, float *r);
+
+void kann_shuffle(int n, int *s);
+float kann_grad_clip(float thres, int n, float *g);
+
+/* common layers */
+kad_node_t *kann_layer_input(int n1);
+kad_node_t *kann_layer_dense(kad_node_t *in, int n1);
+kad_node_t *kann_layer_dropout(kad_node_t *t, float r);
+kad_node_t *kann_layer_layernorm(kad_node_t *in);
+kad_node_t *kann_layer_rnn(kad_node_t *in, int n1, int rnn_flag);
+kad_node_t *kann_layer_lstm(kad_node_t *in, int n1, int rnn_flag);
+kad_node_t *kann_layer_gru(kad_node_t *in, int n1, int rnn_flag);
+kad_node_t *kann_layer_conv2d(kad_node_t *in, int n_flt, int k_rows, int k_cols, int stride_r, int stride_c, int pad_r, int pad_c);
+kad_node_t *kann_layer_conv1d(kad_node_t *in, int n_flt, int k_size, int stride, int pad);
+kad_node_t *kann_layer_cost(kad_node_t *t, int n_out, int cost_type);
+
+kad_node_t *kann_new_leaf(uint8_t flag, float x0_01, int n_d, ...); /* flag can be KAD_CONST or KAD_VAR */
+kad_node_t *kann_new_scalar(uint8_t flag, float x);
+kad_node_t *kann_new_weight(int n_row, int n_col);
+kad_node_t *kann_new_bias(int n);
+kad_node_t *kann_new_weight_conv2d(int n_out, int n_in, int k_row, int k_col);
+kad_node_t *kann_new_weight_conv1d(int n_out, int n_in, int kernel_len);
+
+kad_node_t *kann_new_leaf_array(int *offset, kad_node_p *par, uint8_t flag, float x0_01, int n_d, int32_t d[KAD_MAX_DIM]);
+
+kad_node_t *kann_new_leaf2(int *offset, kad_node_p *par, uint8_t flag, float x0_01, int n_d, ...);
+kad_node_t *kann_layer_dense2(int *offset, kad_node_p *par, kad_node_t *in, int n1);
+kad_node_t *kann_layer_dropout2(int *offset, kad_node_p *par, kad_node_t *t, float r);
+kad_node_t *kann_layer_layernorm2(int *offset, kad_node_t **par, kad_node_t *in);
+kad_node_t *kann_layer_rnn2(int *offset, kad_node_t **par, kad_node_t *in, kad_node_t *h0, int rnn_flag);
+kad_node_t *kann_layer_gru2(int *offset, kad_node_t **par, kad_node_t *in, kad_node_t *h0, int rnn_flag);
+
+/* operations on network with a single input node and a single output node */
+typedef void (*kann_train_cb)(int iter, float train_cost, float val_cost, void *ud);
+int kann_train_fnn1(kann_t *ann, float lr, int mini_size, int max_epoch,
+ int max_drop_streak, float frac_val, int n,
+ float **_x, float **_y, kann_train_cb cb, void *ud);
+float kann_cost_fnn1(kann_t *a, int n, float **x, float **y);
+const float *kann_apply1(kann_t *a, float *x);
+
+/* model I/O */
+void kann_save_fp(FILE *fp, kann_t *ann);
+void kann_save(const char *fn, kann_t *ann);
+kann_t *kann_load_fp(FILE *fp);
+kann_t *kann_load(const char *fn);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/kann/kautodiff.c b/contrib/kann/kautodiff.c
new file mode 100644
index 0000000..d05cc00
--- /dev/null
+++ b/contrib/kann/kautodiff.c
@@ -0,0 +1,2460 @@
+#include "config.h"
+
+#include <stdlib.h>
+#include <assert.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+#include <math.h>
+#include "kautodiff.h"
+#include "blas-config.h"
+
+typedef struct {
+ uint64_t s[2];
+ double n_gset;
+ int n_iset;
+ volatile int lock;
+} kad_rng_t;
+
+/**********************
+ * Graph construction *
+ **********************/
+
+static inline kad_node_t *kad_new_core(int n_d, int op, int n_child)
+{
+ kad_node_t *s;
+ if (n_d >= KAD_MAX_DIM) return 0;
+ s = (kad_node_t*)calloc(1, sizeof(kad_node_t));
+ s->n_d = n_d, s->op = op, s->n_child = n_child;
+ if (s->n_child) s->child = (kad_node_t**)calloc(s->n_child, sizeof(kad_node_t*));
+ return s;
+}
+
+static inline kad_node_t *kad_vleaf(uint8_t flag, float *x, float *g, int n_d, va_list ap)
+{
+ int i;
+ kad_node_t *p;
+ if (n_d > KAD_MAX_DIM) return 0;
+ p = (kad_node_t*)calloc(1, sizeof(kad_node_t));
+ p->n_d = n_d;
+ for (i = 0; i < n_d; ++i)
+ p->d[i] = va_arg(ap, int32_t);
+ p->x = x, p->g = g, p->flag = flag;
+ return p;
+}
+
+kad_node_t *kad_const(float *x, int n_d, ...)
+{
+ kad_node_t *p;
+ va_list ap;
+ va_start(ap, n_d); p = kad_vleaf(KAD_CONST, x, 0, n_d, ap); va_end(ap);
+ return p;
+}
+
+kad_node_t *kad_feed(int n_d, ...)
+{
+ kad_node_t *p;
+ va_list ap;
+ va_start(ap, n_d); p = kad_vleaf(0, 0, 0, n_d, ap); va_end(ap);
+ return p;
+}
+
+kad_node_t *kad_var(float *x, float *g, int n_d, ...)
+{
+ kad_node_t *p;
+ va_list ap;
+ va_start(ap, n_d); p = kad_vleaf(KAD_VAR, x, g, n_d, ap); va_end(ap);
+ return p;
+}
+
+static inline kad_node_t *kad_finalize_node(kad_node_t *s) /* a helper function */
+{
+ int i;
+ if (kad_op_list[s->op](s, KAD_SYNC_DIM) < 0) { /* check dimension */
+ if (s->ptr) free(s->ptr);
+ free(s->child); free(s);
+ return 0;
+ }
+ for (i = 0; i < s->n_child; ++i)
+ if (kad_is_back(s->child[i]))
+ break;
+ if (i < s->n_child) s->flag |= KAD_VAR;
+ return s;
+}
+
+/********** Simple arithmetic **********/
+
+static inline kad_node_t *kad_op2_core(int op, kad_node_t *x, kad_node_t *y)
+{
+ kad_node_t *s;
+ s = kad_new_core(0, op, 2);
+ s->child[0] = x, s->child[1] = y;
+ return kad_finalize_node(s);
+}
+
+static inline kad_node_t *kad_op1_core(int op, kad_node_t *x)
+{
+ kad_node_t *s;
+ s = kad_new_core(0, op, 1);
+ s->child[0] = x;
+ return kad_finalize_node(s);
+}
+
+#define KAD_FUNC_OP2(fname, op) kad_node_t *fname(kad_node_t *x, kad_node_t *y) { return kad_op2_core((op), x, y); }
+
+KAD_FUNC_OP2(kad_add, 1)
+KAD_FUNC_OP2(kad_sub, 23)
+KAD_FUNC_OP2(kad_mul, 2)
+KAD_FUNC_OP2(kad_cmul, 3)
+KAD_FUNC_OP2(kad_matmul, 9)
+KAD_FUNC_OP2(kad_ce_multi, 13)
+KAD_FUNC_OP2(kad_ce_bin, 22)
+KAD_FUNC_OP2(kad_ce_bin_neg, 4)
+KAD_FUNC_OP2(kad_mse, 29)
+
+#define KAD_FUNC_OP1(fname, op) kad_node_t *fname(kad_node_t *x) { return kad_op1_core((op), x); }
+
+KAD_FUNC_OP1(kad_log, 27)
+KAD_FUNC_OP1(kad_exp, 33)
+KAD_FUNC_OP1(kad_sin, 34)
+KAD_FUNC_OP1(kad_square, 5)
+KAD_FUNC_OP1(kad_sigm, 6)
+KAD_FUNC_OP1(kad_tanh, 7)
+KAD_FUNC_OP1(kad_relu, 8)
+KAD_FUNC_OP1(kad_1minus, 11)
+KAD_FUNC_OP1(kad_softmax, 14)
+KAD_FUNC_OP1(kad_stdnorm, 32)
+
+kad_node_t *kad_ce_multi_weighted(kad_node_t *pred, kad_node_t *truth, kad_node_t *weight)
+{
+ kad_node_t *s;
+ s = kad_new_core(0, 13, 3);
+ s->child[0] = pred, s->child[1] = truth, s->child[2] = weight;
+ return kad_finalize_node(s);
+}
+
+/********** Convolution **********/
+
+/* compute output dimension and padding sizes on both sides */
+static inline int conv_find_par(int in_size, int kernel_size, int stride, int pad0, int *new_pad0, int *new_pad1)
+{
+ int out_size, pad_both;
+ /* key equation: out_size = (in_size - kernel_size + pad_both) / stride + 1 */
+ if (pad0 == KAD_PAD_SAME && stride == 1) out_size = in_size;
+ else out_size = (in_size - kernel_size + (pad0 > 0? pad0 : 0) + stride - 1) / stride + 1;
+ pad_both = (out_size - 1) * stride + kernel_size - in_size;
+ *new_pad0 = pad_both / 2;
+ *new_pad1 = pad_both - *new_pad0;
+ return out_size;
+}
+
+typedef struct {
+ int kernel_size, stride, pad[2];
+} conv_conf_t;
+
+static inline conv_conf_t *conv2d_gen_aux(int in_row, int in_col, int kernel_r, int kernel_c, int stride_r, int stride_c, int top_pad, int left_pad)
+{
+ conv_conf_t *cnn;
+ cnn = (conv_conf_t*)calloc(2, sizeof(conv_conf_t));
+ cnn[0].kernel_size = kernel_r, cnn[0].stride = stride_r;
+ cnn[1].kernel_size = kernel_c, cnn[1].stride = stride_c;
+ conv_find_par(in_row, kernel_r, stride_r, top_pad, &cnn[0].pad[0], &cnn[0].pad[1]);
+ conv_find_par(in_col, kernel_c, stride_c, left_pad, &cnn[1].pad[0], &cnn[1].pad[1]);
+ return cnn;
+}
+
+kad_node_t *kad_conv2d(kad_node_t *x, kad_node_t *w, int stride_r, int stride_c, int top_pad, int left_pad)
+{
+ kad_node_t *s;
+ if (x->n_d != 4 || w->n_d != 4) return 0;
+ s = kad_new_core(0, 16, 2);
+ s->child[0] = x, s->child[1] = w;
+ s->ptr = conv2d_gen_aux(x->d[2], x->d[3], w->d[2], w->d[3], stride_r, stride_c, top_pad, left_pad);
+ s->ptr_size = sizeof(conv_conf_t) * 2;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_max2d(kad_node_t *x, int kernel_r, int kernel_c, int stride_r, int stride_c, int top_pad, int left_pad)
+{
+ kad_node_t *s;
+ if (x->n_d != 4) return 0;
+ s = kad_new_core(0, 17, 1);
+ s->child[0] = x;
+ s->ptr = conv2d_gen_aux(x->d[2], x->d[3], kernel_r, kernel_c, stride_r, stride_c, top_pad, left_pad);
+ s->ptr_size = sizeof(conv_conf_t) * 2;
+ return kad_finalize_node(s);
+}
+
+static inline conv_conf_t *conv1d_gen_aux(int in_col, int kernel_c, int stride_c, int left_pad)
+{
+ conv_conf_t *cnn;
+ cnn = (conv_conf_t*)calloc(1, sizeof(conv_conf_t));
+ cnn->kernel_size = kernel_c, cnn->stride = stride_c;
+ conv_find_par(in_col, kernel_c, stride_c, left_pad, &cnn->pad[0], &cnn->pad[1]);
+ return cnn;
+}
+
+kad_node_t *kad_conv1d(kad_node_t *x, kad_node_t *w, int stride, int left_pad)
+{
+ kad_node_t *s;
+ if (x->n_d != 3 || w->n_d != 3) return 0;
+ s = kad_new_core(0, 18, 2);
+ s->child[0] = x, s->child[1] = w;
+ s->ptr = conv1d_gen_aux(x->d[2], w->d[2], stride, left_pad);
+ s->ptr_size = sizeof(conv_conf_t);
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_max1d(kad_node_t *x, int kernel_size, int stride, int left_pad)
+{
+ kad_node_t *s;
+ if (x->n_d != 3) return 0;
+ s = kad_new_core(0, 19, 1);
+ s->child[0] = x;
+ s->ptr = conv1d_gen_aux(x->d[2], kernel_size, stride, left_pad);
+ s->ptr_size = sizeof(conv_conf_t);
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_avg1d(kad_node_t *x, int kernel_size, int stride, int left_pad)
+{
+ kad_node_t *s;
+ if (x->n_d != 3) return 0;
+ s = kad_new_core(0, 28, 1);
+ s->child[0] = x;
+ s->ptr = conv1d_gen_aux(x->d[2], kernel_size, stride, left_pad);
+ s->ptr_size = sizeof(conv_conf_t);
+ return kad_finalize_node(s);
+}
+
+/********** Multi-node pooling **********/
+
+static kad_node_t *kad_pooling_general(int op, int n, kad_node_t **x)
+{
+ int i;
+ kad_node_t *s;
+ s = kad_new_core(0, op, n);
+ s->flag |= KAD_POOL;
+ for (i = 0; i < n; ++i)
+ s->child[i] = x[i];
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_avg(int n, kad_node_t **x) { return kad_pooling_general(10, n, x); }
+kad_node_t *kad_max(int n, kad_node_t **x) { return kad_pooling_general(21, n, x); }
+kad_node_t *kad_stack(int n, kad_node_t **x) { return kad_pooling_general(35, n, x); }
+
+kad_node_t *kad_select(int n, kad_node_t **x, int which)
+{
+ kad_node_t *s;
+ int32_t i, *aux;
+ aux = (int32_t*)calloc(1, 4);
+ *aux = which;
+ s = kad_new_core(0, 12, n);
+ for (i = 0; i < n; ++i) s->child[i] = x[i];
+ s->flag |= KAD_POOL, s->ptr = aux, s->ptr_size = 4;
+ return kad_finalize_node(s);
+}
+
+/********** Dimension reduction **********/
+
+static kad_node_t *kad_reduce_general(int op, kad_node_t *x, int axis)
+{
+ kad_node_t *s;
+ int32_t *aux;
+ aux = (int32_t*)malloc(4);
+ aux[0] = axis;
+ s = kad_new_core(0, op, 1);
+ s->child[0] = x;
+ s->ptr = aux, s->ptr_size = 4;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_reduce_sum(kad_node_t *x, int axis) { return kad_reduce_general(25, x, axis); }
+kad_node_t *kad_reduce_mean(kad_node_t *x, int axis) { return kad_reduce_general(26, x, axis); }
+
+/********** Sampling related **********/
+
+kad_node_t *kad_dropout(kad_node_t *x, kad_node_t *y)
+{
+ kad_node_t *z;
+ z = kad_op2_core(15, x, y);
+ z->ptr = kad_rng(), z->ptr_size = sizeof(kad_rng_t);
+ return z;
+}
+
+kad_node_t *kad_sample_normal(kad_node_t *x)
+{
+ kad_node_t *z;
+ z = kad_op1_core(24, x);
+ z->ptr = kad_rng(), z->ptr_size = sizeof(kad_rng_t);
+ return z;
+}
+
+/********** Miscellaneous **********/
+
+kad_node_t *kad_slice(kad_node_t *x, int axis, int start, int end)
+{
+ kad_node_t *s;
+ int32_t *aux;
+ if (end < start || start < 0) return 0;
+ aux = (int32_t*)malloc(3 * 4);
+ aux[0] = axis, aux[1] = start, aux[2] = end;
+ s = kad_new_core(0, 20, 1);
+ s->child[0] = x;
+ s->ptr = aux, s->ptr_size = 3 * 4;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_concat_array(int axis, int n, kad_node_t **p)
+{
+ kad_node_t *s;
+ int32_t i, *aux;
+ aux = (int32_t*)malloc(4);
+ aux[0] = axis;
+ s = kad_new_core(0, 31, n);
+ for (i = 0; i < n; ++i)
+ s->child[i] = p[i];
+ s->ptr = aux, s->ptr_size = 4;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_concat(int axis, int n, ...)
+{
+ int i;
+ kad_node_t **p, *s;
+ va_list ap;
+ p = (kad_node_t**)malloc(n * sizeof(kad_node_t*));
+ va_start(ap, n);
+ for (i = 0; i < n; ++i) p[i] = va_arg(ap, kad_node_p);
+ va_end(ap);
+ s = kad_concat_array(axis, n, p);
+ free(p);
+ return s;
+}
+
+kad_node_t *kad_reshape(kad_node_t *x, int n_d, int *d)
+{
+ kad_node_t *s;
+ int32_t i, *aux = 0;
+ if (n_d > 0) {
+ aux = (int32_t*)malloc(n_d * 4);
+ for (i = 0; i < n_d; ++i) aux[i] = d? d[i] : -1;
+ }
+ s = kad_new_core(0, 30, 1);
+ s->child[0] = x, s->ptr = aux, s->ptr_size = n_d * 4;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_reverse(kad_node_t *x, int axis)
+{
+ kad_node_t *s;
+ int32_t *aux;
+ aux = (int32_t*)malloc(4);
+ *aux = axis;
+ s = kad_new_core(0, 36, 1);
+ s->child[0] = x, s->ptr = aux, s->ptr_size = 4;
+ return kad_finalize_node(s);
+}
+
+kad_node_t *kad_switch(int n, kad_node_t **p)
+{
+ kad_node_t *s;
+ int32_t i, *aux;
+ aux = (int32_t*)calloc(1, 4);
+ s = kad_new_core(0, 12, n);
+ for (i = 0; i < n; ++i)
+ s->child[i] = p[i];
+ s->ptr = aux, s->ptr_size = 4;
+ return kad_finalize_node(s);
+}
+
+/***********************
+ * Graph linearization *
+ ***********************/
+
+static void kad_mark_back(int n, kad_node_t **v)
+{
+ int i, j;
+ for (i = 0; i < n; ++i) {
+ if (v[i]->n_child == 0) continue;
+ for (j = 0; j < v[i]->n_child; ++j)
+ if (kad_is_back(v[i]->child[j]))
+ break;
+ if (j < v[i]->n_child) v[i]->flag |= KAD_VAR;
+ else v[i]->flag &= ~KAD_VAR;
+ }
+}
+
+static void kad_allocate_internal(int n, kad_node_t **v)
+{
+ int i;
+ kad_mark_back(n, v);
+ for (i = 0; i < n; ++i) {
+ kad_node_t *p = v[i];
+ if (p->n_child == 0) continue;
+ p->x = (float*)realloc(p->x, kad_len(p) * sizeof(float));
+ if (kad_is_back(p)) {
+ p->g = (float*)realloc(p->g, kad_len(p) * sizeof(float));
+ kad_op_list[p->op](p, KAD_ALLOC);
+ }
+ }
+}
+
+int kad_sync_dim(int n, kad_node_t **v, int batch_size)
+{
+ int i, req_alloc = 0, req_sync = 0, old_size = 0;
+ for (i = 0; i < n; ++i) {
+ if (kad_is_feed(v[i])) {
+ old_size = v[i]->d[0]; /* TODO: check if all feeds have the same batch size */
+ if (batch_size > 0 && v[i]->d[0] != batch_size)
+ v[i]->d[0] = batch_size, req_sync = 1;
+ } else if (v[i]->n_child > 0 && req_sync)
+ kad_op_list[v[i]->op](v[i], KAD_SYNC_DIM);
+ }
+ if (old_size < batch_size) req_alloc = 1;
+ for (i = 0; i < n; ++i)
+ if (v[i]->n_child > 0 && v[i]->x == 0) req_alloc = 1;
+ if (req_alloc) kad_allocate_internal(n, v);
+ return batch_size > 0? batch_size : old_size;
+}
+
+#define kvec_t(type) struct { size_t n, m; type *a; }
+
+#define kv_pop(v) ((v).a[--(v).n])
+
+#define kv_push(type, v, x) do { \
+ if ((v).n == (v).m) { \
+ (v).m = (v).m? (v).m<<1 : 2; \
+ (v).a = (type*)realloc((v).a, sizeof(type) * (v).m); \
+ } \
+ (v).a[(v).n++] = (x); \
+ } while (0)
+
+/* IMPORTANT: kad_node_t::tmp MUST BE set to zero before calling this function */
+kad_node_t **kad_compile_array(int *n_node, int n_roots, kad_node_t **roots)
+{
+ int i;
+ kvec_t(kad_node_p) stack = {0,0,0}, a = {0,0,0};
+
+ /* generate kad_node_t::tmp, the count of the parent nodes; shifted by 1; lowest bit to detect fake roots */
+ for (i = 0; i < n_roots; ++i) {
+ roots[i]->tmp = 1; /* mark the root */
+ kv_push(kad_node_p, stack, roots[i]);
+ }
+ while (stack.n) {
+ kad_node_t *p = kv_pop(stack);
+ for (i = 0; i < p->n_child; ++i) {
+ kad_node_t *q = p->child[i];
+ if (q->tmp == 0) kv_push(kad_node_p, stack, q);
+ q->tmp += 1<<1;
+ }
+ }
+
+ /* topological sorting (Kahn's algorithm) */
+ for (i = 0; i < n_roots; ++i)
+ if (roots[i]->tmp>>1 == 0) /* if roots[i]->tmp>>1 != 0, it is not a real root */
+ kv_push(kad_node_p, stack, roots[i]);
+ while (stack.n) {
+ kad_node_t *p = kv_pop(stack);
+ kv_push(kad_node_p, a, p);
+ for (i = 0; i < p->n_child; ++i) {
+ p->child[i]->tmp -= 1<<1;
+ if (p->child[i]->tmp>>1 == 0)
+ kv_push(kad_node_p, stack, p->child[i]);
+ }
+ }
+ free(stack.a);
+ for (i = 0; i < (int)a.n; ++i) { /* check cycles; no cycles if constructed with kad_add() etc */
+ assert(a.a[i]->tmp>>1 == 0);
+ a.a[i]->tmp = 0;
+ }
+
+ /* reverse */
+ for (i = 0; i < (int)a.n>>1; ++i) { /* reverse a.a[] */
+ kad_node_p t;
+ t = a.a[i], a.a[i] = a.a[a.n-1-i], a.a[a.n-1-i] = t;
+ }
+ kad_allocate_internal(a.n, a.a);
+
+ *n_node = a.n;
+ return a.a;
+}
+
+kad_node_t **kad_compile(int *n_node, int n_roots, ...)
+{
+ int i;
+ kad_node_t **roots, **ret;
+ va_list ap;
+
+ roots = (kad_node_t**)malloc(n_roots * sizeof(kad_node_t*));
+ va_start(ap, n_roots);
+ for (i = 0; i < n_roots; ++i) roots[i] = va_arg(ap, kad_node_p);
+ va_end(ap);
+ ret = kad_compile_array(n_node, n_roots, roots);
+ free(roots);
+ return ret;
+}
+
+/************************************
+ * Miscellaneous on compiled graphs *
+ ************************************/
+
+void kad_delete(int n, kad_node_t **a)
+{
+ int i;
+ for (i = 0; i < n; ++i) {
+ kad_node_t *p = a[i];
+ if (p->n_child) {
+ free(p->x); free(p->g);
+ }
+ free(p->child); free(p->ptr); free(p->gtmp); free(p);
+ }
+ free(a);
+}
+
+int kad_size_var(int n, kad_node_t *const* v)
+{
+ int c, i;
+ for (i = c = 0; i < n; ++i)
+ if (kad_is_var(v[i]))
+ c += kad_len(v[i]);
+ return c;
+}
+
+int kad_size_const(int n, kad_node_t *const* v)
+{
+ int c, i;
+ for (i = c = 0; i < n; ++i)
+ if (kad_is_const(v[i]))
+ c += kad_len(v[i]);
+ return c;
+}
+
+/**********************************
+ * Computate values and gradients *
+ **********************************/
+
+static void kad_propagate_marks(int n, kad_node_t **a)
+{
+ int i, j;
+ for (i = n - 1; i >= 0; --i) {
+ kad_node_t *p = a[i];
+ if (p->tmp > 0) {
+ if (kad_is_switch(p)) {
+ int32_t *aux = (int32_t*)p->ptr;
+ if (p->child[*aux]->tmp == 0)
+ p->child[*aux]->tmp = 1;
+ } else {
+ for (j = 0; j < p->n_child; ++j)
+ if (p->child[j]->tmp == 0)
+ p->child[j]->tmp = 1;
+ }
+ }
+ }
+}
+
+void kad_eval_marked(int n, kad_node_t **a)
+{
+ int i;
+ kad_propagate_marks(n, a);
+ for (i = 0; i < n; ++i)
+ if (a[i]->n_child && a[i]->tmp > 0)
+ kad_op_list[a[i]->op](a[i], KAD_FORWARD);
+ for (i = 0; i < n; ++i) a[i]->tmp = 0;
+}
+
+const float *kad_eval_at(int n, kad_node_t **a, int from)
+{
+ int i;
+ if (from < 0 || from >= n) from = n - 1;
+ for (i = 0; i < n; ++i) a[i]->tmp = (i == from);
+ kad_eval_marked(n, a);
+ return a[from]->x;
+}
+
+void kad_grad(int n, kad_node_t **a, int from)
+{
+ int i;
+ if (from < 0 || from >= n) from = n - 1;
+ assert(a[from]->n_d == 0);
+ for (i = 0; i < n; ++i) a[i]->tmp = (i == from);
+ kad_propagate_marks(n, a);
+ for (i = 0; i <= from; ++i) /* set all grandients to zero */
+ if (a[i]->g && a[i]->tmp > 0)
+ memset(a[i]->g, 0, kad_len(a[i]) * sizeof(float));
+ for (i = from, a[i]->g[0] = 1.0f; i >= 0; --i) /* backprop */
+ if (a[i]->n_child && a[i]->tmp > 0)
+ kad_op_list[a[i]->op](a[i], KAD_BACKWARD);
+ for (i = 0; i <= from; ++i) a[i]->tmp = 0;
+}
+
+/***********************
+ * Load and save graph *
+ ***********************/
+
+static void kad_save1(FILE *fp, const kad_node_t *p)
+{
+ fwrite(&p->ext_label, 4, 1, fp);
+ fwrite(&p->ext_flag, 4, 1, fp);
+ fwrite(&p->flag, 1, 1, fp);
+ fwrite(&p->n_child, 4, 1, fp);
+ if (p->n_child) {
+ int32_t j, pre = p->pre? p->pre->tmp : -1;
+ fwrite(&p->op, 2, 1, fp);
+ for (j = 0; j < p->n_child; ++j)
+ fwrite(&p->child[j]->tmp, 4, 1, fp);
+ fwrite(&pre, 4, 1, fp);
+ fwrite(&p->ptr_size, 4, 1, fp);
+ if (p->ptr_size > 0 && p->ptr)
+ fwrite(p->ptr, p->ptr_size, 1, fp);
+ } else {
+ fwrite(&p->n_d, 1, 1, fp);
+ if (p->n_d) fwrite(p->d, 4, p->n_d, fp);
+ }
+}
+
+static kad_node_t *kad_load1(FILE *fp, kad_node_t **node)
+{
+ kad_node_t *p;
+ p = (kad_node_t*)calloc(1, sizeof(kad_node_t));
+ (void) !fread(&p->ext_label, 4, 1, fp);
+ (void) !fread(&p->ext_flag, 4, 1, fp);
+ (void) !fread(&p->flag, 1, 1, fp);
+ (void) !fread(&p->n_child, 4, 1, fp);
+ if (p->n_child) {
+ int32_t j, k;
+ p->child = (kad_node_t**)calloc(p->n_child, sizeof(kad_node_t*));
+ (void) !fread(&p->op, 2, 1, fp);
+ for (j = 0; j < p->n_child; ++j) {
+ (void) !fread(&k, 4, 1, fp);
+ p->child[j] = node? node[k] : 0;
+ }
+ (void) !fread(&k, 4, 1, fp);
+ if (k >= 0) p->pre = node[k];
+ (void) !fread(&p->ptr_size, 4, 1, fp);
+ if (p->ptr_size > 0) {
+ p->ptr = malloc(p->ptr_size);
+ (void) !fread(p->ptr, p->ptr_size, 1, fp);
+ }
+ } else {
+ (void) !fread(&p->n_d, 1, 1, fp);
+ if (p->n_d) (void) !fread(p->d, 4, p->n_d, fp);
+ }
+ return p;
+}
+
+int kad_save(FILE *fp, int n_node, kad_node_t **node)
+{
+ int32_t i, k = n_node;
+ fwrite(&k, 4, 1, fp);
+ for (i = 0; i < n_node; ++i) node[i]->tmp = i;
+ for (i = 0; i < n_node; ++i) kad_save1(fp, node[i]);
+ for (i = 0; i < n_node; ++i) node[i]->tmp = 0;
+ return 0;
+}
+
+kad_node_t **kad_load(FILE *fp, int *_n_node)
+{
+ int32_t i, n_node;
+ kad_node_t **node;
+ (void) !fread(&n_node, 4, 1, fp);
+ node = (kad_node_t**)malloc(n_node * sizeof(kad_node_t*));
+ for (i = 0; i < n_node; ++i) {
+ kad_node_t *p;
+ p = node[i] = kad_load1(fp, node);
+ if (p->n_child) {
+ kad_op_list[p->op](p, KAD_ALLOC);
+ kad_op_list[p->op](p, KAD_SYNC_DIM);
+ }
+ }
+ *_n_node = n_node;
+ kad_mark_back(n_node, node);
+ return node;
+}
+
+/***************
+ * Graph clone *
+ ***************/
+
+static inline kad_node_t *kad_dup1(const kad_node_t *p)
+{
+ kad_node_t *q;
+ q = (kad_node_t*)malloc(sizeof(kad_node_t));
+ memcpy(q, p, sizeof(kad_node_t));
+ q->pre = 0, q->tmp = 0, q->gtmp = 0;
+ if (p->ptr && p->ptr_size > 0) {
+ if (kad_use_rng(p) && !(p->flag & KAD_SHARE_RNG) && p->ptr_size == sizeof(kad_rng_t)) {
+ q->ptr = kad_rng(); /* each time step uses a different RNG */
+ } else {
+ q->ptr = malloc(p->ptr_size);
+ memcpy(q->ptr, p->ptr, p->ptr_size);
+ }
+ }
+ if (q->n_child) {
+ q->x = q->g = 0;
+ q->child = (kad_node_t**)calloc(q->n_child, sizeof(kad_node_t*));
+ }
+ return q;
+}
+
+kad_node_t **kad_clone(int n, kad_node_t **v, int batch_size)
+{
+ int i, j;
+ kad_node_t **u;
+ u = (kad_node_t**)calloc(n, sizeof(kad_node_t*));
+ for (i = 0; i < n; ++i) v[i]->tmp = i;
+ for (i = 0; i < n; ++i) {
+ kad_node_t *p = v[i], *q;
+ q = u[i] = kad_dup1(p);
+ if (p->pre) q->pre = u[p->pre->tmp];
+ if (p->n_child) {
+ for (j = 0; j < p->n_child; ++j)
+ q->child[j] = u[p->child[j]->tmp];
+ } else if (!kad_is_feed(p)) {
+ q->x = (float*)malloc(kad_len(p) * sizeof(float));
+ memcpy(q->x, p->x, kad_len(p) * sizeof(float));
+ q->g = 0;
+ }
+ }
+ for (i = 0; i < n; ++i) v[i]->tmp = 0;
+ kad_sync_dim(n, u, batch_size); /* this will allocate x[] and g[] at internal nodes */
+ return u;
+}
+
+/**************
+ * Unroll RNN *
+ **************/
+
+typedef struct {
+ int32_t n, m;
+ kad_node_t **v;
+} nodes_t;
+
+static inline void push_nodes(nodes_t *w, kad_node_t *p)
+{
+ if (w->n == w->m) {
+ w->m = w->m? w->m<<1 : 16;
+ w->v = (kad_node_t**)realloc(w->v, w->m * sizeof(kad_node_t*));
+ }
+ w->v[w->n++] = p;
+}
+
+static void kad_unroll_helper(int n_v, kad_node_t **v, int i_pivot, kad_node_t **t, int len, nodes_t *w)
+{
+ int i, j, l;
+ uint8_t *flag;
+ kad_node_t **aux;
+
+ assert(kad_is_pivot(v[i_pivot]) && t[i_pivot] == 0);
+ t[i_pivot] = kad_dup1(v[i_pivot]);
+ t[i_pivot]->n_child = len;
+ t[i_pivot]->child = (kad_node_t**)realloc(t[i_pivot]->child, len * sizeof(kad_node_t*));
+
+ flag = (uint8_t*)calloc(n_v, 1);
+ for (i = i_pivot, flag[i] = 16; i >= 0; --i) {
+ if (i < i_pivot && kad_is_pivot(v[i])) continue; /* don't trespass other pivots */
+ if (flag[i]&16) /* flag 16: nodes to unroll */
+ for (j = 0; j < v[i]->n_child; ++j)
+ flag[v[i]->child[j]->tmp] = 16;
+ }
+ for (i = 0; i < i_pivot; ++i) {
+ if (!(flag[i]&16)) continue;
+ if (kad_is_var(v[i]) || kad_is_const(v[i]) || kad_is_pivot(v[i])) flag[i] |= 1; /* external nodes that should not be duplicated */
+ if (v[i]->pre) flag[v[i]->pre->tmp] |= 2;
+ }
+ flag[v[i_pivot]->child[0]->tmp] |= 4;
+ aux = (kad_node_t**)calloc(n_v, sizeof(kad_node_t*));
+ for (l = 0; l < len; ++l) {
+ for (i = 0; i < i_pivot; ++i) {
+ if (!(flag[i]&16) || ((flag[i]&3) && t[i])) continue;
+ t[i] = kad_dup1(v[i]);
+ if (v[i]->n_child)
+ for (j = 0; j < v[i]->n_child; ++j)
+ t[i]->child[j] = t[v[i]->child[j]->tmp];
+ if (flag[i]&4) t[i_pivot]->child[l] = t[i];
+ if (l == 0 && (flag[i]&2)) aux[i] = t[i];
+ if (v[i]->pre) {
+ t[v[i]->pre->tmp] = t[i];
+ if (l == len - 1) t[i]->pre = aux[v[i]->pre->tmp]; /* this forms a cycle! */
+ }
+ push_nodes(w, t[i]);
+ }
+ }
+ push_nodes(w, t[i_pivot]);
+ free(aux); free(flag);
+}
+
+int kad_n_pivots(int n_v, kad_node_t **v)
+{
+ int i, n_pivots = 0;
+ for (i = 0; i < n_v; ++i)
+ if (kad_is_pivot(v[i])) ++n_pivots;
+ return n_pivots;
+}
+
+kad_node_t **kad_unroll(int n_v, kad_node_t **v, int *new_n, int *len)
+{
+ int i, j, n_pivots = 0;
+ kad_node_t **t;
+ nodes_t w = {0,0,0};
+
+ t = (kad_node_t**)calloc(n_v, sizeof(kad_node_t*));
+ n_pivots = kad_n_pivots(n_v, v);
+ for (i = 0; i < n_v; ++i) v[i]->tmp = i;
+ if (n_pivots) {
+ int k, *i_pivots;
+ i_pivots = (int*)calloc(n_pivots, sizeof(int));
+ for (i = k = 0; i < n_v; ++i) /* collect pivots */
+ if (kad_is_pivot(v[i])) i_pivots[k++] = i;
+ for (i = 0; i < n_pivots; ++i) /* unroll each pivot, from the lowest to the highest */
+ kad_unroll_helper(n_v, v, i_pivots[i], t, len[i], &w);
+ free(i_pivots);
+ }
+ for (i = 0; i < n_v; ++i) { /* copy over the rest of nodes */
+ if (t[i]) continue;
+ t[i] = kad_dup1(v[i]);
+ if (v[i]->n_child)
+ for (j = 0; j < v[i]->n_child; ++j)
+ t[i]->child[j] = t[v[i]->child[j]->tmp];
+ push_nodes(&w, t[i]);
+ }
+ free(t);
+ for (i = 0; i < n_v; ++i) v[i]->tmp = 0;
+ for (i = 0; i < w.n; ++i) /* stack may change the output dimension */
+ if (w.v[i]->n_child > 0)
+ kad_op_list[w.v[i]->op](w.v[i], KAD_SYNC_DIM);
+ kad_allocate_internal(w.n, w.v);
+ *new_n = w.n;
+ return w.v;
+}
+
+/********************************
+ * Vector and matrix operations *
+ ********************************/
+
+#ifdef __SSE__
+#include <xmmintrin.h>
+
+static inline float kad_sdot(int n, const float *x, const float *y) /* BLAS sdot using SSE */
+{
+ int i, n8 = n>>3<<3;
+ __m128 vs1, vs2;
+ float s, t[4];
+ vs1 = _mm_setzero_ps();
+ vs2 = _mm_setzero_ps();
+ for (i = 0; i < n8; i += 8) {
+ __m128 vx1, vx2, vy1, vy2;
+ vx1 = _mm_loadu_ps(&x[i]);
+ vx2 = _mm_loadu_ps(&x[i+4]);
+ vy1 = _mm_loadu_ps(&y[i]);
+ vy2 = _mm_loadu_ps(&y[i+4]);
+ vs1 = _mm_add_ps(vs1, _mm_mul_ps(vx1, vy1));
+ vs2 = _mm_add_ps(vs2, _mm_mul_ps(vx2, vy2));
+ }
+ for (s = 0.; i < n; ++i) s += x[i] * y[i];
+ _mm_storeu_ps(t, vs1);
+ s += t[0] + t[1] + t[2] + t[3];
+ _mm_storeu_ps(t, vs2);
+ s += t[0] + t[1] + t[2] + t[3];
+ return s;
+}
+static inline void kad_saxpy_inlined(int n, float a, const float *x, float *y) /* BLAS saxpy using SSE */
+{
+ int i, n8 = n>>3<<3;
+ __m128 va;
+ va = _mm_set1_ps(a);
+ for (i = 0; i < n8; i += 8) {
+ __m128 vx1, vx2, vy1, vy2, vt1, vt2;
+ vx1 = _mm_loadu_ps(&x[i]);
+ vx2 = _mm_loadu_ps(&x[i+4]);
+ vy1 = _mm_loadu_ps(&y[i]);
+ vy2 = _mm_loadu_ps(&y[i+4]);
+ vt1 = _mm_add_ps(_mm_mul_ps(va, vx1), vy1);
+ vt2 = _mm_add_ps(_mm_mul_ps(va, vx2), vy2);
+ _mm_storeu_ps(&y[i], vt1);
+ _mm_storeu_ps(&y[i+4], vt2);
+ }
+ for (; i < n; ++i) y[i] += a * x[i];
+}
+#else
+static inline float kad_sdot(int n, const float *x, const float *y) /* BLAS sdot */
+{
+ int i;
+ float s = 0.;
+ for (i = 0; i < n; ++i) s += x[i] * y[i];
+ return s;
+}
+static inline void kad_saxpy_inlined(int n, float a, const float *x, float *y) // BLAS saxpy
+{
+ int i;
+ for (i = 0; i < n; ++i) y[i] += a * x[i];
+}
+#endif
+
+void kad_vec_mul_sum(int n, float *a, const float *b, const float *c)
+{
+ int i;
+ for (i = 0; i < n; ++i) a[i] += b[i] * c[i];
+}
+
+/* This is actually lapack not cblas, but this definition is used */
+#ifdef HAVE_CBLAS
+#ifndef __APPLE__
+/* As gfortran mangles names */
+#define ssyev ssyev_
+#endif
+extern void ssyev(const char* jobz, const char* uplo, int* n, float* a, int* lda, float* w, float* work, int* lwork, int* info);
+#endif
+
+#ifdef HAVE_CBLAS_SGEMM
+
+#ifdef HAVE_CBLAS_H
+#include "cblas.h"
+#else
+/* Poor man approach, thanks for that Apple */
+enum CBLAS_ORDER {CblasRowMajor=101, CblasColMajor=102 };
+enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112 };
+extern void cblas_sgemm(const enum CBLAS_ORDER Order,
+ const enum CBLAS_TRANSPOSE TA,
+ const enum CBLAS_TRANSPOSE TB,
+ const int M, const int N, const int K,
+ const float alpha, const float *A, const int lda,
+ const float *B, const int ldb, const float beta,
+ float *C, const int ldc);
+#endif
+
+void kad_sgemm_simple(int trans_A, int trans_B, int M, int N, int K, const float *A, const float *B, float *C)
+{
+ cblas_sgemm(CblasRowMajor, trans_A? CblasTrans : CblasNoTrans, trans_B? CblasTrans : CblasNoTrans, M, N, K, 1.0f, A, trans_A? M : K, B, trans_B? K : N, 1.0f, C, N);
+}
+#else
+void kad_sgemm_simple(int trans_A, int trans_B, int M, int N, int K, const float *A, const float *B, float *C) /* simplified BLAS sgemm */
+{
+ static const int x = 16;
+ int i, j, k;
+ if (!trans_A && trans_B) {
+ for (i = 0; i < M; i += x)
+ for (j = 0; j < N; j += x) {
+ int ii, ie = M < i + x? M : i + x;
+ int jj, je = N < j + x? N : j + x;
+ for (ii = i; ii < ie; ++ii) { /* loop tiling */
+ const float *aii = A + ii * K, *bjj;
+ float *cii = C + ii * N;
+ for (jj = j, bjj = B + j * K; jj < je; ++jj, bjj += K)
+ cii[jj] += kad_sdot(K, aii, bjj);
+ }
+ }
+ } else if (!trans_A && !trans_B) {
+ for (i = 0; i < M; ++i)
+ for (k = 0; k < K; ++k)
+ kad_saxpy_inlined(N, A[i*K+k], &B[k*N], &C[i*N]);
+ } else if (trans_A && !trans_B) {
+ for (k = 0; k < K; ++k)
+ for (i = 0; i < M; ++i)
+ kad_saxpy_inlined(N, A[k*M+i], &B[k*N], &C[i*N]);
+ } else abort(); /* not implemented for (trans_A && trans_B) */
+}
+#endif
+
+#ifdef HAVE_CBLAS_SAXPY
+#ifndef HAVE_CBLAS_H
+extern void cblas_saxpy(const int __N,
+ const float __alpha, const float *__X, const int __incX, float *__Y, const int __incY);
+#endif
+
+void kad_saxpy(int n, float a, const float *x, float *y) { cblas_saxpy(n, a, x, 1, y, 1); }
+#else
+void kad_saxpy(int n, float a, const float *x, float *y) { kad_saxpy_inlined(n, a, x, y); }
+#endif
+
+bool kad_ssyev_simple(int N, float *A, float *eigenvals)
+{
+#ifndef HAVE_CBLAS
+ return false;
+#else
+ int n = N, lda = N, info, lwork;
+ float wkopt;
+ float *work;
+
+ /* Query and allocate the optimal workspace */
+ lwork = -1;
+ ssyev ("Vectors", "Upper", &n, A, &lda, eigenvals, &wkopt, &lwork, &info);
+ lwork = wkopt;
+ work = (float*) g_malloc(lwork * sizeof(double));
+ ssyev ("Vectors", "Upper", &n, A, &lda, eigenvals, work, &lwork, &info);
+ /* Check for convergence */
+ if (info > 0) {
+ g_free (work);
+
+ return false;
+ }
+
+ g_free (work);
+
+ return true;
+#endif
+}
+
+/***************************
+ * Random number generator *
+ ***************************/
+
+static kad_rng_t kad_rng_dat = { {0x50f5647d2380309dULL, 0x91ffa96fc4c62cceULL}, 0.0, 0, 0 };
+
+static inline uint64_t kad_splitmix64(uint64_t x)
+{
+ uint64_t z = (x += 0x9E3779B97F4A7C15ULL);
+ z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL;
+ z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL;
+ return z ^ (z >> 31);
+}
+
+static inline uint64_t kad_xoroshiro128plus_next(kad_rng_t *r)
+{
+ const uint64_t s0 = r->s[0];
+ uint64_t s1 = r->s[1];
+ const uint64_t result = s0 + s1;
+ s1 ^= s0;
+ r->s[0] = (s0 << 55 | s0 >> 9) ^ s1 ^ (s1 << 14);
+ r->s[1] = s0 << 36 | s0 >> 28;
+ return result;
+}
+
+static inline void kad_xoroshiro128plus_jump(kad_rng_t *r)
+{
+ static const uint64_t JUMP[] = { 0xbeac0467eba5facbULL, 0xd86b048b86aa9922ULL };
+ uint64_t s0 = 0, s1 = 0;
+ int i, b;
+ for (i = 0; i < 2; ++i)
+ for (b = 0; b < 64; b++) {
+ if (JUMP[i] & 1ULL << b)
+ s0 ^= r->s[0], s1 ^= r->s[1];
+ kad_xoroshiro128plus_next(r);
+ }
+ r->s[0] = s0, r->s[1] = s1;
+}
+
+void kad_srand(void *d, uint64_t seed)
+{
+ kad_rng_t *r = d? (kad_rng_t*)d : &kad_rng_dat;
+ r->n_gset = 0.0, r->n_iset = 0;
+ r->s[0] = kad_splitmix64(seed);
+ r->s[1] = kad_splitmix64(r->s[0]);
+}
+
+void *kad_rng(void)
+{
+ kad_rng_t *r;
+ r = (kad_rng_t*)calloc(1, sizeof(kad_rng_t));
+ kad_xoroshiro128plus_jump(&kad_rng_dat);
+ r->s[0] = kad_rng_dat.s[0], r->s[1] = kad_rng_dat.s[1];
+ return r;
+}
+
+uint64_t kad_rand(void *d) { return kad_xoroshiro128plus_next(d? (kad_rng_t*)d : &kad_rng_dat); }
+
+double kad_drand(void *d)
+{
+ union { uint64_t i; double d; } u;
+ u.i = 0x3FFULL << 52 | kad_xoroshiro128plus_next(d? (kad_rng_t*)d : &kad_rng_dat) >> 12;
+ return u.d - 1.0;
+}
+
+double kad_drand_normal(void *d)
+{
+ kad_rng_t *r = d? (kad_rng_t*)d : &kad_rng_dat;
+ if (r->n_iset == 0) {
+ double fac, rsq, v1, v2;
+ do {
+ v1 = 2.0 * kad_drand(d) - 1.0;
+ v2 = 2.0 * kad_drand(d) - 1.0;
+ rsq = v1 * v1 + v2 * v2;
+ } while (rsq >= 1.0 || rsq == 0.0);
+ fac = sqrt(-2.0 * log(rsq) / rsq);
+ r->n_gset = v1 * fac;
+ r->n_iset = 1;
+ return v2 * fac;
+ } else {
+ r->n_iset = 0;
+ return r->n_gset;
+ }
+}
+
+/*************
+ * Operators *
+ *************/
+
+static inline void kad_copy_dim1(kad_node_t *dst, const kad_node_t *src) /* set the dimension/shape of dst to src */
+{
+ dst->n_d = src->n_d;
+ if (src->n_d) memcpy(dst->d, src->d, src->n_d * sizeof(int));
+}
+
+/********** Arithmetic operations **********/
+
+int kad_op_add(kad_node_t *p, int action)
+{
+ int i, n0, n1;
+ kad_node_t *q[2];
+
+ q[0] = p->child[0], n0 = kad_len(q[0]);
+ q[1] = p->child[1], n1 = kad_len(q[1]);
+ if (action == KAD_SYNC_DIM) {
+ if (n0 % n1 != 0) return -1;
+ kad_copy_dim1(p, q[0]);
+ } else if (action == KAD_FORWARD) {
+ assert(n0 >= n1);
+ memcpy(p->x, q[0]->x, n0 * sizeof(float));
+ for (i = 0; i < n0; i += n1)
+ kad_saxpy(n1, 1.0f, q[1]->x, p->x + i);
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(q[0])) kad_saxpy(n0, 1.0f, p->g, q[0]->g);
+ if (kad_is_back(q[1]))
+ for (i = 0; i < n0; i += n1)
+ kad_saxpy(n1, 1.0f, p->g + i, q[1]->g);
+ }
+ return 0;
+}
+
+int kad_op_sub(kad_node_t *p, int action)
+{
+ int i, n0, n1;
+ kad_node_t *q[2];
+
+ q[0] = p->child[0], n0 = kad_len(q[0]);
+ q[1] = p->child[1], n1 = kad_len(q[1]);
+ if (action == KAD_SYNC_DIM) {
+ if (n0 % n1 != 0) return -1;
+ kad_copy_dim1(p, q[0]);
+ } else if (action == KAD_FORWARD) {
+ assert(n0 >= n1);
+ memcpy(p->x, q[0]->x, n0 * sizeof(float));
+ for (i = 0; i < n0; i += n1)
+ kad_saxpy(n1, -1.0f, q[1]->x, p->x + i);
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(q[0])) kad_saxpy(n0, 1.0f, p->g, q[0]->g);
+ if (kad_is_back(q[1]))
+ for (i = 0; i < n0; i += n1)
+ kad_saxpy(n1, -1.0f, p->g + i, q[1]->g);
+ }
+ return 0;
+}
+
+int kad_op_mul(kad_node_t *p, int action)
+{
+ int i, n0, n1;
+ kad_node_t *q[2];
+
+ q[0] = p->child[0], n0 = kad_len(q[0]);
+ q[1] = p->child[1], n1 = kad_len(q[1]);
+ if (action == KAD_SYNC_DIM) {
+ if (n0 % n1 != 0) return -1;
+ kad_copy_dim1(p, q[0]);
+ } else if (action == KAD_FORWARD) {
+ assert(n0 >= n1);
+ memset(p->x, 0, n0 * sizeof(float));
+ if (q[0]->x != 0 && q[1]->x != 0)
+ for (i = 0; i < n0; i += n1) /* TODO: optimize when n1==1 */
+ kad_vec_mul_sum(n1, p->x + i, q[0]->x + i, q[1]->x);
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(q[0]) && q[1]->x)
+ for (i = 0; i < n0; i += n1)
+ kad_vec_mul_sum(n1, q[0]->g + i, p->g + i, q[1]->x);
+ if (kad_is_back(q[1]) && q[0]->x)
+ for (i = 0; i < n0; i += n1)
+ kad_vec_mul_sum(n1, q[1]->g, p->g + i, q[0]->x + i);
+ }
+ return 0;
+}
+
+int kad_op_cmul(kad_node_t *p, int action)
+{
+ int i, n_a_row, n_b_row, n_col, n_a_col = 1, n_b_col = 1;
+ kad_node_t *q[2];
+
+ q[0] = p->child[0], q[1] = p->child[1];
+ n_col = q[0]->d[q[0]->n_d - 1] > q[1]->d[q[1]->n_d - 1]? q[0]->d[q[0]->n_d - 1] : q[1]->d[q[1]->n_d - 1];
+ for (i = q[0]->n_d - 1; i >= 0; --i) if (n_a_col < n_col) n_a_col *= q[0]->d[i];
+ for (i = q[1]->n_d - 1; i >= 0; --i) if (n_b_col < n_col) n_b_col *= q[1]->d[i];
+ n_a_row = kad_len(q[0]) / n_a_col, n_b_row = kad_len(q[1]) / n_b_col;
+ if (action == KAD_SYNC_DIM) {
+ if (n_a_col != n_b_col) return -1;
+ p->n_d = 2, p->d[0] = n_a_row, p->d[1] = n_b_row;
+ } else if (action == KAD_FORWARD) {
+ memset(p->x, 0, n_a_row * n_b_row * sizeof(float));
+ if (q[0]->x && q[1]->x)
+ kad_sgemm_simple(0, 1, n_a_row, n_b_row, n_col, q[0]->x, q[1]->x, p->x); /* Y = X * trans(W) */
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(q[0]) && q[1]->x)
+ kad_sgemm_simple(0, 0, n_a_row, n_col, n_b_row, p->g, q[1]->x, q[0]->g); /* G_x <- G_y * W */
+ if (kad_is_back(q[1]) && q[0]->x)
+ kad_sgemm_simple(1, 0, n_b_row, n_col, n_a_row, p->g, q[0]->x, q[1]->g); /* G_w <- trans(G_y) * X */
+ }
+ return 0;
+}
+
+int kad_op_matmul(kad_node_t *p, int action) /* TODO: matmul and cmul have different broadcasting rules */
+{
+ int n_a_row, n_b_row, n_a_col, n_b_col;
+ kad_node_t *q[2];
+
+ q[0] = p->child[0];
+ q[1] = p->child[1];
+ n_a_row = q[0]->n_d == 1? 1 : q[0]->d[0];
+ n_b_row = q[1]->n_d == 1? 1 : q[1]->d[0];
+ n_a_col = kad_len(q[0]) / n_a_row;
+ n_b_col = kad_len(q[1]) / n_b_row;
+ if (action == KAD_SYNC_DIM) {
+ if (n_a_col != n_b_row) return -1;
+ p->n_d = 2, p->d[0] = n_a_row, p->d[1] = n_b_col;
+ } else if (action == KAD_FORWARD) {
+ memset(p->x, 0, n_a_row * n_b_col * sizeof(float));
+ if (q[0]->x && q[1]->x)
+ kad_sgemm_simple(0, 0, n_a_row, n_b_col, n_a_col, q[0]->x, q[1]->x, p->x); /* Y = X * W */
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(q[0]) && q[1]->x)
+ kad_sgemm_simple(0, 1, n_a_row, n_a_col, n_b_col, p->g, q[1]->x, q[0]->g); /* G_x <- G_y * trans(W) */
+ if (kad_is_back(q[1]) && q[0]->x)
+ kad_sgemm_simple(1, 0, n_b_row, n_b_col, n_a_row, q[0]->x, p->g, q[1]->g); /* G_y <- trans(A) * G_y */
+ }
+ return 0;
+}
+
+int kad_op_square(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i)
+ p->x[i] = q->x[i] * q->x[i];
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * (q->x[i] + q->x[i]);
+ }
+ return 0;
+}
+
+int kad_op_1minus(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i) p->x[i] = 1.0f - q->x[i];
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ kad_saxpy(n, -1.0f, p->g, q->g);
+ }
+ return 0;
+}
+
+int kad_op_exp(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i) p->x[i] = expf(q->x[i]);
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * p->x[i];
+ }
+ return 0;
+}
+
+int kad_op_log(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i) p->x[i] = logf(q->x[i]);
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] / q->x[i];
+ }
+ return 0;
+}
+
+int kad_op_reduce_sum(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+ int i, j, k, axis, d0, d1;
+
+ assert(p->ptr);
+ axis = *(int32_t*)p->ptr;
+ if (axis < 0 || axis >= q->n_d) return -1;
+ for (i = 0, d0 = 1; i < axis; ++i) d0 *= q->d[i];
+ for (i = axis + 1, d1 = 1; i < q->n_d; ++i) d1 *= q->d[i];
+ if (action == KAD_SYNC_DIM) {
+ p->n_d = q->n_d - 1;
+ for (i = j = 0; i < q->n_d; ++i)
+ if (i != axis) p->d[j++] = q->d[i];
+ } else if (action == KAD_FORWARD) {
+ memset(p->x, 0, kad_len(p) * sizeof(float));
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < q->d[axis]; ++j)
+ for (k = 0; k < d1; ++k)
+ p->x[i * d1 + k] += q->x[(i * q->d[axis] + j) * d1 + k];
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < q->d[axis]; ++j)
+ for (k = 0; k < d1; ++k)
+ q->g[(i * q->d[axis] + j) * d1 + k] += p->g[i * d1 + k];
+ }
+ return 0;
+}
+
+int kad_op_reduce_mean(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+ int i, j, k, axis, d0, d1;
+
+ assert(p->ptr);
+ axis = *(int32_t*)p->ptr;
+ if (axis < 0 || axis >= q->n_d) return -1;
+ for (i = 0, d0 = 1; i < axis; ++i) d0 *= q->d[i];
+ for (i = axis + 1, d1 = 1; i < q->n_d; ++i) d1 *= q->d[i];
+ if (action == KAD_SYNC_DIM) {
+ p->n_d = q->n_d - 1;
+ for (i = j = 0; i < q->n_d; ++i)
+ if (i != axis) p->d[j++] = q->d[i];
+ } else if (action == KAD_FORWARD) {
+ float t = 1.0f / q->d[axis];
+ memset(p->x, 0, kad_len(p) * sizeof(float));
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < q->d[axis]; ++j)
+ for (k = 0; k < d1; ++k)
+ p->x[i * d1 + k] += t * q->x[(i * q->d[axis] + j) * d1 + k];
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ float t = 1.0f / q->d[axis];
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < q->d[axis]; ++j)
+ for (k = 0; k < d1; ++k)
+ q->g[(i * q->d[axis] + j) * d1 + k] += t * p->g[i * d1 + k];
+ }
+ return 0;
+}
+
+/********** Miscellaneous **********/
+
+int kad_op_dropout(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ assert(p->child[1]->n_d == 0);
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_ALLOC) {
+ if (kad_is_back(p->child[0]))
+ p->gtmp = realloc(p->gtmp, n);
+ } else if (action == KAD_FORWARD) {
+ float r = kad_is_const(q) || kad_is_var(q)? 0.0f : *p->child[1]->x, z = 1.0f / (1.0f - r);
+ uint8_t *flag = (uint8_t*)p->gtmp;
+ for (i = 0; i < n; ++i) {
+ int kept = (kad_drand(p->ptr) >= r);
+ p->x[i] = kept? q->x[i] * z : 0.0f;
+ if (flag) flag[i] = kept;
+ }
+ } else if (action == KAD_BACKWARD && kad_is_back(p->child[0])) {
+ float r = kad_is_const(q) || kad_is_var(q)? 0.0f : *p->child[1]->x, z = 1.0f / (1.0f - r);
+ uint8_t *flag = (uint8_t*)p->gtmp;
+ for (i = 0; i < n; ++i)
+ if (flag[i]) q->g[i] += z * p->g[i];
+ }
+ return 0;
+}
+
+int kad_op_sample_normal(kad_node_t *p, int action) /* not tested */
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_ALLOC) {
+ if (kad_is_back(p->child[0]))
+ p->gtmp = realloc(p->gtmp, n * sizeof(float));
+ } else if (action == KAD_FORWARD) {
+ float *r = (float*)p->gtmp;
+ for (i = 0; i < n; ++i) {
+ float z;
+ z = (float)kad_drand_normal(p->ptr);
+ p->x[i] = q->x[i] * z;
+ if (r) r[i] = z;
+ }
+ } else if (action == KAD_BACKWARD && kad_is_back(p->child[0])) {
+ float *r = (float*)p->gtmp;
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * r[i];
+ }
+ return 0;
+}
+
+int kad_op_slice(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+ int32_t *aux, *range;
+ int i, axis, d0, d1;
+
+ assert(p->ptr);
+ aux = (int32_t*)p->ptr, axis = aux[0], range = aux + 1;
+ if (axis < 0 || axis >= q->n_d) return -1;
+ for (i = 0, d0 = 1; i < axis; ++i) d0 *= q->d[i];
+ for (i = axis + 1, d1 = 1; i < q->n_d; ++i) d1 *= q->d[i];
+ if (action == KAD_SYNC_DIM) {
+ if (range[0] >= range[1] || range[0] < 0 || range[1] > q->d[axis]) return -1;
+ kad_copy_dim1(p, q);
+ p->d[axis] = range[1] - range[0];
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < d0; ++i)
+ memcpy(&p->x[i * p->d[axis] * d1], &q->x[(i * q->d[axis] + range[0]) * d1], (range[1] - range[0]) * d1 * sizeof(float));
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < d0; ++i)
+ kad_saxpy((range[1] - range[0]) * d1, 1.0f, &p->g[i * p->d[axis] * d1], &q->g[(i * q->d[axis] + range[0]) * d1]);
+ }
+ return 0;
+}
+
+int kad_op_concat(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+ int32_t *aux;
+ int i, j, k, axis, d0, d1;
+
+ assert(p->ptr);
+ aux = (int32_t*)p->ptr, axis = aux[0];
+ for (i = 0, d0 = 1; i < axis; ++i) d0 *= q->d[i];
+ for (i = axis + 1, d1 = 1; i < q->n_d; ++i) d1 *= q->d[i];
+ if (action == KAD_SYNC_DIM) {
+ for (i = 1; i < p->n_child; ++i) {
+ if (p->child[i]->n_d != q->n_d) return -1;
+ for (j = 0; j < q->n_d; ++j)
+ if (j != axis && q->d[j] != p->child[i]->d[j]) return -1;
+ }
+ kad_copy_dim1(p, q);
+ for (i = 1; i < p->n_child; ++i)
+ p->d[axis] += p->child[i]->d[axis];
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < d0; ++i)
+ for (j = k = 0; j < p->n_child; ++j) {
+ q = p->child[j];
+ memcpy(&p->x[(i * p->d[axis] + k) * d1], &q->x[i * q->d[axis] * d1], q->d[axis] * d1 * sizeof(float));
+ k += q->d[axis];
+ }
+ } else if (action == KAD_BACKWARD) {
+ for (i = 0; i < d0; ++i)
+ for (j = k = 0; j < p->n_child; ++j) {
+ q = p->child[j];
+ if (!kad_is_back(q)) continue;
+ kad_saxpy(q->d[axis] * d1, 1.0f, &p->g[(i * p->d[axis] + k) * d1], &q->g[i * q->d[axis] * d1]);
+ k += q->d[axis];
+ }
+ }
+ return 0;
+}
+
+int kad_op_reshape(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+
+ if (action == KAD_SYNC_DIM) {
+ if (p->ptr) {
+ int32_t *aux = (int32_t*)p->ptr;
+ int i, len = 1, n_missing = 0;
+ p->n_d = p->ptr_size / 4;
+ for (i = 0; i < p->n_d; ++i) p->d[i] = aux[i];
+ for (i = 0; i < p->n_d; ++i)
+ if (p->d[i] <= 0) ++n_missing;
+ else len *= p->d[i];
+ if (n_missing == 0 && len != kad_len(q)) return -1;
+ if (n_missing > 1) { /* attempt to infer missing dimensions except the last one */
+ for (i = 0; i < p->n_d; ++i)
+ if (p->d[i] <= 0 && i < q->n_d) {
+ p->d[i] = q->d[i], len *= p->d[i];
+ if (--n_missing == 1) break;
+ }
+ if (n_missing > 1) return -1;
+ }
+ if (n_missing == 1) { /* infer the last missing dimension */
+ if (kad_len(q) % len != 0) return -1;
+ for (i = 0; i < p->n_d; ++i)
+ if (p->d[i] <= 0) p->d[i] = kad_len(q) / len;
+ }
+ } else kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ memcpy(p->x, q->x, kad_len(p) * sizeof(float));
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ kad_saxpy(kad_len(p), 1.0f, p->g, q->g);
+ }
+ return 0;
+}
+
+int kad_op_reverse(kad_node_t *p, int action)
+{
+ kad_node_t *q = p->child[0];
+ int axis, i, j, n, d0, d1;
+
+ axis = p->ptr? *(int32_t*)p->ptr : 0;
+ if (axis < 0) axis += q->n_d;
+ assert(axis >= 0 && axis < q->n_d);
+ for (i = 0, d0 = 1; i < axis; ++i) d0 *= q->d[i];
+ n = q->d[axis];
+ for (i = axis + 1, d1 = 1; i < q->n_d; ++i) d1 *= q->d[i];
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < n; ++j)
+ memcpy(&p->x[(i * n + n - 1 - j) * d1], &q->x[(i * n + j) * d1], d1 * sizeof(float));
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < d0; ++i)
+ for (j = 0; j < n; ++j)
+ kad_saxpy(d1, 1.0f, &p->g[(i * n + n - 1 - j) * d1], &q->g[(i * n + j) * d1]);
+ }
+ return 0;
+}
+
+/********** Cost functions **********/
+
+int kad_op_mse(kad_node_t *p, int action)
+{
+ kad_node_t *y1 = p->child[0]; /* test */
+ kad_node_t *y0 = p->child[1]; /* truth */
+ int i, n;
+
+ n = kad_len(y0);
+ if (action == KAD_SYNC_DIM) {
+ if (n != kad_len(y1)) return -1;
+ p->n_d = 0;
+ } else if (action == KAD_FORWARD) {
+ double cost = 0.0;
+ for (i = 0; i < n; ++i)
+ cost += (y1->x[i] - y0->x[i]) * (y1->x[i] - y0->x[i]);
+ p->x[0] = (float)(cost / n);
+ } else if (action == KAD_BACKWARD && kad_is_back(y1)) {
+ float t = 2.0f * p->g[0] / n;
+ for (i = 0; i < n; ++i)
+ y1->g[i] += t * (y1->x[i] - y0->x[i]);
+ }
+ return 0;
+}
+
+int kad_op_ce_bin(kad_node_t *p, int action)
+{
+ static const float tiny = 1e-9f;
+ kad_node_t *y1 = p->child[0]; /* test */
+ kad_node_t *y0 = p->child[1]; /* truth */
+ int i, n;
+
+ n = kad_len(y0);
+ if (action == KAD_SYNC_DIM) {
+ if (n != kad_len(y1)) return -1;
+ p->n_d = 0;
+ } else if (action == KAD_FORWARD) {
+ double cost = 0.0;
+ for (i = 0; i < n; ++i) {
+ if (y0->x[i] > 0.0f)
+ cost += y0->x[i] * log(y0->x[i] / (y1->x[i] > tiny? y1->x[i] : tiny));
+ if (1.0f - y0->x[i] > 0.0f)
+ cost += (1.0f - y0->x[i]) * log((1.0f - y0->x[i]) / (1.0f - y1->x[i] > tiny? 1.0f - y1->x[i] : tiny));
+ }
+ p->x[0] = (float)(cost / n);
+ } else if (action == KAD_BACKWARD && kad_is_back(y1)) {
+ float t = p->g[0] / n;
+ for (i = 0; i < n; ++i) {
+ if (y0->x[i] > 0.0f)
+ y1->g[i] -= t * y0->x[i] / (y1->x[i] > tiny? y1->x[i] : tiny);
+ if (1.0f - y0->x[i] > 0.0f)
+ y1->g[i] += t * (1.0f - y0->x[i]) / (1.0f - y1->x[i] > tiny? 1.0f - y1->x[i] : tiny);
+ }
+ }
+ return 0;
+}
+
+int kad_op_ce_bin_neg(kad_node_t *p, int action)
+{
+ static const float tiny = 1e-9f;
+ kad_node_t *y1 = p->child[0]; /* test */
+ kad_node_t *y0 = p->child[1]; /* truth */
+ int i, n;
+
+ n = kad_len(y0);
+ if (action == KAD_SYNC_DIM) {
+ if (n != kad_len(y1)) return -1;
+ p->n_d = 0;
+ } else if (action == KAD_FORWARD) {
+ double cost = 0.0;
+ for (i = 0; i < n; ++i) {
+ if (1.0f + y0->x[i] > 0.0f)
+ cost += .5f * (1.0f + y0->x[i]) * log((1.0f + y0->x[i]) / (1.0f + y1->x[i] > tiny? 1.0f + y1->x[i] : tiny));
+ if (1.0f - y0->x[i] > 0.0f)
+ cost += .5f * (1.0f - y0->x[i]) * log((1.0f - y0->x[i]) / (1.0f - y1->x[i] > tiny? 1.0f - y1->x[i] : tiny));
+ }
+ p->x[0] = (float)(cost / n);
+ } else if (action == KAD_BACKWARD && kad_is_back(y1)) {
+ float t = p->g[0] / n;
+ for (i = 0; i < n; ++i) {
+ if (1.0f + y0->x[i] > 0.0f)
+ y1->g[i] -= .5f * t * (1.0f + y0->x[i]) / (1.0f + y1->x[i] > tiny? 1.0f + y1->x[i] : tiny);
+ if (1.0f - y0->x[i] > 0.0f)
+ y1->g[i] += .5f * t * (1.0f - y0->x[i]) / (1.0f - y1->x[i] > tiny? 1.0f - y1->x[i] : tiny);
+ }
+ }
+ return 0;
+}
+
+int kad_op_ce_multi(kad_node_t *p, int action)
+{
+ static const float tiny = 1e-9f;
+ kad_node_t *y1 = p->child[0]; /* test */
+ kad_node_t *y0 = p->child[1]; /* truth */
+ kad_node_t *c = 0;
+ int i, j, n1, d0;
+
+ n1 = y0->d[y0->n_d - 1];
+ d0 = kad_len(y0) / n1;
+ if (p->n_child == 3) {
+ c = p->child[2];
+ assert(c->n_d == 1 && c->d[0] == n1);
+ }
+ if (action == KAD_SYNC_DIM) {
+ if (kad_len(y0) != kad_len(y1) || y0->d[y0->n_d - 1] != y1->d[y1->n_d - 1]) return -1;
+ p->n_d = 0;
+ } else if (action == KAD_FORWARD) {
+ double cost = 0.0;
+ if (c == 0) {
+ for (j = 0; j < d0; ++j) {
+ float *x1 = &y1->x[j * n1], *x0 = &y0->x[j * n1];
+ for (i = 0; i < n1; ++i)
+ if (x0[i] > 0.0f)
+ cost += x0[i] * log(x0[i] / (x1[i] > tiny? x1[i] : tiny));
+ }
+ } else {
+ for (j = 0; j < d0; ++j) {
+ float *x1 = &y1->x[j * n1], *x0 = &y0->x[j * n1];
+ for (i = 0; i < n1; ++i)
+ if (x0[i] > 0.0f)
+ cost += c->x[i] * x0[i] * log(x0[i] / (x1[i] > tiny? x1[i] : tiny));
+ }
+ }
+ p->x[0] = (float)(cost / d0);
+ } else if (action == KAD_BACKWARD && kad_is_back(y1)) {
+ float t = p->g[0] / d0;
+ if (c == 0) {
+ for (j = 0; j < d0; ++j) {
+ float *g = &y1->g[j * n1], *x1 = &y1->x[j * n1], *x0 = &y0->x[j * n1];
+ for (i = 0; i < n1; ++i)
+ g[i] -= t * x0[i] / (x1[i] > tiny? x1[i] : tiny);
+ }
+ } else {
+ for (j = 0; j < d0; ++j) {
+ float *g = &y1->g[j * n1], *x1 = &y1->x[j * n1], *x0 = &y0->x[j * n1];
+ for (i = 0; i < n1; ++i)
+ g[i] -= t * c->x[i] * x0[i] / (x1[i] > tiny? x1[i] : tiny);
+ }
+ }
+ }
+ return 0;
+}
+
+/********** Normalization **********/
+
+int kad_op_stdnorm(kad_node_t *p, int action)
+{
+ int i, j, n, m;
+ kad_node_t *q = p->child[0];
+ assert(q->n_d > 0);
+ n = q->d[q->n_d - 1];
+ m = kad_len(q) / n;
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_ALLOC) {
+ p->gtmp = realloc(p->gtmp, m * sizeof(float));
+ } else if (action == KAD_FORWARD) {
+ float *si = (float*)p->gtmp;
+ for (j = 0; j < m; ++j) {
+ float *px = &p->x[j * n], *qx = &q->x[j * n];
+ float avg, std_inv;
+ double s;
+ for (i = 0, s = 0.0; i < n; ++i) s += qx[i];
+ avg = (float)(s / n);
+ for (i = 0; i < n; ++i) px[i] = qx[i] - avg;
+ for (i = 0, s = 0.0; i < n; ++i) s += px[i] * px[i];
+ std_inv = s == 0.0? 1.0f : (float)(1.0 / sqrt(s / n));
+ for (i = 0; i < n; ++i) px[i] *= std_inv;
+ si[j] = std_inv;
+ }
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ float *si = (float*)p->gtmp;
+ for (j = 0; j < m; ++j) {
+ float *pg = &p->g[j * n], *qg = &q->g[j * n], *px = &p->x[j * n], std_inv = si[j];
+ double s, t;
+ for (i = 0, s = t = 0.0; i < n; ++i)
+ s += pg[i], t += px[i] * pg[i];
+ s /= n, t /= n;
+ for (i = 0; i < n; ++i)
+ qg[i] += std_inv * (pg[i] - s - px[i] * t);
+ }
+ }
+ return 0;
+}
+
+/********** Activation functions **********/
+
+int kad_op_sigm(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i)
+ p->x[i] = 1.0f / (1.0f + expf(-q->x[i]));
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * (p->x[i] * (1.0f - p->x[i]));
+ }
+ return 0;
+}
+
+int kad_op_tanh(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i) {
+ if (q->x[i] < -20.0f) p->x[i] = -1.0f;
+ else {
+ float y;
+ y = expf(-2.0f * q->x[i]);
+ p->x[i] = (1.0f - y) / (1.0f + y);
+ }
+ }
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * (1.0f - p->x[i] * p->x[i]);
+ }
+ return 0;
+}
+
+int kad_op_relu(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i)
+ p->x[i] = q->x[i] > 0.0f? q->x[i] : 0.0f;
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ if (q->x[i] > 0.0f)
+ q->g[i] += p->g[i];
+ }
+ return 0;
+}
+
+int kad_op_sin(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (i = 0; i < n; ++i) p->x[i] = sinf(q->x[i]);
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (i = 0; i < n; ++i)
+ q->g[i] += p->g[i] * cosf(q->x[i]);
+ }
+ return 0;
+}
+
+int kad_op_softmax(kad_node_t *p, int action)
+{
+ int i, j, n1, d0;
+ kad_node_t *q = p->child[0];
+
+ n1 = q->d[q->n_d - 1];
+ d0 = kad_len(q) / n1;
+ if (action == KAD_SYNC_DIM) {
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ for (j = 0; j < d0; ++j) {
+ float s, max, *x = &q->x[j * n1], *y = &p->x[j * n1];
+ for (i = 0, max = -FLT_MAX; i < n1; ++i)
+ max = max > x[i]? max : x[i];
+ for (i = 0, s = 0.0f; i < n1; ++i) {
+ y[i] = expf(x[i] - max);
+ s += y[i];
+ }
+ for (i = 0, s = 1.0f / s; i < n1; ++i) y[i] *= s;
+ }
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ for (j = 0; j < d0; ++j) {
+ float s, *g = &p->g[j * n1], *y = &p->x[j * n1], *h = &q->g[j * n1];
+ for (i = 0, s = 0.0f; i < n1; ++i)
+ s += g[i] * y[i];
+ for (i = 0; i < n1; ++i)
+ h[i] += y[i] * (g[i] - s);
+ }
+ }
+ return 0;
+}
+
+/********** Multi-node pooling **********/
+
+int kad_op_avg(kad_node_t *p, int action)
+{
+ int i, n;
+ float tmp;
+ kad_node_t *q;
+
+ assert(p->n_child > 0);
+ tmp = 1.0f / p->n_child;
+ q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ for (i = 1; i < p->n_child; ++i)
+ if (kad_len(p->child[i]) != n) return -1;
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ memcpy(p->x, q->x, n * sizeof(float));
+ for (i = 1; i < p->n_child; ++i)
+ kad_saxpy(n, 1.0f, p->child[i]->x, p->x);
+ for (i = 0; i < n; ++i) p->x[i] *= tmp;
+ } else if (action == KAD_BACKWARD) {
+ for (i = 0; i < p->n_child; ++i)
+ if (kad_is_back(p->child[i]))
+ kad_saxpy(n, tmp, p->g, p->child[i]->g);
+ }
+ return 0;
+}
+
+int kad_op_max(kad_node_t *p, int action)
+{
+ int i, n;
+ kad_node_t *q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ int *max_j;
+ for (i = 1; i < p->n_child; ++i)
+ if (kad_len(p->child[i]) != n) return -1;
+ kad_copy_dim1(p, q);
+ max_j = (int*)calloc(n, sizeof(int));
+ p->gtmp = max_j;
+ } else if (action == KAD_FORWARD) {
+ int j, *max_j = (int*)p->gtmp;
+ memset(max_j, 0, n * sizeof(int));
+ memcpy(p->x, q->x, n * sizeof(float));
+ for (j = 1; j < p->n_child; ++j)
+ for (i = 0, q = p->child[j]; i < n; ++i)
+ if (q->x[i] > p->x[i]) p->x[i] = q->x[i], max_j[i] = j;
+ } else if (action == KAD_BACKWARD) {
+ int *max_j = (int*)p->gtmp;
+ for (i = 0; i < n; ++i)
+ p->child[max_j[i]]->g[i] += p->g[i];
+ }
+ return 0;
+}
+
+int kad_op_stack(kad_node_t *p, int action) /* TODO: allow axis, as in TensorFlow */
+{
+ int i, n, axis = 0;
+ kad_node_t *q;
+
+ assert(p->n_child > 0);
+ q = p->child[0];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ for (i = 1; i < p->n_child; ++i)
+ if (kad_len(p->child[i]) != n) return -1;
+ p->n_d = q->n_d + 1;
+ for (i = 0; i < axis; ++i) p->d[i] = q->d[i];
+ p->d[axis] = p->n_child;
+ for (; i < q->n_d; ++i) p->d[i+1] = q->d[i];
+ } else if (action == KAD_FORWARD) { /* TODO: doesn't work when axis != 0 */
+ for (i = 0; i < p->n_child; ++i)
+ memcpy(&p->x[i * n], p->child[i]->x, n * sizeof(float));
+ } else if (action == KAD_BACKWARD) {
+ for (i = 0; i < p->n_child; ++i)
+ if (kad_is_back(p->child[i]))
+ kad_saxpy(n, 1.0f, &p->g[i * n], p->child[i]->g);
+ }
+ return 0;
+}
+
+int kad_op_select(kad_node_t *p, int action)
+{
+ kad_node_t *q;
+ int i, n, which;
+
+ which = *(int32_t*)p->ptr;
+ if (which < 0) which += p->n_child;
+ assert(which >= 0 && which < p->n_child);
+ q = p->child[which];
+ n = kad_len(q);
+ if (action == KAD_SYNC_DIM) {
+ for (i = 0; i < p->n_child; ++i)
+ if (p->child[i]->n_d != q->n_d || kad_len(p->child[i]) != n)
+ break;
+ if (i < p->n_child) return -1;
+ kad_copy_dim1(p, q);
+ } else if (action == KAD_FORWARD) {
+ memcpy(p->x, q->x, n * sizeof(float));
+ } else if (action == KAD_BACKWARD && kad_is_back(q)) {
+ kad_saxpy(n, 1.0f, p->g, q->g);
+ }
+ return 0;
+}
+
+/********** 2D convolution **********/
+
+static void conv_rot180(int d0, int d1, float *x) /* rotate/reverse a weight martix */
+{
+ int i, j;
+ for (i = 0; i < d0; ++i) {
+ float tmp, *xi = &x[i * d1];
+ for (j = 0; j < d1>>1; ++j)
+ tmp = xi[j], xi[j] = xi[d1-1-j], xi[d1-1-j] = tmp;
+ }
+}
+
+static void conv2d_move_1to3(int d[4], const float *x, float *y) /* convert the NCHW shape to the NHWC shape */
+{
+ int i, j, k, l;
+ for (i = 0; i < d[0]; ++i)
+ for (j = 0; j < d[1]; ++j)
+ for (k = 0; k < d[2]; ++k) {
+ int ik = (i * d[2] + k) * d[3], ijk = ((i * d[1] + j) * d[2] + k) * d[3];
+ for (l = 0; l < d[3]; ++l)
+ y[(ik + l) * d[1] + j] = x[ijk + l];
+ }
+}
+
+static void conv2d_add_3to1(int d[4], const float *y, float *x) /* convert the NHWC shape back to NCHW and add to another NCHW-shaped array */
+{
+ int i, j, k, l;
+ for (i = 0; i < d[0]; ++i)
+ for (j = 0; j < d[1]; ++j)
+ for (k = 0; k < d[2]; ++k) {
+ int ik = (i * d[2] + k) * d[3], ijk = ((i * d[1] + j) * d[2] + k) * d[3];
+ for (l = 0; l < d[3]; ++l)
+ x[ijk + l] += y[(ik + l) * d[1] + j];
+ }
+}
+
+#define conv_out_size(in_size, aux) (((in_size) - (aux)->kernel_size + (aux)->pad[0] + (aux)->pad[1]) / (aux)->stride + 1)
+
+#define process_row_for(_xx, _ww, _yy, _wn, _pn, _stride, _pad, _t) do { \
+ int j, l; \
+ if (_stride > 1) { \
+ for (l = 0; l < _wn; ++l) { \
+ const float *xl = &_xx[l - _pad]; \
+ for (j = 0; j < _pn; ++j, xl += _stride) _t[j] = *xl; \
+ kad_saxpy(_pn, _ww[l], _t, _yy); \
+ } \
+ } else for (l = 0; l < _wn; ++l) kad_saxpy(_pn, _ww[l], &_xx[l - _pad], _yy); \
+} while (0)
+
+#define process_row_back_x(_xx, _ww, _yy, _wn, _pn, _stride, _pad, _t) do { \
+ int j, l; \
+ if (_stride > 1) { \
+ for (l = 0; l < _wn; ++l) { \
+ float *xl = &_xx[l - _pad]; \
+ memset(_t, 0, _pn * sizeof(float)); \
+ kad_saxpy(_pn, _ww[l], _yy, _t); \
+ for (j = 0; j < _pn; ++j, xl += _stride) *xl += _t[j]; \
+ } \
+ } else for (l = 0; l < _wn; ++l) kad_saxpy(_pn, _ww[l], _yy, &_xx[l - _pad]); \
+} while (0)
+
+#define process_row_back_w(_xx, _ww, _yy, _wn, _pn, _stride, _pad, _t) do { \
+ int j, l; \
+ if (_stride > 1) { \
+ for (l = 0; l < _wn; ++l) { \
+ const float *xl = &_xx[l - _pad]; \
+ for (j = 0; j < _pn; ++j, xl += _stride) _t[j] = *xl; \
+ _ww[l] += kad_sdot(_pn, _yy, _t); \
+ } \
+ } else for (l = 0; l < _wn; ++l) _ww[l] += kad_sdot(_pn, _yy, &_xx[l - _pad]); \
+} while (0)
+
+/* Forward and backward passes are implemented with two different algorithms.
+ * The first is faster for small kernels with few input channels; otherwise the
+ * second algorithm is faster. Both algorithms should produce identical
+ * results, up to the precision of "float".
+ */
+int kad_op_conv2d(kad_node_t *p, int action) /* in the number-channel-height-width (NCHW) shape */
+{
+#define conv2d_loop1(_x, _w, _y, _tmp, _row_func) do { /* for the NCHW shape */ \
+ int n, c1, c0, i, k, ii; \
+ for (n = 0; n < q->d[0]; ++n) /* mini-batch */ \
+ for (c1 = 0; c1 < w->d[0]; ++c1) /* output channel */ \
+ for (c0 = 0; c0 < w->d[1]; ++c0) /* input channel */ \
+ for (k = 0; k < w->d[2]; ++k) { /* kernel row */ \
+ float *_ww = &(_w)[((c1 * w->d[1] + c0) * w->d[2] + k) * w->d[3]]; \
+ for (i = 0, ii = k - aux[0].pad[0]; i < p->d[2] && ii >= 0 && ii < q->d[2]; ++i, ii += aux[0].stride) { /* output row */ \
+ float *_xx = &(_x)[((n * q->d[1] + c0) * q->d[2] + ii) * q->d[3]]; \
+ float *_yy = &(_y)[((n * p->d[1] + c1) * p->d[2] + i) * p->d[3]]; \
+ if (x_padded) { \
+ memcpy(x_padded + aux[1].pad[0], _xx, q->d[3] * sizeof(float)); \
+ _xx = x_padded + aux[1].pad[0]; \
+ } \
+ _row_func(_xx, _ww, _yy, w->d[3], p->d[3], aux[1].stride, aux[1].pad[0], (_tmp)); \
+ } /* ~i */ \
+ } /* ~k, c0, c1, n */ \
+ } while (0)
+
+#define conv2d_loop2(_x, _w, _y, _code) do { /* for the NHWC shape */ \
+ int n, c1, i, j, k, ii, j_skip = aux[1].stride * q->d[1], m = w->d[3] * w->d[1]; \
+ for (n = 0; n < q->d[0]; ++n) /* mini-batch */ \
+ for (c1 = 0; c1 < w->d[0]; ++c1) /* output channel */ \
+ for (k = 0; k < w->d[2]; ++k) { /* kernel row */ \
+ float *_ww = &(_w)[(c1 * w->d[2] + k) * m]; \
+ for (i = 0, ii = k - aux[0].pad[0]; i < p->d[2] && ii >= 0 && ii < q->d[2]; ++i, ii += aux[0].stride) { /* output and input row */ \
+ float *_xx = &(_x)[(n * q->d[2] + ii) * q->d[3] * q->d[1]]; \
+ float *_yy = &(_y)[((n * p->d[1] + c1) * p->d[2] + i) * p->d[3]]; \
+ if (x_padded) { \
+ memcpy(x_padded + aux[1].pad[0] * q->d[1], _xx, q->d[3] * q->d[1] * sizeof(float)); \
+ _xx = x_padded; \
+ } \
+ for (j = 0; j < p->d[3]; ++j, _xx += j_skip, ++_yy) _code; /* output and input column */ \
+ } /* ~i */ \
+ } /* ~k, c1, n */ \
+ } while (0)
+
+ conv_conf_t *aux = (conv_conf_t*)p->ptr;
+ kad_node_t *q = p->child[0], *w = p->child[1];
+ float *t = 0, *q1 = 0, *w1 = 0, *x_padded = 0;
+ int algo_switch = 0;
+
+ if (action == KAD_FORWARD || action == KAD_BACKWARD) { /* allocate working space */
+ if (w->d[3] * w->d[1] < 16) {
+ t = (float*)malloc(p->d[3] * sizeof(float));
+ x_padded = aux[1].pad[0] + aux[1].pad[1] > 0? (float*)calloc(q->d[3] + aux[1].pad[0] + aux[1].pad[1], sizeof(float)) : 0;
+ } else {
+ q1 = (float*)malloc(kad_len(q) * sizeof(float));
+ w1 = (float*)malloc(kad_len(w) * sizeof(float));
+ x_padded = aux[1].pad[0] + aux[1].pad[1] > 0? (float*)calloc((q->d[3] + aux[1].pad[0] + aux[1].pad[1]) * q->d[1], sizeof(float)) : 0;
+ algo_switch = 1;
+ }
+ }
+ if (action == KAD_SYNC_DIM) {
+ if (q->n_d != 4 || w->n_d != 4) return -1;
+ if (q->d[1] != w->d[1]) return -1; /* unmatched input channels */
+ p->n_d = 4;
+ p->d[0] = q->d[0], p->d[1] = w->d[0], p->d[2] = conv_out_size(q->d[2], &aux[0]), p->d[3] = conv_out_size(q->d[3], &aux[1]);
+ } else if (action == KAD_FORWARD) {
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->x);
+ memset(p->x, 0, kad_len(p) * sizeof(float));
+ if (!algo_switch) { /* this is the first algorithm */
+ conv2d_loop1(q->x, w->x, p->x, t, process_row_for);
+ } else { /* this is the second algorithm */
+ conv2d_move_1to3(q->d, q->x, q1);
+ conv2d_move_1to3(w->d, w->x, w1);
+ conv2d_loop2(q1, w1, p->x, (*_yy += kad_sdot(m, _ww, _xx)));
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->x);
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(p->child[0])) { /* backprop to the input array */
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->x);
+ if (!algo_switch) {
+ conv2d_loop1(q->g, w->x, p->g, t, process_row_back_x);
+ } else {
+ memset(q1, 0, kad_len(q) * sizeof(float));
+ conv2d_move_1to3(w->d, w->x, w1);
+ conv2d_loop2(q1, w1, p->g, kad_saxpy(m, *_yy, _ww, _xx));
+ conv2d_add_3to1(q->d, q1, q->g);
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->x);
+ }
+ if (kad_is_back(p->child[1])) { /* backprop to the weight matrix */
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->g);
+ if (!algo_switch) {
+ conv2d_loop1(q->x, w->g, p->g, t, process_row_back_w);
+ } else {
+ conv2d_move_1to3(q->d, q->x, q1);
+ memset(w1, 0, kad_len(w) * sizeof(float));
+ conv2d_loop2(q1, w1, p->g, kad_saxpy(m, *_yy, _xx, _ww));
+ conv2d_add_3to1(w->d, w1, w->g);
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2] * w->d[3], w->g);
+ }
+ }
+ free(t); free(q1); free(w1); free(x_padded);
+ return 0;
+}
+
+int kad_op_max2d(kad_node_t *p, int action)
+{
+ conv_conf_t *aux = (conv_conf_t*)p->ptr;
+ kad_node_t *q = p->child[0];
+ if (action == KAD_SYNC_DIM) {
+ if (q->n_d != 4) return -1;
+ p->n_d = 4;
+ p->d[0] = q->d[0], p->d[1] = q->d[1], p->d[2] = conv_out_size(q->d[2], &aux[0]), p->d[3] = conv_out_size(q->d[3], &aux[1]);
+ } else if (action == KAD_ALLOC) {
+ p->gtmp = realloc(p->gtmp, kad_len(p) * sizeof(int));
+ } else if (action == KAD_FORWARD) {
+ int rest = 1, len, t, i;
+ int *f = (int*)p->gtmp;
+ len = kad_len(p);
+ for (i = 0; i < len; ++i) p->x[i] = -FLT_MAX;
+ for (i = 0; i < p->n_d - 2; ++i) rest *= p->d[i];
+ for (t = 0; t < rest; ++t) {
+ int i, j, k, l, p_row = p->d[p->n_d - 2], p_col = p->d[p->n_d - 1];
+ for (i = 0; i < p_row; ++i) {
+ int u = (t * p_row + i) * p_col;
+ for (k = 0; k < aux[0].kernel_size; ++k) {
+ int v, v0, v_end, ii = i * aux[0].stride + k - aux[0].pad[0];
+ if (ii < 0 || ii >= q->d[p->n_d - 2]) continue;
+ v0 = (t * q->d[p->n_d - 2] + ii) * q->d[p->n_d - 1];
+ v_end = v0 + q->d[p->n_d - 1];
+ for (l = 0; l < aux[1].kernel_size; ++l)
+ for (j = 0, v = v0 + (l > aux[1].pad[0]? l - aux[1].pad[0] : 0); j < p_col && v < v_end; ++j, v += aux[1].stride)
+ if (p->x[u + j] < q->x[v])
+ p->x[u + j] = q->x[v], f[u + j] = v;
+ } /* ~k */
+ } /* ~i */
+ }
+ } else if (action == KAD_BACKWARD) {
+ int i, len, *f = (int*)p->gtmp;
+ len = kad_len(p);
+ for (i = 0; i < len; ++i) q->g[f[i]] += p->g[i];
+ }
+ return 0;
+}
+
+/********** 1D convolution **********/
+
+static void conv1d_move_1to2(int d[3], const float *x, float *y)
+{
+ int i, j, k;
+ for (k = 0; k < d[0]; ++k)
+ for (j = 0; j < d[1]; ++j)
+ for (i = 0; i < d[2]; ++i)
+ y[(k * d[2] + i) * d[1] + j] = x[(k * d[1] + j) * d[2] + i];
+}
+
+static void conv1d_add_2to1(int d[3], const float *y, float *x)
+{
+ int i, j, k;
+ for (k = 0; k < d[0]; ++k)
+ for (j = 0; j < d[1]; ++j)
+ for (i = 0; i < d[2]; ++i)
+ x[(k * d[1] + j) * d[2] + i] += y[(k * d[2] + i) * d[1] + j];
+}
+
+int kad_op_conv1d(kad_node_t *p, int action) /* in the number-channel-width (NCW) shape */
+{
+#define conv1d_loop1(_x, _w, _y, _tmp, _row_func) do { /* for the NCW shape */ \
+ int n, c1, c0; \
+ for (n = 0; n < q->d[0]; ++n) /* mini-batch */ \
+ for (c1 = 0; c1 < w->d[0]; ++c1) /* output channel */ \
+ for (c0 = 0; c0 < w->d[1]; ++c0) { /* input channel */ \
+ float *_ww = &(_w)[(c1 * w->d[1] + c0) * w->d[2]]; \
+ float *_xx = &(_x)[(n * q->d[1] + c0) * q->d[2]]; \
+ float *_yy = &(_y)[(n * p->d[1] + c1) * p->d[2]]; \
+ if (x_padded) { \
+ memcpy(x_padded + aux->pad[0], _xx, q->d[2] * sizeof(float)); \
+ _xx = x_padded + aux->pad[0]; \
+ } \
+ _row_func(_xx, _ww, _yy, w->d[2], p->d[2], aux->stride, aux->pad[0], (_tmp)); \
+ } /* ~c0, c1, n */ \
+ } while (0)
+
+#define conv1d_loop2(_x, _w, _y, _code) do { /* for the NWC shape */ \
+ int n, c1, j, j_skip = aux->stride * q->d[1], m = w->d[2] * w->d[1]; \
+ for (n = 0; n < q->d[0]; ++n) /* mini-batch */ \
+ for (c1 = 0; c1 < w->d[0]; ++c1) { /* output channel */ \
+ float *_ww = &(_w)[c1 * m]; \
+ float *_xx = &(_x)[n * q->d[1] * q->d[2]]; \
+ float *_yy = &(_y)[(n * p->d[1] + c1) * p->d[2]]; \
+ if (x_padded) { \
+ memcpy(x_padded + aux->pad[0] * q->d[1], _xx, q->d[2] * q->d[1] * sizeof(float)); \
+ _xx = x_padded; \
+ } \
+ for (j = 0; j < p->d[2]; ++j, _xx += j_skip, ++_yy) _code; \
+ } /* ~c1, n */ \
+ } while (0)
+
+ conv_conf_t *aux = (conv_conf_t*)p->ptr;
+ kad_node_t *q = p->child[0], *w = p->child[1];
+ float *t = 0, *q1 = 0, *w1 = 0, *x_padded = 0;
+ int algo_switch = 0;
+
+ if (action == KAD_FORWARD || action == KAD_BACKWARD) { /* allocate working space */
+ if (w->d[2] * w->d[1] < 32) {
+ t = (float*)malloc(p->d[2] * sizeof(float));
+ x_padded = aux->pad[0] + aux->pad[1] > 0? (float*)calloc(q->d[2] + aux->pad[0] + aux->pad[1], sizeof(float)) : 0;
+ } else {
+ q1 = (float*)malloc(kad_len(q) * sizeof(float));
+ w1 = (float*)malloc(kad_len(w) * sizeof(float));
+ x_padded = aux->pad[0] + aux->pad[1] > 0? (float*)calloc((q->d[2] + aux->pad[0] + aux->pad[1]) * q->d[1], sizeof(float)) : 0;
+ algo_switch = 1;
+ }
+ }
+ if (action == KAD_SYNC_DIM) {
+ if (q->n_d != 3 || w->n_d != 3) return -1;
+ if (q->d[1] != w->d[1]) return -1; /* unmatched input channels */
+ p->n_d = 3;
+ p->d[0] = q->d[0], p->d[1] = w->d[0], p->d[2] = conv_out_size(q->d[2], aux);
+ } else if (action == KAD_FORWARD) {
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->x);
+ memset(p->x, 0, kad_len(p) * sizeof(float));
+ if (!algo_switch) { /* this is the first algorithm */
+ conv1d_loop1(q->x, w->x, p->x, t, process_row_for);
+ } else { /* this is the second algorithm */
+ conv1d_move_1to2(q->d, q->x, q1);
+ conv1d_move_1to2(w->d, w->x, w1);
+ conv1d_loop2(q1, w1, p->x, (*_yy += kad_sdot(m, _ww, _xx)));
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->x);
+ } else if (action == KAD_BACKWARD) {
+ if (kad_is_back(p->child[0])) { /* backprop to the input array */
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->x);
+ if (!algo_switch) {
+ conv1d_loop1(q->g, w->x, p->g, t, process_row_back_x);
+ } else {
+ memset(q1, 0, kad_len(q) * sizeof(float));
+ conv1d_move_1to2(w->d, w->x, w1);
+ conv1d_loop2(q1, w1, p->g, kad_saxpy(m, *_yy, _ww, _xx));
+ conv1d_add_2to1(q->d, q1, q->g);
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->x);
+ }
+ if (kad_is_back(p->child[1])) { /* backprop to the weight matrix */
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->g);
+ if (!algo_switch) {
+ conv1d_loop1(q->x, w->g, p->g, t, process_row_back_w);
+ } else {
+ conv1d_move_1to2(q->d, q->x, q1);
+ memset(w1, 0, kad_len(w) * sizeof(float));
+ conv1d_loop2(q1, w1, p->g, kad_saxpy(m, *_yy, _xx, _ww));
+ conv1d_add_2to1(w->d, w1, w->g);
+ }
+ conv_rot180(w->d[0] * w->d[1], w->d[2], w->g);
+ }
+ }
+ free(t); free(q1); free(w1); free(x_padded);
+ return 0;
+}
+
+int kad_op_max1d(kad_node_t *p, int action)
+{
+ conv_conf_t *aux = (conv_conf_t*)p->ptr;
+ kad_node_t *q = p->child[0];
+ if (action == KAD_SYNC_DIM) {
+ if (q->n_d != 3) return -1;
+ p->n_d = 3;
+ p->d[0] = q->d[0], p->d[1] = q->d[1], p->d[2] = conv_out_size(q->d[2], aux);
+ } else if (action == KAD_ALLOC) {
+ p->gtmp = realloc(p->gtmp, kad_len(p) * sizeof(int));
+ } else if (action == KAD_FORWARD) {
+ int rest = 1, len, t, i;
+ int *f = (int*)p->gtmp;
+ len = kad_len(p);
+ for (i = 0; i < len; ++i) p->x[i] = -FLT_MAX;
+ for (i = 0; i < p->n_d - 1; ++i) rest *= p->d[i];
+ for (t = 0; t < rest; ++t) {
+ int j, l, p_width = p->d[p->n_d - 1];
+ int u = t * p_width, v, v0 = t * q->d[p->n_d - 1], v_end = v0 + q->d[p->n_d - 1];
+ for (l = 0; l < aux->kernel_size; ++l)
+ for (j = 0, v = v0 + (l > aux->pad[0]? l - aux->pad[0] : 0); j < p_width && v < v_end; ++j, v += aux->stride)
+ if (p->x[u + j] < q->x[v])
+ p->x[u + j] = q->x[v], f[u + j] = v;
+ }
+ } else if (action == KAD_BACKWARD) {
+ int i, len, *f = (int*)p->gtmp;
+ len = kad_len(p);
+ for (i = 0; i < len; ++i) q->g[f[i]] += p->g[i];
+ }
+ return 0;
+}
+
+int kad_op_avg1d(kad_node_t *p, int action)
+{
+ conv_conf_t *aux = (conv_conf_t*)p->ptr;
+ kad_node_t *q = p->child[0];
+ if (action == KAD_SYNC_DIM) {
+ if (q->n_d != 3) return -1;
+ p->n_d = 3;
+ p->d[0] = q->d[0], p->d[1] = q->d[1], p->d[2] = conv_out_size(q->d[2], aux);
+ } else if (action == KAD_ALLOC) {
+ p->gtmp = realloc(p->gtmp, kad_len(p) * sizeof(int));
+ } else if (action == KAD_FORWARD) {
+ int rest = 1, len, t, i;
+ int *f = (int*)p->gtmp;
+ len = kad_len(p);
+ for (i = 0; i < len; ++i) p->x[i] = 0.0f, f[i] = 0;
+ for (i = 0; i < p->n_d - 1; ++i) rest *= p->d[i];
+ for (t = 0; t < rest; ++t) {
+ int j, l, p_width = p->d[p->n_d - 1];
+ int u = t * p_width, v, v0 = t * q->d[p->n_d - 1], v_end = v0 + q->d[p->n_d - 1];
+ for (l = 0; l < aux->kernel_size; ++l)
+ for (j = 0, v = v0 + (l > aux->pad[0]? l - aux->pad[0] : 0); j < p_width && v < v_end; ++j, v += aux->stride)
+ p->x[u + j] += q->x[v], ++f[u + j];
+ }
+ for (i = 0; i < len; ++i) p->x[i] /= f[i];
+ } else if (action == KAD_BACKWARD) {
+ int rest = 1, t, i;
+ int *f = (int*)p->gtmp;
+ for (i = 0; i < p->n_d - 1; ++i) rest *= p->d[i];
+ for (t = 0; t < rest; ++t) {
+ int j, l, p_width = p->d[p->n_d - 1];
+ int u = t * p_width, v, v0 = t * q->d[p->n_d - 1], v_end = v0 + q->d[p->n_d - 1];
+ for (l = 0; l < aux->kernel_size; ++l)
+ for (j = 0, v = v0 + (l > aux->pad[0]? l - aux->pad[0] : 0); j < p_width && v < v_end; ++j, v += aux->stride)
+ q->g[v] += p->g[u + j] / f[u + j];
+ }
+ }
+ return 0;
+}
+
+/********** List of operators **********/
+
+kad_op_f kad_op_list[KAD_MAX_OP] = {
+ 0,
+ kad_op_add, /* 1: element-wise addition */
+ kad_op_mul, /* 2: element-wise multiplication */
+ kad_op_cmul, /* 3: column multiplication */
+ kad_op_ce_bin_neg, /* 4: binary cross-entropy for (-1,1) */
+ kad_op_square, /* 5: square */
+ kad_op_sigm, /* 6: sigmoid */
+ kad_op_tanh, /* 7: tanh */
+ kad_op_relu, /* 8: ReLU */
+ kad_op_matmul, /* 9: matrix multiplication */
+ kad_op_avg, /* 10: general average pooling (not for ConvNet) */
+ kad_op_1minus, /* 11: 1-x */
+ kad_op_select, /* 12: choose between one of the children */
+ kad_op_ce_multi, /* 13: multi-class cross-entropy */
+ kad_op_softmax, /* 14: softmax */
+ kad_op_dropout, /* 15: dropout */
+ kad_op_conv2d, /* 16: 2D convolution */
+ kad_op_max2d, /* 17: 2D max pooling (for 2D ConvNet) */
+ kad_op_conv1d, /* 18: 1D convolution */
+ kad_op_max1d, /* 19: 1D max pooling (for 1D ConvNet) */
+ kad_op_slice, /* 20: slice data at a dimension */
+ kad_op_max, /* 21: general max pooling */
+ kad_op_ce_bin, /* 22: binary cross-entropy for (0,1) */
+ kad_op_sub, /* 23: element-wise subtraction */
+ kad_op_sample_normal, /* 24: sample from a normal distribution */
+ kad_op_reduce_sum, /* 25 */
+ kad_op_reduce_mean, /* 26 */
+ kad_op_log, /* 27: log() */
+ kad_op_avg1d, /* 28: 1D average pooling (for 1D ConvNet) */
+ kad_op_mse, /* 29: mean square error */
+ kad_op_reshape, /* 30 */
+ kad_op_concat, /* 31 */
+ kad_op_stdnorm, /* 32: layer normalization */
+ kad_op_exp, /* 33: exp() */
+ kad_op_sin, /* 34: sin() */
+ kad_op_stack, /* 35: tf.stack, but on the first axis only */
+ kad_op_reverse /* 36: tf.reverse, but on one axis only */
+};
+
+char *kad_op_name[KAD_MAX_OP] = {
+ 0, "add", "mul", "cmul", "ce_bin_neg", "square", "sigm", "tanh", "relu", "matmul", "avg", "1minus", "select", "ce_multi", "softmax",
+ "dropout", "conv2d", "max2d", "conv1d", "max1d", "slice", "max", "ce_bin", "sub", "sample_normal", "reduce_sum", "reduce_mean", "log",
+ "avg1d", "mse", "reshape", "concat", "stdnorm", "exp", "sin", "stack", "reverse"
+};
+
+/**************************
+ *** Debugging routines ***
+ **************************/
+
+void kad_trap_fe(void)
+{
+#ifdef __SSE__
+ _MM_SET_EXCEPTION_MASK(_MM_GET_EXCEPTION_MASK() & ~(_MM_MASK_INVALID | _MM_MASK_DIV_ZERO));
+#endif
+}
+
+void kad_print_graph(FILE *fp, int n, kad_node_t **v)
+{
+ int i, j;
+ for (i = 0; i < n; ++i) v[i]->tmp = i;
+ for (i = 0; i < n; ++i) {
+ kad_node_t *p = v[i];
+ fprintf(fp, "%d\t%x:%x\t%d\t", i, p->flag, p->ext_flag, p->ext_label);
+ if (p->pre) fprintf(fp, "%d\t", p->pre->tmp);
+ else fprintf(fp, ".\t");
+ fputs("[", fp);
+ for (j = 0; j < p->n_d; ++j) {
+ if (j) fputc(',', fp);
+ fprintf(fp, "%d", p->d[j]);
+ }
+ fprintf(fp, "]\t");
+ if (p->n_child) {
+ fprintf(fp, "%s(", kad_op_name[p->op]);
+ for (j = 0; j < p->n_child; ++j) {
+ if (j) fputc(',', fp);
+ fprintf(fp, "$%d", p->child[j]->tmp);
+ }
+ fprintf(fp, ")");
+ } else fprintf(fp, "%s", kad_is_feed(p)? "feed" : kad_is_var(p)? "var" : kad_is_const(p)? "const" : "N/A");
+ fputc('\n', fp);
+ }
+ for (i = 0; i < n; ++i) v[i]->tmp = 0;
+}
+
+static void kad_add_delta(int n, kad_node_t **a, float c, float *delta)
+{
+ int i, k;
+ for (i = k = 0; i < n; ++i)
+ if (kad_is_var(a[i])) {
+ kad_saxpy(kad_len(a[i]), c, &delta[k], a[i]->x);
+ k += kad_len(a[i]);
+ }
+}
+
+void kad_check_grad(int n, kad_node_t **a, int from)
+{
+ const float eps = 1e-5f, rel = 1e-7f / eps;
+ int i, k, n_var;
+ float *g0, *delta, f0, f_minus, f_plus, s0, s1, rel_err, p_m_err;
+ n_var = kad_size_var(n, a);
+ g0 = (float*)calloc(n_var, sizeof(float));
+ f0 = *kad_eval_at(n, a, from);
+ kad_grad(n, a, from);
+ for (i = k = 0; i < n; ++i)
+ if (kad_is_var(a[i])) {
+ memcpy(&g0[k], a[i]->g, kad_len(a[i]) * sizeof(float));
+ k += kad_len(a[i]);
+ }
+ delta = (float*)calloc(n_var, sizeof(float));
+ for (k = 0; k < n_var; ++k) delta[k] = (float)kad_drand(0) * eps;
+ kad_add_delta(n, a, 1.0f, delta);
+ f_plus = *kad_eval_at(n, a, from);
+ kad_add_delta(n, a, -2.0f, delta);
+ f_minus = *kad_eval_at(n, a, from);
+ kad_add_delta(n, a, 1.0f, delta);
+ s0 = kad_sdot(n_var, g0, delta);
+ s1 = .5f * (f_plus - f_minus);
+ fprintf(stderr, "Gradient check -- %g <=> %g @ %g -- ", s0/eps, s1/eps, f0);
+ if (fabs(s1) >= rel * eps) {
+ rel_err = fabsf(fabsf(s0) - fabsf(s1)) / (fabsf(s0) + fabsf(s1));
+ p_m_err = fabsf(f_plus + f_minus - 2.0f * f0) / fabsf(f_plus - f_minus);
+ fprintf(stderr, "rel_err:%g p_m_err:%g -- ", rel_err, p_m_err);
+ if (rel_err >= rel && rel_err > p_m_err) fprintf(stderr, "failed\n");
+ else fprintf(stderr, "passed\n");
+ } else fprintf(stderr, "skipped\n");
+ free(delta); free(g0);
+}
diff --git a/contrib/kann/kautodiff.h b/contrib/kann/kautodiff.h
new file mode 100644
index 0000000..d7e7133
--- /dev/null
+++ b/contrib/kann/kautodiff.h
@@ -0,0 +1,256 @@
+/*
+ The MIT License
+
+ Copyright (c) 2018-2019 Dana-Farber Cancer Institute
+ 2016-2018 Broad Institute
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+#ifndef KANN_AUTODIFF_H
+#define KANN_AUTODIFF_H
+
+#define KAD_VERSION "r544"
+
+#include <stdio.h>
+#include <stdint.h>
+
+#ifdef __STRICT_ANSI__
+#define inline
+#endif
+
+#define KAD_MAX_DIM 4 /* max dimension */
+#define KAD_MAX_OP 64 /* max number of operators */
+
+/* A computational graph is a directed acyclic graph. In the graph, an external
+ * node represents a variable, a constant or a feed; an internal node
+ * represents an operator; an edge from node v to w indicates v is an operand
+ * of w.
+ */
+
+#define KAD_VAR 0x1
+#define KAD_CONST 0x2
+#define KAD_POOL 0x4
+#define KAD_SHARE_RNG 0x10 /* with this flag on, different time step shares the same RNG status after unroll */
+
+#define kad_is_back(p) ((p)->flag & KAD_VAR)
+#define kad_is_ext(p) ((p)->n_child == 0)
+#define kad_is_var(p) (kad_is_ext(p) && kad_is_back(p))
+#define kad_is_const(p) (kad_is_ext(p) && ((p)->flag & KAD_CONST))
+#define kad_is_feed(p) (kad_is_ext(p) && !kad_is_back(p) && !((p)->flag & KAD_CONST))
+#define kad_is_pivot(p) ((p)->n_child == 1 && ((p)->flag & KAD_POOL))
+#define kad_is_switch(p) ((p)->op == 12 && !((p)->flag & KAD_POOL))
+#define kad_use_rng(p) ((p)->op == 15 || (p)->op == 24)
+
+#define kad_eval_enable(p) ((p)->tmp = 1)
+#define kad_eval_disable(p) ((p)->tmp = -1)
+
+/* a node in the computational graph */
+typedef struct kad_node_t {
+ uint8_t n_d; /* number of dimensions; no larger than KAD_MAX_DIM */
+ uint8_t flag; /* type of the node; see KAD_F_* for valid flags */
+ uint16_t op; /* operator; kad_op_list[op] is the actual function */
+ int32_t n_child; /* number of operands/child nodes */
+ int32_t tmp; /* temporary field; MUST BE zero before calling kad_compile() */
+ int32_t ptr_size; /* size of ptr below */
+ int32_t d[KAD_MAX_DIM]; /* dimensions */
+ int32_t ext_label; /* labels for external uses (not modified by the kad_* APIs) */
+ uint32_t ext_flag; /* flags for external uses (not modified by the kad_* APIs) */
+ float *x; /* value; allocated for internal nodes */
+ float *g; /* gradient; allocated for internal nodes */
+ void *ptr; /* for special operators that need additional parameters (e.g. conv2d) */
+ void *gtmp; /* temporary data generated at the forward pass but used at the backward pass */
+ struct kad_node_t **child; /* operands/child nodes */
+ struct kad_node_t *pre; /* usually NULL; only used for RNN */
+} kad_node_t, *kad_node_p;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Compile/linearize a computational graph
+ *
+ * @param n_node number of nodes (out)
+ * @param n_roots number of nodes without predecessors
+ * @param roots list of nodes without predecessors
+ *
+ * @return list of nodes, of size *n_node
+ */
+kad_node_t **kad_compile_array(int *n_node, int n_roots, kad_node_t **roots);
+
+kad_node_t **kad_compile(int *n_node, int n_roots, ...); /* an alternative API to above */
+void kad_delete(int n, kad_node_t **a); /* deallocate a compiled/linearized graph */
+
+/**
+ * Compute the value at a node
+ *
+ * @param n number of nodes
+ * @param a list of nodes
+ * @param from compute the value at this node, 0<=from<n
+ *
+ * @return a pointer to the value (pointing to kad_node_t::x, so don't call
+ * free() on it!)
+ */
+const float *kad_eval_at(int n, kad_node_t **a, int from);
+
+void kad_eval_marked(int n, kad_node_t **a);
+int kad_sync_dim(int n, kad_node_t **v, int batch_size);
+
+/**
+ * Compute gradient
+ *
+ * @param n number of nodes
+ * @param a list of nodes
+ * @param from the function node; must be a scalar (compute \nabla a[from])
+ */
+void kad_grad(int n, kad_node_t **a, int from);
+
+/**
+ * Unroll a recurrent computation graph
+ *
+ * @param n_v number of nodes
+ * @param v list of nodes
+ * @param new_n number of nodes in the unrolled graph (out)
+ * @param len how many times to unroll, one for each pivot
+ *
+ * @return list of nodes in the unrolled graph
+ */
+kad_node_t **kad_unroll(int n_v, kad_node_t **v, int *new_n, int *len);
+int kad_n_pivots(int n_v, kad_node_t **v);
+
+kad_node_t **kad_clone(int n, kad_node_t **v, int batch_size);
+
+/* define a variable, a constant or a feed (placeholder in TensorFlow) */
+kad_node_t *kad_var(float *x, float *g, int n_d, ...); /* a variable; gradients to be computed; not unrolled */
+kad_node_t *kad_const(float *x, int n_d, ...); /* a constant; no gradients computed; not unrolled */
+kad_node_t *kad_feed(int n_d, ...); /* an input/output; no gradients computed; unrolled */
+
+/* operators taking two operands */
+kad_node_t *kad_add(kad_node_t *x, kad_node_t *y); /* f(x,y) = x + y (generalized element-wise addition; f[i*n+j]=x[i*n+j]+y[j], n=kad_len(y), 0<j<n, 0<i<kad_len(x)/n) */
+kad_node_t *kad_sub(kad_node_t *x, kad_node_t *y); /* f(x,y) = x - y (generalized element-wise subtraction) */
+kad_node_t *kad_mul(kad_node_t *x, kad_node_t *y); /* f(x,y) = x * y (generalized element-wise product) */
+
+kad_node_t *kad_matmul(kad_node_t *x, kad_node_t *y); /* f(x,y) = x * y (general matrix product) */
+kad_node_t *kad_cmul(kad_node_t *x, kad_node_t *y); /* f(x,y) = x * y^T (column-wise matrix product; i.e. y is transposed) */
+
+/* loss functions; output scalar */
+kad_node_t *kad_mse(kad_node_t *x, kad_node_t *y); /* mean square error */
+kad_node_t *kad_ce_multi(kad_node_t *x, kad_node_t *y); /* multi-class cross-entropy; x is the preidction and y is the truth */
+kad_node_t *kad_ce_bin(kad_node_t *x, kad_node_t *y); /* binary cross-entropy for (0,1) */
+kad_node_t *kad_ce_bin_neg(kad_node_t *x, kad_node_t *y); /* binary cross-entropy for (-1,1) */
+kad_node_t *kad_ce_multi_weighted(kad_node_t *pred, kad_node_t *truth, kad_node_t *weight);
+
+#define KAD_PAD_NONE 0 /* use the smallest zero-padding */
+#define KAD_PAD_SAME (-2) /* output to have the same dimension as input */
+
+kad_node_t *kad_conv2d(kad_node_t *x, kad_node_t *w, int r_stride, int c_stride, int r_pad, int c_pad); /* 2D convolution with weight matrix flipped */
+kad_node_t *kad_max2d(kad_node_t *x, int kernel_h, int kernel_w, int r_stride, int c_stride, int r_pad, int c_pad); /* 2D max pooling */
+kad_node_t *kad_conv1d(kad_node_t *x, kad_node_t *w, int stride, int pad); /* 1D convolution with weight flipped */
+kad_node_t *kad_max1d(kad_node_t *x, int kernel_size, int stride, int pad); /* 1D max pooling */
+kad_node_t *kad_avg1d(kad_node_t *x, int kernel_size, int stride, int pad); /* 1D average pooling */
+
+kad_node_t *kad_dropout(kad_node_t *x, kad_node_t *r); /* dropout at rate r */
+kad_node_t *kad_sample_normal(kad_node_t *x); /* f(x) = x * r, where r is drawn from a standard normal distribution */
+
+/* operators taking one operand */
+kad_node_t *kad_square(kad_node_t *x); /* f(x) = x^2 (element-wise square) */
+kad_node_t *kad_sigm(kad_node_t *x); /* f(x) = 1/(1+exp(-x)) (element-wise sigmoid) */
+kad_node_t *kad_tanh(kad_node_t *x); /* f(x) = (1-exp(-2x)) / (1+exp(-2x)) (element-wise tanh) */
+kad_node_t *kad_relu(kad_node_t *x); /* f(x) = max{0,x} (element-wise rectifier, aka ReLU) */
+kad_node_t *kad_softmax(kad_node_t *x);/* f_i(x_1,...,x_n) = exp(x_i) / \sum_j exp(x_j) (softmax: tf.nn.softmax(x,dim=-1)) */
+kad_node_t *kad_1minus(kad_node_t *x); /* f(x) = 1 - x */
+kad_node_t *kad_exp(kad_node_t *x); /* f(x) = exp(x) */
+kad_node_t *kad_log(kad_node_t *x); /* f(x) = log(x) */
+kad_node_t *kad_sin(kad_node_t *x); /* f(x) = sin(x) */
+
+kad_node_t *kad_stdnorm(kad_node_t *x); /* layer normalization; applied to the last dimension */
+
+/* operators taking an indefinite number of operands (e.g. pooling) */
+kad_node_t *kad_avg(int n, kad_node_t **x); /* f(x_1,...,x_n) = \sum_i x_i/n (mean pooling) */
+kad_node_t *kad_max(int n, kad_node_t **x); /* f(x_1,...,x_n) = max{x_1,...,x_n} (max pooling) */
+kad_node_t *kad_stack(int n, kad_node_t **x); /* f(x_1,...,x_n) = [x_1,...,x_n] (stack pooling) */
+kad_node_t *kad_select(int n, kad_node_t **x, int which); /* f(x_1,...,x_n;i) = x_i (select pooling; -1 for the last) */
+
+/* dimension reduction */
+kad_node_t *kad_reduce_sum(kad_node_t *x, int axis); /* tf.reduce_sum(x, axis) */
+kad_node_t *kad_reduce_mean(kad_node_t *x, int axis); /* tf.reduce_mean(x, axis) */
+
+/* special operators */
+kad_node_t *kad_slice(kad_node_t *x, int axis, int start, int end); /* take a slice on the axis-th dimension */
+kad_node_t *kad_concat(int axis, int n, ...); /* concatenate on the axis-th dimension */
+kad_node_t *kad_concat_array(int axis, int n, kad_node_t **p); /* the array version of concat */
+kad_node_t *kad_reshape(kad_node_t *x, int n_d, int *d); /* reshape; similar behavior to TensorFlow's reshape() */
+kad_node_t *kad_reverse(kad_node_t *x, int axis);
+kad_node_t *kad_switch(int n, kad_node_t **p); /* manually (as a hyperparameter) choose one input, default to 0 */
+
+/* miscellaneous operations on a compiled graph */
+int kad_size_var(int n, kad_node_t *const* v); /* total size of all variables */
+int kad_size_const(int n, kad_node_t *const* v); /* total size of all constants */
+
+/* graph I/O */
+int kad_save(FILE *fp, int n_node, kad_node_t **node);
+kad_node_t **kad_load(FILE *fp, int *_n_node);
+
+/* random number generator */
+void *kad_rng(void);
+void kad_srand(void *d, uint64_t seed);
+uint64_t kad_rand(void *d);
+double kad_drand(void *d);
+double kad_drand_normal(void *d);
+void kad_saxpy(int n, float a, const float *x, float *y);
+
+/* debugging routines */
+void kad_trap_fe(void); /* abort on divide-by-zero and NaN */
+void kad_print_graph(FILE *fp, int n, kad_node_t **v);
+void kad_check_grad(int n, kad_node_t **a, int from);
+
+#ifdef __cplusplus
+}
+#endif
+
+#define KAD_ALLOC 1
+#define KAD_FORWARD 2
+#define KAD_BACKWARD 3
+#define KAD_SYNC_DIM 4
+
+typedef int (*kad_op_f)(kad_node_t*, int);
+extern kad_op_f kad_op_list[KAD_MAX_OP];
+extern char *kad_op_name[KAD_MAX_OP];
+
+static inline int kad_len(const kad_node_t *p) /* calculate the size of p->x */
+{
+ int n = 1, i;
+ for (i = 0; i < p->n_d; ++i) n *= p->d[i];
+ return n;
+}
+
+/* Additions by Rspamd */
+void kad_sgemm_simple (int trans_A, int trans_B, int M, int N, int K, const float *A, const float *B, float *C);
+/**
+ * Calculate eigenvectors and eigenvalues
+ * @param N dimensions of A (must be NxN)
+ * @param A input matrix (part of it will be destroyed, so copy if needed), on finish the first `nwork` columns will have eigenvectors
+ * @param eigenvals eigenvalues, must be N elements vector
+ */
+bool kad_ssyev_simple (int N, float *A, float *eigenvals);
+
+#endif
diff --git a/contrib/languages-data/af.json b/contrib/languages-data/af.json
new file mode 100644
index 0000000..136228f
--- /dev/null
+++ b/contrib/languages-data/af.json
@@ -0,0 +1 @@
+{"freq":{"D":9246,"E":2445,"F":2510,"G":3299,"A":6930,"B":3706,"C":2451,"L":2519,"M":3951,"N":3334,"O":2514,"H":3034,"I":2837,"J":2196,"K":3663,"U":687,"T":2336,"W":2258,"V":2714,"Q":182,"P":3097,"S":8234,"R":3039,"Y":252,"X":214,"Z":422,"f":13583,"g":42805,"d":77385,"Feb":207,"e":240974,"b":21626,"c":4896,"a":128566,"n":127153,"o":86673,"l":57433,"m":31352,"j":4048,"k":45378,"h":17527,"i":140621,"w":24930,"v":32618,"u":35166,"t":82606,"s":102389,"r":98861,"q":199,"p":23331,"z":1187,"y":11757,"x":1123,"ï":264,"ë":2903,"ê":1053,"é":765,"á":212,"ü":233,"ö":184,"ó":216,"Eur":318,"Eng":637," l":3565," m":7731," n":16000," o":12065," h":7358," i":23795," j":1325," k":6363," d":33601," e":13358," f":1200," g":11018,"Ñ€":242,"Ñ":306," a":8747,"Ñ‚":161," b":8379," c":434," u":1931," t":8537," w":13128," v":24617," p":4859," s":15482," r":3617," J":2155," K":3559," H":2961," I":2185," N":3120," O":2318," L":2396," M":3803," B":3554," C":2109," A":6365," F":2371," G":3138," D":8986," E":2271,"л":219,"к":266," Z":368," Y":241,"и":371,"о":333,"н":199," S":7708,"Ger":200," R":2881,"в":199," Q":162," P":2912,"а":481," W":2205," V":2322," U":571,"е":266," T":2130,"Fra":1006,"A ":345,"Da":804,"Co":478,"Ch":621,"Du":1025,"Do":201,"De":763,"Di":5828,"Fe":367,"Eu":354,"En":721,"El":212,"Ge":659,"Ga":319,"I ":452,"Fr":1217,"Fo":165,"Fi":216,"II ":246,"C ":278,"Au":486,"Ar":425,"At":187,"As":201,"D ":158,"Ba":648,"Af":2087,"Am":566,"An":491,"Ap":353,"Al":628,"Bu":243,"Br":778,"Ca":399,"Bi":180,"Be":880,"Bo":481,"Bl":161,"Kr":224,"Ko":657,"Le":490,"Li":504,"La":658,"Lu":245,"Lo":347,"Me":800,"Mi":548,"Ma":1360,"Mu":186,"Mo":627,"Ni":257,"Ne":763,"Na":666,"No":1092,"Ok":339,"Ol":206,"Her":157,"Gr":1326,"Go":356,"Ha":534,"He":680,"II":369,"Hi":301,"Ho":503,"Hu":294,"Hy":550,"In":919,"Is":158,"It":218,"Ja":713,"Je":157,"Jo":565,"Ju":623,"Ka":1489,"Ki":194,"Ke":447,"Un":253,"Tu":248,"Tr":236,"To":272,"Th":313,"Te":262,"Ta":276,"V ":280,"Sw":402,"Sy":292,"St":964,"Su":1701,"Wo":181,"Wi":534,"Wa":412,"We":720,"Vo":315,"Vr":251,"Vi":374,"Va":314,"Ve":689,"Pr":551,"S ":157,"Pe":310,"Pa":727,"Po":681,"Pi":230,"Os":236,"Oo":423,"Or":191,"Se":814,"Sc":197,"Si":387,"Sl":222,"Sk":201,"Sp":443,"So":680,"Ru":645,"Ry":194,"Sa":728,"Re":621,"Ri":222,"Ro":746,"SA":233,"Ra":223,"Gre":501,"Gri":383,"Gra":158,"b ":1179,"Gro":254,"a ":7054,"i ":2513,"gd":570,"ge":16432,"ga":1621,"gb":319,"fk":224,"fl":183,"fg":323,"ff":351,"fi":1111,"fh":169,"fs":1224,"fr":2334,"fu":174,"ft":300,"fo":725,"Int":180,"he":6229,"ha":2610,"gn":360,"gl":334,"gi":2135,"gh":921,"gg":418,"gu":592,"gt":1512,"gs":1974,"gr":3459,"go":1385,"dt":211,"du":998,"dw":506,"g ":10256,"ea":936,"eb":3497,"ec":406,"ed":5721,"de":18394,"dd":606,"dg":161,"di":29432,"dh":249,"dj":173,"dm":299,"do":2521,"ds":2062,"dr":1453,"ew":3034,"eu":3603,"ev":2016,"ey":309,"fa":570,"h ":864,"Ind":251,"fd":469,"fe":948,"eh":993,"eg":3187,"ef":995,"ee":12296,"el":15653,"ek":7920,"ei":5726,"ep":2393,"eo":692,"en":27638,"em":4686,"et":10282,"es":15156,"er":33393,"ca":479,"e ":78745,"by":1025,"br":1953,"bu":1057,"bo":2123,"bl":1117,"bi":1966,"bb":156,"be":8513,"db":222,"In ":319,"da":3617,"f ":4067,"ct":207,"co":446,"ck":502,"ci":340,"ch":1526,"ce":547,"c ":311,"az":190,"ay":279,"ba":2057,"d ":15502,"at":11369,"as":9342,"ar":11432,"aw":597,"av":407,"au":883,"ak":2797,"al":9554,"ai":1291,"aj":155,"ap":2087,"am":3989,"an":36357,"ac":615,"ad":4564,"aa":18307,"ab":1064,"ag":2729,"ah":292,"ae":907,"af":1901,"nu":917,"nt":6760,"ns":9243,"nr":212,"no":2885,"nn":1621,"ny":191,"nw":666,"nv":455,"oe":6026,"of":3797,"oc":387,"od":1636,"oa":178,"ob":729,"om":5480,"on":10533,"ok":2525,"ol":5346,"oi":587,"og":2271,"oh":382,"ot":3827,"os":3306,"ov":1152,"ou":2993,"op":4558,"oo":12667,"or":14221,"r ":19504,"ow":1144,"pe":3683,"pg":229,"pa":2371,"pl":1195,"lê":351,"po":1932,"ph":223,"pi":1008,"lo":3369,"lm":315,"ll":2990,"ls":2634,"lp":392,"lw":311,"lv":239,"lu":1548,"lt":993,"ly":716,"o ":2083,"md":261,"ma":3853,"mb":2182,"mg":224,"me":9151,"mi":2940,"mm":802,"mp":1223,"mo":1485,"ië":1437,"mt":249,"ms":966,"mu":1085,"p ":4720,"na":6444,"nb":510,"nc":507,"nd":12581,"ne":5737,"nf":203,"ng":9804,"nh":460,"ni":6127,"nj":300,"nk":2057,"nl":616,"nm":203,"jo":532,"ki":2683,"kh":210,"kg":239,"ke":8584,"ka":6722,"m ":5913,"kw":457,"ky":282,"ks":2318,"kt":2084,"ku":1443,"ko":3908,"kr":2375,"kk":1579,"kl":2200,"km":469,"li":9515,"lh":279,"lk":1158,"lj":705,"le":10290,"ld":1944,"lg":1526,"lf":717,"la":8341,"lb":446,"n ":58065,"hr":313,"ht":702,"hu":1684,"hi":1067,"ho":3048,"dé":160,"id":5034,"ic":1058,"ib":451,"ia":2568,"ig":5540,"if":581,"ie":47836,"hy":348,"k ":9212,"ir":2359,"is":17403,"it":9361,"iu":405,"iv":1008,"iw":219,"ik":8953,"il":3774,"im":1386,"in":25004,"io":1984,"eë":1032,"ip":899,"je":609,"ji":572,"iz":156,"l ":8172,"ja":1960,"wy":994,"z ":242,"wi":1800,"wo":4179,"vy":166,"y ":4684,"wa":9856,"we":6959,"vl":1196,"vi":4040,"vu":178,"vr":662,"vo":4078,"uw":282,"uu":992,"ve":5906,"va":16173,"x ":845,"ui":7822,"uk":678,"ul":2052,"ue":905,"ug":1045,"ur":5410,"us":5098,"ut":907,"um":1711,"un":2596,"up":170,"ty":1434,"tu":2643,"tt":1277,"tw":1177,"tv":217,"ub":1182,"ua":728,"ud":950,"uc":160,"w ":232,"to":5433,"tm":201,"tl":667,"ts":3814,"tr":4026,"tg":532,"te":20430,"tk":279,"tj":177,"ti":5658,"th":1701,"tb":213,"ta":9118,"su":1177,"sv":424,"ss":2799,"st":17122,"sy":1309,"sw":531,"sl":1811,"sk":5006,"sn":242,"sm":693,"sp":2566,"oë":412,"so":3731,"sr":312,"sd":385,"sc":448,"sf":208,"se":15556,"sh":473,"sg":396,"sj":338,"si":8436,"u ":1834,"sa":2367,"sb":577,"rr":652,"rs":6262,"rt":4139,"ru":2543,"rv":1198,"rw":1199,"ry":2450,"rp":1265,"ro":8165,"rn":1586,"rm":2087,"rl":1734,"rk":2996,"ri":11752,"rh":614,"rg":2653,"rf":378,"re":10923,"rd":7372,"rc":234,"rb":955,"ra":7710,"t ":22731,"qu":168,"s ":35284,"px":614,"Hy ":529,"py":231,"pt":765,"pu":844,"pp":1058,"pr":3258,"ps":659,"wê":320,"zi":170,"ze":169,"za":209,"yg":162,"ye":406,"yf":643,"yd":927,"yw":439,"ys":1141,"yn":1041,"yl":288,"yk":1145,"Apr":247,"Aug":272,"Afr":2048,"Ame":464,"Ber":218,"Bel":171,"Bre":163,"Bra":191,"Bri":282,"Des":273,"Daa":460,"Chr":224,"Cha":171,"ër":307,"ël":325,"êr":697,"ë ":1979,"ê ":310,"é ":228,"Dit":1028,"Die":4537,"Dui":918,"Ned":417,"Nas":187,"Nov":238,"Noo":595,"Okt":256,"Oli":158,"Oos":361,"Par":313,"Pro":177,"Pre":186,"SA ":161,"Ita":207,"Jan":348,"Joh":290,"Jul":297,"Jun":245,"Kaa":543,"Kan":220,"Kat":191,"Kar":171,"Ker":270,"Kon":276,"Lat":181,"Lit":162,"Mei":281,"Mar":370,"Maa":286,"Mon":210,"Mid":157,"Wil":165,"Wes":439,"Vry":192,"Vol":161,"êre":674,"Swe":193,"Sy ":252,"Sui":1515,"Sta":443,"Ste":208,"Sep":228,"Spa":253,"Rus":560,"Sch":162,"Rep":214,"Rom":176,"Ver":555,"Uni":236,"The":196,"Tur":159,"bin":400,"blo":205,"bli":525,"bla":215,"boe":246,"boo":276,"bor":587,"bou":330,"ban":283,"bal":289,"bai":191,"baa":372,"bas":270,"bar":272,"beh":366,"beg":372,"bee":325,"bed":285,"ber":1916,"bel":540,"bek":1148,"bew":349,"bev":630,"bes":1308,"bet":510,"bie":1052,"ce ":276,"bri":159,"bro":237,"bra":211,"bre":258,"bru":1062,"bur":584,"by ":693,"am ":1182,"ake":292,"al ":2759,"ain":204,"ak ":856,"aie":241,"agt":446,"anu":467,"ann":632,"ant":1705,"ans":3841,"ane":404,"ang":1856,"ani":742,"anj":191,"ank":961,"ap ":635,"ana":788,"anc":195,"and":5528,"amm":186,"amp":480,"ami":512,"ame":657,"amb":236,"ama":204,"alt":231,"als":160,"all":667,"alk":171,"alg":320,"ali":1276,"ald":217,"ale":2352,"alf":209,"ala":367,"an ":18298,"aks":261,"akt":740,"akl":166,"abe":229,"abi":201,"aby":216,"ae ":624,"aag":175,"aad":172,"aak":679,"aai":350,"aan":6190,"aal":1515,"aam":1083,"aas":579,"aar":5293,"aap":567,"aat":1563,"ad ":2565,"afg":266,"ai ":311,"age":184,"afd":268,"adm":206,"adi":436,"ade":539,"ag ":1304,"ads":176,"ach":166,"ada":249,"af ":494,"at ":6755,"arg":256,"are":965,"ard":1124,"ara":390,"aro":332,"arn":185,"arm":157,"arl":301,"ark":397,"ari":1177,"arv":249,"ars":463,"art":1494,"ary":171,"asi":1669,"ase":210,"aso":169,"ar ":3216,"apa":189,"app":418,"aps":269,"as ":5230,"awe":308,"awi":169,"ata":346,"ast":673,"ass":518,"ato":426,"ate":1382,"ati":871,"ats":404,"atu":409,"aty":167,"aus":156,"jaa":1087,"jar":470,"je ":175,"joe":306,"jin":161,"jie":306,"ito":170,"itt":191,"its":1623,"isk":182,"ism":266,"iss":374,"ist":1582,"ita":608,"ite":1331,"itg":386,"iti":469,"ius":176,"ium":203,"ivi":590,"ive":294,"is ":12546,"ion":1252,"eër":158,"ipa":265,"ir ":1648,"isi":1018,"ise":601,"isa":220,"ire":181,"it ":3772,"kil":644,"kie":536,"kin":914,"km ":266,"kgr":173,"kee":210,"kei":339,"kel":962,"ken":2090,"kep":166,"ker":1342,"ke ":3014,"kra":345,"kse":472,"kry":1085,"kri":662,"kou":249,"kor":369,"kop":214,"koo":391,"kon":866,"kom":903,"kol":246,"koe":157,"ks ":710,"kke":1272,"kki":178,"klu":430,"kle":511,"kla":387,"kli":749,"kat":157,"kar":183,"kas":204,"kap":818,"kan":1256,"kal":611,"kaa":1596,"ka ":1388," Ga":319," Ge":658," Fo":161," Fr":1217," Fi":213," Ha":534," He":680," Go":354," Gr":1318," Hy":549," Hu":294," Ho":502," II":202," Hi":301," Ja":710," Is":157," It":218," In":916,"han":779," Ka":1486,"hal":311," Ke":447,"haw":164," Ki":192,"har":356," Jo":563," Ju":622,"haa":238,"had":164," La":657," Le":488," Li":502," Ko":657," Kr":224," Ma":1348," Mi":547," Me":799,"he ":399," Lo":346," Lu":244," Ne":762," Na":662," Ni":257," Mo":624," Mu":186,"hel":273,"hei":994,"hee":465,"hed":169,"het":2911,"her":350,"hem":255," Ap":349," Am":563," An":491," Al":626," Af":2082," Ba":645," Au":486," At":187," As":200," Ar":422," Be":877,"hie":290," Bi":179," Bl":161," Bo":479," Br":777," Bu":243,"his":173," Ca":384," Ch":612," Co":473," Da":803," Di":5802," De":761," Do":196," Du":1024," El":212," En":720," Eu":354," Fe":367," Wo":179," Wi":530," We":720," Wa":412,"god":193,"gs ":887,"gor":522,"gro":2150,"gra":537,"gri":320,"gre":401," Os":236," Or":191," Oo":422," Po":674," Pi":229," Pe":309," Pa":725,"gst":406," No":1092," Ol":205," Ok":339,"gte":962,"gti":391," Ra":221," Ro":743," Re":620," Ri":222," Pr":547,"gus":284," Sy":292," Sw":400," Su":1700," St":953," Ta":273," Th":307," Te":261," Tr":236," To":270," Ry":194," Ru":645," Sa":724," Si":385," Sc":196," Se":811," So":678," Sp":441," Sk":201," Sl":222," Va":313," Ve":669," Vi":371," Vo":314," Vr":251," Tu":243," Un":253," ja":1102,"ial":357,"ian":256," in":12303,"iaa":736," is":11238," ka":1533," ki":531," ke":481,"id ":2425," ha":612," he":3438," gr":2075," go":365,"ia ":794," hy":292," hi":477," ho":1750," hu":727,"iet":320,"ieu":180,"iew":413," ni":722,"iel":277," ne":437,"ien":998," na":2339,"ier":2228,"ies":4471,"ied":1248,"ief":177,"iek":2103," mu":691,"ig ":1346," mo":667," om":1497," on":2106," of":1952,"ifi":218," no":1205," le":910," li":598," n ":10980," la":1290," ku":387,"ich":258,"ie ":34696," km":407," kl":879,"ica":209," kr":319," ko":1672," me":4100," mi":830,"ids":257," ma":1329," lu":186,"idi":291,"ide":993,"idd":457,"ida":156," lo":197," af":820," aa":2320," ad":269," am":322," an":759," ak":286," al":829," ar":263," at":229," as":2284," ba":599,"il ":459," bi":320," be":5430," bo":565," bl":263," by":612," bu":213," br":340,"ika":2950,"igd":381,"ige":1604,"igh":698,"igi":270,"igg":185,"igt":498,"igs":156,"ik ":2305," en":9738,"imp":231," ei":517," el":502,"ime":187," ek":223," ee":1730,"ind":1030,"ina":506," fa":191,"inn":302," fo":227,"int":638,"ins":1349,"ine":545,"ing":6095," fi":368,"ini":615,"ink":417," ge":8191," ga":169,"inw":455,"ikk":629," ch":185,"ike":1814,"ila":498," da":1923,"in ":12178,"iku":209,"iks":287," do":1111,"ilo":514,"ill":662," dr":523," de":3947,"ilj":228,"ili":684,"ild":294," di":25510,"imb":245,"eë ":693,"io ":196," du":309," wê":298,"hom":166,"hou":360,"hoo":1325,"hoe":410," wy":201,"hul":552,"hui":260,"hri":224,"ht ":578," ru":233," sa":888," se":2315," si":590," sl":329," sk":1250," sp":887," so":2211," ra":237," re":1576," ri":825," ro":614," pr":1589," s ":207," px":614,"hy ":302," ou":447,"hum":674," oo":2639," op":2809," or":325," pe":402," pa":556," pl":641," po":737," lê":242," wa":7840," we":1395," wo":2888," wi":454," va":14670," ve":4043," vo":2359," vr":575," vi":2068," vl":594," ty":439," tw":582," tu":692," ui":1746," ta":895," sw":227," sy":1183," st":4293," su":859," tr":387," to":1857," th":729," ti":190," te":2715,"ffe":165,"fer":157,"fel":155,"fha":158,"fge":290,"fam":176,"fde":429,"eta":359,"ete":1299,"eti":372,"esp":358,"eso":210,"est":2951,"ess":405,"eun":234,"eto":320,"etr":438,"ets":217,"ett":493,"eve":456,"eva":262,"evo":907,"evi":274,"eur":2292,"eus":242,"ewi":337,"ewe":1704,"ewo":449,"ey ":181,"ewa":222,"epe":254,"er ":10617,"epa":228,"eor":221,"es ":4626,"ept":277,"epu":400,"epr":184,"erk":2067,"erl":875,"eri":1765,"erg":1022,"erh":416,"ere":1861,"erf":286,"erd":1514,"era":1470,"erb":529,"et ":6083,"esk":1018,"esl":228,"esi":976,"ese":3607,"eu ":338,"erv":860,"erw":949,"err":349,"ert":1101,"ers":4583,"ern":1142,"erm":861,"erp":342,"ero":382,"ekg":155,"ekk":206,"eko":474,"eks":950,"ekt":701,"en ":13492,"ela":904,"eld":1199,"elf":322,"ele":2593,"eli":1906,"elj":427,"elg":226,"elk":209,"ell":778,"elo":234,"els":1983,"elt":333,"ely":255,"emb":839,"ema":484,"eme":1266,"emo":181,"emi":456,"ep ":699,"ene":1142,"enh":254,"eng":314,"enb":269,"ena":610,"end":3112,"eno":500,"enn":400,"enk":275,"eni":1151,"ens":2864,"ent":2318,"ego":497,"ege":690,"egi":516,"eha":370,"egr":238,"egs":217,"egt":193,"eho":266,"ehe":259,"ek ":1799,"eis":330,"eil":544,"ein":1010,"eie":633,"eid":1307,"el ":3516,"eit":680,"eke":2739,"eka":220,"em ":967,"gin":784,"gie":714,"ght":548,"gep":249,"gen":1564,"get":297,"ger":1248,"ges":2014,"gev":788,"gew":944,"gee":448,"ged":475,"geb":2499,"geh":356,"geg":181,"gem":756,"gel":1995,"gek":350,"gde":427,"ge ":1916,"gaa":266,"gan":539,"ga ":157,"fst":852,"fri":2089,"for":371,"fie":369,"fil":208,"fin":174,"fis":177,"da ":327,"de ":6409,"daa":645,"dag":700,"dae":480,"dat":659,"dan":233,"dam":165,"dde":490,"ch ":316,"cha":160,"ck ":233,"che":490,"ed ":1090,"eba":159,"ebe":354,"ebi":752,"ebo":768,"ebr":1168,"ei ":821,"ega":168,"eek":631,"een":2520,"eel":2072,"eem":410,"eed":587,"ees":884,"eer":3295,"eeu":449,"eet":195,"edi":638,"ede":2561,"eda":161,"eg ":316,"eds":321,"edr":340,"ee ":892,"ef ":280,"dwe":310,"dus":171,"dor":875,"doo":416,"don":160,"dom":227,"ds ":353,"dmi":211,"doe":283,"dst":428,"dui":309,"dri":421,"dra":423,"dry":204,"dsk":181,"dse":527,"dia":294,"der":4829,"des":476,"deu":1676,"dee":1279,"del":1695,"dek":186,"den":1206,"do ":172,"din":875,"dio":177,"dis":425,"dit":656,"die":24964,"dig":1168,"dik":198,"rhe":301,"rga":496,"rgi":335,"rge":595,"ret":312,"res":944,"rg ":777,"rea":245,"ree":1091,"ref":257,"red":294,"rei":545,"reg":1039,"ren":1300,"rek":765,"rel":674,"rep":191,"rf ":180,"rdo":215,"rdi":841,"rde":1873,"re ":2607,"rd ":3667,"ras":532,"rat":587,"rbi":190,"rba":160,"rbe":287,"rag":291,"ran":2011,"ram":317,"ral":832,"rak":247,"raa":1046,"raf":284,"rad":331,"rs ":1922,"ros":273,"rot":330,"rom":305,"ron":1072,"roo":1778,"rop":575,"rou":212,"rov":708,"rod":199,"rol":315,"roe":1277,"rog":195,"rno":196,"rp ":728,"rna":508,"rne":469,"rmo":164,"rma":539,"rme":324,"rmi":175,"rlo":320,"rli":409,"rle":270,"rla":508,"rks":184,"rko":248,"rki":199,"rkl":203,"rke":440,"rka":271,"rm ":692,"rio":174,"rit":493,"ris":571,"riv":501,"rig":863,"ril":278,"rik":3384,"rin":1384,"ria":924,"ric":236,"rie":2029,"rk ":1040,"rwe":410,"rwy":498,"ryf":393,"rui":1143,"rug":256,"rum":244,"ruk":231,"rus":225,"rva":502,"rvl":353,"rvo":192,"rwa":171,"ry ":383,"rsk":872,"rsi":432,"rso":249,"rsp":591,"rsa":225,"rse":478,"rta":186,"rst":1083,"rtk":160,"rto":274,"rte":620,"rti":334,"rua":209,"rty":351,"rt ":1413,"rre":272,"saa":540,"sal":170,"sam":303,"san":408,"sas":204,"sa ":155,"ryw":338,"rys":282,"ryk":576,"sge":305,"sie":4039,"sid":185,"sia":299,"sit":436,"sis":296,"sip":279,"sin":541,"sio":799,"sil":194,"sim":173,"sik":231,"sif":160,"sig":289,"sbu":231,"se ":9840,"sch":268,"ser":501,"ses":400,"set":250,"seu":239,"sea":162,"see":618,"sed":264,"sen":1323,"sem":298,"sel":1093,"sek":186,"spo":405,"spr":756,"spe":934,"spa":260,"sow":508,"som":247,"son":545,"soo":954,"soe":195,"sok":377,"st ":267,"sli":202,"slu":297,"sky":183,"sla":1006,"sle":205,"ski":804,"sko":594,"skr":1152,"sku":244,"ska":1212,"ske":665,"sië":283,"sma":173,"sme":382,"sse":1275,"ssa":198,"ssi":922,"ste":6829,"sta":5065,"sto":805,"sti":1396,"stu":693,"str":1673,"sty":226,"sui":596,"sve":167,"sy ":1199,"swa":313,"tal":1301,"taa":2499,"tad":2323,"tau":165,"tat":456,"tas":164,"tan":1021,"te ":8469,"ta ":339,"pa ":202,"pe ":459,"par":608,"pas":176,"paa":333,"pal":324,"pan":428,"pge":207,"pen":295,"per":1379,"pes":438,"pee":201,"pel":568,"pla":660,"lê ":268,"pli":169,"ple":241,"pie":480,"por":394,"poo":160,"pos":197,"pol":518,"ppy":184,"ppe":636,"pst":229,"pub":435,"pte":575,"pra":251,"pri":484,"pre":726,"pro":1677,"pun":246,"px ":614,"py ":166,"ra ":424,"ngo":161,"ngr":289,"ngs":1292,"nge":2327,"nhe":276,"nel":314,"nen":189,"nem":225,"ner":1014,"net":468,"nes":533,"ng ":4906,"nee":762,"nce":206,"ne ":1530,"ndr":216,"nds":657,"ndo":326,"ndi":878,"nde":5081,"nda":453,"nal":790,"nam":291,"nad":316,"naf":372,"nab":229,"naa":1198,"nd ":4245,"nat":282,"nas":677,"na ":1572,"nwo":542,"nus":209,"nua":266,"ntw":393,"nto":201,"nts":300,"ntr":543,"nti":571,"ntl":164,"nta":457,"nte":1815,"nst":787,"nse":3345,"nsi":1079,"nsl":207,"nsk":498,"nt ":1757,"ns ":2476,"nog":456,"noe":477,"noo":659,"nom":368,"nne":904,"nni":442,"nië":246,"nli":373,"nke":345,"nkl":391,"nks":179,"nkr":453,"nje":156,"nig":640,"nie":1831,"nk ":274,"niv":210,"nis":1512,"nin":804,"ogr":272,"ogi":423,"oi ":216,"oha":228,"oeë":178,"ok ":1432,"ol ":554,"ock":164,"oe ":303,"ode":551,"odi":176,"ods":177,"of ":2323,"oek":499,"oel":276,"oem":563,"oeg":231,"oei":336,"oer":752,"oes":295,"oet":302,"oen":602,"oep":714,"odu":188,"oed":477,"og ":895,"ofs":803,"oew":261,"od ":254,"obe":382,"oud":510,"oue":197,"ote":350,"ott":175,"ots":913,"oto":266,"ost":637,"osi":266,"ose":346,"oss":176,"oso":190,"owa":484,"owe":208,"ovi":678,"ove":370,"ous":302,"our":167,"out":306,"opo":205,"opp":449,"ope":438,"opg":213,"opa":195,"os ":1171,"oon":731,"ool":561,"oom":393,"ook":1376,"ooi":288,"oof":1146,"oog":389,"ood":288,"or ":1152,"oot":1351,"oos":958,"oor":4776,"oop":341,"ork":260,"orl":386,"orm":964,"orp":858,"ord":4583,"ore":773,"org":587,"ori":1212,"ou ":999,"ort":1219,"ors":871,"orw":195,"ot ":1528,"orb":186,"ora":235,"ola":171,"on ":1522,"oli":772,"oll":288,"olk":702,"ole":263,"olg":904,"ols":270,"olo":636,"om ":1870,"okk":553,"ona":980,"ond":1915,"one":1178,"ong":620,"oni":1012,"onl":220,"onk":232,"onn":184,"ono":391,"ons":511,"ont":1339,"oma":425,"ome":845,"omi":324,"omm":454,"omp":297,"oms":595,"op ":2264,"la ":334,"le ":3834,"lf ":175,"lde":601,"laa":982,"lad":180,"lag":434,"lak":490,"lan":4154,"lar":155,"lat":361,"las":433,"ld ":695,"kus":410,"kun":548,"kul":242,"kwe":204,"kwa":191,"kte":822,"kst":257,"ksi":463,"ktr":342,"ktu":210,"kti":247,"kto":369,"ls ":1008,"lon":293,"lom":430,"loo":382,"loe":423,"log":655,"los":274,"lië":349,"lti":157,"lub":411,"lug":221,"lst":643,"lte":252,"lse":623,"lge":754,"lew":250,"leu":193,"les":329,"let":347,"ler":415,"lem":358,"len":1056,"lek":605,"lei":1010,"leg":257,"lee":477,"led":218,"lg ":483,"lo ":169,"lla":325,"lle":1578,"lli":615,"lke":200,"lki":447,"ljo":223,"ll ":176,"lja":430,"lit":831,"lis":504,"leë":449,"lin":1208,"lim":201,"lid":165,"lia":364,"lik":2917,"lig":818,"lie":1618,"ma ":226,"mb ":655,"maa":1244,"mag":221,"mar":331,"mas":207,"mal":270,"man":726,"mat":394,"mba":172,"mbi":179,"mbe":814,"mbo":161,"me ":936,"mde":163,"med":223,"mee":1533,"met":2981,"mes":247,"mer":991,"mel":330,"men":1550,"lui":390,"lus":194,"lwe":213,"lyk":221,"lyn":187,"mpi":220,"mpe":208,"mpo":176,"mpt":267,"ms ":488,"moe":196,"mod":233,"mon":329,"mst":248,"mus":488,"mun":417,"ër ":180,"mge":191,"min":806,"mil":465,"mit":231,"mig":184,"mie":523,"mid":310,"ië ":1136,"mme":353,"wêr":319,"yst":183,"ys ":680,"ywe":370,"ye ":306,"yf ":380,"yde":281,"yds":165,"yd ":230,"yn ":461,"yns":175,"yk ":810,"wys":531,"wor":2620,"woo":760,"won":526,"we ":1260,"wes":799,"wer":1583,"wet":305,"wen":427,"wel":545,"weg":270,"wee":1257,"wis":166,"wit":342,"wie":194,"win":417,"wil":177,"wik":231,"wan":300,"wat":5174,"war":532,"was":2236,"waa":1031,"vry":194,"vro":313,"vir":1570,"vin":921,"vie":880,"vis":289,"vla":709,"vlo":280,"voe":444,"vol":1592,"voo":1083,"vor":625,"ver":4566,"ven":170,"vem":236,"vel":250,"vee":302,"val":319,"van":14723,"vat":155,"vaa":414,"uwe":229,"uur":863,"usl":180,"usi":606,"use":380,"ust":585,"uss":1129,"ute":176,"uto":171,"us ":1998,"ure":395,"urg":669,"uri":191,"urk":167,"uro":352,"urs":211,"urt":189,"ur ":2547,"umb":689,"ume":172,"unt":325,"uns":289,"uni":820,"und":530,"um ":614,"ult":270,"ull":459,"uli":358,"un ":219,"uid":2285,"uik":850,"uim":162,"uis":508,"uk ":200,"uit":3378,"ul ":272,"ugb":161,"ugu":278,"ude":184,"udi":240,"ue ":322,"ug ":159,"ub ":406,"uar":522,"ubl":464,"ud ":181,"tyn":228,"ty ":384,"tur":232,"tus":988,"tuu":617,"tui":232,"tud":171,"tyd":628,"twi":269,"twe":751,"ts ":533,"tre":1022,"tra":1128,"tri":607,"tru":366,"tro":780,"tse":746,"tsk":298,"tsl":425,"tst":993,"tte":641,"tti":226,"to ":272,"tof":244,"toe":713,"tob":268,"tot":1108,"tom":182,"ton":586,"tol":317,"tor":808,"too":280,"til":187,"tik":334,"tie":1846,"tig":1053,"tis":241,"tin":826,"tio":267,"thu":695,"tkl":165,"tli":191,"tla":301,"tem":732,"ten":1059,"tei":844,"tek":528,"tel":2135,"tee":779,"teg":166,"ted":237,"th ":270,"teu":212,"tes":357,"ter":4231,"tge":442,"the":380},"n_words":[1541130,1808182,1328687],"name":"af","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/an.json b/contrib/languages-data/an.json
new file mode 100644
index 0000000..299ca71
--- /dev/null
+++ b/contrib/languages-data/an.json
@@ -0,0 +1 @@
+{"freq":{"D":4007,"E":9481,"F":6264,"G":7528,"A":24874,"B":11416,"C":19854,"L":12511,"M":14032,"N":5019,"O":7507,"H":3089,"I":5567,"J":2444,"K":1244,"U":4208,"T":7124,"W":1232,"V":6098,"Q":391,"P":13954,"S":13768,"R":7862,"Y":2048,"X":966,"Z":2105,"f":30904,"g":39756,"d":198344,"e":418034,"b":46837,"Fed":103,"c":184692,"a":500037,"n":332450,"o":309189,"l":153654,"m":96885,"j":1745,"Fel":132,"k":9548,"h":38996,"Fen":49,"i":297238,"w":1593,"v":28687,"Fer":367,"u":160116,"t":216624,"s":206494,"r":234141,"q":14602,"p":79744,"z":13501,"y":82928,"x":11341,"ª":69,"º":77,"²":4897,"Ã":100,"É":262,"Ã":131,"À":43,"Fil":103,"ï":110,"í":10894,"Fin":74,"ì":53,"ë":76,"ê":61,"é":16262,"è":1639,"ç":714,"ä":54,"Fie":45,"ã":102,"â":100,"Fig":108,"á":13401,"à":575,"ü":700,"ú":1261,"ø":41,"ù":89,"Fis":88,"ö":123,"ô":70,"ò":886,"ó":30243,"ñ":789,"Ä":118,"Ä«":53,"ÅŸ":47,"Å‚":55,"Å¡":45,"Å£":70,"Å«":44,"Fa ":481,"Fan":52,"Fal":65,"Far":69,"Fab":67,"Aín":47,"Era":60,"Eri":53,"Ess":225,"Est":1829,"Esp":2346,"Ern":54,"Esc":294,"É™":50,"Esl":79,"Eus":72,"Eur":458,"Exe":241,"ˈ":99,"El ":405,"Ì":61,"En ":645,"Ele":55,"Enc":143,"μ":51,"ν":114,"ο":175,"Eng":50,"θ":57,"ι":93,"κ":64,"λ":98,"ε":64,"η":68,"α":169,"γ":43,"ά":43,"ί":50,"Emp":168,"Emi":77,"Eli":66,"Ent":56,"ÏŒ":42,"σ":63,"Ï‚":149,"Ï":123,"Ï„":88," l":24121," m":23977," n":9906," o":46236," h":9312," i":7953," j":137," k":5618," d":142714," e":62754," f":15333," g":3703,"ч":80,"Ñ€":160," a":64432,"Ñ":155," b":4777,"Ñ‚":77," c":53853,"у":58," y":53553," x":86," z":803," u":35543," t":11133," w":158," v":4587," q":10572," p":39916," s":31762," r":12559," J":2437," K":1235,"Cál":77," H":3071," I":5555," N":5003," O":7480," L":12489," M":14016," B":11404," C":19641," A":24795,"С":41," F":6210," G":7491," D":3992," E":9455,"л":143," Z":2101,"к":142," Y":2045,"й":58," X":966,"и":262,"п":41,"о":215,"н":140,"м":41," S":13749,"Ges":43,"Ger":648," R":7849,"в":147," Q":385," P":13942,"а":282,"Geo":165," W":1227," V":6094,"Gen":180," U":4207,"е":199,"д":65," T":7101," á":138," ª":48," º":71," Ã":127," À":43,"Gio":49," É":259," Ã":100,"Gil":43,"Gir":65,"Giu":47,"×™":48,"Gan":52,"Gal":553,"Gav":41,"Gau":302,"Gar":1065,"Gas":177,"Gab":72,"Ùˆ":72,"ÙŠ":130,"Ù„":158,"Ù…":103,"Ù†":111,"Fun":74,"Fue":512,"د":70,"ج":41,"Ø­":44,"ب":113,"Ø©":63,"ا":224,"ع":41,"س":66,"ر":91,"Fro":85," ˈ":61,"Flo":152,"Fla":78,"Fra":1775,"Fri":136,"A ":6623,"Fre":184,"Fon":159,"For":357,"F ":41,"Da":680,"Cu":873,"Cy":45,"Cl":510,"Co":3422,"Cr":538,"Ce":1517,"Ch":3175,"Ci":1041,"Ec":152,"Ed":409,"Ea":51,"Eb":284,"Du":470,"Do":659,"Dr":118,"De":985,"Di":657,"Fe":747,"H ":44,"Fa":903,"Ez":49,"Eu":622,"Ev":68,"Ex":340,"Er":278,"Aí":55,"Et":88,"Es":4887,"En":989,"Em":290,"Ei":68,"El":695,"Ge":1116,"Cá":129,"Ga":2449,"I ":1360,"Fu":667,"Fr":2222,"Fo":713,"Bé":72,"Fl":289,"Fi":498,"B ":58," С":41,"II ":724,"C ":328,"Av":152,"Au":1363,"Ar":3874,"Aq":804,"At":949,"As":1097,"D ":277,"Ba":3644,"Az":169,"Ay":82,"Af":232,"Ag":240,"Ab":293,"Ac":412,"Ad":285,"Am":430,"An":2149,"Ap":98,"Ai":212,"Al":5289,"Hit":59,"His":162,"Bu":1318,"Br":1101,"Ca":8068,"E ":158,"Bi":1877,"Be":1779,"Hil":81,"Bo":1120,"Bl":233,"Ku":88,"Kl":47,"Kr":75,"Ko":157,"Le":3160,"Li":1190,"N ":108,"Gú":51,"La":4114,"Lu":751,"Ly":43,"Ll":306,"Lo":1960,"Me":3060,"Mi":1407,"O ":3098,"Ma":5263,"Mc":66,"Mu":1274,"Mo":2562,"Ni":338,"Ne":437,"Na":1458,"P ":71,"Hel":71,"Hei":41,"Nu":796,"No":1516,"Ol":435,"Om":55,"On":170,"Oh":48,"Oc":530,"Od":84,"Hen":171,"Hes":57,"Ob":102,"Her":278,"Gi":355,"Cé":64,"Gl":100,"Gr":1086,"Go":468,"Gu":1597,"J ":56,"Ha":782,"He":720,"Hi":448,"Ho":594,"Hu":367,"Ib":231,"Id":48,"Ig":114,"Ip":108,"Im":754,"In":710,"Il":297,"Is":1033,"It":588,"Ir":296,"Ja":569,"L ":794,"Ji":54,"Je":311,"Jo":919,"Had":44,"Jr":59,"Ju":422,"Fú":72,"Ka":309,"Han":91,"M ":73,"Ham":67,"Kh":65,"Har":170,"Ki":176,"Hau":83,"Ke":203,"Us":67,"Ut":63,"Ur":312,"Un":1997,"Ul":142,"Ue":1325,"Uc":50,"Tu":387,"Tr":784,"Pé":62,"To":1748,"Th":491,"Ti":748,"Te":1399,"Ta":1245,"V ":289,"Sy":56,"St":450,"Su":913,"Wo":126,"Wu":48,"Wi":414,"Sã":53,"Wh":41,"Sé":45,"Wa":266,"We":251,"Vo":163,"Rí":105,"Vi":2398,"X ":302,"Va":2416,"Ve":691,"Uz":41,"Mé":58,"Pu":602,"Pr":1737,"S ":375,"Pe":1603,"Gua":636,"Pa":3522,"Gue":449,"Gub":94,"Lé":44,"Gui":272,"Pl":536,"Po":1463,"Pi":4009,"Ph":120,"Os":1276,"Ot":107,"Ou":63,"Ov":54," ا":100,"Op":71,"Or":1187,"R ":115," ب":40,"Oz":48,"Se":3318,"Sc":255,"Si":1208,"Sh":202,"Sp":125,"So":1654,"Ru":549,"U ":53,"Nù":59,"Sa":4961,"Ná":254,"Re":2929,"Ri":1399,"Ro":1931,"Qu":333,"T ":55,"Mú":79,"Ra":687,"Gre":227,"Gri":72,"Gra":645,"b ":2959,"Gru":82,"Gro":44,"a ":189570,"Ye":1235,"Ya":59,"Yo":603,"Yu":72,"Xa":108,"Tè":61,"Xi":132,"Gon":55,"Gol":63,"Gor":90,"Za":1741,"Ze":94,"Zi":59,"Zo":51,"Zu":79,"God":44,"i ":5556,"bó":122,"gd":66,"ge":1559,"cá":136,"ga":9445,"Inf":101,"bé":284,"fl":536,"bè":98,"fg":41,"ff":208,"fi":10107,"fr":7629,"fu":1368,"ft":72,"Int":142,"fo":3130,"Ins":100,"bí":66,"j ":55,"cú":47,"có":298,"Ipa":95,"dá":285,"he":5671,"ha":11432,"gn":724,"cé":6120,"gm":75,"gl":1957,"gi":469,"gh":348,"gg":120,"gu":5563,"gt":79,"gs":124,"gr":4212,"cí":202,"go":11703,"dt":51,"du":1967,"dv":108,"dw":105,"dy":121,"g ":1073,"ea":2408,"eb":3508,"ec":11322,"ed":4670,"de":103587,"dd":182,"dg":113,"di":13701,"Ile":92,"dh":96,"dm":483,"dl":77,"do":9223,"dn":56,"ds":221,"Ill":119,"dr":2386,"ew":230,"ex":1637,"añ":291,"eu":4920,"ev":2707,"ey":6445,"ez":1432,"fa":2850,"h ":1778,"Ind":230,"bá":102,"fe":3740,"eh":152,"eg":5665,"ef":994,"ee":482,"el":19178,"aç":51,"ek":155,"ej":370,"ei":5762,"ep":5798,"Imp":731,"eo":1144,"en":93499,"em":9197,"et":7416,"es":36862,"aï":66,"er":42074,"aí":1342,"eq":445,"ca":36727,"e ":149038,"by":52,"bs":272,"br":9258,"bu":1135,"bt":83,"bn":53,"bo":2387,"bp":52,"bl":8980,"bf":48,"bi":6912,"bb":79,"bc":225,"bd":79,"be":4754,"dc":59,"da":17903,"f ":486,"cy":72,"cu":2858,"ct":6432,"cs":190,"cq":106,"cr":2764,"co":36494,"cn":93,"ck":597,"cl":2381,"ci":55866,"ch":19957,"ce":5873,"cc":5812,"c ":1440,"aC":163,"az":1496,"ay":2308,"ba":8653,"d ":44350,"at":36951,"as":32668,"ar":48825,"aq":326,"ax":419,"aw":197,"av":3185,"au":6954,"ak":399,"al":37899,"ai":7870,"aj":395,"ao":175,"ap":3319,"am":13303,"an":60138,"ac":18793,"ad":8656,"aa":113,"ab":11063,"ag":9213,"ah":422,"ae":821,"af":2379,"nu":1742,"nt":42111,"ns":10765,"nr":248,"nq":367,"no":15371,"nn":1413,"q ":75,"nz":961,"ny":8884,"nv":500,"oe":616,"of":4127,"oc":10213,"od":4405,"oa":667,"ob":10136,"om":23722,"on":36685,"ok":161,"ol":10871,"oi":2459,"oj":139,"og":2423,"oh":450,"ot":2800,"m²":4897,"os":31113,"ov":10545,"ou":3052,"op":2703,"oo":407,"or":29437,"oq":218,"r ":21128,"ox":224,"ow":337,"oz":1792,"oy":492,"là":56,"lá":4259,"pe":9762,"Igu":42,"pa":17607,"pc":112,"pl":2368,"lè":221,"lé":756,"po":17316,"ph":314,"pi":11513,"lo":14735,"ln":95,"lm":4777,"hé":116,"hè":52,"ll":16721,"ls":3809,"lr":42,"hí":504,"lq":133,"lp":382,"hó":68,"lv":726,"lu":5364,"lt":4369,"lz":119,"ly":296,"o ":118703,"ià":102,"ma":22094,"mb":4820,"iá":1740,"me":16458,"mf":70,"eÅ£":45,"iè":238,"mi":9268,"mn":143,"mm":270,"ié":1837,"mp":6474,"mo":7848,"ms":136,"mu":18950,"iñ":101,"ió":20432,"my":57,"p ":402,"na":42520,"nb":139,"nc":27078,"nd":12460,"ne":13823,"nf":817,"ng":5331,"nh":555,"ni":23728,"nj":111,"nk":294,"nl":294,"nm":99,"Ibe":153,"ju":130,"fí":317,"jo":403,"ki":580,"kh":127,"ke":399,"ka":550,"fú":249,"m ":1910,"gó":1532,"ky":77,"ks":168,"ku":119,"gò":79,"ko":322,"kr":45,"kl":108,"km":5396,"li":16018,"lh":504,"lk":130,"há":74,"le":12141,"ld":1220,"lg":344,"lf":641,"hâ":41,"gü":326,"la":35610,"lc":1027,"lb":1209,"gú":46,"n ":117507,"hr":216,"hs":61,"dí":2145,"dò":202,"dó":142,"ht":207,"hu":6167,"hi":10022,"hn":341,"ho":1603,"hl":125,"dè":107,"dé":369,"hm":76,"id":11407,"ic":35920,"ib":3636,"ia":33192,"ih":91,"ig":5438,"if":2242,"ie":19241,"hy":121,"k ":1459,"eí":56,"iq":449,"ir":8944,"is":13378,"it":41046,"eñ":206,"iu":2673,"iv":4272,"eó":80,"ix":7972,"ii":54,"ij":127,"ik":367,"il":9035,"im":5807,"in":28928,"io":20463,"ip":9686,"je":160,"ji":76,"iz":2558,"iy":76,"l ":26412,"ja":752,"xi":5714,"xo":1208,"tè":189,"té":212,"xm":56,"xp":240,"tí":1477,"tó":3270,"xt":359,"xu":93,"só":139,"sú":82,"z ":3529,"xc":69,"xa":1261,"tá":2483,"xe":1372,"wi":289,"sè":69,"sé":315,"wn":65,"wo":96,"sí":199,"ws":51,"sò":136,"ró":536,"rò":79,"y ":30607,"rú":70,"wa":371,"sá":61,"we":271,"rè":89,"vl":53,"ré":394,"vi":14449,"vu":203,"vr":185,"rí":2661,"vo":2258,"uz":858,"uy":8987,"ux":238,"uv":360,"rá":410,"ve":4086,"rà":73,"va":6899,"qü":104,"x ":772,"ui":5582,"uj":105,"uk":95,"ul":4535,"ue":24260,"uf":231,"ug":3258,"uh":51,"uq":149,"ur":7842,"us":8828,"ut":4110,"um":1883,"un":51530,"uo":167,"up":3752,"ty":165,"tx":92,"tz":1335,"tu":10787,"tt":792,"tw":45,"pó":230,"ub":3284,"ua":11407,"ud":6151,"uc":2944,"w ":229,"to":49861,"tn":174,"tm":126,"tl":1081,"pè":62,"ts":4585,"tr":16094,"pí":43,"tp":51,"tg":118,"tf":100,"te":23226,"ti":24986,"th":1241,"v ":145,"tb":466,"tc":162,"ta":49236,"su":14097,"sv":101,"ss":1947,"st":36036,"sy":54,"sz":61,"sw":69,"sl":1702,"sk":471,"sn":175,"sm":1643,"sp":5348,"so":6479,"sr":79,"oí":58,"sq":228,"sd":259,"sc":6321,"sf":297,"se":12003,"sh":816,"sg":92,"si":17580,"rz":1385,"u ":7771,"sa":8339,"sb":292,"rr":10518,"rs":4137,"rt":18219,"ru":3976,"rv":867,"nò":54,"rw":51,"nó":211,"rx":65,"ry":499,"rq":661,"ní":360,"rp":829,"ro":26141,"rn":3248,"né":3667,"rm":3699,"nè":64,"rl":1733,"rk":850,"nç":561,"ri":26132,"rh":94,"rg":3076,"rf":2952,"re":36472,"ná":154,"rd":4758,"rc":8506,"rb":2086,"ra":47500,"t ":23983,"mú":101,"mó":149,"qu":14376,"mí":198,"mé":389,"má":2404,"s ":90855,"px":43,"pt":487,"pu":3631,"lò":43,"ló":828,"pp":201,"lí":371,"pr":15141,"ps":257,"Hum":43,"IX ":132,"Hue":159,"zá":80,"yú":116,"zí":110,"zó":127,"yá":156,"IV ":156,"yó":1722,"xó":86,"zz":103,"Hor":78,"zh":42,"zi":554,"zb":53,"zc":370,"Hos":42,"ze":662,"vá":44,"za":5335,"Hon":180,"Hol":97,"zu":406,"zt":78,"zo":1662,"zq":48,"ví":116,"zk":61,"vé":50,"zm":43,"vè":48,"ye":26389,"uá":93,"yd":68,"ya":13178,"tú":55,"uñ":102,"yu":88,"yt":84,"ys":168,"yr":129,"uí":262,"yo":9039,"yn":174,"ué":1015,"ym":83,"uè":92,"yl":123,"yi":439,"Arg":208,"Are":82,"Arc":243,"Ard":75,"Ara":1956,"Arb":109,"Arm":172,"Arn":49,"Arl":72,"Ari":148,"Aqu":797,"As ":436,"Ate":44,"Ath":42,"Atl":768,"Ast":163,"Asi":176,"Aso":50,"Asp":82,"Arr":261,"Arq":53,"Art":241,"Ars":45,"Avi":56,"Ave":58,"Aut":173,"Aus":575,"Aur":142,"Aul":197,"Aug":97,"Azu":58,"Bai":742,"Bal":491,"Ban":328,"Bac":68,"Bad":97,"Bay":89,"Bar":1038,"Bat":90,"Bas":270,"Bav":98,"Bau":42,"Abr":40,"Aca":110,"Abe":43,"Aba":96,"Act":184,"Ado":91,"Adr":43,"Afr":187,"Agu":108,"Agr":41,"Ain":41,"Air":59,"Al ":59,"Ala":196,"Alb":564,"Alg":48,"Ali":368,"Alc":398,"Ald":69,"Ale":590,"Alf":219,"Alt":2147,"Alm":205,"All":75,"Alp":176,"Ame":169,"Ama":66,"Amo":43,"Anh":44,"Ang":673,"Ani":69,"Ana":102,"And":336,"Anc":46,"Ans":48,"Ant":494,"Ano":61,"Ann":99,"Bus":74,"Bul":86,"Bun":61,"Bur":806,"Bud":62,"Bue":80,"Bru":135,"² ":4882,"Cab":319,"Cad":58,"Cal":895,"Cam":1021,"Cai":58,"Cas":3464,"Car":802,"Cau":98,"Cat":426,"ª ":69,"Can":590,"Cap":118,"Caz":55,"Bea":178,"Bes":62,"Baï":53,"Bet":64,"Ber":553,"Ben":365,"Bel":385,"Bid":83,"Bie":159,"Bia":400,"Ãn":71,"Biz":81,"Bil":130,"Big":731,"Bis":62,"Bla":163,"Bre":219,"Bra":269,"Bro":165,"Bri":276,"Boc":41,"Boh":42,"Bol":188,"º ":75,"Bon":127,"Bor":282,"Bos":103,"Bou":109,"Cur":50,"Cub":64,"Cue":508,"Cul":58,"Ãx":93,"De ":52,"Des":142,"Del":71,"Dem":144,"Den":147,"Dep":124,"Deb":61,"Év":65,"Ét":123,"Dan":127,"Dar":133,"Dav":113,"Dac":115,"Dal":45,"Cho":110,"Chr":98,"Che":497,"Chi":767,"Chu":642,"às":90,"án":10781,"Cin":435,"áp":259,"Cit":50,"Ciu":227,"Cir":43,"ál":141,"Cis":56,"áb":47,"ác":66,"Civ":71,"ár":205,"át":58,"ás":1227,"ão":95,"Cle":56,"Cla":205,"Cel":82,"Cen":268,"Cer":1047,"à ":342,"á ":406,"Cha":975,"Cri":147,"Cra":64,"Cre":79,"Cru":110,"Cro":121,"Clu":178,"Cob":41,"Cof":46,"Cop":81,"Cos":287,"Cor":738,"Com":767,"Col":268,"Con":936,"Cou":52,"ós":186,"òr":179,"òs":256,"ór":60,"óp":44,"ón":23204,"óm":55,"òc":49,"òm":188,"òn":43,"ó ":6509,"ña":383,"ñe":71,"ño":229,"ò ":54,"ïs":55,"ín":1828,"ío":885,"ít":375,"ís":1863,"íc":52,"íd":62,"ía":5276,"í ":325,"él":127,"ém":47,"én":2272,"és":12234,"ét":78,"ér":268,"éu":47,"év":65,"Edu":48,"éa":65,"éc":86,"ée":59,"Edw":43,"ég":58,"èi":57,"Eda":108,"èl":161,"èm":44,"èr":649,"ès":299,"Edi":90,"èt":179,"èd":50,"Ech":107,"ço":75,"ça":209,"é ":635,"Ebr":275,"Ed ":70,"ç ":366,"Dia":43,"Dic":108,"Dio":78,"Din":113,"Die":65,"Diz":56,"Div":60,"Due":176,"Duc":86,"ür":63,"Dur":53,"üe":263,"üi":139,"ùg":59,"ún":299,"úl":52,"út":328,"ús":134,"úd":56,"Dre":57,"ú ":245,"ER ":56,"Dos":138,"Dou":55,"Don":139,"Dom":136,"Dor":53,"Neg":61,"Neu":41,"Ner":42,"Nat":109,"Nav":689,"Nic":135,"New":87,"Nap":52,"Nar":43,"Nai":88,"Nac":229,"Nue":683,"No ":53,"Nov":50,"Nor":638,"Nom":360,"Nog":123,"Nob":162,"Odr":44,"Ohi":45,"Oci":122,"Oca":40,"Occ":315,"Ovi":40,"Oli":252,"Ons":56,"Ãxa":93,"Os ":984,"Ope":55,"Ort":62,"Ord":119,"Ori":680,"Org":56,"Oro":63,"Oso":45,"Ple":45,"Pla":459,"Pin":194,"Pil":51,"Peñ":79,"Pit":44,"Pis":83,"Pir":3266,"Pie":204,"Pic":78,"Phi":78,"Ped":168,"Paí":579,"Per":591,"Pet":150,"Pen":382,"Pel":40,"Pat":94,"Pas":141,"Par":783,"Pau":504,"Pab":49,"Pac":95,"Pan":179,"Pam":292,"Pap":92,"Pal":487,"Pue":323,"Pui":43,"Pro":378,"Pri":295,"Pre":652,"Pru":131,"Pra":244,"Pob":65,"Pol":283,"Pom":61,"Pon":189,"Poi":57,"Pot":41,"Pop":70,"Por":454,"Poz":58," ال":82,"Mún":76,"Rab":66,"Raf":117,"Rad":50,"Ram":97,"Ran":121,"Qua":92,"Qui":124,"Que":100,"Isa":135,"Irl":118,"Ita":545,"Isl":555,"Isr":43,"Ist":200,"Ira":43,"Irá":41,"Jac":136,"Jar":51,"Jan":70,"Jam":131,"Jer":62,"Jes":44,"Jea":137,"Jos":335,"Jor":54,"Jon":53,"Joh":274,"Jr ":59,"Joa":95,"Jud":52,"Jua":119,"Jus":45,"Jul":90,"LA ":58,"Fút":72,"Kan":52,"Kat":41,"Kar":100,"Ken":92,"Kin":58,"Kha":40,"Éta":96,"Kon":42,"Évr":60,"Let":41,"Les":137,"Ler":40,"Leo":104,"Len":373,"Lem":47,"Lei":300,"Leg":47,"Lec":40,"Lea":60,"Lau":180,"Laz":44,"Le ":98,"Lag":94,"Las":229,"Lat":44,"Lar":146,"Lam":54,"Lan":389,"Lac":69,"Lab":132,"Gúd":51,"La ":2467,"Llo":122,"Lla":76,"Lo ":323,"Lib":230,"Lig":108,"Lim":47,"Lin":136,"Lis":91,"Lit":320,"Ley":1614,"Lus":44,"Lui":166,"Lun":61,"Luc":146,"Lue":59,"Lug":56,"Loí":45,"Lou":130,"Los":419,"Loi":111,"Log":143,"Lor":170,"Lon":283,"Lom":95,"Lob":53,"Men":238,"Mel":100,"Mes":113,"Mer":200,"Met":54,"Med":234,"Mex":142,"Mey":1833,"Man":1211,"Mal":289,"Mam":40,"Mar":1797,"Mas":242,"Mag":110,"Mad":503,"Mai":84,"Mac":135,"May":164,"Max":51,"Mau":231,"Mat":250,"Mol":213,"Mon":1596,"Mos":99,"Mor":353,"Mot":51,"Mie":115,"Mig":96,"Mic":206,"Mir":340,"Mis":69,"Mil":244,"Min":117,"Mun":416,"Mur":389,"Mus":93,"Mue":57,"Mug":163,"Xil":74,"çoi":41,"Xal":52,"XX ":114,"Wur":43,"Wil":218,"Win":54,"ère":155,"èra":204,"èrr":55,"èrs":123,"Wei":53,"Wes":94,"XI ":57,"èth":158,"São":53,"èr ":80,"War":42,"Wal":86,"ès ":220,"èla":91,"éch":44,"Vol":63,"Río":103,"Viz":145,"Vis":59,"Vit":75,"Viv":66,"Viè":41,"évi":40,"éti":41,"Zar":1183,"Zam":440,"én ":2163,"éli":52,"ére":56,"éri":41,"éu ":44,"és ":12144,"之":56,"三":85,"Yor":546,"Ye ":991,"Yer":168,"Sur":84,"Sus":52,"Sup":40,"Sue":132,"Sud":292,"Sui":113,"Str":75,"Sto":48,"Sta":135,"Ste":129,"Ten":108,"Teo":41,"Tel":72,"ão ":95,"Tam":196,"Tan":48,"Tar":597,"Tai":54,"Taj":45,"Tal":63,"She":58,"Sha":58,"Sim":93,"Sil":128,"Sig":48,"Sis":99,"Sir":77,"Sin":151,"Sie":418,"Ser":298,"Seu":68,"Set":46,"Sev":235,"Sep":51,"Sen":840,"Sel":119,"Sem":182,"Sei":53,"Seg":496,"Sec":243,"Soc":173,"Sob":421,"Sot":54,"Sou":61,"Sov":80,"Sol":223,"Som":57,"Son":105,"Sor":351,"Rus":221,"Rum":113,"Rub":41,"Sag":50,"Saf":57,"Sai":418,"Sam":249,"Sal":930,"Sac":112,"Sab":95,"Se ":453,"Sco":41,"Sci":60,"Sch":126,"Sax":87,"Sav":109,"Sau":148,"Sar":312,"Sas":117,"San":2098,"Nùg":59,"Rey":105,"Res":57,"Rev":100,"Rio":251,"Riv":46,"Riu":60,"Ris":43,"Rin":49,"Rib":614,"Ric":155,"Rec":256,"Red":59,"Rei":1060,"Reg":57,"Rem":46,"Ren":132,"Rel":50,"Náp":249,"Rep":613,"Rea":232,"Roi":95,"Rob":204,"Roc":80,"Rod":121,"Roy":73,"Rou":41,"Ros":411,"Ron":93,"Rom":609,"Vel":47,"Ven":264,"Veg":102,"Vas":237,"Van":48,"Val":1946,"Var":71,"Vig":45,"Vid":63,"Vic":256,"Vie":268,"Vir":86,"Vil":1006,"Vin":160,"Ver":197,"Una":265,"Uni":1303,"Un ":378,"Ult":91,"Uru":44,"Urc":58,"Ucr":41,"VI ":101,"Ues":1284,"Tex":48,"Ter":931,"Tes":47,"The":314,"Tho":94,"Tie":516,"Tim":56,"Tir":42,"ça ":138,"Tor":554,"Tol":692,"Tom":76,"Tos":86,"Tou":103,"US ":58,"Tro":100,"Tri":127,"Tre":169,"Tra":328,"Tur":210,"Tud":41,"aC ":158,"вич":54,"às ":80,"biz":69,"bis":197,"bit":3960,"bio":344,"bir":182,"bil":199,"bin":181,"bo ":313,"blu":41,"blo":388,"ble":668,"bli":1408,"bn ":44,"bla":6430,"boc":153,"bol":805,"bió":291,"bpa":44,"bon":258,"bor":360,"bot":129,"bos":126,"bou":51,"áll":83,"be ":906,"ban":1137,"bal":979,"bai":592,"baj":40,"bag":336,"bac":233,"bad":107,"bab":70,"án ":10527,"bau":76,"bat":496,"bas":778,"bar":826,"bch":196,"ánd":139,"ánc":42,"ás ":1198,"ápo":250,"bi ":255,"bei":51,"bec":106,"ber":1735,"ben":218,"bel":1025,"bey":41,"bez":71,"bes":144,"bet":294,"bfa":45,"bia":312,"bib":60,"bic":70,"bid":52,"bie":575,"árb":69,"ca ":14794,"car":2739,"cas":4851,"cat":3216,"cau":248,"can":4948,"cap":1502,"caz":64,"cay":268,"cac":340,"cab":314,"cad":562,"cam":688,"cal":1804,"cag":56,"cai":191,"ce ":491,"bri":1569,"bro":804,"bra":1561,"bre":5163,"bru":86,"bso":41,"bse":48,"bst":107,"bte":55,"bur":411,"bul":93,"bun":55,"bui":89,"buc":81,"bue":92,"bus":122,"buy":48,"aka":66,"am ":354,"ake":58,"aki":53,"akh":42,"ajo":179,"afí":261,"aix":4540,"al ":11240,"aja":109,"aje":50,"ail":226,"aim":120,"ain":870,"air":581,"ais":411,"ait":77,"ak ":65,"aig":94,"aif":44,"aid":83,"aic":166,"aia":53,"adí":134,"aho":59,"acó":65,"adá":67,"aha":105,"agr":176,"agu":831,"agn":198,"ago":5671,"anu":472,"anz":412,"any":6533,"ano":1852,"ann":351,"ant":15027,"ans":1324,"anq":163,"ane":625,"ang":1039,"anh":151,"ani":3885,"anj":43,"ank":148,"anl":160,"ana":6098,"anc":10977,"and":4107,"amu":134,"amm":73,"amo":865,"amp":1732,"ami":2709,"amf":47,"ame":4595,"amb":576,"ama":1728,"ao ":68,"alz":65,"alv":228,"alu":654,"alt":1006,"als":2354,"alq":109,"alp":109,"alo":709,"alm":4203,"all":2888,"alg":57,"alh":70,"ali":4784,"alc":339,"ald":658,"ale":2737,"alf":222,"ala":3055,"alb":415,"an ":6113,"agó":1389,"ako":58,"aba":1272,"abe":873,"abi":3680,"abl":933,"abo":367,"abr":1214,"abs":89,"abu":101,"ae ":319,"aca":1187,"ad ":418,"ac ":612,"ab ":2350,"afo":146,"afr":204,"aff":59,"afe":128,"afi":1228,"afl":109,"ai ":461,"aga":473,"age":155,"ael":246,"aes":53,"aer":81,"ah ":76,"afa":137,"ado":2009,"adr":581,"adm":420,"adh":47,"adi":1004,"adc":41,"ade":908,"adu":331,"adv":85,"acq":78,"aco":595,"acl":40,"ack":101,"aci":10341,"ach":1441,"ace":581,"acc":247,"ada":2435,"act":3007,"acu":155,"acr":135,"acs":115,"azo":174,"azi":123,"azu":65,"azt":42,"aze":141,"aza":531,"azz":58,"axi":147,"axo":137,"atí":551,"asò":111,"az ":147,"ayo":1230,"ató":98,"aya":299,"aye":245,"azí":106,"ba ":2806,"ayú":116,"amó":65,"aqu":292,"amí":58,"at ":8853,"arg":457,"are":871,"ard":1884,"arc":6367,"arb":1022,"ara":7157,"arp":123,"aro":1482,"arn":763,"ané":98,"arm":309,"arl":816,"anç":88,"ark":146,"ari":3225,"aru":81,"arx":55,"aní":220,"arq":324,"arr":4340,"ars":729,"art":10745,"au ":1514,"asa":1087,"ary":98,"arz":796,"asi":1104,"ash":77,"asc":1409,"ase":427,"aso":509,"asq":49,"asp":165,"ask":71,"asm":49,"ar ":6113,"apa":550,"alá":1638,"ape":296,"api":1469,"alé":48,"apl":89,"apo":393,"alí":68,"apr":117,"apt":99,"apu":137,"aló":105,"as ":19306,"amá":261,"ava":1125,"ax ":55,"aux":83,"auv":126,"aut":2694,"arí":594,"avo":154,"avi":1192,"ave":551,"ará":49,"ay ":211,"awa":67,"aró":66,"ata":7591,"asu":67,"ast":7538,"ass":494,"atl":127,"atr":1247,"ato":13605,"ate":1325,"ati":1701,"ath":153,"auc":108,"aub":88,"att":124,"ats":80,"atu":842,"apó":59,"atz":500,"aul":310,"aum":84,"aun":85,"aur":378,"aus":598,"aud":445,"aug":298,"ος":66,"ος ":66,"Ï‚ ":149,"α ":52,"ич ":54,"jas":71,"jau":48,"jar":113,"jan":59,"je ":44,"jos":60,"jor":51,"fía":291,"jo ":179,"itr":81,"ües":75,"ito":14065,"itu":6260,"üen":106,"itt":116,"itz":160,"ity":56,"ipó":66,"eña":141,"iud":1817,"ism":873,"isl":671,"iso":251,"isp":308,"iss":346,"isu":85,"ist":6571,"ita":13238,"itc":43,"ite":2513,"ith":88,"iti":1102,"ivo":1136,"iró":43,"ius":271,"iur":45,"ium":78,"eño":50,"iva":1324,"ix ":335,"ivi":1220,"irá":76,"ive":525,"ipr":52,"ipo":232,"ipp":56,"ipu":138,"ipt":82,"ipi":7426,"ipl":106,"is ":1549,"ion":3766,"iop":54,"ior":777,"ios":2033,"iot":81,"iog":52,"iol":276,"iom":261,"ipa":1088,"ipc":44,"ilá":43,"ipe":312,"iov":42,"ir ":1069,"inó":91,"iru":67,"irv":40,"irr":87,"iro":527,"irm":204,"iné":146,"irl":117,"iri":575,"isi":1148,"ish":214,"ise":307,"isc":524,"isb":61,"isa":284,"iu ":332,"iqu":443,"ire":4699,"irg":44,"ira":1068,"irc":161,"it ":973,"ixó":72,"ja ":289,"iya":43,"üis":117,"ixo":921,"ixi":4792,"ixa":896,"ixe":850,"itá":2129,"iz ":822,"eón":60,"izo":170,"izi":124,"ize":43,"izc":159,"iza":1143,"kil":72,"kin":118,"kis":56,"útb":317,"km ":480,"ús ":123,"ki ":146,"kha":50,"kel":50,"ken":42,"ker":97,"ket":62,"ke ":87,"úne":45,"úni":50," ª ":48,"km²":4894,"ks ":44,"ko ":90," Ãn":69,"kar":110,"kan":40,"kal":59," º ":69,"fút":245,"ka ":210," Ga":2446," Ge":1114," Cá":129," I ":417," Fo":711," Fu":665,"cú ":41," Fr":2182," Fi":496," Fl":289," Bé":72," Ha":781," He":719," J ":52," Go":465," Gr":1065," Gu":1594," Gi":352," Gl":100," Cé":64," Ig":114," Id":48," Ib":231," Hu":365," Ho":591," Hi":444,"ha ":1929," Ji":54," Je":310," L ":757," Ja":568," Ir":296," Is":1032," It":588," Im":754," In":704," Ip":108," Il":295,"ham":219,"han":579,"hap":158," Ka":309,"hai":129," Fú":72,"hal":167,"hau":82," Ke":199," Ki":173,"har":1290,"has":147," Kh":65,"hat":40," Jo":916," Ju":421,"hae":98," Jr":59,"hab":5867,"had":57,"hac":508," Gú":51," La":4106," Le":3156," Li":1187," Kl":47," Ko":156," Kr":75," Ku":88," Mc":66," Ma":5257," O ":3000," Mi":1402," Me":3058,"he ":928,"dá ":99," Lo":1959," Ll":306," Ly":43," Lu":751," Ne":437,"а ":67," Na":1457," Ni":336," Mo":2562," Mu":1272,"hel":358,"hei":72,"heg":254,"hef":53,"hec":353,"hea":69,"heb":242,"hez":41," A ":6341,"het":220,"hes":362,"her":745,"heo":251,"hen":1459,"dán":167,"hem":117,"hi ":92," C ":63," Ap":96," Am":424," An":2142," Al":5280," Ai":211," Ag":239," Af":231," Ac":390," Ad":284," Ab":290," Ba":3640," D ":255," Az":169," Ay":81," Av":152," Au":1361," At":949," As":1091," Ar":3860," Aq":804,"hig":61," Be":1778,"hie":94,"hid":163,"hic":622," Bi":1877,"hib":145,"hia":201,"hip":217,"hio":409,"hin":1554,"him":149," Bl":233," Bo":1120,"hil":241," Br":1100," Bu":1316,"his":710,"hit":396,"hir":613," Ca":8053," Ce":1515," Ci":1040," Ch":3168," Cl":508," Cr":536," Co":3415," Cu":873," Cy":45," Da":678," Di":655," De":984," Dr":118," Do":649," Du":469,"hn ":206," Ea":51," Eb":283," Ec":152," Ed":408," El":689,"hle":43," Ei":68," Et":87," Es":4884," Er":277," Aí":54," En":982," Em":289,"dèr":52," Ez":47," Ex":340," Eu":620," Ev":67,"ho ":186," Fe":743," Fa":900,"go ":3253," Tè":61," Xi":132,"glo":720," Xa":108,"gle":161,"gli":83," Wu":48,"gla":523," Wo":125," Sé":45," Wi":413," Wh":40," Sã":53," We":250," Wa":265,"й ":42," Zu":77,"god":95,"gob":53," Zo":51," Ze":94,"gno":114," Zi":59,"gni":197," Za":1740,"gne":136,"gna":214," Yu":72," Yo":603,"cés":6099," Ya":59," Ye":1232,"cía":153,"gs ":50,"glé":443,"goz":1189,"gol":195,"gon":3334,"gos":1875,"gor":1373,"got":66,"gov":132,"gu ":63,"úda":51," a ":33903,"gro":300,"gru":390,"gra":2556,"gri":582,"gre":209," Oz":48," Ou":62," Ov":54," Os":1274,"gto":64," Ot":107," Or":1187," Op":71," Po":1463," Lé":44,"gui":929," Pl":536," Pi":4008," Ph":119,"gul":273,"có ":168,"gua":1007," Pe":1600,"gub":134," Pa":3520,"gue":1452," Nu":796," No":1504," Ol":435," On":167," Om":55," Oh":48," Od":84," Oc":530," Ob":95," Ra":686," Mú":79,"grí":111," Qu":327,"cón":94," Ro":1930," Ná":254," Re":2927," Ri":1399," S ":115,"gut":43," Pr":1734,"gur":246,"gus":162," Pu":601,"gun":881," Mé":58," Sy":56," Su":912," St":445," Ta":1241," V ":80," Th":489," Ti":744," Te":1398," Tr":784,"gué":212," To":1738," Pé":62," Ru":549," Sa":4961," Nù":59," Sh":196," Si":1206," Sc":255," Se":3315," So":1654," Sp":125,"ún ":198," Uz":41," Va":2415," X ":52," Ve":691," Vi":2397," Rí":105," Vo":162," Tu":387," Uc":50," Ue":1325," Ul":142," Un":1997," Ur":312," Us":67," Ut":63," ja":45," l ":7346,"iam":215,"ial":6627,"ian":2111,"ias":1288,"iar":803,"iau":82,"iat":539," im":548," in":2520," il":210,"ic ":192," ix":548,"iac":505," is":1346,"iad":189," it":1473," ir":178,"iag":113,"ibl":190," fú":245,"ibi":1047," ka":43,"ibo":144," m ":86,"ibn":40,"ibr":375," ki":82,"ibu":235,"id ":208,"iba":752,"ibe":709," ju":44," ha":6493," he":482," gl":81," gr":1637," go":384,"ia ":20555," gu":873," ib":130," id":377," ie":162," ig":40," hi":763," ho":443," dí":123," hu":1100,"iet":455,"ieu":89,"iev":42," ni":192,"iez":72,"iel":3483," ne":482,"iem":1602,"ien":5235," na":4148,"ier":2261,"ies":690,"ied":470,"ieg":1134,"iej":52," mu":11537,"ig ":130," mo":1373,"ibó":41," ol":51,"ifu":46," on":583," oc":5124,"ifo":507," of":3427," ob":670,"ife":480,"ifi":975," nu":622," no":4290,"ifa":124," le":1142,"icr":45,"ict":263," li":1110,"icu":187," n ":127,"ico":6536,"ick":140,"icl":297," la":7106,"ici":16857,"ich":1956,"icc":197,"ice":399,"ie ":3544," km":5375,"ica":8644," me":3304,"idu":108," mi":1891,"idr":120," o ":28468,"ido":565," ma":4705," lu":2504,"idi":602,"idg":67,"ide":1774,"idd":66,"ida":7766," lo":4749," ae":70," af":235," ag":650," ab":1016," ac":2837," ad":973," am":802," an":6430," ap":406," ai":177," al":2449," av":614," au":2876," ar":5285," aq":61," at":946," as":4436," d ":41259," ba":1763," az":45,"il ":1291,"ija":46," bi":684," be":950," bo":292," bl":117," bu":158," br":776," ca":13929," e ":354,"im ":145,"ika":72,"ige":58,"icá":71,"iga":1659,"igl":91,"igh":138,"igi":46,"igu":556,"igr":84,"igo":2086,"ign":388,"icó":83," aC":158,"ik ":59," er":161,"imo":645,"imn":76," eq":225," et":307," es":12368," en":45737," em":791," ep":103,"imp":1430," ei":79,"ime":1567," el":836," ef":58,"imi":832," fe":1998,"inc":9264,"ind":1169,"ina":3383," fa":1854,"imu":52," eu":203," ev":123," ex":831," fu":1130,"inn":131," fr":6946,"ino":2528,"inq":49," fo":1674,"int":2294,"ins":841,"inf":351,"ine":3588," fl":166,"inh":136,"ing":916," fi":1256,"ini":1221,"inl":57," ge":55," ga":614,"ioc":379,"iod":531,"inu":133," i ":370,"inv":192,"iny":434," cl":828,"iko":59," co":25285," cr":940,"iki":75," ce":943," ch":6428,"ike":47," ci":4380,"igü":76,"ila":709,"ilb":85," da":436,"in ":1797," cu":1036,"igò":47," do":1088,"ilo":709,"ill":3569," dr":358,"ilm":50,"ilh":186," de":93040,"ili":1413,"ild":72," di":6008,"ile":486,"ima":777,"imb":156,"ч ":61," ec":252," ed":270,"io ":12068," du":304,"ils":93,"ilv":161,"hs ":41,"día":1990," ví":46," zo":210,"hol":113,"hom":232,"hon":397," za":502,"hos":55," ze":43,"hov":67,"hor":197,"ка":41,"hod":105," ye":24958," ya":136,"hió":4220,"hna":61," tí":352,"ич":60,"dés":276,"hul":634,"hug":321,"hue":1138,"hud":1823,"hua":226,"dó ":56,"ов":67,"hre":40,"hri":107,"ht ":118,"dís":58,"dín":52," ru":519," u ":3972," sa":1314," se":5716," sc":327," si":7787," so":2523," qu":10550,"ви":64," t ":73," ra":569," re":9633," ri":229," ro":846," pu":1468," pr":13974," ps":89," lí":44," s ":862," px":42,"hy ":54," má":765," mé":265,"dón":67," os":5952,"hum":282,"hun":1148,"hus":252," op":121,"hur":185," or":1663," pe":2382," pa":7968," pl":740," po":12254," pi":915,"dòm":187," we":61," y ":28425," sí":53," va":1856," ve":970," vo":365," rí":719," vu":86," vi":1199," ub":66,"ùgo":59," tu":267," us":125," ur":107," un":30714," ue":407," ta":2947," su":13017," tr":2669," to":940," th":178," ti":1270," te":2399,"ffe":52,"fes":224,"fer":1148,"fec":203,"fed":134,"feb":1076,"fem":90,"bán":64,"fen":310,"fel":119,"fei":204,"fib":68,"fga":41,"fas":57,"fat":64,"far":136,"fam":559,"fan":320,"fal":168,"fai":59,"fae":49,"fad":52,"fac":309,"fab":307,"ff ":44,"fe ":106,"eyó":1631,"fa ":651,"eya":1305,"etó":68,"ext":325,"eyo":2607,"eye":329,"esú":73,"exa":137,"ez ":631,"exo":63,"exp":204,"exm":46,"exi":568,"exc":56,"exe":134,"ezu":123,"eza":368,"ezo":65,"ezi":98,"eta":1993,"ete":572,"etc":51,"eti":1075,"eth":102,"etn":131,"esp":2252,"esq":111,"esn":47,"eso":805,"est":15719,"esu":238,"ess":304,"aïs":55,"esv":63,"aña":108,"eud":108,"eui":40,"año":118,"eto":512,"etr":713,"ets":74,"ett":317,"etu":65,"etz":165,"ew ":79,"erá":64,"eve":388,"eva":1219,"eqü":84,"evo":337,"evi":643,"euv":62,"eut":86,"eur":262,"eus":3359,"ex ":75,"ewi":52,"esí":70,"eró":42,"erí":642,"ey ":440,"erú":45,"epe":349,"epc":40,"epi":88,"eph":122,"er ":4019,"epa":3520,"eor":324,"eol":192,"eop":46,"eon":126,"emá":600,"es ":7690,"ept":204,"epu":741,"elè":120,"epp":42,"epo":219,"epr":370,"erk":43,"enç":430,"erl":400,"eri":3958,"erg":428,"ere":1282,"erf":2781,"erc":992,"erd":1295,"era":7654,"erb":545,"et ":1474,"equ":358,"aís":1262,"aín":66,"esl":255,"esm":548,"esf":145,"esh":54,"esi":1456,"esb":43,"esc":3180,"esd":183,"ese":1276,"eu ":796,"esa":2233,"erz":160,"erv":658,"eru":753,"err":3654,"ení":57,"ert":3943,"ers":2076,"ern":1404,"erm":1179,"ené":47,"erp":493,"ero":3393,"egó":100,"en ":45647,"elb":64,"ela":1464,"eld":142,"elc":143,"ele":1251,"eli":1398,"elg":80,"elh":102,"elm":81,"ell":8850,"elo":1006,"elu":165,"elv":192,"els":263,"elt":264,"eo ":161,"emb":1646,"ema":2624,"eme":667,"emo":579,"emi":1471,"emp":1335,"enf":65,"ene":5939,"enh":83,"eng":1616,"enb":80,"ena":1793,"end":3462,"enc":4753,"eno":895,"enn":286,"eni":1492,"enu":197,"env":137,"ens":4894,"ent":19699,"enr":137,"enz":322,"eny":1221,"eog":94,"eod":44,"egl":633,"ego":1470,"ege":41,"egi":69,"egr":354,"egu":1397,"edé":50,"ek ":44,"eic":131,"eis":231,"eir":354,"eim":83,"eil":138,"ein":1974,"eid":281,"eig":91,"eja":198,"el ":3419,"eix":920,"eit":740,"ejo":101,"em ":132,"gin":62,"gio":83,"gia":75,"ght":102,"ghe":49,"gha":41,"ggi":46,"gi ":48,"gen":256,"cán":112,"ger":176,"ges":165,"gh ":89,"gel":473,"ge ":357,"gab":64,"gac":180,"gad":214,"gai":80,"gas":720,"gar":2153,"gau":48,"gat":585,"gam":80,"gal":634,"gan":993,"ga ":3535,"fur":43,"fuo":41,"fus":124,"fut":63,"bón":87,"fue":546,"fun":451,"fra":6871,"fre":244,"fri":349,"fro":68,"fru":71,"for":2092,"fos":84,"fot":124,"fon":296,"fol":141,"foz":70,"bén":225,"fle":62,"fla":55,"flu":281,"bèr":62,"flo":94,"fo ":248,"fic":8082,"fie":199,"fig":50,"fil":651,"fin":464,"fir":81,"fis":211,"fit":61,"fix":143,"da ":4544,"de ":80790,"dac":110,"dad":509,"dal":1070,"dai":51,"dag":63,"dae":153,"daf":41,"dat":9256,"das":725,"dar":409,"dap":72,"dan":479,"dam":162,"dau":66,"dda":65," Év":65," Ét":123,"dch":41,"cup":186,"cun":111,"cul":744,"cum":166,"cui":42,"cue":498,"cub":164,"cuc":89,"cua":224,"ctu":1573,"ctr":587,"cto":2549,"cti":1087,"cte":242,"cta":350,"cy ":62," Ãx":93,"cus":151,"cur":292,"cut":113,"cla":931,"chá":42,"cle":173,"chí":487,"clu":743,"chó":41,"cli":257,"clo":250,"ciá":588,"co ":7064,"ció":9852,"cni":55,"coi":64,"cod":67,"cof":52,"coa":55,"cob":54,"coc":226,"cq ":43,"con":9719,"col":848,"com":14762,"cor":808,"cos":2167,"cop":107,"cov":40,"cot":281,"cou":62,"cs ":170,"cqu":62,"cre":669,"cra":303,"cri":1318,"cru":146,"cro":284,"cci":5499,"cca":49,"cce":203,"cea":126,"ch ":604,"cer":583,"ces":1530,"cet":349,"cen":946,"cep":171,"cel":1113,"ceg":45,"ceb":79,"ced":326,"ci ":68," á ":52,"cha":2607,"chu":4144,"cia":18130,"ck ":381,"cie":3721,"cid":704,"cic":176,"cib":852,"che":3543,"chl":46,"chi":7691,"cho":415,"cht":85,"ciz":58," ár":62,"civ":67,"cil":415,"cim":107,"cif":135,"cir":339,"cis":357,"cit":4565,"ciu":1650,"cin":2607,"cio":3004,"cip":8439,"cke":58,"ed ":199,"eba":1588,"ebe":146,"ebi":129,"ebl":394,"ebo":83,"ebr":993,"ebu":74,"ec ":41,"eac":66,"eag":52,"eae":70,"ean":276,"eal":403,"ear":257,"eas":101,"eat":219,"eau":208,"eb ":59,"ea ":612,"efi":288,"efo":67,"efa":54,"efe":402,"ei ":675,"ega":1451,"efu":50,"een":85,"eer":148,"edi":1178,"edd":40,"ede":794,"eda":712,"edu":87,"edo":984,"edr":427,"ecl":113,"eck":59,"ech":4825,"eci":2528,"ece":250,"ecc":313,"eca":350,"ee ":110,"ef ":40,"ecu":265,"ect":1719,"ecr":84,"eco":638,"ecn":84,"dwi":49,"dwa":45,"dy ":91,"dve":42,"dvo":55,"drí":82,"dré":42,"dur":262,"duq":92,"dus":68,"duy":49,"dor":1504,"dop":45,"don":410,"dom":223,"dol":760,"dou":58,"dot":86,"dos":1016,"ds ":102,"deÅ£":45,"dmi":426,"dió":145,"dob":178,"doc":435,"doe":44,"duo":46,"dul":73,"due":204,"dua":107,"duc":808,"dri":604,"dra":531,"dt ":44,"dre":695,"du ":102,"dro":341,"dge":81,"dic":4356,"did":81,"dia":1190,"dib":46,"dhi":40,"der":1900,"des":5613,"det":240,"deu":41,"dev":209,"dey":218,"dez":184,"deb":265,"dea":184,"ded":106,"dec":377,"def":199,"dei":236,"del":1335,"den":7502,"dem":337,"dep":3598,"deo":89,"di ":271,"do ":4312,"div":679,"diz":245,"dim":719,"din":616,"dio":972,"dip":45,"dir":869,"dis":954,"dit":1042,"die":409,"dif":510,"dig":188,"dil":238,"rgu":256,"rdá":123,"rdè":52,"rdí":75,"rga":811,"ri ":510,"rgi":58,"rge":464,"rcí":64,"rgo":1114,"ret":1001,"res":4668,"rev":232,"reu":243,"rez":147,"rey":634,"rfe":48,"rfi":2743,"rfo":56,"rdu":65,"rds":65,"rg ":290,"reb":488,"rea":344,"ree":87,"ref":461,"rec":6366,"red":815,"rei":1976,"rej":70,"reg":565,"rem":1469,"nán":92,"ren":6329,"rel":727,"raí":55,"req":52,"rer":1624,"rep":629,"rda":881,"rcu":194,"rct":43,"rdo":870,"rdi":439,"rde":1540,"ós ":153,"re ":7388,"rbu":85,"rco":266,"rci":629,"rch":786,"rce":963,"rca":5430,"ray":93,"raz":455,"rd ":524,"rap":218,"raq":58,"rar":802,"ras":2221,"rat":2401,"rau":304,"rav":344,"rbi":229,"rbo":341,"rba":625,"rbe":641,"rai":190,"rah":88,"rag":6073,"ran":12090,"ram":788,"ral":2056,"rab":769,"raf":1535,"rae":83,"rad":1513,"rac":1579,"rpr":167,"rlí":92,"rpo":272,"rs ":2827,"rpe":55,"rpa":54,"rpi":201,"roq":99,"ror":130,"ros":1359,"rot":408,"rom":669,"ron":3295,"rop":1326,"roy":164,"roz":64,"rou":179,"rov":8451,"row":58,"rob":921,"roa":141,"rod":803,"roc":838,"roi":133,"rol":525,"rof":212,"roe":57,"rog":138,"rno":450,"rns":42,"rnu":49,"rna":910,"rne":366,"rni":615,"rmo":194,"nés":3547,"rmu":56,"rió":226,"ro ":6031,"rma":2089,"riá":116,"rme":392,"rmi":794,"rly":44,"rlo":224,"rli":134,"rle":220,"rla":839,"rn ":546,"rks":54,"nço":60,"rki":43,"rke":44,"nça":132,"né ":50,"riz":654,"rl ":72,"rip":80,"rio":3419,"rir":118,"riq":78,"rit":5633,"ris":1199,"riv":286,"riu":101,"rig":256,"ril":1019,"rik":79,"rin":1137,"rim":1306,"ria":3749,"rib":574,"ric":2717,"rid":579,"rie":2036,"rif":80,"rk ":612,"nç ":341,"rté":55,"rtè":42,"rtí":184,"rui":184,"rug":86,"rue":1064,"rud":70,"ruc":376,"rur":59,"rup":477,"run":160,"rum":242,"rul":41,"ruy":43,"ruz":231,"rus":648,"rut":44,"rva":257,"rvi":411,"rve":120,"rvo":53,"ry ":412,"rsi":295,"rso":324,"rsa":318,"rse":122,"rta":4462,"rst":64,"rto":2506,"rte":4811,"rth":164,"rti":4275,"rtz":99,"nó ":165,"rub":49,"rts":44,"rtr":69,"rtu":523,"nín":54,"rmá":65,"nía":237,"rt ":839,"rqu":659,"rné":74,"rro":1014,"rri":1235,"rná":86,"rre":2401,"rra":5371,"ru ":71,"rry":81,"rru":212,"sab":254,"sac":303,"sad":68,"sag":89,"sai":44,"sal":425,"sam":123,"óni":49,"sap":102,"san":614,"sau":128,"sat":386,"sas":492,"sar":1166,"sa ":3938,"ón ":23122,"rze":61,"rza":479,"rzo":759,"sha":111,"òrt":45,"sho":45,"òrr":49,"she":41,"shi":211,"si ":476,"siv":107,"sie":897,"sid":3345,"sic":1265,"sib":103,"sia":1021,"sk ":46,"sit":6032,"òst":117,"sir":64,"sis":499,"siq":107,"sin":887,"sio":389,"sil":374,"sim":339,"sif":156,"sig":443,"scr":1111,"scu":413,"sde":183,"sbo":69,"sbu":127,"se ":5048,"sca":2190,"sce":133,"sci":365,"sch":197,"sco":1662,"scl":134,"sex":85,"sey":101,"ser":1152,"ses":640,"set":627,"seu":313,"sh ":334,"òs ":104,"sfe":195,"sfo":54,"sea":92,"sei":105,"seg":655,"sef":47,"sed":71,"sec":228,"seq":43,"sep":285,"sen":1175,"sem":568,"sel":661,"spu":61,"spo":693,"spr":52,"spe":1130,"spl":78,"spi":211,"spa":3018,"sot":81,"sou":70,"sov":57,"sol":429,"son":2287,"sor":319,"sos":346,"sof":205,"soc":286,"sob":802,"su ":157,"sra":56,"st ":507,"oís":51,"squ":227,"ss ":107,"sli":62,"slo":145,"sla":1276,"sle":207,"ski":109,"sko":57,"ska":167,"ske":47,"sno":42,"sni":58,"smo":1140,"sió":969,"so ":1404,"sma":361,"smi":54,"sme":59,"swi":41,"stí":156,"sté":58,"stè":110,"stá":292,"stó":58,"sse":691,"ssa":416,"sso":402,"ssi":161,"ssu":66,"ste":7013,"stf":50,"sta":9411,"sto":2359,"sti":8938,"stu":608,"str":6362,"sua":62,"sud":745,"sue":210,"sub":387,"suc":192,"sui":69,"suf":112,"sul":509,"sum":63,"sup":2871,"sun":122,"sus":166,"sur":324,"suy":8064,"svi":51,"tai":444,"tal":7927,"taf":42,"tag":121,"tab":863,"tac":1213,"tad":424,"tc ":40,"tba":42,"tay":123,"tax":48,"tav":96,"tau":237,"tat":3988,"tas":2612,"tar":2116,"tap":45,"tan":9479,"tam":5022,"tch":72,"òm ":188,"te ":7101,"tbo":377,"ê°€":45,"ta ":14271,"ño ":46,"pa ":846,"pci":102,"pe ":499,"lá ":57,"par":10175,"pat":307,"pas":460,"là ":40,"pac":352,"pad":63,"pag":64,"pal":1063,"ñor":41,"pai":206,"ños":42,"pap":112,"pan":3073,"ñol":77,"phe":58,"phi":59,"ph ":101,"pañ":78,"peu":90,"pez":143,"pea":157,"pec":832,"ped":144,"lán":4159,"pen":661,"per":6227,"paí":658,"pet":128,"pes":448,"pei":72,"pel":266,"pla":716,"pli":171,"ple":898,"lèr":124,"lès":68,"plo":489,"plu":67,"pia":402,"pid":90,"pic":437,"pie":425,"pil":124,"pin":588,"pio":7267,"pir":221,"pis":101,"pit":1534,"por":5544,"pop":238,"pot":83,"pos":1131,"pon":437,"pol":1645,"pob":6027,"poe":320,"poc":176,"pod":154,"ps ":76,"ppe":91,"pió":148,"lés":609,"lén":62,"po ":1377,"lí ":42,"pta":96,"pse":75,"psi":41,"ló ":140,"pub":839,"pte":85,"pti":114,"pto":155,"ñas":48,"pra":148,"ña ":201,"líd":41,"lín":186,"pri":1181,"pre":3562,"pro":10201,"lón":649,"pur":65,"put":63,"pun":204,"pul":337,"pue":1281,"px ":43,"puz":122,"pué":579,"más":1055,"mán":1305,"més":297,"mía":98,"mín":65,"qua":2094,"mó ":46,"que":9584,"qui":2366,"món":89,"qué":50,"quí":216,"ra ":13591,"mún":92,"ncé":6029,"ngo":337,"ngi":53,"ngl":1038,"ngu":596,"ngr":238,"ngt":66,"ngs":81,"ni ":358,"nge":459,"ngh":51,"nga":1589,"ndí":93,"nho":77,"ndé":233,"ndó":55,"ndò":191,"nha":307,"neg":266,"nei":429,"nel":171,"nen":562,"nem":1044,"neo":91,"ner":1944,"net":586,"nes":1419,"nev":50,"neu":3333,"ng ":522,"nea":100,"neb":239,"nec":186,"ned":220,"nee":129,"nfi":62,"nfo":139,"nfl":201,"nfr":79,"ney":90,"nez":217,"nex":100,"nh ":93,"nfa":138,"nfe":159,"ncr":95,"ncs":41,"nco":784,"nci":13450,"ncl":579,"nce":1594,"nch":1989,"nca":2343,"ne ":2575,"nbu":46,"ndu":163,"ndr":650,"nds":75,"ndo":2567,"ndi":1523,"ïsa":48,"nde":3342,"nda":2934,"nal":2141,"nam":297,"nan":568,"nap":162,"nar":1025,"nac":947,"nad":422,"nae":55,"nag":81,"nai":3351,"nc ":42,"nab":73,"nbe":48,"nd ":533,"nav":217,"nau":204,"nat":1290,"nas":1528,"naz":58,"na ":29914,"ión":15324,"nya":4504,"nye":371,"nyi":95,"ntí":175,"ntó":2943,"nz ":101,"ny ":185,"nvi":113,"nvu":57,"nve":230,"nva":59,"num":304,"nun":319,"nui":47,"nus":137,"nut":82,"nur":49,"nua":50,"nub":51,"nue":339,"nuc":179,"ntz":90,"nto":7609,"ntu":217,"nts":4132,"ntr":3775,"nti":4508,"nth":93,"ntg":68,"ntl":40,"nta":5325,"ntb":47,"nte":2859,"nsu":361,"nsp":80,"nso":356,"nst":724,"nsf":49,"nse":1949,"nsi":3321,"nsk":43,"nsc":53,"nsa":388,"nu ":68,"nry":46,"nri":113,"nre":41,"nt ":9999,"nqu":365,"ns ":3297,"noc":64,"noa":44,"nob":131,"nol":187,"noi":1471,"nop":65,"nom":4566,"non":142,"not":89,"nos":736,"nor":1456,"nov":997,"nou":46,"noy":53,"nne":666,"nna":170,"nno":92,"nni":155,"nns":57,"nió":453,"nli":101,"nn ":180,"nla":104,"ngü":186,"nle":54,"no ":5180,"nke":42,"nki":41,"nja":54,"nig":161,"nif":282,"nie":336,"nid":2791,"nic":8881,"nia":2959,"nk ":109,"niz":323,"niu":41,"niv":269,"nis":1543,"nit":2772,"nir":59,"nio":1015,"nim":685,"nin":419,"nil":164,"ogr":1569,"ogu":112,"océ":67,"ogo":312,"oga":220,"oge":46,"oi ":141,"odí":1685,"ohn":220,"oha":82,"ohe":67,"oix":1425,"ois":211,"oir":170,"oit":63,"oin":160,"oid":74,"ofí":41,"oja":86,"ol ":1565,"oce":431,"och":1133,"oci":755,"ock":200,"ocl":96,"oco":258,"ocr":193,"obt":71,"obs":51,"oe ":44,"oca":1477,"occ":4327,"ode":444,"odi":479,"odo":671,"odr":74,"ocu":240,"oct":667,"of ":232,"oda":219,"oes":72,"oet":294,"oeu":48,"odu":636,"obé":220,"ofi":3286,"ofr":53,"ofo":167,"ofe":194,"ofa":81,"oa ":142,"ob ":41,"nzó":57,"oc ":329,"oan":85,"oal":56,"oac":115,"oba":1120,"od ":84,"oar":74,"obo":70,"obr":1405,"obl":6299,"obi":203,"obc":126,"obe":460,"nyo":3435,"Ø© ":63,"nza":531,"nze":53,"nzi":60,"nzo":99,"nyá":130,"oya":232,"oxi":91,"oxe":41,"oz ":271,"osé":216,"own":58,"ozo":77,"oza":1310,"oyo":79,"otz":224,"opó":65,"oue":43,"oub":45,"ow ":78,"oti":169,"oth":60,"ote":332,"ott":123,"ots":45,"otr":79,"oto":449,"ost":1452,"ota":773,"ov ":53,"osi":694,"ose":613,"osf":51,"osp":46,"oss":155,"osl":56,"oso":499,"osn":62,"oró":198,"oy ":78,"owa":53,"owe":59,"ovi":9219,"orí":319,"ovo":42,"ova":354,"ove":772,"oug":65,"oui":114,"oul":124,"oun":1514,"ous":206,"our":395,"out":72,"opo":466,"opi":489,"ope":454,"oph":55,"opa":516,"os ":25441,"opu":320,"oló":41,"opr":46,"olí":46,"opt":64,"ops":40,"ool":53,"ood":102,"or ":9087,"ork":566,"orl":115,"oné":3235,"orm":1749,"orn":778,"oro":1408,"orp":161,"orq":101,"oní":41,"orr":1698,"orc":390,"ord":1194,"ore":783,"orf":120,"org":706,"ori":3242,"ou ":222,"osa":1430,"osc":233,"ort":2810,"ors":909,"orv":50,"oru":324,"orz":356,"ory":64,"omé":50,"omí":105,"omá":496,"ot ":423,"m² ":4882,"orb":171,"ora":2008,"omú":90,"oqu":218,"íto":354,"ola":1622,"old":202,"olc":47,"on ":9278,"oli":2360,"oll":594,"olk":73,"olf":145,"ole":736,"ols":519,"olt":132,"olm":67,"olo":2074,"olp":41,"olu":385,"olv":43,"íst":73,"om ":111,"ona":5291,"ond":1478,"onc":1207,"onf":246,"one":1454,"onh":79,"ong":765,"oni":1854,"onn":399,"onm":40,"ono":4250,"onq":106,"ons":3093,"ont":2981,"onu":172,"onv":115,"ony":278,"onz":129,"oma":8598,"ome":737,"omb":2217,"omi":687,"omm":82,"omp":969,"omo":2541,"omu":6923,"op ":77,"la ":13225,"gún":43,"ína":74,"íns":58,"ín ":1619,"ío ":786,"le ":1482,"lce":50,"lca":313,"lch":255,"lcu":105,"lco":195,"lf ":51,"lde":545,"lda":117,"ldo":91,"íse":239,"ldi":63,"ldr":41,"lab":279,"lac":7068,"lad":759,"laf":116,"güe":154,"lah":42,"lag":299,"güi":134,"lai":286,"lal":104,"lan":4366,"lam":1159,"lap":58,"lar":1828,"íos":92,"lat":2045,"las":2771,"lau":337,"lav":397,"lay":70,"laz":237,"lba":848,"ld ":212,"lbe":146,"ís ":1479,"lbo":81,"ky ":62,"gòr":69,"ksh":40,"gó ":119,"íde":44,"gón":1401,"llé":182,"llè":44,"llá":2506,"lpe":216,"lph":49,"ls ":3268,"hía":487,"lló":632,"lol":93,"lon":1637,"lom":283,"lop":119,"lor":1072,"lod":42,"loc":1507,"log":342,"loi":46,"lpa":53,"los":2434,"lot":169,"lou":71,"lov":143,"lob":150,"lmo":183,"lme":4078,"liá":885,"lma":308,"lmu":74,"lió":99,"lti":157,"lto":1581,"ltr":243,"ltu":284,"luc":305,"lub":390,"lug":1219,"lue":1296,"lsi":170,"lso":93,"ías":665,"lta":1746,"lte":171,"lu ":50,"lse":50,"lsa":112,"ía ":4569,"lqu":129,"lt ":112,"lho":61,"lhe":92,"lha":188,"lgo":41,"li ":326,"lga":211,"lfr":59,"lfo":195,"lh ":50,"lfa":274,"lez":140,"ley":747,"lex":169,"leu":75,"lev":257,"les":1591,"let":670,"ler":899,"leo":90,"lem":1561,"hán":48,"len":1629,"lel":48,"lei":286,"lej":60,"leg":883,"led":346,"lec":686,"leb":181,"lea":191,"lls":428,"llu":110,"lly":119,"lo ":6451,"lla":7059,"lle":1785,"lli":819,"lhé":44,"llo":2287,"lm ":61,"ll ":601,"lit":2372,"lis":1111,"lir":71,"liq":95,"lip":156,"lio":1048,"lin":1102,"lim":460,"liz":384,"lix":43,"liv":118,"liu":92,"lic":2196,"lid":1116,"lia":2670,"lib":283,"lk ":57,"lil":59,"lig":423,"lie":227,"lif":574,"ma ":4262,"mb ":46,"mac":390,"mai":134,"mad":213,"mag":134,"mar":6853,"mas":1463,"mal":552,"mam":100,"man":3868,"maz":73,"may":1010,"max":40,"mau":44,"mat":2795,"mba":215,"ià ":88,"mbl":71,"mbi":310,"mbe":233,"mbr":3381,"mbo":389,"me ":312,"mbu":159,"med":417,"mea":52,"mec":61,"meu":62,"met":732,"mes":1084,"mer":2230,"mel":507,"ián":1702,"men":10149,"mez":72,"mex":49,"mey":594,"mfi":49,"luy":521,"luz":282,"lva":423,"lve":168,"lvi":79,"lul":85,"lui":166,"lun":337,"lum":164,"lut":90,"lus":184,"lur":85,"luv":44,"ly ":179,"lza":60,"mpi":508,"mpe":1346,"mpr":198,"mpo":1935,"mpl":1206,"mpu":673,"mps":47,"ms ":48,"mog":40,"moc":196,"mod":219,"mon":1273,"mol":200,"mov":182,"mor":989,"mos":648,"mot":101,"mou":58,"mpa":409,"mug":267,"mue":2339,"mud":113,"iña":49,"ió ":5096,"mur":109,"mus":767,"mui":583,"mul":270,"mun":14364,"mi ":66,"min":1797,"mio":915,"mil":953,"mir":128,"mis":230,"mit":490,"mic":569,"mia":389,"mig":100,"mif":80,"mie":1796,"mid":134,"mo ":3815,"ièr":107,"ièl":73,"mno":74,"mp ":61,"iés":139,"ién":1647,"mié":1515,"mma":94,"mme":71,"vía":77,"zqu":48,"zu ":46,"zue":177,"zul":76,"zz ":43,"zi ":55,"zeg":44,"zet":109,"zen":76,"zer":95,"ze ":198,"zca":158,"zcu":130,"zab":60,"zad":59,"zac":289,"zag":449,"zam":70,"zan":466,"zal":125,"zar":348,"zau":49,"zas":154,"zat":450,"zos":77,"zor":59,"zon":343,"zol":64,"ال":87,"zo ":978,"zia":62,"zie":76,"zin":46,"zio":145,"Ãng":58,"uña":55,"uín":41,"ys ":57,"uía":139,"yol":662,"yos":1713,"yor":588,"yon":591,"yod":1669,"uí ":43,"za ":2695,"ye ":23794,"yes":179,"yer":1805,"uán":43,"yen":347,"yev":162,"ya ":11135,"yad":248,"yag":43,"yac":58,"yat":477,"yar":166,"yas":560,"yan":160,"yal":259,"ué ":86,"yo ":3771,"ués":858,"yin":59,"yit":255,"tín":895,"tít":343,"tía":161,"xto":93,"xte":148,"xió":155,"tés":77,"xma":49,"xo ":835,"tèt":120,"xpr":86,"xpo":40,"xpl":51,"xor":48,"xos":144,"xon":140,"tí ":44,"Ù† ":61,"tó ":198,"tón":3024,"tán":2438,"xen":186,"xem":129,"xer":290,"xey":127,"xec":40,"xed":66,"xeb":52,"té ":56,"xis":225,"xit":4235,"xiu":87,"xil":66,"xim":208,"xid":80,"xic":268,"xie":229,"sús":77,"xa ":634,"xe ":402,"xas":176,"xat":41,"xar":156,"xan":151,"só ":81,"sòs":122,"són":50,"sé ":258,"wn ":47,"sèr":48,"sía":111,"woo":45,"wer":45,"sán":43,"wel":61,"wis":40,"wig":45,"wic":75,"win":65,"rú ":40,"wa ":59,"war":116,"rís":312,"río":740,"rín":122,"ría":1416,"vry":61,"vri":44,"vre":45,"ró ":128,"vue":107,"vul":60,"rón":372,"via":506,"vio":76,"vir":277,"vil":743,"vim":180,"vin":8416,"vic":421,"vid":633,"vie":1569,"viv":217,"vit":110,"vis":844,"ré ":45,"rès":47,"vo ":1023,"réc":41,"viè":54,"rén":65,"vié":152,"rés":144,"vió":41,"voc":147,"vog":50,"vol":331,"von":103,"vor":132,"vos":269,"voz":67,"vey":52,"vez":50,"ver":1295,"ves":245,"vet":87,"vei":63,"veg":260,"rán":277,"ven":911,"vel":555,"ved":70,"vec":112,"ve ":292,"rá ":52,"val":1231,"van":360,"vam":114,"var":1013,"vat":345,"vas":972,"qüe":96,"vac":122,"vad":166,"vai":52,"vag":42,"va ":2367,"uza":179,"uzc":121,"uyi":284,"uye":389,"uyo":1635,"uya":6536,"uxe":40,"uz ":427,"ux ":137,"uvi":103,"uva":115,"uve":116,"uy ":68,"urí":43,"uró":53,"usk":97,"ush":233,"usi":1046,"use":320,"usc":110,"usa":376,"usu":133,"ust":1133,"uss":242,"uso":426,"uth":76,"uti":206,"ute":121,"uta":240,"utb":55,"utz":40,"uts":45,"uto":2752,"utr":63,"us ":4557,"umá":123,"uqu":145,"ut ":319,"urb":94,"ura":2237,"urd":85,"urc":256,"ure":412,"urg":1130,"uri":877,"url":50,"urn":118,"uro":825,"urq":156,"urr":231,"urs":199,"urt":257,"uru":86,"urv":46,"uny":294,"uor":52,"uos":46,"upa":191,"ur ":505,"upi":40,"upe":2913,"upo":429,"ump":44,"umu":43,"uió":159,"umi":95,"umo":103,"uma":425,"umb":157,"ume":577,"unt":1291,"uns":190,"unk":72,"uni":12278,"uno":513,"unn":65,"unc":549,"und":1442,"una":19774,"ung":103,"une":139,"um ":208,"ulu":41,"ult":655,"uls":64,"ulo":359,"ulm":116,"ull":355,"uli":893,"ulg":112,"ule":147,"ulc":55,"ula":1293,"un ":14687,"uid":133,"uie":269,"uig":53,"uil":266,"uim":168,"uin":360,"uio":269,"uip":268,"uir":116,"uis":370,"uia":191,"uic":106,"uiv":83,"uiu":45,"uit":1942,"uiz":200,"uix":148,"ul ":264,"ugh":63,"uge":55,"ugo":115,"ucí":66,"ui ":377,"uga":2225,"ugu":646,"uco":62,"ucr":81,"uct":498,"ucu":40,"uda":1999,"ude":796,"udi":2062,"ubo":95,"ubp":52,"ubs":95,"ubr":706,"uca":364,"ue ":10127,"uce":207,"ucc":373,"uci":673,"uch":247,"ucl":167,"uck":53,"uet":490,"uev":956,"uer":4945,"ues":1507,"uey":113,"uez":111,"ufi":110,"udu":147,"udo":192,"ueg":394,"ued":332,"uec":249,"ueb":404,"uen":2417,"uem":50,"uel":1803,"uei":224,"ub ":368,"ua ":575,"uay":40,"uau":59,"uat":5774,"uas":205,"uar":331,"ual":1415,"uan":1770,"ubi":467,"ubl":925,"ube":202,"ubf":47,"uba":115,"ubc":72,"ud ":775,"uai":86,"uad":981,"uac":71,"uc ":85,"tzi":52,"tza":114,"tze":111,"ty ":139,"tró":52,"tuy":78,"tur":1993,"tus":72,"tut":233,"tui":52,"tul":64,"tun":41,"tum":52,"tub":619,"tua":6572,"tud":253,"tuc":165,"tug":422,"tz ":970,"pón":114,"ts ":4329,"tre":3631,"tt ":108,"tra":3520,"tri":4717,"tru":524,"tro":3498,"tu ":83,"tta":116,"tte":197,"tti":104,"tto":79,"tts":75,"pó ":91,"tiá":58,"to ":29963,"tió":3631,"tmo":48,"tié":42,"tni":61,"tna":54,"tod":133,"toc":171,"toi":42,"tog":1133,"tou":1509,"tov":69,"tos":6512,"tot":518,"toz":49,"tom":251,"ton":3336,"tol":876,"tor":4909,"top":212,"til":588,"tif":310,"tie":4123,"tig":1952,"tir":249,"teí":49,"tit":2499,"tis":1361,"tin":1092,"tim":251,"tip":273,"tio":464,"thu":114,"tia":471,"tib":196,"tic":4224,"tid":328,"tiz":57,"tiu":93,"tiv":2043,"tla":799,"tle":198,"tem":761,"ten":3384,"teo":124,"tep":75,"tei":151,"tel":3865,"teg":218,"tea":206,"teb":98,"tec":464,"ted":70,"tfa":44,"th ":346,"tex":101,"tev":102,"teu":43,"tet":109,"tes":1306,"ter":4859,"ti ":579,"thn":65,"tho":92,"the":306,"tha":124,"AS ":76,"zán":49,"yú ":114,"yón":1632,"zón":80,"zín":104,"yán":143,"xón":76,"yó ":84},"n_words":[3992558,4845689,3895070],"name":"an","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/ar.json b/contrib/languages-data/ar.json
new file mode 100644
index 0000000..d2cd2b1
--- /dev/null
+++ b/contrib/languages-data/ar.json
@@ -0,0 +1 @@
+{"freq":{"Ùˆ":674395,"Ù‰":83925,"ÙŠ":1050070,"Ù‹":13534,"ÙŽ":5694,"Ù":4812,"Ù€":6044,"Ù":291773,"Ù‚":234289,"Ùƒ":238048,"Ù„":1258387,"Ù…":769173,"Ù†":600182,"Ù‡":275861,"Ù‘":6098,"Ù":3286,"Ù’":2044,"Ø®":81779,"د":374335,"ج":159763,"Ø­":207504,"ت":462068,"Ø«":62775,"ب":456733,"Ø©":436596,"ئ":43113,"ا":1630465,"ؤ":8533,"Ø¥":87017,"Ø¢":11409,"Ø£":206598,"Ø¡":28935,"غ":62643,"ع":367154,"ظ":26879,"Ø·":114141,"ض":60872,"ص":101344,"Ø´":119185,"س":320648,"ز":83586,"ر":577132,"Ø°":48426,"ØŒ":89553,"Ø›":1398," ØŒ":20335," Ù†":38705," Ù‡":63492," Ù„":88748," Ù…":255388," Ù‚":40529," Ùƒ":61242," Ù":145932," ÙŠ":66494," Ùˆ":214375," ص":12486," Ø´":28169," Ø·":11510," ض":8567," ر":24040," Ø°":6579," س":49888," ز":6199," ع":110158," ظ":1667," غ":11703," Ø¥":56701," ا":619492," Ø¢":6461," Ø£":122164," ج":33981," Ø­":46703," Ø®":15646," د":26005," ب":145150," ت":96353," Ø«":8150,"ã‚¢":1366,"ا، ":7320,"ئة ":2129,"ؤسس":2077,"إن ":1277,"أيض":3023,"أور":2981,"أهم":1974,"أهل":1229,"أول":9618,"أنو":1568,"أنه":3955,"أمي":2439,"ألم":3407,"أما":1795,"ألÙ":1545,"أمر":6318,"أكب":2802,"أكت":1724,"أكث":2574,"Ø£Ùر":1800,"أعل":1292,"أعم":1551,"أغس":1326,"أصل":1634,"اث ":2044,"ئي ":3037,"ب، ":1611,"ات ":57401,"إضا":1407,"اح ":3081,"اج ":4358,"إسب":3960,"ئل ":2536,"اة ":4792,"إسل":3234,"اب ":9098,"إسر":1395,"إدا":2059,"اء ":22733,"إحد":6124,"إذا":1265,"ئر ":2350,"Ø©ØŒ ":15452,"اي ":2212,"بت ":1435,"بب ":1489,"اه ":2688,"بة ":12759,"ان ":40858,"با ":4030,"بد ":6780,"بح ":1410,"اً ":10489,"ا٠":3597,"إيط":2527,"إير":1277,"ام ":31875,"ال ":23428,"ئرة":2003,"اك ":2756,"اق ":4807,"اع ":5640,"إلي":1606,"إلى":18043,"إما":1946,"اض ":1731,"إقل":1614,"اط ":2017,"إنج":4004,"إنت":2754,"إنس":1518,"ار ":15484,"اد ":13875,"اص ":1586,"از ":3651,"اس ":4999,"بي ":11247,"ت، ":2665,"اضي":4506,"اصر":1618,"بو ":3109,"اصم":3245,"اعب":4589,"اعة":3428,"اعت":1731,"اعد":2039,"اعر":1432,"اطع":4959,"اطق":1642,"اطي":1824,"اعي":4248,"ادا":1408,"اخل":1728,"احي":1991,"اخت":2071,"احل":1721,"احت":2360,"احة":4653,"احد":2974,"ارب":3471,"ارة":7735,"ارا":5132,"ادي":11501,"ادة":5295,"بق ":1932,"ادر":1555,"ازي":2304,"است":9208,"اسة":1966,"اسا":1699,"اري":13333,"اره":1487,"ارو":1690,"ارك":3428,"ارع":1412,"ارس":4342,"بل ":7375,"ارد":1984,"ارت":2585,"ارج":1460,"اصة":1686,"اشم":2112,"به ":4466,"اسم":7414,"اشت":1547,"اسي":6201,"اسع":1320,"بن ":14722,"ئلة":1510,"بع ":5351,"ائد":1719,"ائر":5727,"ائز":1678,"ائم":1524,"ائل":4622,"ائي":8565,"ابا":4420,"ابت":1320,"ابة":2266,"ابر":1286,"ابع":5010,"ابي":3084,"ات،":1833,"ابل":1965,"ابق":3015,"ابن":2545,"اتح":2012,"اتب":1961,"ئيس":4458,"اتي":3444,"اته":3712,"ئية":4071,"اجت":1267,"ئيل":1637,"بر ":15051,"بط ":1447,"آخر":1604,"أن ":6708,"آن ":1439,"أس ":2019,"أبر":1929,"أبي":2141,"أبو":3598,"أحد":5818,"أحم":2690,"أخر":2780,"أحي":1348,"أرا":1757,"أرب":1317,"أرض":1852,"أرد":4510,"أست":1490,"أسا":2106,"أسس":2383,"أشه":1227,"أصب":1379,"أو ":17977,"أي ":2409,"جزي":2701,"جدي":2169,"جزء":1922,"حل ":2135,"جرا":1466,"جري":2220,"جزا":2275,"جتم":2280,"ثير":2870,"جات":1361,"ثنا":1548,"جار":2741,"جال":2447,"جان":2465,"جبا":1237,"جام":4767,"جبل":1832,"جلي":4317,"جما":2589,"جلس":2643,"حاÙ":7429,"حال":5921,"حاك":1299,"جهة":1426,"جنو":6846,"جها":2454,"جمي":2428,"حاد":3278,"حار":1507,"جنس":1263,"حاس":1289,"جمع":2692,"جمه":2396,"جمو":3506,"خط ":1642,"خر ":2126,"حي ":2379,"د، ":2106,"دث ":1521,"حصل":1369,"دة ":22913,"دت ":1894,"حسب":1473,"دا ":3896,"حسي":1367,"حسن":1628,"حرك":2813,"حرا":2117,"حرب":2616,"خل ":1874,"حزب":1786,"حري":2754,"دأ ":1419,"حدث":1823,"حدة":6151,"حدا":1890,"حدي":3344,"حدو":1601,"حدى":6092,"حتى":2850,"حتو":1235,"جية":1498,"جيا":1555,"جيد":1284,"جين":1406,"جود":2059,"ختص":1274,"حول":2075,"حوض":3972,"حوا":3753,"خاص":2779,"خار":1757,"حمل":1584,"خاب":1378,"حلي":1951,"حمد":7071,"حما":1348,"حكو":1587,"حكم":2998,"حقي":1342,"دس ":1564,"در ":3643,"دد ":5353,"ذا ":6858,"دن ":3532,"ده ":2855,"دو ":1586,"دى ":7927,"خصي":1891,"ر، ":3918,"دي ":16032,"حيا":3483,"ختل":2576,"حيث":5743,"حية":2840,"حيو":1435,"خدا":1713,"خدم":4681,"خرا":1406,"دل ":1527,"خرج":2035,"خرى":2288,"دم ":12380,"خلي":2944,"خلا":4049,"دان":2670,"دام":2603,"دال":1357,"داي":1553,"داء":1232,"دائ":2836,"دار":5259,"دات":2179,"داخ":1695,"داد":3415,"ذي ":7019,"رج ":3503,"رة ":42156,"رت ":3799,"رد ":2816,"را ":5024,"رب ":12795,"ذه ":5431,"درس":2590,"درج":3597,"درا":3796,"دري":2750,"دول":7090,"دون":2547,"دوا":2452,"دود":1701,"دور":4858,"دها":2546,"دني":4787,"ذات":1939,"ر٠":5073,"زء ":1646,"ديو":1641,"ديم":4297,"دين":17630,"ديا":3615,"دية":10224,"ديس":2111,"ديد":5637,"دير":2378,"ديث":1831,"رس ":4802,"رز ":1453,"رع ":2500,"دما":2598,"رض ":4034,"اقي":2703,"اقت":1926,"اقة":1778,"اÙÙŠ":2956,"اقع":1373,"الق":27005,"الÙ":20852,"الل":14610,"الك":19511,"الي":32249,"ان،":1862,"امت":1485,"امج":2075,"اما":2969,"الن":19094,"الم":106832,"امة":2785,"الو":15207,"اله":10510,"امر":2004,"امع":4584,"الأ":53765,"الآ":3087,"الج":24268,"اكي":1240,"الث":10222,"الت":44136,"الة":4500,"الب":27575,"الا":23279,"اكم":1385,"الإ":23108,"الس":25474,"الش":23985,"الر":18584,"الز":4628,"الد":22592,"الذ":9992,"الح":28530,"الخ":10845,"الع":45326,"الغ":7597,"الط":9353,"الص":10951,"الض":2239,"اÙØ©":3763,"اÙظ":6821,"اين":1399,"ايو":2112,"بتم":1591,"ايا":4981,"اية":7953,"اير":3864,"بدأ":2040,"ØŒ ":87650,"بحي":1333,"بحر":3662,"اهر":2224,"اني":27995,"انه":3927,"انو":4021,"بان":8783,"بال":24841,"باد":1374,"اند":2371,"بار":7325,"انس":1452,"باس":3805,"انا":3741,"باب":2616,"انب":1315,"انة":1432,"بات":4730,"انت":10216,"امي":6789,"باح":1289,"امل":2591,"بائ":1535,"اوي":2785,"اول":1939,"اهي":1800,"تا ":1767,"تب ":3208,"بشك":1596,"بدا":2243,"برا":7053,"برت":1316,"برل":1347,"برو":1928,"برن":1554,"بري":6364,"تر ":3568,"بطو":2790,"بعد":8590,"بعة":3652,"بغد":1205,"بعض":3329,"بني":2271,"بها":4540,"بنا":4249,"ئ ":1205,"ا ":148281,"بلي":1501,"بلا":3530,"ب ":70841,"بلغ":5588,"بلد":6651,"Ø¡ ":25878,"بقا":1527,"Ø£ ":2761,"Ø­ ":16042,"Ø® ":7746,"د ":111902,"Ø° ":7044,"بين":10021,"Ø© ":417779,"بيل":2585,"بيض":1538,"بيع":2501,"ت ":98281,"بيا":2983,"بية":13845,"بير":5823,"بيت":1659,"بون":1363,"Ø« ":14734,"بول":3156,"بور":2404,"ج ":16315,"بوا":1540,"تي ":19035,"ثة ":2508,"تو ":1236,"تى ":3327,"ته ":7416,"تل ":1533,"تم ":5813,"ثر ":3741,"Ø› ":1372,"تأس":2342,"تبر":4383,"تبع":2953,"تال":2664,"تان":2849,"تبا":1684,"تبة":1215,"تار":5301,"تاب":6518,"تاج":2425,"تبل":2737,"تجا":3016,"تخد":5000,"تخب":1329,"تخا":1370,"تحد":6290,"تحر":1572,"تحا":3110,"تحت":3198,"تدا":1332,"ثم ":2995,"تري":1455,"ترو":2478,"ثل ":4130,"ترا":5683,"ترة":2099,"ترك":2885,"تشا":2599,"تسم":2030,"تست":1924,"تصا":3295,"تشر":1532,"جة ":6594,"تشي":1720,"تضم":1238,"تطو":1569,"تعا":1531,"تعت":2311,"تعم":2414,"تعل":2476,"تعر":2059,"تعد":2614,"جد ":3856,"جر ":1340,"تÙا":2566,"تقا":2028,"تقد":2088,"تقر":1287,"تقس":2440,"تقع":8338,"تقل":2844,"تهر":1315,"ثال":2031,"تها":6374,"ثان":4499,"تنظ":2616,"تمي":2924,"تنا":1728,"تمد":1486,"تمر":1589,"تلÙ":3439,"تلك":1400,"تما":2848,"تمب":1576,"تكو":3404,"تلا":1846,"تين":4590,"تية":1561,"تيا":2052,"تون":4528,"توي":1644,"توÙ":2858,"تول":1447,"تور":2920,"توس":1688,"توب":2148,"توا":1906,"تهم":1291,"جه ":1478,"جم ":2117,"جل ":2059,"جي ":1843,"حة ":7424,"حت ":2989,"حر ":2424,"حد ":6903,"ثلا":2751,"ثما":1889,"ثقا":1374,"وي":38534,"ÙŠØŒ":8021,"وو":3027,"وى":2435,"ية":149201,"يب":15166,"يا":76978,"يئ":2097,"يض":5937,"يط":9332,"يع":16206,"يز":13642,"يس":24541,"يش":6638,"يص":2709,"يخ":8388,"يد":33231,"ير":50995,"يت":21919,"يث":8777,"يج":6934,"يح":6752,"يه":17510,"ين":82989,"ًا":1622,"يو":34779,"يك":18257,"يق":23953,"يم":33457,"يل":36829,"ÙŠÙ":12030,"يغ":2332,"يي":7762,"Ùع":3459,"Ùض":3349,"Ùص":2010,"Ùظ":7666,"Ùز":1405,"Ùر":20396,"Ùس":4079,"ÙØ©":10560,"Ùت":8107,"Ù‚ØŒ":1414,"Ùا":19253,"Ùب":1563,"Ù‚Ù":1276,"قع":16814,"قط":5495,"قض":1727,"قص":4820,"قش":1443,"قس":4834,"قر":18930,"قد":24585,"ÙÙ‰":1203,"قت":5868,"ÙÙŠ":137714,"ÙÙ‡":4197,"قب":10150,"قة":17223,"ÙÙˆ":7133,"ÙÙ…":1876,"ÙÙ†":5345,"قا":33043,"ÙÙƒ":2212,"ÙÙ„":7982,"ÙÙ‚":4801,"لأ":58367,"لآ":3323,"ÙƒÙ":1496,"كس":5921,"قي":21786,"كث":5852,"Ù„ØŒ":2207,"كذ":1298,"كر":19872,"كز":7004,"كأ":1420,"قل":10832,"قم":3276,"كا":41128,"قن":3553,"كب":9528,"قه":2295,"كة":14080,"قو":10571,"كت":14016,"قى":1403,"لك":43877,"مؤ":5145,"لق":35996,"Ù„Ù":31832,"لط":12463,"لض":2442,"لص":14130,"لش":25578,"لغ":17940,"لع":56838,"لظ":1228,"لخ":11729,"لد":39289,"لج":27745,"لح":34548,"لز":5989,"لس":38336,"لذ":10795,"لر":20100,"كم":13188,"لا":89050,"كن":10747,"كل":17531,"لإ":25479,"لت":54464,"Ù…ØŒ":3972,"كي":22561,"لث":10985,"لب":37291,"كه":2654,"كو":22542,"لة":26342,"مع":31975,"مغ":4324,"مص":12239,"مض":1909,"مط":2885,"Ù…Ù":2713,"مق":12733,"مك":8220,"مل":28998,"مت":25250,"لى":49564,"مة":21124,"لو":37077,"مج":15409,"ن،":5643,"لي":91040,"مث":7965,"لم":137485,"لل":37112,"مب":12595,"له":28033,"ما":84509,"لن":25730,"مز":2658,"مر":34445,"مش":6760,"مس":26092,"مخ":5223,"مح":22151,"مذ":1262,"مد":31562,"نغ":3178,"نظ":9512,"نع":2681,"نط":11265,"نل":1207,"نم":4711,"نق":5444,"نك":3963,"Ù†Ù":6969,"نح":3012,"نج":9690,"مي":49166,"ه،":2071,"مى":3142,"نت":26741,"نة":26561,"مو":33352,"مه":13330,"نب":6671,"نا":51288,"من":110993,"مم":11520,"نص":4888,"نش":6378,"نس":20514,"نز":3094,"نر":1454,"نذ":4480,"ند":18475,"هـ":2446,"وأ":7843,"وإ":2408,"هل":3624,"هم":13508,"هن":5865,"وا":96512,"هب":2043,"نه":22166,"ها":63427,"هت":1254,"نى":2312,"نو":28243,"هة":2492,"هج":2434,"و،":1267,"ني":68053,"هذ":11164,"هد":6895,"هز":1306,"هر":13839,"وغ":3162,"وظ":1284,"وع":15945,"وق":15305,"وك":13977,"ÙˆÙ":17742,"ون":47917,"وه":18574,"ول":64949,"وم":36461,"هي":33444,"وث":2062,"وت":24045,"هو":37979,"وة":2112,"وب":25979,"ود":18341,"وخ":2267,"وح":8074,"وج":13010,"وس":25063,"وز":7609,"ور":46060,"وذ":2568,"وط":5229,"وض":6816,"وص":4846,"وش":3688,"دة":23918,"خو":3104,"دت":2861,"دا":34518,"دب":3013,"دخ":1822,"دث":2213,"خي":4889,"خل":11579,"خم":2109,"دأ":2148,"خط":4248,"خر":10258,"خد":6707,"خص":5211,"ده":6608,"دو":24057,"دى":7961,"دي":71508,"ر،":3948,"دÙ":3789,"دق":1886,"دك":1306,"دل":4421,"دم":20032,"ذا":11430,"دن":9851,"دع":2356,"دد":6837,"در":20942,"دس":4645,"حث":1327,"جي":13871,"حج":2300,"حة":7643,"جو":9984,"حت":11398,"جن":11985,"حا":28730,"حب":2668,"جه":7398,"جل":12168,"جم":17274,"جع":2397,"جس":2540,"جز":8412,"جر":7944,"جد":8657,"خت":6160,"حي":21851,"د،":2117,"حم":14977,"حن":1639,"خا":10036,"خب":2705,"حو":13676,"Ø­Ù":3106,"حق":4632,"حك":5593,"حل":8199,"حض":1296,"حص":3737,"حز":2507,"حر":15992,"حس":5733,"حد":30065,"تغ":2721,"تÙ":6519,"تم":21550,"تل":11923,"تك":7075,"تق":23507,"تى":3340,"ثة":2642,"تو":27372,"ته":18459,"ثا":9771,"تن":10701,"تج":7596,"تح":21120,"تت":7159,"تر":25584,"تخ":11500,"تد":5897,"تش":11506,"تص":9178,"تز":2287,"تس":9386,"تع":17161,"تض":2140,"تط":3895,"ثق":2026,"ثل":9196,"ثن":2113,"جا":22121,"ثم":5275,"جة":6738,"ثو":2447,"جب":5048,"ثي":5910,"جت":3434,"تي":34720,"ثر":5818,"بغ":2025,"بع":25200,"به":11285,"بن":26227,"بم":5940,"بل":28735,"بك":4252,"بق":7310,"بد":14809,"بج":2196,"اً":11293,"بح":10218,"بت":8320,"اي":29388,"Ø©ØŒ":15567,"بب":2982,"اه":11841,"او":11068,"بة":13305,"بط":8037,"بش":3786,"بص":1566,"بس":4381,"بر":41886,"تا":29084,"تب":19287,"تأ":5378,"بو":18997,"ت،":2681,"بي":60818,"ئد":1726,"ئر":5978,"ئز":1688,"إن":13110,"ئا":1293,"إم":3605,"ئة":2198,"ا،":7371,"إي":6297,"اء":25293,"ئل":4704,"ائ":31094,"ئم":1653,"از":9640,"ار":69575,"اذ":2153,"اد":41536,"اض":8516,"اص":11589,"اش":9130,"اس":39158,"ات":77501,"اة":4919,"ئه":1298,"اب":38588,"اخ":6254,"اح":20482,"اج":12583,"اث":6287,"ئي":14445,"ب،":1623,"اÙ":23866,"بأ":2984,"اق":16552,"ام":64051,"با":69767,"ان":106270,"اك":11934,"ال":726452,"بإ":1475,"اع":27489,"اغ":2741,"اط":14678,"ؤس":2217,"أو":36459,"أه":4128,"أي":8325,"أل":9391,"أك":9086,"أن":19914,"أم":14875,"إس":11365,"إر":1668,"إذ":1752,"إد":2895,"إخ":1619,"إح":7179,"إب":1901,"إل":22468,"إق":2002,"إع":1925,"إض":1479,"آخ":1667,"آل":2561,"أت":2542,"آن":1925,"أب":10779,"أح":11512,"أخ":6156,"أث":2807,"أج":3225,"أر":12567,"أد":4137,"أص":5345,"أش":3739,"أس":15627,"أغ":3162,"أع":5434,"أط":1989,"أق":3836,"Ø£Ù":5069,"غي":8633,"غو":4024,"غن":2949,"غل":3357,"عي":17935,"غد":1582,"غر":13200,"غس":1789,"غز":1267,"عق":2845,"عل":49520,"عم":19388,"غا":8614,"عن":19968,"عه":6016,"غة":3737,"عو":7644,"ظي":3474,"عث":1594,"عت":10284,"عة":23777,"عد":27555,"عش":4458,"عس":1709,"عز":2669,"عر":25424,"عظ":2315,"عط":1205,"عض":5918,"عص":2498,"عا":50253,"ظه":2422,"عب":25388,"ظم":4301,"ظا":4120,"طن":4817,"طل":7662,"طق":11023,"Ø·Ù":1950,"طع":6327,"ظر":2587,"طي":9898,"طو":11014,"ظة":6299,"ضم":8353,"ضل":2264,"ضع":1853,"طس":1527,"طر":9793,"ضي":7171,"طح":1428,"طا":16584,"طب":7274,"طة":4986,"ضو":3284,"صل":10145,"صÙ":5067,"صط":2224,"صص":1416,"صغ":2398,"ضر":2826,"ضة":1642,"صو":8178,"صم":4802,"صن":4515,"ضا":11319,"صي":8440,"Ø´Ù":1822,"شك":5750,"شق":1763,"شع":5311,"شغ":1513,"صح":4260,"صد":4269,"صر":13415,"شم":11602,"صا":13513,"صب":5260,"شه":5994,"صة":3495,"شو":3361,"شي":12032,"سع":7475,"سط":12291,"سس":4815,"سÙ":3591,"شأ":1378,"سي":47561,"شت":5440,"سو":18175,"سه":3635,"شب":4456,"شا":16192,"سن":12898,"سم":24549,"سل":21272,"سك":11128,"شر":23735,"شد":1209,"شخ":3092,"زع":1204,"سب":17052,"زه":1918,"زن":1607,"سا":35091,"ست":30958,"سة":7853,"زو":4517,"زم":3202,"زل":2051,"سد":1627,"سر":8485,"سج":2382,"زي":20894,"رس":13875,"رش":2594,"رز":2717,"رط":1498,"رض":6362,"رع":5610,"رغ":2804,"زء":1924,"رل":3156,"رك":24686,"رق":13754,"رÙ":9661,"رو":32705,"زة":4009,"زب":2789,"ره":9008,"زا":10186,"رن":13036,"رم":7236,"س،":1548,"ري":92759,"رى":6117,"زر":4095,"ذر":1628,"ذك":2116,"رأ":2244,"رئ":5538,"ذل":5170,"رب":36257,"ذه":6871,"را":67798,"رت":12252,"رة":43634,"ذو":1454,"رج":14223,"ذي":9511,"رخ":1356,"رح":4450,"رد":11552,"Ù ":28941,"Ù€ ":3661,"ع ":53161,"غ ":7968,"ص ":6276,"ض ":14917,"Ø· ":15243,"ظ ":1554,"ر ":119691,"ز ":20518,"س ":42343,"Ø´ ":6531,"Ù‹ ":11009,"ÙŠ ":305236,"Ù‡ ":68493,"Ù† ":236663,"Ù‰ ":82765,"Ùˆ ":86344,"Ùƒ ":22299,"Ù‚ ":30798,"Ù… ":131428,"Ù„ ":111126," ØŒ ":18980," Ùˆ ":12423," Ù… ":5345," جن":4512," حا":7431," جه":1218," جم":4339," جي":2378," جو":3451," حت":2592," جد":1981," جز":3197," جر":1610," ثل":1523," ثم":3265," جا":6114," جب":2225," تي":1533," خل":4736," دا":4684," خط":2109," حق":1280," حك":2014," حل":1546," حي":8173," حم":2037," خا":3436," حو":8126," حر":3258," حس":2733," حد":2242," بك":1952," بق":1818," به":5365," بن":17676," بم":5621," بل":8899," بغ":1392," بع":9614," بس":2145," بر":8567," بط":3259," بش":2647," بت":3125," بد":4624," بج":1734," بح":3333," بأ":2828," ال":581886," بإ":1446," با":31729," ان":6245," ام":2207," اع":1488," ار":1569," اس":7954," اب":3029," ات":1745," اح":1418," اخ":1766," تو":6684," ثا":1511," تن":5058," تم":5949," تل":2911," تك":2690," تق":13534," تع":7393," تط":1471," تش":4009," تص":2525," تس":4420," تر":4248," تخ":1540," تد":1553," تج":1794," تح":6360," تت":4407," تا":4988," تب":4366," تأ":3604," بو":6771," بي":13768," أل":4956," أك":6683," أن":14293," أم":6327," أو":27007," أه":2795," أي":6316," آل":1841," أب":8228," أخ":2816," أح":8158," أج":2000," أث":1423," أر":3743," أد":2035," أص":3105," أس":5597," أش":2245," أع":3015," أغ":2285," Ø£Ù":3376," أق":2564," إي":4306," إن":5007," إم":1413," إب":1312," إس":5794," إر":1268," إذ":1294," إد":1530," إح":6465," إل":21224," إق":1208," طو":2165," عش":2714," عر":3955," عد":6773," عا":21951," عب":8841," عي":1567," غر":3853," عل":33952," عم":7142," عن":15313," غا":2188," غي":2774," سع":2415," سي":7136," شب":1886," سو":5683," سم":1931," سن":7850," شا":3992," سك":3794," سل":4129," شر":6157," شخ":1437," شع":1651," شك":1235," شي":2028," شم":4082," صا":1583," شه":2018," صح":1413," ضم":5470," طا":1844," طب":1958," طر":3408," در":2995," دي":6094," دو":6652," ذا":1885," رئ":2370," ذل":2035," را":2673," رس":1784," ري":3022," رق":1403," رو":5273," زي":1372," سب":3177," سا":6895," ست":1885," لك":4551," مؤ":2595," لق":1950," لع":2571," لغ":1253," لص":2532," لج":1545," لح":1350," لد":1501," لب":2260," كو":5523," لت":3577," كي":3645," كل":6741," لإ":1374," كم":7185," كن":1321," لا":10830," مل":4355," مك":2868," مق":7894," مغ":1598," مع":14545," مص":6997," مس":11150," مش":2698," مر":11437," مد":14412," مح":14231," مخ":2723," لي":4897," مث":3231," مج":7634," لو":4068," مت":7729," لن":2133," ما":17956," مب":2224," له":5386," لل":21688," لم":6816," Ù†Ù":1935," نق":1569," نظ":2491," نس":4267," نش":1472," مم":3100," نا":8059," من":85288," مه":1676," مو":12716," مي":4669," نج":1432," وأ":7664," هـ":2402," هن":1965," وا":50314," وإ":2381," هذ":9093," نو":6152," ها":3011," نه":2387," ني":2637," Ùر":6896," Ùب":1504," Ùا":3547," Ùت":2193," قص":1901," قط":1391," ÙÙ‚":1797," ÙÙ„":2626," ÙÙ†":1697," قا":5741," ÙÙˆ":2197," ÙÙ‡":1391," قب":5253," ÙÙŠ":115948," قد":7793," قر":6492," لأ":2928," قل":1628," كت":3116," قو":2125," كب":2590," كا":17231," قي":2400," كث":1288," كر":7439," ود":1531," وخ":1367," وح":3870," وج":3378," هي":20610," وت":16290," هو":22238," وب":6445," وص":1974," وش":2083," وس":5388," وز":1947," ور":2782," وذ":1466," وغ":1624," وع":5254," ون":2492," وه":15877," ول":15982," وم":13412," وق":7666," وك":6757," ÙˆÙ":4804," وو":1985," وي":13320," يا":1484," يب":2139," يح":3014," يج":1805," يت":5470," ير":1696," يد":1787," يص":1234," يش":1764," يس":4063," يع":6186," يل":3428," يم":2984," يق":8446," يك":1977," يو":8148," ين":4302,"Ùس ":1508,"Ùر ":2554,"Ùع ":1454,"Ùا ":1471,"Ù‚ØŒ ":1401,"ÙØ© ":10112,"قع ":14708,"Ùار":1789,"Ùات":2286,"Ùال":1824,"Ùاع":2545,"Ùان":1282,"Ùبر":1355,"Ùتر":2588,"قب ":1566,"قا ":1491,"قت ":1589,"قة ":16768,"ÙÙŠ ":111805,"قد ":7827,"عظم":1719,"عضو":1448,"عشر":3482,"عسك":1369,"عزي":1296,"عري":1226,"عرو":2224,"عرÙ":3788,"عرض":1945,"عرب":8060,"عرا":4113,"عدة":2689,"عدا":2204,"عدد":5430,"عدي":2807,"عتم":1554,"ظيم":2842,"ظهر":1451,"عبر":1705,"عبد":6249,"عبي":2224,"عتب":4439,"عات":3583,"عائ":1805,"عاب":1251,"عال":9007,"عام":18844,"عبا":2776,"عاص":3770,"عار":1586,"عاد":3689,"ظمة":1530,"غسط":1434,"غدا":1272,"غرا":1470,"غرب":8757,"عية":4635,"عيد":1672,"عين":3620,"عمل":7530,"عمو":1379,"عمر":2346,"عها":1818,"عني":1581,"غال":2120,"عند":3134,"عهد":2118,"عود":4342,"علا":3567,"علي":8023,"على":25792,"علو":2893,"عما":5586,"علم":6076,"غني":1336,"غير":5788,"شعر":1240,"شعب":2093,"شما":7339,"شكل":4349,"صال":3820,"شهر":2525,"صبح":1804,"صاد":2357,"صار":2022,"شمي":2083,"صدر":2012,"شير":1659,"شيخ":2626,"طة ":4836,"ضي ":2799,"صطل":1544,"صري":4147,"ضم ":1876,"طب ":1314,"طس ":1466,"صغي":1584,"صول":1823,"صور":2575,"ضاÙ":1670,"ضاء":2485,"صنا":1592,"صمة":2695,"طق ":1799,"صية":1437,"صين":1293,"طي ":1477,"ظة ":6257,"طان":3004,"طال":3847,"طاق":1625,"طار":1511,"طائ":1561,"ضمن":5751,"طبي":3127,"ضية":1599,"طري":4168,"ظم ":1728,"عب ":10724,"عة ":23265,"طعة":4116,"عد ":10757,"عر ":1910,"طلح":1521,"طلق":2117,"طلا":1304,"عض ":2738,"طقة":8653,"طول":4093,"طوي":1533,"طور":2637,"ظام":2445,"طني":2154,"طين":2907,"عل ":1213,"غة ":3658,"عن ":10475,"عه ":1714,"عي ":3208,"س، ":1536,"ري ":14515,"رن ":3048,"زب ":1762,"ره ":3780,"زة ":3835,"رو ":2228,"رى ":5751,"رق ":5370,"رك ":3561,"رجي":1396,"ردن":4580,"ردي":1386,"رتب":1460,"ربع":2964,"ربي":13069,"رجة":3401,"رجا":1686,"ذين":1312,"رتÙ":1376,"رته":1217,"رتي":1328,"ذلك":5018,"ران":5308,"ربا":2177,"راه":1718,"راي":2650,"رة،":1284,"راً":1437,"راط":1969,"راض":2661,"راع":2044,"راÙ":2275,"راك":1677,"راق":4338,"رام":2614,"رال":1755,"راب":2845,"رائ":2827,"راج":1725,"رات":7689,"رئي":4628,"رار":2322,"راد":1959,"راز":1251,"راس":3033,"راء":3735,"ذكر":1364,"رأس":1309,"سس ":1682,"سر ":1409,"زي ":2367,"سة ":7637,"ست ":2491,"رسة":1580,"رسا":1400,"سا ":2325,"رسو":1660,"رسم":1498,"رسي":1211,"سب ":2371,"سم ":8379,"زرا":1870,"سل ":2019,"ريو":1361,"ريم":1723,"رين":5069,"ريك":7429,"ريل":2022,"ريÙ":1889,"ريق":6975,"ريا":9368,"ريب":2334,"ريخ":4312,"ريد":2596,"ريت":1569,"رية":19287,"ريط":2009,"رير":1548,"ريس":2387,"روي":1263,"روÙ":2873,"رون":3533,"روم":2673,"روع":1294,"روس":3818,"روت":1414,"روا":3408,"روب":3276,"رها":3942,"رنس":4822,"زار":1503,"رنا":1951,"زائ":1892,"سع ":1228,"ركي":2730,"ركز":6519,"ركا":1845,"رقم":1561,"ركة":6574,"سط ":4117,"رقي":3306,"رÙÙŠ":1260,"رقة":1238,"شر ":3663,"سن ":1880,"سي ":9223,"سري":1260,"سرا":2022,"صب ":1600,"ساع":1424,"سام":1889,"سال":1558,"سبب":1717,"سان":4418,"سبا":5421,"سبت":1759,"سبة":1761,"ساح":5795,"ساس":2185,"سائ":1433,"سات":2066,"ساب":4056,"ستخ":5235,"ستا":3297,"ستر":2386,"زوج":1305,"زيو":1415,"ستع":2199,"ستق":2335,"زيا":2152,"ستي":2146,"زية":3984,"ستو":3126,"زيز":1328,"زير":3447,"صر ":6441,"سلة":1569,"سكن":1586,"سلا":5866,"سكر":1892,"سلي":1241,"سمب":1534,"سمة":2358,"سلم":2588,"سما":2312,"سلط":1809,"سلس":3040,"سكا":4595,"سطس":1421,"سطي":2518,"صة ":3360,"سعو":3150,"شرك":4217,"شرق":6331,"شرو":1260,"شري":3368,"صل ":5101,"ضة ":1538,"ضا ":2191,"سوÙ":1368,"سون":1558,"شتا":1442,"سوي":1779,"سوا":1477,"سود":1542,"سور":3979,"شاع":1427,"سمى":2128,"سنة":6767,"سمه":2089,"شار":4714,"سمي":3653,"شخص":2650,"ص٠":1872,"سين":3825,"سيم":3305,"سيق":1533,"سيس":1471,"سير":1927,"سيد":1957,"سية":7660,"سيا":7442,"شته":1322,"يين":4613,"يقو":1288,"يقي":3960,"يقع":4212,"يكا":2890,"يكي":7123,"يلع":3028,"يلة":3508,"يكو":2936,"يلا":3946,"يلي":4907,"ين،":2158,"يلم":2210,"يما":3516,"يمة":2518,"يلو":2309,"يمت":1285,"ينا":6385,"يمن":1503,"يمك":1768,"ينت":2412,"يمي":5784,"ينة":14065,"ينو":1382,"يني":7771,"ينم":1347,"ينه":1443,"يها":5753,"يوس":2044,"يور":1429,"يوج":1408,"يوا":1512,"يون":8407,"يوي":1456,"يوم":3633,"يول":2410,"يقا":3504,"يقة":2474,"ÙŠÙÙŠ":1837,"ÙŠÙØ©":1491,"يضا":3644,"يسي":3184,"يسم":2811,"يزي":6603,"يست":3754,"يسا":1431,"يره":2025,"يرو":2802,"يري":3325,"يعي":2030,"يعر":1330,"يعة":1447,"يعت":2839,"يطا":4786,"يجي":2143,"يتا":1503,"يتو":1517,"يتي":2848,"يتم":2214,"يته":1496,"يدي":4565,"يرا":5132,"يرة":7852,"يحي":1325,"يدة":3171,"يدا":2277,"يال":1789,"يبا":1460,"يان":5624,"يام":1663,"ية،":7809,"ياً":2041,"يبل":2267,"يبي":2606,"يئة":1233,"يا،":2810,"ياس":4902,"يار":3005,"ياد":1919,"ياض":3099,"ياء":2370,"يات":14201,"ياب":2680,"يائ":1532,"وز ":1740,"ور ":10877,"ود ":7936,"وض ":4335,"Ù†Ùس":2468,"وس ":4788,"هاد":1315,"هاج":1381,"هاز":1295,"هار":1592,"هاش":2115,"نما":2137,"ها،":1369,"وع ":4025,"نقل":1354,"وط ":1225,"نيس":1670,"نيا":8848,"نية":21320,"نوي":1623,"نون":2797,"نور":1366,"نوب":7417,"نهم":1521,"نوا":4198,"نوÙ":1932,"نوع":3104,"هاي":1376,"هام":1659,"نها":10954,"نهر":1868,"ÙˆÙ ":3389,"هذا":5637,"وق ":2861,"نين":2937,"نيو":3670,"وم ":10050,"ون ":24085,"هرة":1975,"هذه":5272,"هرب":1206,"وك ":1613,"ول ":15598,"وي ":5541,"ÙŠØŒ ":7950,"وى ":2336,"مغر":2269,"معي":2389,"معل":1517,"معه":1532,"معر":3470,"معت":1206,"معة":3514,"معا":4652,"هد ":3199,"هر ":6421,"مقا":7017,"مقر":1875,"مقد":1322,"ملة":1472,"ملا":2140,"مكن":2428,"مكا":1474,"مكت":1317,"منا":3879,"ممل":5220,"نائ":1835,"ناء":3405,"ناد":6151,"منذ":4141,"منت":3699,"نات":4769,"منظ":1521,"منط":9032,"ناط":2245,"ناص":1498,"منص":1619,"ناس":1610,"ناع":1893,"ملك":8686,"مما":1241,"ملي":3442,"ممث":1944,"مهو":2239,"موا":8484,"نبي":1323,"موج":1832,"مور":1727,"مود":1263,"موس":3401,"موع":3576,"موق":1778,"ناك":1264,"نام":2198,"نان":5318,"نبا":1540,"مها":3875,"منه":3913,"ناي":2308,"مني":1297,"نتق":1216,"نتش":1566,"نتخ":2464,"نتج":1837,"نتر":1240,"نتا":3081,"مون":2835,"موم":1267,"مول":1330,"ميل":3548,"ميع":1518,"ميد":1868,"ميز":2165,"مير":3381,"نتم":1247,"ميا":3833,"ميت":1549,"مية":10968,"نتي":2558,"نجل":4522,"مين":4370,"هـ ":2122,"ندا":2054,"ندم":1329,"ندو":1295,"ندر":1700,"ندس":1559,"ندي":3768,"هل ":1310,"هم ":9416,"نسا":3625,"نسب":2400,"وا ":3570,"نسم":2218,"نشا":1687,"نسي":6788,"وب ":7482,"هو ":28172,"وة ":2028,"وت ":2509,"هي ":25821,"نطق":9203,"نظا":2438,"نظر":1914,"نظم":2245,"نظي":2652,"ومن":4650,"ونا":4025,"ومي":3551,"ونس":2979,"وما":5525,"وله":2128,"ولي":9104,"ومة":1640,"ولو":3124,"ولى":4041,"ومت":1425,"ولك":1651,"يع ":3808,"ولة":5953,"وكي":1778,"ولا":11130,"ولد":6407,"وكا":4860,"يط ":1772,"وقا":1242,"ÙˆÙÙ…":1445,"ÙˆÙÙŠ":6567,"يض ":1235,"وقد":4338,"وقع":2237,"ÙˆÙا":1769,"يش ":2562,"يس ":7953,"يق ":5733,"ÙŠÙ ":4553,"وين":2166,"ويو":1325,"ويق":1310,"ويل":2767,"ويس":2024,"ويع":1988,"وية":5123,"ويت":3787,"وير":2162,"ويد":1280,"وهي":6413,"وهو":7384,"وني":7306,"يه ":8938,"يو ":7129,"يم ":12246,"ين ":40630,"ًا ":1445,"يك ":2151,"يل ":12694,"وتو":2723,"وتق":1399,"هير":1949,"وتع":2163,"هول":1332,"وتر":1472,"وتت":1213,"وجه":1246,"وجي":2191,"وجو":2155,"وحد":1804,"هيم":1268,"وجد":2346,"واع":2005,"واس":3039,"وار":3337,"هند":2766,"واد":3067,"واح":3498,"واج":1496,"وات":4032,"واب":1388,"هنا":1458,"وائ":3044,"واء":1341,"هما":1800,"هور":3656,"وبي":4664,"وبل":1243,"وبر":3000,"واي":2060,"واق":2063,"وال":49239,"وان":5702,"وبا":3324,"وري":12364,"وسا":1602,"وزي":2144,"يا ":24320,"وسط":5112,"وسي":5438,"يب ":4787,"ية ":139658,"ودا":1655,"ودة":1389,"ودي":4370,"ورا":3830,"ورد":1319,"ورت":1259,"ورة":4176,"وزا":1263,"ورو":3091,"يد ":18420,"ير ":22585,"يز ":3857,"يث ":7060,"يت ":4897,"يج ":1671,"وطن":2615,"يح ":1259,"وعة":3516,"وعا":1992,"يخ ":5401,"وعي":1203,"لد ":7149,"لس ":3640,"لة ":25605,"كو ":1397,"لت ":2572,"لا ":8651,"كن ":4742,"قسم":1469,"لب ":3341,"قسي":2428,"لح ":4377,"لث ":1339,"Ù…ØŒ ":3933,"كي ":5663,"لق ":2574,"Ù„Ù ":3842,"كثي":2170,"له ":10668,"كسي":1295,"ما ":21645,"كرو":1414,"لم ":13383,"كري":3468,"لك ":9653,"كرا":1233,"كرة":8153,"كات":3795,"قنا":1376,"كار":2777,"لغ ":5250,"قلي":3507,"قيم":1916,"قيق":2120,"كثر":3184,"قية":4744,"كتو":3421,"قيا":3458,"كتب":2651,"كتا":3440,"قوم":2204,"قوا":2182,"كبي":3060,"كبر":3750,"كان":20675,"كام":2151,"كال":2865,"قني":1349,"كر ":2225,"كز ":6044,"كس ":1605,"قل ":2905,"Ùرا":2274,"قم ":1363,"Ùري":3990,"Ùرن":4775,"Ùرق":1262,"كا ":2472,"كة ":13804,"قى ":1356,"Ù„ØŒ ":2187,"قي ":5360,"Ùضل":1264,"Ùظة":6038,"Ùير":1961,"Ùيز":1412,"Ùية":2824,"Ùيد":1310,"Ùيل":3788,"Ùيه":4815,"Ùين":1921,"قتص":1591,"Ùيا":1360,"قدر":1289,"قدم":9632,"قدي":3437,"قرن":2599,"كم ":5038,"قرى":1679,"قري":5990,"كل ":7728,"قرا":2563,"قرب":1568,"Ùلس":2794,"Ùلا":1746,"قاط":5379,"قاÙ":1502,"قال":3148,"قائ":2080,"قاب":1505,"Ùنا":1210,"قات":2176,"قاد":1517,"قار":3777,"Ùمب":1413,"Ùور":1725,"قبل":4422,"قان":1522,"قام":2893,"لمع":7155,"لمغ":2396,"ماء":2699,"لمر":5941,"لمس":9884,"لمش":2559,"لمص":4216,"لمط":1236,"لمت":9057,"للو":1687,"لمة":1580,"لمج":4154,"للي":1280,"لمخ":2131,"لمح":5998,"لمد":5967,"لمؤ":2072,"للم":3559,"لمب":2156,"لله":4901,"لما":10778,"لنÙ":1218,"لنس":1662,"ماس":1244,"لنظ":1668,"ماع":3269,"لمي":7408,"مات":4917,"مار":7604,"ماد":2511,"لند":3683,"لمل":4323,"لمك":1995,"لمق":3111,"لمو":6905,"لمه":2311,"لنب":1670,"لنا":3789,"لمن":8707,"لمم":6914,"مائ":1529,"مؤس":2067,"لكر":4170,"لقي":1879,"لكب":1845,"لكا":2896,"لكت":3059,"لكة":5304,"لقو":2652,"للغ":3104,"للع":2547,"للح":1276,"لكي":2900,"للت":1711,"لكو":3675,"للب":1703,"لكه":1307,"للا":2747,"لكن":3420,"لكل":2395,"للأ":1511,"Ù„Ùر":5137,"Ù„Ùا":3058,"Ù„ÙØ©":1423,"Ù„Ùت":1915,"لقر":6914,"لقد":5675,"لقص":1290,"لقط":1312,"Ù„ÙÙ„":2420,"لقب":2460,"لقا":4848,"Ù„ÙÙ†":2512,"Ù„ÙÙŠ":3665,"نس ":2245,"ند ":3871,"نذ ":4066,"لعب":7815,"لعا":12423,"لعد":3105,"لعش":1359,"لعز":1344,"لعر":10614,"لعص":1325,"لعل":5280,"لغا":2170,"لعم":4185,"لغة":3148,"لغر":4050,"لطب":2889,"لطا":3030,"لطر":1829,"ه، ":2068,"مي ":10289,"لطي":1270,"و، ":1265,"ني ":19921,"نى ":2242,"مصر":6502,"مصط":1846,"نو ":1687,"هة ":2433,"مصا":1239,"مسل":3430,"نه ":6502,"مسي":2261,"مشا":1795,"ها ":46135,"مست":5627,"مسا":7688,"مرك":7198,"مري":7170,"مرو":1205,"مرا":3794,"مرة":1768,"مرب":1566,"مرت":1375,"مدي":14448,"مدر":2755,"مدن":2068,"مخت":2389,"مدا":1536,"محل":1510,"محم":5603,"ليو":6519,"ليه":5228,"مجا":2341,"لين":3186,"مجم":3939,"محا":8346,"مجل":3322,"ليا":10564,"ليب":2394,"لية":13788,"متو":1712,"ليز":4850,"ليس":2021,"ليد":6582,"ليم":5806,"ليل":2383,"مثل":6304,"ليÙ":2446,"لوي":1597,"لون":3102,"متا":1234,"لول":3949,"لوم":4251,"متر":3819,"متد":1293,"متح":5418,"متع":1688,"لهن":1850,"لوا":3317,"لهو":1322,"مبي":1451,"لوج":1868,"لوح":1460,"لور":2028,"لوس":2838,"لوط":2044,"لوك":1356,"مال":13395,"مام":2587,"لنق":1670,"لنو":1865,"مان":13056,"مبا":2715,"لها":8071,"ماي":3281,"لهج":1315,"مبر":6074,"لأص":1592,"لأس":4356,"لأر":8270,"لأد":1635,"لأع":1940,"لأخ":2355,"لأح":2438,"لأب":1984,"لأÙ":1387,"لأل":2944,"لأك":1560,"لأن":3516,"لأم":7460,"لأو":8534,"لأي":1221,"لإس":5236,"كلا":1532,"مر ":4953,"مس ":1733,"مد ":7895,"مة ":20092,"لو ":1775,"مت ":2546,"لى ":49394,"لي ":18230,"ن، ":5582,"مج ":2119,"لسÙ":1264,"لشا":2194,"لسن":1963,"لسك":1528,"لسل":6280,"لسي":5235,"لشب":1219,"لسو":4213,"نب ":1230,"مه ":3791,"لسع":2105,"لسط":2488,"لسا":4332,"من ":76224,"نا ":5184,"لصو":1503,"مى ":3080,"نت ":7260,"لصي":1707,"لشم":4027,"لصا":2983,"لشه":1224,"لصح":2065,"نة ":25901,"لشي":3442,"لشر":6885,"لشع":2442,"لدي":8697,"لدو":6858,"لدر":3930,"لخل":2460,"لدا":2727,"لدة":1345,"لحي":2699,"لري":2867,"لرو":3390,"لرس":1728,"مل ":8293,"لذي":7896,"لرئ":2291,"لرا":2519,"لته":1222,"كيا":1952,"لتن":1720,"لثا":5157,"كية":5479,"لتو":2940,"لتي":15294,"كيل":2229,"كيم":1808,"لجا":2810,"لتا":4876,"كون":7678,"كوم":2797,"لتح":2710,"لتج":1719,"كوي":2650,"لتر":3418,"لتص":1218,"لتش":1260,"لتع":2665,"لتل":1255,"لتق":2650,"لحد":2696,"لجو":1720,"لجي":2140,"لحر":4857,"لحس":1519,"لحم":1993,"لخا":2598,"لحق":1482,"لحك":2052,"لجد":1516,"لجب":1382,"لجز":3847,"لحا":4240,"لجن":5221,"لجه":1784,"لجم":3177,"لاث":2511,"لاج":1537,"لاح":3135,"لاد":4259,"كند":1265,"لار":1311,"لاب":1932,"لات":8075,"لاق":3425,"لاÙ":2118,"لاس":6196,"لاع":5890,"كلي":3267,"لإي":1693,"مع ":8709,"كلم":2454,"لإم":2003,"لإن":6523,"كما":4559,"كور":1677,"لبو":1874,"لبي":3816,"لبل":3617,"لبن":3363,"كول":1392,"لبر":5552,"كهر":1331,"لبح":3898,"لاي":9010,"كنه":1287,"لام":9049,"لان":6278,"لبا":3771,"لال":6356,"لبط":1368},"n_words":[11749565,13990834,9440598],"name":"ar","type":"arab"} \ No newline at end of file
diff --git a/contrib/languages-data/bg.json b/contrib/languages-data/bg.json
new file mode 100644
index 0000000..38d1b9f
--- /dev/null
+++ b/contrib/languages-data/bg.json
@@ -0,0 +1 @@
+{"freq":{"D":2636,"E":1936,"F":2232,"G":2334,"A":4351,"B":3121,"C":4592,"L":2480,"M":3950,"N":1781,"O":1368,"H":1934,"I":6368,"J":1261,"K":1010,"T":2986,"W":1604,"V":1965,"P":3771,"S":5211,"R":2316,"X":1029,"f":3146,"g":5865,"d":9193,"e":32549,"b":4459,"c":9844,"a":30637,"n":21831,"o":21963,"l":16413,"m":12336,"k":7480,"h":8702,"i":28615,"w":2350,"v":3143,"u":15394,"t":17966,"s":19762,"r":22456,"p":5894,"z":1439,"y":4095,"x":1681,"²":3527,"Ì€":1150,"μ":1054,"ν":2280,"ο":2756,"ι":1833,"κ":1014,"λ":1144,"ε":1298,"α":2356,"ί":816,"σ":1479,"Ï‚":1823,"Ï":1221,"Ï„":1963,"ÑŒ":4950,"ÑŽ":16520," o":1042,"Ñ":119927,"ш":24527,"щ":37947,"ÑŠ":118638," k":4461," d":1024,"Ñ„":35061,"Ñ…":26992," e":2141,"ц":57885,"ч":69969,"Ñ€":442208,"Ñ":363493," a":954,"Ñ‚":513431,"у":110117,"Ñ":1026," t":1519," p":1297," s":806,"Й":1921,"И":10722,"Л":9341,"К":22064,"Ð":13530,"Ðœ":19622,"П":22329,"О":13337,"Б":18556,"Ð":21177,"Г":11598,"Ð’":16284,"Е":7594,"Д":15403,"З":5627,"Ж":1534," J":1196," K":898,"Ш":3131," H":1769,"Щ":2903," I":3444," N":1367,"Ю":3236,"Я":1596," O":956," L":2204," M":3517," B":2694,"Т":16259,"У":4389," C":3884,"Р":16110," A":3726,"С":33637," F":2034,"Ц":2988," G":2045,"Ч":3270,"Ф":8267," D":2259," E":1564,"Ð¥":7715,"л":282008,"к":273267,"й":56719,"и":734943,"п":177542,"о":637033,"н":589099,"м":161532,"г":129585," S":4264," R":2038,"в":294348,"б":87024," P":3303,"а":881417," W":1432,"з":132692,"ж":46599," V":1365,"е":647345,"д":212987," T":2518," Ð":17337," Б":18206," Ð’":15997," Г":11364," Д":15029," Е":7181," Ж":1503," З":5448," И":10217," Й":1919," К":20624," Л":9135," Ðœ":19107," Ð":12710," О":12505," П":21699,"Co":972,"I ":2499," б":25135," а":29615," г":41568," в":81473," е":87884," д":46722," з":26978," ж":5999," и":93394," л":8242," к":53137," н":137556," м":36472," п":102853," о":88041," Р":15097," С":31643," Т":15680," У":4153," Ф":7812," Ð¥":7558," Ц":2895," Ч":3246," Ш":3096," Ю":3200," Я":1567," Ñ‚":27331," у":9960," Ñ€":36272," Ñ":111437," ц":6298," ч":13239," Ñ„":16514," Ñ…":6966," ш":3296," щ":6810," ÑŽ":3698," Ñ":2381,"Ca":930,"Ma":1469,"II":1888,"Th":962,"b ":891,"a ":4950,"i ":1873,"ge":970,"he":2112,"ha":1204,"g ":1075,"ea":1397,"ec":826,"ed":916,"de":1775,"di":1045,"do":824,"h ":1014,"el":2062,"en":3274,"et":1200,"es":2494,"er":5287,"ca":1035,"e ":9458,"da":1108,"f ":974,"co":1092,"ci":1241,"ch":1482,"ce":1332,"d ":2383,"at":2587,"as":1457,"ar":3743,"al":2739,"ai":928,"am":1115,"an":4726,"ac":1396,"ad":830,"ae":1227,"nu":977,"nt":2097,"ns":1121,"no":824,"of":927,"om":1215,"on":3740,"ol":1525,"m²":3381,"ot":815,"os":1234,"ou":906,"or":3049,"r ":3015,"pe":805,"lo":1036,"ll":2061,"o ":2327,"ma":1173,"mb":856,"me":1262,"mi":900,"na":1825,"nd":1832,"ne":1694,"ng":1442,"ni":2114,"m ":2526,"km":4286,"li":2736,"le":2314,"la":2281,"n ":4895,"hu":824,"hi":896,"id":1203,"ic":2468,"ia":2261,"ig":1082,"ie":1280,"k ":904,"ir":844,"is":2366,"it":1525,"iu":2316,"il":1887,"in":4071,"io":1821,"l ":2342,"y ":2050,"vi":1029,"ve":1027,"x ":1094,"ul":1045,"ur":1295,"us":5560,"um":1527,"un":918,"tu":1122,"to":1372,"tr":962,"te":2234,"ti":2705,"th":2095,"ta":1616,"ss":891,"st":2061,"se":1041,"si":1173,"rt":1054,"ro":1958,"rn":867,"ri":3533,"re":2281,"rd":988,"ra":2624,"t ":3334,"s ":10393,"² ":3527,"Ï‚ ":1819,"ν ":1021,"К ":979,"Ð’ ":2290,"юг":1081,"юз":1052,"юж":980,"юл":1213,"юн":1123,"ÑŽÑ€":1010,"ÑŽÑ‚":1002,"юц":3295,"юч":2002,"Ñд":1101,"Ñг":923,"Ñв":7999,"Ñн":5481,"Ñм":2556,"Ñл":2107,"Ñк":3070,"ÑÑ…":803,"ÑÑ":1766,"ÑÑ‚":23106,"ÑÑ€":1490,"Ñщ":989,"щи":5926,"ще":7866,"ща":13149,"щт":2297,"що":3015,"щн":816,"ъв":5386,"ъг":8535,"ъд":2915,"ъе":2628,"ъж":1821,"ъз":5682,"ъб":2128,"ÑŠÑ‚":9515,"ъч":1319,"ъц":2116,"ъщ":3958,"ъл":18163,"ък":3158,"ън":6470,"ъм":3577,"ъп":2106,"ъо":998,"ÑŠÑ":7274,"ÑŠÑ€":28553,"ьо":4687,"хе":1611,"хи":3810,"хн":1728,"хо":7132,"Ñ…Ñ€":1667,"ху":1707,"ха":4667,"ци":32071,"цк":2149,"ца":7811,"це":9248,"чл":917,"чн":11748,"чо":1988,"чи":7731,"чк":2263,"чу":949,"чр":804,"цъ":803,"че":23255,"ча":14633,"чв":2213,"шн":3596,"шк":2046,"ши":6517,"ше":4003,"шв":1228,"ша":3103,"Ñк":74214,"Ñм":3598,"Ñл":11817,"Ñо":8891,"Ñн":7769,"ÑÑ€":4324,"Ñп":8021,"Ñв":7943,"Ñе":41908,"Ñи":23052,"рш":1570,"ръ":16028,"Ñ€Ñ":3303,"Ñа":17003,"Ñ€Ñ":15501,"рт":9867,"ру":13758,"рх":2778,"рц":2298,"тн":10904,"тл":2176,"тк":3861,"Ñ‚Ñ":3598,"Ñ‚Ñ€":27985,"то":91021,"те":76446,"тд":804,"тв":20249,"ти":47326,"ÑÑŒ":1346,"ÑÑŠ":18957,"ÑÑ":1659,"та":97852,"тб":3928,"Ñу":2617,"ÑÑ‚":96170,"Ñц":1590,"ур":10561,"уп":6474,"ут":4765,"уÑ":9980,"ум":6401,"ул":6590,"ун":6478,"уи":1236,"уз":3299,"ук":5239,"уд":4037,"уг":3988,"уж":1616,"уе":1334,"уа":3124,"Ñ‚Ñ":2319,"уб":6162,"ув":3514,"Ñ‚ÑŠ":10928,"Ñ‚Ñ‚":3628,"ту":8218,"фу":2995,"Ñ„Ñ":1030,"Ñ„Ñ€":3456,"фо":5112,"фи":11161,"фе":4259,"фа":3044,"уч":6066,"уш":4842,"ух":1233,"уц":1227,"Щ ":2591," I ":819," II":1042," Ma":1457,"а ":358572,"С ":937," Ca":925," Co":962,"ИÑ":1007,"Им":977,"Ин":1158,"к ":21804,"Из":2286,"Ив":1372,"й ":18686,"Ле":1710,"Ли":1795,"Ла":1638,"Ку":1413,"Кл":1286,"Ко":5445,"м ":14788,"Кр":2175,"Ке":836,"Ки":1609,"Ка":6073,"л ":25621,"Йо":1544,"Ðа":5278,"Ðе":1875,"Ðи":2119,"Мо":2461,"о ":136170,"Ма":9260,"Ми":2960,"Ме":2505,"Ло":1497,"н ":69303,"Лу":1452,"Па":3246,"Пе":2744,"Пи":1272,"Пл":3225,"Ñ ":29451,"По":4526,"Оп":1050,"Ñ€ ":30654,"ОÑ":2133,"Ор":1247,"От":1087,"Об":2207,"Ок":2446,"Ðо":2049,"п ":2647,"в ":63211,"Ðм":1040,"Ðн":3455,"Ðл":2956,"Ðв":1300,"Ба":2769,"ÐÑ‚":831,"ÐÑ€":1719,"б ":1391,"ÐЩ":2595,"Во":969,"д ":24331,"Ве":2399,"Ви":2290,"Ð’ÑŠ":2366,"Га":1581,"Бо":2717,"г ":21217,"Бр":1925,"Бе":2785,"Би":1506,"Бъ":3741,"Ва":2968,"Бу":1134,"Ди":2316,"Дж":3197,"Де":2501,"Др":1035,"До":1935,"ж ":1730," Th":951,"Ед":825,"Ев":1551,"Ге":2579,"Гр":2290,"е ":164103,"Го":1786,"Гъ":1180,"Да":1394,"и ":202644,"За":3607,"Ел":966,"з ":12874,"Ша":824,"Юж":1162,"ÑŽ ":929," km":4266,"Ñ ":66327,"Ст":4040,"Та":1626,"Съ":4599,"Ти":1166,"Те":2600,"Ñ„ ":1914,"То":5455,"Тр":1786,"Ту":994,"ТÑ":1180," e ":1798,"Ñ… ":2350,"Пр":4433,"Пъ":1179,"СÐ":2694,"Ра":3306,"Ре":3655,"Ри":2528,"Ñ‚ ":98842,"Ро":3022,"Ру":2132,"СС":822,"Са":2940,"Св":1625,"Си":2245,"Се":4020,"Сл":1252,"Ск":946,"Ср":881,"Сп":1322,"Со":2387,"у ":10195,"Це":1510,"ш ":1936,"Че":1438,"щ ":4203,"ц ":3623,"Фо":984,"Фр":2381,"Фе":819,"Фи":1127,"Фл":885,"Ха":2310,"Хр":971,"Хо":1081,"ч ":2829,"Хе":1295,"лю":6411,"мб":1834,"лÑ":9658,"ма":34017,"мв":3536,"ме":34130,"ми":25035,"лм":3442,"лн":16796,"ло":33848,"лÑ":4866,"лт":3196,"лу":5370,"лъ":2856,"къ":5642,"лв":914,"лб":2941,"ла":34165,"лж":1120,"ле":40750,"лд":1625,"лг":12737,"лк":4727,"ли":64840,"лз":2343,"км":1179,"кн":2476,"кл":6420,"кр":18141,"кÑ":6288,"ко":52495,"кт":11901,"ку":5862,"кц":1433,"ка":68697,"ки":58354,"кв":3364,"ке":7674,"йн":7255,"йо":1892,"йк":1995,"йл":1354,"йм":901,"йÑ":13136,"йт":4652,"иÑ":73064,"ищ":2902,"иш":2457,"йд":1453,"йв":1615,"ио":15275,"ип":3934,"им":23532,"ин":58356,"ик":36656,"ил":29439,"ии":4086,"ий":12560,"иц":13361,"ич":23215,"иф":2446,"их":3360,"ит":63592,"ир":17875,"иÑ":34276,"ри":74729,"рк":5043,"рл":2965,"рм":7553,"рн":12220,"ро":46663,"рп":941,"ра":98873,"рб":2434,"рв":7403,"рг":7600,"рд":5085,"ре":68474,"рж":3084,"рз":1336,"пъ":7097,"пр":53247,"пт":2156,"пÑ":2576,"пу":5784,"пи":12418,"по":49161,"пл":7108,"оÑ":7068,"па":18259,"пе":14376,"ощ":6422,"ош":1648,"оч":6409,"оц":2874,"оÑ":31400,"ор":52184,"оп":12711,"оо":918,"ох":1705,"оф":5618,"оу":1691,"от":71852,"ок":20081,"ол":46187,"ом":18413,"он":40545,"ож":8402,"оз":8385,"ои":10610,"ой":13778,"ов":49786,"ог":14314,"од":35825,"ое":7078,"оа":1786,"нÑ":5762,"об":22311,"нъ":1330,"нц":6022,"нч":1314,"нт":20098,"нÑ":29338,"нф":1197,"ну":2379,"но":61747,"нн":7922,"нр":827,"нк":4229,"нз":932,"ни":106631,"не":37699,"нг":9593,"нд":12141,"мÑ":2035,"на":206302,"мъ":3046,"му":8961,"мÑ":4731,"мп":7095,"мо":14552,"мн":4561,"ге":6356,"ги":12185,"гн":1340,"го":24434,"гл":8226,"гр":23857,"гу":2534,"гъ":1359,"дв":5622,"да":30493,"вг":1251,"вд":1142,"ве":44922,"ви":31125,"вк":2396,"вл":5218,"вн":10484,"во":35579,"вр":11007,"вÑ":5682,"ву":1966,"вт":3431,"вш":985,"въ":12004,"га":25651,"вÑ":3485,"би":11046,"бе":7947,"бр":9346,"бн":1672,"бо":12574,"бл":10831,"бу":4434,"бÑ":2154,"бÑ":950,"ва":58204,"бъ":10139,"бщ":5858,"ад":29187,"ае":3072,"аж":2734,"аз":20177,"аб":6725,"ав":32263,"аг":6621,"ам":19866,"ан":92124,"ап":10532,"аи":1848,"ай":16768,"ак":18833,"ал":43351,"ах":2622,"аф":3137,"ач":7356,"ац":9711,"аÑ":30917,"ар":52528,"ау":3593,"ат":95702,"аÑ":1940,"ба":6691,"ащ":6812,"аш":2543,"зт":3682,"зÑ":1637,"зр":2676,"зп":8092,"зх":1144,"зу":1891,"зк":2737,"зи":16355,"зо":4852,"зн":6991,"зм":2937,"зл":3437,"ив":14438,"иг":8987,"иа":9057,"иб":2301,"иж":2121,"из":36345,"ид":10074,"ие":26794,"зъ":1412,"жо":1971,"жу":937,"жи":10091,"жк":1307,"жн":4179,"за":35424,"зб":1209,"зв":12292,"зг":1227,"зд":6172,"зе":4307,"еф":2109,"ет":45213,"еÑ":32066,"ер":49860,"еп":8513,"ео":5548,"ен":105327,"ем":23126,"ел":47225,"ек":18670,"ей":11615,"еи":962,"ез":20414,"еж":9844,"ее":2077,"жд":7425,"же":11581,"жа":5863,"еÑ":1281,"ещ":3642,"еч":4774,"еш":4262,"ех":2366,"ец":5302,"дÑ":7551,"др":9809,"ду":10303,"дн":14843,"дм":2251,"дп":805,"до":23013,"ди":35808,"дл":987,"дк":815,"де":31362,"дж":3808,"еб":3744,"ев":22477,"ег":6758,"ед":43311,"еа":2059,"дÑ":1235," th":1043,"дъ":7215," ар":2430," аÑ":835," ба":2316," ав":2928," ад":1436," ал":2238," ак":2281," ан":5186," ам":5117," ап":1113," бу":1045," ва":1142," бъ":8445," би":3599," бе":1826," бр":3218," бо":2578," бл":1708," вт":1082," въ":7192," ви":5408," ве":5179," во":6415," вÑ":1918," вр":3197," вл":1868," вк":1271," дв":3064," да":7239," го":11993," гл":1931," гр":14818," ге":2801," ев":904," ед":7019," дъ":3603," дн":1385," до":9466," др":4782," ду":4688," де":7755," ди":3107," же":950," еп":830," ел":1747," ек":959," ез":2706," зе":1022," за":22793," зв":852," жи":3991," зн":1109," иг":2469," ид":823," из":17689," ил":7883," ин":3652," им":7956," ит":1193," иÑ":3053," ка":12665," ки":1971," кр":5742," ко":21829," кн":1333," км":1031," кл":2272," ку":1841," ла":1472," къ":3355," ли":3157," ле":2135," ме":7169," ми":3625," ма":9333," мо":4895," мн":1906," му":6862," ни":1121," не":9125," на":119378," но":5035," ок":9717," оз":981," од":1340," об":13021," нÑ":1648," ощ":1554," оф":1095," от":45416," ор":4392," оÑ":5533," оп":3022," по":33250," пл":5058," пи":3055," пе":4542," па":4088," Ре":3648," Ра":3274," Ро":3019," Ри":2525," Пр":4415," СÐ":2616," Пъ":1174," Пе":2739," Па":3238," Ñ ":12955," По":4518," Пл":3222," Пи":1267," От":1080," ОÑ":2127," Ор":1245," Оп":1048," Те":2595," Ти":1162," То":5392," Тр":1782," Ст":4014," Съ":4598," Та":1620," Св":1624," Си":2198," Се":4013," Сл":1251," Ск":944," Сп":1315," Ср":879," Со":2383," Ру":2131," Са":2935," Фр":2376," Фо":984," Фи":1126," Фл":885," Фе":819," ТÑ":1180," Ту":993," Це":1509," Хр":970," Хо":1079," Хе":1290," Ха":2307," Ша":822," Че":1435," Юж":1162," Ба":2734," ÐÑ‚":830," ÐÑ€":1714," в ":45152," Ðн":3448," Ðм":1038," Ðл":2947," Ðв":1297," Ва":2965," Бъ":3733," Бу":1133," Бо":2714," г ":8207," Бр":1924," Бе":2781," Би":1492," а ":4115," Ед":825," Ев":1550," Ди":2315," Дж":3188," Де":2497," Др":1035," До":1933," Ел":962," Ð’ÑŠ":2363," Га":1572," Ве":2397," Ви":2282," Во":966," Гъ":1180," Да":1391," Ге":2574," е ":70638," Го":1784," Гр":2287," ИÑ":1003," Ин":1152," Им":970," Йо":1543," Ки":1607," Ке":833," Ка":6065," и ":47012," За":3604," Ив":1368," Из":2283," Мо":2455," Ðа":5243," Ðе":1867," Ðи":2110," Ðо":2047," Об":2206," Ок":2446," Кл":1282," Ко":5424," Кр":2158," Ку":1412," Ла":1629," Ле":1707," Ли":1793," Ло":1496," н ":881," Лу":1447," Ма":9223," Ме":2502," Ми":2950," Ð’ ":2161,"Зап":1336,"Ива":1277,"II ":1339,"Кар":1717,"Кал":1241,"Кон":1216,"Кол":892," ра":14338," ре":11569," ри":1504," ро":5884," пр":44389," пÑ":1741," пу":1035," пъ":4915," Ñв":5747," Ñи":8451," Ñе":28117," Ñл":4935," Ñм":1742," Ñк":1388," Ñп":4517," ÑÑ€":3911," Ñо":1964," ру":1874," Ñа":8475," ти":1471," те":8827," то":4709," Ñ‚Ñ€":4417," Ñц":1244," ÑÑ‚":9556," Ñу":971," та":2605," ÑÑŠ":15734," уÑ":1789," уп":1428," ун":1177," ту":1481," Ñ‚Ñ":1501," Ñ‚ÑŠ":1070," фо":2263," Ñ„Ñ€":2272," фу":2793," фе":1675," фи":5383," фа":1634," уч":2779," Ñ…Ñ€":1022," хо":1370," ху":840," хи":1498," ха":1277," це":3524," чо":1060," чл":866," чи":1314," че":4520," ча":4234,"Мак":3324,"Мар":2452," ша":881," ща":6448," юж":900," юг":1007,"Южн":1054,"Ðам":821,"ÐаÑ":1183,"Ðик":1456,"ад ":8166,"ав ":839,"ам ":1105,"ан ":12026,"ак ":1047,"ал ":5489,"ай ":7880,"Окр":2063,"авт":1723,"ага":1670,"авÑ":1374,"агр":1076,"аго":1396,"ада":3119,"ади":3451,"аде":5585,"адо":1154,"адм":1400,"адн":2853,"аем":863,"би ":1015,"ажд":973,"Опе":874,"аба":833,"або":2518,"ава":11168,"авн":2861,"авл":2542,"аво":1409,"аве":2231,"ави":4818,"алÑ":906,"алн":8729,"ало":2628,"алб":1650,"ала":2998,"алк":1807,"али":10040,"але":5565,"амо":2012,"амп":965,"ама":2784,"ами":3947,"аме":7196,"анн":912,"ано":4066,"анÑ":13953,"ант":4813,"анц":1702,"ана":8635,"анд":5902,"анг":4584,"ани":19618,"ане":11702,"анк":1502,"азр":1036,"азп":4124,"азо":828,"азн":1081,"азл":1632,"ази":2620,"азв":2060,"аза":1484,"айÑ":963,"айк":1056,"айо":1501,"айн":2031,"акт":4848,"ако":1531,"акÑ":811,"аке":4784,"аки":1127,"ака":2207,"Пар":949,"Ð°Ñ ":2329,"ар ":3643,"ат ":10948,"Пло":2474,"ба ":1349,"Ð°Ñ ":1362,"ащ ":1410,"Пет":1186,"Пър":1058,"СÐЩ":2595,"Пол":1310,"При":1038,"Пре":1768,"Про":1122,"Рим":1524,"ÐЩ ":2589,"Род":1271,"Раз":1500,"Реп":2237,"Ðле":1070,"Сан":921,"Ðнт":866,"РуÑ":1292,"Соф":1291,"Сев":1817,"Све":1167,"Бра":846,"Съе":2317,"Ð¢Ñ ":1123,"Ста":1292,"Сто":900,"Бел":1301,"Тов":901,"Той":2267,"Вел":1059,"Бъл":3319,"Гра":1071,"Ð’ÑŠÑ‚":1318,"Фра":1531,"Гер":981,"Гео":1043,"Дим":1088,"Хри":815,"Гър":1058,"Джо":1453,"Цен":1121,"Евр":1146,"лбу":1567,"лам":805,"лан":5659,"лаÑ":5904,"лат":3266,"лги":963,"ме ":3219,"лга":11038,"ма ":9888,"Ð»Ñ ":1353,"лав":4235,"лаг":1602,"лад":2109,"къÑ":900,"към":1928,"куÑ":995,"кул":1639,"кци":1405,"коÑ":2700,"кре":815,"кра":5743,"кри":2482,"кръ":8385,"кÑа":1220,"кÑи":1227,"кте":804,"кти":2426,"кто":3328,"ктр":1676,"кла":1490,"ло ":7485,"кло":1049,"клю":1993,"кни":998,"ког":1332,"ков":4810,"ком":4727,"кон":4363,"коп":1296,"кор":1875,"коÑ":1575,"кот":3860,"кое":2107,"кои":3390,"кой":3594,"кол":6405,"кин":1233,"киÑ":7501,"лм ":1747,"кит":4721,"ле ":963,"кед":4374,"ли ":17289,"ква":2572,"кат":20763,"кар":2222,"кан":8113,"как":1566,"кал":2533,"каз":1398,"ла ":6130,"йто":3630,"йÑк":8916,"йÑÑ‚":3543,"кт ":1337,"йна":2642,"йно":1113,"йни":1595,"ÐºÑ ":1165,"йон":1165,"ко ":8475,"км ":856,"ки ":42277,"йво":1025,"иÑÑ‚":14767,"од ":4250,"нац":1700,"нау":1528,"нач":4474,"ог ":1478,"нан":1145,"нам":2383,"нал":6581,"мÑÑ‚":820,"нат":17022,"наÑ":8028,"нар":6159,"нап":1914,"над":1993,"наг":991,"най":5108,"наз":881,"нде":899,"нда":1986,"нгл":4272,"нга":993,"нем":1875,"нен":7192,"ои ":1291,"нер":3755,"неÑ":1561,"нет":4998,"нег":1689,"нев":1009,"нди":2598,"ндо":877,"ндÑ":1234,"ндр":1171,"нив":1213,"низ":3481,"ник":6697,"ниг":977,"ние":16641,"ок ":2212,"ой ":4351,"Ð½Ñ ":922,"ов ":8975,"нав":1281,"нт ":3764,"мпи":1340,"мпе":2702,"мпа":913,"мот":885,"мÑк":3992,"мун":847,"муз":1721,"мик":1065,"мил":1270,"мич":1772,"мин":5435,"миÑ":1421,"мир":3990,"мит":3385,"миÑ":2187,"но ":26769,"мна":831,"мно":2749,"мод":977,"мов":951,"мож":1224,"мон":1361,"мол":819,"моÑ":1285,"мор":2165,"Ð½Ñ ":993,"нд ":1174,"мац":839,"мал":2220,"мак":2180,"май":1485,"лÑÑ‚":839,"мат":5742,"маÑ":872,"мар":1751,"лÑÑ€":941,"нг ":1106,"ман":6016,"лÑн":866,"лÑм":2082,"люц":3280,"люч":1984,"лÑв":2485,"маг":843,"меÑ":1879,"мет":5373,"мен":8396,"ни ":29732,"мер":7582,"мей":1790,"меж":2783,"мед":1422,"мвр":3060,"не ":9731,"на ":142528,"лощ":4100,"му ":4914,"лни":5607,"лно":5548,"лна":4870,"лог":4324,"лож":4131,"лор":940,"лоÑ":1248,"лот":2604,"лом":832,"лон":1406,"лов":4045,"луч":1122,"лÑÑ‚":1405,"лÑк":2670,"лта":884,"лзв":2068,"лиа":1575,"лиг":889,"лив":1397,"лиз":3791,"лим":1095,"лий":4336,"лик":5518,"леж":946,"лев":1784,"лед":5662,"лер":878,"ми ":2984,"лен":16713,"лем":2731,"лек":3978,"лет":2054,"мо ":1651,"лищ":1562,"лиц":2347,"лич":3301,"лиÑ":3220,"лит":6583,"лиф":806,"лин":3617,"лиÑ":3676,"лко":1751,"лка":1568,"оÑÑ‚":3638,"пат":1787,"оÑв":887,"пад":4016,"рг ":1045,"оÑн":1222,"пан":2587,"пар":3257,"рд ":1208,"ре ":1433,"ра ":12349,"пит":1282,"пиÑ":5195,"пла":2637,"пле":1081,"ро ":1581,"пло":2643,"ри ":13979,"пер":6644,"пет":985,"пей":864,"пен":871,"пец":914,"рк ":1201,"ори":9957,"орд":1685,"оре":5253,"орг":4291,"орÑ":1471,"оро":2299,"орм":3070,"орн":2306,"опу":959,"ора":5790,"опе":1825,"опи":2343,"опо":1901,"опр":1742,"опа":1705,"оте":1711,"отк":1219,"отл":842,"оти":2542,"ото":15284,"отн":1748,"отр":1056,"отв":1153,"отб":1366,"ота":1480,"оÑе":1009,"оÑи":1734,"оÑл":2267,"оÑм":1054,"оÑн":3443,"оÑо":2118,"оÑÑ‚":14287,"орт":2090,"оръ":1155,"оÑв":865,"оми":2906,"оме":3032,"олÑ":2830,"ома":3616,"олю":3412,"олу":2069,"олÑ":1164,"олн":1271,"по ":9531,"оло":11644,"олк":1549,"оле":5004,"оли":8455,"олз":2123,"ола":2641,"окр":6747,"окт":1051,"око":5161,"онÑ":990,"онÑ":3193,"онт":1752,"они":6360,"оно":4114,"онн":3311,"она":6297,"онд":803,"оне":3282,"омо":1980,"омп":2499,"оше":916,"очи":922,"очн":3093,"още":1644,"ощт":2295,"офе":923,"офи":2989,"оце":1219,"оци":1428,"нÑк":1962,"нÑв":1202,"ова":9654,"общ":5055,"обр":3456,"обо":1542,"обн":833,"обл":3221,"оби":2313,"обе":2463,"па ":3482,"оит":3156,"ойв":1029,"ойт":3479,"ойÑ":1115,"ойн":2834,"ока":2144,"ожн":1229,"ози":2118,"оза":2042,"озн":2520,"оиз":4705,"одн":3018,"оди":10451,"оду":932,"одр":1677,"одÑ":1049,"одо":2573,"оем":891,"оен":1806,"оет":2833,"оже":5313,"ове":9903,"ови":7840,"ово":5297,"овн":3952,"овÑ":1386,"ога":2254,"оги":3492,"ого":3019,"огр":2846,"ода":3875,"оде":5129,"от ":40884,"нот":4382,"ноÑ":7389,"нош":880,"нор":1012,"Ð¾Ñ ":1874,"ном":2595,"ное":844,"ног":2289,"нов":10487,"ор ":8928,"нно":1572,"нни":2936,"нна":3374,"нко":832,"он ":7439,"нка":817,"ом ":1452,"ниÑ":18624,"нир":1151,"ниÑ":3663,"нит":11345,"ним":1508,"нин":2416,"ол ":1852,"нич":2172,"ниц":3898,"нце":866,"нци":4501,"ощ ":1985,"нтъ":2337,"нуа":804,"нта":3339,"нте":1868,"нти":4317,"нто":1031,"нтр":2358,"нÑк":22095,"нÑÑ‚":3816,"Ñам":1827,"Ñан":2444,"Ñат":1871,"Ñва":1135,"Ñво":1889,"те ":30876,"Ñве":3239,"Ñвъ":838,"Ñев":2490,"Ñед":1178,"Ñел":9974,"Ñек":1165,"Ñеп":870,"ти ":11022,"Ñен":2491,"Ñем":2208,"Ñет":920,"Ñер":1205,"ÑиÑ":1303,"Ñич":1328,"ÑиÑ":1833,"Ñит":2033,"Ñих":1495,"Ñин":1858,"Ñил":2220,"Ñим":1511,"Ñки":43899,"Ñка":20500,"Ñле":5451,"Ñла":2258,"Ñко":7743,"Ñми":843,"Ñлу":1333,"то ":53757,"Ñло":1460,"Ñна":1239,"Ñни":1224,"Ñоб":1154,"Ñов":1181,"Ñок":1395,"Ñно":4759,"Ñпе":2031,"Ñпа":1513,"Ñпи":822,"Ñоф":1092,"Ñоц":1039,"Ñре":3295,"Ñпо":2745,"роц":854,"рот":1758,"роф":1212,"роп":2362,"роÑ":2878,"ÑÑ‚ ":15103,"роÑ":1120,"рта":1478,"рти":2728,"Ñ€Ñк":11358,"Ñ€Ñи":1583,"руг":2898,"руп":3391,"руÑ":2190,"рум":975,"рци":1577,"рхи":885,"ръц":1917,"ръÑ":815,"ръг":7752,"ръж":849,"ръб":829,"та ":68838,"рад":11688,"рае":904,"раж":1539,"раз":12192,"раб":3055,"рав":5786,"рам":2735,"ран":13900,"рай":2815,"рак":2465,"рал":6906,"раф":2143,"рац":1472,"раÑ":2637,"рат":10837,"раÑ":1251,"ращ":1083,"рби":1070,"рва":2274,"рди":900,"рдж":839,"реб":1455,"рев":6384,"рег":1970,"ред":14324,"рет":1976,"реÑ":2290,"реп":1732,"Ñи ":5183,"рен":6538,"рем":3987,"рел":1209,"рек":2832,"рей":892,"рез":11468,"реж":2438,"ржа":2590,"рещ":1470,"реч":1092,"реш":2153,"Ñе ":16807,"рво":979,"рве":1684,"рви":2226,"рга":3761,"рги":1489,"рда":805,"риÑ":9828,"рио":1833,"рим":2306,"рин":4333,"рик":7682,"рил":2477,"рий":2243,"рич":4087,"рит":6765,"рир":851,"риÑ":4939,"рка":815,"риа":2344,"риг":1450,"рив":950,"рие":1786,"рид":1515,"риз":1180,"рни":3773,"рна":4116,"рок":1893,"рол":1507,"ром":2389,"рон":2340,"роз":1160,"рои":4466,"рой":1470,"ров":7381,"рог":1151,"род":7855,"рое":938,"рно":2617,"рла":1189,"рми":2339,"рма":3665,"пра":5058,"при":9598,"пре":21619,"про":14907,"поп":1206,"пор":4301,"поÑ":3943,"пот":1094,"поч":1394,"рт ":2255,"пое":904,"под":4233,"пов":2082,"пон":1889,"пом":919,"пол":13153,"пок":1115,"поз":1787,"пуб":3689,"пул":1018,"пте":935,"пÑи":1481,"Ñа ":8477,"пър":3460,"път":1769,"пъл":1566,"вар":1920,"ват":9299,"ващ":3095,"ван":13950,"вал":2194,"га ":4136,"бщи":2449,"бъд":803,"бър":1419,"бъл":7641,"бща":1581,"бще":1011,"бум":1554,"бук":823,"бÑк":1045,"вто":2970,"вÑи":896,"вÑк":2447,"вÑÑ‚":1194,"вре":3504,"ври":3322,"вро":1874,"връ":1065,"вол":4504,"вой":3658,"вое":1398,"вод":4759,"вот":3890,"вор":2694,"вни":3836,"вна":3375,"вно":2879,"влÑ":1602,"вле":1219,"вли":1041,"вла":1152,"го ":2882,"вкл":1196,"виц":847,"вич":1491,"виÑ":2762,"виз":1513,"виж":1141,"вил":1146,"вин":3278,"виÑ":2864,"вит":3903,"вид":3578,"вие":1422,"веч":1347,"веÑ":4369,"вет":7499,"вер":5871,"вен":9985,"ги ":2787,"вел":1221,"век":3636,"веж":1619,"вед":2219,"вгу":856,"ва ":24595,"бан":1118,"ачи":1132,"аши":924,"аща":2195,"ащи":2202,"ащо":823,"афÑ":811,"афи":915,"ача":3148,"аче":1418,"аци":9279,"апр":1718,"апа":4649,"апо":1282,"апи":1535,"арх":1193,"арÑ":9548,"арт":4677,"аÑа":1041,"аре":2062,"ард":1683,"ара":4896,"арн":1173,"арм":1084,"аро":3343,"ари":12961,"арл":1170,"арк":1896,"аÑÑ‚":14401,"аÑÑ":976,"ата":47391,"аÑи":2434,"аÑе":6138,"аÑо":1029,"ату":1909,"ате":8423,"ати":9387,"ато":12657,"атр":1084,"бол":3321,"бор":2442,"бот":2082,"бро":1543,"бри":1900,"бре":1262,"бра":3810,"бла":3919,"бли":5668,"во ":9926,"ви ":4285,"бен":998,"бер":1182,"без":1026,"бед":1013,"бел":1391,"бек":854,"бит":1770,"бил":1742,"бик":912,"бив":1415,"ве ":3400,"дан":2923,"дар":1829,"дат":3453,"дви":1371,"два":2719,"две":833,"ед ":5333,"дал":1095,"дад":3266,"дав":2214,"ев ":3676,"дее":1168,"дек":992,"дей":1639,"дем":1659,"дел":3837,"ден":12663,"дер":1762,"джи":1277,"ей ":1537,"деÑ":1272,"дет":1508,"ез ":10672,"дÑÑ‚":4125,"дÑк":2604,"дро":826,"дру":2364,"дре":1806,"дри":2347,"дра":1827,"ет ":4265,"душ":3708,"ец ":2918,"ен ":33180,"диÑ":1914,"диц":1640,"ем ":1311,"див":1005,"дим":1128,"дин":15737,"дио":936,"ел ":8445,"диÑ":974,"дит":1804,"дие":842,"ек ":3487,"доб":1818,"дов":2389,"ÐµÑ ":1992,"доÑ":1220,"дор":1353,"док":903,"дон":5500,"дна":5206,"дни":3855,"дне":986,"ер ":6034,"дно":4679,"дми":1636,"да ":13229,"вÑв":1024,"гал":815,"вÑÑ‚":853,"гат":2134,"ган":4672,"гар":12015,"де ":1425,"and":809,"an ":963,"във":2376,"въз":2174,"вър":4758,"гол":3089,"гор":2288,"гов":3764,"год":7938,"гру":3152,"ду ":2227,"гръ":1949,"гра":16257,"гри":1115,"гуÑ":909,"ген":2209,"гер":1710,"ди ":4057,"гио":1272,"гиÑ":3508,"гич":1145,"ati":807,"гле":897,"гла":2323,"до ":6155,"гли":4051,"жан":924,"жав":2301,"за ":15694,"еще":814,"жит":2528,"жив":1957,"жиÑ":1541,"жеÑ":1221,"жду":3122,"зи ":3101,"жен":7044,"жда":3366,"жно":1097,"жни":1433,"жна":1550,"ÐµÑ ":816,"жа ":1065,"дъл":1733,"дър":3663,"еец":1133,"ежи":2367,"ежд":5041,"едÑ":4153,"жи ":1063,"еза":1907,"езо":914,"ези":4043,"ева":1249,"еви":2758,"еве":5028,"его":2858,"едв":1395,"еда":2616,"еде":4183,"еди":11058,"едо":5545,"едн":6376,"евн":1170,"ево":4583,"же ":1283,"евр":1969,"ега":870,"еги":1678,"ент":10211,"енÑ":4280,"енц":1053,"ени":27128,"ено":7268,"енн":2947,"ена":10075,"ене":4173,"енд":1760,"еор":1928,"епт":1100,"епу":3069,"епо":1006,"ерÑ":2306,"ерт":1057,"ерм":2802,"ерн":4552,"еро":3866,"ери":14051,"ерг":986,"ере":1937,"ера":7049,"ерв":1379,"ейн":1799,"ейÑ":4529,"еке":826,"еки":1118,"еко":1156,"ект":4178,"екÑ":3530,"ека":2557,"елн":4078,"ели":9768,"елÑ":1847,"ело":4047,"еле":11585,"елг":908,"ела":1732,"емо":966,"еми":3799,"емÑ":1747,"еме":6592,"елÑ":2163,"ема":3994,"емв":2337,"ехн":1011,"еци":1425,"ече":2969,"ешн":2173,"еща":1146,"еÑа":875,"еÑе":2292,"еÑи":1554,"еÑк":7153,"еÑн":1000,"еÑÑ‚":15331,"ета":4044,"ети":4225,"ете":3470,"етр":1876,"ето":20630,"етн":1476,"етÑ":1612,"етъ":1538,"иве":2646,"иви":1509,"ива":2727,"иал":3813,"иан":3297,"иен":1203,"иет":8077,"иже":812,"иев":1368,"игр":2806,"иго":824,"ида":1273,"иди":1105,"иде":2406,"иво":2091,"ивн":2071,"ивш":976,"ига":2136,"иги":1169,"икн":877,"ико":3994,"йн ":1098,"ики":1147,"ика":16650,"ийÑ":6452,"иит":965,"изъ":865,"изх":1032,"изÑ":1113,"изт":2743,"изп":3253,"изм":1954,"изл":1205,"изо":1731,"изн":1188,"изи":3858,"изк":1474,"изд":1669,"иза":5067,"изв":5752,"ион":10338,"инц":2399,"инÑ":874,"иод":1232,"ине":4216,"ини":7734,"ино":2511,"инÑ":5959,"инт":1527,"ина":17378,"инд":1369,"инг":2575,"ими":3098,"име":5126,"имÑ":2034,"имо":1892,"имп":3052,"имн":924,"има":3880,"или":12587,"иле":1355,"илм":2481,"илн":880,"ило":3073,"ила":2122,"иÑи":1489,"иÑа":3563,"иÑÑ‚":15576,"иÑп":938,"иÑо":1673,"иÑл":1753,"иÑк":1625,"ити":5468,"ите":38859,"ита":6132,"иÑÑŠ":1087,"иÑÑŒ":1263,"иту":932,"ито":6371,"итн":995,"ира":10008,"ири":1911,"иро":1628,"ихо":1462,"ице":1072,"ица":5886,"ици":6007,"ифо":843,"ище":1958,"ичи":1220,"ичк":1339,"ичн":6798,"ича":3197,"иче":8842,"ка ":28086,"ив ":1131,"зав":1648,"зае":830,"ид ":2167,"зви":1188,"зве":5171,"зва":3840,"зац":2538,"зат":1310,"зар":1064,"зап":3505,"зан":2665,"защ":954,"зда":4154,"зво":872,"ие ":14021,"ий ":4437,"ии ":2933,"зем":1420,"зик":4455,"зир":1742,"ил ":4244,"ик ":10314,"ин ":8393,"им ":2180,"зиÑ":1529,"зит":1037,"ип ":824,"зма":848,"ио ":855,"зли":2352,"зна":4787,"зни":1000,"зно":822,"ир ":1662,"зов":1006,"Ð¸Ñ ":2255,"зон":1141,"зпо":5089,"ит ":1689,"зпр":1401,"зра":1726,"зпъ":815,"зÑл":944,"зто":3205,"ич ":1609,"зхо":1090,"Ð¸Ñ ":57202,"ius":2045,"is ":934,"ion":1052,"ьор":2769,"km ":885,"южн":872,"km²":3379,"ъцк":1859,"ÑŠÑ‚Ñ€":1736,"ъще":1730,"ъщо":1266,"he ":1146,"ъде":1407,"ъед":2623,"ъве":1335,"ъзд":2638,"ълн":1684,"ълг":11342,"ълж":1010,"ърн":1183,"ърк":1004,"ърж":2771,"ърз":1179,"ърх":1303,"ърт":1540,"ÑŠÑ€Ñ":856,"ърш":1020,"ърц":1189,"ÑŠÑÑ‚":3892,"ъпр":1168,"ърв":4886,"ърд":852,"ia ":930,"er ":1669,"es ":1352,"Ñко":2322,"Ñло":937,"Ñва":6515,"юци":3278,"ючв":950,"ÑÑ‚ ":9602,"Ñне":821,"Ñни":1186,"Ñма":1227,"Ñта":8869,"Ñто":2957,"уци":861,"уча":2759,"учи":956,"уче":1377,"уши":3537,"феÑ":938,"фер":834,"фан":1085,"фин":933,"физ":1170,"фил":3622,"фиÑ":1796,"фиц":1324,"фут":2220,"фре":1952,"фор":3324,"фон":806,"ца ":4811,"ци ":3879,"хан":805,"хар":1263,"хол":1095,"хор":974,"хов":933,"ход":1906,"хим":863,"Ñтн":3691,"Ñто":10615,"ÑÑ‚Ñ€":11666,"Ñтв":16841,"Ñте":8055,"Ñти":11129,"Ñта":11874,"ÑÑ‚ÑŠ":2246,"Ñту":903,"ÑÑ‚Ñ‚":2922,"Ñце":1326,"Ñъд":995,"Ñъв":1667,"Ñъз":2361,"ÑÑŠÑ":4442,"Ñък":852,"Ñън":863,"Ñъщ":2787,"Ñьо":1331,"тав":5484,"так":1485,"тал":3945,"тан":6731,"тай":879,"тат":3225,"тар":3238,"тбо":3512,"твъ":860,"тво":9227,"тви":1621,"тве":3786,"тва":3607,"тех":952,"тем":3541,"тел":16601,"тео":1008,"тен":6993,"тер":6816,"тет":1731,"теÑ":853,"тез":1494,"тек":1131,"тив":4139,"тие":1898,"тка":1702,"ум ":1519,"тич":5203,"тиц":909,"тиÑ":3555,"тин":3775,"тик":4630,"тил":1307,"тир":2210,"тиÑ":902,"тит":2981,"тла":1006,"тно":3117,"ток":1533,"тол":2748,"той":1268,"тов":5201,"тни":5121,"тна":2467,"тре":4236,"тра":10350,"три":4952,"тор":11467,"тот":821,"том":2089,"тон":2179,"ÑƒÑ ":1180,"топ":1014,"точ":3383,"тоÑ":1817,"тта":3294,"тро":5709,"тру":1929,"Ñ‚Ñк":2626,"тур":4751,"Ñ‚ÑŠÑ€":5898,"Ñ‚ÑŠÑ‚":942,"тън":1062,"ува":2742,"уги":1696,"уга":1360,"уар":1716,"убл":3878,"узи":1937,"уди":922,"удо":1145,"уме":1194,"ума":1453,"улт":1584,"ули":1004,"ула":1122,"укт":810,"ука":1012,"упр":1211,"ура":2417,"ург":1080,"ури":1177,"упа":3307,"уна":1368,"уни":1783,"уÑÑ‚":2872,"утб":2290,"урÑ":982,"урн":1905,"уÑк":2598,"уÑи":1038,"що ":2387,"шни":1037,"шна":1940,"шин":804,"ще ":3484,"шен":1940,"щи ":1591,"ъв ":2817,"ÑŠÑ‚ ":6352,"ÑŠÑ ":1987,"ÑŠÑ€ ":7548,"ън ":3472,"ъм ":3109,"щит":1320,"щин":2379,"ъл ":2252,"ък ":1412,"щен":865,"щеÑ":2508,"ъг ":7573,"щат":7215,"m² ":3381,"on ":1530,"щта":2294,"че ":2916,"цен":4575,"чи ":992,"цел":1258,"цеÑ":947,"цер":828,"циа":2597,"ции":1846,"цио":6596,"цит":1550,"циÑ":11555,"ча ":1350,"цар":1123,"цат":1708,"цки":1353,"чев":1376,"чен":5933,"чеÑ":8800,"чер":977,"ши ":3970,"чет":1965,"чле":872,"чки":1198,"чин":1845,"чит":1626,"ша ":1093,"чва":2149,"чаÑ":5618,"чал":1786,"чан":1529,"чав":2550,"ща ":4581,"чре":802,"чна":4150,"чов":1436,"чни":4370,"чно":3174,"us ":4874,"ter":901},"n_words":[7994134,9177756,6462334],"name":"bg","type":"cyrillic"} \ No newline at end of file
diff --git a/contrib/languages-data/bn.json b/contrib/languages-data/bn.json
new file mode 100644
index 0000000..45b1298
--- /dev/null
+++ b/contrib/languages-data/bn.json
@@ -0,0 +1 @@
+{"freq":{"D":455,"E":376,"F":388,"G":399,"A":894,"B":684,"C":848,"L":430,"M":683,"N":429,"O":309,"H":360,"I":507,"J":241,"K":244,"T":598,"W":240,"V":199,"P":651,"S":947,"R":531,"f":1089,"g":1979,"d":2581,"e":8201,"b":1267,"c":2589,"a":8669,"n":6047,"o":5561,"l":3930,"m":2503,"j":208,"k":699,"h":3040,"i":6685,"w":672,"v":711,"u":2926,"t":5635,"s":4238,"r":5985,"p":1752,"z":245,"y":1310,"x":420,"à¦à¦²à¦¾":514,"ক। ":614,"à¦à¦¤à¦¿":234," l":233," m":235," o":526," i":296," d":312," f":245," a":616," b":228," c":315," t":765," p":493," s":393," r":292," J":235," K":227," H":321," I":408," N":324," O":231," L":375," M":610," B":602," C":719," A":760," F":332," G":372," D":404," E":296," S":771," R":448," P":577," W":226," T":506,"ا":336,"Co":219,"উ":9536,"à¦":27004,"ঃ":514,"ং":15654,"à¦":2141,"ই":18078,"আ":10399,"অ":13627,"চ":16928,"ছ":9717,"ঘ":1893,"ঙ":3088,"ঞ":3794,"ট":30397,"জ":27267,"à¦":790,"ও":9899,"à¦":344,"খ":8927,"গ":20205,"ঔ":204,"ক":78264,"ন":86702,"ফ":6705,"প":46274,"ভ":18017,"ব":70605,"য":69717,"ম":48748,"ড":11071,"ঠ":3258,"ণ":10122,"ঢ":578,"থ":13519,"ত":69776,"ধ":12543,"দ":33053,"স":56165,"হ":25168,"়":39420,"া":175719,"ি":114763,"র":156970,"ল":52543,"Ma":241,"শ":24901,"ষ":17272,"ৎ":1686,"à§":145506,"ৌ":1675,"ো":19195,"ৈ":1879,"ে":113569,"ৃ":4705,"ূ":5615,"à§":27604,"ী":22483,"৭":2295,"৬":2181,"৯":5541,"৮":2706,"৩":2127,"২":3895,"৫":2215,"৪":1978,"১":8858,"০":4887,"৷":254,"In":199,"।":25409,"Th":253,"b ":285,"a ":1054,"i ":350,"ge":332,"ga":252,"he":773,"ha":472,"gh":243,"g ":316,"ea":343,"ec":205,"ed":350,"de":506,"di":332,"à¦à¦›à¦¾":284,"h ":358,"el":402,"en":818,"em":278,"et":317,"es":737,"er":1291,"ca":315,"e ":2258,"be":225,"da":210,"f ":445,"ct":266,"co":263,"ci":201,"ch":362,"ce":347,"c ":220,"à¦à¦Ÿà¦¿":2063,"d ":875,"at":923,"as":401,"ar":1026,"al":912,"ai":231,"am":360,"an":1536,"ac":322,"ad":251,"ag":317," ১":7243,"nt":573,"ns":267,"no":211," ৯":250," ৮":298," ৭":311," ৬":364," ৫":417," ৪":387," ৩":666," ২":2429,"of":407,"om":400,"on":1111,"ol":393,"ot":205,"os":230,"ou":252,"or":795,"r ":1035,"pe":220,"lo":359," ঢ":514," ড":1933,"ll":307," ঠ":230," ধ":2352," দ":9809," থ":3157," ত":9984," ফ":3863," প":28296," ন":11037," য":6674," ম":15133," ভ":9746," ব":28530," ল":4344," র":8416,"o ":382," শ":7392,"ma":464,"mb":342," স":27697,"me":396," হ":13181,"mi":232,"à¦à¦•à¦Ÿ":5136,"à¦à¦•à¦œ":1493,"na":524,"à¦à¦•à¦•":204,"nc":255,"nd":698,"ne":429,"ng":599,"ni":414,"à¦à¦•à¦¾":295," ।":1901,"m ":412," à¦":26382," উ":6430,"à¦à¦–া":461," অ":13571," আ":10126," ই":6407,"li":503," জ":9765," ট":1754,"le":571," ঘ":968," ছ":3198," চ":4584," ক":22576,"la":589," ঔ":203," গ":7129," খ":2591," à¦":343," ও":5820,"n ":1470,"ht":261,"hu":305,"hi":274,"ho":238,"id":213,"ic":689,"ia":484,"ig":326,"ie":307,"ir":212,"is":631,"it":565,"il":352,"in":1040,"io":639,"l ":743,"à¦à¦¬à¦‚":4187,"à¦à¦ªà§":233,"y ":803,"ve":326,"x ":253,"ul":217,"ur":460,"us":340,"ut":207,"um":443,"un":241,"to":352,"tr":334,"te":708,"ti":997,"th":909,"ta":488,"ss":271,"st":614,"se":325,"à¦à¦®à¦¨":239,"si":361,"rt":257,"ry":236,"ro":438,"ri":872,"re":667,"rd":212,"ra":839,"t ":920,"s ":1577,"à¦à¦¦à§‡":277,"à¦à¦‡ ":3233," �":274,"à¦à¦• ":911,"à¦à¦° ":2530,"উনি":291,"উনà§":360,"উদà§":622,"উপা":289,"উপর":460,"উপন":490,"উরো":279,"উলà§":354,"উৎস":239,"উৎপ":278,"উৎ":563,"উর":543,"উপ":2213,"উস":347,"উল":530,"উক":210,"উজ":216,"উচ":390,"উট":503,"উত":973,"উন":827,"উদ":753,"ইন":1626,"ইব":205,"ইম":223,"ইয":591,"ইর":537,"ইল":683,"ইস":741,"উই":375,"à¦à¦›":287,"à¦à¦–":589,"à¦à¦•":9005,"à¦à¦®":465,"à¦à¦¬":4214,"à¦à¦ª":275,"à¦à¦¨":320,"à¦à¦¦":306,"à¦à¦Ÿ":2154,"à¦à¦‡":3366,"ছর ":262,"ংশ":1413,"ংস":1069,"ংল":2600,"ংর":2237,"ংব":422,"ংঘ":236,"ংগ":523,"ংখ":753,"ংক":757,"à¦à¦°":727,"ৎ ":620,"ইত":723,"ইট":668,"ইড":241,"ইজ":233,"আস":454,"আল":1088,"ইক":452,"আয":323,"অà§":840,"আম":516,"আর":1419,"আফ":296,"আব":708,"আধ":289,"আদ":422,"আন":1004,"ইউ":806,"ইং":2317,"আছ":551,"অস":568,"আগ":507,"আক":520,"অল":207,"অর":1154,"অভ":838,"অব":2125,"অফ":360,"অপ":433,"অন":3326,"অধ":754,"আই":743,"অথ":198,"অঞ":597,"অত":316,"অক":381,"অঙ":287,"অং":592,"ঙা":410,"চট":225,"চত":213,"চন":791,"ঙà§":2470,"৩ ":950,"চর":577,"চল":2320,"চী":679,"চà§":374,"চা":2514,"চি":4076,"চে":985,"চà§":2421,"ছব":324,"৪ ":965,"ছর":393,"জগ":223,"ঘট":513,"গা":2158,"গস":249,"গি":590,"গী":642,"গà§":2497,"গো":907,"গà§":3546,"গে":1783,"১ ":1256,"চক":208,"২ ":1015,"à¦à¦¾":497,"গোল":259,"৭ ":899,"টক":320,"টপ":202,"টন":552,"ঞà§":2042,"টব":433,"ঞা":1608,"টা":3413,"৮ ":899,"টর":259,"জন":5856,"ছে":4099,"জধ":330,"ছি":2738,"ছà§":520,"গà§à¦°":2862,"ছা":810,"গà§à¦¯":399,"জর":278,"৫ ":1051,"জল":300,"ছো":231,"জম":242,"জয":281,"জো":267,"জে":2691,"জà§":830,"জী":975,"জি":3650,"জা":4328,"৬ ":1035,"জà§":3683,"গের":766,"ওয":1866,"ক।":744,"à¦à¦¤":235,"à¦à¦¶":223,"à¦à¦²":685,"à¦à¦°":2874,"à¦à¦¸":513,"কà§":11379,"খন":635,"কে":8760,"কো":2299,"কী":339,"কি":3416,"কা":12553,"কৃ":1298,"কà§":1325,"কস":263,"গর":1257,"০ ":1923,"গল":479,"খে":1085,"গন":254,"গব":429,"খà§":2313,"গঠ":636,"খি":334,"খা":2637,"খà§":253,"গড":267,"গত":723,"গণ":853,"কল":1752,"খক":249,"কভ":207,"কম":1122,"কয":216,"কর":9263,"কব":584,"কথ":578,"কদ":234,"কন":446,"কত":422,"কট":5691,"কজ":1571,"কক":371,"পক":777,"নো":1529,"নৈ":281,"নà§":18037,"পথ":251,"পত":1140,"পন":1278,"পদ":1221,"পড":271,"পট":230,"পঞ":305,"নর":378,"৷ ":217,"নন":368,"ধে":496,"নপ":524,"নব":764,"নভ":385,"ধà§":3005,"নম":449,"নয":630,"নà§":2067,"নী":3272,"নে":8228,"নস":741,"নি":10383,"না":9095,"বচ":430,"বছ":386,"বঙ":528,"বক":640,"ফল":533,"ফর":691,"পৌ":262,"পà§":14673,"পো":493,"বপ":409,"বন":1731,"ফে":647,"বদ":516,"বত":795,"ম।":432,"ফà§":801,"বড":289,"ফি":694,"ফা":803,"পশ":1040,"পস":313,"পল":236,"পম":204,"পর":6275,"পে":1955,"পৃ":469,"পূ":1624,"বই":313,"পি":1741,"পা":6024,"পà§":2560,"পী":458,"বং":4443,"ভর":238,"বà§":5813,"মক":698,"মগ":375,"ভি":3000,"ভা":10166,"মত":941,"র।":1267,"ভà§":429,"ভূ":887,"মণ":247,"ভে":813,"মন":1866,"মদ":356,"মধ":1657,"মব":565,"ভো":225,"মপ":208,"বব":837,"ফো":381,"বয":356,"ফà§":917,"বর":3916,"বল":2788,"বশ":202,"বহ":1569,"বস":2742,"বা":13175,"বি":13020,"বী":1634,"বà§":748,"য।":373,"বৃ":690,"বে":6807,"বৈ":512,"বো":1000,"ভব":322,"রগ":376,"রক":3370,"রও":363,"রট":549,"যা":13478,"য়":35819,"রজ":517,"রচ":1226,"যস":235,"রঙ":226,"রদ":1297,"রধ":882,"রত":4811,"রথ":1065,"ল।":866,"যà§":2179,"রণ":2957,"যি":521,"রম":1030,"রয":914,"যো":1322,"রব":1734,"যৌ":231,"রভ":428,"রপ":736,"রন":1614,"যে":3967,"মল":265,"যক":743,"ময":969,"ভà§":389,"মর":648,"মহ":1061,"মস":593,"মূ":1527,"মৃ":459,"যত":845,"যদ":240,"ঙাল":366,"মা":10507,"মি":3934,"মী":801,"মà§":1962,"মো":1026,"যব":1634,"মৌ":259,"যম":725,"মà§":4843,"যন":902,"মে":4592,"ড়":3388,"ডা":1047,"ডি":1739,"ঠি":877,"ঠা":1161,"৯ ":835,"টে":2184,"ঠন":362,"টà§":2862,"টো":693,"টà§":322,"টি":14501,"ণী":616,"ণি":777,"দ।":351,"ণা":1071,"তক":615,"ণত":463,"ঢা":467,"ডà§":286,"ডো":228,"ডে":1416,"ত।":1976,"থা":4072,"থি":1849,"ন।":3725,"থন":275,"তে":6636,"তৈ":476,"থব":514,"তো":511,"তà§":11770,"থম":1047,"দক":1036,"তা":9664,"তি":11706,"তৃ":563,"তী":2327,"তà§":1609,"তথ":334,"তত":465,"তন":1191,"ণে":1383,"ণà§":713,"তম":1948,"তব":730,"তল":215,"তর":2978,"নট":466,"ধা":3367,"নত":805,"নদ":832,"ধি":1235,"ধী":706,"ধà§":525,"দো":593,"দà§":6813,"ধন":270,"দে":7678,"দৈ":238,"নক":807,"নগ":632,"ধর":1424,"নও":200,"দস":274,"দশ":433,"দূ":217,"ধত":234,"দৃ":215,"দী":1553,"দà§":1592,"দা":3394,"দি":3189,"থে":3273,"দন":421,"ছে ":1536,"জন ":2036,"দল":773,"দর":900,"থà§":402,"হন":525,"সে":5003,"সী":651,"সà§":1284,"সূ":479,"হণ":341,"হত":965,"সৃ":307,"সা":9422,"সি":3847,"সহ":471,"হচ":543,"হল":1575,"সà§":12917,"হম":242,"হর":1722,"হয":5736,"সো":556,"হে":854,"হৃ":361,"়।":3879,"হà§":683,"হি":3582,"হী":535,"হা":4314,"হà§":705,"হো":256,"়ক":477,"়ত":462,"়ন":1084,"া।":1709,"়ম":211,"়র":208,"ি।":908,"়à§":308,"়ী":503,"াং":3360,"ছৠ":408,"াà¦":1482,"়ি":1178,"়া":8455,"়ো":436,"াই":3512,"়ে":7915,"াউ":581,"াও":962,"াক":5532,"়à§":205,"াচ":1101,"াছ":570,"াজ":4489,"াখ":950,"াগ":2341,"াঙ":862,"াথ":1203,"াত":6156,"াণ":1382,"ী।":871,"াড":1549,"িং":1140,"াঠ":366,"াট":1770,"াঞ":296,"াভ":908,"াব":4643,"াফ":353,"াপ":2739,"ান":18665,"িউ":1255,"াধ":2334,"াদ":5045,"িক":11087,"িখ":922,"াল":10227,"িও":631,"ার":30756,"িà¦":231,"াম":7273,"ায":8979,"িছ":516,"িজ":2552,"াস":5883,"িচ":1754,"াহ":2942,"িগ":546,"াশ":1723,"াষ":6041,"িট":1865,"িড":639,"িত":12239,"িণ":858,"িদ":2453,"িন":8978,"িধ":463,"িপ":1269,"িব":2698,"িফ":329,"িম":2790,"িভ":1825,"াৎ":271,"িয":6762,"ির":6023,"িল":5504,"ীক":817,"িশ":3794,"ীগ":209,"িষ":2506,"িস":4278,"িহ":941,"à§à¦‡":664,"ীদ":382,"ীত":1782,"ীপ":505,"ীন":1798,"ীয":4089,"িৎ":317,"ীম":347,"ীব":842,"ীল":338,"ীর":2436,"à§à¦—":541,"à§à¦•":2294,"à§à¦–":360,"à§à¦œ":362,"à§à¦Ÿ":930,"রà§":19499,"রো":2284,"রে":15121,"রূ":464,"রà§":2561,"রী":3578,"রা":17630,"রি":11429,"রস":1549,"রহ":932,"রশ":407,"রল":454,"লয":931,"লম":504,"লব":520,"লন":1184,"লত":1046,"লট":257,"লচ":904,"লগ":240,"লক":1697,"লà§":3376,"লো":3053,"লে":9966,"লী":1486,"লà§":671,"লা":9534,"লি":6096,"লস":278,"শক":649,"শব":592,"শন":1507,"ষক":239,"শর":329,"স।":308,"শত":558,"শট":242,"শে":3570,"ষম":327,"শà§":3936,"শো":350,"ষয":380,"সক":584,"শহ":1521,"শà§":848,"শী":865,"সং":3293,"শি":3267,"শা":2252,"ষণ":732,"সন":1003,"ষে":1161,"সফ":273,"সব":1306,"সভ":427,"সম":3977,"ষà§":5176,"সর":1993,"সল":739,"হক":226,"সঙ":453,"ষা":5502,"ষি":1250,"সত":301,"সদ":563,"ৎস":482,"ৎপ":325,"à§à¦¶":719,"à§à¦¸":2033,"à§à¦·":4401,"à§à¦®":4357,"à§à¦­":1012,"à§à¦°":30653,"à§à¦¯":25184,"à§à¦²":2305,"à§à¦¤":14406,"à§à¦£":1498,"à§à¦¡":2937,"à§à¦ ":1626,"à§à¦Ÿ":7869,"à§à¦ž":1750,"à§à¦¬":10276,"à§à¦ª":3453,"à§à¦¨":2407,"à§à¦§":2336,"à§à¦¦":4738,"à§à¦¥":5602,"ৌল":231,"ৌর":302,"à§à¦›":784,"à§à¦œ":1742,"à§à¦š":4084,"à§à¦—":2822,"à§à¦˜":304,"à§à¦•":3885,"োয":771,"োর":1773,"োল":1204,"োব":621,"োভ":229,"োম":910,"োষ":455,"োস":598,"োহ":395,"োà¦":365,"োজ":493,"োড":367,"োট":842,"োদ":303,"োত":491,"োপ":763,"োন":1801,"োধ":294,"োক":1031,"োচ":456,"োগ":1362,"গঠি":273,"খান":1047,"গঠন":344,"ৈর":614,"েশ":5180,"েল":4085,"ের":25063,"েয":1289,"েম":1903,"েভ":263,"েব":2130,"েপ":618,"েন":8060,"ৈত":343,"েহ":255,"েষ":1422,"েস":1309,"েও":994,"েখ":1729,"েক":4154,"েই":1429,"েউ":248,"েট":1435,"েড":786,"েত":1719,"েদ":505,"েগ":411,"েছ":3105,"েজ":2937,"ে।":4052,"ৃথ":340,"ৃত":2338,"ৃহ":465,"ৃষ":756,"ূত":331,"ূপ":414,"ূম":517,"ূর":2094,"ূল":1187,"ূহ":378,"à§à¦¤":1038,"à§à¦¡":349,"à§à¦¨":1462,"à§à¦¦":1447,"à§à¦¬":643,"à§à¦ª":599,"à§à¦¯":1183,"à§à¦°":4508,"à§à¦­":198,"à§à¦®":876,"à§à¦²":3399,"à§à¦¸":1262,"à§à¦·":874,"à§à¦¶":279,"কেট":226,"কেন":738,"কের":985,"কোন":942,"কà§à¦¤":2888,"কà§à¦Ÿ":455,"কà§à¦·":3987,"কà§à¦²":430,"কà§à¦°":1781,"কà§à¦¯":806,"কà§à¦¸":679,"কিছ":467,"কাহ":230,"কিন":953,"কাব":216,"কায":382,"কার":4553,"কাল":1045,"কাশ":812,"কিল":233,"কিস":331,"কà§à¦²":267,"কà§à¦°":331,"কà§à¦®":261,"কৃত":1025,"কৃষ":259,"কে।":511,"কেই":224,"৮০":213,"৭১":230,"৯৯":610,"৯০":387,"৯৫":459,"৯৬":482,"৯৭":685,"৯৮":501,"৯১":321,"৯২":343,"৯৩":359,"৯৪":434,"৮৯":204,"৮৮":200,"গà§à¦°":367,"গà§à¦²":1495,"৩০":296,"২৪":220,"গীত":413,"৫০":212,"গান":542,"গার":291,"গায":230,"১৬":326,"১৫":332,"১৮":996,"১৭":415,"১২":292,"১১":238,"১৪":244,"১৩":256,"১৯":3860,"২০":1142,"০০":1455,"১০":452,"গসà§":210,"à§à¦°à¥¤":452,"à§à¦¯à¦•":735,"à§à¦®à¦¾":901,"à§à¦®à§€":308,"à§à¦®à¦¿":327,"à§à¦¯à¦¨":687,"à§à¦®à§‡":324,"à§à¦¯à¦¤":807,"à§à¦¯à¦®":709,"à§à¦¯à¦¬":1629,"à§à¦°à¦š":617,"à§à¦¯à¦¸":234,"à§à¦°à¦•":1414,"à§à¦¯à§":488,"à§à¦¯à¦¿":244,"à§à¦¯à¦¾":10276,"à§à¦°à¦œ":405,"গবে":354,"à§à¦¬à¦¤":419,"à§à¦¬à¦¨":491,"à§à¦¬à¦ª":359,"à§à¦¬à¦¬":695,"à§à¦¬à¦°":1021,"à§à¦¬à¦¾":1809,"à§à¦¯à¥¤":373,"à§à¦¬à¦¿":841,"à§à¦¬à§€":559,"à§à¦¬à§‡":1025,"à§à¦¬à§‹":213,"à§à¦®à¦—":199,"à§à¦­à¦¾":239,"à§à¦²à¦¾":616,"à§à¦²à¦¿":367,"গরে":240,"à§à¦²à§‡":524,"à§à¦°à¦¬":578,"à§à¦¯à§‹":301,"à§à¦°à¦®":577,"à§à¦°à¦­":304,"à§à¦°à¦¯":464,"à§à¦°à¦¤":2154,"à§à¦°à¦£":361,"à§à¦°à¦¦":1113,"à§à¦°à¦¥":1019,"à§à¦°à¦¨":758,"à§à¦¯à§‡":2625,"à§à¦°à¦§":869,"à§à¦°à¦ª":383,"à§à¦°à¦¾":4952,"à§à¦°à¦¿":3967,"à§à¦°à§€":1344,"à§à¦°à§":685,"খà§à¦¯":1786,"à§à¦°à¦¶":293,"খà§à¦°":509,"à§à¦°à¦¸":463,"à§à¦°à¦¹":735,"à§à¦°à§‹":780,"à§à¦°à§‡":2630,"à§à¦·à¦£":271,"গণি":276,"à§à¦·à¦¿":1061,"à§à¦·à¦¾":679,"à§à¦·à§‡":819,"à§à¦·à¦®":296,"গড়":257,"à§à¦¶à¦¨":406,"খেল":564,"à§à¦¸à§‡":379,"à§à¦¸à¦¿":438,"গত ":349,"খা ":665,"কজন":1518,"খে ":204,"। ":19077,"কলে":496,"কলà§":282,"করত":378,"করণ":527,"করে":4422,"করা":2433,"কলক":407,"করà§":871,"কমà§":406,"কাজ":532,"কাছ":278,"কাত":521,"কান":552,"গে ":596,"গর ":415,"কটি":5240,"ঃ ":340,"ং ":5055,"কবি":409,"ই ":6830,"উ ":312,"কথা":468,"ও ":6690,"ঠ":847,"চ ":769,"খ ":354,"গ ":1629,"ক ":11586,"ত ":12607,"ণ ":3613,"ঠ ":296,"ড ":1706,"ট ":3372,"জ ":1903,"ব ":3202,"ফ ":611,"প ":1284,"ন ":20715,"ধ ":808,"দ ":2233,"থ ":1007,"কা ":1480,"ল ":8450,"ওয়":1861,"ভ ":587,"ম ":6723,"কি ":282,"য ":4809,"র ":54334,"হ ":1019,"় ":12855,"শ ":3191,"খন ":279,"কে ":5302,"ষ ":1259,"স ":4530,"া ":26160,"ী ":7142,"ৠ":2897,"ি ":25805,"ো ":2835,"ৠ":1910,"কল ":227,"ে ":37314,"তর ":887,"ডার":368,"ডিস":334,"ডিয":319,"তন ":462,"ণে ":593,"তম ":977,"ড়া":1089,"ড়ি":682,"ড়ে":451,"১৯০":231,"১৯১":267,"তৠ":433,"১৯৪":366,"১৯৫":361,"১৯২":273,"১৯৩":285,"১৯৮":393,"১৯৯":498,"১৯৬":398,"১৯৭":613,"তী ":298,"তে ":4456,"ডের":406,"ডেন":243,"তি ":2372,"তা ":2207,"ঠান":465,"ঠিত":686,"২০০":842,"দ। ":304,"ণা ":499,"টাব":261,"টান":241,"টিক":566,"টার":1384,"টাল":237,"টিত":421,"টিন":246,"টিশ":274,"টির":990,"টাই":256,"ডে ":223,"ণত ":291,"টোব":204,"টà§à¦°":2092,"টà§à¦¯":377,"টà§à¦Ÿ":258,"টেম":345,"টের":443,"টেল":237,"ঞান":1488,"০০০":220,"ত। ":1457,"ডি ":251,"ঞà§à¦š":1271,"ঞà§à¦œ":764,"টবল":369,"ড় ":618,"টি ":10700,"জà§à¦ž":1748,"টা ":567,"জà§à¦¯":1590,"টে ":252,"ঠন ":205,"à¦à¦¾à¦¯":280,"ঠা ":199,"ছোট":215,"জয়":273,"টন ":232,"জান":513,"জাত":1408,"জার":1034,"জিক":202,"জিত":234,"জীব":714,"জà§à¦²":281,"জà§à¦¨":236,"জেন":272,"জেল":1183,"জের":493,"চà§à¦š":1435,"চà§à¦›":780,"ছবি":323,"ছিল":2594,"ছাড":436,"জধা":329,"ছেন":778,"জনপ":400,"জনী":266,"জনà§":2129,"ছে।":1579,"জি ":2005,"৭১ ":202,"ঙà§à¦—":2125,"ঙà§à¦•":292,"জা ":285,"চনা":453,"চনà§":198,"জে ":350,"চলি":385,"চলে":500,"চলচ":862,"চরি":276,"চিত":2401,"চাল":768,"চার":933,"চিম":924,"The":204,"চেয":454,"চীন":632,"পৃথ":340,"মক ":218,"পূর":1563,"পà§à¦°":1770,"পà§à¦¤":228,"বংশ":207,"পের":502,"পেন":312,"পশà§":957,"পাশ":229,"পিত":257,"পাক":348,"পান":707,"পিউ":331,"পাত":414,"পাদ":424,"পাধ":205,"পার":1335,"পাল":505,"পায":279,"পাও":244,"পরে":327,"পরà§":1509,"বী ":353,"পরি":2733,"য। ":281,"পরব":233,"বে ":2767,"পনà§":745,"পদা":548,"পদà§":310,"বা ":2607,"বি ":661,"পঞà§":305,"বর ":1036,"পড়":268,"বল ":484,"পতà§":595,"পতি":403,"বন ":272,"নà§à¦¤":3819,"নà§à¦¡":1774,"নà§à¦Ÿ":1141,"নà§à¦¸":716,"নà§à¦¥":421,"নà§à¦¦":2863,"নà§à¦§":523,"নà§à¦¨":1373,"নà§à¦®":1138,"নà§à¦¯":3823,"ম। ":348,"নেক":580,"নেত":451,"নের":3172,"নৈত":249,"নোব":216,"নà§à¦¯":400,"নà§à¦·":645,"নà§à¦¸":424,"নিস":305,"নীত":622,"নীয":704,"নীর":259,"নিত":220,"নিজ":306,"নির":1511,"নিয":1391,"নিব":223,"নিম":242,"নাথ":204,"নী।":283,"নাট":333,"নিক":1359,"নাল":376,"নার":884,"নায":377,"নাম":2830,"নিউ":358,"নান":283,"নাই":230,"না।":209,"পে ":392,"বং ":4190,"নয়":574,"ধà§à¦¯":2527,"পি ":200,"নভে":235,"ধà§à¦¬":397,"নপà§":410,"নদী":606,"ধের":201,"নতা":305,"নটি":399,"al ":424,"পর ":878,"ধার":1388,"ধিক":529,"ধান":1581,"ধà§à¦¨":236,"ধীন":509,"and":351,"an ":333,"নগর":339,"ধরà§":622,"নকা":301,"দà§à¦¦":285,"দà§à¦°":1656,"দà§à¦¯":1705,"দà§à¦¬":1242,"দà§à¦­":351,"দà§à¦§":1443,"ধরণ":208,"ধরন":300,"দোল":241,"ধতি":214,"দেখ":386,"দেশ":3340,"দেব":447,"দের":2400,"দেয":262,"দীর":527,"দà§à¦‡":388,"দà§à¦°":229,"পক ":215,"ati":407,"দরà§":464,"০০ ":448,"দলে":293,"নী ":1022,"দশক":219,"নে ":2592,"নো ":750,"দসà§":252,"দায":342,"দান":683,"দিক":556,"দার":1068,"দিয":527,"দিন":460,"নৠ":258,"থিত":1239,"থিব":279,"থান":907,"থার":216,"থাপ":342,"থের":228,"থেক":1914,"থà§à¦¯":318,"নি ":3145,"না ":2078,"তà§à¦¤":2681,"তà§à¦¬":1563,"তà§à¦°":5371,"তà§à¦¯":1749,"তà§à¦®":200,"১০ ":255,"ধি ":218,"দকà§":833,"ধে ":246,"থাক":949,"তাà¦":801,"তি।":212,"তাক":433,"তা।":273,"তিত":496,"তিন":2758,"তিব":469,"তিয":323,"তির":977,"তিহ":514,"তিষ":915,"তিস":234,"তাদ":379,"তান":831,"তাব":456,"তায":251,"তার":2316,"তাল":484,"তিক":1444,"তà§à¦²":199,"তà§à¦°":359,"তীর":252,"তীয":1406,"তৃত":307,"তের":1547,"তৈর":454,"থবি":319,"তরà§":666,"দী ":536,"তরা":670,"তরে":281,"দৠ":322,"ণà§à¦¡":418,"দা ":406,"তমা":738,"দি ":552,"ণà§à¦¯":221,"তবে":327,"দে ":324,"ণিত":430,"দর ":218,"তনà§":360,"ণের":721,"তথà§":200,"ততà§":385,"থা ":838,"ন। ":2670,"থে ":916,"তকে":220,"ঢাক":444,"থম ":866,"রকা":1796,"লত ":375,"রচন":240,"রকে":273,"রকà§":515,"লন ":417,"য়ক":403,"য়ন":1056,"য়ত":420,"রজা":344,"য়র":205,"য়ম":204,"য়ো":399,"য়ে":7441,"য়à§":262,"য়ী":440,"য়ি":464,"য়া":7334,"রচল":394,"রচি":290,"য়।":3790,"যà§à¦—":221,"যà§à¦•":999,"যà§à¦¦":441,"যাক":626," । ":1068,"যাট":233,"যাত":1108,"যাদ":399,"যান":2143,"যাপ":621,"যাব":244,"যাম":399,"যাল":1187,"যার":1249,"যায":1288,"যিক":236,"যাস":933,"রটি":492,"যিন":243,"রথম":978,"রতà§":227,"রতে":1478,"রতি":1816,"রতী":622,"রণে":668,"রণা":328,"রণত":263,"রপত":250,"ion":522,"রনà§":434,"রনা":231,"রনে":444,"যেম":243,"যের":1196,"রধা":866,"রদে":667,"রদা":349,"রভা":262,"লি ":1224,"রবা":369,"রবি":326,"লা ":2174,"রবর":362,"যোগ":846,"রমা":242,"লী ":505,"রয়":679,"লে ":3437,"রহণ":319,"রসà§":785,"লো ":880,"লৠ":250,"রাই":307,"রাক":460,"রাচ":539,"রাজ":2647,"রাখ":212,"রাণ":570,"রাপ":301,"রাব":230,"রান":1015,"রিক":1905,"রার":655,"রাম":984,"রায":1265,"রিজ":258,"রাস":948,"রিচ":1336,"রাহ":202,"রাশ":201,"রাষ":1115,"রিত":670,"রিট":394,"রিম":358,"রিব":807,"রিন":213,"রিস":680,"রিয":1449,"রিল":310,"রীক":256,"রীয":402,"রীর":289,"রà§à¦¯":235,"রà§à¦¤":425,"রূপ":403,"রে।":1025,"রেন":1720,"রের":3004,"রেল":310,"রেশ":198,"রেছ":854,"রেজ":2190,"রেট":233,"রেক":207,"রেস":215,"রোম":235,"রোপ":366,"রোগ":263,"রà§à¦£":1236,"রà§à¦¤":1912,"রà§à¦¡":513,"রà§à¦Ÿ":567,"রà§à¦¬":2473,"রà§à¦§":342,"রà§à¦¨":427,"রà§à¦¥":1669,"রà§à¦¦":480,"রà§à¦²":198,"রà§à¦¯":1584,"রà§à¦®":2023,"রà§à¦­":416,"রà§à¦¸":595,"রà§à¦·":411,"রà§à¦¶":716,"রà§à¦š":340,"রà§à¦œ":782,"রà§à¦•":1521,"রà§à¦˜":284,"রà§à¦—":673,"লকা":488,"লচà§":855,"লকà§":395," Ma":238,"he ":468,"লতে":397," Co":216,"লটি":222,"লনা":313," Th":233,"লয়":917,"লমà§":237,"লাই":481,"লাক":628,"লান":256,"লাদ":1657,"লাস":227,"লিখ":198,"লিক":746,"লার":1528,"লাভ":487,"লায":309,"লাম":488,"লিপ":223,"লিন":254,"লিত":865," of":374," an":214,"লে।":338,"লেও":259,"লের":1832,"লেন":1829,"লেজ":413,"লেখ":829,"লেক":269,"লীন":213,"লিম":293,"লিয":577,"লির":379,"ing":263,"লীয":377,"in ":228,"লà§à¦ª":990,"লà§à¦¡":206,"লà§à¦¯":834,"লà§à¦²":756,"স। ":246,"লোচ":236,"লোম":202,"লোর":312,"লোয":202,"লোক":706,"শন ":671,"hum":238," th":574,"মত ":200,"মন ":560,"ফরা":505,"র। ":1082,"পà§à¦¤":664,"পà§à¦¯":216,"পà§à¦°":13060,"পà§à¦²":304,"পà§à¦Ÿ":290,"পৌর":203,"ভা ":303,"বনি":284,"er ":443,"বনে":288,"বনà§":515,"es ":375,"ফেব":217,"বচে":366,"বঙà§":528,"ent":254,"বছর":385,"ফà§à¦²":203,"বড়":273,"ফà§à¦Ÿ":478,"ফার":283,"বলে":833,"মে ":1672,"বসà§":2112,"বহà§":351,"বহৃ":337,"বহা":619,"ফোর":229,"ববি":644,"বপূ":262,"মি ":517,"ফà§à¦°":706,"মা ":428,"বয়":292,"মী ":371,"বলা":524,"বলত":367,"বরà§":2305,"বৃহ":359,"বেক":203,"বেল":404,"বের":931,"বেশ":937,"বেষ":386,"বোà¦":365,"য় ":12199,"বাই":313,"বাং":2410,"বান":279,"বাদ":861,"বাধ":440,"বার":1699,"বাম":198,"বায":216,"বাঙ":403,"বাজ":214,"বাচ":199,"বিত":730,"বিদ":1525,"বিধ":272,"বিন":362,"বিপ":215,"বিব":278,"বিভ":1099,"বিয":248,"বির":389,"বাল":212,"বিখ":512,"বিক":671,"বাহ":872,"বিচ":203,"বাস":937,"বিজ":1409,"রও ":300,"বীর":327,"বীপ":377,"বিষ":640,"বিস":375,"বিশ":2344,"ভাই":208,"রম ":217,"মগà§":268,"রন ":224,"যে ":1841,"রত ":384,"রণ ":1177,"ল। ":672,"বà§à¦°":1160,"বà§à¦¯":3166,"বà§à¦¦":1107,"যা ":1724,"রা ":4296,"রি ":1578,"ভà§à¦¯":238,"মবঙ":362,"মনà§":664,"ভেম":223,"মধà§":1555,"মতা":307,"ভূম":496,"ভà§à¦•":240,"ভাগ":718,"ভার":2311,"ভাব":1222,"ভাষ":4828,"ভিয":254,"ভিন":1250,"ভিত":372,"ed ":213,"মাই":303,"মিন":199,"মিত":518,"মিট":419,"মাস":327,"মিক":501,"মাল":710,"মার":1752,"মান":3429,"মাত":575,"মাধ":574,"মাণ":369,"মাজ":400,"রো ":310,"মহা":801,"মসà§":221,"রে ":4008,"যকà§":405,"রৠ":494,"রী ":1730,"ময়":958,"মৌল":207,"মà§à¦°":226,"মà§à¦¯":700,"মà§à¦­":243,"মà§à¦®":363,"মà§à¦¬":1398,"মà§à¦ª":1633,"যমে":422,"যনà§":880,"যবস":455,"যবহ":926,"মেন":347,"মের":1210,"মিশ":280,"মিল":414,"মীয":198,"মà§à¦–":293,"মà§à¦•":494,"মà§à¦¦":244,"মà§à¦¸":261,"মূল":1049,"লক ":453,"মূহ":371,"মৃত":392,"যতম":572,"সলা":347,"re ":199,"হচà§":536,"সাং":239,"সাম":709,"সায":256,"সার":942,"সাল":2557,"সিক":500,"সাধ":708,"সান":242,"সাব":458,"সিত":294,"সাহ":727,"সাই":326,"সাথ":746,"সাগ":439,"সীম":247,"সিন":266,"সির":308,"়ক ":242,"সূর":236,"সতà§":211,"সদস":228,"ষেত":374,"সনà§":338,"ষের":340,"সবচ":368,"সবা":223,"ষà§à¦•":275,"ষà§à¦£":260,"ষà§à¦Ÿ":2671,"সভা":316,"ষà§à¦ ":1554,"সময":711,"ষà§à¦¯":236,"সমা":613,"সমূ":375,"সমà§":1299,"সরক":840,"সরà§":566,"়ন ":433,"হিন":860,"হিস":1250,"হাদ":208,"হান":202,"হাম":272,"হায":221,"হার":1136,"হাস":659,"হিত":775,"হাজ":205,"হৃত":340,"সেই":206,"সেম":294,"সেব":1065,"সেপ":226,"সেন":564,"হতে":419,"সৃষ":262,"হতà§":332,"সের":1064,"হয়":5615,"া। ":1439,"সà§à¦•":1611,"সà§à¦Ÿ":2276,"সà§à¦¤":2428,"সà§à¦¥":3469,"সà§à¦ª":563,"সà§à¦¬":1109,"সà§à¦¯":561,"হলে":402,"হলো":289,"হরে":416,"াগ ":429,"াও ":315,"াক ":329,"়া ":2568,"়ি ":218,"হà§à¦¯":294,"হের":399,"াই ":889,"়ে ":3374,"়ী ":420,"ি। ":726,"াল ":1724,"িও ":348,"ার ":13003,"াম ":1773,"াভ ":401," ১০":339,"াব ":352," ৩০":204," ১২":242,"়নে":202," ১৩":220," ১৪":201," ১৫":282," ১৬":286," ১৭":371," ১৮":940," ১৯":3783,"াহ ":308," ২০":1046,"াস ":1335,"াশ ":330,"িক ":5573,"িং ":561,"ী। ":724,"াট ":241,"াজ ":536,"াপ ":211,"ান ":5410,"াদ ":431,"াণ ":268,"াত ":1005," à¦à¦‡":3270," উৎ":560,"ng ":235,"শে ":909," উদ":684," উন":258," উত":959," উচ":373," উল":389," উপ":2191," ইত":669," আস":453," আয":322," অà§":835," আম":509," আর":1370," আল":1083," আধ":289," আদ":422," আন":905," ইউ":775," আফ":294," আব":706," উই":304," ইস":476," ইল":209,"nd ":283," ইর":274," ইয":287," ইন":731," অঙ":286," অক":379," অত":316," অঞ":596," অপ":429," অফ":358," অব":2124," অথ":198," আই":665," অধ":751," অন":3315," অল":206," আক":517," অভ":831," অর":1146," আছ":551," আগ":504," অস":564," ইং":2299,"শি ":343," অং":591,"শী ":336,"ষণ ":226," জà§":458," জা":2134," জি":334," জী":551," জà§":646," জে":1308," জো":204," জল":229," ছো":231," ছি":1929," ছা":399," জন":3064," ছব":300," চা":848," চি":755," চে":275," চল":1081," চর":388," ২ ":205," ১ ":302," গà§":1853," গো":546," গে":282," গà§":745," গি":203," ঘট":372," গা":986," গব":363," খà§":684," খে":607," গণ":589," গঠ":432," খা":596," খà§":226," কা":3277," কি":1369," কà§":537," কৃ":271," কে":1060," কো":1627," কà§":2305," কন":254," কথ":522," কব":408," কর":8000," কম":804," কল":1072," ওয":520," à¦à¦¸":383," à¦à¦¶":223," à¦à¦°":2835," à¦à¦²":645," à¦à¦¤":234," à¦à¦›":286," à¦à¦–":588," à¦à¦•":8988," à¦à¦®":399," à¦à¦¬":4212," à¦à¦ª":273," à¦à¦¨":256," à¦à¦¦":302," à¦à¦Ÿ":2135," ফে":435," বন":336," ফা":463," বড":264," ফি":389," ফà§":675," বছ":364," পো":225," পৌ":257," পà§":11714," ফর":609," ফল":347," বে":1555," বো":669," বৈ":440," বি":7573," বা":7403," বৃ":550," বà§":368," বস":531," বহ":441," ফà§":592," বল":1749," বর":1400," মধ":1533," মন":645," ভে":259," ভূ":442,"শকà§":282," মত":465," ভা":7371," ভি":710,"শকে":227," বà§":3507," মà§":464," মৌ":257," মো":583," মে":1083," মৃ":318," মূ":802," মà§":1346," মি":1118," মা":4397,"সন ":262,"ষে ":226," মহ":923," মর":212," ধা":649," নদ":623," দে":2058," দৈ":202," দà§":1551," ধর":1182," নগ":253," না":3576," নি":3394," নী":245," নে":783," ধà§":292," নভ":274,"of ":368," পঞ":266," পড":228," পদ":913," পত":409," নà§":272," নো":287,"সব ":353," পà§":1605," পা":3594," পি":495," পে":490," বই":234," পূ":997," পৃ":428," পর":4825," পশ":1001," তব":328," তত":253," তথ":332," তি":2653," তা":3867," তà§":325," দক":818," তà§":257," তৈ":474,"ষা ":1614," তে":303," থা":1028," দল":654," দর":261," থে":1958," দি":1421," দা":849," দà§":810," দশ":293," টা":272," টি":345," টে":546," টà§":234," ডি":734," ডা":459," ডে":331," ঢা":461," হয":5669," সà§":3980," সো":334," হল":1487," সি":1221," সা":6870," সহ":408," হচ":537," সে":1681," হত":623," সৃ":267," সূ":402," সà§":981," সী":216," সব":807," সম":3235," সন":329," সর":1532," সঙ":383," সত":207," সদ":329," হো":212," হà§":276," হা":1267," হি":1859," হে":312," শক":251," শা":1199," শি":1188," সং":2886," শà§":645," শহ":1500," সক":314," শে":357," শà§":581," শত":509," শব":479,"on ":532," রি":295," রা":3902," রে":556," রà§":295," রূ":298," রো":452," রক":302," যা":2826," রচ":547," যà§":1137," যি":261," রয":403," রব":219," যো":242," যৌ":226," যে":1328," লি":670," লা":967," লে":811," লো":608," লক":345,"সৠ":266," ও ":4589,"শাখ":311,"শাস":697,"শিত":339,"হর ":728,"শাল":206,"শিক":625,"le ":226,"শহর":1422,"সে ":658,"হন ":263,"সী ":240," ঠ":753,"হণ ":261,"সি ":877,"শবà§":493,"সা ":275,"শনে":210,"শনা":303,"শতক":230,"শতা":251,"শটি":231,"mb ":231,"ষাত":229,"ষিণ":767,"ষায":1936,"ষার":498,"ষাব":214," স ":339,"ষা।":354,"সঙà§":452," র ":316,"়। ":2782,"ষয়":378,"হী ":223,"শà§à¦°":601,"শà§à¦¯":362,"শà§à¦¬":1645,"ষমত":242,"শà§à¦š":1032,"শেষ":751,"শের":1660,"ষণা":351,"শà§à¦°":445,"সংব":214,"হল ":715,"সংস":804,"শিয":443,"শিল":531,"সংক":417,"সংগ":424,"সংখ":731,"সংঘ":234,"শিষ":385,"à§à¦¸à¦¾":325,"à§à¦·à§":372,"à§à¦¸à¦²":207,"ূমি":444,"ূলক":257,"ূলত":394,"ে। ":2956,"ূরà§":1748,"à§à¦•à§":1812,"à§à¦Ÿà¦¿":275,"à§à¦¡à¦¼":269,"à§à¦Ÿà¦¬":355,"à§à¦ªà§":205,"à§à¦¨à¦¿":328,"à§à¦¦à§":1024,"à§à¦¤à§":534,"à§à¦²à¦¾":360,"à§à¦²à¦¿":957,"à§à¦²à§‹":677,"à§à¦°à¦¸":488,"à§à¦°à¦¾":502,"à§à¦°à§":902,"à§à¦°à§‡":214,"à§à¦°à§":475,"à§à¦¯à¦¼":999,"à§à¦®à¦¾":378,"ৃহত":230,"েও ":751,"েক ":677,"েজ ":442,"েট ":476,"েড ":233,"েত ":298,"েন ":2383,"ৃতি":667,"ৃথি":275,"ৃতà§":530,"ৃষà§":660,"েই ":990,"াচী":479,"াঙা":392,"াঙà§":430,"াজে":291,"াজà§":1187,"াজা":637,"াজি":350,"াজধ":330,"াজন":429,"াছে":290,"াখা":510,"াগর":531,"ীন ":996,"াকা":1674,"াকৃ":214,"াকি":417,"াকà§":624,"াকে":1517,"াগা":228,"াগà§":251,"াগে":340,"ীত ":407,"াওয":538,"à§à¦‡ ":270,"াপে":239,"াপি":216,"াপা":332,"াপà§":438,"াবল":223,"াবà§":798,"াবে":1282,"াবি":607,"াবা":550,"ামক":357,"ার।":391,"াথে":795,"াদা":503,"াদি":433,"াদী":233,"াদে":2459,"াদà§":228,"ানক":226,"াধি":292,"াধী":407,"াধা":767,"িউট":438,"াধà§":701,"ানম":216,"ানব":285,"ানা":935,"ানি":1277,"ানী":1247,"ানà§":818,"ানে":2964,"ানà§":3257,"ানো":483,"াপক":268,"াপন":257,"াণী":233,"াণি":288,"াতী":444,"াতি":1024,"াতা":904,"াতন":242,"াণে":256,"ান।":554,"াতà§":1070,"াতে":889,"ীর ":1512,"াটি":687,"াঞà§":296,"াটà§":216,"াড়":1116,"িন ":1620,"িদ ":390,"িত ":5217,"িণ ":578,"িজ ":216,"়ের":988,"়েল":232,"াইল":290,"িস ":391,"়েন":224,"়েত":314,"়েছ":1789,"িশ ":479,"াইক":267,"াইন":398,"াইট":347,"়িত":393,"াà¦à¦°":695,"াংশ":369,"িল ":1310,"াংল":2405,"়াড":214,"াৎ ":225,"়াল":346,"়াম":294,"ির ":3233,"়ার":2357,"়ান":720,"়াত":233,"িম ":623,"িহা":558,"ংশ ":608,"িসà§":1568,"িসা":395,"িসি":199,"িসে":1302,"ূল ":317,"িরি":292,"িরা":288,"িরà§":1412,"িলা":304,"িলি":409,"িলে":1534,"িলà§":792,"িলো":314,"িশন":236,"িষà§":1946,"িশি":298,"িশে":548,"িষয":343,"িশà§":1581,"ীবন":365,"ীমা":216,"ীয়":4081,"ীরà§":354,"ীরে":207,"ৃত ":857,"umb":265,"ীতি":604,"ীতে":555,"ীদে":242,"ীনত":266,"াহা":469,"িচা":733,"াহী":229,"িচি":688,"াহি":1315,"াসà§":708,"াসে":596,"াসি":1223,"াসা":419,"াসী":261,"াষà§":1171,"াসন":338,"াষা":4713,"াশি":537,"à§à¦° ":1172,"িটি":780,"িটা":629,"িজà§":1437,"িজে":324,"িছà§":475,"ালক":364,"ালয":874,"ালন":396,"ারী":1146,"ারি":1557,"ারা":1659,"ারà§":5067,"ারে":2345,"ায়":8671,"ারক":300,"ারন":347,"ারণ":1235,"ারত":1942,"ামি":443,"ামা":748,"ামà§":609,"ামে":1488,"ামী":317,"াশন":211,"িখà§":507,"à§à¦¨ ":561,"িকা":1858,"িকি":198,"িকে":1068,"িকà§":1074,"ালী":661,"ালা":739,"ালি":1410,"ালো":249,"ালে":2817,"ালà§":297,"িপি":219,"িপà§":331,"িনে":565,"িনা":408,"িনি":2917,"িনী":642,"িনয":241,"িনà§":1908,"িমি":290,"িমà§":258,"িমে":259,"িমা":636,"িল।":473,"িয়":6354,"িবা":741,"িবী":293,"িবি":361,"িবে":347,"িবর":417,"িভি":889,"িভা":580,"িমব":382,"tio":423,"thu":236,"িদ।":233,"à§à¦² ":613,"িডি":222,"িত।":1272,"িদà§":1210,"িদে":199,"িনট":209,"িধা":259,"ter":251,"িতী":319,"িতি":339,"িতা":574,"িতে":1209,"িতà§":2907,"the":401,"à¦à¦° ":674,"à§à¦° ":2468,"à§à¦¯ ":4783,"োà¦à¦¾":363,"à§à¦® ":1055,"à§à¦¬ ":1443,"à§à¦ª ":322,"োগà§":438,"à§à¦¨ ":1119,"à§à¦¸ ":725,"à§à¦· ":348,"ংশে":259,"ংসদ":201,"à§à¦Ÿ ":1640,"à§à¦œ ":260,"à§à¦š ":547,"ংলা":2409,"ংরে":2127,"à§à¦¦ ":444,"োকে":215,"à§à¦§ ":572,"à§à¦¤ ":2310,"ংবা":245,"à§à¦¥ ":613,"à§à¦£ ":661,"à§à¦  ":206,"à§à¦¡ ":1005,"à§à¦• ":455,"à§à¦— ":569,"ংকà§":310,"ংখà§":741,"à§à¦žà¦¾":1605,"à§à¦œà¦¾":489,"à§à¦œà¦¿":208,"à§à¦œà§‡":211,"à§à¦œà¦¨":340,"à§à¦›à§‡":599,"à§à¦¤à¥¤":316,"à§à¦¡à¦¿":303,"à§à¦¡à¦¾":322,"à§à¦¡à§‡":556,"à§à¦ à¦¾":754,"à§à¦ à¦¿":421,"à§à¦Ÿà¦¿":1218,"à§à¦Ÿà¦¾":1173,"à§à¦Ÿà§‹":370,"à§à¦Ÿà§":1738,"à§à¦Ÿà§‡":710,"à§à¦¦à§":1049,"আর ":224,"à§à¦¦à§‹":445,"à§à¦¦à¦¾":237,"à§à¦¦à¦¿":541,"à§à¦¦à§€":307,"à§à¦¦à§‡":669,"à§à¦¦à§":407,"à§à¦§à¦¤":232,"à§à¦¦à¦°":217,"à§à¦¥à¦¾":2220,"à§à¦¥à¦¿":1299,"à§à¦¥à§‡":279,"à§à¦¥à¦¨":231,"à§à¦¤à§‡":276,"à§à¦¥à¦¬":329,"à§à¦¤à§":1963,"à§à¦¤à¦¾":1329,"à§à¦¤à¦¿":2212,"à§à¦¤à§€":445,"à§à¦¤à§":740,"à§à¦¤à§ƒ":252,"à§à¦¤à¦¨":456,"à§à¦¤à¦®":911,"à§à¦¤à¦°":2524,"à§à¦¬à¦•":355,"à§à¦ªà§":259,"à§à¦ªà§‡":443,"à§à¦ªà¦¾":452,"à§à¦ªà§€":215,"à§à¦ªà¦¿":552,"à§à¦ªà¦°":340,"à§à¦ªà¦¨":198,"অংশ":574,"à§à¦¨à¦¾":360,"à§à¦¨à¦¿":236,"à§à¦§à§‡":424,"à§à¦§à¦¿":256,"à§à¦§à¦¾":305,"অকà§":342,"ইন ":402,"অঙà§":286,"à§à¦•à¦¿":934,"à§à¦•à¦¾":944,"à§à¦•à§ƒ":335,"à§à¦•à§‡":282,"à§à¦—ত":198,"à§à¦šà¦¾":486,"à§à¦šà¦¿":1888,"à§à¦šà¦²":770,"à§à¦—ে":523,"à§à¦—ী":329,"à§à¦—া":318,"অফ ":315,"ংসà§":691,"োনো":215,"োনা":252,"োপা":229,"োমা":248,"োমি":242,"োয়":741,"োরি":210,"োরà§":375,"োলা":267,"োলন":236,"েছি":696,"েছে":2368,"েজি":1969,"ইংর":2136,"েকà§":594,"আমে":217,"েকে":2070,"েখক":248,"অà§à¦¯":839,"আয়":311,"েখা":876,"আবি":225,"েকট":292,"ইউন":276,"আধà§":208,"ইউর":313,"�":1457,"আনà§":621,"আলো":342,"আরà§":422,"আরব":337,"ইটি":293,"অঞà§":595,"অবস":1435,"অভি":728,"আইন":296,"অধি":383,"অনà§":1003,"অধà§":210,"অনà§":1608,"অনে":547,"আকা":241,"েশ ":1172,"েস ":337,"অরà§":1060,"েষ ":526,"েল ":879,"আছে":543,"ের ":22964,"অসà§":383,"োর ":595,"োন ":788,"োট ":334,"ৈরী":206,"ৈরি":250,"উচà§":373,"ৈতি":259,"োগ ":311,"উটা":283,"োক ":321,"উতà§":954,"েসà§":373,"ইতà§":216,"ইতি":273,"ইনà§":608,"েরা":505,"েরি":683,"েলে":417,"েলি":409,"েলা":1478,"েলো":239,"ইয়":586,"েশন":465,"েশী":418,"েশি":582,"েষণ":370,"ইরা":327,"েশে":1749,"েষà§":229,"েনà§":1439,"েনে":324,"েনি":277,"েনী":276,"েনা":409,"েপà§":238,"েবে":991,"েবà§":240,"েয়":1277,"েমà§":732,"েমি":222,"েমন":290,"ইসল":353,"েটি":246,"েডি":240,"েতা":376,"েতà§":488,"েন।":2598},"n_words":[1969690,2210879,1502429],"name":"bn","type":"devanagari"} \ No newline at end of file
diff --git a/contrib/languages-data/br.json b/contrib/languages-data/br.json
new file mode 100644
index 0000000..7216a43
--- /dev/null
+++ b/contrib/languages-data/br.json
@@ -0,0 +1 @@
+{"freq":{"D":4197,"E":6232,"F":3169,"G":6172,"A":11736,"B":9621,"C":6059,"L":5259,"M":7896,"N":3931,"O":1684,"H":4028,"I":4207,"J":2162,"K":6749,"U":5511,"T":3855,"W":1494,"V":2500,"Q":172,"P":6579,"S":9102,"R":4212,"Y":1003,"X":512,"Z":488,"f":8875,"g":61989,"d":78008,"e":313803,"b":31588,"c":24054,"Fed":26,"a":272837,"n":176116,"o":149856,"l":87783,"Fei":28,"m":47338,"j":4813,"Fel":48,"k":50019,"h":76286,"i":117218,"w":12251,"v":50149,"Fer":128,"u":89344,"t":103349,"s":73080,"r":187209,"q":653,"p":29670,"z":76339,"y":13415,"x":1593,"²":87,"É":103,"Æ":28,"Ã":35,"ß":56,"Ö":36,"Fil":72,"ï":72,"î":145,"Fin":82,"í":410,"ì":30,"ë":83,"ê":3087,"Fir":91,"é":1358,"è":308,"ç":177,"æ":55,"Ã¥":31,"ä":104,"ã":91,"â":320,"á":471,"à":154,"ü":417,"ú":104,"û":28,"ø":33,"ù":12711,"ö":186,"ô":883,"ò":64,"ó":303,"ð":41,"ñ":13114,"Ä“":47,"Ä":179,"ı":57,"Ä«":94,"Å“":24,"Å‚":28,"Å":129,"Å‹":29,"Å¡":64,"Å«":60,"Fas":33,"Fal":52,"Far":46,"Fao":28,"Ere":25,"Eri":80,"Ess":28,"Est":94,"ɪ":92,"Eti":95,"Ern":44,"É‘":37,"É”":56,"É™":176,"Esk":29,"É›":92,"Eus":234,"Eur":413,"Evi":62,"Eve":178,"ʲ":56,"Etr":88,"Ê":32,"ʃ":63,"Eug":29,"Ê’":27,"ˈ":252,"Ë":126,"El ":42,"Eil":157,"Eis":24,"̪":34,"Ì":93,"En ":194,"Ele":79,"μ":110,"ν":272,"ο":341,"Eng":26,"θ":54,"ι":256,"Ene":282,"κ":130,"λ":194,"δ":108,"ε":164,"η":171,"α":367,"β":37,"γ":77,"έ":64,"ά":97,"Emr":46,"ί":114,"ή":58,"Τ":24,"Emi":47,"Π":49,"Σ":33,"Ema":198,"Emb":80,"Îœ":29,"Κ":60,"Emg":113,"Δ":26,"Ε":25,"Α":38,"Eli":87,"Er ":626,"Eos":308,"ω":68,"ÏŽ":38,"Ï":56,"ÏŒ":91,"σ":144,"Enn":75,"Ï‚":295,"Ï":254,"Eno":35,"Ï€":118,"χ":49,"φ":41,"Ï…":86,"Ï„":173,"ÑŒ":74," l":8798," m":10929," n":4691," o":10291,"Ñ":110," h":33728,"ш":32," i":6397," j":591," k":16456,"Ñ‹":48," d":33789," e":65040,"Ñ…":29," f":3147,"ц":29," g":22398,"ч":83,"Ñ€":242," a":67851,"Ñ":221," b":15502,"Ñ‚":159," c":6570,"у":98," y":7164," x":75," z":11510," u":16972," t":10075," w":4118," v":14078," q":70," p":14721," s":18295," r":10530,"К":52,"Ðœ":37,"П":37,"Б":35,"Ð":40,"Ð’":28," J":2150," K":6707,"Gel":33," H":4011,"Gem":30," I":4197," N":3921," O":1674," L":5228," M":7864," B":9590," C":6014,"Р":27," A":11706,"С":33," F":3158," G":6126," D":4174," E":6209,"л":194," Z":483,"к":212,"й":67," Y":1001," X":511,"и":383,"п":52,"Gev":46,"о":298,"н":235,"м":105," S":9083,"г":57,"Ger":233," R":4196,"в":187," Q":170,"б":58," P":6551,"а":436,"Geo":116," W":1492,"з":42," V":2493,"Gen":420," U":5507,"е":253," T":3831,"д":87," à":46," é":34,"Gla":87,"Gha":39," Ã":35," Æ":28,"Gio":61,"Gin":304," É":99,"Gil":36,"Gir":54," Ö":36,"Giu":33,"Öst":25,"Õ¡":37,"×”":46,"ו":58,"ד":24,"×":35,"ב":25,"מ":26,"ל":40,"×™":72,"× ":26,"ש":34,"ר":42,"ת":30,"Gan":365,"Gal":400,"Gau":32,"Gar":135,"Gai":41,"Gae":31,"Gab":63,"Ùˆ":95,"ÙŠ":184,"Ù":25,"Ù‚":36,"Ù„":178,"Ù…":124,"Ù†":112,"Ù‡":24,"Ful":27,"د":75,"ج":40,"Ø­":59,"ت":33,"ب":101,"Ø©":62,"ا":268,"ع":53,"Ø´":34,"س":58,"ز":26,"ر":129,"Fro":29," ˈ":146," Κ":60," Îœ":29," Α":37," Ε":25," Δ":26,"Flo":56,"Fla":330," Ð":40," Б":35," Ð’":28,"Fra":928," К":52,"Fri":125," Ðœ":37,"Fre":98," П":37,"A ":488," Ï„":29," σ":25,"Fon":41,"For":176,"Fou":55," Σ":33," Π":49,"F ":73,"Da":716,"Cu":144,"Cy":124,"Cl":234,"Co":885,"Cr":171,"Ce":272,"Ch":926,"Ci":142,"G ":58,"Ec":91,"Ed":137,"Ea":89,"Eb":325,"Du":515,"Dy":40,"Do":666,"Dr":253,"De":778,"Dj":30,"Di":924,"Dh":31,"Fe":317,"H ":28,"Fa":315,"Ez":36,"Eu":755,"Ev":284,"Ex":30,"Er":901,"Et":209,"Es":214,"En":714,"Em":545,"Ep":30,"Eo":336,"Ei":245,"El":312,"Ek":28,"Eg":106,"Ge":937,"Ga":1188,"I ":590,"Fu":93,"Fr":1205,"Fo":347,"Bé":40,"Fl":420,"Fi":321,"B ":138," Р":27," С":33,"II ":397,"C ":1616,"Av":131,"Au":241,"Ar":3352,"IIv":103,"Aq":24,"At":426,"As":207,"D ":116,"Ba":1056,"Az":390,"Ay":27,"Ae":93,"Af":391,"Ag":119,"Ah":33,"Ab":268,"Ac":90,"Ad":209,"Am":688,"An":2172,"Ao":672,"Ap":114,"Ai":113,"Ak":96,"Al":1482,"Hir":30,"Hit":39,"His":54,"Bu":358,"Br":4619,"Hiz":25,"Ca":1444,"E ":930,"Bh":52,"Bi":459,"Be":1718,"Hil":64,"Bo":882,"Him":29,"Bl":208,"Hip":24,"Ku":315,"Ky":42,"Kn":26,"Kl":164,"Kr":780,"Ko":1315,"Le":1264,"Li":865,"N ":201,"La":1228,"Lu":362,"Ly":48,"Hé":41,"Ll":170,"Lo":978,"Me":1994,"Mi":714,"া":34,"O ":130,"Ma":3078,"Mc":41,"My":178,"Mu":254,"Mo":1332,"Ni":470,"Ne":1118,"Na":820,"P ":94,"Hel":244,"Ny":33,"Hei":53,"Nu":54,"No":1155,"Hea":31,"Ok":98,"Ol":141,"Om":48,"Kê":117,"On":29,"Og":35,"Oi":37,"Heo":48,"Oc":47,"Od":68,"Hem":56,"Of":58,"Hen":313,"Hes":59,"Oa":47,"Her":732,"Ob":101,"Gi":574,"Gh":58,"Gl":175,"Gr":739,"Go":862,"Gu":340,"Gy":27,"Gw":1133,"J ":134,"क":25,"Ha":1093,"He":1705,"Hi":350,"Ho":415,"न":34,"Hu":232,"म":31,"Hy":84,"र":42,"K ":222,"Ib":34,"Ia":195,"Id":60,"Ic":25,"Ie":25,"ि":28,"Ig":31,"ा":67,"Io":60,"Ip":79,"Im":319,"In":1032,"Il":240,"ी":28,"Iw":209,"Iv":208,"à¥":59,"Is":368,"It":343,"Ir":161,"Ja":558,"L ":80,"Iz":313,"Ji":84,"Je":437,"Jo":529,"Had":25,"Haa":25,"Hab":131,"Ju":263,"Hal":46,"Ka":2038,"Hai":67,"Han":276,"M ":61,"Ham":53,"Kh":46,"Har":175,"Ki":287,"Haw":72,"Ke":1388,"Hau":54,"Ut":31,"Ur":2997,"Un":1813,"Uk":44,"Ul":286,"Uh":133,"Ty":53,"Tu":261,"Tr":772,"Pê":43,"To":427,"Th":476,"Ti":378,"Te":503,"Ta":643,"Tc":179,"V ":168,"Sw":47,"Sy":84,"St":1759,"Sv":127,"Su":798,"Wo":96,"Wi":335,"Sã":44,"Wh":28,"Wa":422,"We":489,"Vv":57,"Y ":42,"Vo":153,"Rí":30,"Vr":262,"Vu":27,"Vi":641,"Vl":43,"Ré":29,"X ":71,"Va":400,"Ve":664,"Mé":39,"Qi":24,"Pt":32," Ù…":36,"Pu":130,"Pr":893,"Gus":31,"Ps":36,"Guy":41,"S ":168,"Py":52,"Pe":1224,"Gua":82,"Pf":25,"Pa":1933,"Gue":50,"Gui":57,"Lé":26,"Pl":487,"Po":937," ع":30,"Pi":432,"Ph":183,"Os":107,"Ot":70,"Ou":210," ا":123,"Op":33,"Or":316,"R ":57," ب":27,"Ow":50,"Kö":34,"Se":618,"Sc":261,"Si":922,"Sh":171,"Sm":46,"Sl":65,"Sk":511,"Sr":25,"Sp":346,"So":664,"Ru":420,"U ":31,"Sa":2188,"Gwy":105,"Gwe":600,"Re":1198,"Gwa":119,"Ri":358,"Rh":169,"Ro":1382,"Gwi":279,"Qu":112,"T ":45,"Ra":453,"Mü":57,"Gre":245,"Gri":43,"Gra":263,"Wü":24,"b ":1746,"Gru":58,"Gro":111,"a ":60184,"Ye":198,"Ya":330,"Yo":220,"Yv":36,"Yu":107,"Ys":24,"Gle":29,"Wy":25,"Glo":28,"Xa":37,"Xi":32,"Xv":91,"Gon":36,"Got":28,"Gou":537,"Gor":76,"Za":103,"Ze":159,"Zi":46,"Zo":34,"Zu":68,"Vî":53,"i ":7854,"fy":71,"gd":123,"ge":7989,"ga":15000,"gb":76,"bé":25,"fl":324,"fg":36,"ff":279,"fi":1447,"Ini":174,"fr":1282,"fu":204,"ft":110,"Int":68,"fo":1405,"j ":410,"gy":86,"gw":1619,"hd":50,"Ipa":70,"he":10707,"hc":25,"ha":23155,"gn":2093,"gm":47,"gl":848,"gk":33,"gi":2102,"gh":734,"gg":126,"gu":2092,"Ion":24,"gt":232,"gs":102,"gr":2162,"go":4456,"dt":122,"du":4207,"dv":101,"dw":187,"dy":171,"dz":34,"g ":20767,"ea":2356,"eb":1541,"ec":3453,"ed":9236,"de":16457,"Ili":95,"dd":600,"dg":78,"df":47,"di":9920,"dh":144,"dk":37,"dj":268,"dm":38,"dl":101,"do":5797,"dn":55,"dp":25,"ds":337,"dr":5697,"ew":376,"ex":279,"eu":16675,"añ":10099,"ev":9709,"ey":390,"ez":32964,"fa":1334,"h ":20209,"Ind":651,"fe":1499,"eh":357,"eg":13356,"ef":502,"ee":1354,"el":17644,"ek":11355,"ej":161,"ei":7025,"ep":3262,"eo":10164,"Imp":288,"en":41069,"em":6548,"et":30716,"es":7806,"aï":43,"er":32314,"aí":40,"eq":26,"ca":1129,"e ":52189,"bw":39,"by":108,"bs":76,"br":5995,"bu":921,"bn":37,"bo":2286,"bj":43,"bl":2570,"bh":95,"bi":1782,"bb":73,"bc":25,"bd":41,"be":10849,"db":36,"da":12451,"f ":728,"cy":83,"cu":421,"ct":371,"cs":66,"cq":94,"cr":201,"co":1158,"cm":65,"ck":578,"cl":127,"ci":937,"ch":6076,"ce":962,"cc":154,"c ":11462,"Il ":108,"az":3455,"ay":921,"ba":4699,"d ":20779,"at":6319,"as":6537,"ar":41906,"aq":80,"ax":163,"aw":608,"av":4883,"au":1374,"ak":2454,"al":17234,"ai":2830,"aj":361,"ao":6749,"ap":2035,"am":7774,"an":56453,"ac":2422,"ad":15598,"aa":848,"ab":3465,"ag":10609,"ah":821,"ae":5484,"af":819,"nu":699,"nt":21400,"ns":1811,"nr":251,"nq":24,"np":48,"no":7098,"nn":19156,"q ":55,"nz":429,"ny":637,"nw":203,"nv":5870,"oe":3732,"of":1198,"oc":1802,"od":2285,"oa":9317,"ob":1946,"om":6712,"on":16021,"kê":868,"ok":1528,"ol":8383,"oi":1324,"oj":257,"og":1807,"oh":346,"hÅ":59,"m²":87,"ot":2276,"os":3836,"ov":1541,"ou":23053,"op":1601,"oo":428,"or":10770,"oq":29,"r ":67149,"ox":54,"ow":323,"oz":3965,"oy":250,"pe":10529,"lá":71,"lâ":211,"pg":46,"pa":6028,"pl":2140,"lé":135,"pm":43,"lê":44,"pn":29,"po":2880,"ph":570,"JK ":124,"pi":1360,"pk":99,"lo":6423,"ln":63,"hê":112,"lm":544,"hé":48,"Ida":24,"ll":11038,"ls":344,"lr":74,"lp":246,"lw":139,"lv":1307,"lu":1154,"lt":965,"lz":374,"ly":435,"hô":24,"o ":32003,"md":124,"ma":12097,"mb":2996,"mg":320,"mh":71,"me":8356,"mf":56,"mk":42,"iè":109,"ml":68,"mi":2710,"mn":103,"iê":44,"mm":2195,"ié":44,"mp":2828,"mo":3261,"mr":185,"mt":73,"ms":373,"mv":54,"mu":2992,"iñ":2018,"ió":60,"mw":25,"mz":3729,"my":129,"p ":1373,"na":11870,"nb":331,"nc":1094,"nd":5068,"ne":23869,"nf":226,"ng":4164,"nh":269,"ni":8151,"nj":156,"nk":1557,"nl":441,"Iañ":171,"nm":245,"jv":61,"ju":124,"jp":52,"jo":305,"jk":39,"ki":2545,"kh":260,"kg":81,"kf":26,"ke":6882,"ka":6948,"m ":4186,"kw":151,"ky":90,"ks":726,"kt":473,"ku":843,"kv":64,"ko":9911,"kp":26,"kr":4134,"kk":98,"kl":851,"km":424,"gé":31,"gê":1037,"kn":60,"li":10539,"lh":850,"lk":281,"lj":172,"há":26,"le":14802,"ld":725,"lg":609,"lf":458,"hâ":37,"la":12823,"lc":685,"lb":380,"n ":59707,"hr":1218,"hs":140,"hp":166,"hv":118,"hw":571,"ht":657,"hu":1645,"hk":45,"hh":50,"hi":5476,"hn":188,"ho":10678,"hl":290,"dé":99,"hm":114,"id":3097,"ic":2459,"ib":1666,"ia":11918,"ih":739,"ig":3475,"if":619,"ie":9527,"hy":234,"k ":14348,"iq":109,"ir":3605,"is":9955,"it":5964,"iu":555,"eñ":707,"iv":5470,"iw":1706,"eó":29,"ix":145,"ii":261,"ij":2240,"ik":4708,"il":5550,"im":2753,"in":11502,"io":6614,"ip":861,"je":2425,"ji":570,"iz":11203,"iy":170,"l ":21633,"ja":657,"eü":82,"xi":170,"tê":725,"xo":25,"té":97,"tí":24,"xt":32,"wy":307,"ww":48,"z ":21079,"xa":108,"xe":71,"tá":25,"wg":44,"wi":1081,"wl":28,"sé":62,"wn":92,"wo":182,"wr":234,"ws":56,"wu":29,"ró":27,"vy":51,"y ":1750,"wa":5770,"wd":28,"wc":24,"we":3809,"vl":249,"rè":27,"ré":171,"vi":8261,"vu":454,"vr":4428,"vs":56,"rí":84,"vn":703,"vo":3736,"uz":3040,"uy":131,"ux":133,"uw":90,"uv":260,"uu":46,"rá":48,"ve":20616,"vc":54,"va":5019,"x ":1065,"ui":1518,"uj":140,"uk":544,"ul":3378,"ue":6663,"uf":196,"ug":1545,"uh":578,"uq":27,"ur":24906,"us":15250,"ut":2185,"um":4282,"un":10596,"uo":187,"up":344,"ty":255,"tz":187,"tu":2687,"tt":851,"tw":83,"tv":571,"ub":1283,"ua":3200,"ud":3820,"uc":669,"w ":395,"to":5713,"tn":87,"tm":65,"pé":34,"tl":859,"ts":475,"tr":9199,"tp":62,"tg":52,"tf":46,"te":12319,"td":69,"tk":150,"ti":7302,"th":2179,"où":12686,"v ":6249,"tb":27,"tc":753,"ta":12012,"su":1038,"oñ":231,"sv":158,"ss":1448,"st":12827,"sy":140,"sz":27,"sw":107,"sl":448,"sk":7635,"sn":138,"sm":219,"sp":1928,"so":3042,"sr":249,"sq":42,"sd":77,"sc":650,"sf":62,"se":5114,"sh":1087,"sg":84,"si":4429,"rz":4399,"u ":4057,"sa":6559,"sb":201,"rr":4376,"rs":928,"rt":3750,"ru":2612,"rv":3731,"rw":117,"nô":738,"rx":60,"ry":577,"rq":53,"rp":204,"ro":20230,"rn":4389,"rm":2207,"né":104,"rl":1174,"rk":1313,"nç":124,"rj":101,"ri":15929,"rh":211,"rg":1755,"rf":439,"re":30793,"ná":35,"rd":2128,"rc":2074,"rb":1039,"ra":14858,"t ":46228,"qu":495,"IXv":50,"mé":40,"má":30,"qa":26,"s ":25101,"px":514,"py":38,"pt":406,"pu":902,"pv":51,"pp":317,"lí":31,"pr":1879,"ps":278,"Huo":27,"Hun":82,"IX ":33,"IV ":67,"zz":96,"Hor":50,"zg":66,"zh":24600,"zi":3857,"Hou":36,"zb":56,"zd":80,"ze":9700,"Hog":24,"za":2558,"Hom":30,"Hon":65,"Hol":98,"zv":69,"zs":223,"vî":50,"zr":75,"zu":767,"zt":86,"zo":11706,"zn":961,"zp":33,"zk":64,"zm":26,"vé":27,"zl":48,"yg":40,"ye":6849,"yf":63,"uß":34,"yc":113,"yd":279,"ya":1435,"yb":30,"yw":102,"uñ":26,"yu":169,"yt":99,"ys":381,"yr":208,"yp":102,"yo":385,"yn":492,"ué":33,"ym":158,"yl":291,"yk":38,"yi":226,"Arg":47,"Are":40,"Arc":158,"Ard":33,"Ara":192,"Aro":63,"Arm":115,"Arn":27,"Ark":40,"Ari":62,"Apo":55,"Ate":49,"Ath":31,"Atl":295,"Ast":51,"Ass":62,"Arr":26,"Art":101,"Arv":191,"Arz":47,"Avi":58,"Ave":35,"Aus":28,"Aur":33,"Aug":87,"Aze":65,"Azg":34,"Azi":259,"Bag":26,"Bah":39,"Bai":39,"Bal":109,"Ban":147,"Bab":28,"Bac":26,"Bad":61,"Bae":28,"Bay":41,"Bar":222,"Bas":70,"Bav":68,"Bau":37,"ი":28,"áƒ":24,"Abe":71,"Aba":68,"Abd":28,"Ada":49,"Ado":41,"Ade":35,"Aet":37,"Aga":26,"Afr":330,"Afg":29,"Agn":27,"Air":36,"Al ":187,"Aka":25,"Ala":402,"Alb":169,"An ":1075,"Ali":70,"Alj":93,"Alc":41,"Ale":147,"Alf":49,"Alt":68,"All":74,"Alo":28,"Alp":44,"Ame":389,"Amb":31,"Ama":131,"Ams":36,"Ang":158,"Anj":44,"Ana":144,"And":189,"Anv":85,"Ant":227,"Ann":136,"Aod":213,"Aoz":43,"Ar ":2118,"Aos":340,"Bul":44,"Bun":25,"Bur":93,"Bue":26,"Bry":28,"Bru":240,"² ":87,"Cad":56,"Cae":102,"Cal":419,"Cam":89,"Cas":109,"Car":279,"Cat":102,"Can":103,"Cap":77,"Bea":77,"Bez":111,"Bev":81,"Bes":27,"Bet":78,"Ber":307,"Ben":164,"Bel":409,"Bei":28,"Beh":46,"Beg":25,"Bed":301,"Bib":28,"Bho":30,"Biz":33,"Bil":34,"Big":27,"Bih":155,"Bis":50,"Blo":51,"Ble":54,"Bla":69,"Bre":1870,"Bra":444,"Bro":1738,"Bri":270,"Bod":38,"Bob":30,"Boh":28,"Bol":142,"Bon":77,"Bor":114,"Bos":60,"Bot":49,"Bou":215,"Cyn":66,"Cun":40,"De ":65,"Der":69,"Des":46,"Dev":53,"Deu":57,"Dei":45,"Del":63,"Dem":103,"Den":92,"Dea":33,"Deg":26,"Da ":59,"Dam":34,"Dan":165,"Dao":76,"Dar":80,"Das":35,"Dav":72,"Dag":36,"Dal":37,"Cho":45,"Chr":125,"Che":89,"Chi":180,"Cia":24,"Chu":34,"án":119,"áp":29,"ái":34,"Cit":26,"ár":44,"ân":32,"ât":38,"âr":214,"ão":73,"Châ":25,"Cle":39,"Cla":74,"Cel":27,"Cen":52,"Cer":110,"à ":84,"á ":100,"ße":42,"Cha":389,"Cri":33,"Cra":30,"Cre":37,"Cro":36,"Clu":45,"Ös":26,"Cos":59,"Cor":145,"Com":218,"Col":101,"Coo":25,"Con":166,"Cou":45,"アアア":142,"ôg":720,"ôn":58,"ór":29,"ón":140,"ñs":2047,"ñv":891,"ó ":30,"ñc":70,"ña":31,"ñe":28,"ñj":75,"ñi":45,"ño":24,"ñ ":9866,"în":118,"Dyf":26,"ín":68,"ío":42,"ís":37,"ï ":28,"ía":79,"Egi":77,"í ":88,"êr":2927,"êt":62,"él":92,"éo":59,"ép":36,"ém":53,"én":95,"és":71,"ét":52,"ér":121,"év":46,"Edo":32,"éa":40,"éd":33,"éc":28,"ée":94,"Edw":28,"ég":45,"èn":34,"èr":89,"ès":41,"èg":41,"ê ":35,"Ecu":41,"Ech":27,"ço":86,"ça":53,"é ":426,"Ebr":307,"FE ":165,"Eas":54,"än":24,"Än":45,"Dia":97,"Dic":35,"Dis":208,"Dio":61,"Din":67,"Dim":39,"Dil":24,"Die":74,"Diz":31,"Diw":77,"Div":53,"Ἀ":34,"Dug":54,"Dub":25,"Dun":35,"Dul":31,"ể":119,"ün":95,"ül":25,"üs":76,"ür":103,"üt":25,"üh":27,"Dro":31,"Ð°Ñ ":51,"Du ":257,"ù ":12666,"Dre":155,"Dra":31,"ú ":25,"ör":30,"ön":34,"öl":28,"Dou":218,"Don":216,"Dom":60,"Dor":74,"Ned":28,"Nei":28,"Nev":419,"Neu":75,"Nes":32,"Ner":24,"Nep":142,"Ne ":61,"Nat":66,"Nav":46,"Naz":46,"Nij":37," Ἀ":34,"Nig":136,"Nie":35,"Nic":62,"Nis":25,"Nin":27,"Nik":43,"Nil":32,"New":165,"Mya":136,"Nao":129,"Nap":76,"Nar":28,"Nam":105,"Nan":35,"Nag":59,"Nac":31,"Naf":83,"Nou":26,"Nov":31,"Nor":936,"Not":36,"Nob":26,"Ofi":38,"Oki":39,"Oke":28,"PA ":54,"Obe":44,"Obl":33,"Ott":28,"Our":31,"Oug":38,"Ouz":52,"Owa":32,"Ä« ":39,"Oli":92,"Kêr":116,"Ore":54,"Ori":92,"Plo":191,"Ple":106,"Pla":137,"Pin":52,"Pis":27,"Pir":97,"Pie":124,"Pic":33,"Pho":45,"Phi":80,"Pha":29,"Peu":34,"Ped":27,"Per":261,"Pet":89,"Pem":45,"Pen":568,"Pep":35,"Pel":55,"Pat":67,"Pas":301,"Par":587,"Pau":112,"Pad":32,"Pan":90,"Pap":281,"Pao":63,"Pal":137,"Pak":62,"Pa ":68,"Ån":33,"Åg":26,"Pyr":31,"Pue":30,"Pro":300,"Pri":187,"Pre":189,"Pru":64,"Pra":117,"Poa":24,"Pob":66,"Pol":187,"Pom":34,"Pon":112,"Pot":43,"Pos":46,"Pou":45,"Pop":34,"Por":207,"Pow":35," ال":101,"Rac":24,"Rad":28,"Ram":45,"Ran":131,"Mün":49,"Å« ":25,"Qui":31,"Que":38,"Isa":61,"Ita":306,"Isl":122,"Isr":84,"Ist":57,"Ira":83,"Iwe":203,"Ive":180,"Iza":28,"Ize":269,"Jaf":31,"Jac":102,"Jap":114,"Jan":80,"Jam":74,"Jak":53,"Jen":25,"Jer":130,"Jez":68,"Jea":161,"ã‚":41,"Jos":134,"Jor":74,"Jon":50,"ã‚¢":232,"Joh":156,"Joa":30,"Jud":39,"Jua":30,"Jus":24,"Jul":90,"Kad":30,"Kae":24,"Kab":38,"Kam":138,"Kal":163,"Kao":26,"Kan":725,"Kat":130,"Kas":218,"Kar":359,"Kaz":45,"Kav":60,"Kev":87,"Ker":517,"Ken":236,"Kel":56,"Kem":388,"Keb":26,"Kir":52,"Kin":57,"Kip":33,"Kim":33,"Klo":52,"Kle":50,"Kla":37,"Kon":258,"Kom":385,"Kol":122,"Kor":327,"Kou":52,"Koz":35,"Kre":355,"Kra":64," ã‚¢":26,"Kri":101,"Kro":256,"Kub":35,"Kum":133,"Kur":37,"Kuz":44,"Lev":68,"Les":97,"Leo":254,"Len":235,"Lem":27,"Lei":51,"Lee":34,"Lec":60,"Led":43,"Lau":43,"Lav":32,"Laz":27,"Le ":163,"Lak":70,"Lag":26,"Lat":40,"Lar":56,"Lao":55,"Lap":40,"Lam":58,"Lan":301,"Lad":25,"Lab":87,"La ":252,"Liè":30,"Lla":63,"Lly":57,"Lib":95,"Lia":47,"Lic":25,"Lig":265,"Lil":27,"Lim":38,"Lin":62,"Lis":51,"Lit":44,"Liv":31,"Leó":27,"Liz":35,"Lus":43,"Luk":34,"Lui":42,"Lun":30,"Lud":36,"Luc":61,"Lou":225,"Los":44,"Lot":45,"Loi":45,"Loe":103,"Lod":29,"Loc":62,"Lor":83,"Lop":24,"Lon":123,"Lom":37,"Lok":34,"Mei":50,"Men":173,"Mel":118,"Mes":68,"Mer":275,"Meu":589,"Met":52,"Mec":298,"Meg":27,"Med":94,"Mez":157,"Man":225,"Mao":80,"Mal":272,"Mar":1145,"Mas":64,"Mag":82,"Mad":169,"Mae":313,"Mak":35,"Mah":26,"Mai":71,"Mac":91,"Mab":145,"May":42,"Maz":26,"Mau":69,"Mat":103,"Mod":41,"Moh":29,"Mol":46,"Mon":319,"Mos":55,"Mor":620,"Mou":67,"Mik":53,"Mid":40,"Mig":28,"Mic":152,"Mir":97,"Mis":48,"Mil":93,"Min":123,"Mun":35,"Mur":34,"Mus":85,"çoi":70,"ège":37,"XXv":30,"Wor":34,"Wol":42,"Wil":179,"Win":40,"каÑ":36,"ère":70,"Wei":43,"Wel":28,"Wer":38,"Wes":242,"Wen":65,"São":44,"ène":27,"Was":35,"War":185,"Wat":32,"Wal":100,"ès ":37,"Vve":56,"ée ":61,"Vre":153,"Vro":82,"ées":30,"Vol":32,"Vis":41,"Vit":32,"Viê":40,"Vla":26,"éli":30,"éra":34,"éri":30,"éné":31,"Zel":58,"éon":34,"Zeu":41,"és ":43,"之":110,"並":62,"丘":32,"专":41,"三":151,"ä¸":73,"Yve":30,"Yun":35,"Yuz":30,"Yor":126,"You":64,"É™ ":47,"É™n":29,"Yan":129,"Yao":137,"êr ":2352,"на ":30,"êt ":44,"êrb":145,"êri":399,"Yez":130,"Xve":91,"Syl":30,"Stê":62,"Swa":35,"Sve":120,"Sur":43,"Sus":66,"Sum":29,"Sul":65,"Sun":33,"Sui":93,"Sua":196,"Str":447,"Stu":77,"Sti":57,"Sto":80,"Sta":876,"Ste":130,"Ten":39,"Tem":32,"Teo":30,"Tei":34,"Tel":40,"Tch":179,"ão ":72,"Tam":45,"Tan":101,"Tar":82,"Tai":60,"Tal":111,"Tad":57,"UA ":109,"Ski":27,"Sko":315,"Skr":81,"Ska":32,"Ske":30,"She":24,"Sha":39,"Sim":44,"Sil":48,"Sik":59,"Sig":31,"Sir":75,"Sin":458,"Sie":35,"Sib":58,"Sic":35,"Ser":108,"Set":39,"Sev":58,"Sen":87,"Sel":55,"Sei":79,"Sea":28,"Su ":163,"Spa":218,"Spi":30,"Spe":42,"Sok":47,"Soc":33,"ShÅ":35,"Sou":218,"Sov":76,"Sol":47,"Som":59,"Son":59,"Sor":31,"Slo":44,"Smi":34," 三":55,"Ruz":48,"Rus":272,"Sai":243,"Sak":41,"Sam":85,"Sal":156,"Sac":54,"Sab":28,"Sco":35,"Sci":29,"Sch":148,"Sav":198,"Sau":25,"Sar":106,"San":782,"Sao":292,"ови":34,"SI ":31,"Res":34,"Ret":176,"Reu":24,"Rev":40,"Rho":37,"Rhe":37,"Rio":64,"Riv":24,"Ric":118,"Rhy":27,"Rec":25,"Red":61,"Rei":69,"Reg":63,"Ren":134,"Rep":458,"Rol":26,"Roe":42,"Roa":103,"Rob":85,"Roc":84,"Rod":47,"Roy":27,"Rou":398,"Rot":44,"Ros":112,"Ron":40,"Rom":236,"Rop":51,"SS ":38," 之":38,"SO ":28,"Vel":35,"Ven":128,"Ñка":39,"Van":75,"Val":141,"Var":74,"Vig":36,"Vic":64,"Vie":116,"Vir":42,"Vil":82,"Vin":104,"Ver":120,"Veu":295,"Ul ":233,"Ukr":40,"Uhe":126,"VIv":68,"Una":1019,"Uni":82,"Unv":67,"Un ":607,"Uru":37,"Ur ":2862,"VI ":43,"Teu":28,"Tex":28,"Ter":100,"Tet":30,"Tes":38,"Tha":76,"The":234,"Thi":28,"Tho":61,"Tib":58,"Tie":85,"Tim":40,"Tin":33,"Tit":30,"Tir":57,"Pêr":43,"Tor":68,"Tol":28,"Tom":36,"Ton":31,"Tos":58,"Tou":94,"çai":30,"Tro":186,"Tri":158,"Tre":263,"Tra":119,"Tur":141,"Tun":31,"Tud":24,"ßen":37,"вич":50,"ã‚ã‚":28,"bja":26,"bl ":407,"biz":216,"bis":45,"bit":50,"bio":107,"bir":75,"bik":31,"bil":84,"bin":157,"big":45,"bih":300,"bo ":54,"blo":711,"ble":426,"bli":615,"bla":374,"bod":104,"boa":63,"bob":195,"bok":24,"bol":151,"boe":45,"biñ":26,"br ":57,"bon":174,"bom":59,"bor":273,"bot":53,"bos":68,"bou":909,"be ":93,"bam":27,"ban":1032,"bak":29,"bal":315,"bai":138,"bag":46,"bah":28,"bae":46,"bac":146,"bad":553,"bab":75,"án ":76,"baz":28,"bay":39,"bau":28,"bat":179,"bas":257,"bar":1084,"bao":351,"bea":32,"bi ":70,"bei":36,"beg":400,"bed":495,"bec":37,"ber":2194,"bep":146,"ben":1724,"bem":27,"bel":314,"bek":240,"bez":495,"beu":260,"bev":173,"bes":62,"bet":4067,"ápa":25,"bia":300,"bic":30,"bid":59,"bie":109,"bha":32,"ca ":318,"car":108,"cas":148,"cat":71,"cau":32,"can":105,"cap":50,"cad":36,"cam":25,"cal":102,"ce ":316,"bri":446,"bro":1440,"bra":977,"bre":2698,"bu ":43,"bru":350,"bsb":24,"boù":29,"bua":43,"bur":380,"bul":68,"bun":41,"buh":76,"bug":103,"but":41,"bus":55,"by ":41,"bwe":29,"aka":732,"am ":551,"ake":317,"aki":154,"akh":42,"aji":29,"ajo":39,"al ":3797,"aja":58,"aje":72,"aii":42,"aik":32,"ail":319,"aim":51,"ain":893,"aio":50,"air":137,"ais":419,"ait":72,"aiw":30,"ak ":494,"aig":79,"aie":120,"aid":145,"aia":64,"ahi":68,"ahu":48,"ahr":32,"aho":95,"aj ":136,"ahe":52,"aha":329,"agg":35,"agh":38,"agi":75,"agr":25,"agu":180,"agn":1483,"ago":271,"aog":68,"aol":416,"aok":392,"aod":271,"aoe":315,"anv":4249,"anu":193,"anz":183,"any":89,"ano":519,"ann":3597,"anm":210,"ant":14921,"ans":274,"ane":6479,"anf":35,"ang":1077,"anh":35,"ani":1074,"anj":41,"ank":665,"ap ":252,"ana":1898,"anb":31,"anc":428,"and":2346,"amu":63,"amt":28,"amz":526,"amm":715,"amo":419,"amp":357,"ams":92,"amk":30,"amh":31,"ami":521,"amd":32,"ame":864,"amb":405,"ama":2994,"ao ":87,"alz":151,"alv":701,"alu":178,"alt":325,"als":46,"alp":40,"alo":509,"alm":105,"all":4383,"alk":84,"alg":71,"ali":1896,"alj":57,"alc":477,"ald":244,"ale":1440,"ala":2477,"alb":121,"an ":17839,"aks":123,"akr":86,"aku":87,"akt":174,"ako":115,"aba":962,"abe":777,"abi":225,"abj":25,"abl":133,"abo":381,"abr":134,"abs":38,"abu":59,"ae ":1012,"aca":108,"aaa":146,"aan":41,"aal":34,"aas":35,"aar":52,"aat":389,"ad ":6605,"ac ":497,"aa ":55,"ab ":625,"afo":34,"afr":170,"aff":71,"afe":127,"afi":38,"ai ":239,"aga":729,"agd":27,"age":701,"aen":254,"ael":271,"aes":112,"aer":1001,"aeg":510,"aed":61,"aek":374,"ah ":119,"âte":26,"afa":255,"aet":1430,"aez":325,"ado":1590,"adr":147,"adh":35,"adj":43,"adi":235,"add":93,"ade":2997,"ag ":6969,"adw":46,"ady":24,"ads":127,"adt":40,"adu":2557,"adv":56,"acq":55,"aco":101,"ack":112,"aci":183,"ach":1027,"ace":95,"acc":47,"ada":865,"af ":45,"âre":203,"act":55,"acu":55,"acr":44,"azo":96,"azi":660,"azt":49,"aze":890,"azh":640,"aza":204,"azz":51,"axi":40,"az ":790,"ayo":91,"aym":25,"ays":58,"aya":179,"aye":142,"ba ":235,"bc ":24,"aqu":35,"at ":1878,"arg":317,"arf":40,"are":2408,"ard":882,"arc":571,"arb":175,"ara":1504,"arp":87,"aro":893,"arn":1123,"arm":342,"arl":474,"anç":111,"ark":510,"ari":1893,"aru":121,"arv":1540,"arw":27,"arx":54,"arr":912,"ars":166,"art":2208,"au ":260,"asa":726,"ary":114,"arz":1296,"asi":144,"ash":125,"asc":74,"ase":577,"aso":106,"asp":56,"ask":546,"asm":40,"aon":341,"aoo":69,"aot":235,"aou":1726,"aor":81,"aos":156,"ar ":24115,"aoz":2346,"apa":418,"alá":31,"ape":187,"api":98,"aph":42,"apl":49,"apo":648,"app":89,"apr":28,"aps":87,"apt":56,"apu":61,"as ":2697,"ava":1022,"ax ":65,"aux":32,"aut":155,"arí":35,"avr":39,"avo":170,"avi":298,"ave":2923,"awe":103,"ay ":311,"awa":237,"awr":48,"awi":50,"av ":320,"aoù":110,"ata":1010,"asu":48,"aoñ":77,"ast":1024,"ass":241,"asy":30,"atl":96,"atr":172,"ato":465,"ate":653,"atc":36,"ati":1306,"ath":184,"aw ":56,"auc":67,"aub":40,"att":149,"ats":44,"atu":213,"aul":182,"aum":36,"aun":74,"aur":161,"aus":113,"aud":112,"aug":24,"νη":32,"νο":29,"ολ":24,"πο":32,"ος":144,"ια":35,"ιο":30,"λι":25,"λα":24,"μα":24,"Wür":24,"ει":25,"ος ":144,"ης":30,"αι":28,"αλ":29,"αν":33,"αÏ":25,"ία":31,"Ï‚ ":295,"ν ":51,"ι ":25,"η ":101,"Zub":27,"Vîn":53,"α ":96,"アア":185,"Ïος":25,"ий ":24,"ων":25,"ич ":51,"το":28,"στ":31,"σι":30,"Ïα":33,"Ïο":43,"jed":50,"jer":491,"jek":59,"jen":344,"jet":1133,"jañ":33,"jev":42,"ji ":46,"νη ":24,"jk ":28,"jad":45,"jab":30,"eüs":65,"jap":102,"jar":45,"jal":43,"jan":113,"je ":203,"jou":81,"jon":49,"jor":36,"jpg":45,"Ñк":72,"jio":97,"jin":152,"jik":26,"jie":63,"jia":71,"jib":29,"ти":34,"та":24,"ÑÑ‚":55,"jo ":26,"itl":152,"itr":124,"ito":200,"itu":156,"itt":128,"its":52,"itz":70,"ity":61,"isk":433,"ism":94,"isl":110,"iso":129,"isn":43,"isp":496,"iss":497,"isr":129,"isu":29,"ist":3083,"ioù":3206,"iv ":786,"ita":909,"itc":84,"ite":544,"ith":138,"iti":736,"ivr":37,"ivo":342,"ivy":27,"iwe":401,"iwa":1260,"ius":378,"iur":33,"ium":58,"eñv":618,"iva":1287,"ivc":49,"ix ":114,"ivi":262,"ive":2609,"ipr":52,"ipo":75,"ipp":126,"ipu":31,"ipt":148,"ipi":84,"ipl":28,"is ":2724,"ion":1454,"iop":130,"ior":153,"ios":108,"iot":56,"iou":688,"iog":34,"iol":67,"iom":41,"ipa":143,"ipe":76,"iov":38,"ir ":701,"iru":31,"irv":32,"irr":117,"iro":258,"irm":24,"iri":682,"isi":977,"ish":314,"ise":310,"isc":169,"isb":24,"isa":291,"eñ ":32,"iqu":102,"ire":787,"irg":98,"ira":461,"ird":151,"irc":108,"it ":2544,"ünc":44,"ja ":94,"iya":79,"iye":30,"iz ":5713,"eón":26,"iwo":27,"izu":26,"izo":212,"izi":1082,"izh":3028,"ize":643,"izd":61,"iza":353,"kig":118,"kim":79,"kil":109,"kia":534,"kic":156,"kib":31,"kie":104,"kiz":138,"kl ":27,"kin":273,"kip":95,"kir":270,"kis":131,"kit":175,"keñ":73,"km ":262,"ki ":169,"kgr":43,"kha":110,"kho":57,"kea":37,"ked":186,"kef":27,"keg":356,"kei":93,"kek":447,"kem":714,"kel":791,"keo":32,"ken":1794,"kes":29,"ker":582,"kañ":225,"keu":150,"ket":897,"kev":353,"kez":64,"ke ":191,"kra":328,"kre":866,"kt ":39,"ksa":52,"kse":61,"ku ":65,"kro":809,"kri":2084,"kr ":32,"koz":381,"kov":45,"km²":85,"kot":54,"kou":3139,"kos":289,"kor":683,"kop":87,"kon":591,"kom":3155,"kol":453,"kog":42,"koe":63,"kod":39,"ks ":105,"ể ":40,"klê":29,"kiñ":43,"kme":37,"koa":172,"kob":78,"kni":26,"gêr":1036,"kke":25,"kka":30,"ko ":451,"kle":274,"kla":324,"klo":157,"jus":37,"jvi":55,"joù":51,"ển":39,"kaz":128,"kav":240,"kay":33,"kat":219,"kar":717,"kas":328,"kap":68,"kan":1716,"kao":135,"kal":381,"kam":95,"kai":52,"kae":493,"kad":451,"kac":24,"kab":44,"kaa":150,"ka ":1336," Ga":1180," Ge":925," I ":68," Fo":346," Fu":93," Fr":1201," Fi":320," Fl":419," Bé":40," Ha":1090," He":1698," Gw":1125," Gy":27," J ":114," Go":857," Gr":735," Gu":336," Gh":58," Gi":573," Gl":175," Ig":31," Id":54," Ic":25," Ib":34," Ia":193," K ":68," Hy":84," Hu":232," Ho":413,"ha ":7925," Hi":350," Ji":83," Je":435," L ":45," Ja":555," Iz":312," Iv":30," Iw":209," Ir":161," Is":368," It":343," Im":318," In":1030," Io":58," Ip":79," Il":238,"ham":356,"han":1921," M ":27,"hao":51,"hap":104,"hai":128," Ka":2025,"hak":57,"hal":1676,"hau":60," Ke":1373,"hav":45," Ki":281,"har":1216,"has":180," Kh":45,"hat":349," Jo":527,"haf":27," Ju":260,"hae":120,"hag":6703,"hab":93,"had":772,"hac":45," N ":136," La":1222," Le":1251," Li":860," Kl":163," Kn":26," Ko":1313," Kr":777,"haz":119," Ku":314," Ky":42," Mc":41," Ma":3069," O ":70," Mi":710," Me":1982,"he ":1293," Lo":976," Hé":41," Ll":170," Ly":47," Lu":361," Ne":1118," P ":29,"а ":89," Na":819," Ni":468," Mo":1327," My":178," Mu":252,"hek":226,"hel":1281,"hei":155,"heg":164,"heh":30,"hee":27,"hef":31,"hec":228,"hed":545,"hea":87,"heb":137,"hez":301,"hev":363," A ":247,"hañ":1086,"heu":277,"het":353,"hes":174,"her":2530,"hep":235,"heo":107,"hen":1633,"hem":192,"hi ":652," B ":90," C ":1547," Ao":672," Ap":113," Am":685," An":2165," Ak":92," Al":1478," Ai":112,"hho":32," Ag":119," Ah":33," Ae":93," Af":391," Ac":90," Ad":207," Ab":268," Ba":1051," D ":80," Az":390," Ay":27," Av":131," Au":240," At":426," As":207," Ar":3347," Aq":24,"hig":75," Be":1711,"hie":591,"hid":70,"hic":42," Bi":459," Bh":52,"hia":407,"hip":35,"hio":751,"hin":985,"him":81," Bl":207," Bo":879,"hil":302,"hik":334," Br":4609,"heñ":300," Bu":358,"his":80,"hit":151,"hir":469," E ":710," Ca":1420,"hiz":309," Ce":271," Ci":142," Ch":923," Cl":231," Cr":170," Co":876," Cu":142," Cy":124," F ":30," Da":708,"dé ":30," Di":920," Dh":31," Dj":29," De":772," Dr":253," Do":662," Dy":40," Du":514," Ea":86,"hn ":80," Eb":324," Ec":91," Ed":137," G ":24,"hla":88,"hle":80," El":312," Ek":28," Ei":244,"hli":24," Eg":106," Et":209," Es":211," Er":900,"hlo":42," Ep":30," Eo":336," En":713," Em":541," Ez":35," Ex":30," Eu":753," Ev":283," Fe":315,"ho ":375,"hma":28," Fa":314,"gma":28,"go ":326," Xi":32,"glo":131," Xa":37,"к ":31,"gle":234," Wy":25,"gli":102,"gn ":547,"gla":326," Wo":96," Wi":334," Wh":28," Sã":44," We":489," Wa":422,"й ":42," Y ":37,"gog":60," Vî":53,"goe":139," Zu":66,"god":130,"gob":25,"goa":87," Zo":34,"gnu":34," Ze":158,"gno":502," Zi":46,"gni":42," Za":103," Ys":24,"gne":835," Yv":36," Yu":107,"gna":107," Yo":219," Ya":330,"giñ":44," Ye":198,"gs ":28," Wü":24,"goz":125,"gom":197,"н ":39,"gol":442,"gon":445,"gop":26,"gos":176,"gor":446,"gou":1457,"got":31,"gu ":30," a ":21743,"Ñ€ ":30,"gro":187,"gra":972,"gri":140,"gre":811,"в ":40," Kö":34," Ow":50," Ou":210," Os":107,"gto":52," Ot":70," Or":316," Op":33," Po":935,"gui":62," Lé":26," Pl":484,"gum":1179," Pi":431,"gul":63," Ph":183," Pf":25,"gua":208," Pe":1214,"gue":151," Pa":1928," Ny":33," Nu":54," No":1153," Ol":141," Ok":98," Kê":117," On":29," Om":48," Og":35," Oi":37," Od":68," Oc":46," Of":53,"goù":281," Ob":100,"gta":158," Oa":47," Ra":453," Mü":57," Qu":112,"gwi":305," Ro":1377,"gwe":769," Re":1194,"gwa":450," Ri":357," Rh":169," Py":52,"guz":57," S ":53," Pr":893,"gur":63," Ps":36,"gus":176," Pt":32," Pu":129,"gun":35," Qi":24," Mé":39," Sy":84," Sw":45," Sv":127," Su":797," St":1757," Tc":179," Ta":642," V ":60," Th":473," Ti":377," Te":498," Tr":768," Pê":42," To":425,"gwr":80," Ru":419," Sa":2186," Sh":170," Si":919," Sc":261," Se":617," So":661," Sp":344," Sr":25," Sk":511," Sl":64," Sm":46," Va":397,"и ":29," Ve":661," Vi":639," Vl":43," Ré":29," Rí":30," Vo":153," Vu":27," Vr":262," Tu":257," Ty":52," Uh":133," Uk":44," Ul":286," Un":1810," Ur":2995," Ut":31," ja":187,"iak":59,"ÑŒ ":37," l ":123,"iam":250,"ial":435,"iao":55," iz":369,"ian":2252," ji":41,"ias":133,"iar":101," je":186,"iat":251,"iav":48," im":1409," in":589," il":111,"ic ":292," ij":104," iw":110," eñ":47,"iab":139,"iac":49," iv":1031,"iad":1539," is":1920," it":397,"iae":74,"iag":66," ir":67,"ibl":171,"ibi":176," ka":2835,"ibo":69," m ":227," kh":68,"ibr":42," ki":623,"ibu":38," ke":3570,"iaz":346," jo":32," jp":45,"id ":450,"iba":203,"ibe":842," ju":73," gw":1502," ha":17502," he":5230," gi":303," gl":364," gr":1597," go":1953," gu":1259,"ia ":5683," ib":41," id":83," hh":31,"ib ":56," hi":2337," hl":138," dé":33," ho":3333," hr":812," hs":28," ht":49," hu":259," hv":30," hw":461,"iet":779,"iañ":362,"ieu":132," ni":874,"iez":2096," ng":65," nd":30,"iel":957,"iem":34," ne":1817,"ien":1604," na":567," p ":133,"ier":945,"ies":635,"ied":391,"ieg":782,"iek":656," mu":464,"ig ":786," mo":1518," ok":126," ol":37," om":64," on":28," kê":858,"ifo":196," oc":81," of":821," oa":4658,"ifr":40," ob":921,"ife":146," ny":25,"ifi":60," nu":87," no":744,"ifa":82," le":3053,"icr":29,"ics":47,"ict":94,"icu":113," li":1853," n ":401,"ico":205,"ick":115," la":2328," ku":562,"ici":153,"ich":835,"icc":26,"ice":192," kw":51," gê":983," km":359,"ie ":398," kl":472,"ica":324," kr":1806," ko":5162," me":2877," mi":1053,"idu":47,"idt":41," o ":1644,"idr":25,"Ñ ":78,"ido":268," ma":4658," mb":31,"idj":93," lu":242,"idi":732,"idg":33,"ide":470,"idd":28,"ida":806," lo":922," hê":99,"if ":38," ae":329," af":85," ag":91," aa":47," ab":707," ac":52,"iid":64," ad":591," am":775," an":17401," ao":767," ap":227," ai":30," ak":164," al":3609," av":170," au":58," aw":48," ar":20177," at":205," as":260," d ":4135," ba":1861," az":121," ay":27,"il ":1052,"ija":76," bi":786,"ije":1542," be":6354,"iji":263,"ijk":37," bo":1153," bl":938,"ijo":76," bu":368," br":3925," ca":216," e ":25254,"im ":142,"ika":1513,"igd":59,"ige":995,"iga":163,"ii ":140,"igl":24,"igh":464,"igi":116,"igu":89,"igr":67,"igo":308,"ign":346,"ij ":175," b ":27,"ihe":29,"iha":639,"ik ":1216," c ":4919," er":3046,"imo":147," et":1614,"imm":32," es":229," en":8597," em":1862,"imp":1483," eo":8201," ei":428," el":392,"ime":314," ek":63," ee":65,"imi":166," eg":182," fe":423,"ip ":40,"inc":134," h ":3324,"ind":570,"ina":1494,"inb":38," fa":598," ez":592,"imu":37," eu":10144," añ":114," ev":3777," fu":86,"inn":252,"ino":647," fr":528,"int":619," fo":762,"ins":144,"inf":56,"ine":1835," fl":235,"inh":55,"ing":949," fi":471,"ini":1294,"inl":79,"ink":143," ge":2883," ga":11426,"ioc":83,"iod":24,"inu":125," i ":72,"inv":58,"inw":47,"iny":86,"inz":27," cl":32,"iko":750," cm":63,"ikl":38," co":306," cr":36,"iki":142," ce":115," ch":782,"ike":750," ci":27,"ila":338,"ilb":25," da":8512,"in ":2762," cu":44,"ikt":69,"iku":27,"iks":110," do":1895,"ilo":265,"ill":771," dr":4372,"ilm":190,"ilh":762,"ilg":84," de":7284,"ili":920,"ild":111," dj":44," di":6211,"ile":746,"ima":192,"imb":131," g ":57,"ч ":54," ec":181," ed":82,"io ":457," ea":36," eb":166," du":1193,"ilt":54,"ilu":55,"ilv":37,"ль":30,"ми":24," zo":10353,"ла":32," zu":52,"hpe":153,"ле":32," vî":50," zr":49,"ли":31,"hok":47,"hol":1123,"hom":566,"hon":2468,"hog":52," za":226,"ко":31,"hos":71," ze":301,"hot":72,"hou":757,"hov":24,"hop":42," zi":383,"hor":720," yo":88,"ка":66," yi":44,"hoa":863," yu":97,"ки":34,"hof":24,"hoe":202,"hod":73,"hoc":104," ye":6259,"hno":26," ya":550," z ":116,"ин":40,"hmi":26,"ик":30,"ий":28,"ич":63,"иÑ":27,"ри":28,"hul":35,"hui":68,"hug":38,"ро":50,"hue":39,"hud":36,"hua":110,"htt":30,"htr":24,"hth":32,"hte":44,"hoù":3009,"ор":27,"hsk":47,"hse":48,"ол":26,"ом":26,"ов":76,"hu ":78,"но":24,"hro":118,"hre":688,"ни":34,"hri":212,"ht ":446,"на":57,"hra":123," ru":826," u ":26," sa":3355," se":1968," sc":29,"hya":32," si":1161," sh":141,"hyn":30," sl":74," sk":4322," sp":1119," so":1278," qu":41,"ви":64," t ":34," ra":3095," re":3577," ri":679," ro":2258," pu":120," pr":1459," s ":93," px":514,"hy ":25,"hwa":53,"hwe":448,"hwi":38," ot":171,"hum":675," ou":1145,"hun":195,"hus":124," op":32," or":427,"hur":95,"ан":44," r ":35,"huz":52," pe":8359," lâ":203,"hve":93," pa":1775,"ар":50," pl":522," po":1472,"аÑ":51," pi":232," ph":49," wa":3108," we":680," y ":92," wr":58," wo":36," wi":155," x ":44," va":783," ve":5168," vo":2288," vr":1722," vu":274," vi":3544," vl":184,"еÑ":33," tu":746," us":51," ur":9744," un":6030," uk":30," ul":727," ug":49," uh":238," ta":1167," tc":512," v ":32,"hys":42," sy":28," st":3800," sv":112," su":738,"ев":42," tr":3171," ts":55," to":979," th":740," ti":1088," te":1484,"ffu":30,"ffr":34,"fi ":42,"ffe":36,"ffa":40,"ffi":49,"feu":64,"fet":66,"fes":65,"fer":173,"fed":176,"fem":46,"fen":488,"fel":95,"fei":43,"faz":24,"fas":26,"far":201,"fam":247,"fan":218,"fal":202,"faf":76,"fae":67,"fad":24,"ff ":48,"fe ":207,"fa ":121,"euß":32,"eye":26,"exa":64,"ez ":11148,"exi":24,"ezv":39,"ezu":194,"eza":980,"ezo":388,"ezs":35,"eze":3326,"ezh":15689,"ezi":1049,"eta":840,"ete":2318,"etc":33,"eti":591,"eth":157,"etn":58,"etk":85,"esp":93,"esn":37,"eso":125,"est":1767,"ess":313,"esw":24,"eoù":227,"ev ":1299,"eua":81,"añc":45,"euc":54,"eub":98,"eue":418,"eud":347,"eui":37,"euk":25,"añj":24,"eum":58,"eul":367,"eun":223,"eto":294,"etr":1675,"ets":34,"ett":196,"etu":68,"ew ":174,"eve":3071,"eva":742,"evo":119,"evn":657,"evi":2428,"añv":131,"euv":34,"euw":32,"eut":225,"eur":2890,"añs":950,"eus":10810,"ex ":155,"euz":639,"eux":42,"ewi":41,"ewe":45,"evr":1345,"ey ":241,"ewa":59,"epe":148,"epi":164,"eph":135,"er ":13257,"epa":1461,"eou":101,"eot":99,"eos":32,"eor":152,"eom":25,"eol":459,"eop":50,"eon":456,"aï ":27,"es ":2160,"ept":42,"epu":540,"epk":92,"epm":31,"epp":25,"erk":270,"erl":341,"eri":3029,"erg":379,"erh":59,"ere":4269,"erf":51,"erc":973,"erd":244,"era":623,"erb":525,"et ":24274,"esk":805,"esl":29,"esh":133,"esi":729,"esb":38,"esc":99,"esd":28,"ese":255,"añ ":8904,"eu ":196,"esa":1077,"erz":1304,"ery":87,"erv":1357,"eru":118,"erw":40,"err":913,"ert":421,"ers":473,"ern":1502,"erm":1055,"ené":32,"erp":58,"ero":907,"eki":62,"ekk":24,"ekn":35,"eko":95,"ekr":46,"eks":87,"ekt":111,"eku":26,"ekv":40,"en ":13354,"elb":25,"ela":722,"eld":150,"elc":91,"elf":231,"ele":2592,"eli":634,"elg":331,"elh":32,"elm":94,"elk":40,"ell":2765,"elo":483,"elp":30,"elu":36,"elv":411,"els":97,"elt":214,"ely":60,"elw":63,"eo ":8202,"emb":1318,"ema":1014,"emg":174,"eme":1638,"emd":78,"emm":295,"emo":323,"emi":315,"eiñ":116,"emp":387,"emr":84,"ems":145,"ep ":539,"emz":52,"enf":38,"ene":2818,"enh":96,"eng":662,"enb":206,"ena":1493,"end":451,"enc":313,"eno":539,"enn":14198,"enk":534,"enl":51,"eni":918,"enw":44,"enu":60,"env":509,"ens":884,"ent":3408,"enr":123,"enz":173,"eny":79,"eog":63,"eof":24,"eod":154,"eoc":58,"ego":514,"ege":1345,"egg":36,"egi":351,"eha":29,"egr":44,"egu":61,"eho":118,"ehe":119,"ek ":10245,"eib":27,"eic":80,"eia":58,"eis":824,"eir":312,"eim":59,"eil":495,"ein":413,"eik":24,"eie":159,"eid":119,"eig":28,"eif":24,"eja":35,"el ":8480,"eiz":3938,"eit":125,"eke":323,"eka":196,"em ":600,"ejv":55,"gl ":37,"giz":64,"geñ":32,"git":101,"gis":73,"gir":65,"gil":150,"gim":29,"gip":131,"gin":251,"gio":137,"gid":69,"gie":386,"gia":396,"ght":390,"ghe":49,"gha":96,"ggi":48,"gge":26,"gga":25,"gi ":107,"gen":1791,"geo":127,"get":861,"geu":25,"gañ":198,"ger":1658,"ges":96,"gev":466,"gh ":112,"gez":763,"gea":34,"gee":129,"ged":397,"gei":27,"geg":174,"gef":25,"gem":219,"gel":631,"gek":101,"gda":29,"gde":64,"ge ":374,"gaz":228,"gab":25,"gac":227,"gad":295,"gae":60,"gai":52,"gas":335,"gar":551,"gau":35,"gat":92,"gaw":34,"gav":257,"gak":29,"gam":131,"gal":2464,"gao":44,"gan":9608,"ga ":256,"fyd":28,"fur":99,"foù":33,"fud":31,"ft ":72,"fra":395,"fre":158,"fri":574,"fro":118,"fou":46,"for":444,"fon":658,"fol":97,"fiñ":47,"fle":49,"fla":138,"flu":41,"flo":72,"fic":49,"fie":47,"fil":220,"fin":188,"fis":652,"fiz":82,"da ":6428,"dd ":287,"de ":2341,"dac":243,"dad":448,"dak":29,"dal":1050,"dai":58,"dag":91,"dah":25,"dae":431,"dat":150,"das":255,"dar":633,"dap":31,"dao":544,"dan":1203,"dam":247,"daz":25,"day":28,"dav":252,"dda":76,"dde":59,"ddi":92,"cul":78,"cum":26,"cua":46,"ctu":25,"cto":91,"cti":132,"cte":35,"cta":38,"cus":153,"cur":31,"cks":24,"cki":24,"ckh":38,"ckl":31,"cla":32,"cle":35,"co ":254,"ció":40," Ös":26,"coc":27,"con":207,"col":208,"com":88,"cor":91,"cos":24,"cop":59,"cot":43,"cou":64,"cs ":52,"cqu":76,"cre":28,"cra":43,"cri":37,"cro":75,"cci":49,"cch":32,"cca":25,"cea":54,"ch ":1041,"cer":68,"ces":90,"cen":91,"cep":71,"cel":179,"ci ":114," à ":31,"cha":1216,"chw":69,"chu":220,"chy":27,"cia":161,"ck ":309,"cie":45,"cid":63,"che":1551,"chl":55,"chi":619,"cho":838,"chm":39,"chn":35,"chs":66,"cht":150,"chr":27,"cil":43,"cis":63,"ciu":47,"cin":138,"cio":74,"cip":52,"cm ":60,"ckg":40,"cke":49,"ed ":5626,"eba":92,"ebe":535,"ebi":41,"ebl":61,"ebo":59,"ebr":613,"ebu":44,"ec ":2202,"eac":38,"eae":30,"ead":335,"eaj":43,"ean":370,"eal":95,"eam":25,"ear":95,"eas":306,"eat":155,"eau":133,"eb ":70,"ea ":600,"efi":24,"efl":30,"efo":31,"efa":48,"efe":192,"ei ":147,"ega":544,"efr":31,"eft":47,"eek":223,"een":103,"eel":114,"eeg":74,"eed":85,"eez":83,"eer":364,"eeu":74,"eañ":33,"eet":55,"edi":547,"edd":184,"ede":1648,"ône":24,"eda":435,"eg ":10353,"ης ":30,"edu":46,"edo":498,"edr":110,"eck":63,"ech":877,"eci":30,"eca":36,"ee ":124,"ef ":54,"ecu":28,"ect":87,"eco":53,"dwi":54,"dwa":71,"dy ":65,"dve":63,"dur":2497,"dus":115,"ôge":65,"ôg ":650,"dor":380,"don":562,"dom":53,"dol":90,"dok":40,"dov":47,"dou":1198,"dot":33,"dos":174,"dr ":37,"ds ":105,"diñ":81,"dne":34,"doa":651,"dob":26,"doc":36,"doe":85,"dof":83,"dog":66,"doù":1790,"dst":102,"dth":36,"dun":88,"dui":24,"dul":36,"due":53,"dug":195,"dua":33,"dud":657,"dri":262,"dra":338,"dt ":63,"dre":2258,"du ":415,"dro":2685,"dru":64,"dge":30,"dic":158,"did":92,"dia":1269,"dib":212,"dhi":25,"der":1355,"des":551,"det":713,"deu":1098,"dañ":173,"dev":161,"dez":1065,"dh ":36,"deb":164,"dea":43,"ded":268,"dec":29,"dee":39,"deh":79,"deg":839,"dei":311,"del":1180,"dek":575,"den":4056,"dem":143,"dep":1276,"deo":109,"di ":921,"dle":38,"dla":33,"do ":378,"dja":113,"dje":32,"div":643,"diu":39,"diw":1350,"diz":227,"dim":106,"din":658,"dio":629,"dir":425,"dis":1432,"dit":85,"die":331,"dif":129,"dig":697,"dij":34,"dik":146,"dil":186,"djo":47,"dji":63,"rgu":84,"rhe":42,"rha":43,"rho":37,"rga":269,"ri ":1054,"rgi":254,"rgh":25,"rge":313,"rgr":39,"rgo":143,"rgn":33,"ret":1781,"res":1419,"rev":451,"reu":637,"rañ":634,"rew":29,"rez":4363,"rey":46,"rfa":34,"rfe":82,"rfo":105,"rdu":42,"rds":36,"rdr":49,"rg ":501,"reb":334,"rea":765,"ree":432,"ref":61,"rec":185,"red":2063,"rei":3864,"reg":1192,"rem":696,"ren":3754,"rek":863,"rel":1453,"raí":24,"rer":622,"reo":489,"rep":96,"rf ":139,"rda":144,"rcu":51,"rct":28,"rdo":204,"rdi":421,"rde":436,"rdd":43,"re ":4982,"rbu":43,"rco":34,"rck":34,"rci":70,"rch":168,"rce":81,"rca":52,"raw":45,"rax":26,"ray":119,"raz":505,"rd ":675,"rao":698,"rap":82,"rar":100,"ras":1323,"rat":510,"rau":85,"rav":268,"rbi":84,"rbo":149,"rba":160,"rbe":368,"rc ":1530,"raj":38,"rai":247,"rah":111,"rag":290,"ran":3063,"ram":504,"ral":609,"rak":382,"rab":518,"raf":167,"rae":1511,"rad":877,"rac":186,"rs ":209,"rpe":55,"rpa":30,"rr ":2111,"rpi":29,"ror":29,"ros":301,"rot":175,"rom":591,"ron":1309,"roo":54,"rop":562,"roy":41,"roz":110,"rou":3431,"rov":876,"row":38,"rob":47,"roa":1313,"rod":180,"roc":330,"roi":658,"rol":1503,"rok":136,"rof":62,"roe":745,"rog":203,"rno":714,"rns":59,"rp ":29,"rna":543,"rne":675,"rni":416,"rmo":173,"rms":31,"riñ":374,"rmu":93,"ro ":7159,"rma":433,"rme":1192,"née":33,"rmi":74,"rlu":43,"rlo":138,"rli":304,"rle":290,"rla":227,"rn ":1159,"rkt":34,"rks":233,"rko":43,"nço":78,"rki":200,"rke":237,"nça":45,"rka":144,"rm ":168,"né ":43,"rji":52,"riy":25,"riz":637,"rix":34,"rl ":114,"rip":57,"rio":1388,"rit":468,"ris":758,"riv":1682,"reñ":136,"riu":84,"rig":797,"rij":35,"rii":29,"ril":176,"rik":1404,"rin":891,"rim":123,"ria":2221,"rib":276,"ric":406,"rid":500,"rie":2396,"rif":33,"rhy":25,"rk ":341,"rwe":28,"rwi":26,"rz ":165,"nôg":719,"rxi":44,"rya":37,"ryd":24,"rye":33,"ruj":46,"rui":38,"rug":115,"ruf":51,"rud":504,"ruc":43,"run":197,"rum":500,"ruz":176,"rus":640,"rut":37,"rva":250,"rvi":144,"rve":2286,"rvo":598,"rvu":116,"rwa":33,"ry ":297,"rsk":47,"rsi":111,"rso":78,"rsa":75,"rsh":36,"rse":187,"rta":1407,"rv ":292,"roù":324,"rst":71,"rsu":30,"rto":172,"rte":398,"rth":232,"rti":358,"rub":27,"rts":30,"rtr":52,"rtu":274,"rty":48,"rt ":661,"rqu":49,"rro":294,"rri":521,"rre":814,"rra":478,"ru ":88,"rry":64,"rnô":734,"rru":33,"saa":40,"sab":110,"sac":68,"sad":583,"sae":48,"sah":202,"sai":49,"sak":47,"sal":151,"sam":325,"sba":28,"sbe":56,"sao":1562,"san":655,"sau":75,"sat":83,"sas":62,"sar":136,"sav":1196,"sa ":294,"ón ":120,"rze":238,"rza":44,"rys":28,"ryn":36,"rzu":242,"rzo":285,"rzh":3355,"rzi":42,"sha":218,"sho":75,"she":193,"shi":238,"si ":134,"sga":31,"siz":115,"siv":61,"sie":664,"sid":67,"sic":44,"sia":1269,"sk ":453,"shu":43,"sit":87,"siu":25,"sir":71,"sis":396,"sip":30,"sin":816,"sio":143,"sil":161,"sim":118,"sik":81,"sif":26,"sig":38,"scu":28,"sde":25,"sbo":44,"sbu":48,"se ":1133,"sca":87,"sce":27,"sci":63,"sch":258,"sco":151,"sev":298,"sex":103,"sey":30,"ser":460,"ses":45,"set":371,"seu":286,"sañ":784,"sez":268,"sh ":215,"sfe":27,"sea":33,"sei":175,"seg":263,"see":27,"sed":76,"sec":35,"seb":48,"sep":184,"sen":385,"sem":202,"sel":465,"sek":164,"spo":97,"spr":39,"spe":713,"spl":66,"spi":316,"spa":666,"sot":45,"sou":522,"sov":37,"sol":98,"som":49,"son":1595,"sor":66,"sos":59,"sok":83,"soc":102,"su ":474,"oñ ":47,"sra":211,"st ":1732,"squ":36,"ss ":102,"slo":67,"sla":311,"sle":41,"ski":664,"skl":119,"sko":2999,"skr":1826,"skw":33,"sm ":26,"ska":795,"ske":666,"sno":26,"sne":81,"smo":24,"smu":27,"so ":138,"sma":85,"sme":47,"swi":31,"stê":656,"syl":43,"où ":12664,"sse":388,"ssa":244,"sso":146,"ssk":287,"ssi":196,"soñ":40,"soù":83,"ste":2322,"sth":144,"sta":2593,"sto":981,"sti":600,"stl":193,"stu":875,"str":2529,"sua":28,"sty":49,"sub":34,"oñi":37,"sul":41,"sum":35,"oñj":48,"suk":24,"sup":43,"sun":37,"oñs":58,"sus":65,"sur":177,"sve":118,"swa":45,"tai":326,"tak":108,"tal":1349,"tae":282,"taf":24,"tag":929,"taa":126,"tab":88,"tac":107,"tad":2010,"taz":91,"taw":31,"tav":87,"tau":39,"tat":200,"tas":62,"tar":434,"tap":82,"tao":273,"tan":1117,"tam":1603,"tch":742,"te ":835,"tde":54,"ê°€":25,"ta ":886,"pa ":891,"pe ":5856,"ñs ":1564,"par":1873,"pat":120,"pas":123,"pav":27,"pac":192,"pad":413,"pab":42,"pag":683,"pak":52,"pal":693,"pai":106,"paj":24,"pao":79,"pap":202,"pan":368,"ñve":535,"phe":74,"pha":123,"ñvo":30,"phu":39,"phr":28,"pho":83,"phi":104,"pi ":45,"ñva":108,"ph ":70,"peu":284,"pev":140,"pez":279,"lâr":206,"pg ":46,"pea":32,"pec":47,"ñse":342,"ped":132,"ñso":78,"ñsk":24,"pen":1324,"pem":148,"pep":93,"peo":50,"per":657,"pet":110,"pes":749,"peg":57,"pei":35,"ñv ":186,"pel":375,"pek":88,"pla":359,"pli":1393,"ple":204,"plo":62,"plu":79,"lé ":29,"pke":90,"piz":28,"pl ":35,"phy":24,"pia":162,"pid":57,"pic":35,"pie":64,"pig":24,"pik":145,"pil":44,"pin":212,"pio":73,"pir":52,"pis":281,"pit":78,"poz":26,"por":318,"pop":53,"pou":712,"pot":112,"pos":143,"poi":42,"pon":117,"pol":852,"pob":151,"poe":85,"pod":72,"ps ":90,"ppi":46,"ppo":48,"ppa":34,"ppe":129,"pmu":27,"léo":25,"po ":44,"lêr":32,"poù":33,"pta":38,"pse":81,"psi":47,"pub":539,"pte":76,"pti":106,"pto":37,"pra":108,"pt ":114,"pri":323,"pre":639,"pro":765,"pur":73,"pus":78,"pun":48,"pui":25,"pul":44,"pve":49,"px ":512,"ñch":70,"ñie":36,"ñje":37,"qua":38,"que":308,"qui":93,"ra ":1887,"rb ":171,"ngo":851,"ngi":105,"ngl":237,"ngk":32,"ngu":135,"ngw":30,"ngr":91,"ngt":218,"ngs":57,"ni ":1357,"nge":730,"ngh":68,"nga":534,"nho":63,"ndé":24,"nha":101,"nhe":40,"neg":4522,"nei":121,"nel":551,"nek":2286,"nen":946,"nem":826,"nep":201,"neo":130,"ner":1985,"net":4017,"nes":379,"nev":416,"neu":170,"nañ":1277,"ndy":43,"ng ":876,"nea":366,"neb":309,"nec":60,"ned":1326,"nfo":32,"nfr":50,"ney":67,"nez":3411,"new":34,"nh ":31,"nfa":53,"nfe":39,"nct":33,"nco":81,"nci":162,"nce":259,"nch":229,"nca":55,"ne ":1685,"nbu":121,"ndu":84,"ndr":785,"nds":105,"ndo":360,"ndj":35,"ndh":32,"ndi":1021,"nde":859,"nda":686,"nak":317,"nal":306,"nam":483,"nan":2654,"nao":557,"nap":104,"nar":273,"nac":195,"nad":1521,"nae":258,"naf":53,"nag":110,"nai":155,"nc ":218,"nab":40,"naa":59,"nbo":33,"nbe":67,"nd ":930,"nba":62,"nav":633,"nau":104,"nat":458,"nas":202,"naz":46,"nay":25,"naw":28,"na ":1960,"mys":60,"mze":3020,"mza":29,"mzo":28,"ión":47,"mz ":627,"nya":139,"nye":172,"nyd":33,"nyi":90,"nwy":29,"nz ":54,"ny ":96,"nwe":103,"nwa":28,"nvi":206,"nvr":769,"nve":1542,"nva":229,"nuk":42,"num":58,"nun":42,"nus":273,"nut":34,"nuo":31,"nur":33,"nue":31,"ntv":525,"nto":1560,"ntu":79,"nts":43,"ntr":315,"nti":1416,"nth":89,"noù":2243,"nv ":3093,"nta":1595,"nte":2871,"nsu":30,"nso":629,"nst":225,"nse":99,"nsh":34,"nsi":150,"nsl":25,"nsk":93,"nsc":24,"nsa":96,"nu ":32,"nro":55,"nri":74,"nre":51,"nra":38,"nt ":12714,"ns ":309,"noc":89,"nod":56,"noa":38,"nob":77,"nog":78,"noe":57,"nol":692,"noi":46,"nop":48,"nom":155,"non":174,"not":100,"nos":158,"nor":724,"nov":93,"nou":960,"noz":603,"nne":3434,"nnf":24,"nng":83,"nna":2167,"nnb":29,"nnl":257,"nno":2232,"nni":788,"nnu":55,"nnw":29,"nnv":750,"nns":30,"nny":190,"nma":220,"niñ":141,"nn ":8975,"nla":142,"nle":246,"no ":590,"nke":412,"nkl":44,"nki":148,"nka":174,"nko":102,"nji":28,"nje":58,"nja":37,"nij":187,"nii":52,"nig":259,"nif":46,"nie":1515,"nid":227,"nic":163,"nib":35,"nia":931,"nk ":577,"niz":425,"niu":62,"niv":452,"nis":490,"nit":167,"nio":561,"nip":48,"nim":73,"nin":410,"nik":222,"nil":245,"ogr":98,"ogu":47,"ogw":32,"ogi":341,"ogh":25,"ogl":28,"ogo":182,"ogn":136,"oga":327,"oge":333,"oi ":52,"ohn":97,"oha":110,"ohe":80,"ois":208,"oir":111,"oit":33,"oio":472,"oin":74,"oic":40,"oid":101,"oie":65,"ok ":497,"oia":64,"oji":26,"oje":210,"ol ":836,"oce":86,"och":219,"oci":82,"ock":164,"oco":62,"oe ":1455,"oca":59,"ode":729,"odi":133,"odo":481,"odr":62,"îng":102,"of ":276,"oda":139,"oek":523,"oel":99,"oeg":409,"oei":96,"oer":123,"oet":567,"oen":280,"ody":27,"odu":112,"oed":76,"og ":209,"ofi":656,"ofo":33,"off":41,"ofe":71,"ofa":98,"oa ":4888,"ob ":87,"oc ":1030,"oan":167,"oal":46,"د ":37,"oad":1167,"oab":38,"oba":84,"oaz":875,"od ":515,"oav":413,"oar":1366,"oas":78,"oat":178,"obo":44,"obr":26,"obl":489,"obi":75,"obe":1082,"Ø© ":62,"nza":121,"nze":101,"nzi":83,"nzo":54,"oye":54,"oya":58,"ã‚¢ ":34,"oz ":1606,"owy":28,"osé":34,"own":58,"owi":26,"ozn":946,"ozo":150,"oze":284,"ozh":546,"ozi":67,"oza":314,"oyo":26,"otu":34,"oud":1277,"oue":5169,"oub":196,"ouc":182,"oua":2059,"ow ":55,"oti":138,"oth":123,"ote":529,"ott":209,"ots":48,"otr":253,"oto":262,"ost":1461,"osu":49,"ota":312,"otc":28,"otd":52,"ov ":88,"osi":113,"osh":114,"osk":89,"ose":258,"oss":101,"osl":24,"oso":58,"oy ":63,"owa":47,"owe":66,"ovi":977,"ovo":29,"ouv":131,"ouz":1500,"ova":237,"ove":156,"oug":359,"ouf":27,"oui":317,"ouh":91,"ouk":182,"oum":209,"oul":878,"oun":651,"oup":85,"ous":409,"our":6950,"out":1285,"opo":193,"opi":160,"ope":218,"oph":160,"opa":474,"os ":1217,"opu":67,"opr":35,"opt":102,"ops":55,"ool":28,"ook":38,"ood":29,"or ":2242,"oou":106,"oot":29,"oor":62,"ork":157,"orl":90,"orm":333,"orn":1523,"oro":324,"orp":42,"orr":385,"orc":140,"ord":621,"ore":615,"orf":322,"org":321,"orh":51,"ori":818,"orj":52,"ou ":989,"osa":126,"osc":72,"ort":651,"ors":149,"orv":169,"oru":79,"orz":1153,"ory":50,"m² ":87,"ot ":217,"orb":59,"ora":374,"oqu":24,"ola":324,"old":111,"olc":37,"on ":5066,"oli":1224,"oll":2546,"olk":71,"olf":87,"ole":1214,"ols":36,"olt":143,"olm":61,"olo":1329,"olp":55,"oly":42,"olz":49,"olu":107,"oka":179,"om ":504,"oki":242,"oke":140,"okr":149,"oks":49,"oko":156,"oku":24,"ona":884,"ond":452,"onc":43,"onf":34,"one":3599,"ong":560,"oni":1468,"onk":97,"onn":763,"ono":501,"kêr":864,"onr":43,"ons":267,"ont":1954,"onu":48,"onv":33,"onw":26,"ony":75,"onz":24,"oma":931,"ome":375,"omb":242,"omi":259,"omm":204,"omp":243,"omo":225,"omt":26,"omu":437,"omy":53,"op ":79,"omz":3124,"la ":919,"ín ":37,"ío ":34,"le ":1172,"lch":45,"lf ":74,"lde":258,"lda":59,"ldo":59,"ldi":82,"lc ":543,"laa":65,"lab":398,"lac":162,"lad":1652,"lae":620,"lah":63,"lag":146,"laj":30,"lai":415,"lak":509,"lan":2333,"lam":1300,"lap":113,"lao":231,"lar":488,"lat":874,"las":665,"law":53,"lau":121,"lav":584,"lay":168,"laz":407,"lba":109,"ld ":178,"lbe":121,"lbi":34,"lbo":51,"lbr":32,"kve":49,"kuz":54,"kus":27,"kur":73,"kum":373,"kul":55,"kuk":29,"kwe":50,"kwa":63,"ky ":36,"kta":48,"kte":66,"koù":73,"kst":110,"ksi":64,"ksh":31,"kso":236,"kui":61,"ktr":79,"kti":78,"kto":145,"lpo":25,"llé":30,"lpe":41,"lph":43,"ls ":64,"lol":35,"lok":82,"lon":530,"lom":263,"lop":132,"lor":169,"lod":397,"loc":191,"loe":311,"log":539,"loi":42,"lpa":108,"los":243,"lot":351,"lou":939,"lov":116,"low":33,"loz":39,"loa":974,"lob":38,"hêr":108,"lmo":82,"lme":64,"lma":126,"liñ":39,"lth":34,"lti":120,"lto":44,"ltr":148,"ltu":28,"lud":25,"luc":61,"lub":38,"lua":26,"lue":102,"lsi":26,"lsk":38,"lso":26,"lsp":40,"lst":74,"lv ":131,"loù":329,"lta":234,"lte":150,"lu ":99,"ía ":69,"lre":43,"lt ":168,"lho":64,"lhe":195,"lha":164,"lgr":51,"lgo":96,"lge":35,"lgi":310,"li ":426,"lga":89,"lfr":35,"hât":29,"lfo":49,"lfi":33,"lfe":223,"lh ":403,"lez":1792,"ley":71,"lex":59,"leu":303,"lañ":440,"lev":593,"les":1142,"let":618,"ler":868,"leo":157,"lep":113,"lem":177,"len":2262,"lek":1292,"lel":46,"lei":313,"lej":65,"leg":1616,"lef":51,"lee":58,"led":429,"lec":1300,"leb":38,"lea":170,"llp":41,"lls":74,"llu":48,"llt":30,"llv":37,"lly":81,"lo ":582,"lla":2233,"lld":27,"lle":2420,"llg":38,"lli":505,"llk":31,"llo":648,"lko":32,"lka":49,"lke":60,"lki":37,"lm ":216,"lje":160,"ll ":4679,"lit":627,"lis":471,"lir":32,"lip":162,"lio":552,"lin":703,"lim":91,"liz":963,"liv":703,"leñ":35,"liu":80,"lic":150,"lid":231,"lia":1696,"lib":56,"lk ":62,"lik":678,"lil":31,"lij":1519,"lig":150,"lie":910,"lif":182,"ma ":1461,"mb ":591,"mac":81,"mab":327,"mah":35,"mai":110,"maj":31,"mak":56,"mad":427,"mae":311,"mag":364,"mar":2123,"mas":161,"mal":298,"mam":124,"man":3821,"mao":145,"maz":94,"may":26,"maw":44,"mat":618,"mba":789,"mbl":109,"mbi":183,"mbe":196,"mbr":812,"mbo":192,"me ":317,"mbu":84,"mde":59,"mdr":39,"med":147,"mee":66,"meg":191,"mea":32,"mañ":1356,"meu":785,"met":1513,"mes":334,"mer":2018,"mem":72,"mel":331,"meo":40,"men":1920,"mei":92,"mek":209,"mfe":26,"mez":216,"lva":154,"lve":509,"lvi":71,"luk":41,"lup":28,"lun":99,"lum":66,"lut":69,"lus":336,"lur":47,"ly ":109,"lwe":65,"lvo":48,"lvr":387,"lz ":240,"lwy":34,"lyd":32,"lzi":27,"lza":56,"lze":26,"lyw":60,"lym":32,"lyn":84,"mpi":89,"mph":46,"mpe":243,"mpr":173,"mpo":95,"mpl":1375,"mpv":50,"mps":40,"ms ":55,"mog":45,"moc":36,"moe":45,"mod":89,"mon":521,"mok":123,"moj":168,"mol":78,"mor":813,"mos":78,"mot":89,"mou":587,"mpa":496,"mre":137,"msa":83,"mu ":25,"iñ ":870,"ία ":28,"mst":158,"msk":28,"moù":263,"my ":29,"mur":71,"mus":152,"iñs":1025,"mut":27,"iñv":102,"mui":254,"mul":102,"mun":2223,"muz":74,"mgl":91,"mga":206,"mi ":131,"min":284,"mio":42,"mil":696,"mir":198,"mis":93,"mit":243,"miz":272,"mic":110,"mib":47,"mia":86,"mig":63,"mie":196,"mid":59,"mik":104,"mo ":123,"ièr":50,"ièg":36,"mka":24,"mm ":956,"moa":84,"iêt":40,"mno":39,"mp ":171,"mmu":105,"mmi":51,"mmo":164,"mma":602,"mme":253,"zre":26,"zro":39,"vîn":50,"zte":64,"zoù":296,"zu ":289,"zst":24,"zsk":181,"zve":38,"zue":58,"zul":186,"zun":25,"zuz":70,"zus":84,"zzo":29,"アア ":34,"zgi":47,"zi ":507,"zhi":1481,"zhe":1520,"zha":1503,"zed":202,"zeg":368,"zei":32,"zeb":38,"zdo":72,"zh ":14328,"zev":108,"zañ":742,"zeu":132,"zet":3672,"zes":33,"zez":352,"zen":1263,"zel":1371,"zek":305,"zer":980,"ze ":719,"zab":47,"zad":456,"zac":32,"zbe":30,"zam":89,"zan":193,"zak":37,"zal":175,"zar":107,"zao":47,"zap":54,"zas":136,"zat":105,"zoa":32,"zor":53,"zon":351,"zol":126,"zou":503,"ال":111,"zo ":10196,"ziñ":115,"zne":946,"vé ":24,"zka":30,"zhp":154,"zho":5307,"zhu":132,"zhs":28,"zhv":75,"zia":766,"zif":31,"zie":248,"zid":445,"zig":100,"zin":109,"zim":41,"zil":353,"zik":113,"zio":362,"zir":38,"zis":168,"zit":62,"ziv":302,"ziw":30,"yré":24,"yuz":72,"yth":46,"yst":49,"yso":29,"ysi":55,"yre":24,"ys ":180,"yr ":41,"you":78,"yot":24,"yon":122,"za ":184,"ywe":49,"ye ":71,"uße":32,"ych":49,"yde":28,"ydd":122,"yek":64,"yeg":55,"yer":66,"yen":76,"yez":6381,"yev":24,"ya ":614,"yd ":35,"yar":58,"yan":276,"yao":135,"yal":52,"yam":35,"yak":60,"yah":28,"yn ":147,"yla":30,"yle":56,"yli":31,"yll":66,"ylv":44,"yo ":48,"ymp":27,"ymo":24,"yna":31,"ync":35,"yne":111,"ر ":42,"yi ":39,"yin":88,"yie":31,"têr":714,"ÙŠ ":49,"Ù† ":50,"xi ":24,"té ":71,"xis":56,"xic":25,"xas":26,"xan":47,"wy ":42,"wys":53,"wyn":117,"wyd":57,"wiñ":39,"sé ":43,"wn ":51,"wre":95,"wri":41,"wor":37,"wol":24,"wr ":62,"we ":50,"wez":464,"wev":251,"wes":47,"wer":638,"weu":60,"wen":755,"wel":613,"wek":43,"wei":37,"weg":50,"wed":29,"wee":32,"wec":686,"wg ":36,"wi ":52,"wir":257,"wis":49,"wit":92,"wig":63,"wie":25,"wid":55,"wic":39,"win":99,"wil":166,"wia":38,"vye":26,"wa ":134,"wan":232,"wal":641,"wam":29,"wak":26,"way":43,"wat":35,"war":4112,"was":65,"waz":81,"wag":31,"wai":112,"wae":48,"wad":65,"ría":36,"vru":57,"vro":1897,"vri":160,"vre":1493,"vra":602,"vsk":45,"vus":30,"vuz":29,"vui":173,"vug":63,"vuh":82,"via":194,"vio":268,"vir":66,"vik":47,"vil":263,"vin":176,"vig":93,"vih":155,"vij":194,"vic":93,"vid":194,"vie":274,"viz":2997,"viv":29,"vit":2282,"vis":84,"ré ":55,"vn ":395,"vle":69,"vlo":168,"vo ":111,"rén":27,"viñ":739,"vne":256,"voa":40,"vod":80,"voe":1131,"vog":240,"voj":30,"voi":76,"vol":72,"von":458,"vor":641,"vot":54,"vos":31,"vou":673,"vr ":206,"vi ":33,"vez":7355,"ver":1988,"ves":130,"vet":4050,"veu":266,"vañ":660,"vev":1006,"vei":24,"veg":189,"ven":1240,"vel":2563,"vek":66,"vea":33,"vef":97,"vee":38,"ved":919,"ve ":570,"val":181,"vak":44,"van":668,"vam":45,"vao":84,"var":1289,"vat":272,"vas":122,"vc ":49,"vab":24,"vae":81,"vad":569,"vai":39,"vag":535,"va ":291,"uzu":185,"uzk":33,"uzi":114,"uzh":1308,"uze":459,"uzs":159,"uzo":65,"uza":122,"uzb":30,"uya":31,"uz ":475,"uwe":24,"ux ":103,"uvi":37,"uva":39,"uve":100,"uy ":40,"uvr":49,"usk":479,"ush":47,"usi":500,"use":355,"usc":41,"usa":194,"ust":519,"uss":119,"uso":25,"uth":105,"uti":264,"ute":147,"uta":215,"utt":43,"uts":71,"utu":47,"uto":157,"utr":26,"us ":12885,"ut ":1028,"urb":53,"ura":377,"urd":124,"urc":80,"ure":1465,"urh":24,"urg":533,"uri":1365,"url":187,"urk":288,"urm":291,"urn":83,"uro":614,"urr":1978,"urs":53,"urt":279,"uru":206,"urv":409,"ury":73,"urz":552,"uny":30,"uon":91,"upa":32,"ur ":15791,"upi":44,"upe":63,"upl":36,"umu":1652,"uiñ":100,"umi":41,"umm":904,"umo":35,"uma":310,"umb":711,"ume":152,"ulz":123,"unt":110,"uns":56,"unv":98,"unu":56,"unk":57,"uni":755,"uno":443,"unn":67,"unc":29,"und":259,"una":1290,"ung":318,"une":411,"up ":59,"uks":39,"ukr":56,"uku":44,"uko":46,"ukl":49,"uki":91,"uke":38,"um ":382,"uka":71,"ulv":25,"ulu":113,"ult":103,"uls":40,"ulo":116,"ulm":63,"ull":318,"uli":404,"ulg":56,"ule":376,"ulf":33,"ula":220,"un ":6539,"uid":76,"uie":35,"uig":40,"uik":27,"uil":155,"uin":77,"uio":78,"uis":287,"uk ":47,"uia":185,"uje":42,"uit":103,"uiz":66,"ul ":1239,"uja":41,"ugh":41,"ugi":31,"uge":364,"ugo":67,"ugl":36,"ui ":204,"uga":481,"uhe":503,"ugu":149,"ugr":53,"uha":28,"uco":39,"ucu":26,"uf ":24,"uda":492,"udd":57,"ude":1332,"udi":309,"ubo":31,"uca":58,"ue ":1086,"uce":33,"uci":47,"uch":262,"uck":41,"uet":1306,"uev":34,"uer":664,"ues":294,"uez":1264,"uff":67,"ufo":28,"udu":45,"udo":178,"ug ":229,"udw":33,"ueg":387,"uee":179,"ued":155,"uen":478,"uel":271,"uek":451,"ub ":86,"ua ":372,"uay":86,"uat":78,"uas":116,"uar":925,"uam":161,"ual":78,"uan":1051,"ubi":95,"ubl":610,"ube":190,"uba":184,"ud ":1278,"uai":32,"uaf":53,"uag":34,"uad":128,"uc ":83,"tze":26,"tyl":53,"ty ":132,"tve":544,"tur":438,"tus":231,"tut":38,"tul":25,"tun":47,"tum":651,"tua":71,"tud":498,"tug":248,"tz ":81,"twe":33,"ts ":141,"tre":2978,"tt ":64,"tra":1337,"tri":813,"tru":143,"tro":3557,"tu ":352,"tsa":33,"tse":38,"tsc":49,"tsi":40,"tsk":31,"tso":25,"tsu":28,"toù":279,"tta":120,"tte":308,"tti":121,"tto":83,"ttp":30,"tts":31,"to ":479,"tiñ":48,"tp ":30,"tna":56,"toe":80,"tod":25,"toc":284,"toi":54,"tog":32,"tou":859,"tov":27,"tos":326,"tot":83,"toz":55,"tow":30,"tom":159,"ton":1563,"tok":134,"tol":338,"tor":741,"top":94,"tr ":310,"til":274,"tik":947,"tif":26,"tie":481,"tig":104,"tir":140,"tiq":33,"tit":209,"tis":284,"tin":1250,"tim":79,"tio":509,"thy":51,"thu":684,"tia":703,"tib":461,"tic":184,"tid":98,"tiz":85,"tiu":34,"tiv":312,"tl ":113,"tka":29,"tke":102,"tla":426,"tle":255,"tem":281,"ten":1056,"teo":96,"tep":78,"tei":518,"tek":1918,"tel":1926,"tee":33,"tef":26,"teg":307,"tea":74,"tec":37,"ted":237,"th ":395,"tez":423,"tev":112,"teu":243,"tañ":1689,"tet":90,"tes":340,"ter":3607,"ti ":957,"tho":239,"thr":44,"the":408,"thi":81,"tha":171," アア":26,"並 ":30,"三 ":49,"üs ":59,"之 ":48},"n_words":[2305059,2819439,2226880],"name":"br","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/cs.json b/contrib/languages-data/cs.json
new file mode 100644
index 0000000..e6017de
--- /dev/null
+++ b/contrib/languages-data/cs.json
@@ -0,0 +1 @@
+{"freq":{"D":14883,"E":9868,"F":10603,"G":9505,"A":22558,"B":22848,"C":16455,"L":15978,"M":26754,"N":16476,"O":10801,"H":14139,"I":10554,"J":20213,"K":19386,"U":5533,"T":19206,"W":5350,"V":19456,"P":31985,"S":35760,"R":13848,"Y":1260,"Z":7679,"f":46799,"g":61772,"d":331353,"e":908926,"b":175631,"c":304749,"a":764805,"n":728035,"o":890229,"l":442609,"m":328621,"j":233650,"k":440118,"h":259891,"i":521717,"w":9425,"v":451146,"u":342714,"t":548551,"s":514164,"r":514188,"q":1288,"p":289793,"z":213126,"y":191798,"x":11788,"í":321621,"é":145384,"á":236804,"ý":129650,"ú":13969,"ó":5647,"Ä›":138011,"Ä":2560,"ÄŒ":7702,"Ä":98194,"Å™":105343,"Ř":1405,"ň":6555,"Ž":1649,"ž":81570,"Å¥":3583,"Å ":3572,"Å¡":60015,"ů":48717," l":32594," m":62986," n":107131," o":79037," h":32176," i":15946," j":127655," k":89036," d":65217," e":11656," f":19925," g":5629,"Ñ€":1443," a":90604,"Ñ":1165," b":47368," c":20381," z":85758," u":20743," t":59923," w":1272," v":146123," p":164476," s":159790," r":52674," J":20168," K":19243," H":14014," I":10456," N":16390," O":10658," L":15894," M":26594," B":22644," C":16189," A":22443," F":10496," G":9381," D":14695," E":9809," Z":7643," Y":1252,"и":1928,"о":2213,"н":1523," S":35443," R":13762,"в":1172," P":31807,"а":2293," W":5251," V":19363," U":5519,"е":1554," T":19078," ú":11862," Ä":34240," ÄŒ":7692," ž":9995," Ž":1647," Å ":3567," Å¡":7349," Å™":11127," Ř":1403,"A ":3554,"Da":2148,"Co":3211,"Ce":1274,"Ch":3956,"Do":2726,"De":2027,"Di":1704,"Ev":1845,"Ge":1462,"Ga":1606,"I ":2501,"Fr":2934,"Fo":1409,"Fi":1671,"C ":2405,"Au":1413,"Ar":2392,"As":1426,"D ":1568,"Ba":3474,"Am":1706,"An":3282,"Al":3092,"By":2085,"Bu":2103,"Br":3950,"Ca":2582,"Bi":1362,"Be":3232,"Bo":3007,"Ku":1175,"Kl":1430,"Kr":2544,"Ko":4086,"Le":3365,"Li":3491,"La":2847,"Lu":1426,"Lo":2548,"Me":3379,"Mi":4015,"O ":1633,"Ma":8187,"Mo":4597,"Ni":1492,"Ne":3393,"Na":4223,"P ":1532,"No":2609,"Ob":1767,"Gr":1729,"Ha":2931,"He":2201,"Ho":2992,"Hr":1212,"In":2920,"Ja":4326,"L ":1226,"Ji":2201,"Je":8576,"Jo":1796,"Ju":1244,"Ka":5031,"M ":1302,"Tu":1161,"Tr":2534,"To":2275,"Th":2120,"Ti":1361,"Te":3239,"NÄ›":1339,"Ta":2435,"V ":4585,"St":6172,"Sv":1697,"Su":1422,"Wi":1615,"Wa":1205,"Vy":1686,"Vo":1294,"Vi":2012,"Va":1654,"Ve":3321,"Pr":7552,"S ":3112,"Pe":2524,"Pa":5499,"Pl":1516,"Po":7183,"Pi":1236,"Os":1168,"Or":1234,"R ":1716,"Se":3431,"Sc":1219,"Si":1856,"Sl":2584,"Sk":1212,"Sp":2686,"So":3053,"Ru":1888,"Sa":3516,"Re":2582,"Ná":1404,"Ro":3655,"Ra":2507,"b ":5436,"Zá":1325,"a ":217661,"PÅ™":2063,"Za":1646,"Ze":1335,"i ":90037,"fy":1326,"ge":8367,"ga":7746,"bý":4615,"fi":13067,"aÄ":14478,"fr":4738,"fu":2046,"ft":1555,"fo":9138,"bí":5183,"j ":5728,"gy":1560,"dá":9259,"he":13421,"ha":16982,"bÄ›":8276,"gn":2020,"gl":5355,"gi":10280,"gh":1578,"gu":3569,"gr":6781,"cí":27801,"go":4942,"du":15000,"dv":6535,"dy":13097,"g ":5595,"ea":6165,"eb":20255,"ec":32017,"ed":50406,"de":36807,"dd":1592,"di":24806,"dh":1175,"dk":3086,"dm":3275,"dl":11857,"do":40054,"dn":44007,"dp":2177,"ds":6618,"dr":15992,"ew":1387,"ex":5658,"eu":3247,"ev":22848,"ey":1497,"ez":19185,"fa":3278,"h ":68060,"fe":4502,"bá":2585,"eh":9007,"eg":5751,"ef":3385,"ee":1918,"el":51949,"ek":22014,"ej":27806,"ei":3682,"ep":11390,"eo":6058,"en":103592,"em":54636,"et":33365,"es":44385,"er":89316,"ca":4919,"e ":246825,"bv":2259,"by":26507,"bs":4261,"br":9964,"bu":11101,"bn":7303,"bo":26250,"bj":2483,"bl":11541,"bi":8921,"bc":3044,"bd":1995,"be":13547,"dc":2442,"db":1918,"da":22450,"f ":2962,"cy":1677,"cu":2117,"ct":4541,"cr":1140,"co":10175,"cn":2818,"ck":45489,"ci":24104,"ch":106800,"ce":56738,"c ":9636,"az":14206,"ay":1595,"ba":13431,"d ":32945,"at":54683,"as":30808,"ar":43773,"ax":1424,"av":34552,"au":9271,"ak":35472,"al":52089,"ai":4202,"aj":15958,"ap":12841,"am":28407,"an":76603,"ac":26092,"ad":38615,"ab":9081,"ag":6085,"ah":10143,"ae":4121,"af":3809,"nu":14720,"nt":27386,"ns":25831,"iÄ":7118,"jí":23302,"no":57801,"nn":8311,"nz":2252,"ny":17200,"oe":1227,"ká":21030,"of":6319,"oc":21110,"od":69409,"oa":1996,"ob":47136,"ké":50289,"om":27583,"on":44356,"ok":26216,"ol":47354,"oi":2071,"oj":17875,"og":10937,"oh":13386,"ot":27460,"os":56160,"ov":112940,"ou":73196,"op":21879,"oo":2643,"or":53147,"r ":22705,"ox":1211,"ow":2081,"oz":27164,"lá":15416,"pe":15026,"pa":25836,"ký":35391,"pc":1302,"pl":12514,"lé":7239,"pn":3046,"po":81579,"ph":1918,"pi":14831,"eň":2211,"lo":61865,"ln":20483,"hé":2618,"lm":4934,"ll":7520,"ls":8702,"dů":3418,"lu":15908,"lt":4743,"lz":1355,"ly":9717,"o ":138337,"ma":25523,"eÅ™":2784,"mb":4786,"hý":1516,"me":40325,"iá":5235,"ml":2209,"eÅ¡":4400,"mi":28342,"mn":8112,"mm":1350,"mp":4986,"mo":29298,"mr":1370,"ií":2470,"ms":3506,"mu":15591,"my":5631,"p ":4419,"na":97771,"nc":16128,"nd":14047,"ne":57176,"já":1158,"nf":2490,"ež":8504,"ng":13163,"ni":45831,"nk":8297,"jv":3866,"ju":1525,"eÄ":10324,"js":11164,"jn":5134,"jo":3496,"jm":7494,"ki":3600,"ke":11246,"kd":5064,"kc":3108,"ka":39832,"m ":95402,"ců":2079,"ky":30906,"kt":39344,"ku":33102,"kv":3857,"ko":71233,"kr":19435,"kl":18948,"km":3641,"kn":3161,"li":57563,"lk":10732,"le":77448,"há":9650,"ld":2521,"lf":1159,"la":65008,"dÅ™":2617,"lc":1721,"lb":2945,"n ":43523,"hr":15902,"dí":8010,"hv":1306,"ht":2875,"hu":12412,"hi":7867,"hn":5204,"ho":73846,"hl":13059,"hm":1256,"dé":2586,"id":15111,"ic":65339,"dý":1471,"ib":5266,"aÅ™":4683,"ia":10547,"ih":5555,"ig":6211,"if":3335,"eá":1247,"ie":19796,"hy":6927,"k ":33844,"ir":8411,"is":36532,"it":38607,"iu":2679,"iv":18333,"aÅ¡":1639,"ii":6209,"ij":4185,"ik":23953,"il":27036,"im":9173,"in":70373,"io":12386,"ip":4762,"je":121519,"až":7470,"ji":17967,"iz":8625,"l ":42283,"bÅ™":2940,"ja":17600,"dÄ›":13132,"xi":2953,"té":13825,"tí":14980,"pů":6675,"xt":1600,"z ":24530,"pÅ™":32632,"ož":13354,"tá":14232,"nž":1154,"oÅ¡":2365,"sé":1859,"sí":5664,"rÄ":2879,"nů":2947,"vy":26292,"vz":6628,"y ":96008,"rý":11309,"oÅ™":9947,"wa":2470,"sá":2703,"we":1453,"vl":7295,"ré":10396,"vk":3052,"nÅ¡":1672,"vi":26093,"mž":1362,"vu":5429,"vr":8123,"vs":6209,"vn":24780,"vo":37739,"uz":6709,"mů":3291,"uv":3489,"ve":47740,"rá":30974,"vc":2056,"va":51428,"pÄ›":4347,"x ":2742,"ui":1687,"uj":20016,"uk":7056,"ul":12653,"ue":2392,"ug":1937,"uh":10244,"ur":17880,"us":20839,"ut":14862,"um":13455,"un":12543,"up":13452,"ty":13294,"tz":1768,"tu":25069,"tt":3192,"lů":2536,"tv":17643,"ub":11697,"ua":2086,"ud":12808,"uc":6671,"w ":1552,"to":60100,"tn":18581,"tm":1809,"tl":6035,"ts":8087,"oÄ":8726,"tr":38784,"pí":3163,"te":85042,"pá":2981,"tk":8430,"ti":58581,"lÅ¡":2281,"th":6153,"v ":66402,"tb":1873,"tc":1257,"ta":58747,"nÄ›":45741,"su":6657,"sv":12247,"ss":3706,"st":145711,"sy":7700,"ků":5193,"sl":22654,"sk":92692,"sn":8865,"sm":8745,"sp":20097,"so":35354,"sr":1708,"nÄ":1796,"sc":4718,"se":52884,"sh":2118,"si":15634,"rz":2926,"u ":121101,"mÄ›":22506,"sa":15498,"kÅ™":3224,"sb":1371,"ný":29623,"rr":2082,"rs":10884,"rt":11161,"ru":27381,"rv":11951,"ry":11066,"ní":119912,"rp":2592,"ro":116857,"rn":21334,"rm":10574,"né":26518,"rl":3217,"rk":6663,"ri":39119,"jÅ¡":5173,"rh":1365,"rg":7113,"iž":5869,"ná":41081,"re":36192,"rd":6788,"rc":6965,"mý":1831,"rb":3000,"ra":81349,"t ":42900,"mí":11983,"mé":6077,"iÅ¡":4295,"má":9760,"lý":4646,"s ":40054,"py":1779,"pt":3558,"pu":7668,"hů":2019,"lí":8097,"pr":57083,"ps":5342,"zý":1387,"zá":15336,"už":12787,"vÅ¡":5164,"zí":7043,"vů":2483,"uÅ¡":4102,"tÅ¡":4235,"vÄ›":25419,"tÅ™":15220,"tů":3818,"rž":1454,"vý":29752,"zh":2175,"zi":13719,"rÅ¡":2140,"zb":2038,"zd":9382,"ze":35435,"vá":31782,"tÄ›":14576,"za":24684,"yz":2224,"rů":8094,"zv":6153,"zy":3702,"zs":3415,"uÄ":6510,"zr":1709,"zu":4066,"zt":1618,"zo":8118,"zn":27029,"ví":13118,"zp":5782,"zk":6049,"zm":2157,"vé":19370,"zl":2273,"yh":1425,"yc":3822,"yd":3200,"tý":6426,"yb":3071,"yv":3371,"yu":1291,"yt":7847,"ys":11111,"yr":3273,"yp":5097,"yn":5498,"ym":4086,"yl":23502,"yk":5878,"zÅ¡":1160,"yÅ™":1210,"yÅ¡":3131,"ěžn":1182,"ám":8120,"án":28607,"áp":5503,"áj":1550,"ák":7074,"ál":25430,"áh":2531,"áb":3286,"ác":7677,"ád":11483,"áz":13195,"áv":12832,"ár":9720,"át":14731,"ás":12931,"á ":61028,"íÄ":1699,"ód":1198,"ón":1579,"éž":3751,"ív":7836,"íz":3776,"ín":7431,"ím":27488,"íp":2283,"ír":5728,"ít":5219,"ís":9896,"íh":10920,"ík":10708,"íl":6160,"íj":1652,"íc":36122,"íd":6139,"íb":1336,"í ":168172,"áž":2247,"él":1388,"áš":2438,"ém":16118,"én":5964,"ét":2091,"ér":4128,"ář":4474,"éd":1164,"éh":25602,"é ":80173,"áÄ":1328,"úÄ":1787,"ýc":33402,"ýz":2981,"ýv":5852,"ýs":1457,"ýt":1801,"ýr":2669,"ým":16378,"úz":3790,"ý ":58367,"ús":1640,"íš":1631,"íř":2525,"íž":4061,"Ä› ":45418,"Ä›h":2633,"Ä›j":8834,"Ä›d":3131,"Ä›c":1612,"Ä›z":2258,"Ä›s":10757,"Ä›t":15295,"Ä›v":1529,"Ä›r":3724,"Ä›k":7417,"Ä›l":9917,"Ä›m":5852,"Ä›n":11047,"ěž":3582,"ěř":2184,"ýš":1701,"ÄŒe":5023,"Äi":11688,"Äk":4324,"Äl":2919,"Äe":26348,"Äa":7771,"Ä ":1420,"Ä ":2311,"Äá":8501,"Än":16418,"Äo":3182,"Ät":3695,"Äu":2648,"Äí":6991,"ň ":2419,"Å¡ ":1248,"ří":28277,"řá":2554,"Å™n":1352,"Å™i":14008,"Å™s":2960,"Å™e":44896,"Å™a":3989,"ší":18419,"Å¥ ":1589,"Å¡e":8882,"Å¡a":2373,"Å¡n":3181,"Å¡k":3546,"Å¡l":1812,"Å¡i":4496,"Å¡t":7997,"ňu":1474,"Å™ ":4182,"žs":2453,"žn":8213,"žo":1725,"že":16587,"žd":1803,"žk":1495,"ži":11885,"ž ":17370,"ží":12326,"žá":1346,"ů ":27572,"ům":2630,"ůl":1327,"ůs":3997,"ův":4803,"ůz":2205,"Å¡Å¡":1500,"ůž":1940,"Äás":7287,"Äí ":1292,"Äít":2030,"Äís":1405,"Ätv":1158,"Äuj":2526,"Äní":7738,"Äné":1325,"Äný":1279,"ÄnÄ›":1729,"Ä›ji":2384,"Ä›ko":1330,"Ä›kd":1747,"Ä›lo":1511,"Ä›le":2558,"Ä›kt":1458,"Ä›me":3785,"Ä›ls":1244,"Ä›tÅ¡":3901,"Ä›ta":1179,"Ä›to":2030,"Ä›ný":1390,"Ä›st":9587,"Ä›jÅ¡":4291,"Ä›ní":3741," Ga":1588," Ge":1446," Fo":1382," Fr":2926," Fi":1650," Ha":2917," He":2194," Gr":1717," Hr":1206," Ho":2982," Ji":2194," Je":8552," Ja":4310," In":2868," Ka":5019," Jo":1792," Ju":1242," La":2820," Le":3350," Li":3464," Kl":1383," Ko":4081," Kr":2542," Ku":1171," Ma":8133," Mi":3991," Me":3365," Lo":2542," Lu":1423," Ne":3375," Na":4212," Ni":1486," Mo":4565," Am":1697," An":3275," Al":3081," Ba":3436," Au":1408," As":1420," Ar":2370," Be":3213," Bi":1350," Bo":2981," Br":3935," Bu":2094," By":2085," Ca":2514," Ce":1267," Ch":3924," Co":3154," Da":2135," Di":1684," De":2001," Do":2665," Ev":1844," PÅ™":2056," Wi":1593," Wa":1184," Vy":1684," Ze":1330," Za":1643," a ":53430," Zá":1320," Os":1166," Or":1230," Po":7138," Pl":1502," Pi":1231," Pe":2516," Pa":5464," No":2595," Ob":1764," Ra":2495," Ro":3640," Re":2567," Ná":1395," Pr":7523," Sv":1696," Su":1408," St":6100," NÄ›":1337," Ta":2426," V ":3746," Th":2108," Ti":1351," Te":3210," Tr":2526," To":2242," Ru":1883," Sa":3503," Si":1839," Sc":1168," Se":3406," So":3031," Sp":2664," Sk":1209," Sl":2578," Va":1650," Ve":3306," Vi":1986," Vo":1286," Tu":1143," ja":14129," dÄ›":2397," bÅ™":1481," ji":8392," až":2786," je":93301," in":6001," it":1175," ka":8683," m ":1247," kd":3115," ke":1672," jm":2892," js":6707," ha":1600," bÄ›":2145," he":2509," dá":1402," cí":2459," gr":1666," k ":5918," hi":2201," hl":5156," ho":6862," dí":1705," hr":6552," hu":3075," ni":2426," ne":33309," na":43193," my":1417," mu":2661," mo":9457," mn":2424," ok":4623," oc":1666," od":14999," of":1957," ob":22345," no":3710," le":12637," li":8468," la":3675," dÅ™":1310," kv":1829," ku":2340," kt":24717," kn":2205," km":3399," kl":3809," kr":8199," ko":16866," me":11211," mi":4779," ml":1339," o ":7191," ma":9419," dů":1272," lo":3118," ab":1328," am":3804," an":6347," ap":1198," ak":1935," al":5289," au":3694," ar":3304," at":1332," as":2951," ba":3819," bi":2222," be":2412," bo":3752," by":19260," bu":3054," br":3187," en":1401," el":2125," fa":1772," ex":2023," fu":1558," fr":2890," fo":4715," fi":6203," ge":1763," bý":3017," i ":5273," co":2133," ce":6243," ch":5478," da":4034," do":19631," dn":2609," dl":1361," dr":8732," de":7028," di":4948," dv":4297," du":1944," vý":12688," zk":2698," ví":2124," zp":3908," zn":6476," rů":1942," zv":2770," tÄ›":2639," za":18239," zd":2577," vá":3060," ze":8914," tý":1167," té":5230," pů":4589," z ":20036," pÅ™":29127," vÅ¡":2884," už":1794," zá":11557," tÅ™":3845," vÄ›":4981," ru":2498," ry":1961," u ":3351," sa":3585," mÄ›":10287," kÅ™":2709," se":38618," sc":1843," si":3884," sn":1263," sm":2713," sl":8492," sk":7519," sr":1447," sp":14637," so":12794," mí":3137," ra":2856," ná":11985," re":10229," ro":31079," ní":1455," pr":44618," ps":1456," s ":10437," má":2572," os":6522," ot":1283," ov":1214," op":3355," or":4357," oz":5841," lá":1294," pe":2741," pa":10148," pl":5963," po":56929," vy":18421," vz":6214," sí":2537," pÄ›":1240," va":1643," ve":20353," uv":1309," mů":1354," vo":5885," vr":1606," vi":2111," vl":5324," ty":2521," tv":3467," tu":1727," us":1203," ur":2603," um":2673," un":1389," ta":9553," nÄ›":9405," v ":54233," sy":4875," st":28273," sv":11047," su":1768," pí":1932," tr":6193," to":5957," th":1725," ti":1444," pá":1635," te":10751," ÄŒe":5018," Äí":2219," Äá":5494," Ät":2423," Äe":12226," Äl":2635," Äi":5961," Äa":2613,"ňuj":1462," ús":1372," úz":3683," úÄ":1533," Å¡k":1284," Å™a":1815," Å™e":4682," ří":3390," ži":4198," že":3745,"Evr":1575,"Fra":1982,"šší":1462,"ším":3926,"šíc":3060,"Byl":1846,"šíř":1269,"ší ":8873,"Cha":1224,"Nov":1143,"Par":1206,"Pro":2287,"Pra":2828,"Pod":1248,"Pol":1195,"ůzn":1758,"ůso":2743,"ůvo":3624,"Jed":1974,"Jeh":1143,"Je ":2998,"ům ":1924,"Kar":1595,"Mar":2760,"Å™ez":1271,"Å™ev":1736,"Å™et":1507,"Å™es":3291,"Å™en":6131,"Å™el":1284,"Å™i ":4648,"Å™ed":13088,"Å™ej":1316,"Å™ek":2349,"Å™eb":1351,"Å™ec":3115,"Å™e ":3029,"Å™ad":2628,"ého":25401,"ém ":11396,"Å™sk":2573,"áře":1219,"ému":2422,"éna":1816,"éno":1417,"éri":1326,"Sta":2172,"Spo":1540,"Slo":1556,"Vel":1392,"The":1321,"Å¡ec":1390,"Å¡en":2332,"Å¡e ":1844,"Å¡ak":1283,"Å¡ti":2097,"Å¡ní":1582,"Å¡ko":1848,"Å¡in":1751,"Å¡tÄ›":3036,"átu":1328,"bje":2435,"áte":1666,"átk":2625,"átn":2002,"áto":1624,"bit":1438,"ást":9035,"bil":2518,"bo ":10795,"ávi":1211,"bli":4783,"ávn":2176,"bla":3565,"áva":2824,"bod":1239,"bol":1903,"boj":1202,"blí":1257,"ázv":1186,"ává":2557,"áze":5164,"bor":3758,"bov":1176,"bou":1297,"álk":1854,"áln":9542,"álo":2573,"álu":1211,"ákl":2414,"bal":2672,"áko":1419,"án ":4623,"ále":4254,"bar":2156,"áli":1317,"áns":1949,"bdo":1660,"áno":2096,"ány":1316,"ánu":1360,"ámo":1148,"bce":1301,"ána":2621,"áni":1332,"bec":2884,"ber":2768,"ben":2163,"bez":1903,"ápa":4357,"ámÄ›":1142,"ání":9623,"át ":2061,"áro":4387,"árn":3371,"ámý":1162,"áda":1737,"ách":3663,"áce":1349,"ádá":1824,"ábÄ›":1152,"ca ":1459,"ák ":1153,"ál ":2023,"ce ":37517,"ám ":1274,"bri":1805,"bro":1589,"bra":3506,"bu ":2220,"bní":3217,"bsk":1200,"bsa":1785,"bur":1260,"bud":1860,"bvy":1413,"by ":4307,"byl":18860,"byv":1815,"am ":2742,"ake":1229,"aji":2410,"al ":7387,"adÄ›":2234,"aje":2039,"ak ":5398,"ahu":2375,"ahr":2506,"aha":1562,"ací":4252,"aké":5289,"anu":1370,"any":1635,"ano":3773,"ann":1847,"ant":5679,"ans":3967,"ají":6907,"ane":2662,"ang":5036,"ani":8977,"ank":1763,"ana":4560,"anc":5148,"and":5394,"amu":1194,"amo":2812,"amn":1598,"ami":3729,"ame":8001,"ama":1900,"aly":1179,"als":1998,"alo":7874,"all":1742,"ali":6637,"ale":6714,"ala":3987,"alb":1604,"an ":5076,"aku":1247,"akt":3867,"ako":12804,"abe":1141,"abs":1339,"ae ":1943,"ad ":4944,"afi":1777,"age":1161,"ael":1206,"ado":1550,"adl":1340,"adn":6465,"adi":3651,"ade":3282,"ady":1580,"adu":2552,"aco":1432,"aci":3453,"ach":6559,"ace":7532,"ada":3897,"arÅ¡":1293,"azu":1350,"aze":2771,"azy":2655,"até":1146,"apÅ™":2101,"ba ":3672,"azý":1373,"avÄ›":1439,"atÅ™":2572,"at ":6248,"aná":3560,"are":2437,"ard":3288,"arc":2227,"ara":4391,"aro":4126,"arn":1320,"arm":1290,"ané":4536,"arl":1695,"ark":1851,"ází":4141,"ari":4139,"ars":2390,"art":3122,"asa":1162,"amÄ›":1185,"asi":3609,"aný":5688,"ase":1167,"aso":1587,"asn":2528,"ask":1257,"ar ":2605,"alá":1216,"ape":1349,"alé":1624,"apo":2941,"as ":2058,"alý":1783,"ava":3516,"aut":3726,"avs":1166,"avo":2741,"avn":5290,"avi":3981,"ave":3850,"avy":1957,"avu":1943,"av ":1604,"ata":3060,"anÄ›":2278,"ast":13117,"atn":1969,"atk":1885,"atr":1563,"ato":5121,"ate":11112,"ati":10351,"alÅ¡":1631,"atu":2078,"aur":1970,"řád":2065,"ří ":6262,"řív":1184,"říz":2145,"říc":1240,"říp":1616,"řís":1609,"řím":2008,"řík":1716,"říd":3286,"říž":1421,"až ":2790,"jeh":3911,"jej":4795,"jed":12881,"jek":1841,"jem":2659,"jen":5483,"jev":2523,"ji ":4802,"ažd":1260,"bÅ™e":2487,"dÄ›l":3685,"jak":10723,"dÄ›n":1255,"dÄ›j":2198,"jaz":2568,"je ":83650,"jme":2592,"jno":1257,"jov":2720,"jin":3546,"jih":2344,"jic":2414,"ito":2582,"itu":2506,"its":1828,"ity":1661,"isk":2395,"ism":2509,"isl":2097,"iso":2022,"ist":15778,"inÄ›":2331,"ita":3897,"ite":5726,"iti":3568,"ivo":2672,"ivn":3856,"ium":1381,"iva":2483,"ivi":1448,"ive":2466,"is ":3839,"ion":6938,"iro":1733,"irm":1303,"ise":1488,"iný":1514,"iná":3960,"it ":3805,"dÄ› ":4195,"itý":2115,"ití":1214,"ité":1984,"itá":1643,"izo":2349,"iza":2404,"km ":2378,"kdy":3252,"kej":1210,"kem":3800,"ket":1376,"ke ":1607,"kce":1649,"kde":1621,"kra":5687,"kre":3334,"kt ":1234,"ku ":19613,"kro":2033,"kov":11417,"kou":9659,"kos":4285,"kor":1310,"kop":1321,"kon":10573,"kom":4572,"kol":7494,"úze":3480,"klá":2870,"kni":1628,"klu":1477,"ko ":14736,"kle":2073,"kla":8470,"klo":1299,"jso":6864,"jsk":2959,"eÄn":6772,"jmé":3893,"eÄe":1504,"již":2514,"kaz":1148,"kat":3393,"kar":1707,"kap":1616,"kan":2047,"kal":2034,"kam":1509,"kac":1503,"ka ":20272,"jvÄ›":1879,"ha ":4741,"bÄ› ":2724,"han":1905,"hal":1464,"har":2922,"bÄ›h":2126,"he ":2188,"dá ":2060,"dál":1753,"dáv":1525,"her":2617,"dán":2244,"hem":3497,"his":2398,"běž":1211,"hla":5571,"hle":2243,"hlo":1435,"ho ":41844,"gli":3394,"cí ":18956,"cíc":3683,"gra":4857,"cím":1211,"ial":1377,"ian":1754,"ic ":2965,"ibl":1394,"ia ":4182,"ien":1656,"aří":2027,"ier":1153,"ifi":1720,"ict":1675,"ick":33133,"ici":3981,"ich":6375,"ice":11279,"ie ":13367,"ica":1513,"ido":1643,"idl":1498,"ide":3474,"ida":1748,"il ":4856,"ika":5332,"ii ":5869,"igi":1176,"icí":1825,"iho":2925,"ik ":2696,"imo":1396,"ime":1223,"inc":4325,"ind":2184,"ina":8223,"inn":2167,"ino":5250,"int":2630,"ins":4625,"inf":1536,"ine":3495,"ing":4248,"ini":3264,"ink":1380,"iká":1715,"inu":3551,"iny":6607,"iko":2695,"ikl":2333,"ila":2431,"in ":6095,"iky":3097,"ikt":1203,"iku":2133,"ilo":3489,"ill":1963,"iln":1971,"ilm":2591,"ili":3988,"ile":1526,"hok":1282,"hol":1704,"hot":1459,"hou":1879,"hov":5549,"hor":3514,"dí ":3801,"hod":8706,"hni":1267,"huj":1855,"hud":2458,"hu ":4702,"hro":1617,"hrn":1403,"hra":7196,"díl":2411,"hyb":1243,"hož":1183,"hy ":3116,"hum":1365,"hrá":2941,"evÅ¡":1525,"ext":1409,"etí":2247,"exi":1437,"ezn":2836,"eze":2667,"ezi":7262,"enÄ›":1924,"eta":2522,"ete":4466,"eti":3348,"etn":2128,"esp":1404,"esn":2468,"eso":1292,"est":7801,"ev ":3656,"eto":3093,"etr":4039,"erá":7485,"eve":6187,"evn":2175,"eré":8410,"evi":2322,"erý":10219,"er ":7795,"eor":1444,"es ":7550,"epu":2593,"elé":1267,"epr":1430,"eri":9740,"erg":1674,"ere":2706,"ená":5438,"era":5702,"et ":4584,"emí":4275,"esk":11348,"esi":1360,"ený":6913,"ese":3215,"emÄ›":2087,"erz":1749,"erv":3468,"eru":1909,"ení":16758,"ert":1878,"ers":3684,"ern":7190,"erm":2474,"ené":4823,"ero":6299,"ekl":1253,"eko":2098,"ekt":6413,"en ":12357,"ela":2337,"ele":12125,"eli":4792,"elm":1412,"eln":3331,"elk":5262,"ell":1825,"elo":2426,"els":2269,"ema":1842,"eme":2383,"emn":1840,"emo":2184,"emi":4395,"ene":4580,"ena":4918,"end":1571,"enc":3468,"eno":6243,"enn":2162,"eni":2669,"ens":7642,"ent":12967,"ejí":5227,"eny":2102,"egi":1835,"ej ":1214,"edí":1331,"eho":5425,"ek ":7971,"ein":1192,"el ":8637,"ejs":2194,"ejn":3303,"ejm":2020,"eji":2735,"em ":31739,"ejv":3622,"gin":1170,"gio":1330,"gie":2007,"gic":2475,"gii":1170,"gen":3213,"býv":3044,"být":1189,"gan":3061,"ga ":1144,"íže":1335,"íž ":1472,"fun":1501,"fra":2633,"aÄe":3487,"aÄo":2232,"aÄn":3017,"aÄu":2194,"for":4925,"fot":2015,"bí ":2241,"fic":2581,"fil":3337,"fik":2015,"fin":1574,"fir":1363,"da ":6390,"de ":6350,"dal":3374,"daj":1518,"dat":3084,"dan":2182,"dce":1208,"ctv":1837,"cnÄ›":1190,"cko":3237,"chá":6943,"cky":4430,"ciá":2350,"co ":1224,"cká":6180,"cké":15214,"cov":2906,"cou":2423,"cký":12856,"ch ":62063,"ces":2551,"cen":5710,"cem":1899,"cel":4240,"ci ":10328,"cha":4867,"chu":1777,"chy":1469,"cia":1607,"cie":2249,"che":3452,"chl":1932,"chi":2422,"cho":11778,"chn":2940,"cht":1572,"chr":1838,"ed ":3026,"ebn":2564,"ebo":12455,"ec ":4754,"edl":1223,"edm":1145,"edn":17139,"edi":5930,"ede":6652,"eda":1810,"edy":1655,"eds":1915,"edo":3095,"eck":8996,"ech":12275,"eci":1513,"ecn":1350,"dy ":11421,"drá":1267,"dvo":3395,"dva":1139,"dor":1289,"dop":2114,"dom":1869,"dol":1600,"dok":1496,"dov":6030,"dou":1896,"dos":2695,"dpo":1574,"dna":2226,"dne":2707,"dni":1534,"dno":8163,"dob":7531,"dst":2606,"íře":1323,"dnÄ›":4952,"duc":1342,"dné":1547,"dra":2227,"dná":2916,"du ":8252,"dro":2166,"dní":18004,"dru":7228,"dsk":2773,"dic":1916,"dia":1263,"der":3512,"des":2009,"dev":1882,"deb":1947,"dec":1456,"del":3171,"den":6683,"dem":3354,"di ":2989,"dle":4381,"dla":1946,"do ":8212,"dlo":2844,"div":1579,"din":4766,"dio":1325,"dis":2494,"die":1484,"rga":2761,"rgi":1143,"ižn":3600,"ret":1327,"res":5841,"nás":2254,"náv":1475,"rez":1829,"náz":3803,"raÄ":1242,"iž ":1205,"rea":1647,"nác":1291,"rec":1925,"reg":1788,"nám":4897,"rem":3700,"ren":2252,"rek":1273,"nál":3115,"nár":3570,"rep":3246,"rdi":1273,"ná ":14630,"re ":2546,"rch":3978,"rce":1159,"raz":3775,"rd ":1372,"ras":1475,"rat":6479,"rav":9308,"raj":4559,"rah":1855,"ran":13598,"ram":4289,"ral":1943,"rak":3296,"rab":1667,"raf":2398,"rad":5238,"rac":4788,"ros":9033,"rot":4473,"rom":4735,"ron":3499,"rop":5057,"roz":10708,"rou":3738,"rov":17134,"rob":3794,"rod":11155,"roc":8820,"ní ":79844,"roj":4450,"rol":2078,"rok":8745,"rof":1452,"rog":2059,"rno":1887,"rnu":1168,"rna":1957,"rež":1163,"rni":1331,"ném":2468,"rmo":1370,"jší":4959,"ro ":11368,"rma":3339,"riá":1833,"néh":5048,"né ":18438,"raž":1181,"rit":3911,"ris":3258,"rig":1476,"rik":1662,"rin":2410,"ria":2007,"ric":8579,"rid":1291,"rie":3806,"rk ":1359,"ryc":1171,"ruh":6961,"rum":1222,"ruk":1449,"rus":3302,"rva":1915,"rve":2525,"rvn":4272,"ry ":6263,"rsk":6183,"rnÄ›":2168,"rst":1339,"rto":1577,"rti":1692,"rub":1147,"rtu":1152,"ním":14655,"ník":6579,"níh":7211,"rmá":1574,"níc":9560,"rt ":1561,"rné":1326,"ru ":6692,"rní":7275,"sah":2489,"mÄ›l":2079,"sam":2091,"nýc":8901,"ným":4766,"mÄ›n":2340,"san":1523,"sau":1548,"mÄ›s":9181,"mÄ›r":2410,"sa ":1633,"mÄ› ":2287,"ruž":1312,"ný ":15629,"si ":3518,"měř":1748,"sin":1775,"sil":2216,"se ":32266,"sch":2059,"sev":4239,"ser":1773,"sed":1877,"kří":1234,"sen":1348,"sem":1612,"sel":1714,"spo":10422,"spr":1930,"spe":2260,"spi":1842,"ský":19060,"sou":17153,"sov":4194,"ské":26627,"son":1432,"ská":8637,"sof":1375,"soc":1565,"sob":5198,"su ":2847,"sní":1788,"sné":1153,"st ":11395,"slo":9919,"slu":2567,"sky":9407,"sla":4078,"sle":3342,"skl":1707,"sko":9797,"sku":8770,"ska":4807,"ske":1405,"sni":1352,"smu":2054,"stí":6180,"sté":3269,"stá":6877,"syn":1327,"sys":2806,"stÄ›":3272,"své":1798,"smÄ›":1301,"ste":9093,"snÄ›":1417,"sta":23517,"stn":5308,"sto":14050,"sti":18925,"stl":1904,"stv":6645,"stu":6506,"str":16919,"sts":1291,"ků ":4430,"sty":1890,"sva":1646,"svo":1261,"sy ":1657,"nÄ›j":4079,"tak":7734,"nÄ›k":4566,"nÄ›l":1306,"tal":4758,"tac":1310,"tad":1311,"tba":1700,"tav":8149,"tat":3631,"tas":1264,"tar":5738,"tan":5175,"nÄ›n":3988,"nÄ›m":4105,"te ":2317,"svÄ›":5020,"stÅ™":6161,"éž ":3711,"nÄ› ":25254,"ta ":12315,"ký ":20031,"ouž":6121,"ozá":1627,"pa ":1199,"ovÄ›":6255,"lá ":1917,"kýc":10186,"kým":4999,"par":3389,"pat":3635,"pad":7309,"pak":1199,"pal":1582,"pam":1204,"pan":2500,"pec":1647,"lád":4346,"lán":1802,"pen":1946,"per":4148,"lát":1414,"pla":3524,"ple":1738,"plo":2559,"lé ":2839,"pic":1267,"pin":4585,"pis":3803,"poz":3018,"por":4291,"pop":2679,"pov":4058,"pou":6174,"pot":2565,"pos":5385,"poj":5321,"poh":2575,"pom":2458,"pon":2428,"pok":1746,"pol":12866,"pob":1330,"poc":1426,"pod":13049,"láš":1492,"po ":3633,"lí ":2188,"psk":1538,"hů ":1499,"pub":3056,"poÄ":3912,"pra":9290,"prv":4970,"psa":1291,"pu ":1292,"pri":1503,"pre":2679,"pro":30809,"prá":4755,"poÅ™":1288,"py ":1312,"prů":1838,"lý ":2387,"má ":3426,"mát":1458,"mán":1640,"iÅ¡t":1515,"mén":4741,"mí ":3982,"mís":2771,"mír":1517,"mín":2069,"ra ":9350,"eži":1875,"ngl":4299,"ni ":2504,"nge":1442,"nej":9246,"nek":1332,"nen":1766,"nem":4884,"nep":1631,"ner":3084,"net":2605,"nes":3430,"ež ":1798,"ng ":3190,"neb":12487,"nec":2031,"ned":1356,"nfo":1282,"naÄ":7033,"nez":1422,"nco":2679,"nci":5166,"nce":5874,"ne ":3607,"ndo":1521,"ndi":1883,"nde":1639,"nda":1497,"nak":1732,"nal":3376,"nam":6213,"nan":1425,"nap":3429,"nar":1416,"nac":3724,"nad":3749,"nd ":2404,"nav":1415,"nat":3143,"nas":1417,"naz":1632,"na ":53204,"mys":1461,"mož":1630,"nož":1329,"ny ":15609,"nuj":1460,"nut":2319,"nto":3206,"ntu":1650,"ntr":2728,"nti":4061,"nta":3294,"nte":4199,"nst":3785,"nsk":18177,"nu ":7672,"iÄn":2337,"nné":1576,"nt ":3215,"jím":1621,"jíc":10934,"noh":2321,"nol":1273,"nom":2363,"not":3595,"nos":13323,"nor":2008,"nov":9782,"nou":8746,"než":1270,"nno":1485,"niÄ":2090,"jí ":9261,"neÅ¡":1391,"no ":7023,"nka":1355,"nko":1306,"eží":3799,"nih":1649,"nie":1760,"nic":12884,"niz":2240,"niv":1387,"nis":3526,"nit":3056,"nin":2133,"nik":6308,"ogr":3240,"ogi":4525,"odí":1145,"ohl":1659,"oho":4168,"oha":1909,"obÄ›":2552,"oj ":1676,"ok ":1263,"ohy":1525,"ojo":1142,"ojm":1998,"oji":1192,"oje":7276,"odÄ›":2068,"obÅ™":1215,"ol ":1430,"oce":9242,"och":4480,"oci":2151,"ock":1389,"obs":1745,"obv":2042,"obu":1622,"oby":3426,"ká ":15983,"ode":4045,"odl":3777,"odi":2898,"odo":4607,"odp":1587,"odn":14263,"ods":1272,"odr":1258,"of ":1489,"oda":2197,"kán":1145,"ody":1671,"odv":1246,"odu":5226,"ofi":1587,"obí":2810,"oba":1485,"od ":15166,"obo":2639,"obr":2915,"obl":4657,"obn":3328,"obj":2067,"obi":2455,"obd":1788,"obc":2803,"obe":3958,"ový":12973,"orů":1170,"ové":14190,"ozn":6156,"ozl":1181,"ouÄ":4572,"ozo":1460,"ozd":2636,"oze":3842,"ová":16410,"ozi":1341,"oty":1225,"oud":1374,"oub":1598,"ouc":1601,"otk":1156,"oti":3017,"ote":2572,"oto":6726,"otn":1824,"ost":32198,"ota":1788,"otb":1150,"ov ":3056,"osi":1402,"osk":1491,"ose":1306,"osp":1323,"osm":1339,"osl":4606,"oso":3077,"ovy":1368,"ovi":10902,"ovn":5524,"ovo":6751,"ovs":3742,"ouz":3479,"ova":19936,"ove":6197,"ouh":2082,"oun":1766,"oup":1457,"ous":3223,"out":2402,"opo":2402,"opi":2216,"ope":2692,"opa":2307,"os ":1967,"opu":1150,"opr":3218,"olí":1626,"ops":1833,"or ":5513,"ork":1330,"orm":4795,"orn":2809,"oro":4330,"ord":1649,"ore":3272,"oná":1885,"org":3417,"ori":6242,"ou ":37147,"omÄ›":1557,"osa":2963,"ort":3045,"ors":1818,"oru":3220,"ory":1722,"omá":2183,"ora":3762,"ívá":2944,"íze":1985,"ola":2087,"on ":7329,"oli":8828,"ole":10375,"ols":1995,"oln":2388,"olo":8650,"oly":1795,"olu":2611,"oka":1642,"om ":1785,"íst":3954,"ké ":30775,"ísl":1496,"oke":1629,"íta":1783,"okr":4661,"oko":4002,"íků":1200,"oku":8537,"ona":3352,"ond":1711,"onc":1948,"one":2668,"ong":1188,"oni":3548,"ono":3674,"ons":4146,"ont":3048,"onu":1779,"ony":1528,"oma":2266,"ome":3465,"omi":2447,"kéh":13733,"kém":5563,"omp":1767,"omo":5000,"omu":2175,"íva":3319,"la ":20221,"ími":2048,"íms":1206,"íns":1474,"ím ":20382,"kvÄ›":1156,"ín ":1890,"íle":1385,"le ":15329,"íro":1439,"írk":1295,"íse":1140,"lac":1653,"lad":8838,"ípa":1629,"lan":4986,"lam":1157,"lat":5358,"las":7702,"lav":8047,"krá":4170,"kup":4964,"kum":1330,"kul":2572,"ky ":27061,"ích":18473,"kte":27582,"íce":2062,"ídl":1751,"ců ":1941,"ktr":2141,"ktu":2012,"kti":3026,"kto":1589,"kyt":2072,"ík ":3909,"ící":14415,"ího":9884,"lok":1275,"lon":1573,"lké":1191,"lom":1730,"lod":1650,"loh":1236,"log":5594,"lký":1221,"los":3409,"lou":5777,"lov":17182,"lni":1153,"lež":2967,"liÄ":1203,"lmi":1186,"ltu":1148,"dů ":1701,"lub":1957,"lsk":6596,"lnÄ›":2708,"lné":1553,"lní":11343,"lu ":4880,"lný":1549,"liÅ¡":1271,"li ":6590,"lez":2008,"ház":6195,"lev":1768,"les":3807,"let":9902,"ler":1221,"lem":5693,"len":8028,"lek":3907,"lej":1379,"led":7715,"lec":1964,"eň ":1751,"lo ":9285,"lla":1318,"lle":1365,"lli":1334,"lko":2458,"lky":2194,"lka":1525,"leÄ":4689,"hé ":1767,"lm ":1297,"ll ":1176,"lit":5959,"lis":4420,"lin":6175,"liz":1555,"liv":3315,"lic":7730,"lid":2928,"lia":1350,"lik":5494,"lig":1241,"lie":1691,"ma ":3621,"mac":1223,"maj":1332,"mar":1328,"mal":2994,"man":3894,"mat":5599,"me ":1813,"med":1375,"mec":4748,"met":4900,"mer":5988,"mem":1269,"iál":4186,"men":11122,"mez":6394,"ly ":6413,"lož":5290,"moc":2506,"mob":1461,"mod":2374,"mon":2073,"mov":3077,"mor":1679,"mos":2024,"mot":2793,"mou":1328,"mní":1927,"mu ":8025,"msk":2845,"moÅ™":3176,"my ":2400,"mus":1983,"mun":1943,"mi ":12425,"eÅ¡n":1222,"min":3830,"mil":1861,"mis":1525,"eÅ¡t":1436,"mic":2749,"mo ":1420,"ií ":2195,"mno":2859,"tří":4710,"tÅ¡i":1374,"tší":2640,"ůže":1655,"ÄŒes":3563,"vÄ› ":4735,"vÄ›z":1289,"vÄ›t":11745,"tÅ™i":1228,"vÄ›k":2228,"tÅ™e":8474,"vÄ›d":2018,"výr":2025,"výs":1250,"vým":2640,"výz":2611,"výš":1486,"Äas":5969,"zná":4232,"uÄa":1905,"zní":1327,"víc":1941,"Äen":7616,"Äel":3765,"Äet":1680,"Äes":6771,"Äer":2858,"zný":1440,"zu ":1323,"zsk":2646,"uÄá":1855,"Äi ":4977,"rů ":3148,"zuj":1535,"Äit":1616,"způ":1462,"Äin":2747,"růz":1961,"Äka":1735,"zyk":1998,"Äle":1629,"Äov":3019,"vý ":6798,"Äno":3388,"výc":8997,"zi ":5006,"zaÄ":1189,"zej":2847,"zev":2458,"zen":7446,"ván":11057,"zem":8432,"vál":2700,"zel":1300,"zer":1349,"vá ":12375,"ze ":9310,"zde":1205,"zab":1246,"zac":2566,"tÄ›j":1439,"zah":1842,"tÄ›n":1733,"zal":3135,"tÄ›l":1552,"zas":1298,"ví ":7249,"zor":1418,"zov":3172,"zpr":1480,"vém":1518,"rší":1390,"véh":3281,"zna":13601,"zni":4038,"vé ":13437,"zko":1259,"zkr":2046,"zdí":1212,"zaÅ™":1286,"zin":1828,"zit":1344,"zdÄ›":2035,"těž":1441,"yrá":1302,"yva":1858,"ytu":1246,"ytv":1652,"yto":1723,"yst":4390,"yso":1389,"ysk":1190,"ysl":1771,"tÄ› ":5527,"za ":5089,"ych":2524,"tým":1477,"tý ":2037,"týc":2116,"ykl":2523,"yla":6390,"ylo":2929,"yly":1314,"yl ":10326,"tím":1511,"tém":3602,"tí ":11245,"též":3726,"pův":2875,"půs":2701,"ože":4937,"ožn":1513,"oži":1506,"tán":1850,"pří":8711,"tát":4498,"ož ":2063,"tál":2187,"té ":3506,"xis":1170,"pÅ™ ":1535,"tá ":1819,"pÅ™i":7219,"pÅ™e":14856,"sér":1644,"síd":1352,"rÄe":1253,"oří":2027,"vzn":2563,"vzd":1455,"vyk":2012,"vyr":1739,"vyd":1992,"vys":3366,"vyt":1966,"vyv":1398,"rý ":8817,"vyÅ¡":2053,"rýc":1165,"oÅ™i":1304,"oÅ™e":3041,"vní":15429,"vro":2455,"vrc":1706,"vst":1754,"vsk":4082,"vu ":2977,"nů ":2566,"vnÄ›":2710,"vuj":1416,"vy ":4952,"voÅ™":3595,"vil":1507,"vin":6325,"vic":2325,"vid":2543,"viz":1532,"vit":4611,"vis":2095,"ré ":7668,"vla":2806,"vo ":2147,"vna":1206,"vno":1207,"vni":1709,"vod":8774,"voj":3429,"vol":2921,"vor":1476,"vot":1529,"vov":1657,"vou":5924,"voz":2714,"vlá":2701,"vi ":1398,"ver":8833,"ves":2177,"rát":2382,"ráv":4812,"mž ":1199,"rán":2154,"ven":9163,"vem":1785,"rál":6073,"vel":5581,"ráb":1423,"ved":2640,"rác":2253,"ve ":12748,"rá ":6574,"val":7096,"van":12007,"var":2621,"vat":8883,"pÄ›v":1148,"vaz":1671,"vac":2111,"vad":1154,"vaj":2879,"můž":1309,"va ":9872,"uze":1855,"uzs":2041,"utí":1162,"urÄ":2415,"usk":3978,"use":1494,"umÄ›":1413,"ust":3573,"ute":1270,"mů ":1642,"uto":4678,"us ":7285,"ut ":1210,"ura":2035,"ure":1155,"uri":1389,"urn":1235,"uro":1540,"uru":1160,"ury":1381,"upi":4594,"upe":1451,"upn":1306,"umo":1169,"umb":1314,"ume":1179,"ují":7869,"unk":1600,"uni":2964,"uko":1228,"um ":4532,"ult":2459,"ulo":1228,"uhé":1788,"uli":1303,"ula":1664,"uje":11818,"ucí":1196,"uho":1206,"ude":2608,"udi":1376,"ubo":1407,"uce":1140,"uch":2624,"uh ":1827,"udo":2156,"ub ":1463,"ubl":3299,"ud ":1221,"trů":1395,"tví":6726,"typ":2401,"ty ":7625,"tvr":1530,"oÄí":2254,"tvo":4249,"trá":2691,"tva":2695,"tur":4631,"tuj":2396,"tup":3639,"tud":1736,"pís":1810,"oÄe":1902,"tre":1691,"tra":10732,"tné":1153,"oÄi":1195,"tri":3505,"tru":3242,"tro":11520,"oÄn":1259,"tní":9445,"tu ":7164,"tný":1437,"tsk":6630,"tnÄ›":1562,"lů ":2152,"to ":14761,"lší":1771,"tna":1155,"tno":1779,"toh":1166,"tou":3172,"tov":8578,"tos":1217,"tom":3677,"ton":3319,"tok":1791,"tol":3953,"tor":9066,"top":2079,"til":1384,"tik":3154,"tit":2068,"tis":2205,"tin":7629,"tio":2588,"thu":1152,"tic":11897,"teÄ":1644,"tiv":5666,"tko":1227,"tka":3033,"tli":2690,"tky":1508,"tla":1310,"tem":4570,"ten":3204,"teo":1298,"tej":1972,"tek":2779,"tel":12611,"tec":6325,"ted":1925,"tev":1290,"ter":37331,"ti ":14561,"taÄ":2083,"the":1375,"ží ":4449,"žíc":1641,"žív":5443,"yÅ¡Å¡":1364,"zýv":1380,"žil":1490,"živ":2602,"žit":2707,"žij":1345,"žov":1554,"že ":5417,"žel":1619,"žen":7485,"záv":1903,"záp":4090,"zák":3066,"uži":2255,"ýt ":1211,"ýro":1252,"ým ":11479,"ými":4161,"žní":2919,"ých":32937,"žsk":1623,"žnÄ›":2362,"vší":1568,"zí ":4964,"vÅ¡a":1280,"vÅ¡e":1958,"uží":6581,"tů ":3433,"ýzn":2182,"ývá":1959,"ýva":2863,"uÅ¡e":1228},"n_words":[11333226,13010717,8780627],"name":"cs","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/cy.json b/contrib/languages-data/cy.json
new file mode 100644
index 0000000..a3e1cb8
--- /dev/null
+++ b/contrib/languages-data/cy.json
@@ -0,0 +1 @@
+{"freq":{"Fe ":1539,"D":11333,"E":9591,"F":9129,"G":20345,"A":20881,"B":12687,"C":28561,"L":15711,"M":24164,"N":7579,"O":5361,"H":6663,"I":5301,"J":2920,"K":1955,"U":5682,"T":9914,"W":5139,"V":1175,"Q":331,"P":10920,"S":18433,"R":8785,"Y":6381,"X":156,"Z":358,"f":102454,"g":124045,"d":297794,"e":305845,"b":42206,"c":70455,"Fed":56,"a":340057,"n":303994,"o":193011,"Fei":80,"l":181438,"m":70473,"j":732,"Fel":228,"k":5358,"h":111415,"i":224421,"Fen":103,"w":133473,"v":3889,"u":84011,"Fer":95,"t":96401,"s":97156,"r":275486,"q":393,"p":27532,"z":1653,"y":272195,"x":1194,"Ffo":187,"Ffl":265,"Ffi":428,"Ffe":213,"Ffa":46,"Ffy":47,"Ffu":103,"Ffr":1430,"²":91,"ÃŽ":41,"É":51,"À":445,"Fil":43,"ï":652,"î":418,"Fin":48,"í":198,"ë":206,"ê":1481,"Fir":87,"é":1125,"è":159,"ç":67,"æ":50,"ä":70,"Fie":143,"â":2559,"á":343,"à":109,"ü":121,"ú":96,"ù":63,"ö":248,"ô":3067,"ò":62,"ó":166,"Ä“":42,"Ä":208,"Ä«":83,"Å":123,"ŵ":1259,"Å·":505,"Å«":152,"Faw":386,"Fan":141,"Fal":84,"Far":105,"Fae":78,"Fac":114,"Fai":52,"Erb":44,"Eri":162,"Eth":197,"Eti":126,"Ers":44,"Ery":541,"Esg":154,"Eup":59,"Eur":69,"Eva":147,"Ewr":684,"ˈ":50,"Eit":40,"El ":81,"Eif":65,"Eig":43,"Eid":393,"Ein":91,"Eil":45,"Eis":204,"Ì":115,"Elf":104,"Ele":70,"μ":79,"ν":135,"ο":198,"Eng":69,"ι":95,"κ":64,"λ":95,"δ":38,"ε":83,"η":58,"α":182,"γ":42,"ά":48,"ί":75,"Emi":37,"Emm":39,"Eml":53,"Eli":163,"Ell":61,"Er ":143,"Enw":854,"ÏŒ":61,"σ":58,"Ï‚":162,"Ï":107,"Ï€":56,"Eni":89,"Ï…":45,"Ï„":85," l":10735,"ÑŒ":66," m":19257," n":29792," o":33276,"Ñ":41," h":11620,"Cân":42," i":18800," j":98," k":864," d":30278," e":20097," f":18797," g":42173,"ч":59,"Ñ€":207," a":65561,"Ñ":158," b":14900,"Ñ‚":99," c":26227,"у":60," y":105922," u":7227," t":11355," w":8138," v":206," p":10362," s":19137," r":34831," J":2904," K":1945," H":6645,"Gel":559,"Gem":88," I":5291," N":7566," O":5351," L":15685,"Gei":1020," M":24137," B":12669," C":28504," A":20837,"С":45," F":9117," G":20264," D":11311," E":9519,"Gea":46,"л":170,"к":145," Z":352," Y":6373,"й":66," X":151,"и":260,"о":284,"н":180,"м":52,"г":58," S":18405,"Ger":213," R":8770,"в":159,"б":38," Q":329," P":10887,"а":295,"Geo":264," W":5129," V":1168,"Gen":357," U":5681,"е":260," T":9888,"д":81," â":1668," ô":1446,"Gla":300,"Gha":77," À":445," É":45," ÃŽ":41,"Gil":60," ŵ":72,"×”":41,"ו":50,"×™":46,"Gaz":46,"Gan":372,"Gal":398,"Gam":70,"Gat":72,"Gar":516,"Gas":89,"Gai":99,"Gae":1402,"Gad":59,"Gab":57,"Ùˆ":196,"ÙŠ":291,"Ù":45,"Ù‚":54,"Ùƒ":45,"Ù„":362,"Ù…":201,"Ù†":219,"Ù‡":67,"Ø®":38,"د":148,"ج":68,"Ø­":77,"ت":86,"ب":192,"Ø©":126,"ا":513,"Ø£":38,"ع":70,"Ø·":52,"Ø´":46,"س":144,"ز":53,"ر":230,"Fry":84,"Fro":64,"Fyn":271,"Fwy":63,"Flo":39,"Fla":61,"Fle":45,"Fra":319,"Fri":101,"Fre":191,"A ":1088,"Foo":73,"For":307,"Fou":44,"Foe":122,"F ":59,"Da":2498,"Cu":272,"Cw":585,"Cy":7372,"Cl":935,"Cn":42,"Co":2666,"Cr":1941,"Ce":3498,"Ch":2757,"Ci":277,"G ":104,"Ec":68,"Ed":412,"Ea":170,"Eb":590,"Dw":249,"Du":804,"Dy":1018,"Do":759,"Dr":592,"De":2160,"Dd":958,"Di":1824,"Dh":64,"Fe":2231,"H ":155,"Fa":1155,"Eu":257,"Ev":184,"Ew":725,"Ex":44,"Er":1074,"Et":355,"Es":277,"En":1155,"Em":230,"Ep":63,"Ei":1604,"El":634,"Ef":1230,"Eg":302,"Ge":2651,"Câ":42,"Ga":3313,"I ":676,"Fy":369,"Fw":115,"Fu":118,"Fr":789,"Fo":691,"Fl":182,"Fi":488,"Fh":75,"Ff":2731,"Hi ":175,"B ":244," С":45,"II ":307,"C ":911,"Av":93,"Au":228,"Aw":1145,"Ar":2753,"Aq":45,"At":330,"As":660,"D ":151,"Ba":2551,"Az":66,"Ay":58,"Ae":324,"Af":1863,"Ag":155,"Ab":1337,"Ac":626,"Ad":579,"Am":1812,"An":1298,"Ao":86,"Ap":230,"Ai":814,"Al":5324,"Hir":98,"By":401,"Bw":425,"His":66,"Bu":657,"Br":3063,"Ca":7250,"E ":255,"Bh":283,"Bi":359,"Be":2806,"Hig":55,"Bo":949,"Hil":184,"Him":44,"Bl":698,"Hin":102,"Ku":119,"Ky":127,"Kn":99,"Kr":62,"Ko":131,"Le":1967,"Li":730,"N ":179,"La":1295,"Lu":321,"Lw":47,"Ly":372,"Ll":8526,"Lo":2189,"Me":3068,"Iâ":58,"Mh":722,"Mi":742,"O ":337,"DÅ·":53,"Ma":14605,"Mc":91,"My":761,"Mw":171,"Mu":488,"Mr":121,"Mo":2328,"Nh":168,"Ni":590,"Ng":1975,"Ne":1732,"Na":1719,"P ":102,"Hel":181,"Ny":139,"Hei":49,"Nu":77,"Hed":82,"No":968,"Heb":135,"Hea":71,"Ol":406,"On":113,"Og":270,"Oh":42,"Oc":104,"Od":68,"Oe":2267,"Hen":857,"Of":86,"Hes":38,"Oa":53,"Ob":54,"Her":204,"Gi":265,"Gh":165,"Gl":1637,"Gr":2454,"Go":2441,"Gu":315,"Gy":1683,"Cô":47,"Gw":5026,"J ":134,"Ha":1868,"He":1744,"Hi":789,"Ho":750,"Hu":453,"Hw":138,"Hy":775,"र":40,"K ":39,"Ib":42,"Ia":328,"Id":236,"If":69,"Ie":177,"ा":57,"Io":765,"Im":55,"In":884,"Il":79,"Iw":647,"à¥":49,"Is":833,"It":59,"Ir":229,"Ja":937,"L ":90,"Ji":72,"Je":327,"Jo":1211,"Had":42,"Fô":38,"Jr":44,"Hae":300,"Ju":221,"Haf":198,"Hal":109,"Ka":451,"Hai":43,"M ":130,"Han":221,"Ham":150,"Kh":108,"Har":406,"Ki":402,"Haw":74,"Ke":308,"Hau":114,"Ut":49,"Ur":136,"Un":2471,"Ul":77,"Uc":2348,"W ":83,"Gyr":41,"Ty":474,"Tw":270,"Tu":595,"Gym":1100,"Tr":1825,"Ts":360,"Gyn":163,"Pê":44,"To":757,"Gyd":159,"Th":1526,"Ti":598,"Te":1081,"Gyf":142,"Ta":2097,"V ":167,"Sw":1250,"Sy":524,"St":1498,"Su":393,"Wr":303,"Wo":329,"Wi":1101,"Wh":150,"Wl":140,"Wa":840,"We":1710,"Y ":1318,"Vo":87,"Vi":334,"X ":53,"Va":335,"Grŵ":144,"Ve":206,"Uw":129,"Pw":190,"Pu":305," Ù…":63,"Pr":1739,"S ":764,"Py":119,"Gua":65,"Pe":3405,"Pa":1592,"Gui":67,"Pl":525,"Po":1922,"Pi":467,"Ph":393,"Os":135,"Ot":47,"Ou":45," ا":223,"Op":59,"Or":342,"Gŵ":111,"R ":160," ب":53,"Ow":296,"Oy":53,"Se":1837,"Sc":412,"Si":2884,"Sh":621,"Sg":470,"Sm":109,"Sl":145,"Sk":83,"Sr":86,"Sp":262,"So":559,"Ru":535,"Ry":288,"Gwr":140,"Rw":509,"U ":156,"Sa":6147,"Sb":343,"Gwy":2266,"Gwe":1233,"Re":503,"Ri":460,"Gwa":335,"Rh":3082,"Gwo":134,"Gwn":52,"Gwl":709,"Ro":2605,"Gwi":108,"Qu":256,"Mô":742,"T ":147,"Ra":632,"TÅ·":84,"Gre":610,"Gri":267,"Gra":555,"Glö":96,"b ":3328,"Gru":159,"Gro":691,"a ":51218,"Ye":72,"Ya":90,"Yn":2049,"Ym":722,"Yo":129,"Yu":66,"Yr":659,"Ys":1224,"Gle":844,"Wy":417,"Glo":100,"Gly":254,"Goo":37,"Gol":183,"Gom":67,"Got":62,"Gor":1252,"Za":73,"Ze":78,"Zi":43,"Zu":39,"God":47,"Gog":472,"Goc":85,"i ":38903,"Inv":100,"fw":1978,"fy":10068,"Ioa":44,"gd":137,"ge":9102,"gf":868,"ga":16120,"gb":189,"fl":2826,"fg":93,"ff":11831,"fi":6912,"fs":46,"bî":58,"fr":8643,"fu":2341,"ft":461,"Int":85,"fo":10368,"fn":3165,"j ":48,"gy":9888,"gw":9737,"hf":267,"dâ":43,"hg":350,"hd":1537,"he":14852,"hb":108,"ha":19081,"gn":416,"gm":64,"gl":7224,"gi":3529,"gh":4830,"gg":166,"Ion":616,"gu":1382,"gt":170,"gs":165,"gr":6508,"Ior":53,"go":16514,"dt":42,"du":6001,"dw":8140,"dy":12449,"g ":36598,"ea":4976,"eb":3180,"ec":2955,"ed":44549,"de":16065,"dd":84524,"dg":311,"df":1708,"di":29599,"dh":972,"dm":105,"dl":3195,"do":14938,"dn":1255,"ds":484,"dr":8475,"ew":8995,"ex":424,"eu":11450,"ev":608,"ey":2052,"ez":173,"fa":9608,"h ":26458,"Ind":510,"fd":990,"fe":14177,"fb":102,"eh":1107,"eg":15169,"ef":16678,"ee":943,"el":20723,"ek":164,"ej":41,"ei":27830,"ep":1574,"eo":4494,"en":34977,"em":3744,"et":14347,"es":16689,"er":35301,"ca":8269,"e ":32522,"bw":1267,"by":5500,"bs":99,"br":4597,"bu":1226,"bn":48,"bo":4368,"bl":4571,"bh":358,"bi":1556,"bb":144,"bd":48,"be":6777,"dc":38,"db":94,"da":17715,"f ":18520,"cy":8823,"cw":614,"cu":488,"ct":1580,"cs":549,"cr":2262,"co":5637,"cm":48,"cn":51,"ck":930,"cl":1111,"ci":1587,"ch":26374,"ce":3041,"cc":166,"c ":8637,"az":312,"ay":1012,"ba":8027,"d ":90625,"at":8544,"as":12110,"ar":38569,"ax":139,"aw":9171,"av":851,"au":19336,"ak":678,"al":15639,"ai":28376,"aj":129,"ao":569,"ap":1859,"am":10300,"an":51341,"ac":11287,"ad":20781,"aa":163,"ab":3228,"ag":4220,"ah":1181,"ae":38165,"af":10668,"nu":1924,"nt":12389,"ns":2901,"nr":2284,"Ifa":40,"np":53,"no":14100,"nn":11240,"nz":130,"ny":12248,"nw":13858,"nv":170,"oe":17244,"of":2843,"oc":4164,"od":16702,"oa":475,"ob":4169,"om":3404,"on":23684,"ok":359,"ol":31043,"oi":1499,"og":11702,"oh":1022,"ot":1836,"m²":86,"os":6644,"ov":478,"ou":2364,"op":7305,"oo":863,"or":19871,"r ":97506,"gŵ":65,"ox":76,"ow":1905,"oz":69,"oy":522,"pe":4549,"lâ":62,"pa":7810,"pl":881,"lê":981,"po":1801,"ph":2260,"pi":2423,"lo":8056,"ln":84,"lm":2285,"ll":25680,"ls":592,"lr":78,"lp":247,"lw":7516,"lv":181,"lu":4776,"lt":3355,"Idd":114,"lz":47,"ly":9229,"hô":55,"o ":32658,"dŵ":331,"Idr":57,"mc":302,"md":1313,"ma":12774,"mb":1263,"mg":609,"mh":1304,"me":11191,"iâ":71,"mf":130,"ml":2070,"mi":3619,"mn":407,"mm":469,"mp":1485,"mo":3170,"mr":6007,"mt":173,"ms":740,"mu":2756,"Iei":49,"iô":76,"mw":2666,"my":4653,"p ":2056,"Ies":43,"na":18605,"nb":1462,"nc":2771,"nd":8926,"ne":28781,"nf":2039,"ng":19411,"nh":3006,"ni":14670,"nj":135,"nk":382,"nl":948,"nm":771,"Ieu":53,"fô":44,"ju":54,"fî":107,"jo":72,"ki":563,"kh":206,"gâ":69,"ke":713,"ka":582,"m ":12759,"ky":143,"ks":215,"ku":123,"ko":255,"kr":68,"kk":45,"kl":117,"km":709,"kn":46,"gê":84,"li":13502,"lh":333,"lk":174,"há":37,"le":25308,"ld":3291,"lg":727,"lf":1785,"hâ":88,"la":22265,"lc":1274,"lb":4658,"n ":129973,"dî":57,"hr":4162,"hs":206,"hw":8162,"ht":520,"hu":3049,"hi":7126,"hn":1183,"ho":8111,"hl":2073,"hm":233,"dé":370,"id":14418,"ic":5942,"ib":1579,"ia":26223,"ih":218,"ig":11702,"if":9533,"ie":4390,"hy":13222,"k ":1329,"iq":80,"ir":24552,"is":9056,"it":12154,"iu":514,"iv":491,"iw":5398,"ix":147,"ii":107,"ij":103,"ik":308,"il":12119,"im":1505,"in":29957,"eë":45,"io":13225,"ip":837,"je":73,"Iar":71,"ji":138,"iz":277,"iy":334,"Ian":45,"l ":44324,"ja":217,"Iai":126,"xi":200,"xo":38,"té":52,"tî":45,"xt":49,"wy":36348,"ww":54,"z ":355,"xa":159,"tâ":87,"xe":70,"wg":664,"wi":6651,"wl":4892,"sé":43,"wm":1763,"wn":9601,"sê":51,"wo":2007,"wp":170,"wr":12516,"ws":2657,"sï":46,"wt":289,"wu":53,"rô":114,"y ":43036,"wb":421,"wa":8291,"wd":1844,"wc":1767,"wf":218,"we":14515,"ré":173,"vi":1082,"râ":57,"vu":50,"vr":48,"rï":40,"rë":66,"vo":243,"uz":117,"uy":100,"ux":122,"uw":1502,"uv":98,"ve":1501,"va":690,"x ":538,"ui":873,"uk":137,"ul":2825,"ue":842,"uf":864,"ug":2077,"uh":77,"ur":9114,"us":3544,"ut":1125,"um":1436,"un":9776,"uo":2173,"up":276,"ty":2334,"tz":134,"tu":4774,"tt":1060,"tw":874,"ub":587,"ua":3026,"ud":3346,"uc":1723,"w ":28552,"lÅ·":175,"to":4656,"tn":397,"pê":160,"tm":136,"tl":749,"ts":824,"tr":9852,"tp":42,"tg":191,"tf":144,"te":8913,"tk":40,"ti":6270,"th":34123,"v ":135,"tb":311,"tc":225,"ta":7641,"su":1034,"ss":1407,"Hyw":74,"st":10674,"sy":11549,"sw":1131,"sl":613,"sk":338,"sn":4104,"sm":342,"sp":272,"so":5309,"sr":218,"sd":421,"sc":590,"sf":301,"se":6958,"Hyd":590,"sh":1353,"sg":5428,"si":7703,"rz":64,"u ":38069,"sa":4420,"sb":869,"rr":3139,"rs":2573,"nï":350,"rt":9794,"ru":5227,"rv":239,"rw":8072,"ry":10076,"rq":40,"rp":904,"ro":15586,"në":57,"rn":7321,"né":106,"rm":2771,"rl":5238,"nç":49,"rk":493,"ri":23922,"rh":9744,"rg":2743,"rf":5415,"re":22722,"rd":11710,"rc":3476,"rb":1538,"ra":23899,"t ":12191,"mô":280,"qu":314,"Hwn":76,"mâ":48,"s ":31741,"hÅ·":72,"px":78,"lô":60,"py":248,"lö":131,"pt":1306,"pu":631,"pw":742,"pp":348,"pr":1765,"ps":228,"hÅ«":85,"Hun":62,"Hum":61,"Huw":52,"Hug":142,"tÅ·":48,"IV ":48,"wâ":83,"rŵ":459,"zz":70,"Hor":65,"zh":71,"zi":170,"Hou":114,"zb":37,"ze":198,"za":330,"Hom":49,"yy":47,"Hon":146,"Hol":122,"zu":71,"zo":149,"How":58,"yg":2872,"yh":854,"ye":236,"yf":12232,"yc":3592,"yd":50329,"་":38,"ya":2335,"yb":822,"yw":21420,"yu":51,"yt":2465,"ys":12384,"yr":22478,"yp":196,"yo":362,"yn":74012,"ym":17663,"yl":4484,"yk":65,"IR ":39,"yi":95,"Arg":192,"Are":81,"Arf":161,"Arc":172,"Ard":330,"Ara":645,"Arb":38,"Arm":93,"Ark":49,"Arl":228,"Ari":157,"App":76,"Apo":76,"Ath":193,"Atl":65,"Ast":95,"Ass":90,"Asi":362,"Art":218,"Arw":129,"Avo":40,"Aur":46,"Aug":57,"Awy":60,"Aws":918,"Awd":150,"Bai":38,"Bal":342,"Ban":571,"Bab":49,"Bac":149,"Bad":45,"Bae":152,"Baf":40,"Bay":40,"Bar":705,"Bat":87,"Bas":189,"CC ":319,"BE ":140,"Abr":43,"Aca":159,"Abe":1076,"Aba":80,"Act":324,"Ada":89,"Ach":90,"Adn":48,"Adr":99,"Add":86,"Ade":142,"Ael":205,"Aer":53,"Afo":1268,"Aff":460,"Afg":44,"Ago":41,"Aif":105,"Ail":151,"Ain":442,"Air":55,"Al ":98,"Am ":62,"Ala":129,"Alb":3548,"An ":121,"Alg":86,"Ali":71,"Ale":206,"Alf":52,"Alu":44,"Alm":666,"All":174,"Alp":112,"Amg":74,"Ame":1251,"Ama":68,"Amw":99,"Ams":41,"Ang":201,"Ani":79,"Ana":70,"And":209,"Ant":265,"Ann":197,"Ar ":131,"Aon":53,"But":65,"Bul":39,"Bur":114,"Buc":38,"Bui":58,"Bu ":164,"Bry":805,"Bru":120,"Brw":176,"Byd":283,"² ":91,"Bwl":182,"Bwr":151,"DA ":97,"Cab":43,"Cad":213,"Cae":1032,"DD ":40,"Cal":219,"Cam":227,"Caf":480,"Cai":366,"Cas":915,"Car":1877,"Cat":223,"Can":1350,"Cap":159,"Caw":62,"Bea":150,"Bet":175,"Ber":402,"Ben":733,"Bel":274,"Bei":579,"Bed":337,"Bhr":46,"Bhu":62,"Bil":57,"Bis":38,"Bir":54,"Bha":56,"Bhe":84,"Ài":440,"Blu":41,"Blo":83,"Ble":43,"Bla":500,"Bre":500,"Bra":479,"Bro":495,"Bri":461,"Bod":96,"Bob":38,"Bol":50,"Bon":223,"Boo":44,"Bor":128,"Bos":53,"Bou":115,"Bow":40,"Cyc":57,"Cyd":52,"Cys":83,"Cyr":75,"Cyt":157,"Cyn":1224,"Cym":4367,"Cyl":131,"Cyf":1040,"Cyh":146,"Cur":60,"Cul":80,"ÃŽl":39,"Cwm":473,"De ":401,"Dey":467,"Der":96,"Des":74,"Deu":61,"Dew":73,"Del":66,"Dem":67,"Den":121,"Dea":123,"Dec":105,"Ded":67,"Def":186,"Deh":65,"Ddy":46,"Ddw":51,"Ddu":131,"Ddi":405,"Dda":107,"Dde":168,"Dam":55,"Dan":185,"Dar":231,"Dat":83,"Dav":367,"Daw":158,"Dad":40,"Daf":217,"Dae":153,"Dai":40,"Dal":797,"Cho":221,"Chr":239,"Che":165,"Chi":233,"Chl":64,"Chu":124,"Chw":858,"Chy":89,"án":75,"ái":74,"Cit":58,"Cil":97,"âl":86,"ân":336,"âd":45,"âu":90,"ât":37,"âr":140,"Cle":140,"Cla":218,"Cea":48,"Cei":925,"Cef":291,"Cem":52,"Cel":239,"Cen":423,"Cer":1421,"Cha":693,"â ":1741,"Cri":276,"Cra":198,"Cre":385,"DU ":59,"Cry":52,"Cru":632,"Cro":352,"Cly":91,"Cli":57,"Clo":101,"Clw":225,"Clu":93,"Coc":117,"Coi":141,"Cof":107,"Coe":237,"Cop":81,"Cos":46,"Cor":407,"Com":238,"Col":320,"Con":652,"Cou":119,"アアア":57,"ôr":762,"ôl":1584,"ôn":601,"ón":72,"Dyw":75,"ïo":107,"Dyd":48,"Dyc":49,"în":138,"Dym":227,"Dyn":66,"Dys":44,"Dyf":415,"ïa":463,"Dyl":71,"Dwy":227,"îl":47,"îm":141,"î ":61,"ëw":53,"Egl":261,"Efr":172,"í ":45,"Efy":881,"ëd":42,"Ei ":672,"ëe":56,"ên":983,"êl":271,"êm":109,"êr":91,"él":41,"ép":354,"ém":39,"én":68,"ér":78,"ée":104,"ég":64,"Edw":243,"Edm":38,"èr":41,"ès":40,"Ef ":107,"ço":37,"é ":170,"Ebr":548,"Ear":39,"Eas":62,"Än":56,"Dia":78,"Dic":59,"Dis":198,"Dir":57,"Dio":56,"Din":930,"Dig":90,"Dif":46,"Diw":144,"Dug":61,"Dub":66,"Dun":80,"Dul":102,"Dui":91,"ể":69,"ür":59,"Duw":89,"Dur":65,"Dro":45,"Dru":57,"Dry":63,"Du ":131,"öy":129,"Dri":53,"Dre":173,"Dra":145,"Doc":73,"Dr ":38,"Dou":83,"Dow":41,"Dol":168,"Don":111,"Dom":58,"Dor":76,"Ned":55,"Nev":46,"Neu":46,"Nep":47,"Neo":67,"Ne ":182,"Nat":1075,"Nig":61,"Nid":87,"Nic":115,"Nin":78,"Nha":51,"Nhr":45,"Ngh":1380,"Nge":64,"Ni ":64,"Ngw":317,"Ngo":144,"New":1034,"Myn":603,"Nar":52,"Nan":259,"Nad":69,"OC ":70,"Nyf":95,"OS ":481,"Nor":455,"Not":37,"Nof":207,"Oes":2225,"Ogw":150,"Ogo":40,"Ogl":71,"Off":71,"PA ":37,"Oak":42,"Owa":159,"Owe":132,"Ä« ":39,"Oyk":41,"Oly":162,"Oli":119,"Ope":42,"Ora":40,"Ori":47,"Org":37,"Orl":66,"Plw":64,"Pla":369,"Pin":50,"Pil":40,"Pit":98,"Pie":81,"Pic":69,"Pho":86,"Phi":138,"Phr":55,"Phe":40,"Pea":46,"Ped":89,"Per":357,"Pet":166,"Pen":2562,"Pel":49,"Pat":108,"Pas":79,"Par":525,"Pau":125,"Pab":54,"Pac":49,"Pan":158,"Pap":93,"Pal":184,"Pak":50,"Gŵy":103,"Å ":54,"Pyr":65,"Pwy":112,"Pwl":49,"Pum":82,"Pur":41,"Pug":63,"Pro":243,"Pri":768,"Pre":133,"Pry":452,"Pra":116,"Pob":179,"Pol":106,"Pon":298,"Poi":41,"Pot":63,"Pos":39,"Por":358,"Pow":700," ال":192,"Rac":38,"Rad":90,"Rai":42,"Ram":56,"Ran":141,"Å« ":77,"Môr":383,"Môn":358,"Å· ":235,"ŵl":41,"ŵn":87,"ŵp":443,"ŵr":393,"Que":103,"ŵy":196,"Quo":68,"Å·n":164,"Å·r":76,"Isa":101,"Ise":233,"Isl":207,"Isr":118,"Is ":89,"Ira":166,"Iwe":600,"Jac":132,"Jap":343,"Jan":70,"Jam":257,"Jen":67,"Jer":98,"Jea":66,"Jim":37,"ã‚":42,"Jos":121,"Jon":426,"ã‚¢":100,"Joh":529,"Jr ":44,"Jul":71,"Jun":38,"Kan":77,"Kat":68,"Kas":44,"Kar":90,"Ker":43,"Ken":91,"Kel":37,"Kin":187,"Kil":92,"Kha":59,"Kno":77,"Kyl":83,"Lew":127,"Lev":111,"Les":58,"Ler":63,"Lep":994,"Leo":78,"Lei":113,"Leg":43,"Lee":38,"Lea":111,"Lau":47,"Law":91,"Le ":50,"Lai":41,"Lag":42,"Laf":70,"Las":63,"Lar":55,"Lao":55,"Lam":49,"Lan":381,"Lad":104,"La ":83,"Lli":81,"Llo":1177,"Lla":2896,"Lle":1205,"Lly":1369,"Llw":218,"Llu":548,"Lib":127,"Lia":147,"Lim":38,"Lin":147,"Lit":62,"Lun":66,"Luc":59,"LlÅ·":126,"Llê":901,"Lou":107,"Los":46,"Loi":64,"Loe":137,"Loc":1370,"Lor":55,"Lon":140,"Lom":91,"DÅ· ":53,"Lyd":44,"Lyn":107,"Lyo":122,"Mei":183,"Meh":551,"Men":120,"Mel":122,"Mes":98,"Mer":331,"Mew":209,"Meu":40,"Met":107,"Mec":52,"Mea":441,"Med":618,"Mex":110,"Man":377,"Mao":42,"Mal":270,"Mam":89,"Mar":903,"Mas":96,"Mag":89,"Mad":118,"Mae":10140,"Mah":50,"Mai":677,"Mac":250,"Mab":124,"May":55,"Max":37,"Mau":68,"Mat":309,"Maw":799,"Moc":42,"Mod":56,"Moe":284,"Moh":40,"Mon":614,"Moo":54,"Mos":98,"Mor":865,"Mou":90,"Mot":83,"Mid":69,"Mic":171,"Mit":41,"Mis":80,"Mil":151,"Min":107,"Mhe":110,"Mha":155,"Mhu":72,"Mhr":108,"Mho":230,"Mun":41,"Mul":66,"Mur":72,"Mus":84,"Mud":92,"Mwr":59,"Wyd":174,"Wyn":117,"Wys":47,"Wre":214,"Wor":81,"Woo":67,"Wob":86,"Wla":90,"Whi":104,"Wil":763,"Win":87,"Wic":114,"ère":37,"Web":41,"Wed":39,"Wef":896,"Wei":199,"Wel":99,"Wer":95,"Wes":154,"Wen":117,"Wav":42,"Was":76,"War":117,"Wau":61,"Wat":101,"Wal":281,"ès ":39,"égi":56,"ée ":55,"ées":42,"éné":40,"épa":346,"之":120,"両":62,"並":68,"丘":45,"三":165,"ä¸":71,"ên ":981,"êm ":104,"êl ":267,"Ysb":66,"Ysg":820,"Yst":328,"Yr ":655,"Ym ":65,"Yn ":880,"Yml":71,"Ymr":46,"Ymd":103,"Yme":273,"Ymg":39,"Yma":48,"Yng":84,"You":59,"Yny":1064,"êr ":90,"ëdi":40,"ëeg":55,"Syl":57,"Syr":267,"Sys":57,"Syd":49,"Swy":845,"Swi":176,"Swe":125,"Swa":46,"Sur":47,"Sus":41,"Sum":39,"Sul":51,"Sun":61,"Str":352,"Stu":76,"Sti":42,"Sto":284,"Sta":278,"Ste":342,"Teg":58,"Tec":38,"Ten":65,"Tei":113,"Tel":116,"Tam":59,"Tan":88,"Tat":47,"Tar":132,"Taw":121,"Tay":77,"Tai":74,"Tal":579,"Taf":220,"Tac":506,"Shi":200,"She":116,"Sho":72,"Sgu":236,"Sha":163,"Sim":72,"Sil":92,"Sis":59,"Sir":1857,"Sio":82,"Sin":79,"Sid":94,"Sie":50,"Sib":37,"Sia":245,"Ser":173,"Sgo":69,"Sgi":37,"Sen":310,"Sel":127,"Sem":38,"Sei":620,"Sef":291,"Sea":62,"Sro":50,"TV ":60,"Spa":48,"Spi":76,"Spe":51,"St ":109,"Sof":96,"Soc":69,"Sou":117,"Sol":60,"Som":57,"Son":51,"Sla":57,"Slo":44,"Sma":50,"Siô":76,"Rws":463," 三":37,"Ryg":65,"Ryd":65,"Ryf":105,"Rus":66,"Rut":39,"Ruf":288,"Saf":103,"Sai":1382,"Sah":39,"Sam":100,"Sal":155,"Sac":72,"Sae":3187,"Sad":43,"Sco":119,"Sci":39,"Sch":160,"Sca":61,"Sat":44,"Sau":57,"Sar":112,"San":631,"Sba":324,"Res":39,"SJ ":70,"Rho":422,"Rhu":416,"Rhi":208,"Rha":817,"Rhe":292,"Riv":41,"Ric":257,"Rhy":836,"Rhw":49,"Ras":58,"Ree":51,"Red":74,"Rei":43,"Reg":46,"Ren":42,"SH ":92,"Rea":48,"Roi":127,"Roe":1456,"Rog":56,"Roa":53,"Rob":347,"Roc":57,"Roy":48,"Rou":63,"Ros":111,"Ron":57,"Rom":81,"SS ":37,"ST ":48,"SN ":103,"SO ":61,"Vai":45,"Ven":60,"Van":61,"Val":137,"Vic":94,"Vir":48,"Vin":38,"Ver":74,"Und":296,"Une":566,"Uni":136,"Uno":738,"Un ":695,"Urd":80,"Uwc":129,"Twi":39,"Twr":136,"Twy":43,"Tyw":284,"Tyd":45,"Uch":2348,"Teu":41,"Tex":55,"Ter":240,"Tes":46,"Tey":148,"Tha":114,"The":818,"Thi":43,"Tho":319,"Thr":141,"Tib":106,"Til":75,"Tim":107,"Tit":38,"Tir":127,"Pêl":41,"Tor":142,"Tok":42,"Tom":170,"Ton":77,"Tow":45,"Tou":86,"Try":61,"Tru":44,"Tro":153,"Tri":147,"Tre":944,"Tra":432,"Tsi":288,"Tur":62,"Tun":253,"Tud":152,"вич":38,"bl ":1207,"biw":40,"bit":41,"bio":221,"bir":39,"bil":108,"bin":244,"big":37,"bo ":73,"bly":678,"blw":152,"blo":1344,"ble":366,"bli":117,"bla":667,"bod":1326,"bob":1364,"bol":373,"br ":444,"bon":391,"bor":263,"bot":170,"bos":79,"bou":98,"be ":98,"ban":3482,"bal":155,"bai":223,"bag":40,"bae":396,"bac":337,"bad":90,"bab":43,"án ":53,"baw":50,"bau":111,"bat":176,"bas":114,"bar":2113,"bi ":299,"bei":410,"beg":385,"bed":758,"bec":63,"ber":2389,"ben":1862,"bel":400,"bet":283,"bh ":143,"bia":315,"bid":38,"bie":42,"bha":140,"byw":545,"ca ":1177,"car":1065,"cas":275,"cat":148,"can":2308,"cap":69,"caw":70,"cae":1029,"cad":459,"cam":299,"cal":289,"caf":386,"cai":547,"ce ":639,"bri":1804,"bro":531,"bra":416,"bre":653,"bry":598,"bu ":507,"brw":83,"bur":297,"bul":47,"bum":71,"bud":56,"but":37,"bus":107,"by ":89,"bwl":66,"bwm":84,"bwr":254,"bwy":699,"byd":1610,"byc":1183,"bym":110,"byn":951,"byg":624,"bys":89,"byt":107,"byr":125,"aka":125,"am ":3780,"ake":153,"aki":95,"akh":61,"al ":3450,"ail":1481,"aim":47,"ain":6784,"aeë":39,"air":2398,"ais":581,"ait":4733,"ak ":74,"aig":842,"aif":1555,"aid":7159,"aic":285,"aia":54,"agw":108,"aha":670,"agl":348,"agf":578,"agg":47,"agh":77,"agi":70,"agu":99,"agn":135,"ago":1155,"aoi":58,"aol":79,"aod":56,"aob":43,"anw":1721,"anu":408,"anz":40,"any":320,"ano":3132,"ann":2928,"anm":62,"ant":3519,"ans":1375,"anr":1027,"ane":3419,"anf":695,"ang":2452,"anh":308,"ani":1520,"ank":171,"anl":302,"ap ":434,"ana":2445,"anb":852,"anc":833,"and":2956,"amu":51,"amw":69,"amm":156,"aml":1316,"amo":343,"amp":711,"ams":445,"amr":285,"amh":93,"ami":272,"amg":269,"amd":204,"ame":669,"amb":378,"amc":173,"ama":982,"ao ":50,"aly":92,"alw":828,"alu":142,"alt":188,"als":144,"alp":44,"alo":371,"alm":88,"all":3308,"alk":68,"alg":97,"ali":1102,"alc":202,"ald":305,"ale":2251,"alf":125,"ahâ":37,"ala":2417,"alb":252,"an ":20701,"ako":53,"aba":276,"abe":698,"abh":50,"abi":212,"abl":167,"abo":216,"abr":69,"abw":100,"aby":812,"ae ":13921,"aca":158,"ad ":6988,"âr ":137,"ac ":6256,"ab ":524,"afn":71,"afo":1900,"afr":202,"aff":533,"afe":74,"afi":258,"afl":340,"ai ":2298,"aga":251,"age":183,"afu":183,"afw":88,"afy":180,"aeo":298,"aen":2354,"aem":216,"ael":2752,"aes":3677,"aer":2715,"aeg":1463,"aed":256,"ah ":232,"afa":399,"afb":52,"aew":200,"aet":9257,"ado":1424,"adr":412,"adl":478,"adn":878,"adh":309,"adg":37,"adi":858,"add":2640,"âu ":90,"adf":220,"ade":1321,"aea":937,"ag ":1062,"adw":1093,"ady":61,"adu":1508,"aco":112,"acl":41,"ack":236,"aci":176,"ach":3246,"ace":317,"ada":2456,"af ":6324,"act":455,"acu":37,"acr":47,"acs":87,"azi":49,"aza":106,"awy":402,"ayn":40,"ays":51,"aya":127,"aye":44,"ân ":313,"ba ":592,"âl ":80,"at ":1109,"arh":141,"arg":679,"arf":1468,"are":1082,"ard":3174,"arc":1261,"arb":599,"ara":2389,"arp":203,"aro":1162,"arn":2998,"arm":191,"arl":1098,"anç":39,"ark":227,"ari":1312,"aru":190,"arv":109,"arw":1994,"arr":856,"ars":273,"art":3281,"au ":18148,"asa":524,"ary":397,"asg":690,"asi":753,"ash":208,"asc":78,"ase":115,"asd":54,"aso":389,"asn":260,"ask":48,"asm":43,"aon":63,"aor":71,"aos":40,"ar ":13348,"apa":506,"ape":190,"api":138,"aph":68,"apo":90,"app":56,"apt":39,"apu":247,"as ":6724,"ava":74,"ax ":61,"aut":123,"avo":39,"avi":469,"ave":206,"awe":760,"awd":1261,"ay ":572,"awa":150,"awr":2782,"aws":410,"awn":857,"awl":555,"awf":124,"awg":51,"awi":82,"atb":229,"ata":746,"asu":218,"ast":1476,"ass":281,"asy":50,"asw":75,"atm":41,"atr":340,"ato":500,"âd ":43,"ate":809,"atc":82,"ati":760,"atg":130,"ath":1938,"aw ":1576,"att":185,"ats":96,"atu":1218,"atw":81,"aty":152,"aul":281,"aun":217,"aur":199,"aus":70,"aud":91,"aug":56,"ος":78,"ος ":78,"Ï‚ ":162,"α ":57,"アア":78,"TÅ· ":84,"ич ":40,"jan":42,"Ñк":52,"itl":241,"itr":65,"ito":86,"itu":48,"itt":152,"itz":75,"ity":126,"iw ":670,"ism":50,"isl":41,"iso":170,"isn":521,"iss":179,"ist":1922,"isw":48,"isy":118,"ita":465,"itc":77,"ite":660,"ith":9523,"iti":268,"iwd":40,"iwe":1260,"iwb":73,"iwc":99,"iwa":49,"iwl":119,"iwm":233,"iwi":130,"iwg":119,"ius":367,"ium":95,"iva":70,"ix ":95,"ivi":57,"ive":318,"ipp":108,"ips":45,"ipt":130,"ipi":109,"ipl":76,"is ":2910,"ion":4410,"iop":79,"ior":107,"ios":73,"iot":52,"iog":969,"iol":2152,"ipe":92,"ir ":16769,"irw":755,"irs":63,"irt":106,"irr":47,"iro":1087,"inë":53,"irn":1040,"irk":39,"irl":116,"iri":2683,"isi":1027,"ish":225,"isg":710,"ise":398,"isd":236,"isc":102,"isb":45,"isa":207,"iry":54,"iqu":69,"irf":117,"ire":717,"irg":231,"ira":269,"ird":247,"irc":70,"it ":213,"ja ":42,"iyn":234,"iya":51,"iwy":1090,"iwn":438,"iwt":68,"iwr":950,"iws":49,"izo":45,"iza":90,"kil":38,"kin":180,"kis":60,"km ":586,"ki ":114,"kha":65,"kel":108,"ken":115,"kes":50,"ker":140,"gân":47,"ke ":171,"ku ":48,"km²":84,"ks ":103,"gêm":78,"ko ":44,"kle":37,"kla":44," Ài":440,"fîn":104,"kar":68,"kan":58,"kai":44,"ka ":196," Ga":3309," Câ":42," Ge":2633," Fy":369," Fw":115," I ":222," Fo":687," Fu":118," Fr":788," Fh":75," Fi":485," Ff":2729," Fl":182," Ha":1860," He":1741," Cô":47," Gw":5007," Gy":1682," J ":50," Go":2436," Gr":2427," Gu":311," Gh":165," Gi":263," Gl":1637," If":69," Ie":177," Id":236," Ib":42," Ia":327," Hy":775," Hw":138," Hu":453," Ho":749," Hi":788,"ha ":291," Ji":72," Je":327," L ":42," Ja":936,"ان ":40," Iw":647," Ir":229," Is":832," It":59," Im":55," In":879," Io":765," Il":78,"ham":591," M ":53,"han":7727,"hao":141,"hap":50,"hai":1115," Ka":449,"hak":48,"hal":1482," Ke":303,"hau":1360,"haw":364," Ki":398,"har":2065,"has":612," Kh":108,"hat":129," Jo":1198," Ju":220,"haf":786,"hae":657," Jr":44,"hag":961,"hab":62," Fô":38,"had":434,"hac":101," La":1290," Le":1952," Li":729," Kn":99," Ko":131," Kr":62," Ku":119,"hbe":70," Ky":127," Mc":91," Ma":14593," DÅ·":53," O ":158," Mi":739," Mh":722," Me":3065," Iâ":58,"he ":1354," Lo":2188," Ll":8523," Ly":372," Lw":47," Lu":320," Ne":1728,"а ":68,"hdr":51," Na":1715," Ng":1975," Nh":168," Ni":588," Mr":121,"hda":91," Mo":2327," My":759,"hde":1261," Mu":485," Mw":171,"hel":2523,"hei":954,"heg":242,"hee":57,"hef":2098,"hec":90,"hed":1184,"hea":360,"heb":374,"hey":61,"hew":93," A ":750,"heu":445,"het":177,"hes":532,"her":1843,"heo":518,"hen":1636,"hem":246,"hfe":111,"hfa":83,"hi ":668," B ":201,"hga":94," C ":232,"hgy":60,"hgr":144," Ao":85," Ap":230," Am":1811," An":1296," Al":5312," Ai":814," Ag":155," Ae":323," Af":1863," Ac":625," Ad":573," Ab":1334," Ba":2550," D ":49," Az":66," Ay":58," Aw":1144," Av":92," Au":228," At":329," As":660," Ar":2746," Aq":45,"hig":403,"hif":269," Be":2802,"hie":154,"hid":60,"hic":96," Bi":359,"hia":2000," Bh":283,"hip":49,"hio":970,"hin":783,"him":84," Bl":695," Bo":944,"hil":345," Br":3059,"hiw":167," Bu":657,"his":167,"hit":155," Bw":425,"hir":643," By":399," E ":42," Ca":7220," Ce":3495," Ci":275," Ch":2756," Cn":42," Cl":933," Cr":1937," Co":2656," Cu":270," Cy":7369," Cw":584," Da":2496," Di":1823," Dh":64," De":2156," Dd":954," Dr":592," Do":753," Dy":1018," Du":803," Dw":249,"hn ":464," Ea":167," Eb":590," Ec":68," Ed":412,"hla":1210," El":634,"hle":156," Ei":1604,"hli":122," Eg":302," Ef":1230," Et":355,"hll":37," Es":277," Er":1072,"hlo":125," Ep":63," En":1110," Em":230,"hlu":57,"hlw":214," Ew":723," Ex":43,"hly":120," Eu":257," Ev":183,"ho ":75," Fe":2231,"hma":89," Fa":1154,"go ":385,"glw":795,"glu":132,"gly":189,"glo":190,"gle":4369,"gli":289," Wy":416," Wr":303,"gla":882," Wo":325," Wl":140," Wi":1099," Wh":149," We":1708," Wa":840," Y ":1307,"й ":51,"gog":3238,"gof":323,"goe":182,"god":705,"goc":155,"gob":234," Ze":78,"gno":66," Zi":43,"gni":87," Za":73," Ys":1224," Yr":658,"gne":142," Yu":65,"gna":64," Yo":129," Yn":2049," Ym":722," Ya":90," Ye":72,"gs ":64,"gr ":1217,"gom":145,"gol":3429,"gon":261,"gop":2404,"gos":1215,"gor":3586,"got":51,"gow":97,"gu ":507," a ":25270,"gro":1041,"gry":291,"grw":824,"gru":108,"gra":788,"gri":1359,"gre":514," TÅ·":84," Gŵ":111," Oy":53," Ow":296," Ou":45," Os":133,"gto":136," Ot":47," Or":342," Op":59," Po":1917," Pl":523," Pi":467," Ph":390,"gul":65," Pe":3404,"gua":83,"gue":160," Pa":1581,"gst":47," Ny":139," Nu":77," No":968," Ol":406," On":113," Oh":42," Og":269," Od":68," Oc":104," Of":85," Oe":2267," Ob":54," Oa":50,"gy ":41," Ra":627," T ":37,"grë":37," Mô":741," Qu":254,"gwm":270,"gwl":1426,"gwo":40,"gwn":213,"gwi":169," Ro":2598," Re":502,"gwe":1487," Ri":459,"gwa":1284," Rh":3078,"gwb":37," Py":119," S ":95,"gur":278," Pr":1731,"gus":127," Pu":304,"gun":41," Pw":190," Sy":523," Sw":1250," Su":392," St":1487," Ta":2095," V ":40,"gyb":143,"gyd":2260," Th":1521," Ti":598,"gyf":2828,"gyh":327," Te":1075," Tr":1823," Ts":358,"gyl":394,"gym":698,"gyn":1911," Pê":44," To":755," Rw":509,"gwr":1885," Ry":288," Ru":535," Sb":343,"gwy":2799," Sa":6139," Sg":470," Sh":618," Si":2880," Sc":411," Se":1836," So":559," Sp":260," Sr":86," Sk":82," Sl":145," Sm":109," Uw":129," Va":334," Ve":204,"grŵ":298," Vi":332,"gwâ":55," Vo":86," Tu":591,"gyw":46," Tw":270," Ty":472,"gyt":130,"gys":648,"gyr":399," Uc":2348," Ul":77," Un":2470," Ur":136," Ut":49,"iai":1947,"iam":862,"ial":272,"ian":2596,"iap":95,"ias":109,"iar":462,"iau":4682,"iat":304,"iaw":277," io":61," in":191," il":42,"ic ":408,"iac":97," is":434,"iad":7064,"iae":3103,"iaf":180,"iag":47,"ibl":135,"ibi":164,"ibo":40," m ":291,"ibr":68," ki":48," gâ":47," ke":53,"id ":3479,"iba":122,"ibh":49,"ibe":421," gy":7787," gw":7430," ha":2143," he":3350," gi":324," gl":929," gr":2189," go":7824,"ia ":4053," gu":50," hy":3013," ia":812," id":384," if":86," ie":418," hi":741,"ib ":99," dé":347," ho":683," dî":55," hu":549," hw":571,"iet":350,"ieu":94," nh":758," ni":1109," ng":3326,"iel":338," ne":8387,"ien":309," na":2311,"ier":266,"ies":434,"ied":632,"ieg":134," my":3937,"ief":37,"iei":784," mw":1538," mu":301,"ig ":8077," mo":510,"iec":167,"ifw":100," ol":455," on":1046," og":435," oh":320,"ify":395,"ifo":628," oc":254," od":327," oe":5934," of":830,"ibî":56,"ifr":76,"iff":1472,"ife":1414,"ifl":67," nw":267," ny":117,"ifi":808," no":795,"ifd":919,"ifa":410," le":1343,"icr":127,"ics":52,"ict":213,"icu":74," li":214," n ":12651,"ico":318,"ick":291,"icl":187," la":1153,"ici":221,"ich":1469,"ice":249," gê":83,"ie ":736," km":670,"ica":2272,"iby":308,"idy":707," me":5050," iâ":40,"idw":100," mh":245," mi":1482," ml":96," dŵ":166," o ":21338,"idr":79,"ido":1466," ma":5425,"idl":131," lu":101,"idi":544," lw":161,"idh":303,"idg":97,"idf":47," ly":443,"ide":336,"idd":6222," ll":7015,"ida":808," lo":202,"if ":3195," ae":587," af":757," ag":1206," ab":262," ac":6621," ad":2600," am":4335," an":2101," ap":337," ai":1226," al":881," au":347," aw":854," ar":16211," at":1462," as":456," d ":109," ba":1485,"il ":1444," bi":247," be":2463," bo":2899," bl":1371," by":2374," bw":689," bu":556," br":2722," ca":4243," e ":202,"im ":424,"ika":52,"ige":189,"iga":404,"ii ":78,"igl":50,"igh":415,"igi":940,"igu":58,"igr":271,"igo":654,"ign":71,"igy":145,"igw":328,"iha":170,"ik ":64," c ":146,"imo":119," er":1863," et":513,"iml":45," es":187," en":8086," em":89," ep":59,"imp":49," ei":5329,"ime":308," el":685," ef":594," eh":46,"imi":96," eg":690,"ip ":126," fe":5001,"inc":980," h ":491,"ind":417,"ina":4133," fa":2334,"inb":394," eu":1065," ew":44,"inn":805," fu":348,"inm":467,"ino":1100," fr":934," fo":1170,"int":968,"ins":292,"inf":90," fl":1325,"ine":1198,"ing":1306," fi":507,"ini":3171,"inl":130," ff":4585,"ink":95," ge":5478," ga":9934,"ioc":84,"eëd":39,"iod":501,"ioe":189,"inu":82," i ":16205,"inw":155,"iny":618," fy":1075," fw":1451," cl":534,"iko":49," cn":41," cm":41," co":4045," cr":1706,"iki":37," ce":1490,"ike":47," ch":4604," ci":218,"ila":556," da":4373,"in ":13332," cu":135," cw":407," cy":8555," do":719,"ilo":375,"ill":4682," dr":3411,"ilm":1379,"ilg":87," de":5281," dd":7964,"ili":1564,"ild":117," di":3478,"ilf":180,"ile":398,"ima":234,"imb":55,"ч ":45," ec":224," ed":232,"io ":4492," ea":139,"ilw":317," dw":1202,"ily":579," du":387,"ils":48,"ilt":144,"ilu":39," dy":2712,"ль":41,"hs ":119,"hr ":234,"how":182,"hoy":58,"hol":2341,"hom":352,"hon":1206,"hog":250,"ко":46,"hoi":337,"hos":458,"hot":44,"hou":76,"hoo":77,"hop":78,"hor":447," yn":41808," ym":5719,"hob":365," yw":15955,"hof":49," yr":12153,"hoe":1138,"hod":477,"hoc":66," ys":1828,"hni":112,"hno":169," yc":300," yd":4387,"hna":273," tî":44,"hne":101," wy":632,"ич":41,"hmo":38,"dép":339,"hui":111,"hug":114,"ро":38,"huf":332,"hud":119,"hua":173,"hub":85,"hw ":113,"hto":62,"ов":62,"hu ":783,"hru":48,"hry":410,"dîm":54," tÅ·":48,"hro":487,"hre":1207,"hri":715,"ht ":286,"на":37,"hra":1003," ry":627," rw":43," u ":582," sa":1147," sb":90,"hyf":1088," se":3484,"hyh":50," sc":46,"hyg":114,"hyb":55," si":1946,"hyd":3013," sh":72," sg":392,"hyc":428,"hyn":4010," sm":89,"hym":1892," sp":64,"hyl":212," so":173,"hwr":536,"ви":49," mô":277," t ":111,"hwy":1500," ra":1107," re":611," ri":136," rh":8680," ro":1221," pw":499," pu":181," pr":1508," s ":280," px":78," lô":57,"hy ":169," py":162,"hwa":1012,"hwc":43,"hwe":1597,"hwi":274,"hwn":2898,"hwm":129," os":109,"hum":201,"hun":510,"hus":102,"hut":107," op":73," or":2059,"hur":322,"ан":37," r ":22194," gŵ":50," pe":3258," pa":944," pl":527," po":1095," pi":236," ph":1652," wa":778," we":3663," rô":102," y ":23704," wr":987," wo":70," wi":144," wn":253," sê":49," wl":1199," va":38," ve":41," uw":917," vo":48," vi":53," ré":57," uc":1154,"ер":44," w ":359," ty":390," tw":61," tu":2121," ur":1014," un":3364," ug":115," ta":1382,"hyt":162,"hys":337," sw":506,"hyr":987," sy":9830," st":735,"hyw":645," su":140,"ев":40," tr":2664," pê":155," to":326," th":2240," ti":314," te":1558,"ffu":1091,"fft":332,"ffw":181,"ffr":1792,"fi ":462,"ffy":559,"ffe":1545,"ffa":542,"ffl":109,"ffo":2073,"ffi":2487,"fet":126,"fes":342,"fer":2706,"fey":110,"few":292,"fec":310,"fed":1584,"fen":1795,"fel":4370,"fei":2068,"feg":51,"fia":821,"fga":38,"fgh":43,"fbw":52,"fba":41,"faw":347,"fau":218,"fas":178,"fat":439,"far":1398,"fam":349,"fan":3403,"fal":398,"fai":692,"fag":106,"fae":493,"fad":65,"fac":250,"fab":257,"ff ":917,"fdd":922,"fe ":363,"fa ":921,"eyr":966,"eys":88,"eyd":96,"eye":40,"exa":133,"ez ":93,"ewy":1561,"ews":78,"ewr":200,"exi":125,"eta":251,"ete":326,"eti":221,"eth":10791,"etn":119,"esp":56,"esn":3171,"eso":767,"est":1731,"esu":390,"ess":369,"esy":588,"esw":148,"ev ":37,"eua":272,"euc":47,"eub":64,"eue":51,"eud":983,"eug":110,"eul":766,"euo":648,"eun":167,"eto":117,"etr":1100,"ets":101,"ett":277,"etu":42,"etw":145,"ety":70,"ew ":378,"eve":268,"eva":81,"evi":156,"euw":78,"eut":118,"eur":164,"eus":156,"ex ":115,"euy":40,"ewi":3673,"ewe":71,"ewo":68,"ewn":2724,"ey ":759,"ewa":76,"epe":37,"epi":1038,"eph":182,"er ":10194,"epa":97,"eor":285,"eol":3405,"eop":55,"eon":268,"es ":7548,"ept":38,"epp":38,"erk":43,"erl":567,"eri":3199,"erg":452,"erh":118,"ere":2066,"erf":1312,"erc":671,"erd":1936,"era":2569,"erb":725,"et ":700,"esl":60,"esm":38,"esf":58,"esg":225,"esh":104,"esi":604,"esb":120,"esc":62,"esd":86,"ese":321,"eu ":7692,"esa":201,"ery":875,"erv":60,"eru":112,"erw":570,"err":1317,"ert":3212,"ers":1524,"ern":900,"erm":1779,"erp":114,"ero":902,"en ":8133,"elb":58,"ela":658,"eld":2120,"elf":432,"ele":1904,"eli":827,"elg":144,"elm":49,"ell":3094,"elo":1131,"elp":43,"elu":92,"els":154,"elt":533,"ely":937,"elw":957,"eo ":144,"emb":119,"ema":883,"eme":719,"eml":44,"emo":887,"emi":335,"emp":59,"emy":94,"enf":555,"ene":2395,"enh":570,"eng":830,"enb":102,"ena":618,"end":929,"enc":514,"eno":688,"enm":181,"enn":3626,"enk":49,"enl":180,"eni":1683,"enw":7750,"enu":97,"ens":549,"ent":4104,"enr":449,"enz":60,"eny":811,"eog":53,"eoe":80,"eod":76,"egl":298,"ego":998,"egn":70,"ege":153,"egf":89,"egi":546,"eha":71,"egr":1255,"egu":49,"egw":168,"egy":248,"ehe":930,"ek ":72,"eib":184,"eic":354,"eia":266,"eip":43,"eis":1705,"eir":7366,"eim":111,"eil":1530,"eio":37,"ein":3860,"eid":2056,"eig":925,"eif":224,"el ":7486,"eit":4122,"em ":460,"gl ":325,"git":112,"gis":343,"gir":83,"gil":269,"gip":42,"gin":207,"gio":1071,"gid":96,"gie":91,"gig":167,"gia":697,"ghy":1825,"ghw":91,"ghr":368,"ght":218,"gho":144,"ghe":333,"gha":1381,"gga":62,"gfy":612,"gi ":259,"gfa":38,"gfe":149,"gen":960,"get":83,"ger":2409,"ges":210,"gh ":361,"ged":221,"gei":2736,"geg":490,"gef":158,"gem":288,"gel":883,"gdd":63,"ge ":527,"gbi":171,"gac":62,"gad":298,"gae":2618,"gaf":171,"gai":1331,"gas":141,"gar":1053,"gau":316,"gat":114,"gaw":58,"gam":111,"gal":587,"gan":8980,"gap":38,"ga ":202,"fyw":141,"fys":392,"fwy":1473,"fwr":292,"fyr":1571,"fyl":222,"fyn":2565,"fym":71,"fyg":53,"fyd":4950,"fyf":43,"fur":1066,"fus":43,"fwn":147,"fta":48,"fud":244,"ful":86,"fun":259,"fug":165,"ft ":323,"fra":2027,"frg":79,"frd":59,"fre":1841,"fri":1485,"fu ":398,"fro":1336,"fru":72,"frw":347,"fry":995,"for":3487,"fos":53,"fot":51,"fon":2342,"fol":717,"bî ":57,"fr ":375,"öyn":126,"fna":74,"fne":58,"fnd":51,"fnf":190,"fnu":70,"fni":126,"fno":916,"fod":2910,"fog":67,"foe":238,"fny":1148,"fle":446,"fn ":482,"fla":336,"fli":205,"flu":96,"flo":137,"fo ":353,"fly":274,"flw":1207,"fid":83,"fic":52,"fie":361,"fih":95,"fig":170,"fil":1986,"fin":1391,"fio":735,"fir":153,"fis":223,"fit":156,"fiw":182,"fl ":121,"ffî":106,"da ":2613,"dd ":49936,"de ":2554,"dbw":38,"dac":62,"dad":291,"dal":2984,"dai":3023,"dag":237,"dae":1159,"daf":139,"dat":359,"das":169,"dar":1300,"dan":1183,"dam":229,"day":43,"daw":457,"dau":3395,"dda":2729,"dde":2964,"ddf":808,"ddg":83,"ddh":205,"ddi":10903,"ddl":174,"ddo":8493,"ddr":255,"ddu":2078,"ddw":3166,"ddy":2549,"df ":92,"cul":125,"cud":84,"ctu":101,"ctr":127,"cto":775,"cti":292,"cte":79,"cta":132," ÃŽl":39,"cwl":44,"cwm":218,"cwr":96,"cwe":39,"cy ":84,"cus":119,"cym":1147,"cyl":309,"cyh":309,"cyf":2990,"cyt":158,"cys":475,"cyr":140,"cyn":2866,"cyd":196,"cyc":79,"cwy":42,"cyw":55,"cks":58,"cki":49,"ckl":44,"cla":175,"cle":277,"clu":110,"clw":122,"cli":85,"clo":200,"cly":91,"co ":406,"cod":670,"coe":175,"cof":150,"cob":38,"coc":219,"con":391,"col":301,"com":217,"cor":495,"cos":91,"cop":2154,"cot":157,"cou":73,"cs ":78,"ct ":59,"cre":794,"cra":272,"cri":112,"cru":57,"cro":299,"crw":405,"cry":237,"csa":239,"cso":73,"csi":110,"cci":42,"cca":45,"cea":95,"ch ":6978,"cer":588,"ces":90,"cen":351,"cep":40,"cem":115,"cel":207,"ceg":95,"cef":174,"cei":440,"ced":138,"ci ":202," â ":1666,"chb":78,"cha":3502,"chd":1364,"chw":2809,"chu":496,"chy":2533,"cia":409,"ck ":502,"cie":97,"chg":225,"che":3420,"chf":102,"chl":1181,"chi":932,"cho":903,"chm":87,"chn":386,"chs":64,"cht":200,"chr":1036,"ciw":72,"cil":222,"cis":143,"cit":38,"cin":98,"cio":66,"cip":70,"cm ":41,"cke":114,"ed ":6517,"eba":152,"ebe":278,"ôn ":543,"ebi":94,"ebl":52,"ebo":143,"ebr":165,"ebu":150,"eby":658,"ec ":78,"eac":183,"eag":286,"eaf":76,"eae":48,"ead":313,"eak":38,"eai":225,"ean":995,"eal":663,"eam":55,"ear":1078,"eas":85,"eat":353,"eau":164,"ôl ":1565,"eb ":1399,"ea ":342,"efi":914,"efn":2061,"efo":588,"ôr ":759,"efa":1730,"efe":318,"eff":454,"efy":3744,"ei ":4933,"ega":438,"efr":578,"eft":52,"een":324,"eel":41,"eed":37,"ees":61,"eer":37,"eet":97,"edl":1308,"edn":55,"edi":6449,"edd":25528,"ede":614,"ône":45,"edf":354,"eda":429,"eg ":10783,"edw":785,"edy":568,"edu":747,"edo":543,"edr":525,"eck":79,"ech":1424,"eci":144,"ece":39,"eca":88,"ee ":253,"ef ":6146,"ect":300,"ecs":273,"ecr":59,"eco":330,"dyg":246,"dyf":430,"dyl":322,"dym":701,"dyn":2076,"dys":437,"dyr":98,"dyd":3079,"dyc":204,"dwy":4886,"dwi":169,"dwe":581,"dwc":74,"dwr":1205,"dwl":78,"dwa":882,"dy ":4499,"duw":218,"dur":2144,"dus":635,"dyw":304,"dor":959,"dop":990,"don":2366,"dom":277,"dol":5215,"dow":54,"dos":197,"dr ":1131,"ds ":246,"dna":932,"dne":151," ôl":1443,"dno":132,"doc":100,"dod":1351,"doe":1307,"dog":979,"dso":55,"dw ":187,"dun":58,"duo":40,"dul":211,"dug":82,"dud":167,"dri":671,"dra":2006,"dre":2155,"drw":337,"dry":370,"du ":2339,"dro":1676,"dru":89,"dsh":71,"dha":318,"dge":156,"dgr":60,"dic":115,"did":295,"dia":3817,"dib":66,"dhe":140,"der":2766,"des":358,"det":47,"deu":912,"dew":315,"dey":177,"dh ":410,"dfa":511,"deb":1171,"dea":111,"ded":492,"dec":320,"def":1498,"dee":65,"deh":309,"deg":755,"dei":1883,"del":971,"den":795,"dem":336,"dep":41,"deo":150,"dfu":76,"dfr":146,"dfw":52,"di ":5212,"dga":45,"dfe":259,"dff":43,"dfo":480,"dle":505,"dla":968,"ddŵ":75,"do ":972,"dly":168,"dlw":464,"dlu":272,"dlo":314,"dli":219,"dl ":277,"diu":39,"diw":2623,"dim":202,"din":4906,"dio":2802,"dip":63,"dir":4606,"dis":563,"dit":87,"die":231,"dif":421,"dig":3101,"dil":392,"rgy":156,"rgu":68,"rgw":86,"rhe":939,"rha":3805,"rhi":381,"rhu":44,"rho":350,"rfy":869,"rga":737,"ri ":1789,"rgl":177,"rgi":149,"rgh":39,"rge":464,"rgr":243,"rgo":165,"ret":318,"res":1898,"rev":41,"reu":562,"rew":191,"rey":132,"rfa":309,"rfb":45,"rfe":703,"rff":1405,"rfi":631,"rfl":49,"rfo":973,"rdu":122,"rds":156,"rdr":70,"rdy":486,"rdw":84,"rg ":415,"reb":115,"rea":724,"ree":306,"ref":6655,"rec":445,"red":2528,"rei":2138,"reg":1012,"rem":807,"ren":2559,"rek":37,"rel":299,"rer":65,"reo":299,"rf ":340,"rda":1352,"rcy":41,"rcu":73,"rct":79,"rdo":260,"rdi":824,"rde":771,"rdd":6359,"re ":1508,"rby":562,"rbw":42,"rbu":39,"rbr":42,"rco":44,"rci":169,"rch":2439,"rce":170,"rca":64,"raw":994,"ray":146,"rd ":1148,"rap":92,"rar":127,"ras":623,"rat":596,"rau":1291,"rav":72,"rbi":70,"rbh":58,"rbo":108,"rba":80,"rbe":489,"rc ":329,"rai":5122,"rah":144,"rag":225,"ran":2411,"ram":999,"ral":1583,"rak":64,"rab":559,"raf":654,"rae":3686,"rad":1700,"rac":707,"rpo":41,"rs ":905,"rpw":58,"rpe":88,"rpa":573,"rr ":283,"rpi":50,"ror":608,"ros":1473,"rot":256,"rom":276,"ron":2092,"roo":125,"rop":851,"roy":52,"rou":509,"rov":96,"row":190,"rob":80,"roa":132,"rod":1050,"roc":437,"roi":253,"rol":2368,"rof":352,"roe":2453,"nëe":50,"rog":406,"rno":1027,"rns":37,"rnw":59,"rny":151,"rna":1374,"rng":123,"rne":1846,"rni":383,"rmo":210,"rms":122,"rmw":43,"ro ":1459,"rma":1380,"rme":127,"née":69,"rmi":264,"rly":277,"rlu":234,"rlo":220,"rll":3618,"rli":209,"rld":49,"rle":285,"rla":139,"rn ":2146,"rks":56,"rke":72,"rka":46,"rm ":554,"riz":47,"riw":213,"rix":39,"rl ":122,"rip":190,"rio":1986,"rir":398,"rit":599,"ris":1179,"riv":51,"riu":72,"rig":1620,"ril":699,"rik":45,"rin":1456,"rim":124,"ria":4427,"rib":415,"ric":2311,"rid":570,"rie":637,"rif":5003,"rhw":2579,"rhy":1627,"rk ":218,"rwg":50,"rwe":1416,"rwi":121,"rwp":99,"rwo":224,"rwn":561,"rwm":69,"rws":144,"rwr":297,"rwy":3684,"ryc":824,"ryd":2082,"ryf":329,"rui":70,"rug":759,"ruf":148,"rud":120,"ruc":119,"run":62,"rum":94,"rus":185,"rut":66,"rva":46,"rvi":51,"rve":108,"rwa":160,"rwb":38,"rwc":56,"ry ":1037,"rsi":244,"rso":354,"rsa":276,"nïa":320,"rsh":49,"rse":307,"rsy":112,"rta":429,"rst":96,"rsw":71,"rtn":57,"rto":132,"rte":612,"rth":6409,"rti":407,"rw ":1096,"rts":105,"rtr":278,"rty":80,"rt ":1113,"rqu":38,"rro":150,"rri":1366,"rre":437,"rra":424,"ru ":3404,"rry":313,"rru":55,"rrw":61,"sab":56,"sac":80,"sad":50,"sae":126,"saf":1227,"sai":432,"sak":39,"sal":122,"sam":242,"sba":268,"sbe":72,"sbi":64,"san":708,"sau":254,"sat":40,"sar":175,"say":42,"saw":457,"sa ":271,"ón ":59,"ryw":1557,"rys":874,"ryt":114,"ryr":190,"ryl":141,"rym":301,"ryn":2332,"ryg":165,"sha":130,"sgu":121,"sgw":280,"sgy":278,"sho":46,"sht":41,"she":119,"shi":361,"si ":461,"sga":258,"sgo":1819,"sgl":367,"sgr":1098,"sgf":73,"sge":250,"sgi":173,"siw":382,"siy":220,"sie":542,"sid":204,"sic":303,"sib":67,"sia":2244,"sk ":65,"sit":100,"sir":1014,"sis":167,"sin":210,"sio":756,"sil":315,"sim":40,"sif":99,"sig":463,"sda":48,"sdi":60,"sbr":83,"sbo":63,"sbu":82,"sby":154,"se ":534,"sca":115,"sci":60,"sch":110,"sco":172,"sex":75,"sey":83,"ser":1507,"ses":213,"set":136,"sfa":40,"sh ":444,"sff":143,"sfo":43,"sdr":221,"sg ":658,"sdy":51,"sei":619,"seg":442,"sef":1510,"see":37,"sed":256,"sec":40,"seb":56,"sep":89,"sen":717,"sem":62,"sel":456,"spo":55,"shÅ«":62,"spe":66,"spi":54,"spa":52,"sou":66,"sol":527,"son":872,"sor":102,"sos":82,"soe":1016,"sod":829,"sog":1454,"sof":55,"soc":64,"su ":192,"sra":159,"st ":1362,"ss ":400,"sli":58,"slo":45,"sky":41,"sla":296,"sle":133,"ski":51,"sko":40,"ska":87,"sna":146,"sni":454,"sne":3482,"smo":123,"so ":95,"sma":85,"sme":38,"swr":60,"swo":44,"swm":55,"swl":42,"swi":39,"swe":46,"swy":714,"syc":54,"syd":2408,"syn":267,"syt":47,"sys":246,"sym":497,"syl":933,"sse":383,"ssa":154,"sso":142,"ssi":225,"ste":2086,"sta":1396,"stn":164,"sto":1089,"sti":810,"stl":111,"stu":468,"stw":328,"str":1798,"sty":866,"sul":51,"sut":42,"sus":58,"sur":508,"sy ":6998,"swa":56,"tai":1174,"tal":1078,"tae":112,"taf":985,"tag":117,"tab":136,"tac":100,"tad":428,"tbl":227,"tba":61,"taw":293,"tau":148,"tat":257,"tas":137,"tar":586,"tan":960,"tam":161,"tch":187,"te ":1064,"ta ":832,"pa ":4575,"pe ":114,"par":781,"pat":75,"pas":659,"pau":82,"pac":45,"pal":108,"pai":175,"pao":47,"pap":107,"pan":983,"phe":446,"pha":149,"phw":418,"phu":46,"phr":313,"pho":428,"phl":179,"phi":110,"pi ":55,"ph ":99,"lân":38,"pea":218,"pec":49,"ped":316,"pen":982,"per":2223,"pet":177,"pes":77,"pei":78,"pel":255,"pla":375,"ple":218,"plo":82,"ply":40,"plw":76,"plu":38,"phy":51,"pia":550,"pid":1048,"pig":55,"pil":126,"pin":161,"pio":130,"pis":45,"pit":95,"por":199,"pop":88,"pot":107,"pos":99,"pon":96,"pol":295,"pob":661,"poe":47,"ps ":77,"hÅ« ":64,"ppi":61,"ppl":52,"ppe":110,"po ":38,"lên":948,"psi":40,"pte":1044,"pti":107,"pto":69,"pra":67,"pt ":38,"pry":124,"pri":908,"pre":259,"pro":331,"pwr":173,"pwt":58,"pwl":73,"pwy":372,"lôn":57,"pur":257,"pus":49,"pum":128,"pul":54,"px ":78,"pyn":54,"pyt":61,"pys":41,"löy":126,"qua":43," ŵy":55,"que":158,"qui":76,"môr":276,"ra ":2000,"ngo":2653,"ngi":310,"ngl":561,"ngu":161,"ngw":591,"ngr":197,"ngt":147,"ngs":104,"ni ":1483,"nfy":113,"nge":1187,"ngh":2527,"nga":1043,"ngd":98,"nho":72,"nhy":526,"nhw":208,"nhr":112,"nha":1076,"ngy":306,"nhi":540,"nhe":398,"neg":4193,"nei":264,"nel":633,"nen":226,"nem":48,"ner":1242,"net":231,"nes":2270,"neu":7331,"ndw":46,"ndy":248,"ng ":9397,"nea":151,"neb":716,"nec":67,"ned":7692,"nef":63,"nfi":141,"nfo":387,"nfr":376,"nfu":53,"ney":242,"new":803,"nh ":42,"nfa":429,"nff":206,"nfe":298,"ncr":37,"nct":92,"nco":109,"nci":267,"ncl":53,"nce":463,"nch":147,"nca":318,"ne ":2471,"nby":393,"nbu":88,"ndu":125,"ndr":932,"nds":144,"ndo":502,"ndl":46,"ndi":866,"nde":1235,"ndd":869,"nda":858,"ncy":93,"ncw":75,"nal":455,"nam":268,"nan":722,"nar":638,"nac":478,"nad":788,"nae":541,"naf":1404,"nag":190,"nah":42,"nai":1969,"nc ":1075,"nab":949,"nbr":49,"nbe":186,"nd ":2894,"nba":661,"nau":1855,"nat":502,"nas":4427,"naw":889,"ïo ":43,"na ":2341,"myt":139,"mys":130,"myr":47,"myn":3805,"myl":119,"mwr":262,"mwy":1376,"mwn":673,"iôn":44,"myd":160,"myf":39,"myg":57,"nyb":39,"nyf":51,"nyd":8463,"nyc":85,"nwy":4103,"nwi":220,"nwl":60,"nwo":499,"nwr":453,"nws":101,"ny ":647,"nwg":196,"nwe":332,"nwa":1807,"nve":125,"nul":284,"num":42,"nus":314,"nty":316,"ntw":74,"nw ":6013,"nto":464,"ntu":146,"ïau":380,"nts":154,"ntr":2835,"nti":615,"nth":322,"ntf":38,"ntl":64,"ïai":46,"nta":1485,"nte":453,"nsw":86,"nsy":58,"nso":522,"nst":197,"nse":102,"nsh":86,"nsg":81,"nsi":297,"nsl":63,"nsk":44,"nsc":44,"nsa":259,"nu ":1112,"nry":402,"nrw":38,"nro":38,"nri":910,"nrh":586,"nre":48,"nra":250,"nt ":5305,"ns ":965,"noc":164,"nod":2808,"nob":37,"nog":705,"noe":174,"nof":371,"nol":6920,"noi":38,"nom":300,"non":374,"nos":284,"nor":454,"nov":50,"nou":48,"noy":55,"nne":1057,"nna":2821,"nno":1554,"nnh":61,"nni":1551,"nnu":754,"nnw":1776,"nns":44,"nny":700,"nme":498,"nma":211,"nli":40,"nll":561,"nn ":872,"nla":116,"nle":42,"nly":168,"no ":1182,"ndŵ":83,"nki":55,"nka":58,"nja":59,"nig":2276,"nif":815,"nie":187,"nid":544,"nic":341,"nib":233,"nia":3864,"nk ":137,"niw":268,"niu":56,"niv":47,"nis":599,"nit":132,"nir":305,"nio":1796,"nim":150,"nin":984,"nil":468,"ogr":173,"ogw":78,"ogi":448,"ogl":3819,"ogo":455,"oga":1857,"oge":298,"ogf":54,"oi ":372,"oho":221,"ohn":495,"oha":89,"ohe":171,"ogy":317,"ois":279,"oir":340,"oit":58,"oin":128,"oic":86,"oid":128,"ok ":75,"ol ":19814,"oce":82,"och":2669,"oci":126,"ock":228,"ocl":42,"oco":167,"ocr":146,"ocs":68,"oby":47,"oe ":182,"oca":256,"ode":503,"odf":76,"odl":80,"odi":1787,"odo":830,"odr":901,"oct":65,"of ":754,"odd":3893,"oda":1191,"oel":498,"oeg":1830,"oer":92,"oes":1111,"oet":190,"oen":118,"ody":243,"odu":44,"odw":617,"oed":13126,"og ":4103,"oea":48,"ofn":179,"ofi":339,"ofr":130,"ofo":135,"off":461,"ofe":578,"ofa":165,"oa ":66,"ob ":762,"îl ":44,"oc ":243,"îm ":136,"oan":72,"د ":50,"oad":143,"în ":132,"oba":133,"od ":6442,"oat":50,"obo":63,"obr":327,"obl":2204,"obh":60,"obi":128,"obe":364,"nyn":289,"nym":173,"nyr":93,"nyt":58,"nys":1816,"nyw":409,"Ø© ":125,"oyd":162,"oya":64,"owy":855,"owr":86,"ows":37,"owl":62,"own":249,"oyw":120,"oty":64,"oud":41,"oub":57,"oua":56,"ow ":296,"oti":143,"oth":272,"ote":190,"ott":255,"ots":65,"oto":157,"osy":68,"ost":354,"osw":61,"ota":156,"otb":49,"ov ":55,"osi":398,"osh":69,"ose":562,"osg":209,"osf":80,"oss":202,"osl":48,"oso":1488,"osn":40,"oy ":77,"owa":86,"owe":136,"ovi":136,"ouv":47,"ouz":37,"ox ":45,"ova":45,"ove":172,"oug":154,"oui":121,"oul":126,"oun":623,"ous":235,"our":437,"out":170,"opo":179,"opi":77,"opl":66,"ope":333,"oph":125,"opa":4605,"os ":1942,"opu":42,"opr":39,"opt":1017,"oon":48,"ool":124,"oom":107,"ook":101,"ood":178,"or ":3226,"oot":121,"oor":76,"ork":80,"orl":3097,"orm":633,"orn":345,"oro":694,"orp":53,"orr":343,"orc":174,"ord":2956,"ore":929,"orf":1329,"org":734,"ori":1436,"ou ":122,"osa":670,"osc":72,"osb":238,"ort":1200,"onï":278,"ors":432,"oru":96,"orw":1133,"ory":220,"ot ":344,"m² ":86,"orb":51,"ora":612,"olb":704,"ola":1373,"old":430,"olc":45,"on ":11898,"oli":4195,"oll":572,"olk":46,"olf":383,"ole":1242,"olh":134,"olg":141,"olt":40,"olm":61,"olo":538,"oly":709,"olu":128,"olw":315,"oka":38,"om ":318,"oke":60,"ona":1663,"ond":1690,"onc":97,"onf":152,"one":971,"ong":675,"oni":1332,"onn":630,"ono":1646,"ons":312,"ont":1089,"onw":477,"ony":591,"oma":643,"oo ":37,"ome":1050,"omb":125,"omi":554,"omm":137,"omp":83,"omo":222,"omy":55,"op ":681,"la ":957,"le ":2080,"lbw":166,"lby":104,"lca":76,"lch":1070,"lco":59,"lf ":105,"lde":318,"lda":44,"ldi":2042,"ldw":120,"lds":45,"ldr":142,"lab":92,"lac":485,"lad":4285,"laf":737,"lae":2342,"lah":70,"lag":63,"lai":3136,"lal":87,"lan":5318,"lam":265,"lap":59,"lar":293,"lat":184,"las":992,"law":1750,"lau":895,"lav":69,"lay":112,"lba":3961,"ld ":464,"lbe":138,"lbo":179,"ky ":83,"llê":46,"lpe":48,"lph":58,"ls ":154,"lol":56,"lon":635,"lom":296,"lop":83,"loo":45,"lor":335,"lod":1459,"loc":229,"lof":189,"loe":1682,"log":1629,"loi":55,"lpa":73,"los":162,"lot":96,"lou":84,"low":187,"loy":165,"lob":68,"lmo":40,"lmi":265,"lme":52,"lma":729,"lti":1579,"lto":98,"ltw":61,"ltu":263,"lw ":187,"lud":183,"luc":42,"lub":52,"lue":48,"lsh":144,"lso":43,"lst":89,"lta":499,"lte":173,"lu ":1258,"lse":40,"lsa":39,"lt ":544,"lhe":114,"lha":157,"lgy":51,"lgw":38,"lgo":53,"lge":216,"li ":2035,"lga":220,"lfy":168,"lfr":121,"hân":48,"lfo":118,"lff":365,"lfe":288,"lfa":558,"ley":287,"lew":3221,"lex":112,"leu":338,"lev":58,"les":787,"let":455,"ler":483,"leo":2599,"lem":257,"len":2864,"lei":2402,"leg":1759,"lef":740,"lee":51,"led":5984,"lec":496,"leb":37,"lea":202,"lg ":94,"lls":93,"llu":1630,"llt":2306,"llw":877,"lly":1536,"lo ":540,"lla":2622,"lle":6817,"llf":78,"llg":60,"llh":61,"lli":2844,"llo":886,"lke":43,"lm ":1156,"ll ":5712,"lit":378,"lis":440,"lir":1910,"lip":176,"lio":1087,"lin":1276,"lim":71,"liz":69,"liw":673,"liv":56,"liu":80,"lic":203,"lid":245,"lia":3198,"lk ":57,"lil":79,"lig":359,"lie":476,"lif":562,"dŵr":302,"ma ":1637,"mb ":111,"mac":103,"mab":110,"mai":606,"mad":228,"mae":4308,"mag":118,"map":57,"mar":872,"mas":453,"mal":309,"mam":99,"man":1111,"maw":438,"mau":1430,"mat":779,"mba":96,"mbl":58,"mbi":109,"mbe":242,"mbr":269,"mbo":281,"me ":239,"mca":193,"mbu":37,"mch":98,"mda":65,"mde":487,"mdd":657,"mdr":53,"med":792,"meg":711,"mea":488,"met":1049,"mew":2214,"mes":747,"mer":2496,"mel":276,"men":1156,"mei":882,"mff":63,"mey":41,"lva":80,"lve":64,"luo":1302,"lun":1166,"lum":206,"lus":297,"lur":87,"ly ":477,"lwa":59,"lwb":141,"lwc":190,"lwe":381,"lwg":148,"lwi":478,"lwr":546,"lwn":220,"lwm":69,"lwy":4971,"lws":76,"lyb":68,"lyc":94,"lyd":945,"lyf":715,"lyg":1379,"hôn":42,"lyw":939,"lym":387,"lyn":3191,"lys":623,"lyt":263,"lyr":52,"mpi":350,"mph":82,"mpe":82,"mpl":56,"mpw":155,"mps":75,"mpt":51,"ms ":365,"ëwy":48,"moc":146,"moe":79,"mod":296,"mon":1356,"mol":107,"mor":527,"mos":228,"mot":129,"mou":82,"mpa":381,"mre":767,"mro":136,"mru":3069,"mry":585,"msa":38,"mu ":71,"mse":222,"mra":1424,"mth":126,"mud":331,"mst":38,"mwe":56,"mwd":160,"mwa":49,"my ":104,"mur":150,"mus":72,"mun":2037,"mho":82,"mhl":147,"mhw":50,"mha":708,"mhe":231,"mgy":321,"mgu":85,"mi ":264,"mge":117,"ml ":374,"min":367,"mio":78,"mil":1349,"mir":63,"mis":321,"miw":182,"mit":116,"mic":53,"mia":447,"mig":178,"mie":110,"mo ":102,"mlw":137,"mly":764,"mlu":98,"mle":90,"mla":506,"mni":274,"mne":46,"mmy":53,"mp ":159,"mmu":54,"mmo":59,"mma":119,"mme":106,"wâr":55,"rŵp":443,"zer":50,"ze ":49,"zab":52,"zan":41,"zar":39,"zon":65,"ان":54,"ال":222,"ÂR ":38,"yw ":17417,"yty":72,"ytu":126,"yts":74,"ytr":121,"yti":312,"yth":1577,"yta":92,"ysw":48,"ysy":639,"yst":2007,"ysu":113,"yso":937,"ysl":39,"ysg":2082,"ysf":43,"ysi":788,"ysb":191,"yse":204,"ysa":193,"yry":38,"yri":760,"yro":57,"yrn":1071,"yrs":46,"yrr":204,"yrw":139,"yrc":853,"yrd":987,"yra":2367,"yrh":40,"yre":633,"yrf":123,"ys ":5008,"ypr":60,"yr ":15016,"ype":40,"yon":159,"yny":6318,"ynu":364,"ynw":625,"za ":99,"ywe":457,"ywb":47,"ywa":1109,"ywi":571,"ywu":40,"ywo":763,"ywy":914,"yby":49,"ye ":65,"yca":52,"ych":3349,"ybi":137,"ybl":102,"ybo":259,"ybr":110,"ybu":37,"yf ":458,"yda":2864,"yde":440,"ydf":99,"ydd":28126,"ydi":506,"ycl":60,"yg ":747,"ydo":278,"ydr":1154,"ydl":894,"ydn":127,"ydw":276,"ydy":4654,"yfa":1668,"yfl":563,"yfi":251,"yfe":2566,"yff":1314,"yer":49,"ya ":226,"yb ":51,"yad":44,"yaf":1692,"ybe":40,"yd ":10831,"yan":80,"yal":55,"ym ":3225,"yke":46,"yn ":47219,"yla":317,"ylc":792,"yld":121,"yle":201,"ylf":185,"yli":297,"yll":1653,"ylo":75,"ylw":219,"ylv":55,"ylu":111,"ymc":84,"ymb":284,"yma":1393,"yo ":95,"ymh":887,"ymi":79,"ymf":49,"ymg":221,"ymd":943,"yme":789,"ymr":5566,"ymp":236,"ymo":286,"yml":267,"ymy":322,"ymu":2206,"ymt":125,"ymw":638,"yna":930,"ynd":320,"ync":116,"yni":1100,"ynl":244,"yne":1666,"ynf":195,"yng":7196,"ynh":1161,"ynr":509,"ynt":1723,"ynn":2787,"yno":1438,"yfw":48,"yfu":361,"ر ":53,"yfy":1325,"yfo":259,"yfn":833,"yfr":2540,"ygi":282,"ygl":131,"yga":105,"yge":40,"ygb":169,"ygw":75,"ygy":125,"ygo":197,"ygr":629,"ygu":321,"yhy":100,"yho":721,"yie":41,"yl ":434,"tîm":43,"ÙŠ ":83,"Ù† ":99,"xic":123,"xas":52,"xan":82,"wun":41,"wy ":2968,"wsl":55,"wst":1039,"wsa":63,"sïa":42,"wsi":378,"wse":138,"wto":60,"wty":66,"wth":52,"wyt":933,"wys":3904,"wyr":4356,"wyl":1023,"wym":72,"wyn":4153,"wyo":82,"wyd":14813,"wyb":349,"wyi":44,"wyf":1666,"wyg":83,"wya":1824,"wly":42,"wma":78,"wmb":41,"wmn":259,"wmp":187,"wmw":171,"wna":104,"wne":924,"wnd":93,"wnc":60,"wni":118,"wng":2908,"wnn":120,"wno":71,"wns":141,"wnt":347,"wm ":924,"wlc":168,"wla":2176,"wn ":4514,"wll":145,"wli":167,"wlg":67,"wle":1242,"wlf":163,"ws ":798,"wre":539,"wrc":168,"wrd":645,"wri":768,"wrg":40,"wt ":38,"wra":424,"wrt":1573,"wrs":66,"wry":875,"wrn":594,"wrp":460,"wro":897,"wod":593,"wog":540,"wob":230,"woc":119,"wny":106,"wor":104,"wol":250,"woo":72,"won":68,"wpa":41,"wr ":5367,"wpi":70,"wdh":39,"wdi":51,"wf ":39,"wdd":251,"wcl":96,"wch":1526,"we ":503,"wca":54,"wbl":50,"wfe":98,"wes":112,"wer":1889,"wen":943,"wel":1143,"wei":2145,"wef":1122,"weg":153,"wed":6219,"ŵyr":67,"web":67,"wec":102,"wg ":400,"ŵyl":106,"wdw":185,"wdu":669,"wgr":102,"wga":86,"wi ":316,"wir":738,"wis":414,"wit":92,"wl ":621,"wig":187,"wie":106,"wid":262,"wic":128,"wio":346,"win":3305,"wil":311,"wia":359,"ŵr ":390,"wb ":178,"wa ":108,"wan":225,"wal":228,"way":128,"wau":1633,"war":1946,"was":655,"wba":42,"wd ":517,"wbe":49,"wai":2037,"wah":560,"wae":306,"wad":321,"ŵl ":39,"ŵn ":84,"ŵp ":436,"rôl":105,"via":62,"vil":101,"vin":110,"vic":81,"vid":205,"vie":209,"viu":39,"vis":116,"rég":55,"von":93,"rëw":49,"ver":485,"ves":72,"rân":40,"ven":296,"vel":96,"ve ":424,"val":75,"van":302,"var":50,"va ":105,"uwy":107,"uz ":39,"uwc":992,"uwi":160,"ux ":77,"uve":61,"usk":46,"ush":85,"usi":90,"use":207,"usc":42,"usa":109,"ust":299,"uss":159,"usn":82,"uth":300,"uti":56,"ute":122,"uta":55,"uw ":191,"utt":64,"uts":49,"utu":93,"uto":58,"us ":2318,"ut ":194,"ura":273,"urd":1224,"urc":65,"urf":869,"ure":327,"urg":261,"uri":598,"urk":37,"urn":280,"uro":357,"urr":296,"urs":65,"urt":95,"urw":145,"ury":156,"uny":251,"uoi":80,"uog":87,"uoe":102,"uod":142,"uon":143,"uol":447,"uos":1109,"ur ":3905,"upi":63,"upe":58,"ump":113,"uml":95,"umo":125,"uma":120,"umb":206,"ume":97,"uly":76,"unt":210,"uns":43,"unr":208,"unw":94,"uni":1229,"uno":252,"unn":56,"unc":85,"und":1356,"una":363,"ung":102,"une":2052,"unf":97,"up ":41,"um ":501,"ulu":685,"ult":59,"ulo":75,"ull":591,"uli":246,"ulh":41,"ule":57,"ulf":75,"uld":43,"ula":311,"un ":3244,"uid":128,"uil":132,"uim":38,"uin":123,"uir":49,"uis":142,"uic":91,"uit":55,"ul ":450,"ugh":287,"ugi":102,"uge":143,"ugo":87,"ugl":146,"uga":343,"ugu":87,"uha":41,"uda":109,"udd":714,"udf":71,"ude":80,"udi":647,"udn":61,"ubu":38,"uca":70,"ue ":164,"uce":83,"uci":43,"uch":1304,"uck":68,"uer":86,"ues":64,"uff":193,"ufe":472,"ufa":148,"udu":77,"udo":207,"ug ":773,"udw":146,"uee":83,"ued":160,"uen":92,"uel":100,"ub ":91,"ua ":2061,"uat":48,"uar":137,"ual":68,"uan":197,"ubl":79,"ube":47,"ubh":79,"uba":126,"ud ":1095,"lÅ·r":43,"lÅ·n":132,"uai":74,"uag":90,"uad":147,"uae":87,"tyw":139,"tyf":106,"tyl":97,"tym":66,"tyn":537,"typ":46,"tyr":569,"tys":55,"ty ":569,"tur":1378,"tus":150,"tun":319,"tum":55,"tua":1979,"tud":365,"tyd":81,"tz ":50,"twy":340,"tws":164,"twr":164,"ts ":407,"tre":3761,"tt ":218,"tra":1795,"tri":1062,"tru":77,"tro":965,"Å·r ":76,"tu ":359,"trw":624,"try":340,"tsa":45,"tse":80,"tsi":56,"tsh":38,"tta":92,"tte":273,"tth":42,"tti":111,"ttl":73,"tto":97,"tts":49,"tty":41,"tma":64,"to ":461,"tmo":40,"tne":85,"tna":134,"tno":155,"Å·n ":163,"pêl":157,"tod":481,"toc":94,"toi":46,"tog":93,"tob":164,"tou":42,"tos":66,"tow":75,"tom":329,"ton":1238,"tol":136,"tor":1235,"top":118,"tr ":1173,"til":111,"tif":143,"tie":133,"tig":337,"tir":1682,"tit":55,"tis":234,"tin":490,"tim":109,"tio":758,"thy":2521,"thu":967,"thw":450,"tia":1118,"tic":242,"tid":55,"tiw":232,"tiu":49,"tiv":155,"tl ":156,"tlo":100,"tll":46,"tla":185,"tle":205,"tem":745,"ten":263,"tep":86,"tei":537,"tel":1208,"tef":162,"teg":330,"tea":52,"teb":257,"tec":118,"ted":382,"tfo":51,"th ":17104,"tey":122,"tev":62,"teu":225,"tes":349,"ter":2833,"tgo":37,"ti ":249,"tga":83,"thn":220,"tho":2822,"thl":355,"thm":78,"thr":1553,"ths":111,"thf":157,"thg":113,"thd":135,"the":2439,"thi":2871,"tha":2130,"AS ":68,"AU ":49,"BC ":162,"両 ":46,"Àit":437,"三 ":61,"tÅ· ":48,"之 ":53},"n_words":[3639934,4414620,3548300],"name":"cy","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/da.json b/contrib/languages-data/da.json
new file mode 100644
index 0000000..9f5fa9e
--- /dev/null
+++ b/contrib/languages-data/da.json
@@ -0,0 +1 @@
+{"freq":{"D":31963,"E":16857,"F":19795,"G":13674,"A":25625,"B":25733,"C":16797,"L":17275,"M":22557,"N":16361,"O":9041,"H":24577,"I":12015,"J":9674,"K":25355,"U":7743,"T":19455,"W":7598,"V":12180,"P":17849,"S":49336,"R":17256,"Y":1840,"Z":1413,"f":221888,"g":349504,"d":493190,"e":1436845,"b":166372,"c":57573,"a":587527,"n":746797,"o":488340,"l":487393,"m":293320,"j":42561,"k":286164,"h":119659,"i":613022,"w":11095,"v":176622,"u":183003,"t":621215,"s":567338,"r":850344,"q":1336,"p":139855,"z":8890,"y":86732,"x":5939,"Å":1900,"Ø":2784,"é":2737,"æ":62497,"å":62645,"ä":1464,"ü":1583,"ø":71666,"ö":1579," l":35858," m":66057," n":25041," o":98877," h":49372," i":113380," j":6194," k":49552," d":112412," e":192889," f":104584," g":23827," a":95172," b":68676," c":8562," u":24358," t":58677," w":1082," v":42657," p":52152," s":123480," r":22226," J":9634," K":25242," H":24422," I":11943," N":16289," O":8943," L":17176," M":22363," B":25561," C":16566," A":25498," F":19659," G":13552," D":31783," E":16778," Z":1391," Y":1835," S":49095," R":17196,"а":1047," P":17741," W":7540," V":12125," U":7720," T":19345," å":3941," æ":1756," ø":5351," Å":1898," Ø":2782,"A ":4032,"F ":2501,"Da":6717,"Cl":971,"Co":3707,"Ce":1101,"Ch":3540,"Do":1568,"De":17413,"Di":2020,"Fe":1146,"Fa":2183,"Eu":1618,"Er":1008,"Et":1623,"En":5373,"El":1343,"Ge":1911,"Ga":1871,"I ":3326,"Fr":4782,"Fo":3851,"Fl":1651,"Fi":2355,"C ":1471,"Au":1064,"Ar":2625,"As":2462,"Ba":5317,"Aa":1418,"Am":3646,"An":3500,"Al":3794,"By":1617,"Bu":1524,"Br":3812,"Ca":3808,"Bi":1743,"Be":3864,"Bo":3085,"Bl":1436,"Kl":1826,"Kr":1747,"Ko":8878,"Le":2792,"Li":3203,"La":5071,"Lu":1291,"Lo":2732,"Me":3324,"Mi":3899,"Ma":7557,"Mu":1568,"Mo":3310,"Ni":2023,"Ne":2682,"Na":2422,"No":4840,"Gi":973,"Gr":3579,"Go":1190,"Gu":1554,"Ha":6824,"He":6629,"Hi":1514,"Ho":4306,"Hu":1589,"In":3339,"Is":1195,"Ja":2348,"L ":1217,"Je":1924,"Jo":2770,"Ju":1199,"Ka":3252,"M ":1238,"Ki":2897,"Ke":1081,"Un":2388,"Ty":1556,"Tr":2242,"To":2011,"Th":4228,"Ti":1419,"Te":2146,"Ta":2888,"Sy":2480,"St":10943,"Sv":1640,"Su":1509,"Wo":997,"Wi":1997,"Wa":1860,"We":1192,"Vo":988,"Vi":3450,"Va":1711,"Ve":4334,"Pr":4633,"S ":1850,"Pe":2483,"Pa":3757,"Po":2392,"Pi":1096,"Or":1531,"Kø":2561,"Se":2517,"Sc":1974,"Si":2298,"Sh":1188,"Sl":1960,"Sk":3103,"Sp":1977,"So":7973,"Ru":1537,"Sa":4145,"Re":4467,"Ri":2574,"Ro":3970,"Ra":2483,"b ":7025,"a ":44327,"Yo":1017,"Sø":1957,"bø":1832,"i ":102076,"fy":1735,"gd":2544,"ge":89823,"ga":13601,"gb":1002,"bæ":1473,"fj":1452,"fl":7833,"ff":4196,"fi":15203,"bå":1373,"fh":1152,"fs":1525,"fr":23015,"fu":3863,"ft":15592,"fo":54992,"j ":3361,"gy":1588,"he":22182,"ha":30491,"gn":17858,"gl":6422,"gi":15986,"gh":4618,"gg":16392,"gu":4973,"gt":15761,"gs":16863,"gr":16831,"go":3646,"dt":26827,"du":5081,"dv":6155,"dy":2631,"g ":111365,"ea":8665,"eb":9181,"ec":4319,"ed":75055,"de":218146,"dd":6228,"dg":5250,"df":2420,"di":20969,"dh":1605,"dk":4450,"dj":1187,"dm":1943,"dl":12465,"do":7994,"dn":3058,"ds":29024,"dr":12885,"ew":2085,"ex":1370,"eu":3377,"ev":22509,"ey":2555,"ez":1521,"fa":10884,"h ":6139,"fe":8693,"eh":3909,"eg":21663,"ef":12940,"ee":3853,"el":91687,"ek":16142,"ej":9330,"ei":7265,"ep":6194,"eo":4491,"en":269379,"em":29799,"et":141301,"es":67128,"er":363438,"ca":5613,"e ":256545,"by":15888,"bs":2188,"br":14152,"bu":7411,"bo":14081,"bj":2063,"bl":20954,"bi":8379,"bb":2090,"be":49808,"db":9039,"da":25268,"f ":48816,"cu":1080,"ct":2015,"co":4730,"ck":5340,"ci":7534,"ch":11084,"ce":13957,"c ":1829,"az":1010,"ay":3869,"ba":15017,"d ":83491,"at":56481,"as":18978,"ar":71549,"av":20381,"au":6593,"ak":8835,"al":58276,"ai":4230,"aj":1819,"ap":5563,"am":30979,"an":131829,"ac":6807,"ad":19629,"aa":1902,"ab":11909,"ag":21569,"ah":1853,"ae":3053,"af":53343,"nu":5799,"nt":31384,"ns":60694,"nr":1599,"no":15192,"nn":14628,"jø":1567,"ny":4222,"nv":2608,"oe":2046,"of":9995,"oc":5289,"od":18682,"oa":1343,"ob":5028,"om":68251,"on":52834,"ok":6768,"ol":36629,"oi":1528,"kæ":2283,"og":81449,"oh":1849,"ot":9072,"os":11517,"ov":19468,"ou":6113,"op":19430,"oo":2822,"or":109199,"kø":2883,"r ":303465,"ow":2461,"pe":19212,"pf":1434,"pa":11174,"pl":7694,"po":10483,"ph":1997,"pi":13460,"lå":3180,"læ":7117,"lo":18381,"lm":9161,"ll":54007,"ls":28805,"lr":2728,"lp":1954,"lv":5002,"lu":8894,"lt":13794,"ly":5517,"hø":4198,"o ":13391,"ma":28771,"mb":6952,"mh":1531,"me":79970,"mf":2516,"mk":2192,"ml":2687,"mi":18259,"mn":1374,"mm":27337,"mp":7909,"mo":10889,"mr":3179,"mt":7052,"ms":5339,"mu":15391,"my":1487,"p ":7331,"na":26783,"nb":2591,"nc":5256,"nd":115468,"ne":83213,"nf":2882,"ng":65185,"nh":5629,"ni":34813,"jæ":2156,"nj":1002,"nk":9744,"nl":3415,"nm":3852,"jy":1603,"ju":2882,"js":1575,"jo":3376,"jl":1331,"fæ":2974,"ki":13129,"gå":3551,"ke":62501,"kb":1624,"ka":33369,"m ":58614,"fø":16135,"ky":2485,"ks":11787,"kt":15601,"ku":9611,"kv":2686,"ko":24231,"kr":19731,"kk":11850,"kl":9428,"km":2041,"kn":3763,"li":70659,"hå":1642,"lh":1724,"lk":6427,"hæ":2028,"le":98770,"ld":24706,"lg":4872,"lf":2530,"la":47170,"lb":6128,"gø":1910,"n ":242497,"hr":2287,"hv":9222,"ht":1466,"hu":6980,"dæ":994,"hj":3040,"hi":7688,"hn":1182,"ho":15697,"id":25624,"ic":10962,"ib":6095,"ia":12086,"ig":59928,"if":7649,"ie":27731,"hy":1292,"k ":53472,"dø":3867,"ir":14180,"is":62018,"it":24973,"iu":2399,"iv":18874,"ik":30570,"il":63431,"im":7762,"in":99762,"io":27519,"ip":4261,"je":15216,"jd":2875,"få":1004,"iz":977,"l ":57442,"ja":3097,"tæ":3691,"tå":3749,"z ":1958,"sø":4093,"wi":999,"så":7469,"sæ":4934,"y ":17470,"rø":5891,"wa":2989,"we":1786,"ræ":12201,"rå":5052,"vi":25451,"vs":4402,"vn":10855,"vo":8670,"ve":64168,"vd":1809,"va":25040,"x ":2713,"ui":2354,"uk":3985,"ul":11807,"ue":7203,"uf":1429,"ug":8656,"ur":17879,"us":20761,"ut":8713,"um":12141,"un":39154,"up":6927,"ty":12222,"tz":1053,"tu":11275,"tt":18340,"tv":2718,"ub":5076,"ua":3814,"ud":21845,"uc":3424,"w ":2415,"to":25039,"tn":4704,"tm":1160,"tl":7804,"ts":12139,"tr":31125,"tf":1908,"te":116420,"tj":2329,"pæ":1318,"på":21476,"ti":89903,"th":7738,"v ":18910,"tb":1612,"ta":40351,"su":4879,"sv":6120,"ss":17007,"st":117117,"sy":9417,"sl":10786,"sk":90052,"sn":3225,"sm":6568,"sp":18285,"so":39577,"sr":1826,"sd":1411,"sc":3973,"sf":3067,"se":58416,"sh":5513,"sg":1978,"sj":1417,"si":34046,"u ":4030,"sa":20739,"sb":7435,"rr":10396,"rs":33077,"rt":26331,"ru":25205,"rv":5600,"ry":5622,"rp":2368,"ro":34074,"rn":23969,"rm":13546,"rl":8366,"rk":22061,"næ":3458,"nå":1618,"ri":66660,"rh":5594,"rg":18288,"rf":5902,"re":122058,"rd":25935,"rc":2308,"rb":9120,"ra":54038,"t ":215608,"mø":1338,"qu":972,"mæ":2988,"må":5133,"lø":4835,"s ":87017,"pt":4539,"pu":4408,"pp":6074,"pr":20775,"ps":3450,"zi":1962,"ze":1041,"za":1046,"væ":10057,"yg":7090,"ye":6589,"yd":11050,"ya":1357,"yb":1013,"tø":6010,"yt":3483,"ys":11680,"yr":5643,"yp":2457,"yn":4794,"ym":2136,"yl":4209,"yk":3734,"År":1234,"å ":31271,"Øs":1854,"æv":1666,"æs":6691,"ær":17270,"æt":3678,"æn":8602,"æk":5318,"æl":6220,"æg":5488,"æd":2878,"ån":3755,"ås":1047,"år":11909,"åe":2110,"ål":3106,"åb":1474,"åd":6195,"ør":20663,"øs":5971,"øv":2281,"øj":4875,"øk":983,"øn":5238,"øl":3077,"øm":1629,"øb":5904,"øg":2353,"øe":2140,"ød":12599,"ø ":2608," Ga":1860," Ge":1892," I ":2530," Fo":3830," Fr":4772," Fi":2339," Fl":1645," Ha":6811," He":6623," Go":1179," Gr":3557," Gu":1534," Hu":1582," Ho":4265," Hi":1510," Je":1919," Ja":2336," Is":1181," In":3319," Ka":3227," Ke":1072," Ki":2888," Jo":2762," Ju":1199," La":5040," Le":2776," Li":3188," Kl":1820," Ko":8866," Kr":1743," Ma":7451," Mi":3885," Me":3307," Lo":2724," Lu":1285," Ne":2661," Na":2409," Ni":2020," Mo":3297," Mu":1554," Am":3639," An":3489," Al":3774," Aa":1413," Ba":5300," Au":1061," As":1298," Ar":2597," Be":3849," Bi":1733," Bl":1427," Bo":3066," Br":3787," Bu":1519," By":1610," Ca":3749," Ce":1081," Ch":3522," Co":3667," Da":6687," Di":2014," De":17357," Do":1505," El":1337," Et":1617," Er":1002," En":5365," Eu":1616," Fe":1132," Fa":2149," Sø":1956," Wo":983," Wi":1981," We":1181," Wa":1857," Yo":1015," Kø":2558," Or":1513," Po":2365," Pi":1093," Pe":2468," Pa":3735," No":4834," Ra":2473," Ro":3958," Re":4459," Ri":2570," Pr":4618," Sy":2476," Sv":1637," Su":1503," St":10885," Ta":2881," Th":4217," Ti":1411," Te":2130," Tr":2230," To":1975," Ru":1535," Sa":4126," Sh":1168," Si":2284," Sc":1958," Se":2497," So":7960," Sp":1954," Sk":3100," Sl":1954," Va":1707," Ve":4322," Vi":3432," Vo":984," Ty":1554," Un":2381," ja":1664," få":1001," in":18970," ik":2700," is":1666," ka":10401," fø":12841," ki":4060," ke":5007," jo":1235," fæ":1545," ju":2042," ha":16862," he":5872," gi":1512," gr":8357," gu":1731," dø":1180," hi":2424," hj":2649," ho":5868," hu":1995," hv":8333," ne":2244," na":5975," mu":3617," mo":5717," om":10735," og":60338," of":4764," ny":1418," nu":1920," no":8819," le":3614," li":12024," hå":1306," la":9917," kv":1456," ku":4034," km":1736," kl":2396," kr":4395," ko":12549," me":34653," mi":5228," hø":2581," ma":10653," lu":1103," ly":1210," lo":1691," af":47341," ad":2058," am":5476," an":9937," ap":1120," ak":1320," al":5943," au":1423," ar":5256," at":12715," ba":5812," bi":2763," be":21652," bo":2959," bl":16483," by":7166," bu":1513," br":6861," ca":2244," er":79580," et":23500," en":65390," el":13704," ek":1980," ef":3944," eg":1315," fe":2371," fa":4879," fu":1922," fr":19782," fo":43206," fl":5239," fi":8516," bå":1058," ge":4611," ga":3502," bø":1211," i ":86016," fy":1194," co":1546," ce":1786," ci":1062," da":14483," do":1734," dr":1947," de":85238," di":4041," dy":1311," væ":5438," tæ":998," sø":1940," ru":2426," sa":10706," se":8834," si":12012," sm":1227," sl":3070," sk":10508," sp":8070," so":28866," ra":1828," re":10118," nå":1108," ri":1551," næ":2233," ro":3386," pr":10950," lø":1660," s ":1355," må":3141," ov":4934," op":11111," or":3538," kø":1518," pe":4046," pa":3870," pl":3433," po":4785," lå":2186," pi":1184," læ":1975," så":2369," sæ":2010," va":14642," ve":13203," vo":1548," vi":7015," ræ":1334," ud":14434," ty":5331," tv":1505," un":6550," ta":3468," sy":6433," st":20417," sv":2066," su":1555," tr":6706," to":3836," th":1962," på":21281," ti":28004," te":4720," År":1234," Øs":1852," år":3117," øs":2023,"Årh":1078,"Fil":1000,"Et ":1446,"Eur":1467,"En ":3727,"Fra":1859,"Fre":1891,"For":2210,"Hel":1134,"Her":3429,"Han":2509,"Ind":1212,"Øst":1835,"Hol":1719,"As ":1183,"Bay":1295,"Amt":2376,"And":1120,"Bye":1086,"Car":1184,"Ber":1053,"EF ":1275,"De ":1771,"Det":5562,"Den":6802,"Dan":4528,"Chr":1315,"Cha":1198,"New":1218,"Nor":4041,"Køb":2308,"Par":1507,"Pro":2885,"SA ":2039,"SAs":1153,"Joh":1172,"Kir":1207,"Kon":1132,"Kom":6411,"Lan":2730,"Man":1218,"Mar":2635,"Søn":999,"Syd":1794,"Sve":1310,"Str":999,"Sti":2399,"Sto":1709,"Sta":3406,"Ste":1353,"Tab":1274,"Sog":4806,"Sch":1301,"San":1256,"Reg":1718,"Ros":1109,"åen":1002,"åde":4772,"ål ":1259,"ånd":1467,"åne":2057,"åle":1126,"åre":1034,"ård":2059,"år ":6325,"Ver":993,"Ves":1255,"Uni":1434,"æde":2099,"Tys":1072,"æge":1409,"ægt":2411,"æll":2133,"ækk":2758,"æld":2168,"æns":1603,"æng":3355,"ænd":2342,"ær ":2495,"æsk":1388,"æse":1314,"ært":1291,"ærk":2357,"ære":6027,"ærd":1139,"ætt":1489,"æst":1815,"ævn":987,"The":2590,"bje":1890,"bil":1901,"bin":2468,"blo":1344,"ble":11852,"bli":3619,"bla":2817,"bol":3467,"bog":1211,"bor":5893,"bbe":1641,"be ":1587,"ban":5529,"bal":1468,"bag":1151,"bas":2113,"bar":1770,"bej":2245,"beg":2119,"bef":974,"bed":1902,"ber":10275,"ben":6123,"bel":6286,"bez":990,"bev":1288,"bes":5570,"bet":8125,"ca ":2352,"ce ":2816,"bri":2216,"bro":1760,"bra":1230,"bre":2307,"bru":5207,"bur":1375,"bun":1225,"bum":1745,"by ":6807,"bye":2309,"byg":4928,"am ":2554,"al ":9518,"ain":1470,"aj ":1010,"agt":3576,"anv":1364,"anu":1902,"ann":5158,"anm":3219,"ant":6958,"ans":23197,"ane":4038,"ang":12562,"ani":4609,"ank":4263,"ana":2737,"anc":2097,"and":39458,"amt":2475,"amm":7240,"aml":2016,"amp":2062,"ami":2635,"ame":7259,"ama":1553,"alv":1423,"alt":4977,"als":1981,"alr":2167,"alm":1522,"all":6438,"alg":1439,"ali":5360,"ald":6029,"ale":8208,"ala":1794,"alb":2973,"an ":17044,"akt":3338,"abe":4887,"ad ":3260,"ab ":2197,"aft":1820,"afs":1107,"afh":1105,"afi":1171,"age":8240,"adm":1030,"adi":3138,"ade":6627,"ag ":3450,"ads":1953,"ack":1318,"ach":1746,"ace":1610,"af ":43757,"aye":1830,"at ":17462,"are":4543,"ard":3649,"arb":2537,"ara":2794,"arm":1244,"arl":1801,"ark":7252,"ari":4342,"arv":1066,"arr":1598,"ars":2045,"art":9040,"au ":1067,"asi":1827,"ase":2004,"ask":1161,"ar ":24485,"as ":3170,"avd":1374,"avn":8812,"avi":1508,"ave":5203,"ay ":1009,"av ":1155,"ata":1987,"ast":4061,"ass":3627,"ato":1943,"ate":7241,"ati":16074,"att":4875,"ats":1271,"atu":2072,"jer":5562,"jek":1076,"jem":1846,"jen":1901,"jet":1025,"jan":1060,"je ":1645,"jde":2262,"jor":2291,"fær":971,"fæl":1337,"itu":1780,"itt":1519,"isk":22022,"ism":1473,"iss":2416,"ist":15812,"iv ":2118,"ita":3145,"ite":4554,"iti":5826,"ium":1528,"iva":1011,"ivi":1481,"ive":12562,"ipt":1296,"is ":8187,"ion":21894,"irk":6542,"isi":1236,"ise":3850,"isa":1650,"ire":2379,"it ":2641,"kil":3875,"kib":1205,"kin":2032,"kir":1742,"går":2699,"kis":983,"km ":1307,"ked":1397,"kel":4377,"ken":10401,"kes":1300,"ker":9106,"ket":3222,"ke ":28879,"kra":3109,"kre":5594,"kt ":3134,"kse":1892,"kro":1365,"kri":8221,"kov":1285,"kor":3059,"kon":5299,"kom":8214,"kol":2985,"ks ":2322,"kni":2413,"kke":10909,"klu":1988,"kle":1997,"kla":3032,"kli":1244,"jyl":1350,"jul":1124,"kba":1487,"kat":1697,"kar":1404,"kas":971,"kan":11432,"kal":6243,"kam":1128,"kab":5579,"ka ":2541,"før":6664,"fød":7830,"føl":1523,"ham":1255,"han":5336,"hal":2007,"hav":7061,"har":10554,"he ":3841,"hel":2400,"hed":6692,"her":3250,"hen":2397,"hin":1447,"his":2209,"hje":2112,"gle":3411,"gn ":5722,"gla":1189,"gni":1875,"gne":9082,"gs ":1501,"gsb":1331,"gsk":1233,"gru":5085,"gra":4027,"gt ":9775,"gre":2783,"gst":2115,"gte":3224,"gti":1404,"grø":1000,"gså":4604,"gus":1029,"græ":2818,"ial":2412,"ian":3001,"ic ":1068,"ibo":1076,"ølg":1529,"øn ":1089,"id ":2460,"ibe":2142,"ia ":3660,"øje":1642,"iet":2644,"iel":2523,"ien":9683,"ier":4656,"ig ":12921,"ift":4749,"ør ":3471,"ici":1501,"ich":2160,"ice":1664,"ie ":4513,"ica":1606,"ids":2132,"idt":1926,"idl":3701,"idi":1064,"ide":10399,"idd":1704,"ønd":1642,"øst":4212,"il ":24604,"im ":977,"ika":7541,"ige":19623,"iga":1060,"igh":3457,"igi":1414,"igg":9753,"igt":6078,"igs":1981,"ign":1642,"øre":4495,"ørs":6046,"ørr":1963,"ørt":1508,"ik ":5837,"ørn":1192,"ime":1521,"ind":23824,"ina":3988,"int":3500,"ins":7738,"ine":6155,"ing":36509,"ini":3271,"ink":1131,"iod":1188,"ikl":2029,"ikk":5095,"ike":2959,"ikb":1260,"in ":7601,"ikt":1780,"iks":1269,"ilo":3003,"ill":15072,"ilk":1504,"øve":1134,"ilm":4941,"ilh":993,"ili":3480,"ild":2323,"ile":1492,"io ":1215,"ils":1343,"hol":6737,"hov":3711,"hri":1405,"hvo":4960,"hum":1081,"hun":1070,"hus":3072,"hve":1568,"hvi":2196,"døs":1339,"død":1050,"ffe":2245,"ffi":1018,"fes":1566,"fer":1682,"fen":1036,"fas":1001,"fat":2864,"far":1600,"fam":1418,"fan":1048,"fal":1044,"ezi":1048,"evæ":1177,"eta":3145,"ete":8817,"eti":2099,"esp":2172,"est":19859,"ødt":7615,"ess":3934,"esv":1081,"ev ":11307,"etr":1299,"ets":3278,"ett":4721,"ety":1926,"øen":1132,"ew ":1416,"eve":5776,"eva":1631,"øge":1365,"evi":1173,"øj ":1036,"ey ":1638,"er ":229484,"epa":1085,"eor":1469,"eol":1033,"ød ":1370,"es ":24995,"øbe":3709,"ept":1210,"erk":1552,"erl":2388,"eri":19046,"erg":5102,"erh":1890,"ere":36721,"erf":2743,"erd":3362,"era":4483,"erb":2464,"et ":110723,"esk":4356,"esl":1065,"esi":2775,"øde":1837,"ese":1937,"erv":2764,"eru":2635,"err":4661,"ert":4521,"ers":15033,"ern":16903,"erm":1970,"ero":1448,"egå":1006,"ekr":1629,"eks":4054,"ekt":4817,"en ":174071,"ela":1844,"eld":1116,"ele":6394,"eli":11128,"ell":22146,"elv":2251,"els":21491,"elt":4815,"emb":3551,"ema":1766,"eme":3075,"emm":3388,"emo":1300,"emi":2222,"emt":1052,"emp":1313,"ems":1366,"enf":1295,"ene":9879,"enh":4546,"eng":3572,"enb":1247,"ena":1550,"end":24965,"enc":1233,"enn":5553,"enk":1120,"eni":2823,"ens":17443,"ent":14377,"enr":1262,"egn":6321,"ege":4459,"egi":3950,"eha":1147,"egr":1976,"eho":1258,"eis":2241,"ein":2075,"ejd":2400,"el ":14709,"ejs":1068,"ejl":1164,"eje":2612,"em ":8110,"gis":2194,"giv":4136,"gin":973,"gio":2383,"gie":1747,"ghe":2580,"gge":15510,"gi ":1776,"gen":19301,"geo":1146,"get":6054,"ger":26388,"ges":3499,"gem":1137,"gel":6514,"gde":1133,"ge ":22940,"gad":1042,"gar":1035,"gav":1126,"gam":1188,"gan":5170,"fte":8906,"fun":1949,"ft ":4358,"fra":16811,"fre":2876,"fri":2352,"for":48487,"fol":2294,"fod":2203,"fle":2554,"fla":1188,"flo":1580,"fly":1416,"båd":1117,"fic":1488,"fil":4788,"fik":1836,"fin":3119,"fir":1330,"fis":991,"da ":2439,"dbr":1439,"dbo":3204,"de ":51365,"dby":3214,"dal":1321,"dag":2881,"dat":1966,"dan":12624,"dam":1233,"dda":1566,"dde":3541,"com":1147,"ch ":2261,"cer":3808,"ces":1087,"cen":3123,"cha":1908,"cia":1876,"ck ":2373,"cie":1532,"che":2404,"cke":1007,"ed ":35220,"ebe":1273,"ebo":1021,"ebr":1930,"eal":1397,"eat":1418,"efi":1531,"efo":3934,"eft":4164,"een":1037,"edl":1944,"edi":2262,"ede":18829,"eda":1072,"edt":1636,"eds":6584,"edr":1723,"eci":1115,"ece":1005,"dyr":1288,"dvi":2440,"dve":2150,"don":1291,"dom":2065,"ds ":4746,"dmi":1103,"dni":1787,"dst":6032,"dsp":2071,"dti":2638,"dte":2421,"dtr":1052,"duk":1051,"duc":1998,"dri":1805,"dra":1347,"dt ":18171,"dre":7141,"dro":1133,"dsk":3064,"dsb":2127,"dsa":1334,"dse":2268,"dgi":2184,"dia":1005,"der":53839,"des":7277,"det":27574,"deb":1003,"deh":997,"del":20136,"den":48394,"dem":1809,"di ":1057,"dga":1116,"dle":3278,"dla":1839,"dkr":1756,"dli":6294,"din":2827,"dio":1576,"dis":4268,"dit":1114,"die":2342,"dig":3302,"rhu":1754,"rho":1270,"rga":2811,"ri ":2041,"rgi":1032,"rge":3583,"ret":16711,"res":7617,"rev":3074,"rfa":1970,"rfo":1099,"rds":1442,"rdv":999,"rg ":8008,"reb":1759,"rea":2289,"ref":3055,"red":12454,"rei":2319,"reg":5210,"rem":3072,"ren":13272,"rek":1859,"rel":3506,"rer":8658,"rep":1174,"rda":1335,"rdl":1187,"rdi":2720,"rde":7501,"re ":32700,"rbu":1055,"rbr":1012,"rd ":6821,"ras":1016,"rat":6278,"rbi":1442,"rba":1156,"rbe":2916,"rag":1612,"ran":8553,"ram":2785,"ral":3419,"rak":1777,"raf":2970,"rad":2370,"rs ":4872,"ros":1298,"rot":1028,"rom":2663,"ron":3183,"rop":3118,"rov":4008,"rod":3399,"roc":1755,"rol":2335,"rof":1572,"rog":3109,"rna":2635,"rne":13393,"rni":1274,"ro ":1858,"rma":3375,"rme":5215,"rli":2882,"rla":2011,"rn ":3815,"rks":2467,"rko":1321,"rki":1210,"rke":7276,"rka":1044,"rm ":1940,"næs":1010,"nær":1381,"rip":1436,"rio":1666,"når":1017,"rit":3128,"ris":7929,"riv":2443,"rig":7130,"ril":1328,"rik":9410,"rin":12007,"rim":1705,"ria":2312,"rib":1119,"ric":1788,"rid":1336,"rie":7227,"rif":1151,"rk ":5973,"rug":4144,"rue":1823,"rup":5078,"run":5278,"rum":2612,"ruk":1154,"rus":1691,"rva":1035,"rvi":1176,"rve":2295,"ry ":1399,"rsk":6329,"rsl":985,"rsi":2088,"rso":2553,"rsa":1053,"rse":2094,"rta":1313,"rst":8534,"rte":6848,"rti":3566,"rts":2294,"rt ":7502,"rri":1525,"rre":6580,"sag":1459,"sal":1002,"sam":8006,"sbe":2183,"san":3521,"sat":3398,"ryk":1357,"sho":1591,"sie":1150,"sid":5798,"sk ":33968,"sit":3034,"sis":4442,"sin":4733,"sio":2930,"sik":3367,"sig":4968,"sby":2385,"se ":14360,"sch":1769,"ser":12739,"ses":2382,"set":2238,"sfo":1331,"sep":1174,"sen":14543,"sem":1108,"sel":4270,"sek":1218,"spo":1957,"spr":2788,"spe":2284,"slæ":1631,"spi":8255,"spa":1108,"sol":1008,"som":26634,"son":4747,"sor":1570,"sog":2156,"st ":15643,"sli":981,"slu":1369,"sky":1101,"sla":3115,"sle":2180,"ski":4168,"skl":1367,"sko":3585,"skr":6518,"sku":2545,"ska":8399,"ske":25508,"sni":1817,"sma":1205,"sme":2348,"stæ":1164,"stå":3319,"syd":3714,"stø":3565,"syn":1352,"sys":1619,"sse":5630,"ssa":1137,"sso":1208,"ssi":4603,"ste":30468,"sta":15172,"stn":1107,"sto":7688,"sti":13739,"stj":1076,"stl":3640,"stu":1392,"str":12753,"sty":2236,"sva":1612,"sve":1375,"svi":1847,"tal":8954,"tag":2922,"tab":1032,"tad":2960,"tat":8263,"tar":2918,"tan":5101,"tam":1561,"te ":23992,"ta ":2303,"pa ":1106,"køb":1295,"pe ":3064,"par":4298,"pan":2233,"lå ":2263,"pec":1027,"pen":2557,"per":8319,"pel":1218,"pla":4596,"ple":1353,"læg":3305,"læn":1040,"lær":1638,"pil":7538,"pis":1494,"por":2733,"pop":989,"pos":979,"pol":3456,"ppe":4792,"pst":1130,"pte":1387,"pti":1074,"pri":5464,"pre":2422,"pro":9553,"pun":1123,"præ":2064,"løb":1835,"løs":1373,"mæn":1058,"mær":1139,"mål":1925,"mån":1657,"ra ":16943,"ngi":1084,"ngl":1961,"ngs":7820,"ni ":1743,"nge":21530,"ngd":1418,"nha":2979,"nhe":1266,"nel":5371,"nek":1429,"nen":7128,"nem":2182,"ner":14377,"net":9752,"nes":8157,"ng ":25397,"ned":2644,"nfo":1407,"nce":2989,"ne ":28324,"ndt":10291,"ndr":5292,"nds":9100,"ndo":2080,"ndl":3623,"ndk":1968,"ndi":3782,"nde":48861,"ndb":3975,"nda":1742,"nal":7713,"nan":1050,"nar":1016,"nd ":19511,"nav":5158,"nat":4037,"nas":1015,"na ":3264,"nve":1870,"nus":1183,"nto":1311,"ntr":3209,"nti":2579,"ntl":1237,"nta":2683,"nte":11676,"nsp":982,"nst":8074,"nse":6294,"nsi":1207,"nsk":23035,"nsb":1130,"nt ":5652,"ns ":13305,"nog":1664,"nom":1719,"nor":6469,"nov":1161,"nne":10196,"nni":1944,"nma":3344,"nli":1178,"nla":1269,"nke":3504,"nkt":1750,"jæl":1755,"nie":2401,"nk ":979,"niv":1911,"nis":7071,"nit":1685,"nio":1012,"nin":14093,"nik":1101,"ogs":5045,"ogr":2357,"ogi":3275,"ogl":1041,"ogn":7366,"oge":2091,"ol ":1201,"oci":1344,"ock":2154,"ode":4999,"ods":1493,"of ":2459,"odb":2119,"odu":2868,"og ":59080,"oft":2765,"off":2072,"ofe":1326,"od ":3992,"obe":1539,"nyt":1313,"ote":1650,"ott":1354,"oto":1594,"ost":2113,"ov ":1326,"osk":985,"ose":1440,"ovi":1957,"ovs":2306,"ove":11716,"oun":1213,"our":1852,"opl":974,"ope":1372,"opf":1235,"opa":1354,"os ":2264,"opr":3026,"opt":1105,"ops":1721,"or ":31324,"ork":2427,"orl":1427,"orm":8791,"orn":2079,"ord":14176,"ore":6583,"orf":2453,"org":9163,"orh":1310,"ori":4799,"ort":7693,"ors":6893,"orv":1018,"ot ":1242,"orb":3261,"ora":1624,"old":10067,"on ":17437,"oli":4334,"oll":3644,"olk":2982,"ole":3254,"ols":1497,"olm":1298,"olo":4008,"oka":1272,"om ":32517,"ona":4226,"ond":1810,"one":10149,"ong":2553,"oni":2635,"ono":1620,"ons":4853,"ont":2499,"oma":2195,"ome":4668,"omi":1702,"omh":1267,"omm":13262,"omk":1807,"omp":2560,"omr":2620,"oms":1669,"op ":2057,"la ":1857,"gør":1749,"le ":18274,"lde":9460,"ldt":2484,"lds":2126,"lac":1163,"lad":4353,"lag":4790,"lan":22791,"lam":1129,"lar":1164,"lat":2523,"las":2596,"lav":1728,"lba":1173,"ld ":5587,"lbo":1181,"lbu":1855,"kvi":1085,"kva":1167,"kun":3121,"kul":1926,"kte":2817,"kst":2005,"kso":1301,"kue":1399,"ktr":1143,"kti":4258,"kto":1463,"ktø":1090,"ls ":1795,"lok":1028,"lom":2968,"lod":2232,"log":4064,"los":1196,"lot":1245,"lov":1376,"lmi":1310,"lme":1724,"lti":975,"ltu":1430,"lub":2283,"lsk":4901,"lst":4539,"lta":1027,"lte":2079,"lse":13829,"lre":2112,"lt ":6296,"lge":2455,"li ":1269,"lev":14315,"les":5717,"let":5891,"ler":27840,"lem":6534,"len":6881,"lek":2168,"leg":1270,"led":4289,"lla":4918,"lle":37621,"lli":6167,"lke":3500,"hæn":1282,"lm ":3777,"ll ":2166,"lit":4876,"lis":3436,"lin":9392,"hån":1024,"liv":2785,"lia":1325,"lik":2160,"lil":1691,"lig":34196,"lie":4111,"ma ":1277,"mag":1391,"mar":6718,"mas":1505,"mal":1831,"man":9443,"mat":3311,"mbe":3163,"me ":4862,"med":21647,"meg":1181,"met":7243,"mes":4874,"mer":15160,"mel":6023,"men":16847,"lut":1751,"lys":1458,"høj":2362,"hør":1554,"mpe":2509,"ms ":982,"mod":4612,"mon":1609,"mor":1053,"mt ":4527,"mst":1872,"mrå":2725,"mus":3381,"mul":1155,"mun":9999,"mhe":1090,"min":6763,"mil":3215,"mis":2297,"mid":2156,"mle":1475,"mkr":1669,"mmu":9493,"mme":16141,"vær":6621,"zir":1025,"væg":1206,"ytt":1956,"yst":3197,"ysk":5330,"yre":2022,"yr ":1010,"yde":3637,"yer":2527,"yen":3113,"yd ":1599,"ykk":1287,"yll":1699,"ynd":1352,"ygg":4471,"tør":4175,"tær":1046,"tæn":1204,"tår":2373,"sæt":2008,"så ":5902,"sær":1369,"røn":1765,"rød":1365,"vst":2580,"vir":2222,"vik":2041,"vil":2648,"vin":4397,"vig":1873,"råd":3726,"vid":3491,"vis":5672,"ræs":2950,"ræn":2483,"ræk":2614,"vn ":4721,"vne":3693,"vns":1255,"vok":971,"vor":5368,"ver":19395,"ves":5050,"vet":6585,"vej":1947,"ven":7443,"vem":1052,"vel":2142,"ved":12182,"vde":1584,"ve ":6698,"val":2588,"van":4077,"var":14654,"vat":1120,"usk":1540,"usi":3385,"use":2336,"ust":3006,"uss":1778,"uti":1400,"ute":1658,"us ":6486,"ut ":1499,"ure":2206,"urg":1747,"uri":1026,"urn":1614,"uro":2454,"urt":1085,"ur ":2594,"upp":2843,"umm":1471,"umb":1127,"ume":1120,"uns":1753,"unk":1906,"uni":2675,"und":15503,"una":2326,"ung":3315,"une":7027,"up ":2829,"ukt":2014,"um ":5774,"ult":2011,"uli":1842,"ule":1205,"uld":1327,"un ":2311,"uge":2701,"ugl":1024,"ugu":1009,"ugt":1589,"udb":1145,"udd":1881,"ude":3316,"udg":3260,"udi":978,"ue ":1121,"uce":1989,"uer":2229,"ues":1633,"udv":2056,"uds":2177,"udt":1177,"uel":1000,"ub ":1257,"uar":1965,"ubl":1141,"ubb":1133,"ud ":2370,"typ":1409,"tyr":2052,"tys":3953,"ty ":1687,"træ":3121,"tur":4777,"tut":1397,"tun":1107,"tud":1198,"tyd":2007,"ts ":4804,"tre":5166,"tra":6691,"tri":4615,"tru":5093,"tro":3770,"try":1478,"tv ":1126,"tte":14611,"to ":3441,"tni":3342,"tne":990,"tof":1468,"tod":1141,"tog":1011,"ton":2739,"tol":1446,"tor":9016,"top":1001,"til":27566,"tik":5022,"tif":3307,"tie":1611,"tig":3596,"tit":2754,"tis":8549,"tin":4128,"tio":14633,"thu":1021,"tia":1323,"tid":6831,"tiv":4885,"tje":1256,"tli":4586,"tla":1138,"tle":1766,"tem":5581,"ten":11717,"tek":2453,"tel":3444,"teg":5502,"ted":5198,"th ":1733,"tet":9121,"tes":3522,"ter":41030,"på ":20701,"ti ":3596,"the":2230},"n_words":[9674395,11309170,8090238],"name":"da","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/de.json b/contrib/languages-data/de.json
new file mode 100644
index 0000000..df96884
--- /dev/null
+++ b/contrib/languages-data/de.json
@@ -0,0 +1 @@
+{"freq":{"D":565432,"E":313466,"F":303463,"G":350968,"A":448295,"B":452862,"C":218833,"L":272371,"M":385363,"N":224760,"O":171336,"H":264836,"I":169730,"J":158568,"K":336559,"U":140118,"T":243403,"W":221377,"V":203138,"Q":15370,"P":333284,"S":807982,"R":282432,"Y":16379,"X":9646,"Z":96079,"f":1052401,"g":2011313,"d":3686275,"e":12779820,"b":1254948,"c":2286896,"a":5009484,"n":7859499,"o":2714321,"l":3150399,"m":2035145,"j":85201,"k":1042619,"h":3128015,"i":7539051,"w":833806,"v":645462,"u":3015383,"t":5309288,"s":5151894,"r":6424621,"q":20503,"p":756433,"z":844832,"y":283697,"x":72882,"Ü":10506,"ß":107796,"Ö":13881,"í":8823,"é":38695,"ä":358815,"á":12062,"ü":397123,"ö":249595," l":98174," m":186199," n":127225," o":112357," h":112970," i":1142912," j":27689," k":107425," d":1491422," e":826321," f":160920," g":218820," a":569210," b":294630," c":26306," z":200898," u":467824," t":56510," w":322251," v":371555," p":57089," s":347463," r":56139," J":156270," K":328020," H":253550," I":146651," N":212076," O":159226," L":260916," M":370185," B":437655," C":196607," A":413316," F":291285," G":334059," D":543793," E":298852," Z":93212," Y":15401," S":736164," R":264582," Q":14562," P":316134," W":214460," V":189915," U":132267," T":228797," ä":10185," ö":18488," ü":31572," Ö":12893," Ü":10208,"A ":22760,"Da":90985,"Cl":9563,"Co":64248,"Ch":45017,"G ":11776,"Du":11229,"Do":28882,"Dr":19902,"De":166635,"Di":188563,"Fe":38252,"Fa":46184,"Eu":13931,"Er":67899,"Es":30499,"En":32393,"Ei":81512,"El":17463,"Ge":151044,"Ga":33294,"I ":13640,"Fu":26023,"Fr":58870,"Fo":32048,"Fl":33310,"Fi":36767,"B ":10700,"C ":14623,"Au":73540,"Ar":58134,"At":9258,"As":17569,"D ":9913,"Ba":87454,"Ab":25174,"Ad":11158,"Am":21201,"An":55559,"Ap":14324,"Al":66179,"Bu":55758,"Br":45253,"Ca":33967,"Bi":31885,"Be":143843,"Bo":32121,"Bl":15582,"Ku":30221,"Kl":25047,"Kr":46024,"Ko":56995,"Le":49783,"Li":55021,"La":89578,"Lu":17469,"Hö":9378,"Lo":26047,"Me":64987,"Mi":68245,"Ma":116460,"Mu":29273,"Mo":46837,"Ni":23664,"Ne":44049,"Na":66351,"P ":9001,"No":53500,"Ok":13364,"Ol":9042,"Ob":22144,"Gi":11262,"Gl":11809,"Gr":74845,"Go":17306,"Gu":11007,"Ha":86398,"He":62608,"Hi":21572,"Dé":12333,"Ho":42207,"Hu":11017,"Im":11278,"In":78332,"Is":10695,"Ja":77663,"Je":11227,"Jo":24131,"Ju":32751,"Ka":74688,"Ki":37983,"Ke":14966,"Um":8765,"Un":54096,"Tu":10377,"US":39827,"Tr":32100,"To":24574,"Th":35305,"Ti":19953,"Te":52263,"Ta":27705,"V ":9454,"Sy":15026,"St":193472,"Su":18450,"Wo":21476,"Wi":47351,"Wa":44055,"We":77082,"Vo":36505,"Vi":22665,"Va":14430,"Ve":104056,"Mä":13878,"Pu":10703,"Pr":83211,"S ":48175,"Pe":31242,"Pf":21717,"Pa":56041,"Pl":15401,"Po":48441,"Pi":16961,"Ph":15188,"Os":17404,"Or":58036,"R ":11116,"Kö":16414,"Se":64611,"Sc":120209,"Si":90352,"Sh":10028,"Sp":54743,"So":39076,"Ru":19547,"Sa":63564,"Re":106782,"Ri":24028,"Rh":15491,"Ro":44332,"Qu":12167,"SA":9500,"Ra":38143,"Mü":11354,"b ":47610,"a ":292619,"Sü":22392,"Ze":31894,"Zi":10309,"Zu":19954,"Zw":11243,"i ":187041,"ge":787899,"ga":94286,"fl":45063,"fg":17796,"ff":81186,"fi":73360,"fs":18166,"fr":68277,"fu":19769,"ft":140067,"fo":55471,"he":941699,"hb":12648,"ha":293469,"gn":22363,"gl":78298,"gk":13841,"gi":130788,"gh":20146,"gg":10845,"gu":58531,"gt":76690,"gs":110120,"gr":125748,"go":29970,"dt":100722,"du":88105,"dw":22283,"g ":403400,"ea":71177,"eb":165228,"ec":131420,"aß":17528,"ed":151386,"de":1831853,"dg":9266,"di":398882,"dh":11418,"dk":27018,"dl":45536,"do":69104,"dn":16795,"ds":46937,"dr":54467,"ew":66696,"ex":29126,"eu":212719,"ev":25427,"ey":30641,"ez":98518,"fa":86743,"h ":463041,"fe":136348,"eh":196157,"eg":232810,"ef":68699,"ee":58149,"el":553287,"ek":82123,"ei":1578594,"ep":49360,"eo":41207,"en":2309793,"em":396788,"et":380175,"es":772223,"er":2971302,"ca":38911,"bz":11720,"e ":2082290,"bs":31297,"br":69597,"bu":78689,"bt":20985,"bo":31335,"bl":45392,"bg":11719,"bi":136887,"be":567764,"db":9156,"da":176290,"f ":156375,"cu":10926,"ct":17144,"co":32151,"ck":133244,"ci":25902,"ch":1933159,"ce":45478,"c ":18586,"az":20071,"ay":30095,"ba":134727,"d ":722683,"at":393648,"as":323772,"ar":487358,"ax":8820,"aw":12659,"av":27657,"au":496496,"ak":51121,"al":571224,"ai":78558,"ap":48798,"am":283487,"an":921405,"ac":186135,"ad":167659,"aa":47275,"ab":98737,"ag":124899,"ah":148615,"ae":30527,"af":117969,"nu":74735,"nt":478161,"ns":309893,"nr":20689,"no":90394,"nn":190420,"nz":136146,"ny":13727,"nw":36447,"nv":14218,"oe":10022,"of":64141,"oc":78480,"od":135885,"oa":14277,"ob":51225,"om":176521,"on":667125,"ok":27868,"ol":220141,"oi":24320,"og":71416,"oh":66434,"ot":88470,"os":118609,"ov":54069,"ou":69753,"op":68308,"oo":24531,"or":433879,"r ":1993661,"ow":58643,"oz":19078,"kö":10263,"pe":109404,"pf":29697,"pa":81077,"kü":12292,"pl":26694,"po":71059,"ph":41272,"lä":41208,"pi":98645,"lo":124488,"ln":33301,"lm":52275,"ll":300411,"ls":178640,"lr":10222,"lp":20136,"lv":15360,"lu":97812,"lt":212969,"lz":28989,"hö":49290,"ly":20012,"o ":123676,"iß":9834,"ma":207608,"hü":11610,"mb":77222,"me":410943,"mf":12265,"ml":10335,"mi":238709,"mm":108620,"mp":57358,"mo":55705,"mt":36178,"ms":30817,"mu":48362,"p ":27276,"na":304264,"nb":61884,"nc":42156,"nd":1104110,"ne":802403,"nf":54695,"ng":588331,"nh":45861,"ni":342822,"nk":102273,"nl":39986,"nm":15381,"ki":44691,"ke":186644,"ka":155408,"fü":107647,"m ":660573,"ks":34677,"kt":124687,"ku":45967,"ko":74144,"kr":80423,"kl":44918,"km":33455,"li":562165,"hä":31781,"lh":12595,"lk":36060,"le":501436,"ld":86618,"lg":41891,"lf":32687,"la":312768,"lc":19357,"lb":59934,"n ":2940210,"hr":245096,"hs":72646,"hw":53275,"ht":189239,"hu":73471,"hk":8886,"hi":180846,"hn":163077,"ho":102479,"hl":130889,"hm":50687,"id":80042,"eß":9854,"ic":534870,"ib":46123,"ia":115654,"ih":35970,"ig":262184,"if":71985,"ie":1150659,"hy":13546,"k ":140095,"ir":170539,"is":1295546,"it":570135,"iu":24806,"iv":65304,"ik":165059,"il":298251,"im":289814,"in":1828758,"io":214368,"ip":33386,"je":30947,"fä":14922,"iz":46043,"l ":312169,"ja":19963,"tä":51288,"xi":15762,"z ":144384,"sü":19128,"sä":14628,"wi":176498,"wo":47417,"wu":73059,"rö":38038,"y ":99455,"wa":198781,"rü":63451,"we":246633,"rä":41054,"vi":76382,"vo":299196,"uz":17658,"ve":197477,"va":38019,"x ":21295,"ui":24830,"uk":24301,"ul":92478,"ue":71615,"uf":143869,"ug":74097,"uh":10714,"ur":375391,"us":360270,"ut":230981,"um":178633,"un":921062,"up":64261,"ty":34073,"tz":131414,"tu":172266,"tt":172596,"tw":53648,"ub":43047,"ua":44618,"ud":39409,"uc":128427,"w ":36904,"to":167744,"tn":11002,"tm":18841,"tl":110769,"ts":280176,"tr":215121,"tp":9922,"tg":32640,"tf":25417,"te":1203919,"tk":15341,"ti":420789,"pä":14388,"th":119533,"v ":14300,"tb":22514,"ta":360262,"su":36834,"sv":12561,"ss":289880,"st":1222626,"sy":19371,"sz":17514,"sw":26360,"sl":40519,"sk":49450,"sm":28595,"sp":131491,"so":116426,"sr":21109,"sd":16337,"sc":827974,"oß":20955,"sf":20622,"se":456715,"sh":47160,"sg":56341,"si":322120,"nö":10879,"rz":76103,"u ":108662,"sa":109787,"sb":46727,"rr":85173,"rs":267473,"rt":385464,"ru":174667,"rv":18995,"rw":59814,"ry":19301,"rp":30886,"ro":256692,"rn":187691,"rm":96942,"rl":93135,"rk":131828,"ri":506618,"rh":58397,"rg":185502,"rf":74192,"re":658545,"rd":287337,"rc":86223,"rb":105570,"ra":423969,"t ":1627252,"mö":8912,"qu":17893,"mä":18127,"lü":10612,"s ":1183348,"pt":54028,"pu":30332,"pp":47091,"pr":99340,"ps":12738,"zä":9571,"zö":21005,"wä":21780,"zi":99549,"zb":8992,"ze":213386,"za":20320,"zw":56096,"zu":170081,"zt":32056,"zo":13315,"ye":15230,"uß":27824,"ya":11551,"tü":14407,"yt":8983,"ys":38183,"yr":10616,"yp":13299,"yn":15755,"ym":18550,"yl":11401,"äc":27469,"Üb":9926,"ßb":17940,"ße":45162,"ßt":14023,"ép":12471,"ät":44405,"äu":30285,"äl":29341,"än":69784,"äs":15220,"är":37279,"äd":10841,"äg":13766,"äf":13624,"äh":39944,"ün":62239,"üs":17438,"ür":122114,"üt":16867,"üb":39622,"üc":24866,"üg":9236,"üd":43250,"üh":43336,"öß":14937,"ör":67546,"ös":54393,"ön":22557,"öl":13380,"öm":9902,"öh":14808,"öf":13450," Ga":33168," Ge":150714," Fo":31900," Fu":25992," Fr":58787," Fi":36574," Fl":33229," Ha":86258," He":62512," Go":17225," Gr":74652," Gu":10914," Gi":11192," Gl":11774," Hu":10983," Ho":42078," Dé":12331," Hi":21530," Je":11187," Ja":77578," Is":10643," Im":11222," In":77993," Ka":74525," Ke":14835," Ki":37849," Jo":24054," Ju":32722," La":89327," Le":49575," Li":54819," Kl":25011," Ko":56929," Kr":45960," Ku":30157," Ma":116111," Mi":68090," Me":64810," Lo":25969," Hö":9372," Lu":17420," Ne":43845," Na":66181," Ni":23603," Mo":46674," Mu":29161," Ap":14296," Am":21144," An":55404," Al":65979," Ad":11104," Ab":25090," Ba":87247," Au":73430," At":9235," As":17394," Ar":57936," Be":143584," Bi":31731," Bl":15450," Bo":31991," Br":45134," Bu":55625," Ca":33546," Ch":44814," Cl":9441," Co":63874," Da":90740," Di":188080," De":166199," Dr":19868," Do":28619," Du":11174," El":17420," Ei":81331," Es":30480," Er":67840," En":32299," Eu":13899," Fe":38170," Fa":46079," Sü":22387," Wo":21330," Wi":47141," We":76915," Wa":43907," Zu":19747," Zw":11238," Ze":31828," Zi":10283," Kö":16410," Os":17332," Or":58009," Po":48260," Pl":15331," Pi":16916," Ph":15079," Pf":21701," Pe":31180," Pa":55779," No":53429," Ol":9032," Ok":13357," Ob":22103," Ra":37989," Mü":11350," Qu":12048," Ro":44140," Re":106629," Ri":23976," Rh":15474," Pr":83041," Pu":10674," Mä":13877," Sy":14940," Su":18414," St":192451," Ta":27626," Th":35151," Ti":19864," Te":52071," US":39409," Tr":31991," To":24396," Ru":19497," Sa":63396," Sh":9926," Si":90133," Sc":119896," Se":64226," So":38900," Sp":54582," Va":14382," Ve":103873," Vi":22523," Vo":36449," Tu":10321," Um":8742," Un":54031," je":12055," im":207684," in":455196," is":439990," it":10403," ka":23448," fü":85353," ha":41553," he":33774," gi":8908," gl":12278," gr":31732," ih":21981," hi":13984," ho":9452," ni":22730," ne":14510," na":48500," od":67197," of":20927," nu":10713," no":19195," le":19412," li":42027," la":25753," ku":12534," km":15800," kl":13510," ko":18379," me":26065," mi":107786," ma":27726," ab":26560," am":69685," an":87925," al":101826," au":256133," ba":8860," bi":45326," be":204994," br":14611," bz":9809," er":81246," et":21454," es":15402," en":55368," ei":604709," eh":21203," fr":36324," ge":155650," ch":11442," da":104895," dr":12440," de":1066350," di":258330," du":31949," zu":137283," zw":39753," ze":10132," sü":17881," ru":14611," se":68014," sc":24771," si":106825," sp":27646," so":41648," re":21422," pr":15398," po":16457," wa":91486," we":72625," wu":68805," wi":72114," ve":73784," vo":278143," vi":13609," um":38406," un":417759," st":39848," tr":12290," th":15725," te":8776," Üb":9908," ös":13916," üb":31186,"Fer":11772,"Fil":18131,"Fam":19152,"Es ":23376,"Eur":10652,"Ein":58080,"Eis":15092,"Er ":28197,"Ent":11856,"Gem":51515,"Geb":17190,"Ges":25022,"Ger":10345,"Gen":10761,"Gat":12348,"Fuß":15716,"Flu":13729,"Fra":28567,"Fri":10250,"Fre":14074,"For":20619,"Dép":12076,"Hei":15430,"Her":23305,"Hal":8751,"Han":15746,"Hau":30716,"Gre":10954,"Gra":17232,"Gru":19222,"Gro":14185,"Int":14682,"Ins":19291,"Ind":10322,"In ":14119,"Hoc":11500,"Art":17165,"Aut":13725,"Aus":25209,"Aug":12878,"Auf":14417,"Ban":13867,"Bad":10629,"Bar":8982,"Bas":9343,"Bau":14505,"Als":17057,"Alt":10907,"Ant":9903,"Bun":26145,"Bur":11846,"Car":9966,"Bez":18543,"Bes":11001,"Ber":42078,"Bei":10336,"Beg":14069,"Bre":9847,"Bra":12567,"Dez":9022,"Der":97305,"Deu":28251,"Das":56242,"Chr":9059,"Chi":9883,"Cha":13525,"Com":14437,"Cou":18879,"Die":157652,"Dre":10394,"Dor":10511,"Neu":14698,"Nat":19706,"Nie":12050,"New":10171,"Nam":20762,"Nac":12244,"Nov":10049,"Nor":35685,"Okt":8850,"Obe":15435,"Ort":33777,"Ost":13628,"Pla":10533,"Pfl":9335,"Pfa":9319,"Per":13707,"Par":21713,"Pro":54127,"Pri":9311,"Pol":21608,"Mär":10626,"Jan":11806,"Jah":47539,"Joh":10371,"Jul":9776,"Jun":10796,"Kan":19278,"Kar":13767,"Kir":12749,"Kil":10995,"Kla":9457,"Kon":17432,"Kom":17455,"Kre":22963,"Kri":9328,"Kul":11508,"Lei":11844,"Lan":56584,"Lin":9234,"Lis":12636,"Men":8765,"Met":15567,"Man":15071,"Mal":9073,"Mar":33951,"Mai":15148,"Mon":12046,"Mit":29518,"Min":10015,"Mus":19078,"Süd":20812,"Wil":11137,"Wie":10497,"Wei":14673,"Wel":14947,"Wer":11351,"Wes":22169,"Was":9091,"Wal":11499,"Vor":17501,"Vol":11987,"Zei":17372,"Str":23794,"Stu":11271,"Sta":109473,"Ste":21247,"Tei":19885,"Sit":13505,"Sie":53287,"Ser":9549,"Sep":9592,"Sei":11021,"See":8773,"Spi":16479,"Spr":13123,"Sch":114345,"San":12822,"Rhe":11094,"Rec":10719,"Rei":15453,"Reg":42336,"Rom":8900,"Ver":96243,"Uni":18259,"Unt":29085,"The":20642,"US ":31557,"Tra":12824,"bis":44646,"bil":24249,"bin":15696,"ble":9072,"bli":23577,"be ":15350,"ban":20200,"bal":24830,"bah":12251,"bac":14708,"bau":19849,"bar":18594,"bei":74577,"bef":14886,"bed":10076,"ber":182051,"ben":84728,"bel":16201,"bek":17699,"bez":50353,"bew":10065,"bes":47672,"bet":15480,"bie":27376,"bge":10311,"bzw":9957,"ca ":13669,"ce ":20992,"bt ":13539,"bri":17955,"bra":14742,"bre":12676,"bru":12222,"bst":12657,"bur":42376,"bun":13485,"am ":57496,"al ":67162,"ain":24754,"ais":13530,"ahm":10952,"ahn":21966,"ahl":22279,"ahr":74469,"anu":15776,"anz":50908,"ano":9997,"ann":95807,"ant":49762,"ans":31362,"ane":16424,"ang":74734,"ani":87171,"ank":29449,"ana":30762,"anc":17098,"and":264094,"amt":12217,"amm":42375,"amp":12259,"ami":43004,"ame":78887,"amb":11361,"ama":13307,"alz":12079,"alt":72681,"als":84009,"all":87642,"ali":76090,"ald":16999,"ale":62596,"ala":15461,"alb":20979,"an ":124982,"akt":19509,"abe":38207,"ae ":18008,"aat":34161,"ad ":15688,"ab ":9708,"aft":77933,"aff":9982,"ai ":14358,"age":54066,"adi":20643,"ade":24209,"ag ":17784,"adt":77282,"ack":9729,"ach":145520,"ace":11312,"aye":9954,"at ":58813,"are":33081,"ard":28108,"arc":10050,"arb":19113,"ara":24649,"aro":9777,"arl":16356,"ark":32666,"ari":41626,"arr":11665,"ars":12931,"art":72580,"au ":29621,"asi":19652,"ase":11784,"ar ":124263,"as ":155043,"aut":27334,"ay ":9249,"ata":11051,"ast":27543,"ass":73183,"ato":14091,"ate":62804,"ati":113716,"ath":22536,"auc":75721,"att":42075,"ats":11481,"atu":18974,"atz":18097,"aum":14540,"aup":28995,"aus":149891,"aue":19995,"auf":98508,"itu":19936,"itt":50909,"its":25338,"itz":41656,"ism":9438,"iss":46444,"ist":567635,"ita":30952,"ite":83935,"itg":12750,"iti":55617,"ium":14795,"ivi":10241,"ive":36462,"is ":120788,"ion":169772,"irt":10794,"irk":24645,"isi":17517,"ish":14382,"ise":39335,"isc":434611,"isa":10445,"ire":14375,"ird":46859,"irc":21452,"it ":186965,"itä":20849,"izi":17119,"ize":13649,"kir":9307,"kis":9934,"km ":11509,"ki ":9579,"kei":25216,"kel":18319,"ken":38622,"ker":44613,"key":12420,"ke ":26956,"kra":15849,"kre":43910,"kt ":30020,"kri":11594,"kon":15402,"kom":22572,"ks ":9057,"kma":10480,"kle":18061,"kla":10851,"kat":15038,"für":75640,"kan":77012,"kal":10319,"füh":22557,"ka ":18181,"han":36578,"hal":38788,"hau":37347,"har":21603,"hat":27886,"haf":79189,"hab":9073,"he ":213315,"hel":11970,"hei":73898,"hec":9261,"heu":16839,"hes":34182,"her":185057,"heo":9547,"hen":324438,"hem":37228,"hie":35579,"hic":13745,"hin":28238,"hil":15183,"his":35328,"hl ":14867,"hn ":19219,"hla":28455,"hle":29146,"hli":16905,"hlo":9385,"hlu":9270,"gle":20084,"gli":40281,"gke":10880,"gs ":14145,"gsb":9179,"gsg":11045,"gro":10882,"gru":14203,"gra":25992,"gt ":53228,"gri":24893,"gre":15248,"gst":11232,"gss":10565,"gte":17882,"grö":13018,"grü":19508,"gus":11665,"gun":24014,"ial":21666,"ian":25164,"iat":10334,"ibt":8975,"id ":10846,"ibe":11815,"ia ":36994,"iet":30400,"iel":84385,"ien":123978,"ier":122946,"ies":42285,"ied":66086,"ieg":57690,"ief":8944,"ieh":10954,"ig ":34684,"iec":11989,"ieb":29946,"ift":20078,"iff":26227,"ick":25712,"ich":454249,"ie ":537544,"ica":15678,"ide":33142,"ida":13186,"ieß":9409,"il ":53074,"im ":225234,"ika":58499,"ige":123608,"iga":12204,"igk":10797,"igi":12120,"igu":13208,"igt":19283,"ign":9138,"ihe":9068,"ihr":18848,"ik ":39429,"imm":16766,"ime":10380,"ind":156874,"ina":40189,"inn":28145,"ino":11205,"int":30667,"ins":51098,"inf":10131,"ine":425715,"inh":13926,"ing":93214,"ini":56092,"inl":11619,"ink":12759,"inw":21089,"inz":40189,"ike":34841,"in ":793598,"ilo":17887,"ill":36002,"ilm":27378,"ili":52680,"ild":30027,"ile":21718,"ima":11905,"io ":15324,"hr ":43100,"hol":19495,"hor":9009,"hof":13254,"hoc":13003,"hni":15320,"hnu":17265,"hne":77095,"hme":30297,"hul":13709,"htu":10990,"hts":18003,"hti":8985,"hte":48599,"hst":18465,"hse":23032,"hrt":21145,"hre":87475,"hri":38046,"ht ":81034,"hwa":10659,"hwe":33406,"hum":9803,"hun":26557,"ffe":34346,"fes":13275,"fer":33138,"fen":43852,"fel":18220,"fge":10643,"fas":17434,"fan":11879,"fal":21835,"fah":14992,"ff ":20858,"fe ":14115,"ewä":9115,"eze":57686,"ezi":30267,"eta":11725,"ete":74739,"eti":14295,"eso":11977,"est":128414,"ess":60542,"eue":13801,"eug":13587,"etr":30198,"ett":26303,"etw":18129,"etz":35905,"ew ":11648,"eut":121638,"eur":19487,"ewi":9238,"ewe":18343,"ey ":16388,"erö":10701,"er ":1578023,"eor":17275,"es ":365853,"ept":13647,"epu":9001,"erk":45646,"erl":56900,"eri":118551,"erg":76890,"erh":33322,"ere":140461,"erf":36150,"erd":42731,"era":58724,"erb":63293,"et ":137716,"esi":26333,"esc":45868,"ese":67043,"esa":12858,"erz":20677,"erv":14305,"eru":46595,"erw":47997,"err":51760,"ert":122552,"ers":190257,"ern":139910,"erm":29678,"erp":11457,"ero":15582,"ekt":37882,"en ":1453342,"elb":18844,"ela":19480,"eld":20851,"elc":11326,"elf":9116,"ele":71198,"eli":22926,"elm":9454,"eln":17751,"ell":119434,"els":30215,"elt":58131,"ehö":33094,"eiß":9569,"emb":35370,"ema":36279,"eme":116815,"emi":15129,"enf":15937,"ene":83278,"enh":15362,"eng":41769,"enb":41104,"ena":52256,"end":112707,"eno":10734,"enn":31346,"enk":30270,"enl":8977,"eni":28822,"ens":113000,"ent":181395,"enr":9992,"enz":28843,"ege":66150,"egi":51407,"egr":34898,"egt":35267,"ehm":19125,"ehr":37443,"ehe":47146,"eib":17640,"eic":136250,"eht":23060,"eis":113764,"eim":26661,"eil":79634,"ein":827667,"eih":11402,"eie":15075,"eid":22147,"eig":23796,"el ":101100,"eiz":17159,"eit":174763,"efü":9260,"eka":19118,"em ":162112,"gis":26824,"gin":15255,"gio":29407,"gie":31403,"gen":239581,"get":10602,"ger":96243,"ges":76492,"gew":20591,"geb":50080,"geh":39393,"geg":29832,"gef":16578,"gem":34840,"gel":53887,"ge ":89868,"gab":8826,"gar":15096,"gan":29626,"ga ":12696,"frü":9222,"fte":22800,"ftl":11215,"fun":10974,"fts":18001,"ft ":64901,"fra":26521,"fre":14454,"fri":12770,"for":35090,"fol":14926,"fla":12837,"flu":9809,"fil":11145,"fin":25920,"da ":17513,"de ":197184,"das":76737,"dar":15587,"dam":14425,"ckl":11099,"chä":10997,"ch ":425455,"cha":118322,"chw":51871,"chu":40552,"ck ":31790,"che":671441,"chl":70583,"chi":103178,"cho":20768,"chm":11901,"chn":86780,"chs":68005,"cht":155579,"chr":43254,"cke":51895,"ed ":20514,"eba":8988,"ebe":60498,"ebi":30463,"ebr":17022,"ebu":9376,"eat":10811,"efi":14121,"efe":11305,"ei ":84962,"ega":9107,"eer":10701,"edi":25198,"ede":66213,"eg ":9875,"eck":32288,"ech":81379,"aße":12922,"ee ":17886,"dwe":14218,"dur":31652,"dor":21874,"don":10905,"ds ":13347,"dun":25608,"dri":12382,"dt ":65936,"dre":16392,"dsc":11728,"der":838163,"des":213577,"det":50672,"deu":71719,"del":34612,"den":286768,"dem":104609,"dkr":22973,"dli":27133,"din":14965,"dis":37105,"die":268782,"dig":18151,"rhe":18402,"rha":21798,"rga":19288,"rgi":14417,"rge":50622,"ret":20023,"res":34677,"reu":13265,"rfa":14831,"rfo":9599,"rg ":68697,"rea":10702,"rec":32717,"raß":12628,"rei":196090,"reg":17320,"rem":14552,"ren":145744,"rer":33774,"rf ":19644,"rdn":11514,"rdl":8810,"rdi":14216,"rde":136404,"re ":80803,"rbr":9368,"rch":72681,"rd ":68822,"rap":8929,"rar":8924,"ras":13690,"rat":45378,"rau":34010,"rbi":13086,"rba":21515,"rbe":36419,"rai":10459,"rag":25026,"ran":87263,"ram":18048,"ral":31964,"rab":8938,"raf":18926,"rad":17310,"rac":30425,"rs ":37799,"ros":12129,"rot":11834,"rom":12509,"ron":32055,"rop":24374,"rov":26322,"rod":18103,"rol":12320,"rof":8772,"rog":9738,"rns":12992,"rna":25318,"rne":34331,"rni":11673,"ro ":10671,"rma":31109,"rme":19061,"rmi":10506,"rli":33691,"rle":11502,"rla":23240,"rn ":74871,"rks":11567,"rke":27812,"rm ":16219,"rit":36924,"ris":65792,"rig":20472,"ril":13027,"rik":52744,"rin":60346,"ria":21040,"ric":47382,"rie":98836,"rif":31298,"rk ":38902,"rwe":21555,"rz ":21038,"ruc":12341,"rup":19176,"run":74339,"rum":15222,"rus":14979,"rwa":28214,"ry ":12420,"rsi":21022,"rso":11863,"rsp":15246,"rsc":59984,"roß":19513,"rsa":9024,"rse":16931,"rta":10712,"rst":71108,"rte":100313,"rth":11159,"rti":23089,"rua":8770,"rts":42716,"rtr":18145,"rt ":139444,"rro":10371,"rri":12758,"rre":38535,"rra":9354,"sam":29233,"sbe":20860,"san":12568,"sat":17995,"rze":22030,"sha":10198,"sho":11670,"sge":44768,"sie":44959,"sic":57522,"sit":30295,"sis":59876,"sin":51565,"sio":15881,"sik":20202,"se ":74178,"oße":10850,"sch":817718,"ser":52310,"ses":13066,"set":25883,"seu":12734,"sei":61074,"seh":13958,"see":12215,"sen":104567,"sem":11510,"sel":53325,"spo":10155,"spr":34328,"spe":13644,"spi":46629,"spa":12483,"sow":18519,"sol":10161,"son":33802,"sor":15536,"sre":10655,"st ":524603,"ss ":36271,"sla":20592,"ski":8870,"ska":11035,"so ":11408,"swe":11000,"stä":18451,"sse":110613,"ssa":12522,"sso":12151,"ssi":42403,"sst":37624,"ste":300373,"stf":9710,"sta":116653,"sto":29905,"sti":57375,"stl":38070,"stu":17599,"str":60741,"sun":13352,"sve":8840,"tal":50681,"tag":11693,"taa":33781,"tad":80208,"tbe":10504,"tau":10504,"tat":27116,"tar":23114,"tan":59689,"tam":12219,"te ":230506,"ta ":18923,"pe ":20233,"par":30140,"pan":20821,"läc":9211,"län":14161,"phi":11788,"pen":21299,"per":31265,"pel":10530,"pla":12193,"pie":61526,"por":21385,"pol":21212,"ppe":33120,"pub":8976,"pte":12552,"pts":12222,"pra":20767,"pri":21158,"pre":14559,"pro":24883,"ra ":23418,"ngi":9157,"ngl":32203,"ngs":81612,"ni ":15548,"nge":150664,"nga":14715,"nha":17909,"nhe":16114,"neh":18303,"nel":12298,"nen":124634,"nem":32650,"ner":149716,"net":57023,"nes":53346,"neu":10682,"ng ":256902,"nfo":9002,"nfa":11653,"nce":12344,"nch":15880,"ne ":310275,"nbu":15121,"ndt":9026,"ndu":21347,"ndr":11275,"nds":26772,"ndo":18073,"ndl":16133,"ndk":23064,"ndi":40257,"nde":333302,"nda":14652,"nal":58896,"nam":18687,"nan":39807,"nar":14659,"nac":41308,"nad":10470,"nah":12864,"nbe":18026,"nd ":563951,"nba":15883,"nau":11932,"nat":30766,"na ":34549,"nz ":37982,"nwo":19169,"nve":9948,"nun":33102,"nur":9530,"nua":9860,"nty":16624,"ntw":17792,"nto":23385,"nts":23619,"ntr":27593,"nti":33190,"nth":12699,"ntl":20923,"nta":24938,"nte":156191,"nsp":11166,"nst":73244,"nse":37942,"nsi":11408,"nsc":47879,"nsa":13614,"nsb":9715,"nt ":97967,"ns ":52604,"nom":14191,"nor":17522,"nne":52732,"nni":11264,"nnt":52262,"nns":10154,"nli":11747,"nn ":35057,"nla":20752,"no ":11518,"nke":16356,"nkm":14368,"nkt":16086,"nkr":10153,"nig":30567,"nie":44175,"nic":23391,"nia":9121,"niv":12854,"nis":144657,"nit":17226,"nin":9152,"nik":13041,"ogr":16058,"ogi":20348,"oge":19173,"ohl":8865,"ohn":34927,"och":36284,"ock":28973,"ode":93686,"of ":22612,"odu":15579,"off":17101,"nzö":20567,"obe":24203,"nze":40975,"nzi":11904,"owi":20813,"ozi":9366,"ow ":10053,"oti":8974,"oth":10089,"ote":15159,"ott":13896,"oto":13207,"ost":26924,"ose":16100,"oss":15828,"ovi":27674,"ove":15851,"oun":22912,"our":13749,"oph":11323,"opa":9142,"os ":20626,"or ":51491,"ork":11076,"orm":35190,"orn":17190,"ord":64827,"ore":22869,"orf":23363,"org":24171,"ori":35553,"ort":57075,"ors":22259,"ora":13071,"ola":9778,"old":10163,"on ":380926,"oli":42117,"oll":31645,"olk":11956,"ole":12997,"olg":16083,"olo":30821,"om ":33290,"ona":50054,"ond":30722,"one":41633,"ong":12616,"oni":29183,"onn":12427,"ono":12443,"ons":40673,"ont":24319,"oma":20160,"ome":26297,"omi":13337,"omm":35850,"omp":19252,"omo":11157,"la ":19811,"lb ":10692,"le ":103052,"lch":16014,"lde":27222,"lac":9475,"lag":29983,"lan":114878,"lar":12710,"lat":31357,"las":30573,"lau":19611,"lba":8767,"ld ":26676,"lbe":17468,"kur":12239,"kun":14388,"kte":20495,"ktr":9556,"ktu":9324,"kti":30020,"kto":15493,"ls ":107647,"lon":9380,"lom":14967,"log":27324,"los":22267,"lti":11054,"ltu":37425,"lug":9766,"lsp":9315,"lst":14411,"lte":61749,"lsc":14894,"lt ":63451,"lge":22548,"li ":14630,"les":24231,"let":14648,"ler":100316,"lem":16242,"len":91180,"lek":12889,"lei":58951,"leg":20421,"leb":11626,"lls":25768,"llu":11230,"llt":17543,"lla":17394,"lle":120640,"lli":30333,"ln ":17395,"lm ":19332,"ll ":39906,"lit":40830,"lis":62203,"lin":51826,"lic":170755,"lia":13089,"lik":12998,"lig":41309,"lie":114815,"ma ":15588,"mar":19548,"mal":49935,"man":57586,"mat":30614,"mbe":37028,"me ":36368,"met":20898,"mes":11403,"mer":74569,"mel":10134,"men":133736,"mei":94533,"meh":11820,"mfa":9540,"lve":9179,"lun":37355,"lus":22607,"lz ":10714,"hör":33720,"mpf":9424,"ms ":14667,"mon":13165,"mt ":15320,"mte":10917,"mus":17368,"mun":19505,"min":21460,"mil":28731,"mis":23371,"mit":124264,"mig":9701,"mie":12587,"mmu":13317,"mmt":13008,"mme":52772,"zt ":19861,"zte":9874,"zu ":51921,"zw ":9786,"zug":11138,"zur":37734,"zum":31618,"zun":15443,"zus":11821,"zwe":22370,"zwi":20415,"zei":75965,"zeu":11441,"zes":9884,"zen":43803,"zem":9613,"zel":10023,"zer":23898,"ze ":15773,"zia":11658,"zie":29129,"zig":9649,"zir":18395,"yst":17491,"ußb":14873,"uße":8940,"yer":10735,"tän":11031,"tät":24189,"süd":17890,"wur":70546,"woh":26117,"wes":27777,"wer":58987,"wen":21225,"wel":20147,"wei":83410,"weg":12299,"wir":55249,"wis":33738,"wie":34766,"wic":21374,"wa ":19878,"röß":14321,"wan":13038,"rün":30393,"wal":31257,"war":103047,"rüh":12452,"rüc":12801,"vin":25318,"vie":18884,"von":226355,"vom":22538,"vor":35872,"räg":10007,"ver":144066,"ven":17663,"vem":8808,"ve ":14670,"van":10775,"usi":22265,"usg":18900,"use":25762,"usa":16989,"ust":38480,"uss":45929,"usp":9564,"uti":13537,"ute":38886,"utz":15915,"uts":91328,"uto":20186,"us ":143584,"ut ":20376,"urd":76437,"urc":36382,"ure":16506,"urg":50861,"uri":14126,"urn":9214,"uro":18224,"urs":13370,"urt":10114,"urz":14868,"ur ":74659,"upp":21507,"upt":28543,"ums":9212,"umb":12929,"ume":14114,"umf":9413,"unt":60343,"uns":11344,"unk":20377,"uni":19103,"und":450828,"ung":313251,"ukt":12384,"um ":104918,"ult":19564,"uli":15580,"ule":13595,"ula":10852,"uge":19918,"uft":12442,"ugu":12915,"uf ":70919,"ude":15376,"ue ":12656,"uch":109126,"uck":11441,"uer":24349,"ufe":13329,"ufg":14459,"uen":13989,"uel":10929,"uar":22497,"ubl":12721,"tzt":23155,"tzu":9903,"tze":24373,"ty ":26382,"twa":23393,"tur":43568,"tun":79448,"tum":9791,"tz ":51212,"twi":14763,"twe":10501,"ts ":32847,"tre":34672,"tt ":15987,"tra":75255,"tri":43316,"tru":19097,"tro":30104,"tsc":125606,"tsg":8757,"tsp":10708,"tst":44445,"tte":89035,"tti":9403,"ttu":15931,"to ":13838,"tob":10974,"tom":9404,"ton":34822,"tor":56677,"tik":35201,"tie":28029,"tig":44003,"tit":14488,"tis":66050,"tin":30024,"tim":13478,"tio":114009,"thu":10251,"tiv":26072,"tli":78058,"tla":9588,"tle":19198,"tem":46999,"ten":278529,"tei":81154,"tel":91292,"teh":26143,"tec":9644,"tfa":9048,"th ":15758,"tet":40557,"tes":39452,"ter":318918,"tgl":9160,"tge":13122,"tho":19217,"the":36369,"tha":11467,"zös":20559,"zäh":9158,"épa":12146,"ähl":11359,"ähr":16538,"äch":26798,"äng":21244,"änd":29875,"ält":11293,"ät ":14412,"ärz":9551,"äte":10008,"äuf":9858,"ßte":10219,"ßen":17047,"ßer":9786,"ßba":15077,"ße ":16400,"Übe":9600,"üdl":8991,"ühe":8949,"übe":35449,"ück":19105,"ünd":28744,"ür ":75451,"üns":8869,"ühr":24588,"ößt":9800,"öst":25495,"ösi":20607,"ört":26727,"örd":14857,"öni":10085,"öff":11553,"ürt":8780},"n_words":[87197534,99298261,71857404],"name":"de","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/en.json b/contrib/languages-data/en.json
new file mode 100644
index 0000000..f0c64a5
--- /dev/null
+++ b/contrib/languages-data/en.json
@@ -0,0 +1 @@
+{"freq":{"D":662077,"E":559834,"F":608348,"G":582849,"A":1412556,"B":916671,"C":1498503,"L":649564,"M":1059392,"N":665299,"O":394009,"H":682378,"I":978047,"J":471262,"K":369000,"U":380950,"T":1402307,"W":505824,"V":256072,"Q":51938,"P":880443,"S":1630016,"R":708545,"Y":131400,"X":37224,"Z":73521,"f":5238032,"g":4381944,"d":8729953,"e":27848709,"b":3669334,"c":7841280,"a":23418136,"n":19713516,"o":18673929,"l":10669664,"m":6170962,"j":262547,"k":1633239,"h":10134148,"i":20570816,"w":3362380,"v":2275926,"u":6637499,"t":19408712,"s":16004058,"r":16873084,"q":170855,"p":4621926,"z":397471,"y":4124069,"x":440231,"é":58984," l":780297," m":1079276," n":608651," o":3436911," h":813084," i":3938466," j":78084," k":215197," d":941771," e":740834," f":1754540," g":440359," a":5405782," b":1659799," c":1744132," y":83514," u":322099," t":5058192," w":1900306," v":255788," q":29653," p":1560368," s":2415857," r":962057," J":460420," K":336817," H":654413," I":868613," N":607763," O":345142," L":596797," M":988580," B":847481," C":1361375," A":1263874," F":559603," G":550525," D":599768," E":493444," Z":67911," Y":122764," S":1468740," R":648863," Q":47311," P":804972," W":486015," V":212635," U":359652," T":1336813,"A ":159383,"Da":104182,"Cu":44169,"Cl":60734,"Co":426605,"Cr":65402,"Ce":60359,"Ch":258329,"Ci":62395,"Ed":38074,"Ea":55403,"Du":53228,"Do":65822,"Dr":31956,"De":174035,"Di":112559,"Fe":83008,"Fa":58998,"Eu":42215,"En":125985,"Em":27259,"El":44031,"Ge":134248,"Ga":73890,"I ":75735,"Fr":140633,"Fo":106181,"Fl":37525,"Fi":77847,"B ":28902,"C ":86756,"Au":138720,"Ar":132765,"At":36103,"As":94697,"D ":41383,"Ba":186506,"Af":42815,"Ac":49109,"Ad":33381,"Am":167256,"An":117416,"Ap":56752,"Ai":30417,"Al":127646,"Bu":74783,"Br":187559,"Ca":304473,"E ":36124,"Bi":58070,"Be":129264,"Bo":112086,"Bl":40686,"Ko":45338,"Le":127247,"Li":124649,"La":147235,"Lu":31872,"Lo":122679,"Me":130273,"Mi":151239,"O ":29173,"Ma":405967,"Mu":71213,"Mo":145746,"Ni":43349,"Ne":178415,"Na":143656,"P ":44679,"No":180123,"Ol":47193,"On":40436,"Oc":53105,"Gi":29349,"Gr":130751,"Go":73286,"Gu":48214,"Ha":132380,"He":218817,"II":40680,"Hi":92944,"Ho":127334,"Hu":45621,"K ":32711,"In":253915,"Is":60970,"It":340082,"Ir":53269,"Ja":154594,"L ":34197,"Je":52293,"Jo":101484,"Ju":117086,"Ka":74254,"M ":37484,"Ki":74159,"Ke":51086,"Un":220729,"Tu":36890,"Tr":84116,"US":35610,"To":93419,"Th":850306,"Ti":41292,"Te":99569,"Ta":74444,"V ":32754,"Sw":37200,"Sy":30043,"St":291796,"Su":96654,"Wo":87820,"Wi":99188,"Wh":33410,"Wa":135830,"We":94221,"Vi":82206,"Va":46571,"Ve":37939,"Pu":36227,"Pr":165318,"S ":81629,"Pe":109002,"Pa":220340,"Pl":35710,"Po":107653,"Pi":44494,"Ph":43729,"Or":50633,"R ":32772,"Se":177109,"Sc":119654,"Si":88455,"Sh":112206,"Sp":78520,"So":160970,"Ru":53200,"Sa":155395,"Re":186160,"Ri":92732,"Ro":164209,"Qu":37827,"T ":30323,"Ra":101065,"b ":134105,"a ":2991353,"Yo":71760,"i ":355519,"gd":31577,"ge":721279,"ga":370090,"fl":84104,"ff":180681,"fi":526440,"fr":391564,"fu":82822,"ft":147685,"fo":920673,"gy":74018,"he":4842012,"ha":1007728,"gn":127025,"gl":188283,"gi":377563,"gh":367310,"gg":29821,"gu":248240,"gt":38716,"gs":80931,"gr":275281,"go":177983,"du":289181,"dv":26927,"dw":35481,"dy":74747,"g ":1213593,"ea":1274992,"eb":152577,"ec":841985,"ed":2289411,"de":1320778,"dd":64948,"dg":51752,"di":930651,"dm":40343,"dl":51794,"do":273396,"ds":196380,"dr":141993,"ew":250035,"ex":221189,"eu":76879,"ev":357914,"ey":221145,"fa":236961,"h ":1529402,"fe":315742,"eh":48317,"eg":294475,"ef":194847,"ee":528675,"el":1146347,"ek":64162,"ei":272994,"ep":332185,"eo":187165,"en":2427008,"em":679038,"et":814658,"es":2395636,"er":4179896,"eq":41573,"ca":1058365,"e ":8494237,"by":527627,"bs":58904,"br":217191,"bu":297420,"bo":472510,"bl":324534,"bi":213219,"bb":35633,"be":846369,"da":395082,"f ":2316051,"cy":71900,"cu":238303,"ct":817847,"cs":93807,"cr":254963,"co":1252186,"ck":305854,"cl":251999,"ci":687756,"ch":1106571,"ce":1086909,"cc":111912,"c ":457702,"az":72598,"ay":450984,"ba":470554,"d ":4698126,"at":2664116,"as":2180049,"ar":2492347,"ax":35262,"aw":117859,"av":238618,"au":228752,"ak":197752,"al":2475728,"ai":590954,"aj":50310,"ap":330102,"am":913130,"an":4857931,"ac":715176,"ad":654223,"ab":325448,"ag":433896,"ah":83166,"ae":126319,"af":115896,"nu":217006,"nt":1825754,"ns":843426,"nr":30719,"no":643421,"nn":264870,"nz":30433,"ny":179007,"nv":69977,"oe":69809,"of":2379880,"oc":501750,"od":362363,"oa":174782,"ob":174628,"om":1179222,"on":3432632,"ok":132945,"ol":912097,"oi":135962,"og":247625,"oh":71587,"ot":549343,"os":498414,"ov":435331,"ou":1258409,"op":505111,"oo":421044,"or":2962572,"r ":3075136,"ox":44665,"ow":555778,"oy":75842,"pe":765233,"pa":591162,"pl":399253,"po":579386,"ph":211053,"pi":302192,"lo":761247,"lm":115044,"ll":1129598,"ls":295312,"lp":38509,"lw":35880,"lv":66473,"lu":281619,"lt":228186,"ly":725738,"o ":1535371,"ma":968442,"mb":416311,"me":1451345,"mi":591681,"mm":262444,"mp":468344,"mo":504581,"ms":133483,"mu":261893,"my":62036,"p ":307798,"na":1160193,"nb":27967,"nc":750937,"nd":2690580,"ne":1275364,"nf":87191,"ng":1746068,"nh":37213,"ni":1104016,"nk":109355,"nl":88734,"nm":67028,"ju":44053,"jo":70049,"ki":171410,"kh":26225,"ke":323852,"ka":109789,"m ":983735,"ky":30837,"ks":106786,"ko":46397,"kl":33919,"km":32936,"kn":143680,"li":1420180,"lk":42132,"le":1534709,"ld":351541,"lg":35629,"lf":59414,"la":1421955,"lb":105992,"n ":6374219,"hr":171058,"hw":45980,"ht":193546,"hu":172799,"hi":1052052,"hn":92044,"ho":773733,"hl":46194,"hm":30448,"id":496380,"ic":1849130,"ib":159645,"ia":1169835,"ig":507511,"if":219645,"ie":800933,"hy":82019,"k ":515132,"ir":647827,"is":3249081,"it":1893192,"iu":65009,"iv":578092,"ix":50259,"ik":78883,"il":1065515,"im":394260,"in":4877222,"io":1592954,"ip":270764,"je":50702,"iz":128734,"l ":1934675,"ja":50690,"xi":69443,"xp":49241,"xt":57882,"z ":53637,"xa":40308,"xe":33587,"wh":393518,"wi":429474,"wn":279417,"wo":263879,"wr":100589,"ws":68714,"vy":28150,"y ":3097451,"wa":1012808,"we":415691,"vi":599499,"vo":108876,"ve":1210480,"va":267555,"x ":126181,"ui":189592,"uk":33957,"ul":459768,"ue":281220,"uf":33361,"ug":230153,"ur":968763,"us":936891,"ut":635083,"um":403450,"un":943875,"up":204022,"ty":567576,"tu":446554,"tt":322006,"tw":175696,"ub":264839,"ua":340676,"ud":207254,"uc":282987,"w ":304612,"to":1515648,"tm":51220,"tl":201040,"ts":524135,"tr":893262,"te":2648213,"ti":2353666,"th":4782590,"v ":29410,"tb":79573,"tc":66980,"ta":1180046,"su":341240,"ss":646227,"st":2324937,"sy":102577,"sw":27118,"sl":145841,"sk":108573,"sn":41499,"sm":113246,"sp":316999,"so":700411,"sc":280155,"se":1450470,"sh":709009,"si":1101025,"u ":116514,"sa":232414,"sb":34898,"rr":265014,"rs":761380,"rt":886247,"ru":271327,"rv":158410,"rw":40021,"ry":553810,"rp":87486,"ro":1594919,"rn":650417,"rm":435749,"rl":279627,"rk":261286,"ri":2008460,"rh":32041,"rg":276649,"rf":65494,"re":2611877,"rd":466156,"rc":300833,"rb":95933,"ra":1639206,"t ":3468815,"qu":157005,"s ":7219728,"pt":158295,"pu":265747,"pp":173102,"pr":683558,"ps":94491,"zi":58211,"ze":102677,"za":82368,"zo":27033,"ye":187250,"yc":42157,"yd":28943,"ya":95795,"yt":30002,"ys":153061,"yr":36606,"yp":60686,"yo":54105,"yn":59856,"ym":80995,"yl":75727,"yi":42920,"一":42790," Ga":73074," Ge":133297," Fo":105500," Fr":140138," Fi":76403," Fl":37253," Ha":131725," He":218212," Go":72637," Gr":129383," Gu":47617," Gi":28529," Hu":45416," Ho":126780," II":28541," Hi":92512," Je":52048," Ja":154158," Ir":53197," Is":59983," It":339895," In":252514," Ka":73450," Ke":49770," Ki":73234," Jo":100944," Ju":116938," La":145853," Le":125799," Li":123371," Ko":45184," Ma":403673," Mi":150289," Me":129337," Lo":122023," Lu":31698," Ne":176925," Na":142533," Ni":43042," Mo":144948," Mu":70352," A ":81995," Ap":56659," Am":166952," An":116734," Al":126650," Ai":30214," Af":42668," Ac":48855," Ad":33141," Ba":185398," Au":138482," At":35966," As":93159," Ar":131824," Be":128479," Bi":57574," Bl":40368," Bo":111176," Br":186728," Bu":74408," Ca":300905," Ce":60031," Ci":62055," Ch":257143," Cl":59706," Cr":64523," Co":423247," Cu":43407," Da":103400," Di":111836," De":173171," Dr":31720," Do":63383," Du":52940," Ea":55250," Ed":37865," El":43749," En":125075," Em":27148," Eu":42050," Fe":82594," Fa":58222," Wo":87042," Wi":98480," Wh":33053," We":93550," Wa":135146," Yo":71619," a ":1606658," Or":50217," Po":106745," Pl":35261," Pi":44206," Ph":42887," Pe":108520," Pa":218936," No":179642," Ol":47101," On":40029," Oc":53053," Ra":100251," Qu":37258," Ro":163370," Re":185467," Ri":92437," Pr":164608," Pu":36045," Sy":29848," Sw":37042," Su":96395," St":289114," Ta":73883," Th":847432," Ti":40916," Te":98615," Tr":83585," US":33813," To":92734," Ru":52977," Sa":154637," Sh":111397," Si":87885," Sc":118544," Se":176317," So":160018," Sp":77680," Va":46297," Ve":37650," Vi":81481," Tu":35914," Un":220424," im":47481," in":2124350," is":1535535," it":174191," ki":30820," jo":31616," ju":31891," ha":241917," he":193790," gi":28626," gr":121630," go":63783," gu":29424," hi":208416," ho":110982," hu":32878," ne":121038," na":174050," mu":147201," mo":231837," on":505803," of":2275616," nu":46047," no":232717," le":140717," li":204448," la":217455," kn":129366," km":28301," me":218768," mi":119142," ma":335287," lo":197009," af":58077," ag":45613," ab":70854," ac":136746," ad":69502," am":31088," an":1904322," ap":69597," ai":40857," al":285645," au":57146," ar":316462," at":262226," as":400242," ba":223899," bi":52056," be":361253," bo":296458," bl":27386," by":488337," bu":118496," br":85339," ca":237167," es":49439," en":130005," em":28882," el":86812," fe":73621," fa":169421," ev":56247," ex":119951," fu":49639," fr":341564," fo":746598," fl":49512," fi":320292," ge":92358," ga":85124," cl":107340," co":825577," cr":113302," ce":122706," ch":161778," ci":78624," da":76510," cu":74858," do":65310," dr":48010," de":365522," di":285811," ed":47217," ea":97123," du":78958," ye":60017," ru":58222," sa":63365," se":417230," sc":118543," si":220940," sh":125764," sn":27188," sm":42001," sp":191590," so":270927," qu":28948," ra":127133," re":580460," ri":81201," ro":102378," pu":110299," pr":488102," s ":285424," ot":53672," ou":36365," ov":43302," op":81547," or":335005," ow":28576," pe":132888," pa":238807," pl":194868," po":256546," pi":40433," ph":50321," wa":774642," we":177231," wr":78668," wo":147411," wi":321826," wh":387560," va":54528," ve":51196," vo":40129," vi":105281," ty":26790," tw":69959," us":113343," up":41302," un":149473," ta":60356," sy":54778," st":354336," su":191136," tr":168050," to":791933," th":3629714," ti":77241," te":209958,"Feb":38336,"Eur":34468,"Eng":96944,"Ger":62713,"Geo":33733,"Gen":26192,"Fra":61982,"Fre":54323,"For":54660,"II ":31692,"His":27636,"Hig":26895,"He ":118997,"Her":26981,"Har":34188,"Gre":62525,"Gra":36710,"Int":46918,"Ind":83336,"In ":64014,"Hou":31497,"Arm":26565,"Apr":40277,"Ass":43311,"Aus":73786,"Aug":44651,"Bar":42165,"Afr":28749,"Ame":138898,"Cal":50030,"Car":57990,"Cat":29319,"Can":84273,"Ber":26471,"Bra":33017,"Bro":31697,"Bri":93113,"Dec":45892,"Chr":35874,"Chi":73091,"Cit":38778,"Cen":43272,"Cha":89919,"Cor":34743,"Com":89536,"Col":68792,"Con":74902,"Cou":101975,"Eas":34019,"Dis":41703,"Nat":69269,"New":118080,"Nov":49796,"Nor":97601,"Oct":42904,"Oly":28739,"Pla":26686,"Per":28472,"Pen":27579,"Par":98820,"Pro":69398,"Pri":36739,"Pre":41911,"Pol":34263,"Ita":30913,"Isl":36514,"It ":288844,"Jap":37112,"Jan":53046,"Joh":48960,"Jul":46542,"Jun":47295,"Kin":45793,"Lea":42060,"Lan":29809,"Lin":26493,"Lon":35381,"Man":49459,"Mar":140591,"May":50344,"Mon":37435,"Mic":30191,"Min":35884,"Mus":28824,"Wor":61067,"Wil":46697,"Wes":53739,"War":53697,"Wal":33670,"Yor":47584,"Str":31655,"Sta":170783,"Ste":27246,"She":47105,"Sha":26436,"Ser":32863,"Sep":45219,"Spa":35346,"Sou":76587,"Rus":29193,"Sco":37321,"Sch":55225,"San":42456,"Riv":36843,"Rep":36734,"Rom":34841,"Uni":206527,"The":741033,"Thi":55958,"Tra":32863,"bit":27600,"bil":34265,"bin":32338,"bly":27084,"ble":108674,"bli":153281,"boo":43455,"bor":211694,"bot":33326,"bou":74106,"be ":86424,"ban":103832,"bal":111539,"bac":27179,"bas":116354,"bee":53796,"bec":41647,"ber":372832,"bel":55314,"bes":39082,"bet":76601,"bia":36531,"ca ":71415,"car":71337,"cas":54417,"cat":224104,"can":218593,"cap":33144,"cad":26654,"cam":39022,"cal":268647,"ce ":489631,"bri":35233,"bro":46239,"bra":53964,"bre":38618,"bru":40872,"bur":52830,"bum":61942,"bui":38535,"but":82424,"bus":32037,"by ":517575,"am ":158006,"ake":79597,"aki":36034,"ajo":27606,"al ":1032287,"ail":109529,"ain":263108,"air":75684,"agu":57502,"ago":34773,"anu":71865,"any":116307,"ano":50033,"ann":83296,"ant":217929,"ans":132995,"ane":74984,"ang":153929,"ani":212965,"ank":54539,"ana":169898,"anc":203858,"and":1922995,"amm":33827,"amo":43526,"amp":91990,"ams":33255,"ami":135603,"ame":296400,"amb":32800,"ama":60146,"aly":26784,"alt":66067,"als":156873,"alo":44306,"all":466989,"ali":291050,"ale":104464,"ala":82861,"alb":63960,"an ":1345264,"aba":26164,"abe":27729,"abi":43800,"abl":95465,"abo":73249,"ae ":67161,"ad ":136134,"aft":64376,"aff":27842,"ai ":26417,"aga":52212,"age":211927,"ael":26991,"ado":32963,"adi":130468,"ade":130763,"ack":93180,"aci":48413,"ach":115898,"ace":139311,"acc":38863,"ada":60436,"act":176914,"azi":34919,"ays":38857,"aye":87163,"at ":514237,"arg":65842,"are":299717,"ard":199526,"arc":122863,"ara":130356,"aro":62443,"arn":35663,"arm":40703,"arl":119272,"ark":89799,"ari":222546,"arr":82306,"ars":76523,"art":349130,"ary":226746,"asi":41320,"ash":45659,"ase":201692,"aso":57197,"ask":29635,"ar ":287761,"apa":53686,"ape":43062,"aph":44106,"app":80291,"as ":1288188,"ava":37165,"aut":60210,"avi":60667,"ave":94995,"ay ":243951,"awa":44906,"ata":58365,"ast":279617,"ass":158911,"atr":35326,"ato":75056,"ate":773247,"ati":841381,"ath":123599,"aw ":33098,"att":81527,"atu":81250,"aus":27235,"jec":34085,"jor":28420,"itl":30516,"ito":44622,"itu":81601,"itt":81137,"its":101906,"ity":314774,"ism":33565,"isl":34570,"iso":38810,"iss":83951,"ist":561559,"ita":125707,"ite":286513,"ith":287933,"iti":312557,"ium":39769,"iva":48200,"ix ":27697,"ivi":104219,"ive":414501,"is ":1834908,"ion":1320795,"ior":32367,"iou":46819,"ipa":66227,"ir ":124191,"irs":127746,"isi":109243,"ish":348637,"ise":68787,"isc":59122,"ire":165519,"ird":33893,"irc":26906,"it ":178657,"ize":56700,"iza":46289,"kin":80796,"ker":49025,"ket":62341,"key":30606,"ke ":83682,"ks ":77920,"kno":134011,"ka ":41749,"ha ":26541,"ham":91101,"han":137177,"hai":28881,"hal":42170,"hav":58023,"har":141850,"has":120534,"hat":219394,"had":38461,"he ":3774627,"hel":72126,"hei":75673,"hed":104387,"hea":95197,"hey":47349,"hes":81158,"her":373103,"heo":27238,"hen":82591,"hem":53106,"hig":49277,"hie":27426,"hic":185273,"hip":92853,"hin":140164,"hil":96538,"his":233818,"hit":44255,"hir":71717,"hn ":39410,"ho ":150031,"go ":38352,"gle":57486,"gli":56142,"gn ":28550,"gla":57467,"gne":39182,"gs ":62491,"gov":34838,"gro":78380,"gra":130928,"gre":50553,"gui":30022,"gua":43805,"gue":71236,"gy ":58853,"gus":48388,"iam":54702,"ial":191889,"ian":404233,"iat":85021,"ic ":400287,"ibl":26901,"ibu":29254,"id ":86767,"ibe":47065,"ia ":367003,"iet":50527,"iel":55976,"ien":102314,"ier":62637,"ies":307456,"ied":65179,"ifo":38546,"iff":31829,"ife":37449,"ifi":68240,"ics":90231,"ict":157327,"icu":32368,"ico":35459,"ick":77376,"ici":172909,"ich":222330,"ice":143792,"ie ":67990,"ica":469571,"idi":26260,"ide":209151,"ida":78657,"il ":176048,"im ":36451,"iga":31696,"igh":217420,"igi":75927,"ign":93132,"imp":47812,"ime":121995,"imi":41657,"ip ":77179,"inc":218718,"ind":114260,"ina":201512,"inn":50515,"ino":47946,"int":230917,"ins":127823,"inf":38004,"ine":357110,"ing":1178957,"ini":149339,"inv":30106,"ike":31715,"ila":50860,"in ":2015240,"ilo":31568,"ill":253039,"ilm":70454,"ili":107918,"ild":57179,"ile":87652,"ima":76627,"io ":93766,"ilw":27998,"ily":104441,"ilt":33467,"how":39080,"hol":72031,"hom":45604,"hon":34484,"hos":48768,"hou":71727,"hoo":99559,"hor":109022,"hro":61086,"hre":44973,"hri":42093,"ht ":111042,"hy ":31885,"hum":60594,"hur":46747,"ffe":52374,"ffi":65666,"fes":54841,"fer":101678,"fea":31829,"fam":97344,"fac":45076,"ff ":28943,"fe ":33600,"ext":42471,"exa":34847,"ews":32501,"exp":39847,"exi":35172,"eta":52929,"ete":109151,"eti":91823,"eth":57374,"esp":36940,"est":416254,"ess":250577,"etr":52595,"ett":68152,"etw":81955,"ety":27383,"ew ":152274,"eve":202090,"evi":96051,"ex ":29419,"ey ":184413,"epe":34508,"er ":1640997,"epa":43995,"eor":48705,"eop":40243,"es ":1236398,"ept":72575,"epu":31828,"epr":49381,"erl":52014,"eri":370698,"erg":54150,"ere":274035,"erf":42571,"erc":40232,"era":262856,"erb":34102,"et ":183510,"equ":41108,"esi":112993,"esc":38776,"ese":189976,"ery":53481,"erv":128955,"err":75913,"ert":119791,"ers":454490,"ern":296552,"erm":132045,"ero":60052,"en ":515700,"ela":94480,"eld":88206,"ele":239396,"eli":77371,"ell":173610,"elo":78428,"els":41004,"ely":58392,"emb":232607,"ema":57740,"eme":113005,"emo":49872,"emi":71126,"emp":43662,"ene":128335,"eng":56090,"ena":53361,"end":141343,"enc":194004,"eno":33355,"enn":56995,"eni":62631,"enu":41299,"ens":118987,"ent":917089,"ege":50969,"egi":112559,"ek ":35390,"eir":69644,"ein":59892,"eig":51339,"el ":184102,"em ":65326,"gis":39618,"gin":123942,"gio":71590,"gic":30033,"gia":32230,"ght":158712,"gen":144495,"ger":93467,"ges":68825,"gh ":114066,"ged":27606,"gdo":27360,"ge ":289041,"gas":27663,"gar":46583,"gat":28511,"gam":47831,"gal":29045,"gan":93331,"fte":73646,"ful":30197,"ft ":41960,"fre":30903,"fri":38796,"fro":298934,"fou":123314,"for":682161,"foo":52751,"fol":40754,"fic":131579,"fie":53477,"fil":79707,"fin":56926,"fir":119031,"da ":86635,"de ":228034,"dal":30100,"dae":42405,"dat":50729,"dar":41780,"dan":30719,"day":44277,"cul":69156,"ctu":58584,"cts":42767,"ctr":37744,"cto":137353,"cti":262008,"cte":91935,"cy ":49469,"cus":28281,"cur":73843,"cla":60174,"cle":56340,"clu":89562,"clo":27081,"co ":51753,"coa":30142,"con":327550,"col":81553,"com":369257,"cor":122655,"cov":34759,"cot":39747,"cou":101684,"cs ":92018,"ct ":169935,"cre":89454,"cra":42776,"cri":68130,"cro":41753,"cco":29437,"cce":37436,"cea":26454,"ch ":441284,"cer":71180,"ces":153825,"cen":136988,"cem":49322,"cel":32556,"ced":67120,"cha":166560,"chu":28019,"cia":202890,"ck ":147449,"cie":145817,"che":138582,"chi":112580,"cho":124613,"chn":32667,"cil":38009,"cis":28214,"cit":63842,"cin":40251,"cip":75068,"cke":66684,"ed ":1971122,"ebr":59143,"eac":57373,"eag":49669,"ead":94691,"ean":92098,"eal":72020,"eam":67571,"ear":238611,"eas":225475,"eat":190659,"ea ":97410,"efo":27942,"efe":67003,"ega":48293,"eek":39188,"een":176170,"eed":37937,"eer":50775,"eet":39266,"edi":120330,"ede":54090,"edu":36419,"ech":57443,"eci":121874,"ece":81981,"eca":34537,"ee ":109231,"ecu":34646,"ect":304913,"eco":150097,"dy ":58601,"dur":62131,"don":63380,"dom":45245,"ds ":164811,"duc":125723,"dra":35355,"dre":38221,"dge":45767,"dic":54056,"dia":144125,"der":278249,"des":154773,"dev":51103,"dea":34588,"ded":141601,"def":26810,"del":51924,"den":154840,"dem":42344,"dep":50458,"dle":29935,"do ":34660,"div":36865,"din":170282,"dio":57931,"dir":47066,"dis":162456,"dit":74391,"die":65117,"dif":26218,"rga":62829,"ri ":41893,"rgi":33226,"rge":101036,"ret":62986,"res":357790,"rev":42186,"rfo":28887,"rds":61017,"rg ":35933,"rea":244476,"ree":171216,"ref":62077,"rec":149809,"red":206208,"reg":86193,"rem":55031,"ren":193932,"rel":146404,"rep":68957,"rdi":57659,"rde":80880,"re ":634483,"rch":164110,"rce":59020,"rd ":218199,"rap":56827,"rar":34987,"ras":43044,"rat":260279,"rai":83507,"rag":34608,"ran":265077,"ram":80300,"ral":264928,"rab":26255,"rad":103893,"rac":129420,"rpo":42976,"rs ":371607,"ros":58364,"rot":58999,"rom":337721,"ron":118148,"roo":26751,"rop":109260,"rou":191436,"rov":99028,"row":41881,"roa":55237,"rod":87645,"roc":66994,"rol":65046,"rof":62861,"rog":49055,"rnm":34849,"rna":118629,"rne":61259,"rni":62594,"ro ":47142,"rma":145311,"rme":117007,"rmi":32749,"rly":65353,"rli":51474,"rld":75350,"rle":27840,"rn ":330287,"rks":33396,"rke":46561,"rm ":73619,"rio":79616,"rit":248352,"ris":163763,"riv":53880,"rig":116494,"ril":65623,"rin":255678,"rim":52709,"ria":159780,"rib":60454,"ric":377068,"rid":60669,"rie":181610,"rk ":140894,"ruc":35769,"run":39647,"rum":30319,"rus":32542,"rva":27044,"rvi":62552,"rve":65133,"ry ":509562,"rsi":95891,"rso":35794,"rsh":28300,"rse":60300,"rta":48186,"rst":132558,"rtm":26912,"rte":66419,"rth":190521,"rti":127728,"rua":38894,"rts":59598,"rty":55015,"rt ":230530,"rro":33066,"rri":69478,"rre":97252,"rra":30166,"san":29214,"sha":36063,"sho":85008,"she":117943,"shi":166608,"sid":90354,"sic":111518,"sia":80711,"sit":152904,"sis":68592,"sin":187018,"sio":187210,"sim":26375,"sig":69827,"scr":41549,"se ":323378,"sci":38993,"sch":76599,"sco":58271,"sev":36211,"ser":204330,"ses":72706,"set":56749,"sh ":261718,"sea":111064,"sed":251715,"sec":65010,"sen":110589,"sem":35416,"sel":52566,"spo":60807,"spe":154402,"spi":31807,"spa":33041,"sou":97501,"sol":40004,"som":46889,"son":213206,"sor":49003,"soc":65715,"st ":788491,"ss ":186657,"sla":91489,"sm ":27910,"sna":26235,"so ":118506,"sma":61408,"sys":38243,"sse":101933,"ssa":44414,"sso":82323,"ssi":178639,"ste":308026,"sta":321451,"sto":159447,"sti":202749,"stl":26896,"stu":53381,"str":353511,"sts":54520,"sub":56163,"suc":48817,"sul":26601,"sup":33543,"sus":37393,"sur":52388,"tai":93008,"tak":29757,"tal":158887,"tag":31026,"tab":60969,"tba":73096,"tat":294451,"tar":154358,"tan":148320,"tch":55824,"te ":378459,"ta ":92521,"pe ":62967,"par":214065,"pat":33109,"pac":31182,"pal":68434,"pai":39322,"pan":124559,"phe":35230,"pho":33690,"phi":46768,"pea":63524,"pec":127586,"ped":38131,"pen":76105,"peo":27260,"per":254210,"pet":53680,"pla":213772,"pli":36447,"ple":105992,"plo":26352,"phy":39410,"pic":63629,"pin":46550,"pio":41668,"pit":34080,"por":147134,"pop":69835,"pos":94042,"pon":41677,"pol":103325,"ps ":57068,"ppo":34542,"ppe":56579,"pub":101611,"pte":60939,"pti":34138,"pri":140547,"pre":169814,"pro":351896,"pur":26601,"put":38175,"pul":69919,"qua":54197,"que":60004,"qui":38739,"ra ":108610,"ngi":45147,"ngl":153748,"ngu":51932,"ngt":37246,"ngs":65275,"ni ":33062,"nge":128219,"nga":40074,"ngd":28313,"nel":35766,"nen":30704,"ner":137621,"net":62476,"nes":167423,"ng ":1115424,"nea":48684,"ned":116572,"nee":30138,"nfo":27425,"ney":30714,"new":36703,"nct":31942,"nco":45249,"nci":91672,"ncl":65296,"nce":360792,"nch":88213,"ne ":474805,"ndu":39629,"ndr":33212,"nds":71457,"ndo":66887,"ndi":147835,"nde":264119,"nda":82618,"ncy":32567,"nal":361592,"nam":140852,"nan":41261,"nar":46225,"nad":72347,"nag":33449,"nai":32626,"nd ":1932876,"nat":199905,"na ":142173,"ny ":146188,"nve":33106,"num":41405,"nus":46108,"nua":61979,"nty":82414,"nto":84185,"ntu":58438,"nts":118194,"ntr":155494,"nti":190207,"nth":52495,"ntl":47080,"nta":166897,"nte":282517,"nsu":48967,"nst":113470,"nse":49755,"nsh":53144,"nsi":90541,"nt ":574346,"ns ":352877,"nol":27739,"nom":44734,"non":40510,"not":75828,"nor":109034,"now":162156,"nov":41537,"nne":95671,"nna":27532,"nni":61016,"nme":53495,"nly":54407,"no ":47600,"nic":132031,"nia":137970,"nk ":44974,"niz":43878,"niv":81629,"nis":146601,"nit":215552,"nio":55312,"nin":128702,"ogr":72732,"ogi":44598,"ohn":44056,"ogy":47192,"oin":47463,"ok ":58105,"ol ":109628,"oce":39082,"oci":84252,"ock":92930,"oca":158968,"occ":29313,"ode":77618,"of ":2204484,"odu":81510,"oft":39655,"off":67447,"ofe":46203,"oad":59707,"od ":90721,"obe":66424,"ows":30274,"own":272385,"owi":30580,"ow ":104160,"oti":37576,"oth":132550,"ote":72682,"ott":42106,"oto":39358,"ost":134462,"ota":47221,"otb":59122,"osi":37738,"ose":115458,"oss":51862,"owe":69263,"ovi":113638,"ove":271311,"oug":99629,"oul":30838,"oun":383251,"oup":67359,"ous":158478,"our":185741,"out":265981,"opo":44332,"opi":26584,"opl":40248,"ope":150764,"oph":34738,"os ":57513,"opu":63831,"ool":97648,"ook":83743,"ood":64494,"or ":897485,"oot":80611,"ork":137621,"orl":77957,"orm":213505,"orn":233282,"oro":42745,"orp":39756,"orr":27698,"orc":40804,"ord":187024,"ore":161271,"org":83794,"ori":179394,"ort":324283,"ors":65851,"ory":92826,"ot ":75528,"ora":99397,"ola":48347,"old":74258,"on ":1693252,"oli":168296,"oll":152650,"ole":57272,"olo":130918,"olu":51650,"om ":355568,"ona":295881,"ond":125458,"onc":37498,"onf":27843,"one":226758,"ong":180887,"oni":86395,"onl":50368,"onn":32769,"ono":57151,"ons":362489,"ont":151478,"ony":30391,"oma":89149,"ome":169344,"omb":33063,"omi":82283,"omm":164677,"omp":213665,"omo":39559,"op ":54660,"la ":85626,"le ":464484,"lf ":36467,"lde":36942,"ldi":28865,"lab":36068,"lac":84352,"lag":63397,"lai":31387,"lan":374156,"lar":145329,"lat":207187,"las":100146,"law":29496,"lay":143764,"ld ":226258,"lbu":63795,"ls ":147230,"lon":87604,"lop":54959,"lor":52442,"loc":148176,"log":104122,"los":46581,"low":85754,"lth":30060,"lti":36200,"lud":61069,"lub":38036,"lue":31016,"lso":110778,"lt ":61158,"li ":29468,"ley":50488,"lev":69397,"les":177287,"let":59346,"ler":75366,"lem":45662,"len":64181,"leg":67495,"led":96172,"lec":118736,"lea":145235,"lls":32993,"llu":31038,"lly":155319,"lo ":32182,"lla":120635,"lle":228951,"lli":124524,"llo":71927,"lm ":62508,"ll ":345769,"lit":221318,"lis":254417,"lin":196823,"liz":27437,"liv":42501,"lic":149255,"lia":177899,"lig":44454,"lie":64153,"lif":67751,"ma ":56941,"mb ":40338,"mai":57333,"mad":33441,"mag":32391,"mar":102764,"mas":36713,"mal":90910,"man":305273,"mat":146584,"mbl":26607,"mbi":34922,"mbe":250295,"me ":304088,"med":139063,"mea":33106,"met":85213,"mes":108882,"mer":308315,"mem":71604,"men":353373,"lve":34044,"lum":34502,"lus":42702,"ly ":633235,"lwa":32178,"lym":34483,"mpi":86987,"mpe":67319,"mpo":59756,"mpl":65250,"mpu":32461,"ms ":108315,"mod":36169,"mon":129400,"mol":27569,"mov":30527,"mor":62243,"mos":67786,"mot":45357,"mou":50940,"mpa":78503,"my ":44622,"mus":79882,"mul":30159,"mun":115137,"min":162859,"mil":151172,"mis":47781,"mit":48071,"mic":78126,"mmu":68704,"mmi":37447,"mmo":49689,"mma":30077,"mme":66787,"zed":29181,"zat":36341,"yst":59562,"ysi":32556,"ys ":46066,"ype":26271,"yea":55238,"yed":50493,"yer":52194,"ymp":36974,"yin":35501,"wo ":61016,"wn ":223496,"ws ":47661,"wri":85490,"wor":132758,"wes":66741,"wer":121358,"wel":48031,"wed":28630,"wee":76113,"whe":65586,"whi":161339,"who":156434,"wit":254528,"win":75777,"way":79562,"war":108109,"was":721522,"via":29199,"vil":95978,"vin":109107,"vic":63884,"vid":74216,"vie":42322,"vis":102225,"vol":39542,"ver":416461,"ves":68869,"ven":139602,"vem":57889,"vel":128051,"ved":73656,"ve ":294814,"val":55056,"van":50667,"var":47368,"vat":46841,"usi":127657,"use":190347,"ust":198208,"uss":43860,"uth":195687,"uti":76065,"ute":88005,"us ":269807,"ut ":151960,"ura":72791,"urc":49405,"ure":175159,"urg":39826,"uri":124329,"urn":72450,"uro":43771,"urr":69334,"urs":30553,"urt":42019,"ury":47606,"ur ":96286,"upp":28633,"umm":33986,"uma":31954,"umb":94738,"ume":46174,"uly":40527,"unt":202586,"uni":169488,"unc":71870,"und":253741,"ung":45523,"une":67239,"up ":99562,"um ":146010,"ult":82523,"ull":33469,"ule":32343,"ula":137681,"un ":32424,"uil":50329,"uis":33184,"uit":37221,"ul ":41234,"ugh":110128,"ugu":55721,"uct":69612,"ude":66964,"udi":73540,"uca":35169,"ue ":120838,"uce":56991,"uch":53880,"ues":42067,"uen":42045,"ub ":41295,"uat":42205,"uar":119614,"ual":88610,"ubl":124785,"uag":32210,"typ":31249,"ty ":506251,"tur":219279,"tut":36373,"tua":41791,"tud":63876,"two":81070,"twe":71315,"ts ":465295,"tre":115840,"tra":308513,"tri":221592,"tru":53183,"tro":123988,"try":66671,"tta":29674,"tte":122526,"tti":34349,"ttl":41524,"tme":35635,"to ":731436,"tly":63791,"tob":48945,"tow":72085,"tom":29936,"ton":150493,"tor":304450,"too":27044,"top":29122,"til":53505,"tie":69965,"tit":108778,"tis":139521,"tin":255051,"tim":81225,"tio":971575,"thu":55673,"tia":79923,"tic":276929,"tiv":181662,"tla":29767,"tle":95143,"tem":124758,"ten":154275,"tel":93554,"tee":36958,"tea":71475,"tec":49230,"ted":637757,"th ":648546,"tes":193644,"ter":809390,"ti ":33930,"tho":134583,"thr":87801,"the":3415279,"thi":110969,"tha":240340},"n_words":[260942223,308553243,224934017],"name":"en","type":"latin","flags":["ascii"]} \ No newline at end of file
diff --git a/contrib/languages-data/es.json b/contrib/languages-data/es.json
new file mode 100644
index 0000000..3c87107
--- /dev/null
+++ b/contrib/languages-data/es.json
@@ -0,0 +1 @@
+{"freq":{"D":116547,"E":296654,"F":128129,"G":121338,"A":269964,"B":167407,"C":324676,"L":239740,"M":232523,"N":101570,"O":64162,"H":81475,"I":139475,"J":79180,"K":34991,"U":62793,"T":132122,"W":35590,"V":93781,"Q":9803,"P":223906,"S":275410,"R":140692,"Y":15953,"X":27052,"Z":15636,"f":602083,"g":868874,"d":3877179,"e":8874725,"b":812441,"c":2912236,"a":7916083,"n":5177793,"o":5444424,"l":3848407,"m":1698678,"j":201978,"k":118503,"h":478078,"i":4816050,"w":47097,"v":521828,"u":2624688,"t":3108332,"s":4177405,"r":4307485,"q":288923,"p":1539196,"z":257162,"y":639511,"x":118459,"²":9268,"Ã":8848,"í":298098,"é":232623,"è":9587,"á":241717,"ú":87005,"ó":542725,"ñ":141698," l":925296," m":342591," n":169035," o":224265," h":153575," i":177440," j":57642," k":20882," d":2112466," e":1686012," f":346033," g":128207," a":615947," b":110799," c":841620," y":413167," z":8894," u":509709," t":263819," v":109427," q":191724," p":764411," s":537300," r":229895," J":77814," K":33034," H":77965," I":96403," N":91185," O":55452," L":232579," M":222195," B":158843," C":308303," A":248654," F":121663," G":115612," D":105557," E":280970," Z":14852," Y":15119," X":18791," S":256762," R":131715," Q":9140," P":211939," W":33301," V":79914," U":57665," T":122063," á":26258," é":11575," ú":11089," Ã":8823,"A ":15731,"Da":13474,"Cu":13670,"Cl":11053,"Co":80802,"Cr":14045,"Ce":15444,"Ch":44811,"Ci":15164,"Ed":7168,"Du":7509,"Do":14187,"De":29603,"Di":25181,"Fe":19702,"Fa":9899,"Eu":12005,"Es":99956,"En":28786,"El":96353,"Ge":16269,"Ga":21659,"I ":25126,"Fu":23434,"Fr":30636,"Fo":10911,"Fi":10813,"C ":12178,"Au":18315,"Ar":39333,"At":7482,"As":13866,"D ":7805,"Ba":43405,"Ac":8941,"Am":14049,"An":29222,"Ai":9057,"Al":51515,"Bu":16123,"Br":22960,"Ca":88236,"E ":7609,"Bi":10052,"Be":24042,"Bo":25983,"Le":25199,"Li":24708,"La":111930,"Lu":14073,"Lo":44117,"Me":30598,"Mi":26023,"Ma":85855,"Mu":18246,"Mo":36242,"Ni":10392,"Ne":11654,"Na":26109,"Nu":10474,"No":24130,"Gi":8606,"Gr":21868,"Go":14614,"Gu":21771,"Ha":19115,"He":16228,"II":18780,"Hi":11870,"Ho":13033,"Hu":9391,"In":34662,"Is":11404,"Ja":18733,"Je":8830,"Jo":22070,"Ju":22045,"Ka":8249,"Un":40236,"VI":7056,"Tu":7329,"Tr":17446,"To":19269,"Th":18313,"Ti":12341,"Te":21087,"Ta":17232,"V ":8314,"St":15377,"Su":36610,"Wi":9415,"Wa":8415,"Vi":28361,"X ":8775,"Va":20820,"Ve":16615,"Mé":10810,"Pu":11178,"Pr":36130,"S ":9671,"Pe":30523,"Pa":58336,"Pl":10964,"Po":27786,"Pi":20810,"Or":15403,"Se":51296,"Sc":7275,"Si":21213,"So":22342,"Ru":9143,"Sa":70712,"Re":46909,"Ri":14206,"Ro":31238,"Qu":8096,"Ra":15098,"b ":16974,"a ":2807777,"i ":67723,"cá":7737,"ge":97443,"ga":124206,"fl":18249,"fi":107495,"fr":77696,"fu":123473,"fo":79998,"có":10267,"he":77025,"ha":138548,"gn":24260,"cé":16005,"gl":50338,"gi":128766,"gh":9410,"gu":117756,"gr":97493,"cí":12742,"go":114696,"du":66523,"g ":31294,"ea":126973,"eb":49402,"ec":261065,"ed":147913,"de":2140534,"di":323631,"dm":7851,"do":556174,"dr":51266,"ex":62245,"eu":25398,"añ":100385,"ev":65967,"ey":26516,"ez":45865,"fa":71295,"h ":31038,"fe":72710,"eg":169863,"ef":34480,"ee":18956,"el":821140,"ej":32037,"ei":38528,"ep":114508,"eo":63152,"en":1446857,"em":183658,"et":133793,"es":1262459,"er":848036,"eq":18834,"aí":17948,"ca":518269,"e ":2816707,"bs":7367,"br":179247,"bu":47229,"bo":65362,"bl":117456,"bi":135239,"be":67612,"da":466302,"f ":15176,"cu":158454,"ct":134054,"cr":79888,"co":691321,"ck":21843,"cl":46854,"ci":780154,"ch":115875,"ce":230782,"cc":44533,"c ":23067,"az":30061,"ay":59587,"ba":136047,"d ":184141,"at":211115,"as":567489,"ar":691307,"aq":9531,"av":48388,"au":72787,"ak":11907,"al":659633,"ai":51423,"aj":50703,"ao":8102,"ap":77232,"am":297433,"an":805183,"ac":374463,"ad":631439,"ab":144055,"ag":85149,"ah":13118,"ae":43681,"af":21278,"nu":35537,"nt":779819,"ns":140899,"nq":11336,"no":368063,"nn":24107,"nz":31492,"ny":8769,"nv":18820,"oe":24400,"of":39013,"oc":180019,"od":93463,"oa":18159,"ob":123195,"om":334368,"on":689571,"ol":230551,"oi":24324,"oj":12159,"og":57063,"oh":10801,"m²":9191,"ot":93533,"os":699509,"ov":85328,"ou":52929,"op":77349,"oo":15677,"or":691238,"r ":505089,"ox":9374,"ow":10366,"oz":11222,"oy":19941,"lá":14567,"pe":253260,"pa":326975,"pl":67808,"lé":20848,"po":370364,"ph":12436,"pi":101456,"lo":395718,"lm":47465,"ll":193235,"ls":13390,"lp":10103,"lv":15446,"lu":77232,"lt":59369,"ly":9017,"o ":1816298,"ma":323604,"mb":142619,"me":329898,"mi":236057,"mm":8572,"ié":32355,"mp":132049,"mo":207044,"mu":126184,"ió":324878,"p ":13048,"na":683565,"nc":291584,"nd":277346,"ne":311456,"nf":32397,"ng":105777,"ni":309689,"nj":11352,"nk":8262,"nm":8232,"ju":57258,"fí":9213,"jo":44644,"ki":14501,"ke":14800,"ka":12118,"m ":52750,"gó":7060,"gí":14289,"gé":21878,"km":14607,"li":353994,"le":344098,"ld":22533,"lg":19778,"lf":10189,"la":1037960,"lc":15579,"lb":23157,"gú":7200,"n ":1645057,"hr":7436,"dí":25020,"ht":8163,"hu":24887,"hi":87814,"ho":64406,"dé":12258,"id":350652,"ic":527484,"ib":58759,"ia":458387,"ig":130847,"if":53944,"ie":349344,"k ":32633,"ir":119098,"is":363824,"it":317793,"iu":42028,"eñ":26834,"iv":114899,"eó":10608,"ij":12968,"ik":8874,"il":242265,"im":150543,"in":518205,"io":362002,"ip":90246,"je":41244,"iz":77284,"l ":1070460,"ja":41449,"xi":40771,"té":18357,"xp":12413,"tí":29392,"tó":61297,"xt":20556,"só":7077,"z ":55163,"tá":43968,"sé":9023,"sí":13754,"ró":22130,"y ":494983,"wa":13090,"ré":8377,"vi":184322,"rí":60498,"vo":65147,"uz":11349,"uy":29308,"ux":7768,"uv":14743,"rá":28876,"ve":131509,"va":112048,"x ":22885,"ui":101034,"uj":10219,"ul":130016,"ue":529325,"ug":37794,"ur":196055,"us":135539,"ut":75455,"um":62826,"un":688515,"uo":9672,"up":46545,"tu":178275,"tt":18266,"pó":8915,"ub":69875,"ua":157161,"ud":82055,"uc":86063,"w ":7730,"pú":12838,"to":492464,"tl":11756,"ts":11628,"tr":358077,"te":662982,"ti":404422,"th":33828,"tb":14576,"ta":636505,"su":163099,"ss":27514,"st":570360,"sl":23077,"sk":8256,"sm":35452,"sp":146933,"so":177522,"sd":21641,"sc":98040,"se":328771,"sh":16915,"si":316726,"rz":18465,"u ":97650,"nú":7450,"sa":193871,"rr":132506,"rs":76964,"rt":263306,"ru":82530,"rv":23971,"nó":12071,"ry":14114,"ní":12258,"rq":18301,"rp":16192,"ro":473902,"rn":71986,"né":9922,"rm":118903,"rl":28428,"rk":11610,"ri":592813,"rg":83834,"rf":13796,"ná":7351,"re":689451,"rd":86475,"rc":86488,"rb":27212,"ra":734384,"t ":94930,"mú":13777,"mó":11660,"qu":286748,"mí":7526,"mé":17124,"má":63634,"s ":1974557,"pt":30720,"pu":78951,"ló":19506,"lí":47975,"pr":228655,"ps":9053,"zó":8970,"zi":7657,"za":111482,"zu":8538,"zo":36191,"ye":23683,"ya":28514,"uí":10738,"yo":35768,"ué":9949,"² ":9260,"án":62336,"ál":20287,"áf":8040,"ác":15525,"ár":16431,"át":16282,"ás":49088,"á ":27321,"óg":11484,"ód":8472,"ór":13833,"ón":382883,"óm":11589,"ól":14360,"ó ":71778,"ña":56328,"ño":76416,"ín":33019,"ím":10418,"ío":16786,"ít":28434,"ís":34065,"íf":8702,"íc":18334,"íd":7650,"ía":105050,"í ":14487,"él":11392,"én":55163,"és":52145,"ét":12035,"ér":34634,"éx":11216,"éc":11450,"é ":16494,"ún":21741,"úl":7364,"út":8499,"ús":12464,"úb":12720,"ú ":7505,"一":7134," Ga":21582," Ge":16178," Fo":10836," Fu":23266," Fr":30592," Fi":10737," Ha":19064," He":16181," Go":14559," Gr":21740," Gu":21689," Gi":8540," Hu":9382," Ho":12969," Hi":11833," Je":8808," Ja":18701," Is":11343," In":34554," Ka":8187," Jo":22025," Ju":22012," La":111649," Le":25083," Li":24432," Ma":85482," Mi":25830," Me":30504," Lo":44008," Lu":14044," Ne":11540," Na":26019," Ni":10343," Mo":36161," Mu":18130," Am":14012," An":29155," Al":51409," Ai":9020," Ac":8911," Ba":43282," Au":18286," At":7457," As":13733," Ar":39211," Be":23973," Bi":9959," Bo":25824," Br":22869," Bu":16073," Ca":87586," Ce":15402," Ci":15105," Ch":44683," Cl":10942," Cr":13930," Co":80501," Cu":13492," Da":13336," Di":25027," De":29418," Do":13935," Du":7489," Ed":7143," El":96116," Es":98653," En":28605," Eu":11992," Fe":19655," Fa":9820," Wi":9336," Wa":8349," a ":151113," Or":15358," Po":27637," Pl":10897," Pi":20772," Pe":30452," Pa":58153," Nu":10454," No":24018," Ra":15024," Qu":7915," Ro":30970," Re":46794," Ri":14181," Pr":36030," Pu":11158," Mé":10807," Su":36537," St":14732," Ta":17163," Th":18237," Ti":12302," Te":20935," Tr":17351," To":19041," Ru":9128," Sa":70550," Si":21147," Sc":7137," Se":51093," So":22241," Va":20787," Ve":16562," Vi":28257," Tu":7181," Un":40136," ja":7912," im":16980," in":113728," is":9470," it":12073," ju":44169," ha":71347," he":17319," gr":47141," go":9321," gu":12156," id":8758," hi":30296," ho":20186," hu":9280," ni":9094," ne":10791," na":40693," mu":56375," mo":34821," oc":24698," of":15543," ob":20507," nu":10795," no":85354," le":29852," li":28752," la":626496," gé":18729," km":14107," me":61144," mi":48266," o ":71023," ma":85649," lu":16286," ll":24228," lo":186366," ag":19771," ab":21502," ac":45971," ad":18011," am":14490," an":45970," ap":24125," al":116261," au":26320," ar":40593," at":8359," as":24315," ba":52391," bi":11562," bo":13161," br":12518," ca":157850," e ":10258," er":14855," eq":8506," es":539820," en":614552," em":20859," ej":8137," el":384203," fe":21573," fa":54639," añ":28328," ex":38515," fu":116273," fr":61229," fo":35270," fl":10710," fi":33918," ge":20831," ga":13109," cl":15585," co":462664," cr":30029," ce":31950," ch":12983," ci":56484," da":13407," cu":60477," do":35749," de":1878742," di":146031," ed":14319," du":20922," té":10997," tí":7323," ru":8901," sa":21596," se":189308," si":104740," so":62800," qu":191524," mú":9473," ra":15504," re":170810," ro":18871," pu":38713," pr":177892," lí":9079," má":36795," ot":17424," or":52385," pe":108853," pa":137566," pl":26836," po":242173," pi":19178," y ":406137," va":23514," ve":32789," vo":9768," vi":36979," ub":13189," tu":8296," us":12351," ut":9609," un":466456," ta":42321," su":139495," tr":62181," to":30228," th":7488," ti":31564," te":60962," ár":9835," ál":11624,"Fer":7120,"Es ":20079,"Est":40499,"Esp":30688,"Eur":9266,"El ":89488,"En ":18934,"Gar":7699,"Fue":18788,"Fra":22586,"II ":12947,"Gue":7438,"Gra":11989,"Int":10608,"Ind":7667,"Arg":10516,"Bar":13123,"Ale":9873,"Alt":8713,"And":7070,"Ant":10595,"Cal":12289,"Cam":10802,"Cas":14393,"Car":17937,"Can":11389,"Ber":7187,"Chi":15815,"Cen":8021,"Cha":15858,"Cor":11229,"Com":17537,"Col":10868,"Con":24853,"Nac":11329,"Nue":8768,"Nor":14767,"Pla":8680,"Per":15205,"Par":20522,"Pro":17204,"Pri":7235,"Pre":7665,"Méx":8860,"Jos":9116,"Las":10322,"La ":78398,"Los":18518,"Med":8474,"Man":10902,"Mar":33709,"Mad":9566,"Mon":15534,"Mun":7799,"Su ":13104,"Sai":11675,"Sal":8578,"Se ":18407,"San":32494,"Rep":8544,"Val":13044,"Vil":8238,"Uni":27767,"The":12538,"bit":22599,"bio":8582,"bil":7159,"bo ":7079,"blo":8255,"ble":28350,"bli":27042,"bla":52392,"bol":20729,"bié":25416,"bor":11023,"be ":11171,"ban":26993,"bal":10177,"baj":17768,"bas":16282,"bar":17411,"ber":30080,"bia":11473,"bic":15037,"bie":16653,"ca ":145181,"car":45351,"cas":41067,"cat":14539,"can":94395,"cap":14546,"cac":16670,"cab":10529,"cad":51771,"cam":20670,"cal":48767,"ce ":35472,"bri":27454,"bro":15368,"bra":26651,"bre":105758,"bur":7145,"bum":12078,"am ":8792,"ajo":16624,"al ":274116,"aja":11561,"aje":17588,"ain":19507,"ais":7210,"agu":12491,"ago":28675,"anu":10190,"anz":18126,"ano":79509,"ann":7583,"ant":195066,"ans":17426,"ane":17755,"ang":19562,"ani":47254,"ana":70957,"anc":101262,"and":90623,"amo":12426,"amp":25988,"ami":57872,"ame":88516,"amb":41071,"ama":44395,"alu":9523,"alt":15595,"alo":13492,"alm":37560,"all":38280,"alg":10767,"ali":96797,"alc":7683,"ald":9255,"ale":82063,"ala":31798,"an ":104955,"aba":35637,"abe":14437,"abi":26160,"abl":18696,"abo":13547,"abr":22582,"ae ":28175,"aca":16009,"ad ":111062,"aga":11365,"ado":272062,"adr":17031,"adi":20049,"ade":31132,"adu":7585,"aco":10755,"aci":224093,"ach":10349,"ace":30766,"acc":10027,"ada":154737,"act":41870,"aza":10801,"ayo":25160,"aya":7832,"ba ":19878,"aqu":8935,"at ":7824,"arg":22636,"are":34407,"ard":31172,"arc":32672,"ara":95441,"aro":19229,"arn":7316,"arm":9270,"arl":13962,"ari":83070,"arq":11629,"arr":48459,"ars":9729,"art":119072,"asa":20743,"arz":11183,"asi":22274,"asc":10580,"ase":16265,"aso":13076,"ar ":100623,"apa":19232,"ape":7495,"api":11786,"apo":11441,"apr":7236,"as ":405615,"ava":9975,"aut":18769,"arí":13656,"avi":12074,"ave":14502,"ay ":13782,"ata":37974,"ast":48943,"atr":18116,"ato":31490,"ate":31650,"ati":40488,"atu":16694,"aun":7152,"aur":9494,"aus":7903,"jer":7889,"je ":13092,"jo ":28253,"ito":98982,"itu":57305,"eña":13429,"iud":31065,"ism":26830,"isl":10534,"iso":8290,"isp":12009,"ist":177840,"ita":90631,"ite":23119,"iti":14600,"ivo":27807,"eño":12432,"iva":29645,"ivi":28311,"ive":26878,"ipo":17187,"ipi":23278,"is ":44742,"ion":119701,"ior":15897,"ios":44527,"ipa":22022,"ir ":23370,"iri":18435,"isi":29089,"ise":10419,"isc":18799,"ire":21390,"ira":17066,"ja ":14889,"iza":56748,"km²":9037,"gía":13787,"gén":19973,"jul":9258,"jun":20993,"jue":11292,"ha ":23020,"ham":8200,"han":11281,"har":13249,"has":16766,"hab":28769,"hac":13453,"he ":23960,"her":16542,"hin":11792,"hil":16082,"his":17036,"ho ":17368,"go ":53446,"glo":15485,"gle":10695,"gla":7868,"gob":7093,"gni":7865,"gió":54574,"cés":10035,"glé":12339,"gon":9593,"gos":23034,"gru":17556,"gra":45297,"gri":10321,"gre":9359,"gui":14307,"gua":27766,"gue":27456,"gur":7400,"gun":20675,"iam":7087,"ial":56085,"ian":52472,"ias":29422,"iar":10235,"ic ":7674,"iac":10109,"iad":14476,"ibl":8505,"ibi":8164,"ibr":11003,"ibu":11957,"id ":12492,"ibe":11766,"ia ":260890,"iet":7563,"iel":8991,"iem":41853,"ien":137287,"ier":54165,"ies":19531,"ied":10903,"ieg":9534,"ifo":15425,"ife":9766,"ifi":21950,"ict":9719,"ico":124072,"ici":106469,"ich":18974,"ice":11245,"ie ":46477,"ica":217183,"ido":115195,"idi":11705,"ide":56950,"ida":143468,"il ":29082,"ige":12181,"iga":13218,"igl":16846,"igi":27180,"igu":25326,"igo":7750,"ign":13613,"imo":21293,"imp":19247,"ime":39882,"imi":27512,"inc":70215,"ind":21266,"ina":105516,"ino":52448,"int":60589,"ins":15850,"inf":13445,"ine":33717,"ing":45586,"ini":36645,"iod":9277,"inv":7546,"ila":17117,"in ":30692,"ilo":12791,"ill":61746,"ili":73439,"ile":24670,"ima":28669,"io ":149275,"día":18502,"hom":8228,"hos":7240,"hor":7815,"hum":8623,"fes":10458,"fer":25898,"fec":9155,"feb":8937,"fam":38352,"ext":18108,"ez ":27186,"exp":10760,"exi":15850,"eza":8196,"eta":35458,"ete":19089,"eti":13610,"esp":91539,"eso":20238,"est":182109,"aña":34948,"año":60064,"eto":13724,"etr":18301,"epú":7797,"eve":10842,"eva":20123,"evo":10163,"evi":21593,"erí":14511,"ey ":19031,"epe":9088,"er ":98611,"epa":47324,"eos":9539,"eor":7614,"eon":7647,"emá":15175,"es ":719197,"ept":16454,"epo":8010,"epr":9962,"elí":11498,"eri":85302,"erg":11010,"ere":38036,"erf":9607,"erc":31567,"erd":14302,"era":127455,"erb":7502,"et ":12616,"equ":18716,"aís":14636,"esi":46520,"esc":45995,"esd":20115,"ese":28957,"esa":79475,"erv":17764,"err":53481,"ert":62557,"ers":52410,"ern":44545,"erm":26731,"erp":8929,"ero":107826,"en ":609899,"ela":41071,"ele":43829,"eli":19783,"ell":40806,"elo":21079,"eo ":20655,"emb":41784,"ema":33787,"eme":18576,"emo":14924,"emi":17348,"emp":34242,"ene":94564,"eng":10179,"ena":41586,"end":51128,"enc":77537,"eno":37094,"eni":23612,"ens":46652,"ent":408861,"enz":9528,"ego":28111,"egi":66647,"egr":10013,"egu":24464,"ein":14332,"eja":7548,"el ":611266,"ejo":12599,"eje":7469,"gin":15815,"gio":12026,"gid":10945,"gic":10164,"gen":57105,"ger":8119,"ge ":11010,"gad":14992,"gas":7988,"gar":19699,"gal":9045,"gan":23906,"ga ":30329,"fue":90978,"fun":22112,"fra":50839,"fre":13616,"fri":7302,"for":59914,"flo":7804,"fic":57976,"fil":9492,"fin":19583,"da ":212873,"de ":1556339,"dad":141389,"dal":9998,"dae":14141,"das":36381,"dar":9501,"dan":12985,"dam":10048,"cul":40621,"cue":32934,"cua":33441,"ctu":33625,"ctr":9939,"cto":44819,"cti":20454,"cte":9828,"cta":10659,"cur":10605,"cuy":7974,"cla":14341,"clu":14576,"co ":135190,"ció":209489,"con":251381,"col":25189,"com":184452,"cor":25629,"cos":36158,"cre":24696,"cri":31616,"cro":8764,"cci":38750,"cea":16334,"ch ":11118,"cer":30048,"ces":71319,"cen":32729,"cep":7727,"cel":19017,"ced":7879,"cha":26138,"cia":152068,"ck ":13234,"cie":87706,"cid":70600,"che":19885,"chi":20475,"cho":21852,"cil":9620,"cim":9340,"cir":8507,"cis":9341,"cit":8942,"ciu":26750,"cin":17134,"cio":103441,"cip":45552,"ed ":10329,"ebr":19067,"eae":10702,"ead":15707,"eal":19761,"eas":12813,"eat":7412,"ea ":28070,"efi":9256,"efe":15631,"ega":22427,"edi":50600,"ede":34956,"eda":21258,"edo":13652,"edr":9154,"ech":18820,"eci":89707,"ece":32005,"ecc":12316,"eca":7852,"ecu":14868,"ect":45028,"eco":21596,"dur":19421,"dor":54124,"don":18997,"dou":11963,"dos":96466,"doc":7141,"duc":27199,"dri":13291,"dra":9547,"dre":11310,"dro":9953,"dic":47817,"did":11186,"dia":35235,"der":49978,"des":90830,"deb":8351,"dec":9801,"def":7698,"del":228490,"den":85908,"dem":11684,"dep":50829,"deo":9414,"do ":341916,"div":13995,"din":11969,"dio":35658,"dir":18800,"dis":80875,"dit":9088,"die":14888,"dif":14152,"rga":23318,"ri ":7113,"rge":24597,"rgo":16017,"ret":23611,"res":140547,"rev":10637,"rfi":8102,"rea":43224,"ref":13241,"rec":55408,"red":15261,"rei":7333,"reg":68764,"rem":14749,"ren":51706,"rel":19501,"rer":18242,"rep":14882,"rda":8172,"rdo":13679,"rdi":9872,"rde":29313,"re ":155218,"rci":19308,"rch":7367,"rce":15953,"rca":25800,"rd ":11954,"rar":15085,"ras":50104,"rat":31492,"rav":8075,"rag":12449,"ran":146320,"ram":21212,"ral":52901,"rab":19532,"raf":7329,"rad":60032,"rac":43442,"rs ":11514,"ros":47929,"rot":15483,"rom":15707,"ron":42744,"rop":32071,"rov":41024,"rod":21257,"roc":21679,"rol":20667,"rof":11386,"rog":9763,"rno":13811,"rna":28042,"rne":12661,"rni":8562,"ro ":146408,"rma":60914,"rme":24156,"rmi":20846,"rla":8699,"riz":10435,"rio":82353,"rit":87431,"ris":26175,"rig":32503,"ril":18939,"rin":44691,"rim":37251,"ria":74022,"rib":16779,"ric":53993,"rid":24912,"rie":40300,"rtí":7814,"ruc":9429,"rup":19339,"rus":10637,"rva":8468,"rvi":10259,"ry ":9879,"rsi":16295,"rso":23313,"rse":13455,"rta":69414,"rto":22364,"rte":83807,"rti":44079,"rtu":12987,"nía":7088,"rt ":11603,"rqu":18224,"rro":28597,"rri":26111,"rre":31796,"rra":36840,"sad":15576,"sal":12684,"san":10863,"sas":11791,"sar":21220,"sa ":86942,"rzo":11621,"sie":10373,"sid":35818,"sic":33044,"sia":21182,"sit":47886,"señ":8377,"sis":26321,"sin":18695,"sio":18884,"sil":10556,"sim":10529,"sig":27477,"scr":25119,"scu":15948,"sde":20004,"se ":142516,"sca":14799,"sco":24677,"ser":42063,"ses":15571,"seg":17308,"sec":10078,"sep":14018,"sen":28636,"sem":7053,"sel":10295,"spu":8777,"spo":14448,"spe":51710,"spa":61850,"sol":12812,"son":47400,"sor":10955,"sos":12831,"soc":12868,"sob":16402,"su ":57230,"st ":10396,"sla":17876,"smo":24196,"sió":31343,"so ":44831,"stá":22167,"ste":88396,"sta":179440,"sto":49757,"sti":69501,"stu":17380,"str":118933,"sub":9390,"sul":7853,"sup":14503,"sus":22170,"sur":23147,"tal":73680,"tag":7515,"tab":14559,"tac":23593,"tad":77862,"tat":7964,"tas":38619,"tar":41661,"tan":67218,"tam":79203,"te ":284941,"tbo":12533,"ta ":179252,"pa ":14872,"pe ":7492,"par":158326,"pas":11530,"pac":12615,"pal":23552,"pan":11657,"pañ":59522,"pec":57888,"pen":14739,"peo":7168,"per":107782,"paí":9997,"pes":10236,"pel":18436,"pla":29458,"pli":11132,"ple":15398,"plo":9048,"pic":11535,"pie":8675,"pin":12467,"pio":24607,"pit":12508,"por":173351,"pop":7479,"pos":33566,"pon":17239,"pol":27348,"pob":43702,"pod":7343,"lés":13215,"po ":39204,"pub":10168,"pti":14042,"lít":16259,"líc":10958,"lín":8248,"pri":52537,"pre":58569,"pro":108338,"put":7419,"pul":11629,"pue":30894,"mát":8971,"más":38263,"mán":10709,"mér":9468,"que":227040,"qui":47867,"quí":8097,"ra ":231606,"mús":8775,"ncé":9514,"ngo":7965,"ngl":22080,"ngu":15031,"ni ":7789,"nge":13697,"nga":8189,"nen":11810,"neo":11720,"ner":65372,"net":8127,"nes":80792,"ng ":18654,"nea":12399,"nec":28654,"nfo":10116,"nez":7381,"nco":17791,"nci":146898,"ncl":10729,"nce":61430,"nch":9730,"nca":12577,"ne ":57285,"ndu":7292,"ndr":12605,"ndo":65513,"ndi":38911,"nde":54981,"nda":66093,"ncu":16386,"nal":70953,"nam":8218,"nan":12157,"nar":27443,"nac":46410,"nad":35305,"naj":8529,"nd ":14500,"nat":21175,"nas":35377,"na ":398062,"ión":292845,"ntó":37020,"nve":10579,"nue":9759,"nto":136463,"ntu":7064,"ntr":93850,"nti":67583,"nta":102723,"nte":292971,"nso":9665,"nst":30664,"nse":26710,"nsi":26727,"nsa":11053,"nt ":24387,"nqu":11277,"ns ":13747,"noc":38585,"nom":49236,"nos":42446,"nor":29422,"nov":19327,"nne":10554,"no ":158177,"nif":10693,"nie":9958,"nid":51044,"nic":69869,"nia":31660,"niz":14512,"niv":15892,"nis":28903,"nit":8537,"nio":20490,"nim":13518,"ogr":18966,"ogo":8779,"ol ":38964,"oce":17267,"och":8312,"oci":56829,"ock":8945,"oco":9047,"oca":41229,"ode":14611,"odi":11580,"odo":26635,"ocu":9100,"oct":12604,"of ":7638,"oda":9018,"oes":12161,"odu":18410,"ofi":7512,"ofe":9434,"oba":8003,"obr":25648,"obl":50514,"obi":12794,"obe":8533,"nza":20160,"ote":14228,"otr":17397,"oto":14558,"ost":31662,"ota":19189,"osi":18739,"ose":15508,"oso":14720,"ovi":57483,"orí":9114,"ove":16922,"oun":14708,"our":10655,"opo":9237,"opi":14838,"ope":13312,"opa":10564,"os ":567590,"opu":9913,"olí":19344,"or ":251715,"orm":64912,"orn":13575,"oro":14041,"orr":19323,"ord":30127,"ore":48358,"org":21075,"ori":66357,"osa":21383,"ort":65622,"m² ":9185,"ora":46259,"ola":30987,"on ":190943,"oli":25682,"oll":17825,"ole":17479,"olo":38775,"olu":11781,"ogí":11606,"ona":104550,"ond":38242,"onc":23216,"onf":9258,"one":74442,"ong":11339,"onj":7098,"oni":32298,"ono":52521,"ons":50424,"ont":59611,"onv":7285,"oma":33534,"ome":21150,"omb":45255,"omi":25678,"omp":41070,"omo":82407,"omu":58045,"la ":610751,"gún":7107,"le ":58880,"lab":11220,"lac":64631,"lad":24486,"lag":7484,"lan":69592,"lam":24610,"lar":41436,"lat":24612,"las":124120,"lbu":12473,"lon":23409,"lom":10655,"lor":23128,"loc":26941,"log":23375,"los":171022,"lme":34946,"lti":12566,"lto":11513,"ltu":9464,"luc":12530,"lug":7366,"lta":15082,"lgu":7617,"lev":15617,"les":91476,"let":11653,"ler":15144,"lem":26257,"len":30658,"leg":14703,"lec":26734,"lea":8424,"lo ":93016,"lla":74669,"lle":47097,"lli":13816,"llo":31751,"ll ":10675,"lit":23261,"lis":29055,"lio":17662,"lin":23082,"lim":8340,"liz":32987,"lic":48449,"lid":38365,"lia":73957,"lib":11554,"lig":10555,"lie":8152,"ma ":72422,"mac":13622,"mad":34215,"mar":42452,"mas":23691,"mal":12192,"man":63359,"may":20759,"mat":18421,"mba":10176,"mbi":42599,"mbr":71287,"me ":12825,"med":24892,"met":16848,"mes":19527,"mer":60276,"men":167009,"mex":7078,"lva":8274,"lus":8634,"mpi":8494,"mpe":21376,"mpr":14573,"mpo":32948,"mpl":23586,"mpu":10144,"mod":10208,"mon":23296,"mor":11861,"mos":17488,"mpa":16447,"ió ":27428,"mus":8143,"mun":83842,"min":50965,"mil":54218,"mis":20947,"mit":22265,"mic":21219,"mie":39274,"mo ":116388,"ién":27517,"zad":34608,"zac":11303,"zan":7115,"zar":10021,"zon":9132,"zo ":17792,"yor":11730,"za ":34624,"ya ":16470,"yo ":17934,"ués":7665,"tín":7356,"tér":8916,"tón":43482,"tán":19529,"xim":7461,"xic":17547,"tá ":17299,"sí ":7618,"rís":7245,"río":10310,"ría":28707,"rón":9102,"via":12659,"vil":17697,"vin":37424,"vic":9152,"vid":23929,"vie":21693,"viv":7676,"vis":25393,"vo ":33305,"vol":10345,"vos":8624,"ver":42481,"ves":9401,"ven":28441,"vel":14981,"ve ":15016,"val":14785,"van":9128,"var":20642,"vas":9770,"vad":11386,"va ":31522,"uye":9933,"uy ":8325,"usi":15007,"use":7793,"usa":13025,"ust":26126,"uso":9699,"uti":16202,"uta":12332,"uto":18658,"us ":49172,"ura":67622,"ure":8908,"urg":11824,"uri":19845,"uro":19849,"ur ":25587,"upe":15682,"upo":17158,"uma":10015,"ume":13099,"unt":24294,"uni":64830,"uno":27366,"unc":10467,"und":48857,"una":267868,"une":8988,"um ":20862,"ult":21405,"ulo":17762,"uli":17880,"ula":47246,"un ":217532,"uid":13351,"uie":12890,"uil":7952,"uin":8672,"uip":8207,"uis":11000,"ueñ":8502,"uit":13101,"uga":15579,"ugu":12215,"uct":12213,"uda":37565,"udi":17314,"ubr":12417,"uca":7683,"ue ":294774,"uce":7415,"ucc":8218,"uci":25554,"uch":12624,"uev":15569,"uer":46622,"ues":36069,"ueg":19032,"ued":16010,"ueb":8677,"uen":38522,"uel":28446,"púb":11593,"ua ":14731,"uat":7912,"uar":13380,"ual":41530,"uan":21270,"ubi":18564,"ubl":12045,"ud ":8479,"uad":38503,"tur":45480,"tul":9339,"tub":10280,"tua":54339,"tud":17240,"tug":8561,"tre":58815,"tra":116327,"tri":81313,"tru":16821,"tro":72914,"to ":261732,"tod":17241,"tos":55424,"tom":10133,"ton":20463,"tor":94895,"til":28629,"tie":38174,"tig":17405,"tir":9162,"tit":18235,"tis":12154,"tin":45124,"tim":14112,"tip":10749,"tio":15498,"tia":11282,"tic":87300,"tid":24642,"tiv":46603,"tem":32370,"ten":66323,"tel":23707,"teg":9394,"tea":9283,"tec":13760,"th ":7178,"tes":66100,"ter":127000,"the":10645,"éti":7504,"éxi":11167,"éne":20764,"én ":28809,"éri":13573,"érm":8428,"és ":45633,"áti":14202,"án ":27367,"álb":11283,"áni":16413,"ás ":40719,"úsi":8717,"útb":7936,"úbl":12344,"ún ":11275,"óni":12420,"ólo":7805,"ón ":361830,"ógi":7057,"ño ":26522,"ños":18182,"ñol":28121,"ña ":43408,"íti":20855,"íst":9984,"íne":7430,"ín ":16078,"ío ":10995,"ís ":12371,"ícu":14416,"ías":8590,"ía ":90214},"n_words":[70286890,82926999,60413548],"name":"es","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/et.json b/contrib/languages-data/et.json
new file mode 100644
index 0000000..533fe04
--- /dev/null
+++ b/contrib/languages-data/et.json
@@ -0,0 +1 @@
+{"freq":{"D":3604,"E":9114,"F":3428,"G":4270,"A":13995,"B":6049,"C":4433,"L":9649,"M":10132,"N":6533,"O":2986,"H":6531,"I":5403,"J":4779,"K":13384,"U":2720,"T":11888,"W":2107,"V":9682,"Q":266,"P":11422,"S":15988,"R":6826,"Y":596,"X":461,"Z":653,"f":12728,"g":77107,"d":141364,"e":425996,"b":40246,"c":10761,"a":496421,"n":259681,"o":223900,"l":256122,"m":147242,"j":63354,"k":181311,"h":68975,"i":428108,"w":2663,"v":88215,"u":213745,"t":258531,"s":344443,"r":180124,"q":542,"p":61630,"z":3134,"y":5816,"x":1328,"²":200,"Ã…":72,"Ä":128,"Ã":51,"Ãœ":1682,"ß":76,"Õ":259,"Ö":167,"í":335,"ì":56,"ë":66,"é":703,"è":77,"ç":89,"æ":123,"Ã¥":190,"ä":42375,"ã":61,"á":537,"à":86,"ü":30512,"ú":88,"ø":326,"ö":8126,"ô":78,"õ":40539,"ó":356,"ð":63,"ñ":47,"Ä“":125,"Ä—":44,"Ä€":44,"Ä":654,"Ä":90,"ı":50,"Ä«":336,"ÅŸ":74,"Å„":54,"Å‚":96,"Å":306,"Ž":57,"ž":1084,"Å ":453,"Å¡":2423,"Å«":213,"ÇŽ":46,"É™":85,"Ì":52,"μ":94,"ν":197,"ο":232,"ι":145,"κ":95,"λ":130,"δ":59,"ε":108,"η":90,"α":274,"γ":53,"ά":69,"ί":80,"ω":53,"ÏŒ":79,"σ":105,"Ï‚":232,"Ï":156,"Ï€":59,"Ï…":62,"Ï„":124," l":23928,"ÑŒ":570," m":38337,"Ñ":66,"ÑŽ":76," n":13986," o":51528,"Ñ":590," h":8823,"ш":180," i":10696," j":29617," k":61170,"Ñ‹":415,"Ñ„":166," d":4378," e":23050,"Ñ…":292,"ц":201," f":3289," g":1891,"ч":785,"Ñ€":2377," a":29616,"Ñ":1744,"Ñ‚":1384," b":2333," c":602,"у":739," y":132," x":127," z":161," u":3277," t":29638," w":134," v":33321,"Ñ–":130,"Ñ‘":86," q":53," p":29550," s":33963," r":13644,"И":192,"Л":141,"К":318,"Ð":192,"Ðœ":293,"П":266,"О":115,"Б":203,"Ð":448,"Г":287,"Ð’":339,"Е":66,"Д":138,"З":71," J":4764," K":12980," H":6494,"Ш":75," I":5378,"Ю":47," N":6511,"Я":49," O":2964," L":9614," M":10069,"Э":79," B":6003,"Т":143," C":4377,"У":66,"Р":154," A":13953,"С":395," F":3376," G":4219,"Ч":60,"Ф":101," D":3559," E":9045,"Ð¥":61,"л":1746," Z":652,"к":1535," Y":594,"й":835," X":432,"и":3293,"п":426,"о":3362,"н":2533,"м":786,"г":616," S":15877," R":6777,"в":1990," Q":264,"б":447," P":11356,"а":3415," W":2074,"з":300," V":9645,"ж":145," U":2711,"е":2779," T":11830,"д":957," ä":1217," õ":2318," ö":381," ü":7159," Ä€":44," Ã":51," Ã…":72," Ä":128," Ö":164," Õ":255," Ãœ":1674," ž":64," Ž":57," Å ":453," Å¡":125,"Õ¶":64,"Õ¡":118,"ו":53,"×™":74,"ר":53,"Ùˆ":125,"ÙŠ":278,"Ù":48,"Ù„":301,"Ù…":185,"Ù†":161,"Ù‡":60,"د":159,"Ø­":106,"ت":46,"ب":196,"Ø©":66,"ا":449,"ع":100,"Ø´":48,"س":100,"ر":186," Ð":444," Б":203," Ð’":336," Г":277," Д":137," Е":66," З":71," И":192," К":315," Л":139," Ðœ":291," Ð":191," О":113,"A ":1491," П":265,"F ":209,"Da":636,"Cu":143,"Cl":241,"Co":1022,"Cr":170,"Ce":161,"Ch":852,"Ci":177,"G ":207,"Ec":84,"Ed":303,"Ea":53,"Du":151,"Do":581,"Dr":153,"De":809,"Di":576,"Fe":322,"H ":271,"Fa":384,"Eu":1055,"Ev":105,"Ex":60,"Er":534,"Et":156,"Es":507,"En":458,"Em":268,"Ep":67,"Ei":131,"El":689,"Ek":108,"Ee":3423,"Eh":79,"Eg":183,"Ge":880,"Ga":592,"I ":1170,"Fu":158,"Fr":833,"Fo":411,"Fl":194,"Fj":45,"Fi":490," б":52," г":58," в":53," д":53," и":51," к":97," н":58," м":69," п":140,"B ":254," о":107," Р":152," С":394," Т":143," У":64," Ф":99," Ð¥":61," Ч":60," Ш":75," Э":77," Ю":47," Я":48,"C ":532," Ñ":79,"Av":172,"Au":837,"Ar":1361,"At":438,"As":1550,"D ":280,"Ba":1524,"Az":50,"Ae":128,"Af":163,"Ag":213,"Ah":170,"Aa":1076,"Ab":517,"Ac":135,"Ad":407,"Am":1404,"An":1589,"Ap":221,"Ai":278,"Aj":171,"Ak":327,"Al":2030,"Bu":590,"Br":896,"Ca":1008,"E ":421,"Bi":534,"Be":1094,"Bo":722,"Bl":189,"Kv":67,"Ku":1642,"Gö":61,"Ky":76,"Kn":47,"Kl":321,"Kr":1260,"Ko":2411,"Le":1403,"Hä":127,"Li":2750,"N ":331,"Gü":52,"La":1509,"Lu":547,"Ly":58,"Hõ":54,"Lo":1151,"Me":1361,"Dž":49,"Mi":1243,"O ":457,"Ma":3890,"Hü":108,"Mc":62,"My":97,"Mu":881,"Mo":1371,"Jä":515,"Ni":728,"Ne":1120,"Na":1009,"P ":412,"Ny":45,"Jõ":269,"Nu":172,"No":1468,"Ok":125,"Ol":339,"Om":123,"On":147,"Oh":110,"Kä":196,"Oi":46,"Od":88,"Of":51,"Jü":111,"Ob":154,"Gi":310,"Gl":141,"Gr":855,"Go":370,"Gu":494,"Gy":49,"J ":73,"Ha":1920,"He":1336,"Hi":1244,"Ho":963,"Hu":297,"Hy":59,"K ":351,"Dü":69,"Id":463,"Ig":87,"Io":55,"Im":143,"In":1490,"Il":192,"Ii":222,"Iv":146,"Is":449,"It":383,"Ir":247,"Ja":1273,"L ":301,"Ji":117,"Je":477,"Jo":991,"Ju":835,"Fü":63,"Ka":3521,"Fö":134,"M ":228,"Kh":87,"Ki":1209,"Ke":1467,"Us":124,"Ut":61,"Ur":136,"Um":60,"Un":420,"Uk":201,"Ul":110,"Ud":58,"Pü":264,"W ":104,"Pö":45,"Ty":76,"Põ":1331,"Tv":68,"Tu":682,"Tr":728,"Ts":169,"To":953,"Pä":660,"Th":747,"Ti":538,"Te":1883,"Ta":4354,"V ":865,"Sy":110,"St":1249,"Sv":158,"Su":1472,"Wo":216,"Wi":599,"Wh":48,"Sä":63,"Rü":62,"Wa":532,"We":428,"Rõ":64,"Vo":452,"Vu":47,"Rä":65,"Vi":1892,"Vl":115,"X ":185,"Va":3312,"Ve":2352,"Uu":347,"Mä":340,"Lü":129,"Pt":77,"Pu":606," Ù…":56,"Pr":1526,"Ps":58,"S ":641,"Lõ":671,"Py":58,"Pe":1263,"Pa":2287,"Kü":279,"Pl":289,"Po":1247," ع":51,"Pi":1126,"Ph":224,"Lä":1080,"Os":432,"Ot":174,"Ou":77," ا":173,"Oo":46,"Op":127,"Or":500,"R ":183," ب":58,"Kõ":416,"Kö":81,"Se":3048,"Sc":514,"Si":1380,"Sh":515,"Sm":85,"Sl":182,"Sk":219,"Sp":264,"So":1693,"Ru":426,"Nõ":704,"U ":154,"Sa":3549,"Nü":46,"Re":985,"Ri":1129,"Nä":146,"Rh":72,"Ro":1769,"Qu":132,"Mõ":229,"T ":213,"Ra":1769,"Mü":121,"TÅ¡":407,"b ":14100,"a ":101956,"Tõ":185,"Tö":129,"Tü":236,"Ya":114,"Yo":251,"Yu":48,"Z ":57,"Sõ":312,"Sö":54,"Sü":198,"Xi":96,"Tä":315,"TÅ":50,"Za":127,"Ze":122,"Zh":62,"Vä":634,"Zi":61,"Zo":49,"Zu":94,"Võ":414,"bö":50,"i ":67391,"gd":177,"ge":11091,"ga":17189,"bü":159,"fj":156,"fl":261,"fg":90,"ff":336,"fi":3834,"bä":233,"fs":80,"fr":799,"fu":505,"ft":358,"fo":2004,"j ":178,"gy":115,"he":15992,"ha":9696,"gn":1067,"gm":312,"gl":3003,"gk":631,"gj":169,"gi":15351,"gh":514,"gg":188,"gv":123,"gu":12657,"gt":248,"gs":906,"gr":3204,"gp":765,"go":1593,"dt":600,"du":15412,"dv":357,"dw":178,"dy":147,"dz":76,"g ":7339,"ea":10628,"eb":4711,"ec":855,"ed":12410,"de":25494,"dd":200,"dg":178,"di":20365,"dh":181,"dk":488,"dj":432,"eK":296,"dm":1746,"dl":1518,"do":2346,"dn":649,"dp":77,"ds":1055,"dr":3163,"ew":508,"ex":366,"eu":1294,"ev":10732,"ey":521,"aõ":117,"ez":242,"fa":820,"aü":201,"h ":1684,"fe":1177,"eh":9506,"eg":11045,"ef":1060,"ee":30805,"el":44868,"ek":15889,"ej":1876,"ei":11617,"ep":3591,"eo":4613,"en":30509,"em":17859,"et":21350,"es":49834,"er":30979,"eq":50,"ca":1187,"e ":97234,"by":175,"bs":372,"br":3541,"bu":2050,"bt":50,"bn":97,"bo":1301,"bj":462,"bk":63,"bl":1788,"bh":62,"bi":4858,"bb":142,"bd":109,"be":5461,"db":83,"da":21215,"f ":1128,"cz":59,"cy":123,"cu":367,"ct":474,"cs":86,"cr":248,"co":978,"cm":60,"ck":1210,"cl":163,"ci":834,"ch":3094,"ce":1021,"cc":199,"c ":475,"az":442,"ay":578,"ba":4886,"d ":44215,"at":32525,"as":52565,"ar":30297,"aq":50,"ax":179,"aw":265,"av":11952,"au":7185,"ak":18860,"al":53791,"ai":13608,"aj":8772,"ao":2265,"ap":5317,"am":18369,"an":38127,"ac":1403,"ad":21341,"aa":42535,"ab":9670,"ag":6434,"ah":11107,"ae":3739,"af":1986,"nu":11347,"nt":12596,"ns":4828,"nr":469,"np":151,"no":4692,"nn":17075,"q ":46,"nz":355,"jö":51,"ny":360,"jõ":2814,"nw":57,"nv":314,"oe":1975,"of":1882,"oc":974,"od":6416,"oa":902,"ob":2993,"om":10938,"on":61110,"ok":4533,"ol":25486,"oi":2595,"oj":1236,"og":7246,"kä":1444,"oh":4211,"ot":7307,"hÅ":45,"m²":200,"os":13528,"ov":3905,"ou":1255,"op":3770,"oo":36377,"or":18504,"r ":11002,"ox":103,"ow":446,"kö":136,"oz":161,"oy":237,"kõ":3255,"pe":8582,"pf":53,"kü":2678,"pa":9082,"pl":1823,"pm":182,"lé":50,"pn":209,"po":8371,"lä":2964,"ph":615,"pi":8736,"pk":178,"lo":11129,"ln":1607,"lm":6318,"ll":19635,"ls":3471,"lr":311,"lp":708,"lv":1889,"lu":12030,"lt":11142,"lz":55,"ly":347,"hõ":638,"o ":5439,"hü":580,"ma":35693,"mb":5069,"mg":52,"dž":411,"mh":80,"me":23829,"mf":109,"hÄ":79,"mk":376,"ml":264,"mi":40706,"eÅ¡":58,"mj":183,"mn":780,"mm":2852,"mp":2268,"mo":4294,"mr":80,"mt":306,"ms":1112,"mv":97,"mu":8559,"iõ":260,"my":155,"p ":1639,"na":27821,"iü":173,"nb":439,"nc":950,"nd":30717,"ne":35835,"nf":748,"ež":190,"ng":16763,"nh":382,"jä":4035,"ni":36973,"nj":320,"nk":2416,"nl":419,"nm":235,"jt":102,"ju":8195,"jn":184,"jo":2230,"jm":158,"kj":201,"ki":13463,"kh":323,"kf":60,"ke":20124,"kd":44,"kb":60,"fü":589,"ka":27697,"m ":10376,"fö":145,"ky":200,"gõ":66,"gö":54,"ks":20790,"kt":6626,"ku":25416,"kv":1230,"ko":25420,"kp":298,"kr":3838,"kk":4692,"kl":3700,"km":2283,"kn":633,"li":57000,"hä":827,"lh":474,"lk":2832,"lj":4573,"le":37295,"ld":8105,"lg":4950,"lf":372,"gü":68,"la":31670,"lc":95,"lb":823,"n ":49802,"hr":621,"hs":195,"hv":2638,"hw":103,"ht":6604,"hu":4453,"hj":2882,"hk":5994,"hh":1052,"hi":9059,"hn":1001,"ho":2898,"hl":335,"hm":902,"dé":47,"id":25348,"ic":2524,"ib":3313,"dü":272,"ia":15748,"ih":2638,"ig":11090,"if":1034,"ie":3981,"hy":186,"dõ":82,"k ":16382,"iq":93,"ir":11061,"is":67525,"it":26582,"iu":2821,"iv":6458,"iw":71,"ix":158,"ii":25093,"aÅ¡":218,"ij":1535,"ik":35922,"il":29934,"im":19741,"in":53773,"io":9587,"ip":3170,"je":2855,"až":108,"ji":256,"fä":213,"iz":330,"iy":72,"eõ":59,"l ":31660,"eü":118,"ja":41991,"tä":3641,"xi":267,"té":49,"xt":76,"sö":241,"sõ":3307,"ww":59,"z ":542,"sü":7550,"xa":160,"xe":69,"sä":382,"oÅ¡":66,"wi":436,"sé":45,"wn":69,"wo":143,"ws":111,"vv":89,"rõ":329,"vy":59,"rö":278,"y ":2468,"rø":55,"wa":716,"rü":1335,"rÄ":49,"we":364,"vl":251,"vm":177,"ré":93,"vj":46,"vk":322,"rä":668,"vi":9510,"vg":104,"vt":88,"vu":4481,"vr":127,"vs":891,"vp":221,"rí":55,"vn":468,"vo":2748,"uz":144,"uy":48,"ux":121,"uv":4464,"uu":16971,"ve":10196,"vd":60,"va":34297,"x ":556,"ui":3667,"uj":2076,"uk":5480,"ul":16978,"ue":1272,"uf":259,"ug":4237,"uh":3669,"ur":13993,"us":50648,"ut":13872,"um":10739,"un":13735,"uo":494,"up":1948,"ty":350,"põ":3410,"tz":299,"pö":275,"tu":27878,"tt":4340,"tw":94,"tv":618,"ub":5783,"pü":734,"ua":3006,"ud":21571,"uc":397,"w ":514,"to":11780,"tn":1099,"tm":2505,"tl":5476,"ts":21273,"tr":8177,"tp":259,"tg":91,"tf":183,"te":48864,"tk":949,"tj":657,"ti":30264,"pä":3196,"th":1862,"v ":7055,"tb":129,"tc":91,"ta":41990,"su":20276,"sv":2315,"ss":8143,"st":67054,"sy":172,"sz":75,"sw":139,"sl":2942,"sk":10212,"sn":1521,"sm":4289,"sp":2885,"so":5912,"sr":794,"sd":174,"sc":930,"sf":485,"se":62785,"sh":1322,"sg":140,"sj":1063,"kÅ¡":54,"si":30745,"rz":159,"u ":17617,"sa":24031,"nü":262,"sb":374,"rr":3973,"rs":3109,"rt":7363,"ru":7114,"rv":5729,"rw":105,"ry":602,"nõ":791,"rp":648,"ro":11751,"rn":3712,"rm":4085,"né":46,"rl":1539,"rk":3920,"nç":57,"rj":4639,"ri":41739,"rh":803,"nä":2052,"iž":51,"rg":9073,"mÄ":44,"rf":347,"re":19761,"rd":6093,"rc":505,"rb":1857,"mü":731,"ra":27209,"t ":40235,"mõ":3716,"mö":208,"qu":338,"iÅ¡":133,"lÄ":51,"mä":4794,"lü":1913,"s ":83808,"px":82,"lõ":2040,"py":45,"lö":179,"pt":1489,"pu":3682,"pp":2089,"pr":5736,"hÅ«":50,"ps":1178,"yÅ":62,"uÅ¡":116,"tÅ¡":984,"vö":298,"zz":105,"vü":141,"vä":6112,"zh":68,"zi":492,"ze":455,"uü":74,"za":431,"uõ":62,"zy":50,"võ":10296,"zs":86,"zu":144,"zo":270,"zn":53,"zm":49,"yg":56,"ye":201,"yc":105,"yd":131,"tü":1113,"ya":457,"yb":52,"tö":1641,"tõ":1288,"yt":138,"ys":416,"yr":185,"yp":88,"yo":142,"yn":260,"ym":152,"yl":325,"yk":76,"yi":121,"რ":63,"áƒ":50,"ნ":45,"ი":145,"ლ":57,"ე":73,"áƒ":150,"² ":198,"án":134,"ä ":222,"ár":59,"äb":871,"Ãœh":852,"Ãœl":582,"á ":83,"Ãœ ":171,"Õi":103,"Õh":47,"Õp":48,"Ök":67,"アアア":314,"ö ":312,"õ ":118,"ón":116,"ín":74,"ía":57,"í ":51,"én":82,"és":48,"ér":80,"ää":6210,"é ":189,"ät":739,"äv":318,"äm":341,"äl":3418,"äo":127,"än":4791,"äp":377,"äs":977,"är":10885,"äe":2551,"äd":252,"äg":1326,"äi":4721,"äh":3974,"äk":203,"üü":2774,"Än":168,"Är":63,"õš":48,"Ä ":58,"öö":3301,"ể":72,"üo":89,"ün":5015,"üm":1677,"ül":5410,"üs":2571,"ür":1208,"üp":727,"üv":138,"üt":1243,"üb":167,"üa":118,"üf":48,"üg":358,"üd":830,"ük":2636,"üh":5389,"øy":105,"õõ":933,"öv":135,"ør":74,"øn":53,"õr":3591,"õs":403,"õt":1896,"õm":351,"õn":2300,"õo":45,"õp":1548,"õi":12796,"õj":1645,"õk":165,"õl":2454,"õe":2011,"õg":861,"õh":4577,"õb":159,"õd":527,"öt":705,"ör":689,"ös":334,"öp":203,"ön":372,"öl":253,"öm":99,"ök":454,"öi":89,"ög":138,"öe":80,"öd":707,"öb":82,"ḩ":72,"õz":69,"õv":176,"õu":3856,"Ä« ":118,"Ä«n":58,"Å¡ ":464,"Å¡e":571,"Å¡a":283,"Å¡o":109,"Å¡k":120,"Å¡i":439,"Å¡u":129,"Å¡t":112,"Å a":47,"Å o":91,"Å v":165,"Åk":47,"Å ":113,"žu":64,"žo":59,"že":84,"ža":274,"ži":342,"ž ":81,"Å« ":64,"ი ":59,"ã‚":71,"ã‚¢":442,"가가 ":46," ã‚¢":53,"ä¹™":67,"之":249,"丹":47,"临":68,"並":138,"丘":83,"专":99,"三":373,"ä¸":204," 三":86," ä¸":77,"倉":51," 之":80,"ã‚ã‚":46,"ος":107,"ος ":107,"Ï‚ ":231,"α ":91,"アア":377,"Ñн":79,"ый":89,"ье":98,"ÑŒÑ":48,"ьн":72,"ха":91,"ци":80,"че":89,"Ñк":479,"Ñл":56,"Ñо":89,"Ñе":96,"Ñи":145,"рь":62,"Ñа":149,"Ñ€Ñ":87,"рт":59,"ру":71,"Ñ‚Ñ€":186,"то":214,"те":148,"тв":71,"ти":150,"та":205,"ÑÑ":83,"ÑÑ‚":346,"ур":101,"уÑ":74,"ул":50,"ун":50,"уд":52,"уб":58,"Ñ‚ÑŒ":45,"фо":51," Ga":590," Ge":860," I ":313," Fo":396," Fu":156," Fr":828," Fi":477," Fl":192," Fj":45," Ha":1914," He":1326," Gy":49," Go":366," Gr":850," Gu":492," Gi":307," Gl":139," Ig":87," Id":463," Dü":69," Hy":59," Hu":297," Ho":962," Hi":1244," Ji":117," Je":475," L ":50," Ja":1269," Iv":146," Ir":247," Is":445," It":383," Im":143," In":1484," Io":55," Ii":222," Il":189," M ":95," Fö":134," Fü":63," Ka":3502," Ke":1459," Ki":1200," Kh":87," Jo":986," Ju":831," N ":212," Gü":52," La":1495," Le":1398," Hä":127," Li":2733," Kl":320," Kn":46," Ko":2402," Kr":947," Kv":67," Ku":1637," Gö":60," Ky":76," Mc":62," Ma":3863," Hü":108," O ":93," Mi":1232," Dž":49," Me":1354," Lo":1146," Ly":58," Hõ":54," Lu":546," Ne":1115," P ":83,"а ":653," Na":1003," Jä":515," Ni":722," Mo":1366," My":96," Mu":873," A ":257," B ":117," C ":205," Ap":221," Am":1400," An":1581," Ak":327," Al":2024," Ai":275," Aj":171," Ag":212," Ah":170," Ae":126," Af":163," Ac":133," Ad":404," Aa":1075," Ab":516," Ba":1515," D ":91," Az":50," Av":171," Au":835," At":438," As":1512," Ar":1360," Be":1090," Bi":531," Bl":186," Bo":717," Br":891," Bu":586," E ":253," Ca":996," Ce":158," Ci":166," Ch":848," Cl":225," Cr":170," Co":1011," Cu":137," F ":65," Da":635," Di":564," De":805," Dr":153," Do":564," Du":150," Ea":51," Ec":83," Ed":300," G ":69," El":689," Ek":107," Ei":131," Eh":78," Eg":183," Ee":3409," Et":156," Es":505," Er":526," Ep":67," En":453," Em":267," Ex":60," Eu":1052," Ev":105," Fe":322," Fa":376," H ":102," Xi":80," Tä":312," Sü":194,"к ":130," Sõ":311," Sö":54,"Ив":73," Wo":210," Wi":583," Wh":45," Sä":63," We":424," Rü":62," Wa":532,"й ":637," Rõ":64," Zu":94,"Ле":48," Võ":412," Zo":49,"Ку":52," Ze":122," Zh":62," Vä":632," Zi":58,"Ко":96,"м ":90," Za":127," Yu":48,"Ка":72," Yo":250," Tü":235," Ya":113," Tö":129," Tõ":184,"л ":110,"Ðи":84,"Мо":47,"о ":255," TÅ¡":404,"Ма":85,"Ми":98," TÅ":50,"н ":434,"Па":47,"Пе":79,"По":48," a ":162,"Ñ ":91,"Ñ€ ":253," R ":62,"в ":424,"Ðн":87," Kõ":416," Kö":81,"Ðл":161," Ou":77," Os":431," Ot":172," Or":497," Oo":46,"Ðб":44," Op":127," Po":1235," Pl":283," Pi":1123," Ph":212," Lä":1080," Pe":1250," Pa":2281," Kü":278," Ny":45," Jõ":269," Nu":172," No":1462," Ol":339," Ok":124," On":146," Om":122," Oh":110," Kä":196," Oi":46," Od":87," Of":45," Ob":154," Jü":111," Mõ":229,"Вл":51," Ra":1755," Mü":121," T ":66,"д ":86,"Ви":55," Qu":130," Ro":1766," Re":978," Ri":1126," Nä":145," Rh":72," Lõ":669," Py":57," S ":141,"Бо":74,"г ":58," Pr":1513," Ps":52," Pt":76," Pu":606,"Ва":94," Lü":129," Mä":338," Sy":110," Sv":156," Su":1464," St":1223," Ta":4345," V ":98," Pä":659," Th":744," Ti":535," Te":1863," Tr":722," Ts":168," To":951," Nõ":702," Ru":426,"Ге":81,"Гр":45," Sa":3543," Nü":45,"е ":239," Sh":511," Si":1350," Sc":495," Se":3040," So":1682," Sp":263," Sk":219," Sl":182," Sm":84," Uu":347," Va":3308," X ":95,"и ":220," Ve":2347," Rä":65," Vi":1878," Vl":115," Vo":452," Vu":47," Tu":679," Tv":68," Ty":76," Põ":1331," Pö":45," W ":53," Pü":263," Ud":58," Uk":200," Ul":110," Um":60," Un":417," Ur":135," Us":122," Ut":61," ja":20477," l ":68,"ÑŒ ":130," io":82," im":299," in":4488," il":865," ii":156," is":1859," it":246," fü":370," ka":13902," fö":77," m ":441," ki":4980," ke":11286," jn":167," jo":514," jm":152," jt":94," ju":2953," ha":2640," he":1467," gl":111," gr":524," go":49," k ":146,"Ñ‹ ":90," ib":56," dü":205," id":1072," ig":443," hi":1143," ho":971," hu":967," jä":3184," ni":6713," nd":118," ne":1909," na":1142," p ":48," mu":3133," mo":1838," mm":67," ok":758," ol":7096," om":1854," on":34245," oh":172," kä":1164," oj":83," of":466," ob":937," jõ":1996," nu":369," nt":126," no":1262," nn":71," le":1944," lk":65," hä":549," li":6989," n ":104," la":5115," kv":246," ku":9586," km":1544," kl":989," kr":1633," ko":10755," me":3580," dž":65," mi":14942,"Ñ ":380," hü":385," ma":7824," lu":810," hõ":577," lo":3309," ae":363," af":69," ag":306," ah":192," aa":4258,"Ст":49," ab":842," ac":46," ad":192," am":855," an":2331," ap":575," ai":1079," aj":1910," ak":483," al":5568," av":698," au":1797," ar":3514," at":197," as":4137," d ":153," ba":664," 가가":46," bi":566," be":162," bo":152," bl":78," bu":241," br":141," ca":117," e ":147,"Ñ… ":65," b ":66,"Ñ‚ ":156,"Ро":57,"Се":84,"Со":64," er":2077," et":1438," es":2246," en":2378," em":344," ep":75," ei":852," el":2672," ek":498," ef":88," ee":3011," eh":5575," eg":105," fe":136," fa":280," eu":119," ev":98," fu":315," fr":185," fo":525," fl":88," fj":44," fi":1214," bä":181," ge":720," ga":319," i ":993," cm":54," co":181," ch":69," da":137," do":440," dr":125," de":2173," eK":291," di":908,"ч ":585," ed":721," eb":183," du":101,"ль":338,"ма":140,"ме":79,"ми":161," vö":80,"лл":50,"ло":200," vü":80,"ла":291," zo":50," zu":50,"ле":285," võ":8473,"ли":217,"кÑ":169,"ко":414," vä":4759,"ка":276,"ки":323," tõ":919," tö":1012," tü":520,"йÑ":53,"иÑ":126," tä":3294,"им":129,"ин":324,"ик":203," sõ":2172,"ил":197," sö":81,"ии":45,"ий":374,"ич":632,"их":108,"ит":98,"ир":115,"иÑ":145," sü":5410,"ри":296,"рк":48,"рн":83,"ро":406,"ра":317,"рг":121,"рд":63,"ре":208,"пр":87,"по":93,"па":67,"пе":55,"оÑ":221,"ор":399,"оп":51,"от":68,"ок":58,"ол":330,"ом":142,"он":240,"ой":87,"ов":918,"ог":103,"од":117,"ое":100,"об":67,"ны":111,"нт":92,"нÑ":154,"но":345,"нн":108,"нк":74,"ни":279,"не":99,"нг":60,"нд":193,"на":391," tÅ¡":115,"мо":119,"ге":109," ru":423," nõ":449,"ги":57," u ":56,"го":156," sa":6645," nü":110," se":7005,"гу":45," sc":47," si":2589," sh":125," sl":105," sk":246," sp":570," so":1701,"да":103,"ве":150,"ви":646," mõ":2887," mö":122,"вн":105,"во":155," t ":61," mü":545," ra":5080,"вÑ":81," re":2103," ri":3272," nä":1534,"га":89," ro":1241,"бе":93," pu":1946," pr":4024," ps":246,"бо":55," s ":402," lö":96," px":82," lõ":1781," mä":2882," lü":990,"ва":236," os":2510,"ад":144," ot":471,"ае":61,"аз":52," oo":592," op":302,"аб":45,"ав":150," or":1834,"аг":45,"ам":81,"ан":559,"ай":127," kõ":2868,"ак":68,"ал":255," kö":73," pe":3640," kü":1892," pa":3723,"аÑ":154,"ар":332,"ат":153," pl":542," po":5154,"аÑ":206,"ба":89," pi":4298," lä":2715," rü":649," rõ":177," y ":67," rö":76,"ив":76,"иг":52,"ид":49,"ие":98," sä":314," x ":107," va":12420," ve":4280," uu":1024," vo":1317,"за":70," vu":102," rä":537," vi":1606," vk":47," vm":99,"ет":178," pü":607,"еÑ":139,"ер":394,"ео":53,"ен":381,"ем":79,"ел":192," pö":222," põ":3044,"ек":180,"ей":122," tu":3292,"ее":74," us":346," um":867," un":274," uk":117," ul":323," uj":60," ta":4816,"др":169,"до":115,"ди":121," sy":46," st":1273,"де":141," su":5118,"ев":407," tr":1088," ts":358,"ед":85," to":2603," pä":1847," th":387," ti":918," te":10216," Õp":48," Õh":47," Õi":101," Ök":64," Ãœ ":59," Ãœl":578," Ãœh":851," är":293," ää":835," öe":60," ök":219," õl":119," õi":938," õh":408," õp":648," õn":74," öö":76," ür":128," ül":2321," üm":508," ük":1493," üh":2633," ê°€":51,"ê°€":148,"Åky":44," Å v":165," Å o":91," Å a":47," ža":50,"د ":87,"Ø© ":66,"ان":46,"ال":176,"ر ":45,"ÙŠ ":68,"Ù† ":97," アア":51,"AS ":102," ко":46,"BA ":64," по":46," Ро":57," Пе":79," Па":47," По":48,"가가":97," Ст":49," Се":84," Со":64,"AO ":53," Ðн":87," Ðл":161," Ðб":44," Ва":94," Бо":74," Ви":55," Вл":50," Ге":81," Гр":45," Ка":71," Ив":73," Мо":47," Ðи":84," Ко":95," Ку":52," Ле":47," Ма":83," Ми":98,"Fel":59,"Fer":121,"Fil":156,"Fin":145,"Fir":68,"Fan":92,"Fal":44,"Fai":50,"Era":62,"Eri":232,"Est":182,"Ern":93,"Esi":185,"Eur":956,"Eva":52,"Ehi":44,"Ele":225,"Ela":50,"Eks":56,"End":51,"Ena":97,"Eng":50,"Ene":66,"Emm":77,"Ema":72,"Eli":126,"Ell":63,"Ent":53,"Ger":95,"Geo":499,"Gen":130,"Gio":51,"Gil":46,"Ива":71,"Öko":66,"Gan":54,"Gal":104,"Gam":62,"Gar":101,"Gab":56,"Flo":75,"Fla":65,"Fra":377,"Fri":218,"Fre":186,"Foo":47,"Fon":48,"For":124,"IP ":48,"II ":579,"His":301,"Hii":583,"Hil":89,"Hin":100,"Hel":430,"Hei":240,"Hea":65,"Hen":145,"Her":270,"Haa":140,"Hab":68,"Hal":209,"Hai":81,"Han":256,"Ham":105,"Har":682,"Hau":60,"Gus":86,"Gua":87,"Gui":119,"Grö":69,"Gre":178,"Gri":80,"Gra":216,"Gru":170,"Gro":104,"Gol":71,"Got":58,"Gor":73,"Ing":355,"Inf":74,"Ini":62,"Int":306,"Ins":151,"Ill":72,"Ind":377,"Imp":71,"Iis":118,"Iir":94,"Ida":408,"Hum":47,"IV ":90,"Hor":121,"Hoo":71,"Hom":80,"Hon":86,"Hol":356,"Arg":115,"Arh":59,"Are":119,"Arc":52,"Ara":180,"Arm":142,"Arn":51,"Ark":65,"Ari":84,"App":49,"Apo":82,"Ate":50,"Atl":232,"Asu":1049,"Ast":103,"Ass":139,"Ase":89,"Art":168,"Arv":91,"Aru":68,"Ava":88,"Aut":116,"Aus":402,"Aug":149,"Bai":96,"Bak":55,"Bal":343,"Ban":110,"Bab":69,"Bad":75,"Bar":269,"Bat":57,"Bas":115,"Bau":47," пр":54,"Aar":73,"Aas":446,"Aaf":295,"Aad":67,"Abr":47,"Abe":61,"Aba":83,"Abd":79,"Abi":49,"Ada":90,"Adv":74,"Ado":56,"Ade":47,"Aer":49,"Aeg":46,"Age":54,"Afg":70,"Ain":50,"Air":92,"Al ":56,"Aja":140,"Aka":119,"Akt":63,"Ala":283,"Alb":201,"Alg":178,"Ali":85,"Ale":472,"Alf":50,"Alu":59,"Alt":115,"All":206,"Alp":102,"Ame":1051,"Amb":81,"Ama":69,"Ang":173,"Ani":70,"Ana":136,"And":351,"Ans":55,"Ant":450,"Ann":191,"Bus":55,"Bul":94,"Bur":160,"Bud":75,"Мих":58,"Bru":68,"Cal":198,"Cam":87,"Cas":93,"Car":284,"Cat":48,"Can":91,"Bea":78,"CH ":77,"Ber":414,"Ben":187,"Bel":194,"Bil":84,"Bis":73,"Bir":85,"Bio":136,"CO ":51,"Bla":86,"Bre":116,"Bra":266,"Bro":108,"Bri":271,"Ðик":72,"Bol":76,"Bon":63,"Bor":154,"Bos":66,"Bou":71,"Õig":83,"EE ":44,"Det":56,"Des":76,"Dev":50,"Del":63,"Dem":159,"Den":66,"Dep":48,"ан ":122,"Dan":112,"Dar":75,"Dav":137,"ай ":47,"Chr":190,"Che":117,"Chi":112,"Cit":59,"Cla":86,"Cen":74,"Cha":324,"Cri":46,"DV ":45,"Clu":51,"Cor":172,"Com":152,"Col":177,"Con":247,"Cou":57,"FA ":50,"ади":61,"аев":60,"Ege":46,"Egi":118,"али":51,"аль":66,"ано":90,"Ees":3340,"анд":117,"ани":46,"Edu":80,"Edw":63,"Ede":52,"FC ":88,"Dia":62,"Dis":117,"Dio":47,"Dig":58,"Die":54,"Div":45,"Ð°Ñ ":201,"Dre":44,"Dra":54,"Пет":51,"Doy":59,"Don":122,"Dom":77,"Doo":55,"Dor":65,"Nee":123,"Nen":81,"Nel":76,"Nei":118,"Nev":47,"Neu":75,"Net":49,"Nat":157,"Nii":60,"Nig":95,"Nic":90,"Nim":92,"Nik":161,"Jär":285,"New":291,"Nap":53,"Nar":263,"Nan":47,"Nag":44,"Nad":109,"Jõe":111,"Jõg":96,"Jää":129,"OS ":46,"Nov":157,"Nor":978,"Noo":108,"Ðле":136,"Kär":84,"Obe":69,"Jür":85,"Ott":71,"Ote":47,"Kõi":90,"Kõr":209,"Oli":46,"Ole":65,"On ":49,"Oma":92,"Olü":53,"Ope":62,"Ora":58,"Osc":88,"Osa":64,"Ord":51,"Ori":67,"Org":113,"Ost":56,"Osl":81,"Oss":50,"Lää":780,"Ple":51,"Pla":187,"Pin":124,"Pil":73,"Pik":86,"Pii":306,"Pir":78,"Pih":114,"Pie":94,"Pho":46,"Phi":103,"Läh":64,"Lät":200,"Pea":138,"Ped":50,"Per":283,"Pet":350,"Pen":99,"Pel":65,"Pee":136,"Pat":127,"Pas":71,"Par":721,"Pav":48,"Pau":155,"Paa":107,"Pad":62,"Pan":241,"Pai":107,"Pal":313,"Kül":163,"Pak":101,"Lõu":592,"Pto":68,"Pun":149,"Pur":44,"Pue":49,"Puh":59,"Puu":79,"Pro":404,"Pri":219,"Pre":257,"Pra":604,"Pol":248,"Pom":45,"Poh":46,"Pot":67,"Pos":114,"Poo":305,"Por":187,"RO ":68,"Mär":85," ال":145,"Män":102,"Вла":51,"SA ":770,"ВаÑ":49,"Raa":184,"Rad":71,"Rai":65,"Rah":536,"Ram":65,"Mün":49,"Ran":128,"Rak":98,"SD ":47,"Mõn":60,"Mõi":70,"Isa":72,"Ise":51,"Ita":340,"Isl":190,"Ira":139,"Iva":115,"Jac":79,"Jaa":509,"Jar":46,"Jan":141,"Jam":143,"Jal":74,"Jak":74,"Jen":64,"Jer":96,"Jea":76,"Jee":58,"KP ":62,"Jos":138,"Jor":57,"Joo":51,"Jon":103,"Joh":467,"Joa":63,"Jug":45,"Juh":63,"Juu":106,"Jur":60,"Jul":218,"Jum":146,"Föd":118,"Kaa":255,"Kad":123,"Kab":68,"Kai":254,"Kah":71,"Kag":66,"Kam":188,"Kal":368,"Kak":46,"Kap":63,"Kan":466,"Kau":239,"Kat":178,"Kas":307,"Kar":687,"Ker":134,"Kes":554,"Ket":49,"Ken":126,"Kel":51,"Kem":44,"Kei":251,"Keh":55,"Kee":118,"Kir":431,"Kit":75,"Kin":160,"Kiv":96,"Kil":83,"Kih":48,"Kii":162,"Klo":53,"Kli":49,"Kle":53,"Kla":98,"Klu":55,"Koo":227,"Kon":442,"Kom":339,"Kol":258,"Kos":162,"Kor":274,"Kop":81,"Kog":69,"Kod":97,"Kok":46,"Koi":60,"Koh":156,"Kr ":310,"Kot":63,"Kre":312,"Kra":186,"Kri":267,"Kro":87,"Kru":51,"Kui":148,"Kul":317,"Kun":277,"Kur":369,"Kuu":241,"Kva":53,"Lev":81,"Les":48,"Lep":73,"Leo":109,"Len":251,"Lem":45,"Lei":135,"Leh":50,"Lee":284,"Lea":56,"Lau":185,"Le ":49,"Lak":57,"Lai":145,"Lag":77,"Lah":103,"Lae":57,"Las":93,"Lar":49,"Lap":96,"Lam":60,"Lan":154,"Lad":56,"Laa":70,"La ":89,"ML ":59,"Lib":76,"Lie":81,"Lih":74,"Lig":47,"Lii":1363,"Lil":58,"Lim":58,"Lin":573,"Lip":63,"Lis":111,"Lit":56,"Liv":63,"Lut":62,"Luu":51,"Luk":48,"Lui":44,"Lud":75,"Luc":57,"Lou":126,"Los":88,"Lot":49,"Loh":45,"Lor":56,"Loo":276,"Lon":195,"Lom":53,"Lok":47,"NA ":78,"Mei":81,"Meh":138,"Men":72,"Mel":110,"Mes":92,"Mer":322,"Met":201,"Med":80,"Mee":102,"Man":356,"Mal":230,"Mar":1023,"Mas":191,"Mag":212,"Hüd":45,"Mad":279,"Maj":106,"Mak":124,"Mah":61,"Mai":109,"Mac":68,"Maa":607,"Max":73,"Mau":82,"Mat":240,"Mod":67,"Moh":45,"Mol":99,"Mon":346,"Moo":103,"Mos":352,"Mor":136,"Mot":44,"Mih":126,"Mik":120,"Mic":227,"Mit":88,"Mis":98,"Mil":150,"Min":245,"Mul":70,"Muh":59,"Muu":174,"Mur":95,"Mus":281,"Tän":126,"Täh":115,"лав":45,"лад":53,"ль ":49,"TÃœ ":47,"ääm":126,"ääl":395,"ään":2004,"ääk":111,"ääd":51,"ääg":344,"ääb":251,"äät":82,"ääv":158,"äär":2288,"ääs":242,"ää ":101,"Sõn":78,"кÑа":105,"Sõj":66,"Sõr":64,"ков":99,"кол":81,"кий":216,"Wor":93,"Wol":71,"Wil":257,"Win":131,"каÑ":100,"Wei":45,"Weh":50,"Wes":117,"Was":59,"War":74,"Wat":77,"Wal":182,"йÑк":48,"ко ":58,"Vor":103,"Voo":53,"Vol":205,"Vis":61,"Vit":59,"Vla":113,"наÑ":59,"ое ":83,"ндр":145,"ой ":73,"Väl":103,"Väi":301,"Väs":46,"Vär":64,"ов ":254,"мир":58,"мов":45,"ман":65,"Yor":178,"You":47,"Töö":118,"Tür":179,"льн":72,"на ":209,"льÑ":44,"三三":54,"Tõn":45,"лов":104,"лив":45,"лен":58,"лек":145,"ра ":49,"Sve":99,"Suu":805,"Sur":85,"Sul":92,"Sup":46,"Sun":86,"Sud":52,"Str":229,"Stu":106,"Sto":184,"Sta":395,"Ste":226,"Tee":206,"Teh":108,"Teg":101,"Tea":178,"Ted":50,"Ten":61,"Tem":174,"Teo":71,"Tei":326,"Tel":114,"Tam":134,"Tan":148,"Tat":66,"Tar":920,"Tav":135,"Tai":140,"Tal":1376,"Tag":78,"Taa":310,"Tab":46,"Tad":48,"Ta ":667,"ори":54,"оро":72,"Ska":93,"Shi":143,"She":124,"Sha":138,"Sim":113,"Sil":212,"Sii":184,"Sih":45,"Sig":69,"Sis":180,"Sir":66,"оÑÑ‚":84,"Sin":138,"Sie":46,"Sib":150,"оль":64,"Ser":245,"Sev":74,"оло":46,"оли":53,"ола":72,"Sep":56,"Sen":68,"Sel":786,"Sem":74,"Sei":87,"Sed":193,"See":1098,"Sea":119,"TV ":56,"äb ":221,"Spa":44,"Spe":59,"Spo":69,"Sof":45,"Soc":51,"Sot":162,"Sou":50,"Sol":103,"Som":77,"Son":96,"Soo":896,"TO ":49,"Slo":106,"äga":245,"äev":922,"ова":78,"äet":56,"äes":339,"äea":55,"äed":62,"äel":380,"äht":1060,"äid":319,"äib":55,"Nõu":525,"ähe":2126,"ähk":56,"ähi":637,"Nõm":138,"äge":296,"ägi":746,"äe ":449,"äbi":596,"ный":73,"SV ":478,"Rus":63,"äda":144,"Rum":92,"äde":48,"änn":112,"änu":150,"Sag":67,"Sai":131,"Sah":102,"Sak":1209,"Sam":213,"Sal":247,"Saa":614,"Sab":44,"Sad":48,"ämm":58,"äna":797,"äni":236,"äng":1228,"äne":1640,"änd":528,"äpp":49,"äps":237,"Sco":47,"Sch":374,"äol":71,"Sav":86,"Sau":94,"Sar":178,"San":281,"är ":134,"äit":1511,"äis":253,"äir":146,"äin":438,"äil":189,"äik":1090,"äij":45,"äig":275,"äie":224,"äiv":81,"ове":46,"älu":94,"ält":126,"ови":380,"äli":683,"älj":1874,"овн":48,"овÑ":48,"äme":148,"ого":52,"äki":100,"TA ":65,"älg":52,"äld":112,"äle":277,"ävi":138,"SI ":60,"Res":61,"äva":99,"Rev":69,"Näi":96,"нов":187,"ор ":52,"Ris":101,"Rii":508,"Rin":80,"Ric":153,"Rid":55,"ärg":1735,"äri":1473,"ärj":460,"ärk":656,"ärm":139,"ära":1650,"ärd":69,"äre":1128,"Ras":57,"Rau":107,"Rap":154,"äsi":369,"äsk":115,"äse":53,"нко":46,"он ":72,"ärv":1633,"ärn":371,"äru":76,"ärt":890,"ärs":325,"ärr":78,"ätm":48,"äti":288,"ätk":94,"äst":200,"äss":64,"äsu":88,"Rec":53,"Rei":185,"Reg":77,"Ren":63,"Rel":79,"Rep":45,"ätt":141,"äts":56,"Rea":67,"Roh":92,"Rob":131,"Roc":60,"Ros":150,"Rom":120,"Roo":904,"SS ":97,"SO ":46,"нÑк":92,"Ñан":105,"Vab":795,"Vaa":45,"Vai":341,"Vah":167,"Vel":86,"Ven":1754,"Vee":94,"Ñил":46,"Ñки":218,"Ñка":112,"Ñко":106,"Vas":243,"Van":479,"Val":835,"Var":232,"Vih":45,"Vig":101,"Vii":222,"Vid":54,"Vic":79,"Vie":71,"Vir":448,"Vil":366,"Vik":195,"Vin":105,"Ñов":48,"Ver":155,"Ves":127,"Ukr":194,"Ung":127,"Uni":210,"VR ":44,"Uus":233,"Uur":54,"Uue":46,"Tve":67,"Põl":225,"Põh":1046,"Püh":181,"рий":44,"риÑ":45,"Pür":45,"ров":215,"VI ":61,"Ter":274,"Tes":65,"The":470,"Päi":111,"Tho":103,"Pär":428,"Tih":59,"Tii":133,"Tim":77,"Pää":51,"Too":201,"Tor":167,"Tol":64,"Tom":75,"Ton":64,"Toi":63,"Tru":76,"Tro":136,"Tri":130,"Tre":108,"Tra":219,"Tur":159,"Tuu":92,"Tul":130,"Tun":106,"Å¡i ":155,"Å¡et":55,"Å¡ev":66,"Å¡el":50,"Å¡er":71,"Å¡ee":52,"ван":86,"Å¡eh":151,"Å¡a ":68,"Å¡at":45,"Å ve":145,"вÑк":69,"вна":76,"вич":569,"Å¡ii":58,"ва ":83,"аÑи":52,"ато":47,"во ":47,"bje":380,"bja":66,"bis":218,"bit":234,"biv":187,"bio":537,"bip":80,"bir":67,"bik":204,"bil":578,"bim":157,"bin":301,"bii":222,"bij":70,"bo ":72,"blo":54,"ble":274,"bli":550,"bn ":58,"bla":847,"ев ":82,"bod":50,"bok":45,"bol":236,"ей ":78,"bon":137,"boo":67,"bor":247,"bot":147,"bos":56,"bou":65,"be ":205,"bam":130,"ban":365,"bak":284,"bal":373,"bai":111,"baj":47,"bah":73,"bac":66,"bad":310,"baa":286,"án ":48,"bav":70,"bat":316,"bas":493,"bar":1055,"bi ":1023,"bee":78,"bed":61,"bec":51,"ber":2586,"ben":167,"bem":51,"bel":642,"bek":156,"bes":968,"bet":314,"bia":533,"bib":89,"bid":126,"bie":117,"bha":48,"дро":45,"ет ":44,"ca ":308,"car":187,"cas":78,"cat":96,"can":142,"cal":161,"ce ":506,"bri":2225,"bro":86,"bra":389,"bre":198,"bu ":80,"bru":574,"bso":114,"bse":67,"bst":74,"дим":49,"bub":112,"bur":697,"bul":127,"bun":75,"bum":298,"bud":140,"but":54,"bus":333,"дор":45,"by ":122,"aka":1199,"am ":1037,"ake":1191,"aki":1056,"ajo":642,"aju":375,"al ":9437,"aja":7056,"aje":136,"aaž":50,"aik":874,"ail":2117,"aim":1277,"ain":2391,"aio":97,"air":125,"ais":922,"ait":1457,"aiu":220,"aiv":79,"ak ":324,"aig":556,"aie":299,"aid":1846,"aia":317,"ahn":114,"ahk":234,"ahl":63,"ahi":395,"ahj":153,"ahh":236,"ahu":752,"ahv":2279,"ahs":82,"aht":765,"ahr":54,"aho":103,"ahe":4825,"aha":671,"agi":522,"agr":165,"agu":1145,"agn":333,"ago":338,"akä":47,"aol":220,"aok":108,"anu":1617,"anz":132,"ajõ":234,"any":60,"ano":601,"ann":2713,"anm":52,"ant":3838,"ans":1284,"anr":93,"ane":2890,"ang":1938,"ajä":246,"ani":8218,"anj":107,"ank":687,"anl":132,"ap ":65,"ana":2831,"anc":369,"and":7769,"amu":491,"amt":68,"amm":1132,"aml":63,"amo":227,"amp":318,"ams":141,"amj":57,"ami":6164,"adž":93,"ame":2122,"amb":788,"ama":4891,"ao ":126,"alv":587,"alu":1954,"alt":2286,"als":1219,"alr":157,"alp":242,"alo":1054,"aln":583,"alm":1124,"all":6989,"alk":711,"alg":2752,"alh":218,"ahä":77,"ali":9008,"alj":729,"ald":4688,"ale":3211,"alf":70,"ala":5975,"alb":322,"an ":2385,"akv":79,"aks":8379,"akr":533,"aku":957,"akt":1317,"ako":2809,"akp":79,"akk":543,"akl":109,"aba":1901,"abe":658,"abi":1296,"abl":140,"abo":165,"abr":186,"abs":170,"abu":92,"ae ":265,"aca":68,"aab":988,"aaf":943,"aag":470,"aad":3122,"aae":152,"aaj":148,"aak":2014,"aai":1233,"aan":4143,"aal":7268,"aam":1949,"aas":6140,"aar":5151,"aap":765,"aav":886,"aau":47,"aat":3358,"ad ":6207,"ac ":51,"aa ":3615,"ab ":4871,"afo":83,"afr":355,"aft":134,"aff":76,"afe":44,"afi":983,"ai ":831,"aga":2664,"agd":52,"age":905,"aen":249,"ael":441,"aes":76,"aer":147,"aeg":1173,"aee":54,"aed":90,"aek":78,"aeh":45,"ah ":172,"afa":57,"aev":851,"aet":163,"ado":272,"adr":398,"adl":409,"adm":484,"adj":65,"adi":3217,"ade":4451,"ag ":78,"ads":103,"adu":2361,"adv":107,"aco":80,"ack":227,"aci":112,"ach":400,"ace":151,"acc":48,"ada":2882,"af ":128,"act":96,"azo":46,"azi":122,"гор":67,"Å ot":73,"avä":555,"aza":97,"др ":56,"avõ":220,"avö":76,"azz":52,"atä":55,"axi":69,"asõ":365,"asü":91,"atõ":66,"atö":93,"atü":106,"aya":58,"aye":71,"ba ":651,"atÅ¡":101,"at ":2216,"amõ":135,"arh":476,"anä":68,"arg":455,"arf":44,"are":3776,"ard":1446,"arc":202,"arb":519,"ara":2982,"amü":61,"arp":144,"aro":486,"arn":587,"arm":512,"arl":676,"ark":1022,"anç":55,"arj":940,"ari":5535,"aru":930,"arv":2193,"arr":543,"ars":766,"art":3541,"au ":240,"asa":1531,"anõ":84,"ary":130,"asi":2682,"ash":208,"asc":73,"ase":3414,"aso":395,"asn":174,"asp":282,"ask":962,"asj":302,"asm":178,"asl":169,"aot":356,"aor":55,"aos":1245,"ar ":2000,"akü":111,"apa":845,"ape":400,"api":685,"aph":50,"apn":98,"apl":208,"apo":517,"app":368,"apr":632,"aps":286,"apt":144,"apu":179,"alõ":58,"as ":16624,"alü":198,"amä":398,"ava":6484,"ax ":62,"aut":985,"avs":163,"avo":221,"avl":84,"avi":977,"ave":501,"ay ":224,"arü":99,"awa":117,"arö":60,"avy":49,"arõ":46,"avu":364,"av ":2085,"ata":5828,"asu":6462,"ast":15872,"ass":1978,"asr":56,"asv":597,"atm":137,"atn":57,"atk":174,"atl":435,"atr":525,"ato":1488,"ate":6661,"ati":3865,"ath":254,"apä":659,"aua":310,"aub":254,"att":452,"ats":3184,"atv":82,"atu":6581,"apõ":81,"aul":1069,"aum":65,"aun":225,"aup":177,"aur":292,"aus":715,"aud":1021,"aue":64,"auf":46,"aug":1111,"auh":299,"auk":196,"TÅk":46,"TÅ¡u":55,"TÅ¡i":63,"TÅ¡e":232,"Ãœhi":101,"Ãœhe":728,"Võr":202,"Või":74,"еев":71,"Ãœli":294,"Ãœld":58,"Ãœle":201,"еви":205,"ени":90,"енн":63,"ерн":50,"ерг":55,"екÑ":141,"ель":84,"етр":46,"ико":106,"ина":63,"ими":56,"иль":70,"иÑÑ‚":52,"иха":59,"ка ":82,"ив ":46,"ие ":51,"ий ":339,"ин ":106,"ич ":575,"Ð¸Ñ ":116,"jee":65,"jer":66,"jek":711,"jel":510,"jem":199,"jen":398,"jes":189,"jet":51,"jev":327,"ji ":46,"jad":794,"jaa":1553,"jab":52,"jat":1299,"jas":1811,"jav":659,"jap":328,"jao":620,"jar":182,"jal":3824,"eük":52,"jak":1121,"jan":2708,"jam":517,"jah":222,"jag":412,"jaj":121,"jai":178,"jaz":54,"je ":209,"jms":73,"jne":167,"jok":49,"joo":1404,"jon":315,"jor":223,"jia":91,"jm ":66,"fää":209,"itn":116,"itm":881,"itl":1066,"itr":232,"ito":1263,"itu":2865,"itt":840,"its":4841,"itz":125,"ity":156,"ipõ":57,"iud":60,"isk":1562,"isj":238,"ism":2164,"isl":320,"iso":452,"isn":380,"üdr":236,"isp":759,"iss":1297,"isr":458,"isu":1149,"üdn":80,"ist":16111,"üdi":214,"isv":570,"iv ":471,"ita":4226,"ite":4040,"ith":114,"ipä":225,"iti":2606,"itj":280,"ivs":528,"ivo":234,"ivn":346,"ivu":91,"ühe":2500,"irü":49,"iwa":47,"ühh":198,"ühi":1384,"üha":330,"iup":58,"ius":891,"ium":1351,"iul":72,"iut":86,"iva":1768,"ix ":96,"ügi":120,"ivi":1653,"ive":618,"üga":160,"ipr":172,"ipo":148,"ipp":426,"ipu":354,"ips":94,"ipt":204,"ipi":209,"ipl":275,"ilõ":45,"is ":17934,"ion":1904,"ioo":5170,"iop":63,"ior":225,"ios":453,"iot":121,"iog":121,"iok":89,"iol":466,"iom":85,"ipa":243,"ikü":95,"ipe":504,"iov":51,"ikõ":47,"ir ":371,"iru":697,"irv":50,"irs":78,"irt":114,"iro":264,"irm":315,"irn":410,"irk":973,"irl":76,"iri":2513,"irj":2489,"isi":2505,"ish":182,"ise":17961,"isc":260,"isa":2662,"üda":146,"iu ":159,"inõ":65,"iqu":72,"übi":94,"imä":330,"ire":897,"inä":160,"irg":342,"ira":660,"ird":488,"irc":67,"it ":2043,"imõ":379,"ünn":379,"üno":56,"ünk":61,"üng":61,"ünd":3848,"üp ":170,"üna":270,"ümp":310,"ümm":93,"ümn":303,"itÅ¡":421,"ümi":74,"üme":56,"ümf":54,"ümb":638,"ülr":51,"üpo":95,"üpp":83,"üpe":67,"üpi":189,"ünt":213,"ja ":25408,"ül ":45,"itü":46,"itõ":50,"itö":136,"isü":229,"ühr":50,"isõ":144,"ühm":571,"üht":317,"üla":1693,"üle":1718,"üld":476,"ülg":296,"üli":561,"가가가":51,"ülj":132,"ülm":94,"üll":158,"ülo":65,"üks":2052,"ivõ":345,"ivä":146,"üki":160,"ize":56,"ükl":190,"ükk":136,"iza":126,"üm ":72,"kii":670,"kih":644,"kik":66,"kij":101,"kim":281,"kil":445,"kk ":305,"kia":138,"kib":147,"kie":46,"kid":1460,"kiv":697,"kin":1384,"kip":117,"kir":3701,"kis":700,"kit":731,"kiu":59,"kja":72,"km ":1345,"ki ":1992,"kha":79,"kj ":53,"kho":116,"kea":512,"ked":219,"kee":6776,"keh":762,"kei":247,"kek":94,"kej":56,"kem":268,"kel":1252,"ken":530,"kep":122,"kes":5621,"ker":962,"ket":387,"kev":86,"füü":427,"ke ":2101,"kra":1232,"kre":1046,"kt ":481,"ksa":2425,"kse":4827,"ku ":4118,"kro":695,"kru":92,"kri":683,"kpa":66,"kr ":45,"kov":236,"km²":193,"kot":103,"kos":501,"kor":2423,"kop":353,"koo":4137,"kon":8471,"kom":1331,"kol":1558,"kok":400,"koj":93,"koh":2395,"kog":2107,"koe":132,"kod":636,"ööp":158,"öör":516,"öös":287,"ööt":635,"ks ":9327,"ööv":121,"ööb":59,"ööd":419,"öög":123,"kpo":74,"kpi":89,"öök":163,"ööm":71,"ööl":141,"öön":269,"kme":409,"kmi":180,"koa":59,"kob":69,"kne":492,"öö ":232,"kku":1792,"kke":775,"kka":353,"kko":768,"kki":577,"klu":516,"ko ":325,"kma":69,"kle":259,"kla":1096,"klo":288,"kli":1510,"jut":1345,"jus":656,"juv":119,"juu":1477,"jul":461,"juk":80,"jun":529,"jum":545,"jur":167,"jub":134,"juh":1325,"jug":89,"jud":292,"ju ":890,"jt ":90,"kav":342,"kat":1523,"kau":1221,"kar":907,"füs":56,"kas":4782,"kap":253,"kan":1568,"kao":86,"kal":1965,"kam":275,"kaj":243,"kak":404,"kah":1019,"kai":1321,"kag":415,"kae":301,"kad":579,"kab":194,"kaa":1822,"ka ":8332,"föd":110,"ha ":1068,"ham":380,"han":1084,"hap":399,"hai":439,"haj":90,"hak":332,"hal":1865,"hau":155,"hav":82,"har":1689,"has":829,"hat":244,"haf":57,"hae":123,"hag":90,"hab":58,"haa":386,"had":171,"he ":2035,"hek":639,"hel":4104,"hei":379,"heg":134,"hee":149,"hed":836,"hea":267,"hev":124,"het":632,"hes":861,"her":737,"hep":112,"heo":122,"hen":4035,"hem":646,"hi ":803,"hhi":366,"hho":288,"hha":319,"hk ":5071,"hig":50,"hie":100,"hid":172,"hic":69,"hib":91,"hia":196,"hip":104,"hio":120,"hin":1818,"him":313,"hil":853,"hik":677,"hii":392,"hiv":50,"his":1677,"hit":1195,"hir":223,"hja":2408,"hje":45,"hju":407,"hka":48,"hm ":244,"hke":354,"hkl":54,"õõd":171,"hkr":47,"hku":160,"hkv":87,"hn ":212,"õõt":409,"õõr":156,"õõs":83,"hla":75,"hle":125,"hli":44,"ho ":78,"hma":426,"gma":96,"go ":269,"gme":81,"glu":61,"glo":135,"gle":108,"gli":2195,"gla":352,"gko":571,"gno":68,"gni":103,"gne":544,"gna":249,"gmi":89,"glü":44,"gs ":53,"gpa":728,"gol":120,"goo":292,"gon":121,"gos":166,"gor":359,"got":49,"gov":53,"ый ":88,"gu ":2469,"gse":607,"gro":161,"gru":407,"gra":1940,"gt ":50,"gri":373,"gre":257,"gto":66,"gug":104,"gui":112,"guk":582,"gum":403,"gul":521,"gua":174,"gub":77,"gue":138,"gud":1131,"gy ":51,"guv":167,"gut":1041,"guu":62,"gur":250,"gus":4478,"gup":144,"gun":555,"guo":121,"gvi":72,"iai":81,"iah":71,"iak":307,"iaj":141,"iam":367,"ial":852,"iao":48,"dün":222,"ian":848,"iap":68,"ias":2407,"iar":200,"iau":97,"iat":663,"iav":133,"ic ":272,"iaa":1098,"iab":61,"iac":55,"iad":147,"iae":81,"iag":258,"ibl":123,"ibi":192,"ibo":94,"ibn":54,"ibr":97,"ibu":187,"id ":9461,"iba":199,"ibe":483,"ia ":7687,"ib ":1767,"iet":453,"iev":106,"iel":576,"iem":193,"ien":617,"ier":409,"ies":329,"iee":79,"ied":249,"iek":65,"ig ":195,"iec":79,"ifu":44,"ifo":208,"ife":183,"ifi":363,"ifa":59,"icr":74,"ics":57,"ict":114,"icu":98,"ico":150,"ick":168,"ici":96,"ich":758,"ice":186,"ie ":546,"ica":492,"idu":2141,"ids":89,"idr":54,"ido":139,"idm":73,"idn":61,"idl":140,"idi":1042,"idg":52,"ide":8097,"ida":3662,"iib":405,"iia":256,"iif":48,"iig":4040,"iid":2067,"iie":248,"iik":3861,"aÅ¡i":75,"iin":2367,"iil":963,"iim":756,"iis":1521,"iir":3180,"iip":91,"iiv":1986,"iiu":178,"iit":2428,"il ":6055,"ija":1055,"ije":50,"ijo":86,"iju":112,"im ":1389,"ika":6374,"ige":1448,"iga":2716,"ii ":657,"igl":90,"igm":62,"igh":187,"igi":4211,"igu":1608,"igr":133,"igo":112,"ign":238,"ihe":679,"iha":299,"ihk":182,"ihm":65,"ihh":111,"ihi":253,"iht":687,"ihu":144,"iho":52,"ik ":7328,"imo":225,"imn":64,"imm":144,"iml":60,"ims":252,"imp":192,"idž":109,"ime":7025,"imk":210,"imi":4107,"ip ":128,"inc":157,"ind":2723,"ina":4896,"inb":48,"imt":70,"imu":1850,"inn":6824,"ino":606,"inr":85,"int":1835,"ins":667,"inf":422,"ine":12596,"ijä":97,"inh":82,"ing":10735,"inj":49,"ini":4649,"inl":74,"ink":241,"ioa":102,"inu":4443,"inv":62,"iny":57,"ьев":85,"iko":1851,"ikn":407,"ikm":345,"ikl":1543,"ikk":1983,"iki":2101,"ikh":52,"ike":2715,"ila":1146,"ilb":76,"in ":2156,"ikv":103,"ikt":211,"iku":8444,"ikr":276,"iks":1858,"ilp":103,"ilo":1161,"ill":5963,"ilk":54,"iln":65,"ilm":2925,"ilh":117,"ilj":833,"ili":8126,"ild":409,"ile":1595,"ima":3230,"imb":81,"io ":525,"ils":210,"ilt":420,"ilu":306,"ilv":181,"hol":538,"hom":193,"hon":119,"hoi":327,"hos":103,"hot":60,"hou":76,"hov":70,"hoo":630,"hop":48,"hor":259,"hob":86,"hof":75,"hoe":47,"hod":56,"hni":355,"hno":248,"hnu":57,"hna":53,"hiü":45,"hme":82,"hmi":96,"øya":60,"huk":196,"hul":1525,"hua":81,"htu":834,"htt":114,"hts":526,"htr":54,"htp":46,"htn":93,"htm":114,"htl":468,"hti":872,"htk":120,"hte":1012,"hta":927,"hv ":86,"hst":88,"hu ":482,"hrm":49,"hro":62,"hre":114,"hri":254,"ht ":1267,"hra":59,"hvu":1043,"hwa":51,"hum":318,"hun":86,"hus":694,"hut":421,"hur":174,"huu":56,"huv":197,"hva":963,"hve":131,"hvi":258,"hvk":76,"fi ":130,"ffe":80,"ffi":97,"fes":255,"fer":227,"fen":116,"fek":256,"fel":93,"fia":527,"bän":220,"fga":88,"far":57,"fan":78,"fak":63,"aül":78,"fal":91,"fai":103,"aüh":102,"fac":50,"faa":159,"ff ":69,"fe ":58,"etÅ¡":80,"fa ":46,"etü":153,"etö":63,"etõ":90,"eys":46,"esü":78,"exa":132,"ez ":79,"esõ":110,"etä":142,"exi":65,"evõ":561,"evä":227,"ezi":53,"eta":4185,"ete":3476,"eti":2019,"eth":183,"epä":212,"etn":244,"etl":824,"etk":122,"esp":119,"esn":50,"eso":307,"est":15172,"esu":447,"esr":46,"ess":2291,"esw":67,"ev ":1881,"eud":58,"euk":45,"eum":209,"eto":650,"etr":1191,"ets":1555,"ett":1369,"etu":2430,"etv":60,"epõ":75,"ew ":294,"eve":552,"eva":4542,"evo":358,"evk":55,"evi":1332,"eut":162,"eur":217,"eus":372,"ex ":72,"evu":1013,"ey ":327,"erü":78,"epe":305,"epi":674,"elä":86,"eph":149,"er ":4533,"ekü":242,"epa":593,"eot":431,"eos":1290,"eor":477,"eom":115,"eol":658,"eok":113,"eop":87,"eoo":298,"eon":217,"emä":170,"es ":19178,"ept":722,"eps":46,"epu":101,"epl":50,"epp":174,"epo":236,"epr":147,"erk":288,"erl":431,"eri":8620,"erj":357,"erg":1464,"enä":125,"erh":103,"ere":3347,"erf":89,"erc":92,"erd":390,"era":2994,"erb":550,"et ":2344,"emõ":84,"esk":3814,"esl":149,"esm":728,"esh":61,"esi":3721,"esc":105,"ese":2495,"esa":695,"erz":44,"enõ":60,"ery":69,"erv":749,"eru":688,"err":901,"ert":1083,"ers":969,"ern":1144,"erm":1039,"erp":165,"ero":598,"eki":846,"ekk":441,"ekl":132,"eko":1441,"ekr":148,"eks":6278,"ekt":2822,"eku":1116,"ekv":84,"en ":1782,"elb":55,"ela":2671,"eld":1386,"elf":47,"ele":10866,"eli":5612,"elj":466,"elg":374,"elm":278,"eln":482,"elk":721,"ell":3694,"elo":690,"elp":87,"elu":1130,"elv":392,"els":698,"elt":4591,"eo ":141,"emb":1631,"ema":5853,"eme":1702,"emm":105,"emn":126,"emo":658,"emi":4057,"emj":89,"emk":49,"emu":471,"emp":531,"ems":73,"ep ":59,"enf":67,"ene":6080,"enh":138,"ejä":142,"eng":640,"enb":279,"ena":1669,"end":9301,"enc":218,"eno":318,"enn":1302,"enk":163,"enl":78,"eni":2414,"enj":57,"enu":916,"ens":876,"ent":3483,"enr":170,"enz":141,"ejõ":394,"eny":49,"eog":500,"eod":99,"eob":44,"egl":195,"ego":223,"egn":128,"ege":2106,"egi":861,"eha":1078,"egr":208,"egs":221,"egu":1523,"ehn":541,"ehm":64,"ehk":5017,"ehr":86,"eho":88,"ehe":590,"ehi":903,"ehh":383,"ek ":209,"eib":48,"eic":64,"eia":95,"eht":613,"eip":60,"eis":4050,"eir":116,"eim":514,"eil":387,"ein":1247,"eii":45,"eik":96,"eie":63,"eid":2255,"eig":71,"eja":948,"el ":10310,"eit":602,"eiu":148,"eiv":45,"ejo":57,"eje":77,"eke":406,"eka":1625,"em ":2077,"eju":213,"gjo":58,"öta":380,"ötl":187,"ötm":68,"gji":57,"gl ":95,"öst":232,"git":756,"gis":1905,"gir":78,"giv":225,"gil":1090,"gim":411,"gij":262,"gik":642,"gip":428,"gin":524,"gio":325,"gid":469,"gie":61,"gib":455,"gih":62,"gii":53,"gig":185,"örl":72,"gia":2005,"örs":50,"ght":152,"öra":67,"örd":139,"gha":121,"öri":170,"ös ":62,"gga":46,"gi ":5321,"öpm":55,"ör ":108,"gen":1194,"geo":730,"get":321,"geu":170,"ger":573,"ges":288,"gev":1367,"gh ":77,"gee":427,"ged":631,"geb":241,"gei":271,"gem":670,"gel":1762,"gek":61,"gej":286,"öni":115,"gda":70,"önd":159,"gde":51,"ömi":48,"ge ":1988,"gaz":58,"öli":74,"gab":187,"gad":306,"gai":51,"gaa":657,"gas":1340,"gar":467,"bür":50,"gat":540,"gav":357,"gak":106,"gaj":153,"gam":272,"gal":712,"gan":2019,"öko":300,"gap":72,"ga ":9681,"öid":47,"ögi":100,"öel":65,"fur":44,"fta":81,"fti":45,"fun":331,"ft ":139,"fra":194,"fre":107,"fri":412,"fro":53,"for":851,"fos":114,"fot":258,"fon":233,"foo":243,"fol":120,"fla":75,"fli":57,"fo ":79,"fid":70,"fic":72,"fie":44,"fii":98,"fil":1891,"fik":301,"fin":213,"fir":227,"fis":69,"fit":113,"fjo":141,"õla":152,"õle":240,"õlg":110,"õli":251,"õlk":134,"õll":298,"õlm":435,"õlt":294,"õlu":48,"õlv":413,"õju":446,"õkk":88,"õne":668,"õng":76,"õna":926,"õnd":49,"õnn":136,"õni":203,"õnk":86,"õnu":108,"da ":7682,"õmb":51,"õmm":206,"õpe":513,"õpi":263,"õpu":202,"de ":10486,"õpp":472,"dad":930,"daa":286,"dab":1585,"dak":424,"dal":2308,"dai":94,"daj":449,"dag":307,"dae":79,"dat":1521,"das":1349,"dar":435,"dap":141,"dao":265,"dan":602,"dam":1690,"dav":923,"õri":60,"õrj":81,"õrk":196,"dde":47,"õrg":1625,"õra":75,"õrb":100,"õrd":333,"õt ":51,"õrv":475,"õru":187,"õrr":267,"õrm":46,"õda":213,"õde":121,"cul":68,"ctu":66,"õe ":1023,"cto":118,"cti":143,"cy ":83,"õet":187,"ões":152,"õel":416,"õen":137,"õdu":141,"cus":131,"cur":49,"õhk":139,"õhe":49,"õhj":2576,"õhi":1112,"õge":171,"õgi":631,"õi ":5148,"õja":1187,"õiv":218,"õit":583,"õis":2033,"õim":1468,"õik":622,"õie":258,"õid":332,"õig":1596,"õib":513,"õhu":552,"cks":55,"cki":151,"ckh":122,"cla":52,"cle":64,"co ":192,"con":154,"col":137,"com":86,"cor":129,"cos":67,"cot":61,"cs ":57,"öd ":110,"ct ":57,"cra":52,"cro":105,"ödi":95,"öda":96,"öde":377,"õst":108,"cci":47,"õu ":118,"õsa":163,"õtu":46,"õtt":671,"cea":61,"õte":379,"õta":58,"õtm":273,"õtl":100,"õtj":101,"õus":427,"õut":47,"ch ":578,"ces":126,"õua":73,"cen":106,"õue":77,"õud":589,"õug":83,"cel":80,"õuk":759,"õul":55,"õun":1596,"ci ":105,"õva":82,"õve":54,"cha":386,"chw":86,"õtü":68,"chu":131,"cia":158,"ck ":487,"cie":98,"che":615,"chl":138,"chi":411,"cho":153,"chm":45,"chn":57,"chs":85,"cht":224,"chr":50,"cis":100,"cin":92,"cm ":52,"cke":185,"cka":54,"õzs":53,"ed ":5646,"eba":352,"ebe":121,"ebi":253,"ebl":44,"ebo":77,"ebr":684,"ebu":73,"eab":210,"eaa":424,"eag":132,"eae":122,"ead":2747,"eak":483,"eaj":63,"eai":99,"eah":87,"ean":733,"eao":48,"eal":1914,"eam":934,"ear":190,"eas":399,"eap":99,"eav":225,"eat":919,"eau":103,"eb ":2987,"ea ":655,"efi":173,"efo":256,"efa":50,"efe":291,"eff":54,"ei ":1558,"ega":4991,"eft":49,"eej":166,"eek":1260,"eeh":72,"een":1833,"eel":7467,"eem":2642,"eeb":812,"eea":50,"eeg":511,"eed":1212,"ees":3817,"eer":4899,"eep":225,"eev":114,"eet":2444,"edi":900,"ede":2096,"eda":2449,"eg ":436,"edu":707,"edo":157,"edr":231,"eck":136,"ech":210,"eci":98,"ee ":3170,"ef ":57,"ect":151,"eco":81,"dwi":78,"dvu":88,"dwa":83,"dy ":101,"dve":99,"duv":409,"duu":64,"dur":309,"dut":208,"dus":9107,"dva":107,"dor":276,"doo":134,"don":412,"dom":146,"dol":150,"dok":340,"dow":80,"dov":88,"dot":70,"dos":91,"dr ":97,"ds ":128,"dmi":448,"dmu":204,"dne":328,"dni":181,"dnu":82,"dsu":120,"dso":79,"dte":403,"dun":212,"dum":683,"dul":361,"duk":455,"õbe":86,"dub":375,"dua":181,"dud":1109,"dri":1541,"drh":50,"dra":532,"dt ":79,"dre":399,"du ":1711,"dro":385,"dru":114,"dsi":69,"dsa":56,"dse":425,"dha":80,"dge":90,"dgl":50,"dic":84,"did":1413,"dia":1187,"der":1254,"des":3846,"det":1487,"dev":307,"deb":74,"dea":150,"ded":103,"def":181,"dee":433,"deg":743,"dei":268,"del":2624,"dek":762,"den":1386,"dem":499,"dep":286,"deo":437,"di ":4266,"dle":97,"dla":684,"dko":283,"dki":67,"dme":789,"dma":248,"do ":334,"dlu":270,"dli":419,"dja":309,"div":260,"diu":154,"dim":305,"din":5058,"dio":737,"dip":162,"dir":107,"dis":3167,"dit":1143,"die":158,"dif":137,"dig":325,"dii":121,"dik":678,"dil":754,"dka":57,"dju":74,"eKr":290,"rgu":1208,"rhe":141,"näd":70,"rha":144,"näg":90,"näh":356,"näi":1177,"rhi":386,"när":92,"rho":51,"näo":109,"rga":2008,"ri ":5455,"rgk":63,"rgl":133,"rgi":2791,"rgh":68,"rge":1513,"rgs":57,"rgr":60,"rgo":122,"rgm":113,"rgn":141,"ret":637,"res":3334,"rev":282,"reu":99,"müü":230,"rfa":51,"rfe":44,"rfi":65,"rfo":51,"rdu":636,"rds":305,"rdr":70,"rg ":613,"reb":76,"rea":973,"ree":2061,"ref":247,"rec":94,"red":566,"rei":666,"reg":889,"rem":1488,"ren":1721,"rek":855,"rel":1476,"rer":134,"reo":148,"rep":269,"rda":591,"rcu":58,"rdo":137,"rdn":203,"rdk":56,"rdm":90,"rdl":149,"rdi":1642,"rde":1014,"re ":3338,"rbu":230,"rbr":301,"rch":187,"rce":98,"rd ":981,"rao":70,"rap":222,"mür":81,"rar":113,"ras":2388,"müt":281,"rat":2331,"rau":610,"rav":669,"rbi":388,"rbl":48,"rbo":167,"rba":241,"rbe":348,"raj":969,"rai":575,"rah":2331,"rag":255,"ran":3582,"mün":66,"ram":1127,"ral":1912,"rak":1264,"rab":296,"raa":4053,"raf":126,"rae":588,"rad":970,"rac":160,"rpu":103,"rpr":48,"rpo":141,"rs ":316,"rpe":48,"rpa":108,"rr ":55,"rpi":60,"ror":118,"ros":627,"rot":969,"rom":675,"ron":932,"roo":2596,"rop":430,"rou":140,"rov":1089,"row":56,"rob":239,"roa":117,"rod":447,"roc":250,"roj":238,"roi":109,"rol":630,"rok":255,"rof":315,"roe":51,"roh":391,"rog":576,"rno":131,"rns":89,"rnu":569,"rna":702,"riü":55,"rež":141,"rne":1143,"rni":528,"rjä":49,"rmo":236,"rms":58,"rmu":175,"ro ":398,"rma":1727,"rme":627,"rmi":820,"rlo":160,"nää":74,"rli":332,"rld":67,"rle":265,"rla":365,"rn ":318,"rgõ":53,"rkv":211,"rku":265,"rkt":136,"rks":128,"rkm":44,"rko":1059,"rki":431,"rkk":66,"rke":333,"rka":216,"rm ":305,"rju":905,"rjo":50,"rja":2847,"rje":754,"riz":44,"rl ":207,"rip":347,"rio":550,"rir":293,"rit":4861,"ris":3546,"riv":668,"riu":963,"rih":61,"rig":747,"rij":404,"rii":5626,"ril":3640,"rik":3181,"rin":3123,"rim":2490,"ria":2118,"rib":484,"ric":609,"rid":1612,"rie":633,"rif":87,"rk ":868,"rtü":56,"rug":66,"rue":78,"rud":172,"ruc":52,"rup":396,"run":460,"rum":712,"rul":170,"ruk":513,"ruu":777,"ruv":117,"rus":1487,"rut":261,"rva":1548,"rvi":795,"rve":1159,"rvp":144,"rvl":46,"rvu":1089,"rwa":50,"ry ":413,"rsk":243,"rsi":519,"rso":171,"rsc":53,"rsa":171,"rsh":62,"rse":561,"rta":274,"rv ":755,"rst":515,"rss":168,"rsu":190,"rtl":107,"rtm":58,"rtn":92,"rto":217,"rte":1543,"rth":281,"rti":1255,"rub":87,"rua":490,"rts":1053,"rtr":65,"rtu":1315,"rtt":57,"rt ":810,"rro":167,"rri":784,"rre":476,"rra":2127,"ru ":1080,"rry":114,"rru":164,"saa":4775,"sab":140,"sad":632,"sae":148,"sag":380,"sah":226,"sai":500,"saj":656,"sak":2002,"sal":1925,"sam":2292,"sbe":92,"sap":131,"óni":51,"san":1221,"sau":105,"sat":1244,"sas":2183,"sar":1496,"sav":213,"sa ":3636,"ón ":53,"rze":44,"nõu":458,"nõr":64,"nõl":71,"nõi":66,"rvü":53,"sha":261,"sho":138,"she":80,"shi":370,"si ":5360,"sgr":60,"sja":511,"siv":324,"sju":179,"sjo":145,"sfä":198,"sie":142,"sid":2303,"sic":71,"sib":74,"sia":2206,"sk ":538,"shu":57,"sit":1318,"siu":225,"sir":212,"sis":2918,"sip":251,"sin":1981,"sio":5046,"sil":1686,"sim":1606,"sij":144,"sik":2762,"sih":204,"sii":1362,"sif":138,"sig":349,"sda":47,"sdo":45,"sbo":51,"sbu":115,"se ":21232,"sca":191,"sci":49,"sch":453,"sco":130,"sev":717,"sey":55,"ser":1068,"ses":5796,"set":1978,"seu":254,"nüü":243,"sh ":174,"sfi":86,"sfo":68,"sea":1804,"sei":2222,"seh":77,"seg":1695,"see":2074,"sed":3928,"seb":254,"sep":903,"seo":645,"sen":2101,"sem":2170,"sel":10065,"sek":3272,"sej":332,"spu":121,"spo":772,"spr":203,"spe":493,"spi":353,"spa":687,"skü":66,"sot":518,"sou":51,"sov":164,"skõ":91,"sol":473,"som":95,"son":907,"soo":2060,"sop":124,"sor":585,"sos":135,"sog":196,"sof":91,"skä":80,"soa":50,"soc":67,"sob":104,"su ":776,"sri":388,"sre":70,"sra":250,"st ":19438,"smõ":57,"smä":672,"ss ":961,"sli":1453,"slo":247,"slu":141,"sla":864,"sle":179,"ski":1330,"skk":688,"skj":46,"skm":251,"skl":222,"sko":1611,"skn":98,"skp":86,"sks":70,"skr":240,"sku":1933,"skv":335,"sm ":509,"ska":1203,"ske":1189,"sno":114,"sjõ":49,"snu":95,"sna":125,"sni":235,"sjä":141,"sne":888,"smo":245,"smu":122,"so ":126,"sma":1250,"smi":1155,"sme":256,"swi":78,"ssü":155,"ssö":85,"ssõ":135,"stõ":73,"stö":240,"svä":103,"svõ":164,"svö":80,"sse":2400,"ssa":886,"sso":712,"ssk":48,"ssi":2279,"ssu":260,"sst":65,"ste":12441,"stf":123,"spä":99,"sta":12430,"stm":389,"stn":305,"sto":736,"sti":11558,"stj":94,"stk":190,"stl":1467,"stv":203,"stu":4098,"str":2584,"sts":374,"sua":230,"sud":110,"sub":2124,"suh":654,"sug":926,"sul":1089,"sum":385,"suk":274,"sup":112,"sun":473,"suu":3246,"sut":3768,"sus":4456,"sur":769,"suv":746,"sva":1352,"sve":77,"svi":156,"svo":72,"svu":249,"sy ":64,"tai":1177,"taj":1586,"tak":4223,"tal":2877,"tae":166,"taf":74,"tag":732,"tah":420,"taa":2158,"tab":1730,"tac":49,"tad":1273,"tba":65,"tav":3721,"tau":131,"tat":7322,"tas":2453,"tar":1636,"tap":159,"tao":157,"tan":2647,"tam":2975,"tch":74,"te ":12807,"ta ":4265,"Ñто":56,"ÑÑ‚Ñ€":66,"Ñтв":57,"pa ":804,"Ñта":51,"pe ":459,"par":1885,"küt":108,"pat":370,"küs":137,"pas":347,"pau":44,"pad":102,"paa":872,"pab":98,"pag":125,"pae":47,"pak":331,"kül":2089,"pal":1654,"pai":924,"paj":61,"pap":44,"küm":165,"pan":1223,"kün":49,"phe":92,"läb":569,"pha":122,"lät":92,"pho":86,"phi":121,"läh":1032,"pi ":489,"küü":67,"ph ":74,"lä ":65,"pea":2994,"peb":54,"pec":50,"ped":165,"pen":309,"per":2019,"pet":866,"pes":209,"pee":735,"pej":56,"pei":56,"pel":221,"pek":193,"pla":985,"pli":241,"lää":1062,"ple":335,"plo":204,"pko":106,"тан":48,"phy":67,"pia":480,"pid":668,"pie":55,"pig":83,"pii":2460,"pik":984,"pil":982,"pim":106,"pin":1660,"pio":63,"pir":110,"pis":249,"pit":216,"por":922,"pop":279,"poo":3433,"pot":178,"pos":489,"poi":69,"poj":56,"pom":75,"pon":250,"pok":78,"pol":2011,"poe":314,"ps ":115,"ppu":86,"ppi":265,"ppl":73,"ppo":76,"ppa":134,"ppe":738,"тер":49,"pme":107,"po ":62,"pni":140,"pne":50,"pp ":583,"psu":187,"pta":45,"pse":289,"psi":131,"pso":44,"ptu":183,"pub":112,"pud":66,"pte":686,"pti":246,"pto":149,"pts":116,"pra":973,"pru":93,"psa":102,"pu ":143,"pri":927,"pre":785,"pro":2923,"psü":231,"lõh":137,"pur":257,"pus":270,"put":156,"pun":658,"pui":197,"pul":391,"puh":552,"px ":81,"тов":47,"puu":696,"тор":53,"lõp":469,"lõi":316,"lõu":988,"тро":97,"löö":149,"lüm":359,"lül":150,"lük":99,"lüh":844,"lüü":275,"mäe":703,"mäg":380,"män":1220,"mäl":265,"mär":1565,"mää":557,"qua":76,"que":158,"qui":88,"mõe":265,"mõi":1422,"mõj":443,"mõl":89,"mõn":454,"mõr":58,"mõt":400,"mõõ":559,"ra ":2421,"möö":180,"rb ":55,"ngo":283,"ngj":112,"eži":152,"ngi":2407,"ngl":2197,"ngk":485,"ngv":53,"ngu":1971,"ngr":466,"ngt":111,"ngs":158,"ni ":8348,"nge":1413,"ngh":131,"nga":1383,"ngd":44,"nho":71,"jäl":102,"jät":125,"jär":2920,"nha":193,"jäi":81,"nhe":51,"neg":216,"nei":559,"nel":854,"nek":392,"nen":1253,"nem":2691,"nep":231,"neo":254,"ner":1383,"net":1372,"nes":1547,"nev":1928,"neu":153,"ndv":49,"ng ":5293,"nea":209,"neb":1735,"ned":454,"nee":1283,"nfi":63,"nfo":399,"nfl":72,"ney":104,"nez":55,"nfe":71,"nco":82,"nci":160,"nce":327,"nch":174,"ne ":18973,"nbu":170,"ndt":47,"ndu":3781,"ndr":1767,"nds":245,"ndn":73,"ndo":567,"ndl":703,"ndm":736,"ndj":199,"ndk":76,"ndi":9574,"nde":2285,"nda":5981,"ncy":54,"nak":847,"nal":1444,"iül":54,"nam":1141,"nan":791,"nao":524,"nap":666,"nar":696,"nac":71,"nad":869,"nae":164,"naf":95,"nag":661,"nah":165,"iüh":71,"nai":421,"naj":121,"nab":249,"naa":1844,"nbe":157,"nd ":4383,"nba":54,"nav":684,"nau":173,"nat":1014,"nas":5608,"na ":9461,"iõp":167,"ê°€ ":51,"nya":69,"jõe":1151,"nyi":68,"jõg":658,"nz ":90,"nsü":49,"ny ":159,"nux":47,"nve":213,"nuk":391,"nul":392,"num":581,"nun":66,"nui":73,"nuj":150,"nus":1407,"nut":211,"nuv":56,"nur":279,"nua":567,"nud":6307,"nto":594,"ntn":44,"ntu":1152,"nts":3190,"ntr":629,"nti":1551,"nth":107,"ntl":53,"nta":1333,"nte":1954,"nsu":194,"nsp":183,"nso":192,"nst":1654,"nss":87,"nse":418,"nsh":72,"nsi":398,"nsk":423,"nsa":519,"nsb":47,"nu ":581,"nri":214,"nra":73,"nt ":1663,"npr":44,"ns ":389,"nod":52,"nob":45,"nog":106,"nof":50,"nok":61,"nol":406,"noi":73,"noo":957,"nop":56,"nom":241,"non":176,"not":137,"nos":276,"nor":508,"nov":777,"nr ":71,"nne":1409,"nna":9091,"nnm":60,"nno":142,"nni":2211,"nnu":1256,"nns":51,"nme":49,"nma":125,"ndž":60,"nli":99,"jää":730,"nn ":2722,"nla":220,"no ":545,"nke":134,"nkl":53,"nki":267,"nka":177,"nku":165,"nko":182,"nks":69,"nkt":721,"nkr":123,"nji":51,"nja":120,"njo":51,"nij":103,"nii":935,"nih":114,"nig":327,"nif":70,"nie":199,"nid":1206,"nic":153,"nib":61,"nia":1970,"nk ":386,"nix":49,"niu":138,"niv":309,"nis":5167,"nit":1025,"nir":49,"nio":206,"nip":222,"nim":5797,"nin":5067,"nik":3655,"nil":1691,"ogs":64,"ogr":1310,"ogu":2080,"ogt":71,"ogi":2325,"ogl":51,"ogo":115,"ogn":110,"oga":201,"oge":329,"oi ":80,"oht":1247,"kät":121,"käs":428,"kär":67,"ohv":181,"ohu":379,"ohk":157,"ohj":63,"käi":618,"ohi":105,"oho":123,"ohn":161,"oha":1453,"käe":74,"ohe":234,"ois":197,"oir":72,"oiu":71,"oit":264,"oin":102,"oik":52,"oim":998,"oil":59,"oid":535,"ok ":100,"oia":64,"oju":137,"oje":372,"oja":659,"ol ":1399,"oce":44,"och":80,"oci":105,"ock":501,"oco":61,"obs":62,"obu":202,"oe ":109,"ode":1123,"odk":64,"odi":1047,"odo":182,"odn":64,"ods":77,"odr":58,"of ":621,"oda":609,"oel":103,"oeg":287,"oer":126,"oes":47,"oet":688,"oen":399,"odu":2480,"oee":44,"og ":463,"ofi":557,"oft":95,"ofo":118,"off":59,"ofe":202,"oa ":93,"ob ":139,"oan":65,"oam":51,"oal":156,"oak":80,"oaj":111,"oba":165,"od ":481,"oar":70,"oas":64,"obo":116,"obr":390,"obl":782,"obj":309,"obi":400,"obe":361,"jõk":47,"jõu":952,"nza":48,"nze":88,"nzi":54,"kõi":997,"otü":52,"oya":45,"osü":192,"ows":73,"own":56,"kõr":1655,"kõv":81,"oyl":61,"kõl":116,"kõn":322,"otu":561,"ow ":114,"otl":94,"otk":48,"otj":93,"oti":581,"oth":115,"ote":630,"ott":288,"ots":2879,"otr":134,"oto":793,"otm":209,"ost":1657,"osu":161,"ota":435,"otb":45,"ov ":391,"osi":662,"osh":53,"osk":556,"ose":1045,"osf":200,"osp":69,"oss":687,"osm":201,"osl":192,"oso":739,"osn":672,"oy ":48,"owa":44,"owe":70,"ovi":1506,"ovg":80,"ovn":57,"ovo":225,"ovu":110,"ovs":142,"ox ":49,"ova":496,"ove":767,"oug":79,"oui":114,"oul":81,"oun":169,"oup":53,"ous":121,"our":234,"out":116,"opo":355,"opp":88,"opi":377,"opk":103,"opl":127,"ope":659,"oph":152,"opa":1002,"os ":1676,"olü":473,"opu":229,"opr":90,"opt":153,"ops":80,"oon":7646,"ool":6457,"oom":3730,"ooj":448,"ook":1060,"oof":591,"oog":2684,"ood":3903,"oob":808,"ooa":118,"or ":1504,"oov":321,"oot":2071,"oos":2409,"oor":2416,"oop":1217,"ork":293,"orl":106,"orm":1403,"orn":581,"oro":441,"orp":251,"orr":1904,"orc":46,"ord":2640,"ore":923,"orf":173,"org":2150,"ori":3030,"orj":99,"ou ":125,"osa":4585,"onü":106,"ort":1076,"ors":316,"orv":260,"oru":308,"ory":79,"omä":44,"ot ":233,"m² ":198,"omö":62,"orb":147,"ora":643,"ola":1318,"old":420,"on ":39161,"olj":64,"oli":8483,"oll":1578,"olk":349,"olf":126,"ole":2988,"kää":76,"olg":132,"ols":1007,"olt":1144,"olm":1122,"oln":368,"olo":2600,"olu":1563,"olv":65,"ofü":93,"oka":297,"om ":370,"okk":531,"oki":231,"oke":623,"okr":399,"oks":957,"oko":247,"okl":55,"okt":664,"oku":340,"ona":1303,"ond":3491,"onf":139,"one":1232,"ong":640,"oni":6425,"onl":72,"onk":273,"onn":4600,"ono":803,"onp":75,"onr":44,"ons":816,"ont":1242,"onu":172,"onv":154,"ony":92,"oma":3656,"oo ":462,"ome":2088,"omb":266,"omi":1618,"omm":613,"omp":670,"omn":106,"omo":342,"omt":108,"omu":606,"oms":246,"op ":270,"la ":5991,"le ":12620,"lf ":96,"ldg":52,"lde":484,"lda":2949,"ldo":95,"ldn":65,"ldm":61,"ldk":243,"ldj":62,"ldi":712,"ldu":1916,"ldt":53,"lds":102,"ldr":58,"laa":1373,"lab":404,"lac":145,"lad":1522,"lae":732,"lah":1640,"lag":388,"laj":204,"lai":1062,"lal":666,"lak":450,"lan":3578,"güm":58,"lam":1233,"lap":307,"lao":158,"lar":440,"kyÅ":50,"lat":2238,"las":6761,"lau":1302,"lav":840,"lay":76,"lba":164,"ld ":1107,"lbe":196,"lbi":58,"lbo":45,"lbu":264,"lbr":50,"kvi":101,"kve":117,"kva":933,"kuv":271,"kuu":2381,"kut":1032,"kus":4511,"kur":510,"kup":256,"kun":2636,"kum":1199,"kul":2990,"kuk":399,"kuj":1401,"ky ":44,"kta":184,"kte":370,"ksp":172,"ksu":1368,"kst":539,"ksk":77,"ksj":60,"ksi":1286,"ksh":44,"kso":186,"ksn":51,"ksm":52,"ksl":97,"kub":299,"kud":842,"kug":145,"kuh":220,"kui":2011,"kua":93,"ktr":1077,"kts":955,"ktu":547,"kti":1734,"kto":1139,"gõz":53,"ksü":131,"kyl":45,"lpo":71,"lpe":157,"lpi":167,"lph":47,"ls ":120,"llü":71,"lpt":60,"lol":75,"lok":121,"lon":328,"lom":526,"lop":192,"loo":6341,"lor":264,"lod":57,"loc":108,"loe":826,"loh":62,"log":273,"loi":81,"los":787,"lot":130,"lov":353,"lkõ":156,"lni":82,"lne":1147,"lob":91,"lnu":266,"lmn":103,"lmi":1561,"lme":771,"lma":2491,"liõ":138,"lmu":534,"lms":87,"lti":513,"lto":103,"ltr":71,"lts":498,"ltu":933,"lud":201,"lub":1064,"lua":79,"lug":367,"lue":62,"lsi":290,"lsk":106,"lso":81,"lss":61,"lst":139,"lsu":210,"lv ":77,"lta":252,"lte":588,"lri":102,"lu ":1464,"lse":1684,"lsa":658,"ía ":44,"lt ":7949,"lra":58,"lhu":155,"häv":119,"lho":73,"här":83,"lhe":124,"häi":143,"lha":82,"lgu":794,"lgs":300,"lgn":56,"lgl":54,"lgr":91,"lgo":86,"lgp":735,"lge":1193,"lgi":247,"li ":8571,"lga":1157,"lfr":46,"lfi":78,"lfa":51,"ley":127,"lex":145,"leu":72,"lev":2102,"les":6911,"let":1229,"ler":467,"leo":191,"lep":522,"lem":2382,"len":1713,"lek":2924,"lel":939,"lei":662,"lej":650,"leh":741,"leg":615,"lef":123,"lee":784,"led":276,"lec":56,"leb":814,"lea":143,"lg ":112,"llp":58,"lls":97,"llu":829,"lly":101,"lo ":301,"lla":3994,"lle":7577,"hää":354,"lli":4723,"llk":46,"llo":330,"lko":770,"lku":106,"lka":667,"lke":205,"lki":380,"lkl":73,"lju":697,"ljo":186,"lm ":614,"lje":847,"ll ":1528,"lja":2809,"lit":3006,"lis":12972,"lir":127,"lip":448,"lio":191,"lin":10479,"lim":654,"liz":82,"liv":165,"liu":274,"lic":156,"lid":1460,"lia":1378,"lib":211,"lk ":259,"lik":8420,"lil":1193,"lii":5503,"lij":133,"lig":496,"lih":443,"lie":227,"lif":188,"hõõ":63,"ma ":4860,"mb ":129,"maa":11983,"mac":100,"mab":268,"mah":237,"чеÑ":55,"mai":1139,"maj":980,"mak":919,"hüd":167,"mad":1582,"mae":114,"maf":51,"mag":417,"hüp":189,"map":146,"mar":885,"mas":3246,"mal":2280,"hüm":55,"mam":324,"man":2399,"mav":524,"mat":3086,"mba":251,"mbl":329,"mbi":269,"mbe":2044,"mbr":1530,"mbo":263,"me ":2925,"mbu":182,"med":1125,"mee":3676,"meg":594,"mea":100,"met":4426,"mev":67,"mep":72,"mes":2849,"mer":2169,"mem":157,"mel":1042,"men":2636,"mei":782,"meh":405,"mek":592,"hüü":97,"lva":713,"lve":678,"lvi":208,"lul":347,"luk":457,"lup":150,"luo":49,"lun":505,"lum":505,"lut":683,"lus":3490,"lur":311,"luv":1330,"luu":781,"ly ":163,"lrü":75,"lvk":62,"hõb":45,"hõi":206,"hõl":310,"mpi":452,"mph":51,"mpe":554,"mpr":66,"mpo":299,"mpl":310,"mpu":89,"mps":61,"ms ":217,"mog":59,"mob":84,"moe":58,"mod":191,"moo":1204,"mon":658,"mok":297,"mom":61,"mol":409,"mov":44,"mor":367,"mos":365,"mot":173,"mpa":252,"msa":75,"mu ":287,"mse":406,"mtu":96,"mud":268,"mub":235,"mst":77,"msu":90,"mso":47,"msi":100,"mte":76,"my ":109,"mur":342,"mus":2450,"mut":357,"muu":2318,"muv":182,"mui":212,"muj":55,"muk":66,"mul":772,"mum":89,"mun":762,"džu":46,"dža":146,"mi ":3313,"dži":112,"mjo":107,"min":5859,"mio":159,"mil":5861,"mim":121,"mir":250,"mis":17764,"mip":65,"miv":72,"mit":2152,"mic":74,"mia":602,"mig":164,"mie":74,"mid":2544,"mik":1090,"mii":356,"mo ":151,"mli":87,"mle":44,"mla":87,"mn ":55,"mko":173,"mka":66,"mm ":329,"mnu":50,"mni":67,"mna":105,"mne":450,"mmy":82,"mp ":57,"mmu":480,"mmi":519,"mmo":72,"mma":633,"mme":654,"tÅ¡ ":386,"tÅ¡e":222,"tÅ¡i":133,"tÅ¡a":111,"zst":65,"võt":836,"võs":96,"võr":892,"võn":91,"võl":153,"või":7974,"võe":178,"vöö":291,"võõ":64,"vür":134,"zi ":155,"väl":2397,"väh":435,"väi":1090,"väg":389,"väe":751,"zen":74,"zer":96,"ze ":109,"zab":50,"uüh":53,"zan":66,"zar":62,"zon":61,"zo ":48,"vää":478,"vär":503,"väs":44,"zia":44,"zin":86,"yst":121,"ysi":65,"ys ":119,"ylä":47,"za ":111,"tüü":554,"yer":56,"ya ":162,"töö":1589,"tüt":130,"tür":79,"yan":90,"tük":102,"tüh":106,"yn ":51,"yle":91,"ylo":45,"yne":47,"yin":76,"tõe":238,"tõm":70,"tõl":200,"tõk":57,"tõu":195,"tõs":109,"tõt":177,"tõr":100,"tän":706,"täh":1822,"täi":727,"xi ":91,"süü":243,"xim":52,"tär":60,"täp":255,"söö":193,"süs":1324,"süt":71,"sük":585,"sül":70,"süm":231,"sün":4272,"xan":102,"süd":144,"süh":336,"süg":181,"sõl":377,"sõj":1120,"sõp":113,"sõn":939,"sõd":233,"sõi":351,"sää":83,"ws ":61,"wor":58,"rüü":199,"wer":86,"wen":48,"säi":178,"wit":44,"wig":128,"wic":50,"win":68,"rõh":178,"röö":217,"wa ":114,"wan":110,"rün":141,"rüo":64,"wal":97,"rük":165,"way":55,"rüt":71,"war":196,"rüh":567,"vri":47,"vsu":116,"vst":161,"vse":403,"vsk":184,"vu ":181,"vut":793,"vus":2887,"vud":63,"vum":73,"vuk":112,"vul":243,"vy ":55,"vib":72,"via":291,"vio":50,"vip":54,"vir":136,"vik":721,"vil":657,"vim":530,"vin":1205,"vig":116,"vih":89,"vii":1360,"vic":51,"vid":522,"vie":84,"viv":99,"vit":1084,"vis":990,"vka":64,"vko":169,"vkj":47,"vla":98,"rää":402,"vli":75,"vo ":249,"vms":66,"vne":367,"vna":66,"voj":65,"vol":333,"vok":90,"von":554,"voo":711,"vor":540,"vos":69,"vpa":151,"rän":173,"vi ":1273,"vgo":75,"veo":115,"ver":1595,"ves":902,"vet":333,"vei":261,"veg":47,"ven":2478,"vem":547,"vel":383,"vek":127,"vea":44,"vee":1876,"ved":504,"ve ":854,"val":8945,"vak":669,"van":2026,"vam":407,"vap":226,"var":1747,"vat":2727,"vas":3460,"vav":166,"vaa":1097,"vab":718,"vae":151,"vad":3860,"vai":1190,"vaj":399,"vag":98,"vah":2967,"va ":3296,"uvõ":268,"uvä":105,"usõ":228,"usü":545,"uuk":89,"uun":1174,"uul":3063,"uum":754,"uub":214,"uua":104,"uug":75,"uud":1301,"uue":262,"ux ":63,"uus":2149,"uur":5973,"uup":59,"uuv":44,"uut":1103,"uvi":326,"uvo":58,"uva":2014,"uve":246,"uvu":186,"usl":1435,"usm":668,"usj":225,"usk":1361,"ush":270,"usi":2532,"usf":104,"usg":54,"usd":49,"use":12729,"usc":44,"usa":1306,"uu ":547,"uv ":1217,"usv":989,"usu":604,"ust":10829,"uss":984,"usr":193,"usp":536,"uso":240,"usn":79,"utk":47,"utl":151,"utm":97,"utn":50,"uth":113,"upä":61,"uti":1297,"ute":1328,"uta":4946,"utz":51,"upõ":82,"utt":121,"uts":1366,"utv":93,"utu":2313,"uto":959,"utr":121,"us ":14520,"umä":125,"ut ":606,"urb":400,"ura":699,"urd":430,"ure":1850,"urg":757,"urj":163,"uri":3709,"url":67,"urk":267,"urm":383,"urn":372,"uro":1163,"urr":134,"urs":299,"urt":336,"uru":948,"urv":239,"uol":59,"uot":45,"uor":83,"uos":126,"ukü":84,"upa":311,"ur ":1500,"upi":356,"upe":239,"upo":146,"upp":283,"upr":65,"upl":50,"upu":118,"ump":59,"umu":239,"umi":3810,"umm":293,"umo":96,"uma":2384,"umb":1389,"ume":800,"unt":871,"uns":729,"unu":969,"unk":808,"uni":2914,"uno":48,"unn":618,"unc":54,"und":1856,"una":2903,"ung":491,"ujä":52,"une":1105,"up ":125,"uks":640,"ukr":117,"uku":265,"ukt":617,"uko":1820,"ukk":131,"ukl":81,"uki":384,"uke":268,"um ":1368,"uka":800,"uju":1494,"ulu":2649,"ult":2105,"uls":130,"ulp":70,"ulo":139,"uln":45,"ulm":184,"ull":465,"ulk":645,"ulj":361,"uli":2783,"ulg":972,"ule":2072,"uld":337,"ula":1652,"un ":198,"uid":849,"uie":51,"uig":91,"uil":85,"uim":112,"uin":284,"uis":365,"uht":1356,"uhu":769,"uk ":199,"uiv":119,"uit":151,"ul ":2180,"uja":426,"ugh":62,"ugi":162,"uge":590,"ugo":56,"ugl":46,"ui ":1419,"uga":910,"uhi":560,"uhe":222,"uho":49,"uhk":168,"ugu":2195,"ugr":50,"uha":362,"uct":46,"uda":566,"ude":2760,"udi":687,"udm":209,"ubu":163,"uca":46,"ue ":313,"uci":50,"uch":100,"uck":66,"uet":68,"uer":113,"ues":193,"püü":159,"ufo":48,"ufi":80,"udu":959,"uds":80,"udt":385,"udo":126,"ug ":76,"udw":51,"uee":109,"uen":148,"uel":155,"pöö":270,"ub ":3808,"põõ":56,"ua ":229,"uat":112,"uas":207,"püs":236,"uar":1172,"pür":52,"uam":44,"ual":222,"uan":196,"ubi":508,"ubj":82,"ubl":189,"ube":338,"uba":529,"ud ":15469,"uak":53,"püh":249,"uai":64,"uad":62,"uaa":347,"tvõ":44,"tvä":73,"tzi":57,"tze":47,"põh":2641,"põi":53,"põl":539,"põr":58,"ty ":254,"trü":150,"tvu":92,"tvo":55,"tve":93,"tva":221,"tur":775,"tus":7900,"tut":545,"tuu":2287,"tuv":618,"tuj":83,"tui":121,"tul":1640,"tuk":346,"tun":1878,"tum":1084,"tup":139,"tub":376,"tua":183,"tud":7345,"tuh":72,"tug":512,"tsü":333,"tz ":121,"tsõ":45,"ts ":1169,"tre":529,"tt ":402,"tra":1815,"tri":3128,"tru":752,"tro":1685,"tu ":1887,"tsa":1214,"tse":5653,"tsc":70,"tsi":9068,"tsj":165,"tsm":159,"tsk":220,"tsl":83,"tso":358,"tsu":2329,"tst":212,"tta":440,"tte":2371,"tti":343,"ttl":46,"tto":143,"ttp":55,"tts":70,"ttu":276,"tme":1276,"tma":236,"to ":641,"tmu":55,"tmo":97,"tmi":756,"tni":437,"tne":300,"tp ":49,"tna":202,"tnu":77,"tno":51,"tof":48,"toe":213,"tod":315,"toc":161,"toi":1340,"toh":46,"tog":150,"toa":50,"tov":120,"tos":269,"tot":150,"tom":456,"ton":1159,"tok":224,"tol":1335,"tor":2218,"too":2528,"top":205,"tkü":52,"tij":48,"tii":1991,"til":2148,"tik":5067,"tif":152,"tie":142,"tih":290,"tig":255,"tir":123,"tit":780,"tis":3212,"tin":1198,"tim":859,"tip":427,"tio":925,"thu":237,"tia":704,"tic":223,"tid":1567,"tiu":100,"tiv":496,"tja":582,"tki":109,"tko":214,"tku":135,"tka":199,"tke":169,"tli":1813,"pää":190,"tlu":1138,"tla":898,"tle":1553,"tem":2042,"ten":994,"teo":974,"tep":235,"tei":2648,"tej":88,"tek":2590,"tel":4094,"tee":4577,"tef":53,"teg":3418,"teh":952,"tea":2198,"tec":57,"ted":509,"tfo":51,"tfa":78,"th ":373,"tev":754,"tet":709,"tes":4048,"ter":4969,"ti ":9417,"tho":135,"thr":60,"pär":1925,"päe":892,"the":496,"thi":183,"päi":92,"tha":245,"Än ":65,"üüp":380,"üür":188,"üüs":557,"üüt":421,"üüa":70,"üüb":92,"üüd":395,"üüg":68,"üüh":62,"üüm":176,"üül":234,"üün":56,"並 ":57,"žik":69,"žis":93,"三 ":165,"žii":61,"žan":88,"ä¸ ":49,"žaa":84,"ži ":55,"üve":65,"ürs":166,"ürt":60,"üro":72,"ürk":57,"ürg":241,"üri":378,"üre":55,"üra":58,"üs ":51,"ütt":66,"ütu":70,"üti":151,"ütl":183,"ütm":69,"üto":339,"üta":158,"üte":90,"üss":79,"üst":1343,"üsi":1015,"之 ":88},"n_words":[4341644,4941492,4175920],"name":"et","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/eu.json b/contrib/languages-data/eu.json
new file mode 100644
index 0000000..ca99f7e
--- /dev/null
+++ b/contrib/languages-data/eu.json
@@ -0,0 +1 @@
+{"freq":{"D":12636,"E":31626,"F":41733,"G":24671,"A":49254,"B":32314,"C":24793,"L":25422,"M":31193,"N":19703,"O":18871,"H":20234,"I":17795,"J":9363,"K":12674,"U":5646,"T":16850,"W":3928,"V":9991,"Q":1167,"P":21838,"S":30092,"R":9657,"Y":2494,"X":2077,"Z":5511,"f":39789,"g":194786,"d":496450,"e":1172759,"b":212072,"c":39575,"a":1770525,"n":845494,"o":639351,"l":395235,"m":170469,"j":17374,"k":463043,"h":111048,"i":893736,"w":6134,"v":18456,"u":464998,"t":693891,"s":298290,"r":847892,"q":2827,"p":125668,"z":349493,"y":24758,"x":24609,"²":121,"´":110,"É":768,"Ã":135,"í":1797,"ì":278,"ë":206,"ê":267,"é":6456,"è":4480,"ç":407,"ä":628,"â":543,"á":1849,"à":612,"ü":636,"ú":522,"ù":141,"ö":568,"ô":3132,"ò":128,"ó":1337,"ñ":2145,"Ä“":126,"Ä":699,"Ä«":353,"Å":492,"Å¡":115,"Å«":427,"ÇŽ":126,"Ì":383,"μ":233,"ν":549,"ο":715,"ι":506,"κ":273,"λ":400,"δ":171,"ε":392,"η":259,"α":749,"γ":157,"ά":203,"ί":245,"ω":158,"ÏŒ":170,"σ":279,"Ï‚":638,"Ï":505,"Ï€":199,"φ":112,"Ï…":181,"Ï„":391," l":25023,"ÑŒ":254," m":32235," n":11168," o":23087,"Ñ":187," h":48767," i":53849," j":11234,"Ñ‹":136," k":40202," d":213721," e":179842," f":18132," g":36773,"ч":225,"Ñ€":831," a":103410,"Ñ":671," b":117209,"Ñ‚":523," c":1597,"у":285," y":441," x":499," z":66161," u":47027," t":18277," w":614," v":880," q":216," p":25257," s":29279," r":2003,"К":143,"Ð":116,"Ð’":111," J":9317," K":12636," H":20099," I":17702," N":19654," O":18798," L":25277," M":31072," B":32085," C":24661," A":48997,"С":176," F":41666," G":24576," D":12574," E":30981,"л":615," Z":5479,"к":647," Y":2491,"й":238," X":2059,"и":1033,"п":143,"о":1160,"н":760,"м":252," S":29950,"г":226," R":9631,"в":628," Q":1162,"б":218," P":21762,"а":1315," W":3894," V":9966," U":5599,"е":865," T":16758,"д":267," Ã":134," É":748,"ו":119,"×™":145,"Ùˆ":342,"ÙŠ":584,"Ù":132,"Ù‚":122,"Ù„":670,"Ù…":459,"Ù†":404,"Ù‡":130,"د":321,"ج":158,"Ø­":170,"ت":166,"ب":429,"Ø©":205,"ا":1078,"ع":171,"Ø´":113,"س":238,"ز":110,"ر":498," Ð":116," Ð’":111," К":143,"A ":2297,"F ":196,"Da":1856,"Ct":233,"Cu":578,"Cy":474,"Cl":878,"Cn":233,"Co":5762,"Cr":1202,"Ce":1047,"Ch":5833,"Ci":545,"G ":191,"Ec":112,"Ed":572,"Ea":166,"Eb":131,"Du":1120,"Do":3494,"Dr":849,"De":2359,"Di":1558,"Fe":1310,"H ":330,"Fa":940,"Ez":530,"Eu":5638,"Ev":124,"Ex":216,"Er":7909,"Et":707,"Es":7502,"En":1222,"Em":589,"Ep":195,"Ei":284,"El":2242,"Ek":1089,"Eg":1307,"Ge":4595,"Ga":8150,"I ":1890,"Fu":637,"Fr":34398,"Fo":1536,"Bé":204,"Fl":778,"Fi":1608,"B ":768," С":176,"C ":551,"Av":693,"Au":3356,"Ar":7948,"At":1721,"As":1845,"D ":286,"Ba":9918,"Az":821,"Ay":124,"Af":752,"Ag":995,"Ah":190,"Ab":1086,"Ac":453,"Ad":9951,"Am":3102,"An":4722,"Ap":642,"Ai":1869,"Ak":667,"Al":6525,"Bu":1690,"Br":3204,"Ca":6336,"E ":494,"Bi":4638,"Be":8067,"Bo":3491,"Bl":641,"Ku":829,"Kl":325,"Kr":801,"Ko":3438,"Le":5234,"Li":3143,"N ":436,"La":7002,"Lu":2001,"Ly":440,"Hé":453,"Ll":255,"Lo":6499,"Me":5309,"Mi":2877,"O ":483,"Ma":13695,"Mu":2429,"Iñ":132,"Mo":5680,"Ni":1359,"Ne":2222,"Na":11411,"P ":389,"Nu":354,"No":2161,"Ok":8891,"Ol":1020,"Om":122,"On":616,"Oh":168,"Oi":1025,"Od":228,"Of":128,"Ob":229,"Gi":3338,"Gl":406,"Gr":2264,"Go":2393,"Gu":2262,"Gy":276,"Cô":768,"Gw":139,"J ":172,"Ha":6267,"He":6657,"Hi":2597,"Ho":2280,"Hu":1033,"Hy":385,"Dô":481,"K ":469,"Ib":698,"Id":259,"Ig":331,"ा":121,"Ip":1284,"Im":160,"In":4524,"Ik":340,"Il":578,"Iv":141,"Is":1295,"It":2278,"Ir":2852,"Ja":2843,"L ":365,"Iz":862,"Ji":328,"Je":1151,"Jo":2759,"Fé":111,"Ju":1950,"Ka":4370,"M ":322,"Kh":198,"Ki":1106,"Ke":755,"Us":185,"Ut":139,"Ur":1743,"Un":1309,"Uk":242,"Ul":214,"Uh":348,"Ud":554,"Ty":618,"Tx":2595,"Tu":924,"Tr":2492,"Pé":178,"To":2471,"Th":1746,"Ti":995,"Te":1911,"Ta":2499,"V ":486,"Sy":260,"St":1667,"Su":1641,"Wo":340,"Wi":1463,"Wh":118,"Sé":155,"Sè":286,"Wa":880,"We":721,"Vo":900,"Vi":3787,"Ré":151,"X ":613,"Va":2746,"Ve":1775,"Uz":154,"Mé":297,"Qi":124,"Pt":120," Ù…":129,"Pu":1219,"Pr":2635,"Ps":373,"S ":435,"Py":117,"Pe":2734,"Pa":5996,"Lé":212,"Pl":1086,"Po":3314,"Pi":2764,"Ph":767,"Os":761,"Ot":444,"Ou":336,"Ov":109," ا":406,"Op":333,"Or":2924,"R ":202," ب":155,"Ox":154,"Oz":622,"Se":3621,"Sc":1754,"Si":1956,"Sh":1018,"Sl":127,"Sk":128,"Sp":738,"So":2835,"Ru":564,"U ":162,"Sa":12778,"Re":1715,"Ri":1242,"Rh":1366,"Ro":2679,"Qu":659,"T ":234,"Ra":1463,"b ":834,"a ":418310,"Ye":164,"Ya":233,"Yo":1349,"Yv":208,"Yu":192,"QÅ«":205,"Xe":154,"Xa":263,"Xi":357,"Za":1318,"Ze":1350,"Zh":191,"Zi":1026,"Zo":305,"Zu":1064,"i ":75419,"gd":272,"ge":26042,"ga":31469,"gb":147,"bé":122,"fl":595,"fg":114,"ff":773,"fi":7825,"fr":4122,"fu":1726,"ft":461,"fo":4955,"j ":153,"gy":318,"gz":158,"he":25799,"ha":32541,"gn":3188,"gm":257,"gl":1391,"gi":27050,"gh":1165,"gg":388,"gu":20475,"gt":292,"gs":391,"gr":5318,"cí":127,"go":71226,"dt":177,"du":58038,"dw":263,"dy":552,"g ":4240,"ea":90696,"eb":7278,"ec":2369,"ed":20845,"de":134965,"dd":530,"dg":314,"di":46366,"dh":268,"dm":9929,"dl":187,"do":27634,"dn":167,"ds":538,"dr":5710,"ew":1215,"aô":1078,"ex":2335,"añ":358,"eu":10850,"ev":2468,"ey":2501,"ez":30732,"fa":13204,"h ":3096,"fe":4135,"eh":10866,"eg":27624,"ef":2297,"ee":7857,"el":32829,"ek":58090,"ej":999,"ei":12396,"ep":39259,"eo":5671,"en":302618,"em":15409,"et":97162,"es":80312,"er":181118,"aí":122,"ca":4335,"e ":126190,"by":276,"bs":905,"br":4958,"bu":10932,"bn":150,"bo":17651,"bj":339,"bl":4003,"bg":166,"bh":132,"bi":50340,"bb":248,"bd":240,"be":37939,"da":201401,"f ":1461,"cy":423,"cu":1624,"ct":2467,"cs":124,"cq":317,"cr":939,"co":4763,"ck":1858,"cl":657,"ci":5204,"ch":8436,"ce":4449,"cc":457,"c ":2780,"aB":147,"aE":312,"az":38405,"ay":2915,"ba":82298,"d ":7993,"at":138353,"as":38078,"ar":234987,"aq":229,"ax":1187,"aw":605,"av":3400,"au":28656,"ak":137603,"al":150790,"ai":71296,"aj":772,"ao":1106,"ap":9383,"am":53259,"an":300148,"ac":7614,"ad":15049,"aa":638,"ab":24555,"ag":67873,"ah":5467,"ae":11540,"af":5163,"nu":3790,"nt":87610,"ns":6135,"nr":517,"nq":161,"np":6374,"no":18233,"nn":4720,"q ":190,"nz":1004,"ny":2628,"nx":162,"nw":144,"nv":608,"oe":35138,"of":3518,"oc":3225,"od":6707,"oa":47947,"ob":8716,"om":15442,"on":56967,"ok":34406,"ol":29688,"oi":15827,"oj":392,"og":7848,"oh":2732,"hÅ":150,"m²":115,"ot":18274,"os":23317,"ov":2405,"ou":8969,"op":11236,"oo":1321,"or":55477,"oq":186,"r ":35138,"ox":827,"ow":1113,"oz":6401,"oy":1012,"lá":134,"pe":17100,"pa":48349,"pl":4223,"lè":695,"lé":417,"po":11630,"ph":5789,"pi":13999,"lo":23311,"ln":225,"lm":2191,"hé":316,"ll":14813,"ls":1249,"lp":1461,"lw":166,"lv":1388,"lu":13777,"lt":11988,"lz":205,"ly":1619,"hô":322,"o ":239621,"ià":195,"ma":42350,"mb":1990,"mg":110,"me":60276,"iá":151,"hÄ":111,"iè":1602,"mi":30640,"mn":549,"mm":1879,"ié":152,"mp":2538,"mo":12040,"mt":139,"ms":406,"mu":11394,"iñ":300,"ió":290,"iö":115,"my":536,"p ":1016,"na":89833,"nb":5719,"nc":5864,"nd":77531,"ne":54150,"nf":1744,"ng":20200,"jä":211,"nh":729,"ni":56445,"nj":622,"nk":3770,"nl":20585,"nm":259,"ju":1296,"jo":3182,"ki":41699,"kh":562,"ke":15259,"ka":61958,"m ":4321,"ky":241,"kz":9751,"ks":428,"kt":5287,"ku":47447,"ko":209669,"kr":3078,"kk":1095,"kl":2132,"km":634,"kn":545,"li":47744,"lh":523,"lk":11786,"lj":263,"le":105272,"ld":63524,"lg":2023,"hâ":367,"lf":1145,"la":65016,"lc":483,"lb":3705,"n ":374467,"hr":1439,"hs":188,"hw":153,"ht":906,"hu":4025,"hi":26055,"hn":657,"ho":11118,"hl":1392,"dé":379,"hm":308,"id":28391,"ic":6403,"ib":22289,"ia":177930,"ih":1356,"ig":10103,"if":3391,"ie":37824,"hy":1010,"k ":62634,"iq":528,"ir":40658,"is":41795,"it":88626,"iu":1861,"eñ":229,"iv":1736,"iw":111,"ix":1809,"ii":901,"ij":1221,"ik":65069,"il":61174,"im":14806,"in":101260,"io":30374,"ip":9370,"je":1468,"ji":1295,"iz":65709,"iy":270,"l ":19248,"ja":9289,"xk":501,"xi":8569,"xo":2798,"té":228,"tí":194,"xt":269,"xu":1487,"wy":210,"ww":215,"z ":36063,"xa":3213,"xe":3776,"wi":721,"sè":421,"sé":400,"wn":243,"wo":287,"ws":184,"vy":165,"rô":347,"y ":9063,"wa":1578,"we":945,"rè":474,"ré":955,"vi":5771,"vu":143,"vr":1081,"vs":130,"rí":340,"vo":1538,"uz":19517,"uy":977,"ux":2209,"uv":1191,"ve":4565,"va":4080,"x ":3431,"ui":4962,"uj":404,"uk":15052,"ul":13460,"ue":46850,"uf":629,"ug":6475,"uh":2527,"ur":58487,"us":31658,"ut":35746,"um":6170,"un":31293,"uo":392,"up":1366,"ty":2264,"tx":11953,"tz":106075,"tu":90564,"tt":2560,"tw":311,"ub":8862,"ua":92243,"ud":39922,"uc":1798,"w ":1243,"to":23073,"tn":499,"tm":498,"tl":1525,"ts":16546,"tr":26770,"tp":196,"tg":181,"tf":199,"te":85168,"ti":60433,"th":5295,"v ":481,"tb":891,"tc":244,"ta":203261,"su":14353,"oñ":152,"sv":152,"ss":4629,"st":55835,"sy":515,"sz":245,"sw":227,"sl":3070,"sk":52536,"sn":1906,"sm":3054,"sp":8612,"so":18734,"sr":212,"sq":219,"sd":220,"sc":2520,"sf":603,"se":20022,"sh":2489,"sg":1046,"sj":125,"si":25419,"rz":597,"u ":41051,"sa":30547,"sb":1326,"rr":133906,"rs":4346,"rt":84154,"ru":29410,"rv":1353,"rw":161,"rx":118,"ry":1952,"rq":209,"rp":2429,"ro":53014,"rn":9800,"né":358,"rm":7397,"rl":6227,"rk":10937,"nç":272,"rj":253,"ri":132692,"rh":504,"rg":8992,"rf":596,"ná":137,"re":124545,"rd":16275,"rc":2417,"rb":5362,"ra":172109,"t ":54109,"qu":2270,"mé":209,"qi":160,"s ":48513,"py":188,"pz":138,"pt":2020,"pu":7547,"pp":706,"lí":113,"pr":9929,"ps":2339,"zè":218,"zz":285,"zg":277,"zh":568,"zi":104084,"zb":829,"ze":72397,"za":55517,"zy":159,"zu":15760,"zt":33176,"zo":6743,"zp":2277,"zk":17416,"zl":2872,"yg":376,"ye":1014,"yc":830,"་":163,"yd":772,"ya":1650,"yu":249,"uñ":753,"yt":384,"ys":908,"yr":1417,"yp":1406,"yo":623,"yn":833,"ym":693,"ué":150,"yl":2141,"yk":360,"yi":1221,"Éc":146,"Ét":205,"àn":218,"án":687,"ái":148,"ál":138,"ár":132,"ât":411,"à ":197,"á ":169,"アアア":219,"ö ":127,"ôm":838,"ôn":1427,"ón":676,"ó ":189,"ña":982,"ñe":634,"ño":321,"ín":439,"ío":159,"ís":174,"ía":492,"í ":198,"éz":196,"él":236,"éo":163,"ém":241,"én":464,"és":320,"ét":218,"ér":1074,"év":257,"èz":354,"éa":185,"éd":183,"éc":258,"ée":453,"ég":228,"èn":113,"èr":1733,"ès":933,"èv":648,"èg":419,"ço":189,"ça":162,"é ":1538,"är":266,"Än":289,"ür":178,"ú ":187,"ôt":781,"Ä« ":109,"Ä«n":117,"Ån":111,"Å ":172,"Å« ":285,"ã‚¢":394,"ä¹™":208,"之":886,"丹":146,"临":277,"並":364,"丘":302,"专":624,"三":1377,"ä¸":715,"万":373," 丘":117," 专":142," 三":359," ä¸":261,"倉":119," 之":199,"大":136,"ος":294,"ος ":290,"Ï‚ ":631,"ν ":159,"α ":219,"アア":304,"Ñк":207,"ÑÑ‚":182," Ga":8127," Ge":4583," I ":500," Fo":1533," Fu":636," Fr":34376," Fi":1595," Fl":773," Bé":203," Ha":6216," He":6623," Cô":768," Gw":139," Gy":275," J ":124," Go":2376," Gr":2249," Gu":2252," Gi":3327," Gl":404," Ig":331," Id":259," Ib":692," K ":344," Hy":384," Dô":481," Hu":1024," Ho":2274," Hi":2583," Ji":326," Je":1147," L ":160," Ja":2821," Iz":849," Iv":140," Ir":2841," Is":1292," It":2274," Im":160," In":4497," Ip":1284," Ik":324," Il":575," M ":119," Ka":4363," Ke":748," Ki":1096," Kh":198," Fé":110," Jo":2739," Ju":1944," N ":227," La":6984," Le":5208," Li":3050," Kl":325," Ko":3426," Kr":791," Ku":827," Ma":13659," O ":208," Mi":2855," Me":5297," Lo":6484," Hé":453," Ll":255," Ly":440," Lu":1998," Ne":2204,"а ":256," Na":11386," Ni":1355," Mo":5669," Mu":2390," Iñ":129," A ":442," B ":131," C ":208," Ap":641," Am":3098," An":4660," Ak":544," Al":6496," Ai":1863," Ag":994," Ah":190," Af":752," Ac":451," Ad":9947," Ab":1037," Ba":9875," Az":814," Ay":124," Av":691," Au":3286," At":1715," As":1839," Ar":7825," Be":7803," Bi":4537," Bl":636," Bo":3476," Br":3193," Bu":1682," Ca":6284," Ce":1039," Ci":542," Ch":5831," Cn":231," Cl":859," Cr":1191," Co":5729," Cu":573," Ct":233," Cy":473," Da":1841," Di":1550," De":2338," Dr":847," Do":3472," Du":1118," Ea":161," Eb":127," Ec":112," Ed":572," El":2228," Ek":1058," Ei":282," Eg":1300," Et":695," Es":7460," Er":7880," Ep":193," En":1209," Em":587," Ez":526," Ex":211," Eu":5218," Ev":121," Fe":1297," Fa":936," H ":110," Xi":356," Xe":150," Xa":262," Wo":331," Sé":155," Sè":285," Wi":1455," Wh":115," We":715," Wa":873,"й ":179," Zu":1054," Zo":304," Ze":1333," Zh":191," Zi":1025," Za":1318," QÅ«":205," Yv":208," Yu":192," Yo":1348," Ya":232," Ye":164,"н ":112," a ":8305,"в ":122," Oz":621," Ox":152," Ou":336," Ov":109," Os":758," Ot":443," Or":2903," Op":333," Po":3289," Lé":212," Pl":1082," Pi":2760," Ph":764," Pe":2717," Pa":5978," Nu":351," No":2154," Ol":1015," Ok":8850," On":612," Om":121," Oh":168," Oi":1023," Od":227," Of":119," Ob":227," Ra":1451," Qu":655," Ro":2671," Re":1711," Ri":1240," Rh":1366," Py":112," Pr":2625," Ps":372," Pt":120," Pu":1219," Qi":123," Mé":297," Sy":259," Su":1637," St":1657," Ta":2484," V ":128," Th":1731," Ti":988," Te":1895," Tr":2477," To":2450," Pé":178," Ru":561," Sa":12748," Sh":1015," Si":1935," Sc":1747," Se":3599," So":2830," Sp":735," Sk":123," Sl":127," Uz":153," Va":2741," X ":118," Ve":1772," Vi":3778," Ré":151," Vo":900," Tu":917," Ty":618," Tx":2595," Ud":553," Uh":347," Uk":211," Ul":214," Un":1298," Ur":1698," Us":185," Ut":139," ja":7967," l ":317," iz":22098," je":446," ip":2762," im":117," in":7560," ik":2938," il":408," is":831," it":3666," ir":5256," ka":6507," m ":200," ki":4014," ke":272," jo":2357," ju":380," ha":15787," he":11845," gi":2915," gl":237," gr":2766," go":3412," gu":3064," ib":2762," ia":184," id":4126," ih":124," ig":680," hi":13695," ho":6235," hu":1046," ni":298," ne":1575," na":6665," mu":7219," mo":3550," ok":389," ol":651," om":239," on":4235," oh":1483," oi":1278," od":151," of":1484," ob":692," nu":269," no":1940," le":6898," li":3011," n ":330," la":8148," ku":1542," km":546," kl":781," kr":1091," ko":25212," me":9646," mi":3328,"Ñ ":139," ma":8141," lu":3558," lo":2463," af":170," ag":1571," ah":1271," ab":5206," ad":2544," am":1664," an":37224," ap":2038," ai":1625," ak":945," al":7726," au":6652," ar":17725," at":1816," as":2486," d ":1527," ba":61785," az":4149," bi":31183," be":18081," bo":1808," bl":186," bu":3139," br":903," ca":373," e ":115," er":27809," et":48874," es":44512," en":3242," em":2495," ep":416," el":3773," ek":9675," eh":379," eg":10971," fe":664," fa":8966," ez":8057," eu":4631," ex":330," fu":1398," fr":2586," fo":1255," fl":294," fi":2901," ge":12607," ga":11697," i ":197," co":450," cr":121," ch":186," da":131136," do":2083," dr":141," de":49829," di":15772,"ч ":164," ed":12800," ea":1283," eb":259," du":13144," zo":627,"ла":160," zu":10484," za":4120,"ко":120," ze":24755," zi":26100,"ка":189,"ин":111,"ич":173," xe":141," xa":124,"ро":162,"ра":136,"ов":295,"но":113," ru":136," sa":10788," se":2907," sc":161," si":3486," sh":276," sp":132," so":7153," qu":182,"ви":184," ra":630," re":402," ri":131," ro":580," pu":1255," pr":7926," ps":237," s ":207," os":6513," ot":1043," op":380," or":3982,"ан":144," ox":142," oz":163," pe":3345,"аÑ":113," pa":3424," pl":1311," lè":553," po":4955," pi":2498," we":277," y ":323," x ":114," va":276," ve":153," uz":1119," vo":221," vi":176," ud":33375,"ер":141," tx":4570," tu":424," us":349," ut":439," ur":7726," um":156," un":1012," uk":258," ul":113," ug":588," uh":1778," ta":4238," st":216," su":3722,"ев":111," tr":2688," to":1288," th":540," ti":1534," te":2817," Ét":203," Éc":146,"ê°€":192,"د ":145,"Ø© ":203,"ان":117,"ال":397,"ÙŠ ":165,"Ù† ":205,"가가":128,"Fed":255,"Fel":133,"Fen":110,"Fer":565,"Fil":503,"Fin":531,"Fis":168,"Ezk":195,"Fal":128,"Far":152,"Era":611,"Erd":679,"Ere":283,"Eri":170,"Erk":205,"Ess":209,"Est":2782,"Esp":2765,"Err":5471,"Ern":111,"Esc":150,"Esk":1235,"Eus":2856,"Eur":2345,"Etx":413,"Ez ":126,"El ":382,"Ele":228,"Ela":199,"Eki":655,"Eko":224,"Emi":244,"Ema":110,"Elo":131,"Eli":450,"Elk":412,"Ent":296,"Enp":155,"Eno":127,"Gek":939,"Gel":126,"Geh":141,"Ger":1702,"Get":138,"Geo":759,"Gen":514,"Gla":168,"Gin":141,"Gip":1442,"Gil":209,"Gir":724,"Giz":302,"Gaz":575,"Gan":238,"Gal":1278,"Gam":113,"Gau":462,"Gar":4067,"Gas":607,"Gai":282,"Gab":184,"Fut":140,"Fun":140,"Fro":202,"Flo":355,"Fla":260,"Fra":33371,"Fri":275,"Fre":391,"Fon":318,"For":678,"Fou":190,"II ":948,"Hir":971,"Hit":208,"His":306,"Hiz":289,"Hig":126,"Hil":265,"Hip":128,"Heg":983,"Hel":358,"Hei":184,"Hez":131,"Hem":285,"Hen":358,"Hes":115,"Her":3806,"Hal":393,"Hai":353,"Han":786,"Ham":279,"Has":208,"Har":1000,"Hau":2610,"Gym":213,"Côt":760,"Gur":199,"Guz":741,"Gua":381,"Gue":128,"Gui":276,"Gre":836,"Gri":154,"Gra":882,"Gro":247,"Glo":115,"Gon":350,"Gol":169,"Goi":319,"Got":125,"Gou":140,"Gor":655,"Gob":162,"Ing":1090,"Inf":149,"Inp":655,"Int":602,"Ins":196,"Ipa":1240,"Ill":286,"Ind":1394,"Ika":163,"Ign":110,"Ida":112,"Iba":263,"Ibe":221,"Hyd":269,"Dôm":479,"Hur":125,"Hun":272,"IX ":263,"IV ":188,"Hor":488,"Hou":158,"Hot":121,"Hom":153,"Hon":528,"Hol":339,"Arg":594,"Are":343,"Arc":161,"Ard":862,"Ara":1880,"Arb":146,"Aro":492,"Arm":450,"Arn":154,"Ark":223,"Ari":603,"Apo":201,"Ate":172,"Ath":126,"Atl":685,"Atr":335,"Ast":557,"Ass":118,"Ata":133,"Ash":115,"Asi":444,"Ask":171,"Asp":121,"Arr":879,"Art":881,"Avi":122,"Ave":440,"Auz":146,"Aut":408,"Aus":617,"Aur":352,"Aud":619,"Aug":208,"Aub":618,"Aza":115,"Azp":147,"Azk":248,"Bai":588,"Bak":203,"Bal":660,"Ban":555,"Bad":228,"Baz":195,"Bay":133,"Bar":2121,"Bat":3332,"Bas":1042,"Bav":137,"Bau":120,"Aca":159,"Abe":336,"Aba":192,"Ada":158,"Adm":9191,"Adi":259,"Aga":387,"Afr":600,"Agu":144,"Agi":187,"Ain":131,"Ais":678,"Air":369,"Ait":206,"Aiz":109,"Al ":169,"Aka":163,"Aki":200,"Ala":306,"Alb":472,"An ":143,"Alg":117,"Ali":138,"Alj":125,"Ald":917,"Ale":1684,"Alf":227,"Alt":401,"Alm":128,"All":581,"Alo":136,"Alp":727,"Ame":1656,"Ami":117,"Ama":372,"Amo":119,"Amp":481,"Ang":544,"Ana":309,"And":973,"Ant":1538,"Ano":454,"Ann":178,"Apa":132,"Bus":154,"Bul":138,"Bur":532,"Bud":110,"Bue":186,"Bru":377,"Cab":148,"Cad":128,"Cal":1977,"Cam":634,"Cai":113,"Cas":692,"Car":985,"Cau":156,"Cat":248,"Can":647,"Cap":209,"Bea":612,"Bes":339,"Bet":367,"Ber":3897,"Ben":412,"Bel":1201,"Bei":190,"Beh":504,"Bid":263,"Bie":141,"Bib":135,"Bia":112,"Biz":1309,"Bil":1194,"Big":333,"Bis":146,"Bir":303,"Bio":163,"Bi ":209,"Blo":124,"Bla":371,"Bre":742,"Bra":786,"Bro":433,"Bri":733,"Bol":414,"Boi":402,"Bon":360,"Bor":485,"Bos":337,"Bot":216,"Bou":715,"Cyr":266,"Cur":113,"Der":132,"Des":202,"Deu":416,"Del":171,"Dem":199,"Den":399,"Deb":142,"EB ":427,"Dam":258,"Dan":487,"Dar":179,"Dat":205,"Dav":212,"EBe":213,"Dal":132,"Cho":209,"Chr":241,"Che":1052,"Chi":641,"Cit":117,"Châ":343,"Cle":128,"Cla":350,"Cel":174,"Cen":305,"Cer":368,"Cha":3073,"Cri":128,"Cra":131,"Cre":406,"Cro":283,"Cte":231,"Clu":134,"Cne":218,"Cos":145,"Cor":1166,"Com":567,"Col":1978,"Con":778,"Cou":568,"Drô":329,"Egu":578,"Egi":536,"Edi":139,"Dia":142,"Dis":216,"Dip":284,"Dio":112,"Din":121,"Die":165,"Dub":337,"Dun":167,"Dur":225,"Dra":223,"Dou":638,"Don":1289,"Dom":330,"Dor":804,"Neu":356,"Ner":122,"Nat":287,"Nav":137,"Naz":944,"Nig":111,"Nic":160,"Nik":161,"Nil":110,"New":858,"Nap":194,"Nar":6638,"Nan":255,"Nag":310,"Naf":1923,"Niè":290,"Nov":113,"Nor":1256,"Not":129,"Nob":153,"Ois":735,"Ohi":123,"Okz":8717,"Oze":547,"Oli":468,"Ola":182,"Ono":139,"Ond":186,"Or ":694,"Ope":128,"Ora":139,"Ort":156,"Ore":134,"Ord":215,"Ori":184,"Org":128,"Orn":527,"Oro":257,"Oso":146,"Ple":223,"Pla":722,"Pin":245,"Pil":160,"Pir":1128,"Phy":140,"Pie":535,"Pic":114,"Pia":119,"Phi":231,"Phr":148,"Ped":255,"Per":814,"Pet":525,"Pen":376,"Pel":246,"Pat":265,"Pas":1075,"Par":2304,"Pau":404,"Pac":140,"Pan":351,"Pal":580,"Pue":117,"Puy":565,"Pro":1149,"Pri":464,"Pre":388,"Pru":183,"Pse":223,"Pra":311,"Pol":1076,"Pom":111,"Pon":385,"Poi":164,"Pot":148,"Pos":133,"Pou":137,"Pop":124,"Por":717," ال":346,"Rai":109,"Ram":376,"Ran":155,"Qui":200,"Que":317,"Irl":1084,"Iri":152,"Iru":781,"Ita":1023,"Isl":300,"Isr":115,"Ist":111,"Ira":616,"Isè":386,"Itu":232,"Its":696,"Itz":194,"Iza":331,"Ize":360,"Jac":241,"Jav":138,"Jau":467,"Jat":166,"Jar":142,"Jap":555,"Jan":172,"Jam":243,"Jak":114,"Jai":339,"Jer":194,"Jes":231,"Jea":485,"Jos":894,"Jor":132,"Jon":241,"Jok":251,"Joh":504,"Joa":335,"Jua":434,"Jus":123,"Jur":582,"Jul":330,"Jun":115,"Kai":138,"Kam":110,"Kal":524,"Kap":120,"Kan":885,"Kat":1063,"Kas":158,"Kar":856,"Kaz":123,"Ker":134,"Ken":177,"Kem":181,"Kir":246,"Kin":214,"Kil":130,"Kim":109,"Kla":112,"Kon":1335,"Kom":296,"Kol":432,"Kos":152,"Kor":646,"Kop":181,"Kre":127,"Kri":277,"Kro":231,"Kul":139,"Kur":185,"Let":186,"Les":516,"Ler":221,"Lep":592,"Leo":391,"Len":128,"Lem":125,"Lek":123,"Lei":412,"Leg":497,"Leh":498,"Lea":127,"Lau":560,"Lav":171,"Le ":868,"Lak":109,"Lag":212,"Las":259,"Lat":204,"Lar":562,"Lap":493,"Lam":255,"Lan":1366,"Lac":384,"Lab":307,"MA ":123,"La ":1524,"Hér":386,"Lib":309,"Lif":129,"Lie":116,"Lig":406,"Lim":186,"Lin":293,"Lio":514,"Lis":187,"Lit":211,"Liv":131,"Liz":228,"Luz":124,"Lur":628,"Lui":341,"Luc":199,"Loz":203,"Lou":492,"Los":296,"Lot":763,"Loi":2917,"Log":117,"Lor":274,"Lon":848,"NA ":1300,"Lyg":192,"Men":1204,"Mel":250,"Mes":401,"Mer":455,"Meu":1170,"Met":319,"Med":480,"Mex":492,"Man":1508,"Mal":618,"Mar":6226,"Mas":420,"Mag":310,"Mad":1098,"Mak":109,"Mah":123,"Mai":913,"Mac":300,"May":341,"Max":125,"Maz":151,"Mau":432,"Mat":548,"Mod":145,"Mol":190,"Mon":2148,"Mos":1550,"Mor":687,"Mou":317,"Mot":148,"Mik":294,"Mig":214,"Mic":585,"Mit":157,"Mir":210,"Mis":141,"Mil":404,"Min":480,"Mun":792,"Mul":124,"Mut":117,"Mur":267,"Mus":488,"Mug":257,"Xià":110,"Xab":116,"ège":392,"XX ":192,"Wor":153,"Sèv":275,"èvr":599,"Wik":612,"Wil":429,"Win":144,"ère":1699,"Wes":294,"Was":125,"War":190,"Wal":229,"ès ":918,"之三":148,"ée ":379,"éco":116,"Vos":480,"Vol":182,"èze":351,"Vit":132,"évi":152,"Zor":154,"Zeh":114,"Zee":161,"Zar":377,"Zah":179,"Zal":136,"éli":109,"éra":508,"ére":128,"éri":214,"Zie":209,"Zib":154,"Zin":153,"Zel":160,"Zen":285,"Zer":305,"és ":183,"Zab":118,"QÅ« ":204,"Yve":193,"Yor":688,"Yon":444,"三三":257,"三之":120,"ä¸ä¸‰":109,"Sur":139,"Sul":122,"Sue":239,"Sui":315,"Str":342,"Sto":163,"Sta":507,"Ste":414,"Ten":223,"Teo":128,"Tei":132,"Tel":274,"Tan":225,"Tar":886,"Tai":174,"Tak":137,"Tal":334,"Shi":203,"She":176,"Sha":388,"Sim":208,"Sil":273,"Sis":185,"Sir":222,"Sin":296,"Sie":134,"Sib":121,"Ser":500,"Sev":217,"Saô":1077,"Sen":362,"Sel":120,"Sem":121,"Sei":1420,"Seg":141,"Sph":400,"Soc":136,"Sob":257,"Sou":293,"Soz":190,"Sol":243,"Som":871,"Son":154,"Sor":289,"Sai":5007,"Sam":232,"Sal":766,"Sab":129,"Sco":113,"Sci":1122,"Sch":335,"Sax":145,"Sav":636,"Sat":139,"Sau":359,"Sar":1149,"Sas":116,"San":2399,"ови":110,"Res":126,"Rhi":762,"Rha":154,"Rio":118,"Riv":186,"Ria":109,"Rib":112,"Ric":281,"ärv":206,"Rec":114,"Rei":155,"Reg":177,"Rem":135,"Ren":337,"Rea":142,"Rob":311,"Roc":325,"Rod":194,"Roy":117,"Rou":411,"Ros":328,"Rom":283,"Rhô":310,"Vel":146,"Ven":718,"Vau":379,"Vas":132,"Van":188,"Val":1311,"Var":518,"Vig":138,"Vic":317,"Vie":959,"Vip":162,"Vir":171,"Vil":1162,"Vin":200,"Ver":591,"Ukr":188,"Uha":317,"Uni":1067,"Uro":222,"Urt":224,"Urr":361,"Urk":120,"Uri":138,"Urd":163,"Ura":118,"Txa":393,"Typ":542,"Txi":1766,"Txe":254,"Uda":518,"VI ":183,"Ter":439,"Tes":138,"Tha":159,"The":888,"Thi":181,"Tho":316,"Tib":109,"Tim":132,"Tir":125,"Pér":118,"Tor":335,"Tok":213,"Tol":699,"Tom":137,"Tow":179,"Tou":411,"Tro":782,"Tri":446,"Tre":367,"Tra":727,"Tur":461,"aEu":198,"вич":155,"àn ":157,"bje":319,"biz":22873,"bis":1045,"bit":2242,"bio":874,"bir":655,"bik":563,"bil":6749,"bin":3474,"big":994,"bih":468,"bo ":895,"blo":238,"ble":654,"bli":2524,"bn ":137,"bla":489,"boa":985,"bok":9893,"bol":1370,"boi":169,"boe":180,"bs ":545,"bon":636,"bor":1560,"bot":625,"bos":584,"bou":296,"boz":129,"be ":1452,"ban":2718,"bak":3303,"bal":5966,"bai":7451,"bag":126,"bae":319,"bac":240,"bad":1178,"bab":550,"án ":329,"baz":996,"bax":116,"bat":43650,"bas":1155,"bar":12363,"bea":507,"áng":124,"bi ":1854,"bei":305,"beh":2621,"beg":284,"bed":218,"bec":150,"ber":19208,"ben":1757,"bel":1463,"bek":859,"bez":2002,"bes":5139,"bet":1729,"bia":2180,"bid":4505,"bie":1454,"bgu":159,"ca ":1045,"car":483,"cas":288,"cat":226,"cau":179,"can":540,"cap":126,"cad":117,"cak":136,"cam":158,"cal":436,"cag":240,"ce ":1311,"bri":2477,"bro":342,"bra":732,"bre":1164,"bu ":155,"bru":149,"bst":212,"bur":7454,"bul":964,"buk":216,"bum":126,"but":176,"bus":313,"buz":1067,"by ":112,"aka":3860,"am ":849,"ake":2522,"aki":3275,"akh":163,"ajo":184,"aix":538,"aiz":1199,"al ":8930,"aja":231,"aje":125,"aik":2668,"ail":16466,"aim":225,"ain":22745,"aio":2613,"aip":526,"air":1724,"ais":1887,"ait":6762,"ak ":35410,"aig":330,"aie":1989,"aid":440,"aib":241,"aia":7883,"ahi":1974,"ahu":235,"aho":676,"aha":1972,"agh":132,"agi":3410,"agr":297,"agu":6313,"agn":992,"ago":50304,"aoi":123,"anu":801,"anz":393,"any":212,"anp":1062,"ano":2915,"ann":1154,"ant":46628,"ans":1575,"ane":3569,"anf":114,"ang":4075,"ani":22937,"anj":269,"ank":984,"anl":19832,"ana":6367,"anb":593,"anc":2276,"and":13202,"amu":416,"amm":343,"amo":901,"amn":123,"amp":1235,"ams":133,"ami":9813,"ame":32856,"amb":651,"ama":5578,"ao ":342,"alv":863,"alu":1501,"alt":3287,"als":169,"alp":282,"alo":1294,"alm":822,"all":1873,"alk":1496,"alg":595,"ali":14200,"alc":178,"ald":57921,"ale":42820,"alf":261,"ala":12819,"alb":844,"akz":215,"an ":170348,"akr":254,"aku":2585,"akt":1284,"ako":87760,"aba":5416,"abd":115,"abe":8846,"abi":6317,"abl":275,"abo":739,"abr":604,"abs":119,"abu":1781,"ae ":7943,"aca":415,"aar":137,"ad ":651,"ac ":1453,"ab ":142,"afo":576,"afr":334,"aff":141,"afe":133,"afi":865,"ai ":2679,"aga":4252,"age":1716,"aen":488,"aem":396,"ael":880,"aer":1127,"ah ":241,"âte":302,"afa":2878,"aet":117,"ado":2055,"adr":1175,"adm":643,"adi":3700,"ade":1430,"adu":1159,"acq":143,"aco":349,"ack":319,"aci":488,"ach":1442,"ace":785,"ada":3736,"act":1592,"acu":160,"acr":160,"azp":1356,"azo":562,"azi":7825,"azl":2452,"azk":3543,"azu":329,"azt":5546,"aze":1070,"aza":3082,"azz":159,"axi":278,"axo":276,"axu":131,"az ":12242,"axa":121,"ayo":136,"ays":139,"aya":303,"aye":536,"ba ":1649,"aqu":160,"at ":33651,"arg":3519,"are":45955,"ard":3975,"arc":902,"arb":1084,"ara":18217,"arp":378,"aro":3911,"arn":4158,"arm":1625,"arl":1757,"ark":3693,"anç":220,"ari":25386,"aru":371,"arv":202,"arr":46687,"ars":715,"art":52635,"au ":3167,"asa":2228,"ary":240,"arz":155,"asg":196,"asi":3937,"ash":374,"asc":404,"ase":1361,"aso":2684,"asp":509,"ask":3157,"asm":635,"asl":1054,"aon":151,"ar ":18677,"apa":1371,"ape":2743,"api":1618,"aph":271,"apl":270,"apo":1684,"app":159,"apr":133,"aps":163,"apt":114,"apu":638,"as ":4585,"ava":748,"ax ":148,"auz":2790,"aux":633,"auv":282,"aut":4572,"arí":190,"avo":638,"avi":1105,"ave":561,"ay ":1280,"awa":295,"ata":5981,"asu":3797,"ast":11887,"ass":921,"atm":119,"atl":299,"atr":577,"ato":3893,"ate":18483,"ati":19404,"ath":563,"aua":391,"auc":470,"aub":143,"att":437,"ats":845,"atu":34742,"atx":565,"atz":18521,"aul":1718,"aum":331,"aun":1227,"aup":140,"aur":6720,"aus":1048,"aud":2046,"aue":1117,"aug":815,"auk":876,"Zur":179,"Zuz":173,"Zub":282,"ич ":157,"jer":179,"jek":326,"jel":120,"jen":358,"jab":248,"jat":1353,"jas":739,"jau":579,"jap":648,"jar":1747,"jal":123,"jak":490,"jan":459,"jai":2132,"jaz":157,"joa":281,"joe":132,"jol":254,"jok":1248,"jot":496,"jor":115,"jio":380,"jin":276,"jia":219,"jo ":284,"itm":162,"itr":368,"ito":2098,"itu":32563,"itt":399,"its":2904,"itz":12198,"ity":169,"itx":1037,"eña":150,"isk":1566,"ism":1753,"isl":341,"iso":671,"isn":727,"isp":164,"iss":773,"isu":768,"ist":19392,"ita":22759,"ite":6146,"ith":384,"iti":6015,"ivo":164,"ius":683,"iur":210,"ium":623,"iva":357,"ix ":589,"ivi":458,"ive":562,"ipo":429,"ipp":233,"ipu":2084,"ips":176,"ipt":634,"ipi":478,"ipl":363,"is ":8540,"ion":6034,"iop":428,"ior":622,"ios":459,"iot":1950,"iog":146,"ioi":472,"iok":992,"iol":1372,"ipa":3058,"ipe":1493,"iox":130,"ioz":931,"ir ":1292,"iru":3334,"irt":277,"irr":1309,"iro":2123,"irm":114,"irk":440,"irl":941,"iri":11303,"isi":1862,"ish":426,"ise":2016,"isc":611,"isb":459,"isa":1271,"iqu":500,"ire":8235,"irg":171,"ira":10391,"ird":162,"it ":1135,"ja ":330,"iya":143,"ixo":397,"ixk":401,"ixa":122,"ixe":126,"iz ":3294,"izu":662,"izt":21049,"izp":401,"izo":1014,"izl":132,"izk":5110,"izi":6510,"izg":172,"ize":7802,"iza":19239,"kig":334,"kik":669,"kim":840,"kil":2662,"kia":6057,"kib":168,"kie":1433,"kid":1763,"kiz":579,"kin":6825,"kio":719,"kip":627,"kir":776,"kis":461,"kit":2888,"km ":412,"ki ":14622,"kha":202,"kho":112,"kea":556,"kee":629,"kei":150,"kek":347,"kel":306,"keo":173,"ken":1434,"kes":365,"ker":2731,"ket":6095,"kez":458,"ke ":1678,"kra":849,"kre":293,"ku ":1173,"kro":731,"kri":1134,"koz":874,"kov":156,"km²":111,"kot":1230,"kos":1322,"kor":2545,"kop":563,"koo":115,"kon":8009,"kom":1476,"kol":2022,"kok":4933,"koi":1642,"koe":2023,"kod":227,"kiö":114,"koa":15917,"kob":165,"kni":322,"kno":187,"kko":963,"klu":109,"ko ":166136,"kle":410,"kla":832,"klo":486,"kli":257,"jur":356,"jua":336,"jud":181,"kaz":1190,"kat":17259,"kau":137,"kar":12671,"kas":3232,"kap":656,"kan":3807,"kal":6938,"kam":178,"kak":2781,"kai":3049,"kag":867,"kae":232,"kad":1070,"kab":331,"ka ":7200,"ha ":447,"ham":2422,"han":4212,"hap":332,"hai":2518,"hak":119,"hal":2041,"hau":3280,"har":11639,"has":2581,"hat":743,"hae":346,"hag":159,"hab":197,"had":140,"hac":168,"haz":745,"he ":3675,"hek":291,"hel":1598,"hei":419,"heg":3098,"hec":165,"hed":603,"hea":271,"heb":248,"hez":537,"hev":153,"het":258,"hes":714,"her":9041,"heo":124,"hen":3743,"hem":423,"hi ":2061,"hig":352,"hie":1401,"hid":400,"hic":295,"hia":1281,"hip":234,"hio":194,"hin":1789,"him":161,"hil":1588,"hik":564,"hii":240,"his":2862,"hit":1675,"hir":8155,"hiz":2483,"hn ":341,"hle":145,"hlo":1029,"ho ":201,"gma":140,"go ":26432,"glo":370,"gle":317,"gli":281,"gla":304,"gog":353,"goe":28566,"god":354,"gob":600,"goa":4107,"gny":701,"gno":312,"gni":265,"gne":942,"gna":765,"goz":493,"goi":1856,"gok":1203,"gom":158,"gol":1108,"gon":1318,"gos":518,"gor":3038,"got":736,"gu ":476,"gro":244,"gra":2474,"gri":255,"gre":2219,"gto":194,"gui":372,"gul":293,"gua":626,"gue":1210,"gud":543,"gy ":143,"guz":1681,"gut":1860,"gur":3120,"gus":2569,"gun":7491,"gzh":121,"iak":58472,"iam":434,"ial":8884,"ian":17793,"iap":140,"ias":645,"iar":18221,"iau":120,"iat":2790,"ic ":497,"iab":177,"iac":288,"iad":428,"iag":1598,"ibl":211,"ibi":2091,"ibo":10573,"ibr":326,"ibu":3077,"iaz":1327,"id ":402,"iba":3588,"ibe":2135,"ia ":66121,"iet":4538,"ieu":535,"iev":194,"iez":349,"iel":657,"iem":109,"ien":15589,"ier":10660,"ies":673,"iee":130,"ied":378,"ieg":332,"iei":288,"iek":1321,"ig ":225,"iea":142,"ifo":600,"iff":119,"ife":501,"ifi":1639,"ifa":243,"icr":332,"ict":263,"icu":365,"ico":799,"ick":329,"ici":329,"ich":1326,"ice":649,"ie ":1575,"ica":1267,"idu":909,"idr":287,"ido":1820,"idi":703,"idg":234,"ide":10967,"ida":12783,"iid":341,"il ":2896,"ija":210,"iji":583,"ijo":114,"im ":459,"ika":16573,"ige":670,"iga":3099,"ii ":406,"igl":239,"igh":367,"igi":1126,"igu":1200,"igr":300,"igo":1186,"ign":1443,"ihe":190,"iha":384,"ihu":414,"iho":241,"ik ":16195,"imo":875,"imm":120,"ime":3172,"imi":1266,"inc":1845,"ind":5098,"ina":14551,"inb":1383,"imu":195,"inn":283,"inp":1720,"ino":3873,"int":13017,"ins":1000,"inf":881,"ine":11821,"inh":165,"ing":7209,"ini":17586,"inl":550,"ink":1408,"ioa":6380,"iob":288,"ioc":123,"iod":290,"ioe":954,"inu":1280,"inv":181,"iny":1050,"iko":21260,"ikl":464,"iki":4819,"ike":2410,"ila":10425,"ilb":1403,"in ":15912,"ikz":303,"ikt":193,"iku":2386,"ikr":256,"ilo":4660,"ill":6632,"ilk":7089,"ilm":637,"ilh":252,"ilg":475,"ili":12072,"ild":890,"ile":8785,"ima":8169,"imb":209,"io ":8484,"ils":121,"ilt":2974,"ilu":1485,"hol":600,"hom":373,"hon":2777,"hog":384,"hoi":120,"hos":554,"hot":751,"hou":397,"hop":188,"hor":3428,"hob":344,"hoa":178,"hod":259,"hoc":115,"dée":208,"hua":295,"hth":235,"hry":189,"hro":550,"hri":326,"ht ":301,"hra":124,"hyd":145,"hyl":274,"hy ":139,"hum":243,"hun":759,"hus":732,"hut":205,"hur":1153,"ffe":167,"ffi":176,"fes":515,"fer":1274,"fed":363,"fen":469,"fek":715,"fel":165,"fia":757,"fga":109,"fas":196,"far":2984,"fam":8183,"fan":210,"fak":148,"fal":486,"fab":275,"ff ":133,"fe ":256,"fa ":176,"exu":444,"eyr":541,"exa":425,"ez ":12298,"exo":122,"aôn":1078,"exi":839,"exe":123,"ezu":483,"ezb":637,"eza":6134,"ezp":276,"ezt":764,"eze":550,"ezi":5130,"ezk":3961,"ezl":261,"eta":71146,"ete":2569,"eti":4795,"eth":290,"etn":393,"esp":3014,"esn":776,"eso":556,"est":11662,"esu":1628,"ess":912,"ev ":129,"aña":175,"eud":448,"euf":120,"eui":361,"euk":128,"eul":175,"eun":111,"eto":1392,"etr":5219,"ets":1121,"ett":760,"etu":368,"etx":2177,"etz":1163,"ew ":848,"eve":525,"eva":377,"evo":137,"evi":1107,"euv":304,"eut":239,"eur":2218,"eus":5316,"ex ":208,"eux":721,"ey ":1366,"epe":753,"epi":3559,"eph":603,"er ":6409,"epa":30994,"eot":118,"eor":1271,"eom":283,"eol":759,"eok":178,"eop":146,"eon":704,"es ":12081,"ept":728,"eps":204,"epu":1540,"epo":499,"erk":2593,"erl":1568,"eri":9060,"erg":1467,"ere":16377,"erf":172,"erc":707,"erd":4566,"era":40409,"erb":2356,"et ":5547,"esk":37262,"esl":821,"esm":224,"esf":120,"esh":127,"esi":2406,"esb":376,"esc":377,"ese":3688,"eu ":330,"esa":3808,"erz":150,"ery":187,"erv":411,"eru":942,"err":62362,"ert":11444,"ers":2709,"ern":3239,"erm":1856,"erp":616,"ero":11175,"eki":8183,"ekk":958,"ekl":121,"ekn":504,"eko":32688,"ekr":109,"ekt":3089,"eku":1445,"ekz":206,"en ":169009,"elb":764,"ela":7880,"eld":843,"elf":319,"ele":6128,"eli":2171,"elg":473,"elm":234,"elk":1631,"ell":3887,"elo":1639,"elu":1274,"elv":114,"els":360,"elt":1391,"ely":133,"eo ":860,"emb":233,"ema":8298,"eme":1995,"emo":994,"emi":1594,"emu":1285,"emp":192,"emy":190,"enf":133,"ene":16371,"enh":206,"eng":2970,"enb":2943,"ena":41627,"end":44877,"enc":765,"eno":2247,"enp":1606,"enn":1479,"enk":297,"eni":2624,"enu":163,"ens":1539,"ent":12453,"enr":290,"eoa":338,"enz":249,"eny":137,"eog":358,"eoe":128,"eod":110,"ego":7214,"ege":2822,"egi":10698,"eha":3263,"egr":361,"egu":4291,"eho":226,"ehe":4597,"ehi":2102,"ek ":6903,"eic":110,"eia":366,"ehu":489,"eis":293,"eir":418,"eim":319,"eil":722,"eio":204,"ein":3804,"eik":189,"eij":125,"eie":157,"eid":253,"eig":270,"eja":123,"el ":3223,"eiz":933,"eit":1960,"eke":1121,"eka":2666,"em ":347,"eju":612,"giz":1772,"git":4426,"gis":854,"gir":652,"gil":2535,"gim":660,"gik":1305,"gip":786,"gin":5407,"gio":349,"gid":465,"gie":433,"gia":5348,"ght":231,"ghe":212,"gha":252,"ggi":124,"gi ":1669,"gen":9845,"geo":710,"get":173,"ger":4696,"ges":1287,"gh ":273,"gez":400,"gea":1197,"gee":110,"gei":542,"geh":1437,"geg":119,"gel":2960,"gek":219,"ge ":1979,"gaz":2792,"gab":1043,"gad":132,"gae":256,"gai":4004,"gas":712,"gar":10218,"gau":1336,"gat":2266,"gak":724,"gam":476,"gal":2723,"gan":2119,"ga ":2307,"fur":143,"fus":174,"fut":598,"fun":655,"ftw":197,"ft ":115,"fra":2359,"fre":282,"fri":962,"fro":345,"fru":141,"for":3048,"fos":110,"fon":526,"fol":298,"foa":306,"fle":119,"fla":235,"flu":128,"fo ":357,"fic":126,"fie":126,"fil":1255,"fik":2128,"fin":1035,"fis":784,"fit":221,"fiz":1025,"da ":83681,"dd ":135,"de ":14213,"dac":1167,"dad":591,"dab":383,"dak":4129,"dal":34505,"dai":1813,"dag":46918,"dae":7409,"dat":4371,"das":372,"dar":7697,"dap":284,"dan":1747,"dam":370,"daz":3020,"dau":2348,"ddo":115,"cul":269,"cty":1123,"ctu":376,"cto":282,"cti":251,"cta":224,"cy ":306,"cus":776,"cur":130,"cut":132,"cla":138,"cle":140,"clu":182,"clo":110,"co ":929,"ció":186,"cod":133,"con":497,"col":516,"com":276,"cor":390,"cos":170,"cop":272,"cot":155,"cou":1017,"cqu":221,"cra":135,"cri":123,"cru":146,"cro":386,"cci":190,"cea":325,"ch ":1027,"cey":125,"cer":590,"ces":263,"cen":434,"cep":383,"cel":570,"cha":1044,"chw":110,"chu":371,"chy":392,"cia":581,"ck ":966,"cie":258,"cid":1189,"che":2829,"chl":124,"chi":830,"cho":437,"cht":276,"chr":456,"cil":203,"cis":344,"cin":1476,"cio":340,"cke":298,"ed ":562,"eba":3556,"ebe":1499,"ebg":158,"ebi":808,"ebo":188,"ebr":497,"ebu":253,"ec ":189,"eag":605,"eae":255,"ead":117,"eak":5006,"ean":53210,"eal":880,"ear":5919,"eas":238,"eat":764,"eau":1023,"eaz":319,"eb ":149,"ea ":21907,"efi":386,"efo":313,"efa":131,"efe":1060,"ei ":1830,"ega":1847,"eek":733,"eei":184,"een":2608,"eel":320,"eer":450,"eet":3066,"edi":3117,"edd":167,"ede":1453,"ône":1395,"eda":1186,"edu":758,"edo":12587,"edr":806,"eck":189,"ech":649,"eci":177,"eca":138,"ôme":834,"ee ":210,"ecu":151,"ect":278,"eco":286,"dyl":181,"dwa":144,"dy ":206,"dré":154,"dur":2122,"dut":4392,"dus":429,"duz":254,"dor":3977,"dop":318,"don":1432,"dom":352,"dol":488,"dok":827,"doz":381,"dou":190,"dot":620,"dos":896,"ds ":300,"dmi":9816,"doa":2009,"dob":254,"doc":225,"doe":371,"doi":1104,"dun":1704,"dum":256,"dui":109,"dul":792,"duk":1954,"due":5451,"dug":218,"dua":33103,"dri":1545,"dra":754,"dre":1584,"du ":7012,"dro":1415,"dge":253,"dic":166,"did":286,"dia":9396,"dib":563,"ôte":774,"der":7825,"des":2086,"det":1562,"dez":1959,"deb":3140,"dea":45982,"dec":379,"def":455,"dee":1641,"deg":553,"dei":1488,"del":2329,"dek":7786,"den":11216,"dem":540,"dep":31070,"deo":330,"di ":3931,"do ":13628,"diu":152,"diz":2212,"dim":351,"din":3293,"dio":1674,"dip":225,"dir":6847,"dis":1910,"dit":6852,"die":3603,"dif":136,"dig":733,"dik":2769,"dil":919,"rgu":720,"rhi":113,"rho":158,"rga":1667,"ri ":30074,"rgi":2575,"rgh":113,"rge":1357,"rgo":1529,"ret":4609,"res":7061,"rev":255,"reu":617,"rez":4457,"rey":281,"rfo":144,"rdu":1468,"rds":187,"rdy":168,"rg ":780,"reb":423,"rea":5485,"ree":1399,"ref":864,"rec":235,"red":748,"rei":1098,"reg":3035,"reh":258,"rem":1526,"ren":57161,"rek":7795,"rel":1168,"rer":1714,"reo":216,"rep":4742,"rf ":117,"rda":944,"rcu":273,"rdo":1564,"rdi":6252,"rde":3329,"re ":19143,"rbu":270,"rco":298,"rci":264,"rch":425,"rce":466,"rca":271,"ray":301,"raz":13993,"rd ":1974,"rao":152,"rap":758,"rar":3528,"ras":8250,"rat":19217,"rau":2390,"rav":257,"rbi":2074,"rbo":540,"rba":974,"rbe":1362,"raj":132,"rai":9556,"rah":192,"rag":3181,"ran":42992,"ram":1927,"ral":6492,"rak":9219,"rab":9119,"raf":1044,"rae":295,"rad":2088,"rac":1198,"rpu":497,"rpo":196,"rs ":1994,"rpe":728,"rpa":194,"rpi":260,"rph":358,"ror":761,"ros":1392,"rot":2296,"rom":2243,"ron":4833,"roo":173,"rop":4279,"roy":129,"roz":1007,"rou":490,"rov":593,"row":119,"rob":3623,"roa":6616,"rod":811,"roc":757,"roi":1273,"rol":3561,"rok":9546,"rof":646,"roe":542,"rog":1119,"rno":850,"rnu":670,"rna":1877,"rne":4341,"rni":878,"riè":489,"rmo":557,"rmu":500,"ro ":5889,"rma":3800,"rme":1506,"rmi":806,"rlu":114,"rlo":918,"rli":1036,"rld":114,"rle":728,"rla":3010,"rn ":837,"rku":647,"rko":637,"nço":166,"rki":3202,"rke":2368,"rka":2965,"né ":169,"riz":2281,"rix":382,"rl ":161,"rip":367,"rio":4087,"rir":622,"rit":7871,"ris":4325,"riv":160,"riu":525,"rig":1164,"rij":116,"ril":4692,"rik":14158,"rin":4139,"rim":739,"ria":43986,"rib":3119,"ric":1194,"rid":2808,"rie":4686,"rif":356,"rk ":865,"rya":116,"rui":249,"rug":753,"rue":502,"rud":764,"ruc":148,"rur":560,"rup":118,"run":1514,"rum":622,"rul":119,"ruk":7712,"ruz":1278,"rus":3454,"rut":2747,"rva":299,"rvi":571,"rve":398,"rrè":303,"ry ":1109,"rsk":111,"rsi":579,"rso":234,"rsa":310,"rsb":123,"rsh":119,"rse":377,"rta":37313,"rst":238,"rto":962,"rte":16058,"rth":1501,"rti":3170,"rtz":8628,"rub":137,"rua":4504,"rts":3910,"rtr":191,"rtu":8524,"rtx":1302,"rt ":2199,"rqu":185,"rro":8594,"rrh":151,"rri":59865,"rre":25667,"rra":27202,"ru ":3378,"rry":287,"rru":11643,"sab":386,"sac":395,"sad":276,"sag":755,"sai":9052,"sak":1159,"sal":1071,"sam":544,"sba":383,"sbe":333,"san":1883,"sau":475,"sat":4584,"sas":2876,"sar":2841,"saz":146,"sa ":3578,"ón ":558,"rze":132,"rza":167,"ruñ":573,"ryp":120,"ryn":159,"sha":293,"sho":242,"she":237,"shi":909,"si ":2705,"sga":355,"sge":484,"siz":297,"sie":2276,"sid":627,"sic":145,"sib":219,"sia":4506,"sk ":177,"shu":112,"sit":1781,"sir":233,"sis":2072,"sin":2608,"sio":1888,"sil":990,"sim":204,"sik":4135,"sig":316,"scu":158,"sbo":201,"sbu":282,"se ":3435,"sca":504,"sci":309,"sch":523,"sco":757,"sex":360,"sey":232,"ser":1632,"ses":1995,"set":590,"seu":341,"sez":2416,"sh ":330,"sfe":412,"sfo":111,"sea":404,"sei":1006,"seg":464,"sed":159,"sep":278,"seo":240,"sen":2206,"sem":882,"sel":1814,"sek":1181,"spo":227,"spe":2584,"spl":190,"spi":825,"spa":4514,"sot":612,"sou":390,"sov":121,"soz":627,"sol":1417,"som":367,"son":3257,"sop":239,"sor":5121,"soe":201,"sof":846,"soi":556,"sok":569,"soa":1638,"sob":130,"su ":1117,"sra":160,"st ":1348,"squ":206,"ss ":363,"sli":115,"slo":268,"sla":1331,"sle":1317,"ski":965,"sko":4744,"skr":385,"sku":35531,"ska":8859,"ske":1695,"sna":476,"sni":281,"sne":1033,"smo":1970,"so ":2308,"sma":633,"smi":193,"sme":185,"sze":109,"sse":1415,"ssa":845,"sso":606,"ssi":746,"ssu":264,"ssy":167,"ste":10155,"sta":12361,"sto":3482,"sti":12235,"stl":148,"stu":1961,"str":13494,"sua":1580,"sue":1113,"sub":323,"sui":142,"sug":112,"sul":894,"sum":1240,"suk":305,"sup":126,"sun":3563,"sut":122,"sus":779,"sur":2717,"sy ":268,"tai":3219,"tak":18285,"tal":9875,"tae":444,"taf":399,"tag":801,"tab":681,"tac":177,"tad":501,"taz":975,"tav":112,"tau":653,"tat":11576,"tas":3495,"tar":13099,"tap":321,"tan":42023,"tam":31263,"tch":169,"te ":16458,"tbo":740,"ta ":64927,"ozè":187,"ño ":122,"pa ":796,"pe ":842,"par":37920,"pat":1309,"pas":522,"pau":187,"paz":432,"pac":116,"pad":210,"pag":296,"pak":754,"pal":618,"pai":3956,"pap":195,"pan":730,"phe":439,"pha":737,"phu":411,"pht":217,"pho":650,"phl":1004,"phi":1924,"pi ":440,"ph ":172,"pez":1232,"pea":802,"ped":964,"pen":3866,"per":5212,"pet":1004,"pes":475,"peg":247,"pei":178,"pel":1577,"pek":338,"pla":1480,"pli":1410,"ple":493,"lès":615,"plo":475,"plu":342,"lé ":113,"piz":190,"phy":150,"pia":822,"pid":4283,"pic":144,"pie":488,"pig":222,"pik":760,"pil":704,"pin":1928,"pio":428,"pir":1470,"pis":933,"pit":729,"por":1550,"pop":436,"pot":370,"pos":1322,"pon":1624,"pok":216,"pol":3723,"poa":264,"poe":553,"pod":317,"ps ":1451,"ppi":156,"ppe":293,"po ":711,"ñak":177,"psi":410,"pso":126,"ptu":244,"pub":1994,"pte":196,"pti":290,"pto":1097,"pra":285,"ña ":328,"psa":122,"pri":796,"pre":2596,"pro":6098,"ñea":278,"ñek":147,"pur":933,"pus":334,"put":783,"pun":714,"pui":142,"pul":574,"puz":1734,"pzi":136,"qua":275,"que":1375,"qui":483,"ra ":35431,"ngo":4685,"ngi":806,"ngl":421,"ngu":2685,"ngr":203,"ngt":258,"ngs":265,"ni ":1052,"nge":3635,"ngh":311,"nga":2908,"ngd":140,"ndé":266,"jär":205,"nha":305,"ngy":127,"ngz":152,"nhe":147,"neg":587,"nei":353,"nel":533,"nek":3786,"nen":5074,"nem":1122,"neo":478,"ner":12607,"net":5039,"nes":2224,"nev":226,"neu":1060,"ng ":2807,"nea":4726,"neb":129,"nec":148,"ned":379,"nee":138,"nfi":148,"nfo":733,"nfl":131,"ney":289,"nez":3775,"nfa":134,"nfe":453,"nct":118,"nco":700,"nci":1712,"nce":1222,"nch":1193,"nca":298,"ne ":11184,"nbu":575,"ndu":36540,"ndr":2320,"nds":159,"ndo":4551,"ndi":10387,"nde":10020,"nda":11174,"ncy":130,"ncu":199,"nak":8535,"nal":3860,"nam":614,"nan":1735,"nar":6462,"nac":646,"nad":808,"nae":224,"naf":779,"nag":2948,"nah":1954,"nai":1342,"nab":663,"nbo":866,"nbe":572,"nbi":1065,"nd ":1618,"nba":2467,"nav":317,"nau":412,"nat":3667,"nas":994,"naz":1755,"nay":387,"na ":51123,"mys":135,"ión":245,"nya":287,"nyi":1026,"nz ":115,"ny ":1034,"nvi":456,"nuk":355,"num":238,"nus":716,"nur":159,"nua":755,"nue":380,"ntx":477,"ntz":53194,"nto":2625,"ntu":3025,"nts":4328,"ntr":1766,"nti":3678,"nth":439,"nta":4382,"nte":6419,"nsu":131,"nsm":113,"nso":335,"nst":1117,"nse":442,"nsh":183,"nsi":830,"nsk":396,"nsa":275,"nu ":553,"nri":259,"nt ":6455,"nqu":144,"npe":1521,"npi":336,"npl":1447,"npo":1351,"npr":973,"npu":200,"ns ":1708,"noc":367,"nod":214,"noa":1389,"nob":252,"nog":132,"noe":258,"nok":336,"nol":1301,"noi":432,"nop":687,"nom":2855,"non":1208,"not":726,"nos":1504,"nor":1438,"nov":276,"nou":160,"noz":134,"npa":532,"nne":2767,"nna":510,"nno":193,"nni":412,"niè":150,"nn ":470,"nla":655,"nle":19805,"no ":4249,"nke":316,"nki":526,"nka":819,"nku":162,"nko":1451,"nji":179,"nje":125,"nja":149,"nig":202,"nif":230,"nie":1873,"nid":1634,"nic":527,"nib":1314,"nia":21004,"nk ":215,"niz":435,"niu":163,"niv":129,"nis":11960,"nit":1609,"nir":121,"nio":1723,"nim":7615,"nin":1144,"nik":2843,"nil":338,"ogr":1140,"ogu":264,"ogi":3112,"ogl":187,"ogo":1114,"ogn":260,"oga":472,"oge":994,"oi ":1867,"ohi":1496,"oho":168,"ohn":393,"oha":408,"ohe":156,"oix":147,"ois":1228,"oir":3115,"oit":1745,"oin":2360,"oik":468,"oil":486,"oig":146,"oih":139,"oid":551,"oie":762,"ok ":552,"oia":1590,"ol ":1542,"oiz":813,"oce":551,"och":579,"oci":236,"ock":754,"oco":220,"obu":191,"oe ":120,"oca":267,"ode":1086,"odi":779,"odo":1366,"odr":254,"of ":673,"oda":1173,"oek":753,"oel":238,"oem":135,"oei":191,"oer":727,"oes":154,"oet":2140,"oen":30360,"ody":112,"odu":1464,"og ":109,"ofi":1327,"oft":279,"ofo":462,"off":123,"ofe":486,"oa ":23154,"oc ":215,"oan":5469,"oal":1996,"oak":7788,"oai":315,"oag":249,"oaf":179,"oad":180,"oba":971,"oaz":457,"od ":223,"oar":7570,"oat":134,"obo":225,"obr":314,"obl":583,"obj":240,"obi":4309,"obe":1621,"nza":293,"oye":159,"oya":225,"oxi":323,"oxe":134,"oxa":143,"oz ":1399,"osé":257,"own":214,"ozt":385,"ozk":775,"ozo":304,"oze":1252,"ozi":1536,"oza":425,"oty":475,"otz":3173,"otx":166,"otu":1228,"oud":180,"oue":152,"oub":647,"ouc":291,"oua":149,"ow ":180,"oti":1497,"oth":414,"ote":1876,"ott":442,"ots":1405,"otr":282,"oto":1274,"ost":3761,"osu":155,"ota":4700,"ov ":231,"osi":954,"osh":196,"osk":831,"ose":2342,"osg":604,"osf":232,"osp":1434,"oss":573,"osm":148,"osl":159,"oso":2977,"osn":183,"oy ":351,"owe":287,"ovi":716,"ouv":335,"oux":333,"ouz":161,"ova":475,"ove":679,"oug":264,"oui":410,"oul":669,"oun":317,"oup":179,"ous":984,"our":2427,"out":459,"opo":1448,"opi":1772,"opl":283,"ope":1031,"oph":2032,"opa":1875,"os ":3052,"opu":866,"opt":221,"ops":1174,"ool":150,"ook":209,"ood":215,"or ":2026,"oor":211,"ork":1333,"orl":371,"orm":2863,"orn":1125,"oro":1987,"orp":1026,"orr":8683,"orc":229,"ord":4422,"ore":7122,"orf":275,"org":1446,"ori":6643,"ou ":585,"osa":5006,"osc":282,"ort":9314,"ors":420,"orv":338,"oru":798,"ory":148,"ot ":1204,"orb":627,"ora":3882,"oqu":170,"ola":6129,"old":2635,"on ":9316,"oli":5769,"oll":736,"olk":977,"olf":204,"ole":2038,"ols":202,"olt":171,"olm":155,"olo":6056,"olp":119,"oly":472,"olu":2010,"oka":5664,"om ":243,"oki":11899,"oke":354,"okr":376,"oko":14411,"okl":115,"okz":240,"okt":241,"oku":440,"onb":522,"ona":9127,"ond":7536,"onc":465,"onf":339,"one":4593,"ong":1876,"oni":6318,"onk":362,"onn":1364,"ono":3870,"onp":1137,"ons":1225,"ont":7687,"onu":301,"onv":206,"ony":150,"onz":159,"oma":3483,"ome":4236,"omb":366,"omi":2449,"omm":1081,"omp":326,"omo":1653,"omu":1205,"op ":248,"la ":13924,"ín ":239,"ío ":118,"le ":29964,"lca":121,"lf ":110,"lde":53459,"lda":3988,"ldo":569,"ldi":2939,"ldu":1826,"lab":1670,"lac":584,"lad":550,"lae":479,"lah":118,"lag":1101,"lai":2536,"lal":149,"lak":5012,"lan":12052,"lam":935,"lap":544,"lar":14549,"lat":4673,"las":2232,"lax":140,"lau":1492,"lav":391,"lay":294,"laz":1246,"lba":511,"ld ":516,"lbe":387,"lbi":632,"lbo":1200,"lbu":874,"kuz":165,"kut":584,"kus":1014,"kur":1078,"kup":125,"kun":4291,"kum":688,"kul":2168,"kuk":191,"kta":288,"kte":291,"kub":403,"kud":351,"kue":274,"kui":374,"kua":34372,"ktr":750,"ktu":1773,"kti":747,"kto":1401,"kzi":9745,"llé":141,"lpe":1008,"lpi":131,"lph":125,"ls ":415,"lol":163,"lok":422,"lon":2710,"lom":2677,"lop":1545,"lor":2845,"lod":422,"loc":168,"loe":182,"log":3913,"loj":116,"loi":515,"lpa":119,"los":1717,"lot":1891,"lou":253,"lov":277,"low":117,"loa":880,"lob":424,"lmo":204,"lmi":305,"liè":124,"lme":519,"lma":675,"lti":598,"lto":617,"lts":230,"ltu":1223,"ltz":6760,"ltx":178,"lub":1731,"lua":1210,"lue":348,"lso":139,"lst":144,"lta":966,"lte":571,"lu ":1176,"ía ":420,"lt ":691,"lhe":161,"lha":201,"lgu":141,"lgo":926,"lge":122,"lgi":369,"li ":1303,"lga":398,"hât":347,"lfo":360,"lfa":358,"lez":813,"ley":359,"lex":439,"leu":374,"lev":222,"les":5236,"let":2340,"ler":37899,"leo":696,"lep":680,"lem":3502,"len":2886,"lek":3605,"lel":174,"lei":663,"leh":2756,"leg":2211,"lef":184,"lee":849,"led":224,"lec":200,"leb":1036,"lea":7810,"lls":158,"llu":240,"lly":586,"lo ":1720,"lla":2591,"lle":6822,"lli":1822,"llo":1275,"lko":462,"lku":282,"lka":8759,"lke":746,"lki":1220,"lm ":353,"lje":187,"ll ":864,"lit":5235,"lis":2569,"lir":147,"lip":493,"lio":1757,"lin":3474,"lim":313,"liz":2323,"lix":128,"liv":211,"liu":172,"lic":425,"lid":704,"lia":18816,"lib":1181,"lk ":177,"lik":3840,"lij":389,"lig":725,"lie":1726,"lif":1422,"ma ":5443,"mb ":125,"mac":227,"mab":228,"mah":164,"mai":4561,"mak":1772,"mad":637,"mae":329,"mag":822,"mar":5241,"mas":1187,"mal":8071,"man":6689,"maz":985,"mat":5384,"mba":280,"mbl":156,"mbi":246,"mbe":401,"mbr":369,"mbo":218,"me ":3651,"mbu":137,"iàn":147,"med":627,"mee":219,"mea":1209,"met":5777,"mes":758,"mer":3360,"mel":403,"meo":147,"men":42606,"mei":212,"mek":454,"mez":354,"luz":1919,"lva":1011,"lve":149,"lvi":156,"lul":394,"luk":181,"lun":936,"lum":401,"lut":259,"lus":2243,"lur":2397,"lux":127,"ly ":703,"lwa":119,"lyc":377,"hôn":315,"mpi":239,"mph":670,"mpe":165,"mpr":155,"mpo":233,"mpl":143,"mps":237,"ms ":126,"mog":116,"moi":170,"moe":126,"mod":1434,"mon":2086,"mop":136,"mok":406,"mol":594,"mor":1418,"mos":401,"mot":1295,"mou":184,"mpa":321,"moz":332,"mu ":428,"mug":1811,"mua":312,"iña":144,"my ":184,"mur":529,"mus":3234,"mut":703,"mui":219,"mul":1435,"mun":2365,"mi ":314,"min":12260,"mio":152,"mil":9541,"mir":441,"mis":679,"mit":1601,"mic":192,"mia":1625,"mig":137,"mie":282,"mid":994,"mik":1980,"mo ":1194,"ièv":316,"ièr":845,"ièg":369,"moa":1809,"mno":287,"mp ":117,"mmu":161,"mmo":170,"mma":310,"mme":989,"zte":6650,"zti":2572,"zta":21734,"ztu":2068,"zto":118,"zu ":506,"zub":389,"zua":330,"zuh":271,"zue":6583,"zur":964,"zuk":745,"zul":747,"zun":824,"zuz":1928,"zut":2275,"zy ":123,"zga":145,"zi ":3451,"zed":168,"zee":198,"zeg":625,"zeh":1509,"zei":931,"zej":597,"zea":4408,"zeu":174,"zet":1270,"zes":644,"zez":804,"zen":47887,"zel":1984,"zek":4796,"zer":3205,"zep":260,"zeo":158,"ze ":2487,"zab":1288,"zad":174,"zac":133,"zae":141,"zaz":774,"zbe":719,"zai":5026,"zag":3454,"zah":516,"zam":162,"zan":15520,"zak":3577,"zal":4025,"zar":6420,"zap":297,"zau":482,"zat":4596,"zoa":1994,"zot":212,"zor":775,"zon":1162,"zok":314,"zol":134,"zoi":237,"zoe":174,"zpe":474,"zpa":171,"zpi":1584,"zo ":1317,"zke":2202,"zka":3073,"zko":8159,"zki":1871,"zku":2058,"zle":2768,"zho":218,"zib":216,"zia":43966,"zie":3257,"zid":476,"zig":450,"zin":2910,"zim":160,"zil":633,"zik":1618,"zio":10069,"zip":247,"zir":2573,"zis":301,"zit":32762,"ziz":641,"uñe":454,"uña":238,"yst":148,"yro":439,"yrt":231,"yra":176,"yre":199,"ys ":403,"yph":1017,"ypt":141,"yon":185,"za ":8723,"ye ":281,"ych":465,"yda":109,"yco":120,"ydr":356,"yer":179,"yen":319,"ya ":555,"yar":131,"yas":121,"yan":296,"yal":138,"yko":242,"yn ":138,"yla":109,"yle":229,"yli":146,"yll":153,"ylv":131,"ylu":1217,"ymn":245,"yne":176,"yno":210,"yi ":127,"ygo":225,"yin":1034,"tín":140,"xu ":295,"xo ":287,"xor":162,"xot":335,"xon":327,"xoa":1198,"xua":342,"xur":507,"xi ":475,"xen":218,"xer":312,"xet":179,"xea":732,"xeb":306,"xek":523,"xel":128,"xka":395,"té ":122,"xir":793,"xis":324,"xit":138,"xil":657,"xik":2037,"xin":2912,"xim":206,"xia":292,"xie":194,"xa ":433,"xe ":943,"xat":249,"xar":321,"xak":258,"xal":109,"xap":662,"xan":503,"xab":132,"xag":128,"wyn":124,"sé ":310,"wn ":153,"sèr":390,"wor":110,"wer":265,"web":234,"wic":120,"win":111,"wil":123,"wa ":219,"wan":193,"wal":111,"way":199,"war":517,"ría":207,"vre":805,"vsk":110,"vy ":146,"rôm":333,"via":694,"vir":147,"vil":2088,"vin":415,"vig":216,"vic":152,"vid":247,"vie":427,"vit":352,"vis":236,"ré ":295,"rèz":299,"vo ":165,"viè":127,"voi":624,"von":289,"vi ":294,"vey":349,"ver":1032,"ves":311,"vet":138,"veg":284,"ven":884,"vel":488,"ve ":552,"val":523,"vak":167,"van":773,"var":579,"vat":112,"vad":796,"vai":119,"va ":528,"uzk":3402,"uzi":824,"uze":3853,"uzt":5216,"uzo":1817,"uza":960,"uya":140,"uxe":167,"uxi":112,"uz ":3148,"ux ":1723,"uvi":405,"uva":124,"uve":444,"uy ":638,"uvr":124,"usk":7231,"ush":119,"usi":7071,"use":2014,"usc":130,"usa":632,"usu":203,"ust":3020,"uss":762,"usp":226,"uso":148,"uth":327,"uti":2642,"ute":9518,"uta":14646,"utb":708,"utx":862,"utz":2136,"utt":153,"uts":624,"utu":1500,"uto":1475,"utr":268,"us ":9795,"ut ":685,"urb":881,"ura":8810,"urd":1157,"urc":233,"ure":3312,"urg":1560,"uri":5079,"url":279,"urk":2210,"urm":306,"urn":368,"uro":1987,"urp":319,"urr":7898,"urs":288,"urt":8830,"uru":8984,"ury":170,"upa":176,"ur ":5462,"upi":127,"upe":490,"umi":217,"umo":363,"uma":1860,"umb":322,"ume":1972,"unt":5181,"uns":142,"unk":576,"uni":2485,"uno":187,"unn":124,"unc":158,"und":3967,"una":6699,"unb":140,"ung":1494,"une":4343,"up ":152,"ukr":242,"uku":113,"ukt":390,"uko":10845,"ukl":257,"uki":593,"uke":693,"um ":1084,"uka":1067,"ulu":958,"ult":3635,"ulp":177,"ulo":533,"ulm":122,"ull":366,"uli":995,"ulg":169,"ule":1208,"ula":4220,"un ":5328,"uid":216,"uil":962,"uin":1030,"uir":110,"uis":870,"uk ":736,"uia":127,"uji":118,"uit":797,"uiz":145,"ul ":480,"uja":166,"ugh":140,"ugi":893,"uge":534,"ugo":167,"ui ":164,"uga":3668,"uhi":142,"ugu":650,"uha":2168,"uco":199,"uf ":132,"uda":33965,"ude":2755,"udi":1792,"ubo":127,"ubs":680,"ubr":1485,"ubu":129,"uca":203,"ue ":1044,"uce":126,"uci":117,"uch":496,"ucl":185,"uck":125,"uet":3427,"uev":148,"uer":1318,"ues":754,"uez":197,"uff":155,"udu":480,"udo":382,"ued":519,"ueb":168,"uen":35965,"uel":1556,"uek":1147,"uei":226,"ub ":167,"ua ":13559,"uaz":311,"uat":379,"uar":5388,"ual":33640,"uan":34449,"ubi":1182,"ubl":2352,"ube":1089,"uba":1426,"ud ":222,"uak":3352,"uai":215,"uag":257,"uad":292,"tzi":45858,"tzo":1861,"tzu":3457,"tza":20297,"tze":31257,"tyl":1154,"typ":482,"ty ":368,"twa":207,"tuz":3752,"tx ":291,"tur":8124,"tus":1524,"tut":13606,"tul":337,"tuk":1405,"tun":1118,"tum":153,"tub":959,"tua":12060,"tud":397,"tue":26377,"tug":593,"txu":656,"txo":1698,"txi":4866,"txe":2626,"txa":1807,"tz ":3179,"ts ":1033,"tre":1843,"tt ":158,"tra":13798,"tri":2959,"tru":540,"tro":7382,"tu ":19929,"tsa":4489,"tse":2634,"tsi":2184,"tso":2535,"tsu":3293,"tta":369,"tte":927,"tti":344,"tto":194,"tts":160,"to ":2344,"tmo":266,"tni":243,"tna":149,"toe":252,"tod":580,"toc":190,"toi":544,"tog":246,"tob":359,"toa":986,"tou":170,"tos":463,"tot":514,"toz":229,"tom":728,"ton":2972,"tok":1652,"tol":2657,"tor":7132,"top":659,"til":1069,"tik":17820,"tif":454,"tie":8056,"tig":462,"tir":1298,"tiq":263,"tit":1526,"tis":1599,"tin":3308,"tim":1547,"tip":247,"tio":1317,"thu":295,"tia":4907,"tib":10581,"tic":667,"tid":1081,"tiz":562,"tiv":202,"tla":841,"tle":497,"tem":2289,"ten":15717,"teo":777,"tei":891,"tek":9552,"tel":3847,"tee":1553,"teg":2380,"teh":241,"tea":12289,"teb":185,"ted":602,"th ":729,"tez":2099,"tev":241,"teu":131,"tet":1953,"tes":1845,"ter":11656,"ti ":3145,"tho":471,"thr":250,"the":2245,"thi":275,"tha":655,"Än ":123,"专 ":284,"並 ":155,"三 ":381,"ä¸ ":173,"万 ":297,"zèr":200,"之 ":273},"n_words":[10862913,12392263,10572779],"name":"eu","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/fa.json b/contrib/languages-data/fa.json
new file mode 100644
index 0000000..6acdfa3
--- /dev/null
+++ b/contrib/languages-data/fa.json
@@ -0,0 +1 @@
+{"freq":{"Ù¬":970,"Ù¾":57795,"Ùˆ":425936,"ÙŠ":735907,"Ù‹":1698,"ÙŽ":4564,"Ù":2202,"Ù":144242,"Ù‚":86257,"Ùƒ":2176,"Ù„":186547,"Ù…":383367,"Ù†":505254,"Ù‡":501741,"Ù‘":1083,"Ù":3108,"Ù”":10105,"Ø®":65646,"د":475980,"ج":87595,"Ø­":43181,"ت":373990,"Ø«":6476,"ب":289206,"Ø©":899,"ئ":9436,"ا":1111531,"Ø¢":65724,"Ø£":1456,"Ø¡":1000,"غ":15714,"ع":91539,"ظ":7379,"Ø·":45348,"ض":9683,"ص":43813,"Ø´":310066,"س":370480,"ز":155766,"ر":687985,"Ø°":11817,"ØŒ":43281,"Ø›":1040,"Û²":32624,"Û³":25683,"Û°":28783,"Û±":63091,"Û¶":17216,"Û·":19045,"Û´":17256,"Ûµ":18080,"Û¸":24510,"Û¹":42744,"Ú¯":80261,"Ú©":285107,"Ú†":25230,"Ú˜":11180," ØŒ":7698," Ú¯":30777," Ú©":148021," Û¸":2786," Û¹":2591," Û±":47632," Û²":17985," Û³":5368," Û´":3375," Ûµ":3167," Û¶":2742," Û·":2925," Ù†":61607," Ù‡":83469," Ù„":8629," Ù…":155700," Ù‚":22376," Ùƒ":1185," Ù":28537," ÙŠ":42307," Ùˆ":134423," ص":10068," Ø´":162851," Ø·":22340," ض":1200," ر":51088," Ø°":1128," س":118308," ز":19079," ع":23850," غ":6362," ا":284499," Ø¢":62929," ج":29359," Ø­":15097," Ø®":24724," د":158860," ب":152331," ت":57718," Ú˜":5441," Ú†":17369," Ù¾":40145,"کا ":2349,"کت ":2257,"ژوئ":1909,"Ú˜ÙŠ ":877,"Ú˜Ù‡ ":1588,"ژان":2068,"گيت":1091,"گير":2604,"کلا":1036,"کلي":1793,"Ú©ÙˆÚ†":1236,"کيل":3244,"کمي":2269,"کنا":990,"کند":3501,"کنو":1241,"کنن":2076,"کوم":883,"کوه":1843,"کرا":939,"Ú©Ø´Ù":46287,"کشو":4351,"کست":856,"کزي":3453,"کرد":5509,"کار":7408,"کات":897,"کان":3075,"کاي":1649,"کام":1480,"کال":1004,"کبي":947,"کتر":1843,"کتا":2868,"کتب":2985,"Ú©Ù… ":997,"Ú©Ù„ ":1774,"Ú©Ù‡ ":49534,"Ú©Ù† ":1159,"Ú©ÙŠ ":15027,"کز ":2384,"کس ":931,"گور":1227,"گون":2525,"گوي":1532,"گلي":2047,"Ú¯Ùت":2210,"گست":862,"گري":2038,"گرو":2216,"گزا":1135,"گرد":3447,"گرا":2354,"گرÙ":2543,"گذش":1619,"گذا":1619,"گار":1384,"گاه":8669,"گان":3798,"Ú¯ÙŠ ":4154,"گر ":3440,"ا، ":2188,"پان":3222,"پاد":1299,"پار":1859,"Û¹Û¹ ":1855,"Û¹Û¸ ":1731,"Û¹Û· ":955,"Û¹Û¶ ":844,"Û¹Û² ":854,"Û¹Û± ":894,"ئي ":825,"ات ":10939,"اح ":1044,"اج ":937,"Û¹Û° ":940,"پرا":1102,"پرد":1609,"پرو":1607,"اب ":5407,"ئن ":919,"پاي":4002,"پتا":3629,"پنج":5682,"Û¸Û¹ ":864,"Û¸Û¸ ":992,"Û¸Û³ ":1400,"Û¸Ûµ ":947,"Û¸Û° ":846,"Û¸Û± ":878,"اي ":55848,"بت ":1678,"اه ":14391,"او ":4256,"با ":13059,"ان ":92173,"اً ":1671,"ا٠":1676,"پيو":992,"پيش":2404,"پير":873,"ام ":15467,"ال ":24240,"اق ":2509,"اع ":1667,"اط ":832,"ار ":39983,"اد ":13676,"اش ":1063,"از ":51085,"اس ":3550,"اسک":971,"بي ":8868,"ت، ":2631,"اضي":879,"ارگ":1672,"اصل":2729,"اسپ":971,"اعا":905,"اعت":1127,"اطل":2273,"اعي":1202,"Û¹Û¹Û¹":1558,"Û¹Û¹Û¸":1457,"ادا":1809,"احي":1808,"اخت":6534,"احم":966,"احت":888,"ارب":1131,"ارا":5657,"ادي":8285,"اده":8465,"ادب":902,"بق ":16057,"ادر":1545,"ادش":1271,"ازي":4465,"است":105076,"ازه":1433,"ازن":1098,"اسا":2986,"ازم":1894,"ازد":2915,"اري":13153,"اره":35608,"ازا":956,"ارن":1623,"ارو":2419,"ارم":2814,"ارز":893,"ارس":5553,"ارش":1311,"ارص":1726,"بل ":1849,"ارد":9154,"ارت":4139,"اشن":1730,"ارک":47474,"اشي":1130,"اشد":5423,"به ":41777,"اسم":943,"اسل":2665,"اشا":865,"اشت":3533,"اسي":5578,"بن ":2042,"اسر":1003,"بع ":2965,"ائي":1624,"ابا":1519,"ابت":1082,"ابر":3253,"ابع":2818,"ابي":1834,"ابل":1424,"ابق":1003,"ابو":1132,"اتر":1045,"اثر":1056,"اتي":2176,"اتو":1748,"اجر":1089,"ئيه":1089,"بر ":33260,"آب ":1128,"آثا":830,"آبا":1911,"آزا":908,"آذر":4073,"آن ":10516,"آما":970,"آلم":1164,"آمر":3443,"آمد":1740,"آمو":1076,"آنه":16519,"آور":2786,"آهن":945,"آغا":1416,"Ú†Ù‡ ":1398,"جزي":1617,"خت ":2202,"حل ":927,"جرا":1045,"جري":3087,"جار":1185,"جاد":1192,"جان":5597,"جام":2439,"جاه":2146,"جاي":1843,"حال":1317,"جنو":3614,"جها":2193,"جمي":1716,"جمع":1997,"جمه":2045,"جمو":2028,"خش ":8012,"حي ":1372,"د، ":4587,"خي ":1696,"دا ":1969,"خه ":1063,"حسي":842,"حزب":975,"حرم":1392,"حده":1001,"حدو":1445,"جنگ":1615,"جوا":942,"جود":1945,"ختر":883,"خان":4015,"خار":1012,"خاب":828,"حمد":2876,"در ":87864,"دد ":1157,"دن ":19684,"تگا":1213,"خست":1155,"ده ":55594,"دو ":4399,"ر، ":2027,"دي ":13399,"خته":2886,"ختل":1697,"خدا":1090,"دل ":1012,"دم ":1329,"خلي":855,"دان":9861,"دام":1683,"دال":1294,"داي":1703,"دار":14265,"داز":1707,"داس":1080,"داش":2836,"داخ":1028,"داد":4687,"خوا":3065,"خود":3802,"خور":1687,"دبي":1292,"رج ":831,"درگ":1116,"رت ":5024,"رد ":14918,"را ":12544,"دست":3550,"دسا":1658,"رب ":1902,"دشا":1262,"درس":984,"درج":847,"درا":1451,"دري":2646,"درو":1489,"دول":1633,"دون":991,"دوم":3136,"دوي":2445,"دهم":3138,"دوا":1480,"دود":1919,"دور":3332,"دهٔ":2250,"دها":1941,"دهد":1019,"دني":1772,"دهس":4371,"دمي":947,"دند":2383,"ذار":1352,"ر٠":16580,"ديو":1392,"ديم":1100,"دين":2614,"ديا":931,"ديد":3091,"دير":977,"رش ":1790,"رس ":3514,"رز ":943,"دما":1034,"دلب":839,"اقي":1204,"اÙÙŠ":974,"اقع":6290,"الل":1166,"ان،":3374,"الي":6506,"الن":1060,"اما":3065,"الم":2177,"اله":1801,"امب":7272,"امر":1231,"الت":2172,"الب":1010,"الا":3840,"الد":1099,"الع":862,"اÙز":1059,"اÙر":840,"اÙت":2627,"اين":37088,"ايل":1631,"ايي":11337,"ايه":1592,"ايس":842,"انگ":3856,"ايش":2420,"ايا":4829,"ايت":2340,"ايج":4814,"انک":896,"ايد":1943,"ايز":1142,"اير":12515,"ØŒ ":41769,"بخش":8437,"اپ ":1341,"اهر":1198,"اهش":1637,"اني":16598,"باي":4838,"انه":7584,"انو":5568,"اهد":836,"انق":1002,"اها":4950,"بان":6032,"انن":2230,"بال":3191,"اند":8054,"باد":2479,"باز":4359,"انز":1366,"بار":4430,"انش":5048,"باش":6504,"انس":4593,"باس":1834,"انص":1766,"امن":1026,"انا":2522,"امه":4047,"امو":1040,"انت":3295,"بات":1162,"امي":5640,"انج":1903,"امل":2517,"اوي":1440,"اول":3794,"اور":2685,"الک":1567,"اوت":2392,"اهي":3496,"اوا":1252,"امپ":1281,"تا ":5488,"بسي":1565,"برگ":2294,"ايگ":1002,"بدا":1231,"برا":9676,"برد":2157,"برخ":1084,"برو":884,"برن":1758,"بري":2753,"بزر":3175,"بست":2397,"تر ":6150,"بعد":1118,"بني":1199,"بنا":1507,"بند":1464,"ا ":87899,"ب ":20308,"Ø­ ":3843,"Ø® ":4523,"د ":139796,"بين":3092,"بيم":902,"بيس":5590,"بيش":2092,"ت ":137024,"بيا":1448,"بير":1134,"Ø« ":1238,"بور":941,"ج ":5421,"بود":10929,"تي ":8134,"اک ":1290,"تن ":2136,"ته ":15461,"تم ":2196,"ثر ":1178,"Ø› ":940,"تبر":18681,"تال":1624,"تبا":2109,"تان":31858,"تام":3922,"تاه":3675,"تاي":3199,"تاد":4918,"تاش":890,"تار":5154,"تاب":4019,"تخا":898,"تحد":1114,"تري":7869,"ترو":1889,"ترا":2202,"ترن":839,"جا ":15939,"تصا":1521,"تشر":878,"ترک":1945,"تصد":3250,"تشک":1423,"تعد":829,"تغي":867,"اژه":1048,"تÙا":3357,"تقا":902,"تقو":993,"تهر":2967,"تها":877,"ثار":853,"تند":2837,"تمي":4044,"تلÙ":1376,"تما":2359,"تلا":1147,"تيم":1139,"تين":1737,"تيا":863,"تون":1309,"تول":2859,"تور":2332,"توس":3860,"توا":4394,"پس ":2219,"تيک":1005,"اکن":1376,"اکت":3167,"اکس":936,"جه ":2283,"جي ":1241,"حت ":897,"پت":4019,"پا":14295,"پس":3568,"پد":1266,"پر":7121,"Ú¯ ":9285,"پو":2856,"په":1230,"پن":7042,"پي":8594,"پل":1723,"Ú© ":69398,"Ù…Ú©":3060,"ÙŠØŒ":6981,"وي":29381,"وو":995,"يب":4572,"يا":96279,"يع":3352,"يز":10370,"Ù†Ú¯":14413,"يس":23115,"يش":9793,"يص":2081,"يخ":3819,"يد":17899,"ير":33847,"يت":13742,"يج":6291,"يح":1015,"Ù†Ú©":2007,"يه":14576,"ين":90729,"ÙˆÚ†":1800,"يو":11558,"يق":4160,"يم":13074,"يل":23315,"ÙŠÙ":3191,"وپ":2022,"يي":14011,"ÙˆÚ˜":952,"ÙˆÚ©":2822,"Ù¾ ":2440,"ÙˆÚ¯":1347,"ÙŠÚ†":1842,"يپ":878,"ÙŠÚ˜":1013,"ÙŠÚ©":36397,"ÙŠÚ¯":5574,"Ùع":2000,"Ùض":915,"Ùز":1100,"Ùر":14718,"Ùس":1042,"Ùد":1178,"Ùت":17008,"Ùا":9831,"قع":6601,"قط":1598,"قش":1281,"قس":1226,"قر":7195,"قد":3410,"قت":1605,"ÙÙŠ":8057,"قب":1853,"ÙÙ‡":2747,"ÙÙˆ":4734,"ÙÙ†":1501,"قا":9232,"ÙÙ„":2016,"ÙÙ‚":1179,"Ù„ØŒ":834,"قي":10959,"قل":3288,"قم":3612,"قه":3627,"قو":3406,"لق":1885,"Ù„Ù":2767,"لط":1095,"لع":1660,"لد":3211,"لح":1516,"لز":963,"لس":4361,"لر":1340,"لا":28241,"لت":4427,"Ù…ØŒ":1074,"لب":3816,"مع":25874,"مغ":887,"مص":1370,"مط":1157,"Ù…Ù":852,"مق":2922,"مل":8979,"مت":12979,"لو":11064,"مج":4572,"ن،":5004,"لي":31085,"لم":9307,"لل":2270,"له":9135,"مب":9205,"لن":3401,"ما":76186,"مز":2088,"مر":23149,"مش":4307,"مس":6882,"مخ":2590,"مح":8770,"مد":9936,"نظ":3887,"نع":1089,"نط":1891,"نم":4839,"نق":3830,"Ù†Ù":2887,"نج":25669,"ه،":2861,"مي":78888,"نت":9394,"مو":18704,"مه":13616,"نب":2742,"نا":34982,"من":11960,"نص":3159,"نش":8281,"نس":7940,"نز":3319,"نر":3727,"ند":45085,"نخ":1510,"مپ":1882,"Ù‡Ù":7528,"هل":3945,"هم":15355,"وئ":3203,"وا":39119,"هن":8401,"نه":35756,"هب":1561,"ها":75779,"نن":4681,"هت":1153,"نو":29290,"هج":3131,"ني":39040,"هد":4476,"هز":15706,"هر":26999,"هش":9098,"هس":7389,"هص":1630,"وع":5118,"وق":2152,"ÙˆÙ":3392,"ون":20803,"وه":7280,"ول":18645,"وم":15237,"هي":7368,"وت":7324,"Ù…Ú†":1311,"هو":6697,"هه":1141,"وب":8255,"ود":37999,"وح":978,"Ù„Ú©":3276,"وج":4297,"وس":20717,"Ù„Ú¯":942,"وز":11168,"ور":41200,"وط":1483,"وض":931,"وص":985,"Ù‡Ù”":10019,"وش":6987,"Ú† ":1702,"خو":10471,"دت":1296,"دا":45095,"خه":1201,"دب":2008,"دخ":1393,"خي":3813,"خل":2639,"خط":1338,"خر":3213,"خد":1570,"خص":2448,"خش":9472,"خس":1563,"تگ":2018,"ده":71326,"دو":21727,"ر،":2098,"دي":30900,"دÙ":1399,"دق":1055,"دل":3645,"دم":4441,"دن":24729,"ذا":2266,"دد":1619,"در":101792,"دش":2498,"دس":6715,"جي":2986,"جو":5371,"حت":1860,"حا":5566,"جن":6990,"جه":5500,"حب":830,"جل":2106,"جم":10111,"جس":1114,"جز":2725,"جر":5927,"جد":2691,"بک":2261,"خت":10589,"حي":3848,"د،":4720,"تک":1874,"حم":4596,"خا":9456,"حو":1605,"حق":1478,"حل":2638,"حص":1359,"حز":1102,"حر":3367,"حس":2640,"حد":3791,"اژ":1490,"تغ":1106,"تÙ":4099,"تم":9655,"تل":4375,"تق":4035,"تو":18858,"ته":20979,"تن":7644,"ثا":1467,"تج":1247,"تح":3912,"تر":24685,"تخ":2986,"تد":1435,"تش":4284,"تص":5949,"تس":1020,"تع":2873,"جا":32040,"جب":888,"جت":900,"تي":16351,"اک":9717,"ثر":1600,"اگ":2405,"بع":5160,"به":44819,"بن":6697,"بل":4961,"بق":17149,"بخ":9337,"بد":4503,"اً":1680,"بت":3170,"اي":143043,"اه":29729,"او":19244,"بط":1115,"بش":1384,"بز":4117,"بس":4538,"بر":58307,"اپ":4030,"تا":68031,"تب":22518,"بو":16880,"بي":27336,"ت،":2720,"ا،":2256,"ائ":3170,"از":65740,"ار":177821,"اد":40749,"اض":1760,"اص":6315,"اش":15541,"اس":126776,"ات":19285,"ئو":1601,"اب":23054,"ئن":1012,"اخ":9486,"اح":6972,"اج":4954,"ئي":4511,"اث":1612,"اÙ":9698,"اق":12353,"ام":46608,"ان":164721,"با":53255,"ال":53226,"اع":7941,"اغ":1903,"اط":5875,"اظ":989,"آب":4072,"آث":840,"Ù” ":10096,"آذ":4112,"آر":1739,"آس":1327,"آز":1293,"آغ":1527,"Ø¢Ù":915,"آل":2867,"آم":7747,"آو":3278,"آن":29292,"آه":1034,"آي":2113,"غي":2298,"عي":7120,"غر":4096,"عل":6634,"عم":5201,"غا":3393,"عن":4842,"عه":5570,"عت":3100,"عد":3656,"عر":22149,"عض":1560,"عا":7561,"عب":3193,"ظا":1618,"طل":3586,"طق":2226,"Ø´Ú¯":3723,"ظر":2221,"Ø´Ú©":5793,"طي":2187,"طه":1305,"طو":2844,"طر":3523,"ضي":1604,"سک":2829,"طا":2296,"طب":17352,"ضو":1678,"سپ":5713,"صل":3684,"صÙ":2604,"ضر":876,"زگ":1068,"صو":4924,"صن":1205,"ضا":2719,"صي":2649,"Ø´Ù":46579,"شص":3397,"شش":4939,"شع":1023,"رک":59526,"صد":15627,"صر":2425,"رگ":10959,"شم":40834,"شن":7520,"صا":3027,"شه":20211,"شو":16524,"صت":1929,"شي":10582,"سع":1222,"سط":4899,"دگ":4253,"سÙ":2299,"رپ":863,"سي":82351,"شت":20229,"رچ":966,"سو":9339,"شب":2714,"سه":5822,"شا":14113,"سن":5406,"سم":5040,"سل":6428,"شر":11318,"شد":71050,"شخ":1686,"زش":2575,"زه":4419,"سب":3293,"زن":6220,"سا":40397,"ست":157670,"زو":2068,"زم":7708,"سد":1583,"سر":10053,"زي":19577,"سخ":1157,"دک":1541,"رس":21469,"رش":6643,"رر":874,"رز":5465,"رص":2328,"رض":1657,"رع":995,"رل":1461,"رق":7164,"رÙ":22038,"رو":41463,"ره":48555,"زب":4854,"زا":26314,"رن":9102,"رم":13248,"ري":64671,"زر":4516,"زد":7554,"ذر":4784,"ذش":1625,"رآ":1627,"رب":15211,"را":79279,"رت":10552,"رج":4101,"ذي":851,"رخ":3587,"رح":1384,"Ø­Ú©":1290,"رد":33206,"Ù ":71140,"ع ":15503,"غ ":1398,"ص ":2346,"Ø· ":7167,"ظ ":1176,"ر ":213685,"ز ":65089,"س ":20728,"Ø´ ":24544,"Ù‹ ":1683,"ÙŠ ":278753,"Ù‡ ":289561,"Ù† ":207616,"Ùˆ ":126016,"Ù‚ ":24581,"Ù… ":45325,"Ù„ ":55421,"Û³Û°":2261,"Û³Û±":1756,"Û³Û²":1475,"Û²Û·":1672,"Û²Û¶":1845,"Û²Û¹":1827,"Û²Û¸":1810,"Û²Û³":1837,"Û²Û²":2072,"Û²Ûµ":2015,"Û²Û´":2310,"Û´Û²":932,"Û´Û³":828,"Û´Û°":1179,"Û´Û±":851,"Û³Û¹":874,"Û³Û¸":3317,"Û³Û·":1442,"Û³Û¶":1469,"Û³Ûµ":1620,"Û³Û´":1319,"Û³Û³":1412,"Û±Û°":3498,"Û°Û±":1251,"Û°Û°":7262,"Û°Û³":960,"Û°Û²":955,"Û°Ûµ":991,"Û°Û´":954,"Û°Û·":971,"Û°Û¶":1030,"Û°Û¹":925,"Û°Û¸":1126,"Û²Û°":6610,"Û²Û±":2252,"Û±Û´":2656,"Û±Û³":8246,"Û±Û²":3243,"Û±Û±":2760,"Û±Û¸":4100,"Û±Û·":2716,"Û±Û¶":2670,"Û±Ûµ":2740,"Û±Û¹":21618,"Û·Û´":853,"Û·Û³":1099,"Û·Û¶":1037,"Û·Ûµ":996,"Û·Û°":1130,"Û·Û²":902,"Û·Û±":971,"Û¶Û¸":828,"Û¶Û¹":864,"Û¸Û·":1462,"Û¸Û¶":1335,"Û¸Ûµ":1706,"Û¸Û´":1157,"Û¸Û³":1867,"Û¸Û²":1238,"Û¸Û±":1392,"Û¸Û°":1426,"Û·Û¹":1133,"Û·Û·":1205,"Û·Û¸":1210,"ÛµÛ°":1455,"ÛµÛ²":836,"ÛµÛ±":867,"ÛµÛ´":849,"ÛµÛ³":863,"Û´Û´":839,"Û´Ûµ":921,"Û´Û¸":900,"Û´Û¹":832,"Û¶Û±":861,"Û¶Û°":1533,"Û¶Ûµ":892,"ÛµÛ·":978,"ÛµÛ¸":881,"ÛµÛµ":896,"ÛµÛ¶":829,"ÛµÛ¹":829,"Û¸Û¹":1660,"Û¸Û¸":1649,"Û¹Û°":1972,"Û¹Û³":2257,"Û¹Û´":1978,"Û¹Û±":1915,"Û¹Û²":1938,"Û¹Û·":3928,"Û¹Û¸":5762,"Û¹Ûµ":1920,"Û¹Û¶":2485,"Û¹Û¹":9457,"Û¸ ":9234,"Û· ":8195,"Û¹ ":8815,"Ú©ÙŠ":21693,"کس":3863,"Ú©Ø´":52864,"کر":10838,"کز":5951,"کد":1167,"کت":10947,"Ú©Ùˆ":9889,"Ú©Ù‡":50638,"Ú©Ù†":11029,"Ú©Ù…":5441,"Ú©Ù„":6076,"کب":1527,"کا":21352,"Ú¯Ù":2232,"Ú¯Ù„":4014,"Ú¯Ù†":1256,"Ú¯Ù‡":1120,"Ú¯Ùˆ":7699,"Ú¯ÙŠ":9941,"گذ":3598,"گز":1726,"گر":18517,"گش":860,"گس":1433,"گا":16359,"Ûµ ":8279,"Û¶ ":7892,"Û³ ":8356,"Û´ ":8000,"Û± ":8359,"Û² ":7902,"Û° ":11742,"Ú†ÙŠ":2258,"Ú†Ù†":2891,"Ú†Ù‡":10435,"Ú†Ùˆ":1224,"Ú†Ú©":1330,"چا":2573,"ژا":3316,"Ú˜Ù‡":1974,"Ú˜ÙŠ":1330,"Ú˜Ùˆ":2750,"چين":1123,"چني":1160,"چها":6583,"چهل":2101,"چند":1391,"چار":831," ØŒ ":7275," Ø® ":829," Ùˆ ":107907," حا":2893," جن":6563," جه":2834," جل":883," جم":4408," جو":1700," جد":1121," جز":2226," جر":1081," جا":5217," تي":2383," اک":4067," خل":1477," خي":1014," خو":9064," دا":22315," خر":1716," خد":1173," خط":1264," حق":954," تک":1140," حم":1000," خا":5629," حو":858," حز":987," حر":1265," حس":1494," حد":1443," به":41159," بن":5266," بل":1919," بع":1479," بز":3261," بس":2873," بر":23110," اي":60506," اه":1491," او":12009," بخ":8934," بد":2040," اق":1653," اÙ":3495," ال":9180," با":32041," ان":13519," ام":5995," اط":2539," اع":2096," اد":2220," ار":5819," از":47219," اس":93321," اش":2136," اص":3471," اب":2944," ات":1821," اث":1151," اج":1670," اح":1812," اخ":1959," تو":11109," ته":3565," تن":2119," تم":1148," تل":1644," تق":2105," تع":1944," تش":1637," تص":1087," تر":7158," تخ":1066," تج":1001," تح":1857," تا":10536," تب":1758," بو":12627," بي":13492," آم":7369," آل":2819," آو":2892," آه":1013," آن":28846," آي":1817," آذ":4109," آر":1633," آث":831," آب":3531," آغ":1486," آس":1300," آز":1278," Ø´Ú©":1757," طو":1797," عض":849," عر":3450," عا":1962," عب":2630," غر":3490," عل":4955," عم":2914," عن":2426," غي":1131," سط":837," سÙ":999," سي":58158," شب":2526," سه":2789," سو":5501," سم":1003," سن":2452," شا":6389," سل":2036," شر":8538," شخ":981," شد":64662," شص":1903," شش":4604," شو":11010," شي":3635," شم":35388," شن":2904," شه":16707," صد":3342," سپ":4409," صÙ":1164," صو":2043," صن":981," طب":16842," طر":1891," دس":4487," در":93973," دي":6271," دو":15789," ده":8568," دن":1430," دل":1101," Ø­Ú©":1033," را":15933," رس":3186," رش":1541," ري":3051," رÙ":1064," رو":19239," زب":3421," زا":2852," رم":1079," زي":3601," سد":907," سر":6965," زم":4334," سب":1207," زن":2927," سا":27260," ست":1210," Û²Û±":1529," Û²Û°":5779," Û±Û¹":20860," Û±Ûµ":1981," Û±Û¶":1865," Û±Û·":1946," Û±Û¸":3370," Û±Û±":2053," Û±Û²":2491," Û±Û³":7504," Û±Û´":1848," Û³Û°":1169," Û²Û¸":971," Û²Û¹":1004," Û²Û¶":1106," Û²Û·":943," Û²Û´":1600," Û²Ûµ":1220," Û²Û²":1386," Û²Û³":1186," Û±Û°":2514," Û· ":1034," Û¶ ":835," Û¹ ":923," Û¸ ":918," پل":1189," پن":6100," پو":1775," پي":6231," پا":11101," پر":5507," پد":1204," پس":3103," لا":2038," مل":2782," مق":2722," مع":22757," مط":1111," مص":1331," مس":5314," مش":3603," مر":12704," مد":3474," مح":8486," مخ":2396," لي":1375," مج":4247," لو":1161," مت":6834," ما":10891," مب":1057," Ù†Ù":1957," نق":2575," نم":3573," نظ":2990," نخ":1392," نر":962," نز":1060," نس":1428," نش":1872," نص":1002," نا":13232," من":6720," مه":4441," مو":8861," مي":39390," Ù‡Ù":7486," هن":3049," وا":10473," هم":7875," هر":3177," هز":15471," هس":2372," هش":6627," نو":12914," ها":28631," نه":5384," ني":6690," هج":2983," Ùر":9595," Ùع":1613," Ùا":3850," ÙÙ„":1560," قا":3036," ÙÙˆ":3446," قب":1076," ÙÙŠ":4001," قد":1747," قر":6081," قم":3484," قل":1179," قو":1126," ÙŠÚ©":22683," وج":1401," هي":1336," هو":2401," وس":1484," وز":1216," ور":1520," ول":1293," Ù…Ú©":1516," وي":5201," يا":14595," Ù†Ú¯":1352," يو":1770," Û± ":999," Ûµ ":990," Û´ ":1026," Û³ ":1037," Û² ":1134," کت":2907," کر":7960," Ú©Ø´":52338," Ú©Ù„":3525," Ú©Ù†":7370," Ú©Ù…":2467," Ú©Ùˆ":5973," Ú©Ù‡":48270," Ú©ÙŠ":3060," کب":1107," کا":11187," گا":3551," Ú¯Ù„":1221," Ú¯Ù":2153," Ú¯ÙŠ":3912," Ú¯Ùˆ":4533," گر":10421," گذ":2142," ژا":2692," Ú˜Ùˆ":2172," چا":2040," Ú†ÙŠ":1498," Ú†Ù†":1727," Ú†Ù‡":9110,"Û¶Û° ":1024,"Û²Û°Û°":3965,"Û³Û¸Û³":931,"Û²Û¸ ":1132,"Û²Û¹ ":1101,"Û²Û¶ ":1164,"Û²Û· ":991,"Û²Û´ ":1524,"Û²Ûµ ":1126,"Û°Û°Û°":1569,"Û²Û³ ":1040,"Û²Û² ":1199,"Û²Û± ":1215,"Û²Û° ":1572,"Û³Û± ":837,"Û±Û¹Û¸":3784,"Û±Û¹Û¹":7313,"Û±Û¹Û³":1178,"Û±Û¹Û²":829,"Û±Û¹Ûµ":879,"Û±Û¹Û´":946,"Û±Û¹Û·":2720,"Û±Û¹Û¶":1373,"Û±Û³Û¸":2507,"Û³Û° ":1283,"Û°Û° ":2532,"Û±Û² ":1158,"Û±Û± ":1041,"Û±Û³ ":1130,"Û±Û´ ":1282,"Û±Ûµ ":1270,"Û±Û¶ ":1284,"Û±Û· ":1225,"Û±Û¸ ":1153,"Û±Û¹ ":1122,"Û±Û° ":1507,"Ùر ":1868,"Ùت ":4170,"قع ":6048,"Ùار":2458,"Ùاد":2141,"Ùاع":864,"Ùتا":2708,"Ùته":4673,"Ùتم":1892,"Ùتص":1637,"ÙÙ‡ ":1283,"ÙÙŠ ":1976,"عرو":1419,"عرÙ":15753,"عرب":2186,"عرا":897,"عدا":1093,"عبد":885,"عات":1421,"عال":2039,"عبا":1366,"عاد":1232,"غرب":3482,"عيت":1749,"عمل":1014,"عنا":1018,"عمو":1947,"عنو":2161,"غان":879,"عني":1446,"غاز":1437,"علا":887,"علي":2182,"علو":970,"عما":1036,"علم":1702,"غير":1279,"صي ":884,"شما":36739,"شهر":16753,"شنا":4807,"صاد":1085,"شمي":1876,"رکت":2292,"رکز":5506,"رکي":1378,"شور":5763,"شود":8535,"شهو":994,"شون":1223,"شير":1302,"شيد":1499,"شين":1156,"ضي ":873,"رگا":1053,"رگذ":955,"رگر":1703,"رگز":904,"سپت":3558,"سپا":1035,"صول":885,"صور":2307,"ضاي":1046,"صلي":1295,"طي ":1075,"طه ":1047,"ظر ":999,"طبق":15902,"طرا":958,"عت ":1441,"عد ":843,"طلس":913,"طلا":2183,"طقه":1497,"طور":1362,"ظام":977,"Ø´Ú©ÙŠ":1859,"Ø´Ú©Ù„":1326,"شگا":3095,"عه ":4310,"عي ":3265,"زد ":1250,"ري ":28175,"رن ":1348,"ذشت":1620,"زب ":861,"ره ":40871,"رو ":1622,"ديگ":2478,"ديک":1038,"رق ":1211,"ذرب":3735,"رم ":3802,"رجه":899,"ردم":1119,"رده":3960,"ردن":2001,"ردي":2386,"ردا":4731,"رتب":927,"ربي":3967,"رتي":854,"راک":1311,"ران":23644,"ربا":5086,"راه":2985,"راو":1025,"راي":12001,"ربر":1171,"راÙ":1247,"رام":1664,"رال":1192,"راب":3402,"رائ":1115,"راح":1002,"رات":2813,"رار":4645,"راد":1759,"راز":1040,"راس":2348,"زش ":1033,"رÙت":3769,"سر ":1698,"زي ":8767,"رصد":2148,"رشي":1128,"ست ":82115,"رسا":1284,"رست":10190,"رشت":1369,"رسم":1031,"رسي":3685,"زه ":3212,"رزش":844,"سم ":1194,"زدي":1099,"زده":4341,"ريک":4778,"روپ":963,"ريه":3010,"ريو":908,"رين":6859,"ريل":1861,"ريق":887,"ريا":4939,"ريخ":2752,"ريت":1617,"ريز":1529,"رنگ":1040,"ريس":1451,"روه":2711,"روي":3959,"ري،":1064,"روÙ":1550,"رون":2388,"روم":1326,"رور":969,"روز":5199,"رود":4045,"رهٔ":1083,"روس":7942,"روش":1626,"رهن":1422,"روا":2639,"زاي":1000,"زان":1010,"رها":4101,"زبا":3526,"زار":18512,"رند":2708,"زاد":3573,"رمي":2432,"رنا":1487,"رما":3636,"رمز":959,"سط ":3314,"رقي":4879,"شد ":26077,"شر ":1296,"شش ":1401,"سن ":1047,"سه ":5126,"زرگ":3314,"شت ":4426,"سي ":13405,"ستگ":1285,"Ø´Ù ":46304,"زيک":1213,"سري":988,"سرو":889,"سرا":2106,"دگي":1827,"شه ":1221,"دگا":1978,"زما":4471,"سام":2387,"سال":16867,"سان":4639,"ساي":1790,"زند":3098,"ساخ":3059,"ساس":1619,"ساز":3943,"زنا":1261,"زمي":2413,"ساب":1251,"ستا":39209,"ستب":15399,"ستر":2167,"ست،":1386,"ستÙ":2194,"ستن":2498,"زيا":1067,"ستم":1666,"ستي":2717,"سته":2853,"ستو":1258,"زيس":843,"زير":3410,"رگ ":3861,"صر ":1320,"سلا":2748,"سمت":1146,"شي ":3088,"صت ":1755,"صد ":14363,"رک ":48005,"شرق":5570,"صل ":1169,"شده":27411,"شدن":16377,"شصت":1903,"شرک":1669,"شصد":1494,"ششم":1955,"ششص":1494,"شتا":2276,"سوم":2657,"سوي":1270,"شتر":2340,"شتص":1588,"شبه":953,"شاه":3881,"شان":2849,"شام":1210,"شار":1758,"سند":1687,"شاخ":862,"سمي":1232,"شخص":1423,"سيل":1493,"سين":2125,"سيم":1189,"سيق":1188,"سيو":851,"شتم":2005,"سيص":1797,"سنگ":854,"سيس":2418,"سير":939,"سيد":2026,"شتي":869,"سيا":50847,"شته":5322,"يکا":4258,"ÙŠÚ©Ù…":1999,"ÙŠÚ©ÙŠ":12431,"يگر":3114,"يگا":1094,"ÙŠÚ† ":1170,"ÙˆÚ†Ú©":1078,"يقي":1448,"يلا":5819,"يلي":2362,"يلم":2163,"يما":2897,"يله":1142,"يلو":2447,"يند":2629,"ينج":15434,"ينا":1201,"يمن":836,"ينت":984,"يمي":2048,"ينو":1227,"يني":2869,"ينه":1667,"يهٔ":843,"يوس":866,"يوا":1011,"يون":3682,"ينگ":934,"وپا":979,"يصد":1797,"يشه":906,"يشت":1412,"يسي":2944,"Ù†Ú¯ÙŠ":1776,"Ù†Ú¯Ù„":2493,"يسه":1037,"يسن":1399,"يزي":2379,"يست":11437,"يزه":870,"يسا":894,"نگا":2016,"يره":2645,"يرو":2532,"يري":2723,"يزد":1123,"يعي":880,"يجا":4729,"يتا":2607,"يتي":884,"يده":2751,"يدن":1012,"يدي":1302,"يرا":13163,"يرد":1088,"يخي":912,"يدا":2030,"يدل":878,"ياÙ":1602,"يال":2762,"يان":11075,"يبا":1219,"يام":955,"ياه":1109,"ياي":3214,"ياس":2249,"يار":48948,"ياز":1404,"ياد":1729,"يات":2044,"ياب":1358,"ÙŠÚ© ":14180,"وز ":3949,"ور ":11858,"ود ":25900,"نقش":885,"Ù‡Ù” ":10011,"Ù†Ùر":1610,"وش ":1865,"وس ":2204,"نمو":979,"نند":4531,"هاس":15525,"هار":8046,"نما":2997,"ها،":830,"وع ":1666,"نقل":1074,"وط ":936,"نيز":2529,"نيس":1397,"نير":1331,"نيا":4775,"ني،":840,"نوي":5351,"نون":2065,"نور":1647,"نود":1940,"نوب":3858,"نهم":1851,"نوا":6754,"نوع":1934,"نوش":1776,"نوز":853,"هاي":33269,"هان":4405,"نها":19728,"نهص":1628,"هدا":927,"ÙˆÙ ":1708,"نيک":1404,"نين":1804,"نيم":949,"نيو":887,"هجر":2343,"هست":7247,"وم ":4571,"هري":1954,"هشت":6878,"ون ":8984,"هرا":3739,"هرم":1056,"هره":1437,"هزا":15536,"هرس":8280,"ول ":5820,"وي ":9400,"ÙŠØŒ ":6814,"وه ":3934,"هشم":1388,"هصد":1629,"معي":1766,"معم":1600,"معن":1773,"معر":16933,"معا":1278,"هد ":1760,"هر ":8315,"مقا":1363,"منا":1581,"ناب":867,"مند":1285,"ناخ":1262,"منت":1332,"منط":1759,"ناس":3745,"نار":1067,"ملل":1010,"مله":1083,"ملي":2015,"مهم":1073,"مهو":1618,"موا":1351,"موج":939,"مور":2297,"موز":1395,"مود":1287,"موس":2124,"موع":2058,"نام":12986,"نان":3607,"ناي":2070,"نتش":1291,"نتر":1378,"مون":1614,"مول":1953,"ميل":5431,"ميد":1645,"مير":1559,"ميا":3140,"نتي":1863,"نجم":2428,"نجا":19529,"مين":23588,"ندا":3431,"ند،":1301,"نخس":1148,"ندو":1099,"نده":6796,"ندر":1455,"ندس":875,"ندي":2668,"هل ":2462,"نزد":2165,"هم ":4541,"نسا":1177,"نست":1454,"ندگ":2266,"نسو":981,"نشا":1366,"وب ":4521,"نصد":1688,"وت ":2984,"هي ":3587,"نطق":1746,"نظا":1022,"نظر":1892,"نشگ":2179,"ونا":1554,"ومي":5536,"وند":2535,"ولي":4939,"ومت":2798,"يع ":1026,"ولت":1350,"ولا":2437,"ولد":1454,"يش ":4417,"يس ":3929,"Ù†Ú¯ ":4653,"يق ":1173,"ويژ":863,"ÙŠÙ ":1505,"ويچ":897,"مکا":1239,"ويي":925,"وين":2044,"ويه":2289,"ويم":1400,"ويز":853,"ويس":4678,"وير":1085,"وني":3417,"ونه":1782,"وها":1116,"يي ":11355,"يه ":12282,"يو ":1257,"يم ":5175,"ين ":61177,"يل ":7962,"Ù‡Ùت":6908,"هنگ":2686,"واژ":1094,"Ù…Ú†Ù†":1005,"وتب":889,"وجو":2093,"لکت":1073,"هنر":1330,"وار":3450,"واز":2127,"هند":2551,"واد":2014,"همي":4888,"وئي":1673,"واب":2536,"وئن":956,"هما":980,"هور":2961,"وبي":1735,"همچ":1222,"هوا":1704,"واه":871,"واي":1826,"واق":6339,"وال":1232,"وان":9374,"وام":2251,"وري":9228,"وست":8309,"وزه":1349,"وزن":1058,"وزي":1584,"يا ":15452,"وسط":3323,"وسي":3688,"وشت":1878,"يب ":1599,"ود،":1062,"ودر":855,"ودن":1173,"ودي":1606,"وده":3280,"ورا":3355,"ورز":1216,"ورش":1426,"ورد":3566,"ورت":2597,"وره":2981,"ورو":1136,"يد ":7931,"ير ":7783,"يز ":4213,"يت ":7713,"يج ":845,"يخ ":2265,"وعي":1123,"وعه":1942,"لد ":1738,"لس ":1850,"لت ":2709,"لا ":1918,"لب ":937,"Ù…ØŒ ":1039,"Ù„Ù ":1009,"له ":6963,"ما ":2248,"لم ":2584,"قمر":3187,"قلا":1065,"قيق":1249,"قوي":1155,"Ùعا":1470,"Ùرد":869,"Ùرا":3846,"Ùرو":1305,"Ùري":1360,"Ùرم":1232,"Ùره":1581,"Ùزا":918,"قه ":2804,"قي ":7279,"Ùيز":835,"Ùيل":2814,"قدي":1049,"قرن":906,"قرا":4415,"قال":936,"قاب":1236,"Ùوت":1061,"Ùور":2032,"قان":1259,"لله":1097,"لما":2054,"ماع":870,"لمي":1688,"مات":1140,"مار":39512,"لند":1567,"ماد":1634,"لمل":950,"نس ":921,"نش ":1280,"ند ":23182,"ه، ":2793,"مي ":37976,"نج ":1914,"ني ":21211,"مشه":1314,"مرک":5626,"نو ":876,"نه ":11112,"مسي":1271,"ها ":10366,"مست":1149,"مسا":1677,"مري":7089,"مرو":1152,"مرد":2093,"مرا":2087,"مرب":1215,"مدي":1420,"مدر":931,"مخت":1330,"مدا":1385,"محر":1442,"محل":1270,"محم":2301,"ليو":834,"لين":2134,"مجم":2201,"مجل":977,"ليت":1597,"ليا":2311,"متو":1555,"ليس":3144,"ليد":1495,"ليل":1108,"لوي":1612,"لوم":2898,"متر":3763,"متح":1213,"مال":4589,"مام":1588,"مان":14800,"ماه":2044,"ماي":4595,"مبر":7239,"مد ":3240,"لو ":1092,"مت ":2632,"ن، ":4905,"لي ":12663,"مه ":7560,"نا ":1758,"من ":2003,"نت ":1363,"مل ":2783,"لاح":1096,"لاد":4785,"لار":867,"لاب":1067,"لات":2844,"لاق":1169,"لاÙ":888,"لاس":997,"لاع":939,"لبر":1058,"لاي":1580,"لام":3536,"لان":2170,"لال":908},"n_words":[8069793,10004435,6796528],"name":"fa","type":"arab"} \ No newline at end of file
diff --git a/contrib/languages-data/fi.json b/contrib/languages-data/fi.json
new file mode 100644
index 0000000..2c587bd
--- /dev/null
+++ b/contrib/languages-data/fi.json
@@ -0,0 +1 @@
+{"freq":{"D":15745,"E":23140,"F":13820,"G":13964,"A":40046,"B":22964,"C":22892,"L":28771,"M":37698,"N":19249,"O":13807,"H":28813,"I":21256,"J":16698,"K":43440,"U":7805,"T":36899,"W":9333,"V":24290,"P":38188,"S":77764,"R":24157,"Y":10043,"X":1821,"Z":1858,"f":29615,"g":68514,"d":158713,"e":1194104,"b":43435,"c":44492,"a":1814181,"n":1349748,"o":934203,"l":885783,"m":433706,"j":314669,"k":691662,"h":239330,"i":1579260,"w":11494,"v":295575,"u":751889,"t":1169051,"s":1099978,"r":475344,"q":1633,"p":248621,"z":7860,"y":250687,"x":8131,"é":2342,"ä":426790,"ö":56679,"š":2606," l":70812," m":89685," n":43615," o":180629," h":39470," i":21438," j":161666," k":172428," d":8045," e":69090," f":6572," g":3193," a":72510," b":5870," c":2712," y":39165," u":13744," t":117334," v":98965," p":89643," s":126171," r":34331," J":16375," K":41900," H":27293," I":17045," N":17648," O":11538," L":26471," M":33953," B":21331," C":19265," A":36093," F":12406," G":12952," D":13481," E":21275," Z":1697," Y":9517," S":71766," R":22636," P":35182," W":8721," V":22294," U":6908," T":34378," ä":2798,"A ":2604,"Da":2655,"Co":4892,"Ch":3205,"Do":1694,"De":2688,"Di":2497,"Fa":1585,"Eu":3024,"Et":2349,"Es":2046,"En":2950,"El":2967,"Ge":2029,"Ga":2319,"I ":2891,"Fr":2159,"Fo":2415,"Fi":2245,"C ":2932,"Au":2202,"Ar":3564,"As":1696,"D ":1819,"Ba":4270,"Am":2679,"An":4463,"Al":7577,"Bu":1605,"Br":4547,"Ca":3954,"Bi":1625,"Be":3488,"Bo":2874,"Ku":5528,"Ky":1627,"Kr":2024,"Ko":6559,"Le":5504,"Hä":4384,"Li":5491,"La":6972,"Lu":2444,"Lo":3355,"Me":4707,"Mi":5018,"Ma":12276,"Mu":3221,"Mo":4384,"Ni":3815,"Ne":4302,"Na":2575,"P ":1899,"No":3154,"Gr":2750,"Ha":6094,"He":6887,"II":1871,"Hi":2221,"Ho":2925,"In":4312,"Is":2133,"It":2586,"Ja":4546,"Jo":4525,"Ju":3085,"Ka":13363,"M ":1871,"Ki":5554,"Ke":4172,"Un":1530,"Tu":4391,"Tr":2204,"To":3873,"Th":5318,"Ti":2566,"Te":3907,"Ta":5984,"St":4721,"Su":13153,"Wi":2315,"Wa":2313,"Vu":2187,"Vi":4932,"Va":6002,"Ve":5654,"Pu":2910,"Pr":3317,"S ":3929,"Pe":5861,"Pa":6975,"Po":6867,"Pi":3820,"Se":20320,"Si":6786,"Sh":1709,"So":3993,"Ru":2993,"Sa":10443,"Re":3883,"Ri":2393,"Ro":4731,"Ra":6455,"b ":4269,"a ":521046,"Yh":4604,"Tä":2310,"i ":187878,"ge":9008,"ga":10572,"fi":6772,"fr":2666,"fo":3746,"hd":23028,"he":38383,"ha":33064,"gn":1824,"gl":5520,"gi":17723,"gh":2779,"gu":2320,"gr":3324,"go":5014,"du":5956,"dy":8630,"g ":5897,"ea":17917,"eb":2211,"ec":3419,"ed":14005,"de":60149,"di":25899,"do":15043,"ds":2160,"dr":3184,"ew":1828,"eu":16676,"ev":21342,"ey":5808,"fa":2058,"h ":5313,"fe":3904,"eh":18148,"eg":3976,"ef":1593,"ee":73047,"el":141223,"ek":30007,"ej":4585,"ei":64027,"ep":7092,"eo":9816,"en":311708,"em":29073,"et":103119,"es":102393,"er":103228,"ca":4693,"e ":102313,"br":3098,"bu":11145,"bo":3540,"bl":1718,"bi":4927,"be":5208,"da":16197,"f ":4183,"cu":1883,"ct":2366,"co":5044,"ck":6770,"ci":3249,"ch":7835,"ce":5416,"c ":2562,"az":1738,"ay":3699,"ba":5388,"d ":12256,"at":82974,"as":103098,"ar":90264,"av":32524,"au":43994,"ak":51088,"al":188447,"ai":159817,"aj":32823,"ao":2348,"ap":28311,"am":44148,"an":226249,"ac":6773,"ad":13018,"aa":139980,"ab":3846,"ag":5956,"ah":19189,"ae":6848,"af":2453,"nu":19006,"nt":79878,"ns":48167,"nr":2630,"np":4065,"no":34422,"nn":76172,"ny":10224,"nv":4829,"oe":6505,"of":6585,"oc":5124,"od":21394,"oa":12733,"ob":3163,"om":49202,"on":220535,"ok":63100,"ol":88347,"oi":101083,"oj":13195,"og":7134,"oh":26450,"kä":31248,"ot":48882,"os":73162,"ov":18707,"ou":20507,"op":25063,"oo":22586,"or":48012,"r ":15286,"ow":2927,"kö":4127,"oy":1883,"pe":39610,"pa":50151,"pl":2749,"po":19855,"lä":49440,"ph":2309,"pi":40558,"lo":48127,"lm":34905,"ll":141072,"ls":4754,"lp":5969,"lv":6764,"lu":74348,"lt":40738,"ly":10157,"o ":46103,"ma":107323,"mb":5971,"iä":11662,"me":68275,"mi":93628,"mm":26269,"mp":15152,"mo":20189,"mu":39868,"iö":6377,"my":13711,"p ":3699,"na":88104,"nc":4077,"nd":15417,"ne":115628,"nf":1558,"ng":26517,"nh":5001,"jä":31523,"ni":80364,"nj":4418,"nk":37841,"nl":4563,"nm":4904,"ju":18610,"jo":83376,"ki":95289,"ke":65747,"ka":163517,"m ":8821,"ky":14212,"ks":65607,"kt":5970,"ku":90075,"ko":85957,"kr":5885,"kk":48142,"kl":2002,"kn":1816,"li":159675,"hä":6582,"lk":30410,"lj":7180,"le":81668,"ld":2991,"la":154382,"lb":9921,"n ":656910,"hr":2420,"ht":42173,"hu":10726,"hj":14036,"hk":3966,"dä":2655,"hi":25267,"hn":1754,"ho":13688,"hl":1555,"hm":8011,"id":26485,"ic":10236,"ib":2776,"ia":70345,"eä":2738,"ih":17748,"ig":6859,"if":2884,"ie":58968,"hy":6074,"k ":7419,"ir":40208,"is":262822,"it":142893,"iu":5141,"iv":30487,"ii":85125,"ij":23513,"ik":78594,"il":97254,"im":65870,"in":288307,"io":39520,"ip":11864,"je":25016,"ji":8870,"l ":13054,"ja":144917,"tä":77533,"sä":41776,"vy":5926,"y ":23865,"wa":2667,"rä":7609,"vi":49369,"vu":43029,"vo":19782,"uv":30695,"uu":71208,"ve":29647,"va":118391,"x ":5327,"ui":22148,"uj":5727,"uk":45990,"ul":59174,"ue":19283,"ug":2138,"uh":10868,"ur":38607,"us":87239,"ut":63413,"um":30283,"un":78305,"uo":96118,"up":18134,"ty":54901,"tu":95699,"tt":139450,"ub":2312,"ua":10701,"ud":16569,"uc":1658,"w ":2157,"to":89637,"tl":2100,"ts":24815,"tr":16390,"te":142275,"tk":13593,"ti":135644,"th":11168,"pä":16825,"ta":258052,"su":49746,"sv":13201,"ss":117492,"st":215124,"sy":12228,"sl":7481,"sk":39138,"sn":1670,"sm":5723,"sp":7200,"so":30621,"sr":2112,"sc":2341,"se":139811,"sh":5275,"sj":2476,"si":171906,"nö":1934,"u ":38729,"sa":155918,"rr":9979,"rs":10921,"rt":23316,"ru":25282,"rv":12255,"ry":7951,"rp":3591,"ro":37223,"rn":7182,"rm":9935,"rl":3938,"rk":30938,"rj":32055,"ri":107434,"rh":7195,"nä":20276,"rg":6568,"re":35490,"rd":7095,"rc":2086,"rb":1938,"ra":66068,"t ":86930,"mä":30120,"s ":74921,"px":2388,"py":4209,"lö":4854,"pt":2602,"pu":33575,"pp":18507,"pr":6312,"ps":3220,"yä":2288,"yö":17077,"vä":26642,"yy":14572,"yh":29899,"ye":9120,"yd":4538,"ya":1921,"tö":12963,"yv":6669,"yt":27603,"ys":23779,"yr":6697,"yp":4560,"yn":15218,"ym":10818,"yl":20580,"yk":18251,"yi":8456,"ä ":141477,"äa":1925,"ö ":8048,"ää":49135,"ät":13038,"äv":12353,"äy":16618,"äm":13914,"äl":20535,"äo":2938,"än":46392,"äp":3824,"äs":18046,"är":23375,"äe":2662,"äi":25110,"äh":12610,"äk":13701,"äj":7869,"öö":1835,"öy":2134,"öt":2960,"ör":2049,"ös":12022,"ön":8537,"öl":2694,"öm":1579,"ök":3180,"öh":1825,"öi":4386," Ga":2281," Ge":2007," Fo":2388," Fr":2153," Fi":2200," Ha":6075," He":6845," Gr":2717," Ho":2907," Hi":2219," Ja":4528," Is":2125," It":2580," In":4291," Ka":13310," Ke":4144," Ki":5524," Jo":4519," Ju":3077," La":6921," Le":5486," Hä":4383," Li":5091," Ko":6537," Kr":2018," Ku":5494," Ky":1626," Ma":12197," Mi":4965," Me":4668," Lo":3341," Lu":2432," Ne":4266," Na":2563," Ni":3790," Mo":4358," Mu":3189," Am":2670," An":4363," Al":7546," Ba":4234," Au":2195," As":1683," Ar":3519," Be":3463," Bi":1601," Bo":2840," Br":4522," Bu":1591," Ca":3824," Ch":3183," Co":4800," Da":2631," Di":2463," De":2676," Do":1656," El":2955," Et":2345," Es":2034," En":2918," Eu":3021," Fa":1558," Tä":2303," Wi":2296," Wa":2289," Yh":4598," Po":6838," Pi":3788," Pe":5834," Pa":6894," No":3136," Ra":6410," Ro":4711," Re":3856," Ri":2382," Pr":3297," Pu":2902," Su":13126," St":4422," Ta":5960," Th":5301," Ti":2528," Te":3868," Tr":2184," To":3832," Ru":2990," Sa":10406," Sh":1689," Si":6699," Se":20290," So":3959," Va":5984," Ve":5638," Vi":4890," Vu":2183," Tu":4371," ja":74622," in":3365," il":6233," is":1907," it":4582," ka":41758," ki":21829," ke":20426," jo":60523," ju":13868," ha":11651," he":10326," hy":3213," ih":2463," hi":4639," ho":2199," hu":2999," jä":12144," ni":12782," ne":5502," na":3130," my":11678," mu":27269," mo":4701," ol":30354," om":3788," on":110013," kä":15552," oh":5375," oi":2526," of":2700," ny":2745," nu":2351," no":7612," le":7943," hä":3832," li":11414," n ":4861," la":17994," ku":35062," ky":5412," kr":3070," ko":26681," me":12340," mi":8054," ma":22033," lu":14733," ly":2009," lo":4509," am":2608," an":4850," ai":10388," aj":3176," al":26047," av":3055," au":2714," ar":5107," as":8761," er":7980," et":8124," es":9200," en":13349," ei":4106," el":17426," fi":2104," de":3555," di":1711," ed":3633," vä":7263," ym":1774," yl":9242," yk":6097," yh":19431," tä":4808," ru":4569," ry":2522," sa":23518," se":22713," si":27049," so":8389," ra":14800," re":3082," ri":4303," nä":4180," ro":4580," pu":9566," pr":4112," s ":2297," px":2387," py":2739," mä":2284," os":9956," ot":1666," ov":7670," op":3466," or":1769," pe":20224," pa":14359," po":10381," pi":14437," lä":10166," sä":4588," va":34158," ve":7508," uu":2456," vo":6724," vu":30542," vi":12396," ty":5276," tu":19858," us":4512," ur":1664," ul":2520," ta":38855," sy":5905," st":3683," su":25086," to":18439," th":5224," pä":9746," ti":8862," te":13642," ää":2036,"Ete":1912,"Eur":2720,"For":1614,"Hel":2818,"Int":2022,"Alb":1684,"Bri":2185,"Nor":1531,"Per":1731,"Par":1783,"Poh":2941,"Ran":1941,"Kal":1586,"Kan":2320,"Kau":1621,"Kar":2566,"Kir":1567,"Kun":1618,"Hän":3486,"Mar":3439,"ään":18825,"ääk":4426,"äät":2724,"äär":4190,"ääs":2129,"ää ":10320,"Vuo":2097,"Yhd":3250,"Suo":9477,"Sta":1658,"Sen":4830,"äht":3405,"ähd":1555,"ähe":3536,"ähi":1621,"Ruo":1578,"änn":2773,"äns":2545,"änt":2999,"Sak":1930,"ämi":2674,"äni":2131,"äjä":6711,"äne":3700,"Se ":11672,"äs ":1575,"ämä":7782,"äos":2009,"äka":1964,"äis":10749,"äin":8551,"äiv":1941,"ält":2814,"äli":6173,"älk":2158,"äll":6310,"äks":2395,"äki":2767,"än ":27597,"äve":2052,"ävi":2773,"ärä":1957,"äyt":13595,"äri":6126,"ärj":7258,"ät ":5667,"äsi":4968,"äse":2119,"ärv":3216,"äst":3327,"äss":3767,"ätt":2038,"ävä":6336,"Ven":3558,"Val":1869,"The":3947,"Tur":1761,"ber":2683,"ce ":2305,"bri":1634,"bum":8758,"aka":6269,"ake":7201,"aki":5604,"aji":6912,"ajo":3036,"al ":4672,"aja":18366,"aje":2029,"aih":3713,"aik":14735,"ail":13159,"aim":1862,"ain":38398,"ais":49469,"ait":14001,"aiv":4010,"aid":2730,"ahm":1834,"aht":3966,"ahd":4670,"aha":3293,"anu":4220,"ano":5725,"ann":13433,"anm":2516,"ant":14494,"ans":18785,"ane":3551,"ang":3334,"anh":3197,"ani":13808,"anj":1817,"ank":6571,"ana":14302,"anc":1814,"and":6936,"amm":4533,"amo":2056,"amp":3132,"ami":8254,"ame":4479,"ama":17130,"alv":4153,"alu":15472,"alt":16363,"alo":7802,"alm":6123,"all":43431,"alk":10647,"ali":18156,"ale":9704,"ala":40225,"alb":7609,"an ":106763,"aks":12179,"aku":8003,"akt":1737,"ako":3232,"akk":5053,"ae ":1542,"aaj":5209,"aak":5208,"aai":4807,"aan":46070,"aal":13155,"aam":4542,"aas":4039,"aar":7457,"aav":4444,"aat":11110,"aa ":30258,"ai ":13611,"ael":1895,"adi":4196,"ack":1788,"ada":3189,"at ":26536,"arh":2305,"are":4235,"ard":3175,"ara":7612,"arp":1607,"aro":2059,"arm":2145,"arl":1651,"ark":11086,"arj":11397,"ari":15453,"aru":3626,"arv":4834,"arr":3315,"ars":2606,"art":6849,"asa":4886,"asi":8287,"ase":5553,"aso":1854,"ask":4888,"ar ":2391,"apa":10451,"ape":2308,"api":2097,"app":6574,"apu":2355,"as ":6418,"ava":19468,"aut":8391,"avo":2634,"avi":5086,"avu":2827,"ata":7640,"asu":5324,"ast":26494,"ass":30398,"asv":4804,"atk":3390,"ato":4227,"ate":6056,"ati":12192,"att":13242,"ats":1884,"atu":3704,"aul":4866,"aup":10709,"aur":1674,"aus":6854,"aud":2731,"auh":2050,"auk":3794,"jel":4529,"jen":8431,"jes":7295,"jet":1556,"ji ":2297,"jaa":5054,"jat":4403,"jas":4641,"jal":7569,"jak":4713,"jan":11104,"jai":10124,"jou":4303,"joh":4795,"joe":2427,"jol":3356,"jok":24849,"joi":16729,"jon":9520,"jot":6559,"jos":6649,"jia":1786,"itk":2090,"ito":6246,"itu":9149,"itt":43813,"its":13426,"ity":9715,"isk":6128,"ism":2973,"isl":1855,"iso":4721,"isp":1964,"iss":30541,"isu":9410,"ist":90841,"isy":1690,"ita":17408,"ite":16460,"iti":5058,"ivo":1659,"ivu":2547,"isä":6664,"iva":11789,"ivi":6748,"ive":3576,"ipp":2283,"ipu":1889,"ilä":2764,"ilö":2966,"is ":8294,"ion":10934,"iop":2837,"ios":4401,"iot":1986,"ikä":1979,"ioi":3973,"iol":2452,"ipa":1808,"ipe":1630,"irt":2993,"iro":1572,"irk":4448,"iri":6484,"irj":10801,"isi":26753,"ise":59209,"isa":5742,"ire":2058,"inä":3666,"ira":5520,"it ":5916,"ja ":90836,"itä":10709,"ivä":3912,"kii":3407,"kik":3240,"kij":2090,"kim":2775,"kil":12288,"kia":4204,"kie":9439,"kiv":2180,"kin":14280,"kio":1639,"kir":13427,"kis":5163,"kit":6283,"ki ":12342,"kea":4096,"kee":4463,"keh":4191,"kei":5212,"kem":2757,"kel":5236,"ken":9401,"kes":10316,"ker":7027,"keu":3655,"ket":2834,"kev":1653,"ke ":2125,"kre":1700,"ksa":4741,"kse":20115,"ku ":3951,"kot":3952,"kou":3762,"kos":5488,"kor":5268,"koo":4082,"kon":11074,"kom":3392,"kol":7289,"kok":7259,"koj":1821,"koi":14746,"koh":3422,"koe":1980,"kku":3944,"kke":3841,"kka":15414,"kko":10268,"kki":11721,"ko ":7803,"jul":11521,"kat":4283,"kau":16540,"kar":4289,"kas":12031,"kap":6608,"kan":20757,"kal":9712,"kam":1710,"kak":4624,"kah":2946,"kai":21386,"kaa":14665,"ka ":40830,"ha ":1581,"han":4990,"hai":2275,"hal":6945,"har":4587,"hah":1631,"haa":1766,"he ":6389,"hdo":2167,"hdy":6548,"hde":8081,"hdi":4168,"hel":4032,"hei":7575,"hee":1552,"het":1965,"her":2750,"hen":6953,"hem":2103,"hin":8148,"his":2803,"hit":5065,"hja":5248,"hje":3366,"hjo":5005,"gle":2192,"gla":2321,"gra":1797,"ial":8370,"ian":13283,"ias":7499,"ic ":1670,"iaa":4991,"ia ":28344,"iet":9042,"iel":9758,"iem":2939,"ien":20960,"ier":2897,"ies":3794,"ied":2257,"ieh":1857,"iek":1604,"eä ":1619,"ich":1751,"ie ":2310,"ica":1971,"idi":2094,"ide":16311,"ida":3705,"iid":1915,"iik":12079,"iih":2036,"iin":34905,"iil":1871,"iim":1601,"iis":3357,"iir":7370,"iip":1650,"iiv":3159,"iit":11463,"ija":15115,"ijo":3212,"ika":17282,"ii ":2634,"igh":2049,"ihe":3769,"iha":1816,"ihm":2273,"ihi":4976,"iht":1902,"imo":4420,"imm":10403,"imp":1982,"ime":14110,"imi":21669,"ind":1904,"ina":17571,"imu":3688,"inn":9244,"ino":7309,"int":21117,"ins":3814,"ine":62207,"ijä":3525,"ing":10847,"ini":7961,"ink":6694,"ioa":2138,"inu":2993,"inv":2235,"iko":8699,"ikk":20680,"iki":6034,"ike":7607,"ila":13660,"in ":123439,"iku":7225,"iks":5910,"ilp":4235,"ilo":6040,"ill":27290,"ilm":11405,"ilj":1888,"ili":8485,"ile":2304,"ima":7267,"io ":5938,"ilt":4359,"ilu":7728,"hmä":2450,"hol":1937,"hon":2485,"hoi":2109,"hmi":2965,"hmo":1850,"hty":9392,"htu":3085,"hto":2163,"hti":9794,"hte":8765,"hta":4518,"htä":1902,"huo":1796,"hum":3543,"hyv":1693,"etä":4989,"evä":1690,"eta":9667,"ete":9398,"eti":3164,"est":38413,"ess":23541,"eud":1841,"eto":4814,"etr":5809,"ets":2385,"ett":36313,"etu":5501,"ety":2077,"erä":4263,"eve":1697,"eva":7898,"evi":4106,"euv":2196,"eut":3244,"eur":5135,"eus":2033,"esä":2111,"ey ":1541,"evy":4434,"elä":9755,"er ":8358,"eor":2097,"eol":1627,"emä":1894,"es ":8151,"erk":7307,"eri":25262,"erg":2239,"erh":1625,"enä":7730,"ere":5494,"era":4710,"et ":16346,"esk":9110,"esi":14483,"ese":2664,"erv":1561,"eru":10609,"err":4242,"ert":7186,"ers":4983,"ern":2916,"erm":2674,"ero":5460,"eki":2975,"ekk":1967,"ekn":1637,"eko":1843,"eks":9087,"ekt":2330,"en ":216998,"ela":5301,"ele":9915,"eli":28185,"elj":2641,"elm":10985,"ell":36182,"elo":8595,"elu":6803,"els":2931,"elt":10887,"ely":1692,"ema":7235,"eme":2562,"emm":5417,"emo":1546,"emi":5347,"emp":2813,"ene":8165,"eng":2981,"ena":5955,"end":1729,"enn":9973,"enk":6752,"eni":7989,"ens":15471,"ent":18077,"ekä":5887,"ehd":2924,"ehi":4544,"eht":6057,"eis":18168,"eim":3841,"eil":7078,"ein":8482,"eik":4720,"eid":4335,"eja":2716,"el ":2656,"eit":8348,"öss":1852,"gis":3146,"gin":6890,"gia":4299,"ght":1865,"ös ":7736,"gen":2252,"ger":1575,"ön ":5342,"gas":2278,"gan":2134,"fri":1576,"for":2137,"fil":1743,"da ":1964,"de ":3246,"daa":2346,"dal":1725,"das":2350,"dan":3795,"ck ":3175,"ed ":1799,"ean":1991,"eal":1781,"eat":2428,"ea ":2811,"ei ":3830,"een":37770,"eel":9070,"ees":6176,"eet":6503,"edi":2493,"ede":4270,"edu":2200,"edo":1603,"ee ":9590,"dys":6375,"dus":2698,"don":2736,"dol":2034,"dos":3976,"dia":2516,"der":2863,"des":8657,"det":3714,"del":8280,"dek":2312,"den":24947,"dem":1603,"di ":1662,"din":5138,"dio":5021,"dis":6234,"rhe":3181,"rha":2376,"näj":3467,"näk":2091,"näi":1855,"ri ":14183,"rgi":2577,"ret":2238,"res":3703,"nä ":7166,"rea":2509,"ree":2885,"rei":4803,"ren":6049,"rel":2990,"rdi":1756,"re ":2848,"rd ":1882,"ras":4899,"rat":4668,"rau":3008,"raj":2533,"rai":2122,"ran":13051,"ral":4384,"rak":6377,"raa":6078,"rad":3325,"rs ":1794,"ros":3755,"rot":3070,"rom":2805,"ron":4039,"roo":4243,"rov":1686,"roc":2312,"roi":1996,"rna":1810,"rne":1656,"rni":2033,"ro ":2723,"rma":3198,"rme":1785,"riä":3343,"rmi":2119,"rko":8356,"rki":6693,"rkk":7003,"rke":4145,"rka":1913,"rjo":6361,"rja":16764,"rje":7814,"rio":2073,"rit":12816,"ris":12311,"rii":3347,"ril":5111,"rik":9617,"rin":19152,"rim":2324,"ria":8783,"ric":1741,"rie":2399,"näy":2474,"ruo":2205,"run":2801,"ruu":3504,"rus":12209,"rva":2385,"rvi":3707,"rve":3560,"rvo":2466,"ry ":2410,"rsi":3925,"rta":4520,"rto":3696,"rte":2990,"rti":3436,"rtt":1850,"rt ":1812,"rro":1797,"rre":2394,"rra":3831,"saa":11689,"sai":4605,"sak":3935,"sal":8568,"sam":4120,"san":10417,"sat":2386,"sas":5453,"sar":9940,"sav":3217,"sa ":88364,"ryh":2885,"si ":33316,"siv":2877,"sie":3768,"sia":14727,"sit":13812,"sis":15595,"sin":22898,"sio":5975,"sil":7537,"sim":10671,"sij":10631,"sik":6439,"sii":15427,"se ":7339,"sev":3309,"ser":1954,"ses":19353,"set":8009,"seu":4708,"sei":5713,"see":16044,"sen":44413,"sem":6469,"sel":8992,"sek":9571,"spa":2113,"sot":3315,"sol":2285,"son":3580,"sop":1591,"sos":3124,"sod":2739,"sof":1780,"soi":4463,"st ":1959,"sli":1829,"sla":3001,"ski":7594,"sko":4863,"sku":10246,"ska":6718,"ske":7078,"sma":1538,"siä":2875,"smi":2339,"ssä":20775,"stä":20839,"stö":6003,"syn":3354,"syy":2751,"sse":2037,"ssa":81782,"sso":2391,"ssi":7227,"ste":34759,"sta":74007,"sto":18448,"sti":27582,"stu":19044,"str":3547,"sty":7941,"suk":6559,"suo":12490,"suu":15921,"sut":2418,"sva":8410,"svi":2330,"tai":23079,"taj":9294,"tak":4197,"tal":15729,"taa":33105,"tav":11283,"tau":2602,"tat":3910,"tas":5911,"tar":15686,"tap":4732,"tan":13725,"tam":13647,"te ":4200,"ta ":96137,"pa ":1685,"par":4524,"paa":2641,"pah":2670,"pak":1785,"pal":14678,"pai":9231,"pan":6844,"läp":1604,"län":3940,"läm":2523,"läi":5398,"läh":6121,"pi ":2782,"lä ":21592,"per":21853,"pet":1794,"pel":8616,"lää":3392,"pia":3590,"pid":1828,"pie":3232,"pii":5873,"pil":2184,"pin":5017,"pis":5110,"pit":5090,"por":1632,"poi":2560,"poh":4180,"pol":4180,"ppu":2417,"ppi":4768,"ppa":7267,"ppe":1921,"pro":3960,"pur":1783,"pus":1680,"pun":10901,"puo":7107,"pul":1657,"puh":2589,"px ":2372,"puu":2017,"mä ":7257,"mäi":6703,"män":3876,"mäs":1650,"mää":5076,"ra ":5712,"ngi":7939,"ngl":4558,"ni ":6341,"nge":2486,"nga":3203,"jän":3177,"jäl":4462,"jäs":2729,"jär":9906,"nha":2067,"nei":4215,"nel":5383,"nen":63104,"ner":2732,"net":11819,"nes":5079,"neu":1692,"ng ":3916,"nee":7398,"jä ":5912,"nce":1684,"ne ":6951,"ndo":1777,"ndi":2475,"nde":2171,"nda":1665,"nak":1691,"nal":6875,"nan":9972,"nai":6761,"naa":4824,"nd ":4059,"nat":3023,"nas":6250,"na ":39872,"myö":9305,"iö ":2528,"ntä":3484,"nsä":4958,"nva":1615,"num":1758,"nus":3629,"nut":8387,"nty":5978,"nto":10581,"ntu":4377,"ntt":4440,"nti":13349,"nta":21023,"nte":11336,"nso":2855,"nss":4946,"nse":1616,"nsi":11891,"nsk":3706,"nsa":12409,"nnä":1593,"nt ":1924,"ns ":1897,"nol":2730,"noi":8508,"nom":2361,"non":3636,"not":1919,"nos":4123,"nne":16039,"nna":31714,"nno":6361,"nni":12231,"nnu":4777,"nme":1667,"nma":1873,"jää":2989,"nla":2791,"no ":2124,"nke":2631,"nki":13461,"nka":13056,"nko":2285,"nja":2100,"nii":4930,"nie":2772,"nia":6492,"nis":11071,"nit":5273,"nim":13793,"nin":16562,"nik":3354,"nil":2835,"ogi":4118,"oi ":3679,"oht":5278,"käs":5110,"kär":1779,"ohj":13237,"oho":2246,"oiv":1619,"ois":23808,"oir":1655,"oit":25419,"oin":13137,"oik":5181,"oim":14211,"oil":4642,"oih":1702,"oid":5498,"käy":11169,"oje":6434,"oja":5307,"ock":3160,"ode":9371,"odi":1977,"odo":4669,"of ":2718,"oda":3354,"oel":2434,"oen":2647,"ofi":1682,"kä ":7989,"oa ":4765,"oal":3299,"nyk":2547,"nyt":4401,"nvä":1760,"otu":2486,"otk":3192,"oti":5562,"ote":5798,"ott":10457,"ots":2991,"oto":4571,"ost":14630,"ota":8496,"osi":11836,"osk":3907,"ose":3031,"oss":12304,"oso":3047,"ovi":3135,"ova":10956,"ove":2498,"ouk":4542,"oul":4682,"oun":1887,"ous":2951,"out":1762,"opp":4159,"opi":5727,"ope":4607,"opa":3651,"os ":5287,"opu":2089,"oon":7289,"ool":1601,"oom":1672,"or ":1961,"oot":2091,"oos":2166,"oop":3454,"ork":3647,"orm":3104,"orn":1628,"oro":2190,"ord":2513,"ore":3159,"org":2171,"ori":12794,"osa":15087,"ort":3782,"ot ":3256,"ora":3303,"ola":6203,"on ":146201,"oli":32153,"oll":16426,"ole":10977,"kää":1980,"olt":1744,"olm":4749,"olo":5842,"oly":2026,"olu":4176,"oka":27584,"okk":2554,"oki":3937,"oke":2183,"oks":6588,"oko":8846,"oku":9166,"ona":6281,"one":7607,"ong":1915,"oni":10795,"onk":9689,"onn":20892,"ono":2022,"ons":4191,"ont":4754,"oma":18393,"ome":15724,"omi":7452,"omu":1601,"la ":45674,"le ":23672,"laa":8510,"lah":2594,"laj":7083,"lai":41589,"lal":1802,"lak":2624,"lan":15670,"lam":1915,"lat":4073,"las":9219,"lau":6037,"lbu":8824,"kuv":12915,"kuu":20315,"kut":5564,"kus":5968,"kup":2448,"kuo":2056,"kun":18716,"kul":5464,"kuk":1991,"ksi":34515,"kso":2000,"kue":1696,"kui":3945,"kti":3367,"kyi":1827,"kyl":5052,"llä":17871,"lok":9030,"lon":3666,"lom":4409,"lop":2359,"log":4285,"loi":6423,"lpa":4160,"los":4127,"lot":1540,"lou":2403,"ljä":2124,"lmi":8971,"lme":5378,"lma":14737,"lti":5458,"ltt":2378,"lue":13574,"lsi":2865,"lta":20774,"lu ":5034,"lmä":3813,"hän":3427,"li ":38792,"lev":10595,"les":3513,"let":2489,"ler":1630,"lem":3429,"len":7655,"lek":2031,"lel":2977,"lei":7201,"leh":3152,"lee":7823,"llu":3725,"lo ":3012,"lla":45545,"lle":23098,"lli":38165,"llo":8003,"lko":3906,"lku":3582,"lka":14232,"lke":3142,"lki":3385,"ljo":1577,"lje":1833,"ll ":2046,"lit":6604,"lis":35039,"lip":2086,"lio":3340,"lin":25643,"lim":2511,"liv":3460,"lia":6855,"lik":3266,"lil":3081,"lii":12989,"lij":5839,"lie":2208,"ma ":15016,"mb ":3008,"maa":25280,"mah":1595,"mai":6076,"mak":3049,"mar":3413,"mas":6492,"mal":15003,"man":17338,"mat":9985,"me ":2801,"mee":1671,"met":10390,"mes":9381,"mer":13701,"mel":4799,"men":18334,"mei":2490,"iä ":10723,"lve":3316,"lvi":1540,"luk":6577,"lui":2536,"luo":7753,"lun":5262,"lut":6301,"lus":6284,"luv":9141,"luu":4990,"ltä":6950,"lyh":1732,"lym":1752,"mpi":5319,"moo":2276,"mon":6257,"mpa":1715,"mmä":7757,"mpä":2564,"mua":1679,"mus":7282,"mut":3853,"muu":6734,"mui":3639,"muk":6397,"muo":5752,"mi ":13252,"min":18710,"mil":5082,"mis":25017,"mit":6945,"mia":4076,"mie":6732,"mik":3016,"mii":3025,"mo ":3241,"mmi":7415,"mma":5671,"mme":2557,"väl":6253,"vä ":5493,"vää":1811,"vän":2110,"vät":4019,"yvi":1744,"yty":2014,"ytt":9230,"yte":5446,"ysv":6099,"yst":4193,"ysi":1754,"yri":2968,"yt ":5088,"ymä":1567,"ys ":5219,"ylä":5863,"yny":2515,"yvä":4334,"yyl":1824,"yys":2076,"yyp":2141,"yyt":1690,"yy ":2993,"ye ":3730,"yde":2513,"yee":3728,"yks":11428,"yky":3192,"yn ":4172,"yle":4657,"yli":5834,"yll":2070,"ymi":2202,"ymp":3885,"yne":1918,"ynt":3210,"yi ":2699,"yhm":2893,"yhd":7827,"yht":15566,"yis":3850,"tön":2023,"tös":1710,"tö ":2585,"täm":6823,"tän":1885,"täh":1876,"täj":3398,"tä ":34022,"tää":13926,"täy":1596,"tär":1635,"täv":7438,"sää":2844,"sä ":25127,"säl":3359,"säk":2119,"säv":1927,"vuo":32425,"vun":2514,"vul":3123,"vy ":2051,"via":2642,"vio":1556,"vir":4221,"vil":2731,"vin":6040,"vii":6538,"vie":4226,"vit":3912,"vis":7340,"voi":10127,"von":1808,"vos":2863,"räi":2223,"vi ":2985,"ver":6645,"ves":2270,"ven":5385,"vel":7217,"ve ":1928,"val":26751,"vak":1862,"van":11048,"vap":1548,"var":8953,"vat":20575,"vas":8886,"vaa":7863,"vai":7839,"va ":18614,"uuk":1525,"uun":9740,"uul":10457,"uud":5759,"uus":11398,"uur":9061,"uut":11944,"uvi":3760,"uvo":2424,"uva":18498,"uvu":5385,"usl":2474,"usk":5912,"usi":8604,"use":5829,"usa":2048,"uu ":9094,"ust":26922,"uss":6964,"utk":3663,"uti":2698,"ute":5204,"uta":9377,"utt":14676,"uts":1830,"utu":7065,"uto":3590,"us ":19751,"ut ":13926,"ura":6710,"ure":2812,"urh":2266,"uri":10072,"uro":4160,"uru":1961,"uod":11474,"uon":20919,"uol":10441,"uom":19569,"uok":4826,"uot":13021,"uor":5597,"uos":6389,"upe":2549,"upu":10042,"umi":12508,"uma":5258,"umb":3567,"ume":3010,"unt":10189,"unu":3435,"unk":6506,"uni":5890,"unn":17239,"und":1617,"una":4707,"ung":5391,"une":2125,"uks":15716,"uku":7377,"uko":2518,"ukk":6046,"uki":2552,"uke":1610,"um ":2055,"uka":8994,"ulu":17346,"ult":3742,"ulo":2329,"ull":6128,"ulk":14188,"uli":3634,"ule":2174,"ula":6501,"un ":17537,"uin":4870,"uis":6404,"uht":1985,"uhu":1855,"uje":1776,"uit":1991,"uja":2763,"ui ":2881,"uha":2389,"ude":10287,"udi":3267,"ue ":4574,"uet":2172,"uee":7376,"työ":3659,"ua ":4041,"uas":1855,"tyv":3209,"tyy":6164,"tye":7861,"tyi":5279,"tyk":3846,"tym":2979,"tyn":4512,"tys":6857,"ty ":6437,"tur":3509,"tus":10527,"tut":5144,"tuu":8595,"tuv":3012,"tuj":1880,"tui":4969,"tul":5059,"tuk":7469,"tun":13048,"tum":4274,"tuo":5296,"tua":2491,"tud":2510,"ttö":2890,"ttä":18426,"tra":3772,"tri":7061,"tro":3179,"tu ":15401,"tsa":2112,"tse":11636,"tsi":5021,"tsu":1775,"tta":37250,"tte":15326,"tti":29985,"tto":7420,"ttu":17456,"tty":9077,"to ":10851,"tiö":3363,"toj":3671,"toi":21414,"tkä":1786,"toa":3897,"tos":4171,"tot":2445,"tom":3430,"ton":10160,"tok":5162,"tol":5536,"tor":8049,"too":2936,"top":1727,"tii":20914,"til":10225,"tik":4482,"tie":14185,"tit":2814,"tis":9101,"tin":13829,"tim":2695,"tio":13178,"thu":3317,"tia":5511,"tiv":2672,"tki":4098,"tka":4817,"pää":9932,"tem":2906,"ten":19694,"teo":3855,"tei":13515,"tek":5492,"tel":25974,"tee":16061,"teh":4545,"th ":1573,"tet":23609,"tes":1638,"ter":13778,"ti ":27214,"pär":2295,"the":3039,"päi":2568,"yön":1725,"yös":8569,"yöh":1625,"yä ":1547},"n_words":[15184556,16912812,13033049],"name":"fi","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/fr.json b/contrib/languages-data/fr.json
new file mode 100644
index 0000000..ee27c50
--- /dev/null
+++ b/contrib/languages-data/fr.json
@@ -0,0 +1 @@
+{"freq":{"D":116102,"E":120408,"F":121384,"G":111406,"A":238769,"B":169565,"C":285959,"L":345504,"M":204991,"N":98243,"O":65813,"H":89479,"I":164982,"J":77783,"K":45111,"U":61602,"T":117987,"W":42348,"V":73826,"Q":14589,"P":196230,"S":238878,"R":128546,"Y":18365,"X":15982,"Z":13917,"f":625832,"g":841835,"d":2804209,"e":9206578,"b":583197,"c":1872040,"a":5160230,"n":5070939,"o":3525396,"l":3535844,"m":1717377,"j":148113,"k":150020,"h":692514,"i":4746975,"w":64787,"v":659313,"u":3519294,"t":4403208,"s":4479915,"r":4208721,"q":428703,"p":1557910,"z":87206,"y":309342,"x":204437,"É":44421,"ï":13787,"î":22540,"ê":38749,"é":1751958,"è":218696,"ç":101170,"â":19710,"à":277569,"û":11801,"ù":7682,"ô":30698,"œ":8733," l":1232220," m":320276," n":242807," o":228420," h":94558," i":171950," j":98827," k":16664," d":1997342," e":1227865," f":345437," g":129922," a":696872," b":124278," c":594010," y":7295," u":524523," t":219372," w":8382," v":133967," q":138426," p":728591," s":616330," r":286749," J":75893," K":42648," H":85423," I":133876," N":84973," O":54606," L":335414," M":194304," B":161567," C":266111," A":214898," F":109790," G":105282," D":105238," E":106937," Z":13187," Y":17633," X":12755," S":214333," R":117248," Q":13527," P":183259," W":40332," V":62074," U":57563," T":104796," à":274935," î":9932," é":209998," ê":8160," É":44088,"A ":15041,"Da":22440,"Cl":11868,"Co":65041,"Cr":12941,"Ce":29429,"Ch":51129,"Ci":6652,"Du":7894,"Do":13538,"De":21502,"Di":17162,"Fe":8715,"Fa":10235,"Eu":11691,"Es":11694,"En":18617,"El":32899,"Ge":14692,"Ga":20211,"I ":15646,"Fr":44421,"Fo":14809,"Fl":7090,"Fi":9942,"C ":21532,"Au":19631,"Ar":26711,"At":6889,"As":11086,"D ":7702,"Ba":35723,"Am":14264,"An":31376,"Al":37486,"Bu":10594,"Br":27743,"Ca":55063,"Bi":10547,"Be":29116,"Bo":31176,"Bl":7237,"Le":129356,"Li":24542,"La":91448,"Lu":9746,"Lo":35045,"Me":21247,"Mi":27542,"Ma":80328,"Mu":9199,"Mo":39904,"Ni":11148,"Ne":15700,"Na":15519,"No":31277,"On":6946,"Gi":8289,"Gr":22075,"Go":10803,"Gu":15739,"Ha":29460,"He":19781,"II":12760,"Hi":8034,"Ho":13597,"In":22404,"Il":64459,"Is":6832,"It":7525,"Ja":19690,"L ":40055,"Je":19939,"Jo":19144,"Ju":9216,"Ka":11065,"Un":43594,"Tr":15916,"To":17150,"Th":22226,"Ti":6718,"Te":13283,"Ta":12863,"V ":9181,"St":21014,"Su":20295,"Wi":11355,"Wa":11101,"We":6825,"Vi":21333,"Ré":14676,"Va":15441,"Ve":10793,"Pr":25219,"S ":10997,"Pe":15558,"Pa":62058,"Pl":7693,"Po":24431,"Pi":18433,"Ph":7701,"Ou":6784,"Or":12154,"Se":26414,"Sc":10127,"Si":15134,"Sh":8197,"Sp":6886,"So":23717,"Ru":6655,"Sa":56793,"Re":19635,"Ri":11472,"Ro":36071,"Qu":12001,"Ra":14786,"b ":23149,"a ":715548,"Yo":8948,"i ":246446,"ge":153192,"ga":79217,"bé":13640,"fl":14307,"ff":44067,"fi":98156,"fr":118819,"fu":33433,"fo":104257,"he":135815,"ha":144005,"gn":87721,"cé":45974,"cè":9053,"gl":38751,"gi":139568,"gh":16776,"gu":62416,"gr":93539,"go":42919,"du":266692,"g ":46447,"ea":84524,"eb":15026,"ec":142148,"ed":28114,"de":1288806,"di":214915,"dm":11618,"do":95110,"ds":13995,"dr":52545,"ew":13282,"ex":53989,"eu":311458,"ev":34640,"ey":24158,"ez":12097,"fa":75922,"h ":43257,"fe":44423,"eg":22246,"ef":23663,"ee":14949,"el":261273,"ei":65911,"ep":62933,"eo":13735,"en":1013332,"em":298038,"et":487438,"aï":8404,"es":1468583,"aî":9959,"er":561408,"ca":174843,"e ":4165476,"by":10945,"bs":8710,"br":109877,"bu":41947,"bo":48835,"bl":80107,"bi":56922,"be":65791,"da":250547,"f ":44719,"cy":11950,"cu":58711,"ct":159489,"cs":7871,"cq":7986,"cr":84742,"co":398094,"ck":30588,"cl":49217,"ci":192973,"ch":228563,"ce":285004,"cc":27518,"c ":86602,"az":14616,"ay":48306,"ba":99079,"d ":356687,"at":402587,"as":164469,"ar":580946,"aq":13150,"ax":8225,"av":102326,"au":313806,"ak":16011,"al":437033,"ai":491839,"aj":9782,"ao":15479,"ap":112014,"am":169689,"an":984305,"ac":155600,"ad":94574,"ab":78438,"ag":129146,"ah":12966,"ae":24602,"af":19038,"nu":48492,"nt":781378,"ns":405957,"nr":20379,"nq":10241,"no":177623,"nn":194465,"nz":7383,"ny":16039,"nv":29268,"oe":7051,"of":34594,"oc":112798,"od":60253,"oa":9189,"ob":47107,"om":320717,"on":978231,"ok":8561,"ol":182032,"oi":182944,"og":68867,"oh":10520,"ot":93297,"os":119510,"ov":56208,"ou":495961,"op":94277,"oo":30133,"or":397296,"oq":6742,"r ":618020,"ox":6981,"ow":11940,"oy":26184,"pe":191299,"pa":336824,"lè":14938,"pl":106081,"lé":66665,"po":236682,"ph":72167,"pi":70246,"lo":174519,"hé":40769,"lm":23015,"hè":8383,"ll":348027,"ls":35163,"lp":13659,"lv":7890,"lu":105730,"lt":35681,"ly":20798,"o ":123736,"ma":245390,"mb":101583,"me":421982,"iè":67396,"mi":181182,"mm":149391,"ié":40043,"mp":121199,"mo":139179,"mt":11044,"ms":10324,"mu":103041,"my":7264,"p ":21047,"na":254840,"nc":212957,"nd":247872,"ne":645258,"nf":26140,"ng":123325,"ni":252369,"nk":9186,"ju":35235,"jo":42449,"fé":31064,"ki":18400,"ke":23430,"ka":17710,"m ":101180,"ko":10158,"gé":41926,"li":366432,"le":1033424,"ld":17364,"lg":18291,"lf":6672,"la":653743,"lb":19702,"n ":1301691,"hr":18002,"ht":13842,"hu":32067,"hi":111052,"hn":14386,"ho":82392,"hl":7215,"dé":209539,"id":89273,"ic":228813,"ib":37956,"ia":126735,"ig":111904,"if":68146,"ie":498651,"hy":15296,"k ":41221,"iq":189141,"ir":210609,"is":657338,"it":513778,"iu":11403,"iv":116504,"ix":21397,"ik":10626,"il":316521,"im":87035,"in":595401,"io":414857,"ip":60451,"je":33885,"iz":8646,"l ":543603,"ja":25992,"xi":22025,"tè":15940,"té":190035,"xp":12895,"xt":10929,"z ":20913,"xa":6853,"xe":16992,"wi":9300,"sé":88528,"rô":7192,"y ":99277,"wa":16319,"we":7882,"rè":35139,"ré":242093,"vi":188474,"vu":6788,"vr":43893,"rê":8320,"vo":57579,"ux":99749,"uv":71595,"ve":234012,"va":93360,"x ":120860,"ui":230155,"uj":8246,"ul":117513,"ue":407659,"uf":8137,"ug":28998,"ur":515467,"us":227928,"ut":191898,"um":73719,"un":620155,"up":64955,"ty":19621,"tu":167178,"tt":77237,"ub":50581,"ua":51699,"ud":59261,"uc":57765,"w ":13938,"to":173702,"pé":53976,"tl":8663,"pè":21910,"ts":102393,"tr":321129,"te":602785,"ti":612679,"th":96741,"v ":6801,"où":7384,"tb":13422,"tc":11543,"oû":9564,"ta":328001,"su":146804,"ss":210582,"st":753670,"sy":23798,"sl":12189,"sk":12848,"sn":9845,"sm":19680,"sp":78469,"so":222811,"sq":17522,"sc":60065,"se":403227,"sh":19364,"si":325826,"u ":509520,"sa":131590,"rr":83294,"rs":168244,"rt":289803,"ru":65582,"rv":27573,"ry":20802,"rq":9625,"rp":15297,"ro":341033,"rn":84154,"né":177701,"rm":89580,"rl":33141,"rk":16207,"nç":94772,"ri":487470,"rg":66626,"rf":12647,"re":763207,"rd":96680,"rc":70956,"rb":28741,"ra":505286,"t ":1634972,"qu":422607,"mê":10277,"mé":91911,"mè":10378,"s ":1913426,"pt":48826,"pu":65796,"pp":68088,"pr":230460,"ps":19103,"zi":10741,"ze":10497,"za":12596,"zo":10854,"vé":17663,"ye":16656,"yc":12861,"ya":27740,"yt":7956,"ys":37929,"yr":15599,"yp":14150,"yo":8853,"yn":14467,"ué":92385,"ym":16691,"yl":12792,"Ét":26242,"ât":11174,"à ":276969,"éé":15388,"îl":9937,"êm":12225,"êt":20220,"él":51000,"éo":28234,"ép":86639,"ém":44293,"én":55491,"és":107336,"ét":140932,"éq":14455,"ér":173640,"év":51715,"éb":22743,"éa":37956,"éd":70000,"éc":110985,"éf":19719,"ée":230327,"ég":99948,"èm":27722,"èn":11542,"èr":67257,"ès":34878,"èt":16045,"èv":6684,"èg":9962,"èc":26515,"ço":8440,"ça":89329,"é ":372600,"ût":9917,"ù ":7557,"ôt":12300,"œu":7751,"一":9376," Ga":20103," Ge":14539," Fo":14685," Fr":44293," Fi":9743," Fl":7048," Ha":29378," He":19697," Go":10717," Gr":21904," Gu":15637," Gi":8161," Ho":13528," Hi":7850," Je":19786," L ":35977," Ja":19604," Is":6751," It":7499," In":22151," Il":64299," Ka":10962," Jo":19013," Ju":9181," La":90821," Le":128061," Li":24245," Ma":79831," Mi":27364," Me":21128," Lo":34906," Lu":9698," Ne":15538," Na":15371," Ni":11097," Mo":39732," Mu":9105," C ":11416," Am":14164," An":31164," Al":37253," Ba":35535," Au":19508," At":6802," As":10968," Ar":26331," Be":28995," Bi":10366," Bl":7174," Bo":30964," Br":27614," Bu":10535," Ca":54551," Ce":29219," Ch":50964," Cl":11738," Cr":12852," Co":64524," Da":22266," Di":16958," De":21321," Do":13217," Du":7843," El":32823," Es":11656," En":18307," Eu":11642," Fe":8673," Fa":10157," Wi":11260," We":6753," Wa":11001," Yo":8931," a ":54049," Ou":6671," Or":12055," Po":24205," Pl":7593," Pi":18345," Ph":7574," Pe":15479," Pa":61683," No":30970," On":6799," Ra":14631," Qu":11878," Ro":35894," Re":19473," Ri":11439," Pr":25007," Su":20225," St":20131," Ta":12776," Th":22049," Te":13049," Tr":15810," To":17022," Sa":56588," Sh":8106," Si":14799," Sc":9966," Se":26149," So":23336," Sp":6786," Va":15389," Ve":10579," Vi":21221," Ré":14605," Un":43425," ja":19099," l ":229130," je":16653," im":17160," in":90112," il":33523," is":6946," it":13099," jo":28748," fé":16770," ju":33224," ha":21033," gr":51361," go":9168," gu":9190," hi":17321," dé":147001," ho":26370," hu":11315," ne":11993," na":26573," mu":26866," mo":87695," on":19443," oc":20805," of":17687," ob":10074," no":93418," le":471758," li":50278," n ":10043," la":426807," gé":20899," me":31539," mi":31668," ma":94790," lu":10283," lo":32481," af":11272," ag":11352," ab":10657," ac":41475," ad":17435," am":34528," an":80768," ao":8597," ap":47388," ai":15942," al":42510," av":56190," au":168550," ar":49629," at":13572," as":18267," d ":197071," ba":44107," bi":13538," be":11213," bo":15457," bl":6713," bu":7813," br":18993," ca":62413," et":342046," es":474914," en":328723," em":8216," el":14313," fe":12823," fa":58915," eu":10657," ex":33671," fu":26386," fr":103021," fo":71702," fl":7457," fi":46439," ge":16870," ga":14369," cl":20052," co":290856," cr":32388," ce":54595," ch":72895," ci":26401," da":165596," cu":9192," cy":6653," do":48107," dr":11453," de":1120211," di":79077," du":223778," té":10776," ru":15648," sa":43475," se":95411," sc":20238," si":97963," sp":20322," so":138476," qu":138128," ra":22818," re":79730," ri":12843," né":88113," ro":35414," pu":26022," pr":170981," s ":23984," mê":10232," mé":23004," ou":87553," op":9623," or":44807," pe":62711," pa":204808," lé":7304," pl":62779," po":141674," pi":14858," ph":17219," sé":19742," va":14483," ve":21390," vo":19720," vi":65672," ré":111803," ut":13716," un":501687," ta":13187," où":7340," sy":15329," st":19632," su":114473," tr":64495," pé":8718," to":29268," th":29162," ti":14570," te":41930," Ét":26213," à ":274908," êt":8158," év":15420," éq":11336," ét":84332," ép":12565," él":18021," éd":13385," éc":33500," ég":11829," îl":9913,"Eur":7732,"En ":11566,"Ell":27452,"Fra":34835,"For":6793,"II ":8125,"Her":7505,"Hau":10362,"Gra":11197,"Ind":7362,"Il ":58031,"Bar":7002,"Bas":7999,"All":7697,"Ang":8461,"Cal":11943,"Car":9781,"Can":10563,"Ber":8508,"Bel":7820,"Bou":9492,"Dan":8952,"Chi":7925,"Cen":6692,"Cet":9727,"Cha":27535,"Cor":8959,"Com":17296,"Col":7940,"Con":15750,"Cou":6779,"New":7835,"Nor":17044,"Pie":8398,"Par":29868,"Pro":9444,"Ita":6863,"Jea":13129,"Les":29698,"Le ":85946,"Lan":7053,"La ":64716,"Lou":8888,"Loi":7362,"Man":8487,"Mar":33987,"Mon":17275,"Mic":7917,"Sud":6967,"Sta":7499,"Son":7414,"Sai":21598,"San":8752,"Val":6713,"Uni":28327,"Un ":8531,"The":11621,"bit":13396,"bil":9380,"ble":36471,"bli":30327,"bor":11093,"bou":14756,"be ":11661,"ban":14082,"bal":23571,"bat":8684,"bas":21924,"bar":8519,"bec":7102,"ber":20504,"bel":11445,"bie":9676,"ca ":7169,"car":22613,"cat":26927,"can":25116,"cap":8531,"cad":7391,"cal":21842,"cai":32564,"ce ":152975,"bri":16551,"bra":7577,"bre":74429,"bum":11288,"but":12601,"by ":8395,"am ":12290,"al ":83242,"ail":20993,"ain":126816,"air":72297,"ais":165056,"ait":69408,"aie":8332,"acé":7099,"agi":9624,"agn":32293,"ago":7536,"anv":10025,"ano":12803,"ann":37434,"ant":198270,"ans":198981,"ane":14041,"ang":47054,"ani":45704,"ana":30499,"anc":97402,"and":98722,"amm":17042,"amp":23230,"ami":37168,"ame":15452,"amb":9570,"ama":14537,"alt":7552,"alo":12342,"all":56591,"ali":100273,"ale":107350,"ala":20659,"alb":12381,"an ":76640,"abe":9364,"abi":15595,"abl":19961,"abo":9408,"abr":8483,"ae ":16557,"ac ":8751,"aff":8179,"ai ":16674,"aga":7441,"age":55470,"ado":7415,"adm":9823,"adi":21924,"ade":15733,"aci":10019,"ach":16602,"ace":28163,"acc":10028,"ada":13056,"act":43926,"até":6761,"ays":15243,"aya":8788,"aqu":12595,"at ":36684,"arg":12036,"are":21800,"ard":35646,"arc":26378,"arb":8126,"ara":32792,"aro":14440,"arn":8384,"arm":11551,"arl":15187,"anç":91515,"ari":62241,"aru":7211,"arq":6770,"arr":20556,"ars":16927,"art":134713,"au ":118568,"asi":7486,"ase":11574,"ar ":120208,"api":10314,"aph":15548,"apo":12040,"app":42097,"apr":9346,"as ":37163,"amé":27799,"ava":24401,"aux":51164,"aut":60224,"avr":9575,"avo":10413,"avi":13083,"ave":40354,"ay ":10427,"ata":13379,"aoû":8368,"ast":22288,"ass":51465,"atr":15439,"ato":12284,"ate":44406,"ati":192812,"ath":17738,"auc":6672,"att":16852,"ats":23409,"atu":15332,"aul":9810,"aum":7753,"aur":11354,"aus":19732,"aud":9105,"jet":8119,"jeu":19460,"jan":10693,"fév":8788,"fér":12432,"jou":35022,"itr":12118,"ito":10908,"itu":87676,"itt":10859,"its":10569,"ism":13926,"iso":19540,"isp":7420,"iss":50166,"ist":125656,"ita":62348,"ite":57973,"ith":6663,"iti":60924,"isé":36821,"iva":20746,"ix ":16528,"ivi":25932,"ive":55895,"is ":216454,"ion":363871,"iol":6761,"ipa":18735,"ipe":13929,"ir ":34075,"iro":12325,"iné":18867,"iri":8297,"isi":28415,"ise":101298,"isc":8611,"isa":28074,"iqu":188846,"ire":120253,"ira":8690,"irc":8347,"it ":122098,"ité":60635,"gén":18856,"jus":7076,"jui":19529,"ham":23355,"han":29050,"hau":9292,"har":26334,"hab":12359,"he ":52407,"hel":8715,"hef":7684,"hes":11848,"her":19826,"hie":13715,"hin":15050,"hil":13377,"his":19375,"hiq":6944,"dé ":18734,"déb":7151,"déc":35759,"go ":7689,"cée":7075,"céd":16748,"gle":10717,"gli":6716,"gla":15951,"gno":9326,"gni":10059,"gne":50271,"gna":8886,"gou":8451,"gro":28558,"gra":41927,"gre":10583,"gui":7723,"gue":37672,"ial":35445,"ian":23438,"iat":15622,"ic ":12032,"ibl":8749,"ibu":7144,"ia ":21961,"ieu":43579,"iel":22526,"ien":138676,"ier":92906,"ies":16655,"iff":14012,"ifi":22387,"ict":17983,"icu":12669,"ico":11245,"ick":6729,"ici":37411,"ich":23588,"ice":26902,"ie ":169464,"ica":66266,"idi":10544,"ide":31414,"ida":13896,"if ":16206,"il ":58832,"ige":6657,"igh":9785,"igi":21897,"igu":11721,"ign":41717,"idé":15214,"imp":16150,"ime":18408,"imi":12406,"inc":41459,"ind":21942,"ina":36398,"ino":12991,"int":75915,"ins":44722,"inf":11189,"ine":105011,"ing":34736,"ini":40626,"ila":9244,"in ":120797,"ilo":10879,"ill":136561,"ilm":14297,"ili":45030,"ile":18930,"ima":15975,"io ":17855,"ils":11236,"hol":10188,"hom":18219,"hon":13352,"hor":8438,"dév":10462,"dée":10321,"déf":8940,"dém":9364,"dép":53945,"dér":20431,"dés":15740,"ht ":8492,"hum":15199,"ffe":7771,"ffi":11627,"fes":12569,"fer":7877,"fam":26250,"fai":22917,"fac":6862,"ext":9480,"ez ":9334,"exp":11699,"exi":10548,"eta":8136,"ete":7713,"eti":12647,"esp":33176,"est":489584,"ess":57266,"eul":7484,"ett":36757,"ew ":8505,"eve":10111,"eva":7495,"evi":7979,"euv":6803,"eut":10699,"eur":177418,"eus":14865,"eux":38960,"ey ":17748,"er ":177809,"es ":856843,"ept":19543,"epu":14012,"elé":11860,"epr":16261,"erl":8779,"eri":17387,"erg":13600,"ere":10113,"erc":19130,"era":14672,"erb":9918,"et ":392437,"esc":6799,"eu ":31850,"erv":19978,"err":46160,"ert":38461,"ers":78150,"ern":42947,"erm":29874,"ero":7440,"en ":352124,"ela":11282,"ele":8299,"eli":13751,"elg":9644,"ell":93604,"elo":21097,"els":10727,"emb":53506,"ema":21014,"eme":149235,"emi":31587,"emp":25298,"ene":6850,"ena":20675,"end":38663,"enc":42120,"enn":54421,"eni":9389,"enu":8876,"env":9265,"ens":48323,"ent":371396,"enr":18661,"eil":18380,"ein":23896,"eig":7921,"el ":58534,"gis":10364,"giq":10802,"gin":20723,"gio":57044,"gie":22112,"ght":8779,"gen":26739,"ger":14731,"ges":19429,"ge ":72294,"gar":9985,"gal":16426,"gan":17016,"ga ":6882,"fus":7249,"fut":21703,"fra":95189,"fri":7633,"for":45771,"fon":27020,"foo":11436,"foi":12370,"fic":24328,"fil":25221,"fin":20051,"ffé":6845,"da ":14864,"de ":945562,"dai":12767,"dae":8896,"dat":11688,"dan":177922,"cul":27372,"ctu":20091,"ctr":11700,"cto":18088,"cti":59878,"cte":30379,"cré":24660,"cla":14396,"cle":15005,"clu":9348,"ché":8767,"co ":9868,"cié":10273,"con":120854,"col":28490,"com":148564,"cor":20969,"cou":37141,"cs ":7503,"cqu":7392,"ct ":7934,"cra":8267,"cri":32184,"cro":10112,"cci":7074,"cea":6989,"ch ":14828,"cer":17030,"ces":36299,"cet":8987,"cen":24433,"cem":11283,"cel":14093,"cha":54442,"cia":30366,"ck ":14947,"cie":55297,"cid":7429,"che":76684,"chi":31813,"cho":10460,"chn":7132,"cir":7170,"cis":7169,"cit":9428,"cin":16999,"cip":23717,"cke":8035,"ed ":9999,"ec ":39615,"ean":16858,"eau":41781,"eff":6903,"ech":13667,"ef ":8454,"ect":52251,"eco":14578,"dur":7123,"don":36748,"dom":8813,"dou":8007,"ds ":11646,"dmi":10099,"doc":6954,"dui":12311,"duc":16031,"dri":6694,"dra":6827,"dre":21768,"du ":216503,"dro":13106,"dic":10492,"dia":18227,"der":20812,"des":229646,"deu":19332,"dev":7361,"del":7755,"den":24656,"dep":13089,"di ":9657,"do ":8222,"div":14424,"din":10138,"dio":11927,"dir":13444,"dis":29420,"dit":31296,"die":29137,"dif":14101,"rga":14359,"ri ":10992,"rgi":6778,"rge":19992,"ret":19172,"res":111968,"reu":15390,"rg ":10612,"rea":7335,"rec":29178,"reg":9663,"rem":41279,"ren":49926,"rel":19572,"rer":6909,"rep":18635,"rdi":12299,"rde":11104,"re ":401675,"rco":8273,"rch":25796,"rce":14227,"rd ":49795,"rap":22951,"ras":10406,"rat":61280,"rav":14677,"rbe":7330,"rai":37514,"rag":12671,"ran":191224,"ram":14978,"ral":42593,"rab":8579,"rad":17757,"rac":18041,"rs ":108005,"ros":12152,"rot":13239,"rom":20207,"ron":45128,"rop":30358,"rou":56658,"rov":25371,"rod":19072,"roc":18144,"roi":33087,"rol":7836,"rof":11414,"rog":8618,"rna":26829,"rne":26551,"rni":14539,"riè":7237,"nér":15715,"ro ":12314,"rma":27965,"née":51678,"rme":33077,"rmi":8879,"rle":10727,"rla":9260,"nça":87637,"né ":83238,"rip":8171,"rio":12752,"riq":23356,"rit":49518,"ris":61412,"riv":19174,"rig":30515,"ril":14567,"rin":32694,"rim":9133,"ria":17879,"rib":8141,"ric":64397,"rid":9964,"rie":84165,"rk ":9714,"ruc":9646,"rus":10779,"rvi":9893,"rve":7324,"ry ":13884,"rsi":16639,"rso":15966,"rse":12239,"rta":21499,"rto":8213,"rte":72339,"rth":9356,"rti":82832,"rts":8395,"rtu":7199,"rmé":10475,"rt ":67054,"rqu":9541,"rro":9520,"rri":15769,"rre":35748,"rra":13162,"sac":6821,"sai":15869,"san":31853,"sat":19662,"sa ":20364,"si ":23508,"sie":23358,"sid":13927,"sic":8539,"sit":96579,"sis":11253,"siq":13014,"sin":20761,"sio":46892,"sil":9278,"sig":20886,"scr":8195,"se ":185840,"sca":7156,"sci":11470,"sco":9152,"ser":33436,"ses":32384,"seu":18614,"sea":8788,"sei":15724,"sec":11236,"sep":15495,"sen":31564,"sem":23920,"sel":11350,"spo":15398,"spe":6813,"spa":15911,"sou":38682,"sol":8864,"son":104745,"sor":19583,"soi":7516,"soc":22585,"st ":465410,"squ":17455,"sla":8458,"siè":10850,"sme":14217,"stè":10526,"sys":7499,"où ":7381,"sse":73979,"ssa":25709,"sso":20070,"ssi":65158,"ssu":9524,"ste":86262,"sta":32395,"spé":11204,"sto":25947,"sti":51286,"spè":18097,"str":64918,"sud":12096,"sui":13416,"sul":7775,"sup":9440,"sur":74845,"tai":73243,"tal":50318,"tag":11022,"tab":9930,"oût":9507,"tba":12570,"tat":51850,"tar":13094,"tan":65935,"tam":9043,"tch":9033,"te ":229351,"ta ":14870,"pe ":50543,"par":234438,"pat":9116,"pas":16097,"pay":9300,"pag":21493,"pal":18293,"pan":7834,"phe":9797,"pha":8008,"pho":13907,"phi":21732,"peu":15687,"pen":18365,"per":42656,"pet":8398,"pes":12928,"pel":20669,"pla":26028,"pli":9068,"ple":16401,"plo":8634,"plu":41224,"lé ":12711,"phy":6923,"pio":10675,"pir":7633,"pit":8267,"por":43095,"pop":8694,"pou":69236,"pos":40507,"poi":8423,"pon":18257,"pol":30878,"ps ":10703,"ppo":8721,"ppa":15829,"ppe":26475,"lév":6751,"lég":7457,"lée":13954,"pub":20867,"pte":18796,"pti":14976,"pri":39208,"pre":41906,"pro":84520,"pui":23064,"pul":10006,"prè":17131,"pré":40058,"mé ":9595,"mêm":9754,"mée":9397,"méd":12609,"mét":11009,"mér":37720,"qu ":18907,"qua":24258,"que":267169,"qui":99689,"qué":9551,"ra ":22507,"ngl":20995,"ngu":17741,"ni ":11818,"nge":18618,"nga":7486,"ndé":17148,"nel":18137,"nen":6872,"nem":17361,"ner":14652,"net":8307,"nes":51020,"neu":14088,"ng ":26468,"nfo":8493,"nct":9124,"nco":15938,"nci":43333,"nce":105515,"nch":14886,"ne ":496030,"ndu":10752,"ndr":20713,"ndo":10176,"ndi":34156,"nde":56096,"nda":37414,"nal":54750,"nan":26247,"nar":12696,"nad":12377,"nag":10703,"nai":30209,"nd ":51106,"nau":12991,"nat":48959,"na ":18193,"nté":16371,"ny ":7135,"nvi":18890,"nve":7658,"nue":12712,"nto":18893,"nts":41197,"ntr":72123,"nti":46425,"nta":41915,"nte":98733,"nsu":6773,"nso":9167,"nst":29806,"nse":30814,"nsi":22894,"nsc":6653,"nu ":15262,"nné":23464,"nre":13930,"nt ":427954,"nqu":6929,"ns ":274209,"nol":9967,"noi":11530,"nom":60135,"non":15485,"not":12329,"nor":18442,"nov":11258,"nou":9106,"nne":96393,"nna":28812,"nni":16118,"nnu":18917,"no ":11795,"nif":7747,"nie":43637,"nic":15379,"niv":18250,"nis":60898,"nit":15799,"niq":25898,"nio":9717,"nim":8520,"nin":8256,"ogr":17742,"ogi":23895,"ogn":7673,"oi ":11549,"ois":67090,"oir":45748,"oit":23632,"oin":16797,"ol ":10751,"och":11107,"oci":26047,"ock":12402,"oca":13060,"occ":8851,"ode":18688,"oct":13097,"of ":10906,"odu":17793,"off":9016,"ofe":9519,"obr":10777,"obi":7081,"oye":7463,"oya":11498,"osé":9652,"oue":24956,"ouc":7361,"oti":7550,"ote":10769,"oto":12004,"opé":11013,"ost":16493,"ota":13987,"otb":12106,"osi":13992,"ose":15543,"oss":13431,"ovi":29641,"ouv":52919,"ove":15893,"oug":6908,"oui":10077,"oul":19390,"oup":35466,"ous":44108,"our":142367,"out":30934,"opo":10738,"opp":14092,"ope":8783,"oph":15160,"os ":20357,"opu":8920,"or ":16010,"oot":13483,"ork":6806,"orm":40453,"orn":11249,"orr":9505,"orc":7615,"ord":43495,"ore":17630,"org":20050,"ori":52286,"ou ":81997,"ort":98537,"ors":17665,"omé":6755,"ot ":11871,"ora":19855,"ola":10628,"on ":413868,"oli":37368,"oll":18132,"ole":20481,"olo":39631,"olu":16814,"om ":32719,"ona":46449,"ond":64776,"onc":25758,"onf":7947,"one":18885,"ong":21441,"oni":23679,"onn":93395,"ono":18517,"ons":103783,"ont":114426,"ony":7110,"oma":27935,"ome":15382,"omb":20959,"omi":18767,"omm":120540,"omp":47351,"omo":9810,"omt":10618,"la ":409873,"le ":650000,"lab":8668,"lac":18171,"lag":11480,"lai":40035,"lan":67550,"lam":6667,"lar":10935,"lat":30778,"las":17285,"ld ":8004,"lbu":11677,"llé":6984,"ls ":27190,"lon":33452,"lom":9149,"lop":13744,"lor":20550,"loc":9951,"log":34746,"loi":10734,"los":7562,"lié":10397,"héo":7867,"lti":6832,"lub":8118,"lue":8111,"li ":8120,"leu":30125,"les":193810,"let":24641,"ler":15063,"lem":54863,"len":12174,"lec":18621,"lo ":9843,"lla":30932,"lle":228673,"lli":25721,"llo":13913,"lm ":12927,"ll ":25434,"lit":56867,"lis":75238,"liq":19529,"lio":7237,"lin":24717,"lim":6784,"liv":7668,"lic":17589,"lia":14317,"lib":8142,"lig":13313,"lie":66363,"lif":7647,"ma ":11524,"mb ":9981,"mai":41522,"mag":14210,"mar":34078,"mas":7661,"mal":10704,"man":59224,"mat":42658,"mba":7370,"mbl":12331,"mbr":52251,"mbo":7187,"me ":138747,"met":15733,"mes":29770,"mer":17959,"mem":10190,"men":192406,"lui":7872,"lut":8740,"lus":46191,"mpi":17291,"mpr":7983,"mpo":28665,"mpl":18530,"mps":7384,"mpt":6961,"ms ":7519,"moi":10596,"mod":8999,"mon":39570,"mor":34638,"mot":9918,"mou":10824,"mpa":13773,"mmé":6819,"mus":17057,"mul":7053,"mun":72605,"mi ":7506,"min":39720,"mil":42847,"mis":18157,"miq":9951,"mit":10387,"mie":22944,"ièr":39842,"ièm":14714,"ié ":12315,"mmu":65417,"iét":13514,"miè":11922,"mma":8876,"mme":60859,"yst":10800,"ys ":16246,"yen":7179,"yan":8923,"ué ":23787,"uéb":10376,"uée":52636,"tér":21675,"tés":15841,"tél":10092,"tée":11073,"tèr":7376,"tèm":7268,"té ":120644,"sée":25892,"sér":11984,"sé ":33254,"réé":15299,"vri":19364,"vre":15576,"vra":7557,"vir":10748,"vil":38576,"vin":28833,"vic":9378,"vid":11495,"vie":25896,"vit":11987,"vis":24817,"ré ":18385,"rès":23870,"réc":11353,"rée":15149,"réf":7765,"réa":27396,"rén":7293,"rég":60534,"rét":7637,"rés":42404,"rép":7979,"voi":21769,"vol":18467,"ver":63394,"ves":12081,"ven":32893,"vem":17149,"vel":23577,"vea":6695,"vec":28438,"ve ":39545,"val":17132,"van":23469,"var":6981,"vai":21124,"uté":12504,"ux ":89405,"uve":48851,"uvr":11649,"usi":30125,"use":23041,"ust":23088,"uss":29122,"uti":33117,"ute":37011,"uto":14655,"utr":16881,"us ":97767,"ut ":58626,"ura":21007,"ure":61643,"urg":15629,"uri":18044,"urn":17785,"uro":15520,"urs":51288,"urt":11255,"ur ":263487,"upe":34564,"uma":8198,"umb":11805,"ume":16466,"uni":34130,"una":10830,"une":273015,"um ":21626,"ult":20316,"uli":14760,"ule":23978,"ula":21824,"un ":279711,"uil":20013,"uin":15450,"uip":9137,"uis":42982,"uit":34639,"ul ":11106,"ui ":87556,"uct":19646,"ude":13798,"udi":11213,"ue ":252733,"uch":11064,"ueu":15091,"uer":16125,"ues":65033,"uen":9815,"uel":31495,"ub ":7072,"uat":10437,"uar":8894,"uan":14849,"ubl":25725,"ud ":22607,"tué":65263,"typ":6932,"ty ":7203,"tré":9963,"tur":39432,"tut":8273,"tud":13647,"tue":18585,"ts ":96064,"tre":117480,"tra":79361,"tri":46224,"tru":15862,"tro":41105,"tta":10904,"tte":35818,"ttr":6953,"pée":8023,"péc":10887,"to ":12714,"pér":17722,"tiè":7263,"toi":21756,"tob":11135,"tou":31577,"tom":9528,"ton":34164,"tor":25952,"til":20545,"tif":22436,"tie":47885,"tir":8477,"tiq":62118,"tit":40283,"tis":20099,"tin":33045,"tim":9651,"tio":235114,"thu":11875,"tia":8957,"tic":25452,"tiv":30333,"thé":16932,"pèc":18042,"tem":70682,"ten":37907,"tel":13125,"tec":11914,"th ":10711,"teu":75009,"tes":56728,"ter":82954,"ti ":26511,"tho":14369,"the":16191,"Éta":23397,"ège":6746,"èce":21913,"ère":67047,"ète":9916,"ène":11368,"ème":27688,"ès ":31596,"édé":24476,"éga":13682,"égi":62907,"écé":13741,"édi":27690,"éce":13512,"éci":16144,"ée ":173192,"écu":7487,"écr":22748,"éco":23770,"éfi":6764,"ées":42716,"éen":8156,"ébe":6903,"éal":20524,"éve":11864,"évi":9923,"éré":8391,"évo":13471,"évr":9038,"éta":51936,"éti":16412,"éte":7748,"étr":10893,"étu":6864,"été":39872,"éme":8284,"émi":12731,"émo":10213,"éna":8958,"éni":9260,"éli":8934,"éle":12690,"éma":11177,"éo ":6991,"équ":14445,"éra":46783,"ére":13799,"éri":83768,"éro":16442,"éné":26266,"ése":23781,"ési":28918,"épa":51263,"épu":10400,"épo":7454,"élé":15180,"és ":45990,"ême":12159,"êtr":10139,"çai":86644,"ût ":9033,"ôte":8179,"île":9715,"éé ":7606},"n_words":[66338594,78580813,56850284],"name":"fr","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/ga.json b/contrib/languages-data/ga.json
new file mode 100644
index 0000000..73d5eb1
--- /dev/null
+++ b/contrib/languages-data/ga.json
@@ -0,0 +1 @@
+{"freq":{"fág":374,"fái":240,"fán":17,"fás":43,"Bá ":22,"D":3670,"E":1972,"F":3366,"G":4685,"A":5367,"B":7488,"C":9211,"L":3210,"M":5923,"N":2064,"O":1295,"H":1083,"I":7769,"J":910,"K":695,"U":762,"T":5807,"W":639,"V":513,"Q":100,"P":3001,"S":6241,"R":3017,"Y":140,"X":81,"Z":115,"f":10484,"g":35594,"Fea":348,"d":35722,"e":80219,"b":22235,"c":48153,"a":205992,"n":112182,"o":48493,"Fei":128,"l":55972,"m":27549,"j":233,"k":1737,"h":92271,"i":133366,"w":1117,"v":1580,"u":29979,"Fer":45,"t":60494,"s":65408,"r":82828,"q":111,"p":7714,"z":576,"y":2584,"x":359,"Fha":60,"Ã":207,"É":2404,"Fho":14,"Fhe":48,"Fhi":99,"Ã":813,"fá ":15,"Fia":48,"Ú":160,"Fhr":107,"Ö":14,"Ó":660,"Fil":63,"Fin":103,"í":22652,"ë":49,"Fio":121,"é":21077,"Fir":16,"è":30,"ç":26,"Ã¥":17,"ä":37,"ã":17,"â":15,"á":26187,"à":53,"ü":78,"ú":11150,"ù":19,"ö":111,"ò":31,"ó":13717,"ð":28,"ñ":23,"Ä“":22,"féi":573,"Ä":45,"ı":15,"Ä«":25,"Å‚":23,"че":14,"Å":29,"Å¡":15,"Å«":19,"Ñк":41,"то":18,"ÑÑ‚":21,"Fat":14,"féa":120,"Far":31,"Fao":27,"Fad":18,"Fai":28,"itn":37,"ipé":81,"itl":172,"itr":218,"ipí":17,"ito":21,"itu":20,"itt":39,"its":92,"itz":45,"ity":81,"Eri":32,"ism":40,"isl":100,"iso":33,"isn":120,"isp":63,"iss":59,"ist":1852,"iv ":26,"ita":62,"itc":21,"ite":2146,"ith":4604,"iti":613,"irí":823,"Esp":20,"iré":62,"ɪ":16,"isé":44,"ius":114,"ium":34,"Ern":19,"iva":35,"ix ":29,"É™":39,"ivi":35,"É›":15,"ive":99,"ilí":349,"ipp":21,"ips":24,"ipt":76,"ipi":79,"Eur":16,"ilé":72,"Eva":15,"Eve":20,"is ":8915,"ion":1312,"iop":36,"ior":150,"ios":247,"iot":378,"iof":15,"iog":41,"iol":50,"iom":483,"ipe":96,"ir ":10292,"irv":17,"irs":353,"irt":1857,"iní":139,"iro":33,"irp":47,"irm":319,"iné":195,"irn":286,"irk":27,"irl":227,"iri":2132,"isi":1776,"ish":105,"isf":50,"ise":1193,"isc":1361,"isb":24,"isa":22,"iu ":157,"iqu":19,"ˈ":48,"imí":77,"imé":209,"irf":27,"ire":4833,"irg":383,"irb":139,"ira":42,"ird":294,"irc":279,"Ë":17,"it ":669,"El ":21,"ünc":20,"Eil":84,"Eis":23,"Ì":82,"ja ":23,"μ":37,"ν":74,"ο":100,"Eng":16,"θ":18,"ι":51,"κ":26,"λ":62,"δ":20,"ε":36,"η":30,"α":107,"β":19,"γ":29,"έ":19,"ά":19,"ί":36,"ití":138,"Emi":14,"ité":82,"Eo ":105,"isí":148,"Eli":30,"Ell":16,"Eor":261,"Eol":17,"Eog":51,"ivé":51,"Eoi":40,"Eoc":19,"iza":25,"ω":17,"Ï":15,"ÏŒ":27,"σ":40,"Ï‚":68,"Ï":66,"Ï€":27,"φ":26,"Ï…":24,"Ï„":48," l":8985,"ÑŒ":28," m":7148,"Ñ":32," n":15246," o":1707,"Ñ":18," h":3099," i":18215," k":158,"Ñ‹":19," d":12271," e":3594,"Ñ…":18," f":6713,"ц":14," g":7608,"ч":57,"Ñ€":135," a":48402,"úsa":108,"Ñ":109,"kie":20," b":10871,"Ñ‚":81," c":12912,"у":41," y":42," z":14," u":1021,"kin":66," t":9735," w":69," v":164," p":2643," s":15941," r":5178,"úta":17,"úth":153,"km ":105,"ús ":237,"ki ":55,"И":15,"К":15,"П":17,"Ð":21,"Ð’":16,"Д":14," J":910," K":682," H":1081," I":7545," N":2056," O":1160," L":3193," M":5910," B":6902," C":8023,"úra":117,"kha":23," A":4758,"С":21," F":3245," G":4206," D":3525," E":1742,"Gea":306,"л":108," Z":111,"к":120," Y":140,"й":72," X":80,"и":188,"п":19,"о":212,"н":176,"м":39,"Cás":28," S":5918,"г":35," R":3015,"úrs":169,"в":118,"Ger":46," Q":98,"б":29," P":2788,"а":242,"Geo":92," W":633,"з":23," V":510,"Gen":35," U":701,"úrt":165,"е":136," T":5610,"д":38," á":890," í":1985," é":7791,"únt":454,"gái":142,"gál":25,"kel":20,"gán":133,"ken":22," ó":1986,"kes":25,"gás":18,"Cé ":51,"ker":42,"ket":22," ú":1600,"key":21,"úpa":264,"úr ":181,"úpl":35,"Gla":76,"ke ":58,"Ghe":106,"Gha":320,"úlr":31,"Gho":18,"últ":26,"Ghl":88,"Ghi":30,"Ghu":20,"Ghr":519," Ã":761,"úna":570,"Gio":40,"Gin":21," É":1719," Ã":149,"Gil":20," Ó":649,"únm":24," Ö":14,"hÃi":46," Ú":148,"únd":18,"ku ":25,"kov":15,"kol":18,"ks ":25,"hÃs":21,"hÃo":25,"×”":23,"ו":21,"×":14,"ל":17,"×™":36,"géi":147,"úsá":344,"útó":18,"gí ":33,"ר":17,"Gao":19,"Gal":229,"Ö¸":14,"géa":181,"Ö¼":16,"Gar":111,"Gai":188,"Gae":1050,"hÉi":662,"Gab":25," Ãi":88," Ãr":98," Ãt":543,"Ùˆ":29,"Bó ":14,"ÙŠ":52,"fór":53,"fós":55,"fón":15,"Ù„":53,"Ù…":48,"Ù†":39,"Fui":17,"Fua":59,"د":34,"Ø­":14,"ت":14,"ب":21,"Ø©":18,"ا":87,"fói":40,"ع":14,"س":25,"fód":32,"ر":41,"Bót":22," ˈ":34,"fío":57,"fír":23,"fís":212,"kar":18,"kan":18,"Béa":950,"Flo":29,"Fhé":16,"Fla":23,"Fle":15,"fút":14,"ka ":67," Ð":21," Ð’":16,"Fra":387," Д":14," И":15," К":15,"Fri":39,"Fre":40,"A ":162," П":17,"Bío":106,"Foi":71,"Foo":31,"For":64," α":14," Ga":1406,"F ":106," Bú":24," Ge":490," Cá":50,"Da":525,"Dé ":27,"Cu":433," Bó":52," I ":184,"Cy":22,"Cl":805," Bí":110,"Cn":52," Fo":226,"Co":2077," Fu":111,"Cr":445," Fr":436," Fh":371," Fi":384,"Ce":563," Fl":82,"Ch":2601," Bé":806,"Ci":377," Ha":296,"G ":74," Dá":241," He":239,"Ed":73,"Ea":766," Có":81," Gw":17,"Du":229," Cú":124," J ":15," Go":230,"Do":507," Cí":18,"Dl":37," Gr":356,"Dr":164," Gu":114," Gh":1129," Gi":149,"De":754," Gl":184,"Di":257," Cé":88," Gn":25,"Dh":350,"cúp":30,"cúr":63,"Fe":557,"Bá":48," Id":99,"H ":53,"Déa":55,"Fa":174," Ia":276," Dú":161," K ":14,"cúi":261,"Eu":46,"cúl":23,"Ev":36,"Ex":17,"Er":77," Dó":33," Hu":86,"Es":55,"En":53,"Em":36,"Eo":495," Dí":30," Ho":194,"Ei":150,"El":97," Dé":97," Hi":140,"ha ":3363,"Cá":56,"Ge":520," Ji":54,"Ga":1705," Fá":64," Je":87,"Bú":28,"I ":379," Ja":245,"Bó":57,"Fu":117," Iv":17,"Fr":485,"Bí":112," Ir":128," Is":5161,"Fo":242," It":14," Im":262,"Bé":956,"Fl":84," In":364," Io":352,"Fi":403,"Déi":15,"Fh":373," Il":46,"ham":346," M ":16,"han":1176,"hao":727,"hap":30," Ka":137,"hai":6276,"hal":414," Ke":187," Ki":123,"har":2134,"has":684,"B ":234,"hat":707," Kh":43," Fé":97," С":21," Jo":392,"II ":143," Ju":72,"hae":331," Jr":24,"hag":201,"hab":166,"had":297," Fó":592,"hac":666," N ":85," La":605," Le":483," Li":406,"C ":207,"hba":15," Kn":16," Ñ€":15,"hd ":21," Ko":48," Kr":33,"hbh":223," Ku":37," Mc":98,"Au":67," Ma":1326,"Ar":549,"At":157," O ":116,"As":333,"D ":237," Mi":402,"Ba":1930," Mh":1153," Me":1282,"Ae":65,"Af":126,"dá ":242,"he ":2867,"Ag":62," Lo":611,"Ab":180," Hé":19,"Ac":138,"Ad":72," Ll":24,"Am":211,"An":878," Ly":37,"Ao":683,"Ap":36,"Ai":875,"hch":612," Lu":335,"Al":673,"By":31,"Hit":18," Ne":228,"а ":44,"Bu":392," Iú":336,"Br":787," Na":383,"Ca":1185,"E ":51," Ni":185,"Bh":1538,"Bi":171,"Be":744,"hda":17,"Hig":22," Mo":403," My":20,"hde":119,"Bo":264,"Hil":25,"Bl":146," Mu":284,"hdi":36,"Hio":16,"hdh":155,"dál":94,"Ku":37,"hel":99,"hei":1299,"dái":241,"Kn":16,"hea":3474,"Kr":33,"Hay":19,"Ko":50,"Le":491,"hew":16," A ":89,"Li":408,"N ":133,"dát":17,"hes":67,"her":184,"dár":15,"haí":305,"heo":669,"hen":86,"dán":99,"hem":16,"La":609,"Lu":335,"Ly":38,"hfh":57,"hfi":58,"Hé":19,"Ll":24,"Lo":614,"hfe":113,"Dá ":15,"hfa":66,"Me":1284,"hi ":227,"Mh":1155,"Mi":403," B ":218,"hfr":27,"O ":148,"hfu":664,"hfo":149,"Ma":1327,"Mc":98,"hgh":63,"My":20,"Mu":285,"Mo":406," C ":40,"Ni":186,"Ne":229,"Na":387,"P ":39,"Iú":345," Ao":620," Ap":32," Am":200," An":855,"Dál":164,"Hel":19," Al":487," Ai":787,"Hei":32,"Dái":50," Ag":59," Ae":60,"Nu":291," Af":79," Ac":123,"No":499," Ad":67,"Hea":28," Ab":148," Ba":1677,"Ol":192," D ":160,"On":32,"Oi":439,"Oc":20,"Od":23,"Hen":60," Au":67," At":140," As":265," Ar":510,"Her":56,"Ob":21,"Gi":165,"hig":33," Be":713,"heá":241,"Gh":1136,"hie":19,"Cé":98,"hid":79,"Gn":26,"hic":130," Bi":165,"Gl":218," Bh":1534,"hia":602,"Cí":19,"hip":47,"hio":429,"Gr":417,"hin":549,"Go":237," Bl":127,"him":81,"Gu":127," Bo":242,"hil":430," Br":722," Bu":374,"Có":94,"Gw":18,"his":511,"Cú":234,"hit":119,"J ":18,"hir":287," By":31,"Ha":296," Ca":1052,"Dá":246,"He":239,"Hi":140," Ce":495," Ci":334,"Dé":102," Ch":2590,"Ho":194," Cn":49,"Dí":30," Cl":788," Cr":398,"Hu":87,"hfé":14," Co":1382,"Dó":33," Cu":398,"hfó":38," Cy":22,"K ":26,"Cúi":194,"Dú":193," F ":85,"Ib":14,"Ia":309,"Id":105,"dé ":34,"hm ":163," Da":489," Di":255,"Io":441," Dh":344,"Im":285,"In":411," De":739,"Il":49,"Iv":17," Dr":151,"Is":5163," Dl":36,"It":14," Do":485,"Ir":130,"Ja":246,"L ":51," Du":227,"hn ":148," Ea":709,"Ji":54," Ed":71,"Fá":72,"Je":87,"hla":719," El":96,"hle":240,"Jo":392," Ei":130,"hli":2004,"Fé":97,"Fó":594," Es":55,"hlo":95," Er":76,"Jr":24," Eo":356," En":50,"Ju":72," Em":35,"Hal":38,"hlu":153,"Ka":139," Ex":17,"M ":83," Eu":45,"Han":37,"Ham":35," Ev":35,"Kh":43,"ho ":34," Fe":551," Bá":46,"Har":97,"Ki":125,"hma":28,"déa":414," Fa":169,"hmc":14," H ":25,"Ke":192,"céa":541," Tó":45,"go ":1892,"Ur":48," Tí":110,"Un":145," Té":54,"Ul":134,"Ui":83,"glu":75,"glo":14," Tá":2087,"ghé":59,"Ua":134,"ghá":14,"gle":54," Sú":23," Só":54,"Pó":40,"gli":33,"Tu":566," Wy":14,"Tr":269," Sí":148,"Pé":28,"gla":380," Wo":76,"To":290,"Th":1249," Sé":182,"Ti":174," Wi":221,"Pá":248,"Te":461," Wh":33," We":93," Sá":44,"Ta":319,"Úis":14," Wa":161," Rú":279,"V ":65,"Sw":14,"й ":53,"Sy":24,"St":806," Ró":193,"Su":269,"Sí":182,"Wo":77,"gob":19,"Wi":224,"cí ":85,"Wh":33,"Sé":185," Ví":26,"Wa":162,"hOs":21,"Rú":279," Ze":18,"We":94,"Sá":50," Zi":16,"Ró":193,"hOl":21,"Y ":15,"hOi":42,"gni":28," Za":30,"Vo":36,"Rí":362,"gne":64," Uí":112,"giú":420,"gna":72,"Vi":167," Yo":40,"Ré":222,"X ":33,"Va":115," Ya":26,"Ve":93,"Rá":59," Ye":18," Tú":22,"gmh":16,"céi":99,"Má":456,"gs ":22,"Qa":19,"glú":20,"úba":28,"úd ":102,"Mé":25,"Gré":160," Ù…":17,"Pu":37,"Pr":303,"Lí":32,"glé":50,"о ":14,"Lú":395,"S ":72,"Gua":19,"Pe":309,"Lá":163,"Pa":425,"Gui":35,"Lé":53,"Pl":114,"Po":635,"úc ":28,"Pi":190,"Ph":604,"Os":120,"Ot":24," ا":42,"н ":32,"Op":26,"gon":18,"Or":118,"R ":47,"gos":16,"gor":55,"Ow":22,"Ox":16,"Se":734,"Sc":406,"Si":351,"Sh":551,"Sg":27,"Sn":28,"Sm":30,"Sl":185,"Sk":30,"Sr":230,"úda":213,"Oí":18,"Sp":350,"So":189," a ":8371,"gní":96,"Ru":470,"Ñ€ ":23,"úcr":20,"Ry":20,"gné":36,"Cór":28,"U ":28,"gnó":47,"Sa":1373,"úca":22,"Re":192,"gra":412,"Ná":110,"Ri":337,"gt ":23,"úch":359,"Rh":17,"Cón":21,"gri":67,"Né":26,"Ro":461,"gre":39,"gná":104,"Ní":182,"Cói":21,"Mí":197,"cío":21,"cín":25,"Qu":70,"Mó":140,"Grú":25,"T ":38,"cís":24,"Mú":20,"cít":16,"Ra":236,"Mü":22," R ":39,"в ":19," Ow":22," Ox":16,"gto":41," Os":95," Ot":23," Or":112," Op":26,"Gre":42," Po":543," Lé":53," Pl":107,"Gri":35," Pi":185," Ph":599,"có ":37,"gua":61," Pe":292," Lá":163,"Gra":99," Pa":406,"gue":52," Nu":290," No":498,"gth":443," Ol":168," On":31,"gte":20," Oi":369," Od":23," Oc":18," Ob":21,"hUa":22,"gta":637,"gy ":14,"grú":149," Ra":236," Mü":22," Mú":20," Mí":196," Mó":140," Qu":68," Né":26,"cón":291,"cói":80," Ní":182,"údá":21,"b ":1035," Ro":460," Re":192," Ná":110,"cód":14," Ri":337," Rh":17," S ":17," Lú":395,"Gru":17,"Úcr":31,"gut":34," Lí":32,"Gro":24,"gur":328," Pr":285,"gus":7532," Pu":36,"gré":40,"a ":43319," Mé":25,"Úda":15," Qa":19,"grá":39," Má":456," Sy":24,"Céi":17,"Tú":25,"úl ":165," Sw":14," Su":231," St":802,"úit":93,"Ye":18,"úis":399," Ta":308,"Ya":26,"úir":269," Th":1246,"Yo":40," Ti":162," Pá":205," Te":438," Tr":254,"Giú":44," To":282,"Uí":112," Pé":25,"Gle":76,"cós":67,"Ghá":16,"Sú":30,"cór":164," Ry":20,"cóp":18,"Wy":14,"Só":54," Ru":470,"Tá":2089," Sa":1332,"gsú":141," Sg":27," Sh":547,"úin":409,"Glu":16,"Té":56," Si":309,"úil":1485," Sc":405," Se":630,"úig":450," So":181,"Tó":46," Sp":350,"Go ":71,"Céa":22,"úic":75," Oí":17," Sr":217,"úid":61," Sk":30," Sl":167," Sm":30,"Tí":131," Sn":28,"ún ":707,"úla":387," Va":113," X ":22," Ve":93,"Gor":57," Rá":59," Vi":167," Ré":221," Rí":362," Vo":36," Tu":471," Pó":39,"Za":30,"Ze":18," Ua":104,"Zi":16,"Ví":26," Ui":69," Ul":132," Un":136,"God":16," Ur":43,"Gob":16,"iai":2569,"i ":7943,"iam":415,"ial":677,"bó":145,"ian":964,"fy":18,"gc":1529,"gd":28,"hE":204,"cá":614,"ge":2607,"hF":113,"ias":115,"iar":1266,"gf":27," fá":646,"Iod":189,"bú":25,"hA":434,"iat":624,"ga":4696,"gb":67," io":626,"bé":131," im":729,"fl":50," in":3888," il":62,"ff":123,"fi":830,"ic ":568,"fh":1623,"Ini":60,"fr":651,"iab":149,"Inn":16,"fu":1205,"ft":48,"iac":136,"Int":32,"iad":874," is":4270,"fo":1220,"Ins":58," it":46,"bí":296,"iag":42," ir":76,"ibl":72,"j ":21,"cú":400," fú":16,"ibi":111,"gy":22,"có":705," m ":25,"hf":1201,"hg":74,"ibr":504,"hd":373,"dá":713,"he":9095," hÃ":49,"hb":245,"hc":639," gá":43,"ha":17871,"hO":88,"gn":464,"gm":31,"cé":763,"gl":642," fí":105,"gi":1171,"Iom":65,"hI":157,"gh":5251,"gg":41,"id ":2038," fé":577,"Ion":38,"gu":8062," fó":153,"hU":40,"gt":1166,"gs":210,"Ios":63,"gr":790,"cí":192,"Ior":55,"ibh":672,"ibe":76,"go":2060,"dt":1341,"du":779,"dw":67," có":303,"dúl":59," cú":315,"dy":102,"dúi":243,"g ":4176,"dúr":62," ha":566,"ea":25239,"eb":101,"dún":41,"ec":179," he":103," dá":331,"ed":379,"de":6172," gi":157,"dd":146,"dg":84," hI":155," gh":896,"df":76,"di":2454," cé":299," hO":88," gn":249,"dh":7736," gl":224,"dm":58," cí":22,"dúc":72," gr":303,"dl":217,"do":1640," go":1885,"dn":21,"ia ":680," gu":367,"ds":128,"Ill":14," hU":40,"dr":674,"ew":190,"ex":98,"eu":111,"ev":181,"ey":348,"ez":71," dú":293," ia":1300,"fa":1504,"Inb":19,"Inc":22," id":903,"h ":22971,"Ind":128,"fe":932,"bá":362,"gC":1143,"eh":31,"eg":159," hi":225,"ib ":82,"ef":77,"ee":269,"el":1747,"ek":71," dé":333,"ei":8910," ho":149,"ep":165,"Imp":113," dí":197,"eo":5144,"en":2705,"Imr":156,"em":248,"et":550," ht":48," hu":37,"es":1134,"er":2470,"aí":5441," dó":278,"iet":28,"cb":18,"ca":3365," ni":34,"e ":26562," ng":227,"iel":96," nd":382,"eál":273," ne":306,"ien":60,"eán":1781,"by":43,"bs":46,"br":1157," na":8256,"ier":82,"eár":16,"bu":707,"ies":49,"bt":15," iú":22,"eád":40,"bo":363,"ied":26,"bp":167,"ieg":18,"eái":390,"bl":1578,"иÌ":22," mu":97,"cG":32,"bh":11551,"bi":450,"ig ":954,"cC":35,"bb":61," mo":107," mn":21,"eác":154," nO":20,"be":1029,"dc":62,"db":293," ol":129,"da":3031,"In ":39," on":26,"f ":269," oi":794,"ifo":72," oc":122,"cy":33,"ibé":23," of":143,"cu":1138,"ct":536,"ibí":99,"ifr":59,"cs":370," ob":69,"dT":187,"cr":1405,"co":2084,"iff":28,"ife":99,"cm":91,"cn":208,"ck":346,"ifi":245,"cl":898,"ci":1417," nu":480,"ch":26341," no":28,"ce":4885,"cf":27,"cc":47," le":4947," há":141,"icr":56,"c ":2108,"ics":128,"ict":334,"icu":19," li":530," n ":179,"icn":56,"ico":53,"ick":139,"icl":81," la":460," mB":573,"icm":54,"ici":351,"ich":719,"ice":1644," gé":72,"eá ":311,"ie ":209," km":111,"ica":132," hÉ":663," hÃ":55," me":369," nG":425,"az":83,"ay":273," mh":955," nI":25,"ba":2824," mi":457,"ids":14,"d ":7267,"idt":208,"at":6355,"as":7371,"idr":97," hú":26,"ar":22782,"ido":20," nA":40,"ax":41,"idm":20," ma":2135,"aw":83," mb":882,"av":283," nD":124," nE":15,"au":284,"ak":145," lu":291,"idi":1657,"al":6550,"idh":1444,"ai":28977,"aj":44,"idg":41,"bP":203,"ao":4221,"ap":476,"ide":714,"idd":27,"am":5107,"an":31829,"idb":243,"ac":15816," hé":28,"ad":8059,"ida":56,"aa":74," lo":308,"ab":2446,"ag":12220,"ah":164,"ae":3237,"af":197,"nu":707," ae":136,"nt":6330,"ns":1273," ag":9107,"nr":644,"np":33," ab":662,"no":556," ac":887,"nn":13453," ad":142," am":1152," an":14934," bP":203," ao":494,"nz":50," ap":19," ai":2945,"ny":183,"nw":29,"nv":31," al":169,"oe":87,"of":346,"oc":3503,"od":1311," ar":6072,"oa":84,"ob":1794," at":2168," as":1109,"om":4389,"on":9178," d ":589," ba":1293,"ok":119,"ol":4027,"oi":8724,"oj":15,"og":1022,"il ":4239,"oh":246," bh":6443,"ot":1050," bi":158,"os":2328,"ov":241,"ou":522,"op":291,"oo":309," be":526,"or":3909," bp":167,"r ":26836," bo":185," bl":423,"ox":26,"ow":261,"oz":32,"oy":150,"ifí":195,"lá":2770,"pe":951," bu":524,"pa":1122,"pc":34," br":367,"pl":469," ca":803,"lé":691,"po":724,"ph":1119,"im ":440,"pi":497,"ika":15,"lo":964,"ln":33,"ige":988,"lm":247,"hé":1982,"ll":4477,"ls":671,"iga":43,"lr":97,"hí":2469,"ii ":17,"lp":55,"hó":1143,"lw":14,"lv":106,"icé":38,"lu":1017,"lt":2738,"Idi":92,"igh":3179,"hö":28,"igi":784,"ly":208,"igg":14,"hú":882,"igu":24,"nA":42,"igt":17,"o ":4410,"nD":124,"mc":74,"icí":106,"nE":15,"md":20,"igo":16,"ma":4924,"mb":1185,"ign":27,"mg":14,"mh":8902,"nI":25,"me":1225,"iá":23,"mf":48,"nG":433,"ml":170,"mi":1584," b ":71,"mn":295,"nO":21,"mm":166,"mp":948,"mo":376,"mr":638,"mt":67,"ms":365,"mu":293,"idé":131,"ió":116,"my":76,"p ":418,"iú":2995,"na":20087,"nb":151,"nc":1202,"nd":1981,"ne":5883,"idí":295,"nf":133,"ng":2368,"nh":89,"ni":2645,"ik ":21,"nj":39,"nk":107," c ":38,"nl":294,"nm":973,"imo":55,"imn":120," et":14,"imm":26,"fó":227,"ims":271," en":17,"imr":401," eo":211,"imp":429,"fí":323," ei":675,"imf":18,"ime":312,"jo":35,"fé":693,"imi":716,"imh":994," gC":1143,"ki":194,"kh":56," fe":599,"ip ":117," bá":278,"inc":452," h ":53,"ke":227,"gá":342,"ind":132,"hÃ":50,"ina":1555," fa":1032,"inb":24,"ka":183,"imt":31,"fú":18,"m ":2796,"аÌ":17," fu":504,"inn":3418,"gó":548,"inm":801,"ky":50,"ino":76," fr":394,"gö":27,"ks":87," bí":103,"inr":60,"ku":55," fo":815,"int":1223,"ins":402,"inf":38,"ko":112," bé":67," fl":33,"hÃ":55,"ine":2867,"inh":20,"gí":69,"ing":553,"kr":26," fi":426,"kk":18," fh":1406,"hÉ":663,"ini":1121,"kl":45,"gé":329,"inl":22,"km":117,"ink":22," cá":319,"li":6865," ge":143,"lh":37," gc":1511,"lk":48," hE":204,"há":2539,"le":11033," ga":579,"iob":126,"ioc":151,"ld":296,"lg":924,"iod":20,"lf":136," hA":434,"la":10153,"mB":573,"inu":20," i ":6264,"lc":271,"lb":620,"iny":32,"gú":29,"n ":44303," bó":54,"igí":38,"hr":4331," cl":537,"hs":439," cn":57,"hp":68,"iko":20,"dí":554,"igé":189," co":1181,"dó":754,"hw":24," cr":447,"ht":5712,"hu":4839,"iki":22,"hk":20,"hi":3686," ce":1239,"ike":19,"hn":1628," ch":6174,"ho":3923,"hl":3740,"hm":370," ci":449,"dé":521,"ila":50,"id":7038,"ilb":41,"ic":4509,"ib":1678,"ia":8556,"ih":33,"in ":9648,"ig":6298," da":665,"if":738,"eá":2972,"ie":631," dT":187," cu":711,"hy":66,"dú":564,"k ":471,"iq":21," do":1320,"ir":22505,"ilo":20,"is":15956,"it":9092," dl":128,"ill":1658,"iu":350," dr":195,"iv":271,"eó":36,"ilm":27,"ix":45,"ii":26,"ilg":818,"ij":27,"ik":120," de":4551,"ili":1289,"il":13368,"ild":34,"im":4227,"in":22865,"ilc":72,"ilf":25,"io":3173," di":196,"ile":3808," dh":1220,"ip":561,"ima":59,"je":23,"fá":699,"imb":35,"ч ":34,"Iar":271," ea":2630,"io ":128,"ji":36,"iz":73,"iy":39," dt":1102," du":676,"ils":248,"ilt":514,"l ":9412,"ja":59,"ilv":45,"ль":14,"hlú":76,"xi":32,"té":533,"Dúi":63,"tí":1793,"Dún":93,"tó":758,"ми":19,"xt":17,"wy":58,"hlé":82,"ww":87,"só":186,"sú":492,"z ":155,"hph":63,"Dúc":24,"hló":128,"xa":66,"tá":3852,"xe":16,"hlí":29,"wi":117," ví":29,"how":16,"wl":15,"sé":2086,"ла":34,"wn":96,"hlá":205,"wo":78,"ле":18,"sí":985,"wr":17,"ws":41,"wt":17,"ró":676,"hol":348,"hom":626,"hon":716,"hog":133,"hoi":807,"y ":1513,"ко":39,"hos":256,"rú":779,"hot":43,"wa":203,"hop":25,"we":124,"sá":626,"hor":473,"ré":2568,"dí ":288,"vi":359,"hob":241,"vr":17,"vs":40,"ки":31,"rí":3930,"hod":30,"vn":14,"vo":75,"hoc":113,"hni":60," tú":106,"uz":23,"ux":50,"uv":36,"uu":19,"rá":2009,"ve":416,"hno":28,"va":328,"hnu":19,"x ":157,"ui":6290,"uk":43," té":207,"ul":621,"ue":219,"uf":51,"ug":2486,"uh":17,"hiú":107," tó":85,"ur":1536,"us":8283,"hna":260,"ut":560,"um":955,"un":1808,"uo":20,"hne":600," tí":564,"up":67,"ty":172,"hme":74," sú":61,"tz":64,"hmh":34,"déi":40,"tu":987,"hmi":33,"ик":15,"tt":384,"tw":33,"pó":261," ww":43," só":70,"ий":37,"ub":394,"ua":4295,"ич":35,"ud":441," tá":647,"uc":433,"w ":201,"pú":26,"to":1115,"tn":113,"hul":73,"pé":170,"hui":1613,"tm":76,"tl":375,"hug":337,"ts":494,"tr":2509,"pí":75,"ро":21,"tp":64,"huc":17,"ра":26,"tf":21,"hua":936,"pá":620,"te":7229,"hub":28,"ti":2504,"th":14962,"dó ":211,"v ":84,"oú":35,"tb":247,"tc":62,"ta":12304,"htt":53,"su":1468,"htr":64,"ss":379,"st":5067,"sy":34,"sz":28,"sw":26,"sl":560,"hth":78,"sk":149,"tO":20,"sn":666,"sm":219,"sp":789,"so":753,"hte":89,"sr":416,"tS":312,"oí":60,"sd":35,"sc":3980,"hta":2099,"sf":132,"se":3651,"sh":1228,"tI":37,"sg":76,"si":3838,"rz":27,"u ":794,"hst":33,"nú":375,"tA":118,"sa":7835,"sb":93,"hsp":79,"rr":1332,"оÑ":14,"ор":16,"rs":1355,"rt":5685,"ru":2167,"rv":57,"nó":3141,"rw":39,"rx":21,"ry":348,"ní":1203,"hse":111,"rp":389,"hsf":34,"ол":16,"ом":15,"hsc":29,"ro":1820,"rn":1077,"он":24,"rm":1323,"né":446,"rl":1840,"rk":221,"nç":15,"hsh":62,"ri":5360,"ов":54,"rh":40,"hu ":168,"rg":994,"rf":169,"hsa":44,"re":8800,"ná":2686,"rd":1927,"rc":1015,"hnú":19,"rb":776,"hru":236,"ra":9135,"t ":8611,"mú":53,"hnó":36,"hné":52,"mó":1228,"qu":91,"hní":184,"но":37,"hro":198,"mí":387,"mé":353,"hre":546,"hná":193,"Dón":21,"nÉ":14,"hri":328,"má":744,"ht ":2941,"на":27,"hra":1213,"dír":63,"s ":29560,"lú":564,"dís":17,"pt":179,"pu":104,"díl":15,"ló":456,"dín":53,"pp":84,"lí":989,"pr":684,"dío":88,"ps":108," ru":1283,"htó":70,"Hum":14," nó":2812,"htú":179," sa":4679," nú":48," tA":117," sf":15," se":1093," sc":1162," si":1192," tI":37," sh":787," tO":20," sn":472," sm":68," sl":198," sr":276," tS":311," sp":393," so":340," mí":271,"dór":16," mó":1033,"ви":37," t ":559," mú":31," ra":690," re":138," ná":1297,"htá":92," ri":660," né":46," ro":398," ní":458," pu":29,"hrí":388," pr":520," lí":138," lú":62,"hró":88," s ":76,"hy ":43,"hrú":276," má":90,"dóc":22," mé":92,"dói":476," nÉ":14," os":81," ot":23,"hum":273,"hun":734,"hus":58,"hut":20,"ав":21,"hur":315," or":279,"ан":35," r ":14,"ай":20," lá":1959," pe":214,"IV ":29,"hrá":475," pa":99," lé":265," pl":119," po":531,"hré":565," pi":69," ph":761," rú":25," sá":66," ró":83," y ":28," sí":561," sé":1980," va":18,"dú ":74," ve":45," rá":141,"zz":17," vo":28," rí":368," vi":31," ré":1371,"Hor":17,"zh":29,"zi":69," ua":370,"еÑ":16,"ер":17,"Hou":27,"ze":68,"vá":31,"ен":14,"ек":14,"za":84," pó":35," tu":399,"Hol":53," ur":39," um":41,"zo":52,"ví":54," ui":526,"vé":90,"yg":21," ta":838,"ye":100,"uá":49,"yf":14,"yc":51,"yd":69,"ya":136,"yb":15,"tú":630,"huí":244," st":1040,"оÌ":24," su":1397,"yw":35," pí":25,"ев":18," tr":1087," ts":243,"yt":19," pé":22,"ys":107,"yr":61,"uí":380," to":270," th":2550,"yp":14,"yo":56," ti":480,"yn":135,"ym":43,"ué":20,"yl":73," pá":201," te":1172,"yk":16,"yi":42,"Are":14,"Arc":17,"Ard":151,"Ara":133,"Arm":66,"gCú":110,"ffy":15,"ffe":33,"ffi":21,"bás":179,"fer":28,"faí":35,"feo":21,"fea":451,"bán":27,"As ":53,"bái":111,"fei":332,"fhr":56,"fhu":70,"Ath":59,"Atl":73,"fia":38,"Ast":182,"Ass":25,"fhe":162,"fhi":169,"fho":376,"fha":173,"Art":45,"Asa":26,"gCu":31,"gCr":41,"gCl":15,"gCo":682,"gCi":43,"gCe":68,"gCa":115,"Óg ":20,"Aus":24,"far":64,"fao":645,"fan":33,"Aug":23,"fai":206,"fad":258,"fac":71,"Ógl":34,"ff ":26,"fe ":57,"fa ":140,"Ba ":684,"eys":19,"exa":49,"ez ":54,"ews":23,"ewt":15,"Bai":726,"Bal":86,"Ban":106,"Bay":32,"Bar":140,"Bat":24,"Bas":44,"eta":28,"ete":77,"eti":18,"eth":90,"eso":16,"est":189,"ess":89,"eto":24,"etr":56,"ets":16,"ett":84,"etu":17,"ew ":88,"eve":60,"eva":24,"evo":14,"Aca":46,"evi":62,"eut":17,"Abh":106,"eur":21,"eus":24,"ex ":32,"Ada":23,"Ach":57,"ey ":289,"ewa":22,"epe":15,"eph":65,"er ":853,"epa":17,"eos":35,"eor":359,"Ado":17,"eom":21,"eol":1069,"eon":124,"aíd":33,"es ":655,"Aer":41,"epp":19,"erl":77,"eri":131,"erg":65,"erh":16,"ere":93,"ená":60,"Aga":17,"erf":70,"erc":31,"erd":48,"era":154,"erb":43,"et ":98,"Afr":106,"aít":187,"aío":2153,"aín":34,"aíl":74,"esh":15,"esi":19,"esc":30,"ese":21,"esa":23,"ery":28,"erv":17,"err":112,"ert":175,"ers":225,"ern":128,"erm":66,"erp":23,"ero":49,"eks":14,"en ":1506,"Aib":300,"elb":39,"ela":302,"eld":65,"Aif":16,"Aig":121,"elf":14,"Aid":28,"ele":76,"eli":66,"elm":20,"Aim":21,"Ail":68,"Ain":61,"ell":274,"Ais":61,"elo":57,"Air":154,"Ait":29,"els":60,"elt":152,"Al ":17,"eo ":814,"emb":18,"ema":39,"eme":32,"emo":20,"emi":38,"emp":22,"ene":80,"enh":19,"eng":17,"enb":18,"ena":216,"end":83,"enc":57,"eno":22,"enn":128,"eni":51,"ens":89,"ent":221,"enr":49,"Ala":73,"Alb":377,"aí ":2937,"An ":507,"eoi":2441,"eog":109,"Ali":19,"eod":36,"eoc":94,"Ale":50,"Alt":15,"ego":31,"ege":29,"All":35,"egi":20,"Alp":16,"Amh":81,"Ama":63,"ek ":36,"Ang":71,"eib":67,"eic":489,"Ana":16,"And":78,"eip":74,"eis":2272,"eir":1778,"eim":365,"eil":1930,"ein":197,"Ant":93,"Anr":30,"eid":455,"eig":136,"Ann":43,"eif":17,"Aoi":37,"el ":546,"Aod":28,"eit":1087,"Ar ":54,"Aon":605,"em ":43,"cé ":113,"Bré":15,"Brá":20,"Bus":17,"Bul":20,"Bun":205,"Bur":29,"git":32,"gis":154,"gir":14,"Bui":33,"gil":38,"gip":32,"gin":265,"Bua":24,"gio":95,"gid":19,"gig":26,"gia":23,"ghu":53,"ghr":185,"ght":98,"ghs":14,"hIm":15,"hIn":39,"ghn":371,"hIo":62,"gho":20,"ghi":177,"ghl":361,"ghe":310,"hIa":25,"gha":654,"ghc":42,"ghd":172,"gcú":68,"gcó":255,"gcé":46,"Bru":49,"gcá":19,"gfo":19,"gaí":93,"cán":103,"gen":61,"hFo":15,"geo":88,"ger":72,"hFr":49,"ges":26,"cás":29,"Byr":24,"Buí":18,"gh ":2708,"gea":414,"gei":17,"cái":420,"hFi":19,"gel":59,"cál":28,"hEo":135,"hEa":45,"hEi":18,"gcr":73,"gco":270,"gcl":65,"gcu":132,"gca":128,"ge ":1840,"gci":53,"gch":18,"gce":392,"Brú":19,"gba":47,"gab":28,"gac":522,"gad":1739,"gai":436,"gas":69,"gar":258,"gat":20,"gal":128,"gao":46,"gan":561,"hAf":47,"hAb":16,"öl ":26,"hAs":63,"hAr":24,"hAo":33,"hAn":19,"hAl":173,"hAi":35,"ga ":749,"Cab":19,"Cad":42,"Cae":22,"Cal":103,"Cam":61,"Cai":194,"Cas":82,"Car":213,"Cat":300,"Cao":35,"Can":62,"Cap":27,"Bea":469,"bót":50,"Ber":55,"Ben":44,"Bel":48,"Bei":56,"fré":32,"frá":23,"fur":17,"bói":55,"bón":20,"Ãt":545,"Ãr":101,"Bhi":15,"Bhr":197,"Ãi":134,"Bhu":117,"Bho":33,"fud":114,"fua":289,"fun":18,"fui":747,"Bin":21,"Bil":27,"Big":43,"Bir":28,"Bio":15,"ft ":28,"fra":143,"fre":342,"fri":80,"bío":179,"bít":82,"Bha":357,"Bhe":114,"Bhó":20,"for":412,"fos":28,"Bhí":533,"fon":18,"Blo":17,"foi":441,"Bhé":114,"fol":56,"béi":58,"fiú":44,"foc":169,"Bhá":20,"fog":33,"Bli":37,"Bla":57,"fhá":105,"fhé":430,"fhí":42,"Bre":227,"fo ":28,"Bra":117,"fhó":29,"Bro":105,"béa":62,"Bri":217,"fid":30,"fic":105,"fie":34,"fig":207,"fil":119,"fin":52,"fio":69,"fir":44,"fis":60,"fit":18,"Bob":22,"Bol":34,"Boi":21,"Bon":27,"Boo":20,"Bor":28,"Bou":17,"Boy":27,"övs":26,"Ó ":534,"Óg":62,"Ói":16,"da ":574,"dd ":46,"Cur":28,"dbh":268,"Crí":132,"de ":3405,"dac":276,"dad":34,"dal":63,"dai":439,"Cua":50,"dae":36,"dat":54,"das":53,"dar":472,"Cui":121,"dao":417,"dan":122,"dam":218,"Cun":15,"day":18,"Cum":170,"dde":19," Éi":1640,"ddi":20,"ddl":19,"ddy":21,"Ãs":59,"Ão":123,"dch":53," Éa":60,"cun":17,"Éa":62,"cul":63,"cum":194,"cui":390,"cua":56,"ctr":133,"De ":104,"cto":75,"cti":62,"cte":57,"cth":78,"cta":42,"dTu":94,"dTr":14,"dTí":21,"Der":45,"Des":14,"Dei":367,"crú":31,"cy ":21,"Dem":25,"Den":24,"Dea":77," Ãs":38," Ão":95,"cré":18,"crí":673,"crá":38,"cus":41,"cur":146,"Éi":2322,"cté":51,"Dam":30,"É ":47,"Dan":89,"Dao":73,"Dar":147,"Dav":75,"Dai":34,"Dal":29," Ó ":534,"cks":22,"Chn":17,"Cho":745,"Chr":206,"ài":32,"Che":174,"cki":21,"Chi":387,"Chl":125,"cla":165,"Cia":62,"cle":66,"chá":513," Óg":54,"Chu":175," Ói":14,"cky":14,"ám":250,"chí":120,"án":4189,"Cin":46,"áp":43,"Cio":16,"Cit":56,"ái":8489,"clu":187,"chó":321,"ál":1018,"áe":16,"cli":29,"áf":24,"ág":444,"Cil":132,"áb":389,"ché":598,"ác":287,"clo":44,"ád":259,"cme":28,"cmh":19,"co ":63,"ár":1109,"chú":264,"át":991,"ás":613,"ció":20,"cni":17,"cne":34,"cna":87,"ciú":86,"DN ":19,"ão":15,"cnu":17,"coi":647,"cod":26,"Cla":68,"cog":48,"cob":20,"con":157,"col":164,"com":586,"cor":131,"cos":144,"cot":56,"cou":17,"clá":107,"Cea":371,"clé":67,"Cei":73,"clí":17,"Cel":18,"cló":73,"Ceo":53,"Cen":23,"clú":142,"cs ":59,"cmí":15,"ct ":18,"cre":91,"cná":17,"cra":247,"cri":85,"á ":8021,"cru":138,"cro":73,"cu ":204," Úc":26," Úi":16,"dTe":20,"cse":49,"csa":161," Ús":41,"Cha":583," Úr":42,"csi":66,"Clú":19,"Cló":22,"Cri":31,"cea":2323,"Cra":92,"Cre":44,"Ús":41,"Úr":44,"ch ":8976,"cer":23,"ces":38,"ceo":483,"cen":28,"Úc":31,"Úd":15,"caí":154,"cel":32,"Úi":18,"Cru":51,"cei":451,"Cro":66,"ci ":143," á ":180,"Co ":17,"Chú":59,"chb":68,"cha":3675,"chd":28,"Cli":497,"Clo":33,"Ché":62,"Chó":43,"Clu":118,"chu":2165,"Ciú":36,"cia":133,"ck ":201,"ceá":740,"cie":25,"che":1628,"chl":544,"chi":421,"cho":1057,"chm":18,"chn":85,"chs":16,"cht":5396,"chr":402," ár":46,"Coi":126,"Cog":108,"cil":150,"Cno":22," áb":129,"cim":18,"cig":39," áf":23,"cir":45,"cis":192," ái":498,"cit":27,"ciu":24,"cin":312,"cio":113,"Cnu":23,"cip":31,"cke":40,"Clá":26,"Cos":39,"Cor":193,"Com":288,"Col":188,"Coo":21,"Con":1012,"Cou":24,"ed ":132,"ós":524,"ót":285,"eba":20,"óv":19,"ebe":20," é ":6560,"ec ":22,"eac":4120,"eab":592,"eag":1122,"ób":33,"eaf":21,"eae":15,"ead":1421,"eai":31,"ean":6477,"ói":3006,"eal":1578,"óg":454,"eam":933,"ear":3955,"eas":1483,"eap":208,"ód":167,"óc":115,"ór":1746,"óp":56,"eat":955,"eau":27,"ón":1301,"óm":776,"ól":143,"íú":73,"ea ":2266,"ó ":5058,"eff":24,"ei ":24,"ega":26," í ":1887,"een":63,"eel":21," éi":443,"eed":16,"eer":16,"eaí":20,"eet":19,"edh":19,"edi":38,"ín":1128,"ím":31,"edd":25,"ede":40,"ío":6870,"ír":1028,"ít":709,"ís":504,"eg ":16," éa":781,"edy":19,"éú":20,"edo":20,"edr":22,"ech":43,"ee ":87,"íg":26,"íf":18,"íl":526,"ect":37,"íc":229,"íd":182,"ía":41,"eco":18,"íb":395,"dtú":90,"ës":14,"dtá":26,"í ":10858,"dté":16,"dtí":383," ís":30," ío":64,"dwi":23,"él":14,"éi":5025,"dró":22,"éo":14,"ép":16,"én":18,"és":18,"dwa":33,"ét":14,"ér":37,"dy ":84,"ë ":21,"drá":86,"éb":23,"éa":6775,"ée":17,"Edw":27,"ég":14," ór":14," ón":600," óg":50," ói":41,"é ":9003,"Ear":38,"Eas":45,"Eal":32,"Ean":323," ó ":1226,"Eag":152,"áí":25,"Eac":31,"Eab":120,"FC ":14,"dor":65,"don":416,"dom":343,"dol":27,"dos":18,"Dhú":133,"dlí":99,"Diú":26,"ds ":70,"dlú":36," ós":29,"dió":66,"diú":99,"doc":36,"dog":20,"doi":23,"Dhá":14," úd":122,"Dho":65,"Dhe":46,"dso":14,"Dia":55," ús":341,"dto":27," úr":105,"Dic":15,"dti":49,"dte":287,"Dhu":42,"dta":63,"Dis":41,"Dir":17,"Din":17,"Dil":14,"dtu":328,"dtr":51,"Die":22,"dun":14,"dui":527,"dul":80,"dub":16,"dua":69,"duc":14," ú ":1012,"dri":103,"dra":170,"dre":124,"du ":18,"dro":93,"dru":42,"Dha":18,"Dub":47,"Dua":39,"dha":260,"dhb":24,"Dun":28,"Dui":58,"dge":49,"ün":29,"ür":27,"Drá":15,"dhu":100,"dic":27,"dia":293,"dhi":80,"dhe":442,"dhr":104,"dho":62,"dhn":28,"dhm":310,"dhl":50,"daí":220,"der":172,"des":36,"dh ":5455,"dfa":52,"dea":701,"Dro":81,"dei":449,"del":62,"den":1153,"deo":76,"úp":306,"ún":1833,"úm":23,"úl":626,"di ":53,"út":197,"ús":719,"úr":674,"úi":3261,"úg":21,"úd":355,"úb":49,"úc":437,"dle":41,"dhá":244,"dla":24,"öv":27,"Dlí":35,"dmh":31,"dhú":132,"do ":629,"Dre":21,"Dra":14,"dhó":20,"dhí":53,"dhé":348,"ú ":2631,"Doi":47,"Dob":22,"dim":32,"din":294,"dio":40,"dir":1209,"dis":147,"dit":21,"deá":44,"die":30,"dif":40,"dig":48,"dil":18,"ör":19,"öl":32,"Dr ":17,"Dou":14,"Don":78,"Dom":200,"Dor":64,"rdá":19,"órá":63,"Nea":38,"rha":17,"Nei":30,"rdí":26,"rbó":47,"rga":146,"ri ":89,"rgi":106,"óva":19,"rgh":71,"rcá":36,"rge":229,"rgt":18,"rgn":54,"nát":222,"ret":35,"res":104,"rew":25,"rey":30,"óth":174,"rfa":28,"óta":81,"rfh":18,"rfi":21,"rfo":63,"rdu":18,"Nas":14,"Nat":55,"rds":35,"Nav":15,"rg ":211,"rea":4475,"ósa":37,"ree":55,"nác":45,"rec":16,"red":50,"nád":57,"nái":868,"rei":578,"reg":26,"rem":34,"nám":42,"ren":116,"nán":213,"óst":232,"rek":22,"nál":147,"rel":65,"raí":921,"rer":16,"nár":33,"reo":588,"óra":749,"Nig":14,"rda":143,"órc":14,"Nic":75,"Nia":22,"rdo":44,"órt":164,"órs":81,"rdi":66,"Nio":18,"rdh":169,"rdc":33,"rde":172,"ós ":247,"re ":2165,"ná ":1044,"ómá":33,"rbu":15,"rbr":21,"rco":14,"rcl":38,"rci":49,"rch":211,"rce":106,"rca":315,"ópa":18,"ór ":620,"rd ":1105,"rao":356,"rap":41,"rar":34,"ras":586,"rat":223,"rav":21,"rbi":53,"rbh":278,"rbo":26,"rba":184,"rbe":37,"rc ":189,"New":90,"rai":2238,"rah":38,"rag":65,"ran":640,"ram":134,"ral":89,"rak":15,"rab":86,"raf":72,"rae":116,"rad":543,"rac":1041,"rlí":86,"rpo":23,"rs ":166,"rlá":17,"rpe":23,"rpa":218,"rr ":255,"rph":42,"ros":108,"rot":96,"rom":262,"ron":135,"roo":19,"rop":32,"rou":48,"rov":34,"row":44,"rod":23,"roc":87,"ní ":309,"roi":637,"rol":45,"rok":18,"rog":54,"rno":16,"rnr":15,"rns":15,"riú":336,"rp ":54,"rna":252,"rne":186,"rni":111,"rml":62,"rmo":22,"rmu":15,"ro ":90,"rma":359,"néa":182,"rme":66,"rmh":95,"rmi":88,"néi":221,"Nao":110,"Nap":16,"Nai":48,"rlo":46,"rli":111,"rle":214,"rla":1273,"rn ":133,"rku":14,"rgí":14,"rks":24,"rki":19,"rgá":88,"rke":35,"rka":14,"né ":24,"rm ":191,"Iúi":305,"ótó":16,"Na ":69,"riz":14,"rl ":40,"rip":32,"rio":414,"rit":613,"ris":438,"riu":30,"rig":165,"ril":27,"rik":16,"rin":795,"rim":139,"rdú":27,"ria":919,"rib":71,"ric":942,"rid":144,"reá":343,"rie":109,"rif":21,"rk ":90,"nóg":29,"óg ":175,"rsá":56,"nól":18,"nói":152,"rwi":20,"nón":22,"rsé":16,"nót":37,"nós":133,"Nua":266,"rté":20,"rtí":355,"rtú":33,"rui":134,"rug":1110,"ruf":14,"rud":172,"ruc":68,"run":22,"rum":55,"óch":56,"rus":59,"rut":246,"óca":32,"rva":21,"rrá":17,"rve":15,"óda":35,"nóc":14,"nód":16,"ry ":291,"rrú":28,"ódh":24,"rsk":16,"rsi":104,"rso":40,"rsm":24,"rsc":217,"rsa":287,"rsh":40,"rse":211,"rta":707,"rst":140,"roí":25,"rtl":15,"rto":55,"ód ":76,"rte":314,"rth":1771,"rti":96,"nó ":2699,"rua":226,"rts":27,"rtr":15,"óbh":15,"rty":31,"rmé":23,"nín":101,"níl":49,"rmá":327,"níc":30,"rt ":2182,"rmó":19,"nít":71,"nío":623,"rné":17,"rro":35,"rri":84,"rre":71,"rná":209,"rra":630,"rry":95,"rrt":19,"rnó":52,"rní":19,"rrs":58,"sc ":327,"sab":21,"sac":654,"sad":54,"sae":23,"sag":75,"sai":592,"sal":68,"sam":141,"tAo":24,"ómh":688,"óma":33,"ónt":14,"sbh":16,"ónn":18,"sao":275,"óng":28,"san":1622,"sat":24,"sas":19,"sar":67,"óna":427,"óp ":28,"sa ":3740,"núi":159,"ógá":48,"tAb":14,"tAi":42,"nús":88,"óla":44,"nún":35,"ón ":775,"óim":116,"óil":212,"óig":90,"óis":269,"óir":1464,"óit":73,"óin":227,"nú ":75,"óip":36,"ódó":17,"óib":55,"óic":15,"óid":394,"ól ":82,"óiv":47,"ógr":30,"óga":179,"Nor":78,"ógt":15,"Nol":338,"Nob":19,"sha":314,"scó":82,"sho":57,"shr":63,"she":203,"shi":156,"shl":100,"si ":38,"scé":347,"scí":15,"scá":85,"sgh":17,"sfé":91,"seá":101,"sie":17,"sid":18,"sic":138,"sib":26,"sia":514,"sk ":19,"shu":50,"sit":136,"sir":78,"sis":198,"sin":1068,"sio":159,"sil":94,"sim":57,"sig":443,"scr":521,"sct":40,"scu":23,"Oid":16,"Oib":81,"Oif":26,"Oil":208,"Oir":93,"sbu":25,"se ":1138,"sca":692,"sce":1106,"sci":152,"sch":129,"sco":383,"scn":26,"scl":50,"sey":40,"ser":32,"set":23,"sh ":132,"sea":1145,"sei":128,"see":18,"saí":406,"sep":38,"seo":837,"sen":41,"sel":32,"slé":30,"slí":29,"spr":39,"sph":19,"slá":31,"spe":143,"spl":89,"spi":78,"spa":83,"sol":68,"son":394,"sor":16,"sof":18,"soi":82,"oí ":21,"soc":67,"sru":24,"tSu":38,"sné":54,"tSl":17,"tSi":42,"tSe":104,"sná":21,"sre":22,"tSa":39,"sra":197,"st ":494,"oíc":18,"ss ":98,"sli":105,"shí":50,"shó":29,"slu":19,"sgö":26,"sky":14,"sla":192,"shá":17,"sle":131,"ski":48,"ska":40,"sna":496,"siú":811,"sne":60,"smu":24,"so ":31,"sma":98,"smh":17,"smi":37,"swi":14,"stí":140,"sté":91,"stá":507,"stú":37,"stó":65,"suí":51,"Owa":15,"szo":14,"sse":71,"ssa":61,"sso":60,"ssi":66,"íú ":73,"spá":57,"ste":1218,"sta":1246,"spé":48,"stm":17,"sto":201,"sti":488,"stl":22,"stu":43,"spó":185,"str":448,"sua":60,"sub":21,"sui":1192,"sul":46,"sun":29,"srá":131,"tSí":34,"sy ":15,"tai":2305,"tal":289,"tae":1311,"tag":109,"tab":44,"tac":773,"tad":69,"Oll":155,"tbh":212,"tba":28,"tat":53,"tas":487,"tar":1806,"tap":16,"tao":264,"tan":292,"tam":111,"tch":56,"Ont":15,"te ":2398,"oú ":35,"ê°€":17,"Ope":20,"Ora":17,"ta ":3767,"Osc":19,"Ord":25,"Org":16,"Ost":52,"Osp":25,"Léi":39,"Plu":14,"pa ":431,"Phá":131,"Pla":56,"Pin":38,"Pil":15,"Pir":16,"Pie":43,"Pic":15,"Pia":16,"Phl":25,"Pho":215,"pch":27,"Phi":49,"Phr":41,"Phe":38,"lá ":1233,"pe ":40,"Pha":68,"par":151,"pat":36,"pas":37,"pac":105,"pad":34,"pag":31,"pal":41,"pai":149,"pan":23,"Pea":55,"phe":133,"pha":75,"Lár":36,"Per":28,"Pet":70,"Pen":31,"phr":237,"Pei":90,"pho":285,"Lái":95,"phl":74,"phi":39,"pi ":15,"ph ":45,"Lá ":25,"pea":396,"Pat":75,"Pas":17,"Par":135,"Pau":68,"pen":39,"lán":235,"lám":61,"per":79,"lár":422,"paí":59,"lát":214,"lás":33,"lái":556,"pei":268,"pel":21,"pla":221,"ple":113,"phá":114,"phí":17,"Pac":17,"phé":19,"Pan":18,"phó":39,"Pai":26,"Pal":24,"lé ":37,"phy":24,"pia":37,"pid":51,"pic":45,"peá":30,"pin":38,"pio":53,"pir":179,"pit":35,"por":56,"RA ":16,"pop":47,"pos":23,"poi":71,"pol":359,"pob":89,"ps ":19,"plí":36,"ppi":17,"plé":24,"plá":39,"ppe":27,"léi":437,"léa":199,"lí ":340,"Lú ":20,"pta":38,"pso":42,"ló ":37,"pub":20,"pte":41,"pti":15,"pth":22,"pto":20,"Pró":17,"lís":63,"lít":64,"pra":40,"pt ":15,"lío":233,"lín":268,"Prí":87,"pri":56,"pre":46,"pro":19,"lór":25,"lóv":14,"lóg":58,"lói":259,"lón":22,"Pro":75,"pus":14,"pun":14,"Pri":61,"Pre":38,"prá":19,"prí":405,"pró":88,"lú ":248,"Lí ":19,"Pob":269,"Pol":146,"Poi":14,"Pot":15,"Por":97,"Pow":27,"lún":59,"lút":57,"lúd":46,"lúc":14,"lúi":104,"Mí ":149," ال":36,"má ":30,"mát":17,"más":38,"mán":258,"mái":372,"Mái":98,"Már":312,"nÉi":14,"RC ":57,"Lún":278,"Lút":94,"Má ":17,"méi":105,"méa":226,"mí ":94,"mít":19,"míc":15,"mín":39,"mío":44,"míl":152,"Rac":22,"Rai":49,"Ram":22,"Ran":22,"Mün":20,"mó ":604,"Qué":14,"que":47,"qui":28,"móg":26,"mói":77,"mór":457,"món":35,"Qui":16,"Que":32,"Mói":32,"Mór":89,"Míc":34,"múi":17,"ra ":1834,"rb ":96,"ngn":22,"Isa":18,"ngo":25,"ngi":39,"ngl":144,"ngu":44,"ngr":50,"ncí":19,"ngt":83,"ngs":41,"ni ":60,"Iri":72,"nge":211,"ngf":22,"ngh":86,"nga":974,"ndí":14,"ndé":38,"ndó":18,"nha":48,"Isl":17,"ndá":65,"neg":14,"nei":19,"Is ":5111,"nel":42,"nen":32,"neo":447,"ner":75,"naí":1112,"net":45,"nes":120,"ndy":16,"ng ":349,"nea":1340,"ned":36,"nfh":67,"Ire":45,"ney":97,"nfa":14,"nfe":14,"nco":31,"nci":177,"nce":204,"nch":428,"nca":121,"ne ":3335,"ndu":14,"ndr":118,"nds":22,"ndo":64,"ndl":20,"ndi":217,"nde":329,"nda":522,"ncy":16,"ncu":14,"nal":159,"nam":348,"nan":389,"nao":175,"nap":14,"nar":134,"nac":2846,"nad":430,"nag":54,"nai":1426,"nc ":149,"nab":27,"nbe":25,"nbh":87,"nd ":363,"nat":73,"nas":642,"na ":12179,"iúd":104,"iúc":163,"iún":891,"iúl":283,"iúi":885,"iúr":175,"iút":14,"iú ":460,"Jac":51,"Jar":14,"Jan":31,"Jam":111,"msí":64,"ión":27,"ntú":67,"Jer":36,"nyi":30,"nté":57,"ntí":102,"Jea":24,"Fái":49,"ntó":92,"nz ":20,"ntá":73,"nsí":44,"nrú":15,"ny ":111,"nvi":15,"nró":30,"nrí":45,"nui":29,"nus":32,"nua":572,"nty":14,"nto":113,"ntu":65,"Jim":41,"ntr":159,"nti":241,"nth":35,"nta":3920,"nte":565,"nsp":24,"nso":37,"nst":156,"nse":204,"nsh":22,"nsi":157,"nsl":20,"nsk":16,"nsc":131,"nsa":161,"nnú":15,"nry":35,"nní":68,"nné":19,"nri":70,"nná":276,"nre":31,"nra":389,"nt ":782,"Féi":77,"nph":26,"ns ":220,"noc":72,"nod":15,"nol":15,"noi":128,"nom":39,"non":20,"nos":23,"nor":35,"nov":43,"Féa":15,"nne":1006,"nnf":20,"nna":2785,"nnc":59,"nnl":27,"nnm":15,"nno":36,"nni":557,"nnu":27,"nnt":268,"nns":21,"nnr":29,"nny":46,"nma":16,"nmn":111,"nmh":268,"niú":176,"nli":38,"nn ":8140,"nla":210,"nle":19,"no ":106,"ngá":45,"nm ":546,"ngó":117,"ngé":45,"Jos":64,"Jon":27,"nja":20,"ã‚¢":18,"Joh":203,"Joe":27,"Jr ":24,"Joy":18,"nig":486,"nie":54,"neá":219,"nid":36,"nic":278,"nia":209,"ndú":111,"nk ":33,"Joa":15,"niu":167,"niv":27,"nis":728,"nit":168,"nir":17,"nio":42,"nim":85,"nin":41,"nil":25,"ogr":39,"ogh":289,"ogl":14,"oga":398,"oge":56,"oi ":566,"odó":26,"ohn":167,"oha":45,"odá":193,"ohe":14,"ois":848,"oir":3309,"oit":255,"oin":1191,"oim":354,"oil":1296,"oif":183,"oig":99,"oib":236,"oic":136,"oid":224,"ok ":22,"ofá":17,"Jul":20,"ol ":417,"och":2829,"oci":28,"ock":58,"ocl":57,"ocr":39,"ocs":92,"oe ":46,"oca":254,"odg":15,"odh":743,"ode":33,"odo":22,"odr":47,"Fóm":579,"of ":150,"oda":77,"og ":142,"oft":18,"obá":17,"ofa":122,"oa ":16,"ob ":24,"oc ":64,"oan":17,"oad":17,"oba":305,"od ":80,"obr":83,"obl":440,"obh":750,"obi":26,"obe":98,"Ø© ":18,"Kan":19,"Kat":31,"Kar":34,"Kev":15,"Key":15,"oyd":15,"oya":32,"Ken":62,"Kel":29,"Kea":20,"otá":130,"osú":142,"owy":22,"osó":14,"ows":15,"osé":18,"LG ":24,"own":87,"oyl":19,"Kin":36,"otu":20,"ow ":44,"oth":323,"ott":66,"ots":23,"Kil":36,"otr":21,"oto":30,"ost":375,"ota":316,"otb":27,"ov ":33,"osh":28,"ose":75,"éú ":20,"osf":14,"osp":42,"oss":42,"osr":51,"osl":98,"oso":32,"osn":18,"oró":83,"oy ":46,"owa":27,"osá":17,"owe":32,"ovi":50,"oré":21,"ox ":19,"ova":45,"orá":51,"ove":70,"oug":50,"oui":57,"oul":26,"oun":65,"ous":31,"our":113,"out":79,"opo":21,"ítí":24,"ope":29,"olá":174,"oph":46,"opa":58,"opc":14,"olú":33,"os ":781,"oló":41,"opt":22,"oon":34,"ool":43,"oom":21,"ísí":16,"ook":30,"ood":67,"or ":345,"Khö":26,"oot":38,"oos":16,"oor":30,"ork":35,"orl":75,"orm":172,"orn":219,"oro":53,"orp":272,"orr":167,"orc":278,"ord":273,"oná":62,"ore":96,"orf":22,"org":242,"ori":126,"onú":23,"ou ":33,"osa":407,"osc":122,"ort":590,"ors":62,"oru":46,"onó":59,"ory":25,"omá":246,"ot ":44,"orb":118,"ora":509,"omó":111,"ola":1711,"old":55,"ítr":15,"olc":90,"on ":2106,"íth":89,"oli":70,"oll":644,"íti":59,"olf":34,"ole":55,"olg":25,"olr":30,"ols":26,"olt":377,"olm":81,"olo":58,"olu":29,"Kra":14,"íst":16,"om ":147,"ísh":22,"ísl":16,"oke":30,"ogá":20,"íte":478,"ona":897,"ond":268,"onc":148,"onf":22,"one":117,"ong":260,"oni":64,"onl":192,"onn":1898,"onm":22,"ono":40,"onr":302,"ons":354,"ont":2217,"ony":50,"oma":307,"ome":52,"omc":38,"omb":131,"omi":43,"omh":2862,"omm":49,"oml":86,"omp":156,"omn":22,"omo":20,"omr":44,"oms":14,"op ":47,"la ":2317,"ímh":16,"íne":254,"íni":119,"ínt":26,"ín ":561,"íle":271,"íli":20,"ílt":21,"le ":6126,"há ":229,"lca":35,"ít ":29,"lch":115,"lf ":34,"íní":128,"íre":446,"lde":34,"íri":75,"lda":20,"ldo":21,"íse":228,"ldi":15,"ldr":25,"íon":1757,"íol":105,"íom":1097,"íof":67,"lc ":14,"íog":60,"mBa":252,"íob":247,"lab":207,"íoc":2004,"lac":1652,"íod":679,"lad":283,"laf":29,"mBe":30,"lae":75,"lah":15,"lag":166,"lai":2053,"ír ":462,"mBl":19,"lan":820,"lam":183,"lap":14,"lao":133,"mBo":21,"mBr":58,"íos":602,"lar":224,"íor":199,"lat":121,"íot":31,"las":488,"lau":20,"mBu":18,"lav":45,"lay":20,"lba":399,"ld ":134,"lbe":35,"lbh":90,"ís ":182,"lbo":37,"gói":147,"Les":14,"Leo":49,"Len":15,"ky ":41,"Lei":85,"Ãsi":55,"Lee":32,"Lea":169,"ífh":16,"ích":119,"íci":16,"Lau":16,"íce":62,"gó ":16,"íde":34,"Le ":36,"göl":26,"Lai":234,"Lat":22,"Lar":18,"Ãos":108,"Lao":78,"Lan":50,"Lac":25,"íl ":179,"Lab":43,"Ãoc":14,"gól":15,"gór":363,"ídí":15,"La ":40,"llí":62,"llá":36,"lph":21,"llú":34,"ls ":42,"híc":20,"lló":16,"lon":282,"lom":21,"loo":18,"lor":72,"loc":108,"log":67,"loi":139,"los":46,"lot":18,"lou":14,"low":25,"loy":15,"hí ":1705,"lmo":15,"héi":849,"lme":15,"lmh":99,"héa":1120,"lma":25,"liú":299,"ML ":17,"lth":18,"lti":78,"Lib":24,"lto":55,"Lia":109,"íd ":117,"Lif":21,"ltr":42,"Lim":18,"Lin":50,"Lio":85,"hó ":54,"Lit":21,"luc":142,"Liv":18,"Leó":14,"lub":148,"lua":385,"íbh":382,"lsi":100,"íc ":20,"lso":25,"lst":34,"lta":1256,"ltb":209,"lte":433,"lse":63,"lsc":185,"lsa":41,"híl":84,"ía ":22,"hír":114,"hís":67,"hín":94,"hío":369,"lt ":143,"lra":80,"lhe":14,"Lui":93,"Lua":43,"Lud":14,"Luc":96,"Lug":29,"lge":761,"lcá":58,"lgh":15,"li ":52,"lga":54,"lfr":17,"lfo":14,"lbá":32,"lfa":28,"ley":85,"lex":35,"lev":17,"les":180,"hás":82,"let":44,"hát":26,"laí":1250,"hár":49,"ler":93,"leo":208,"háp":15,"lem":28,"mBé":150,"len":233,"hán":441,"lek":16,"hál":49,"hái":1498,"lei":1619,"hág":33,"leg":26,"hád":25,"lec":18,"háb":67,"lea":1609,"lg ":39,"lls":203,"llt":63,"lly":96,"lo ":74,"lla":935,"llc":15,"lle":424,"lli":518,"llm":62,"llo":69,"ln ":15,"lgá":15,"lgé":23,"Lou":71,"Lov":14,"Los":30,"lm ":60,"Loi":15,"ll ":1866,"Loc":214,"Lor":30,"Lon":181,"lit":361,"lis":292,"lir":387,"lip":69,"lio":164,"lin":417,"lim":385,"liz":19,"liv":28,"liu":30,"lic":201,"lid":19,"lia":3592,"lib":16,"lk ":16,"lil":22,"lig":150,"leá":626,"lie":64,"lif":258,"ma ":472,"húc":86,"húd":16,"húi":311,"húl":55,"hún":169,"hús":42,"húr":40,"mac":612,"mai":598,"mad":253,"mae":32,"mag":35,"mar":1718,"mas":175,"mal":86,"mam":24,"man":516,"mao":49,"mat":79,"mba":240,"mbl":503,"mbi":32,"mbe":81,"mbr":65,"mbo":25,"me ":182,"nDa":34,"nDe":15,"mbu":66,"nDo":14,"mch":67,"mdh":15,"mea":458,"met":27,"maí":234,"mes":142,"mer":79,"meo":30,"men":88,"mei":43,"nGe":21,"nGa":292,"mh ":1936,"Lyn":18,"mbó":35,"nGr":52,"mbí":90,"mbé":18,"nGl":33,"lva":29,"lve":18,"lvi":21,"lui":189,"lun":17,"lum":19,"lus":40,"Mei":735,"ly ":90,"Men":20,"Mel":55,"Mer":24,"hód":18,"hóg":71,"hói":282,"Mea":58,"Med":17,"lsú":44,"hór":401,"lsí":98,"hón":68,"hót":81,"hós":139,"ltú":88,"ltí":65,"ltó":312,"lvé":32,"lyw":29,"lym":15,"lyn":33,"luí":23,"Man":130,"Mao":38,"Mal":52,"Mar":371,"Mas":30,"Mag":40,"Mad":44,"Mak":15,"Mah":18,"Mai":143,"höv":26,"Mac":269,"McG":26,"McC":34,"hú ":144,"Mau":18,"Mat":68,"mpi":170,"mph":15,"mlá":60,"mpe":309,"mpr":28,"mlí":34,"mpl":157,"mps":56,"mpt":19,"ms ":24,"Mhá":43,"moi":28,"mod":31,"mon":137,"mol":22,"mor":39,"Mhí":40,"mot":24,"mou":21,"mpa":114,"Mhó":86,"mná":23,"mre":320,"Mol":14,"Mon":181,"Moo":22,"mní":56,"Mos":37,"Mor":60,"Mou":21,"msa":14,"mse":115,"mra":22,"Mik":20,"Meá":332,"Mid":25,"Mic":162,"mth":59,"Mit":15,"ió ":67,"Mis":38,"Mil":49,"Min":33,"Mio":27,"Mhi":56,"mst":20,"Mhe":373,"Mha":337,"msh":17,"msi":78,"mpá":18,"Mhu":157,"Mho":39,"mrí":232,"my ":60,"mur":22,"mus":69,"mui":145,"mun":25,"mrá":27,"mho":79,"nIo":16,"mhp":53,"mhr":513,"mhl":221,"mhn":220,"mhs":290,"mht":62,"mhu":94,"Mun":14,"nDú":32,"Mul":17,"mhb":105,"mha":2867,"Mum":29,"Mui":108,"mhg":44,"mhi":314,"mhd":145,"Mur":63,"Mus":20,"mhc":392,"mhf":40,"mhe":469,"mi ":37,"min":375,"mio":137,"mil":143,"mir":224,"mis":103,"mit":76,"mic":290,"mia":46,"meá":129,"mie":35,"mid":17,"mhú":61,"mo ":38,"mhó":274,"mhí":205,"mhé":50,"mle":33,"mhá":463,"mla":28,"mni":119,"nOi":18,"mna":22,"mne":64,"mmy":33,"miú":91,"mmi":21,"mmo":23,"mma":33,"mme":29,"Téa":54,"Tái":16,"Tá ":2060,"Súi":16,"Sói":47,"кий":26,"Sín":135,"Sío":26,"Wor":21,"Woo":16,"Wol":21,"Séa":149,"Whi":17,"Wik":16,"Wil":138,"Win":32,"TÉ ":39,"Sé ":26,"Sái":26,"Wes":45,"Rúi":254,"Was":16,"War":22,"Wal":73,"Róm":117,"Rói":56,"éi ":35,"vír":30,"éig":552,"éif":26,"éid":461,"éic":46,"éib":77,"éin":788,"éim":366,"éil":829,"éiv":16,"éit":178,"éir":1174,"éis":462,"éip":14,"ée ":15,"Vol":22,"Rí ":158,"Río":178,"ébe":15,"éal":1141,"éam":173,"éan":937,"éar":2079,"éas":293,"éad":1071,"éag":793,"éac":146,"éab":116,"Ré ":18,"Réi":59,"Réa":133,"zi ":14,"vái":19,"zer":17,"ze ":17,"zab":15,"zar":14,"zon":14,"Ãth":545,"ال":39,"véa":37,"véi":51,"Ãra":86,"之":46,"Ãit":16,"Ãis":79,"Ãir":27,"yst":15,"三":49,"ä¸":24,"uío":293,"ys ":50,"yon":20,"uí ":65,"za ":23,"Uíb":33,"ywo":25,"ye ":18,"yce":17,"ydd":20,"uái":30,"yes":20,"yer":38,"uán":14,"ya ":45,"tún":49,"túr":132,"tús":176,"túi":132,"You":19,"yd ":20,"Uí ":79,"yan":35,"yal":16,"yn ":41,"yle":30,"uéb":14,"ymo":15,"yne":26,"на ":23,"yin":31,"Tóg":25,"tín":149,"tío":154,"tíl":50,"tír":606,"téi":199,"téa":299,"tí ":809,"Ù† ":19,"Tír":114,"tú ":120,"tói":556,"tóg":93,"tón":40,"tór":36,"tán":115,"tár":64,"tát":339,"tád":22,"táb":174,"tál":25,"tái":738,"Stá":384,"té ":28,"Srá":139,"Sut":21,"Sur":14,"Sum":35,"Sul":19,"Sun":21,"Sui":19,"Sua":83,"Str":47,"Stu":20,"Sti":47,"Sto":43,"Sta":139,"Ste":88,"Spá":239,"sún":98,"súl":153,"súi":176,"súr":33,"Tea":279,"Pád":61,"Ten":26,"Páp":16,"Teo":19,"Tei":25,"Pái":139,"Tel":19,"ão ":15,"Tao":30,"Tam":21,"Tan":19,"Tar":51,"sú ":27,"Tai":37,"Tal":28,"Tag":65,"tá ":2371,"xas":26,"xan":25,"ww ":43,"Shi":41,"Shl":44,"She":97,"Shr":17,"Sho":28,"Sha":212,"Sim":55,"Sil":27,"Sir":43,"Sio":65,"Sin":55,"Seá":101,"Sib":15,"Sic":20,"Sia":23,"wys":21,"wyn":15,"Ser":18,"Sev":15,"Scé":14,"Sgi":14,"www":43,"sór":14,"TG ":26,"sói":127,"Scr":150,"Seo":77,"оно":15,"Sei":86,"Sea":356,"Sra":58,"séa":65,"séi":59,"Sru":15,"TV ":15,"Spi":26,"Spe":25,"sé ":1940,"Spr":18,"Spo":16,"Slé":27,"Sló":23,"Oíc":14,"wn ":71,"St ":18,"síc":29,"ws ":28,"sít":28,"sío":289,"sín":66,"Soc":18,"séú":20,"Sou":43,"Sol":26,"Som":19,"Son":17,"Sor":26,"Sla":28,"sí ":546,"Shó":16,"wor":22,"woo":34,"Shé":39,"Sli":87,"Smi":23,"sás":15,"wer":23,"sár":23,"sán":45,"wen":17,"sál":14,"wel":39,"sái":497,"Rya":15,"sác":17,"Éad":37,"Rus":14,"Rug":352,"wic":33,"Rua":57,"win":29,"Sag":39,"Sai":92,"Sam":333,"Sal":39,"TD ":24,"Sac":45,"Sab":15,"Sco":64,"Sci":16,"Sch":44,"Sca":96,"Sat":14,"Sar":19,"Sas":352,"San":139,"Sao":103,"rú ":278,"rúi":83,"rún":65,"rús":14,"rúp":255,"Éid":18,"Éig":34,"Éis":16,"Éir":2239,"wa ":20,"rúc":52,"rúd":20,"wan":20,"wal":16,"ови":23,"way":37,"war":66,"Sa ":139,"wai":21,"rís":49,"rít":177,"rír":17,"río":1887,"rín":63,"ríd":89,"ríc":21,"ríb":349,"ном":14,"нов":15,"vsg":26,"Rio":16,"Riv":17,"Rit":19,"ró ":41,"Ris":24,"Rin":78,"Ria":85,"Ric":56,"ríú":71,"Rat":26,"Ray":17,"rót":49,"rós":27,"róc":18,"ród":40,"róg":49,"ról":47,"rói":314,"róp":18,"róm":19,"rón":46,"Red":20,"Rei":34,"Nái":100,"Reg":14,"Ren":19,"Rep":16,"Rea":41,"via":20,"vil":33,"vin":52,"vic":53,"vid":56,"vie":19,"Roi":64,"Rog":14,"Roa":15,"Rob":88,"Roc":23,"Rod":16,"Ní ":122,"vit":24,"vis":41,"Roy":25,"Ros":122,"Ron":28,"ré ":111,"Níl":43,"réa":1091,"réi":1345,"rí ":1171,"vol":16,"von":30,"Néi":23,"ver":165,"ves":15,"rás":53,"rát":69,"vei":42,"rái":750,"rán":608,"ven":46,"rám":112,"vel":29,"rál":140,"rád":32,"rác":51,"ve ":67,"rá ":172,"val":28,"van":117,"var":26,"vac":33,"vad":16,"vai":30,"Ñки":27,"va ":51,"Van":14,"Val":47,"Vic":66,"Vir":27,"Vil":15,"Vin":17,"Ver":42,"Rát":50,"ux ":34,"uve":14,"usk":14,"ush":22,"use":38,"usc":28,"usa":33,"unú":106,"ust":89,"uss":40,"usp":33,"uth":369,"uti":18,"uta":20,"Uil":36,"utt":29,"us ":7938,"Ung":31,"Uni":93,"ut ":27,"urb":96,"ura":189,"urc":62,"ure":45,"urg":90,"uri":53,"url":38,"urn":75,"uro":30,"urp":15,"urr":39,"urt":87,"ury":23,"Ula":97,"ur ":612,"umh":128,"umm":17,"uma":512,"umb":45,"ume":17,"unt":128,"uns":24,"unr":31,"uni":30,"unn":30,"unc":40,"und":110,"una":569,"ung":63,"up ":17,"um ":194,"ulu":22,"ult":111,"ulo":15,"ull":75,"uli":38,"ulg":25,"ule":17,"ulf":16,"ula":104,"un ":599,"uid":1064,"uig":163,"uil":876,"uim":340,"uin":941,"uip":23,"uir":1098,"uis":276,"uic":175,"uib":54,"uit":1252,"ul ":152,"ugh":92,"ugb":50,"uge":17,"ugl":17,"uga":1558,"ugu":24,"ugt":630,"uha":14,"Pól":20,"uda":51,"udd":19,"ude":22,"udi":21,"Trá":40,"ubs":20,"Trí":30,"uca":20,"ue ":74,"uci":14,"uch":302,"uck":27,"uer":27,"uff":30,"udo":19,"ug ":71,"uee":24,"uen":19,"uel":34,"ub ":137,"ua ":465,"uat":139,"uas":240,"uar":105,"uam":28,"ual":155,"uan":118,"ubl":49,"ubh":89,"uba":46,"ubb":14,"Ua ":15,"ud ":277,"uai":2599,"uag":16,"uad":53,"uac":355,"uc ":20,"Uac":82,"Uai":24,"tuá":17,"ty ":159,"trú":23,"tró":28,"tré":177,"trí":558,"trá":247,"tur":96,"tus":32,"tut":21,"tui":74,"tua":318,"tug":303,"tz ":31,"pós":16,"two":15,"pór":40,"VI ":18,"tsí":19,"pói":180,"Tex":30,"Pár":28,"Ter":20,"ts ":73,"pío":31,"pín":27,"tre":362,"tt ":71,"tra":441,"tri":301,"tru":91,"tro":254,"tu ":58,"tsa":41,"tse":67,"tsc":23,"tsi":118,"Tha":57,"tsr":40,"The":480,"tso":36,"Thi":133,"tsu":19,"Tho":176,"Thr":22,"tta":38,"Thu":219,"tte":89,"tth":14,"tti":29,"ttl":27,"tto":23,"Tia":32,"ttp":53,"tts":17,"Tio":42,"Tim":32,"Tit":14,"péa":48,"tma":53,"thú":153,"to ":131,"péi":112,"tne":54,"tiú":375,"tp ":53,"tna":46,"toc":35,"Thá":100,"toi":111,"tog":48,"tob":29,"Thí":47,"tos":71,"tow":49,"tom":35,"ton":323,"tok":16,"tol":18,"tor":167,"top":31,"Péi":18,"til":72,"tie":23,"teá":53,"tig":135,"tir":335,"tit":122,"tis":104,"tin":198,"tim":310,"tio":296,"thy":18,"thu":979,"tia":170,"tic":139,"Tor":40,"Tol":19,"Tom":78,"Ton":21,"Tos":30,"tiu":19,"tiv":17,"Tog":21,"Toi":20,"tli":127,"thé":99,"thí":114,"thó":153,"tla":91,"thá":326,"tle":130,"tem":15,"ten":53,"pán":35,"teo":1066,"tep":28,"tei":324,"pái":517,"tel":48,"tee":15,"tea":2655,"Tro":44,"ted":69,"Tri":35,"Tre":38,"Tra":56,"th ":2421,"tev":28,"tes":48,"pás":47,"ter":367,"taí":552,"ti ":132,"thn":307,"tho":317,"thl":87,"thm":25,"thr":797,"ths":70,"Tur":38,"thf":27,"thg":15,"thd":15,"the":3504,"thi":663,"Tui":81,"Tul":23,"thc":164,"thb":46,"tha":4634,"Tua":184,"Tug":194,"вич":31,"àid":20,"Ìн":22,"Ìч":14,"áth":595,"bis":88,"bit":81,"ást":38,"bir":14,"bil":25,"bin":47,"áta":69,"bhú":16,"bhó":78,"bhé":79,"bhí":1218,"árú":23,"bhá":244,"ble":36,"bli":926,"bla":511,"boc":23,"bol":50,"boi":15,"bog":74,"blí":56,"bpo":38,"bpr":43,"bló":16,"bpa":22,"bpl":21,"bon":33,"bor":45,"bou":52,"bbe":22,"ált":86,"cCa":16,"be ":60,"áma":128,"ámh":113,"bam":96,"ban":414,"bal":290,"bai":1183,"bac":109,"bad":23,"án ":2509,"ála":655,"bas":43,"bar":115,"ánr":41,"áns":33,"ánt":45,"ánm":44,"ánn":32,"bea":484,"ánd":17,"ánc":23,"ána":1388,"ás ":419,"bei":63,"ár ":430,"bec":19,"ber":213,"baí":52,"beo":79,"bel":41,"bet":26,"ápa":41,"bh ":1159,"bhf":1122,"ása":96,"bhi":557,"bhl":1983,"bho":90,"bhn":402,"bhs":39,"bhr":844,"bhu":498,"árt":347,"bht":84,"árs":39,"bia":125,"ásc":33,"át ":311,"bhF":112,"árn":34,"ári":39,"bha":2405,"bhe":575,"ára":142,"bhc":24,"áda":66,"ách":270,"ág ":24,"ádr":77,"áfa":23,"ága":18,"ágt":374,"ca ":266,"áig":19,"áid":933,"áib":37,"áic":28,"ádú":57,"car":214,"cas":83,"cat":333,"can":421,"cao":97,"cap":21,"ál ":251,"cac":181,"áip":28,"cab":30,"áis":885,"cae":15,"áir":1722,"cad":202,"áit":897,"cam":73,"cal":252,"áim":58,"cag":16,"áil":1302,"cai":992,"áin":2550,"ce ":736,"cbh":16,"bri":171,"bro":48,"bra":206,"bre":499,"bru":63,"bst":23,"bpá":25,"bua":84,"bur":99,"bun":439,"bui":16,"bus":26,"brá":19,"brí":91,"bró":17,"by ":34,"brú":28,"ád ":33,"ábl":63,"ábh":315,"buí":16,"aka":23,"am ":799,"ake":39,"agá":70,"ajo":14,"afó":14,"al ":1358,"ail":3174,"aim":543,"ain":7349,"aip":101,"air":5994,"ais":2778,"ait":3108,"aiv":22,"adú":40,"ak ":15,"adó":396,"aig":2716,"aif":67,"aid":1852,"aic":689,"aib":509,"aho":27,"acó":14,"ahe":17,"adá":61,"aha":38,"agl":179,"agm":19,"agh":478,"agr":310,"ags":144,"agt":22,"agu":7542,"agn":79,"ago":48,"aoi":1842,"aof":47,"aog":45,"aol":452,"aom":117,"bPa":18,"aod":21,"bPe":17,"aob":407,"aoc":48,"anu":67,"anz":22,"any":20,"ano":191,"ann":7828,"anm":43,"ant":1573,"ans":231,"anr":74,"ane":93,"anf":24,"ang":1024,"anh":18,"ani":113,"ank":64,"anl":23,"ap ":67,"ana":2087,"anb":49,"anc":353,"and":712,"amu":114,"amm":39,"amo":44,"amp":203,"ams":48,"amh":2467,"ami":50,"ame":171,"amb":62,"ama":964,"alv":19,"alu":30,"alt":1341,"als":101,"alr":23,"alo":60,"alm":34,"all":1574,"alk":22,"alg":42,"ali":175,"alc":59,"ald":89,"ale":108,"ala":1139,"alb":131,"an ":16537,"agó":420,"ako":19,"aba":124,"abe":33,"abh":1558,"abi":26,"abl":14,"abo":23,"abr":55,"ae ":1425,"aca":623,"ad ":2107,"ac ":653,"ab ":562,"afo":26,"ai ":46,"aga":645,"agc":16,"age":73,"acá":56,"aen":85,"aem":24,"ael":663,"aes":47,"aer":113,"aed":15,"aei":796,"ah ":42,"afa":69,"aet":39,"ado":66,"adr":74,"adl":17,"adh":3910,"adi":55,"add":28,"adc":24,"adf":52,"ade":47,"ag ":2140,"ady":15,"ads":15,"adt":14,"adu":21,"aco":19,"acn":59,"acl":31,"ack":100,"aci":53,"ach":13609,"ace":48,"ada":1067,"adb":26,"af ":23,"act":78,"acu":209,"acr":43,"acs":125,"azi":25,"avá":25,"aza":20,"asó":66,"asú":74,"atá":1959,"ayn":15,"ató":45,"aya":22,"aye":50,"ba ":436,"amó":47,"at ":114,"arg":165,"arf":40,"aná":386,"are":86,"ard":1063,"arc":302,"arb":337,"ara":1215,"arp":24,"aro":41,"arn":274,"arm":678,"arl":1377,"ark":117,"anç":14,"ari":179,"aru":23,"arv":15,"anó":82,"arx":20,"arr":979,"ars":293,"art":2216,"au ":32,"anú":140,"asa":1535,"ary":92,"asg":16,"asi":30,"ash":50,"asc":579,"ase":61,"bPá":43,"asd":17,"aso":16,"asn":49,"asp":71,"ask":20,"asm":52,"asl":40,"aon":895,"bPo":90,"aot":134,"aor":130,"bPr":17,"aos":25,"ar ":12570,"apa":112,"ape":18,"alá":30,"api":16,"aph":25,"apo":27,"app":15,"apt":53,"aló":74,"as ":3979,"alú":82,"amá":22,"ava":69,"ax ":19,"arí":40,"avi":97,"ará":352,"ave":39,"asá":71,"arú":93,"ay ":128,"awa":22,"aró":146,"ata":651,"aoú":35,"ast":501,"ass":83,"asr":26,"atm":35,"atn":41,"atl":30,"atr":129,"ato":56,"apá":67,"ate":101,"ati":172,"ath":2853,"aw ":16,"att":85,"ats":25,"atu":25,"apó":24,"aul":83,"aur":29,"aus":26,"aud":27,"aug":19,"ον":16,"ος":28,"ος ":28,"ον ":15,"BC ":31,"AC ":14,"Úrs":30,"AD ":23,"Ï‚ ":68,"AM ":43,"Úsá":39,"ν ":30,"ÃC ":14,"α ":44,"三 ":16,"アア":15,"ий ":34,"之 ":27,"ич ":33,"το":15,"Ïα":19},"n_words":[1346129,1623635,1308708],"name":"ga","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/hi.json b/contrib/languages-data/hi.json
new file mode 100644
index 0000000..3f575b1
--- /dev/null
+++ b/contrib/languages-data/hi.json
@@ -0,0 +1 @@
+{"freq":{"ौदà¥":366,"ोली":389,"ोरà¥":465,"ोमी":388,"à¥à¤¤à¤®":479,"à¥à¤¤à¤¨":384,"à¥à¤¤à¤°":2639,"à¥à¤¤à¥":1391,"à¥à¤¥à¤²":430,"à¥à¤¤à¤¾":2582,"à¥à¤¤à¥€":679,"à¥à¤¤à¤¿":2706,"à¥à¤¤à¥":954,"à¥à¤¥à¤¿":2834,"à¥à¤¥à¤¾":3750,"à¥à¤¦à¥":869,"à¥à¤¦à¥€":1870,"à¥à¤¦à¥‚":492,"à¥à¤¦à¥‡":907,"à¥à¤Ÿà¥":2750,"à¥à¤Ÿà¥‡":7603,"à¥à¤Ÿà¥€":634,"à¥à¤Ÿà¤¿":711,"à¥à¤¤à¤•":539,"à¥à¤¬à¤¨":468,"à¥à¤®à¤•":445,"à¥à¤¯à¤•":2308,"à¥à¤®à¤¾":2864,"à¥à¤®à¤¿":1000,"à¥à¤°à¤‚":360,"à¥à¤®à¥€":437,"à¥à¤¯à¤¤":638,"à¥à¤¯à¤¯":385,"à¥à¤¯à¤®":558,"à¥à¤°à¤•":2642,"à¥à¤¯à¤µ":609,"à¥à¤°à¤š":370,"à¥à¤°à¤œ":367,"à¥à¤¯à¥":778,"à¥à¤¯à¤¾":6530,"à¥à¤§à¤¾":551,"à¥à¤§à¤¿":463,"à¥à¤¨à¤¾":572,"à¥à¤ªà¤¨":572,"à¥à¤ªà¤¾":591,"à¥à¤ªà¥":3747,"à¥à¤—त":419,"à¥à¤•à¤¾":992,"à¥à¤•à¥ƒ":823,"à¥à¤•à¤°":450,"à¥à¤œà¤¾":403,"à¥à¤žà¤¾":1890,"à¥à¤Ÿà¤°":764,"à¥à¤šà¤¿":1116,"ोंन":705,"ौर ":706,"ं":116003,"ः":743,"à¤":3754,"आ":15110,"इ":21960,"अ":27216,"ऊ":1116,"ई":8670,"उ":14369,"à¤":27071,"ओ":3623,"ऑ":695,"à¤":1181,"ग":41430,"ख":13380,"क":215389,"औ":11975,"छ":5467,"च":24607,"घ":4688,"ट":40532,"ञ":2576,"à¤":1114,"ज":58287,"ठ":3871,"ड":21061,"ढ":1924,"ण":16159,"त":129370,"थ":26984,"द":62970,"ध":21789,"न":137720,"प":89801,"फ":9525,"à¥à¤— ":934,"ब":39694,"भ":28885,"म":108014,"य":96626,"र":228209,"ल":79901,"व":82288,"ष":22409,"श":41726,"ह":118206,"स":149246,"़":11159,"ि":139433,"ा":290518,"े":193119,"ू":22463,"ृ":6345,"ी":110466,"à¥":44034,"ौ":4425,"à¥":228350,"ो":68898,"ै":59521,"ॉ":2831,"।":45019,"०":5718,"१":6322,"à¥à¤• ":738,"६":1789,"७":1738,"८":1943,"९":4350,"२":3762,"३":1587,"४":1437,"५":1969,"à¥à¤¨ ":1500,"ोजन":549," ख":3003," ग":13738," औ":11821," क":119739," ओ":938," à¤":1154," ऑ":666," ट":5844," ज":31363," à¤":690," च":7337," छ":3781," घ":3397," इ":19284," आ":12686," अ":26917," à¤":21008," ऊ":806," उ":13706," ई":1344,"à¥à¤« ":515,"à¥à¤® ":3545,"ोड़":535,"à¥à¤¯ ":9507,"à¥à¤° ":7477,"ोते":895,"ोती":1270,"ोता":2537," २":2506," ३":490," १":5155," ।":6381,"à¥à¤µ ":2678,"ोनो":409,"ोने":1142," प":54146," फ":4760," न":21169," म":54702," य":22634," ब":23290,"à¥à¤· ":1625," भ":19961," ड":2225," द":21719," ध":2884," त":12822," थ":8999," ह":72412," स":78945," ल":14663," र":25548," श":12231," व":27670,"à¥à¤¸ ":1196,"à¥à¤š ":578,"à¥à¤Ÿ ":1436,"à¥à¤ž ":468,"à¥à¤¡ ":1049,"à¥à¤  ":1088,"à¥à¤£ ":1354,"à¥à¤¥ ":1814,"ोगि":441,"ोगो":559,"à¥à¤¤ ":4240,"à¥à¤§ ":2352,"à¥à¤¦ ":1588,"à¥à¤¸à¤¾":631,"à¥à¤·à¥":370,"à¥à¤¹à¥‡":874,"à¥à¤¹à¥‹":949,"à¥à¤¸à¥":3660,"à¥à¤²à¥€":1260,"à¥à¤²à¤¾":820,"à¥à¤²à¥‡":511,"à¥à¤°à¤¦":1892,"à¥à¤°à¤¥":561,"à¥à¤°à¤¤":2596,"à¥à¤°à¤£":709,"à¥à¤¯à¥‚":989,"à¥à¤°à¤ª":384,"à¥à¤°à¤®":2375,"à¥à¤°à¤­":518,"à¥à¤¯à¥‹":1594,"à¥à¤°à¤¯":1925,"à¥à¤°à¤µ":977,"à¥à¤°à¤¶":494,"à¥à¤°à¤¹":1076,"à¥à¤°à¤¸":1694,"à¥à¤°à¤¾":9061,"à¥à¤°à¤¿":3453,"à¥à¤°à¥€":4034,"à¥à¤°à¥ˆ":386,"à¥à¤°à¥‡":9372,"à¥à¤°à¥‹":1860,"à¥à¤·à¤£":540,"à¥à¤·à¥‡":3114,"à¥à¤·à¤¿":1460,"à¥à¤·à¤¾":1408,"à¥à¤µà¤¤":875,"à¥à¤µà¤°":1044,"à¥à¤µà¤µ":540,"à¥à¤µà¤ª":609,"à¥à¤µà¥€":1298,"à¥à¤¶à¤¨":1985,"à¥à¤µà¥‡":430,"à¥à¤µà¤¿":632,"à¥à¤µà¤¾":6601,"०० ":579,"२००":1298,"१९९":360,"१९६":420,"१९७":415,"ंत ":1187,"ंड ":1010,"ंग ":1786,"ंक ":375,"ंभ ":447,"ंद ":452,"ंश ":574,"ंस ":426,"ंह ":485,"आप ":923,"अंत":806,"अंग":1106,"अकà¥":469,"इन ":809,"ंचत":1845,"ंचा":2297,"ंगà¥":1940,"ंगा":789,"ंगी":464,"ं। ":4299,"ंको":451,"ंकà¥":1624,"ंखà¥":1085,"ंगल":563,"ंटे":1825,"ंडि":369,"ंजा":609,"ंने":716,"ंबं":450,"ंपा":1007,"ंतà¥":1705,"ंति":457,"ंतर":775,"ंदà¥":705,"ंदी":875,"ंदि":932,"ंदर":556,"ंबर":485,"ंयà¥":368,"ंसà¥":2087,"ंसा":366,"�":2362,"थे ":819,"दन ":568,"थी ":791,"था ":4899,"तॠ":405,"थम ":523,"तंत":641,"तो ":1178,"ं ":59244,"ः ":691,"ताओ":415,"तान":1545,"ताब":607,"तार":557,"तिक":1335,"दो ":833,"तिज":375,"तिन":613,"ठ":2207,"तिय":1006,"तिर":505,"तिह":787,"आ ":1631,"नई ":675,"धन ":595,"दी ":4353,"दू ":610,"तरी":448,"तरा":413,"तरह":445,"दा ":857,"तरà¥":854,"दि ":1483,"तमा":404,"तमि":373,"णà¥à¤¡":966,"णों":461,"दर ":576,"दल ":742,"तथा":2577,"ततà¥":438,"णाल":368,"थ ":4109,"द ":7269,"ध ":3357,"न ":42221,"ड ":6850,"था।":3059,"ठ ":1539,"थाप":1123,"ण ":9157,"थान":1787,"त ":29579,"थित":2566,"धी ":490,"ज ":2925,"दकà¥":984,"ञ ":468,"धा ":464,"नट ":1785,"ट ":6235,"धि ":2431,"थवा":400,"च ":2310,"छ ":1066,"क ":31192,"तों":455,"ग ":7730,"ख ":1929,"तà¥à¤¸":696,"तà¥à¤µ":1830,"तà¥à¤ª":791,"तà¥à¤°":11745,"तà¥à¤¯":2756,"तà¥à¤®":761,"तà¥à¤¤":3587,"ठ":4565,"तीन":446,"तीय":3767,"तीस":449,"ई ":6870,"ै ":18248,"दार":977,"दान":855,"दाय":435,"दिल":1253,"दिश":630,"दिर":1086,"े ":101676,"दिय":1540,"दित":1005,"दिन":569,"नॠ":786,"दà¥à¤°":430,"ू ":2066,"दूर":671,"दूस":592,"ि ":14043,"नी ":5102,"ी ":75983,"ॠ":3758,"ा ":97285,"़ ":2122,"दसà¥":366,"ह ":13867,"ने ":12971,"स ":15469,"ष ":2544,"श ":4366,"व ":7299,"ल ":18408,"नि ":406,"दरà¥":1109,"ना ":7450,"र ":61007,"थी।":911,"य ":22189,"म ":13770,"भ ":1015,"नव ":411,"ब ":2705,"फ ":1049,"थे।":1504,"प ":5685,"टना":394,"डल ":378,"डी ":506,"डा ":698,"टरà¥":362,"ड़ ":675," �":578,"टती":1811,"ञान":1620,"टà¥à¤°":7028,"टà¥à¤Ÿ":617,"टेश":7359,"ढ़ ":636,"टीक":471,"णॠ":439,"णी ":648,"णा ":355,"तक ":1940,"ति ":4868,"ता ":14635,"तॠ":1219,"ती ":9089,"ते ":6248,"तन ":484,"ड़े":606,"ड़ी":1222,"ड़ा":1450,"डिय":1027,"तम ":378,"तर ":1819,"जे ":3791,"जा ":1910,"ज़ ":425,"जी ":1578,"चेन":368,"चीन":1126,"चà¥à¤¨":405,"जो ":3111,"चिक":478,"चित":1154,"चार":1698,"चाल":2122,"चिम":876,"जंक":1375,"चà¥à¤š":632,"छोट":541,"जनà¥":1234,"टक ":496,"जधा":578,"जनस":468,"जनी":553,"छूट":1815,"जसà¥":392,"टन ":357,"जहा":441,"ज़ी":477,"जिस":3147,"जिल":1663,"जिन":942,"जित":402,"जार":493,"जाब":559,"जान":2165,"जात":5721,"जैस":743,"जà¥à¤¯":2693,"जà¥à¤ž":2457,"जीव":1088,"टर ":1798,"टे ":2183,"टि ":384,"टी ":1658,"टा ":740,"ंघ":459,"ंख":1287,"ंग":7039,"ंक":4013,"ंड":2775,"ंट":3074,"ंज":1716,"ंच":5039,"केन":435,"ंश":984,"ंस":3555,"ंह":717,"ंव":676,"ॠ":2686,"खने":369,"ंध":1776,"ंद":4729,"ंन":763,"ंथ":399,"ंत":5036,"ंय":471,"ंप":2156,"ंभ":782,"ंब":1984,"ो ":17876,"à¤à¤š":523,"कोई":460,"कों":1045,"आत":950,"इं":1275,"कृत":1818,"आज":527,"अस":694,"अव":2719,"आक":698,"अल":1444,"अर":2196,"अभ":1051,"अम":1289,"अब":415,"अप":3105,"आई":372,"अध":2198,"अन":4741,"अथ":432,"इत":740,"इट":387,"आस":422,"इक":606,"आव":1056,"आम":383,"आय":903,"आर":1257,"आप":1303,"कृष":566,"आद":1231,"आध":1054,"आन":511,"अं":2476,"आं":508,"अत":785,"अक":781,"अग":575,"ं।":6630,"उप":2592,"उस":2266,"इन":2901,"इल":681,"इस":13512,"आ।":435,"ईस":387,"उत":2684,"उन":3336,"उद":1042,"कोच":453,"कोड":3985,"à¤à¤‚":534,"कोल":359,"à¤à¤":425,"कोश":370,"à¤à¤•":15570,"कà¥à¤Ÿ":677,"कà¥à¤¤":3551,"कà¥à¤¯":529,"कà¥à¤¶":1460,"कà¥à¤·":8266,"कà¥à¤°":2787,"कà¥à¤¸":4316,"ओर":445,"ओं":2444,"à¤à¤¸":714,"à¤à¤µ":3043,"à¤à¤²":360,"à¤à¤¸":427,"गर":2860,"गय":3634,"गल":1055,"गव":557,"खे":1130,"खो":528,"गम":494,"खà¥":2681,"गभ":616,"खा":2157,"गठ":414,"खि":516,"खी":461,"घं":1798,"गढ":741,"गण":767,"गत":1406,"गद":372,"गà¤":383,"कà¥":22746,"कै":618,"के":37285,"खन":943,"गई":554,"को":17366,"कॉ":440,"कि":12948,"की":20919,"गं":513,"का":35945,"कृ":2446,"कà¥":3003,"कू":402,"कस":766,"कव":956,"कह":3138,"कल":2049,"खक":372,"कम":1276,"कर":12591,"कप":515,"कथ":448,"कन":1065,"कड":382,"खं":419,"कत":2310,"कट":630,"और":11428,"कई":777,"कं":723,"चà¥":923,"ची":1816,"जं":1545,"चि":3214,"चा":5438,"चे":1055,"चौ":426,"चà¥":1117,"० ":1980,"जग":567,"गढ़":657,"चत":1993,"चन":1667,"चर":750,"चल":1479,"घा":478,"चं":451,"। ":33428,"गति":373,"चक":424,"गा":3720,"गह":412,"गी":1381,"गि":1175,"गà¥":1958,"गो":2483,"गà¥":4970,"गे":620,"गै":364,"घर":423,"टन":928,"ञा":1890,"टत":1909,"टा":1764,"टल":396,"टर":2646,"४ ":778,"à¤à¤¾":416,"३ ":798,"टक":827,"जो":3936,"जै":1167,"जे":4339,"जू":402,"जी":3381,"जà¥":839,"जा":12605,"जि":7022,"ज़":2064,"घंट":1796,"२ ":974,"जà¥":5651,"जन":5056,"जध":594,"छू":1841,"छा":395,"जह":523,"जस":483,"१ ":954,"जर":910,"जल":806,"जब":740,"छो":713,"जय":526,"जम":644,"ठा":568,"५ ":1070,"टे":10515,"ठन":369,"टà¥":8239,"टो":662,"टी":2663,"टि":1894,"ड़":5317,"डा":1474,"डि":2023,"डी":831,"डल":514,"६ ":930,"गभग":596,"तं":976,"तः":367,"ढ़":1322,"७ ":816,"डà¥":669,"डो":499,"डे":781,"डà¥":358,"णि":851,"णी":758,"णà¥":577,"णा":1135,"तक":2674,"८ ":811,"णन":433,"९ ":917,"तव":480,"ति":11566,"ता":20016,"तà¥":2358,"ती":14469,"तत":547,"तथ":2628,"तप":364,"तन":1343,"णà¥":1311,"तम":1364,"णो":495,"तल":566,"तर":5109,"थव":456,"था":11678,"थी":1994,"थि":3214,"ते":6961,"तो":1865,"तà¥":23294,"थम":587,"थल":512,"दक":1341,"दस":512,"दश":363,"दृ":463,"दू":2227,"दà¥":1528,"दी":5182,"दि":8815,"दा":4680,"थो":462,"थे":2431,"दन":776,"दल":1038,"दर":2412,"थà¥":491,"धा":4715,"नट":1830,"नत":1382,"खें":445,"नद":975,"धी":935,"धि":5621,"धà¥":599,"दो":2067,"दौ":382,"दà¥":15486,"नई":683,"दे":6364,"धन":912,"नक":3879,"नग":1441,"धर":1283,"नर":642,"नल":477,"नव":1823,"नन":633,"नप":443,"धà¥":2713,"नम":981,"नी":7183,"पं":1090,"नà¥":2801,"ने":15948,"नस":1797,"नह":1786,"ना":16910,"नि":11710,"पक":1243,"नो":2338,"नà¥":16749,"पत":2217,"पन":4849,"पद":2167,"पड":542,"पट":576,"पश":1179,"पह":3589,"पस":581,"पल":544,"पय":630,"पर":12235,"बई":388,"पे":1223,"पै":501,"पू":3275,"पृ":1255,"पा":8076,"पि":1966,"पी":1550,"बं":1558,"पà¥":5232,"फल":545,"फर":388,"पà¥":26390,"पौ":356,"पो":892,"बन":3146,"फे":662,"बद":869,"फि":861,"बड":1139,"फी":943,"फा":687,"बज":3798,"फ़":1678,"फो":368,"फà¥":1074,"बर":2061,"बल":818,"गया":3134,"भग":1053,"बस":1999,"बह":1601,"बि":1756,"बा":6278,"बà¥":789,"बी":1868,"मं":2240,"बे":1048,"बै":523,"बो":1031,"भर":525,"बà¥":3946,"मक":1369,"भव":561,"भी":5128,"यं":862,"भि":1870,"भा":13836,"मत":879,"मण":522,"भू":2019,"भà¥":462,"मन":2122,"मध":940,"मद":711,"भौ":389,"भो":416,"मल":693,"यक":2875,"भà¥":381,"मय":1214,"मर":1260,"मश":384,"मस":725,"मह":3586,"मृ":555,"यत":1327,"मू":1671,"खà¥à¤¯":2528,"यद":785,"मि":6854,"मा":15028,"मà¥":5029,"मी":3565,"रं":1975,"मो":1712,"यम":1479,"मà¥":5925,"मे":33130,"यन":1546,"यप":564,"मै":980,"रख":1463,"यव":775,"रग":385,"रक":5780,"यय":399,"यर":1126,"या":29338,"रज":1027,"यह":10437,"रच":1272,"रद":2279,"रथ":786,"रत":13386,"यू":1784,"रण":4290,"यà¥":3494,"यी":833,"यि":596,"रय":1959,"रम":3670,"रभ":636,"यो":8068,"रब":740,"रप":839,"ये":5448,"रन":4212,"लम":734,"लय":1888,"लब":457,"लन":1624,"लत":1279,"लग":2035,"लक":1497,"रà¥":27657,"रो":5583,"रै":502,"रॉ":358,"रे":18745,"गां":472,"रू":3986,"लं":804,"री":10641,"रà¥":2131,"रि":9664,"रा":30478,"रह":3970,"रस":3428,"रश":605,"रव":1845,"रल":964,"वं":3802,"लà¥":5184,"लो":3980,"चन ":356,"लै":601,"ले":6664,"लà¥":426,"ली":6031,"लि":10617,"ला":9912,"लव":4226,"शब":1373,"वो":877,"वै":1045,"शन":10719,"वे":7461,"वà¥":3139,"शर":623,"वह":1716,"वव":621,"वश":922,"वस":1629,"वृ":595,"शत":728,"वा":18296,"वि":15882,"वी":3437,"वप":710,"वन":1882,"वध":1955,"शक":894,"वल":726,"वर":5872,"वय":393,"वज":391,"वक":455,"वत":1673,"षे":3151,"सन":2988,"सप":658,"सभ":1408,"सब":1580,"सम":9143,"षà¥":6623,"सर":4398,"सल":818,"सव":526,"षा":3622,"सट":463,"षि":1967,"सत":766,"सद":867,"शै":363,"शे":1186,"शà¥":6992,"शो":892,"षय":360,"सक":10544,"शह":1374,"शी":1148,"सं":12512,"शà¥":1050,"शा":4600,"शि":3780,"गवा":390,"षण":1876,"है":50827,"हे":2158,"हà¥":7130,"ही":5060,"हि":5996,"हा":10508,"हà¥":694,"हो":8889,"हन":827,"से":22788,"सà¥":1954,"सी":5308,"हत":2956,"सू":1760,"चल ":435,"सि":5630,"सा":11391,"सह":931,"सस":498,"हव":548,"हस":476,"हल":1812,"हम":833,"सà¥":33294,"हर":3284,"सै":435,"सो":953,"़त":356,"ात":13410,"ाथ":2015,"ाण":2291,"ाठ":433,"िं":3492,"ाड":1607,"ाट":1520,"ाब":2717,"ाभ":451,"ाप":4193,"ाफ":948,"ान":23672,"ाद":7103,"गà¥à¤°":518,"ाध":1672,"ाव":4478,"िख":1296,"िक":17099,"ाल":13682,"ार":38751,"ाय":6583,"िà¤":2749,"ाम":8252,"िज":2534,"ाह":4362,"िच":656,"ास":7258,"ाष":4485,"िग":560,"ाश":1899,"़ी":1936,"ां":6856,"़ा":2260,"़ि":839,"ाà¤":2505,"़ो":448,"ाइ":2283,"ाई":2655,"़े":807,"ाउ":506,"ाओ":2028,"ाक":3759,"़à¥":460,"ाà¤":1076,"ाच":1779,"ाज":7904,"ाग":3365,"ाख":925,"ीड":730,"à¥à¤‚":2936,"ा।":4894,"à¥à¤ˆ":845,"ीत":2135,"à¥à¤†":2021,"ीप":1050,"ीन":3683,"à¥à¤":1050,"ीम":1042,"ीय":7229,"ीब":439,"ील":1370,"à¥à¤“":472,"ीर":2043,"à¥à¤—":719,"à¥à¤–":2469,"ीव":1580,"गीत":642,"à¥à¤•":2798,"à¥à¤œ":614,"à¥à¤›":884,"ीस":1060,"à¥à¤Ÿ":385,"िट":1437,"ीं":3186,"गिक":498,"िण":1209,"गाल":415,"ित":17874,"िद":3772,"िध":2081,"िन":9694,"िप":1804,"िब":505,"िभ":1498,"िम":3142,"िर":6200,"िय":18344,"िल":8308,"ीक":2676,"िश":4603,"िव":3402,"िस":8212,"िष":1899,"ीच":800,"िह":1477,"ीज":592,"ीट":1012,"ेव":2738,"ेश":12536,"ैक":523,"ेल":9698,"ेय":725,"ेर":2606,"ेम":795,"ेब":355,"ेप":664,"ेन":6508,"ैद":560,"ैत":376,"ैज":408,"ेह":634,"ेस":4417,"ेष":1173,"ैर":490,"ैल":1189,"ैन":999,"े।":1956,"ॉन":366,"ैस":1223,"ें":31320,"ेक":2519,"ेख":1744,"गों":941,"ेट":1393,"ेड":787,"ैं":11503,"ेत":4873,"ेद":951,"ेग":371,"ेज":1606,"ृत":2832,"ृष":1881,"गोल":437,"à¥à¤¤":2428,"à¥à¤£":594,"à¥à¤¡":541,"à¥à¤¨":2123,"à¥à¤§":368,"à¥à¤¦":1873,"à¥à¤ª":1093,"à¥à¤°":6463,"à¥à¤®":1318,"गà¥à¤²":427,"à¥à¤²":2104,"à¥à¤·":645,"à¥à¤¸":2362,"à¥à¤µ":1118,"गà¥à¤°":3597,"ूच":523,"ूट":2497,"ूत":791,"ी।":1527,"ून":1187,"ूप":2981,"ूब":457,"ूम":650,"ूर":5314,"ूल":1317,"ूष":1076,"ूस":930,"ूह":483,"à¥à¤µ":16523,"à¥à¤¶":2596,"à¥à¤·":9849,"à¥à¤¸":6797,"à¥à¤¹":2245,"à¥à¤­":823,"à¥à¤®":10658,"à¥à¤¯":26066,"à¥à¤°":57172,"à¥à¤²":4149,"ची ":462,"ौत":437,"ौद":469,"ोर":2026,"ोल":2394,"ोब":410,"ोम":1176,"ोस":741,"ोष":453,"ोह":815,"ोश":547,"ोव":518,"à¥à¤£":2398,"à¥à¤¤":17955,"à¥à¤¡":2149,"à¥à¤Ÿ":15553,"à¥à¤ ":1556,"à¥à¤ž":2460,"à¥à¤«":981,"à¥à¤¬":2118,"à¥à¤ª":6879,"à¥à¤§":4156,"à¥à¤¨":3593,"à¥à¤¥":9797,"à¥à¤¦":7520,"ौर":1279,"à¥à¤œ":1507,"à¥à¤š":2621,"à¥à¤—":2050,"à¥à¤•":4547,"ों":15790,"ै।":21739,"ोई":504,"ॉर":357,"ॉल":485,"ोज":1283,"ोड":4892,"ोट":1429,"ोद":429,"ोत":5502,"ोप":971,"ोध":664,"ोन":2390,"ोक":1863,"ोच":771,"ोग":4303,"००":2062,"१०":362,"१८":536,"१९":2833,"२०":1612,"९६":475,"९५":380,"९८":427,"९७":465,"९९":438,"जन ":1003,"जब ":495,"चती":1825,"चना":962,"गई ":460,"के ":35060,"का ":19876,"कि ":2326,"की ":19870,"कल ":368,"कम ":375,"कर ":3798,"और ":11340,"कित":562,"किन":799,"काफ":651,"काम":601,"कार":8097,"काल":1674,"काश":1174,"काव":390,"कास":610,"किप":365,"किय":4402,"किल":674,"किस":2621,"कीय":409,"कà¥à¤›":880,"कà¥à¤²":589,"कà¥à¤®":405,"कां":514,"कान":492,"कहत":1259,"कहल":373,"कहा":1242,"कला":836,"गी ":381,"कवि":783,"कमà¥":429,"करà¥":878,"गा ":602,"करा":372,"करे":537,"करत":2060,"करण":990,"करन":3111,"कथा":387,"कता":1005,"कते":720,"गर ":1575,"खंड":407,"गत ":745,"खा ":869,"कंप":421,"को ":9463,"ओं ":2427,"à¤à¤µà¤‚":2948,"à¤à¤•à¥":3566,"कई ":776,"ओर ":406,"उसक":842,"उसे":365,"उपन":475,"उपय":550,"à¤à¤ ":417,"à¤à¤• ":11574,"आदि":993,"आधा":542,"आरà¥":522,"आवश":646,"उन ":374,"इति":479,"अथव":383,"अधि":1401,"अनà¥":2084,"अधà¥":641,"अनà¥":1665,"अने":519,"अपन":2305,"इस ":3686,"अभि":793,"अमे":560,"अलग":451,"अरà¥":1665,"अवध":1855,"उनà¥":1250,"उदà¥":651,"उनक":1565,"उतà¥":2559,"उस ":606,"इनक":960,"इनà¥":786,"इसक":6193,"इसम":1168,"इसी":419,"इसे":1083,"इसà¥":411,"ेशà¥":531,"ेशो":391," ओर":436," कं":518," कई":773," और":11423," कन":365," कम":1021," कर":8774," कल":965," कव":757," कह":3125," की":14772," कि":9638," का":20292," कृ":765," कà¥":2268," के":32645," कै":557," गई":551," को":14690,"ेशन":8153," गà¤":382," कà¥":4800,"ेवा":915," à¤à¤•":15549," à¤à¤µ":3020," à¤à¤¸":714,"ै। ":17713,"ोई ":492," चौ":410," जग":453," चà¥":690," ची":475," जं":1512," चि":1251," चा":1105," चे":563," जर":355," जल":669," जब":682," छो":700," जह":416," छू":1830," जन":2746," जà¥":953," जि":5438," जा":9396," जà¥":639," जी":1466," जो":3609," जै":1100," गर":416," गय":3607," खे":428," गण":526," घं":1793," खा":713," गो":922," गà¥":1394," गà¥":1570," गा":1413," । ":3304," चल":837," चर":408," अं":2469," अब":411," अप":3085," अन":4720," अध":2196," अथ":432," आक":690,"ेता":622," अल":1382," अर":2119," अम":1263," अभ":1046," आज":524," अस":670," अव":2716," आत":825,"ेतà¥":3123," इं":1185," अक":778," अग":573," आं":486," अत":780,"ेपा":393,"ेनà¥":1060,"ेना":477,"ेयर":374," इस":13352," इल":428," इन":2514,"ों ":14584," इत":729," आस":419," आर":1183," आम":378," आय":881," आव":1052," आन":491," आद":1214," आध":1052," आप":1294,"ेलव":3726," उस":2203," उप":2582," उन":3214," उद":990," उत":2673,"ेरि":755," वा":3866," वी":554," वि":11396," वृ":367," शत":617," वस":497," वह":1459," वà¥":2079," शर":525," वे":1801," वै":880," शब":1343," शà¥":774," सं":11345," शि":1818," शा":2156," शह":1270," सक":1827," शà¥":1605,"ैसे":672," शक":380," वर":2508," लग":1362," ला":1268," लि":5639," ले":1874," लो":1934," रख":1043," रज":470," या":6370," यह":10383," रच":735," यू":702," यà¥":969," यो":670," ये":2040," रा":8448," रि":478," रह":2139," रे":6398," रू":2374," रà¥":449," लं":500," रो":897,"ोर ":430," हो":7690," हि":3553," ही":2071," हा":1189," हà¥":3748," हे":505," है":50701," सम":6692," सभ":777," सब":1516," सन":1812," सर":2776," सट":433," सद":698," सत":571," हम":409," सà¥":16029," हर":704," सो":571," सा":6452," सि":2848," सह":892," से":17371," सी":1037," सà¥":1695," सू":1297," दर":820," थे":2324," दू":1295," दृ":400," दी":559," दà¥":776," दा":748," दि":4337," दक":983," तà¥":639," तो":1173," था":4674," थी":1702," तम":403," तर":977," तथ":2624," तत":379," ता":833," ति":580,"ोग ":2225," ती":840," तà¥":355," तक":1670," डा":569," डि":508," टà¥":3950,"ोच ":425," मो":707," मे":29038," मै":859," मू":889," यद":611," मा":5157," मि":3653," मी":694," रं":422," मà¥":2959," मह":3455," मर":397," भौ":360," मन":1109," मध":857," भू":1742," भी":4132," भा":10893," भर":368," बà¥":1174," बे":646," बै":464," बो":710," बा":3995," बि":1259," बी":1196," मं":1754," बà¥":605," भग":427," बह":1547," बस":477," फà¥":403," बर":576," बल":422," बद":685," बन":2421," फे":481," फा":416," बज":3787," फ़":822," फि":728," बड":1101," पà¥":19145," पो":399," पि":711," पा":4190," पà¥":2921," पी":496," बं":729," पे":601," पै":416," पृ":1225," पू":2103,"ोड ":4034," पर":10662," पश":1088," पह":3539," पड":455," पट":420," पद":1827," पत":1117," नà¥":487," नह":1717," नि":5215," ना":4741," पं":1035," नी":536," ने":3413," धà¥":387," नव":592," धा":800," नद":808," दे":3244," दà¥":5503," दो":1318," धर":1010," नग":840,"ेंट":533,"ेंद":417," ई ":361,"ेल ":4266,"ेर ":787,"ेस ":3757,"ेष ":525,"ेश ":2529,"े। ":1586,"ेकि":361,"ेकà¥":824,"ैंड":368,"ैं।":6253,"ेज़":466,"ेजी":411," व ":823,"ृषà¥":1735,"ें ":29625,"ृति":1045,"ृतà¥":636,"ेन ":4101,"ेद ":371,"ेज ":388,"ेट ":613,"ैं ":4429,"ेक ":779,"à¥à¤·à¥":405,"à¥à¤µà¤¾":512,"à¥à¤²à¤¾":407,"à¥à¤°à¤¾":1009,"à¥à¤°à¤¸":482,"à¥à¤°à¥":818,"à¥à¤°à¥":791,"à¥à¤°à¥‚":359,"à¥à¤®à¤¾":600,"à¥à¤¨à¤¿":743,"à¥à¤¨à¤¾":530,"à¥à¤¦à¥":1350,"à¥à¤¤à¥":627,"ूसर":611,"ूषण":1023,"ूरà¥":2859,"ूरी":482,"ूटत":1807,"à¥à¤¸à¥":700,"à¥à¤¸à¤¾":1149,"िकल":459,"ाला":1369,"ालि":2288,"ाली":1852,"ाले":1192,"ावि":362,"ावा":595,"à¥à¤¤ ":1429,"िका":3179,"िकि":844,"िकी":876,"ावर":399,"िको":367,"िकà¥":1744,"ाषा":2056,"ासन":527,"ाषà¥":2240,"ाशि":573,"ासक":514,"ाहि":1577,"ासि":587,"ासा":469,"ासी":381,"ाहर":429,"ासà¥":1433,"िजà¥":1407,"à¥à¤° ":2020,"िता":1136,"िति":559,"à¥à¤² ":595,"ितà¥":3274,"िदà¥":3069,"िधि":924,"िधा":871,"िनट":1785,"िना":538,"िनि":795,"िनà¥":3490,"िपी":396,"िभि":546,"िभा":779,"ियन":355,"ियम":677,"िमी":551,"िमा":800,"ियो":3134,"िये":2350,"िया":10338,"िरà¥":2740,"िरा":423,"िलत":401,"िले":1007,"िलà¥":2138,"िलो":516,"िला":1775,"िसम":787,"िषà¥":1088,"ी। ":1157,"िशा":959,"िशà¥":1982,"िशे":677,"िसक":1052,"िवा":1432,"िवे":381,"ीका":588,"िवर":417,"ूप ":2670,"िहा":1270,"ून ":537,"िसे":557,"िसी":1508,"िसà¥":2756,"ीडि":468,"à¥à¤‚ब":372,"à¥à¤‚च":1928,"ीटर":673,"ूल ":705,"ीति":750,"ूर ":759,"à¥à¤†à¥¤":435,"ूह ":393,"à¥à¤“ं":451,"ीरà¥":408,"ीमा":484,"à¥à¤•à¥":1845,"ीवन":532,"à¥à¤–à¥":1068,"ृत ":890,"ीं ":2774,"ित ":12024,"िण ":703,"िन ":1347,"िल ":1061,"ीक ":803,"ांग":686,"ांस":610,"िम ":1068,"ांत":1630,"िय ":463,"िर ":1319,"ीच ":562,"ाà¤à¤":397,"िश ":406,"िस ":650,"ा। ":4040,"à¥à¤† ":1392,"ीत ":751,"ागर":849,"ाकà¥":373,"ाकि":1058,"ाका":641,"ाकर":472,"ाओं":1931,"िंद":1176,"ाड़":911,"िंह":577,"à¥à¤ ":894,"िंग":982,"ाटक":402,"ाजा":802,"ाजि":402,"ाजà¥":2259,"ीप ":552,"ाजन":843,"ाजध":594,"ाजस":382,"ाची":795,"ीन ":2404,"à¥à¤ˆ ":687,"ाने":2242,"ाना":1838,"ानि":2065,"ानी":2791,"à¥à¤– ":1161,"ानव":590,"ानस":622,"ाधà¥":419,"ापन":842,"ानà¥":1734,"ानो":532,"ादी":544,"ादा":388,"ादि":1157,"ानत":633,"ाधि":410,"ानक":524,"ाति":524,"ाता":4840,"ाती":1316,"ाते":918,"ातà¥":3549,"ील ":655,"ीय ":6964,"ीर ":855,"ारी":1527,"ारि":1016,"ारा":6234,"ारस":394,"ारà¥":5244,"ारो":922,"ारू":465,"ारे":707,"ालय":1677,"ामि":517,"ामा":1032,"ायक":414,"ामà¥":377,"ायन":449,"ामी":544,"ाया":1613,"ायी":366,"ारक":420,"ारण":1386,"ारत":7763,"ाबा":770,"ाबà¥":555,"ामक":498,"à¥à¤› ":876,"ापा":517,"ापà¥":901,"ाफी":651,"ां ":1658,"़ी ":1669,"ाई ":2362,"़े ":628,"हों":935,"है।":21737,"होत":4643,"होन":1130,"हà¥à¤®":356,"़ा ":1290,"ाठ":1565,"ाग ":1189,"ाथ ":1651,"ाद ":3112,"ाण ":964,"ात ":1397,"ान ":8758,"ाज ":1013,"ाश ":423,"ाव ":1068,"िक ":8100,"ाह ":929,"ास ":2767,"ाम ":3709,"िठ":2584,"ाब ":747,"ाल ":3360,"ार ":11273,"ाय ":1113,"समà¥":515,"समा":1211,"समू":429,"सरक":1452,"समे":1977,"समà¥":2440,"सरà¥":919,"सबस":1305,"समय":1034,"षà¥à¤¯":501,"षà¥à¤£":694,"षà¥à¤ ":1362,"सभी":533,"सभा":753,"षà¥à¤Ÿ":3181,"षेत":2985,"सनà¥":672,"हतà¥":826,"हते":1441,"सेन":499,"सें":450,"सीम":423,"सिर":645,"सूर":646,"सूच":437,"सां":361,"साह":1353,"सिद":1352,"सित":526,"साध":457,"सार":1977,"साम":1236,"साय":536,"सिक":706,"साग":441,"सिं":774,"साथ":1234,"सहा":390,"ससे":419," १८":501," १९":2789," २०":1512,"हला":582,"हले":581,"हरा":369,"सà¥à¤µ":2405,"सà¥à¤¯":682,"सà¥à¤®":485,"सà¥à¤²":386,"सà¥à¤¥":6865,"सà¥à¤ª":4184,"सà¥à¤Ÿ":9331,"सà¥à¤¤":5132,"सà¥à¤•":2303,"सेव":720,"हैं":10612,"हें":816,"हà¥à¤¤":957,"हà¥à¤ˆ":797,"हà¥à¤":951,"हà¥à¤‚":1933,"हà¥à¤†":1762,"हिन":1798,"हित":1670,"हीं":1893,"हास":1027,"हाव":428,"हार":1703,"हान":874,"हिं":1060,"हाà¤":938,"हां":1185,"हे ":650,"है ":18074,"सकी":3365,"सका":2827,"सके":2127,"सतà¥":357,"षिण":921,"सटी":407,"षित":444,"हो ":1330,"शेष":769,"शों":440,"हा ":1528,"सकत":1521,"ही ":2790,"शà¥à¤µ":2006,"शà¥à¤°":1917,"शà¥à¤¯":1102,"शà¥à¤š":1235,"शहर":1243,"से ":20466,"सी ":3799,"हर ":1676,"संच":2031,"शिय":571,"संक":898,"संग":1492,"संख":1034,"शिव":510,"संघ":401,"संब":719,"संप":1355,"संय":444,"संस":2227,"शित":453,"शाह":383,"शास":1407,"शिक":1243,"शाल":480,"वेश":377,"वेद":482,"शता":469,"सर ":563,"सा ":1620,"वà¥à¤¯":2741,"शबà¥":1328,"वों":401,"वरà¥":3433,"षा ":2805,"वरी":565,"ववि":516,"शकà¥":405,"वशà¥":752,"वसà¥":1030,"सन ":1657,"वाद":1110,"वान":862,"वाच":378,"विक":1766,"वाल":2753,"वास":1153,"वार":6241,"वाय":555,"वाम":390,"वित":981,"विद":1842,"विध":1382,"विज":1255,"वाह":694,"विच":389,"वीं":679,"विष":806,"विश":2580,"विस":375,"विव":625,"विभ":1112,"वीप":490,"वता":365,"वधि":1770,"वपू":515,"शा ":783,"षण ":1678,"शी ":479,"वंश":456,"शन ":10109,"वे ":5027,"वि ":440,"वा ":1780,"वी ":1441,"लà¥à¤ª":398,"लà¥à¤®":761,"लà¥à¤¯":463,"लà¥à¤²":1775,"लोक":1046,"लोग":827,"लों":837,"वह ":1195,"लोम":397,"वल ":387,"लेख":932,"लेक":881,"लिय":3022,"वर ":730,"लाक":373,"लात":527,"लिà¤":2317,"लाल":465,"लिख":696,"लिक":529,"लित":2228,"लिप":379,"वन ":938,"लवे":3731,"लना":388,"लता":462,"लती":425,"वं ":2957,"लगभ":597,"लगा":459,"रेस":3665,"रोग":432,"रों":2025,"रà¥à¤¶":1019,"रà¥à¤µ":3266,"रà¥à¤¸":388,"रà¥à¤·":1579,"रà¥à¤®":3443,"रà¥à¤­":362,"रà¥à¤¯":2627,"रà¥à¤¥":2552,"रà¥à¤¦":1098,"रà¥à¤§":360,"रà¥à¤¨":513,"रà¥à¤«":531,"रà¥à¤¬":360,"रà¥à¤Ÿ":763,"रà¥à¤¡":507,"रà¥à¤£":1686,"रà¥à¤¤":1633,"रà¥à¤—":1726,"रà¥à¤•":924,"रà¥à¤œ":853,"रà¥à¤š":479,"रीक":801,"रिव":843,"रिय":2208,"रीय":1696,"रीर":445,"रà¥à¤ª":452,"रूप":2871,"रें":640,"रेज":901,"रेल":6023,"रेन":3698,"रसा":555,"रसि":1094,"रहत":499,"रहा":649,"रहे":489,"रसà¥":963,"ले ":3374,"रां":1067,"रान":1776,"राप":737,"रात":590,"राण":691,"राज":5799,"राच":861,"राक":432,"रिट":356,"रित":976,"राष":2127,"रास":461,"राम":1378,"राय":1077,"रार":802,"रिक":2624,"राव":460,"ला ":4369,"रयो":985,"रयà¥":635,"रमà¥":388,"रमा":570,"रमà¥":1096,"रवा":666,"ली ":4841,"रने":2577,"रना":959,"रदे":1187,"रदा":576,"रभा":427,"योज":542,"योग":2669,"यों":3685,"या।":1162,"यà¥à¤•":1333,"यà¥à¤¤":419,"यà¥à¤¦":407,"यान":790,"याप":561,"यात":2560,"याद":888,"यास":655,"यिक":384,"याल":1423,"यार":471,"याय":535,"रता":1516,"रति":1618,"रती":4166,"रते":811,"रतà¥":447,"रथम":512,"लय ":1619,"यूट":437,"यून":379,"रणा":516,"रचन":643,"रकà¥":994,"रखे":433,"रजि":411,"यां":725,"याà¤":525,"यहा":1528,"लन ":775,"रे ":1761,"मसà¥":459,"महा":2097,"महत":756,"यकà¥":1167,"रॠ":369,"यका":377,"री ":6215,"मृत":432,"मूह":442,"मूल":727,"मà¥à¤¦":734,"मà¥à¤–":2339,"रंभ":445,"मà¥à¤‚":391,"मीट":638,"रंग":680,"मिल":1800,"मित":732,"मिन":2223,"मार":2023,"माल":612,"मिक":732,"रो ":378,"माण":1011,"माध":371,"मात":697,"मान":5551,"माज":766,"मां":436,"मों":431,"लग ":468,"यदि":507,"में":27306,"मेर":842,"मेल":1950,"यता":597,"रका":3522,"मà¥à¤°":418,"मà¥à¤ª":956,"मà¥à¤¬":1357,"मà¥à¤®":1860,"रत ":4510,"रण ":3158,"या ":17033,"यॠ":436,"यी ":716,"भिन":962,"भाव":792,"भाष":2240,"भार":7799,"भाग":1261,"रम ":841,"यंत":450,"ये ":4867,"मधà¥":813,"भूम":411,"भूष":1010,"रल ":672,"रा ":9045,"मरà¥":364,"रह ":1087,"मा ":1358,"भगव":386,"मी ":1758,"मे ":1785,"यन ":960,"बहà¥":1203,"बसे":1366,"बाल":449,"बार":924,"बाद":1868,"बां":380,"यर ":634,"मंत":477,"मंद":933,"बिह":404,"बीच":558,"मंड":367,"यम ":869,"रक ":417,"यह ":8676,"बोल":432,"बà¥à¤°":1124,"बà¥à¤¦":1896,"पà¥à¤¯":460,"पà¥à¤°":23891,"पà¥à¤¤":1355,"भा ":824,"भी ":4811,"मन ":544,"फ़ि":394,"बजे":3647,"फिल":452,"मय ":1094,"बड़":987,"बदल":617,"यक ":1025,"फेर":414,"बना":1428,"बनà¥":526,"फà¥à¤°":464,"पहà¥":2050,"पहल":1024,"पशà¥":999,"बी ":558,"बा ":412,"परà¥":1203,"परम":473,"पयो":528,"परि":1837,"परा":615,"मक ":839,"पृष":917,"पà¥à¤¸":378,"पूर":2834,"पà¥à¤¤":428,"पà¥à¤°":3647,"पीड":441,"बंग":529,"बंध":817,"पास":574,"पित":787,"पाक":1033,"पान":519,"पात":387,"पाद":1538,"पार":1007,"पाल":950,"पकà¥":435,"फी ":766,"नà¥à¤¤":2330,"नà¥à¤§":640,"नà¥à¤¨":1957,"नà¥à¤¦":3701,"नà¥à¤¯":2792,"नà¥à¤®":1190,"नà¥à¤¹":2015,"नों":1406,"पदà¥":1215,"पदा":458,"भग ":605,"पनी":874,"पना":1055,"पनà¥":743,"पने":1546,"बर ":1142,"पतà¥":1249,"पति":612,"पड़":437,"नमे":432,"नवर":402,"नसं":440,"नदी":699,"धà¥à¤¯":2164,"नà¥à¤¸":1344,"पंज":567,"नीत":724,"नेप":393,"नेत":472,"नेक":552,"बई ":387,"नही":1669,"नसभ":406,"निव":567,"निर":2352,"निय":1801,"निध":617,"नित":1221,"नाव":392,"निक":2637,"नाय":738,"नाम":2777,"नार":680,"नान":715,"नात":423,"नाथ":458,"नाट":450,"नाड":359,"नाग":557,"दà¥à¤¦":569,"दà¥à¤µ":5706,"दà¥à¤°":2473,"दà¥à¤¯":2403,"दà¥à¤®":1046,"दà¥à¤§":2842,"धरà¥":969,"नका":1135,"नकी":794,"नके":1054,"देख":471,"देव":1125,"देश":3481,"दों":393,"दोन":372,"धित":713,"धिय":418,"धार":1754,"धिक":1763,"धान":1811,"पर ":7035,"नता":441,"नते":555,"नगर":1205},"n_words":[3436892,4107546,2722787],"name":"hi","type":"devanagari"} \ No newline at end of file
diff --git a/contrib/languages-data/hr.json b/contrib/languages-data/hr.json
new file mode 100644
index 0000000..c42497a
--- /dev/null
+++ b/contrib/languages-data/hr.json
@@ -0,0 +1 @@
+{"freq":{"D":8690,"E":5169,"F":5109,"G":8706,"A":14720,"B":12556,"C":10193,"L":6822,"M":14153,"N":10511,"O":10729,"H":9113,"I":13451,"J":5359,"K":11989,"U":4743,"T":10711,"W":3179,"V":5850,"Q":585,"P":15109,"S":23054,"R":9319,"Y":1450,"X":701,"Z":4251,"f":16193,"g":88687,"d":148916,"e":436858,"b":60942,"c":64868,"a":586818,"n":342270,"o":415815,"l":177559,"m":139424,"j":258985,"k":204303,"h":49394,"i":501698,"w":7347,"v":140168,"u":209613,"t":201984,"s":238514,"r":271701,"q":1755,"p":126499,"z":85944,"y":9646,"x":2022,"í":886,"é":1412,"á":1674,"ó":546,"Ä‘":8171,"ć":16773,"ÄŒ":895,"Ä":46294,"Ž":887,"ž":23555,"Å ":1432,"Å¡":26211," l":9617," m":20259," n":49295," o":45630," h":4791," i":61249," j":56434," k":46966," d":31478," e":6507," f":5286," g":21986,"Ñ€":912," a":19142,"Ñ":742," b":17477,"Ñ‚":629," c":5752," z":19710," u":45274," t":21468," w":820," v":15539," p":67380," s":79773," r":20001," J":5152," K":11465," H":8434," I":11848," N":9837," O":7822," L":5986," M":13514," B":11861," C":9435," A":13018," F":4631," G":8304," D":7573," E":4598,"л":648," Z":4104,"к":817," Y":1407," X":526,"и":1209,"о":1253,"н":943," S":19621," R":8667,"в":563," Q":537," P":14387,"а":1816," W":3048," V":5326," U":4477,"е":917," T":9972," Ä":5596," ÄŒ":887," ž":5028," Ž":838," Å ":1403," Å¡":4901,"A ":857,"Da":1321,"Co":1834,"Cr":1328,"Ce":588,"Ch":1843,"Du":764,"Do":1328,"Dr":1133,"De":1138,"Di":1002,"Fe":631,"H ":574,"Fa":563,"Eu":879,"Ge":716,"Ga":972,"I ":821,"Fr":1113,"Fo":779,"Fi":649,"Au":808,"Ar":1684,"At":729,"As":560,"D ":829,"Ba":2710,"Af":545,"Am":1341,"An":1467,"Al":1655,"Bu":1011,"Br":2042,"Ca":2160,"Bi":1321,"Be":1685,"Bo":2115,"Ku":844,"Kr":1748,"Ko":2488,"Le":881,"Li":1283,"La":1685,"Lu":593,"Lo":1036,"Me":1818,"Mi":1887,"O ":2633,"Ma":5853,"Mu":793,"Mo":1972,"Nj":820,"Ni":1096,"Ne":1404,"Na":3401,"No":2003,"Ob":956,"Gl":526,"Gr":2253,"Go":1536,"Gu":881,"Gv":612,"Ha":1340,"He":1374,"Hi":570,"Ho":905,"IS":2154,"Hr":2959,"Hu":536,"In":4619,"Is":978,"Ja":1450,"L ":681,"Iz":610,"Je":1026,"Jo":711,"Ju":1563,"Ka":3681,"Ki":952,"Tu":1072,"Tr":1314,"To":1609,"Th":1212,"Ti":993,"Te":1282,"Ta":1685,"St":1916,"Sv":940,"Su":1303,"Wi":722,"Wa":1107,"Vo":662,"Vi":1040,"Va":1022,"Ve":1522,"Pu":746,"Pr":3018,"S ":720,"Pe":1604,"Pa":2898,"Pl":816,"Po":3639,"Pi":867,"Os":790,"Ov":672,"Op":1011,"Or":735,"Se":1283,"Sj":776,"Si":1385,"Sh":659,"Sl":1203,"Sr":1393,"Sp":846,"So":1277,"Ru":1026,"U ":1401,"Sa":3386,"Re":2977,"Ri":1491,"Ro":1097,"SO":2074,"Ra":1418,"b ":1522,"a ":202747,"Za":2365,"Ze":593,"i ":135497,"aÄ‘":2045,"gd":857,"ge":5429,"ga":10157,"ać":1902,"fi":4106,"fs":661,"fr":2013,"aÄ":10342,"fu":639,"fo":2404,"j ":16449,"he":4881,"ha":5859,"gn":997,"gl":6895,"gi":6788,"gh":714,"gu":4729,"gr":14473,"go":15835,"du":5516,"dv":2741,"g ":19299,"ea":3988,"eb":4045,"ec":3773,"ed":22606,"de":10224,"dg":624,"di":30316,"dj":3005,"dm":1005,"dl":806,"do":12624,"dn":15638,"ds":4759,"dr":13255,"ew":844,"eu":1755,"ev":10791,"ey":838,"ez":15435,"fa":1476,"h ":20168,"fe":2011,"eh":1279,"eg":7983,"ef":1181,"ee":1826,"el":24518,"ek":14461,"ej":2546,"ei":1534,"ep":5699,"eo":3348,"en":44674,"em":19226,"et":18394,"es":16599,"er":32266,"ca":14154,"e ":160018,"br":6048,"bu":5481,"bn":1412,"bo":6809,"bj":1695,"bl":6540,"bi":13442,"be":6031,"db":626,"da":22107,"f ":1430,"cu":2623,"ct":683,"cr":1908,"co":2679,"ck":2130,"ci":21232,"ch":4164,"ce":10558,"c ":2983,"az":15697,"ay":1948,"ba":10149,"d ":22851,"at":29447,"as":22343,"ar":30709,"aw":1498,"av":28227,"au":3869,"ak":15591,"al":35013,"ai":3078,"aj":16895,"ao":6448,"ap":10100,"am":17849,"an":70440,"ac":13980,"ad":21936,"aa":799,"ab":3879,"ag":6276,"ah":2814,"ae":1668,"af":2077,"nu":8144,"nt":10450,"ns":14593,"iÄ":14290,"no":45797,"nn":1399,"nz":952,"ny":715,"oe":678,"of":2150,"oc":4342,"od":40790,"oa":2451,"ob":12953,"om":28810,"on":23346,"ok":14795,"ol":17826,"oi":3961,"oj":40426,"og":22833,"oh":1159,"ot":12114,"os":23349,"ov":31034,"ou":2922,"op":11191,"oo":1568,"or":33494,"jÄ":526,"r ":11028,"ow":1054,"oz":7848,"pe":6182,"pa":15403,"pl":8816,"pn":1287,"po":36246,"ph":606,"pi":11043,"pj":1476,"lo":18651,"ln":5989,"lm":1222,"ll":3111,"ls":2651,"lu":8623,"lt":2358,"o ":61519,"ma":31717,"mb":2862,"dž":894,"me":24452,"ml":1963,"eÅ¡":3371,"mi":10591,"mj":5104,"mn":1665,"mp":2261,"mo":11753,"mr":1000,"ms":2000,"mu":3945,"p ":2149,"na":87240,"nc":5972,"nd":11672,"ne":33576,"nf":891,"ež":1706,"ng":7899,"ni":54450,"nj":27833,"nk":2801,"ić":2802,"jv":1605,"eÄ‘":3970,"ju":19590,"eÄ":2671,"js":8691,"jn":2724,"jo":4699,"eć":4631,"jk":855,"ki":28341,"ke":16969,"kc":1026,"ka":37663,"m ":37899,"ks":2734,"kt":4893,"ku":14523,"kv":1615,"ko":63668,"kr":10108,"kl":4333,"km":1244,"kn":1083,"li":41243,"lk":741,"lj":24378,"le":17764,"ld":889,"lg":643,"la":37196,"lb":2414,"n ":24345,"hr":2448,"hv":1089,"ht":807,"hu":1937,"hi":4652,"hn":862,"ho":4536,"id":4699,"ic":20781,"ib":2814,"ia":4852,"ih":18991,"ig":5433,"if":1923,"ie":2060,"k ":14010,"ir":12272,"is":21664,"it":16417,"iu":848,"iv":14019,"aÅ¡":3959,"ij":57037,"ik":23850,"il":22797,"im":27723,"in":47299,"io":10059,"ip":4978,"je":134763,"až":1877,"ji":26676,"iz":19026,"l ":6966,"ja":39549,"z ":7997,"ož":2534,"oÅ¡":2893,"wi":861,"rÄ":1709,"y ":3377,"wa":2878,"we":956,"vl":4881,"vj":3877,"vk":562,"vi":24026,"vu":3419,"vr":6622,"vs":1342,"vn":9959,"vo":21455,"uz":3112,"uv":1349,"ve":22017,"vc":593,"va":35406,"x ":785,"ui":1893,"uj":4079,"uk":4559,"ul":5933,"pć":2758,"ue":1973,"ug":8425,"uh":1544,"ur":9708,"us":10059,"ut":7404,"um":7672,"un":8708,"uo":923,"up":11863,"tu":9961,"tt":1393,"tv":7233,"oÄ‘":1449,"ub":5211,"ua":3882,"ud":5733,"uc":1692,"w ":858,"to":28843,"tn":5773,"tm":575,"tl":1860,"oć":686,"ts":8790,"tr":16932,"oÄ":4671,"tp":896,"te":22658,"tk":2453,"tj":1700,"ti":34096,"th":3097,"v ":4865,"ta":38983,"su":13292,"sv":6451,"ss":1881,"st":55588,"sl":7897,"sk":52271,"sn":7159,"sm":3278,"sp":5046,"so":5247,"sr":3449,"sc":1522,"se":24429,"sh":2381,"sj":4697,"si":10381,"rz":1007,"u ":87089,"sa":16094,"rr":1162,"rs":8936,"rt":5687,"ru":17434,"rv":8435,"ry":930,"rp":1334,"ro":34834,"rn":10194,"rm":2849,"rl":1655,"rk":2924,"rj":894,"ri":44000,"rh":1312,"rg":3269,"iž":1025,"re":32485,"rd":3070,"rc":1885,"rb":2322,"ra":64303,"t ":14252,"qu":1390,"iÅ¡":6125,"s ":15910,"pt":1447,"pu":6628,"pp":658,"pr":27731,"ps":2712,"už":5556,"uÅ¡":2233,"rž":4670,"zg":1072,"rÅ¡":1722,"zi":22879,"zb":2089,"zd":2018,"ze":4698,"za":19270,"zv":4353,"uÄ":5083,"zr":2246,"zu":2322,"zo":3146,"zn":8371,"zm":2468,"zl":2200,"uć":2114,"ye":523,"ya":1738,"yo":520,"á ":773,"é ":601,"ć ":1732,"ći":5749,"ću":1057,"ća":4248,"će":3354,"Äi":7487,"Äj":2822,"Äk":13325,"Äl":554,"Äe":5735,"Äa":5202,"Ä ":1473,"Ä‘e":2942,"Ä‘a":1580,"Än":7182,"Äu":1818,"Ä‘u":2924,"Å¡ ":1063,"šć":1355,"Å¡e":2480,"Å¡a":1860,"Å¡p":635,"Å¡n":3132,"Å¡k":2964,"Å¡l":591,"Å¡i":3525,"Å¡u":567,"Å¡t":7272,"žu":2110,"žn":3428,"žb":852,"že":4351,"ža":6201,"ži":5092,"Ä‘en":1909,"Ä‘er":597,"Ä‘u ":1688,"Ä‘un":708,"Äun":831," Ga":961," Ge":702," Fo":742," Fr":1109," Fi":643," Ha":1336," He":1372," Go":1534," Gr":2247," Gu":878," Gv":612," Gl":525," Hu":528," IS":2110," Hr":2946," Ho":895," Hi":565," Je":1023," Ja":1447," Iz":607," Is":977," In":4605," Ka":3666," Ki":914," Jo":707," Ju":1556," La":1670," Le":867," Li":1265," Ko":2481," Kr":1745," Ku":843," Ma":5817," Mi":1882," Me":1804," Lo":1032," Lu":587," Ne":1391,"а ":562," Na":3399," Nj":820," Ni":1084," Mo":1968," Mu":787," Am":1333," An":1450," Al":1637," Af":533," Ba":2689," Au":794," At":724," As":558," Ar":1651," Be":1676," Bi":1296," Bo":2110," Br":2033," Bu":1004," Ca":2140," Ce":585," Ch":1828," Cr":1326," Co":1819," Da":1296," Di":998," De":1131," Dr":1125," Do":1300," Du":759," Eu":878," Fe":623," Fa":554," Wi":714," Wa":1098," Ze":593," Za":2362," a ":4274," Ov":665," Os":789," Or":732," Op":1008," Po":3623," Pl":812," Pi":864," Pe":1599," Pa":2881," No":2002," Ob":956," Ra":1412," Ro":1089," Re":2970," Ri":1491," Pr":3007," Pu":743," Sv":937," Su":1301," St":1875," Ta":1677," Th":1189," Ti":990," Te":1277," Tr":1295," To":1604," Ru":1022," Sa":3371," U ":1282," Sh":655," Si":1373," Sj":776," Se":1273," So":1270," Sp":836," Sr":1393," Sl":1201," Va":1018," Ve":1517," Vi":1035," Vo":661," Tu":1066," ja":1334," iz":11704," je":51007," im":3911," in":3359," il":7148," is":4046," ka":7800," ki":1128," jo":810," ju":3163," ha":604," he":608," gl":3276," gr":8161," go":6544," ih":786," ig":1120," hi":648," ho":696," hr":1744," nj":3220," ni":1874," ne":6451," na":34503," mu":1067," mo":4787," mn":528," ok":4011," on":1068," od":15714," of":856," ob":6528," no":2407," le":827," lj":2291," li":2543," la":2172," ku":1972," kn":829," km":1048," kl":1410," kr":4664," ko":26854," me":3676," mi":2188," mj":1905," o ":1165," ma":4891," lo":1177," am":2645," an":1498," ak":848," al":3416," au":1442," ar":1418," at":615," ba":2490," bi":5965," be":1110," bo":2421," bl":1053," bu":691," br":3222," ca":552," et":559," en":2154," el":1166," ek":584," fa":621," fr":941," fo":883," fi":1669," ge":1007," gd":551," ga":1134," i ":27993," cr":1408," ce":1531," ci":1052," da":4574," do":6878," dr":6375," de":2927," dj":1233," di":5389," dv":2035," du":1555," zn":2259," zr":767," zv":702," za":12881," ze":1108," zb":633," už":620," ru":1327," u ":32404," sa":9931," se":16886," sj":3429," si":2544," sn":698," sm":1890," sl":3915," sk":4532," sr":2957," sp":2436," so":944," ra":7698," re":4140," ri":4141," ro":2280," pu":1716," pr":23703," s ":3500," os":3950," ot":2810," ov":1276," op":3149," or":1765," oz":1224," pe":1664," pa":2832," pl":5604," po":28804," pi":966," pj":1218," va":1250," ve":3149," uz":1739," vo":2912," vr":3315," vi":2766," vj":724," vl":1147," ud":648," tv":902," tu":1007," us":1218," ut":721," ur":562," up":1243," um":849," un":999," uk":1055," ul":803," ug":653," ta":2808," st":9162," sv":5803," su":10580," tr":4273," to":2817," th":929," ti":2177," te":5426," Äe":1938," Äl":530," Äi":1960," Å¡t":1607," Å¡i":1148," Å¡p":620," ži":2289," že":880," žu":1586,"Eur":771,"Fra":831,"Her":776,"Gra":1106,"Gor":695,"Ind":3698,"Hrv":2864,"ISO":2034,"Ara":578,"šće":547,"Ame":663,"Car":617,"Bra":541,"Bri":609,"Bos":708,"Chi":621,"Cha":594,"Nal":560,"Nje":555,"Nov":1094,"Opć":766,"Per":568,"Par":726,"Pro":696,"Pri":721,"Pre":828,"Juž":532,"Kal":564,"Kan":524,"Kar":752,"Kra":872,"Man":821,"Mal":573,"Mar":1189,"Mad":576,"Zag":834,"Sta":754,"Sje":767,"Srb":562,"Sre":533,"Slo":726,"Rus":582,"San":791,"Rep":1934,"SO ":2049,"Vel":771,"The":928,"Å¡en":553,"Å¡e ":976,"Å¡to":1559,"Å¡te":2571,"Å¡ti":879,"Å¡ta":884,"Å¡tv":1041,"Å¡ko":1181,"Å¡nj":2421,"Å¡in":692,"Å¡ir":1400,"Å¡ki":826,"bje":654,"bja":1027,"bit":2079,"biv":585,"bio":1738,"bil":3046,"bin":696,"bij":1580,"blj":1003,"bli":4296,"bla":688,"bol":1257,"boj":638,"bog":903,"biÄ":654,"bno":610,"bor":1232,"be ":1311,"ban":1537,"bal":1792,"bav":860,"bar":843,"bi ":1256,"ber":918,"ben":1811,"ca ":10450,"car":656,"can":1075,"ce ":5505,"bri":882,"bro":2066,"bra":1828,"bu ":732,"bum":1939,"buh":796,"aka":2848,"am ":2209,"ake":1450,"aki":683,"aji":1629,"ajn":1118,"ajs":690,"aju":3559,"ajv":1325,"al ":2419,"aja":1758,"aje":2556,"ain":623,"ak ":2449,"aj ":1426,"aha":673,"agr":1324,"agu":525,"ago":1080,"anu":1384,"ano":4034,"ant":2834,"ans":7065,"ane":2342,"ang":1485,"ani":9755,"anj":10858,"ank":1213,"ana":11419,"anc":2218,"and":3683,"amo":1853,"ami":1538,"ame":3781,"amb":801,"ama":5199,"ao ":5760,"alu":766,"alt":538,"als":661,"alo":1706,"aln":4903,"all":806,"ali":7309,"alj":2503,"ale":2883,"ala":6700,"alb":1948,"an ":10347,"aku":618,"akt":1211,"ako":4291,"aba":762,"abi":791,"ae ":719,"aca":3812,"ad ":3345,"ac ":1484,"afs":539,"afi":554,"ai ":709,"aga":937,"age":778,"ado":1169,"adr":1028,"adn":3155,"adi":2550,"ade":1153,"ads":935,"adu":1422,"aci":6004,"ach":727,"ada":5689,"azn":685,"azi":7453,"azl":1063,"azv":1013,"azu":592,"aze":804,"aza":1040,"azb":1010,"aya":527,"ba ":2218,"at ":2601,"arh":564,"are":1147,"ard":1409,"ara":6301,"aro":3575,"arn":1712,"ark":1048,"ari":4189,"aru":579,"ars":3100,"art":1393,"asa":826,"asi":1226,"ash":669,"ase":3354,"asn":1043,"asp":540,"ask":1064,"asl":534,"ar ":2545,"apa":4410,"api":1243,"apo":975,"apr":526,"aps":650,"apu":900,"as ":2262,"ava":8033,"aut":1108,"avo":1516,"avn":4358,"avl":3013,"avi":4083,"ave":3087,"ay ":531,"awa":719,"avu":1466,"av ":1100,"ata":3446,"ast":9058,"atn":1424,"atk":562,"atr":1028,"ato":2245,"ate":2791,"ati":6984,"ats":4995,"atu":1171,"aus":601,"jeg":2764,"jed":10627,"jec":1103,"jer":2418,"jek":4521,"jel":6299,"jem":5086,"jen":9322,"jez":10141,"jes":4061,"jet":3631,"jev":7451,"jaÄ":750,"ji ":16045,"ažn":588,"jat":632,"jav":2585,"jal":1858,"jak":822,"jan":5965,"jam":1147,"je ":61320,"jeÅ¡":1621,"jni":1204,"joj":1250,"jom":1082,"jiv":803,"jim":3670,"jin":1845,"jih":2364,"jeÄ":1963,"jeć":1172,"eća":1346,"eće":940,"eći":1617,"ito":2406,"itu":706,"its":544,"isk":1391,"isl":604,"iso":539,"isn":807,"isp":569,"isu":779,"ist":10340,"iv ":1925,"ita":3250,"ite":2457,"iti":4001,"ivo":1642,"ivn":1941,"iva":3738,"ivi":1447,"ive":1512,"is ":1700,"ion":3221,"ipa":1930,"ir ":678,"iro":1457,"iri":1540,"isi":976,"ish":564,"ise":556,"isc":574,"isa":1555,"ire":1683,"ira":5142,"it ":890,"ja ":23119,"iz ":4867,"izu":877,"izv":2232,"izr":793,"izo":774,"izn":759,"izm":2019,"izl":668,"izi":2042,"izd":806,"iza":2198,"kih":5870,"kim":2628,"kin":753,"km ":900,"ki ":16671,"ked":578,"ke ":14258,"kci":941,"kra":4133,"kre":1063,"ku ":4982,"kro":1000,"kru":1711,"kri":1186,"kov":2238,"kot":547,"kor":2342,"kop":1083,"kon":3505,"kom":6854,"kol":2347,"koj":25320,"kog":6388,"kod":903,"knj":735,"ko ":9564,"kla":1495,"klo":701,"klj":755,"eÄ‘u":2748,"joÅ¡":665,"jve":805,"eÄ‘e":812,"juj":863,"jug":1520,"jud":1914,"jsk":7836,"jst":574,"eÄi":628,"ju ":11452,"eÄe":573,"kaz":825,"kat":974,"kar":1304,"kas":686,"kan":1885,"kao":3422,"kal":1067,"kam":671,"kak":633,"kad":1513,"kac":594,"juž":1378,"ka ":22352,"juÄ":696,"juć":551,"ha ":913,"han":1101,"har":926,"he ":1979,"her":717,"hin":716,"hit":527,"go ":855,"gle":1361,"gla":3728,"god":2754,"gom":912,"gon":803,"gos":769,"gor":1668,"gov":5098,"gu ":1902,"gru":1369,"gra":9421,"gre":1427,"grÄ":1302,"ian":1493,"ic ":604,"iba":598,"ibe":551,"ia ":1869,"ifi":824,"ih ":17225,"icu":773,"ici":6028,"ich":645,"ice":4697,"ie ":604,"ica":6876,"idi":521,"ide":1150,"ida":944,"aÅ¡n":1360,"aÅ¡t":576,"il ":644,"ija":16185,"ije":24598,"iji":7426,"ijo":787,"ijs":5406,"iju":2114,"im ":10462,"ika":7961,"ige":790,"iga":952,"igi":547,"igr":1302,"iho":1185,"ik ":6489,"imo":723,"imp":613,"ime":3556,"imi":856,"inc":1602,"ind":1666,"ina":9909,"ino":2039,"int":1177,"ins":3389,"ine":10127,"ing":2574,"inj":1865,"ini":6764,"inu":1508,"iko":2904,"iki":780,"ike":2426,"ila":3734,"in ":2960,"iku":1705,"ilo":2029,"ill":863,"ilm":540,"ilj":1386,"ili":10385,"ile":865,"ima":9207,"io ":4778,"ils":642,"ilu":562,"hov":1213,"hrv":1503,"hva":1061,"fer":566,"ez ":610,"ezu":527,"eza":1009,"ezn":818,"eze":1246,"ezi":9990,"eta":3704,"ete":898,"eti":2144,"etn":2390,"etl":530,"etk":539,"esn":1243,"est":6673,"eto":1100,"etr":912,"ets":1480,"etu":996,"etv":760,"eve":4353,"eva":1803,"evo":961,"evn":623,"evi":1701,"er ":4432,"epa":580,"es ":2168,"epu":2109,"epo":647,"epr":650,"eri":7215,"erg":698,"ere":1639,"erc":963,"era":4007,"et ":2066,"esk":1461,"esm":595,"esi":709,"ese":1073,"esa":706,"erv":605,"eru":1435,"ert":600,"ers":1367,"ern":3520,"erm":720,"ero":2614,"eki":802,"eko":2778,"eks":1589,"ekt":2568,"eku":789,"en ":4377,"ela":3182,"ele":2918,"eli":4944,"elj":5961,"ell":698,"elo":2967,"elu":1458,"ema":3624,"eme":6131,"eml":1258,"emo":840,"emi":1481,"ene":3561,"eng":1640,"ena":7266,"end":836,"enc":1218,"eno":6088,"eni":7665,"enj":3407,"enu":1213,"ens":1955,"ent":3912,"ego":1890,"egi":1521,"ek ":753,"el ":1052,"ejs":573,"eke":1782,"eka":2350,"em ":3991,"gl ":618,"gij":2556,"gin":523,"gi ":831,"gen":1325,"ger":827,"gdj":702,"ge ":1704,"gar":946,"gal":662,"gan":1740,"ga ":4609,"aÄ‘a":855,"aÄ‘e":658,"fra":927,"aÄa":1970,"aÄe":950,"aÄk":3501,"fri":721,"aÄi":1814,"fsk":622,"aÄu":958,"for":1658,"aÄ ":583,"aće":585,"aća":1021,"fil":896,"fik":566,"fin":592,"da ":9164,"de ":2788,"dal":1347,"daj":785,"dat":824,"dar":1385,"dan":5150,"dam":626,"dav":567,"cus":960,"co ":549,"cu ":955,"cea":713,"ch ":519,"ces":1492,"cen":997,"ceg":685,"ci ":6216,"cha":750,"ck ":1290,"che":856,"chi":796,"cij":9280,"cim":1128,"cir":569,"cio":1059,"ed ":1254,"eba":923,"ebe":581,"ean":810,"ea ":773,"duž":619,"ega":1085,"edn":7588,"edi":4719,"ede":1425,"eda":3130,"eg ":2170,"eds":1513,"edo":1202,"eci":935,"eca":935,"dvi":741,"dvo":690,"dva":1103,"drž":4140,"don":1767,"dom":1052,"dol":970,"dok":659,"dov":1189,"dos":617,"diÅ¡":1956,"dna":2624,"dne":953,"dni":3564,"dnj":1612,"dno":6490,"dob":1518,"dst":1252,"dra":1195,"dre":1145,"du ":2280,"dru":5517,"dsk":2664,"dic":3889,"der":1371,"des":966,"den":1450,"di ":4014,"do ":2419,"dje":2862,"dim":541,"din":5519,"dio":1844,"dis":1126,"dij":7896,"raÄ‘":876,"rga":1240,"ri ":5834,"rgi":678,"ret":1800,"res":1271,"rev":690,"rez":931,"rać":609,"raÄ":1555,"reb":1913,"rea":805,"ree":521,"red":6133,"reg":1745,"rem":2802,"ren":2797,"rek":1056,"rel":612,"rep":724,"rdi":692,"re ":4525,"rce":928,"raz":4813,"rd ":686,"rap":1294,"ras":1520,"rat":4060,"rav":4237,"rbi":882,"raj":3616,"rag":678,"ran":11269,"ram":1816,"ral":3163,"rak":1775,"raf":1147,"rad":6716,"rac":1934,"ros":2073,"rot":1439,"rom":2618,"ron":1652,"rop":1763,"roz":1266,"rov":2438,"rob":584,"roa":633,"rod":8265,"roc":1470,"roj":2300,"roi":1622,"rol":793,"rok":933,"rog":1195,"rno":2932,"riÄ":3609,"rna":1479,"rne":1371,"rnj":845,"rni":2710,"ro ":1317,"rma":1223,"rmi":655,"rlo":551,"rn ":540,"rkv":521,"rka":695,"reÄ‘":981,"reć":667,"raž":797,"rje":601,"riz":799,"rip":1837,"rio":889,"rir":879,"rit":2510,"ris":3174,"riv":1004,"rig":523,"rij":11045,"raÅ¡":597,"ril":1430,"rik":1383,"rin":1848,"rim":1849,"ria":651,"rib":805,"ric":1431,"rid":583,"rug":4029,"rup":1471,"run":520,"ruk":844,"rus":1028,"rva":5151,"rvi":1279,"rve":1037,"rvo":605,"ry ":556,"rsk":5201,"rta":726,"rst":2436,"rti":871,"rt ":714,"ru ":2471,"sad":738,"sam":1694,"san":1142,"sat":604,"sas":3039,"sav":1014,"sa ":5478,"ruž":889,"ruÅ¡":721,"ruÄ":2722,"rzi":592,"shi":607,"si ":1399,"sje":4603,"sis":550,"sin":1783,"sil":686,"sim":836,"sij":991,"se ":14671,"ser":1178,"set":665,"sh ":559,"seb":745,"sel":4338,"spo":2263,"spr":578,"spe":611,"spa":672,"sov":613,"son":650,"sob":1342,"su ":8627,"sre":2282,"st ":3926,"slj":632,"sli":1331,"slo":1832,"slu":1752,"sla":2093,"ski":17159,"skl":527,"sko":15307,"skr":553,"sku":5305,"ska":7019,"ske":5955,"sno":3094,"sna":922,"sni":2255,"sne":697,"smj":1004,"sma":1116,"ste":2346,"sta":16592,"sto":9996,"sti":9753,"stv":3654,"stu":2357,"str":6456,"sus":1535,"sva":788,"sve":1415,"svi":1441,"svj":804,"svo":1576,"taj":996,"tak":2161,"tal":4191,"tac":782,"tav":5990,"tat":1128,"tar":4163,"tan":6149,"tam":583,"te ":6089,"ta ":9339,"pa ":1847,"pe ":1308,"par":1595,"pat":647,"pad":4813,"pal":580,"pan":3273,"pi ":571,"per":1812,"pet":668,"pla":1804,"plj":816,"pli":1041,"ple":3986,"plo":1052,"pje":1454,"pij":889,"pin":4572,"pis":2255,"poz":2191,"por":5631,"pop":1191,"pov":2463,"pot":1881,"pos":3892,"poj":1457,"pog":612,"pom":933,"pon":975,"pok":1642,"pol":3220,"pod":6429,"po ":2082,"psk":1887,"pub":2165,"pti":586,"poÄ":712,"pra":3170,"prv":1850,"pri":7027,"pre":6249,"pro":8677,"put":1060,"pun":676,"pul":698,"iÅ¡e":1062,"iÅ¡n":1267,"iÅ¡t":2503,"qui":570,"ra ":9271,"ngo":698,"ngl":1753,"ni ":16130,"nga":751,"nej":882,"nek":2622,"nen":652,"nep":734,"ner":1509,"net":1026,"nes":1293,"ng ":2367,"naÄ":2757,"nez":1162,"nci":2911,"nce":931,"ne ":19985,"ndu":534,"ndo":1052,"ndi":5037,"nde":837,"nda":1627,"ncu":940,"nak":2307,"nal":5099,"nam":1658,"nan":1732,"nap":971,"nar":3446,"nac":3781,"nad":1618,"nag":771,"naj":3818,"nd ":1313,"nav":631,"nat":3622,"nas":6912,"naz":2805,"na ":42477,"mož":877,"nut":1280,"nto":781,"ntr":1002,"nti":2109,"nta":2295,"nte":1843,"nst":1771,"nsk":10758,"nu ":5412,"iÄn":3327,"iÄk":8118,"iÄi":1329,"iÄa":967,"nt ":937,"niÅ¡":631,"ns ":584,"noa":520,"nog":6035,"noj":3137,"nom":6601,"nos":6166,"nor":535,"nov":4240,"niÄ":2517,"ića":608,"no ":15032,"nka":782,"nji":3978,"nje":12914,"nja":6205,"ić ":1370,"nju":1904,"njs":879,"njo":1594,"nij":6542,"naÅ¡":1014,"nih":6201,"nic":5054,"niz":1589,"nis":1359,"nit":879,"nir":714,"nim":4415,"nin":1303,"nik":4325,"ogr":2375,"ogu":995,"ogi":1476,"ogl":635,"ogo":1802,"oga":1651,"oj ":12832,"ois":1145,"oim":653,"ok ":1418,"oju":1294,"ojs":576,"ojo":759,"ojn":1052,"oji":10382,"oje":6734,"oja":6046,"ol ":664,"oiz":1012,"oce":973,"oci":1114,"ock":1115,"obu":893,"ode":1274,"odi":8132,"odo":921,"odn":4204,"ods":1138,"odr":4259,"of ":888,"oda":2801,"odu":910,"og ":13004,"oan":559,"oba":2782,"od ":14807,"obo":933,"obr":1117,"obl":1881,"obn":555,"obj":1316,"obi":2599,"oz ":529,"ozn":3312,"ozi":1205,"oza":1421,"oti":1557,"ote":985,"otr":921,"otp":614,"oto":4095,"ost":8908,"ota":912,"ov ":1004,"osi":1554,"ose":1141,"osl":1958,"oso":1519,"osn":2717,"ovj":940,"ovi":7022,"ovn":2887,"ovr":829,"ovo":6487,"ova":7007,"ove":3130,"opć":1972,"opo":690,"opi":1778,"opl":1169,"ope":858,"opa":775,"os ":1176,"opu":936,"opr":664,"ops":861,"or ":2008,"orm":1284,"orn":2821,"oro":4229,"ord":815,"ore":2892,"org":1291,"ori":7402,"osa":1092,"ort":1833,"ors":1607,"oru":1306,"ot ":522,"ora":3600,"ola":1856,"on ":4329,"olj":2400,"oli":4446,"ole":1119,"ols":789,"olo":3261,"olu":1664,"oka":2684,"om ":17269,"oke":734,"okr":3164,"oko":3273,"oku":2145,"ona":3954,"ond":619,"one":1941,"ong":1053,"onj":777,"oni":2936,"ono":2047,"ons":1609,"ont":1171,"onu":585,"oma":2281,"ome":2911,"omi":1374,"omp":729,"omo":1785,"la ":9649,"le ":3757,"lac":1275,"lad":1494,"lag":644,"lak":731,"lan":3996,"lam":781,"lar":880,"lat":2578,"las":3015,"lav":4030,"laz":5016,"lbu":1947,"kva":701,"kup":5219,"kul":1164,"ksi":936,"ktr":973,"koÄ‘":596,"ktu":654,"kti":1269,"kuć":703,"lok":712,"lon":791,"lom":1506,"lop":558,"log":2016,"lov":4118,"lno":1832,"lni":2255,"lne":712,"lob":538,"liÄ":1936,"lna":885,"ltu":737,"lub":608,"lsk":2111,"lu ":2550,"liÅ¡":638,"lj ":865,"li ":12760,"les":1936,"let":657,"lem":4158,"len":1436,"lek":2007,"led":614,"lo ":4167,"lla":657,"lle":663,"lli":529,"ljs":545,"lju":4071,"lje":10652,"ll ":619,"lja":4832,"lji":2209,"lit":2066,"lis":1339,"lin":3083,"lim":1508,"liz":1420,"lic":2147,"lia":609,"lik":6521,"lij":3213,"lig":541,"ma ":15645,"luž":1294,"mac":714,"maj":623,"mak":533,"mar":837,"mas":812,"mal":2079,"man":3666,"mat":2524,"mbi":905,"me ":5240,"med":673,"met":3414,"mer":3791,"mel":710,"men":6548,"maÄ":1334,"lum":944,"loÅ¡":641,"mpi":753,"mog":976,"mon":1050,"mor":2726,"mos":601,"mot":653,"mu ":1457,"msk":1673,"mun":787,"mi ":1004,"meÄ‘":2505,"mje":5038,"min":2090,"mil":697,"mir":1036,"mis":832,"eÅ¡t":1289,"mit":939,"eÅ¡k":669,"mij":1629,"mo ":1420,"mlj":1522,"mno":900,"rža":4071,"Äa ":974,"Äan":800,"Äar":558,"Äaj":693,"uÄa":545,"zra":1677,"Äe ":853,"uÄj":2470,"uÄi":687,"Äav":1111,"Äen":1159,"Äet":1250,"Äes":595,"zu ":768,"zva":612,"zvi":939,"Äi ":1484,"zvo":1982,"zum":625,"Äij":837,"Äic":566,"Äit":844,"Äin":2554,"Äko":4001,"Äka":1912,"Äke":2028,"Äki":5096,"Äju":1626,"ÄeÅ¡":556,"Äla":542,"Äni":1699,"Äno":3306,"Äna":751,"Äne":1006,"zi ":3239,"zem":1449,"zer":1084,"ze ":973,"zbo":641,"zda":1154,"zac":771,"zbe":843,"zaj":1076,"zam":819,"zan":918,"zal":859,"zap":3007,"zav":531,"zas":565,"zon":794,"zme":1381,"zna":6328,"zno":753,"ziÄ":837,"zni":942,"zla":762,"uća":616,"zli":1013,"ući":766,"zic":1241,"zij":2320,"rÅ¡i":715,"zin":1079,"zil":1675,"zik":7193,"zir":891,"ziv":3073,"za ":7407,"ya ":644,"ože":1175,"ću ":879,"ćin":2919,"ći ":2008,"rÄk":1144,"oÅ¡ ":708,"oÅ¡k":698,"vrÅ¡":1066,"wa ":713,"viÅ¡":1245,"vrt":868,"vrs":1358,"vri":916,"vre":806,"vsk":989,"vu ":2689,"vir":547,"vil":1092,"vim":1483,"vin":3680,"vih":903,"vij":3997,"vic":888,"vid":903,"vit":817,"vis":1291,"već":1856,"vje":3831,"vla":1440,"vlj":3383,"vo ":3548,"vne":801,"vna":1303,"vno":3450,"vić":621,"vni":4024,"vod":2717,"vog":1164,"voj":3212,"vol":582,"vom":1226,"vor":5059,"vot":928,"voz":1030,"vi ":4716,"vaÄ":1765,"vez":1580,"ver":4932,"vet":804,"vać":849,"ven":3207,"vel":1857,"ved":700,"ve ":5319,"val":1770,"vak":779,"van":5697,"var":1878,"vat":5563,"vac":751,"vaj":1521,"va ":12711,"uz ":1098,"usk":1818,"usi":743,"ust":3150,"uti":590,"ute":706,"uta":1448,"uto":1403,"us ":1654,"ut ":910,"ura":1400,"ure":1189,"uri":1158,"urn":947,"uro":1486,"urs":550,"uru":580,"upa":2662,"upi":4500,"upe":980,"upo":759,"upr":748,"upn":605,"umi":603,"umj":671,"uma":1379,"umb":688,"ume":724,"unu":749,"uni":993,"und":654,"una":2557,"up ":535,"uko":757,"ukl":657,"um ":2138,"uka":702,"uju":1023,"ult":1215,"pći":2432,"uli":687,"ula":1753,"uhv":776,"uje":2393,"ugi":1126,"uge":731,"ugo":2301,"ugl":583,"uga":1630,"ugu":1137,"uda":924,"udi":2722,"ue ":562,"uci":556,"ug ":588,"ua ":834,"ual":550,"uan":840,"ubl":2337,"tvu":537,"tvr":1039,"tvo":2139,"tve":1133,"oÄ‘e":1121,"tva":2106,"tur":2517,"tup":876,"tud":1103,"tre":2014,"oÄe":731,"tra":6680,"će ":1908,"oÄi":564,"tri":3139,"tru":1417,"tro":2790,"oÄn":2463,"tu ":3006,"tsk":7605,"toÄ":2840,"ćen":895,"to ":5478,"tni":2342,"ća ":3122,"tna":778,"tiÄ":3116,"tno":1901,"toc":579,"toj":1710,"toi":723,"tog":984,"tov":1949,"tom":1889,"ton":1847,"tok":3601,"tol":1732,"tor":3179,"top":785,"ćan":554,"tij":2254,"til":813,"tik":1009,"tih":692,"tir":1327,"tit":1406,"tis":916,"tin":3058,"tim":1630,"tio":1426,"tic":2096,"tiv":2430,"tje":1395,"tko":583,"tka":775,"tla":700,"tem":1590,"ten":2148,"tek":931,"tel":2393,"th ":723,"tet":818,"ter":4781,"ti ":9529,"the":1111,"živ":2628,"žni":894,"žno":1666,"že ":1842,"žbe":582,"žav":3922,"žan":794,"ži ":866,"žen":1273,"užb":678,"uže":864,"uži":889,"užn":2163,"žup":1446,"uÅ¡t":795},"n_words":[5153330,5928363,4281211],"name":"hr","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/hu.json b/contrib/languages-data/hu.json
new file mode 100644
index 0000000..74a903a
--- /dev/null
+++ b/contrib/languages-data/hu.json
@@ -0,0 +1 @@
+{"freq":{"D":13198,"E":18562,"F":13797,"G":13466,"A":98368,"B":24663,"C":21970,"L":15870,"M":28130,"N":17752,"O":8481,"H":13530,"I":14875,"J":7703,"K":21077,"U":4237,"T":19473,"W":4434,"V":10406,"P":22295,"S":35593,"R":12093,"Y":1303,"X":1480,"Z":4534,"f":108322,"g":317259,"d":216557,"e":996693,"b":219927,"c":121096,"a":1016786,"n":579901,"o":467276,"l":670215,"m":298372,"j":133885,"k":454857,"h":131050,"i":506132,"w":6657,"v":203150,"u":147118,"t":700784,"s":637262,"r":536637,"q":1410,"p":129287,"z":429255,"y":242745,"x":7186,"É":1936,"Ã":2646,"í":56628,"é":327830,"á":372527,"ü":54519,"ú":35451,"ö":107023,"ó":117740,"Å‘":81190,"ű":27405," l":36854," m":86086," n":55262," o":27009," h":46816," i":30318," j":25021," k":101750," d":20317," e":90097," f":58652," g":17434,"Ñ€":1511," a":216424,"Ñ":1133," b":38681," c":32807," z":5071," u":7248," t":73839," v":67319," p":25823," s":75268," r":41479," J":7511," K":20282," H":12799," I":11800," N":16438," O":7214," L":14736," M":26568," B":23082," C":19809," A":94593," F":12742," G":12445," D":12015," E":17107,"л":1197," Z":3901,"к":1191," Y":1159,"и":1900,"о":2301,"н":1605," S":32225,"в":1255," R":11088,"а":2749," P":20591," W":4161," V":9030," U":3712,"е":1700," T":17702," á":18269," í":4682," é":57281," ö":8570," ó":3751," ü":2190," ú":3722," Ã":2459," É":1858," Å‘":1879,"A ":56345,"Da":1481,"Cs":2770,"Co":4107,"Ce":1232,"Ch":2651,"Du":1186,"Do":1533,"De":2255,"Di":1628,"Fe":3066,"Fa":1242,"Ez":1912,"Eu":1996,"Er":1727,"El":1696,"Eg":3224,"Ge":1854,"Ga":2079,"I ":2334,"Fr":2249,"Bí":1300,"Fo":1548,"Fi":1170,"B ":1118,"C ":1489,"Au":1680,"Ar":2744,"Ba":3859,"Az":15235,"Ac":1577,"Am":2415,"An":3658,"Al":3816,"Bu":2584,"Br":2522,"Ca":4180,"Bi":1847,"Be":4063,"Bo":3301,"Kr":1187,"Ko":2867,"Le":3145,"Li":2545,"La":4095,"Lo":1996,"Me":3040,"Mi":3545,"Ma":10861,"Mu":1674,"Mo":3340,"Ne":3381,"Na":3799,"Ny":3322,"No":1352,"Ol":1338,"Gr":1713,"Go":1444,"Gy":1283,"Ha":3592,"He":2397,"II":1513,"Dé":1163,"Ho":2637,"In":4213,"Is":1406,"Ja":1924,"Je":1294,"Jo":1519,"Ka":3817,"Ki":3017,"Ke":1650,"Tu":1174,"Tr":2483,"To":1807,"Th":2533,"Ti":1466,"Te":2749,"Ta":2278,"V ":1123,"Sz":12584,"St":3173,"Wi":1325,"Vi":2150,"Va":2270,"Ve":1656,"Pr":2035,"S ":1580,"Pe":2500,"Pa":5475,"Po":2747,"Pi":1727,"Or":1697,"Kö":2908,"Se":1702,"Sc":1225,"Si":1430,"Sp":1130,"So":1614,"Sa":3389,"Re":2307,"Né":1630,"Ro":2833,"Ra":1803,"b ":16527,"a ":242203,"bö":1784,"i ":117842,"bó":6751,"ge":24009,"gf":2198,"ga":25428,"gb":5764,"bé":2495,"ff":1119,"fi":9218,"fr":3611,"fu":1929,"fo":23259,"j ":9703,"gy":101312,"gz":1512,"dá":7767,"he":27807,"ha":38767,"gn":5176,"gm":1339,"cé":2844,"gl":4237,"gk":2851,"gj":4145,"gi":17202,"gh":4135,"gg":3431,"gv":1711,"gu":3889,"gt":2295,"gs":3570,"gr":6943,"cí":4683,"go":14059,"dt":3427,"du":4251,"dv":2240,"dz":1317,"g ":45451,"ea":6887,"eb":8727,"ec":7541,"ed":20502,"de":31392,"dd":1273,"di":23518,"dj":21891,"do":18282,"dn":1552,"ds":5503,"dr":4675,"ex":2202,"eu":3957,"ev":16778,"ey":1469,"ez":42514,"fa":14955,"h ":4414,"bá":5025,"fe":22515,"eh":6429,"eg":88315,"ef":4014,"ee":2573,"el":138888,"ek":55065,"ej":8422,"ei":11205,"ep":13034,"eo":3448,"en":123802,"em":38648,"et":109362,"es":66857,"er":112461,"ca":6636,"e ":99285,"bs":1600,"br":4886,"bu":6371,"bo":9382,"bl":2460,"bi":10991,"bb":21421,"bd":1747,"be":56227,"db":1466,"da":33610,"f ":3042,"cu":2732,"ct":3944,"cs":39940,"cr":1412,"co":5597,"ck":3543,"ci":18130,"ch":10237,"ce":11684,"cc":1311,"c ":4126,"az":52261,"ay":1779,"ba":62023,"d ":20001,"at":73551,"as":35129,"ar":73118,"av":6465,"au":6414,"ak":76786,"al":112498,"ai":30999,"aj":22921,"ao":1282,"ap":21474,"am":34363,"an":99045,"ac":9905,"ad":30543,"aa":1143,"ab":12613,"ag":47002,"ah":3611,"ae":13109,"af":4252,"nu":6592,"nt":48640,"ns":9377,"no":16464,"nn":10307,"nz":2533,"ny":61139,"gÅ‘":1259,"jó":1888,"ká":8866,"of":2805,"oc":6235,"od":12616,"ob":7903,"jú":2913,"ké":20553,"om":30097,"on":54876,"ok":33935,"ol":57719,"oi":1954,"og":16189,"oh":1599,"ot":26104,"os":55594,"ov":15603,"ou":3364,"op":11263,"oo":1729,"or":73234,"kí":3018,"kú":6878,"r ":40419,"ow":1527,"kó":3434,"kö":37773,"oz":36942,"lá":37200,"pe":16459,"pa":16699,"kü":4521,"pc":1541,"pl":5060,"lé":24839,"po":17893,"ph":4251,"pi":8966,"pj":2198,"pk":2115,"lo":28055,"ln":6206,"lm":14166,"hé":2687,"ll":38085,"ls":10986,"lr":2355,"hí":1890,"lv":12472,"lu":8529,"lt":43305,"lz":1116,"ly":54235,"o ":10039,"ma":45288,"mb":11466,"me":75090,"iá":11532,"ml":3581,"mi":28248,"mn":1344,"mm":4364,"mp":6217,"mo":16948,"ms":1960,"mu":7004,"fÅ‘":4516,"ió":9250,"mz":4734,"p ":5660,"na":50058,"nb":3564,"nc":11966,"nd":37316,"ne":56223,"já":30832,"nf":2395,"ng":19748,"nh":1725,"ni":26541,"nj":1322,"nk":8011,"nl":3257,"jé":12386,"jt":5016,"ju":2496,"jn":3362,"jo":4214,"jl":2908,"fé":13187,"ki":30779,"kh":2892,"ke":42527,"gá":10089,"kc":1158,"kb":6547,"fü":2239,"ka":34729,"m ":27269,"fö":2942,"jz":1414,"gó":4573,"dÅ‘":5431,"gö":3372,"ks":5542,"kt":6729,"ku":13047,"kv":1892,"ko":37181,"gí":1165,"kr":7237,"kk":6940,"kl":3302,"gé":11029,"km":3318,"kn":3693,"li":35208,"lh":5532,"lk":11727,"lj":5903,"le":93067,"há":9402,"ld":10308,"lg":5358,"lf":2782,"la":70668,"lc":4145,"lb":4771,"gú":1727,"n ":153664,"hr":1649,"dí":4512,"dó":5588,"ht":1620,"hu":2691,"hi":9568,"hn":1454,"ho":21034,"dé":9871,"id":22338,"ic":13315,"ib":5712,"ia":30517,"ih":1907,"ig":15930,"if":9201,"eá":1335,"ie":7180,"dö":1970,"hy":1869,"k ":156709,"ir":14921,"is":42797,"it":23842,"iu":6023,"iv":7173,"bÅ‘":3863,"ii":3341,"ij":1228,"ik":46279,"il":27833,"im":8040,"in":53934,"io":8559,"ip":5320,"fá":1392,"je":23937,"jd":2673,"iz":8720,"l ":91636,"ja":22570,"xi":1625,"té":34673,"tí":4348,"tó":18526,"sö":2411,"só":2652,"sú":4418,"z ":78459,"sü":3136,"tá":43166,"wi":1167,"sé":27679,"sí":6485,"ró":12762,"rö":8024,"y ":64628,"rú":3555,"rü":11432,"wa":1485,"sá":25703,"ré":19532,"vi":19889,"vt":2486,"rí":1306,"vo":15593,"mű":8899,"nÅ‘":6159,"ve":52514,"rá":28508,"va":38110,"x ":2815,"ui":1873,"uk":4884,"ul":23432,"ue":1945,"ug":7493,"ur":14404,"us":30910,"ut":12436,"um":11242,"un":8785,"up":2459,"ty":4352,"tu":13656,"tt":50460,"tv":7094,"ub":2532,"ua":1854,"pü":4993,"ud":8185,"uc":2378,"to":52152,"tn":2524,"pé":3054,"tm":2785,"tl":6382,"ts":8729,"tr":17609,"pí":3835,"tf":1412,"te":87104,"pá":7384,"tk":5205,"tj":7119,"ti":40828,"th":9478,"v ":9584,"tb":6187,"tc":1163,"ta":76171,"su":4875,"sv":2378,"ss":22768,"st":26226,"sz":174760,"lÅ‘":14903,"sl":2577,"sk":7704,"sn":2526,"sm":5506,"sp":4512,"so":26449,"sr":3213,"sd":1132,"sc":3372,"sf":1789,"se":34881,"sh":2900,"si":20642,"nö":2962,"rz":5827,"u ":5228,"sa":40529,"sb":5688,"rr":6334,"rs":21922,"rt":47601,"ru":7619,"rv":8829,"nó":1459,"kÅ‘":1313,"ry":3017,"rp":2572,"ro":47249,"rn":9437,"né":20022,"rm":20825,"rl":4279,"rk":7954,"rj":4223,"ri":45410,"rh":2668,"rg":7757,"rf":3837,"re":67985,"ná":12107,"rd":9230,"rc":7289,"rb":5639,"ra":55047,"t ":159290,"mú":1197,"mó":2093,"qu":1194,"mí":1861,"mé":15355,"má":27550,"lü":5880,"s ":157503,"lö":4000,"pt":5892,"pu":3980,"ló":16644,"pp":2547,"lí":3237,"pr":7474,"ps":3440,"yÅ‘":1310,"zü":5181,"zú":1280,"zá":31274,"yú":1973,"yü":3112,"zé":19023,"zí":5603,"zö":10272,"zó":20498,"vű":1517,"yá":13547,"yí":1801,"yé":5272,"yó":2584,"vÅ‘":2313,"tű":4358,"vö":1632,"zz":3816,"sű":1526,"tÅ‘":14413,"zf":1314,"zg":2806,"zh":3151,"zi":25268,"zb":1632,"zd":4010,"ze":71368,"vá":29898,"za":30193,"yz":1903,"zv":1698,"rű":2676,"sÅ‘":8855,"zs":12765,"zu":3963,"zt":36837,"zo":19843,"zn":6608,"ví":2765,"zp":2246,"zk":4768,"vé":17783,"zm":2937,"zl":11760,"yg":1834,"yh":1922,"ye":40237,"uá":1400,"yf":2056,"yc":1116,"ya":26830,"tü":4027,"yb":3764,"tú":2750,"tö":12442,"rÅ‘":4773,"yv":4653,"yu":3843,"yt":3497,"ys":7991,"yr":1838,"yp":1398,"yo":12260,"yn":3914,"ym":1973,"yl":2832,"yk":2442,"yi":17109,"zÅ‘":8108,"yű":1984,"Ãl":1439,"ám":8760,"án":50938,"áp":2119,"áj":7251,"ák":18338,"ál":59120,"ág":29984,"áh":4265,"áb":34134,"ác":4739,"ád":14308,"áz":7779,"áv":4411,"áu":1148,"ár":52483,"át":21141,"ás":47264,"á ":2111,"óz":2079,"ós":5892,"ót":4344,"óv":1296,"óa":1234,"ób":3397,"ój":4542,"ói":2619,"óg":4159,"óf":2044,"ód":5384,"óc":1867,"ór":2907,"óp":3032,"ón":2969,"óm":2752,"ól":15303,"ók":6328,"ó ":44110,"ív":3409,"íz":3248,"ín":4459,"ím":4915,"ír":8949,"ít":21446,"íg":1483,"ík":1149,"íl":1523,"íj":1882,"éz":4750,"ék":26204,"él":30309,"éj":1119,"ép":16482,"ém":6476,"én":36437,"és":84643,"ét":19103,"ér":22242,"év":16532,"éb":17184,"éd":3694,"éc":1321,"éh":4561,"ég":31689,"é ":4130,"úz":1141,"ün":2126,"ül":37857,"ür":1396,"üt":3250,"üz":1153,"üg":2645,"ük":3502,"úl":1994,"új":3185,"út":2986,"ús":1934,"úr":1723,"úa":6976,"úg":2023,"öv":8122,"öz":26938,"ú ":9189,"öt":9388,"ör":17835,"ös":7442,"ön":10766,"öl":8906,"öm":1336,"ök":5801,"ög":3454,"öd":2116,"öb":4016,"Å‘ ":26112,"Å‘v":2032,"Å‘r":2463,"Å‘s":10918,"Å‘t":2983,"Å‘z":2353,"Å‘d":2691,"Å‘b":2103,"Å‘g":1179,"Å‘e":1463,"Å‘l":9494,"Å‘k":4433,"Å‘j":2432,"Å‘i":2735,"Å‘n":2715,"ű ":13555,"űk":1684,"űe":1599,"űs":1405,"űr":1219,"űv":2905,"一":1237," Ãl":1439," Ga":2065," Ge":1837," Bí":1298," Fo":1535," Fr":2248," Fi":1161," Ha":3587," He":2393," Gy":1279," Go":1438," Gr":1700," Ho":2631," Dé":1163," Je":1288," Ja":1919," Is":1400," In":4206," Ka":3809," Ke":1641," Ki":3002," Jo":1515," La":4057," Le":3135," Li":2524," Ko":2859," Kr":1178," Ma":10822," Mi":3523," Me":3025," Lo":1988," Ne":3354," Na":3783," Mo":3325," Mu":1669," A ":54120," Am":2399," An":3650," Al":3798," Ac":1573," Ba":3855," Az":15203," Au":1680," Ar":2712," Be":4051," Bi":1842," Bo":3283," Br":2520," Bu":2577," Ca":4107," Ce":1230," Ch":2631," Co":4068," Cs":2760," Da":1472," Di":1622," De":2234," Do":1490," Du":1183," El":1692," Eg":3214," Er":1720," Ez":1906," Eu":1996," Fe":3060," Fa":1224," Wi":1310," a ":114242," Kö":2904," Or":1691," Po":2728," Pi":1722," Pe":2496," Pa":5457," Ny":3316," No":1346," Ol":1338," Ra":1786," Né":1627," Ro":2824," Re":2297," Pr":2030," Sz":12567," St":3106," Ta":2270," Th":2520," Ti":1460," Te":2728," Tr":2468," To":1786," Sa":3377," Si":1425," Sc":1193," Se":1692," So":1606," Sp":1107," Va":2262," Ve":1647," Vi":2142," Tu":1156," ja":1594," je":10035," in":4519," il":2853," is":10067," ir":2442," fü":1523," ka":7871," fö":1646," ki":17579," ke":13720," jo":1938," fé":4201," gy":6161," ha":19271," he":6504," cé":1611," cí":4405," gr":1159," id":3467," ig":3043," hi":3276," dé":2801," ho":8126," dí":1823," ne":18984," já":7979," na":5781," fÅ‘":4149," mu":2133," mo":4422," ok":1477," ol":6794," ké":11002," of":1186," ny":10296," no":1409," há":4031," le":17706," li":2353," la":7930," ku":2128," gö":2536," km":2432," kr":1168," ko":9227," me":30782," mi":10796," ma":18644," lo":1099," hí":1381," ad":3539," am":14965," an":8296," ak":3658," al":19507," au":2171," ar":2746," as":2187," ba":7443," az":39674," bi":2851," be":18442," bo":2566," bu":1302," er":6741," es":6369," en":1270," em":3989," el":22279," eg":37244," fe":16017," bá":1342," fa":10457," ez":6961," fu":1228," fr":2101," fo":11286," bé":1167," fi":3986," ge":1465," ga":1837," ci":1469," da":2485," cs":21597," do":1754," de":6050," di":2713," vé":3902," ví":1450," vá":10832," ze":2004," tö":9167," té":2457," tá":3920," nö":1405," sa":2016," se":2587," sp":2365," so":5755," ra":2479," re":19463," né":13996," ro":3461," pr":3595," má":8283," mé":4247," os":10567," kí":1660," or":4197," kö":27883," pe":2664," lá":2021," kü":3017," pa":5441," lé":4236," po":4686," pi":1682," ró":1121," rö":2141," va":20643," ve":9072," nÅ‘":1649," mű":3958," vo":11603," vi":8365," ré":10099," tu":3046," ut":3907," ur":1101," ta":23929," sz":51220," st":2457," su":1760," tr":1755," pé":1624," to":2214," ti":2324," te":19251," pá":3006," át":2658," ál":11522," év":4437," ér":4015," és":39564," én":2210," ép":2514," él":3293," ír":3245," ók":1246," ön":2076," ös":3774," ót":1104," új":1846,"Å‘l ":7803,"Å‘je":1778,"Å‘k ":2201,"Å‘ne":2127,"Å‘sz":2500,"Å‘s ":1549,"Å‘sí":1918,"Å‘sö":1177,"Å‘tt":1492,"Å‘bb":1227,"Å‘i ":1993,"Ãll":1152,"Eur":1570,"Ez ":1152,"Fra":1443,"Bír":1270,"Int":2403,"Az ":14884,"Ame":1184,"Bud":1205,"Car":1146,"Cse":1140,"Egy":3037,"Nem":1251,"Nag":2455,"Nye":1809,"űvé":1367,"Köz":2207,"Pas":2299,"Par":1124,"űve":1432,"űkö":1244,"Lad":1325,"Mar":2612,"Mag":3698,"éhe":3730,"ék ":14028,"ége":6434,"égb":1296,"égi":4433,"égy":1459,"él ":2635,"éde":1275,"ég ":10442,"ébe":12999,"éba":2195,"éve":6732,"ésé":4210,"év ":4758,"ész":20296,"éte":4730,"élÅ‘":1204,"ést":1350,"ésr":1270,"épü":1343,"épí":1687,"étr":1526,"éze":2382,"érÅ‘":1195,"éme":4235,"ép ":2557,"éne":8073,"éni":1139,"ént":3323,"ény":11240,"ékh":1465,"éke":3632,"éko":2177,"égé":2685,"én ":10291,"éli":2101,"éle":4096,"élk":1173,"ély":3515,"élt":1344,"ére":3670,"érf":1845,"ét ":8716,"ért":4687,"érs":1117,"éri":1359,"ése":7910,"ési":1788,"épe":3380,"ér ":1942,"éps":1130,"élé":8715,"és ":42926,"ésű":1277,"Szl":3665,"Sza":1311,"Sze":2596,"The":1583,"ául":1145,"áts":1437,"áto":1839,"biz":1578,"áss":1306,"ást":1822,"ász":3797,"áró":1875,"ásá":7057,"árá":5593,"áva":2230,"bol":1519,"átó":1225,"áté":2725,"áz ":1260,"ázi":1122,"bor":3409,"áza":2446,"áll":11176,"bbi":2623,"ált":12559,"bba":2240,"ály":14827,"be ":8763,"áma":1434,"ban":38532,"bal":3567,"baj":2665,"áku":1940,"bad":1154,"án ":13154,"ála":2623,"bar":1493,"áli":4198,"álh":2859,"áno":1436,"ány":14546,"ánu":1287,"ámo":1867,"ána":12065,"bda":1545,"áni":2143,"ájá":1672,"áló":2208,"ás ":11271,"bi ":2519,"ár ":6296,"ber":6263,"ben":26680,"bel":7243,"bes":1771,"bet":1775,"ása":4529,"árt":3175,"árs":2861,"áso":5295,"ási":3907,"ásb":2951,"át ":7693,"ámí":1237,"áro":12322,"árm":2982,"ári":3231,"ára":5673,"ács":1314,"áci":2905,"ág ":10677,"ádj":10076,"ábó":2969,"ága":2054,"ágb":3476,"ca ":1909,"ágo":3161,"ági":2804,"ák ":11048,"áho":4041,"ája":2888,"ál ":2859,"ágá":1372,"áki":3435,"ce ":2292,"ám ":1135,"bri":1367,"bra":1211,"bur":1175,"bum":2164,"ád ":2121,"ábo":1867,"ább":3556,"ába":23979,"ajz":1349,"aka":2195,"am ":2190,"agá":2325,"aki":3799,"afé":1601,"ajn":3205,"ajt":2171,"al ":17585,"aja":1802,"ajd":2547,"ain":2508,"ait":1256,"ak ":42544,"adó":2898,"aj ":6376,"agy":29473,"adá":3539,"agj":1385,"ago":2554,"anu":1841,"any":6283,"ano":1477,"ann":3140,"ant":4051,"ans":1439,"ane":1370,"ajá":1272,"ang":7307,"ani":3989,"ank":1265,"ana":3610,"anc":3826,"and":5977,"amm":1385,"amo":3002,"amp":1099,"ami":6863,"ame":12652,"amb":1439,"ama":2681,"alv":1163,"alu":2639,"alt":1806,"alr":1104,"alo":6119,"alm":6541,"all":3854,"alk":5861,"ali":4354,"alc":1755,"ale":2695,"ala":25236,"alb":2678,"an ":50569,"akr":1682,"aku":2116,"akt":1697,"ako":5426,"akk":1936,"aba":1856,"abb":3158,"abd":1557,"ae ":11230,"ad ":2344,"ai ":23587,"aga":3600,"ado":2879,"adi":3222,"ag ":2506,"adt":1359,"aci":1651,"ach":1191,"ace":1933,"ada":7719,"acs":1177,"azo":3899,"azi":1454,"azt":1117,"azg":1460,"aza":1771,"azd":1290,"azz":1187,"az ":35775,"asú":1719,"atá":8809,"ató":6586,"azá":1314,"ba ":10534,"bb ":12760,"at ":13186,"are":1125,"ard":2003,"arc":2552,"ara":12278,"aro":4590,"arm":1949,"arl":1467,"ark":2718,"ari":4156,"aru":1342,"arr":1237,"art":20941,"asa":1245,"aso":1347,"ar ":7389,"akö":2063,"apa":2996,"akú":5988,"alá":20616,"ape":1622,"apc":1186,"apj":1950,"api":1102,"apo":2602,"apt":1225,"aló":3345,"as ":5129,"ava":2849,"aut":1403,"avi":1292,"ará":1634,"asá":3004,"arú":1364,"asz":10871,"atb":1153,"ata":7891,"ast":1692,"ass":5488,"atl":1204,"apí":1933,"atr":1521,"ato":8994,"apá":1330,"ate":2686,"ati":9317,"ath":1161,"att":4210,"aur":1366,"jed":1383,"jel":12132,"jez":1371,"jes":2679,"jai":1887,"je ":3339,"fér":1780,"fél":10120,"job":1168,"jno":2503,"jog":1359,"jle":1493,"itr":1237,"ito":1407,"itt":1847,"isk":1278,"ism":3684,"iss":1661,"ist":5127,"isz":6268,"ita":2682,"ite":2144,"iti":1991,"ius":3051,"ium":2277,"inÅ‘":1958,"iva":2264,"ügg":1434,"irá":5579,"ive":1942,"is ":16119,"ion":3768,"ilá":4340,"iro":2666,"ise":2504,"ire":1148,"ira":1291,"it ":5468,"ült":4085,"ülé":2870,"izá":1108,"ja ":16271,"ül ":11962,"bÅ‘l":3435,"ük ":2123,"itá":2293,"üle":10807,"izo":1300,"ize":1447,"kif":1538,"kik":1160,"kil":1182,"kia":2258,"kin":1795,"kir":2041,"kis":3777,"kit":1528,"úsz":1202,"km ":2029,"ki ":6708,"khe":1740,"ked":3722,"kel":6145,"gál":2697,"ken":2595,"gán":1121,"kes":2444,"gás":1179,"ker":9767,"gár":1661,"ket":5479,"kez":5201,"ke ":3579,"kra":2981,"koz":3084,"kot":2662,"kos":4432,"kor":12163,"kon":4605,"kom":1428,"kol":2622,"kok":2194,"kod":1381,"gés":1802,"kna":1132,"kiá":3411,"géb":1114,"gép":2262,"gén":2847,"kke":2328,"kka":1690,"kbe":2008,"kba":2863,"kat":6062,"kar":2888,"kas":1212,"kap":3160,"kan":1437,"kal":4211,"kai":6519,"füg":1411,"ka ":5099,"föl":2822,"gyé":3341,"gyá":1929,"gyü":2858,"ha ":1609,"han":3792,"haj":2261,"hal":5044,"har":4293,"has":5102,"hat":10778,"hag":1327,"had":1308,"gyű":1250,"he ":2263,"hel":6775,"heg":1841,"hez":5105,"het":6443,"dás":3281,"dár":1447,"her":2201,"hiv":1273,"gla":1955,"gna":2223,"gió":1365,"cél":1462,"úak":6906,"gok":1862,"gol":3486,"gon":2487,"gos":1601,"got":1210,"gra":2375,"gre":1699,"cím":4480,"gsz":2045,"gy ":37866,"gus":1307,"gya":12084,"gye":12610,"gyh":1320,"gyi":8702,"gyk":1565,"gym":1148,"gyo":4080,"úgó":1191,"gys":4096,"iai":5083,"ial":1462,"ian":2020,"iat":1462,"iad":1714,"id ":1248,"ibe":1135,"ia ":14894,"ig ":5385,"ifo":5710,"ife":1711,"ics":1889,"ici":1201,"ich":2336,"ice":1195,"ie ":1156,"ica":2195,"ide":4043,"ida":9550,"ika":8060,"ige":3208,"iga":2520,"ii ":1284,"idé":1418,"ik ":22104,"imp":1521,"inc":1595,"ind":5305,"ina":5669,"ino":2798,"int":11506,"ine":3645,"ing":3990,"ini":2901,"ink":1323,"iká":2172,"inu":1420,"iko":1515,"ike":2523,"ila":1887,"in ":5995,"idÅ‘":1933,"iku":4841,"ilo":2015,"ill":7614,"ilm":1881,"ili":3110,"ile":2438,"hoz":7697,"hol":2205,"hon":1202,"hog":3581,"hos":1589,"hor":2048,"dék":1979,"dél":3343,"dés":3202,"dó ":3604,"dít":1872,"díj":1758,"fen":1212,"bán":1642,"fek":1857,"fel":10451,"fej":4036,"fia":1202,"ezÅ‘":2970,"fal":3069,"faj":8821,"etű":3115,"etÅ‘":6086,"ezé":5371,"etü":1875,"esü":1885,"ez ":6988,"eté":6862,"erű":1796,"ezd":1515,"erÅ‘":1876,"evé":4056,"ezt":2772,"eze":13658,"ezh":1979,"ezi":1930,"etb":3254,"eta":1302,"ete":14739,"eti":8705,"etl":3489,"etk":2537,"est":5914,"ess":3758,"elÅ‘":7242,"esz":10684,"epü":3424,"etr":2609,"ets":2876,"ett":18422,"etv":2394,"erá":1228,"eve":8161,"eré":4519,"eur":1117,"esí":1394,"erü":9565,"epe":2041,"epi":1162,"er ":12986,"es ":29789,"elü":5526,"epl":1338,"elé":3820,"erk":1552,"eri":12722,"erj":2568,"erg":1768,"ere":20225,"erc":2570,"erd":1315,"era":2862,"erb":2020,"et ":26086,"emé":3302,"esi":1683,"ese":5978,"erz":2655,"erv":3920,"err":1968,"ert":6946,"ers":4641,"ern":2834,"erm":4064,"ené":1314,"ero":1595,"eki":1276,"egé":3370,"ekt":2496,"en ":52180,"ela":1439,"ele":27299,"eli":3526,"elj":2109,"elh":1530,"ehé":1275,"elm":4378,"eln":2146,"elk":2863,"ell":9606,"elo":1349,"elv":9038,"els":6778,"elt":4823,"ely":23022,"emb":4386,"ema":1264,"eme":6876,"eml":2503,"emi":1303,"emp":1725,"emz":4591,"ene":6376,"eng":4545,"end":20330,"enc":3429,"ejé":1183,"enn":3397,"enl":1598,"eni":2643,"ens":3074,"ent":14391,"eny":3762,"egk":1751,"egj":2290,"egn":2926,"ege":4339,"egf":1828,"egi":1596,"egh":1485,"egr":1473,"egs":1610,"egt":1312,"egy":47388,"edé":2069,"ehe":2302,"ek ":33389,"ein":1947,"el ":15987,"eit":1126,"ejt":1562,"ejl":2010,"eje":2098,"eke":7178,"ekb":1825,"em ":9006,"ött":6080,"gje":1874,"öss":4328,"gja":1441,"git":1317,"gia":2402,"ört":3402,"örz":1373,"öná":1325,"gha":1393,"ös ":1742,"gi ":5691,"ör ":1444,"gen":2834,"get":4617,"ger":3980,"ges":2578,"gel":2023,"öny":3378,"gek":1625,"önb":1254,"ge ":3243,"gbe":1509,"öld":3665,"gaz":3489,"gba":3483,"ölt":2040,"gas":2172,"gar":1902,"gat":6541,"gal":3993,"ga ":3541,"ök ":2555,"ög ":1983,"ból":6420,"fra":1911,"özö":5568,"özé":2985,"for":12637,"fon":1546,"fol":3847,"özi":2766,"öze":2220,"özs":4673,"özp":1522,"özl":1118,"övé":1547,"fog":4106,"özt":1797,"örü":1396,"fil":2271,"fin":1712,"övi":1986,"öve":4144,"örö":5177,"da ":3729,"de ":4718,"dal":6673,"dae":8117,"dat":1797,"das":1500,"dar":6693,"dap":1398,"cti":1331,"csá":1388,"ciá":1338,"ció":4016,"cs ":2427,"öbb":3612,"cse":2576,"csa":18273,"cso":5464,"csi":2762,"csk":1549,"cea":1428,"ch ":1376,"cer":1367,"cen":1114,"ci ":1364,"cha":1323,"cia":3115,"ck ":1275,"che":1523,"chi":1879,"ed ":1845,"ebb":4696,"ebe":1132,"eae":1455,"ea ":1332,"efo":1148,"ei ":4004,"ega":1438,"edi":4113,"ede":6411,"eg ":8600,"ech":1220,"ect":1158,"ecs":1616,"dor":1576,"don":3362,"dom":3952,"dol":1610,"dot":2319,"djá":9950,"djé":9456,"dsz":3860,"dul":1851,"dt ":1182,"dia":1805,"der":2519,"des":1303,"det":5300,"dez":2835,"del":4174,"den":3777,"di ":3532,"dja":1819,"din":2017,"dig":1984,"dik":7071,"rga":1547,"ri ":7426,"rgi":1214,"rge":1233,"ret":2640,"res":5836,"óta":1799,"rfi":1520,"red":5622,"ósz":1200,"reg":2954,"rem":1197,"ren":20766,"rek":3056,"rel":1449,"nál":7068,"rep":4197,"rde":1294,"ós ":2694,"re ":12803,"rci":1283,"rce":1814,"ópa":1701,"rd ":1700,"ras":2085,"rat":3432,"rba":1762,"rbe":1412,"raj":1914,"rai":1551,"rag":1242,"ran":7745,"ram":2669,"ral":2852,"rak":6278,"rab":2185,"rad":2745,"rac":1357,"rs ":1186,"ror":2542,"ros":13308,"rot":1270,"rom":4552,"ron":3143,"rop":1359,"roz":4272,"rov":1366,"rod":2730,"roc":1647,"rol":2608,"rok":2835,"rog":1810,"rny":1765,"rna":1947,"rne":1337,"rni":1311,"nél":1390,"ném":3320,"nép":2342,"név":7964,"rma":4453,"rme":7442,"riá":1542,"nég":1166,"rla":1110,"rke":1521,"rje":2570,"rit":2732,"ris":3358,"ril":1221,"rik":5031,"rin":5644,"ria":3879,"ric":1848,"rid":1666,"rie":1106,"rif":2877,"rk ":1195,"rsé":1351,"rtá":1222,"rté":4693,"rul":1250,"rus":2059,"rva":1143,"rve":3364,"ry ":1179,"rsa":2647,"rse":2444,"rsz":11249,"rta":3812,"rtj":2461,"rto":16676,"rte":3860,"rti":1302,"óba":1208,"rmé":1800,"rmá":4220,"rt ":8024,"rre":1962,"rra":1492,"sab":1704,"sai":1775,"sak":2444,"sal":14426,"óma":1626,"sba":3589,"sap":1951,"san":2277,"ójá":1365,"sat":1742,"sas":1771,"sa ":7871,"óko":1362,"ók ":2577,"ól ":13016,"ója":2231,"rvá":1234,"rze":1233,"ógi":2210,"ói ":1956,"növ":1656,"rzs":1300,"rvé":1214,"si ":8670,"sid":1199,"sil":1988,"sik":1599,"rzÅ‘":1316,"se ":7438,"ser":4397,"set":1567,"seb":2714,"sen":5233,"sem":1195,"sel":3187,"sek":2815,"spo":1141,"spa":1306,"sol":2274,"son":4142,"sop":2367,"sor":6863,"sod":2739,"sok":5478,"st ":5212,"sle":1305,"sko":1582,"ska":1477,"ske":1576,"sme":3505,"ssé":2608,"ssá":1740,"sz ":8870,"lÅ‘t":1770,"lÅ‘s":2401,"stá":1316,"sza":14173,"svá":1185,"sze":35971,"szo":9131,"szt":27632,"szu":2442,"szi":9598,"szl":6370,"szk":3277,"szn":4368,"sse":3446,"ssa":2570,"ssz":9449,"ste":4883,"sta":3478,"sto":1355,"sti":2035,"str":2018,"sug":1404,"lÅ‘ ":3629,"sré":1266,"tai":1258,"tak":4038,"tal":17363,"tag":2404,"tbe":3293,"tba":1669,"tat":5289,"tar":19006,"tan":4747,"te ":9032,"ta ":15756,"szá":22866,"szü":3008,"szé":7434,"szí":4720,"szö":3515,"szó":5026,"ozó":12847,"ozá":2383,"kúa":5586,"pa ":1509,"pcs":1390,"par":3913,"pat":2664,"kül":3938,"pai":1203,"pap":1164,"pan":2344,"pha":1287,"láb":2286,"ped":1503,"lád":12468,"lán":2463,"pen":1323,"per":4376,"lát":1961,"pes":3470,"lás":5210,"lág":4159,"pel":1327,"lál":4812,"plo":1196,"pia":1507,"por":4386,"pos":1197,"pon":4538,"pol":3552,"lét":2548,"lés":5210,"lén":1237,"lék":10125,"lég":1202,"pjá":1227,"psz":1291,"plÅ‘":1147,"ló ":6853,"pte":2465,"lít":2551,"pri":1456,"pre":1161,"pro":3235,"lós":1760,"lóg":2572,"pus":1929,"lód":1195,"lön":2431,"lül":5325,"már":3617,"más":7074,"mán":10674,"máj":1466,"még":1185,"mét":1869,"mér":2829,"més":1202,"mél":2519,"mén":4256,"mít":1327,"mód":1366,"ra ":14404,"ngo":3206,"ni ":7437,"nge":3949,"ngy":1650,"jáh":2978,"neg":1502,"nel":1635,"nek":17344,"ják":1953,"ján":4452,"nem":11176,"jár":7148,"ner":1420,"net":3267,"ját":5567,"nes":2093,"nev":9590,"ng ":3420,"jáb":8030,"ncs":2007,"nci":3666,"nce":2318,"ne ":3513,"ndr":1401,"nds":3385,"ndo":2353,"ndj":9787,"ndi":2220,"nde":6999,"nda":1524,"nak":21124,"nal":2777,"nap":2250,"nae":1236,"nag":5845,"nai":2208,"nd ":4041,"nba":1457,"nat":2233,"na ":6605,"mze":3907,"iós":1792,"iój":1245,"nyb":1400,"nya":6957,"nye":12400,"nyi":4978,"nté":5969,"ny ":9196,"nul":2608,"nus":1717,"nty":1140,"nto":3218,"nti":4462,"ntj":1553,"nta":2379,"nte":5991,"nsz":1210,"nse":1158,"nt ":15590,"ns ":1607,"nká":1105,"nok":4021,"nop":1219,"nos":3762,"nov":1101,"nne":2064,"nna":2413,"nni":1136,"nny":1741,"niá":1146,"jéb":6198,"jén":1719,"jéh":3083,"nle":1203,"no ":1255,"nka":1134,"nid":1544,"nic":1428,"nia":2714,"nk ":1863,"nis":2046,"nik":2308,"ogr":1697,"ogl":2068,"oga":3293,"ogy":4073,"ok ":16218,"ol ":3707,"ock":1291,"ode":1450,"odi":3096,"odo":1217,"of ":1189,"oda":2951,"kál":1176,"kán":1252,"káb":1589,"nyí":1276,"obb":3582,"nyo":6087,"nyv":3444,"nyu":2805,"nyt":1370,"nys":1658,"ntÅ‘":1470,"jú ":1653,"nyá":1816,"nyé":1280,"oz ":6136,"osí":1185,"köv":1585,"köt":1840,"ozt":1266,"köz":23037,"kön":3656,"köl":1449,"kör":4099,"ozo":1583,"köd":1357,"ová":8369,"ozi":2582,"oza":5292,"ott":13732,"oto":1925,"osz":17179,"ost":2095,"ítÅ‘":1100,"oss":3713,"oso":1195,"orú":1625,"osá":2471,"ovi":1122,"ova":1834,"ove":1321,"orá":3688,"íté":2953,"opo":3410,"olá":1371,"ító":2222,"os ":17347,"oló":3495,"opt":1890,"or ":6588,"ítá":3043,"orm":10619,"orn":2184,"oro":8033,"orr":2134,"ord":2929,"oná":1361,"ore":1109,"org":1652,"ori":4752,"osa":4828,"ort":5009,"ors":11126,"orv":1840,"omá":8665,"ot ":4277,"orb":1551,"ora":2591,"ízi":1186,"íto":2971,"ola":3942,"old":2725,"olc":1133,"on ":14829,"oli":4956,"oll":1909,"olg":2986,"ols":1101,"olt":11581,"oln":1426,"olo":2069,"oly":9282,"olu":1943,"okb":1937,"oka":3243,"om ":4999,"okk":1292,"okr":1131,"oks":2384,"íte":4142,"oko":2728,"ív ":1366,"okt":1201,"író":2841,"ona":4624,"ond":2206,"one":1152,"ong":1637,"oni":3479,"ono":2799,"ons":1750,"ont":8687,"ony":4114,"oma":1728,"omb":2831,"omi":1564,"omm":1333,"kék":1183,"kép":5158,"kén":3827,"írá":2059,"omo":2114,"két":4231,"kés":3743,"ímű":2569,"la ":5399,"le ":4396,"lcs":2466,"írt":1125,"lda":1565,"ldi":1183,"lab":1923,"lad":1437,"lag":3296,"laj":1625,"lal":2932,"lak":15098,"lan":5737,"lam":7155,"lap":6224,"lat":10133,"las":4338,"ld ":2188,"lbu":2257,"dÅ‘ ":2164,"kus":4984,"kul":4809,"ksá":2221,"ksz":1790,"gó ":1820,"gör":2727,"llí":1578,"llá":2364,"lló":2888,"lon":1455,"lom":5872,"lor":1462,"log":1806,"los":2605,"lot":1131,"lov":8327,"lne":1797,"ljá":1491,"lmi":2811,"lme":1597,"lma":4921,"lna":1780,"lto":3300,"lsz":1378,"lta":8440,"lte":3310,"lu ":1887,"lmé":1286,"lre":1816,"lt ":21002,"ldá":1177,"lha":4212,"li ":5160,"ház":2853,"lev":1937,"les":5249,"let":23881,"hár":2058,"lep":3720,"lem":6723,"len":17272,"lek":3977,"lel":1659,"leh":1502,"leg":14221,"háb":1477,"lla":10657,"lle":8630,"lli":2652,"llo":1404,"lko":4418,"lka":2131,"lke":2520,"lgá":2219,"lje":1805,"ll ":3110,"lja":2062,"lit":2107,"lis":6935,"lin":2766,"lim":1567,"lid":1319,"lia":3789,"lik":2243,"lyó":1710,"ma ":5513,"mai":2750,"maj":1342,"mad":7309,"mag":9145,"mar":2289,"mas":1406,"mal":2064,"man":2348,"maz":4614,"mat":4172,"mba":2545,"mbe":4662,"me ":1765,"meg":18261,"iáb":6342,"met":6641,"mes":7852,"mer":9128,"mel":20048,"men":4193,"mek":1223,"mez":3626,"lva":1622,"lve":3172,"lul":1401,"lus":1938,"ly ":11120,"lvt":1957,"ltá":1280,"lsó":1363,"lya":5925,"lyb":1133,"lyi":2296,"lye":12093,"lté":1277,"lvá":1133,"lyo":1466,"lyn":2379,"lys":1166,"lyt":1183,"lsÅ‘":6342,"lyá":9265,"mpi":1413,"mpl":1595,"mon":2735,"mok":3026,"mos":2616,"mot":1595,"moz":1966,"ió ":2881,"mus":1737,"mut":1302,"mun":2084,"mi ":6215,"min":10577,"mil":1543,"mit":2000,"mia":1836,"mik":1748,"mma":1863,"tÅ‘ ":4350,"tÅ‘l":2706,"tÅ‘s":1505,"tÅ‘n":1924,"sű ":1297,"zná":3356,"zt ":3494,"víz":2022,"zte":5806,"zti":1839,"zta":2495,"ztr":2410,"zto":1911,"zsi":1572,"sÅ‘ ":5513,"zul":1547,"ztá":11872,"zté":2691,"rű ":1401,"zsé":5276,"zza":1374,"zga":2049,"zi ":4058,"zhe":2186,"zet":19059,"zes":1480,"zen":10515,"ván":2981,"zem":3342,"zel":4337,"vál":6881,"zek":2284,"vák":7162,"vár":8676,"zer":19180,"ze ":4791,"zda":1164,"zab":2585,"zad":1388,"zak":6902,"zal":1294,"zat":8108,"zot":2710,"zor":2657,"zom":1281,"zon":5431,"zok":2301,"zol":2432,"zpo":1664,"vén":5253,"véd":2112,"vég":2913,"vét":1848,"vés":2023,"zió":1200,"zke":1119,"zle":1674,"zlo":7817,"zig":3735,"zin":2369,"zil":1212,"zik":7225,"zis":1133,"yve":2842,"yug":2984,"ysz":2663,"yok":1279,"yom":2043,"yol":1936,"yos":2298,"yob":2167,"za ":3143,"rÅ‘l":1883,"ysé":2805,"ye ":3860,"yei":1140,"yek":4530,"yed":1362,"yes":5416,"yer":1837,"yen":2702,"yel":9386,"yez":2468,"yet":5968,"ya ":3038,"yag":1568,"ybe":1927,"yba":1297,"yar":8873,"yan":3850,"tül":2293,"yal":1350,"yak":3842,"yai":1283,"yhá":1178,"yne":2282,"yi ":5728,"yik":7147,"tív":1267,"téz":2544,"tér":3350,"tét":2097,"tés":9077,"tén":6927,"tél":1509,"ték":6622,"töb":3419,"tör":6044,"tó ":7302,"tól":4069,"tán":3887,"tár":7602,"tás":11209,"táv":1125,"ták":3376,"tál":11084,"táj":1226,"sú ":1586,"sül":2209,"só ":1339,"sök":1579,"ség":20101,"sén":1506,"sér":1918,"sét":1189,"sít":5115,"sár":2700,"sát":1861,"sán":1607,"ság":11788,"sáb":4560,"rög":2754,"rök":1818,"röv":1876,"rúg":1325,"rül":11029,"ró ":3203,"vtu":1747,"ról":2801,"róp":2151,"vir":1492,"vil":4652,"vid":4003,"vis":3046,"réb":2703,"rég":2911,"rés":10041,"vol":10664,"von":3245,"vi ":1300,"vez":9822,"ver":7753,"rás":8381,"ves":3105,"vet":5802,"veg":1557,"rág":1665,"ven":5494,"rán":5049,"rál":4428,"vel":6375,"vek":3167,"ráb":2450,"ve ":7296,"val":9128,"van":3020,"var":2245,"vat":2280,"vas":2577,"vad":1432,"vag":11793,"va ":3234,"műv":2529,"műk":1274,"nÅ‘s":2336,"utó":1685,"utá":2191,"nÅ‘i":1248,"nÅ‘ ":1717,"mű ":3511,"uró":2193,"usz":5860,"ust":1128,"uta":3446,"uto":1342,"us ":17567,"ura":2400,"urg":1418,"uri":1391,"ulá":1430,"uma":1476,"umb":1113,"unk":2047,"uni":1375,"um ":4397,"ult":3281,"ula":2897,"uk ":1640,"ul ":10317,"uga":4002,"uda":1985,"udo":3577,"pül":4756,"tvá":1162,"trá":1500,"tve":3654,"tur":1775,"tus":2989,"tul":1430,"tum":1120,"tud":4114,"tté":1199,"ttá":1546,"tsé":3581,"pít":3633,"tre":3344,"tt ":30966,"tra":4155,"tri":3477,"tro":2690,"tsz":2410,"tta":4727,"tte":6235,"tti":1842,"to ":1211,"tjá":2955,"tos":4461,"tot":7575,"tkö":1448,"toz":15230,"tom":3113,"ton":3995,"tok":4029,"tol":3082,"tor":5700,"til":2521,"tik":5482,"tis":1923,"tin":5413,"tio":2102,"tiz":1109,"tja":3013,"tke":1731,"tla":1565,"tle":4022,"tem":4106,"pán":1462,"ten":6944,"tei":1486,"tek":7221,"pál":1428,"tel":10568,"teg":2075,"th ":1102,"tet":11629,"tes":7803,"ter":19863,"pár":2273,"ti ":14986,"the":2658,"tha":1854,"zÅ‘ ":4019,"zöt":4004,"zöv":1986,"zör":1244,"zül":3298,"zás":4765,"zár":4252,"zám":6211,"zál":1821,"zág":9532,"záz":1557,"yüt":2463,"zó ":13810,"vű ":1466,"zít":1657,"zín":2823,"zép":2406,"zér":1246,"zés":6463,"zén":2337,"zél":1670,"zék":2328,"ülö":2566,"ütt":2514,"ülÅ‘":1827,"yáb":1633,"yán":7191,"yár":2315,"tű ":2810,"vÅ‘ ":1785,"yó ":1102,"yéb":2439},"n_words":[10929783,12338513,8457220],"name":"hu","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/id.json b/contrib/languages-data/id.json
new file mode 100644
index 0000000..818be59
--- /dev/null
+++ b/contrib/languages-data/id.json
@@ -0,0 +1 @@
+{"freq":{"D":29898,"E":10468,"F":12839,"G":16234,"A":46756,"B":45618,"C":22345,"L":19113,"M":41589,"N":19093,"O":9096,"H":16882,"I":48750,"J":24390,"K":57099,"U":11179,"T":39498,"W":10090,"V":6089,"Q":1321,"P":55685,"S":65231,"R":21435,"Y":7072,"X":1468,"Z":2501,"f":36925,"g":365165,"d":485064,"e":840847,"b":246328,"c":70473,"a":2100570,"n":996656,"o":305761,"l":430448,"m":362974,"j":64260,"k":378208,"h":280863,"i":864649,"w":50622,"v":31442,"u":508685,"t":522270,"s":482635,"r":589018,"q":2068,"p":245252,"z":10934,"y":153007,"x":3585,"é":2192," l":29162," m":98895," n":17327," o":22384," h":14754," i":38006," j":15556," k":88933," d":239066," e":8087," f":10185," g":12086," a":116958," b":88325," c":8643," y":62811," z":1713," u":21958," t":81597," w":11107," v":3008," p":104408," s":135289," r":13775," J":23985," K":55803," H":16124," I":42065," N":16611," O":7318," L":17821," M":38172," B":43424," C":19640," A":41294," F":11657," G":14895," D":28065," E":8736," Z":2373," Y":6861," S":61440," R":19599," Q":1212," P":52451," W":9492," V":4735," U":10169," T":36300,"ا":1474,"A ":3175,"Da":7353,"Co":3999,"Ce":1371,"Ch":3483,"Ci":2915,"Du":2494,"Do":1602,"De":6798,"Di":6852,"Fe":1983,"Fa":1382,"Er":1543,"Ge":3292,"Ga":2686,"I ":4993,"Fr":1829,"Fo":1312,"Fi":2806,"C ":1897,"Au":2055,"Ar":4506,"As":2577,"D ":1256,"Ba":19721,"Ag":2188,"Ab":1488,"Ac":1714,"Ad":1447,"Am":5332,"An":4093,"Ap":1629,"Al":4784,"Bu":4215,"Br":3076,"Ca":4296,"Bi":2492,"Be":8472,"Bo":3000,"Ku":3549,"Kr":1685,"Ko":9825,"Le":3207,"Li":3131,"La":6311,"Lu":1669,"Lo":2427,"Me":8886,"Mi":3917,"Ma":15663,"Mu":4166,"Mo":3510,"Ni":1382,"Ne":3573,"Na":5130,"P ":1286,"Nu":1142,"No":3476,"Ok":1174,"Gr":2085,"Go":1818,"Gu":2419,"Ha":5613,"He":2039,"II":2521,"Hi":2733,"Ho":2454,"Hu":2080,"Ib":1304,"Ia":3224,"In":24134,"Is":3276,"It":2124,"Ir":1207,"Ja":10872,"Je":6536,"Jo":2100,"Ju":3132,"Ka":22425,"M ":1852,"Ki":1891,"Ke":13666,"Ut":2746,"Un":3136,"Tu":2495,"Tr":1946,"To":2609,"Th":2978,"Ti":8403,"Te":9113,"Ta":6615,"V ":1165,"Sy":1338,"St":3734,"Su":10264,"Wo":1285,"Wi":2707,"Wa":3162,"We":1219,"Vi":1573,"Pu":4020,"Pr":10851,"S ":1994,"Pe":15767,"Pa":12458,"Po":3505,"Pi":2342,"Or":1349,"Se":18572,"Si":5355,"Sh":2006,"Sp":1564,"So":3044,"Ru":2528,"Sa":8331,"Re":4622,"Ri":1937,"SM":1268,"Ro":3385,"T ":1420,"Ra":5456,"b ":5402,"a ":312256,"Ya":1618,"Yo":2051,"Yu":2161,"i ":261146,"ge":17252,"ga":100134,"fi":10254,"fr":1949,"fu":1399,"ft":1686,"fo":4136,"he":8947,"ha":50757,"gn":2069,"gl":1807,"gk":15524,"gi":27056,"gh":4457,"gg":24987,"gu":19428,"gs":5762,"gr":8256,"go":7739,"du":22038,"dy":1437,"g ":127207,"ea":8168,"eb":53877,"ec":18556,"ed":14792,"de":45859,"dd":1120,"di":143405,"do":23309,"ds":1430,"dr":3663,"ew":5523,"eu":3102,"ev":3178,"ey":2387,"fa":4277,"h ":162608,"fe":4040,"eh":16895,"eg":14415,"ef":2165,"ee":3361,"el":59121,"ek":27553,"ej":8227,"ei":5982,"ep":21232,"eo":11093,"en":149316,"em":59371,"et":34760,"es":60149,"er":212475,"ca":27397,"e ":42795,"br":3158,"bu":59966,"bo":6681,"bl":3420,"bi":20522,"be":67748,"da":219838,"f ":6725,"cu":3577,"ct":1798,"co":4648,"ck":2674,"ci":8787,"ch":7724,"ce":8123,"c ":1919,"az":1822,"ay":22914,"ba":74854,"d ":16186,"at":148965,"as":93999,"ar":156214,"aw":19383,"av":2545,"au":33011,"ak":85647,"al":170145,"ai":50563,"aj":9897,"ap":26307,"am":87049,"an":494871,"ac":7825,"ad":111448,"aa":15819,"ab":26392,"ag":30377,"ah":179811,"ae":6490,"af":4082,"nu":12849,"nt":61715,"ns":21631,"no":11352,"nn":7780,"nz":1307,"ny":39684,"oe":1902,"of":3903,"oc":2742,"od":8904,"oa":2254,"ob":5836,"om":19983,"on":61017,"ok":11631,"ol":35874,"oi":1595,"og":9163,"oh":4041,"ot":23779,"os":12530,"ov":13798,"ou":5529,"op":9093,"oo":2776,"or":39625,"r ":75373,"ow":2582,"oy":1610,"pe":68224,"pa":90070,"pl":2952,"po":12602,"ph":1712,"pi":13857,"lo":16018,"ln":1524,"lm":6776,"ll":7645,"ls":1781,"lu":24730,"lt":3200,"ly":1874,"o ":23320,"ma":94460,"mb":26120,"me":93044,"ml":1630,"mi":26777,"mn":2215,"mm":2111,"mp":24012,"mo":8478,"mu":28574,"p ":11473,"na":77075,"nc":11437,"nd":56574,"ne":39080,"nf":2387,"ng":238792,"ni":56068,"nj":13796,"nk":3871,"ju":15959,"jo":2256,"ki":23132,"kh":4590,"ke":56774,"ka":127350,"m ":50860,"ky":1397,"ks":9835,"kt":10395,"ku":22809,"ko":27878,"kr":3523,"kk":1592,"kl":3754,"km":2765,"kn":3265,"li":60829,"lk":2913,"le":44927,"ld":2217,"la":192799,"lb":1849,"n ":334938,"hr":1702,"ht":1424,"hu":19954,"hk":1872,"hi":19755,"hn":2129,"ho":5306,"hl":1668,"id":19549,"ic":8689,"ib":13081,"ia":81624,"ih":9019,"ig":10843,"if":6720,"ie":7742,"k ":77693,"ir":31102,"is":68271,"it":40678,"iu":5028,"iv":4491,"iw":2077,"ij":2865,"ik":61148,"il":48904,"im":30225,"in":117362,"io":16681,"ip":11977,"je":5678,"ji":4743,"iz":1367,"iy":1200,"l ":56560,"ja":34513,"z ":1416,"wi":10306,"wo":1229,"y ":11456,"wa":31269,"we":3398,"vi":15930,"vo":1681,"ve":8269,"va":3932,"x ":1658,"ui":4586,"uj":4080,"uk":44283,"ul":29290,"ue":2818,"uf":1241,"ug":8297,"uh":9753,"ur":39636,"us":36462,"ut":30979,"um":25849,"un":84334,"up":32836,"ty":2279,"tu":61107,"tt":3054,"ub":9153,"ua":56784,"ud":13238,"uc":2268,"w ":1917,"to":20692,"tn":2005,"tl":1402,"ts":2489,"tr":20074,"te":90779,"tk":2568,"ti":55141,"th":5635,"ta":176003,"su":25989,"ss":5417,"st":32797,"sy":2246,"sw":1219,"sl":3160,"sk":6158,"sn":2055,"sm":3761,"sp":3394,"so":6557,"sc":1962,"se":100046,"sh":4276,"si":103815,"u ":69102,"sa":100996,"rr":2177,"rs":13384,"rt":29454,"ru":41105,"rv":1161,"rw":1931,"ry":4130,"rp":3731,"ro":34136,"rn":12088,"rm":13211,"rl":14368,"rk":15615,"rj":5203,"ri":102805,"rh":3147,"rg":10415,"rf":1249,"re":30519,"rd":11856,"rc":2704,"rb":14366,"ra":143028,"t ":74952,"qu":1325,"s ":74416,"pt":3193,"pu":23123,"pr":13816,"ps":1371,"zi":2204,"za":3196,"ye":4611,"ya":121025,"yu":3182,"ys":1566,"yo":3138,"yi":2638,"一":3113," Ga":2665," Ge":3275," Fo":1303," Fr":1822," Fi":2802," Ha":5588," He":2029," Go":1803," Gr":2062," Gu":2405," Ib":1298," Ia":3220," Hu":2077," Ho":2444," II":1738," Hi":2728," Je":6511," Ja":10843," Ir":1206," Is":3262," It":2123," In":24109," Ka":22393," Ke":13629," Ki":1879," Jo":2086," Ju":3129," La":6277," Le":3183," Li":3097," Ko":9805," Kr":1679," Ku":3543," Ma":15603," Mi":3888," Me":8848," Lo":2414," Lu":1657," Ne":3562," Na":5110," Ni":1377," Mo":3486," Mu":4147," Ap":1628," Am":5319," An":4070," Al":4777," Ag":2185," Ac":1711," Ad":1437," Ab":1471," Ba":19679," Au":2049," As":2567," Ar":4490," Be":8434," Bi":2483," Bo":2979," Br":3065," Bu":4209," Ca":4213," Ce":1368," Ci":2897," Ch":3462," Co":3963," Da":7319," Di":6832," De":6775," Do":1546," Du":2487," Er":1536," Fe":1977," Fa":1366," Wo":1267," Wi":2689," We":1211," Wa":3147," Yu":2156," Yo":2043," Ya":1612," Or":1346," Po":3467," Pi":2336," Pe":15730," Pa":12421," Nu":1139," No":3465," Ok":1169," Ra":5409," SM":1160," Ro":3363," Re":4599," Ri":1930," Pr":10827," Pu":4013," Sy":1332," Su":10248," St":3681," Ta":6595," Th":2952," Ti":8388," Te":9090," Tr":1942," To":2586," Ru":2526," Sa":8312," Sh":1982," Si":5333," Se":18549," So":3026," Sp":1555," Vi":1551," Tu":2469," Un":3130," Ut":2746," ja":5269," je":2335," in":24841," il":1562," is":2175," it":1708," ka":17469," kh":1422," ki":3127," ke":41380," ju":6965," ha":6291," gr":1294," gu":1601," ib":1921," ia":3559," hi":4380," hu":2034," ne":5231," na":8812," mu":5887," mo":2739," ol":12623," of":1811," no":1685," le":5736," li":3289," la":14587," ku":3200," km":2201," kl":1852," ko":17290," me":68772," mi":4388," ma":16460," lu":3991," lo":1345," ag":1308," ab":1554," ad":60411," an":11749," ap":1720," ak":5080," al":4506," aw":1213," ar":2749," at":18641," as":3479," ba":29062," bi":8314," be":42050," bo":2101," bu":6066," ca":3227," en":1364," ek":1536," fo":1399," fi":5527," ge":4483," ga":3124," co":1499," ce":1281," da":93500," do":1270," de":28323," di":109723," du":4523," za":1414," ya":62479," ru":1822," sa":30188," se":78628," si":6482," sp":1184," so":1196," ra":4660," re":4944," ro":1171," pu":5843," pr":10257," or":4567," pe":52401," pa":26959," po":5413," pi":2294," wa":3549," wi":6744," tu":4514," ut":3268," um":2172," un":12469," ta":18998," st":3288," su":11745," tr":2305," to":2153," th":1170," ti":9886," te":42313,"Fil":1716,"Ger":1344,"II ":2070,"Han":1227,"Har":1306,"Ing":3276,"Int":1699,"Ind":17352,"Ia ":3146,"Ara":1730,"Aus":1437,"Bah":2306,"Bal":1493,"Ban":5008,"Bar":4958,"Bat":1605,"Agu":1534,"Ame":4085,"Ang":1356,"Ber":2116,"Ben":1538,"Bel":2649,"Bri":1173,"Des":2124,"Dal":2000,"Cha":1289,"Dia":1644,"Neg":1421,"Nov":1144,"Per":8357,"Pem":1634,"Pen":3260,"Pas":1289,"Par":2408,"Pad":2035,"Pan":1988,"Pul":1756,"Pro":7786,"Pre":1253,"Pol":1113,"Ita":1993,"Isl":1234,"Jan":1394,"Jak":1458,"Jep":2416,"Jer":2596,"Jaw":4820,"Jul":1161,"Kab":8059,"Kal":5302,"Kan":1250,"Kat":1399,"Kar":2399,"Ker":1724,"Kep":1438,"Kel":1127,"Kec":4407,"Kon":1421,"Kom":1142,"Kor":1136,"Kot":4453,"Lau":1252,"Men":2499,"Mer":1254,"Man":2093,"Mal":2386,"Mar":3752,"Mas":1454,"Mus":1132,"Yun":1506,"一一":1315,"Sur":1395,"Sum":1934,"Sul":1400,"Sun":2471,"Sta":2065,"Ten":4508,"Tan":2075,"Sin":1312,"Ser":4461,"Sep":1413,"Sel":5571,"Sem":1247,"Seb":1132,"Rus":1267,"Sam":1292,"San":2004,"Rep":1522,"Rom":1304,"Uni":2252,"Uta":2484,"Ter":1521,"The":1859,"Tim":5598,"bis":1615,"bit":1641,"bil":2430,"bin":3990,"bih":2863,"bli":2292,"bol":2158,"ban":15219,"bak":1477,"bal":2565,"bai":2045,"bag":18039,"bah":11866,"bad":2660,"baw":2021,"bat":5802,"bas":2119,"bar":6280,"beb":2207,"bed":1156,"ber":44372,"ben":5468,"bel":6123,"bek":1139,"bes":5623,"bia":3415,"bid":1129,"ca ":1339,"car":5372,"can":2310,"cam":11583,"ce ":2031,"bu ":2451,"bua":22699,"bup":9962,"bur":1967,"bul":1642,"buk":3601,"bun":4326,"bum":1823,"buh":2607,"but":6527,"aka":36392,"am ":27176,"aki":3783,"akh":2140,"al ":32136,"aja":7514,"aik":1989,"ain":9801,"air":2314,"ais":1591,"ait":2311,"ak ":26644,"ahk":1403,"ahi":5150,"ahu":13410,"aha":23711,"agi":8426,"agu":2010,"anu":3628,"any":11108,"ano":1219,"ann":4127,"ant":18067,"ans":3241,"ane":1770,"ang":133187,"ani":10218,"anj":5111,"ank":1863,"ap ":5145,"ana":17620,"anc":4861,"and":16448,"amu":2163,"amp":6554,"ami":2968,"ame":2230,"amb":4802,"ama":38387,"alu":3924,"alo":1291,"all":1612,"ali":20004,"ale":3013,"ala":101814,"an ":259764,"aks":3177,"aku":4198,"akt":4736,"aba":6181,"abi":1719,"abu":12017,"ae ":1556,"aca":2365,"aan":12880,"aat":2166,"ad ":3820,"ab ":3122,"afi":1142,"ai ":28231,"aga":15818,"age":1445,"aer":3528,"ah ":131776,"adi":10057,"ade":2164,"ach":1161,"ace":1304,"ada":91074,"ayu":1260,"aya":17559,"ba ":1212,"at ":42706,"arg":2616,"are":4682,"ard":2694,"ara":50200,"aro":1477,"arn":2327,"arl":1195,"ark":4616,"ari":39797,"aru":4864,"ars":1240,"art":10017,"au ":21880,"asa":28749,"ary":2056,"asi":28272,"ase":1504,"ask":2058,"ar ":23543,"apa":12599,"api":3283,"apu":2004,"as ":18260,"aut":2942,"ay ":1252,"awa":16458,"awi":1328,"ata":52964,"asu":4212,"ast":4229,"ass":1429,"asy":1183,"atk":1321,"ato":2873,"ate":14253,"ati":9780,"aua":1165,"atu":20191,"aup":1527,"aus":1167,"jen":2753,"jad":5570,"jaa":2051,"jab":1150,"jar":5491,"jal":2507,"jak":2749,"jan":3425,"jo ":1155,"ito":1897,"itu":7300,"ism":1635,"isu":1798,"ist":11957,"ita":13383,"ite":4451,"iti":3900,"iwa":1676,"ium":1147,"iun":2068,"ivi":1396,"ive":2266,"ipu":1250,"ipi":1541,"is ":23018,"ion":11007,"ipa":2509,"ipe":2501,"ir ":10536,"irk":1299,"iri":9503,"isi":11065,"ise":5218,"isa":7947,"ire":1684,"ira":4533,"it ":5451,"ja ":5112,"kim":1495,"kil":1164,"kin":2100,"kir":1475,"kis":1425,"kit":4346,"km ":1129,"ki ":8986,"khi":1575,"keb":2854,"kec":8512,"ked":2012,"kek":1191,"kem":4126,"kel":7010,"ken":6683,"kep":3726,"kes":1631,"ker":5150,"ket":3912,"ke ":5970,"ksa":1691,"ku ":4682,"kot":9086,"kon":4219,"kom":5415,"kol":2400,"koh":1184,"ks ":1167,"kny":1410,"kka":1328,"ko ":1260,"kla":1828,"juk":1809,"jun":1767,"jum":1474,"jua":1961,"jug":4990,"kaw":1325,"kat":13174,"kar":11177,"kas":4851,"kap":2757,"kan":62967,"kal":5122,"kam":1705,"kai":2738,"kad":1254,"kab":2943,"ka ":12884,"ha ":2059,"ham":2420,"han":14832,"hak":1134,"hal":1973,"har":6037,"has":12725,"hat":1841,"haa":1690,"had":1765,"he ":2996,"her":1241,"hi ":1844,"hid":1896,"hin":3929,"hir":6378,"hka":1709,"go ":1252,"gku":1576,"gor":1284,"got":2192,"gsa":3115,"gu ":1706,"gra":2940,"gri":3408,"gur":1410,"gus":2168,"gun":9840,"iam":1372,"ial":6166,"ian":16893,"iap":1176,"ias":3614,"iat":1370,"ibi":1510,"ibu":4263,"id ":2050,"iba":3456,"ibe":2500,"ia ":44404,"ier":1178,"ies":1321,"ifi":1731,"ih ":5574,"ich":1544,"ie ":1390,"ica":1838,"idu":1993,"idi":3417,"ide":3199,"ida":6661,"if ":2797,"il ":7886,"ija":1503,"im ":3888,"ika":23689,"iga":3238,"igu":3825,"iha":2592,"ik ":17419,"imp":2775,"ime":1812,"imi":1994,"ind":4767,"ina":9693,"imu":7350,"inn":1767,"ino":1354,"int":9752,"ins":11171,"ine":2698,"ing":27890,"ini":22825,"ink":1199,"iny":3792,"iko":1474,"iki":7614,"ike":5441,"ila":14804,"in ":16335,"iku":2328,"ill":2052,"ilm":5734,"ili":13211,"ile":1187,"ima":9934,"io ":1907,"hny":1246,"hub":1262,"hun":11971,"hus":1117,"fat":1257,"eta":14851,"ete":4068,"eti":4646,"est":3135,"esu":1622,"ess":1214,"etu":1676,"evi":1298,"ey ":1761,"ewa":3433,"epe":3912,"er ":19709,"epa":9668,"eor":7717,"es ":8727,"ept":1302,"epu":3274,"erk":7438,"erl":11827,"eri":25792,"erj":4499,"erg":3829,"erh":2837,"ere":6493,"erc":1371,"erd":7020,"era":35580,"erb":12574,"et ":5262,"esi":21745,"ese":3800,"esa":15867,"eru":21470,"ert":16167,"ers":10880,"ern":7132,"erm":8938,"erp":2943,"ero":2183,"eki":2304,"ekn":1342,"eko":2677,"eks":2683,"ekt":2574,"eku":2083,"en ":21818,"ela":27699,"ele":4883,"eli":4636,"ell":1619,"elo":2885,"elu":8272,"emb":14950,"ema":7624,"eme":9102,"emo":1229,"emi":9424,"emu":4943,"emp":7479,"ene":6504,"eng":44211,"ena":14758,"end":12541,"enc":3844,"eni":5826,"enj":6735,"enu":4939,"ens":2961,"ent":14866,"eny":6205,"ege":2018,"egi":2598,"ehi":1472,"ek ":3218,"ein":1306,"eja":5845,"el ":6722,"eke":1577,"eka":8278,"em ":3699,"gka":11656,"git":1200,"gin":2402,"gio":1293,"gia":7505,"gha":1865,"ggu":3578,"ggr":3634,"ggo":2263,"ggi":3715,"gga":11266,"gi ":10809,"gen":4341,"ger":4633,"gem":1548,"gel":2369,"ge ":1412,"gah":5405,"gai":13025,"gas":1879,"gar":10391,"gat":2080,"gam":3520,"gal":5628,"gan":35620,"gap":1320,"ga ":16084,"for":2821,"fil":3873,"fik":2255,"fis":1267,"da ":32556,"de ":5057,"dak":4551,"dal":74713,"dah":2729,"dae":2768,"dat":2016,"das":2265,"dar":32215,"dap":6287,"dan":52470,"dam":1256,"day":1876,"ch ":1540,"cer":1324,"cha":1339,"ck ":1406,"che":1327,"chi":1177,"cil":1660,"cis":2553,"ed ":1578,"eba":14092,"ebe":8142,"ebi":3337,"ebr":1132,"ebu":26049,"ea ":1655,"ei ":1901,"ega":7504,"eh ":13927,"edi":3728,"ede":1589,"eda":4173,"edu":2208,"eci":1667,"eca":14054,"dus":1247,"don":16622,"dok":1141,"dun":4005,"dup":1272,"dul":1434,"duk":5272,"dua":3278,"dud":2657,"dra":1669,"du ":1319,"did":3706,"dia":10670,"dib":6007,"der":4755,"des":7802,"del":1249,"dek":2240,"den":19332,"dem":1173,"di ":63731,"do ":1904,"dim":2715,"din":3800,"dio":1486,"dip":4666,"dir":8308,"dis":11623,"dit":5425,"dig":4149,"dik":8070,"dil":3331,"rha":2148,"rga":5059,"ri ":42938,"rge":1577,"ret":2887,"res":4698,"rg ":1147,"rea":2388,"rej":1691,"ren":4953,"rek":3031,"rda":4163,"rdi":3109,"re ":2800,"rbu":1584,"raw":1318,"rd ":2133,"rap":3154,"rar":1749,"ras":8094,"rat":11399,"rbi":1817,"rba":6850,"rbe":3317,"raj":2995,"rai":2568,"rah":9741,"rag":1442,"ran":36962,"ram":3611,"ral":3814,"rak":7284,"rab":2762,"raa":1221,"raf":1467,"rad":6359,"rs ":1283,"rpe":1250,"ros":2826,"rot":1395,"rom":1135,"ron":3571,"rop":2512,"rov":9832,"rod":2363,"rog":1358,"rny":1495,"rna":5736,"rmu":1260,"ro ":1992,"rma":9740,"rle":9267,"rla":2281,"rn ":1378,"rki":1226,"rke":4730,"rka":5975,"rja":3509,"rip":1275,"rio":1652,"rit":4490,"ris":9168,"ril":2736,"rik":14926,"rin":9510,"rim":1709,"ria":5936,"rib":1211,"ric":1356,"rid":1193,"rie":1621,"rk ":1641,"rya":1832,"ruh":2011,"rup":16158,"run":3050,"rum":2131,"ruk":1365,"rus":4704,"rut":3005,"ry ":1765,"rsi":3955,"rsa":2467,"rse":3931,"rta":11752,"rte":3710,"rti":7667,"rua":2111,"rtu":2198,"rt ":1478,"ru ":3013,"saa":3516,"sah":3766,"sai":1253,"sak":1265,"sal":14489,"sam":5305,"san":11536,"sat":14608,"sas":2791,"sar":11521,"saw":1247,"sa ":26535,"shi":1399,"si ":37996,"sid":1884,"sia":23497,"sit":3002,"siu":2036,"sir":1424,"sis":6271,"sin":5946,"sio":5312,"sil":3164,"sim":1558,"sik":5796,"sih":1154,"sif":1869,"se ":1750,"ser":7000,"ses":3161,"set":3675,"seh":1426,"sed":1550,"sec":2609,"seb":38986,"sep":5282,"seo":6619,"sen":4315,"sem":4262,"sel":6726,"sek":6032,"sej":4109,"spe":1482,"son":1660,"st ":1847,"ss ":1345,"sli":1139,"sla":1693,"ska":2806,"smi":1231,"sme":1407,"sya":1486,"ssa":1575,"ste":6115,"sta":5899,"sto":1411,"sti":6848,"stu":1610,"str":8471,"sua":5405,"sum":1276,"suk":5658,"sun":2892,"sut":1776,"sus":2249,"sur":1552,"tai":3534,"tak":12396,"tal":4630,"tah":16201,"tab":1684,"tau":16309,"tat":1881,"tas":11773,"tar":15390,"tap":2003,"tan":48487,"tam":7817,"te ":2719,"ta ":30047,"pa ":5126,"par":4662,"pat":19772,"pas":2458,"pad":19421,"pak":16990,"pal":3741,"pai":3130,"pan":11608,"pi ":3245,"pen":20640,"pem":8687,"per":28237,"pes":2547,"pel":2885,"pil":1386,"pin":3678,"pis":1169,"por":1652,"pop":1746,"pos":1337,"pon":1531,"pok":1819,"pol":2768,"pua":1330,"pub":1942,"pte":1271,"pri":2203,"pre":1413,"pro":9219,"pur":2238,"pus":1843,"put":3676,"pun":4895,"pul":5693,"ra ":33891,"ngo":1466,"ngi":5434,"ngk":15500,"ngu":4341,"ngs":5639,"ni ":27725,"nge":5034,"ngg":24626,"ngh":3138,"nga":45848,"neg":5658,"nen":1469,"ner":3870,"net":1984,"nes":16997,"ng ":124434,"nci":3629,"nce":1815,"nca":3309,"ne ":3872,"ndu":6040,"ndr":1343,"ndo":17880,"ndi":8610,"nde":4365,"nda":13200,"ncu":1191,"nak":8053,"nal":12498,"nam":10027,"nan":12711,"nar":2327,"nah":2814,"nai":1146,"nd ":3909,"nat":2083,"nas":5218,"na ":14205,"nya":32061,"nye":2679,"nyi":2079,"nul":1194,"nun":2161,"nus":2178,"nur":1832,"nua":1507,"nto":2680,"ntu":18006,"ntr":1617,"nti":5215,"nta":22900,"nte":7428,"nst":1796,"nse":1390,"nsi":12613,"nt ":2342,"ns ":1443,"nol":1247,"nom":2318,"nny":5115,"no ":2934,"nka":1820,"nja":9105,"nju":2843,"nia":6702,"niv":1591,"nis":7959,"nit":2077,"nin":2190,"nik":2713,"ogr":1789,"ogi":3408,"ok ":3549,"ol ":2833,"ode":3528,"of ":1858,"odu":2149,"oh ":1516,"obe":1842,"ote":1332,"oto":2549,"ota":16038,"osi":2975,"ose":2201,"oso":1148,"ovi":10349,"ove":2391,"oun":1267,"ope":1909,"opa":1279,"os ":1902,"opu":1750,"or ":7039,"orm":2622,"oro":1416,"ord":1550,"ore":2472,"org":2231,"ori":3198,"ort":1972,"ora":11685,"ola":5593,"on ":12856,"oli":4593,"ole":13717,"olo":5388,"oka":1673,"om ":1677,"oko":1983,"ona":7292,"ond":2230,"one":17406,"ong":6397,"oni":2889,"ono":3029,"ons":2868,"ont":3055,"oma":3069,"ome":2094,"omb":1474,"omi":2626,"omp":4769,"omo":1445,"omu":1988,"la ":8248,"le ":2787,"lah":85384,"lag":1642,"lai":7338,"lal":2146,"lak":4119,"lan":18336,"lam":23788,"lap":1829,"lar":2452,"lat":9794,"las":5560,"law":2074,"lau":6152,"lay":9124,"kut":2035,"kus":1523,"kur":1949,"kup":1223,"kun":2321,"kum":1981,"kul":1651,"kuk":1717,"kte":1325,"ksi":4460,"kua":1980,"ktr":1281,"ktu":2650,"kti":1994,"kto":2651,"lok":1231,"lon":2362,"lom":2722,"log":4305,"lny":1353,"lmu":1446,"lua":4860,"lta":1301,"lu ":3267,"li ":7633,"lev":1141,"les":2673,"let":9284,"ler":1546,"lem":2195,"len":2677,"lek":2289,"leh":12733,"leb":3217,"lla":1957,"lle":1326,"lli":1227,"lka":1991,"lm ":4360,"ll ":1780,"lit":5426,"lis":7079,"lir":1515,"lip":1370,"lin":6938,"lim":6542,"lia":5362,"lik":11324,"lih":1781,"ma ":21495,"mah":2164,"mai":3887,"mak":3420,"mad":1850,"mar":3160,"mas":10730,"mal":1793,"man":22564,"mat":17173,"mba":10647,"mbi":2114,"mbe":6808,"me ":2693,"mbu":4891,"med":1539,"met":2294,"mes":1768,"mer":26378,"mem":15519,"mel":4982,"men":35186,"luk":2200,"lui":1193,"lun":1652,"lum":2455,"lus":1551,"lur":4662,"mpi":3325,"mpe":2325,"mpo":2760,"mpu":6558,"mod":1185,"mon":1666,"mor":1409,"mpa":7594,"mu ":1626,"mud":2039,"mua":1660,"mur":7325,"mus":3149,"muk":2006,"mul":2774,"mum":2269,"mun":4790,"mi ":4545,"min":4316,"mil":8646,"mis":1673,"mit":1227,"mia":1700,"mik":1662,"mla":1505,"mny":1911,"zam":1194,"yu ":1225,"ya ":33584,"yat":1924,"yar":2043,"yan":64564,"yak":4682,"yah":7474,"yai":2823,"yi ":1127,"wi ":1323,"wil":6167,"wa ":9713,"wan":5957,"wal":2130,"wak":2165,"wat":1765,"war":3553,"was":1762,"wah":1890,"vin":9845,"vis":1830,"ver":3522,"usi":7004,"use":1166,"usa":6560,"usu":2654,"ust":3983,"uti":2003,"ute":2083,"uta":7869,"utu":1851,"utr":2712,"us ":12050,"ut ":11892,"ura":9315,"uri":1909,"urk":1245,"uru":7051,"uny":2077,"upa":26147,"ur ":13684,"upu":1725,"ump":1843,"umu":2414,"umi":1414,"uml":1462,"umn":1617,"uma":4069,"umb":3461,"ume":1307,"unt":12867,"unu":2087,"uni":6942,"unc":1388,"und":1967,"una":10560,"ung":21765,"une":1335,"up ":3437,"uks":1606,"uku":6493,"uko":1586,"uki":1626,"um ":7458,"uka":7728,"uju":3230,"ulu":3880,"ult":1653,"uli":4656,"ula":12496,"un ":20290,"uk ":22268,"ul ":3263,"ui ":2114,"uga":6118,"uha":3495,"uda":4275,"udi":2630,"ubu":2471,"uh ":4643,"udu":3551,"ua ":5771,"uat":8034,"uas":4449,"uar":6223,"ual":1917,"uan":9121,"ubl":2137,"uba":1945,"uah":19677,"ty ":1866,"tur":4927,"tus":3183,"tuj":1345,"tul":1791,"tuk":15174,"tun":2681,"tum":1823,"tua":3005,"ts ":1124,"tra":8844,"tri":6158,"tru":1229,"tro":2826,"tu ":22458,"to ":3427,"tny":1162,"tob":1155,"ton":2911,"tok":1669,"tol":2048,"tor":4842,"til":2355,"tik":7507,"tif":2658,"tig":1707,"tit":1315,"tis":2474,"tin":8673,"tim":3584,"tio":3164,"tia":2861,"tid":3087,"tiv":1124,"tka":2458,"tem":9574,"ten":16531,"tek":2322,"tel":6089,"th ":1540,"tet":1414,"tes":1167,"ter":45895,"ti ":10357,"the":1811},"n_words":[11077227,12709440,9643042],"name":"id","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/is.json b/contrib/languages-data/is.json
new file mode 100644
index 0000000..5fd568b
--- /dev/null
+++ b/contrib/languages-data/is.json
@@ -0,0 +1 @@
+{"freq":{"Aþe":44,"D":2516,"E":3951,"F":4399,"G":3894,"A":5835,"B":5986,"C":2240,"L":4227,"M":5323,"N":3677,"O":1245,"H":8629,"I":1566,"J":2209,"K":4574,"U":1075,"T":3307,"W":992,"V":3210,"Q":137,"P":2862,"S":11572,"R":3778,"Y":369,"X":126,"Z":178,"f":81003,"g":100010,"d":53734,"e":187693,"b":27005,"c":5204,"a":263792,"n":216937,"o":68668,"l":133209,"m":99769,"j":33645,"Fel":69,"k":84595,"h":42906,"Fen":29,"i":195706,"w":1317,"v":51876,"u":132172,"Fer":95,"Bár":54,"t":142015,"s":170723,"r":261058,"q":297,"p":24810,"z":1088,"y":26307,"x":2164,"Óði":53,"²":107,"Ã":3558,"É":51,"Æ":142,"Ã":1654,"Ã":39,"Þ":3879,"Ú":294,"Ö":390,"Ó":667,"Fil":53,"í":48796,"Fin":139,"Fim":41,"Fir":29,"é":6992,"è":36,"æ":24190,"Ã¥":50,"ä":93,"á":41692,"à":32,"þ":19539,"ü":180,"ý":7865,"ú":17028,"Fja":106,"ø":114,"ö":23897,"Fis":39,"ó":30363,"ð":97792,"Eyj":294,"Eys":84,"Eyr":71,"Ä":102,"Eyv":33,"Ä«":50,"Å":51,"Å¡":31,"Å«":35,"Fan":34,"Fal":50,"Far":50,"Að ":48,"Erl":41,"Elí":30,"Aða":120,"Eur":33,"ˈ":29,"Evr":590,"Ein":322,"Eir":81,"FL ":35,"Ì":73,"Eld":72,"Ekk":41,"μ":51,"End":39,"ν":134,"ο":164,"Eng":344,"ι":127,"κ":62,"λ":87,"δ":52,"ε":87,"η":55,"α":162,"ά":44,"ί":34,"Π":31,"Ell":80,"Er ":52,"ω":34,"ÏŒ":43,"Ens":32,"σ":73,"Ï‚":149,"Ï":115,"Ï€":60,"Ï…":43,"Ï„":83," l":13188,"ÑŒ":55," m":17205," n":9875,"Ñ":55," o":24158," h":25667," i":1873," j":2491," k":10596,"Ñ‹":69," d":4603,"Ñ…":30," e":51762," f":25096," g":9131,"ч":38,"Ñ€":151,"Ñ":242," a":22647," b":11739,"Ñ‚":158," c":312,"у":71," y":1803," x":58," z":29," u":8441," t":16530," w":71," v":22667," p":2352," s":52785," r":7093,"К":36,"Önn":32,"Ðœ":32," J":2203," K":4567," H":8585," I":1555," N":3667," O":1188," L":4213,"Gei":146," M":5293," B":5951," C":2201,"Р":33,"С":41," A":5808," F":4383," G":3869," D":2495," E":3934,"л":143,"к":145," Z":177," Y":365,"й":61,"и":226," X":119,"п":31,"о":257,"н":183,"м":63,"г":31," S":11508,"Ges":41," R":3756,"в":111,"Ger":69,"б":78," Q":135," P":2823,"а":337,"Geo":100,"з":43," W":981,"Gen":80," V":3197,"е":181," U":1070,"д":70," T":3273," æ":1194," á":19136," í":21823," é":48," ö":1771," ó":1123," ý":438," þ":16844," ú":4082,"Gla":37," Ã":1653," Æ":142," É":49," Ã":3550,"Gil":84," Ó":667," Ö":387," Ú":293," Þ":3878," Ã":38,"Búr":44,"×”":29,"ו":29,"Ölf":40,"×™":40,"Gal":86,"Gam":94,"Gat":36,"Gau":70,"Gar":113,"Gag":40,"Fræ":55,"Frá":36,"Ùˆ":81,"ÙŠ":130,"Ù„":172,"Ù…":100,"Ù†":92,"Ful":38,"د":85,"Ø­":41,"ت":33,"ب":97,"ا":239,"ع":51,"Ø´":31,"س":65,"ر":96,"Fru":56,"Fro":37,"Fyr":272,"Fyl":44,"Frí":39,"Bók":91,"Flu":50,"Flo":84,"Flj":91,"Fla":98,"Fle":86,"Bær":105,"Bæj":56,"Fra":556," К":36,"Fri":161," Ðœ":32,"Fre":118,"A ":369,"Fló":77,"Fos":51,"For":261,"Fjö":95," Π":31,"Fjó":52,"F ":112,"Da":796,"Cu":62,"Cl":179,"Dæm":64,"Co":498,"Cr":129,"Ce":91,"Ch":432,"Ci":87,"G ":277,"Ed":148,"Ea":44,"Du":121,"Dy":59,"Do":199,"Dr":201,"De":344,"Dj":72,"Di":273,"Aþ":47,"Bá":77,"Fe":260,"H ":152,"Fa":299,"Ey":536,"Hlu":52,"Eu":56,"Ev":632,"Ex":43,"Er":232,"Et":30,"Að":243,"Es":92,"En":512,"Em":79,"Ei":511,"El":310,"Ek":54,"Ef":192,"Hlj":179,"Eg":213,"Ge":554,"Ga":560,"Bú":144,"Hjá":32,"I ":270,"Bö":36,"Fy":316,"Bó":133,"Fu":127,"Fr":1134,"Bí":44,"Fo":403,"Fl":550,"Fj":284,"Bæ":200,"Fi":347,"B ":133," о":38," Р":33," С":41,"II ":73,"C ":290,"Av":40,"Au":691,"Ar":661,"At":288,"As":369,"D ":98,"Ba":1277,"Af":349,"Ag":60,"Aa":38,"Ab":97,"Ac":57,"Ad":101,"Am":376,"An":599,"Ap":140,"Ai":48,"Ak":259,"Al":997,"By":66,"Hit":42,"His":29,"Bu":149,"Br":1094,"Ca":498,"E ":99,"Hja":125,"Bi":297,"Be":746,"Hil":53,"Bo":829,"Bl":365,"Hin":97,"Bj":353,"Hip":31,"Kv":148,"Ku":90,"Gö":52,"Ky":159,"Kn":101,"Kl":180,"Kr":557,"Gí":124,"Ko":534,"Há":323,"Le":589,"Lj":88,"Hæ":71,"Li":540,"N ":121,"La":1290,"Lu":200,"Hó":220,"Ly":57,"Hö":355,"Hé":115,"Lo":531,"Hí":33,"Me":774,"Mi":929,"Mj":65,"O ":134,"Hú":1094,"Ma":1619,"Mc":44,"My":193,"Mu":181,"Ið":49,"Mo":567,"Nj":57,"Ni":233,"Já":37,"Ne":516,"Na":464,"P ":275,"Jö":155,"Hel":374,"Jó":681,"Hei":302,"Nu":48,"No":1524,"Hea":36,"Ok":35,"Ol":56,"On":43,"Oh":30,"Od":80,"Ká":86,"Hen":69,"Of":128,"Jú":86,"Hes":54,"Her":215,"Gi":186,"Gj":38,"Gn":42,"Gl":211,"Gr":968,"Go":225,"Gu":618,"Gv":43,"Gy":51,"J ":42,"Ha":2194,"He":1196,"Hi":344,"Dæ":71,"Hj":179,"Hl":321,"Hn":67,"Ho":620,"Hr":479,"Hu":359,"Hv":448,"Dó":65,"Dö":29,"K ":93,"Dý":44,"Ic":56,"Im":36,"In":761,"Il":85,"Is":113,"Eð":29,"Ja":517,"L ":154,"Fá":50,"Je":231,"Jo":341,"Fí":29,"Fæ":128,"Fé":158,"Fó":50,"Hag":49,"Haf":163,"Ju":93,"Hal":249,"Ka":1200,"Fö":37,"Han":1145,"M ":133,"Ham":103,"Har":211,"Kj":148,"Ki":277,"Hau":84,"Ke":369,"Ut":32,"Ur":47,"Up":198,"Um":210,"Un":333,"Ul":39,"W ":56,"Ty":102,"Guð":291,"Pó":142,"Tv":79,"Tu":195,"Pí":33,"Tr":409,"Pé":116,"To":307,"Th":490,"Ti":314,"Tj":67,"Pá":120,"Te":275,"Ta":376,"V ":127,"Sy":96,"St":1866,"Sv":974,"Su":942,"Sí":179,"Wo":95,"Wi":310,"Sæ":90,"Wh":42,"Sé":40,"Wa":265,"Rú":290,"We":159,"Sá":78,"Ró":242,"Rö":41,"Vo":162,"Rí":181,"Úlf":48,"Vi":653,"Ré":30,"X ":104,"Úkr":30,"Va":711,"Rá":45,"Ve":1144,"Má":196,"Lý":96,"Grá":30,"Græ":154,"Mæ":40,"Gun":117,"Ló":43,"Pu":82,"Pr":335,"Lí":171,"Lú":62,"S ":238,"Lö":102,"Lá":110,"Pe":379,"Gua":30,"Pa":545,"Pl":227,"Po":355,"Pi":180,"Ph":126,"Gul":70,"Læ":49,"Os":65,"Ot":31," ا":112,"Op":55,"Kí":164,"Or":413,"R ":119,"Kú":67,"Kó":214,"Ox":46,"Kö":70,"Se":576,"Sc":214,"Sj":305,"Si":722,"Sh":148,"Sn":277,"Sm":127,"Sl":218,"Sk":1512,"Sp":372,"So":351,"Ru":106,"Nó":36,"U ":71,"Sa":1406,"Ný":191,"Nú":101,"Mý":140,"Re":1401,"Ná":93,"Ri":256,"Rh":36,"Næ":38,"Ro":345,"Ní":89,"Mí":43,"Qu":74,"Grí":106,"Mó":77,"Mö":80,"T ":79,"Mú":128,"Gró":29,"Mü":48,"Ra":615,"Gre":171,"Gri":130,"Gra":178,"b ":287,"Gru":62,"Gro":42,"a ":54859,"Gnú":34,"Tú":51,"Tö":63,"Yf":66,"Ya":33,"Yo":164,"Gle":35,"Sú":126,"Sö":178,"Só":163,"Glo":38,"Tá":38,"Sý":65,"Té":39,"Tæ":41,"Tó":200,"Tí":127,"Vö":72,"Gol":33,"Got":52,"Za":29,"Ze":49,"Vé":39,"Ví":264,"bö":256,"i ":42436,"fv":268,"bó":1425,"fy":4976,"gd":575,"ge":6266,"gf":407,"bú":1553,"ga":14643,"gb":298,"bý":357,"fk":117,"Ing":198,"fj":4059,"bæ":1840,"fm":226,"fl":4455,"fg":129,"ff":584,"fi":7839,"fh":173,"fs":2374,"fr":11940,"Inn":76,"fu":5606,"ft":4797,"Int":121,"fo":2567,"fn":8061,"Ins":29,"bí":505,"j ":69,"gy":295,"hf":59,"he":10053,"dá":141,"hb":32,"ha":7740,"gn":2900,"gm":495,"gl":2693,"gk":130,"gj":2483,"gi":7861,"gh":415,"gg":3399,"gv":803,"gu":10964,"gt":1944,"gs":3263,"gr":4868,"go":889,"dt":82,"du":6343,"dv":327,"dw":47,"dy":153,"g ":30278,"ea":698,"eb":435,"ec":343,"ed":647,"de":2371,"dd":1771,"dg":199,"df":200,"di":10756,"dh":181,"dk":373,"dj":161,"dm":128,"dl":570,"do":802,"dn":635,"ds":4031,"Ill":42,"dr":2023,"aó":39,"ew":295,"ex":734,"eu":292,"ev":474,"ey":6435,"aö":53,"ez":75,"fa":5499,"h ":796,"aú":77,"Ind":243,"fd":62,"bá":511,"fe":3046,"fb":250,"aþ":342,"eh":101,"eg":10758,"ef":8971,"ee":293,"aá":119,"el":7041,"ek":4309,"aæ":251,"ej":35,"ei":27077,"ep":1965,"eo":285,"en":19500,"em":17579,"et":5721,"að":23009,"es":9022,"er":48228,"eq":33,"aí":380,"ca":618,"e ":4753,"by":1518,"bs":81,"br":3631,"bu":941,"bn":48,"bo":3424,"bj":538,"bl":1526,"bi":1485,"bb":255,"be":3254,"db":244,"da":11080,"f ":7700,"cy":44,"cu":238,"ct":305,"cs":53,"cr":172,"co":503,"cm":37,"ck":446,"cl":64,"ci":425,"ch":1138,"ce":634,"cc":89,"c ":322,"az":129,"ay":341,"ba":3436,"d ":7343,"at":7217,"as":9693,"ar":51349,"aq":51,"ax":603,"aw":133,"av":2668,"au":7198,"ak":4372,"al":17385,"ai":546,"aj":229,"ao":105,"ap":1426,"am":11839,"an":36124,"ac":749,"ad":1746,"aa":265,"ab":2521,"ag":8003,"ah":1785,"ae":748,"af":17312,"nu":14078,"nt":3812,"ns":14817,"nr":473,"np":59,"no":4303,"nn":28163,"q ":62,"jö":4211,"nz":284,"ny":302,"nw":40,"jó":9027,"nv":1399,"oe":103,"ká":825,"of":4047,"oc":382,"od":513,"oa":97,"ob":315,"jú":991,"om":2016,"on":7027,"ok":2788,"ol":2476,"oi":174,"oj":70,"kæ":150,"og":21658,"oh":219,"ot":3409,"m²":103,"os":2326,"ov":311,"ou":631,"op":1077,"oo":305,"or":15125,"kí":733,"kú":380,"r ":98244,"ox":157,"ow":306,"kó":2090,"oz":31,"kö":623,"oy":162,"lá":981,"pe":1831,"pf":85,"pg":70,"pa":2152,"pb":43,"ký":317,"pl":1650,"lé":431,"pm":157,"pn":706,"po":550,"ph":775,"pi":1889,"pj":95,"læ":985,"pk":33,"lo":2737,"ln":490,"hé":660,"lm":1436,"ll":11846,"ls":4614,"lr":408,"hí":33,"lp":267,"hó":925,"lv":1365,"lu":9341,"lt":2655,"hö":2143,"lz":40,"ly":1089,"hú":1293,"o ":2043,"md":435,"ma":10956,"hý":74,"mb":2218,"mg":109,"mh":618,"me":10367,"iá":30,"mf":545,"mk":703,"ml":1350,"mi":7819,"mj":551,"mn":514,"mm":1678,"mp":1250,"mo":647,"mr":361,"mt":1727,"ið":22433,"ms":4462,"mv":384,"mu":3055,"my":2358,"p ":1704,"na":24623,"nb":713,"iþ":48,"nc":512,"nd":28901,"ne":5566,"já":2743,"nf":978,"ng":20121,"nh":697,"ni":18092,"nj":745,"nk":1747,"nl":1765,"nm":750,"fó":868,"fð":1006,"ju":3468,"fí":157,"jo":90,"fé":1623,"fæ":1715,"Dýr":35,"gæ":257,"kj":4569,"ki":11936,"kh":360,"kg":80,"kf":253,"ke":4269,"gá":474,"kd":187,"fþ":53,"kb":82,"ka":13629,"fú":58,"m ":40783,"fö":400,"gó":341,"Ice":47,"ky":1074,"gö":726,"ks":1875,"kt":2936,"gð":1600,"ku":9598,"kv":2762,"ko":4051,"gí":357,"kr":3178,"kk":5618,"kl":2343,"km":1291,"kn":2921,"li":12184,"lh":740,"lk":1973,"hæ":1097,"lj":3578,"há":1640,"le":15573,"ld":8186,"lg":1902,"lf":2514,"la":24339,"lc":60,"gþ":48,"lb":662,"gú":251,"n ":36944,"hr":2163,"hs":37,"dí":356,"hv":2491,"dó":1251,"hw":52,"ht":210,"hu":1451,"dæ":682,"hj":645,"hi":2023,"hn":524,"ho":1432,"hl":4630,"hm":61,"id":1046,"ic":1301,"ib":562,"dý":970,"ia":1044,"ih":1334,"ig":5591,"if":2163,"ie":864,"dö":206,"hy":512,"dú":164,"k ":6366,"iq":49,"ir":20666,"is":11681,"eð":11441,"it":9396,"iu":360,"iv":434,"eó":56,"ix":96,"ii":120,"ij":65,"ik":5708,"il":15253,"im":4416,"in":45324,"io":856,"ip":2056,"je":151,"fá":352,"ji":208,"iz":110,"iy":35,"l ":12639,"ja":12542,"tæ":3059,"xi":212,"xn":82,"xl":137,"té":125,"xp":31,"tí":3010,"tó":2681,"xt":373,"xu":39,"sö":1819,"só":1189,"ww":36,"sú":642,"z ":258,"sþ":235,"sý":1460,"xa":252,"xf":48,"xe":64,"tá":725,"wi":159,"sæ":968,"sé":1567,"wn":54,"wo":89,"sí":2536,"ws":87,"ró":3050,"rö":1573,"y ":1711,"rú":1794,"rý":331,"wa":277,"rþ":186,"sá":1042,"we":162,"ré":1220,"ræ":6045,"vi":9174,"rð":12661,"vu":610,"vr":778,"rí":6044,"vn":41,"vo":2712,"uz":51,"uy":35,"ux":78,"uv":609,"uu":36,"rá":5069,"ve":15820,"va":13225,"x ":601,"ui":242,"uj":51,"uk":1254,"ul":3466,"ue":357,"uf":941,"ug":3646,"uh":528,"ur":34056,"us":5972,"ut":4027,"um":24785,"un":17263,"uo":98,"up":2929,"ty":774,"tz":87,"pö":182,"tu":13668,"tt":12048,"tw":67,"pó":246,"tv":2024,"ub":622,"pý":31,"ua":273,"ud":463,"uc":259,"w ":311,"pú":114,"to":3449,"tn":2675,"Hví":124,"tm":438,"tl":2066,"ts":2200,"tr":5330,"pí":449,"tp":67,"tg":495,"tf":506,"te":7118,"pá":304,"td":76,"tk":513,"pæ":160,"tj":2442,"ti":20846,"th":1669,"v ":113,"tb":686,"tc":63,"ta":23969,"su":3147,"sv":4635,"ss":7478,"st":37992,"oð":819,"sy":891,"sl":8316,"sk":20851,"sn":2092,"sm":2339,"sp":2463,"so":2878,"sr":592,"sd":792,"nþ":148,"sc":305,"sf":1106,"se":20267,"sh":1736,"sg":382,"sj":2608,"si":7701,"rz":66,"nö":212,"u ":23701,"nú":1706,"sa":9556,"ný":623,"sb":788,"rr":3768,"rs":10257,"rt":4340,"ru":11212,"rv":1275,"rw":51,"nó":446,"ry":982,"ní":1001,"rp":1277,"ro":2458,"rn":7451,"rm":2415,"né":65,"rl":3496,"rk":5904,"rj":1626,"næ":1092,"ri":23025,"rh":1825,"rg":5415,"rf":4667,"re":10640,"ná":1602,"rd":1531,"rc":245,"mþ":95,"rb":1584,"mý":131,"ra":19142,"t ":24652,"mú":288,"mö":974,"mó":746,"qu":172,"mí":442,"mæ":806,"Dóm":48,"má":3190,"Hrú":49,"lý":968,"lþ":583,"s ":20225,"lú":270,"Hró":53,"py":298,"lö":2699,"pt":1531,"pu":2231,"pv":50,"ló":1489,"pp":4302,"lí":4068,"pr":1694,"ps":1356,"Hun":69,"Hum":33,"Hve":94,"Hva":208,"Hug":182,"yð":586,"xá":78,"Hri":58,"Hre":99,"Hra":144,"xí":115,"Hlí":31,"vö":1344,"zz":45,"Hor":96,"zi":76,"uþ":108,"ze":74,"za":123,"Hon":37,"Hol":265,"zu":44,"zo":77,"ví":5140,"How":29,"zk":225,"væ":2362,"vé":379,"yg":1822,"ye":103,"uá":36,"yf":2085,"yc":47,"tþ":77,"yd":149,"ya":182,"yb":37,"tý":358,"tú":766,"tö":3200,"yv":43,"yt":1440,"uð":6077,"ys":971,"Hof":73,"yr":6845,"yp":342,"yo":75,"yn":4442,"ym":390,"yl":1475,"yk":1688,"yj":1636,"uæ":73,"yi":92,"Arg":37,"Are":29,"Ara":128,"Arn":175,"Ari":100,"Alþ":272,"App":83,"Atl":210,"Ast":30,"Ass":31,"Art":46,"Aus":516,"Auk":30,"Aug":34,"Así":208,"Auð":65,"Óly":64,"Óma":39,"Óla":271,"Bak":55,"Bal":91,"Ban":640,"Bar":205,"Bas":64,"Ósk":37,"Afr":254,"Afg":30,"Air":29,"Akr":70,"Aku":116,"Ala":63,"Alb":62,"Alg":63,"Ale":88,"Alf":93,"Alm":57,"All":93,"Ame":231,"Ama":30,"Ang":85,"Ani":38,"Ana":32,"And":211,"Ant":112,"Ann":73,"Bur":43,"Bry":34,"Bru":46,"Byg":36,"Ãþr":48,"² ":107,"Brú":53,"Cal":54,"Cam":73,"Cas":34,"Car":150,"Cat":30,"Can":52,"Bea":37,"à ":621,"Bey":33,"Bet":31,"Ber":266,"Ben":118,"Bel":130,"Bei":52,"Ãs":264,"Ãr":416,"Bib":37,"Ãg":72,"Ãl":120,"Bja":184,"Bil":56,"Bis":37,"Bir":67,"CO ":32,"Bla":79,"Blö":85,"Bló":44,"Bre":500,"Bra":209,"Bro":90,"Bri":92,"Bjö":145,"Bol":62,"Boo":38,"Bor":456,"Bos":43,"Bot":40,"Blá":56,"Ól":368,"Óm":42,"Ód":30,"Ãþ":49,"Ãb":166,"Ãt":191,"Ãs":2281,"Ãr":171,"De ":44,"Dev":30,"à ":655,"Dei":35,"Del":38,"Den":31,"Dan":318,"Dar":51,"Æv":33,"Dav":95,"Æt":40,"Dag":64,"Dal":141,"Ãð":37,"Chr":82,"Che":50,"Chi":122,"ám":763,"án":1343,"áp":70,"Cit":32,"ái":182,"ák":1231,"ál":5705,"áe":70,"áf":560,"ág":530,"áh":466,"áa":140,"áb":249,"ád":63,"áv":511,"áu":67,"ár":6093,"át":2849,"ás":1809,"Þæ":99,"Þá":57,"Þý":326,"Þó":307,"æ ":266,"Cla":106,"Cen":41,"Þa":1137,"Þe":606,"Þi":250,"Þj":278,"Þo":417,"Þu":34,"Þr":200,"Þy":38,"Þv":74,"á ":17048,"Cha":121,"Cri":38,"Úr":59,"Út":101,"Úk":30,"Úl":56,"Ós":82,"Öl":88,"Ön":69,"Ör":97,"Ös":31,"Óð":61,"Öx":50,"Cor":85,"Com":79,"Col":95,"Con":100,"Cou":37,"ós":1395,"ót":2821,"アアア":152,"óu":215,"óv":375,"ö ":331,"ðá":72,"óa":487,"ób":406,"ój":46,"ói":177,"óh":245,"óg":351,"óf":1069,"óe":87,"ód":134,"ór":3904,"óp":1247,"ón":3499,"óm":3062,"ól":4371,"ók":2164,"íð":2645,"íó":153,"íþ":437,"ðr":2069,"ðs":3396,"ðp":30,"ðv":1344,"ðt":171,"ðu":11785,"ó ":1141,"ðl":1590,"ðm":400,"ðn":620,"ðo":29,"ðh":571,"ði":13616,"ðj":880,"ðk":295,"ðd":125,"ðe":430,"ðf":1240,"ðg":495,"ða":17064,"ðb":974,"ív":134,"íu":2135,"ín":2687,"ím":2131,"íp":113,"ío":119,"ír":336,"éð":84,"ít":1441,"ís":5447,"ð ":40406,"íg":419,"íh":106,"íe":115,"íf":1316,"ík":5310,"íl":945,"íd":173,"ía":767,"íb":632,"Egy":100,"Egi":67,"Eft":63,"í ":21136,"æð":6732,"Efn":44,"ék":185,"él":2012,"én":89,"és":167,"ét":1646,"ér":1779,"éu":87,"éb":41,"éf":124,"ég":31,"Edd":90,"Ef ":37,"é ":624,"æv":174,"æx":41,"æs":710,"ær":4290,"æt":3260,"æn":1633,"æp":103,"æk":1777,"æj":307,"æm":1338,"æl":1029,"æg":1139,"æf":427,"æi":109,"æd":792,"áð":1784,"áæ":82,"FC ":43,"ýð":730,"þá":1196,"þí":45,"þæ":401,"þé":152,"þó":497,"þú":323,"þý":829,"þö":79,"Dia":31,"Dis":97,"ýl":453,"ýk":129,"ýj":265,"ýf":75,"ýd":43,"ýa":30,"ýv":32,"ýs":2406,"ýt":324,"ýr":1984,"ýp":92,"ým":480,"ýn":556,"þj":1460,"þi":534,"þe":4740,"þa":4830,"þv":1549,"þy":303,"þr":1838,"úð":432,"þu":272,"þo":314,"ý ":120,"ün":52,"ür":71,"þ ":167,"Ð°Ñ ":31,"úp":309,"ún":2987,"úm":548,"úl":849,"úk":435,"úv":113,"úu":46,"öð":2279,"út":3137,"ús":2168,"úr":2334,"úa":1374,"úi":308,"úf":229,"úg":271,"úd":160,"úe":84,"úb":118,"óð":3019,"öx":90,"ør":44,"Dre":57,"Dra":80,"óþ":62,"ú ":1027,"Djú":38,"ðþ":39,"ðö":84,"öt":1074,"öu":59,"ör":3745,"ös":255,"öp":100,"ön":3508,"öl":4517,"öm":308,"ök":1689,"öf":2560,"ög":3309,"öd":29,"Don":53,"Nef":35,"Net":32,"Nes":42,"Nas":29,"Nat":57,"Nau":55,"Nic":32,"Neð":45,"Nin":66,"Nik":35,"New":193,"Myn":130,"Nar":29,"Naf":146,"Jök":86,"Jóh":104,"Jót":38,"Jór":29,"Jón":432,"Jól":33,"Njá":30,"OS ":34,"Nor":1398,"Not":36,"Kár":32,"Odd":60,"Oft":93,"Jör":55,"Júl":36,"Orð":204,"Kór":59,"Kóp":83,"Kól":42,"Oxf":36,"Kín":133,"Org":32,"Ork":38,"Pla":169,"Læk":37,"Pin":45,"Pho":31,"Phi":60,"Lár":54,"Per":153,"Pet":79,"Pen":61,"Pat":43,"Par":152,"Pau":49,"Pan":54,"Pap":35,"Pal":72,"Pak":41,"Lög":81,"Pro":96,"Pri":96,"Pre":57,"Líf":63,"Pol":31,"Pot":52,"Por":129," ال":82,"Mál":122,"Már":34,"Lýð":56,"SA ":48,"Múl":61,"Mús":34,"Raf":70,"Rad":29,"Rag":108,"Mün":41,"Ran":132,"Que":34,"Isl":57,"Jaf":30,"Jac":46,"Jar":64,"Jap":87,"Jan":110,"Jam":61,"Jak":50,"Jen":40,"Jer":68,"Jes":43,"Fél":151,"ã‚":61,"Fær":114,"Jos":55,"Jon":44,"ã‚¢":218,"Joh":167,"Jul":35,"Kam":65,"Kal":223,"Kap":39,"Kan":178,"Kau":134,"Kat":86,"Kas":91,"Kar":285,"Ker":45,"Ket":54,"Ken":76,"Kep":34,"Kel":35,"Kef":52,"Kir":145,"Kin":55,"Kja":80,"Kna":56,"Kle":57,"Kla":45,"LP ":162,"Kon":176,"Kom":44,"Kol":166,"Kos":29,"Kor":44,"Kr ":66,"Kjö":29,"Gís":50,"Gín":44,"Kra":61," ã‚¢":29,"Kri":234,"Kro":41,"Kyn":37,"Kyr":107,"Kró":49,"Ãbú":146,"Kve":46,"Kvi":48,"Ãsa":105,"Ãrl":73,"Hát":37,"Let":29,"Hás":150,"Les":33,"Ãta":188,"Len":37,"Hál":31,"Hák":37,"Lei":237,"Ãsl":2081,"Ãsr":41,"Lau":192,"Lax":122,"Ãra":76,"Le ":30,"Lag":87,"Lat":43,"Lao":32,"Lam":35,"Lan":545,"La ":46,"Hér":92,"ML ":30,"Lim":33,"Lin":79,"Lis":104,"Lit":139,"Lun":86,"Lou":37,"Los":48,"Lof":40,"Lon":229,"Ljó":82,"Hör":55,"NA ":58,"Hún":965,"Hól":152,"Höf":247,"Men":138,"Mel":83,"Mes":43,"Mer":93,"Met":61,"Meg":45,"Med":40,"Mex":54,"Hús":107,"Man":190,"Mal":118,"Mar":589,"Mas":67,"Mag":199,"Mad":42,"Mak":33,"Mai":40,"Mac":73,"Max":34,"Mat":90,"Mið":393,"Mol":36,"Mon":128,"Mos":92,"Mor":172,"Mot":32,"Mjó":31,"Mik":116,"Mic":123,"Með":113,"Mis":52,"Mil":62,"Min":98,"Mun":57,"Mus":51,"Ték":31,"Sýr":30,"Sög":42,"Sön":96,"Sók":35,"Sól":71,"Síð":58,"Wor":55,"Wil":110,"Win":83,"Wii":32,"Wei":29,"Wes":69,"Rús":196,"Rún":33,"Was":36,"War":47,"Wal":110,"Róm":176,"ék ":30,"él ":93,"éf ":34,"éfa":46,"Vol":51,"Rík":122,"Vis":32,"Veð":31,"Ráð":30,"Við":210,"ést":58,"étr":52,"étt":1263,"étu":131,"Vík":49,"Vín":30,"Vís":51,"Víe":38,"ékk":137,"éla":1571,"én ":35,"éli":43,"élt":37,"éra":473,"érh":132,"ét ":143,"érs":261,"éru":58,"éu ":77,"ér ":713,"élö":131,"之":47,"並":35,"三":94,"Yor":108,"You":39,"æð ":244,"Yfi":66,"æða":580,"æði":5160,"æðu":478,"æðs":139,"æðr":99,"Tóm":44,"Tón":101,"Töl":55,"Tím":60,"Svæ":31,"Sví":230,"Suð":653,"Stó":139,"Stö":59,"Stæ":32,"Þýs":303,"Svi":95,"Sve":238,"Sva":354,"Sum":80,"Sup":30,"Sun":82,"Sty":51,"Str":276,"Stu":161,"Stj":104,"Sti":42,"Sto":146,"Sta":383,"Spá":114,"Ste":372,"Ten":40,"Pál":100,"Tel":34,"Tan":53,"Tal":83,"Skj":90,"Ski":81,"Sko":140,"Skr":62,"Sku":31,"Ska":549,"Ske":116,"Shi":29,"Sha":56,"Sim":64,"Sil":74,"Sig":433,"Sin":41,"Ser":78,"Set":31,"Sey":30,"Sen":51,"Sel":128,"Sem":35,"Sei":38,"Seg":32,"Snæ":134,"Spa":50,"Skú":52,"Spi":49,"Spe":34,"Slí":40,"Slé":43,"Sló":37,"Smá":43,"St ":29,"Sjá":155,"Sjö":30,"Sjó":78,"Sno":80,"Ská":150,"Sog":33,"Sof":40,"Sou":36,"Sov":61,"Skó":140,"Sol":36,"Son":39,"Skí":37,"Sky":37,"Smi":30,"обл":31,"Sag":99,"Saf":35,"Sam":545,"Sal":86,"Sco":36,"Sch":99,"Sax":56,"Sav":41,"Sau":95,"Sar":46,"San":244,"Nýj":100,"SI ":80,"Rey":1063,"Nát":43,"Rit":75,"Ric":60,"Rau":132,"Mýv":29,"Mýr":98,"SG ":207,"Rei":54,"Reg":63,"Rob":50,"Roc":43,"Ros":47,"Rom":33,"áæt":78,"Ven":79,"Veg":63,"Vei":40,"Vat":207,"Van":47,"Val":271,"Var":102,"Vig":35,"Vir":57,"Vil":122,"Vik":42,"Vin":79,"áð ":233,"Ver":208,"Ves":624,"Vet":35,"Um ":130,"áða":140,"áðg":34,"áðh":171,"áði":225,"áðu":798,"áðs":118,"Und":51,"Ung":70,"Uni":181,"Upp":182,"æg ":91,"ædd":776,"æfi":141,"Pól":106,"æfe":104,"æfa":50,"Tró":34,"Trö":38,"Tyr":85,"ægj":69,"ægi":110,"æga":135,"ægr":102,"ægt":328,"ægu":162,"æju":29,"æfð":38,"Tví":41,"æin":69,"æja":272,"æl ":48,"æli":334,"æll":56,"ælt":61,"ælu":92,"æma":80,"æmd":120,"æki":736,"ækj":217,"ækk":69,"ækn":356,"ægð":100,"ækt":237,"æku":92,"æn ":110,"æla":272,"æld":77,"ænu":227,"ænt":81,"æns":354,"ænm":30,"ænl":161,"ænn":155,"æni":89,"æng":118,"ænd":99,"æna":190,"æmu":71,"æmt":313,"æmi":690,"æpl":30,"ær ":1321,"æsi":41,"æsl":32,"æsk":45,"ærr":162,"ærs":892,"ært":38,"æru":151,"æra":217,"æri":815,"ære":175,"ætt":2050,"ætu":294,"æta":147,"æti":456,"ætl":282,"æst":554,"Tex":43,"Ter":34,"ævi":136,"æva":29,"ærð":475,"Tha":29,"The":298,"æxl":41,"Tho":106,"Til":161,"Tim":38,"Tin":49,"Tit":29,"Pét":113,"Tor":78,"Tom":37,"Tos":37,"Tjö":35,"Try":40,"Tro":45,"Tri":44,"Tre":49,"Tra":74,"Tur":46,"Tun":86,"Þrí":29,"Þrá":37,"Þve":45,"ã‚ã‚":41,"áum":45,"átu":169,"bju":54,"átt":1937,"bja":121,"áti":93,"átr":47,"bis":218,"bit":57,"áss":31,"ást":312,"bir":182,"bik":50,"bil":637,"bin":115,"áta":175,"ble":145,"áve":79,"árá":48,"bli":128,"árð":42,"bla":414,"áva":365,"bol":146,"bog":60,"átö":52,"bjö":156,"bjó":189,"átí":162,"ásö":38,"blí":66,"bló":334,"ávö":33,"blö":167,"blá":155,"blæ":36,"bon":36,"bor":2681,"bot":163,"álk":336,"áll":88,"álm":334,"bbi":37,"álp":80,"áls":457,"álu":194,"álv":69,"bba":96,"be ":48,"áma":116,"bbu":35,"ámi":57,"bam":50,"ban":1603,"bak":368,"bal":62,"ákn":429,"bah":47,"áko":53,"áku":34,"bac":32,"bad":40,"áks":66,"ákv":456,"án ":338,"ála":688,"ágú":123,"álf":1089,"bau":182,"ále":131,"bat":32,"áld":466,"bas":167,"bar":589,"áli":555,"álh":106,"álg":55,"áns":123,"ánn":67,"ánu":150,"áms":328,"ámu":55,"ánd":108,"ána":282,"áni":194,"ás ":143,"álí":29,"bi ":52,"bei":420,"ár ":710,"ber":1931,"ben":97,"bel":260,"bek":34,"bey":64,"álæ":106,"bes":144,"bað":32,"bet":171,"ása":364,"árv":81,"áru":387,"árr":36,"árs":356,"ásk":711,"ási":75,"bib":31,"áse":31,"bif":37,"át ":80,"árm":104,"árn":118,"árk":58,"árl":107,"ári":3035,"árg":32,"árh":61,"áre":46,"árf":67,"árd":141,"ára":516,"áfa":149,"ábó":66,"áfu":225,"áfe":61,"áfr":66,"áhe":82,"áha":37,"ágs":40,"ágr":106,"águ":57,"ca ":193,"áhu":51,"áhr":211,"car":60,"cas":39,"cat":48,"can":86,"ál ":816,"áir":32,"cal":71,"cag":30,"áin":106,"ce ":210,"áka":90,"ám ":162,"brj":92,"bri":289,"bro":253,"bra":622,"bre":1260,"bry":35,"bu ":44,"bru":150,"boð":197,"bur":454,"bun":269,"bus":56,"áa ":42,"brá":62,"bræ":116,"bré":110,"bró":94,"brö":90,"by ":34,"brú":361,"áan":33,"áar":36,"ábe":37,"áby":43,"byl":138,"byg":1051,"bys":77,"ábr":30,"byr":205,"aka":724,"am ":1285,"ake":553,"aki":430,"afæ":30,"afé":292,"afð":193,"afí":36,"afó":34,"al ":1641,"aja":65,"ail":51,"ain":143,"air":55,"ais":50,"ak ":482,"ahy":29,"Þór":281,"adó":89,"adý":30,"ahl":129,"adæ":81,"ahv":94,"adí":88,"ahr":268,"aho":81,"agv":46,"ahe":110,"aha":506,"agj":32,"agk":31,"agl":139,"agf":101,"agg":36,"agi":919,"agr":297,"ags":903,"agt":139,"agu":274,"agn":1177,"ago":116,"anv":402,"anu":572,"ajö":66,"ano":128,"ann":7640,"anm":237,"ant":646,"ans":2276,"anr":62,"ane":475,"anf":63,"ang":1850,"anh":50,"ani":508,"anj":69,"ank":389,"anl":325,"ap ":71,"ana":1067,"anb":53,"anc":186,"and":12634,"amu":36,"amt":977,"amv":106,"amy":55,"amm":631,"aml":867,"amo":77,"amn":326,"amp":188,"ams":715,"amr":194,"amk":501,"amh":353,"ami":326,"amf":296,"amg":65,"amd":67,"ame":964,"amb":945,"ama":1875,"ahú":86,"ahö":183,"aly":37,"alv":169,"alu":337,"alt":530,"als":1039,"alr":40,"alp":48,"alo":66,"aln":159,"alm":385,"all":4558,"alk":134,"alg":346,"alh":154,"ali":1259,"ald":2532,"ale":741,"ahá":51,"alf":139,"ala":1875,"alb":102,"agö":58,"an ":5852,"agó":47,"akv":56,"aks":182,"akr":86,"aku":48,"akt":192,"agð":277,"ako":186,"akk":529,"akj":37,"akm":75,"akl":519,"aba":247,"abb":91,"abe":147,"abi":501,"abj":35,"abl":134,"abo":208,"abr":215,"abu":41,"aby":70,"ae ":196,"aaf":34,"aal":29,"aar":39,"ad ":170,"ac ":69,"afn":2524,"afm":183,"afo":148,"afr":1163,"abí":243,"aft":662,"afs":676,"aff":148,"abá":34,"afe":261,"afh":109,"afj":658,"abæ":120,"afi":1278,"afl":1145,"afk":73,"ai ":55,"abú":61,"aga":1887,"agb":56,"age":255,"afu":287,"abó":285,"afy":94,"ael":113,"aef":43,"aei":56,"ah ":67,"afa":1337,"afb":140,"aey":189,"ado":94,"adr":85,"adi":100,"add":86,"ade":268,"ag ":1083,"ads":38,"aco":29,"ack":101,"aci":77,"ach":162,"ace":117,"ada":509,"af ":5722,"act":62,"avé":39,"aví":1064,"avö":101,"atæ":44,"axi":148,"axl":39,"axn":54,"atí":446,"Þát":29,"axt":48,"asó":42,"asö":150,"az ":37,"axa":101,"asý":141,"atá":67,"auð":861,"atö":90,"ató":78,"aya":40,"aye":40,"ba ":118,"bb ":35,"axá":78,"Þær":73,"amó":187,"amí":123,"amú":39,"at ":305,"amö":61,"arh":891,"arg":845,"arf":2166,"are":429,"ard":705,"amþ":87,"arc":90,"arb":562,"ara":2184,"arp":734,"aro":178,"arn":1542,"arm":909,"arl":1420,"ark":1210,"arj":31,"ari":2716,"aru":75,"arv":495,"anó":79,"aní":198,"arr":610,"ars":2758,"art":990,"au ":547,"anú":173,"asa":1107,"ary":75,"asi":329,"ash":111,"anþ":38,"ase":151,"aso":291,"asn":37,"asp":70,"ask":904,"asj":57,"asm":56,"asl":51,"akí":50,"aos":34,"akó":58,"ar ":24659,"apa":379,"ape":84,"alæ":30,"api":60,"aph":36,"apl":48,"apo":29,"app":288,"alí":469,"apr":187,"apu":54,"aló":52,"alö":117,"as ":455,"alþ":280,"alý":31,"amá":227,"amæ":216,"ava":456,"ax ":56,"aut":633,"arð":2612,"arí":1423,"avo":132,"aré":99,"avn":36,"avi":270,"aræ":58,"ave":508,"ará":337,"arþ":116,"arú":66,"ay ":120,"awa":61,"arö":154,"aró":106,"así":119,"atb":114,"ata":1386,"asu":87,"ast":4650,"ass":502,"asy":41,"asv":193,"atn":1309,"atk":61,"atl":50,"atr":203,"ato":180,"ate":387,"atf":38,"ati":491,"ath":330,"att":952,"ats":86,"atv":191,"atu":294,"apö":55,"aul":90,"aum":222,"aun":895,"aup":528,"aur":125,"aus":1761,"auf":120,"aug":671,"auk":646,"ος":71,"ος ":71,"Ï‚ ":149,"ν ":35,"Víð":50,"α ":38,"Völ":29,"アア":185,"Það":730,"Þet":130,"Þes":144,"Þei":197,"Þeg":77,"Þin":237,"Þor":406,"Þjó":274,"Þau":111,"Þar":262,"jen":38,"fán":134,"fás":32,"jað":148,"ji ":76,"jad":61,"jaf":886,"jab":64,"jat":92,"jas":441,"jav":807,"jap":160,"jar":3369,"jal":1563,"jak":139,"jan":1062,"jam":60,"jah":111,"fá ":59,"fél":1458,"fék":83,"fæð":108,"Ñк":66,"fé ":69,"fæt":90,"fæs":142,"fær":603,"fæd":750,"ÑÑ‚":78,"itn":80,"eðn":127,"itm":37,"itl":270,"eðl":410,"eðr":140,"itr":160,"ito":53,"itv":30,"eðv":33,"eðu":188,"itu":683,"itt":1458,"its":233,"eðs":115,"itz":45,"ity":146,"isk":905,"isj":46,"ism":523,"isl":359,"iso":62,"isn":159,"isp":55,"iss":785,"isr":186,"isu":36,"ist":4346,"isv":198,"eða":6425,"ita":1646,"ite":114,"eðf":92,"itg":57,"ith":325,"eði":184,"iti":2561,"eðj":70,"irð":784,"irí":91,"isá":31,"ius":175,"ium":116,"iva":54,"ix ":65,"ivi":61,"iræ":48,"ive":235,"ilí":122,"ipp":147,"ipu":275,"ips":70,"ipt":795,"ipi":107,"ipl":30,"ilö":36,"is ":1869,"ion":501,"ior":56,"ipa":275,"ipe":46,"ir ":13932,"iru":61,"irv":49,"irs":247,"irt":855,"iní":40,"irr":770,"iro":49,"irm":92,"irn":320,"irk":1284,"irl":451,"iri":354,"isi":625,"ish":290,"isg":63,"isf":275,"ise":129,"isd":44,"isc":101,"isb":133,"isa":319,"iu ":46,"iqu":44,"irf":191,"ire":187,"irh":41,"irg":228,"irb":292,"ira":203,"eð ":3577,"it ":1254,"ünc":35,"ja ":3453,"ití":89,"isú":45,"iza":39,"kif":29,"kim":46,"kil":1176,"kk ":327,"úse":29,"úsa":165,"úsd":34,"kie":30,"úsn":29,"úsi":225,"úsk":37,"úsl":41,"kin":2085,"úsu":332,"kip":1436,"kir":1170,"kis":1049,"keð":39,"úss":498,"kit":45,"úst":268,"úta":123,"öði":221,"úti":93,"úth":65,"kja":2742,"útg":318,"útf":37,"gæf":29,"útd":41,"útb":80,"útl":88,"öðl":46,"km ":337,"öðv":305,"útv":167,"kju":1220,"öðu":713,"útu":109,"útr":45,"gæs":64,"öðr":512,"gæt":90,"úts":144,"úpv":33,"ús ":380,"ki ":3751,"kgr":34,"khl":111,"út ":1602,"öð ":459,"úra":87,"kha":33,"úrg":32,"úrf":29,"úre":62,"úrk":41,"kdó":160,"úrl":34,"úrs":50,"úní":148,"kho":40,"úrv":36,"úrt":29,"úru":307,"kef":136,"gáf":290,"únu":54,"keg":32,"kei":325,"kem":386,"kel":195,"ken":1184,"kep":493,"kes":42,"gár":91,"ker":1044,"gát":46,"ket":42,"kað":419,"key":170,"kfa":29,"úpa":59,"úr ":1479,"kaþ":93,"úpu":52,"úps":35,"kfr":105,"úlí":196,"úpi":33,"ke ":136,"úlu":109,"úml":92,"úmm":51,"úme":150,"úmi":46,"úp ":32,"úna":560,"úms":48,"úið":119,"únn":35,"úni":599,"kra":945,"kre":121,"kt ":1361,"gð ":585,"kmö":30,"gíu":82,"gít":108,"gís":71,"gíg":49,"kse":36,"ksf":65,"kry":53,"ku ":2256,"knú":46,"kro":107,"kru":154,"kri":1166,"kkó":35,"km²":99,"kot":277,"kos":362,"kor":407,"kop":46,"kon":1246,"kom":1243,"kol":168,"ks ":435,"klú":36,"kló":34,"klæ":58,"kið":1002,"kmy":402,"úsí":34,"kna":1143,"kme":201,"kmi":111,"gæð":40,"knu":434,"kjó":69,"kjö":344,"útí":111,"kob":49,"kjá":188,"kne":96,"kni":971,"kkv":35,"kku":899,"kkt":826,"kks":204,"kkr":232,"úum":39,"kn ":153,"kke":95,"kkb":29,"kka":637,"kkn":107,"kko":71,"kkl":305,"kkj":140,"kkh":101,"kki":1529,"klu":225,"khó":38,"khú":66,"ko ":34,"kma":92,"úrí":36,"kle":411,"úve":99,"kla":939,"klo":62,"kli":470," Ãl":120," Ãg":72," Ãr":415," Ãs":264,"fór":224,"fót":90,"fós":48,"fón":31,"fól":411,"jus":220,"juv":64,"jul":273,"jun":745,"jum":817,"jur":390," à ":609,"fðu":130,"jub":47,"juh":41,"jug":89,"juf":31,"jud":50,"fða":298,"fði":429,"fðb":90,"ju ":578,"fín":37,"fð ":55,"kbr":32,"kav":64,"kat":183,"kau":363,"kar":1751,"kas":646,"kap":274,"kan":720,"kal":1475,"kam":694,"kak":79,"kah":74,"kaf":322,"kag":595,"kad":110,"kab":52,"föð":56,"fús":45,"ka ":5563,"fóð":29,"för":70,"fös":57,"föt":31,"fön":37,"föl":135," Ga":560," Bú":143," Ge":549," Fy":316," Bó":133," I ":60," Bö":36," Bí":44," Fo":401," Fu":126," Fr":1133," Fi":346," Fl":547," Fj":284," Bæ":200," Ha":2193,"gyð":102," He":1194," Gy":51," Go":223," Gr":964," Gu":613," Gv":43," Gi":180," Gj":38," Gl":210," Gn":42," Ic":56," Dý":44," K ":33," Dö":29," Hv":447," Dó":65," Hu":336," Hr":478," Hn":67," Ho":620," Hl":320," Dæ":71," Hj":179,"ha ":73," Hi":343," Fá":50," Je":228," L ":32," Ja":516," Is":113," Im":35," In":759," Il":85,"ham":273,"han":2326," M ":34," Fö":37," Ka":1198,"hal":899,"hau":175," Ke":368," Ki":275,"har":340," Kj":147,"has":39,"hat":96," Fé":158," Fæ":127," Fí":29," Jo":341,"haf":2871," Ju":93,"hae":47,"hag":381," Fó":49," N ":56," La":1289," Há":323," Le":588," Li":537," Lj":87," Hæ":71," Kl":180," Kn":101," Ko":532," Kr":556," Gí":124," Kv":148," Ku":90," Gö":52," Ky":159," Mc":44," Ma":1610," Hú":1094," O ":35," Mi":927," Mj":65," Me":769,"he ":479," Hí":33," Lo":529," Hé":115," Hö":355," Ly":57," Hó":219," Lu":199," Já":37," Ne":513,"а ":49," Na":464," Nj":57," Ni":233,"hf ":56," Ið":49," Mo":559," My":192," Mu":180,"hel":997,"hei":4035,"dái":46,"heg":81,"hef":1628,"heb":56,"hey":346," A ":141,"het":78,"hes":156,"her":942,"hen":1048,"hem":44,"hi ":50," B ":65," C ":121," Ap":140," Am":374," An":597," Ak":256," Al":992," Ai":48," Ag":60," Af":348," Ac":57," Ad":101," Aa":38," Ab":95," Ba":1274," D ":30," Av":40," Au":690," At":285," As":369," Ar":655,"hig":37," Be":741,"hic":69," Bi":297,"hia":38,"hip":34," Bj":353,"hin":765," Bl":363,"him":136,"hil":183," Bo":816," Br":1093," Bu":149,"his":35,"hit":297," By":64,"hir":173,"hja":45," E ":35," Ca":484,"dæg":46," Ce":87," Ci":85," Ch":431," Cl":177," Cr":127,"dæm":557,"dæl":69," Co":494," Cu":55," F ":53," Da":794," Di":273," Dj":72," De":342," Dr":200," Do":188," Dy":59," Du":120," Ea":43,"hn ":90," Ed":147," G ":36,"hla":226,"hle":106," El":310," Ek":54," Ei":508,"hli":228," Eg":213," Ef":192,"hlj":1680," Et":30," Að":243," Es":91," Er":231," En":509," Em":78,"hlu":2009," Ey":536," Ex":43," Eu":55," Ev":632," Bá":77,"ho ":29," Fe":260," Aþ":47," Fa":298," H ":70,"gma":107," Tó":200,"go ":107,"gme":79," Tí":126," Té":39,"glu":412," Tæ":41," Tá":38,"gls":98," Sý":65,"gle":410," Sú":125," Só":162,"gli":346,"glj":57," Sö":178,"gn ":453,"gla":791," Sí":179," Wo":94,"ggð":568," Sé":40," Sæ":89," Wi":308," Wh":41," We":157," Sá":78,"gko":58," Wa":263," Rú":290,"й ":44," Rö":41," Ró":242,"gog":30," Vé":39,"gjö":146," Ví":264,"gnu":408,"gnv":46,"gns":98," Ze":49,"gni":304,"gnh":31," Za":29,"gnf":78,"gjá":29,"gne":63,"gna":1067,"úa ":496," Yo":162,"gmy":138,"gið":678," Ya":33," Yf":65," Tö":59," Tú":51,"glö":41,"úbb":42,"gs ":890,"gmá":109,"glý":46,"glæ":66,"úar":691,"glí":50,"úaf":94," Vö":72,"gon":51,"gos":180,"gor":133,"got":57,"údd":41,"gsb":67,"úde":39,"gsa":85,"gu ":2071,"gnú":139,"gsk":208,"gsh":92,"gsi":265,"gsf":96,"úda":42,"gsd":42," a ":135,"gse":69,"gro":36,"gru":389,"gra":798,"gt ":1312,"gri":532,"grj":36,"gre":1498," R ":43," Kú":67,"úel":31," Kö":70," Kó":213," Ox":46,"gto":58," Os":65," Ot":31," Kí":164," Or":369," Op":55," Po":343,"guf":112," Pl":227,"guh":41," Læ":47,"gum":2005," Pi":180," Ph":118,"gul":515," Lá":110,"gua":30," Pe":377,"gue":40," Pa":536,"gsv":75," Jö":155,"gst":295,"goð":247,"gsu":50," Jó":679,"gsl":236," Nu":48,"gsm":187,"gsr":110,"gss":198," No":1523,"gsp":47," Ol":55," Ok":35," On":42," Oh":30," Od":79," Of":128," Ká":86," Jú":86,"gta":478,"gy ":31,"grú":33," Mö":79,"gró":151," Mü":48," Ra":615," Mú":128,"grö":62,"úgu":29," Mí":43," Mó":77," Qu":74,"grí":914," Næ":38," Ní":89," Ro":344,"gsæ":96," Re":1399," Ná":93," Mý":140," Ri":247," Rh":36," Lö":101," S ":42," Lú":62," Lí":171,"gur":4137," Pr":333,"úfu":85,"gus":223," Pu":82,"gun":1431," Ló":43," Mæ":40,"gré":42,"úi ":32,"græ":199,"gvi":54,"úga":156," Lý":96,"grá":134,"gve":169," Má":195,"gva":364," Sy":96," Sv":974," Su":940," St":1837,"gtö":37," Ta":373," Tj":67,"úfé":45," Th":487," Ti":313," Pá":120," Te":268," Tr":405," Pí":33," To":304,"gyp":133," Pé":116," Nó":36," Ru":106," Nú":101," Ný":191," Sa":1402," Sh":147,"úin":138," Si":722," Sj":303," Sc":205," Se":571," So":351," Sp":370," Sk":1509," Sl":218," Sm":127," Sn":277,"gvö":64,"ún ":1473,"úla":181," Va":709,"úkr":68," X ":47," Rá":45," Ve":1143,"úku":33,"úlk":110," Vi":651,"úll":94," Ré":29," Rí":181," Vo":162,"úlf":51,"úli":36," Tu":190," Pó":142," Tv":79," Ty":102,"guð":201,"úm ":49,"úkl":39," Ul":39,"gví":39," Um":208,"úkd":157," Un":333,"úka":42," Up":196,"gvé":77," Ur":46," Ut":32," ja":1316,"ÑŒ ":33,"iam":73,"ial":59,"ian":234,"ias":35," fá":200,"iat":58," in":1466," il":75,"ic ":168," is":51," eð":5981,"ibl":77,"ibj":53,"ibi":39," ka":1977,"dýp":59," fö":181," m ":286," gæ":107,"dýr":894," kj":379,"ibr":56," ki":467," gá":35," ke":1446,"ibu":44," fí":60," fæ":1198," fé":473,"id ":120," fó":715,"iba":30,"ibe":53," ju":184," gy":80," ha":4851,"dún":44," he":6396," dá":76," gi":256," gj":246," gl":267," gr":1982," go":395,"ia ":441," gu":299," hy":49," dö":163," dú":43," dý":362," hi":1085," dæ":326," hj":450," hl":3298," hn":173," ho":514," hr":1011," hu":968," hv":1445," dó":235,"iet":44," ni":345,"iel":71," ne":1620," já":131,"ien":104," na":898,"ier":105,"ies":63,"ied":45,"ief":45," my":1128,"iei":96," mu":272," ið":125,"ig ":2070," mo":182," mm":37," ok":208," ol":65,"ifu":195," og":20861," kæ":55,"ifo":174," od":33," ká":29," of":1574," jú":303,"ifs":149,"ift":225,"ifr":86," jö":359,"iff":48,"ife":72,"ifl":73," jó":154,"ifj":45," ny":134,"ifi":210," nu":54," no":3458,"ifa":566," le":2444," há":1191," lj":434," hæ":774,"icr":57,"ics":37,"ict":48,"icu":82," li":1725," n ":78,"ico":79,"ick":89," la":3474," kv":906," ku":88,"ici":64,"ich":282," gö":256," ky":352,"ice":85," gó":105," kn":246," km":437,"ie ":195,"ica":290," kl":476," kr":671," gí":129,"iby":31," ko":2181," me":6197," mi":3548," mj":459,"Ñ ":46," o ":46," hú":877," ma":2672," hý":48," lu":44,"idi":32," hó":647,"idg":43," ly":173,"ide":137,"idd":384," hö":1280," hé":535,"ida":195," lo":635,"if ":187," af":6312," ag":34," ab":43," am":104," an":1387," ap":172," ak":50," al":2511," au":1349," ar":311," at":450," as":67," d ":204," ba":1579,"il ":5889," bi":592," be":1105," bo":1713," bl":752," bj":275," by":1013," bu":123," br":1739," ca":62," e ":310,"im ":624,"ika":773,"ige":85,"ibý":65,"iga":307,"ii ":75,"igl":259,"igh":129,"igi":601,"igf":50,"igg":668,"igu":503,"igs":49,"igr":192,"ign":307," b ":35,"ihe":1081,"iha":82,"ihl":41,"ik ":458,"imo":50," er":28966,"imn":59,"imm":338," að":9614," et":46,"iml":33," en":4649,"ims":1087," em":145,"imp":63,"imf":72," ei":5897," el":620,"ime":190," ek":1007,"imk":47," ef":2947," eh":31,"imi":735," eg":82,"ip ":220," bá":176," fe":1285,"inc":97,"ind":1888,"ina":1616," fa":994,"inb":291," ey":827,"imu":329," ev":138,"imy":162," fu":598,"inn":11792,"inm":78," fr":7589,"ino":134,"inr":53," bí":110," fo":1797,"int":511,"ins":4795,"inf":252,"ine":220," fl":2079,"inh":343,"ing":10532," fj":2182," bæ":1033,"inj":88," fi":969,"ini":973,"inl":365,"ink":753," ge":3864," ga":1049," bý":113," bú":500,"inu":3925," i ":55,"inv":49," bö":146,"iny":29," fy":4531," bó":727,"iko":33,"ikn":392," cm":36,"ikm":465,"ikl":338," co":79,"ikk":205,"ikj":227,"iki":1110,"ikh":77,"ikf":51," ch":40,"ike":140,"ila":590,"ilb":133," f ":217," da":1144,"in ":6373,"ikv":111,"igð":208,"ikt":88,"iku":599,"ikr":111,"iks":474," do":59,"ilo":79,"ilr":73,"ill":2619,"ilk":107," dr":748,"iln":105,"ilm":73,"ilh":348,"ilg":369,"ilj":142," de":781,"ili":745,"ild":1200," dj":84," di":134,"ilf":193,"ile":949,"ima":446,"imb":81,"io ":167," dv":45,"ily":95," du":109,"ils":420,"ilt":422,"ilu":204,"ilv":300," dy":47,"hlý":34," vö":461,"hlí":219," vé":126," væ":160," ví":914,"ла":36,"hol":488,"hom":68,"hon":246,"hou":37,"hop":49,"hor":323," yn":69,"hjó":152,"ка":44,"dí ":50,"hjú":47," yr":29," yt":52," ys":51,"hni":63," tú":84," tö":739," yf":1550,"hno":44,"hns":33," tæ":256," tó":1014,"hna":38,"hne":91,"hjá":398," tí":1122," sú":468," sö":1067," só":322," tá":346," sý":615,"hið":78,"hug":970,"ра":31,"dó ":54,"оÑ":32,"ов":39,"hnú":49,"об":32,"hru":56,"hry":131,"hní":30,"hro":60,"hre":864,"hri":511,"ht ":90,"на":33,"hra":356,"dís":130,"díu":40,"dín":33," ru":152," ry":31," nó":204," u ":29," nö":39," sa":4334," ný":430," nú":672," se":17046,"hyg":213," sj":1921," si":1522," sn":851," sm":466," sl":415," sk":4722,"hyl":69," sp":941," so":265," mí":73,"dót":514,"dór":123," mó":335," mö":457," t ":217," mú":64," ra":1335," mý":45," ná":971," re":1271," ri":857," næ":757," né":32," ro":246," ní":127," ló":69,"hvo":342,"hrí":33," pu":136," pr":628," lí":1770," lú":49," s ":184,"бл":47," lö":1089,"hy ":40," má":1082," lý":420,"dóe":37," mæ":323,"dón":58,"dóm":399," os":39,"hum":74,"hun":188,"hus":67," op":322," kí":261,"hur":44," or":937,"ан":42," kú":89," kó":124," kö":324," pe":303,"hva":437," lá":392,"hve":1438," pa":204,"hrá":47,"аÑ":31," ký":34," lé":191," pl":325,"аÑ":31," po":204," pi":60,"hræ":36," læ":234," rý":32," rú":443," sá":460," ró":306," rö":312," sí":1634," sæ":382," sé":1534," x ":48," va":8042," ve":5521," rá":293," vo":1291," rí":972," vi":6117," ræ":385," ré":421,"hvö":38," pö":85," ty":60," pó":129," tv":1119," tu":588," ut":212," ur":156," up":1941," um":4447," un":1589," ul":34," ta":1361,"hyr":120," sy":488," st":9503," sv":2024," su":1520," pí":89," tr":698,"dök":34,"hví":219,"dön":95," to":200," th":207," ti":6298," tj":75," te":2098," pá":51,"dög":48,"óðv":149,"óðu":414,"ffr":124,"fi ":1557,"óðm":36,"óðr":133,"óðs":215,"óðe":41,"ffe":40,"óðf":210,"óðg":53,"óðh":71,"óði":237,"ffa":53,"óðk":45,"óðl":242,"óða":595,"ffi":118,"óðb":40,"fet":34,"fað":546,"bát":196,"fes":284,"fer":1310,"fey":29,"feb":142,"fen":195,"bál":229,"fel":818,"fei":90,"fhe":36,"fhl":44,"aþó":58,"bæ ":192,"fga":65,"aþá":42,"fge":31,"fbr":125,"fbe":30,"fas":294,"fat":65,"far":1064,"fam":53,"fan":667,"fal":741,"fah":32,"fag":101,"faf":45,"óð ":473,"ff ":51,"aþj":48,"aþi":69,"eyð":283,"exí":65,"fa ":1735,"aút":75,"ext":253,"eyr":688,"eys":419,"eyp":145,"eyn":309,"eym":192,"eyj":1341,"eyk":1077,"eyi":44,"eyf":333,"eyg":150,"esú":56,"exa":95,"ez ":34,"eyt":859,"aöl":43,"etb":29,"eta":600,"aða":2137,"aðf":330,"ete":97,"aðe":248,"etj":138,"eti":412,"aði":1858,"eth":57,"aðh":97,"aðg":231,"aðn":42,"etn":430,"aðl":109,"etl":247,"esp":60,"eso":32,"est":3452,"ess":1992,"eum":32,"eto":75,"aðr":337,"etr":663,"ets":81,"aðs":858,"ett":1496,"etu":910,"aðu":2529,"aðv":31,"ew ":192,"eve":76,"erá":64,"eva":56,"evo":39,"evi":74,"eus":66,"ex ":266,"esí":96,"eró":30,"erí":413,"evr":168,"erð":2864,"ey ":517,"erú":35,"epi":31,"eph":64,"er ":26993,"epa":67,"eor":111,"eon":46,"aíd":31,"es ":1150,"ept":231,"epl":35,"epp":1389,"elí":74,"erk":1822,"erl":415,"eri":1572,"erj":619,"erg":583,"erh":55,"ere":120,"erf":1459,"erc":66,"erd":76,"era":1234,"erb":160,"et ":347,"að ":14115,"emí":69,"aín":58,"esj":33,"esk":983,"esf":32,"esh":83,"esi":419,"esb":59,"esc":36,"ese":247,"eu ":59,"esa":148,"ery":35,"erv":161,"eru":4787,"err":409,"ení":97,"ert":870,"ers":1962,"ern":801,"erm":315,"erp":109,"ero":113,"eki":1159,"ekj":180,"ekk":2070,"ekn":178,"eks":107,"ekt":94,"egð":96,"eku":210,"en ":4249,"ela":212,"eld":1621,"elf":107,"ele":208,"eli":198,"elj":279,"elg":458,"elm":97,"ell":1350,"elo":54,"elp":39,"elu":227,"els":1013,"elt":302,"eo ":32,"emb":731,"ema":216,"eme":166,"emd":57,"emm":172,"emo":53,"emi":212,"eið":2061,"emu":458,"emp":64,"ems":115,"enf":36,"ene":240,"enh":41,"eng":2379,"enb":71,"ena":189,"end":2651,"enc":88,"eno":72,"enn":4534,"enk":42,"eni":177,"enj":285,"enu":71,"ens":3046,"ent":778,"enr":50,"aí ":153,"enz":239,"egj":131,"egl":570,"ego":116,"egn":789,"ege":76,"egg":655,"egi":1272,"egr":290,"egs":216,"egt":454,"egu":1949,"egy":33,"ehf":31,"ek ":94,"eip":47,"eis":901,"eir":2011,"eim":2679,"eil":1098,"ein":8206,"eik":3123,"eid":336,"eig":1161,"eif":473,"el ":631,"eit":4783,"efð":193,"aæt":246,"efá":91,"eka":127,"em ":15176,"gfú":42,"ötu":646,"ött":156,"ötn":84,"gju":374,"ötv":30,"gfé":47,"ötl":36,"gl ":222,"öst":120,"gja":1901,"geð":49,"git":30,"gis":534,"gir":792,"ösu":31,"gil":555,"gim":55,"gin":2704,"ösk":74,"gib":68,"gdý":42,"gif":68,"örl":66,"örn":587,"örp":62,"örs":47,"öru":372,"ört":84,"örv":91,"öry":39,"ght":99,"öt ":64,"gho":33,"örd":65,"ghe":48,"örf":205,"örg":409,"gha":53,"öri":68,"örk":384,"ggv":60,"ggu":598,"ggt":85,"ggs":75,"ggl":31,"ggj":980,"ggi":554,"ggd":40,"gge":45,"gga":155,"gfr":149,"öpu":50,"öpp":29,"ölæ":50,"gi ":2174,"gbú":105,"gfa":30,"gaþ":60,"gfe":34,"ör ":105,"gfi":32,"gen":859,"get":826,"gað":150,"ger":1807,"ges":57,"gey":329,"gh ":58,"gaö":31,"gg ":105,"geb":47,"gei":373,"geg":382,"gef":1039,"gel":153,"gek":42,"gdr":32,"öns":279,"önt":102,"önk":81,"gdu":56,"önn":762,"gda":103,"önd":1211,"öng":1010,"gdi":40,"ömu":145,"ömm":54,"búð":112,"öml":44,"ge ":230,"ölv":557,"öku":681,"ökv":108,"öld":1331,"gd ":235,"ölb":96,"ölm":167,"öll":1141,"ölg":41,"ölf":107,"ölu":629,"öls":201,"býl":253,"gbl":50,"gbr":62,"býr":47,"öln":31,"gab":121,"gad":130,"öfð":390,"gaf":494,"gag":255,"gah":95,"gaa":32,"gas":712,"gar":5221,"öki":70,"gau":40,"ökl":108,"ökk":171,"gat":286,"ökf":51,"gav":94,"gak":142,"ögð":187,"gam":445,"gal":380,"gan":1061,"gap":30,"öl ":78,"búd":35,"búa":689,"ga ":4778,"ök ":413,"bús":74,"bún":284,"búi":183,"ögu":1452,"ögs":34,"ögr":85,"öfu":1400,"bú ":35,"öfr":36,"ögm":139,"ögn":233,"ögi":82,"ögl":33,"ögf":50,"ögg":153,"öfn":464,"öfl":142,"fuð":815,"ög ":817,"bök":39,"bör":116,"bön":81,"ftæ":51,"bót":132,"fsí":41,"fyr":4233,"fyl":713,"ftó":31,"fræ":3955,"fvi":41,"fré":80,"fve":203,"frá":3018,"fur":2375,"fus":91,"ból":159,"bók":1011,"bón":65,"frö":89,"frí":321,"fró":162,"fta":843,"fti":2157,"fst":735,"fss":270,"fsm":103,"fsv":103,"fuf":32,"fub":43,"fuk":55,"ful":355,"fum":292,"fun":902,"fug":341,"fts":95,"ftu":254,"ft ":1106,"fra":2612,"fre":467,"frj":180,"fri":247,"bíl":118,"bíu":86,"bís":172,"fu ":237,"fsa":40,"fsd":78,"fse":170,"fsf":61,"fsh":40,"fsi":147,"fsj":32,"fsk":78,"fro":66,"fru":674,"for":2274,"fos":163,"fol":50,"fló":302,"flæ":88,"fmæ":37,"flú":30,"fs ":273,"flö":79,"fið":525,"fna":2112,"fne":55,"fjá":227,"fnd":868,"fng":60,"fnf":164,"fns":126,"fnv":120,"fjó":323,"bæð":272,"fnt":253,"fnu":1196,"fnh":30,"fni":1953,"fjö":1748,"fle":884,"fn ":987,"fla":445,"flj":256,"fli":90,"flu":781,"flo":1007,"fly":309,"öxt":60,"fma":136,"öun":59,"fil":143,"fin":1853,"fim":307,"fir":2556,"fis":690,"báð":53,"fit":83,"feð":29,"fl ":57,"fja":1696,"bæk":87,"ffæ":88,"bæj":204,"bæi":95,"bæn":134,"bær":563,"bæt":285,"fju":37,"örð":1075," Ãð":37,"da ":2461,"dd ":334,"dba":39,"dbr":34,"dbo":31,"de ":373,"dad":43,"dab":91,"dak":98,"dal":1432,"dag":946,"dah":92," Æt":40,"dae":100,"daf":226,"dat":82," Æv":33,"das":501,"dar":2968,"dap":31,"dan":878,"dam":444,"dav":102,"dau":223,"dda":212,"ddi":256,"dds":49,"ddu":785,"cul":53,"cum":33,"ctu":33,"cto":64,"cti":110," à ":611," Ãr":171," Ãt":191," Ãs":2273," Ãb":166,"cus":90," Ãþ":49,"cks":45,"ckl":29," Ól":368," Óm":42," Ód":30," Ós":82,"co ":104," Ön":69," Ör":95," Ös":31," Öl":87,"con":78,"col":55,"com":44,"cor":78," Öx":50,"cot":33," Óð":61,"cs ":45,"ct ":31,"cra":34,"cri":30,"cro":75," Úl":56," Úk":30,"öf ":67," Út":101," Úr":58,"cea":68,"ch ":237,"cer":51,"ces":67,"cet":54,"cen":40,"cel":70," Þv":74," Þy":38,"ci ":39," á ":11624," Þo":417," Þr":200," Þu":34," Þi":250," Þj":278," Þa":1137," Þe":606,"cha":138,"chw":37,"chu":59,"cia":66,"ck ":218,"cie":48,"che":262,"chi":127,"cho":60,"chn":39,"cht":62," át":481," ás":462," ár":4236," áv":94,"cil":37," áb":154," ág":152," áh":379," áf":139," ák":361,"cis":51," ál":132,"cin":65," án":237,"cm ":36," Þó":307,"cke":55," Þý":325," Þá":57," Þæ":99,"ed ":214," æt":868,"ebe":38," æs":38," æx":33,"ebr":277," æv":104," áæ":48,"eae":45," áð":574,"ead":32,"eak":31,"ean":82,"eal":37,"ear":81,"eas":45,"eat":104,"eau":29,"ea ":112,"efi":869,"efj":94,"efl":96," æð":121,"efn":3337,"efa":155,"eff":31,"ei ":96,"ega":3391,"efr":52,"eft":1944,"efs":237," í ":19047,"efu":1553,"een":87,"eed":29,"aár":34,"eet":35,"edi":112,"edd":31,"ede":88,"eda":35,"eg ":669,"edo":29,"edr":32,"eck":48,"ech":72,"ee ":58,"ef ":241,"ect":77,"eco":32,"dys":33,"duð":125," ít":165," ís":1832," ír":55,"dsö":53," íb":414," íl":31,"dró":35,"drú":32,"dy ":43,"dvi":51,"drá":54,"dve":148,"drí":37,"dræ":56,"dré":44,"duv":56,"dur":3307,"dus":104,"dva":88," ór":37," ól":197," óm":33," óh":75," óg":40," óf":82," óe":43," ób":82," íþ":190,"ðár":38,"dor":109,"don":304,"dom":32,"dow":49,"dos":38," öf":73," ök":39," öl":609," ön":266," ör":241,"ds ":1667,"dmi":55," ót":48," ós":171," óv":94,"dið":637,"dna":116,"dni":37,"dnu":50,"ðöl":79,"djú":102,"dsv":141,"dss":318,"dst":207,"dsr":47,"dsp":43," úr":1554," öð":482," út":2473,"dun":333,"dum":1677,"dul":119,"duf":29,"duh":45,"duc":33," óþ":38,"dri":390,"dra":458,"dt ":40,"dná":398,"dre":520,"dry":91,"du ":383,"dro":176,"dru":103,"dsh":112,"dsj":55,"dsi":362,"dsl":272,"dsk":119,"dsn":42,"dsm":154,"dsb":51,"dsa":115,"dsd":51,"dsf":53,"dse":98," þo":209," þu":239," þr":1309," þy":189," þv":1520," þa":4775," þe":4557,"dge":78," þi":195,"dgo":49," þj":806,"dic":66,"dia":76,"dhe":43,"der":294,"des":336,"det":32,"dað":218,"dey":65,"dea":33," þ ":166,"deg":113,"dei":506,"del":110,"den":250,"dem":107,"dfr":39," ým":413,"di ":5025,"dbú":64,"daþ":31,"dfj":56,"dle":192," þö":45," þý":700,"dla":206," þú":310," þó":393," þé":142," þæ":322," þá":953,"dkn":291,"dgæ":30,"do ":152,"dlu":40,"dli":96,"dim":46,"din":1824,"dio":43,"dir":2045,"dis":659,"die":47,"dik":52,"dil":68,"rgv":83,"rgu":338,"órí":40,"rhe":181,"óve":202,"rha":190,"óvi":58,"rdæ":68,"rhl":419,"rhr":181,"órð":209,"rhu":32,"óró":29,"rho":90,"rdí":30,"rfy":82,"rbó":57,"rfu":285,"rft":34,"óun":156,"rbý":31,"rga":1056,"rbú":62,"ri ":5309,"rgl":50,"rgi":863,"rgj":52,"óva":44,"rge":305,"rgf":73,"rgs":155,"rgt":31,"rgr":329,"rað":905,"ret":461,"nát":321,"ótm":41,"nás":32,"res":811,"reu":56,"óti":277,"ótl":55,"rey":1506,"óte":161,"rfa":643,"óta":345,"rfe":164,"rfi":1445,"rfj":246,"rbæ":241,"ótu":133,"rfl":183,"rfo":103,"ópí":47,"rfr":289,"óts":159,"rfs":230,"ótt":1174,"rdu":33,"ósm":64,"rds":39,"ósl":49,"ósk":121,"ósi":102,"rg ":1633,"óse":107,"rea":127,"ósa":223,"ree":99,"ref":289,"red":93,"rei":2743,"reg":939,"nág":58,"rem":334,"nám":587,"nán":98,"ren":1042,"rek":593,"nák":43,"óst":252,"ósu":45,"nál":173,"rel":158,"raí":49,"rep":672,"rf ":244,"óra":323,"órb":108,"rda":542,"mþy":72,"óre":82,"órf":38,"rcu":48,"órh":40,"óri":369,"órn":1251,"rdo":29,"órr":30,"óní":98,"órt":130,"rdj":41,"órs":264,"órv":45,"óru":242,"rdi":87,"rde":130,"ós ":169,"ná ":105,"re ":456,"rby":65,"rbu":32,"mýr":102,"rbr":230,"rch":57,"rce":64,"ót ":306,"ópa":167,"ór ":572,"rd ":235,"ópi":56,"rap":103,"rar":1340,"ópe":33,"ras":718,"rat":554,"rau":1255,"rav":117,"rbi":30,"ólí":218,"rbj":47,"óps":195,"rbl":48,"rbo":361,"rba":200,"rbe":117,"ópu":659,"rai":43,"rah":258,"rag":405,"ran":2332,"ram":2670,"ral":525,"rak":489,"rab":445,"raf":954,"rae":86,"rad":211,"rac":77,"rpu":127,"rpr":62,"rps":409,"rlí":129,"rpo":41,"rs ":1328,"rlö":185,"rlá":68,"rpe":36,"rpa":113,"rkú":29,"rr ":186,"rlæ":104,"rpi":226,"rkí":63,"ror":58,"ros":443,"rot":445,"rom":128,"ron":188,"rop":81,"roy":30,"rou":80,"rov":35,"row":39,"rod":104,"roc":48,"ní ":161,"rjú":69,"rol":93,"rok":333,"rof":43,"rog":34,"rno":60,"rnl":105,"rnm":279,"rnr":31,"rns":265,"rnv":34,"rjó":300,"næð":35,"rnt":68,"rnu":692,"rjö":122,"rp ":163,"rnb":71,"rna":2301,"rng":308,"rnf":82,"rne":499,"rjá":342,"rnd":175,"rni":1126,"rml":167,"rmo":35,"rms":59,"rið":4639,"rmu":102,"rmy":87,"rhö":75,"ro ":119,"rhú":92,"rma":627,"rme":281,"rmi":222,"rly":31,"rhó":65,"rlu":136,"rls":75,"rlo":52,"rlm":64,"rhé":29,"rlj":29,"rhæ":116,"rli":393,"rld":50,"rhá":110,"rle":987,"rla":936,"rn ":955,"rgö":94,"rkv":63,"rku":705,"rkr":89,"rgí":51,"rgð":52,"rkt":84,"rks":157,"rkn":154,"rkm":132,"rko":161,"rkj":1072,"rki":911,"rkl":152,"rke":391,"rgá":37,"rkf":85,"rka":797,"rm ":127,"né ":49,"næs":243,"nær":514,"rju":301,"næt":37,"rfð":143,"rfæ":63,"næm":31,"rfé":382,"rja":455,"næf":123,"næg":56,"ótí":55,"rl ":136,"rip":130,"rio":79,"rir":3442,"náð":134,"reð":86,"rit":1626,"ris":1095,"riu":44,"rih":31,"rig":346,"ril":173,"rik":328,"rin":4044,"rim":103,"ria":195,"rdý":141,"rib":57,"ric":218,"rid":143,"rie":134,"rif":777,"rhv":114,"rdó":74,"ósí":50,"rhy":62,"rk ":749,"rsá":81,"nól":35,"rsæ":104,"nót":79,"rsí":75,"rsó":269,"nóv":141,"rsö":95,"rtá":75,"rsý":233,"ófa":126,"rtæ":610,"óev":37,"rtí":153,"óað":58,"ófo":31,"rtó":34,"ófr":36,"rtö":112,"ófl":62,"rtú":143,"ófi":133,"ryd":53,"ófe":115,"ruh":42,"rug":140,"ruf":120,"rud":31,"ruc":30,"rur":213,"rup":45,"run":1571,"rum":2163,"rul":208,"ruv":97,"rus":350,"rut":38,"rva":375,"rvi":233,"rræ":279,"óf ":138,"rve":420,"rrá":98,"rrí":167,"rré":62,"rvo":44,"ry ":262,"rsk":1287,"rsl":476,"rsi":446,"rsj":74,"rso":176,"rsp":74,"rsm":68,"rsn":93,"rsd":88,"rsa":290,"rsb":37,"rsh":96,"rse":406,"rsf":44,"rsy":38,"rta":441,"rst":3509,"rss":354,"rsv":330,"rsu":120,"óar":52,"óas":59,"óat":33,"rtl":50,"óan":31,"rto":127,"rte":220,"rth":122,"rti":364,"nó ":64,"óbe":269,"rub":37,"óba":44,"rts":104,"rtr":57,"rtu":242,"nín":81,"rmá":390,"níf":29,"nía":80,"rlý":62,"rmú":75,"rt ":1301,"óa ":210,"rmö":79,"rmó":68,"níu":415,"nít":33,"nís":81,"rro":51,"rri":789,"rrk":38,"rná":49,"rre":213,"rra":1384,"ru ":5901,"rry":72,"rnö":83,"rru":200,"rnó":42,"rrv":80,"rní":108,"ómv":228,"sab":56,"óms":871,"sac":50,"óið":33,"ómu":162,"saf":750,"sag":708,"ómp":698,"sak":243,"ómk":72,"sal":365,"sam":4354,"ómi":78,"ómb":34,"óme":237,"óma":424,"óly":43,"sba":82,"sbe":82,"ónu":716,"ónv":387,"óns":366,"sbi":51,"nýj":114,"ónn":30,"nýl":85,"ónl":482,"sbl":34,"óni":207,"óng":36,"san":529,"sau":114,"sat":94,"óne":67,"sas":112,"ónd":59,"sar":943,"óna":507,"sav":82,"óp ":40,"núa":192,"ókr":63,"óks":153,"óku":123,"sa ":884,"óki":198,"núi":49,"ókm":151,"ókn":521,"óka":293,"ryð":51,"óm ":126,"óls":237,"ólu":206,"ný ":52,"ólk":565,"óll":128,"óli":764,"ólm":135,"nús":141,"núp":46,"óla":1201,"ólg":69,"nún":502,"núm":106,"ólf":325,"óle":64,"núl":72,"núv":68,"ón ":402,"nút":151,"nú ":311,"óin":48,"ók ":536,"óju":31,"ól ":221,"óge":44,"nöf":78,"ógi":34,"ógr":30,"rys":58,"ófs":262,"ruð":130,"ófu":99,"ryk":114,"ryl":29,"ói ":77,"ryn":55,"ryg":260,"óga":70,"óhe":37,"rvö":60,"ódí":33,"ógu":78,"rví":87,"nöl":72,"óha":107,"sha":333,"sho":46,"shr":178,"she":190,"shi":231,"sdæ":76,"shl":141,"si ":1082,"sga":69,"sfy":34,"sbó":49,"sgr":89,"sge":55,"nþá":53,"sja":169,"sl ":180,"sju":30,"sfé":34,"sid":34,"sic":68,"sdý":41,"sia":58,"sk ":1182,"shy":73,"sdó":359,"shv":58,"sit":427,"sir":292,"sis":245,"sin":3946,"sio":58,"sil":235,"sig":687,"nþr":30,"núð":42,"sda":221,"sde":55,"nýr":57,"sbr":123,"sbo":107,"sbu":68,"nýs":66,"nýt":153,"sby":61,"se ":199,"sca":45,"sch":105,"sco":77,"sex":134,"sey":275,"ser":203,"ses":89,"set":1580,"sað":68,"sfa":35,"sh ":114,"sbæ":68,"sfj":305,"sfi":93,"sfe":87,"sfo":31,"sfr":321,"sfl":100,"sea":30,"sei":316,"seg":634,"sef":91,"sep":228,"sen":611,"sem":15452,"sel":291,"sek":38,"saæ":40,"sló":155,"spu":111,"spy":259,"slö":108,"spo":83,"slé":92,"slí":192,"spr":262,"slá":52,"spe":881,"slæ":31,"spj":61,"spi":313,"skú":111,"ský":245,"spa":131,"sou":29,"skó":1556,"skö":121,"sol":30,"son":2333,"sop":47,"skí":253,"sor":180,"ská":595,"sof":60,"skæ":55,"sjú":230,"soc":29,"su ":308,"snú":516,"sní":36,"snæ":42,"sri":40,"sre":88,"sra":90,"smö":45,"oð ":82,"st ":7277,"smó":46,"smí":94,"smæ":45,"smá":328,"ss ":1445,"sli":370,"slo":65,"shó":47,"slu":1624,"sgö":54,"sky":454,"sn ":57,"sla":2777,"sle":2574,"shá":42,"ski":2451,"skj":398,"skl":38,"sko":719,"sks":67,"skr":1500,"sku":3717,"skt":636,"skv":172,"sm ":71,"ska":5867,"ske":638,"sjö":285,"sjó":945,"sna":202,"sni":260,"snj":58,"sne":753,"sjá":875,"smo":37,"smy":129,"smu":253,"sið":390,"shú":35,"sma":628,"sly":46,"shö":100,"smi":324,"sme":283,"ssí":67,"ssý":210,"ssö":35,"stí":390,"sté":71,"stæ":1897,"stá":53,"stý":228,"stú":122,"stö":1473,"stó":1005,"syn":591,"suð":680,"sys":40,"syl":37,"syk":82,"íþj":172,"sví":186,"íþr":259,"svæ":826,"svö":99,"sse":290,"sný":67,"ssa":788,"ssn":223,"sso":1722,"ssl":175,"ssj":54,"ssk":305,"ssi":507,"ssv":116,"ssu":314,"sst":880,"ssp":30,"ssy":180,"ste":1923,"spá":57,"stf":191,"sth":38,"sta":8651,"oða":349,"stb":39,"stm":156,"stn":133,"oðn":31,"sto":2031,"sti":2147,"oði":87,"spæ":137,"stj":1787,"stk":49,"stl":62,"stv":37,"oðu":86,"stu":5049,"str":2496,"spí":66,"sts":239,"oðs":85,"sty":391,"sum":731,"sun":1042,"sus":74,"sur":86,"sva":378,"sve":2028,"srá":129,"svi":516,"svo":582,"srí":153,"tai":47,"tak":1550,"tal":2080,"tae":43,"taf":1306,"tag":148,"tah":147,"tab":188,"tad":164,"tbe":34,"tba":41,"tav":183,"tau":127,"tat":143,"tas":1447,"tar":3875,"tap":85,"tan":2188,"tam":380,"tch":58,"te ":264,"tbo":47,"tbr":96,"tbu":123,"tda":38,"ê°€":30,"ta ":6721,"syð":69,"ký ":32,"kút":86,"kúr":39,"kúl":122,"pa ":507,"oxí":40,"íð ":441,"kýr":177,"pe ":53,"lá ":83,"kýj":35,"par":415,"pat":41,"pas":115,"pav":105,"pac":29,"pad":30,"paf":53,"pak":38,"pal":112,"pap":68,"pan":400,"phe":36,"pha":447,"phi":81,"pi ":583,"íði":373,"íðn":37,"íðu":289,"íðs":99,"íó ":37,"pen":326,"lán":85,"lár":68,"per":484,"lát":267,"pað":131,"pes":31,"lás":107,"peg":29,"íða":1262,"lág":191,"pel":119,"lák":76,"pek":637,"pla":766,"pn ":39,"pli":59,"ple":140,"pgö":32,"pja":37,"læg":317,"læm":64,"læk":184,"læp":38,"læt":66,"lær":134,"phy":56,"íói":42,"pil":337,"pin":465,"pir":132,"pis":61,"pit":31,"por":171,"pop":61,"pos":65,"pon":36,"pol":74,"plö":432,"ps ":194,"pps":271,"ppr":373,"ppu":465,"ppt":170,"pph":431,"ppi":417,"ppl":285,"ppn":424,"ppa":270,"ppe":128,"ppf":56,"ppg":58,"pið":157,"lét":224,"lés":58,"lén":49,"lék":31,"ðuð":62,"pma":115,"lí ":172,"pnu":104,"læð":115,"ðví":60,"pni":409,"pp ":769,"pna":137,"pss":172,"pst":155,"pta":402,"pse":49,"psi":86,"psn":40,"psk":239,"pso":41,"ptu":116,"ló ":63,"pub":42,"pte":249,"pti":394,"ppí":56,"líu":494,"lís":172,"lít":407,"pra":73,"pt ":185,"líf":935,"plý":155,"lía":90,"lín":354,"lím":87,"lík":1027,"pru":359,"psa":36,"pu ":453,"pri":135,"pre":517,"pro":73,"lór":49,"lót":52,"lóv":63,"lói":51,"lók":52,"lóm":402,"lón":87,"psþ":115,"ptö":103,"pur":707,"pus":125,"put":29,"líð":255,"pum":165,"pun":389,"puk":38,"pul":139,"pve":36,"prí":173,"lóa":225,"pró":302,"lóð":350,"pyr":266,"puð":74,"lög":1326,"lön":726,"lök":35,"löt":398,"ðju":320,"ðfæ":123,"ðfé":50,"ðji":36,"lús":36,"lút":54,"löð":154,"ðja":484,"ðis":677,"ðir":1217,"lúb":39,"ðin":3092,"ðim":112,"ðil":607,"ðig":61,"ðih":890,"ðid":32,"ðib":80,"ðdý":38,"ðlu":118,"ðhö":65,"ðle":272,"ðli":420,"ðla":663,"ðgö":37,"ðko":89,"ðke":36,"ðki":33,"ðka":49,"ðnu":157,"ðne":31,"ðna":212,"lúð":37,"ðni":136,"ðið":1005,"lþj":313,"lþi":147,"ðmu":161,"ðma":32,"má ":229,"lýt":37,"lýs":582,"ðmi":61,"lýð":232,"ðs ":277,"ðlæ":31,"mát":62,"már":47,"más":73,"mán":166,"mál":2457,"lþý":88,"ðri":810,"ðsa":58,"ðu ":1241,"ðru":553,"ðmæ":37,"ðra":457,"ðto":53,"mæl":428,"ðub":33,"mæt":102,"mær":230,"ðsk":523,"ðsl":355,"ðsm":81,"ðsp":39,"ðse":398,"ðsf":48,"ðsh":47,"ðsi":217,"ðta":31,"ðst":779,"ðss":159,"ðsv":67,"ðsy":35,"ðré":42,"ðræ":84,"ðvi":83,"ðsá":34,"ðrú":35,"ðul":76,"ðuk":30,"ðuf":35,"ðuh":32,"ðug":67,"ður":7875,"ðus":182,"ðun":524,"ðum":1484,"ðuv":112,"ðve":852,"ðva":303,"ðsö":131,"mís":58,"míu":36,"mín":126,"míl":29,"qua":30,"míð":92,"ða ":7924,"que":67,"qui":57,"mób":30,"mót":447,"món":57,"ðal":1210,"ðak":71,"ðan":1528,"ðam":182,"ðab":236,"ðad":86,"ðaf":336,"ðae":34,"ðah":156,"ðag":69,"ðba":46,"ðas":709,"ðat":38,"ðar":3764,"ðau":278,"ðav":103,"ðbo":469,"ðbr":88,"ðbu":122,"mög":91,"mön":242,"mör":555,"móð":114,"ðaá":38,"ðei":268,"ðen":29,"ðað":227,"ðey":75,"ðfa":52,"ðfe":384,"ðaþ":30,"ðfl":57,"ðfj":46,"ðbæ":76,"ðfr":439,"ði ":5726,"ðbó":84,"ðge":167,"ðga":93,"múl":68,"ra ":5434,"ðgu":32,"ðgr":104,"mún":52,"ðhe":185,"mús":74,"ðha":43,"múr":37,"ðho":42,"ðhv":60,"ngn":161,"ngm":109,"ngo":35,"ngj":642,"ngi":2358,"ngl":694,"ngk":66,"ngv":376,"ngu":4585,"ngr":646,"ngt":285,"ngs":1362,"ni ":6328,"nfy":38,"nge":478,"ngf":42,"ngh":122,"nga":4769,"ngb":69,"ngd":467,"ndí":91,"nhl":58,"ndó":106,"nhv":283,"nha":96,"nhi":29,"nhe":69,"ndá":32,"neg":62,"nei":195,"nel":88,"jál":1132,"ják":79,"nen":108,"ján":150,"nem":287,"jár":442,"ner":236,"ját":39,"nað":2058,"net":289,"nes":1695,"jás":35,"jáv":273,"neu":45,"ndv":196,"ng ":2515,"nea":43,"ned":41,"nef":1415,"nfj":30,"nfl":71,"nfr":379,"nfu":33,"ney":381,"naú":37,"naþ":36,"nfa":160,"nfe":55,"nco":35,"nci":82,"nce":184,"nch":109,"ne ":377,"já ":476,"nbu":51,"ndu":3833,"ndr":595,"nds":3224,"ndn":520,"ndo":373,"ndl":456,"ndm":49,"ndk":337,"ndh":138,"ndi":7817,"ndf":105,"ndg":40,"nde":320,"ndd":33,"ndb":149,"nda":5929,"nak":181,"nal":486,"nam":345,"nan":1706,"nap":74,"nar":7258,"nac":33,"nad":255,"nae":106,"naf":1278,"nag":145,"nah":376,"naj":48,"nab":189,"nbo":78,"nbl":83,"nbr":60,"nbe":288,"nbj":30,"nd ":4229,"nav":232,"nau":139,"nat":892,"nas":978,"na ":7656,"myr":42,"myn":2237,"msó":36,"msæ":58,"msö":32,"msý":70,"iðö":77,"mtö":286,"mtí":120,"ntö":45,"ntý":78,"ntí":118,"ntó":67,"nsö":46,"nz ":37,"jö ":94,"nsý":72,"nsæ":244,"jól":491,"jóm":1532,"jón":1264,"nsí":40,"jór":2056,"jós":611,"jót":508,"nsó":384,"ny ":83,"jóf":63,"nsá":38,"nré":59,"nvi":93,"nrí":48,"nve":746,"nva":467,"nul":144,"num":5240,"nun":1626,"nug":63,"nuh":50,"nus":525,"nuv":51,"nur":921,"nua":31,"nub":32,"nuf":123,"nud":81,"jó ":290,"nto":119,"ntu":324,"nts":226,"ntr":104,"nti":309,"nth":63,"nta":596,"nte":421,"nsu":61,"nsv":84,"nsy":38,"nsn":109,"nsm":56,"nsp":46,"nso":41,"nst":1135,"nss":607,"nsf":108,"nse":233,"nsh":91,"nsg":31,"nsj":37,"nsi":284,"nsl":375,"nsk":4454,"nsd":160,"nsa":227,"nsb":64,"nu ":4087,"nní":45,"nri":142,"nná":48,"nra":58,"nt ":1180,"nmö":126,"nmá":266,"nlí":51,"ns ":5692,"nok":465,"noi":41,"nom":35,"non":64,"not":1703,"nos":55,"nor":1577,"nnd":365,"nne":296,"nnf":115,"nng":84,"nna":7028,"nnb":60,"nnl":189,"nno":53,"nnh":72,"nni":6598,"nnk":88,"nnu":1613,"nnt":643,"njó":114,"nnv":87,"nns":1691,"nnr":157,"nny":33,"nme":186,"nma":83,"nið":1052,"nmy":38,"nli":460,"nn ":8805,"nla":365,"nle":833,"no ":154,"nke":308,"ngá":73,"nki":143,"nka":436,"ngó":130,"nku":284,"nky":63,"ngö":59,"nko":60,"nkt":196,"nfæ":48,"nja":244,"nfö":56,"nju":309,"nih":310,"nig":1566,"nif":72,"nie":59,"nid":40,"nic":97,"ndý":173,"nia":97,"ndú":78,"nk ":108,"niu":46,"niv":147,"nis":1128,"nit":151,"neð":186,"nir":1570,"nio":41,"nip":29,"nim":227,"nin":4763,"nik":72,"nil":175,"ogs":89,"ogr":32,"ogu":44,"ogi":191,"ogl":33,"ogn":32,"oga":158,"oge":37,"ogg":37,"ohn":133,"oha":44,"ois":44,"oid":32,"ok ":125,"kær":45,"kæn":58,"ol ":139,"och":58,"oci":55,"ock":125,"oco":29,"ode":87,"odi":36,"odo":53,"of ":366,"odd":103,"oda":50,"kák":62,"kál":657,"odu":43,"og ":20914,"ofn":1635,"ofi":70,"ofu":107,"ofs":67,"oft":1402,"off":70,"ofa":240,"júk":314,"jún":149,"júl":154,"júg":38,"júf":76,"ob ":50,"júp":173,"د ":39,"oba":45,"od ":57,"obi":55,"obe":61,"nyr":104,"nuð":960,"nyt":33,"jöf":126,"jög":408,"nvæ":34,"jök":280,"nzk":209,"jöl":1329,"jör":1720,"jöu":59,"jöt":144,"nvö":31,"jóð":2051,"jú ":59,"oya":34,"ã‚¢ ":30,"kót":31,"kór":108,"kós":45,"ows":60,"kól":1488,"own":47,"köt":45,"köm":42,"ové":67,"kön":66,"kök":33,"köl":238,"kör":99,"köp":65,"otu":454,"otv":37,"kó ":88,"ow ":78,"otl":108,"otk":150,"oti":132,"oth":76,"ote":93,"ott":415,"ots":82,"oto":46,"otn":229,"ost":320,"ota":1324,"ov ":34,"osi":138,"osh":54,"osk":246,"ose":94,"osf":59,"oss":397,"oso":80,"osn":180,"oy ":61,"kóg":196,"owe":53,"ovi":54,"orí":119,"orð":3303,"ox ":46,"ova":32,"orá":31,"ove":83,"oug":45,"oui":32,"oul":47,"oun":72,"ous":64,"our":135,"out":62,"kíð":62,"opn":231,"opo":41,"opp":155,"opi":251,"ope":40,"oph":87,"opa":52,"ítö":52,"os ":473,"olí":72,"opt":43,"ool":48,"ook":64,"ood":45,"or ":435,"okö":33,"ítá":50,"íví":47,"ork":465,"orl":173,"orm":710,"orn":1435,"oro":81,"orp":260,"orr":707,"orc":31,"ord":318,"ore":423,"orf":203,"org":2899,"ori":386,"ou ":62,"osa":139,"ort":719,"ors":867,"orv":93,"oru":1072,"ory":62,"kíl":193,"kín":122,"kíf":146,"ot ":172,"m² ":103,"orb":110,"ora":164,"kís":41,"kír":55,"olb":58,"ola":137,"old":173,"ítr":55,"ítt":43,"on ":2981,"ítu":112,"oli":166,"oll":558,"olk":48,"íti":396,"éði":33,"ítl":29,"olf":120,"ole":105,"ols":62,"olt":453,"olm":30,"olo":103,"oly":34,"olu":106,"olv":29,"oka":194,"ísu":62,"íst":50,"om ":494,"okk":1938,"ísi":469,"oki":67,"ísl":2033,"ísk":2316,"íta":480,"oks":33,"okt":196,"oku":100,"ona":1076,"ond":305,"one":201,"ong":207,"oni":201,"onn":87,"ono":62,"ons":298,"ont":181,"onu":1209,"ony":41,"oma":392,"ome":121,"íus":154,"íur":45,"omb":46,"omi":303,"íun":207,"íum":114,"íul":62,"omm":235,"íuh":63,"omp":59,"omn":42,"omo":42,"omu":187,"íva":36,"oms":54,"op ":66,"íma":1376,"íme":53,"la ":3270,"ími":124,"íms":207,"ímu":159,"ímy":31,"ína":797,"íne":65,"íni":88,"gús":189,"íns":125,"ínu":752,"ínv":127,"ím ":33,"íka":550,"íkj":759,"íkl":107,"góð":87,"íki":1509,"íkr":40,"íks":66,"ígð":53,"íkt":208,"íku":987,"ín ":498,"íle":41,"íld":62,"íla":168,"íll":70,"íli":51,"íls":42,"há ":44,"le ":477,"lby":30,"ít ":92,"éð ":29,"íra":38,"lf ":135,"írs":66,"ldf":77,"ldg":92,"lde":90,"ldb":66,"lda":1630,"ísa":218,"ldn":83,"ldm":49,"ldl":84,"íu ":1396,"ldk":29,"ldi":1971,"ínó":48,"ldv":44,"ldu":1350,"íru":32,"lds":601,"ldr":327,"íon":35,"lab":255,"lac":81,"lad":110,"laf":643,"lae":76,"lah":156,"lag":2929,"lai":54,"ír ":94,"lal":147,"lak":210,"íkó":69,"lan":8388,"lam":321,"lap":48,"lar":1445,"íos":29,"lat":1416,"las":1408,"lax":34,"lau":1468,"lav":277,"lay":48,"lba":94,"ld ":1392,"lbe":135,"ís ":151,"lbo":49,"íló":181,"ípu":47,"lbr":155,"kvi":617,"kve":660,"krá":299,"kva":323,"íet":48,"kuv":29,"kut":72,"íg ":45,"kus":117,"kur":3957,"kup":220,"kun":523,"kum":1574,"kul":461,"ífi":116,"ífl":51,"ksá":78,"ífr":77,"ífs":158,"íft":46,"ífu":123,"ífv":204,"ky ":34,"krú":51,"kró":126,"krö":33,"kvu":31,"ífa":134,"krí":69,"íff":195,"ífe":48,"gða":277,"kta":448,"kte":85,"kss":123,"ksv":50,"koð":120,"kst":476,"ksk":82,"ksi":136,"ksh":75,"ksm":107,"ksl":35,"kub":39,"kuf":44,"kug":52,"gó ":40,"íf ":93,"kts":31,"ktu":499,"gðu":319,"gði":360,"kti":182,"íde":43,"ída":53,"kto":78,"gön":268,"göm":74,"göl":30,"kvæ":806,"gög":58,"kuð":160,"kyr":42,"kyn":662,"ík ":916,"göt":244,"kví":214,"gör":37,"íl ":215,"kvö":70,"ígu":52,"ígs":41,"ígr":36,"ígi":50,"gól":117,"íga":62,"íbú":416,"íhy":55,"kyl":290,"któ":189,"llí":79,"lpe":30,"lph":37,"ls ":828,"llý":48,"lló":29,"lpu":35,"lok":1278,"lon":100,"lom":31,"lop":33,"lor":130,"loc":48,"lof":376,"log":101,"lpa":98,"los":137,"lot":189,"lou":29,"lkó":30,"low":39,"lni":132,"ljá":29,"lne":42,"ljú":103,"ljó":2991,"hæð":207,"lnu":78,"lmi":211,"hél":52,"lme":420,"lma":336,"lna":176,"lmu":132,"hér":502,"lms":88,"hét":88,"lið":1440,"íba":75,"lti":430,"ltr":104,"lts":172,"ltu":112,"lub":100,"luh":51,"lug":511,"luf":94,"íbe":59,"lue":48,"lsh":178,"lsi":379,"lsj":39,"lsk":731,"lsl":62,"lsm":64,"lsn":115,"lss":277,"lst":946,"lsu":92,"lsv":167,"ías":50,"íal":40,"ían":123,"lta":329,"lte":303,"lri":85,"lu ":1523,"lsf":58,"lse":116,"lsd":64,"lsb":58,"lsa":85,"ía ":451,"lmö":34,"lre":29,"lt ":907,"lra":219,"ldó":125,"lhl":81,"lho":108,"lhe":309,"lhj":75,"lha":33,"lgu":75,"lgr":315,"lge":419,"lgj":181,"lgi":219,"li ":3368,"lga":439,"lbú":63,"lfv":36,"lfr":406,"lfs":536,"lft":177,"lfu":334,"lfo":43,"lfj":68,"lfl":58,"lfi":167,"lfb":34,"lfe":71,"lbá":43,"lfa":315,"ley":649,"lex":80,"leu":35,"lev":29,"les":811,"hás":508,"lað":1400,"hát":389,"let":264,"ler":229,"hár":141,"lep":51,"lem":83,"len":3455,"lek":97,"hál":239,"lei":4428,"laæ":29,"leg":4445,"lef":174,"lea":40,"lls":642,"llr":182,"llu":1097,"llt":609,"llv":107,"lly":78,"lo ":81,"lla":2838,"llb":51,"lld":150,"lle":713,"llg":128,"llh":48,"lli":2397,"llj":272,"llk":55,"llm":34,"lln":58,"llo":157,"lgí":110,"lko":96,"lku":305,"lks":187,"lky":33,"lka":170,"lke":74,"lki":628,"lkj":58,"lkn":60,"hæs":106,"hær":33,"lju":52,"hæt":240,"lm ":74,"ll ":1950,"lja":366,"hæk":32,"hæf":189,"hæg":247,"leð":89,"lit":720,"háð":110,"lis":1592,"lir":354,"lip":103,"lin":2452,"lim":235,"liv":30,"liu":91,"lic":99,"lid":33,"lia":157,"ldý":49,"lib":33,"lk ":248,"lik":141,"lil":61,"lig":590,"lie":155,"lif":428,"ma ":1535,"hún":569,"hús":642,"mb ":57,"mab":539,"mah":39,"mak":53,"mad":43,"mae":55,"maf":42,"mag":278,"mar":1996,"mas":392,"mal":260,"mam":64,"man":3392,"mav":111,"mat":635,"mba":627,"md ":63,"mbl":54,"mbi":46,"mbe":570,"hýs":53,"mbr":131,"mbo":170,"me ":192,"mbu":134,"mby":39,"húð":63,"mda":120,"mde":37,"mdu":54,"mdi":69,"med":51,"mef":118,"meg":340,"met":614,"mað":1182,"maí":169,"mes":450,"mer":1104,"mel":116,"men":2186,"mei":865,"maæ":41,"mfe":123,"mfa":71,"mey":48,"mfr":56,"mbí":46,"mbæ":186,"mfj":29,"lva":264,"lve":225,"lvi":125,"lul":121,"luk":133,"lun":888,"lum":1455,"lut":2348,"lus":353,"lur":1152,"luv":69,"ly ":105,"lsá":123,"hóf":338,"lræ":29,"lvu":443,"lsí":62,"hóp":300,"hól":244,"lyk":142,"lyf":122,"ltö":53,"hög":61,"höf":1628,"höl":107,"lvæ":179,"lym":116,"lyn":76,"lys":63,"luð":344,"lyt":300,"lyr":91,"hön":281,"lví":102,"hör":44,"mpi":49,"mpe":55,"mpr":35,"mpl":736,"mpu":43,"mps":68,"ms ":324,"mon":182,"mol":39,"mor":175,"mos":64,"mou":30,"mpa":70,"mri":72,"mru":37,"msa":120,"mu ":328,"mse":173,"msb":39,"mmá":75,"mt ":874,"ið ":16040,"mmú":58,"mra":69,"mtu":58,"iðu":850,"iðv":49,"iðs":853,"iðt":99,"mpí":104,"iðr":196,"iðm":78,"iðn":200,"iðk":105,"iðl":242,"iði":994,"mti":77,"iðj":659,"iðh":142,"mss":231,"mst":639,"msu":130,"msp":568,"msj":68,"msk":329,"msl":119,"msm":255,"msf":81,"msh":58,"msi":108,"iðd":64,"iðf":164,"mte":29,"iðe":79,"iðb":270,"iða":1209,"mta":215,"msv":831,"mvi":66,"mræ":144,"msá":58,"my ":29,"mur":1133,"mus":94,"mul":139,"mum":344,"mun":905,"mva":32,"mve":268,"mhl":52,"mhv":209,"mha":163,"mdæ":65,"mhe":130,"mi ":1234,"mbö":37,"mfy":45,"mfé":153,"min":1199,"mil":1646,"mir":185,"mis":788,"mit":132,"með":4188,"mic":43,"mik":821,"mo ":39,"mlu":66,"mli":71,"mle":912,"mla":218,"mgö":49,"mky":51,"mkv":450,"mki":55,"mke":47,"mm ":233,"mjö":303,"mjú":31,"mnu":39,"mjó":184,"mni":205,"mna":60,"mne":186,"mmy":31,"mms":259,"mmu":172,"mið":1660,"mmt":327,"mmi":66,"mmo":32,"mma":212,"mme":102,"xár":48,"xík":59,"xá ":30,"víd":42,"vía":47,"vít":367,"vís":868,"vív":36,"víu":65,"vír":31,"víl":52,"vík":1324,"vín":173,"vím":40,"víg":117,"víþ":176,"víð":290,"vö ":226,"vöx":63,"vöt":84,"vör":454,"völ":332,"vök":88,"アア ":30,"vöð":43,"ví ":1408,"væð":1152,"Ãsg":52,"Ãst":139,"ال":93,"vél":295,"vét":58,"Ãra":30,"Ãri":106,"Ãrm":44,"Ãrn":150,"væt":40,"vær":197,"zku":191,"væg":225,"væm":521,"væl":58,"væn":135,"Ãgú":61,"Ãlf":73,"yrð":218,"yvi":33,"uðu":1637,"ytu":186,"uðv":232,"uðs":298,"ytt":292,"ytr":53,"uðr":201,"uðm":148,"uðn":68,"uðk":47,"uðl":128,"ytj":179,"yti":525,"uði":226,"uðg":34,"uðf":78,"uðb":470,"uða":444,"yta":127,"yss":114,"yst":379,"ysu":42,"ysk":93,"ysl":34,"ysi":128,"ynþ":31,"ysa":75,"yrk":276,"yrj":263,"yri":3304,"yrp":48,"yrn":424,"yrt":41,"yrs":1207,"yrr":562,"yru":38,"uð ":1944,"yra":258,"yrg":59,"ys ":74,"ypi":37,"ypt":168,"yps":42,"yr ":47,"yjó":34,"uðá":45,"týr":322,"yf ":34,"ydd":64,"yfa":39,"yfl":41,"yfi":1843,"yfj":42,"yer":35,"ya ":46,"túd":45,"tún":89,"töð":1090,"túr":354,"túg":124,"túl":110,"yan":50,"uæt":68,"yju":348,"ykh":37,"yki":114,"ykj":958,"ykk":283,"yks":31,"ykr":37,"yku":91,"ygð":31,"ykt":79,"yn ":65,"yld":242,"yle":30,"ylf":29,"ylg":269,"ylk":497,"yll":160,"ylv":38,"ylt":92,"yma":72,"ymi":55,"yms":49,"ymp":115,"yna":50,"ynd":2435,"yni":439,"ynj":135,"ynl":57,"ynf":55,"yng":529,"yns":204,"ynt":58,"ynn":253,"ر ":29,"yft":54,"ygi":89,"ygg":1542,"ygl":52,"ygj":36,"yin":76,"yja":1253,"tím":1376,"tín":493,"tíl":107,"tíu":187,"tís":148,"tíf":31,"tíg":107,"xtu":66,"xta":178,"xti":64,"tæð":489,"xne":38,"xna":31,"xið":31,"tét":65,"ÙŠ ":37,"xlu":43,"töf":197,"tön":65,"tök":743,"töl":1034,"tör":51,"Ù† ":58,"tóð":129,"tó ":38,"tíð":431,"tóm":77,"tón":860,"tók":231,"tól":186,"tób":197,"tóf":35,"tór":803,"tót":73,"sþá":113,"xfo":37,"tán":117,"tár":67,"ták":413,"tál":58,"xla":43,"tæt":90,"tær":1323,"tæk":1071,"tæp":54,"xin":82,"súl":58,"sút":44,"súr":97,"xa ":51,"sú ":352,"sýr":127,"sýs":821,"sýn":431,"sýk":65,"sþi":29,"tá ":55,"xas":32,"xar":46,"xan":81,"síð":895,"sök":67,"söl":167,"söm":115,"sön":374,"söf":68,"sög":1012,"sót":79,"sós":78,"són":245,"sól":222,"sók":506,"sér":1103,"séu":75,"sæt":356,"sé ":343,"wn ":40,"síg":43,"ws ":58,"sía":114,"sís":110,"síu":323,"sík":31,"sím":100,"síl":50,"sín":753,"wor":29,"rúð":68,"rþe":39,"rþi":59,"sá ":403,"rýt":47,"rýs":98,"rýn":71,"rým":56,"wer":56,"sár":272,"sát":78,"sál":193,"sæl":271,"sæk":59,"sæn":178,"wic":43,"rön":643,"röl":81,"rös":68,"rög":86,"röf":82,"rök":168,"róð":187,"rú ":190,"rúi":57,"rún":199,"rúm":310,"rús":221,"röð":352,"rút":141,"wa ":41,"rúa":419,"rúg":52,"rúf":57,"way":31,"war":95,"wai":35,"ríu":241,"rív":32,"rís":1547,"rít":76,"réð":35,"rír":32,"río":31,"rím":339,"rín":121,"rík":2512,"ríl":176,"ríh":63,"ríf":39,"ríb":73,"ría":168,"rð ":1989,"vu ":91,"rðv":334,"ró ":38,"rðr":296,"rðs":472,"rðu":3047,"rðg":39,"rðf":294,"rði":2526,"rðh":35,"rðm":67,"rðl":335,"rða":3018,"rðb":53,"rðe":35,"ríð":409,"vus":33,"vur":48,"vuo":30,"ríó":57,"vum":75,"vun":101,"vul":145,"vró":716,"rót":502,"rós":105,"róu":170,"róa":188,"róf":441,"rók":92,"ról":88,"rói":38,"rój":32,"róp":739,"róm":241,"rón":128,"vip":108,"vir":486,"vik":659,"vil":263,"vin":1308,"vig":107,"vic":33,"vid":89,"vif":41,"veð":482,"ráð":822,"vit":214,"vis":416,"ræn":875,"ræl":54,"ræm":47,"ræk":248,"ræf":40,"ræg":184,"ré ":89,"ræt":349,"ræv":35,"vo ":541,"réf":119,"rét":867,"við":5233,"rés":59,"ræð":4116,"rí ":42,"vog":188,"vol":114,"vok":77,"von":122,"vop":132,"vor":1409,"vot":61,"vi ":117,"vex":159,"vep":209,"ver":6921,"rár":206,"rás":260,"ves":1169,"vet":255,"rát":145,"vað":288,"ráv":43,"rái":33,"vei":2637,"veg":1474,"ven":751,"rán":98,"vem":147,"vel":1100,"rák":60,"vek":61,"ráb":38,"ráa":38,"vef":239,"ve ":176,"rá ":3167,"val":1084,"vak":173,"van":556,"vam":93,"var":9040,"vat":1043,"vas":74,"vav":70,"vax":219,"Ãðu":37,"vaf":68,"vad":53,"vag":69,"va ":334,"uvö":31,"uví":69,"utæ":43,"utí":62,"ux ":46,"uvi":38,"uva":134,"uve":291,"urá":81,"urí":54,"urð":755,"usl":30,"usk":240,"ush":40,"usi":109,"use":190,"usa":273,"usv":66,"ust":3311,"uss":141,"usp":64,"usn":74,"utl":60,"utn":112,"uth":88,"uti":783,"upá":53,"ute":116,"utf":76,"uta":1193,"utt":454,"uts":66,"utv":219,"utu":201,"uto":43,"utr":39,"us ":1167,"ulý":49,"umá":434,"umæ":45,"umö":32,"ut ":328,"urb":253,"ura":213,"urd":67,"urf":235,"ure":339,"urh":524,"urg":224,"urj":30,"uri":1241,"url":562,"urk":192,"urm":112,"urn":504,"uro":90,"urp":40,"urr":390,"urs":1140,"urt":381,"uru":95,"urv":93,"ury":37,"uor":53,"upa":89,"ur ":26291,"upi":81,"upe":58,"ulá":50,"upp":2005,"ulí":46,"upm":133,"ulö":48,"ups":167,"ums":276,"umr":110,"umu":278,"umt":34,"umv":35,"umi":189,"umh":215,"umk":61,"umm":97,"uml":76,"umn":32,"uma":561,"umb":222,"umd":81,"ume":325,"umf":103,"uhö":65,"unt":112,"uns":75,"unv":62,"unu":1350,"unk":272,"uni":524,"unn":2545,"unc":32,"und":5296,"una":2110,"unb":61,"ung":2211,"une":148,"umy":38,"up ":193,"uku":56,"ukt":40,"ukn":82,"ukk":98,"uki":102,"uke":171,"um ":21458,"uka":181,"ulu":90,"ult":54,"uls":171,"ulr":34,"ulo":30,"ull":717,"ulj":36,"uli":154,"ulg":31,"uhá":62,"ule":1154,"ulf":47,"uld":119,"ula":290,"un ":2337,"uin":39,"uis":52,"uk ":357,"ufé":80,"uit":37,"ul ":138,"ugf":52,"ugg":223,"ugh":68,"ugi":102,"ugb":111,"uge":80,"ugn":84,"ugl":480,"ugm":142,"ufu":94,"ubó":48,"ufr":151,"uft":32,"uga":631,"ufy":58,"uhe":55,"uhr":48,"uhl":89,"ugv":167,"ugu":342,"ugt":564,"ugs":232,"ugr":64,"uha":44,"uct":39,"uda":130,"udd":31,"ude":123,"udi":40,"ubo":41,"ubr":59,"uca":53,"ue ":86,"uch":32,"uck":49,"uer":36,"ues":38,"uff":37,"ufe":30,"ufa":76,"ufo":48,"ubæ":33,"ufj":77,"ufi":29,"ufl":123,"ug ":195,"uef":41,"uel":43,"uar":31,"ual":30,"uan":43,"ubl":130,"ube":67,"uba":61,"ubb":30,"tvö":262,"tvæ":181,"tví":220,"pön":127,"pör":37,"tyk":37,"tyr":286,"tuð":583,"tyt":114,"ty ":229,"trö":549,"trý":35,"trú":459,"tró":64,"tré":135,"træ":183,"trí":436,"tvo":138,"trá":56,"tve":811,"tvi":179,"tva":215,"tur":6286,"tus":346,"tut":290,"tul":115,"tuk":92,"tun":2035,"tum":1349,"tub":54,"tud":78,"tuf":48,"tuh":68,"tug":285,"ttú":315,"ttá":38,"tz ":61,"pós":50,"two":32,"pól":144,"pía":68,"ts ":398,"tmæ":47,"tmá":108,"pít":55,"pír":83,"píp":39,"píu":114,"tre":326,"tt ":3973,"tra":1427,"trj":59,"tri":1095,"tru":188,"tro":207,"tu ":1914,"try":105,"tsa":40,"tsb":48,"tse":120,"tsd":95,"tsi":93,"tsj":49,"tsh":220,"tsm":42,"tsk":176,"tsl":69,"tsp":261,"tsv":50,"tsu":54,"toð":80,"tst":183,"tss":149,"tta":1700,"ttb":296,"tte":221,"ttf":40,"tth":142,"tti":1896,"ttk":177,"ttl":443,"ttm":78,"ttn":145,"tto":50,"ttp":29,"tts":368,"ttr":107,"ttu":1777,"ttt":56,"ttv":68,"tme":42,"tma":134,"to ":193,"thö":259,"tið":769,"tmi":41,"tni":1019,"tne":174,"tjá":158,"tna":432,"tmy":32,"tjó":1537,"tns":509,"tnu":113,"tof":1728,"toc":34,"tog":155,"tjö":220,"tos":79,"tom":29,"ton":377,"tok":74,"tol":63,"tor":456,"top":73,"tlæ":32,"til":6806,"tik":46,"tif":55,"tie":49,"tig":294,"tir":3353,"tit":169,"tis":756,"tin":2196,"tim":103,"tio":397,"thy":59,"thu":143,"thv":123,"tia":90,"tic":138,"tid":39,"tfæ":31,"tju":44,"pæn":140,"tiv":71,"tja":445,"tkv":226,"tku":148,"Ætt":33,"tke":39,"tgá":259,"Ævi":29,"tli":156,"tlu":263,"tn ":382,"tla":1003,"thá":36,"tle":549,"tem":310,"pán":114,"ten":1249,"tep":34,"tei":1039,"tek":688,"tel":552,"tef":480,"teg":799,"ted":71,"tfl":34,"tfj":120,"tfi":62,"tfe":37,"tbá":198,"tfa":120,"th ":216,"tey":170,"tex":159,"tað":3001,"tet":95,"tes":176,"ter":920,"pár":60,"tgr":38,"tge":143,"ti ":5397,"tbý":74,"tga":31,"tfr":38,"tho":74,"thl":29,"thr":34,"the":336,"thi":41,"tha":162,"þús":303,"þýs":403,"þýt":31,"þýð":375,"ýða":33,"ýðr":52,"ýði":302," アア":29,"ýðs":32,"ýðu":113,"ýðv":179,"þét":142,"þör":71,"þól":58,"þór":44,"þót":75,"þó ":313,"þá ":592,"þær":240,"þæt":143,"þát":560," об":31,"þrá":74,"þve":149,"þró":578,"þrí":169,"þræ":66,"þur":132,"þun":127,"þrý":79,"því":1355,"þyr":48,"þyn":97,"þyk":156,"三 ":35,"ýnt":39,"ýr ":423,"ýrd":38,"ýra":596,"ýrl":53,"ýri":416,"ýrt":38,"ýrs":37,"ýju":59,"ýki":49,"ýli":248,"ýle":93,"ými":230,"ýms":227,"ýni":230,"ýnd":168,"ýna":59,"ýja":206,"úða":105,"úðu":100,"úði":56,"úð ":64,"þro":69,"þre":237,"þrj":165,"þri":320,"þjó":1391,"þol":52,"þot":31,"þor":186,"þjá":53,"þin":513,"xíð":39,"þeg":905,"þei":1439,"þek":897,"þen":84,"það":1705,"þet":98,"þes":1283,"þar":1999,"þan":673,"þak":46,"yða":47,"yðj":117,"yði":287,"yðr":29,"þau":382,"yðs":65,"ýrð":81,"ýva":31,"ýti":83,"ýtt":104,"ýtu":56,"ýta":55,"ýsl":836,"ýsk":757,"ýsi":434,"ýst":220,"ýru":217,"ýsa":87},"n_words":[2831948,3326341,2728347],"name":"is","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/it.json b/contrib/languages-data/it.json
new file mode 100644
index 0000000..6371347
--- /dev/null
+++ b/contrib/languages-data/it.json
@@ -0,0 +1 @@
+{"freq":{"D":94947,"E":67243,"F":92398,"G":104535,"A":191905,"B":119758,"C":221153,"L":203562,"M":160653,"N":84931,"O":53123,"H":49392,"I":191375,"J":30427,"K":33649,"U":59673,"T":99599,"W":33788,"V":71164,"Q":12325,"P":151391,"S":231227,"R":109065,"Y":11206,"X":16332,"Z":10278,"f":518611,"g":920085,"d":2363397,"e":6056669,"b":494596,"c":2150346,"a":6155041,"n":4141668,"o":4513622,"l":3766393,"m":1338197,"j":22860,"k":110896,"h":452585,"i":5976536,"w":49843,"v":585518,"u":1754682,"t":3725316,"s":2419415,"r":3264659,"q":101625,"p":1281861,"z":475996,"y":114701,"x":30592,"È":18809,"ì":9651,"é":22484,"è":321778,"à":93588,"ù":30198,"ò":27447,"ó":6194," l":332142," m":240472," n":389032," o":184207," h":31309," i":549255," k":13980," d":1574321," e":446281," f":277424," g":138336," a":703335," b":75718," c":722915," z":6698," u":458470," t":223758," v":125009," q":64842," p":570240," s":706865," r":244819," J":29715," K":31685," H":46792," I":159850," N":78119," O":44164," L":197779," M":152789," B":113710," C":207226," A":159909," F":86857," G":100079," D":87417," E":59714," Z":9584," Y":10426," X":11662," S":213852," R":103083," Q":11829," P":141839," W":31372," V":61312," U":56839," T":91377," è":311053," È":18792,"A ":27808,"Da":15987,"Cu":5620,"Cl":7833,"Co":63300,"Cr":10676,"Ce":10902,"Ch":29704,"Ci":11685,"Du":6024,"Do":13966,"De":15564,"Di":21598,"Fe":11756,"Fa":10362,"Eu":7860,"Es":8695,"En":5820,"El":6080,"Ge":15850,"Ga":14081,"I ":31564,"Fu":8170,"Fr":19816,"Fo":12748,"Fi":15192,"C ":10682,"Au":9085,"Ar":21848,"As":8912,"D ":5691,"Ba":27783,"Am":10014,"An":18633,"Al":34245,"Bu":8230,"Br":18440,"Ca":56504,"Bi":8237,"Be":19580,"Bo":19802,"Le":25820,"Li":19778,"La":88640,"Lu":9671,"Lo":25268,"Me":19948,"Mi":23733,"Ma":59219,"Mu":9761,"Mo":30287,"Ni":7514,"Ne":23857,"Na":17893,"No":19720,"Gi":22185,"Gl":5829,"Gr":18092,"Go":9531,"Gu":8537,"Ha":17031,"He":8149,"II":13036,"Ho":8979,"In":30727,"Il":66084,"Is":9021,"It":12730,"Ja":8255,"L ":27703,"Jo":9534,"Ka":7944,"Un":28071,"Tr":15672,"To":14627,"Th":17010,"Ti":6315,"Te":15797,"Ta":10754,"UA":13360,"V ":7192,"St":32757,"Su":12977,"Wi":7930,"Wa":7819,"Vo":7254,"Vi":20021,"Va":13663,"Ve":14211,"Pu":5612,"Pr":25455,"S ":9118,"Pe":16282,"Pa":38366,"Po":20730,"Pi":22494,"Or":10767,"Se":20797,"Sc":24538,"Si":21736,"Sh":7322,"Sp":9077,"So":16973,"Ru":7444,"Sa":46373,"Re":30166,"Ri":14281,"Ro":32671,"Qu":10556,"Ra":11841,"b ":11012,"a ":2254795,"i ":1483483,"ge":88624,"ga":67666,"fl":8575,"ff":35291,"fi":134642,"fr":78002,"fu":41990,"fo":66122,"he":195889,"ha":63814,"gn":70158,"gl":120822,"gi":234393,"gh":24919,"gg":70910,"gu":55229,"gr":73387,"go":71768,"du":52294,"g ":29202,"ea":91863,"eb":24191,"ec":137759,"ed":128701,"de":782039,"dd":10103,"di":854362,"do":146993,"ds":9108,"dr":32135,"ew":7817,"ex":7461,"eu":18301,"ev":52077,"ey":12656,"ez":26509,"fa":71893,"h ":27064,"fe":57833,"eg":172770,"ef":19354,"ee":16955,"el":899715,"ei":89786,"ep":27874,"eo":41068,"en":625515,"em":132418,"et":253878,"es":434221,"er":718824,"eq":6537,"ca":399283,"e ":2094441,"br":59095,"bu":40562,"bo":29867,"bl":42370,"bi":141440,"bb":48744,"be":43765,"da":306546,"f ":15395,"cu":62362,"ct":11024,"cq":7195,"cr":59094,"co":609419,"ck":21526,"cl":55423,"ci":319342,"ch":241963,"ce":223557,"cc":104089,"c ":16295,"az":131617,"ay":15144,"ba":66599,"d ":145378,"at":664795,"as":217940,"ar":494405,"av":77540,"au":53901,"ak":11813,"al":679169,"ai":64110,"ap":84421,"am":172413,"an":700812,"ac":102427,"ad":97495,"ab":100158,"ag":138596,"ah":7533,"ae":26244,"af":30789,"nu":38735,"nt":634979,"ns":87660,"nq":7056,"no":374486,"nn":92022,"nz":76063,"ny":7751,"nv":12211,"oe":12640,"of":30817,"oc":106098,"od":71766,"oa":9353,"ob":27798,"om":291133,"on":807692,"ok":6829,"ol":302572,"oi":49498,"og":89760,"oh":6209,"ot":113582,"os":173480,"ov":109964,"ou":37230,"op":117448,"oo":15155,"or":452213,"r ":177153,"ow":12656,"oz":6935,"oy":5914,"pe":242170,"pa":234524,"pl":24851,"po":213425,"ph":7578,"pi":128997,"lo":210541,"lm":44463,"ll":700194,"ls":12139,"lp":10018,"lv":11056,"lu":64908,"lt":90822,"ly":7335,"o ":1645460,"ma":263312,"mb":52388,"me":350394,"mi":170562,"mm":42029,"mp":106005,"mo":153835,"mu":105465,"p ":14900,"iù":27078,"na":431794,"nc":198192,"nd":220346,"ne":784634,"nf":25150,"ng":109071,"ni":335162,"nk":7990,"ki":12045,"ke":15323,"ka":11574,"m ":76534,"ko":6777,"km":10394,"li":478088,"le":450199,"ld":17670,"lg":10033,"lf":7624,"la":677771,"lc":22009,"lb":26576,"n ":671873,"hr":6559,"ht":9126,"hu":11107,"hi":93831,"hn":5725,"ho":18964,"id":103779,"ic":450726,"ib":38087,"ia":461965,"ig":105629,"if":52751,"ie":183937,"k ":32183,"ir":101293,"is":299704,"it":487446,"iu":44781,"iv":129892,"ik":7413,"il":319428,"im":199167,"in":697560,"io":541230,"ip":107291,"iz":100179,"l ":908043,"ja":6741,"z ":9054,"tà":85596,"wi":6023,"vv":10000,"y ":65822,"wa":12380,"we":6401,"vi":166809,"vo":78376,"uz":20664,"ve":175153,"va":133125,"x ":19811,"ui":79181,"ul":72491,"ue":88946,"uf":10300,"ug":23193,"ur":134766,"us":100555,"ut":118876,"um":71437,"un":561799,"uo":58100,"up":47527,"ty":10365,"tu":164890,"tt":381279,"ub":48232,"ua":156844,"ud":43002,"uc":34894,"w ":10150,"to":743121,"tl":8532,"ts":9120,"tr":295588,"te":580573,"ti":577841,"th":33664,"ta":698330,"su":137066,"sv":16162,"ss":213977,"st":447357,"sl":9152,"sk":10657,"sm":16619,"sp":87152,"so":218207,"sc":157523,"sf":9364,"se":363584,"sh":16912,"si":401810,"rz":16932,"u ":59520,"sa":125213,"rr":72120,"rs":82412,"rt":207644,"ru":63955,"rv":21724,"ry":12041,"rp":15121,"ro":352219,"rn":64837,"rm":75961,"rl":22967,"rk":10623,"ri":614338,"rg":44544,"rf":10278,"re":622304,"rd":75640,"rc":66009,"rb":30426,"ra":583297,"t ":103375,"qu":99430,"s ":156468,"pu":55098,"pp":84071,"pr":255723,"ps":6191,"zz":76375,"zi":221598,"ze":20937,"za":99515,"zo":37143,"ya":7485,"È ":18782,"à ":92132,"ò ":26518,"ì ":8878,"é ":9885,"è ":314074,"ù ":29830," Ga":14003," Ge":15739," I ":16475," Fo":12451," Fu":8160," Fr":19775," Fi":15103," Ha":16981," He":8121," Go":9484," Gr":17959," Gu":8468," Gi":22095," Gl":5798," Ho":8931," L ":24925," Ja":8214," Is":8984," It":12700," In":30290," Il":65842," Ka":7887," Jo":9477," La":88189," Le":25663," Li":19618," Ma":58913," Mi":23636," Me":19838," Lo":25207," Lu":9633," Ne":23724," Na":17829," Ni":7466," Mo":30189," Mu":9681," A ":5979," Am":9971," An":18493," Al":34065," Ba":27690," Au":9040," As":8840," Ar":21718," Be":19500," Bi":8121," Bo":19624," Br":18373," Bu":8165," Ca":56136," Ce":10868," Ci":11600," Ch":29611," Cl":7713," Cr":10596," Co":62928," Da":15272," Di":21504," De":15426," Do":13718," Du":5966," El":6042," Es":8654," En":5749," Eu":7835," Fe":11726," Fa":10272," Wi":7870," Wa":7761," a ":110030," Or":10710," Po":20608," Pi":22451," Pe":16220," Pa":38161," No":19629," Ra":11758," Qu":10458," Ro":32569," Re":30042," Ri":14247," Pr":25353," Su":12949," St":32098," Ta":10678," UA":13306," Th":16944," Ti":6253," Te":15675," Tr":15562," To":14480," Ru":7431," Sa":46303," Sh":7238," Si":21606," Sc":24393," Se":20659," So":16887," Sp":9003," Va":13628," Ve":14047," Vi":19885," Vo":7231," Un":27982," l ":52713," im":22658," in":295683," il":151179," is":14605," it":18185," ha":23853," gi":32407," gl":14007," gr":40025," go":5702," gu":9712," id":6931," ne":275511," na":25351," mu":20050," mo":56127," ol":7856," om":7514," og":8277," oc":9930," of":10734," nu":14604," no":68303," le":52423," li":35936," la":143866," km":9911," me":51316," mi":29028," o ":42065," ma":81024," lu":16900," lo":28809," af":7222," ag":11338," ab":64448," ac":16193," ad":25196," am":20786," an":81988," ap":29511," ai":8950," al":183776," av":17618," au":20712," ar":36060," at":28661," as":31346," d ":20488," ba":32317," bi":8838," be":7889," bo":6391," br":13522," ca":114268," e ":219384," er":25454," et":12895," es":52932," en":13079," ep":5911," el":14730," fe":18710," fa":56978," fu":35844," fr":63957," fo":40785," fi":57936," ge":26299," ga":8284," i ":33110," cl":12326," co":356836," cr":19933," ce":24853," ch":106561," ci":55235," da":211997," cu":28570," do":30811," de":613533," di":670575," ec":32925," ed":39893," du":23486," ru":12186," sa":17824," se":114506," sc":42709," si":147221," sp":37866," so":80656," qu":64678," ra":31475," re":103663," ri":75848," ro":21093," pu":36089," pr":197319," os":7051," ot":10340," ov":5976," op":14803," or":49602," pe":114303," pa":91709," po":66694," pi":52800," va":18746," ve":35685," vo":17500," vi":51221," tu":14428," us":12554," ut":9880," un":408903," ul":5818," ta":13154," st":115114," sv":15397," su":116897," tr":76246," to":12684," th":12060," ti":18079," te":75278," È ":18766," è ":310990,"Eur":6532,"Gio":7152,"Fra":14090,"For":6200,"II ":9095,"Gra":9265,"Int":6283,"In ":9661,"Il ":64107,"Bas":6130,"Alt":6625,"Cal":8462,"Cam":7263,"Cas":9838,"Car":10034,"Can":7902,"Chi":9585,"Cen":5742,"Cha":12533,"Cor":7549,"Com":13539,"Col":6543,"Con":19924,"Dis":7107,"Nel":12164,"Nor":11147,"Per":7527,"Par":12134,"Pro":9921,"Pre":7483,"Que":6463,"Ita":11783,"Le ":10050,"La ":72068,"Man":6976,"Mar":24416,"Mon":12589,"Sta":17125,"UA ":13253,"Si ":7499,"Sai":6635,"Sco":15601,"San":19570,"Reg":7644,"Rom":11444,"Ven":6082,"Val":6880,"Uni":19062,"The":12364,"Tra":6212,"bit":77836,"bil":24167,"bli":33195,"bor":7965,"bbl":30259,"bbe":6273,"be ":7531,"bbr":5663,"ban":14724,"bal":6018,"bat":9698,"bas":13208,"bar":7268,"ber":18766,"bia":7881,"ca ":152253,"car":46789,"cas":12364,"cat":57544,"can":40911,"cap":15975,"caz":8224,"cam":17670,"cal":28832,"ce ":48205,"bri":12930,"bro":7660,"bra":15040,"bre":21272,"bum":18883,"am ":7918,"al ":120078,"ain":14219,"aio":8047,"agl":13391,"agg":47904,"agi":14545,"agn":23437,"ago":14318,"anz":27062,"ano":80778,"ann":47068,"ant":158501,"ans":9496,"ane":24020,"ang":15547,"ani":57213,"ana":44733,"anc":100934,"and":75292,"amm":16889,"amo":10418,"amp":23753,"ami":24192,"ame":55568,"amb":14366,"ama":14741,"alt":24272,"alo":9405,"alm":19827,"all":157126,"ali":107974,"alc":17413,"ald":5979,"ale":157930,"ala":17702,"alb":21166,"an ":45440,"abb":6484,"abi":71766,"abo":7042,"ae ":8196,"ad ":23773,"aff":9278,"afi":13627,"ai ":19697,"aga":10288,"age":5874,"aes":7300,"ado":12933,"adr":12192,"adi":16689,"ade":11907,"acq":6540,"aco":8070,"aci":10043,"ach":7116,"ace":12782,"acc":33357,"ada":10503,"acr":5776,"azi":114771,"azz":11554,"at ":6931,"arg":6155,"are":73118,"ard":28986,"arc":21226,"ara":44876,"aro":14834,"arn":6502,"arm":9201,"arl":10615,"ari":92056,"arr":17226,"ars":9758,"art":120284,"asa":12008,"asi":18399,"asc":31970,"ase":8737,"ar ":13924,"api":8043,"apo":15567,"app":40715,"as ":11094,"ava":20339,"aut":20581,"avo":14059,"avi":12649,"ave":19075,"ay ":7096,"avv":6341,"ata":122912,"ast":41964,"ass":66552,"atr":11363,"ato":251278,"ate":43614,"ati":96839,"att":98894,"atu":21242,"aur":8293,"aus":7205,"ito":68066,"itu":73079,"itt":66676,"ism":9170,"iso":15645,"isp":27507,"iss":18851,"ist":119446,"ita":148002,"ite":23677,"iti":31286,"ivo":23510,"ius":6126,"ium":7439,"iun":5847,"iut":11741,"iva":34645,"ivi":28519,"ive":42061,"ipo":9185,"ipi":8806,"is ":24623,"ion":300543,"ior":37852,"ios":7307,"ipa":67260,"ipe":7498,"iov":6740,"iro":6686,"iri":10262,"isi":24222,"ise":9930,"isc":25942,"isa":8063,"ire":38092,"ira":11739,"irc":16765,"ità":54075,"izz":49660,"izi":46425,"km ":6935,"ha ":20767,"ham":6367,"han":10716,"har":10403,"he ":154969,"het":5812,"her":11167,"hi ":17347,"hie":13497,"hia":17408,"hin":6740,"hil":8553,"hit":6455,"go ":24875,"gle":14019,"gli":101433,"gno":22948,"gni":13103,"gne":7169,"gna":23696,"gol":13409,"gon":12443,"gru":16613,"gra":38555,"gre":12522,"gui":11363,"gua":15610,"gue":16931,"gur":5768,"iam":18301,"ial":41395,"ian":67167,"ias":18220,"iar":13087,"iat":27828,"ic ":7518,"iac":6819,"ibi":9969,"ibr":7455,"ibu":6636,"iaz":6940,"ibe":7052,"ia ":244503,"iet":16601,"iem":9287,"ien":42565,"ier":26963,"ies":15028,"iff":8035,"ife":10392,"ifi":25651,"ico":88289,"ici":66289,"ich":34469,"icc":12587,"ice":31036,"ie ":52789,"ica":189725,"ido":7371,"idi":19231,"ide":49954,"ida":12753,"il ":154014,"igl":35255,"igh":7328,"igi":24518,"igu":7702,"ign":12535,"imo":36242,"imm":6781,"imp":19629,"ime":69205,"imi":17819,"inc":72864,"ind":26998,"ina":84297,"ino":48220,"int":56511,"ins":16076,"inf":11205,"ine":59357,"ing":57070,"ini":52386,"ioc":18011,"inv":7003,"ila":12536,"in ":183514,"ilo":8196,"ill":25371,"ilm":20062,"ili":45534,"ile":34997,"ima":41217,"io ":144991,"ilu":7192,"ffe":9164,"ffi":11476,"fes":8200,"fer":25051,"fia":6078,"fas":16462,"fat":6791,"far":5672,"fam":16618,"fan":7946,"età":10656,"ezz":10468,"ezi":12836,"eta":25147,"ete":11093,"eti":19239,"esp":10424,"eso":7266,"est":82383,"ess":82490,"eto":8361,"etr":19381,"ett":136219,"eve":12042,"eva":15707,"evo":6501,"evi":15209,"eur":6939,"ey ":9468,"er ":129403,"eor":7000,"es ":37108,"epu":6598,"eri":107300,"erg":9054,"ere":60624,"erf":7449,"erc":19250,"era":90400,"et ":14734,"equ":6340,"esi":40808,"esc":25003,"ese":123684,"esa":17222,"erz":5997,"erv":15974,"err":35138,"ert":35208,"ers":54254,"ern":33474,"erm":28661,"erp":7756,"ero":53515,"en ":21820,"ela":13747,"ele":33688,"eli":13565,"ell":470580,"elo":7068,"eo ":15585,"emb":16989,"ema":19885,"eme":23252,"emo":10570,"emi":32972,"emp":19288,"ene":64829,"eng":5768,"ena":16730,"end":39915,"enc":6461,"eno":18323,"enn":25781,"eni":17944,"enu":8919,"ens":28995,"ent":321381,"enz":41200,"egl":30932,"ego":7039,"egn":18128,"egg":11766,"egi":67882,"egu":13973,"el ":344686,"giu":12377,"gis":8322,"gin":21943,"gio":125636,"gic":7947,"gia":27457,"ght":6164,"ghi":6769,"ghe":8042,"ggi":57223,"gge":11031,"gi ":14154,"gen":34367,"get":10150,"ger":10735,"ge ":13833,"gar":8686,"gat":8038,"gan":16471,"ga ":14336,"fra":61392,"fu ":20614,"for":37768,"fon":16839,"fic":48822,"fig":8947,"fil":22511,"fin":27532,"da ":136024,"de ":72497,"dal":94565,"dai":6473,"dat":26742,"dar":7604,"dan":11831,"cun":8420,"cul":6650,"cui":21314,"cur":6089,"cla":11488,"cli":30380,"co ":105436,"cog":6308,"con":172605,"col":60224,"com":159257,"cor":35975,"cos":24601,"cop":26258,"cqu":7001,"cre":13257,"cra":6453,"cri":26431,"cro":12281,"cci":18510,"cch":12126,"cco":24015,"cca":13415,"cce":29193,"ch ":7853,"cer":13344,"ces":73387,"cen":46057,"cel":12841,"ced":6856,"ci ":29747,"cha":6310,"cia":75274,"ck ":12816,"cie":22050,"cid":13163,"che":152251,"chi":60026,"cil":6171,"cir":15709,"cis":9575,"cit":48155,"ciu":9832,"cin":18966,"cio":18635,"cip":33626,"ed ":34167,"ebb":8622,"ebr":5973,"ean":5957,"eal":11897,"eat":17904,"ea ":31185,"efi":7167,"ei ":71643,"ega":13314,"edi":40474,"ede":36599,"ecl":13991,"eci":22204,"ece":13292,"ecc":23316,"eca":6015,"ee ":7306,"eco":39931,"dur":10435,"dut":6584,"duz":6137,"dor":7110,"dop":9690,"don":12153,"dov":9865,"dot":18765,"ds ":7341,"due":12229,"dri":5744,"dra":9112,"dre":9661,"dro":6212,"dic":30670,"dia":36200,"der":31418,"des":28656,"det":11258,"dec":5661,"def":6138,"deg":19555,"dei":52282,"del":486810,"den":37113,"deo":6268,"di ":549365,"do ":64324,"div":20925,"diz":13866,"din":21333,"dio":28106,"dip":45197,"dir":19963,"dis":42122,"dit":11525,"die":7033,"dif":12635,"rga":12245,"ri ":84427,"rgi":7768,"rge":8891,"rgo":7492,"ret":51197,"res":79309,"rev":9081,"rfi":5771,"rds":5842,"rea":31808,"rec":19082,"red":9839,"reg":68923,"rem":13549,"ren":40494,"rel":11321,"rda":7163,"rdo":8984,"rdi":19945,"rde":7719,"re ":253820,"rco":11417,"rci":11839,"rch":14217,"rca":18883,"raz":24666,"rd ":22088,"rap":12296,"rar":11138,"ras":20760,"rat":94270,"rav":8469,"rbi":17348,"rai":8283,"rag":15270,"ran":117955,"ram":19333,"ral":33868,"raf":16450,"rad":21432,"rac":15856,"rpr":6371,"rs ":8921,"ros":16960,"rot":12347,"rom":18807,"ron":34271,"rop":27028,"rov":41355,"rod":22675,"roc":16930,"roi":16969,"rol":9837,"rof":8044,"rog":13121,"rno":15812,"rna":22371,"rne":10164,"rni":10544,"ro ":93036,"rma":38511,"rme":10556,"rmi":17565,"rla":7860,"riz":22621,"rio":45747,"rit":48367,"ris":50562,"riv":16323,"rig":24728,"ril":9910,"rin":43294,"rim":44423,"ria":61665,"rib":8788,"ric":75415,"rid":11323,"rie":41936,"rif":9551,"rk ":5699,"rup":18084,"rus":6787,"rut":6215,"rva":7812,"rvi":6676,"rve":5812,"ry ":9246,"rsi":24369,"rso":28714,"rsa":7860,"rse":7118,"rta":21350,"rto":33655,"rte":47947,"rti":80543,"rt ":10303,"rro":11867,"rri":17046,"rre":16669,"rra":22088,"sal":9289,"san":12698,"sat":15313,"sar":8168,"sa ":55092,"rzo":7427,"si ":104255,"siv":14099,"sie":10542,"sid":14077,"sic":27928,"sia":21469,"sit":73146,"sis":21720,"sin":20546,"sio":35066,"sil":9310,"sim":20855,"sig":13260,"scr":20145,"se ":150356,"sca":15235,"sce":19663,"sci":47092,"sch":12701,"sco":34425,"ser":44878,"ses":5630,"set":14434,"seg":23827,"sed":9878,"sec":24038,"sen":42117,"sem":26599,"spo":16816,"spe":38890,"spi":8159,"spa":13861,"sot":9626,"sol":26000,"son":52029,"sop":6399,"sor":13321,"soc":11137,"su ":15907,"st ":20759,"smo":8359,"so ":74484,"sse":52141,"ssa":30610,"sso":53075,"ssi":64743,"ssu":6194,"ste":66836,"sta":130887,"sto":49939,"sti":78547,"stu":12684,"str":82216,"sua":16483,"sud":8747,"suc":8338,"sul":27351,"sup":11429,"suo":21188,"sur":6713,"svi":7639,"svo":5978,"tal":71028,"tag":17728,"taz":13579,"tav":10674,"tat":99866,"tas":10040,"tar":29860,"tan":110605,"tam":12199,"te ":231097,"ta ":297475,"pa ":11956,"pe ":6955,"par":126674,"pat":12618,"pas":6581,"pag":17667,"pal":29532,"pan":6881,"pi ":12873,"pec":11403,"pen":11283,"per":160544,"pet":24231,"pes":8448,"pli":9280,"ple":7659,"pia":16405,"pic":12371,"pie":7531,"pin":8248,"pio":15425,"pir":5858,"pit":11446,"por":30452,"pop":13161,"pot":6913,"pos":33968,"poi":5808,"pon":18443,"pol":43083,"poc":5679,"ppr":9310,"ppi":6671,"ppo":29526,"ppa":24732,"ppe":7033,"po ":47633,"più":26636,"pub":29314,"pra":8794,"pri":70446,"pre":86755,"pro":88068,"put":6255,"pun":7305,"qua":45208,"que":36643,"qui":15702,"ra ":147867,"ngo":19829,"ngl":15051,"ngu":15413,"ni ":109423,"nge":12593,"ngh":7552,"nga":7243,"neg":12898,"nei":14330,"nel":251833,"nen":19286,"nem":6968,"ner":29386,"net":10607,"nes":22568,"ng ":19777,"nea":10437,"nfi":6288,"nco":13364,"nci":58066,"ncl":16907,"nce":63829,"nch":32511,"nca":8638,"ne ":381891,"ndu":5794,"ndr":9579,"ndo":51716,"ndi":44494,"nde":40952,"nda":41143,"nal":55844,"nam":6348,"nan":9718,"nar":20198,"nag":9973,"nd ":21375,"nat":56733,"nas":7852,"naz":16138,"na ":222844,"iù ":27049,"nve":7397,"num":9332,"nut":9355,"nto":104437,"ntr":54646,"nti":140368,"nta":97179,"nte":203922,"nso":6392,"nse":23034,"nsi":31575,"nt ":19040,"nqu":6994,"ns ":8742,"nol":12956,"nom":37789,"non":21278,"not":13854,"nos":16869,"nor":18667,"nov":9880,"nne":24481,"nna":15245,"nno":16579,"nni":28556,"no ":229968,"nif":7757,"nie":7902,"nic":33089,"nia":32535,"niz":16719,"niv":11731,"nis":28026,"nit":37389,"nio":10735,"nim":18502,"ogr":18460,"ogi":20692,"ogo":10374,"ogn":10473,"oge":8216,"ogg":8490,"oi ":11644,"oir":5807,"oid":15471,"ol ":6484,"oce":16535,"och":7177,"oci":18254,"ock":8603,"oco":10221,"oca":18569,"occ":17275,"ode":8882,"odi":13909,"odo":23349,"of ":9685,"oda":9043,"odu":9647,"obi":7836,"nza":33286,"nze":8647,"nzi":18694,"nzo":13784,"oti":8184,"ote":14816,"ott":46439,"oto":17159,"ost":54505,"ota":15242,"osi":22379,"ose":9147,"oss":25102,"oso":10625,"ovi":33581,"ova":26672,"ove":33842,"oun":6402,"our":8721,"opo":34311,"opp":9241,"ope":36127,"os ":8709,"opr":18087,"or ":15387,"orm":33668,"orn":18923,"oro":17981,"orr":13295,"ord":34705,"ore":83573,"org":19011,"ori":84933,"osa":10641,"osc":18331,"ort":43269,"ors":13700,"orb":16181,"ora":29450,"ola":56821,"on ":116999,"oli":48219,"oll":21894,"ole":18360,"olt":34437,"olo":80006,"olu":13657,"ona":73743,"ond":62359,"onc":11700,"onf":10972,"one":239015,"ong":9187,"oni":83803,"onn":10167,"ono":78642,"ons":27379,"ont":70151,"oma":36812,"ome":71129,"omb":8981,"omi":23159,"omm":10870,"omp":38351,"omo":18571,"omu":77250,"la ":502615,"le ":302428,"lci":6000,"lcu":7598,"lab":6177,"lac":10059,"lan":31109,"lam":6780,"lar":23739,"lat":27051,"las":21662,"lav":9746,"laz":14283,"ld ":6147,"lbu":19083,"lpi":7148,"lon":12253,"lom":6131,"lor":18602,"loc":10691,"log":22625,"los":5990,"lme":20869,"lti":14409,"lto":12474,"ltr":18612,"lta":21939,"lte":12298,"li ":126320,"lev":10337,"les":31571,"let":25767,"ler":10315,"lem":6948,"len":14630,"leg":16217,"lo ":116614,"lla":366876,"lle":95889,"lli":26880,"llo":51317,"lm ":18058,"ll ":147686,"lit":48006,"lis":18952,"lio":27570,"lin":48858,"lim":8378,"liz":23775,"liv":6284,"lic":53902,"lia":69213,"lib":8948,"lig":7530,"lie":14089,"ma ":77830,"mag":30019,"mar":19705,"mas":10652,"mal":8482,"man":52234,"maz":7709,"mat":39139,"mba":7166,"mbi":12940,"mbr":16743,"me ":86094,"med":14225,"met":29612,"mes":11952,"mer":34722,"mem":5945,"men":156332,"lup":7071,"luo":8412,"lun":8513,"lus":8431,"mpi":23542,"mpe":12689,"mpr":10131,"mpo":28943,"mpl":10446,"mod":11161,"mon":36802,"mol":12121,"mor":12412,"mos":9186,"mot":7018,"mpa":14258,"mus":15113,"mun":79273,"mi ":19370,"min":44712,"mil":12857,"mis":11697,"mit":12512,"mic":16950,"mia":19829,"mig":18385,"mo ":49530,"mmi":13753,"mma":13393,"mme":10561,"zzo":8684,"zza":55596,"zi ":8253,"ze ":11813,"zaz":5784,"zat":34965,"zon":11265,"zo ":22095,"zia":26147,"zie":6922,"zio":172644,"za ":45565,"tà ":85470,"vve":6371,"via":17063,"vil":14796,"vin":31306,"vic":6909,"vid":11446,"vie":12590,"viz":5900,"vit":13527,"vis":26147,"vo ":24368,"vol":30564,"vor":8888,"vi ":11655,"ver":60261,"ves":9627,"ven":39876,"vel":9674,"ve ":30393,"val":16987,"van":19808,"vam":6468,"var":11852,"vat":11790,"va ":54959,"uzi":17041,"usi":20123,"use":8740,"usc":8066,"usa":11990,"ust":15108,"uss":11959,"uti":17848,"ute":8898,"uta":15229,"utt":27857,"uto":37177,"us ":15875,"ura":49253,"ure":16806,"urg":6288,"uri":11168,"uro":17633,"uog":8121,"uol":7555,"uov":6581,"ur ":9581,"upe":12180,"upp":25887,"umb":6348,"ume":24760,"uo ":15893,"unt":14471,"uni":39506,"uno":18306,"una":118143,"ung":13147,"une":69547,"um ":26907,"ult":16466,"ull":18007,"ula":8409,"un ":268027,"uin":7055,"uis":8264,"uit":22856,"ul ":12226,"ui ":25799,"udi":18251,"ue ":29020,"ucc":13459,"uer":10140,"ues":18932,"uff":8520,"uen":11155,"uel":14803,"ua ":25910,"uat":61583,"uar":12747,"ual":28454,"uan":11940,"ubi":6952,"ubb":30175,"ud ":7866,"uad":8171,"ty ":9152,"tur":35100,"tut":19514,"tui":7703,"tun":9180,"tua":67153,"tud":14358,"ttà":16625,"tre":43645,"tra":110475,"tri":61377,"tru":14883,"tro":62451,"tta":55228,"tte":61167,"tti":63839,"tto":137927,"ttr":17103,"ttu":21997,"to ":551865,"tog":6135,"tos":6930,"tom":8273,"ton":24892,"tol":23856,"tor":90775,"til":16701,"tif":7476,"tie":11090,"tig":6347,"tir":6554,"tit":35734,"tis":12185,"tin":33317,"tim":54555,"tip":9392,"tio":18064,"tia":7439,"tic":92757,"tiz":5615,"tiv":40919,"tem":30137,"ten":60350,"tel":26672,"tea":13828,"tec":11962,"ted":10497,"th ":7445,"tes":26748,"ter":139440,"ti ":212534,"the":11405},"n_words":[55820958,65476626,49460182],"name":"it","type":"latin","flags":["diacritics"]}
diff --git a/contrib/languages-data/lt.json b/contrib/languages-data/lt.json
new file mode 100644
index 0000000..53c8ce9
--- /dev/null
+++ b/contrib/languages-data/lt.json
@@ -0,0 +1 @@
+{"freq":{"Å«du":237,"Å«do":102,"Å«di":671,"Å«da":331,"D":8684,"E":5566,"F":3594,"G":7812,"A":19441,"B":11003,"C":5201,"L":15923,"M":12432,"N":7515,"O":2898,"H":3427,"I":6943,"J":8588,"K":17851,"U":3005,"T":11425,"W":951,"V":11358,"Q":106,"P":20146,"S":16914,"R":10122,"Y":441,"X":1264,"Z":1079,"f":17928,"g":102928,"d":143421,"e":363311,"b":73275,"c":39025,"Fed":229,"a":700709,"n":341806,"o":414082,"l":214716,"m":192290,"j":140065,"k":242084,"h":15703,"i":803156,"w":1450,"v":137211,"Fer":122,"u":266546,"t":305542,"s":495974,"r":352720,"q":282,"p":146221,"z":20490,"y":92511,"x":937,"Å«gn":67,"Å«gi":136,"­":175,"²":445,"Fil":257,"Fin":83,"í":134,"é":265,"ä":96,"á":197,"à":65,"ü":86,"ú":64,"ö":70,"ó":158,"Ä—":94567,"Ä™":9237,"Ä":164,"Ä…":20880,"ÄŒ":1561,"Ä":23545,"Ä«":70,"Ä®":732,"į":24134,"Å":147,"ų":90375,"Ž":2451,"ž":44691,"Å ":7077,"Å¡":70424,"Ū":76,"Å«":25506,"Fal":71,"Far":66,"Å«br":365,"Fab":73,"Eri":121,"Est":230,"Eti":120,"Eur":1489,"Ekv":92,"Ele":243,"Eko":63,"μ":79,"ν":128,"ο":198,"ι":119,"κ":65,"λ":109,"ε":95,"α":204,"Ent":76,"σ":88,"Ï‚":168,"Ï":114,"Ï€":65,"Ï„":94,"ÑŒ":90," l":18168," m":39544," n":26364,"Ñ":147," o":7927," h":2325," i":50039," j":10300,"Ñ‹":100," k":69096," d":33767," e":11412,"ц":67," f":7201,"ч":71," g":24899,"Ñ€":400,"Ñ":408," a":48847,"Ñ‚":349," b":21159,"у":184," c":5176," y":9991," x":101," z":358," u":9812," t":37056," w":140," v":43820," p":74653," s":59667," r":27511," J":8568," K":17825," H":3382,"Gel":228," I":6930," N":7489," O":2834,"Geg":68," L":15878," M":12380," B":10949," C":5111,"Ged":147,"С":93," A":19385," F":3572," G":7762," D":8642," E":5548,"л":327,"к":381," Z":1072,"й":180," Y":439,"и":579," X":1253,"п":122,"о":721,"н":471,"м":156,"г":113," S":16766,"в":327,"Ger":157," R":10064,"б":111," Q":106,"а":757," P":20081,"з":104,"Geo":114," W":927,"Gen":295," V":11305,"е":519," U":2992,"д":172," T":11383," Ä…":118," Ä":811," ÄŒ":1561," Ä—":303,"Gla":64,"Gib":75,"Gin":94,"Gil":90,"Gir":131," ž":11466," Ž":2446," ų":74," Ū":76," Å«":618," Å ":7067," Å¡":19822," Ä®":722," į":17189,"šų ":222,"Gan":127,"Gal":319,"Gam":189,"Gaj":89,"Gau":103,"Gar":244,"Gai":76,"Gab":77,"ÙŠ":99,"Ù„":106,"Ù…":65,"Ù†":73,"ب":63,"ا":149,"ر":65,"Å¡Å«n":177,"Flo":97,"Fra":186,"Fri":67,"Fre":105,"A ":1074,"For":525,"F ":250,"Da":2187,"Cu":139,"Cy":71,"Cl":158,"Co":831,"Cr":212,"Ce":857,"Ch":1267,"Ci":314,"G ":220,"Ed":144,"Dv":187,"Du":605,"Dz":157,"Dy":151,"Do":667,"Dn":85,"Dr":666,"De":900,"Di":2059,"Fe":543,"H ":197,"Fa":428,"Eu":1613,"Ev":98,"Ex":73,"Er":363,"Et":189,"Es":386,"En":323,"Em":143,"Ep":87,"Ei":125,"El":573,"Ek":234,"Eg":301,"Ge":1133,"Ga":1414,"I ":1579,"Fu":220,"Fr":454,"Fo":751,"Fl":218,"Fi":559,"B ":521," С":93,"II ":866,"C ":1065,"Av":215,"Au":2682,"Ar":1944,"At":910,"As":631,"D ":380,"Ba":3819,"Az":731,"Af":806,"Ag":282,"Ab":409,"Ac":198,"Ad":552,"Am":1804,"An":2621,"Ap":788,"Ai":520,"Aj":72,"Ak":583,"Al":1985,"Bu":1123,"Br":1448,"Ca":774,"E ":242,"Bi":838,"Be":1548,"Bo":1206,"Bl":293,"Him":100,"Hip":99,"Kv":191,"Ku":1466,"Ky":76,"Kn":86,"Kl":830,"Kr":1957,"Ko":2668,"Le":2735,"Li":7587,"N ":208,"La":2652,"Lu":489,"Ly":250,"Lo":724,"DÅ«":73,"Me":1902,"Dž":420,"Mi":1792,"O ":471,"Ma":4741,"My":151,"Mu":533,"Mo":1998,"Ni":720,"Ež":273,"Ne":1803,"Na":1866,"P ":639,"Hel":151,"Ny":218,"Nr":561,"Nu":901,"No":779,"Ok":221,"Ol":265,"Om":89,"On":141,"JÄ…":106,"Oc":104,"Od":117,"Of":81,"Hen":73,"Ob":214,"Her":326,"Gi":594,"Gl":239,"Gr":2248,"Go":422,"Gu":450,"Gv":487,"Gy":327,"J ":126,"Ha":879,"He":851,"Hi":517,"Ho":543,"Hu":227,"Hy":110,"K ":521,"Ib":64,"Id":102,"ा":68,"Ig":304,"Im":227,"In":1584,"Ik":112,"Il":691,"AÅ¡":111,"à¥":69,"Iv":75,"Is":624,"It":425,"Ir":401,"Ja":1783,"L ":366,"DÄ—":112,"Iz":149,"Ji":1102,"Je":435,"Jo":1434,"BÅ«":168,"Ju":1787,"Hal":105,"Ka":7199,"M ":415,"Han":93,"Ham":121,"Har":160,"Ki":1470,"Hav":88,"Ke":841,"Us":82,"MÅ«":111,"Ut":266,"Ur":330,"Uo":118,"Up":176,"Un":265,"Uk":462,"Ul":82,"Ug":88,"W ":71,"Ty":98,"Tv":72,"Gyv":237,"Tu":954,"Tr":1269,"To":1061,"Th":580,"Ti":954,"Te":1633,"Ta":3988,"V ":1768,"Sy":113,"St":1651,"KÅ«":349,"Sv":277,"Su":2454,"Wo":154,"Wi":274,"Wh":63,"Wa":211,"We":128,"Vy":520,"Vo":1251,"Vi":4324,"Vl":71,"X ":603,"Va":2935,"Ve":1368,"Uz":69,"Gva":257,"Gvi":230,"IÅ¡":781,"Jį":73,"Pu":558,"Pr":4041,"Ps":108,"KÄ—":213,"S ":1115,"Pe":1585,"Gua":94,"ėžt":110,"Pa":7638,"Gud":68,"Pl":1104,"Po":1535,"Pi":2755,"Ph":112,"ėži":636,"Os":252,"Ot":63,"Op":146,"Or":562,"JÄ—":115,"R ":1502,"Se":1883,"Sc":300,"Si":1578,"Sh":180,"Sn":82,"Sm":286,"Sl":299,"Sk":1024,"Sr":63,"Sp":553,"So":1804,"Ru":2114,"JÅ«":181,"Ry":867,"MÄ—":156,"U ":260,"Jų":127,"Sa":3106,"Re":2289,"Ri":771,"Ro":1443,"LÄ—":71,"T ":426,"Ra":1932,"Gre":184,"Gri":532,"Gra":983,"VÄ—":216,"b ":435,"Už":349,"Gru":297,"Gro":162,"a ":85927,"SÄ—":67,"Yo":68,"Yr":183,"Z ":96,"SÄ…":373,"Gol":68,"SÅ«":95,"TÄ—":65,"Za":514,"Ze":220,"Zi":68,"Zo":74,"RÅ«":230,"aÄ—":112,"i ":72374,"fy":103,"gd":980,"ge":8770,"ga":24984,"Inf":65,"fl":318,"fg":67,"ff":102,"fi":4585,"fr":1236,"aÄ":2434,"fu":1782,"ft":398,"fo":4629,"Int":267,"Ins":77,"j ":778,"bÄ™":413,"bÄ—":6516,"gy":5384,"gz":596,"he":3549,"ha":2884,"gn":1795,"gm":391,"gl":2484,"bÄ…":398,"gi":19462,"gh":277,"gv":1056,"gu":4898,"gt":1721,"gs":555,"gr":11904,"gp":278,"go":7511,"du":8502,"dv":2782,"dy":7581,"dz":385,"g ":2674,"ea":3623,"eb":2049,"ec":2366,"ed":9131,"de":15402,"dg":238,"Ilg":538,"di":31560,"dh":99,"dk":208,"dm":1790,"dl":210,"do":13968,"dn":150,"ds":320,"dr":6349,"ew":210,"ex":138,"eu":1022,"ev":4073,"ey":228,"ez":1754,"fa":1478,"h ":675,"Ind":876,"fe":2397,"eh":155,"eg":6969,"ef":1143,"ee":377,"el":20835,"ek":14743,"ej":2777,"ei":27949,"ep":4431,"Imp":87,"eo":2521,"en":55954,"em":13103,"et":36257,"es":24051,"er":37980,"ca":1365,"bz":182,"e ":76709,"by":382,"bs":381,"br":2910,"bu":7186,"bt":432,"bn":67,"bo":6948,"bj":1259,"bl":3193,"bi":8863,"bd":129,"be":9419,"da":31040,"f ":470,"cy":129,"cu":428,"ct":550,"cr":288,"co":1164,"cm":472,"ck":632,"cl":149,"ci":18836,"ch":6035,"ce":6724,"cc":125,"Iki":75,"c ":600,"az":2975,"ay":393,"ba":18685,"d ":4075,"at":26947,"as":91731,"ar":60917,"ax":94,"aw":113,"av":20318,"au":60068,"ak":21549,"al":70175,"ai":71518,"aj":9247,"ao":400,"ap":19804,"am":34880,"an":55907,"ac":11205,"ad":17624,"aa":279,"ab":6312,"ag":11672,"ah":654,"ae":3599,"af":1358,"nu":16862,"nt":40253,"ns":5838,"iÄ":2386,"nr":387,"np":87,"no":25328,"nn":299,"iÄ—":81,"fų":79,"nz":353,"ny":7794,"nv":1496,"oe":521,"of":2043,"oc":3010,"od":8826,"oa":712,"ob":3847,"om":18799,"on":30734,"ok":15352,"ol":18423,"gÅ¡":230,"oi":681,"oj":37495,"jÄ…":2263,"og":8954,"oh":445,"ot":12743,"m²":434,"os":91367,"gÅ«":892,"ov":9611,"ou":825,"op":6292,"oo":622,"or":24858,"jÄ—":931,"gų":1741,"jÄ™":663,"r ":33992,"ox":93,"ow":356,"oz":1228,"oy":145,"pd":423,"pe":8966,"Ign":235,"gž":880,"pg":158,"pa":38606,"pc":152,"pl":7261,"pm":79,"pn":207,"po":11055,"ph":742,"pi":24691,"kÄ…":1329,"pj":353,"pk":273,"lo":17284,"ln":7253,"lm":1772,"ll":1349,"ls":6461,"dÅ«":239,"lp":828,"lv":2437,"lu":5702,"lt":6648,"dų":1603,"lz":63,"ly":8935,"gÄ™":183,"gÄ—":2321,"o ":113444,"md":246,"ma":41541,"mb":3140,"mg":95,"dž":9497,"me":27128,"mf":234,"mk":64,"ml":224,"mi":27912,"eÅ¡":4905,"mn":423,"mm":415,"mp":5761,"mo":36514,"mt":1706,"ms":5086,"mu":8277,"gį":95,"mz":185,"my":1666,"hÄ—":71,"p ":5826,"na":39063,"nb":286,"nc":4749,"nd":18751,"ne":20163,"nf":1054,"ež":4639,"ng":14841,"nh":168,"ni":75394,"iÄ…":1524,"nj":255,"nk":20248,"nl":177,"nm":383,"jy":130,"dį":460,"ju":6480,"eÄ":2839,"bÅ«":3463,"jo":44031,"ki":36060,"kh":180,"kg":90,"ke":8017,"kd":1358,"kc":1366,"ka":55987,"m ":17519,"bų":1576,"ky":6899,"ks":8217,"cÅ«":948,"kt":10687,"ku":20625,"eį":147,"kv":1386,"ko":29827,"kr":12212,"kl":14764,"km":7611,"kn":695,"li":63975,"lh":105,"lk":4623,"lj":376,"gÄ…":598,"le":15356,"ld":6869,"lg":3321,"lf":324,"la":33178,"lc":275,"lb":5099,"fÄ—":82,"cų":73,"n ":4899,"hr":575,"hs":179,"ht":431,"hu":755,"hi":2459,"hn":698,"ho":2226,"hl":151,"hm":207,"id":19607,"ic":4825,"ib":3663,"ia":66700,"ih":166,"ig":7460,"if":2042,"ie":61163,"hy":403,"k ":2911,"cÄ—":232,"ir":43336,"is":76128,"it":24630,"iu":21162,"iv":9558,"ix":156,"aÅ¡":6163,"ii":607,"dÄ…":519,"ij":49065,"ik":38607,"il":20049,"im":33709,"in":120546,"io":45938,"ip":8649,"je":37765,"až":6015,"ji":8060,"iz":7107,"dÄ—":5783,"dÄ™":233,"l ":5566,"ja":28492,"sÄ…":2757,"xi":80,"pÅ¡":499,"sÄ":2453,"pÅ«":264,"rį":1236,"ww":153,"rÄ™":1878,"z ":658,"rÄ—":10366,"ož":891,"šį ":141,"nž":291,"oÅ¡":750,"wi":168,"rÄ…":1320,"wn":67,"wo":98,"rÄ":361,"ws":118,"vy":6021,"nų":7287,"vz":413,"y ":1490,"wa":302,"we":130,"vl":94,"vk":143,"nÅ¡":207,"vi":38809,"mž":705,"vu":2039,"pį":89,"vr":219,"vs":237,"vn":98,"vo":18294,"pÄ—":6097,"mų":3539,"uz":1745,"ux":118,"uv":14825,"ve":15404,"va":45949,"pÄ™":314,"x ":613,"mÅ¡":101,"ui":3816,"uj":5784,"pÄ…":197,"uk":12269,"ul":12572,"ue":864,"uf":303,"Ä—Å¡r":96,"ug":10980,"lž":345,"uh":117,"ur":34041,"pÄ":97,"us":51555,"mÅ«":487,"ut":11996,"um":9287,"un":10381,"uo":35316,"up":8302,"ty":13326,"lų":4078,"tz":73,"nį":1610,"tu":28297,"tt":447,"tw":77,"tv":4512,"ub":4404,"ua":1611,"ud":10289,"uc":1769,"w ":260,"to":36142,"tn":718,"tm":1109,"tl":1570,"ts":3686,"lÅ«":569,"tr":20341,"oÄ":849,"tp":243,"tg":277,"tf":256,"te":29101,"tk":518,"tj":104,"lÅ¡":275,"ti":64800,"th":1088,"v ":492,"nÄ™":1600,"kų":5621,"nÄ—":25365,"tb":1204,"tc":76,"ta":61331,"su":18269,"sv":2479,"kÅ«":2551,"ss":694,"st":45024,"sy":1283,"sz":67,"sl":4978,"sk":13287,"sn":3503,"sm":4814,"sp":7479,"so":8499,"sr":2349,"nÄ":4435,"sd":318,"sc":661,"sf":460,"se":19656,"sh":403,"sj":177,"nÄ…":1594,"kÅ¡":7202,"si":47811,"rz":154,"jų":7476,"mÄ—":4474,"u ":18088,"mÄ™":301,"sa":23030,"sb":164,"mÄ":71,"rr":385,"jÅ«":2283,"rs":5538,"rt":15475,"lį":874,"ru":13317,"rv":2004,"ry":13171,"rp":4617,"ro":31644,"rn":3965,"rm":7673,"rl":1174,"rk":4790,"mÄ…":3705,"rj":224,"ri":79105,"rh":115,"rg":5620,"iž":716,"rf":267,"re":18683,"rd":4525,"rc":1831,"rb":9032,"ra":68430,"t ":8619,"lÄ™":789,"ių":36065,"lÄ—":8350,"kį":293,"qu":225,"iÅ«":2482,"lÄ":267,"lÄ…":1161,"iÅ¡":22230,"kÄ—":5197,"s ":278368,"kÄ™":593,"px":100,"py":2379,"pt":3572,"pu":6072,"jį":548,"pv":639,"pp":160,"kÄ":96,"pr":22301,"ps":4103,"IX ":219,"zÄ™":74,"zÄ—":693,"už":4062,"yÄ":2372,"vÅ«":491,"vų":1508,"uÅ¡":1079,"tž":178,"IV ":137,"tÅ¡":139,"sų":1592,"vÄ—":5454,"vÄ™":713,"tų":12691,"tÅ«":4346,"vÄ…":321,"rž":1586,"rų":3959,"sÅ«":452,"vÄ":186,"Hor":71,"zg":207,"zi":7116,"rÅ¡":2441,"zb":85,"zd":1810,"ze":985,"tÄ™":1071,"za":3048,"Hom":79,"yz":303,"Hon":129,"tÄ—":5769,"Hol":87,"zy":69,"rÅ«":3551,"zr":101,"uÄ":735,"zu":1165,"tį":886,"zo":1897,"zn":153,"zk":67,"zm":1647,"zl":136,"pž":94,"yg":3318,"ye":94,"yc":258,"yd":2201,"ya":257,"yb":11044,"sÄ™":282,"sÄ—":3998,"pų":330,"yv":6788,"sį":230,"yt":11083,"ys":9392,"yr":12733,"yp":1504,"yo":203,"yn":5647,"ym":5531,"yl":2089,"yk":7720,"yj":6842,"tÄ…":2285,"yi":98,"Arg":584,"Are":73,"Arc":130,"Ara":258,"Arm":184,"Ark":155,"Ari":111,"Å¡Äi":2647,"Apo":92,"Atl":385,"Asu":66,"Ast":264,"Art":193,"Avi":81,"Aut":259,"Aus":646,"Auk":724,"Aug":852,"Ats":131,"yž":529,"Aze":82,"zÅ«":96,"Azi":595,"zų":230,"yÅ¡":830,"Å¡Ä— ":158,"Å¡Ä—s":100,"Å¡Ä—j":66,"Bag":85,"Bah":87,"Bai":88,"Bak":129,"Bal":1866,"Ban":272,"Bab":93,"Bad":64,"Bar":458,"Bat":90,"Bas":130,"Bau":99,"Abi":168,"Ada":67,"Ach":87,"Adm":203,"Adr":71,"Aga":104,"Afr":719,"Å¡Ä… ":73,"Air":332,"Aka":93,"Akm":194,"Ala":187,"Alb":193,"Alg":65,"Ali":208,"Ale":201,"Alt":143,"Aly":220,"Alk":99,"Alp":174,"Ame":1274,"Ama":241,"Ang":426,"Ank":145,"Ana":165,"And":549,"Any":154,"Ant":913,"Api":205,"Apa":86,"Buv":78,"But":90,"Bul":113,"Bur":231,"Bud":102,"Bue":112,"Bui":103,"Bru":71,"² ":443,"Cac":84,"Cad":64,"Cal":67,"Cam":66,"Car":189,"Can":84,"Cap":73,"Bet":72,"Ber":374,"Ben":312,"Bel":409,"Baž":77,"Bil":78,"Bir":299,"Bla":117,"Bre":103,"Bra":609,"Bro":169,"Bri":429,"Bol":312,"Bon":72,"Bor":174,"Bos":142,"Bot":70,"Dei":68,"Del":74,"Dem":131,"Dep":64,"Deg":63,"Dam":137,"Dan":260,"Dar":403,"Dau":478,"Dab":105,"Dai":135,"Dal":128,"Cho":100,"Chr":185,"Che":286,"Chi":117,"Chu":119,"Cit":67,"Cen":563,"Cer":118,"Cha":378,"Cro":65,"DP ":99,"Cor":127,"Com":218,"Col":154,"Con":133,"アアア":149,"FA ":185,"ón":71,"Dze":98,"Egi":189,"é ":79,"Dni":78,"Daž":276,"Ä… ":16360,"Dia":66,"Did":1104,"Dis":102,"Dir":98,"Din":76,"Die":275,"Dub":146,"Dun":110,"Dvi":80,"Dus":63,"Dva":68,"Dru":132,"Dri":158,"Dra":241,"Dod":83,"ES ":63,"Don":122,"Dom":99,"Nem":374,"Nev":213,"Neu":81,"Net":114,"Ä™s":3467,"Ner":231,"Nep":196,"Ä—z":158,"Ä™ ":5692,"Ä—c":117,"Ä—d":1595,"Ä—l":4129,"Ä—k":704,"Ä—n":2911,"Ä—m":2033,"Ä—g":1127,"Ä—j":12119,"Ä—t":2531,"Ä—s":37968,"Ä—v":406,"Ä—p":181,"Ä—r":671,"Äų":69,"Nat":155,"Nau":859,"−":522,"Nig":114,"Niu":165,"Å«Å¡y":191,"Nik":109,"Å«Å¡i":1752,"Ä—Ä":63,"Eže":263,"New":68,"ėž":954,"Ä—Å¡":258,"Nar":128,"Nam":132,"Nag":68,"Nac":163,"Nuo":622,"Ä…l":250,"Ä…j":1470,"Ä…m":118,"Ä…n":83,"Ä…s":1224,"Ä…r":696,"Ä…v":409,"Nyd":143,"ÄŒi":814,"ÄŒe":428,"ÄŒa":214,"Äi":21375,"Äk":154,"Äe":803,"Äa":249,"Ä ":409,"ąž":190,"Äo":121,"Äu":90,"Ä— ":26547,"ųjų":1526,"OS ":98,"Nr ":561,"Nov":113,"Nor":386,"Not":108,"ÄÄ—":116,"įž":184,"JÄ… ":106,"Oke":81,"Och":90,"Obe":83,"Ä® ":110,"į ":15271,"Ä®e":85,"Ä®k":141,"įg":465,"įl":469,"įk":1035,"įm":672,"įd":140,"įe":122,"Oli":102,"Ä®s":221,"įr":905,"įp":182,"įv":1510,"įs":2207,"įt":836,"Ori":91,"Org":75,"Oro":67,"Ost":83," −":501,"Po ":75,"Plu":194,"Å¡ ":7298,"Plo":267,"Pli":66,"Ple":71,"Pla":439,"Pin":159,"Pil":450,"PaÅ¡":81,"Pir":463,"Pie":1313,"Pic":67,"šį":142,"Per":807,"Pet":236,"Pen":132,"Pel":183,"Å¡Ä":2693,"Å¡Ä…":98,"Å¡Ä—":408,"Pat":292,"Pas":767,"Par":1014,"Pav":466,"Pau":206,"Pag":484,"Pab":107,"Pad":139,"Pan":697,"Pam":122,"Pap":991,"Paj":94,"Pal":1658,"Pak":297,"Å¡g":221,"Å¡e":5219,"Å¡d":199,"Å¡a":5331,"Å¡b":74,"Å¡o":2052,"Å¡p":366,"Å¡m":981,"Å¡n":613,"Å¡k":10936,"Å¡l":1712,"Å¡i":16792,"Å¡v":1932,"Å¡u":1367,"Å¡t":7877,"Å¡s":1233,"Å¡r":983,"Å¡y":1194,"Å e":341,"Å a":886,"Å l":99,"Å o":70,"Å i":3391,"Å k":111,"JÄ—z":95,"Å u":220,"Å t":155,"Å v":1543,"Å r":75,"PrÅ«":117,"KÄ—d":210,"Å ":76,"Pun":96,"Pus":118,"Pue":76,"Pro":479,"Pri":882,"Pre":234,"Jį ":73,"Pra":2269,"Pod":91,"Pol":415,"Pon":120,"Pot":100,"Pos":67,"Pop":119,"Por":237,"žr":232,"žs":417,"žt":693,"žu":2236,"žn":2332,"žo":2063,"žp":171,"žv":1312,"žy":1527,"žb":100,"že":6838,"žd":1624,"ža":3699,"žk":289,"žm":1670,"žl":158,"ži":16869,"Žm":91,"Žy":116,"Žu":122,"Žv":178,"Ža":482,"Ži":399,"Že":955,"RS ":294,"IÅ¡s":132,"IÅ¡t":197,"ž ":1080,"žų":376,"IÅ¡ ":185,"žį":113,"žą":94,"žė":568,"SA ":111,"Å«Äi":238,"Rad":254,"Rai":78,"Raj":66,"Rag":90,"Ram":470,"Ran":143,"Å« ":138,"šų":238,"Å¡Å«":280,"Å«g":553,"Å«d":1733,"Å«b":446,"Å«z":1060,"Å«s":1429,"Å«t":1461,"Å«v":180,"Å«p":172,"Å«r":9049,"Å«k":1535,"Å«l":283,"Å«m":598,"Å«n":4467,"ų ":88718,"Å«Ä":245,"ųj":1568,"Å«Å¡":2032,"ūž":93,"Ita":399,"Isp":323,"Isl":135,"įpr":139,"įma":63,"įmo":598,"Ira":230,"įsi":807,"įsk":80,"įkÅ«":191,"įst":1175,"įta":351,"įte":278,"įtr":77,"įtv":117,"įra":525,"įre":301,"įro":74,"Izr":81,"įga":117,"įgy":242,"DÄ—l":85,"įei":119,"įdu":68,"įku":691,"Jav":80,"Jau":108,"Jas":71,"įla":456,"Jap":569,"Jan":213,"Jam":221,"Jak":178,"Jel":102,"Jer":74,"Jis":502,"Jie":135,"Ji ":425,"ã‚":83,"Jo ":148,"Ä®st":114,"Ä®si":88,"Jos":270,"Jon":566,"ã‚¢":227,"Joh":97,"BÅ«d":72,"Ä®ku":124,"Jug":66,"Jup":99,"Juo":530,"Jur":232,"Juk":63,"Jun":554,"Ä®ei":85,"Kad":98,"Kab":68,"Kai":1116,"Kam":412,"Kal":1098,"Kap":211,"Kan":625,"Kau":985,"Kat":242,"Kas":171,"Kar":1509,"Kaz":340,"国 ":82,"Ker":168,"Ket":93,"Ken":88,"Kel":250,"Kir":183,"Kit":124,"Kin":537,"Kip":63,"Kie":93,"Kil":126,"Kli":79,"Kle":64,"Kla":523,"Klu":106,"Kon":597,"Kom":332,"Kol":450,"Kos":140,"Kor":624,"Kop":80,"Kov":100,"LR ":167,"Kre":575,"Kra":414,"Kri":437,"Kro":173,"Kru":166,"Kry":125,"Kub":74,"Kul":193,"Kun":174,"Kup":188,"Kur":371,"Kva":100,"Les":78,"Lep":69,"Leo":93,"Len":1971,"Lei":123,"Lau":223,"Laz":170,"MC ":504,"Lai":279,"Lat":571,"Lao":67,"Lap":196,"Lam":138,"Lan":215,"Lab":226,"Kuž":106,"La ":159,"Lib":176,"Lia":138,"Lie":6085,"Lig":81,"Lim":100,"Lin":276,"Lit":89,"Liu":198,"Liv":74,"Luk":93,"Lua":75,"Los":103,"Lot":96,"Lon":143,"Å«gÅ¡":200,"Å«s ":568,"Å«pi":97,"Å«ra":1708,"Å«se":158,"Å«ry":183,"Å«ru":361,"Å«ro":3121,"Å«ri":1981,"Å«ks":471,"Å«ka":130,"Å«ki":544,"Å«kl":137,"Å«ma":254,"Å«na":1137,"Å«mi":118,"Å«dž":157,"Lyg":63,"Å«nu":353,"Å«ni":1251,"Å«ne":67,"Å«no":452,"Å«rÄ—":579,"Å«rÄ™":259,"Mek":335,"Men":159,"Mel":164,"Mes":186,"Mer":401,"Met":158,"Med":222,"Mez":64,"Å«zi":829,"Džo":137,"Å«rų":466,"Dža":76,"Dži":72,"Dže":84,"Å«kÅ¡":76,"Å«si":141,"Å«nÄ…":68,"Å«st":200,"Å«nÄ—":216,"Å«ta":74,"Å«te":79,"Man":463,"Mal":406,"Å«ti":914,"Mar":1387,"Mas":288,"Mag":384,"Mad":147,"Maj":79,"Mak":226,"Mah":72,"Mai":142,"Å«ty":109,"Mac":86,"Å«mų":111,"Å«vi":114,"Mau":95,"Mat":219,"Å«nų":861,"Å«rÄ…":276,"Mok":227,"Mol":370,"Mon":426,"Mos":65,"Mor":209,"Mot":313,"Moz":127,"Å«zų":144,"NR ":79,"Mik":194,"Mie":142,"Mia":99,"Mic":168,"Mit":95,"Mir":79,"Mis":99,"Mil":131,"Min":395,"Å«tų":160,"Å«sų":232,"Maž":500,"Mur":145,"Mus":128,"MiÅ¡":117,"SÄ…j":308,"XX ":204,"XV ":71,"кий":66,"Wor":95,"Wil":72,"Win":96,"XI ":73,"War":82,"Vyr":158,"Vyt":150,"Viž":91,"Vok":795,"Vol":265,"ViÅ¡":70,"Vis":391,"Vit":100,"Äkų":69,"Zar":256,"Zel":141,"之":194,"並":122,"三":326,"ä¸":115,"Yra":183,"三三":72,"Sve":90,"Sva":109,"Sur":111,"Sus":251,"Suv":802,"Sum":113,"Suk":74,"Sup":79,"Suo":198,"Sun":92,"Sud":274,"Suc":101,"Sub":64,"Str":249,"Stu":148,"Sti":184,"Sto":157,"Sta":653,"Ste":219,"Ten":81,"Tei":320,"Tel":387,"Tek":128,"Tam":147,"Tan":110,"Tar":749,"Tau":535,"Tai":1790,"Tak":96,"Tal":105,"Tad":89,"Ski":160,"Skr":105,"Sku":184,"Ska":362,"Sim":140,"Sil":118,"Sir":117,"Sin":192,"Sid":156,"Sie":159,"Sib":270,"Sic":64,"Ser":315,"Sep":68,"Sen":642,"Sel":68,"Sem":92,"Sei":374,"TV ":98,"KÅ«n":234,"国":86,"Spa":162,"TS ":71,"Spi":75,"Spe":132,"Spo":74,"Sof":72,"Sok":963,"Soc":156,"Sol":99,"Son":84,"Sos":108,"Sky":69,"Slo":158,"Smi":70,"Sma":84," 三":86,"Ryt":687,"Ryg":105,"Jų ":127,"JÅ«r":171,"Rus":1573,"Rug":83,"Rud":101,"Rum":121,"Sak":119,"Sam":191,"Sal":563,"Sac":85,"Sco":107,"Sav":191,"Sat":80,"Sau":518,"Sar":175,"San":807,"MÄ—n":80,"RaÄ":79,"SI ":77,"Res":1390,"Rio":84,"Rin":77,"Rib":131,"Rie":98,"Ras":159,"Rau":279,"Rec":68,"Red":71,"Rei":156,"Reg":116,"Ren":82,"Rok":173,"Rob":64,"Rod":82,"SR ":568,"Ros":182,"Rom":573,"SS ":87,"Äų ":69," 之":78,"SO ":120,"Vai":313,"Vad":65,"Vel":220,"Ven":367,"Vei":167,"Ñки":73,"Vas":102,"Van":212,"Val":695,"Vak":682,"Var":578,"VaÅ¡":70,"Vid":634,"Vie":542,"Vir":184,"Vil":1774,"Vik":201,"Vin":109,"Ver":344,"Ves":109,"Ukr":322,"Ukm":131,"Uni":208,"Uru":89,"Ura":110,"Ute":210,"UpÄ—":66,"VD ":82,"VI ":165,"Ter":267,"Tet":74,"Tes":113,"The":467,"Tib":113,"Tie":67,"Tik":149,"Til":74,"Tim":101,"Tin":67,"Tir":87,"Tiu":72,"Tor":126,"Tok":232,"Tol":111,"Tom":115,"Ton":98,"Tru":94,"Tro":116,"Tri":330,"Tre":156,"Tra":520,"Tur":537,"Tun":69,"Å¡ga":146,"Å¡i ":168,"Å¡el":123,"Å¡er":281,"Å¡ei":4330,"Å¡a ":145,"Å¡e ":72,"Å¡as":283,"Å¡ar":241,"Å¡au":451,"Å¡ac":68,"Å¡ai":175,"Å¡ak":893,"Å¡al":2584,"Å¡am":125,"Å¡an":259,"Å vÄ":176,"Å ve":932,"Å vi":88,"Å¡to":898,"Å¡tr":99,"Å¡te":875,"Å¡ti":1747,"Å¡kų":474,"Å¡ta":1289,"Å¡un":139,"Å¡uo":95,"Å¡ul":170,"Å¡um":70,"Å¡us":224,"Å¡uj":65,"Å¡uk":112,"Å¡tu":1243,"Å¡ty":912,"Å¡vi":810,"Å¡ve":671,"Å¡va":235,"Å¡ut":236,"Å¡uv":83,"Å¡vy":117,"Å¡pa":109,"Å¡kÄ…":187,"Å¡pl":90,"Å¡on":165,"Å¡or":222,"Å¡os":190,"Å¡iÅ¡":128,"Å¡kÄ—":457,"Å¡re":107,"Å¡ra":138,"Å¡ri":577,"Å¡iÅ«":137,"Å¡ių":982,"Å¡si":699,"Å¡sp":63,"Å¡sk":346,"Å¡ru":91,"Å¡sa":81,"Å¡mÄ—":100,"Å¡ku":531,"Å¡ky":77,"Å¡ko":2173,"Å¡le":715,"Å¡li":309,"Å¡la":532,"Å¡me":157,"Å¡eÅ¡":251,"Å¡mi":172,"Å¡o ":946,"Å¡ma":81,"Å¡mo":366,"Å¡ni":181,"Å¡iÄ…":67,"Å¡ne":70,"Å¡na":94,"Å¡ny":138,"Å¡no":80,"Å¡oj":128,"Å¡ok":166,"Å¡om":149,"Å¡ia":7811,"Å¡ie":268,"Å¡in":2480,"Å¡io":1156,"Å¡il":344,"Å¡im":653,"Å¡ik":111,"Å¡iu":646,"Å¡ir":121,"Å¡is":1523,"Å¡dÄ—":102,"Å¡ka":2905,"Å¡ki":3659,"Å¡ke":337,"Å¡tų":188,"Å¡tÄ—":332,"Å¡tį":112,"Å¡yt":666,"Å¡ys":236,"Å¡ym":121,"Å¡tÄ…":94,"Ä™s ":3114,"Ä™si":184,"Ä™st":139,"bje":1237,"baž":702,"bis":206,"bit":262,"biu":413,"bio":445,"bip":90,"bir":575,"bik":90,"bil":1165,"bim":181,"bin":2312,"bij":590,"bo ":831,"blo":199,"ble":254,"bli":2467,"bla":154,"bod":66,"bok":148,"bol":1634,"boj":747,"bež":91,"biÅ¡":132,"bon":209,"bom":266,"bor":294,"bot":239,"bos":2270,"be ":431,"bam":151,"ban":1542,"bak":168,"bal":1952,"bai":1573,"baj":127,"bac":144,"bad":71,"baz":180,"bau":383,"bat":206,"bas":1718,"bar":1644,"bda":77,"bi ":151,"bei":2412,"ber":929,"ben":2915,"bel":329,"bek":139,"bev":245,"bes":726,"bet":875,"bia":875,"bib":123,"Ä—do":412,"bie":263,"Ä—di":261,"Ä—da":433,"buÄ":70,"bzd":176,"− ":502,"ca ":297,"buž":82,"car":324,"cas":107,"cat":74,"can":137,"cac":140,"cal":88,"ce ":305,"bių":787,"bri":867,"bro":454,"bra":835,"bre":106,"bry":134,"bu ":188,"bru":157,"bso":103,"bse":75,"bta":77,"bst":94,"bti":224,"btr":68,"buo":284,"bur":719,"bul":328,"buk":109,"bum":669,"bui":94,"bud":276,"buv":3564,"but":190,"bus":430,"brÄ—":285,"byl":146,"bys":99,"abų":128,"aka":10783,"am ":1782,"ake":779,"akc":437,"aki":1290,"aji":120,"Ä—za":80,"ajo":4851,"aju":423,"aiz":904,"al ":2864,"adÄ—":694,"aja":2169,"aje":126,"aik":6012,"ail":788,"aim":3610,"ain":2341,"aip":4816,"air":2272,"ais":9853,"ait":1788,"aiv":2833,"Ä—sÄ":100,"ak ":92,"aig":1833,"aid":2977,"aib":279,"ahi":72,"aho":88,"Ä—vi":102,"aj ":639,"abÄ—":174,"agv":331,"agy":177,"Ä—va":79,"aha":210,"agl":74,"agm":87,"agi":657,"Ä—nų":1107,"agr":3158,"agu":296,"agn":741,"ago":977,"anu":591,"anz":118,"any":235,"ano":3129,"ann":125,"anm":118,"ant":17439,"ans":1524,"aiÄ":1223,"anr":199,"ane":1374,"ang":3590,"ani":4762,"anj":123,"ank":4229,"ap ":76,"ana":4088,"anc":1441,"and":6903,"amu":630,"Ä—tų":311,"amt":459,"amy":484,"amz":131,"amm":86,"amo":3569,"amp":1185,"ams":2382,"ami":5086,"adž":530,"ame":5511,"amb":1223,"ama":10174,"ao ":130,"adų":80,"agÄ—":263,"aly":3612,"alv":1945,"alu":1766,"alt":2672,"als":5664,"alp":263,"alo":3517,"aln":4883,"alm":300,"all":289,"alk":1267,"alg":429,"ali":17593,"alc":176,"ald":5080,"ale":3230,"Å an":75,"ala":5633,"alb":4521,"Å al":318,"Å ak":157,"an ":1340,"Å au":91,"Ä—tį":118,"akv":76,"aky":230,"aks":329,"akr":1125,"Å ar":99,"aku":724,"akt":1972,"ako":1771,"akn":187,"akm":413,"akl":356,"aba":2740,"abe":207,"abi":1011,"abl":158,"abo":330,"abr":414,"abs":143,"abu":396,"abz":181,"ae ":3025,"aca":72,"ad ":1165,"ac ":144,"ab ":182,"afo":102,"afr":63,"aft":132,"afi":672,"ai ":28957,"aga":3563,"age":494,"afy":94,"aen":65,"ael":136,"aei":134,"afa":115,"ado":1953,"adr":251,"adm":1262,"adi":7918,"ade":803,"ady":144,"adu":285,"adv":445,"ack":137,"aci":7737,"ach":900,"ace":1807,"ada":1767,"act":213,"azm":131,"azo":425,"azi":1049,"arÅ¡":379,"azl":78,"auÄ":212,"arÅ«":95,"atÄ—":313,"aze":155,"azg":73,"aza":391,"azd":314,"Ä—cÄ—":111,"arų":1990,"arž":560,"apÅ¡":64,"asÄ":85,"arÄ—":673,"arÄ™":105,"atÄ…":166,"Ä—go":219,"Ä—gi":261,"apų":68,"asÄ—":868,"Ä—ga":161,"avų":296,"Ä—kt":249,"Å ta":114,"Ä—n ":149,"Å v ":293,"Ä—la":197,"Ä—li":1870,"auž":306,"ba ":7831,"azÄ—":203,"Ä—ly":383,"Ä—gÄ—":107,"Å ud":88,"Ä—me":99,"Ä—dž":122,"Ä—mi":1028,"Ä—ms":611,"atÅ«":905,"Ä—ja":1671,"Ä—l ":1089,"Ä—dÄ—":151,"Ä—ji":1633,"Ä—je":5476,"atÅ¡":113,"asų":302,"Ä—jo":1675,"avÄ—":225,"avÄ™":277,"Ä—ju":616,"atž":107,"Ä—ki":72,"auÅ¡":245,"atų":809,"Ä—kl":135,"Ä—km":115,"alÄ":210,"alÄ—":817,"at ":1471,"Ä—pa":71,"arg":513,"aiž":139,"are":1358,"ard":2315,"arc":907,"arb":6774,"ara":6684,"arp":3635,"aro":3645,"arn":1263,"Ä—gų":151,"arm":594,"arl":418,"ark":2565,"amÄ…":414,"Ä—jÄ™":513,"arj":115,"ari":8416,"alį":375,"aru":2631,"arv":357,"Å ie":85,"Å ia":1607,"arr":83,"ars":1153,"ajÅ«":107,"art":6116,"au ":2801,"asa":3470,"ary":2268,"ajų":380,"akÅ¡":168,"asi":8082,"ash":79,"ase":867,"asd":82,"aso":710,"asn":256,"asp":307,"ask":1323,"anÄ…":115,"asm":1404,"Ä—s ":35616,"asl":491,"Ä—nu":240,"Ä—no":201,"aos":78,"agÅ«":453,"ajÄ—":352,"ar ":4730,"Ä—ni":505,"agų":187,"apa":1300,"Ä—ne":311,"ape":335,"Ä—na":175,"apd":357,"akÄ…":119,"api":6467,"apg":138,"apl":1752,"apk":227,"apo":1183,"apr":1671,"aps":2502,"apt":510,"apu":404,"apv":201,"apy":1793,"akÄ—":76,"as ":68022,"Å i ":192,"aiÅ¡":684,"alÄ…":244,"Ä—ny":160,"Ä—to":359,"ava":4842,"apÄ—":321,"amų":755,"auz":79,"Ä—ty":82,"aut":4104,"Å ko":96,"avo":3989,"Ä—ta":441,"avi":8394,"amž":667,"Ä—ti":999,"ave":644,"ay ":151,"Å eÅ¡":127,"avy":790,"anų":720,"avu":579,"arÄ":154,"arÄ…":182,"anž":85,"anÄ—":190,"akų":454,"ata":3198,"asu":274,"ast":4236,"ass":235,"anÄ":3200,"asy":280,"asv":107,"Å il":407,"atm":389,"atn":97,"atk":211,"atl":931,"Ä—ra":344,"Å ip":64,"atr":904,"Å io":192,"ato":2724,"Å im":69,"atp":71,"Ä—lÄ—":390,"ate":2024,"atf":194,"Å is":287,"Å ir":132,"Ä—ri":244,"ati":4429,"atg":129,"Å iu":193,"ath":131,"alų":2328,"Ä—sn":115,"aub":219,"att":75,"ats":3002,"alÅ«":263,"atv":1480,"atu":978,"Ä—st":385,"aty":1953,"aul":3204,"aum":246,"aun":1743,"auo":93,"aup":243,"Ä—mÄ—":214,"aur":9854,"aus":15793,"Ä—jų":416,"aud":4737,"Ä—si":184,"aug":7354,"Ä—se":1427,"auj":2881,"auk":5802,"ος":93,"ος ":93,"VÄ—l":110,"Ï‚ ":168,"Ä—Äi":63,"RÅ«d":99,"α ":85,"SÅ«d":82,"アア":188,"ий ":71,"jei":143,"jer":295,"jek":1582,"jen":63,"jet":69,"jev":166,"ji ":3840,"aža":639,"ažd":237,"aže":494,"aži":1113,"ažn":2141,"ažo":516,"ažu":326,"ažy":107,"jad":82,"izÄ—":124,"jas":2376,"jav":90,"jau":1901,"jap":205,"jar":157,"jak":120,"jan":2657,"jam":5082,"jag":120,"jai":1573,"je ":35197,"ažį":104,"ažų":112,"jog":127,"jok":177,"joj":10950,"jon":4897,"jom":502,"jot":90,"jos":20733,"jor":274,"Ñк":151,"jis":383,"jim":2557,"jin":736,"jie":392,"ÑÑ‚":86,"jo ":6159,"itn":63,"itm":290,"itk":77,"ioÄ":215,"itr":489,"ito":5645,"itv":127,"inį":1158,"itu":2883,"itt":74,"ity":2401,"ilų":166,"iub":80,"iuc":232,"ilž":195,"iud":174,"iuj":1926,"iuk":566,"iui":902,"isk":1115,"inÄ…":194,"ism":716,"isl":291,"iso":747,"isp":344,"iss":63,"ikÅ«":653,"inÄ":492,"isu":1429,"ist":10402,"isv":554,"isy":283,"ikų":1702,"inÄ—":20322,"inÄ™":1356,"ita":3101,"ite":2150,"ith":115,"iti":4567,"ivo":225,"inų":1093,"ivy":313,"ivu":89,"irÄ…":66,"inž":95,"ius":6964,"iur":510,"ium":511,"iul":202,"iuo":4507,"iun":199,"imų":1088,"iuz":146,"iut":1006,"iuv":85,"iva":5713,"ipÄ—":458,"ix ":136,"ivi":1601,"ive":1142,"ipr":458,"ipo":522,"ipu":252,"ips":676,"ipt":461,"ipi":440,"ikÄ…":480,"igž":771,"ipl":309,"ikÄ™":172,"is ":52601,"ikÄ—":859,"ion":5845,"iop":388,"ior":133,"ios":8346,"igÅ«":296,"iot":445,"iog":457,"ijÄ…":1411,"ioj":3109,"iok":271,"iol":667,"iom":700,"ijÄ™":103,"ipa":679,"ipe":332,"iov":183,"igų":264,"ioz":68,"ir ":24988,"iru":470,"irv":355,"irs":602,"ijÅ«":154,"irt":3182,"iro":972,"irp":210,"irm":1827,"irn":109,"irk":393,"iri":3779,"imÄ…":1476,"ikÅ¡":1311,"isi":2798,"ish":114,"ise":370,"isd":71,"isc":198,"isa":1339,"iu ":2871,"imÄ—":266,"ijų":2456,"iry":540,"ire":616,"irg":245,"irb":684,"ira":1224,"ird":228,"it ":169,"ilÄ—":771,"ilÄ™":379,"ašų":113,"iuÅ¡":73,"dÄ—t":870,"dÄ—s":969,"itų":1031,"dÄ—j":1386,"dÄ—m":219,"dÄ—n":263,"dÄ—l":1132,"dÄ™ ":83,"ivÄ—":295,"isų":290,"ivų":75,"dÄ™s":150,"ja ":13854,"itÄ…":217,"ipų":79,"isÄ—":1814,"isÄ™":115,"isÄ…":156,"irÄ™":81,"irÄ—":261,"irž":446,"irų":76,"izu":821,"itį":88,"irÅ«":86,"izo":451,"izn":83,"izm":1490,"izi":1251,"irÅ¡":1690,"izg":80,"ize":116,"izd":791,"iza":1796,"itÄ™":494,"itÄ—":412,"dÄ— ":795,"kaÅ¡":89,"kik":107,"kij":2691,"kim":1590,"kil":1486,"kia":4293,"kie":2272,"kiv":63,"kin":5088,"kio":2376,"kip":128,"kir":3853,"kis":920,"kit":2704,"kiu":661,"kaž":63,"km ":5939,"ki ":3521,"kdy":223,"kg ":77,"kea":110,"ked":124,"kei":560,"kel":3590,"ken":372,"kep":138,"kes":439,"ker":661,"ket":958,"ke ":844,"kci":1331,"kda":895,"kdo":201,"kra":4663,"kre":1055,"klÄ—":428,"kt ":364,"kių":2205,"kiÅ¡":1820,"klÄ…":309,"ksa":796,"kse":134,"kry":1180,"kmÄ—":162,"ku ":438,"kro":983,"kru":318,"kri":3232,"kov":891,"km²":423,"kot":816,"kos":8611,"kor":653,"kop":332,"koo":136,"kon":3138,"kom":3814,"kol":1241,"kok":423,"koj":2216,"koh":148,"kod":1418,"ks ":271,"kmu":184,"kme":718,"kmi":91,"kny":405,"kni":155,"kiÄ…":80,"klu":945,"ko ":5671,"kly":141,"kle":472,"kla":7712,"klo":1474,"kli":2387,"bų ":1576,"jyb":83,"jus":1760,"jun":1716,"juo":1236,"jur":161,"juj":263,"jui":107,"jud":642,"bÅ«n":290,"bÅ«k":89,"bÅ«r":626,"bÅ«s":338,"bÅ«t":996,"eÄi":2678,"dį ":451,"ju ":392,"bÅ«d":1044,"kaz":236,"kav":389,"kat":1246,"kau":876,"kar":10769,"kas":6028,"kap":1244,"kan":2442,"kal":11413,"kam":1843,"kaj":116,"kak":332,"kai":11174,"kag":106,"kad":1450,"kac":439,"kab":221,"ka ":5403," Ga":1407,"bÄ—t":104,"bÄ—s":4168,"bÄ—n":81,"bÄ—m":94," Ge":1126,"bÄ—j":670," I ":269,"bÄ—g":144,"bÄ—c":110," Fo":747," Fu":218,"bÄ™ ":399," Fr":454," Fi":557," Fl":215," Ha":877," He":844," Gy":327," J ":93," Go":421," Gr":2243," Gu":448," Gv":487," Gi":589," Gl":237," Ig":304," Id":102," Ib":64,"guž":295," K ":135," Hy":110," Hu":227," Ho":536,"ha ":309," Hi":515," Ji":1102," Je":431," L ":82," Ja":1776," Iz":149," DÄ—":110," Iv":74," Ir":401," Is":623," It":425," Im":226," In":1579," AÅ¡":111," Ik":112," Il":691,"ham":166," M ":181,"han":534,"hai":104," Ka":7194,"haj":70,"hal":255,"hau":98," Ke":839," Ki":1466,"har":550,"has":169,"hat":88," Jo":1429," Ju":1786," BÅ«":168,"hab":63,"had":70," La":2647," Le":2732," Li":7563," Kl":830," Kn":86," Ko":2662," Kr":1957," Kv":191," Ku":1464," Ky":76," Ma":4719," O ":103," Mi":1782," Dž":420," Me":1896," DÅ«":73,"he ":653," Lo":721," Ly":249," Lu":488," Ne":1796,"а ":158," P ":142," Na":1859," Ež":273," Ni":719," Mo":1991," My":150," Mu":530,"hel":177,"hei":83,"hev":142," A ":384,"het":95,"hes":72,"her":871,"heo":356,"hen":157,"hem":647,"hi ":89," B ":160," C ":148," Ap":776," Am":1804," An":2616," Ak":583," Al":1977," Ai":520," Aj":72," Ag":281," Af":804," Ac":198," Ad":550," Ab":408," Ba":3796," D ":97," Az":731," Av":213," Au":2682," At":910," As":629," Ar":1939," Be":1541,"hie":74,"hid":370," Bi":835,"hia":94,"hip":216,"hin":342," Bl":291,"him":102,"hil":142," Bo":1202,"hij":135," Br":1443," Bu":1122,"his":79,"hit":343,"hir":118," E ":80," Ca":760," Ce":856," Ci":314," Ch":1252," Cl":153," Cr":197," Co":818," Cu":130," Cy":71," Da":2178," Di":2044," De":898," Dr":663," Do":658," Dn":85," Dy":151," Dz":157," Du":604," Dv":187," Ed":144," G ":65," El":573," Ek":234," Ei":124," Eg":301," Et":188," Es":385,"hlo":65," Er":362," Ep":87," En":317," Em":142," Ex":72," Eu":1612," Ev":98," Fe":541,"ho ":213,"hma":111," Fa":424," H ":80,"gma":128,"go ":1512,"gme":171," SÄ…":373,"glo":300,"gle":181,"gli":959,"gla":443," Wo":146," Wi":267," We":123," Wa":210,"й ":139," Vy":520," RÅ«":228,"gog":103," Zo":73," Ze":220,"gno":577," Zi":68," TÄ—":65,"gni":373," Za":514," Yr":183,"gne":245,"gna":449," Yo":68," SÄ—":67,"giÅ¡":189,"gpj":199," VÄ—":216,"о ":63,"gr ":135," SÅ«":95,"goj":625,"gom":579,"gol":244,"gon":466,"gos":3067,"gor":332,"got":295,"gov":127,"gu ":264," a ":1580,"gro":326," Už":349,"gry":429,"gru":2484,"gra":4017,"gių":416,"glÄ—":83,"gri":3118,"gre":877," R ":90," JÄ—":115,"gty":102,"glų":187,"gto":154," Os":251," Or":560," Op":146," Po":1526,"gui":113," Pl":1099," Pi":2753,"gum":1158," Ph":102,"gul":484," Pe":1581,"gua":121,"gub":128," Pa":7618,"gue":64,"gst":140," Ny":218," Nr":561," Nu":899," No":770," Ol":265," Ok":221," On":138," Om":89," JÄ…":106,"gti":868," Od":117," Oc":104," Of":78," Ob":214,"gta":465," LÄ—":70," Ra":1919,"gvo":218," Ro":1436,"grÄ…":89," Re":2283," Ri":769," KÄ—":213," S ":144,"guv":99,"gut":67,"gur":151," Pr":4036," Ps":108,"gus":1460," Jį":73," Pu":558,"gun":99,"guo":261," IÅ¡":781,"gvi":98,"gva":685," Sy":113," Sv":277," Su":2443,"grį":216," St":1586," KÅ«":349," Ta":3980,"gsÄ—":181," V ":128,"gyb":293,"gyd":154," Th":577," Ti":951," Te":1623," Tr":1262,"gyl":77,"gyn":197," To":1060," Ry":866," JÅ«":181," Ru":2112," Sa":3099," Jų":127,"е ":76," MÄ—":156," Sh":178," Si":1574," Sc":292," Se":1876," So":1791," Sp":549," Sr":63," Sk":1023," Sl":298," Sm":286," Sn":82,"grÅ«":76," Uz":68," Va":2931," X ":111,"и ":92," Ve":1358," Vi":4303," Vl":71," Vo":1248," Tu":948,"gyv":4200," Tv":72," Ty":93,"gys":254,"bÄ— ":1092,"gzi":495," Ug":88," Uk":462," Ul":82," Un":258," Uo":118," Up":172," Ur":330," Us":82," MÅ«":111," Ut":265," ja":1311,"iai":12600,"iak":2489,"iaj":841,"iam":6822," dÄ—":1157,"ial":2617," iz":109,"ian":3835," ji":819,"iap":102,"ias":3810,"iar":592,"iau":21521," je":150,"iat":599,"iav":1428," im":762," in":5586," ik":3172," il":1403,"ic ":202," aÅ¡":299,"iab":113,"iac":1098," is":1471,"iad":246," it":186," ir":24988,"iag":1048,"ibl":199," ka":22789,"ibi":638,"ibo":741," m ":8180," kg":76,"ibr":360," ki":5307," ke":3545,"ibu":210," jo":2067,"id ":126,"iba":407," bÅ«":2814,"ibe":354," ju":2553," gy":4128," bÄ—":99," ha":363," he":768," gi":1781," gl":388," gr":5317," go":132,"ia ":6457," gu":260," gv":148," k ":116," id":308," ie":140," hi":591," ho":365," hu":138,"iet":17789,"iev":1199," ni":198," ež":2821,"iel":363,"iem":1768," ne":7202,"ien":11441,"iep":530," na":5031,"ier":804,"ies":9563,"ied":842,"ieg":235," my":64,"iek":3632,"iej":2087," mu":1417," mo":7083," mm":210,"ieb":372," ok":189," ol":297," jÄ…":256," oj":106,"ifo":912," od":155," of":644," ob":1166,"ife":169," ny":106,"ifi":598," nu":12007," no":1368,"ifa":104," le":2689,"icr":108,"icu":77," li":4372,"ico":274,"ick":163," la":7451," kv":492," ku":12047,"ici":1931," kt":309,"ich":922," ky":193,"ice":275," kn":370,"ie ":6151," km":6372," kl":2310,"ica":672," kr":4858," ko":8974," me":9020,"idy":563," dž":138," eÅ¡":161,"idu":2683," mi":7229,"idv":63," ml":158," gÄ—":342,"Ñ ":102," o ":1212,"idr":494,"ido":937,"idm":185," ma":4864," lu":92,"idi":2189," ly":1967,"ide":3721,"ida":3570," lo":631," ag":258,"aÅ¡a":470," ab":508,"iid":108," ac":88,"aÅ¡e":79," ad":1540," am":912,"idÄ…":63,"aÅ¡k":334," an":3700," ap":10764,"aÅ¡i":756," ai":453,"aÅ¡o":756," ak":1627,"aÅ¡l":96,"aÅ¡m":98,"iim":317," al":1456," av":222," au":6951," ar":9760," at":6438,"aÅ¡u":283,"aÅ¡t":1920," as":2056," d ":1323,"aÅ¡y":781," ba":4469,"il ":133,"idÄ—":1297,"ija":13519," bi":992,"ije":95," be":6829,"iaž":208,"iji":627," bo":424," bl":292,"ijo":29798,"ieÄ":1593," by":148,"ibÅ«":258," bu":4118,"iju":808," br":918," ca":122,"ibų":249," e ":82,"im ":74,"ika":5325,"ige":301,"iga":1803,"ii ":82,"igl":111,"igm":118,"igh":167,"igi":1406,"igu":322,"igt":307,"igr":206,"igo":701,"ign":206,"dÄ… ":495,"ibÄ—":80,"igy":112,"ik ":1008,"imo":10288," er":623,"imn":310," et":431," es":1712," en":778,"ims":264," em":226," ep":239,"imp":910,"idž":3127," ei":683,"imf":152,"ime":1400," el":1508,"imd":91," ek":915," ef":150,"imi":2582,"ieÅ¡":2115," eg":645," fe":604,"ip ":3767,"inc":1582,"ind":4737,"ina":9499," fa":728,"imt":970,"imu":2073," eu":161," ev":173,"imy":291," fu":1594,"inm":71,"ino":4836," fr":209,"aÅ¡Ä":176," fo":1905,"int":7918,"ins":1597,"inf":613," fl":132,"ine":2681,"iež":504,"ing":5394," fi":1973,"ini":42857,"ink":9182," ge":4705," ga":6719,"ioc":68,"iod":332,"inu":851,"inv":260,"iny":2814,"aÅ¡Ä—":95,"iko":8237," cm":466,"ikm":102,"ikl":4355," co":131,"iki":8010," ce":2733,"ike":332," ch":821," ci":727,"ila":1055,"in ":677," da":11917,"iky":490," cu":64,"ikt":1187,"iku":935,"ikr":1885,"iks":1325," do":606,"ilp":157,"ilo":1045,"ill":367," dr":1386,"ilk":807,"iln":1654,"ilm":942,"ilg":1557,"ilj":190," de":4787,"ili":7113,"ild":935," di":7861,"ile":510,"ima":10774,"imb":476," g ":911,"idų":320,"igÄ—":179,"io ":24537," dv":1810,"ily":138," du":1631,"ils":105,"ilt":617,"ilu":734," dy":1060,"ilv":228," sÅ«":347," zo":229," rÅ«":2140,"hol":535," tÄ—":174,"hom":130,"hon":117,"ко":78," tÄ™":151,"hos":152,"hot":89,"hov":154,"hop":73,"hor":425," yp":378,"ка":91,"ки":93," yr":9482," sÄ—":250,"hni":331,"hno":260," sÄ…":2199," pÅ«":86,"ин":66," ww":64,"ий":86," ož":66,"ра":115,"hua":109,"htt":75," už":2934,"hst":134,"ол":71,"ов":122,"hry":109,"но":71,"hro":216,"ни":79,"hri":103," tÅ«":466,"ht ":128,"на":78,"hra":72," vÄ—":1295," ru":1285," jÅ«":1646," ry":5467," mÄ—":866," jų":977," sa":11017," sf":110," se":4482," sc":162," si":4889," sn":168," sm":866," sl":1116," sk":5183," sr":2054,"hyl":73," sp":3493," so":1538,"ве":64," lÄ—":404," t ":238," ra":8074," re":7211," ri":2152," mÄ…":77," ro":1078," pv":390," pu":2293," jį":257," pr":18161," ps":398," s ":108," px":100," lÄ…":374," iÅ¡":11701," os":113,"hum":232," ov":65,"hun":68," op":537," or":3230," jÄ—":239,"ан":71,"ал":63," pe":3678," pa":29839,"ар":72," pl":4167," po":4807," pi":10334," pj":71," vy":3074," y ":87," x ":92," va":20321," ve":4247," pÄ—":140," vo":600,"cÄ—l":111," vu":113," vi":14126,"ер":72,"ен":65," ty":567," tv":701," tu":2991," mÅ«":318," ur":214," uo":741," up":3500," un":1322," ul":71," ug":910," ta":10594," nÄ—":349,"hyt":188," st":5898," kÅ«":1165," sv":1124," su":14551," tr":3585," lÅ«":96," to":1975," th":302," ti":5451," te":9733,"fes":435,"fer":701,"fed":364,"fen":78,"fek":526,"fel":82," ÄŒa":214," ÄŒi":814," ÄŒe":428,"fga":67,"faz":69,"fas":187,"far":115,"žų ":324,"fan":250,"fak":286,"fal":119,"fai":102,"ezÄ—":113,"fe ":71,"etų":4160,"esų":111,"evÄ—":459,"evų":124,"fa ":109,"esÄ—":92,"erį":83,"etÄ…":266,"erÄ—":268,"erÄ":118," Äe":566,"esÄ":286," Äi":215,"epÅ¡":367,"esÄ…":106,"ezu":126,"erų":398,"erž":348,"etÄ—":1071,"eza":64,"esį":127,"ezo":319,"eze":135,"erÅ¡":197,"ezi":795,"eta":5454,"enÄ™":98,"ete":1059,"elÅ¡":173,"eti":4554,"etn":281,"esp":1704,"esn":1467,"eso":815,"est":4994,"esu":549,"enÄ":626,"ess":157,"esy":202,"enÄ—":1439,"ekų":115,"eud":63,"euk":102,"eto":2887,"etr":2354,"ett":92,"enį":206,"etu":10440,"etv":1625,"ety":134,"ew ":79,"eve":542,"eva":902,"evo":715,"evi":699,"eut":158,"eur":366,"eus":130,"epÄ—":63,"emų":168,"erÄ…":156,"evr":154,"ey ":158,"evy":281,"enų":1350,"epe":177,"epc":101,"epi":513,"eph":64,"er ":2219,"epa":1258,"eot":67,"eor":538,"eom":87,"eol":734,"eop":149,"eon":218,"eiÅ¡":1009,"ekÄ—":269,"es ":8000,"ekÄ™":92,"ept":436,"eps":96,"epu":165,"epo":412,"epr":642,"erk":587,"erl":418,"eri":10880,"emÄ…":219,"erg":1284,"ere":1184,"erf":77,"erc":545,"erd":659,"era":5900,"erb":981," Ä—j":280,"elÄ™":124,"elÄ—":2221,"et ":1493,"ekį":64,"esj":92,"enÄ…":643,"esk":317,"esm":259,"ekÅ¡":67,"esi":2143,"ese":290,"esa":2125,"ery":234,"emÄ—":1363,"ejų":738,"erv":903,"elį":206,"eru":488,"err":173,"ert":1937,"ers":2707,"ern":1320,"erm":1142,"erp":330,"ero":2197,"eki":1989,"ekl":411,"ekm":65,"eko":1042,"ekr":192,"eks":1896,"ekt":4207,"eku":345,"ekv":315,"eky":288,"en ":710,"elb":215,"ela":746,"eld":365,"elf":88,"ele":3424,"eli":8265,"elj":72,"elg":477,"elm":261,"eln":456,"elk":671,"ell":316,"elo":518,"elu":87,"elv":84,"els":353,"elt":670,"ely":474,"eo ":170,"egÄ—":73,"edų":106,"emb":301,"ema":2008,"edž":1549,"eme":1016,"emd":90,"emo":2200,"emi":2274,"emt":191,"emu":703,"emp":946,"ems":853,"emy":348,"ene":2754,"eng":1816,"enb":111,"ena":6485,"end":4440,"enc":1042,"eno":4885,"enm":89,"enk":5159,"enl":92,"eni":6797,"enu":670,"env":1006,"ens":1444,"ent":11928,"enr":107,"eiÄ":280,"enz":118,"eny":1624,"eog":245,"eod":89,"egl":182,"ego":396,"ege":449,"egi":3283,"egz":550,"ebÄ—":292,"egr":181,"egu":769,"egy":91,"ek ":622,"eic":241,"eip":170,"eis":5132,"eir":124,"eim":4571,"eil":568,"ein":1861,"eik":5224,"eid":1499,"eig":1358,"eja":503,"edÄ—":140,"el ":389,"eiz":76,"eit":977,"eiv":749,"ejo":153,"eji":683,"eje":65,"eke":125,"ekc":170,"eka":2352,"em ":150,"ejy":83,"eju":497," į ":8903,"gl ":220,"giu":292,"git":144,"gis":1794,"gir":399," Ä® ":86,"gil":204,"gim":1321,"gij":3190,"gik":123,"gip":251,"gin":4262,"gio":2880,"gie":143,"gia":3193,"ght":134,"bÄ… ":394,"gi ":457,"gen":3944,"geo":416,"get":168,"ger":1023,"ges":352,"geb":197,"gei":69,"geg":248,"gem":66,"gel":1714,"gdo":74,"gdy":714,"gda":73,"ge ":442,"gab":134,"gac":108,"gad":87,"gai":1374,"gas":1974,"gar":1729,"gau":1337,"gat":335,"gav":268,"gam":2232,"gal":9014,"gan":3227,"ga ":2878," įv":1493," įt":810," įs":1931," įr":895," įp":146," Ä®s":221," įd":139," įe":113," įg":452," įk":1033," įl":469," įm":645,"fys":93," Ä®e":85," Ä®k":141,"fut":1079," įž":65,"fun":535,"fto":75,"ft ":191,"aÄa":64,"fra":263,"aÄk":85,"aÄi":1928,"fri":745,"fro":133,"for":3388,"fos":117,"fot":172,"fon":490,"fol":145,"aÄ ":218,"fla":74,"fli":110,"fo ":188,"fic":473,"fig":118,"fij":466,"fil":952,"fik":605,"fin":1043,"fir":67,"fit":88,"fiz":526,"da ":3356,"de ":955,"dac":195,"dad":96,"dab":1199,"dak":155,"dal":5226,"dai":1928,"dag":146,"dae":1242,"dat":189,"das":3048,"dar":5510,"dan":2672,"dam":1100,"dav":1026,"dau":2555,"cul":112,"cto":80,"cti":127,"cta":124,"cus":68,"cur":66,"cko":72,"co ":242,"con":95,"col":102,"com":76,"cor":163,"cos":201,"cop":83,"ciÅ¡":133,"cro":177,"cea":1680,"ch ":208,"cer":417,"ces":658,"cet":70,"cen":3061,"cep":179,"cel":113,"ceg":78,"cha":955,"chu":89,"cia":2358,"ck ":279,"cie":95,"cid":198,"che":1211,"chl":85,"chi":1035,"cho":966,"chm":91,"chn":602,"chs":167,"cht":194,"chr":188,"civ":350,"cij":9068,"cik":433,"cil":163,"cif":247,"cis":123,"cit":220,"cin":3236,"cio":1497,"cip":355,"cm ":459,"ed ":292,"eba":313,"ebe":309,"ebi":367,"ebo":120,"ebr":205,"ebu":212,"eag":73,"eae":1640,"eak":217,"ean":215,"eal":365,"ear":134,"eap":89,"eat":373,"eau":64,"ea ":188,"efi":76,"efo":293,"efa":70,"efe":515,"ei ":3954,"ega":737,"eed":76,"edi":2343,"ede":757,"eda":2475,"edy":257,"edu":236,"edo":437,"edr":226,"eck":65,"ech":882,"eci":846,"ece":95,"eca":89,"ect":147,"eco":118,"dyg":104,"dyk":422,"dym":1573,"dyn":853,"dys":311,"dyt":634,"dyd":692,"dyb":2813,"drÄ—":138,"drÄ…":94,"dvy":133,"dvi":1481,"dve":133,"dvo":211,"duv":159,"dur":2035,"dut":428,"dus":534,"dva":622,"dvÄ—":188,"dzi":233,"dor":1140,"don":1347,"dom":1366,"dol":160,"dok":455,"dow":94,"dov":1045,"dot":521,"dos":2310,"ds ":126,"diÅ¡":216,"deÅ¡":1558,"dmi":1463,"dni":66,"dob":88,"doe":78,"doj":2119,"dum":210,"duo":2102,"duj":255,"dui":167,"dul":493,"duk":700,"dug":90,"dub":205,"dua":83,"dri":926,"dra":1708,"dre":173,"dry":71,"du ":892,"dro":2362,"dru":787,"dge":113,"dic":606,"did":3151,"dia":454,"der":1524,"des":572,"det":124,"dev":201,"deb":268,"dea":123,"ded":1735,"deg":595,"dei":369,"del":1942,"dek":567,"den":3068,"dem":709,"dep":764,"deo":162,"di ":260,"dme":216,"do ":2940,"div":255,"diu":103,"diz":157,"dim":1824,"din":11534,"dio":390,"dip":85,"dir":773,"dis":1529,"dit":234,"die":4136,"dif":150,"dij":4732,"dik":613,"dil":259,"dka":165,"daž":1194,"ižy":171,"rbÄ—":133,"rgu":182,"mÄ… ":3290,"rga":2638,"iža":147,"ri ":2109,"rgl":72,"iži":186,"rgi":812,"rbÄ…":117,"rge":1017,"rgo":391,"ret":1342,"res":1263,"rev":180,"rez":564,"rfi":80,"rfo":87,"raÄ":122,"rdu":515,"rds":64,"rdv":219,"reb":105,"rea":608,"ree":109,"ref":432,"rec":183,"red":291,"rei":3237,"lÄ—Å¡":194,"rej":92,"reg":2937,"rem":808,"ren":2189,"rek":801,"rel":806,"rer":127,"reo":63,"rep":489,"rda":868,"rcu":85,"rdo":569,"rdi":1074,"rde":192,"re ":1146,"rby":63,"rbt":207,"rbu":464,"rco":114,"rci":353,"rch":1033,"rce":143,"raz":577,"rd ":418,"rao":69,"rap":372,"rar":206,"ras":10157,"rat":2588,"rau":2880,"rav":918,"rbi":1182,"rbl":168,"rbo":606,"rba":5655,"rbe":227,"raj":5239,"rai":3447,"rah":84,"rag":1079,"ran":8258,"ram":2837,"ral":3029,"rak":1364,"rab":639,"raf":875,"rae":172,"rad":3322,"rac":2843,"rpt":814,"rpu":483,"rpr":64,"rps":98,"rpo":225,"rkÄ—":212,"rs ":981,"rpe":180,"rpa":257,"rkÄ…":115,"rpi":679,"rgž":70,"ror":154,"ros":5272,"rot":753,"rom":1205,"ron":1539,"rop":2336,"roz":106,"rgų":68,"rou":123,"rov":2454,"row":67,"rob":326,"roa":194,"rod":1480,"roc":897,"roj":2662,"roi":152,"rol":1077,"rok":757,"rof":604,"roe":102,"rog":1825,"rno":408,"riÄ":321,"rnu":106,"rny":471,"rp ":1398,"rna":1385,"rež":195,"rne":498,"riÄ…":492,"rni":813,"rmo":1239,"rmu":712,"rdų":72,"ro ":7225,"rgÄ—":150,"rma":2602,"rme":740,"rdž":190,"rmi":1316,"rly":64,"rlo":116,"rli":270,"rld":79,"rle":65,"rla":454,"rn ":72,"rky":100,"rkv":63,"rku":184,"rkt":264,"rks":74,"rko":615,"rki":550,"rkl":319,"rke":407,"rka":1300,"rbų":71,"mÄ…s":197,"reÄ":569,"mÄ…j":193,"raž":304,"rje":132,"riz":580,"rix":85,"rdÄ—":104,"rip":333,"rio":6643,"rit":7580,"ris":4114,"riv":457,"riu":3503,"rig":767,"rij":7031,"rdÄ…":70,"raÅ¡":3717,"rii":272,"ril":196,"rik":6018,"rin":15816,"rim":1853,"ria":7346,"rib":1061,"ric":629,"rid":769,"rie":6574,"rif":330,"rk ":142,"rož":108,"jÅ«Ä":213,"rsÄ—":70,"jų ":7448,"ryb":1588,"ryd":120,"rui":138,"rug":582,"rud":269,"rur":65,"rup":2436,"ruo":2554,"run":372,"rum":1117,"ruk":710,"ruz":216,"ruv":66,"rus":3390,"rut":405,"rva":500,"rmų":317,"rvi":416,"rve":467,"rvo":162,"rnų":71,"rvu":153,"ry ":248,"rsk":222,"rsl":371,"rkÅ¡":158,"rsi":1214,"rso":508,"rsm":221,"jÅ«n":185,"rsa":416,"rse":236,"rkų":131,"rta":4621,"rst":803,"jÅ«r":1765,"rsu":138,"rtm":86,"rtn":140,"rto":2230,"rte":485,"rth":95,"rti":4648,"rub":105,"rtr":75,"rtu":1225,"rty":412,"riÅ¡":698,"rt ":258,"rių":3254,"mÄi":71,"rmÄ…":324,"rre":91,"riž":94,"rra":105,"lį ":872,"ru ":632,"rmÄ—":253,"sac":151,"sad":119,"sag":124,"sai":543,"saj":93,"sak":1089,"sal":3421,"sam":580,"ryÅ¡":526,"sap":78,"san":3811,"sau":3166,"sat":201,"sas":1914,"sar":840,"sav":6083,"sa ":609,"ryÄ":1308,"rvų":140,"mÄ—g":132,"rvÄ—":96,"rsų":124,"mÄ™ ":264,"rtų":277,"mÄ—t":142,"mÄ—s":1624,"mÄ—l":387,"mÄ—m":110,"mÄ—j":427,"mÄ—n":862,"rtÄ—":217,"mÄ— ":740,"rys":1185,"ryt":5807,"ryo":78,"ryp":869,"ryk":138,"ryl":68,"rym":161,"ryn":188,"ryj":340,"rtÄ…":376,"rtį":75,"nÄ… ":1490,"shi":127,"si ":5878," ê°€":71,"siv":352,"sjo":73,"nÄ…j":98,"sie":2665,"sid":3080,"sic":542,"sib":166,"sia":6343,"kÅ¡t":3974,"sit":1385,"siu":976,"sir":1172,"sis":6798,"sip":235,"sin":3201,"kÅ¡n":209,"sio":4893,"kÅ¡o":166,"kÅ¡l":78,"sil":885,"kÅ¡m":294,"sim":839,"sij":3056,"sik":1642,"sii":91,"sif":333,"sig":288,"sda":106,"sdi":133,"se ":12437,"ryž":454,"sce":120,"sci":144,"sch":242,"ser":955,"ses":253,"set":159,"seu":67,"sez":169,"sh ":92,"sfe":280,"sfo":80,"sei":720,"sed":68,"sep":203,"sen":3202,"sem":199,"sel":216,"sek":790,"spu":1602,"skÄ—":68,"spy":159,"spo":957,"spr":582,"spe":1156,"spi":670,"sjÄ—":92,"spa":2100,"sot":100,"sov":81,"sol":244,"som":660,"son":563,"sop":70,"sor":222,"sos":1164,"sod":242,"sof":371,"soj":156,"soc":1068,"su ":4472,"smÄ—":297,"sru":176,"sro":178,"sri":1777,"nÄi":4255,"siž":207,"sra":172,"sių":1944,"slÄ—":668,"st ":506,"siÅ«":210,"siÅ¡":352,"ss ":140,"sli":978,"slo":811,"slu":296,"sky":900,"sla":1643,"sle":271,"ski":4050,"skl":344,"sko":1593,"skr":2283,"sku":545,"skv":165,"ska":2455,"ske":580,"kÅ¡Ä":2245,"sno":220,"sny":134,"snu":80,"sna":277,"sp ":78,"sni":2039,"siÄ…":193,"sne":141,"smo":495,"smu":706,"so ":3418,"sma":829,"smi":538,"sme":1637,"syb":334,"kų ":5583,"syn":63,"syt":71,"sys":136,"syk":221,"stÄ…":248,"stÄ™":193,"syv":319,"stÄ—":1387,"nÄ— ":11072,"kÅ«g":75,"sse":159,"ssa":96,"kÅ«n":731,"sso":72,"ssi":140,"kÅ«s":84,"kÅ«r":1568,"snÄ—":370,"skų":182,"ste":4835,"sta":11594,"sto":6918,"sti":7618,"snį":163,"stu":1797,"str":5446,"sua":240,"sty":3413,"slų":200,"sud":2168,"sue":135,"sub":489,"sui":78,"suf":148,"sug":205,"sul":523,"sum":815,"suj":169,"suk":1852,"sup":534,"sun":407,"suo":1175,"sut":1183,"sus":2952,"sur":378,"suv":210,"spÄ—":63,"smų":124,"sva":1159,"sve":370,"svi":243,"svo":286,"svu":88,"svy":123,"tai":9707,"taj":152,"tak":2532,"tal":2568,"tag":213,"tab":401,"tac":699,"tad":469,"tav":1620,"tau":2680,"tat":2848,"tas":15287,"tar":6407,"tap":484,"tan":2917,"tam":3320,"te ":2806,"tbo":1115,"nÄ™ ":1530,"svÄ—":126,"ê°€":170,"stų":690,"nÄ—m":645,"nÄ—n":104,"nÄ—l":246,"nÄ—j":3435,"stÅ«":94,"nÄ—s":9359,"nÄ—t":145,"nÄ—r":303,"ta ":7677,"suž":77,"nÄ™s":68,"ovų":204,"jÄ™s":545," Å¡t":72," Å¡v":1206," Å¡u":230," Å¡i":9378,"pa ":504," Å¡l":369," Å¡o":352," Å¡a":3536," Å¡e":4400," Å v":1543," Å u":220," Å t":147," Å r":75," Å o":70," Å l":99," Å k":111,"otų":425," Å i":3390," Å e":340,"jÄ—s":91," Å a":886,"jÄ—g":559,"ovÄ™":68,"jÄ™ ":118,"ovÄ—":1571,"pdo":296,"pci":139,"pe ":192,"par":5103,"pat":3653,"pas":6180,"pav":2954,"pau":1209,"pac":165,"pad":1171,"paa":84,"pab":319,"pag":4995,"pak":1745,"pal":2163,"pai":635,"paj":498,"pap":2360,"pam":953,"pan":2096,"ozÄ—":194,"pha":125,"kÄ… ":1267,"pho":90,"phi":136,"gžd":845,"paÄ":638,"pec":747,"ped":487,"pen":679,"per":4439,"pet":193,"pes":135,"pei":362,"pel":1215,"pek":276,"pla":1821,"pli":2030,"ple":626,"plo":1805,"ply":260,"plu":196,"pkr":163,"paž":694,"phy":268,"pib":499,"pia":510,"pid":180,"pie":9064,"pig":92,"paÅ¡":357,"pij":468,"pik":263,"pil":4079,"pim":702,"pin":2920,"pio":1030,"pir":1825,"pis":533,"pit":515,"piu":812,"poz":347,"pr ":158,"por":1711,"pop":634,"pov":195,"pot":290,"pos":1432,"poj":445,"pog":134,"pom":76,"pon":986,"pok":315,"pol":1919,"pob":138,"poe":104,"poc":74,"pod":139,"po ":1517,"psu":123,"pst":87,"pta":1134,"pse":76,"psi":725,"psn":657,"psk":1854,"ptu":124,"pty":174,"pua":68,"pub":1670,"pte":200,"pti":1385,"pto":309,"plÅ«":91,"pra":5043,"pių":716,"plÄ—":383,"piÅ¡":241,"pjÅ«":231,"pru":94,"psa":387,"pu ":180,"jį ":548,"kÄi":82,"pri":9679,"pre":1608,"pro":5447,"poÅ¡":212,"pož":275,"pyg":410,"pur":225,"pus":2091,"put":192,"pun":96,"puo":365,"pup":123,"puk":97,"pul":563,"px ":100,"pva":210,"pvz":390,"kÄ™ ":172,"kÄ—j":853,"kÄ—n":99,"kÄ—l":172,"kÄ—s":2221,"ptÅ«":85,"kÄ—t":269,"puÅ¡":167,"pyn":242,"pyj":119,"pyl":1154,"pyk":93,"pyv":70,"pyr":87,"pyt":97,"kÄ— ":1457,"prÅ«":156,"puÄ":92,"kÄ™s":421,"lÄ… ":721,"iÅ¡ ":6273,"lÄ…s":439,"iÅ¡d":131,"iÅ¡e":155,"iÅ¡g":193,"iÅ¡a":318,"iÅ¡m":207,"iÅ¡l":1019,"iÅ¡o":261,"iÅ¡n":213,"iÅ¡i":468,"iÅ¡k":9640,"iÅ¡t":673,"iÅ¡v":411,"iÅ¡p":207,"iÅ¡s":1093,"iÅ¡r":686," Ži":399," Že":954," Ža":482," Žy":115," Žu":120," Žv":178," Žm":91," ži":2342," žm":1523," ža":1939," že":2052," žy":592," žv":1024," žo":907," žu":959,"iÅ« ":112," ųj":70,"kį ":292,"lÄi":255,"iÅ«t":153,"iÅ«r":499,"iÅ«n":1343,"iÅ«l":184,"que":92,"qui":80," šį":69,"ių ":35485,"lÄ— ":2188," Å«k":511,"iųj":519,"lÄ—g":91,"lÄ—j":539,"lÄ—d":122,"lÄ™ ":354,"lÄ—v":65,"lÄ—t":585,"lÄ—s":3311,"lÄ—n":507,"lÄ—m":235,"lÄ—l":106,"lÄ—k":280,"ra ":13206,"lÄ™s":405,"ngo":2538,"ngi":2530,"eži":1071,"ngl":1130,"ngv":440,"ngu":1035,"ežu":89,"ngr":450,"ežt":146,"ngt":1224,"ngs":123,"ni ":526,"eže":2899,"nge":541,"eža":160,"nga":2804,"ežd":87,"nha":74,"iÄ… ":1321,"neg":409,"nei":1033,"nel":523,"nek":262,"nen":435,"nem":377,"nep":854,"neo":246,"ner":1909,"net":3370,"nes":2353,"nev":652,"neu":225,"ndy":107,"ng ":636,"nea":242,"neb":233,"nec":126,"ned":530,"nef":72,"nfo":543,"nfl":92,"nfr":63,"nez":420,"nfe":266,"nco":109,"nci":2521,"nce":603,"nch":181,"ne ":5137,"nbu":79,"ndu":1568,"ndr":3362,"nds":72,"ndo":1993,"ndi":4736,"nde":2551,"nda":1898,"nak":285,"nal":3006,"nam":4510,"nan":2181,"nap":314,"nar":1177,"nac":1442,"nad":530,"nae":88,"naf":82,"nag":711,"nai":2792,"naj":138,"nc ":104,"nab":100,"nbe":88,"nd ":453,"nav":974,"nau":2982,"nat":1500,"nas":9068,"naz":237,"na ":6125,"muÅ¡":82,"myr":132,"myn":393,"myl":81,"ê°€ ":71,"mzd":138,"myk":132,"myb":757,"fų ":79,"nyb":586,"ntÄ…":186,"nyj":775,"nyi":66,"nyg":399,"ny ":199,"nvi":1009,"nux":72,"nve":361,"nva":85,"nuk":354,"nul":255,"num":449,"nun":66,"nug":200,"nui":231,"nus":1958,"nut":401,"nuv":180,"nuo":11505,"nur":161,"nty":1460,"ntv":111,"nto":3213,"ntu":1006,"nts":156,"ntr":4642,"nti":13882,"nth":149,"ntg":70,"nta":6698,"nte":3240,"nsu":225,"nkų":1281,"nsp":335,"nso":235,"nst":1864,"nse":182,"nkÅ¡":139,"nsi":456,"nsl":149,"nsk":533,"nsa":277,"nu ":681,"nro":72,"iÄi":2192,"nri":70,"nra":165,"nių":11248,"nt ":3145,"niÅ«":1065,"niÅ¡":1026,"nkÄ…":172,"ns ":1187,"nkÄ—":2543,"nod":224,"nog":120,"nok":84,"nol":827,"noi":70,"noj":1649,"nop":168,"nom":2167,"non":256,"not":507,"nos":4995,"nor":1222,"nov":1035,"ngų":481,"noz":151,"nne":81,"než":127,"nni":71,"nme":211,"nma":125,"neÅ¡":331,"ndž":626,"ngÄ…":233,"nla":91,"ndų":534,"ngÄ—":262,"no ":11621,"ndÅ«":86,"nke":587,"nkl":1886,"nki":5054,"nkc":507,"nka":2908,"nku":626,"neį":137,"nky":223,"nko":1384,"nks":891,"ncÅ«":939,"nkt":1190,"nkr":206,"iÄ…j":181,"naž":94,"nja":76,"ndÄ—":404,"njo":112,"ndÄ…":97,"nij":5825,"naÅ¡":624,"nig":783,"nif":163,"nie":779,"nid":254,"nic":409,"nia":7994,"nk ":388,"niz":2108,"niu":3647,"niv":780,"nis":15940,"nit":387,"nir":72,"nio":7526,"nip":69,"nim":4778,"nin":8141,"nik":1554,"nil":147,"ogs":78,"ogr":2210,"ogu":324,"ogi":3408,"ogl":119,"ogo":286,"ogn":64,"oga":839,"oge":235,"oho":167,"oha":70,"ohe":76,"obÄ—":124,"oj ":68,"jÄ… ":2223,"gÅ¡t":167,"odÄ…":88,"oid":309,"ok ":122,"oju":555,"obÅ«":144,"ojo":4241,"oji":3887,"oje":21105,"oja":6655,"odÄ—":378,"ol ":243,"oce":779,"och":261,"oci":1478,"ock":185,"obs":81,"obu":232,"odg":104,"ode":1122,"odk":163,"odi":1127,"odo":1175,"odr":68,"of ":299,"oda":2306,"oel":84,"oet":66,"oeu":79,"ody":417,"odu":589,"og ":261,"ofi":842,"oft":141,"ofo":158,"ofe":379,"ofa":95,"oa ":103,"nyÄ":782,"oac":66,"oba":242,"od ":72,"oar":69,"oat":135,"obo":229,"obl":199,"obj":1006,"obi":1248,"obe":189,"nyn":1169,"nyk":427,"nyr":226,"nyt":99,"nys":2919,"ntÄ—":672,"nzi":126,"nzo":87,"ntį":195,"nsų":145,"ntÅ«":104,"ntų":1010,"gų ":1699,"osÄ—":119,"orÄ—":417,"ows":92,"orÄ…":78,"orų":130,"orÅ«":115,"ozo":156,"ozi":309,"jÄ— ":190,"otÄ—":267,"oza":345,"otÄ…":235,"olų":162,"oty":482,"otu":369,"ow ":66,"oti":3180,"ote":1275,"ott":72,"otr":264,"oto":2263,"otn":65,"okų":103,"ost":1854,"osu":63,"osv":137,"ota":3370,"onÄ—":1445,"osi":4557,"okÅ¡":697,"osk":117,"onÄ…":313,"ose":10057,"osf":255,"osp":155,"oss":88,"gÅ«r":160,"osm":327,"osl":114,"oso":811,"osn":165,"gÅ«n":307,"ovy":172,"onų":1034,"ovi":2936,"onÅ¡":70,"ovo":1368,"ovu":118,"ovs":154,"omų":264,"opÄ—":75,"ox ":72,"ova":2312,"ove":473,"oun":122,"oup":69,"ous":97,"our":160,"out":83,"opo":1809,"opi":1192,"opl":208,"ope":871,"oph":333,"opa":456,"os ":71850,"okÄ—":208,"opu":410,"opr":78,"opt":255,"ops":178,"ool":67,"ood":82,"or ":364,"ojÄ—":85,"ogų":154,"oor":180,"ork":297,"orl":173,"orm":3710,"orn":376,"oro":1427,"orp":329,"orc":113,"ord":1011,"ore":636,"orf":147,"org":2280,"ori":8425,"omÄ…":125,"omÄ—":233,"ojų":757,"ou ":101,"osa":489,"gÅ«b":364,"ort":1674,"ors":672,"orv":193,"oru":254,"ory":137,"olÄ…":105,"m² ":432,"ot ":142,"olÄ—":893,"orb":205,"ora":1549,"olÄ™":89,"okį":63,"ola":1063,"old":223,"on ":1119,"oli":7420,"oll":172,"olk":163,"olf":87,"ole":1072,"olg":102,"olt":96,"olm":81,"oln":64,"olo":5241,"oly":304,"odų":160,"ogÄ—":746,"olu":523,"oka":1176,"om ":163,"oki":2742,"oke":340,"okr":516,"oks":1989,"oko":845,"okl":512,"oky":4231,"okt":115,"oku":1484,"ona":5705,"ond":604,"onc":437,"onf":314,"one":3050,"ong":677,"oni":5683,"onk":450,"ono":5947,"ons":941,"ont":1350,"onu":771,"onv":131,"ony":229,"onz":95,"oma":4058,"ome":2921,"omb":322,"omi":4219,"odž":830,"omp":1868,"omo":2822,"omu":462,"oms":893,"omy":305,"op ":190,"la ":3891,"kyÄ":84,"ksų":107,"kvÄ—":157,"ktų":516,"ktÅ«":882,"le ":987,"lci":77,"lde":301,"lda":470,"ldo":1438,"ldi":380,"ldu":86,"lab":885,"lac":354,"lad":233,"lag":301,"laj":360,"lai":7095,"lal":283,"lak":544,"lan":3678,"lam":672,"lap":1033,"lar":290,"lat":1022,"las":4962,"lau":4662,"lav":1958,"lay":83,"laz":248,"lba":2195,"ld ":140,"lbe":120,"kyÅ¡":114,"lbi":260,"lbo":1044,"lbu":643,"kvi":414,"kve":103,"kva":547,"kuv":68,"kut":500,"kus":1457,"kur":11466,"kup":293,"kuo":1084,"kun":573,"kum":831,"kul":3442,"krÄ…":130,"kvo":154,"kta":2002,"kte":286,"cÅ«z":939,"ksp":237,"ksu":217,"kst":1638,"ksi":1031,"kso":492,"ksn":481,"ksm":470,"ksl":1822,"kub":70,"kui":142,"kty":770,"klų":780,"ktr":1113,"ktu":872,"kti":2012,"kto":1582,"kyt":491,"kyr":514,"kys":407,"ktÄ—":87,"krÅ«":445,"kuÄ":63,"krų":140,"cų ":73,"kyb":405,"kyd":213,"kyk":2742,"kyj":122,"ktÄ…":149,"kym":1532,"kyl":177,"ksÄ":304,"lpo":78,"lpn":97,"lpi":224,"lkÄ—":206,"ls ":120,"lpt":73,"lok":724,"lon":920,"lom":534,"lop":556,"lor":335,"lod":121,"loc":79,"log":3426,"loj":732,"loi":69,"lpa":143,"los":2610,"lot":1612,"lou":74,"lov":512,"lno":410,"liÄ…":195,"lni":4011,"lež":588,"lne":64,"lob":276,"lny":212,"lnu":597,"lmo":217,"lmi":148,"lme":109,"ldž":460,"lma":561,"lna":1027,"lmu":116,"lti":1386,"lto":827,"ltr":83,"loÄ":162,"ltu":191,"lty":140,"lub":724,"lkÅ¡":220,"lsi":157,"lsk":87,"lso":116,"dÅ«r":138,"lkÅ«":97,"lst":3935,"lsv":208,"lnÄ—":167,"lkų":772,"lta":1286,"lte":434,"lu ":327,"lmÄ—":408,"lsa":135,"liÅ¡":786,"liÅ«":227,"lių":4116,"lt ":76,"lbÄ—":158,"gÄ… ":583,"lgu":66,"lgy":105,"lgo":332,"lge":222,"lbÄ…":82,"lgi":1731,"li ":2175,"lga":584,"laÄ":341,"lfi":69,"lfa":89,"lez":63,"leu":86,"lev":356,"les":733,"let":811,"ler":662,"leo":285,"lep":84,"lem":868,"len":3529,"lek":1990,"lel":445,"lei":2618,"leg":368,"lef":117,"led":360,"lec":77,"ldy":3239,"lls":89,"llu":67,"lo ":4298,"lla":282,"lle":203,"lli":227,"llo":104,"lko":1340,"lku":68,"lks":80,"ln ":170,"lka":889,"lke":67,"lki":640,"lkl":70,"lbų":491,"lje":178,"ll ":250,"lja":125,"lit":2868,"lis":6123,"lip":320,"lio":6518,"lin":10945,"lim":1681,"ldÄ—":95,"liz":1115,"liv":318,"liu":3368,"lic":390,"lid":423,"lia":9617,"lib":160,"lik":3451,"laÅ¡":117,"lij":2779,"lig":956,"lie":4974,"lif":303,"ma ":8966,"gÄ™s":112,"mb ":104,"lvų":129,"mac":657,"mai":3495,"maj":238,"mak":310,"mad":111,"mag":648,"mar":995,"mas":16980,"mal":1077,"mam":360,"man":2648,"maz":236,"mav":409,"mau":158,"mat":2440,"mba":403,"mbl":299,"mbi":672,"mbe":160,"mbr":361,"mbo":656,"me ":4745,"mbu":328,"mdo":70,"mdi":69,"med":2275,"meg":289,"mec":166,"met":7146,"mes":916,"mer":2496,"mem":105,"mel":455,"men":7636,"mei":219,"mez":71,"mfo":93,"lmų":71,"lpÄ—":80,"lva":657,"lve":135,"lvi":553,"luk":80,"lui":136,"lup":118,"luo":731,"lun":291,"lum":619,"lut":328,"lus":1933,"lur":73,"luv":66,"lnų":522,"ly ":90,"lvo":651,"lyb":79,"lyd":329,"dų ":1583,"ltÄ…":68,"lyj":2459,"lyk":200,"lyg":1880,"lsÄ":1443,"gÄ— ":262,"lyv":644,"lyp":129,"lym":88,"lyn":910,"lys":1218,"lyt":728,"gÄ—l":263,"gÄ—j":295,"gÄ—g":66,"gÄ™ ":71,"lvÄ—":163,"ltų":271,"gÄ—r":187,"ltÅ«":1732,"gÄ—s":1188,"mpi":2031,"mph":67,"mpe":889,"mpr":78,"mpo":545,"mpl":367,"mpu":163,"ms ":4671,"mog":880,"moc":111,"mob":923,"mod":511,"mon":3483,"mok":6158,"moj":1912,"mom":231,"mol":407,"mor":477,"mos":8683,"mot":735,"mpa":1170,"mu ":1142,"gį ":94,"miÅ¡":1246,"mių":389,"mt ":92,"mto":342,"mtm":156,"mti":479,"mso":66,"msi":145,"mta":393,"mur":124,"mus":1181,"mut":199,"mui":800,"mul":740,"mum":317,"mun":878,"muo":1470,"muz":1069,"mpÄ—":146,"džo":105,"dža":265,"mga":68,"eÅ¡ ":414,"mi ":2927,"dži":8825,"dže":173,"meÄ":426,"mbÅ«":66,"maž":1405,"min":8866,"mio":695,"eÅ¡o":276,"mil":606,"mim":392,"eÅ¡m":321,"mir":512,"mis":3697,"eÅ¡p":103,"eÅ¡t":271,"mit":1456,"eÅ¡u":171,"miu":166,"mic":65,"mia":1046,"eÅ¡a":283,"mig":185,"eÅ¡e":206,"mie":3684,"mid":149,"mik":627,"eÅ¡k":262,"mij":1021,"eÅ¡i":2201,"maÅ¡":150,"mo ":11787,"mln":124,"mm ":215,"mna":274,"meÅ¡":73,"vėž":564,"tÅ¡a":98,"įve":191,"įva":943,"įvy":373,"tÅ«r":3760,"tÅ«n":64,"tÅ«k":418,"sÅ«n":301,"sÅ«r":75,"sų ":1585,"vÄ— ":1134,"ÄŒek":141,"ÄŒer":143,"vÄ™s":499,"ÄŒiu":96,"vÄ—r":82,"vÄ—s":1539,"ÄŒik":109,"vÄ—d":103,"ÄŒil":228,"vÄ—m":70,"vÄ—n":180,"vÄ—p":105,"ÄŒia":184,"vÄ—j":505,"vÄ—l":1093,"vÄ™ ":211,"rža":90,"rže":425,"rži":124,"ržo":186,"ržu":185,"ržy":332,"vÄ… ":302,"ržų":128,"vÄ ":177,"zra":92,"zmÄ…":73,"uÄi":685,"rÅ¡Å«":168,"Äem":518,"rÅ«d":121,"rÅ«g":209,"tį ":848,"rÅ«t":179,"rÅ«s":316,"rÅ«v":85,"rÅ«k":126,"rÅ«n":147,"rÅ«m":473,"rÅ«p":93,"zmų":92,"zuo":830,"zul":113,"Äia":6305,"Äiu":3945,"Äin":247,"rų ":3880,"Äio":4188,"rÅ«Å¡":1748,"rųj":78,"ÄiÄ…":252,"Äių":6037,"ÄiÅ«":111,"zga":86,"rÅ¡ ":371,"zdu":307,"zdy":84,"zdo":187,"zeu":81,"zen":162,"zel":75,"zer":347,"ze ":133,"zda":224,"zdi":311,"zde":65,"zac":1511,"zai":89,"zam":101,"zan":208,"zal":69,"zar":93,"zau":206,"zav":191,"zas":215,"zos":160,"zot":82,"zon":766,"zol":127,"zo ":283,"zma":593,"zmo":623,"zme":95,"zdž":483,"zna":91,"zmu":127,"rÅ¡a":115,"zia":212,"zie":245,"zid":291,"zic":125,"zij":2812,"rÅ¡i":542,"zin":555,"zil":400,"zik":1298,"rÅ¡k":199,"zio":150,"zis":547,"rÅ¡t":245,"zit":198,"rÅ¡u":471,"yvu":368,"ynų":327,"yvy":146,"yvo":316,"yve":3277,"yvi":885,"yva":1082,"ymų":245,"ytu":3236,"yto":736,"yti":2477,"yta":2174,"ynÄ—":548,"yst":1756,"ynÄ…":129,"ysk":104,"ysl":216,"ysi":1379,"ykÅ¡":318,"yse":638,"sį ":224,"ymÄ—":134,"ymÄ…":285,"yri":1443,"yro":395,"yru":239,"ylÄ—":72,"yra":10061,"yre":153,"ys ":5011,"ykÄ™":197,"ypt":748,"ygų":179,"ypa":451,"yop":72,"yny":165,"ynu":272,"tÄ™s":608,"yvÅ«":473,"yvų":140,"za ":200,"tÄ—l":64,"tÄ—n":94,"tÄ—m":94,"tÄ—j":863,"ytų":2089,"tÄ—s":2264,"tÄ—v":185,"tÄ™ ":460,"yzd":234,"yrų":212,"ytÄ…":75,"tÄ— ":2107,"ytÄ—":146,"ysÄ":138,"yrÄ—":91,"ybi":1941,"ybo":1219,"yda":247,"yde":256,"ydi":267,"ydo":238,"ydr":112,"ydy":152,"ya ":66,"sÄ™s":72,"rįž":117,"ybe":375,"yba":1117,"ydį":315,"ybų":501,"yka":208,"ykd":1149,"yki":766,"ykl":3525,"yko":378,"yks":518,"yku":494,"yn ":82,"yla":244,"yli":1437,"ygÄ…":95,"yll":79,"ylo":78,"yma":917,"ydų":119,"ymi":564,"ydž":370,"yme":67,"ymo":2861,"ymu":353,"yna":1411,"yni":762,"yne":693,"yno":1069,"ygi":962,"ygl":225,"ybÄ…":107,"yga":678,"ybÄ—":5361,"tÄ… ":2180,"ygo":549,"ygu":328,"ybÄ™":375,"yin":78,"tÄ…j":82,"yje":6777,"sÄi":2451,"pÅ«s":102,"rį ":999,"sÄ— ":786,"sÄ—d":208,"sÄ—k":228,"sÄ—j":581,"sÄ—s":2025,"sÄ—t":101,"sÄ™ ":210,"rįs":115,"pų ":324,"sÄ… ":505,"ožy":124,"ože":207,"oža":97,"oži":226,"rėž":276,"sÄ…v":390,"sÄ…r":690,"sÄ…s":356,"sÄ…j":351,"sÄ…m":110,"sÄ…l":240,"sÄ…n":78,"pÅ¡i":366,"rÄ™s":410,"rÄ—g":75,"rÄ—l":160,"rÄ—j":1557,"rÄ—n":386,"rÄ—m":119,"rÄ—t":168,"rÄ—s":6344,"rÄ™ ":1445,"rÄ—d":146,"ww ":76,"rąž":93,"rÄ— ":1022,"www":76,"ws ":91,"rÄi":327,"nžu":77,"rÄ… ":1079,"nži":161,"rÄ…j":82,"oÅ¡t":122,"oÅ¡i":132,"oÅ¡e":197,"oÅ¡a":95,"vyl":67,"vyk":2515,"vyn":610,"vyr":1040,"vyd":92,"vyj":96,"vys":564,"vyt":65,"vyz":235,"nųj":77,"war":74,"viÅ¡":606,"vro":160,"vių":2220,"vsk":219,"vu ":109,"pį ":89,"vus":1133,"vuo":278,"vum":127,"vul":245,"vz ":411,"nų ":7207,"vyb":651,"vož":79,"via":688,"vio":508,"vir":3137,"vik":248,"vil":1104,"vim":2439,"vin":4831,"vig":137,"nÅ¡i":88,"vij":1156,"vic":102,"vid":2656,"vie":10516,"viz":527,"viv":2783,"viu":186,"vit":264,"vis":3495,"važ":200,"vka":106,"vo ":7394,"vež":182,"viÄ":254,"voj":2810,"vol":423,"vok":960,"von":160,"vor":253,"vot":192,"vos":5796,"Ä…ra":686,"vi ":818,"Ä…jį":197,"vaÄ":96,"mži":691,"Ä…jÄ…":384,"ver":3346,"ves":601,"vet":199,"vej":358,"vei":3674,"veg":178,"ven":4988,"vel":635,"vek":95,"ved":568,"Ä…mo":118,"Ä…na":83," − ":501,"ve ":425,"Ä…ly":225,"val":10598,"vak":4952,"van":3055,"vam":251,"vap":91,"var":5998,"vat":733,"vas":2439,"vav":423,"vau":1068,"vaz":104,"vab":435,"vac":68,"vad":6655,"vai":6631,"vaj":535,"Ä…ju":613,"vag":166,"uvų":343,"Ä…ja":230,"va ":1330,"utų":550,"pÄ—s":2048,"pÄ—j":200,"pÄ—m":214,"pÄ—n":79,"uvÄ™":282,"pÄ—d":630,"usų":229,"pÄ™ ":262,"uvÄ—":461,"urž":199,"mÅ«Å¡":172,"mųj":304,"urų":123,"urÅ¡":132,"uzi":1325,"usį":91,"uza":114,"uzd":74,"utÄ—":482,"pÄ— ":2903,"urį":786,"mų ":3232,"usÄ™":91,"usÄ—":442,"usÄ…":115,"uož":251,"urÄ—":7194,"urÄ™":1318,"uoÅ¡":293,"umų":278,"upÄ™":254,"ux ":100,"upÄ—":4550,"uvi":2426,"uvk":94,"uvo":8973,"uva":1382,"uve":227,"uvy":103,"unų":111,"uvu":401,"usl":117,"usm":292,"usk":802,"ukÅ¡":4009,"usi":16156,"mÅ«g":65,"usd":94,"use":242,"usa":1585,"unÄ—":100,"ukų":215,"usy":348,"usv":157,"usu":325,"ust":2671,"uss":74,"mÅ«s":116,"ukÅ«":174,"mÅ«r":92,"usr":197,"uso":1941,"uti":3065,"ute":1211,"uta":1298,"utb":1100,"ulų":95,"uty":105,"uts":108,"utu":465,"uto":2462,"utr":654,"uoÄ":243,"us ":25324,"ukÄ—":207,"ut ":98,"ulÄ—":940,"urb":304,"ura":1290,"urd":135,"ure":433,"urg":609,"umÄ…":619,"uri":13437,"pÄi":95,"urk":677,"urm":174,"urn":819,"uro":2323,"urp":92,"urs":335,"urt":2267,"uru":270,"ulį":98,"urv":125,"ury":263,"ujų":216,"uog":115,"uod":1799,"uob":148,"uop":102,"uon":468,"uol":1850,"uom":2229,"uoj":2916,"ujÄ…":85,"uok":472,"uot":3875,"uos":9387,"upa":260,"ugų":171,"uoz":64,"ur ":478,"ukÄ…":82,"upi":1336,"upe":880,"upo":164,"upr":184,"upy":138,"upt":113,"upu":131,"ump":675,"umu":499,"umi":565,"umo":1874,"uma":2535,"umb":630,"ume":1015,"udž":589,"uly":692,"ugÄ—":81,"uo ":11031,"ugÄ™":74,"unt":286,"uns":125,"unu":78,"unk":1010,"uni":2717,"uno":1046,"unc":101,"und":514,"una":778,"ung":2769,"une":377,"up ":104,"uks":488,"ukr":338,"uku":1424,"ukt":1044,"uko":768,"ukm":63,"ukl":276,"uki":835,"ukc":193,"uke":516,"um ":371,"uka":1363,"ubų":81,"uju":231,"ulv":76,"ulu":205,"ult":2376,"uls":117,"ulp":163,"ulo":319,"ulm":98,"ulk":1598,"uli":4261,"ulg":118,"ule":258,"ula":686,"un ":91,"uid":65,"uik":63,"uil":83,"uin":83,"uis":152,"uic":69,"Ä…vo":220,"uje":2229,"uji":238,"ujo":901,"Ä…ve":118,"Ä…va":67,"uit":183,"uiz":104,"ul ":100,"udÄ—":847,"uja":1817,"ugi":2043,"Ä…si":447,"lži":190,"ugd":727,"uge":478,"ugn":281,"ugo":862,"ugp":256,"ugl":79,"ui ":2863,"uga":3449,"ugy":234,"ugv":81,"ugu":1147,"ugs":210,"ugr":96,"uha":63,"pÄ… ":193,"Ä…st":536,"uda":2525,"ude":508,"udi":1843,"ubo":317,"ubt":123,"ubr":94,"ubu":247,"ue ":115,"uci":1337,"uch":229,"uer":154,"ufo":165,"udu":90,"udr":145,"udo":3114,"ug ":571,"udy":189,"udz":149,"uen":154,"uel":163,"uei":67,"tyÄ":85,"tuÅ¡":104,"ua ":181,"uau":101,"uar":229,"ual":389,"uan":260,"ubi":273,"ubj":203,"ubl":1778,"ube":224,"uba":739,"uac":101,"trų":261,"trÅ«":89,"tyv":1223,"tyg":105,"tyj":2629,"tyk":592,"tyl":66,"tym":1125,"tyn":745,"tyr":732,"tys":1652,"tyt":1130,"tvÄ—":152,"ty ":191,"tvy":70,"tve":795,"tvi":1569,"tva":1876,"tur":3389,"tus":4707,"tut":476,"tuv":8023,"tui":321,"tul":676,"tuk":242,"tun":200,"tum":1081,"tup":188,"tuo":4132,"tub":127,"tua":342,"tud":729,"tuc":1071,"tug":166,"tyb":2995,"lų ":4053,"trÄ—":213,"trÄ…":137,"ts ":284,"tiÅ¡":751,"tre":865,"tt ":65,"tra":6670,"tri":3774,"oÄi":767,"tru":2118,"tro":4397,"nį ":1595,"tu ":2200,"try":1680,"tsa":349,"lÅ«d":70,"tsi":864,"lÅ«n":117,"lÅ«k":105,"tsk":627,"tsp":242,"tst":1087,"lÅ«s":124,"tte":91,"ttp":75,"tme":480,"tma":182,"to ":9437,"tmo":148,"tmi":171,"tni":215,"tne":192,"tp ":76,"tna":162,"tno":94,"tod":622,"toc":126,"toj":3705,"tog":985,"tob":201,"tov":2012,"tos":4057,"tot":663,"toz":74,"tom":2117,"ton":1654,"tok":1091,"tol":2595,"tor":6343,"top":234,"tr ":100,"tpa":67,"tij":3726,"lÅ¡i":179,"taÅ¡":256,"til":880,"tik":5456,"tif":144,"tie":4693,"tig":115,"tir":719,"tit":1932,"tis":9550,"tin":18689,"tim":2115,"tip":1038,"tio":733,"thu":146,"tia":180,"tib":66,"tic":399,"tid":346,"taž":90,"tiz":264,"tiu":112,"tiv":191,"tko":75,"tku":65,"tka":159,"tli":717,"tla":468,"tle":281,"tem":2567,"ten":1501,"teo":504,"tep":206,"tei":4906,"tek":2029,"tel":3046,"teg":305,"tea":212,"teb":306,"tec":578,"ted":149,"tfo":195,"th ":187,"tez":110,"tet":1080,"tes":640,"ter":7958,"ti ":12579,"tga":173,"taÄ":773,"tho":100,"the":357,"thi":68,"tha":124,"yži":382,"zÅ«r":65,"žė ":79,"zų ":230,"žėj":93,"žės":302,"yÅ¡u":129,"yÅ¡k":285,"yÅ¡i":281,"žįs":104,"AR ":460,"AT ":96,"AV ":1248,"zÄ™ ":74,"zÄ—s":255,"BA ":84,"AB ":129,"가가":99,"Žie":168,"Žmo":87,"Žal":202,"Žai":105,"Žem":792,"ža ":218,"vųj":76,"vų ":1432,"Žva":101,"žli":98,"žka":118,"žin":2776,"žim":518,"žik":242,"žir":69,"žio":3609,"žiu":1205,"žis":284,"ždÄ—":292,"žia":4861,"žie":756,"三 ":108,"žpa":73,"žos":282,"zÄ— ":373,"žny":789,"žoj":209,"žol":428,"žod":694,"žmo":1552,"žiÄ…":130,"žni":1016,"žna":456,"žo ":203,"ždž":217,"žde":109,"žda":511,"žas":486,"žba":64,"žai":1499,"žal":557,"žan":496,"žar":129,"ži ":73,"žer":3193,"žes":178,"žet":91,"žei":363,"žel":344,"žem":2279,"žen":275,"ždy":272,"ždu":70,"ždi":76,"už ":1010,"uža":101,"užd":391,"uže":68,"užs":382,"užt":281,"užr":186,"užu":95,"užk":126,"uži":622,"užp":129,"užn":124,"žys":63,"žym":728,"žyg":68,"žyd":166,"žyb":321,"žtv":87,"žud":129,"žuv":1113,"žut":66,"žur":382,"žuo":217,"žve":120,"žva":821,"žvi":225,"žvy":103,"žra":185,"žių":1603,"žiÅ«":506,"žsi":337,"žta":167,"žte":70,"žti":234,"yÄi":2350,"vÅ«n":458,"užė":257,"tųj":219,"tų ":12460,"uÅ¡Ä—":81,"之 ":65,"uÅ¡t":63,"uÅ¡k":90,"uÅ¡i":396,"uÅ¡a":109,"tžv":91},"n_words":[6266541,7160065,6094403],"name":"lt","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/lv.json b/contrib/languages-data/lv.json
new file mode 100644
index 0000000..4442428
--- /dev/null
+++ b/contrib/languages-data/lv.json
@@ -0,0 +1 @@
+{"freq":{"Å«ci":182,"Å«du":26,"Å«dr":44,"Å«do":28,"Å«de":716,"Å«da":90,"D":3059,"E":2166,"F":1831,"G":1835,"A":5495,"B":3355,"C":2145,"L":4567,"M":3974,"N":1829,"O":1507,"H":1820,"I":2327,"J":1801,"K":4616,"U":852,"T":4509,"W":365,"V":3715,"Q":38,"P":4759,"S":5447,"R":3339,"Y":65,"X":234,"Z":1747,"f":8161,"g":29438,"d":58295,"e":131519,"b":30435,"Fed":124,"c":22500,"a":237698,"n":95937,"o":88352,"l":73126,"m":69285,"j":43124,"k":78380,"Fel":22,"h":5764,"Fen":23,"i":199506,"w":553,"v":49552,"u":101354,"Fer":33,"t":118682,"s":178123,"r":127678,"q":293,"p":52161,"z":35062,"y":1297,"x":396,"²":54,"Fil":64,"í":35,"Fin":30,"Fir":23,"é":81,"ç":26,"ä":65,"â":42,"á":40,"ü":91,"ö":83,"ó":42,"Fiz":28,"Ä“":27728,"Ä’":143,"Ä€":409,"Ä":76597,"ÄŒ":300,"Ä":1638,"ı":70,"Ä·":3799,"Ķ":255,"Eze":42,"Ä»":56,"ļ":8048,"Ä¢":128,"Ä£":3732,"Ī":133,"Ä«":34346,"Ä¢er":28,"Ä¢eo":39,"Ä¢en":28,"ÅŸ":30,"ņ":7202,"Å…":124,"Å":55,"Ž":134,"ž":4793,"Å ":1043,"Å¡":16459,"Ū":37,"Å«":7903,"Equ":185,"Eri":30,"Ern":25,"Eur":70,"Eir":595,"Ì":38,"Ele":79,"Eks":36,"Eko":28,"μ":50,"ν":83,"End":31,"ο":116,"ι":60,"κ":31,"λ":61,"δ":42,"ε":43,"η":40,"α":118,"β":23,"γ":49,"ί":37,"Emi":22,"Eli":48,"ω":27,"Ent":23,"ÏŒ":36,"σ":41,"Ï‚":97,"Ï":66,"Ï€":29,"φ":23,"Ï…":22,"Ï„":47," l":9777,"ÑŒ":65," m":8646," n":13651," o":3224,"Ñ":84," h":1222," i":26578," j":3868," k":23420,"Ñ‹":62," d":13444," e":3216,"Ñ…":31,"ц":23," f":3413," g":7521,"ч":56,"Ñ€":268," a":22507,"Ñ":262," b":8086,"Ñ‚":170," c":4249,"у":116," y":33," x":65," z":3907," u":13570," t":13126," w":93," v":19041," q":33," p":25406," s":20595," r":6772,"HK ":23,"Ūde":29,"Л":28,"К":30,"Ð":52,"Ðœ":37,"О":27,"Б":29,"Ð":26,"Ð’":29," J":1795," K":4607," H":1796," I":2309," N":1813," O":1473," L":4536," M":3956," B":3324," C":2058,"Р":35," A":5469,"С":193," F":1822," G":1815,"Ф":23," D":3031," E":2156,"л":187," Z":1739,"к":221,"й":81," Y":64," X":228,"и":382,"п":57,"о":387,"н":271,"м":85," S":5368,"г":53,"Ger":28," R":3325,"в":222,"б":51," Q":37," P":4729,"а":390,"Geo":50," W":350,"з":76," V":3691,"Gen":23," U":847,"е":335," T":4452,"д":133,"šža":24," Ä":501," Ä€":409," Ä":467," ÄŒ":299," Ä’":143," Ä“":282,"HL ":22," ž":311," Ž":134," Ū":37," Å«":565," Å ":1042," Å¡":2083," Ä»":56," ļ":329," Ä·":663," Ķ":254," Ī":131," Ä«":702," Ä¢":128," Ä£":1094," Å…":120," ņ":41,"×™":25,"Gan":49,"Gal":115,"Gam":34,"Gau":47,"Gar":70,"Gai":64,"Fut":176,"Ùˆ":37,"ÙŠ":53,"Ù„":68,"Ù…":47,"Ù†":33,"د":35,"ب":27,"Ø©":22,"ا":101,"س":25,"ر":50,"FrÄ«":41,"Å¡Å«n":142,"Flo":40," Ð":25," Б":28," Ð’":29,"Fra":343," К":29,"Fri":44," Л":27," Ðœ":37," Ð":52," О":26,"Fre":87,"A ":493,"For":93," α":37,"F ":117,"Da":634,"Cu":37,"Cl":54,"Co":292,"Cr":43,"Ce":347,"Ch":135,"Ci":209,"G ":59,"Ed":95,"Eb":22,"Du":106,"Dz":229,"Do":208,"Dr":103,"De":400,"Di":679,"IP ":41,"Fe":227,"H ":468,"Fa":110,"Ez":43,"Eu":85,"Ev":42,"Ex":29,"Er":98,"Eq":185,"Et":48,"Es":44,"En":123,"Em":90,"Ei":677,"El":193,"Ek":93,"Eg":26,"Ge":158,"BÄ":56,"Ga":460,"I ":434,"Fu":230,"Fr":536,"Fo":212,"Fl":70,"Fi":194," о":23,"B ":133," Р":34," С":176," Ф":23,"II ":185,"C ":377," Ñ":29,"Av":74,"Au":759,"Ar":519,"At":405,"As":243,"D ":170,"Ba":1010,"Az":34,"Af":50,"Ag":51,"Ah":37,"Ab":147,"Ac":82,"Ad":148,"Am":328,"An":739,"Ap":260,"Ai":200,"Ak":154,"Al":535,"Hit":49,"His":22,"Bu":240,"Br":526,"Ca":279,"E ":106,"Bi":333,"Be":447,"Bo":341,"Hil":23,"Bl":116,"Him":40,"Hip":27,"Kv":49,"Ku":433,"Kl":267,"Kr":1045,"Ko":785,"GÄ":24,"Le":274,"Li":1187,"N ":114,"La":2106,"Lu":187,"Ly":26,"Lo":258,"Me":656,"HÄ":28,"Dž":311,"Mi":487,"O ":256,"Ma":1352,"My":39,"HÄ“":35,"Mu":257,"Mo":497,"Ni":163,"Ne":339,"Na":331,"P ":315,"Hel":44,"Hei":53,"Nu":37,"No":488,"Ok":85,"Ol":289,"On":42,"Og":53,"Oh":28,"Od":50,"JÄ":93,"Hen":52,"Of":29,"Ob":41,"Her":113,"Gi":68,"Gl":88,"Gr":518,"Go":173,"IA ":51,"Gu":165,"Gv":31,"BÄ“":72,"J ":48,"Ha":442,"He":346,"Hi":235,"Ho":305,"BÄ«":48,"Hu":55,"CÄ“":86,"K ":99,"Ib":23,"Id":25,"DÄ":72,"Ie":161,"Ig":193,"Im":77,"In":597,"Ik":26,"Il":62,"DÄ“":23,"Iv":35,"Is":56,"It":130,"Ir":159,"Ja":606,"L ":193,"Iz":255,"Je":180,"Jo":238,"Hab":30,"Ju":235,"Hal":59,"Hai":24,"Ka":1308,"Han":77,"M ":140,"Ham":27,"Har":76,"Ki":234,"Hav":40,"Ke":175,"MÅ«":173,"Ur":133,"Up":46,"Un":191,"Uk":57,"Ul":29,"Ug":37,"PÄ":146,"W ":36,"Tu":367,"NÄ«":78,"Tr":348,"LÅ«":27,"To":528,"Th":183,"Ti":330,"Tj":24,"Te":412,"Ta":1094,"V ":507,"BÄ“r":58,"GrÄ«":41,"Sy":41,"St":811,"Sv":222,"Su":204,"Wo":58,"Wi":99,"Wa":68,"RÄ":30,"We":47,"Vo":206,"PÄ«":38,"Vu":40,"Vi":1349,"Vl":38,"X ":140,"Va":795,"Ve":528,"PÄ“":222,"Uz":97,"LÄ":40,"Pu":149,"Pr":574,"Ps":29,"Gus":31,"S ":509,"Pe":330,"KÄ":157,"Pa":1460,"Gui":25,"Pl":246,"Po":522,"Pi":543,"Ph":56,"Gul":51,"Os":199,"Ot":132," ا":37,"Ov":28,"Op":65,"Or":179,"R ":242,"šņu":59,"JÄ“":211,"NÄ":75,"Se":449,"Sc":140,"Si":396,"Sh":52,"Sn":22,"Sm":71,"Sl":112,"Sk":318,"MÄ«":29,"Sp":308,"So":292,"Ru":198,"JÅ«":171,"MÄ“":139,"U ":122,"Sa":1320,"GrÄ":24,"Re":684,"MÄ":133,"Ri":309,"Rh":25,"Ro":567,"LÄ«":215,"Qu":25,"LÄ“":28,"T ":118,"Ra":377,"Gre":140,"Gri":112,"Gra":91,"VÄ«":58,"b ":1431,"Gru":54,"Gro":39,"a ":45978,"SÄ“":77,"TÄ":889,"Yo":26,"TÄ“":78,"SÄ«":40,"Z ":47,"RÄ“":60,"SÄ":102,"PÅ«":23,"RÄ«":858,"Gol":28,"Got":33,"VÄ":356,"VÄ“":122,"Za":126,"Ze":604,"Zi":554,"TÄ«":49,"Zo":72,"Zu":34,"God":26,"RÅ«":43,"Zv":222,"Å¡Ä·Ä":29,"i ":20844,"aÄ“":38,"gd":52,"ge":675,"bÄ":1878,"ga":11955,"gb":23,"Ing":34,"Inf":36,"fl":161,"fg":38,"ff":54,"aÄ":32,"fi":2006,"Å¡Ä·Ä“":67,"fs":184,"fr":742,"aÄ":195,"fu":915,"ft":143,"Int":119,"fo":1630,"Ins":32,"j ":431,"aÄ·":196,"gz":413,"bÄ“":351,"Ir ":43,"cÄ":282,"he":981,"aļ":2174,"ha":893,"gn":346,"gm":103,"gl":1302,"gk":23,"aÄ£":126,"gi":1023,"gh":75,"gg":39,"gv":152,"gu":2538,"gt":345,"gs":1978,"gr":3115,"Å¡Ä·Ä«":179,"aÄ«":165,"go":1269,"dt":34,"du":3460,"dv":153,"dy":39,"dz":8558,"g ":378,"Ima":40,"ea":831,"eb":2145,"ec":2362,"ed":5035,"de":6326,"dd":61,"dg":22,"di":7508,"dh":33,"dk":50,"dm":437,"dl":81,"do":4137,"dn":387,"dp":40,"ds":1737,"dr":2632,"ew":72,"ex":66,"eu":138,"ev":3202,"ey":101,"ez":2104,"fa":698,"h ":326,"Inc":26,"Ind":290,"fe":924,"eh":574,"eg":1917,"ef":579,"ee":260,"el":8559,"ek":9435,"ej":1682,"ei":6080,"ep":1928,"eo":1018,"Imp":27,"en":17231,"em":12435,"et":9035,"es":13667,"er":13266,"cb":46,"ca":1325,"e ":9104,"bv":92,"by":23,"bs":478,"br":1873,"bu":2780,"bn":90,"bo":2478,"bp":134,"bj":321,"bk":204,"bl":923,"bi":5030,"bb":37,"bd":41,"be":3600,"da":13240,"f ":227,"cy":80,"cu":688,"ct":320,"cs":106,"cr":118,"co":534,"cp":28,"cm":65,"ck":210,"cl":78,"ci":9940,"ch":496,"ce":3798,"cc":58,"c ":1617,"ZÄ«":28,"az":1867,"ay":138,"ba":7096,"d ":1260,"at":15301,"as":46560,"ar":22107,"Å¡Ä·e":37,"ax":44,"aw":25,"av":4016,"au":11606,"Å¡Ä·a":58,"ak":5105,"al":13877,"ai":16755,"aj":4377,"ao":99,"ap":6109,"Å¡Ä·i":792,"am":7193,"an":13591,"ac":1971,"ad":7331,"aa":123,"Å¡Ä·u":61,"ab":3333,"ag":2079,"ah":438,"ae":799,"af":317,"nu":3766,"nt":6816,"ns":6634,"iÄ":143,"nr":301,"hÄ«":43,"np":87,"no":13676,"nn":366,"nz":252,"ny":68,"iÄ“":45,"nv":1299,"oe":196,"jÄ":6443,"of":937,"DÄn":37,"oc":1480,"od":5344,"oa":231,"ob":1666,"DÄr":22,"om":4560,"on":9439,"ok":3012,"ol":6437,"oi":175,"gÅ¡":291,"iÄ£":203,"oj":3579,"og":1772,"oh":231,"Iga":176,"ot":8233,"m²":53,"gÅ«":417,"os":7017,"ov":1717,"ou":258,"op":3592,"oo":225,"or":8642,"iÄ·":475,"r ":23056,"ox":43,"jÄ“":253,"ow":104,"oz":1358,"oy":26,"pd":241,"pe":2624,"kÄ":9200,"pg":761,"gž":90,"pa":12196,"pb":92,"iļ":351,"pc":59,"pl":2446,"iņ":2232,"pm":337,"pn":123,"po":2922,"ph":171,"pi":8931,"pj":212,"pk":534,"lo":5141,"ln":1238,"eņ":1120,"lm":668,"ll":864,"ls":6050,"dÅ«":151,"lr":53,"fÄ«":68,"lp":774,"lv":2188,"lu":3183,"lt":2348,"lz":417,"ly":68,"gÄ“":101,"o ":13523,"md":123,"ma":11255,"mb":1027,"dž":490,"mg":152,"me":8257,"mf":62,"hÄ":197,"mk":81,"ml":84,"mi":7529,"eÅ¡":2372,"mj":254,"mn":294,"mm":568,"mp":1937,"mo":3385,"mt":1499,"ms":4163,"mv":67,"Iek":27,"mu":4579,"hÄ“":85,"mz":26,"Ied":23,"Ier":30,"p ":1187,"Ies":23,"na":11100,"nb":173,"nc":1308,"gļ":378,"nd":3610,"ne":6367,"nf":496,"iÄ":1355,"ež":1461,"ng":2033,"nh":159,"ni":10637,"nj":103,"nk":1409,"nl":178,"nm":119,"jv":54,"jt":24,"ju":6450,"jr":40,"eÄ":136,"bÅ«":646,"js":1745,"jp":49,"dÄ«":2457,"jn":108,"jo":2718,"jl":29,"jk":33,"ki":2861,"eÄ£":460,"kh":87,"kg":105,"fÄ":86,"ke":1430,"kd":52,"kc":519,"kb":27,"ka":21595,"m ":14604,"fÄ“":229,"ky":24,"cÅ«":51,"ks":6047,"kt":4783,"ku":9917,"kv":498,"ko":8251,"kp":53,"eÄ«":44,"kr":3493,"kk":49,"kl":2204,"dņ":25,"km":1006,"kn":291,"li":12111,"lh":104,"lk":789,"lj":234,"le":5914,"ld":1879,"lg":521,"lf":344,"gÄ":2261,"la":14259,"eļ":2084,"lc":198,"lb":519,"n ":12333,"eÄ·":397,"hr":242,"hs":176,"bÄ«":818,"ht":159,"hu":177,"hi":1019,"hn":345,"ho":773,"hl":97,"aņ":799,"hm":55,"id":6988,"ic":1972,"ib":840,"ia":987,"ih":374,"ig":1651,"if":662,"dÄ":3118,"ie":43851,"hy":49,"cÄ“":786,"k ":3032,"cÄ«":1751,"iq":24,"ir":20819,"is":22114,"it":6916,"iu":210,"iv":1914,"dÄ“":1715,"ix":47,"ii":76,"aÅ¡":1551,"ij":18002,"ik":8262,"il":7272,"im":5227,"in":12037,"io":3152,"ip":1361,"je":2277,"jd":65,"až":1131,"eÄ":550,"ji":2430,"iz":8191,"l ":908,"ja":18785,"xi":24,"pÅ¡":285,"rÄ£":265,"pÅ«":149,"rÄ«":5569,"xt":23,"rÄ“":1505,"ww":72,"z ":3167,"ož":188,"sÄ":1658,"nž":71,"wi":65,"oÅ¡":2487,"Å¡Ä«s":96,"Å¡Ä«r":28,"wo":40,"Å¡Ä«n":152,"ws":45,"vv":135,"vz":34,"y ":470,"wa":100,"pļ":34,"rÄ":7122,"we":66,"vl":36,"vm":57,"oņ":431,"vj":161,"vk":55,"vi":15879,"nÅ¡":154,"vg":67,"vu":1511,"vr":110,"vs":665,"nÅ«":52,"vp":135,"pÄ«":487,"vn":277,"vo":2198,"Å¡Ä«b":168,"uz":3239,"ux":47,"pÄ“":3898,"uv":1138,"uu":163,"ve":6316,"vd":128,"vc":24,"oļ":231,"va":13478,"oÄ·":34,"x ":250,"oÄ£":891,"ui":330,"mÅ¡":154,"uj":549,"uk":2577,"ul":3970,"ue":159,"pÄ":3136,"uf":42,"ug":2904,"lž":37,"uh":49,"oÄ«":103,"ur":8826,"mÅ«":945,"us":8406,"ut":3554,"um":10369,"un":12883,"uo":66,"up":1299,"ty":109,"tz":181,"tu":9581,"tt":1263,"tw":38,"tv":2682,"ub":1129,"ua":219,"ud":1801,"uc":777,"w ":104,"to":9558,"tn":1900,"tm":416,"tl":676,"lÅ«":587,"ts":7290,"tr":9995,"oÄ":89,"nÄ«":2204,"tp":327,"tg":235,"tf":75,"te":8645,"td":205,"tk":771,"tj":107,"ti":16287,"lÅ¡":164,"th":431,"v ":977,"Å¡Ä« ":103,"tb":1541,"tc":58,"ta":17648,"su":2355,"sv":864,"ss":1501,"kÅ«":96,"st":22568,"sy":29,"sz":152,"sw":23,"nÄ“":1546,"sl":2038,"sk":10907,"sn":958,"sm":2964,"sp":4072,"so":1970,"sr":95,"nÄ":276,"mÄ«":785,"sd":451,"sc":379,"sf":321,"nÄ":8240,"se":3495,"sh":197,"sg":86,"sj":78,"si":5921,"kÅ¡":1455,"rz":732,"u ":34678,"sa":11922,"sb":260,"rr":239,"rs":5613,"jÅ«":1021,"rt":4290,"ru":7404,"rv":1310,"CÄ“s":56,"mÄ“":2975,"ry":160,"lÄ«":4107,"rp":1821,"ro":10142,"rn":1729,"kņ":114,"rm":3278,"rl":568,"rk":1664,"rj":126,"jÅ¡":27,"ri":15479,"rh":439,"rg":2074,"iž":275,"mÄ":4269,"rf":127,"re":6968,"rd":2322,"rc":590,"kļ":994,"rb":2090,"ra":18562,"t ":6209,"lÄ“":2235,"qu":266,"kÄ«":37,"iÅ¡":513,"lÄ":5128,"s ":99264,"kÄ“":55,"pz":438,"pt":1346,"pu":3008,"pv":476,"pp":139,"jÄ«":54,"pr":4678,"ps":1123,"zÄ«":3820,"zÄ":1002,"IX ":48,"Å¡Ä£i":28,"zÄ“":1137,"už":113,"uÅ¡":785,"IV ":42,"tÅ¡":221,"uļ":295,"uņ":333,"vÄ«":926,"tÅ«":1454,"vÄ":2847,"tļ":184,"uÄ£":166,"rž":79,"zz":61,"sÅ«":160,"BÄ«b":25,"vÄ“":3474,"tņ":104,"uÄ":341,"Hor":54,"zg":379,"zi":7929,"rÅ¡":1224,"zb":206,"Hou":27,"zc":577,"zd":606,"ze":4296,"za":1506,"Hom":27,"Hon":23,"Hok":34,"Hol":58,"zv":1734,"uÄ“":34,"rÅ«":541,"zs":1624,"uÄ":23,"zr":375,"zu":648,"zt":460,"zo":976,"zn":964,"tÄ«":5903,"zp":657,"zk":332,"zj":64,"zm":1200,"zl":791,"yg":24,"ye":62,"tÄ":11917,"rļ":24,"yc":58,"yd":22,"ya":64,"rÄ·":279,"sÄ“":2104,"tÄ“":2602,"yt":35,"ys":126,"yr":30,"sÄ«":1152,"yp":31,"yo":86,"yn":43,"ym":26,"rņ":171,"yl":66,"yi":22,"Arg":44,"Arh":40,"Are":22,"Ard":29,"Ara":25,"Arm":42,"Ari":31,"Å Ä«s":37,"Apv":89,"Apo":23,"Atk":35,"Atl":83,"Atr":91,"Ato":22,"Ast":53,"Ata":28,"Asi":33,"Aso":76,"Art":56,"Avo":25,"Aut":94,"Aus":451,"Aug":149,"AlÅ«":46,"ArÄ":51,"Å¡Ä“j":249,"zÅ¡":91,"zÅ«":24,"zņ":392,"Å Ä·i":29,"AtÅ¡":29,"Bak":26,"Bal":501,"Ban":57,"Bab":25,"Baz":45,"Bar":119,"Bas":49,"Bau":40,"Abr":31,"Act":23,"Ada":38,"Å¡Ä ":313,"Adr":25,"Å¡Äd":125,"Å¡Äk":184,"Å¡Äm":60,"Å¡Äs":183,"Å¡Äv":22,"Afg":26,"Aiz":104,"Aka":34,"Ala":26,"Alb":79,"Ale":93,"Alf":40,"Alt":55,"All":36,"Alp":29,"Ame":197,"Ama":49,"Å Ä« ":93,"Ang":135,"Ana":95,"And":123,"Ant":228,"Ann":55,"Ar ":101,"ОР":57,"Bul":41,"Bur":79,"BlÅ«":22,"Bru":49,"BrÄ«":92,"² ":53,"BrÄ":30,"Cal":37,"Cam":27,"Car":39,"Cau":23,"Can":86,"Cap":22,"CH ":201,"Bez":37,"Ber":134,"Ben":73,"Bel":45,"Bie":67,"Biz":31,"Å Äd":55,"Bil":27,"Bir":42,"Bio":33,"Blo":34,"CP ":24,"CO ":32,"Bla":25,"Bre":53,"Bra":125,"Bro":55,"Bri":92,"Bol":40,"Bon":23,"Bor":50,"Bos":71,"Bov":25,"Der":41,"Des":26,"Dei":40,"Del":44,"Dem":45,"Den":34,"Deg":23,"Dan":52,"Dar":42,"Dat":63,"Dau":261,"Dab":33,"Chr":32,"Cit":38,"Cir":22,"Cil":47,"Cel":26,"Cen":195,"Cet":27,"Cer":41,"Cha":50,"Cor":106,"Com":69,"Col":30,"Con":48,"FA ":41,"Ä£ip":59,"Ä£in":756,"Ä£io":352,"Ä£it":52,"Ä£is":449,"Ä£im":77,"Ä£ij":982,"Dze":94,"Dzi":93,"Ä£i ":28,"Ä£er":123,"Ä£el":28,"Ä£eo":161,"Ä£en":236,"Edv":25,"Edu":28,"Ä£a ":65,"DzÄ«":27,"Äg":215,"Daž":59,"Äf":379,"Äc":3184,"Äd":3136,"Äb":862,"Äp":472,"Äm":4331,"Än":2529,"Äk":5743,"Äl":5581,"Äj":3661,"Äv":1981,"Ät":4781,"Äs":7161,"Är":6698,"Äz":697,"Dib":24,"Dis":48,"Din":26,"Ä€b":22,"Dig":22,"Ä€d":23,"Ä€g":35,"Ä€f":113,"Die":362,"Ä€z":143,"Ä€r":50,"Div":37,"Ä ":23455,"Dub":30,"ür":23,"Ð°Ñ ":29,"Ä£u ":60,"СР":64,"Dra":31,"Dob":31,"ES ":40,"Don":29,"Dom":61,"OH ":75,"Nea":33,"Ä“Ä£":190,"Nei":26,"Net":40,"Ner":49,"Nep":30,"Neo":28,"Ä’Ä£":41,"Nat":30,"−":111,"Ä«di":166,"Ä«dr":32,"Ä«ds":186,"Ä«du":170,"Ä«da":185,"Ä«de":154,"Nin":27,"Nik":51,"ēž":23,"Ä«ci":383,"Å«Å¡u":22,"Ä«ca":276,"Å«Å¡a":102,"Ä«ce":148,"Ä«bn":58,"Ä“Å¡":773,"Ä«bi":22,"Ä«bu":1222,"Ä«bv":37,"Ä«d ":59,"Ä«be":36,"Ä«ba":3544,"ēļ":376,"New":31,"Ä“Ä·":56,"ēņ":27,"Ä«cÄ":40,"Ä«bÄ":960,"Ä«gi":496,"Ä«ga":1676,"Ä«gu":405,"Ä«go":298,"Ä«gs":407,"Ä«dz":1603,"Ä«la":22,"Nam":26,"Ä«gÄ":813,"Ä«le":67,"Ä«li":124,"Nac":121,"Ä«ka":42,"Ä«kl":241,"Ä«ko":106,"Ä«ks":151,"Ä«ja":305,"Ä£u":81,"Ä«ji":84,"Ä«ju":700,"Ä«dÄ«":105,"Ä«m ":246,"Ä£i":2798,"Ä£e":601,"Ä£a":66,"Ä«dÄ":112,"Ä«cÄ«":70,"Ä¢e":106,"Äļ":238,"Äņ":597,"ÄÄ":113,"ÄŒi":48,"ÄŒe":124,"ÄŒa":35,"ÄŒu":25,"Äi":150,"Äe":516,"Īri":58,"Äa":247,"ÄÅ¡":567,"Äž":93,"No ":69,"Ä“ ":2668,"Īsl":26,"Äo":33,"Äs":42,"Äu":562,"Ä“z":349,"Ä“t":4896,"Ä“s":1279,"Ä“v":258,"ÄÅ«":27,"Čī":26,"OS ":28,"Ä“m":2242,"Ä“n":867,"Ä“k":1496,"Ä“l":2212,"Ä“r":3544,"Ä“p":82,"Nov":61,"Ä“c":1136,"Ä“d":833,"Nor":163,"Ä“j":4209,"Ä“g":174,"Nos":50,"Not":22,"Ä“b":25,"Nok":22,"Ä’r":64,"Nob":33,"Ä·e":541,"Ä·a":229,"Ä·u":510,"Ä·s":30,"Ä·o":78,"Odi":27,"Ä·i":1644,"ļ ":195,"Ogr":36,"JÄn":51,"ļa":2291,"ļi":696,"ļk":39,"ļj":32,"ļe":71,"ļd":32,"Ä·Ä":31,"Ķī":170,"ļr":180,"ļs":48,"ļl":36,"ļo":646,"Ä·Ä“":171,"ļu":2107,"ļv":154,"Ä»e":28,"Oke":30,"Ä·Ä«":541,"ļķ":37,"ļī":23,"ļģ":54,"ļē":91,"PA ":36,"ļÄ":656,"ļš":133,"ļū":169,"ļļ":89,"ļņ":176,"ņv":39,"ņi":716,"ņk":42,"ņj":25,"ņo":260,"ņn":28,"ņu":2226,"ņs":60,"ņr":139,"Å…u":43,"ņa":1766,"ņb":29,"ņd":47,"ņe":432,"Otr":101,"Ovi":22,"Ä« ":3304,"ģē":120,"ģī":39,"JÄ“k":140,"Īs":48,"Īr":60,"Ä«b":5893,"JÄ“z":37,"Ä«c":960,"Ä«d":2819,"Ä«j":1202,"Ä«g":4125,"Ä«l":317,"Ä«k":663,"Ä«n":1786,"Ä«m":2002,"Ä«p":453,"Ä«r":757,"Ä«t":3369,"Ä«s":1871,"Ä«v":3050,"Ä«z":500,"ģīt":31,"Oli":250,"ı ":35,"JÄņ":29,"Ä«Ä£":38,"Ä«Ä":97,"īņ":368,"PS ":25,"īļ":122,"Ä«Å¡":592,"Ope":52,"Org":41,"Ost":30," −":67,"Osk":35,"Osm":120,"Ķe":55,"Å¡ ":1382,"Ple":55,"Pla":145,"Pil":83,"PaÅ¡":73,"Pit":22,"Pir":196,"QL ":27,"Pie":153,"Pha":28,"ģēt":22,"ģēr":24,"KÄr":70,"ģēl":37,"Å¡Ä«":572,"Å¡Ä£":28,"Per":109,"šļ":41,"Pet":30,"Pen":59,"Pek":25,"Pel":45,"Å¡Ä·":1261,"Å Ä«":134,"KÄ ":42,"Å¡Ä":893,"Å¡Ä“":273,"Å Ä·":45,"Å Ä":65,"Paz":31,"Pat":54,"Pas":386,"Par":468,"Pav":27,"Pau":41,"Pad":103,"Pan":44,"Pap":26,"Pal":61,"Pak":46,"Å¡g":29,"Å¡e":180,"Å¡d":202,"Å¡a":5315,"Å¡o":571,"Å¡p":217,"Å¡m":110,"Å¡n":89,"Å¡k":157,"Å¡l":112,"Å¡i":1623,"Å¡v":170,"Å¡u":2014,"Å¡t":354,"Å¡s":357,"Å¡r":148,"Å¡z":36,"Å e":104,"Å a":199,"Å o":132,"Å i":163,"Å t":29,"Å v":81,"ņš":324,"Å…Å«":40,"ņģ":77,"PrÅ«":54,"ņē":348,"ņķ":245,"ņÄ":297,"PrÄ":22,"Pro":228,"Pri":121,"Pre":78,"Pra":58,"Pol":274,"Pos":25,"Pop":24,"Por":59,"žr":80,"žs":96,"žu":691,"žn":24,"žo":773,"že":338,"ža":902,"žk":75,"žm":27,"ži":613,"Ža":46,"Ži":23,"Že":35,"RS ":104," ال":23,"ž ":41,"žū":26,"žņ":118,"žī":67,"žģ":48,"žÄ":790,"SA ":43,"Rad":110,"Rai":35,"Rag":24,"Rak":35,"Å« ":140,"šņ":106,"Ūd":31,"šž":26,"Å¡Å«":152,"Å«g":56,"Å«d":971,"Å«c":206,"Å«z":629,"Å«s":815,"Å«t":957,"Å«v":256,"Å«p":130,"Å«r":2415,"Å«k":417,"Å«l":106,"Å«m":112,"Å«n":373,"Å«Å¡":146,"ūž":54,"Isl":22,"IrÄ":62,"Iva":31,"Izv":72,"Izr":29,"ItÄ":112,"Jac":27,"Jav":26,"Jau":217,"Jap":129,"Jan":40,"Jam":45,"Jag":28,"Ja ":27,"Jel":80,"Jer":41,"Jor":27,"Jon":25,"ã‚¢":23,"Joh":102,"KS ":29,"Jug":32,"Jup":83,"Jur":42,"Jum":22,"Kad":24,"Kab":22,"Kai":30,"Kam":79,"Kal":190,"Kap":52,"Kan":182,"Kau":72,"Kat":99,"Kas":57,"Kar":357,"Kaz":79,"Ker":28,"Ken":34,"Kem":29,"Kei":25,"Kir":46,"Kin":77,"Kij":27,"Kli":34,"Kle":26,"Kla":44,"Klu":132,"Kon":195,"Kom":112,"Kol":59,"Kos":58,"Kor":100,"Kop":106,"Kod":29,"Kok":40,"Kre":35,"Kra":70,"Kri":714,"Kro":35,"Kru":113,"KrÄ":28,"Kul":67,"Kur":259,"Kva":27,"Let":23,"Leo":37,"Lej":26,"Led":38,"Lau":64,"Lak":51,"Lai":103,"Lag":34,"Lat":1556,"Lar":29,"Lap":25,"Lam":24,"Lan":56,"Lab":56,"ML ":46,"Lib":28,"Lie":695,"Lim":30,"Lin":97,"Lit":29,"Liv":157,"Lut":24,"Luk":24,"Lud":28,"Lug":27,"Lor":22,"Lon":64,"Lok":22,"Å«s ":24,"Å«pn":62,"Å«ra":1491,"Å«t ":294,"Å«sd":328,"Å«nÄ":49,"Å«rs":28,"Å«ru":316,"Å«rv":45,"Å«rm":32,"Å«rn":45,"Å«rg":22,"Å«ri":123,"Å«ks":153,"Å«ku":78,"Å«ka":36,"Å«ko":44,"Å«ma":25,"Å«li":71,"Å«mu":36,"Å«na":154,"Å«ns":22,"Å«nu":55,"Å«ni":45,"Mek":48,"Mei":30,"Men":40,"Mel":144,"Mer":74,"Met":75,"Å«tÄ":37,"Med":58,"Mez":22,"Å«zi":561,"Džo":145,"Å«zu":22,"Å«tÄ«":135,"Džu":25,"Dže":88,"Å«si":58,"Å«sm":68,"Å«su":37,"Å«st":237,"Å«ta":126,"Å«te":28,"Man":129,"Mal":86,"Å«ti":73,"Mam":22,"Mar":393,"Å«to":32,"Mas":145,"Å«tn":56,"Mag":45,"Å«ts":52,"Mad":99,"Mak":70,"Mai":65,"Å«tu":111,"Mac":23,"Ä«zÄ":35,"Ä«zÄ“":29,"Å«vi":42,"Maz":108,"Mat":72,"Å«ve":114,"Å«rÄ":242,"Ä«vÄ":296,"Ä«tÄ«":76,"Ä«zi":177,"Ä«ze":130,"Ä«tÄ“":23,"Mod":95,"Mol":50,"Mon":122,"Mos":23,"Mor":85,"Mot":31,"Moz":28,"NS ":43,"Ä«vÄ«":139,"Mež":62,"Ä«vÄ“":41,"Mih":26,"Mik":53,"Mie":24,"Ä«rÄ":42,"Mic":50,"Ä«vn":200,"Ä«vm":36,"Ä«vp":28,"Ä«vo":1046,"Ä«vu":146,"Mir":23,"Ä«vs":154,"Mil":64,"Min":133,"Ä«va":467,"Ä«ve":148,"Ä«vi":298,"Å«vÄ“":84,"Ä«tÄ":552,"NO ":49,"Ä«z ":81,"Ä«sÄ":29,"Ä«lÄ«":22,"Ä«rs":118,"Ä«ru":139,"Ä«ri":268,"Ä«sa":30,"Ä«nÄ":148,"Ä«mÄ“":659,"Mur":22,"Mus":124,"Ä«ra":120,"Ä«t ":399,"Ä«tu":205,"Ä«nÄ«":56,"Ä«tr":41,"Ä«ts":565,"Ä«tn":23,"Ä«to":115,"Ä«ti":392,"Ä«st":852,"Ä«ss":36,"Ä«mÄ«":130,"Ä«sl":28,"Ä«sk":119,"Ä«si":181,"Ä«kÅ¡":28,"Ä«te":151,"Ä«ta":808,"Ä«ne":41,"Ä«ni":507,"Ä«no":33,"Ä«ns":285,"Ä«nu":169,"Ä«me":446,"Ä«mi":343,"Ä«mj":45,"Ä«mo":31,"Ä«mu":34,"Ä«ms":40,"Ä«na":462,"Ä«kÄ":26,"Ä«s ":468,"Ä«jÄ":106,"Ä«pa":426,"SÄk":57,"RÄ“z":54,"XX ":24,"кий":32,"Wor":39,"Wil":32,"Win":29,"War":28,"Vul":29,"Vor":36,"Vol":134,"Viņ":206,"Viļ":33,"Vis":230,"Vit":48,"Vla":36,"Äu ":517,"Zaļ":27,"Zie":391,"Zin":62,"Zil":29,"Zel":41,"Zem":525,"之":22,"TÄ“r":50,"三":24,"на ":32,"TÄ ":634,"TÄl":53,"TÄs":150,"SÄ“r":28,"SÄ“l":24,"RÄ«g":820,"SvÄ“":102,"Sys":26,"Sve":45,"SpÄ“":56,"Sul":33,"SpÄ":87,"Str":115,"Stu":53,"Sti":60,"Sto":87,"Sta":349,"Ste":80,"Teh":22,"Tec":22,"Tem":25,"Teo":25,"Tei":33,"Tel":43,"Tam":51,"Tan":40,"Tas":583,"Tar":26,"Tau":65,"Tai":63,"Taj":81,"Tak":27,"Tal":71,"Sko":119,"Sku":25,"Ska":110,"Sim":41,"Sil":58,"Sig":27,"Sir":32,"Sin":63,"Sid":28,"Sie":22,"Sib":28,"NÄc":29,"Ser":100,"Sen":122,"Sel":26,"Sem":34,"Sek":22,"TV ":22,"Spa":42,"TS ":22,"Spi":27,"Spe":35,"Spo":27,"Soc":53,"Sol":25,"Som":114,"Son":25,"Slo":56,"Smi":29,"TP ":38,"MÄ“r":47,"MÄ“n":64,"JÅ«r":108,"JÅ«l":32,"SV ":351,"Run":31,"Rum":56,"Rub":29,"Sai":29,"Sah":30,"Sak":57,"Sam":35,"Sal":189,"Sac":23,"Sab":29,"Sae":27,"Sad":27,"Sco":41,"Sci":36,"Sch":39,"Sav":262,"Sat":49,"Sau":200,"Sar":143,"Sas":60,"San":93,"ови":22,"MÄr":46,"MÄk":28,"SI ":51,"Res":30,"Rie":159,"Rau":22,"Rec":23,"Red":33,"Rei":58,"Ren":27,"Rep":321,"Rob":99,"Rod":49,"SR ":106,"Ros":76,"Ron":23,"Rom":176,"SS ":24,"LÄ«g":32,"LÄ«d":101,"ÄÅ«s":27,"ReÄ£":30,"SO ":60,"PÄ“c":116,"PÄ“t":62,"Vai":65,"Vel":97,"Ven":119,"Vei":37,"Vec":107,"Ñки":39,"Ñка":25,"Ñко":22,"Vas":82,"Van":50,"Val":406,"Var":89,"VaÅ¡":22,"Vid":291,"Vie":179,"Vir":65,"Vil":141,"Vik":70,"Vin":38,"Ver":44,"Ves":70,"Ukr":55,"Ung":47,"Uni":125,"Urs":73,"MÅ«z":68,"MÅ«s":71,"Ä’rg":30,"Uz ":28,"Čīl":23,"Ä“cÄ«":76,"Ä“dÄ":27,"TrÄ«":27,"Ä“j ":73,"Ä“gu":33,"Ä“gt":73,"Ä“kl":37,"Ä“ko":23,"Ä“ki":215,"Ä“ku":376,"Ä“kt":35,"Ä“ks":180,"Ä“m ":819,"Ä“ka":504,"Ä“ji":561,"Ä“ju":643,"Ä“js":400,"Ä“jr":35,"Ä“jp":24,"Ä“jo":477,"Ä“l ":109,"Ä“dÄ“":151,"Ä“jd":44,"Ä“ja":1223,"Ä“mu":365,"Ä“ma":558,"Ä“mi":221,"Ä“li":303,"Ä“lo":171,"Ä“ln":72,"Ä“ls":126,"Ä“lu":113,"Ä“la":172,"Ä“le":379,"Ä“c ":930,"Ä“ci":41,"Ä“du":26,"Ä“dz":296,"PÄv":29,"PÄr":113,"Ä“da":98,"VI ":40,"Ä“de":49,"Ä“di":142,"Ter":125,"Tet":23,"Ä“vs":32,"Ä“rÄ“":61,"Ä“rÄ":124,"Ä“rÄ·":215,"Ä“sÄ“":144,"The":138,"Ä“rÄ«":91,"Ä“sÄ":29,"Ä“ze":148,"Tib":26,"Tie":134,"Ä“sÄ«":28,"Tim":41,"Ä“tÄ":792,"Tir":25,"Ä“zi":135,"Ä“rÅ¡":71,"Ä“tÄ«":178,"Ä“zu":41,"Ä“vÄ“":143,"To ":233,"Ä“ne":214,"Ä“ni":174,"Ä“na":102,"Ä“ns":137,"Ä“nu":91,"Top":34,"Ä“jÄ":633,"Tor":80,"Tom":55,"Tos":31,"Ä“kÄ":56,"Ä“r ":112,"Ä“jÄ«":33,"Ä“po":30,"Ä“lÄ":185,"Ä“s ":404,"Ä“t ":252,"Ä“lÄ“":492,"NÄ«d":49,"Ä“ra":721,"Ä“rb":56,"Ä“rd":31,"Ä“re":37,"Ä“mÄ":142,"Ä“ri":669,"Ä“rk":23,"Tro":39,"Ä“rn":117,"Ä“rp":23,"Ä“ro":293,"Ä“lÄ«":53,"Ä“rt":389,"Tri":83,"Ä“rs":217,"Ä“rv":101,"Ä“ru":97,"Ä“mÄ“":83,"Tre":69,"Ä“rz":31,"Tra":108,"Ä“nÄ":38,"Ä“kÅ¡":30,"Ä“sl":23,"Ä“su":45,"Ä“st":566,"Ä“nÄ“":31,"Ä“te":130,"Ä“ta":1832,"Ä“tn":71,"Ä“to":79,"Ä“ti":598,"Ä“tk":66,"Ä“tu":414,"Ä“tr":34,"Ä“ts":396,"Tur":208,"Tuv":48,"Tuk":33,"Tul":26,"Å¡dz":152,"Å¡de":24,"Å¡i ":834,"Å¡el":77,"Å¡a ":338,"Å¡as":321,"Å¡au":228,"Å¡ah":27,"Å¡ai":236,"Å¡aj":205,"Å¡am":126,"Å¡an":3757,"Ä“Ä£i":168,"Å ve":65,"Å¡te":79,"Å¡ta":234,"Å¡st":29,"Å¡um":55,"Å¡us":57,"Å¡vi":22,"Å¡ve":23,"Å¡va":72,"Å¡pa":34,"Å¡kÄ":87,"Å¡pi":125,"Å¡os":132,"Å¡ot":31,"Ä’Ä£i":41,"Å¡pu":29,"Å¡s ":266,"Å¡re":66,"Å¡ru":52,"Å¡sa":28,"Å¡u ":1883,"Å¡la":74,"Å¡me":77,"вич":26,"Å¡o ":346,"Å¡ni":32,"Å¡no":38,"Å¡ie":334,"Å¡if":27,"Å¡in":270,"Å¡im":31,"Å¡is":80,"ēļa":42,"ēļ ":153,"Ä“Ä·i":49,"Å¡vÄ":48,"Å¡ze":25,"ēļu":170,"bju":45,"bje":249,"biz":35,"bis":145,"bit":116,"bio":155,"bir":59,"bik":26,"bil":410,"bin":496,"bij":2367,"bo ":36,"blo":48,"ble":47,"bli":610,"bla":73,"bku":154,"bok":22,"bol":1514,"boj":283,"bni":61,"bež":545,"bs ":212,"biļ":23,"bpi":124,"biņ":28,"bon":217,"bor":108,"bot":84,"bos":35,"bov":23,"be ":419,"bam":32,"ban":81,"bak":69,"bal":1602,"bai":173,"baj":47,"bag":46,"bac":26,"bad":23,"baz":174,"bau":36,"bat":100,"bas":2875,"bar":192,"azÄ«":447,"bi ":165,"bei":323,"bed":22,"ber":482,"ben":195,"bel":118,"bek":85,"bez":361,"bes":245,"bet":713,"bib":56,"bie":860,"brÅ«":72,"− ":42,"buļ":79,"−C":44,"ca ":396,"car":55,"cas":201,"cat":29,"cau":337,"can":76,"cab":38,"cam":74,"cal":52,"ce ":431,"cba":35,"blÄ“":52,"bri":385,"bro":61,"bra":298,"bre":171,"bu ":1529,"blÄ«":48,"bru":342,"bsk":33,"bso":54,"bse":25,"bst":77,"bur":350,"bul":250,"bun":25,"bum":299,"bud":74,"but":64,"bus":88,"boļ":39,"bva":41,"brÄ":110,"brÄ«":410,"aka":880,"am ":2221,"ake":96,"akc":136,"aki":53,"aji":783,"ajo":724,"ajs":38,"adÄ«":860,"aju":73,"adÄ“":277,"aiz":832,"al ":300,"aja":469,"aje":41,"aij":61,"aik":1200,"ail":137,"aim":532,"ain":809,"aip":45,"acÄ«":139,"air":752,"ais":4315,"ait":1125,"aiv":51,"aig":508,"adÄ":819,"aie":25,"aid":426,"aic":101,"aib":25,"ahi":35,"ahs":67,"aht":36,"abÄ«":81,"ahr":22,"abÄ“":108,"aha":93,"agl":88,"abÄ":487,"agi":39,"agr":218,"ags":92,"agu":84,"agn":169,"ago":152,"ajÄ":2224,"anv":34,"anu":718,"anz":65,"ano":394,"ann":140,"ant":1775,"ans":674,"anr":123,"ane":123,"ang":611,"anh":30,"ani":1185,"anj":22,"ank":275,"anl":23,"ap ":332,"ana":3035,"anc":401,"and":1589,"amu":130,"amm":408,"amo":175,"amp":185,"ams":603,"ami":532,"ahÄ":45,"adž":91,"ame":775,"amb":217,"ama":1351,"ao ":25,"alv":1226,"alu":519,"alt":791,"als":1977,"alp":208,"alo":1383,"aln":628,"alm":121,"all":272,"alk":231,"alg":97,"ali":1208,"alc":61,"ald":1013,"ale":584,"alf":96,"agÄ":163,"ala":1330,"alb":172,"Å aj":103,"an ":784,"aks":1610,"akr":121,"Å ar":24,"aku":213,"akt":789,"ako":151,"akn":91,"akm":124,"akl":47,"aba":1096,"abe":95,"abi":630,"abl":44,"abo":176,"abp":131,"abr":109,"abs":116,"abv":34,"abu":136,"ae ":655,"aca":38,"aau":77,"ad ":480,"ac ":37,"afr":25,"aft":53,"afi":105,"ai ":5352,"aga":835,"age":107,"ael":27,"aei":29,"ah ":32,"afa":26,"ado":609,"adr":151,"adl":29,"adn":49,"adm":304,"adi":802,"ade":142,"adz":161,"ads":379,"adu":517,"ack":35,"aci":939,"ach":47,"ace":507,"ada":1586,"act":88,"acu":34,"azn":207,"azm":31,"azo":137,"azi":230,"arÅ¡":263,"azu":55,"atÄ«":1487,"azs":68,"atÄ“":140,"aze":46,"aza":217,"azd":25,"atņ":23,"avÄ“":54,"avÄ":267,"arž":51,"arÄ«":2864,"arÄ“":195,"az ":62,"asÄ":83,"asÄ«":143,"asÄ“":82,"Å ob":33,"atÄ":330,"aye":23,"auž":79,"ba ":1526,"azÄ":276,"atÅ«":468,"auņ":57,"avÄ«":79,"atÅ¡":186,"auÅ¡":105,"at ":261,"alÄ“":137,"arh":312,"aiž":27,"arg":217,"amÄ":271,"are":686,"ard":512,"arc":143,"akļ":72,"arb":1417,"ara":2574,"arp":1342,"aro":671,"arn":121,"arm":206,"arl":146,"ark":821,"arj":66,"ari":685,"aru":580,"Å ie":27,"arv":25,"amÄ“":73,"alÄ«":910,"arr":34,"ars":430,"art":883,"au ":159,"asa":1301,"ary":36,"anÄ":994,"asf":22,"akÅ¡":415,"asi":550,"ash":28,"asc":35,"ase":553,"aso":90,"asn":297,"amÄ«":170,"asp":375,"ask":709,"asm":50,"asl":59,"ar ":6199,"apb":88,"apa":727,"Å ep":43,"akÄ":218,"ape":72,"apd":220,"aiļ":41,"apj":56,"api":222,"aph":30,"apg":734,"aiņ":241,"apm":273,"Å ei":27,"apl":157,"apk":491,"apo":44,"app":36,"apr":520,"aps":483,"apt":349,"apu":118,"apv":358,"apz":415,"as ":38125,"alÄ":492,"aiÅ¡":99,"ava":1146,"ax ":25,"auz":73,"auv":33,"aut":1629,"avs":157,"apÄ«":65,"avr":23,"avo":299,"avp":76,"anÅ¡":45,"avi":843,"ave":232,"ay ":49,"Å o ":72,"avv":82,"avu":357,"arÄ":580,"av ":331,"atb":314,"ata":845,"asu":217,"ast":3489,"ass":194,"anÄ":183,"anÄ“":182,"asv":32,"atm":195,"atn":153,"atk":493,"atl":205,"anÄ«":82,"atr":1790,"ato":1405,"atp":124,"ate":661,"atf":33,"Å is":89,"atc":32,"atd":107,"ati":895,"atj":54,"atg":171,"ath":38,"auc":511,"att":979,"ats":614,"alÅ«":32,"atv":2153,"atu":973,"atz":132,"aul":1187,"aum":53,"aun":922,"aup":26,"aur":568,"aus":1232,"aud":1114,"apÄ":234,"aug":1739,"auj":444,"auk":1608,"VÄ“s":43,"VÄ“r":24,"ος":59,"ος ":59,"Ï‚ ":97,"ν ":22,"Zvi":157,"Zva":55,"α ":62,"VÄr":66,"VÄc":268,"Ä“Å¡a":752,"Ð ":147,"ий ":35,"ич ":26,"jeb":1453,"jer":106,"jek":399,"jel":23,"jen":64,"jet":57,"jev":88,"eÄn":337,"eÄl":99,"eÄt":94,"ji ":637,"aža":89,"aži":139,"ažk":37,"ažo":204,"ažr":77,"ažu":75,"ļģi":26,"jad":67,"izÄ“":509,"jas":8640,"jau":557,"jap":53,"jar":34,"jak":65,"jan":85,"jam":931,"dÄ“Å¡":109,"jai":593,"izņ":119,"izÅ¡":65,"jda":46,"jni":88,"jol":22,"jon":751,"jom":89,"jot":574,"jos":350,"jor":71,"js ":1627,"dÄ«b":475,"dÄ«c":65,"jiņ":29,"jpu":34,"Ñк":94," zÄ«":183,"jis":272,"jie":1464,"то":24,"ÑÑ‚":46,"jko":24,"jo ":387,"ažÄ":482,"itm":69,"itl":241,"itr":232,"inÄ«":182,"itp":32,"ito":983,"itu":694,"itt":30,"its":171,"ity":36,"imÅ¡":47,"isk":6070,"ism":951,"isl":163,"iso":147,"isn":115,"imÄ«":127,"isp":195,"iss":145,"inÄ":41,"isu":253,"ist":3658,"isv":78,"inÄ“":330,"ita":561,"ite":695,"ith":30,"iti":844,"ilÅ¡":47,"inÅ«":22,"ivs":26,"ivr":48,"ipÄ«":32,"ivp":30,"ivo":171,"ivv":36,"ivu":117,"irÄ":703,"inž":48,"ius":102,"iur":38,"ium":46,"ipÄ“":33,"iva":204,"ivd":29,"dÄ“ ":186,"ix ":38,"ivi":487,"inÅ¡":34,"ivj":45,"ivk":27,"ive":291,"ipr":208,"ipo":63,"ipu":70,"ips":96,"ipt":115,"ipi":127,"igž":89,"ipl":132,"cÄ«b":793,"cÄ«g":288,"ilÄ":148,"is ":8439,"ion":1944,"iop":23,"aÅ¡Ä«":283,"ior":31,"igÅ«":45,"ios":84,"iot":138,"ijÄ":2772,"iog":37,"iok":51,"iol":177,"iom":40,"ipa":229,"ikÄ":1450,"ipe":167,"iov":23,"ir ":14672,"imÄ“":165,"iru":216,"irs":599,"irt":236,"ilÄ«":94,"irr":24,"iro":905,"irm":929,"irn":124,"irk":171,"iri":558,"isi":459,"ikÅ¡":84,"ish":36,"inÄ":3097,"ise":117,"isc":158,"isb":151,"isa":605,"irz":314,"cÄ«t":112,"cÄ«n":190,"cÄ«j":23,"cÄ«k":97,"imÄ":119,"ire":99,"irg":402,"irb":90,"ira":259,"ird":185,"irc":39,"it ":228,"ilÄ“":113,"cÄ«z":63,"itÅ«":160,"ivÄ«":107,"izÄ":483,"dēļ":202,"ja ":7647,"itÄ":1332,"irÄ«":150,"dÄ“s":22,"dÄ“t":124,"dÄ“v":144,"dÄ“j":602,"dÄ“n":29,"dÄ“m":188,"cīņ":106,"dÄ“l":80,"isÄ":208,"irÄ“":31,"iz ":254,"ivÄ":236,"itļ":164,"izz":43,"izu":55,"izv":798,"itÄ«":222,"izr":274,"izs":752,"izt":200,"izp":558,"izo":166,"izn":131,"izm":1077,"izl":692,"izk":139,"izj":45,"irÅ¡":41,"izi":513,"izg":245,"ize":159,"izd":376,"izc":301,"izb":41,"iza":170,"itÄ“":104,"isÄ«":84,"eÄ£i":394,"kij":77,"kil":129,"kia":33,"kie":620,"eÄ£e":52,"kin":233,"ļÄs":38,"kip":24,"kir":36,"kis":66,"ļÄm":48,"keÄ":336,"km ":504,"kga":63,"ki ":1566,"kgr":28,"kaļ":69,"kaÄ·":31,"kho":57,"kaņ":374,"kej":204,"kel":71,"ken":50,"kes":27,"ker":86,"ket":457,"Ä¼Ä ":507,"fÄt":30,"ke ":111,"kci":508,"kdi":30,"kra":920,"kre":163,"klÄ“":111,"kt ":222,"eÄ«n":30,"klÄ":396,"ksa":250,"kse":83,"kmÄ“":88,"ku ":2253,"kro":296,"klÄ«":65,"kru":265,"kri":985,"koz":38,"kov":64,"kot":513,"km²":49,"kos":513,"kor":259,"kop":1110,"koo":61,"kon":1216,"kom":1199,"kol":578,"kok":286,"koj":143,"koh":29,"kog":34,"koe":27,"kod":306,"ks ":1859,"kme":260,"kne":183,"klu":183,"kls":75,"ko ":1627,"kle":136,"kla":494,"klo":153,"kli":573,"ļēj":87,"dÄ«Å¡":155,"joÅ¡":397,"jus":320,"jum":2601,"jur":89,"jvi":27,"jsi":45,"bÅ«v":217,"bÅ«s":22,"bÅ«t":380,"eÄu":47,"ju ":3050,"dÄ«g":518,"dÄ«j":452,"dÄ«n":38,"dÄ«t":668,"dÄ«z":28,"jra":36,"eÄe":56,"kaz":48,"kav":162,"kat":1010,"kau":385,"kar":1806,"kas":6913,"kap":301,"kan":621,"kal":1136,"kam":395,"kaj":1054,"kak":37,"kai":2823,"kad":430,"kab":198,"ka ":3723,"juÅ¡":280," Ga":455," BÄ":54," Ge":158," I ":103,"guļ":52," Fo":211," Fu":230," Fr":535,"ļš ":124," Fi":190," Fl":70," Ha":441,"aÄ·u":37," He":345," BÄ“":72,"aÄ·i":22,"guÅ¡":31," J ":40," Go":172,"aÄ·e":127," Gr":516," Gu":162," Gv":31," Gi":68," Gl":86," Ig":193," DÄ":72," Ie":156," Id":25," Ib":23," K ":22," CÄ“":86," Hu":54," BÄ«":48," Ho":304,"ha ":116," Hi":232," Je":179," Ja":604," Iz":255," Iv":35," DÄ“":23," Ir":159," Is":56," It":130," Im":77," In":594," Ik":26," Il":59,"ham":85," M ":45,"han":159,"hai":42," Ka":1303,"hal":100," Ke":174,"hau":35," Ki":234,"har":152,"has":51,"hat":24,"aļ ":30," Jo":237," Ju":233,"hae":23," N ":55," La":2100," Le":272," GÄ":24," Li":1175," Kl":266," Ko":784," Kr":1044," Kv":49," Ku":433," Ma":1344," O ":55,"aļu":335," Mi":483,"aļv":40," Dž":308," Me":654," HÄ":28,"he ":281," Lo":254,"aļa":796," Ly":24," Lu":187,"aļi":184," Ne":334," P ":63,"а ":100," Na":328," Ni":163,"cÄ ":43," Mo":496," My":39," Mu":257," HÄ“":35,"hek":42,"hel":84,"hei":58," A ":102,"С ":41,"het":54,"her":196,"heo":49,"hen":64,"hem":51,"cÄk":136,"cÄm":35,"hi ":25," B ":49,"cÄs":52," C ":203," Ap":260," Am":327," An":736," Ak":149," Al":533," Ai":198," Ag":50," Ah":36," Af":50," Ac":82," Ad":148,"aļÄ":556," Ab":144," Ba":1003,"aļē":89," D ":68," Az":34," Av":74," Au":758," At":403," As":243," Ar":519," Be":444,"hie":34,"hid":182,"hic":23,"hib":51," Bi":327,"hip":128,"hin":98," Bl":114,"him":44,"hil":44," Bo":338,"hij":70," Br":524," Bu":238,"his":79,"hit":143,"aļķ":22," E ":56," Ca":270," Ce":344," Ci":203," Ch":131," Cl":40," Cr":39," Co":292," Cu":37," F ":24," Da":629," Di":673," De":398," Dr":102," Do":202," Dz":227," Du":106,"hn ":22," Eb":22," Ed":95," El":193," Ek":92," Ei":676," Eg":26," Et":48," Es":44," Er":98,"hlo":75," Eq":185," En":121," Em":90," Ez":43," Ex":28," Eu":85," Ev":42," Fe":226,"aņa":180," Fa":110," H ":140,"aņd":30,"gma":49,"go ":413,"gme":45," RÄ«":858," PÅ«":23,"glu":49,"glo":94," SÄ":102,"gls":48," Z ":22,"gle":241," RÄ“":60,"gli":344,"gla":305," Wo":56," Wi":95," We":44," RÄ":30," Wa":65,"й ":60," RÅ«":43,"gog":33," Zu":31,"god":68," Zv":222," Zo":72,"aģē":26," TÄ«":49," Ze":604,"gno":38," Zi":551,"gni":35,"bÄÅ¡":52," Za":124," SÄ«":40," TÄ“":78,"gna":34," Yo":26," TÄ":889," SÄ“":77," VÄ«":58,"gs ":1150,"glÄ":76,"о ":42," VÄ“":121,"н ":27,"gol":71,"gon":119,"gos":60,"gor":256," VÄ":354,"got":83,"gov":43,"gu ":870,"gnÄ":78," a ":72,"Ñ€ ":25,"glÄ«":122,"gro":67,"gru":596,"gra":755,"gt ":56,"gri":552,"gre":213,"aÄ«s":153,"gtu":35," R ":37," JÄ“":211," Ov":27," Os":199,"gto":46," Ot":132," Or":178,"gts":50," Op":65," Po":516," Pl":245,"gum":401," Pi":541," Ph":55,"gul":224," KÄ":157,"gua":28," Pe":329,"gub":86," Pa":1447,"gud":66,"gst":733,"gnÄ“":109,"gsn":33," Nu":37," No":486," Ol":289," Ok":85," On":42," Oh":28," Og":53," Od":50," JÄ":93," Of":27," Ob":41,"gta":111," Ra":371,"д ":27," LÄ“":28," Qu":23,"goÅ¡":45," LÄ«":215," Ro":565," Re":681,"grÄ":671," Ri":308," Rh":25," MÄ":133," S ":101,"guv":148,"gur":196," Pr":570," Ps":28,"gus":188," Pu":147,"gun":194,"gvi":84," LÄ":40,"gva":31,"bÄ“r":112," Sy":39,"grÄ«":79,"bÄ“t":95," Sv":220," Su":203," St":793," Ta":1092," V ":48," Tj":24," Th":177," Ti":329," Te":399," Tr":346," LÅ«":27,"ļūt":29,"ļūs":33," NÄ«":78," To":521," MÄ“":139," JÅ«":171," Ru":196,"ļūd":92,"grÄ“":114," Sa":1313," U ":30,"е ":61," Sh":52," Si":395," Sc":134," Se":442," NÄ":75," So":290," Sp":307," MÄ«":29,"bÄ“j":25," Sk":316," Sl":112," Sm":70,"bÄ“m":82," Sn":22," PÄ“":221,"grÅ«":45," Uz":96," Va":790," X ":41,"и ":54," Ve":526," Vi":1347," Vl":38," PÄ«":38," Vo":205," Vu":40," Tu":361," W ":26," PÄ":146," Ug":37," Uk":57," Ul":29,"gzd":28," Un":191," Up":44," Ur":133,"gzn":376," MÅ«":173," ja":682,"cÄ“Å¡":42,"iak":26,"ÑŒ ":24,"iam":59,"ial":119," iz":4946,"ian":220,"ias":27," je":1511,"iat":108," im":336," in":1344," ik":136," il":190,"ic ":183," dÄ“":302," iv":22," is":92," it":111," cÄ«":105,"iag":25," ir":14600,"ibl":76," ka":7945,"ibi":285,"ibo":30," m ":158,"ibr":57," ki":241," fÄ":22,"ibu":58," ke":72,"iaz":26," jo":407," dÄ«":55,"id ":25,"iba":66," bÅ«":407,"ibe":125," ju":119," bÄ“":90," ha":132," he":218," aÄ£":61," gl":287," gr":1494," go":110," gu":155,"ia ":270," gs":71," cÄ“":148," id":175," dÄ":101," ie":4455," ig":94," hi":297," hl":48," ho":304," bÄ«":86," hr":108," ht":33," hu":35,"iet":4005,"iev":1792," ni":63,"iez":227,"iel":3446,"iem":8120," ne":2383,"ien":7676,"iep":380," na":986,"ier":1197," p ":50,"ies":2658,"iee":100,"ied":2745,"ieg":992,"iek":4508," mu":445,"iej":39,"iea":35," mo":718,"iec":1164," mm":35,"ieb":72," ok":512," ol":76," on":40,"dÄv":49," og":135,"ifo":180,"dÄm":160," of":307," jÄ":140,"dÄn":30,"ifs":22,"dÄs":334,"dÄt":198,"dÄr":218,"ifr":46," ob":243,"dÄf":28,"ife":75,"dÄl":53,"dÄj":251,"ifi":245," nu":108," no":9749," gÄ":212," le":458,"icr":40,"ics":32,"ict":31," li":2852,"icu":63," n ":74,"ico":88,"ick":37," eļ":37,"icl":22," la":3298," kv":261," ku":4521,"ici":543," cÅ«":37,"ich":68,"ice":118,"ie ":1698," km":554,"ica":238," kl":531," kr":1914," ko":4523," me":1393,"idz":312," dž":28,"idu":626," mi":1711,"ids":336,"Ñ ":56,"idr":632,"ido":1694,"idp":27,"idm":53," ma":1802,"idn":29," lu":115,"idi":317," lv":27,"ide":638,"idd":33,"ida":1167," lo":437,"dÄ ":1681," af":23," ag":155,"aÅ¡a":197,"idÄ":226," ab":209," ac":105," ad":392," am":414," an":722,"aÅ¡i":302," ap":4517,"aÅ¡n":47," ai":846," ak":503,"aÅ¡l":70," al":593," av":171,"aÅ¡s":53," au":2262,"aÅ¡r":63,"icÄ«":260," ar":6236,"aÅ¡v":86," at":4655,"aÅ¡u":167," as":496," d ":35," ba":1482,"idÄ“":338,"il ":50,"ija":12639," bi":2895,"ije":54," be":1390,"iji":165," bo":133," bl":146,"idÄ«":348,"ijs":425," bu":382,"iju":1715," br":766," ca":410," e ":62,"im ":312,"ika":3320,"ige":58,"iga":284,"ii ":23,"igl":56,"igm":40,"igh":40,"igi":68,"igu":73,"igs":27,"igr":58,"igo":33,"ign":102,"ij ":163,"igz":407," b ":53,"iha":58,"ihi":54,"ihs":70," ZÄ«":28,"iho":104,"ibÄ«":64,"ik ":182,"icÄ“":211," c ":63," er":23,"imo":88,"imn":195," et":118," es":227,"ims":83," en":251," em":85," ep":55,"imp":593,"idž":50,"imf":36," ei":45," el":849,"ime":522," ek":494," ef":77,"imi":811,"ieÅ¡":1802," fe":288,"ip ":24,"inc":281,"ind":539,"ina":1252," fa":282," ez":655,"imt":1282,"imu":237," ev":61," fu":829,"inn":34,"ino":729," fr":351,"ļļu":39,"inr":24,"int":1230," fo":740,"ins":796,"inf":318,"aÅ¡Ä":181,"ine":633," fl":69,"inh":61,"ing":589,"iež":570,"inj":23," fi":803,"dÄÅ¡":37,"ini":1119,"ink":89," ge":43,"cÄ« ":27," bÄ":233,"ioa":45," ga":5014,"iod":270,"ļļa":40,"inu":278,"inv":33," i ":44,"iko":225,"ikn":25," cm":50,"ikm":90,"ikl":215," co":68,"iki":144,"ikg":62," ce":962," ch":25,"ike":59,"ikd":24," ci":2295,"ila":297,"ilb":52," da":4272,"in ":197,"ieÄ·":378,"ikv":30," cu":46,"ikt":677,"iku":1045,"ikr":195,"iks":380," do":494,"ilp":256,"ilo":443,"ill":168," dr":234,"ilk":121,"iln":265,"ilm":237,"ieņ":198,"ilh":58,"ilg":175," de":1205,"ilj":175,"ili":579,"ild":409,"ilc":81,"ieļ":23,"igÄ":206," di":3087,"ile":125,"ima":242,"imb":308,"ч ":32," g ":36,"io ":204," eb":89," du":128,"ilz":47,"idÅ«":60," dz":3456,"ils":2044,"ilt":346,"ilu":127,"ilv":752,"ль":27,"hs ":112,"bÄ«g":65," sÅ«":56,"bÄ«d":30," vÄ“":1059,"ми":22,"bÄ«b":430,"ло":25," vÄ":1227," tÄ«":340," zo":142,"ла":30," zu":23,"ле":23," rÅ«":113,"ли":29," zv":544,"hok":186,"hol":226,"hom":35,"hon":30," za":193,"ко":45,"hos":37," ze":740,"hot":32," zi":2010,"hop":25,"hor":75,"ка":54," tÄ“":247,"ки":52," sÄ«":84,"hni":146," sÄ“":228," tÄ":1796,"hno":151,"aņu":241," pÅ«":63," rÄ«":133,"aņe":35,"hme":29,"ин":34,"aņi":61," ww":36,"ий":38," sÄ":685,"ич":34,"aņo":46,"ри":31,"ро":53,"ра":42,"ре":36,"htt":39," zÄ":53,"hst":50,"оÑ":22,"ор":37,"ол":32,"aņģ":36,"ов":87,"hu ":43,"ое":22,"aņē":59,"нÑ":22,"но":31,"hro":158,"aņÄ":78,"ни":29,"hri":23," tÅ«":103," vÄ«":192,"ht ":30,"на":44,"bÄ«r":30,"bÄ«t":96,"bÄ«s":120,"bÄ«n":33,"cÄ“t":118," ru":206," jÅ«":849," mÄ“":678," u ":116," sa":6813," nÄ":219," sf":46," se":1483," sc":42," si":1753," sh":33," sn":112," sm":289," sl":596," sk":1357," mÄ«":111," sp":2239," so":351,"ве":29," qu":23,"ви":41," lÄ“":170," t ":61,"во":24," ra":2342," kļ":152," re":2229," mÄ":826," ri":726,"cÄ“n":32," kņ":49,"cÄ“m":41,"cÄ“l":179," ro":913,"cÄ“j":228," lÄ«":2107," pu":1266," pr":3156," ps":120," s ":86," lÄ":230,"ļņu":56,"ва":29," os":114," gÅ«":38," ot":330,"ļņi":27,"hum":50," op":221,"ав":30," or":1147,"ан":46,"ļņa":76," jÄ“":154,"ал":24," pe":651,"cÄ“ ":120," kÄ":2571," pa":8411,"ар":38," pc":24," pl":1389,"аÑ":29," po":1148," pi":5908," rÄ":73," x ":60," va":7044," ve":2127," pÄ“":1285," uz":2903," vo":159," pÄ«":83," vu":52," vi":7158,"еÑ":31,"ер":46,"ен":52," tv":38," tu":790," mÅ«":750," ut":55," ur":52," up":232," un":10091," ug":66," pÄ":1765," ta":1745," st":3025," kÅ«":39," sv":412," su":879," nÄ«":29," tr":1203," lÅ«":49," to":1082," th":155," ti":3365," te":2144,"aÄt":29,"fi ":28,"fes":176,"fer":133,"feo":25,"fed":168,"ņi ":361,"feb":35,"fen":84,"fek":185,"fel":37," ÄŒa":35,"ņin":63," ÄŒi":48,"ņie":244," ÄŒe":124,"fga":33,"faz":23,"fas":28,"ezÄ«":69,"far":40,"fan":93,"fak":97,"fal":45,"fai":36,"ņbr":28,"fab":102,"ņem":387,"ņda":33,"fe ":34,"ņos":47,"ņot":124,"ņoj":49," Čī":26,"evÄ“":231,"evÄ«":174," Ä“z":45," Ä“t":31,"ņra":135,"fa ":173," Ä’r":64,"ņs ":24," Ä“d":44," Ä“r":63," Ä“k":65,"esÄ“":107,"esÄ«":292,"erņ":62,"etÄ":683," ÄŒu":25,"ez ":111,"erÄ“":156,"erÄ«":393," Äe":400,"ņji":23,"erÄ£":206," Äa":23,"esÄ":195,"ezv":29,"ezu":188,"evÄ":40,"eza":38,"ezd":35,"etÄ“":183,"ezm":33,"ezn":141,"ezo":141,"ezp":31,"etÄ«":89,"erÅ«":27,"ezt":29,"eze":817,"ezg":53,"erÅ¡":24,"ezi":201,"ezk":125,"etb":351,"eta":734,"ete":624,"eti":845,"elÅ¡":89,"eth":26,"etn":170,"etl":29,"esp":338,"emÄ«":84,"esn":43,"eso":286,"est":937,"ņoÅ¡":22,"esu":212,"enÄ":28,"esr":33,"ess":340,"enÄ“":213,"esv":35,"ev ":24,"epÄ":130,"emÅ¡":50,"eto":1378,"enÄ«":969,"etr":698,"ets":222,"ett":58,"etu":1584,"etv":168,"ew ":38,"eve":83,"eva":376,"evo":146,"evn":29,"evi":1543,"enÅ¡":59,"eut":31,"eus":24,"ex ":26,"erÄ":709,"evu":397,"evr":24,"evs":91,"ey ":55,"epe":71,"ekÄ":290,"epi":258,"epj":76,"eph":37,"er ":1104,"epa":258,"eot":23,"egÅ«":271,"eos":34,"eor":322,"ņu ":2105,"eom":53,"eol":176,"eop":39,"eon":61,"elÄ":1125,"es ":8988,"ept":197,"eps":32,"epu":444,"epl":47,"epo":71,"epr":190,"erk":66,"erl":157,"eri":2539,"erg":167,"ere":536,"emÄ":318,"erf":37,"ekļ":537,"erc":178,"erd":99,"era":1626,"erb":238,"et ":1093,"elÄ“":112,"esk":278,"esl":110,"esm":388,"enÄ":1113,"ņus":89,"esi":374,"ekÅ¡":656,"esc":24,"ese":289," Ä’Ä£":41,"esa":337,"erz":36,"ery":29,"erv":331,"eru":391,"emÄ“":508,"ņve":32,"err":36,"elÄ«":103,"ert":417,"ers":1648,"ern":467,"erm":855,"erp":67,"ero":595,"eki":366,"ekl":427,"ekm":150,"ekn":62,"eko":416,"ekr":305,"eks":1397,"ekt":1750,"eku":811,"ekv":109,"en ":402,"elb":111,"ela":1449,"eld":81,"egÄ":92,"elf":85,"ele":1367,"eli":980,"elj":22,"elg":122,"elm":98,"eln":210,"elk":50," Ä¢e":106,"ell":209,"elo":182,"elp":198,"elu":464,"elv":59,"els":567,"elt":409,"elz":306,"eo ":63,"emb":176,"ema":412,"edž":46,"emg":143,"ehÄ":103,"eme":2607,"emd":28," Ä£e":323,"emn":25,"emo":348," Ä£i":768,"emi":389,"emj":71,"emt":133,"emu":60,"emp":493,"ems":89,"ep ":41,"ene":1004,"eng":203,"enb":97,"ena":2039,"end":388,"enc":397,"eno":1358,"enp":40,"enm":45,"enn":52,"enk":331,"enl":88,"eni":1040,"enu":734,"env":1058,"ens":2464,"ent":2847,"ehÄ«":26,"enr":79,"enz":110,"egÅ¡":47,"eog":110,"eof":26,"ejÄ":89,"eod":60,"egl":203,"ego":210,"ege":31,"egi":29,"eaÄ£":26,"egr":83,"egs":36,"egt":159,"egu":482,"egv":44,"ehn":276,"eho":29,"ecÄ":200,"ehi":58,"ek ":1542,"eib":23,"eic":317,"ecÄ“":53,"ecÄ«":715,"eis":229,"eir":180,"eim":142,"eil":42,"ein":253,"eih":26,"eik":735,"eie":54,"eid":2544,"eig":207,"edÄ":218,"eja":808,"el ":113,"edÄ“":86,"eiz":614,"eit":215,"eiv":56,"ejs":177,"ebÅ«":23,"edÄ«":123,"ejo":81,"ejn":76,"eji":80,"eke":31,"ekc":120,"eka":402,"em ":6412,"eju":260,"ejv":23,"gaž":22,"git":41,"gis":49," Ä«s":256,"gin":40,"aÄ£e":63,"gie":240," Ä«p":415,"aÄ£i":37,"ght":32," Īr":60," Īs":47,"gaļ":93,"bÄs":254,"bÄt":57,"bÄz":178,"gi ":519,"bÄj":44,"bÄl":44,"bÄk":295,"bÄn":43,"bÄm":208,"gen":244,"ger":118,"ges":34,"gel":54,"gej":38,"bÄ ":646,"ge ":89,"gab":646,"gad":2505,"gai":779,"gas":1682,"gar":786,"gau":305,"gat":362,"gav":387,"gaj":108,"gam":98,"gal":1339,"gan":1451,"ga ":1326,"frÄ«":26," Ä·Ä“":68,"aÄ“l":30," ļo":206," Ķī":169,"frÄ“":39," ļa":118,"žņu":81," Ä·Ä«":292,"Å…uj":43,"fut":631," Ä»e":28,"fta":36,"fun":189,"ft ":61,"fra":303,"fre":65,"fri":213," Ķe":55,"fu ":37," Ä·i":28,"fro":69,"aÄu":132," Ä·e":273,"for":1089,"fos":41,"fot":77,"fon":280,"fol":75,"ņak":24,"ņam":33,"ņas":543,"ņaz":78," ņe":38," Å…u":39,"fs ":175,"ņai":51,"ņa ":1018,"fle":24,"fla":25,"fli":29,"flo":42,"fic":294,"fig":49,"fij":235,"fil":435,"fik":181,"fin":189,"fir":37,"fis":214,"fiz":287,"da ":3140,"de ":574,"dac":25,"dab":346,"dak":25,"dal":823,"dai":150,"dag":58,"dae":523,"dat":711,"das":2562,"dar":1471,"dap":69,"dan":60,"dam":271,"dau":771,"dda":35,"cul":28,"cum":72,"cty":34,"cto":75,"cti":74,"cte":48,"cus":46,"cyo":60,"ceļ":571,"cle":31,"co ":36,"ciÄ":742,"cog":76,"con":50,"col":50,"com":55,"cor":52,"cop":28,"cot":97,"ciņ":28,"cs ":85,"ct ":32,"cro":67,"cu ":436,"cea":28,"ch ":124,"cer":171,"ces":699,"cet":115,"cen":1090,"cep":69,"cek":125,"cem":35,"cel":374,"ced":32,"ci ":164,"cha":72,"cia":105,"ck ":59,"cie":767,"che":97,"chi":53,"cho":24,"chn":24,"civ":69,"cij":3939,"cik":179,"cil":946,"cim":32,"cif":53,"cir":29,"cis":561,"cit":901,"ciu":34,"cin":303,"cio":768,"cip":250,"cm ":47,"cke":32,"cka":44,"ed ":130,"eba":49,"ebe":168,"ebi":87,"ebk":175,"ebo":22,"ebr":247,"ebs":44,"eak":105,"ean":79,"eal":83,"ear":51,"eas":36,"eap":54,"dzÄ«":2001,"eat":203,"eau":46,"eb ":1292,"dzÄ":72,"dzÄ“":373,"ea ":66,"efi":131,"efo":131,"efa":30,"efe":198,"ei ":338,"ega":166,"efs":31,"eej":99,"een":27,"edi":335,"ede":1042,"eda":646,"edz":1195,"eds":41,"edu":254,"edo":120,"edr":685,"eck":45,"ech":44,"eci":633,"ece":209,"eca":89,"ee ":59,"ecu":116,"ect":70,"ecp":24,"eco":62,"drÄ«":349,"dz ":1238,"doÅ¡":285,"drÄ":113,"dy ":22,"dvi":33,"dve":34,"doņ":93,"dur":107,"dus":742,"dva":54,"duÅ¡":31,"dzv":49,"dzs":170,"dzu":124,"dzo":52,"dzn":78,"duÄ":33,"dzi":2913,"dze":1097,"dza":294,"dor":118,"don":402,"dom":421,"dol":232,"dok":188,"dow":29,"dov":34,"dot":762,"dos":455,"diņ":89,"dpo":22,"ds ":1372,"dmi":319,"dne":44,"dni":257,"diÄ":75,"dob":64,"dod":87,"doe":23,"doj":533,"dnÄ«":68,"dun":24,"dum":173,"dul":78,"duk":158,"dug":33,"dub":73,"dua":23,"duc":94,"dri":640,"diž":52,"dra":350,"dre":139,"du ":1832,"dro":640,"drs":87,"dru":292,"dsi":206,"dsk":23,"dsm":48,"daļ":1548,"dic":163,"dia":195,"dib":246,"der":1382,"des":875,"det":70,"dev":375,"deb":147,"dea":22,"ded":49,"dec":38,"def":128,"deh":29,"deg":170,"dej":161,"dei":89,"del":244,"dek":100,"den":1101,"dem":139,"dep":138,"deo":95,"di ":661,"deļ":51,"dko":25,"dma":88,"do ":341,"deņ":285," Ä€z":143,"div":810,"diu":34," Ä€r":50,"dim":118,"din":621,"dio":398,"dip":35,"dir":59,"dis":577,"dit":40,"die":2511," Ä€f":113,"dif":53," Ä€g":35,"dig":41," Ä€d":23," Ä€b":22,"dij":481,"dik":129,"dil":24," Är":250," Ät":154," Äp":26,"daž":632," Äd":29,"ižu":32,"rgu":304,"raļ":69,"rhe":62,"raÄ·":99,"rha":55,"rhi":273,"rbÄ«":529,"mÄz":34,"raÄ“":35,"mÄt":506,"iža":92,"rga":943,"jÅ¡ ":26,"ri ":1063,"ižk":30,"rgi":83,"iže":26,"rge":111,"rbÄ":31,"rgs":168,"rgo":78,"ret":778,"res":696,"rev":103,"reu":25,"rez":384,"rfa":28,"mÄc":507,"mÄj":191,"mÄk":627,"mÄl":154,"mÄm":145,"mÄn":158,"rfo":45,"mÄr":102,"mÄs":220,"rdu":246,"rds":349,"rg ":40,"rdz":219,"reb":22,"rea":246,"ree":37,"ref":132,"rec":144,"red":582,"rei":822,"rej":271,"reg":233,"rem":185,"ren":387,"rek":208,"rel":259,"rer":29,"reo":23,"rep":144,"mÄ ":1508,"rda":365,"kļu":259,"rct":35,"rdo":96,"rdn":63,"rdi":228,"rde":305,"re ":399,"ņÄm":78,"ņÄs":22,"rbu":231,"rbs":88,"rco":80,"kļo":52,"kļi":136,"rci":226,"rch":63,"rce":59,"rca":34,"kļa":421,"rax":25,"raz":76,"rd ":116,"rap":69,"rar":28,"ras":4121,"rat":1049,"rau":810,"rav":242,"rbi":268,"rbo":531,"rba":232,"rbe":87,"raj":717,"lÄ“Å¡":123,"Å†Ä ":197,"rai":643,"rah":53,"rag":283,"ran":1071,"ram":1238,"ral":520,"rak":1567,"rab":128,"raf":152,"rad":1141,"rac":100,"rpt":521,"rpu":97,"rpr":64,"rpp":23,"rpo":72,"rs ":2853,"rpe":30,"rpa":43,"riÄ·":45,"riņ":229,"rpl":24,"rpi":90,"rkÄ":93,"ror":63,"ros":500,"rot":765,"rom":435,"ron":900,"roo":23,"rop":1052,"roz":140,"rou":41,"rov":266,"rob":624,"roa":30,"rod":1744,"roc":512,"lÄ« ":122,"roj":521,"roi":35,"riÄ£":56,"rol":216,"rok":423,"rof":236,"roe":58,"rog":545,"rno":46,"rns":56,"rnu":187,"rp ":573,"rgļ":35,"rna":307,"rež":135,"riÄ":400,"rne":333,"rni":347,"mÄÅ¡":32,"rmk":26,"mÄņ":51,"rmo":304,"rms":203,"kņu":24,"rmu":139,"ro ":308,"kņa":79,"rma":773,"rme":418,"rdž":57,"rmi":499,"reÅ¡":174,"rls":24,"rlo":32,"rli":126,"rgÄ":217,"ižÄ":28,"rld":35,"rle":24,"rla":201,"rn ":84,"rku":265,"rkt":251,"rks":248,"kļū":96,"rkn":35,"rko":61,"rki":73,"reÄ£":361,"rkl":58,"rke":63,"rka":377,"rbÅ«":24,"reÄ":45,"rdÄ«":30,"rja":44,"reÄ":83,"raž":305,"rje":48,"riz":128,"rdÄ“":22,"rip":74,"rio":330,"rit":1293,"ris":2568,"riv":81,"riu":29,"rih":80,"rig":98,"rij":2366,"raÅ¡":66,"ril":99,"rik":885,"rin":683,"rim":290,"ria":161,"rib":107,"ric":188,"rid":231,"rie":3740,"rif":114,"rdÄ":161,"rk ":63,"roÅ¡":448,"mÄ“d":92,"rož":23,"rsÄ":44,"mÄ“j":294,"mÄ“m":78,"mÄ“l":71,"mÄ“n":131,"mÄ“s":84,"mÄ“r":1231,"mÄ“t":199,"rsÄ“":59,"rtÄ":193,"rpÄ":40,"rud":44,"ruc":33,"rur":31,"roÄ«":36,"rup":644,"run":357,"rum":1852,"ruk":350,"ruz":42,"rpÄ“":106,"rus":878,"rut":84,"rva":432,"mÄ“ ":626,"rvi":327,"rve":210,"rvs":22,"lÄ«Ä":64,"roņ":62,"rvu":63,"ry ":107,"rsk":108,"rsl":46,"jÅ«l":25,"rsi":451,"rkÅ¡":28,"rso":354,"rsp":142,"rsm":215,"rsn":57,"jÅ«d":35,"rsd":23,"rsa":148,"rse":69,"rnÄ":130,"rta":1012,"jÅ«t":39,"rst":685,"rss":51,"jÅ«r":881,"rnÄ“":53,"rsv":73,"rsu":128,"rtl":32,"rtn":59,"rto":248,"rnÄ«":124,"rte":168,"rth":30,"rti":792,"rts":333,"rtr":97,"roÄ":76,"rtu":328,"lÄ«m":186,"lÄ«n":460,"riÅ¡":83,"lÄ«j":172,"lÄ«g":392,"lÄ«c":164,"lÄ«d":1562,"lÄ«b":213,"rt ":437,"lÄ«z":66,"lÄ«v":121,"lÄ«t":444,"rri":36,"rre":54,"rmÄ":762,"rra":96,"ru ":2726,"rmÄ“":77,"rlÄ«":45,"sab":330,"sac":442,"sad":310,"sag":154,"sai":857,"mÄ“Å¡":100,"saj":138,"sak":466,"sal":1082,"sam":248,"sbi":144,"rzÄ«":41,"sap":162,"san":223,"sau":2277,"sat":285,"sas":1660,"sar":747,"sav":1100,"saz":48,"sa ":1042,"mÄ“Ä£":49,"rsÅ«":33,"rvÄ“":141,"ruÅ¡":59,"ruņ":155,"rze":250,"ruÄ":39,"rza":50,"rtÄ“":197,"lÄ«Å¡":88,"rvÄ":75,"rzs":23,"rtÄ«":278,"rzi":284,"saņ":59,"sho":31,"shi":22,"si ":826,"kÅ¡ ":48,"sga":35,"nÄv":198,"nÄt":1807,"nÄs":1024,"sgr":33,"saÄ«":148,"siz":24,"saž":44,"sie":635,"sid":107,"kÅ¡d":149,"sic":34,"sib":36,"kÅ¡a":264,"sia":56,"sk ":23,"kÅ¡t":50,"sit":335,"kÅ¡u":53,"sir":69,"sis":1127,"kÅ¡s":52,"kÅ¡p":164,"sip":27,"sin":756,"kÅ¡n":32,"sio":148,"sil":229,"kÅ¡l":23,"kÅ¡m":75,"sim":594,"sij":423,"sik":99,"kÅ¡k":106,"sih":129,"sif":65,"sig":89,"nÄ ":1513,"sda":48,"sdi":356,"sbu":40,"se ":542,"sca":22,"sce":66,"sci":117,"sch":88,"sco":39,"sev":271,"ser":334,"ses":362,"set":54,"sfa":26,"sez":117,"sh ":43,"nÄj":705,"nÄd":289,"nÄc":166,"sfo":68,"nÄr":126,"nÄl":1149,"nÄk":422,"nÄm":427,"sdz":22,"sei":104,"seg":72,"sed":36,"sec":62,"sep":119,"sen":630,"sem":62,"sel":189,"sek":324,"sej":83,"spu":61,"spo":584,"siņ":49,"kšņ":49,"spr":198,"skÄ":2459,"spe":377,"spl":61,"spi":604,"kÅ¡Ä·":38,"spa":151,"sot":106,"sov":22,"sol":116,"som":91,"son":503,"sor":227,"sos":35,"sod":41,"sof":59,"kÅ¡Ä£":28,"soj":40,"soc":383,"su ":834,"smÄ“":44,"nÄu":194,"sjÅ«":46,"slÄ«":114,"kšž":25,"smÄ":173,"nÄa":33,"sra":53,"st ":664,"slÄ“":278,"mÄ«t":118,"mÄ«n":84,"mÄ«k":32,"mÄ«l":44,"mÄ«g":202,"slÄ":344,"mÄ«b":243,"mÄ«d":29,"ss ":875,"sli":399,"slo":148,"slu":68,"sfÄ“":192,"sla":592,"sle":81,"ski":1153,"sko":1104,"sks":400,"skr":170,"sku":481,"ska":4650,"ske":410,"sno":53,"kÅ¡Ä“":181,"sns":27,"sna":41,"nÄÅ¡":340,"sni":516,"sng":29,"sne":180,"kÅ¡Ä":24,"smo":229,"shÄ“":40,"sms":242,"smu":340,"so ":75,"sma":1140,"smi":429,"seÅ¡":62,"sme":291,"soÅ¡":166,"nÄ“t":549,"nÄ“s":72,"nÄ“m":160,"nÄ“j":281,"nÄ“z":87,"stÄ":3123,"sze":116,"suÄ":44,"stÄ“":996,"stÄ«":1710,"svÄ":52,"sse":108,"ssa":247,"sso":61,"ssk":34,"ssi":62,"ssv":26,"kÅ«v":24,"smÄ«":51,"ssp":22,"snÄ“":32,"ste":1133,"sta":3898,"stm":49,"stn":212,"sto":1125,"stp":85,"sti":2606,"stl":22,"stv":62,"stu":1969,"str":3208,"snÄ«":38,"sts":1496,"sud":63,"sub":69,"spÄ":202,"sug":619,"sul":139,"sum":128,"sup":49,"sun":42,"sus":127,"sur":82,"spÄ“":1745,"suv":25,"nÄ“ ":308,"sva":373,"sve":113,"svi":110,"soņ":53,"spÄ«":75,"nÄ“Å¡":36,"tai":828,"taj":702,"tak":124,"tal":256,"taf":30,"tag":177,"tab":171,"tac":309,"tad":222,"tbi":216,"tba":91,"tav":303,"tau":931,"tat":537,"tas":3364,"tar":1920,"tap":52,"tan":395,"tam":934,"tce":25,"te ":724,"tbo":1157,"tbr":38,"tdi":61,"tda":95,"svÄ“":166,"suņ":88,"stÅ«":113,"svÄ«":41,"ta ":6165,"ozÄ":24," Å¡t":206," Å¡i":160,"pa ":730," Å¡o":253," Å¡a":314,"iÄ·u":54," Å¡e":87,"iÄ·i":331," Å v":81,"iÄ·a":30," Å t":29,"iÄ·e":39," Å o":131," Å i":163," Å e":104,"ovÄ«":25," Å a":199,"ovÄ“":187,"osÅ«":27," Å Ä·":45,"pdr":26," Å¡Ä":147,"iļu":60," Å Ä«":134,"kÄ ":4239,"iļi":22,"pci":27," Å Ä":65,"iļa":27,"pe ":174,"par":3884,"ozÄ«":528,"pat":548,"pas":2092,"pav":354,"pau":149,"paz":430,"pba":51,"pac":124,"pad":181,"paa":70,"pab":25,"pag":415,"pak":786,"pal":445,"pai":59,"pap":138,"pam":682,"pan":195,"pc ":24,"iļÄ":27,"paļ":30,"pha":38,"paņ":40,"phi":54,"pga":664,"kÄz":22,"pi ":142,"pgr":35,"kÄd":424,"kÄb":641,"kÄc":170,"kÄp":196,"kÄn":152,"kÄt":32,"kÄs":1168,"kÄr":1333,"paÄ":29,"kÄm":189,"kÄl":251,"kÄk":39,"kÄj":89,"pea":50,"pec":223,"ped":72,"pdz":192,"pen":158,"per":1152,"pet":33,"pes":264,"pei":23,"pel":224,"pek":146,"peļ":35,"pla":1300,"pli":106,"pgÄ":32,"ple":241,"plo":77,"plu":29,"pka":321,"pko":110,"pja":69,"iļņ":141,"pjo":49,"pju":63,"pie":4300,"pig":22,"paÅ¡":762,"pij":105,"pil":2275,"pin":133,"pio":305,"pir":966,"pis":387,"pit":151,"poz":106,"por":630,"pop":247,"pot":153,"pos":221,"poj":154,"pog":70,"pon":210,"pok":25,"pol":767,"pod":51,"ps ":241,"ppu":25,"ppl":37,"piņ":42,"pkÄ":95,"iņu":475,"iņs":42,"iņo":51,"kÄņ":216,"iņi":157,"iņj":22,"pme":52,"pma":68,"iņa":746,"po ":65,"gžņ":89,"pni":63,"pne":24,"psu":31,"pst":387,"iņķ":170,"pta":564,"pse":42,"ņķo":49,"ņķi":80,"psi":158,"psk":78,"ņķa":92,"pso":28,"ptu":237,"ptv":61,"pub":494,"pte":121,"pti":177,"pto":59,"pnÄ«":27,"plÅ«":201,"plÄ“":184,"pra":501,"jÄ«g":32,"plÄ":202,"pmÄ“":120,"pru":34,"psa":72,"pu ":321,"pmÄ":37,"iņÄ":150,"pri":1091,"pre":942,"pro":1797,"plÄ«":101,"poÅ¡":65,"pož":46,"prÄ“":65,"prÄ«":60,"psÄ“":31,"pur":145,"pmÅ«":26,"pus":720,"put":341,"pum":233,"pun":290,"ņģē":37,"iņš":323,"pul":349,"poļ":35,"pva":35,"pvi":404,"prÄ":125," Å…Å«":40,"ptÅ«":22,"pzi":58,"ptÄ«":28,"prÅ«":42,"ņģe":27,"pzÄ«":361,"lÄ ":1304,"lÄr":401,"lÄs":590,"lÄt":209,"lÄm":367,"lÄn":215,"lÄp":55,"lÄj":219,"lÄk":1056,"lÄg":85,"lÄd":144,"lÄc":180,"lÄv":61,"lÄz":36,"iÅ¡a":70,"iÅ¡i":31,"iÅ¡u":56," Ži":23," Že":35," Ža":46,"lÄÄ":85,"lÄÅ¡":22," ža":153," že":29," žu":96,"lÄņ":65,"iÅ¡Ä·":223,"išņ":36,"ņēm":347,"qua":27,"lÄ“ ":420,"quu":124,"que":29,"qui":79," Å¡Ä«":215," Å¡Ä·":525,"lÄ“d":76,"lÄ“c":25,"lÄ“m":141,"lÄ“n":178,"lÄ“k":87,"lÄ“l":48,"lÄ“j":178,"lÄ“g":138,"lÄ“s":295,"lÄ“t":383,"lÄ“r":42,"lÄ“p":64," Å¡Å«":130," Ūd":31," Å«d":556,"ra ":3874,"ežo":374,"ngo":130,"ngi":47,"eži":331,"ngl":303,"ngv":61,"ežu":144,"ngu":183,"ngr":212,"ngt":36,"ngs":108,"ni ":1261,"nge":139,"eža":263,"nga":229,"ncÄ“":85,"nha":38,"nhi":24,"nhe":54,"neg":80,"nej":92,"nei":201,"nel":264,"nek":293,"nen":247,"nem":97,"nep":336,"neo":76,"ner":698,"net":418,"nes":1590,"nev":218,"ndz":31,"ng ":184,"nea":224,"neb":55,"nec":44,"ned":153,"nfi":27,"nfo":273,"iÄn":89,"iÄl":1004,"nfl":40,"nfr":48,"nez":35,"iÄc":169,"nfe":88,"nci":571,"gļi":38,"nce":476,"nch":29,"nca":32,"ne ":1014,"nbu":52,"ndu":273,"ndr":359,"nds":68,"ndo":248,"ndi":727,"nde":437,"nda":780,"gļu":239,"nak":85,"nal":235,"nam":315,"nan":170,"nar":136,"nac":383,"nad":66,"nae":90,"naf":23,"nag":57,"nai":923,"naj":193,"nab":23,"nbe":61,"nd ":206,"nav":456,"nau":117,"nat":306,"nas":3073,"naz":22,"na ":4384,"muļ":23,"muÅ¡":43,"hÄ“m":45,"mtÄ":193,"ntÄ":395,"nsÄ“":48,"noÅ¡":193,"ny ":39,"nvi":1034,"nux":29,"nve":133,"nva":62,"nuk":32,"nul":25,"num":139,"nus":295,"noÄ«":26,"nto":950,"ntu":416,"nts":925,"ntr":875,"nti":950,"nth":28,"nta":799,"nte":768,"nsu":59,"nsv":36,"nsp":162,"nso":88,"nst":540,"nss":27,"nkÅ«":25,"nsf":27,"nse":131,"nsg":24,"nsi":192,"nsl":44,"nsk":554,"nsa":85,"nu ":3157,"nmÄ“":32,"iÄu":24,"iÄs":33,"nrs":66,"iÄi":25,"nri":78,"iÄa":38,"nra":85,"nt ":98,"hÄ«d":23,"nkÄ":234,"niņ":82,"ns ":4213,"noc":58,"nod":505,"noa":58,"nob":42,"nog":200,"nof":35,"nok":292,"nol":424,"noj":304,"nop":85,"nom":610,"non":115,"not":1518,"nos":1301,"nor":435,"nov":804,"noz":696,"npa":23,"niÄ·":23,"nne":83,"nna":118,"ngļ":211,"nno":31,"nni":41,"nme":24,"nma":29,"iÄņ":39,"ežģ":48,"neÅ¡":108,"ndž":62,"ežī":26,"ežÄ":198,"ngÄ":136,"nn ":32,"nla":94,"neļ":30,"nle":25,"no ":5915,"nke":26,"nki":45,"nkc":180,"nka":173,"nku":121,"nko":53,"gļū":52,"nks":81,"nkt":272,"nkr":126,"nja":39,"ndÄ«":35,"njo":28,"nij":1586,"nig":62,"nif":36,"ndÄ":228,"nie":3147,"nid":123,"nic":83,"nia":73,"nk ":29,"niz":488,"ndÄ“":98,"niv":242,"nis":2463,"nit":151,"nir":27,"nio":28,"nip":37,"nim":128,"nin":84,"nik":329,"nil":57,"ogs":183,"ogr":713,"ogu":116,"ogi":122,"ogl":96,"ogo":65,"ogn":46,"oga":163,"obÄ":56,"ņš ":323,"obÄ«":23,"ocÄ“":27,"oho":29,"ohn":25,"oha":113,"odÄ“":120,"ois":41,"oin":31,"ocÄ«":54,"odÄ":459,"iÄ£i":165,"iÄ£e":27,"ok ":31,"gÅ¡a":114,"oju":1293,"odÄ«":31,"ojo":586,"oji":109,"oje":183,"oja":1186,"ol ":51,"oce":578,"och":26,"oci":581,"ock":58,"oco":46,"obs":66,"obu":91,"oca":32,"ode":548,"odi":313,"odo":337,"odn":41,"ods":132,"odr":212,"ocy":51,"jÄ ":4204,"of ":167,"oda":2367,"oel":45,"oef":31,"oei":29,"oen":27,"odz":39,"odu":575,"jÄm":1042,"jÄn":27,"ofi":331,"jÄj":22,"jÄs":923,"ofs":60,"oft":50,"ofo":72,"ofe":168,"jÄd":51,"jÄb":32,"ofa":27,"nzÄ“":40,"oal":24,"oak":87,"oba":54,"od ":93,"obo":33,"obr":164,"obl":72,"obj":208,"obi":154,"obe":704,"nsÄ«":306,"ntÄ“":146,"Ø© ":22,"nza":50,"nzi":41,"nzo":53,"ntÄ«":340,"nvÄ":46,"ntÅ«":58,"otÄ":1010,"orÄ·":25,"orÄ«":50,"opÅ¡":220,"jÄ“j":29,"osÄ":36,"jÄ“d":148,"orÄ“":91,"ows":32,"ovÄ":78,"gÅ«Å¡":56,"ozu":30,"otÄ«":57,"ozo":189,"oze":115,"ozi":143,"oza":288,"otÄ“":115,"orņ":38,"otu":305,"oti":1556,"oth":36,"ote":602,"ott":59,"olÅ«":125,"ots":723,"otr":369,"onÄ«":135,"oto":694,"otn":222,"onÄ“":180,"ost":506,"gÅ«t":168,"osu":40,"ota":947,"osi":149,"osk":156,"ose":88,"osf":203,"onÄ":1526,"osp":73,"oss":48,"gÅ«s":114,"gÅ«r":45,"osm":456,"osl":225,"oso":109,"orÄ":211,"owe":22,"ovi":389,"ovg":31,"opÄ«":120,"ovs":143,"opÄ“":176,"ox ":29,"jÄ“ ":37,"ova":643,"ove":137,"opÄ":322,"oun":55,"ous":25,"our":37,"out":44,"opm":30,"opo":347,"opi":253,"opl":68,"ope":301,"okÄ":278,"oph":40,"opa":835,"os ":3856,"opu":475,"opr":71,"opt":86,"ops":93,"ojÄ":198,"or ":99,"oot":22,"oor":77,"ork":107,"orl":52,"orm":1086,"orn":222,"oro":248,"orp":128,"olÄ«":129,"orr":39,"orc":49,"okļ":89,"ord":492,"ore":247,"omÄ":452,"orf":64,"org":872,"orh":26,"ori":1689,"osa":935,"ort":723,"ors":691,"orv":185,"oru":295,"omÄ“":208,"orz":33,"ory":23,"olÄ":244,"ot ":1449,"m² ":52,"orb":148,"ora":653,"olÄ“":98,"olb":29,"ola":1498,"old":135,"on ":659,"oli":1155,"oll":82,"olk":110,"olf":57,"ogÄ":71,"ole":441,"olg":32,"ols":417,"olt":46,"olm":107,"oln":31,"olo":1146,"olu":413,"ogÄ“":50,"oka":339,"om ":55,"oki":85,"okh":49,"oke":542,"okr":192,"oks":506,"oko":208,"okl":145,"okm":47,"okt":89,"oku":309,"ona":1194,"ond":224,"ogļ":73,"onc":136,"onf":111,"one":173,"ong":158,"oni":1530,"onk":159,"onn":56,"ono":626,"onr":30,"ons":1271,"ont":491,"onu":567,"onv":60,"gÅ¡Ä“":65,"ony":39,"onz":36,"oma":1047,"ome":546,"omb":95,"omi":730,"omm":51,"omj":98,"omp":487,"omo":191,"omt":26,"omu":284,"oms":200,"op ":56,"la ":3657,"eÄ·u":303,"kuļ":50,"ktÅ«":291,"kuÅ¡":110,"eÄ·i":82,"le ":615,"eļd":25,"lce":40,"eļe":30,"eļa":635,"lci":93,"eļi":128,"eļv":32,"eļu":421,"eļo":286,"eļr":165,"gÄ ":654,"lde":303,"lda":256,"ldo":68,"ldn":139,"ldi":101,"ldv":22,"ldu":71,"lds":71,"ldr":27,"lab":686,"lac":74,"lad":63,"lah":38,"lag":32,"laj":474,"lai":2379,"lak":285,"lan":766,"lam":314,"lap":246,"lar":131,"lat":1524,"las":2361,"lau":462,"lav":247,"lay":44,"laz":38,"lba":32,"ld ":81,"lbe":128,"lbu":171,"lbr":107,"kvi":68,"kve":97,"kva":316,"kuu":25,"kut":36,"kmÅ«":49,"kus":576,"kur":4124,"kup":73,"kun":181,"kum":1594,"kul":627,"kuk":32,"kuj":25,"koÅ¡":92,"krÄ":588,"koņ":106,"kta":560,"kte":51,"knÄ“":45,"kss":117,"kmÄ«":26,"ksp":156,"ksu":120,"kst":1947,"ksk":31,"cÅ«k":37,"ksi":433,"kso":59,"ksn":205,"ksm":118,"ksl":373,"ktr":699,"kts":634,"ktu":492,"kti":662,"kto":408,"ksÄ«":103,"ktÄ“":73,"kuÄ£":144,"krÅ«":64,"ktÄ«":473,"ksÄ":78,"krÄ“":127,"ktÄ":172,"krÄ«":67,"fÄ“r":208,"ksÄ“":49,"lpo":192,"liņ":107,"lps":165,"lkÄ":113,"lpe":42,"lpi":52,"dÅ« ":66,"ls ":2358,"lpu":76,"lok":186,"lon":250,"lom":194,"lop":98,"lor":215,"lod":1255,"loc":172,"log":309,"loj":39,"liÄ£":139,"lpa":98,"los":267,"lot":233,"lov":125,"loz":171,"lno":58,"lnk":23,"lni":256,"lne":55,"lob":58,"lnv":34,"lnu":164,"lns":198,"eņo":30,"lmi":75,"eņi":138,"lme":73,"eņe":150,"eņb":28,"lma":269,"eņa":350,"gÄļ":23,"lna":214,"lmu":99,"eņu":143,"eņr":134,"lms":33,"ltk":80,"lti":424,"ltn":68,"lto":53,"ltr":29,"lts":217,"lnÄ«":100,"ltu":221,"lud":54,"luc":32,"lub":111,"lug":36,"lpÄ":54,"lsi":33,"lkÅ¡":22,"lsk":140,"lsm":135,"lsn":30,"lso":131,"lsp":34,"lss":53,"lst":1735,"lsu":49,"eņķ":69,"lv ":26,"lta":371,"lte":188,"lkņ":24,"lmÄ":69,"lu ":1807,"eņē":35,"lnÄ":79,"lse":64,"dÅ«d":48,"lsa":85,"lt ":35,"lhe":59,"eļÄ":42,"lgu":44,"lgs":36,"lgt":25,"lgr":44,"lgo":65,"lbÄ":28,"lge":25,"li ":1126,"gÄz":197,"lga":178,"gÄr":144,"lfr":22,"gÄs":198,"gÄt":113,"gÄn":110,"gÄj":129,"gÄk":229,"gÄl":68,"gÄm":215,"gÄc":40,"gÄd":124,"lfa":193,"lez":112,"lev":95,"les":1626,"let":176,"ler":213,"leo":70,"lep":67,"lem":358,"len":329,"lek":1222,"lei":141,"lej":190,"leg":36,"lef":47,"led":254,"lec":71,"lea":23,"lfÄ«":26,"lls":27,"ldÅ«":50,"llu":78,"lo ":480,"lla":140,"lle":160,"lgÄ":68,"lli":182,"leņ":73,"llo":82,"lko":184,"lku":31,"lks":97,"lka":119,"lke":27,"lki":33,"eļš":95,"leÄ£":37,"lkl":39,"leÄ":51,"ljo":123,"ldÄ«":542,"eļļ":42,"lje":30,"ll ":123,"lja":61,"eļņ":24,"lit":895,"lis":1779,"lir":22,"lip":94,"lio":99,"lin":517,"lim":573,"liz":308,"ldÄ“":33,"liv":35,"lic":208,"lid":326,"lia":94,"lib":49,"lik":929,"laÅ¡":379,"eļģ":25,"lij":964,"lig":100,"lie":3537,"lif":131,"ldÄ":34,"ma ":2997,"mac":23,"mai":882,"maj":174,"mak":194,"mad":68,"mag":255,"map":23,"lzÄ«":23,"mar":276,"mas":1459,"mal":305,"mam":184,"man":1880,"maz":508,"mat":1537,"mba":149,"mbl":29,"mbi":138,"mbe":47,"mbr":158,"mbo":287,"me ":570,"mbu":166,"mda":33,"mde":27,"mdo":23,"med":310,"meg":22,"met":1247,"mes":1625,"mer":791,"mem":29,"mel":157,"men":1391,"mei":147,"meh":105,"mek":209,"mez":28,"hÄr":41,"mfo":25,"hÄn":110,"lva":502,"lve":648,"lvi":41,"luk":23,"loÄ£":845,"lup":35,"luo":34,"lum":401,"lut":87,"lus":425,"lur":30,"loÄ«":24,"ly ":28,"loÅ¡":29,"loņ":26,"lvo":84,"lvu":57,"lož":83,"ltÄ":300,"gÄ“n":73,"lsÄ“":1122,"lzi":33,"lza":40,"lzc":242,"lsÄ«":24,"lvÄ":83,"ltÄ«":58,"lzs":43,"lvÄ“":725,"ltÅ«":225,"mpi":549,"mpe":254,"mpr":55,"mpo":156,"miņ":109,"mpl":189,"mpu":71,"mps":29,"mpt":22,"ms ":3789,"mog":37,"moc":27,"mob":84,"mod":286,"mon":399,"mop":26,"mok":98,"moj":43,"mom":24,"mol":217,"mor":220,"mos":1307,"mot":190,"mpa":147,"eÅ¡Ä·":123,"miÄ·":31,"mmÄ":27,"mmÄ“":68,"msa":67,"mu ":2868,"mt ":26,"mtu":58,"mts":51,"mti":48,"mug":77,"mpÄ":152,"mss":33,"mst":25,"msu":24,"msk":121,"mte":23,"mta":1054,"mvi":24,"moÅ¡":35,"mur":78,"mus":558,"mut":78,"mui":109,"mul":121,"mum":318,"mun":151,"mpÄ“":250,"muz":106,"maņ":208,"džu":100,"džs":35,"mga":146,"dža":84,"mi ":937,"maÄ£":28,"dži":70,"dže":114,"mju":201,"ml ":23,"mje":32,"min":1418,"mio":29,"eÅ¡o":30,"mil":392,"mir":718,"eÅ¡s":32,"mis":1428,"eÅ¡v":29,"mit":475,"eÅ¡u":1242,"miz":31,"mic":35,"eÅ¡a":400,"mig":64,"mie":997,"eÅ¡d":35,"mid":33,"mik":313,"mij":472,"maÅ¡":145,"eÅ¡i":317,"mo ":301,"meņ":339,"meļ":1113,"džÄ":37,"mko":36,"mm ":35,"mni":192,"mna":27,"eÅ¡Ä":114,"mež":103,"mmu":74,"mma":291,"mme":33,"vÄ« ":36,"tÅ¡Ä·":209,"uņi":48,"uņo":84,"uņu":131,"tÅ«t":53,"tÅ«r":1186,"tÅ«k":66,"tÅ«c":93,"vÄ«n":52,"vÄ«t":70,"vÄ«r":237,"vÄ«g":77,"vÄ«d":56,"vÄ«b":202,"vÄ«z":153,"vÄ“ ":134,"Ä«Äu":22,"Ä«Äa":67,"sÅ«t":95,"vÄ“l":461,"vÄ“k":693,"vÄ“j":244,"vÄ“c":27,"ÄŒeh":54,"vÄ“s":501,"vÄ“t":425,"vÄ“r":790,"vÄ“n":31,"vÄ“Ä£":76,"ÄŒik":22,"uļo":37,"uļv":73,"uļu":71,"uļi":36,"uļa":32,"vÄ“Å¡":46,"tļa":51,"tļu":59,"tļi":52,"tļo":22,"vÄ ":578,"vÄj":43,"vÄc":471,"vÄl":70,"vÄk":140,"vÄn":38,"vÄm":138,"vÄr":1021,"vÄt":107,"vÄs":151,"vÄv":76,"ržu":43,"uÄ£a":52,"uÄ£i":59,"uÄ£u":36,"tņu":72,"Äa ":145,"zra":210,"zru":92,"zjÅ«":39,"tÄ«d":188,"zlÄ":29,"tÄ«b":1094,"zs ":189,"tÄ«t":1190,"tÄ«s":433,"tÄ«v":934,"tÄ«r":77,"tÄ«k":237,"tÄ«n":159,"tÄ«m":268,"tÄ«g":275,"tÄ«j":350,"zte":106,"zti":66,"ztu":134,"Äem":249,"ztv":96,"Äet":144,"znÄ«":263,"Äer":57,"zsa":231,"rÅ«d":56,"znÄ":215,"zmÄ“":101,"zu ":183,"rÅ«t":48,"zst":360,"rÅ«s":59,"zsv":33,"znÄ“":22,"zsl":25,"zsk":600,"rÅ«k":50,"rÅ«n":90,"rÅ«m":55,"zsp":47,"rÅ«p":88,"zva":673,"zvi":325,"zve":584,"Äi ":69,"zul":164,"zum":137,"zpÄ“":51,"zus":63,"Äie":24,"zsÄ":72,"tīņ":237,"zrÄ":47,"tÄ«Ä£":32,"zzi":39,"zuÄ":39,"Ä«Ä£e":36,"tÄ«Å¡":166,"zvÄ“":121,"rÅ«Å¡":31,"Äs ":40,"zgl":134,"zga":142,"rÅ¡ ":626,"zi ":317,"uÄr":73,"uÄl":181,"zaļ":79,"zgu":59,"zef":34,"zej":232,"zdz":26,"zeb":60,"zdo":186,"uÄc":50,"zes":430,"zen":285,"zem":1268,"zel":450,"zek":170,"zer":903,"ze ":308,"zce":422,"zbr":68,"zda":107,"zde":234,"zci":73,"zab":39,"īļu":88,"zai":49,"tÄ“Å¡":90,"zah":57,"zam":111,"zan":24,"zak":23,"zar":328,"zau":128,"zav":27,"zas":275,"zat":43,"zod":22,"zob":42,"tÄ« ":235,"zos":43,"zot":69,"zor":28,"zop":22,"zom":26,"zon":281,"zol":154,"zof":144,"ziļ":81,"ziÄ·":104,"zpa":78,"zpr":45,"ال":32,"zpl":262,"ziņ":261,"zkÄ":32,"zpi":173,"zo ":48,"zma":918,"zme":35,"zmi":84,"zna":38,"zmu":41,"zno":68,"zne":198,"rÅ¡Ä":79,"zni":156,"zbÅ«":81,"zka":34,"zkl":48,"zkr":193,"zla":642,"zgÄ":25,"zli":78,"zeļ":55,"zeņ":47,"rÅ¡a":176,"zie":1720,"zid":135,"zij":625,"rÅ¡i":100,"zin":1174,"zim":1926,"zil":123,"zik":669,"rÅ¡o":29,"zio":48,"zcÄ«":73,"rÅ¡r":51,"zir":324,"zis":229,"zit":58,"rÅ¡u":90,"ziv":58,"tÄ“ ":318,"yst":46,"rņÄ":31,"sÄ«k":51,"sÄ«j":64,"sÄ«m":27,"sÄ«n":31,"sÄ«s":28,"sÄ«t":105,"sÄ«v":85,"sÄ«b":502,"sÄ«d":98,"sÄ«g":56,"tÄž":25,"yon":65,"sÄ« ":32,"tÄ“Ä£":40,"īņÄ":28,"za ":232,"Äža":23,"Äžu":50,"īņu":254,"īņa":76,"sīļ":23,"tÄ“k":46,"tÄ“l":335,"tÄ“m":821,"tÄ“n":44,"tÄ“j":348,"tÄ“t":204,"tÄ“s":73,"tÄ“v":63,"tÄ“r":175,"tÄ“z":37,"Ä«Å¡a":499,"Ä«Å¡u":52,"tÄ ":2912,"tÄc":148,"tÄd":527,"tÄk":538,"tÄl":1306,"tÄj":1760,"rÄ·u":27,"rÄ·e":31,"rÄ·i":179,"yla":29,"tÄļ":48,"rņa":66,"rņu":41,"tÄÅ¡":25,"tÄv":1332,"tÄz":22,"tÄp":79,"tÄm":538,"tÄn":411,"tÄs":1054,"tÄt":740,"tÄr":419,"rÄ«z":162,"rÄ«n":188,"rÄ«j":40,"rÄ«k":162,"rÄ«l":26,"rÄ«v":387,"rÄ«s":243,"rÄ«t":190,"rÄ«g":665,"rÄ«b":593,"rÄ«d":236,"rÄ«c":210,"pÅ«t":23,"sÄļ":43,"rÄ« ":2361,"rÄ«Å¡":55,"pÅ«Å¡":30,"sÄ“ ":157,"sÄ“j":316,"sÄ“k":27,"sÄ“m":89,"sÄ“n":52,"sÄ“d":44,"sÄ“s":30,"sÄ“r":135,"sÄ“t":1211,"ožu":58,"oža":54,"sÄt":137,"sÄs":67,"sÄr":68,"sÄp":24,"sÄn":37,"pÅ¡ ":215,"sÄc":43,"sÄj":28,"sÄk":643,"sÄl":103,"sÄm":92,"ožÄ":27,"pÅ¡u":22,"rÄ£i":226,"pÅ¡a":33,"rÄ£e":23,"rÄ“Ä·":47,"sÄ ":333,"rÄ“Å¡":122,"rÄ“ ":110,"ww ":36,"Äļu":173,"Äļi":45,"www":36,"rÄ“t":449,"rÄ“s":31,"rÄ“m":93,"rÄ“n":56,"rÄ“j":380,"rÄ“k":33,"rÄ“l":48,"rÄ“d":88,"rÄļ":69,"rÄņ":44,"oÅ¡Ä":198,"rÄÅ¡":23,"ws ":30,"oÅ¡Ä«":35,"wor":28,"rÄ ":1557,"pļa":22,"rÄk":869,"rÄl":881,"rÄm":402,"rÄn":164,"rÄg":38,"rÄj":67,"rÄc":432,"rÄd":816,"rÄf":309,"rÄb":117,"wer":27,"nže":53,"rÄr":31,"rÄp":24,"rÄv":135,"rÄs":553,"rÄt":492,"rÄz":74,"oÅ¡s":181,"oÅ¡u":115,"oÅ¡i":415,"oÅ¡o":147,"oÅ¡a":1342,"vze":33,"vvÄ“":32,"Äņi":41,"Äņu":468,"vuÅ¡":26,"Äņa":65,"war":33,"pÄ«t":24,"pÄ«r":89,"pÄ«n":23,"pÄ«l":42,"viÅ¡":195,"pÄ«g":88,"pÄ«d":81,"pÄ«b":56,"vs ":473,"vri":37,"vst":90,"vsk":63,"vu ":956,"vus":141,"vmÅ«":35,"vum":201,"vuk":61,"vul":71,"vva":94,"voÅ¡":86,"pīļ":68,"ÄÅ¡a":527,"via":49,"nÅ¡a":32,"vio":33,"vir":905,"vik":41,"vil":371,"vin":313,"vig":52,"vij":2101,"vic":31,"vid":1832,"vie":6969,"viz":69,"nÅ¡u":53,"vit":157,"nÅ¡t":29,"vis":1620,"vji":56,"vju":71,"vo ":339,"oņa":59,"veÅ¡":46,"oņi":162,"oņu":145,"viÄ":31,"vna":31,"vni":238,"viÄ":61,"vod":34,"voj":423,"vol":130,"vok":149,"von":248,"vor":27,"vot":667,"vos":64,"vpa":23,"viļ":118,"vpi":79,"viņ":404,"vpr":25,"vaņ":38,"vaļ":151,"vi ":493,"vgo":31,"ver":992,"ves":531,"vet":45,"vej":30,"vei":2843,"ven":961,"vem":34,"vel":158,"vek":42,"ved":123,"vec":250,"oļu":172," − ":35,"vda":91,"uzņ":254,"ve ":176,"val":3215,"vak":43,"van":198,"vam":39,"var":1672,"vat":135,"vas":1579,"uzÄ“":26,"vab":47,"vad":1354,"vai":3704,"vaj":267,"pēļ":144,"va ":982,"mūž":31,"uvÄ“":31,"uvÄ":81,"uzv":164,"uzl":72,"uzk":39,"urÅ¡":530,"uzi":73,"uzg":36,"uze":128,"uzt":192,"uzs":479,"uzr":64,"utÄ«":62,"uzm":28,"utÄ“":47,"uzb":144,"uzc":22,"uzd":109,"utÄ":171,"usÄ«":47,"pÄ“t":433,"usÄ“":169,"usÄ":96,"pÄ“d":188,"pÄ“c":941,"pÄ“j":691,"urÄ£":36,"pÄ“m":32,"pÄ“l":736,"pÄ“k":384,"pÄ“r":307,"urÄ«":204,"urÄ“":196,"uz ":1335,"urÄ":908,"umÅ¡":52,"uum":27,"upÄ":104,"upÄ“":50,"ux ":33,"uus":124,"uvi":141,"uvo":31,"uva":357,"uve":337,"upÄ«":23,"uvu":135,"usl":182,"usm":71,"usj":49,"usk":104,"mÅ«k":26,"ukÅ¡":120,"usi":609,"unÄ":343,"usd":32,"use":127,"usa":206,"mÅ«z":468,"usz":100,"usv":36,"usu":107,"ust":2158,"mÅ«s":296,"uss":395,"mÅ«r":69,"umÄ«":24,"uso":49,"mÅ«n":33,"utn":283,"uth":28,"uti":652,"ute":201,"uta":279,"utb":797,"utt":67,"uts":90,"utv":22,"utu":102,"uto":580,"utr":30,"unÄ«":25,"us ":3775,"ulÄ":424,"oÄ«d":99,"ulÄ“":350,"ut ":97,"urb":65,"ura":1608,"urd":30,"urc":141,"umÄ":913,"ure":246,"urg":216,"uiž":110,"urj":22,"uri":851,"url":46,"urk":327,"urn":303,"uro":291,"urp":107,"ulÄ«":38,"urs":344,"urt":245,"uru":1149,"urv":97,"umÄ“":24,"urz":200,"unz":34,"ugÅ¡":195,"ujÄ":42,"uor":22,"upa":461,"ur ":573,"upj":23,"upi":127,"ukÄ":121,"upe":293,"upo":29,"upu":109,"ump":111,"ums":2443,"umu":2190,"umt":47,"umv":39,"umi":1048,"umk":31,"pÄņ":98,"umm":30,"uml":29,"umo":1060,"umn":30,"uma":1511,"umb":191,"umd":66,"ume":317,"udž":74,"unt":48,"uns":163,"unr":34,"unv":30,"unu":158,"unl":25,"unk":486,"uni":663,"uno":67,"ugļ":37,"unc":25,"und":348,"una":264,"ung":150,"une":47,"up ":25,"uks":271,"uku":836,"ukt":791,"uko":36,"ukl":91,"uki":42,"ukc":69,"um ":175,"uka":140,"ulv":63,"ulu":136,"ult":666,"uls":122,"ulp":39,"ulo":29,"ulm":42,"ull":58,"ulk":243,"uli":203,"ulg":48,"ule":823,"ulf":35,"ugÄ":140,"ulc":29,"uld":110,"ula":397,"ulb":50,"un ":9898,"uid":83,"oÄ£e":28,"udÄ":23,"oÄ£i":861,"uil":24,"uis":38,"ucÄ“":126,"Ä·a ":175,"mÅ¡a":94,"uji":27,"ujo":73,"udÄ“":94,"ul ":24,"uja":223,"ubÄ":54,"ugi":71,"ugo":30,"ugl":76,"pÄr":1984,"pÄs":27,"uga":887,"ugu":553,"ugs":799,"uha":22,"uj ":134,"uco":23,"pÄ ":453,"uda":124,"ude":104,"udi":282,"Ä·ir":550,"Ä·is":433,"Ä·in":48,"ubs":106,"Ä·ij":62,"Ä·im":24,"ubu":76,"Ä·ie":106,"Ä·id":184,"uca":88,"ue ":35,"uce":66,"uci":96,"uer":36,"pÄc":26,"pÄn":369,"pÄm":68,"pÄj":65,"udu":86,"ÄÄu":68,"udr":142,"udo":32,"ug ":37,"udz":752,"uel":29,"Ä·et":39,"Ä·es":25,"Ä·er":270,"Ä·en":35,"Ä·el":35,"tuÅ¡":25,"ua ":31,"tzÄ«":84,"uar":51,"ual":31,"uan":35,"ubi":59,"ubj":31,"ubl":522,"ube":111,"uba":77,"uag":22,"uc ":316,"Ä·i ":217,"tvÄ":37,"tzi":22,"tuÄ":134,"ttÄ«":266,"trÅ«":52,"ttÄ“":198,"tza":38,"ttÄ":196,"nÄ«Å¡":44,"tyl":39,"tvÄ“":108,"Ä·u ":470,"ty ":51,"toņ":39,"tve":395,"toļ":81,"tvi":1976,"tva":158,"tur":1909,"tus":646,"tuv":673,"Ä·o ":45,"tul":280,"tuk":47,"tun":72,"Ä·eÅ¡":66,"tum":1336,"tub":43,"tud":188,"tuc":22,"tug":34,"tpÅ«":25,"trÄ«":313,"trÄ“":132,"two":22,"toÅ¡":435,"trÄ":1294,"nÄ«c":430,"nÄ«d":58,"nÄ«b":728,"ts ":6404,"nÄ«j":53,"nÄ«g":331,"nÄ«t":121,"nÄ«s":25,"nÄ«r":102,"nÄ«n":27,"nÄ«m":63,"tlÄ“":35,"tre":229,"tt ":106,"oÄa":27,"tra":1972,"tri":1251,"oÄi":28,"trs":398,"tru":1883,"tro":2406,"tlÄ«":43,"tu ":4075,"tmÄ“":32,"tsa":92,"tse":174,"lÅ«c":74,"lÅ«d":44,"lÅ«g":25,"lÅ«k":137,"tsk":152,"tsl":24,"lÅ«p":24,"tsp":162,"tsv":27,"tsu":24,"lÅ«t":49,"tst":123,"lÅ«s":138,"lÅ«z":46,"tnÄ“":293,"tta":34,"tte":67,"tti":291,"ttl":22,"tto":22,"tnÄ«":27,"ttp":39,"Ä·us":35,"tma":80,"to ":1892,"tms":32,"tmo":123,"tml":23,"tmi":87,"tni":753,"tne":504,"tp ":39,"tna":35,"tns":112,"tnu":123,"tno":42,"tof":38,"tod":192,"toc":137,"toj":671,"tog":69,"nÄ« ":207,"tob":116,"tov":52,"tos":547,"tot":998,"toz":23,"tom":540,"ton":588,"tok":294,"tol":346,"tor":2083,"top":326,"tpe":28,"tkÄ":47,"tpi":87,"tiÄ·":219,"tpa":63,"tpl":32,"tiņ":164,"tpr":24,"tij":940,"til":601,"tik":1958,"tif":82,"tie":4403,"tig":30,"tir":195,"tit":273,"tis":2277,"tin":593,"tim":104,"tip":378,"tio":312,"tia":75,"lÅ¡a":104,"tic":277,"tid":70,"teÄ":78,"tju":24,"tiz":110,"lÅ¡u":27,"tiv":158,"tja":44,"tki":59,"tkl":212,"tko":27,"tkr":96,"tku":37,"tka":280,"tli":341,"teņ":82,"tla":152,"tgÄ":22,"tle":76,"tem":480,"ten":587,"teo":285,"tep":65,"tei":672,"tej":47,"tek":587,"tel":542,"teg":123,"teh":268,"teb":48,"tec":92,"ted":53,"tfo":35,"th ":68,"tet":45,"tes":1146,"ter":2575,"tgr":47,"ti ":2942,"tga":132,"taÄ":123,"tho":40,"the":191,"thi":29,"taļ":38,"tha":41," α ":31,"Äpa":64,"Äpe":142,"ÄkÄ":884,"Äpi":34,"ÄjÄ“":32,"Är ":34,"ÄjÄ":201,"Äno":72,"Äns":365,"Änu":327,"Äre":82,"ÄmÄ":78,"Ärg":36,"Ära":470,"Ärb":33,"Äkļ":103,"Ärd":904,"Ärm":72,"Ärn":280,"Äro":74,"Ärp":114,"Äri":461,"Ärk":89,"Ärl":118,"Ät ":234,"ÄlÄ":1101,"Äpu":44,"Äps":37,"Äs ":6616,"Äld":44,"ÄgÄ":35,"Äle":181,"Äli":1326,"Äla":1215,"Äks":485,"Äkt":61,"Äku":495,"Äko":642,"Äki":268,"Äka":1716,"Äm ":3525,"Äjs":680,"ÄbÅ«":29,"ÄdÄ«":293,"Äju":1139,"Äji":502,"Äjo":89,"Äjp":22,"Äni":891,"Äna":615,"Änd":34,"Ämu":42,"Äms":160,"Ämi":68,"Äma":396,"Älu":633,"Äls":579,"Älo":317,"ÄrÄ“":202,"ÄsÄ":40,"ÄpÅ¡":36,"ÄrÄ«":93,"Ävu":149,"žÄs":33,"ÄrÄ":289,"žÄd":455,"žÄk":194,"žÄm":34,"ÄtÄ“":54,"Äze":265,"ÄrÅ¡":275,"Äzi":216,"Äzu":50,"zÅ¡Ä·":85,"ÄtÄ«":67,"ÄtÄ":439,"Ärņ":54,"Äv ":562,"Å¾Ä ":38,"Äta":627,"Äst":95,"Äsu":62,"Ätn":541,"Ätr":318,"Äto":42,"Äte":469,"Äti":825,"Ärz":92,"Äsa":252,"ÄlÄ«":22,"Ärr":75,"Ärs":844,"Ärt":1313,"Äru":144,"Ärv":488,"ÄnÄ":124,"Äsi":27,"ÄkÅ¡":30,"Ävd":67,"Äva":113,"ÄpÄ“":74,"Ävs":30,"Ävo":161,"Ävj":71,"Ävi":182,"Äve":205,"Äts":679,"Ätu":408,"Ä€zi":142,"Äfi":261,"Äfr":63,"Äfs":37,"Äga":57,"Ägs":41,"Ägo":45,"Äj ":38,"ÄbÄ“":84,"ÄcÄ“":37,"Äk ":1029,"ÄdÄ":677,"ÄcÄ«":199,"ÄdÄ“":157,"Äja":844,"Äje":33,"Äbi":54,"Äbj":26,"Äbe":514,"Äba":24,"Äca":27,"Äbu":87,"Äci":2513,"Äcb":31,"Äda":636,"Äcu":337,"Ädn":33,"Ädo":104,"Äde":246,"Ädi":416,"Äds":129,"Ädu":390,"ļin":31,"ļie":252,"ļi ":260,"zņe":149,"ļda":25,"ļav":42,"ļau":478,"ļai":47,"ļas":391,"ļam":83,"ļa ":1215,"ÄzÄ“":88,"zņē":241,"žīm":24,"žģī":37,"Ätņ":41,"ÄvÄ“":149,"ÄvÄ":189,"ÄvÄ«":87,"AU ":30,"":23,"zÄ“Å¡":55,"BA ":99,"Ä·Ä« ":22,"ļot":251,"ļos":305,"ļiņ":134,"ļoj":28,"ļri":165,"Ķīn":125,"ļu ":1814,"zÄ«Å¡":29,"ļsk":34,"ļve":81,"ļvi":28,"ļum":33,"ļus":125,"ļuv":123,"Ä·Ä“r":58,"Ä·Ä“n":27,"Ä·Ä“d":39,"ļoÅ¡":38,"Žan":29,"zÄ«s":345,"zÄ«t":108,"zÄ«v":1468,"zÄ«l":49,"zÄ«j":29,"zÄ«m":1070,"zÄ«d":65,"zÄ«c":55,"zÄ«b":324,"zÄ«g":242,"AP ":26,"zÄc":457,"zÄ ":57,"ža ":310,"zÄk":210,"zÄl":65,"zÄm":26,"zÄn":65,"zÄs":75,"žku":26,"zÄ“ ":172," С ":34,"žis":66,"Ä·Ä«d":71,"Ä·Ä«r":23,"Ä·Ä«s":84,"žie":84,"Ä·Ä«n":49,"Ä·Ä«m":271,"žos":33,"žot":153,"žor":51,"žkÄ":37,"Ä€ge":33,"žon":63,"žoj":249,"Ä€fr":113,"zÄ“l":36,"zÄ“j":212,"zÄ“m":74,"zÄ“t":500,"zÄ“s":27,"zÄ“r":47,"žni":23,"žo ":82,"žas":293,"žag":66,"žai":25,"žan":122,"žam":29,"ži ":383,"žer":29,"žes":28,"žet":61,"žei":43,"žel":23,"žen":74,"uža":37,"užu":32,"žus":34,"žur":90,"žoÅ¡":80," Ð ":25,"žs ":70,"žre":77,"žu ":492,"uÅ¡Ä":109,"uÅ¡u":56,"uÅ¡o":68,"uÅ¡i":230,"uÅ¡a":268},"n_words":[2106271,2428959,2036825],"name":"lv","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/mr.json b/contrib/languages-data/mr.json
new file mode 100644
index 0000000..2f32d7e
--- /dev/null
+++ b/contrib/languages-data/mr.json
@@ -0,0 +1 @@
+{"freq":{"णां":79,"णाच":69,"थे ":276,"तका":129,"था ":217,"ोलà¥":72,"ोलि":66,"ोला":81,"ोरà¥":136,"ोबर":168,"णता":197,"तंत":167,"ोवà¥":196,"तो ":552,"णजे":209,"à¥à¤¤à¤°":330,"ं ":115,"à¥à¤¤à¥":568,"à¥à¤¤à¥‡":64,"à¥à¤¥à¤³":62,"तां":107,"à¥à¤¤à¤¾":389,"ः ":139,"à¥à¤¤à¥€":202,"ताक":90,"à¥à¤¤à¤¿":145,"à¥à¤¤à¥":82,"ताच":344,"à¥à¤¦à¤°":73,"तात":957,"तान":226,"à¥à¤¥à¤¿":115,"ताम":63,"à¥à¤¥à¤¾":569,"तार":104,"ताल":160,"तिक":153,"à¥à¤¥à¥‡":76,"तीं":64,"à¥à¤¦à¤¾":73,"а":72,"à¥à¤§à¤¤":82,"तिह":109,"तीच":131,"तिस":79,"à¥à¤Ÿà¥":957,"à¥à¤Ÿà¥‹":150,"à¥à¤Ÿà¥‡":199,"à¥à¤Ÿà¥€":161,"à¥à¤Ÿà¤¾":154,"à¥à¤Ÿà¤¿":89,"इ ":181,"तसे":151,"धन ":83,"à¥à¤¤à¤•":79,"दी ":549,"à¥à¤¯à¤‚":164,"à¥à¤®à¤¨":141,"à¥à¤¯à¤•":342,"दू ":113,"à¥à¤®à¤¾":304,"à¥à¤®à¤¿":128,"à¥à¤°à¤‚":90,"à¥à¤¯à¤¤":102,"à¥à¤®à¥‡":88,"à¥à¤¯à¤ª":104,"à¥à¤¯à¤®":126,"à¥à¤°à¤•":379,"à¥à¤¯à¤µ":153,"à¥à¤°à¤œ":340,"à¥à¤¯à¥":195,"à¥à¤¯à¤¾":9729,"à¥à¤§à¤¾":165,"तया":65,"à¥à¤¨à¥‡":69,"तरे":84,"à¥à¤¨à¤¾":140,"तरा":162,"तरर":91,"à¥à¤ªà¤¨":111,"दा ":131,"à¥à¤ªà¤°":85,"तले":94,"तला":76,"à¥à¤ªà¤¾":80,"णà¥à¤¯":949,"ا":67,"तमि":228,"à¥à¤•à¤¾":190,"à¥à¤•à¥ƒ":130,"à¥à¤•à¤°":75,"दर ":72,"णून":182,"तदा":122,"à¥à¤žà¤¾":188,"दल ":81,"ततà¥":79,"णार":809,"णात":73,"à¥à¤—ा":89,"à¥à¤šà¤¿":216,"à¥à¤šà¤¾":100,"णà¥à¤•":137,"à¥à¤šà¤¨":73,"थ ":288,"द ":645,"ध ":495,"न ":5349,"ड ":671,"थील":100,"ठ ":317,"थाप":202,"ण ":1146,"थान":236,"त ":6358,"थित":69,"धी ":120,"ज ":339,"दकà¥":266,"ञ ":122,"धा ":98,"ट ":1322,"ं":13502,"ः":232,"à¤":308,"आ":6891,"इ":1323,"अ":4895,"ऊ":224,"ऋ":69,"ई":607,"उ":1473,"घ ":103,"à¤":2374,"ओ":357,"ऑ":510,"à¤":63,"ग":7424,"ख":2772,"क":20151,"औ":108,"छ":206,"च":12137,"घ":1111,"ट":6169,"ञ":320,"à¤":981,"ज":8381,"ठ":2153,"ड":3743,"ढ":440,"ण":6906,"त":27784,"थ":2477,"द":9254,"ध":4679,"न":18404,"प":12607,"फ":1499,"ब":5130,"à¥à¤— ":233,"भ":4247,"म":15733,"य":22975,"र":35779,"ळ":2991,"ल":18817,"व":16505,"ष":4156,"श":6670,"ह":17826,"स":18689,"ऽ":116,"ि":18218,"ा":66714,"थवा":122,"े":31183,"ॅ":601,"ू":4430,"ृ":896,"ी":21357,"च ":843,"à¥":7163,"ौ":422,"à¥":44808,"ो":8505,"ै":961,"ॉ":916,"०":1926,"१":3312,"à¥à¤• ":91,"६":758,"७":829,"८":1020,"९":2445,"२":1611,"३":695,"४":690,"५":705,"क ":4288,"ग ":912,"ख ":378,"तà¥à¤¸":89,"तà¥à¤µ":365,"तà¥à¤ª":68,"तà¥à¤°":2148,"तà¥à¤¯":1716,"तà¥à¤®":72,"तà¥à¤¤":815,"तà¥à¤¨":66,"à®®":77,"த":78,"ா":90,"ி":93,"à®°":79,"तून":246,"க":89,"तà¥à¤°":76,"ॉरà¥":122,"à¯":211,"तीत":68,"तीन":98,"तीय":438,"तील":2444,"तेल":85,"ई ":267,"ै ":165,"à¥à¤¨ ":140," ख":774," ग":1923," औ":107,"ोजी":152," क":6366," ओ":265," ऑ":506," ट":487," ज":3863," à¤":541," च":2075," छ":154," घ":462," इ":1199," आ":6783," अ":4797," à¤":2269," ऊ":68," उ":1326," ई":99,"दार":306,"दिग":99,"दान":93,"दाच":70,"दिल":153,"दिव":143,"à¥à¤« ":77,"दिर":79,"े ":16291,"दीच":62,"नंत":140,"दà¥à¤°":84,"दà¥à¤¸":164,"ोठे":238,"à¥à¤® ":214,"ू ":729,"à¥à¤¯ ":1111,"à¥à¤° ":868,"ि ":987,"नी ":1364,"ोणा":90,"ी ":12189,"ोतà¥":125,"ोतो":104,"ोते":752,"ोती":123,"ोता":310,"ॠ":263,"ा ":17571,"ोधन":72," ८":95," ९":93," ६":115," ७":115," ४":132," ५":115," २":1066," ३":209," ०":70," १":2735,"à¥à¤µ ":312," प":6615," फ":822," न":2997," म":6902," य":3998,"à¥à¤· ":149," ब":2263," भ":2543," ठ":228," ड":419,"ह ":265," द":3178," ध":461," त":3644," थ":112,"ोपा":104," ह":7628," स":7738," ल":2032," र":3046," श":2241," व":5406,"ने ":992,"à¥à¤¸ ":322,"स ":1697,"ष ":215,"थà¥à¤µ":73,"श ":822,"व ":2502,"à¥à¤š ":156,"दरम":106,"ळ ":724,"à¥à¤Ÿ ":345,"ल ":4586,"à¥à¤ž ":122,"दरà¥":246,"ना ":828,"à¥à¤¡ ":88,"र ":6250,"य ":2749,"à¥à¤  ":232,"म ":1312,"à¥à¤£ ":136,"ोका":76,"ोकस":391,"भ ":104,"ब ":134,"à¥à¤¥ ":178,"फ ":180,"à¥à¤¤ ":411,"à¥à¤§ ":328,"प ":291,"à¥à¤¦ ":119,"टना":76,"डू ":84,"डी ":137,"डा ":115,"ठे ":245,"ठी ":788,"ञान":146,"टोब":135,"टà¥à¤¯":132,"टà¥à¤°":1051,"टà¥à¤Ÿ":87,"टेड":67,"टेक":65,"तः ":64,"डे ":134,"टां":121,"टें":125,"टात":71,"टार":107,"टिक":111,"ठà¥à¤¯":96,"णी ":301,"णि ":782,"णा ":114,"à¥à¤¹à¥":230,"à¥à¤¸à¤¿":71,"à¥à¤¹à¤£":631,"à¥à¤¹à¤¾":235,"à¥à¤¹à¤¿":103,"à¥à¤¹à¥‡":192,"à¥à¤¸à¥":66,"ठिक":87,"डणà¥":118,"à¥à¤²à¤¿":183,"à¥à¤²à¥€":63,"à¥à¤²à¤¾":198,"à¥à¤²à¥‡":93,"à¥à¤°à¤¦":304,"à¥à¤°à¤¥":62,"à¥à¤°à¤¤":188,"à¥à¤°à¤£":89,"à¥à¤¯à¥‚":155,"à¥à¤°à¤ª":582,"à¥à¤¯à¥‡":1183,"à¥à¤°à¤®":557,"à¥à¤¯à¥‹":94,"à¥à¤°à¤²":83,"à¥à¤°à¤µ":174,"à¥à¤°à¤¶":146,"à¥à¤°à¤¹":141,"à¥à¤°à¤¸":352,"à¥à¤°à¤¾":1606,"à¥à¤°à¥":153,"à¥à¤°à¤¿":802,"à¥à¤²à¤‚":83,"à¥à¤°à¥€":882,"à¥à¤°à¥‡":549,"à¥à¤°à¥‹":151,"à¥à¤°à¥":109,"à¥à¤·à¤£":152,"à¥à¤¶à¤¿":77,"à¥à¤·à¥‡":198,"à¥à¤·à¤¿":311,"à¥à¤·à¤¾":344,"à¥à¤·à¥€":88,"à¥à¤µà¤¤":172,"à¥à¤µà¤œ":70,"à¥à¤µà¤°":132,"à¥à¤µà¥€":309,"à¥à¤µà¥‡":282,"à¥à¤µà¤¿":88,"à¥à¤µà¤¾":1006,"डà¥à¤¯":63,"ता ":928,"ती ":831,"ते ":2330,"णे ":394,"डाच":85,"डात":62,"डिय":72,"डिस":128,"डील":75,"तर ":662,"डून":170,"जे ":259,"जा ":105,"जी ":439,"चीन":184,"चित":727,"चार":281,"चाल":111,"चिन":184,"चिम":207,"जगा":147,"चà¥à¤¯":2911,"चà¥à¤š":223,"जनà¥":279,"टक ":89,"जनत":86,"जधा":220,"टन ":79,"जाग":68,"जां":62,"जिल":278,"जास":142,"जार":66,"जान":157,"जात":654,"जाण":139,"जरा":68,"जरà¥":134,"जवळ":124,"जोड":73,"जà¥à¤¯":1261,"जà¥à¤ž":320,"जीव":120,"जà¥à¤¨":111,"जà¥à¤²":121,"जून":124,"टर ":126,"à¤à¤¾à¤²":422,"टी ":174,"टा ":105,"ठा ":104,"ंघ":431,"ंख":180,"ंग":1699,"ंक":394,"ंड":876,"केत":187,"ंट":262,"ंज":187,"ंच":1704,"केच":132,"केट":130,"à¤à¤¡":81,"ंश":217,"ंस":555,"ंव":461,"ॠ":692,"ंध":318,"ंद":1200,"ंन":966,"ंथ":109,"ंत":1644,"ंम":356,"केल":701,"ंप":555,"ंभ":92,"ंब":953,"ो ":900,"à¤à¤—":72,"आण":837,"आढ":90,"इं":419,"कृत":201,"अस":1528,"अश":181,"आग":71,"अव":109,"आक":127,"अल":143,"अर":282,"अभ":461,"अम":428,"अप":77,"अध":195,"अन":311,"अथ":144,"इत":287,"आश":86,"आह":3959,"आल":245,"आय":156,"आर":178,"आप":132,"आफ":117,"कृष":91,"आध":103,"अं":259,"आं":194,"कें":67,"अत":143,"अण":68,"अक":80,"उर":65,"उप":257,"उल":75,"ऊन":63,"ऊर":66,"इस":98,"उं":94,"खले":96,"उच":118,"उत":389,"उन":70,"उद":171,"कोण":88,"कोल":77,"à¤à¤•":1893,"à¤à¤–":65,"à¤à¤ª":100,"कà¥à¤•":92,"कà¥à¤Ÿ":201,"कà¥à¤¤":320,"कà¥à¤¯":169,"कà¥à¤·":1291,"कà¥à¤°":523,"कà¥à¤¸":174,"ऑस":91,"ऑफ":67,"ऑक":141,"ऑग":114,"गर":430,"गल":148,"गळ":139,"गव":115,"खे":276,"गप":74,"गम":74,"खà¥":538,"गट":66,"खा":505,"खि":83,"गड":87,"खी":84,"गण":286,"गत":154,"खर":91,"कà¥":2945,"खल":175,"के":1566,"खन":94,"कॅ":149,"को":604,"कॉ":161,"कि":847,"की":630,"गं":74,"का":3922,"कृ":301,"कà¥":375,"कू":79,"कस":544,"कव":162,"कश":95,"कल":330,"खक":117,"कम":167,"कर":1579,"कप":84,"कथ":172,"कन":175,"कड":224,"खं":163,"कण":68,"कत":150,"कक":66,"कं":129,"ओळ":135,"ची":1602,"चि":1243,"चा":1991,"चे":2243,"चौ":88,"चà¥":3182,"० ":717,"जग":235,"जक":120,"चन":187,"चर":81,"चल":81,"चव":64,"घा":367,"घे":138,"चं":156,"घट":138,"गा":1549,"गस":133,"घड":69,"गी":328,"गि":172,"गू":80,"गà¥":294,"गो":373,"गà¥":1223,"गे":325,"घर":90,"टन":228,"ञा":188,"टा":652,"टल":177,"टर":206,"खाद":93,"४ ":416,"à¤à¤¾":550,"à¤à¤¿":70,"टच":62,"३ ":377,"टक":258,"जो":179,"जे":499,"जू":189,"जी":702,"जà¥":275,"जा":1637,"जि":541,"२ ":421,"जà¥":1646,"जन":601,"खान":75,"जध":220,"खाल":87,"जस":63,"जव":226,"१ ":522,"जर":265,"जल":91,"जय":81,"जम":98,"डच":80,"ठा":219,"ठि":103,"ठव":67,"५ ":430,"टे":445,"टà¥":1360,"टो":293,"टी":396,"टि":297,"डा":413,"डि":310,"डी":307,"डल":139,"डळ":73,"डव":71,"६ ":435,"ठà¥":106,"डम":84,"डर":65,"ठी":879,"डत":76,"डण":186,"डन":65,"ठे":307,"तं":195,"तः":94,"ढा":78,"णज":221,"णक":97,"७ ":492,"ढळ":87,"डà¥":156,"डो":138,"डॉ":73,"डे":278,"डू":298,"डà¥":64,"णि":910,"णी":405,"णà¥":223,"णू":264,"णा":1365,"तक":277,"८ ":406,"णप":91,"णत":272,"९ ":481,"तव":72,"तस":223,"ति":798,"ता":3296,"तू":362,"तà¥":325,"ती":4269,"दं":76,"तद":128,"तत":106,"तप":146,"तन":88,"णे":550,"तम":332,"णà¥":982,"तय":70,"तल":255,"तळ":94,"तर":1232,"थव":138,"था":849,"थी":145,"थि":153,"ते":2652,"तो":665,"तà¥":5456,"खेळ":198,"थळ":77,"दक":326,"दृ":82,"धत":108,"दू":226,"दà¥":367,"नं":194,"दी":767,"दि":707,"दा":1004,"थे":411,"दन":81,"दव":72,"दल":156,"दर":524,"थà¥":121,"धा":812,"नच":194,"नत":190,"नद":208,"धी":622,"नड":78,"धि":310,"धू":73,"धà¥":110,"दो":255,"दà¥":1977,"धन":165,"दे":1492,"नक":199,"नग":152,"धर":260,"धल":79,"नर":109,"नल":207,"नव":362,"धे":84,"नय":86,"नम":96,"धà¥":1350,"नी":1612,"पं":155,"नà¥":263,"गळà¥":76,"ने":2153,"नस":171,"ना":2993,"नि":1438,"पक":409,"नो":289,"नै":75,"नà¥":1414,"पत":355,"पण":229,"पन":409,"पद":339,"पड":106,"पट":647,"पश":225,"पह":161,"पस":114,"पल":219,"पर":957,"बई":129,"पे":273,"पै":233,"पॉ":77,"पू":594,"पृ":102,"पॅ":69,"पा":1577,"पि":254,"बं":239,"पी":379,"पà¥":752,"बच":72,"फळ":72,"पà¥":3134,"पो":211,"बन":164,"फे":175,"बद":156,"बत":64,"फà¥":80,"फि":95,"फा":147,"फो":62,"फà¥":393,"बर":861,"बह":109,"बि":244,"बा":902,"बà¥":115,"मं":347,"बी":172,"बे":403,"बॉ":75,"बो":197,"भर":75,"बà¥":696,"मक":100,"गरा":120,"गरी":78,"मग":69,"मज":82,"मच":80,"यं":250,"भि":399,"भा":2637,"मत":229,"भू":282,"मण":132,"भे":177,"मन":439,"मध":1532,"मद":153,"भौ":79,"भो":73,"मल":215,"यक":493,"भà¥":94,"मर":604,"मह":1126,"यत":203,"मृ":108,"मू":271,"मॅ":66,"खà¥à¤¯":500,"मि":1010,"यट":74,"मा":2886,"मà¥":1239,"रं":435,"मी":463,"मो":679,"यम":237,"मà¥":1246,"यन":341,"मे":875,"यप":140,"मै":72,"यव":231,"रख":99,"रग":70,"रक":670,"यल":98,"यर":147,"या":14036,"रज":391,"रच":375,"यस":75,"रध":68,"रद":356,"रथ":65,"रत":1594,"यू":189,"रण":1061,"यà¥":675,"यी":118,"यि":121,"रय":68,"रम":767,"रभ":78,"रब":97,"यो":389,"रफ":64,"रप":674,"ये":1927,"रन":85,"गाच":88,"लय":143,"लब":72,"लन":141,"लढ":74,"गात":319,"लच":92,"लग":102,"लक":174,"रà¥":5914,"रो":716,"रॉ":92,"रे":1413,"गां":194,"रू":387,"लं":299,"री":2531,"रà¥":575,"रि":1770,"रा":6808,"रह":201,"रस":797,"रश":254,"रव":382,"रल":306,"रर":139,"ळà¥":150,"ळे":264,"ळा":565,"वं":149,"ळी":258,"ळू":62,"ळव":128,"लà¥":2173,"लो":767,"चन ":87,"लू":62,"ळण":107,"ळत":82,"लै":137,"ले":3399,"ळन":66,"लà¥":220,"ली":1530,"लि":1122,"ला":2983,"लव":72,"ळख":135,"ळक":78,"वो":76,"शब":124,"वै":95,"वे":911,"शन":181,"शव":74,"षक":96,"वà¥":1218,"शर":79,"वश":88,"गसà¥":118,"वस":571,"वू":99,"वृ":105,"शत":95,"वा":3822,"वि":1941,"वी":773,"वन":263,"शक":187,"वळ":226,"वल":308,"वर":1642,"वय":91,"वज":113,"वच":65,"वक":74,"वण":225,"वत":410,"वड":312,"वट":66,"सन":187,"षे":500,"सप":188,"सभ":348,"षà¥":1679,"सम":806,"सर":1179,"सल":632,"सव":108,"सच":99,"षा":600,"षि":375,"षी":127,"सण":108,"सत":526,"सद":113,"सध":64,"शे":245,"शà¥":798,"शो":167,"षय":121,"सक":135,"शह":705,"सं":1892,"शी":422,"शà¥":72,"शा":1405,"शि":914,"षण":199,"हे":6136,"हॅ":73,"हà¥":162,"ही":1380,"हि":1219,"हा":3206,"हà¥":871,"हो":1733,"हन":73,"से":980,"सà¥":643,"सी":265,"हत":249,"सू":619,"हण":691,"सि":714,"सा":2701,"सह":169,"हव":64,"हस":71,"हम":90,"सà¥":3801,"हय":143,"हर":898,"सै":65,"सो":298,"ात":6409,"ाथ":69,"ाढ":83,"ाण":917,"ाठ":965,"िं":1222,"ाड":364,"ाट":442,"ाब":339,"ाभ":170,"ाप":1160,"ान":3426,"ाद":925,"ाध":343,"गà¥à¤°":68,"ाव":2259,"िख":96,"िक":2722,"ाळ":594,"ाल":2412,"ार":6697,"ाय":1081,"ाम":1731,"िज":229,"ाह":1051,"िच":184,"ास":2359,"ाष":1328,"ाश":369,"िग":160,"ां":4665,"ाà¤":113,"ाऊ":66,"ाइ":80,"ाई":239,"ाउ":96,"ाक":602,"ाच":4155,"ाà¤":79,"ाज":2078,"ाग":1046,"ाख":206,"ीण":65,"ीठ":87,"à¥à¤‚":330,"गिर":64,"ीत":755,"ीप":214,"ीन":664,"ीम":234,"ीय":1104,"ील":3494,"ीर":265,"à¥à¤—":140,"à¥à¤–":527,"ीव":305,"à¥à¤•":528,"गीत":150,"à¥à¤œ":89,"ीस":155,"à¥à¤Ÿ":125,"िट":204,"गाव":277,"ीं":192,"िड":79,"गाल":127,"िण":383,"ित":1967,"गाय":92,"िद":477,"िध":193,"िन":1260,"िप":347,"िब":126,"घटन":111,"िभ":130,"िम":634,"िर":694,"िय":1422,"िल":1282,"ीक":319,"िळ":425,"िश":552,"िव":770,"िस":721,"िष":365,"िह":250,"ीच":727,"ीज":85,"ेव":793,"ेश":1384,"ेळ":324,"ैक":225,"ेल":2762,"ेय":96,"ेर":672,"ेम":206,"ेब":175,"ेप":128,"ेन":599,"ैद":75,"ॉं":62,"ेह":98,"ेस":492,"ेष":313,"ॉक":64,"गेल":207,"ैव":63,"ैन":90,"ॉन":105,"ॉट":71,"ॅर":127,"ॅन":107,"ें":768,"ेक":722,"ेख":448,"ेट":491,"ेड":164,"ेत":2226,"ेण":224,"ेद":177,"ेथ":380,"ेग":137,"ेच":757,"ेज":75,"ृथ":73,"ृत":438,"ृष":221,"गोल":138,"à¥à¤¤":238,"à¥à¤£":210,"à¥à¤¢":64,"à¥à¤¡":67,"ूं":81,"à¥à¤¨":719,"à¥à¤§":64,"à¥à¤¦":440,"à¥à¤ª":192,"à¥à¤°":1052,"à¥à¤®":183,"à¥à¤³":257,"गà¥à¤²":179,"ूक":85,"à¥à¤²":420,"à¥à¤·":125,"à¥à¤¸":491,"à¥à¤µ":264,"गà¥à¤°":670,"ूच":81,"गà¥à¤¦":71,"गà¥à¤¨":84,"ूत":126,"ून":1612,"ूप":102,"ूम":143,"ूर":779,"ूल":112,"ूळ":67,"ूह":94,"चा ":1450,"à¥à¤µ":2745,"à¥à¤¶":274,"à¥à¤·":1475,"à¥à¤¸":788,"à¥à¤¹":1660,"à¥à¤®":1315,"à¥à¤¯":13761,"à¥à¤°":9096,"à¥à¤²":826,"ची ":1393,"ोर":443,"ोय":88,"ोल":499,"ोब":307,"ोम":148,"ोस":168,"ोष":97,"ोह":121,"ोळ":97,"ोश":68,"ोव":325,"à¥à¤£":384,"à¥à¤¤":2563,"à¥à¤¡":225,"à¥à¤Ÿ":2277,"à¥à¤ ":301,"à¥à¤":104,"à¥à¤ž":320,"à¥à¤«":142,"à¥à¤¬":125,"à¥à¤ª":569,"à¥à¤§":714,"à¥à¤¨":549,"à¥à¤¥":1118,"à¥à¤¦":543,"ौर":111,"à¥à¤œ":211,"à¥à¤š":600,"à¥à¤—":488,"à¥à¤•":801,"ों":135,"ॉर":159,"ॉल":165,"ोज":285,"ोड":211,"ोट":232,"ोठ":384,"ोद":77,"ोण":200,"ोत":1519,"ोप":280,"ोध":220,"ोन":394,"ोक":697,"ोच":112,"ोग":276,"चे ":2125,"घात":147,"घाच":71,"चंद":96,"घेत":69,"०६":62,"०७":100,"०४":65,"००":610,"०१":76,"०८":75,"०९":89,"१०":143,"१५":108,"१६":128,"१७":139,"१८":321,"११":87,"१२":105,"१३":94,"१४":109,"१९":1554,"२१":65,"२०":603,"८०":89,"८६":64,"९०":106,"९६":205,"९५":154,"९८":288,"९७":177,"९२":135,"९१":161,"९४":175,"९३":129,"८८":69,"८९":125,"९९":420,"३०":86,"२३":65,"२२":68,"२५":80,"२७":76,"२६":63,"२८":64,"५०":65,"६०":76,"जन ":67,"चना":79,"के ":109,"का ":542,"ओळख":133,"की ":396,"खक ":110,"कर ":232,"कन ":100,"काह":112,"किन":92,"काम":326,"कार":1020,"काय":62,"काल":221,"काळ":222,"काश":148,"कास":81,"किम":109,"किल":76,"कीय":113,"कां":335,"काद":84,"कात":158,"काण":89,"किं":328,"काच":171,"कवी":78,"कसं":90,"कशा":76,"कसभ":296,"कला":81,"कलà¥":115,"गी ":87,"करà¥":219,"गा ":89,"करा":112,"करू":89,"करी":67,"三":73,"करत":181,"करण":522,"कथा":145,"कडे":62,"कडू":62,"गर ":134,"खंड":160,"कंप":101,"à¤à¤ªà¥":100,"à¤à¤•à¤¾":149,"à¤à¤–ा":65,"ऑफ ":63,"ऑसà¥":91,"ऑगस":114,"ऑकà¥":139,"०९ ":83,"०८ ":72,"०७ ":92,"ऊरà¥":63,"११ ":69,"१२ ":81,"१३ ":67,"१० ":113,"१६ ":75,"१७ ":70,"१४ ":80,"१५ ":76,"०० ":116,"०४ ":63,"à¤à¤• ":1511,"आणि":780,"आपल":113,"आफà¥":114,"२००":459,"आले":120,"आरà¥":98,"आहे":3954,"इति":65,"इतर":127,"अथव":122,"अति":63,"१९१":93,"१९८":221,"१९९":347,"१९६":128,"१९७":149,"१९४":143,"१९५":124,"१९२":109,"१९३":101,"अधि":146,"अने":154,"अभि":344,"अभà¥":70,"अमे":278,"अमि":68,"अरà¥":160,"आका":68,"असल":335,"असत":304,"असण":73,"अशा":75,"अशी":64,"असे":343,"असू":247,"असा":90,"इंड":62,"इंग":258,"आढळ":87,"००९":79,"००७":86,"௠":95,"उचà¥":113,"उदà¥":130,"३० ":73,"उतà¥":376,"१९ ":73,"२७ ":71,"ऊन ":62,"२८ ":62,"२५ ":73,"इसà¥":65,"२० ":116,"ेशा":514,"ेशि":132," ओळ":134," कं":103,"ेषà¥":213," खं":102," कथ":124," कम":84," कर":956,"ैकी":208," कल":112," कव":97," कि":683," का":1385," कृ":78,"ेवर":160," कà¥":249," के":933," कॅ":146," को":333," कॉ":122," कà¥":525,"ेवा":214," à¤à¤•":1893," à¤à¤–":65," à¤à¤ª":100," ऑक":140," ऑग":114," ऑफ":67,"ेसà¥":75," ऑस":91," चौ":88," चà¥":195," जग":208," ची":128," चि":913," चा":258," चे":120," जर":148," जम":72," १ ":124," जन":432," जà¥":461," २ ":80," जि":352," जा":1062," जू":107," जà¥":247," जी":146," जे":116," जो":163," ३ ":78," à¤à¤¾":447," खे":215," गण":130," गट":64," खा":206," घर":66," गो":192," गà¥":328," गे":212," गà¥":208," गा":404," चं":92," घे":131," घा":63,"ंट ":79," अं":254,"ेथे":251,"ेथी":97," अप":73,"ेते":395," अन":300," अध":194," अथ":143," आक":124,"ेता":130," अल":139,"ेती":391," अर":236," अम":426," अभ":454," अस":1524," अव":109," अश":180," आग":68,"ेतà¥":247," आढ":90," आण":837," इं":409,"ेणा":90," अक":80," आं":187,"ेणà¥":83," अण":68," अत":140,"ेतल":69,"६० ":62,"ंत ":465,"ंड ":204,"ेने":94,"ेनà¥":114,"ेनि":72,"ंग ":304,"ेबà¥":104,"ेळा":116," इस":89,"ेलà¥":528,"ेलि":98,"ेली":343," इत":279,"ेला":618," आह":3959,"ेले":852," आश":86," आर":162," आय":154," आल":243,"ंच ":100," आध":97," आप":129," आफ":117," उल":66," उप":255," उद":168," उत":384,"ेरि":321,"ेरी":87," उच":118,"ंघ ":80," वा":841,"ोत ":71," वि":1279," शत":75," वस":293," वà¥":476," शर":73," वे":251," वै":84," शब":119," सं":1488," शि":338," शा":307," शह":692," शे":119," शà¥":176," शो":63," शक":86," वर":325," वन":73,"ोन ":196," ला":353," लि":318," ले":317," लो":622," या":2852," रच":72," यà¥":309," यो":83," ये":604," रश":94," रा":1893," रि":79," रस":83," रे":153," रà¥":64," रो":269,"ंद ":67,"ोर ":89," हो":1650," हà¥":509," हि":526," ही":801," हा":1542," हे":1892,"ोल ":64," सम":640," सप":137," सर":815," सद":67," सध":64," सत":74," सà¥":1248," हर":99," हय":132," सो":189," सा":1287," सि":219," सह":142," से":175," सी":112," सà¥":443," सू":90," दर":173," दृ":63," दà¥":235," दा":117,"ोक ":70," दि":467," दक":264," तà¥":1085," तो":143," ते":655," तय":65," तम":233," तर":237," तत":72," ता":327," ति":182,"ोग ":64," ती":172," तà¥":116," तस":163," ७ ":73," ६ ":68," डि":141," टे":67," टà¥":135," टो":93," ५ ":63," ठि":88," ४ ":80," टा":72," मो":576," मà¥":694," मे":240," मै":64," मू":114," मृ":78," मा":1083," मि":294," रं":86," मà¥":622," मह":1114," मल":145," मर":508," भौ":66," मन":77," मध":589," मत":139," भू":208," भा":1980," बà¥":283," बे":267," बो":108," बा":493," बि":111," मं":217," बह":106," फà¥":212," बर":139," बद":64," बन":145," फे":128," फा":98," फà¥":65," फि":73," बच":70," पà¥":2295," पो":147," पि":143," पा":818," पà¥":600," बं":158," पे":127," पॉ":75," पृ":96," पू":277," पर":492," पश":206," पह":159," पड":69," पट":81," पद":228," पत":76," नà¥":105," नो":178," पक":276," नि":731," ना":905," पं":133," ने":304," नव":130," धा":110," नद":157," दे":1033," दà¥":193," दो":206," धर":205,"ॅरि":66,"ेंट":68,"ेंद":165,"ेंब":371,"८९ ":68," द ":78," इ ":175,"ेल ":174,"८० ":71,"ेर ":66,"ेस ":273,"ेश ":484,"ेव ":85," स ":136,"ेचà¥":192,"ेचे":150,"ेचा":92,"ेची":66,"ेकड":79,"ेखन":78,"ेकà¥":200,"ेखक":117,"ेटा":76,"ॉन ":63,"९८ ":66,"९९ ":71,"९६ ":77,"९१ ":71," व ":1440,"ृषà¥":213,"ृथà¥":72,"ृती":81,"ृतà¥":188,"अंत":115,"ेत ":845,"ेन ":118,"ेट ":229,"ेड ":81,"९९६":63,"ेक ":319,"ेख ":111,"ेच ":240,"आंत":108,"à¥à¤·à¥":79,"à¥à¤¸à¤°":130,"à¥à¤µà¤¾":185,"à¥à¤³à¥‡":148,"à¥à¤²à¤¾":65,"à¥à¤²à¥ˆ":113,"à¥à¤°à¤¾":109,"à¥à¤°à¤¸":95,"à¥à¤°à¥‹":111,"à¥à¤°à¥":256,"à¥à¤°à¥":156,"à¥à¤°à¥‚":78,"à¥à¤®à¤¾":140,"à¥à¤ªà¥":76,"à¥à¤¨à¤¿":101,"à¥à¤¨à¥€":63,"à¥à¤¨à¤¾":87,"à¥à¤¦à¥":358,"à¥à¤¤à¥":100,"ंचà¥":531,"ंचा":365,"ंची":306,"ंचे":348,"ंगà¥":429,"ंगा":332,"ंघट":78,"ंगी":166,"ंघा":242,"ंका":93,"ंखà¥":149,"ंगल":89,"ंगण":71,"ंडा":180,"ंडळ":71,"ंना":305,"ंनी":632,"ंपै":165,"ंबई":124,"ूरà¥":429,"ंपा":63,"ंपर":71,"ंपन":86,"ंतà¥":351,"ंता":140,"ंती":82,"ंतर":401,"ंदà¥":289,"ंदी":266,"ंदू":116,"ंदा":71,"ंदि":113,"ंदर":105,"ंबर":448,"ंमध":305,"ंसà¥":364,"ंसा":99,"à¥à¤¸à¥":136,"ंवर":89,"à¥à¤¸à¤¾":139,"ंवा":298,"ंशो":74,"ाळी":71,"ाळा":189,"िकन":68,"ाला":601,"ालि":128,"ाली":458,"ालà¥":126,"ालà¥":170,"ाले":279,"ावि":118,"ावी":70,"ावा":374,"ावà¥":94,"ावे":148,"ावण":64,"िका":572,"ावर":588,"ावल":82,"िको":101,"िकà¥":215,"िके":418,"ाषा":187,"ासत":84,"ाषे":285,"ाषà¥":810,"ाशि":80,"ाशी":94,"िगà¥":108,"ासक":92,"ाही":306,"ाहि":257,"िचा":69,"ाहà¥":62,"à¥à¤¨ ":350,"ासि":69,"ासा":570,"ासà¥":84,"ाहत":75,"ासू":218,"ासà¥":488,"िजे":63,"ींच":65,"िणे":98,"िता":173,"िती":211,"ितà¥":984,"िदà¥":408,"िना":177,"िनि":68,"िनी":230,"िने":334,"िनà¥":135,"िपी":179,"िभा":112,"ियन":201,"िमे":82,"ियम":83,"िमी":103,"िमा":152,"ियो":66,"िया":833,"िरà¥":245,"िरी":73,"िरा":70,"िले":194,"िलà¥":507,"िला":170,"िली":83,"िसर":109,"िषà¥":172,"िषय":111,"िशà¥":136,"िशे":76,"िवस":72,"िवा":161,"िवि":81,"ीका":68,"िवड":238,"ीचे":213,"ीची":91,"िहि":83,"िहा":133,"ीचा":113,"ून ":1516,"ीचà¥":262,"िसे":127,"िसà¥":228,"à¥à¤‚ब":177,"ीती":162,"ूर ":255,"ीने":102,"ीरा":80,"ीवर":113,"à¥à¤•à¥":227,"à¥à¤•à¤¾":154,"à¥à¤–à¥":276,"ृत ":101,"ीला":83,"ित ":454,"िण ":169,"िन ":167,"िध ":64,"िल ":211,"िळ ":165,"ीक ":118,"ांड":163,"ांच":1386,"ांक":146,"ांग":206,"ांस":157,"ांव":124,"िम ":197,"ांम":292,"ांब":110,"ांप":219,"ांन":910,"ांद":82,"ांध":110,"ांत":540,"िय ":84,"िर ":71,"िश ":190,"िस ":90,"ागा":289,"ीत ":405,"ागर":198,"ाखा":79,"ाकà¥":63,"ाका":132,"ाकर":80,"िंद":422,"ाडू":108,"िंव":279,"ाठी":839,"िंग":264,"ाटक":140,"ाजा":172,"ाजी":169,"ाजà¥":867,"ाजध":220,"ाजव":101,"ाचा":713,"ाची":807,"ाचे":1121,"ाचà¥":1245,"ाजक":91,"ीन ":336,"ाने":765,"ाना":350,"ानि":81,"ानी":337,"ानà¥":71,"ानल":114,"ानव":85,"à¥à¤– ":209,"ाधà¥":84,"ापन":141,"ानà¥":247,"ादी":203,"ानं":83,"ानत":66,"ाधि":74,"ाधा":87,"ानच":62,"ानक":91,"ादà¥":145,"ाता":198,"ाती":1964,"ातू":202,"ाणे":107,"ाणà¥":123,"ातल":106,"ाते":315,"ातà¥":194,"ातो":153,"ील ":3327,"ीय ":1038,"ातं":85,"ीर ":82,"ाणी":153,"ाणा":214,"ारी":709,"ारि":99,"ारा":1018,"ारस":180,"ारà¥":1216,"ारे":318,"ालय":129,"ामा":334,"ायक":105,"ामà¥":126,"ायन":99,"ामी":72,"ारं":64,"ामà¥":161,"ाया":101,"ारच":84,"ारख":74,"ारक":98,"ारण":233,"ारत":1080,"ाबा":143,"ामध":252,"ामन":110,"ापर":215,"ापू":104,"ापी":79,"ापा":183,"ापà¥":78,"ीस ":70,"ाई ":106,"होत":1397,"होण":97,"हà¥à¤®":91,"हà¥à¤¯":743,"ाक ":79,"ाग ":209,"ाद ":178,"ाण ":173,"ात ":2992,"ान ":859,"ाज ":109,"ाच ":165,"ाट ":100,"ाव ":432,"िक ":1026,"ाळ ":124,"ास ":473,"ाम ":369,"ाल ":196,"ार ":1261,"ाय ":102,"सले":457,"सलà¥":84,"समà¥":164,"समा":227,"समू":87,"समà¥":80,"सरा":98,"सरà¥":792,"सपà¥":140,"सभे":108,"षà¥à¤¯":98,"षà¥à¤£":139,"षà¥à¤ ":291,"सभा":231,"षà¥à¤Ÿ":1050,"सधà¥":62,"षेत":366,"हतà¥":160,"सेच":158,"सेन":81,"सें":161,"सीम":75,"सà¥à¤¨":150,"हणू":177,"सà¥à¤°":133,"सà¥à¤®":66,"हणत":183,"सून":460,"हणज":210,"सां":148,"साह":175,"सिद":219,"साध":112,"सार":316,"साम":328,"साय":147,"सिक":101,"साव":80,"साल":226,"साग":141,"सिं":119,"साठ":416,"सात":110," १२":89," ११":78," १४":90," १३":81," १६":113," १५":99," १८":311," १७":121," १९":1529," २०":568,"हरा":269,"हरà¥":68,"सà¥à¤µ":512,"हया":139,"सà¥à¤¯":64,"सà¥à¤²":87,"सà¥à¤¥":759,"सà¥à¤ª":226,"सà¥à¤Ÿ":559,"सà¥à¤¤":1031,"सà¥à¤•":382,"सेव":68," १०":100,"हेत":649,"हें":134,"हिल":252,"हिन":68,"हित":204,"हास":245,"हाव":84,"हाय":65,"हार":516,"हाम":139,"हान":148,"हिं":378,"हात":102,"हे ":5167,"षां":107,"सणा":80,"सतो":69,"सते":145,"सता":120,"सतà¥":153,"षाच":186,"षिण":264,"शेष":68,"शोध":130,"हा ":1574,"ही ":1273,"शà¥à¤µ":169,"शà¥à¤°":226,"शà¥à¤¯":73,"शà¥à¤š":243,"शहर":683,"से ":365,"सी ":115,"हर ":423,"संत":74,"शिय":317,"संक":81,"संग":244,"शिव":108,"संख":143,"संघ":417,"संब":106,"संप":91,"संस":355,"संश":79,"शात":171,"शाच":270,"शास":375,"शिक":213,"शां":163,"वेळ":85,"वेल":78,"वेश":67,"वेग":99,"वेद":75,"शतक":63,"वृत":90,"सा ":185,"वà¥à¤¹":547,"वà¥à¤¯":626,"शबà¥":122,"वरà¥":390,"षा ":197,"वरा":67,"वरू":76,"वरी":175,"षी ":90,"वले":122,"वसा":93,"वसल":181,"वसà¥":189,"सन ":83,"वां":120,"वात":562,"वाद":207,"वान":200,"वाच":234,"वाज":104,"विक":171,"वाल":80,"वास":154,"वाप":201,"वार":452,"वाय":134,"वाम":62,"वित":108,"विन":71,"विद":197,"विध":115,"विज":139,"वाह":173,"विच":82,"विष":207,"विश":185,"वीच":70,"विल":73,"विव":106,"विम":91,"विभ":103,"वीप":110,"वडू":94,"वडण":125,"वणा":82,"वणà¥":85,"वती":67,"वता":101,"वना":93,"शा ":95,"षण ":107,"शी ":306,"वंश":67,"ळात":171,"ळाड":66,"षक ":73,"ळà¥à¤¯":142,"शन ":88,"वे ":211,"वा ":653,"वी ":367,"लà¥à¤ª":162,"लà¥à¤¯":1022,"लà¥à¤²":251,"लà¥à¤µ":86,"लà¥à¤¸":80,"लà¥à¤¹":320,"ळना":64,"लेश":64,"लोक":579,"वळ ":98,"लेल":1062,"लेख":355,"लिह":91,"लिश":121,"लिय":136,"लील":65,"लà¥à¤•":126,"वर ":789,"लां":197,"लाग":85,"लाच":70,"लिं":80,"लाप":66,"लाव":113,"लिक":135,"लास":65,"लिन":79,"लिप":197,"वत ":73,"ळखल":119,"ळे ":200,"ळा ":98,"ळी ":148,"रेस":151,"रोज":164,"रोप":113,"रà¥à¤¶":238,"रà¥à¤µ":1106,"रà¥à¤¸":68,"रà¥à¤·":184,"रà¥à¤®":630,"रà¥à¤¯":552,"रà¥à¤²":103,"रà¥à¤¥":343,"रà¥à¤¦":88,"रà¥à¤§":121,"रà¥à¤¨":169,"रà¥à¤«":104,"रà¥à¤Ÿ":116,"रà¥à¤¡":113,"रà¥à¤£":231,"रà¥à¤¤":217,"रà¥à¤—":412,"रà¥à¤•":189,"रà¥à¤œ":148,"रà¥à¤š":115,"रीक":109,"रिय":267,"रिल":109,"रिस":109,"रीत":101,"लंड":195,"रीय":372,"रील":210,"रà¥à¤¨":72,"रà¥à¤ª":68,"रà¥à¤µ":148,"रून":145,"रूप":76,"रें":77,"रेक":125,"रेट":62,"रेल":183,"रसा":80,"रसि":236,"रसà¥":202,"ले ":1654,"लै ":119,"रां":490,"रान":176,"राठ":475,"राट":76,"रात":566,"राण":185,"राज":1474,"राच":427,"रिट":93,"रित":141,"राष":801,"रास":112,"राह":156,"राम":258,"राय":119,"रिक":669,"राव":320,"रलि":71,"ला ":1877,"ररा":122,"रमà¥":122,"रमा":263,"रमà¥":208,"रसं":149,"रशि":112,"रशा":97,"रवा":143,"रले":92,"ली ":1229,"रपट":518,"येष":193,"येण":75,"येथ":336,"येत":173,"रदे":228,"योग":175,"यà¥à¤•":84,"यà¥à¤°":149,"यà¥à¤¨":102,"यà¥à¤¦":90,"याच":1413,"याम":178,"यान":560,"याप":224,"यात":1507,"याद":157,"यास":414,"याव":251,"यिक":66,"याल":203,"यार":164,"रता":731,"रति":74,"रती":443,"रते":65,"रतà¥":99,"लय ":91,"रणा":308,"रणे":72,"रणà¥":354,"रचन":71,"यवस":94,"रकà¥":111,"रजा":126,"रजी":94,"यां":1609,"रजà¥":107,"रचà¥":105,"लन ":62,"रे ":511,"महा":885,"महत":141,"यकà¥":261,"रू ":76,"मले":80,"री ":1362,"मृत":96,"मूह":88,"मà¥à¤³":150,"मà¥à¤²":69,"मà¥à¤¦":185,"रॠ":478,"मà¥à¤–":511,"रंप":65,"रंथ":84,"मà¥à¤‚":140,"रंग":175,"मिळ":390,"मित":196,"मिन":63,"मार":537,"माल":161,"मिक":114,"माव":85,"माह":78,"माण":213,"मात":178,"मान":677,"माच":73,"माज":179,"मां":206,"मोठ":381,"में":100,"मेर":300,"मेल":94,"यतà¥":103,"रका":358,"मà¥à¤°":173,"मà¥à¤¯":205,"मà¥à¤¹":682,"मà¥à¤®":64,"रत ":129,"यू ":94,"रण ":190,"या ":6887,"भिन":289,"भाव":89,"भाष":512,"भार":1098,"भाग":467,"रम ":63,"यंत":200,"ये ":856,"मधी":410,"मधà¥":993,"भेव":86,"मनी":88,"भूत":65,"भूम":113,"मतद":118,"मरा":500,"रा ":574,"भà¥à¤¯":77,"मा ":144,"बरà¥":157,"मी ":268,"यन ":212,"मे ":125,"बहà¥":71,"बिय":93,"बाज":80,"बाब":72,"बार":87,"बाद":98,"बां":115,"मंत":114,"मंद":83,"मंड":86,"यम ":64,"रक ":67,"बेर":65,"बेट":143,"बोध":67,"बà¥à¤°":411,"बà¥à¤¦":139,"पà¥à¤°":2788,"पà¥à¤¤":110,"पà¥à¤Ÿ":127,"भा ":239,"मण ":66,"मन ":141,"बचà¥":64,"यक ":87,"बदà¥":68,"बनव":83,"फेब":101,"फà¥à¤°":309,"पहि":119,"पशà¥":206,"पलà¥":82,"बी ":93,"बा ":68,"परà¥":325,"परि":145,"परा":78,"परं":68,"�":133,"पैक":208,"पृथ":73,"पà¥à¤¸":66,"पूर":541,"पà¥à¤£":116,"पà¥à¤¤":63,"पà¥à¤°":350,"पीठ":86,"बंग":106,"बंध":84,"पास":320,"पाच":82,"पाण":105,"पान":76,"पात":141,"पाद":66,"पार":155,"पाल":72,"पाय":64,"पां":102,"पकà¥":286,"नà¥à¤¨":159,"नà¥à¤¯":422,"नà¥à¤®":293,"नà¥à¤¹":93,"नà¥à¤¸":215,"नोव":141,"पदà¥":103,"पदा":135,"पनी":76,"पना":113,"पणे":87,"बर ":537,"पतà¥":186,"पती":143,"पटा":141,"नले":90,"नवी":100,"पी ":206,"नदी":146,"धà¥à¤¯":1211,"नà¥à¤¸":148,"नीच":95,"निस":85,"नेत":475,"नेश":67,"नेव":132,"नेक":163,"बई ":90,"निव":276,"निर":218,"निय":224,"नास":64,"नाह":82,"नाव":453,"निक":308,"नाय":79,"नाम":119,"नार":159,"नात":134,"नाट":176,"नाड":71,"नाच":211,"नाग":125,"नां":144,"दà¥à¤¦":122,"दà¥à¤µ":204,"दà¥à¤°":575,"दà¥à¤¯":488,"दà¥à¤§":540,"धरà¥":201,"पट ":338,"देव":241,"देश":972,"देण":76,"दोन":169,"धात":66,"धार":222,"धिक":210,"धान":301,"धील":412,"पर ":71,"नता":99,"नगर":128,"पण ":79,"नचà¥":79},"n_words":[573395,652939,442285],"name":"mr","type":"devanagari"} \ No newline at end of file
diff --git a/contrib/languages-data/ms.json b/contrib/languages-data/ms.json
new file mode 100644
index 0000000..216d2c3
--- /dev/null
+++ b/contrib/languages-data/ms.json
@@ -0,0 +1 @@
+{"freq":{"D":19572,"E":5893,"F":8324,"G":10072,"A":33896,"B":36297,"C":14172,"L":22330,"M":42865,"N":13239,"O":4712,"H":13017,"I":28961,"J":16075,"K":40946,"U":8179,"T":34101,"W":6762,"V":4890,"Q":1060,"P":46573,"S":49712,"R":17401,"Y":3291,"X":535,"Z":1931,"f":21309,"g":267175,"d":316152,"e":658563,"Feb":550,"b":199729,"Fed":85,"c":52318,"a":1572499,"n":737214,"o":176914,"l":322914,"m":264030,"j":47182,"Fel":252,"k":282587,"h":223514,"i":560632,"w":38651,"v":16909,"Fer":162,"u":382193,"t":354175,"s":303833,"r":438412,"q":1773,"p":194581,"z":11534,"y":121423,"x":2864,"²":570,"É":92,"ß":112,"Fil":1891,"Fin":201,"í":159,"ë":593,"é":1443,"Fir":84,"è":291,"ç":161,"ä":134,"Fie":111,"â":119,"á":238,"ü":561,"ö":608,"ô":979,"ó":201,"Ä":412,"Ä«":194,"Å":211,"Å«":189,"Fas":81,"Fat":132,"Faz":217,"Fan":166,"Fal":106,"Far":151,"Fai":104,"Eri":93,"Est":118,"Ero":494,"É™":85,"Eur":130,"El ":142,"Ì":146,"Ele":136,"Eks":136,"Eko":134,"ν":86,"ο":114,"Eng":424,"α":136,"Emp":510,"Emi":118,"Ent":129,"Ï‚":105,"Ï":83," l":23228," m":77479," n":10921,"Ñ":98," o":15254," h":12749," i":30341," j":11345," k":65931," d":162664," e":5363," f":6534," g":7177,"Ñ€":170,"Ñ":230," a":48871,"Ñ‚":130," b":61081,"у":83," c":5105," y":44531," x":98," z":926," u":11761," t":69161," w":8735," v":1459," q":106," p":65928," s":95750," r":8778," J":16031," K":40896," H":12965,"Gel":192,"Gem":153," I":28925," N":13195," O":4683," L":22273," M":42791," B":36217," C":14046," A":33790," F":8281," G":9989," D":19485," E":5868,"л":171,"к":214," Z":1920," Y":3279,"и":251," X":518,"о":231,"н":181," S":49556,"в":121," R":17319,"Ger":418,"б":106," Q":1053,"а":416," P":46450," W":6726,"Geo":314," V":4864,"Gen":396,"е":192," U":8167," T":33952,"Gha":143," É":91,"Gil":114,"Gir":155,"×™":86,"Gan":153,"Gal":238,"Gam":239,"Gaj":170,"Gay":116,"Gar":385,"Gad":110,"Gab":156,"Ùˆ":473,"ÙŠ":643,"Ù":123,"Ù‚":139,"Ùƒ":121,"Ù„":805,"Ù…":542,"Ù†":527,"Ù‡":177,"د":337,"ج":123,"Ø­":205,"ت":219,"ب":378,"Ø©":256,"ا":1272,"Ø£":90,"ع":180,"Ø´":82,"س":358,"ر":545,"Flo":106,"Fle":177,"Fra":351,"Fri":170,"Fre":163,"A ":1902,"Fon":180,"For":435,"F ":465,"Da":7113,"Cu":422,"Cy":178,"Cz":168,"Cl":327,"Co":2781,"Cr":469,"Ce":818,"Ch":3522,"Ci":1462,"G ":398,"Ec":309,"Ed":294,"Ea":212,"Du":1497,"Do":1267,"Dr":1224,"De":3041,"Di":4070,"Dh":92,"Fe":1229,"H ":344,"Fa":1374,"Eu":254,"Ev":110,"Ex":177,"Er":726,"Et":118,"Es":290,"En":930,"Em":828,"Ei":91,"El":547,"Ek":289,"Eh":94,"Ge":1691,"Ga":1915,"I ":1850,"Fu":334,"Fr":767,"Fo":867,"Fl":397,"Fi":2562,"B ":922,"II ":729,"C ":1235,"Av":219,"Au":1348,"Aw":291,"Ar":3162,"Aq":140,"At":570,"As":2214,"D ":620,"Ba":15003,"Az":1110,"Ay":310,"Ae":197,"Af":831,"Ag":889,"Ah":1002,"Ab":2089,"Ac":1560,"Ad":704,"Am":3400,"An":3447,"Ap":1094,"Ai":809,"Aj":96,"Ak":735,"Al":5346,"By":81,"Hir":81,"Hit":715,"His":108,"Bu":5575,"Br":2582,"Ca":2733,"E ":430,"IM ":85,"Bh":473,"Bi":1727,"Hid":126,"Be":7488,"Hig":156,"Hij":131,"Hik":136,"Hil":325,"Bo":2057,"Bl":490,"Hin":334,"Ku":4706,"Ky":121,"Kl":380,"Kr":939,"Ko":4380,"Le":2149,"Li":1937,"N ":1325,"La":11882,"Lu":2055,"Ly":128,"Lo":3499,"Me":9475,"Mi":2501,"O ":757,"Ma":21779,"Mc":109,"My":256,"Mu":4039,"Mo":3221,"Ni":961,"Ng":163,"Ne":3358,"Na":4580,"P ":1043,"Hel":170,"Ny":154,"Nu":712,"No":2652,"Hea":82,"Ok":544,"Ol":491,"Om":176,"On":254,"Og":727,"Oc":198,"Hen":128,"Of":107,"Her":410,"Ob":148,"Gi":647,"Gh":173,"Gl":218,"Gr":2002,"Go":908,"Gu":1935,"Cô":450,"J ":232,"Ha":4713,"He":1106,"Hi":2278,"Ho":2093,"Hu":2027,"Hy":124,"K ":459,"Ib":1386,"Ia":4453,"Id":210,"Ic":100,"Ip":196,"Im":670,"In":13413,"Ik":260,"Il":316,"IC ":92,"Is":3633,"It":1183,"Ir":732,"Ja":4941,"L ":509,"Ji":423,"Je":6467,"Jo":1642,"Had":169,"Hab":110,"Ju":2146,"Hal":335,"Hak":118,"Haj":534,"Hai":91,"Ka":14796,"M ":4357,"Han":529,"Ham":347,"Kh":882,"Hat":137,"Kg":139,"Has":393,"Har":1041,"Ki":2010,"Haw":106,"Ke":11600,"Hau":497,"Us":180,"Ut":2903,"Ur":241,"Up":83,"Um":385,"Un":2182,"Uk":86,"Ul":440,"Ud":348,"Ub":107,"W ":157,"Tu":2321,"Tr":1038,"Ts":110,"To":1817,"Th":2070,"Ti":5675,"Te":14128,"Ta":5699,"V ":895,"Côt":448,"Sw":424,"Sy":2644,"St":2134,"Su":9154,"Wo":465,"Wi":2174,"Wh":136,"Wa":2486,"Rü":119,"We":1119,"Y ":91,"Vo":1700,"Vi":1229,"X ":350,"Va":902,"Ve":612,"Uz":98,"Qa":209,"Qi":91,"Gun":810,"Pu":4399," Ù…":146,"Gur":224,"Pr":5037,"S ":1438,"Py":150,"Gua":329,"Pe":16586,"Pa":11390," س":132,"Gui":240,"Pl":433,"Po":3845,"Pi":2607,"Ph":402,"Gul":81,"Os":257,"Ot":149,"Ou":112," ا":503,"Op":211,"Or":706,"R ":526," ب":135,"Se":15468,"Sd":126,"Sc":1468,"Si":4011,"Sh":1862,"Sm":89,"Sl":322,"Sk":239,"Sr":539,"Sp":508,"So":1552,"Ru":1209,"U ":260,"Sa":7125,"Re":2654,"Ri":1317,"Rh":2165,"Ro":2065,"Qu":662,"T ":567,"Ra":4724,"Gre":1162,"Gra":462,"b ":4501,"Gro":225,"a ":210782,"Ye":197,"Ya":1244,"Yo":982,"Yu":587,"Glo":89,"Sü":116,"Gon":110,"Gol":142,"Gor":139,"Za":844,"Ze":342,"Zi":166,"Zo":133,"Zu":270,"God":87,"i ":164203,"gd":604,"ge":16197,"ga":76046,"fk":164,"Ing":1603,"Inf":85,"fl":433,"fg":113,"ff":772,"fi":6374,"Ini":272,"fs":278,"fr":1213,"fu":556,"ft":567,"fo":1845,"Int":557,"Ins":318,"j ":686,"gy":603,"hd":641,"he":6433,"hb":328,"ha":43380,"gn":1513,"gm":169,"gl":1474,"gk":8707,"gj":210,"gi":15794,"gh":2760,"gg":14722,"gu":11142,"gt":306,"gs":5117,"gr":2072,"go":5967,"dt":171,"du":14415,"dv":115,"dw":512,"dy":480,"dz":256,"g ":103343,"Ima":490,"ea":5087,"eb":49697,"ec":13590,"aß":86,"ed":9261,"de":23626,"Ili":91,"dd":794,"dg":528,"di":100113,"dh":461,"dk":453,"dm":900,"dl":344,"do":13268,"dn":327,"ds":769,"dr":1709,"ew":3500,"ex":782,"eu":1721,"ev":2199,"ey":1335,"ez":1091,"fa":3015,"h ":128258,"Ind":10245,"fe":1363,"eh":14770,"eg":12184,"ef":868,"ee":2807,"el":48944,"ek":16873,"ej":6455,"ei":5276,"ep":15359,"Imp":84,"eo":5366,"en":99039,"em":39155,"et":34806,"es":39797,"er":193978,"eq":83,"ca":17842,"Ika":175,"e ":34293,"by":357,"bs":314,"br":1904,"bu":54229,"bt":124,"bn":513,"bo":5898,"bj":669,"bk":504,"bl":1565,"bi":16194,"bb":374,"bd":1187,"be":37633,"dc":103,"db":909,"da":137597,"f ":4203,"cy":378,"cu":2092,"ct":1148,"cs":129,"cr":524,"co":3401,"ck":2599,"cl":364,"ci":8015,"ch":6428,"ce":6430,"cc":744,"c ":1819,"az":2511,"ay":31162,"ba":73332,"d ":17846,"at":94936,"as":52637,"ar":111647,"aq":379,"ax":268,"aw":17249,"av":1840,"au":24752,"ak":88368,"al":109743,"ai":38299,"aj":9650,"ao":666,"ap":27642,"am":64984,"an":422498,"ac":4688,"ad":56503,"aa":11612,"ab":20825,"ag":21007,"ah":136447,"ae":8470,"af":2405,"nu":8816,"nt":38718,"ns":10123,"nr":1326,"np":780,"no":5581,"nn":4359,"q ":402,"nz":372,"ny":25233,"nw":221,"nv":296,"oe":721,"of":2166,"oc":1646,"od":3040,"oa":966,"ob":2206,"om":14310,"ké":379,"on":34226,"ok":8406,"ol":25149,"oi":2136,"oj":754,"og":4180,"oh":4267,"m²":560,"ot":8011,"os":7149,"ov":6595,"ou":3897,"op":4862,"oo":1785,"or":25529,"r ":54659,"ox":186,"ow":1588,"oz":259,"oy":682,"pe":46877,"pa":95395,"pl":1464,"pm":86,"pn":114,"po":7399,"ph":1253,"pi":8396,"pk":282,"lo":8490,"ln":692,"lm":890,"ll":5253,"ls":1529,"lp":1203,"lw":82,"lv":509,"lu":18399,"lt":2503,"lz":145,"ly":913,"hô":463,"o ":11973,"Idr":97,"ma":65762,"mb":18534,"me":75198,"mf":114,"mk":292,"ml":733,"iè":101,"mi":11715,"mn":744,"mm":2305,"mp":19625,"mo":4147,"mr":122,"mt":113,"ms":473,"mu":20069,"my":327,"p ":5732,"na":48153,"nb":1690,"nc":10083,"nd":46633,"ne":28218,"nf":758,"ng":182026,"nh":406,"ni":30289,"nj":7927,"nk":2659,"nl":387,"nm":699,"ju":11519,"jr":154,"jo":1281,"jl":522,"Ibr":373,"ki":14104,"kh":3991,"Ibu":746,"ke":44562,"kb":86,"ka":108120,"Ibn":142,"m ":42736,"fö":176,"kw":208,"ky":1117,"ks":3853,"kt":4143,"ku":13823,"ko":15955,"kp":103,"kr":1887,"kk":1128,"kl":2495,"km":1674,"kn":2015,"li":41181,"lh":268,"lk":1910,"lj":112,"le":52683,"ld":1859,"lg":478,"lf":422,"la":150999,"lc":220,"lb":2609,"n ":280458,"hr":1238,"hs":425,"hw":450,"ht":1165,"hu":13402,"hk":2033,"hi":14383,"hn":1231,"ho":3729,"hl":3211,"hm":1374,"id":12680,"ic":6147,"ib":7125,"ia":71186,"ih":7528,"ig":7857,"if":3795,"ie":5178,"hy":546,"k ":62478,"iq":398,"ir":20180,"is":34419,"it":31463,"iu":2005,"iv":2344,"iw":1960,"ix":314,"ii":592,"ij":1917,"ik":34589,"il":33625,"im":21680,"in":69236,"io":6266,"ip":10779,"je":4970,"ji":4809,"iz":1664,"iy":922,"l ":29072,"ja":22803,"xi":565,"xo":133,"té":82,"xt":102,"ww":150,"z ":1327,"xa":185,"Ia ":4390,"xe":102,"wi":8007,"wn":252,"wo":532,"ws":319,"wu":517,"vy":83,"rö":281,"y ":7880,"wa":23770,"we":3150,"ré":209,"vi":7596,"vu":82,"vr":113,"vo":1142,"uz":1379,"uy":326,"ux":309,"uw":287,"uv":269,"uu":95,"ve":5122,"va":1988,"x ":1392,"ui":3189,"uj":3571,"uk":26920,"ul":24992,"ue":2226,"uf":806,"ug":5578,"uh":8031,"uq":123,"ur":29560,"us":17527,"ut":20841,"um":18481,"un":56650,"uo":172,"up":38959,"ty":1373,"tz":380,"tu":37333,"tt":1895,"tw":249,"tv":82,"ub":6360,"ua":55927,"ud":9184,"uc":1791,"w ":1480,"to":8847,"tn":1031,"tm":404,"tl":983,"ts":1145,"tr":7210,"tp":463,"tf":106,"te":79676,"td":166,"tk":3583,"ti":36996,"th":4853,"v ":487,"tb":138,"tc":212,"ta":118349,"su":13746,"ss":3665,"st":16805,"sy":3492,"sw":1296,"sl":3506,"sk":3092,"sn":2184,"sm":1932,"sp":2237,"so":3953,"sr":496,"sd":136,"sc":1307,"sf":360,"se":78194,"sh":2942,"sg":539,"sj":1138,"si":55053,"rz":381,"u ":48453,"sa":62323,"sb":776,"rr":2104,"rs":8875,"rt":16822,"ru":47839,"rv":432,"rw":832,"ry":2374,"rp":3350,"ro":15649,"rn":7372,"rm":10746,"né":180,"rl":23672,"rk":11585,"rj":2659,"ri":56760,"rh":2483,"rg":8124,"rf":712,"re":18314,"rd":8278,"rc":2216,"rb":22662,"ra":108362,"t ":48210,"qu":886,"qi":117,"qa":200,"s ":44138,"px":487,"py":85,"pt":2000,"pu":18990,"pp":606,"pr":4620,"ps":516,"Hut":228,"Hus":197,"Hun":189,"Hub":115,"Hul":888,"IV ":93,"zz":209,"Hor":109,"zh":286,"zi":2979,"Hou":88,"zb":112,"Hos":142,"ze":1129,"za":3097,"yy":396,"Hon":251,"yz":105,"Hol":986,"zu":864,"zo":647,"zn":118,"zm":124,"zl":139,"yg":94,"yh":122,"ye":3480,"yc":226,"yd":229,"ya":87677,"yb":95,"yu":3240,"yt":295,"ys":11634,"yr":594,"yp":206,"yo":1324,"yn":309,"ym":262,"yl":472,"yi":2320,"Arg":299,"Are":112,"Ard":238,"Ara":1408,"Arm":211,"Ari":140,"Aqu":111,"Apr":696,"Atl":206,"Ast":233,"Ass":149,"Ata":113,"Ash":102,"Asi":970,"Art":249,"Asa":432,"Aut":127,"Aus":928,"Awa":270,"Aze":184,"Azi":185,"Azu":461,"Aye":93,"Aya":126,"CA ":82,"Bag":475,"Bah":1958,"Bai":201,"Bak":335,"Bal":892,"Ban":3541,"Bab":232,"Bac":93,"Bad":566,"Baw":87,"Bay":277,"Bar":4153,"Bat":1275,"Bas":494,"Bau":97,"Abu":317,"Aca":95,"Abb":90,"Aba":147,"Abd":1143,"Abj":137,"Abi":121,"Ada":244,"Ace":1263,"Adv":82,"Adi":102,"Aer":103,"Age":96,"Aga":333,"Afr":636,"Afg":103,"Aha":85,"Agu":275,"Ago":91,"Ahm":452,"Ahl":397,"Air":528,"Al ":922,"Aka":166,"BN ":115,"Akt":218,"Ala":509,"Alb":985,"Alg":101,"Ali":409,"Ale":191,"Alu":88,"Alt":207,"Alm":117,"All":368,"Alo":177,"Alp":987,"Ame":2205,"Ami":153,"Ama":359,"Amu":131,"Amp":255,"Ang":606,"Ani":107,"Ana":284,"And":339,"Anu":311,"Ant":1304,"Ann":89,"Api":115,"Apa":124,"But":135,"Buk":2249,"Bul":563,"Bum":345,"Bun":442,"Bur":1072,"Bud":212,"Bua":152,"Bru":298,"² ":569,"DA ":144,"Cab":88,"Cal":547,"Cam":344,"Cas":261,"Car":564,"Cat":148,"Can":269,"Cap":134,"Bea":159,"Beb":103,"Bes":855,"Bet":145,"Ber":2024,"Ben":734,"Bel":2825,"Bek":161,"Bei":136,"Bid":98,"Bia":120,"Bin":573,"Bil":164,"Bis":119,"Bit":82,"Bir":213,"Bio":85,"Bha":303,"Bhd":125,"Bla":207,"Bre":159,"Bra":1009,"Bro":237,"Bri":839,"Bol":345,"Bon":245,"Boo":88,"Bor":302,"Bos":100,"Bot":109,"Bou":267,"Boy":81,"Cze":166,"De ":309,"Der":103,"Des":621,"Dev":116,"Dew":569,"Del":214,"Dem":195,"Den":384,"Dea":94,"Dam":225,"Dan":256,"Dar":1495,"Das":103,"Dat":1195,"Dau":165,"Dav":161,"Day":273,"Dae":1400,"Dah":132,"Dag":88,"Dai":144,"Dal":1240,"Cho":141,"Chr":187,"Che":558,"Chi":1206,"Chu":173,"Cin":942,"Cit":148,"Cir":82,"Cla":127,"Cem":85,"Cen":218,"Cer":238,"ße":95,"Cha":1124,"Cre":111,"Cro":154,"Coc":102,"Cos":83,"Cor":310,"Com":467,"Col":444,"Con":500,"Cou":501,"アアア":393,"ôn":487,"FA ":123,"ër":184,"ém":402,"én":144,"ér":118,"ë ":295,"ée":141,"èr":124,"Edi":123,"Eck":180,"é ":327,"Eas":113,"Do ":123,"Än":100,"Dia":606,"Dib":90,"Dic":128,"Dit":98,"Dis":1199,"Dir":367,"Dip":183,"Din":297,"Dil":85,"Dig":111,"Di ":495,"Dua":114,"Dun":670,"ể":84,"ür":122,"Dur":174,"Dus":152,"üg":114,"üd":115,"Dra":745,"ôt":453,"ör":194,"Dr ":339,"öe":242,"Dou":90,"Dol":113,"Don":404,"Dom":132,"Neg":1871,"Neu":122,"Net":159,"Nas":509,"Nat":234,"Nav":84,"Naz":81,"Nig":91,"Nic":169,"Nin":95,"Nik":104,"New":731,"Mya":98,"Nar":191,"Nak":108,"Nam":1815,"Nan":583,"Nag":142,"Nai":85,"Naj":112,"Nab":285,"Nun":101,"Nur":239,"Nus":207,"Nov":710,"Nor":1237,"Noo":205,"Oct":88,"Ogo":658,"Okt":487,"PA ":297,"Oli":227,"Oma":141,"Ope":136,"Ora":305,"Ori":90,"Pla":316,"Pin":1148,"Pil":423,"Pis":85,"Pie":157,"Pic":109,"Pia":318,"Phi":161,"Ped":98,"Per":10202,"Pes":304,"Pet":530,"Pem":974,"Pen":3021,"Pej":89,"Pek":281,"Pel":589,"Peg":132,"Pay":578,"Pat":250,"Pas":1115,"Par":2423,"Pav":366,"Pau":192,"Pah":601,"Pag":90,"Pad":1577,"Pan":1375,"Pam":171,"Pap":263,"Pal":1707,"Pak":472,"RA ":106,"Å ":90,"Pyr":101,"Puc":123,"Pun":218,"Pur":154,"Put":1136,"Pus":493,"Pue":122,"Pul":1812,"Pro":3699,"Pri":366,"Pre":620,"Pra":231,"Pok":1401,"Pol":629,"Pon":245,"Pot":126,"Pos":316,"Pop":197,"Por":490,"Pow":89," ال":410,"RM ":2481,"RI ":158,"RC ":114,"SA ":131,"Rab":140,"Rad":225,"Rai":103,"Raj":807,"Rah":353,"Ram":356,"Ran":504,"Rak":486,"Que":146,"Qur":321,"Isa":108,"Ita":1073,"Isk":108,"Ism":235,"Isl":1820,"Isr":240,"Ist":853,"Ipo":169,"Ira":444,"Ire":182,"Jac":170,"Jab":343,"Jar":133,"Jan":881,"Jam":523,"Jal":1059,"Jak":191,"Jel":140,"Jem":105,"Jen":531,"Jep":1274,"Jer":4043,"Jea":98,"Jay":337,"Jaw":691,"Jin":94,"ã‚":296,"Jos":163,"ã‚¢":682,"Joh":1111,"KR ":83,"Jua":218,"Jul":729,"Jun":707,"Jum":163,"Kad":192,"Kab":4567,"Kai":155,"Kam":1464,"Kal":3891,"Kak":93,"Kaj":109,"Kap":368,"Kan":1394,"Kau":151,"Kat":284,"Kas":307,"Kar":941,"Kaz":84,"Kaw":364,"Kay":157,"Ker":1245,"Kes":641,"Ket":677,"Ken":331,"Kep":862,"Kej":260,"Kel":1647,"Kem":873,"Kec":1697,"Keb":1923,"Ked":783,"Kg ":138,"Kea":187,"가가 ":106,"Kis":136,"Kir":134,"Kit":181,"Kin":961,"Kid":196,"Kil":86,"Kim":88,"Kha":610,"Kla":249,"Kon":712,"Kom":706,"Kol":252,"Kor":569,"Kot":1614,"Kob":83,"Kre":356,"Kra":141," ã‚¢":106,"Kri":351,"Kua":1867,"Kub":239,"Kuc":105,"Kui":101,"Kum":474,"Kul":245,"Kun":284,"Kup":104,"Kus":108,"Kur":232,"Kut":677,"Les":206,"Lep":84,"Leo":109,"Len":99,"Lem":434,"Lee":107,"Leb":345,"Lau":1422,"Law":147,"Lay":85,"Le ":130,"Lak":332,"Lal":252,"Lag":398,"Lah":252,"Las":115,"Lat":437,"Lar":140,"Lao":109,"Lap":5633,"Lam":637,"Lan":870,"Lad":118,"Lab":297,"MA ":87,"La ":382,"MP ":115,"Lib":174,"Lia":116,"Lig":184,"Lim":361,"Lin":364,"Lit":120,"Liv":87,"Lui":82,"Lum":953,"Lub":167,"Lua":151,"Lud":173,"Luc":91,"Lou":140,"Los":104,"Lot":137,"MS ":96,"Loi":929,"Lor":631,"Lon":654,"Lom":266,"Lok":116,"NG ":89,"Mek":105,"Mei":562,"Men":2125,"Mem":236,"Mel":2611,"Mes":469,"Mer":1020,"Met":202,"Mec":896,"Meg":95,"Med":514,"Mex":375,"Man":1319,"Mal":11656,"Mam":120,"Mar":2036,"Mas":1315,"Mag":286,"Mad":347,"Maj":610,"Mak":620,"Mah":703,"Mai":531,"Mac":771,"May":201,"Maz":85,"Mau":200,"Mat":615,"Mod":145,"Moh":709,"Mon":755,"Mos":127,"Mor":550,"Mou":293,"Mot":160,"Mik":122,"Mid":149,"Mic":276,"Mit":90,"Mir":149,"Mis":255,"Mil":498,"Min":689,"NO ":270,"Muz":273,"Mun":209,"Mul":257,"Muk":242,"Muh":672,"Mut":94,"Mur":403,"Mus":909,"Mud":234,"Mua":519,"Süd":110,"Wor":178,"Won":85,"Wik":155,"Wil":1358,"Win":285,"Wit":159,"ère":118,"Wei":137,"Wes":721,"Wah":96,"Rüg":112,"Was":81,"War":783,"Wat":152,"Wan":460,"Wak":137,"Wal":495,"ées":91,"Vos":513,"Vor":961,"Vol":89,"Vis":137,"émo":374,"Zea":181,"Zai":220,"Zam":306,"éné":87,"ä¹™":83,"之":243,"並":163,"丘":125,"三":343,"ä¸":209,"Yus":150,"Yun":286,"Yor":143,"Yon":245,"Yog":442,"Yam":136,"Yan":424,"Yah":251,"Syr":106,"Sys":90,"Sya":1981,"Sye":220,"Swi":148,"Swe":223,"Sur":638,"Sus":89,"Sum":1496,"Sul":3001,"Suk":631,"Sup":214,"Sun":2363,"Sud":110,"Sub":100,"Sua":114,"Str":327,"Stu":188,"Sto":152,"Sta":593,"Ste":575,"Teb":161,"Tea":82,"Ten":4201,"Tem":412,"Teo":86,"Tel":885,"Tek":246,"Tap":195,"Tam":976,"Tan":1919,"Tas":214,"Tat":100,"Tar":410,"Taw":155,"Tau":110,"Tai":272,"Taj":110,"Tak":203,"Tal":221,"Tah":219,"Tab":349,"TM ":239,"Shi":244,"She":230,"Sho":202,"Sha":998,"Sim":492,"Sil":414,"Sik":106,"Sit":261,"Sis":302,"Sir":341,"Sip":89,"Sin":1116,"Sid":166,"Sie":82,"Sib":139,"Sia":209,"Ses":148,"Ser":1397,"Set":535,"Sdn":124,"Sep":1336,"Sen":771,"Seo":93,"Sel":5561,"Sem":1319,"Sej":326,"Sek":1892,"Sei":202,"Seg":241,"Sed":118,"See":220,"Seb":744,"Sec":184,"Sea":113,"Sri":518,"TV ":585,"Spa":92,"Spi":83,"Spe":180,"St ":234,"Sou":282,"Sov":188,"Sol":221,"Som":81,"Son":251,"Sos":91,"Sle":97,"TO ":91,"Slo":108,"TP ":81," 三":99,"Rus":662,"Rum":183,"Sai":1025,"Sah":128,"Sak":152,"Sam":586,"Sal":707,"Saa":187,"Sab":832,"Sco":163,"Sch":1149,"Say":155,"Sat":343,"Sau":314,"Sar":891,"Sas":105,"San":1065,"Sao":85,"SI ":111,"Res":223,"Rev":110,"Rhi":1420,"Rhe":239,"Rio":130,"Riv":156,"Rin":102,"Ria":192,"Ric":211,"Rid":172,"Rat":181,"Ras":224,"Rap":81,"Raw":128,"Raz":189,"Ray":672,"Red":114,"Rei":85,"Rem":165,"Ren":489,"Rek":84,"Rep":799,"Rob":171,"Roc":116,"Roy":88,"Rou":105,"Rot":215,"Ros":222,"Rom":673,"SS ":126,"SM ":492,"Rhô":457,"Ven":208,"Van":91,"Val":299,"Var":384,"Vic":157,"Vie":298,"Vir":90,"Vil":217,"Ver":240,"Una":261,"Und":396,"Uni":1237,"Unt":117,"Ula":82,"Ulu":237,"Uma":96,"Umu":153,"Uta":2604,"Uth":159,"Uda":328,"Ter":7545,"Tet":82,"Tha":659,"The":1090,"Tho":125,"Tid":104,"Tig":175,"Tim":3943,"Tin":840,"Tit":106,"Tir":130,"UN ":205,"Tor":170,"Tok":365,"Tol":171,"Tom":154,"Ton":166,"Tow":90,"Tou":95,"Tob":83,"Toj":142,"Tro":148,"Tri":323,"Tre":155,"Tra":341,"Tur":534,"Tuj":92,"Tul":101,"Tum":183,"Tun":599,"Tua":360,"Tuh":175,"ße ":87,"ã‚ã‚":178,"bka":467,"bje":424,"bja":244,"bis":253,"bit":3310,"bio":260,"bir":1273,"bil":2147,"bim":123,"bin":2755,"bih":1923,"bij":157,"bo ":233,"blo":96,"ble":283,"bli":991,"bn ":236,"bla":135,"bok":140,"bol":2635,"boj":88,"boh":101,"bni":142,"bon":529,"bom":143,"bor":733,"bot":679,"bou":178,"bbe":102,"bba":131,"be ":164,"ban":31880,"bak":943,"bal":1621,"bai":843,"baj":91,"bag":10103,"bah":10190,"bac":413,"bad":1572,"baa":211,"bab":844,"bd ":101,"bay":335,"baw":1427,"bau":218,"bat":5257,"bas":903,"bar":5076,"bap":179,"beb":1241,"bea":94,"bdu":982,"bi ":755,"beg":153,"ber":24299,"ben":3312,"bel":3864,"bek":999,"bez":543,"bes":2425,"bet":286,"bia":1988,"bib":110,"bic":100,"bid":732,"ca ":950,"car":3573,"cas":256,"cat":393,"cau":90,"can":1363,"cap":475,"cay":362,"cac":97,"cab":306,"cad":172,"cak":355,"cam":7892,"cal":653,"cai":202,"cah":474,"ce ":1976,"bri":257,"bro":107,"bra":750,"bre":165,"bu ":2368,"bru":584,"bsi":118,"bua":29593,"bup":5795,"bur":3331,"bul":1177,"buk":2032,"bun":2518,"bum":1048,"buh":3221,"bud":788,"but":1884,"bus":270,"by ":195,"aka":43030,"am ":24035,"ake":622,"aki":2473,"akh":1543,"ajl":518,"aji":1471,"ajo":353,"aju":766,"aiz":292,"al ":13357,"aja":6197,"aij":188,"aik":1144,"ail":1086,"aim":441,"ain":8906,"aip":152,"air":1407,"ais":498,"ait":2354,"aiw":1231,"ak ":31786,"ahy":162,"ahw":146,"aid":269,"aib":213,"aia":983,"ahm":450,"ahn":440,"ahk":975,"ahl":1517,"ahi":2365,"ahu":8672,"ahs":174,"ahr":292,"aj ":128,"ahe":94,"aha":18129,"agh":136,"agi":6470,"agu":1752,"agn":508,"ago":378,"aq ":213,"anu":3170,"anz":119,"any":7283,"anp":517,"ano":698,"ann":2528,"anm":135,"ant":12519,"ans":983,"ane":1046,"ang":113274,"ani":5398,"anj":3436,"ank":1191,"anl":96,"ap ":2995,"ana":12204,"anb":114,"anc":5840,"and":21721,"amu":780,"amm":801,"aml":124,"amo":344,"amn":175,"amp":5004,"ams":199,"amk":171,"ami":1889,"ame":1030,"amb":3055,"ama":27062,"ao ":367,"alv":310,"alu":2781,"alt":434,"als":328,"alp":100,"alo":978,"aln":492,"alm":207,"all":932,"alk":853,"alg":121,"ali":15559,"alc":93,"ald":621,"ale":1649,"ala":69871,"alb":641,"an ":229755,"akw":163,"aky":793,"aks":1768,"akr":316,"aku":1710,"akt":1506,"ako":830,"akn":464,"akk":560,"akm":163,"akl":477,"aba":7456,"abe":538,"abi":1679,"abj":107,"abk":443,"abl":125,"abo":245,"abr":124,"abu":7287,"ae ":779,"aca":1070,"aah":88,"aan":10415,"aar":216,"aat":274,"ad ":3530,"ac ":681,"aa ":82,"ab ":2395,"aft":225,"afs":96,"aff":132,"afi":651,"ai ":18726,"aga":10764,"age":668,"aen":91,"ael":376,"aer":6596,"aed":290,"ah ":102722,"afa":559,"ado":684,"adr":137,"adm":127,"adh":132,"adi":4970,"add":126,"ade":739,"adz":85,"adu":655,"aco":144,"ack":348,"aci":257,"ach":779,"ace":635,"acc":89,"ada":44005,"adb":874,"af ":525,"act":193,"acu":251,"azi":1065,"aze":259,"azh":86,"aza":605,"az ":156,"ayi":95,"ayo":257,"ays":11200,"ayu":2352,"aya":15557,"aye":177,"ba ":1121,"at ":26256,"arh":224,"arg":2527,"arf":134,"are":909,"ard":1182,"arc":474,"arb":558,"ara":39644,"arp":171,"aro":691,"arn":2199,"arm":521,"arl":1083,"ark":4152,"arj":275,"ari":23309,"aru":4058,"arv":121,"arr":588,"ars":575,"art":3752,"au ":15214,"asa":23237,"ary":969,"asi":8187,"ash":536,"asc":171,"ase":633,"aso":149,"asn":586,"asp":182,"ask":1160,"asj":1090,"asm":720,"asl":278,"aos":96,"ar ":23228,"apa":19852,"ape":224,"api":1939,"aph":119,"apl":112,"apk":212,"apo":341,"app":137,"apt":214,"apu":1299,"as ":9010,"ava":423,"ax ":100,"aut":4080,"avo":84,"avi":800,"ave":328,"awe":1878,"ay ":1141,"awa":14399,"awi":616,"ata":35618,"asu":2536,"ast":2146,"ass":961,"asr":128,"asy":799,"atm":81,"atn":277,"atk":1026,"atl":166,"atr":347,"ato":1267,"ate":9872,"atd":105,"ati":6436,"ath":634,"aw ":128,"aua":793,"auc":85,"aub":89,"att":488,"ats":175,"atu":11913,"aul":410,"aum":504,"aun":617,"aup":666,"aur":353,"aus":360,"aud":602,"aue":165,"auf":124,"auh":298,"auk":81,"Ï‚ ":105,"Zul":114,"Zui":82,"アア":532,"jer":102,"jek":688,"jel":278,"jem":250,"jen":3319,"jet":96,"ji ":1240,"jad":2581,"jaa":1731,"jab":2775,"jat":469,"jas":271,"jau":469,"jar":2821,"jal":1171,"jak":1017,"jan":2215,"jam":827,"jah":1005,"jag":216,"jaj":319,"jai":194,"jay":698,"jaw":545,"joh":284,"jol":87,"jon":105,"jor":354,"Ñк":92,"jis":137,"jir":304,"jin":516,"jik":249,"jil":131,"jid":1157,"jia":666,"jib":155,"jo ":323,"jli":517,"itk":2435,"itr":182,"ito":856,"itu":5709,"itt":386,"its":183,"itz":235,"ity":553,"iub":115,"isk":246,"ism":665,"isl":386,"iso":280,"isn":315,"iss":300,"üdl":82,"isu":481,"ist":4387,"isy":1010,"ita":6918,"ite":4203,"ith":286,"iti":5204,"iwa":1799,"ius":385,"iur":92,"ium":973,"iul":90,"iva":248,"üge":111,"ix ":251,"ivi":886,"ive":1052,"ipo":113,"ipp":136,"ipu":557,"ips":84,"ipt":701,"ipi":916,"ipl":145,"is ":15255,"ion":3684,"iop":132,"ior":170,"ios":146,"iot":81,"iol":260,"ipa":5888,"ipe":1538,"ir ":5942,"iru":594,"iro":343,"irm":96,"irn":148,"irk":679,"irl":158,"iri":4503,"isi":4078,"ish":1001,"ise":2265,"isc":471,"isb":98,"isa":2790,"iqu":224,"ire":1729,"irg":93,"ira":5164,"ird":90,"irc":278,"it ":3986,"ja ":3286,"iya":450,"iz ":428,"가가가":123,"izi":398,"ize":101,"iza":490,"iyy":293,"kim":1100,"kil":906,"kia":289,"kib":224,"kiy":103,"kin":1562,"kip":238,"kir":1682,"kis":850,"kit":2779,"km ":794,"ki ":4083,"khl":192,"khi":1569,"khe":118,"khb":203,"kha":646,"khu":413,"kht":143,"kho":147,"kea":516,"keb":1435,"kec":7220,"ked":1408,"kee":288,"keg":389,"keh":350,"kek":487,"kej":496,"kem":2040,"kel":6287,"ken":4505,"kep":5037,"kes":1492,"ker":4473,"keu":171,"ket":2676,"kew":229,"key":94,"kh ":421,"ke ":4792,"kra":370,"kre":451,"ksa":750,"kse":201,"ku ":3568,"kro":241,"kri":758,"kov":81,"km²":556,"kot":1484,"kos":215,"kor":487,"kop":296,"koo":92,"kon":2602,"kom":4887,"kol":2649,"kok":1924,"koh":327,"kod":435,"ks ":713,"kmu":121,"kna":192,"kny":1047,"kni":375,"kno":375,"kka":941,"kki":112,"klu":484,"ko ":261,"kma":107,"kle":1198,"kla":510,"klo":121,"kli":164,"jut":411,"jul":111,"juk":2111,"jun":1303,"jum":727,"jur":705,"jua":1342,"juh":227,"jug":3370,"jud":682,"ju ":351,"jra":136,"kaw":5261,"kay":507,"kat":7066,"kau":601,"kar":4772,"kas":2281,"kap":1840,"kan":65462,"kal":3979,"kam":1403,"kaj":351,"kak":575,"kah":502,"kai":1379,"kae":207,"kad":826,"kac":140,"kab":1684,"kaa":721,"ka ":8387,"för":174," Ga":1910," Ge":1684," I ":473," Fo":858," Fu":333," Fr":765," Fi":2557," Fl":395," Ha":4690," He":1102," Cô":450," J ":133," Go":899," Gr":1994," Gu":1933," Gh":173," Gi":642," Gl":218," Id":208," Ic":100," Ib":1384," Ia":4449," K ":99," Hy":123," Hu":2023," Ho":2089,"ha ":1152," Hi":2274," Ji":422," Je":6459," L ":153," Ja":4925,"ان ":99," Ir":730," Is":3629," It":1183," Im":670," In":13399," Ip":196," Ik":259," Il":313,"ham":2871,"han":12119," M ":391,"hap":427,"hai":1915," Ka":14783,"haj":358,"hak":893,"hal":1601," Ke":11588,"hau":162,"haw":466," Ki":2005,"har":6324," Kg":139,"has":7477," Kh":876,"hat":1258," Jo":1631," Ju":2140,"haf":92,"hae":147,"hah":683,"hag":2675,"hab":563,"haa":110,"had":1334," N ":147," La":11856," Le":2143," Li":1930," Kl":380,"hba":248,"hd ":532," Ko":4376,"haz":127," Kr":939,"hay":446," Ku":4698," Ky":121," Mc":109," Ma":21745," Mi":2499," Me":9466,"he ":2354," Lo":3490," Ly":128," Lu":2053," Ne":3341," P ":266," Na":4572," Ng":163," Ni":955," Mo":3210," My":254," Mu":4029,"hek":90,"hel":428,"hei":620,"hee":103,"hea":136,"heb":90," A ":513,"het":112,"hes":177,"her":867,"hen":757,"hem":241,"hi ":1051," B ":222," C ":269," Ap":1093," Am":3390," An":3440," Ak":734," Al":5334," Ai":808," Aj":96," Ag":886," Ah":1001," Ae":195," Af":829," Ac":1559," Ad":700," Ab":2084," Ba":14981," D ":146," Az":1109," Ay":309," Aw":291," Av":216," Au":1348," At":566," As":2194," Ar":3152," Aq":140," Be":7481,"hie":97,"hid":1769,"hic":143,"hib":221," Bi":1723," Bh":473,"hia":286,"hip":145,"hin":4765,"him":801," Bl":487," Bo":2041,"hil":602,"hik":165,"hij":260," Br":2580," Bu":5571,"his":190,"hit":603," By":81,"hir":2908," E ":101," Ca":2706," Ce":812," Ci":1452," Ch":3498," Cl":318," Cr":466," Co":2741," Cu":414," Cy":178," Cz":168," F ":213,"hka":1860," Da":7101," Di":4059," Dh":92," De":3032," Dr":1222," Do":1228,"hko":119," Du":1487,"hn ":373," Ea":212," Ec":309," Ed":292," G ":103,"hla":260,"hle":884," El":545," Ek":289," Ei":91,"hli":1784," Eh":94," Et":117," Es":289," Er":725," En":922," Em":827,"hlu":168," Ex":176," Eu":253," Ev":109,"ho ":208," Fe":1224,"hma":998," Fa":1363," H ":142,"gma":83,"go ":689,"glo":121," Sü":116,"gle":434,"gli":352,"gla":485," Wo":453,"gku":1265," Wi":2169," Wh":135," We":1105,"gko":307," Wa":2483," Rü":119,"gkr":129,"gke":128,"gki":641,"gog":193," Zu":270," Zo":131,"gny":427," Ze":341," Zi":163,"gni":98," Za":841,"gne":632,"gna":169," Yu":587," Yo":978," Ya":1240," Ye":195,"gs ":118,"gol":750,"gon":466,"gos":757,"gor":1882,"got":513,"gov":355,"gsa":3814,"gu ":1642,"gsi":463," a ":258,"gro":268,"gra":1121,"gri":124,"gre":287," R ":136,"gtu":144," Ou":112,"gto":99," Os":256," Ot":149," Or":704," Op":210,"gug":107," Po":3822,"gui":166," Pl":428,"guk":111," Pi":2601,"gum":149,"gul":272," Ph":397,"gua":691," Pe":16557,"gub":432,"gue":251," Pa":11353,"gsu":450," Ny":144,"gsl":154," Nu":712," No":2649," Ol":489," Ok":544," On":249," Om":175," Og":727," Oc":198," Of":98," Ob":147," Ra":4674," T ":97,"grö":241," Qu":659," Ro":2055," Re":2651," Ri":1308," Rh":2165," Py":149," S ":208,"gut":128," Pr":5024,"gur":850,"gus":381," Pu":4397,"gun":5744," Qi":91," Qa":209," Sy":2644," Sw":421," Su":9144," St":2094," Ta":5685," V ":104,"gya":453," Th":2050," Ti":5667," Te":14107," Tr":1030," Ts":110," To":1798," Ru":1202," Sa":7117," Sh":1855," Si":3998," Sc":1446," Sd":126," Se":15446," So":1545," Sp":492," Sr":539," Sk":238," Sl":322," Sm":87," Uz":98," Va":900," X ":203," Ve":610," Vi":1221," Vo":1699," Tu":2312," Ub":106," Ud":348," Uk":86," Ul":439," Um":384," Un":2175," Up":83," Ur":240," Us":180," Ut":2902," ja":4995,"iai":1585,"iah":796,"iak":522," l ":89,"iam":821,"ial":13523,"ian":10128," ji":417,"iap":745,"ias":2287,"iar":1823," je":1297,"iau":2147,"iat":661,"iaw":324," im":275," in":10019," ik":452," il":277,"ic ":717,"iaa":121,"iac":153,"iad":926," is":1283," it":1237,"iag":345,"ibl":93,"ibi":920," ka":14437,"ibo":150," m ":97,"ibn":261," kh":806,"ibr":155," ki":2905," ke":34954,"ibu":1830,"id ":2411,"iba":1857,"ibb":83,"ibe":991," ju":4534," ha":7358," he":560," gi":259," gl":145," gr":258," go":642,"ia ":33962," gu":802," ib":1091," ia":15199," id":186," hi":2680,"ib ":619," ho":310," hu":1664,"iet":538,"ieu":98," ni":525,"iel":330," ne":5519,"ien":485," na":3187,"ier":702,"ies":1302,"ied":208,"iej":108," mu":4333,"ig ":949," mo":1128," ok":83," ol":9058," on":137," oc":95,"ifo":215," of":1351,"ifs":152," ob":344,"iff":148,"ife":181,"ifk":145," ny":301,"ifi":877," nu":298,"ih ":4063," no":914,"ifa":630," le":4083,"icr":152,"ics":105,"ict":304,"icu":180," li":1439,"ico":699,"ick":394," la":15380," ku":3370,"ici":517,"ich":943,"ice":662," km":1352,"ie ":1007,"ica":1361," kl":234," kr":296," ko":7504," me":61077,"idz":82,"idu":1218," mi":1388,"Ñ ":83,"idr":191,"ido":222,"idm":710," ma":9348," lu":1062,"idi":1336,"idg":114,"ide":1853,"idd":108,"ida":4190," lo":1081,"if ":1278," ag":1211," ah":1386," ab":756," ac":279," ad":16501," am":705,"iik":146," an":6006," ap":1132," ai":662," aj":154," ak":2442," al":2643," au":286," aw":789," ar":1148," at":10219," as":1761," d ":615," ba":24359," 가가":103," ay":339,"il ":5677,"ija":979," bi":4229," be":24484,"iji":355," bo":2914," bl":104,"ijr":146," bu":4711,"iju":223," br":115," ca":1681," e ":109,"im ":3199,"ika":13080,"ige":492,"iga":1804,"ii ":168,"igh":815,"igi":732,"igu":2207,"igs":159,"igr":98,"igo":173,"ign":291,"iha":2790,"ihi":368,"ihu":86,"ik ":10067,"imo":347," er":382,"imn":174," et":757," en":1147," em":632," ep":143,"imp":1571," el":748,"ime":2177," ek":760,"imi":887," fe":189,"ip ":575,"inc":592,"ind":2963,"ina":9402," fa":1094,"inb":149,"imu":5422,"inn":244," fu":221,"inm":85," fr":306,"ino":771," fo":731,"int":4295,"ins":5288,"inf":140,"ine":3802," fl":155,"inh":91,"ing":16551," fi":3762,"inj":440,"ini":10223,"inl":149,"ink":392," ge":3276,"iq ":87," ga":1706,"inu":307," i ":105,"iny":1962,"iko":560,"ikn":492," co":867,"ikl":232,"ikk":91,"iki":2641,"ikh":399," ce":822," ch":277,"ike":4994," ci":740,"ila":13068," da":59600,"in ":11178," cu":456,"ikt":166,"iku":1306,"ikr":147,"iks":241," do":472,"ilo":508,"ill":1539,"ilk":662," dr":364,"ilm":266,"ilh":97," de":14986,"ili":5625,"ild":212," di":83692,"ile":5293,"ima":6615,"imb":1113," ed":217,"io ":1306,"ily":94," du":2752,"ilu":254," zo":121,"hol":201,"hom":180,"hon":510," za":721,"hos":188,"hot":110,"hou":148,"hoo":173,"hor":1332,"hny":592,"ка":100," ya":44378,"hmu":111,"hme":81,"hmi":93,"huk":240,"hul":645,"hui":194,"huj":251,"hud":233,"hua":247,"hub":485,"htt":96,"hte":100,"hta":151,"hsi":131,"hu ":365,"hru":143,"hro":130,"hre":90,"hri":265,"ht ":578,"hra":480," ru":967," sa":14469," se":65781," si":4360,"hya":165," sh":165," sk":326," sp":1202," so":731," ra":4498," re":1765," ri":882," ro":601," pu":2878," pr":3226," s ":516," px":487,"hwa":232,"hwe":84,"hwi":126," ot":94,"hup":170,"hum":860,"hun":7536,"hus":555,"hut":907," op":389,"hur":599," or":3496," pe":36568," pa":17415," pl":444," po":3206," pi":1442," wa":2574," we":290," wu":266," wi":5410," va":145," ve":377," vo":304," vi":594," ud":384," ub":143," tu":2924," us":324," ut":1972," ur":128," up":118," um":752," un":7312," uk":109," ul":360," ta":11300," sw":110," sy":970," st":1611," su":5235," tr":1008," to":641," th":1478," ti":5407," te":46306,"fi ":402,"ffa":97,"ffi":302,"fes":299,"fer":388,"fen":164,"fel":156,"fia":152,"fgh":100,"fas":143,"fat":355,"far":311,"fam":394,"fan":307,"fak":149,"fal":282,"fai":192,"fah":482,"ff ":196,"fe ":96,"fa ":136,"exa":138,"ez ":176,"ews":100,"ewu":103,"exi":402,"ezu":133,"eza":588,"eta":22591,"ete":2221,"eti":3058,"eth":196,"etn":374,"esp":92,"eso":358,"est":2090,"esu":1224,"ess":1053,"esy":124,"esw":856,"eum":158,"eun":154,"eto":293,"etr":602,"ets":101,"ett":299,"etu":1742,"etw":105,"ew ":783,"eve":355,"eva":165,"evo":467,"evi":1067,"eut":155,"eur":163,"eus":217,"eup":121,"ex ":120,"ewi":120,"ey ":956,"ewa":2199,"epe":2484,"epi":447,"eph":179,"er ":9482,"epa":7540,"eor":3629,"eol":252,"eop":100,"eon":238,"es ":6607,"ept":643,"epu":3262,"epp":83,"epo":201,"erk":5331,"erl":22148,"eri":15969,"erj":2346,"erg":1984,"erh":2054,"ere":4833,"erf":289,"erc":938,"erd":5231,"era":31612,"erb":21481,"et ":3047,"esk":117,"esl":106,"esh":228,"esi":13843,"esc":110,"ese":3188,"eu ":111,"esa":9499,"erz":234,"ery":101,"erv":184,"eru":33986,"erw":586,"err":623,"ert":10703,"ers":7669,"ern":3919,"erm":8575,"erp":1899,"ero":1667,"eki":1159,"ekn":672,"eko":2990,"eks":1079,"ekt":1163,"eku":1531,"en ":13321,"elb":797,"ela":24547,"eld":531,"elf":99,"ele":4153,"eli":4806,"elg":122,"elm":87,"ell":1081,"elo":901,"elu":8313,"els":167,"elt":96,"eo ":687,"emb":9560,"ema":5467,"eme":4035,"emo":659,"emi":3299,"emu":2900,"emp":6552,"ep ":352,"ene":3392,"enh":146,"eng":28628,"enb":1340,"ena":10322,"end":7875,"enc":2582,"eno":673,"enp":206,"enm":121,"enn":571,"enk":219,"eni":4899,"enj":3357,"enu":2960,"ens":1346,"ent":11262,"enr":1199,"enz":142,"eny":4231,"eog":153,"egl":90,"ego":878,"ege":3600,"egi":1125,"ej ":300,"eha":360,"egr":182,"egu":379,"ehk":131,"ehe":137,"ehi":1102,"ek ":1904,"eic":82,"eis":698,"eir":227,"eim":272,"eil":160,"ein":1673,"eik":182,"eij":135,"eid":126,"eif":187,"eja":2806,"el ":2913,"ejo":362,"eje":2245,"eke":565,"eka":5600,"em ":6452,"eju":647,"gka":6162,"gja":188,"giu":105,"git":589,"gis":912,"gir":419,"gil":557,"gim":123,"gik":836,"gin":1308,"gio":226,"gic":109,"gig":81,"gia":2714,"ghu":271,"ght":623,"ghi":183,"örd":174,"gha":1099,"ggu":2160,"ggr":373,"ggo":583,"ggi":2148,"gge":1788,"gga":7632,"gi ":7590,"gen":3765,"geo":235,"get":640,"ger":6793,"ges":802,"gh ":228,"gea":90,"ged":153,"geb":235,"gem":770,"gel":1459,"gek":139,"gdo":544,"ge ":840,"gab":488,"gac":87,"gad":225,"gag":159,"gah":4102,"gai":9563,"gaa":525,"gas":1063,"gar":7736,"gau":156,"gat":1264,"gaw":574,"gay":288,"gak":392,"gaj":331,"gam":2696,"gal":2392,"gan":31725,"gap":1128,"ga ":11059,"fte":85,"fta":174,"fsw":151,"fun":317,"ft ":199,"fra":229,"fre":154,"fri":738,"for":1053,"fot":84,"fon":311,"foo":101,"fol":96,"fli":95,"flo":92,"fic":314,"fie":153,"fil":3257,"fik":1106,"fin":217,"fir":98,"fis":84,"fiz":345,"fka":157,"da ":27370,"dbi":867,"de ":2083,"dad":217,"daa":403,"dak":3335,"dal":29698,"dai":362,"dag":474,"dah":2813,"dae":5534,"daf":161,"dat":1149,"das":913,"dar":25178,"dap":4274,"dan":33544,"dam":453,"day":1071,"daw":132,"dau":298,"ddh":97,"ddi":484,"cup":101,"cun":117,"cul":407,"cum":151,"cuk":241,"cub":107,"cua":246,"ctu":131,"cto":255,"cti":335,"cy ":279,"cus":161,"cur":169,"cut":119,"cks":91,"ckl":1007,"cle":147,"co ":813,"con":891,"col":233,"com":238,"cor":311,"cos":92,"cop":111,"cot":159,"cou":214,"cs ":109,"ct ":200,"cre":115,"cra":101,"öe ":241,"cro":203,"cu ":125,"ccu":84,"cci":124,"cce":280,"cea":276,"ch ":1178,"cer":686,"ces":635,"cet":263,"cen":347,"cep":121,"cem":134,"cel":216,"ceh":1264,"cec":129,"ced":146,"ci ":407,"cha":755,"chw":84,"chu":185,"chy":82,"cia":390,"ck ":760,"cie":186,"che":1276,"chl":916,"chi":1006,"cho":416,"cht":142,"civ":87,"cil":1184,"cir":431,"cis":3697,"cit":186,"cin":441,"cio":99,"cip":618,"cke":401,"ed ":1818,"eba":11892,"ebe":4137,"ebi":2395,"ebo":229,"ebr":729,"ebs":109,"ebu":29935,"eac":114,"eag":133,"eae":222,"ead":560,"eak":120,"ean":542,"eal":407,"eam":160,"ear":468,"eas":200,"eat":650,"eau":189,"eb ":216,"ea ":1116,"efi":144,"efo":203,"efe":130,"ei ":1170,"ega":5716,"eft":107,"eek":338,"een":939,"eel":102,"eem":212,"eec":90,"eed":132,"eh ":12714,"eer":155,"eet":128,"edi":2073,"ede":990,"ône":474,"eda":2175,"eg ":84,"edu":1494,"edo":329,"edr":119,"eck":1080,"ech":443,"eci":1191,"aße":85,"ece":504,"eca":9518,"ee ":526,"ef ":96,"ecu":162,"ect":304,"eco":162,"dwi":284,"dwe":130,"dwa":87,"dy ":371,"dur":258,"dut":214,"dus":523,"dor":515,"don":9812,"dom":753,"dol":193,"dok":339,"dow":266,"dos":301,"ds ":392,"dmi":140,"dun":2327,"dup":672,"dul":1503,"duk":2609,"dua":2723,"dud":2109,"duc":125,"dri":381,"dra":600,"dt ":116,"dre":255,"du ":1066,"dro":350,"dsb":197,"dha":216,"dge":491,"dic":635,"did":1487,"dia":6402,"dib":2542,"dhi":87,"ôte":452,"der":2200,"des":4618,"dew":286,"deb":180,"dea":352,"ded":138,"dec":284,"def":91,"deg":127,"del":494,"dek":1033,"den":10314,"dem":474,"dep":132,"deo":515,"di ":54108,"dle":103,"dn ":127,"dkr":218,"dma":725,"do ":591,"dli":128,"div":279,"diu":557,"diw":190,"dim":977,"din":2811,"dio":830,"dip":2297,"dir":2791,"dis":3174,"dit":5839,"die":663,"dif":84,"dig":2704,"dih":643,"dii":332,"dij":600,"dik":7651,"dil":2398,"dka":218,"rgu":308,"rhe":129,"rha":1676,"rhi":196,"rhu":353,"rho":81,"rfu":133,"rga":3299,"ri ":22851,"rgk":87,"rgi":456,"rge":1062,"rgo":543,"ret":1067,"res":2048,"rev":326,"reu":161,"rew":103,"rez":84,"rey":120,"rfa":107,"rfi":182,"rdu":156,"rds":117,"rdy":187,"rdw":115,"rg ":2166,"reb":133,"rea":1054,"ree":677,"raß":86,"rec":165,"red":517,"rei":890,"rej":223,"reg":650,"reh":132,"rem":580,"ren":2521,"rek":2078,"rel":624,"rer":249,"rep":182,"rf ":161,"rda":3685,"rcu":188,"rdo":180,"rdi":1361,"rde":1385,"re ":3550,"rbu":880,"rco":88,"rci":124,"rch":578,"rce":252,"rca":846,"raw":990,"ray":1762,"raz":756,"rd ":938,"rap":1559,"raq":133,"rar":213,"ras":4078,"rat":7211,"rau":320,"rav":203,"rbi":3232,"rbo":304,"rba":16478,"rbe":1672,"raj":2760,"rai":1892,"rah":13007,"rag":489,"ran":31080,"ram":3117,"ral":1707,"rak":4425,"rab":2634,"raa":1169,"raf":913,"rae":299,"rad":1746,"rac":391,"rpu":377,"rpo":1155,"rs ":614,"rpe":834,"rpa":577,"rpi":203,"ror":104,"ros":1224,"rot":661,"rom":519,"ron":1956,"roo":126,"rop":1379,"rou":319,"rov":4057,"row":385,"rob":171,"roa":181,"rod":471,"roc":332,"roj":312,"roi":110,"rol":484,"rok":385,"rof":362,"roe":206,"roh":152,"rog":606,"rno":175,"rnm":315,"rny":606,"rna":2876,"rnf":173,"rne":905,"rnk":114,"rni":698,"rmo":257,"rmu":1254,"ro ":1007,"rma":7923,"rme":289,"née":98,"rmi":310,"rlu":660,"rlo":176,"rli":1376,"rld":143,"rle":19283,"rla":1739,"rn ":1339,"rku":436,"rko":237,"rki":661,"rke":2263,"rkh":746,"rka":6421,"rm ":579,"rju":506,"rja":1818,"rje":230,"riz":119,"rix":133,"rl ":182,"rip":4424,"rio":461,"rit":2934,"ris":4313,"riv":103,"riu":208,"rih":81,"rig":839,"ril":887,"rik":7517,"rin":4389,"rim":859,"ria":3082,"rib":675,"ric":726,"rid":481,"rie":868,"rif":442,"rk ":625,"rwo":83,"rya":582,"ruj":1475,"rui":92,"ruh":1229,"rug":133,"ruf":342,"rud":184,"ruc":88,"rup":29747,"run":3036,"rum":1098,"rul":804,"ruk":474,"ruy":112,"rus":2650,"rut":1552,"rva":107,"rvi":143,"rve":148,"rwa":662,"ry ":1503,"rsk":111,"rsi":2063,"rso":232,"rsa":1745,"rsb":121,"rsh":121,"rse":3167,"rta":6444,"rst":183,"rsu":233,"rtn":93,"rto":428,"rte":1378,"rth":768,"rti":3790,"rub":489,"rua":1248,"rts":116,"rtu":2053,"rty":170,"rt ":1334,"rro":201,"rri":312,"rre":320,"rra":913,"ru ":2865,"rry":211,"saa":2288,"sab":137,"sac":82,"sad":132,"saf":204,"sah":1841,"sai":2187,"sak":765,"sal":6322,"sam":2727,"sba":93,"sbe":123,"sap":108,"san":12183,"sau":293,"sat":8021,"sas":1598,"sar":5174,"say":288,"saw":965,"sa ":16822,"rzo":153,"sha":523,"sho":148,"she":167,"shi":687,"si ":12617,"sge":511," ê°€":110,"sji":1115,"sie":915,"sid":1093,"sic":184,"sib":161,"sia":23942,"sk ":160,"sit":1085,"siu":88,"sir":1920,"sis":2215,"sip":1088,"sin":2640,"sio":1252,"sil":1595,"sim":1250,"sik":1183,"sih":825,"sif":646,"sig":163,"sbu":405,"se ":1244,"sca":245,"sch":553,"sco":283,"sew":140,"ser":3709,"ses":2265,"set":1888,"seu":109,"sh ":916,"sfe":203,"sea":187,"sei":253,"seh":881,"seg":534,"see":90,"sed":838,"sec":1466,"seb":39199,"sep":2892,"seo":3063,"sen":3081,"sem":4327,"sel":4711,"sek":3390,"sej":3739,"spo":247,"spe":1275,"spi":326,"spa":153,"sot":113,"sou":89,"sol":388,"som":96,"son":1077,"sop":111,"sor":696,"sos":286,"sod":99,"sof":189,"sok":132,"soa":82,"su ":534,"sra":377,"st ":1090,"ss ":420,"sli":668,"slo":101,"slu":163,"sky":86,"sla":2463,"sle":93,"ski":268,"sko":166,"skr":273,"sku":127,"ska":1853,"sny":1578,"sna":139,"sni":143,"sne":275,"smo":84,"so ":382,"sma":631,"smi":706,"sme":415,"swi":860,"sye":1001,"sya":1661,"syu":126,"syt":127,"syi":154,"syh":85,"sse":728,"ssa":1040,"sso":702,"ssi":559,"ste":4462,"sta":2889,"stm":167,"sto":598,"stp":330,"sti":3901,"stu":237,"str":2762,"sua":2264,"sud":961,"sub":558,"suc":350,"suh":294,"sul":629,"sum":580,"suk":2918,"sup":206,"sun":1377,"sut":148,"sus":1037,"sur":1688,"sy ":170,"swa":407,"tai":2173,"taj":254,"tak":21498,"tal":2397,"taf":141,"tag":153,"tah":9752,"taa":959,"tab":947,"tac":174,"tad":1206,"tay":304,"taw":490,"tau":9325,"tat":797,"tas":2227,"tar":10465,"tap":1660,"tan":37185,"tam":5004,"tch":182,"te ":3845,"tda":113,"ê°€":342,"ta ":10995,"pa ":2327,"pe ":281,"par":2140,"pat":12545,"pas":2802,"pau":127,"pay":722,"pac":225,"pad":20739,"pab":507,"pag":366,"pah":868,"pak":30609,"pal":3287,"pai":927,"pap":231,"pam":148,"pan":16670,"phe":134,"pha":505,"pho":147,"phi":165,"pi ":1888,"pea":108,"pec":204,"ped":430,"pen":13849,"pem":3764,"pep":189,"per":20160,"pet":458,"pes":2868,"peg":272,"pej":473,"pel":2806,"pek":821,"pla":686,"pli":230,"ple":359,"plo":124,"pka":268,"phy":98,"pia":417,"pid":110,"pic":128,"pih":341,"pik":502,"pil":749,"pim":153,"pin":1830,"pio":81,"pir":975,"pis":530,"pit":518,"por":689,"pop":931,"pou":100,"pot":165,"pos":307,"poh":486,"pom":970,"pon":661,"pok":948,"pol":1646,"ps ":173,"ppi":85,"ppe":181,"po ":181,"pny":87,"pta":664,"psi":179,"pua":664,"pub":846,"pte":935,"pti":180,"pto":107,"pra":188,"pu ":412,"pri":989,"pre":663,"pro":2699,"pur":2588,"pus":1163,"put":1968,"pun":6347,"puk":143,"pul":4553,"px ":483,"رة ":90,"qua":138,"que":442,"qui":261,"ra ":25300,"ngn":355,"ngo":1514,"ngj":195,"ngi":2924,"ngl":959,"ngk":8614,"ngu":2703,"ngr":216,"ngt":156,"ngs":4906,"ni ":12040,"nge":2966,"ngg":14541,"ngh":1384,"nga":40112,"ngd":572,"nha":170,"neg":5789,"nei":292,"nel":1414,"nek":199,"nen":1045,"nem":406,"neo":158,"ner":1969,"net":1133,"nes":9659,"neu":163,"ng ":99565,"nea":314,"ned":91,"nfo":133,"nfl":104,"ney":393,"nez":159,"nfe":134,"nco":417,"nci":4468,"nce":1649,"nch":449,"nca":2247,"ne ":4670,"nbu":1352,"ndu":4122,"ndr":524,"nds":418,"ndo":10238,"ndi":3985,"nde":2188,"nda":19923,"ncy":259,"ncu":421,"nak":4741,"nal":5652,"nam":4608,"nan":8824,"nap":273,"nar":1813,"nac":196,"nad":1121,"naf":114,"nag":573,"nah":2523,"nai":809,"naj":140,"nab":295,"naa":932,"nbe":130,"nd ":4902,"nba":120,"nau":433,"nat":2102,"nas":1399,"nay":387,"naw":275,"na ":10713,"ê°€ ":113,"nya":20158,"nye":1690,"nyi":1836,"ny ":576,"nwa":109,"nvi":88,"nve":165,"nuk":445,"nul":732,"num":373,"nun":1462,"nug":514,"nuh":642,"nus":1343,"nut":397,"nur":346,"nua":1039,"nub":164,"nty":389,"nto":1247,"ntu":9594,"nts":152,"ntr":493,"nti":4527,"nth":267,"nta":14735,"nte":5062,"nsu":721,"nsy":119,"nsn":707,"nso":443,"nst":1059,"nse":539,"nsh":129,"nsi":4092,"nsk":184,"nsa":240,"nsb":162,"nu ":1047,"nre":1118,"nt ":2047,"npl":189,"ns ":1395,"noh":123,"nol":621,"noi":90,"nop":117,"nom":1343,"non":340,"not":257,"nos":240,"nor":470,"nov":416,"npa":520,"nne":1190,"nna":293,"nno":98,"nni":227,"nny":2270,"nme":425,"nma":233,"nn ":167,"nla":231,"no ":1002,"nke":98,"nki":218,"nka":1483,"nku":327,"nji":842,"nje":264,"nja":4869,"nfö":173,"nju":1774,"njo":126,"nig":119,"nie":150,"nid":172,"nic":456,"nia":3904,"nk ":304,"niy":199,"niu":157,"niv":669,"nis":5362,"nit":1992,"nio":219,"nip":112,"nim":789,"nin":1570,"nik":1402,"nil":494,"ogr":689,"ogu":135,"ogt":141,"ogi":1196,"ogo":280,"ogn":241,"oga":366,"oge":265,"oi ":269,"oho":915,"ohn":385,"ohd":380,"oha":945,"ogy":484,"ois":234,"oir":1071,"oin":141,"oid":154,"ok ":3077,"ojo":198,"oje":321,"oja":137,"ol ":1539,"oce":85,"och":315,"oci":139,"ock":424,"oco":179,"oe ":208,"oca":188,"occ":131,"ode":810,"odi":278,"odo":173,"of ":1293,"oda":299,"oei":98,"oen":92,"odu":321,"og ":188,"ofi":103,"oft":130,"oh ":1421,"off":261,"ofe":269,"oa ":251,"ob ":122,"oan":123,"oal":112,"د ":141,"oad":171,"oba":366,"od ":778,"obo":201,"obl":99,"obj":270,"obi":241,"obe":749,"nyo":518,"nyu":376,"Ø© ":254,"nza":129,"oya":277,"ã‚¢ ":125,"ows":189,"own":225,"owi":119,"oyo":90,"ow ":425,"otl":114,"oti":361,"oth":207,"ote":516,"ott":294,"oto":989,"otn":85,"ost":425,"ota":4428,"ov ":225,"osi":886,"osh":144,"osk":108,"ose":885,"osg":503,"osf":111,"osp":283,"oss":236,"osl":105,"oso":466,"oy ":167,"owa":312,"owe":160,"ovi":3651,"ovo":102,"ova":317,"ove":2144,"oug":180,"oui":99,"oul":186,"oun":730,"oup":135,"ous":420,"our":774,"out":586,"opo":452,"opi":468,"opl":109,"ope":851,"oph":272,"opa":651,"os ":2032,"opu":974,"opt":304,"ops":100,"oon":251,"ool":149,"oké":372,"ook":216,"ood":234,"or ":5219,"oot":168,"oor":392,"ork":389,"orl":178,"orm":1207,"orn":616,"oro":810,"orp":1180,"orr":678,"orc":95,"ord":1125,"ore":1134,"orf":206,"org":773,"ori":1675,"ou ":243,"osa":542,"osc":127,"ort":1677,"ors":277,"oru":170,"orw":94,"ory":594,"m² ":559,"ot ":726,"orb":185,"ora":7028,"ola":3886,"old":231,"on ":7262,"oli":2647,"oll":548,"olk":89,"olf":106,"ole":11812,"ols":765,"olt":108,"olo":2453,"olu":604,"oka":760,"om ":1649,"oki":234,"oke":184,"okr":214,"oks":233,"oko":2518,"oky":119,"okt":111,"oku":358,"ona":2279,"ond":1093,"onc":169,"onf":186,"one":9980,"ong":5686,"onj":101,"oni":1220,"onk":139,"onn":659,"ono":1193,"ons":1416,"ont":1746,"onu":81,"onv":152,"ony":334,"oma":1160,"oo ":109,"ome":1328,"omb":1378,"omi":1343,"omm":1191,"kém":368,"omp":2047,"omo":486,"omu":3500,"op ":434,"la ":8163,"le ":2220,"lf ":112,"lde":213,"lda":235,"ldk":204,"ldi":123,"ldr":97,"laa":247,"lab":751,"lac":351,"lad":420,"laf":100,"lah":41764,"lag":1649,"laj":805,"lai":4507,"lal":1588,"lak":3883,"lan":17980,"lam":20229,"lap":6088,"lar":2085,"lat":9127,"las":2647,"law":2684,"lau":5752,"lav":163,"lay":19664,"laz":215,"lba":1538,"ld ":768,"lbe":124,"lbu":792,"kut":2151,"kus":252,"kur":1183,"kup":264,"kun":678,"kum":2053,"kul":690,"kuk":537,"kwa":145,"ky ":133,"kta":496,"ksy":183,"ksp":152,"ksu":782,"ksi":749,"kso":141,"kub":97,"kud":134,"kue":85,"kug":115,"kui":193,"kua":1683,"ktr":757,"ktu":771,"kti":1117,"kto":892,"kyo":126,"kya":786,"lpe":958,"ls ":210,"lok":357,"lon":1793,"lom":1385,"lop":336,"lor":590,"loc":124,"loh":134,"log":1685,"loi":125,"lpa":103,"los":237,"lot":172,"lou":94,"lov":170,"low":135,"lob":141,"lny":581,"lma":261,"lmu":271,"lti":290,"lto":86,"lud":118,"luc":152,"lub":115,"lua":5030,"luh":485,"lue":130,"lst":756,"lta":1323,"lte":276,"lu ":2269,"lsa":202,"lt ":289,"lha":114,"lge":136,"lgi":113,"li ":9076,"lga":143,"lfo":84,"ley":274,"lew":115,"lex":130,"leu":205,"lev":943,"les":2125,"let":19322,"ler":519,"leo":109,"lep":1429,"lem":5627,"len":2626,"lek":1317,"lel":468,"lei":110,"lej":185,"leh":11245,"leg":545,"lef":234,"lee":98,"led":158,"lec":148,"leb":2160,"lea":321,"lls":96,"llu":129,"lly":170,"lo ":657,"lla":1553,"lle":1246,"lli":666,"llo":405,"lka":1572,"lki":104,"lm ":121,"ll ":825,"lit":3181,"lis":3844,"lir":917,"lip":802,"lio":243,"lin":3503,"lim":5664,"liz":154,"liv":116,"liu":162,"lic":650,"lid":486,"lia":4614,"lib":614,"lik":3976,"lil":385,"lig":376,"lih":1595,"lie":274,"lif":337,"ma ":13530,"mb ":646,"maa":523,"mac":144,"mah":1764,"mai":2470,"maj":1120,"mak":2827,"mad":1643,"mag":284,"map":110,"mar":1635,"mas":5847,"mal":1537,"mam":694,"man":17867,"maz":97,"may":121,"mau":220,"mat":13126,"mba":6501,"mbi":2016,"mbe":4139,"mbr":177,"mbo":1368,"me ":1188,"mbu":3564,"med":795,"mee":451,"meg":240,"mec":162,"met":1241,"mew":781,"mes":766,"mer":37899,"mem":9158,"mel":2698,"men":19175,"mek":226,"mej":156,"lva":321,"lve":96,"lul":249,"luk":1418,"lui":805,"lun":825,"lum":2206,"lut":262,"lus":1161,"lur":2937,"ly ":453,"lya":87,"hôn":460,"lym":81,"mpi":2087,"mpe":1217,"mpr":185,"mpo":1192,"mpl":253,"mpu":8478,"ms ":187,"mod":652,"mon":1281,"mok":188,"mol":218,"mor":397,"mos":294,"mot":314,"mou":108,"mpa":5954,"mu ":410,"mud":1128,"mua":910,"my ":225,"mur":5500,"mus":917,"mut":295,"mui":219,"muk":990,"mul":2530,"mum":892,"mun":5382,"muz":686,"ër ":162,"mi ":2729,"min":2074,"mil":3082,"mim":282,"mir":325,"mis":572,"mit":382,"mic":168,"mia":690,"mig":85,"mid":185,"mik":768,"mo ":213,"ièr":83,"mla":599,"mka":269,"mny":568,"mmu":120,"mmi":85,"mma":792,"mme":1112,"ور":123,"zue":132,"zur":484,"zza":91,"アア ":119,"zi ":161,"zha":131,"zec":167,"zen":271,"zer":331,"ze ":110,"بن":82,"zaa":88,"zah":203,"zam":678,"zan":280,"zak":354,"zal":148,"zar":263,"zon":176,"zog":154,"ان":141,"ال":473,"zia":91,"zin":101,"zim":257,"zil":726,"zik":1108,"ziu":148,"ziz":173,"yré":84,"yum":99,"yun":210,"yur":228,"yus":82,"yua":148,"yti":119,"yst":183,"ysi":10703,"yu ":2215,"yri":256,"ys ":580,"yok":119,"yol":401,"yon":283,"za ":754,"سو":84,"yya":320,"را":86,"رة":90,"ye ":102,"yeb":310,"yed":369,"yer":661,"yen":1160,"yem":89,"yel":428,"ya ":19984,"yaa":759,"yaw":106,"yat":1718,"yar":4015,"yas":133,"yap":186,"yan":47012,"yal":266,"yam":513,"yak":3202,"yah":7404,"yai":2092,"yn ":94,"yle":84,"yll":91,"ylo":98,"yo ":269,"ymp":86,"ر ":108,"yi ":1148,"yhu":83,"yin":165,"yim":134,"yik":137,"yid":111,"yia":421,"ÙŠ ":156,"xon":87,"Ù‡ ":83,"Ù† ":270,"Ù… ":108,"xic":389,"xan":91,"wuj":426,"wn ":203,"ws ":263,"wor":251,"woo":89,"we ":145,"wes":1940,"wer":159,"wel":124,"wei":92,"wed":219,"web":258,"wi ":588,"wir":85,"wit":297,"wig":1003,"win":269,"wil":5229,"wia":82,"röe":241,"wa ":2659,"wan":4391,"wal":2126,"wam":340,"waj":117,"wak":1437,"way":450,"wat":2105,"wau":86,"war":2035,"was":5549,"wai":624,"wah":1233,"wab":269,"via":591,"vir":115,"vil":492,"vin":3415,"vic":160,"vid":769,"vie":296,"vit":347,"vis":1097,"vo ":146,"rén":87,"vol":481,"vok":203,"vi ":88,"ver":1914,"ves":180,"vet":142,"ven":1162,"vem":496,"vel":571,"ve ":410,"val":198,"van":369,"var":225,"vat":140,"vas":121,"vad":278,"va ":432,"uzi":987,"uya":163,"ux ":226,"uve":119,"uwa":153,"usl":307,"usk":402,"ush":174,"usi":3335,"use":449,"usa":3092,"usu":1400,"ust":2174,"uss":842,"uso":135,"usn":368,"utn":101,"uth":367,"uti":1850,"ute":2173,"uta":5785,"utt":161,"uts":114,"utu":1541,"uto":578,"utr":336,"us ":4638,"ut ":7583,"urb":296,"ura":6397,"urd":90,"urc":329,"ure":519,"urh":114,"urg":2337,"uri":1172,"urk":876,"urn":367,"uro":302,"urr":119,"urs":201,"urt":279,"uru":5838,"ury":103,"uny":2843,"upa":36302,"ur ":9875,"upi":131,"upe":318,"upu":963,"ump":3501,"umu":874,"umi":1006,"umk":95,"uml":587,"umo":90,"umn":298,"uma":2885,"umb":3220,"ume":429,"uly":87,"unt":6686,"uns":519,"unu":1520,"unk":553,"unj":444,"uni":4206,"uno":217,"unn":144,"unc":664,"und":1844,"una":5768,"ung":15003,"une":463,"up ":964,"uku":3080,"ukt":432,"uko":385,"ukn":323,"ukk":353,"ukl":355,"uki":2400,"ukh":102,"uke":89,"um ":5354,"uka":5655,"uju":2959,"ulu":2906,"ult":1554,"ulo":336,"ull":522,"ulk":198,"uli":2651,"ulg":85,"ule":183,"ula":12934,"un ":15688,"uid":135,"uih":107,"uil":237,"uin":352,"uis":395,"uhu":184,"uk ":13558,"uji":209,"uit":287,"ul ":3175,"uja":272,"ugh":152,"ugi":310,"uge":603,"ugo":103,"ui ":1412,"uga":3887,"uhi":252,"uhr":221,"uhn":192,"uhk":782,"ugu":316,"uha":3353,"uct":124,"ucu":163,"uf ":428,"uda":2440,"udd":449,"ude":159,"udg":308,"udi":1380,"udk":154,"ubo":151,"ubu":2193,"uca":268,"ue ":523,"ucc":281,"uci":281,"uch":381,"uck":113,"uer":235,"ues":185,"uh ":2913,"ufa":96,"udu":2586,"udo":102,"udw":165,"uen":605,"uel":282,"ub ":287,"ua ":4778,"uay":140,"uat":3203,"uas":2848,"uar":5742,"uam":156,"ual":2564,"uan":6824,"ubi":233,"ubj":123,"ubl":997,"ube":118,"uba":1808,"ud ":1277,"uak":209,"uah":28489,"uai":435,"uad":168,"uac":88,"tze":150,"ty ":1209,"tur":2470,"tus":1028,"tut":762,"tuj":696,"tul":1448,"tuk":8612,"tun":1428,"tum":1714,"tup":264,"tub":1391,"tua":2911,"tud":425,"tue":190,"tuh":346,"tug":473,"tz ":139,"two":138,"ts ":530,"tre":416,"tt ":153,"tra":2815,"tri":1776,"tru":429,"tro":1657,"tu ":13080,"tsu":260,"tta":304,"tte":701,"tti":139,"ttl":217,"tto":116,"ttp":95,"tme":175,"tma":120,"to ":1901,"tni":159,"tne":114,"tp ":97,"tna":265,"tny":365,"tno":94,"toc":160,"toi":105,"toh":336,"tob":555,"tov":117,"tos":314,"tot":99,"tow":94,"tom":534,"ton":1740,"tok":501,"tol":501,"tor":1363,"top":201,"tph":309,"til":1110,"tik":4794,"tif":1686,"tie":154,"tih":1384,"tig":1070,"tir":325,"tiq":113,"tit":1049,"tis":1131,"tin":5400,"tim":2572,"tip":149,"tio":1887,"thu":783,"tia":2252,"tib":138,"tic":397,"tid":1658,"tiw":247,"tiu":167,"tiv":497,"tka":3394,"tli":157,"tla":512,"tle":246,"tem":4640,"ten":10423,"teo":252,"tep":379,"tei":1032,"tek":974,"tel":4483,"teg":559,"teh":223,"tea":171,"teb":211,"tec":87,"ted":825,"th ":1378,"tet":1024,"tes":1074,"ter":49142,"ti ":8689,"tho":286,"thm":261,"thr":109,"the":1215,"thi":258,"tha":367," アア":100,"AS ":389,"BB ":113,"BC ":165,"AC ":83,"AB ":193,"가가":229,"AM ":100,"AL ":81,"AN ":625,"AP ":166,"三 ":110,"ã‚ã‚ã‚":113},"n_words":[8074753,9381487,7891301],"name":"ms","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/ne.json b/contrib/languages-data/ne.json
new file mode 100644
index 0000000..9c1aba5
--- /dev/null
+++ b/contrib/languages-data/ne.json
@@ -0,0 +1 @@
+{"freq":{"ेशà¥":122,"ेषà¥":121,"दन ":955," कम":272," कर":171," कल":142," कि":313,"ेशक":101," का":2467," कृ":130," कà¥":353," के":400," को":746," कà¥":668,"ेवा":114,"ेवी":117," à¤à¤•":2232," à¤à¤®":84," à¤à¤¸":66," à¤à¤µ":82,"ेही":103," चौ":73," चà¥":87," ची":65," चि":272," चा":174," छन":436,"थी ":98," जर":83,"तको":395," छो":111," जस":287," १ ":94,"तका":126," छा":110," जन":834," जà¥":137," २ ":106," जि":1031," जा":297," जà¥":184," जी":93," जे":102,"था ":777," छ।":1176," जो":201," ३ ":81," à¤à¤¾":69," गर":1726," खो":93," खे":113," गण":154," गत":96," खा":152," घर":283," गौ":72," गो":262," गà¥":152,"ैति":183," गà¥":176," गा":503," । ":1729,"दछ ":91," चल":134,"णमा":89," चर":93," चन":91,"ोलà¥":69," अं":111,"ोलि":69,"ोला":99,"ोलन":95," अप":72," अन":738," अध":277," अथ":96," आक":83,"ेता":128," अल":77," अर":427," अम":148," अभ":65,"ोरà¥":161," अस":323," अव":627,"ेतà¥":416," अक":100," अग":100," अत":71," अञ":333,"ेपा":2443,"ेनà¥":518,"ोबा":73,"थल ":101,"ेना":77,"ंग ":129,"ेमा":93," à¤à¤‰":305,"तॠ":79,"थम ":65," इल":78," इन":145,"ेली":90,"ेला":163,"तो ":127," आर":92," आय":68," आन":91," आद":140," आध":94," आफ":173," उह":329,"ेरà¥":71," उप":369," उन":300," उद":118," उत":328,"ेरि":117," उच":89,"ेरै":136,"ंघ ":125," वा":585," वी":67," वि":1830," वà¥":93,"à¥à¤¤à¤°":540," वà¥":234," शर":146,"ं ":520,"ैशा":79," वै":119," शब":115," शà¥":75,"à¥à¤¤à¥‹":97," सं":1338,"à¥à¤¤à¥":507," शि":206," शा":289,"à¥à¤¤à¥ˆ":88," शह":107,"à¥à¤¥à¤²":136,"à¥à¤¤à¤¾":424," सक":211,"à¥à¤¤à¥€":153," शà¥":338,"ताक":144,"à¥à¤¤à¤¿":514,"à¥à¤¤à¥":122,"à¥à¤¥à¥":92,"तान":131,"à¥à¤¦à¤›":254,"à¥à¤¥à¤¿":236,"ताम":82,"à¥à¤¥à¤¾":656,"तार":119,"ताल":259,"तिक":679,"à¥à¤¥à¥€":106,"ताह":141,"à¥à¤¦à¥‹":111,"à¥à¤¦à¥":640," वट":348,"तिन":112,"तिम":220,"ठ":497,"तिब":72,"à¥à¤¦à¤¾":387,"तिल":64," वर":390,"à¥à¤¦à¥€":92,"à¥à¤¦à¤¿":109,"तिर":95,"à¥à¤¦à¥‚":82,"तीक":85,"तिह":119," वन":80,"à¥à¤¦à¥ˆ":77,"à¥à¤¦à¥‡":123,"à¥à¤Ÿà¥":320," लग":144,"à¥à¤Ÿà¥€":291,"à¥à¤Ÿà¤¾":74," लल":75,"à¥à¤¡à¤²":215," लà¥":106," ला":569," लि":251," ले":262,"इ ":252," लो":97,"à¥à¤¡à¤•":81,"à¥à¤£à¤•":65," या":166," यस":2003," यह":103," यि":70,"धन ":65," यà¥":243," यो":2991,"à¥à¤¤à¤•":221," रा":1721," रह":505," रे":111," रू":232,"दै ":146," रà¥":253," रो":152,"à¥à¤¬à¤¨":84,"दी ":721,"à¥à¤¬à¤¿":65,"à¥à¤®à¤•":109,"à¥à¤®à¤¨":119,"à¥à¤¯à¤•":669,"à¥à¤®à¤¾":313,"à¥à¤®à¤¿":89,"à¥à¤®à¥€":132,"à¥à¤¯à¤¤":93,"à¥à¤®à¥‡":80,"à¥à¤¯à¤®":169,"à¥à¤°à¤•":2903,"à¥à¤¯à¤µ":169,"à¥à¤¯à¤¸":223,"à¥à¤¯à¤¹":70,"à¥à¤°à¤œ":90,"à¥à¤¯à¥":227,"à¥à¤¯à¤¾":1225," हो":3798,"à¥à¤§à¤¾":79,"à¥à¤§à¤¿":77,"à¥à¤§à¥":64,"तया":66," हि":324," हा":314," हà¥":1822," हे":105,"à¥à¤¨à¥‡":717,"à¥à¤¨à¥":381,"तरी":81,"à¥à¤¨à¤¾":84,"तरा":100,"à¥à¤¨à¥‹":104," सम":2258," सभ":305," सब":222,"à¥à¤ªà¤¨":76," सन":367,"à¥à¤ªà¤¤":195," सर":306,"दा ":445," सद":429," सत":75,"तरà¥":239," सà¥":1058," हर":182,"à¥à¤«à¤¬":94," सो":117,"à¥à¤ªà¤¾":1027," सा":1310," सि":468," सह":215," से":264,"à¥à¤ªà¥":73," सà¥":497," सू":97,"दि ":71," दल":430," दर":95," दà¥":249," दा":223,"ोक ":69," दि":387," दक":141," तà¥":468," ते":64," था":199," थि":1391," तर":320," तथ":624," ता":221,"à¥à¤•à¥":76," ति":156,"ोग ":226,"तमा":143,"à¥à¤•à¥‹":75," ती":93,"à¥à¤•à¤¾":162,"à¥à¤•à¥ƒ":122,"à¥à¤•à¥€":66,"णà¥à¤¡":575,"ैनि":66," ठà¥":72," ठू":98," डा":80,"à¤à¤•à¥‹":90," टे":70,"ैभन":80,"तपà¥":159," ५ ":87," ठा":328,"तनà¥":137," ४ ":90,"७१ ":81," मो":133," मे":182,"à¥à¤œà¤¾":70," मा":2199," मि":226," मी":71," मà¥":393,"à¥à¤œà¥€":68,"à¥à¤›à¥¤":1182," मह":623," भो":79," मन":295," भे":81," मध":395,"ैमा":131," भà¥":80," मण":175," भू":134," भि":128," भा":1679," मज":70," भर":94," बà¥":200," भद":64," बे":149," भन":724," बै":74," बो":103,"à¥à¤žà¤¾":169," बा":593," बि":392," मं":65," बी":67," बà¥":143," भग":64," बह":81," बस":196,"à¥à¤Ÿà¤°":85," फà¥":93," भà¤":2250," बर":162,"दल ":217," बन":332," फा":77," बज":106,"तथा":612," बढ":88," पà¥":5303," पो":110," फल":67," पि":113," पा":1007," पà¥":600," पे":81," पृ":77," पू":312,"ौं ":109," पर":802," पश":191," पह":651," पछ":122," पन":637," पद":103,"णाल":87," पत":3652,"ौठ":65," नà¥":72," नै":168," नि":657," ना":582," पं":74,"à¥à¤šà¤²":413," ने":2763," धे":138,"à¥à¤›à¤¨":169," नय":92," नर":78,"à¥à¤šà¤¿":231," नव":75,"à¥à¤šà¤¾":256," धा":133,"à¥à¥¤ ":184," नद":186," दै":64," दे":732," धन":78," दà¥":298," दो":108," धर":161," नग":197,"थ ":223,"दछ।":89,"द ":562,"ध ":304,"दछन":85,"न ":5506,"ड ":307," छ ":575,"ठ ":146,"थाल":88,"थिà¤":221,"थाप":321,"ण ":876,"थान":166,"थिय":1145,"त ":3231,"थित":211,"धी ":65,"ज ":416,"दकà¥":143,"ट ":1241,"धि ":82,"ं":3288,"ः":73,"à¤":2200,"आ":1341,"इ":1404,"अ":4126,"ऋ":113,"ई":1594,"उ":3478,"घ ":163,"à¤":6874,"ओ":539,"à¤":69,"ग":8111,"ख":2732,"क":41084,"औ":111,"छ":5495,"च":3790,"ङ":845,"घ":968,"ट":4637,"ञ":822,"à¤":319,"ज":7684,"ठ":1551,"ङ ":292,"ड":2764,"ढ":364,"ण":2637,"त":20032,"थ":4532,"द":12121,"ध":3905,"न":29922,"प":22723,"फ":1251,"à¥à¤— ":136,"ब":5465,"भ":7912,"म":22298,"य":17139,"र":42105,"ल":18318,"व":12132,"ष":4104,"श":6654,"ह":13763,"स":20352,"ि":29268,"ा":66742,"े":14995,"थवा":92,"ू":2666,"ृ":815,"ी":11554,"च ":186,"à¥":12537,"ौ":1228,"à¥":49626,"ो":28329,"ै":2713,"।":9235,"०":2064,"१":1785,"à¥à¤• ":72,"छ ":1403,"६":623,"७":619,"८":678,"९":972,"२":2170,"३":565,"४":638,"५":791,"क ":5557,"ेल ":167,"ग ":763,"ख ":276,"ेर ":327,"तà¥à¤¸":67,"तà¥à¤µ":197,"तà¥à¤ª":68,"तà¥à¤°":5347,"तà¥à¤¯":758,"तà¥à¤¤":449,"ठ":239,"तीय":69,"दॠ":189,"ेस ":70,"उ ":287,"ई ":1278,"ेश ":318,"ै ":1366,"à¥à¤¨ ":467,"नै ":335," ख":612," ग":3519," औ":90," क":6153," ओ":97," à¤":65," ट":271," ज":3597," à¤":164," च":1230," छ":2605," घ":502," इ":461," आ":1284," अ":3988," à¤":2988," ऋ":77," उ":1784," ई":92,"दार":162,"नो ":211,"दान":85,"दीक":207,"दिर":84,"े ":3604,"दिन":243,"नॠ":626,"दà¥à¤ˆ":79,"ौà¤à¤®":309,"दà¥à¤°":326,"ेकप":97,"à¥à¤® ":374,"ू ":585,"ेका":344,"ेको":828,"à¥à¤¯ ":1998,"ेखि":464,"à¥à¤° ":942,"ेखा":65,"ि ":3525,"नी ":612,"ेटि":110,"ी ":7016,"नॠ":137,"ोतà¥":73,"ॠ":1173,"ा ":19102,"à¤à¤‰ ":227," ८":76," ९":73," ६":100," ७":169," ४":169," ५":165," २":1308," ३":186," ०":277," १":1083," ।":2222,"à¥à¤µ ":234,"ेजी":71,"दसà¥":318," प":14289," फ":558," न":5247," म":5367," य":5733,"à¥à¤· ":268," ब":2996," भ":5836," ढ":70," ठ":526," ड":306,"ह ":271," द":2964," ध":656," त":2303," थ":1702," ह":6806," स":10170," ल":1764," र":4971," श":1531," व":3977,"ने ":1547,"à¥à¤¸ ":116,"स ":1459,"ष ":373,"थà¥à¤¯":109,"श ":446,"à¥à¤› ":694,"व ":550,"à¥à¤š ":74,"दरम":107,"à¥à¤Ÿ ":232,"ल ":2508,"दलह":69,"नि ":671,"दरà¥":91,"दलक":97,"ना ":698,"à¤à¤— ":83,"र ":6666,"à¥à¤¡ ":120,"य ":3205,"à¥à¤  ":89,"म ":1712,"à¥à¤£ ":171,"ोखर":85," र ":1581,"ब ":130,"à¥à¤¥ ":94,"फ ":86,"à¥à¤¤ ":382,"à¥à¤§ ":250,"प ":228,"ो। ":2741,"à¥à¤¦ ":129,"ृषà¥":136,"डल ":72,"डी ":201,"डा ":124,"ञà¥à¤œ":95,"ञà¥à¤š":499,"ृति":122,"ृतà¥":80,"ञान":153,"ेत ":110,"ठमा":517,"टà¥à¤°":339,"टà¥à¤Ÿ":87,"ढी ":85,"ेक ":82,"अकà¥":91,"टिन":116,"अञà¥":333,"टीक":104,"à¥à¤µà¤¾":235,"ूको":117,"à¥à¤²à¤¾":154,"à¥à¤²à¥‹":69,"à¥à¤²à¥":89,"à¥à¤²à¥‡":184,"à¥à¤°à¤¾":226,"à¥à¤°à¥‹":69,"à¥à¤°à¥":237,"à¥à¤°à¥€":67,"à¥à¤°à¥":159,"à¥à¤°à¤®":86,"à¥à¤®à¥":151,"à¥à¤°à¤•":79,"à¥à¤®à¤¾":298,"णी ":110,"à¤à¤¦à¥ˆ":80,"à¥à¤­à¤¯":124,"णा ":67,"डलक":101,"à¥à¤­à¤":420,"à¥à¤ªà¥":75,"à¥à¤ªà¤¾":81,"à¥à¤ªà¤®":101,"à¥à¤¨à¥":1483,"à¥à¤¨à¥ˆ":129,"à¥à¤¨à¥‡":151,"à¥à¤¨à¥":515,"à¥à¤¨à¤¿":176,"à¥à¤¨à¤¾":77,"à¥à¤¦à¥":231,"à¥à¤¸à¤¾":87,"à¥à¤¸à¥‡":64,"à¥à¤·à¥":65,"à¤à¤®à¤¾":361,"à¥à¤¸à¥":64,"ठाउ":291,"ठूल":104,"à¥à¤²à¤¾":1132,"à¥à¤²à¥‹":92,"à¥à¤²à¥‡":68,"ंचा":97,"à¥à¤°à¤¦":218,"à¥à¤°à¤¥":82,"à¥à¤°à¤¤":719,"à¥à¤°à¤£":90,"à¥à¤°à¤ª":72,"à¥à¤°à¤¨":77,"à¥à¤¯à¥‡":172,"à¥à¤°à¤§":142,"à¥à¤°à¤®":518,"à¥à¤¯à¥‹":226,"ंगà¥":118,"à¥à¤°à¤¯":179,"à¥à¤°à¤µ":165,"à¥à¤°à¤¶":109,"à¥à¤°à¤¹":151,"à¥à¤°à¤¸":389,"à¥à¤°à¤¾":928,"ंगा":83,"à¥à¤°à¤¿":4096,"à¥à¤°à¥€":584,"ठà¥à¤²":72,"à¥à¤°à¥ˆ":117,"à¥à¤°à¥‡":422,"à¥à¤°à¥‹":266,"à¥à¤·à¤•":74,"ीहर":197,"à¥à¤·à¤°":74,"ंगठ":68,"à¥à¤·à¥‡":409,"à¥à¤·à¤¿":335,"à¥à¤·à¤¾":151,"à¥à¤·à¥€":92,"à¥à¤µà¤•":82,"à¥à¤—à¥":67,"à¥à¤µà¤¤":106,"à¥à¤µà¤°":184,"à¥à¤µà¤µ":84,"à¥à¤µà¤®":76,"à¥à¤µà¤¯":164,"à¥à¤µà¥€":175,"à¥à¤µà¤¾":1398,"डौà¤":365,"डौं":74,"ूला":68,"ूलो":82,"ति ":1256,"ता ":557,"णको":70,"तॠ":113,"ूमि":65,"ूमा":80,"ती ":337,"ूरà¥":396,"ूपम":142,"तै ":115,"ते ":88,"ूदà¥":136,"ंमा":238,"ंसà¥":240,"तर ":294,"à¥à¤¸à¥":101,"à¥à¤¸à¤¾":127,"à¥à¤¹à¥":425,"ंवि":168,"ाला":279,"ालि":277,"ाली":1573,"ालà¥":175,"ाले":372,"छ। ":2152,"ावि":101,"ाशन":833,"िकृ":90,"िका":4480,"ाशक":884,"िको":207,"िकà¥":271,"ाषा":1077,"ासन":94,"ासद":191,"ाषà¥":296,"ाशि":795,"ासक":88,"ाही":67,"ाहि":278,"ाहा":83,"à¥à¤¨ ":257,"ासि":356,"ासी":103,"ाहर":380,"ासà¥":236,"जा ":203,"चलक":256,"िजà¥":95,"चरà¥":65,"िजय":135,"जी ":92,"à¥à¤° ":568,"ङà¥à¤—":239,"ितप":65,"छनà¥":559,"िता":146,"िति":411,"ितà¥":493,"िदà¥":368,"िधि":130,"िधा":249,"िना":260,"िनि":160,"िनी":149,"िने":242,"िनà¥":78,"िनà¥":1022,"चीन":87,"िभि":96,"िभा":91,"ियन":73,"िमà¥":132,"ियम":114,"िमा":914,"ियो":1315,"िया":360,"िरà¥":332,"िरा":176,"िले":173,"िलà¥":1069,"िलो":573,"चित":254,"िला":204,"चाय":66,"चार":351,"चाल":100,"चिव":79,"चिन":100,"चिम":183,"िषà¥":174,"िशà¥":356,"िशे":71,"िवा":232,"ीका":189,"ीको":490,"िहा":169,"िसà¥":142,"ूल ":110,"ीति":294,"चà¥à¤š":109,"जमा":145,"ीदà¥":108,"छोर":78,"जनà¥":230,"जनक":98,"जनत":72,"जधा":122,"जनव":107,"जनै":185,"जनी":264,"जना":168,"जदà¥":67,"ीमा":357,"ीया":65,"à¥à¤•à¥":205,"à¥à¤•à¥‹":176,"à¥à¤•à¤¾":171,"à¥à¤–à¥":130,"ृत ":152,"ीले":86,"ीला":83,"जसà¥":157,"जसल":64,"ित ":1545,"िण ":71,"जिल":993,"जिक":146,"जार":186,"जान":79,"जात":246,"जरà¥":77,"जयी":89,"ाà¤à¤‰":242,"ाà¤à¤š":67,"ाà¤à¤•":137,"िन ":363,"जवा":81,"ाइन":178,"िल ":92,"जोड":89,"ाउà¤":218,"ाउं":237,"ाउन":366,"ीक ":78,"जà¥à¤¯":351,"ाà¤à¤¸":67,"ांग":75,"जà¥à¤ž":223,"ाà¤à¤²":66,"िम ":169,"िय ":243,"िर ":148,"जील":69,"जीव":121,"ाà¤à¤•":100,"जà¥à¤¨":124,"टर ":108,"िव ":68,"िस ":86,"ागà¥":194,"ागà¥":64,"ागि":189,"ीत ":128,"ागर":78,"ागम":84,"ाको":3606,"ाकà¥":157,"ाका":303,"ाओव":397,"ाडी":141,"िंह":72,"ाठम":478,"ाटन":82,"टी ":233,"टा ":741,"ाजा":131,"ाजि":146,"ाजà¥":244,"ाजन":456,"ाजध":126,"ाजव":83,"ीन ":153,"ाचन":103,"ाङà¥":108,"à¥à¤ˆ ":73,"ाने":97,"ाना":169,"ानि":283,"ानी":322,"ानà¥":103,"ानव":65,"à¥à¤– ":149,"ानस":161,"ानम":146,"ाधà¥":103,"ापन":228,"ानà¥":702,"ानो":110,"ादे":93,"ादी":738,"ादà¥":180,"ादन":914,"ानक":94,"ादà¥":158,"ाति":178,"ाता":140,"ाती":68,"ाणà¥":134,"ातà¥":408,"ादक":75,"ीय ":427,"ाडौ":440,"ारी":407,"ारि":163,"ारा":1342,"ारà¥":2184,"ारे":69,"ालक":765,"ालम":256,"ालय":324,"ालद":78,"ामा":889,"ायक":72,"ामà¥":197,"ायण":151,"ायत":156,"ाया":71,"ारक":244,"ारम":182,"ारण":184,"ारत":376,"ाबा":106,"िà¤à¤•":373,"ामक":80,"ामय":74,"ामम":66,"ापा":186,"ापà¥":232,"à¥à¤™ ":64,"ंघ":232,"à¤à¤¸":83,"ौ ":65,"ंख":88,"ंग":634,"ंक":193,"à¤à¤²":78,"ंच":134,"केन":240,"à¤à¤¦":154,"à¤à¤¡":80,"à¤à¤®":363,"ंस":389,"ंह":86,"ंव":201,"ॠ":1181,"ंत":94,"ंम":241,"केह":83,"à¤à¤•":178,"ो ":20357,"à¤à¤—":166,"à¤à¤š":76,"à¤à¤‰":243,"कृत":264,"अस":341,"अव":627,"आक":84,"अल":82,"अर":461,"अभ":65,"आà¤":70,"अम":151,"घर ":251,"अप":74,"अध":281,"अन":771,"अथ":96,"इत":80,"ाइ ":156,"इà¤":81,"आय":68,"आर":95,"आफ":174,"कृष":151,"आद":141,"आध":94,"आन":112,"ाई ":1080,"अं":111,"अञ":333,"अत":71,"अक":101,"अग":107,"उम":66,"उप":377,"उह":331,"ऋत":72,"इन":366,"इर":70,"इल":137,"इस":102,"उà¤":251,"उं":238,"उक":79,"उच":90,"उट":311,"उत":340,"हो।":2612,"उन":717,"उद":141,"कोट":86,"à¤à¤‰":305,"à¤à¤•":5494,"à¤à¤®":113,"à¤à¤ª":66,"कà¥à¤•":84,"कà¥à¤¤":425,"कà¥à¤¯":129,"कà¥à¤¨":67,"कà¥à¤·":1267,"कà¥à¤°":426,"कà¥à¤¸":110,"ाठ":310,"à¤à¥¤":67,"à¤à¤µ":82,"à¤à¤°":147,"à¤à¤¸":80,"गर":2258,"गल":135,"गव":76,"खे":215,"गन":66,"खो":112,"गम":263,"खà¥":345,"खा":431,"गठ":95,"खि":580,"खी":82,"खà¥":123,"ाग ":123,"गढ":68,"गण":181,"गत":326,"गको":91,"खर":150,"कà¥":2722,"खम":75,"गक":127,"कै":214,"के":654,"को":14452,"कि":691,"की":474,"का":12832,"कृ":425,"कà¥":618,"कू":91,"कस":66,"कव":78,"कह":90,"कल":273,"कम":415,"कर":463,"कप":247,"कन":76,"कत":154,"ाङ ":105,"कक":112,"ओव":398,"चà¥":124,"ची":172,"चि":731,"चा":795,"छन":717,"चौ":90,"चà¥":216,"चो":89,"० ":345,"जक":89,"चन":324,"ङà¥":397,"चर":129,"चल":599,"घा":108,"गणà¥":66,"ङम":69,"गते":76,"। ":7045,"गा":955,"गी":218,"गि":310,"गà¥":302,"गो":351,"गौ":84,"गà¥":809,"गे":161,"गै":75,"घर":335,"टन":160,"ञà¥":596,"ञा":169,"टा":992,"टर":235,"४ ":302,"à¤à¤¾":105,"३ ":263,"टक":135,"जो":225,"जे":208,"छ।":2479,"जी":387,"जà¥":351,"जा":939,"जि":1330,"२ ":297,"जà¥":670,"जन":1499,"खान":74,"जद":90,"जध":126,"छि":255,"छा":205,"जस":334,"जव":93,"१ ":351,"जर":120,"जल":76,"गठन":79,"छो":118,"जय":160,"जम":205,"ठा":393,"डक":133,"ाथ ":102,"५ ":351,"ठन":83,"टे":203,"ठम":522,"टà¥":498,"टो":148,"टी":408,"टि":258,"डा":364,"डि":270,"डी":265,"डल":256,"६ ":313,"ाद ":282,"ठà¥":73,"ठू":104,"ढी":114,"ढा":72,"ाण ":70,"णक":119,"७ ":328,"डà¥":123,"डौ":474,"डो":86,"डे":171,"णि":132,"णी":170,"णा":196,"ात ":197,"तक":594,"८ ":352,"णम":93,"तव":80,"९ ":264,"तह":110,"ति":2878,"ता":1722,"तà¥":292,"ती":680,"तथ":624,"तप":208,"तन":233,"तम":226,"णà¥":631,"तय":73,"तल":91,"तर":935,"थव":101,"दछ":271,"था":1584,"थी":140,"थि":1694,"ते":197,"तै":125,"तो":181,"थम":91,"तà¥":7154,"थल":165,"खेल":103,"दक":289,"दस":347,"दू":145,"दà¥":584,"दी":1175,"दि":689,"दा":996,"दन":1011,"दव":74,"दल":480,"दर":324,"दम":71,"थà¥":174,"नज":74,"धा":907,"नत":94,"नद":266,"धी":98,"धि":828,"ान ":852,"धà¥":156,"दो":255,"दौ":70,"दà¥":2831,"दे":1366,"धन":183,"दै":256,"नक":650,"नग":337,"धर":218,"नर":137,"नल":223,"नव":289,"ाज ":178,"धे":169,"नन":100,"नप":96,"नब":64,"नय":150,"नम":477,"धà¥":787,"पं":75,"नी":1179,"नà¥":1548,"ने":4740,"नस":329,"नह":97,"ना":2286,"नि":2559,"पक":159,"नो":316,"नै":540,"नà¥":7416,"पत":4116,"पन":1101,"पद":120,"न।":118,"पछ":277,"पट":80,"पश":213,"पह":665,"पस":130,"पल":132,"पम":272,"पर":1026,"पे":166,"पू":427,"पृ":79,"पा":5213,"पि":272,"पी":141,"पà¥":1182,"फल":131,"फर":70,"पà¥":6004,"फब":94,"पो":196,"बन":449,"फे":70,"बत":86,"फà¥":76,"बढ":88,"फू":67,"फा":131,"बज":120,"फà¥":316,"भà¤":2692,"बर":285,"बल":88,"भक":78,"भग":82,"बस":232,"बह":195,"बि":521,"बा":1458,"बà¥":218,"मं":75,"बी":142,"भद":79,"भन":878,"बे":211,"गरम":92,"बै":272,"बो":141,"गरप":94,"भय":191,"भर":147,"गरे":321,"बà¥":486,"मक":339,"गरी":134,"गरि":502,"मग":77,"मज":101,"भि":375,"भा":2323,"मत":163,"मण":318,"भू":158,"भà¥":153,"गरà¥":821,"मन":572,"भे":108,"मध":481,"मद":72,"भो":89,"मप":65,"मल":187,"यक":819,"मम":151,"भà¥":133,"मय":212,"मर":128,"मस":126,"मह":673,"मृ":68,"यत":299,"मू":114,"यण":158,"खà¥à¤¯":245,"यद":91,"मि":1113,"मा":10729,"ाट ":785,"मà¥":840,"मी":348,"रं":73,"मो":273,"यम":527,"मà¥":2421,"मे":530,"यन":231,"यप":89,"मै":88,"यव":183,"रख":100,"रग":116,"रक":3696,"यल":80,"यर":101,"या":2168,"रज":106,"यह":189,"रच":84,"गमा":170,"यस":2268,"रध":173,"रद":310,"रथ":102,"रत":1236,"रण":521,"यà¥":657,"यी":175,"यि":191,"रय":191,"रम":1296,"रभ":77,"रब":131,"यो":5061,"रप":226,"रन":174,"ये":226,"लम":462,"लय":351,"लब":81,"लद":123,"लन":243,"गाउ":130,"लच":65,"लग":235,"लक":1487,"रà¥":6868,"रो":703,"रै":297,"रे":1245,"गाà¤":224,"रू":1254,"री":1735,"रà¥":1451,"रि":5597,"रा":5311,"रह":868,"रस":574,"रश":114,"रव":321,"रल":193,"वं":88,"िक ":1742,"लà¥":1854,"लो":1078,"चन ":69,"ले":2083,"लà¥":247,"ली":1940,"लि":1013,"ला":3708,"लल":168,"लह":138,"लस":69,"शब":122,"वै":233,"शन":968,"वे":271,"षक":102,"शम":116,"वà¥":325,"शर":176,"वह":113,"वव":90,"वस":368,"वà¥":106,"वा":3817,"वि":2520,"वी":434,"वप":72,"वन":350,"वध":419,"शक":1037,"वल":144,"वर":826,"वय":169,"वम":117,"वक":170,"वत":227,"वट":383,"ाह ":84,"सन":601,"षे":417,"सप":72,"सभ":377,"सब":248,"षà¥":905,"सम":2673,"सर":429,"सल":312,"सव":65,"हक":127,"सच":90,"षा":1291,"षि":415,"षी":120,"िङ ":81,"ास ":619,"सत":83,"सद":673,"शे":152,"शà¥":1168,"षर":77,"सग":64,"सक":2028,"शह":143,"सं":1482,"शी":150,"शà¥":136,"शा":630,"शि":1111,"सà¤":111,"षण":89,"हे":701,"हà¥":2366,"ही":279,"हि":1570,"हा":1720,"िठ":165,"ाम ":560,"हà¥":87,"हो":3850,"से":478,"हन":162,"सà¥":586,"सी":345,"हत":108,"सू":105,"सि":1173,"चल ":150,"सा":2163,"सह":275,"हज":68,"हल":85,"सà¥":3576,"हर":1967,"सै":149,"सो":307,"ात":1181,"ाथ":320,"ाण":404,"ाठ":517,"िं":132,"ाड":760,"ाल ":1016,"ाट":1019,"ाब":231,"ाभ":93,"ाप":949,"ान":3357,"ाद":2589,"ाध":265,"गà¥à¤°":71,"िख":71,"ाव":435,"िक":7298,"ाल":5422,"ार":6852,"ाय":812,"ाम":2329,"िà¤":681,"िज":372,"ाह":1078,"िच":143,"िङ":184,"ास":1934,"ाष":1391,"ाश":2631,"िग":115,"ां":267,"ाà¤":1104,"ाइ":698,"ाई":1298,"ाउ":964,"ाओ":403,"ाक":4316,"ाà¤":185,"गà¥à¤¨":74,"ाच":265,"ाज":1659,"ाग":1013,"ाख":288,"ाङ":244,"ाघ":70,"ार ":1305,"à¥à¤":124,"ीद":163,"गिर":76,"à¥à¤ˆ":88,"ीत":508,"ीप":148,"ीन":313,"ीम":461,"ीय":520,"ीब":79,"ील":304,"ीर":227,"à¥à¤—":188,"à¥à¤–":347,"ीव":147,"गीत":100,"à¥à¤•":691,"ीह":209,"ीस":78,"à¥à¤™":145,"à¥à¤Ÿ":214,"िट":172,"ाय ":70,"िण":163,"ित":2827,"िद":484,"गाय":88,"िध":410,"िन":2481,"िप":245,"िब":181,"िभ":260,"िम":1517,"िर":1038,"िय":2231,"िल":2254,"ीक":872,"िश":533,"िव":468,"िस":542,"िष":301,"िह":253,"ेव":427,"ेश":820,"ेल":681,"सला":171,"ेर":844,"ेम":230,"ेब":84,"ेप":2592,"सले":93,"ेन":806,"ैत":257,"ैज":79,"ेह":196,"ेस":274,"ेष":206,"ैश":81,"ैर":64,"ैल":110,"ैभ":84,"ैम":134,"ैन":181,"ेक":1516,"ेख":723,"ेट":234,"ेड":98,"ेत":801,"ेद":151,"ेग":72,"ेज":215,"समà¥":85,"समि":387,"समा":459,"ृथ":65,"ृत":389,"ृष":194,"गोर":127,"सरक":167,"गोल":71,"समे":69,"समà¥":1386,"à¥à¤¤":130,"à¥à¤£":70,"à¥à¤¨":2984,"à¥à¤¦":383,"à¥à¤¬":76,"à¥à¤ª":397,"à¥à¤°":1722,"à¥à¤®":556,"à¥à¤­":589,"ूक":155,"गà¥à¤²":85,"à¥à¤²":730,"à¥à¤·":88,"à¥à¤¸":363,"à¥à¤µ":409,"गà¥à¤°":367,"à¥à¤¶":84,"à¥à¤¹":538,"सरà¥":105,"गà¥à¤¨":160,"ूद":147,"ून":67,"ूप":214,"ूम":171,"ूर":453,"ूल":388,"ूह":74,"सबै":187,"à¥à¤µ":2939,"à¥à¤¶":82,"à¥à¤·":1678,"à¥à¤¸":579,"à¥à¤¹":81,"à¥à¤­":100,"à¥à¤®":1441,"à¥à¤¯":5693,"à¥à¤°":13669,"à¥à¤²":1547,"à¥à¥¤":480,"समय":115,"षà¥à¤®":69,"षà¥à¤£":122,"षà¥à¤ ":171,"षà¥à¤Ÿ":483,"सभा":365,"सदà¥":160,"ौत":91,"ोर":513,"ोल":515,"सदस":314,"ोब":119,"ोम":148,"ोस":122,"ोह":153,"सदर":107,"ोश":104,"ोव":69,"à¥à¤£":427,"à¥à¤¤":3287,"à¥à¤¡":711,"à¥à¤Ÿ":1245,"à¥à¤ ":192,"à¥à¤ž":223,"à¥à¤«":210,"à¥à¤¬":412,"à¥à¤ª":1700,"à¥à¤§":702,"à¥à¤¨":1912,"षेत":396,"à¥à¤¥":1433,"ो।":3741,"à¥à¤¦":2288,"ौल":71,"ौर":128,"à¥à¤œ":453,"à¥à¤›":2152,"à¥à¤š":1134,"à¥à¤—":579,"à¥à¤–":113,"à¥à¤•":866,"सनà¥":376,"ोज":238,"ौं":159,"ोड":185,"ोट":228,"ौà¤":401,"ोद":66,"ोत":169,"ोप":186,"ोध":78,"ोन":98,"ोख":109,"ोक":287,"ोग":453,"हतà¥":91,"सेन":118,"छन ":99,"सà¥à¤¨":123,"सिर":66,"सिम":95,"सिन":119,"सà¥à¤°":172,"साह":147,"सिद":138,"सान":163,"साद":218,"साप":131,"सार":311,"साम":357,"सिक":403,"साल":247,"सिं":91,"साथ":94,"सहर":108," १८":152," १९":516," २०":1012,"०६":103,"०४":114,"०५":217,"०२":500,"०३":93,"००":262,"०१":255,"१०":116,"१५":81,"१७":82,"१८":226,"१२":66,"१४":88,"१९":555,"२०":1126," ७१":72,"हरू":762,"हरà¥":794,"हरि":77,"ङमा":65,"७१":90,"सà¥à¤µ":373,"९०":64,"छि ":189,"९६":77,"९५":70,"९८":66,"९१":112,"सà¥à¤°":74,"सà¥à¤¯":402,"सà¥à¤¨":77,"सà¥à¤¥":1019,"सà¥à¤ª":226,"सà¥à¤Ÿ":105,"सà¥à¤¤":727,"९९":85,"सà¥à¤•":351,"२३":69,"२२":66,"२५":83,"२७":122,"२६":80,"२९":70,"२८":115,"सोज":81,"सेव":93," १०":76," ०५":134,"हेक":471,"हेन":82,"जन ":98,"हà¥à¤¨":2172,"हà¥à¤":68,"हिम":124,"हिल":691,"हिन":285,"हित":161,"हास":142,"हिक":125,"हाल":183,"हार":140,"हान":72,"हाद":225,"हाड":113,"हाà¤":430,"चना":64,"चनà¥":135,"सचि":83,"सकि":79,"सका":115,"सको":1596,"सकà¥":148,"कै ":184,"का ":4257,"कि ":84,"षाक":932,"की ":394,"षिण":127,"हो ":1122,"षिक":187,"ओवा":396,"शेष":71,"हा ":81,"ही ":218,"शà¥à¤µ":419,"शà¥à¤°":376,"शà¥à¤¯":86,"शà¥à¤š":215,"शहर":129,"सी ":177,"हर ":124,"संग":215,"संख":67,"संघ":224,"संव":197,"संस":296,"शाख":89,"शित":796,"शाह":92,"शास":187,"सà¤à¤—":107,"शिक":120,"सो ":69,"शार":66,"वैश":69,"वेश":81,"वà¥à¤²":80,"शरà¥":111,"सा ":76,"वà¥à¤¯":273,"शबà¥":114,"वरà¥":460,"षा ":211,"किन":120,"काम":260,"कार":2072,"काल":377,"किक":104,"काश":2575,"कास":405,"किस":96,"कà¥à¤¨":166,"कà¥à¤°":99,"कà¥à¤®":159,"ववि":79,"शको":75,"कान":97,"काठ":489,"काक":1844,"वसà¥":272,"वहा":72,"सन ":103,"गि ":141,"वाद":803,"वान":150,"वाच":132,"विक":476,"वाल":112,"वास":237,"कला":91,"वार":1366,"वाम":64,"वित":78,"विन":80,"विद":224,"विध":297,"विज":192,"वाह":64,"गी ":84,"विष":89,"विश":380,"विस":90,"विम":83,"विर":102,"विभ":130,"वीर":70,"कमà¥":158,"कमा":129,"करà¥":161,"गा ":93,"करण":105,"कपà¥":78,"कपा":103,"वधि":404,"कता":70,"वमा":91,"गर ":180,"वयम":124,"सं ":192,"शी ":67,"गत ":162,"खि ":378,"वटा":364,"खी ":68,"खा ":132,"को ":13998,"शन ":893,"वि ":102,"à¤à¤®à¤¾":79,"वा ":501,"वी ":188,"लà¥à¤¨":81,"लà¥à¤ª":107,"लà¥à¤¯":159,"लà¥à¤²":1131,"à¤à¤µà¤‚":67,"à¤à¤•à¥€":88,"à¤à¤•à¤¾":367,"à¤à¤•à¤¿":98,"à¤à¤•à¥‹":2830,"लोक":92,"लेख":158,"शक ":887,"लेक":101,"लेट":101,"लिम":70,"लà¥à¤®":71,"वर ":134,"लाई":880,"लाइ":94,"लाग":378,"लाक":490,"लाम":331,"लाल":114,"लिक":155,"लाह":236,"लिङ":98,"लित":141,"लिन":92,"वन ":146,"लहर":130,"à¤à¤‰à¤Ÿ":304,"ललि":72,"लमा":361,"लदà¥":81,"लका":156,"वं ":67,"लगा":129,"लको":1159,"लकà¥":65,"रैम":78,"रेस":80,"रेष":101,"रोप":82,"रोग":68,"रà¥à¤¶":68,"रà¥à¤µ":558,"रà¥à¤¸":92,"रà¥à¤·":409,"रà¥à¤®":544,"रà¥à¤¯":1351,"रà¥à¤¥":286,"रà¥à¤¦":429,"रà¥à¤¨":750,"रà¥à¤«":154,"रà¥à¤Ÿ":316,"रà¥à¤£":301,"रà¥à¤¤":237,"रà¥à¤—":285,"रà¥à¤–":68,"रà¥à¤•":192,"रà¥à¤œ":198,"रà¥à¤›":112,"रà¥à¤š":149,"रीक":121,"रिव":148,"रिय":408,"रीम":87,"रीय":210,"रà¥à¤•":171,"रà¥à¤®":97,"रà¥à¤ª":191,"रà¥à¤²":94,"रूक":141,"रूद":136,"रूप":207,"रूम":82,"रूल":87,"रेक":307,"रेज":78,"रेल":67,"रेर":97,"रेन":80,"रसा":265,"रसि":109,"रहे":467,"रहर":171,"रसà¥":71,"ले ":1440,"राई":127,"रान":448,"राप":77,"रात":83,"राण":115,"राख":96,"राज":1244,"राक":104,"रिन":282,"रित":70,"राष":293,"राह":72,"राम":287,"रिà¤":176,"राय":214,"राल":136,"रिक":3842,"लो ":852,"रला":76,"लि ":69,"ला ":807,"रयो":160,"रमà¥":75,"रमा":715,"रमà¥":241,"रवा":142,"रले":64,"ली ":1594,"रधा":161,"येक":79,"रदे":155,"रमण":71,"यो।":1126,"रबा":70,"योग":340,"रपा":108,"यà¥à¤•":77,"यà¥à¤°":75,"यà¥à¤¨":112,"यà¥à¤¦":113,"याङ":69,"याक":103,"याम":150,"यान":128,"याप":111,"यात":120,"याद":68,"यास":84,"यिक":111,"याल":253,"यार":124,"याय":70,"यिन":65,"रति":595,"रतà¥":114,"रथम":70,"लय ":258,"यà¥à¤µ":76,"रतक":192,"रणा":86,"यसक":1478,"यसै":79,"यसल":145,"रगत":68,"यवस":100,"रकà¥":100,"रको":493,"रजा":74,"याà¤":127,"यहा":98,"यसà¥":93,"लन ":125,"रे ":127,"महे":70,"महा":221,"महि":223,"महत":77,"यको":226,"यकà¥":275,"रू ":452,"रॠ":592,"यका":195,"मले":67,"री ":1056,"मà¥à¤¦":90,"मà¥à¤–":298,"मà¥à¤•":206,"मिल":98,"मित":526,"मिन":83,"मार":349,"माल":263,"मिक":100,"रो ":251,"मास":338,"मिट":74,"माण":135,"माड":449,"माध":70,"माथ":111,"मात":198,"मान":717,"माओ":399,"माक":73,"माज":262,"रै ":187,"मोर":76,"यनà¥":64,"यदà¥":67,"मेत":64,"मेर":107,"मेल":71,"रका":2994,"यमि":88,"ऋतà¥":72,"मà¥à¤°":98,"मà¥à¤¯":162,"यमा":180,"मà¥à¤ª":1201,"मà¥à¤¬":311,"मà¥à¤®":335,"रत ":106,"मको":161,"रण ":320,"या ":441,"उहा":325,"यी ":158,"भिन":116,"भित":82,"भाव":75,"यो ":3433,"भास":199,"भाष":1089,"भार":394,"भाग":184,"उपत":120,"रम ":92,"मजद":66,"ये ":122,"मधà¥":444,"मना":80,"मनà¥":256,"भूम":78,"मणà¥":216,"मती":69,"मयि":71,"रा ":1447,"०० ":67,"रि ":116,"ममा":144,"मा ":6826,"बरà¥":135,"यत ":118,"भकà¥":69,"मी ":185,"यन ":118,"बसà¥":105,"बहा":133,"बाल":77,"बाह":65,"बास":89,"बिन":70,"बाट":745,"बाग":79,"बार":117,"बिह":66,"यम ":171,"बेल":114,"यस ":334,"भनि":212,"भने":219,"भनà¥":420,"बैभ":80,"भयो":166,"बà¥à¤°":111,"बà¥à¤¯":100,"बà¥à¤¦":140,"à¤à¤° ":132,"पà¥à¤¯":67,"पà¥à¤°":5545,"पà¥à¤¤":252,"भा ":133,"मन ":82,"बजा":98,"मय ":66,"यक ":69,"बने":91,"बना":154,"बनà¥":175,"à¤à¤• ":2017,"भà¤à¤•":2580,"फà¥à¤¨":128,"फà¥à¤°":99,"आदि":87,"पहि":540,"पहा":109,"पशà¥":183,"आनà¥":94,"परà¥":460,"आफà¥":129,"इà¤à¤•":72,"२०२":474,"परि":275,"२०१":229,"परा":119,"२००":176,"पमा":255,"�":311,"फबा":94,"पोख":90,"उन ":91,"पृथ":65,"पà¥à¤¸":117,"पूर":347,"पà¥à¤—":79,"पà¥à¤°":744,"पित":80,"पाक":119,"पान":108,"पात":131,"पाद":966,"पार":420,"पाल":2675,"पाइ":114,"बै ":101,"अथव":91,"१९१":90,"१९९":68,"१९६":68,"अधि":111,"अनि":120,"अनà¥":226,"अधà¥":166,"अनà¥":365,"नà¥à¤š":76,"नà¥à¤›":1928,"नà¥à¤¤":1150,"नà¥à¤Ÿ":89,"नà¥à¤§":228,"नà¥à¤¨":383,"नà¥à¤¥":109,"नà¥à¤¦":1479,"नà¥à¤¯":282,"नà¥à¤®":233,"नà¥à¤¸":147,"नà¥à¥¤":473,"नैत":179,"अमे":66,"अरू":140,"अरà¥":230,"अवस":167,"अवध":415,"पनी":71,"पना":230,"पनि":614,"पनà¥":133,"असो":81,"असà¥":166,"पतà¥":3819,"पति":105,"पता":130,"पछि":199,"पा ":229,"नले":102,"नया":92,"नवा":108,"ननà¥":74,"०२७":93,"०२८":90,"नदी":180,"धेर":139,"नमा":333,"धà¥à¤¯":728,"नà¥à¤¹":426,"उटा":296,"नà¥à¤ª":72,"नà¥à¤­":561,"नà¥à¤¸":135,"निस":101,"निष":82,"नीत":296,"नेप":2459,"नेत":159,"नेक":221,"फल ":80,"उनी":101,"उनà¥":99,"उने":183,"उनल":77,"उदà¥":70,"उनक":91,"नसà¥":108,"निर":312,"निय":235,"निम":64,"निन":391,"उतà¥":329,"निध":76,"नाल":106,"निक":384,"नाम":395,"नार":232,"नाथ":91,"नाक":103,"नाउ":100,"नाइ":96,"दà¥à¤¦":65,"दà¥à¤µ":1093,"दà¥à¤°":692,"दà¥à¤¯":285,"दà¥à¤§":407,"धरà¥":124,"नकप":71,"इने":87,"इनà¥":241,"नका":130,"नको":327,"इला":81,"देख":530,"देव":249,"देश":451,"दैन":81,"१८ ":75,"दोल":117,"२७ ":96,"धित":72,"धिम":418,"धार":244,"धिक":138,"धान":515,"२८ ":93,"नता":76,"नगर":305,"न। ":91,"उंम":222},"n_words":[614665,704688,490631],"name":"ne","type":"devanagari"} \ No newline at end of file
diff --git a/contrib/languages-data/nl.json b/contrib/languages-data/nl.json
new file mode 100644
index 0000000..6f4124a
--- /dev/null
+++ b/contrib/languages-data/nl.json
@@ -0,0 +1 @@
+{"freq":{"D":295391,"E":101171,"F":127232,"G":105530,"A":221658,"B":178586,"C":193400,"L":124657,"M":162943,"N":137196,"O":83395,"H":199830,"I":105198,"J":61744,"K":84088,"U":22509,"T":111990,"W":76478,"V":96433,"P":145276,"S":218863,"R":114676,"Y":11382,"Z":56681,"f":358313,"g":1325585,"d":2549536,"e":9521882,"b":675678,"c":885504,"a":4428747,"n":5174765,"o":3044475,"l":2174413,"m":1301830,"j":487381,"k":957853,"h":1157193,"i":4146613,"w":618312,"v":1136367,"u":1226136,"t":3691802,"s":3058116,"r":3420928,"q":14380,"p":897097,"z":331215,"y":211813,"x":48411,"²":22665,"ï":7308,"í":7570,"ë":58538,"é":54151,"è":10784,"ä":6605,"á":12476,"ü":12035,"ö":12215,"ô":9257,"ó":9442," l":108039," m":271385," n":113728," o":359305," h":429647," i":863484," j":55299," k":151715," d":1022256," e":865604," f":60480," g":362770,"р":6363," a":271750," b":268848," c":85027," z":129440," u":151127," t":287237," w":287907," v":736662," p":253369," s":272741," r":136806," J":58839," K":80686," H":197094," I":88140," N":131673," O":75874," L":120300," M":156963," B":171895," C":177462," A":209281," F":122253," G":98641," D":289943," E":96010," Z":55679," Y":11046,"и":8389,"о":8805,"н":6323," S":205129," R":108525," P":136936,"а":10448," W":74232," V":87489," U":20421,"е":6745," T":105528," é":7785,"A ":9463,"Da":14497,"Cu":5824,"Cl":7287,"Co":57452,"Cr":6279,"Ce":10546,"Ch":35435,"Ci":5986,"Du":33850,"Do":13572,"Dr":5649,"De":198026,"Di":16644,"Fe":6923,"Fa":6211,"Eu":11420,"Er":8327,"En":17423,"El":7917,"Ee":19326,"Ge":21620,"Ga":13267,"I ":14576,"Fr":73223,"Fo":9924,"Fi":11740,"C ":12680,"Au":12206,"Ar":23539,"Ba":28977,"Am":55987,"An":22709,"Al":28901,"Bu":10850,"Br":34564,"Ca":38690,"Bi":11873,"Be":44722,"Bo":26862,"Bl":5807,"Kr":9527,"Ko":15418,"Le":21847,"Li":19176,"La":31976,"Lu":12150,"Lo":28450,"Me":24563,"Mi":26609,"Ma":59234,"Mu":7133,"Mo":28002,"Ni":14152,"Ne":59633,"Na":17889,"No":27858,"Ol":9544,"Gi":5965,"Gr":28322,"Go":11925,"Gu":7667,"Ha":30031,"He":95269,"II":10036,"Hi":30742,"Ho":27037,"Hu":6810,"In":31628,"Is":5913,"It":15844,"Ja":19607,"Je":8912,"Jo":19365,"Ju":6270,"Ka":21546,"Ki":7967,"Ke":10121,"Un":6926,"Tu":6976,"Tr":10760,"Ts":9146,"To":14361,"Th":19966,"Ti":10118,"Te":13194,"Ta":10688,"V ":6513,"St":35202,"Su":9985,"Wo":6733,"Wi":21345,"Wa":15248,"We":22896,"Vo":12732,"Vi":16383,"Vl":8595,"Va":15790,"Ve":22438,"Pr":15997,"S ":6945,"Pe":17479,"Pa":35566,"Po":30054,"Pi":14013,"Oo":13070,"Op":6344,"Or":9483,"Se":15433,"Sc":16133,"Si":16384,"Sh":6081,"Sl":6946,"Sp":19561,"So":16434,"Ru":16114,"Sa":42083,"Re":17106,"Ri":17272,"Rh":6524,"Ro":32707,"Ra":12538,"b ":17772,"a ":223995,"Yo":6270,"Ze":12835,"Zi":6830,"Zu":10609,"Zw":13009,"i ":117137,"gd":23906,"ge":622849,"ga":53344,"fl":6356,"fg":6573,"ff":17069,"fi":41042,"fs":14314,"fr":11929,"fu":6506,"ft":32454,"fo":24009,"j ":85100,"he":517783,"ha":140501,"gn":22691,"gl":11647,"gi":125274,"gh":17920,"gg":9680,"gu":24589,"gt":35908,"gs":36659,"gr":72480,"go":35093,"dt":42741,"du":30903,"dw":10868,"g ":209248,"ea":48028,"eb":81350,"ec":86712,"ed":198243,"de":1265409,"dd":20989,"dg":6934,"di":281697,"dh":8510,"dk":6604,"do":126396,"ds":99321,"dr":53877,"ew":31588,"ex":17157,"eu":69138,"ev":102331,"ey":15758,"ez":66338,"fa":30179,"h ":79224,"fd":18426,"fe":38002,"eh":28491,"eg":168269,"ef":50847,"ee":952339,"el":605553,"ek":125240,"ei":156611,"ep":103855,"eo":27881,"en":2016971,"em":320296,"et":590121,"es":318427,"er":1223846,"ca":46321,"e ":2100607,"bs":5841,"br":58403,"bu":44669,"bo":58794,"bl":25851,"bi":87840,"bb":9494,"be":276525,"da":122637,"f ":90198,"cu":23421,"ct":84409,"cr":11311,"co":70369,"ck":28358,"cl":14976,"ci":82756,"ch":396490,"ce":87968,"c ":16305,"az":12546,"ay":18941,"ba":76157,"d ":427738,"at":401880,"as":187038,"ar":458993,"aw":6737,"av":29958,"au":65384,"ak":138064,"al":362702,"ai":53889,"aj":7035,"ap":71362,"am":172784,"an":1131186,"ac":99694,"ad":102907,"aa":662152,"ab":30531,"ag":85506,"ah":10878,"ae":23635,"af":47867,"nu":31030,"nt":435391,"ns":284103,"nr":14733,"no":95599,"nn":67675,"nz":11234,"ny":9083,"nw":88548,"nv":13925,"oe":163750,"of":83411,"oc":46361,"od":56125,"oa":14423,"ob":29747,"om":159575,"on":522696,"ok":52068,"ol":157894,"oi":33928,"og":70046,"oh":14048,"ot":108603,"m²":22643,"os":81239,"ov":88551,"ou":131520,"op":166634,"oo":370075,"or":483257,"r ":577062,"ow":27555,"oz":6517,"oy":5912,"pe":152921,"pg":12730,"pa":116725,"pl":132966,"po":60048,"ph":12783,"pi":48220,"lo":99408,"lm":24809,"ll":128927,"ls":119057,"lp":13038,"lv":15042,"lu":37402,"lt":126691,"ly":17100,"o ":155926,"md":19425,"ma":206926,"mb":60844,"me":492457,"mi":91034,"mm":32963,"mp":51654,"ië":52728,"mo":56705,"mt":19679,"ms":35581,"mu":25483,"my":6479,"p ":141470,"na":201890,"nb":21559,"nc":69092,"nd":473575,"ne":270200,"nf":9979,"ng":259192,"nh":15116,"ni":168878,"nk":43137,"nl":16899,"nm":6930,"jv":11516,"ju":22953,"js":16655,"jn":70736,"jo":13034,"jk":115620,"ki":53710,"kh":5862,"ke":182363,"ka":118671,"m ":155936,"kw":7713,"ks":46927,"kt":131352,"ku":22530,"ko":60158,"kr":23259,"kk":20620,"kl":29172,"km":27830,"kn":6593,"li":338485,"lh":5934,"lk":34257,"le":302236,"ld":95337,"lg":36498,"lf":20122,"la":398196,"lc":8500,"lb":22338,"n ":2543334,"hr":30422,"ht":107458,"hu":34327,"hi":99792,"hn":9649,"ho":105347,"id":125660,"ic":175476,"ib":14735,"ia":95834,"ig":142126,"if":16130,"ie":477304,"k ":193827,"ir":54494,"is":722822,"it":333187,"iu":13498,"iv":35131,"iw":8456,"ix":5720,"ij":364327,"ik":93516,"il":169468,"im":40230,"in":879621,"io":144997,"ip":26974,"je":41083,"jd":31240,"jf":10910,"iz":17029,"l ":299819,"ja":41826,"xi":7194,"z ":13752,"wi":58960,"wn":7387,"wo":161887,"ws":8197,"y ":100628,"wa":155603,"wd":6545,"we":181101,"vl":34784,"ré":7748,"vi":112804,"vr":17131,"vo":180923,"uz":15014,"ux":8237,"uw":44437,"uv":7894,"uu":52115,"ve":243007,"va":525240,"x ":20927,"ui":271759,"uk":10265,"ul":55545,"ue":25713,"ug":30034,"ur":165500,"us":143221,"ut":46184,"um":51805,"un":100952,"up":10064,"ty":46646,"tz":8930,"tu":100531,"tt":45242,"tw":33429,"tv":9249,"ub":30475,"ua":33926,"ud":45500,"uc":29841,"w ":28312,"to":174226,"tm":7826,"tl":14300,"ts":211260,"tr":144811,"tg":20068,"te":770637,"tk":6386,"tj":6074,"ti":257739,"th":76245,"v ":8380,"tb":23716,"ta":261366,"su":28639,"sv":9064,"ss":138251,"st":562933,"sy":14252,"sz":5928,"sw":6865,"sl":43368,"sk":33909,"sn":9526,"sm":15035,"sp":74738,"so":59040,"sr":6378,"sd":20593,"sc":205927,"se":425364,"sh":26990,"sg":6293,"sj":16338,"si":98258,"rz":20638,"u ":30645,"sa":44187,"sb":15979,"rr":71892,"rs":228779,"rt":187734,"ru":73560,"rv":44462,"rw":18490,"ry":14720,"rp":27084,"ro":286166,"rn":60686,"né":6017,"rm":70366,"rl":117376,"rk":74650,"ri":419079,"rh":16037,"rg":84498,"rf":11818,"re":392586,"rd":257870,"rc":25722,"rb":33548,"ra":285112,"t ":1445388,"qu":12987,"s ":1176543,"pt":24055,"pu":20157,"pp":39364,"pr":99751,"ps":14478,"zi":116688,"ze":78298,"za":31109,"zu":15798,"zo":49094,"ye":6127,"yc":6275,"ya":8170,"ys":18578,"yr":9415,"yp":6946,"yn":9021,"ym":12848,"yl":8352,"² ":22661,"éé":7636,"én":13925,"ë ":43502,"é ":8856,"一":6916," Ga":13216," Ge":21536," Fo":9883," Fr":73181," Fi":11695," Ha":29997," He":95105," Go":11879," Gr":28204," Gu":7630," Gi":5928," Hu":6795," Ho":26988," II":6497," Hi":30725," Je":8879," Ja":19556," Is":5895," It":15839," In":31514," Ka":21414," Ke":10042," Ki":7899," Jo":19317," Ju":6264," La":31862," Le":21646," Li":18941," Ko":15401," Kr":9517," Ma":59034," Mi":26529," Me":24479," Lo":28411," Lu":12123," Ne":59527," Na":17815," Ni":14127," Mo":27929," Mu":7091," Am":55975," An":22679," Al":28811," Ba":28861," Au":12111," Ar":23451," Be":44588," Bi":11811," Bl":5783," Bo":26752," Br":34502," Bu":10803," Ca":38284," Ce":10523," Ci":5893," Ch":35359," Cl":7183," Cr":6209," Co":57213," Cu":5704," Da":14480," Di":16596," De":197691," Do":13329," Du":33818," El":7895," Ee":19260," Er":8309," En":17358," Eu":11409," Fe":6899," Fa":6133," Wo":6660," Wi":21285," We":22829," Wa":15180," Zu":10598," Zw":13007," Ze":12821," Zi":6778," Yo":6266," Or":9466," Oo":13036," Op":6330," Po":29977," Pi":14002," Pe":16009," Pa":35403," No":27810," Ol":9537," Ra":12376," Ro":32636," Re":17067," Ri":17238," Rh":6521," Pr":15952," Su":9973," St":34839," Ta":10649," Th":19903," Ti":10088," Te":13125," Tr":10698," Ts":9134," To":14233," Ru":16099," Sa":42032," Sh":6028," Si":16291," Sc":16056," Se":15382," So":16378," Sp":19500," Sl":6935," Va":15750," Ve":22303," Vi":16318," Vl":8585," Vo":12506," Tu":6926," Un":6865," ja":26885," in":468125," is":385652," ka":34673," ki":8773," ke":15418," ju":21208," ha":22481," he":342958," gr":38760," go":7687," hi":19625," ho":31679," hu":11200," ni":13445," ne":8341," na":55514," mu":11657," mo":19517," ok":9997," om":21439," on":84624," of":50488," nu":5724," no":28282," le":24338," li":33990," la":36262," ku":10447," km":25898," kl":14578," kr":8283," ko":24113," me":105531," mi":19475," ma":109683," lo":6645," af":19870," aa":54537," ac":15329," an":19082," ap":11762," al":65830," au":17140," ar":47665," ba":14963," bi":41311," be":166033," bo":19486," bl":6588," bu":9669," br":10013," ca":7354," e ":8358," er":11018," et":7087," en":320059," ei":19716," el":9897," ee":469095," fe":12855," fa":15977," fo":5946," fi":16445," ge":303157," ga":6353," co":32521," ce":11923," ch":6021," ci":15694," da":51374," do":72769," dr":11539," de":740739," di":128156," du":9386," zo":26839," zu":11699," za":9390," ze":21870," zi":55419," ru":5958," sa":9420," se":22896," sc":26028," si":12277," sl":6445," sp":31376," so":18785," ra":8521," re":91305," ri":13940," ro":16553," pr":57908," s ":10712," ou":7125," ov":18421," oo":33765," op":114061," or":11383," pe":20368," pa":24621," pl":113070," po":21982," wa":118659," we":90321," wo":59080," wi":19128," va":481040," ve":76969," vo":131543," vr":12217," vi":22015," vl":9586," tw":15968," tu":14723," ui":144553," ta":9836," sy":6615," st":111138," su":9964," tr":11805," to":58623," th":14839," ti":18789," te":138599," éé":7588,"Eur":9786,"Eng":13833,"Fra":63542,"II ":7320,"Hij":21364,"Het":68660,"Her":7011,"Gri":6704,"Gra":6848,"Gro":9812,"Ind":7590,"In ":14751,"Hon":6999,"Hol":6098,"Bar":6393,"Alp":5725,"Ame":44932,"Ams":6535,"Ant":6487,"Cal":8797,"Car":7946,"Ber":10327,"Bel":16135,"Bra":7962,"Bri":11166,"Bou":5646,"De ":168413,"Dez":5967,"Dee":6161,"Chi":8065,"Cen":7181,"Cha":14656,"Cor":5822,"Com":7136,"Con":6541,"Cou":26493,"Een":16908,"Dui":29012,"Ned":43974,"Nat":6561,"New":6417,"Nor":10502,"Noo":11547,"Oly":6017,"Oos":10737,"Per":5867,"Par":11966,"Poo":9123,"Ita":15181,"Joh":7769,"Lan":8583,"Man":6806,"Mar":24247,"Mon":9837,"Mid":7429,"Wil":9135,"Wer":5760,"Wes":9343,"Vla":7545,"Ze ":6085,"Sta":16762,"Sin":5858,"Spa":11230,"Rus":11905,"Sai":9817,"Sch":12674,"San":8826,"Rij":7380,"Rom":5676,"Ver":12912,"Uni":5818,"The":11210,"Tsj":8380,"bis":7734,"bin":13224,"bij":36202,"bli":10301,"bla":6463,"boe":8360,"bon":5829,"bor":10260,"bou":11481,"bbe":6915,"ban":15026,"bal":21186,"baa":7632,"bas":9398,"bar":8019,"beh":12808,"beg":7886,"bee":12385,"bed":18175,"ber":72483,"ben":9807,"bel":14825,"bek":19147,"bev":17462,"bes":52050,"bet":13589,"bie":17101,"ca ":12312,"car":7029,"cat":5698,"ce ":27218,"bri":6013,"bro":7591,"bra":10008,"bre":7181,"bru":26538,"bur":21107,"bum":8070,"am ":47277,"ake":12657,"al ":74743,"ail":5969,"ain":18706,"air":10562,"ais":6111,"ak ":11574,"agt":10140,"agn":7735,"anu":16559,"ano":8039,"ann":16248,"ant":55265,"ans":143853,"ane":9267,"ang":46635,"ani":27973,"ank":15316,"ap ":25134,"ana":20565,"anc":16651,"and":178626,"amm":8429,"amp":20053,"ams":9681,"ami":28047,"ame":31772,"amb":6788,"ama":9230,"alv":6355,"alt":35976,"als":49851,"all":30537,"ali":53705,"alc":5747,"ald":14462,"ale":38623,"ala":12213,"alb":11148,"an ":555423,"aks":6554,"akt":91851,"abe":6263,"abi":6431,"ae ":14725,"aaf":9902,"aag":19367,"aad":5831,"aak":81870,"aan":153725,"aal":41110,"aam":28835,"aas":6793,"aar":125046,"aat":183677,"ad ":41911,"afs":8757,"age":22392,"adi":11143,"ade":19021,"ag ":18180,"ach":45489,"ace":13447,"ada":5737,"af ":10774,"act":17739,"at ":119635,"are":27820,"ard":36005,"arc":10579,"arb":9145,"ara":15993,"aro":16161,"arn":8226,"arm":7787,"arl":11777,"ark":11271,"ari":63150,"arr":46599,"ars":14646,"art":89778,"au ":9748,"asi":6239,"ase":7343,"ar ":70080,"apa":7428,"app":11428,"apr":10161,"as ":104628,"aut":12900,"avi":8222,"ave":10618,"ay ":8013,"ata":7505,"ast":27312,"ass":19856,"ato":8717,"ate":41676,"ati":71342,"ath":9399,"att":6300,"ats":113866,"atu":9691,"aug":10264,"Zwe":6862,"Zui":9741,"jec":10400,"jk ":66346,"jaa":10547,"jar":7927,"jan":13106,"jd ":7717,"je ":16212,"jde":16692,"jns":7518,"js ":7214,"jn ":50205,"jks":11628,"jke":29064,"itt":7963,"its":47572,"ity":14389,"iss":57247,"ist":78828,"ita":16110,"ite":28930,"itg":14085,"iti":26080,"ium":6105,"ivi":15930,"ive":12826,"is ":431822,"ion":45782,"ir ":7395,"isi":11995,"ish":6473,"ise":13845,"isc":89426,"isa":6882,"ire":19679,"it ":148319,"iwo":7073,"ize":7008,"kin":23405,"ki ":12463,"kee":5719,"kel":22383,"ken":67439,"ker":29057,"ke ":43164,"kt ":89985,"kse":13533,"kri":8432,"km²":22590,"kor":8329,"kon":7670,"kom":21374,"ks ":13043,"kke":14372,"kle":13907,"kla":8538,"jul":9669,"jun":10013,"jve":8311,"jst":5685,"kan":26234,"kam":11248,"kaa":47537,"ka ":11569,"ham":9052,"han":20214,"hap":31501,"hal":11321,"har":17692,"haa":16281,"had":6694,"he ":78504,"hel":11001,"hei":31175,"hee":28005,"het":305470,"her":18043,"hen":9035,"hem":8872,"hie":14941,"hin":11368,"hil":13952,"hij":15137,"his":12860,"gne":9405,"gna":8227,"gon":7132,"gsd":8056,"gro":30942,"gra":24600,"gt ":25727,"gri":8059,"gre":7606,"gst":8398,"gus":11645,"ial":9076,"ian":13694,"iat":6807,"iaa":21077,"id ":39722,"ia ":31791,"iet":21274,"ieu":11845,"iev":7556,"iel":12150,"iem":5824,"ien":50882,"ier":48258,"ies":24371,"ied":23758,"ief":9097,"iek":35699,"ig ":26250,"ict":38543,"icu":11018,"ico":6343,"ici":9264,"ich":59319,"ice":15194,"ie ":201388,"ica":19828,"idi":13237,"ide":28170,"idd":13665,"ida":12788,"il ":19641,"ijd":30969,"ije":8022,"ijf":10866,"ijk":115115,"ijn":70104,"ijs":16040,"ijv":11367,"im ":7764,"ika":50912,"igd":11510,"ige":49060,"igh":9499,"igi":9202,"igt":11964,"ign":9284,"ij ":80452,"ik ":10619,"ime":7803,"inc":33586,"ind":41153,"ina":22287,"inn":14337,"ino":8487,"int":30127,"ins":19772,"ine":33694,"ing":150853,"ini":14846,"ink":10381,"ioe":9616,"inw":77340,"ikk":7357,"ike":6423,"ila":17614,"in ":405294,"ikt":10786,"ilo":7646,"ill":44186,"ilm":12944,"ili":39663,"ild":8065,"ile":5842,"ima":5906,"io ":71745,"hol":9679,"hou":16715,"hoo":29440,"hor":9181,"hoe":5979,"hui":10622,"hts":6548,"hth":9745,"hti":8381,"hte":21726,"hre":7796,"hri":17051,"ht ":49854,"hum":5764,"ffe":6190,"ffi":6818,"feb":9777,"fen":6207,"fam":16628,"fde":6413,"eze":22604,"ezi":32218,"etb":16830,"eta":11379,"ete":33498,"eti":8945,"eth":6048,"esp":11543,"est":111009,"ess":14653,"etr":9667,"ets":6968,"ett":13669,"ew ":6999,"eve":54896,"eva":7494,"evo":21109,"evi":14484,"euw":17939,"eur":21284,"ewe":9375,"ewo":5810,"ey ":9860,"epe":6552,"er ":333095,"epa":50447,"eor":9695,"es ":93024,"ept":12657,"erk":43542,"erl":61155,"eri":98353,"erg":28574,"erh":11977,"ere":86052,"erd":117197,"era":27762,"erb":15858,"et ":468269,"esl":15451,"esi":12100,"esc":18281,"ese":16953,"erz":13202,"erv":34773,"erw":12350,"err":17024,"ert":36463,"ers":189637,"ern":29282,"erm":20928,"erp":10525,"ero":16315,"eks":13762,"ekt":9522,"en ":1373884,"ela":20546,"eld":60315,"elf":9763,"ele":68456,"eli":59882,"elg":17311,"elk":7255,"ell":29565,"elo":7411,"els":48151,"elt":76311,"emb":35339,"ema":13115,"eme":213429,"emd":11991,"emi":9436,"ep ":11271,"ene":28524,"enh":7989,"eng":11580,"enb":15906,"ena":20359,"end":77370,"enc":8130,"eno":20895,"enn":18048,"enk":9463,"enl":7298,"eni":29460,"enw":7353,"ens":81592,"ent":262064,"enr":11987,"ego":6580,"ege":50512,"egi":74199,"eho":15902,"ek ":28609,"eis":7935,"eil":21113,"ein":30892,"eid":39967,"eig":6260,"el ":169543,"eiz":7508,"eit":10871,"eke":48960,"em ":17335,"gis":18136,"gin":20458,"gio":63608,"gie":9618,"gge":6516,"gep":6253,"gen":119618,"geo":6626,"get":5657,"ger":43727,"ges":41787,"gev":26798,"gew":8731,"gez":31827,"gee":9960,"ged":10354,"geb":49406,"geh":10722,"gem":117467,"gel":56245,"gek":5735,"gde":10792,"ge ":59115,"gd ":7871,"gaa":11487,"gan":14683,"ft ":25649,"for":15464,"fic":10500,"fie":5840,"fil":14418,"da ":11415,"de ":684651,"daa":13978,"dag":8773,"dae":7793,"dat":38507,"dan":11068,"dam":12288,"dde":16386,"cti":18928,"cte":10141,"cus":11312,"clu":9316,"co ":6846,"con":15186,"com":22591,"ct ":40340,"cea":6367,"ch ":51967,"ces":6110,"cen":17525,"cem":10124,"cha":49925,"cia":10705,"ck ":10471,"cie":34119,"che":90373,"chi":44310,"cho":18169,"cht":101737,"chr":21384,"cit":10702,"ed ":30096,"eba":5684,"ebe":6342,"ebi":14653,"ebo":12775,"ebr":30709,"eau":7092,"ei ":13562,"eft":21131,"eek":11355,"een":586471,"eel":149087,"eem":7130,"eef":23903,"eed":19633,"ees":22270,"eer":87604,"eeu":10006,"eet":5732,"edi":16531,"ede":93412,"eda":9229,"eg ":11274,"eds":11135,"edo":6698,"edr":18455,"ech":31918,"eci":6720,"ece":11419,"ee ":19989,"ef ":10385,"ect":16877,"eco":6380,"dwe":6219,"dor":15099,"doo":63254,"don":8072,"dom":7397,"ds ":31717,"dië":7942,"doc":6254,"doe":8162,"dst":15322,"dui":6443,"duc":8131,"dri":17764,"dra":19458,"dt ":39781,"dro":6043,"dsc":11185,"dse":26450,"dic":13497,"dia":8442,"der":192741,"des":13390,"dez":10434,"dec":12054,"dee":111320,"del":33701,"den":135340,"dep":43953,"di ":6337,"do ":6228,"din":21044,"dio":7262,"dis":79203,"dit":10793,"die":85548,"dig":22881,"rha":6318,"rga":14305,"ri ":27658,"rgi":6243,"rge":20743,"ret":8087,"res":26172,"rev":8098,"rdt":29462,"rg ":25954,"rea":7679,"ree":20672,"rec":16838,"red":11064,"rei":15261,"reg":76191,"ren":91873,"rek":7399,"rel":18716,"rda":12870,"rdo":6848,"rdi":16416,"rde":80930,"re ":58121,"rch":10326,"rd ":94954,"ras":7988,"rat":17581,"rbi":10377,"rbe":8163,"rag":7326,"ran":87841,"ram":11943,"ral":17891,"raa":43729,"rad":10061,"rac":17496,"rs ":114967,"rpe":6605,"ros":6652,"rot":12676,"rom":9424,"ron":74395,"roo":21289,"rop":13868,"rou":11473,"rov":34172,"rod":10405,"roc":11802,"rol":11209,"roe":27621,"rog":7078,"rno":7457,"rp ":10130,"rna":17955,"rne":14489,"rni":6410,"rmo":6164,"ro ":8067,"rma":25601,"rme":14051,"rlo":10187,"rli":43352,"rle":10232,"rla":47955,"rn ":7785,"rko":8091,"rke":15492,"rm ":8899,"rip":6607,"rio":7162,"rit":17759,"ris":26171,"riv":7770,"rig":10543,"rij":60217,"ril":12296,"rik":53920,"rin":39022,"ria":15453,"ric":60714,"rid":8288,"rie":42534,"rk ":23012,"rwe":7689,"rwi":6024,"rui":23623,"rug":6639,"rum":7291,"rus":7171,"rva":8120,"rve":6455,"rvl":18128,"rvo":6709,"ry ":9896,"rsi":7125,"rso":10142,"rsp":13174,"rsc":16580,"rse":16920,"rta":7455,"rst":29072,"rto":9036,"rte":64198,"rth":8072,"rti":18904,"rua":9937,"rts":6565,"rt ":52830,"rro":41654,"rri":6371,"rre":14498,"sam":7637,"sat":5699,"shi":8681,"sje":10417,"sie":18702,"sit":8909,"sis":17171,"sin":18433,"sig":6448,"sdi":9330,"se ":245903,"sch":188901,"sco":5845,"ser":23257,"sh ":6367,"sei":5692,"see":12427,"sep":12185,"sen":46205,"sem":40984,"sel":17905,"spo":12356,"spr":14426,"spe":34342,"spi":6360,"son":16537,"soo":14382,"st ":68174,"sla":28873,"ski":9665,"ske":6724,"sme":5826,"sse":83731,"ssa":7327,"sso":6523,"ssi":25931,"ste":153650,"sta":134037,"sto":24680,"sti":36773,"stu":49771,"str":76216,"sus":9037,"tai":6439,"tal":42563,"taa":79421,"tad":33209,"tba":18995,"tat":25594,"tar":9576,"tan":24920,"te ":245946,"ta ":14225,"pe ":6304,"par":65094,"paa":13416,"pan":11481,"pge":11800,"pec":7014,"pen":27399,"per":54393,"pes":8975,"pee":13839,"pel":26450,"pla":118636,"ple":6523,"pij":6479,"pio":11106,"pis":8712,"por":10136,"poo":7190,"pon":6968,"pol":16543,"ppe":30477,"pub":6405,"pte":13585,"pri":21984,"pre":14246,"pro":59321,"que":5834,"ra ":17662,"ngr":7754,"ngt":6149,"ngs":24028,"ni ":14308,"nge":85205,"nga":9441,"nha":5738,"nel":9354,"nen":38308,"nem":7134,"ner":92669,"net":10080,"nes":17246,"ng ":100478,"nee":12320,"nci":34445,"nce":13701,"nch":8196,"ne ":58920,"nbu":5898,"ndr":9897,"nds":53318,"ndo":10563,"ndi":65506,"nde":170234,"nda":12836,"nal":19401,"nam":19197,"nan":5995,"nad":6211,"naa":50268,"nbe":7893,"nd ":125333,"nat":22785,"na ":37268,"nwo":81256,"ny ":6026,"num":6116,"nua":11671,"nty":25239,"ntw":10449,"nto":19506,"nts":8743,"ntr":16443,"nti":22690,"nta":19603,"nte":152583,"nsu":9180,"nst":31383,"nse":133099,"nsc":14189,"nri":8599,"nt ":140657,"ns ":66627,"noe":11549,"noo":19104,"nom":9568,"nov":12875,"nne":42144,"nna":5969,"nni":9362,"nië":12014,"nla":8754,"no ":10798,"nke":14949,"nkr":6212,"nig":15471,"nie":28378,"nic":7027,"nia":7786,"nk ":5831,"niv":5902,"nis":35078,"nin":21782,"ogr":8544,"ogi":8951,"oge":15686,"oiw":7018,"ois":5765,"oir":6590,"ok ":26496,"ol ":12318,"och":15236,"ock":8061,"ode":16327,"ods":9122,"of ":51317,"oek":14333,"oel":8698,"oem":15572,"oeg":10025,"oer":17651,"oet":21912,"oen":24849,"oep":14118,"odu":7865,"oed":12901,"og ":18782,"off":7132,"ofd":11478,"oal":5868,"od ":7190,"obe":14031,"own":6969,"oud":23948,"oth":6889,"ote":16152,"ott":8284,"ots":8865,"oto":6753,"ost":29293,"ota":7234,"ose":6693,"ovi":34676,"ouw":19904,"ove":41666,"oun":29232,"ous":5891,"our":16621,"out":8877,"opp":20304,"ope":17398,"opg":12331,"os ":12938,"oon":17563,"ool":15370,"oom":10898,"ook":25159,"ooi":6071,"oof":13207,"oog":15636,"ood":7845,"or ":123656,"oot":15244,"oos":12524,"oor":222103,"oop":6797,"ork":12244,"orl":12130,"orm":37127,"orn":13425,"orp":13450,"ord":94351,"ore":21033,"org":20784,"ori":18821,"ou ":6777,"ort":47654,"ors":16087,"ot ":44568,"m² ":22639,"ora":14333,"ola":5947,"old":6867,"on ":90823,"oli":23091,"oll":13695,"olk":19645,"ole":13281,"olg":11475,"ols":13292,"olo":16700,"om ":28845,"okt":10174,"ona":29547,"ond":144852,"one":97393,"ong":27845,"oni":32623,"onn":9985,"ono":10092,"ons":14560,"ont":39227,"oma":10713,"ome":31512,"omb":6304,"omi":12146,"omm":12297,"omp":15805,"omt":12605,"oms":11302,"op ":79508,"la ":16552,"le ":71663,"lf ":5640,"lde":29685,"laa":121868,"lac":19544,"lad":5854,"lag":14890,"lai":7568,"lak":20347,"lan":133760,"lar":5948,"lat":13164,"las":13782,"ld ":39055,"lbu":9341,"kun":16374,"kwa":5842,"kte":24985,"kst":6852,"kto":10642,"lpe":6305,"ls ":61450,"lon":9738,"lom":7994,"loo":10759,"lor":5860,"loe":7778,"log":16741,"los":5784,"lië":7455,"lub":9599,"lst":25217,"lte":6994,"lse":20318,"lt ":100197,"lge":14115,"lgi":13560,"li ":13675,"lev":15162,"les":15192,"let":7945,"ler":20832,"lem":10140,"len":59840,"lei":20020,"leg":19493,"lee":17156,"led":8570,"lec":6605,"lo ":6322,"lla":18593,"lle":62567,"lli":22272,"llo":6485,"lks":5894,"lki":11362,"lm ":10134,"ll ":9099,"lit":19124,"lis":19817,"lip":6344,"lin":41169,"lic":12334,"lia":22468,"lij":95486,"lig":34482,"lie":40870,"ma ":13279,"maa":102890,"mar":12201,"mal":14307,"man":28303,"mat":12974,"md ":13491,"mbe":34190,"me ":20464,"med":7669,"mee":138208,"met":69732,"mes":6603,"mer":72670,"mel":11918,"men":146532,"mei":13406,"lve":5791,"lym":7416,"mpi":18269,"mpe":6753,"mpo":7375,"ms ":12121,"moe":5824,"mon":14109,"mt ":13658,"mst":14316,"muz":8520,"min":20688,"mil":24349,"mis":8333,"mit":6248,"mig":6106,"mie":5852,"mid":8678,"ië ":43280,"mma":6643,"mme":17999,"zui":10334,"zee":6519,"zet":5935,"zen":14435,"zel":8377,"ze ":24809,"zan":8216,"zoo":6798,"zon":12147,"zoe":7835,"zie":38234,"zic":12949,"zij":41913,"yst":6749,"ys ":5784,"ymp":7552,"wn ":6464,"wod":7070,"wor":46676,"woo":12376,"won":83488,"woi":6978,"wes":10367,"wer":76690,"wet":5976,"wen":7541,"wel":18559,"weg":14286,"wee":29661,"wit":7604,"win":7741,"wij":18720,"wat":9640,"war":13183,"was":80249,"waa":29854,"vro":6819,"vil":13878,"vin":41497,"vie":14100,"vis":16850,"vla":21113,"vli":6207,"voe":25269,"vol":30369,"von":6589,"voo":92340,"vor":15339,"ver":131519,"ven":47801,"vem":9979,"vel":12213,"vee":14817,"ve ":13676,"val":33641,"van":455616,"vat":5892,"vaa":8390,"uzi":9073,"uwe":13657,"uwd":6413,"uur":48711,"usi":5945,"use":7901,"ust":24814,"uss":30621,"ute":13524,"uw ":14870,"uto":6696,"us ":63461,"ut ":6738,"ure":14305,"urg":25394,"uri":10844,"url":29792,"uro":10683,"urt":8342,"ur ":31687,"umb":6654,"unt":32613,"uns":8285,"uni":16955,"und":14286,"um ":25713,"ult":7652,"uli":14819,"ule":6125,"ula":6204,"un ":7353,"uid":29632,"uik":15052,"uis":15533,"uit":185570,"ugu":11498,"ude":20442,"udi":6485,"ue ":5767,"uch":9124,"ub ":8370,"uar":23535,"ubl":7698,"ty ":41728,"tur":6970,"tus":27504,"tuu":41013,"tud":6157,"twi":6118,"twe":21595,"ts ":120548,"tre":20553,"tra":31398,"tri":58554,"tru":9236,"tro":22667,"tse":38669,"tsc":8839,"tst":17796,"tte":25599,"to ":11397,"toe":15554,"tob":10290,"tot":36145,"tow":6284,"ton":34234,"tor":20776,"tij":29417,"tie":72965,"tig":17350,"tit":9886,"tis":23784,"tin":21180,"tio":32273,"thu":8037,"tic":17261,"tle":5983,"tem":61681,"ten":116036,"tei":13819,"tek":11255,"tel":112278,"tee":16372,"teg":10988,"ted":9251,"th ":11037,"teu":8413,"tes":8744,"ter":133625,"tge":17358,"tho":11376,"the":28515,"tha":9247,"én ":8357,"één":7621},"n_words":[56157687,65372177,47614417],"name":"nl","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/no.json b/contrib/languages-data/no.json
new file mode 100644
index 0000000..757fc8e
--- /dev/null
+++ b/contrib/languages-data/no.json
@@ -0,0 +1 @@
+{"freq":{"D":67298,"E":31132,"F":46085,"G":30699,"A":51244,"B":53160,"C":43132,"L":37752,"M":50816,"N":44173,"O":29040,"H":46630,"I":28485,"J":19280,"K":42898,"U":16316,"T":46770,"W":15084,"V":29381,"Q":2727,"P":33476,"S":98250,"R":33889,"Y":5384,"X":3613,"Z":4254,"f":354544,"g":680290,"d":751947,"e":2892960,"b":291896,"c":77603,"a":1296256,"n":1640496,"o":1106000,"l":1037936,"m":608425,"j":148253,"k":716879,"h":226424,"i":1317183,"w":19861,"v":434834,"u":407647,"t":1430364,"s":1268259,"r":1753294,"q":3203,"p":350711,"z":18142,"y":203712,"x":10440,"²":2235,"Ã…":2838,"Ø":5129,"í":2375,"é":5770,"æ":27321,"Ã¥":157055,"ä":7866,"á":3631,"à":2467,"ü":2355,"ø":174400,"ö":5464,"ó":2058," l":79574," m":127350," n":66294," o":215607," h":91447," i":286253," j":14786," k":126841," d":163994," e":389283," f":217023," g":55928,"Ñ€":2932," a":164625,"Ñ":2244," b":141242," c":8452," u":55940," t":125598," v":100006," p":120765," s":286186," r":49569," J":19242," K":42735," H":46471," I":28413," N":43960," O":28901," L":37540," M":50591," B":52872," C":42604," A":51010," F":45901," G":30503," D":67077," E":31023,"л":2294," Z":4215,"к":2313," Y":5370," X":3570,"и":3772,"о":4246,"н":2997," S":97781,"в":2674," R":33770," Q":2713,"а":4944," P":33276," W":14969," V":29296,"е":3204," U":16274," T":46489," Ã¥":24460," ø":14695," Ã…":2834," Ø":5122,"A ":7964,"Da":7415,"Co":15888,"Ch":7594,"Do":3375,"Dr":2174,"De":44977,"Di":3865,"Fe":2733,"Fa":3197,"Eu":2611,"Et":2524,"En":8728,"El":2986,"Ge":3683,"Ga":4272,"I ":9052,"Fy":2932,"Fr":8731,"Fo":9701,"Fl":2786,"Fj":2610,"Fi":5958,"C ":3097,"Au":3599,"Ar":6717,"As":3221,"Ba":9331,"Am":2967,"An":7679,"Al":7567,"By":3184,"Bu":3866,"Br":7415,"Ca":9043,"Bi":3113,"Be":11885,"Bo":6452,"Bl":2411,"Ku":2113,"Kr":5137,"Ko":7893,"Le":5322,"Li":8435,"La":9531,"Lu":3278,"Lo":6840,"Me":7383,"Mi":7116,"Ma":15908,"Mu":2812,"Mo":8218,"Ni":3496,"Ne":6662,"Na":6162,"No":20842,"Ol":2369,"Gr":7570,"Go":2542,"Gu":4480,"Ha":15522,"He":10062,"Hi":2665,"Ho":7455,"Hu":4099,"In":7421,"Is":2952,"Ir":2049,"Ja":4954,"L ":8381,"Je":2911,"Jo":5435,"Ju":2288,"Ka":7574,"M ":2289,"Ki":8846,"Ke":2144,"Un":4807,"W ":3224,"Ty":3546,"Tr":8937,"To":6250,"Th":6517,"Ti":3470,"Te":5634,"Ta":4262,"V ":2848,"Sy":2174,"St":18668,"Sv":4781,"Su":4159,"Wi":3769,"Wa":4239,"We":2978,"Vi":7414,"Va":4893,"Ve":9972,"Pr":5274,"S ":4165,"Pe":5166,"Pa":8226,"Po":5247,"Pi":2410,"Os":6058,"Op":2483,"Or":3270,"Se":7511,"Sc":2844,"Si":5113,"Sh":4054,"Sl":2066,"Sk":6894,"Sp":4048,"So":11528,"Ru":4355,"Sa":11798,"Re":6477,"Ri":4763,"Ro":9292,"Ra":4607,"b ":4101,"a ":140249,"Yo":2095,"Sø":4628,"Xi":2231,"bø":2634,"i ":283647,"fy":8788,"gd":9301,"ge":149531,"ga":32513,"fj":12532,"fl":14388,"ff":8428,"bÃ¥":3627,"fi":25866,"fr":47803,"fu":8120,"ft":23386,"fo":101580,"j ":2796,"gy":3318,"he":44416,"ha":66151,"gn":19968,"gl":10219,"gj":12645,"gi":33050,"gh":8862,"gg":35921,"gv":2904,"gu":9872,"gt":8325,"gs":33529,"gr":39232,"go":8838,"dt":33464,"du":13333,"dv":5679,"dy":4144,"g ":242136,"ea":17194,"eb":16045,"ec":4860,"ed":108322,"de":299418,"dd":13784,"df":2274,"di":40822,"dh":3225,"dk":3004,"dm":5071,"dl":19375,"do":17360,"dn":4555,"ds":31560,"dr":31419,"ew":4011,"ex":3115,"eu":5509,"ev":23237,"ey":5833,"fa":23302,"h ":12462,"fe":26099,"eh":7528,"eg":49132,"ef":18320,"ee":8701,"el":199057,"ek":50272,"ei":44782,"ep":20746,"eo":9834,"en":582197,"em":46141,"et":320684,"es":146652,"er":695279,"ca":10711,"e ":500246,"by":34947,"br":29081,"bu":14609,"bo":17004,"bl":54931,"bi":19197,"bb":5548,"be":63706,"db":3584,"da":47371,"f ":9303,"cu":2103,"ct":3488,"co":8095,"ck":10053,"ci":6056,"ch":18251,"ce":8996,"c ":3649,"az":2995,"ay":5287,"ba":35656,"d ":150657,"at":91048,"as":69325,"ar":172213,"av":121888,"au":14940,"ak":30193,"al":135078,"ai":12693,"aj":2677,"ao":2477,"ap":29140,"am":63029,"an":267240,"ac":9777,"ad":37337,"aa":2792,"ab":12532,"ag":40280,"ah":4661,"ae":7154,"af":16520,"nu":7442,"nt":92694,"ns":124075,"nr":3026,"no":52873,"nn":98330,"jø":13856,"ny":13664,"nv":3459,"oe":7618,"of":17045,"oc":8436,"od":26153,"oa":3647,"ob":11128,"om":188777,"on":123342,"ok":27219,"ol":75500,"oi":3014,"og":163889,"oh":4581,"ot":32564,"m²":2226,"os":28900,"ov":41178,"ou":20519,"op":34634,"oo":5024,"or":234371,"r ":604948,"ow":4407,"oy":2125,"pe":55903,"pa":27179,"pl":13813,"pn":2118,"po":25761,"ph":3721,"lä":2860,"pi":29184,"lÃ¥":4732,"læ":3598,"lo":47607,"lm":11687,"ll":125327,"ls":54920,"lp":4961,"lv":17299,"lu":16394,"lt":42371,"ly":13905,"hø":10888,"o ":38269,"ma":52812,"mb":14541,"me":153343,"mf":5780,"mk":2245,"ml":5863,"mi":35740,"mn":2923,"mm":59754,"mp":15073,"mo":24316,"mr":8156,"mt":8238,"ms":13233,"mu":35413,"my":4118,"p ":18651,"na":63523,"nb":10179,"nc":6842,"nd":155382,"ne":210727,"nf":8666,"ng":145804,"nh":6700,"ni":70736,"nj":3359,"nk":18937,"nl":14559,"nm":3689,"ju":8303,"jo":49289,"kj":21128,"gÃ¥":9251,"ki":30043,"kh":3599,"ke":149522,"ka":75120,"m ":148254,"fø":39544,"ky":6546,"ks":31928,"kt":45246,"ku":18940,"kv":7398,"ko":71373,"kr":37123,"kk":42089,"kl":19159,"km":10210,"kn":9496,"li":148110,"lh":5181,"lk":27090,"lj":4011,"le":206454,"ld":27006,"lg":10977,"lf":6606,"la":117768,"lb":9872,"n ":508629,"hr":3491,"hv":8983,"ht":2318,"hu":13547,"hj":3760,"hi":16514,"hn":2241,"ho":28328,"id":53829,"ic":16512,"ib":6764,"ia":42013,"ig":91825,"if":12391,"ie":49167,"hy":2076,"k ":129587,"dø":14496,"ir":32357,"is":145060,"it":66902,"iu":3847,"iv":28029,"ii":2629,"ij":2484,"ik":79231,"il":114997,"im":14324,"in":224355,"io":25793,"ip":8856,"je":54644,"ji":3380,"l ":114552,"ja":10997,"xi":3017,"tÃ¥":6348,"z ":3331,"sø":16304,"sÃ¥":13214,"wi":2306,"y ":43971,"rø":14616,"wa":4985,"we":2854,"vh":2078,"rÃ¥":14698,"vi":56505,"vt":2733,"vs":5579,"vn":14394,"vo":12525,"ve":142305,"vd":3441,"va":71132,"x ":3406,"ui":4966,"uk":21664,"ul":26959,"ue":11327,"uf":3039,"ug":10093,"ur":41090,"us":50014,"ut":47498,"um":19336,"un":102304,"up":9818,"ty":27626,"tu":28284,"tt":93499,"tv":14967,"ub":13117,"ua":11212,"ud":11995,"uc":2853,"w ":4541,"to":63371,"tn":11784,"tm":2957,"tl":16216,"ts":42441,"tr":65044,"tg":7470,"tf":6385,"te":283104,"td":2594,"tk":2587,"tj":4066,"ti":143857,"pÃ¥":54079,"th":14355,"v ":99425,"tb":9067,"ta":95087,"su":13370,"sv":19448,"ss":49933,"st":245546,"sy":11979,"sl":28878,"sk":191864,"sn":6441,"sm":11084,"sp":37365,"so":108796,"sr":4235,"sd":5831,"sc":5435,"sf":8108,"se":148118,"sh":13228,"sg":2307,"sj":47719,"si":74309,"nø":2380,"u ":10252,"sa":44310,"sb":10119,"rr":19009,"rs":78883,"rt":82606,"ru":50990,"rv":11610,"ry":11156,"rp":5076,"ro":69758,"rn":31836,"rm":21775,"rl":20062,"rk":47716,"næ":4051,"ri":132605,"nÃ¥":4464,"rh":6919,"rg":35522,"rf":12509,"re":256059,"rd":66008,"rc":4262,"rb":18116,"ra":128776,"t ":465642,"mø":2258,"qu":2140,"mÃ¥":7691,"lø":8427,"s ":146195,"pt":7260,"pu":11071,"pp":34390,"pr":51296,"ps":7773,"vÃ¥":3732,"za":2150,"væ":7493,"yg":16794,"ye":16616,"yd":10423,"ya":7947,"tø":14974,"yt":13580,"ys":21183,"yr":13910,"yp":7213,"yn":9715,"ym":5956,"yl":14093,"yk":8767,"yi":4209,"² ":2226,"Ã¥ ":86331,"Øs":2809,"ær":24440,"Ã¥p":3244,"Ã¥n":3530,"Ã¥t":7047,"Ã¥s":2737,"Ã¥r":24111,"Ã¥e":2573,"Ã¥l":6435,"Ã¥k":4798,"Ã¥d":11397,"än":3083,"øy":21642,"ør":54354,"øs":14419,"øp":5028,"øv":7903,"øt":3062,"øk":5634,"øn":8821,"øl":4965,"øm":4740,"øe":2478,"ød":34288,"ø ":2765,"之":2379,"专":2738,"三":3026," Ga":4247," Ge":3655," Fy":2931," I ":6572," Fo":9668," Fr":8721," Fi":5928," Fl":2776," Fj":2609," Ha":15503," He":10039," Go":2524," Gr":7534," Gu":4458," Hu":4096," Ho":7447," Hi":2663," Je":2895," Ja":4943," Is":2884," In":7378," Ka":7550," Ke":2113," Ki":8813," Jo":5426," Ju":2284," La":9489," Le":5267," Li":8383," Ko":7880," Kr":5128," Ku":2108," Ma":15819," Mi":7093," Me":7355," Lo":6813," Lu":3268," Ne":6630," Na":6059," Ni":3491," Mo":8183," Mu":2787," Am":2962," An":7665," Al":7546," Ba":9272," Au":3596," As":2828," Ar":6669," Be":11852," Bi":3097," Bl":2409," Bo":6415," Br":7392," Bu":3840," By":3181," Ca":8810," Ch":7563," Co":15800," Da":7402," Di":3846," De":44915," Dr":2166," Do":3289," El":2979," Et":2521," En":8708," Eu":2608," Fe":2724," Fa":3178," Xi":2228," Sø":4627," Wi":3744," We":2962," Wa":4211," Yo":2085," Os":5937," Or":3262," Op":2481," Po":5205," Pi":2406," Pe":5104," Pa":8187," No":20722," Ol":2367," Ra":4586," Ro":9267," Re":6462," Ri":4754," Pr":5244," Sy":2169," Sv":4778," Su":4150," St":18500," Ta":4246," Th":6493," Ti":3455," Te":5588," Tr":8905," To":6181," Ru":4349," Sa":11773," Sh":4026," Si":5095," Sc":2823," Se":7484," So":11504," Sp":4034," Sk":6885," Sl":2061," Va":4882," Ve":9942," Vi":7397," Ty":3541," Un":4794," ja":4722," in":34065," ik":4460," is":2212," ka":20700," fø":33238," kj":12239," gÃ¥":5771," ki":10103," jo":2540," ju":4702," ha":40329," he":11962," gi":4943," gj":7272," gr":19932," gu":2109," dø":7855," hi":3888," hj":2877," ho":10427," hu":4249," hv":8245," ne":7120," na":12532," my":2654," mu":6899," mo":13920," ok":2382," ol":3868," om":17822," og":141787," of":9223," ob":2162," ny":3021," no":33974," le":11218," li":27765," la":25410," kv":4363," ku":8567," km":9490," kl":4277," kr":10076," ko":42531," me":70036," mi":10095," hø":6629," ma":17890," lu":2098," lo":3520," ad":3981," am":9313," an":19228," ap":2222," ak":3076," al":9858," av":89135," au":3693," ar":13091," at":7530," ba":13278," bi":6342," be":29509," bo":6968," bl":40680," by":21015," bu":2338," br":16487," ca":3932," er":153435," et":63825," en":123670," ei":4836," el":30282," ek":4165," eg":3293," fe":8817," fa":9591," fu":3699," fr":43218," fo":78205," fl":9728," fj":6013," bÃ¥":2550," fi":13808," ge":3067," ga":9278," i ":237285," fy":7547," da":14064," do":2621," dr":4255," de":122624," di":7830," dy":2072," væ":4635," sø":12596," ru":9067," sa":19419," se":29156," sj":3216," si":17713," sl":7113," sk":21745," sp":19430," so":87507," ra":3039," re":25505," ri":2962," nÃ¥":3726," næ":2896," ro":6266," pu":2169," pr":27989," lø":2099," mÃ¥":3587," ov":11878," op":17267," or":7497," pe":8091," pa":7750," pl":5274," po":9080," pi":4926," lä":2565," sÃ¥":2458," va":41741," ve":33600," vo":3210," vi":14710," ty":6900," tu":3382," ut":27318," un":21502," ta":8589," sy":5687," st":43167," sv":7377," su":2425," tr":14296," to":9228," th":2455," pÃ¥":53815," ti":62638," te":14306," Øs":2807," Ã¥ ":15026," Ã¥r":6691," øs":6280," øy":4235,"Fin":2794,"Eur":2192,"En ":4024,"Eng":3095,"Fyl":2572,"Fra":3815,"Fre":2185,"Fol":3808,"For":3758,"Hel":2604,"Han":5694,"Har":2577,"Ind":2121,"Øst":2801,"Bar":2295,"And":2152,"Car":2172,"Ber":4934,"De ":3390,"Det":18734,"Den":19203,"Dan":2408,"Cha":2366,"Cou":8345,"OL ":7076,"New":2594,"Nor":18501,"Osl":4630,"Par":3554,"Pro":2168,"SA ":4753,"Joh":2414,"Kar":2117,"Kin":6314,"Kon":2546,"Kom":2130,"Kri":2217,"Lan":3221,"MW ":2903,"Lon":2680,"Man":2484,"Mar":5164,"Sør":3934,"Sve":3734,"Str":2300,"Sto":5079,"Sta":5059,"Ste":2680,"Sko":2364,"Som":4932,"Rus":2959,"San":3826,"än ":2530,"Rom":2726,"Ã¥de":9646,"Ã¥l ":2493,"Vei":2618,"Ã¥le":2191,"Vin":2983,"Ã¥re":3629,"Ã¥rd":2572,"Ã¥r ":14295,"Ves":3148,"Ã¥tt":3275,"Uni":3264,"Tys":2479,"ær ":4620,"ært":3809,"ære":8422,"The":4012,"Tro":3352,"bis":3511,"bil":4902,"bin":5018,"blo":2551,"ble":31774,"bli":12888,"bla":6396,"bok":3234,"bor":4144,"bbe":3093,"ban":9995,"bak":3210,"bal":7414,"bas":4309,"bar":6636,"bei":5432,"beg":3804,"ber":17512,"ben":4646,"bel":3914,"bev":2185,"bes":8730,"bet":9507,"ca ":4720,"ce ":3774,"bri":5623,"bra":2313,"bre":4128,"bru":13492,"bur":2443,"bun":2536,"bum":3444,"by ":8318,"byd":2738,"bye":5451,"byg":13374,"byp":2127,"am ":5629,"ake":6088,"al ":22413,"ain":3128,"ak ":2688,"agt":4288,"anu":3347,"ano":2720,"ann":21260,"ant":16537,"ans":34997,"ane":10156,"ang":34028,"ani":10694,"ank":5395,"anl":5003,"ap ":5392,"ana":7215,"anc":2796,"and":71633,"amt":3120,"amm":13306,"aml":4533,"amp":3404,"ami":6970,"ame":12312,"amb":2328,"ama":4584,"alv":2901,"alt":13338,"als":4480,"all":28689,"alg":3345,"ali":12960,"ald":4724,"ale":19089,"ala":6862,"alb":4116,"an ":35027,"aks":3242,"akt":7856,"akk":3187,"abl":2978,"ae ":2689,"ad ":7446,"aft":9172,"afi":2584,"ai ":3427,"aga":2271,"age":11424,"adm":2981,"adi":6411,"add":5083,"ade":7303,"ag ":10292,"ach":2606,"ada":2399,"at ":12379,"are":11421,"ard":7601,"arb":5329,"ara":7138,"arn":4304,"arm":4027,"arl":3706,"ark":12605,"ari":9703,"arr":4082,"ars":6026,"art":19222,"asi":5334,"ase":4616,"ask":2682,"asj":20288,"ar ":65150,"apa":4754,"ape":8916,"app":2255,"as ":8910,"ava":2465,"aut":2096,"avn":11841,"avi":3695,"ave":7468,"ay ":2288,"av ":85297,"ata":4831,"ast":9209,"ass":12223,"ato":4866,"ate":21584,"ati":14442,"att":15952,"ats":2979,"atu":5294,"aug":3319,"jer":9769,"jek":2468,"jel":9659,"jem":3205,"jen":18638,"jan":3394,"je ":3919,"jon":34428,"jor":11663,"itu":2594,"itt":16474,"its":2537,"ity":2756,"isk":51560,"isj":4819,"ism":2335,"iss":5696,"ist":35845,"iv ":3591,"ita":7242,"ite":15976,"iti":9684,"iva":2856,"ivi":3570,"ive":13599,"ipp":2048,"is ":12915,"ion":14718,"ir ":6919,"irk":9296,"isi":4522,"ish":2413,"ise":12425,"isa":4298,"ire":7035,"it ":2143,"kil":3815,"kk ":10113,"kin":8354,"kip":2377,"kir":5305,"gÃ¥r":7228,"kis":2484,"kje":16281,"km ":7242,"kel":6677,"ken":15267,"kes":10195,"ker":29257,"ket":9398,"ke ":65844,"kra":11143,"kre":8431,"kt ":12793,"kse":4519,"kro":2450,"kri":13226,"kot":2092,"km²":2186,"kor":5874,"kon":13827,"kom":31756,"kol":9129,"ks ":3032,"kny":2137,"kjø":3238,"kni":5015,"kke":25288,"klu":3574,"kle":4386,"kla":6195,"kli":3140,"jul":2456,"kat":3942,"kar":4133,"kas":2638,"kap":17702,"kan":20091,"kal":14105,"kam":2092,"ka ":6242,"før":14702,"fød":22348,"føl":2302,"ha ":2473,"ham":3082,"han":12460,"hal":3654,"hav":6160,"har":25645,"had":4958,"he ":6327,"hel":5939,"hei":3009,"het":10209,"her":5545,"hen":7004,"hin":2784,"his":4361,"hje":2935,"gle":2190,"gn ":3125,"gla":3596,"gjø":2529,"gni":2494,"gne":11554,"gs ":3264,"gru":12215,"gra":9273,"gt ":5657,"gre":14854,"gst":4516,"gsÃ¥":9706,"gus":2418,"ial":4669,"ian":8957,"øke":2125,"ølg":2438,"id ":6821,"ibe":2226,"ia ":20599,"iet":4979,"iel":5273,"ien":17663,"ier":7794,"ies":2351,"ig ":24484,"ift":6890,"ør ":13261,"øpe":2597,"ømm":2930,"ich":3831,"ie ":7138,"ica":2979,"ids":3533,"idr":3478,"ønn":3150,"idl":8890,"ide":18687,"idd":2770,"ida":3822,"ønd":2352,"øst":12196,"il ":45404,"im ":2951,"ika":14540,"ige":27059,"iga":2145,"igh":5558,"igi":2075,"igg":17522,"igs":3095,"ign":3006,"øre":10636,"ørs":12878,"ørt":2899,"ik ":6168,"ime":2903,"ind":10332,"ina":14021,"inn":35697,"ino":2582,"int":10160,"ins":19112,"ine":16471,"ing":71844,"ini":7560,"ink":2146,"iod":2244,"iny":3773,"ikl":3957,"ikk":22590,"ike":13916,"in ":20575,"ikt":9421,"iks":3390,"ilo":2911,"ill":23109,"ilk":2217,"øve":6392,"ilm":4862,"ilh":3403,"ili":8732,"ild":3649,"ile":4123,"io ":2961,"ils":3090,"ilt":4137,"ørø":2134,"øy ":5016,"hol":9462,"hov":6717,"øye":5083,"øya":3991,"hvo":3807,"hun":2960,"hus":4616,"hve":2835,"døs":2243,"død":8337,"ffe":3959,"ffi":2131,"fes":3648,"fer":4921,"fem":2188,"fen":2692,"fek":3989,"fel":4217,"fat":5769,"far":3333,"fam":3942,"fal":2369,"eta":7185,"ete":19743,"eti":2646,"etn":3543,"esp":3357,"eso":2351,"est":46619,"ødt":21901,"ess":8652,"esv":2859,"etr":2759,"ets":9941,"ett":37893,"ety":3237,"ew ":2555,"eve":11454,"eva":2230,"evi":2446,"ey ":3642,"er ":431766,"epa":2788,"eor":2578,"eol":2061,"ød ":7105,"es ":40484,"ept":2696,"epu":5041,"epr":2644,"erk":15694,"erl":6414,"eri":32585,"erg":11340,"erh":2771,"ere":60500,"erf":5751,"erd":9568,"era":10673,"erb":3057,"et ":226950,"esk":8307,"esi":10382,"øde":3134,"ese":10631,"erv":4618,"eru":2639,"err":6355,"ert":31938,"ers":27995,"ern":17742,"erm":4003,"ero":2745,"ekk":7221,"ekn":2204,"ekr":3313,"eks":10659,"ekt":15683,"en ":368973,"ela":6680,"eld":5215,"ele":18548,"eli":14582,"elg":2392,"ell":56973,"elo":2842,"elv":10426,"els":37467,"elt":17365,"emb":7048,"ema":3872,"eme":6179,"emm":4042,"emo":2614,"emi":3720,"emp":3266,"ems":2210,"enf":4002,"ene":43923,"enh":3807,"eng":13869,"ena":2893,"end":24201,"enn":18704,"enk":3130,"eni":5336,"ens":38448,"ent":42518,"egn":9280,"ege":7883,"egg":4893,"egi":7310,"eha":2568,"egr":3736,"eis":4556,"eim":2591,"ein":6378,"eie":6921,"eid":6829,"el ":17607,"eke":3082,"em ":6695,"gje":8413,"git":6162,"gis":5345,"gin":2581,"gio":4769,"ghe":4290,"gge":29980,"gi ":3980,"gen":31946,"get":10089,"ger":52848,"ges":4527,"gg ":2739,"gel":10922,"gde":3615,"ge ":33172,"gas":3030,"gar":3339,"gat":3035,"gam":2397,"gal":2375,"gan":11800,"ga ":3023,"fyl":7035,"fte":8757,"fun":3666,"ftv":7006,"ft ":3864,"fra":35021,"fre":5013,"fri":6048,"for":85448,"fot":5733,"fol":6844,"fle":5300,"fly":4535,"fil":5351,"fik":2963,"fin":6087,"fir":2657,"fis":4937,"fje":5241,"fjo":6945,"da ":8120,"de ":60427,"dal":11372,"dag":6176,"dat":3189,"dar":2553,"dan":8876,"dde":9644,"co ":2205,"ch ":3122,"cha":2322,"ck ":3691,"che":4052,"chi":2081,"cke":2693,"ed ":55809,"eba":2196,"ebe":2240,"ebr":3393,"eal":3627,"eat":2589,"efi":2051,"efo":4434,"efe":4659,"ei ":7806,"een":2210,"edl":3176,"edi":4356,"ede":17856,"eda":3620,"eg ":6746,"eds":6107,"edr":4047,"dve":3394,"dus":5910,"don":4573,"dom":4754,"ds ":5042,"dmi":3335,"dni":2776,"dst":4471,"dte":2377,"duk":4468,"dri":5473,"dra":3627,"dt ":28714,"dre":18932,"dsk":5623,"dia":3579,"der":53613,"des":6374,"det":47571,"del":41297,"dek":2450,"den":73325,"dem":4267,"di ":2586,"dle":5037,"dla":2367,"dli":11521,"din":4634,"dio":3152,"dis":11321,"die":2789,"dig":4896,"rga":6400,"ri ":3980,"rgi":2280,"rge":14583,"ret":27902,"res":20361,"rev":6468,"rfa":4418,"rds":3327,"rdv":2179,"rg ":6581,"rea":5698,"ree":2115,"ref":5778,"red":9285,"rei":3228,"reg":12569,"rem":5839,"ren":35981,"rek":8172,"rel":4778,"rer":20318,"rep":8966,"rda":5684,"rdl":3552,"rdi":4958,"rde":19629,"re ":73979,"rbu":2376,"rd ":17240,"ras":7043,"rat":9184,"rav":3015,"rbi":3863,"rbe":5664,"rag":2711,"ran":19588,"ram":7456,"ral":8750,"rak":4191,"raf":12560,"rad":5981,"rs ":8675,"ros":5462,"rom":6997,"ron":6959,"rop":5525,"rov":9073,"rod":9193,"rol":4308,"rof":2729,"rog":4064,"rna":5941,"rne":13070,"ro ":3052,"rma":5722,"rme":6882,"rli":7449,"rle":2859,"rla":5982,"rn ":5061,"rks":2379,"rko":2612,"rki":3108,"rke":16934,"rka":2079,"rm ":3363,"nær":3936,"rio":3486,"rit":8311,"ris":17487,"riv":5194,"rig":8523,"ril":2995,"rik":24390,"rin":21593,"rim":2649,"ria":5352,"ric":2697,"rid":3352,"rie":12327,"rif":3440,"rdø":2401,"rk ":15343,"rup":6154,"run":12301,"rum":4066,"ruk":11189,"rus":6418,"rva":2898,"rvi":2442,"rve":4784,"ry ":3420,"rsk":24627,"rsi":6764,"rso":5206,"rse":3955,"rta":2228,"rst":17535,"rte":20285,"rti":8810,"rua":2277,"rts":5729,"rt ":35053,"rri":3179,"rre":7643,"rra":4011,"sak":3680,"sal":2138,"sam":15315,"sbe":2623,"san":6993,"sat":4359,"sas":3809,"sa ":2445,"ryk":2399,"sha":2088,"shi":3201,"sje":5058,"sjo":34525,"sie":4330,"sid":8350,"sia":4578,"sk ":75137,"sit":9775,"sis":16636,"sin":9850,"sik":9289,"sda":2876,"sby":2166,"se ":19672,"sch":2395,"ser":35054,"ses":4849,"set":9508,"sfo":2877,"seg":6362,"sep":3173,"sen":41736,"sem":4309,"sel":10690,"sek":3507,"spo":8496,"spr":5617,"spe":5862,"spi":12660,"spa":2392,"sol":2063,"som":85971,"son":9679,"sor":2986,"sjø":3410,"st ":39863,"ss ":6097,"sli":3561,"slo":6037,"slu":2332,"sky":2874,"sla":9825,"sle":4538,"ski":8246,"skj":4074,"skl":2812,"sko":10936,"skr":10481,"sku":3651,"ska":22269,"ske":46720,"sma":2688,"sme":3758,"stÃ¥":5531,"stø":6651,"syn":2627,"sys":2439,"syk":3044,"sse":15330,"ssa":2824,"sso":3173,"ssl":2943,"ssk":3059,"ssi":8369,"sst":2689,"ste":66334,"stf":2230,"sta":38189,"stn":2344,"sto":16688,"sti":20167,"stl":6400,"stu":3784,"str":25344,"sty":4151,"sun":2773,"sut":4111,"sva":4458,"sve":9604,"svi":2603,"tak":2812,"tal":21730,"tab":3588,"tad":7590,"tba":6294,"tav":2813,"tat":16792,"tas":5433,"tar":7233,"tan":11218,"tam":2411,"te ":58351,"ta ":8154,"pa ":2052,"pe ":5528,"par":10024,"pas":4090,"pan":5764,"län":2721,"pen":10042,"per":17920,"pet":8954,"pes":3294,"pel":3770,"pla":9218,"lær":3470,"pil":11406,"pin":7171,"pis":4864,"por":10569,"pol":7076,"ppr":5964,"ppl":2678,"ppe":9671,"pp ":3799,"pub":5466,"pte":2999,"pri":10360,"pre":12096,"pro":23686,"prÃ¥":2961,"løp":3837,"mÃ¥l":4336,"ra ":37023,"ngi":2137,"ngl":4160,"ngr":2861,"ngt":2215,"ngs":15959,"ni ":4401,"nge":45140,"nga":2868,"ngd":4050,"nhe":2871,"nel":12450,"nen":27435,"ner":33385,"net":21165,"nes":25627,"ng ":54251,"ned":4860,"nfo":4282,"nce":2258,"ne ":72733,"nby":5798,"ndt":4870,"ndr":12413,"nds":12659,"ndo":5458,"ndl":2779,"ndi":6968,"nde":56337,"nda":5748,"nal":12130,"nan":2828,"nar":2470,"nad":2448,"nd ":38901,"nav":9346,"nat":5154,"nas":7061,"na ":15576,"nyi":3711,"nua":2135,"nty":8509,"nto":3515,"ntr":8273,"nti":5299,"ntl":2516,"nta":6663,"nte":28676,"nsp":2257,"nst":14726,"nss":3014,"nse":21534,"nsj":2053,"nsi":3034,"nsl":2182,"nsk":39260,"nsa":2994,"nt ":24653,"ns ":20153,"noe":3540,"nom":6845,"nor":29131,"nov":2911,"nne":42931,"nna":2105,"nnb":5914,"nnl":4919,"nno":3889,"nni":5129,"nnk":2501,"nns":7691,"nma":2510,"nli":4969,"nn ":14981,"nla":6112,"nle":2679,"no ":2847,"nke":3676,"nkr":4191,"nia":6220,"niv":5311,"nis":15475,"nit":2397,"nin":25664,"nik":2124,"ogs":10014,"ogr":5237,"ogi":5246,"ogn":3334,"oge":2102,"ok ":7509,"ol ":2411,"ock":4853,"ode":7882,"of ":4012,"oen":3787,"odu":8565,"og ":134463,"oft":3759,"off":4398,"ofe":2299,"od ":2196,"obl":2263,"obe":3133,"nyt":4280,"jøe":2169,"jør":5841,"ote":3613,"ott":4146,"ots":2202,"oto":3139,"ost":4414,"otb":5531,"osi":2968,"ose":3901,"oss":3129,"ovi":8837,"ove":25154,"oun":9480,"our":3311,"opp":20548,"ope":3654,"opa":2135,"os ":5236,"or ":56677,"ork":4146,"orl":2445,"orm":10647,"orn":3102,"orr":3175,"ord":42679,"ore":13522,"orf":4509,"org":16810,"ori":9530,"ort":20720,"ors":26414,"m² ":2219,"ot ":7999,"orb":6069,"ora":3709,"ola":2381,"old":10833,"on ":35130,"oli":8827,"oll":6799,"olk":9387,"ole":9725,"ols":3475,"olm":2556,"olo":8072,"oly":3133,"oka":3954,"om ":107366,"okk":3118,"oks":3517,"okt":2231,"ona":9739,"ond":6224,"one":22668,"ong":10465,"oni":5288,"ono":3591,"ons":14426,"ont":6196,"oma":5119,"ome":6397,"omb":2444,"omi":3410,"omf":2921,"omm":37033,"omp":3178,"omr":6915,"oms":5919,"la ":5793,"le ":57070,"lde":10166,"lds":2282,"lad":2570,"lag":17793,"lan":58009,"lar":3088,"lat":7640,"las":9906,"lav":2641,"lba":2444,"ld ":6657,"lbu":3945,"kvi":3664,"kva":2672,"kur":2362,"kun":5248,"kul":5655,"kte":10988,"kst":5101,"ksj":6623,"ksi":2580,"ktu":4360,"kti":6575,"kto":3659,"kys":2223,"ls ":2559,"lok":3826,"lon":2509,"lom":14627,"log":6983,"los":2313,"lov":2717,"lme":3805,"lto":5869,"ltu":2394,"lub":3807,"lsk":13894,"lst":8591,"lv ":4083,"lta":2646,"lte":5587,"lse":19815,"lt ":21046,"lge":4495,"li ":4167,"lev":4182,"les":12321,"let":13065,"ler":55284,"lem":5548,"len":25387,"lek":9666,"lei":2319,"leg":6026,"led":5805,"lls":3758,"lo ":5941,"lhø":2797,"lla":5878,"lle":65120,"lli":10408,"llk":3343,"llo":12212,"lke":16528,"lkn":2938,"lm ":3452,"lje":2230,"ll ":17353,"lit":14639,"lis":9353,"lir":3649,"lin":18549,"liv":2248,"lia":5498,"lik":12256,"lig":60844,"lie":7273,"ma ":3762,"mai":2118,"mar":11919,"mas":4336,"mal":3128,"man":16587,"mat":5209,"mbe":7011,"me ":8290,"med":37095,"met":13813,"mes":7734,"mer":33081,"mel":15249,"men":33002,"mfa":2204,"lva":3696,"lve":4358,"lut":2852,"ly ":2856,"lym":3245,"lys":2102,"høy":5492,"hør":3562,"mpi":3576,"mpe":4413,"ms ":2196,"mod":2416,"mon":3257,"mor":2125,"mot":10268,"mt ":3487,"mst":3459,"mrÃ¥":7270,"mus":6700,"mun":25496,"min":11788,"mil":8071,"mis":4562,"mid":4108,"mle":3222,"mmu":24025,"mme":31877,"vær":6865,"ytt":5943,"yte":2764,"yst":6626,"ysk":7598,"yrk":2373,"yre":3443,"ypr":2466,"yr ":3697,"ype":2571,"ye ":2367,"yde":4954,"yer":2533,"yen":9116,"ya ":4898,"ykk":3634,"ylk":9083,"ymp":3267,"ygg":10777,"ygd":2788,"yin":3851,"tør":7504,"tøv":4228,"tÃ¥r":3969,"sør":11014,"sÃ¥ ":11194,"røs":2166,"røn":4125,"røy":2397,"vir":2566,"rÃ¥k":3246,"vik":8483,"vil":4773,"vin":16505,"vid":3526,"rÃ¥d":8908,"vit":4787,"vis":10758,"vn ":5466,"vne":6818,"vol":2228,"vok":2535,"vor":4361,"ver":56905,"ves":12440,"vet":7953,"vei":7599,"veg":2114,"ven":15144,"vem":2207,"vel":5384,"ved":23018,"ve ":6401,"val":6677,"van":11534,"var":40025,"vat":3521,"va ":5156,"utø":4075,"usi":6124,"use":8793,"ust":8108,"uss":8811,"utg":6539,"ute":5272,"utt":5236,"uts":3096,"utv":4169,"us ":10711,"ut ":5919,"ure":6307,"urg":2892,"uri":2750,"urn":3039,"uro":3768,"urr":2556,"ur ":5709,"upp":6292,"ume":4177,"unt":9250,"uns":3775,"unk":3135,"uni":6920,"unn":14136,"und":31193,"ung":4670,"une":22585,"uks":4735,"ukt":5470,"uke":5765,"um ":8855,"ult":4324,"ull":5554,"uli":4543,"ule":2555,"un ":3708,"ugl":2152,"ugu":2387,"ude":2985,"udi":2221,"ue ":2256,"ues":2288,"uar":4525,"uan":2334,"ubl":5816,"ubb":3409,"ud ":2293,"typ":2478,"tyr":5657,"tys":4599,"ty ":11596,"tve":8575,"tvi":4656,"tur":14995,"tus":2293,"tun":2106,"tud":2813,"ts ":7521,"tre":18842,"tt ":26597,"tra":17913,"tri":10126,"tru":4763,"tro":7380,"try":2883,"tse":4787,"tsk":2860,"tsu":3903,"tst":7808,"tta":2344,"tte":44536,"tti":2906,"tts":8940,"to ":8546,"tni":6514,"tne":2955,"tob":2085,"ton":7030,"tok":6614,"tol":4307,"tor":21379,"til":54443,"tik":6372,"tif":3581,"tie":3348,"tig":5426,"tit":4010,"tis":16494,"tin":10509,"tio":5365,"tia":3220,"tid":14688,"tiv":7883,"tje":3021,"tli":7658,"tla":4715,"tle":2120,"tem":9786,"ten":44932,"tei":3878,"tek":5962,"tel":9339,"teg":7791,"ted":11925,"tfo":3043,"th ":3609,"tet":33199,"tes":6700,"ter":82933,"tgi":4175,"pÃ¥ ":52716,"ti ":4056,"the":4578},"n_words":[20399254,23799460,17069273],"name":"no","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/pa.json b/contrib/languages-data/pa.json
new file mode 100644
index 0000000..3699c75
--- /dev/null
+++ b/contrib/languages-data/pa.json
@@ -0,0 +1 @@
+{"freq":{"ਾਜ਼":17,"ਾਜਧ":12,"ਾਜਾ":13,"ੀਲ ":18,"D":18,"E":20,"F":29,"G":23,"A":52,"B":35,"C":59,"L":38,"M":47,"N":37,"O":37,"H":31,"I":60,"U":14,"T":45,"W":18,"P":47,"S":55,"R":24,"f":103,"g":125,"d":221,"e":655,"b":81,"c":194,"ੀਰ ":33,"a":561,"n":492,"o":398,"l":256,"m":260,"j":16,"k":37,"h":252,"i":537,"w":71,"v":42,"u":236,"t":467,"s":328,"r":400,"p":124,"z":13,"y":106,"x":19,"ਿਆ।":22,"ਾਤਰ":17,"ਾਤਾ":27,"ਾਤੀ":12,"ਿਆਣ":31,"ਿਆਦ":20,"ਿਆਨ":47,"ਿਆਲ":13,"ਿਆਰ":28,"ਿਆਵ":14,"ਿਆਸ":12,"ਾਣਕ":12,"ਾਣੂ":104,"ਿਆਂ":76,"ਾਣੀ":40,"ਾਣਿ":27,"ਾਣਾ":22,"ਾਨੇ":13,"ਾਨਾ":31,"ਿਊਟ":14,"ਾਨੀ":48,"੩੧ ":18," । ":128,"੩੦ ":23,"ਾਦੀ":22,"ਾਦਾ":19,"ਾਨਕ":26,"à©à¨ ":21,"।ਇਸ":14,"ium":27,"is ":22,"ion":36,"ੀਤ ":15,"਼à©à¨°":22,"੦੦੮":24,"ੀਪ ":375,"ੀਨ ":57,"ੀਮ ":51,"ਟੀ ":57,"ਜੋਂ":19,"ਾਗਰ":20,"ਾਕੀ":370,"ਾਕਿ":34,"ਟਾ ":22,"੨੩ ":13," m":24," o":57," h":17," i":57," d":21," e":15," f":30," a":133," b":21," c":34," t":120," w":40," p":41," s":54,"੨੨ ":14," r":16,"੨੧ ":14,"੨੫ ":14,"਼ਸੀ":29," H":25," I":54," N":32," O":31," L":21," M":43,"੨੪ ":13," B":32," C":54," A":46," F":17," G":21," D":15," E":17,"ਿਰ ":73," S":47," R":22,"਼ਹਿ":50," P":44," W":15," T":36,"ਜੀਵ":20,"੨੭ ":13,"੨੬ ":13,"ਿਲ ":25,"ਜà©à¨²":54,"ਜਿੰ":30,"੨੯ ":16,"ਜੂਨ":35,"਼ਾਂ":36,"਼ਾਹ":32,"੨੮ ":15,"਼ਿਆ":14,"ੀਕ ":34,"ਜ਼ਮ":12,"਼ਿਲ":75,"ਜ਼ਾ":33,"ਜ਼ਿ":88,"ਾਂਦ":190,"ਜਾਂ":231,"ਜ਼ੀ":72,"ਾਂਤ":20,"ਾਂਸ":16,"ਜਾਬ":125,"਼à©à¨°":27,"ਜਾਤ":13,"ਜਾਣ":55,"ਜਾਦ":21,"ਜਿਮ":22,"ਜਿਲ":12,"ਜੀਅ":19,"ਿਸ ":72,"ਾਅਦ":30,"ਜਿਸ":70,"ਜਿਹ":19,"ਾਇਆ":59,"ਾਇਣ":62,"੨੦ ":14,"ਾਇਲ":17,"ਟਰ ":67,"ਾਈਟ":14,"ਾਈਡ":27,"ਾਉਂ":14,"ਾਈਨ":29,"ਾਉਣ":23,"਼ਬਦ":26,"ੀਂ ":52,"ਿਤ ":82,"ਟਨ ":17,"ਚੰਦ":21,"੧੮ ":12,"ਛੋਟ":13,"੧੯ ":14,"ੀਆ ":53,"ਿਨ ":1101,"਼ਰਵ":39,"ਜਨਮ":32,"ਜਨਵ":39,"ਜਨਸ":13,"ਜਦੋ":15,"ਜਧਾ":12,"ਿਬ ":69,"ੀਤਾ":41,"Co":14,"ੀਤੀ":42,"à©à¨†à¨°":46," In":28," Ma":13,"he ":69,"ਾੜੀ":12,"ੀਟਰ":38,"Ma":13,"Ol":12," Co":14,"In":29,"L ":13,"।":1840,"ੀਕਲ":22,"ੂਪ ":25,"ੀਕਨ":13,"ਿਲੀ":21,"ਿਲਾ":83,"Th":15,"ਿਲà©":13,"ਿਲੋ":46,"ਿਲੇ":38,"ੂਨ ":35,"ਿਲਦ":12,"ਵ":3164,"ਲ":4004,"ਰ":5772,"ਿ":5178,"ਾ":10004,"਼":1111,"ਹ":4455,"ਸ":5034,"ਦ":5926,"ਧ":297,"ਤ":3377,"ਥ":271,"ਢ":42,"ਣ":777,"ਠ":69,"ਡ":853,"ਮ":2207,"ਯ":188,"ਬ":2031,"ਭ":430,"ਪ":1926,"ਫ":303,"ਨ":4410,"ਕ":3798,"ਗ":2047,"ਖ":660,"à¨":62,"ਓ":66,"à¨":61,"ਜ":2041,"ਟ":625,"ਘ":134,"ਛ":70,"ਚ":1522,"ਅ":1321,"ਆ":1270,"ਇ":2053,"ਂ":2880,"à¨":233,"ਈ":490,"ਉ":422,"ਊ":48,"ੱ":1924,"à©°":2555," a ":18,"੦":200,"੧":273,"੪":62,"ਿਮਨ":20,"à©«":77,"੨":239,"à©©":93,"à©®":92,"੯":96,"੬":69,"à©­":58,"੨੦੦":38,"à©œ":191,"à©€":4636,"à©":1825,"à©‚":1115,"ੇ":3761,"ੈ":2081,"à©‹":1676,"à©Œ":206,"à©":1046,"ੂਲ ":22,"ੀਜਿ":21," Ol":12,"ੂਰ ":25,"b ":18,"ਿਹਾ":91,"a ":87,"ਿਸੇ":29,"ਿਸ਼":57,"ਿਸਾ":23," Th":15,"ਿਸਤ":34,"ਚੌਂ":12,"ਚੋਂ":38,"ਿਵੇ":63,"ਿਵਾ":13,"ੀਕਾ":45,"i ":27,"ਿਟੀ":38,"ge":13," in":40,"ic ":12,"fi":14," is":12,"fo":16,"ਚਾਈ":23,"he":95,"ha":27,"gh":19,"go":13,"g ":34,"ea":30,"ਚਾਰ":34,"ec":17,"ਚਾਲ":16,"ed":45,"de":37,"di":36,"ia ":20,"ev":13,"h ":33,"Ind":16,"ee":14,"el":41,"ei":12,"en":68,"em":14,"et":19,"ੀਆਂ":178,"es":53,"er":114,"ੀਅਤ":34,"ca":15,"e ":169,"ਚੀਨ":33,"be":16,"da":20,"f ":44," of":43,"ct":18,"cs":14,"co":29,"ck":12,"ch":22,"ce":32,"c ":16,"ics":14,"d ":98,"at":58,"as":29,"ar":52,"al":54,"ai":16,"am":68,"an":97,"ac":18,"ad":16,"ab":12,"ਿਨਾ":26,"nt":47,"ns":26," am":51," an":24,"ਿਨੇ":13,"ੈ। ":755,"of":44,"om":42,"on":91,"ol":28," ਅ":1094,"os":19," ਇ":1749," ਆ":274,"ou":18,"or":54," à¨":119,"r ":72," ਉ":313,"ow":13," ਈ":30," ਕ":1460,"pe":18," ਖ":182," ਗ":918,"ਿਨà©":12," à¨":59," ਓ":47," ਜ":1208," à¨":16," ਟ":112,"po":12," ਘ":42,"pi":19," ਚ":259," ਛ":39,"lo":16,"ਜੋ ":82,"ll":20,"igh":12,"ly":15,"o ":26,"ma":17,"mb":17,"me":42,"mi":19,"mp":27,"mu":48,"na":37,"nc":21,"nd":68,"ne":28,"ng":50,"ni":32,"ਿਤਾ":77,"ਿਤੀ":12," ।":190,"ੀਅਮ":40,"m ":58,"ੀਅਨ":27,"ਿਥਿ":27,"ine":16,"ing":33,"li":35,"le":33,"ld":21,"la":31,"n ":137," co":22,"ht":18,"hu":17,"hi":29,"ho":16,"id":13,"ic":60,"ia":36,"ig":20,"in ":38,"ie":22,"k ":17,"ir":14,"is":39,"it":46,"iu":27,"il":21,"in":129,"io":41,"l ":48,"ਾਰਕ":16,"ਾਰਚ":41,"ਾਰਤ":101,"ਾਰਨ":27,"wo":13,"ਾਮਿ":15,"y ":61,"wa":12,"ve":29,"ਾਰੀ":54,"ur":21,"us":14,"ਾਰਾ":51,"ut":16,"um":56,"un":15,"ty":14,"ਾਰੇ":39,"ua":16,"to":25,"ts":12,"tr":30,"te":65,"ti":67,"ਾਬਕ":377,"th":116,"ta":28,"st":52,"se":26,"sh":12,"si":30," ੨":195," à©©":56," ੪":26," à©«":29,"u ":55,"ਚਰਲ":20," ੬":17," à©­":14," à©®":15," ੯":19,"rs":22,"rt":24,"ry":21," ੧":210,"ro":42,"ri":64,"ਚਲਾ":12,"re":47,"rd":17,"à©à¨¤ ":26,"ra":24,"t ":72," ਹ":2872," ਸ":2699," ਵ":2408,"ਾਬਲ":23,"ht ":12," ਰ":498," ਲ":715," ਬ":868," ਭ":339,"s ":155,"à©à¨£ ":18," ਮ":1134," ਯ":133," ਨ":1023," ਪ":1056," ਫ":184," ਤ":951," ਥ":21,"ਜੇ ":27," ਦ":4245," ਧ":86," ਡ":55,"ਾਬੀ":48," ਢ":16,"pr":12,"ਾਬਾ":23,"ਿਗਿ":35,"à©à¨° ":13," s ":16,"ਾਹੀ":40,"ਾਹਿ":88,"ਿਚਾ":17,"hum":14,"ਾਸਿ":34,"ਾਸ਼":82,"ਾਸੇ":16,"ਜਾ ":63,"ਚਨਾ":19,"ਾਲੇ":56,"ਾਲਾ":46,"ਾਲੀ":54,"ਜ਼ ":52,"ਜੀ ":155,"ਿਖੇ":23,"ਾਵਾ":47,"ਿਖਾ":19,"ਾਵਲ":13,"ਿਕਾ":29," th":90,"ym":13,"ਿਕਸ":14,"ਹਾਂ":94,"ਹੀਰ":14,"ਹੀਨ":13,"ਹà©à¨£":12,"ਹà©à¨¤":24,"ਹਿਰ":60,"ਹਿਲ":50,"ਹਿਨ":24,"ਹਿਬ":75,"ਹਿਸ":19,"ਹਾਲ":13,"ਹਾਰ":25,"ਹਾਨ":18,"ਹਿਤ":27,"ਹੀਂ":44,"ਹਾਸ":57,"ਹਾਈ":26,"ਜਨ ":22,"਼ਨ ":23,"ਹਰਿ":36,"ਹਨਾ":25,"er ":48,"es ":28,"ਸà©à¨°":19,"ers":16,"en ":20,"ਸਿੱ":77,"ਸਿੰ":78,"ਸੂਰ":32,"ਸੂਬ":12,"ਸਿਧ":13,"ਸਿਰ":19,"ਹਨ।":438,"ent":21,"ght":14,"ਸੀ।":71,"ਸ਼ਨ":37,"ਸ਼ਤ":17,"ਸ਼ਰ":12,"ਸ਼ਬ":26,"ਸ਼ਹ":65,"ਸ਼à©":34,"ਸ਼ੀ":30,"ਸਾਂ":13,"ਸ਼ਿ":19,"ਸ਼ਾ":149,"ਸ਼ਖ":29,"ਸ਼ਟ":21,"ਸਾਨ":18,"ਸਾਰ":46,"ਸਿਕ":33,"ਸਿਖ":14,"ਸਾਲ":1144,"ਸਾਹ":90,"ਸਿਟ":36,"ਸੀਅ":29,"ਸਾਇ":73,"ਸ਼ੇ":16,"ਸ਼à©":23,"ਸਿਆ":13,"ਸਰਕ":13,"ਸਮੇ":23,"ਸਮਾ":24,"ਗੜà©":19,"ਸਰੀ":12,"ਸਲਾ":21,"ਸਤੰ":39,"ਸਦੇ":13,"ਸਦਾ":17,"ਸਦੀ":15,"ਸਨੂ":15,"ਗà©à¨°":167,"ਗੋਬ":26,"for":13,"ਸਟਾ":14,"ਗà©à¨°":502,"ਸਨ।":37,"ਚਾ ":12,"ਸਤਾ":54,"ਸਥਾ":37,"ਸਥਿ":13,"ਗਰੇ":16,"ਗਰਾ":30,"ਗਰੀ":415,"ਿਚ ":162,"ਾਹ ":17,"ਾਸ ":54,"਼ਟਰ":18,"ਿਕ ":134,"ਕੰਪ":19,"ਾਲ ":1369,"੧੬ ":14,"cti":12,"ਗਸਤ":63,"੧੭ ":15,"ਾਰ ":238,"੧੪ ":14,"੧੫ ":13,"੧੧ ":12,"ਾਮ ":62,"ਗਿਆ":104,"੧੨ ":16,"ਗਾਂ":14,"੧੩ ":12,"਼ਖ਼":29,"ਾਬ ":99,"੧੦ ":14,"ਹੱਦ":12,"ਖਾਂ":21,"ਖ਼ਸ":29,"ਖਿਆ":40,"ਾਨ ":155,"com":13,"ਗਣਿ":17,"ਿਆ ":232,"ਾਦ ":29,"੦੮ ":24,"cs ":14,"ਾਣ ":28,"ਾਤ ":23,"ਖੇਡ":37,"ਖੇਤ":37,"ਾਜ ":52,"ਖੋਂ":14,"ed ":32,"ਕੌਮ":15,"ਾਗ ":23,"ਕà©à¨°":37,"ਸੰਸ":37,"੦੦ ":21,"ਘਰ ":13,"ਾਈ ":117,"ਸੰਖ":18,"ਕਾਂ":22,"ਸੰਬ":52,"ਸੰਤ":12,"ਕਿਤ":57,"ਾਂ ":1353,"਼ੀ ":81,"ਹੋà¨":48,"ਹੋਇ":70,"ਹੋਈ":20,"ਕਾਲ":30,"ਕਾਸ":23,"ਕਾਬ":25,"ਕਾਰ":103,"ਕੀਤ":87,"ਹੋਰ":25,"ਹੋਣ":23,"ਕਿਲ":58,"ਕਿਸ":74,"ਕਿਹ":19,"ਕà©à¨":20,"ਕà©à©±":13,"dia":15,"਼ਾ ":51,"ਹਿੰ":31,"ਹੈ।":922,"ਕੇਸ":12,"ਹà©à©°":468,"ਕੋਈ":24," ਅਪ":47," ਅਨ":20," ਆਉ":20," ਅਧ":17,"ੱਖੀ":14,"ੱਖਾ":16," ਅਮ":64," ਅਰ":39," ਅਸ":17,"ੱਖਰ":18,"ਕਦੀ":15," ਅਜ":33," ਅਗ":67," ਅਕ":54," ਅਤ":411,"ੰਸਥ":15,"ਗਾ ":22,"ਕਨੀ":13,"ੰਸਾ":14,"re ":16,"ੱਚੋ":29,"ੱਛਮ":12,"ਕਤੀ":13,"ਕਤੂ":34,"ੱਜੋ":16,"rs ":13," ਉੱ":56," ਉਹ":51," ਉਸ":69," ਇੱ":255," ਇੰ":25," ਉਪ":25," ਉਨ":48," ਉਤ":19," à¨à¥¤":18," ਇਹ":533," ਇਸ":723," ਇਲ":20," ਅੰ":217," ਅੱ":34,"ਾ। ":18," ਇਨ":39," ਇਤ":23," ਇਥ":21,"rig":16,"ਗਰ ":22," ਆਮ":22,"ੱਖਣ":15," ਆਰ":19," ਇਕ":94," ਆਦ":14," ਆਪ":76," ਆਬ":12,"ਸਟ ":13," ਕਰ":183," ੨ ":19," ਕਲ":399," ਕਹ":19," ਕਾ":86," ਕੀ":109," ਕਿ":260," ਕà©":68," ਗਈ":32," ਕੇ":73," ਕੈ":22," ਕੋ":64," ਕੌ":24," à©© ":14," ਕà©":23," ਗà¨":21,"ਕਸ਼":26," ਓਲ":31," ਕਈ":21," ੧ ":21,"ry ":17,"ਸਤ ":69,"ਕਲਾ":14,"ਵਰਗ":12,"ਕਲੰ":379," à¨à¨«":13," à¨à¨¨":15," à¨à¨¸":15,"ਵਰਸ":36," à¨à¨—":22," à¨à¨²":14,"ਵਰਤ":32,"ਵਰੀ":76," ਜ਼":109," ਜਿ":159," ਜਾ":364," ਜà©":64," ਜੀ":148," ਜੂ":37," ਜੇ":15," ਜੋ":96," ਜੰ":39,"ਸਨ ":38,"ਕਰਕ":17," à©® ":14,"ਕਰਦ":41,"ੱਡਾ":23,"ਕਰਨ":55," ਚਿ":16," ਚਾ":51," ਚà©":16," ਚੀ":38," ਚੰ":37," ਚੱ":12," ਜਰ":13," ਛੋ":14,"ਕਰੀ":18," ੯ ":15,"ਕਰੋ":16," ਜਨ":89," ਜਦ":19,"ਕਲਚ":23," ੬ ":12,"ੱਤਰ":42," ਚਲ":15," à©­ ":13,"ਗੀ ":17,"ਸਭ ":25," ਗਰ":23," ਕੰ":44," ੪ ":15," ਖੇ":84," ਗਣ":16," ਖਾ":28," ਖਿ":13,"ੱਥੇ":12," ਖ਼":15,"ੱਤਾ":19," ਘਰ":13," ਗà©":436," à©« ":15," ਗੋ":35," ਗà©":188," ਗਿ":86,"ੱਤੇ":24," ਤਰ":31,"ੰਖਿ":16," ਤਾ":41," ਤਿ":24,"ੰਗਾ":15,"ੰਗਰ":20," ਤਕ":27," ਤੱ":66," ਦਰ":56," ਦੂ":37," ਦਾ":950," ਦਿ":1171," ਦੀ":419," ਦà©":51," ਦਸ":54," ਤੌ":38," ਤੋ":497," ਤੇ":160," ਟੀ":56," ਟਰ":13,"ੰਗà©":60," ਮਈ":37," ਬੇ":15," ਬੋ":36," ਬਾ":492," ਬਿ":25," ਬੀ":32," ਬà©":18," ਭਗ":25," ਬਹ":34," ਫà©":14," ਪੱ":30," ਪੰ":163," ਬਲ":14," ਬਰ":25," ਬਨ":14," ਫ਼":66," ਪੜ":16," ਫਾ":12," ਫਿ":18," ਫà©":20," ਬਣ":71," ਪੋ":18," ਪà©":133," ਮੌ":17," ਮੋ":24," ਮੈ":37," ਮੀ":19," ਮà©":481," ਮਾ":131," ਮਿ":97," ਮਹ":95," ਮਸ":25," ਬੰ":15,"st ":12," ਭੌ":12," ਮਨ":31," ਮਤ":12," ਭਾ":230," ਭੀ":16," ਬà©":13," ਭਰ":19," ਨਹ":38," ਨਿ":106," ਨਾ":302," ਨੂ":313," ਨੇ":127," ਦੱ":14," ਨਵ":51,"ਗਠ":15," ਦੇ":1425," ਦੋ":31," ਧਰ":56," ਪà©":69," ਪੀ":18," ਪਿ":76," ਪਾ":125," ਪੈ":40," ਪੇ":13," ਪੂ":34," ਪਰ":200," ਪਹ":57," ਪਟ":13,"ਕੋ ":15," ਲੱ":19,"ਈ ":351,"ੰਦੇ":65," ਵਖ":19,"ੰਦੀ":31,"ੰਦਾ":409," ਵਰ":63," ਵਧ":14," ਵੀ":103," ਵਿ":1204," ਵਾ":793,"ੰਦਰ":40," ਵਸ":13," ਵੇ":12," ਵੈ":16," ਵੰ":18," ਵੱ":108," ਸਕ":45,"ੰਬਰ":125," ਯਾ":23," ਰਚ":14," ਯੂ":72,"ਖੇ ":23," ਮੰ":56," ਰਾ":142," ਰਿ":33," ਰਸ":75," ਰਹ":56," ਰੇ":12," ਲਈ":76," ਰà©":23,"str":14,"ੰਪਿ":40," ਰੂ":37," ਰੋ":38," ਲਗ":21," ਰੱ":19," ਲਾ":29," ਲਿ":55," ਲੀ":380,"ਠ":114," ਲੇ":15," ਲੈ":20," ਲੋ":37,"Oly":12,"ਗਤ ":30,"ਲੰਪ":29,"ਲੰਡ":379,"ਲੱਗ":14,"ਂ ":2409,"ੰਜੀ":16,"ੰਜਾ":127," ਸਭ":32," ਸਬ":19," ਸਮ":69," ਸਨ":75," ਸਪ":16," ਸਰ":55," ਸਟ":13," ਸਤ":61," ਸਥ":20," ਸਦ":12,"ੰਤਰ":46," ਹਰ":55," ਸà©":21," ਸੋ":27,"ਅ ":12," ਸਾ":1296," ਸਿ":210," ਸ਼":257,"ਖੀ ":31," ਹਨ":519," ਸੇ":33," ਸੂ":56," ਸੀ":135," ਸà©":40," ਹੋ":233,"ੰਡਲ":14,"ੰਡੀ":26," ਸੱ":24,"ਆ ":416," ਸੰ":107,"ੰਡਾ":15," ਹਾ":41," ਹਿ":46," ਹੀ":66," ਹà©":491," ਹੇ":15," ਹੈ":1352,"ੰਡਰ":381,"ਟ ":103,"ਠ":29," ਠ":29,"ਜ ":145,"ਕਸ ":17,"ਚ ":1074,"ਘ ":72,"ਹਿ ":31,"ਗ ":162," ਚ ":20,"ਹਾ ":59,"ਖ ":140,"ਕ ":1139,"ਹੀ ":112,"ਓ ":12,"ੀ। ":55,"pic":12,"ਕਰ ":44,"ਵੰਡ":19,"ਰ ":1488,"ਵੱਖ":25,"ਹੇ ":27,"ਵੰਬ":36,"ਵੱਜ":17,"ਵੱਡ":34,"ਭ ":31,"ਹੈ ":423,"ਵੱਲ":20,"ਸਕਦ":19,"ਗਈ ":25,"ਮ ":350,"ਫ ":31,"ਕੇ ":100,"ਬ ":211,"ਓਲੰ":29,"ਪ ":461,"ਧ ":64,"ਨ ":1907,"ਥ ":48,"ਕਾ ":68,"ਦ ":171,"ਕਿ ":66,"ਣ ":217,"ਕੀ ":396,"ਹੋ ":30,"ਤ ":537,"ਠ ":22,"ਡ ":132,"ਖਣ ":24,"ਾ ":2802,"ਵਿਸ":21,"ਵਾਰ":49,"ਵਿਕ":13,"à© ":14,"ਵਾਲ":96,"ਵਾਸ":13,"ਵਿਗ":34,"ਵਿਖ":22,"ਵਿਚ":200,"à©€ ":2902,"ਵਿਦ":16,"ਿ ":127,"ਵਾਨ":14,"ਵਾਂ":742,"ਹ ":633,"ਸਰ ":20,"਼ ":152,"ਵ ":71,"ਸਾ ":23,"ਵੇਦ":48,"ਸ ":997,"ਵੇਂ":22,"ਸ਼ ":84,"ਲ ":1735,"ਸਸ ":13,"ਵਿੱ":870,"ਸੇ ":56,"ਹਨ ":79,"ਂ।":15,"ਕਨ ":15,"à©‹ ":213,"ਸੀ ":89,"à© ":22,"ਹਰ ":16,"ੇ ":2940,"ੈ ":444,"à©‚ ":303,"ਕਟ ":14,"ਰਸ਼":34,"ਰਸਾ":69,"ਰਸਿ":50,"ਰਹਿ":56,"ng ":24,"ਰਹੇ":17,"nce":16,"ne ":14,"ndi":15,"ਰਾਂ":87,"ਰਾਜ":89,"ਰਾਬ":13,"ਰਾਨ":31,"ਰਿਆ":72,"ਰਾਣ":19,"ਰਾਸ":25,"ਰਾਹ":24,"ਰਿਕ":21,"ਰਾਵ":14,"ਰਾਮ":35,"nd ":32,"ਰਿਤ":27,"ਰੀਆ":20,"ਰਮਾ":139,"ਮੰਡ":22,"ਮੰਨ":14,"ਲੇ ":121,"ਰਵਾ":29,"ਰਵਰ":37,"ਰੈਲ":39,"ਰੋਜ":15,"ਰੋਮ":30,"ਰੋੜ":12,"nte":17,"ੰਕ ":58,"ns ":12,"ਰੀਸ":12,"ਰੀਬ":18,"ਰੀਕ":94,"ਰਿਸ":18,"ਰਿਹ":15,"ੰਗ ":69,"। ":1125,"ਕਈ ":21,"ਰੂਪ":26,"ੰਘ ":71,"ਰੈਗ":367,"ਰੈਕ":12,"ਰੇਗ":15,"ਰੇਜ":79,"ੰਜ ":22,"ਲਮ ":13,"ਰਕੇ":20,"of ":41,"ਲਣ ":22,"ਰਕਾ":23,"à¨à¨—ਰ":22,"ਰਨਾ":16,"ਲਾ ":163,"ਰਦà©":28,"ਰਦੀ":16,"ਰਦਾ":42,"ਰਦੇ":29,"ਰਮਨ":15," ਈ ":16,"ਰਬੀ":20,"ਲੀ ":116,"ਯੋਗ":18," ਆ ":21,"on ":41,"ਰਤਾ":15,"ਰਤਿ":15,"ਰਤੀ":63,"ona":13,"ons":12,"ਯੂਨ":34,"ਯੂਰ":14,"ਯੂਲ":19,"ਲੀਪ":371,"ੱਤ ":62,"ਲੀਵ":25,"ੱਦ ":13,"ਲਾਂ":56,"ਲਾਈ":52,"ਵਲ ":18,"ਲਿਆ":33,"ਲੀਅ":22,"ld ":18,"ਲੀਆ":17,"ਲਾਵ":16,"ਲਿਖ":25,"ੱਧ ":29,"ਵੇ ":16,"ਲà©à¨¹":16,"ੱਲ ":23,"ਵਾ ":26,"ਲੋਮ":22,"ਲੋਂ":20,"ਲੋਕ":25,"ਲੋਗ":23,"ਵੀ ":125,"ਲੇਖ":15,"ਲੈਂ":19,"ੰਤ ":18,"mb ":14,"ਵਖ ":12,"ੰਡ ":46,"ਲਚਰ":23,"mer":15,"ਲਗਾ":13,"ੰਨ ":21,"lym":12,"ੰਧ ":16,"।ਇ":25,"ੰਦ ":43,"ੰਥ ":14,"mpi":14,"ਰੱਖ":17,"ੱਕ ":255,"ਰੰਥ":15,"ਵਨ ":15,"ੱਖ ":91,"mu ":47,"ੱਚ ":827,"ਆਉਂ":16,"ਅਨà©":13,"à©à¨¹à¨¾":96,"ਇਸ ":693,"ਰਾ ":103,"ਮਨੇ":20,"ਇਹ ":527,"ਮਨà©":16,"ਮਨੀ":13,"ਅਤੇ":407,"ਰਲ ":24,"ਭਾਈ":15,"ਭਾਗ":17,"ਭਾਰ":147,"ਭਾਵ":14,"ਭਾਸ":44,"ਅਜਿ":15,"ਇਲ ":16,"ਰਨ ":81,"ਰਮ ":38,"ਅਰਥ":18,"ਅਮਰ":51,"ਯਾ ":16,"ਬੋਲ":29,"ਰਡ ":23,"ਅਪà©":39,"ਰਤ ":82,"ਬà©à¨°":15,"à©à¨°à©‹":27,"à©à¨°à©‡":90,"à©à¨°à©ˆ":416,"ਰਣ ":20,"à©à¨°à¨®":14,"à©à¨°à¨¦":16,"à©à¨°à©€":65,"à©à¨°à¨¿":60,"à©à¨°à¨¾":30,"à©à¨°à¨¸":16,"à©à¨°à¨¹":30,"ਰਥ ":20,"à©à¨°à©°":13,"ਮੇਂ":24,"ਆਨੀ":21,"à¨à¥¤ ":23,"ਆਪਣ":51,"ਆਦਿ":12,"ਆਦਾ":18,"ਆਣਾ":27,"ਮੈਨ":17,"ਮਾਨ":46,"ਮਾਤ":26,"ਮਾਣ":116,"ਮਿਕ":21,"ਮਾਰ":62,"ਮਾਲ":18,"ਮਿਥ":28,"ਮਾਂ":39,"ਇਆ।":31,"ਮà©à©±":38,"ਮਿਲ":41,"ਮਿਸ":19,"ਮੀਟ":34,"ਮà©à¨–":14,"ਮà©à¨•":29,"ਮà©à¨¤":380,"ਰੋ ":14,"ਮਹਾ":43,"ਮਹਿ":32,"ਮਹੀ":13,"ਆਵਾ":13,"ਰੂ ":145,"ਭੌਤ":12,"ਆਰਾ":46,"ਰੀ ":651,"ਬੰਧ":25,"ਰੇ ":75,"ਲਈ ":78,"ਉਣ ":23,"ਮਰੀ":43,"ਇਥੇ":16,"ਬਣਾ":43,"ਇਨà©":18,"ਪੜà©":12,"ਫà©à¨Ÿ":13,"ਫ਼ਰ":40,"ਇਣਕ":56,"ਉਸ ":51,"ਉਹ ":40,"ਈਆਂ":13,"ਇਤਿ":19,"ਮਨ ":35,"ਇਸਦ":16,"ਪà©à¨°":176,"ਪੈਦ":15,"ਅੰਤ":31,"ਅੰਦ":15,"ਅੰਗ":91,"ਅੰਕ":65,"ਭੀ ":15,"ਇਲਾ":16,"ਈਨਾ":22,"ਰਜ ":32,"ਰਚ ":44,"ਬੀਜ":26,"ਰਗ ":14,"ਬਿੰ":25,"ਬਾਦ":22,"ਬਾਰ":32,"ਬਾਬ":12,"ਰਕ ":18,"ਬਾਕ":372,"ਬਾਲ":27,"ymp":12,"ਈਡਰ":15,"ਬਾਅ":27,"ਬਾਈ":14,"ਉਂਦ":22,"ਬਹà©":25,"ਭਗਤ":25,"ਮੇ ":12,"ਬਲਾ":20,"ਪੱਛ":12,"ਪੰਜ":152,"ਬਰਾ":15,"ਫà©à¨°":14,"ਮੀ ":18,"ਮਾ ":32,"ਸਮ":81,"ਸਭ":32,"ਸਬ":22,"ਸਫ":15,"ਸਪ":32,"ਸਨ":101,"ਸਵ":19,"ਸਲ":51,"ਸਰ":94,"ਾ।":27,"ਸਟ":58,"ਸਦ":49,"ਸਤ":207,"ਸਥ":50,"ਸਕ":73,"ਵੱ":108,"ਵੰ":62,"ਵੈ":22,"ਵੇ":119,"ਉਨà©":43,"ਵਸ":16,"ਵਿ":1234,"ਵੀ":154,"ਵਾ":1015,"ਵਨ":21,"ਵਧ":14,"ਵਰ":201,"ਵਲ":39,"ਵਖ":19,"ਲੰ":416,"ਲੱ":23,"ਲੜ":14,"ੌਰ ":34,"ਲੋ":133,"ਲà©":24,"ਲੇ":163,"ਲੈ":43,"ਲਿ":106,"ਲਾ":368,"ਲà©":15,"ਲੀ":580,"ਲਹ":12,"ਰੱ":21,"ਰੰ":40,"ਲਵ":25,"ਲਬ":13,"ਲਮ":25,"ਲਤ":16,"ਲਣ":25,"ਲਦ":24,"ਲਚ":23,"ਲਕ":25,"ਲਗ":28,"ਰੈ":428,"ਰੋ":128,"ਰà©":22,"ਰੀ":839,"ਰà©":36,"ਰੂ":205,"ਲਈ":79,"ਰੇ":213,"ਰਹ":94,"ਰਸ":179,"ਰਿ":221,"ਰਾ":502,"ਮੱ":15,"ਮੰ":60,"ਰਲ":37,"ਰਵ":92,"ੀਤ":114,"à©à¨†":59,"ੀਦ":20,"ੀਬ":34,"ੀਮ":70,"ੀਨ":92,"ੀਪ":396,"à©à¨–":20,"ੀਵ":67,"à©à¨•":56,"ੀਰ":67,"ੀਲ":39,"ੇ।":14,"à©à¨":24,"à©à¨Ÿ":25,"ੀਸ":29,"à©à¨œ":22,"ੀਆ":241,"ਿਥ":45,"ਿਦ":23,"ਿਣ":20,"ੀਅ":117,"ਿਤ":204,"ੀਂ":64,"ਿਡ":14,"ਿਟ":58,"ਿਮ":51,"ਿਬ":77,"ਬਰ ":175,"ਿਪ":16,"ਿਧ":20,"ਿਨ":1162,"ੀਗ":18,"ਿਵ":98,"ੀਕ":136,"ਿਲ":265,"ਿਰ":130,"ੀਟ":47,"ਾੜ":27,"ੀਜ":44,"ਿਹ":111,"ਿਸ":263,"ਾਡ":16,"ਇੰਟ":13,"ਾਣ":249,"ਿਅ":43,"ਾਤ":101,"ਿਆ":517,"ਾਦ":96,"ਾਧ":15,"ਿਉ":16,"ਾਨ":318,"ਿਊ":33,"ਾਪ":44,"ਾਬ":594,"ਾਮ":104,"ਾਰ":660,"ਾਲ":1569,"ਾਵ":109,"ਿਖ":72,"ਿਕ":215,"ਿਗ":45,"ਾਹ":169,"ਿਚ":202,"ਾਸ":245,"ਿਜ":25,"਼ਾ":203,"਼ਿ":116,"ਾਅ":32,"਼ੀ":109,"ਾਂ":1630,"਼à©":43,"ਾਈ":221,"ਾਉ":53,"਼ੇ":19,"ਾਇ":185,"਼ੋ":16,"਼ੈ":21,"ਬਲ ":16,"਼à©":25,"ਾਕ":443,"ਾਗ":63,"ਾਖ":18,"ਾਜ":126,"ਾਚ":13,"਼ਨ":40,"਼ਤ":27,"਼ਟ":24,"਼ਸ":37,"਼ਹ":70,"਼ਵ":12,"਼ਰ":65,"਼ਬ":33,"਼ਮ":21,"਼ਖ":29,"ਹੱ":25,"ਹੰ":15,"ਹੂ":17,"ਹà©":520,"ਹੈ":1355,"ਹੇ":48,"ਹਾ":329,"ੀ।":87,"ਹੀ":203,"ਹਿ":360,"ਸੰ":167,"ਸੱ":26,"ਹੋ":237,"ਹੌ":13,"ਸੂ":66,"ਸà©":43,"ਸੀ":222,"ਸੇ":97,"ਹਨ":550,"ਸਹ":12,"ਸਸ":15,"ਸਿ":313,"ਸਾ":1477,"ਸ਼":623,"ਹਲ":14,"ਸੋ":30,"ਹਰ":67,"ਹਮ":13,"ਸà©":26,"ਦਸ":57,"ਦਿ":1225,"ਦਾ":1748,"ਦà©":90,"ਦੀ":615,"ਦੂ":48,"ਥੇ":53,"ਦਨ":53,"ਥੋ":13,"ਦਰ":129,"ਤੰ":44,"ਦਲ":21,"ਤੱ":67,"ਧਾ":77,"ਨਜ":12,"ਨਡ":15,"ਧੀ":16,"ਧਿ":28,"ਨਦ":13,"ਨਤ":17,"ਦੇ":1613,"ਦੋ":54,"ਨà©à¨¸":12,"ਧਰ":65,"ਨਕ":44,"ਤਸ":17,"ਤਵ":19,"ਤੀ":231,"ਤੂ":46,"ਤਿ":96,"ਤਾ":728,"ਣੇ":53,"ਤਨ":20,"ਤਪ":13,"ਤਤ":12,"ਤਰ":186,"ਤਲ":37,"ਤਮ":27,"ਉਹਨ":12,"ਥੀ":15,"ਥਿ":46,"ਥਾ":70,"ਤੋ":518,"ਤੇ":631,"ਨà©à©±":16,"ਤੌ":43,"ਇੱਕ":245,"ਣਕ":72,"ਨੂੰ":329,"ਡੇ":30,"ਣਾ":137,"ਣੂ":108,"ਣਿ":62,"ਣੀ":85,"ਤਕ":48,"ਨੇਂ":14,"ਟà©":17,"ਟੇ":26,"ਨੇਜ":20,"ਟੀ":130,"ਟਿ":29,"ਡਿ":29,"ਡੀ":68,"ਨ।":476,"ਡਾ":86,"ਡਲ":20,"ਡਰ":410,"ਡਦ":12,"ਮਰ":69,"ਮਲ":18,"ਬੰ":44,"ਮਸ":29,"ਮਹ":96,"ਮà©":499,"ਮੀ":82,"ਮਿ":154,"ਮਾ":391,"ਮੂ":19,"ਮੈ":49,"ਮੇ":59,"ਮà©":15,"ਮੌ":19,"ਮੋ":28,"ਰਕ":76,"ਰਖ":17,"ਰਗ":45,"ਰਚ":72,"ਰਜ":81,"ਰਟ":18,"ਯਾ":37,"ਰਡ":28,"ਯੂ":73,"ਰਣ":36,"ਯà©":14,"ਰਥ":37,"ਰਤ":198,"ਰਦ":131,"ਰਨ":116,"ਰਫ":22,"ਰਪ":20,"ਯੋ":27,"ਰਬ":46,"ਰਮ":236,"ਪੰ":164,"ਬਲ":53,"ਪੱ":37,"ਬਰ":215,"ਫà©":15,"ਬਹ":34,"ਭਗ":29,"ਬà©":20,"ਬੀ":120,"ਬਿ":55,"ਬਾ":578,"ਬੋ":46,"ਬੈ":14,"ਬੇ":32,"ਮਈ":41,"ਮਕ":17,"ਬà©":15,"ਭਰ":22,"ਮਜ":12,"ਮਤ":30,"ਭਾ":256,"ਭੀ":18,"ਭਿ":14,"ਭੌ":12,"ਮਦ":18,"ਮਨ":98,"ਪਲ":21,"ਨੰ":12,"ਪਹ":57,"ਪਸ":12,"ਪਰ":222,"ਪੂ":52,"ਪੈ":44,"ਪੇ":23,"ਪੀ":37,"ਪà©":90,"ਪਾ":168,"ਪਿ":130,"ਬਕ":380,"ਪੋ":22,"ਪà©":176,"ਨà©à¨¹":75,"ਫਰ":17,"ਫਲ":16,"ਬਦ":46,"ਬਨ":25,"ਫ਼":106,"ਪੜ":21,"ਫਾ":21,"ਫਿ":21,"ਫà©":23,"ਬਣ":71,"ਨਵ":102,"ਦੱ":14,"ਨਲ":14,"ਨਰ":14,"ਨਮ":39,"ਨੇ":196,"ਨੂ":341,"ਨੀ":200,"ਨà©":33,"ਨਾ":539,"ਨਿ":163,"ਨਸ":36,"ਨਹ":40,"ਨà©":89,"ਨੈ":12,"ਨੋ":19,"ਪਨ":15,"ਪਣ":56,"ਪਤ":28,"ਪਟ":24,"ਬਦ ":20,"ਕਰ":232,"ਕਮ":20,"੨ ":59,"ਕਲ":439,"ਕਨ":33,"ਕਦ":33,"ਕਟ":31,"ਕਤ":65,"੧ ":77,"ਕਈ":21,"ਕੱ":19,"ਕੰ":48,"ਗਲ":25,"ਗਰ":511,"੪ ":44,"ਖੋ":29,"ਖੇ":111,"ਗਦ":22,"ਗਣ":22,"ਗਤ":41,"ਖੀ":35,"ਖਾ":81,"ਖਿ":54,"ਖ਼":60,"ਕੜ":15,"ਕà©":37,"ਗà¨":21,"ਖਰ":32,"ਕੌ":24,"à©© ":43,"ਕੋ":104,"ਗਈ":32,"ਖਦ":15,"ਕੈ":22,"ਕੇ":135,"ਕà©":80,"ਕੂ":15,"ਖਣ":34,"ਖਤ":12,"ਕਾ":323,"ਕਿ":322,"ਕੀ":517,"ਕਹ":19,"ਕਵ":12,"ਕਸ":64,"à¨à¨²":14,"à¨à¨—":22,"ਪਹਿ":45,"à¨à¨¸":17,"ਓਲ":31,"੦ ":87,"ਜੇ":38,"ਜੈ":24,"ਜੋ":120,"ਜ਼":304,"ਜਾ":547,"ਜਿ":213,"ਜੀ":234,"ਜà©":64,"ਜੂ":46,"ਪਿਤ":20,"à©° ":328,"ਪਾਸ":19,"ਪਿਕ":29,"ਜਦ":23,"ਪਾਰ":18,"ਜਧ":12,"ਜਨ":116,"ਪਿਊ":14,"ਪਾਣ":18,"੯ ":52,"ਛੋ":17,"ਚੱ":12,"ਪਾਕ":32,"ਚੰ":38,"ਜਲ":16,"ਜਰ":27,"ਟਬ":13,"ਟਨ":25,"ਪਾਈ":14,"ਟਾ":78,"ਟਰ":121,"ਜੰ":41,"ੋਗਰ":27,"ਗੜ":29,"੬ ":50,"ਚਕ":12,"ਗਸ":65,"ਗਾ":75,"ਗੂ":16,"ਗà©":197,"ਗੀ":24,"ਗਿ":132,"ਪੂਰ":40,"ਗੇ":24,"ਗà©":505,"ਗੋ":52,"à©« ":50,"ਘਰ":19,"ਚਿ":28,"ਚਾ":99,"ਚà©":16,"ਚੀ":48,"ਚੇ":17,"ਪਿੰ":41,"ਪà©à¨°":63,"ਚੋ":42,"ਚੌ":23,"à©® ":73,"ਛਮ":12,"ਜਗ":13,"ਚਨ":19,"ਚਰ":32,"à©­ ":49,"ਚਲ":24,"ਅਤ":454,"ਆਂ":297,"ਅਜ":34,"ਅਗ":69,"ਅਕ":61,"ਆਣ":33,"ਇਆ":135,"ਆਖ":12,"ਅਸ":23,"ਅਮ":113,"ਅਰ":63,"ਅਲ":18,"ਅਦ":42,"ਆਇ":12,"ਅਧ":17,"ਆਉ":22,"ਅਨ":55,"ਅਪ":47,"ਪਣੇ":34,"ਈਆ":13,"ਇਥ":21,"ਇਤ":25,"ਇਣ":62,"ਆਸ":22,"ਪਣੀ":15,"ਆਰ":95,"ਆਮ":24,"ਇਕ":101,"ਆਵ":17,"ਆਲ":16,"ਆਨ":54,"ਆਦ":35,"ਆਬ":13,"ਆਪ":78,"ਆ।":54,"ਬਾ ":22,"ਂਕ":18,"ਂਗ":31,"ਈ।":12,"ਂਟ":25,"ਂਡ":36,"ਂਸ":30,"ਂਤ":24,"ਂਦ":244,"ਉੱ":56,"à©‹à©œ ":16,"ਬੀ ":78,"à¨à¨¨":15,"à¨à¨«":13,"ਇਨ":48,"ਅੱ":34,"ਇਲ":37,"ਅੰ":220,"ਈਟ":14,"ਇਹ":542,"ਮਈ ":41,"ਇਸ":741,"ਉਂ":46,"ਈਡ":27,"ਈਨ":32,"à¨à¥¤":33,"ਬੇ ":13,"ਉਦ":21,"ਉਨ":51,"ੋਇਆ":71,"ਉਣ":28,"ਉਤ":19,"ਉਪ":26,"à©œ ":44,"ਇੱ":255,"ਇੰ":32,"ਉਸ":76,"ਉਹ":56,"ਊਟ":14,"ਪਰਮ":113,"ਨਕਸ":15,"ਪਤ ":12,"ਧਰਮ":31,"ਧਰਤ":22,"ਪਰ ":49,"ਧਾਂ":14,"ਉੱਤ":40,"ੋਬਿ":25,"ਦà©à¨†":48,"ਦਿੱ":27,"ਦà©à¨¨":24,"ੋਮੀ":24,"ਦੂਸ":12,"ਦੂਜ":16,"ੋਮਨ":25,"੦੦":66,"੦੮":25,"੧੦":19,"੧੩":12,"੧੪":15,"੧੧":14,"੧੨":18,"੧੭":17,"੧੮":24,"੧੫":17,"੧੬":17,"੧੯":42,"੨੧":14,"੨੦":53,"ਦਾਂ":14," ੧੮":24," ੧੭":16,"ਦਾਨ":17," ੧੬":17," ੧੫":16," ੧੪":15," ੧੩":12," ੧੨":16," ੧੧":13,"ੋਲੀ":19,"ਦਾਰ":26," ੧੯":41," ੨੦":53," ੨੧":14,"ਦਿਆ":31," ੨੭":12," ੨੬":12," ੨੯":17," ੨੮":14," ੨੩":13," ੨੨":14," ੨੫":13," ੨੪":14,"ਦਾਸ":19," ੩੦":25," ੩੧":16,"ਦੀਆ":75,"ਦਿਨ":1097,"umb":16,"à©à¨¹ ":21," ੧੦":19,"ਦੋਂ":17,"੩੧":18,"੩੦":26,"um ":31,"੨੯":17,"੨੮":15,"੨੭":13,"੨੬":13,"੨੫":14,"੨੪":14,"੨੩":13,"੨੨":14,"ੱਤ":172,"ੱਢ":13,"ੱਡ":45,"ੱਧ":49,"ੱਦ":25,"ੱਥ":25,"ੱਗ":23,"ੱਕ":286,"ੱਖ":185,"ੱਛ":18,"ੱਜ":30,"ੰਸ":52,"ੱਚ":879,"ੱਟ":28,"ੱਠ":21,"ੱਲ":52,"ੱਸ":22,"ੰਥ":18,"ੰਤ":85,"ੰਧ":36,"ੰਦ":618,"ੰਡ":502,"ੰਬ":155,"ੰਮ":32,"ੰਨ":45,"ੰਪ":49,"ੰਕ":75,"ੰਖ":19,"ੰਗ":218,"ਦੇਸ":44,"ਦੇਵ":25,"ੰਜ":177,"ੰਟ":22,"ੰਘ":85,"ਨਵਰ":42,"ਨੀਆ":32,"ਨੀਅ":13,"ਨਿਵ":62,"ਨੀਕ":15,"ਨਿਆ":24,"ਬਕ ":378,"ਨਾਨ":32,"ਨਾਲ":179,"ਨਾਵ":16,"ਨਿਕ":18,"ਨਾਮ":38,"à©œà©":38,"ਨਾਂ":88,"ੜੀ":29,"ੜਾ":29,"ੜੇ":20,"ty ":13,"ਨਸੰ":13,"ਨਹੀ":36,"à¨à¨« ":13,"ਨਵੰ":36,"à©à¨¦":13,"ੂਆ":13,"à©à¨¨":42,"à©à¨¤":412,"à©à¨£":26,"à©à¨®":18,"à©à¨°":279,"à©à¨¸":47,"ਿੰ":258,"à©à¨²":97,"ਿੱ":1026,"ੈ।":922,"ੂਜ":17,"ੂਨ":78,"ੂਦ":14,"ੂਰ":124,"à¨à¨² ":13,"ੂਬ":50,"ੂਪ":31,"ੂਸ":23,"ੂਲ":53,"à©à©°":490,"à©à©±":117,"à©‚à©°":335,"ੇਂ":94,"ੇਕ":12,"ੇਖ":36,"ੇਦ":53,"ੇਤ":52,"ੇਡ":47,"ੈਂ":49,"ੇਟ":14,"ੇਜ":108,"ੇਗ":24,"ੇਰ":47,"ੇਲ":44,"ੈਕ":39,"ੇਵ":52,"ੈਗ":377,"ਧਾਰ":30,"ੇਨ":27,"ਧਾਨ":15,"ਧਿਆ":15,"ੈਣ":13,"ੈਦ":24,"ੇਸ":109,"ੈਟ":17,"ੈਲ":73,"ੈਰ":17,"ੈਨ":38,"ੈਸ":23,"ੋਂ":617,"ੋਇ":77,"ੋਈ":46,"tio":34,"thu":15,"ੋਟ":27,"ੌਂ":47,"ੋਡ":13,"ੋਜ":32,"ੋਧ":14,"ੋਨ":32,"ੋਪ":22,"ੋਣ":30,"ੋਤ":20,"à©‹à¨":51,"ੋਗ":55,"ੋਚ":14,"ੋਕ":33,"ੌਜ":14,"ੌਤ":23,"ੋਮ":61,"ੋਬ":36,"ੋਲ":71,"ੋਰ":92,"ੋਵ":23,"ੋਹ":24,"ੋਸ":21,"à©‹à©œ":31,"ੌਮ":17,"ੌਰ":50,"ted":14,"à©à¨¹":146,"à©à¨°":834,"ter":25,"the":71,"ਆ। ":38,"ਤੂਬ":34,"ੇਲ ":13,"ੇਰ ":12,"ਤੀਆ":19,"ੇਸ ":12,"ਤਿਹ":21,"ਤਿਆ":30,"ਤਾਨ":49,"ਤਿਕ":18,"ਤਾਬ":383,"ਨਕ ":14,"ਤਾਰ":18,"ਤਾਂ":61,"ੇਵ ":19,"ਨਮ ":32,"ਤੌਂ":25,"ਤੋਂ":485,"ੈਨ ":18,"ਤੌਰ":15,"ੈਲ ":41,"ਨਾ ":113,"ਥਾਂ":13,"ਥਿਤ":12,"ਥਿਹ":26,"ਥਾਨ":18,"ਦਸੰ":39,"ਨੀ ":107,"ੇਗਰ":13,"ਦਰਿ":26,"ਨੇ ":143,"ਤੰਬ":41,"ਤੱਕ":12,"ਤੱਤ":53,"ਂਟ ":12,"ੇਜੀ":27,"ੇਜ਼":73,"ੇਡਾ":24,"ਂਡ ":22,"ੈਂਡ":21,"ੇਡਦ":12,"ਂਗ ":18,"ੇਦਨ":48,"ੇਤੀ":16,"ੇਤਰ":23,"ੋਂ ":614,"ਤੋ ":20,"ਣਕਾ":14,"ਤਕਨ":13,"ਥੇ ":48,"ਦਨ ":52,"ੈਕਟ":16,"ੋਈ ":39,"ce ":16,"ੈਗਰ":366,"ੋਠ":45,"ਥਾ ":23,"ੇਵਾ":19,"ੇਸ਼":80,"am ":12,"ੋਕ ":20,"al ":27,"ਣਿਤ":17,"ਣਿਆ":40,"and":32,"amu":47,"an ":23,"ੈਦਾ":18,"ਦਰ ":34,"ਣਾਇ":20,"ੌਂ ":41,"ੋਣ ":17,"ਦੋ ":25,"ਤਰਾ":24,"ਤਰੀ":36,"ੋਪ ":15,"ਦੇ ":1528,"at ":19,"ਦੀ ":510,"as ":15,"ੋਰ ":44,"ੋਲ ":12,"ਦਾ ":1619,"ati":17,"ਦਿ ":15,"ੜਾ ":14,"à©à¨Ÿà¨¬":13,"ੀਸਟ":12,"ੀਵਨ":13,"ੀਵਰ":32,"à©à¨•à¨¾":27,"ਤਕ ":19,"ੜੀ ":26,"ੀਮਾ":12,"�":48,"à©à¨°à¨¦":34,"ਡਦੀ":12,"ੜੇ ":18,"à©à¨¨à¨¿":12,"à©à¨¨à©€":20,"ਣੀ ":69,"à©à¨¤à¨¾":380,"ਣਾ ":86,"ਡਰਾ":13,"ਣੂ ":105,"ਣੇ ":51,"ਆਂ ":277,"à©à¨¸à¨¼":14,"ਂਦਰ":14,"ਂਦੀ":34,"ਂਦਾ":168,"ਂਦੇ":20,"ੈ।ਇ":14,"à©‚à©° ":327,"ਤਰ ":83,"ਅਨ ":34,"ਡਾਂ":31,"à©à¨°à©‚":142,"à©à¨°à¨¾":23,"à©à¨°à¨¸":21,"ਿੰਦ":69,"ਿੰਡ":45,"ਿੰਘ":68,"ਿੰਗ":49,"ਡੀਅ":12,"ਿੱਤ":44,"ਿੱਧ":18,"ਿੱਖ":61,"ਅਤ ":35,"ਿੱਚ":863,"à©à¨²à¨¾":52,"à©à¨²à©€":15,"ਅਦ ":27,"ੂਰਜ":24,"ੂਰਬ":14,"ੂਬਰ":34,"ਤਾ ":184,"ਤੀ ":175,"ੂਨਿ":14,"ੂਨੀ":19,"ਤੇ ":613,"ਜੰਤ":20,"ਜੰਗ":15,"ਡਰ ":388,"ਆਨ ":22,"ੂਲੀ":19,"ਇਆ ":100,"ਆਪ ":24,"ਅਮ ":45,"à©à©±à¨–":54,"ਅਰ ":17,"à©à©°à¨¦":474,"ਡਲ ":18,"ਟਬਾ":13,"ਡੀ ":25,"ਡਾ ":35,"ਨ। ":77,"ੇਂ ":52,"ਡੇ ":21,"à©œà©à¨¹":37,"ਟਰੀ":23,"ਟਾਂ":13,"ਟਾਇ":14,"ਅਕਤ":41,"ਅਕਾ":15,"ਅਗਸ":63,"ਆਮ ":19,"ਆਰ ":21,"ਣਕ ":58,"ਟਿਆ":13,"ਇਕ ":86,"ਟੀਮ":54,"ਟà©à¨°":17},"n_words":[112478,136533,89577],"name":"pa","type":"devanagari"} \ No newline at end of file
diff --git a/contrib/languages-data/pl.json b/contrib/languages-data/pl.json
new file mode 100644
index 0000000..3ddb2f6
--- /dev/null
+++ b/contrib/languages-data/pl.json
@@ -0,0 +1 @@
+{"freq":{"D":76546,"E":48349,"F":59499,"G":97533,"A":113161,"B":112599,"C":110974,"L":74803,"M":131493,"N":86269,"O":63924,"H":50582,"I":84977,"J":54373,"K":109538,"U":33818,"T":71466,"W":122172,"V":29111,"P":225853,"S":204600,"R":87715,"Y":5329,"X":17841,"Z":57656,"f":154643,"g":760584,"d":1304112,"e":3541226,"b":467514,"c":1792078,"a":3833729,"n":2529442,"o":3537464,"l":1140017,"m":1150421,"j":965827,"k":1483844,"h":558914,"i":3732766,"w":2322567,"v":42704,"u":1069834,"t":1619824,"s":1819258,"r":2204575,"p":1082020,"z":1828688,"y":1372315,"x":13069,"é":11522,"á":5335,"ü":5350,"ö":4590,"ó":396123,"Ä™":271422,"ć":60605,"Ä…":370153,"Å›":279181,"Åš":15543,"Å„":123462,"Å‚":591480,"Å":9157,"ż":255164,"Å»":6256,"ź":20372," l":139321," m":204189," n":260999," o":286725," h":33638," i":177645," j":136928," k":213385," d":243257," e":37159," f":50076," g":203060,"Ñ€":5358," a":148427," b":95754," c":133679," z":321847," u":97002," t":172994," w":969288," p":661390," s":339763," r":209751," J":52915," K":104698," H":47495," I":52640," N":80062," O":56243," L":69035," M":123296," B":105665," C":84648," A":95983," F":55238," G":73890," D":70098," E":42554," Z":55304,"к":5011," X":11746,"и":6265,"о":7740,"н":5938," S":182443," R":76715," P":214389,"а":9102," W":115231," V":20154," U":30651,"е":5223," T":64394," ż":19337," Å»":6158,"ęśc":13317," Å›":51887," Åš":15346," Å":9064," Å‚":17467,"A ":12056,"Da":8795,"Cz":10287,"Co":11705,"Ce":7894,"Ch":21362,"Ci":4668,"G ":10044,"Du":4576,"Do":17200,"Dr":5145,"De":9899,"Di":5692,"GC":12708,"Fe":4746,"Eu":8774,"Ge":6043,"Ga":8896,"I ":22055,"Fr":25795,"Fo":4852,"Fi":6166,"B ":5264,"C ":19133,"Au":8116,"Ar":11839,"Ba":23485,"Am":7571,"An":13752,"Al":15057,"Bu":9080,"Br":16403,"Ca":12445,"Bi":14211,"Be":12406,"Bo":15278,"Ku":5947,"Gó":8233,"Kr":17850,"Ko":31811,"Le":13922,"Li":15420,"La":13916,"Lu":9458,"Lo":10236,"Me":12645,"NG":5663,"Mi":29200,"O ":4778,"Ma":41474,"Mu":5320,"Mo":18522,"Ni":19737,"Ne":7072,"Na":24418,"P ":7412,"No":17077,"Ol":5489,"Od":7399,"PG":5183,"Ob":6870,"Gm":6336,"Gr":14758,"Go":7919,"Ha":11794,"He":11337,"II":18254,"Hi":5668,"Ho":9461,"In":12447,"Ja":16186,"L ":4866,"Je":18291,"Jo":8139,"Ka":25307,"M ":4990,"Ki":5813,"Un":7211,"VI":4920,"W ":15791,"Tu":5606,"Tr":10180,"To":9177,"Th":7956,"Te":9915,"Ta":10132,"V ":6364,"Sz":17267,"Sy":6237,"St":34755,"Su":6926,"Wo":12341,"Ws":4692,"Wi":28290,"Wa":16039,"We":8452,"Vi":5052,"X ":6678,"Pu":5975,"Pr":25703,"S ":11205,"Pe":7756,"Pa":34023,"Po":99382,"Pi":15484,"Os":6465,"Or":5855,"Se":13365,"Sc":7388,"Si":11311,"Sk":5337,"Sp":7038,"So":10398,"Ru":5061,"Rz":5267,"Sa":22530,"Re":17869,"Ro":19242,"Ra":13392,"b ":46793,"a ":1141880,"Wy":14854,"Za":17995,"SÅ‚":20200,"i ":569208,"bó":4844,"gd":5266,"ge":34328,"ga":71454,"ać":8134,"fi":50271,"fr":12808,"fu":7503,"fo":23920,"j ":229252,"bÄ™":5925,"có":12082,"gw":10035,"he":43730,"ha":50953,"gn":13137,"gm":84581,"gl":15240,"bÄ…":7905,"gi":77017,"gh":5124,"gu":37274,"gr":81461,"go":201815,"du":56828,"dw":18508,"dy":56165,"dz":249442,"g ":46202,"ea":22981,"eb":20189,"ec":208980,"ed":135395,"de":106238,"dd":7144,"di":38677,"dk":20908,"dm":14590,"dl":41406,"do":162506,"dn":125390,"dp":5891,"ds":17240,"dr":43512,"ew":111327,"eu":15671,"ev":5608,"ey":6513,"ez":85170,"fa":15386,"h ":260015,"fe":16025,"eg":232853,"ef":12705,"ee":8458,"el":150281,"ek":109837,"ej":291522,"ei":21799,"ep":45340,"eo":16834,"en":264211,"em":150465,"et":92062,"es":172911,"er":319668,"ca":105725,"e ":876959,"by":41730,"bs":19604,"br":47012,"bu":41705,"bn":8407,"bo":39861,"bl":23444,"bi":69531,"be":48450,"dc":10689,"db":7638,"da":129820,"f ":12810,"cz":335815,"cy":107837,"cu":14533,"ct":15111,"co":51690,"cn":22110,"ck":80920,"ci":275500,"cj":115414,"ch":414464,"ce":179393,"c ":34764,"az":93181,"ay":6838,"ba":43868,"d ":139962,"at":193861,"as":161357,"ar":314898,"aw":119971,"av":7852,"au":36829,"ak":99776,"al":185695,"ai":24623,"aj":148717,"ap":33608,"am":123297,"an":453452,"ac":219676,"ad":149992,"ab":34080,"ag":38473,"ah":5887,"ae":10185,"af":22452,"nu":23194,"nt":96712,"ns":36134,"nr":4954,"no":152706,"nn":38516,"ny":249928,"oe":4808,"of":20662,"oc":101234,"od":293823,"oa":8273,"ob":82692,"iÄ™":123546,"om":108472,"on":318937,"ok":137945,"ol":232069,"oi":39360,"jÄ…":114675,"oj":102542,"og":59938,"oh":7680,"ot":70221,"os":163642,"ov":6489,"ou":18871,"op":86592,"oo":8474,"or":254276,"jÄ™":19551,"r ":119247,"ow":537395,"kó":27850,"oz":62053,"pe":42396,"pa":107114,"pc":8415,"pl":33619,"pn":11150,"po":424408,"ph":5594,"iÅ‚":21358,"pi":86460,"kÄ…":16568,"iÅ„":19637,"lo":81679,"ln":79071,"lm":9683,"ll":32626,"ls":113022,"hó":9384,"lu":76951,"lt":13727,"o ":461084,"dź":5330,"eÅ›":75573,"mc":10115,"ma":121066,"mb":17385,"me":89660,"mi":333776,"mn":11655,"mm":6313,"mp":24445,"mo":86555,"ms":11588,"mu":41658,"ió":8421,"my":18150,"p ":18753,"na":529472,"nb":5437,"nc":75955,"nd":63112,"ne":249253,"eż":40953,"nf":7374,"ng":54651,"gÅ‚":47489,"ni":692138,"iÄ…":60315,"nk":45573,"jw":6916,"ju":20103,"js":79998,"jn":43747,"jo":26898,"jm":12943,"dÅ‚":15854,"ki":412466,"ke":10550,"kc":19574,"ka":229229,"m ":343638,"kw":12402,"gó":15509,"ks":37081,"kt":82034,"ku":103176,"ko":241820,"kr":97482,"kl":22672,"km":9933,"eÅ‚":15052,"li":209435,"eÅ„":19329,"lk":31238,"le":182455,"ld":11857,"lg":4525,"la":173548,"lc":6468,"lb":16636,"n ":138864,"hr":17682,"dó":7283,"ht":5694,"hu":10241,"cÄ…":8049,"aÅ„":46393,"aÅ‚":115023,"hi":35646,"hn":15999,"ho":81700,"id":59864,"ic":201937,"aÅ›":6694,"ib":17573,"ia":315115,"ig":21890,"if":5037,"ie":1136068,"hy":6565,"k ":111343,"ir":31982,"is":139600,"it":68750,"iu":39428,"iv":6169,"iw":23202,"ii":58678,"ij":14562,"dÄ…":5425,"ik":74901,"il":54561,"im":169852,"in":280897,"io":133204,"ip":17035,"aź":5844,"jc":7619,"aż":13250,"je":192771,"jd":20070,"ji":76024,"iz":35250,"l ":52041,"ja":94283,"sÄ…":9617,"tó":52700,"wz":9495,"wy":175884,"só":9957,"rÄ™":14692,"z ":208589,"oż":86148,"wi":428233,"pÅ‚":15785,"rÄ…":23256,"wk":11864,"wn":80692,"wo":213117,"wr":13531,"ws":100565,"wu":6452,"ró":58140,"y ":457874,"wa":271461,"oÅ›":108251,"wc":28885,"we":125766,"oÅ„":26624,"oÅ‚":121062,"vi":10565,"uz":15301,"uw":4835,"ve":14040,"va":7846,"x ":7519,"ui":5984,"uj":65669,"uk":36207,"ul":35319,"ue":10441,"ug":25222,"ur":117881,"us":71220,"ut":47463,"um":45758,"un":53367,"up":30095,"ty":157784,"tz":6072,"tu":85903,"tt":12443,"pó":31075,"tw":127984,"ub":63557,"ua":10045,"ud":56663,"uc":35683,"w ":734288,"to":228890,"tn":41685,"tl":10672,"ts":8380,"tr":134088,"te":190777,"kż":7374,"tk":33051,"ti":40445,"th":19993,"nÄ™":6037,"ta":271329,"su":26228,"ss":16223,"st":440956,"sy":40789,"sz":185716,"sw":7100,"sl":4597,"sk":362869,"sn":10482,"sm":8241,"sp":72745,"so":57024,"sc":108490,"se":50637,"sh":8501,"nÄ…":21831,"sj":10343,"si":154595,"rz":330244,"u ":285894,"mÄ™":6084,"sa":75170,"rr":8977,"rs":83617,"rt":71318,"ru":79284,"nó":11750,"rw":35761,"ry":128901,"rp":13436,"ro":332963,"rn":56890,"rm":40806,"rl":14366,"rk":29749,"ri":79907,"kÅ‚":24507,"rg":38504,"rf":5604,"re":197175,"iż":7681,"rd":33902,"rc":41834,"jÅ›":4511,"rb":12273,"ra":338168,"t ":152714,"lÄ™":5780,"mó":8836,"lÄ…":12242,"s ":109715,"kÄ™":4736,"py":18332,"pt":9892,"pu":38310,"pr":207712,"ps":10578,"yÅ‚":33312,"yÅ„":8163,"zÄ…":41404,"zÄ™":35066,"uż":24054,"wÅ‚":15632,"zó":6325,"wÄ™":7097,"uÅ‚":11203,"wÄ…":11216,"wó":72388,"zg":11821,"zi":142936,"sÅ‚":33435,"zb":26540,"zc":34117,"zd":18869,"ze":326759,"tÄ™":19569,"za":220009,"yz":12669,"zw":54934,"zy":216261,"zr":6357,"zu":31373,"zt":75735,"zo":93735,"zn":149848,"zp":13878,"zk":54785,"zj":15781,"zm":18594,"zl":6622,"yg":10175,"yf":6219,"yc":219665,"yd":32633,"yb":20679,"yw":52492,"yt":47612,"ys":92766,"yr":14041,"yp":20913,"yn":66941,"ym":103256,"yl":20573,"yk":74193,"yj":45803,"tÄ…":4964,"yż":9849,"yÅ›":4897,"zÅ‚":13341,"ów":161116,"ób":6381,"ój":5872,"ód":78413,"óc":5492,"ór":65730,"ól":16369,"ć ":59245,"Ä…d":23612,"Ä…c":135745,"Ä… ":117382,"óż":8608,"ół":34199,"Ä™b":8256,"Ä™c":19527,"Ä™d":32895,"Ä™g":11685,"Ä™k":11873,"Ä™p":16514,"Ä™t":22906,"Ä™s":9306,"Ä™ ":96973,"Ä™z":11858,"ęś":18258,"ęż":7758,"Ä…g":22092,"Ä…s":12009,"Ä…t":12097,"Ä…z":17763,"ąż":21246,"Å‚ ":59292,"Å‚u":50120,"Å‚t":4875,"Å‚y":40224,"Å‚k":15386,"Å‚n":20167,"Å‚o":189649,"Å‚e":22223,"Å‚c":7515,"Å„ ":16610,"Å‚a":119873,"Å„s":75332,"Å„c":31072,"łó":33659,"Å›l":22153,"Å›n":19502,"Å›r":30296,"Å›w":24072,"Å›c":92180,"Åšw":8984,"Å› ":45778,"ść":40444,"Å‚Ä™":5560,"Å‚Ä…":10057,"źn":6441,"źd":5042,"ż ":20942,"żu":6315,"ży":40208,"żo":72896,"żn":19632,"żs":5910,"że":31935,"ża":27767,"żą":12050,"ąża":15191,"Ä™ci":10430,"Ä™dz":16930,"Ä™du":4976,"Ä™zy":9477,"Ä™pu":8551,"Ä™ks":6674,"Ä™st":6004,"Å‚ec":8332,"Å‚a ":38802,"Å‚an":5477,"Å‚ad":27517,"Å‚ac":9469,"Å‚aw":16966,"Å‚as":4900,"Å‚oż":68106,"Å‚oÅ›":19952,"Å‚y ":21321,"Å‚yw":5534,"Å‚u ":9627,"Å‚oÅ„":16090,"Å‚ud":14450,"Å‚ug":11020,"Å‚ow":21447,"Å‚os":9246,"Å‚on":8256,"Å‚no":15191,"Å‚od":6015,"Å‚ka":8122,"Å‚o ":21533," Ga":8827," Ge":5971," I ":4633," Fo":4786," Fr":25744," Fi":5918," Ha":11737," He":11280," Go":7898," Gr":14678,"Å„có":5655," Gm":6328," Ho":9424," II":9642," Hi":5632," Je":18244," Ja":16067," In":12303," Ka":25219," Ki":5713," Jo":8117," La":13845," Le":13833," Li":15290," Ko":31759," Kr":17816," Ku":5884," Gó":8232," Ma":41192," Mi":29080," Me":12575," NG":5574," Lo":10195," Lu":9428," Ne":6981," Na":24184," Ni":19681," Mo":18451," Mu":5268," Am":7370," An":13699," Al":15025," Ba":23126," Au":8090," Ar":11761," Be":12343," Bi":14133," Bo":15194," Br":16331," Bu":9050," Ca":12220," Ce":7873,"Å„cz":4853," Ci":4561," Ch":21239," Co":11557," Cz":10255,"Å„ce":15534," Da":8706," Di":5639," De":9843," Dr":5105," Do":16986," Du":4568," Eu":8764," Fe":4580," Wy":14786," Ws":4674," Wo":12135," Wi":28155," We":8391," Wa":15970," SÅ‚":20157," Za":17851," a ":33564," Os":6455," Or":5834," Po":99137," Pi":15394," Pe":7708," Pa":33893," No":17025," Ol":5468," Od":7369," PG":4658," Ob":6845," Ra":13288," Ro":19091," Re":17774," Pr":25548," Pu":5949," Sz":17219," Sy":6223," Su":6907," St":33766," Ta":10078," Th":7821," Te":9806," Tr":10135,"Å„st":9625," To":9099,"Å„sk":65260," Ru":5052," Sa":22468," Rz":5260," Si":11239," Sc":7241," Se":13251," So":10324," Sp":6969," Sk":5318," Vi":4980," Tu":5482," W ":13679," Un":6885," ja":24161," je":81709," im":8378," in":25684," is":7255," ka":27845," ki":9981," gw":7366," j ":15130," ha":5671," he":5463," gm":81906," gr":42165," go":8008," hi":10476," hr":4991," gÅ‚":25015," ni":54044," na":181345," mu":9539," mo":21735," ok":44273," on":4611," og":5234," od":78508," of":7838," ob":36134," no":11764," le":15807," li":34396," la":36395," ku":10205," kt":33550," ks":6999," kw":7071," km":9336," kl":10638," kr":29471," ko":64410," me":15598," mi":86248," o ":18463," ma":52176," lu":42219," lo":7327," ad":8368," am":10571," an":21235," ak":7205," al":15945," au":9619," ar":12965," as":16623," ba":14299," bi":14788," be":7689," bo":7340," by":20334," bu":7038," br":13094,"Å‚uż":7054," ca":5242," el":9782," fa":6215," fu":5153," fr":6875," fo":9292," fi":17828," ge":6292," ga":16786," i ":123913," co":7528," ce":14033," ch":18489," ci":24028," da":17665," cy":4833," cz":55270," do":83523," dn":7531," dl":8990," dr":18598," de":38500," di":6649," dw":10226," du":6950," dz":27524," dy":8440," zm":8262," zo":12577," zn":26389," zw":23230," za":89284," zd":5616," ze":19904," zb":9872," zi":4717," sÅ‚":12543," sÄ…":8630," z ":107756," wy":79384," wz":8136," wÅ‚":13150," uż":7103," ru":4802," ry":6135," rz":19343," sa":14213," se":14117," si":84234," sk":26716," sp":31645," so":11139," ra":15542," re":46718," ro":89060," pu":7297," pr":169964," os":21891," ot":4775," op":16859," or":30563," r ":7257," jÄ™":8026," oz":4712," pe":8239," pa":50520," pl":22703," po":329900," pi":36353," wa":17275," we":29944," wc":10065," ró":17486," wr":10056," wo":81271," ws":27715," wi":85470," pÅ‚":9284," w ":591873," ty":20926," tw":5981," pó":15190," tu":7399," us":6046," ut":8073," ur":32556," uk":7357," ul":5488," ta":20050," sw":5466," sz":26144," sy":14406," st":72132," su":7093," tr":22707," to":40581," th":4944," te":42334,"łów":28362," Å‚a":6906,"Å‚Ä…c":7474," Åšw":8979," Å›r":24528," Å›w":19258," że":5562," ży":7572,"GC ":12479,"Eur":7453,"Fra":21145,"II ":13664,"Her":4920,"Gra":4795,"Gmi":6178,"Bar":4693,"Bra":4520,"Cha":6684,"Dol":5831,"Nie":14925,"Now":7712,"Nor":6043,"PGC":4549,"Pie":5284,"Par":14525,"PoÅ‚":4889,"Pro":6911,"Prz":9834,"Pod":4669,"Pol":63410,"Pow":4846,"Rad":5296,"Jan":4677,"Jes":5251,"Kar":5182,"Kon":5190,"Kra":7526,"KoÅ›":5328,"Gór":8184,"NGC":5540,"Mar":15306,"Mon":4650,"Mie":5310,"Mis":5057,"Wys":5223,"Wie":13213,"War":8050,"SÅ‚o":18142,"Str":5084,"Sta":19220,"Sie":4994,"Sai":4713,"Sch":5152,"Ros":4617,"Uni":6296,"The":5470,"bio":14193,"bli":16339,"bor":8274,"bow":6707,"bar":9707,"bec":8656,"ber":14109,"bel":6901,"bez":6239,"bia":7547,"bie":21077,"ca ":84053,"cac":4624,"byÅ‚":19691,"ce ":116313,"bro":10887,"bra":11568,"bry":5366,"brz":9909,"bsz":5466,"bst":6710,"bur":11294,"bum":8181,"bud":9663,"by ":10708,"aka":5857,"am ":8178,"akc":4640,"aki":8717,"ajo":5413,"ajm":4898,"ajw":6681,"aju":12289,"al ":15001,"aja":6999,"ajd":19542,"aje":5093,"ain":11213,"ak ":12839,"aj ":9008,"agr":6880,"ago":6833,"ajÄ…":56423,"anu":7044,"any":57351,"ano":19191,"ann":5995,"ant":15544,"ans":10106,"ane":52267,"ang":21057,"ani":105534,"ank":9718,"ana":52659,"anc":30280,"and":28542,"amo":13297,"amp":4556,"ami":37349,"ame":32849,"ama":9031,"alo":8151,"aln":38047,"all":6281,"ali":35299,"ale":28164,"ala":17506,"alb":10365,"an ":25897,"akt":18470,"ako":21085,"abi":5944,"abs":7005,"ae ":4965,"ad ":24951,"ac ":5802,"afi":15590,"aga":5833,"ado":9262,"adm":7603,"adi":7361,"ade":12127,"ady":8896,"adz":20389,"ack":10335,"acj":42190,"aci":10932,"ach":91172,"ada":31955,"acz":24570,"acy":17521,"azo":11989,"azu":9191,"azw":14194,"aza":4626,"azd":9097,"azy":7482,"az ":24099,"ba ":9607,"at ":32096,"are":9332,"ard":17168,"arc":24556,"ara":29532,"aro":21478,"arn":16904,"arm":8931,"arl":5792,"ark":13034,"ari":18485,"aru":4729,"anó":4796,"ars":25559,"art":41017,"asa":22529,"ary":12822,"arz":32133,"asi":8550,"aso":5103,"ask":9308,"ar ":9731,"api":8704,"apo":6161,"as ":15649,"aut":8273,"awa":20666,"aws":12931,"awn":15933,"awo":12547,"awi":30734,"asz":13814,"ata":29924,"ść ":40186,"ast":57235,"asy":6947,"atk":6138,"atr":8063,"ato":27910,"ate":18972,"akż":7374,"ati":9455,"aw ":7482,"atu":20800,"aty":21623,"auk":5346,"Åšwi":8845,"Å›ci":91962,"Å›ni":16799,"Å›li":5074,"Å›ro":7633,"Å›re":20628,"Å›lÄ…":5357,"Å›wi":21339,"jeg":6331,"jej":4751,"jed":34575,"jen":6642,"jew":58578,"jes":38109,"ji ":74801,"jal":7148,"jak":18567,"jaw":9057,"aźd":4510,"je ":30364,"jdu":17294,"jmu":7356,"jna":5939,"jny":19293,"jne":11196,"jow":13446,"jon":8504,"ito":6446,"ity":14838,"isk":17265,"ist":59047,"isz":10261,"ita":8740,"ite":12873,"iwe":4858,"iwi":4661,"ius":4598,"ium":6118,"is ":13674,"ion":48352,"iop":4645,"ior":18228,"ios":6430,"iot":6025,"ipc":5025,"ikó":5855,"iow":26535,"isi":4518,"isa":8456,"iu ":24955,"ire":4546,"ira":6326,"ja ":38334,"izy":4891,"izo":6649,"izm":4986,"iza":9287,"kim":128411,"kic":24931,"kie":126958,"dÅ‚u":9193,"km ":8293,"ki ":116429,"kaÅ„":16086,"kcj":11696,"kra":24036,"kre":18070,"kry":8710,"krz":7284,"ku ":72509,"kro":6037,"kow":64383,"kos":4831,"kor":8298,"kop":10012,"kon":28370,"kom":18077,"kol":18048,"kok":4910,"klu":5529,"ko ":39683,"kle":4640,"kla":7237,"jsk":40065,"jsz":8170,"ju ":15477,"jsc":28565,"kaz":5339,"kat":10533,"kar":21380,"kan":15572,"kal":9168,"kam":6459,"kad":5909,"kac":12564,"ka ":110422,"ha ":5601,"han":8170,"har":13309,"he ":11732,"her":6722,"aÅ‚ ":24279,"cÄ… ":7445,"aÅ„ ":4989,"aÅ‚e":8027,"aÅ‚a":29091,"aÅ‚o":22494,"his":8897,"aÅ‚y":11839,"aÅ„c":7994,"aÅ„s":33252,"go ":160963,"god":4635,"gni":4919,"gmi":81864,"gos":5577,"gor":5388,"gow":8194,"gu ":24037,"gro":9171,"grz":5388,"gry":6845,"gru":20718,"gra":31694,"gwi":7275,"ców":11188,"iaj":4940,"iam":4548,"ial":8076,"ian":22005,"ias":25407,"iar":9378,"iat":20282,"ic ":6481,"iac":9488,"iad":9916,"iaz":7546,"id ":16421,"ia ":151774,"iet":14221,"iew":12649,"iel":59211,"iem":53337,"ien":34053,"ier":72550,"ies":27602,"ied":28632,"ieg":53169,"iek":15875,"iej":135012,"iec":108952,"icy":13868,"ict":6284,"icj":4509,"ick":18455,"ici":5695,"ich":42480,"ice":16432,"ie ":425948,"ica":18194,"ide":5884,"ida":21189,"icz":59703,"ijs":6419,"im ":137877,"ika":22144,"ii ":57658,"ibÄ…":4979,"iaÅ‚":33757,"ik ":16857,"imp":4698,"imi":11049,"inc":8772,"ind":7280,"ina":57100,"inn":9746,"ino":8623,"int":11381,"ins":5688,"ine":11157,"ież":14661,"ing":19340,"ini":80940,"iny":19314,"iko":7564,"iki":9122,"in ":24060,"ilo":5237,"ill":10795,"ieÅ„":7511,"ilm":5161,"ili":7909,"ieÅ›":51901,"io ":7198,"how":11191,"hol":4719,"hor":6372,"hod":42292,"hni":9648,"hra":5522,"dów":6742,"fia":9588,"ewó":56727,"ez ":38609,"ews":7573,"eze":8964,"ezi":7185,"ezj":6360,"eta":10069,"etn":9079,"esp":11621,"eso":4594,"est":61386,"esz":24722,"eto":22583,"etr":10559,"ety":8345,"ew ":5885,"ewi":12221,"ewo":9270,"ewn":8260,"ewa":5038,"er ":34400,"epa":15408,"ejÄ…":6188,"es ":24102,"epu":5035,"epr":8058,"eri":17872,"erg":8537,"ere":16329,"erc":5702,"era":33404,"erb":5967,"et ":10917,"esk":6857,"esi":11275,"esa":4705,"erz":21989,"ery":23381,"eru":5751,"erw":27139,"ert":7815,"ers":24326,"ern":18633,"erm":6952,"erp":6669,"ero":30451,"eki":4817,"eko":5980,"eks":10430,"ekt":18859,"eku":6941,"en ":29550,"ela":11200,"ele":25422,"eli":15142,"eln":10771,"elk":19485,"ell":8172,"elo":10644,"elu":7577,"els":11724,"emc":9224,"ema":7081,"eme":7011,"emo":5689,"emi":25674,"ene":12261,"ena":8388,"end":7106,"enc":29197,"eno":4952,"enn":11782,"enk":5319,"eni":82488,"egÅ‚":18236,"ens":6693,"ent":47936,"ego":161252,"egi":24719,"ej ":196844,"egu":4666,"ek ":37209,"ein":6239,"el ":16434,"ejs":51321,"ejo":11481,"ejm":7156,"eje":7376,"eka":10766,"em ":73512,"gio":21748,"gie":12191,"gic":6767,"gii":7756,"gia":4982,"bÄ… ":6293,"gi ":9182,"gen":12957,"gel":5153,"gar":6319,"gat":9377,"gaj":4881,"gal":8113,"gan":14815,"ga ":14947,"fun":4925,"fra":5853,"for":16996,"fic":8573,"fil":7378,"fik":6148,"ać ":7866,"czÄ…":12269,"da ":43439,"czÅ‚":5966,"de ":15481,"dby":5063,"czÄ™":22437,"dal":5635,"daj":6034,"dar":8045,"dan":24834,"daw":15145,"dcz":6407,"ctw":8655,"cy ":54622,"cus":4654,"cym":5360,"cyj":21911,"cyc":12366,"cz ":10143,"czy":47263,"czk":8277,"czn":94844,"czo":18714,"cza":38835,"cze":61835,"cki":56501,"chó":9144,"co ":6133,"cni":7419,"ciÄ…":23459,"cne":4800,"cny":4546,"cow":30428,"cjÄ™":4623,"cez":4969,"ch ":250098,"cer":7119,"ces":6923,"cen":13171,"cej":5254,"cel":7686,"ceg":6128,"ci ":63750,"cha":21398,"cia":18501,"cie":119729,"che":16574,"chi":12341,"cho":63783,"chn":12122,"chr":6017,"ciw":5743,"cja":35582,"ciu":5042,"cin":7038,"cio":9246,"cka":9993,"cji":61836,"ed ":7038,"ebi":5142,"ec ":12319,"dzÄ…":8566,"ega":9608,"edl":5574,"edn":51333,"ede":12015,"eda":5582,"edz":17029,"edy":9486,"eds":6095,"ecj":5624,"eck":33379,"ech":31282,"eci":73659,"ece":7263,"ecz":26497,"ect":5183,"ecn":7076,"dyn":9974,"dys":5848,"dyc":5708,"dy ":20176,"dzy":15488,"dzt":56173,"dzo":7609,"dzk":13570,"dzi":114188,"dze":17343,"dza":11866,"dor":6162,"dom":7183,"dol":8096,"dok":4665,"doz":5829,"dow":41632,"dos":5363,"dmi":12552,"dna":7598,"dne":6153,"dni":85396,"dno":17791,"dny":5925,"dob":7583,"dst":8461,"duj":19180,"duk":7030,"dra":4842,"drz":4777,"du ":16409,"dro":12893,"dru":8195,"dia":9387,"der":13407,"del":4853,"dek":8141,"den":21584,"dem":8020,"dep":15043,"dle":22301,"dla":14281,"dko":7298,"do ":56353,"dio":7032,"die":6315,"rga":13673,"rgi":6819,"ret":5375,"res":16392,"rez":11929,"rg ":8389,"rdz":8372,"rea":6346,"rec":6681,"red":24223,"rej":11845,"reg":27968,"rem":8867,"ren":18517,"rek":5432,"rep":5124,"rcz":7138,"re ":17858,"rci":7329,"rch":9881,"rce":4547,"rca":6614,"raw":21807,"raz":29604,"rd ":6953,"ras":7862,"rat":14245,"raj":22570,"rai":5724,"ran":52894,"ram":15553,"ral":18585,"rak":14728,"rab":9860,"raf":17050,"rad":12311,"rac":33149,"rpn":4743,"ros":15610,"rot":7718,"rom":11621,"ron":23271,"rop":15650,"roz":22873,"row":46481,"rob":7374,"rod":59324,"roc":14309,"roj":6342,"roi":15518,"rol":7056,"rok":41872,"rog":11103,"rno":5279,"rny":6200,"rna":10748,"rne":9659,"rni":18640,"ro ":7330,"rma":13967,"reÅ›":7847,"rmi":12636,"rla":5043,"rki":6506,"rka":6181,"riu":5278,"rii":14143,"rin":4886,"ria":19003,"kÅ‚a":22407,"ric":4906,"rie":5586,"rwo":5268,"rws":10740,"nów":11177,"rz ":14438,"ryb":4752,"ryc":13818,"rug":6274,"rud":6357,"ruc":5354,"rup":16289,"run":6993,"rum":5890,"ruk":5290,"rus":7391,"rwa":7707,"rwc":4554,"ry ":30832,"rsk":40769,"rsz":11669,"rta":19705,"rst":9077,"rto":10205,"rte":6255,"rti":6031,"rtu":4870,"rty":7882,"rt ":8136,"ru ":8305,"rzÄ™":9880,"sad":8969,"sam":12218,"san":8261,"sar":4951,"sa ":26422,"rzÄ…":15714,"rze":166268,"rza":14480,"rzc":6541,"ryw":7519,"rys":7800,"ryt":14172,"ryk":17536,"rym":6312,"ryn":5422,"rzy":76075,"rzo":14189,"nÄ… ":19254,"si ":6951,"sie":37561,"sia":7216,"sk ":6051,"sin":6607,"se ":4707,"sce":55140,"sch":21250,"sco":24974,"ser":11407,"sen":9224,"spo":28928,"spr":6284,"spe":6485,"spi":8532,"sow":15205,"son":7183,"sok":5885,"siÄ™":66198,"sob":7483,"su ":8612,"skÅ‚":11877,"st ":49786,"ski":241685,"sko":33991,"skr":5762,"sku":9283,"ska":44791,"siÄ…":6562,"sz ":5749,"syn":5333,"sys":6083,"sza":29086,"stÄ™":16122,"sze":30322,"szc":22167,"szp":5112,"szo":8648,"szt":14601,"szk":19152,"szy":32568,"ste":40839,"sta":110434,"stn":10874,"sto":55229,"sti":7248,"stk":11925,"stu":7827,"spó":13799,"stw":35135,"str":55283,"sty":30683,"sy ":5848,"tak":16147,"tal":13415,"tac":30341,"tad":4771,"taw":14561,"tat":11340,"tar":27950,"tan":40782,"tam":18213,"te ":12262,"ta ":47407,"jÄ™z":8303,"pa ":7030,"owÄ…":6832,"jÄ™ ":5820,"pca":4745,"par":31402,"pas":19500,"pac":4930,"pad":8610,"pal":4646,"pan":7407,"kÄ… ":14644,"paÅ„":7582,"pec":4830,"per":13425,"pej":4544,"pla":21906,"ple":4628,"iÅ„s":18739,"pie":29091,"iÅ‚k":7468,"pio":5728,"pir":4743,"pis":14610,"poz":13437,"pow":94565,"por":15922,"pop":9036,"pot":7235,"pos":20600,"poj":4594,"pom":22714,"pon":6930,"pok":5372,"pol":56274,"poc":15903,"pod":44935,"piÅ‚":8661,"po ":9878,"pni":8916,"pub":6917,"pra":25179,"prz":116916,"pu ":5270,"pre":13822,"pro":44748,"put":4642,"puj":8407,"poÅ‚":82723,"py ":16720,"lÄ…s":7826,"mów":5303,"ra ":38610,"ngi":4912,"ni ":23063,"nge":9484,"iÄ… ":10996,"neg":51936,"nej":47963,"nek":13069,"nem":5441,"ner":9906,"net":21971,"nes":4536,"ndy":5147,"ng ":20546,"eży":7778,"nci":16716,"ncj":34860,"nce":9629,"ne ":77120,"eż ":13347,"ndr":6665,"ndo":5824,"ndi":8916,"nde":9089,"nda":6965,"nak":5404,"nal":20391,"nam":4778,"nan":15192,"nap":5117,"nar":14632,"nac":26022,"nad":18221,"nag":7300,"naj":37748,"nd ":10560,"nau":6824,"nat":12135,"nas":8986,"naz":14802,"naw":6217,"na ":315412,"moż":6857,"nyc":66044,"ny ":143123,"noÅ›":21837,"nty":9827,"nto":7930,"ntu":6194,"ntr":10458,"nta":17677,"nte":16419,"nst":10393,"nu ":12191,"nt ":17005,"ns ":4556,"noc":23507,"nom":6299,"nos":13706,"nor":4839,"now":33236,"nne":10038,"nna":4663,"nni":6637,"niÄ™":6603,"nny":11159,"głó":21438,"no ":23520,"nki":7495,"nkc":4543,"nka":8221,"nku":6543,"nko":6939,"eżą":8181,"iÄ…g":20115,"iÄ…z":16848,"nii":15486,"nie":303754,"nic":58316,"nia":115515,"niz":10946,"niu":18651,"nis":21291,"nio":33347,"gÅ‚o":21325,"nim":8023,"nin":5902,"nik":44866,"ogr":14773,"ogi":15434,"ogo":4557,"oga":7462,"jÄ… ":20069,"oid":31094,"ok ":8415,"ojs":6215,"ojn":8775,"oje":68149,"jÄ…c":92228,"oce":7209,"och":19137,"oci":8093,"ock":7255,"ocn":14524,"obs":6969,"oby":5467,"ode":8229,"odk":11049,"odl":22987,"odo":21990,"odp":5300,"odn":35244,"ods":5690,"odr":4605,"ocz":25360,"of ":6324,"odc":6738,"odb":6509,"oda":13640,"odz":61637,"ody":9005,"odu":13447,"ofi":6001,"iÄ™k":9725,"iÄ™c":10576,"iÄ™d":13216,"iÄ™t":12440,"od ":47389,"obo":8696,"obr":12863,"obl":4516,"obn":4740,"obi":10703,"obe":11907,"nym":35914,"iÄ™ ":65182,"owy":75588,"osó":4510,"ków":24138,"ows":40460,"own":11322,"owo":53873,"owi":138750,"ozy":6621,"ozw":5341,"ozn":9119,"osÅ‚":6618,"ozb":6448,"oty":9500,"ote":7310,"otr":5518,"oto":11888,"otn":8681,"osz":15296,"ost":51897,"ota":6378,"osi":14132,"osk":7450,"ose":9237,"osp":5848,"oso":13740,"oró":5598,"owc":6201,"owa":124010,"owe":69144,"opo":27225,"opi":9321,"ope":10070,"opa":11132,"os ":8863,"opu":5185,"opr":8595,"or ":15758,"ork":4605,"orm":16889,"orn":6157,"oro":17951,"orc":4621,"ord":8495,"ore":9264,"org":11803,"ori":12364,"osa":8380,"ort":15205,"ors":22187,"oru":7711,"orz":34348,"ory":13310,"ora":31522,"ola":10017,"on ":32944,"oli":42609,"ole":26115,"ols":92465,"oln":14866,"olo":22365,"olu":5605,"oka":12522,"om ":7335,"oki":5566,"okr":39604,"oko":19282,"ogó":4810,"oku":40113,"ona":86386,"ond":4877,"one":29526,"oni":52759,"onk":6206,"onn":4626,"ono":16680,"ons":8848,"ont":10876,"onu":6969,"ony":38958,"oma":14351,"ome":9149,"omi":22956,"omp":9870,"omo":23578,"omu":5636,"la ":29248,"le ":27634,"lac":9698,"lak":7036,"lan":38327,"lam":5773,"lar":10967,"lat":37023,"las":14320,"ld ":4729,"lbu":8442,"koÅ‚":12944,"kul":5213,"kuj":7417,"kwi":7243,"krÄ…":16281,"koÅ›":9426,"kró":6396,"kte":5141,"ksz":11351,"ksi":5778,"kty":11231,"ktr":5362,"ktu":7021,"kto":8566,"krÄ™":7083,"gól":4564,"gór":7545,"któ":37445,"lok":4711,"lon":16670,"log":15244,"lot":5895,"low":11925,"lno":13692,"lni":16273,"leż":15711,"lne":15681,"lny":18798,"lna":11882,"lud":5047,"lub":38824,"lsk":52175,"lu ":15294,"lsc":50391,"li ":18475,"lew":7878,"les":10216,"let":6569,"ler":6678,"lem":11032,"len":14544,"lek":14961,"lej":10577,"leg":25572,"lec":7985,"lla":5033,"lle":8878,"lli":5642,"lko":12509,"eÅ„s":10006,"lka":5102,"lki":9010,"ll ":4740,"lit":23937,"lis":17192,"lip":7716,"lin":28801,"lim":6337,"liz":9545,"liw":7216,"lic":38891,"lia":7515,"eÅ„ ":8305,"lik":8729,"lii":5868,"ma ":13583,"mac":9833,"eÅ› ":42823,"maj":8752,"mar":13792,"mas":6086,"mal":6141,"man":12874,"maz":13085,"mat":15538,"me ":5269,"eÅ›c":9065,"eÅ›n":12795,"eÅ›l":8074,"mcz":8811,"met":12557,"mer":18450,"men":33777,"lut":5364,"hód":6854,"mpi":5561,"miÅ„":6040,"moc":8677,"mod":4841,"mon":7419,"mow":13205,"mor":22691,"mu ":12663,"msk":7954,"my ":8132,"muj":8914,"mun":5245,"muz":6245,"maÅ‚":6431,"mi ":36519,"min":109517,"mis":7482,"mit":5081,"mic":8508,"mia":35513,"mie":85656,"miÄ™":17160,"mni":5567,"wÄ… ":9465,"wód":59175,"wór":6020,"źdz":4969,"zta":5990,"ztw":56537,"zu ":8060,"zuj":5309,"zur":6895,"zy ":52536,"zwa":16748,"zwi":19757,"zwy":6194,"zyw":7464,"zys":27664,"zym":15419,"zyn":18487,"zyk":17972,"zyl":5653,"zyj":5225,"zyc":26077,"zyd":4698,"zyÅ‚":4808,"zi ":13438,"zaÅ‚":9317,"zgr":4705,"zec":34126,"zed":16305,"zeg":13280,"zej":12005,"zeb":5093,"zdo":9053,"zes":26033,"zez":38687,"zew":14699,"zen":40191,"zem":9711,"zel":6475,"zek":13631,"zer":16763,"ze ":43444,"zch":5824,"zbi":11943,"zcz":25210,"zac":28997,"zaw":17852,"zaj":18917,"zam":9131,"zan":18287,"zak":6986,"zal":7485,"zar":15092,"zap":6541,"zas":21035,"zny":34365,"zos":17618,"zon":31868,"zow":26765,"zpo":4827,"zeÅ›":12448,"zmi":5909,"zna":56249,"zno":5266,"zne":31393,"zni":17047,"zm ":5171,"zka":12896,"zko":15029,"zki":16987,"zeÅ„":5484,"zib":8100,"zia":21275,"sÅ‚a":11498,"zie":58529,"zin":14757,"sÅ‚o":8768,"zio":8015,"sÅ‚u":9392,"zja":5084,"zji":8382,"yty":7484,"ytu":9487,"yto":6663,"źni":4961,"yte":4862,"yta":5297,"ysz":7085,"yst":44231,"yso":5982,"ysp":7012,"ysk":11600,"tÄ™p":15379,"za ":37664,"yzn":5243,"ywa":26508,"ywi":7128,"ywn":5067,"yce":6751,"ych":133947,"yci":10363,"ycz":50193,"yda":15003,"żaj":15742,"yck":5423,"ycj":6652,"że ":16017,"yjs":11565,"yka":23821,"ym ":69059,"yki":7812,"ykl":4610,"yko":9261,"yn ":8183,"yli":7761,"ymi":14584,"yms":4916,"yna":17425,"yni":13954,"yno":4822,"żen":5078,"yk ":13655,"yjn":27661,"tów":16736,"tór":35547,"sÄ… ":7387,"ożo":65228,"oży":4895,"oże":6845,"wy ":42246,"wsp":8605,"wsz":18473,"wst":10921,"wsc":10075,"wsk":48535,"rąż":16283,"wys":19278,"wym":21711,"wyk":10398,"wyn":5788,"wyd":14033,"wyb":6431,"wyc":28770,"sów":5741,"woÅ›":26886,"wo ":32774,"wna":7537,"wne":21421,"wni":31749,"wiÄ…":20802,"wno":4837,"wka":4779,"wrz":6674,"wod":23078,"wiÄ™":20120,"wny":9160,"wow":8334,"wor":17209,"wol":5953,"woj":77026,"wcz":7095,"oÅ›l":6671,"wch":7307,"we ":41958,"oÅ›c":57589,"wca":6919,"wer":9502,"wej":33041,"weg":17678,"wed":4775,"waÅ‚":10230,"ość":33539,"wi ":5607,"pÅ‚y":9060,"wis":10022,"wiz":5342,"wie":266248,"wid":6688,"wic":20177,"win":12633,"wia":38637,"wa ":75032,"wan":92067,"wal":11772,"waj":5843,"wat":12396,"war":32065,"wac":6719,"róż":7313,"wad":9110,"rów":28327,"ród":6783,"ról":5670,"oÅ‚a":8333,"oÅ‚o":72193,"oÅ‚e":8524,"oÅ‚u":20038,"oÅ„c":21153,"ver":4892,"uzy":7699,"usk":8960,"usz":14475,"ust":14884,"ute":10402,"utw":6376,"uto":11126,"us ":13666,"ura":9619,"ure":5261,"urg":10533,"uro":16796,"urs":7537,"ury":7673,"urz":9377,"ujÄ…":40031,"upa":5160,"ur ":29893,"upy":9651,"umi":4996,"ume":5036,"unk":16732,"uni":8824,"und":6279,"une":6969,"uko":7594,"um ":20351,"uka":4715,"ult":6004,"uli":7259,"ula":8673,"uje":18646,"uja":4953,"ugi":6634,"ucz":7047,"udn":21514,"uch":15770,"udo":10709,"udz":6914,"ub ":30727,"pół":22236,"ubl":8802,"ube":7781,"tyw":7322,"tyj":4614,"tyk":20395,"tyl":6759,"tym":7939,"tyn":8462,"typ":6891,"tys":5969,"tyt":6677,"twó":5271,"ty ":25638,"twa":27679,"tur":19565,"tun":9184,"tyc":48848,"two":26167,"pól":4883,"twi":61091,"tre":5469,"tra":36373,"tri":6420,"tru":11588,"tro":23044,"trz":32044,"tu ":25361,"try":9123,"to ":54716,"tni":28308,"toc":7538,"toi":15874,"tos":10191,"tow":38410,"tom":6513,"ton":10929,"tok":8683,"tol":14627,"tor":34275,"top":9511,"tin":4571,"tio":8819,"tki":10465,"tko":7816,"tka":9207,"tle":5081,"tem":19837,"ten":10208,"tej":7771,"tek":9091,"tel":10074,"kże":7372,"teg":14724,"tec":7568,"ter":68917,"the":6053,"taÅ‚":27215,"żąc":10258,"zÅ‚o":10911,"yÅ‚ ":11114,"ży ":9032,"zęś":17553,"życ":6860,"yÅ‚a":8647,"żyn":4577,"żyw":6989,"yÅ‚y":6907,"yÅ„s":6994,"żni":7269,"żon":68291,"zÄ™d":7177,"zÄ™s":4784,"ższ":5421,"zÄ…d":13275,"zÄ…c":15081,"zÄ… ":5842,"zÄ…t":4518,"uży":12473,"zów":5144,"wÅ‚a":9785,"óżn":6877,"ół ":8476,"ółn":14744,"ów ":112668,"óra":6553,"óre":15215,"órn":8319,"óry":15904,"ór ":5776,"ówk":5666,"ówn":37744,"ódz":62547,"ód ":10612,"ób ":4595,"óln":9187,"Ä…gu":15768,"Ä…dz":6409,"Ä…cz":8602,"Ä…cy":47223,"Ä…ce":25280,"Ä…ca":47593,"Ä…zk":9275,"Ä…za":4528,"Ä…sk":8772,"Ä…tk":5004},"n_words":[44927968,50956492,36530760],"name":"pl","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/pt.json b/contrib/languages-data/pt.json
new file mode 100644
index 0000000..5203a33
--- /dev/null
+++ b/contrib/languages-data/pt.json
@@ -0,0 +1 @@
+{"freq":{"D":84998,"E":149956,"F":100967,"G":92106,"A":261572,"B":132089,"C":229572,"L":100908,"M":162595,"N":83569,"O":115743,"H":53006,"I":86169,"J":66643,"K":27233,"U":54108,"T":95611,"W":30243,"V":63770,"Q":7457,"P":198289,"S":200312,"R":102796,"Y":7968,"X":13707,"Z":9504,"f":420930,"g":646820,"d":3123972,"e":5421795,"b":522239,"c":1736812,"a":5855900,"n":2824771,"o":4680584,"l":1547279,"m":2105049,"j":94292,"k":164391,"h":490405,"i":3472398,"w":44026,"v":454134,"u":1899338,"t":2255026,"s":2945228,"r":3035316,"q":191465,"p":1075380,"z":175309,"y":94919,"x":113550,"²":81071,"É":13693,"Ã":9519,"í":209507,"ê":67610,"é":451892,"è":5136,"ç":214936,"ã":317848,"â":37738,"á":187832,"à":26465,"ü":5538,"ú":40758,"ô":23605,"õ":33734,"ó":129413," l":122890," m":228220," n":368310," o":307851," h":135081," i":114948," j":44060," k":90787," d":1636882," e":668130," f":265709," g":86172," a":553690," b":90195," c":638137," u":471782," t":166023," v":73530," q":119308," p":552163," s":358148," r":175170," J":65167," K":25787," H":50638," I":64806," N":77349," O":109788," L":96197," M":155484," B":122895," C":211790," A":232527," F":95595," G":81500," D":78496," E":140343," Z":9058," Y":7429," X":9721," S":182999," R":96218," Q":6965," P":187689," W":28455," V":54881," U":50226," T":85637," á":66191," à":25954," é":287636," ú":5989," Ã":9484," É":13626,"A ":78627,"Da":9542,"Cu":6462,"Cl":8427,"Co":62457,"Cr":10302,"Ce":12150,"Ch":23975,"Ci":8714,"Ed":5016,"Do":10375,"De":23490,"Di":17511,"GC":5835,"Fe":14358,"Fa":9224,"Eu":7410,"Es":76180,"En":6707,"Em":8917,"El":11312,"Ge":12507,"Ga":11449,"I ":12898,"Fu":6893,"Fr":18796,"Fo":24543,"Fl":6042,"Fi":9055,"C ":14112,"Au":8700,"Ar":20685,"At":7248,"As":11784,"D ":5749,"Ba":33758,"Ag":5894,"Ab":6380,"Ac":5079,"Am":13589,"An":22486,"Al":38766,"Bu":7329,"Br":31392,"Ca":66526,"E ":5464,"Bi":7032,"Be":17271,"Bo":16310,"Le":17360,"Li":18877,"La":20252,"Lu":9478,"Lo":21075,"Me":18166,"NG":5897,"Mi":22675,"O ":59245,"Ma":62167,"Mu":14513,"Mo":23160,"Ni":6791,"Ne":10091,"Na":16341,"P ":5903,"No":30608,"Ol":8785,"Gi":5566,"Gr":19098,"Go":9769,"Gu":13341,"Ha":12782,"He":10653,"II":8816,"Hi":6333,"Ho":11804,"In":21460,"Ja":20297,"Je":5208,"Jo":23876,"Ju":10807,"Ka":6332,"Um":5798,"Un":17690,"Tu":6480,"Tr":12997,"To":12084,"Th":10714,"Ti":5247,"Te":19234,"Ta":10759,"UA":15545,"V ":6036,"St":11159,"Su":24311,"Wi":7399,"Sã":16301,"Wa":7041,"Vi":19753,"Va":11369,"Ve":14368,"Pr":23453,"S ":9133,"Pe":21497,"Pa":48708,"Pl":5522,"Po":57527,"Pi":15167,"Os":8721,"Or":9792,"Se":27899,"Sc":6387,"Si":12578,"Sh":5677,"So":14955,"Sa":39239,"Re":33199,"Ri":19155,"Ro":19693,"Qu":6140,"T ":5229,"Ra":8949,"b ":50847,"a ":2181662,"i ":168232,"ge":70306,"ga":87148,"bé":16913,"fl":8986,"fi":74478,"fr":59032,"fu":27785,"fo":117850,"dá":5256,"he":73854,"ha":210111,"gn":21302,"cê":5480,"gl":18191,"cç":8441,"gi":116568,"gh":7414,"gu":120239,"gr":61974,"cí":25911,"go":74169,"du":48733,"g ":26553,"ea":109055,"eb":27192,"ec":114155,"ed":73900,"de":1384296,"di":203085,"dm":36748,"do":671969,"ds":6146,"dr":29810,"ew":5596,"ex":56960,"eu":52678,"ev":46416,"ey":9665,"ez":25288,"fa":47754,"h ":22234,"fe":50703,"eg":161615,"ef":22289,"ee":12045,"el":215678,"aç":106998,"ej":12312,"ei":173784,"ep":64753,"eo":27810,"en":615409,"em":301673,"et":118849,"es":604827,"er":508227,"eq":14954,"aí":13564,"ca":322979,"e ":2017914,"bs":7191,"br":100433,"bu":28485,"bo":46844,"bl":22439,"bi":100249,"be":49470,"da":652111,"f ":9921,"cu":52185,"ct":31468,"cr":50241,"co":536282,"ck":15123,"cl":33604,"ci":328384,"ch":65018,"ce":215952,"c ":14270,"az":18902,"ay":13866,"ba":74690,"d ":43293,"at":195898,"as":441529,"ar":395911,"aq":7417,"av":49201,"au":56223,"ak":8531,"al":413966,"ai":134662,"aj":6551,"ao":31258,"ap":60289,"am":235425,"an":557322,"ac":100712,"ad":561485,"ab":137795,"ag":58484,"ah":7769,"ae":25110,"af":14102,"nu":24993,"nt":522402,"ns":188107,"no":344725,"nn":15103,"nz":5976,"ny":7562,"nv":17155,"oe":16285,"of":27077,"oc":103353,"od":80574,"oa":21115,"ob":40172,"om":360608,"on":341990,"ol":141927,"oi":109737,"oj":7341,"og":52000,"oh":5263,"ot":49005,"m²":81019,"os":507875,"ov":77341,"ou":126561,"op":64022,"oo":11470,"or":489375,"r ":303972,"ox":5797,"ow":11301,"oz":5717,"pe":192004,"lá":12510,"pa":207724,"pl":31072,"lé":9270,"lê":6741,"po":262183,"ph":9247,"pi":69522,"lo":175844,"lm":34639,"ll":48668,"ls":10434,"lp":6052,"lv":20781,"lu":42856,"lt":38629,"ly":6623,"o ":2000634,"ma":495019,"mb":74173,"iã":65284,"me":284803,"iá":6420,"iç":26049,"mi":135455,"mm":5698,"mp":92721,"mo":132550,"mu":111484,"p ":8620,"na":437221,"nc":193122,"nd":299787,"ne":119195,"nf":17343,"ng":76996,"nh":95890,"ni":224974,"nj":6755,"nk":6105,"ju":23595,"fí":5753,"jo":21637,"ki":12050,"ke":10814,"ka":8676,"m ":610401,"ko":5815,"gé":5229,"km":88160,"gê":12280,"li":257137,"lh":65430,"le":173629,"ld":17597,"lg":14230,"lf":5772,"la":234272,"lc":7172,"lb":14242,"n ":112737,"ht":6380,"hu":13446,"hi":44146,"dê":5692,"ho":87126,"dé":6382,"id":307392,"ic":326922,"ib":24713,"ia":390572,"ig":76460,"if":27051,"ie":76108,"k ":20205,"ir":178791,"is":322059,"it":260738,"iu":12350,"iv":114919,"ix":17582,"ik":5904,"eç":8830,"il":161901,"im":114838,"in":395415,"io":223313,"ip":55696,"je":12713,"iz":80563,"l ":281164,"ja":26393,"xi":25832,"tê":5442,"xo":8355,"té":18873,"xp":7291,"tí":13967,"tó":26791,"xt":10508,"z ":28371,"xc":16696,"xa":14045,"tã":10541,"tâ":11963,"tá":22795,"xe":8953,"sã":39079,"wi":7047,"sé":18935,"sí":5599,"ró":40309,"y ":44219,"wa":10938,"sá":5639,"ré":9925,"rç":11146,"vi":94338,"rã":8523,"vr":9835,"rí":29707,"rê":8332,"vo":58913,"uz":14799,"ux":5541,"uv":5858,"ve":130840,"rá":18536,"va":112071,"x ":15240,"ui":98859,"uj":5584,"ul":126725,"ue":171782,"ug":35554,"ur":125836,"us":87304,"ut":96190,"um":493013,"un":209874,"up":26206,"ty":5162,"tu":132302,"tt":15069,"pó":5919,"ub":41087,"ua":116112,"ud":26863,"uc":23327,"w ":7530,"pú":7028,"to":326748,"pé":22989,"tl":9288,"ts":8324,"tr":260066,"te":557844,"ti":259190,"th":25055,"ta":456660,"su":99840,"ss":133040,"st":452789,"sl":7670,"sk":8628,"sm":23244,"sp":74972,"so":134893,"sq":6495,"sd":6659,"sc":72024,"se":292294,"sh":13868,"si":227032,"u ":130375,"sa":133204,"sb":7374,"rr":74864,"rs":43158,"rt":193480,"ru":48286,"rv":18129,"ry":9625,"ní":7223,"rq":17132,"rp":10793,"ro":344915,"rn":53960,"rm":73617,"né":7825,"rl":15891,"nç":36309,"rk":7248,"ri":409856,"nã":10618,"rg":47427,"rf":6634,"ná":10363,"re":480208,"rd":53701,"rc":51264,"rb":27665,"ra":596837,"t ":61702,"mú":6929,"qu":189373,"mí":22028,"mé":15139,"má":8216,"mã":9155,"s ":1150995,"pt":9571,"pu":45299,"ló":14756,"lí":26351,"pr":175784,"ps":6678,"zi":14725,"ze":25298,"vá":5473,"za":80538,"zo":10261,"ví":24973,"ya":6552,"ys":5731,"yr":7114,"uí":16711,"uê":10113,"uç":8683,"² ":81055,"É ":12059,"ã ":5975,"ál":15193,"ác":6980,"ád":5021,"áv":6927,"áx":6998,"ár":84913,"át":10444,"ás":9114,"âm":6223,"ân":30468,"ão":308442,"à ":22252,"á ":25281,"ós":7667,"ôm":5559,"ôn":14217,"ói":30866,"óg":6061,"ód":6423,"ór":23808,"óp":8429,"ón":16555,"ól":8310,"ív":5979,"ín":37401,"ím":8935,"íp":23888,"ío":18396,"ít":18007,"ís":21982,"íf":5629,"íl":22187,"íc":11953,"íd":15269,"çõ":21574,"çã":131394,"ên":31853,"êm":5074,"ês":26627,"él":7493,"ém":24476,"én":6312,"és":8228,"ét":9564,"ér":30738,"éd":7542,"éc":31275,"ço":21084,"ça":38992,"é ":308148,"ún":5650,"ús":10148,"úb":7903,"õe":33653," Ga":11344," Ge":12440," Fo":24420," Fu":6872," Fr":18748," Fi":9007," Fl":5988," Ha":12738," He":10614," Go":9722," Gr":18935," Gu":13279," Gi":5513," Ho":11760," Hi":6304," Je":5193," Ja":20267," In":21384," Ka":6274," Jo":23796," Ju":10788," La":20143," Le":17264," Li":18789," Ma":61882," O ":56830," Mi":22595," Me":18100," NG":5699," Lo":21011," Lu":9458," Ne":9995," Na":16245," Ni":6760," Mo":23061," Mu":14430," A ":56821," Am":13551," An":22425," Al":38669," Ag":5878," Ac":5031," Ab":6362," Ba":33333," Au":8681," At":7218," As":11678," Ar":20591," Be":17207," Bi":6949," Bo":16224," Br":31284," Bu":7302," Ca":66026," Ce":12123," Ci":8645," Ch":23864," Cl":8347," Cr":10239," Co":62169," Cu":6320," Da":9487," Di":17388," De":23395," Do":10149," El":11281," Es":76102," En":6655," Em":8884," Eu":7403," Fe":14330," Fa":9149," Wi":7332," Sã":16295," Wa":6979," a ":159392," Os":8676," Or":9772," Po":57372," Pl":5480," Pi":15147," Pe":21311," Pa":48469," No":30533," Ol":8779," Ra":8887," Qu":6094," Ro":19582," Re":33096," Ri":19127," Pr":23377," Su":24284," St":10640," Ta":10684," UA":14219," Th":10656," Ti":5222," Te":19124," Tr":12898," To":12004," Sa":39159," Sh":5584," Si":12511," Sc":6287," Se":27797," So":14864," Va":11341," Ve":14300," Vi":19683," Tu":6372," Um":5787," Un":17669," ja":9410," im":11343," in":69292," il":7613," it":11439," jo":14428," ju":16927," ha":94775," gr":27016," go":7301," gu":5619," hi":11250," ho":15720," ne":9802," na":144146," mu":41919," mo":24736," on":8262," oc":7515," of":9658," ob":9893," nu":6152," no":190063," le":13833," li":23905," la":21762," gê":9642," km":87834," me":40602," mi":24825," o ":110021," ma":77227," lu":6557," lo":47174," ag":9251," ab":11690," ac":15052," ad":42206," am":26285," an":52016," ao":28506," ap":22430," al":29345," av":5103," au":17149," ar":22234," at":33443," as":63845," ba":35433," bi":6595," be":6338," bo":7718," br":28760," ca":66098," e ":241281," er":11510," et":6896," es":129714," en":44317," em":145368," el":17511," fe":16956," fa":35767," ex":43118," fu":24478," fr":49866," fo":99597," fi":29419," ge":13813," ga":14076," cl":11956," co":383123," cr":20621," ce":57678," ch":14448," ci":62716," da":258975," cu":12445," do":238864," de":1021034," di":92269," ed":7807," du":15330," sa":10534," se":193144," si":34152," so":24812," qu":119214," mú":6854," ra":8844," re":136757," nã":9894," ri":7562," ro":14358," pu":7134," pr":135007," lí":6039," os":51421," ou":60851," op":5907," or":37748," pe":115763," pa":93579," pl":9433," po":171127," pi":7518," sã":13196," sé":13507," va":8521," ve":22883," vo":11121," vi":22819," us":9297," ut":7175," um":444668," un":7084," ta":23369," su":56894," tr":33259," to":19262," th":6383," ti":9760," te":60106," É ":12023," à ":21932," ár":53832," ál":7915," é ":284550,"GC ":5727,"Est":57555,"Esp":12609,"Eur":5804,"Ele":5431,"Em ":5626,"Ger":5645,"Fra":12238,"Foi":10640,"For":9044,"II ":6274,"Gra":10045,"Int":5948,"Amé":5088,"Bai":6138,"Bar":6969,"Ale":12779,"Alt":5502,"Ant":8344,"Cal":7739,"Cam":11986,"Cas":10156,"Car":12659,"Cat":5648,"Can":7596,"Bra":20750,"Den":4991,"Chi":5974,"Cen":6997,"Cha":10337,"Cor":7280,"Com":13375,"Col":5164,"Con":24999,"Dis":5047,"Nov":9350,"Nor":13657,"Os ":6443,"Per":7552,"Par":15095,"Pau":10573,"Pal":5074,"Pro":8377,"Pol":9021,"Pos":21873,"Por":13937,"Jan":9313,"Jos":5715,"Jog":5307,"Lan":5276,"NGC":5668,"Man":7960,"Mar":25108,"Mon":7943,"Min":6224,"Mun":8484,"São":16298,"Sul":8673,"UA ":15445,"Sai":5389,"San":15530,"Rio":11763,"Val":5795,"Vil":5627,"Ver":7558,"Uni":16626,"Ter":5697,"The":7418,"Tra":5593,"bit":68890,"bil":5849,"bo ":5900,"bli":15859,"bol":16054,"bor":6814,"be ":8829,"ban":15903,"bal":9064,"bai":9744,"bas":9817,"bar":9261,"ber":18628,"bel":6221,"ca ":99422,"car":23995,"cas":22552,"cat":6662,"can":45210,"cap":11240,"cad":24057,"cam":12174,"cal":52109,"ce ":17573,"bri":17515,"bro":31772,"bra":35217,"bre":13206,"bur":6170,"bum":7625,"am ":33009,"aix":9798,"al ":180975,"ain":14382,"aio":19228,"air":8502,"ais":60324,"aia":5333,"ago":14405,"anu":5896,"ano":74765,"ant":129336,"ans":13455,"ane":21736,"ang":13849,"anh":31372,"ani":21090,"ana":43001,"anc":58051,"and":76029,"amo":8042,"amp":20451,"ami":6934,"ame":89963,"amb":25048,"ama":26496,"ao ":25066,"alt":10265,"alo":6981,"alm":23414,"all":8441,"alg":7598,"alh":11152,"ali":86609,"ald":5965,"ale":20569,"ala":17691,"an ":23333,"aba":10377,"abe":9561,"abi":55871,"abo":5404,"abr":10253,"ae ":16585,"aca":7889,"ab ":40485,"ai ":5437,"aga":6957,"age":17901,"ado":217915,"adr":8357,"adm":35408,"adi":9559,"ade":147619,"adu":9382,"aco":11338,"aci":33260,"ach":9385,"ace":11134,"ada":120249,"act":10015,"até":8607,"ató":5142,"ba ":7163,"aqu":7206,"amí":18994,"arg":7520,"are":23525,"ard":21302,"arc":12276,"ara":72529,"aro":8382,"arn":4993,"arm":6518,"arl":7205,"anç":24866,"ari":33288,"arq":9519,"arr":16718,"art":76275,"asa":6049,"asi":39800,"asc":20114,"ase":9902,"ar ":59582,"apa":10937,"alá":8272,"ape":8629,"api":9537,"apo":6995,"apr":7853,"as ":272706,"ava":17923,"aut":16096,"arç":7034,"avi":9973,"ave":10994,"ata":23226,"ast":51032,"ass":26073,"atr":16524,"ato":21654,"ate":18241,"ati":65753,"atu":18602,"aul":12887,"aus":5992,"jet":6359,"jan":5629,"jog":9592,"ito":67486,"itu":23834,"ism":9783,"isp":8481,"iss":14904,"ist":137228,"ita":115808,"ite":14296,"iti":9264,"ivr":6273,"ivo":19750,"isã":7956,"iva":52169,"ivi":13782,"ive":20602,"ipo":7067,"is ":98587,"ion":54065,"ior":20141,"ios":21003,"ipa":29796,"ipe":6360,"ir ":14429,"irr":7079,"iro":63701,"iri":6848,"ise":5201,"isc":10067,"isa":6981,"iu ":6072,"ire":22818,"ira":48023,"ja ":10608,"ixa":7209,"itâ":5590,"iz ":5440,"iza":68411,"km ":6889,"ki ":6235,"km²":80918,"gên":12105,"jul":5237,"jun":11429,"ha ":54934,"ham":13772,"han":7220,"har":10472,"has":8814,"hab":92261,"he ":16106,"hei":5849,"hec":22728,"her":9140,"hin":8200,"his":10200,"ho ":40610,"go ":31208,"cçã":8036,"gna":9579,"giã":60677,"gos":20347,"gov":5137,"gru":8390,"gra":31116,"gre":13275,"cíp":21053,"gui":6993,"gua":14762,"gue":29368,"gun":46422,"guê":7234,"iai":5206,"iam":5743,"ial":29154,"ian":36430,"ias":36377,"iad":18351,"ibu":6231,"ibe":5599,"ia ":236755,"ien":14065,"ier":7590,"ies":5878,"ied":5977,"iaç":6604,"ife":6300,"ifi":12350,"icu":5216,"ico":76077,"ici":43382,"ich":7968,"ice":7761,"ie ":27537,"ica":143651,"ido":58535,"idi":8244,"ide":55475,"ida":173600,"il ":28740,"im ":15779,"ige":5893,"iga":14060,"igi":13843,"igu":5678,"icí":20778,"igo":8521,"ign":14724,"imo":12687,"imp":12501,"ime":38876,"imi":7471,"inc":33960,"ind":23589,"ina":69514,"ino":22963,"int":49765,"ins":15328,"inf":7949,"ine":18603,"inh":19122,"ing":32761,"ini":53005,"inu":8866,"ila":11890,"in ":16212,"ilo":7283,"ill":18435,"ilm":5403,"ilh":18906,"ili":24989,"ile":27199,"ima":19294,"io ":111339,"hom":5231,"hos":8051,"hor":13996,"hum":7243,"fes":5911,"fer":17361,"fei":6807,"fam":20697,"ext":9301,"ez ":8287,"exp":6312,"exi":7134,"exc":16402,"eze":10386,"eta":21527,"ete":19661,"eti":12522,"esp":45070,"est":117047,"ess":36172,"eto":18111,"etr":18837,"eve":21643,"eva":5679,"evi":13265,"eus":10550,"eró":29181,"erí":19798,"ey ":6913,"er ":64588,"epa":35649,"açõ":12626,"eon":5303,"es ":232451,"epr":6051,"enç":6954,"eri":53238,"erg":9605,"ere":26129,"erc":25527,"erd":8470,"era":58345,"et ":10811,"equ":14020,"aís":9202,"esm":7907,"esi":25700,"esc":27105,"esd":5456,"ese":28703,"eu ":29870,"esa":56958,"erv":11462,"err":29563,"ert":32707,"ers":31036,"ern":27425,"erm":19525,"ero":27972,"en ":19630,"ela":57757,"ele":31916,"eli":10608,"elh":19021,"ell":12189,"elo":35477,"eo ":6077,"emb":29421,"ema":25633,"eme":11201,"emo":8721,"emi":9916,"emp":20050,"ene":11086,"enh":10752,"ena":27141,"end":70599,"enc":33249,"eno":17065,"eni":7335,"env":8603,"ens":111144,"ent":263509,"açã":84894,"ecç":6791,"ego":8230,"egi":67782,"egr":7999,"egu":58530,"eia":6927,"eis":10113,"eir":97869,"eio":5873,"ein":12110,"eja":8929,"el ":27319,"eit":17574,"em ":185457,"gin":9613,"gio":6344,"gic":7282,"gia":12807,"gen":19808,"ger":11186,"gem":16786,"ge ":8655,"gad":9856,"gas":7737,"gar":10290,"gal":15478,"gan":11614,"ga ":19137,"fut":9060,"fun":15140,"fra":39155,"fre":11644,"for":36958,"foi":67238,"bém":15726,"fic":31195,"fil":13806,"fin":10276,"fis":5455,"da ":386105,"de ":1113463,"dad":172007,"dal":5052,"dae":8716,"das":50851,"dan":7654,"dam":5121,"cul":25961,"cto":6036,"cti":5064,"cta":7890,"cur":7265,"cla":7926,"clu":9168,"cli":8238,"co ":77432,"con":114590,"col":16761,"com":260529,"cor":23539,"cos":23065,"cre":8780,"cri":27018,"cro":6255,"cea":7114,"ch ":7362,"cer":21587,"ces":47284,"ceu":5073,"cen":80958,"caç":9087,"cel":14050,"cei":7700,"cha":18824,"cia":84536,"ck ":7938,"cie":24606,"cid":84373,"che":14945,"chi":8861,"cim":6171,"cis":5750,"cin":23953,"cio":45369,"cip":32198,"ebo":13090,"ead":6607,"ean":5576,"eal":11859,"eat":4993,"ea ":57325,"efe":11937,"ei ":8171,"ega":11879,"edi":19796,"ede":22533,"eda":8333,"edo":8760,"ecl":7415,"eci":35040,"ece":14294,"ecu":5255,"ect":14984,"eco":11351,"dur":9714,"duz":6967,"dor":36541,"don":5597,"dos":101787,"diç":6127,"dmi":36030,"dua":7331,"dri":7071,"dra":5496,"dre":6857,"dro":6768,"dic":13696,"did":7827,"dia":53753,"der":26564,"des":61760,"dez":6564,"dec":9473,"def":5074,"dei":8639,"del":8957,"den":70173,"dem":10237,"dep":38994,"do ":499886,"div":11366,"din":7506,"dio":15896,"dir":15370,"dis":34873,"dit":6840,"dif":6362,"rga":9198,"ri ":5474,"rgi":6642,"rge":9033,"não":10106,"rgo":7927,"ret":18367,"res":88172,"rev":9016,"rg ":6617,"rea":66834,"ref":8795,"rec":27943,"red":8922,"rei":27335,"reg":80521,"rem":9993,"ren":24469,"raç":16220,"rel":14080,"nár":5684,"rep":8548,"rda":6649,"rdo":6927,"rdi":10382,"rde":15782,"re ":57889,"rci":7180,"rce":9738,"rca":20138,"rd ":8761,"rar":6504,"ras":65057,"rat":52626,"rav":11402,"rbi":16469,"rba":5129,"rai":12729,"rag":7162,"ran":101236,"ram":28519,"ral":30318,"rab":8082,"raf":5059,"rad":42430,"rac":11489,"rs ":6575,"ros":32986,"rot":8757,"rom":13204,"ron":14637,"rop":15939,"rou":5772,"rov":31049,"rod":14281,"roc":11812,"rol":6958,"rof":9288,"nçã":8381,"rog":7781,"rno":11572,"rna":23454,"rne":8579,"rmo":8053,"ro ":148425,"rma":34000,"rme":10601,"rmi":14095,"nça":24469,"riz":8006,"rio":44890,"rit":46897,"ris":18600,"rig":17891,"ril":10251,"rin":40225,"rim":24339,"ria":73821,"rib":7744,"ric":60975,"rid":13442,"rie":17826,"rup":9891,"rus":5606,"rva":5441,"rvi":6787,"ry ":6483,"rsi":7243,"rso":14017,"rta":49041,"rto":14307,"rte":57600,"rti":26907,"rtu":25752,"rt ":7638,"rqu":17086,"rro":14834,"rri":9266,"rre":21970,"rra":23654,"sad":9694,"san":5546,"sas":9730,"sar":5392,"sa ":76681,"sid":59438,"sic":18066,"sia":15495,"sit":17633,"sis":13892,"sin":13902,"sio":8799,"sil":41142,"sim":8594,"sig":13420,"scr":14392,"scu":5033,"sde":5601,"se ":99012,"sca":8201,"sce":12058,"sci":8690,"sco":17645,"ser":25139,"ses":9941,"set":9939,"seu":20458,"seg":45509,"sed":9373,"sen":34160,"sem":9716,"sel":5712,"spo":14510,"spe":12903,"spi":7965,"spa":15990,"sol":6214,"son":13806,"sor":8549,"sos":34321,"soa":5579,"soc":9148,"sob":10181,"st ":5646,"squ":6464,"smo":15213,"so ":33266,"ssã":5427,"stá":10797,"stã":5171,"stó":7858,"sse":16316,"ssa":18526,"sso":30852,"ssi":23013,"ssu":30134,"ste":119936,"sta":117494,"spé":16790,"sto":24915,"sti":39339,"stu":7367,"str":101454,"sua":22858,"sub":11326,"sui":27030,"sul":9138,"sup":6250,"sur":5128,"tai":5662,"tal":56545,"tad":72109,"tat":5635,"tas":21431,"tar":22207,"tan":74838,"tam":58144,"te ":182044,"ta ":108579,"pa ":8506,"pe ":6181,"par":108733,"pas":6728,"pac":5622,"pal":27847,"pan":19668,"láx":5857,"pec":9282,"pen":12180,"per":70169,"paí":6811,"pet":6411,"pes":15522,"pel":52058,"pla":10972,"ple":7590,"plo":5321,"pic":9020,"pin":7594,"pio":22913,"pir":7271,"pit":9245,"por":117200,"pop":23863,"pos":31387,"pon":14947,"pol":21044,"pod":12218,"po ":20718,"lês":5033,"pub":5596,"lít":10135,"pri":47853,"pre":42479,"pro":71854,"put":8461,"pul":26066,"mão":6273,"mér":6337,"míl":18580,"qua":26640,"que":121041,"qui":34023,"ra ":178784,"mús":6480,"ngo":5638,"ngl":12535,"ngu":12950,"ni ":5382,"nge":10906,"nga":5764,"nho":23066,"nha":43874,"nhe":24382,"nei":17925,"naç":10350,"nen":5777,"ner":21401,"net":5963,"nes":11864,"ng ":14902,"nco":14783,"nci":80668,"ncl":5010,"nce":69648,"nch":7394,"nca":5561,"ne ":24199,"ndr":9286,"ndo":94739,"ndi":27820,"nde":77208,"nda":61455,"nal":50464,"nam":7322,"nan":6772,"nar":8444,"nac":9251,"nad":24850,"nag":7446,"nai":6712,"nd ":10811,"nat":14635,"nas":35071,"na ":240257,"ny ":5694,"nsã":7776,"nvo":7994,"nve":5937,"num":5422,"nut":6804,"nto":104039,"ntu":19077,"ntr":64145,"nti":32354,"nta":52915,"nte":218759,"nso":37106,"nst":25404,"nse":18037,"nsi":55540,"nsa":5243,"nt ":11783,"ns ":20234,"nom":29572,"not":5470,"nos":46255,"nor":21961,"nov":11058,"nne":6439,"no ":209889,"nid":27726,"nic":53412,"nia":29100,"niz":6361,"niv":8244,"nis":48078,"nio":9439,"nim":8021,"nin":5656,"ogr":11168,"ogi":9366,"ogo":16032,"oga":7825,"oi ":78403,"ois":10929,"oje":5415,"ol ":15035,"oce":8227,"oci":14692,"oco":7976,"oca":52787,"ode":20265,"odi":6628,"odo":29483,"of ":5154,"oda":5699,"oes":5612,"odu":11939,"ofi":9565,"oa ":8425,"obr":14956,"oví":22195,"ote":8672,"oto":8649,"ost":25890,"ota":9466,"osi":9200,"oss":38213,"oso":8522,"ovi":11009,"ovo":6748,"ova":12269,"ove":21270,"ous":5680,"our":9672,"out":17508,"opo":7367,"ope":8960,"opa":6108,"os ":396479,"opu":24150,"oló":8455,"olí":12107,"or ":155288,"orm":37181,"orn":14580,"oro":7726,"orr":17850,"ord":17760,"ore":31129,"org":12371,"ori":29601,"ou ":76190,"osa":9113,"ort":72366,"m² ":81009,"orb":15491,"ora":44778,"ola":17798,"on ":35303,"oli":16249,"ole":10370,"olo":20061,"olu":6559,"olv":8428,"om ":129315,"ona":61961,"ond":42834,"onc":16183,"one":12259,"onh":24723,"ong":8421,"oni":11825,"ono":11037,"ons":37243,"ont":48875,"oma":22403,"ome":35533,"omb":7493,"omi":12319,"omp":29869,"omo":47819,"omu":62085,"la ":68801,"le ":30025,"lac":17518,"lad":10810,"lag":5216,"lan":32402,"lam":5387,"lar":20429,"lat":13280,"las":19631,"ld ":5145,"lbu":8512,"lon":10273,"lor":10947,"loc":44637,"log":14014,"los":17433,"lme":28363,"lti":6810,"lto":6807,"ltu":6502,"lub":6858,"lta":10527,"lho":31407,"lhe":5407,"lha":24850,"lgu":5605,"lev":8296,"les":20832,"let":10999,"ler":5749,"lem":18432,"len":11746,"laç":21723,"lei":31609,"leg":6064,"lec":6739,"lo ":60493,"lla":8917,"lle":13121,"lli":8289,"ll ":7390,"lit":15034,"lis":21665,"lio":5219,"lin":27495,"lim":6007,"liz":58462,"liv":7687,"lic":27450,"lid":13334,"lia":47125,"lig":7684,"ma ":312319,"mai":41100,"mad":18812,"mar":24207,"mas":19571,"mal":6120,"man":38877,"mat":11364,"mba":7627,"mbi":6786,"mbr":26443,"mbo":5222,"me ":30188,"med":7143,"met":15815,"mes":16176,"mer":33993,"mem":6344,"mel":7946,"men":135706,"mei":21546,"maç":6372,"mbé":15876,"lva":5641,"lve":5109,"lvi":6861,"mpi":7951,"mpe":15384,"mpr":13673,"mpo":26049,"mpl":10714,"içã":16989,"mod":7097,"mon":12314,"mor":10927,"mos":10632,"mpa":8900,"mui":6353,"mul":5806,"mun":87543,"ião":65139,"min":67980,"mil":11247,"mis":7225,"mit":9135,"mic":17167,"mia":5205,"mo ":71790,"vín":22165,"zem":7471,"zaç":7006,"zad":51320,"zon":5106,"uíd":7919,"uçã":7240,"za ":11487,"uês":7863,"tón":8038,"tór":13405,"tão":9310,"tân":11804,"tár":6234,"té ":9213,"xim":6106,"xia":6380,"xa ":5938,"xce":15412,"tá ":7267,"séc":5490,"sér":7647,"são":39006,"río":17761,"rói":28842,"róp":5175,"via":10682,"vil":9868,"vim":5528,"vid":18086,"vis":17869,"rço":7357,"vo ":18229,"vol":13908,"vos":6610,"rão":7259,"vez":5961,"ver":46986,"ves":7125,"vei":6022,"ven":17920,"vem":8996,"vel":15436,"ve ":14933,"val":11487,"van":5589,"var":6549,"vas":5897,"vad":9906,"va ":59685,"uzi":7134,"utó":6849,"usi":7047,"use":5760,"usa":11181,"ust":14662,"uss":6347,"uti":10782,"ute":14945,"uta":12601,"utu":10263,"uto":22443,"utr":11202,"us ":31309,"ura":55930,"ure":6495,"urg":9235,"uri":7874,"uro":13305,"ur ":5714,"upe":7143,"upo":9984,"uma":271540,"ume":7805,"unt":10008,"uni":47036,"und":70729,"una":50258,"unh":9065,"um ":201899,"ult":13394,"ulo":24134,"uli":6278,"ulh":9149,"ula":44299,"uil":6385,"uin":8363,"uip":5474,"uis":7425,"uia":5524,"uit":14697,"ul ":16765,"ui ":29672,"uga":10215,"ugu":19601,"uda":6128,"ude":5928,"ubr":7378,"uca":5380,"ue ":100430,"uer":13174,"ues":23125,"udo":7316,"uen":9436,"uel":7358,"púb":6723,"ua ":33662,"uas":11459,"uar":8119,"ual":23558,"uan":11328,"ubl":7183,"ube":7379,"uai":5297,"uad":12055,"tur":41731,"tus":5101,"tui":5138,"tul":5524,"tub":7669,"tua":23176,"tud":7868,"tug":23030,"tre":31460,"tra":95106,"tri":60532,"tru":9663,"tro":48136,"péc":16451,"to ":188185,"tod":9530,"tou":5354,"tos":40605,"tom":5946,"ton":10767,"tor":47860,"til":14634,"tig":9433,"tir":6653,"tit":12090,"tis":7919,"tin":26516,"tim":13089,"tip":5927,"tio":9200,"tia":5519,"tic":53927,"tid":12974,"tiv":62127,"tem":39133,"ten":72655,"tei":7527,"taç":9386,"tel":22078,"teg":5010,"teb":12305,"tec":8043,"th ":5190,"tes":82988,"ter":103003,"the":7042,"ço ":13183,"ém ":22415,"édi":7117,"éci":19647,"écu":6135,"éti":5315,"éri":21433,"ênc":16544,"êne":10481,"ês ":26475,"ção":131304,"ão ":306157,"ça ":19591,"çad":10194,"áti":8887,"áve":5112,"áxi":6942,"álb":7607,"áli":5630,"ári":27812,"áre":51471,"âni":17684,"úsi":6572,"úbl":7260,"ões":32837,"ôni":11441,"óri":16535,"óno":6648,"óni":7908,"óid":28287,"íti":12961,"íst":6738,"ínc":23886,"íng":5418,"íli":21725,"íod":17716,"ípi":21652,"ís ":6687,"íci":9019,"íde":5475,"çõe":21567},"n_words":[49778514,58587553,42469388],"name":"pt","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/ro.json b/contrib/languages-data/ro.json
new file mode 100644
index 0000000..6cbc837
--- /dev/null
+++ b/contrib/languages-data/ro.json
@@ -0,0 +1 @@
+{"freq":{"D":14206,"E":15258,"F":12709,"G":12710,"A":32029,"B":21807,"C":31096,"L":13478,"M":26711,"N":11195,"O":8313,"H":8411,"I":15485,"J":5893,"K":4820,"U":8262,"T":15341,"W":4878,"V":8350,"Q":589,"P":22438,"S":30214,"R":23700,"Y":1325,"X":1901,"Z":1899,"f":89747,"g":84289,"d":270437,"e":933940,"b":76172,"c":339926,"a":811577,"n":568157,"o":427370,"l":434670,"m":224465,"j":16485,"k":15291,"h":47000,"i":839847,"w":7236,"v":72644,"u":459551,"t":525861,"s":321476,"r":567673,"q":978,"p":199529,"z":55715,"y":12885,"x":14642,"²":280,"ÃŽ":4242,"É":135,"ß":93,"î":69716,"í":464,"é":1581,"è":290,"ç":171,"ä":380,"ã":94,"â":33080,"á":1243,"à":152,"ü":921,"ú":132,"ö":583,"ó":581,"ñ":107,"Ä‚":92,"ă":161614,"Ä":266,"ć":89,"ı":136,"Ä«":130,"ÅŸ":69795,"Å‚":112,"Å":143,"Å¡":140,"Å£":70418,"Å«":99,"ÇŽ":351,"Ș":1233,"Èš":844,"É™":124,"ˈ":89,"Ì":280,"μ":153,"ν":326,"ο":443,"ι":274,"κ":132,"λ":209,"δ":99,"ε":222,"η":124,"α":406,"γ":123,"ά":91,"ί":118,"ÏŒ":118,"σ":174,"Ï‚":360,"Ï":245,"Ï€":122,"Ï…":100,"Ï„":203," l":54431,"ÑŒ":237," m":47832," n":26804," o":49358,"Ñ":249," h":3267," i":28305," j":7583," k":2023,"Ñ‹":136," d":156105," e":67780,"Ñ…":112," f":51072,"ц":117," g":13468,"ч":249,"Ñ€":882," a":134125,"Ñ":828," b":11159,"Ñ‚":574," c":116614,"у":311," y":145," x":142," z":3353," u":43415," t":26873," w":1170," v":14303,"Ñ–":84," q":90," p":90611," s":84706," r":35629,"К":133,"Ð":83,"Ðœ":137,"П":110,"Б":103,"Ð":114,"Ð’":101," J":5857," K":4744," H":8304," I":15348," N":11099," O":8211," L":13327," M":26422," B":21644," C":30661,"Р":101," A":31745,"С":176," F":12566," G":12513," D":13925," E":15142,"л":649," Z":1885,"к":766," Y":1319,"й":327," X":1885,"и":1244,"п":176,"о":1327,"н":924,"м":284," S":29902,"г":187," R":23579,"в":707," Q":580,"б":205," P":22131,"а":1480," W":4821,"з":146," V":8268," U":8214,"е":1081," T":15157,"д":332," î":68405," Ä‚":90," É":134," ÃŽ":4219," Å£":2046," ÅŸ":40504,"×”":105,"ו":138,"×":92,"ל":89,"×™":166," Ș":1227," Èš":843,"ר":104,"Ùˆ":163,"ÙŠ":302,"Ù„":318,"Ù…":220,"Ù†":216,"د":146,"Ø­":87,"ب":195,"ا":469,"ع":89,"س":117,"ر":219," Ð":114," Б":103," Ð’":101," К":131," Ðœ":137,"A ":4154," П":110,"F ":568,"Da":2015,"Cu":1954,"Cy":113,"Cl":1608,"Co":8574,"Cr":1776,"Ce":2628,"Ch":2680,"Ci":1438,"G ":509,"Ec":471,"Ed":733,"Ea":729,"Du":1500,"Do":1908,"Dr":850,"De":3817,"Di":2658,"Fe":1913,"H ":435,"Fa":1302,"Eu":3402,"Ev":451,"Ex":957,"Er":513,"Et":184,"Es":2728,"En":827,"Em":497,"Ep":280,"Ei":240,"El":1974,"Ef":84,"Eg":290,"Ge":3177,"Cá":254,"Câ":416,"Ga":1765,"I ":3107,"Fu":513,"Fr":2787,"Fo":1889,"Fl":1292,"Fi":1643,"Bâ":129,"B ":735," Р":100," С":176,"C ":1390,"Av":423,"Au":2210,"Aw":103,"Ar":4794,"At":636,"As":1391,"D ":881,"Ba":4270,"Az":208,"Ae":279,"Af":611,"Ag":465,"Ah":119,"Ab":553,"Ac":3223,"Ad":1097,"Am":1924,"An":3647,"Ap":1490,"Ai":410,"Ak":87,"Al":4305,"Bu":3694,"Br":3092,"Ca":7562,"E ":767,"Bi":3068,"Be":2955,"Bo":2204,"Bl":574,"Ku":328,"Ky":124,"Kn":99,"Kl":149,"Kr":348,"Ko":662,"Le":1965,"Gă":138,"Li":3218,"N ":854,"La":3044,"Lu":1753,"Ly":104,"Ll":318,"Lo":1983,"Me":2961,"Mi":4169,"O ":1394,"Ma":9710,"Mc":201,"My":160,"Mu":2543,"Mo":4971,"Ni":1761,"Ne":2385,"Na":2454,"P ":736,"Q ":153,"Nu":750,"No":2851,"Ol":1045,"Om":198,"On":376,"Oh":91,"Oc":1033,"Od":206,"Of":171,"Oa":113,"Ob":305,"Gi":853,"Bă":725,"Gh":757,"Gl":444,"Gr":2243,"Go":1170,"Gu":1179,"Gy":91,"J ":354,"Ha":2364,"Dâ":128,"He":1745,"Că":581,"Hi":659,"Ho":1571,"Hr":249,"Hu":882,"Hy":112,"K ":358,"Ib":100,"Ia":1254,"Id":117,"Ie":183,"Ig":92,"Io":1307,"Im":1022,"In":3678,"Il":431,"Iu":1265,"Iv":162,"Is":1017,"It":790,"Ir":549,"Ja":1638,"L ":645,"Iz":203,"Ji":337,"Je":630,"Jo":1867,"Ju":935,"Ka":1128,"M ":833,"Kh":125,"Fă":159,"Ki":625,"Ke":541,"Ut":114,"Ur":420,"Up":83,"Um":91,"Un":5673,"Ul":182,"Pă":635,"Uc":368,"W ":229,"Ty":98,"Tu":1266,"Tr":2949,"To":1782,"Th":2212,"Ti":1500,"Te":2836,"Pâ":232,"Ta":1163,"V ":1269,"Sw":112,"Sz":144,"Sy":285,"St":5254,"Su":2579,"Wr":229,"Wo":691,"Wi":1273,"Ră":813,"Wh":234,"Wa":1039,"Sâ":220,"We":859,"Y ":147,"Vo":713,"Vr":133,"Vu":111,"Râ":933,"Vi":1974,"Vl":297,"X ":1079,"Va":2283,"Ve":1567,"Lă":421,"Pu":736,"Pr":4511,"S ":1482,"Pe":2677,"Pa":6072,"Pl":931,"Po":3427,"Pi":1539,"Ph":445,"Os":331,"Ot":294,"Ou":154," ا":202,"Ov":120,"Op":423,"Or":1772,"R ":901,"Kö":111,"Sf":926,"Se":3730,"Sc":1718,"Si":2479,"Sh":841,"Nă":243,"Sm":167,"Sl":471,"Sk":113,"Sp":3075,"So":2382,"Ru":1542,"U ":435,"Sa":3082,"Re":6657,"Ri":1089,"Mă":769,"Rh":203,"Ro":9754,"Qu":329,"T ":645,"Mü":187,"Ra":1639,"Wü":172,"b ":5072,"a ":212743,"Ye":128,"Tă":194,"Ya":168,"Yo":679,"Yu":120,"Z ":124,"Tâ":356,"Să":464,"Vă":154,"Za":383,"Ze":392,"Zi":317,"Vâ":376,"Zo":278,"i ":179645,"gd":205,"ge":13414,"câ":4391,"ga":11234,"gb":146,"fl":4423,"ff":711,"bâ":136,"fi":21259,"fr":4539,"fu":3903,"ft":996,"fo":28249,"j ":2040,"gy":251,"dâ":214,"he":10237,"ha":6256,"gn":1318,"gm":382,"gl":4043,"gi":18379,"gh":4326,"bă":2234,"gg":253,"gv":296,"gu":7182,"gt":188,"gs":381,"gr":9341,"cî":121,"go":3415,"dt":248,"du":15963,"dv":200,"dw":264,"dy":391,"dz":86,"g ":5818,"ea":59379,"eb":4646,"ec":30872,"ed":13772,"de":117725,"dd":311,"dg":263,"di":66800,"dh":111,"dj":215,"dm":1691,"eM":86,"dl":291,"do":13587,"dn":210,"ds":959,"dr":6694,"ew":1340,"ex":7454,"eu":5496,"ev":8185,"ey":1419,"ez":15187,"fa":8662,"h ":3714,"fe":10587,"eh":1815,"eg":14417,"ef":4744,"ee":3471,"el":50297,"ek":775,"ej":687,"ei":30112,"ep":15349,"eo":6294,"en":63988,"em":24435,"et":20416,"es":79718,"er":87843,"ca":58257,"e ":363120,"by":328,"bs":1372,"br":14880,"bu":7988,"bt":195,"bn":117,"bo":5650,"bl":6057,"bm":100,"bi":12351,"bb":308,"bc":108,"bd":241,"be":6318,"db":261,"da":18159,"f ":3092,"cz":119,"cy":195,"cv":646,"cu":45953,"ct":20001,"cs":291,"cq":100,"cr":11320,"co":42083,"cm":243,"cn":187,"ck":2747,"cl":5298,"ci":32813,"ch":13925,"ce":51893,"cc":2745,"c ":18619,"aP":84,"aC":95,"az":9879,"ay":1659,"ba":11641,"d ":22818,"at":93723,"as":21874,"ar":105841,"aq":90,"ax":1099,"aw":499,"av":8565,"au":21651,"ak":1416,"al":87116,"ai":18198,"aj":3933,"ao":374,"ap":15113,"am":19806,"an":87690,"ac":26532,"ad":16915,"aa":641,"ab":6836,"ag":7060,"ah":1692,"ae":3048,"af":7326,"nu":32036,"nt":67674,"ns":20163,"nr":1134,"np":113,"no":17936,"nn":2298,"q ":85,"nz":1824,"ny":836,"nw":156,"nv":2089,"oe":1765,"of":5818,"oc":21548,"od":10761,"oa":21531,"ob":5951,"om":43493,"on":56179,"ok":801,"ol":30865,"oi":8296,"oj":309,"og":8118,"oh":1090,"m²":273,"ot":11364,"os":27341,"ov":12192,"ou":7338,"op":17725,"oo":1964,"or":85929,"r ":49086,"ox":1908,"ow":1237,"oz":3227,"oy":555,"pe":42586,"lâ":532,"pa":25762,"pc":98,"pl":7066,"lé":112,"po":22942,"ph":1089,"pi":11751,"lo":40730,"ln":822,"lm":3343,"ll":6217,"ls":1221,"lp":822,"lv":2229,"lu":48097,"lt":11451,"lz":261,"ly":862,"o ":37058,"mc":90,"ma":44395,"mb":17891,"eÅŸ":9887,"hă":248,"me":40474,"mf":257,"eÅ£":7364,"ml":160,"iè":112,"mi":28493,"mn":4278,"mm":955,"mp":17238,"mo":11698,"mt":225,"ms":601,"mu":21624,"my":299,"p ":5018,"na":37617,"nb":568,"nc":20662,"nd":32264,"ne":45632,"nf":4807,"ng":14388,"nh":479,"ni":65371,"nj":736,"nk":1221,"nl":1126,"nm":330,"ju":7523,"jo":2219,"jl":509,"bÅ£":531,"ki":1735,"kh":269,"fă":1862,"cÅŸ":118,"gâ":388,"kf":113,"ke":1642,"ka":1464,"m ":15258,"ky":316,"ks":661,"kt":260,"ku":385,"ko":995,"kr":318,"kk":161,"cÅ£":4949,"kl":292,"km":1457,"li":54376,"lh":237,"gă":2958,"lk":383,"lj":135,"le":70613,"ld":4148,"lg":2038,"lf":1236,"hâ":94,"la":55152,"lc":2339,"lb":4110,"n ":164001,"hr":1429,"hs":252,"hw":257,"ht":1909,"hu":2313,"că":22340,"hi":13680,"hn":1591,"ho":3277,"hl":404,"hm":291,"id":10613,"ic":65212,"ib":4937,"ia":63950,"dă":2289,"ih":1392,"ig":9132,"aÅŸ":9011,"if":6185,"ie":62856,"hy":354,"k ":4658,"iq":165,"eî":135,"ir":13340,"is":36911,"it":62439,"iu":24448,"iv":15138,"iw":97,"ix":744,"ii":33158,"ij":1160,"aÅ£":19894,"ik":1190,"il":42983,"im":26881,"in":122257,"io":22518,"ip":8638,"je":1168,"fâ":980,"ji":658,"iz":12018,"iy":146,"l ":109782,"ja":1735,"pÅ£":503,"să":10006,"xi":3113,"xo":371,"té":92,"tî":216,"xp":1513,"xt":2105,"xu":441,"ww":338,"z ":3980,"xc":491,"xa":1180,"tâ":2269,"xe":1609,"oÅŸ":1503,"ră":15639,"wi":755,"oÅ£":1099,"sé":93,"wn":249,"wo":518,"sî":86,"wr":185,"ws":439,"vy":161,"y ":7000,"wa":1979,"sâ":398,"we":941,"vl":85,"ré":180,"nÅ£":14138,"vi":18551,"râ":2794,"nÅŸ":236,"vu":1822,"vr":1272,"rî":85,"vs":260,"rí":114,"vn":113,"vo":6165,"uz":4516,"uy":132,"ux":811,"uw":93,"uv":2854,"uu":147,"ve":20356,"va":13083,"x ":3188,"ui":36731,"uj":940,"mÅ£":245,"uk":412,"ul":105505,"ue":3558,"uf":779,"ug":3958,"pă":8327,"uh":313,"ur":41535,"us":16414,"ut":20794,"um":22214,"un":77098,"uo":207,"up":12202,"ty":1026,"tz":478,"tu":41090,"tt":2205,"tw":494,"tv":129,"ub":9499,"ua":12620,"ud":10993,"uc":12122,"w ":1362,"to":36489,"tn":678,"tm":622,"tl":1983,"ts":1249,"tr":54687,"tp":270,"pâ":2492,"tf":804,"te":135412,"td":104,"lÅ£":1071,"tk":89,"ti":57663,"th":5025,"v ":5358,"tb":1584,"tc":809,"ta":57242,"su":20142,"sv":111,"ss":3065,"st":121541,"sy":354,"sz":231,"sw":164,"sl":2146,"sk":1369,"sn":714,"sm":3046,"sp":10131,"so":10359,"sr":365,"sd":200,"sc":21675,"sf":2108,"se":33580,"sh":1776,"nă":16784,"sg":163,"si":25637,"rz":902,"u ":55858,"sa":21555,"sb":549,"rr":1909,"rs":9764,"rt":23612,"ru":36664,"rv":2469,"rw":210,"ry":1487,"rp":2476,"ro":43127,"rn":8536,"rm":17803,"né":122,"rl":4592,"nç":103,"rk":1642,"rj":246,"ri":114051,"mă":7183,"rh":1830,"rg":8792,"rf":1291,"nâ":643,"re":125851,"rd":9049,"rc":6829,"rb":4129,"ra":67426,"t ":87746,"qu":761,"mé":97,"iÅ£":6799,"mâ":11991,"lă":11508,"iÅŸ":3714,"s ":29143,"px":1165,"py":91,"pt":7489,"pu":17959,"pp":664,"pr":42710,"ps":1182,"ză":8661,"zâ":331,"uÅ£":3113,"xă":383,"uÅŸ":1691,"vă":2660,"zz":266,"sÅ£":297,"zf":124,"vâ":2188,"rÅŸ":635,"zg":128,"uă":2010,"zi":14356,"zb":1622,"zd":236,"ze":7824,"vá":88,"za":10358,"zv":1283,"zy":102,"zu":1714,"zt":122,"zo":3689,"zn":131,"rÅ£":3519,"zm":105,"zl":93,"tă":34729,"ye":514,"yc":205,"yd":240,"ya":763,"yb":116,"yw":120,"yu":112,"yt":228,"ys":611,"yr":295,"yp":159,"yo":376,"yn":435,"ym":285,"yl":584,"yi":202,"² ":275,"ÃŽ ":96,"ÃŽn":3944,"ÃŽm":147,"án":283,"ác":265,"ár":117,"ás":83,"âl":535,"âm":971,"ân":23884,"âi":324,"âu":2278,"ât":1719,"âr":2512,"à ":112,"á ":120,"アアア":92,"ón":162,"ó ":105,"în":66365,"ín":93,"îi":280,"îl":220,"îm":1778,"ía":115,"î ":282,"âş":625,"él":95,"én":106,"és":86,"ér":215,"ée":141,"èr":89,"é ":392,"ăc":1712,"ăd":1353,"ăi":1593,"ăj":113,"ăg":390,"ăm":1728,"ăn":1902,"ăl":2480,"ăr":15029,"ăp":813,"ăv":254,"ău":2670,"ăt":8950,"ăs":5402,"ăz":2233,"ăb":253,"Ä‚ ":96,"ă ":108634,"ün":238,"ür":317,"ör":105,"ön":99,"öl":118,"îş":473,"ăţ":4694,"ăş":1107,"ÅŸo":1467,"ÅŸt":10708,"ÅŸu":3626,"ÅŸi":42994,"ÅŸn":265,"ÅŸm":86,"ÅŸa":1934,"ÅŸc":1152,"ÅŸe":2787,"ÅŸ ":4403,"şă":140,"Å£e":5578,"Å£i":45859,"Å£u":4697,"Å£a":7179,"Å£ ":1101,"ţă":5758,"ã‚¢":157,"ÇŽ ":253,"Èši":110,"Èša":292,"Ș ":94,"Și":132,"Șo":113,"Șt":390,"Èš ":94,"Șc":133,"Șa":135,"Șe":144,"Ță":266,"之":98,"三":194,"ä¸":102,"ος":175,"ος ":175,"Ï‚ ":359,"ν ":94,"α ":133,"アア":124,"Ñк":290,"та":92,"ÑÑ‚":158," Ga":1748," Câ":415," Ge":3154," Cá":254," I ":808," Fo":1868," Fu":504," Fr":2770," Fi":1622," Bâ":129," Fl":1278," Ha":2353," He":1731," Dâ":128," Gy":84," J ":235," Go":1158," Gr":2202," Gu":1163," Bă":725," Gh":755," Gi":841," Gl":431," Ig":92," Ie":183," Id":117," Ib":97," Ia":1248," K ":217," Hy":109," Hu":880," Hr":248," Ho":1557," Că":576," Hi":652," Ji":335," Je":625," L ":250," Ja":1632," Iz":203," Iu":1263," Iv":159," Ir":548," Is":1004," It":785," Im":1017," In":3630," Io":1287," Il":430," M ":262," Ka":1125," Ke":531," Ki":614," Kh":124," Fă":159," Jo":1852," Ju":930," N ":206," La":3007," Le":1942," Gă":138," Li":3175," Kl":144," Kn":98," Ko":661," Kr":346," Ku":321," Ky":124," Mc":200," Ma":9571," O ":853," Mi":4107," Me":2921," Lo":1967," Ll":318," Ly":104," Lu":1743," Ne":2351," P ":197,"а ":320," Na":2442," Ni":1749," Mo":4944," My":159," Mu":2527," A ":2659," B ":442," C ":439," Ap":1483," Am":1916," An":3623," Ak":86," Al":4275," Ai":406," Ag":455," Ah":117," Ae":277," Af":571," Ac":3199," Ad":1079," Ab":530," Ba":4239," D ":276," Az":205," Aw":102," Av":410," Au":2198," At":631," As":1386," Ar":4774," Be":2935," Bi":3051," Bl":566," Bo":2181," Br":3074," Bu":3675," E ":267," Ca":7457," Ce":2586," Ci":1423," Ch":2660," Cl":1578," Cr":1755," Co":8451," Cu":1909," Cy":110," F ":199," Da":1990," Di":2603," De":3729," Dr":841," Do":1853," Du":1487," Ea":726," Ec":468," Ed":725," G ":189," El":1961," Ei":240," Eg":286," Ef":83," Et":180," Es":2724," Er":511," Ep":280," En":810," Em":492," Ex":936," Eu":3393," Ev":441," Fe":1900," Fa":1274," H ":241," Să":460," Tâ":354,"к ":95," Wr":229," Wo":669," Wi":1265," Ră":813," Wh":231," We":849," Sâ":220," Wa":1026," Y ":114,"й ":231," Zo":278," Ze":389," Vâ":375," Zi":312," Za":381," Yu":120," Yo":676," Ya":164," Tă":194," Ye":128," Wü":172,"о ":126,"н ":131," Vă":153," a ":36875," R ":300,"в ":116," Kö":111," Ou":147," Ov":103," Os":331," Ot":292," Or":1764," Op":422," Po":3392," Pl":908," Pi":1531," Ph":433," Pe":2652," Pa":6041," Q ":118," Nu":741," No":2833," Ol":1044," On":368," Om":193," Oh":91," Od":204," Oc":1030," Of":168," Ob":298," Oa":113," Mü":186," Ra":1619," T ":190," Qu":324," Ro":9725," Re":6614," Ri":1081," Mă":767," Rh":203," S ":442," Pr":4358," Pu":723," Lă":359," Sz":142," Sy":283," Sw":111," Su":2564," St":5193," Ta":1158," V ":299," Th":2181," Ti":1481," Pâ":232," Te":2810," Tr":2917," To":1756," Ru":1539," Sa":3068," U ":212,"е ":128," Sh":834," Nă":243," Si":2443," Sc":1686," Se":3700," Sf":922," So":2361," Sp":3054," Sk":111," Sl":463," Sm":160," Va":2273," X ":295,"и ":98," Ve":1551," Vi":1939," Râ":933," Vl":295," Vo":709," Vu":110," Vr":132," Tu":1248," Ty":98," W ":133," Uc":367," Pă":634," Ul":182," Um":91," Un":5644," Ur":414," Ut":114," ja":470," l ":435," iz":399," io":137," ip":103," im":2637," in":15177," il":210," iu":2496," is":1936," it":606," ir":289," ka":124," m ":620," fă":1009," ki":195," gâ":148," jo":1049," ju":5955," ha":692," he":341," gi":362," gh":227," bă":568," gl":415," gr":4643," cî":96," go":322," gu":1027," ia":2677," id":624," ie":296," dă":97," aÅŸ":533," că":5014," hi":694," ho":634," hr":508," ht":170," ni":1017," ne":2985," na":3334," p ":138," mu":6108," mo":5925," ol":415," om":2225," on":472," oc":2425," od":198," of":2241," oa":721," ob":2230," nr":213," nu":8582," no":5141," le":5248," gă":633," li":8299," n ":4015," la":23505," km":1385," me":7797," eÅŸ":107," mi":5999," o ":24882,"Ñ ":186," ma":19097," lu":7950," lo":8243," ae":559," af":3722," ag":621," ab":871," ac":11612," ad":3606," am":3470," an":10041," ap":7602," ai":935," aj":609," al":27010," av":3789," au":7831," ax":133," ar":7993," at":2213," as":3512," d ":818," ba":2886," az":461," bi":2225," be":668," bo":1156," bl":302," bu":1418," br":1785," ca":30202," e ":424," c ":92," er":1837," et":1185," es":44667," en":2470," em":716," ep":805," ei":565," el":3115," ef":535," eg":319," fe":3014," fa":5201," eu":1087," ev":1448," ex":4580," fu":2227," fr":3215," fo":24062," fl":777," fi":11367," ge":4653," câ":3570," ga":964," i ":494," cl":2039," cm":100," co":29681," cr":3733," cc":163," ce":15081," ch":1829," ci":2979," f ":104," da":4946," cu":21872," do":6118," dr":1888," de":93131," di":45684,"ч ":147," ec":2186," ed":1091," ea":295," eb":276," du":3328,"ль":104," vă":281," zo":1012,"ла":120,"ли":124,"ко":168," ze":739," zb":153," zi":1040," zf":120," vâ":719,"ка":188,"ки":145," tă":177," să":3897,"ин":138,"ик":85," ww":167,"ий":142," tâ":352,"ич":160,"ри":89,"ро":151,"ра":156,"ре":83,"оÑ":104,"ор":119,"ол":94,"ов":307," uÅŸ":254,"но":129,"ни":117,"на":162," ru":1689," sa":12280," sf":678," se":17401," sc":5833," si":9687," sh":138," nă":1457," sl":456," sp":5202," so":3426,"ви":183,"во":101," ra":2561," re":17790," ri":1713," mă":1822," ro":8367," pu":4423," pr":28214," ps":399," s ":2092," px":1165," lă":254," mâ":207," os":197," ot":110," op":1754," or":11013,"ан":170," ox":160,"ал":98," pe":25650," lâ":421," pa":9591,"ар":142," pl":2462," po":12520,"аÑ":98," pi":2615," wa":449," sâ":329," we":213," wr":124," wi":111," ră":1828," x ":110," va":2539," ve":4691," uz":162," vo":2209," vr":384," vu":171," râ":1545," vi":3164," uc":310,"еÑ":86,"ер":137,"ен":133," tu":1391," us":154," ut":956," ur":2048," um":561," un":37329," ul":1504," pă":1596," ta":986," st":9754," su":13217,"ев":109," tr":8292," to":2637," th":2187," ti":4195," te":6492," pâ":1627,"Țăr":258," ÃŽ ":96," ÃŽm":146," ÃŽn":3919," în":65579," îl":200," îm":1722," îi":236," î ":223," îş":424," Ä‚ ":90," Å£a":471," ÅŸt":950," ÅŸo":136," ÅŸi":37976," ÅŸc":219," ÅŸe":338," ÅŸa":764," Å£i":391," Å£e":161," ţă":1008," Ță":265," Èša":292," Èši":110," Șe":142," Șc":133," Șa":134," Èš ":92," Șt":387," Șo":113," Și":132," Ș ":91,"ال":185,"ÙŠ ":110,"Ù† ":132,"AS ":90,"BC ":103,"Feb":468,"Fed":536,"Fel":141,"Fer":333,"Fes":98,"Bâr":109,"Fil":558,"Fin":381,"Fir":91,"Fie":114,"Ext":451,"Fam":192,"Fan":103,"Fal":98,"Far":165,"Fac":313,"Fab":84,"Era":97,"Eri":131,"Est":2527,"Eur":3119,"Eva":149,"Eve":110,"Eug":130,"Exp":186,"Exi":117,"Exc":103,"Evu":86,"El ":710,"Ele":546,"Enc":124,"Eng":174,"Ene":132,"Emi":260,"Elv":237,"Eli":210,"Epi":189,"Ent":184,"Câm":221,"Cân":103,"Các":221,"Ger":1707,"Geo":684,"Gen":482,"Gla":99,"Ghe":506,"Băl":198,"Ghi":176,"Băt":252,"Gil":105,"Gir":238,"Giu":159,"Gaz":102,"Gal":559,"Gam":141,"Gav":159,"Gar":269,"Gab":147,"Fun":252,"Fru":86,"Fro":141,"Flo":463,"Fla":640,"Fra":1957,"Fri":249,"Fre":285,"Fon":202,"Fot":362,"For":809,"Fox":86,"II ":1544,"Dâm":116,"Căl":223,"His":116,"Hil":108,"Hel":236,"Hei":175,"Hea":139,"Hen":243,"Hes":101,"Her":541,"Hal":221,"Hai":98,"Han":325,"Ham":243,"Har":682,"Hau":104,"Gur":143,"Guv":353,"Gua":120,"Gui":207,"Gre":662,"Gri":261,"Gra":547,"Gru":280,"Gro":360,"ţă ":4334,"Glo":227,"Goo":112,"Gol":285,"Got":84,"ţăt":101,"Gor":250,"ţăr":1089,"ţăm":207,"Inv":138,"Ioa":445,"Inf":292,"Ini":100,"Int":1173,"Ins":803,"Ion":531,"Ios":98,"Ior":163,"Ili":165,"Ill":92,"Inc":150,"Ind":606,"Imp":850,"In ":158,"IaÅŸ":404,"Ier":125,"Ian":535,"Ial":83,"Hun":383,"Hum":90,"IX ":412,"Hug":91,"IV ":391,"IT ":137,"Hor":215,"Hou":138,"Hot":160,"Hom":127,"Hon":117,"Hol":365,"Hr ":133,"Arg":385,"Arh":350,"Are":1696,"Arc":194,"Ard":100,"Ara":584,"Arm":402,"Ari":241,"Apo":195,"Apr":477,"Ate":96,"Atl":247,"Ast":296,"Ass":154,"Asi":398,"Aso":236,"Art":517,"Au ":88,"Avi":111,"Ave":127,"Aut":413,"Aus":786,"Aur":237,"Apă":85,"Aug":551,"Bai":170,"Bal":441,"Ban":574,"Bab":153,"Bac":358,"Bad":439,"Baz":111,"Bay":92,"Bar":873,"Bat":155,"Bas":347,"Bav":161,"CD ":118,"Abr":91,"Aca":558,"Act":175,"Ada":136,"Ace":1958,"Acc":188,"Adu":103,"Adm":250,"Ado":166,"Adr":161,"Ade":100,"Afa":103,"Aer":222,"Age":153,"Afr":381,"Agr":117,"Air":208,"Al ":217,"Ala":198,"Alb":1104,"Alg":107,"Ali":309,"Alc":89,"Ale":1016,"Alf":175,"Alt":180,"Alm":145,"All":248,"Alp":203,"Ame":1318,"Amb":85,"Ama":153,"Ang":951,"Ani":130,"Ana":326,"And":938,"Ant":732,"Ann":166,"Apa":378,"But":87,"Bus":115,"Buz":155,"Bul":406,"Bun":176,"Bur":449,"Buc":1806,"Bud":195,"Bru":284,"Bră":148,"Ca ":119,"Cab":93,"Cal":738,"Cam":777,"Cas":1102,"Car":1723,"Cau":101,"Cat":1040,"Can":883,"Cap":564,"Bea":150,"Bet":104,"Ber":735,"Ben":285,"Bel":1128,"Bib":207,"Bil":178,"Bih":202,"Bis":1833,"Bir":155,"Bio":98,"Blo":128,"CN ":94,"CO ":85,"Bla":274,"Bre":357,"Bra":1081,"Bro":311,"Bri":819,"Bog":147,"Boe":111,"Bol":190,"Bon":168,"Boo":93,"Bor":292,"Bos":170,"Bot":287,"Bou":149,"Cuv":164,"Cur":363,"Cup":386,"Cul":405,"De ":453,"Dez":104,"Der":87,"Det":101,"Des":342,"Dev":138,"Deu":92,"Del":231,"Dem":343,"Den":254,"Dep":258,"Dea":199,"Dec":652,"Dam":105,"Dan":549,"Dar":174,"Dat":151,"Dav":338,"Dac":187,"Dal":97,"Chr":305,"Che":298,"Chi":1153,"Cip":91,"Cin":138,"Cio":191,"Cit":251,"Ciu":252,"Civ":93,"DN ":145,"Cle":88,"Cla":363,"Cea":181,"Ceh":224,"Cel":451,"Cen":839,"Cet":187,"Cer":452,"Cha":723,"Cri":558,"Cra":341,"Cre":321,"Cu ":179,"Cru":182,"Cro":263,"Cli":120,"Clo":93,"şă ":93,"Clu":841,"Coc":83,"Coa":119,"Cod":224,"Cop":211,"Cos":228,"Cor":1033,"Com":2355,"Col":727,"Coo":87,"Con":2677,"Cou":282,"Cot":116,"Cov":111,"Ea ":436,"FA ":238,"Egi":189,"FI ":85,"Edu":88,"Edi":420,"Eco":139,"Ech":197,"Å£ur":116,"Eas":194,"Å£ui":121,"Å£ul":4343,"FC ":245,"Å£ar":732,"Å£at":1470,"DeÅŸ":155,"Å£e ":1287,"Å£a ":4852,"Dia":147,"Dic":236,"Dis":609,"Dir":129,"Dio":102,"Din":623,"Dim":132,"Die":129,"Div":260,"Å£it":365,"Å£iu":2393,"Å£ir":210,"Duc":160,"Å£in":4093,"Å£io":5262,"Dup":205,"Å£il":2341,"Å£im":464,"Dun":357,"Dum":429,"Å£ii":7297,"Å£if":426,"EX ":88,"Dur":97,"Ð°Ñ ":95,"Å£ia":6859,"Å£ie":10551,"Å£ei":1511,"Å£en":180,"Å£el":1678,"Å£es":190,"Dre":250,"Dra":255,"Å£i ":5299,"Doi":184,"Dob":158,"Dou":102,"Dol":170,"Don":271,"Dom":358,"Dor":224,"Å£ea":461,"Nea":360,"Neg":223,"Nev":114,"Neu":159,"Net":137,"Nep":85,"Nas":96,"Nat":327,"Nav":140,"Nig":112,"Nic":784,"Nis":298,"Nin":99,"Nik":102,"NaÅ£":910,"New":711,"Nap":289,"Nam":92,"Num":477,"OS ":93,"Nou":391,"Nov":172,"Nor":1220,"Not":118,"Noi":520,"Nob":138,"ÃŽnc":117,"Oct":512,"ÃŽnt":212,"Ode":88,"PC ":107,"Oce":295,"ÃŽmp":136,"Obe":86,"ÃŽn ":3310,"Oto":155,"Olt":287,"Oli":317,"Ola":252,"Ono":115,"One":96,"Ope":278,"Ora":462,"Ort":215,"Osc":136,"Ord":181,"Ori":268,"Org":211,"PeÅŸ":117,"Plo":152,"Ple":87,"Pla":572,"Pin":170,"Pit":107,"Pir":96,"Pie":423,"Pic":155,"Pia":306,"Pho":97,"Phi":218,"Ped":83,"Per":522,"Pet":738,"Pen":513,"Pel":85,"Pe ":267,"Pat":391,"Pas":174,"Par":3240,"Pav":117,"Pau":327,"Pac":294,"Pan":274,"Pap":271,"Pal":567,"Pub":157,"Pur":83,"Put":143,"Pro":1436,"Pri":1498,"Pre":1075,"Pru":161,"Pra":285,"Pod":203,"Poa":126,"Pol":793,"Pom":101,"Pon":142,"Poi":169,"Pot":185,"Pos":133,"Pop":623,"Por":507," ال":165,"Lăc":323,"SA ":143,"Rac":113,"Rad":382,"Rai":238,"Ram":106,"Mün":150,"Ran":107,"SD ":117,"Que":155,"Isa":101,"Irl":266,"Ita":687,"Isl":175,"Isr":173,"Ist":357,"Ira":132,"Iug":109,"Iva":127,"Iul":580,"Iun":435,"Izv":134,"Jae":83,"Jac":383,"Jap":315,"Jan":217,"Jam":257,"Jer":140,"Jea":190,"Jim":122,"Jos":531,"Jon":151,"Joh":545,"Joc":288,"Jud":261,"Jus":113,"Jur":102,"Jul":125,"Jun":115,"Kal":121,"Kan":96,"Kat":124,"Kar":336,"Ken":149,"Kir":86,"Kin":185,"Kie":90,"Kon":94,"Kos":163,"Kre":87,"Kra":118,"Lew":83,"Lev":90,"Let":83,"Les":120,"Leo":277,"Len":176,"Lei":93,"Leg":332,"Lee":87,"Lea":120,"Lau":147,"Laz":87,"Le ":90,"Las":86,"Lat":109,"Lar":154,"Lam":95,"Lan":338,"Lac":259,"Lab":85,"La ":1181,"Lle":258,"Lib":362,"Lic":139,"Lie":89,"Lig":414,"Lim":410,"Lin":461,"Lis":431,"Lit":301,"Liv":198,"MI ":96,"Lux":121,"Lup":107,"Lum":137,"Lun":558,"Lud":146,"Luc":356,"Lou":179,"Lov":137,"Los":244,"Lot":89,"Loc":483,"Lor":132,"Lon":361,"Meh":106,"Men":147,"Mem":162,"Mel":209,"Mes":271,"Mer":299,"Met":461,"Mec":135,"Med":595,"Mex":216,"Man":1131,"Mal":342,"Mar":4668,"Mas":368,"Mag":383,"Mad":271,"Maj":121,"Mai":903,"Mac":356,"May":100,"Max":155,"Mau":132,"Mat":338,"Mod":176,"Mol":1838,"Mon":1277,"Mos":318,"Mor":392,"Mou":115,"Mot":179,"Mih":609,"Mik":106,"Mij":92,"Mid":157,"Mig":86,"Mic":936,"Mit":232,"Mir":321,"Mis":279,"Mil":373,"Min":651,"Muz":367,"Mun":1060,"Mul":94,"Mur":508,"Mus":300,"NU ":88,"Săl":172,"Săr":113,"Târ":305,"XX ":206,"XV ":95,"Wre":180,"Wor":386,"Wol":146,"Whi":111,"Răd":101,"Răz":528,"Răs":89,"Wil":518,"Win":333,"Wie":86,"Wit":97,"ère":85,"Web":84,"Wes":374,"Sân":132,"Was":121,"War":230,"Wat":92,"Wal":290,"Vra":93,"ée ":112,"Vol":202,"Voi":185,"Vis":124,"Vit":135,"Vla":243,"Ziu":91,"Zon":137,"Zee":92,"Vâl":191,"Vâr":128,"âşt":551,"Yor":388,"You":200,"на ":94,"Stă":112,"Sys":110,"Stî":142,"Sur":132,"Sus":204,"Sul":83,"Sup":346,"Sun":219,"Sue":270,"Sud":589,"Suc":243,"Sub":164,"Str":664,"Stu":292,"Sti":105,"Sto":394,"Sta":2644,"Ste":728,"Teh":119,"Tea":170,"Tec":113,"Ten":121,"Tem":164,"Teo":264,"Tel":395,"Tan":144,"Tat":111,"Tar":189,"Tai":83,"Tal":112,"UA ":470,"Sfâ":498,"Shi":108,"She":197,"Năs":219,"Sho":130,"Sha":300,"Sim":365,"Sil":252,"Sig":138,"Sit":102,"Sis":180,"Sir":233,"Sin":438,"Sie":127,"Sib":370,"Sfi":241,"Sez":141,"Ser":760,"Sev":325,"Sf ":135,"Scr":179,"Sep":495,"Sen":217,"Sel":98,"Sem":99,"Sec":315,"Sea":111,"TV ":299,"Spa":2255,"Spi":211,"Spe":223,"Spr":121,"Spo":178,"Sof":165,"Soa":134,"Soc":483,"Sou":289,"Sov":352,"Sol":260,"Som":154,"Son":175,"Sor":138,"Sla":128,"Slo":264,"RoÅŸ":199,"Rus":998,"Sai":239,"Sam":225,"Sal":380,"Sab":86,"Se ":674,"Sco":358,"Sci":132,"Sch":680,"Sca":185,"Sax":130,"Sav":134,"Sat":330,"Sau":85,"Sar":201,"San":758,"ови":108,"TA ":103,"Rez":129,"Res":168,"Ret":90,"Rev":440,"Măn":337,"Măr":265,"Rhe":108,"Riv":111,"Rin":182,"Ric":330,"Rap":96,"Ref":116,"Rec":405,"Red":97,"Rei":117,"Reg":1929,"Rem":121,"Ren":298,"Rel":102,"Rep":1978,"Rea":273,"Rol":118,"Rob":343,"Roc":197,"Rod":127,"Roy":87,"Rot":120,"Ros":294,"Rom":7794,"SS ":239,"SO ":102,"ReÅŸ":139,"Vel":106,"Ven":242,"Vec":214,"Ñки":101,"Vas":382,"Van":141,"Val":1157,"Var":247,"Vic":330,"Vie":288,"Vir":168,"Vil":251,"Vin":215,"Râu":810,"Ver":423,"Ves":364,"Păm":274,"Păd":123,"Ung":582,"Uni":3993,"Un ":792,"VD ":93,"Ucr":344,"VI ":204,"Tex":109,"Ter":953,"Tes":175,"Pân":88,"Pâr":129,"Tha":108,"The":1600,"Thi":130,"Tho":227,"Tib":83,"Tim":723,"Tin":109,"Tit":147,"Tir":112,"Top":132,"Tor":259,"Tok":94,"Tol":279,"Tom":280,"Ton":126,"Tot":116,"Tou":121,"Tru":137,"Tro":183,"Tri":361,"Tre":328,"Tra":1888,"Tur":781,"Tul":107,"Tun":93,"Tud":91,"вич":136,"biz":191,"bis":684,"bit":942,"biu":329,"bio":569,"bir":494,"baÅ£":118,"bil":3719,"bin":1111,"bii":385,"beÅŸ":168,"bo ":147,"blu":442,"ÅŸa ":427,"blo":266,"ble":593,"bli":4345,"bla":309,"boa":326,"bol":798,"boi":1231,"bog":143,"biÅŸ":155,"biÅ£":97,"ÅŸe ":206,"ÅŸca":505,"ÅŸal":128,"ÅŸan":384,"ÅŸap":297,"bon":331,"bom":116,"bor":1299,"bot":233,"bos":94,"bov":171,"ÅŸar":126,"ÅŸas":296,"bou":197,"box":118,"ÅŸat":99,"be ":463,"ban":1063,"bal":2277,"bai":156,"baj":238,"bac":607,"bab":295,"án ":151,"baz":1371,"bat":704,"bas":635,"bar":766,"bdi":107,"bea":119,"bi ":576,"bei":94,"bee":100,"bec":120,"ber":2623,"ben":568,"bel":997,"bes":190,"bet":600,"bia":703,"bib":169,"bic":703,"bie":1278,"áce":223,"ÅŸnu":136,"buÅ£":387,"buÅŸ":122,"ÅŸoa":686,"ÅŸor":235,"ÅŸov":403,"ca ":12197,"car":20537,"cas":995,"cat":4867,"cau":568,"can":4982,"cap":1522,"caz":602,"cav":140,"cac":106,"cab":250,"cad":1721,"cam":769,"cal":7164,"caf":88,"cai":123,"ÅŸu ":299,"ce ":13744,"bri":8393,"bro":479,"ÅŸco":241,"bra":1280,"bre":732,"bu ":154,"ÅŸea":100,"bru":3685,"ÅŸed":992,"bso":227,"bse":314,"ÅŸez":265,"ÅŸev":87,"bst":592,"ÅŸel":294,"ÅŸef":250,"ÅŸer":161,"ÅŸen":171,"ÅŸi ":40695,"bur":1502,"bul":939,"bun":893,"bum":1812,"bui":746,"buc":290,"but":536,"bus":305,"buz":102,"ÅŸcă":238,"ÅŸie":311,"ÅŸii":223,"by ":257,"ÅŸia":83,"bră":230,"ÅŸit":599,"ÅŸir":239,"ÅŸin":563,"ÅŸil":101,"aka":186,"am ":1398,"ake":307,"aki":175,"ajo":811,"aju":1101,"al ":30968,"aja":297,"aje":549,"adă":644,"ail":713,"aim":241,"ain":2042,"aio":622,"air":263,"ais":301,"ait":135,"aiu":141,"ak ":275,"aie":333,"aid":284,"aic":780,"aib":127,"aia":581,"ahn":102,"ahi":154,"acă":1581,"ahu":103,"aho":221,"aj ":993,"adâ":150,"ârâ":145,"aha":317,"agl":117,"agm":128,"agh":501,"abă":348,"agi":1223,"agr":589,"agu":482,"agn":543,"ago":775,"anu":8793,"anz":399,"any":226,"ano":1227,"ann":872,"ant":7062,"ans":5187,"ane":4573,"ang":1746,"anh":83,"ani":17950,"anj":319,"ank":595,"ap ":209,"ana":3541,"anc":4078,"and":7634,"amu":1451,"amm":250,"amo":500,"amn":539,"amp":1577,"ams":162,"ami":2968,"ame":7488,"amb":1018,"ama":1598,"alz":98,"alv":263,"alu":3357,"alt":3527,"als":199,"alp":187,"alo":2252,"aln":118,"alm":409,"all":1429,"alk":125,"alg":190,"agă":206,"ali":13360,"alc":1323,"ald":503,"ale":15313,"alf":367,"ala":3376,"alb":2405,"an ":14955,"aku":88,"ako":128,"acÅ£":1206,"ârÅŸ":461,"aba":554,"abe":900,"abi":2311,"abl":317,"abo":655,"abr":755,"abs":332,"abu":239,"ae ":1225,"aca":773,"aaa":86,"aal":84,"aar":122,"ad ":1372,"ânt":3632,"ÅŸur":726,"ânu":94,"ânz":207,"ÅŸul":2492,"âns":100,"ac ":1040,"ÅŸtr":244,"âmt":99,"ÅŸti":6783,"ÅŸte":3540,"âmp":537,"âng":896,"âne":1748,"ÅŸta":102,"ând":5547,"âni":6072,"ab ":204,"ânc":472,"âna":293,"afr":149,"aft":132,"aff":130,"afe":369,"afi":1982,"afl":2509,"ai ":11311,"aga":899,"age":1103,"afu":111,"âur":92,"âul":1913,"aen":112,"ael":563,"aes":133,"aer":621,"ah ":371,"âte":342,"afa":1595,"ado":765,"ârs":182,"adr":1167,"ârt":98,"adm":1348,"adj":104,"adi":2476,"ârz":238,"âu ":266,"ade":2813,"ag ":301,"ână":2880,"adt":92,"adu":1730,"aco":1240,"acl":100,"ack":689,"aci":1819,"ach":1150,"ace":7711,"ât ":1298,"acc":1560,"ârb":268,"ada":3740,"ârf":276,"ârg":288,"af ":209,"acv":115,"ârn":129,"act":5256,"acu":1523,"ârl":126,"acr":543,"azo":188,"azi":1570,"arÅ£":1475,"azu":425,"aze":593,"avâ":822,"aza":1518,"azd":83,"azz":139,"avă":214,"asă":1423,"axi":362,"axo":262,"az ":338,"axa":101,"atâ":459,"ată":12359,"âi ":95,"ays":88,"aya":143,"aye":215,"ân ":1771,"âlc":198,"ba ":2953,"âmb":183,"ază":4808,"âln":280,"âin":172,"at ":24564,"amă":335,"arh":1164,"arg":1153,"arf":116,"are":42152,"ard":2497,"arc":2307,"arb":848,"ara":6502,"arp":413,"aro":1047,"arn":629,"arm":1225,"arl":2466,"anç":90,"ark":583,"ari":10760,"aru":1463,"arv":129,"arr":585,"ars":566,"art":13731,"au ":13208,"asa":2050,"ary":262,"arz":166,"asi":2001,"ană":4433,"ash":375,"asc":1871,"ase":2773,"aso":520,"asn":213,"asp":360,"ask":118,"asm":175,"asl":121,"ar ":8918,"apa":2757,"ape":1246,"api":1462,"aph":142,"apl":456,"apo":1977,"app":137,"apr":2890,"aps":139,"apt":1565,"apu":270,"as ":1819,"ală":6884,"ava":1432,"ax ":193,"auz":385,"aux":116,"aut":2879,"avr":217,"avo":325,"anÅ£":3393,"avi":1654,"anÅŸ":96,"ave":2326,"ay ":775,"awa":162,"avy":131,"avu":1095,"ară":4267,"av ":216,"ata":4096,"asu":901,"ast":6214,"ass":610,"asy":113,"atm":162,"alÅ£":185,"atl":201,"atr":2501,"ato":5035,"ate":24962,"atf":138,"atc":92,"ati":8684,"ath":466,"aw ":115,"aua":249,"auc":144,"att":328,"ats":134,"atu":9240,"aul":521,"aum":109,"aun":296,"aur":1060,"aus":649,"aud":358,"aug":1299,"apă":1796,"amÅ£":177,"Wür":170,"ка ":90,"ий ":128,"ич ":137,"jec":116,"jel":228,"jen":194,"fâr":376,"fân":580,"ji ":99,"jat":278,"jap":309,"jar":183,"jan":170,"jaz":89,"je ":321,"joa":246,"joc":694,"joz":167,"jos":164,"jor":690,"jit":146,"jin":221,"jaÅ£":96,"bÅ£i":531,"jo ":103,"jlo":496,"itm":199,"itl":818,"itr":663,"ito":6294,"itu":8009,"itt":280,"its":137,"itz":191,"ity":420,"iub":117,"iuc":160,"iua":221,"iud":230,"ipă":404,"isk":89,"ism":2124,"isl":772,"iso":596,"isn":143,"isp":881,"iss":507,"isr":101,"isu":227,"ist":19214,"iv ":3796,"ita":17021,"itc":116,"ite":7821,"ith":307,"iti":4614,"ivo":234,"ivu":281,"ius":647,"iur":519,"ium":322,"iul":7954,"iun":9474,"iva":1811,"ix ":362,"ivi":3232,"inÅ£":4376,"ive":4336,"ipr":212,"ipo":338,"ipp":144,"ipu":386,"ips":239,"ipt":541,"ipi":1233,"aţă":1525,"ipl":436,"is ":3857,"ion":10900,"iop":116,"ior":1475,"ios":402,"iot":422,"iou":159,"iog":188,"iol":1112,"ipa":2480,"ipe":948,"iov":274,"ir ":799,"iru":345,"irs":100,"irt":172,"iro":730,"irm":483,"eîn":134,"irk":94,"irl":333,"iri":2227,"isi":1374,"ish":315,"ină":2514,"isf":103,"ise":3028,"isc":2092,"isa":444,"iu ":4493,"iqu":132,"ilă":788,"inâ":388,"ire":5707,"imă":577,"irg":179,"ira":895,"irc":947,"it ":8220,"ünc":112,"iză":370,"ja ":337,"ită":7145,"ixt":86,"isă":796,"ixe":89,"iz ":119,"ivă":1340,"izu":147,"izv":208,"izo":952,"izi":2792,"ize":859,"iza":6312,"kil":158,"kin":314,"kir":111,"kis":140,"km ":1110,"ki ":642,"făc":542,"făr":519,"kel":127,"ken":233,"kes":118,"ker":360,"ket":144,"fă ":122,"gân":291,"kfu":88,"ke ":452,"kra":120,"kre":109,"kt ":111,"kov":198,"km²":262,"kol":93,"ks ":224,"făş":535,"cÅ£i":4945,"ko ":243,"jut":351,"jus":113,"jul":508,"jun":382,"jum":207,"jur":1124,"jud":4033,"juc":683,"kar":128,"kan":145,"kai":124,"kad":96,"ka ":524,"ha ":718,"ham":492,"han":1154,"hai":779,"hak":92,"hal":402,"hau":169,"har":1141,"has":93,"hat":192,"hae":287,"hag":86,"hab":90,"he ":3331,"hel":915,"hei":851,"hee":120,"hed":164,"hea":469,"hez":295,"hev":85,"het":520,"hes":378,"her":1057,"heo":670,"hen":622,"hem":280,"că ":14688,"hi ":1098,"dân":197,"căi":96,"căl":499,"căd":116,"căz":100,"căp":117,"căs":168,"căr":2073,"cău":418,"căt":3820,"hie":477,"hid":863,"hic":510,"hib":86,"hia":1153,"hip":1598,"hio":209,"hin":1334,"him":1433,"hil":698,"hii":152,"hiu":429,"hiv":233,"his":738,"hit":1266,"hir":636,"hiz":150,"hn ":414,"hle":150,"ho ":172,"gma":139,"go ":432,"gme":179,"glo":451,"gle":2336,"gli":771,"gn ":157,"gla":327,"gog":138,"goa":126,"gnu":85,"gno":150,"gni":185,"câş":533,"gne":487,"gna":170,"geÅŸ":236,"gs ":160,"gol":308,"gon":427,"gos":413,"gor":814,"got":99,"gov":160,"gu ":263,"gro":318,"gru":1821,"gra":4239,"gri":471,"gre":2106,"gto":159,"gui":168,"gum":179,"gul":1451,"gua":163,"gue":323,"gy ":135,"gră":290,"guv":925,"gur":1708,"gus":1778,"gvi":266,"iam":458,"ial":8383,"ian":7496,"ias":341,"iar":3383,"iau":106,"iat":2458,"ic ":11072,"iab":156,"iac":516,"iad":260,"iag":132,"ibl":375,"ibi":1287,"ibo":95,"ibr":438,"ibu":1167,"iaz":367,"id ":1405,"iba":208,"ibe":1115,"ia ":37192,"iet":2227,"ieu":92,"iev":591,"iew":117,"iez":102,"iel":766,"iem":1503,"ien":2907,"iep":154,"ier":4481,"ies":1526,"ied":283,"ief":128,"iei":9680,"aÅŸ ":2678,"ig ":320,"iec":2094,"ifu":325,"ifo":500,"ifr":208,"iff":103,"ife":1805,"ifi":2918,"dă ":1783,"ifa":105,"icr":452,"ics":163,"ict":2757,"icu":1908,"ico":2006,"ick":421,"icl":554,"ici":10582,"ich":1701,"ice":8554,"ie ":34669,"ica":13932,"idu":1244,"idr":458,"ido":239,"idi":1327,"idg":110,"ide":3884,"ida":1312,"if ":119,"iic":157,"iaÅŸ":207,"iaÅ£":1579,"idă":222,"iin":4917,"iil":3254,"iit":1059,"il ":2791,"ija":114,"iji":265,"ijl":490,"ijo":87,"im ":1225,"ika":140,"aÅ£a":1077,"aÅŸe":616,"ige":485,"aÅŸc":90,"aÅŸa":330,"iga":1279,"ii ":23496,"igm":125,"igh":1464,"igi":3078,"aÅŸi":1717,"icâ":83,"igu":863,"aÅŸu":2407,"aÅŸt":704,"igr":211,"aÅŸo":348,"igo":379,"ign":421,"dăc":87,"iha":624,"ică":10472,"ihi":105,"dău":132,"iho":423,"dăr":156,"ik ":233,"imo":864,"imn":359,"imm":132,"imp":4726,"ieÅŸ":578,"imf":123,"ime":4318,"ieÅ£":356,"imi":3490,"ip ":1099,"inc":6719,"ind":7549,"ina":7598,"imu":2173,"inn":213,"inm":129,"ino":1626,"int":13234,"ins":3523,"inf":1447,"ine":10869,"inh":83,"ing":4928,"ini":7437,"inl":374,"ink":229,"ioa":4790,"ioc":312,"iod":317,"inu":2953,"inv":735,"iny":91,"inz":228,"iko":237,"icÅ£":367,"aÅ£i":16926,"iki":186,"ike":152,"aÅ£e":270,"ila":1467,"ilb":98,"in ":44693,"ilo":9977,"ill":1766,"ilm":2143,"igă":285,"ilh":99,"ili":9285,"ild":180,"ile":12078,"ima":4046,"imb":4580,"io ":1569,"ily":129,"ilt":143,"ilu":581,"ilv":1166,"hiÅŸ":314,"how":106,"hol":376,"hom":292,"hon":245,"hos":162,"hot":319,"hou":159,"hov":191,"hoo":89,"hop":132,"hor":542,"hoe":88,"hoc":86,"hni":581,"hno":368,"hne":86,"heÅ£":86,"hul":158,"hua":109,"htt":211,"hte":132,"hro":140,"hre":117,"hri":377,"ht ":1261,"hra":577,"hiÅ£":84,"hy ":109,"hwa":155,"hum":1416,"hus":146,"hur":185,"fi ":1715,"ffe":164,"ffi":135,"feu":99,"fet":117,"fes":1040,"fer":4398,"fec":1006,"fed":383,"feb":993,"fem":538,"fen":538,"fel":943,"fib":88,"fia":458,"faz":136,"fas":102,"fat":182,"far":402,"fap":490,"fam":1470,"fan":745,"fal":367,"fai":144,"fac":2220,"fab":663,"ff ":157,"fe ":258,"euÅŸ":129,"eză":2753,"fa ":130,"exu":301,"ext":1548,"etă":1447,"exa":920,"ez ":2054,"ews":115,"exp":1323,"epÅ£":305,"esă":354,"exi":1412,"exc":376,"exe":917,"ezv":899,"ezu":584,"evă":275,"eza":830,"ezo":739,"eze":4121,"ezi":2997,"erÅ£":230,"eta":3771,"epâ":360,"ete":2601,"etc":489,"eti":2973,"eth":248,"etn":363,"esp":1890,"eso":846,"est":55654,"esu":952,"ess":1010,"ev ":251,"euc":86,"eud":222,"epă":213,"eum":115,"eul":878,"eun":627,"eto":805,"etr":2728,"ets":89,"ett":511,"etu":1024,"etw":102,"ew ":746,"eve":1810,"eva":1191,"evo":1396,"enÅ£":4104,"evi":2580,"eut":437,"eur":1309,"eus":264,"ex ":508,"ewi":114,"eră":1931,"evr":439,"ey ":1079,"ewa":103,"epe":1003,"epi":751,"eph":288,"er ":7653,"epa":1109,"eot":142,"eos":392,"eor":2351,"eom":197,"eol":739,"eop":176,"eon":471,"elă":165,"eiÅ£":151,"es ":5970,"ept":3637,"epu":3456,"epl":448,"epp":122,"epo":532,"epr":2782,"erk":178,"erl":756,"eri":23643,"erg":1582,"emă":409,"erh":102,"ere":9729,"erf":436,"erc":1961,"erd":628,"era":10049,"erb":1083,"et ":2968,"esk":95,"esl":106,"esf":637,"enă":298,"esh":139,"esi":1831,"esb":88,"esc":5445,"ese":2325,"eu ":1098,"esa":2035,"erz":182,"ery":156,"erv":1894,"eru":1701,"erw":96,"err":761,"ert":2020,"ers":6219,"ern":5307,"erm":6392,"erp":946,"ero":1890,"eki":83,"ecÅ£":1050,"en ":4825,"elb":109,"ela":2761,"eld":314,"elf":84,"ele":22448,"eli":2942,"elg":1037,"egă":470,"elm":166,"eln":87,"ell":1629,"elo":7272,"elu":2902,"elv":133,"els":286,"elt":287,"eo ":496,"emb":7566,"ema":2057,"eme":3406,"emn":2329,"emo":1281,"ehă":102,"emi":3276,"emu":1154,"emp":1201,"ems":94,"ep ":186,"ene":5641,"enh":144,"eng":1760,"enb":358,"ena":2023,"end":1865,"enc":997,"eno":950,"enn":450,"enk":132,"eni":7113,"enu":3506,"ens":2167,"ent":26508,"enr":218,"eoa":176,"enz":462,"eog":396,"eod":283,"eoc":172,"egl":156,"ego":650,"egn":92,"ege":2159,"ecâ":351,"egi":6402,"egh":185,"egr":796,"egu":603,"ehn":787,"eho":111,"ehe":163,"ehi":357,"ecă":398,"ek ":382,"eic":203,"eia":920,"eis":644,"eir":131,"eim":251,"eil":863,"ein":887,"eii":140,"edă":119,"eaÅ£":502,"eie":409,"eid":483,"eig":145,"eaÅŸ":214,"eja":336,"el ":7309,"eit":209,"eiu":167,"eke":85,"em ":1443,"eju":102,"giz":650,"giu":3192,"git":384,"gis":1283,"gir":91,"gil":280,"gim":999,"gaÅ£":227,"gip":292,"gin":3668,"gio":683,"gid":91,"gie":1712,"gic":1515,"gii":481,"gia":2426,"bău":105,"băr":238,"ght":1266,"băt":475,"băn":99,"ghi":1143,"ghe":1448,"gha":167,"cât":1186,"gi ":458,"câi":94,"cân":2264,"câm":225,"gen":4213,"geo":702,"get":450,"ger":3349,"ges":353,"gh ":216,"bă ":1169,"gea":351,"geb":118,"gem":253,"gel":950,"gda":132,"ge ":2096,"gaz":511,"gby":87,"gas":177,"gar":1662,"gau":115,"gat":2233,"gaj":154,"gam":133,"gal":1275,"gan":2759,"ga ":1522,"îşi":423,"fuz":427,"fur":551,"fus":129,"ful":658,"fun":1914,"ftw":294,"ft ":458,"fra":2347,"fre":553,"fri":632,"fiÅ£":85,"fro":290,"fru":551,"for":8285,"fos":13009,"fot":1551,"fon":2006,"fol":2314,"flă":1101,"fiÅŸ":196,"feÅ£":166,"foc":121,"foa":663,"fle":282,"fla":1128,"fli":157,"flu":1381,"flo":329,"fic":5576,"fie":1608,"fig":239,"fii":4052,"fil":2800,"faÅ£":1332,"fin":1931,"fir":582,"fis":97,"fit":276,"fiu":394,"fix":216,"fiz":803,"cuÅ£":360,"da ":5431,"dba":199,"de ":81669,"dac":568,"dad":169,"dal":1004,"dai":112,"daj":204,"dag":131,"dae":325,"dat":5418,"dar":2392,"dap":257,"dan":788,"dam":518,"day":88,"dav":93,"dau":120,"cup":2017,"cun":3394,"cul":7448,"cum":2248,"cui":2182,"cuf":114,"cub":97,"cuc":191,"cua":310,"ctu":3820,"ctr":1989,"cto":4340,"cti":3912,"cte":2267,"cta":977,"coÅ£":184,"cră":288,"cy ":117,"cve":360,"cvi":112,"cva":166,"cus":289,"cur":6209,"cut":5650,"cuv":696,"ctă":260,"Șco":105,"cks":236,"ckh":87,"cla":1563,"cle":639,"clu":1691,"cli":610,"clo":419,"ceÅŸ":351,"cmi":98,"co ":1054,"cni":95,"cod":494,"coe":86,"cof":296,"cog":235,"coa":845,"cob":156,"coc":134,"con":13122,"coo":319,"col":4664,"com":13771,"cor":2665,"cos":422,"cop":2252,"cov":597,"cot":422,"cou":153,"cs ":186,"clă":363,"cqu":87,"ct ":2355,"cre":3245,"cra":1687,"cri":4361,"cru":523,"cro":1167,"cu ":14527,"cci":346,"cca":249,"cce":1997,"cea":5425,"cez":1350,"ch ":1051,"cev":105,"cer":3600,"ces":6831,"cet":999,"ceu":226,"cen":3570,"cep":2469,"cem":1612,"cel":8680,"cei":1228,"ceh":124,"cee":645,"ced":641,"ci ":3845,"Ști":135,"cha":1202,"Ște":246,"chw":220,"chu":202,"cia":8398,"ck ":1749,"cie":2502,"cid":710,"cic":613,"che":3039,"chl":127,"chi":6835,"cho":240,"chn":170,"chs":144,"cht":297,"ciz":320,"civ":411,"caÅ£":977,"cil":1200,"cim":273,"cif":735,"caÅŸ":448,"cii":3633,"cir":703,"cis":679,"cit":1218,"ciu":1361,"cin":2022,"cio":753,"cip":3172,"cm ":96,"cke":198,"ed ":964,"eba":172,"ebe":338,"ebi":493,"ebo":152,"ebr":2253,"ebu":793,"ec ":965,"eac":1155,"eag":683,"eae":329,"ead":219,"eak":166,"ean":4116,"eal":2295,"eam":780,"ear":779,"eas":2985,"eap":381,"eav":360,"eat":1944,"eau":812,"eaz":4296,"eb ":255,"ea ":37128,"efi":943,"efl":126,"efo":468,"efa":467,"efe":1778,"eff":109,"ei ":24037,"ega":2463,"eft":125,"efu":229,"eek":131,"een":318,"eel":266,"eea":861,"eed":149,"eer":336,"eep":102,"eet":278,"edi":6693,"ede":2885,"eda":988,"eg ":239,"edu":500,"edo":913,"edr":363,"ecl":455,"eck":180,"ech":2954,"eci":4375,"ece":4160,"eca":1262,"ee ":711,"ef ":336,"ecv":378,"ecu":2928,"ect":7556,"ecr":556,"eco":3176,"dwi":90,"dwa":128,"dy ":306,"dve":124,"dur":1540,"dut":183,"dus":2699,"duÅŸ":94,"dor":982,"dop":268,"don":1156,"dom":1511,"dol":361,"dox":527,"dow":256,"dov":2135,"dou":2054,"dos":139,"ds ":634,"diÅ£":1218,"diÅŸ":157,"deÅ£":4616,"dmi":1594,"dne":111,"doa":843,"dob":218,"doc":617,"dof":143,"doi":1173,"Èšar":280,"dun":278,"dum":103,"dup":2223,"dui":145,"dul":3772,"dub":195,"dua":160,"duc":3456,"dri":740,"dra":1452,"dt ":159,"dre":2042,"du ":925,"dro":713,"dru":1503,"dge":174,"dic":3233,"did":167,"dia":4181,"der":5158,"des":4508,"det":654,"deu":131,"dev":1104,"dez":1907,"deb":451,"dea":1065,"ded":225,"dec":2801,"def":691,"dee":144,"deg":147,"dej":85,"dei":744,"del":2089,"den":4447,"dem":1462,"dep":2099,"deo":975,"di ":446,"dle":126,"dla":95,"deÅŸ":387,"do ":923,"dja":87,"div":1213,"diu":1854,"diz":147,"dim":595,"din":41384,"dio":1557,"dip":176,"dir":1178,"dis":3885,"dit":1572,"die":1235,"dif":1731,"dig":295,"dii":366,"daÅ£":232,"dil":110,"rgu":556,"rhe":302,"rj ":109,"rha":273,"rcă":369,"rhi":1066,"măr":2035,"măs":677,"măt":499,"măn":620,"rfu":326,"rga":2411,"ri ":15694,"rgi":1524,"rgh":746,"rbă":345,"rge":1425,"rgo":215,"ret":2582,"res":5915,"rev":1477,"reu":1156,"rew":92,"rez":5275,"rey":127,"mă ":3118,"rfa":103,"rfe":202,"rfi":117,"nân":632,"rfo":213,"rdu":767,"rds":382,"rg ":1476,"reb":516,"rea":18674,"ree":777,"ref":1708,"rec":5448,"red":1424,"rei":3494,"rej":104,"reg":5973,"rem":2573,"ren":2941,"rek":143,"rel":3576,"rer":308,"reo":342,"rep":4378,"rf ":235,"rda":852,"rcu":1210,"rct":172,"rdo":623,"rdi":1685,"rde":964,"re ":51071,"rbu":436,"rco":232,"rci":1263,"rch":609,"rce":1470,"rca":1029,"ray":96,"raz":629,"rd ":3327,"rap":865,"rar":2929,"ras":1358,"rat":9747,"rau":625,"rav":765,"rbi":1165,"rbo":575,"rba":839,"rbe":545,"rc ":287,"raj":371,"rai":1767,"rah":296,"rag":920,"ran":9556,"ram":2869,"ral":5890,"rak":183,"rab":1135,"raf":2847,"rae":350,"rad":3282,"rac":2468,"rpu":387,"rpr":755,"rpo":278,"rs ":1565,"rpe":182,"rpa":307,"rpi":136,"ror":724,"ros":1102,"rot":1489,"rom":6745,"ron":3428,"roo":164,"rop":6403,"roz":338,"rou":752,"rov":4132,"row":176,"rox":732,"rob":828,"roa":1307,"rod":3362,"roc":2541,"roi":1094,"rol":1987,"rof":1423,"roe":237,"roh":90,"rog":1247,"rno":327,"rnu":701,"rp ":237,"rna":2969,"rne":1758,"rni":1303,"reÅ£":1190,"rmo":436,"rmu":522,"ro ":2108,"rma":9885,"rme":3062,"reÅŸ":4337,"rmi":1348,"rls":94,"rlo":266,"rgă":207,"rli":413,"rld":353,"rle":420,"rla":2658,"rn ":698,"rks":134,"rke":254,"rm ":799,"riz":1053,"rl ":211,"rip":540,"rio":5018,"rir":448,"rit":7787,"ris":4701,"riv":1528,"riu":2574,"rdă":172,"rig":3530,"raÅŸ":5092,"rij":321,"rii":7673,"ril":8587,"rik":126,"raÅ£":2453,"rin":9619,"rim":5400,"ria":7770,"rib":1255,"ric":11723,"rid":1224,"rie":15408,"rif":356,"rk ":829,"roÅŸ":363,"rsă":312,"rui":1637,"rug":366,"rud":206,"ruc":1749,"rur":434,"rup":3037,"run":754,"rum":1184,"rul":9603,"ruz":121,"rux":123,"rus":1703,"rut":1017,"rva":737,"rvi":837,"rve":659,"rvo":119,"ry ":1178,"rsk":120,"rsi":1656,"rso":2085,"rsc":90,"rsa":860,"rnă":515,"rsh":97,"rse":1189,"rta":3149,"rst":426,"rsu":1104,"rto":1146,"rte":6836,"rth":550,"rti":6445,"rua":1449,"rts":182,"rtr":166,"rtu":1485,"rtt":156,"riÅ£":1112,"riÅŸ":356,"rmâ":178,"rt ":2203,"rro":293,"rmă":1457,"rri":340,"rre":470,"rra":339,"ru ":12849,"rry":345,"sc ":2232,"sab":243,"sac":340,"sad":91,"sag":137,"sai":83,"saj":124,"sal":1207,"sam":459,"sbe":121,"san":693,"sau":8354,"sat":4809,"sas":155,"sar":1304,"sa ":3104,"ón ":120,"ruÅŸ":110,"rze":109,"rtă":914,"rzi":434,"sha":218,"sho":144,"năr":653,"năs":2089,"năt":775,"she":136,"scă":1232,"shi":636,"năl":262,"si ":1523,"sfâ":443,"siv":865,"sie":984,"sid":975,"sic":666,"sib":419,"sia":1164,"sk ":250,"nău":325,"sit":6826,"siu":1567,"sir":294,"sis":2018,"sip":93,"sin":2491,"sio":987,"sil":2030,"sim":1237,"sih":314,"sii":129,"sif":302,"sig":587,"scr":3431,"scu":7764,"sbu":216,"se ":12289,"sca":1105,"sce":729,"sci":701,"sch":1873,"sco":2491,"sex":343,"sey":122,"ser":5132,"ses":902,"set":368,"seu":253,"sfa":100,"sez":381,"sh ":403,"nă ":12507,"sfi":168,"sfe":495,"sfo":294,"sea":1239,"sei":529,"see":104,"sed":775,"sec":2976,"seb":398,"sep":1669,"seo":135,"sen":1576,"sem":2294,"sel":1265,"spu":788,"spo":1513,"spr":1609,"spe":3508,"spi":782,"spa":1327,"sou":172,"sov":534,"sol":1405,"som":99,"son":2390,"sop":133,"sor":985,"sos":101,"sod":147,"sof":666,"soa":1403,"soc":1613,"su ":139,"sra":283,"st ":20718,"ss ":769,"sli":132,"slo":207,"slu":183,"sla":1454,"sle":117,"ski":424,"sfă":558,"sko":122,"sm ":736,"ska":303,"sna":139,"sni":276,"sne":177,"smo":155,"seÅ£":118,"smu":996,"so ":251,"sma":281,"seÅŸ":442,"smi":416,"sme":397,"soÅ£":294,"stâ":421,"stă":4284,"sse":604,"ssa":443,"sso":336,"ssi":591,"ssu":98,"ste":53928,"spâ":306,"stf":531,"sta":11097,"stm":85,"sto":3769,"sti":8907,"stl":380,"stu":4079,"str":12855,"sty":123,"sud":1668,"sue":207,"sub":3715,"suc":517,"spă":179,"suf":275,"sul":3772,"sum":370,"sup":2843,"sun":3645,"sut":208,"sus":705,"sur":1681,"suv":86,"sy ":155,"tai":479,"taj":164,"tal":7011,"taf":116,"tag":288,"tab":988,"tac":696,"tad":477,"tc ":460,"tba":1494,"tax":122,"tav":190,"tau":320,"tat":21973,"tas":470,"tar":6168,"tap":132,"tan":6974,"tam":968,"tch":206,"te ":79600,"suÅŸ":110,"ta ":8238,"oză":144,"pa ":2141,"oxă":260,"pe ":11904,"par":10822,"pat":2004,"pas":403,"pay":83,"pac":752,"pad":108,"pab":102,"pag":502,"pal":1642,"pai":83,"pap":190,"pam":183,"pan":5506,"phe":183,"pha":169,"pho":127,"phi":253,"pi ":238,"ph ":186,"lân":506,"pea":2043,"pec":3297,"ped":445,"pen":9052,"per":11036,"pet":1065,"pes":1059,"pei":1300,"pel":911,"pla":2428,"pli":1624,"ple":1254,"plo":453,"plu":926,"phy":102,"pia":1002,"pid":288,"pic":1543,"pie":1703,"pii":550,"paÅ£":844,"pil":610,"pin":728,"pio":918,"pir":800,"pis":848,"pit":1452,"piu":839,"poz":1416,"por":4692,"pop":3746,"pov":414,"pot":1327,"pos":1095,"poi":429,"pog":111,"pom":116,"pon":1731,"pol":4578,"poa":1462,"poe":647,"poc":495,"pod":314,"ps ":147,"ppe":285,"peÅŸ":178,"po ":105,"pta":1006,"pse":287,"psi":410,"pso":85,"ptu":1231,"pub":3609,"pte":2615,"pti":445,"pto":319,"pra":2308,"pt ":1178,"plă":316,"pru":251,"psa":94,"pri":13510,"pre":11982,"pro":14449,"pră":140,"ptă":561,"pur":1199,"pus":1351,"put":2590,"pun":2005,"pul":6398,"px ":1151,"puÅŸ":180,"puÅ£":344,"mân":11918,"lă ":9449,"iÅŸ ":380,"lăc":209,"lăd":384,"lăr":280,"lău":157,"lăt":576,"iÅŸa":158,"iÅŸc":590,"iÅŸe":100,"iÅŸi":457,"iÅŸo":462,"iÅŸn":143,"iÅŸu":248,"iÅŸt":1090,"iÅ£e":420,"iÅ£i":4507,"iÅ£a":1141,"lăţ":94,"iţă":635,"qua":125,"que":368,"qui":218,"ra ":10628,"rb ":100,"ngo":269,"ngi":1205,"ngl":3065,"ngv":280,"ngu":1282,"ngr":328,"ngt":167,"ngs":258,"ni ":7085,"nge":1701,"ncâ":95,"ngh":727,"nga":1437,"nha":227,"ncă":762,"neg":502,"nei":3106,"nel":2471,"nen":1433,"nem":658,"nep":189,"neo":674,"ner":4256,"net":2067,"nes":1905,"nev":324,"neu":474,"ng ":2660,"nea":6161,"nec":713,"ned":727,"nee":362,"nef":188,"nfi":1313,"nfo":1463,"nfl":766,"nfr":253,"nfu":123,"ney":389,"nez":1743,"nex":158,"nfa":94,"nfe":701,"ncr":242,"nct":978,"nco":1033,"nci":6536,"ncl":974,"nce":5421,"nch":1298,"nca":697,"ne ":15507,"nbu":225,"ndu":3412,"ndr":2268,"nds":282,"ndo":1104,"ndi":3514,"nde":6309,"ndb":189,"nda":4140,"ncy":92,"ncu":853,"nal":8677,"nam":764,"nan":953,"nap":143,"nar":3126,"nac":272,"nad":606,"nag":295,"nah":123,"nai":648,"naj":750,"nc ":203,"nbe":172,"nd ":10185,"nav":597,"nau":373,"nat":5278,"nas":876,"naz":192,"na ":10566,"muÅŸ":108,"moÅ£":117,"nyi":101,"ntă":3762,"nz ":336,"ntâ":712,"nsă":705,"noÅ£":101,"ny ":515,"noÅŸ":120,"nvi":271,"nux":134,"nve":1249,"nva":170,"nul":11142,"num":9716,"nun":659,"nui":2359,"nus":463,"nut":1282,"nuu":86,"nuv":87,"nur":524,"nua":2364,"nue":133,"nuc":341,"nty":205,"nto":1327,"ntu":5345,"nts":104,"ntr":20429,"nti":5120,"nth":232,"nta":5926,"nte":13468,"nsu":2110,"nsn":130,"nsm":304,"nsp":737,"nso":628,"nst":5554,"nsf":385,"nse":1387,"nsh":116,"nsi":3286,"nsl":126,"nsk":220,"nsc":408,"nsa":2214,"nu ":2393,"nru":101,"nri":188,"nre":465,"nt ":10809,"niÅ£":1334,"niÅŸ":600,"ns ":1634,"noc":153,"nod":134,"noa":802,"nob":170,"nog":217,"nol":681,"noi":1449,"nop":228,"nom":2958,"non":580,"not":496,"nos":3264,"nor":3599,"nov":557,"nou":744,"noz":170,"nr ":214,"nne":796,"nna":357,"nno":163,"nni":320,"nny":106,"nme":156,"nma":86,"neÅ£":165,"neÅŸ":1107,"nli":309,"ngă":685,"nn ":394,"nla":407,"no ":1211,"nlo":244,"nkf":88,"nke":129,"ncÅ£":1362,"nki":120,"nka":125,"nkt":91,"înÅ£":344,"nje":175,"nja":194,"nju":195,"nii":3788,"ndă":489,"naÅŸ":348,"nig":141,"nif":691,"nie":6578,"nid":183,"nic":6607,"nib":123,"nia":13403,"nk ":409,"niz":1868,"nix":112,"niu":1843,"niv":2033,"nis":4310,"nit":7595,"nir":583,"nio":766,"nim":2048,"nin":1227,"naÅ£":2629,"nik":97,"nil":2118,"ogr":2357,"ogu":334,"ogi":3138,"ogl":113,"ogo":258,"ogn":118,"oga":345,"ogd":87,"oge":523,"oi ":1939,"ohi":99,"oho":87,"ohn":440,"oha":219,"ocă":128,"ohe":86,"ois":381,"oir":138,"oiu":600,"oit":118,"oin":312,"oaÅ£":87,"oil":1036,"oaÅŸ":230,"odă":270,"înă":265,"oii":90,"oic":184,"oid":249,"oie":2586,"ok ":171,"oia":306,"obÅ£":455,"ol ":1950,"oiz":87,"oce":1371,"och":606,"oci":2584,"ock":1201,"ocl":282,"ocm":112,"oco":593,"împ":1524,"ocr":595,"obs":247,"obu":209,"oe ":171,"oca":5500,"occ":161,"îmb":234,"ode":1683,"odi":1388,"înv":645,"odo":1005,"înt":5727,"înr":506,"îns":1196,"odr":168,"ocu":4799,"înl":252,"oct":1467,"îng":438,"înf":1221,"of ":1300,"înd":673,"înc":2936,"îna":811,"oda":803,"oel":100,"oem":170,"oes":95,"oet":442,"oen":174,"ody":97,"odu":3682,"og ":550,"ofi":1691,"ofu":93,"oft":569,"ofo":587,"oez":194,"off":117,"ofe":1241,"ofa":91,"nzâ":121,"ob ":212,"îl ":203,"nză":237,"oc ":1888,"oap":522,"oan":2626,"oam":565,"oal":517,"oai":251,"oad":3000,"oac":410,"în ":51095,"oba":769,"od ":1364,"oar":8811,"oas":2028,"oat":2201,"obo":275,"obr":179,"obl":493,"obi":2216,"obe":608,"nza":270,"nze":236,"nuă":93,"nzi":410,"nzo":124,"nvă":302,"îi ":276,"otă":208,"oya":123,"oxi":1058,"oxe":145,"oz ":239,"ows":299,"own":215,"oră":461,"owi":83,"ovă":83,"orÅ£":934,"ozo":459,"oze":129,"ouă":1877,"ozi":1701,"oza":446,"otu":494,"oua":772,"ow ":278,"olÅ£":86,"oti":981,"oth":247,"ote":1814,"ott":429,"ots":89,"otr":617,"oto":1723,"ost":14678,"osu":176,"ota":1624,"otb":1480,"ov ":924,"osi":2418,"osh":105,"onă":692,"ose":1383,"osf":245,"osp":184,"oss":270,"osm":217,"osl":251,"oso":685,"osn":103,"oy ":170,"owa":92,"owe":145,"ovi":5242,"onÅ£":708,"ovo":380,"ovs":162,"ouv":83,"ox ":338,"ova":2953,"ove":2093,"orâ":88,"oug":124,"oui":176,"oul":710,"oun":598,"oup":188,"ous":347,"our":791,"out":476,"opo":1954,"opp":88,"opi":1812,"opl":160,"ope":4845,"oph":214,"opa":1115,"os ":2802,"opu":4445,"opr":1065,"opt":660,"ops":95,"oon":154,"ool":264,"ook":218,"ood":367,"or ":26373,"oot":145,"oor":339,"oop":114,"ork":590,"orl":449,"orm":7578,"orn":1037,"oro":968,"orp":872,"orr":316,"orc":286,"ord":5185,"ore":2827,"orf":352,"org":3181,"omă":1147,"ori":16103,"orj":132,"ou ":829,"osa":293,"osc":3302,"ort":5002,"ors":359,"orv":285,"oru":3894,"ory":244,"olă":311,"omâ":10245,"m² ":271,"ot ":1395,"orb":953,"ora":7147,"ola":2095,"old":2473,"olc":184,"on ":8021,"olj":96,"oli":6258,"oll":628,"olk":148,"olf":429,"ole":2350,"olg":83,"ols":165,"olt":1104,"olm":146,"oln":127,"olo":7169,"oly":117,"olu":4417,"olv":270,"om ":2396,"oki":92,"oke":139,"ona":10266,"ond":4478,"onc":1725,"onf":1659,"one":3635,"ong":702,"onj":211,"oni":6032,"onl":252,"onn":355,"ono":3183,"ons":6444,"ont":4103,"onu":2421,"onv":453,"ony":208,"onz":212,"oma":4275,"oo ":114,"ome":3698,"omb":2289,"omi":3631,"omm":249,"omp":6241,"omn":706,"omo":1504,"omu":6796,"op ":1017,"la ":23354,"lb ":181,"le ":40374,"lce":389,"lca":429,"lcl":134,"lch":83,"lci":164,"lcu":522,"lco":212,"lf ":227,"lde":317,"lda":182,"ldo":2003,"ldi":182,"ldu":132,"lab":636,"lac":1028,"lad":507,"lae":470,"lah":161,"lag":391,"laj":309,"lai":336,"lal":366,"lan":7785,"lam":2475,"lap":209,"lar":3822,"lat":4828,"las":2223,"lax":136,"lau":569,"lav":737,"lay":251,"lba":949,"ld ":1067,"lbe":538,"lbi":174,"lbo":115,"lbu":1909,"ky ":197,"kso":178,"lpi":329,"lph":128,"ls ":385,"lpt":166,"lon":2782,"lom":608,"lop":364,"loo":88,"lor":17380,"lod":381,"loc":8139,"log":3938,"loi":220,"los":2548,"lot":440,"lou":173,"lov":661,"low":107,"loz":395,"lni":416,"lne":136,"loa":1027,"lob":416,"lmo":117,"lmi":101,"leÅ£":90,"lme":740,"leÅŸ":291,"lma":385,"lna":86,"lmu":506,"lth":83,"lti":1438,"lto":515,"ltr":114,"ltu":1401,"lud":310,"luc":1053,"lub":644,"lua":671,"lug":175,"lue":1286,"lsi":131,"lso":99,"lst":136,"lta":1833,"lte":3356,"lu ":1358,"lse":84,"lsa":93,"liÅ£":378,"ía ":107,"liÅŸ":264,"lt ":1778,"găt":711,"gău":109,"găr":308,"găs":508,"lhe":118,"lcă":318,"lj ":83,"lgo":101,"lge":272,"lgi":975,"lbă":158,"li ":1453,"lga":583,"lfu":196,"lfo":121,"gă ":1042,"lfa":415,"lez":1735,"ley":333,"lex":1098,"leu":326,"lev":1072,"les":2557,"let":1046,"ler":1357,"leo":330,"lep":98,"lem":2259,"len":1650,"lek":93,"lel":1961,"lei":1502,"leg":2868,"lef":302,"lee":148,"led":336,"lec":2987,"leb":331,"lea":5194,"lls":187,"llu":109,"lly":238,"lo ":746,"lla":922,"lle":1509,"lli":1027,"llo":392,"ln ":118,"lm ":1367,"ll ":1537,"lit":10843,"lis":3477,"lir":181,"lip":665,"lio":847,"lin":3950,"lim":4821,"liz":3596,"liv":266,"liu":800,"lic":6840,"lid":584,"lia":5811,"lib":815,"lk ":118,"lik":113,"laÅ£":3598,"lil":379,"lii":609,"lig":1016,"laÅŸ":630,"lie":5757,"lif":704,"ma ":3957,"mb ":1486,"mac":332,"mai":9505,"maj":528,"mad":521,"mae":84,"mag":1419,"mar":6545,"mas":1035,"mal":1356,"mam":301,"man":9186,"max":174,"mat":6955,"mba":2864,"mbl":374,"mbi":1641,"mbe":588,"mbr":9100,"mbo":699,"me ":5049,"mbu":511,"med":2490,"meg":129,"mea":1453,"mec":632,"eÅŸ ":976,"met":2646,"mes":1043,"mer":5757,"mem":2715,"mel":4206,"men":12391,"mei":953,"hă ":127,"mez":105,"mex":145,"mfo":85,"luz":516,"lva":1356,"lve":558,"lvi":203,"lul":6901,"luj":787,"lui":26427,"lup":481,"lun":1806,"lum":1944,"lut":376,"lus":1014,"lur":909,"lux":157,"luv":232,"ly ":423,"ltă":720,"luÅ£":840,"mpi":1771,"mpe":1922,"mpr":847,"mpo":2882,"mpl":2315,"mpu":2400,"mps":120,"mpt":100,"ms ":246,"mog":102,"moc":469,"mob":459,"mod":2074,"mon":3526,"mom":281,"mol":693,"mov":314,"mor":1244,"mos":450,"mot":668,"mou":110,"mpa":3076,"mnâ":163,"miÅŸ":960,"miÅ£":248,"mto":98,"mst":145,"mnă":376,"moÅŸ":151,"my ":245,"mur":1183,"mus":438,"mut":273,"mpă":755,"mul":9511,"mun":7969,"muz":1842,"eÅŸt":6641,"eÅŸu":215,"eÅŸa":195,"mi ":621,"eÅŸi":561,"mbă":561,"eÅŸe":1204,"min":5735,"mio":138,"mil":3851,"mir":1594,"mis":1523,"mit":6241,"miu":551,"miz":90,"mix":115,"eÅ£ ":436,"mic":3196,"mia":589,"mig":156,"maÅŸ":321,"mif":173,"mie":1057,"mid":167,"maÅ£":1711,"mij":400,"mii":918,"mo ":311,"mn ":1067,"eÅ£u":4047,"eÅ£i":1636,"eÅ£e":1089,"eÅ£a":97,"mm ":112,"moa":417,"mnu":348,"mni":697,"mna":983,"mne":568,"mmy":134,"mp ":917,"mmo":100,"mma":189,"meÅŸ":407,"mme":257,"ăţă":263,"vă ":1622,"văr":375,"văz":128,"sÅ£i":293,"văţ":340,"ziÅ£":770,"zvo":1221,"zua":152,"zur":207,"zul":782,"zut":213,"zz ":105,"rÅŸi":489,"zi ":1628,"vâr":386,"vân":1791,"zec":677,"zei":499,"zea":690,"zdu":96,"uă ":1773,"zeu":674,"zes":98,"zen":2310,"zel":386,"zer":601,"ze ":1470,"zbo":1342,"zbu":165,"zf ":120,"zac":138,"zah":127,"zam":95,"zan":432,"zar":1712,"zau":120,"zat":4549,"zoa":213,"zot":117,"zor":484,"zom":97,"zon":1721,"zol":318,"zof":297,"rţă":100,"zo ":122,"rÅ£e":443,"rÅ£a":169,"rÅ£i":2629,"rÅ£u":83,"rÅ£ ":87,"uăz":94,"zib":96,"zia":1295,"zie":586,"zid":234,"zic":2773,"zii":433,"zin":2192,"zim":92,"zil":806,"zaÅ£":846,"zio":247,"zir":91,"zis":507,"zit":1014,"ziu":1413,"yst":189,"ysi":92,"ys ":152,"tăţ":3951,"yon":90,"za ":1974,"ywo":86,"ye ":94,"tă ":27261,"yer":209,"ya ":253,"yan":149,"yn ":128,"yle":187,"yo ":86,"yne":94,"tăl":523,"tăn":89,"tăm":184,"tăz":367,"tăt":383,"tăr":1670,"tăp":108,"yin":96,"tîn":184,"xt ":219,"xtr":935,"xtu":126,"xte":487,"xti":276,"xpr":511,"xpo":137,"xpl":405,"xpe":325,"xon":265,"ăţi":4022,"ăţe":252,"ăşe":185,"ăşi":118,"ăşo":168,"ăşu":379,"ăşt":194,"xul":217,"xua":180,"săm":195,"săn":94,"săi":149,"tât":368,"târ":286,"tân":1123,"să ":6878,"tâi":116,"tâl":280,"tâm":92,"xem":547,"xer":167,"xec":276,"xel":184,"pÅ£i":497,"xis":976,"xil":150,"xim":851,"xid":122,"xic":439,"xig":85,"săr":830,"săs":204,"săp":211,"săt":291,"său":989,"xcl":154,"xce":187,"xe ":275,"xat":91,"xan":620,"xac":149,"ww ":168,"www":168,"oÅ£i":926,"oÅ£e":90,"wn ":181,"ws ":286,"wre":154,"wor":208,"woo":193,"răş":270,"răţ":90,"sân":187,"ră ":10046,"wer":171,"wel":112,"wei":114,"web":177,"oÅŸ ":108,"răb":152,"răd":292,"răc":168,"răg":121,"răj":88,"răi":753,"răn":163,"răm":369,"oÅŸt":362,"oÅŸu":230,"oÅŸa":206,"oÅŸi":435,"sâr":193,"wis":119,"wig":94,"wic":83,"win":151,"rău":112,"răs":796,"răt":330,"răr":765,"răz":877,"wa ":123,"wan":150,"wal":599,"way":113,"war":714,"viÅ£":222,"viÅŸ":91,"vro":102,"vri":167,"vre":765,"vra":124,"vsk":216,"vut":1014,"vul":660,"vy ":150,"via":1245,"nÅ£ ":202,"vio":597,"vir":425,"vaÅ£":256,"vil":1004,"vin":4712,"vig":218,"vii":288,"vic":1489,"vid":1022,"vie":1344,"viz":1818,"viu":343,"vit":1797,"vis":1422,"nÅ£a":4036,"nÅ£e":2750,"nÅ£i":4753,"nÅ£u":270,"vo ":269,"veÅ£":303,"veÅŸ":233,"voa":145,"voc":768,"vod":247,"voi":344,"vol":2372,"von":307,"vor":1265,"vot":169,"vos":108,"nţă":2104,"vi ":279,"rât":86,"râu":1467,"râr":88,"vez":158,"ver":5755,"ves":2284,"vet":135,"râm":188,"rân":915,"vei":383,"veh":194,"veg":396,"ven":3368,"vel":1246,"vea":1522,"ved":733,"vec":1443,"ve ":2034,"val":1848,"van":2141,"var":1789,"vat":1229,"vas":355,"vaz":94,"vac":381,"vad":175,"vai":89,"uză":207,"uzâ":97,"va ":4378,"usÅ£":288,"uzu":128,"uzi":2633,"uvâ":532,"uze":512,"uza":616,"ută":2040,"uxe":267,"usă":1693,"uz ":167,"ură":2750,"upă":3161,"ux ":398,"uvi":580,"unÅ£":1150,"uve":1576,"urâ":83,"ună":2252,"ush":108,"usi":1228,"use":1795,"usc":466,"usa":280,"uu ":89,"usu":380,"ust":4305,"uss":282,"uso":94,"ulÅ£":391,"uth":338,"uti":1787,"ute":2406,"uta":1302,"utt":99,"uts":136,"utu":1616,"uto":3287,"utr":237,"us ":5190,"ulă":561,"ut ":7246,"urb":404,"ura":6173,"urd":294,"urc":802,"ure":4226,"umă":1226,"urg":1586,"uri":11434,"urk":100,"urm":1472,"urn":1161,"uro":4600,"urp":138,"urr":101,"urs":2050,"urt":1427,"uru":770,"ury":127,"urz":103,"unz":236,"upa":1044,"ur ":1460,"păş":90,"upi":117,"upe":1889,"upo":171,"upr":3078,"upl":134,"upt":549,"upu":1021,"ump":261,"umu":1472,"umi":4694,"umo":274,"umn":200,"uma":1213,"umb":1802,"ume":7127,"unt":4936,"uns":365,"unu":4147,"unk":173,"uni":10730,"uno":4024,"unn":89,"unc":3006,"und":2612,"una":3959,"ung":1996,"une":10686,"up ":833,"ucÅ£":871,"um ":3644,"ulu":21791,"ult":5763,"uls":125,"ulp":273,"ulo":572,"ulm":181,"ull":195,"uli":2417,"ugă":238,"ulg":558,"ule":2260,"ulc":332,"ula":6764,"ulb":106,"un ":26513,"uid":145,"uie":791,"uaÅ£":284,"uil":227,"uin":534,"uir":346,"uis":305,"păt":304,"păs":368,"uk ":108,"mÅ£ ":178,"uia":763,"uit":3995,"ul ":62901,"ucâ":92,"ugh":328,"ugi":103,"ugb":125,"uge":384,"ugo":240,"ufu":127,"ui ":29161,"uga":410,"păi":109,"ucă":1628,"păd":164,"păr":2478,"păm":252,"ugu":1856,"uha":84,"uj ":675,"uco":175,"ucr":1114,"uct":1068,"ucu":1790,"uda":563,"ude":4922,"udi":2446,"ubo":209,"ubm":96,"ubs":626,"ubt":149,"ubu":425,"uca":1046,"ue ":469,"uce":1968,"ucc":519,"uci":711,"uch":237,"ucl":385,"uck":119,"uer":254,"ues":246,"pă ":4354,"uff":102,"ufe":113,"ufi":109,"ufl":99,"udu":394,"udo":402,"ug ":96,"ued":469,"uea":236,"uen":1166,"uel":339,"ub ":2463,"tuÅ£":713,"ua ":1456,"uat":4356,"uar":3767,"ual":1937,"uan":471,"ubi":491,"ubl":4031,"ube":197,"uba":209,"ubc":93,"ubd":102,"ud ":2013,"uad":93,"uc ":293,"tze":90,"tyl":114,"tuÅŸ":101,"ty ":780,"twa":283,"trâ":424,"tur":9303,"tus":270,"tut":920,"tui":1608,"tul":18833,"tun":827,"tum":175,"tua":5218,"tud":1979,"tue":106,"tug":400,"tz ":249,"two":138,"toÅ£":88,"tră":1759,"toÅŸ":149,"ts ":552,"tiÅ£":743,"tiÅŸ":169,"tre":13329,"tt ":332,"tra":10048,"tri":7435,"tru":15106,"tro":4167,"tu ":332,"tsc":140,"tsu":91,"tst":102,"tta":206,"tte":678,"tti":236,"ttl":111,"tto":150,"ttp":214,"tts":95,"teÅŸ":321,"tme":92,"tma":135,"to ":1187,"tmo":189,"tni":304,"tne":121,"tp ":211,"tna":149,"tno":89,"tof":98,"tod":1039,"toc":696,"toi":238,"tog":562,"tob":142,"toa":4019,"tou":153,"tov":167,"tos":511,"tot":1088,"tox":87,"tom":2691,"ton":3163,"tol":1579,"tor":17808,"top":634,"tr ":2261,"tii":1559,"til":2704,"taÅ£":1027,"tif":641,"tie":3100,"taÅŸ":87,"tig":849,"tir":1365,"tit":3982,"tis":1483,"tin":5976,"tim":3231,"tip":1320,"tio":2187,"thu":1487,"tia":853,"tib":219,"tic":12965,"tid":1357,"tiz":272,"tiu":313,"tiv":7387,"lÅ£i":875,"tli":252,"tlu":762,"tla":333,"tle":501,"tem":5382,"ten":2986,"teo":833,"tep":232,"tei":2254,"tej":251,"tel":8378,"tee":83,"tef":321,"teg":901,"teh":698,"tea":10690,"teb":131,"tec":1715,"ted":479,"pân":2359,"tfo":164,"tfe":489,"tfa":104,"th ":1080,"tez":721,"tex":467,"tev":339,"teu":83,"tet":387,"tes":961,"ter":17000,"ti ":4750,"pâr":90,"tho":249,"thr":172,"the":1255,"thi":201,"tha":314,"ăpi":117,"ăr ":639,"ărg":110,"ări":6448,"ărc":171,"ăre":1669,"ărb":346,"ăra":1104,"ăpu":119,"ăpt":200,"ălă":333,"ămâ":1238,"ăsp":468,"ăsu":674,"ăst":933,"ăta":498,"ăte":138,"ăpâ":120,"ăti":182,"ălÅ£":396,"ăto":3911,"ăro":211,"ăru":1791,"ărt":197,"ărs":101,"ău ":1645,"ăsc":1578,"ăse":603,"ănă":998,"ăsi":241,"ăut":251,"ărâ":207,"ătu":1360,"ătr":2034,"ăud":174,"ăul":112,"ăsă":746,"ără":1000,"ăzu":203,"ărÅ£":804,"ăzi":480,"ăze":118,"ăzd":88,"ăzb":1237,"ătă":689,"ăuÅ£":165,"ăce":168,"ăca":435,"ăci":357,"ăcu":469,"ăde":218,"ădi":504,"ădu":392,"ăi ":268,"ăia":213,"ăie":323,"ăcă":141,"ăl ":131,"ăin":267,"ăit":151,"ădă":164,"ăil":231,"ăgă":140,"ăle":134,"ălc":90,"ăld":111,"ăla":258,"ălb":84,"ăma":126,"ămi":144,"ăli":530,"ălu":247,"ăne":410,"ănc":84,"ăni":157,"ză ":7802,"zăr":410,"zăt":197,"zău":117,"zân":327,"ürt":162,"uÅŸ ":157,"xă ":377,"uÅ£i":2814,"uÅŸu":97,"uÅŸt":92,"uÅŸo":225,"uÅŸe":140,"uÅŸc":117,"uÅŸi":662},"n_words":[8213281,9706468,7977386],"name":"ro","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/ru.json b/contrib/languages-data/ru.json
new file mode 100644
index 0000000..944e03f
--- /dev/null
+++ b/contrib/languages-data/ru.json
@@ -0,0 +1 @@
+{"freq":{"D":12949,"E":10306,"F":8444,"G":9635,"A":18320,"B":11812,"C":18729,"L":10133,"M":18025,"N":9350,"O":8480,"H":7852,"I":36250,"K":4555,"U":4649,"T":15138,"W":7160,"V":11078,"P":15071,"S":24620,"R":10933,"X":14985,"f":12270,"g":18709,"d":30289,"e":107154,"b":13058,"c":32324,"a":88793,"n":72221,"o":77565,"l":46717,"m":27155,"k":11440,"h":29689,"i":78874,"w":9055,"v":9950,"u":33414,"t":65000,"s":53078,"r":72396,"p":19491,"z":3846,"y":15658,"x":6168,"Ì":88446,"ÑŒ":418052,"Ñ":80439,"ÑŽ":190059," o":3966,"Ñ":756306,"ш":164289,"щ":122233,"ÑŠ":10879,"Ñ‹":569718,"Ñ„":163421,"Ñ…":364215,"ц":246632,"ч":362951,"Ñ€":2010938," a":3919,"Ñ":1933070,"Ñ‚":1865754,"у":741481," t":5450,"Ñ‘":73990,"И":44043,"Л":43881,"К":96587,"Ð":57501,"Ðœ":84561,"П":106733,"О":53561,"Б":68591,"Ð":88041,"Г":58489,"Ð’":87514,"Е":15516,"Д":50036,"З":22723,"Ж":9219,"Ш":21342," H":6358," I":14743,"Ю":9600," N":6330,"Я":13672," O":4736," L":7278,"Э":22580," M":13492,"Т":45273," B":8977,"У":23856," C":13678,"Р":75819," A":12906,"С":148731,"Ц":10320," F":6714,"Ч":15862," G":7609,"Ф":36383," D":9169,"Ð¥":20421," E":6845,"л":1373509,"к":1271742,"й":669125,"и":2990916," X":9849,"п":840034,"о":3520831,"н":2386826,"м":997958,"г":667312," S":16699,"в":1520663," R":8106,"б":419069,"а":2917550," P":10356," W":6218,"з":510343,"ж":242497," V":5002,"е":2737376,"д":981077," T":11223," Ð":74197," Б":66425," Ð’":84271," Г":55440," Д":46951," Е":13561," Ж":8620," З":21445," И":40121," К":90300," Л":41939," Ðœ":81442," Ð":51890," О":47190," П":101697,"Co":3768,"I ":12785," б":96560," а":114180," г":188817," в":459004," е":32581," д":177939," з":81625," ж":31277," й":4814," и":369580," л":56771," к":233167," н":240854," м":177553," п":455990," о":265653," Р":63171," С":125419," Т":41894," У":21149," Ф":32286," Ð¥":19430," Ц":9599," Ч":15364," Ш":14654," Э":20859," Ю":9415," Я":13422,"C ":3880," Ñ‚":131031," у":81588," Ñ€":185096," Ñ":401251," ц":26307," ч":72261," Ñ„":71734," Ñ…":30938," ш":31141," ÑŽ":9238," Ñ":35568," Ñ":51707,"Ma":3809,"II":10412,"VI":3879,"Th":4668,"X ":5588,"S ":3829,"a ":12175,"i ":4290,"he":9109,"g ":4678,"ea":4162,"ec":3745,"de":5482,"h ":4087,"el":5609,"en":10589,"et":4586,"es":8452,"er":17575,"e ":29556,"f ":4058,"ch":4543,"ce":3891,"c ":3917,"d ":10245,"at":8929,"as":4202,"ar":11277,"al":8675,"am":4333,"an":13622,"ac":3987,"nt":7777,"ns":3721,"of":4484,"om":4497,"on":14150,"ol":4531,"os":3740,"ou":4174,"or":10679,"r ":11184,"ll":5038,"o ":6694,"ma":4022,"me":5234,"na":4836,"nd":7645,"ne":5797,"ng":6066,"ni":4386,"m ":5088,"li":5421,"le":7129,"la":5868,"n ":18576,"ic":8588,"ia":4454,"ie":3873,"k ":4036,"is":6529,"it":5343,"il":4505,"in":14086,"io":6997,"l ":9526,"y ":8801,"ve":4522,"x ":4062,"ur":4265,"us":4229,"to":4590,"te":9298,"ti":9123,"th":7132,"ta":6068,"st":7180,"ro":7763,"ri":8232,"re":8155,"ra":7850,"t ":13376,"s ":22614,"Ìв":5893,"Ìд":3913,"Ìй":4535,"Ìм":3791,"Ìл":9850,"Ìн":15831,"ÌÑ€":9052,"ÌÑ":4186,"ÌÑ‚":6032,"К ":4112,"О ":4223,"Ð ":10572,"Ð’ ":18440,"ÑŒÑ":15305,"ью":22763,"Ñл":12007,"Ñк":10864,"ÑÑ€":3980,"Ñн":8177,"ÑÑ‚":23791,"юб":4435,"юг":4684,"юд":5402,"юз":4895,"юл":6476,"юн":7127,"ÑŽÑ€":4992,"ÑŽÑ":3863,"ÑŽÑ‚":24842,"юч":8444,"ющ":38941,"Ñж":4621,"Ñе":26767,"Ñд":10334,"Ñв":18798,"Ñб":17686,"Ñн":29130,"Ñм":15306,"Ñл":9628,"Ñз":22046,"ÑÑ…":9639,"ÑÑ":6449,"ÑÑ‚":41978,"ÑÑ€":7070,"ÑÑŽ":12156,"Ñщ":14702,"щи":42463,"ще":53089,"ща":18897,"ъе":8051,"ый":115257,"ык":21356,"ые":44083,"ыд":3688,"ыв":20957,"ыш":9845,"ыч":6916,"Ñ‹Ñ…":88893,"Ñ‹Ñ‚":10971,"Ñ‹Ñ":15933,"Ñ‹Ñ€":7427,"ып":8198,"ын":9614,"ым":40979,"ыл":22285,"ьк":10517,"ьд":4034,"ье":17860,"ьз":12362,"ьб":9420,"ьш":16573,"ÑŒÑ":28658,"ьт":14115,"ьн":93902,"ьм":17794,"хе":5192,"хи":15842,"хн":9628,"хо":65229,"Ñ…Ñ€":8875,"Ñ…Ñ":5190,"ху":8026,"Ñ„Ñ„":4256,"ха":22339,"ци":120621,"цк":12640,"цо":4729,"цу":6546,"ца":29752,"це":46729,"чл":3954,"чн":40293,"чи":44063,"чк":5394,"чт":9754,"цы":8412,"че":142550,"ча":74585,"шо":6953,"шн":6580,"шк":7840,"шл":7340,"ши":40508,"шт":17550,"ше":40253,"чё":6388,"ша":19003,"Ñк":366421,"Ñм":15604,"Ñл":64229,"Ñо":139920,"Ñн":41282,"ÑÑ€":13223,"Ñп":78471,"Ñв":31144,"рё":5020,"Ñб":4953,"Ñе":102280,"Ñи":96102,"рш":7321,"ры":45781,"рь":12146,"Ñа":63371,"Ñ€Ñ":46850,"рр":14529,"Ñ€Ñ":52200,"рт":57700,"ру":105041,"рх":14747,"рц":4365,"тн":65472,"тм":3983,"тл":9354,"тк":17989,"Ñ‚Ñ":84335,"Ñ‚Ñ€":155968,"то":273813,"те":235202,"тд":4232,"тв":124455,"ÑÑ‘":6893,"ти":199676,"ÑÑ‹":9941,"ÑÑŒ":18264,"та":266195,"ÑÑ":89398,"тб":4549,"Ñу":38119,"ÑÑ„":3977,"ÑÑ":88379,"ÑÑ‚":540751,"Ñч":5733,"Ñш":4240,"ÑÑ…":8473,"Ñц":3788,"ур":58652,"уп":39068,"ут":30026,"уÑ":56106,"ум":19318,"ул":47107,"ун":41314,"уи":4250,"уз":23986,"ук":30175,"уд":42687,"уг":33231,"уж":19840,"уе":15435,"уа":10851,"Ñ‚Ñ":20201,"уб":26698,"ув":5212,"Ñ‚Ñ‹":37440,"Ñ‚ÑŒ":74835,"тч":3850,"Ñ‚Ñ‚":6006,"ту":57964,"фу":8016,"Ñ„Ñ€":15690,"фо":29756,"фи":45878,"фе":25068,"фа":16306,"ую":32657,"ущ":17182,"уч":38493,"уш":8215,"ух":10059,"Ñ‘Ñ‚":8989,"ён":22517,"Ñ‘Ñ€":12412,"ёл":4881,"ём":5538," II":3728," Ma":3710,"а ":616120,"Р ":10559,"С ":6434,"ИÑ":5682,"Ин":6739,"к ":120218,"Из":4203,"Ив":3823,"й ":507309,"Ле":11105,"Ли":8362,"Ла":6540,"Ку":8515,"Ко":21705,"м ":237652,"Кр":10087,"Ки":8491,"Ка":27548,"л ":64722,"Ðа":20616,"Ðе":6593,"Ðи":9075,"Му":3926,"Мо":21246,"о ":404148,"Ма":25177,"Ми":12922,"Ме":9781,"Ло":5700,"н ":135483,"Лу":3901,"Па":15703,"Пе":17521,"Пи":4313,"Пл":3876,"Ñ ":91974,"По":29660,"Ñ€ ":85184,"ОÑ":6849,"Ор":3751,"От":4582,"Об":5237,"Он":3901,"Ол":8342,"Ðо":9337,"п ":8470,"в ":366455,"Ðм":4416,"Ðн":11755,"Ðл":14756,"Ðв":5862,"Ба":12391,"ÐÑ€":8812,"б ":7229,"д ":64292,"Во":11807,"Ве":14286,"Ви":9847,"Га":7333,"Бо":12192,"г ":35799,"Бр":14227,"Бе":12321,"Би":3781,"Ва":10241,"Бу":4848,"Ди":3742,"Дж":8980,"Де":6705,"ж ":6087,"До":7659," Th":4595,"Ев":5669,"Ге":11433,"Гр":11587,"Го":10198,"е ":450788,"Да":6449,"и ":574901,"За":11852,"з ":53551,"ÑŒ ":133456,"еÌ":15696,"Ñ‹ ":130752,"иÌ":16589,"Ñ‘ ":6265,"Яв":5487,"ÑŽ ":63842,"Ñ ":490850,"Эт":4232,"Ст":11217,"Су":5933,"Та":9444,"Ти":3758,"Те":7001,"Ñ„ ":6051,"То":6501,"Тр":6028,"Ту":4280,"Ук":4529,"Ñ… ":211491,"Пр":23361,"Ра":11292,"Ре":10364,"Ри":7652,"СР":8553,"Ñ‚ ":206463,"Ро":21953,"Ру":5315,"СС":13685,"СШ":6418,"Са":18244,"Св":4560,"Си":8236,"Се":17477,"у ":114144,"Со":23127,"Це":5862,"ш ":6132,"аÌ":24040,"Че":6673,"ШÐ":6418,"ц ":11865,"Фр":9086,"Фе":6645,"Фи":4961,"Ха":6486,"ч ":21859,"Хо":3899,"ль":255358,"лы":10808,"лю":22731,"мб":8994,"лÑ":104684,"ма":130982,"лё":9664,"ме":172608,"ми":140084,"мл":6471,"лл":23806,"лн":13959,"ло":163517,"лÑ":10825,"лт":4507,"лу":35325,"ла":158459,"лж":4211,"ле":205849,"лд":3683,"лг":5390,"лк":10879,"ли":255185,"км":7931,"кн":9321,"кк":4284,"кл":31050,"кр":60959,"кÑ":31731,"ко":383857,"кт":66101,"ку":41628,"кц":9152,"ка":219076,"кж":14683,"ки":210538,"кв":18587,"ке":35320,"йн":21970,"йо":19765,"йр":3918,"йк":4778,"йл":4612,"йш":7189,"йÑ":69884,"йт":6056,"иÑ":163126,"ию":29435,"ищ":4465,"иш":6262,"уÌ":5838,"йд":4105,"ио":61311,"ип":31810,"им":121890,"ин":226639,"ик":130372,"ил":122490,"ии":133366,"ий":180537,"иц":62824,"ич":108247,"иф":12791,"их":84356,"ит":178471,"иу":4714,"ир":73443,"иÑ":176435,"ри":235653,"рк":23089,"рл":11046,"рм":37955,"рн":63599,"ро":363111,"рп":4622,"ра":429505,"рб":11729,"рв":24770,"рг":34028,"рд":18157,"ре":279331,"рж":10477,"пь":4901,"пы":11436,"пр":223152,"пп":19192,"пт":6272,"пÑ":5990,"пу":32229,"пи":56651,"пн":7871,"по":247631,"пл":31561,"ою":7698,"оÑ":26385,"па":81900,"пе":95232,"ощ":8932,"ош":11972,"оч":34323,"оц":13939,"оÑ":5861,"оÑ":271102,"ор":307758,"оп":60188,"оо":12934,"ох":15431,"оф":19467,"оу":5082,"от":162519,"ок":93705,"ол":228863,"ом":207667,"он":199509,"ож":50393,"оз":54318,"ои":31770,"ой":205541,"ов":359690,"нё":5120,"ог":196172,"од":289964,"ое":90767,"нÑ":35079,"оа":5488,"об":119246,"нь":18690,"ны":243593,"нц":33951,"нч":4962,"нт":90634,"нÑ":97833,"нф":7203,"ну":26904,"но":402371,"нн":163382,"нр":4657,"нк":27682,"нз":4096,"ни":399245,"не":163042,"нг":32092,"нд":56240,"мё":3976,"нв":7556,"на":409072,"мÑ":17291,"мь":4683,"мы":32208,"му":54274,"мÑ":10476,"мп":42062,"мо":86128,"мн":19897,"мм":15488,"ге":39235,"гд":7818,"ги":64206,"гн":5587,"го":307537,"гл":27873,"гм":4198,"гр":76619,"гу":22493,"дв":21897,"да":163117,"вг":7437,"ве":186015,"ви":112054,"вк":13130,"вл":60895,"вм":3919,"вн":71300,"во":212143,"вп":4179,"вр":34804,"вÑ":46803,"ву":21309,"вт":16905,"вх":16990,"вш":21902,"вы":73241,"вь":4164,"га":61976,"вÑ":19117,"вё":4031,"би":27123,"бе":43659,"бр":63607,"бн":9059,"бо":89921,"бл":44500,"бу":21528,"бÑ":7153,"ва":210954,"бы":35755,"бъ":8402,"бщ":13191,"ад":83035,"ае":38603,"аж":22477,"аз":106308,"аб":41193,"ав":144996,"аг":30571,"ам":106564,"ан":372615,"ап":45088,"аи":20853,"ай":49922,"ак":101386,"ал":220430,"ах":49626,"аф":15259,"ач":35451,"ац":44113,"аÑ":161556,"ар":181597,"ау":19997,"ат":182329,"ба":30240,"аÑ":142316,"аю":29293,"ащ":9761,"аш":12824,"зÑ":6594,"зр":12483,"зу":17726,"зк":4107,"зи":46141,"зо":48202,"зн":42107,"зм":17172,"зл":7624,"ив":68972,"иг":39987,"зÑ":3693,"иа":42640,"иб":20393,"иж":14048,"из":124756,"ид":43626,"ие":128645,"зы":33503,"жо":5749,"жÑ":4009,"жу":6473,"жи":33668,"жн":29430,"за":99672,"зв":49033,"жё":3790,"зг":3804,"зд":32492,"зе":22277,"еф":7165,"еу":4109,"ет":190550,"еÑ":211743,"ер":322951,"еп":23032,"ео":22288,"ен":392846,"ем":131477,"ел":205217,"ек":97566,"ей":99541,"еи":5939,"ез":47927,"еж":36786,"ее":34825,"жд":33436,"же":89890,"жа":21003,"еÑ":12292,"ещ":8165,"еч":24475,"еш":11291,"ех":17181,"ец":24531,"дÑ":40426,"др":39150,"ду":60618,"дн":71944,"дм":11329,"дп":4848,"до":102671,"ди":135662,"дл":31466,"дк":7688,"де":155382,"дд":3909,"оÌ":17497,"дж":10265,"еб":21454,"ев":101298,"дё":3920,"ег":62011,"ед":133360,"дь":6956,"еа":12623,"дÑ":7893,"дш":3897,"ды":16920," Яв":5485," ар":13568," ба":10710," ав":18221," ад":9137," ал":10374," ак":9085," ан":12535," ам":9443," ап":7635," бу":7250," ва":6199," бы":23597," би":6156," бе":13045," бр":8596," бо":19871," бл":6242," вт":4810," вх":16859," га":5609," вы":32582," ви":13232," ве":27020," во":59783," вÑ":13316," вр":12829," вл":4862," вк":4169," вн":5530," дв":15611," да":12181," гу":4609," го":108387," гл":8578," гр":29774," ге":11208," ги":5523," ед":6587," ег":8307," до":35767," др":17969," ду":6602," де":43890," ди":14485," дл":25835," же":14087," за":57830," зв":5213," жи":10062," зн":7274," иг":13915," из":63621," ил":26819," ин":23235," им":21022," иÑ":39365," их":4602," ию":10395," ка":46349," ки":10920," кр":19920," ко":101579," кн":6962," км":7101," кл":11050," ку":8839," ла":5400," ли":22008," ле":16122," ло":3818," ме":49176," ми":21788," лю":4902," ма":35222," мо":29145," мн":7010," му":26862," ни":8598," не":49337," на":155501," но":20718," ок":24600," он":5812," од":28184," об":63971," от":54560," ор":15811," оÑ":31704," оп":15294," по":164771," пл":14871," пи":11060," пе":48196," па":21414," Ре":10285," Ра":11205," Ро":21862," Ри":7649," Пр":23192," Пе":17464," Па":15586," Ñ ":51024," По":29407," Пл":3863," Пи":4292," От":4556," ОÑ":6825," Ор":3729," Те":6933," Ти":3719," То":6441," Тр":5998," Ст":11107," Су":5919," Та":9379," Св":4548," Си":8202," Се":17430," у ":4211," Со":23029," Ру":5277," СС":6857," СШ":6404," Са":18190," Фр":9072," Фи":4937," Фе":6613," Ук":4507," Ту":4270," Це":5856," Хо":3885," Ха":6470," Че":6653," Эт":4213," Ба":12348," ÐÑ€":8769," в ":244550," Ðн":11697," Ðм":4402," Ðл":14708," Ðв":5843," Ва":10204," Бу":4830," Бо":12018," г ":7009," Бр":14207," Бе":12299," Би":3719," а ":10629," Ев":5656," Ди":3719," Дж":8956," Де":6645," До":7584," Га":7281," Ве":14245," Ви":9808," Во":11759," Да":6384," Ге":11389," Го":10155," Гр":11539," ИÑ":5607," Ин":6668," к ":15541," Ки":8443," Ка":27418," и ":157390," За":11752," й ":4397," Ив":3814," Из":4180," Му":3903," Мо":21203," о ":7496," Ðа":20476," Ðе":6542," Ðи":9034," Ðо":9287," Об":5188," Он":3896," Ол":8330," Ко":21565," Кр":10028," Ку":8476," Ла":6504," Ле":11080," Ли":8319," Ло":5687," Лу":3887," Ма":25091," Ме":9720," Ми":12850," Ð’ ":17470,"II ":7232," ра":87254," ре":41392," ро":33130," пр":177960," пÑ":3748," пу":8711," Ñв":25177," Ñи":22645," Ñе":47025," Ñл":16395," Ñм":6218," Ñк":7720," Ñп":19254," ÑÑ€":11080," Ñн":4707," Ñо":89660," ру":13053," Ñа":14863," ти":6431," те":38937," то":20252," Ñ‚Ñ€":22707," ÑÑ‚":53938," Ñу":12938," та":27006," ук":4204," уÑ":11044," уп":6557," ур":3864," ун":4890," ул":9872," ту":5700," фо":12379," Ñ„Ñ€":10534," фу":6744," фе":9085," фи":20989," фа":9016," уч":17593," Ñ…Ñ€":4873," хо":7233," ху":6092," ха":4633," ци":3784," це":18264," чт":6942," чи":6042," че":25547," ча":26885," ша":3808," шт":13545," Ñк":9725," Ñл":8825," ÑÑ‚":18390," юг":4276," Ñн":5667," Ñз":10931," Ñв":12558,"аз ":5770,"ад ":7406,"Явл":5341,"ав ":5373,"ам ":14841,"ан ":31178,"ак ":26116,"ал ":21311,"ай ":4239,"авш":9597,"авт":9645,"ага":8115,"аго":7572,"ада":10767,"ади":12630,"аде":10895,"адм":7561,"адн":6641,"адь":3777,"аем":7453,"ает":25830,"аже":6271,"ажд":5556,"аба":3999,"або":17739,"абр":6755,"ава":11200,"авн":21035,"авл":26690,"авÑ":4043,"аво":12671,"аве":7803,"авг":5378,"ави":21691,"ало":13331,"алл":5996,"ала":22307,"али":48595,"але":13975,"амм":7674,"амо":8686,"амы":4469,"алÑ":5952,"ама":7226,"аль":73407,"ами":28699,"аме":19983,"анн":37419,"ано":18426,"анÑ":39188,"ант":20680,"анц":15483,"аны":7448,"ань":4229,"ана":29768,"анд":25091,"анг":9722,"ани":99562,"ане":11955,"анк":9278,"азр":7025,"азо":13307,"азн":9789,"азл":4533,"ази":10605,"азе":3818,"азд":5347,"азв":17619,"аза":10657,"аин":6236,"аиб":4114,"азы":8101,"айÑ":4889,"айо":18909,"айн":4793,"акт":16378,"ако":12308,"акж":14674,"аки":5796,"ака":8072,"ах ":31165,"Ð°Ñ ":6451,"ар ":4822,"ат ":18392,"Ð°Ñ ":136821,"ба ":4571,"Пол":8430,"Пор":6259,"При":5525,"Пре":6670,"Про":7157,"РоÑ":14485,"РаÑ":4325,"РеÑ":3751,"Сан":8587,"ССС":5587,"ССР":7399,"СШÐ":6399,"СоÑ":6600,"Сов":3947,"Сев":6002,"Сер":4458,"Ста":4121,"Укр":4080,"Фед":3907,"Фра":5981,"ШР":6384,"аÌн":5689,"лам":6144,"лан":15129,"лаÑ":29897,"лат":7877,"ме ":7744,"Ð»Ñ ":50911,"ма ":24448,"лав":17972,"лаг":7649,"лад":11258,"лы ":5725,"ль ":38018,"кую":3788,"куÑ":4925,"кул":8752,"ктÑ":6372,"кци":8948,"кре":6974,"кра":17934,"кри":5557,"кру":16231,"кро":7021,"лу ":5744,"кры":6225,"кÑа":6759,"кÑи":5075,"кта":5223,"кте":4216,"кти":10807,"кто":14126,"ктр":8094,"кту":4971,"кла":9863,"кло":3891,"ло ":19382,"клю":8248,"кни":4141,"ког":53446,"ков":40843,"ком":58598,"кон":29711,"коп":5499,"кор":15168,"коÑ":5742,"кот":33255,"кое":17818,"кой":65146,"кол":25577,"кие":12731,"кин":10160,"ким":14411,"кий":81796,"ких":35222,"ле ":19103,"кже":14673,"ли ":56435,"ква":7601,"ках":4702,"кат":7467,"кар":8017,"кам":8824,"кан":23037,"как":20319,"кал":11552,"каз":10972,"каÑ":36891,"каб":6283,"кад":5211,"ла ":40418,"йши":4432,"йÑÑ":5983,"йÑк":44468,"йÑÑ‚":14738,"кт ":8556,"ку ":9345,"йны":8079,"ÐºÑ ":4079,"йон":18007,"ко ":19496,"км ":4431,"ки ":43486,"ке ":21167,"иÑм":6120,"иÑÑ…":4963,"нва":6008,"од ":33146,"наÑ":57610,"нах":9504,"нац":5517,"нау":8200,"нач":22853,"ог ":6440,"нан":6213,"нам":8570,"нал":25338,"нат":9581,"наÑ":17157,"нар":16443,"нап":11298,"над":9721,"нак":5181,"наи":4066,"наз":20814,"нде":5878,"нда":8693,"нгл":9288,"нге":3994,"нга":3873,"ое ":58821,"ней":12822,"нек":4897,"нем":9855,"нен":19222,"нер":13236,"неÑ":9300,"нет":5791,"нег":6018,"нев":4773,"нее":5864,"нди":11172,"ндо":5836,"ндр":8570,"нив":5586,"нии":26244,"низ":11864,"ник":36850,"ний":18207,"ниг":4932,"ние":74066,"ок ":21671,"ой ":187929,"ны ":35382,"нь ":8902,"Ð½Ñ ":11152,"мый":5262,"мых":5149,"мыш":3735,"ов ":92702,"нт ":11498,"мпо":5172,"мпи":13008,"мпе":6698,"мпа":7653,"мот":4724,"ну ":6285,"мпь":3765,"мÑк":6474,"мун":14544,"мул":3800,"муз":9742,"мик":7960,"мил":4443,"мии":4851,"мич":7046,"мин":28165,"мир":17908,"мит":4140,"но ":62002,"мму":3810,"мно":11450,"мод":4772,"мог":5035,"мов":6153,"мой":4768,"мож":6250,"мон":9329,"мом":4520,"мол":6893,"моÑ":8228,"мор":8527,"нд ":4096,"мац":4061,"лÑÑŽ":8990,"маÑ":9013,"лÑл":3785,"мал":9300,"лÑе":22475,"мат":21959,"маÑ":5866,"лÑÑ€":4924,"мар":12011,"ман":23543,"люч":8195,"мец":4888,"меÑ":13130,"мет":17123,"мен":50820,"ни ":19557,"мер":29417,"мез":5962,"меж":12542,"мед":8168,"не ":38359,"лён":4990,"мы ":9798,"льн":93765,"на ":153332,"Ð¼Ñ ":11127,"льм":14104,"льк":7925,"льз":12358,"льд":3831,"льб":7821,"льÑ":5362,"льш":14923,"льт":14050,"льÑ":23881,"лощ":4920,"му ":16323,"лок":5500,"лог":21990,"лод":5038,"лож":18221,"лор":4516,"лоÑ":11300,"лот":8157,"лом":9527,"лон":6111,"лов":35245,"луч":9778,"луж":6113,"лÑÑ":9183,"лиг":4087,"лив":6013,"лиз":10430,"лии":15024,"лим":9347,"лий":6287,"лик":17837,"лез":6815,"лей":13214,"лев":9323,"лег":4328,"лед":13548,"лее":8350,"леÑ":4757,"лер":5420,"ми ":51806,"лен":60261,"лем":10309,"лек":22756,"лет":13846,"лли":5515,"лла":3879,"лле":7201,"лиц":18482,"лич":17052,"лиÑ":14036,"лит":29052,"лин":18545,"лиÑ":7551,"лко":3705,"паÑ":4125,"оÑщ":5066,"оÑб":5483,"пад":12246,"пал":13951,"рг ":4862,"оÑн":3986,"пан":12047,"пар":16176,"ре ":17661,"ра ":44965,"оюз":3745,"пий":7004,"пио":5573,"пиÑ":24111,"пла":9596,"пле":8010,"ро ":9844,"пло":9025,"пед":4071,"ри ":19004,"пер":55359,"пеÑ":4491,"печ":4195,"пец":6528,"ори":37323,"орд":5846,"оре":20523,"орг":13852,"орÑ":6547,"оро":76961,"орм":17625,"орн":11312,"опу":5280,"ора":20394,"опе":10688,"опи":6454,"опо":11431,"опр":11046,"оор":3720,"опа":4525,"отд":3834,"оте":8329,"отк":7498,"отл":4401,"оти":8154,"ото":45196,"отн":11373,"отр":8447,"оÑу":15201,"отв":5315,"ота":11881,"оÑе":5320,"оÑи":7543,"оÑк":15979,"оÑл":18494,"оÑн":17062,"оÑо":11923,"оÑп":4485,"оÑÑ":25551,"оÑÑ‚":122012,"ору":9047,"орт":19622,"оры":19028,"орÑ":6038,"оÑв":3854,"омм":4672,"оми":12071,"оме":14471,"ома":19204,"олÑ":5909,"оль":52719,"олу":10916,"олн":13122,"по ":36706,"оло":54184,"олл":5951,"олж":3930,"оле":21412,"оли":27404,"ола":9666,"окр":16259,"окт":7805,"оку":5419,"око":16003,"ооб":4126,"онÑ":12501,"онт":8231,"онц":5168,"они":16817,"онк":3985,"оно":19353,"онн":16949,"она":44812,"онд":5190,"оне":18296,"омо":11288,"омп":18935,"ому":11330,"оше":5000,"пы ":9126,"очи":4013,"оче":6235,"очн":15161,"оща":4802,"офе":5906,"офи":7745,"оты":3797,"оце":7042,"оци":5856,"охо":7896,"нÑÑ‚":6570,"нÑе":3761,"ова":79059,"обы":7576,"общ":12061,"объ":7061,"обр":22953,"обо":16065,"обн":4315,"обл":17532,"оби":5216,"обе":11035,"ных":63297,"ные":28042,"ным":27407,"ный":86741,"па ":9132,"оит":3855,"оиÑ":6658,"оим":4265,"ойÑ":6061,"ойн":7471,"оке":4198,"ока":10728,"ожи":4007,"ожн":11252,"озд":12188,"озв":5021,"ози":6006,"озн":9808,"оиз":10355,"одн":39144,"оди":52480,"оду":22578,"одр":3912,"одÑ":13394,"одо":20011,"оды":5756,"оед":4040,"оев":9413,"одÑ":6688,"оен":8911,"оек":3819,"оже":24113,"ожд":8010,"ове":40857,"овк":4823,"овл":5356,"ови":24912,"ово":42247,"овн":14676,"овр":5254,"овÑ":17603,"овы":17385,"ога":5667,"овÑ":4334,"огд":4881,"оги":20707,"огл":3937,"ого":132906,"огр":15611,"ода":58550,"оде":17121,"от ":31995,"ноÑ":6122,"ноÑ":44637,"нор":3684,"нол":4077,"ном":37955,"ной":71982,"ное":29866,"ног":57728,"нод":4260,"нов":54046,"нны":69394,"нно":59505,"ор ":24212,"нни":6472,"нна":20166,"нко":5353,"он ":29089,"нкт":5008,"нка":4499,"ом ":104456,"ниÑ":84752,"нию":8268,"нир":5623,"ниÑ":17636,"нит":8771,"ним":17208,"нин":5963,"нич":11022,"них":9088,"ниц":23421,"нце":5996,"нци":17698,"нцу":5080,"ную":9327,"нфо":3773,"нтÑ":5255,"нут":4116,"нта":17028,"нте":10866,"нти":10987,"нто":10091,"нтр":15601,"нÑк":63751,"нÑÑ‚":16529,"Ñам":12116,"Ñан":12409,"Ñ€Ñд":7767,"Ñат":6375,"Ñво":14816,"те ":15248,"Ñве":4519,"ÑвÑ":9460,"Ñев":8467,"Ñел":21166,"ти ":46330,"Ñен":10794,"Ñем":7371,"Ñет":3970,"Ñер":16452,"ÑиÑ":11359,"Ñит":9045,"Ñий":14008,"Ñии":9496,"Ñин":6230,"Ñил":7978,"Ñим":6904,"Ñкв":8290,"Ñки":131938,"Ñка":46869,"Ñли":3844,"Ñле":23336,"Ñла":10537,"Ñку":8391,"Ñко":157451,"Ñме":6005,"Ñлу":9361,"Ñло":15249,"то ":29238,"ÑнÑ":3968,"Ñоб":15821,"Ñов":27452,"Ñоз":9281,"Ñок":8059,"Ñом":3851,"Ñно":26195,"Ñ‚Ñ€ ":8572,"Ñны":3885,"Ñпе":13580,"Ñпа":4761,"Ñпи":4683,"ÑоÑ":21656,"Ñор":6482,"Ñоо":5100,"Ñон":7291,"Ñоц":4226,"Ñре":10218,"ту ":8116,"Ñпу":6484,"Ñпо":39170,"Ñпр":6952,"рри":9506,"роц":5562,"рош":3692,"рот":12263,"роф":6754,"рох":5095,"роп":12392,"роÑ":28947,"ÑÑ‚ ":16255,"рта":13460,"Ñ€ÑÑ‚":15956,"рто":4691,"рти":13560,"Ñ€Ñк":18805,"Ñ€Ñо":4243,"Ñ€Ñи":7173,"рую":4208,"рту":7614,"рук":7417,"руг":20792,"руд":5213,"руж":6808,"руп":20038,"руÑ":13228,"рхи":6150,"ÑÑŒ ":14955,"ÑÑ ":83299,"та ":65491,"рыт":4303,"рых":5096,"рый":5524,"рые":4885,"тв ":5853,"рад":12120,"раж":10828,"раз":53144,"раб":18768,"рав":32412,"рам":17118,"ран":56502,"раи":8567,"рай":21830,"рак":10369,"рал":23925,"рах":8851,"раф":10748,"рац":10644,"раÑ":32059,"рат":31627,"раÑ":5022,"ращ":3710,"рбу":3964,"рва":4083,"пью":3789,"реб":6936,"рев":19639,"рег":20011,"ред":53953,"реа":5135,"рет":10005,"реÑ":14958,"реп":5228,"Ñи ":5105,"рен":15519,"рем":25502,"рел":11664,"рек":12317,"рей":7082,"рез":11794,"реж":11533,"ржа":3754,"реч":6532,"рво":6073,"Ñе ":6838,"рвы":9033,"рга":14493,"рге":4800,"рги":4021,"риÑ":13567,"рию":4048,"рио":6627,"рим":11681,"рин":20707,"рик":20481,"рил":3781,"рии":19852,"рий":6871,"рич":8657,"рит":22893,"рир":3686,"риÑ":16618,"рка":4003,"риа":11930,"риг":4581,"рив":4645,"рид":5404,"риз":6655,"Ñк ":4696,"рни":9740,"рна":12681,"рок":11103,"рол":12179,"ром":23332,"рон":16716,"рож":7867,"роз":4092,"рои":18841,"рой":14224,"ров":68815,"рог":16486,"род":57888,"рое":10921,"рны":14512,"рно":18416,"рла":4760,"рко":4371,"рми":8750,"рма":15602,"Ñо ":3903,"ппы":6617,"пра":26585,"при":52484,"пре":53035,"ру ":6954,"про":87029,"поп":4906,"пор":17269,"поÑ":28755,"пот":5403,"поÑ":3864,"ппа":6927,"рт ":5103,"The":3957,"под":23464,"пов":12924,"пон":8324,"пом":6183,"пол":68859,"пок":4701,"поз":9391,"пуб":8028,"пуÑ":5784,"пут":5691,"пул":3714,"ры ":15775,"Ñа ":16206,"Ñ€Ñ ":29812,"рь ":4987,"вар":15705,"ват":16606,"ваÑ":7821,"ваю":8055,"ге ":6711,"вае":10913,"вав":4587,"ван":65485,"вал":18282,"быт":4123,"быч":3846,"был":18105,"га ":14808,"бъе":7043,"бще":10673,"вы ":7062,"бур":7473,"брÑ":21845,"вто":16228,"вÑе":9508,"вÑк":27242,"вÑÑ‚":7859,"вра":8893,"вре":17466,"вро":6581,"вол":13485,"вок":4441,"вой":25086,"вои":3759,"воз":12251,"вое":18754,"вод":28316,"вог":6893,"вов":10300,"вны":14251,"вор":11524,"воÑ":15224,"воп":3740,"вом":8183,"вни":4616,"вне":8106,"вна":13669,"вно":25418,"влÑ":30877,"вле":21719,"вла":3874,"го ":153532,"вкл":4066,"вка":4191,"вич":20022,"виÑ":4450,"виж":4984,"вил":7784,"вин":10044,"виÑ":5933,"вит":12981,"вид":14202,"веÑ":15630,"вет":22334,"вер":37523,"вен":36297,"ги ":6381,"вел":6076,"век":17069,"вед":16797,"вгу":5781,"ва ":46587,"ающ":19084,"ают":9616,"ачи":5855,"бы ":3828,"ащи":4099,"аще":3705,"ауч":3787,"аук":4642,"афи":6466,"ахо":7975,"ача":13504,"аче":11447,"аци":41912,"апр":13134,"апа":11764,"апо":3746,"апи":8487,"арх":7888,"арÑ":18625,"арт":22698,"арь":4529,"арÑ":9072,"аре":7851,"ард":6979,"ара":21925,"арн":5879,"арм":4715,"аро":18219,"ари":19835,"арл":4156,"арк":7904,"аÑÑ":17733,"аÑÑ‚":73387,"аÑÑŒ":5439,"ата":17783,"аÑи":4710,"аÑе":9811,"аÑл":4318,"аÑп":18234,"аÑн":7236,"ату":10612,"аты":6297,"ать":13519,"ате":41110,"ати":28209,"атн":5152,"ато":21641,"атр":7249,"бол":22933,"бом":6832,"бой":5049,"бор":14402,"бот":13748,"бно":3859,"бро":3695,"бри":6979,"бре":4187,"бра":24036,"бла":17775,"бли":15168,"бле":4490,"во ":31067,"ви ":4378,"беÑ":5023,"бер":15432,"бел":4899,"бит":4733,"бил":3862,"ве ":17457,"даю":4004,"дах":4951,"дан":21423,"дам":3898,"дар":18766,"дат":8984,"дви":7106,"дал":8909,"дав":5333,"ев ":8761,"дек":7455,"дей":9763,"дем":5618,"дел":26493,"ден":32485,"дер":21083,"ей ":52679,"деÑ":4085,"дет":5015,"деÑ":8136,"дву":4656,"дво":4445,"ее ":25157,"ез ":4626,"ды ":12967,"дь ":4234,"ех ":3705,"дÑÑ‚":24032,"дÑк":13686,"дро":6400,"дру":10418,"дре":7577,"дра":7639,"ет ":49624,"дун":5024,"ец ":6315,"ен ":23878,"диÑ":4777,"диц":4594,"ем ":32646,"диа":3739,"див":3812,"дим":5230,"дин":31699,"ел ":6867,"дио":3858,"диÑ":4374,"дит":31866,"дии":5066,"дил":7726,"ек ":8976,"дны":11957,"дож":6200,"дов":21587,"доÑ":6949,"дор":10512,"дол":8011,"док":5562,"дон":5574,"дом":8881,"дна":11456,"дни":9495,"дне":7494,"ер ":23656,"дно":27634,"длÑ":23991,"дми":9130,"вып":7258,"вый":9785,"вым":6060,"вых":10632,"выÑ":8860,"al ":4210,"да ":70397,"вÑз":5698,"гал":7699,"вÑÑ‚":8003,"ган":16467,"гар":4062,"де ":21659,"гда":5300,"вую":5927,"вхо":16916,"вше":5430,"вша":3781,"вши":12432,"вые":6905,"гон":3740,"гол":6702,"гоÑ":13811,"гот":3744,"гор":32983,"гов":9981,"год":69990,"гру":17285,"ду ":34904,"гро":3890,"гра":40520,"гре":6189,"гуÑ":6354,"ген":9872,"гер":7302,"ди ":8816,"гии":7384,"гио":11815,"гич":7565,"гих":4581,"гла":12247,"до ":13248,"гли":6452,"жан":5670,"еÑÑ‚":7722,"за ":17832,"еще":5057,"жит":5272,"жив":7938,"жиÑ":4312,"жеÑ":7568,"жет":3940,"жду":12980,"жел":6716,"жен":38841,"жде":11274,"жда":4735,"жÑк":3945,"жно":12596,"жни":4413,"жны":6194,"жур":3704,"ежи":6716,"ежд":16924,"еду":5404,"едÑ":18751,"еза":4036,"езн":5938,"езо":10562,"езд":5637,"ези":9501,"ева":10267,"еви":11883,"еве":16643,"еат":3835,"дÑщ":4359,"его":29841,"еда":12960,"еде":22810,"еди":27267,"едо":10112,"едн":12312,"евн":10381,"же ":22970,"ево":16350,"евр":9678,"евÑ":6183,"евы":4574,"еге":5203,"еги":14109,"ент":47594,"енÑ":9645,"енц":4019,"ени":132856,"ено":11907,"енн":84711,"ена":18893,"емÑ":7341,"ене":16352,"енд":7186,"емь":4324,"емы":10998,"еор":5073,"ены":6940,"ень":8052,"епо":3992,"ерх":5155,"ерр":10216,"ерÑ":17748,"ерт":9897,"ерл":3718,"ерм":10872,"ерн":25326,"еро":19985,"ери":40656,"ерк":7071,"ерг":7482,"ерж":8654,"ере":44413,"ера":37126,"ерв":20819,"ерб":7899,"ейн":5541,"ейÑ":18646,"еки":4636,"еко":11088,"ект":27081,"екÑ":15558,"ейш":5926,"ека":19196,"ели":21048,"ело":19740,"еле":41611,"ела":9361,"емл":3751,"емо":8767,"еми":9433,"ему":3888,"емп":6664,"ель":74971,"еме":23861,"елÑ":16614,"ема":12283,"елё":3840,"ехн":5852,"ецк":9009,"еци":7333,"еча":3853,"ечн":3687,"ече":11632,"еше":3757,"еÑе":5474,"еÑк":73489,"еÑн":5953,"еÑп":9644,"еÑÑ":15061,"еÑÑ‚":79162,"еÑÑ":5372,"ета":21699,"ети":10490,"ете":10421,"етр":11442,"ето":9221,"етн":10174,"етÑ":48640,"еты":4087,"ибо":7839,"иве":7414,"иви":3809,"ива":16306,"иал":20951,"иан":8939,"иже":6481,"идо":4070,"иев":3803,"ием":14066,"ией":9608,"игр":16265,"иго":4097,"ида":4859,"иде":14015,"иво":7871,"ивн":17940,"ивш":6456,"ига":5856,"иги":5053,"икл":4956,"икр":3804,"ико":26837,"ике":6203,"ики":16651,"ика":34115,"ийÑ":37107,"изм":10707,"изо":8828,"изн":6897,"изи":10371,"изд":4964,"иза":12843,"изв":18761,"ион":41500,"инц":6456,"ины":10703,"инÑ":5979,"иод":4236,"ине":14798,"ини":31286,"инн":3826,"ино":20897,"инÑ":24611,"инт":7204,"инф":4063,"ина":38691,"инд":6333,"инг":12127,"ими":15581,"име":21870,"имÑ":3708,"имо":9905,"имп":12615,"има":14708,"иль":23825,"или":46988,"иле":5170,"илÑ":4907,"илл":5632,"ило":11583,"ила":9561,"иÑи":5473,"иÑа":15121,"иÑÑ…":5676,"иÑÑ‚":74674,"иÑÑ":10637,"иÑп":16317,"иÑо":5087,"иÑл":10745,"иÑк":11427,"ити":13816,"ите":56212,"ита":24816,"иÑÑŒ":6230,"иту":6461,"ито":18876,"итÑ":8689,"ипа":14414,"ипе":3791,"ира":11140,"ире":4647,"иру":8486,"ири":4818,"иро":27497,"иха":3800,"ихо":4447,"ице":7291,"ица":16979,"ици":25302,"ицы":6049,"ить":5325,"ифи":3877,"ичи":6000,"ичн":11017,"ича":4726,"иче":66707,"июн":5092,"июл":5316,"ка ":55202,"ив ":4402,"зав":10564,"ид ":4294,"зви":4153,"зве":16359,"зва":15033,"зац":9007,"зат":4555,"зап":11760,"зан":11015,"зам":3927,"зак":6826,"зде":5994,"зда":15915,"ие ":94285,"зво":7967,"ий ":139271,"зер":5619,"ии ":131775,"зем":4097,"из ":38626,"зид":4005,"зил":7018,"ил ":8940,"ик ":27882,"ин ":25833,"им ":24359,"зиÑ":4934,"зит":4573,"зме":6834,"зли":4448,"зна":20529,"зни":4774,"зно":8729,"ир ":5392,"зны":4336,"зов":19642,"Ð¸Ñ ":4567,"зон":5974,"зор":7083,"ит ":29545,"зра":8237,"зÑк":6535,"их ":71178,"зуе":5324,"ич ":17567,"зыв":8397,"ию ":18909,"зык":19420,"Ð¸Ñ ":146910,"ьшо":3872,"ьши":3819,"ьше":5810,"ÑŒÑн":5268,"ьют":3946,"ьма":3914,"ьна":7658,"ьни":4302,"ьно":44690,"ьны":33869,"ько":6149,"ion":5234,"ьзу":6190,"ьзо":5415,"ьту":4384,"ÑŒÑÑ‚":6382,"ÑŒÑк":16866,"Ñм ":5239,"ÑŽÑ‚ ":10384,"Ñко":4504,"Ñле":8958,"Ñто":14311,"Ñ‹Ñ… ":86372,"he ":5907,"ыва":15848,"ье ":4950,"ые ":43888,"ыл ":8892,"ым ":27422,"ый ":115051,"ычн":4405,"ÑŒÑ ":5879,"ью ":17235,"ьбо":5724,"ьев":4427,"ьер":4030,"ьм ":7924,"ыка":9496,"ыла":5382,"ыми":11026,"ыпу":5527,"Ñ‹Ñо":4693,"ённ":14954,"er ":6599,"es ":4533,"Ñза":3917,"Ñет":23887,"Ñзы":12445,"Ñвл":14387,"Ñбр":17050,"ÑÑ… ":9267,"ÑŽÑ‚Ñ":8148,"юте":4109,"юща":7138,"ющи":22555,"юще":8725,"юча":3971,"юлÑ":5121,"ÑÑ‚ ":7009,"юнÑ":4749,"ÑÑŽÑ‚":4520,"Ñющ":7517,"Ñще":6980,"Ñщи":4896,"ÑнÑ":6220,"Ñнв":5611,"Ñни":5071,"Ñми":7921,"Ñрн":3999,"Ñто":4282,"Ñти":7069,"Ñте":9306,"уще":14055,"уча":15033,"учн":4074,"учи":4977,"уче":7765,"феÑ":5738,"фев":5319,"фер":4649,"ующ":10346,"фин":4428,"физ":4024,"фил":15705,"фик":3811,"фиц":4922,"фре":4266,"фра":6501,"фор":17745,"фон":4486,"ца ":19779,"це ":5162,"хан":4526,"хар":5112,"хра":4915,"хно":4200,"хни":3959,"хож":3747,"хов":4677,"ход":41602,"цы ":7957,"худ":5932,"ÑÑк":14412,"ÑÑи":32096,"ÑÑо":7973,"ÑÑа":7005,"ÑÑе":5970,"Ñтн":20518,"Ñто":67851,"ÑÑ‚Ñ€":70127,"Ñтв":108223,"Ñте":31686,"Ñти":71515,"Ñта":85694,"ÑÑÑ‚":5885,"Ñуд":15247,"ÑÑ‚ÑŒ":44809,"ÑÑ‚Ñ‹":4236,"Ñту":9860,"Ñущ":7215,"Ñхо":7227,"Ñ‚Ñ‹ ":16675,"Ñ‚ÑŒ ":55254,"тав":37473,"так":23239,"тал":22250,"там":8524,"тан":36504,"тай":3709,"тат":27244,"уг ":6960,"тар":13509,"тбо":4051,"тву":7862,"ÑÑ‘Ñ€":3801,"тво":31741,"тви":10153,"тве":41916,"тва":21790,"тех":6332,"тем":17566,"тел":76816,"тео":3852,"тен":11539,"тер":49417,"теп":3783,"тет":16644,"тек":8246,"тей":6713,"тив":26421,"тие":8972,"ук ":4128,"тка":5169,"тич":24503,"тиÑ":7560,"тии":4630,"тий":5054,"тин":12258,"тик":14625,"тил":7099,"тир":5823,"тиÑ":5149,"тип":5281,"тит":7029,"тки":3943,"тно":26382,"ток":11951,"тол":12417,"той":7374,"тны":17802,"тов":29921,"тог":9531,"тни":13284,"тна":5384,"ут ":4381,"тре":17053,"тра":54933,"три":18286,"тор":95430,"том":20153,"тон":9780,"ÑƒÑ ":4688,"точ":11868,"тоÑ":12927,"Ñ‚ÑÑ‚":5913,"Ñ‚ÑÑ":53866,"тро":41481,"тру":10653,"Ñ‚Ñк":19176,"туг":4888,"туп":6027,"тур":21906,"тью":9623,"тый":5760,"ую ":18777,"уго":4888,"уги":7037,"уга":8925,"уда":16459,"Ñ‚Ñб":11568,"убл":8850,"убе":4682,"узы":8785,"узÑ":4917,"уже":5996,"ует":9543,"уем":3815,"уди":4265,"удо":10125,"уме":6894,"уль":15447,"улÑ":5536,"ули":11772,"укт":4529,"уко":4826,"ука":4047,"упп":14974,"упн":6779,"упр":5417,"ура":6947,"ург":8767,"уре":4620,"унк":5527,"уна":6858,"уни":18074,"уÑÑ‚":16092,"уÑÑ":16109,"ута":4019,"уры":4919,"урн":11401,"уро":5529,"уÑк":4624,"ших":7365,"шир":4576,"шин":5562,"ший":10857,"шен":11201,"шаÑ":6983,"шта":14038,"щих":9037,"щие":5309,"щий":14604,"щин":3869,"щее":7976,"щей":4091,"щег":3871,"щен":12451,"щеÑ":17297,"щаÑ":10075,"щад":4691,"on ":6654,"цен":15360,"чи ":4183,"цел":4906,"цев":3998,"цеÑ":4907,"цер":5569,"циа":13419,"ций":4974,"ции":29185,"цие":3800,"цип":12774,"цио":18812,"циÑ":20642,"ча ":3694,"цар":3955,"цуз":4865,"цко":4797,"цки":5412,"чем":5556,"чен":25493,"чел":10826,"чеÑ":78218,"чер":6254,"чет":6065,"чле":3904,"чин":7894,"чив":3802,"чиÑ":7524,"чит":8014,"ше ":5808,"чаю":4561,"чаÑ":36891,"чат":3981,"чал":9929,"чае":6846,"чны":14169,"чно":19736,"что":8092,"tio":4098,"Кар":6020,"Кра":3888,"Кор":4309,"Кон":5008,"Мар":7555,"Мин":4281,"МоÑ":11770,"ÐаÑ":4395,"Ðик":4718,"Ðов":5069,"Оли":6672,"Пар":6298,"СР ":8410,"Пет":6304,"Пер":7246,"Ðле":6898,"Бра":9561,"Бол":4163,"Бел":4463,"Вел":5787,"Гра":3978,"Гер":5952,"Джо":3835,"Евр":4076},"n_words":[36763344,40893832,29165701],"name":"ru","type":"cyrillic"} \ No newline at end of file
diff --git a/contrib/languages-data/sl.json b/contrib/languages-data/sl.json
new file mode 100644
index 0000000..4131d88
--- /dev/null
+++ b/contrib/languages-data/sl.json
@@ -0,0 +1 @@
+{"freq":{"D":7697,"E":4326,"F":7483,"G":7482,"A":12478,"B":11289,"C":8095,"L":10635,"M":14480,"N":8171,"O":6564,"H":9046,"I":7126,"J":7268,"K":11757,"U":2722,"T":7824,"W":1905,"V":10234,"Q":288,"P":16672,"S":26591,"R":9751,"Y":433,"X":356,"Z":6077,"f":21271,"g":83131,"d":174445,"e":575804,"b":82172,"Fed":58,"c":56405,"a":599039,"n":410604,"o":476628,"l":247955,"m":151651,"j":244223,"k":242702,"Fel":92,"h":59332,"i":522865,"w":2524,"v":222867,"Fer":370,"u":129396,"t":241091,"s":268170,"r":314054,"q":489,"p":164089,"z":94676,"y":6418,"x":1284,"²":123,"ÃŽ":185,"É":129,"Ã":62,"Fil":230,"í":2013,"Fin":151,"ë":81,"ê":309,"é":3055,"Fir":82,"è":584,"ç":174,"ä":365,"â":114,"á":3560,"à":133,"ü":557,"ú":870,"ø":79,"ö":488,"ô":419,"ò":234,"ó":1670,"Ä“":62,"Ä‘":109,"Ä":87,"Ä":111,"ć":1063,"ÄŒ":1470,"Ä":56247,"Å•":73,"Å‘":245,"Å‚":69,"Å":86,"Ž":2008,"ž":30614,"Å ":2528,"Å¡":51762,"Fak":387,"Fal":70,"Far":111,"Eri":80,"Est":69,"Eti":62,"Ern":82,"Eur":142,"Eva":78,"Evr":794,"Ein":58,"Ì":345,"Ele":144,"Eko":86,"μ":124,"ν":240,"Ena":60,"ο":342,"ι":214,"κ":105,"λ":169,"δ":61,"ε":131,"η":95,"α":330,"β":59,"γ":116,"ά":79,"ί":88,"Emi":63,"Eli":70,"ÏŒ":99,"σ":119,"Ï‚":264,"Ï":222,"Ï€":98,"φ":61,"Ï…":78,"Ï„":148," l":19333,"ÑŒ":113," m":28795," n":57198,"Ñ":125," o":41952," h":4815,"ш":89," i":55210," j":67383," k":54205,"Ñ‹":63," d":42036,"Ñ„":78,"Ñ…":75," e":9710,"ц":79," f":9447," g":14912,"ч":253,"Ñ€":642,"Ñ":417," a":19069," b":18884,"Ñ‚":428," c":4312,"у":369," y":127," x":151," z":31775," u":19039," t":27202," w":136," v":56888," p":90919," s":76633," r":20918,"HK ":93,"И":69,"К":110,"Ð":65,"Ðœ":132,"П":82,"Б":103,"Ð":111,"Г":80,"Ð’":98,"Д":60," J":7259," K":11717," H":8994," I":7097," N":8139," O":6489," L":10542," M":14428," B":11242," C":8027,"С":106," A":12432," F":7459," G":7430," D":7664," E":4306,"л":503,"к":526," Z":6067,"й":182," Y":432," X":347,"и":1084,"п":111,"о":912,"н":714,"м":224,"г":177," S":26492," R":9719,"Ger":142,"в":657," Q":287,"б":103," P":16617,"а":1195,"з":72," W":1888,"Geo":282," V":10198,"Gen":320," U":2704,"е":796,"д":264," T":7802," Ä":5716," ÄŒ":1467," Ä":87,"Gla":255,"Gia":71,"HL ":80," Ã":62,"Gio":131," É":127," ÃŽ":185,"Gir":61,"Giu":113," ž":7506," Ž":2008," Å ":2526," Å¡":7903,"Õ¡":58,"Gan":63,"Gal":184,"Gam":76,"Gar":174,"Gab":108,"Ùˆ":76,"ÙŠ":148,"Ù„":189,"Ù…":127,"Ù†":117,"Fun":63,"د":71,"ب":101,"ا":258,"ر":124,"Flo":89,"Fla":62," Ð":111," Б":103," Ð’":98," Г":71,"Fra":1714," Д":60," И":69," К":110,"Fri":169," Ðœ":132," Ð":65,"A ":1521," П":82,"Fre":167,"For":1812," α":63,"F ":287,"Da":1043,"Cu":190,"Cv":64,"Cy":61,"Cl":275,"Co":1643,"Cr":347,"Ce":1243,"Ch":1233,"Ci":465,"G ":285,"Ed":331,"Ea":63,"Dv":143,"Du":856,"Do":1622,"Dr":1054,"De":1300,"Di":918,"Fe":729,"H ":307,"Fa":834,"Eu":245,"Ev":1000,"Er":320,"Et":115,"Es":220,"En":413,"Em":164,"Ep":63,"Ei":124,"El":403,"Ek":172,"Eg":130,"Ge":887,"Ga":902,"I ":771,"Fu":219,"Fr":2141,"Fo":2047,"Fl":225,"Fi":681,"B ":327," С":104,"II ":321,"C ":846,"Av":1043,"Au":489,"Ar":1294,"At":316,"As":365,"D ":417,"Ba":1844,"Az":412,"Af":338,"Ag":178,"Ah":87,"Ab":563,"Ac":172,"Ad":372,"Am":987,"An":1820,"Ap":398,"Ai":256,"Aj":81,"Ak":264,"Al":2250,"Hit":92,"Bu":797,"Br":2247,"Ca":1535,"E ":345,"Bi":1004,"Hid":63,"Be":2195,"Bo":1910,"Hil":81,"Bl":438,"Bj":272,"Hip":72,"Kv":109,"Ku":509,"Kn":259,"IE ":65,"Kl":445,"Kr":2275,"Ko":3816,"Le":3189,"Lj":1723,"Li":1298,"N ":264,"La":1624,"Lu":757,"Ly":76,"Lo":1512,"Me":2219,"Dž":96,"Mi":1869,"Ml":211,"O ":680,"Ma":6025,"Mc":133,"Mu":580,"Mr":101,"Mo":2460,"Nj":399,"Ni":828,"Já":138,"Ne":1598,"Na":2980,"P ":622,"Hel":136,"Ny":85,"Hei":136,"Nu":95,"No":1554,"Ok":258,"Ol":321,"Om":113,"On":187,"Og":200,"Oh":58,"Oc":80,"Od":1568,"Hen":142,"Her":380,"Ob":922,"Gi":553,"Gl":510,"Gr":1909,"Go":1659,"Gu":367,"Gv":83,"Gy":129,"Cô":91,"J ":297,"Ha":1563,"He":1085,"Hi":543,"Ho":701,"Hr":4064,"Hu":331,"K ":438,"Ib":78,"Id":108,"Ig":180,"Im":513,"In":1624,"Il":230,"Iv":415,"Is":1036,"It":703,"Ir":277,"Ja":1804,"L ":443,"Iz":883,"Ji":68,"Je":1541,"Jo":1290,"Hab":58,"Ju":2001,"Hal":126,"Haj":64,"Ka":2508,"Han":157,"M ":389,"Ham":119,"Har":293,"Ki":622,"GyÅ‘":92,"Hau":402,"Ke":544,"Us":272,"Ur":262,"Up":188,"Um":80,"Un":907,"Uk":147,"Ul":103,"W ":109,"Ty":77,"Tu":672,"Tr":1582,"To":1468,"Th":691,"Ti":704,"Te":1011,"Ta":1070,"V ":1619,"Côt":91,"Sw":66,"Sz":701,"Sy":147,"St":3308,"Sv":2528,"Su":652,"Wo":195,"Wi":724,"Wh":64,"Wa":405,"We":272,"Vz":150,"Vo":999,"Vr":488,"Vs":193,"Vu":139,"Vi":1620,"Vl":200,"X ":215,"Va":913,"Ve":3902,"Má":81,"Pt":177,"Pu":321,"Pr":3992,"S ":1529,"Pe":1571,"Pa":2197,"Gui":73,"Lé":59,"Pl":789,"Po":5526,"Pi":1027,"Ph":198,"Os":738,"Ot":263," ا":112,"Op":322,"Or":646,"R ":294,"Oz":78,"Se":2093,"Sc":592,"Si":1170,"Sh":254,"Sn":82,"Sm":234,"Sl":6573,"Sk":484,"Sr":839,"Sp":1374,"So":1617,"Ru":971,"GrÄ":81,"U ":245,"Sa":2421,"Re":3735,"Rd":107,"Ri":922,"Rh":128,"Ro":1660,"Qu":211,"T ":330,"Ra":1534,"Mü":85,"Gre":332,"Gri":94,"Gra":810,"Grb":113,"b ":3960,"Gru":126,"Gro":222,"a ":211856,"Yo":229,"Gle":73,"Z ":240,"Glo":95,"Gol":160,"Gor":907,"Gos":120,"Za":1749,"Zd":593,"Ze":881,"Zi":330,"Zg":337,"Vé":82,"Zm":60,"Zl":152,"Zo":102,"Zn":141,"Zu":99,"God":61,"Zr":80,"Zv":306,"i ":134220,"gd":128,"ge":5872,"ga":24650,"fj":134,"Inf":332,"fl":264,"ff":331,"fi":4842,"fs":419,"fr":3897,"aÄ":5802,"fu":678,"ft":369,"fo":2045,"Int":382,"fn":91,"j ":12717,"gy":387,"dá":73,"he":3525,"ha":5343,"gn":1368,"gm":166,"gl":6996,"gi":8855,"gh":503,"gg":192,"gv":88,"gu":2562,"gt":124,"gs":203,"bÄ":5121,"gr":10356,"cí":78,"go":15666,"dt":312,"du":5066,"dv":5016,"dw":173,"dy":144,"dz":369,"g ":4531,"Ima":116,"ea":2500,"eb":7309,"ec":6168,"ed":29381,"de":25427,"Ili":79,"dd":738,"dg":429,"di":24057,"dh":181,"dk":1177,"dj":1456,"dm":1512,"dl":1112,"do":16087,"dn":14568,"dp":1050,"ds":3632,"dr":13867,"ew":457,"ex":237,"eu":1040,"ev":19283,"ey":856,"ez":12153,"fa":2439,"h ":28964,"aú":301,"Ind":547,"fe":2538,"bá":91,"eh":4032,"eg":21346,"ef":1414,"ee":651,"Ime":316,"el":42176,"ek":16159,"ej":9407,"ei":1868,"ep":9382,"eo":3145,"en":61076,"em":38275,"et":32921,"es":22879,"er":40015,"ca":8624,"bz":65,"e ":176361,"bv":197,"by":89,"bs":2013,"br":6519,"bu":2540,"bt":104,"bn":3084,"bo":9119,"bj":937,"bk":143,"cL":77,"bl":11842,"bm":547,"bh":211,"bi":17028,"bb":158,"bd":603,"be":10505,"dc":181,"db":1176,"da":28148,"f ":2764,"cz":61,"cy":123,"cv":189,"cu":951,"ct":549,"cs":478,"cq":69,"cr":248,"co":5124,"cm":121,"cn":95,"ck":1329,"cl":245,"ci":15112,"ch":3183,"ce":12343,"cc":353,"c ":6745,"az":10124,"ay":618,"ba":6779,"d ":25792,"at":30196,"as":24786,"ar":39381,"ax":151,"aw":283,"av":30027,"au":2021,"ak":12407,"al":45711,"ai":1701,"aj":23033,"ao":367,"ap":6069,"am":13386,"an":71011,"ac":6592,"ad":26926,"aa":226,"ab":7824,"ag":7152,"ah":7624,"ae":1031,"af":1621,"nu":5151,"nt":10999,"ns":25512,"iÄ":13659,"nr":295,"np":329,"no":51231,"nn":1495,"nz":984,"ny":526,"nw":82,"jó":58,"nv":255,"oe":569,"ká":135,"of":3946,"oc":3247,"od":39028,"oa":584,"ob":20785,"om":19239,"on":23036,"ok":14837,"ol":27351,"oi":2548,"oj":11831,"og":11726,"oh":1523,"m²":117,"ot":18168,"os":30813,"ov":55859,"ou":2158,"op":10262,"oo":1164,"or":35498,"jÄ":131,"r ":20683,"ox":97,"kó":141,"ow":586,"kö":75,"oz":8175,"oy":187,"pd":65,"lá":269,"pe":10198,"pa":26706,"Igr":74,"pc":146,"pl":6471,"lé":474,"pn":2144,"po":53400,"ph":657,"pi":10229,"pj":64,"pk":185,"lo":38721,"ln":9297,"lm":1635,"hé":133,"ll":3272,"ls":3383,"lp":1148,"lv":458,"lu":4829,"lt":2975,"lz":195,"ly":422,"hô":58,"Idr":67,"o ":117173,"mc":91,"md":104,"ma":23441,"mb":4609,"dž":1661,"iá":58,"me":39489,"mf":152,"mk":917,"iè":113,"ml":1947,"mi":12739,"eÅ¡":4276,"mj":85,"mn":2345,"mm":463,"mp":2668,"mo":14761,"hÄ":66,"mr":915,"mt":95,"ms":4458,"mv":69,"mu":5539,"my":115,"p ":1343,"na":91417,"nb":277,"nc":8502,"nd":6315,"ne":43250,"já":118,"nf":942,"ež":5003,"ng":4878,"nh":345,"ni":72174,"nj":24738,"nk":3254,"ić":821,"nl":278,"nm":267,"jv":1409,"jt":127,"ju":14494,"eÄ":5728,"js":10383,"jp":458,"jn":4767,"jo":15551,"jl":168,"jm":397,"jk":398,"kj":752,"ki":56467,"kh":121,"ke":31686,"kd":477,"kc":1346,"ka":42561,"fü":98,"m ":32866,"jz":162,"ky":130,"ks":2035,"kt":5285,"ku":8273,"kv":2313,"ko":52366,"gí":90,"kr":10395,"kk":78,"kl":5176,"km":1749,"gé":62,"kn":1425,"dÅ¡":398,"li":49096,"lh":245,"lk":2648,"lj":32952,"le":34235,"há":163,"ld":976,"lg":1256,"lf":539,"hâ":70,"la":37335,"lc":2469,"lb":823,"n ":53330,"hr":1878,"hs":96,"dí":74,"hw":70,"ht":692,"hu":981,"hj":61,"hk":1523,"hi":3365,"hn":1375,"ho":9636,"hl":379,"dè":58,"dé":139,"hm":293,"id":5268,"ic":16189,"ib":3404,"ia":4880,"ih":24332,"ig":5375,"if":1125,"ie":2994,"hy":222,"dú":72,"k ":19129,"iq":110,"ir":15616,"dÄ":73,"is":20587,"it":17935,"iu":664,"iv":10504,"ix":194,"aÅ¡":9037,"ii":2454,"ij":44668,"ik":30544,"il":26890,"im":25029,"in":69797,"io":5253,"ip":4264,"jc":271,"jb":886,"je":114329,"jd":485,"až":1346,"bÅ¡":153,"ji":18381,"jh":438,"iz":20523,"l ":16581,"ja":44343,"tä":161,"pÅ¡":106,"xi":105,"té":252,"tí":147,"tó":164,"ww":69,"z ":12173,"xa":126,"ož":4663,"tá":178,"nž":333,"wi":424,"oÅ¡":3587,"sé":150,"wn":164,"wo":138,"sí":94,"rÄ":573,"ws":163,"ró":226,"rô":65,"vz":2935,"y ":2958,"wa":490,"sá":124,"we":348,"rè":76,"vl":5625,"vm":123,"ré":314,"vj":1191,"vk":1293,"vh":61,"nÅ¡":866,"vi":26446,"vg":688,"vt":1201,"vu":1200,"vr":6944,"vs":7139,"vp":572,"rí":254,"vn":18643,"vo":20029,"uz":834,"ux":225,"uv":1705,"uu":61,"ve":45929,"rá":407,"vd":234,"vc":816,"vb":122,"va":33280,"x ":755,"mÅ¡":2061,"ui":868,"uj":5428,"uk":2181,"ul":7777,"ue":1036,"uf":172,"ug":6200,"lž":478,"uh":1992,"ur":10254,"pÄ":78,"us":9210,"ut":2587,"um":3595,"un":5703,"uo":144,"up":17758,"ty":284,"tz":304,"tu":11745,"tt":1249,"tw":125,"pó":70,"tv":10431,"ub":7050,"ua":3183,"ud":8270,"uc":1029,"w ":442,"to":33340,"tn":10733,"pé":68,"tm":2491,"tl":2152,"ts":2917,"oÄ":7724,"tr":18852,"tp":111,"tg":72,"tf":115,"te":38423,"pá":79,"td":278,"tk":3275,"tj":2887,"lÅ¡":176,"ti":34585,"th":1742,"v ":46495,"tc":114,"ta":42943,"su":2438,"sv":4904,"ss":1521,"st":64846,"sy":100,"sz":642,"sw":95,"sl":11373,"sk":60513,"sn":5408,"sm":2123,"sp":12768,"so":15794,"nÄ":1620,"sr":3075,"sd":114,"sc":1363,"sf":304,"se":40074,"sh":762,"sj":389,"kÅ¡":233,"si":9098,"rz":2218,"u ":23884,"sa":10107,"sb":1404,"mÄ":427,"rr":991,"rs":12425,"rt":8497,"ru":11754,"rv":10562,"rw":89,"nó":68,"ry":877,"ní":142,"rp":731,"ro":31529,"rn":11113,"né":172,"rm":4944,"rl":1460,"rk":7297,"nç":122,"rj":7769,"jÅ¡":3373,"ri":44436,"rh":1257,"rg":4613,"iž":2554,"rf":351,"ná":202,"re":43763,"rd":5068,"rc":1442,"rb":2537,"ra":68896,"t ":20636,"qu":409,"mí":90,"lÄ":258,"mé":211,"iÅ¡":12414,"má":431,"s ":18299,"px":59,"pt":1806,"pu":4524,"ló":186,"pp":444,"lí":193,"pr":43317,"ps":1569,"Hum":81,"yÅ‘":97,"vž":60,"zá":92,"už":4277,"vÅ¡":416,"uÅ¡":1692,"Hrv":3876,"yí":60,"Hra":77,"rž":2418,"zz":215,"vÄ":549,"zg":2406,"Hor":93,"zh":2141,"zi":10015,"rÅ¡":3989,"zb":1418,"zc":70,"zd":3868,"ze":5649,"vá":315,"za":21344,"Hon":64,"Hok":73,"Hol":102,"zv":5632,"zs":530,"zr":2612,"uÄ":1996,"zu":2176,"zt":382,"zo":6065,"zn":9166,"ví":186,"zp":931,"zk":568,"zj":291,"zm":3200,"vé":151,"zl":2570,"ye":263,"yc":118,"yd":123,"ya":395,"yt":90,"ys":299,"Hoc":85,"yr":276,"yp":87,"yo":155,"yn":206,"ym":124,"yl":250,"yk":172,"yj":189,"yi":151,"Arg":159,"Arh":66,"Are":64,"Ard":164,"Å¡Äu":315,"Ara":107,"Arm":203,"Å¡Äo":81,"Å¡Än":64,"Ari":101,"Å¡Äi":1771,"Apo":252,"Å¡Äe":2731,"Å¡Äa":1225,"Atl":156,"Ast":131,"Ass":80,"Art":149,"Avt":105,"Avs":709,"Ave":81,"Auv":73,"Aut":64,"Aug":99,"zÅ¡":796,"Azi":191,"Azu":144,"Bak":78,"Bal":229,"Ban":193,"Bab":79,"Bad":69,"Bar":430,"Bat":109,"Bas":111,"Bav":61,"Aba":278,"Ada":80,"Ado":61,"Afr":271,"Air":92,"Al ":78,"šće":104,"Aka":59,"Akv":95,"Ala":127,"Alb":257,"Ali":77,"Ale":389,"Alf":117,"Alt":76,"All":122,"Alo":87,"Alp":662,"Ame":656,"Ama":87,"Ang":426,"Ana":133,"And":409,"Å¡Ä ":144,"Ant":584,"Ann":63,"Buz":68,"Buk":74,"Bur":210,"Bud":88,"Bru":110,"Bož":77,"² ":120,"DA ":787,"DD ":66,"Cal":180,"Cam":157,"Cas":187,"Car":397,"Cat":66,"Can":211,"Cap":85,"Bea":66,"Bes":153,"Ber":630,"Beo":125,"Ben":301,"Bel":559,"Biz":86,"Bje":262,"Bil":214,"Bis":245,"Bit":75,"Bio":71,"Blo":64,"Ble":72,"Bla":204,"Bre":545,"Bra":506,"Bro":313,"Bri":606,"Boh":133,"Bog":157,"Bol":225,"Boj":60,"Bon":91,"Bor":510,"Bos":176,"Bou":105,"ÃŽl":184,"Der":74,"Des":135,"Dev":75,"Dek":70,"Del":334,"Dem":64,"Den":114,"Deb":67,"Dam":75,"Dan":339,"Dar":97,"Dav":176,"Dal":121,"Chr":147,"Che":154,"Chi":125,"ám":76,"án":655,"Cit":86,"áj":126,"Cir":117,"ák":172,"ál":309,"ác":115,"ád":136,"áz":116,"áv":123,"ár":751,"át":278,"ás":315,"ât":67,"Châ":62,"Cla":118,"Cel":355,"Cen":243,"Cer":450,"Ces":88,"à ":58,"á ":69,"Cha":633,"Cre":137,"Cor":603,"Com":262,"Col":181,"Con":211,"Cou":99,"ós":120,"ót":110,"óv":101,"ôm":59,"ôn":89,"DuÅ¡":60,"ód":99,"ór":148,"ón":273,"óm":66,"ól":124,"ók":78,"ó ":118,"Drž":130,"ív":88,"íz":94,"ín":324,"ír":134,"ít":126,"ís":96,"ík":80,"íl":246,"íj":224,"íd":88,"ía":72,"Egi":80,"ên":58,"êr":99,"éz":94,"ék":134,"él":241,"éj":87,"ém":138,"én":587,"és":167,"ét":261,"ér":344,"év":100,"éd":144,"ée":75,"Edw":93,"èn":67,"èr":126,"ço":94,"é ":306,"ät":162,"Do ":87,"ć ":449,"Dia":61,"Dic":151,"Dis":100,"Dir":87,"Dio":72,"Din":68,"Die":63,"Div":106,"Dub":388,"Dun":175,"ün":108,"ür":209,"Dvo":99,"Dru":220,"ún":88,"új":321,"úr":86,"Dre":102,"Dra":452,"íš":61,"Dob":200,"ôt":129,"ör":97,"Dou":72,"Dol":339,"Don":353,"Dom":230,"Dor":129,"Ned":79,"Nea":69,"Nem":513,"Nek":87,"Nev":71,"Neu":84,"Jás":105,"Nep":74,"Nas":351,"Nat":199,"Nav":148,"Nic":123,"Niz":179,"Nik":208,"OJ ":106,"New":248,"Nap":119,"Nar":283,"Nam":213,"Nan":68,"Nag":201,"Nah":254,"Naj":294,"Nad":104,"Na ":449,"ći":324,"OV ":124,"ća":95,"će":161,"ÄŒi":127,"ÄŒe":527,"ÄŒa":315,"ÄŒu":69,"ÄŒr":316,"Äi":13203,"Äj":3069,"Äk":2033,"Äl":1507,"Äe":9878,"Äa":7103,"Äb":207,"Ä ":3958,"Ä‘e":65,"Äu":65,"Än":11436,"Äo":315,"Är":1069,"Äu":2190,"Äv":95,"Nji":74,"Nje":320,"Nov":818,"Nor":440,"Not":61,"Odv":1174,"Ogr":131,"ObÄ":247,"PL ":89,"Okr":139,"Nyí":60,"Obs":170,"Obi":89,"Obr":75,"Obo":60,"Od ":141,"ÃŽle":184,"Oto":151,"Oli":186,"Ont":118,"Ope":134,"Ore":118,"Org":93,"Ost":65,"Osj":214,"Osm":58,"Osn":93,"Po ":423,"Å¡ ":428,"Pli":67,"Ple":143,"Pla":466,"Pin":72,"Pik":79,"Pit":66,"Pis":66,"Pir":250,"Pie":165,"Phi":96,"Ped":76,"Per":309,"Pes":215,"Pet":518,"Pen":134,"Pel":66,"Å¡Ä":6357,"šć":151,"Pat":125,"Pas":151,"Par":669,"Pav":183,"Pau":156,"Pad":61,"Pac":78,"Pan":157,"Pap":82,"Pal":206,"Pak":100,"Å¡e":3798,"Å¡a":2845,"Å¡o":760,"Å¡p":1066,"Å¡n":1922,"Å¡k":22840,"Å¡l":713,"Å¡i":4197,"Å¡j":684,"Å¡v":183,"Å¡u":178,"Å¡t":5493,"Å e":283,"Å a":280,"Å m":149,"Å o":105,"Å i":182,"Å k":346,"Å u":75,"Å t":406,"Å v":259,"Å p":321,"Pož":327,"Ptu":112,"Pro":707,"Pri":1370,"Pre":1114,"Prv":307,"Pru":73,"Å‘r":96,"Pra":329,"Pod":622,"Pok":159,"Pol":1336,"Pom":102,"Pon":195,"Pog":199,"Poi":167,"Poj":93,"Pot":652,"Pos":334,"Pov":181,"Pop":91,"Por":385,"Poz":75,"žr":68,"žu":4511,"žn":3757,"žo":181,"žc":62,"žb":1014,"že":6604,"žd":91,"ža":4974,"žk":205,"žj":1185,"žl":145,"žg":130,"ži":6432,"Žu":1147,"Ža":131,"Ži":199,"Že":372,"RS ":268," ال":88,"ž ":1046,"Rac":68,"Rad":357,"Ram":58,"Mün":75,"Ran":79,"Rak":65,"Que":120,"Irs":68,"Ita":679,"Isl":158,"Ist":728,"Ira":123,"InÅ¡":104,"Iva":381,"Izv":187,"Izr":254,"Izd":68,"Jac":161,"Jad":111,"Jav":92,"Jar":108,"Jap":208,"Jan":529,"Jam":191,"Jak":171,"Jel":130,"Jer":198,"Jes":136,"Jez":169,"Jea":145,"IzÅ¡":73,"Je ":506,"Jos":354,"Jor":75,"Jon":77,"Joh":406,"Jug":908,"Jud":70,"Jup":160,"Jur":234,"Jul":171,"Jož":199,"LA ":59,"Juž":280,"Kam":278,"Kal":271,"Kap":166,"Kan":372,"Kat":241,"Kas":83,"Kar":544,"Kaz":101,"Kav":68,"Ker":149,"Ken":111,"Kis":102,"Kir":77,"Kit":180,"Kin":85,"Klo":59,"Kli":73,"Kle":103,"Kla":125,"Kon":517,"Kom":582,"Kol":204,"Kos":201,"Kor":867,"Kop":552,"Kov":80,"Kot":149,"Koz":113,"Knj":131,"Kob":74,"Koc":70,"Kre":107,"Kra":1273,"Kri":388,"Krk":87,"Kro":118,"KrÅ¡":94,"KoÅ¡":77,"KoÄ":102,"Kul":64,"Kun":76,"Kur":69,"Kva":82,"Lev":100,"Let":1929,"Les":99,"Leo":137,"Len":143,"Lau":76,"Law":92,"Le ":107,"Lag":58,"Lah":83,"Las":91,"Lat":85,"Lar":113,"Lam":67,"Lan":280,"Lab":97,"La ":179,"Lju":1716,"Lib":144,"Lig":77,"Lim":128,"Lin":177,"Lip":125,"Lit":222,"Luk":154,"Lui":81,"Lun":67,"Lud":105,"Luc":114,"Lou":148,"Lov":122,"Los":65,"Lot":115,"MS ":61,"Loi":123,"Log":104,"Lor":123,"Lon":188,"Lok":174,"LiÄ":69,"Lež":206,"Meh":106,"Men":114,"Mel":123,"Mes":284,"Mer":173,"Met":220,"Med":836,"MaÄ":73,"Mez":77,"Man":399,"Mal":462,"Mar":2349,"Mas":191,"Mag":152,"Mad":1011,"Maj":111,"Mak":235,"Mai":69,"Mac":176,"McL":66,"Max":58,"Mau":68,"Mat":387,"Mla":152,"Mod":111,"Moh":80,"Moj":64,"Mol":94,"Mon":601,"Mos":493,"Mor":297,"Mou":64,"Mot":314,"Mih":227,"Mik":175,"Mic":218,"Mit":62,"Mir":154,"Mis":119,"Mil":386,"Min":343,"NK ":115,"Mur":270,"Mus":87,"MoÅ¡":81,"çoi":67,"Wor":66,"Wol":82,"Wik":71,"Wil":329,"Win":160,"ère":90,"Wes":77,"War":73,"Wal":129,"Vzh":106,"Vse":94,"Vrb":98,"Vra":69,"Vrh":141,"Vol":145,"Voj":511,"Vod":126,"ViÅ¡":166,"VeÄ":124,"Vis":126,"Vit":109,"Vla":188,"Zla":131,"Äuj":826,"Äun":696,"Vél":58,"Äut":108,"Ärk":423,"Äre":113,"Ärt":288,"éte":58,"Ärn":193,"Äu ":419,"Zna":123,"Zdr":568,"ény":95,"Zap":71,"Zar":126,"Zas":101,"Zav":104,"Zag":314,"Zah":111,"Zak":109,"Zal":123,"ékt":72,"én ":269,"éli":106,"Zgr":67,"Zgo":249,"éra":62,"Zim":226,"Zel":179,"Zem":564,"ов ":64,"ZaÄ":142,"之":72,"三":107,"Zad":119,"Za ":213,"Yor":161,"на ":82,"Szo":98,"Sza":480,"Sys":60,"Sve":2328,"Sup":72,"Sud":142,"Str":416,"Stu":150,"Sti":77,"Sto":225,"Sta":2102,"Ste":244,"Teh":61,"Ten":108,"Tem":109,"Teo":88,"Tel":99,"Tek":83,"Tam":78,"Tan":81,"Tar":117,"Tak":138,"Tal":65,"Ta ":195,"Sko":62,"Skr":66,"Sku":204,"Ska":62,"Sha":82,"Sim":142,"Sil":93,"Sis":205,"Sir":188,"Sin":157,"Sib":59,"Sez":419,"Ses":154,"Ser":176,"Sev":340,"Sen":145,"Sel":241,"Sem":91,"Sei":131,"Sed":75,"Srb":301,"Sre":466,"TV ":116,"Sv ":98,"Spa":211,"Spl":481,"Spi":65,"Spe":164,"Spr":96,"Spo":313,"Sod":63,"Sok":63,"Soc":117,"Sob":124,"Sou":80,"Sov":270,"Sol":119,"Som":85,"Son":199,"Sop":149,"Sor":59,"Sla":201,"TO ":84,"Slo":6226,"Sli":73,"So ":69,"Rož":72,"Rus":577,"Rud":148,"Sai":290,"Sam":204,"Sal":223,"Sad":71,"Sco":63,"Sch":394,"Sav":417,"Sat":60,"Sau":74,"Sar":260,"San":495,"ови":138,"RaÄ":75,"SI ":76,"Res":82,"Rev":59,"нов":79,"Rim":348,"Rib":92,"Ric":128,"ät ":150,"Ras":66,"Rav":91,"Raz":271,"Rde":102,"SG ":79,"Rec":63,"Red":122,"Rei":91,"Reg":444,"Ren":132,"Rek":120,"Rep":2306,"Rog":117,"Rob":196,"Roc":78,"Rod":85,"Rou":102,"Ros":172,"Ron":234,"Rom":237,"SS ":448,"SO ":89,"Vel":2994,"Ven":157,"Ñки":72,"Vas":75,"Van":123,"Val":340,"Var":221,"Vid":128,"Vic":103,"Vie":59,"Vir":205,"Vil":211,"Vik":75,"Vin":206,"Ver":276,"Ves":184,"Ukr":102,"Uni":842,"Ura":80,"Ust":217,"Upo":141,"Trž":77,"Ter":179,"The":362,"Tho":193,"Tih":67,"Tim":73,"Tis":131,"Tir":83,"To ":334,"Top":167,"Tor":193,"Tok":64,"Tol":157,"Tom":179,"Ton":82,"Tou":87,"Tru":58,"Trs":127,"Tro":210,"Trn":82,"Tri":282,"Trg":65,"Tre":271,"Tra":244,"Tur":330,"Tuk":62,"Å¡i ":887,"Å¡ev":379,"Å¡em":193,"Å¡el":270,"Å¡en":350,"Å¡es":555,"Å¡er":80,"Å¡eg":118,"Å¡ek":215,"Å¡a ":1145,"Å¡e ":1531,"Å¡ar":175,"Å¡av":187,"Å¡ah":216,"Å¡aj":99,"Å¡al":222,"Å¡an":614,"Å ve":109,"Å vi":128,"Å¡to":95,"Å¡tr":65,"Å¡te":2122,"Å¡ti":888,"Å¡ta":268,"Å¡uj":72,"Å¡tv":1659,"Å¡tu":262,"Å¡vi":77,"Å¡ve":96,"Å¡pa":358,"Å¡ov":61,"Å¡po":639,"Å¡t ":64,"Å¡u ":58,"Å¡ko":7029,"Å¡lj":265,"Å¡la":312,"вич":166,"Å¡o ":156,"Å¡ić":67,"Å¡ni":492,"Å¡nj":513,"Å¡ne":308,"Å¡na":327,"Å¡iÄ":190,"Å¡no":281,"Å¡ol":499,"Å¡ic":246,"Å¡ib":61,"Å¡in":843,"Å¡il":171,"Å¡im":108,"Å¡ik":69,"Å¡ih":534,"Å¡it":122,"Å¡ir":718,"Å¡je":233,"Å¡ja":121,"Å¡ji":274,"Å¡ka":2789,"Å¡ki":6172,"Å¡ke":6766,"Ìн":79,"cLa":71,"bju":136,"bje":482,"bja":288,"áto":72,"biz":60,"bis":206,"bit":738,"biv":1271,"bio":487,"bir":767,"ász":140,"bil":9355,"bim":90,"bin":672,"bij":500,"bo ":1111,"blj":4814,"blo":131,"ble":384,"bli":5531,"bn ":78,"bla":906,"bod":537,"bok":127,"bol":2141,"boj":794,"bog":423,"boh":73,"biÄ":489,"bno":809,"bna":528,"bni":923,"bne":712,"bmo":523,"biÅ¡":76,"bon":136,"bom":159,"bor":1964,"áza":65,"bot":328,"bos":204,"bov":492,"bou":88,"áln":82,"be ":2097,"bam":109,"ban":872,"bak":214,"bal":951,"baj":107,"áko":72,"bah":107,"bac":170,"án ":124,"baz":156,"bav":207,"bat":235,"bas":207,"bar":1233,"ánt":85,"bdo":308,"áno":61,"ány":67,"bde":133,"azÅ¡":326,"ánd":61,"bda":111,"bi ":1760,"bej":129,"beh":88,"ár ":304,"bec":216,"ber":3398,"ben":1839,"bel":842,"bez":109,"bes":1297,"bet":155,"baú":258,"bho":160,"bia":67,"bib":116,"bic":125,"áro":114,"ári":73,"áci":80,"buÅ¡":62,"ca ":7059,"car":308,"cas":90,"cat":115,"can":153,"cam":193,"cal":177,"cah":276,"ce ":3980,"bri":1142,"bro":951,"brn":171,"bra":2116,"bre":1080,"bu ":321,"brs":91,"bru":767,"bsk":752,"bso":165,"bse":534,"bst":433,"boÄ":145,"bur":549,"bul":151,"bum":224,"buj":619,"bud":138,"buc":60,"bus":181,"bve":153,"by ":66,"bož":173,"aka":2233,"am ":1501,"ake":764,"akc":307,"aki":595,"ajk":170,"ajl":126,"aji":853,"ajo":3508,"ajp":438,"ajm":203,"ajn":1032,"ajs":1806,"ajt":95,"aju":1775,"ajv":1227,"al ":5680,"ajb":861,"aja":4631,"ajd":410,"ajc":106,"aje":1321,"ajh":414,"ail":145,"ain":656,"air":120,"ais":193,"ak ":1483,"ahk":1261,"ahl":62,"ahi":99,"ahu":67,"aht":170,"aho":1733,"aj ":3209,"agy":182,"aha":1141,"agl":148,"agm":59,"agi":325,"agr":2630,"agu":186,"agn":486,"ago":1560,"anu":2488,"anz":168,"ano":4496,"ann":617,"ant":2576,"ans":8949,"ane":3760,"ang":2192,"anh":58,"ani":12168,"anj":11003,"ank":1463,"ap ":82,"ana":5327,"anc":5066,"and":2328,"amu":229,"amm":99,"amo":2039,"amn":424,"amp":456,"ams":594,"amk":68,"ami":1934,"adž":1259,"ame":3673,"amb":620,"ama":1428,"ao ":196,"alv":164,"alu":459,"alt":357,"als":1671,"alp":219,"alo":3988,"aln":5663,"alm":581,"all":563,"alk":590,"alg":235,"ali":12519,"adÅ¡":352,"alj":2204,"alc":1638,"ald":278,"ale":2748,"alf":59,"Å am":81,"ala":5384,"alb":313,"an ":7080,"aks":331,"akr":499,"aku":1530,"akt":954,"ako":3028,"akn":93,"akl":239,"aba":528,"abe":1164,"abi":922,"abl":2808,"abn":285,"abo":950,"abr":232,"abs":510,"abu":83,"ae ":446,"aca":87,"ad ":1592,"ac ":892,"ab ":93,"afo":67,"afr":130,"aft":161,"afs":248,"aff":75,"afe":70,"afi":418,"ai ":215,"aga":832,"age":434,"ael":296,"ah ":2832,"afa":85,"ado":1208,"adr":500,"adl":177,"adk":221,"adn":2260,"adm":341,"adg":67,"adj":251,"adi":2931,"add":208,"adc":70,"ade":1675,"ag ":119,"adz":178,"ads":309,"adu":492,"aco":154,"ack":237,"aci":3927,"ach":405,"ace":463,"acc":103,"ada":12281,"adb":350,"af ":280,"act":73,"azn":554,"azm":458,"azp":368,"azo":464,"arÅ¡":186,"azi":1682,"azl":1367,"azk":65,"azv":987,"azu":362,"azr":436,"azt":209,"azs":228,"aze":313,"azg":172,"aza":617,"Å pa":241,"azb":60,"azd":489,"avÄ":119,"azz":92,"az ":748,"ayl":61,"aye":106,"Å ta":81,"Å te":144,"ba ":1670,"Å tu":63,"at ":1858,"arh":471,"arg":219,"are":1974,"ard":2757,"arc":477,"arb":245,"ara":3352,"arp":61,"aro":2967,"arn":2439,"arm":683,"arl":640,"anç":98,"ark":1199,"arj":3449,"ajÅ¡":619,"ari":4347,"aru":188,"arv":516,"arr":314,"ars":3622,"art":3803,"au ":230,"asa":521,"ary":261,"akÅ¡":197,"asi":1326,"ash":118,"asc":90,"asb":1150,"ase":7736,"aso":654,"asn":756,"asp":358,"ask":223,"asm":118,"asl":1005,"ar ":4861,"apa":403,"Å en":181,"ape":643,"api":1573,"aph":75,"apn":71,"apl":264,"apo":1357,"app":73,"apr":1152,"aps":75,"apt":76,"apu":142,"as ":2234,"avc":158,"avb":92,"ava":3257,"ax ":66,"aux":63,"aut":447,"avs":1220,"avt":891,"avr":171,"Å ko":256,"avo":1996,"avn":8963,"avk":259,"avl":3158,"avi":5083,"anÅ¡":152,"avj":154,"avg":550,"ave":2196,"Å ma":126,"ay ":236,"awa":58,"avz":193,"avu":60,"arÄ":77,"awn":81,"anž":60,"av ":1266,"ata":1793,"asu":583,"ast":7399,"ass":309,"anÄ":577,"atm":280,"atn":621,"atk":2238,"atl":115,"atr":682,"ato":4303,"ate":7456,"alÅ¡":59,"ati":8392,"atj":90,"ath":285,"att":232,"ats":603,"atu":979,"aul":252,"aum":70,"aun":59,"aur":182,"aus":178,"aud":100,"auk":85,"ος":138,"ος ":138,"Ï‚ ":264,"ν ":77,"Zve":255,"α ":118,"еви":63,"ий ":84,"ич ":167,"až ":103,"jeg":1394,"jej":1047,"jed":501,"jec":86,"jep":94,"jer":828,"jek":653,"jel":447,"jem":3606,"jen":7756,"jez":1544,"jes":258,"jet":1512,"jev":3484,"jaÄ":189,"ji ":9921,"aža":200,"ažd":84,"aže":590,"ažj":94,"aži":70,"ažn":142,"jhe":109,"jhn":247,"jad":100,"jat":493,"jas":144,"jav":1972,"jap":253,"jar":116,"jal":4024,"jak":772,"jan":7670,"jam":569,"jah":261,"jaj":1388,"jaz":69,"jbo":832,"jce":82,"je ":90645,"izÅ¡":397,"jci":81,"jde":159,"jda":105,"jna":637,"jeÅ¡":262,"jmo":79,"jni":1155,"jne":1049,"jiÄ":83,"jno":1777,"eÄ ":1193,"jol":65,"jon":462,"jos":83,"jor":101,"jpo":317,"jpr":102,"Ñк":123,"jiv":675,"jit":167,"jis":201,"jim":790,"jin":1168,"bÅ¡k":61,"jik":186,"jil":336,"jaÅ¡":1441,"jij":72,"jig":518,"jih":2539,"jic":888,"те":99,"jeÄ":100,"ÑÑ‚":65,"ул":59,"jn ":65,"jko":128,"jka":118,"jo ":14224,"jma":181,"jlo":62,"itn":496,"itm":98,"itl":62,"itk":153,"itr":810,"ito":1788,"itv":1047,"itu":620,"itt":146,"its":553,"itz":130,"ity":105,"isk":1191,"ism":148,"isl":637,"iso":1260,"isn":596,"isp":122,"iss":283,"inÄ":105,"isu":398,"ist":8943,"isz":119,"iv ":665,"ita":4085,"itd":78,"ite":3294,"ith":151,"iti":3232,"itj":163,"ivo":647,"ivn":1608,"ivu":289,"inž":250,"ius":336,"ium":162,"iva":2833,"ix ":146,"inÅ¡":354,"ivi":1461,"ivj":69,"ivk":187,"ivl":505,"ive":1813,"ipr":269,"ipo":673,"ipp":99,"ipu":59,"ips":60,"ipt":181,"ipi":261,"ipl":628,"is ":1968,"ion":2321,"iop":95,"ior":98,"ios":95,"iot":214,"iog":97,"iok":133,"iol":587,"iom":79,"ipa":1086,"ipe":338,"iov":126,"ir ":1030,"iru":579,"irs":381,"irt":62,"iro":1658,"irn":1433,"irk":3666,"iri":1414,"irj":549,"isi":572,"ish":146,"ise":765,"isc":429,"isa":2843,"iu ":69,"iqu":102,"ilÄ":111,"ire":909,"irg":67,"ira":3405,"irc":132,"it ":737,"ünc":71,"ivÄ":108,"ür ":68,"ivÅ¡":143,"ja ":24669,"iz ":4501,"izu":488,"izv":2556,"izr":1033,"izs":209,"izp":299,"izo":951,"izn":244,"izm":1841,"izl":440,"izk":366,"izj":155,"irÅ¡":113,"izi":2488,"izh":387,"izg":268,"ize":500,"izd":905,"izb":341,"iza":2030,"kaÅ¡":110,"kih":6752,"kij":103,"kim":1029,"kil":157,"kie":91,"kiv":97,"kin":454,"kip":232,"kir":103,"kis":399,"kit":420,"kaž":123,"kje":723,"km ":698,"ki ":46414,"ked":200,"keg":3700,"kej":985,"kem":8923,"kel":218,"ken":160,"kes":145,"ker":562,"ket":264,"kev":340,"key":87,"kaÄ":293,"ke ":15947,"kci":1307,"kda":435,"kra":4811,"krb":350,"kre":318,"kt ":433,"ksa":263,"kse":141,"ku ":1833,"kro":1818,"krv":182,"kri":2242,"koz":528,"kov":6012,"km²":98,"kot":4172,"kos":954,"kor":1260,"kop":980,"koo":149,"kon":3042,"kom":2571,"kol":3036,"kok":1914,"koj":101,"koh":100,"kog":105,"kof":1624,"kod":500,"ks ":201,"kmu":212,"kme":268,"kmo":345,"koc":91,"kob":135,"kne":116,"kni":115,"knj":1121,"klu":447,"ko ":24162,"kma":99,"kle":646,"kla":1462,"klo":710,"kli":1093,"klj":712,"jvo":146,"jut":62,"jus":126,"jul":555,"jun":698,"jur":257,"jve":859,"jvi":346,"joÄ":433,"jub":2164,"juj":406,"jug":1198,"jud":1091,"jsk":8627,"jst":1435,"eÄj":1199,"eÄk":404,"eÄi":887,"eÄn":355,"eÄo":63,"eÄu":68,"ju ":6445,"jse":265,"jiÅ¡":131,"eÄa":390,"eÄe":1036,"již":650,"kaz":528,"kav":355,"kat":5637,"für":83,"kar":3117,"kas":410,"kap":270,"kan":1808,"kal":3239,"kam":632,"kaj":559,"kak":434,"kah":366,"kai":81,"kad":433,"kac":404,"juž":772,"ka ":23634,"juÄ":601,"jze":62," Ga":900," Ge":881," I ":207," Fo":2042," Fu":219," Fr":2139," Fi":677," Fl":224," Ha":1561," He":1083," Cô":91," Gy":129," J ":84," Go":1655," Gr":1898," Gu":360," Gv":83," Gi":550," Gl":510," Ig":179," Id":108," Ib":77," K ":116," Hu":329," Hr":4060," Ho":699,"ha ":480," Hi":543," Ji":67," Je":1541," L ":110," Ja":1800," Iz":882," Iv":415," Ir":276," Is":1032," It":703," Im":512," In":1618," Il":228,"ham":322,"han":759," M ":155,"hai":79," Ka":2499,"haj":1477,"hal":392," Ke":540,"hau":83," Ki":619,"har":950,"has":60,"hat":80," Jo":1288," Ju":1997,"haf":95,"hae":137,"hab":68,"had":64," N ":83," La":1551," Le":3180," Li":1291," Lj":1723," Kl":443," Kn":255," Ko":3810," Kr":2271," Kv":109," Ku":508," Mc":133," Ma":6006," O ":269," Ml":210," Mi":1860," Dž":96," Me":2209,"he ":889," Lo":1511," Ly":76," Lu":757," Já":138," Ne":1590,"а ":247," P ":279," Na":2970," Nj":398," Ni":827," Mr":101," Mo":2454," Mu":575,"hek":94,"hel":368,"hei":125,"heb":110," A ":314,"het":73,"hes":152,"her":719,"heo":145,"hen":330,"hem":187,"hi ":171," B ":212," C ":442," Ap":398," Am":984," An":1817," Ak":264," Al":2235," Ai":256," Aj":81," Ag":178," Ah":85," Af":338," Ac":171," Ad":366," Ab":561," Ba":1832," D ":134," Az":412," Av":1041," Au":488," At":316," As":363," Ar":1291," Be":2192,"hie":110,"hid":239,"hic":86," Bi":1002,"hia":98,"hip":210,"hio":70," Bj":272,"hin":407,"him":147," Bl":437," Bo":1900,"hil":201,"hik":78,"hij":109," Br":2242," Bu":794,"his":178,"hit":605,"hir":188," E ":92," Ca":1521," Ce":1243," Ci":464," Ch":1220," Cl":266," Cr":346," Co":1629," Cu":189," Cv":64," Cy":61," F ":123," Da":1039," Di":915," De":1297," Dr":1054,"hkr":181," Do":1606,"hko":1232," Du":855," Dv":142," Ea":63,"hn ":214," Ed":330," G ":65,"hla":193," El":398," Ek":172," Ei":122," Eg":129," Et":115," Es":219," Er":320," Ep":63," En":408," Em":164," Eu":245," Ev":1000," Fe":726,"ho ":80,"hma":181," Fa":830," H ":159,"gma":95,"go ":1697,"glo":304," Z ":182,"gle":2479,"gli":810,"glj":228,"gla":2978," Wo":190," Wi":722," Wh":62," We":269," Wa":403,"й ":140," Vz":150,"gog":353," Zr":80," Zu":99,"god":1736," Zv":306,"gob":81," Vé":82," Zm":60," Zl":152," Zo":102," Zn":141," Zd":593," Ze":880," Zg":337,"giÄ":79,"gno":136," Zi":327,"gni":103,"gnj":147," Za":1745,"gne":563,"gna":300," Yo":229,"gs ":62,"о ":66,"goz":506,"goj":282,"н ":123,"gom":421,"gol":311,"gon":470,"gos":3146,"gor":2864,"got":348,"gov":2780,"gu ":806," a ":458,"Ñ€ ":62,"gro":564,"grm":63,"gru":125,"bÄu":59,"grs":221,"gra":6878,"grb":61,"bÄi":4969,"gri":272,"gre":942," R ":122,"в ":115," Oz":78," Os":737,"gto":98," Ot":263," Or":643,"goÄ":401," Op":320," Po":5507," Lé":59," Pl":785," Pi":1026,"gul":122," Ph":190,"gua":77," Pe":1565,"gub":149," Pa":2188,"gue":188," Ny":85," Nu":95," No":1550," Ol":321," Ok":257," On":182," Om":107," Oh":58," Og":200," Od":1566," Oc":80," Ob":907," Ra":1525," Mü":85," T ":114," Qu":210,"új ":254,"goÅ¡":89," Ro":1658," Re":3734," Rd":107," Ri":916," Rh":128," S ":293," Pr":3986,"gur":145,"gus":676," Pt":177," Pu":321,"gun":145," Má":81," Sz":701," Sy":146," Sw":66," Sv":2526," Su":652," St":3290," Ta":1068," V ":1176,"gya":99," Th":689," Ti":702," Te":1006," Tr":1582,"gyk":115," To":1461," Ru":970," Sa":2415," U ":88,"е ":69," Sh":246," Si":1155," Sc":584," Se":2090," So":1610," Sp":1368," Sr":838," Sk":482," Sl":6566," Sm":233," Sn":82," Va":911,"и ":110," X ":144," Ve":3898," Vi":1616," Vl":200," Vo":997," Vu":138," Vr":488," Vs":192," Tu":669," Ty":77,"grÅ¡":1077," Uk":147," Ul":103," Um":80," Un":907," Up":188," Ur":251," Us":272," ja":3029," l ":65,"iam":240,"ial":1334," iz":12947,"ian":961," ji":1172,"ias":76,"iar":121," je":57692,"iat":484," io":104," ip":64," im":7503," in":30104," il":218,"ic ":1048,"iac":130," is":755," it":1465,"iag":159," ir":117,"ibl":600," fü":65,"ibi":274," ka":7604,"ibo":889," m ":545,"ibn":134,"ibr":170," kj":700," ki":23502,"ibu":174," ke":816," jo":1895,"id ":690,"iba":621,"ibe":314," ju":3473," ha":275," he":621," gi":540," gl":3153," gr":2915," go":2718,"ia ":1100," gu":114," k ":178," ib":60," id":224," ig":1379,"ib ":92," hi":1051," hk":181," hl":97," ho":1169," hr":955," hu":193,"iet":196,"ieu":64," nj":2131," ni":1612,"iel":266," ne":7311,"ien":545," na":42601,"ier":585,"ies":172,"ied":168,"ieg":71,"иÌ":65," mu":575," mr":220,"ig ":209," mo":4823," mn":787," mm":80," ok":4185," ol":657,"ifu":62," om":674," on":135," og":566," oh":232,"ifo":187," oc":331," od":6669," of":580," ob":12756,"ifr":67,"ife":161,"ifi":406,"ih ":22740," nu":342," no":2512," np":247,"ifa":63," le":11324," lj":1128,"icr":71,"ics":75,"ict":217," li":1970,"icu":137,"icn":80," n ":309,"ico":974,"ick":188," la":3283," kv":554," ku":716,"ici":3252,"ich":665,"ice":3027," kn":1159,"ie ":718," km":1016,"ica":6322," kl":1067," kr":5141," ko":11511," me":12878," dž":64,"idu":106," mi":1837,"ids":75," ml":614,"Ñ ":69,"idr":346," o ":887,"ido":732," ma":6202,"idn":485," lu":270,"idi":434,"idg":71,"ide":1356,"ida":711," lo":1132," af":94," ag":154,"aÅ¡a":593," ab":327," ac":59," ad":284,"aÅ¡e":251," am":1331,"aÅ¡k":6766," an":2354,"aÅ¡i":200," ap":760,"aÅ¡n":368,"iin":107," ak":715,"iim":2227," al":6858," av":1858," au":60," ar":1254," at":457,"aÅ¡t":146," as":1847," d ":320," ba":1756," az":58,"il ":5175,"ija":12085," bi":10040,"ije":12707," be":1503,"iji":5965," bo":2432," bl":915,"ijo":6465,"ijs":4363," bu":186,"iju":165," br":1850," ca":173," e ":85,"im ":3856,"ika":6461,"ige":666,"iga":1104,"aÅ¡ ":78,"igl":183,"igh":214,"igi":601,"igu":193,"igr":1537,"igo":249,"ign":280,"ij ":2784,"Ñ‚ ":70," b ":94,"ihe":100,"iha":530,"ihi":87,"iho":729,"ik ":9136,"у ":103," c ":70," er":65,"imo":1733,"imn":223," et":360," es":185," en":4762,"ims":2333," em":109," ep":251,"imp":926,"imf":91," el":1413,"ime":9448," ek":723,"imk":790,"imi":2451,"ip ":337," fe":1007,"inc":748,"ind":925,"ina":10483," fa":1370,"imu":325,"аÌ":109," ev":586," fu":480,"inn":113," fr":3146,"ino":5575,"aÅ¡Ä":462," fo":718,"int":1271,"ins":4445,"inf":316," fl":126,"ine":5190,"inh":91,"ing":1310,"inj":1193," fi":2355,"ini":4151,"ink":465," ge":1321," ga":4003,"iod":295,"inu":433," i ":115,"inv":62,"inz":59," cl":64,"iko":4804," cm":100,"ikl":979," co":254,"iki":3287," ce":2735," ch":72,"ike":4355," ci":552,"ila":7057,"ilb":66," f ":61,"in ":31978," da":5215,"ikv":58," cv":150,"ikt":162,"iku":799,"ikr":245,"iks":94," do":7621," dn":241,"ilo":4202,"ill":1121,"ilk":457," dr":7206,"iln":1875,"ilm":587,"ilh":107," de":11720,"ilj":766,"ili":2960,"ild":76,"ilc":146," di":5516,"ile":1135,"ima":2334,"imb":293,"ч ":172," ed":1001,"io ":829," dv":2054," du":1905,"ils":366,"ilt":85,"ilu":372,"ilv":125,"ль":58," vÄ":201," zm":655," zl":487,"ла":90," zo":151," zn":3542," zu":280,"ле":83," uÄ":541," zr":511,"ли":66," zv":1609,"hok":781,"hol":497,"hom":250,"hon":167," za":14763,"ко":128,"hos":66," zd":1135,"hot":761," ze":976," zb":804,"hov":2393,"ку":58,"hop":61," zi":186,"hor":342," zg":1780,"ка":91,"ки":115,"hod":3942,"hni":523,"hno":289,"hnu":58,"hna":106,"hne":92," z ":4835,"ин":98,"ик":81,"ий":90," ož":121,"ич":191,"ри":87,"ро":86,"ра":120,"ре":58,"htt":71,"hto":63,"htn":72,"hte":153," už":81,"ор":104,"ол":82,"ов":288,"hu ":190,"hrv":430,"но":113,"hro":110,"hre":73,"ни":87,"hri":399,"ht ":188,"на":128,"hra":653,"hiÅ¡":212," ru":1131," u ":188," sa":1902," se":18649," sc":148," si":3326," sh":137," sn":739," sm":996," sl":6094," sk":4932," sr":1993," sp":10659," so":10507,"ви":220," t ":95," ra":7736," re":7014," rd":178," ri":2298," ro":2318," pt":209," pu":582," pr":31964," ps":336," s ":4586," px":59,"hy ":96,"ва":103,"ад":75," os":4746," ot":1470,"hum":246," ov":185,"hun":84,"hus":107," op":2076," or":2176,"hur":142,"ан":143,"ак":68," oz":3235," pe":3830," pa":8196,"ар":96," pl":2669," po":40217," pi":2589," y ":101," vz":1551," x ":129," va":2039," ve":6450," uv":1301," vn":129," vo":6456," vp":464," vr":2767," vs":3256," vi":2299," vk":283," vl":673," vm":85," ud":201,"ет":79,"ер":77,"ен":63," tv":236," tu":5865," us":2479," ut":152," ur":2875," up":8796," um":682," un":617," uk":539," ul":238," ug":179," ta":4305," v ":29906," st":7117," sv":3870,"оÌ":61," su":765,"ев":143," oÄ":210," tr":3796," tl":193," to":2461," th":371," ti":1200," tk":120," te":8405,"fi ":65,"ffe":78,"ffi":69,"fes":342,"fer":516,"fed":154,"feb":634,"fen":145,"fek":403,"fel":89," ÄŒa":315," ÄŒi":126," ÄŒe":525,"faz":112,"fat":64,"far":143,"fan":669,"fak":920,"fal":122,"ff ":62,"fe ":105," Äu":65,"fa ":189,"aúj":298," Äu":115," Är":759,"eyr":71," ÄŒu":69,"exa":83,"ez ":873," ÄŒr":316," Äe":1273," Äl":1304," Äi":437," Äa":1729,"ezu":507,"eza":1493,"ezd":639,"ezn":2303,"ezo":3022,"eví":88,"euÄ":161,"eze":1111,"erÅ¡":61,"ezi":1770,"eta":9837,"ete":2260,"etd":101,"etj":1541,"eti":2887,"elÅ¡":67,"eth":104,"etn":3058,"etl":581,"etk":435,"esp":164,"esn":1865,"eso":719,"est":9497,"esu":218,"enÄ":254,"esr":69,"ess":341,"ev ":4515,"emÅ¡":2022,"eto":5019,"etr":1803,"ets":629,"ett":335,"etu":861,"etv":352,"ew ":247,"eve":3581,"evd":110,"evc":189,"eva":3659,"evo":978,"evn":851,"evl":113,"evk":344,"evj":120,"enÅ¡":165,"evi":3431,"euv":65,"eut":92,"eur":130,"eus":174,"ex ":78,"evu":106,"evr":568,"evs":223,"evt":159,"ey ":565,"evz":91,"epe":218,"epi":485,"eph":184,"er ":9909,"epa":2146,"eos":79,"eor":822,"eom":284,"eol":485,"eop":91,"eon":196,"es ":3592,"ept":897,"epu":2717,"epl":115,"epn":163,"elé":65,"epp":110,"epo":984,"epr":1096,"erk":1130,"erl":288,"ejÅ¡":1389,"eri":6828,"erj":1569,"erg":1075,"erh":74,"ere":2880,"erf":64,"erc":387,"erd":237,"era":3623,"erb":290,"et ":2922,"esj":60,"esk":362,"esl":322,"esm":246,"esi":510,"esc":198,"ese":3032,"eu ":71,"esa":1428,"erz":1265,"ery":65,"erv":377,"eru":570,"emÄ":403,"err":420,"ert":841,"ers":1249,"ern":2752,"erm":750,"erp":166,"ero":2923,"eki":619,"ekl":439,"ekm":687,"eko":1561,"ekr":138,"eks":912,"ekt":2647,"eku":579,"ekv":164,"en ":7706,"elb":111,"ela":2552,"eld":165,"elc":327,"elf":63,"ele":5343,"eli":7355,"elj":12080,"elg":274,"ehé":68,"elm":113,"eln":478,"elk":390,"ell":946,"elo":6099,"elu":1481,"els":386,"elt":135,"eo ":194,"emb":2938,"ema":2409,"edž":154,"eme":2514,"emd":94,"eml":1047,"emn":341,"emo":1461,"emi":2775,"emu":847,"emp":655,"ems":675,"ep ":100,"ene":4626,"enh":70,"eng":267,"enb":171,"ena":6583,"end":853,"enc":1426,"eno":7193,"enn":350,"enk":486,"enl":165,"eni":11655,"enj":3527,"enu":1052,"ens":9125,"ent":4611,"enr":131,"enz":343,"eog":377,"eod":241,"eob":101,"egl":515,"ego":1324,"egn":116,"ege":392,"egi":3464,"ej ":1349,"eha":369,"egr":207,"egu":231,"egy":59,"ehn":551,"ehr":149,"eho":984,"ehi":163,"ek ":3893,"eic":68,"eis":179,"eir":92,"eim":143,"eil":148,"ein":591,"eid":153,"eja":1145,"el ":3464,"eiz":90,"eit":89,"ejs":725,"ejo":1467,"ejn":247,"ebÅ¡":60,"eji":700,"eje":1829,"ekd":397,"eke":552,"ekc":78,"eka":3433,"em ":19835,"eju":371,"gl ":122,"git":143,"gis":130,"gir":72,"gim":692,"gij":4610,"gik":62,"gip":136,"gin":441,"gio":182,"gie":59,"gib":460,"gih":484,"gia":72,"ght":197,"gha":74,"ggi":66,"gaÄ":142,"gi ":1057,"gen":1655,"geo":621,"get":148,"ger":535,"ges":132,"gh ":98,"geb":124,"geg":139,"gem":133,"gel":572,"gej":96,"gda":62,"ge ":1547,"gac":68,"gad":460,"gah":82,"gas":134,"gar":537,"gat":408,"gaj":272,"gam":198,"gal":701,"gan":2184,"ga ":19235,"fur":60,"fte":66,"fun":414,"ft ":152,"aÄb":149,"aÄa":451,"fra":3096,"fre":206,"aÄe":1435,"aÄj":77,"aÄk":163,"fri":439,"aÄi":1402,"fsk":383,"fro":120,"aÄn":463,"aÄr":174,"aÄu":1108,"fov":120,"for":1149,"fos":82,"fot":167,"fon":272,"fol":119,"aÄ ":265,"fiÄ":180,"fla":71,"fic":219,"fie":63,"fig":93,"fij":889,"fil":1101,"fik":288,"fin":542,"fit":105,"fiz":1064,"fja":62,"db ":97,"da ":14922,"dbe":281,"dba":322,"dbi":164,"dbo":232,"de ":3505,"dac":99,"dal":2088,"daj":1290,"dag":370,"dah":122,"dae":198,"dat":2652,"dar":1727,"dan":3799,"dam":251,"dav":343,"dda":258,"dde":307,"dce":69,"cul":97,"cto":99,"cti":246," ÃŽl":184,"cy ":79,"cve":165,"cus":130,"cur":63,"cks":59,"cko":94,"cla":60,"cle":108,"co ":1155,"con":195,"col":193,"com":102,"cor":143,"cos":2856,"cot":60,"cou":95,"cs ":326,"cqu":61,"cro":134,"cu ":453,"cci":147,"cca":61,"cea":385,"ch ":526,"cev":1776,"cer":935,"ces":1318,"cet":77,"cen":988,"cep":196,"cej":81,"cem":898,"cel":1340,"ceg":119,"ced":151,"ci ":2670,"cha":428,"chw":61,"chu":146,"cia":974,"ck ":555,"cie":148,"cid":94,"che":837,"chl":72,"chi":339,"cho":135,"chm":179,"chn":116,"cht":110,"civ":109,"cij":6609,"cik":850,"cil":290,"cim":128,"cif":150,"cih":145,"cir":298,"cis":476,"cit":587,"cin":475,"cio":727,"cip":225,"cm ":87,"cke":249,"cka":102,"ed ":7632,"eba":410,"ebe":1030,"ebi":1286,"ebl":102,"ebn":705,"ebo":328,"ebr":1109,"ebu":659,"ec ":3986,"eac":70,"eag":60,"eae":122,"ead":77,"eak":245,"ean":473,"eal":308,"ear":308,"eas":76,"eap":68,"eat":181,"eau":166,"eb ":1452,"ea ":232,"efi":249,"efo":206,"efa":145,"efe":483,"ei ":95,"ega":14282,"een":162,"eh ":1490,"eer":68,"eev":58,"edk":372,"edl":222,"edm":691,"edn":4723,"edh":90,"edi":2962,"edj":165,"ede":4374,"ône":77,"eda":2590,"edb":168,"eg ":627,"edt":128,"eds":1765,"edv":775,"edu":552,"edp":144,"edo":1120,"edr":537,"eck":137,"ech":121,"eci":430,"ece":981,"eca":75,"ee ":129,"ef ":166,"ecu":64,"ect":136,"eco":75,"dož":69,"dvs":488,"dwa":102,"dy ":102,"dvi":1792,"dve":1124,"dvo":628,"dur":65,"dus":252,"dva":885,"duÅ¡":167,"drž":2000,"dzo":143,"dzi":74,"dze":62,"dor":425,"dop":136,"don":666,"dom":1025,"dol":2617,"dok":396,"doz":167,"dow":104,"dov":2766,"dot":187,"dos":834,"dr ":66,"dpi":182,"dpr":535,"dpo":228,"ds ":150,"diÅ¡":1126,"dmi":400,"dmo":356,"dna":1989,"dne":2475,"dni":3893,"dež":1743,"dnj":2661,"dno":3481,"diÄ":320,"dob":2688,"doc":83,"dod":289,"dog":505,"dst":1636,"dso":71,"dte":147,"dun":61,"duj":151,"dul":104,"duk":196,"duh":1475,"duc":137,"dri":615,"dra":2046,"dt ":61,"dre":1648,"du ":2258,"dro":1558,"drs":225,"dru":5537,"dsk":1257,"dse":412,"dge":82,"dgo":243,"dic":712,"did":62,"dia":709,"dho":97,"ôte":98,"der":1069,"des":1368,"det":180,"dev":559,"dez":147,"deb":135,"dea":152,"ded":131,"dec":783,"def":191,"dej":655,"del":8200,"dek":748,"den":2287,"dem":894,"dep":1917,"deo":172,"di ":7874,"dle":116,"dla":418,"dko":292,"dkr":290,"dki":137,"dme":496,"dma":184,"do ":2879,"dlo":208,"dlj":134,"dli":196,"dja":348,"dje":967,"div":552,"diu":58,"diz":86,"dim":344,"din":3211,"dio":320,"dip":467,"dir":3392,"dis":608,"dit":557,"die":123,"dif":117,"dig":180,"dih":219,"dij":1635,"dik":285,"dil":1003,"dka":143,"dke":226,"dju":78,"deÄ":365,"rgy":70,"rgu":403,"rhe":114,"rha":142,"rhi":372,"rhu":89,"rhn":68,"rho":225,"iža":226,"rga":1747,"ri ":6205,"rgl":58,"iži":369,"rgi":606,"ižj":188,"iže":597,"rge":583,"rgo":351,"ižn":1012,"rgn":87,"ret":1927,"res":1802,"rev":1974,"reu":328,"rez":1267,"rh ":216,"rfi":77,"rfo":61,"raÄ":1077,"rdu":102,"rds":107,"rg ":550,"iž ":81,"reb":2051,"rea":620,"ree":132,"ref":492,"rec":915,"red":9699,"rei":333,"rej":1975,"reg":4478,"reh":767,"rem":2494,"ren":2516,"rek":2213,"rel":1062,"rer":139,"reo":180,"rep":1325,"rda":399,"rcu":159,"rdo":364,"rdn":359,"rdi":2193,"rde":564,"re ":2956,"rbu":279,"rbs":360,"rci":266,"rch":173,"rce":392,"rca":244,"raz":7036,"rd ":765,"rap":609,"rar":714,"ras":1404,"rat":4465,"rau":89,"rav":10605,"rbi":646,"rbo":438,"rba":318,"rbe":206,"rc ":65,"raj":3231,"rai":122,"rah":639,"rag":448,"ran":10813,"ram":1883,"ral":3846,"rak":1045,"rab":4754,"raf":1035,"rae":188,"rad":7153,"rac":1106,"rpu":173,"rpo":96,"rs ":406,"rpe":109,"rpa":103,"ror":143,"ros":2112,"rot":1454,"rom":2391,"ron":2361,"roo":117,"rop":1736,"roz":633,"rou":216,"rov":3453,"row":90,"rob":744,"roa":69,"rod":3371,"roc":719,"roj":1187,"roi":1061,"rol":591,"rok":1027,"rof":772,"roe":143,"rog":1831,"rno":2826,"jÅ¡Ä":126,"riÄ":1125,"rns":80,"rnu":76,"rna":2170,"rež":741,"rne":1804,"rnj":672,"rni":3001,"rmo":552,"rmu":1839,"ro ":1744,"rma":1564,"rme":357,"rmi":362,"reÅ¡":389,"rlo":204,"rlj":61,"rli":272,"rle":264,"rla":328,"rn ":268,"rkv":475,"rku":166,"rkt":89,"rks":59,"rkn":66,"nço":87,"rko":671,"rki":446,"rke":698,"rka":3906,"rm ":91,"reÄ":850,"rju":560,"rji":433,"rja":3706,"raž":658,"rje":3010,"riz":1024,"rl ":197,"rip":1323,"jÅ¡o":114,"rio":714,"rir":527,"rit":2886,"ris":2096,"riv":1041,"riu":103,"rih":1145,"rig":966,"rij":4080,"raÅ¡":557,"jÅ¡i":1298,"rii":2351,"ril":2003,"rik":1886,"jÅ¡n":99,"rin":1694,"rim":3965,"jÅ¡a":896,"ria":1305,"rib":1637,"ric":1600,"rid":1029,"rie":837,"jÅ¡e":763,"rif":96,"rk ":603,"roÅ¡":477,"rož":1407,"ruj":117,"ruh":77,"rug":3066,"rud":146,"ruc":91,"rup":276,"run":235,"rum":444,"rul":77,"ruk":405,"ruz":173,"rus":1264,"rut":93,"rva":5491,"rvi":1217,"rve":3036,"rvo":519,"rvn":190,"ry ":643,"rsk":7463,"rsi":153,"rso":356,"rsa":151,"rse":525,"rta":677,"rst":3047,"rtm":1893,"rtn":868,"rto":742,"rte":567,"rth":222,"rti":1649,"rub":82,"rua":643,"rts":92,"roÄ":1581,"rtu":402,"rtv":101,"riÅ¡":1946,"rt ":931,"rro":107,"mÄi":401,"rri":145,"rre":246,"riž":633,"rra":237,"ru ":1442,"rry":141,"sab":104,"sac":80,"sad":165,"saj":415,"sak":747,"sal":910,"sam":1606,"sba":85,"sbe":912,"sbi":124,"san":1213,"sat":1223,"sas":75,"sar":861,"sav":254,"sa ":1996,"ruž":2445,"ón ":126,"ruÅ¡":505,"rze":647,"rza":241,"ryj":60,"rzo":76,"rzi":1047,"sha":61,"sho":59,"shr":72,"she":68,"shi":263,"si ":1301,"sje":270,"siv":155,"seÄ":117,"sid":220,"sic":368,"sia":89,"sk ":194,"sit":181,"sir":202,"sis":1373,"sip":188,"sin":800,"kÅ¡n":168,"sio":244,"sil":1296,"sim":507,"sij":849,"sik":175,"sih":502,"saÅ¡":156,"sif":95,"sig":135,"sbo":112,"sbu":112,"se ":9988,"sca":143,"sce":180,"sci":213,"sch":525,"sco":197,"sev":1928,"ser":946,"ses":2057,"set":1089,"sez":2688,"sh ":159,"sfe":159,"sfo":66,"sei":64,"seh":402,"seg":907,"sed":3642,"sec":285,"seb":3164,"sep":883,"sen":803,"sem":2072,"sel":7746,"sek":379,"sej":258,"spu":64,"spo":2044,"spr":1492,"spe":1038,"spl":748,"spi":220,"spa":7101,"sot":293,"sou":74,"sov":1364,"sol":487,"som":341,"son":987,"sop":193,"sor":556,"sos":106,"sod":1527,"sof":102,"sok":589,"soj":102,"soc":318,"sob":183,"su ":1066,"nÄn":750,"nÄi":273,"nÄe":232,"sre":2496,"srb":412,"nÄa":248,"st ":5603,"ss ":196,"sli":984,"slo":6036,"slu":603,"sla":2779,"sle":851,"ski":17344,"skl":1236,"sko":15385,"skr":668,"sku":2764,"skv":191,"ska":8402,"ske":14199,"siÄ":233,"sno":1962,"sna":435,"sni":1800,"snj":102,"sež":377,"sne":1079,"smo":133,"smr":229,"smu":290,"so ":8342,"sma":396,"smi":413,"sme":609,"sz ":97,"sza":136,"sze":92,"szt":71,"sse":338,"ssa":253,"sso":262,"ssi":329,"ste":4516,"sta":14712,"std":76,"stm":122,"stn":2769,"sto":9875,"sti":10401,"stj":760,"stk":173,"stl":589,"stv":6827,"stu":585,"soÄ":125,"str":7508,"sub":140,"suh":69,"sul":130,"sum":64,"suj":230,"sup":155,"sun":61,"sur":243,"sve":2955,"svi":91,"svo":1687,"tai":89,"taj":1653,"tak":1296,"tal":5739,"taf":62,"tag":120,"tah":112,"tab":357,"tac":360,"tad":115,"td ":92,"taz":72,"tav":4146,"tat":2093,"tas":324,"tar":2532,"tap":65,"tan":6147,"tam":415,"tch":88,"te ":3456,"tde":172,"ta ":16983,"ovÅ¡":201," Å¡t":2696," Å¡v":175," Å¡i":477,"pa ":6188," Å¡k":1366," Å¡o":471," Å¡p":977," Å¡a":246," Å¡e":1284," Å v":259," Å u":75," Å t":405," Å p":321," Å o":105," Å m":149," Å k":346," Å i":182," Å e":283," Å a":280,"ovÄ":87," Å¡Ä":159,"pci":101,"pe ":794,"par":3565,"pat":325,"pas":565,"pav":120,"paz":284,"pac":108,"pad":8267,"pah":117,"pak":187,"pal":492,"paj":410,"pap":279,"pan":5531,"phe":97,"pha":91,"pho":59,"phi":118,"pi ":445,"ph ":153,"pev":467,"paÄ":83,"pea":111,"pec":334,"ped":485,"pen":956,"pep":58,"per":2309,"pet":1059,"lás":63,"pes":1184,"peh":707,"pel":589,"pek":319,"pla":1692,"plj":349,"pli":1265,"ple":1082,"plo":1730,"piz":105,"peÄ":83,"phy":92,"pia":104,"pid":74,"pic":81,"pih":119,"pij":931,"pik":108,"pil":764,"pin":2417,"pio":83,"pir":404,"pis":3588,"pit":455,"poz":877,"pr ":382,"por":6507,"pop":763,"pov":2801,"pou":108,"pot":2623,"pos":4743,"poi":280,"poj":1082,"pog":1928,"pom":2891,"pon":1163,"pok":980,"pol":5783,"pob":187,"poe":84,"pod":14532,"ps ":70,"plé":235,"ppo":61,"ppe":190,"lén":278,"peÅ¡":381,"po ":5444,"piÄ":147,"pno":567,"pnj":141,"pež":186,"pni":1125,"pne":206,"pna":93,"pse":98,"psi":273,"psk":942,"pso":79,"ptu":77,"pub":2962,"pte":795,"pti":568,"pto":218,"poÄ":106,"pra":8108,"pt ":65,"piÅ¡":190,"prv":4499,"prs":213,"prt":260,"pru":88,"pu ":357,"pri":11604,"pre":12368,"pro":5665,"poÅ¡":266,"pož":71,"pur":71,"pus":357,"put":68,"pun":78,"pul":238,"px ":59,"puÅ¡":308,"már":283,"iÅ¡ ":113,"iÅ¡e":281,"iÅ¡a":220,"iÅ¡l":273,"iÅ¡n":372,"iÅ¡i":543,"iÅ¡k":6388,"iÅ¡j":568,"iÅ¡t":1077," Ži":199," Že":372," Ža":131," Žu":1147,"išć":99,"iÅ¡Ä":2416," ži":1623," žl":73," ža":202," že":1263," žu":4184,"mén":60,"lÄn":96,"qua":66,"que":223,"qui":88,"ra ":5234,"rb ":124,"ežn":508,"ngo":291,"ežj":186,"ngi":146,"eži":1395,"ngl":1754,"ežk":163,"ngu":231,"ngr":157,"ngt":104,"ngs":96,"ni ":21439,"eže":1628,"nge":731,"ngh":66,"nga":200,"eža":393,"nha":154,"nj ":974,"nhe":59,"neh":90,"neg":6715,"nej":1610,"nei":60,"nel":292,"nek":1786,"nen":595,"nem":5832,"nep":765,"neo":239,"ner":2095,"net":1642,"nes":1470,"nev":638,"neu":164,"ndv":192,"ež ":616,"ng ":946,"nea":167,"neb":386,"nec":499,"ned":247,"nfo":286,"nfr":73,"naÄ":1875,"ney":179,"nez":634,"nfa":314,"nfe":121,"nco":3080,"nci":1963,"nck":74,"nce":1597,"nch":251,"nca":769,"ne ":16673,"nbu":104,"ndu":365,"ndr":735,"nds":176,"ndo":673,"ndi":1187,"nde":738,"nda":1053,"ncu":165,"nak":2105,"nal":3824,"nam":2274,"nan":3844,"nap":1555,"nar":4271,"nac":538,"nad":2224,"nag":2320,"nah":1010,"nai":103,"naj":4748,"nc ":424,"nab":409,"nbe":101,"nd ":921,"nav":1486,"nau":169,"nat":1633,"nas":9591,"naz":549,"na ":45810,"muÄ":308,"mož":442,"nyi":95,"nz ":124,"nož":611,"ny ":285,"nve":127,"nuk":69,"num":110,"nun":197,"nuj":664,"nus":215,"nut":274,"nua":1914,"nud":60,"ntv":66,"nto":1165,"ntn":394,"ntu":151,"nts":282,"noÄ":122,"ntr":823,"nti":1732,"nth":129,"ntj":84,"nta":1740,"nte":2274,"nsp":149,"nso":136,"nst":4879,"nsf":59,"nse":185,"nsi":206,"nsk":18646,"nsc":84,"nsa":368,"nu ":1386,"iÄu":99,"iÄn":8484,"iÄk":636,"njÅ¡":744,"iÄi":660,"nri":146,"niž":206,"iÄe":649,"iÄa":1089,"nt ":1816,"niÅ¡":3134,"npr":251,"ns ":498,"noc":81,"nod":149,"noa":98,"nob":153,"nog":856,"nof":95,"nok":237,"nol":557,"noi":86,"noj":94,"noo":307,"nop":217,"nom":2492,"non":367,"not":1702,"nos":5345,"nor":656,"nov":7837,"noz":157,"iÄ ":1910,"nne":487,"než":185,"nna":175,"nić":78,"nno":127,"nni":311,"niÄ":1334,"nma":70,"neÅ¡":91,"ići":278,"nlj":152,"nn ":230,"no ":28820,"nke":357,"nki":201,"nm ":145,"nkc":434,"nka":820,"nku":97,"nko":712,"nkt":84,"nkr":119,"nji":3942,"njk":62,"nje":10740,"nja":5163,"ić ":428,"nju":1491,"neÄ":59,"njs":1007,"njo":572,"nij":12636,"naÅ¡":856,"nih":7675,"nig":117,"nif":78,"nie":143,"nid":118,"nic":4360,"nia":191,"nk ":305,"niz":1781,"niv":1064,"nis":1438,"nit":923,"nir":658,"nio":254,"nim":2568,"nin":1324,"nik":9988,"nil":489,"obÄ":4860,"ogr":2005,"ogu":149,"ogi":1571,"ogl":820,"ogo":3723,"ogn":253,"oga":915,"oge":466,"ohr":268,"ohl":58,"ohi":154,"oho":271,"ohn":289,"oha":244,"ohe":84,"oj ":853,"ois":217,"oir":158,"oit":129,"oin":114,"oim":560,"oid":870,"ok ":1437,"ojz":107,"ojv":145,"oju":183,"ojs":1259,"ojo":151,"ojn":3019,"ojm":105,"oji":1325,"oje":1842,"oja":2676,"ol ":641,"oiz":309,"oce":920,"och":145,"oci":1214,"ock":526,"oco":76,"obs":723,"obv":173,"obu":288,"oca":63,"odg":241,"ode":2090,"odk":490,"odl":499,"odi":2311,"odj":811,"odo":2908,"odp":861,"odm":304,"odn":5885,"ods":542,"odt":73,"odr":2425,"of ":1863,"odd":348,"odc":60,"odb":552,"oda":3554,"oel":114,"oen":93,"odz":124,"odv":669,"odu":1390,"og ":1603,"ofi":975,"ofj":119,"ofs":134,"oft":92,"ofo":143,"oh ":61,"oev":59,"off":75,"ofe":287,"ofa":151,"oa ":64,"ob ":1309,"oc ":81,"oam":62,"oak":62,"oba":1124,"od ":12681,"oar":69,"obo":1557,"obr":1689,"obl":2408,"obn":1165,"obm":522,"obh":168,"obj":732,"obi":1733,"obd":521,"obe":1617,"nza":121,"nze":101,"nzi":167,"nzo":136,"nzu":292,"oz ":588,"ows":139,"own":81,"ozv":174,"ozm":90,"ozn":2233,"ozl":77,"ouÄ":124,"ozo":660,"ozd":363,"oze":956,"ozj":60,"orÅ¡":73,"ozi":1844,"oza":922,"otu":189,"oud":99,"ouc":89,"ow ":91,"otl":190,"otj":79,"oti":1413,"oth":99,"ote":2905,"ott":253,"ots":334,"otr":1161,"oto":3788,"otn":1790,"ost":12934,"osu":137,"osv":805,"ota":1246,"ov ":9875,"osi":707,"osk":3606,"ose":3687,"osf":145,"osp":608,"oss":155,"onÄ":622,"osr":908,"osm":266,"osl":2901,"oso":665,"osn":920,"ovz":1051,"owe":104,"ovj":805,"ovi":6438,"ovn":6797,"ovl":1128,"ovk":184,"ovr":972,"ovp":60,"ovo":2629,"ovs":1169,"ova":8912,"ovc":416,"ove":14862,"olž":454,"oug":85,"oui":145,"oul":129,"oun":202,"ous":295,"our":376,"out":124,"opn":423,"opo":1059,"opi":1707,"opk":100,"opl":563,"ope":1736,"oph":167,"opa":1127,"os ":1104,"opu":601,"opr":1200,"opt":284,"ops":761,"ook":176,"ood":98,"or ":2915,"oot":78,"oos":261,"oor":234,"ork":374,"orl":98,"orm":2841,"orn":2246,"oro":2344,"orp":318,"orr":124,"orc":198,"ord":1013,"ore":2187,"orf":214,"org":1762,"ori":3690,"orj":1749,"ou ":161,"osa":1004,"osc":65,"ort":1525,"ors":2919,"orv":191,"oru":646,"orz":678,"ory":80,"m² ":116,"ot ":4531,"orb":248,"ora":6836,"olÄ":72,"ola":1198,"old":256,"olc":287,"on ":4034,"olj":3299,"oli":8521,"oll":233,"olk":1067,"olf":200,"ole":2539,"olh":61,"olg":662,"ols":682,"olt":165,"olm":216,"oln":1077,"olo":4662,"olp":199,"olz":68,"olu":478,"okc":353,"oka":3564,"om ":4672,"oki":527,"oke":1295,"okr":2091,"oks":347,"oko":2939,"okl":420,"okv":386,"okt":717,"oku":681,"ona":3944,"ond":566,"onc":719,"onf":152,"one":1109,"ong":354,"onj":581,"oni":3533,"onk":153,"onn":242,"ono":2372,"ons":2332,"ont":1027,"onu":380,"onv":124,"ony":108,"onz":388,"oma":3776,"ome":4659,"omb":385,"omi":1226,"omm":146,"oml":116,"omp":471,"omn":379,"omo":1779,"omt":60,"omu":612,"omr":152,"oms":494,"op ":353,"la ":14406,"kuž":64,"ína":99,"ín ":66,"ílo":74,"kuÅ¡":168,"le ":7709,"lce":1290,"lca":262,"lci":518,"lcs":234,"lf ":143,"Å‘r ":86,"lde":127,"lda":104,"ldo":98,"ldi":83,"lab":207,"lac":514,"lad":2741,"lah":1475,"lag":668,"laj":371,"lai":176,"lal":158,"lak":556,"lan":4164,"lam":493,"lap":158,"lao":145,"lar":855,"lat":1921,"las":3314,"lau":130,"lav":3720,"lay":77,"laz":215,"lba":139,"ld ":368,"lbe":231,"lbi":66,"lbo":79,"lbu":240,"kvi":678,"kve":490,"kva":1108,"kus":268,"kur":144,"kup":2763,"kun":201,"kum":211,"kul":2114,"kuj":269,"koÅ¡":266,"ky ":76,"kta":268,"kte":244,"ksp":155,"kst":183,"ksi":550,"kso":225,"ksn":135,"kuh":70,"ktr":1174,"koÄ":387,"ktu":785,"kti":841,"ktn":114,"kto":1394,"krÅ¡":290,"íja":164,"kož":132,"lpo":59,"lps":155,"lpe":443,"lpi":97,"lph":69,"ls ":116,"lol":70,"lok":331,"lon":766,"lom":1330,"lop":499,"lor":376,"lod":293,"loc":74,"loh":68,"log":2821,"loj":149,"lpa":140,"los":423,"lot":1148,"lou":85,"lov":16172,"loz":578,"lno":1801,"lić":59,"lnj":113,"lni":3993,"lež":843,"lne":1558,"lob":711,"liÄ":1905,"lmo":111,"lmi":113,"leÅ¡":1366,"lme":146,"lma":647,"lp ":83,"lna":1726,"lmu":66,"hér":83,"lms":222,"lti":179,"lto":145,"ltr":80,"loÄ":1559,"lts":67,"ltu":477,"luc":178,"lub":512,"lug":80,"lue":65,"lsk":2568,"lso":81,"lst":393,"lta":305,"lte":1274,"ljÅ¡":529,"liž":879,"lu ":1284,"liÅ¡":2924,"ía ":66,"lt ":201,"lhe":70,"lj ":3087,"lha":78,"lgo":251,"lge":200,"lgi":319,"li ":11443,"lga":376,"lfr":59,"laÄ":361,"hât":62,"lfo":59,"lfi":89,"lfa":90,"ház":65,"lez":1147,"ley":245,"lex":102,"lev":543,"les":1759,"let":8204,"ler":672,"leo":160,"lep":918,"lem":1328,"len":1784,"lek":2164,"lel":63,"lei":115,"lej":216,"leh":60,"leg":676,"lef":95,"led":2297,"lec":1220,"leb":105,"lea":97,"lg ":68,"lls":71,"llu":110,"lly":120,"lo ":8977,"lla":550,"lle":786,"lli":672,"llo":334,"lko":692,"lku":60,"ln ":62,"lka":768,"lke":292,"lki":170,"ljs":1060,"leÄ":258,"lju":2873,"ljo":339,"ljn":358,"lm ":297,"lje":13663,"ll ":501,"lja":8525,"ljk":59,"laž":224,"lji":2324,"lit":3130,"lis":1861,"lir":252,"lip":265,"lio":237,"lin":3453,"lim":1141,"liz":841,"liv":1015,"liu":90,"lic":2658,"lid":143,"lia":493,"lib":162,"lk ":580,"lik":9667,"dÅ¡k":353,"lil":332,"laÅ¡":191,"lij":4137,"lig":667,"lih":765,"lie":274,"lif":220,"ma ":6156,"luž":388,"mb ":135,"mac":507,"mah":62,"maj":3291,"mak":294,"mad":706,"mag":926,"mar":1529,"mas":537,"mal":1048,"mam":116,"man":2972,"maz":61,"mat":4492,"mba":322,"mbi":346,"mbe":1994,"mbr":581,"mbo":391,"mbn":601,"me ":3191,"mbu":131,"mde":94,"med":6987,"meg":105,"mec":75,"met":4065,"mev":70,"mes":5031,"mer":4601,"mem":950,"mel":2712,"meo":80,"men":7879,"meh":365,"mek":1496,"mej":987,"mez":436,"maÄ":311,"mfo":74,"luz":65,"lva":240,"lve":110,"lvi":73,"luk":109,"luj":862,"lun":128,"lum":243,"lut":187,"lus":353,"ly ":206,"loÅ¡":1311,"ltä":132,"lož":762,"lza":60,"luÄ":58,"luÅ¡":63,"mpi":770,"mpe":599,"mpo":176,"mpl":480,"mpu":59,"mpt":60,"ms ":130,"mog":546,"mob":319,"mod":647,"mon":864,"mok":221,"moj":62,"mom":240,"mol":590,"mov":1124,"mor":2946,"mos":831,"mot":853,"mou":135,"mpa":312,"moz":59,"mre":315,"mrl":143,"mrt":287,"mu ":1595,"miÅ¡":516,"moÄ":1268,"mso":95,"msk":4107,"moÅ¡":595,"my ":71,"mur":404,"mus":157,"mut":85,"mul":1968,"mun":583,"muz":183,"dža":1295,"mi ":4700,"dži":218,"meÄ":117,"maž":60,"min":1835,"eÅ¡n":268,"mio":71,"mil":655,"mim":87,"mir":615,"mis":564,"mit":679,"eÅ¡t":114,"miz":89,"mic":185,"eÅ¡a":172,"eÅ¡e":255,"mie":124,"mid":113,"eÅ¡k":2738,"mik":628,"mij":927,"maÅ¡":150,"eÅ¡i":154,"mih":245,"mo ":2844,"mlj":1176,"mle":124,"mla":591,"mki":710,"mka":116,"mm ":88,"eÅ¡Ä":404,"miÄ":519,"mni":612,"mnm":116,"mno":883,"mna":379,"mne":300,"meÅ¡":231,"mma":103,"mme":132,"ÄŒe ":73,"ÄŒep":77,"ÄŒeÅ¡":144,"ÄŒrn":256,"rža":2068,"rže":110,"rži":130,"Äa ":1709,"Äal":131,"Äam":203,"Äan":1056,"vÄn":97,"Äar":720,"Äas":2119,"Äaj":758,"Äak":128,"vÄe":97,"vÄa":256,"zre":618,"uÄe":431,"uÄa":350,"zra":1527,"Äe ":2812,"uÄu":301,"zro":341,"uÄn":253,"uÄi":543,"Äat":86,"Äav":96,"Äba":88,"Äbe":68,"víl":96,"ziÅ¡":265,"zte":155,"Äeg":102,"Äen":2815,"Äem":362,"Äel":601,"Äek":421,"Äev":1119,"Äet":926,"Äes":137,"Äer":105,"zto":106,"Äep":104,"zse":225,"zu ":354,"zst":173,"zva":406,"zvi":2202,"zve":1702,"Äi ":940,"zvr":346,"zvo":918,"zuj":323,"Äez":81,"zur":194,"zul":378,"zum":402,"zun":254,"zus":115,"Äij":767,"Äih":292,"Äic":410,"Äk ":142,"Äit":716,"Äis":78,"Äin":8235,"Äil":961,"Äim":249,"Äko":555,"Äkr":195,"Äka":504,"zzo":59,"Äke":298,"Äki":315,"zza":68,"Äjo":263,"Äju":736,"Äja":509,"Äje":920,"Äji":637,"ÄeÅ¡":144,"Älo":587,"Äo ":156,"Äle":236,"Äla":635,"Äob":73,"ÄiÄ":263,"Äić":71,"Äni":3672,"Äno":2056,"Äna":1890,"Äne":3797,"ÄiÅ¡":160,"zgl":160,"zi ":788,"zaÄ":773,"zha":357,"zgu":93,"zgr":400,"zgo":1653,"zej":178,"zdr":1230,"zdj":83,"zdo":198,"zdn":338,"zet":178,"zen":780,"ván":80,"zem":1646,"zel":858,"vár":138,"zer":530,"ze ":1270,"zbo":448,"zbi":532,"zbu":82,"zbr":206,"zda":703,"zdi":107,"zde":1030,"zab":392,"zad":779,"zac":850,"zaz":93,"zd ":136,"zbe":87,"zai":81,"zaj":329,"zag":420,"zah":1609,"zam":289,"zan":1549,"zak":553,"zal":882,"zar":791,"zap":901,"zav":812,"zas":719,"zat":657,"zod":127,"zob":347,"zor":519,"zom":118,"zon":2846,"zol":213,"zof":467,"zpe":152,"zpa":113,"zoz":300,"zov":527,"zpr":114,"zpo":448,"ال":104,"zo ":291,"zma":791,"zmn":81,"zmo":266,"zme":1501,"zmi":301,"zna":6251,"zmu":231,"zno":574,"ziÄ":152,"rÅ¡Ä":1404,"zne":577,"zni":1693,"zka":85,"zko":152,"zkl":76,"zki":60,"zku":97,"zla":344,"zli":1716,"zle":63,"zlo":375,"zho":1743,"rÅ¡a":147,"zia":58,"rÅ¡e":205,"zid":236,"zic":70,"zij":1942,"rÅ¡j":95,"zaÅ¡":153,"rÅ¡i":594,"rÅ¡n":88,"zin":229,"zim":197,"zil":483,"zik":2477,"rÅ¡k":1367,"zio":176,"zir":1419,"zis":483,"zit":356,"ziv":530,"zja":105,"zje":158,"yst":102,"ysi":60,"yro":69,"yon":68,"za ":8558,"ye ":58,"yer":80,"ya ":130,"yar":104,"yku":71,"yle":59,"yi ":114,"yje":66,"yja":58,"Ù† ":69,"ožu":71,"ože":1171,"oža":441,"ožb":309,"ožn":777,"oži":853,"ožj":690,"ožg":76,"ož ":115,"té ":62,"tät":153,"ći ":280,"xan":70,"oÅ¡Ä":370,"wn ":129,"ws ":103,"rÄe":97,"rÄi":287,"wor":59,"wer":110,"wel":68,"nže":273,"oÅ¡ ":92,"wis":86,"oÅ¡t":692,"oÅ¡i":152,"oÅ¡e":119,"wic":65,"oÅ¡o":62,"oÅ¡n":419,"win":68,"oÅ¡k":1384,"oÅ¡a":203,"vzo":115,"vzn":60,"vzr":280,"vzp":183,"vzg":93,"vze":217,"vzd":233,"vrÅ¡":1631,"vzh":1577,"vza":88,"wal":60,"war":181,"viÅ¡":902,"vrt":231,"vrs":1856,"vrn":62,"vro":1339,"vri":65,"vrh":387,"vre":731,"vra":482,"vso":118,"vst":1541,"vse":2779,"vsk":1809,"vsi":97,"vu ":881,"vsa":763,"vto":922,"vtr":73,"voÄ":112,"vts":62,"vul":228,"via":94,"vk ":203,"vio":100,"vir":2583,"vik":86,"vil":3700,"vim":485,"vin":3336,"vig":158,"vih":947,"vaÅ¡":4759,"vij":2768,"vic":1147,"vid":829,"vie":118,"nÅ¡e":75,"vja":192,"viz":807,"nÅ¡t":314,"vit":1128,"vis":1138,"veÄ":3002,"vje":763,"vka":371,"vju":170,"vko":123,"vke":182,"vkl":325,"vla":601,"vle":130,"vlo":204,"vlj":4644,"vo ":5275,"vme":74,"rén":67,"veÅ¡":647,"vež":91,"vne":3166,"vna":1676,"vno":7416,"vić":274,"vnj":103,"vni":6241,"nÅ¡Ä":326,"viÄ":1157,"vob":458,"vod":2329,"vog":94,"voj":6252,"vol":768,"vok":302,"von":637,"vom":445,"vor":1714,"vot":255,"vos":442,"vov":212,"voz":481,"vpi":58,"vpl":277,"vpr":143,"vgu":581,"vi ":4563,"vaÄ":184,"vey":59,"vez":2827,"ver":4490,"ves":828,"vet":5958,"vdo":109,"vej":290,"veh":446,"veg":583,"rán":91,"ven":15616,"vem":1214,"vel":2593,"vek":749,"ved":1600,"vec":1132,"vcu":59,"vca":96,"ve ":3712,"vci":431,"vce":194,"val":4949,"vak":400,"van":6929,"vam":248,"var":2538,"vat":726,"vas":1162,"vaz":61,"vac":503,"vad":923,"vai":77,"vaj":1562,"vah":328,"va ":7766,"uzi":197,"urÅ¡":301,"uze":286,"uza":111,"urÄ":177,"ux ":175,"uva":93,"uve":379,"uvr":1131,"usl":98,"usm":167,"usk":1090,"usi":676,"use":332,"usa":242,"usu":60,"ust":3507,"uss":205,"usp":414,"uso":223,"usn":197,"utl":65,"utn":316,"uth":139,"uti":197,"ute":579,"uta":264,"utt":69,"uts":102,"uto":194,"utr":115,"us ":1853,"ut ":342,"urb":124,"ura":3309,"urd":58,"ure":1237,"urg":652,"urj":220,"uri":925,"url":67,"urk":122,"urm":63,"urn":892,"uro":433,"urs":478,"urt":136,"uru":97,"ury":61,"upa":4858,"ur ":618,"upi":1896,"upe":339,"upo":4232,"upr":4636,"upl":66,"upn":1389,"umr":152,"umu":107,"umi":296,"umo":158,"uma":367,"umb":237,"ume":1232,"unt":159,"uns":232,"unk":535,"unj":73,"uni":1649,"uno":147,"unc":267,"und":490,"una":1305,"ung":160,"une":190,"up ":120,"uks":65,"ukr":140,"uku":79,"ukt":476,"uko":214,"ukn":60,"ukl":92,"uki":118,"ukc":123,"uke":60,"um ":819,"uka":277,"ulu":100,"ult":2100,"uls":66,"ulo":296,"ull":127,"ulk":70,"ulj":355,"uli":1131,"ule":1791,"ulf":70,"uld":69,"ula":1090,"un ":304,"ukv":294,"uig":66,"mÅ¡k":1972,"uil":118,"uin":78,"uir":59,"uis":202,"uk ":155,"uje":4627,"uji":154,"ujo":137,"ujs":75,"uit":156,"ul ":297,"uja":181,"ugh":94,"ugi":1041,"lži":298,"uge":696,"lžn":64,"ugo":2841,"ugl":86,"uga":776,"uhi":80,"uho":1525,"ugu":295,"uha":183,"uj ":60,"uda":313,"ude":487,"udj":82,"udi":5813,"udn":106,"ubo":133,"ubn":112,"ubs":149,"ubr":361,"uca":72,"ue ":195,"uce":114,"uci":418,"uch":129,"uck":88,"uer":142,"ues":152,"uh ":100,"uds":601,"udo":411,"ug ":215,"ued":59,"uen":108,"uel":148,"ub ":374,"uar":2656,"ual":170,"uan":113,"ubi":368,"ubl":4817,"ube":341,"uba":207,"ud ":197,"trž":74,"ty ":183,"tvu":608,"tvo":3047,"tve":2705,"tvi":733,"tva":3299,"tur":2565,"tus":332,"tut":264,"tuj":422,"tul":174,"tun":135,"tum":104,"tub":74,"tua":118,"tud":5466,"tuc":67,"tug":184,"tz ":159,"toÅ¡":66,"ts ":250,"tiÅ¡":235,"tmá":241,"trd":395,"tre":2501,"oÄe":1653,"tt ":133,"oÄa":1047,"tra":4316,"trj":88,"oÄj":1644,"oÄk":616,"oÄl":103,"će ":102,"trm":72,"trg":328,"tri":4216,"oÄi":1373,"trs":733,"oÄu":60,"tru":1087,"trt":467,"tro":4048,"trn":84,"oÄn":740,"tu ":1700,"try":92,"tsc":117,"tsk":2341,"tta":143,"tte":299,"tti":218,"tto":129,"ttp":71,"tts":81,"toÄ":930,"tma":1947,"to ":6420,"tmo":86,"tmi":151,"teÅ¡":130,"tež":380,"tni":5182,"tne":1620,"ća ":63,"tp ":71,"tna":1556,"tiÄ":5076,"tno":2333,"tod":350,"toc":762,"toj":799,"toi":353,"toh":82,"tog":270,"tob":807,"tou":124,"tov":5531,"tos":1124,"tot":329,"toz":100,"tom":1530,"ton":1758,"tok":2514,"tol":3650,"tor":3549,"top":2051,"oÄ ":359,"tij":686,"til":753,"tik":2804,"tif":115,"tie":119,"tih":1408,"tig":120,"tir":1113,"tit":785,"tis":2436,"tin":2891,"tim":668,"tip":613,"tio":1015,"thu":192,"tia":170,"tib":79,"tic":1448,"tid":147,"tji":207,"tju":277,"teÄ":443,"tjo":443,"tiz":729,"tiv":1588,"tje":1033,"tja":867,"tki":1874,"tko":496,"tku":163,"tka":483,"tke":192,"tlj":122,"tli":831,"tlo":345,"tla":447,"tle":327,"tem":4827,"ten":1364,"teo":775,"tep":103,"tei":263,"tej":504,"tek":2989,"tel":4105,"tef":109,"teg":1181,"teh":652,"tea":89,"teb":91,"tec":139,"ted":342,"th ":436,"tez":323,"tev":3243,"tet":1626,"tes":590,"ter":10610,"ti ":9411,"tho":148,"the":481,"thi":96,"tha":151,"zÅ¡l":321,"zÅ¡i":325,"zÅ¡e":149,"Živ":99,"yÅ‘r":92,"Žel":286,"ža ":604,"Žup":1073,"žko":65,"žle":58,"žlj":64,"žju":249,"žke":66,"žin":1764,"žim":160,"žil":222,"žir":92,"živ":1723,"žit":230,"žis":155,"žja":222,"žje":479,"žji":178,"žic":387,"žig":82,"žij":171,"žiÄ":188,"žnj":265,"žni":1372,"žno":1191,"žna":269,"žne":657,"žo ":93,"žeÅ¡":277,"že ":817,"žbi":268,"žbe":388,"žbo":86,"žav":2001,"žba":234,"žaj":196,"žal":97,"žan":339,"žar":1500,"žga":114,"ži ":1033,"žev":1038,"žej":69,"žek":75,"žel":1057,"žem":871,"žen":2078,"žef":104,"ždi":82,"žuž":106,"užb":674,"uže":844,"užu":116,"uži":1380,"užn":1159,"žuj":195,"žup":4080,"žiÅ¡":141,"žu ":73,"vÅ¡Ä":108,"vÅ¡k":96,"vÅ¡e":78,"vÅ¡i":66,"yír":60,"uÅ¡Ä":304,"uÅ¡t":314,"uÅ¡n":118,"uÅ¡k":172,"uÅ¡i":260,"uÅ¡e":213,"uÅ¡a":205},"n_words":[5788075,6773679,5606921],"name":"sl","type":"latin","flags":["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/so.json b/contrib/languages-data/so.json
new file mode 100644
index 0000000..1be8d8b
--- /dev/null
+++ b/contrib/languages-data/so.json
@@ -0,0 +1 @@
+{"freq":{"YO ":13,"jec":34,"jee":32,"D":313,"E":183,"F":66,"G":214,"A":673,"B":249,"C":240,"L":152,"M":367,"N":163,"O":122,"H":180,"I":236,"J":129,"K":173,"U":82,"T":107,"W":226,"V":11,"Q":76,"P":22,"S":486,"R":114,"Y":96,"X":120,"Z":10,"f":458,"g":2154,"d":5233,"e":4497,"b":2102,"c":900,"a":24510,"n":3878,"o":5982,"l":3786,"m":2460,"j":397,"k":2897,"h":3132,"i":6615,"w":2306,"v":27,"u":3829,"t":1545,"s":2871,"r":2895,"q":718,"p":77,"z":23,"y":3607,"x":1698,"jaa":13,"jab":16,"jar":10,"jam":12,"Xam":11,"joo":14,"Xas":10,"jis":14,"jir":95,"jii":13,"jid":17,"jo ":15,"Far":12,"isk":69,"ism":12,"isl":25,"iso":22,"isu":42,"ist":67,"ita":17,"is ":71,"ion":20,"ir ":84,"irs":56,"irt":28,"iro":22,"irk":32,"iri":56,"isi":32,"ish":96,"ise":18,"isb":17,"Wux":23,"isa":134,"ire":16,"ira":131,"iyi":10,"iyo":394,"iya":423,"iye":65,"ixi":16," l":598," m":880,"kii":161," n":189," o":537," h":365," i":795," j":267," k":1328," d":1214," e":328," f":95," g":401," a":1317," b":593," c":361," y":296," x":283," u":599," t":376," w":1834," q":291," p":20," s":807," r":112,"km ":14," J":125," K":142," H":119," I":161," N":93," O":34," L":81," M":322," B":217,"khd":24," C":229,"kha":11," A":275," F":59," G":169," D":236," E":41," Z":10," Y":40," X":90," S":438," R":66," Q":69," P":18," W":211," U":33," T":83,"kee":20,"key":11,"kh ":38,"Web":10,"Waa":56,"ku ":434,"kor":15,"Wax":40,"koo":94,"War":17,"XEE":11,"مد":16,"Gal":22,"Ùˆ":25,"ÙŠ":76,"Ù":13,"Ù‚":12,"Ù„":77,"Ù…":62,"Ù†":31,"Ù‡":13,"د":46,"Ø­":26,"ب":37,"Ø©":21,"ا":98,"Ø£":11,"ع":29,"Ø´":21,"س":23,"ر":49,"kar":49,"kas":30,"kan":49,"kal":143,"kam":32,"kad":48,"kac":14,"kab":10,"kaa":81,"ka ":1268,"A ":83," Ga":53," Ge":18,"Da":59,"DU":11,"Cu":18,"Co":13,"DE":11," Fi":17,"Ce":13,"DH":15,"Ci":23," Ha":35,"Du":13,"EY":13," Go":61," Gu":12,"EG":11,"De":45,"EE":45,"EL":14,"Di":29,"Dh":36,"H ":16,"GA":19,"Fa":23," IY":15,"Er":12," Ho":29,"ha ":334," Hi":37,"Ge":18," Ji":25,"Ga":53,"حم":18,"HA":35,"I ":13," Ja":63," KA":16," Is":32," It":29,"GM":12," In":35,"Fi":17,"ham":43,"han":102," Ka":28,"hal":48,"haw":17,"hax":44,"haq":58," Ki":19,"har":45,"has":76," Kh":10," Ju":19,"hah":12,"hab":77,"haa":189,"had":144,"hac":36,"AS":15,"AR":23," MA":17,"AX":27," La":22,"AY":15,"BA":11," Li":11,"C ":10,"AD":43,"AA":51,"AB":14,"AG":11," Ko":23,"AH":23,"hay":333,"AL":37," Ku":26,"AM":13,"AN":35," Ma":180,"Ax":18,"Ar":12,"D ":22,"بن":10," Mi":27,"Ba":101,"CA":15,"Af":65,"بد":10,"he ":25,"Aa":22,"Ab":33,"Ad":10,"Am":17," Lu":25,"Al":38," Ne":14,"Bu":30," Na":32,"Ca":127,"DA":43,"E ":30,"Bi":19,"Be":25,"hda":27,"Bo":30,"Hin":18," Mu":78,"hel":22,"Ku":26,"hee":112,"Ko":23,"hey":26,"hex":72,"Li":11,"N ":26,"her":11,"MA":41,"La":22,"Lu":25,"hi ":27,"Mi":27,"NK":10,"ال":51,"O ":34,"NA":12,"Ma":180,"Mu":79,"Ne":14,"Na":32," Am":16," Al":38,"Nu":16," Af":65,"No":12,"OO":18," Ad":10," Aa":22," Ab":33," Ba":101," CA":12," Ax":18," Ar":12,"hig":23," Be":25,"hid":12," Bi":19,"hin":40,"Go":61,"him":17,"Gu":12," Bo":30,"hii":170," Bu":30,"his":24,"hir":31,"Ha":35," Ca":127,"Hi":37," Ce":13," DE":10," Ci":23,"IN":12,"Ho":29," DH":13,"IS":10," Co":12," Cu":18,"IY":20," Da":59," Di":29," Dh":36,"In":36," De":45,"Is":32,"It":30,"Ja":63,"KA":33," Du":13,"Ji":25," Er":12,"Ju":19,"LA":35,"Ka":28,"Kh":10,"ho ":53,"Har":14,"Ki":19,"LE":16," Fa":23,"gma":64,"go ":32," Xi":13," Xa":51,"UU":11,"yuu":26," Wu":23,"To":11,"Th":10," Wi":15," We":12,"Ta":37," Wa":133,"St":13,"Su":23,"Wu":23,"gob":97,"Wi":16,"Wa":133,"XA":19,"We":12,"XE":12,"Y ":18,"yst":29," Yu":14,"yso":15," Ya":10,"WA":26,"gmo":41,"ysa":93,"Qa":26,"Qo":17," Ù…":12,"RA":10,"S ":18," ع":21," ا":48,"goo":52,"R ":20," ب":13,"gsa":14,"gu ":229,"Si":17,"Sh":86,"gsi":12,"So":180,"Ru":12,"U ":11,"Sa":70,"TA":13,"Re":13,"SH":11,"Ro":11,"yoo":24,"Qu":16,"SA":16,"Ra":20,"gud":22," Nu":16," No":12,"gta":43," Ra":20," Qu":16,"b ":130," Ro":11," Re":13,"guu":20,"gun":12,"a ":5909," Qo":17," Qa":26,"شي":10," Su":23," St":13," Ta":37,"Ya":10," Th":10,"Yu":14," To":11," Ru":12," Sa":70,"Xa":51,"YO":15," Sh":86," Si":17,"Xi":13," So":180," WA":20,"ري":12,"Gob":48," ja":60,"i ":853,"ye ":36,"ian":11," iy":365," ji":127,"ge":93," je":47,"ga":1135,"fk":16,"Ing":16," im":15," in":148," il":54," ii":23,"ic ":14,"fi":49,"fr":45,"fu":47,"ft":29,"fo":18," is":155," ka":688," kh":13,"hd":44,"he":286," ki":46," ke":11,"ha":1580,"gn":11,"gm":108," jo":14,"gl":15,"gi":72,"id ":171,"gu":305,"iba":32,"gt":52,"gs":27,"gr":15," ju":17,"go":196,"du":188,"dw":36,"dy":13,"g ":83," ha":190,"ea":16,"eb":72,"yee":61,"ec":51," he":28,"ed":360,"de":252,"dd":113,"di":494,"dh":632,"dk":189,"dl":33," go":117,"do":234,"dn":22," gu":55,"ia ":36,"ex":102,"ey":554,"fa":110,"h ":441," id":15,"fe":17,"eh":54,"ib ":32,"eg":202," hi":20,"ee":1263,"el":242,"ek":35," ho":120,"ei":12,"yey":26,"en":172,"em":31,"et":26,"es":93,"er":287,"ya ":266,"ca":427," ni":37,"e ":881," ne":15,"bs":21," na":54,"br":36,"bu":104,"bt":55,"bn":18,"bo":234,"bk":30,"bl":13," mu":48,"ig ":10,"bi":355,"bb":15,"bd":41,"be":201,"db":11,"da":2087," og":18,"f ":98,"cy":18," of":16,"cu":41,"ct":11,"cs":27,"co":62,"cm":24,"cn":13,"cl":19,"ci":73," nu":10,"ch":33," no":73,"ce":64,"cd":20,"yad":111,"yag":10," le":91,"c ":51,"yaa":287," la":334,"icm":22," ku":465,"ici":14," km":14,"ica":25," ko":88," me":49,"az":10,"ay":1458,"idu":13," mi":187,"ba":817,"d ":893,"at":134,"as":580,"yd ":29,"ido":43,"ar":1307,"aq":237," ma":590,"ax":1066,"aw":157,"idk":12,"yay":52," lu":25,"ak":76,"al":1647,"idi":35,"yaw":11,"idh":19,"ai":29,"aj":59,"yar":45,"am":590,"an":1951,"yaq":50,"yan":13,"ac":260,"ida":140,"ad":2243,"aa":4171," lo":138,"ab":630,"ag":664,"ah":1152,"yah":134,"af":128,"iib":15,"nu":38,"iic":11,"nt":263," af":45,"ns":59," ah":473," aa":208,"iig":13," ab":31,"iid":50,"no":160,"nn":18," ad":49,"q ":34," am":103," an":18,"iik":48,"iin":164,"ny":57,"yka":17,"iil":93," al":21,"iim":26,"iis":199,"iir":65,"of":78,"iiq":14,"oc":29," ax":10,"od":156," ar":26,"ob":291," aq":21," as":29,"om":340,"on":186," ba":344,"ok":16,"ol":273," ay":246,"og":129,"il ":80,"ot":41,"os":90," bi":107,"op":10,"oo":1738," be":63,"or":236,"oq":49,"yn ":105," bo":34,"r ":475,"ox":10,"ow":125,"oy":128," bu":35,"pa":14," ca":238,"im ":21,"ika":50,"lo":386,"ige":10,"lm":39,"ll":110,"ls":27,"iga":247,"ii ":339,"lw":14,"lu":48,"igi":31,"yo ":488,"ly":56,"igu":13,"igt":12,"o ":2012,"ma":1465,"mb":52,"mh":21,"me":199,"mk":39,"mi":333,"mp":19,"mo":102,"yna":98,"mu":85,"ihi":82,"yni":14,"na":851,"nb":30,"yne":30,"nc":10,"nd":137,"ne":107,"nf":30,"ng":58,"ynt":29,"ni":213,"nk":312,"nl":21,"imo":20,"ju":17,"jo":31," ee":295,"imi":21,"ki":203,"kh":95,"ke":48,"ind":29,"ina":80," fa":48,"yga":15,"ka":1778,"yi ":19,"m ":103," fu":10,"ino":13,"kt":20," fo":12,"ku":558,"int":102,"ins":10,"ko":130,"ine":14,"ing":16," fi":17,"ini":10,"km":16,"ink":82," ge":36,"li":577,"lk":332,"le":352," ga":186,"ld":23,"lg":22,"inu":15,"la":1306,"lb":52,"iny":13,"n ":1478," co":22,"ht":11,"hu":92,"ikh":54," ce":15,"hi":387,"hn":16,"ho":217," ci":36,"ila":160,"id":471,"ic":103,"yin":59,"ib":108,"ia":61,"ih":88,"in ":262,"ig":350," da":424,"if":21,"yih":49,"yig":21," cu":34,"hy":12,"k ":24,"iq":21," do":45,"ilo":13,"ir":438,"is":630,"it":49,"ill":18,"ilk":32,"ix":28,"ilm":12,"ii":1062,"ij":21,"ik":134," de":120,"ili":51,"il":385,"im":170,"in":663,"io":30," di":70,"yir":13," dh":511,"ima":76,"je":69,"ji":178,"iy":896," du":39,"l ":398,"ja":82,"xi":123,"xo":56,"xm":34,"xw":27,"xu":185,"xb":18,"xa":850,"xe":161,"xd":67,"wg":11,"wi":81,"how":15,"wl":60,"wo":26,"wu":102,"hog":13,"y ":1137,"wa":1722,"wd":13,"hoo":55,"we":185,"hor":60," yi":55," yu":13,"uy":12,"ux":164,"uw":34,"uu":720," ye":13,"ve":10," ya":211,"x ":140," xo":33,"uj":15,"uk":28,"ul":200,"uf":20," xi":90,"ug":210,"uh":16,"uq":90,"ur":259,"hna":12," xu":39,"us":114,"ut":54,"um":90,"un":214,"tu":47,"ub":104,"ua":11,"ud":145,"uc":17," xe":16,"w ":59," xa":103,"to":175,"hul":37,"tr":25,"te":120,"ti":246,"th":37,"ta":784,"su":111,"ss":19,"st":173,"sw":12,"sl":47,"sk":106,"sm":25,"so":371,"sr":10,"sc":17,"se":101,"sh":456,"ÙŠ ":20,"xme":19,"si":404,"xma":13,"u ":1296,"sa":722,"sb":21,"rr":20,"rs":115,"rt":160,"ru":77,"rw":11,"rx":11,"ry":27,"ro":144,"rn":40,"rm":32,"rl":22,"rk":200,"ri":397,"hu ":11,"rg":35,"re":258,"rd":49,"rc":12,"rb":25,"ra":754,"t ":51,"qu":35,"qs":10,"xoo":44,"qo":163,"IYO":15,"qi":33,"qe":23,"qa":334,"qd":61,"s ":240,"pu":15,"pr":14," ru":12," u ":194," sa":221," se":17," si":157," sh":112," so":259," qu":21,"xya":13," ra":48," re":33,"Ù† ":17," ro":11," qe":14," qa":168," qo":69," qi":18," oo":464," or":10,"huu":29," wa":1582," we":88," wo":12," wu":102," wi":39," uu":195,"xud":12,"xuu":133,"Hoo":12," tu":36," us":16," ur":10,"Ù… ":11," um":12," un":11," ug":131,"yg":19," ta":231,"ye":133,"yd":48,"ya":998,"yb":27,"xwe":21,"xy":17," su":25,"yu":34,"ys":166," to":18," th":15," ti":62,"yo":522,"yn":280," te":11,"yk":19,"yi":189,"fee":11,"xey":58,"xee":54,"far":32,"fad":21,"faa":24,"Suu":12,"Axm":14,"xir":17,"xis":13,"xil":26,"xii":17,"xid":14,"xig":24,"Sta":10,"xa ":169,"eyb":17,"eya":63,"eys":74,"Tal":11,"eyn":163,"eyo":14,"eyk":10,"xda":51,"eyd":16,"eye":14,"exa":10,"exd":12,"exe":51,"xe ":46,"xar":38,"Ban":18,"Baa":14,"Bad":22,"xam":54,"xan":16,"Bar":23,"xay":166,"xba":16,"xaa":341,"xad":27,"xag":13,"wux":100,"Aas":11,"Shi":22,"She":12,"Sha":50,"ex ":21,"Af ":19,"ey ":159,"er ":103,"es ":21,"eri":33,"ere":30,"era":49,"Afr":32,"esh":28,"esa":10,"ers":11,"ern":14,"ekh":16,"en ":89,"ela":47,"ele":26,"eli":17,"ell":42,"elo":15,"emb":19,"ena":28,"wla":53,"eny":12,"egm":90,"ego":14,"egt":11,"Som":32,"Soo":136,"woq":10,"el ":65,"wda":13,"Buu":11,"Bur":11,"we ":12,"gir":17,"gii":26,"wey":124,"wee":27,"gey":15,"gee":44,"wi ":14,"wis":10,"wii":22,"Sal":11,"gab":12,"gac":45,"gad":26,"DA ":20,"gaa":436,"gar":35,"gay":21,"gal":70,"gan":69,"ga ":388,"San":27,"wa ":22,"Cab":27,"waq":26,"wan":30,"wal":39,"wax":715,"way":45,"Cal":18,"war":52,"was":18,"Car":40,"waa":581,"wad":168,"Bel":10,"fur":37,"Bis":12,"fri":39,"fii":15,"Boo":10,"fka":13,"da ":918,"de ":22,"dad":131,"daa":159,"dab":19,"dal":113,"WAX":16,"dag":65,"dah":101,"dar":51,"dan":291,"dam":39,"day":61,"dax":79,"daw":32,"Cum":10,"dda":74,"dde":11,"ddi":17,"cun":14,"EEY":13,"EEL":14,"EGM":11,"Deg":30,"cyo":15,"uxu":126,"Daa":22,"Dag":10,"Dal":10,"uxa":15,"uun":88,"uul":63,"uum":13,"uug":15,"uud":50,"uux":10,"ux ":12,"uus":29,"uur":74,"uuq":18,"uut":24,"uwa":28,"co ":26,"cma":23,"ush":13,"usi":11,"use":13,"uu ":316,"usu":26,"uso":11,"uti":16,"uta":19,"cod":10,"com":11,"uqa":33,"uqd":36,"ura":37,"ure":10,"uri":31,"urk":17,"urt":32,"uru":37,"ur ":39,"csi":14,"uma":56,"unt":32,"unk":27,"uni":11,"una":85,"cel":30,"uka":13,"cee":17,"uls":10,"ulo":20,"ull":14,"ulk":27,"uli":14,"ule":16,"ula":26,"un ":29,"che":12,"ul ":36,"ciy":12,"cii":28,"uga":40,"ugu":128,"ugs":11,"ed ":184,"ebi":20,"uf ":13,"uda":33,"udi":12,"eb ":12,"udu":37,"ug ":18,"ega":53,"ub ":32,"eek":25,"een":99,"eel":138,"eem":18,"eeb":23,"eeg":65,"eed":229,"eey":113,"eh ":42,"ees":56,"eer":157,"edk":18,"edi":12,"ede":22,"eda":72,"uba":39,"ubb":11,"edu":15,"ud ":36,"edo":11,"ecl":12,"ece":25,"ee ":319,"dwe":25,"dwa":11,"duu":57,"tuu":22,"doo":96,"dow":37,"tri":10,"The":10,"dna":12,"to ":75,"Dhe":14,"Dhu":12,"dun":12,"dul":20,"dug":23,"too":69,"du ":45,"tii":59,"tig":10,"tir":66,"dha":335,"tio":16,"tic":26,"dhu":33,"dib":25,"dhi":112,"dhe":122,"dho":21,"der":19,"dex":18,"dey":16,"dee":48,"deg":96,"den":15,"di ":38,"dle":11,"dla":17,"tee":36,"dku":14,"dki":33,"do ":77,"ter":36,"diy":39,"din":26,"ti ":29,"dir":60,"dis":51,"dig":42,"dii":165,"dil":12,"dka":134,"the":16,"rga":14,"ri ":48,"rge":14,"rey":42,"ree":110,"rda":15,"rdh":16,"re ":77,"rco":10,"rax":25,"ray":99,"rar":15,"ras":44,"rat":10,"rba":11,"rah":41,"ran":54,"ram":17,"rak":12,"rab":82,"raa":165,"rad":87,"rs ":11,"roo":48,"rna":16,"rne":11,"rni":10,"ro ":63,"rma":23,"Nab":15,"rla":13,"rku":10,"rko":10,"rki":41,"rke":18,"rka":117,"riy":58,"ris":28,"rig":31,"rii":110,"rik":46,"rin":21,"ric":16,"rya":13,"rur":10,"run":18,"ruu":10,"ry ":11,"rsi":16,"rsa":63,"rsh":15,"rta":110,"rto":18,"rte":11,"rti":11,"rub":12,"saa":120,"sab":11,"sad":52,"sag":23,"sah":11,"sal":49,"sam":47,"sbi":14,"san":191,"sas":14,"sar":33,"say":43,"sa ":99,"sha":242,"sho":46,"she":41,"shi":83,"si ":68,"siy":42,"sid":91,"shu":10,"sil":13,"sim":38,"sii":82,"sig":32,"se ":61,"sh ":17,"see":14,"sow":16,"som":59,"soo":214,"soc":14,"su ":25,"sla":30,"sku":37,"ska":59,"so ":55,"sma":15,"حمد":15,"ste":15,"sta":66,"sto":28,"sti":41,"sub":11,"suf":12,"sug":13,"sul":11,"suu":22,"tal":42,"tag":10,"tah":87,"taa":194,"tad":13,"tay":60,"tar":33,"tan":31,"tam":13,"te ":13,"ta ":272,"bka":23,"biy":71,"bis":28,"bir":12,"bil":48,"bin":31,"big":38,"bii":37,"bo ":47,"bol":129,"bna":15,"boo":24,"bba":12,"be ":19,"ban":61,"bal":43,"bah":27,"bad":232,"baa":96,"bab":12,"bay":35,"bax":34,"bas":10,"bar":156,"bdi":25,"bdu":11,"bi ":69,"bee":145,"ber":11,"bey":12,"ca ":55,"car":35,"cas":13,"can":24,"cay":13,"cab":20,"cad":53,"caa":145,"cal":33,"cag":16,"bri":13,"bra":15,"bsa":11,"bta":33,"bti":13,"bur":20,"bul":12,"buu":52,"aka":19,"am ":40,"aki":23,"aji":27,"ajo":16,"qa ":12,"al ":136,"ahi":41,"qar":20,"qay":16,"aho":10,"qad":44,"qab":47,"qaa":149,"ahd":20,"qan":14,"qal":17,"ahe":26,"aha":697,"agm":13,"agt":24,"agu":76,"ago":29,"aq ":22,"qdi":38,"qda":17,"any":23,"ano":51,"ann":10,"ant":70,"ans":32,"ane":21,"ang":10," ال":46,"ani":87,"ank":185,"ana":385,"anb":26,"and":92,"amu":23,"amo":10,"amk":32,"amh":19,"ami":82,"ame":93,"amb":16,"ama":257,"aly":20,"qey":14,"alo":160,"alm":17,"all":22,"alk":165,"alg":17,"ali":424,"ald":14,"ale":110,"ala":480,"alb":42,"an ":924,"aba":194,"abd":37,"abe":56,"abi":146,"abk":18,"abo":40,"abt":38,"abu":36,"aca":130,"aab":114,"aac":13,"aaa":15,"aaf":38,"aag":64,"aad":398,"aaj":28,"aak":21,"aah":75,"aan":742,"aal":743,"aam":113,"aas":211,"aar":259,"aaq":41,"aaw":32,"aat":37,"aay":89,"aax":19,"ad ":334,"qiy":15,"ac ":19,"aa ":1110,"qii":10,"ab ":33,"afr":11,"aft":15,"afi":18,"aga":458,"age":12,"ah ":325,"afa":38,"ado":85,"adl":23,"adk":153,"adn":12,"adh":26,"adi":223,"add":96,"ade":66,"ag ":29,"adw":22,"adu":44,"aci":16,"ace":10,"Qar":12,"acd":15,"ada":1138,"af ":19,"acy":15,"acs":19,"qor":48,"qoo":60,"qof":24,"axi":13,"axm":15,"axo":15,"axu":15,"axa":702,"axb":16,"axd":50,"axe":90,"ayi":11,"ayo":52,"ayn":115,"ays":84,"ayu":13,"axy":16,"axw":26,"ayb":10,"aya":151,"ayg":11,"ayd":32,"aye":26,"ba ":84,"qur":24,"at ":11,"arg":25,"are":96,"ard":30,"arb":14,"ara":357,"aro":72,"arn":19,"arm":17,"arl":10,"ark":135,"ari":153,"aru":20,"ars":39,"art":72,"asa":99,"ary":14,"asi":106,"ash":156,"ase":12,"aso":31,"ask":17,"ar ":198,"as ":80,"aqa":111,"aqi":13,"aqo":51,"ax ":98,"awe":20,"ay ":932,"awa":46,"awl":31,"awi":33,"ata":37,"asu":12,"ast":33,"ato":18,"ate":17,"ra ":58,"ati":34,"ngi":20,"ni ":47,"Isl":11,"neh":11,"ng ":11,"nee":16,"nfu":25,"ney":14,"ne ":43,"ndh":18,"ndi":22,"nan":17,"nac":45,"nad":83,"nah":41,"nab":18,"naa":131,"Ito":28,"nbe":15,"nd ":69,"AXE":10,"AY ":10,"nba":11,"AXA":12,"nay":47,"nax":11,"na ":412,"Jab":13,"Jan":13,"Jam":22,"KA ":11,"KAL":10,"nya":38,"AAL":13,"ADA":25,"nuu":21,"nto":13,"nti":37,"nta":176,"nte":24,"nsi":15,"nsa":22,"AHA":14,"noo":67,"noq":18,"nna":11,"ALA":17,"nle":12,"no ":59,"nki":22,"nka":271,"AN ":16,"nii":13,"nih":11,"nig":39,"niy":10,"nis":15,"nim":17,"nin":39,"ogu":24,"oga":60,"Jub":11,"ol ":60,"oco":11,"odi":15,"of ":38,"oda":43,"ofe":10,"LA ":12,"د ":29,"oba":86,"od ":60,"obo":134,"obi":38,"Ø© ":21,"oyi":94,"oya":10,"owl":29,"ow ":45,"ost":14,"ota":10,"ose":28,"os ":15,"oon":114,"ool":98,"oom":198,"oof":13,"oog":60,"ood":123,"oob":124,"or ":39,"ooy":111,"oow":16,"oot":14,"oos":65,"oor":31,"Koo":13,"ore":44,"ori":14,"osa":11,"ort":21,"oqo":37,"oqd":11,"ora":61,"ola":52,"on ":52,"olk":99,"ole":20,"olo":14,"oly":10,"ona":28,"onf":25,"oni":16,"onk":11,"ons":12,"ont":14,"oma":298,"oo ":749,"omp":12,"la ":241,"le ":159,"laa":281,"lab":61,"lac":11,"lad":232,"laf":10,"lah":96,"lag":116,"lal":23,"lan":88,"lam":27,"las":21,"lay":70,"lba":15,"lbe":31,"kuw":22,"kuu":18,"kun":22,"kul":14,"kto":17,"MAD":13,"lom":11,"loo":176,"lmo":12,"lmi":13,"lma":10,"lsh":13,"Luu":11,"li ":92,"lga":16,"ley":29,"leh":35,"lee":98,"lo ":165,"lla":49,"lle":32,"lka":311,"lki":14,"lis":19,"lin":48,"lim":15,"liy":204,"lid":28,"lia":24,"lib":24,"lil":40,"lii":17,"lig":30,"ma ":133,"maa":361,"mac":36,"mah":24,"mad":229,"mag":226,"mar":193,"mas":14,"mal":133,"man":32,"may":23,"max":25,"mba":26,"mbe":10,"me ":19,"med":68,"mee":72,"mey":24,"luq":12,"luu":17,"مد ":15,"lya":33,"lyo":10,"Mar":22,"Mas":10,"Mag":51,"Mad":20,"Maa":17,"Max":25,"moo":35,"muq":17,"muu":16,"mul":10,"Mux":13,"mhu":20,"Muq":24,"Mud":14,"mi ":19,"min":17,"mil":14,"mis":11,"miy":27,"mig":18,"mid":170,"mij":10,"mii":25,"mo ":60,"mka":33},"n_words":[94077,109135,83288],"name":"so","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/sq.json b/contrib/languages-data/sq.json
new file mode 100644
index 0000000..d14e9b5
--- /dev/null
+++ b/contrib/languages-data/sq.json
@@ -0,0 +1 @@
+{"freq":{"D":2951,"E":2593,"F":3211,"G":3160,"A":5867,"B":4180,"C":1638,"L":2905,"M":4987,"N":3399,"O":1229,"H":1921,"I":3066,"J":1447,"K":6663,"U":1057,"T":3497,"W":518,"V":1916,"Q":766,"P":4792,"S":7666,"R":2957,"Y":251,"X":569,"Z":898,"f":14736,"g":26110,"d":50009,"e":167179,"b":16462,"c":10479,"a":127255,"n":113931,"o":70925,"l":52723,"m":56721,"j":47732,"k":46937,"h":74009,"i":153617,"w":760,"v":25346,"u":51367,"t":143796,"s":94347,"r":123043,"q":14863,"p":40169,"z":11328,"y":8479,"x":1327,"Ë":504,"Ç":673,"Fil":198,"ë":138684,"ç":2147,"Fja":191,"Evr":427,"ο":279,"α":259," l":7602," m":19599," n":36778," o":2746," h":2298," i":13440," j":4180," k":16392," d":18386," e":22497," f":7928," g":6344," a":8088," b":5231," c":3040," z":2178," u":2660," t":29267," v":9770," q":6524," p":21985," s":22399," r":6234," J":1246," K":6396," H":1660," I":2018," N":2979," O":928," L":2729," M":4746," B":3697," C":1374," A":5029," F":3007," G":2987," D":2696," E":2074," Z":866," Y":222," X":455," S":7145," R":2508," Q":712," P":4598," W":483," V":1621," U":882," T":3065,"Gje":385," ç":814,"Gji":334," ë":7724,"Gju":218," Ç":648," Ë":309,"Gja":201,"Fra":312,"A ":555,"For":238,"Da":377,"Co":247,"Ch":193,"Du":229,"Do":262,"Dr":303,"De":624,"Di":445,"Dh":185,"Fe":366,"Fa":366,"Ev":475,"El":231,"Ga":322,"I ":544,"Fr":441,"Fo":342,"Fl":194,"Fj":196,"Fi":584,"BA":277,"C ":177,"Au":418,"Ar":625,"At":240,"Ba":1072,"Af":186,"Am":331,"An":632,"Ai":327,"Aj":203,"Al":611,"Bu":398,"Br":503,"Ca":335,"E ":302,"Bi":247,"Be":668,"Bo":499,"Ku":625,"Ky":228,"Kr":523,"Ko":2407,"Le":440,"Li":756,"La":432,"Lu":465,"Lo":267,"Me":757,"Mi":743,"Ma":1783,"Mu":398,"Mo":476,"Nj":248,"Ni":232,"Nd":207,"Ne":392,"Na":320,"No":435,"Gj":1228,"Gr":581,"Go":208,"Gu":206,"Ha":528,"He":303,"II":206,"Hi":261,"Ho":278,"In":443,"Is":481,"It":205,"Ja":385,"Je":182,"Jo":291,"Ju":322,"Ka":1334,"Kj":278,"Ki":240,"Un":312,"Tu":198,"Tr":315,"To":375,"Pë":415,"Th":341,"Ti":609,"Te":638,"Ta":357,"St":538,"Su":265,"Wi":200,"Vi":295,"Va":291,"Ve":412,"Qe":273,"Pu":185,"Pr":997,"Pe":613,"Pa":1058,"Po":615,"Pi":181,"Kë":300,"Se":537,"Si":523,"Sh":3256,"Sk":218,"Sp":262,"So":375,"Ru":267,"Rr":230,"Sa":579,"Re":810,"Ri":264,"Në":821,"Ro":386,"Ra":365,"Gre":313,"b ":407,"a ":28448,"Xh":218,"i ":33796,"ge":656,"ga":7640,"fj":347,"fl":382,"fi":3904,"fs":1052,"fr":816,"fu":1400,"ft":1033,"fo":1696,"bë":2042,"j ":4723,"hf":238,"he":19006,"ha":4203,"gl":638,"gj":7884,"gi":492,"gu":1606,"gr":2543,"cë":728,"go":1368,"du":2130,"dy":756,"g ":1263,"ea":970,"eb":315,"ec":617,"ed":3466,"de":5126,"di":8348,"dh":17111,"dj":1045,"dm":281,"do":4139,"ds":204,"dr":2528,"ew":200,"ex":233,"eu":592,"ev":3916,"ez":1764,"fa":1842,"h ":2770,"fe":1185,"eh":617,"eg":2448,"ef":474,"el":4768,"ek":4482,"ej":2627,"ei":482,"ep":1848,"eo":855,"en":13277,"em":3767,"et":19558,"es":10616,"er":15019,"eq":1060,"ca":1358,"e ":72317,"br":1628,"bu":1107,"bo":1748,"bj":242,"bl":1253,"bi":2183,"be":1978,"da":2581,"f ":429,"co":479,"ck":313,"ci":4788,"ch":654,"ce":1081,"c ":318,"az":1629,"ay":236,"ba":3411,"d ":2270,"at":11806,"as":8047,"ar":20865,"aq":1562,"av":2909,"au":845,"ak":4327,"al":8585,"ai":935,"aj":4288,"ap":2172,"am":3780,"an":16760,"ac":1088,"ad":3850,"ab":980,"ag":1063,"ah":884,"ae":194,"af":1439,"nu":1486,"nt":6015,"ns":1390,"jë":8682,"no":2581,"nn":299,"q ":809,"ny":268,"nx":351,"oe":253,"of":894,"oc":773,"od":2400,"oa":235,"ob":841,"om":4147,"on":11462,"ok":1665,"ol":4317,"oi":918,"oj":2100,"og":2207,"oh":3303,"ot":2941,"os":4887,"ov":2272,"ou":371,"op":2168,"oo":285,"kë":4598,"or":15900,"oq":456,"r ":22499,"oz":751,"pe":3763,"pa":6781,"pl":799,"lë":3066,"po":4293,"pi":2577,"pj":1679,"lo":4348,"hë":4530,"lm":750,"ll":8126,"ls":381,"lu":2680,"lt":1030,"ly":235,"o ":4990,"ma":6825,"mb":3911,"me":13242,"iç":252,"mi":11206,"mj":687,"mp":1128,"mo":2299,"mr":960,"mt":825,"ms":189,"mu":2606,"p ":820,"na":4803,"nc":1647,"nd":14065,"ne":6011,"nf":360,"ng":8085,"ni":9080,"nj":10024,"nk":623,"jy":374,"jv":200,"jt":2182,"ju":2055,"jr":203,"js":193,"jn":989,"fë":409,"jo":2391,"jm":201,"kj":232,"ki":2183,"ke":4651,"ka":7856,"m ":4929,"ky":261,"ks":1435,"kt":3378,"ku":6445,"ko":7127,"gë":1191,"kr":3597,"kl":551,"km":406,"kn":219,"li":13297,"lk":295,"lj":608,"le":5962,"ld":260,"lg":209,"la":6555,"lb":400,"n ":21363,"hr":608,"hs":268,"hp":932,"hq":3267,"hv":498,"ht":18411,"hu":3735,"hj":1397,"hk":4243,"hi":5167,"hn":261,"ho":1827,"dë":2976,"hm":1740,"id":1902,"ic":1580,"ib":803,"ia":5265,"ih":920,"ig":1500,"if":611,"ie":1546,"hy":320,"k ":3634,"iq":788,"ir":3308,"is":14282,"it":23622,"iu":865,"iv":2165,"ij":2183,"eç":454,"ik":8974,"il":7041,"im":10832,"in":18975,"io":3678,"ip":5685,"je":13383,"ji":4180,"iz":2757,"l ":3714,"ja":7433,"xh":647,"të":37539,"z ":666,"së":6850,"y ":1757,"wa":196,"vl":237,"vj":627,"vi":5977,"vr":700,"rë":7665,"vo":819,"uz":609,"uv":188,"ve":12217,"vd":565,"va":1908,"x ":256,"ui":282,"uj":950,"uk":1866,"ul":3717,"ue":2301,"uf":993,"ug":1106,"uh":1399,"uq":312,"ur":9836,"us":3067,"ut":2692,"um":2889,"un":3916,"që":4381,"up":1314,"ty":1124,"tu":7405,"tt":246,"ub":1079,"ua":6884,"ud":1227,"uc":369,"w ":197,"pë":9674,"to":7414,"tm":513,"ts":310,"tr":5518,"te":16922,"tj":1903,"ti":17731,"th":4122,"v ":372,"tb":272,"ta":11255,"su":1269,"sv":254,"ss":508,"st":7650,"sy":189,"sl":488,"sk":843,"sn":215,"sm":700,"sp":784,"so":3394,"sq":194,"sc":195,"se":5448,"sh":35309,"sj":384,"si":11685,"u ":4144,"sa":3279,"rr":4311,"rs":2938,"rt":4337,"ru":3266,"rv":415,"ry":2015,"rq":210,"rp":488,"ro":7859,"në":24491,"rn":818,"rm":3136,"rl":428,"rk":1734,"rj":983,"ri":20960,"rh":205,"rg":2062,"rf":941,"re":16088,"rd":2048,"rc":625,"rb":1536,"ra":14927,"t ":30601,"qy":921,"qu":769,"më":7228,"qj":229,"qi":4293,"qe":2941,"qa":297,"s ":14264,"pt":2183,"pu":2168,"pr":4557,"ps":322,"zë":930,"zg":391,"zh":1019,"zi":2576,"zb":204,"ze":1031,"za":1391,"zy":222,"zu":773,"zo":1184,"vë":1622,"zj":202,"zm":366,"ye":1323,"yt":1297,"ys":927,"yr":1681,"yp":187,"yn":208,"ym":190,"yl":202,"Art":179,"Aut":217,"Bas":348,"Ai ":287,"Ame":229,"Ber":210,"Bot":188,"Ës":295,"Çm":287,"アアア":185,"ër":16534,"çë":180,"ëp":625,"ëm":1771,"ën":7250,"ël":747,"ëz":648,"ëv":1509,"ës":19082,"ët":3696,"ëh":461,"ëj":261,"ëd":282,"ë ":85317,"çm":262,"çi":247,"çe":269,"ça":442,"ç ":230,"Nob":257,"Per":268,"Pas":194,"Par":336,"Pro":243,"Pri":459,"Pre":232,"Ish":253,"Ita":196,"ア":259,"Jug":202,"Ka ":178,"Kal":215,"Kar":206,"Kjo":277,"Kon":248,"Kom":428,"Kos":1014,"Kor":248,"Ky ":207,"Lig":189,"Mal":292,"Mar":396,"Maq":235,"Mad":210,"Min":180,"Ësh":295,"çmi":226,"ëhe":440,"ëm ":968,"ël ":264,"ëll":356,"ën ":3829,"ënt":362,"ëng":554,"ënd":1225,"ëna":195,"ëmi":309,"ëmb":216,"ëdh":247,"Sta":228,"Shk":524,"Shq":1268,"Sht":446,"Ser":197,"Rep":364,"Në ":708,"Uni":285,"The":241,"Tir":480,"Për":406,"çan":218,"bje":222,"bis":202,"bim":300,"bin":241,"ble":244,"bli":768,"bol":404,"bot":888,"be ":208,"ban":768,"baj":338,"baz":296,"bas":738,"bar":627,"bi ":668,"ber":314,"bel":434,"bet":408,"ca ":491,"cav":177,"cak":266,"ce ":197,"bri":501,"bra":276,"bre":593,"bur":304,"bul":254,"aka":205,"am ":292,"ake":288,"aki":184,"afë":206,"ajo":282,"ajt":819,"al ":1170,"aja":239,"ak ":563,"ahi":215,"aj ":1938,"agj":197,"ago":189,"ano":603,"ant":917,"ans":360,"ane":1253,"ang":643,"ani":2814,"anj":226,"ana":956,"anc":511,"and":1227,"amu":220,"amp":346,"ami":877,"ame":814,"amb":181,"ama":345,"alt":205,"alo":479,"all":1341,"ali":1967,"ale":1756,"ala":461,"an ":2721,"aks":181,"aku":485,"akt":1546,"ako":591,"aba":207,"abe":271,"aft":183,"afi":534,"ai ":361,"adm":223,"adh":1978,"adi":505,"ade":333,"aci":658,"ada":241,"azi":334,"aze":285,"azh":332,"atë":1708,"azë":204,"at ":2869,"arg":323,"are":2127,"ard":768,"ara":2483,"aro":307,"anë":3784,"arm":193,"arl":201,"ark":631,"arj":231,"ari":2009,"aru":212,"arr":782,"ars":590,"art":1488,"asa":351,"asi":536,"ash":2587,"ar ":6112,"apa":246,"alë":446,"apo":1058,"as ":2094,"aqe":844,"aqi":214,"aqj":186,"amë":284,"ava":276,"aut":366,"arë":1850,"avi":245,"ave":2116,"ata":707,"asu":211,"ast":1173,"atr":379,"ato":1090,"apë":205,"ate":982,"ati":2851,"atu":362,"aty":276,"アア":222,"jed":481,"jeo":227,"jer":1714,"jek":498,"jel":290,"jen":1698,"jes":2300,"jet":2586,"jev":418,"ji ":371,"jat":998,"jas":292,"jar":261,"jal":704,"jak":272,"jan":2402,"je ":2467,"joh":964,"jon":213,"fër":208,"jit":1409,"jis":586,"jim":228,"jin":428,"jik":319,"jih":193,"jo ":859,"ito":557,"ipë":1386,"itu":2464,"iud":180,"iso":211,"ist":3511,"iv ":257,"ita":1245,"ite":1604,"ith":1252,"iti":3696,"itj":291,"irë":662,"iut":194,"iva":248,"ivi":385,"ive":1169,"ilë":642,"ipt":1558,"ipi":226,"is ":537,"ion":2522,"ikë":1269,"ior":219,"ipa":1258,"ipe":572,"ir ":232,"inë":1756,"iro":194,"iri":583,"isi":327,"ish":5381,"ise":279,"isa":571,"iu ":213,"imë":180,"ire":227,"ira":796,"it ":11180,"ja ":2116,"itë":896,"isë":2854,"izu":355,"izo":367,"izm":294,"izi":684,"iza":492,"kim":794,"kin":237,"kip":221,"kis":422,"km ":346,"ken":688,"ket":291,"ke ":3158,"kra":414,"kre":377,"kt ":208,"kry":877,"ku ":990,"kro":261,"kru":297,"kri":1336,"kov":192,"gët":303,"kos":230,"kor":588,"kon":2239,"kom":1483,"gël":243,"kol":575,"koh":623,"kod":386,"gë ":240,"ko ":239,"kla":195,"jtë":442,"jut":199,"jtj":180,"jtu":499,"jua":282,"juh":823,"jug":449,"jta":181,"jti":303,"jnë":775,"jt ":307,"kat":718,"kar":404,"kas":189,"kan":1846,"kal":946,"kam":249,"kak":195,"ka ":2506,"jyr":194," Ga":321," Fo":338," Fr":441," Fi":583," Fl":194," Fj":187," Ha":528," He":302," Go":206," Gr":579," Gu":204," Gj":1228," Ho":278,"ha ":1068," Hi":261," Je":182," Ja":384," Is":478," It":205," In":442,"ham":267,"han":292,"hap":354," Ka":1328," Ki":237,"har":380," Kj":278,"has":365,"hat":909," Jo":291," Ju":322," La":426," Le":440," Li":748," Ko":2406," Kr":523," Ku":622," Ky":226," Ma":1780," Mi":739," Me":757,"he ":11876," Lo":267," Lu":464," Nd":206," Ne":390," Na":320," Nj":246," Ni":229," Mo":474," Mu":395,"hek":391,"hel":179,"hej":224,"het":2896,"hes":233,"her":715,"heq":210,"hen":1147,"hem":522,"hfa":178,"hi ":531," Am":329," An":631," Al":605," Ai":327," Aj":202," Af":185," Ba":1069," Au":417," At":239," Ar":624," Be":668," Bi":243,"hin":747,"him":565,"hil":181," Bo":497," Br":498," Bu":397,"his":436,"hit":1517,"hir":335," Ca":327,"hje":1227," Ch":191," Co":245,"hka":488," Da":377," Di":443,"hke":445," Dh":185," De":621,"hki":188," Dr":303,"hkr":988," Do":261,"hko":663,"hku":970," Du":229," El":231," Ev":475," Fe":366,"dë ":206," Fa":359,"cë ":308," Xh":218,"gli":277," Wi":198,"gjë":270,"gon":186,"cës":254,"gos":180,"gor":539," a ":180,"gru":420,"gra":1045,"gri":269,"gre":682," Kë":300," Po":615," Pi":181,"gul":267," Pe":612," Pa":1054," No":435," Ra":364," Në":821," Ro":384," Re":809," Ri":264," Pr":996,"gur":313,"gus":310," Pu":185," Qe":273," Su":263," St":520," Ta":356," Th":340," Ti":608," Te":634," Tr":311," To":373," Pë":415," Rr":230," Ru":266," Sa":578," Sh":3248," Si":519," Se":536," So":373," Sp":261," Sk":217," Va":291," Ve":411," Vi":293," Tu":194," Un":311," ja":2600,"ial":691,"ian":1176," je":756," in":1156," is":1788," it":245," ka":3973,"ibr":249," ki":698," ke":222," jo":218," ju":582," ha":507," he":538," gj":3833," gr":1301,"ia ":2790," gu":290," hi":627," dë":296," ho":209," hu":257," nj":8261," ni":252," ng":5742," nd":3078,"iel":198," ne":1135,"ien":214," na":398,"ier":413," mu":1092," mo":751," of":214,"ifi":230," nu":833," no":407," le":585," li":3581," la":1142," ku":2373,"ici":344,"ich":192," ky":232," km":370,"ie ":357," kl":275,"ica":291," kr":2025," ko":4104," me":6491," mi":1023," mj":371," ma":2979," mb":1927," lu":944,"idi":290,"idh":520,"ide":545," ll":324," lo":338," af":314," ad":323," am":508," an":1204," ap":925," ai":237," aj":220," ak":811," al":253," au":397," ar":1194," at":742," as":478," ba":1617,"il ":230,"ija":200," bi":487,"ije":259," be":279," bo":1013," bu":403,"iju":269," br":668," ca":214," e ":17735,"im ":2164,"eça":298,"ika":1647,"iga":202,"igj":563,"igu":244,"icë":252,"ij ":910,"ihe":650,"ik ":1689,"imo":407," et":343," es":291," en":262," em":840," el":514,"ime":1522," ek":723,"imi":5114,"ip ":230," fe":444,"ind":2986,"ina":1004," fa":908,"imt":634," fu":1154," fs":638,"ino":317,"ijë":223," fr":355,"int":466," fo":1164," bë":626,"ins":260,"inf":213,"ine":1441," fl":312,"ing":422," fj":334," fi":2335,"inj":211,"ini":1146,"iq ":474," ga":620,"inu":257," i ":9701,"iko":537,"iki":386,"ike":2366," ci":2530,"ila":1202," da":481,"in ":7905,"ikt":220,"iku":640," do":799,"ilo":291,"ill":1867," dr":678,"ilm":536," de":1917,"ilj":319,"ili":1491," di":2033," dh":10408,"ile":229,"ima":293," ed":1307,"io ":303," du":817," dy":651,"hpr":194," vë":300," zo":209,"hpe":278," zy":211," za":245,"dës":713," zb":179,"hkë":454," zh":535,"hoq":360,"hor":539,"dër":1831," zg":360," të":19747,"hme":1293,"hul":218,"hua":438,"hty":181,"htu":748,"htr":565,"htm":182,"hti":611,"htj":309,"hte":3639,"hta":810,"hsh":200,"hro":441,"ht ":2187,"hqi":3061," ru":213," rr":1833," u ":1417," sa":775," se":1591," si":4496," sh":10305," sk":290," sp":362," so":432," qu":416," qy":665," t ":287," ra":1563," re":1537," ri":358,"htë":9010," në":16417," ro":441," pu":708," pr":3527," qe":1420," më":4730," os":1270,"hum":1047," kë":1907," or":779,"hur":1522," pe":1738," pa":4231,"hvi":461," pl":373," lë":638," po":2196," pi":398," pj":1282," së":2396," va":567," ve":3244," vd":524," vo":419," rë":271," vi":3777," vj":545," vl":199," ty":371," tu":272," us":358," që":3732," uj":303," ta":476," st":879," su":385," tr":1582," to":411," pë":7156," th":1109," ti":1065," tj":633," te":3252,"fes":283,"fer":357,"faq":704,"fam":375,"fak":216,"ezë":244,"ez ":343,"etë":2083,"ezu":183,"eza":184,"ezo":208,"eze":303,"ezi":227,"eta":1185,"ete":1425,"etj":331,"eti":2581,"eth":892,"eso":355,"est":534,"esv":178,"eto":1111,"etr":567,"etu":501,"eve":3023,"erë":1553,"evi":375,"esë":1317,"epe":182,"er ":1621,"eor":217,"ekë":226,"eqi":478,"es ":2822,"epu":459,"elë":206,"epr":500,"eri":4116,"erg":266,"ere":548,"era":1561,"erb":418,"et ":8412,"emë":345,"esm":180,"esh":1754,"esi":1453,"ese":746,"eu ":239,"esa":635,"err":624,"ert":290,"ers":1252,"ern":432,"erm":871,"enë":512,"ero":390,"eki":248,"ekn":209,"eko":404,"egë":220,"eks":618,"ekt":1016,"eku":676,"en ":3312,"ela":426,"ele":1238,"eli":682,"ell":702,"elu":266,"emb":225,"ema":440,"eme":707,"emo":232,"emi":771,"emr":507,"ene":497,"ena":430,"end":3296,"enc":779,"eno":271,"eni":647,"enj":342,"ens":203,"ent":2477,"eog":217,"egj":763,"ego":445,"ej ":1063,"egu":355,"ek ":360,"eja":234,"el ":689,"ejt":732,"eka":189,"em ":322,"gju":704,"gjy":338,"gje":2061,"gji":3096,"gja":1196,"gim":207,"gaz":249,"gar":602,"gat":279,"gan":669,"ga ":5443,"ftë":453,"fus":270,"fut":207,"fun":698,"fra":187,"fri":223,"fsh":1041,"bër":688,"for":1268,"bët":491,"bëh":244,"fil":1245,"fik":505,"fin":217,"fis":186,"fit":614,"fiz":441,"fja":339,"da ":488," Çm":287,"de ":479,"dal":409,"dat":201,"dar":567,"dan":203," Ës":295,"ces":191,"ci ":188,"cia":487,"cil":2535,"cio":782,"ean":206,"eat":241,"ega":205,"edh":1894,"edi":471,"ede":284,"eda":183,"edo":295,"eci":308," çm":240,"dyt":203,"dy ":376,"dur":609," ës":7645,"dor":1271,"don":593,"dom":180,"dok":214,"dos":301,"dmi":238,"dod":448,"duk":778,"dua":191,"dri":384,"dra":292,"dre":608,"dry":602,"dro":361,"dha":308,"dhu":653,"dia":304,"dhj":875,"dhi":1463,"dhe":12079,"der":1468,"des":314,"det":819,"dh ":431,"deg":199,"del":210,"dek":243,"den":513,"dem":246,"di ":1814,"do ":585,"dhë":1093,"dje":856,"dim":1054,"din":518,"dio":362,"diq":424,"dis":945,"dit":1503,"dik":388,"rga":634,"ri ":4362,"rgj":700,"ret":2035,"res":1012,"rev":322,"rez":402,"rfa":431,"rbë":484,"rfs":234,"rg ":266,"rea":282,"rej":1398,"reg":1272,"reh":210,"rem":230,"ren":1060,"rek":369,"req":525,"rdo":695,"rdi":283,"rdh":548,"re ":5892,"rca":229,"rd ":234,"rap":224,"raq":331,"ras":547,"rat":1584,"rav":877,"rbi":290,"rba":177,"rbe":317,"raj":258,"rah":340,"ran":2020,"ram":546,"ral":554,"rak":530,"rab":220,"raf":580,"rad":1584,"rs ":444,"ror":1129,"ros":215,"nës":866,"nët":322,"rot":314,"rom":357,"ron":1148,"nën":960,"rop":794,"rov":207,"rod":677,"roc":243,"roj":419,"roi":191,"rol":305,"rok":233,"rof":239,"roh":382,"rog":307,"rne":248,"rmo":182,"rmu":349,"ro ":340,"në ":21951,"rmb":202,"rma":1098,"rme":349,"rmi":261,"rku":392,"rko":294,"rki":195,"rke":238,"rja":228,"rje":742,"riz":365,"rip":239,"rio":403,"rit":2857,"ris":2678,"riv":179,"riu":462,"rih":296,"rig":343,"rij":533,"ril":452,"rik":1578,"rin":1760,"rim":1881,"ria":1506,"rie":292,"rk ":231,"rtë":586,"rye":967,"rue":227,"rur":358,"rup":598,"rus":297,"rve":228,"rrë":463,"rsi":760,"rso":384,"rsa":288,"rsh":704,"rse":194,"rta":523,"rto":212,"rte":551,"rth":191,"rti":1091,"rua":870,"rtu":307,"rt ":743,"rmë":369,"rri":1024,"rrj":335,"rre":1446,"rra":326,"rru":264,"saj":699,"san":214,"sat":207,"sa ":1426,"rys":615,"sha":1190,"shm":1425,"sho":753,"shp":846,"shq":1997,"shr":422,"sht":17551,"she":1082,"shf":232,"shi":1551,"shk":3709,"si ":3797,"sje":269,"sid":277,"sia":412,"shu":1255,"sit":1207,"sir":214,"sis":1576,"sip":1484,"sin":587,"sio":712,"sim":613,"sik":262,"se ":3292,"ser":504,"set":189,"sh ":1239,"sen":315,"spo":232,"spe":251,"sot":202,"sov":1143,"son":618,"sor":814,"st ":714,"shë":1713,"sla":211,"ske":209,"sma":252,"sme":323,"stë":596,"sse":192,"ste":1041,"sta":1312,"sto":730,"sti":1254,"stu":543,"str":1367,"sua":229,"sue":178,"sur":369,"sve":238,"taj":547,"tal":816,"tav":260,"tat":899,"tas":209,"tar":4736,"tan":685,"te ":5516,"tbo":238,"ta ":2326,"pa ":427,"pe ":427,"par":1922,"pat":286,"pas":2622,"pak":267,"pan":389,"pi ":351,"per":1941,"pet":184,"pes":328,"pla":231,"plo":258,"pje":1425,"pju":181,"pia":517,"pik":379,"pin":244,"pio":290,"pit":346,"poz":244,"lër":206,"por":810,"pop":489,"lëv":248,"lët":262,"lës":678,"pos":377,"lën":745,"pon":243,"pol":674,"lë ":644,"po ":983,"pta":1447,"pub":598,"pti":317,"pto":261,"pra":544,"pri":797,"pre":1386,"pro":1746,"pun":526,"pul":588,"qar":179,"qe ":553,"qet":282,"qev":185,"qer":357,"qen":825,"qed":256,"qi ":290,"qit":232,"qip":2962,"qis":408,"më ":5423,"mës":389,"mër":628,"mën":311,"qua":358,"quh":192,"qyt":799,"ra ":3887,"ncë":386,"ngj":549,"ngl":339,"ngu":279,"ngr":213,"ni ":1941,"nge":207,"nga":5420,"ndë":2245,"nen":1014,"ner":340,"net":653,"nes":253,"nev":297,"ng ":438,"nez":219,"nci":280,"nce":340,"nca":359,"ne ":2694,"ndu":806,"ndr":1255,"ndo":1605,"ndj":868,"ndi":3173,"nde":1242,"nda":1141,"nal":768,"nar":589,"nd ":1287,"nav":195,"nat":905,"nas":219,"na ":1164,"ntë":218,"nxh":177,"nuk":387,"num":496,"nua":315,"nto":514,"ntr":261,"nti":1877,"nta":460,"nte":1912,"nst":368,"nsi":197,"nt ":562,"nom":633,"non":199,"jës":1000,"jër":345,"nor":452,"nov":244,"një":7204,"jë ":6765,"no ":251,"ngë":383,"nji":251,"nje":980,"nja":288,"njo":1016,"nie":218,"nic":209,"nia":391,"niz":506,"niv":445,"nis":1563,"nit":1162,"nim":624,"nin":724,"nik":917,"ogr":753,"ogj":624,"oi ":722,"ohu":860,"ohe":1613,"oj ":252,"ojn":599,"oje":249,"oja":193,"ol ":185,"oci":316,"odh":931,"ode":218,"odi":416,"of ":206,"odu":227,"ofe":213,"obe":363,"otë":804,"ovë":1008,"ozi":268,"otu":316,"oti":443,"ote":245,"oto":318,"opë":263,"ost":299,"ota":201,"osh":522,"ose":1363,"oso":1152,"ovi":292,"orë":1117,"ova":484,"ove":273,"oqë":260,"opo":219,"opi":438,"ope":206,"os ":591,"opu":565,"kën":763,"okë":307,"or ":3887,"kët":1025,"kës":1506,"kër":189,"orm":1202,"onë":412,"orr":382,"orc":230,"ord":255,"ore":3570,"org":551,"ori":2679,"ort":629,"oru":235,"ot ":268,"ora":318,"ola":251,"on ":2800,"oli":1087,"oll":1071,"ole":276,"olo":874,"ohë":394,"oka":403,"ogë":224,"oku":205,"ona":1138,"ond":187,"one":812,"onj":684,"oni":2539,"ojë":384,"ono":475,"ons":340,"ont":1418,"oma":638,"kë ":735,"ome":460,"omb":682,"omi":782,"omp":538,"omo":183,"omu":542,"la ":1588,"le ":1504,"lan":809,"lam":463,"lar":790,"lat":987,"las":448,"lav":342,"lba":191,"kut":329,"kus":201,"kur":1434,"kup":473,"kun":397,"kul":958,"ky ":230,"kth":196,"kte":440,"ksi":671,"kuf":336,"kua":615,"ktr":411,"ktu":532,"kti":608,"kto":809,"llë":931,"lon":395,"hën":1076,"hëm":639,"lor":960,"hër":431,"log":828,"loj":558,"loi":197,"hës":946,"lot":202,"lmi":262,"ltu":283,"lua":941,"luf":367,"lue":177,"lsi":217,"li ":2003,"lez":179,"lev":346,"les":310,"let":699,"ler":367,"lem":300,"len":674,"lek":623,"lls":213,"llu":548,"hë ":1014,"lla":1048,"lle":383,"lli":2212,"llo":1314,"lm ":208,"lje":416,"ll ":881,"lja":178,"lit":1752,"lis":1010,"lir":324,"lio":179,"lin":3413,"lim":1223,"liz":412,"lic":188,"lid":353,"lia":455,"lib":286,"lik":872,"lig":270,"ma ":556,"maj":406,"mak":224,"mad":655,"mar":852,"mas":295,"mal":684,"man":1367,"mat":1185,"mba":779,"mbl":257,"mbi":849,"mbe":401,"mbr":294,"me ":7511,"mbu":225,"med":203,"met":1333,"mev":393,"mes":776,"mer":1044,"mel":355,"men":1333,"mbë":676,"lum":373,"mpj":180,"mpi":353,"ëtë":434,"mpo":182,"mon":405,"mor":732,"mos":301,"mri":623,"mra":263,"mua":310,"mta":664,"mur":234,"mul":232,"mun":1237,"muz":308,"ës ":6600,"ëpi":229,"ër ":5836,"mi ":2352,"ëse":300,"ësa":205,"ërt":763,"ërs":761,"ërp":333,"ënë":345,"ëro":630,"ërr":212,"mje":546,"ërk":679,"ërm":698,"ërg":545,"ëri":1847,"min":2761,"ërf":795,"mil":488,"mim":764,"ërd":700,"mir":472,"ëra":457,"mis":535,"ërb":763,"ët ":1589,"mit":2427,"mik":541,"mij":213,"ëti":364,"ëto":188,"ëpë":198,"ëta":971,"ësh":8836,"ësi":2416,"ëso":446,"ësu":202,"ërë":456,"ëve":1277,"zua":506,"zyr":215,"zgj":372,"zi ":247,"zet":248,"ze ":310,"zan":199,"zak":278,"zat":320,"vës":631,"zon":428,"zoh":205,"vë ":592,"zmi":236,"zhv":458,"zim":386,"zik":591,"zis":248,"zit":331,"yrë":392,"yte":900,"ysh":673,"yrt":226,"yra":227,"yre":469,"za ":311,"ytë":280,"yes":421,"yer":268,"yeq":202,"të ":32538,"tëp":239,"tër":1782,"tët":229,"tës":1610,"tëv":188,"tën":440,"tëm":288,"xha":211,"xhi":228,"Çmi":287,"së ":5936,"sëm":198,"sën":372,"vro":528,"veç":250,"vil":586,"vin":191,"vic":177,"viz":481,"vit":3504,"vis":321,"vje":622,"rë ":3679,"vog":268,"rën":1295,"rët":460,"rës":1368,"rëv":444,"rëz":253,"vep":456,"ver":1289,"vet":682,"ven":1374,"vel":341,"vdi":416,"ve ":7283,"val":324,"var":488,"va ":428,"uzi":400,"urë":434,"ush":1610,"usi":271,"ust":434,"uti":207,"ute":336,"utb":206,"uto":619,"us ":390,"umë":772,"ut ":820,"ura":1239,"ure":247,"urg":207,"uri":1038,"uro":455,"unë":434,"urr":247,"urt":615,"qër":295,"qës":313,"ur ":4520,"upi":325,"upt":364,"umr":331,"umi":458,"uma":183,"umb":309,"ume":412,"që ":3460,"uni":521,"ujë":192,"und":1633,"una":206,"up ":226,"uku":221,"ukt":232,"uke":519,"ult":579,"uhë":641,"ull":1873,"uli":199,"ula":307,"un ":316,"uk ":422,"ujt":177,"ugo":283,"uft":506,"uhe":480,"uha":223,"udh":352,"udi":562,"ues":1871,"ufi":398,"ug ":248,"ua ":660,"uar":4550,"ual":287,"uan":427,"ubl":681,"uaj":694,"tyr":784,"tur":3146,"tul":217,"tua":1367,"tud":541,"tue":838,"tre":747,"tra":1541,"tri":1729,"tru":612,"tro":732,"tu ":717,"tme":342,"to ":593,"pë ":222,"toj":273,"toh":424,"pës":324,"tom":222,"ton":869,"tok":323,"tol":198,"për":8954,"tor":3712,"tij":916,"til":200,"tik":1967,"tit":3342,"tis":395,"tin":4094,"tim":1479,"tio":325,"thu":272,"tia":238,"tiv":962,"tje":1286,"tja":349,"thë":407,"tem":772,"ten":475,"tek":589,"tel":426,"th ":586,"tev":1136,"tet":4737,"tes":527,"ter":1775,"ti ":2991,"tho":197,"ths":192,"the":828,"thi":467,"tj ":257,"tha":760,"BA ":261,"zë ":417},"n_words":[1760559,2076420,1518161],"name":"sq","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/sr.json b/contrib/languages-data/sr.json
new file mode 100644
index 0000000..4164f4e
--- /dev/null
+++ b/contrib/languages-data/sr.json
@@ -0,0 +1 @@
+{"freq":{"Зид":109,"Зим":114,"Зла":429,"­":109,"²":236,"í":92,"é":175,"á":86,"ó":108,"Ä‘":1104,"Ä":126,"Штр":168,"ć":1891,"Што":91,"ÄŒ":99,"Ä":4854,"Шта":397,"Ште":107,"Зај":154,"Зап":1249,"Зал":315,"Зак":122,"Зах":97,"ЗаÑ":107,"ž":2132,"Зав":113,"Зад":197,"Заг":404,"Å ":118,"Å¡":2843,"Шпр":125,"Зво":126,"Зве":122,"Збо":90,"Зем":492,"Зел":167,"Зег":102,"Шум":153,"Шко":107,"Шиб":265,"Шир":110,"Шпа":200,"Шле":1282,"Шма":86,"Зор":122,"Шва":417,"Шве":146,"Шал":83,"Шап":121,"Шам":410,"Шат":197,"Шар":556,"μ":130,"ν":216,"ο":340,"ι":229,"κ":120,"λ":186,"δ":101,"ε":168,"η":99,"α":396,"Шен":174,"Шер":106,"γ":103,"ά":102,"Шек":94,"Зуб":86,"σ":141,"Ï‚":235,"Ï":204,"Ï€":100,"Ï„":159,"ш":94950,"Ñ„":29785,"Ñ…":46445,"ц":68619,"ч":65628,"Ñ€":436156,"Ñ":355754,"Ñ‚":321024,"у":311365,"ÑŸ":1310,"Ñ™":35971,"ј":308122,"Ñ›":19548,"Ñš":47495,"Или":225,"Ñ’":11394,"Ј":17149,"Љ":527,"Њ":798,"Ћ":246,"Ð":1148,"Ђ":626,"И":5904,"Л":8844,"К":19940,"Ð":11262,"Ðœ":15719,"П":32017,"О":9935,"Б":27505,"Ð":14540,"Г":10785,"Ð’":12826,"Е":5430,"Д":13309,"З":6537,"Ж":1460,"Ш":7137,"Т":9123,"У":5062,"Р":18801,"С":31653,"Изр":126,"Ц":3268,"Ч":1717,"Ф":11461,"Ð¥":16282,"л":188879,"к":286042,"и":710913,"п":216690,"о":641802,"н":481173,"м":187091,"г":133308,"в":226917,"б":84282,"а":837777,"з":105986,"ж":36683,"е":690251,"д":236545," ć":228," Ä":635," ÄŒ":97,"Инд":349,"ИнÑ":87,"Инт":201,"Има":166,"Име":214,"Имп":88," ž":304," Å ":118," Å¡":593,"Ива":318,"Иак":105,"Изе":84,"Изв":129,"Ù„":83,"ا":150,"Ита":429,"ИрÑ":118,"Ира":123,"ИÑÑ‚":900," Ð":14517," Б":27484," Ð’":12808," Г":10748," Д":13291," Е":5410," Ж":1459," З":6528," И":5862," К":19904," Л":8835," Ðœ":15676," Ð":11244," О":9864," П":31853," Ђ":625," Љ":525," Ј":17141," Ћ":244," Њ":798," Ð":1146," α":83," б":24266," а":18333," г":25445," в":20362," е":9609," д":51232," з":20203," ж":6804," и":86886," л":8350," к":56435," н":57584," м":28129," п":94827," о":106930," Р":18752," С":31602," Т":9091," У":5047," Ф":11453," Ð¥":15981," Ц":3264," Ч":1710," Ш":7130," Ñ›":799," Ñš":14264," Ñ™":998," ј":100589," ÑŸ":140," Ñ‚":19933," у":90576," Ñ€":27310," Ñ":117389," ц":5947," ч":5875," Ñ„":9403," Ñ…":4218," ш":6192,"Кад":128,"Кат":231,"Кап":162,"Као":168,"КаÑ":339,"Кар":1306,"Кал":923,"Как":158,"Кан":452,"Кам":420,"Каз":105,"Кај":141,"Ла ":84,"Куп":172,"Кур":248,"Кук":124,"Кул":160,"Кут":93,"Куз":132,"Кри":449,"КрÑ":142,"Кро":295,"Кру":300,"Кра":1299,"Кре":190,"Коњ":201,"Кош":104,"Кох":109,"Кот":211,"КоÑ":2912,"Кор":985,"Коп":201,"Кон":882,"Ком":414,"Кол":434,"Коз":193,"Код":100,"Ков":122,"Коб":165,"Кне":149,"Клу":161,"Кла":296,"Кли":168,"Кле":200,"Кло":88,"КиÑ":96,"Кир":248,"Кин":302,"Кер":95,"Кел":116,"Кен":184,"Лај":133,"Лав":93,"Лан":943,"Лау":243,"Лаз":155,"Лак":94,"Лен":108,"Леп":116,"Лео":127,"ЛеÑ":143,"Лет":239,"ÐК ":130,"Либ":219,"Лив":97,"Лим":199,"Лит":89,"ЛиÑ":112,"Лип":144,"Лин":222,"Лич":200,"Лиц":223," ја":1826," је":93577,"Лоа":476," љу":968,"Лоз":115,"Лон":298," њи":786,"Лот":96,"Лор":361," ње":13337," јо":756," ју":4367,"ÐУ ":87," њо":101," ће":734,"Луд":233,"Луц":83,"Лук":374," ра":8653," ре":13918," ри":921," пј":104," ро":2604," пр":45536," пт":139," пÑ":608," пу":1943," Ñв":6584," Ñи":3294," Ñе":23555," Ñл":3656," Ñм":1501," Ñк":1772," Ñп":2834," ÑÑ€":16502," Ñн":827," Ñо":2475," ру":1082," Ñа":24552," тз":108," ти":1237," Ñј":428," тв":397," те":6410," то":2768," Ñ‚Ñ€":4085," тк":114," Ñц":352," ÑÑ…":94," ÑÑ‚":16984," Ñу":11465," та":3595," тј":250," ук":888," уз":918," уж":105," уг":826," уд":840," уÑ":2032," ут":916," уп":1401," ур":494," ун":1058," уо":216," ул":992," ум":1152," ту":890," ув":475," уб":353," фо":1385," Ñ„Ñ€":763," фу":2034," фе":853," уј":191," фи":2888," фл":239," фа":1231," уч":922," уш":182," Ñ…Ñ€":1331," хо":601," ху":177," хи":633," хе":1026," хл":116," ха":273," цр":1684," ци":926," цв":109," це":2162," ца":1024," чу":291," чо":353," чл":773," чи":1933," че":1951," чв":130,"Ðа ":568," ча":434,"Мал":538,"Мак":940,"Мад":118,"Маз":83,"МаÑ":129," шв":112,"Мар":1490," ша":327,"Мат":232,"Ман":533," ше":436," шк":548," ши":2335," шп":265,"Мач":118," шт":1839," шу":256,"Мађ":286,"Маг":109,"Мај":781,"Мез":119,"Мед":226,"МеÑ":117,"Мет":642,"Мел":120,"Мек":1028,"Мер":390,"Мен":198,"Међ":362,"Мла":173,"Мик":159,"Мил":998,"Мих":270,"Мин":381,"Мит":253,"МиÑ":115,"Мир":309,"Моз":210,"Мод":99,"МоÑ":300,"Мор":533,"Мол":130,"Мон":516,"² ":229,"Мур":119,"Ðац":137,"Ðаг":213,"Ðак":122,"Ðаз":173,"Ðал":1049,"ÐаÑ":418,"Ðап":155,"Ðар":455,"Ðев":147,"Од ":276,"Ðај":450,"Ðин":100,"Ðиш":535,"Ðид":112,"Ðик":531,"Ðем":539,"Ðек":205,"Ðер":93,"Ðор":920,"Ðов":1182,"Ðоб":88,"аз ":491,"Ðој":432,"ад ":7870,"ав ":760,"Он ":195,"аг ":169,"аб ":148,"ао ":7785,"ап ":117,"ам ":2217,"ан ":10929,"ак ":3202,"ал ":1554,"РЈ ":221,"Оаз":95,"Ово":323,"Ове":241,"Обр":159,"Обе":308,"Оби":118,"Ова":554,"Па ":301,"Оли":280,"Окр":343,"Оде":172,"авц":213,"аву":2060,"ага":974,"аги":364,"авј":94,"ављ":3634,"аге":519,"агр":1237,"агл":272,"агм":99,"агн":334,"аго":977,"агу":495,"ада":15920,"ади":3490,"аде":2838,"аду":3144,"адр":910,"адÑ":1188,"адо":1331,"адм":506,"адн":4147,"Опш":1002,"Ора":138,"Орг":87,"адњ":419,"аел":202,"Орл":171,"би ":1331,"аер":148,"аеÑ":355,"ажа":324,"аже":556,"ажд":168,"ОÑв":136,"ажи":552,"ОÑк":121,"ажн":580,"ОÑн":326,"ОÑм":190,"ај ":2186,"ОÑÑ‚":672,"ажу":110,"Опе":165,"аба":540,"По ":294,"абл":408,"або":463,"абр":657,"абе":439,"аби":574,"ава":9441,"абу":169,"авн":5705,"авл":255,"абљ":86,"авк":265,"авÑ":891,"Оно":129,"авр":714,"бе ":991,"аво":2548,"аве":14288,"авд":102,"авг":575,"Она":164,"ави":16802,"алÑ":1372,"алт":594,"алп":116,"алн":9423,"бо ":129,"ало":1695,"алм":639,"алц":195,"алу":774,"алф":120,"алв":200,"алб":659,"ала":10846,"алк":547,"али":7292,"але":3111,"алд":1018,"алг":361,"амÑ":415,"аму":268,"амн":217,"амо":1815,"амп":780,"амб":552,"ама":5553,"ами":1178,"аме":2791,"анр":169,"ано":13152,"ану":5736,"анÑ":7532,"ант":3514,"анч":263,"анш":366,"анц":5831,"ана":8086,"анд":4157,"анг":1007,"ани":15172,"анз":207,"ане":2427,"анл":86,"анк":1530,"амј":97,"анџ":130,"аор":109,"аоц":85,"аоб":176,"анђ":145,"аон":418,"азу":880,"азо":538,"аљ ":458,"азн":730,"азм":274,"ажњ":102,"азл":1135,"ази":7146,"азе":707,"азг":86,"азд":502,"азб":116,"азв":1306,"аза":1339,"ањ ":588,"бл ":88,"аил":149,"аку":907,"акт":1696,"акц":534,"акш":91,"акл":273,"акн":144,"акм":1192,"ако":4032,"акÑ":2478,"акр":275,"акв":306,"аке":1255,"аки":556,"ака":2323,"ах ":1503,"аф ":127,"ć ":324,"ач ":958,"ац ":3942,"Пал":2844,"Пак":125,"Пар":781,"Ð°Ñ ":1347,"Пан":253,"Пав":221,"Паз":97,"ПаÑ":164,"Пат":127,"ар ":7524,"ау ":935,"ат ":5190,"СР ":195,"Пла":286,"Пли":97,"Пле":203,"ба ":1983,"Пиј":157,"Пит":117,"Пик":455,"Пин":119,"Пир":890,"аш ":324,"Пећ":96,"Пет":882,"ПеÑ":116,"Пер":500,"Пен":104,"Пел":147,"Пчи":125,"Ðеј":167,"Са ":86,"ТВ ":182,"Поб":94,"Пов":174,"Поа":233,"Пож":269,"Под":816,"Пок":135,"Поз":146,"Пон":117,"Пол":384,"Пом":1052,"Пој":141,"Пољ":527,"Пор":439,"Поп":218,"Пот":421,"ПоÑ":2261,"При":2140,"Пре":8305,"Прв":934,"Про":918,"Пра":388,"Прњ":91,"Рим":354,"РиÑ":97,"Риб":123,"Риј":90,"Рот":220,"Рођ":159,"Роз":94,"Рон":456,"Ром":159,"Роб":215,"Ä ":155,"Рог":222,"Род":147,"Рач":90,"РаÑ":264,"Рат":180,"Раш":217,"Рад":602,"Раз":142,"Рав":210,"Ран":169,"Рам":88,"Рак":105,"Рај":3464,"Рег":203,"Реч":123,"Реп":6121,"Рен":315,"Рем":99,"Рек":143,"Ðкв":410,"Ðка":112,"Ðле":602,"Ðлз":106,"Ðли":168,"Ðлб":402,"Ðла":116,"Ðлт":379,"Ðлп":608,"Ðмб":86,"Ðма":100,"Ðме":543,"Ðнд":310,"Ðнг":149,"Сад":314,"Саб":121,"Ðна":188,"Сав":772,"Сан":433,"Сао":220,"Сал":228,"Сам":255,"Сак":1964,"ÐнÑ":83,"Ðнт":461,"Сау":85,"СаÑ":150,"Сар":597,"Ðнх":325,"Ðнђ":98,"Ðпо":93,"РуÑ":825,"Рум":494,"Рук":87,"Руд":247,"Ðга":84,"Ðдм":355,"Ðзи":247,"Ðзу":140,"То ":455,"Сло":1071,"Сме":197,"Сма":124,"Ску":98,"Ско":232,"Сли":85,"Сла":799,"Ска":113,"Срп":4201,"Спо":152,"Срб":3494,"Сре":1205,"Сон":87,"Соц":253,"Спа":137,"Спл":118,"Спи":169,"Сов":165,"Сок":210,"Сом":238,"Сол":177,"Сев":1163,"Ðор":106,"Ðон":221,"Све":1041,"Сви":102,"Сва":118,"Сил":112,"Сим":270,"Син":185,"Сир":125,"СиÑ":300,"ВЈ ":118,"Сек":103,"Сел":387,"Сед":576,"Сер":251,"Сем":83,"Сен":765,"Бру":152,"Бри":492,"Бро":346,"Бра":1319,"Брд":115,"Бре":491,"Å¡ ":86,"Тар":199,"Тау":93,"Тан":138,"Так":280,"БоÑ":5750,"Бон":160,"Бор":452,"Бок":92,"Бол":109,"Бож":132,"Тај":125,"Боб":83,"Бод":117,"Бог":382,"Бој":140,"Бла":154,"Биб":121,"БиÑ":191,"Бит":519,"Бир":183,"Бин":118,"Био":412,"Бил":291,"šć":176,"Ðјф":238,"Ðјх":175,"Биј":195,"Беч":173,"БиХ":287,"Ста":912,"Сти":143,"Сте":486,"Стр":358,"Сто":282,"Сту":125,"Без":118,"Суд":249,"Бел":521,"Бек":95,"Бен":320,"Сун":370,"Бео":1668,"Бер":907,"Суп":90,"Бет":112,"Бањ":358,"Бај":268,"Бач":119,"Бау":104,"Бат":131,"Бар":635,"Тро":185,"Бал":373,"Трн":93,"Бан":484,"Три":351,"Бад":1635,"Тре":550,"Баб":113,"Бав":2112,"Тра":430,"Тул":87,"Тур":462,"Топ":250,"Том":239,"Тор":137,"Ток":171,"Тит":113,"Тих":87,"Тир":1042,"Тим":218,"Сје":411,"ÐÑ„Ñ€":304,"ÐуÑ":436,"Ðут":151,"Ðти":273,"Ðтл":356,"ФК ":379,"Ðта":245,"Ðрв":84,"Ðрг":112,"Ðра":348,"Тек":85,"Ðрд":374,"Тео":217,"Ðри":102,"Тел":164,"ТеÑ":146,"Тер":251,"Ðрм":86,"Ðрт":149,"Ðрх":112,"Врх":140,"Врб":141,"Вра":229,"Вул":164,"Вук":248,"Вор":124,"Вод":118,"Вол":243,"Вог":102,"Вој":653,"Вид":142,"Вин":286,"Вик":124,"Вил":360,"Виз":162,"Вит":360,"Вир":1331,"ВиÑ":149,"Виш":338,"Бје":189,"Виј":137,"Вла":763,"Вер":279,"ВеÑ":907,"Вел":1090,"Вен":217,"Вез":142,"Тут":88,"Вај":429,"Ваљ":108,"ВаÑ":189,"Вар":502,"Вал":502,"Ван":167,"УÑÑ‚":141,"Упр":237,"Унт":105,"Уни":389,"Укр":132,"Бук":174,"Бул":140,"Бур":592,"Буг":354,"Буд":232,"Буз":86,"Буј":88,"ГиÑ":85,"Др ":85,"До ":877,"Гли":92,"Гла":528,"Фак":100,"Фар":105,"Гра":1813,"Грб":96,"Гре":246,"Гри":146,"Гро":269,"Гру":261,"Грч":384,"Год":132,"Гор":2705,"ГоÑ":206,"Гот":135,"Гол":158,"Фер":187,"Фед":1501,"Фил":509,"Фир":118,"Фин":97,"Ује":238,"Фле":186,"Фоч":111,"Фон":130,"Фор":279,"Фра":5371,"Фри":175,"Гал":207,"Фре":186,"Гар":335,"Гац":85,"Гај":99,"Фуд":232,"Гер":273,"Гео":143,"Ген":166,"ã‚¢":111,"Диј":83,"Дит":144,"Дир":130,"ДиÑ":105,"Дио":94,"Дип":97,"Дим":140,"Дин":131,"Ен ":161,"Ер ":150,"Хар":369,"ХаÑ":91,"Хал":153,"Хам":142,"Хан":247,"Доњ":1845,"Дон":322,"Дом":284,"Дол":169,"Дор":179,"Доб":482,"Дри":109,"Дре":135,"Држ":378,"Дра":512,"Дру":869,"Дун":151,"Дуб":334,"Дуг":84,"Душ":140,"Хај":361,"Хип":87,"Хил":214,"Хел":153,"Хен":110,"Хем":86,"Хер":6097,"ХеÑ":450,"Хрв":3210,"Хри":165,"Хое":93,"Хор":121,"Хол":1488,"Хох":95,"Дал":436,"Дав":86,"Дан":479,"Дам":130,"Дар":218,"Дво":149,"Хун":181,"Дев":98,"Дем":212,"Дел":197,"Дер":133,"Ден":132,"Деч":93,"ДеÑ":132,"Деј":200,"Цар":168,"ЕÑÑ‚":109,"Епи":167,"Епа":99,"Енг":213,"Енд":111,"Еми":93,"Еле":145,"Ели":91,"Цер":116,"Цет":121,"Цел":161,"Цен":445,"Црв":222,"Црк":191,"Црн":1110,"Еге":91,"Еги":147,"Евр":809,"Чар":97,"Чеш":130,"ЧеÑ":89,"Чет":114,"Чел":87,"Жан":101,"За ":264,"Жив":194,"Жир":124,"Жен":115,"Жер":101,"лај":601,"лањ":114,"лбр":87,"лбу":532,"лва":233,"лаз":5049,"лаж":295,"лал":90,"лам":651,"лак":571,"лар":955,"лан":5765,"лау":200,"лаÑ":3936,"лат":4598,"лаш":484,"лач":526,"лац":1124,"лах":123,"лба":351,"лбе":269,"лађ":217,"лго":231,"лги":122,"лда":152,"лде":413,"лди":101,"лдб":91,"ме ":4180,"лви":91,"лге":138,"лга":84,"ма ":25410,"лаб":226,"лав":5743,"лаг":702,"лад":2173,"кши":181,"кућ":321,"куј":354,"куÑ":263,"кут":125,"куш":199,"кул":1691,"кум":252,"кун":133,"лц ":163,"куп":1701,"кур":384,"кци":1655,"кош":268,"кох":141,"кођ":505,"коњ":158,"кој":46468,"лт ":421,"крв":211,"кре":1287,"кра":4330,"кри":1665,"кру":15138,"лу ":3102,"кро":885,"крÑ":287,"крш":123,"кÑа":656,"кÑе":256,"кÑи":900,"кÑо":2107,"кÑн":118,"кÑÑ‚":498,"кÑп":269,"кÑу":269,"кта":463,"кте":692,"кти":1369,"ктн":244,"кто":1404,"ктр":790,"кту":4907,"кла":1570,"ло ":9505,"кло":812,"кли":774,"кле":1389,"клу":905,"кми":1187,"кне":270,"кни":220,"коб":278,"ког":8112,"ков":3602,"кну":107,"ком":10764,"кон":3999,"Ð»Ñ ":170,"коо":112,"коп":907,"кор":2532,"коÑ":446,"кот":489,"код":1658,"кож":96,"коз":151,"кок":257,"кол":2479,"кид":186,"кив":177,"кин":661,"ким":2487,"кил":447,"киј":370,"кић":129,"кињ":125,"лм ":881,"кип":134,"кир":398,"киÑ":271,"ких":13568,"киш":101,"ле ":4253,"кво":109,"кви":2552,"кед":760,"кеа":264,"кеј":173,"ли ":11349,"кен":534,"кел":235,"кет":218,"кер":419,"кањ":169,"лд ":1252,"кај":91,"кве":793,"ква":1006,"кат":1296,"кау":101,"кар":2613,"каÑ":751,"као":4364,"кап":295,"кам":527,"кан":1752,"как":575,"кал":1288,"каж":125,"каз":991,"каш":243,"кац":648,"каб":115,"кав":369,"кад":1624,"лб ":126,"ла ":11363,"кт ":360,"ку ":4779,"ићу":167,"ÐºÑ ":487,"иљу":112,"иња":733,"иње":1591,"ињи":131,"ињÑ":371,"ко ":11086,"ињу":98,"ића":1051,"иће":486,"ићи":1351,"ићк":102,"ије":18649,"ија":26005,"км ":243,"ијо":1259,"ији":11828,"ијÑ":4230,"ију":3169,"иља":489,"иље":460,"иљн":91,"иљк":320,"ки ":15059,"ке ":16319,"нва":203,"од ":26954,"нбу":2054,"нбе":511,"нај":4359,"нањ":237,"нађ":92,"нба":320,"нах":325,"нац":1791,"нау":1444,"нач":3335,"наш":893,"нао":86,"ог ":17762,"нан":525,"нам":1269,"нал":7610,"нат":5956,"наÑ":17476,"нар":3487,"нап":984,"наж":99,"нае":414,"над":1326,"наг":937,"нак":1382,"наи":83,"наз":1926,"нде":1146,"нда":1820,"нгу":126,"нгт":100,"нго":298,"нгÑ":87,"нгр":162,"нгл":815,"нги":1043,"нге":1637,"нгв":164,"нга":390,"нви":125,"нве":215,"неш":165,"нже":149,"нек":2076,"нел":235,"нем":1023,"нен":721,"нео":346,"неп":695,"нер":1881,"неÑ":797,"нет":980,"неу":227,"неф":87,"нец":104,"нег":385,"нев":413,"неб":279,"неа":117,"нез":804,"неж":223,"нед":404,"нди":1494,"оз ":466,"ндо":1219,"ндÑ":442,"ндр":728,"нду":438,"нив":1382,"низ":2908,"ник":12010,"нид":108,"ниг":102,"ниж":128,"нзо":88,"ок ":1611,"нза":186,"нзи":538,"нзе":237,"неј":878,"оа ":446,"ов ":1648,"нав":745,"наб":169,"об ":174,"мпÑ":94,"мпо":410,"нт ":1409,"мпл":372,"моћ":512,"мпи":804,"мпе":520,"мпа":795,"моч":117,"моц":131,"моу":89,"мот":526,"мрт":332,"мро":192,"ну ":15940,"мпј":83,"мрл":89,"мре":223,"мÑк":2421,"三":157,"ä¸":86,"нц ":300,"мун":1141,"мул":381,"муз":898,"муÑ":132,"мур":117,"муч":159,"муш":457,"нш ":306,"мци":104,"мењ":500,"мељ":149,"мећ":198,"нк ":147,"мик":370,"мил":886,"миз":88,"мид":103,"мич":1375,"миц":536,"миш":727,"мин":3851,"мим":241,"мио":91,"миÑ":825,"мир":1661,"мит":950,"миј":1533,"мињ":179,"мић":122,"Љуб":331,"мла":438,"но ":31892,"мна":346,"мне":116,"мни":309,"мно":823,"моб":172,"мод":539,"мог":1314,"мов":616,"моз":267,"мож":885,"мон":1010,"мом":420,"мол":368,"мок":358,"моÑ":2613,"мор":2662,"Ð½Ñ ":374,"маљ":216,"мај":1350,"маћ":267,"мањ":980,"мбе":1510,"мби":327,"мбу":200,"мбо":455,"нд ":1390,"мбр":845,"маш":262,"мац":1084,"мач":12755,"мах":140,"мађ":207,"мба":1378,"мал":1443,"мак":415,"мад":341,"мат":3328,"маÑ":545,"мар":1689,"нг ":1021,"мао":200,"ман":7905,"мам":150,"маг":535,"Јов":351,"меј":166,"међ":2287,"мех":195,"меш":617,"меÑ":3136,"мет":3201,"мен":7145,"ни ":34617,"мер":4603,"мек":210,"мел":176,"мем":128,"мед":1023,"мев":268,"мде":84,"не ":27271,"Јон":92,"лшт":1289,"Јад":173,"Јаб":184,"Јав":91,"Јаг":89,"Јан":158,"Јап":177,"Јак":157,"ЈаÑ":196,"лце":157,"лци":100,"лцл":114,"лха":148,"Јел":179,"на ":57921,"Јез":144,"Јед":11541,"Јев":137,"Јер":216,"лпи":612,"лој":304,"лпÑ":100,"лош":1201,"лоч":242,"му ":1793,"лни":2506,"лне":2152,"лно":2998,"лму":125,"лмÑ":356,"лна":907,"лоз":717,"лок":686,"лог":3009,"лод":251,"лож":580,"лоп":543,"лор":387,"лоÑ":501,"лот":376,"лол":105,"лом":1605,"лон":784,"Ð¼Ñ ":231,"лну":1948,"лов":5701,"лоб":930,"лут":258,"луÑ":300,"луч":438,"луц":348,"лул":94,"лук":379,"луп":108,"луо":194,"лун":148,"лум":724,"луг":281,"луб":1026,"луж":723,"луз":136,"луј":149,"лфр":88,"лфе":131,"лфи":86,"лфа":117,"лÑÑ‚":128,"лÑк":1883,"лÑа":106,"лту":760,"лто":95,"лти":229,"лте":881,"лта":559,"лећ":152,"лза":115,"лиг":1118,"лид":159,"либ":192,"лив":576,"лиз":1529,"лиж":310,"лил":191,"лим":1734,"лик":11361,"лдо":133,"лдÑ":84,"леа":196,"леб":189,"леж":344,"лез":1467,"лев":722,"Југ":1256,"лег":313,"лед":1839,"Јуж":557,"леÑ":1364,"лер":1174,"леп":226,"Јур":133,"лео":258,"ми ":812,"лен":2926,"лем":1370,"лел":128,"Јул":83,"лек":2393,"леш":105,"леч":125,"леф":108,"леу":115,"лет":1196,"леј":185,"леђ":138,"лкр":243,"кљу":617,"књи":1112,"мо ":1125,"лма":835,"лмо":236,"лме":86,"лих":503,"лиц":2558,"лич":2364,"лиш":305,"лиÑ":1427,"лит":3381,"лиф":251,"лин":3445,"лио":440,"лип":305,"лир":152,"лић":519,"лиј":3989,"лко":146,"лка":612,"паÑ":269,"пат":651,"пац":228,"паш":107,"пав":93,"пад":14312,"паг":84,"паж":106,"пак":284,"пам":140,"пал":338,"пао":90,"рг ":5715,"пан":3250,"пар":6224,"пап":134,"пај":236,"пањ":487,"рд ":408,"ођа":440,"ре ":3735,"ође":2085,"ра ":11124,"рб ":143,"пил":207,"пик":84,"пир":430,"пио":186,"рл ":93,"пин":427,"пит":636,"пиÑ":10327,"пиш":133,"оја":11213,"оје":9859,"ојв":646,"оји":9116,"пиј":1084,"ојк":186,"ојн":1254,"пић":150,"ојм":157,"ојо":880,"ојÑ":808,"оју":1151,"ојт":161,"ојц":174,"ојш":127,"оља":608,"оље":731,"ољи":259,"ољÑ":462,"рн ":846,"ољо":106,"ољн":234,"ољу":345,"пла":1577,"оња":1770,"оњи":617,"пли":958,"пле":749,"оње":264,"пло":805,"ро ":1261,"оће":109,"оћи":182,"пед":320,"пев":364,"ри ":8692,"пер":2735,"пеÑ":743,"пет":618,"пек":270,"пел":585,"пен":599,"пец":408,"пеш":318,"пећ":129,"рк ":422,"пиз":88,"ори":9358,"орз":85,"орд":1293,"оре":3647,"орв":124,"орг":2732,"орÑ":1818,"оро":1886,"пу ":455,"орп":370,"орм":2476,"орн":1996,"опљ":214,"орк":370,"опш":35593,"опу":756,"опт":202,"опÑ":1008,"орб":607,"ора":4078,"нџа":125,"опе":978,"опи":8685,"опн":248,"опо":1046,"опр":619,"опл":534,"ооÑ":176,"оор":106,"опа":674,"оте":1270,"отк":354,"отл":140,"оти":2195,"оÑј":1754,"ото":1607,"отп":577,"отн":337,"оту":206,"отр":1043,"отÑ":417,"оÑÑ„":169,"оÑу":428,"отв":376,"ота":1137,"орђ":198,"оÑе":1479,"оÑи":1742,"орј":175,"оÑк":422,"оÑл":4652,"орњ":1483,"оÑм":2386,"оÑн":8035,"оÑо":1921,"оÑп":343,"оÑÑ€":231,"орџ":88,"оÑÑ‚":12354,"ору":1182,"орт":1154,"орф":1119,"орц":137,"оÑа":1233,"оÑв":640,"омн":289,"омл":99,"оми":1500,"оме":4149,"омб":447,"ома":3081,"олш":1216,"олц":286,"олу":1267,"олф":220,"олÑ":304,"олт":167,"олн":318,"по ":2745,"оло":4064,"олк":138,"олм":112,"оле":1477,"оли":4441,"олд":233,"ола":2070,"окр":16250,"окÑ":337,"окт":753,"оку":1463,"око":3712,"онÑ":1810,"онт":1672,"онц":357,"ону":5169,"онф":228,"они":5926,"онз":280,"онк":246,"пр ":139,"оно":2541,"она":7689,"онв":188,"онб":88,"онд":758,"онг":338,"оне":1433,"омо":1831,"омп":1100,"омÑ":686,"ому":652,"оша":382,"оши":323,"оше":324,"ошк":1178,"ошл":182,"ошт":538,"оча":344,"очи":758,"оче":1097,"очн":3498,"оцј":1629,"очу":83,"ошћ":293,"офе":855,"офа":229,"оуч":265,"офт":139,"офо":217,"офÑ":147,"офи":650,"оуг":194,"оуп":90,"оуз":104,"оца":171,"оце":938,"оци":1593,"оха":159,"охе":282,"охр":88,"охи":547,"охо":201,"оба":2774,"оат":253,"оар":591,"оан":147,"овг":89,"ова":11175,"обу":821,"обр":1721,"обо":1375,"обн":584,"обл":2580,"оби":2413,"обз":108,"обе":1299,"Њен":125,"па ":2144,"Њег":294,"оиÑ":1849,"оим":1253,"оћ ":160,"нљи":100,"оки":368,"окл":297,"оке":600,"ока":1581,"окв":1772,"ожи":330,"ој ":44878,"ожђ":85,"озв":134,"ози":1003,"озе":676,"оза":1830,"ожј":112,"озо":1076,"озн":2962,"оид":1008,"оиз":729,"Њуј":106,"одм":327,"одн":4424,"одл":481,"оди":8562,"оду":1182,"одр":3378,"одÑ":582,"одо":1134,"одњ":237,"оен":147,"пи ":1059,"оже":1557,"ожд":83,"ожа":629,"ове":6236,"обљ":196,"овл":133,"ови":13959,"обј":733,"ово":5370,"пе ":1159,"овн":11503,"ову":980,"овр":1263,"овÑ":742,"овч":112,"овц":697,"ога":1828,"нђе":269,"оге":687,"оги":1839,"овј":1495,"овљ":259,"огл":656,"огн":99,"ого":1533,"огр":4802,"огт":183,"огу":1193,"одб":375,"ода":2535,"одг":540,"одв":356,"оде":2259,"оу ":269,"нро":158,"нри":135,"ноћ":177,"от ":628,"нпр":136,"ној":19200,"нот":238,"ноÑ":7574,"ноф":97,"нош":420,"ноп":192,"нор":440,"нон":275,"ноо":172,"Ð¾Ñ ":2917,"нол":379,"ном":7402,"нок":125,"ноз":97,"нож":162,"ног":6852,"нод":143,"ноб":292,"нов":16704,"ноа":173,"ор ":3231,"оп ":397,"нла":112,"нли":105,"он ":5316,"нко":648,"нкр":96,"нкт":107,"нку":283,"мљу":151,"нкц":621,"мља":354,"нка":850,"нки":337,"мљи":386,"мље":717,"нке":437,"ом ":22463,"нић":400,"ниј":12404,"мје":4955,"нир":517,"ниÑ":2501,"нит":625,"нил":406,"ним":4578,"нин":1402,"нио":253,"ол ":661,"нич":2890,"ниш":4604,"ниф":141,"них":6787,"ниц":5694,"ншт":373,"нче":230,"нчи":115,"нча":188,"нце":702,"нци":2151,"нца":651,"нцу":5237,"нхе":115,"нха":822,"ош ":929,"нхо":113,"нфе":414,"нфо":322,"нуа":555,"нум":116,"нул":184,"нук":195,"нуÑ":137,"нуо":120,"нут":1477,"нÑÑ„":160,"нта":3191,"нте":2285,"нти":6056,"нтÑ":456,"нту":323,"нто":871,"нтн":454,"нтр":2081,"нÑа":348,"нÑб":322,"нÑк":21333,"нÑи":1508,"нÑе":243,"нÑÑ‚":2320,"нÑÑ€":148,"нÑп":163,"нÑо":221,"оф ":272,"Ñаз":225,"Ñак":730,"Ñал":373,"Ñам":1848,"Ñан":1886,"Ñао":546,"Ñаб":166,"Ñав":12830,"Ñаг":142,"Ñад":723,"Ñач":444,"Ñар":487,"Ñат":1303,"ÑаÑ":2383,"Ñау":119,"Ñац":438,"Ñањ":362,"Ñај":140,"тд ":136,"Ñба":232,"Ñбе":307,"Ñва":1042,"Ñбу":542,"Ñво":2081,"те ":6016,"Ñвр":218,"Ñве":3473,"Ñви":792,"рђа":295,"Ñвј":110,"рђе":430,"Ñдо":330,"Ñеб":683,"Ñев":3037,"Ñег":95,"Ñез":327,"Ñед":1857,"Ñел":1535,"Ñек":671,"Ñеп":618,"ти ":10122,"Ñео":160,"Ñен":1313,"Ñем":200,"Ñет":704,"ÑеÑ":115,"Ñер":1186,"Ñец":168,"Ñеч":136,"Ñељ":14369,"Ñеј":202,"Ñећ":267,"Ñењ":205,"Ñиг":253,"Ñид":147,"Ñив":302,"рја":155,"рје":274,"Ñич":228,"ÑиÑ":1247,"Ñит":283,"Ñир":1615,"Ñих":600,"Ñиф":177,"Ñик":192,"Ñин":1494,"Ñио":533,"Ñил":738,"Ñим":906,"Ñкв":125,"Ñке":7702,"Ñки":22959,"рљи":86,"Ñка":16636,"Ñиј":1688,"Ñиљ":91,"Ñињ":146,"Ñић":118,"рњи":454,"Ñли":1408,"рње":277,"Ñле":1944,"рња":1169,"Ñла":3221,"Ñку":1728,"Ñкр":678,"Ñко":23421,"Ñкл":305,"Ñми":426,"Ñме":837,"Ñма":1176,"ÐЈ ":245,"Ñлу":1437,"Ñло":3155,"то ":10724,"Ñкљ":114,"Ñна":1061,"Ñне":1878,"Ñни":6625,"Ñмо":2197,"Ñмр":275,"Ñоб":1187,"Ñов":2446,"Ñок":416,"Ñол":242,"Ñом":308,"Ñ‚Ñ€ ":138,"Ñно":4183,"Ñну":121,"Ñпе":1119,"Ñпа":703,"Ñпл":139,"Ñпи":697,"Ñоф":144,"Ñор":646,"Ñоп":287,"Ñон":2669,"Ñоц":896,"Ñре":14058,"ту ":6826,"Ñро":147,"Ñри":173,"Ñпу":160,"Ñпо":2462,"Ñпр":728,"рпу":136,"рпÑ":6643,"рпр":93,"Ñу ":17733,"роц":2456,"роч":246,"рош":621,"рот":1629,"роу":489,"роф":1077,"роп":1727,"рор":221,"роÑ":2133,"ÑÑ‚ ":4278,"рпо":293,"рој":2351,"рођ":922,"рпе":118,"рпа":88,"рта":1010,"Ñ€ÑÑ‚":3136,"Ñ€ÑÑ…":160,"ртн":153,"ртм":4491,"ртÑ":303,"ртр":100,"рто":516,"рте":1621,"ртв":127,"рти":1376,"Ñ€Ñа":168,"Ñ€Ñк":8319,"Ñ€Ñл":95,"Ñ€Ñо":156,"Ñ€Ñн":119,"Ñ€Ñб":203,"Ñ€Ñд":115,"Ñ€Ñе":238,"Ñ€Ñи":314,"руш":1238,"руч":948,"рфи":96,"руј":409,"рфе":114,"рфо":135,"рту":311,"руа":505,"руб":149,"руз":135,"рук":1051,"руг":17782,"руд":207,"руж":1022,"руп":2147,"руÑ":775,"рут":180,"рум":339,"рун":279,"рца":147,"рце":6006,"рцг":111,"рцв":90,"рци":365,"рцо":146,"рха":299,"рхе":608,"рхи":1027,"рхо":272,"ÐД ":350,"рху":132,"ршт":219,"ршн":146,"рши":1285,"ршк":166,"рше":452,"рша":332,"рчк":1174,"рчи":96,"рча":87,"ÐП ":94,"та ":24494,"рад":15222,"рае":112,"раж":1067,"раз":4405,"раб":340,"рав":5728,"раг":886,"рам":3023,"ран":15671,"рао":659,"рап":559,"рак":1642,"рал":3181,"рах":410,"раф":1317,"рач":1292,"рац":2825,"раÑ":1831,"рар":136,"рау":311,"рат":5078,"рба":695,"раш":666,"рби":3548,"рбе":475,"рађ":1238,"рбо":328,"рањ":1384,"раћ":751,"рај":4293,"раљ":1551,"рбу":351,"рва":5445,"рдÑ":106,"рду":314,"рдн":105,"рдо":475,"рди":1053,"рде":834,"реб":1934,"рев":1509,"рег":7841,"ред":21162,"реа":613,"рдф":196,"реф":4512,"реу":146,"рет":2436,"реÑ":1742,"рер":230,"реп":1358,"Ñи ":1735,"рео":463,"рен":3928,"рем":12715,"рел":899,"рек":3409,"реи":116,"рез":1388,"реж":768,"рже":168,"ређ":1332,"ржа":14645,"реч":1314,"реш":535,"рех":110,"рец":293,"рвн":173,"Ñе ":16756,"рво":1461,"рве":1500,"рви":1598,"рга":2899,"рву":294,"рвц":130,"рго":513,"рге":622,"рги":440,"рдв":110,"рда":653,"ргу":233,"риш":1433,"пје":186,"рио":1442,"Ñл ":142,"рип":10260,"рим":2792,"рин":3442,"рик":1650,"рил":1573,"риц":1301,"рич":2529,"риф":333,"рих":484,"рит":3451,"рир":799,"риÑ":3789,"пља":339,"рка":895,"рки":242,"пљи":88,"ркв":1089,"рке":414,"пље":324,"риј":9688,"рињ":113,"рић":295,"рза":189,"ржи":374,"реј":705,"рећ":815,"рењ":498,"риб":747,"риг":881,"рив":1495,"рид":656,"риз":1001,"риж":129,"рзи":967,"рзо":108,"Ñк ":101,"рнк":113,"рни":2602,"рне":1342,"рнб":105,"рна":2309,"рму":235,"рмÑ":114,"рмо":317,"рок":955,"рол":836,"ром":2696,"рон":2160,"рож":168,"роз":1124,"рои":2993,"ров":3353,"рог":2212,"род":5445,"роа":193,"роб":830,"рну":227,"рно":3320,"рле":211,"рли":396,"рла":605,"рку":269,"ркт":206,"рко":589,"рме":465,"рми":1450,"рма":1859,"рло":695,"Ñо ":105,"рлÑ":110,"пра":4356,"прв":2403,"при":15354,"пре":17090,"про":11337,"ру ":5752,"поп":7996,"пор":3359,"поÑ":5163,"пот":2258,"пох":118,"поч":1072,"пош":263,"пој":1887,"пољ":1013,"рт ":1206,"пно":461,"пое":108,"под":4064,"пог":482,"пов":2442,"поб":402,"Ñ€Ñ ":235,"пон":1141,"пом":1383,"пол":3072,"пок":1589,"поз":2822,"оћу":157,"оћн":181,"пне":188,"пна":116,"пни":227,"пуш":177,"рч ":111,"пуњ":88,"пуб":6548,"пту":139,"пун":422,"рц ":99,"пуÑ":393,"пут":1505,"пук":115,"пул":497,"пта":194,"пÑÑ‚":361,"рх ":203,"пто":209,"пти":395,"пте":647,"пÑа":91,"прÑ":111,"пру":342,"пÑо":194,"рф ":929,"пÑк":7893,"пÑи":594,"пÑе":135,"пшт":36823,"пче":98,"пци":231,"пхо":118,"Ñа ":8016,"вао":651,"вар":4393,"ваÑ":221,"ват":5276,"вац":1889,"вач":2332,"ваљ":162,"вај":2438,"вањ":3929,"ге ":1625,"вад":467,"ван":5104,"вам":274,"вал":2173,"вак":1047,"ваз":458,"важ":509,"га ":15953,"бух":516,"буш":92,"бун":179,"бум":473,"бул":126,"бук":180,"буÑ":106,"бур":3775,"буд":350,"буг":130,"буб":88,"врђ":432,"вÑк":2068,"вÑÑ‚":354,"вра":364,"врд":211,"вре":2511,"ври":173,"врл":169,"врÑ":1575,"гу ":3405,"вро":1134,"врх":345,"врт":434,"врш":1750,"вођ":799,"вој":4438,"вољ":474,"вол":443,"вок":137,"воз":273,"вод":3129,"вог":1645,"вов":377,"воб":187,"воа":91,"вот":981,"воу":181,"вор":3081,"воÑ":856,"вом":1630,"вон":796,"вни":13918,"вне":1253,"вна":1125,"вну":306,"вно":4495,"гр ":172,"вла":1894,"го ":717,"вко":88,"бље":277,"вка":213,"бља":290,"вић":3354,"вињ":156,"виј":2182,"виш":1683,"виц":1060,"вич":586,"бје":512,"виђ":131,"бја":446,"виз":1047,"вил":1408,"вик":313,"вин":8161,"вим":1946,"вио":870,"гл ":123,"виÑ":1551,"вир":2019,"вит":1026,"вих":1104,"вив":88,"виг":1474,"вид":967,"већ":2150,"веш":575,"веч":174,"вец":113,"веÑ":693,"вет":3813,"вер":6835,"вен":4778,"ги ":980,"вео":533,"вел":2599,"вем":722,"век":1758,"вез":13652,"веж":185,"вег":170,"вед":952,"веб":100,"вде":113,"вда":83,"вгу":597,"ва ":17876,"бак":147,"баз":278,"бан":881,"бам":152,"бал":2686,"бав":1092,"баб":84,"ачн":488,"ачи":1962,"ачк":16987,"ачу":748,"аше":304,"аша":706,"ашн":83,"ашк":841,"аши":740,"ашт":708,"ашћ":92,"ашњ":1176,"афа":126,"ауч":504,"ауц":95,"ауÑ":384,"аут":1011,"аур":188,"аун":269,"аул":105,"аум":107,"аук":567,"ауз":840,"ауе":242,"ауд":139,"ахв":170,"аха":309,"афр":145,"афÑ":527,"афи":664,"афе":332,"аца":895,"аце":128,"ахо":294,"аху":91,"ахт":162,"ахи":208,"ача":2246,"ачв":93,"аче":1192,"аци":9797,"ацк":104,"апу":221,"апÑ":380,"апт":123,"апр":1018,"апе":351,"апа":4871,"апо":620,"апл":119,"апи":905,"арх":1394,"арц":510,"арÑ":6391,"арт":6536,"ару":699,"аÑа":1276,"арш":342,"аре":1441,"ард":1546,"арб":390,"ара":7231,"арг":210,"арв":113,"арн":2120,"арм":468,"арп":116,"аро":3543,"бу ":460,"ари":4397,"арл":1030,"арк":1412,"аÑу":214,"аÑÑ‚":10621,"ата":4681,"аÑи":1271,"аÑе":15255,"аÑл":611,"аÑк":500,"аÑп":712,"аÑо":748,"аÑн":1469,"ату":1549,"ауб":95,"атв":223,"ате":3408,"ати":9099,"атл":474,"атк":530,"атн":1447,"атм":93,"ато":2391,"атÑ":4507,"атр":1269,"бож":165,"бол":831,"бом":290,"бок":146,"бог":924,"бод":695,"бов":463,"бор":1861,"бон":136,"бот":221,"боÑ":184,"бни":336,"бна":134,"вр ":135,"бно":688,"брн":93,"бро":2384,"ву ":4383,"брз":284,"аџи":118,"бри":1202,"бре":580,"брд":174,"бра":2962,"бру":739,"бољ":521,"бој":722,"бођ":100,"аља":600,"аљи":242,"аље":1718,"ајњ":180,"аљо":121,"аљу":134,"биш":118,"бич":725,"ајб":530,"аја":2689,"ајг":83,"ајв":1567,"аје":3121,"ајд":438,"ајз":386,"аји":1314,"биј":4013,"биљ":565,"ајк":465,"бињ":212,"ајл":672,"бић":203,"ајм":1497,"ајн":7063,"ајо":227,"ајп":549,"ајр":167,"ајÑ":1200,"ајт":407,"ају":3999,"ајф":161,"ајх":127,"ајц":375,"ајч":467,"аћа":316,"аћи":320,"аће":606,"бла":1682,"ања":4048,"бли":8473,"ањи":632,"бле":479,"ање":5548,"бло":178,"ањо":120,"во ":5192,"ањÑ":162,"ању":1132,"беј":91,"беђ":131,"ви ":16486,"бео":120,"бен":1184,"бет":151,"беÑ":248,"бер":3715,"без":706,"бед":465,"бел":853,"бек":184,"бег":170,"биц":333,"бир":422,"бит":1370,"биÑ":240,"бим":119,"бил":6721,"био":4368,"бин":1064,"биб":102,"бив":1525,"бзи":113,"ве ":5887,"бањ":123,"бај":140,"бат":207,"баÑ":218,"бар":2176,"бац":280,"бах":889,"баш":126,"бач":343,"ађу":117,"ађи":181,"ађе":817,"ађо":95,"ађа":1192,"дбе":122,"дби":132,"дај":799,"даљ":844,"дањ":368,"дба":1697,"дач":129,"дац":201,"даш":393,"дан":4942,"дам":540,"дап":114,"ег ":1363,"дао":250,"дар":1892,"дат":1152,"дви":424,"два":1043,"две":625,"дбр":248,"ед ":1208,"дбо":137,"еб ":222,"дак":211,"дал":853,"дав":818,"даг":152,"ев ":380,"дев":361,"деб":169,"дек":279,"дем":734,"дел":3577,"део":1026,"ден":4057,"дер":3109,"деп":4528,"деј":406,"дећ":331,"дељ":596,"деф":396,"деÑ":1778,"дет":316,"деш":161,"дец":815,"еж ":110,"дго":498,"дво":732,"ез ":996,"дгр":142,"еа ":228,"ех ":100,"дÑј":100,"дÑÑ…":127,"дÑÑ‚":2337,"еф ":157,"дÑк":2156,"дÑе":669,"дÑб":213,"дрш":85,"еу ":86,"дро":606,"дру":3934,"држ":14521,"дре":1505,"дри":545,"дрв":213,"дра":1821,"доњ":201,"ет ":2403,"еш ":152,"дфр":132,"дха":112,"дуј":1810,"еч ":332,"дућ":111,"дур":108,"дуÑ":249,"дуц":108,"дух":450,"душ":198,"дуа":101,"дуб":168,"дув":92,"дуг":396,"дуж":674,"дуз":216,"дук":334,"дул":103,"дун":303,"ец ":275,"гље":108,"ен ":11412,"диш":14641,"дич":217,"диц":1561,"диф":139,"ем ":4511,"дињ":631,"дић":425,"диј":3395,"диг":489,"див":514,"дим":783,"дин":8562,"ел ":2175,"дио":1175,"дип":178,"дир":430,"диÑ":935,"дит":511,"дид":113,"диз":259,"дик":313,"дил":663,"ек ":825,"доб":1676,"дож":120,"доз":192,"дов":1958,"дог":363,"дод":480,"доп":210,"ÐµÑ ":778,"доÑ":743,"дор":1358,"дол":780,"док":1240,"дон":1598,"дом":1333,"дош":123,"дна":2901,"дни":4153,"дне":1150,"дно":18824,"ер ":5027,"дну":350,"дма":127,"дме":197,"дми":757,"дмо":469,"еп ":122,"дла":144,"дле":110,"дли":278,"ео ":1513,"дло":145,"длу":219,"да ":21560,"гад":292,"гал":396,"гам":317,"гањ":112,"гај":141,"гађ":246,"гач":132,"гаш":101,"гаÑ":227,"гат":525,"гау":144,"ган":3079,"гао":220,"гар":1084,"гво":89,"де ":3882,"гви":147,"где":607,"гда":83,"вук":158,"вуч":103,"вуј":109,"вца":402,"вце":222,"вци":497,"вцу":236,"вше":911,"вшо":105,"вши":448,"гон":372,"гол":312,"гом":499,"гоÑ":1688,"гот":321,"гор":1661,"гош":127,"гов":8734,"год":5194,"гог":627,"гои":426,"гоз":827,"гно":115,"др ":192,"гну":250,"гна":134,"гне":319,"гни":146,"гру":1851,"ду ":4304,"гро":436,"грч":867,"грб":106,"гра":17750,"гри":166,"гре":1035,"гој":432,"гоњ":233,"гту":131,"гто":97,"гÑл":98,"гÑб":103,"гха":104,"гућ":509,"гуј":168,"гуÑ":706,"гур":326,"гум":88,"гун":84,"гул":187,"губ":222,"гет":151,"геј":143,"геб":295,"гед":238,"гез":107,"ген":3069,"гел":241,"гер":454,"ди ":2469,"гео":722,"гим":517,"гиÑ":230,"гит":213,"гин":376,"гио":6793,"гип":234,"вља":2757,"вље":1142,"вљу":193,"вљи":116,"вја":130,"вје":4514,"гич":129,"гих":425,"гиј":3453,"гињ":124,"гме":124,"гма":140,"гле":1228,"гла":3020,"глу":599,"гло":280,"до ":2950,"гли":311,"жањ":128,"жба":86,"жбе":201,"жај":441,"жби":131,"жан":1034,"жал":90,"жар":134,"жав":14295,"зв ":108,"ος":117,"за ":8613,"ешћ":540,"еју":172,"еља":1079,"еље":14036,"ељк":93,"ељи":403,"ељн":90,"ељу":544,"ељÑ":195,"жит":83,"жич":92,"жиц":127,"жиш":102,"еја":612,"ејв":144,"еје":348,"еји":839,"жиј":121,"ејк":85,"ејл":91,"ејм":173,"ος ":117,"ејн":153,"ејо":108,"ејÑ":626,"ејт":90,"жив":4026,"жир":301,"жиÑ":135,"жио":111,"жим":106,"жин":429,"жил":151,"жењ":609,"жељ":130,"жем":122,"жел":174,"зи ":3995,"жен":1361,"жев":999,"жег":248,"еђу":2825,"жди":88,"жде":98,"зе ":1198,"еђе":1098,"еђи":194,"еђа":194,"иХ ":284,"зу ":549,"жрт":88,"жно":1981,"жни":671,"жне":211,"жна":151,"зо ":136,"ењу":566,"ењÑ":222,"ењи":241,"ење":2556,"ења":1959,"ећо":157,"ећу":200,"ећи":1696,"ећа":1119,"еће":1229,"Ï‚ ":235,"жуп":1922,"жут":87,"жуј":115,"жа ":368,"еак":214,"ежа":300,"ежн":187,"еј ":684,"ежи":857,"еже":462,"еду":2402,"едр":262,"едÑ":2840,"жи ":752,"Ђур":142,"едњ":1513,"α ":172,"езу":767,"еин":162,"еир":106,"езб":299,"еза":1560,"езн":11980,"ељ ":395,"езо":593,"езв":1298,"ежђ":133,"езг":91,"езд":473,"езе":1288,"ези":2451,"ебу":238,"ебр":919,"ебо":119,"ебн":548,"ева":2735,"ебљ":174,"еви":3027,"еве":5206,"еар":187,"Ђор":119,"еат":111,"еал":333,"еан":354,"еба":904,"еби":695,"ебе":787,"егр":445,"его":7667,"егу":205,"едв":232,"едб":129,"еда":4271,"еде":3246,"едг":91,"еди":18581,"едо":1827,"едн":17721,"едм":296,"едл":104,"евн":1038,"же ":1650,"ево":1640,"евр":650,"евÑ":729,"еву":170,"евц":292,"ега":1064,"еге":504,"еги":7687,"егл":144,"евљ":87,"ент":9797,"ену":1523,"енр":171,"енÑ":2955,"енц":1314,"енч":83,"енф":253,"енх":510,"емљ":1229,"енк":694,"енл":133,"ени":9840,"ено":12442,"енг":675,"енв":171,"енб":2610,"ена":7015,"енз":581,"ене":4034,"енд":1652,"еоÑ":145,"енљ":86,"еол":657,"еом":497,"еон":274,"еоп":167,"еор":1067,"еоб":155,"еод":197,"еог":2215,"енш":218,"епт":829,"епу":6492,"епр":922,"епо":878,"епи":567,"епе":474,"епа":4865,"ерш":235,"еру":849,"ерф":172,"ерх":186,"ерц":6125,"ерп":274,"ерÑ":1727,"ерт":854,"ерл":469,"ерм":1235,"ерн":3846,"жу ":162,"еро":4629,"ери":8354,"ерз":950,"ерк":557,"ерд":412,"ерг":2878,"ере":2380,"ера":7017,"ерв":952,"ерб":428,"дје":329,"ећ ":287,"еке":903,"еки":816,"дљи":143,"екл":1752,"еко":3685,"екр":251,"ект":6659,"екÑ":2140,"еку":831,"ека":2874,"екв":261,"елн":382,"елм":190,"елк":306,"ели":7167,"дњи":405,"елх":90,"дњу":139,"елу":1959,"елт":171,"елÑ":443,"ело":4882,"дњо":511,"екц":169,"елд":739,"дње":682,"еле":3555,"елг":118,"дња":475,"ела":3270,"елб":151,"емо":949,"емн":211,"еми":2164,"ему":676,"емп":302,"емÑ":396,"еме":3254,"ема":24141,"емб":3007,"ехн":475,"ехо":95,"еха":297,"еца":115,"еци":860,"еце":786,"еча":464,"ецу":92,"ечк":150,"ечн":680,"ечи":726,"ече":483,"еша":500,"ешт":2328,"ешн":216,"ешк":621,"ечј":98,"еши":203,"еше":271,"еÑа":822,"еÑв":112,"еÑе":1676,"еÑи":981,"еÑк":1340,"ерњ":218,"еÑл":277,"еÑн":2049,"еÑм":470,"еÑп":273,"еÑо":641,"еÑу":289,"еÑÑ‚":11967,"еÑÑ…":103,"ета":4650,"етв":788,"ети":2810,"ете":1561,"етл":282,"етк":797,"етр":1300,"етп":147,"ето":2098,"етн":2013,"ету":4517,"етÑ":2919,"етх":191,"еуд":103,"еук":143,"етљ":101,"етњ":220,"еуз":123,"еут":176,"еур":88,"ефе":4639,"ефа":304,"ефо":247,"ефи":517,"И ":215,"ибу":225,"ибл":386,"ибо":357,"ибр":131,"иве":1702,"иви":1641,"ива":4282,"иак":116,"иби":356,"ибе":609,"иба":325,"К ":688,"Л ":171,"Ðœ ":193,"ижа":175,"иже":927,"ижи":106,"ижн":162,"Ð ":104,"идо":324,"идн":616,"иду":243,"идÑ":126,"идр":302,"игр":1750,"игÑ":188,"игм":91,"игл":124,"О ":291,"иго":261,"игн":387,"игу":383,"ида":820,"иди":470,"идл":111,"иде":1216,"ивр":422,"иво":1982,"ивн":2641,"П ":278,"ивш":1479,"иву":230,"ига":1246,"ивј":1819,"ивљ":215,"иги":748,"иге":674,"икл":343,"икÑ":296,"икр":200,"ико":2826,"ике":3772,"Ð ":487,"ики":922,"ика":16078,"ић ":4301,"зје":128,"Б ":209,"アア":86,"Ð’ ":325,"изу":761,"изр":754,"изм":2464,"изл":469,"иљ ":194,"изо":941,"изн":934,"изи":1871,"изд":704,"изг":522,"изе":216,"иза":4363,"изв":1974,"изб":400,"Г ":98,"иј ":194,"иом":122,"ион":10502,"иок":107,"иол":450,"иор":162,"иоп":96,"иот":219,"инц":1099,"инх":152,"иод":791,"иог":169,"Д ":487,"ине":13774,"инж":128,"ини":19900,"имљ":315,"инк":386,"ино":2092,"инÑ":14428,"инт":1353,"ину":1584,"инф":389,"ина":25163,"Е ":100,"инв":151,"инд":1081,"инг":3441,"ими":1186,"име":4873,"иму":393,"имÑ":660,"имо":935,"имп":1016,"имн":293,"илу":422,"имб":441,"има":9275,"или":9465,"илд":262,"иле":1239,"илÑ":244,"илт":96,"илм":1501,"илн":453,"ило":6433,"икш":116,"ику":1972,"икт":253,"ила":5198,"илв":95,"иÑи":951,"иÑе":634,"иÑа":3378,"иÑц":582,"иÑÑ…":121,"иÑу":8052,"иÑÑ‚":15326,"иÑп":745,"иÑо":651,"иÑн":740,"иÑм":208,"иÑл":783,"иÑк":1574,"ити":3953,"итв":152,"итд":137,"ите":7069,"ита":4182,"итб":246,"Ј ":683,"иту":1129,"ито":3248,"итÑ":285,"итр":493,"итл":226,"итк":519,"итн":680,"итм":218,"ипа":10217,"ипе":470,"иоц":158,"ипт":214,"ипÑ":176,"ипр":120,"ипу":126,"ипи":313,"ипо":546,"ипл":633,"ира":4573,"ирг":162,"ире":985,"ирÑ":409,"иру":1957,"ирт":1182,"ирх":398,"ирц":88,"ири":2830,"ирк":377,"ирн":181,"ирм":121,"иро":1739,"ихв":144,"иха":399,"ихи":202,"ихе":169,"ихт":106,"ихо":1238,"ице":3082,"ица":6758,"ици":5856,"ицу":455,"ицо":160,"ифа":124,"ифе":473,"ифи":670,"ифо":192,"ифр":1763,"ифх":92,"ишљ":461,"ишњ":1193,"ишћ":818,"ичу":145,"ичи":1889,"ичк":7995,"ичн":3458,"ича":2409,"иче":1493,"ишу":252,"ишт":18544,"ишк":400,"иши":415,"ишн":213,"иша":620,"ише":2071,"ка ":39689,"ив ":1671,"зав":1036,"заб":470,"зад":497,"заг":167,"заз":185,"збу":130,"ид ":941,"збо":708,"зањ":158,"зви":1871,"зве":860,"зва":825,"заш":441,"зач":117,"зац":2143,"зах":250,"зау":217,"зат":525,"заÑ":773,"зар":677,"зап":3143,"иг ":1521,"зао":108,"зан":1598,"зам":1010,"зал":371,"зак":756,"зби":265,"зај":1067,"збе":327,"зго":178,"згр":481,"зде":188,"зди":94,"зда":994,"здв":108,"зву":144,"зво":1776,"звр":300,"згл":149,"жђа":128,"зећ":156,"зељ":93,"зеј":114,"зет":340,"зер":989,"зел":607,"зен":1338,"зем":1233,"зду":396,"из ":14002,"здр":240,"зид":143,"зив":2326,"зим":680,"зил":849,"зик":1839,"зир":341,"зио":440,"ил ":1271,"зин":605,"ик ":5538,"ин ":3321,"им ":10232,"зиј":1899,"жје":87,"жја":87,"зиц":570,"зиш":103,"зич":979,"зит":988,"ип ":474,"зму":125,"зми":443,"зме":1661,"зма":563,"зло":335,"ио ":7229,"зли":1074,"жња":91,"зла":578,"зна":5348,"зне":283,"зни":867,"зно":12141,"ир ":1849,"зов":1108,"зод":97,"зом":405,"зол":148,"Ð¸Ñ ":1009,"зон":699,"зор":503,"зоф":485,"ит ":777,"зра":882,"зре":249,"зро":128,"их ":25752,"зул":422,"зуз":162,"зуч":91,"зур":195,"иц ":714,"зум":622,"зуј":637,"ич ":252,"иш ":433,"шћ":1913,"шњ":2514,"шљ":564,"хе":2594,"хи":2850,"хл":164,"хн":515,"хо":3887,"Ñ…Ñ€":1677,"Ñ…Ñ":192,"Ñ…Ñ‚":445,"ху":632,"Ñ„Ñ…":116,"ха":4269,"хв":996,"ци":25708,"цн":139,"цк":319,"цл":158,"цр":1743,"цо":468,"цу":6249,"хш":139,"цб":191,"ца":10662,"це":15022,"цг":117,"цв":296,"чл":795,"чм":103,"чн":8899,"чо":443,"чи":8450,"цј":1649,"чк":26459,"чу":1749,"че":8488,"ча":7145,"чв":283,"шо":406,"шп":317,"шн":886,"шк":4442,"чњ":99,"шл":364,"ши":6327,"чј":809,"шч":215,"шт":64295,"шу":691,"ше":5258,"шв":269,"ша":3738,"Ñк":73819,"рљ":151,"рј":516,"рћ":85,"Ñм":5030,"Ñл":11320,"рњ":1982,"Ñо":9678,"Ñн":13876,"ÑÑ€":17094,"рџ":149,"Ñп":6049,"Ñв":7773,"Ñб":1179,"Ñд":436,"рђ":833,"Ñг":131,"Ñе":44770,"Ñи":12767,"рш":2918,"Ñа":33351,"Ñ€Ñ":13440,"рт":11412,"ру":33347,"рф":1371,"рх":2668,"рц":7114,"рч":1587,"тн":6309,"тм":5097,"тл":1743,"тк":2545,"Ñ‚Ñ":9086,"Ñ‚Ñ€":21007,"тп":835,"то":37020,"те":36966,"тд":161,"тв":12471,"Ñј":2502,"ти":79813,"тз":114,"та":69266,"тб":348,"Ñу":21121,"ÑÑ„":694,"ÑÑ‚":82417,"ÑÑ…":1012,"Ñц":1134,"ур":15198,"уп":9618,"ут":7405,"уÑ":13259,"ум":6429,"ул":6994,"Ñ‚Ñš":299,"уо":641,"ун":8172,"уи":452,"уз":4197,"ук":5623,"Ñ‚Ñ™":189,"тј":481,"уд":6767,"уг":23579,"уж":4717,"уе":639,"уа":1983,"уб":10441,"ув":1407,"ту":17614,"Ñ‚Ñ„":592,"Ñ‚Ñ…":390,"фу":2311,"Ñ„Ñ‚":307,"Ñ„Ñ":704,"уџ":113,"Ñ„Ñ€":3373,"фо":3036,"ућ":1931,"уњ":301,"фл":420,"уљ":267,"уј":6353,"фи":5806,"фе":8951,"уђ":267,"фа":2762,"уч":4458,"уш":3354,"ух":1368,"уц":1110,"уф":272,"џе":228,"џа":369,"џи":363,"њу":2111,"њо":1184,"ÑšÑ":793,"ћи":4498,"ћк":165,"ће":4723,"ћа":3529,"љу":3257,"Ñ™Ñ":735,"јџ":83,"љн":429,"љо":421,"јњ":183,"љк":483,"њи":5652,"ње":25640,"ња":11305,"ћу":1016,"ћн":479,"ћо":225,"јш":265,"ља":6896,"љи":2197,"ље":20267,"јл":855,"јк":776,"ји":23301,"јп":587,"јо":3515,"јн":8532,"јм":1846,"ју":14903,"јт":676,"јÑ":6913,"јр":268,"јч":534,"јц":574,"јх":352,"јф":416,"ја":44066,"јб":617,"јв":2381,"јг":134,"јд":673,"је":146704,"јз":552,"ђу":3076,"ђо":146,"ђи":555,"ђе":5047,"ђа":2396,"Ју":2303,"Јо":811,"Ја":1517,"Је":12368,"Љу":413,"Њу":211,"Њи":110,"Ње":449,"а ":310619,"Р ":421,"С ":431,"Т ":144,"У ":1965,"Ђа":97,"Ф ":135,"Ђо":174,"Ђу":229,"Ð¥ ":378,"Ц ":102,"Ир":324,"ИÑ":1207,"Ит":448,"Им":514,"Ин":925,"к ":12340,"Ик":105,"Ил":414,"Из":699,"Иг":191,"Ив":390,"Иб":105,"Иа":105,"Зу":210,"Зр":87,"Зо":292,"Зл":467,"Зи":401,"Ле":1326,"Ли":1997,"Ла":2339,"Ку":1279,"Кл":913,"Кн":244,"Ко":7323,"м ":42843,"Кр":2869,"Ке":678,"Кв":150,"Ки":1180,"Ка":4883,"л ":7046,"Ðа":4229,"Ðе":1665,"Ðи":1775,"Мр":192,"Му":528,"о ":97605,"Мо":2251,"Ма":5725,"Мл":210,"Ми":2919,"Ме":3482,"н ":35328,"Ло":1918,"Књ":126,"Лу":1170,"Па":5381,"Пе":2174,"Пи":1995,"Пл":687,"По":7950,"Ñ ":9421,"Оп":1309,"Ñ€ ":19435,"ОÑ":1730,"Ор":832,"От":286,"Ох":85,"Оа":99,"Об":952,"Од":735,"Ов":1319,"Ог":84,"Оз":108,"Ом":122,"Он":644,"Ок":613,"Ол":521,"п ":2907,"Ðо":2845,"Ðу":145,"в ":4786,"Ðп":242,"Ðм":917,"Ðн":1871,"Ðк":748,"Ðл":2798,"Ðз":496,"Ðд":582,"Ðв":348,"Ðг":213,"Ðб":271,"Ба":6851,"ÐÑ…":118,"ÐÑ„":342,"Ðу":783,"ÐÑ‚":942,"ÐÑ":250,"ÐÑ€":1905,"б ":1710,"Ðо":462,"Ðи":97,"Ðе":413,"Вл":769,"Ð’Ñ€":797,"Во":1407,"д ":41112,"Ве":3053,"Ви":3647,"Бј":217,"Гв":148,"Га":1178,"Ву":646,"Ðј":796,"Бл":374,"Бо":7744,"г ":28611,"Бр":3072,"Бе":4333,"Би":2691,"Ва":2337,"Бу":2024,"Ди":1375,"Де":1690,"Ду":1074,"Др":2279,"ж ":448,"До":4484,"Ег":317,"Ед":208,"Ев":965,"Ги":403,"Ге":866,"Гр":3342,"Гл":769,"е ":235734,"Го":3576,"Гу":327,"Да":1824,"Дв":240,"Жи":530,"и ":186242,"Жу":184,"Зб":101,"Зв":315,"За":3384,"Зе":1084,"Ек":250,"Ем":298,"Ел":532,"з ":16545,"Ен":728,"Ер":728,"Еп":349,"Ет":201,"ЕÑ":301,"Еу":95,"Жа":255,"Же":368,"Шл":1326,"Шк":142,"Ши":626,"Шп":414,"Шо":300,"Шм":122,"Шу":240,"Шт":864,"Шв":589,"Ша":1762,"Ше":616,"Ст":2384,"Су":1195,"Та":1272,"Сј":431,"Ти":1785,"Тв":83,"Те":1308,"Ñ„ ":1728,"То":1598,"Тр":1845,"Ту":877,"Ук":239,"Уз":91,"Уд":151,"Уг":187,"Ут":114,"УÑ":287,"Ур":216,"Уп":276,"Ñ… ":27795,"Ун":652,"Ум":120,"Ул":137,"Пр":12883,"ПÑ":86,"Пу":267,"Пч":135,"Ра":5749,"Ре":7578,"Ри":1147,"Пљ":86,"Ñ‚ ":18408,"Ро":2099,"Ру":1955,"Са":5493,"Св":1412,"Си":1482,"Се":3616,"См":461,"Сл":2083,"Ск":586,"Ср":8976,"Сп":690,"Со":1389,"у ":163872,"Цв":112,"Це":942,"Ца":232,"ш ":2445,"Цр":1575,"Ци":219,"Че":601,"Ча":537,"Чу":105,"Чо":83,"Чи":273,"Фа":508,"ц ":5646,"Фо":828,"Фр":5849,"Фу":461,"Фе":1908,"Фи":915,"Уј":251,"Фл":377,"Ха":1787,"Хр":3553,"ч ":1834,"Хо":2226,"Ху":405,"Хи":666,"Хе":7160,"мб":4820,"ма":59507,"ме":28451,"мд":107,"ми":14537,"мл":591,"мк":113,"књ":1128,"кћ":91,"лм":2649,"лн":10609,"ло":27832,"лп":842,"лÑ":2735,"лт":3191,"лу":8870,"лф":579,"лх":258,"лц":781,"лш":1376,"лв":428,"лб":1478,"ла":45745,"ле":21446,"лд":2395,"лг":595,"кљ":675,"лк":1280,"ли":48289,"лз":191,"км":1561,"кн":662,"кл":5535,"кр":24074,"кÑ":5786,"ко":94888,"кх":134,"кт":10243,"ку":10596,"кш":274,"кц":1692,"ка":57877,"ки":34381,"кв":4521,"ке":19190,"Ñ› ":4823,"иџ":130,"иј":65389,"иљ":1736,"ињ":3046,"ић":7527,"иш":25602,"иђ":179,"Ñš ":751,"ио":20368,"ип":13525,"им":29885,"ин":88842,"ик":32559,"ил":26990,"зј":195,"иц":17142,"ич":17794,"иф":3473,"их":28103,"ит":23396,"ир":17174,"иÑ":35020,"ри":59126,"пј":236,"рк":4376,"пљ":819,"рл":2194,"рм":4605,"рн":11097,"ро":38332,"рп":7588,"ра":86875,"рб":5621,"рв":10734,"рг":10628,"рд":4407,"ре":76600,"рж":15319,"рз":1501,"пш":36896,"пч":155,"пр":51210,"пт":1690,"пÑ":9353,"пф":93,"пу":10490,"пц":293,"пх":131,"ој":81044,"пк":142,"ољ":2891,"пи":15225,"пн":1028,"по":43591,"оњ":2765,"пл":4178,"оћ":912,"па":29402,"пе":8683,"ођ":2674,"ош":4387,"оч":6007,"оц":4534,"оÑ":40837,"ор":39900,"нџ":226,"оп":51144,"оо":430,"ох":1509,"оф":2614,"оу":1143,"от":10383,"ок":28856,"нљ":100,"ол":17145,"ом":36538,"он":34353,"ож":2953,"оз":8406,"ои":5016,"ов":55896,"нђ":336,"ог":30691,"од":53599,"ое":545,"оа":1915,"об":14901,"нц":9206,"нх":1153,"нш":840,"нч":575,"нт":17280,"нÑ":27485,"нф":1024,"ну":19119,"но":93266,"ÑŸ ":107,"нр":483,"нп":198,"нл":270,"нм":157,"мј":5028,"мљ":1678,"нк":3713,"нз":1151,"ни":94588,"не":40351,"нж":264,"нг":6034,"нд":8864,"нб":3109,"нв":605,"на":115144,"мш":112,"мц":179,"мф":180,"му":5395,"мÑ":2960,"мр":1007,"мп":3351,"мо":14075,"мн":1634,"ге":7187,"гд":811,"вј":4674,"вљ":4218,"гз":88,"ги":13875,"гн":968,"го":22475,"гл":5570,"гм":306,"гÑ":345,"гт":294,"гр":22464,"гх":130,"гу":5955,"гш":129,"дг":708,"дв":2905,"дб":2411,"да":35561,"вг":764,"вд":274,"ве":45977,"ви":47733,"бј":964,"бљ":690,"вк":390,"вл":2091,"бњ":88,"вн":21104,"во":25024,"вр":9405,"вÑ":2479,"ву":5035,"вч":192,"вц":1358,"вш":1624,"га":23320,"гв":336,"бз":146,"би":23648,"бе":9199,"ађ":2448,"бд":119,"аџ":248,"бр":8596,"бн":1295,"бо":7576,"бл":10968,"ањ":12252,"аћ":1384,"ај":30020,"аљ":3397,"бу":6812,"бÑ":108,"ва":49542,"ад":42109,"ае":747,"аж":2528,"аз":15348,"аб":3706,"ав":58459,"аг":5540,"ам":16256,"ан":80786,"Ñ’ ":91,"ао":8763,"ап":8907,"аи":605,"ак":19402,"ал":40672,"ах":2983,"аф":2148,"ач":24712,"ац":14985,"аÑ":34373,"ар":47766,"ау":5673,"ат":35129,"ба":12073,"аш":5217,"зр":1331,"иХ":289,"зу":2900,"жј":268,"зи":16003,"Ñ™ ":1114,"зо":3900,"зн":18686,"зм":2905,"жњ":238,"зл":2056,"ив":18317,"иг":7473,"иа":237,"иб":2549,"иж":1542,"из":30460,"ид":5409,"ие":162,"жо":177,"ј ":48286,"жр":91,"жу":2380,"еј":4470,"жи":6697,"ењ":5600,"ељ":16871,"жн":3064,"ећ":4760,"за":24448,"зб":1569,"зв":5905,"жђ":219,"зг":986,"зд":2099,"зе":6370,"еф":6020,"еу":929,"ет":26936,"еÑ":21968,"ер":49978,"еп":15485,"ео":7262,"ен":68315,"ем":41012,"дњ":2216,"ел":25958,"ек":21254,"дљ":162,"дј":352,"еи":490,"ез":21921,"еж":2298,"жд":302,"же":5505,"еђ":4350,"жа":16716,"жб":477,"еч":3067,"еш":5057,"ех":1267,"ец":2261,"дÑ":5798,"др":23445,"ду":9933,"дн":27401,"дм":1566,"до":15789,"ди":38559,"гњ":86,"дл":918,"гљ":169,"де":26907,"дз":137,"еб":4721,"ев":16201,"ег":19300,"ед":54814,"еа":1573,"дф":213,"дх":157,"ђи ":137,"ђе ":1005,"ђа ":629,"ђор":83,"ђењ":699,"ђив":257,"ђев":272,"ђен":2693,"ђел":175,"ђањ":91,"ђај":335,"ђач":140,"ђар":463,"ђан":442,"ђав":223,"ђу ":1809,"ушћ":97,"ха ":436,"уха":230,"ухо":341,"ухв":534,"уце":178,"уци":804,"уча":728,"учу":355,"учн":763,"учи":534,"уче":1175,"уша":487,"ушт":1121,"ушн":189,"учј":690,"ушк":628,"уше":325,"уши":195,"хе ":209,"уђе":173,"феÑ":847,"фед":157,"феб":471,"фер":989,"фен":546,"фел":1161,"фек":4515,"фар":175,"фам":285,"фан":502,"фал":512,"фак":464,"фаз":142,"фаб":126,"фло":155,"уће":434,"ућа":330,"уње":115,"уња":132,"фов":93,"ући":802,"ућн":239,"ућу":84,"фиг":103,"ујо":136,"ују":1186,"ује":4514,"фиј":646,"уља":126,"фин":837,"фит":117,"фир":149,"физ":617,"фил":2015,"фик":680,"уја":305,"фиц":170,"фич":243,"фур":105,"фуд":1381,"фун":652,"фхо":95,"фри":594,"фре":160,"фра":725,"фот":183,"фор":2033,"фол":134,"фон":348,"фтв":101,"Ñ„Ñк":663,"ху ":202,"фро":157,"фру":1715,"ца ":8462,"хер":413,"хел":163,"ци ":5975,"хео":230,"хем":656,"хен":509,"хев":99,"це ":3788,"хва":984,"хал":405,"хам":134,"хан":387,"хар":360,"хау":611,"хаф":91,"хаг":129,"хаи":135,"хај":1188,"хри":538,"хра":374,"хрв":394,"хоф":175,"хол":640,"хом":223,"хок":106,"хоп":195,"хор":321,"хно":141,"хни":346,"хов":1243,"ход":493,"хла":98,"хиј":1039,"хид":139,"хит":284,"хип":394,"хин":106,"хим":207,"хич":96,"хшт":83,"хум":204,"хте":253,"Ñ…ÑÑ„":89,"хро":311,"цу ":1011,"Ñрп":2456,"Ñтм":124,"Ñто":18048,"ÑÑ‚Ñ€":8594,"Ñтв":4838,"Ñте":5340,"Ñти":11681,"Ñта":26605,"Ñур":199,"ÑуÑ":280,"Ñуп":505,"Ñум":91,"Ñун":122,"Ñук":182,"Ñул":102,"Ñуд":629,"Ñув":97,"Ñуб":180,"Ñуа":202,"ÑÑ‚Ñ„":499,"Ñту":2121,"Ñфо":135,"Ñуј":205,"Ñуђ":102,"Ñфе":494,"Ñуш":111,"Ñут":178,"Ñхо":131,"Ñха":631,"Ñце":455,"Ñци":572,"Ñца":94,"уа ":129,"уб ":744,"тав":5969,"тад":668,"таб":573,"так":3250,"тал":4047,"там":556,"тан":13504,"таз":140,"тау":158,"тат":1577,"таф":105,"тап":100,"тао":464,"уг ":985,"таÑ":541,"тар":5550,"тач":1717,"тац":1165,"таш":269,"уд ":223,"тањ":1253,"тај":2712,"таљ":208,"тбу":301,"уж ":93,"тву":884,"твр":969,"уе ":91,"тво":2793,"тви":271,"тве":1074,"тва":6461,"теф":236,"тех":457,"теч":207,"теш":240,"тем":4118,"тел":3005,"тео":771,"тен":2926,"тер":8164,"теп":335,"тет":6112,"теÑ":265,"тед":132,"тег":425,"теж":433,"теи":123,"тез":220,"тек":983,"тев":356,"уз ":472,"тиг":306,"тив":3595,"тид":114,"тиб":241,"ук ":347,"тзв":108,"тељ":925,"тењ":110,"теј":125,"тј ":244,"тећ":113,"тка":747,"тке":238,"тиј":2163,"тиљ":126,"тињ":565,"тић":314,"ум ":1118,"тиш":108,"тич":4693,"тиц":1520,"Ñја":94,"Ñје":2389,"тиз":382,"тим":1937,"тин":41889,"тик":1442,"тил":706,"тир":1382,"тиÑ":676,"тио":325,"ул ":556,"тип":584,"тиф":212,"тих":863,"тит":5421,"тлу":83,"уо ":134,"тло":233,"тме":159,"тма":4738,"ткр":261,"тку":270,"тко":705,"ун ":634,"тки":298,"тле":453,"тли":403,"тла":526,"тну":158,"тно":2527,"ур ":576,"тоз":134,"тод":371,"ток":2271,"тол":1112,"тои":1244,"тов":1934,"тог":1199,"тоб":664,"тмо":118,"уп ":544,"тне":555,"тни":2279,"тна":775,"тпу":223,"тпр":138,"тпо":276,"ут ":1197,"тре":2615,"трг":210,"трв":1452,"тра":8057,"трк":102,"три":3258,"трж":83,"тор":6320,"тоÑ":262,"тот":172,"том":2063,"тон":1779,"ÑƒÑ ":844,"топ":627,"тох":508,"точ":3504,"тој":1764,"тпи":109,"Ñ‚ÑÑ‚":153,"ух ":119,"тро":3120,"тру":1654,"Ñ‚Ñк":8634,"уф ":92,"уш ":86,"тхо":231,"тха":116,"туг":141,"туд":536,"тул":330,"тум":385,"туп":985,"тур":6672,"тун":177,"туа":281,"туј":202,"тфо":135,"тут":228,"туÑ":171,"туц":235,"тфа":417,"фа ":267,"уво":94,"уви":157,"убј":98,"убљ":204,"уве":506,"ува":512,"уго":4228,"угл":556,"уги":1217,"уге":625,"уга":13358,"уда":1291,"угу":2310,"угр":109,"уал":444,"уац":116,"уан":156,"уар":1082,"уби":913,"убл":6600,"уба":618,"убе":181,"убр":360,"убо":453,"убу":129,"узе":1365,"узд":102,"ужј":100,"узи":1311,"узо":105,"узр":146,"узв":99,"уза":304,"уже":711,"ужи":830,"ужн":2081,"ужа":406,"ужб":369,"уер":98,"уел":160,"уен":239,"уду":209,"удÑ":407,"уди":1350,"уде":530,"удв":207,"удб":1648,"удр":214,"удо":485,"удн":163,"умб":202,"уме":1673,"уми":518,"ума":1270,"улÑ":111,"улт":1736,"улу":266,"тњи":173,"ули":840,"улк":196,"укљ":356,"уло":814,"ула":1767,"уле":373,"укÑ":201,"укт":434,"укр":138,"укц":281,"уку":549,"уки":197,"тљи":143,"уко":1171,"укл":262,"уке":369,"ука":1212,"тје":162,"уиÑ":163,"упо":628,"Ñ„Ñ‚ ":102,"упн":661,"упу":323,"упÑ":196,"упр":1269,"упш":224,"ура":1239,"ург":4051,"урб":165,"ури":4983,"уре":1096,"упљ":436,"урк":103,"уоп":85,"уоÑ":180,"упа":3575,"упе":909,"упи":580,"унÑ":388,"уно":440,"унк":639,"унц":425,"уну":749,"унт":86,"уоб":88,"умо":176,"умр":281,"умÑ":122,"уму":713,"уна":2155,"уни":1710,"унг":142,"унд":396,"уне":181,"утр":371,"утÑ":99,"уту":188,"уÑÑ‚":2999,"уÑу":97,"уÑо":230,"уÑп":514,"уто":1313,"утн":844,"ути":1135,"уте":570,"утв":274,"ута":1163,"урш":100,"урÑ":607,"урт":171,"уру":316,"урн":928,"уро":361,"уÑл":737,"уÑк":5922,"уÑн":112,"уÑм":134,"уÑи":776,"уÑв":94,"уÑе":189,"уÑа":477,"шло":150,"шку":123,"чња":95,"шке":532,"шки":1029,"шко":1898,"шов":116,"шни":421,"шно":249,"шна":90,"шењ":346,"шка":843,"чју":367,"шић":458,"шиј":103,"чје":235,"чја":148,"ших":85,"шиф":1728,"шиц":139,"шир":819,"шин":1269,"шио":130,"шил":146,"шва":162,"шев":523,"шег":353,"шеÑ":305,"шет":83,"шен":657,"шај":207,"шањ":397,"шав":680,"шах":118,"шар":232,"шао":182,"шан":312,"шам":104,"шал":110,"шчи":96,"штр":216,"што":1671,"шту":218,"шум":244,"шпа":238,"шој":123,"штв":5051,"ште":3994,"шти":37563,"шта":15543,"шћу":343,"шће":923,"шћа":591,"шњи":882,"шње":857,"шњо":257,"шња":479,"шље":424,"шља":86,"цге":112,"цве":139,"цва":144,"че ":1166,"цен":2833,"чи ":1951,"цел":553,"цем":639,"цег":5749,"цеÑ":608,"цер":167,"цеп":196,"циљ":469,"циј":13318,"циз":111,"циг":120,"цив":149,"цип":664,"цио":2122,"циÑ":184,"цир":296,"цил":157,"цик":239,"цин":417,"цим":814,"цит":296,"циф":257,"ча ":1135,"цањ":160,"цбу":108,"цај":333,"цар":1250,"цам":251,"цуÑ":5145,"цла":132,"цко":142,"цна":120,"цог":137,"цом":173,"црв":233,"црн":309,"црк":917,"чу ":279,"црт":240,"чењ":1369,"чев":703,"чег":147,"чем":184,"чен":757,"чек":99,"чел":592,"чеÑ":1070,"чер":91,"чео":115,"ши ":1042,"чет":1463,"чеш":599,"чко":16119,"чки":5326,"чке":2191,"чла":792,"чку":322,"чић":352,"чињ":485,"цје":1643,"чиј":1063,"чка":2488,"чим":154,"чил":179,"чио":107,"чин":2262,"чив":224,"чит":1082,"чиц":356,"ша ":940,"чвр":128,"ше ":2479,"чва":109,"чај":1188,"чаÑ":347,"чак":300,"чар":1858,"чан":988,"чав":1025,"чне":725,"чна":776,"чов":351,"чни":2024,"чну":146,"чно":5223,"чув":312,"чуј":364,"чун":594,"шу ":312,"џер":98,"џиј":83,"џа ":116,"ћим":236,"ћин":617,"ћих":347,"ћел":450,"ћен":999,"ћем":203,"ћер":180,"ћењ":211,"ћев":207,"ћег":174,"ћој":145,"ћно":252,"ћни":156,"ђуÑ":133,"ђут":104,"ђун":548,"ђуј":344," ар":1531," аÑ":1495," ат":789," ау":1060," аф":120," ба":1782," аг":204," ав":1075," ае":132," ад":525," аз":106," ал":2165," ак":1263," ан":1456," ам":1558," ап":880," бу":724," ва":1641," би":13849," бе":1509," бр":3177," бо":2496," бл":699," ву":141," га":1017," ви":3100," ве":6374," во":2889," вр":3580," вл":1525," дв":1695," да":6529,"ÑšÑк":762," гу":273," го":5940," гл":2733," гр":13040," ге":1414," гд":614," вј":1068," ги":217,"ћу ":880," ев":376," ег":176," дн":118," до":8150," др":17143," ду":1399," де":12761," ди":3032," же":911," жа":185,"њој":397," еф":184," еÑ":167," ет":301," еп":599," ер":161," ен":4749," ел":1099," ем":401," дј":115," ек":993," зд":192," зг":134," зе":1269,"њом":267," за":14329," зв":844," зб":573," жу":2033,"њов":202," жр":88," жи":3507," зи":192," зл":353," зо":272," зн":1754," зр":158,"њев":193," иа":112,"њег":1511," иг":1416," ид":487," из":20909," ил":6136," ик":91," ин":3129," им":4445," ит":477," иÑ":6583," ир":131," их":414,"ће ":2252," ка":9529," ки":953," кв":334," ке":139," кр":4983," ко":34802," кн":423," км":304," кл":2025," ку":1563," ла":2233," кљ":83," ли":3146," ле":1584," лу":259,"њиц":291," књ":1120," кћ":91," ло":1091,"њих":951," ме":6800," ми":3211,"њић":169," мл":339," ма":5632," мо":4593," мн":508," му":1999," мр":337," ни":1843,"њиж":739," не":7050,"њим":687,"њин":185," мј":4621," на":44905,"њив":145,"њиг":412," ну":317," но":3063," нп":131," ол":413," ок":19137," он":950," ом":368," оз":712," ог":275," ов":1780," од":28239," об":6316," ош":91," оц":139," оч":225," оф":198," от":977,"њер":162,"ћи ":3090," ор":2923,"њен":1992," оÑ":5852,"њем":13062," оп":36889," по":35696,"њењ":256,"�":306," пл":2060," пи":1443," пе":3239," па":2758," Ре":7567," Ра":5738," Ро":2092," Ри":1146," Пљ":86," Пу":260," Пр":12819," ПÑ":86," Пч":135," Пе":2104," Па":5377," По":7935," Ñ ":397," Пл":685," Пи":1992," Ох":85," От":286," ОÑ":1726," Ор":831," Оп":1276," Те":1302," Ти":1784," Сј":431," То":1597," Тр":1837," Ст":2368," Су":1194," Та":1268," Св":1404," Си":1481," Се":3613," Сл":2081," См":461," Ск":586," Сп":690," Ср":8968," Со":1387," у ":76526," Ру":1950," Са":5488," Фр":5848," Фу":460," Фо":827," Уј":250," Фи":914," Фл":377," Фе":1907," Фа":506,"ћањ":215,"ћај":354," Ут":114," УÑ":286," Ур":216," Уп":276," Ун":652," Ум":118," Ул":135," Ук":237," Уз":91," Уд":150," Уг":187," Ту":874," Цр":1572,"ћав":181," Ци":218," Ца":232," Цв":112," Це":942," Хр":3549," Хо":2223," Ху":404," Хи":661," Хе":7159," Ха":1786,"ћан":574,"њуј":187," Ше":614," Шв":589," Ша":1760," Шо":300," Шп":414," Шм":122," Шк":141," Шл":1326," Ши":625," Шт":864," Шу":240," Че":600," Ча":535," Чо":83," Чи":272," Чу":105," Ба":6848," ÐÑ„":342," ÐÑ…":117," ÐÑ‚":942," Ðу":782," ÐÑ€":1901," ÐÑ":250," Ðп":241," Ðн":1869," Ðм":916," Ðл":2792," Ðк":746,"љиш":98," Ðз":496,"љиц":161," Ðд":581," Ðг":213," Ðв":348," Ðб":270,"љим":98," Ва":2332,"љин":408,"љив":745," Бу":2024," Бо":7741," Бр":3064," Ðј":794," Бл":374," Бе":4332," Би":2681,"јње":140," а ":3795,"љке":117," Ðи":97," Ðо":461,"љка":256," Ðе":412,"љни":159," Ед":207," Ег":317," Ев":963,"љно":170," Ди":1374," Де":1685," Ду":1072," Др":2274," До":4482," Же":368," Жа":255," Ек":250," Ел":532," Ем":298," ЕÑ":301," Ет":200," Еу":95," Ен":715," Еп":349," Ер":727," Ву":631," Гв":147," Га":1175," Ве":3050," Ви":3645," Бј":217," Вл":762," Ð’Ñ€":797," Во":1406," д ":244," Гу":327,"љон":115,"љоп":103," Да":1819," Дв":240," Ге":857," Ги":403," Гл":761," е ":253," Го":3576," Гр":3337," ИÑ":1205," Ир":324,"љаÑ":133," Ит":448,"љал":133," Ин":917,"љам":117," Им":500,"љан":705,"љао":106," Ил":412,"љак":314," Ик":105,"љаш":109,"љач":143," Ки":1180,"љав":466," Ке":678," Кв":150," Ка":4863," и ":42425," Жу":183," Жи":529,"ње ":8309," За":3380," Зб":101," Зв":313," Зе":1083," Зр":87," Зу":210,"љањ":379," Зи":400,"љај":470," Зо":292," Зл":467," Ив":384," Иг":188," Иа":105," Иб":105," Из":696," Мр":192," Му":523," о ":1337," Мо":2248,"љев":1384," Ðа":4218," Ðе":1661," Ðи":1773," Ðу":144," п ":1147," Ðо":2843," Од":732," Ог":84," Ов":1314," Об":952," Оа":97," Он":642," Ом":122," Ол":520," Ок":611," Оз":107," Кл":909," Кн":244," Ко":7314," Кр":2862," Ку":1279," Ла":2334," Ле":1325," Ли":1997," н ":236,"њи ":1901," Ло":1918," Књ":125,"љен":9209,"љем":246," Лу":1168," Ма":5708," Ме":3478,"љењ":432," Мл":210," Ми":2909," У ":1834," С ":120,"ћа ":2102,"њав":306," Ђу":228,"њац":140,"њач":180," Ђо":174,"њан":194,"њак":423," Ђа":97," Љу":412,"њу ":1891," Ја":1517," Је":12367," Ју":2300,"Ñ™ÑÑ‚":94,"Ñ™Ñк":641," Јо":807,"људ":763,"љуб":277,"ључ":650,"љуј":500," Њи":110," Ње":449," Њу":211,"јна":3212,"јнб":154,"јно":645,"јну":95,"јни":1500,"јне":363,"јма":315,"јмо":183,"јмÑ":112,"јло":133,"јли":84,"јле":319,"јка":209,"јко":160,"јке":100,"јиÑ":141,"јих":1002,"јић":104," Ð ":144,"јил":86,"јин":1068,"јим":1235,"јио":131," Б ":110,"јзе":206,"јзн":168,"јеÑ":4944,"јет":1944,"јен":3254,"љи ":494,"јеп":114,"јер":1439,"јеч":245,"јеш":1172,"јењ":122,"јељ":149,"јећ":109,"јез":2070,"јек":1138,"јел":2648,"јем":1164,"јев":2265,"јег":367,"јед":10980,"јдо":91,"јду":106,"јде":199," Ðœ ":110,"јво":585,"ље ":8612,"јви":622,"јве":956,"јва":185,"јбо":385,"јањ":589,"јај":268,"јаш":298,"јак":354,"јал":3500,"јам":1141,"јан":2625,"јао":110,"јап":268,"јар":388,"јаÑ":623,"јат":622,"јац":247,"јач":377,"јаг":137,"јав":1529,"јад":132,"ља ":3633,"ња ":9657,"јшт":110,"јца":205,"јцн":120,"јче":458,"јућ":704,"јфе":389,"јхÑ":103,"јте":185,"јул":555,"јум":460,"јун":673,"јур":125,"јуÑ":143,"јут":86,"југ":1799,"јуж":1274,"љу ":927,"јра":101,"јÑк":5249,"јÑÑ‚":654,"јÑе":169,"јÑи":92,"јон":495,"јом":1166,"јор":180,"јнш":97,"јој":768,"јпо":385,"још":604,"јм ":973,"јн ":2046,"ји ":19262,"је ":112273,"јд ":136,"ја ":30644,"ју ":8819,"јц ":119,"Ñ˜Ñ ":427,"јт ":210},"n_words":[7556811,8870200,7383394],"name":"sr","type":"cyrillic"} \ No newline at end of file
diff --git a/contrib/languages-data/stop_words b/contrib/languages-data/stop_words
new file mode 100644
index 0000000..0aad88d
--- /dev/null
+++ b/contrib/languages-data/stop_words
@@ -0,0 +1,4153 @@
+{
+ "fr": [
+ "seront",
+ "eûmes",
+ "mes",
+ "auront",
+ "étante",
+ "notre",
+ "été",
+ "eusses",
+ "même",
+ "vous",
+ "furent",
+ "nous",
+ "votre",
+ "avions",
+ "ayants",
+ "par",
+ "eussions",
+ "eues",
+ "moi",
+ "fut",
+ "fussiez",
+ "serions",
+ "sommes",
+ "qu",
+ "serons",
+ "serai",
+ "aurions",
+ "ayantes",
+ "qui",
+ "aurons",
+ "avaient",
+ "aurai",
+ "fûtes",
+ "avez",
+ "sont",
+ "aurait",
+ "aura",
+ "eusse",
+ "eût",
+ "étiez",
+ "était",
+ "ait",
+ "étés",
+ "tes",
+ "dans",
+ "aux",
+ "serait",
+ "est",
+ "sera",
+ "aurais",
+ "elle",
+ "êtes",
+ "auraient",
+ "eue",
+ "étée",
+ "avais",
+ "fussent",
+ "eus",
+ "ayant",
+ "serais",
+ "toi",
+ "ayez",
+ "soyons",
+ "seraient",
+ "fusses",
+ "étions",
+ "avons",
+ "eurent",
+ "auriez",
+ "aie",
+ "une",
+ "seriez",
+ "fûmes",
+ "eux",
+ "leur",
+ "ton",
+ "pour",
+ "étaient",
+ "étant",
+ "ses",
+ "ont",
+ "soit",
+ "eûtes",
+ "soyez",
+ "soient",
+ "aient",
+ "eussiez",
+ "eut",
+ "avec",
+ "sur",
+ "fussions",
+ "seras",
+ "mon",
+ "ayante",
+ "ayons",
+ "ces",
+ "aviez",
+ "aies",
+ "auras",
+ "fusse",
+ "fût",
+ "avait",
+ "suis",
+ "serez",
+ "étantes",
+ "fus",
+ "pas",
+ "étants",
+ "eussent",
+ "aurez",
+ "étées",
+ "étais"
+ ],
+ "id": [
+ "mungkinkah",
+ "gunakan",
+ "sebelumnya",
+ "penting",
+ "percuma",
+ "sekecil",
+ "masalahnya",
+ "teringat",
+ "sedangkan",
+ "melalui",
+ "boleh",
+ "kamilah",
+ "depan",
+ "amatlah",
+ "bermaksud",
+ "rasanya",
+ "harusnya",
+ "bermacam-macam",
+ "betulkah",
+ "berarti",
+ "dari",
+ "lanjut",
+ "haruslah",
+ "tegasnya",
+ "keinginan",
+ "mengibaratkan",
+ "dikatakan",
+ "tadinya",
+ "terdahulu",
+ "tunjuk",
+ "naik",
+ "yang",
+ "berapakah",
+ "berikutnya",
+ "diakhiri",
+ "sekaligus",
+ "mengibaratkannya",
+ "sayalah",
+ "guna",
+ "bukanlah",
+ "terjadilah",
+ "belumlah",
+ "bagaimanapun",
+ "sesampai",
+ "semisal",
+ "bukannya",
+ "selanjutnya",
+ "kedua",
+ "kenapa",
+ "seseorang",
+ "menyampaikan",
+ "kembali",
+ "menjadi",
+ "benarlah",
+ "itukah",
+ "pastilah",
+ "sekadar",
+ "memberi",
+ "katanya",
+ "bermula",
+ "tinggi",
+ "meski",
+ "sebegini",
+ "digunakan",
+ "tersampaikan",
+ "bawah",
+ "seenaknya",
+ "bahwa",
+ "diberikannya",
+ "sejak",
+ "menginginkan",
+ "setempat",
+ "bagai",
+ "mempergunakan",
+ "mau",
+ "disampaikan",
+ "banyak",
+ "dipunyai",
+ "memihak",
+ "seperti",
+ "mengucapkannya",
+ "sekalipun",
+ "sajalah",
+ "ialah",
+ "dibuat",
+ "dimulainya",
+ "dia",
+ "ditunjuk",
+ "pula",
+ "ada",
+ "inikah",
+ "kepadanya",
+ "bagaikan",
+ "agak",
+ "menyiapkan",
+ "kamu",
+ "seringnya",
+ "sekiranya",
+ "bertanya",
+ "suatu",
+ "aku",
+ "rata",
+ "ibaratkan",
+ "meyakinkan",
+ "tentulah",
+ "wong",
+ "sepanjang",
+ "tak",
+ "tidakkah",
+ "memerlukan",
+ "menunjuki",
+ "sendirinya",
+ "padanya",
+ "keseluruhannya",
+ "keadaan",
+ "sangat",
+ "melakukan",
+ "masing-masing",
+ "sebegitu",
+ "tentunya",
+ "secara",
+ "bung",
+ "begitu",
+ "dua",
+ "disini",
+ "sebetulnya",
+ "lainnya",
+ "masih",
+ "tempat",
+ "mengatakannya",
+ "jelaslah",
+ "semacam",
+ "memungkinkan",
+ "tepat",
+ "terhadapnya",
+ "datang",
+ "sinilah",
+ "turut",
+ "kapankah",
+ "sejauh",
+ "tandasnya",
+ "kelihatannya",
+ "semuanya",
+ "jelasnya",
+ "ditunjukkan",
+ "jumlah",
+ "awal",
+ "lalu",
+ "asalkan",
+ "berujar",
+ "karenanya",
+ "memperlihatkan",
+ "secukupnya",
+ "mendatangi",
+ "sehingga",
+ "apakah",
+ "hingga",
+ "bilakah",
+ "atau",
+ "tertuju",
+ "menyatakan",
+ "tetap",
+ "ingin",
+ "kalaulah",
+ "perlukah",
+ "tiba",
+ "ujar",
+ "mempunyai",
+ "asal",
+ "dong",
+ "inginkan",
+ "menunjuknya",
+ "akankah",
+ "soal",
+ "diperbuat",
+ "atas",
+ "macam",
+ "menanyakan",
+ "hendak",
+ "buat",
+ "tambahnya",
+ "diminta",
+ "semata",
+ "jadinya",
+ "ibaratnya",
+ "akhirnya",
+ "seterusnya",
+ "setengah",
+ "menjawab",
+ "ternyata",
+ "tandas",
+ "ditanyakan",
+ "hendaklah",
+ "nyaris",
+ "jumlahnya",
+ "cukuplah",
+ "kemudian",
+ "ketika",
+ "begitupun",
+ "ditunjuknya",
+ "tanpa",
+ "kesampaian",
+ "mempersiapkan",
+ "memisalkan",
+ "diketahui",
+ "umumnya",
+ "inginkah",
+ "seorang",
+ "sesudah",
+ "cuma",
+ "perlu",
+ "berkenaan",
+ "berlalu",
+ "seberapa",
+ "jawab",
+ "waktunya",
+ "pihak",
+ "sebagaimana",
+ "maka",
+ "ataupun",
+ "memperkirakan",
+ "tetapi",
+ "bakalan",
+ "kini",
+ "sedang",
+ "siapapun",
+ "dipersoalkan",
+ "yakin",
+ "apalagi",
+ "sebagainya",
+ "terus",
+ "pun",
+ "bagaimanakah",
+ "mana",
+ "dekat",
+ "nyatanya",
+ "terlalu",
+ "tambah",
+ "apa",
+ "jadilah",
+ "sedikit",
+ "tidak",
+ "masing",
+ "keterlaluan",
+ "serta",
+ "hendaknya",
+ "serupa",
+ "semampu",
+ "ibarat",
+ "sampai-sampai",
+ "diakhirinya",
+ "berkeinginan",
+ "dini",
+ "sekitar",
+ "disinilah",
+ "menunjukkan",
+ "sendirian",
+ "seingat",
+ "ibu",
+ "ditujukan",
+ "apaan",
+ "tuturnya",
+ "sekurangnya",
+ "masihkah",
+ "melainkan",
+ "ditunjuki",
+ "terdapat",
+ "mirip",
+ "sangatlah",
+ "sebuah",
+ "memintakan",
+ "disebut",
+ "dikira",
+ "begitulah",
+ "diucapkannya",
+ "mempertanyakan",
+ "sudahlah",
+ "balik",
+ "bersama",
+ "kapan",
+ "seluruh",
+ "dapat",
+ "didapat",
+ "tiap",
+ "kira-kira",
+ "dimintai",
+ "malahan",
+ "karena",
+ "kala",
+ "mengucapkan",
+ "menghendaki",
+ "mengatakan",
+ "memulai",
+ "menuju",
+ "semakin",
+ "bersama-sama",
+ "ucap",
+ "menunjuk",
+ "mengungkapkan",
+ "terutama",
+ "keseluruhan",
+ "diperlihatkan",
+ "toh",
+ "memastikan",
+ "beginian",
+ "bisakah",
+ "apabila",
+ "satu",
+ "lanjutnya",
+ "terjadinya",
+ "katakanlah",
+ "ditunjukkannya",
+ "mulanya",
+ "kita",
+ "bermacam",
+ "selama-lamanya",
+ "itu",
+ "melihat",
+ "beginikah",
+ "keluar",
+ "pernah",
+ "sesuatunya",
+ "berjumlah",
+ "anda",
+ "sekurang-kurangnya",
+ "nantinya",
+ "belakangan",
+ "antar",
+ "kira",
+ "sering",
+ "apatah",
+ "entahlah",
+ "bolehkah",
+ "sedemikian",
+ "sesegera",
+ "oleh",
+ "terlebih",
+ "dikarenakan",
+ "cara",
+ "beri",
+ "semata-mata",
+ "lama",
+ "betul",
+ "dibuatnya",
+ "tanyanya",
+ "nah",
+ "misal",
+ "persoalan",
+ "mulailah",
+ "pertanyaan",
+ "kepada",
+ "terhadap",
+ "menanti",
+ "dijawab",
+ "walaupun",
+ "berkehendak",
+ "khususnya",
+ "sebut",
+ "biasa",
+ "kalaupun",
+ "seharusnya",
+ "berikan",
+ "sebabnya",
+ "misalkan",
+ "dimulailah",
+ "berbagai",
+ "masa",
+ "hanya",
+ "mengetahui",
+ "sekali",
+ "ini",
+ "dirinya",
+ "diingat",
+ "menggunakan",
+ "terlihat",
+ "sesudahnya",
+ "dilalui",
+ "sebaik",
+ "akhir",
+ "mampu",
+ "jika",
+ "antara",
+ "justru",
+ "bersiap-siap",
+ "diibaratkannya",
+ "lagi",
+ "setidak-tidaknya",
+ "manakala",
+ "luar",
+ "kebetulan",
+ "dikatakannya",
+ "setiba",
+ "besar",
+ "merasa",
+ "inilah",
+ "sesama",
+ "sejenak",
+ "jauh",
+ "hari",
+ "semasa",
+ "tertentu",
+ "yaitu",
+ "saatnya",
+ "bisa",
+ "sebagai",
+ "kemungkinan",
+ "dimaksudnya",
+ "cukupkah",
+ "sebanyak",
+ "kecil",
+ "sewaktu",
+ "kalau",
+ "walau",
+ "baru",
+ "berawal",
+ "menantikan",
+ "umum",
+ "sekitarnya",
+ "jelas",
+ "semua",
+ "itulah",
+ "sudahkah",
+ "semampunya",
+ "selalu",
+ "jangan",
+ "sama-sama",
+ "semaunya",
+ "sini",
+ "sedikitnya",
+ "sampai",
+ "tahu",
+ "diberikan",
+ "diingatkan",
+ "tentu",
+ "tentang",
+ "segala",
+ "segera",
+ "lagian",
+ "seluruhnya",
+ "sebenarnya",
+ "jelaskan",
+ "tapi",
+ "menaiki",
+ "tersebutlah",
+ "ditambahkan",
+ "sementara",
+ "makanya",
+ "kalian",
+ "enggak",
+ "kurang",
+ "diinginkan",
+ "berupa",
+ "seperlunya",
+ "dijelaskannya",
+ "punya",
+ "dimaksudkannya",
+ "memang",
+ "setelah",
+ "menyeluruh",
+ "berlainan",
+ "mengingatkan",
+ "pukul",
+ "saja",
+ "dilakukan",
+ "berapalah",
+ "paling",
+ "meyakini",
+ "menuturkan",
+ "makin",
+ "sekali-kali",
+ "olehnya",
+ "tahun",
+ "ditanya",
+ "daripada",
+ "menyangkut",
+ "ungkapnya",
+ "bagian",
+ "lamanya",
+ "terbanyak",
+ "caranya",
+ "malah",
+ "kamulah",
+ "siap",
+ "ingat-ingat",
+ "berturut",
+ "saling",
+ "manalagi",
+ "melihatnya",
+ "tampaknya",
+ "kapanpun",
+ "dimungkinkan",
+ "dituturkannya",
+ "sesuatu",
+ "diperlukan",
+ "diri",
+ "sekalian",
+ "mereka",
+ "kelihatan",
+ "bahkan",
+ "baik",
+ "namun",
+ "terjadi",
+ "dulu",
+ "mempersoalkan",
+ "keduanya",
+ "jawaban",
+ "empat",
+ "kok",
+ "didatangkan",
+ "bukan",
+ "berlangsung",
+ "begini",
+ "saya",
+ "berakhir",
+ "dipergunakan",
+ "wahai",
+ "belum",
+ "termasuk",
+ "lebih",
+ "pantas",
+ "mengira",
+ "terakhir",
+ "bakal",
+ "merupakan",
+ "panjang",
+ "ucapnya",
+ "ke",
+ "benar",
+ "bapak",
+ "sepihak",
+ "pasti",
+ "adapun",
+ "amat",
+ "mengapa",
+ "selamanya",
+ "bolehlah",
+ "diperbuatnya",
+ "kami",
+ "bagi",
+ "maupun",
+ "beberapa",
+ "mendatangkan",
+ "sepantasnyalah",
+ "tegas",
+ "kiranya",
+ "siapa",
+ "diantara",
+ "menandaskan",
+ "dalam",
+ "disebutkan",
+ "berapapun",
+ "ditegaskan",
+ "demikian",
+ "meminta",
+ "harus",
+ "telah",
+ "mula",
+ "menanyai",
+ "selain",
+ "sama",
+ "sebutlah",
+ "bukankah",
+ "seolah",
+ "sela",
+ "membuat",
+ "selaku",
+ "seketika",
+ "kata",
+ "hanyalah",
+ "biasanya",
+ "diibaratkan",
+ "diketahuinya",
+ "sebutnya",
+ "ditanyai",
+ "sebelum",
+ "ditandaskan",
+ "menjelaskan",
+ "benarkah",
+ "dimisalkan",
+ "pentingnya",
+ "sebab",
+ "menegaskan",
+ "masalah",
+ "mengenai",
+ "tanyakan",
+ "tampak",
+ "disebutkannya",
+ "misalnya",
+ "juga",
+ "entah",
+ "tanya",
+ "lewat",
+ "semula",
+ "siapakah",
+ "sebaliknya",
+ "ungkap",
+ "pertama",
+ "dikerjakan",
+ "berdatangan",
+ "kemungkinannya",
+ "enggaknya",
+ "mulai",
+ "padahal",
+ "sambil",
+ "semisalnya",
+ "katakan",
+ "berikut",
+ "mendapatkan",
+ "supaya",
+ "usah",
+ "agaknya",
+ "beginilah",
+ "tiga",
+ "tiba-tiba",
+ "waduh",
+ "usai",
+ "dimaksud",
+ "kitalah",
+ "demikianlah",
+ "kelima",
+ "tadi",
+ "seolah-olah",
+ "nanti",
+ "selama",
+ "hampir",
+ "bulan",
+ "dijelaskan",
+ "akan",
+ "seusai",
+ "sekadarnya",
+ "andalah",
+ "merekalah",
+ "pertama-tama",
+ "pertanyakan",
+ "rasa",
+ "bila",
+ "berkali-kali",
+ "diperkirakan",
+ "diucapkan",
+ "terasa",
+ "berapa",
+ "jangankan",
+ "kelamaan",
+ "agar",
+ "ataukah",
+ "pak",
+ "diberi",
+ "sampaikan",
+ "setinggi",
+ "menyebutkan",
+ "menurut",
+ "dengan",
+ "menanya",
+ "lima",
+ "sepertinya",
+ "dituturkan",
+ "diperlukannya",
+ "berakhirlah",
+ "dimulai",
+ "sempat",
+ "sejumlah",
+ "setidaknya",
+ "dialah",
+ "adalah",
+ "berlebihan",
+ "lah",
+ "sudah",
+ "berturut-turut",
+ "ujarnya",
+ "mengakhiri",
+ "awalnya",
+ "setiap",
+ "sendiri",
+ "sana",
+ "diungkapkan",
+ "sebaik-baiknya",
+ "memperbuat",
+ "berkata",
+ "meskipun",
+ "tutur",
+ "diantaranya",
+ "yakni",
+ "akulah",
+ "mungkin",
+ "demi",
+ "bersiap",
+ "dimaksudkan",
+ "menambahkan",
+ "berakhirnya",
+ "mengerjakan",
+ "tidaklah",
+ "setibanya",
+ "semasih",
+ "kasus",
+ "janganlah",
+ "jadi",
+ "jikalau",
+ "sebisanya",
+ "dilihat",
+ "minta",
+ "sebaiknya",
+ "soalnya",
+ "mendatang",
+ "bertanya-tanya",
+ "dipertanyakan",
+ "ikut",
+ "terkira",
+ "terdiri",
+ "dipastikan",
+ "antaranya",
+ "sesekali",
+ "rupanya",
+ "hal",
+ "berada",
+ "sesaat",
+ "tersebut",
+ "sebesar",
+ "adanya",
+ "mampukah",
+ "ingat",
+ "sebagian",
+ "bekerja",
+ "untuk",
+ "waktu",
+ "pihaknya",
+ "jawabnya",
+ "bahwasanya",
+ "wah",
+ "mengingat",
+ "segalanya",
+ "menanti-nanti",
+ "pada",
+ "kinilah",
+ "memberikan",
+ "dahulu",
+ "mendapat",
+ "begitukah",
+ "perlunya",
+ "cukup",
+ "belakang",
+ "teringat-ingat",
+ "artinya",
+ "bertutur",
+ "sepantasnya",
+ "lain",
+ "tengah",
+ "bagaimana",
+ "akhiri"
+ ],
+ "en": [
+ "until",
+ "his",
+ "those",
+ "who",
+ "any",
+ "you've",
+ "it's",
+ "that",
+ "aren",
+ "after",
+ "down",
+ "couldn",
+ "itself",
+ "ours",
+ "about",
+ "whom",
+ "won't",
+ "isn't",
+ "what",
+ "this",
+ "wouldn't",
+ "than",
+ "its",
+ "don't",
+ "too",
+ "such",
+ "wasn",
+ "by",
+ "doing",
+ "you'd",
+ "you",
+ "some",
+ "where",
+ "you'll",
+ "against",
+ "she",
+ "been",
+ "into",
+ "but",
+ "that'll",
+ "they",
+ "more",
+ "wasn't",
+ "and",
+ "here",
+ "to",
+ "during",
+ "him",
+ "my",
+ "our",
+ "she's",
+ "if",
+ "yourself",
+ "hers",
+ "there",
+ "out",
+ "yours",
+ "aren't",
+ "should've",
+ "because",
+ "own",
+ "couldn't",
+ "these",
+ "should",
+ "ourselves",
+ "few",
+ "them",
+ "haven",
+ "between",
+ "both",
+ "shouldn't",
+ "just",
+ "wouldn",
+ "when",
+ "didn",
+ "off",
+ "hasn",
+ "how",
+ "did",
+ "once",
+ "above",
+ "yourselves",
+ "you're",
+ "other",
+ "below",
+ "shan't",
+ "with",
+ "don",
+ "haven't",
+ "through",
+ "again",
+ "each",
+ "ain",
+ "then",
+ "myself",
+ "the",
+ "won",
+ "we",
+ "can",
+ "now",
+ "their",
+ "herself",
+ "only",
+ "mustn",
+ "very",
+ "shan",
+ "themselves",
+ "were",
+ "didn't",
+ "hasn't",
+ "nor",
+ "not",
+ "does",
+ "weren't",
+ "mightn't",
+ "having",
+ "doesn't",
+ "needn't",
+ "further",
+ "while",
+ "before",
+ "hadn't",
+ "mustn't",
+ "why",
+ "theirs",
+ "your",
+ "himself",
+ "which",
+ "being",
+ "from",
+ "up",
+ "it",
+ "same",
+ ],
+ "ar": [
+ "هنالك",
+ "لم",
+ "أنت",
+ "بكما",
+ "هيا",
+ "اللتان",
+ "اللواتي",
+ "بمن",
+ "وهو",
+ "Ùلا",
+ "عليه",
+ "بهما",
+ "لعل",
+ "حيث",
+ "اللتين",
+ "هنا",
+ "كأي",
+ "لاسيما",
+ "كيت",
+ "آها",
+ "سوÙ",
+ "لستما",
+ "أنا",
+ "الذي",
+ "لستم",
+ "إلا",
+ "ذاك",
+ "هيت",
+ "إيه",
+ "كلاهما",
+ "Ùإن",
+ "والذي",
+ "هو",
+ "الذين",
+ "يا",
+ "Ùإذا",
+ "مما",
+ "هؤلاء",
+ "إذ",
+ "ولكن",
+ "أو",
+ "هذان",
+ "عند",
+ "وما",
+ "التي",
+ "أينما",
+ "بها",
+ "ماذا",
+ "لسن",
+ "إنه",
+ "بس",
+ "ثمة",
+ "بنا",
+ "كليكما",
+ "Ùيه",
+ "لدى",
+ "لستن",
+ "هناك",
+ "ذان",
+ "تلكم",
+ "بي",
+ "منه",
+ "لهم",
+ "ليسا",
+ "إليك",
+ "تين",
+ "وإن",
+ "اللذين",
+ "إذا",
+ "لكم",
+ "حين",
+ "وإذا",
+ "عدا",
+ "تلكما",
+ "قد",
+ "أكثر",
+ "به",
+ "دون",
+ "إما",
+ "ها",
+ "عل",
+ "هاتين",
+ "ذلكن",
+ "على",
+ "كلتا",
+ "كليهما",
+ "لهن",
+ "هذين",
+ "بك",
+ "ذينك",
+ "كأنما",
+ "كيÙما",
+ "اللذان",
+ "لكن",
+ "مذ",
+ "لما",
+ "ولا",
+ "آي",
+ "هاته",
+ "أولاء",
+ "نحن",
+ "هي",
+ "عليك",
+ "تينك",
+ "هذه",
+ "كي",
+ "ذانك",
+ "أي",
+ "هاتي",
+ "أيها",
+ "ذلكم",
+ "هن",
+ "آه",
+ "Ùيم",
+ "ذين",
+ "نحو",
+ "إذما",
+ "إليكن",
+ "إلى",
+ "ليسوا",
+ "أن",
+ "هاهنا",
+ "كذلك",
+ "ذلكما",
+ "عما",
+ "هكذا",
+ "لسنا",
+ "أوه",
+ "لئن",
+ "لكيلا",
+ "Ùيها",
+ "سوى",
+ "ذواتي",
+ "لا",
+ "أنتن",
+ "إي",
+ "ذي",
+ "هل",
+ "اللتيا",
+ "إن",
+ "ÙÙŠ",
+ "حاشا",
+ "كل",
+ "ذه",
+ "حبذا",
+ "خلا",
+ "بما",
+ "كأين",
+ "ذواتا",
+ "والذين",
+ "منذ",
+ "لوما",
+ "هذا",
+ "بكن",
+ "هاتان",
+ "لك",
+ "أنتما",
+ "أقل",
+ "اللائي",
+ "أنتم",
+ "كذا",
+ "لن",
+ "نعم",
+ "بهن",
+ "له",
+ "أولئك",
+ "Ø£Ù",
+ "ذوا",
+ "ته",
+ "لست",
+ "بل",
+ "كما",
+ "لكي",
+ "مهما",
+ "بلى",
+ "حيثما",
+ "عن",
+ "ومن",
+ "اللاتي",
+ "هما",
+ "ذا",
+ "بين",
+ "شتان",
+ "لي",
+ "ممن",
+ "تي",
+ "بهم",
+ "حتى",
+ "كلما",
+ "ليستا",
+ "ذات",
+ "ليت",
+ "إنما",
+ "هيهات",
+ "Ùيما",
+ "ريث",
+ "بعض",
+ "لنا",
+ "ما",
+ "ليست",
+ "مع",
+ "ذو",
+ "لها",
+ "إذن",
+ "عسى",
+ "أم",
+ "لولا",
+ "هذي",
+ "إليكما",
+ "هم",
+ "ذلك",
+ "بيد",
+ "Ùمن",
+ "بخ",
+ "ولو",
+ "مه",
+ "تلك",
+ "إليكم",
+ "من",
+ "بعد",
+ "متى",
+ "ليس",
+ "لهما",
+ "منها",
+ "هلا",
+ "بماذا",
+ "هاك",
+ "لكنما",
+ "لكما",
+ "إنا",
+ "غير",
+ "كيÙ",
+ "ألا",
+ "ثم",
+ "كأن",
+ "كلا",
+ "لو",
+ "وإذ"
+ ],
+ "no": [
+ "noka",
+ "ingi",
+ "kva",
+ "deira",
+ "uten",
+ "vært",
+ "noko",
+ "noe",
+ "noen",
+ "mellom",
+ "somt",
+ "varte",
+ "nokon",
+ "hossen",
+ "kvifor",
+ "deg",
+ "medan",
+ "siden",
+ "Ã¥",
+ "hvem",
+ "sånn",
+ "deires",
+ "korleis",
+ "vere",
+ "blei",
+ "kvi",
+ "inkje",
+ "mykje",
+ "sjøl",
+ "korso",
+ "hoss",
+ "fordi",
+ "blitt",
+ "hvilke",
+ "sidan",
+ "verte",
+ "hvilken",
+ "inni",
+ "etter",
+ "somme",
+ "hvordan",
+ "eit",
+ "vors",
+ "deim",
+ "enn",
+ "kven",
+ "nå",
+ "dykkar",
+ "hva",
+ "inn",
+ "dykk",
+ "ble",
+ "kvar",
+ "både",
+ "eitt",
+ "bare",
+ "hvorfor",
+ "sitt",
+ "vore",
+ "hver",
+ "opp",
+ "kvarhelst",
+ "samme",
+ "hjå",
+ "hennar",
+ "før",
+ "hadde",
+ "dere",
+ "ved",
+ "båe",
+ "eg",
+ "seg",
+ "vort",
+ "nokre",
+ "kom",
+ "begge",
+ "nokor",
+ "slik",
+ "elles"
+ ],
+ "kz": [
+ "өзім",
+ "дүрÑ",
+ "емеÑ",
+ "мен",
+ "қайÑыбір",
+ "күрт",
+ "Ñарт-Ñұрт",
+ "ғана",
+ "әрине",
+ "Ñіздердің",
+ "Ñол",
+ "әншейін",
+ "барша",
+ "бізге",
+ "әлдене",
+ "Ñен",
+ "дүңк",
+ "ешқашан",
+ "алатау",
+ "әлдеқашан",
+ "ай",
+ "паһ-паһ",
+ "Ñіз",
+ "пай",
+ "әй",
+ "біздердің",
+ "алайда",
+ "өзінің",
+ "арбаң-арбаң",
+ "арÑ",
+ "Ñізге",
+ "өзіне",
+ "ірк",
+ "кірт",
+ "Ñарт",
+ "моһ",
+ "ÑÑ‹Ò£Ò›",
+ "болп",
+ "алдақашан",
+ "ешқайÑÑ‹",
+ "батыр-бұтыр",
+ "арÑалаң-арÑалаң",
+ "бәрі",
+ "былп",
+ "кә",
+ "құрау-құрау",
+ "біз",
+ "өзімнің",
+ "мыңқ",
+ "ешқандай",
+ "ешкім",
+ "менің",
+ "Ñіздер",
+ "шіңк",
+ "шаңқ-шұңқ",
+ "тарбаң-тарбаң",
+ "өй",
+ "аһа",
+ "жалт-жалт",
+ "әйтпеÑе",
+ "беу",
+ "оÑылай",
+ "Ñізбен",
+ "олардың",
+ "тағы",
+ "әлденеше",
+ "пай-пай",
+ "уау",
+ "Ñенің",
+ "арнайы",
+ "қолп",
+ "өз",
+ "бірақ",
+ "жаракімалла",
+ "бірдеме",
+ "ана",
+ "Ó™",
+ "кейбір",
+ "ÑÑ…",
+ "Ñондай",
+ "біздер",
+ "бүгжең-бүгжең",
+ "өзге",
+ "ойпырмай",
+ "өзің",
+ "Ñ‚Ñ‹Ñ€Ñ",
+ "онымен",
+ "жалп",
+ "ештеме",
+ "менімен",
+ "шаңқ-шаңқ",
+ "әлдеқалай",
+ "мына",
+ "әрне",
+ "Ñ‚Ñ‹Ò£Ò›",
+ "әрқайÑÑ‹",
+ "барлық",
+ "пфша",
+ "Ñ‹Ñ€Ñ",
+ "Ñенімен",
+ "бізбен",
+ "әркім",
+ "құрау",
+ "жоқ",
+ "жалт-жұлт",
+ "уай",
+ "Ñона",
+ "қыңқ",
+ "қорÑ",
+ "түге",
+ "күңк",
+ "менде",
+ "анау",
+ "қаңғыр-күңгір",
+ "кәне",
+ "еш",
+ "бұндай",
+ "пішту",
+ "Ñіздерге",
+ "қалт-қалт",
+ "шәйт",
+ "оһо",
+ "олармен",
+ "Ñорап",
+ "тәк",
+ "кейбіреу",
+ "қап",
+ "борт",
+ "мынау",
+ "әттегенай",
+ "Ñіздермен",
+ "дегенмен",
+ "бүйт",
+ "Ñенде",
+ "Ñй",
+ "Ñонау",
+ "түгел",
+ "біздерден",
+ "қайқаң-құйқаң",
+ "Ñенен\tонан",
+ "барқ",
+ "гүрÑ",
+ "арÑ-ұрÑ",
+ "бар",
+ "ох",
+ "шек",
+ "алақай",
+ "пырÑ",
+ "шіркін",
+ "қош-қош",
+ "Ñаңқ",
+ "Ñонымен",
+ "оÑынау",
+ "тек",
+ "біздерге",
+ "морт",
+ "әрқалай",
+ "маңқ",
+ "Ñіздерден",
+ "олар",
+ "Ñебебі",
+ "желп",
+ "қалт-құлт",
+ "ол",
+ "мұндай",
+ "әлдеқайдан",
+ "әттең",
+ "мышы",
+ "Ñолай",
+ "Ñалаң-Ñұлаң",
+ "қана",
+ "біздермен",
+ "кәнеки",
+ "уа",
+ "әрбір",
+ "құр",
+ "мәÑÑаған",
+ "ал",
+ "Ñ‹Ò£Ò›",
+ "е",
+ "оÑÑ‹",
+ "Ñізден",
+ "ура",
+ "әукім",
+ "пішә",
+ "шырт",
+ "митың-митың",
+ "біреу",
+ "әлдекім",
+ "шаңқ",
+ "әттеген-ай",
+ "далаң-далаң",
+ "өзіме",
+ "қаңқ-қаңқ",
+ "кәһ",
+ "өзі",
+ "әйткенмен",
+ "онда",
+ "өйткені",
+ "ербелең-ербелең",
+ "тарÑ-тұрÑ",
+ "Ñпырмай",
+ "па",
+ "күллі",
+ "қаңқ-құңқ",
+ "кәні",
+ "тарÑ",
+ "бүкіл",
+ "айтпақшы",
+ "ыржың-тыржың",
+ "ах",
+ "бұл",
+ "әйда",
+ "ие",
+ "бізден",
+ "Ñенен",
+ "қызараң-қызараң",
+ "үйт",
+ "ешбір",
+ "аÑтапыралла",
+ "тағыда",
+ "дәнеңе",
+ "таңқ",
+ "Ñ‹Ñ€Ò›",
+ "маÑқарай"
+ ],
+ "hu": [
+ "szinte",
+ "lehetett",
+ "talán",
+ "csak",
+ "nagy",
+ "ill",
+ "illetve",
+ "kívül",
+ "cikkeket",
+ "utána",
+ "nagyobb",
+ "sokkal",
+ "vagy",
+ "melyek",
+ "aztán",
+ "ennek",
+ "õ",
+ "azonban",
+ "azért",
+ "míg",
+ "éppen",
+ "jobban",
+ "mellett",
+ "benne",
+ "után",
+ "ehhez",
+ "mindenki",
+ "mikor",
+ "azon",
+ "ison",
+ "alatt",
+ "pedig",
+ "hanem",
+ "neki",
+ "elsõ",
+ "igen",
+ "szemben",
+ "lenne",
+ "számára",
+ "lenni",
+ "vagyis",
+ "egyéb",
+ "ezek",
+ "még",
+ "szerint",
+ "mindent",
+ "milyen",
+ "másik",
+ "maga",
+ "ott",
+ "új",
+ "mindig",
+ "nekem",
+ "volna",
+ "jó",
+ "vissza",
+ "ismét",
+ "rá",
+ "azt",
+ "egy",
+ "emilyen",
+ "vannak",
+ "én",
+ "voltunk",
+ "ellen",
+ "belül",
+ "magát",
+ "valami",
+ "így",
+ "egyik",
+ "mivel",
+ "viszont",
+ "nagyon",
+ "ezen",
+ "valaki",
+ "kellett",
+ "és",
+ "ezzel",
+ "persze",
+ "sok",
+ "õk",
+ "abban",
+ "miért",
+ "arról",
+ "át",
+ "újra",
+ "össze",
+ "volt",
+ "azok",
+ "ilyen",
+ "sokat",
+ "néha",
+ "ekkor",
+ "ill.",
+ "cikk",
+ "erre",
+ "egyetlen",
+ "vagyok",
+ "voltam",
+ "amelyeket",
+ "annak",
+ "ezt",
+ "semmi",
+ "ez",
+ "ahhoz",
+ "minden",
+ "lett",
+ "ilyenkor",
+ "aki",
+ "egyre",
+ "ahogy",
+ "amolyan",
+ "keressünk",
+ "egész",
+ "mintha",
+ "néhány",
+ "nincs",
+ "õket",
+ "teljes",
+ "által",
+ "hogy",
+ "mely",
+ "között",
+ "egyes",
+ "legyen",
+ "amíg",
+ "amelyek",
+ "továbbá",
+ "akkor",
+ "valamint",
+ "voltak",
+ "amelynek",
+ "bár",
+ "hogyan",
+ "azután",
+ "tehát",
+ "ezért",
+ "utolsó",
+ "saját",
+ "lesz",
+ "jól",
+ "általában",
+ "hiszen",
+ "vele",
+ "tovább",
+ "ami",
+ "elõször",
+ "újabb",
+ "nélkül",
+ "cikkek",
+ "elõ",
+ "elõtt",
+ "amelyekben",
+ "elég",
+ "kell",
+ "keresztül",
+ "amely",
+ "már",
+ "felé",
+ "ebben",
+ "több",
+ "mert",
+ "való",
+ "amelyet",
+ "azzal",
+ "úgy",
+ "legalább",
+ "eddig",
+ "amikor",
+ "arra",
+ "ahol",
+ "olyan",
+ "közül",
+ "ugyanis",
+ "lehet",
+ "itt",
+ "amit",
+ "mint",
+ "akik"
+ ],
+ "tr": [
+ "niye",
+ "mu",
+ "belki",
+ "nasıl",
+ "niçin",
+ "mü",
+ "ile",
+ "nerede",
+ "aslında",
+ "nerde",
+ "bazı",
+ "gibi",
+ "ÅŸey",
+ "yani",
+ "mı",
+ "kez",
+ "hep",
+ "defa",
+ "ama",
+ "neden",
+ "ise",
+ "diye",
+ "veya",
+ "için",
+ "hepsi",
+ "tüm",
+ "çok",
+ "birkaç",
+ "sanki",
+ "acaba",
+ "ÅŸu",
+ "çünkü",
+ "hiç",
+ "eÄŸer",
+ "nereye"
+ ],
+ "it": [
+ "avrai",
+ "erano",
+ "suoi",
+ "fossero",
+ "siete",
+ "fecero",
+ "sarà",
+ "fareste",
+ "faceva",
+ "nello",
+ "aveva",
+ "quelle",
+ "starei",
+ "stando",
+ "faresti",
+ "tutti",
+ "eri",
+ "avevamo",
+ "sarebbe",
+ "miei",
+ "fanno",
+ "avessimo",
+ "sul",
+ "furono",
+ "ebbero",
+ "stesti",
+ "quanti",
+ "fu",
+ "nostra",
+ "come",
+ "sarei",
+ "vostra",
+ "facevamo",
+ "queste",
+ "faremo",
+ "nelle",
+ "stiamo",
+ "nei",
+ "che",
+ "non",
+ "faremmo",
+ "tutto",
+ "stavamo",
+ "avendo",
+ "avessi",
+ "starete",
+ "negli",
+ "stava",
+ "avrei",
+ "facessi",
+ "aveste",
+ "facemmo",
+ "avevate",
+ "ebbi",
+ "avemmo",
+ "quanto",
+ "è",
+ "nella",
+ "anche",
+ "stetti",
+ "negl",
+ "sono",
+ "quella",
+ "avevo",
+ "faranno",
+ "farò",
+ "ero",
+ "agli",
+ "facesse",
+ "facciamo",
+ "stiate",
+ "stai",
+ "stesse",
+ "nell",
+ "stavi",
+ "farete",
+ "sue",
+ "avrà",
+ "staremo",
+ "facesti",
+ "col",
+ "stessero",
+ "sarai",
+ "più",
+ "farebbero",
+ "stavate",
+ "feci",
+ "starebbe",
+ "mia",
+ "avevi",
+ "questa",
+ "dal",
+ "avrebbe",
+ "hai",
+ "vostre",
+ "sei",
+ "abbiano",
+ "starai",
+ "facessimo",
+ "mio",
+ "faccia",
+ "nostre",
+ "loro",
+ "stavo",
+ "faceste",
+ "stette",
+ "dall",
+ "sullo",
+ "siate",
+ "faccio",
+ "dove",
+ "farebbe",
+ "siano",
+ "saresti",
+ "eravate",
+ "avesti",
+ "avute",
+ "avuti",
+ "questo",
+ "facciate",
+ "stessi",
+ "dagl",
+ "sareste",
+ "abbia",
+ "dalla",
+ "tue",
+ "farà",
+ "dello",
+ "stanno",
+ "foste",
+ "fosti",
+ "abbiamo",
+ "facevi",
+ "essendo",
+ "avrebbero",
+ "sulle",
+ "avrò",
+ "sui",
+ "tra",
+ "degli",
+ "avranno",
+ "saremmo",
+ "avrete",
+ "staremmo",
+ "nel",
+ "avuto",
+ "starebbero",
+ "fai",
+ "sugli",
+ "saremo",
+ "siamo",
+ "agl",
+ "facciano",
+ "quello",
+ "delle",
+ "farei",
+ "stessimo",
+ "stettero",
+ "sta",
+ "quanta",
+ "facessero",
+ "lei",
+ "vostri",
+ "perché",
+ "sto",
+ "avete",
+ "sulla",
+ "avremmo",
+ "avessero",
+ "hanno",
+ "sarete",
+ "sugl",
+ "stia",
+ "facevano",
+ "abbiate",
+ "dell",
+ "sarò",
+ "facendo",
+ "stareste",
+ "staranno",
+ "saranno",
+ "dallo",
+ "eravamo",
+ "sull",
+ "della",
+ "ebbe",
+ "vostro",
+ "degl",
+ "stavano",
+ "avremo",
+ "contro",
+ "nostro",
+ "quelli",
+ "staresti",
+ "avresti",
+ "questi",
+ "dagli",
+ "stemmo",
+ "coi",
+ "stiano",
+ "quante",
+ "steste",
+ "avuta",
+ "suo",
+ "chi",
+ "fummo",
+ "gli",
+ "dov",
+ "fece",
+ "tuoi",
+ "fossi",
+ "dai",
+ "starà",
+ "dalle",
+ "quale",
+ "facevo",
+ "starò",
+ "avesse",
+ "farai",
+ "sarebbero",
+ "avevano",
+ "fossimo",
+ "facevate",
+ "avreste",
+ "allo"
+ ],
+ "de": [
+ "musste",
+ "ob",
+ "ihrem",
+ "jeden",
+ "solchen",
+ "seine",
+ "sich",
+ "sonst",
+ "kann",
+ "seinen",
+ "eine",
+ "indem",
+ "für",
+ "solche",
+ "dann",
+ "unser",
+ "würden",
+ "dieselben",
+ "im",
+ "hab",
+ "derselbe",
+ "können",
+ "würde",
+ "seiner",
+ "nun",
+ "jenen",
+ "zum",
+ "ihrer",
+ "derselben",
+ "nichts",
+ "welchen",
+ "deinem",
+ "über",
+ "gewesen",
+ "jetzt",
+ "ihm",
+ "oder",
+ "eurer",
+ "mir",
+ "anderm",
+ "jene",
+ "dieselbe",
+ "jede",
+ "etwas",
+ "hatte",
+ "welche",
+ "wirst",
+ "ihnen",
+ "eurem",
+ "sondern",
+ "derer",
+ "von",
+ "hatten",
+ "selbst",
+ "weg",
+ "hin",
+ "euch",
+ "keinem",
+ "einen",
+ "einmal",
+ "einigen",
+ "noch",
+ "diesem",
+ "aller",
+ "wenn",
+ "wo",
+ "meinem",
+ "ist",
+ "einige",
+ "eures",
+ "allem",
+ "unseres",
+ "seines",
+ "zwischen",
+ "denn",
+ "dessen",
+ "wollen",
+ "wir",
+ "weil",
+ "unserem",
+ "sind",
+ "haben",
+ "sie",
+ "ich",
+ "ihres",
+ "sollte",
+ "manche",
+ "hat",
+ "aber",
+ "habe",
+ "anderen",
+ "manchen",
+ "ihn",
+ "sein",
+ "nicht",
+ "hinter",
+ "dort",
+ "wird",
+ "auf",
+ "machen",
+ "anderem",
+ "demselben",
+ "uns",
+ "bei",
+ "keines",
+ "damit",
+ "bin",
+ "mancher",
+ "ihr",
+ "anderes",
+ "eines",
+ "deines",
+ "anderer",
+ "dasselbe",
+ "manches",
+ "dazu",
+ "bist",
+ "allen",
+ "muss",
+ "warst",
+ "anders",
+ "manchem",
+ "meinen",
+ "diesen",
+ "zwar",
+ "einiges",
+ "unsere",
+ "einigem",
+ "und",
+ "jedes",
+ "mich",
+ "kein",
+ "also",
+ "diese",
+ "ins",
+ "meine",
+ "meiner",
+ "dieser",
+ "unseren",
+ "jenes",
+ "dein",
+ "zur",
+ "einiger",
+ "aus",
+ "zu",
+ "wollte",
+ "bis",
+ "keiner",
+ "welches",
+ "dich",
+ "andern",
+ "ohne",
+ "sehr",
+ "deine",
+ "jener",
+ "welchem",
+ "deinen",
+ "wieder",
+ "jedem",
+ "ander",
+ "euer",
+ "ihren",
+ "eure",
+ "keinen",
+ "durch",
+ "jenem",
+ "anderr",
+ "jeder",
+ "deiner",
+ "welcher",
+ "unter",
+ "mein",
+ "gegen",
+ "dies",
+ "denselben",
+ "keine",
+ "werde",
+ "einig",
+ "nur",
+ "seinem",
+ "daß",
+ "solches",
+ "weiter",
+ "solchem",
+ "einer",
+ "desselben",
+ "werden",
+ "während",
+ "nach",
+ "war",
+ "auch",
+ "einem",
+ "ihre",
+ "meines",
+ "solcher",
+ "viel",
+ "dieses",
+ "soll",
+ "euren",
+ "könnte",
+ "öffnen",
+ "fahren",
+ "fährt",
+ "anhängen",
+ "anhang",
+ "angehangen",
+ "nächste",
+ "nämlich",
+ "spät",
+ "später",
+ "gehören",
+ "hören",
+ "früh",
+ "früher",
+ "möglich",
+ "plötzlich",
+ "schön",
+ "überall",
+ "zurück",
+ "fünf",
+ "außen",
+ "außer",
+ "spaß",
+ "süß",
+ "hallo",
+ "guten",
+ "freundliche",
+ "freundlichen",
+ "besten"
+ ],
+ "sv": [
+ "varför",
+ "deras",
+ "någon",
+ "inom",
+ "ert",
+ "något",
+ "inte",
+ "till",
+ "dess",
+ "vilket",
+ "att",
+ "än",
+ "hon",
+ "vilka",
+ "dessa",
+ "vid",
+ "vars",
+ "denna",
+ "sina",
+ "vara",
+ "är",
+ "vilken",
+ "sådant",
+ "vilkas",
+ "där",
+ "jag",
+ "ju",
+ "mycket",
+ "dina",
+ "utan",
+ "sedan",
+ "detta",
+ "vad",
+ "vem",
+ "och",
+ "samma",
+ "några",
+ "våra",
+ "varje",
+ "vårt",
+ "mellan",
+ "varit",
+ "sitta",
+ "själv",
+ "sådana",
+ "över",
+ "kunde",
+ "för",
+ "mina",
+ "här",
+ "Ã¥t",
+ "från",
+ "icke",
+ "allt",
+ "hade",
+ "blivit",
+ "när",
+ "ej",
+ "hur"
+ ],
+ "ru": [
+ "Ñам",
+ "тогда",
+ "него",
+ "мой",
+ "ну",
+ "он",
+ "мне",
+ "впрочем",
+ "чтобы",
+ "не",
+ "ÑебÑ",
+ "моÑ",
+ "она",
+ "наÑ",
+ "к",
+ "про",
+ "была",
+ "какой",
+ "иногда",
+ "было",
+ "хорошо",
+ "чем",
+ "вÑÑŽ",
+ "они",
+ "здеÑÑŒ",
+ "чуть",
+ "тот",
+ "об",
+ "над",
+ "них",
+ "опÑÑ‚ÑŒ",
+ "надо",
+ "теперь",
+ "был",
+ "уже",
+ "вам",
+ "еÑли",
+ "тебÑ",
+ "чтоб",
+ "почти",
+ "тоже",
+ "ему",
+ "или",
+ "вот",
+ "по",
+ "можно",
+ "но",
+ "перед",
+ "три",
+ "еÑÑ‚ÑŒ",
+ "конечно",
+ "там",
+ "ли",
+ "да",
+ "их",
+ "а",
+ "один",
+ "вÑего",
+ "ж",
+ "никогда",
+ "Ñту",
+ "же",
+ "без",
+ "у",
+ "потом",
+ "его",
+ "от",
+ "тут",
+ "так",
+ "какаÑ",
+ "чего",
+ "Ñтой",
+ "то",
+ "тем",
+ "что",
+ "им",
+ "ÑейчаÑ",
+ "нибудь",
+ "через",
+ "вÑегда",
+ "Ñтого",
+ "того",
+ "ни",
+ "даже",
+ "уж",
+ "Ñ",
+ "Ñ‚Ñ‹",
+ "будто",
+ "зачем",
+ "при",
+ "вдруг",
+ "Ñвою",
+ "разве",
+ "под",
+ "Ñтот",
+ "нельзÑ",
+ "много",
+ "в",
+ "может",
+ "вÑех",
+ "как",
+ "ваÑ",
+ "были",
+ "нет",
+ "ведь",
+ "из",
+ "два",
+ "нее",
+ "быть",
+ "поÑле",
+ "больше",
+ "Ñти",
+ "лучше",
+ "на",
+ "более",
+ "длÑ",
+ "за",
+ "ее",
+ "куда",
+ "том",
+ "вÑе",
+ "между",
+ "до",
+ "Ñебе",
+ "где",
+ "мы",
+ "ней",
+ "другой",
+ "когда",
+ "Ñо",
+ "ним",
+ "будет",
+ "только",
+ "хоть",
+ "раз",
+ "потому",
+ "и",
+ "во",
+ "ничего",
+ "бы",
+ "Ñтом",
+ "ÑовÑем",
+ "еще",
+ "кто",
+ "наконец",
+ "менÑ",
+ "такой",
+ "вы"
+ ],
+ "az": [
+ "olan",
+ "onların ",
+ "bəzi",
+ "ay",
+ "arasında",
+ "harada",
+ "haqqında",
+ "onsuzda",
+ "beÅŸ",
+ "otuz",
+ "əslində",
+ "faiz",
+ "heç",
+ "yaxşı",
+ "çox",
+ "yəni",
+ "bilər",
+ "kimi",
+ "təəssüf",
+ "xan",
+ "yoxdur",
+ "kimÉ™",
+ "sən",
+ "olmuÅŸdur",
+ "həmin",
+ "nəhayət",
+ "isÉ™",
+ "olmadı",
+ "yalnız",
+ "bəy",
+ "üç",
+ "onun",
+ "onu",
+ "buradan",
+ "yenÉ™",
+ "olmaz",
+ "edir",
+ "ən",
+ "ı",
+ "ondan",
+ "sizlər",
+ "özü",
+ "bəlkə",
+ "iki",
+ "lap",
+ "deyil",
+ "bəli",
+ "bizim",
+ "doqquz",
+ "həm",
+ "amma",
+ "sənin",
+ "ilk",
+ "hÉ™",
+ "mənə",
+ "düz",
+ "olur",
+ "indi",
+ "sizin",
+ "xanım",
+ "həmişə",
+ "bir",
+ "vÉ™",
+ "artıq",
+ "etmək",
+ "nÉ™",
+ "olduÄŸu",
+ "məhz",
+ "səhv",
+ "gilÉ™",
+ "olaraq",
+ "dəqiqə",
+ "buna",
+ "idi",
+ "altmış",
+ "səkkiz",
+ "zaman",
+ "mən",
+ "bəzən",
+ "bundan",
+ "dək",
+ "niyÉ™",
+ "qədər",
+ "yeddi",
+ "əlbəttə",
+ "ildÉ™",
+ "onlardan",
+ "yoxsa",
+ "dörd",
+ "dedi",
+ "saniyÉ™",
+ "xeyr",
+ "istifadÉ™",
+ "bütün",
+ "qırx",
+ "ü",
+ "obirisi",
+ "etmÉ™",
+ "olar",
+ "öz",
+ "əgər",
+ "mirÅŸey",
+ "bizlər",
+ "bunların",
+ "biraz",
+ "oradan",
+ "cı",
+ "lakin",
+ "É™",
+ "sənə",
+ "üçün",
+ "bunun",
+ "altı",
+ "hər",
+ "sonra",
+ "çünki",
+ "edən",
+ "doqsan",
+ "sadəcə",
+ "əlli",
+ "dÉ™",
+ "etdi",
+ "dən",
+ "yüz",
+ "belÉ™",
+ "görə",
+ "iyirmi",
+ "olsun",
+ "elÉ™",
+ "onlar",
+ "səksən",
+ "yetmiÅŸ",
+ "yox",
+ "oldu",
+ "qarşı",
+ "ilÉ™",
+ "ona",
+ "bunu",
+ "bax",
+ "cü"
+ ],
+ "fi": [
+ "sinun",
+ "teiltä",
+ "nämä",
+ "teissä",
+ "näille",
+ "heistä",
+ "olisit",
+ "minusta",
+ "minulle",
+ "tuona",
+ "tätä",
+ "kenen",
+ "miltä",
+ "teihin",
+ "siinä",
+ "keitä",
+ "häneen",
+ "kenet",
+ "siihen",
+ "vaikka",
+ "meidät",
+ "sinut",
+ "tuon",
+ "hän",
+ "joiden",
+ "sinua",
+ "vaan",
+ "niille",
+ "keiksi",
+ "minussa",
+ "tästä",
+ "tallä",
+ "siitä",
+ "jolla",
+ "keihin",
+ "olisimme",
+ "jotka",
+ "mutta",
+ "keissä",
+ "tuossa",
+ "keiltä",
+ "tuolta",
+ "ovat",
+ "noille",
+ "missä",
+ "mille",
+ "olette",
+ "olimme",
+ "meille",
+ "kanssa",
+ "keinä",
+ "teidän",
+ "heillä",
+ "minulta",
+ "tähän",
+ "teitä",
+ "itse",
+ "jolle",
+ "niissä",
+ "keneen",
+ "niiltä",
+ "oli",
+ "hänet",
+ "olin",
+ "minuun",
+ "teidät",
+ "niiksi",
+ "keille",
+ "siksi",
+ "tässä",
+ "niihin",
+ "johon",
+ "tälle",
+ "joina",
+ "niinä",
+ "kenenä",
+ "eivät",
+ "noista",
+ "meihin",
+ "mistä",
+ "noiksi",
+ "meissä",
+ "noihin",
+ "hänen",
+ "näitä",
+ "meiltä",
+ "minulla",
+ "näiltä",
+ "meitä",
+ "poikki",
+ "meidän",
+ "vai",
+ "teille",
+ "näissä",
+ "näihin",
+ "näiksi",
+ "jota",
+ "noilla",
+ "olit",
+ "mitä",
+ "tältä",
+ "joissa",
+ "joilta",
+ "näinä",
+ "olisi",
+ "joita",
+ "niitä",
+ "kuin",
+ "olisin",
+ "kenessä",
+ "olen",
+ "olisivat",
+ "sinussa",
+ "minua",
+ "noiden",
+ "heitä",
+ "heidän",
+ "teillä",
+ "mikä",
+ "jos",
+ "joka",
+ "nyt",
+ "sen",
+ "sinulta",
+ "tuoksi",
+ "jona",
+ "joksi",
+ "tämän",
+ "niiden",
+ "täksi",
+ "ette",
+ "keneltä",
+ "keistä",
+ "tuosta",
+ "sille",
+ "keneksi",
+ "siltä",
+ "tuolla",
+ "ollut",
+ "keillä",
+ "sinulle",
+ "häntä",
+ "tuotä",
+ "kenestä",
+ "joille",
+ "hänellä",
+ "kenelle",
+ "josta",
+ "minun",
+ "kuka",
+ "tämä",
+ "olet",
+ "sinusta",
+ "heissä",
+ "tuohon",
+ "että",
+ "olleet",
+ "minut",
+ "teistä",
+ "heiltä",
+ "nuo",
+ "olivat",
+ "heihin",
+ "näiden",
+ "jossa",
+ "joista",
+ "tai",
+ "noina",
+ "mukaan",
+ "niillä",
+ "häneltä",
+ "joihin",
+ "sekä",
+ "joiksi",
+ "yli",
+ "emme",
+ "näistä",
+ "meillä",
+ "millä",
+ "heille",
+ "mitkä",
+ "noin",
+ "hänessä",
+ "sitä",
+ "jonka",
+ "miksi",
+ "noilta",
+ "noissa",
+ "ketä",
+ "näillä",
+ "meistä",
+ "sinuun",
+ "noita",
+ "tänä",
+ "olisitte",
+ "mihin",
+ "hänestä",
+ "olla",
+ "heidät",
+ "keiden",
+ "sinulla",
+ "olemme",
+ "niistä",
+ "olitte",
+ "joilla",
+ "jolta",
+ "koska",
+ "kenellä",
+ "tuolle",
+ "hänelle"
+ ],
+ "da": [
+ "noget",
+ "hos",
+ "anden",
+ "jo",
+ "thi",
+ "været",
+ "ind",
+ "sit",
+ "hende",
+ "blive",
+ "bliver",
+ "havde",
+ "jer",
+ "hvad",
+ "af",
+ "ud",
+ "nogle",
+ "ham",
+ "hendes",
+ ],
+ "es": [
+ "estuvierais",
+ "esa",
+ "fuimos",
+ "tuviese",
+ "habíais",
+ "hubieses",
+ "nosotras",
+ "estuviera",
+ "tenemos",
+ "eso",
+ "estáis",
+ "habiendo",
+ "otra",
+ "habrías",
+ "seamos",
+ "fueron",
+ "eras",
+ "fueras",
+ "estando",
+ "hubieras",
+ "están",
+ "otro",
+ "sentida",
+ "tuvimos",
+ "tuviera",
+ "sintiendo",
+ "tenidas",
+ "suya",
+ "tuvieron",
+ "erais",
+ "tuvieseis",
+ "contra",
+ "tenido",
+ "tuviésemos",
+ "cuando",
+ "durante",
+ "las",
+ "serán",
+ "sí",
+ "vuestra",
+ "hemos",
+ "fuéramos",
+ "estabais",
+ "estemos",
+ "desde",
+ "tuviéramos",
+ "suyo",
+ "estada",
+ "quienes",
+ "sobre",
+ "soy",
+ "todos",
+ "tuvierais",
+ "estados",
+ "nuestra",
+ "estuvieran",
+ "tendríamos",
+ "esas",
+ "hubieron",
+ "tengáis",
+ "fuera",
+ "pero",
+ "hubieseis",
+ "tendrían",
+ "fueseis",
+ "esos",
+ "estábamos",
+ "también",
+ "hubiésemos",
+ "tuyas",
+ "fueran",
+ "estés",
+ "habré",
+ "eran",
+ "habido",
+ "vuestras",
+ "hubiéramos",
+ "estuvo",
+ "antes",
+ "algunos",
+ "habidas",
+ "nuestras",
+ "estuvimos",
+ "hubierais",
+ "unos",
+ "estarás",
+ "estaremos",
+ "tuyos",
+ "tanto",
+ "hubiese",
+ "estuvieses",
+ "tuvieses",
+ "estaba",
+ "les",
+ "sentid",
+ "estaríamos",
+ "estuviésemos",
+ "estarían",
+ "hayas",
+ "seas",
+ "quien",
+ "seríais",
+ "teniendo",
+ "tendrás",
+ "tendremos",
+ "tuvieras",
+ "teníais",
+ "estoy",
+ "hubiera",
+ "estaban",
+ "hubimos",
+ "esto",
+ "cual",
+ "sentidas",
+ "tendrá",
+ "serás",
+ "sentido",
+ "mí",
+ "habrán",
+ "tendrías",
+ "habían",
+ "ese",
+ "otros",
+ "tus",
+ "tengamos",
+ "había",
+ "habrá",
+ "tendréis",
+ "tienes",
+ "estar",
+ "hubisteis",
+ "míos",
+ "los",
+ "estuviéramos",
+ "otras",
+ "tendrán",
+ "muchos",
+ "eres",
+ "hayáis",
+ "mías",
+ "porque",
+ "habríamos",
+ "hasta",
+ "fueses",
+ "estuvieras",
+ "tuve",
+ "estabas",
+ "nosotros",
+ "teníamos",
+ "nada",
+ "estaréis",
+ "habríais",
+ "habidos",
+ "tuviesen",
+ "tened",
+ "nuestro",
+ "tenías",
+ "tengo",
+ "hubo",
+ "hubiste",
+ "vuestro",
+ "habremos",
+ "estará",
+ "algunas",
+ "ellos",
+ "suyos",
+ "habida",
+ "serías",
+ "estarías",
+ "estuviste",
+ "habéis",
+ "tengas",
+ "estás",
+ "él",
+ "vosostros",
+ "tú",
+ "estuvisteis",
+ "estuviesen",
+ "estuvieron",
+ "tuvieran",
+ "muy",
+ "fuésemos",
+ "ellas",
+ "suyas",
+ "fuisteis",
+ "estarán",
+ "estadas",
+ "nuestros",
+ "hayan",
+ "seréis",
+ "tuyo",
+ "mucho",
+ "fuesen",
+ "habréis",
+ "hubiesen",
+ "siente",
+ "hay",
+ "tendríais",
+ "tuviste",
+ "tuvo",
+ "fuerais",
+ "tenéis",
+ "ante",
+ "estado",
+ "estaría",
+ "poco",
+ "vuestros",
+ "fuese",
+ "tendré",
+ "esté",
+ "mío",
+ "tienen",
+ "tenida",
+ "yo",
+ "tuya",
+ "tiene",
+ "mía",
+ "estad",
+ "hubieran",
+ "seáis",
+ "habrás",
+ "tenidos",
+ "habías",
+ "estos",
+ "vosostras",
+ "estén",
+ "tendría",
+ "tenga",
+ "habíamos",
+ "fuiste",
+ "sentidos",
+ "seré",
+ "mis",
+ "qué",
+ "tengan",
+ "habrían",
+ "estaríais",
+ "tuvisteis",
+ "hayamos",
+ "algo",
+ "estuviese",
+ "haya",
+ "tenía",
+ "serían",
+ "estéis",
+ "estuve",
+ "sería",
+ "estuvieseis",
+ "habría",
+ "estaré",
+ "tenían",
+ "hube",
+ "fue",
+ "donde"
+ ],
+ "pt": [
+ "nossas",
+ "depois",
+ "sou",
+ "houveria",
+ "em",
+ "há",
+ "tiverem",
+ "teu",
+ "foi",
+ "fossem",
+ "meus",
+ "nosso",
+ "nós",
+ "fomos",
+ "eles",
+ "tivéssemos",
+ "hajam",
+ "estivermos",
+ "sejam",
+ "esteja",
+ "tivermos",
+ "estavam",
+ "vocês",
+ "houverão",
+ "formos",
+ "minha",
+ "estivemos",
+ "houveremos",
+ "estivera",
+ "tivéramos",
+ "hei",
+ "terão",
+ "tem",
+ "fora",
+ "aquela",
+ "essas",
+ "também",
+ "num",
+ "houvesse",
+ "dele",
+ "nossos",
+ "tiver",
+ "estejamos",
+ "tém",
+ "pelos",
+ "mas",
+ "estivesse",
+ "teve",
+ "teus",
+ "tivesse",
+ "estive",
+ "pelas",
+ "tuas",
+ "tiveram",
+ "nossa",
+ "ao",
+ "tivessem",
+ "aquilo",
+ "houveríamos",
+ "teríamos",
+ "uma",
+ "aquele",
+ "mesmo",
+ "tenham",
+ "estiverem",
+ "teria",
+ "minhas",
+ "tinha",
+ "isto",
+ "serei",
+ "seriam",
+ "muito",
+ "numa",
+ "fôssemos",
+ "esses",
+ "houverem",
+ "já",
+ "qual",
+ "forem",
+ "temos",
+ "só",
+ "tenha",
+ "teriam",
+ "lhe",
+ "tinham",
+ "elas",
+ "seria",
+ "aos",
+ "houveriam",
+ "aqueles",
+ "são",
+ "terei",
+ "delas",
+ "houvéssemos",
+ "dos",
+ "às",
+ "essa",
+ "estivéramos",
+ "tive",
+ "havemos",
+ "estiveram",
+ "suas",
+ "isso",
+ "tínhamos",
+ "tivera",
+ "houve",
+ "você",
+ "estou",
+ "estes",
+ "houver",
+ "seus",
+ "estivessem",
+ "nas",
+ "estiver",
+ "tivemos",
+ "não",
+ "serão",
+ "foram",
+ "tenho",
+ "houvera",
+ "houvemos",
+ "lhes",
+ "aquelas",
+ "houverá",
+ "quem",
+ "esteve",
+ "pela",
+ "fôramos",
+ "estava",
+ "quando",
+ "ela",
+ "houvermos",
+ "houvessem",
+ "houveram",
+ "hão",
+ "pelo",
+ "deles",
+ "estejam",
+ "até",
+ "esse",
+ "haja",
+ "seja",
+ "terá",
+ "estivéssemos",
+ "tenhamos",
+ "houvéramos",
+ "estávamos",
+ "houverei",
+ "teremos",
+ "estão",
+ "sejamos",
+ "seu",
+ "hajamos"
+ ],
+ "ne": [
+ "आफू",
+ "यी",
+ "रहेका",
+ "चाहिà¤",
+ "तिमी",
+ "अनà¥à¤¯à¤¤à¥à¤°",
+ "तà¥à¤°à¥à¤¨à¥à¤¤à¥ˆ",
+ "छ",
+ "थियो",
+ "तà¥à¤¸à¤ªà¤›à¤¿",
+ "हà¥à¤¨à¥",
+ "भनà¥à¤¨à¥‡",
+ "आतà¥à¤®",
+ "अरà¥à¤²à¤¾à¤ˆ",
+ "नौ",
+ "थिà¤",
+ "बरà¥",
+ "जहाà¤",
+ "बिशेष",
+ "लगभग",
+ "र",
+ "राखà¥à¤›",
+ "कि",
+ "यसबाहेक",
+ "पनि",
+ "पहिलà¥à¤¯à¥ˆ",
+ "कसरी",
+ "तिर",
+ "नजिकै",
+ "न",
+ "पटक",
+ "बिरà¥à¤¦à¥à¤§",
+ "देखिनà¥à¤›",
+ "तापनी",
+ "गरà¥à¤¨à¥à¤ªà¤°à¥à¤›",
+ "जान",
+ "तà¥à¤¯à¥‹",
+ "पछि",
+ "निमà¥à¤¨",
+ "म",
+ "साà¤à¤šà¥à¤šà¥ˆ",
+ "सबै",
+ "भनà¥à¤›à¥",
+ "जब",
+ "उदाहरण",
+ "पाà¤à¤šà¥Œà¤‚",
+ "सात",
+ "संगै",
+ "पकà¥à¤•à¤¾",
+ "बाहेक",
+ "औं",
+ "à¤à¤•à¤¦à¤®",
+ "रामà¥à¤°à¥‹",
+ "कसैले",
+ "à¤à¤•",
+ "पà¥à¤°à¤¤à¥‡à¤•",
+ "जसलाई",
+ "गरà¥à¤¦à¥ˆ",
+ "पूरà¥à¤µ",
+ "बारे",
+ "हà¥à¤¨à¥à¤›",
+ "यस",
+ "छà¥",
+ "सधै",
+ "तिनिहरà¥à¤²à¤¾à¤ˆ",
+ "दिनà¥à¤­à¤à¤•à¥‹",
+ "à¤à¤‰à¤Ÿà¥ˆ",
+ "भà¤",
+ "पहिले",
+ "उहालाई",
+ "सकà¥à¤›",
+ "थिà¤à¤¨",
+ "तेसà¥à¤°à¥‹",
+ "को",
+ "भितà¥à¤°",
+ "रहेको",
+ "यदि",
+ "साथ",
+ "ले",
+ "माथि",
+ "छू",
+ "हà¥à¤¨à¥‡",
+ "वासà¥à¤¤à¤µà¤®à¤¾",
+ "भनà¥",
+ "गरी",
+ "उप",
+ "गरà¥à¤›",
+ "तिनी",
+ "अरà¥à¤¥à¤¾à¤¤à¥",
+ "तपाईको",
+ "अà¤à¥ˆ",
+ "नै",
+ "ओठ",
+ "भनà¥à¤¨à¥à¤­à¤¯à¥‹",
+ "किनभने",
+ "आयो",
+ "हरेक",
+ "आजको",
+ "चाहनà¥à¤›à¥",
+ "समय",
+ "आदि",
+ "अरà¥à¤¥à¤¾à¤¤",
+ "कम से कम",
+ "कोही",
+ "नयाà¤",
+ "राखे",
+ "बीच",
+ "ती",
+ "देखि",
+ "तर",
+ "अब",
+ "बने",
+ "भने",
+ "बाहिर",
+ "ततà¥à¤•à¤¾à¤²",
+ "अकà¥à¤¸à¤°",
+ "वरीपरी",
+ "जे",
+ "पकà¥à¤•à¥ˆ",
+ "भितà¥à¤°à¥€",
+ "दिà¤",
+ "दà¥à¤ˆ",
+ "त",
+ "बीचमा",
+ "जसमा",
+ "सायद",
+ "सà¥à¤ªà¤·à¥à¤Ÿ",
+ "गरà¥à¤¨à¥",
+ "दिनà¥à¤¹à¥à¤¨à¥à¤›",
+ "अनà¥à¤¤à¤°à¥à¤—त",
+ "तपाई",
+ "यहाà¤",
+ "पहिलो",
+ "के",
+ "अरà¥à¤•à¥‹",
+ "यसरी",
+ "गरà¥à¤¨à¥‡",
+ "कतै",
+ "गरौं",
+ "नि",
+ "मा",
+ "यदà¥à¤¯à¤ªà¤¿",
+ "जबकि",
+ "सही",
+ "पà¥à¤²à¤¸",
+ "तà¥à¤¸à¥ˆà¤²à¥‡",
+ "गरà¥à¤¨",
+ "उनले",
+ "गैर",
+ "यहाà¤à¤¸à¤®à¥à¤®",
+ "निमà¥à¤¤à¤¿",
+ "गà¤",
+ "लाई",
+ "जताततै",
+ "यसको",
+ "चाले",
+ "आà¤",
+ "यसो",
+ "उनको",
+ "चाहनà¥à¤¹à¥à¤¨à¥à¤›",
+ "परà¥à¤¥à¥à¤¯à¥‹",
+ "जसà¥à¤¤à¥‹à¤¸à¥à¤•à¥ˆ",
+ "तीन",
+ "मलाई",
+ "तिनीहरà¥à¤•à¥‹",
+ "यथोचित",
+ "रूप",
+ "ठीक",
+ "फेरी",
+ "कृपया",
+ "गरà¥à¤›à¥",
+ "केही",
+ "भनà¥à¤›à¤¨à¥",
+ "गरि",
+ "संग",
+ "भन",
+ "गरेर",
+ "गयौ",
+ "यसà¥à¤¤à¥‹",
+ "गरेको",
+ "आफà¥à¤¨à¥‹",
+ "जसबाट",
+ "समà¥à¤­à¤µ",
+ "पà¥à¤°à¤¤à¤¿",
+ "जो",
+ "कà¥à¤°à¤®à¤¶à¤ƒ",
+ "मà¥à¤–à¥à¤¯",
+ "परà¥à¤›",
+ "लागि",
+ "छैन",
+ "आफà¥à¤¨à¥ˆ",
+ "या",
+ "जà¥à¤¨",
+ "निमà¥à¤¨à¤¾à¤¨à¥à¤¸à¤¾à¤°",
+ "रही",
+ "मातà¥à¤°",
+ "अनà¥à¤¸à¤¾à¤°",
+ "छनà¥",
+ "देखेर",
+ "जसà¥à¤¤à¥‹",
+ "कà¥à¤°à¤¾",
+ "साथै",
+ "देखेको",
+ "हरे",
+ "कà¥à¤¨à¥ˆ",
+ "हो",
+ "आफूलाई",
+ "यति",
+ "अरà¥",
+ "तथा",
+ "शायद",
+ "समà¥à¤®",
+ "हà¥à¤¨",
+ "तà¥à¤¯à¤¹à¤¾à¤",
+ "सोही",
+ "दोसà¥à¤°à¥‹",
+ "यसपछि",
+ "कहाà¤à¤¬à¤¾à¤Ÿ",
+ "जसà¥à¤¤à¥ˆ",
+ "चार",
+ "मेरो",
+ "पाà¤à¤š",
+ "यो",
+ "परà¥à¤¯à¤¾à¤ªà¥à¤¤",
+ "तदनà¥à¤¸à¤¾à¤°",
+ "जसले",
+ "देखियो",
+ "अनà¥à¤¯",
+ "भनà¥à¤¦à¤¾",
+ "तल",
+ "भà¤à¤•à¥‹",
+ "देखे",
+ "धेरै",
+ "गरेका",
+ "अगाडी",
+ "छौं",
+ "जाहिर",
+ "नतà¥à¤°",
+ "निरà¥à¤¦à¤¿à¤·à¥à¤Ÿ",
+ "किन",
+ "सबैलाई",
+ "सो",
+ "भर",
+ "सारा",
+ "कहिलेकाहीं",
+ "तेसà¥à¤•à¤¾à¤°à¤£",
+ "जसको",
+ "पछिलà¥à¤²à¥‹",
+ "अलग",
+ "कसै",
+ "तिनीहरू",
+ "अनà¥à¤¯à¤¥à¤¾",
+ "सटà¥à¤Ÿà¤¾"
+ ],
+ "nl": [
+ "zich",
+ "zou",
+ "hebben",
+ "reeds",
+ "ge",
+ "veel",
+ "worden",
+ "daar",
+ "aan",
+ "want",
+ "zonder",
+ "het",
+ "omdat",
+ "niet",
+ "voor",
+ "iets",
+ "zijn",
+ "zij",
+ "wordt",
+ "nog",
+ "dit",
+ "ik",
+ "tegen",
+ "heb",
+ "ben",
+ "ze",
+ "ons",
+ "uw",
+ "maar",
+ "hij",
+ "iemand",
+ "naar",
+ "geweest",
+ "doen",
+ "mijn",
+ "een",
+ "werd",
+ "zo",
+ "mij",
+ "haar",
+ "door",
+ "meer",
+ "dus",
+ "niets",
+ "kon",
+ "zal",
+ "toen",
+ "uit",
+ "ook",
+ "wezen",
+ "wil",
+ "met",
+ "geen",
+ "bij",
+ "moet",
+ "onder",
+ "zelf",
+ "eens",
+ "toch",
+ "kunnen",
+ "altijd",
+ "deze",
+ "wat",
+ "heeft"
+ ],
+ "ro": [
+ "fii",
+ "aceea",
+ "tăi",
+ "acestia",
+ "altfel",
+ "citi",
+ "atita",
+ "numai",
+ "cîtva",
+ "toata",
+ "pînă",
+ "către",
+ "acelea",
+ "atunci",
+ "acest",
+ "îmi",
+ "imi",
+ "după",
+ "putini",
+ "parca",
+ "aia",
+ "unele",
+ "atit",
+ "decit",
+ "fi",
+ "altceva",
+ "ala",
+ "două",
+ "tuturor",
+ "cei",
+ "să",
+ "prin",
+ "citeva",
+ "căci",
+ "unuia",
+ "doilea",
+ "mîine",
+ "deja",
+ "inca",
+ "mele",
+ "unul",
+ "ea",
+ "până",
+ "uneori",
+ "ăstea",
+ "alte",
+ "tale",
+ "unui",
+ "ceilalti",
+ "care",
+ "deci",
+ "sai",
+ "dată",
+ "prima",
+ "vostru",
+ "cind",
+ "abia",
+ "aceÅŸti",
+ "iar",
+ "oriunde",
+ "nostru",
+ "pai",
+ "fiţi",
+ "ati",
+ "chiar",
+ "atata",
+ "dacă",
+ "tocmai",
+ "despre",
+ "cat",
+ "ca",
+ "nici",
+ "nimeni",
+ "toţi",
+ "pina",
+ "vă",
+ "deÅŸi",
+ "drept",
+ "doi",
+ "niste",
+ "fără",
+ "noÅŸtri",
+ "nimic",
+ "acea",
+ "său",
+ "atat",
+ "cînd",
+ "sint",
+ "cîţi",
+ "isi",
+ "ÅŸi",
+ "aÅŸ",
+ "celor",
+ "mâine",
+ "nou",
+ "patru",
+ "cea",
+ "lor",
+ "voÅŸtri",
+ "pe",
+ "foarte",
+ "anume",
+ "cine",
+ "dintr-",
+ "mereu",
+ "în",
+ "inapoi",
+ "undeva",
+ "toate",
+ "aceeasi",
+ "adica",
+ "sunt",
+ "carora",
+ "noua",
+ "uneia",
+ "deasupra",
+ "dintr",
+ "alta",
+ "câţi",
+ "cum",
+ "toată",
+ "că",
+ "îţi",
+ "cărui",
+ "asupra",
+ "câtva",
+ "unei",
+ "va",
+ "dau",
+ "fata",
+ "acestei",
+ "zice",
+ "zi",
+ "nouă",
+ "câte",
+ "acela",
+ "fara",
+ "vouă",
+ "toti",
+ "printr-",
+ "totul",
+ "atitea",
+ "atitia",
+ "desi",
+ "cît",
+ "acei",
+ "fiu",
+ "ii",
+ "îi",
+ "orice",
+ "căror",
+ "caruia",
+ "oricând",
+ "acel",
+ "ceva",
+ "cand",
+ "pot",
+ "patra",
+ "cite",
+ "mea",
+ "mai",
+ "insa",
+ "aceia",
+ "catre",
+ "ale",
+ "cel",
+ "poate",
+ "îl",
+ "puţin",
+ "asta",
+ "mult",
+ "ăsta",
+ "avem",
+ "vreo",
+ "doar",
+ "tău",
+ "unora",
+ "atare",
+ "pentru",
+ "daca",
+ "alea",
+ "aţi",
+ "treilea",
+ "careia",
+ "avea",
+ "alti",
+ "ălea",
+ "altul",
+ "spre",
+ "apoi",
+ "aceÅŸtia",
+ "multa",
+ "Å£i",
+ "noastră",
+ "mă",
+ "unor",
+ "oricât",
+ "ul",
+ "inainte",
+ "face",
+ "fim",
+ "intre",
+ "oricine",
+ "sau",
+ "citiva",
+ "aceste",
+ "voastră",
+ "unii",
+ "noastre",
+ "acestui",
+ "multă",
+ "aveţi",
+ "dintre",
+ "fie",
+ "altcineva",
+ "sale",
+ "spate",
+ "ar",
+ "eÅŸti",
+ "sintem",
+ "Å£ie",
+ "ori",
+ "ului",
+ "sunteţi",
+ "oricare",
+ "sub",
+ "intr",
+ "puţină",
+ "asa",
+ "totusi",
+ "acestea",
+ "atatia",
+ "aceasta",
+ "voastre",
+ "atatea",
+ "fiecare",
+ "trei",
+ "mulţi",
+ "lângă",
+ "incit",
+ "oricum",
+ "multi",
+ "multe",
+ "niÅŸte",
+ "cărei",
+ "astfel",
+ "aici",
+ "cita",
+ "oricînd",
+ "ba",
+ "unu",
+ "cineva",
+ "dă",
+ "treia",
+ "altii",
+ "unde",
+ "mei",
+ "ăştia",
+ "cam",
+ "acesta",
+ "iti",
+ "aibă",
+ "primul",
+ "dupa",
+ "acele",
+ "această",
+ "sa-mi",
+ "fost",
+ "cele",
+ "ceea",
+ "prea",
+ "totuÅŸi",
+ "cit",
+ "dar",
+ "acelasi",
+ "puţina",
+ "peste",
+ "cât",
+ "astea",
+ "lîngă",
+ "săi",
+ "oricît",
+ "suntem",
+ "sa-ti",
+ "vreun",
+ "cumva",
+ "cîte",
+ "ăla",
+ "pic",
+ "tine",
+ "avut"
+ ],
+ "cs": [
+ "aÄkoli",
+ "ahoj",
+ "alespoň",
+ "anebo",
+ "aspoň",
+ "během",
+ "blízko",
+ "bohužel",
+ "brzo",
+ "brzy",
+ "bude",
+ "budeme",
+ "budeš",
+ "budete",
+ "budou",
+ "budu",
+ "byl",
+ "byla",
+ "byli",
+ "bylo",
+ "byly",
+ "bys",
+ "být",
+ "celá",
+ "celé",
+ "celý",
+ "Äeská",
+ "Äi",
+ "Äísel",
+ "Äísla",
+ "Äíslo",
+ "dál",
+ "dále",
+ "další",
+ "dát",
+ "děkujeme",
+ "děkuji",
+ "dělá",
+ "dělat",
+ "dík",
+ "díky",
+ "dnes",
+ "dobrý",
+ "docela",
+ "dokud",
+ "druhá",
+ "druhé",
+ "druhý",
+ "druzí",
+ "hned",
+ "hodnÄ›",
+ "chce",
+ "chceme",
+ "chceš",
+ "chcete",
+ "chci",
+ "chtějí",
+ "chtít",
+ "já",
+ "jak",
+ "jaká",
+ "jaké",
+ "jaký",
+ "jde",
+ "jeho",
+ "její",
+ "jejich",
+ "jemu",
+ "jen",
+ "jenom",
+ "jenž",
+ "jestli",
+ "jestliže",
+ "ještě",
+ "jí",
+ "jich",
+ "jím",
+ "jiná",
+ "jinak",
+ "jiné",
+ "jiný",
+ "jít",
+ "již",
+ "jsem",
+ "jsi",
+ "jsme",
+ "jsou",
+ "jste",
+ "každá",
+ "každé",
+ "každý",
+ "kde",
+ "kdo",
+ "kdy",
+ "když",
+ "kolik",
+ "kromÄ›",
+ "která",
+ "které",
+ "který",
+ "kteří",
+ "kvůli",
+ "má",
+ "mají",
+ "málo",
+ "mám",
+ "máme",
+ "máš",
+ "máte",
+ "mé",
+ "mÄ›",
+ "měl",
+ "mém",
+ "mezi",
+ "mí",
+ "místo",
+ "mít",
+ "mnÄ›",
+ "mnou",
+ "moc",
+ "mohl",
+ "mohou",
+ "mohu",
+ "možná",
+ "možné",
+ "možný",
+ "můj",
+ "musí",
+ "může",
+ "můžeme",
+ "můžeš",
+ "můžete",
+ "nad",
+ "nade",
+ "nám",
+ "námi",
+ "naproti",
+ "nás",
+ "náš",
+ "naše",
+ "naši",
+ "našich",
+ "našim",
+ "našimi",
+ "nebo",
+ "nebyl",
+ "nebyla",
+ "nebyli",
+ "nebyly",
+ "něco",
+ "nějak",
+ "nějaká",
+ "nějaké",
+ "nějakou",
+ "nějaký",
+ "někam",
+ "někde",
+ "někdo",
+ "nemají",
+ "nemáme",
+ "nemáš",
+ "nemáte",
+ "neměl",
+ "němu",
+ "není",
+ "než",
+ "nich",
+ "nikam",
+ "nikde",
+ "nikdo",
+ "ním",
+ "nimi",
+ "nová",
+ "nové",
+ "nový",
+ "pak",
+ "paní",
+ "pod",
+ "podle",
+ "pokud",
+ "pořád",
+ "potom",
+ "pozdravem",
+ "proÄ",
+ "prosím",
+ "prostÄ›",
+ "proti",
+ "protože",
+ "prvé",
+ "první",
+ "před",
+ "přejeme",
+ "přeji",
+ "přes",
+ "přese",
+ "při",
+ "příloha",
+ "přílohou",
+ "přílohy",
+ "příloze",
+ "přiložená",
+ "přiložené",
+ "přiloženém",
+ "přiložený",
+ "přiložených",
+ "přímo",
+ "sám",
+ "skoro",
+ "smí",
+ "smíme",
+ "smíš",
+ "smíte",
+ "snad",
+ "spolu",
+ "svá",
+ "svého",
+ "svému",
+ "svoje",
+ "svoji",
+ "svou",
+ "svůj",
+ "tady",
+ "také",
+ "takhle",
+ "taková",
+ "takové",
+ "takový",
+ "takto",
+ "taky",
+ "takže",
+ "tamhle",
+ "tamhleto",
+ "tamto",
+ "tÄ›",
+ "tebou",
+ "teÄ",
+ "tedy",
+ "tento",
+ "této",
+ "tobÄ›",
+ "tohle",
+ "trošku",
+ "třeba",
+ "tvá",
+ "tvé",
+ "tvoje",
+ "tvůj",
+ "urÄitÄ›",
+ "už",
+ "vám",
+ "vámi",
+ "vás",
+ "váš",
+ "vaše",
+ "vaši",
+ "vaší",
+ "vážená",
+ "vážené",
+ "vážení",
+ "vážený",
+ "vÄera",
+ "vědět",
+ "však",
+ "vše",
+ "všeho",
+ "všechen",
+ "všechna",
+ "všechno",
+ "všechny",
+ "všichni",
+ "vždy",
+ "zaÄ",
+ "zatím",
+ "zatímco",
+ "zda",
+ "zítra",
+ "žádám",
+ "žádná",
+ "žádné",
+ "žádný",
+ "že"
+ ],
+ "sl": [
+ "avgust",
+ "bila",
+ "bile",
+ "bili",
+ "bilo",
+ "biti",
+ "blizu",
+ "bomo",
+ "boste",
+ "bova",
+ "boš",
+ "brez",
+ "celi",
+ "celo",
+ "daleÄ",
+ "deset",
+ "težak",
+ "težka",
+ "težki",
+ "težko",
+ "Äesto",
+ "Äetrta",
+ "Äetrtek",
+ "Äetrti",
+ "Äetrto",
+ "Äigav",
+ "Å¡est",
+ "Å¡esta",
+ "Å¡esti",
+ "Å¡esto",
+ "Å¡tiri",
+
+ ],
+}
diff --git a/contrib/languages-data/sv.json b/contrib/languages-data/sv.json
new file mode 100644
index 0000000..dbcdfcb
--- /dev/null
+++ b/contrib/languages-data/sv.json
@@ -0,0 +1 @@
+{"freq":{"D":81614,"E":57703,"F":65307,"G":58568,"A":101184,"B":88354,"C":71198,"L":68342,"M":88836,"N":55651,"O":30870,"H":85820,"I":53417,"J":41709,"K":67803,"U":32439,"T":73345,"W":29206,"V":48264,"P":64540,"S":183432,"R":54519,"Y":8819,"X":3563,"Z":5647,"f":573497,"g":796472,"d":1393951,"e":3122256,"b":426497,"c":439610,"a":2769748,"n":2605268,"o":1513455,"l":1576016,"m":1015154,"j":171593,"k":981769,"h":549646,"i":1975038,"w":38244,"v":702277,"u":641405,"t":2130744,"s":2008119,"r":2700083,"q":5367,"p":544314,"z":28076,"y":232177,"x":48501,"Å":5245,"Ö":9894,"é":11827,"å":315734,"ä":562602,"ü":3814,"ö":429328," l":92862," m":210837," n":78082," o":280066," h":123124," i":363557," j":42422," k":154387," d":271388," e":324754," f":359134," g":79790," a":307677," b":150256," c":22076," y":5622," u":95544," t":189989," v":181674," p":161981," s":494735," r":78305," J":40952," K":63008," H":83187," I":43524," N":52283," O":27251," L":64682," M":82331," B":82669," C":63320," A":84049," F":59275," G":56022," D":77148," E":54653," Z":5389," Y":8451,"и":3560,"о":3458," S":163518," R":51071," P":59385,"а":4143," W":28309," V":40400," U":30253," T":68207," å":25155," ä":214016," ö":25267," Å":5217," Ö":9837,"A ":15096,"F ":4011,"Da":13315,"Cl":3705,"Co":16319,"Ce":3786,"Ch":12336,"Ed":3462,"Do":5286,"De":40359,"Di":6082,"Fe":3321,"Fa":6453,"Eu":5433,"Er":5959,"En":12408,"El":5544,"Ge":7613,"Ga":6615,"I ":11291,"Fr":15828,"Fo":6429,"Fl":4338,"Fi":8854,"B ":3976,"C ":5688,"Au":4696,"Ar":9978,"As":4625,"D ":3210,"Ba":14125,"Ad":3948,"Am":3905,"An":15253,"Al":15279,"Bu":5535,"Br":13255,"Ca":15038,"Bi":5634,"Be":16801,"Bo":13961,"Bl":4691,"Ku":4607,"Gö":6526,"Kr":7039,"Ko":8264,"Le":10243,"Li":14030,"La":13793,"Lu":7481,"Lo":9958,"Me":11631,"Mi":11197,"Ma":32867,"Mu":4647,"Mo":11964,"Ni":7405,"Ne":10761,"Na":8237,"P ":3714,"Ny":3435,"No":15326,"Ol":4960,"Gr":10737,"Go":5035,"Gu":8716,"Ha":34281,"He":15981,"II":4483,"Hi":4225,"Ho":12929,"Hu":4943,"K ":4256,"In":13186,"Is":3759,"Ja":10130,"Je":5266,"Jo":15004,"Ju":4068,"Ka":19254,"Fö":6773,"M ":4536,"Ki":6491,"Ke":4271,"Up":4420,"Un":6136,"Ty":4282,"Tu":3610,"US":9839,"Tr":7442,"To":8743,"Th":13440,"Ti":6372,"Te":8278,"Ta":5863,"V ":6654,"Sy":4319,"St":38584,"Sv":20759,"TV":4696,"Su":6305,"Wo":3225,"Wi":9262,"Wa":7328,"We":4981,"Vi":10913,"Va":8115,"Ve":4995,"Pr":8839,"S ":6544,"Pe":11351,"Pa":14568,"Po":7593,"Pi":4124,"Or":4779,"Se":8850,"Sc":5830,"Si":8063,"Sh":4489,"Sk":9057,"Sp":7173,"So":11179,"Ru":3598,"Sa":15777,"Re":10230,"Ri":6924,"Ro":13031,"SA":10320,"Ra":8269,"b ":9647,"a ":475159,"Yo":5029,"Sö":3933,"Vä":9609,"bö":8648,"i ":355245,"fy":6327,"gd":12439,"ge":171880,"gf":3700,"ga":96208,"gb":3970,"fj":3473,"fl":21495,"ff":12688,"bå":4938,"fi":46296,"bä":6295,"fr":73976,"fu":7874,"ft":39725,"fo":44582,"j ":14814,"gy":3273,"he":67749,"ha":96551,"gn":17191,"gl":17131,"gj":3942,"gi":45303,"gh":13879,"gg":27749,"gu":21320,"gt":19887,"gs":51929,"gr":63119,"go":20646,"dt":3864,"du":15217,"dv":9506,"dy":5840,"g ":162227,"ea":25581,"eb":31218,"ec":33642,"ed":130488,"de":463243,"dd":77303,"dg":4171,"df":3528,"di":70271,"dh":3483,"dk":3943,"dj":7587,"dm":5988,"dl":16308,"do":35107,"dn":14977,"ds":66886,"dr":50264,"ew":8421,"ex":20287,"eu":8550,"ev":26002,"ey":12032,"fa":41395,"h ":210556,"fe":29350,"eh":9823,"eg":38958,"ef":28657,"ee":11526,"el":240548,"ek":41076,"ej":4317,"ei":17866,"ep":29857,"eo":14539,"en":790036,"em":80183,"et":324009,"es":162582,"er":632238,"ca":14258,"e ":392431,"by":19363,"br":42376,"bu":29326,"bo":47200,"bl":35898,"bi":33086,"bb":11237,"be":121628,"db":6093,"da":116179,"f ":20176,"cu":4292,"ct":5938,"co":13763,"ck":84782,"ci":26831,"ch":224513,"ce":44300,"c ":6962,"az":4871,"ay":9160,"ba":47465,"d ":342945,"at":219537,"as":105952,"ar":493878,"ax":4752,"aw":3514,"av":157525,"au":25162,"ak":38031,"al":200396,"ai":14768,"aj":13462,"ap":42925,"am":154793,"an":490452,"ac":25353,"ad":157516,"aa":3218,"ab":17368,"ag":71089,"ah":8954,"ae":7341,"af":19685,"nu":26246,"nt":115132,"ns":230163,"nr":8077,"no":73752,"nn":76927,"jö":13694,"ny":12036,"nv":25830,"oe":5372,"of":28837,"oc":231970,"od":38865,"oa":7955,"ob":24414,"om":258987,"on":206933,"ok":32662,"ol":124858,"oi":5452,"kå":10126,"oj":3321,"og":35630,"kä":17405,"oh":11968,"ot":63544,"os":44373,"ov":30611,"ou":23986,"op":33718,"oo":9810,"or":224835,"r ":824959,"ow":7736,"kö":11089,"pe":96760,"pg":3698,"pa":50445,"pl":23211,"pn":3697,"po":41753,"ph":7115,"lä":54929,"pi":19978,"lå":14272,"lo":57125,"ln":12525,"lm":43916,"ll":251645,"ls":70931,"lp":6851,"lv":21024,"lu":27907,"lt":40778,"hö":19038,"ly":18029,"o ":47987,"md":3318,"ma":128458,"mb":42798,"mg":3435,"mh":5236,"me":195253,"mf":8701,"mk":4460,"ml":20883,"mi":57579,"mn":28583,"mm":63704,"mp":23791,"mo":40002,"mr":10561,"mt":14449,"ms":22455,"mu":35770,"my":8276,"p ":30112,"na":190258,"nb":8963,"nc":14129,"nd":274187,"ne":121340,"nf":12519,"ng":233628,"jä":20365,"nh":9404,"ni":141019,"nj":5464,"nk":26276,"nl":21248,"nm":4189,"ju":37966,"jo":14819,"gå":19638,"ki":37850,"kh":15721,"ke":98699,"ka":235170,"m ":249455,"fö":204508,"ky":14497,"gö":7789,"ks":31609,"kt":90094,"ku":21062,"kv":10463,"ko":98732,"kr":52042,"kl":32687,"km":8676,"kn":24158,"li":194145,"hå":8698,"lh":8848,"hä":12770,"lk":26056,"lj":22122,"le":171301,"ld":48550,"lg":7505,"lf":10995,"la":251597,"lb":23488,"n ":932062,"hr":6279,"ht":5130,"hu":27638,"hj":3586,"dä":13148,"då":7895,"hi":28029,"hn":5105,"ho":40609,"hl":4048,"id":81847,"ic":45251,"ib":12524,"ia":49113,"ig":122539,"if":22472,"ie":80770,"dö":42849,"hy":3755,"k ":153527,"ir":29485,"is":203197,"it":107910,"iu":7188,"iv":58134,"ix":3284,"ii":3814,"ik":122112,"il":185111,"im":21187,"in":352330,"io":82024,"ip":11363,"je":24984,"få":5419,"fä":10915,"iz":3965,"l ":174554,"ja":31220,"tä":25279,"xi":4781,"tå":13823,"xt":11670,"sö":12206,"z ":7095,"xa":4078,"xe":8556,"sä":22008,"wi":3916,"så":23262,"rö":18726,"y ":50286,"wa":8836,"we":6979,"vl":8639,"rä":38698,"rå":61767,"vi":89468,"vt":4439,"vu":11778,"vr":3344,"vs":16546,"vn":4095,"vo":9064,"uv":14989,"ve":165060,"vd":3502,"va":154942,"x ":11441,"ui":6645,"uk":16033,"ul":41627,"ue":11748,"ug":17372,"ur":68778,"us":77613,"ut":71499,"um":42001,"un":129135,"up":38948,"ty":33055,"tu":36176,"tt":213949,"tv":19150,"ub":16763,"ua":23575,"ud":29487,"uc":9950,"w ":8904,"to":108377,"tn":19214,"tm":5831,"tl":17707,"ts":67257,"tr":105466,"tg":11122,"tf":8472,"te":298793,"tk":5397,"tj":8416,"ti":267169,"på":66737,"th":27907,"v ":145502,"tb":14691,"ta":250228,"su":13835,"sv":52306,"ss":78482,"st":354837,"sy":19796,"sl":47405,"sk":296865,"sn":10814,"sm":23482,"sp":69363,"so":182040,"sr":7516,"sd":11605,"sc":12968,"sf":11899,"se":125434,"sh":21233,"sg":4215,"sj":15800,"si":100666,"nö":3871,"u ":12792,"sa":104819,"sb":15311,"rr":35803,"rs":138379,"rt":88500,"ru":68780,"rv":15260,"ry":24021,"rp":8643,"ro":113836,"rn":86552,"rm":33875,"rl":42446,"rk":70077,"rj":9549,"ri":267243,"nå":6703,"nä":22292,"rh":9656,"rg":57899,"rf":20073,"re":276377,"rd":70096,"rc":6855,"rb":28710,"ra":275879,"t ":541739,"mö":6674,"qu":3576,"må":14078,"mä":17850,"s ":365755,"lö":8782,"pt":22670,"pu":12367,"pp":63825,"pr":69684,"ps":16974,"vä":52142,"zi":3697,"vå":20606,"za":3995,"yg":18262,"yf":3366,"yc":9359,"yd":14899,"ya":7829,"tö":15637,"yt":14189,"ys":34208,"yr":23125,"yp":6499,"yn":12472,"ym":10053,"yl":9018,"yk":5316,"å ":92466,"äc":6371,"Ös":5286,"ö ":8541,"én":3279,"åv":3406,"ån":81191,"åt":17902,"ås":6794,"år":43331,"åg":13520,"åe":3379,"ål":17121,"åk":8868,"åd":24022,"ät":25342,"äv":21618,"äx":8215,"äm":18322,"äl":34618,"än":82390,"äp":6744,"äs":34864,"är":272645,"äd":8120,"äg":24196,"äk":15670,"öv":17529,"öt":11908,"ör":194602,"ös":15048,"öp":9470,"ön":13721,"öl":7954,"öm":7511,"öj":4231,"ök":5353,"ög":10789,"öd":115283,"一":3570," Ga":6575," Ge":7551," I ":6565," Fo":6382," Fr":15807," Fi":8815," Fl":4322," Ha":34254," He":15951," Go":5001," Gr":10670," Gu":8675," Hu":4927," Ho":12906," Hi":4166," Je":5254," Ja":10108," Is":3740," In":13126," Fö":6755," Ka":19203," Ke":4195," Ki":6438," Jo":14976," Ju":4056," La":13680," Le":10167," Li":13966," Ko":8252," Kr":7023," Ku":4598," Gö":6518," Ma":32732," Mi":11127," Me":11596," Lo":9918," Lu":7453," Ne":10698," Na":8152," Ni":7391," Mo":11908," Mu":4615," Am":3890," An":15211," Al":15206," Ad":3934," Ba":14044," Au":4688," As":4526," Ar":9902," Be":16738," Bi":5612," Bl":4680," Bo":13888," Br":13201," Bu":5515," Ca":14547," Ce":3768," Ch":12294," Cl":3646," Co":16148," Da":13258," Di":6045," De":40274," Do":5127," Ed":3453," El":5523," Er":5935," En":12351," Eu":5425," Fe":3308," Fa":6390," Sö":3929," Wi":9211," We":4953," Wa":7296," Vä":9600," Yo":5022," Or":4762," Po":7530," Pi":4113," Pe":11321," Pa":14488," Ny":3423," No":15289," Ol":4952," Ra":8225," Ro":12950," Re":10191," Ri":6904," Pr":8806," Sy":4310," Sv":20662," TV":4398," Su":6295," St":38280," Ta":5840," Th":13403," Ti":6347," Te":8213," US":9725," Tr":7389," To":8642," Ru":3585," Sa":15732," Sh":4450," Si":8029," Sc":5776," Se":8798," So":11126," Sp":7111," Sk":9040," Va":8077," Ve":4954," Vi":10865," Tu":3571," Ty":4268," Un":6118," Up":4407," ja":13806," få":4608," fä":3840," in":71154," is":5201," ka":33444," fö":177054," gå":6683," ki":4333," ke":3269," jo":4366," ju":20755," ha":54753," he":14640," gi":5926," gr":27602," gu":3989," dö":38312," id":4924," dä":12563," då":6671," hi":6661," ho":7645," hu":13035," ne":4922," na":17184," my":5673," mu":11845," mo":16924," ok":9806," ol":7701," om":30114," kä":11926," oc":192923," of":14077," ny":4446," nu":5991," no":24830," le":9968," hä":8278," li":23241," la":18204," kv":5049," ku":9406," ky":7530," km":5693," kl":6892," kr":12841," ko":48113," me":100026," mi":15594," ma":40584," hö":10368," lo":3574," ad":5086," am":17761," an":46161," ap":9654," ak":3619," al":18398," av":131499," au":10184," ar":19850," at":35597," ba":19765," bi":15445," be":42377," bo":12025," bl":19693," by":8036," br":16393," e ":3326," et":57787," en":197679," el":30240," ef":10304," eg":6206," fe":13408," fa":16157," ex":7709," fu":4697," fr":64090," fo":25076," fl":15697," bå":3785," fi":26001," ge":16400," ga":6755," i ":270663," bö":6965," fy":5095," ce":5878," ci":5961," da":13590," do":7611," dr":6511," de":166176," di":13002," vä":22816," yt":3626," tä":6373," sö":6783," ru":4704," ry":5559," sa":28037," se":37040," sj":9848," si":31916," sm":3812," sl":17774," sk":41581," sp":27636," so":132908," mö":3254," ra":6370," re":30692," ri":11458," nå":5590," nä":11487," ro":9439," pu":3945," pr":32472," ps":3376," s ":6480," mä":6492," må":7724," or":14441," kö":4010," pe":10275," pa":12607," pl":6656," po":22401," lå":10106," lä":20032," rö":4188," så":14446," sä":12510," va":97104," ve":11661," rä":4287," vi":43226," ty":10671," tv":8751," tu":4474," ut":38370," ur":7073," up":19177," un":29257," ta":17457," sy":11768," st":66122," sv":36306," su":3592," tr":20781," to":9251," th":7205," på":64716," ti":85531," te":14430," Ös":5280," år":16994," åt":5457," än":4390," äl":3304," är":187240," äv":12511," ös":4997," öv":12751,"Fin":4121,"Eri":4081,"Eur":4760,"En ":5663,"Eng":3595,"Öst":5201,"Fra":6710,"Fre":3915,"Hel":4484,"Her":3363,"Han":20213,"Har":3270,"Gra":3275,"Ind":4121,"Hon":4948,"Alb":3593,"And":5658,"Car":7211,"Ber":6424,"De ":4186,"Det":11763,"Den":16516,"Dan":4412,"Cha":4718,"Cou":3952,"New":5646,"Nor":12355,"Per":3613,"Pet":3811,"Par":5960,"Pro":3506,"SA ":9788,"Joh":8659,"För":6368,"Kal":4090,"Kar":6529,"Göt":5300,"Lin":4919,"Man":3676,"Mal":3953,"Mar":11224,"Söd":3244,"Wil":4251,"Väs":5464,"Yor":3342,"Sve":19151,"Str":3346,"Sto":17434,"Sta":8398,"Ste":4189,"TV ":4653,"äga":3677,"äge":8468,"ägg":3687,"äck":6350,"änn":6181,"äns":11234,"ämn":7168,"äms":3895,"äng":10799,"änd":34964,"äpp":6344,"Sch":3849,"San":4701,"är ":211888,"älv":4078,"äll":14085,"äkt":7912,"äkn":3856,"äld":3913,"än ":11102,"äve":13254,"ävl":6172,"äxt":6157,"ärk":4071,"ärl":8853,"ärm":4235,"ära":8245,"ärd":6093,"äre":3225,"ärn":6279,"ärs":4830,"äst":24789,"ätt":18556,"åde":18255,"ågo":4233,"åna":9220,"ång":25948,"åll":6899,"ån ":40117,"åre":4622,"åt ":4155,"ård":7005,"år ":24938,"åte":4737,"ått":4014,"Upp":4062,"Tys":3259,"The":8834,"USA":9498,"bis":3915,"bil":16029,"bin":3462,"ble":7136,"bli":9670,"bla":13299,"bok":5630,"bol":12338,"bor":13470,"bbe":3314,"ban":15887,"bas":6749,"bar":10200,"beg":3230,"ber":59590,"ben":7083,"bel":10187,"bes":11143,"bet":17916,"ca ":4920,"ce ":7249,"bri":10671,"bro":7219,"bra":3701,"bre":3338,"bru":13260,"bur":4739,"bun":4581,"bum":12276,"by ":6960,"byg":8032,"aka":4483,"am ":15544,"ake":4072,"al ":27066,"ain":4963,"aj ":9347,"ags":6901,"agn":5084,"anv":11075,"anu":13080,"ano":4964,"ann":22378,"ant":22398,"ans":71523,"ane":7517,"anf":3613,"ang":11657,"ani":15707,"ank":9321,"anl":8379,"ap ":4055,"ana":13418,"anc":5238,"and":127973,"amt":9744,"amm":21193,"aml":16952,"amo":3480,"amn":18156,"amp":5088,"amh":4267,"ami":11713,"amf":3558,"ame":25638,"amb":4074,"ama":6369,"alv":3571,"alt":10698,"als":5130,"alo":3365,"alm":9277,"all":37991,"alk":5184,"ali":21293,"ald":6403,"ale":25866,"ala":19113,"alb":12779,"an ":132556,"akt":15324,"ad ":44388,"aft":3394,"aff":3221,"afi":4365,"aga":8161,"age":18047,"adm":3257,"adi":7918,"ade":78504,"ag ":17369,"ads":8986,"ack":8031,"ach":4677,"ace":4160,"ada":3703,"af ":3707,"at ":35834,"are":94208,"ard":11975,"arb":9854,"ara":27724,"aro":4452,"arn":18125,"arm":4137,"arl":13544,"ark":16487,"ari":37821,"arr":8387,"ars":21054,"art":35107,"asi":5423,"ase":5556,"ask":3632,"ar ":171511,"apa":8845,"ape":7645,"app":4146,"apr":9022,"as ":45411,"ava":3381,"avs":7927,"avi":4830,"ave":5436,"ay ":3818,"av ":125744,"ata":7543,"ast":21648,"ass":11560,"ato":8615,"ate":21603,"ati":48093,"att":63033,"ats":14038,"atu":6856,"aur":3255,"aug":8539,"jer":3396,"jen":6427,"fäl":3203,"fär":6347,"jan":14542,"je ":5239,"jor":7802,"itu":3337,"itt":20776,"ity":3491,"isk":74955,"ism":4171,"iss":12117,"ist":54624,"iv ":6421,"ita":12733,"ite":17827,"iti":20829,"ius":3596,"iva":11600,"ivi":7901,"ive":20061,"is ":19720,"ion":60198,"ir ":3453,"irk":5584,"isi":5287,"ish":4776,"ise":7902,"isa":8098,"ire":6030,"it ":10727,"ja ":4918,"kil":7942,"kiv":4415,"kin":6276,"gån":6463,"går":10145,"kis":6751,"kho":12716,"kel":5730,"ken":19496,"kes":3520,"ker":29895,"ket":13180,"key":3901,"ke ":13529,"kra":8741,"kre":6291,"kt ":28682,"ksa":5304,"ksd":4444,"kro":5638,"kri":26131,"kot":3776,"kor":12173,"kon":20182,"kom":35824,"kol":9820,"ks ":3388,"kna":9166,"kni":12636,"klu":5282,"kla":15409,"kli":6794,"jul":9684,"jun":10738,"jur":5620,"kat":7694,"kar":20696,"kas":4078,"kap":15909,"kan":45226,"kal":21132,"kam":3425,"kad":6591,"ka ":105354,"för":137814,"föd":63483,"ha ":4594,"ham":7622,"han":27562,"hal":5140,"hav":3856,"har":31426,"had":5793,"he ":12764,"hel":8034,"het":17433,"her":10299,"hen":4180,"hem":4216,"då ":5689,"där":12388,"hin":3839,"his":5950,"gli":6286,"gla":6605,"gni":3551,"gna":6982,"gs ":13333,"gon":4582,"gor":4044,"gsk":4034,"gru":20067,"gra":18317,"gt ":15114,"gre":14703,"gst":6826,"gsm":3229,"gus":10096,"grä":4757,"ial":7807,"ian":12040,"ic ":4032,"ibl":3325,"id ":28142,"ibe":3733,"ia ":17620,"iet":8507,"iel":7300,"ien":33298,"ier":11523,"ies":3414,"ig ":23812,"ift":11636,"ick":11163,"ici":4587,"ich":7049,"ice":6107,"ie ":9770,"ica":5181,"ids":4482,"idr":4877,"idn":3610,"idi":9977,"ide":12497,"ida":10207,"il ":15253,"ika":38088,"ige":28072,"iga":25749,"igh":9261,"igi":4197,"igg":9635,"igt":12741,"ik ":16104,"ime":3935,"ind":16350,"ina":18896,"inn":24862,"ino":16003,"int":17796,"ins":22420,"inf":4654,"ine":12860,"ing":135978,"ini":12277,"inl":6319,"ink":4368,"inv":10239,"ike":30688,"ila":3896,"in ":36372,"ikt":12966,"iks":9950,"ilo":5583,"ill":91188,"ilk":6078,"ilm":10451,"ilj":12364,"ili":8683,"ild":15424,"ile":4047,"io ":6788,"ils":3516,"hol":17007,"hon":3609,"hri":3262,"hum":3706,"hus":7589,"huv":8614,"död":37434,"fes":6198,"fer":3302,"feb":8379,"fat":13524,"far":5867,"fam":7188,"fal":3610,"ext":4721,"exe":4418,"eta":15891,"ete":27600,"eti":4494,"esp":9596,"est":29943,"ess":19229,"ev ":6530,"etr":3912,"ets":17582,"ett":72343,"ety":4212,"ew ":5717,"eve":6119,"eva":3607,"evi":3537,"ey ":7588,"elä":5847,"er ":284777,"eor":5154,"es ":68641,"ept":9611,"epp":4218,"epr":4795,"erk":19481,"erl":7951,"eri":65274,"erg":20456,"erh":4470,"enä":3549,"ere":6711,"erf":4436,"era":63522,"erb":6250,"et ":167149,"esk":4221,"esi":7482,"ese":7369,"erv":5509,"err":9816,"ert":13225,"ers":49047,"ern":40094,"erm":7787,"ero":5248,"eki":3784,"eko":7026,"ekt":14604,"en ":561421,"ela":36480,"ele":19021,"eli":7809,"eln":6244,"ell":67565,"els":32415,"elt":6933,"emb":26969,"ema":5588,"eme":8121,"emm":3946,"emo":6706,"emi":7410,"emp":5979,"ene":8729,"enh":4318,"eng":9041,"enb":3805,"ena":11883,"end":16477,"eno":10317,"enn":10256,"eni":7932,"ens":78815,"ent":44100,"enr":3651,"ege":11795,"egi":9515,"egr":5273,"eis":3601,"ein":4280,"el ":33296,"em ":8949,"öte":5512,"gjo":3575,"öst":10660,"git":4157,"gis":8031,"giv":6533,"gin":4809,"gio":4718,"gic":3318,"gif":3672,"örs":34131,"öra":7478,"örb":6194,"örd":7646,"ghe":6517,"öre":25188,"örf":10182,"örj":5505,"örk":3758,"ggn":3889,"gge":11686,"gga":4221,"gi ":4509,"öpi":3748,"ör ":72432,"gen":70552,"get":15734,"ger":32986,"ges":7614,"gel":15568,"gde":4040,"ge ":21010,"ön ":5859,"öm ":3264,"gas":5062,"gar":34878,"gat":4458,"gan":12485,"ga ":26894,"ögs":4071,"ödr":4070,"bör":6225,"frå":39311,"frä":4616,"fte":17736,"fta":8409,"fun":3567,"ft ":9311,"fra":14521,"fri":9705,"for":23834,"fot":7918,"fol":8477,"fle":7024,"flo":3656,"fly":6056,"fic":5656,"fil":11844,"fik":3556,"fin":14513,"fis":3388,"öve":12758,"da ":24533,"dd ":64224,"de ":113906,"dad":11525,"dal":5199,"dag":13716,"dat":8203,"das":4305,"dar":13228,"dan":22463,"dam":6624,"dda":3257,"dde":6028,"cks":6976,"ckh":12838,"ckn":5609,"ckl":6273,"öd ":37381,"öde":6673,"ödd":63162,"ch ":194706,"cer":9280,"cen":9841,"cem":8851,"cha":5427,"cia":6069,"ck ":16616,"cie":3837,"che":9375,"chi":3577,"cir":4840,"cke":18899,"cka":6286,"ed ":60806,"ebo":7090,"ebr":11024,"ean":3203,"eat":4659,"ea ":3260,"efo":3657,"eft":11252,"edl":4748,"edi":6332,"edd":3460,"ede":13539,"eda":22012,"edr":4279,"eck":13530,"eci":3498,"ece":9407,"dvä":3688,"dor":6126,"don":6243,"dom":6715,"ds ":21182,"dmi":3679,"dni":12167,"dst":6539,"duc":4844,"dri":7974,"dra":24350,"dre":6617,"dro":7925,"dsk":11908,"dia":4497,"der":69482,"des":48453,"det":49713,"dec":9237,"del":42235,"den":110439,"dem":7457,"dle":5138,"dla":5392,"dli":4421,"din":7458,"dio":6335,"dis":13125,"dit":3226,"die":4951,"dig":14250,"dju":3500,"näm":4493,"när":11567,"näs":3945,"rga":9150,"ri ":24498,"rgi":4148,"rge":9457,"rgs":6893,"ret":18000,"res":20396,"rev":5759,"rfa":11243,"rds":5163,"rg ":18322,"rea":6436,"red":13615,"reg":14934,"rem":4053,"ren":33831,"rek":11220,"rel":8037,"rer":8988,"rep":9910,"rda":6312,"rdn":3815,"rdi":6203,"rde":19138,"re ":105451,"rbu":4396,"rd ":16211,"rar":16692,"ras":12995,"rat":27144,"rav":4107,"rbe":10058,"rag":4677,"ran":40758,"ram":18971,"ral":14704,"rak":6953,"rab":3374,"raf":9108,"rad":32041,"rs ":30266,"rr ":4068,"rlä":3700,"ror":5868,"ros":5853,"rot":10704,"rom":7155,"ron":11508,"rop":10007,"rov":7683,"rod":8744,"roc":6434,"rol":8031,"rof":6006,"rog":7715,"rna":45582,"rne":7715,"rni":4580,"ro ":5673,"rma":11324,"rme":6990,"rli":7045,"rld":8530,"rle":4342,"rla":6287,"rn ":13803,"rks":6561,"rko":6401,"rki":4271,"rke":8992,"rka":21148,"rm ":4966,"rja":6006,"rl ":5718,"rio":4391,"rit":18193,"ris":26661,"riv":11699,"rig":24904,"någ":4585,"ril":11007,"rik":49805,"rin":29008,"rim":3215,"ria":11330,"ric":6800,"rid":7589,"rie":22621,"rif":3813,"rk ":11840,"rup":12524,"run":19578,"rum":7564,"ruk":6602,"rus":3564,"rva":5265,"rvi":3304,"ry ":6473,"rsk":18149,"rsi":7847,"rso":8438,"rsp":7925,"rsa":13567,"rse":4696,"rta":8444,"rst":26007,"rss":3568,"rte":14318,"rth":3501,"rti":15874,"rua":8438,"rts":3730,"rt ":25088,"rri":6493,"rre":7456,"rra":8312,"sak":4471,"sal":8723,"sam":44053,"san":6209,"sat":10892,"sar":8127,"sa ":10458,"rys":4892,"sho":4590,"shi":4912,"sju":4204,"sie":4171,"sid":6241,"sk ":90513,"sit":12645,"sis":10405,"sin":20425,"sio":10464,"sik":13865,"sig":10296,"sda":5839,"sde":3304,"se ":11017,"sch":5874,"ser":32108,"ses":3294,"set":6663,"sed":10233,"sep":9935,"sen":30980,"sel":3990,"spo":4913,"spr":13840,"slä":13781,"spe":39400,"spa":4154,"som":124363,"son":32242,"sor":7813,"skå":7923,"soc":6710,"st ":46043,"ss ":9668,"sli":3748,"slo":3319,"slu":6612,"sky":3403,"sla":13504,"sle":3801,"ski":13633,"skl":5724,"sko":16207,"skr":17282,"sku":3455,"skt":14119,"sfö":5604,"ska":106537,"ske":8526,"sjö":6673,"sni":4681,"sjä":3770,"sma":7643,"sme":3479,"stå":11548,"stä":10967,"syd":3945,"stö":8124,"sys":5192,"svä":4118,"sse":11502,"ssa":10371,"sso":16553,"ssl":3516,"ssi":9151,"sst":3551,"ssp":3681,"ste":53976,"sta":88293,"stn":5297,"sto":24174,"sti":35352,"stu":7051,"str":43893,"sty":3941,"sva":8262,"sve":36524,"tal":35934,"tag":10048,"tad":34944,"tav":3738,"tat":22284,"tas":6110,"tar":38387,"tan":25694,"tam":3270,"te ":29236,"tbo":6832,"ta ":57380,"pa ":4951,"par":19834,"pas":4154,"pan":9819,"läg":9136,"lär":4866,"läp":5670,"län":20491,"läk":8254,"pen":15225,"per":22100,"pet":5946,"pel":41073,"pla":14119,"pin":6560,"lån":4645,"pis":3811,"låt":5580,"por":8039,"pop":3841,"pos":4574,"pol":17194,"pps":6631,"ppt":7044,"ppl":4401,"ppa":4036,"ppe":13206,"pp ":11986,"pub":4680,"pte":14474,"pru":4841,"psa":5456,"pri":16288,"pre":11182,"pro":27839,"prå":5570,"män":8000,"mäs":4442,"mål":5318,"mån":4648,"ra ":70040,"ngl":9266,"ngr":3616,"ngt":3834,"ngs":25507,"ni ":11042,"nge":55986,"nga":25508,"ngd":5985,"jäl":6437,"jär":7781,"nhe":3391,"neh":3788,"nel":8830,"nen":21817,"ner":30394,"net":14471,"nes":10240,"ng ":83505,"neb":3747,"ned":3616,"nce":6161,"ne ":14860,"ndr":19487,"nds":29703,"ndo":7016,"ndl":5881,"ndi":11752,"nde":95443,"nda":25018,"nal":16599,"nam":15523,"nan":10186,"nar":23024,"nad":14500,"nd ":63041,"nat":21915,"nas":10608,"na ":68634,"ny ":3572,"num":3857,"nus":4249,"nua":9868,"nty":4644,"nto":5785,"ntr":13553,"nti":10623,"nta":12228,"nte":31914,"nsp":4485,"nst":23393,"nss":4316,"nse":13545,"nsi":5055,"nsl":4609,"nsk":92387,"nsa":6209,"nri":3216,"nt ":21290,"ns ":54594,"nom":26142,"nor":19884,"nov":9987,"nne":21868,"nna":22185,"nno":3272,"nni":10650,"nns":9117,"nli":10002,"nn ":5037,"nla":5932,"no ":3529,"nke":3380,"ngå":3281,"nkt":4581,"nkr":3354,"nfö":4263,"nie":10939,"nia":3660,"niv":6000,"nis":23117,"nit":5715,"nio":3817,"nin":62497,"nik":4407,"ogr":10374,"ogi":6891,"ohn":3565,"kän":11729,"oha":5651,"kåd":7525,"ok ":3568,"ol ":3524,"och":189597,"oci":4767,"ock":32660,"ode":11255,"of ":8033,"odu":7243,"og ":7992,"oft":6611,"off":4118,"ofe":5148,"od ":6498,"obe":13311,"nvä":12582,"nvå":8371,"jör":3329,"köp":4602,"ote":5241,"ott":18376,"ots":4197,"oto":5671,"ost":8837,"ota":4879,"otb":6812,"osi":3701,"ose":3675,"oss":3221,"ovi":7790,"ove":12742,"oun":6352,"our":5919,"opp":7590,"ope":5756,"opa":4338,"os ":9800,"or ":33039,"ork":5444,"orm":13532,"orn":10771,"orr":11303,"ord":31398,"ore":8252,"org":22560,"ori":16030,"osa":3703,"ort":28485,"ors":16477,"ot ":11261,"ora":9653,"ola":11528,"on ":79327,"oli":25700,"oll":18015,"olk":10298,"ole":5201,"ols":3442,"olm":16608,"olo":12711,"oly":3210,"oka":3360,"om ":168644,"oke":3223,"okr":4246,"okt":9964,"ona":12877,"ond":7866,"one":32789,"ong":8629,"oni":7172,"ono":6812,"ons":23539,"ont":10940,"oma":9422,"ome":9673,"omb":3421,"omi":4970,"omm":30299,"omk":3355,"omp":5523,"omr":7897,"oms":4749,"op ":3482,"la ":37263,"le ":15487,"lde":11008,"lda":11000,"lds":6297,"ldr":3289,"lac":3318,"lad":21513,"lag":21635,"lan":79933,"lam":3972,"lar":35655,"lat":17536,"las":17797,"lba":3463,"ld ":9319,"lbu":12650,"kvi":3264,"kva":4810,"kus":3478,"kun":6039,"kul":5875,"kså":3945,"kta":4658,"kte":14929,"kti":15289,"kto":12389,"kyr":9176,"gör":5454,"ls ":8684,"lom":6295,"lor":5534,"lod":5577,"log":11812,"los":3900,"lot":3451,"lni":4907,"lme":6188,"lma":4184,"lms":5047,"lti":5165,"lub":5423,"lsk":12154,"lss":5166,"lst":11990,"lta":4364,"lte":4556,"lse":12155,"lsa":4733,"lt ":15274,"häl":3201,"här":4070,"li ":11521,"lev":10258,"les":8492,"let":19531,"ler":49986,"lem":6956,"len":25248,"lek":6522,"led":11323,"lls":18041,"llt":8125,"llv":5128,"lhö":4242,"lla":46043,"lle":53525,"llh":4880,"lli":11251,"lln":3548,"lkr":4019,"ln ":4441,"lke":7097,"lm ":18320,"lje":9323,"ll ":78820,"lja":3349,"lit":24473,"lis":16271,"lin":35763,"liv":5593,"lic":4558,"lia":6013,"lik":14533,"hål":7111,"lig":48482,"lie":9952,"ma ":10795,"maj":9292,"mar":31731,"mas":5427,"mal":5574,"man":40392,"mat":14610,"mbe":28393,"me ":3701,"med":67946,"met":18659,"mes":8746,"mer":42686,"mel":15875,"men":33082,"lva":3742,"lve":7060,"lun":3206,"lut":8771,"lyg":4277,"hög":7787,"hör":6448,"mpi":3263,"mpe":6539,"mpo":3301,"ms ":6348,"mod":5091,"mon":5448,"mok":4020,"mor":4951,"mot":12947,"mt ":8137,"mst":8336,"mrå":8280,"mus":13621,"mun":18113,"mfö":3853,"min":18579,"mil":15426,"mis":6696,"mit":4808,"mli":14006,"mla":4634,"mn ":9157,"mni":3641,"mne":9027,"mmu":16790,"mma":26822,"mme":13993,"vå ":7032,"väg":7022,"vän":13490,"vär":10896,"väs":8080,"väx":7295,"vån":8847,"ytt":4640,"yta":3770,"yst":7068,"ysk":15364,"yrk":11273,"yra":3515,"yde":3835,"yck":8174,"ya ":3709,"ygg":9192,"xte":5064,"tör":12969,"täl":6340,"xem":4510,"tår":5717,"tär":4618,"täv":5108,"söd":5258,"så ":8428,"sån":9241,"sät":7762,"röm":5410,"rör":3335,"vs ":5015,"vud":9343,"rät":6983,"råk":6071,"vik":5560,"vil":12065,"rån":39221,"vin":14334,"råd":11141,"vid":23789,"vit":5788,"vis":16981,"vli":3664,"rän":8989,"räk":4195,"räm":4496,"räd":4698,"ver":61891,"vet":10570,"ven":64039,"vem":8723,"vec":5468,"ve ":6856,"val":12123,"van":15891,"var":99905,"vat":6662,"va ":9694,"uvu":9284,"usi":13675,"use":6334,"ust":23739,"utg":8418,"uti":3815,"ute":6086,"uta":8659,"utt":3405,"uts":5227,"utv":5038,"us ":21399,"ut ":10089,"ura":3453,"ure":6299,"urg":5379,"uri":5728,"urn":5209,"uro":7154,"urs":7342,"ur ":12946,"upp":32257,"umb":4012,"ume":9374,"unt":7357,"unk":5142,"uni":16744,"unn":5336,"und":51074,"ung":19464,"une":3661,"ukt":5611,"um ":17518,"ult":6645,"ull":6732,"uli":10267,"un ":12274,"ugu":9934,"ude":4661,"udi":4906,"uce":4932,"uds":4702,"udo":3784,"uar":18907,"ubl":5383,"ubb":6201,"två":7001,"typ":3540,"tyr":3920,"tys":8073,"ty ":8321,"trö":5020,"tve":6253,"trä":9790,"tur":14907,"tun":3943,"tud":5778,"tyd":4220,"ts ":26900,"tre":12611,"tt ":116227,"tra":36819,"tri":14508,"tru":8774,"tro":12216,"try":3989,"tse":5043,"tsk":5040,"tst":3437,"tta":33856,"tte":27085,"tti":9635,"ttn":5259,"tts":5893,"ttr":3221,"to ":4895,"tni":11444,"tjä":4558,"tna":3229,"tod":3739,"toc":13127,"tog":4722,"tob":9412,"tom":4310,"ton":13637,"tol":5901,"tor":39112,"til":75428,"tik":17763,"tif":5408,"tie":6576,"tig":8660,"tit":6965,"tis":30207,"tin":14073,"tio":41843,"thu":4634,"tia":4024,"tic":3207,"tid":19952,"tiv":14860,"tli":6668,"tla":6760,"tem":18447,"ten":56126,"tek":5853,"tel":9950,"teb":4880,"tec":5494,"th ":6931,"tex":3539,"tet":22695,"tes":9480,"ter":115723,"tgi":4603,"på ":62169,"ti ":13119,"the":7679},"n_words":[31862602,36956776,26222440],"name":"sv","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/languages-data/sw.json b/contrib/languages-data/sw.json
new file mode 100644
index 0000000..e5cefd2
--- /dev/null
+++ b/contrib/languages-data/sw.json
@@ -0,0 +1 @@
+{"freq":{"jer":348,"jen":305,"ji ":6234,"D":1805,"E":874,"F":1081,"G":1202,"A":4461,"B":2717,"C":2251,"L":1530,"M":12761,"N":2782,"O":860,"H":1677,"I":2605,"J":2641,"K":12188,"U":3120,"T":5185,"W":4730,"V":1116,"P":2090,"S":3343,"R":1632,"Y":517,"Z":395,"f":11048,"g":13829,"d":15034,"e":46694,"Feb":214,"b":19688,"c":9784,"a":289584,"n":90468,"o":57043,"l":42025,"m":53651,"j":21456,"k":76835,"h":32492,"i":164978,"w":60984,"v":3863,"u":57506,"t":40551,"s":35298,"r":27443,"p":13501,"z":18893,"y":38832,"x":501,"jar":185,"jan":137,"jaw":201,"é":167,"jim":1500,"jin":4267,"jil":163,"jij":492,"jia":221,"jib":3854,"ito":288,"itu":317,"itw":269,"isp":140,"ist":592,"ita":1061,"ite":213,"iti":334,"ivy":133,"iwa":2430,"ius":183,"ipo":224,"ipi":265,"is ":521,"ion":720,"iop":279,"ipa":165,"ipe":219,"iro":173,"iri":997,"isi":902,"ish":5756,"isa":694,"ire":164,"ira":314,"ja ":1529,"iyo":4644,"iye":227,"izo":242,"izi":413,"iza":568," l":8602,"kif":518," m":27935," n":19872," o":327,"kik":333," h":7652," i":9059,"kij":166,"kim":258," j":5212,"kil":389," k":27977," d":1010," e":802," f":914,"kia":390," g":257," a":6533," b":1252," c":2191,"kiw":279," y":17767," z":2257,"kin":442," u":4361,"kio":148," t":2402,"kip":379," w":34366," v":1482,"kis":520," p":2154,"kit":315," s":6097," r":837,"ki ":2193," J":2627," K":12017," H":1638," I":2128," N":2678," O":803," L":1487," M":12665," B":2646," C":2112," A":4277," F":1046," G":1172," D":1740," E":782," Z":375," Y":513,"и":142," S":3229," R":1588,"а":137," P":2015," W":4707," V":1031," U":3052," T":5117,"kea":156,"kem":150,"ke ":1988,"ku ":187,"kri":520,"kon":141,"koa":3734,"ko ":1214,"ل":165,"ا":240,"juu":155,"jul":257,"jum":177,"kaz":5045,"kaw":137,"kat":14149,"kar":374,"kas":316,"kan":2795,"kao":197,"kal":354,"kam":1048,"kad":160,"kab":375,"ka ":19783," Ga":196,"Da":365," Ge":229,"Co":364," Fr":177,"Ch":770," Ha":622," He":218," Go":142,"Do":469," Gr":177," Gu":142,"De":497,"Di":169,"Fe":311," Id":148,"Fa":160," Hu":173," Ho":177," II":154,"ha ":2668," Hi":392,"Ge":229," Ji":535,"Ga":198," Je":286,"I ":397," Ja":792,"Fr":177," Ir":284," Is":141," It":181," In":316," Ik":143," Il":224,"ham":522,"han":444,"hap":154," Ka":2225,"hai":238,"haj":163,"hak":611,"hal":314," Ke":708," Ki":3568,"har":1714,"has":255,"hat":148," Jo":255,"II ":207," Ju":691,"hag":267,"hab":181,"had":740," La":231," Le":207," Li":441," Ko":414," Ku":695," Kw":4009,"Au":181," Ma":4258," Mb":461,"Ar":475,"As":222," Mk":3388,"Ba":771," Mi":685," Mj":478," Me":615,"Af":445,"he ":544,"Ag":372," Lo":213,"Am":241,"An":463,"Ap":290," Lu":315,"Al":840," Ne":518,"Bu":429,"Br":278," Na":464,"Ca":592," Ni":435,"Bi":308," Mt":420,"Be":362," Mp":146," Mo":643,"Bo":282," Mu":471," Mw":545,"Ku":695,"Kw":4009,"Ko":415,"hez":299,"Le":210,"Li":441,"hes":336,"her":275,"hen":226,"hem":395,"La":231,"Lu":315,"Lo":213,"Me":621,"hi ":3880,"Mi":690,"Mj":478,"Mk":3388,"Ma":4263,"Mb":461,"Mw":546,"Mu":475,"Mt":420,"Mp":146,"Mo":643,"Ni":437,"Ne":518,"Na":466," Ap":290," Am":240," An":463," Al":833,"Ny":247," Ag":372," Af":443,"No":466," Ba":766,"Ok":277," Au":181," As":222," Ar":474," Be":362," Bi":308,"hio":2603,"Gr":177,"Go":143,"hin":1991,"him":244,"hil":432,"Gu":142," Bo":282,"hii":230," Br":278," Bu":429,"his":266,"hir":394,"Ha":622," Ca":582,"hiy":239,"He":219,"II":286,"Hi":393," Ch":768,"Ho":179,"Hu":173," Co":362,"K ":152,"Id":148," Da":365," Di":167,"In":317," De":495,"Ik":143,"Il":226,"Is":141,"It":181," Do":469,"Ir":284,"Ja":792,"Ji":536,"Je":286,"Jo":255,"Ju":691,"Ka":2234,"Has":225,"ho ":334," Fe":311,"Ki":3577," Fa":159,"Ke":708,"Us":172,"Ut":325,"Ur":181,"go ":920,"Un":355,"Uk":150,"Ul":189,"Ui":244,"Uj":249,"Uh":170,"Uf":251,"Uc":175,"Tu":237,"To":205,"Th":275,"Te":258," Wi":3377,"Ta":3841," We":188," Wa":1003,"St":260,"Su":178,"Wi":3380,"Wa":1003,"We":189," Zi":141," Za":152,"Vi":670," Yo":250,"Pr":150,"Pe":270,"goz":233,"Pa":858,"Po":195,"Pi":163,"gom":190,"gon":205,"gos":279,"gor":306,"Se":532,"gu ":424,"Si":424,"Sh":518,"So":239,"Ru":370,"Sa":668,"Re":188,"Ri":138,"Ro":385,"Ra":354," Po":195,"guj":253," Pi":163," Pe":270," Pa":857," Ny":247," No":466," Ok":277," Ra":354,"b ":211," Ro":385,"gwe":166," Re":188," Ri":138,"gwa":280,"guz":429," Pr":150,"a ":143240," Su":178," St":248," Ta":3838," Th":274,"Yo":250," Te":257," To":205," Ru":370," Sa":668," Sh":517," Si":421," Se":528," So":239," Vi":666," Tu":231,"Za":152,"Zi":141," Uc":175," Uf":251," Uh":170," Ui":243," Uj":249," Uk":150," Ul":189," Un":355," Ur":181," Us":172," Ut":325," ja":134,"iak":142,"i ":52347,"ian":874," ji":4522,"ias":364,"ge":1928,"iar":235," je":226,"ga":2900," im":145," in":3363," ik":274," il":4878,"fi":1075,"fr":504,"fu":1927,"fo":752,"ibl":142,"ibi":603," ka":16147,"gw":483," ki":3027,"he":2541,"ibu":4111,"ha":8898,"gl":145,"gi":1836,"gh":1233,"gu":1858,"iba":566," ju":300,"go":2336,"du":838,"dw":136,"g ":607," ha":1606,"ea":1091,"eb":539," he":144,"ec":251,"ed":686,"de":1841,"di":4816,"dh":617,"do":1639,"ia ":9119,"dr":203,"ew":912,"ex":163,"eu":261,"ev":332,"ey":739,"ez":1828,"fa":6104,"h ":704," id":219,"fe":174,"eh":737," hi":990,"eg":644,"ef":303,"ee":307,"el":2120,"ek":2577,"ej":155," ho":139,"ei":650,"ep":643,"eo":1165,"en":9965,"em":2423,"et":1296," hu":4749,"es":2258,"er":4147," nj":147,"ca":364," ni":9330,"e ":10467," ng":147," nd":690,"bw":843," nc":2455," na":6269,"br":408,"bu":5373,"bo":2905," mw":6857,"bl":321," mu":4335," mt":648," ms":331,"bi":2134," mp":280," mo":680," mn":1501,"be":1280," mm":157,"ifu":393,"da":3239,"f ":246,"ifo":606," of":164,"co":390," ny":523,"ck":301,"ci":283,"ch":7388,"ce":365,"ifa":585," le":184,"c ":192," li":859," la":7153," ku":5668,"ich":830," kw":2736," km":140,"ica":140," ko":150," me":184," mf":368,"az":6015,"ay":5308," mi":1257,"ba":6016," mj":5191," mk":1617,"d ":1205,"at":22079,"as":4908,"ar":9773," ma":3449," mb":469,"aw":1490," mc":155,"av":414,"au":1667," lu":341,"ak":14678,"al":8458,"idi":551,"ai":5267,"aj":1998,"ao":6210,"ap":5739,"ide":157,"am":9111,"an":29556,"ac":1224,"ad":3126,"ida":813,"aa":1773,"ab":2568,"ag":1596,"ah":1414,"ae":682,"af":1092,"nu":591,"nt":1270,"ns":4895,"no":1318,"nn":478," am":1335," an":488,"nz":5093," ai":153,"iin":242,"ny":7307," aj":134," ak":183," al":2589,"of":4380," au":941,"oc":308,"od":678,"oa":4118,"ob":631," at":195," as":220,"om":1846,"on":3853,"ok":2328," ba":679,"ol":1930,"oi":1488,"oj":1425,"og":855,"oh":360,"ija":140,"ot":1280," bi":222,"os":1066,"ov":580,"ou":534,"ije":137,"op":845,"oo":318,"or":2938,"iji":1232,"r ":1622,"ow":244,"oz":397,"oy":154,"pe":836,"pa":6921,"po":1264,"ph":151,"pi":2193,"ika":13864,"lo":1408,"lm":337,"Ida":135,"ll":791,"ls":182,"iga":224,"ii ":525,"lu":868,"lt":178,"igh":170,"igi":384,"ly":147,"o ":24303,"mc":173,"igo":169,"ma":8274,"mb":6660,"mh":261,"me":2630,"mf":564,"mk":1733,"ml":210,"mi":3477,"mj":5199,"mn":1546,"mm":321,"mp":578,"ihe":138,"mo":6079,"mr":140,"mt":753,"ms":447,"mu":6394,"mw":6988,"ihi":187,"p ":352,"na":23279,"nc":2788,"nd":5575,"ne":2353,"ng":6858,"ni":24361,"nj":567,"nk":135,"imo":196," es":141," en":369,"ju":713,"imf":161,"ime":354," el":223,"jo":133,"imi":180,"ki":6922,"kh":154,"ind":834,"ke":2748,"ina":8001," fa":353,"ka":45110,"imu":392,"m ":727," fu":177,"kw":3124,"ino":181,"ks":210,"kt":463,"ku":10532,"ins":133,"ko":5804,"ine":479,"ing":1959,"kr":669," fi":274,"ini":4598,"km":156,"li":17984,"le":2997,"ld":221,"lf":159,"la":14880,"lb":250,"iny":275,"n ":3144,"iko":612,"hw":492,"ht":198,"hu":6825,"iki":2488,"hi":11111," ch":2090,"hn":150,"ho":1180,"ila":4379,"id":1813,"ic":1403,"ib":5595,"ia":11251,"ih":490,"in ":378,"ig":1252," da":146,"if":1790,"ie":672,"iku":2496,"k ":628,"ilo":373,"ir":1982,"is":9376,"it":2904,"ill":288,"iu":466,"iv":385,"iw":2556,"ii":989,"ij":1580,"ik":19966," de":224,"ili":8251,"il":13887,"im":4832,"in":17333,"io":4395,"ile":321,"ip":1169,"ima":914,"je":934,"imb":2471,"io ":2960,"ji":17145,"iz":1362,"iy":4997," du":302,"l ":1018,"ja":2368,"z ":191,"wi":1773,"wo":202,"vy":671," za":1702,"y ":1239,"wa":56175," zi":456,"we":2203,"vi":1632,"vu":418,"vo":138,"uz":1451,"uw":2877,"uv":252,"uu":3068," ye":258,"ve":578," ya":17428,"va":328,"x ":213,"ui":563,"uj":4429,"uk":1643,"ul":2575,"ue":357,"uf":741,"ug":901,"uh":626,"ur":1919,"us":3274,"ut":2784,"um":5397,"un":5099,"uo":368,"up":1077,"ty":166,"tu":2287,"tt":391,"tw":473,"ub":1112,"ua":2111,"ud":534,"uc":476,"w ":435,"to":4407,"huk":345,"hul":146,"tl":220,"ts":343,"tr":455,"te":2280,"ti":12092,"th":999,"ta":14867,"su":644,"ss":500,"st":1842,"sw":308,"sl":142,"sk":865,"sm":139,"sp":289,"so":683,"sc":179,"se":5649,"sh":8151,"si":4764,"u ":13704,"sa":7736,"rr":220,"rs":467,"rt":620,"ru":2279,"ry":287,"ro":1786,"rn":619,"rm":257,"rl":223,"rk":320,"ri":8157,"rg":403,"re":3855,"rd":556,"rc":143,"rb":136,"ra":5018,"t ":1231,"s ":3025,"pt":348,"pu":357,"pw":193,"pr":381," sa":589," se":4480," si":369," sh":318," ra":432," ri":188,"hwa":473,"huo":175,"hum":2789,"hun":282,"hus":506,"hur":418,"huu":1333," pe":176," pa":632," pi":931," wa":33135," we":275," vy":396," wi":862," vi":1013," uc":144,"zi":8597,"ze":368,"za":8043," tu":189,"zw":257," us":165," ut":249," up":502," um":247,"zu":272," un":1571," uk":210,"zo":952," ul":573," uh":139," ta":1410,"ye":2395,"ya":24129,"yu":306," to":170," th":289,"yo":5888," te":201,"yi":4283,"Apr":266,"Asi":146,"Aru":195,"far":316,"fam":283,"fan":4203,"fal":292,"fa ":488,"eya":259,"Bah":237,"Bar":140,"eza":1136,"ezo":172,"ezi":237,"eta":229,"ete":154,"eti":253,"est":247,"ett":212,"ew ":355,"evi":165,"ewe":148,"ey ":361,"ewa":358,"er ":615,"epa":149,"es ":640,"ept":299,"eri":650,"ere":660,"era":456,"Afr":406,"esh":359,"ese":306,"esa":279,"eru":498,"Ago":254,"ert":152,"ers":339,"eku":184,"en ":297,"ela":204,"ele":786,"eli":360,"ell":177,"eo ":852,"emb":1055,"ema":157,"eme":314,"emi":276,"emu":365,"ene":704,"eng":671,"ena":283,"end":498,"eno":221,"eni":486,"ens":4087,"ent":441,"eny":1803,"Ali":478,"ege":351,"Ame":158,"ehe":647,"Ana":176,"el ":260,"eke":267,"eka":1754,"giz":193,"gir":232,"gin":349,"gid":165,"ght":136,"gha":925,"gi ":572,"gen":204,"ger":781,"ge ":611,"gaz":140,"gar":155,"gan":693,"ga ":1334,"Cal":307,"fup":194,"Bib":137,"fua":317,"fum":143,"fun":167,"fri":445,"fu ":810,"for":356,"fo ":342,"fil":269,"fik":168,"fiz":146,"da ":1525,"de ":752,"dad":386,"dae":220,"dar":151,"dan":305,"dam":173,"Des":272,"Dar":167,"Chi":216,"Chu":136,"Cha":300,"ch ":165,"cha":2430,"chu":596,"ck ":143,"che":571,"chi":3152,"cho":370,"ed ":154,"ebr":313,"ea ":663,"ei ":346,"efu":197,"edi":297,"ee ":156,"don":150,"dom":308,"dol":151,"dog":335,"dun":335,"dha":302,"dia":330,"dhi":240,"der":146,"deg":261,"del":152,"di ":2661,"do ":429,"Dod":240,"diy":201,"din":291,"dis":387,"dik":302,"ri ":2373,"rez":420,"rea":148,"ref":154,"reh":266,"ren":163,"rek":1672,"re ":305,"rd ":213,"ras":256,"rat":173,"Ni ":218,"New":381,"rai":160,"ran":867,"ram":226,"rab":297,"rad":150,"ron":135,"rog":253,"rne":169,"rni":283,"ro ":593,"riw":166,"ris":508,"ril":300,"rik":1688,"rin":373,"ria":769,"rib":1011,"ric":160,"rk ":191,"ruf":262,"rum":452,"ruk":315,"rus":423,"ry ":194,"rse":228,"Nya":144,"rua":234,"rt ":160,"ru ":273,"sab":458,"sac":139,"san":482,"sas":180,"sa ":5643,"Nov":242,"sha":1745,"sho":271,"she":240,"shi":5099,"si ":1365,"siw":355,"sia":608,"shw":458,"shu":187,"sis":157,"sin":881,"sil":283,"sim":158,"sik":319,"sey":212,"ser":175,"set":147,"Okt":259,"seh":319,"sen":4083,"sem":335,"spa":151,"son":242,"su ":198,"st ":167,"sko":136,"ska":599,"so ":134,"ssa":198,"ste":192,"sta":295,"sto":444,"sti":401,"str":197,"swa":181,"tai":280,"taj":233,"tak":462,"tal":339,"taa":220,"tab":242,"taw":344,"tat":292,"tar":668,"tao":3872,"tan":641,"tam":288,"te ":507,"ta ":6480,"pa ":765,"pat":4120,"pak":235,"pap":248,"pam":300,"pan":895,"pi ":233,"ped":156,"Pap":368,"pia":789,"pil":189,"pin":267,"pis":162,"pit":144,"po ":743,"pte":287,"pri":298,"pwa":189,"Rai":176,"ra ":1932,"ngo":958,"ngi":1065,"ngu":1084,"ngw":363,"ni ":18823,"Iri":209,"nge":937,"nga":1742,"Ita":147,"neo":505,"nes":161,"ng ":405,"nch":2504,"ne ":911,"ndu":263,"ndo":574,"ndi":1835,"nde":1085,"nda":1162,"nak":251,"nal":257,"nam":1855,"nan":221,"nao":1457,"nap":185,"nac":183,"nad":288,"naf":402,"nai":158,"naj":196,"nd ":409,"nat":353,"nas":439,"nay":454,"na ":15738,"Jan":271,"Jam":281,"nya":1379,"Jer":215,"nye":1338,"nyi":4239,"nus":133,"nua":282,"Jim":174,"Jin":277,"nti":403,"nta":151,"nte":177,"nsi":211,"nsa":4269,"nt ":232,"ns ":140,"nne":236,"no ":948,"nji":138,"nja":269,"Joh":134,"nia":4199,"nis":530,"ogo":593,"ois":1291,"oji":173,"oja":1149,"Jul":285,"Jun":259,"odo":288,"of ":150,"ofu":134,"ofa":3991,"oa ":3810,"oan":188,"oba":375,"nza":3817,"nzi":1111,"Kai":144,"Kag":175,"Kal":167,"Kan":354,"Kat":474,"Kas":372,"Kar":232,"Ken":632,"ozi":165,"Kis":329,"Kir":165,"Kit":204,"Kin":148,"Kib":138,"Kia":309,"ote":378,"Kik":287,"Kil":453,"Kim":202,"oto":331,"Kig":295,"Kii":249,"ost":309,"ota":195,"ove":320,"opo":325,"os ":178,"or ":161,"Kon":197,"orn":300,"oro":673,"ore":188,"ori":369,"ort":147,"ora":378,"ola":427,"on ":838,"oli":431,"ole":357,"olo":331,"oka":1580,"oke":163,"oko":236,"oku":141,"ona":230,"ond":383,"one":151,"ong":860,"oni":784,"oma":766,"omb":303,"omi":249,"omo":182,"op ":143,"la ":8089,"le ":1011,"Kwa":3975,"laa":157,"lai":293,"lak":564,"lan":660,"lam":497,"lat":186,"lay":3727,"Kus":393,"lba":165,"kuz":236,"kuw":2713,"kuu":1305,"kut":1795,"kus":492,"kur":190,"kup":186,"kun":409,"kum":210,"kul":297,"kuj":187,"kwe":591,"kwa":2512,"kub":762,"kuf":233,"kuh":134,"kua":620,"kto":308,"lom":136,"loj":136,"lme":241,"Lin":225,"lug":350,"lu ":155,"li ":2787,"lez":192,"lew":193,"lev":140,"les":155,"leo":178,"lem":198,"len":254,"lek":133,"lo ":347,"lla":138,"lle":153,"lli":198,"ll ":147,"lit":241,"lis":337,"lip":257,"lio":738,"lin":627,"lim":922,"liz":411,"liy":4415,"liw":979,"lic":340,"lia":1497,"lik":2742,"lil":529,"lih":179,"lif":397,"ma ":2611,"mb ":139,"maa":449,"maj":397,"mak":522,"mad":206,"mae":140,"mag":342,"mar":439,"mas":613,"mal":159,"mam":161,"man":1055,"mat":406,"mba":3047,"mbi":361,"mbe":389,"mbo":2343,"me ":516,"mbu":267,"mch":170,"met":211,"mer":252,"men":492,"mfa":152,"mez":387,"mfu":373,"Mei":250,"Man":216,"Mar":1940,"Mas":472,"Mag":282,"Mak":206,"Mac":287,"Mbe":273,"mpi":142,"mon":163,"moj":1127,"mpa":160,"Mor":279,"mu ":1602,"mtu":175,"mto":226,"Mic":182,"Mis":147,"msh":144,"mta":228,"mwe":383,"mwi":345,"Mko":3178,"mwa":6205,"Mku":138,"Mji":464,"muj":3839,"muz":374,"mhu":232,"Mtw":147,"mi ":359,"mji":5175,"min":192,"mil":749,"Mwa":460,"mit":295,"mia":630,"mik":321,"mo ":4413,"mku":1038,"mko":539,"mna":1501,"mmo":145,"Wik":149,"Wil":3077,"Wan":148,"zwa":252,"zi ":5785,"zai":249,"zaj":254,"zam":177,"zan":3194,"zal":783,"zar":173,"zo ":612,"zia":533,"zin":815,"zil":197,"zik":548,"zis":240,"一":303,"yof":3874,"yot":286,"za ":2981,"ye ":1320,"yen":237,"ya ":21762,"yar":252,"yan":567,"yao":167,"yam":250,"yak":657,"yo ":973,"yin":213,"yik":3954,"一一":144,"Tan":3407,"Tab":164,"Shi":315,"Sin":201,"Sep":283,"we ":401,"wez":265,"wen":1037,"wim":286,"wil":741,"Sal":197,"vyo":257,"wa ":33121,"wap":4111,"wan":3901,"wal":617,"wam":169,"wak":9923,"way":141,"wat":368,"war":238,"was":172,"wai":2667,"wah":176,"vu ":165,"vya":351,"vil":200,"vin":183,"vit":187,"vis":284,"Rom":180,"vem":244,"Vij":328,"uzi":743,"uza":470,"Uje":235,"uwa":2760,"uvu":174,"ush":417,"usi":1319,"use":183,"usa":176,"uu ":2892,"usu":216,"ust":207,"uso":141,"uti":211,"ute":137,"uta":560,"Uin":218,"utu":215,"uto":1436,"us ":536,"Ung":252,"ura":183,"ure":140,"uri":491,"uru":630,"unz":137,"Ula":150,"upa":554,"upi":311,"umu":162,"umi":484,"umo":2705,"uma":686,"umb":661,"ume":297,"uo ":238,"uni":940,"und":747,"una":1741,"ung":1193,"uku":302,"uko":457,"uki":429,"uka":247,"ulu":258,"uli":1405,"ule":192,"ula":478,"ukw":139,"uhu":267,"uji":4010,"uja":302,"Utu":261,"ugh":514,"ufu":352,"uhi":136,"ugu":137,"udi":174,"ubw":695,"uch":343,"ufa":176,"ufi":189,"ua ":369,"uat":317,"uar":494,"uan":690,"uba":185,"Uch":175,"ty ":146,"twa":450,"tur":369,"tun":270,"tum":424,"Ufa":219,"ts ":214,"tu ":896,"The":164,"tts":142,"to ":986,"tob":268,"tom":167,"ton":281,"tok":1553,"tol":482,"tor":246,"tik":8147,"tis":158,"tin":351,"tio":199,"thu":171,"tia":156,"tem":384,"ten":273,"tel":171,"th ":160,"ter":432,"ti ":2389,"the":225,"thi":213,"biw":209,"bis":191,"bil":315,"bin":256,"bo ":2326,"bli":173,"bor":262,"be ":229,"bam":230,"ban":516,"bal":619,"bah":147,"baa":227,"bab":179,"bay":333,"bar":432,"bao":277,"bi ":662,"ber":216,"bel":151,"bey":251,"bia":222,"ce ":176,"bu ":4649,"bru":221,"bur":149,"bun":177,"bwa":786,"aka":10583,"am ":337,"ake":1982,"aki":644,"aji":1355,"aju":170,"al ":304,"aja":293,"ain":393,"air":222,"ais":2933,"aif":267,"aid":437,"ahi":308,"aha":751,"agh":475,"agu":395,"aoi":1233,"anu":344,"anz":4756,"any":4453,"ano":638,"ann":141,"ant":323,"ans":490,"ane":261,"ang":1660,"ani":7747,"anj":260,"ana":4702,"anc":133,"and":2300,"amu":1047,"amo":1890,"amp":179,"amh":222,"ami":838,"ame":637,"amb":1658,"ama":1868,"ao ":4649,"alo":269,"alm":262,"all":133,"ali":5324,"ale":476,"ala":1026,"alb":152,"an ":1167,"akr":376,"aku":502,"ako":215,"aba":751,"abe":140,"abi":660,"abo":208,"abu":582,"ae ":291,"aad":302,"aan":389,"aal":140,"aam":185,"aar":236,"aa ":361,"afi":303,"ai ":477,"aga":223,"age":227,"afu":225,"aen":162,"ael":172,"afa":411,"ado":269,"adh":288,"adi":1538,"ach":840,"ada":637,"azo":205,"azi":5401,"aza":186,"ayo":638,"aya":4140,"aye":284,"ba ":2178,"are":1998,"ard":317,"ara":2057,"aro":249,"ari":3153,"aru":316,"art":243,"au ":993,"asa":1084,"asi":1169,"ash":895,"ask":665,"ar ":568,"apa":4869,"api":162,"apo":406,"as ":271,"aut":148,"awa":1126,"awi":190,"ata":10070,"ast":167,"ass":197,"ato":634,"ate":225,"ati":9962,"ath":135,"atu":749},"n_words":[1316698,1560317,1165243],"name":"sw","type":"latin"} \ No newline at end of file
diff --git a/contrib/languages-data/tr.json b/contrib/languages-data/tr.json
new file mode 100644
index 0000000..5b8f9e4
--- /dev/null
+++ b/contrib/languages-data/tr.json
@@ -0,0 +1 @@
+{"freq":{"D":20468,"E":16965,"F":15212,"G":16978,"A":46399,"B":33447,"C":16597,"L":11614,"M":29609,"N":10463,"O":12233,"H":17902,"I":6437,"J":5964,"K":30894,"U":5006,"T":27864,"W":5022,"V":7330,"P":19836,"S":32251,"R":13212,"Y":12032,"Z":2430,"f":59766,"g":101779,"d":361700,"e":798707,"b":176889,"c":86462,"a":1029574,"n":718304,"o":261031,"l":623439,"m":275808,"j":10372,"k":334154,"h":85819,"i":785261,"w":7611,"v":106213,"u":275921,"t":334795,"s":315971,"r":646028,"q":1181,"p":80800,"z":105754,"y":255623,"x":3410,"²":1600,"Ç":4715,"Ãœ":2583,"Ö":3499,"î":1753,"é":1705,"ç":69086,"â":3605,"ü":157542,"ö":58180,"ÄŸ":69432,"ı":399538,"Ä°":20804,"ÅŸ":111613,"Åž":5518," l":7220," m":26771," n":32386," o":55007," h":21337," i":59190," k":78855," d":82641," e":36355," f":17358," g":42591,"Ñ€":1499," a":67898,"Ñ":1190," b":117978,"Ñ‚":957," c":5691," y":76657," z":4398," u":11497," t":53835," v":55379," p":15531," s":57822," r":8817," J":5925," K":30750," H":17802," I":6388," N":10386," O":12150," L":11515," M":29458," B":33275," C":16419," A":46229," F":15133," G":16827," D":20328," E":16897,"л":1101," Z":2402,"к":1097," Y":11995,"и":2029,"о":2304,"н":1554," S":31973,"в":1191," R":13120," P":19696,"а":2550," W":4958," V":7287,"е":1678," U":4989," T":27682," ç":18143," ö":14326," ü":15280," Ç":4706," Ö":3487," Ãœ":2581," ı":6625," Ä°":20776," ÅŸ":10933," Åž":5509,"ÙŠ":1130,"Ù„":1185,"Ù†":965,"ا":2068,"A ":2665,"Da":3349,"Cu":2398,"Co":2920,"Ce":1839,"Ch":2248,"Do":2641,"De":6139,"Di":2765,"Fe":2199,"Fa":2297,"Ey":1528,"Er":2288,"Es":1409,"En":2354,"El":1347,"Ek":1512,"AÄŸ":1411,"Ge":3246,"Ga":2711,"Bü":1152,"I ":2251,"Bö":1239,"Fr":4109,"Fo":1223,"Fi":2407,"B ":1016,"C ":1464,"Av":3034,"Ar":5311,"At":1678,"As":1760,"D ":3341,"Ba":8245,"Ay":1472,"Af":1012,"Ab":1087,"Ad":2089,"Am":3467,"An":5110,"Ak":1709,"Al":8286,"Bu":5544,"Br":2172,"Ca":3556,"Bi":5816,"Be":4617,"Bo":2818,"Ku":5039,"Gö":1216,"Kr":1520,"Ko":3871,"Le":1961,"Li":3051,"Gü":3195,"La":2872,"Lo":1867,"Me":5154,"Mi":3881,"Ma":11016,"Mu":2388,"Mo":2807,"Ni":2456,"Ne":2578,"Na":1799,"P ":982,"No":1828,"Ok":1001,"Ol":1014,"Oc":1091,"Gi":1074,"Gr":2260,"Go":1145,"Ha":7669,"He":2472,"Hi":1980,"Ho":2868,"Dü":2155,"In":1300,"Ja":2318,"Jo":1650,"Ka":10073,"M ":1217,"Ki":1749,"Ke":1920,"Ul":1131,"Tu":1634,"Tr":1523,"To":2482,"Th":2500,"Ti":1452,"Te":4160,"Ta":4376,"V ":1033,"St":3068,"Su":2048,"Wi":1541,"Wa":1156,"Vi":2393,"Va":1979,"Ve":1434,"Pr":2222,"S ":1244,"Pe":2037,"Pa":5059,"Kü":1314,"Po":6260,"Pi":1406,"Os":2243,"Or":2331,"Kö":1204,"Se":4094,"Sc":1963,"Si":3179,"Sh":999,"Sp":940,"So":3019,"Ru":3002,"Sa":7772,"Re":2740,"Ri":1066,"Ro":3278,"Kı":2617,"Ra":1699,"Mü":1802,"b ":1498,"a ":176902,"Ye":1685,"Tü":7528,"Ya":4483,"Yo":1238,"Yu":2386,"bö":7750,"i ":134793,"ge":29308,"aÄŸ":18792,"bü":7597,"ga":7799,"fl":1604,"ff":1053,"fi":9181,"fr":3323,"fu":6151,"ft":2020,"fo":3898,"he":11849,"ha":24248,"cü":3165,"gl":1385,"gi":16689,"gh":1246,"gu":5867,"gr":6009,"go":2702,"du":14570,"dy":2265,"g ":6921,"ea":4032,"eb":5387,"ec":6696,"ed":29273,"de":95875,"dd":2946,"di":58969,"dl":2907,"do":12268,"dr":3254,"ew":1275,"eu":1087,"ev":14605,"ey":30087,"ez":8326,"fa":8497,"h ":5851,"fe":6028,"eh":5944,"eg":3973,"ef":3639,"ee":1898,"el":56887,"ek":41993,"aç":5873,"ei":4450,"ep":3432,"eo":3151,"en":100462,"em":27716,"et":47279,"es":52024,"er":131153,"ca":18661,"e ":189222,"br":3182,"bu":23594,"bo":7205,"bl":1863,"bi":65897,"be":19027,"da":97587,"f ":5395,"cu":6717,"ct":1404,"cr":1803,"co":3001,"ck":3008,"ci":12969,"ch":6435,"ce":16707,"c ":1516,"az":19018,"ay":42685,"ba":34644,"d ":11781,"at":38962,"as":49850,"ar":157561,"av":10941,"au":2292,"ak":57066,"al":73836,"ai":6322,"aj":1917,"ap":20439,"am":37584,"an":184545,"ac":10260,"ad":37380,"aa":2080,"ab":13369,"ag":3250,"ah":16357,"ae":2002,"af":13702,"nu":18358,"nt":17557,"ns":13849,"nr":3598,"no":7521,"nn":3019,"nz":1833,"ny":12611,"hı":1036,"iÄŸ":11558,"of":3409,"oc":3225,"od":4927,"ob":3075,"om":13542,"on":43168,"ok":10977,"kç":3190,"ol":59486,"oi":1195,"oj":3578,"og":3689,"oh":1112,"m²":1589,"ot":7864,"os":10639,"ov":4695,"ou":3530,"op":8235,"oo":1893,"or":32678,"r ":170339,"ow":1477,"oz":2422,"kö":4303,"oy":8048,"pe":6240,"kü":6018,"pa":15264,"pl":7151,"pm":1729,"po":6918,"ph":1672,"pi":5799,"lç":5891,"lo":10260,"lm":31640,"ll":28962,"ls":3083,"fı":9951,"lu":32826,"lt":9362,"ly":5619,"o ":14289,"mc":2028,"md":5246,"hü":1497,"ma":67090,"mb":3028,"eÅŸ":11455,"mh":2240,"me":45214,"iç":12229,"ml":15462,"mi":33126,"mm":2739,"mp":4822,"mo":6599,"ms":3576,"mu":11366,"gı":2023,"my":1356,"p ":11912,"na":47407,"nb":3039,"nc":18538,"nd":102802,"ne":51401,"nf":1257,"ng":12672,"ni":53546,"nk":5100,"nl":28972,"nm":10641,"dı":37931,"ki":41309,"eÄŸ":6350,"ke":25138,"ka":45970,"m ":32916,"ky":1403,"gö":9935,"ks":8093,"kt":21304,"ku":22990,"ko":13147,"kr":3294,"kk":1610,"kl":25669,"km":5012,"kn":1431,"li":71923,"lk":10655,"le":122046,"ld":15695,"lg":12154,"lf":964,"la":144587,"gü":8845,"lc":1649,"lb":3731,"n ":243619,"hr":4105,"bı":1392,"ht":2266,"hu":4538,"hi":16646,"hn":1197,"ho":2852,"hl":3714,"hm":1584,"id":21397,"ic":9431,"ib":5859,"dü":13708,"ia":8374,"ih":7127,"ig":4782,"aÅŸ":24105,"if":4811,"ie":4567,"dö":3928,"k ":83031,"ir":107452,"is":40125,"it":22698,"iu":1112,"iv":5224,"cı":8286,"ii":1103,"ij":1085,"ik":40360,"eç":4865,"il":86504,"im":27783,"in":138518,"io":5300,"ip":8100,"je":1713,"ji":4659,"iz":21969,"iy":28989,"l ":42232,"ja":1035,"tç":1871,"ÄŸlu":963,"rı":43249,"sö":2312,"z ":18621,"sü":4852,"oÅŸ":947,"wi":1830,"sç":1475,"pı":8243,"vv":1012,"vy":1154,"y ":15418,"wa":1412,"rü":8173,"ÄŸla":2983,"vl":4067,"rç":3762,"vi":9343,"nÅŸ":1036,"vu":2607,"vr":5677,"vo":1424,"uz":10570,"uy":4807,"uv":2249,"ve":59460,"va":15490,"x ":1999,"ui":1273,"uk":6582,"ul":36937,"ue":3947,"oÄŸ":10206,"ug":2004,"ÄŸit":1160,"uh":1861,"ur":35478,"ÄŸin":3622,"us":21592,"ut":7622,"um":12243,"un":39859,"up":6378,"ty":1262,"tu":11035,"tt":7164,"nı":44489,"ub":5081,"ua":1900,"ud":5831,"uc":4903,"w ":1459,"to":17485,"tm":6406,"tl":13327,"ts":2378,"tr":10162,"te":52587,"pç":1206,"tk":2785,"ti":53158,"th":4577,"v ":2143,"tb":2499,"ta":68248,"su":11906,"sv":930,"ss":3728,"st":34655,"sy":6610,"mı":21024,"sw":1258,"sl":8558,"sk":7111,"sn":1024,"sm":5546,"sp":5003,"so":11037,"sc":1810,"se":25089,"sh":2001,"si":63532,"rz":1185,"u ":42818,"sa":39618,"nü":10410,"rr":2777,"rs":8400,"rt":19992,"ru":21036,"rv":1617,"lı":53226,"ry":3722,"rp":1215,"ro":18824,"rn":5785,"rm":14124,"rl":25348,"rk":23839,"nç":1698,"rj":994,"ri":89903,"rh":1272,"rg":7962,"rf":1575,"re":45975,"rd":23811,"rc":3479,"rb":3537,"mü":12609,"ÄŸi ":7346,"ra":89176,"t ":36013,"kı":16796,"ÄŸer":2592,"iÅŸ":22142,"lü":12025,"s ":31408,"pt":3720,"pu":1218,"pr":5018,"ps":1980,"ÄŸun":2143,"ÄŸus":1809,"ÄŸum":954,"zı":6859,"zü":1410,"yı":27954,"yü":11063,"ÄŸu ":5997,"zö":2165,"ÄŸlı":7152,"ÄŸre":1586,"uÅŸ":11784,"yâ":1423,"yö":4435,"ÄŸiÅŸ":1548,"uÄŸ":6358,"zg":2173,"rÅŸ":2653,"zi":13389,"zc":5363,"zd":2307,"ze":21253,"za":12673,"tı":26503,"zy":1958,"zu":2578,"zo":2918,"zm":3788,"zl":5972,"yg":2621,"ye":34397,"yd":5178,"tü":11302,"ya":90167,"yb":1899,"tö":1628,"sı":43368,"yv":967,"yu":8007,"ys":1270,"yr":3168,"yo":15057,"yn":5330,"ym":1117,"yl":13689,"yk":1018,"uç":1551,"yi":7908,"² ":1587,"ğı ":9316,"ğın":3141,"Çi":1350,"Ça":1454,"âl":1634,"Ãœn":1471,"Öz":1299,"çı":7148,"çü":4634,"î ":1515,"ço":5321,"çm":1158,"çl":3470,"çi":13394,"çe":14606,"ça":9969,"ç ":5469,"üş":3641,"üğ":1561,"üç":3404,"ün":27564,"üm":12601,"ül":12799,"üs":5069,"ür":27019,"üp":974,"üt":2907,"üz":15474,"üy":5418,"üc":2767,"üf":4960,"üd":3476,"öğ":1623,"ük":10837,"ü ":14372,"öy":3363,"öz":7289,"ör":11792,"ös":1882,"ön":12083,"öl":13407,"ök":2015,"ÄŸ ":1364,"ÄŸu":12250,"ÄŸr":3621,"ÄŸe":3325,"ÄŸd":979,"ÄŸa":3324,"ÄŸl":11320,"ÄŸi":14714,"ğı":15723,"ğü":1540,"ığ":10531,"ış":19918,"ı ":102139,"Ä°n":4248,"Ä°m":1089,"Ä°l":1863,"ın":99943,"ım":16215,"ıp":1333,"ıl":42504,"ık":17416,"ıf":1477,"ıb":1069,"ıd":8898,"ıc":3354,"Ä°t":1484,"Ä°r":962,"Ä°s":7391,"ıy":7698,"ız":6701,"ıs":11942,"ıt":2777,"ır":43940,"ÅŸt":19651,"ÅŸu":2786,"ÅŸi":12692,"ÅŸl":8667,"ÅŸk":4781,"ÅŸm":7053,"ÅŸa":13302,"ÅŸe":7623,"Åžu":1166,"Åža":2212,"Åže":1159,"ÅŸ ":18890,"şı":10920,"şü":1419," Ga":2694," Bü":1149," AÄŸ":1410," Ge":3227," Bö":1239," Fo":1207," Fr":4097," Fi":2393," Ha":7643," He":2466," Go":1137," Gr":2225," Gi":1067," Dü":2153," Ho":2861," Hi":1971," Ja":2310," In":1293," Ka":10047," Ke":1904," Ki":1728," Jo":1635," Gü":3188," La":2844," Le":1944," Li":3034," Ko":3863," Kr":1518," Ku":5027," Gö":1215," Ma":10963," Mi":3862," Me":5127," Lo":1852," Ne":2557," Na":1779," Ni":2442," Mo":2790," Mu":2369," Am":3459," An":5094," Ak":1699," Al":8247," Af":1009," Ad":2080," Ab":1075," Ba":8219," Ay":1468," Av":3029," At":1662," As":1754," Ar":5292," Be":4599," Bi":5804," Bo":2788," Br":2154," Bu":5532," Ca":3515," Ce":1828," Ch":2228," Co":2886," Cu":2385," Da":3328," Di":2745," De":6126," Do":2600," El":1340," Ek":1512," Es":1407," Er":2282," En":2338," Ey":1526," Fe":2195," Fa":2283," Wi":1521," Wa":1142," Yu":2383," Yo":1225," Tü":7497," Ya":4478," Ye":1681," a ":1377," Kö":1204," Os":2235," Or":2329," Po":6216," Pi":1404," Pe":2029," Pa":5030," Kü":1311," No":1821," Ol":1012," Ok":999," Oc":1088," Ra":1684," Mü":1801," Kı":2610," Ro":3268," Re":2723," Ri":1059," Pr":2202," Su":2042," St":2970," Ta":4359," Th":2488," Ti":1443," Te":4145," Tr":1513," To":2461," Ru":2999," Sa":7750," Sh":981," Si":3171," Sc":1940," Se":4078," So":2983," Va":1973," Ve":1421," Vi":2383," Tu":1612," Ul":1129," im":933," in":10201," ik":4076," il":20054," is":5383," ka":23639," ki":5592," ke":6355," eÄŸ":1244," dı":1534," ha":11824," he":4045," gi":4830," gr":3277," dö":3823," dü":5597," id":1007," aÅŸ":930," hi":2120," ni":6427," nd":4080," ne":3117," na":1507," mu":1078," mo":2617," ok":1391," ol":35321," on":2426," of":1496," nu":1915," no":1087," le":1165," li":3624," gü":6331," la":1347," ku":18460," gö":9671," km":3282," ko":8297," me":8098," mi":3110," iç":9390," hü":1380," ma":6917," ad":10374," am":2886," an":8617," ai":1874," ak":2362," al":14807," ar":9911," at":1881," as":2441," d ":1776," ba":23010," ay":4744," bi":52706," be":11424," bo":2633," bu":15855," ca":1538," e ":1478," er":1285," et":4327," es":3101," en":6931," el":3328," ek":1681," aç":1971," fe":1277," fa":4483," ey":3388," ev":1229," fu":1090," fr":1901," fo":1846," fi":5962," aÄŸ":1494," ge":16462," bü":4020," ga":1355," i ":957," bö":7468," ci":1625," da":23947," do":7108," de":26605," di":9385," ed":6003," du":2416," za":2588," yo":2330," sı":5340," ye":11522," tü":4969," ya":37365," sö":2004," sü":3398," yı":13517," yü":5685," yö":4365," nü":5050," sa":17392," se":8759," si":6463," sp":996," so":7754," kı":5201," ra":1785," mü":4235," re":3088," ro":2639," pr":3082," iÅŸ":3101," ot":2226," or":5403," oy":4366," kö":3581," pe":1612," kü":2525," pa":4625," pl":965," po":2333," pi":1501," va":2885," ve":49979," uz":2734," uy":2267," vi":955," nı":8640," tu":1641," un":2824," ul":1616," ta":27334," st":1812," su":2833," to":3720," th":1268," ti":1727," te":11491," Ä°t":1484," Ä°s":7380," Ä°r":961," ın":4768," Ä°n":4243," Ä°l":1857," Ä°m":1089," Çi":1348," Ça":1452," Öz":1296," Ãœn":1468," çe":4156," ça":3855," ço":4379," çi":1440," çı":3500," öl":1632," ön":4253," ör":1000," öz":4179," ür":2793," üs":1211," ün":2173," ül":2545," üz":3783," öğ":1347," üç":1563," ÅŸi":1684," ÅŸe":4952," ÅŸa":3567," Åža":2210," Åže":1156," Åžu":1166,"Ä°st":2249,"Ä°ta":1353,"Ä°sp":2587,"Ä°ng":3329,"Ä°mp":936,"ıca":1274,"ıda":1227,"ılı":15802,"ısa":1923,"ıra":2874,"ırl":2487,"ırm":2227,"ını":15301,"ımı":5151,"ıya":1118,"ırı":3492,"ız ":2539,"ıl ":1920,"ıdı":7570,"ık ":8525,"ıcı":2074,"ıla":12183,"ın ":32401,"ıld":3344,"ılm":5978,"ıll":2939,"ım ":3853,"ıka":2439,"ıkl":3108,"ıkt":1193,"ınl":2820,"ıp ":1063,"ınd":38554,"ına":8579,"ınm":927,"ıma":1331,"ımc":938,"ıml":3642,"ıs ":2159,"ır ":29899,"ıyı":1283,"ızı":1437,"ısı":6837,"ıyl":4411,"ızl":1239,"Çin":1025,"ığı":10132,"ış ":6594,"ışt":5588,"ışa":989,"ışm":2843,"ışı":2617,"Fil":1047,"Eyl":1137,"Eki":1120,"End":927,"AÄŸu":1069,"Gen":1382,"Gal":926,"Böl":1142,"Fra":3208,"II ":1197,"Haz":1331,"Her":938,"Hal":1040,"Har":940,"Dün":1753,"Hol":1623,"Ara":2840,"Avr":1832,"Bar":1214,"Bat":956,"BD ":2271,"Ada":1186,"Alm":3781,"Ame":2710,"Ana":1277,"Ant":1215,"Bu ":2451,"şıl":1169,"şık":1556,"şın":2204,"şı ":3074,"Bel":1045,"Bil":1105,"BaÅŸ":989,"Bir":3328,"Cum":2030,"Dev":1876,"Den":1321,"Cha":1022,"DoÄŸ":1141,"Nis":1117,"Oca":1015,"Ort":943,"Osm":1787,"Par":1928,"Pro":985,"Por":4336,"Kan":1554,"Kas":1322,"Kar":2603,"Kon":1226,"Kra":1061,"Kur":1360,"Kuz":1311,"Gün":1851,"Mer":1131,"Man":1052,"Mar":3200,"May":1678,"Mil":1269,"çok":3674,"çeÅŸ":1324,"çla":1690,"çim":1273,"çin":7152,"Yun":1817,"Tür":7269,"Yar":1131,"Sta":1148,"Tem":1343,"Rus":2328,"Sch":1519,"Sav":1634,"San":1732,"Rom":1453,"Ulu":930,"The":1923,"ça ":3264,"çal":2740,"çe ":2241,"çes":2357,"çer":2328,"çev":1541,"çek":2799,"çi ":1341,"biy":1472,"bit":1248,"bir":44645,"bil":11436,"bin":1380,"baÅŸ":7621,"ÅŸa ":1752,"bol":2761,"ÅŸam":1786,"ÅŸan":2545,"ÅŸar":3761,"boy":1344,"ban":1510,"bak":1768,"bal":1557,"baz":1091,"bay":1165,"azı":4805,"bat":3330,"bas":2581,"bar":2153,"Åžub":979,"bi ":3364,"ber":3678,"ben":1194,"bel":7483,"bes":1722,"baÄŸ":8489,"ÅŸla":4267,"ÅŸle":3059,"ÅŸma":4426,"ÅŸme":1364,"ca ":7674,"car":1245,"can":1925,"cak":4366,"ce ":9532,"bra":943,"bu ":5377,"ÅŸek":1911,"ÅŸeh":2781,"ÅŸi ":1181,"bur":1706,"bul":11239,"bun":1530,"buc":1673,"ÅŸid":1048,"ÅŸit":1478,"ÅŸir":1286,"ÅŸin":976,"ÅŸim":2323,"ÅŸil":1127,"ÅŸik":1949,"ÅŸki":1113,"ÅŸke":1220,"ÅŸka":1851,"aka":3723,"am ":5523,"aki":6742,"adı":13553,"al ":11243,"ail":1471,"air":932,"ait":1247,"acı":3398,"ak ":21809,"abı":1039,"ahi":5926,"aha":4061,"anu":1353,"any":6219,"ano":1022,"ann":1052,"anm":4730,"ant":4742,"ans":6131,"ane":2508,"ang":2190,"ani":4880,"ank":1605,"anl":12840,"ap ":1282,"ana":13099,"anb":2176,"anc":4224,"and":10443,"aml":2039,"amp":1789,"ami":2705,"ame":1745,"ama":13353,"aly":2496,"afı":8781,"alt":4529,"alm":2361,"all":3746,"alk":2064,"ali":6616,"ald":2079,"ale":5493,"ala":14214,"alb":2602,"an ":84156,"aks":970,"akt":8868,"akk":924,"akl":5036,"aba":3739,"abe":1525,"abi":3498,"abu":1168,"ae ":1012,"aca":2693,"ad ":1275,"ÅŸtu":4101,"ÅŸti":6965,"afi":1276,"ÅŸtı":7023,"ah ":1276,"ado":1356,"adl":2298,"adi":1602,"add":1987,"ade":4140,"ady":944,"ada":8970,"azi":2552,"azl":1262,"atı":8461,"aze":1261,"arÅŸ":2391,"aza":4710,"az ":2571,"ayn":2937,"ayl":1883,"ayr":2190,"ası":27693,"arı":35962,"aya":12842,"ayd":1055,"aye":1092,"âle":1449,"ba ":1088,"ayı":12386,"akı":5066,"at ":6328,"are":4111,"ard":9709,"arc":1066,"ara":45240,"arm":1435,"arl":5899,"ark":6902,"ari":9836,"alı":12304,"ars":1642,"art":5112,"asa":4734,"ary":1104,"asi":2326,"ask":1950,"ar ":19629,"apa":2097,"apm":1588,"apl":1156,"apo":1295,"apt":1792,"as ":3211,"ava":5024,"avr":943,"arç":1233,"avi":1237,"ay ":3414,"avu":1433,"apı":7957,"ata":3117,"ast":3088,"asy":2788,"amı":5927,"atm":1041,"apç":991,"atl":2228,"atr":1379,"ato":2437,"ate":2841,"ati":5438,"att":1422,"anı":18491,"Ãœni":1310,"Åžar":1029,"ji ":1449,"jis":950,"itl":1849,"öğr":1364,"ito":956,"cı ":3493,"ism":1455,"ist":11668,"ita":3780,"ite":3995,"iti":3699,"cıl":1667,"üfu":4682,"iva":1157,"ive":2871,"ilç":2129,"is ":3578,"ion":3161,"ir ":71671,"irm":3093,"irk":1523,"irl":6644,"iri":12312,"isi":13092,"ise":2824,"isa":3166,"ire":3266,"ira":2633,"ird":1851,"it ":3783,"ünl":1704,"üni":930,"ünc":1623,"ünd":2862,"üne":6398,"üml":1023,"üme":1310,"ült":1479,"ür ":5331,"üny":3730,"iyi":1021,"ül ":1714,"iyl":2890,"iyo":3774,"iya":6327,"iye":14035,"üdü":2475,"ük ":5512,"cıy":983,"iz ":6631,"üle":2031,"ülk":2638,"üll":944,"üks":1988,"ün ":4631,"izm":2299,"izl":1160,"izi":3733,"izc":3709,"ükl":1155,"iza":1126,"üm ":2100,"kim":2642,"kil":4381,"kiy":3334,"kiz":4406,"kin":3419,"kis":1066,"kit":1608,"km ":1709,"ki ":15464,"eÄŸe":1107,"eÄŸi":4944,"kel":2350,"ken":7944,"kes":1806,"ker":2097,"ket":3347,"kez":3142,"ke ":1164,"kra":1708,"kiÅŸ":2390,"kse":2982,"klı":3118,"km²":1573,"kor":1133,"kon":4434,"kom":1557,"kol":1873,"kle":9482,"kla":8235,"kli":3448,"dız":949,"dıy":1257,"dır":19814,"dın":2440,"dı ":5738,"kaz":1358,"kay":2732,"kat":3255,"kar":8976,"kas":2234,"kap":1914,"kan":7174,"kal":4961,"kam":982,"kad":3577,"kab":1637,"dış":1185,"ka ":4513,"dığ":4219,"ha ":3071,"cü ":1507,"ham":1081,"han":2727,"hak":1219,"hal":3743,"hav":968,"har":3475,"has":1125,"hat":1036,"haz":1057,"hay":1743,"he ":3168,"her":2799,"hen":1336,"hem":1060,"hi ":1444,"hip":4144,"hin":2793,"hil":1264,"hir":2302,"hle":2209,"ağı":5375,"gru":2866,"gra":2358,"gul":1221,"gue":2075,"dül":2427,"ian":1361,"dün":2104,"dür":3900,"ibi":3243,"düz":2369,"id ":1061,"iba":969,"ia ":4696,"aÅŸ ":1969,"ig ":1399,"ici":2058,"ich":1442,"ice":1186,"ie ":1088,"ica":1674,"idi":13477,"ide":3373,"ida":1813,"if ":1449,"düş":1411,"il ":5350,"im ":8509,"ika":5895,"aÅŸa":4663,"aÅŸl":3782,"aÅŸm":1272,"aÅŸk":2892,"aÅŸt":1881,"ihl":1333,"ihi":3044,"ik ":16892,"iml":3173,"imp":924,"ime":2281,"imd":1567,"imi":5215,"ip ":4274,"inc":5308,"ind":30574,"ina":3996,"aşı":5939,"ino":995,"int":1436,"ins":2898,"ine":16354,"ing":3240,"ini":20786,"inl":1777,"iko":990,"ikl":5637,"iki":4685,"eçi":1614,"eçe":1013,"ila":2643,"in ":47078,"ikt":2801,"ilo":1062,"ill":5588,"ilk":4218,"ilm":10420,"ilg":3153,"ili":20573,"ild":2510,"ile":24870,"ima":2176,"io ":955,"ily":1842,"hri":2688,"hur":2425,"dör":1115,"dön":2425,"fes":1014,"fer":1326,"far":1401,"eyâ":1363,"fa ":1026,"eyb":1488,"eya":8242,"eyl":966,"eyi":4871,"eyd":2007,"eye":3542,"ez ":1852,"ezo":1005,"ezi":2727,"eta":1696,"ete":2836,"eti":16248,"etm":3594,"etl":4019,"etk":1732,"est":3861,"ess":1296,"esw":1087,"ev ":1032,"etr":1981,"ett":2719,"eve":1133,"eva":954,"evl":3752,"erç":1693,"evi":3588,"evr":2587,"ey ":4964,"er ":28073,"es ":5122,"erk":5212,"erl":5910,"eri":48739,"erg":2108,"ere":9775,"erc":963,"erd":6317,"era":4204,"erb":1053,"et ":9016,"açı":2094,"esk":1828,"esl":1542,"esm":989,"esi":30715,"ese":2043,"esa":1128,"ert":1494,"ers":4580,"ern":2620,"erm":2340,"eki":11678,"ekl":5425,"açl":1635,"ekn":1222,"eko":1029,"eks":1790,"ekt":6067,"en ":46837,"ela":1161,"eld":2163,"ele":20479,"eli":10508,"elm":1130,"ell":5734,"els":966,"ema":2676,"eme":6814,"eml":3228,"emm":1228,"emi":6693,"ene":8543,"eng":1269,"ena":1544,"end":6506,"enc":1546,"enm":2094,"enk":1143,"enl":4600,"eni":13441,"ens":1925,"ent":5851,"enz":1180,"egu":1647,"ehr":2303,"ehi":2131,"ek ":9883,"aç ":1092,"ein":1910,"el ":11201,"eke":1777,"eka":935,"em ":2604,"öst":1805,"gis":1807,"gir":1360,"gil":5370,"önü":972,"geç":2375,"gin":1021,"gib":2269,"ört":1016,"öre":4482,"ölü":2891,"ölç":2720,"gi ":2386,"ör ":949,"gen":4928,"ger":3542,"ges":4599,"aÄŸa":1186,"gel":7731,"aÄŸl":9979,"önc":1623,"öne":7107,"ge ":2812,"aÄŸ ":1140,"gaz":1147,"ölg":6551,"gar":1277,"büy":3008,"büm":2701,"gan":1892,"böl":7461,"fus":4728,"fut":963,"öyü":1007,"fre":2029,"for":2226,"öze":3441,"özl":926,"örü":2439,"fil":4052,"da ":52350,"de ":46250,"dak":5123,"dal":2767,"dah":2619,"das":1177,"dar":3775,"dan":21526,"dam":1680,"day":1218,"dde":1533,"cul":955,"cus":1172,"cre":977,"cu ":1846,"ch ":1055,"ces":998,"cek":1367,"cel":1540,"ci ":4802,"ck ":1696,"che":1266,"chl":1164,"cil":1965,"cis":1558,"cin":1363,"ed ":1577,"ebe":1387,"ebi":2611,"efe":1034,"edi":16774,"ede":8053,"eda":1066,"eci":1467,"ece":3057,"dyo":1078,"dur":5671,"duÄŸ":3327,"don":972,"dol":1859,"dok":977,"diÄŸ":3682,"doÄŸ":5145,"dra":1187,"dlı":1288,"du ":2273,"daÄŸ":1454,"der":6369,"des":1911,"dev":2820,"ded":1153,"del":2710,"dek":4272,"den":20219,"dem":1663,"di ":6071,"dla":1190,"do ":1033,"diz":1879,"diy":5465,"din":3013,"dir":24487,"dis":2432,"dik":1433,"dil":6616,"deÄŸ":2872,"rga":1818,"ri ":19287,"rgi":1489,"rge":959,"ret":5334,"res":4290,"rev":1391,"rdu":2054,"rg ":1524,"rec":1008,"red":1084,"reg":1836,"ren":6966,"rek":4543,"rel":1942,"rda":4941,"rdi":3615,"rde":5880,"re ":11153,"ray":2290,"müz":3407,"rd ":1472,"rap":1757,"rar":2927,"ras":9820,"rat":4222,"rbi":1042,"rba":1205,"mün":3661,"ran":10600,"ram":4044,"ral":5875,"rak":15812,"rab":1433,"raf":10463,"rad":2346,"rac":1216,"rs ":1268,"ros":1512,"rot":967,"rom":2121,"ron":1936,"rol":2443,"rkç":1463,"rog":1323,"rna":1248,"rne":1926,"ro ":1843,"rma":6313,"rme":4031,"rmi":1407,"rlu":1118,"rli":4427,"rle":9208,"rla":6927,"rki":3194,"rkl":1461,"rke":5713,"rka":1543,"rdı":3751,"riy":3959,"rit":1940,"ris":5765,"rih":5101,"müş":1050,"raÅŸ":1079,"ril":8019,"rik":4914,"rin":23896,"rim":3407,"ria":1426,"rdü":1150,"ric":1498,"rid":4314,"rk ":5291,"lıl":999,"lık":8208,"lın":9443,"lım":1113,"lır":2906,"rya":1228,"rup":3018,"run":1284,"rum":2945,"rul":5547,"ry ":1364,"rsi":2903,"rsa":1025,"rta":4786,"rte":4662,"rti":1858,"lı ":21277,"rub":2412,"rt ":3885,"rkı":3548,"ru ":1610,"rlı":1952,"sab":1070,"sad":1069,"nüf":4498,"sah":5074,"sal":4864,"nüm":1062,"san":6337,"nün":1717,"sat":1316,"sar":2328,"say":5210,"sav":1790,"sa ":4002,"nü ":1409,"lış":2586,"lığ":4204,"si ":18237,"saÄŸ":1738,"siz":1136,"siy":3676,"sid":3408,"sia":1857,"sit":3495,"sis":2770,"sin":18799,"sil":2083,"sim":2963,"sik":1925,"se ":2953,"ser":3758,"ses":2131,"sen":2086,"sem":1040,"sel":4229,"sek":2392,"spo":1131,"spa":2896,"son":6601,"su ":5315,"st ":3445,"slu":1935,"sla":3121,"sle":1769,"ski":2726,"ske":1671,"sma":2231,"smi":1955,"mın":4103,"swi":1113,"stü":1161,"sya":2595,"syo":3350,"ste":9865,"sta":9233,"sto":2278,"sti":3825,"stl":934,"str":1911,"mı ":4565,"sun":2855,"tak":3619,"tal":5550,"tab":2400,"tad":4726,"tay":1623,"tat":1503,"tas":2835,"tar":17331,"tap":1242,"tan":13930,"tam":2113,"te ":6912,"tbo":1793,"ta ":6535,"mış":10404,"pa ":2055,"par":4696,"kül":1832,"küm":1078,"pan":3919,"per":2301,"küç":956,"pla":3634,"ple":961,"plu":1414,"lçe":2541,"piy":2262,"por":1907,"pon":1081,"pol":1613,"lçü":2547,"pti":1099,"pra":1013,"pro":2802,"ptı":1375,"lü ":3559,"lüm":2580,"lül":1275,"lük":1015,"iÅŸ ":5374,"iÅŸi":4832,"iÅŸk":1088,"iÅŸl":2771,"iÅŸt":5462,"kı ":2158,"kıl":1124,"kım":2874,"kıs":2851,"kın":2434,"kıy":1259,"ra ":9857,"mü ":1475,"ngi":4542,"ni ":12017,"nge":1612,"ncü":1344,"nel":4863,"nek":1315,"nen":2962,"nem":4576,"ner":2342,"net":4444,"nes":2204,"ng ":2974,"ned":1504,"ney":4102,"nci":4377,"nce":5236,"nca":4009,"ne ":19615,"nbu":2438,"ndu":1472,"ndr":1044,"ndo":1041,"ndi":6541,"nde":35967,"nda":48013,"ncu":1481,"nak":1647,"nal":3192,"nam":1052,"nan":13136,"nar":2141,"nad":2145,"nd ":2524,"nat":3318,"nas":1441,"nay":1104,"na ":14803,"muÅŸ":4297,"nya":10222,"nun":5903,"nus":1510,"nuc":968,"nto":981,"ntr":1143,"nti":4165,"nta":1967,"nte":3585,"nmı":2809,"nst":987,"nse":1596,"nsi":1251,"nsa":3394,"nu ":4463,"nlı":4882,"nra":2468,"nt ":2440,"niÅŸ":970,"ns ":1946,"nlü":1339,"nom":2114,"nne":1221,"nme":1944,"nma":4498,"nmi":973,"nli":2503,"nla":14247,"nle":4649,"nlu":1296,"nka":1369,"ndı":4155,"nic":987,"ndü":1181,"niy":1052,"niz":3806,"ncı":1170,"niv":2157,"nis":1976,"nir":2294,"nim":1125,"nin":20306,"nik":2175,"nil":944,"ogr":1717,"ok ":4093,"oji":2722,"ol ":4011,"ock":1051,"ode":1635,"of ":1645,"iÄŸi":9467,"iÄŸe":1929,"obi":987,"nsı":2063,"köy":1994,"oyu":4442,"oyn":1402,"oto":3517,"osy":1120,"ost":1210,"osu":1204,"ovi":1205,"ova":1326,"opl":2685,"os ":3304,"çıl":1025,"çık":4300,"or ":3792,"ork":1035,"orl":1782,"orm":2268,"ord":2039,"ore":1088,"org":1822,"ori":2004,"ort":8149,"oru":2784,"m² ":1578,"ora":1605,"ola":24963,"old":3573,"olc":925,"on ":12904,"oli":2380,"oll":2123,"ole":1383,"ols":1253,"olm":3895,"olo":4194,"olu":8661,"om ":1843,"kçe":1715,"okt":974,"oku":2095,"ona":2977,"ond":1625,"one":1566,"oni":1789,"onl":1840,"ono":2239,"onr":2464,"ons":1619,"ont":1468,"onu":7635,"ony":1563,"oma":3785,"ome":1724,"omi":1625,"omo":1197,"la ":11386,"le ":19263,"lde":3058,"ldi":2187,"ldu":3454,"lab":1412,"lac":926,"lad":1565,"lah":1109,"lak":1159,"gün":5583,"lan":43556,"lam":9121,"lar":57894,"lat":2773,"las":2417,"lay":4711,"ld ":958,"kuz":3055,"kur":7540,"kul":7562,"kta":6976,"kte":6402,"ksi":2396,"ktr":1368,"kti":2217,"gös":1720,"gör":5828,"ktı":2504,"lon":1166,"liÄŸ":4787,"loj":2682,"lmi":5563,"lme":4109,"leÅŸ":5292,"lma":11588,"lmu":3175,"lst":1073,"lmı":3998,"lta":1114,"lte":1773,"lu ":7438,"llı":1332,"liÅŸ":3210,"lt ":1120,"lge":7089,"lgi":3114,"li ":16772,"lbü":2715,"lga":999,"ley":2404,"lev":1310,"les":3725,"let":9791,"ler":45886,"lem":4975,"len":16099,"lek":2724,"led":5291,"lec":1154,"lo ":939,"lla":11633,"lle":6955,"lli":5594,"lke":3179,"lm ":1714,"ldı":4073,"ll ":1428,"lit":1562,"lis":4164,"lir":4755,"lin":9226,"lim":4866,"liz":3501,"liy":1446,"lid":1846,"lia":946,"lk ":5135,"lik":9172,"lil":1002,"laÅŸ":3620,"ma ":9869,"mac":2119,"mak":8947,"mad":3045,"mar":2763,"mas":9043,"mal":4481,"mam":1405,"man":15375,"may":3231,"mat":2471,"me ":6043,"mda":1303,"mde":1460,"mdi":1706,"med":2057,"eÅŸ ":1039,"met":4942,"mes":5862,"mer":6138,"mel":3135,"men":5922,"mek":5999,"maç":1261,"mey":2238,"çüm":2197,"çük":1139,"luk":1924,"lup":1601,"lun":8327,"lum":1008,"lus":2340,"fın":8889,"lya":3819,"luÄŸ":2080,"ltı":2446,"luÅŸ":5045,"mpi":1862,"mod":1329,"mon":1220,"mpa":1588,"mu ":1170,"miÅŸ":7571,"mun":1116,"muz":1291,"mhu":2129,"eÅŸm":1123,"eÅŸt":1717,"mi ":6405,"eÅŸi":5221,"min":6521,"mil":2577,"mir":1395,"mis":1028,"mcı":1046,"mit":1204,"mid":2070,"mik":1507,"mlu":936,"mli":2428,"mle":5174,"mla":5844,"içi":8066,"içe":2201,"mmu":1223,"uÄŸu":5772,"tı ":4071,"zun":1539,"tıl":3307,"tın":2882,"tır":10360,"tıs":1266,"zyo":962,"tığ":1565,"zi ":3100,"zet":1053,"zey":4798,"zen":2488,"zel":3272,"zer":6678,"ze ":1245,"zce":3650,"zde":1081,"zam":2429,"zan":2260,"zak":1080,"zar":3022,"zon":1273,"zme":978,"rşı":2081,"zla":2044,"zgü":956,"zle":1804,"zin":1617,"zik":2532,"zir":1408,"zis":1290,"yum":1152,"yun":4454,"sı ":14744,"ynı":1581,"ylü":1204,"yol":2904,"yor":1353,"yon":7554,"yrı":1788,"sıd":1928,"sıl":1373,"sım":1355,"sır":2328,"sın":16353,"sız":2708,"sıy":986,"ye ":8402,"yda":1138,"yed":1753,"yes":4583,"yer":7744,"yen":3683,"yel":1262,"yet":5659,"ya ":30890,"rış":2333,"yba":1101,"yaz":5444,"yay":4498,"yat":3352,"yar":6010,"tür":5785,"yas":4408,"yap":11061,"tün":1161,"yan":10579,"yal":5013,"tüm":1022,"yak":2748,"ydı":961,"yla":6744,"yle":4770,"yo ":1109,"yna":2824,"yi ":2354,"ygu":1552,"yin":3381,"yaÅŸ":2680,"tör":1578,"rı ":14655,"rım":1734,"rın":16497,"rıl":3423,"rıs":1452,"sür":2901,"söz":1499,"sça":1363,"wig":1137,"rü ":1492,"rün":2219,"rül":1328,"vru":1839,"vri":1244,"vre":1449,"vra":964,"pıl":3787,"pım":2266,"vil":989,"vaÅŸ":2523,"viz":960,"vis":1122,"rça":1249,"rçe":1701,"vle":3463,"vi ":1409,"vey":5886,"ver":8628,"vet":1190,"ven":1213,"ve ":40015,"val":1203,"van":2336,"var":3116,"va ":2329,"uzu":1436,"uze":4115,"uyu":1005,"uza":1667,"uyg":1403,"uya":1017,"uz ":2525,"usç":954,"uva":1017,"usl":3168,"usa":1561,"usu":5699,"ust":2940,"utb":1431,"us ":4706,"ut ":1302,"ura":2627,"urd":1061,"urg":1888,"uri":2897,"urm":935,"uro":982,"urt":1354,"uru":9050,"upa":2517,"ur ":10047,"umu":1897,"umh":2127,"uml":1645,"uma":1385,"unu":6201,"unl":2635,"unm":1881,"unc":2040,"und":5507,"una":8329,"up ":2721,"ukl":1910,"um ":2957,"ulu":12477,"ult":959,"ulm":2979,"ull":6976,"ula":6334,"un ":10772,"uk ":2310,"ul ":3821,"uha":1114,"ucu":1955,"udi":996,"ubu":2444,"uca":2065,"oÄŸu":5104,"oÄŸr":1470,"ues":1785,"udu":3053,"oÄŸa":1679,"oÄŸl":957,"uba":1499,"tur":6494,"nır":2256,"nıl":5540,"nın":19519,"nım":1872,"nıf":1036,"tre":1553,"tra":2671,"tri":1839,"tro":3501,"tte":1278,"tti":2653,"nı ":11130,"tme":3621,"tma":1695,"to ":1507,"tiÄŸ":1447,"tos":1656,"tom":1446,"ton":2840,"tol":991,"tor":3257,"top":3422,"til":2825,"tik":5330,"tif":1158,"taÅŸ":2105,"tir":10735,"tis":1905,"tin":8295,"tim":3897,"tio":1845,"tic":1549,"tid":1024,"tiy":1388,"tki":2141,"pça":1027,"tli":1970,"tla":3863,"tle":6323,"tem":6048,"ten":3191,"tei":1620,"tek":7857,"tel":3048,"ted":3509,"th ":1012,"tes":4105,"ter":10441,"ti ":6632,"the":1663,"üşü":1357,"üğü":1444,"zı ":1890,"zıl":2487,"üç ":948,"zöl":2079,"üçü":1824,"yı ":2418,"yım":2005,"yıl":14867,"yın":3653,"yıs":2728,"yük":5408,"yüz":3683,"yön":3982,"üzö":2079,"üyü":3989,"üzi":2596,"üze":7514,"üye":1004,"ütü":1249,"üsü":1036,"ürü":4447,"ürk":7688,"ürl":1211,"üre":5286,"ümü":6554,"ülü":2192,"üs ":1086,"üst":1769,"ünü":3990,"uÅŸ ":2901,"uÅŸu":1743,"uÅŸt":4122,"uÅŸa":1668,"yâl":1370},"n_words":[9192208,10449574,7620193],"name":"tr","type":"latin","flags":["diacritics"]}
diff --git a/contrib/languages-data/uk.json b/contrib/languages-data/uk.json
new file mode 100644
index 0000000..3a0cdfd
--- /dev/null
+++ b/contrib/languages-data/uk.json
@@ -0,0 +1 @@
+{"freq":{"D":2686,"E":45335,"F":1931,"G":2251,"A":4545,"B":2792,"C":4259,"L":2592,"M":3964,"N":23611,"O":1971,"H":1743,"I":29162,"T":3063,"V":2399,"P":3688,"S":26730,"R":2282,"X":2695,"f":3088,"g":5064,"d":7920,"e":26938,"b":4218,"c":9825,"a":26142,"n":18042,"o":19695,"l":12837,"m":7697,"k":2703,"h":7896,"i":22805,"w":2062,"v":2426,"u":10520,"t":16747,"s":15016,"r":19222,"p":6493,"y":4268,"x":1842,"Ì":45988,"ÑŒ":314859,"ÑŽ":121712,"Ñ":259056,"ш":55425,"щ":34352,"Ñ„":70318,"Ñ…":152145,"ц":190380,"ч":157844,"Ñ€":767814,"Ñ":616542,"Ñ‚":784384,"у":465611," t":1584,"Ñ”":83027,"Ñ—":138148,"Ñ–":1047876," p":1613,"Є":3706,"І":14787,"Й":2127,"Л":22219,"К":34440,"Ð":39528,"Ðœ":28217,"П":47409,"О":15488,"Б":23123,"Ð":35851,"Г":15610,"Ð’":31676,"Е":8630,"Д":40073,"З":11772,"Ж":3520,"Ш":12613," I":24900,"Ю":2003,"Я":3891," L":1894," M":2910," B":2299,"Т":14847," C":3270,"У":18037,"Р":27850," A":3247,"С":43546," F":1599,"Ц":5942,"Ч":5807," G":1797,"Ф":32198," D":1850,"Ð¥":6705,"л":484150,"к":541520,"й":174507," X":1787,"и":768429,"п":339521,"о":1248992,"н":1274404,"м":381536,"г":226618," S":3616," R":1762,"в":579991,"б":162019," P":2630,"а":1251055,"з":257293,"ж":95383,"е":806087," T":2138,"д":454853,"Ò":1990,"Ò‘":4623," Ð":31736," Б":22701," Ð’":30948," Г":15174," Д":39397," Е":7917," Ж":3460," З":11194," Й":2051," К":32754," Л":21825," Ðœ":27530," Ð":36830," О":14175," П":46014," Є":3619," І":13525,"EE":21655,"I ":2684," б":36681," а":43891," г":34855," в":154625," е":34325," д":117491," з":113779," ж":8768," й":6819," л":26339," к":61923," н":123337," м":87814," п":148797," о":74913," Р":21137," С":39028," Т":13848," У":16956," Ф":31501," Ð¥":6271," Ц":5651," Ч":5720," Ш":10618," Ю":1964," Я":3806," Ñ—":5196," Ñ–":83499," Ñ”":13641," Ñ‚":92032," у":89129," Ñ€":101219," Ñ":120858," ц":13483," ч":24594," Ñ„":40198," Ñ…":11362," ш":8917," щ":18497," ÑŽ":2119," Ñ":36739," Ò":1984,"E ":21977,"NS":21624,"II":2127,"IN":21640,"SE":21638,"a ":4767,"i ":1589,"he":1740,"el":1571,"en":2535,"es":2308,"er":4659,"e ":6591,"d ":1983,"at":2458,"ar":2758,"al":2232,"an":3512,"ac":1574,"nt":1885,"on":3299,"or":2694,"r ":2633,"o ":1648,"na":1574,"ni":1555,"li":1877,"le":1913,"la":1826,"n ":4271,"ic":2243,"ia":1621,"is":1891,"in":3535,"io":1638,"l ":1929,"y ":1830,"us":2293,"um":1631,"te":2376,"ti":2183,"th":1945,"ta":1627,"st":1645,"ro":2012,"ri":2866,"re":1930,"ra":2584,"t ":3059,"s ":6868,"Ì ":2290,"Ìв":3741,"Ìд":2078,"Ìк":2065,"Ìй":3031,"Ìм":2120,"Ìл":4305,"Ìн":8997,"ÌÑ€":4428,"ÌÑ":2026,"ÌÑ‚":2528,"Ð ":2905,"Ð’ ":2961,"ьє":2885,"юв":4387,"юд":3425,"ÑŽÑ€":3284,"ÑŽÑ‚":14456,"юч":4396,"ÑŽÑ”":2722,"Ñд":6152,"Ñг":3108,"Ñв":2990,"ÑŽÑŽ":1542,"Ñн":10771,"Ñм":9107,"Ñл":2194,"Ñк":26414,"Ñз":4329,"ÑÑ…":3443,"ÑÑ‚":9738,"ÑÑ€":2263,"Ñч":4477,"ÑÑ”":2354,"ші":2899,"щи":3380,"ще":5720,"ща":2636,"що":19501,"щі":1589,"ьк":115778,"Ñ–Ì":5805,"ьб":1841,"ьв":2256,"ьш":4727,"ÑŒÑ":24456,"ьт":4512,"ьн":50183,"ьм":4831,"ьп":5070,"ьо":12701,"Ñ„Ñ–":13912,"хи":2735,"хн":7629,"хо":17158,"Ñ…Ñ€":2067,"ху":3902,"ха":7837,"ци":27077,"Ñ…Ñ–":11714,"цт":3698,"цу":23008,"це":16672,"чл":2502,"чн":59778,"чо":6493,"ці":93529,"чи":16012,"чк":3842,"чу":3146,"ць":14520,"че":19603,"цÑ":6177,"ча":25750,"цю":1804,"шо":7889,"шн":3282,"шк":3613,"шл":2107,"ши":11930,"шт":3498,"шу":1879,"чч":1777,"чі":4713,"ше":8200,"ша":4745,"Ñк":15952,"Ñм":3119,"Ñл":40211,"Ñо":21185,"Ñн":22933,"Ñп":20278,"Ñв":10260,"Ñе":44728,"Ñи":21636,"рі":55323,"рш":6088,"рю":2898,"Ñа":14767,"Ñ€Ñ":9090,"рр":2512,"Ñ€Ñ":15180,"рт":37524,"ру":28810,"рх":8486,"рц":1712,"тн":17002,"тл":3183,"тк":8278,"Ñ‚Ñ":3967,"Ñ‚Ñ€":48477,"то":75584,"те":71719,"тв":21013,"ти":129473,"ÑÑ–":43933,"ÑÑŒ":82109,"та":153881,"ÑÑ":27435,"ÑÑŽ":2514,"Ñу":15433,"ÑÑ":3485,"ÑÑ‚":199868,"ÑÑ…":2897,"Ñц":3318,"ур":24405,"уп":9084,"ут":35010,"уÑ":11964,"ум":8171,"ул":17666,"ун":34351,"Ñ‚Ñ–":36808,"уз":30153,"ук":20646,"уд":13417,"уг":6510,"уж":3934,"Ñ‚ÑŽ":2714,"уа":9949,"Ñ‚Ñ":7369,"уб":6901,"ув":18706,"Ñ‚ÑŒ":44694,"Ñ‚Ñ‚":6769,"ту":74115,"фу":3164,"Ñ„Ñ‚":2346,"Ñ„Ñ€":25365,"фо":9781,"фе":6303,"ує":8184,"фа":4406,"ую":4341,"уч":6062,"уш":1880,"ух":4146,"іш":7237,"іц":26077,"іч":43837,"Ñ–Ñ”":28690,"їв":4260,"Ñ–ÑŽ":3757,"Ñ–Ñ":30142,"ін":70416,"ім":15479,"іл":34864,"ік":38962,"ій":44252,"із":21994,"іж":6237,"Ñ–Ñ…":3168,"Ñ–Ñ„":1617,"Ñ–Ñ‚":52153,"Ñ–Ñ":42072,"Ñ–Ñ€":18028,"іп":25092,"іо":58208,"ів":71042,"іг":5632,"ід":79007,"іа":11156,"іб":26830,"Ñ—Ñ—":1916,"Ñ–Ñ—":55045,"їн":18326,"Ñ—Ñ…":2476,"єю":25186,"єм":3400,"єн":2727,"єт":11907,"єр":2623,"єд":3075,"єк":1686,"єв":4497," IN":21609,"а ":272631,"Р ":3804,"У ":2439,"Єв":1986,"Ів":2523,"Ін":2522,"Іл":1807,"к ":38380,"й ":109949,"Ле":2968,"Ла":4470,"Ку":2193,"Ко":9293,"м ":60876,"Кр":3269,"Ки":4604,"Ка":8275,"л ":10034,"Йо":1971,"Ðа":28658,"Ðе":1779,"Мі":3387,"Мо":5475,"о ":171358,"Ма":9118,"Ль":1829,"Ми":3400,"Лі":2822,"Ме":3642,"н ":48334,"Ло":1601,"Лу":5897,"Па":5129,"Пе":6590,"По":6892,"Ñ ":17750,"Ñ€ ":43282,"ОÑ":1646,"Ор":1949,"ÐÑ–":1710,"Ол":3009,"Ðо":2027,"п ":4596,"в ":98488,"Ðм":1689,"Ðн":4049,"Ðк":1666,"Ðл":7021,"Ðв":2330,"Ба":3984,"ÐÑ€":5755,"б ":28913,"Во":3994,"д ":30432,"Ве":7201,"Ви":2444,"Бі":2125,"Га":3476,"Бо":3837,"г ":7123,"Бр":2821,"Бе":3301,"Ва":4390,"Бу":4720,"Дж":1933,"Де":3400,"До":4198,"ж ":10290,"Ð’Ñ–":6492,"Ге":2702,"Гр":3253,"Го":2499,"е ":51777,"Да":23258,"и ":147695,"За":6302,"з ":38125,"Ен":2243,"ÑŒ ":77560,"еÌ":6375,"Ша":6268,"Ше":1818,"иÌ":5895,"ÑŽ ":78537,"Ñ ":164366,"Ст":3877,"Су":1648,"Та":2989,"Сі":1751,"Те":3497,"То":1720,"Ñ„ ":2068,"Тр":1849,"Ук":9250,"Ñ… ":93221,"РС":2535,"Пр":9085,"Пу":2178,"Ра":2736,"Ре":2944,"Пі":12505,"Ри":1682,"СР":4182,"Ñ‚ ":74217,"Ро":9385,"Ру":1597,"СШ":1868,"Са":5497,"Св":1748,"Си":1634,"Се":8345,"Со":4417,"у ":182563,"Це":4350,"ш ":3438,"аÌ":12498,"Че":2490,"ШÐ":1865,"Фр":25493,"Фе":1857,"Ха":2266,"ч ":13144,"ль":97373,"лю":13039,"мб":2910,"ма":43992,"лÑ":27148,"ме":56645,"лі":100876,"ми":33489,"мл":2674,"мк":1772,"лл":4063,"ло":47531,"лу":11894,"ла":56466,"ле":64714,"лк":3429,"ли":41628,"кі":53446,"км":1768,"кн":2648,"кл":18117,"кр":33069,"кÑ":9526,"ко":160847,"кт":21418,"ку":30802,"кц":4113,"ка":63693,"ки":83769,"кв":7446,"ке":9654,"йн":15208,"йо":12518,"йк":1565,"йл":1986,"йм":3040,"йÑ":17098,"иÑ":1553,"ищ":6034,"иш":3403,"иї":2680,"уÌ":2267,"иє":2892,"йд":1750,"йб":2838,"ип":31662,"им":32760,"ин":52951,"ик":67385,"ил":14249,"ий":79035,"иц":14181,"ич":30391,"иф":2474,"их":75898,"ит":52472,"ир":15232,"иÑ":66975,"ри":69723,"рк":9097,"рл":2627,"рм":12337,"рн":24183,"ро":132547,"рп":3819,"ра":159212,"рб":3520,"рв":5932,"рг":12811,"рд":11789,"ре":97753,"рж":5968,"пі":30461,"пр":61701,"пт":2856,"пÑ":1790,"пу":8363,"ої":42339,"пи":15610,"пн":6007,"по":70389,"пл":11153,"ою":42263,"оÑ":1743,"па":69692,"оє":2366,"пе":54812,"ощ":2597,"ош":4991,"оч":9700,"оц":9082,"оÑ":101143,"ор":94904,"оп":23531,"оо":1738,"ох":8570,"оф":6129,"от":23405,"ок":38039,"ол":59373,"ом":93606,"он":134604,"ож":13014,"ні":182081,"оз":26801,"ой":3869,"ов":119233,"ог":97492,"од":63564,"ое":3185,"ню":3932,"нÑ":95444,"об":38077,"нь":43392,"нц":54211,"нш":5309,"нч":1583,"нт":57484,"нÑ":58539,"нф":2536,"ну":21092,"но":153491,"нн":89867,"нк":13365,"мі":70237,"ни":149376,"не":44107,"нг":6365,"нд":19006,"на":216248,"му":49313,"мÑ":3566,"мп":12768,"мо":32397,"мн":5261,"мм":1713,"ге":9243,"ві":92696,"ги":4209,"гн":2639,"го":97928,"гл":6748,"гр":20635,"гу":9031,"дв":4924,"дб":1844,"да":33332,"вд":7688,"ве":48786,"вж":2823,"бі":16889,"ви":75994,"вк":5403,"вл":11314,"вн":39402,"Ñ” ":23248,"во":52805,"вп":2102,"вр":5405,"вÑ":21071,"ву":11922,"вт":7129,"вч":4263,"вц":1693,"га":24943,"вÑ":2583,"би":5176,"аї":20255,"ає":15656,"бе":15560,"бр":8733,"бн":4172,"бо":24369,"бл":17974,"бк":1672,"бу":17942,"бÑ":1904,"ва":79944,"ад":39043,"аж":6697,"аз":20777,"аб":18055,"ав":68705,"аг":13743,"ам":54378,"ан":186015,"ап":12553,"ай":22198,"ак":29485,"ал":112843,"ах":20860,"аф":6570,"ач":13762,"ац":41300,"аÑ":69626,"ар":92013,"ау":6764,"ат":76559,"ба":13923,"аю":6775,"аш":6416,"зт":3423,"зр":3339,"зп":3833,"зу":8061,"зк":2540,"зи":11355,"жі":2914,"зо":12242,"зн":22505,"зм":7721,"ив":25953,"иг":5812,"иб":4671,"иж":2800,"зі":9568,"из":10688,"ид":11644,"зь":26870,"жо":4282,"жу":4654,"еї":6545,"жи":9686,"жн":10415,"за":70302,"зб":4513,"зв":12953,"зг":1786,"зд":4686,"зе":9240,"еф":2808,"ет":48754,"еÑ":21933,"ер":123318,"еп":28664,"Ñ— ":106019,"ео":7564,"ен":200818,"ем":25862,"ел":52238,"ек":44785,"ей":12736,"ез":15415,"ді":60214,"еж":8691,"же":36749,"еÑ":2396,"жа":10671,"еч":5677,"еш":1971,"ех":4631,"ец":12416,"дÑ":7505,"др":11609,"ду":13511,"дн":33993,"дм":3635,"дп":3899,"Ñ– ":217878,"до":70187,"ди":35057,"дл":9508,"дк":6162,"де":83166,"дз":2175,"гі":41117,"оÌ":9048,"дж":30353,"еб":4669,"ев":15708,"ег":29992,"ед":62088,"дÑ":4582,"еа":4570," ар":4635," ба":6422," аб":10662," ав":3504," ад":2033," ал":3311," ак":3721," ан":4873," ам":2325," бу":8984," ва":2979," бе":6626," бр":1983," бо":3438," бл":2347," ву":2932," га":4235," бі":6025," ви":31633," ве":7581," во":6259," Ñ” ":4843," вÑ":3250," вл":2695," дв":2986," да":3768," гу":2423," го":6549," гр":11679," ге":4251," ві":52058," до":45102," Ñ– ":33511," др":3031," де":37135," гі":3754," ди":4078," дл":8069," ел":3252," ек":24873," ді":8202," зд":2257," зе":1996," за":56665," зв":4345," зб":3336," жо":2604," жи":2736," зм":2726," зо":3274," зн":5228," зі":1585," йо":3069," ка":7109," кв":3523," кр":6792," ко":25840," кн":1684," кл":3559," ку":3737," ла":4131," кі":5040," ли":5653," ле":3016," ме":10350," лі":7010," ми":2991," лю":4680," ма":12887," мо":13377," му":26214," ни":3719," мі":18386," не":11146," на":101285," но":3271," ок":3237," оз":1668," ні":2363," од":9316," об":14076," от":1642," ор":5923," оÑ":30485," оп":3801," по":44438," пл":4775," пи":3225," пе":16844," па":8207," Ре":2937," Пі":12498," Ра":2732," Ро":9378," Ри":1682," Пу":2177," Пр":9060," Пе":6585," Па":5105," По":6869," ОÑ":1642," Ор":1944," Ñ€ ":1902," Те":3490," Сі":1746," То":1716," Тр":1847," Ст":3864," Су":1643," Та":2975," Св":1745," Си":1629," Се":8339," Со":4411," у ":67844," Ру":1597," СШ":1865," Са":5492," Фр":25493," Фе":1855," Ук":9235," Це":4345," Ха":2264," Ше":1818," Ша":6265," Че":2486," Ñ ":1678," Ба":3980," ÐÑ€":5747," в ":34140," Ðн":4045," Ðм":1689," Ðл":7017," Ðк":1665," Ðв":2326," Ва":4387," Бу":4716," Бо":3831," Бр":2819," Бе":3297," а ":3713," Дж":1930," Де":3390," До":4191," Ен":2238," з ":28463," Га":3472," Ве":7185," Ви":2437," Бі":2122," Во":3987," Да":23253," Ге":2696," Ð’Ñ–":6479," Го":2496," Гр":3249," Йо":1970," Ки":4602," Ка":8259," За":6236," й ":3420," Мо":5471," Ðа":28633," Ðе":1775," Мі":3382," Ðо":2025," Ол":3009," ÐÑ–":1705," Ко":9278," м ":2294," Кр":3260," Ку":2189," Ла":4467," Ле":2966," Ло":1600," Лу":5896," Ль":1829," Ма":9106," Лі":2816," Ме":3636," Ми":3397," У ":1855," Єв":1986," Ів":2522," Іл":1806," Ін":2499," Ð’ ":2749,"Шам":2267,"Шар":2655,"INS":21596,"Кар":1691,"Кор":2456,"Киї":2310," єк":1604," єд":2480," Ñ—Ñ…":2188," Ñ–Ñ":5151," із":3768," ім":4088," ін":33850," Ñ—Ñ—":1897,"Луа":4691," ра":11678," ре":33133," пі":17870," ри":2122," ро":38150," пр":49254," Ñв":7646," рі":9410," Ñи":9133," Ñе":11547," Ñл":3167," Ñм":1572," Ñк":6896," Ñп":12079," Ñо":4130," ру":3107," Ñа":3132," ÑÑ–":4244," ти":4115," тв":2718," те":13347," то":7539," Ñ‚Ñ€":8770," ÑÑ…":2630," ÑÑ‚":41843," Ñу":7442," ÑÑŽ":1727," та":50935," ук":7367," Ñ‚Ñ–":1840," уÑ":2008," ут":2182," ун":1655," фо":4647," Ñ„Ñ€":23569," фу":2570," фа":2336," уч":1717," Ñ…Ñ€":1639," хо":1776," ху":1552," Ñ„Ñ–":5255," ха":2328," ци":1770," це":7308," чо":1959," чл":2302," чи":4976," ці":2356," че":5747," ча":8926,"Льв":1624,"Мар":4041," шт":1850," що":17557," Ñн":1885," Ñз":3165," Ñк":23816," ÑÑ‚":2201,"Мик":1636,"Мон":1603,"ÐаÑ":21926,"ад ":3964,"ав ":5473,"EE ":21630,"ам ":4361,"ан ":7774,"ак ":2945,"ал ":2859,"ай ":1706,"Оле":2078,"авч":1579,"авт":2891,"ага":5972,"аві":2505,"аго":2958,"ада":8317,"ади":3697,"аде":2606,"аду":2146,"адÑ":2241,"адо":2036,"адм":1600,"адÑ":2411,"аді":3733,"би ":2028,"ажа":1619,"або":10777,"абе":1768,"ава":4168,"авн":9017,"авл":3400,"авÑ":2814,"ає ":8442,"аво":4040,"аве":24071,"ави":3333,"бо ":10197,"ало":3744,"алу":3259,"ала":5763,"али":6052,"акі":1706,"але":7492,"амо":2377,"амп":2753,"ама":2782,"аль":45514,"ами":10086,"аме":25683,"алі":34159,"анн":20969,"ано":8600,"ану":2414,"анÑ":11789,"ант":9369,"анц":47104,"ань":4811,"ана":9630,"анд":6858,"анг":2311,"ани":9657,"амі":2404,"ане":2623,"анк":2826,"ані":36761,"азу":2305,"азо":3692,"ази":2579,"азв":4425,"аза":2057,"азі":1724,"айÑ":1694,"айо":6278,"айн":1941,"айб":2377,"акт":6740,"ако":7371,"ака":2878,"ах ":10031,"Пар":1681,"Ð°Ñ ":3290,"ар ":5874,"ат ":4127,"СР ":2617,"ба ":1557,"Пет":1744,"Пер":2277,"Пол":2322,"РСР":2369,"При":2113,"Про":3790,"Пуа":1541,"Пік":2365,"Пір":3646,"РоÑ":2415,"Роз":1581,"Рон":3384,"Пів":4853,"Ðль":5551,"Сан":1563,"СШÐ":1862,"Сер":1775,"Сен":4479,"Ста":1577,"Ðрд":2815,"Тер":1748,"Вол":2219,"Вер":4116,"Вел":1833,"Укр":9121,"Бур":2431,"Фра":24714,"Дан":22031,"ШР":1853,"аÌн":3396,"Цен":2677,"Чер":1615,"NSE":21595,"лам":1827,"лан":5122,"лаÑ":12503,"лат":2370,"ма ":7369,"Ð»Ñ ":13589,"лав":3656,"лад":10782,"кці":3993,"ль ":10569,"кул":3808,"кур":1627,"кою":25572,"кої":15644,"кре":3214,"кра":21169,"кри":4236,"кро":1703,"лу ":3819,"кÑа":1966,"кте":2165,"кти":3833,"кто":5082,"ктр":2798,"кту":3015,"кла":11761,"ло ":6223,"клю":1692,"клі":1698,"ког":16518,"ков":13165,"ком":15826,"кон":32464,"коп":2254,"кор":9858,"коÑ":2132,"кож":4742,"кол":8207,"ким":4246,"кий":27960,"ких":8957,"кві":3598,"ле ":3371,"кі ":7394,"ли ":7619,"кер":1599,"ква":2483,"ках":2083,"кат":2360,"кар":5682,"кам":2944,"кан":6220,"кал":2575,"каз":1777,"кад":1810,"ла ":13040,"Іва":2158,"йÑÑŒ":13547,"кт ":1727,"ку ":18769,"йна":1852,"йно":3774,"йни":5036,"йов":2120,"йог":2770,"йон":6057,"ко ":7248,"иїв":2473,"ки ":39616,"ке ":4049,"йбі":2012,"од ":2452,"нах":3061,"нац":23841,"нау":3625,"наф":1792,"нач":7238,"ог ":2185,"нан":4567,"нам":2788,"нал":31101,"нат":2955,"наÑ":4427,"нар":6454,"нап":3620,"над":4116,"нак":1904,"най":5486,"наз":4802,"нде":1705,"нда":2557,"ож ":4256,"нгл":1926,"неї":3826,"нен":4632,"нер":5081,"неÑ":1886,"нет":1977,"нец":1734,"нев":2650,"нез":1549,"нді":4444,"ні ":100559,"ндр":3680,"мії":2239,"ник":15713,"ний":35532,"ок ":8024,"мір":2620,"міÑ":12191,"мік":1956,"мін":9879,"міч":24763,"між":4232,"нь ":34487,"Ð½Ñ ":85321,"ню ":1561,"ов ":5346,"нав":26133,"об ":3066,"мпо":1641,"нт ":27692,"мпе":2663,"мпа":3905,"ну ":12904,"мпі":1622,"мÑÑŒ":1988,"мун":23434,"муз":3552,"ліз":3946,"лій":2290,"лік":3856,"лід":25282,"лів":5148,"літ":32189,"ліÑ":3353,"лін":5016,"лії":2011,"миÑ":2927,"мир":2021,"но ":17011,"мно":2222,"мод":1683,"мог":2124,"мов":8204,"мож":2842,"мон":3152,"мол":2071,"мор":4169,"Ð½Ñ ":2855,"має":2263,"лÑÑ”":1872,"мац":1743,"мал":3075,"мад":2394,"лÑд":1770,"мат":7712,"маÑ":1664,"лÑÑ€":1752,"мар":2087,"лÑн":1603,"ман":6730,"люч":1579,"маг":1762,"люд":2454,"лют":2538,"мет":6826,"мен":33354,"ни ":22900,"мер":6133,"меж":1784,"мі ":2838,"не ":13395,"льп":5070,"льн":50159,"льо":2192,"на ":73406,"льм":2015,"льк":3474,"льш":4684,"льт":4502,"льÑ":6409,"му ":17492,"лок":1590,"лог":8013,"лод":3460,"лор":1601,"лоÑ":2987,"лот":1979,"лом":2920,"лон":2251,"лов":10255,"луж":1851,"ків":10057,"кій":3619,"кіл":3020,"кіп":21719,"кін":3986,"кіÑ":1867,"лив":4838,"лиз":2163,"лик":4841,"лі ":6988,"леж":2754,"ми ":23542,"лен":36965,"лем":3035,"лек":7457,"лиц":3452,"лиш":2141,"лиÑ":4376,"лин":3727,"лип":2133,"пат":1536,"пад":4424,"пал":24506,"пан":5073,"пар":26316,"ре ":2016,"ра ":10978,"пиÑ":6483,"пла":3785,"пле":2508,"пло":1933,"ро ":4356,"пед":22902,"ри ":9926,"пер":22098,"печ":1592,"ори":10852,"опі":2491,"орд":2209,"оре":6973,"орг":5783,"орÑ":3875,"оро":12661,"орм":6382,"орн":2519,"опу":1571,"ора":4398,"опе":4273,"опи":2385,"опо":5348,"опа":3672,"оÑÑ–":28049,"оте":2368,"оти":4264,"ото":4402,"отр":2511,"ота":2076,"орі":12506,"оÑи":1994,"оÑк":2565,"оÑл":27847,"оÑн":4625,"оÑо":4627,"оÑп":1773,"оÑÑ‚":18846,"ору":3825,"орт":4236,"орÑ":2411,"оми":3494,"олі":11097,"оме":3420,"ома":8391,"олÑ":2505,"олю":1788,"оль":6830,"олу":1961,"по ":3091,"оло":19189,"оле":3051,"оли":6076,"окі":1858,"ола":2363,"окр":3962,"оку":9695,"око":4015,"оні":30331,"онÑ":3562,"онт":3739,"ону":5669,"они":2616,"омі":28058,"оно":28802,"онн":3725,"она":36879,"онд":1886,"оне":2765,"омо":4256,"омп":5253,"ому":14955,"оча":2902,"очи":1547,"оці":5589,"офе":1770,"оце":2765,"офі":2407,"охо":3822,"оба":1591,"нÑÑ‚":1850,"нÑм":4193,"ова":23572,"обу":2657,"обр":3128,"обо":4431,"обн":1797,"обл":9987,"оби":2069,"ою ":40879,"ньо":6069,"па ":1808,"оки":2050,"оке":1657,"ока":3381,"ожн":3109,"озв":2698,"нів":6803,"ніз":4328,"ози":2146,"оза":1905,"озт":3392,"ніт":1771,"ніÑ":9671,"ніц":22879,"ніш":2881,"ніч":5481,"нік":1930,"ній":8198,"озм":1625,"нім":3177,"озн":3472,"озр":1991,"озп":1689,"нії":3375,"ніÑ":2921,"одн":11088,"оди":15822,"одж":2752,"огі":6272,"оду":4615,"одо":6110,"ої ":40186,"пи ":6419,"оді":8902,"оже":2141,"обі":3439,"ове":4729,"овл":2741,"ови":27036,"ово":14293,"овн":9596,"овт":2617,"ову":5401,"овÑ":3796,"ога":1966,"ові":15249,"ого":77853,"огр":5122,"ода":5921,"оде":2118,"ної":16682,"ною":5373,"ноÑ":8914,"ноп":1707,"ном":33988,"нок":2048,"нні":5141,"ног":41554,"нов":14623,"ннÑ":64956,"нно":4474,"ор ":9067,"нни":8535,"нна":3011,"SEE":21595,"нко":3013,"он ":7906,"нку":1909,"нка":2370,"ом ":22352,"ним":10232,"нин":2727,"нич":2031,"них":49028,"ниц":6236,"нши":2462,"нці":27792,"нцу":22511,"нув":2948,"нті":3002,"нÑÑŒ":25045,"нта":5969,"нте":3274,"нти":4476,"нту":1658,"нто":2721,"ох ":2654,"нтр":7071,"нÑÑ‚":25118,"Ñам":2553,"Ñ€Ñм":1886,"Ñан":2886,"Ñ€Ñд":3770,"Ñво":2566,"ÑвÑ":1631,"Ñві":5195,"ÑÑ– ":3512,"Ñел":26979,"ти ":13731,"Ñен":1784,"Ñер":9261,"рів":9362,"рід":3016,"ріа":3973,"рій":3132,"різ":4766,"ріо":2440,"ріш":1652,"річ":3119,"ріÑ":3137,"ÑиÑ":3972,"Ñит":2195,"рії":5126,"Ñин":2791,"Ñил":2927,"Ñка":1987,"Ñли":1774,"Ñла":3451,"Ñко":3258,"Ñкл":4964,"Ñлі":25511,"ÑлÑ":1724,"Ñлу":2674,"Ñло":3957,"то ":6783,"Ñни":4268,"ÑнÑ":2333,"Ñоб":4324,"Ñов":3112,"Ñні":1707,"Ñок":1638,"Ñно":9584,"Ñ‚Ñ€ ":4211,"Ñну":2111,"Ñпе":2794,"Ñор":1743,"Ñон":2424,"Ñоц":1647,"ту ":33552,"Ñпі":4109,"Ñпу":1543,"Ñпо":6481,"Ñпр":2669,"Ñу ":3870,"роц":5357,"рот":5051,"роф":2524,"роп":4979,"роÑ":8219,"ÑÑ‚ ":6153,"рпн":1909,"Ñ€ÑÑŒ":7310,"рта":23604,"Ñ€ÑÑ‚":1652,"рти":2701,"Ñ€Ñи":2327,"рух":1575,"рту":2197,"рті":2581,"рук":2570,"руг":2655,"руд":3764,"руп":2957,"руÑ":1928,"рхі":2138,"рхн":4291,"рши":1950,"ÑÑŒ ":1944,"та ":53895,"ÑÑ ":24676,"рад":5772,"раж":1608,"раз":3483,"рав":11695,"рам":5248,"ран":55851,"рай":6264,"рак":4056,"рал":6105,"рах":2738,"раф":3312,"рац":4988,"раÑ":1585,"рат":9434,"раї":18701,"рі ":5250,"рде":3344,"ргі":2098,"реб":1606,"рев":3520,"рег":26086,"ред":9949,"реа":2012,"рет":3016,"реÑ":6263,"Ñи ":1816,"рен":12198,"рем":5175,"рел":2059,"рек":2766,"рез":6938,"рді":3024,"реж":3159,"ржа":4954,"реч":1677,"рец":2506,"рвн":2312,"рга":5371,"ргу":2442,"рим":4841,"рин":4051,"рик":6384,"рил":1738,"рий":1712,"рич":3064,"рит":7476,"рир":1816,"риÑ":11068,"рка":1676,"пів":7244,"під":9422,"риб":1872,"риг":1977,"рив":2681,"риз":3342,"піл":3701,"піÑ":2425,"рмі":3148,"рни":5921,"рне":1575,"рна":4926,"рок":13067,"рол":3802,"ром":8841,"рон":6202,"роз":14973,"рні":3617,"ров":13779,"рог":3625,"род":18027,"роб":8445,"рно":5153,"рко":1653,"рма":4259,"пра":9765,"при":17085,"пре":5034,"про":26547,"ру ":4801,"поп":1576,"пор":8542,"поÑ":5211,"пот":2142,"пох":2319,"поч":2731,"пош":1607,"рт ":1813,"под":5278,"пов":10451,"пнÑ":3871,"пон":3944,"пом":2310,"пол":10620,"пок":1979,"поз":3035,"пуб":2176,"пуÑ":1543,"прÑ":1882,"Ñа ":2954,"вар":4866,"ват":3699,"вач":1754,"ває":1673,"вав":2297,"ван":27620,"вал":6746,"важ":2723,"га ":2428,"бут":2028,"бул":3726,"бур":1929,"буд":3912,"був":3407,"Ìн ":1990,"вÑÑ":2929,"вто":3926,"втн":2139,"вÑÑ‚":3544,"вÑÑŒ":11822,"гу ":1846,"вро":2652,"вою":2829,"вої":4164,"вол":4005,"вні":6649,"вод":5394,"вог":4666,"вов":1600,"внÑ":4991,"вор":9030,"воÑ":3392,"вом":2439,"вон":1910,"вни":11918,"вне":1755,"вна":3546,"вно":8046,"влі":1867,"влÑ":2045,"вле":3353,"вла":2838,"го ":74789,"вка":1932,"вищ":4417,"вич":10270,"виз":2384,"вий":5063,"вил":1963,"вик":6641,"вин":5363,"вим":3289,"вип":2153,"виÑ":3214,"вир":3300,"вит":2461,"вих":6912,"вив":1786,"виг":1941,"вид":7090,"біл":7304,"вец":2207,"вер":12033,"ги ":1955,"вел":4069,"вед":23850,"ві ":8320,"вде":5262,"ва ":20184,"ают":4986,"баг":2186,"аці":38441,"ашо":3325,"аук":3811,"аті":2815,"афт":2014,"ахо":3481,"афі":2408,"ача":3308,"аче":5139,"ахі":3064,"апр":2675,"апа":1903,"апо":2274,"апи":1583,"арх":2803,"арÑ":2742,"арт":27902,"аре":1829,"ард":4062,"ара":11795,"арн":4172,"аро":9064,"ари":4536,"арк":3550,"аÑÑ‚":19420,"ата":3063,"аÑи":3444,"арі":4603,"аÑе":23473,"аÑл":1823,"аÑо":2485,"аÑн":7019,"ату":6561,"ать":1702,"ате":4441,"аÑÑ–":1565,"ати":34490,"атк":2377,"атн":2400,"Ìль":1996,"ато":8833,"атр":2250,"бол":1926,"бор":3014,"бни":2582,"бро":2133,"ву ":2725,"бра":3585,"блі":3564,"бла":6820,"бли":3503,"бле":1613,"во ":6738,"ви ":5236,"аєт":5513,"бер":8329,"без":3385,"аїн":17837,"ве ":2027,"дає":2865,"дач":1562,"дан":4800,"дар":3127,"дат":2935,"дал":2067,"дав":3568,"дем":2009,"ден":35742,"дер":7440,"деп":22255,"дже":27277,"ей ":5110,"дво":2066,"ез ":2842,"ді ":6167,"дÑÑŒ":3872,"дÑÑ‚":2708,"дрі":2262,"дро":2499,"дру":2324,"дра":1728,"дпо":2072,"ет ":25079,"дко":1659,"ен ":10176,"ем ":4029,"див":2382,"гіч":3315,"дин":11940,"дит":3892,"гії":2091,"оÌв":1799,"гіо":23749,"гір":2225,"гід":1618,"днÑ":2562,"доб":2151,"дні":5130,"дов":8472,"доÑ":25281,"дор":2173,"док":2841,"дон":2228,"дом":6507,"дна":4835,"дмі":2603,"дни":7067,"дне":1547,"дно":10257,"ер ":7534,"длÑ":8246,"да ":6361,"газ":2730,"гал":4240,"гат":2475,"вÑÑ‚":1631,"ган":6922,"де ":6060,"вул":1831,"вує":1869,"вча":1606,"вче":1615,"гол":4270,"гоÑ":1776,"гор":5348,"гов":2480,"год":1736,"гру":5182,"ду ":6755,"гро":3158,"гра":8609,"гре":2241,"гун":2442,"ген":3541,"ди ":6794,"гео":1703,"вів":2194,"вік":22506,"віл":2297,"вій":5760,"від":28365,"віт":10410,"вір":2465,"віÑ":2374,"він":3115,"глÑ":1806,"до ":11544,"жав":4567,"за ":31382,"жит":2833,"жив":2319,"жин":1953,"жі ":1618,"жен":31144,"зу ":2376,"жно":2243,"жни":2399,"жна":2353,"жні":1682,"жов":2516,"ежа":1636,"ежн":1674,"ежи":2240,"едÑ":1966,"еї ":5804,"еді":22755,"діÑ":9605,"діє":22354,"дії":2193,"діл":4733,"езн":2478,"езп":2124,"дів":4952,"дій":3323,"ева":2033,"еви":2106,"еат":1536,"дÑн":2535,"еда":2858,"еде":24252,"еди":1957,"егі":24559,"едо":1597,"едн":2110,"евн":1571,"же ":2405,"ево":2649,"еві":2180,"ент":38060,"енÑ":2476,"енц":1664,"енк":2033,"ени":9106,"ено":4866,"енн":55011,"ена":6246,"емі":4010,"ене":8942,"енд":1893,"еор":1718,"ені":27559,"ень":28255,"епа":22831,"ерш":4377,"ерх":5234,"ерп":2312,"ерÑ":4919,"ерт":3088,"ерм":2838,"ерн":8048,"еро":3883,"ери":8614,"ерк":2529,"ерд":2064,"ерг":2332,"ерж":5295,"ере":27737,"ера":11283,"ерв":4795,"ерб":1797,"ейÑ":2515,"еко":24801,"ект":7509,"екÑ":5740,"ели":5579,"ело":1860,"еле":29499,"ела":1663,"емл":1627,"емо":2244,"емн":1709,"еми":2190,"елі":3500,"ель":6771,"еме":2758,"ема":4374,"ехн":2533,"ець":7827,"еці":2099,"ері":9985,"еÑн":2797,"еÑп":2017,"еÑо":1640,"еÑÑ‚":4336,"ета":3510,"еÑÑ–":2297,"ети":3023,"ете":1577,"етр":4098,"ето":3229,"ету":1803,"ива":5433,"иді":1970,"иго":1963,"ида":2853,"ивÑ":1559,"иво":3293,"ивн":5798,"иві":2780,"икл":3021,"ико":10700,"ики":26994,"ика":9412,"изь":1617,"изн":4148,"ині":5661,"имі":1780,"ини":11405,"инн":2229,"ино":4309,"инÑ":2389,"ину":2117,"ина":9917,"ими":8406,"илі":2149,"имо":2293,"има":1943,"иль":2368,"икі":3590,"или":2166,"ило":1638,"ила":2574,"иÑе":1748,"иÑа":1892,"иÑÑ‚":43720,"иÑо":2640,"иÑл":3795,"иÑк":1974,"ити":3061,"ите":2660,"ита":3840,"иÑÑ":2391,"иÑÑŒ":3153,"иту":23904,"итт":1716,"ито":5821,"ипа":23599,"ипн":2085,"ире":1798,"иро":6079,"ихо":1721,"ицт":1948,"иць":3344,"ить":5744,"ище":2694,"иці":4564,"ичи":1676,"ичн":15682,"ицÑ":3075,"ича":2349,"ка ":31484,"ив ":2824,"зав":2331,"заб":1710,"заг":2313,"ид ":1647,"зви":3864,"зва":3561,"зац":3958,"зах":3661,"заÑ":5087,"зап":2437,"зан":2419,"зал":3333,"зак":3350,"ий ":76977,"зер":2249,"зем":2575,"зді":2313,"зі ":4174,"зик":1993,"ик ":11149,"ин ":8446,"им ":13202,"зич":2000,"зна":12042,"зни":3492,"змі":3801,"зно":1802,"знÑ":3181,"зов":3243,"зон":2606,"зпе":1731,"зпо":1609,"зро":1734,"зта":3312,"их ":71895,"ич ":8041,"зьк":25702,"ьме":1976,"ьна":3031,"ьни":9629,"ьно":31998,"ька":11941,"ьке":2474,"ьки":32360,"ько":60206,"ькі":5969,"ьту":1942,"ÑŒÑÑ":17960,"ÑŒÑÑŒ":5340,"ьог":3861,"ьов":1687,"ьні":3450,"ьпи":4442,"Ñк ":5568,"Ñм ":4621,"ÑŽÑ” ":1572,"юва":4241,"ÑŽÑ€ ":1687,"er ":1536,"Ñка":3149,"Ñки":6457,"Ñко":4501,"Ñкі":4711,"Ñч ":2564,"юто":2149,"ÑÑ… ":2449,"ÑŽÑ‚ÑŒ":10792,"ючи":2094,"ÑнÑ":3724,"Ñми":2547,"ÑÑ‚ÑŒ":2425,"Ñти":2263,"уча":3150,"уєт":3972,"феÑ":1886,"фер":1827,"уют":3197,"фун":1616,"фра":23347,"фор":6458,"фік":1922,"філ":3022,"фіз":1799,"це ":3714,"хан":1700,"хар":2307,"хні":3188,"хов":2787,"ход":8360,"хнÑ":2286,"Ñто":20885,"ÑÑ‚Ñ€":14612,"Ñтв":9052,"Ñте":10268,"Ñти":58155,"Ñта":42984,"ÑÑ‚Ñ–":17422,"ÑÑ‚Ñ":1612,"ÑÑ‚ÑŽ":2000,"ÑÑ‚ÑŒ":9349,"Ñту":4755,"Ñце":2414,"Ñ‚ÑŒ ":23970,"Ñ‚ÑŽ ":2076,"Ñьк":76644,"Ñ‚Ñ ":4033,"Ñьм":2215,"ÑÑŽÑ€":1594,"ув ":2334,"тав":4291,"так":7239,"тал":7016,"там":24076,"тан":13432,"тат":26660,"тах":1715,"тар":4879,"таш":3389,"тво":10485,"тва":6033,"тех":2451,"тец":1853,"тем":5996,"тел":3217,"тен":2843,"тер":16300,"теп":1791,"тет":24922,"тек":2381,"тей":1760,"Ñ‚Ñ– ":19387,"Ñіч":2428,"тив":7452,"Ñій":4342,"ук ":2255,"Ñів":2280,"Ñіб":22500,"тка":1831,"тич":9159,"ÑÑ–Ñ—":1716,"тий":2070,"тин":8122,"тик":26054,"тил":2041,"тир":1541,"тиÑ":25636,"тип":1715,"тит":24505,"тку":2518,"тко":2210,"тла":1548,"ур ":1549,"тно":2899,"тні":2826,"тод":1678,"ток":3298,"тол":4612,"тов":10097,"тог":4926,"тнÑ":4250,"тни":4639,"тре":1761,"тра":13299,"три":6247,"тор":19453,"тоÑ":2194,"том":6059,"тон":3478,"топ":3418,"тою":1721,"Ñ‚ÑÑŒ":3003,"тро":10751,"тру":4586,"трі":6027,"тув":1637,"туп":2434,"тур":8289,"Ñ‚Ñ‚Ñ":4038,"тут":22979,"Ñ‚ÑŒÑ":17965,"ує ":4149,"ува":15273,"уго":1668,"уар":5114,"уат":1738,"убл":2373,"узь":23065,"узе":1844,"тій":2025,"узи":2433,"тіл":1694,"тів":6962,"уді":1635,"удо":3223,"удн":2817,"уме":1763,"уль":5593,"улÑ":1849,"ули":2646,"уло":1791,"ула":2199,"укт":2271,"укр":7476,"уко":2812,"упн":1546,"ура":2553,"ург":3415,"ури":2733,"упа":1808,"унк":2435,"уні":25084,"умо":1836,"унд":2657,"уту":22317,"уÑÑ‚":2645,"утв":1947,"урн":4525,"уро":1539,"що ":17655,"шов":4224,"ших":3225,"шир":2574,"ший":2090,"ще ":2338,"шен":2816,"ші ":1719,"шта":1833,"щин":2002,"щен":2787,"цен":4287,"чи ":3745,"цев":1886,"цеÑ":2327,"цер":1781,"ці ":10080,"хід":5700,"цип":23018,"ць ":4743,"Ñ†Ñ ":4768,"ча ":2185,"цуз":22494,"цтв":3628,"ців":2368,"ціа":3468,"ціо":25893,"цій":6921,"ціÑ":7173,"чен":10697,"чер":4797,"чі ":2791,"чле":2375,"чка":1597,"чин":4805,"ціє":1783,"ції":31741,"чиÑ":2289,"цьк":8704,"ша ":1587,"ше ":3944,"чає":2804,"чаÑ":12463,"чат":2049,"чай":1572,"ща ":1848,"чні":4741,"чна":4519,"чнÑ":2280,"чни":34595,"чно":11306,"us ":1758,"Ñ–ÑŽ ":3147,"їв ":1974,"іал":5781,"іан":1803,"Ñ–Ñ ":24064,"іде":1874,"ідж":22706,"ідк":2690,"ідн":11513,"Ñ–Ñ— ":54256,"ідр":2425,"ідп":3775,"ідо":5503,"івд":5855,"іве":2535,"ібн":1843,"івÑ":5463,"івн":8627,"ійн":9751,"ійÑ":12117,"іжн":2207,"ізо":1615,"ізн":5205,"ізм":1906,"ізи":1715,"іза":4100,"інд":1760,"іне":2360,"іна":3489,"інн":4758,"іно":3712,"імі":1726,"інф":1596,"інц":2809,"інÑ":24263,"інт":1810,"іль":17941,"іме":3447,"ілÑ":2402,"ілі":1831,"імп":2489,"іле":1779,"ікі":22043,"іло":3466,"іка":6429,"іки":2076,"іко":1702,"Ñ–Ñц":2429,"Ñ–ÑÑ‚":27606,"Ñ–Ñл":1997,"Ñ–Ñн":3890,"іре":3719,"ірн":2678,"іпе":22100,"іні":5468,"іод":1626,"інш":3831,"інь":1631,"іон":51386,"ітн":4596,"іто":2264,"Ñ—Ñ… ":2010,"Ñ–Ñ‚Ñ‚":1981,"іту":1943,"іта":3402,"іте":26865,"іти":4695,"ішн":1712,"іше":2070,"ічн":39482,"іці":2111,"іци":22860,"іб ":22595,"ів ":41252,"ід ":15090,"із ":3976,"іж ":3000,"ій ":20369,"ік ":2836,"ім ":3593,"ін ":5333,"Ñ–Ñ€ ":2760,"Ñ–Ñ… ":2419,"єю ":25174,"єдн":2196,"єкт":1631,"єть":11566,"Ñ—Ñ— ":1912,"ією":24943,"Ñ–Ñч":2557,"ївÑ":1781,"їнÑ":8303,"їни":6169},"n_words":[15331232,17151725,12469252],"name":"uk","type":"cyrillic"} \ No newline at end of file
diff --git a/contrib/languages-data/ur.json b/contrib/languages-data/ur.json
new file mode 100644
index 0000000..62296aa
--- /dev/null
+++ b/contrib/languages-data/ur.json
@@ -0,0 +1 @@
+{"freq":{"Ù¹":7565,"Ù¾":19909,"Ù°":348,"Ùˆ":91887,"ÙŠ":164757,"Ù‹":561,"ÙŽ":343,"Ù":600,"Ù€":316,"Ù":15396,"Ù‚":19382,"Ùƒ":314,"Ù„":61015,"Ù…":85213,"Ù†":73500,"Ù‡":210,"Ù‘":218,"Ù":1208,"Ø®":10520,"د":42999,"ج":29799,"Ø­":15399,"ت":63424,"Ø«":2772,"ب":46160,"ئ":12622,"ا":210246,"ؤ":741,"Ø¢":6835,"Ø¡":3457,"غ":3607,"ع":26594,"ظ":4173,"Ø·":9560,"ض":4853,"ص":11646,"Ø´":17043,"س":59068,"ز":13283,"ر":100269,"Ø°":2393,"ØŒ":8584,"Ø›":310,"Û":81948,"Ûƒ":194,"Û‚":442,"Û’":72723,"Û“":1226,"Û”":21285,"Ú¯":13942,"Ú©":96247,"Ú¾":18217,"Úº":33709,"Ú†":6431,"Úˆ":3486,"Ú‘":2986,"Ú˜":212," ØŒ":2410," Û”":1318," Û":33487," Ú¾":395," Ú¯":5518," Ú©":67994," Ù†":12353," Ù„":8072," Ù…":43729," Ù‚":5408," Ùƒ":206," Ù":4429," ÙŠ":9396," Ùˆ":11189," ص":3954," Ø´":8375," Ø·":3503," ض":1387," ر":7523," Ø°":1057," س":22509," ز":3628," ع":9286," ظ":270," غ":946," ا":57235," Ø¡":2548," Ø¢":6281," ج":19940," Ø­":6228," Ø®":5235," د":11120," ب":20334," ت":17083," Ø«":374," Ú†":3475," Úˆ":1133," Ù¹":932," Ù¾":14925,"کا ":9732,"کت ":188," ØŒ ":2340," Ø¡ ":2420," Ùˆ ":1367," بھ":3806," جن":2286," حا":1179," جل":205," بÛ":901," جم":727," جي":597," جو":3440," جد":255," جس":2699," جز":358," جر":340," اے":201," اÛ":629," جا":6670," جب":836," تي":855," اک":704," بڑ":974," اگ":491," خل":943," تÛ":199," تھ":4319," خي":280," خو":966," دا":1316," خص":169," خر":184," خد":283," خط":402," حق":234," حي":633," تک":1112," حم":243," خا":1564," حر":509," حس":437," حد":214," حض":717," حص":647," بے":210," بن":2795," بل":1071," بغ":190," بع":1385," اÙ":270," بر":1803," اي":8027," او":11360," بد":254," بح":524," اق":467," اÙ":961," ال":4248," با":2739," ان":6816," ام":1333," اط":314," اع":876," اد":879," ار":1330," از":241," اس":10716," اش":424," اص":842," اض":219," اب":1381," ات":313," اث":175," اج":297," اح":507," اخ":805," تو":1190," بچ":211," تن":378," تم":482," تق":681," تÙ":194," تع":1588," تش":261," تص":444," تر":1553," تخ":375," تج":367," تح":884," تا":1192," تب":411," اپ":1467," اٹ":312," بو":485," بي":1555," آپ":775," آل":254," آن":292," آي":279," آخ":168," آر":246," آت":368," آج":212," آئ":402," آب":1319," Ø¢Ù":180," آس":249," آز":331," عÛ":219," سے":9513," Ø´Ú©":419," طو":1200," ظا":186," عظ":314," عر":1083," عد":298," عث":218," عا":1056," عب":533," Ø´Û":1939," عي":293," عل":2998," عم":1153," عن":249," عو":213," غي":270," سع":194," سط":261," دھ":223," سÙ":224," سي":1366," سو":980," سم":787," دÛ":248," شا":2109," سن":832," سل":1069," شر":734," شخ":260," شع":372," شي":299," شم":1497," صا":255," صر":324," رک":1032," صح":380," صد":878," صل":252," صÙ":254," صو":1376," رÛ":1328," ضر":190," ضل":1004," طا":301," طب":638," سک":803," طر":1051," دس":404," در":2025," دي":2436," دو":2534," دن":789," دل":217," ذر":344," جگ":269," Ø­Ú©":746," ذي":205," جÛ":593," را":903," جھ":293," رس":334," ري":940," زر":261," رق":405," رو":1485," زب":1015," زا":201," رن":190," زي":935," سر":1466," زم":612," سب":960," سا":2953," زن":311," ست":266,"گھر":200,"Ú¯Û ":210," ÛÛ’":16945,"Ú¯ÙˆÚº":240,"گيا":1207," ٹا":182," Ù¾Ú‘":329," پن":536," پو":662," پي":2142," پا":3033," پت":168," پر":5135," پش":170," پس":187," ٹي":317," Ù„Ù":878,"ں، ":508," لغ":168," لح":265," لئ":237," لا":1034," مل":1468," مق":1520," Ù…Ù":313," مغ":611," مع":1708," مط":921," مض":238," مص":513," مس":1572," مش":1902," مر":2119," مز":309," مد":642," مذ":261," مح":1355," مخ":914," لي":2681," مث":406," مج":498," لو":614," مت":1246," ما":1837," Ù†Ù":215," نق":288," نم":568," نظ":923," نس":373," نش":178," مم":601," من":1456," نا":2714," نب":179," مو":1833," مي":17546," نج":183," وا":3957," Ù„Ú‘":168," نو":730," ني":646," Ùض":168," Ùر":1011," Ùا":763," Ùت":168," قس":291," قص":318," ÙÙ„":520," قا":1019," ÙÙ†":193," ÙÙˆ":398," قب":691," ÙÙŠ":493," قد":579," قر":855," ÙÙ¹":171," قل":240," قو":565," قي":355," ÙˆÛ":1582," ÙŠÛ":4254," Ù„Ú©":550," وج":661," Ù„Ú¯":411," وس":430," وز":353," ور":376," وغ":293," ول":235," وق":563," ÙˆÙ":272," Ù…Ú©":558," وي":428," Ù„Û’":206," Ù…Ú¯":362," Ù…Û":365," يا":2941," Ù†Ú©":275," Ù†Ú¯":248," يع":665," Ù†Û":1220," يو":1100," Ù†Û’":2696," Ú¯Ú¾":362," ÛÙˆ":7614," ÛÙ…":409," ÛÙ†":745," ÛÙŠ":5912," Ûز":243," Ûر":384," Ûا":430," Ûج":240," کت":582," کر":5496," کس":1214," Ú©Ø´":187," Ú©Ù„":1235," Ú©Ù†":341," Ú©Ù…":719," Ú©Ùˆ":6421," Ú©ÙŠ":15836," کا":11009," کئ":348," Ú©Û":5408," Ú©Ú¾":713," گا":490," گئ":863," Ú©Û’":17634," Ú©Ú†":321," Ú¯Ù†":214," Ú¯Ù„":202," Ú¯ÙŠ":1388," Ú¯Ùˆ":519," گر":807," Û” ":1018," Ù¾Û":1353," Ù¾Ú¾":659," چت":240," چا":498," Ú†Ù„":252," Ú†ÙŠ":492," Ú†Ùˆ":467," Ú†Ù†":303," ڈا":359," ÚˆÙŠ":297," Ú†Ú¾":653," Ú†Ú©":241,"کٹر":175,"Ú©ÙŠÛ’":174,"Ú©ÙŠÛ”":282,"کلا":210,"کلو":610,"کلي":168,"کيا":2343,"کيو":230,"کيم":299,"کين":186,"کيل":348,"کمي":186,"کمل":193,"کنا":202,"کوئ":495,"کور":176,"کوم":779,"کرا":507,"کتے":245,"کسي":1209,"کست":1507,"کزي":242,"کري":262,"کرت":1247,"کرد":421,"کرن":1152,"کرک":265,"کار":1107,"کائ":219,"کئي":281,"کان":309,"کام":558,"کال":425,"کتي":199,"کثر":209,"کتا":937,"Ú©Ù… ":404,"Ú©Ù„ ":646,"Ú©Ù† ":715,"Ú©ÙŠ ":12997,"Ú©Ùˆ ":5510,"کر ":2045,"کز ":207,"کس ":278,"گور":172,"گري":1479,"گرو":225,"گرد":220,"گرا":170,"گاÛ":224,"گئے":444,"گار":222,"گئي":576,"Ú¯ÙŠ ":628,"Ú©ÛÙ„":354,"Ú©Ûت":572,"Ú©Ûا":1480,"کھا":567,"کھت":425,"Ú©Ú¾Ùˆ":307,"Ú©Ú¾Ù†":402,"Ú©Ú¾ÙŠ":483,"گر ":1024,"گا ":209,"Ú©Û’ ":18223,"Ú©Ú†Ú¾":310,"Ú©Ú¾ ":350,"Ú©Û ":4209,"ا، ":388,"ھا۔":1135,"ھائ":250,"ÚºÛ” ":2401,"ھان":195,"ھار":432,"عÛد":221,"ھتے":283,"پان":548,"پار":326,"پاس":169,"پائ":259,"ÛØŒ ":321,"پي ":246,"ئي ":3847,"ات ":5026,"اح ":512,"اج ":434,"ئم ":385,"پرو":236,"پري":221,"ئل ":289,"اب ":1615,"اء ":706,"اؤ ":172,"پاک":1528,"پنج":401,"پنا":177,"ئع ":214,"پور":484,"پني":552,"ئش ":369,"Ú¾ÙŠ ":3674,"ئر ":279,"بت ":362,"Ùر ":258,"ان ":9264,"با ":254,"Ùظ ":915,"ÛÙ… ":759,"اً ":534,"ÛÙˆ ":977,"پنے":725,"ا٠":395,"پيش":361,"پيد":1076,"ام ":5824,"ال ":3321,"ÛÙŠ ":1279,"اق ":313,"اظ ":454,"اع ":255,"Ûائ":399,"Ûات":210,"Ûار":307,"ار ":4602,"اخ ":174,"Ùت ":347,"اد ":2459,"Ûاں":875,"اص ":271,"از ":1002,"اس ":6443,"Ûان":212,"ارے":550,"اطا":177,"ھوا":216,"اسک":1125,"ت، ":322,"بي ":1989,"اضي":270,"بو ":188,"اصط":318,"اصل":1371,"ارÛ":558,"اسے":795,"اعت":501,"اعد":311,"اعر":317,"ھوٹ":341,"ھيل":511,"اطي":187,"Ú¾Ù†Û’":355,"اعظ":290,"Ú¾ÙˆÚº":204,"اعل":259,"قع ":1257,"Ùار":420,"Ú¾ÙŠÚº":343,"Ùات":329,"Ùاظ":273,"ادا":654,"اخل":199,"اتھ":922,"Ú¾ÙŠÛ”":577,"احي":174,"اخت":714,"احم":281,"احت":209,"ئي۔":457,"Ûا ":1654,"ارا":821,"ادي":1677,"ئيں":363,"ادل":266,"ادب":182,"ادت":200,"بق ":617,"ادر":214,"اتے":523,"ازي":292,"است":1978,"اري":1870,"ارن":318,"ارو":652,"Ûت ":499,"ارÙ":181,"ارس":382,"بل ":520,"ارد":993,"ارت":911,"ارک":342,"اشي":236,"اسم":259,"ادÛ":769,"اسل":804,"اشا":172,"اسي":947,"ارٹ":196,"بن ":873,"بع ":355,"ائد":205,"ائر":395,"ائش":451,"ائع":219,"ائن":582,"ائم":404,"ائل":355,"ائي":3099,"ئنس":292,"ابت":325,"ابر":254,"اؤں":202,"Ûد ":227,"ابي":488,"ابل":415,"ابق":678,"ابو":358,"ابن":262,"اتح":175,"اتا":3152,"Ûر ":2189,"اثر":224,"ؤں ":253,"اتي":1903,"قت ":631,"ائے":1094,"اجا":187,"بر ":849,"ÙÙŠ ":604,"عظي":276,"عظم":356,"Ù¾Ú‘Ú¾":180,"عري":274,"عرو":255,"عرب":766,"عرا":186,"عدا":424,"عدد":310,"عثم":220,"ظيم":431,"عبد":345,"عات":242,"Ø´ÛÙˆ":678,"Ø´Ûر":1734,"عال":536,"عام":681,"عاش":251,"عار":177,"صے ":201,"Ù¾ÛÙ†":208,"Ù¾ÛÙ„":895,"غرب":546,"Ø·Û ":206,"عيا":204,"عيس":213,"پھر":309,"Ù¾Ú¾ÙŠ":228,"عمل":531,"عمو":341,"عمي":283,"عمر":256,"غان":180,"عني":881,"غاز":188,"علا":1570,"علق":602,"علي":1229,"علو":368,"عما":1154,"علم":738,"آتا":178,"آئي":238,"آبا":1188,"آخر":190,"آزا":306,"ھر ":618,"آن ":225,"ھا ":1233,"غير":796,"آيا":202,"عے ":290,"آپ ":665,"Ø¹Û ":760,"جسم":317,"جزي":258,"خت ":294,"جرا":223,"جري":262,"شعب":197,"جزا":198,"جرم":182,"بکÛ":367,"ثيت":188,"Ø°Ûب":185,"شما":1408,"جبک":363,"اÛÙ…":638,"جائ":411,"اÛÙ„":181,"جات":4030,"اÛÙˆ":304,"جاب":370,"جاس":338,"اÛÙŠ":375,"جار":388,"جان":1341,"جام":298,"رے ":1358,"بÛت":604,"رکز":477,"بÛا":219,"صبÛ":185,"جما":288,"رکا":241,"صحا":176,"رکي":303,"صدر":477,"حال":384,"جنو":753,"ÛÛ’Û”":9056,"صدي":342,"حاد":180,"حاص":632,"حاظ":206,"جمع":225,"جمو":257,"شيا":452,"خط ":169,"بھي":3076,"بھا":585,"ضي ":277,"صطل":323,"جسے":332,"رکھ":862,"صرÙ":317,"جسک":274,"Ø²Û ":268,"طب ":280,"تک ":956,"خي ":172,"حرک":215,"حضر":635,"حصي":251,"دت ":251,"حدÛ":191,"سٹي":260,"حسا":181,"دا ":1143,"حسن":171,"سٹر":203,"حرا":239,"حري":437,"طح ":229,"ØªÛ ":472,"تھ ":814,"صوص":325,"ÛÛ’ØŒ":923,"صول":176,"حدي":195,"صور":542,"صوب":1118,"رÛÙŠ":215,"رÛÙ†":216,"جمÛ":270,"رÛا":394,"جنگ":528,"جيس":335,"رÛت":193,"صلا":189,"جود":775,"ضرت":619,"ضرو":182,"خان":852,"خاص":306,"خار":176,"حمد":801,"حقي":267,"تے ":3295,"رÛÛ’":389,"صيل":310,"دس ":176,"در ":1119,"دد ":492,"حصÛ":359,"دن ":286,"ٹا ":239,"دو ":1561,"ظر ":265,"خصو":323,"ر، ":449,"دي ":2904,"حير":216,"ضلع":1009,"حيا":329,"ختل":505,"حيث":191,"ختي":341,"سے ":11230,"طان":608,"طال":493,"طاب":536,"طبي":334,"دل ":339,"دم ":215,"خلي":574,"سکت":863,"تÛا":216,"سکا":323,"سکي":300,"خلا":443,"سکو":402,"دان":840,"دال":423,"تھے":1383,"دائ":580,"دار":1737,"داز":181,"داد":564,"طرح":391,"طرز":173,"طرÙ":300,"خوا":291,"خود":282,"طري":194,"ظم ":393,"Ø´Û ":199,"سکے":358,"خيا":180,"عت ":511,"تھي":1189,"عد ":1126,"تھا":1925,"رج ":392,"ٹي ":724,"رت ":2032,"رد ":384,"طلب":258,"طلا":420,"عض ":255,"رح ":408,"طور":1195,"ظام":601,"طنت":356,"Ø´Ú©Ù„":258,"را ":761,"دست":238,"رب ":643,"ظاÛ":223,"درس":185,"درج":475,"درا":374,"دري":711,"Ø¬Û ":949,"درم":359,"درو":190,"دون":230,"ظري":241,"ØµÛ ":469,"دور":724,"دوس":950,"دني":624,"دوں":249,"ر٠":993,"ديو":334,"ديم":368,"دين":825,"ديل":228,"ديا":819,"ديت":278,"ديد":212,"ٹر ":1167,"رس ":290,"عي ":320,"دما":182,"جے ":273,"اقو":437,"اقي":177,"اقت":219,"اقا":176,"اÙÙŠ":307,"اقع":1323,"الق":182,"الÙ":464,"الل":695,"الي":1802,"ان،":219,"امت":248,"اما":469,"الن":229,"الم":986,"الو":429,"امر":725,"الج":351,"الت":351,"الب":611,"الا":1303,"الس":179,"الر":180,"الد":643,"الح":530,"الع":407,"ري ":3579,"جگÛ":221,"اÙر":440,"اÙت":417,"ٹري":255,"انے":1223,"ايم":221,"انÛ":1162,"ايو":177,"ايس":1069,"انگ":1569,"ايش":229,"امÛ":269,"ايا":902,"ايت":316,"انک":290,"اير":272,"ØŒ ":8315,"بحي":221,"اوÛ":305,"رو ":291,"بحر":243,"اقے":283,"ديگ":275,"اني":2788,"انو":1426,"ديک":357,"رق ":406,"باً":193,"اقÛ":415,"بان":1448,"بال":507,"اند":1077,"باد":1658,"باز":180,"بار":1042,"انس":905,"امن":202,"انا":720,"انب":309,"امو":288,"انت":736,"بات":454,"امي":1316,"انج":263,"امل":979,"بائ":435,"الے":826,"بتد":219,"اوي":243,"انڈ":193,"انچ":276,"الÛ":394,"اون":198,"اول":497,"اوق":178,"اور":10590,"الک":535,"رم ":316,"ذري":244,"اوا":211,"تا ":6076,"Ø­Ú©Ùˆ":764,"Ø­Ú©Ù…":277,"رحد":298,"ردو":947,"ردي":295,"ردا":370,"رتے":586,"رتا":498,"ايک":5551,"بدا":306,"راچ":302,"ربع":318,"بدي":235,"برا":685,"ربي":855,"ذيل":186,"راÛ":513,"برط":276,"برق":294,"بري":212,"رتي":676,"راک":232,"جÛا":530,"تر ":480,"ران":1556,"ربا":206,"راع":230,"راÙ":173,"رام":330,"رال":643,"راب":283,"رائ":562,"راج":355,"رات":605,"راث":218,"رار":318,"راد":611,"راص":190,"راس":245,"ÛÙŠÚº":6226,"بعد":1053,"تح ":170,"ÛÙˆÛ”":237,"جھي":195,"بعض":255,"رآن":223,"بني":578,"بنا":831,"ÛÙˆÚ¯":285,"ÛÙˆÚº":722,"بند":385,"ا ":36261,"بلن":294,"ÛÙ†Ú†":177,"بلو":291,"بلي":191,"ÛÙ„Û’":442,"سر ":306,"بلا":185,"ب ":6862,"Ø¡ ":3203,"Ûوئ":1384,"Ûور":1098,"Ûوت":2264,"Ûوا":926,"Ûون":793,"Ûلي":355,"Ûند":711,"Ûمي":171,"ؤ ":175,"زي ":1743,"رطا":394,"Ø­ ":1752,"Ú¾Û’ ":736,"Ûلا":565,"Ø® ":788,"رسٹ":177,"د ":10666,"بوں":222,"بيٹ":246,"ست ":854,"Ø° ":232,"بين":440,"بيل":268,"سا ":510,"رست":341,"بيع":188,"ت ":16193,"بيا":321,"بير":253,"رسي":336,"سب ":865,"رجÛ":199,"Ø« ":408,"بول":436,"ج ":1790,"بلک":176,"سم ":677,"Ø¯Û ":2099,"تي ":4132,"سل ":210,"اک ":371,"ريÛ":436,"تو ":819,"ٹلي":171,"ريک":876,"دھ ":232,"روں":847,"رپ ":187,"ريع":323,"رين":690,"ريل":294,"ريÙ":299,"ريق":376,"رنے":958,"تم ":196,"ريا":1581,"ريب":696,"ريخ":486,"ريت":171,"ريز":1341,"ريش":173,"رنگ":273,"روÙ":378,"رون":314,"روم":315,"روع":324,"رور":309,"روز":221,"روس":205,"روا":481,"اپن":1352,"Ûزا":285,"Ûري":293,"زبا":959,"زار":465,"رند":247,"زاد":421,"Ûرا":229,"رمي":582,"رمن":177,"رنا":383,"زائ":318,"رما":229,"اٹل":178,"Ûتے":651,"ثر ":284,"Ø› ":306,"سط ":176,"رقي":620,"رقب":303,"Ûجر":225,"ئے ":2320,"تبد":215,"تان":2396,"تبا":381,"تار":774,"تاب":521,"اچي":318,"دے ":517,"سن ":294,"ÛÛ’ ":7219,"اں ":1697,"سو ":217,"تحر":360,"شت ":170,"تحا":210,"تحص":253,"سي ":3401,"تدا":329,"Ø§Û ":1042,"تري":474,"ترا":469,"جا ":352,"تصا":242,"جب ":430,"تصو":215,"ترک":411,"اڑي":213,"ستÛ":193,"تظا":196,"سري":335,"تعا":188,"سرح":298,"سرا":421,"تعم":1291,"تعل":965,"تعد":362,"دگي":389,"جد ":384,"شن ":295,"زما":353,"سام":409,"سال":748,"سان":757,"زند":314,"جس ":1740,"سائ":617,"زمي":343,"سات":868,"ساب":328,"ستا":2494,"تقا":197,"سجد":245,"تقر":347,"تقس":240,"ستع":1050,"زيا":694,"ستي":192,"ستو":208,"زيد":178,"اے ":199,"زير":703,"تمل":297,"ا۔ ":2384,"صر ":373,"تلÙ":484,"تما":488,"اۓ ":304,"سلا":945,"تين":302,"سمج":250,"سلي":236,"سمب":188,"سلم":718,"سما":225,"تيس":201,"تيا":780,"سلط":553,"سلس":275,"توا":248,"شي ":269,"سطح":252,"تيں":193,"پر ":4411,"اکي":196,"اکا":303,"اکس":1424,"سرے":373,"توں":404,"صد ":229,"Ù¹Û’ ":294,"اگر":397,"جن ":581,"بڑا":364,"رک ":436,"Ø¨Û ":1879,"اکھ":210,"Ø±Û ":1976,"جي ":406,"شرق":569,"شرو":345,"شري":335,"صل ":1226,"جو ":2871,"بڑي":256,"حت ":262,"ئے۔":600,"بڑے":246,"حد ":400,"Ú¾Û’Û”":835,"سوي":178,"شتر":182,"سوا":232,"سور":248,"شاع":346,"شام":808,"سمن":228,"شائ":191,"سند":323,"شاخ":186,"سمي":191,"ثلا":193,"شخص":263,"ثما":224,"ذکر":233,"اÛر":591,"اÛد":216,"سين":345,"سيم":308,"شاÛ":679,"شتم":303,"سيد":218,"بے ":441,"سيا":755,"ڑا ":500,"Ù¹Ú¾":533,"پت":249,"پا":3394,"پس":366,"پش":179,"پر":5467,"Ù¹Û’":304,"Ú¯ ":1702,"Ù¹Ú©":243,"Ù¾Ú‘":366,"پو":978,"پن":2069,"پي":2662,"پل":260,"ٹي":1294,"ٹو":369,"ٹن":221,"ٹل":297,"ٹر":1769,"ٹا":643,"Ú© ":9401,"Ù„Û’":2164,"يں،":396,"Ù…Ú¯":395,"Ù…Ú©":804,"Ù†Úˆ":840,"وي":2835,"ÙŠØŒ":686,"Ù†Ú†":605,"وو":294,"يب":1474,"يا":18357,"Ù…Û":1477,"يئ":237,"يع":1465,"يز":2169,"يس":3006,"Ù†Ú¯":3438,"يش":1424,"يص":226,"يخ":630,"يد":2958,"ير":4679,"يت":2515,"يث":342,"يج":682,"Ù†Ú©":1176,"يح":396,"ين":5944,"يو":3744,"ÙˆÚ†":363,"Ù†Ú¾":192,"يق":966,"يم":3458,"Ù†Û":3293,"يل":3575,"ÙŠÙ":992,"وپ":411,"وٹ":898,"ÙˆÚ‘":349,"Ù†Û’":8036,"ÙˆÚˆ":316,"يٹ":1722,"ÙˆÚ©":763,"Ù¾ ":1225,"يٰ":198,"ÙˆÚ¯":857,"ÙˆÛ":2577,"ÙŠÚ†":216,"ÙŠÚˆ":589,"ÙˆÚº":7039,"يپ":229,"ÙˆÛ”":276,"ÙŠÚ©":8185,"ÙŠÚ¯":581,"ÙŠÛ":6841,"ÙŠÚº":24637,"ÙŠÛ”":1652,"ÙŠÛ“":606,"ÙŠÛ’":1374,"Ùع":234,"Ùض":206,"Ùظ":975,"Ùر":2018,"Ùس":304,"Ùت":956,"Ùا":1934,"ÙŠÛا":454,"قع":1462,"قط":251,"قص":525,"قس":674,"قر":1456,"قد":958,"قت":918,"ÙÙŠ":1664,"قب":1358,"ÙÙˆ":716,"قا":2876,"ÙÙ†":259,"ÙÙ„":626,"ÙÙ‚":170,"ÙÙ¹":222,"قي":2110,"Ù„ØŒ":228,"قل":641,"قو":1321,"لق":994,"Ù„Ù":1951,"لط":721,"ÙŠÚºÛ”":3302,"لغ":238,"لع":1546,"لد":859,"لج":430,"لح":953,"لز":212,"لس":945,"لر":265,"لئ":340,"لا":9180,"لت":997,"لب":1083,"مع":2282,"مغ":639,"مص":529,"مض":257,"مط":1001,"Ù…Ù":327,"مق":1607,"مل":3799,"مت":2835,"لو":3381,"مج":828,"لي":9899,"ن،":378,"مث":441,"لم":3135,"لل":728,"مب":937,"لن":899,"ما":9656,"مز":517,"مر":3546,"مش":2018,"مس":1772,"مخ":947,"مح":1397,"مذ":278,"مد":1699,"نظ":1248,"نع":313,"نل":169,"Ù‚Û":970,"نم":856,"نق":463,"Ù†Ù":515,"Û’ØŒ ":1325,"نج":1093,"مي":23019,"نت":1838,"مو":3519,"نب":733,"نا":7139,"من":2588,"ÙÛ":638,"مم":631,"نص":409,"نش":418,"نس":2278,"نز":337,"نر":275,"ند":4576,"Ù‚Û’":375,"مپ":203,"وئ":2114,"وا":9079,"نن":295,"نو":3979,"ني":8247,"و،":187,"Ù¹ ":1461,"Ù„Ú‘":176,"وغ":354,"نٹ":588,"وع":906,"وق":1147,"ÙˆÙ":901,"ون":4062,"ول":2928,"وم":3387,"Ù„Û":2193,"وت":3089,"وب":2645,"ود":2057,"Ù„Ú©":2192,"وح":206,"وج":2206,"Ù„Ú¯":620,"وس":2774,"وز":895,"ور":18253,"وط":241,"وض":250,"وص":445,"وش":613,"ÙŠÚ©Ú¾":326,"Ú‘ ":321,"ÙŠÚ©Ù†":476,"ÙŠÚ©ÙŠ":320,"يگر":286,"Ú† ":481,"ÙŠÚ©Û":301,"Úˆ ":843,"خو":1180,"دت":297,"دا":6455,"دب":338,"خي":646,"خل":1333,"خم":176,"تÛ":894,"تھ":5631,"خط":526,"خر":474,"خد":295,"خص":603,"دو":4578,"ر،":467,"دي":7480,"دÙ":241,"دل":705,"دم":634,"ذا":327,"دن":1041,"تے":3362,"دع":168,"دد":552,"در":3958,"دش":266,"دس":695,"جي":1296,"جو":4658,"حت":458,"جن":2503,"حا":2227,"حب":298,"جل":349,"بÛ":2825,"جم":1355,"بھ":4084,"ا۔":3031,"اۓ":335,"اے":213,"جس":2866,"جز":533,"جر":1045,"جد":723,"بک":601,"خت":1742,"حي":1318,"تک":1171,"حم":1392,"خا":1967,"خب":324,"حو":294,"حق":534,"حل":497,"حض":792,"حص":974,"بے":458,"حر":1330,"حس":621,"حد":976,"تÙ":294,"تم":1316,"تل":864,"تق":1058,"تو":2257,"بچ":213,"ثا":431,"تن":759,"تج":398,"تح":1528,"تر":2642,"تخ":631,"تد":422,"اڑ":396,"تش":356,"تص":699,"تس":246,"تظ":210,"تع":3085,"اں":1731,"ثل":246,"جا":8260,"ثم":243,"اÛ":3738,"جب":952,"ثي":412,"تي":6367,"اک":3503,"ثر":502,"اگ":766,"بڑ":994,"ئے":3028,"ؤں":259,"بغ":203,"بع":1847,"بن":3272,"بم":183,"بل":2052,"بق":773,"بد":925,"اً":548,"بج":198,"بح":566,"بت":774,"اي":9647,"او":12882,"بط":281,"بص":181,"بز":199,"بس":224,"اÙ":281,"بر":3415,"اپ":1892,"اٹ":526,"تا":10999,"اچ":510,"تب":1015,"بو":1546,"ت،":328,"بي":4832,"ئد":211,"ئر":536,"ئش":454,"ا،":402,"اء":804,"اؤ":588,"ئل":407,"ائ":7576,"ئم":430,"ئع":221,"از":2184,"ار":13604,"اد":7175,"اض":766,"اص":2408,"اش":1157,"اس":13319,"ات":12391,"اب":5157,"ئن":668,"اخ":1510,"اح":1946,"اج":1405,"اث":517,"ئي":5602,"اÙ":2468,"اق":3776,"ام":10852,"با":7594,"ان":23472,"ال":15347,"اع":2513,"اغ":318,"اط":844,"اظ":619,"آپ":794,"آئ":413,"آب":1377,"آت":374,"آج":214,"آخ":195,"آر":259,"آس":262,"آز":333,"Ø¢Ù":182,"آل":275,"آم":177,"آن":518,"آي":281,"عے":301,"Ù° ":195,"عÛ":1012,"Ø·Û":234,"غي":910,"غل":246,"عي":1223,"غر":737,"صے":202,"عق":239,"عل":4649,"صÛ":487,"عم":2601,"غا":645,"عن":1246,"عو":588,"عث":370,"ظي":492,"عت":818,"عد":2145,"عز":175,"عر":1942,"عظ":634,"عض":358,"عا":2408,"عب":841,"ظم":515,"Ø´Û":2846,"ظا":851,"طن":541,"سÛ":287,"طل":743,"سے":11317,"ظر":549,"Ø´Ú©":686,"طي":706,"طو":1493,"ضم":182,"زÛ":344,"ضل":1190,"رے":1402,"طر":1375,"ضي":496,"سک":2597,"طح":262,"طا":2021,"طب":814,"ضو":288,"سپ":375,"صل":1848,"صÙ":401,"صط":357,"سٹ":824,"ضر":887,"صو":2504,"رÛ":3575,"صن":434,"ضا":582,"صي":766,"دے":538,"شع":464,"رک":3328,"صح":404,"صد":1147,"صر":962,"رگ":411,"شم":1789,"Ø°Û":258,"صا":794,"شن":774,"صب":345,"شو":358,"شي":1319,"سع":252,"سط":727,"دگ":457,"سÙ":448,"دھ":774,"رپ":455,"رٹ":425,"رڈ":213,"سي":6156,"شت":955,"رچ":305,"سو":1823,"شا":3056,"سن":1275,"دÛ":2389,"سم":1996,"سل":3485,"شر":1691,"شد":243,"شخ":304,"Ø°Ú©":274,"سب":1244,"سا":5309,"زن":603,"ست":5770,"زو":460,"زم":950,"زل":234,"سر":2705,"سج":282,"زي":3808,"دک":188,"رس":1755,"رش":298,"رر":201,"رز":482,"جے":273,"رط":461,"رص":252,"رض":357,"رل":427,"رق":1634,"رÙ":1286,"رو":5304,"زب":1100,"رن":2342,"زا":1886,"رم":1656,"ري":12753,"زر":534,"ذر":554,"جگ":285,"رآ":320,"جھ":599,"جÛ":1619,"رب":2493,"را":9267,"رت":4343,"رج":1190,"ذي":412,"رخ":287,"رح":1014,"Ø­Ú©":1119,"رد":2466,"Ù ":3475,"ع ":4219,"Ú‘ÙŠ ":543,"غ ":230,"ص ":761,"ض ":697,"Ø· ":839,"ظ ":1429,"ر ":38086,"ز ":2573,"س ":11232,"Ø´ ":1550,"Ù ":847,"Ù‹ ":542,"ÙŠ ":57056,"Ù† ":18794,"Ùˆ ":15262,"Ù‚ ":2735,"Ù… ":14225,"Ù„ ":13265,"ينÛ":361,"ينے":248,"يوں":1103,"ÙˆÛ ":2016,"يقي":265,"يلا":274,"يلي":664,"يما":422,"يلو":184,"ينا":221,"Ù†ÛÙŠ":1033,"يمي":391,"Ù†ÛÙˆ":438,"يٹ ":181,"يني":579,"يقÛ":243,"وچس":192,"يور":512,"يوا":288,"يون":715,"ينڈ":257,"ÙˆÚº ":6861,"ÛÛ’":17450,"يٰ ":189,"وٹي":197,"يعے":209,"ÙˆÚ¯ ":226,"يرÛ":738,"يشي":298,"يشن":306,"يسي":670,"Ù†Ú¯ÙŠ":216,"Ù†Ú¯Ù„":243,"يسو":203,"يسر":205,"Û’ØŒ":1342,"نگر":1405,"يزي":1326,"نگا":269,"يسا":524,"يرو":314,"يري":407,"يعن":649,"يسے":586,"۔ا":238,"يثي":187,"يا۔":1172,"نکا":269,"Û’Û”":10861,"يتي":187,"ياں":353,"يرا":565,"Ù†Ú©Û":250,"يدا":1270,"Ù†Û’ ":7909,"يال":283,"يان":856,"يبا":352,"يام":192,"ياس":748,"يار":881,"ياد":1222,"ياض":232,"ياء":241,"يات":2312,"ياب":173,"يائ":508,"Ú©Ú†":345,"Ú©Ù¹":398,"Ú©ÙŠ":17718,"ÙŠÛ” ":1247,"کس":3186,"Ú©Ø´":375,"کر":6688,"کز":505,"Ú©Ø«":328,"کت":1905,"Ú©Ùˆ":8220,"Ú©Ù†":1311,"Ú©Ù…":1336,"Ú©Ù„":1933,"کئ":359,"کب":357,"کا":13105,"ÙŠÛ“ ":585,"ÙŠÛ’ ":1282,"Ú¯Û":320,"Ú¯Ú¾":468,"Ú¯Ù„":520,"Ú¯Ù†":309,"Ú¯Ùˆ":1092,"Ú¯ÙŠ":2325,"گز":223,"گر":3536,"گئ":1042,"گا":1385,"Ú©Û’":18417,"Ú©Ú¾":2829,"Ú©Û":6908,"ں،":524,"Ú¯Û’":188,"ÙŠÛ ":6011,"ÙŠÚˆÙŠ":227,"Ú¾Û’":1744,"Ûز":291,"Ûر":3373,"Ûج":298,"Ûت":1501,"Ûد":644,"Ûا":4675,"Ûب":255,"ÛÙŠ":8193,"ÛÙ„":1738,"ÛÙ…":1207,"ÛÙ†":1510,"ÛÙˆ":9694,"ÚºÛ”":3376,"Ú¾Ù…":206,"Ú¾Ù„":258,"Ú¾ÙŠ":5556,"Ú¾Ùˆ":1393,"Ú¾Ù†":692,"ÛØŒ":328,"ھر":965,"ÙŠÚº ":20810,"ھا":3850,"ھت":572,"Úº ":29650,"ڈر":227,"Ú†ÙŠ":975,"Ú†Ù†":411,"Ú†Ùˆ":627,"ÙˆÛا":172,"ڈا":533,"Ú†Ú©":371,"Ú†Û":230,"Ú†Ú¾":1116,"ÚˆÙˆ":318,"ÚˆÙŠ":856,"Ù¾Ú¾":684,"Ù¾Û":1407,"چس":237,"چا":849,"چت":269,"Ú†Ù„":282,"ڑا":619,"Ûƒ ":184,"Û‚ ":441,"Ú‘Û’ ":367,"يٹر":982,"Ú‘ÙŠ":670,"ÙŠÚ© ":5968,"Ú¾ ":2275,"Ú†Û’":173,"Û ":29676,"ÙˆÚ¯Ùˆ":199,"Ú‘Ú©":206,"Ú‘Ú¾":467,"Ú‘Û’":384,"Û’ ":59880,"Û“ ":1150,"Û” ":15851,"ÙˆÛ” ":200,"وز ":174,"ور ":14043,"ود ":1161,"ÚˆÙŠ ":274,"Ù„Ú¯ ":206,"وس ":423,"چين":228,"نما":532,"وع ":503,"نيا":1482,"نوي":380,"نون":235,"نور":347,"نوب":664,"نوا":373,"نٹ ":234,"ÙˆÙ ":378,"نيÛ":422,"نيو":458,"نوں":1111,"وم ":918,"Ù„Û ":1961,"ون ":1017,"چند":174,"ول ":1088,"Ù†Úˆ ":285,"وي ":1068,"ÙŠØŒ ":669,"Ù†Ú† ":177,"مغر":519,"معل":267,"معن":336,"معر":258,"مشÛ":684,"معا":596,"چست":201,"مقا":854,"مقد":219,"چان":177,"ملت":253,"ملا":357,"چتر":233,"Ù‚Û’ ":368,"منا":208,"نائ":404,"مند":339,"نات":285,"منت":213,"نار":304,"مما":299,"ملي":227,"موا":174,"ملک":619,"موج":660,"مور":225,"موس":304,"موع":240,"نام":2044,"نان":513,"ناي":217,"نتق":217,"نتظ":204,"نتخ":209,"مون":289,"موم":337,"مول":250,"ميل":269,"ميد":233,"مير":641,"ميا":812,"ميت":203,"نتي":206,"نجا":562,"مين":762,"ميٹ":952,"موں":292,"ندا":677,"ميں":16302,"ندو":448,"ندر":713,"ندي":569,"چار":237,"نسا":448,"Ú†ÙŠ ":376,"وا ":736,"ندگ":273,"ندÛ":249,"نسي":397,"نسل":289,"ندھ":311,"وب ":577,"وت ":377,"نظا":416,"نظر":493,"وج ":244,"نظي":172,"Ù„Ú© ":878,"ونا":319,"ومي":882,"ونس":207,"وما":454,"ولي":616,"ولو":168,"ومت":782,"يع ":214,"ولا":396,"وقت":472,"ÙˆÙا":282,"يش ":423,"يس ":618,"Ù†Ú¯ ":881,"ويں":364,"يق ":261,"ÙŠÙ ":611,"Ù…Ú©Ù…":188,"ونے":813,"Ù†ÚˆÙŠ":216,"Ú†Û ":203,"ونک":263,"Ú†Ú¾ ":368,"وني":567,"وٹ ":170,"ونو":241,"ÙˆÚˆ ":203,"يو ":296,"Ù†Û ":1535,"يم ":2131,"ين ":3424,"مگر":341,"يل ":1806,"Ù„Û’ ":2120,"وئے":873,"واں":205,"وتي":759,"وتا":1184,"وبÛ":880,"وجي":169,"وجو":858,"وا۔":228,"وار":699,"واز":279,"واد":234,"واج":227,"وئي":1076,"واب":205,"وائ":356,"وبي":420,"واي":171,"واق":1340,"وال":2546,"وان":598,"وبا":263,"وام":369,"وري":875,"Ù…Û ":911,"وست":275,"لگا":240,"وزي":317,"يا ":8493,"وسر":713,"وسط":295,"ورپ":286,"ودÛ":176,"وسي":366,"يب ":717,"ورÛ":195,"Ù„Ú©ÙŠ":197,"وتے":500,"ودي":279,"ورا":750,"وجÛ":615,"ورس":204,"ورت":506,"Ù„Ú©Û":196,"ورن":218,"Ù„Ú©Ú¾":453,"وغي":292,"يد ":1164,"ير ":2092,"يز ":500,"يت ":1838,"يج ":176,"يح ":219,"يخ ":423,"لد ":289,"قصب":216,"لت ":358,"لا ":1251,"قسم":285,"لب ":395,"قسي":239,"�":423,"لق ":599,"Ù„Ù ":570,"Ú†Ú¾Ùˆ":396,"قوں":260,"ما ":498,"لم ":1252,"لع ":962,"قيق":283,"قيا":359,"قوم":305,"قوا":311,"قل ":209,"Ùرا":633,"Ùرو":181,"Ùري":361,"قي ":950,"Ù„ØŒ ":225,"Ùيص":180,"قبÛ":284,"قدر":190,"قدي":366,"قري":696,"قرآ":222,"قرا":218,"قال":197,"قائ":507,"قاب":316,"قات":245,"Ùلم":218,"Ùوج":255,"قبو":176,"قبل":223,"قان":205,"قبا":188,"قام":703,"ÙÙ¹ ":202,"لما":635,"ماع":315,"لمي":331,"مات":656,"مار":1346,"ماد":258,"لند":373,"مائ":379,"Ù„Ùظ":904,"Ù„Ùا":303,"نس ":426,"لعÛ":239,"ند ":917,"لطا":204,"مي ":1920,"لطن":355,"مطل":234,"ني ":4359,"و، ":185,"مطا":719,"مصن":197,"مصر":194,"مرک":583,"نو ":281,"مذÛ":183,"مشر":609,"مسل":679,"مشت":387,"مست":254,"مسج":242,"مسا":197,"Ù‚Û ":928,"مري":646,"مجھ":254,"مرا":743,"مرب":331,"مرت":186,"مدي":169,"ليے":1014,"ليۓ":470,"مدد":185,"ليک":538,"ليÛ":571,"مخت":598,"لوگ":423,"محم":603,"لوں":448,"لوچ":226,"لين":425,"مجم":256,"لنے":191,"ليت":222,"متي":175,"ليا":836,"ماÛ":260,"ليم":656,"مثل":227,"لوي":317,"لوم":894,"للÛ":672,"متح":211,"متع":399,"مال":2414,"مام":560,"مان":1742,"مبا":175,"ماي":231,"مبر":417,"مر ":315,"مد ":873,"لو ":198,"Û’Û” ":8070,"مت ":1234,"لي ":3595,"ن، ":362,"لسل":386,"نب ":260,"نا ":1656,"من ":398,"ÙÛ ":313,"نت ":572,"لدي":343,"لحک":303,"مل ":1925,"لتي":176,"لاک":279,"لئے":321,"لاÛ":228,"لتا":182,"لحا":247,"لاح":555,"لاد":170,"لاز":192,"لائ":397,"لات":751,"لاق":1241,"لاÙ":367,"مع ":244,"لاو":376,"لام":1294,"لان":427,"لبا":312},"n_words":[1602570,1999510,1324903],"name":"ur","type":"arab"} \ No newline at end of file
diff --git a/contrib/languages-data/vi.json b/contrib/languages-data/vi.json
new file mode 100644
index 0000000..d3a4d75
--- /dev/null
+++ b/contrib/languages-data/vi.json
@@ -0,0 +1 @@
+{"freq":{"D":18934,"E":10094,"F":9985,"G":23207,"A":42579,"B":48257,"C":72224,"L":45665,"M":47574,"N":70917,"O":10000,"H":57302,"I":15743,"J":5371,"K":22108,"U":5400,"T":104616,"W":8071,"V":30021,"Q":14744,"P":61692,"S":43824,"R":17604,"Y":3509,"X":6207,"Z":3364,"f":32992,"g":525146,"d":205922,"e":428748,"b":182777,"c":671453,"a":692878,"n":1382200,"o":437498,"l":442210,"m":420259,"j":4762,"k":100510,"h":937660,"i":759263,"w":22119,"v":178461,"u":410719,"t":952446,"s":286409,"r":444790,"q":33677,"p":174320,"z":16996,"y":192811,"x":29082,"²":689,"ÃŽ":330,"É":501,"Ã":3556,"Â":2373,"Ã":1288,"ß":648,"Ú":1274,"Ô":1751,"í":49339,"ì":32480,"ê":66073,"é":38397,"è":3862,"ç":233,"ä":808,"ã":19808,"â":112923,"á":116051,"à":337783,"ü":3187,"ý":5197,"ú":10002,"ù":30808,"ö":2319,"ô":98885,"õ":1117,"ò":13544,"ó":78081,"ñ":390,"Ä‘":198318,"Ä":41649,"ă":37793,"Ä":557,"Ä©":9562,"Ä«":249,"Å":772,"Å©":5403,"Å«":650,"Æ°":147894,"Æ¡":25607,"Ì":204,"ο":236,"ι":146,"λ":139,"α":217,"Ï‚":203,"Ï":170," l":242575,"ÑŒ":159," m":173509," n":244497,"Ñ":194," o":12009," h":134731," i":13655," j":644,"Ñ‹":142," k":76050," d":93354," e":7007," f":8319," g":49741,"ч":190,"Ñ€":638,"Ñ":522," a":22488," b":142456,"Ñ‚":441,"у":252," c":262807," y":1958," x":17955," z":365," u":1259," t":566783," w":4693," v":156401," q":28586," p":59115," s":94989," r":14869," J":5319," K":22031," H":57162," I":15633," N":70771," O":9935," L":45508," M":47408," B":48077," C":71972," A":42454,"С":156," F":9904," G":23091," D":18825," E":10050,"л":506,"к":551," Z":3238,"й":211," Y":3492,"и":885," X":6141,"о":889,"н":659,"м":235,"г":168," S":43565,"в":508," R":17493," Q":14713,"б":149,"а":1192," P":61533," W":8016," V":29925," U":5377,"е":792,"д":263," T":104253," á":1499," â":3662," í":541," ô":2220," ý":503," ă":627," Ä":41607," Ä‘":198156," Â":2373," Ã":3556," É":500," ÃŽ":330," Ô":1748," Ú":1274," Ã":1277," Æ°":1144,"ÙŠ":271,"Ù„":264,"Ù…":187,"Ù†":142,"ا":398,"ر":178,"A ":2159,"F ":494,"Da":2569,"Cu":1591,"Cy":963,"Cl":1245,"Co":10944,"Cr":1746,"Ce":2576,"Ch":19039,"Ci":1005,"G ":637,"Ec":908,"Ed":361,"Ea":530,"Du":1267,"Dy":152,"Do":2470,"Dr":1247,"De":2217,"Di":2894,"Bà":792,"Bá":858,"Fe":902,"H ":455,"Fa":1831,"Eu":2126,"Ex":464,"Er":617,"Et":367,"Es":894,"En":771,"Em":460,"Ep":314,"Ei":335,"El":872,"Cá":2732,"Ge":2583,"Câ":290,"Cà":157,"Ga":3052,"I ":2174,"Fu":749,"Fr":2327,"Bí":387,"Bì":1773,"Fo":1217,"Bé":191,"Fl":1104,"Fi":1024,"B ":642," С":156,"C ":1227,"Av":428,"Au":2997,"Ar":5387,"Aq":1658,"At":1067,"As":1434,"D ":745,"Ba":13610,"Az":1190,"Ay":139,"Ae":265,"Af":962,"Ag":750,"Ah":192,"Ab":577,"Ac":1193,"Ad":611,"Am":2029,"An":10483,"Ap":626,"Ai":1117,"Ak":164,"Al":6955,"By":154,"Bu":3220,"Br":4012,"Ca":14082,"E ":3496,"Bh":176,"Bi":2488,"Be":3517,"Bo":4388,"Bl":876,"Gò":247,"Ku":658,"Gö":233,"Ky":391,"Kn":144,"Kl":231,"Kr":781,"Ko":952,"Hã":340,"Hà":3871,"Há":6912,"Le":5532,"Li":5716,"N ":1502,"La":11255,"Lu":1591,"Hó":455,"Hò":671,"Ly":614,"BÆ°":341,"Hé":321,"Lo":11096,"Hì":160,"Me":5622,"Mi":7578,"CÆ¡":380,"O ":859,"Ma":14564,"CÆ°":440,"My":979,"Mu":2726,"Mo":6001,"Nh":15252,"Ni":3602,"Ng":8870,"Ne":5246,"Na":14732,"P ":1136,"DÆ°":2196,"Æ°u ":1732,"Ny":327,"Nu":290,"No":7260,"Ok":174,"Ol":595,"Om":201,"On":483,"Oh":203,"Oi":748,"Oc":485,"Od":183,"Oe":270,"Oa":270,"Ob":773,"Gi":6635,"Gh":326,"Gl":496,"Gr":4171,"Go":1400,"Gu":2462,"Gy":286,"Cô":3480,"Có":402,"Cú":325,"J ":219,"Ha":7544,"Dâ":1283,"He":2483,"Hi":2709,"Ho":13976,"Hu":4398,"Hy":1927,"Dô":340,"K ":324,"Ib":265,"Id":176,"Ic":220,"Im":190,"In":3623,"Il":428,"Is":1691,"It":2092,"Ir":1003,"Ja":1797,"L ":642,"Ji":230,"Je":710,"Jo":1104,"Ju":1037,"Ka":2594,"M ":626,"Kh":4970,"Æ°ng":2590,"Ki":2990,"Ke":1180,"Ut":638,"Ur":404,"Um":166,"Un":919,"Uk":408,"Ul":140,"Ug":456,"W ":203,"Ty":201,"Tw":155,"Tu":2972,"Tr":21381,"To":2690,"Th":42787,"Ti":3875,"Te":2952,"Ta":3822,"V ":758,"Sw":579,"Sy":613,"St":3687,"Su":1921,"Wo":921,"Wi":1647,"Wh":297,"Sé":358,"Sè":329,"Wa":2024,"Sâ":534,"Sá":354,"We":2017,"Sà":247,"Y ":322,"LÆ°":806,"Vo":1915,"Vu":318,"Vi":16219,"Ré":208,"X ":401,"Va":2214,"Ve":2267,"Uy":394,"Lă":215,"Mã":1030,"Má":214,"Lý":664,"Lü":151,"Mé":245,"Pt":637,"Pu":1475,"Pr":3920,"Ps":557,"S ":1842,"Py":3299,"Pe":2603,"Là":800,"Lã":279,"Pf":1720,"Lâ":833,"Pa":7810,"Lé":142,"Pl":1384,"Po":3931,"Lê":927,"Pi":2275,"Ph":31098,"Os":655,"Ot":301,"Ou":271,"Ov":238," ا":171,"Op":394,"Or":2938,"R ":429,"Kô":237,"Se":3444,"Sc":3122,"Si":2745,"Nă":720,"Sh":1345,"Sm":208,"Sl":522,"Sk":307,"Sr":367,"Sp":2000,"So":3317,"Ru":1170,"Nô":330,"Ry":155,"Nó":11015,"U ":337,"HÆ°":909,"Sa":11442,"Re":2085,"Ri":2194,"Rh":4441,"Ro":3835,"Qu":14208,"Mô":839,"T ":649,"Mù":252,"Ra":1904,"Mü":183,"SÆ¡":2030,"SÆ°":226,"Yê":502,"Xã":2079,"Wü":541,"Xô":549,"Æ°a ":1620,"Xí":179,"VÄ©":456,"b ":1444,"a ":226759,"TÆ°":1027,"VÅ©":589,"Tù":160,"Tú":283,"Tô":962,"Xy":153,"NÆ°":140,"Ye":349,"Ya":435,"Yp":239,"Yo":874,"SÄ©":507,"Yv":187,"Yu":292,"MÆ°":225,"Só":240,"Sô":413,"Tâ":9624,"Tà":542,"Xe":229,"Tá":302,"Sü":308,"Xa":216,"Tê":676,"Xi":420,"Tò":164,"Xu":1093,"Tí":259,"Vù":202,"Vă":1172,"Za":1272,"Và":236,"Ze":739,"Zh":185,"Zi":458,"Vâ":484,"TÄ©":188,"Zu":199,"Võ":191,"i ":212782,"bó":2113,"cà":270,"gd":295,"cá":22579,"ge":11280,"câ":2231,"bú":210,"ga":12467,"bé":224,"fl":1616,"bã":190,"fg":215,"ff":1242,"fi":2497,"fs":160,"fr":3324,"fu":1164,"ft":684,"fo":4267,"bê":1228,"bí":976,"bì":10910,"j ":155,"cù":1871,"cú":386,"gy":681,"có":47444,"gw":161,"cô":5046,"dâ":28874,"dã":502,"dà":2537,"he":41577,"hb":191,"ha":43156,"gn":5754,"gm":577,"gl":1738,"gk":293,"gi":33763,"bă":263,"gh":11779,"gg":570,"cò":3292,"gu":10662,"gt":510,"gs":1016,"gr":2654,"go":8645,"dt":805,"du":4489,"dw":715,"dy":507,"g ":364697,"ea":15752,"eb":3649,"ec":8564,"ed":11541,"de":24490,"dd":556,"dg":298,"df":167,"di":32181,"dh":710,"dk":145,"dm":293,"dl":1043,"do":14691,"dn":264,"ds":1755,"dr":3533,"ew":3323,"ex":3581,"eu":6762,"ev":2827,"ey":3123,"ez":1216,"fa":5205,"h ":178273,"bà":2398,"bá":2351,"fe":2871,"eh":679,"eg":4460,"ef":1217,"ee":4546,"el":24442,"ek":799,"ej":311,"ei":13045,"ep":4148,"eo":14079,"en":49087,"em":10013,"et":12592,"es":40569,"er":51401,"eq":167,"ca":30360,"XÆ°":246,"e ":134279,"bw":303,"by":1138,"bs":628,"br":3660,"bu":6071,"bt":170,"bn":144,"bo":4357,"bl":1961,"bf":332,"bh":146,"bi":46026,"bb":920,"be":11159,"db":293,"da":38163,"f ":9188,"cy":1022,"cu":9388,"ct":7017,"cs":417,"cq":226,"cr":3098,"co":14606,"cm":248,"cn":200,"ck":4238,"cl":1934,"ci":14195,"ch":129179,"ce":13915,"cc":1731,"VÆ°":1101,"c ":282171,"az":1913,"ay":20261,"ba":27162,"d ":28581,"at":21123,"as":21308,"ar":46226,"aq":534,"ax":1123,"aw":1934,"av":3839,"au":17184,"ak":3208,"al":35399,"ai":30021,"aj":1274,"ao":21790,"ap":5292,"am":43172,"an":108096,"ac":18572,"ad":10374,"aa":985,"ab":4752,"ag":8446,"ah":2616,"ae":34413,"af":1148,"nu":5357,"nt":26825,"ns":10879,"nr":434,"nq":250,"np":246,"no":9747,"hÄ©":3597,"nn":7993,"q ":277,"nz":1727,"dÆ°":2566,"ny":2595,"nx":314,"nw":217,"nv":655,"oe":1614,"of":8008,"oc":9998,"od":5406,"oa":17234,"ob":3593,"ké":466,"om":14342,"kê":1778,"on":100425,"ok":1733,"kè":139,"ol":16267,"oi":9431,"oj":178,"og":4447,"oh":1218,"ot":8950,"m²":659,"hÅ":191,"os":13276,"ov":4515,"ou":16281,"kì":215,"op":11878,"oo":2896,"or":34485,"oq":166,"gÅ©":395,"kí":1225,"r ":23753,"ox":666,"ow":4522,"oz":1526,"oy":925,"là":155485,"lá":810,"pe":13642,"pf":220,"lâ":707,"lã":1014,"pa":8707,"ký":795,"lè":430,"pl":2303,"lé":393,"lê":819,"po":6064,"ph":59995,"pi":7555,"lo":48870,"ln":332,"hê":913,"lm":1499,"hé":1262,"hè":446,"ll":28312,"ls":5922,"hí":22956,"lp":3661,"hì":4519,"hó":4714,"lw":160,"hò":5943,"lv":1420,"lu":8343,"lt":3135,"bÆ°":9431,"lz":2455,"hö":169,"ly":7272,"hô":9522,"hú":4659,"hù":1344,"o ":85116,"iß":177,"ià":688,"hü":788,"ma":19960,"mb":6117,"mg":163,"hă":499,"me":15863,"iá":6621,"cÆ¡":2439,"iè":809,"ml":141,"mi":19185,"mn":579,"iê":19338,"mm":3914,"ié":147,"mp":6181,"mo":8059,"mt":201,"ms":1171,"mu":3841,"iô":221,"ió":194,"cÆ°":776,"my":849,"p ":57766,"iú":305,"na":36440,"nb":2672,"nc":9194,"nd":29108,"ne":33625,"nf":985,"ng":422545,"nh":179739,"ni":21672,"nj":602,"nk":2141,"nl":3093,"nm":685,"ju":396,"jo":698,"ki":21387,"kh":39807,"gã":188,"gâ":871,"ke":3413,"gá":300,"gà":10665,"ka":4069,"m ":167316,"gó":565,"gô":2554,"ky":750,"ks":1201,"kt":384,"ku":945,"ko":1602,"cÅ©":2937,"kr":861,"kk":293,"kl":1148,"km":4687,"kn":974,"li":29575,"lh":510,"lk":1201,"le":28117,"há":50648,"hà":29720,"ld":3791,"hã":1199,"lg":997,"hâ":38191,"lf":811,"la":39723,"lc":719,"lb":2760,"n ":440934,"hr":3391,"hs":2273,"dò":649,"hw":1075,"ht":2057,"hu":96505,"hk":147,"hh":235,"că":337,"hi":59002,"hn":1134,"ho":33760,"hl":2514,"hm":909,"dé":1520,"id":34385,"ic":24617,"ib":3478,"ia":42159,"ih":435,"ig":7165,"if":3529,"ie":18418,"hy":2981,"dù":2108,"k ":5941,"iq":1214,"ir":10741,"is":30398,"it":19216,"iu":2880,"iv":3060,"iw":152,"eó":2057,"ix":1054,"ii":6395,"ij":559,"ik":1883,"il":38718,"im":11936,"in":70002,"io":10270,"ip":6214,"je":426,"ji":826,"iz":1625,"iy":348,"l ":17786,"ja":1818,"nÆ¡":1994,"să":488,"xi":2236,"tê":7401,"xo":824,"té":201,"tì":8961,"xp":151,"tí":18614,"tò":332,"xt":572,"xu":3759,"mÆ°":601,"sô":2011,"ww":348,"só":466,"z ":4461,"xc":234,"xa":2141,"tâ":11580,"tá":3122,"xe":1418,"tà":3376,"wh":1250,"ră":636,"wi":4079,"sè":470,"wl":199,"sé":203,"wn":1808,"sê":269,"wo":1048,"ws":516,"wt":183,"rò":801,"rõ":266,"rô":669,"lÆ°":3756,"rö":667,"rù":551,"y ":123996,"rú":838,"rü":313,"wa":4838,"sâ":1032,"sá":4087,"we":3258,"rè":166,"ré":3552,"vi":14749,"râ":232,"rã":310,"vu":13359,"vr":721,"rì":2865,"rí":1219,"rê":17007,"vo":1414,"uz":955,"uy":40068,"ux":1932,"uw":188,"uv":1489,"ve":8349,"rá":1135,"rà":746,"va":5946,"x ":5085,"ui":7198,"uj":271,"uk":987,"ul":11719,"ue":8283,"uf":658,"ug":2145,"uh":313,"mÅ©":152,"ur":22852,"us":22737,"ut":11022,"um":8525,"un":37448,"uo":379,"up":2448,"ty":3384,"tz":1632,"tu":10835,"tt":6267,"tw":506,"tv":273,"ub":3534,"ua":12317,"ud":3186,"uc":4195,"w ":3929,"to":19941,"tn":373,"tm":750,"tl":2439,"ts":3488,"tr":168490,"lÅ©":299,"tp":612,"tg":165,"tf":340,"te":34125,"ti":45992,"th":244395,"v ":848,"tb":456,"tc":539,"ta":22518,"su":6738,"sv":185,"ss":11481,"st":25304,"sy":900,"sw":1422,"sl":2460,"sk":1966,"sn":3570,"sm":2007,"sp":6570,"so":7204,"sr":482,"sq":267,"oà":40273,"sd":636,"oß":224,"sc":6190,"sf":434,"se":16078,"oá":2469,"nă":29159,"sh":6976,"sg":779,"oã":161,"si":19668,"hÆ°":18279,"rz":821,"u ":92839,"nú":1257,"sa":12158,"sb":1450,"rr":9518,"rs":7765,"rt":11056,"ru":33603,"rv":1083,"rw":648,"nó":3472,"nô":498,"ry":4815,"rq":206,"rp":2369,"lÄ©":607,"ro":79309,"nê":766,"rn":10149,"né":3611,"rm":6127,"rl":2540,"nç":154,"rk":3126,"hÆ¡":3447,"ri":45411,"rh":1015,"rg":11132,"nâ":296,"rf":2041,"re":31859,"rd":9664,"nà":32104,"า":167,"rc":4893,"rb":3024,"ra":48559,"mù":496,"t ":240212,"gÆ°":23033,"mó":270,"mô":1896,"qu":33189,"ร":145,"mé":23563,"mì":570,"má":2261,"mã":976,"lý":2656,"mà":2647,"s ":83922,"lú":457,"lô":11642,"px":1137,"py":386,"pt":4331,"pu":3257,"lò":168,"pp":2831,"lí":1363,"pr":2803,"ps":2406,"hÅ«":297,"vÆ°":1017,"zè":171,"vÅ©":680,"tÆ°":4765,"sÆ¡":283,"yê":5321,"sÆ°":822,"xá":866,"xâ":942,"xã":6617,"rÆ°":7067,"xé":140,"vÄ©":1312,"xí":212,"vă":2376,"vù":22603,"zz":263,"vâ":231,"zh":389,"zi":1581,"uý":467,"zb":261,"và":46943,"ze":2177,"za":3062,"yz":169,"vò":489,"võ":378,"vô":1031,"zu":1605,"zo":1512,"zn":191,"ví":191,"vì":1173,"zl":195,"yg":495,"yh":272,"tă":382,"ye":2354,"uá":1249,"uâ":5412,"yf":154,"yc":1911,"yd":753,"ya":3754,"yb":311,"tú":167,"tù":418,"nÆ°":35853,"tô":1034,"xy":294,"yx":252,"uô":12078,"yu":633,"yt":872,"ys":3871,"yr":5371,"yp":2222,"sÄ©":1785,"yo":1215,"yn":1357,"uê":505,"ym":1622,"ué":319,"yl":3067,"yi":713,"xÆ°":530,"yÅ«":142,"² ":668,"à ":1673,"Ãv":245,"Ão":1420,"Âu":2129,"Âm":150,"ÃŽl":330,"àn":34510,"ào":9571,"àm":2558,"ài":40656,"ã ":15139,"ày":41288,"àu":3190,"ám":1101,"án":28484,"áo":4300,"áp":27542,"ái":6121,"ác":31049,"áy":2255,"áu":552,"át":6111,"âm":6839,"ân":71359,"âu":6947,"ât":496,"ây":27219,"ãn":2862,"ão":485,"ãi":672,"ãy":559,"à ":205928,"á ":8250,"ße":294,"Úc":1243,"à ":1285,"Ôn":1610,"アアア":304,"ôi":2574,"ôm":11781,"ôn":48478,"óa":2709,"õ ":900,"ói":993,"óc":731,"óp":293,"ón":6670,"óm":1531,"ô ":34032,"òa":5612,"òm":318,"òn":6337,"ó ":64834,"ña":207,"ò ":1069,"ín":13049,"ít":636,"ìn":20626,"ìm":8528,"íc":17864,"ía":12239,"í ":5056,"ên":48870,"êm":9168,"êu":3630,"éz":154,"ì ":3075,"él":169,"éo":764,"ép":2326,"ém":434,"én":3441,"és":341,"ét":23592,"ér":825,"év":213,"éb":181,"éd":152,"éc":499,"ée":3394,"ég":181,"èn":257,"èo":280,"èr":1433,"ès":603,"èv":444,"ê ":4050,"é ":1396,"Äưể":826,"è ":371,"är":168,"ăm":28997,"ăn":8563,"ăk":142,"đưể":28774,"Än":253,"ý ":5155,"ể":1508247,"ün":249,"ür":1564,"üt":177,"üc":207,"üd":322,"ùn":27807,"ùi":235,"ùa":1067,"úp":776,"ún":1686,"úy":219,"út":637,"úa":606,"ùy":248,"úi":1359,"úc":3127,"ù ":1286,"ú ":1439,"ôt":1632,"öt":180,"ör":366,"ös":634,"ön":312,"öl":155,"Ä‘Ä©":597,"đó":3828,"đô":27703,"đú":182,"đă":220,"ÄÆ°":848," ể":80643,"Ä‘Æ°":29695,"Ä‘Æ¡":1908,"tưể":2647,"Äa":443,"Äo":196,"Äi":1420,"Ä‘e":478,"Ä‘a":1708,"Ä‘o":2470,"Ä‘i":17430,"Ä ":295,"Äì":322,"Äà":1638,"Äá":165,"Äâ":3555,"đí":392,"đì":460,"đê":8064,"đã":4348,"Äă":188,"đá":3730,"đâ":1604,"đà":1035,"Äô":12692,"Ä‘u":489,"Cể":8674,"Dể":759,"Bể":9876,"Hể":9168,"Gể":170,"Lể":3634,"Kể":5925,"Ä© ":4602,"Ä©a":3526,"Ä©n":1420,"Ä©nh":1417,"dể":12219,"Ä©a ":3523,"cể":67050,"bể":45726,"mể":131131,"Å© ":2155,"lể":23218,"kể":7585,"iể":166241,"hể":264262,"gể":17545,"Sể":1192,"Tể":7402,"Rể":525,"Å ":339,"Mể":5140,"vưể":475,"Nể":1896,"Xể":142,"Vể":1294,"rể":50434,"sể":54959,"tể":83275,"uể":92128,"nể":21381,"Å« ":399,"oể":14719,"Å©i":200,"Å©n":2955,"vể":53705,"专专 ":169,"xể":2191,"yể":31245,"Æ¡i":3100,"Æ¡m":181,"Æ¡n":18072,"Æ¡ ":4182,"ã‚":270,"ã‚¢":496,"가가 ":224,"Æ°Æ¡":12314,"Æ° ":8494,"Æ°a":1622,"Æ°n":2592,"Æ°u":1733,"Äể":19679,"để":91689,"Äan":304,"Ä‘a ":614,"Æ¡m ":179,"Æ¡n ":5934,"Ä‘ai":310,"Ä‘an":664,"Æ¡i ":3096,"Ä‘i ":743,"Æ¡ng":12128,"Ä‘en":294,"Ä‘o ":419,"Ä‘oà":1003,"Ä‘ua":287,"ä¹™":857,"乘":179,"之":2791,"丹":642,"临":665,"中":142,"並":1114,"丙":309,"丘":935,"丛":252,"专":2269,"且":143,"丈":196,"三":4224,"ä¸":2034,"万":989,"亞":563,"亂":327,"ä¾":215,"ểy ":15070,"ểu ":43780,"ểt ":179200,"Äây":3553,"Äào":412,"Äài":765,"Äà ":277,"Äôn":3441,"圓":161,"Äô ":9162,"Äìn":322,"冲":171,"đêm":8016," 丘":335,"đíc":372," 专":413," 三":1027," ä¸":795,"đìn":459,"đây":1563,"đá ":2316,"倉":569,"đán":1131,"đã ":4325,"đào":392,"đài":257,"đàn":318,"đún":140," 倉":244,"ưể":121112," ä¾":158,"đón":976,"đôi":461,"đôn":8728," 並":342," 临":320," 丹":351," 之":799," ä¹™":411,"đó ":2805,"đô ":18495," 亂":152,"大":197,"ểa ":61642,"ểc ":245982,"ểch":9163,"ểi ":112695,"ểk ":174,"ển ":203101,"ểm ":59200,"ểp ":25010,"ểng":124564,"ểnh":41635,"ểo ":9528,"Äiể":1233,"ã‚ã‚":156,"Ä‘iể":16552,"Ä‘oể":920,"đăn":218,"Ä‘Ä©a":596,"Ï‚ ":202,"Ä‘Æ¡n":1904," ểc":15788," ển":5390," ểm":264," ểy":799,"アア":391," ể ":57997,"Ä‘Æ°Æ¡":308,"Ä‘Æ°a":612,"Ñк":150," vÆ°":1017," xÆ°":529," Ão":1420," Ãv":245," à ":1673," Ga":3043," Câ":289," Cá":2724," Ge":2570," Cà":157," I ":489," Bì":1773," Bí":387," Fo":1209," Fu":749," Fr":2319," Fi":1014," Fl":1075," Bé":190," Ha":7534," He":2474," Dâ":1282," Cô":3477," Có":398," Gy":285," Cú":325," J ":162," Go":1392," Gr":4156," Gu":2454," Gh":322," Gi":6622," Gl":488," Id":176," Ic":219," Ib":265," Hy":1924," Dô":340," Hu":4380," Ho":13959," Hi":2699," Ji":227," Je":694," L ":288," Ja":1791," Ir":1002," Is":1688," It":2091," Im":186," In":3565," Il":425," M ":229," Ka":2587," Ke":1169," Ki":2975," Kh":4951," Jo":1090," Ju":1029," N ":147," La":11244," Hà":3868," Há":6905," Le":5519," Hã":339," Li":5628," Kl":231," Kn":141," Ko":946," Kr":777," Gò":247," Ku":656," Gö":233," Ky":391," Ma":14506," O ":139," CÆ¡":380," Mi":7555," Me":5610," Hì":157," Lo":11086," Hé":321," BÆ°":341," Ly":614," Hó":455," Hò":670," Lu":1590," Ne":5220,"а ":296," P ":285," Na":14703," Ng":8844," Nh":15231," Ni":3593," Mo":5991," CÆ°":440," My":978," Mu":2713," A ":801," B ":314," C ":460," Ap":624," Am":2023," An":10467," Ak":163," Al":6944," Ai":1106," Ag":749," Ah":189," Ae":262," Af":957," Ac":1189," Ad":608," Ab":573," Ba":13586," D ":281," Az":1190," Ay":139," Av":424," Au":2992," At":1064," As":1419," Ar":5374," Aq":1657," Be":3504," Bi":2420," Bh":176," Bl":874," Bo":4376," Br":4006," Bu":3207," By":153," E ":175," Ca":14032," Ce":2572," Ci":1003," Ch":19001," Cl":1229," Cr":1732," Co":10896," Cu":1575," Cy":959," F ":217," Da":2548," Di":2876," De":2206," Dr":1245," Do":2436," Dy":152," Du":1262," Ea":529," Ec":908," Ed":358," G ":173," El":867," Ei":335," Et":365," Es":889," Er":614," Ep":314," En":763," Em":454," Ex":456," Eu":2122," Bà":785," Bá":846," Fe":894," Fa":1822," H ":178," Xu":1090," Tò":163," Tí":259," Tê":651," Xi":413," Tà":542," Xe":204," Tá":302," Tâ":9623," Sü":308," Xa":216," Só":240," Sô":411," MÆ°":225," Wo":905," Sé":358," Sè":329," Wi":1632," Wh":294," Sá":350," We":2007," Sâ":533," Sà":247," Wa":2017,"й ":151," Y ":281," LÆ°":805," Võ":190," TÄ©":188," Và":232," Ze":737," Vâ":484," Zh":184," Zi":450," Za":1260," Yv":187," Yu":290," Yp":238," Yo":868," SÄ©":506," Ya":432," Ye":349," NÆ°":140," Tô":962," Xy":153," Tú":282," Tù":160," Xí":179," VÄ©":455," Xô":549," Wü":541," Xã":2079," Vù":202," Vă":1148," VÅ©":587," TÆ°":1025," a ":4962," Yê":501," SÆ°":226," SÆ¡":2029," R ":199," Kô":237," Ou":268," Ov":236," Os":655," Ot":299," Or":2938," Op":391," Po":3911," Lê":922," Lé":142," Pl":1371," Pi":2271," Ph":31029," Lã":279," Pf":1720," Lâ":831," Pe":2592," Là":800," Pa":7774," DÆ°":2196," Ny":327," Nu":290," No":7248," Ol":594," Ok":173," On":479," Om":200," Oh":202," Oi":748," Od":181," Oc":483," Oe":269," Ob":773," Oa":270," Ra":1894," Mü":182," T ":156," Mù":252," Mô":838," Qu":14182," Ro":3827," Re":2068," Ri":2185," Rh":4439," Py":3297," S ":230," Pr":3910," Ps":554," Pt":637," Pu":1471," Mé":244," Lý":663," Lü":151," Má":214," Lă":214," Mã":1026," Sy":607," Sw":578," Su":1915," St":3635," Ta":3810," V ":169," Th":42680," Ti":3850," Te":2931," Tr":21258," To":2625," Nó":11014," Ry":154," Nô":330," Ru":1162," Sa":11420," HÆ°":907," U ":154," Nă":720," Sh":1334," Si":2727," Sc":3070," Se":3434," So":3300," Sp":1984," Sr":367," Sk":306," Sl":517," Sm":206," Uy":393," Va":2209," X ":200," Ve":2261," Vi":16180," Ré":207," Vo":1912," Vu":317," Tu":2947," Tw":154," Ty":200," Ug":456," Uk":408," Ul":140," Um":165," Un":913," Ur":403," Ut":638," ja":224," l ":250," im":322," in":6639," is":5408," it":773," ka":329," m ":439," kh":38847," ki":18267," gâ":429," ke":373," gá":282," ju":151," cô":5020," có":47410," cú":382," cù":1871," ha":11491," dã":502," dâ":28872," he":621," dà":2524," gi":28288," bă":261," gh":729," gl":236," gr":899," go":250," gu":378," cò":3288," hy":276," dù":2107," că":333," hi":6669," dé":1144," ho":9851," ht":218," hu":16965," dò":646," nh":45517," ni":1927," ng":51853," ne":507," na":15500," cÆ°":775," mu":1888," mo":1972," ké":437," on":1221," kê":1776," oc":375," of":6994," ob":204," dÆ°":2565," nu":602," no":1173," hã":1017," hà":8670," há":923," le":1986," li":4644," la":5104," gó":504," kn":913," km":4476," cÅ©":2936," ko":246," me":1022," mi":9905," cÆ¡":2426," hù":342,"Ñ ":139," hú":203," ma":3748," lu":1700," hó":2026," hò":4796," ly":382," hô":283," bÆ°":9431," hè":160," lo":35801," hì":2834," ag":160," ab":359," ac":560," ad":220,"Hểu":669," am":614,"Hểp":191," an":7056," ap":254," ai":278,"Hển":822," al":2427," au":545,"Hểi":3718," ar":2712,"Hểc":428," aq":275," at":395," as":1145," d ":2111," ba":20628," 가가":227," bi":39922," be":1810," bo":1086," bl":299," by":542," bu":1037," br":705," ca":18336," XÆ°":246,"Hể ":3004," VÆ°":1101," er":165," et":2265," es":168," en":1522," em":442," el":437," bà":2390," fe":289," bá":2320," fa":2087," eu":165," ex":489," fu":610," fr":2001," bí":964," bì":10908," fo":1760," bê":1224," bé":146," fl":771," fi":679," bã":190," ge":1821," cá":22553," câ":2231," cà":270," ga":2783," bú":206," bó":2103," cl":550," cm":154," co":7147," cr":532," ce":1064," ch":72993," ci":299," da":4945," cu":4863," cy":157," do":4301," dr":248," de":8416," di":19653," ec":238," ed":190," ea":304," du":2623," 三三":209," vù":22601," vă":2376," ví":173," vì":1170," võ":378," vô":1027," vò":489," và":46932," vâ":230," sÄ©":1785,"ка":178," tù":416," tú":155," tô":909," nÆ°":35851," tă":379," xo":207," tê":7377," nÆ¡":1991," să":488," tò":331," xu":3665," tí":18540," tì":8960," mÆ°":601," ww":167," só":461," sô":2005," tâ":11572," tà":3363," tá":3073," xe":994," xa":907," tÆ°":4765," vÅ©":680," 三之":172," sÆ°":822,"ов":213," yê":400," sÆ¡":282," rÆ°":159," vÄ©":1312," xã":6617," xâ":941," xá":864," ru":574," nô":442," nó":3467," hÆ°":1543," sa":4809," nú":1256," se":1352," sc":583," si":4586," sh":865," nă":29145," sn":2777," sm":592," sl":244," sp":4116," so":2264," qu":28552," mó":260,"ви":141," mô":1890," mù":492," ra":5117," re":1311," nà":32102," nâ":291," ri":1195," hÆ¡":1747," né":300," ro":791," lÄ©":607," nê":751," lò":161," pu":610," pr":1132," lí":1307," lú":446," s ":1052," lô":926," px":1128," py":148," mã":975," má":2233," mà":2646," lý":2654," mì":566," mé":12707," ot":153," kì":215," op":485," kí":1225," or":1862," lá":722," pe":891," là":155391," lã":1013," lâ":684," pa":1485," ký":784," lè":332," pl":606," po":832," lê":796," pi":462," ph":51377," wa":721," sâ":1031," sá":4080," we":797," rõ":266," y ":288," lÆ°":3756," wo":408," wi":1339," wh":1197," sê":264," va":1100," ve":759," uy":141," vo":357," vu":12917," rã":285," vi":8649," ty":993," tu":4206," us":140," mÅ©":152," up":179," un":373," ta":1679," v ":243," sy":262," st":916," su":3809," lÅ©":299," tr":156052," to":6595," th":230858," ti":21975," te":899," Ä‘u":484," Ä‘a":1700," Ä‘e":474," Ä‘i":17419," Ä‘o":2466," Äá":164," Äà":1637," Äâ":3555," Äì":322," Äo":196," Äi":1416," Äa":443," Ä‘Æ¡":1908," Ä‘Æ°":29691," ÄÆ°":847," đí":392," đê":8032," đì":460," đá":3730," đâ":1604," đã":4347," Äă":188," đà":1035," Äô":12690," đă":220," đú":182," đô":27698," đó":3827," Ä‘Ä©":596,"Lể ":569,"Lểi":178,"Lển":215,"Lểp":1280,"Lểc":1143," Bể":9869," Kể":5918," Hể":9143,"Kể ":5821," Gể":165," Cể":8663," Dể":757," Âm":150," Âu":2129," ÃŽl":330," Ôn":1608,"Nể ":198,"Nển":165,"Nểi":1235," Úc":1243," à ":1274," áp":355," án":814," âm":3643," ít":431," ô ":353," ôn":1711,"Mể ":2527," ý ":503,"Mểc":807,"Mểt":1153,"Mểu":174,"Mển":285,"Mểi":146," ăn":626," ê°€":331," Æ°Æ¡":433," Æ°u":158,"ê°€":830," hể":63957," gể":12621," lể":23214," kể":7585," cể":67038," dể":12218," bể":45724," Vể":1292," Tể":7366,"Rểp":306," Xể":142," Nể":1895," Lể":3615," Mể":5122," Rể":522," Sể":1093," vể":53702," xể":2191," yể":935," mể":131122," nể":21375," rể":4289," sể":54954," tể":83225,"Tểt":172,"Tểp":225,"Tểc":152,"Tểa":1075,"Tển":2741,"Tểi":245,"Tể ":2617,"Sể ":900,"Vểt":162,"Vển":442,"Vểi":163,"Vể ":500,"ưểt":382,"ưểu":183,"ưển":18802,"ưểm":9501,"ưểi":24657,"ưểc":67509," Äể":19647," để":91683," ưể":501,"Ãvi":245,"ال":152,"Ão ":1417,"三三 ":318,"三万 ":202,"Âu ":2118,"三专 ":318,"Âm ":149,"Æ°Æ¡n":12134,"Æ°Æ¡i":146,"가가":499,"Bể ":2169,"AO ":231,"AN ":149,"Bểt":140,"Bểc":4036,"Bển":2788,"Bểo":450,"Cể ":946,"Cểu":671,"Cểp":339,"Cểm":232,"Cển":5510,"Cểc":787,"Dể ":256,"Dểc":190,"Bà ":315,"Bá ":150,"Bài":310,"Bàn":152,"Bác":305,"Fel":162,"Fen":155,"Fer":312,"Fis":282,"Ext":290,"Fas":526,"Fal":176,"Far":151,"Fab":275,"Eri":170,"Ess":158,"Est":392,"Eth":271,"Eup":437,"Eur":1040,"El ":188,"Ele":184,"Eng":198,"Epi":248,"Ent":180,"Các":1801,"Ger":929,"Cát":277,"Geo":862,"Gen":359,"Gla":140,"Gha":245,"Gia":2324,"Gil":159,"Gir":424,"oể ":155,"oểt":2492,"Cá ":259,"oểc":3719,"Gan":147,"Gal":295,"Gam":240,"Gau":147,"Gar":1513,"oển":3941,"oểi":4411,"Gab":195,"Fus":252,"Fro":378,"Flo":588,"Fla":166,"Fle":174,"Å©i ":200,"Fra":932,"Fri":349,"Å©ng":2953,"Fre":553,"Bín":197,"Bìn":1748,"Fon":217,"For":497,"Fou":186,"Dân":1220,"nểu":216,"II ":1176,"Hil":320,"Him":177,"Hin":145,"Hip":145,"nể ":1462,"Hel":390,"nểi":2679,"Hei":256,"nểm":15373,"nển":1144,"Hem":163,"Hen":196,"Hes":405,"nểa":304,"Her":528,"Cúp":236,"Hal":332,"Hai":401,"Han":467,"Ham":344,"Has":179,"Har":748,"Haw":533,"Hau":3745,"Côt":1585,"Guy":153,"Cô ":190,"Gua":732,"Có ":275,"Gui":1036,"Côn":1681,"Gre":1012,"Gri":151,"Gra":2331,"Gro":483,"Glo":169,"Giá":926,"Gon":156,"Goo":205,"Gol":188,"Gom":158,"Úc ":1233,"Inn":161,"Int":561,"Ins":197,"Ill":204,"Ind":2106,"mểc":13680,"mểm":11638,"mểi":2779,"mểu":420,"mểt":99606,"mển":1814,"mể ":1110,"Ibe":205,"亞 ":440,"Hyp":280,"Hyd":396,"Dôm":338,"Hy ":979,"Hun":464,"Huy":2731,"Hue":217,"IV ":160,"Hoà":1902,"Hor":294,"Hou":153,"Hom":153,"Hon":555,"Hok":148,"Hol":1270,"Hoa":8475,"Arg":562,"Are":201,"Arc":363,"Ard":1593,"Ara":960,"Arm":257,"Ari":480,"Aqu":1651,"Apo":203,"Ath":149,"Atl":649,"Ast":281,"Ass":392,"Asi":256,"Arr":175,"Art":235,"Ave":196,"Auv":549,"Aus":949,"Aur":157,"Aud":253,"Aug":204,"Aub":473,"lể ":2800,"Azu":1003,"lểp":5146,"lểt":155,"lểi":2389,"lển":5223,"lểa":430,"lểc":6489,"Ba ":889,"lểy":462,"Bai":215,"Bal":574,"Ban":6419,"Bac":146,"Bad":1148,"Bay":966,"Bar":929,"Bat":270,"Bas":1136,"Bau":155,"Abr":165,"Aca":233,"Acr":321,"Ach":200,"Ade":163,"Ai ":341,"Aga":200,"Afr":687,"Afg":186,"Air":488,"Ala":309,"Alb":681,"An ":1388,"Alg":185,"Ali":170,"Ale":299,"Alv":153,"Als":515,"Alt":493,"Alm":187,"All":366,"Alp":2942,"Ame":689,"Amb":210,"Ama":449,"Amp":158,"Anh":5927,"Ang":642,"Ana":331,"And":666,"Ant":595,"Ano":159,"Ann":219,"Bus":168,"Bul":625,"Bur":1284,"Buc":492,"Bru":412,"kể ":5372,"Cab":144,"kểt":1768,"Cae":179,"Cal":3283,"Cam":1757,"Cai":141,"Cas":3059,"Car":1359,"Cau":236,"Cat":533,"Cao":644,"Can":1671,"kểc":374,"Cap":594,"Bea":394,"Ber":1074,"Ben":559,"Bel":748,"Bin":143,"Bil":278,"Bis":150,"Bit":253,"Bir":234,"Blu":260,"CP ":196,"Biê":192,"CN ":900,"Bla":380,"Bre":721,"Bra":1413,"Bro":439,"Bri":863,"Bol":471,"Boi":232,"Bon":251,"Bor":635,"Bos":277,"Bot":225,"Bou":1517,"Cyp":409,"Cur":167,"Cub":268,"Cun":184,"Cup":164,"EE ":154,"Des":235,"Deu":424,"Del":231,"Dem":185,"Den":364,"Dam":166,"Dan":842,"Dar":270,"Dav":172,"Dal":154,"Cho":474,"Chr":351,"Che":539,"Chi":4142,"Cic":417,"Chu":975,"Cit":157,"Châ":2351,"Cle":204,"Cla":633,"iểt":19441,"iểu":22084,"iểy":197,"Cel":188,"iểm":11357,"iểp":3887,"iển":94986,"Cen":1295,"Cer":828,"iểc":3158,"iểi":7620,"iểa":1763,"Cha":5749,"Cri":167,"Cra":378,"Cre":395,"ChÆ°":286,"Cro":546,"Chù":202,"Chú":672,"Chí":1230,"Coc":220,"Coe":184,"Cop":262,"Cos":1089,"Cor":1423,"Com":752,"Col":1893,"Con":3653,"Cou":867,"FA ":242,"Drô":335,"iể ":1731,"Edw":139,"Ông":1552,"Ect":200,"Ecu":469,"Eas":364,"Do ":273,"Diê":175,"Dic":144,"之三三":139,"Dit":152,"Dis":351,"hểo":739,"hểp":5189,"hểm":3873,"Dip":407,"hển":23819,"Dio":154,"hểy":11164,"hểu":6003,"Die":258,"hểt":14285,"hểa":1704,"hểi":18265,"hểc":23691,"Di ":179,"hể ":155404,"Dun":239,"Duy":294,"Du ":216,"Dri":386,"Dre":146,"Dra":192,"Dou":158,"Don":392,"Dom":257,"Dor":723,"CÆ°Æ¡":311,"Nev":160,"Neu":541,"Net":191,"Nep":895,"Neo":280,"Nas":316,"Nat":595,"Nav":156,"Nig":396,"Nie":1241,"Nic":337,"Nin":1068,"Nhi":250,"Nha":5926,"Nga":1678,"Ngh":1164,"Ngu":2389,"Ngo":273,"New":2499,"Myr":349,"xể ":903,"Mya":329,"Nak":165,"Nam":11941,"Nan":196,"Nag":208,"Na ":357,"xểp":906,"xểy":267,"NhÆ°":191,"Nym":236,"Nhó":149,"Nhà":676,"Nhâ":3285,"Ngô":642,"Ngà":567,"Ngâ":218,"NgÆ°":724,"NhÄ©":654,"Nov":205,"NgÅ©":256,"Nor":4126,"Not":381,"Nob":171,"Noc":1779,"Oec":143,"DÆ°Æ¡":1846,"PG ":211,"Ois":715,"Ohi":151,"Oah":152,"Occ":238,"Obe":662,"ÃŽle":330,"Ott":171,"Ovu":166,"Kôn":235,"Oly":147,"Oli":201,"Giể":2035,"Ont":199,"Or ":550,"Opo":183,"Ora":158,"Ore":369,"Orc":304,"Ori":453,"Orn":566,"Ost":243,"Phú":926,"Phù":169,"Phò":192,"Phó":180,"Phá":21296,"Ple":538,"Phâ":188,"Pla":641,"Hiể":1343,"Lê ":891,"Pin":370,"Pit":142,"Phy":669,"Pie":506,"Pic":810,"Pho":672,"Phi":3036,"Pha":566,"Lãn":143,"vểy":569,"Lâm":707,"vển":2537,"vểt":17681,"Pfa":1704,"vểa":164,"vểc":13644,"vểi":7805,"Per":956,"Pet":409,"Pen":544,"Pel":140,"Lào":383,"vể ":11302,"Pay":1416,"Là ":298,"Pat":164,"Pas":888,"Par":1717,"Pau":274,"Pac":219,"Pan":548,"Pap":721,"Pal":1091,"Pak":192,"Pyr":3193,"Huể":468,"Pte":513,"Pun":221,"Pup":166,"Pue":171,"Puy":394,"Pro":2087,"Pri":425,"Pre":247,"PhÆ°":678,"Pse":388,"Hoể":173,"Pra":808,"Pol":756,"Pom":148,"Pon":298,"Poi":1472,"Pot":227,"Por":309,"uểc":63038,"uểi":2825,"Lăn":196,"uểt":6647,"uển":17332,"Mã ":868,"uể ":2223,"Lý ":659,"Mùa":206,"SA ":174,"Ram":194,"Ran":617,"Quá":225,"Quâ":725,"Môn":650,"Quý":277,"Qua":706,"Qui":159,"Que":829,"Quy":457,"Ita":500,"Isl":594,"Isr":263,"It ":1456,"Ira":481,"Ire":363,"Isè":447,"tểm":507,"tển":38505,"tểi":9573,"tểc":3283,"tểa":923,"tể ":24294,"Jac":218,"Jav":319,"Jan":148,"Jam":357,"tểo":2243,"tểp":1747,"tểt":2043,"Jer":195,"Jea":217,"Biể":504,"Jos":191,"Jor":166,"Joh":334,"Jul":480,"sể ":37621,"sển":14954,"sểm":218,"sểc":1290,"sểa":158,"Kai":155,"Kam":181,"Kal":268,"Kan":358,"Kau":180,"Kat":153,"Kas":184,"Kar":496,"Kaz":203,"sểt":255,"sểp":188,"Ken":722,"Kir":203,"Kit":143,"Kin":907,"Kim":609,"Kho":491,"Khu":1438,"Kha":464,"Khi":230,"Chể":1285,"Khê":158,"Khá":594,"Khô":380,"Kon":253,"Kor":151,"Kre":398,"Gòn":201,"Cuể":281,"Kus":145,"Kur":182,"Hàn":1241,"Leu":183,"Les":367,"Lep":631,"Leo":339,"Len":152,"Hán":6856,"Lei":182,"rểc":1530,"Lea":285,"rển":39207,"rểm":291,"rểi":1137,"Hà ":2463,"Lau":510,"rể ":6793,"Le ":619,"Lak":163,"Lai":390,"Las":248,"Lat":742,"Lar":215,"Lao":140,"Lam":368,"Lan":4696,"Lac":165,"Lab":180,"倉 ":180,"La ":2717,"Liê":1950,"Hér":279,"Diể":335,"Lib":574,"Lie":155,"Lig":183,"Lim":511,"Lin":692,"Lio":171,"Lis":148,"Lit":507,"Liv":154,"Leó":2050,"Hãn":333,"rểt":1287,"Hòa":572,"Lud":184,"Luc":212,"Loà":1669,"Hìn":160,"Loz":163,"Lou":367,"Los":193,"Lot":580,"MS ":483,"Loi":3201,"Lor":2131,"Lon":1378,"Lom":271,"Loa":517,"Ma ":267,"Hóa":437,"Luâ":253,"Lyc":258,"Mei":187,"Men":179,"Mel":544,"Mes":227,"Mer":503,"Meu":1122,"Met":367,"Mec":579,"Meg":242,"Med":321,"Mex":918,"Man":2248,"Mal":1668,"Mar":4888,"Mas":469,"Mag":297,"Mad":710,"Mah":444,"Mai":1077,"Mac":532,"NE ":2847,"May":519,"Mau":531,"Mat":259,"Miê":150,"Mol":400,"Mon":1606,"Mos":1419,"Mor":654,"Mou":470,"Mot":424,"Moz":414,"Mid":2002,"Mic":1239,"CÆ¡ ":351,"Mit":747,"Mir":161,"Mis":522,"Mil":319,"Min":1719,"Mun":171,"Mur":1848,"Mus":259,"Xià":174,"Phể":1906,"Tây":8477,"Tân":1027,"Tào":168,"Tàu":164,"Süd":288,"Sôn":404,"Sóc":214,"Wor":462,"Wol":216,"Séc":256,"Sèv":318,"Whi":209,"èvr":358,"Wik":174,"Wil":393,"Win":429,"Wie":149,"Wit":233,"ère":1422,"Sài":216,"Web":172,"Wei":433,"LÆ°Æ¡":358,"Wes":961,"Sân":446,"Was":264,"War":327,"Wal":818,"ès ":590,"LÆ°u":290,"Dưể":277,"èn ":152,"èo ":278,"ém ":263,"之ä¸":166,"QÄ ":168,"之三":478,"之万":178,"ée ":472,"之专":315,"之之":217,"ées":2904,"Vos":497,"Vor":516,"Vol":589,"éc ":283,"Nhể":3766,"Ngể":811,"Viê":165,"évi":139,"TÄ©n":188,"ép ":1113,"Zea":402,"Zar":282,"Zam":723,"én ":239,"éo ":609,"éra":396,"ét ":23407,"éri":203,"éné":2942,"Zim":303,"Zel":153,"épa":1146,"Vân":484,"ên ":47944,"êm ":9120,"Yps":167,"Quể":10633,"Yve":176,"並三":153,"SÄ© ":505,"Yor":442,"You":143,"Yon":155,"ênh":268,"êng":589,"专专":219,"专三":225,"êu ":3625,"Yel":146,"三万":220,"三ä¸":213,"三三":777,"三专":394,"Tô ":277,"Xuâ":498,"三之":347,"Tôn":652,"Túc":179,"ä¸ä¸“":144,"ä¸ä¸":142,"Tên":660,"ä¸ä¸‰":258,"Tín":160,"ä¸ä¹‹":215,"Xuy":470,"ãi ":669,"Syn":143,"Syr":151,"Swi":186,"Swa":238,"Sur":243,"Sum":294,"Sul":327,"Sun":218,"Sud":207,"Str":573,"Stu":169,"Sti":380,"Sto":266,"Sta":800,"Ste":1346,"Ten":237,"Tel":150,"ãnh":975,"ãng":1319,"ão ":484,"Tam":802,"Tan":929,"Tas":235,"Tar":601,"ãn ":565,"Tai":151,"Tak":148,"Ski":144,"ãy ":559,"Khể":614,"Shi":359,"She":144,"Năm":661,"Sho":220,"Sha":353,"Sim":166,"Sil":271,"Sin":736,"Sie":417,"Sib":258,"Sic":157,"Ses":155,"Ser":590,"Sen":317,"Sel":193,"HÆ°Æ¡":208,"Sei":684,"Seg":435,"Sri":344,"TV ":172,"Spa":202,"Spi":140,"Sph":1036,"Spe":218,"Spr":184,"Sou":1403,"Sol":431,"Som":283,"Son":366,"Sor":294,"Kiể":703,"Slo":396,"Nôn":230,"Rus":514,"Nó ":10964,"Sai":3614,"Sam":307,"Sal":1166,"Saa":431,"Sac":786,"Sab":139,"Sco":422,"Sci":154,"Sch":2083,"Sca":289,"Sax":162,"Sav":348,"Sat":196,"Sau":596,"Sar":1013,"Sap":212,"San":1415,"Sao":173,"HÆ°n":444,"Sa ":225,"TA ":219,"Res":158,"Rhi":678,"Rhe":1850,"Riv":248,"Ris":546,"Rie":160,"Ric":612,"Red":234,"Rei":173,"Reg":216,"Ren":178,"Rep":253,"Rob":306,"Roc":356,"Rou":1296,"Rot":151,"Ros":295,"Rom":626,"SS ":368,"Rhô":1640,"Ven":831,"Vau":301,"Van":289,"Val":1014,"Var":300,"Vic":424,"Vie":766,"Vir":376,"Vil":850,"Vin":383,"Ver":694,"Vex":266,"Ukr":405,"Uni":646,"Miể":206,"Uy ":300,"Utt":447,"Tră":223,"Luể":194,"Trá":373,"Trà":189,"Trâ":155,"Trì":245,"Bưể":300,"TrÆ°":1573,"Uga":441,"Tex":717,"Ter":1194,"Tha":1705,"The":9028,"Thi":2418,"Tho":351,"Thu":1030,"Til":178,"Tim":168,"Tin":432,"Thà":2636,"Thá":3215,"Liể":150,"Thü":723,"Thô":412,"Tiê":963,"Tor":743,"Tok":152,"Tol":222,"Tom":166,"Tou":295,"ThÆ°":1102,"Tru":10246,"Tro":2116,"Tri":2061,"Tre":371,"Tra":816,"Toà":178,"Tuy":629,"Tur":1442,"Tun":141,"Mưể":195,"ày ":41264,"TÆ°Æ¡":169,"àu ":3188,"gểa":154,"gểc":1587,"gểi":5837,"gểp":336,"gểm":3968,"gển":2170,"gểy":212,"gểt":591,"ành":21823,"àng":7133,"gể ":2467,"ào ":9552,"àn ":5519,"àm ":2556,"ài ":40596,"dể ":1928,"bis":208,"bit":429,"biu":210,"bio":195,"biq":379,"bir":196,"bil":353,"bin":1156,"bii":441,"dểa":1072,"dểc":2450,"bo ":249,"dểi":171,"dểm":153,"dển":5451,"dểu":482,"dểy":237,"áy ":2251,"blo":143,"ble":788,"bli":464,"bla":343,"boa":544,"bol":192,"biê":756,"bon":625,"bom":252,"bop":200,"bor":571,"bot":181,"bos":169,"bou":707,"bbe":548,"be ":978,"áo ":4269,"ban":14532,"bal":489,"bai":195,"bac":1588,"bad":244,"bab":362,"án ":11049,"bay":2258,"bat":528,"bas":507,"bar":886,"bao":2943,"bea":250,"áp ":27496,"ánh":4002,"áng":13329,"bi ":208,"bei":269,"bee":755,"bed":546,"bec":391,"ber":5184,"ben":538,"bel":1263,"bek":171,"bes":289,"bet":259,"bfa":299,"áu ":550,"bia":2262,"bic":185,"bid":364,"át ":6081,"ách":6363,"áce":165,"ái ":6094,"ca ":5266,"car":2682,"cas":523,"cat":809,"cau":258,"can":2819,"cao":13635,"cap":271,"cac":552,"cae":343,"cad":152,"cam":352,"cal":1785,"cai":449,"ce ":3602,"ám ":1096,"bri":1032,"bro":469,"bra":914,"bre":597,"bru":440,"bsi":207,"bur":2940,"bul":333,"bun":198,"bum":722,"but":706,"bus":353,"by ":708,"bwe":288,"ác ":24475,"aka":675,"am ":27480,"ake":553,"aki":591,"akh":416,"aji":229,"ajo":231,"al ":6772,"aja":635,"aii":453,"ail":3696,"ain":10996,"air":827,"ais":1435,"ait":246,"ak ":354,"aig":201,"aid":285,"aic":180,"aia":148,"ây ":27201,"ahi":144,"ahu":249,"ahr":141,"aho":292,"aha":852,"agi":293,"agr":420,"agu":637,"agn":2270,"ago":1655,"aq ":141,"anu":911,"anz":946,"any":498,"ano":1266,"ann":1274,"anm":363,"ant":5267,"ans":1570,"ane":1647,"ang":23334,"anh":11504,"ani":4224,"anj":313,"ank":983,"ap ":200,"ana":4781,"anc":3852,"and":17225,"amu":239,"amm":581,"amo":823,"amp":3495,"ams":266,"ami":3273,"ame":2454,"amb":1921,"ama":1996,"ao ":21236,"alz":1868,"aly":679,"alv":524,"alu":519,"alt":1192,"als":583,"alp":280,"alo":1138,"alm":374,"all":4541,"alk":430,"alg":162,"ali":4610,"alc":275,"ald":946,"ale":3407,"alf":143,"ala":5532,"alb":1038,"an ":27435,"aku":180,"ako":211,"aba":1197,"abe":439,"abi":699,"abl":307,"abo":447,"abr":553,"abu":216,"abw":286,"ae ":31054,"aca":592,"aal":305,"aar":275,"ad ":1523,"ac ":1152,"âng":210,"ab ":257,"aft":152,"aff":222,"ai ":10857,"aga":1564,"age":954,"aeo":180,"aen":693,"ael":524,"aes":144,"aer":162,"VÆ°Æ¡":859,"aei":212,"ah ":482,"âte":406,"ado":1640,"adr":334,"adi":656,"âu ":6939,"ade":2368,"aea":378,"aec":197,"ag ":159,"ady":140,"adt":318,"adu":484,"aco":486,"ack":866,"aci":2053,"ach":4974,"ace":5973,"acc":246,"ada":2077,"af ":167,"act":895,"acu":424,"acr":438,"azo":203,"azi":604,"aze":208,"aza":466,"axi":240,"axo":326,"az ":215,"ayo":158,"ays":2306,"aya":1069,"aye":1477,"ân ":71107,"ba ":1967,"âm ":6828,"aqu":368,"at ":2191,"arh":179,"arg":1494,"are":4513,"ard":3817,"arc":1390,"arb":768,"ara":4118,"arp":777,"aro":1967,"arn":2152,"arm":452,"arl":768,"ark":1469,"ari":8943,"aru":469,"arv":290,"arr":2293,"ars":863,"art":4067,"au ":6244,"asa":466,"ary":947,"arz":380,"asi":1358,"ash":1142,"asc":1250,"ase":497,"aso":213,"asp":226,"ask":259,"asm":354,"aon":161,"ar ":3587,"apa":481,"ape":779,"api":791,"aph":710,"apl":205,"apo":627,"app":349,"apt":205,"apu":632,"as ":6065,"ava":1308,"ax ":315,"aux":509,"auv":229,"aut":4199,"avo":316,"avi":854,"ave":987,"awe":151,"ay ":14622,"awa":1004,"awi":403,"ata":3801,"asu":225,"ast":6128,"ass":2666,"atr":716,"ato":1406,"ate":3193,"atc":165,"ati":4840,"ath":1382,"aua":167,"auc":575,"aub":164,"att":609,"ats":390,"atu":1696,"aty":238,"aul":975,"aum":254,"aun":342,"aur":1043,"aus":1014,"aud":522,"aue":262,"aug":153,"aui":145,"bể ":9115,"Wür":527,"bểy":259,"bểt":2371,"bểu":602,"bểi":3447,"bểo":909,"Xã ":2077,"bển":18106,"bểc":10801,"Thể":19271,"Tiể":1514,"Trể":2534,"VÄ©n":400,"Xíc":159,"Võ ":184,"cểu":3163,"cểt":394,"cểp":4267,"cểc":593,"cểa":46999,"cển":3734,"cểm":741,"cểi":1406,"Hưể":237,"Vùn":202,"cể ":5697,"Văn":1172,"SÆ° ":192,"Viể":12501,"VÅ© ":518,"TÆ° ":617,"Tuể":240,"Xô ":542,"Yên":481,"SÆ¡n":1964,"ji ":324,"jar":408,"jan":213,"biể":38109,"jo ":152,"itr":907,"ito":2297,"itu":456,"itt":1509,"its":449,"itz":828,"ity":598,"ism":206,"isl":191,"iso":476,"isp":337,"iss":3448,"ist":3624,"ita":3373,"itc":146,"ite":2019,"ith":2136,"iti":2423,"ivo":162,"ius":1319,"ium":1206,"iva":362,"ix ":837,"ivi":819,"ive":1549,"ipo":143,"ipp":1792,"ipu":171,"ipt":450,"ipi":254,"iph":349,"ipl":287,"ilô":10662,"is ":16454,"ion":4473,"iop":856,"ior":202,"ios":885,"iot":545,"iou":177,"ioi":173,"iol":823,"ipa":1057,"ipe":657,"ir ":1809,"iru":298,"irs":241,"ück":162,"iro":773,"irk":203,"irl":235,"iri":455,"isi":750,"ish":1324,"ise":1480,"isc":1025,"isa":361,"iqu":1196,"ire":3687,"irg":448,"ira":1107,"irc":652,"it ":1527,"ja ":441,"iya":187,"iz ":244,"eón":2056,"가가가":272,"izo":396,"ize":388,"iza":272,"kim":1056,"kil":10715,"kia":494,"kin":3453,"kip":325,"kir":466,"kis":481,"km ":3828,"chể":21772,"ki ":1183,"khi":3505,"út ":627,"khe":165,"kha":834,"khu":15081,"kho":6571,"gày":9377,"gái":216,"kel":215,"ken":731,"kes":174,"ker":740,"ket":226,"key":304,"gân":416,"gây":413,"ke ":634,"úp ":776,"gàn":962,"úng":1614,"kra":492,"kre":255,"kt ":229,"cÅ©n":2434,"ểa":61690,"ku ":366,"km²":641,"kot":155,"kor":174,"kom":159,"kok":195,"ks ":691,"ể ":376292,"úy ":216,"cÅ© ":497,"kno":896,"kka":183,"khô":4285,"khó":360,"khí":730,"khú":467,"ko ":151,"khá":4090,"kle":735,"kla":228,"buể":254,"ểo":9548,"ểp":25031,"ểk":175,"ểm":59218,"ển":369560,"ểi":112828,"ểc":255331,"ểy":15079,"ểt":179295,"ểu":43805,"kaz":157,"gà ":270,"kat":204,"kar":238,"kas":219,"kan":739,"kal":182,"kam":185,"kai":366,"ka ":1184,"cùn":1840,"cúp":216,"ha ":8300,"ùng":27727,"ham":3795,"han":4769,"hao":550,"hap":378,"hai":4690,"hal":2163,"hau":2057,"hav":347,"har":4673,"has":721,"hat":786,"hae":501,"hag":376,"hab":337,"had":177,"hac":331,"hay":7652,"he ":18327,"dàn":837,"dài":1594,"hel":1664,"hei":3775,"hec":208,"hed":210,"hea":373,"hey":228,"hev":147,"het":420,"hes":679,"her":2719,"heo":10006,"hen":1600,"hem":538,"hi ":13142,"dây":222,"dân":28557,"dãy":398,"căn":327,"hig":332,"hie":278,"hid":1412,"hic":705,"hia":1555,"hip":346,"hio":999,"hin":3454,"him":2986,"ùy ":240,"hil":2244,"hik":232,"hii":300,"hiu":173,"his":1013,"hit":572,"hir":444,"hn ":286,"hla":242,"hle":1267,"hli":520,"hlo":240,"ho ":7256,"hma":350,"gma":174,"go ":1433,"giá":5603,"gme":307,"già":408,"glo":256,"gle":536,"gli":276,"gla":529,"gko":162,"gog":725,"gny":420,"ghÄ©":2919,"gno":258,"gni":256,"gne":3863,"giú":291,"gna":708,"úa ":604,"giô":172,"gs ":297,"goz":262,"úc ":3114,"gom":139,"gol":384,"gon":1688,"gos":596,"gor":601,"gov":334,"gu ":148,"goà":839,"gro":663,"gra":1031,"gri":527,"gre":280,"gto":303,"gui":371,"gum":182,"gul":499,"có ":47334,"gua":512,"gue":1969,"gy ":222,"cô ":368,"guy":4577,"gur":185,"gus":453,"gun":159,"còn":3275,"úi ":1356,"côn":4644,"gyr":164,"iai":626,"iam":328,"ial":1014,"iao":1227,"ian":5609,"ias":397,"iar":265,"iat":696,"ic ":3037,"iac":1407,"iae":280,"ibl":173,"ibi":314,"ibo":164,"ibu":345,"id ":1469,"iba":306,"ibb":546,"ibe":1206,"ia ":29704,"iet":540,"ieu":473,"iel":1226,"ien":2075,"ier":2747,"ies":4767,"ied":1831,"ieg":187,"ig ":1226,"ifo":1413,"iff":148,"ife":614,"ifl":166,"ifi":693,"icr":1054,"ics":295,"ict":897,"icu":1741,"ico":2316,"ick":622,"ici":3591,"ich":3932,"ice":765,"ie ":3741,"ica":6048,"idu":239,"ids":175,"ido":567,"idi":2598,"ide":2455,"ida":26239,"iid":4750,"il ":3121,"ija":174,"iji":157,"im ":5476,"ika":371,"ige":770,"iga":742,"ii ":1370,"igm":416,"igh":923,"igi":710,"igu":318,"igs":187,"igr":220,"igo":300,"ign":1169,"iha":223,"ik ":169,"imo":742,"imm":158,"imp":567,"ime":1906,"imi":643,"ip ":810,"inc":1322,"ind":1704,"ina":4704,"inb":264,"imu":305,"inn":513,"ino":1469,"int":4526,"ins":1803,"inf":237,"ine":10837,"inh":13778,"ing":8141,"ini":3074,"inl":2046,"ink":351,"ioc":292,"iod":167,"inu":932,"inv":156,"inx":181,"iny":208,"iko":368,"iki":420,"ike":257,"ila":1297,"ilb":181,"in ":13262,"ilo":550,"ill":9682,"ilh":181,"ili":3385,"ild":355,"ile":3779,"ima":1310,"imb":551,"io ":1175,"ily":2239,"ils":2324,"ilu":323,"how":204,"hol":800,"hom":456,"hon":2401,"hoi":164,"hos":411,"hot":229,"hou":479,"hoo":159,"hop":241,"hor":2606,"hoa":5214,"hof":203,"hoe":206,"hod":441,"hoc":207,"hni":161,"hne":266,"dée":277,"hme":233,"hmi":203,"hiê":5040,"dép":1137,"hua":252,"htt":326,"htr":324,"hth":197,"hte":186,"hst":227,"hse":1600,"hoá":860,"hoà":1969,"hu ":18061,"hry":423,"hro":1154,"hre":321,"hri":577,"ùa ":1067,"ht ":687,"hra":468,"hya":190,"huê":158,"hyl":1269,"dòn":604,"hy ":171,"hwa":665,"hwe":313,"hum":443,"hun":1586,"hus":902,"hut":182,"hur":289,"huy":20578,"Vưể":237,"dùn":1738,"dù ":368,"hyt":176,"hys":223,"hyr":202,"huô":174,"ùi ":234,"ffe":313,"ffi":237,"fer":812,"báo":721,"bác":152,"fen":465,"bán":1218,"fel":1035,"fgh":188,"bà ":200,"fas":191,"far":156,"fam":2105,"fal":2041,"bày":158,"bàn":328,"bào":422,"bài":1285,"ff ":218,"eya":142,"ext":206,"eyr":325,"eyh":211,"eye":179,"exa":986,"ez ":316,"exi":1325,"exc":181,"ezu":350,"ezi":175,"eta":842,"ete":743,"eti":1014,"eth":666,"etn":165,"etl":616,"esp":684,"esn":283,"eso":340,"est":3808,"ess":1434,"esw":957,"ev ":202,"euc":337,"eud":540,"eui":263,"eum":201,"eto":465,"etr":999,"ets":459,"ett":1071,"etu":203,"etw":156,"etz":312,"ew ":2560,"eve":622,"eva":566,"evi":1144,"euv":222,"eut":564,"eur":1318,"eus":1354,"ex ":606,"euz":190,"eux":671,"ey ":1664,"ewa":232,"erö":586,"epe":234,"epi":495,"eph":665,"er ":9592,"epa":418,"eot":213,"eor":527,"eom":471,"eol":262,"eop":530,"eon":709,"es ":26910,"ept":1404,"epu":235,"epo":154,"erk":202,"erl":752,"eri":6361,"erg":3169,"erh":199,"ere":2445,"erf":557,"erc":1171,"erd":573,"era":3401,"erb":858,"et ":4591,"equ":162,"esl":202,"esh":1269,"esi":2021,"esc":975,"ese":753,"eu ":303,"esa":259,"erz":260,"ery":556,"erv":528,"eru":1012,"erw":278,"err":2985,"ert":1566,"ers":4623,"ern":4630,"erm":2107,"erp":282,"ero":2376,"eki":196,"en ":12018,"elb":206,"ela":2654,"eld":1258,"elf":146,"ele":1267,"eli":1798,"elg":160,"elm":262,"elk":404,"ell":9746,"elo":933,"elu":208,"els":642,"elt":280,"ely":341,"eo ":10593,"eiß":164,"emb":1047,"ema":1158,"eme":2947,"emm":256,"emo":539,"emi":1585,"emp":238,"ems":155,"emy":139,"enf":291,"ene":1945,"enh":484,"eng":730,"enb":2082,"ena":1558,"end":2473,"enc":2127,"eno":956,"enn":3389,"enk":343,"enl":192,"eni":2073,"enu":1664,"ens":4503,"ent":10489,"enr":274,"enz":420,"eny":515,"eoc":157,"ego":1228,"ege":478,"egg":139,"egi":670,"egr":229,"egu":327,"ek ":217,"eic":752,"eis":833,"eir":787,"eim":1035,"eil":952,"ein":4903,"eie":545,"eid":730,"eig":203,"eif":191,"el ":3452,"eiz":186,"eit":258,"em ":1658,"öst":587,"giu":194,"gis":185,"gil":310,"gin":1363,"gio":553,"gid":1547,"gic":199,"gia":8820,"ght":794,"băn":187,"gho":161,"ghi":3850,"ghe":189,"gha":488,"ggi":146,"gge":165,"câu":470,"cây":1629,"gi ":387,"gen":5062,"cán":798,"cáo":198,"ger":1519,"ges":1085,"gh ":607,"các":18946,"geb":333,"cái":701,"gem":243,"gel":544,"cá ":1776,"ge ":1808,"gae":147,"gai":1612,"gas":1474,"gar":1812,"gau":202,"gat":452,"gay":317,"gam":376,"gal":707,"gan":1692,"gap":295,"ga ":2819,"bút":167,"Tưể":239,"fur":264,"fus":239,"bón":2056,"ful":164,"fun":375,"ft ":439,"fra":309,"fre":507,"fri":815,"bín":794,"fro":1604,"fou":879,"for":2385,"fon":152,"fol":504,"bìn":10852,"bên":1143,"bí ":160,"fle":146,"fla":250,"fli":289,"flo":441,"fly":418,"fic":547,"fie":487,"fil":195,"fin":314,"fis":489,"da ":3697,"de ":8611,"dac":762,"dad":140,"dal":972,"dai":189,"dag":446,"dae":24708,"dat":480,"dar":398,"dap":183,"dan":4922,"dam":377,"cun":718,"cul":2298,"cum":328,"cua":546,"cty":284,"ctu":2164,"ctr":238,"cto":873,"cti":1458,"cte":647,"cta":670,"cy ":251,"cus":673,"cur":399,"cut":255,"cyc":140,"cks":263,"cki":166,"ckl":786,"cla":486,"chá":201,"cle":346,"châ":14860,"cky":262,"chí":8076,"chò":225,"clu":420,"chó":180,"cli":182,"ché":314,"chì":193,"clo":464,"chù":312,"co ":1548,"chú":1406,"coi":717,"cod":256,"coa":287,"cob":146,"coc":214,"con":3655,"col":1035,"com":2135,"cor":1035,"cos":565,"cop":801,"cot":443,"cou":1134,"coz":183,"cs ":366,"ct ":454,"cre":241,"cra":438,"chÆ¡":592,"cri":479,"cro":1725,"chÆ°":1623,"ccu":244,"cci":838,"cco":265,"cca":156,"cea":4824,"ch ":37737,"cer":1071,"ces":531,"cet":142,"cen":1512,"cep":386,"XÆ°Æ¡":220,"cel":1002,"ced":395,"cha":3033,"chw":543,"chu":6009,"chy":360,"cia":1452,"ck ":1564,"cie":3416,"cid":2524,"che":4294,"chl":1776,"chi":13490,"cho":7644,"chm":217,"chn":259,"chs":1959,"cht":448,"chr":711,"cil":2248,"cif":277,"cis":311,"cit":448,"cin":1085,"cio":750,"cip":1009,"cm ":153,"cke":753,"ed ":4935,"eba":285,"ebe":884,"ôn ":3722,"ebi":183,"ebo":301,"ebr":934,"ebs":216,"ebu":294,"ec ":362,"eac":334,"ôm ":271,"eag":170,"eaf":201,"eae":4646,"ead":460,"ean":1098,"eal":621,"ear":1037,"eas":698,"eat":882,"eau":1120,"eb ":303,"ea ":3947,"efo":223,"efe":249,"ei ":970,"ega":773,"een":1179,"eel":145,"eed":257,"ees":218,"eer":175,"eep":145,"eet":728,"edi":1118,"ede":1955,"ône":1757,"ông":42975,"eda":404,"eg ":341,"edt":284,"edo":1988,"edr":212,"eck":1046,"ech":639,"eci":3537,"ece":167,"eca":304,"ee ":1150,"ôme":730,"ecu":225,"ect":1210,"eco":767,"dwi":249,"dwe":183,"dwa":258,"dy ":278,"dur":468,"dus":275,"duy":1143,"ôi ":2569,"dor":2168,"dop":306,"don":2791,"dom":306,"dol":520,"dow":395,"dov":304,"dou":172,"dos":360,"ds ":1070,"doa":646,"doc":1372,"dog":488,"dun":458,"dul":398,"duc":230,"dri":731,"dra":523,"dt ":665,"dre":808,"du ":1084,"dro":986,"dha":326,"dge":229,"dic":469,"did":312,"dia":1872,"ôte":1594,"der":3477,"des":3199,"ômé":10659,"dea":701,"ded":323,"dec":231,"del":1465,"den":4157,"dem":911,"deo":220,"di ":3427,"dle":656,"dla":192,"do ":4210,"diu":260,"din":1244,"dio":353,"dis":2108,"dit":379,"die":2847,"dil":249,"rgy":150,"rgu":287,"rhe":221,"rha":208,"rho":196,"rga":532,"ri ":2301,"rgi":1202,"rgh":152,"rge":1773,"rgo":1290,"rgn":567,"ret":1055,"res":3724,"rev":398,"reu":912,"rex":297,"rey":288,"rfa":190,"rfl":255,"nân":179,"rdu":164,"rds":270,"rdr":159,"này":31504,"rg ":4949,"reb":939,"rea":2006,"ree":1006,"ref":247,"rec":523,"red":781,"rei":1635,"reg":1203,"rem":776,"ren":4076,"rel":1428,"rer":300,"reo":199,"rep":265,"rf ":1207,"rda":570,"rcu":715,"rct":554,"rdo":799,"nào":548,"rdi":1755,"rde":2543,"re ":9446,"rbu":335,"rco":417,"rci":274,"rch":1874,"rce":427,"rca":280,"ray":496,"raz":574,"rd ":2860,"rao":655,"rap":552,"raq":160,"rar":494,"ras":1804,"rat":1838,"rau":875,"rav":215,"rbi":1026,"rbo":389,"rba":580,"rbe":418,"rai":3535,"rah":190,"rag":1656,"ran":8266,"ram":1112,"ral":2485,"rak":217,"rab":584,"raf":217,"rae":749,"rad":1615,"rac":3183,"rpu":270,"rpo":754,"rs ":2330,"rpe":165,"rpa":404,"rpi":173,"rph":340,"ror":201,"ros":1675,"rot":1221,"rom":2423,"ron":57984,"lÄ©n":607,"roo":669,"rop":3656,"roy":149,"rou":1200,"rov":1781,"row":593,"rob":744,"roa":358,"rod":644,"roc":1935,"roi":481,"rol":951,"rof":216,"rog":648,"nên":713,"rno":358,"rns":159,"rna":1180,"rne":2695,"rni":1419,"riè":172,"ném":189,"riê":408,"rmo":497,"rms":184,"ro ":929,"rma":3617,"née":2905,"rme":709,"rmi":637,"rly":155,"rlo":157,"rli":457,"rld":399,"rle":351,"rla":627,"rn ":3576,"hÆ¡n":1695,"hÆ¡i":801,"rki":204,"rke":393,"rka":202,"né ":169,"rm ":302,"riz":418,"rix":394,"rl ":161,"rip":365,"rio":1072,"rit":3591,"ris":2983,"riv":318,"riu":456,"rig":1218,"rii":3008,"ril":1020,"rin":4090,"rim":450,"ria":5391,"rib":1175,"ric":5324,"rid":4461,"rie":2155,"rif":320,"rk ":1670,"hÆ¡ ":851,"rwe":155,"nói":685,"rz ":208,"hÆ° ":5084,"nôn":377,"rya":216,"ryc":442,"rug":231,"rue":707,"ruc":377,"rup":171,"run":25178,"rum":705,"rul":242,"ruy":2675,"ruz":213,"rus":1414,"rva":295,"rvi":473,"rve":224,"rwa":347,"ry ":2417,"rsi":891,"rso":252,"roß":201,"rsc":361,"rsd":187,"rsa":1123,"rsb":257,"rsh":319,"rse":899,"rta":737,"óc ":726,"rst":546,"rto":415,"rtb":145,"rte":1749,"rth":2805,"rti":1215,"nó ":2643,"rub":265,"rts":371,"rtr":430,"rtu":180,"rtt":513,"rt ":1957,"óa ":2701,"rqu":185,"rro":1564,"rrh":197,"rri":1788,"rre":2216,"rra":3074,"ru ":477,"rry":300,"rru":147,"sab":140,"sac":1864,"sai":281,"sak":172,"sal":439,"sam":384,"sba":297,"sbe":425,"sao":776,"óng":4090,"san":1255,"sau":2887,"sat":228,"sas":328,"sar":660,"oà ":337,"óp ":288,"sa ":2015,"núi":1191,"óm ":1521,"ón ":2564,"rze":166,"hÆ°a":582,"rys":387,"ryo":172,"ryp":167,"ryl":162,"ói ":993,"ryn":172,"hÆ°n":1523,"sha":619,"năm":27971,"năn":1182,"sho":293,"shr":233,"sht":381,"she":876,"shi":1264,"si ":634,"oãn":159,"sge":580,"sie":368,"sid":739,"sic":687,"sia":3700,"sk ":591,"shw":300,"shu":294,"sit":709,"sis":3021,"sip":265,"sin":5091,"sio":465,"sil":1895,"sim":553,"sii":258,"sif":151,"sig":220,"scr":307,"scu":281,"òng":2223,"oài":34230,"oàn":5706,"sdo":465,"sbu":460,"se ":4575,"oá ":380,"sca":1351,"sce":236,"sci":1194,"sch":1792,"sco":719,"sey":198,"ser":1028,"ses":272,"set":418,"oát":273,"seu":551,"sh ":1939,"sfe":321,"sea":715,"sei":161,"see":284,"sed":245,"sec":229,"sep":202,"sen":3196,"oán":1638,"sem":1371,"sel":2101,"hÆ°Æ¡":4318,"spo":375,"shÅ«":247,"spr":259,"sph":157,"spe":3907,"spi":1229,"spa":351,"sot":292,"sou":945,"sol":467,"som":376,"son":1789,"sop":308,"sor":191,"sof":194,"soi":556,"soc":217,"su ":286,"sra":294,"st ":2960,"squ":256,"ss ":872,"sli":218,"sky":167,"kiể":2684,"sla":1569,"sle":323,"ski":288,"sks":212,"khể":3237,"ska":369,"sna":2980,"sni":207,"sne":250,"smo":583,"siê":264,"so ":947,"sma":1028,"smi":179,"swi":1048,"syn":165,"syl":224,"sse":3231,"soá":206,"ssa":1676,"sso":1413,"ssi":3339,"ssu":544,"ste":5951,"stf":168,"sth":161,"sta":3335,"sto":2068,"stp":174,"sti":4685,"stl":165,"stu":445,"str":4349,"sts":262,"sty":144,"sub":863,"sul":490,"sum":304,"sup":191,"sun":266,"sus":998,"sur":1961,"suy":191,"òa ":5602,"sy ":172,"swa":243,"tai":3180,"tak":236,"tal":2442,"tae":263,"tag":338,"tah":173,"tab":252,"tac":777,"tad":502,"tay":436,"tax":151,"tau":185,"tat":1298,"tas":231,"tar":1441,"tap":149,"tan":2995,"tam":442,"tch":446,"òm ":317,"te ":8300,"tbu":170,"òn ":4112,"ta ":6742,"ký ":793,"ozè":153,"pa ":420,"làm":1846,"làn":610,"lá ":624,"pe ":1111,"par":2272,"pat":396,"pas":155,"là ":152991,"pac":448,"pag":1801,"pal":1430,"pan":1032,"phe":584,"pha":2087,"phu":325,"phr":290,"pho":3352,"phn":208,"phi":5842,"pi ":273,"lãn":895,"ph ":150,"lâu":392,"lâm":192,"pea":417,"pec":3297,"ped":468,"pen":678,"per":2665,"pet":646,"pes":3121,"pel":638,"pla":748,"hiể":23255,"pli":245,"phâ":2752,"phá":7538,"ple":773,"lès":368,"phí":12064,"phê":328,"plo":417,"phé":481,"phò":631,"phó":453,"phy":787,"pia":524,"pid":686,"pic":879,"pie":144,"pil":547,"pin":2627,"pio":231,"pir":455,"pis":429,"pit":358,"por":1000,"pop":359,"pot":426,"pos":699,"pom":618,"pon":447,"pol":658,"poc":162,"pod":1012,"ps ":851,"hÅ« ":248,"ppi":1358,"ppo":224,"ppe":777,"phú":224,"po ":176,"lí ":1217,"lên":761,"pta":227,"pse":181,"psi":853,"pso":273,"ptu":208,"pua":385,"pub":253,"puc":309,"pte":1527,"pti":1324,"pto":542,"ptr":151,"pra":304,"hoể":8808,"pt ":251,"phÆ°":1981,"pri":732,"pre":749,"pro":835,"huể":53256,"lôn":185,"lôm":10657,"pur":817,"pus":345,"pun":212,"pul":573,"lô ":685,"px ":1130,"pyr":166,"lý ":2650,"lúc":371,"mà ":1433,"màn":167,"máy":1955,"mã ":867,"màu":877,"mét":23053,"méo":221,"mìn":517,"qua":5837,"mô ":773,"quy":3085,"que":2033,"qui":2180,"món":229,"môn":561,"môi":520,"quâ":3680,"quá":966,"quê":311,"quý":155,"mùa":374,"ra ":16536,"ngo":2830,"ngi":1741,"ngl":604,"ngk":252,"ngu":5422,"ngr":255,"ngt":407,"ngs":604,"ni ":1381,"nge":3237,"ngh":7783,"nga":2179,"nho":251,"ndé":306,"nhu":514,"nha":2433,"nhi":5815,"nhe":263,"neg":239,"nei":512,"nel":1599,"nen":680,"nem":262,"neo":282,"ner":1201,"net":904,"nes":4373,"nev":164,"neu":463,"ng ":357107,"nea":1462,"neb":278,"nec":300,"ned":346,"nee":182,"nfo":163,"ney":270,"nez":428,"nh ":132657,"nfe":333,"nct":449,"nco":534,"nci":1276,"ncl":257,"nce":3207,"nch":2383,"nca":570,"ne ":19277,"nbu":1513,"ndu":416,"ndr":1581,"nds":990,"ndo":2398,"ndl":619,"ndh":176,"ndi":4609,"nde":3318,"nda":1999,"ncy":170,"nal":1151,"nam":13111,"nan":913,"nar":774,"nac":1244,"nad":927,"nae":1024,"nag":697,"nai":3249,"nbo":145,"nbe":462,"nd ":12230,"nba":380,"nav":164,"nau":516,"nat":2078,"nas":580,"nay":2459,"na ":6835,"iúp":291,"ê°€ ":328,"cÆ° ":415,"iôn":176,"myc":163,"nya":523,"nyi":193,"nz ":267,"ny ":1333,"nvi":479,"nx ":222,"nul":298,"num":443,"nus":3075,"nut":506,"nty":550,"nto":2015,"ntu":367,"nts":334,"ntr":2166,"nti":3110,"nth":1365,"ntl":140,"nta":3026,"nte":4904,"nsu":730,"nsy":214,"nso":325,"nst":981,"nsf":180,"nse":669,"nsh":461,"nsi":2204,"nsl":567,"nsk":139,"nsc":152,"nsa":421,"nsb":373,"nhÆ°":5336,"nt ":8221,"ngÆ°":22262,"nqu":229,"ns ":3001,"noc":411,"nod":303,"hÄ©a":2885,"nob":365,"nol":370,"noi":432,"nop":786,"nom":551,"non":799,"not":862,"nos":776,"nor":939,"now":1063,"nov":236,"nou":281,"nne":5126,"nna":750,"nno":345,"nni":716,"nns":289,"nma":453,"niê":790,"nhâ":2593,"nn ":413,"nla":2159,"nhá":489,"nhà":6343,"nhó":1299,"nly":637,"no ":1110,"hÄ© ":710,"nhì":354,"ngâ":223,"nke":271,"nki":377,"ngà":10009,"nka":598,"ngô":1896,"nkt":202,"nja":290,"nii":438,"nig":289,"nif":242,"nie":473,"nid":3745,"nic":2037,"nia":5524,"nk ":242,"nix":164,"niu":181,"niv":273,"nis":1435,"nit":1158,"nio":409,"nim":283,"nin":1178,"nik":149,"nil":357,"ogr":267,"ogu":214,"ogi":228,"ogl":349,"ogo":319,"ogn":1408,"oga":317,"oge":346,"oi ":1139,"ohn":303,"oha":229,"ohe":168,"ogy":195,"ois":1103,"oir":3370,"oit":1455,"oin":300,"oil":149,"oid":1389,"oie":174,"ok ":359,"ol ":657,"oce":820,"och":1882,"oci":330,"ock":741,"ocl":146,"oco":568,"ocr":154,"obu":172,"oe ":211,"oca":1110,"occ":493,"ode":1045,"odi":450,"odo":1089,"odr":210,"oct":1918,"ocy":317,"of ":6954,"oda":691,"dÆ°Æ¡":848,"oen":367,"odu":285,"oed":204,"og ":551,"oft":282,"off":252,"ofe":161,"oa ":13629,"oc ":1250,"oan":1921,"oad":230,"oba":818,"od ":1168,"oar":504,"oas":324,"oat":240,"obo":175,"obr":145,"obl":325,"obi":968,"obe":655,"nym":211,"nza":905,"nze":176,"oya":255,"oxy":147,"oxi":202,"oz ":147,"guể":1293,"ows":383,"owl":142,"own":1769,"owi":183,"ozo":255,"oza":724,"otu":288,"oud":174,"oub":187,"ouc":419,"oua":154,"ow ":1038,"oti":1060,"oth":1496,"ote":697,"ott":720,"ots":282,"otr":764,"oto":966,"ost":2982,"osu":324,"ota":1031,"ov ":258,"osi":491,"osh":227,"ose":2073,"osg":493,"osp":425,"oss":826,"osm":547,"oso":413,"osn":140,"oy ":273,"owe":547,"ovi":954,"ovo":159,"ouv":263,"oux":242,"ova":705,"ove":2204,"oug":763,"oui":391,"oul":679,"oun":2537,"oup":387,"ous":2368,"our":3149,"out":2276,"opo":1519,"opp":188,"opi":1120,"opl":220,"ope":1422,"oph":3191,"opa":411,"os ":2470,"opu":314,"opt":1403,"ops":1255,"oon":440,"ool":189,"oom":247,"ook":283,"oog":197,"ood":513,"or ":3376,"oot":347,"oor":232,"ork":631,"orl":473,"orm":2975,"orn":1953,"oro":870,"orp":1047,"orr":2602,"orc":598,"ord":3003,"ore":2045,"orf":1239,"org":708,"ori":3584,"ou ":1707,"osa":1094,"osc":366,"ort":2762,"ors":1266,"oru":667,"ory":1225,"kín":427,"kíc":629,"ot ":1197,"goể":939,"m² ":639,"orb":599,"ora":2404,"oqu":150,"ola":2194,"old":517,"kê ":1566,"giể":12764,"on ":15999,"oli":2444,"oll":1993,"olf":295,"ole":1581,"ols":1043,"olt":149,"olm":148,"olo":2220,"oly":684,"olz":312,"olu":1314,"olv":159,"oka":251,"ghể":2336,"om ":2129,"okk":143,"oki":163,"oke":162,"oku":288,"ona":2358,"ond":2416,"onc":461,"onf":214,"one":2805,"ong":59905,"oni":4577,"onl":675,"onn":2036,"kên":176,"ono":1589,"ons":1629,"ont":3656,"onu":697,"onv":312,"ony":549,"gÅ© ":341,"kí ":158,"oma":3160,"oo ":152,"ome":1667,"omb":1058,"omi":1212,"omm":2408,"omp":760,"omo":798,"kéo":312,"omu":461,"omy":255,"op ":502,"kì ":200,"la ":10761,"ính":11878,"ín ":1003,"há ":535,"le ":10516,"lca":204,"ít ":628,"lch":163,"lf ":224,"lde":761,"ldb":159,"lda":161,"hào":150,"ldo":198,"hàn":21780,"hàm":435,"hài":213,"ldi":250,"lab":417,"lac":1406,"lad":738,"lae":401,"lah":182,"lag":424,"laj":292,"lai":1546,"lal":167,"lan":10384,"lam":1066,"lap":209,"lao":259,"lar":4422,"lat":2293,"las":1229,"law":585,"lau":512,"lav":645,"lay":1413,"lba":423,"hà ":7105,"ld ":1873,"lbe":635,"lbi":189,"lbo":527,"lbu":797,"gô ":876,"góc":144,"ky ":446,"ích":17834,"cuể":3904,"gôi":662,"góp":269,"gôn":1009,"hìn":3185,"llé":141,"hìm":147,"lpe":2886,"lpi":216,"lph":225,"ls ":3004,"híc":534,"hía":11942,"lok":158,"lon":3179,"lom":1001,"lop":1213,"lor":1602,"lod":250,"loc":750,"loe":167,"log":693,"loi":242,"los":940,"lot":496,"lou":372,"lov":598,"low":793,"hêm":276,"lob":302,"hí ":2215,"liê":1520,"hép":885,"lmo":317,"lme":324,"lma":282,"hì ":1109,"lti":486,"lto":187,"hó ":498,"lud":253,"luc":198,"lue":351,"lso":351,"lst":1066,"lta":363,"lte":462,"lu ":183,"lse":196,"loà":31500,"lsa":619,"ía ":12221,"hín":8169,"lt ":1144,"lhe":150,"lha":193,"hãn":1016,"lge":195,"li ":845,"lga":417,"hât":447,"hâu":5143,"hân":32338,"hâm":245,"lfe":172,"ley":541,"lex":375,"leu":725,"lev":279,"les":4804,"hát":4309,"let":890,"ler":1633,"leo":683,"háo":303,"lep":286,"háp":24791,"hám":208,"lem":456,"len":2814,"hán":13040,"lel":163,"lei":443,"hái":3807,"leg":371,"lef":140,"led":786,"hác":3511,"lec":514,"leb":324,"lea":958,"lls":526,"llu":1871,"lly":728,"lo ":703,"lla":8252,"llb":208,"lle":6069,"lli":4980,"llo":3130,"lks":330,"hê ":529,"diể":17680,"lka":390,"lm ":305,"ll ":2030,"hè ":211,"lit":1006,"lis":1811,"lip":1960,"lio":1480,"lin":3515,"lim":803,"liz":196,"liv":616,"liu":661,"lic":1528,"lid":3931,"lia":3554,"lk ":142,"lik":204,"lii":571,"lig":457,"lie":1623,"lif":1017,"ma ":3578,"húa":466,"húc":1589,"hún":1289,"hút":209,"húy":143,"mb ":269,"ìm ":8521,"mac":644,"mai":885,"maj":203,"mad":498,"mae":190,"ìn ":392,"mag":301,"hür":727,"mar":2810,"mas":488,"mal":1566,"man":6200,"maz":173,"mat":1698,"mba":981,"mbl":196,"mbi":1974,"mbe":1329,"mbr":351,"mbo":504,"me ":3769,"iá ":976,"iße":139,"mbu":270,"ình":20204,"iàn":475,"med":457,"meg":343,"mea":185,"iác":256,"mec":144,"met":1158,"iáp":2001,"mes":1080,"mer":2891,"iám":415,"mem":171,"mel":1053,"iáo":2698,"ián":260,"men":4045,"mei":235,"luy":242,"hòa":4573,"lva":958,"hô ":347,"lve":227,"lvi":160,"lul":235,"lun":153,"lum":1803,"lut":702,"lus":2401,"ly ":5413,"hóa":2227,"hòm":296,"hòn":1026,"lz ":1932,"hôi":152,"hón":422,"hóm":1446,"hôn":8959,"luô":144,"lyp":567,"lym":264,"lyn":185,"hù ":377,"hú ":960,"hùn":400,"hùa":429,"mpi":548,"mph":909,"mpe":699,"mpr":166,"mpo":339,"mpl":212,"mpu":526,"mps":257,"ms ":540,"moc":217,"mod":141,"mon":2835,"mop":418,"mol":919,"mor":1191,"mos":484,"mot":349,"mou":592,"mpa":2137,"mu ":149,"mua":252,"mst":192,"my ":346,"mur":466,"mus":478,"mul":387,"mun":1488,"hăm":190,"hăn":308,"mi ":443,"min":2275,"mil":2845,"mir":352,"mis":870,"mit":1161,"cÆ¡ ":2355,"mic":1181,"mib":195,"mia":637,"mie":192,"mid":343,"mo ":242,"ièr":623,"mm ":173,"iêu":3120,"mni":139,"iêm":793,"iên":15316,"mno":232,"mmu":465,"mmi":508,"miê":868,"mmo":992,"mma":550,"mme":1127,"xâm":158,"xây":774,"xã ":6613,"xác":753,"thể":105642,"tiể":16334,"vÄ© ":1285,"Äể ":4107,"suể":581,"Äểt":393,"Äểu":259,"Äểa":425,"Äểc":8666,"Äểi":2655,"Äển":2345,"Äểo":612,"văn":2376,"soể":246,"vô ":966,"vòn":456,"zue":365,"zur":1019,"ruể":225,"võ ":369,"hưể":6739,"vùn":22597,"vây":180,"vào":6478,"vàn":747,"zen":390,"zel":179,"zer":396,"ze ":541,"vài":326,"và ":39376,"zam":416,"zan":947,"zak":194,"zar":156,"ví ":164,"zon":909,"zo ":189,"vì ":1170,"zna":154,"riể":3937,"zia":217,"zie":194,"zin":147,"zil":497,"yré":2878,"yx ":207,"yth":348,"yst":455,"yso":284,"ysi":882,"yri":846,"yro":276,"yra":354,"yrg":171,"yre":315,"ys ":1775,"yph":446,"ypt":526,"ypr":406,"ypo":300,"ype":314,"yon":322,"uý ":440,"za ":604,"gưể":22834,"uôn":11800,"uôi":272,"quể":14720,"ye ":286,"uá ":683,"yca":261,"yce":225,"ych":303,"ycl":287,"yco":153,"yct":483,"ydr":465,"yer":937,"uán":332,"yen":509,"ya ":1624,"yat":304,"yan":957,"yal":151,"uê ":487,"yla":353,"yle":207,"yli":383,"yll":1078,"ylo":482,"ylv":243,"ylu":187,"yma":185,"sÄ© ":1783,"yo ":184,"yme":163,"ymp":602,"ymn":164,"yna":220,"yne":184,"yno":257,"uân":5407,"yi ":308,"tăn":371,"yho":211,"yin":196,"tín":2825,"để ":24120,"tíc":15647,"đểy":569,"xtr":322,"đểu":7023,"đểt":4542,"đểa":6133,"đểi":9089,"đểc":6272,"đển":28910,"đểo":4093,"đểp":378,"đểm":552,"tên":7356,"tìn":624,"tìm":8334,"xon":351,"xoa":253,"tù ":245,"tô ":345,"tòa":324,"xuy":162,"xun":238,"tôn":573,"săn":488,"xi ":160,"tây":9977,"tâm":1507,"tán":178,"xem":772,"tác":2521,"tái":142,"tàn":309,"tàu":1978,"nÆ¡i":1970,"phể":19501,"xil":335,"xin":153,"xic":980,"xa ":336,"tài":1059,"xce":150,"xe ":304,"xas":712,"xan":855,"ww ":171,"www":171,"són":349,"sôn":1977,"wo ":144,"yểu":888,"yểt":2656,"yển":27686,"sên":250,"wn ":1674,"sèr":449,"ws ":411,"wor":456,"woo":295,"we ":324,"sân":742,"wes":568,"wer":566,"sáu":219,"sát":605,"wen":170,"sán":1258,"wel":193,"wei":504,"wed":169,"wee":268,"web":356,"sác":1906,"whe":719,"whi":453,"răn":535,"sâu":259,"wi ":419,"wit":1057,"wig":1125,"wid":265,"wic":172,"win":477,"wil":183,"rös":588,"lÆ°u":884,"vuô":11318,"dưể":1608,"rùn":419,"wa ":340,"rúc":638,"wan":646,"wal":651,"way":264,"wat":547,"war":864,"was":494,"wai":552,"rüc":145,"ría":161,"vre":519,"rò ":634,"vua":1478,"vul":314,"rõ ":263,"ròn":162,"rôm":369,"via":1024,"vir":262,"vil":2132,"vin":903,"vig":163,"vic":424,"vid":453,"vie":338,"ngể":4112,"vit":212,"vis":359,"nhể":20937,"ré ":197,"niể":631,"rén":2896,"viê":2261,"rì ":216,"rên":16855,"rí ":838,"voi":343,"vol":291,"von":231,"vor":176,"rìn":2591,"vi ":814,"râu":162,"rãi":238,"ver":2511,"ves":443,"vet":169,"ràn":171,"rào":384,"rái":592,"rán":159,"ven":2308,"vel":746,"rác":292,"ve ":1535,"val":496,"vak":211,"van":1011,"var":683,"vat":444,"rà ":170,"vad":360,"vai":638,"va ":1421,"uyê":4853,"cưể":274,"uze":169,"uzn":158,"uya":181,"uxe":167,"uxo":172,"muể":253,"ux ":1397,"uvi":315,"uve":890,"uy ":4510,"usk":685,"ush":495,"usi":1086,"use":2256,"usc":486,"usa":422,"usu":188,"ust":2067,"uss":2125,"utm":305,"uth":2069,"uti":986,"ute":4039,"uta":488,"utt":513,"uts":241,"utu":168,"uto":356,"us ":12540,"ut ":1468,"urb":527,"ura":1745,"urc":370,"ure":2141,"urg":4458,"uri":3154,"urk":298,"urn":505,"uro":993,"urp":172,"urr":1188,"urs":436,"urt":1515,"uru":494,"ury":261,"ur ":3979,"uph":337,"upi":208,"upe":461,"upl":256,"umi":350,"umo":202,"uma":655,"umb":1047,"ume":427,"unt":1093,"uns":220,"uni":1404,"unn":257,"unc":850,"und":2117,"una":410,"ung":28650,"une":861,"up ":561,"uki":214,"um ":5338,"ulu":810,"ult":827,"ulo":594,"ulm":200,"ull":475,"uli":2981,"ulg":343,"ule":468,"ulc":170,"ula":2994,"ulb":224,"miể":7160,"un ":876,"uid":1911,"uil":817,"uin":1133,"uis":537,"uit":1805,"ul ":704,"ugh":635,"uge":202,"ugl":142,"ui ":422,"uga":249,"ugu":343,"uco":341,"uct":140,"ucu":155,"uda":464,"ude":797,"udi":503,"ubs":175,"ubr":196,"uca":423,"ue ":2123,"uce":268,"ucc":527,"uci":266,"uch":1204,"ucl":244,"uck":465,"uet":161,"uev":189,"uer":585,"ues":940,"uff":186,"udo":607,"udw":163,"uee":563,"ued":1142,"ueb":187,"uen":891,"uel":938,"ub ":179,"ua ":4516,"uay":450,"uat":643,"uar":339,"ual":262,"uan":4521,"ubi":473,"ubl":385,"ube":682,"ubf":310,"uba":515,"ud ":256,"uai":209,"uad":968,"tze":378,"tyl":438,"typ":380,"bưể":9335,"trÆ°":5334,"trù":418,"ty ":2056,"trú":686,"trò":743,"trí":887,"trì":2564,"trê":16809,"trá":625,"trà":444,"tvi":145,"tuy":1965,"tur":1038,"tus":1579,"tut":187,"tui":1819,"tul":991,"tun":226,"tum":525,"tud":242,"tuc":215,"luể":1046,"tz ":904,"two":235,"tră":326,"ts ":2029,"lÅ©n":205,"tre":2316,"loể":3367,"tt ":262,"tra":13075,"thÆ¡":816,"tri":6787,"tru":17464,"tro":55540,"thÆ°":5419,"tu ":263,"try":204,"toá":1052,"tsc":237,"toà":1057,"tsu":330,"tsw":157,"tta":873,"tte":2605,"tti":475,"ttl":380,"tto":705,"ttp":325,"tts":284,"thă":146,"tme":444,"tma":160,"thú":506,"thù":145,"to ":4755,"thô":2398,"tiê":3084,"tp ":325,"tna":209,"toe":205,"tod":450,"toc":558,"toi":213,"tog":204,"tob":196,"tou":1449,"tos":429,"tot":153,"tow":612,"tom":1725,"ton":3013,"tol":487,"tor":2310,"top":602,"tr ":327,"tii":370,"til":3476,"tif":586,"tie":501,"tig":741,"tir":204,"tiq":562,"tit":511,"tis":1648,"tin":5106,"tim":1567,"tip":239,"tio":2405,"thy":410,"thu":53072,"thw":183,"tia":1585,"tic":3507,"tid":1768,"tiu":237,"tiv":602,"tli":260,"thê":286,"thé":143,"thí":558,"thì":784,"liể":1722,"tla":967,"thâ":11890,"thà":11154,"tle":1005,"thá":12170,"tem":2257,"ten":2299,"tep":155,"tei":2098,"tel":2040,"tee":338,"teg":403,"tea":605,"tec":172,"ted":1492,"tfa":159,"th ":4032,"tev":154,"teu":168,"tet":184,"tes":3544,"ter":9430,"ti ":720,"tho":1935,"thm":205,"thr":908,"the":21046,"thi":5961,"tha":4235,"之三 ":187,"之万 ":161,"rưể":6336,"之专 ":252,"ăk ":141,"ăm ":28988,"ăn ":5275,"ăng":3284,"xÆ°a":183,"vÆ°Æ¡":541,"nưể":35806,"丘 ":240,"mưể":240,"专 ":1368,"xuể":3053,"並 ":377,"tÆ°Æ¡":996,"lưể":2704,"三 ":1277,"ä¸ ":542,"万 ":771,"uyể":30286,"ä¹™ ":247,"sÆ° ":673,"tÆ° ":1117,"zèr":162,"vÅ© ":670,"ürt":532,"üri":758,"viể":4820,"rÆ°Æ¡":293,"rÆ°n":427,"tuể":1284,"trể":43603,"yêu":353,"之 ":589,"yên":4958,"sÆ¡ ":163},"n_words":[13809827,17315344,13396979],"name":"vi","type":"latin", "flags": ["diacritics"]} \ No newline at end of file
diff --git a/contrib/lc-btrie/CMakeLists.txt b/contrib/lc-btrie/CMakeLists.txt
new file mode 100644
index 0000000..a5fe9e5
--- /dev/null
+++ b/contrib/lc-btrie/CMakeLists.txt
@@ -0,0 +1,6 @@
+SET(LCTRIESRC btrie.c)
+ADD_LIBRARY(lcbtrie STATIC ${LCTRIESRC})
+
+SET(LCTRIE_CFLAGS "-DBUILD_RSPAMD")
+
+set_target_properties(lcbtrie PROPERTIES COMPILE_FLAGS "${LCTRIE_CFLAGS}") \ No newline at end of file
diff --git a/contrib/lc-btrie/btrie.c b/contrib/lc-btrie/btrie.c
new file mode 100644
index 0000000..81b69b2
--- /dev/null
+++ b/contrib/lc-btrie/btrie.c
@@ -0,0 +1,2644 @@
+/* Level-Compressed Tree Bitmap (LC-TBM) Trie implementation
+ *
+ * Contributed by Geoffrey T. Dairiki <dairiki@dairiki.org>
+ *
+ * This file is released under a "Three-clause BSD License".
+ *
+ * Copyright (c) 2013, Geoffrey T. Dairiki
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * * Neither the name of Geoffrey T. Dairiki nor the names of other
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GEOFFREY
+ * T. DAIRIKI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*****************************************************************
+ *
+ * This code implements a routing table conceptually based on a binary
+ * trie structure. Internally, the trie is represented by two types
+ * of compound nodes: "multibit nodes", which contain the top few
+ * levels of an entire binary subtree; and "level compression" (LC)
+ * nodes which represent a (potentially long) chain of out-degree one
+ * (single child) binary nodes (possibly ending at a terminal node).
+ *
+ * The multibit nodes are represented using a "Tree Bitmap" structure
+ * (more on this below), which is very efficient --- both in terms of
+ * memory usage and lookup speed --- at representing densely branching
+ * parts of the trie. The LC nodes can efficiently represent long
+ * non-branching chains of binary trie nodes. Using both node types
+ * together results in efficient representation of both the sparse and
+ * dense parts of a binary trie.
+ *
+ * Graphically, here's the rough idea:
+ *
+ * ........
+ * .LC o .
+ * . / . LC nodes can
+ * . o . <= represent long chains
+ * . \ . of (non-branching) binary
+ * . o . trie nodes
+ * . / .
+ * . o .
+ * ......../.....
+ * .TBM o .
+ * . / \ . TBM nodes can represent
+ * . o * . <= several levels of densely
+ * . / \ . branching binary trie nodes
+ * . o o .
+ * ......./.....\.......
+ * .TBM o .. o LC.
+ * . / \ .. \ .
+ * . o o .. o .
+ * . / / \ .. \ .
+ * . * o *.. o .
+ * ...../....... / .
+ * . o LC. . o .
+ * . \ . .....\......
+ * . * . . o TBM.
+ * ........ . / \ .
+ * . o o .
+ * . / \ \ .
+ * .* * *.
+ * ...........
+ *
+ * Terminology
+ * -----------
+ *
+ * node
+ * Usually, in the comments below, "node" will be used to refer to
+ * a compound node: either a multibit (TBM) node or an LC node.
+ *
+ * "internal node" or "prefix"
+ * The terms "prefix" or "internal node" are used to refer to
+ * a node in the binary trie which is internal to a multibit (TBM)
+ * node.
+ *
+ * ----------------------------------------------------------------
+ *
+ * Internal Representation of the Nodes
+ * ====================================
+ *
+ * Multibit (TBM) Nodes
+ * ~~~~~~~~~~~~~~~~~~~~
+ *
+ * The multibit nodes are represented using a "Tree Bitmap" (TBM)
+ * structure as described by Eatherton, Dittia and Varghese[1]. See
+ * the paper referenced below for basic details.
+ *
+ * A multibit node, represents several levels of a binary trie.
+ * For example, here is a multibit node of stride 2 (which represent
+ * two levels of a binary trie.
+ *
+ * +------- | ------+
+ * | multi o |
+ * | bit / \ |
+ * | node / \ |
+ * | o * |
+ * +--- / \ - / \ --+
+ * O
+ *
+ * Note that, for a multibit node of stride S, there are 2^S - 1 internal
+ * nodes, each of which may have data (or not) associated with them, and
+ * 2^S "external paths" leading to other (possibly compound nodes).
+ * (In the diagram above, one of three internal node (the one denoted by "*")
+ * has data, and one of four extending paths leads to an external node
+ * (denoted by the 'O').)
+ *
+ * The TBM structure can represent these bitmaps in a very memory-efficient
+ * manner.
+ *
+ * Each TBM node consists of two bitmaps --- the "internal bitmap" and the
+ * "extending paths bitmap" --- and a pointer which points to an array
+ * which contains both the extending path ("child") nodes and any
+ * internal prefix data for the TBM node.
+ *
+ * +--------+--------+
+ * TBM | ext bm | int bm |
+ * Node +--------+--------+
+ * | pointer |----+
+ * +-----------------+ |
+ * |
+ * |
+ * +-----------------+ |
+ * | extending path | |
+ * | node[N-1] | |
+ * +-----------------+ |
+ * / ... / |
+ * / ... / |
+ * +-----------------+ |
+ * | extending path | |
+ * | node[0] | |
+ * +-----------------+<---+
+ * | int. data[M-1] |
+ * +-----------------+
+ * / ... /
+ * +-----------------+
+ * | int. data[0] |
+ * +-----------------+
+ *
+ * The extending paths bitmap (or "ext bitmap") has one bit for each
+ * possible "extending path" from the bottom of the multibit node. To
+ * check if a particular extending path is present, one checks to see if
+ * the corresponding bit is set in the ext bitmap. The index into the
+ * array of children for that path can be found by counting the number
+ * of set bits to the left of that bit.
+ *
+ * Similarly, the internal bitmap has one bit for each binary node
+ * which is internal to the multibit node. To determine whether there
+ * is data stored for an internal prefix, one checks the corresponding
+ * bit in the internal bitmap. As for extending paths, the index into
+ * the array of internal data is found by counting the number of set
+ * bits to the left of that bit.
+ *
+ * To save space in the node structure, the node data array is stored
+ * contiguously with the node extending path array. The single
+ * ("children") pointer in the TBM structure points to the beginning
+ * of the array of extending path nodes and to (one past) the end of
+ * the the internal data array.
+ *
+ * The multibit stride is chosen so that the entire TBM node structure fits
+ * in the space of two pointers. On 32 bit machines this means the stride
+ * is four (each of the two bitmaps is 16 bits); on 32 bit machines the
+ * stride is five.
+ *
+ * Note that there are only 2^stride - 1 internal prefixes in a TBM
+ * node. That means there is one unused bit in the internal bitmap.
+ * We require that that bit must always be clear for a TBM node. (If
+ * set, it indicates that the structure represents, instead, an LC
+ * node. See below.)
+ *
+ * ----------------------------------------------------------------
+ *
+ * Level Compression (LC) Nodes
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * LC nodes are used to represent a chain of out-degree-one (single
+ * child) prefixes in the binary trie. The are represented by a bit
+ * string (the "relative prefix") along with its length and a pointer
+ * to the extending path (the next node past the LC node.)
+ *
+ *
+ * Non-Terminal LC Node:
+ *
+ * +------------------+-------+
+ * | relative prefix |1|0|len|
+ * +------------------+-------+
+ * | ptr.child |--+
+ * +--------------------------+ |
+ * |
+ * |
+ * +--------------------------+ |
+ * | Next node - | |
+ * | either LC or TBM | |
+ * | | |
+ * +--------------------------+<-+
+ *
+ * The Relative Prefix
+ * -------------------
+ *
+ * The maximum relative prefix per LC node is selected so that (again)
+ * the entire node structure fits in the space of two pointers. On 32 bit
+ * machines, the maximum relative prefix is 24 bits; on 62 bit machines
+ * the limit is 56 bits.
+ *
+ * In the LC node structure, the relative prefix is stored as an array
+ * of bytes. To avoid some bit-shifting during tree searches, these
+ * bytes are byte-aligned with the global prefix. In other words, in
+ * general there are (pos % 8) "pad" bits at the beginning of the
+ * relative prefix --- where pos "starting bit" (or depth in the
+ * binary tree) of the LC node --- which really belong to the parent
+ * node(s) of the LC node. For efficiency (so that we don't have to
+ * mask them out when matching) we require that these pad bits be
+ * correct --- they must match the path which leads to the LC node.
+ *
+ * The relative prefix length stored in the LC node structure does not
+ * count the pad bits.
+ *
+ * Terminal Node Compression
+ * -------------------------
+ *
+ * For memory efficiency, we also support "terminal LC" nodes. When
+ * the extension path from an LC node consists a single terminal node,
+ * we store that terminal nodes data directly in the parent LC node.
+ *
+ * Instead of this:
+ *
+ * +------------------+-------+
+ * | relative prefix |1|0|len|
+ * +------------------+-------+
+ * | ptr.child |--+
+ * +--------------------------+ |
+ * |
+ * +--------------------------+ |
+ * | Terminal Node (TBM node, | |
+ * | empty except for the | |
+ * +--| root internal node.) | |
+ * | +--------------------------+<-+
+ * |
+ * +->+--------------------------+
+ * | terminal node data |
+ * +--------------------------+
+ *
+ * We can do this:
+ *
+ * +------------------+-------+
+ * | relative prefix |1|1|len|
+ * +------------------+-------+
+ * | terminal node data |
+ * +--------------------------+
+ *
+ * Terminal LC nodes are differentiated from non-terminal LC nodes
+ * by the setting of the is_terminal flag.
+ *
+ * Node Structure Packing Details
+ * ------------------------------
+ *
+ * The LC and TBM node structures are carefully packed so that the
+ * "is_lc" flag (which indicates that a node is an LC node)
+ * corresponds to the one unused bit in the internal bitmap of the TBM
+ * node structure (which we require to be zero for TBM nodes).
+ *
+ * ----------------------------------------------------------------
+ *
+ * References
+ * ==========
+ *
+ * [1] Will Eatherton, George Varghese, and Zubin Dittia. 2004. Tree
+ * bitmap: hardware/software IP lookups with incremental
+ * updates. SIGCOMM Comput. Commun. Rev. 34, 2 (April 2004),
+ * 97-122. DOI=10.1145/997150.997160
+ * http://doi.acm.org/10.1145/997150.997160
+ * http://comnet.kaist.ac.kr/yhlee/CN_2008_Spring/readings/Eath-04-tree_bitmap.pdf
+ *
+ ****************************************************************/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <setjmp.h>
+#if defined(TEST) && defined(NDEBUG)
+# warning undefining NDEBUG for TEST build
+# undef NDEBUG
+#endif
+#include <assert.h>
+
+#include "btrie.h"
+#include "libutil/mem_pool.h"
+
+#ifdef __SIZEOF_POINTER__
+#define SIZEOF_VOID_P __SIZEOF_POINTER__
+#else
+#if defined(__ILP32__) || defined(__ILP32) || defined(_ILP32)
+# define SIZEOF_VOID_P 4
+#elif defined(__ILP64__) || defined(__ILP64) || defined(_ILP64)
+# define SIZEOF_VOID_P 8
+#elif defined(__LLP64__) || defined(__LLP64) || defined(_LLP64) || defined(_WIN64)
+# define SIZEOF_VOID_P 8
+#elif defined(__LP64__) || defined(__LP64) || defined(_LP64)
+# define SIZEOF_VOID_P 8
+#elif defined(UINTPTR_MAX) && defined(UINT64_MAX) && (UINTPTR_MAX == UINT64_MAX)
+# define SIZEOF_VOID_P 8
+#else
+# define SIZEOF_VOID_P 4
+#endif
+#endif
+
+#if SIZEOF_VOID_P == 4
+# define TBM_STRIDE 4
+#elif SIZEOF_VOID_P == 8
+# define TBM_STRIDE 5
+#else
+# error "Unsupported word size"
+#endif
+
+#ifndef NO_STDINT_H
+# if TBM_STRIDE == 4
+typedef uint16_t tbm_bitmap_t;
+# else
+typedef uint32_t tbm_bitmap_t;
+# endif
+#else /* NO_STDINT_H */
+# if TBM_STRIDE == 4
+# if SIZEOF_SHORT == 2
+typedef short unsigned tbm_bitmap_t;
+# else
+# error "can not determine type for 16 bit unsigned int"
+# endif
+# else /* TBM_STRIDE == 5 */
+# if SIZEOF_INT == 4
+typedef unsigned tbm_bitmap_t;
+# elif SIZEOF_LONG == 4
+typedef long unsigned tbm_bitmap_t;
+# else
+# error "can not determine type for 32 bit unsigned int"
+# endif
+# endif
+#endif
+
+#define TBM_FANOUT (1U << TBM_STRIDE)
+#define LC_BYTES_PER_NODE (SIZEOF_VOID_P - 1)
+
+typedef union node_u node_t;
+
+/* The tbm_node and lc_node structs must be packed so that the the
+ * high bit (LC_FLAGS_IS_LC) of lc_flags in the the lc_node struct
+ * coincides with bit zero (the most significant bit) of tbm_node's
+ * int_bm. (This bit is how we differentiate between the two node
+ * types. It is always clear for a tbm_node and always set for an
+ * lc_node.)
+ */
+
+struct tbm_node
+{
+#ifdef WORDS_BIGENDIAN
+ tbm_bitmap_t int_bm; /* the internal bitmap */
+ tbm_bitmap_t ext_bm; /* extending path ("external") bitmap */
+#else
+ tbm_bitmap_t ext_bm; /* extending path ("external") bitmap */
+ tbm_bitmap_t int_bm; /* the internal bitmap */
+#endif
+ union
+ {
+ node_t *children; /* pointer to array of children */
+ const void **data_end; /* one past end of internal prefix data array */
+ } ptr;
+};
+
+struct lc_node
+{
+ /* lc_flags contains the LC prefix length and a couple of bit flags
+ * (apparently char-sized bit fields are a gcc extension)
+ */
+# define LC_FLAGS_IS_LC 0x80
+# define LC_FLAGS_IS_TERMINAL 0x40
+# define LC_FLAGS_LEN_MASK 0x3f
+#ifdef WORDS_BIGENDIAN
+ btrie_oct_t lc_flags;
+ btrie_oct_t prefix[LC_BYTES_PER_NODE];
+#else
+ btrie_oct_t prefix[LC_BYTES_PER_NODE];
+ btrie_oct_t lc_flags;
+#endif
+ union
+ {
+ node_t *child; /* pointer to child (if !is_terminal) */
+ const void *data; /* the prefix data (if is_terminal) */
+ } ptr;
+};
+
+union node_u
+{
+ struct tbm_node tbm_node;
+ struct lc_node lc_node;
+};
+
+struct free_hunk
+{
+ struct free_hunk *next;
+};
+
+#define MAX_CHILD_ARRAY_LEN (TBM_FANOUT + TBM_FANOUT / 2)
+
+struct btrie
+{
+ node_t root;
+
+ rspamd_mempool_t *mp;
+ struct free_hunk *free_list[MAX_CHILD_ARRAY_LEN];
+ jmp_buf exception;
+ /* mem mgmt stats */
+ size_t alloc_total; /* total bytes allocated from mempool */
+ size_t alloc_data; /* bytes allocated for TBM node int. prefix data */
+ size_t alloc_waste; /* bytes wasted by rounding of data array size */
+#ifdef BTRIE_DEBUG_ALLOC
+ size_t alloc_hist[MAX_CHILD_ARRAY_LEN * 2]; /* histogram of alloc sizes */
+#endif
+
+ /* trie stats */
+ size_t n_entries; /* number of entries */
+ size_t n_tbm_nodes; /* total number of TBM nodes in tree */
+ size_t n_lc_nodes; /* total number of LC nodes in tree */
+};
+
+/****************************************************************
+ *
+ * Memory management
+ *
+ * We will need to frequently resize child/data arrays. The current
+ * mempool implementation does not support resizing/freeing, so here
+ * we roll our own.
+ */
+
+static inline void _free_hunk(struct btrie *btrie, void *buf, unsigned n_nodes)
+{
+ struct free_hunk *hunk = buf;
+
+ hunk->next = btrie->free_list[n_nodes - 1];
+ btrie->free_list[n_nodes - 1] = hunk;
+}
+
+static inline void *
+_get_hunk(struct btrie *btrie, unsigned n_nodes)
+{
+ struct free_hunk *hunk = btrie->free_list[n_nodes - 1];
+
+ if (hunk != NULL)
+ btrie->free_list[n_nodes - 1] = hunk->next;
+ return hunk;
+}
+
+/* Get pointer to uninitialized child/data array.
+ *
+ * Allocates memory for an array of NDATA (void *)s followed by an
+ * array of NCHILDREN (node_t)s. The returned pointer points to to
+ * beginning of the children array (i.e. it points to (one past) the
+ * end of the data array.)
+ */
+static node_t *
+alloc_nodes(struct btrie *btrie, unsigned nchildren, unsigned ndata)
+{
+ size_t n_nodes = nchildren + (ndata + 1) / 2;
+ node_t *hunk;
+
+ assert(n_nodes > 0 && n_nodes <= MAX_CHILD_ARRAY_LEN);
+
+ hunk = _get_hunk (btrie, n_nodes);
+ if (hunk == NULL) {
+ /* Do not have free hunk of exactly the requested size, look for a
+ * larger hunk. (The funny order in which we scan the buckets is
+ * heuristically selected in an attempt to minimize unnecessary
+ * creation of small fragments)
+ */
+ size_t n, skip = n_nodes > 4 ? 4 : n_nodes;
+ for (n = n_nodes + skip; n <= MAX_CHILD_ARRAY_LEN; n++) {
+ if ((hunk = _get_hunk (btrie, n)) != NULL) {
+ _free_hunk (btrie, hunk + n_nodes, n - n_nodes);
+ goto DONE;
+ }
+ }
+ for (n = n_nodes + 1; n < n_nodes + skip && n <= MAX_CHILD_ARRAY_LEN;
+ n++) {
+ if ((hunk = _get_hunk (btrie, n)) != NULL) {
+ _free_hunk (btrie, hunk + n_nodes, n - n_nodes);
+ goto DONE;
+ }
+ }
+
+ /* failed to find free hunk, allocate a fresh one */
+ hunk = rspamd_mempool_alloc0 (btrie->mp, n_nodes * sizeof(node_t));
+ if (hunk == NULL)
+ longjmp (btrie->exception, BTRIE_ALLOC_FAILED);
+ btrie->alloc_total += n_nodes * sizeof(node_t);
+ }
+
+ DONE: btrie->alloc_data += ndata * sizeof(void *);
+ btrie->alloc_waste += (ndata % 2) * sizeof(void *);
+#ifdef BTRIE_DEBUG_ALLOC
+ btrie->alloc_hist[2 * nchildren + ndata]++;
+#endif
+
+ /* adjust pointer to allow room for data array before child array */
+ return hunk + (ndata + 1) / 2;
+}
+
+/* Free memory allocated by alloc_nodes */
+static void free_nodes(struct btrie *btrie, node_t *buf, unsigned nchildren,
+ unsigned ndata)
+{
+ size_t n_nodes = nchildren + (ndata + 1) / 2;
+
+ assert(n_nodes > 0 && n_nodes <= MAX_CHILD_ARRAY_LEN);
+
+ _free_hunk (btrie, buf - (ndata + 1) / 2, n_nodes);
+
+ btrie->alloc_data -= ndata * sizeof(void *);
+ btrie->alloc_waste -= (ndata % 2) * sizeof(void *);
+#ifdef BTRIE_DEBUG_ALLOC
+ btrie->alloc_hist[2 * nchildren + ndata]--;
+#endif
+}
+
+/* Debugging/development only: */
+#ifdef BTRIE_DEBUG_ALLOC
+static void
+dump_alloc_hist(const struct btrie *btrie)
+{
+ unsigned bin;
+ size_t total_alloc = 0;
+ size_t total_free = 0;
+ size_t total_bytes = 0;
+ size_t total_waste = 0;
+ size_t total_free_bytes = 0;
+
+ puts("hunk alloc free alloc wasted free");
+ puts("size hunks hunks bytes bytes bytes");
+ puts("==== ====== ====== ======== ======== ========");
+
+ for (bin = 1; bin < 2 * MAX_CHILD_ARRAY_LEN; bin++) {
+ size_t n_alloc = btrie->alloc_hist[bin];
+ size_t bytes = n_alloc * bin * sizeof(void *);
+ size_t waste_bytes = (bin % 2) * n_alloc * sizeof(void *);
+ size_t n_free = 0, free_bytes;
+ if (bin % 2 == 0) {
+ const struct free_hunk *hunk;
+ for (hunk = btrie->free_list[bin / 2 - 1]; hunk; hunk = hunk->next)
+ n_free++;
+ }
+ free_bytes = n_free * bin * sizeof(void *);
+
+ printf("%3zu: %6zu %6zu %8zu %8zu %8zu\n", bin * sizeof(void *),
+ n_alloc, n_free, bytes, waste_bytes, free_bytes);
+
+ total_alloc += n_alloc;
+ total_free += n_free;
+ total_bytes += bytes;
+ total_waste += waste_bytes;
+ total_free_bytes += free_bytes;
+ }
+ puts("---- ------ ------ -------- -------- --------");
+ printf("SUM: %6zu %6zu %8zu %8zu %8zu\n",
+ total_alloc, total_free, total_bytes, total_waste, total_free_bytes);
+}
+#endif
+
+/****************************************************************
+ *
+ * Bit twiddling
+ *
+ */
+
+static inline tbm_bitmap_t bit(unsigned b)
+{
+ return 1U << ((1 << TBM_STRIDE) - 1 - b);
+}
+
+/* count the number of set bits in bitmap
+ *
+ * algorithm from
+ * http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
+ */
+static inline unsigned count_bits(tbm_bitmap_t v)
+{
+ /* Count set bits in parallel. */
+ /* v = (v & 0x5555...) + ((v >> 1) & 0x5555...); */
+ v -= (v >> 1) & (tbm_bitmap_t) ~0UL / 3;
+ /* v = (v & 0x3333...) + ((v >> 2) & 0x3333...); */
+ v = (v & (tbm_bitmap_t) ~0UL / 5) + ((v >> 2) & (tbm_bitmap_t) ~0UL / 5);
+ /* v = (v & 0x0f0f...) + ((v >> 4) & 0x0f0f...); */
+ v = (v + (v >> 4)) & (tbm_bitmap_t) ~0UL / 17;
+ /* v = v % 255; */
+#if TBM_STRIDE == 4
+ /* tbm_bitmap_t is uint16_t, avoid the multiply */
+ return (v + (v >> 8)) & 0x0ff;
+#else
+ return (v * (tbm_bitmap_t) (~0UL / 255)) >> ((sizeof(tbm_bitmap_t) - 1) * 8);
+#endif
+}
+
+static inline unsigned count_bits_before(tbm_bitmap_t bm, int b)
+{
+ return b ? count_bits (bm >> ((1 << TBM_STRIDE) - b)) : 0;
+}
+
+static inline unsigned count_bits_from(tbm_bitmap_t bm, int b)
+{
+ return count_bits (bm << b);
+}
+
+/* extracts a few bits from bitstring, returning them as an integer */
+static inline btrie_oct_t RSPAMD_NO_SANITIZE extract_bits(const btrie_oct_t *prefix, unsigned pos,
+ unsigned nbits)
+{
+ if (nbits == 0)
+ return 0;
+ else {
+ unsigned v = (prefix[pos / 8] << 8) + prefix[pos / 8 + 1];
+ return (v >> (16 - nbits - pos % 8)) & ((1U << nbits) - 1);
+ }
+}
+
+static inline unsigned extract_bit(const btrie_oct_t *prefix, int pos)
+{
+ return (prefix[pos / 8] >> (7 - pos % 8)) & 0x01;
+}
+
+/* get mask for high n bits of a byte */
+static inline btrie_oct_t high_bits(unsigned n)
+{
+ return (btrie_oct_t) -(1U << (8 - n));
+}
+
+/* determine whether two prefixes are equal */
+static inline int prefixes_equal(const btrie_oct_t *pfx1,
+ const btrie_oct_t *pfx2, unsigned len)
+{
+ return (memcmp (pfx1, pfx2, len / 8) == 0
+ && (len % 8 == 0 ||
+ ((pfx1[len / 8] ^ pfx2[len / 8]) & high_bits (len % 8)) == 0));
+}
+
+/* determine length of longest common subprefix */
+static inline unsigned common_prefix(const btrie_oct_t *pfx1,
+ const btrie_oct_t *pfx2, unsigned len)
+{
+ /* algorithm adapted from
+ * http://graphics.stanford.edu/~seander/bithacks.html#IntegerLogLookup
+ */
+ static btrie_oct_t leading_zeros[] =
+ { 8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, };
+ unsigned nb;
+
+ for (nb = 0; nb < len / 8; nb++) {
+ unsigned diff = *pfx1++ ^ *pfx2++;
+ if (diff != 0)
+ return 8 * nb + leading_zeros[diff];
+ }
+ if (len % 8) {
+ unsigned n = leading_zeros[*pfx1 ^ *pfx2];
+ if (n < len % 8)
+ return 8 * nb + n;
+ }
+ return len;
+}
+
+/****************************************************************
+ */
+
+static inline int is_empty_node(const node_t *node)
+{
+ return node->tbm_node.ext_bm == 0 && node->tbm_node.int_bm == 0;
+}
+
+static inline int is_lc_node(const node_t *node)
+{
+ return (node->lc_node.lc_flags & LC_FLAGS_IS_LC) != 0;
+}
+
+static inline int is_tbm_node(const node_t *node)
+{
+ return !is_lc_node (node);
+}
+
+/* is node a TBM node with internal data? */
+static inline int has_data(const node_t *node)
+{
+ return is_tbm_node (node) && node->tbm_node.int_bm != 0;
+}
+
+static inline unsigned base_index(unsigned pfx, unsigned plen)
+{
+ assert(plen < TBM_STRIDE);
+ assert(pfx < (1U << plen));
+ return pfx | (1U << plen);
+}
+
+/* initialize node to an empty TBM node */
+static inline void init_empty_node(struct btrie *btrie, node_t *node)
+{
+ memset(node, 0, sizeof(*node));
+ btrie->n_tbm_nodes++;
+}
+
+/* get pointer to TBM internal prefix data */
+static inline const void **
+tbm_data_p(const struct tbm_node *node, unsigned pfx, unsigned plen)
+{
+ unsigned bi = base_index (pfx, plen);
+
+ if ((node->int_bm & bit (bi)) == 0)
+ return NULL; /* no data */
+ else {
+ return &node->ptr.data_end[-(int) count_bits_from (node->int_bm, bi)];
+ }
+}
+
+/* add an element to the internal data array */
+static void tbm_insert_data(struct btrie *btrie, struct tbm_node *node,
+ unsigned pfx, unsigned plen, const void *data)
+{
+ /* XXX: don't realloc if already big enough? */
+ unsigned bi = base_index (pfx, plen);
+ unsigned nchildren = count_bits (node->ext_bm);
+ int ndata = count_bits (node->int_bm);
+ unsigned di = count_bits_before (node->int_bm, bi);
+ node_t *old_children = node->ptr.children;
+ const void **old_data_beg = node->ptr.data_end - ndata;
+ const void **data_beg;
+
+ assert((node->int_bm & bit (bi)) == 0);
+
+ node->ptr.children = alloc_nodes (btrie, nchildren, ndata + 1);
+ data_beg = node->ptr.data_end - (ndata + 1);
+ data_beg[di] = data;
+ node->int_bm |= bit (bi);
+
+ if (nchildren != 0 || ndata != 0) {
+ memcpy(data_beg, old_data_beg, di * sizeof(data_beg[0]));
+ memcpy(&data_beg[di + 1], &old_data_beg[di],
+ (ndata - di) * sizeof(data_beg[0])
+ + nchildren * sizeof(node_t));
+ free_nodes (btrie, old_children, nchildren, ndata);
+ }
+}
+
+/* determine whether TBM has internal prefix data for pfx/plen or ancestors */
+static inline int has_internal_data(const struct tbm_node *node, unsigned pfx,
+ unsigned plen)
+{
+# define BIT(n) (1U << ((1 << TBM_STRIDE) - 1 - (n)))
+# define B0() BIT(1) /* the bit for 0/0 */
+# define B1(n) (BIT((n) + 2) | B0()) /* the bits for n/1 and its ancestors */
+# define B2(n) (BIT((n) + 4) | B1(n >> 1)) /* the bits for n/2 and ancestors */
+# define B3(n) (BIT((n) + 8) | B2(n >> 1)) /* the bits for n/3 and ancestors */
+# define B4(n) (BIT((n) + 16) | B3(n >> 1)) /* the bits for n/4 and ancestors */
+
+ static tbm_bitmap_t ancestors[] =
+ { 0, B0(), B1(0), B1(1), B2(0), B2(1), B2(2), B2(3), B3(0), B3(1), B3(2),
+ B3(3), B3(4), B3(5), B3(6), B3(7),
+# if TBM_STRIDE == 5
+ B4(0), B4(1), B4(2), B4(3), B4(4), B4(5), B4(6), B4(7), B4(8), B4(
+ 9), B4(10), B4(11), B4(12), B4(13), B4(14), B4(15),
+# elif TBM_STRIDE != 4
+# error "unsupported TBM_STRIDE"
+# endif
+ };
+# undef B4
+# undef B3
+# undef B2
+# undef B1
+# undef B0
+# undef BIT
+
+ return (node->int_bm & ancestors[base_index (pfx, plen)]) != 0;
+}
+
+/* get pointer to TBM extending path */
+static inline node_t *
+tbm_ext_path(const struct tbm_node *node, unsigned pfx)
+{
+ if ((node->ext_bm & bit (pfx)) == 0)
+ return NULL;
+ else
+ return &node->ptr.children[count_bits_before (node->ext_bm, pfx)];
+}
+
+/* resize TBM node child array to make space for new child node */
+static node_t *
+tbm_insert_ext_path(struct btrie *btrie, struct tbm_node *node, unsigned pfx)
+{
+ unsigned nchildren = count_bits (node->ext_bm);
+ unsigned ci = count_bits_before (node->ext_bm, pfx);
+ int ndata = count_bits (node->int_bm);
+ node_t *old_children = node->ptr.children;
+ const void **old_data_beg = node->ptr.data_end - ndata;
+
+ assert((node->ext_bm & bit (pfx)) == 0);
+
+ node->ptr.children = alloc_nodes (btrie, nchildren + 1, ndata);
+ init_empty_node (btrie, &node->ptr.children[ci]);
+ node->ext_bm |= bit (pfx);
+
+ if (nchildren != 0 || ndata != 0) {
+ const void **data_beg = node->ptr.data_end - ndata;
+ memcpy(data_beg, old_data_beg,
+ ndata * sizeof(data_beg[0]) + ci * sizeof(node_t));
+ memcpy(&node->ptr.children[ci + 1], &old_children[ci],
+ (nchildren - ci) * sizeof(old_children[0]));
+ free_nodes (btrie, old_children, nchildren, ndata);
+ }
+
+ return &node->ptr.children[ci];
+}
+
+static inline int lc_is_terminal(const struct lc_node *node)
+{
+ return (node->lc_flags & LC_FLAGS_IS_TERMINAL) != 0;
+}
+
+static inline unsigned lc_len(const struct lc_node *node)
+{
+ return node->lc_flags & LC_FLAGS_LEN_MASK;
+}
+
+static inline void lc_init_flags(struct lc_node *node, int is_terminal,
+ unsigned len)
+{
+ assert((len & ~LC_FLAGS_LEN_MASK) == 0);
+ node->lc_flags = LC_FLAGS_IS_LC | len;
+ if (is_terminal)
+ node->lc_flags |= LC_FLAGS_IS_TERMINAL;
+}
+
+static inline void lc_add_to_len(struct lc_node *node, int increment)
+{
+ unsigned new_len = lc_len (node) + increment;
+ assert((new_len & ~LC_FLAGS_LEN_MASK) == 0);
+ node->lc_flags = (node->lc_flags & ~LC_FLAGS_LEN_MASK) | new_len;
+}
+
+static inline unsigned lc_shift(unsigned pos)
+{
+ return pos / 8;
+}
+
+static inline unsigned lc_base(unsigned pos)
+{
+ return 8 * lc_shift (pos);
+}
+
+static inline unsigned lc_bits(const struct lc_node *node, unsigned pos)
+{
+ return pos % 8 + lc_len (node);
+}
+
+static inline unsigned lc_bytes(const struct lc_node *node, unsigned pos)
+{
+ return (lc_bits (node, pos) + 7) / 8;
+}
+
+static inline unsigned lc_leading_bits(const struct lc_node *node, unsigned pos,
+ unsigned nbits)
+{
+ return extract_bits (node->prefix, pos % 8, nbits);
+}
+
+/* Initialize a new terminal LC node
+ *
+ * If prefix is too long to fit in a single LC node, then a chain
+ * of LC nodes will be created.
+ */
+static void init_terminal_node(struct btrie *btrie, node_t *dst, unsigned pos,
+ const btrie_oct_t *prefix, unsigned len, const void *data)
+{
+ struct lc_node *node = &dst->lc_node;
+ unsigned nbytes = (len + 7) / 8;
+
+ while (nbytes - lc_shift (pos) > LC_BYTES_PER_NODE) {
+ memcpy(node->prefix, prefix + lc_shift (pos), LC_BYTES_PER_NODE);
+ lc_init_flags (node, 0, 8 * LC_BYTES_PER_NODE - pos % 8);
+ node->ptr.child = alloc_nodes (btrie, 1, 0);
+ pos += lc_len (node);
+ node = &node->ptr.child->lc_node;
+ btrie->n_lc_nodes++;
+ }
+
+ memcpy(node->prefix, prefix + lc_shift (pos), nbytes - lc_shift (pos));
+ lc_init_flags (node, 1, len - pos);
+ node->ptr.data = data;
+ btrie->n_lc_nodes++;
+}
+
+/* merge chains of multiple LC nodes into a single LC node, if possible.
+ *
+ * also ensure that the leading nodes in the LC chain have maximum length.
+ */
+static void coalesce_lc_node(struct btrie *btrie, struct lc_node *node,
+ unsigned pos)
+{
+ while (!lc_is_terminal (node) && lc_bits (node, pos) < 8 * LC_BYTES_PER_NODE
+ && is_lc_node (node->ptr.child)) {
+ struct lc_node *child = &node->ptr.child->lc_node;
+ unsigned spare_bits = 8 * LC_BYTES_PER_NODE - lc_bits (node, pos);
+ unsigned end = pos + lc_len (node);
+ unsigned shift = lc_shift (end) - lc_shift (pos);
+ if (lc_len (child) <= spare_bits) {
+ /* node plus child will fit in single node - merge */
+ memcpy(node->prefix + shift, child->prefix, lc_bytes (child, end));
+ lc_init_flags (node, lc_is_terminal (child),
+ lc_len (node) + lc_len (child));
+ node->ptr = child->ptr;
+ free_nodes (btrie, (node_t *) child, 1, 0);
+ btrie->n_lc_nodes--;
+ }
+ else {
+ /* can't merge, but can take some of children bits */
+ unsigned cshift = lc_shift (end + spare_bits) - lc_shift (end);
+
+ memcpy(node->prefix + shift, child->prefix,
+ LC_BYTES_PER_NODE - shift);
+ lc_add_to_len (node, spare_bits);
+ if (cshift)
+ memmove(child->prefix, child->prefix + cshift,
+ lc_bytes (child, end) - cshift);
+ assert(lc_len (child) > spare_bits);
+ lc_add_to_len (child, -spare_bits);
+
+ pos += lc_len (node);
+ node = child;
+ }
+ }
+}
+
+static void init_tbm_node(struct btrie *btrie, node_t *node, unsigned pos,
+ const btrie_oct_t pbyte, const void **root_data_p, node_t *left,
+ node_t *right);
+
+/* given an LC node at orig_pos, create a new (shorter) node at pos */
+static void shorten_lc_node(struct btrie *btrie, node_t *dst, unsigned pos,
+ struct lc_node *src, unsigned orig_pos)
+{
+ assert(orig_pos < pos);
+ assert(lc_len (src) >= pos - orig_pos);
+ assert(dst != (node_t * )src);
+
+ if (lc_len (src) == pos - orig_pos && !lc_is_terminal (src)) {
+ /* just steal the child */
+ node_t *child = src->ptr.child;
+ *dst = *child;
+ free_nodes (btrie, child, 1, 0);
+ btrie->n_lc_nodes--;
+ }
+ else {
+ struct lc_node *node = &dst->lc_node;
+ unsigned shift = lc_shift (pos) - lc_shift (orig_pos);
+ if (shift) {
+ memmove(node->prefix, src->prefix + shift,
+ lc_bytes (src, orig_pos) - shift);
+ node->lc_flags = src->lc_flags;
+ node->ptr = src->ptr;
+ }
+ else {
+ *node = *src;
+ }
+ lc_add_to_len (node, -(pos - orig_pos));
+ coalesce_lc_node (btrie, node, pos);
+ }
+}
+
+/* convert LC node to non-terminal LC node of length len *in place*
+ *
+ * on entry, node must have length at least len
+ */
+static void split_lc_node(struct btrie *btrie, struct lc_node *node,
+ unsigned pos, unsigned len)
+{
+ node_t *child = alloc_nodes (btrie, 1, 0);
+
+ assert(lc_len (node) >= len);
+ shorten_lc_node (btrie, child, pos + len, node, pos);
+
+ lc_init_flags (node, 0, len);
+ node->ptr.child = child;
+ btrie->n_lc_nodes++;
+}
+
+/* convert non-terminal LC node of length one to a TBM node *in place* */
+static void convert_lc_node_1(struct btrie *btrie, struct lc_node *node,
+ unsigned pos)
+{
+ btrie_oct_t pbyte = node->prefix[0];
+ node_t *child = node->ptr.child;
+ node_t *left, *right;
+
+ assert(lc_len (node) == 1);
+ assert(!lc_is_terminal (node));
+
+ if (extract_bit (node->prefix, pos % 8))
+ left = NULL, right = child;
+ else
+ left = child, right = NULL;
+ init_tbm_node (btrie, (node_t *) node, pos, pbyte, NULL, left, right);
+ free_nodes (btrie, child, 1, 0);
+ btrie->n_lc_nodes--;
+}
+
+/* convert an LC node to TBM node *in place* */
+static void convert_lc_node(struct btrie *btrie, struct lc_node *node,
+ unsigned pos)
+{
+ unsigned len = lc_len (node);
+
+ if (len >= TBM_STRIDE) {
+ unsigned pfx = lc_leading_bits (node, pos, TBM_STRIDE);
+ struct tbm_node *result = (struct tbm_node *) node;
+
+ /* split to LC of len TBM_STRIDE followed by child (extending path) */
+ split_lc_node (btrie, node, pos, TBM_STRIDE);
+ /* then convert leading LC node to TBM node */
+ result->int_bm = 0;
+ result->ext_bm = bit (pfx);
+ btrie->n_lc_nodes--;
+ btrie->n_tbm_nodes++;
+ }
+ else if (lc_is_terminal (node)) {
+ /* convert short terminal LC to TBM (with internal data) */
+ unsigned pfx = lc_leading_bits (node, pos, len);
+ const void *data = node->ptr.data;
+ node_t *result = (node_t *) node;
+
+ init_empty_node (btrie, result);
+ tbm_insert_data (btrie, &result->tbm_node, pfx, len, data);
+
+ btrie->n_lc_nodes--;
+ }
+ else {
+ assert(len > 0);
+ for (; len > 1; len--) {
+ split_lc_node (btrie, node, pos, len - 1);
+ convert_lc_node_1 (btrie, &node->ptr.child->lc_node, pos + len - 1);
+ }
+ convert_lc_node_1 (btrie, node, pos);
+ }
+}
+
+static void insert_lc_node(struct btrie *btrie, node_t *dst, unsigned pos,
+ btrie_oct_t pbyte, unsigned last_bit, node_t *tail)
+{
+ struct lc_node *node = &dst->lc_node;
+ btrie_oct_t mask = 1 << (7 - (pos % 8));
+ btrie_oct_t bit = last_bit ? mask : 0;
+
+ if (mask != 0x01 && is_lc_node (tail)) {
+ /* optimization: LC tail has room for the extra bit (without shifting) */
+ assert((tail->lc_node.prefix[0] & mask) == bit);
+ *node = tail->lc_node;
+ lc_add_to_len (node, 1);
+ return;
+ }
+
+ /* add new leading LC node of len 1 */
+ node->prefix[0] = pbyte | bit;
+ lc_init_flags (node, 0, 1);
+ node->ptr.child = alloc_nodes (btrie, 1, 0);
+ node->ptr.child[0] = *tail;
+ btrie->n_lc_nodes++;
+
+ if (is_lc_node (tail))
+ coalesce_lc_node (btrie, node, pos);
+}
+
+/* given:
+ * pbyte: the bits in the prefix between lc_base(pos) and pos
+ * pfx: the next TBM_STRIDE bits in the prefix starting at pos
+ * returns:
+ * the bits in the prefix between lc_base(pos + plen) and pos + plen
+ */
+static inline btrie_oct_t next_pbyte(btrie_oct_t pbyte, unsigned pos,
+ unsigned pfx)
+{
+ unsigned end = pos + TBM_STRIDE;
+
+ if (end % 8 != 0) {
+ btrie_oct_t nbyte = (btrie_oct_t) pfx << (8 - end % 8);
+ if (end % 8 > TBM_STRIDE)
+ nbyte |= pbyte & high_bits (pos % 8);
+ return nbyte;
+ }
+ return 0;
+}
+
+/* construct a new TBM node, given the data and children of the
+ * root prefix of the new node.
+ */
+static void init_tbm_node(struct btrie *btrie, node_t *dst, unsigned pos,
+ const btrie_oct_t pbyte, const void **root_data_p, node_t *left,
+ node_t *right)
+{
+ struct tbm_node *node = &dst->tbm_node;
+ unsigned nchildren = 0;
+ unsigned ndata = 0;
+ node_t children[TBM_FANOUT];
+ const void *data[TBM_FANOUT - 1];
+ tbm_bitmap_t ext_bm = 0;
+ tbm_bitmap_t int_bm = 0;
+ unsigned i, d, pfx_base;
+
+ if (left && is_lc_node (left) && lc_len (&left->lc_node) < TBM_STRIDE)
+ convert_lc_node (btrie, &left->lc_node, pos + 1);
+ if (right && is_lc_node (right) && lc_len (&right->lc_node) < TBM_STRIDE)
+ convert_lc_node (btrie, &right->lc_node, pos + 1);
+
+ /* set internal data for root prefix */
+ if (root_data_p) {
+ data[ndata++] = *root_data_p;
+ int_bm |= bit (base_index (0, 0));
+ }
+ /* copy internal data from children */
+ for (d = 0; d < TBM_STRIDE - 1; d++) {
+ if (left && has_data (left)) {
+ for (i = 0; i < 1U << d; i++) {
+ const void **data_p = tbm_data_p (&left->tbm_node, i, d);
+ if (data_p) {
+ data[ndata++] = *data_p;
+ int_bm |= bit (base_index (i, d + 1));
+ }
+ }
+ }
+ if (right && has_data (right)) {
+ for (i = 0; i < 1U << d; i++) {
+ const void **data_p = tbm_data_p (&right->tbm_node, i, d);
+ if (data_p) {
+ data[ndata++] = *data_p;
+ int_bm |= bit (base_index (i + (1 << d), d + 1));
+ }
+ }
+ }
+ }
+
+ /* copy extending paths */
+ for (pfx_base = 0; pfx_base < TBM_FANOUT; pfx_base += TBM_FANOUT / 2) {
+ node_t *child = pfx_base ? right : left;
+ if (child == NULL) {
+ continue;
+ }
+ else if (is_lc_node (child)) {
+ unsigned pfx = pfx_base + lc_leading_bits (&child->lc_node, pos + 1,
+ TBM_STRIDE - 1);
+ /* child is LC node, just shorten it by TBM_STRIDE - 1 */
+ shorten_lc_node (btrie, &children[nchildren++], pos + TBM_STRIDE,
+ &child->lc_node, pos + 1);
+ ext_bm |= bit (pfx);
+ }
+ else if (!is_empty_node (child)) {
+ /* convert deepest internal prefixes of child to extending paths
+ * of the new node
+ */
+ for (i = 0; i < TBM_FANOUT / 2; i++) {
+ const void **data_p = tbm_data_p (&child->tbm_node, i,
+ TBM_STRIDE - 1);
+ node_t *left_ext = tbm_ext_path (&child->tbm_node, 2 * i);
+ node_t *right_ext = tbm_ext_path (&child->tbm_node, 2 * i + 1);
+ if (data_p || left_ext || right_ext) {
+ node_t *ext_path = &children[nchildren++];
+ unsigned pfx = pfx_base + i;
+ btrie_oct_t npbyte = next_pbyte (pbyte, pos, pfx);
+
+ ext_bm |= bit (pfx);
+ if (left_ext == NULL && right_ext == NULL) {
+ /* only have data - set ext_path to zero-length terminal LC node */
+ lc_init_flags (&ext_path->lc_node, 1, 0);
+ ext_path->lc_node.prefix[0] = npbyte;
+ ext_path->lc_node.ptr.data = *data_p;
+ btrie->n_lc_nodes++;
+ }
+ else if (data_p || (left_ext && right_ext)) {
+ /* have at least two of data, left_ext, right_ext
+ * ext_path must be a full TBM node */
+ init_tbm_node (btrie, ext_path, pos + TBM_STRIDE,
+ npbyte, data_p, left_ext, right_ext);
+ }
+ else if (left_ext) {
+ /* have only left_ext, insert length-one LC node */
+ insert_lc_node (btrie, ext_path, pos + TBM_STRIDE,
+ npbyte, 0, left_ext);
+ }
+ else {
+ /* have only right_ext, insert length-one LC node */
+ insert_lc_node (btrie, ext_path, pos + TBM_STRIDE,
+ npbyte, 1, right_ext);
+ }
+ }
+ }
+ btrie->n_tbm_nodes--;
+ free_nodes (btrie, child->tbm_node.ptr.children,
+ count_bits (child->tbm_node.ext_bm),
+ count_bits (child->tbm_node.int_bm));
+ }
+ }
+
+ assert(count_bits (int_bm) == ndata);
+ assert(count_bits (ext_bm) == nchildren);
+
+ node->ptr.children = alloc_nodes (btrie, nchildren, ndata);
+ memcpy(node->ptr.data_end - (int )ndata, data, ndata * sizeof(data[0]));
+ memcpy(node->ptr.children, children, nchildren * sizeof(children[0]));
+ node->ext_bm = ext_bm;
+ node->int_bm = int_bm;
+ btrie->n_tbm_nodes++;
+}
+
+static enum btrie_result add_to_trie(struct btrie *btrie, node_t *node,
+ unsigned pos, const btrie_oct_t *prefix, unsigned len, const void *data)
+{
+ for (;;) {
+ if (is_lc_node (node)) {
+ struct lc_node *lc_node = &node->lc_node;
+ unsigned end = pos + lc_len (lc_node);
+ unsigned cbits = common_prefix (prefix + lc_shift (pos),
+ lc_node->prefix, (len < end ? len : end) - lc_base (pos));
+ unsigned clen = lc_base (pos) + cbits; /* position of first mismatch */
+
+ if (clen == end && !lc_is_terminal (lc_node)) {
+ /* matched entire prefix of LC node, proceed to child */
+ assert(lc_len (lc_node) > 0);
+ node = lc_node->ptr.child;
+ pos = end;
+ }
+ else if (clen == end && len == end && lc_is_terminal (lc_node)) {
+ /* exact match for terminal node - already have data for prefix */
+ return BTRIE_DUPLICATE_PREFIX;
+ }
+ else {
+ assert(clen < end || (lc_is_terminal (lc_node) && len > end));
+ /* Need to insert new TBM node at clen */
+ if (clen > pos) {
+ split_lc_node (btrie, lc_node, pos, clen - pos);
+ node = lc_node->ptr.child;
+ assert(is_lc_node (node));
+ pos = clen;
+ }
+ convert_lc_node (btrie, &node->lc_node, pos);
+ }
+ }
+ else if (is_empty_node (node)) {
+ /* at empty TBM node - just replace with terminal LC node */
+ init_terminal_node (btrie, node, pos, prefix, len, data);
+ btrie->n_entries++;
+ btrie->n_tbm_nodes--;
+ return BTRIE_OKAY;
+ }
+ else {
+ struct tbm_node *tbm_node = &node->tbm_node;
+ unsigned end = pos + TBM_STRIDE;
+
+ if (len < end) {
+ unsigned plen = len - pos;
+ unsigned pfx = extract_bits (prefix, pos, plen);
+
+ if (tbm_data_p (tbm_node, pfx, plen) != NULL)
+ return BTRIE_DUPLICATE_PREFIX; /* prefix already has data */
+ else {
+ tbm_insert_data (btrie, tbm_node, pfx, plen, data);
+ btrie->n_entries++;
+ return BTRIE_OKAY;
+ }
+ }
+ else {
+ unsigned pfx = extract_bits (prefix, pos, TBM_STRIDE);
+
+ /* follow extending path */
+ node = tbm_ext_path (tbm_node, pfx);
+ if (node == NULL)
+ node = tbm_insert_ext_path (btrie, tbm_node, pfx);
+ pos = end;
+ }
+ }
+ }
+}
+
+static const void *
+search_trie(const node_t *node, unsigned pos, const btrie_oct_t *prefix,
+ unsigned len)
+{
+ /* remember last TBM node seen with internal data */
+ const struct tbm_node *int_node = 0;
+ unsigned int_pfx = 0, int_plen = 0;
+
+ while (node) {
+ if (is_lc_node (node)) {
+ const struct lc_node *lc_node = &node->lc_node;
+ unsigned end = pos + lc_len (lc_node);
+ if (len < end)
+ break;
+ if (!prefixes_equal (prefix + lc_shift (pos), lc_node->prefix,
+ end - lc_base (pos)))
+ break;
+
+ if (lc_is_terminal (lc_node))
+ return lc_node->ptr.data; /* found terminal node */
+
+ pos = end;
+ node = lc_node->ptr.child;
+ }
+ else {
+ const struct tbm_node *tbm_node = &node->tbm_node;
+ unsigned end = pos + TBM_STRIDE;
+ if (len < end) {
+ unsigned plen = len - pos;
+ unsigned pfx = extract_bits (prefix, pos, plen);
+ if (has_internal_data (tbm_node, pfx, plen)) {
+ int_node = tbm_node;
+ int_pfx = pfx;
+ int_plen = plen;
+ }
+ break;
+ }
+ else {
+ unsigned pfx = extract_bits (prefix, pos, TBM_STRIDE);
+ if (has_internal_data (tbm_node, pfx >> 1, TBM_STRIDE - 1)) {
+ int_node = tbm_node;
+ int_pfx = pfx >> 1;
+ int_plen = TBM_STRIDE - 1;
+ }
+ pos = end;
+ node = tbm_ext_path (tbm_node, pfx);
+ }
+ }
+ }
+
+ if (int_node) {
+ const void **data_p = tbm_data_p (int_node, int_pfx, int_plen);
+ while (data_p == NULL) {
+ assert(int_plen > 0);
+ int_pfx >>= 1;
+ int_plen--;
+ data_p = tbm_data_p (int_node, int_pfx, int_plen);
+ }
+ return *data_p;
+ }
+
+ return NULL;
+}
+
+struct btrie *
+btrie_init(rspamd_mempool_t *mp)
+{
+ struct btrie *btrie;
+
+ if (!(btrie = rspamd_mempool_alloc0 (mp, sizeof(*btrie)))) {
+ return NULL;
+ }
+
+ btrie->mp = mp;
+ btrie->alloc_total = sizeof(*btrie);
+
+ /* count the empty root node */
+ btrie->n_tbm_nodes = 1;
+
+ return btrie;
+}
+
+enum btrie_result btrie_add_prefix(struct btrie *btrie,
+ const btrie_oct_t *prefix, unsigned len, const void *data)
+{
+ enum btrie_result rv;
+ if ((rv = setjmp (btrie->exception)) != 0)
+ return rv; /* out of memory */
+
+ return add_to_trie (btrie, &btrie->root, 0, prefix, len, data);
+}
+
+const void *
+btrie_lookup(const struct btrie *btrie, const btrie_oct_t *prefix, unsigned len)
+{
+ return search_trie (&btrie->root, 0, prefix, len);
+}
+
+/****************************************************************
+ *
+ * btrie_stats() - statistics reporting
+ */
+
+#ifdef BTRIE_EXTENDED_STATS
+
+/* Define BTRIE_EXTENDED_STATS to get extra statistics (including
+ * trie depth). This statistics require a traversal of the entire trie
+ * to compute, and so are disabled by default.
+ */
+
+struct stats {
+ size_t max_depth;
+ size_t total_depth;
+#ifndef NDEBUG
+ size_t n_lc_nodes;
+ size_t n_tbm_nodes;
+ size_t n_entries;
+ size_t alloc_data;
+ size_t alloc_waste;
+#endif
+};
+
+static void
+node_stats(const node_t *node, size_t depth, struct stats *stats)
+{
+ if (depth > stats->max_depth)
+ stats->max_depth = depth;
+ stats->total_depth += depth;
+
+ if (is_lc_node(node)) {
+#ifndef NDEBUG
+ stats->n_lc_nodes++;
+#endif
+ if (!lc_is_terminal(&node->lc_node))
+ node_stats(node->lc_node.ptr.child, depth + 1, stats);
+#ifndef NDEBUG
+ else
+ stats->n_entries++;
+#endif
+ }
+ else {
+ unsigned i;
+ unsigned nchildren = count_bits(node->tbm_node.ext_bm);
+#ifndef NDEBUG
+ unsigned ndata = count_bits(node->tbm_node.int_bm);
+
+ stats->n_tbm_nodes++;
+ stats->n_entries += ndata;
+ stats->alloc_data += ndata * sizeof(void *);
+ stats->alloc_waste += (ndata % 2) * sizeof(void *);
+#endif
+ for (i = 0; i < nchildren; i++)
+ node_stats(&node->tbm_node.ptr.children[i], depth + 1, stats);
+ }
+}
+#endif /* BTRIE_EXTENDED_STATS */
+
+#ifndef NDEBUG
+static size_t count_free(const struct btrie *btrie)
+{
+ size_t total = 0;
+ unsigned sz;
+ for (sz = 1; sz <= MAX_CHILD_ARRAY_LEN; sz++) {
+ const struct free_hunk *free = btrie->free_list[sz - 1];
+ size_t n;
+ for (n = 0; free; n++)
+ free = free->next;
+ total += sz * n;
+ }
+ return total * sizeof(node_t);
+}
+#endif /* not NDEBUG */
+
+const char *
+btrie_stats(const struct btrie *btrie, guint duplicates)
+{
+ static char buf[128];
+ size_t n_nodes = btrie->n_lc_nodes + btrie->n_tbm_nodes;
+ size_t alloc_free = (btrie->alloc_total + sizeof(node_t) /* do not double-count the root node */
+ - n_nodes * sizeof(node_t) - btrie->alloc_data - btrie->alloc_waste
+ - sizeof(*btrie));
+#ifdef BTRIE_EXTENDED_STATS
+ struct stats stats;
+ double average_depth;
+
+ memset(&stats, 0, sizeof(stats));
+ node_stats(&btrie->root, 0, &stats);
+ average_depth = (double)stats.total_depth / n_nodes;
+
+#ifndef NDEBUG
+ /* check the node counts */
+ assert(stats.n_lc_nodes == btrie->n_lc_nodes);
+ assert(stats.n_tbm_nodes == btrie->n_tbm_nodes);
+ assert(stats.n_entries == btrie->n_entries);
+ assert(stats.alloc_data == btrie->alloc_data);
+ assert(stats.alloc_waste == btrie->alloc_waste);
+#endif /* not NDEBUG */
+#endif /* BTRIE_EXTENDED_STATS */
+
+#ifndef NDEBUG
+ /* check that we haven't lost any memory */
+ assert(alloc_free == count_free (btrie));
+#endif
+
+#ifdef BTRIE_DEBUG_ALLOC
+ dump_alloc_hist(btrie);
+#endif
+
+
+#ifdef BTRIE_EXTENDED_STATS
+ snprintf(buf, sizeof(buf),
+ "ents=%lu tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu"
+ " depth=%.1f/%lu"
+ ,(long unsigned)btrie->n_entries, (long unsigned)btrie->n_tbm_nodes,
+ (long unsigned)btrie->n_lc_nodes, (double)btrie->alloc_total / 1024,
+ (long unsigned)alloc_free, (long unsigned)btrie->alloc_waste
+ , average_depth, (long unsigned)stats.max_depth);
+#else
+ snprintf(buf, sizeof(buf),
+ "ents=%lu dup=%u tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu",
+ (long unsigned)btrie->n_entries,
+ duplicates,
+ (long unsigned)btrie->n_tbm_nodes,
+ (long unsigned)btrie->n_lc_nodes, (double)btrie->alloc_total / 1024,
+ (long unsigned)alloc_free, (long unsigned)btrie->alloc_waste
+ );
+#endif
+ buf[sizeof(buf) - 1] = '\0';
+ return buf;
+}
+
+/****************************************************************/
+
+#ifndef NO_MASTER_DUMP
+
+struct walk_context
+{
+ btrie_walk_cb_t *callback;
+ void *user_data;
+
+ btrie_oct_t prefix[(BTRIE_MAX_PREFIX + 7) / 8];
+};
+
+static void
+walk_node(const node_t *node, unsigned pos, struct walk_context *ctx);
+
+static void walk_tbm_node(const struct tbm_node *node, unsigned pos,
+ unsigned pfx, unsigned plen, struct walk_context *ctx)
+{
+ btrie_oct_t *prefix = ctx->prefix;
+ int pbyte = pos / 8;
+ btrie_oct_t pbit = 0x80 >> (pos % 8);
+ const void **data_p = tbm_data_p (node, pfx, plen);
+
+ if (pos >= BTRIE_MAX_PREFIX) {
+ /* This can/should not happen, but don't overwrite buffers if it does. */
+ return;
+ }
+
+ if (data_p)
+ ctx->callback (prefix, pos, *data_p, 0, ctx->user_data);
+
+ /* walk children */
+ if (plen < TBM_STRIDE - 1) {
+ /* children are internal prefixes in same node */
+ walk_tbm_node (node, pos + 1, pfx << 1, plen + 1, ctx);
+ prefix[pbyte] |= pbit;
+ walk_tbm_node (node, pos + 1, (pfx << 1) + 1, plen + 1, ctx);
+ prefix[pbyte] &= ~pbit;
+ }
+ else {
+ /* children are extending paths */
+ const node_t *ext_path;
+ if ((ext_path = tbm_ext_path (node, pfx << 1)) != NULL)
+ walk_node (ext_path, pos + 1, ctx);
+ if ((ext_path = tbm_ext_path (node, (pfx << 1) + 1)) != NULL) {
+ prefix[pbyte] |= pbit;
+ walk_node (ext_path, pos + 1, ctx);
+ prefix[pbyte] &= ~pbit;
+ }
+ }
+
+ if (data_p)
+ ctx->callback (prefix, pos, *data_p, 1, ctx->user_data);
+}
+
+static void walk_lc_node(const struct lc_node *node, unsigned pos,
+ struct walk_context *ctx)
+{
+ btrie_oct_t *prefix = ctx->prefix;
+ unsigned end = pos + lc_len (node);
+ btrie_oct_t save_prefix = prefix[lc_shift (pos)];
+
+ if (end > BTRIE_MAX_PREFIX) {
+ /* This can/should not happen, but don't overwrite buffers if it does. */
+ return;
+ }
+
+ /* construct full prefix to node */
+ memcpy(&prefix[lc_shift (pos)], node->prefix, lc_bytes (node, pos));
+ if (end % 8)
+ prefix[end / 8] &= high_bits (end % 8);
+
+ if (lc_is_terminal (node)) {
+ ctx->callback (prefix, end, node->ptr.data, 0, ctx->user_data);
+ ctx->callback (prefix, end, node->ptr.data, 1, ctx->user_data);
+ }
+ else
+ walk_node (node->ptr.child, end, ctx);
+
+ prefix[lc_shift (pos)] = save_prefix; /* restore parents prefix */
+ if (lc_bytes (node, pos) > 1)
+ memset(&prefix[lc_shift (pos) + 1], 0, lc_bytes (node, pos) - 1);
+}
+
+static void walk_node(const node_t *node, unsigned pos,
+ struct walk_context *ctx)
+{
+ if (is_lc_node (node))
+ walk_lc_node (&node->lc_node, pos, ctx);
+ else
+ walk_tbm_node (&node->tbm_node, pos, 0, 0, ctx);
+}
+
+/* walk trie in lexicographical order
+ *
+ * calls callback twice (once preorder, once postorder) at each prefix
+ */
+void btrie_walk(const struct btrie *btrie, btrie_walk_cb_t *callback,
+ void *user_data)
+{
+ struct walk_context ctx;
+
+ memset(&ctx, 0, sizeof(ctx));
+ ctx.callback = callback;
+ ctx.user_data = user_data;
+
+ walk_node (&btrie->root, 0, &ctx);
+}
+
+#endif /* not NO_MASTER_DUMP */
+
+
+#ifdef TEST
+/*****************************************************************
+ *
+ * Unit tests
+ *
+ */
+#include <stdio.h>
+
+#ifndef UNUSED
+# define UNUSED __attribute__((unused))
+#endif
+
+/* bogus replacements mp_alloc for running self-tests */
+void *
+mp_alloc(UNUSED struct mempool *mp, unsigned sz, UNUSED int align)
+{
+ return malloc(sz);
+}
+
+#if 0
+# define PASS(name) puts("OK " name)
+#else
+# define PASS(name) fputs(".", stdout); fflush(stdout)
+#endif
+
+const char * pgm_name = "???";
+
+static void
+test_struct_node_packing()
+{
+ node_t node;
+
+ assert(sizeof(struct tbm_node) == 2 * sizeof(void *));
+ assert(sizeof(struct lc_node) == 2 * sizeof(void *));
+ assert(sizeof(node_t) == 2 * sizeof(void *));
+
+ /* The lc_node bit must be an alias for bit zero of int_bm, since
+ * that is the only unused bit in the TBM node structure.
+ */
+ memset(&node, 0, sizeof(node));
+ assert(node.tbm_node.int_bm == 0);
+ lc_init_flags(&node.lc_node, 0, 0);
+ assert(node.tbm_node.int_bm == bit(0));
+
+ PASS("test_struct_node_packing");
+}
+
+static void
+test_bit()
+{
+ tbm_bitmap_t ones = ~(tbm_bitmap_t)0;
+ tbm_bitmap_t high_bit = ones ^ (ones >> 1);
+
+ assert(bit(0) == high_bit);
+ assert(bit(1) == high_bit >> 1);
+ assert(bit(8 * sizeof(tbm_bitmap_t) - 1) == 1);
+ PASS("test_bit");
+}
+
+static void
+test_count_bits()
+{
+ unsigned max_bits = sizeof(tbm_bitmap_t) * 8;
+ tbm_bitmap_t ones = ~(tbm_bitmap_t)0;
+
+ assert(count_bits(0) == 0);
+ assert(count_bits(1) == 1);
+ assert(count_bits(2) == 1);
+ assert(count_bits(3) == 2);
+ assert(count_bits(ones) == max_bits);
+ assert(count_bits(~1) == max_bits - 1);
+
+ /* count_bits(0x5555....) */
+ assert(count_bits(ones / 3) == max_bits / 2);
+ /* count_bits(0x3333...) */
+ assert(count_bits(ones / 5) == max_bits / 2);
+ /* count_bits(0x0f0f...) */
+ assert(count_bits(ones / 17) == max_bits / 2);
+ /* count_bits(0x1010...) */
+ assert(count_bits(ones / 255) == max_bits / 8);
+
+ PASS("test_count_bits");
+}
+
+static void
+test_count_bits_before()
+{
+ unsigned max_bits = sizeof(tbm_bitmap_t) * 8;
+ tbm_bitmap_t ones = ~(tbm_bitmap_t)0;
+ unsigned i;
+
+ for (i = 0; i < max_bits; i++) {
+ assert(count_bits_before(0, i) == 0);
+ assert(count_bits_before(ones, i) == i);
+ }
+
+ PASS("test_count_bits_before");
+}
+
+static void
+test_count_bits_from()
+{
+ unsigned max_bits = sizeof(tbm_bitmap_t) * 8;
+ tbm_bitmap_t ones = ~(tbm_bitmap_t)0;
+ unsigned i;
+
+ for (i = 0; i < max_bits; i++) {
+ assert(count_bits_from(0, i) == 0);
+ assert(count_bits_from(ones, i) == max_bits - i);
+ }
+
+ PASS("test_count_bits_from");
+}
+
+static void
+test_extract_bits()
+{
+ static btrie_oct_t prefix[] = {0xff, 0x55, 0xaa, 0x00};
+ unsigned i;
+
+ for (i = 0; i < 32; i++)
+ assert(extract_bits(prefix, i, 0) == 0);
+
+ for (i = 0; i < 8; i++)
+ assert(extract_bits(prefix, i, 1) == 1);
+ for (i = 8; i < 16; i++)
+ assert(extract_bits(prefix, i, 1) == i % 2);
+ for (i = 16; i < 24; i++)
+ assert(extract_bits(prefix, i, 1) == (i + 1) % 2);
+ for (i = 24; i < 32; i++)
+ assert(extract_bits(prefix, i, 1) == 0);
+
+ assert(extract_bits(prefix, 2, 6) == 0x3f);
+ assert(extract_bits(prefix, 3, 6) == 0x3e);
+ assert(extract_bits(prefix, 4, 6) == 0x3d);
+ assert(extract_bits(prefix, 5, 6) == 0x3a);
+ assert(extract_bits(prefix, 6, 6) == 0x35);
+ assert(extract_bits(prefix, 7, 6) == 0x2a);
+ assert(extract_bits(prefix, 8, 6) == 0x15);
+
+ PASS("test_extract_bits");
+}
+
+static void
+test_high_bits()
+{
+ assert(high_bits(0) == 0x00);
+ assert(high_bits(1) == 0x80);
+ assert(high_bits(2) == 0xc0);
+ assert(high_bits(3) == 0xe0);
+ assert(high_bits(4) == 0xf0);
+ assert(high_bits(5) == 0xf8);
+ assert(high_bits(6) == 0xfc);
+ assert(high_bits(7) == 0xfe);
+ assert(high_bits(8) == 0xff);
+ PASS("test_high_bits");
+}
+
+static void
+test_prefixes_equal()
+{
+ btrie_oct_t prefix1[LC_BYTES_PER_NODE];
+ btrie_oct_t prefix2[LC_BYTES_PER_NODE];
+ unsigned i;
+ memset(prefix1, 0xaa, LC_BYTES_PER_NODE);
+ memset(prefix2, 0xaa, LC_BYTES_PER_NODE);
+
+ for (i = 0; i < 8 * LC_BYTES_PER_NODE; i++) {
+ assert(prefixes_equal(prefix1, prefix2, i));
+ prefix1[i / 8] ^= 1 << (7 - i % 8);
+ assert(!prefixes_equal(prefix1, prefix2, 8 * LC_BYTES_PER_NODE));
+ assert(prefixes_equal(prefix1, prefix2, i));
+ if (i + 1 < 8 * LC_BYTES_PER_NODE)
+ assert(!prefixes_equal(prefix1, prefix2, i + 1));
+ prefix1[i / 8] ^= 1 << (7 - i % 8);
+ }
+ PASS("test_prefixes_equal");
+}
+
+static void
+test_common_prefix()
+{
+ btrie_oct_t prefix1[LC_BYTES_PER_NODE];
+ btrie_oct_t prefix2[LC_BYTES_PER_NODE];
+ unsigned i;
+ memset(prefix1, 0x55, LC_BYTES_PER_NODE);
+ memset(prefix2, 0x55, LC_BYTES_PER_NODE);
+
+ for (i = 0; i < 8 * LC_BYTES_PER_NODE; i++) {
+ assert(common_prefix(prefix1, prefix2, i) == i);
+ prefix1[i / 8] ^= 1 << (7 - i % 8);
+ assert(common_prefix(prefix1, prefix2, 8 * LC_BYTES_PER_NODE) == i);
+ if (i + 1 < 8 * LC_BYTES_PER_NODE)
+ assert(common_prefix(prefix1, prefix2, i+1) == i);
+ prefix1[i / 8] ^= 1 << (7 - i % 8);
+ }
+ PASS("test_common_prefix");
+}
+
+static void
+test_base_index()
+{
+ assert(base_index(0,0) == 1);
+ assert(base_index(0,1) == 2);
+ assert(base_index(1,1) == 3);
+ assert(base_index(0,2) == 4);
+ assert(base_index(1,2) == 5);
+ assert(base_index(2,2) == 6);
+ assert(base_index(3,2) == 7);
+ PASS("test_base_index");
+}
+
+static void
+test_has_internal_data()
+{
+ struct tbm_node node;
+ unsigned plen, pfx, bi;
+ for (plen = 0; plen < TBM_STRIDE; plen++) {
+ for (pfx = 0; pfx < 1U << plen; pfx++) {
+ tbm_bitmap_t ancestor_mask = 0;
+ for (bi = base_index(pfx, plen); bi; bi >>= 1) {
+ node.int_bm = bit(bi);
+ ancestor_mask |= bit(bi);
+ assert(has_internal_data(&node, pfx, plen));
+ }
+ node.int_bm = ~ancestor_mask;
+ assert(!has_internal_data(&node, pfx, plen));
+ }
+ }
+ PASS("test_has_internal_data");
+}
+
+/****************************************************************/
+static const btrie_oct_t numbered_bytes[] = {
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
+ 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
+ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+ 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
+};
+
+static void
+check_non_terminal_lc_node(struct lc_node *node, unsigned len)
+{
+ assert(is_lc_node((node_t *)node));
+ assert(!lc_is_terminal(node));
+ assert(lc_len(node) == len);
+}
+
+static void
+check_terminal_lc_node(struct lc_node *node, unsigned len, const void *data)
+{
+ assert(is_lc_node((node_t *)node));
+ assert(lc_is_terminal(node));
+ assert(lc_len(node) == len);
+ assert(node->ptr.data == data);
+}
+
+static void
+test_init_terminal_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ node_t node;
+ struct lc_node *head = &node.lc_node;
+
+ init_terminal_node(btrie, &node, 0,
+ numbered_bytes, 8 * LC_BYTES_PER_NODE, data);
+ check_terminal_lc_node(head, 8 * LC_BYTES_PER_NODE, data);
+ assert(memcmp(head->prefix, numbered_bytes, LC_BYTES_PER_NODE) == 0);
+
+ init_terminal_node(btrie, &node, 7,
+ numbered_bytes, 8 * LC_BYTES_PER_NODE, data);
+ check_terminal_lc_node(head, 8 * LC_BYTES_PER_NODE - 7, data);
+ assert(memcmp(head->prefix, numbered_bytes, LC_BYTES_PER_NODE) == 0);
+
+ init_terminal_node(btrie, &node, 0,
+ numbered_bytes, 2 * 8 * LC_BYTES_PER_NODE, data);
+ check_non_terminal_lc_node(head, 8 * LC_BYTES_PER_NODE);
+ assert(memcmp(head->prefix, numbered_bytes, LC_BYTES_PER_NODE) == 0);
+ {
+ struct lc_node *child = &head->ptr.child->lc_node;
+ check_terminal_lc_node(child, 8 * LC_BYTES_PER_NODE, data);
+ assert(memcmp(child->prefix, &numbered_bytes[LC_BYTES_PER_NODE],
+ LC_BYTES_PER_NODE) == 0);
+ }
+
+ init_terminal_node(btrie, &node, 15,
+ numbered_bytes, 8 * LC_BYTES_PER_NODE + 15, data);
+ check_non_terminal_lc_node(head, 8 * LC_BYTES_PER_NODE - 7);
+ assert(memcmp(head->prefix, &numbered_bytes[1], LC_BYTES_PER_NODE) == 0);
+ {
+ struct lc_node *child = &head->ptr.child->lc_node;
+ check_terminal_lc_node(child, 7, data);
+ assert(child->prefix[0] == numbered_bytes[LC_BYTES_PER_NODE + 1]);
+ }
+
+ PASS("test_init_terminal_node");
+}
+
+static void
+test_coalesce_lc_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ node_t node;
+ struct lc_node *head = &node.lc_node;
+
+ /* test merging */
+ init_terminal_node(btrie, &node, 0,
+ numbered_bytes, 8 * (LC_BYTES_PER_NODE + 1), data);
+ check_non_terminal_lc_node(head, LC_BYTES_PER_NODE * 8);
+ lc_add_to_len(head, -8);
+ coalesce_lc_node(btrie, head, 8);
+ check_terminal_lc_node(head, LC_BYTES_PER_NODE * 8, data);
+ assert(head->prefix[LC_BYTES_PER_NODE - 1]
+ == numbered_bytes[LC_BYTES_PER_NODE]);
+
+ /* test bit stealing */
+ init_terminal_node(btrie, &node, 0,
+ numbered_bytes, 8 * (2 * LC_BYTES_PER_NODE), data);
+ check_non_terminal_lc_node(head, LC_BYTES_PER_NODE * 8);
+ lc_add_to_len(head, -15);
+ coalesce_lc_node(btrie, head, 15);
+ check_non_terminal_lc_node(head, LC_BYTES_PER_NODE * 8 - 7);
+ assert(memcmp(head->prefix, numbered_bytes, LC_BYTES_PER_NODE - 1) == 0);
+ assert(head->prefix[LC_BYTES_PER_NODE - 1]
+ == numbered_bytes[LC_BYTES_PER_NODE]);
+ {
+ struct lc_node *child = &head->ptr.child->lc_node;
+ check_terminal_lc_node(child, 8 * (LC_BYTES_PER_NODE - 1), data);
+ assert(memcmp(child->prefix, &numbered_bytes[LC_BYTES_PER_NODE + 1],
+ LC_BYTES_PER_NODE - 1) == 0);
+ }
+
+ PASS("test_coalesce_lc_node");
+}
+
+static void
+test_shorten_lc_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ node_t node, shorter;
+
+ /* test shorten without shift */
+ init_terminal_node(btrie, &node, 0,
+ numbered_bytes, 8 * LC_BYTES_PER_NODE, data);
+ memset(shorter.lc_node.prefix, 0xff, LC_BYTES_PER_NODE);
+ shorten_lc_node(btrie, &shorter, 7, &node.lc_node, 0);
+ check_terminal_lc_node(&shorter.lc_node, LC_BYTES_PER_NODE * 8 - 7, data);
+ assert(memcmp(shorter.lc_node.prefix, numbered_bytes, LC_BYTES_PER_NODE)
+ == 0);
+
+ /* test shorten with shift */
+ init_terminal_node(btrie, &node, 7,
+ numbered_bytes, 8 * LC_BYTES_PER_NODE, data);
+ memset(shorter.lc_node.prefix, 0xff, LC_BYTES_PER_NODE);
+ shorten_lc_node(btrie, &shorter, 9, &node.lc_node, 7);
+ check_terminal_lc_node(&shorter.lc_node, LC_BYTES_PER_NODE * 8 - 9, data);
+ assert(memcmp(shorter.lc_node.prefix, &numbered_bytes[1],
+ LC_BYTES_PER_NODE - 1) == 0);
+
+ {
+ /* test child stealing */
+ struct lc_node head;
+ node_t tail, shorter;
+
+ lc_init_flags(&head, 0, 7);
+ head.ptr.child = &tail;
+ init_empty_node(btrie, &tail);
+
+ shorten_lc_node(btrie, &shorter, 7, &head, 0);
+ assert(is_empty_node(&shorter));
+ }
+
+ PASS("test_shorten_lc_node");
+}
+
+static void
+test_split_lc_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ struct lc_node node;
+
+ init_terminal_node(btrie, (node_t *)&node, 1, numbered_bytes, 25, data);
+ split_lc_node(btrie, &node, 1, 8);
+ check_non_terminal_lc_node(&node, 8);
+ check_terminal_lc_node(&node.ptr.child->lc_node, 16, data);
+
+ /* test conversion of terminal to non-terminal */
+ init_terminal_node(btrie, (node_t *)&node, 7, numbered_bytes, 10, data);
+ split_lc_node(btrie, &node, 7, 3);
+ check_non_terminal_lc_node(&node, 3);
+ check_terminal_lc_node(&node.ptr.child->lc_node, 0, data);
+
+ PASS("test_split_lc_node");
+}
+
+static void
+test_convert_lc_node_1()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ struct lc_node head;
+
+ /* test tail is left */
+ lc_init_flags(&head, 0, 1);
+ head.prefix[0] = 0;
+ head.ptr.child = alloc_nodes(btrie, 1, 0);
+ init_terminal_node(btrie, head.ptr.child, 1, numbered_bytes, 1, data);
+ convert_lc_node_1(btrie, &head, 0);
+ {
+ node_t *result = (node_t *)&head;
+ assert(is_tbm_node(result));
+ assert(result->tbm_node.ext_bm == 0);
+ assert(result->tbm_node.int_bm == bit(base_index(0, 1)));
+ assert(*tbm_data_p(&result->tbm_node, 0, 1) == data);
+ }
+
+ /* test tail is right */
+ lc_init_flags(&head, 0, 1);
+ head.prefix[0] = 1;
+ head.ptr.child = alloc_nodes(btrie, 1, 0);
+ init_terminal_node(btrie, head.ptr.child, 8, numbered_bytes, 10, data);
+ convert_lc_node_1(btrie, &head, 7);
+ {
+ node_t *result = (node_t *)&head;
+ assert(is_tbm_node(result));
+ assert(result->tbm_node.ext_bm == 0);
+ assert(result->tbm_node.int_bm == bit(base_index(4, 3)));
+ assert(*tbm_data_p(&result->tbm_node, 4, 3) == data);
+ }
+
+ PASS("test_convert_lc_node_1");
+}
+
+static void
+test_convert_lc_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ node_t node;
+
+ /* if (len >= TBM_STRIDE) */
+ init_terminal_node(btrie, &node, 7, numbered_bytes, TBM_STRIDE + 7, data);
+ convert_lc_node(btrie, &node.lc_node, 7);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == bit(0));
+ assert(node.tbm_node.int_bm == 0);
+ check_terminal_lc_node(&tbm_ext_path(&node.tbm_node, 0)->lc_node, 0, data);
+
+ /* if (lc_is_terminal(node)) */
+ init_terminal_node(btrie, &node, 0, numbered_bytes, 0, data);
+ convert_lc_node(btrie, &node.lc_node, 0);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == 0);
+ assert(node.tbm_node.int_bm == bit(base_index(0, 0)));
+ assert(*tbm_data_p(&node.tbm_node, 0, 0) == data);
+
+ /* else */
+ lc_init_flags(&node.lc_node, 0, TBM_STRIDE - 1);
+ node.lc_node.prefix[0] = 0;
+ node.lc_node.ptr.child = alloc_nodes(btrie, 1, 0);
+ init_empty_node(btrie, node.lc_node.ptr.child);
+ tbm_insert_data(btrie, &node.lc_node.ptr.child->tbm_node, 0, 0, data);
+
+ convert_lc_node(btrie, &node.lc_node, 0);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == 0);
+ assert(node.tbm_node.int_bm == bit(base_index(0, TBM_STRIDE - 1)));
+ assert(*tbm_data_p(&node.tbm_node, 0, TBM_STRIDE - 1) == data);
+
+ PASS("test_convert_lc_node");
+}
+
+static void
+test_insert_lc_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ node_t node, tail;
+
+ /* test optimized case, last_bit == 0 */
+ init_terminal_node(btrie, &tail, 9, numbered_bytes, 17, data);
+ insert_lc_node(btrie, &node, 8, 0, 0, &tail);
+ check_terminal_lc_node(&node.lc_node, 9, data);
+ assert(memcmp(node.lc_node.prefix, &numbered_bytes[1], 2) == 0);
+
+ /* test optimized case, last_bit == 1 */
+ init_terminal_node(btrie, &tail, 7, &numbered_bytes[0x12], 15, data);
+ insert_lc_node(btrie, &node, 6, 0x10, 1, &tail);
+ check_terminal_lc_node(&node.lc_node, 9, data);
+ assert(node.lc_node.prefix[0] == 0x12);
+ assert(node.lc_node.prefix[1] == 0x13);
+
+ /* test with shift */
+ init_terminal_node(btrie, &tail, 0, numbered_bytes, 8, data);
+ insert_lc_node(btrie, &node, 7, 0x40, 1, &tail);
+ check_terminal_lc_node(&node.lc_node, 9, data);
+ assert(node.lc_node.prefix[0] == 0x41);
+ assert(node.lc_node.prefix[1] == numbered_bytes[0]);
+
+ /* test with TBM node */
+ init_empty_node(btrie, &tail);
+ insert_lc_node(btrie, &node, 6, 0x40, 0, &tail);
+ check_non_terminal_lc_node(&node.lc_node, 1);
+ assert(is_tbm_node(node.lc_node.ptr.child));
+
+ PASS("test_insert_lc_node");
+}
+
+static void
+test_next_pbyte()
+{
+ assert(next_pbyte(0xff, 0, 1) == 0x80 >> (TBM_STRIDE - 1));
+ assert(next_pbyte(0xff, 1, 1) == (0x80 | (0x80 >> TBM_STRIDE)));
+ assert(next_pbyte(0xff, 2, 1) == (0xc0 | (0x80 >> (TBM_STRIDE + 1))));
+ assert(next_pbyte(0xff, 8 - TBM_STRIDE, 1) == 0);
+ assert(next_pbyte(0xff, 9 - TBM_STRIDE, 1) == 0x80);
+
+ PASS("test_next_pbyte");
+}
+
+static void
+test_init_tbm_node()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ unsigned lr;
+ node_t node;
+
+ /* test root data */
+ init_tbm_node(btrie, &node, 0, 0, &data, NULL, NULL);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == 0);
+ assert(node.tbm_node.int_bm == bit(base_index(0, 0)));
+ assert(*tbm_data_p(&node.tbm_node, 0, 0) == data);
+
+ for (lr = 0; lr < 2; lr++) {
+ node_t child;
+ node_t *left = lr ? NULL : &child;
+ node_t *right = lr ? &child : NULL;
+ unsigned base = lr ? (1U << (TBM_STRIDE - 1)) : 0;
+ unsigned pfx;
+
+ /* test with long LC node child */
+ init_terminal_node(btrie, &child, 1, numbered_bytes, TBM_STRIDE + 1, data);
+ init_tbm_node(btrie, &node, 0, 0, NULL, left, right);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == bit(base));
+ assert(node.tbm_node.int_bm == 0);
+ check_terminal_lc_node(&tbm_ext_path(&node.tbm_node, base)->lc_node,
+ 1, data);
+
+ /* test with short LC node children */
+ init_terminal_node(btrie, &child, 1, numbered_bytes, TBM_STRIDE - 1, data);
+ init_tbm_node(btrie, &node, 0, 0, NULL, left, right);
+ assert(is_tbm_node(&node));
+ assert(node.tbm_node.ext_bm == 0);
+ assert(node.tbm_node.int_bm == bit(base_index(base >> 1, TBM_STRIDE-1)));
+ assert(*tbm_data_p(&node.tbm_node, base >> 1, TBM_STRIDE-1) == data);
+
+ /* construct TBM node with all eight combinations of having data,
+ * left_ext and/or right_ext in its extending paths */
+ init_empty_node(btrie, &child);
+ for (pfx = 0; pfx < 8; pfx++) {
+ if (pfx & 1)
+ tbm_insert_data(btrie, &child.tbm_node, pfx, TBM_STRIDE - 1, data);
+ if (pfx & 2) {
+ btrie_oct_t prefix0 = 0;
+ init_terminal_node(btrie,
+ tbm_insert_ext_path(btrie, &child.tbm_node, 2*pfx),
+ TBM_STRIDE + 1,
+ &prefix0, TBM_STRIDE + 2, data);
+ }
+ if (pfx & 4) {
+ btrie_oct_t prefix0 = 0x80 >> TBM_STRIDE;
+ init_terminal_node(btrie,
+ tbm_insert_ext_path(btrie, &child.tbm_node, 2*pfx+1),
+ TBM_STRIDE + 1,
+ &prefix0, TBM_STRIDE + 3, data);
+ }
+ }
+ init_tbm_node(btrie, &node, 0, 0, NULL, left, right);
+ for (pfx = 0; pfx < 8; pfx++) {
+ unsigned base = lr ? (1U << (TBM_STRIDE - 1)) : 0;
+ node_t *ext_path = tbm_ext_path(&node.tbm_node, base + pfx);
+ if (pfx == 0)
+ assert(ext_path == NULL);
+ else if (pfx == 1)
+ check_terminal_lc_node(&ext_path->lc_node, 0, data);
+ else if (pfx == 2) {
+ check_terminal_lc_node(&ext_path->lc_node, 2, data);
+ assert(ext_path->lc_node.prefix[0] == 0);
+ }
+ else if (pfx == 4) {
+ check_terminal_lc_node(&ext_path->lc_node, 3, data);
+ assert(ext_path->lc_node.prefix[0] == (0x80 >> TBM_STRIDE));
+ }
+ else {
+ tbm_bitmap_t int_bm = 0;
+ assert(is_tbm_node(ext_path));
+ if (pfx & 1) {
+ int_bm |= bit(base_index(0, 0));
+ assert(*tbm_data_p(&ext_path->tbm_node, 0, 0) == data);
+ }
+ if (pfx & 2) {
+ int_bm |= bit(base_index(0, 2));
+ assert(*tbm_data_p(&ext_path->tbm_node, 0, 2) == data);
+ }
+ if (pfx & 4) {
+ int_bm |= bit(base_index(4, 3));
+ assert(*tbm_data_p(&ext_path->tbm_node, 4, 3) == data);
+ }
+ assert(ext_path->tbm_node.int_bm == int_bm);
+ }
+ }
+ }
+
+ PASS("test_init_tbm_node");
+}
+
+static void
+test_add_to_trie()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data = (void *)0xdeadbeef;
+ enum btrie_result result;
+ unsigned pfx, plen;
+ node_t root;
+
+ /* test initial insertion */
+ init_empty_node(btrie, &root);
+ result = add_to_trie(btrie, &root, 0,
+ numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE, data);
+ assert(result == BTRIE_OKAY);
+ check_non_terminal_lc_node(&root.lc_node, 8 * LC_BYTES_PER_NODE);
+ check_terminal_lc_node(&root.lc_node.ptr.child->lc_node,
+ 8 * LC_BYTES_PER_NODE, data);
+
+ /* test can follow LC node to tail, and then detect duplicate prefix */
+ result = add_to_trie(btrie, &root, 0,
+ numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE, data);
+ assert(result == BTRIE_DUPLICATE_PREFIX);
+
+ /* test can insert new TBM node within existing LC node */
+ result = add_to_trie(btrie, &root, 0,
+ &numbered_bytes[1], 16, data);
+ assert(result == BTRIE_OKAY);
+ check_non_terminal_lc_node(&root.lc_node, 7);
+ assert(is_tbm_node(root.lc_node.ptr.child));
+
+ /* test can convert terminal LC node to TBM node */
+ init_terminal_node(btrie, &root, 0, numbered_bytes, 12, data);
+ result = add_to_trie(btrie, &root, 0, numbered_bytes, 24, data);
+ assert(result == BTRIE_OKAY);
+ check_non_terminal_lc_node(&root.lc_node, 12);
+ assert(is_tbm_node(root.lc_node.ptr.child));
+
+ /* test can insert internal prefix data in TBM node */
+ for (plen = 0; plen < TBM_STRIDE; plen++) {
+ for (pfx = 0; pfx < (1U << plen); pfx++) {
+ btrie_oct_t prefix0 = plen ? pfx << (8 - plen) : 0;
+ init_empty_node(btrie, &root);
+ init_terminal_node(btrie, tbm_insert_ext_path(btrie, &root.tbm_node, 0),
+ TBM_STRIDE,
+ numbered_bytes, 8, data);
+ result = add_to_trie(btrie, &root, 0, &prefix0, plen, data);
+ assert(result == BTRIE_OKAY);
+ assert(is_tbm_node(&root));
+ assert(root.tbm_node.ext_bm == bit(0));
+ assert(root.tbm_node.int_bm == bit(base_index(pfx, plen)));
+ assert(*tbm_data_p(&root.tbm_node, pfx, plen) == data);
+
+ result = add_to_trie(btrie, &root, 0, &prefix0, plen, data);
+ assert(result == BTRIE_DUPLICATE_PREFIX);
+ }
+ }
+
+ /* test can add extending paths to TBM node */
+ for (pfx = 0; pfx < (1U << TBM_STRIDE); pfx++) {
+ btrie_oct_t prefix0 = pfx << (8 - TBM_STRIDE);
+ init_empty_node(btrie, &root);
+ tbm_insert_data(btrie, &root.tbm_node, 0, 0, data);
+ result = add_to_trie(btrie, &root, 0, &prefix0, 8, data);
+ assert(result == BTRIE_OKAY);
+ assert(is_tbm_node(&root));
+ assert(root.tbm_node.ext_bm == bit(pfx));
+ assert(root.tbm_node.int_bm == bit(base_index(0, 0)));
+ check_terminal_lc_node(&tbm_ext_path(&root.tbm_node, pfx)->lc_node,
+ 8 - TBM_STRIDE, data);
+
+ result = add_to_trie(btrie, &root, 0, &prefix0, 8, data);
+ assert(result == BTRIE_DUPLICATE_PREFIX);
+ }
+
+ /* test can follow extending path */
+ init_empty_node(btrie, &root);
+ init_terminal_node(btrie,
+ tbm_insert_ext_path(btrie, &root.tbm_node, 0), TBM_STRIDE,
+ numbered_bytes, 8, data);
+ result = add_to_trie(btrie, &root, 0, numbered_bytes, 7, data);
+ assert(result == BTRIE_OKAY);
+ assert(root.tbm_node.ext_bm == bit(0));
+ assert(root.tbm_node.int_bm == 0);
+ check_non_terminal_lc_node(&root.tbm_node.ptr.children[0].lc_node,
+ 7 - TBM_STRIDE);
+
+ PASS("test_add_to_trie");
+}
+
+static void
+test_search_trie()
+{
+ struct btrie *btrie = btrie_init(NULL);
+ const void *data01 = (void *)0xdead0001;
+ const void *data11 = (void *)0xdead0101;
+ const void *data = (void *)0xdeadbeef;
+ unsigned plen, pfx;
+ node_t root;
+
+ /* test can follow chain of LC nodes to an exact match */
+ init_empty_node(btrie, &root);
+ add_to_trie(btrie, &root, 0,
+ numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE, data);
+
+ assert(search_trie(&root, 0, numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE)
+ == data);
+ assert(search_trie(&root, 0, numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE + 1)
+ == data);
+ assert(search_trie(&root, 0, numbered_bytes, 8 * 2 * LC_BYTES_PER_NODE - 1)
+ == NULL);
+ assert(search_trie(&root, 0, &numbered_bytes[1], 8 * 2 * LC_BYTES_PER_NODE)
+ == NULL);
+
+ /* test can follow extending path to an exact match */
+ for (pfx = 0; pfx < (1U << TBM_STRIDE); pfx++) {
+ btrie_oct_t prefix0 = pfx << (8 - TBM_STRIDE);
+ init_empty_node(btrie, &root);
+ tbm_insert_data(btrie, &root.tbm_node, 0, 1, data01);
+ tbm_insert_data(btrie, &root.tbm_node, 1, 1, data11);
+ add_to_trie(btrie, &root, 0, &prefix0, 8, data);
+ assert(search_trie(&root, 0, &prefix0, 8) == data);
+ /* test that last matching TBM internal prefix gets picked up */
+ if (prefix0 & 0x80)
+ assert(search_trie(&root, 0, &prefix0, 7) == data11);
+ else
+ assert(search_trie(&root, 0, &prefix0, 7) == data01);
+ prefix0 ^= 1 << (8 - TBM_STRIDE);
+ if (prefix0 & 0x80)
+ assert(search_trie(&root, 0, &prefix0, 8) == data11);
+ else
+ assert(search_trie(&root, 0, &prefix0, 8) == data01);
+ }
+
+ /* test finding of TBM internal prefixes */
+ init_empty_node(btrie, &root);
+ tbm_insert_data(btrie, &root.tbm_node, 0, 1, data01);
+ tbm_insert_data(btrie, &root.tbm_node, 1, 1, data11);
+
+ assert(search_trie(&root, 0, numbered_bytes, 0) == NULL);
+ for (plen = 1; plen < TBM_STRIDE; plen++) {
+ for (pfx = 0; pfx < (1U << TBM_STRIDE); pfx++) {
+ btrie_oct_t prefix0 = pfx << (8 - plen);
+ if (prefix0 & 0x80)
+ assert(search_trie(&root, 0, &prefix0, plen) == data11);
+ else
+ assert(search_trie(&root, 0, &prefix0, plen) == data01);
+ }
+ }
+
+ PASS("test_search_trie");
+}
+
+static int
+unit_tests()
+{
+ test_struct_node_packing();
+ test_bit();
+ test_count_bits();
+ test_count_bits_before();
+ test_count_bits_from();
+ test_extract_bits();
+ test_high_bits();
+ test_prefixes_equal();
+ test_common_prefix();
+ test_base_index();
+ test_has_internal_data();
+
+ test_init_terminal_node();
+ test_coalesce_lc_node();
+ test_shorten_lc_node();
+ test_split_lc_node();
+ test_convert_lc_node_1();
+ test_convert_lc_node();
+ test_insert_lc_node();
+ test_next_pbyte();
+ test_init_tbm_node();
+ test_add_to_trie();
+ test_search_trie();
+
+ puts("\nOK");
+ return 0;
+}
+
+/*****************************************************************
+ *
+ * btrie_dump: print out the trie structure (for testing)
+ *
+ */
+#define INDENT_FILL "....:....|....:....|....:....|....:....|"
+
+static void dump_node(const node_t *node, unsigned pos, btrie_oct_t *prefix,
+ int indent);
+
+static void
+dump_prefix(btrie_oct_t *prefix, unsigned len, int indent, const char *tail)
+{
+ unsigned i;
+
+ printf("%*.*s0x", indent, indent, INDENT_FILL);
+ for (i = 0; i < len / 8; i++)
+ printf("%02x", prefix[i]);
+ if (len % 8)
+ printf("%02x", prefix[len / 8] & high_bits(len % 8));
+ printf("/%u%s", len, tail);
+}
+
+/* the opposite of extract_bits, sets a short string of bits from integer */
+static void
+insert_bits(btrie_oct_t *prefix, unsigned pos, btrie_oct_t pfx, unsigned nbits)
+{
+ if (nbits != 0) {
+ unsigned v = (prefix[pos / 8] << 8) + prefix[pos / 8 + 1];
+ unsigned mask = (1U << nbits) - 1;
+ unsigned shift = 16 - (pos % 8) - nbits;
+ v = (v & ~(mask << shift)) | (pfx << shift);
+ prefix[pos / 8] = v >> 8;
+ prefix[pos / 8 + 1] = (btrie_oct_t)v;
+ }
+}
+
+static void
+dump_tbm_node(const struct tbm_node *node, unsigned pos,
+ btrie_oct_t *prefix, int indent)
+{
+ unsigned pfx = 0, plen = 0;
+
+ dump_prefix(prefix, pos, indent, " [tbm]\n");
+
+ for (;;) {
+ if (plen < TBM_STRIDE) {
+ const void **data_p = tbm_data_p(node, pfx, plen);
+ if (data_p) {
+ insert_bits(prefix, pos, pfx, plen);
+ dump_prefix(prefix, pos + plen, indent, "");
+ printf(" [%u/%u] (%s)\n", pfx, plen, (const char *)*data_p);
+ }
+ plen++;
+ pfx <<= 1;
+ }
+ else {
+ const node_t *ext_path = tbm_ext_path(node, pfx);
+ if (ext_path) {
+ insert_bits(prefix, pos, pfx, TBM_STRIDE);
+ dump_node(ext_path, pos + TBM_STRIDE, prefix, indent + 1);
+ }
+ while (pfx & 1) {
+ if (--plen == 0)
+ return;
+ pfx >>= 1;
+ }
+ pfx++;
+ }
+ }
+}
+
+static void
+dump_lc_node(const struct lc_node *node, unsigned pos,
+ btrie_oct_t *prefix, int indent)
+{
+ unsigned end = pos + lc_len(node);
+ btrie_oct_t save_prefix = prefix[lc_shift(pos)];
+
+ memcpy(&prefix[lc_shift(pos)], node->prefix, lc_bytes(node, pos));
+
+ if (lc_is_terminal(node)) {
+ dump_prefix(prefix, end, indent, "");
+ printf(" (%s)\n", (const char *)node->ptr.data);
+ }
+ else {
+ dump_prefix(prefix, end, indent, "\n");
+ dump_node(node->ptr.child, end, prefix, indent + 1);
+ }
+
+ prefix[lc_shift(pos)] = save_prefix;
+ if (lc_bytes(node, pos) > 1)
+ memset(&prefix[lc_shift(pos) + 1], 0, lc_bytes(node, pos) - 1);
+}
+
+static void
+dump_node(const node_t *node, unsigned pos, btrie_oct_t *prefix, int indent)
+{
+ if (is_lc_node(node))
+ dump_lc_node(&node->lc_node, pos, prefix, indent);
+ else
+ dump_tbm_node(&node->tbm_node, pos, prefix, indent);
+}
+
+static void
+btrie_dump(struct btrie *btrie)
+{
+ btrie_oct_t prefix[(BTRIE_MAX_PREFIX + 7) / 8];
+
+ memset(prefix, 0, sizeof(prefix));
+ dump_node(&btrie->root, 0, prefix, 0);
+ puts(btrie_stats(btrie));
+}
+
+/****************************************************************
+ *
+ * test program - just enough to construct a trie and preform a lookup
+ *
+ */
+
+#include <arpa/inet.h>
+
+static int
+parse_prefix(const char *arg, btrie_oct_t prefix[16], unsigned *len)
+{
+ char addrbuf[128];
+ return sscanf(arg, "%127[0-9a-fA-F:]/%u", addrbuf, len) == 2
+ && inet_pton(AF_INET6, addrbuf, prefix) == 1;
+}
+
+static int
+test_btrie(int argc, char *argv[])
+{
+ struct btrie *btrie = btrie_init(NULL);
+ int i;
+ btrie_oct_t prefix[16];
+ unsigned len;
+
+ for (i = 1; i < argc-1; i++) {
+ if (!parse_prefix(argv[i], prefix, &len)) {
+ fprintf(stderr, "Can not parse arg '%s'\n", argv[i]);
+ return 1;
+ }
+ btrie_add_prefix(btrie, prefix, len, argv[i]);
+ }
+
+ btrie_dump(btrie);
+
+ if (argc > 1) {
+ const void *data;
+
+ if (!parse_prefix(argv[argc-1], prefix, &len)) {
+ fprintf(stderr, "Can not parse arg '%s'\n", argv[argc-1]);
+ return 1;
+ }
+ data = btrie_lookup(btrie, prefix, 128);
+ printf("lookup(%s) => %s\n", argv[argc-1], (const char *)data);
+ }
+
+ return 0;
+}
+
+int
+main(int argc, char *argv[])
+{
+ if ((pgm_name = strrchr(argv[0], '/')) != NULL)
+ pgm_name++;
+ else
+ pgm_name = argv[0];
+
+ if (argc > 1)
+ return test_btrie(argc, argv);
+ else
+ return unit_tests();
+}
+
+#endif /* TEST */
diff --git a/contrib/lc-btrie/btrie.h b/contrib/lc-btrie/btrie.h
new file mode 100644
index 0000000..370a03e
--- /dev/null
+++ b/contrib/lc-btrie/btrie.h
@@ -0,0 +1,83 @@
+/* Level-Compressed Tree Bitmap (LC-TBM) Trie implementation
+ *
+ * Contributed by Geoffrey T. Dairiki <dairiki@dairiki.org>
+ *
+ * This file is released under a "Three-clause BSD License".
+ *
+ * Copyright (c) 2013, Geoffrey T. Dairiki
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * * Neither the name of Geoffrey T. Dairiki nor the names of other
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GEOFFREY
+ * T. DAIRIKI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+#ifndef _BTRIE_H_INCLUDED
+#define _BTRIE_H_INCLUDED
+
+#include "config.h"
+
+#include <stdint.h>
+typedef uint8_t btrie_oct_t;
+
+/* maximum length of bit string btrie_walk() can handle
+ *
+ * note: this limit is necessitated by the use of fixed length buffers
+ * in btrie_walk() --- btrie_add_prefix() and btrie_lookup() impose no
+ * limit on the length of bitstrings
+ */
+#define BTRIE_MAX_PREFIX 128
+
+struct btrie;
+struct memory_pool_s;
+
+struct btrie * btrie_init(struct memory_pool_s *mp);
+
+enum btrie_result
+{
+ BTRIE_OKAY = 0,
+ BTRIE_ALLOC_FAILED = -1,
+ BTRIE_DUPLICATE_PREFIX = 1
+};
+
+enum btrie_result btrie_add_prefix(struct btrie *btrie,
+ const btrie_oct_t *prefix, unsigned len, const void *data);
+
+const void *btrie_lookup(const struct btrie *btrie, const btrie_oct_t *pfx,
+ unsigned len);
+
+const char *btrie_stats(const struct btrie *btrie, guint duplicates);
+
+#ifndef NO_MASTER_DUMP
+typedef void btrie_walk_cb_t(const btrie_oct_t *prefix, unsigned len,
+ const void *data, int post, void *user_data);
+
+void btrie_walk(const struct btrie *btrie, btrie_walk_cb_t *callback,
+ void *user_data);
+#endif /* not NO_MASTER_DUMP */
+
+#endif /* _BTRIE_H_INCLUDED */
diff --git a/contrib/libev/CMakeLists.txt b/contrib/libev/CMakeLists.txt
new file mode 100644
index 0000000..b2d33fe
--- /dev/null
+++ b/contrib/libev/CMakeLists.txt
@@ -0,0 +1,82 @@
+SET(LIBEVSRC ev.c)
+
+CHECK_INCLUDE_FILES(sys/types.h HAVE_SYS_TYPES_H)
+CHECK_INCLUDE_FILES(sys/inotify.h HAVE_SYS_INOTIFY_H)
+CHECK_INCLUDE_FILES(sys/epoll.h HAVE_SYS_EPOLL_H)
+CHECK_INCLUDE_FILES("sys/types.h;sys/event.h;sys/time.h" HAVE_SYS_EVENT_H)
+CHECK_INCLUDE_FILES(sys/queue.h HAVE_SYS_QUEUE_H)
+CHECK_INCLUDE_FILES(sys/stat.h HAVE_SYS_STAT_H)
+CHECK_INCLUDE_FILES(sys/signalfd.h HAVE_SYS_SIGNALFD_H)
+CHECK_INCLUDE_FILES(port.h HAVE_PORT_H)
+CHECK_INCLUDE_FILES(poll.h HAVE_POLL_H)
+CHECK_INCLUDE_FILES(memory.h HAVE_MEMORY_H)
+CHECK_INCLUDE_FILES(sys/select.h HAVE_SYS_SELECT_H)
+CHECK_INCLUDE_FILES(sys/eventfd.h HAVE_SYS_EVENTFD_H)
+CHECK_INCLUDE_FILES(sys/timerfd.h HAVE_SYS_TIMERFD_H)
+CHECK_INCLUDE_FILES(linux/fs.h HAVE_LINUX_FS_H)
+CHECK_INCLUDE_FILES(linux/aio_abi.h HAVE_LINUX_AIO_ABI_H)
+
+IF(HAVE_SYS_INOTIFY_H)
+ CHECK_SYMBOL_EXISTS(inotify_init "sys/types.h;sys/inotify.h" HAVE_INOTIFY_INIT)
+ENDIF()
+IF(HAVE_SYS_EPOLL_H)
+ CHECK_SYMBOL_EXISTS(epoll_ctl "sys/types.h;sys/epoll.h" HAVE_EPOLL_CTL)
+ENDIF()
+IF(HAVE_SYS_EVENT_H)
+ CHECK_SYMBOL_EXISTS(kqueue "sys/types.h;sys/event.h;sys/time.h" HAVE_KQUEUE)
+ENDIF()
+IF(HAVE_PORT_H)
+ CHECK_SYMBOL_EXISTS(port_create port.h HAVE_PORT_CREATE)
+ENDIF()
+IF(HAVE_POLL_H)
+ CHECK_SYMBOL_EXISTS(poll poll.h HAVE_POLL)
+ENDIF()
+IF(HAVE_SYS_SELECT_H)
+ CHECK_SYMBOL_EXISTS(select sys/select.h HAVE_SELECT)
+ENDIF()
+IF(HAVE_SYS_EVENTFD_H)
+ CHECK_SYMBOL_EXISTS(eventfd sys/eventfd.h HAVE_EVENTFD)
+ENDIF()
+IF(HAVE_SYS_SIGNALFD_H)
+ CHECK_SYMBOL_EXISTS(signalfd sys/signalfd.h HAVE_SIGNALFD)
+ENDIF()
+IF(HAVE_SYS_TIMERFD_H)
+ CHECK_SYMBOL_EXISTS(timerfd_create sys/timerfd.h HAVE_TIMERFD)
+ENDIF()
+IF(HAVE_LINUX_FS_H)
+ CHECK_SYMBOL_EXISTS(RWF_SUPPORTED linux/fs.h HAVE_KERNEL_RWF_T)
+ENDIF()
+CHECK_SYMBOL_EXISTS(time.h nanosleep HAVE_NANOSLEEP)
+
+# check first without rt
+CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME)
+
+CHECK_LIBRARY_EXISTS(rt clock_gettime "" HAVE_LIBRT)
+# then check with rt
+CHECK_LIBRARY_EXISTS(rt clock_gettime "" HAVE_CLOCK_GETTIME)
+CHECK_LIBRARY_EXISTS(m ceil "" HAVE_LIBM)
+
+CONFIGURE_FILE(config.h.in libev-config.h)
+
+IF(ENABLE_STATIC MATCHES "ON")
+ ADD_LIBRARY(rspamd-ev STATIC ${LIBEVSRC})
+ELSE()
+ ADD_LIBRARY(rspamd-ev SHARED ${LIBEVSRC})
+ENDIF()
+target_link_libraries(rspamd-ev "${RSPAMD_REQUIRED_LIBRARIES}")
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+ADD_DEFINITIONS("-DEV_CONFIG_H=\"libev-config.h\""
+ -DEV_MULTIPLICITY=1
+ -DEV_USE_FLOOR=1
+ -DEV_NO_THREADS=1 # We do not have threads in Rspamd!
+ -DEV_FEATURES=127 # Enable all features
+ )
+IF(HAVE_EVENTFD)
+ ADD_DEFINITIONS(-DEV_USE_EVENTFD=1)
+ENDIF()
+
+IF(ENABLE_FULL_DEBUG MATCHES "ON")
+ ADD_DEFINITIONS(-DEV_VERIFY=3)
+ENDIF()
+
+INSTALL(TARGETS rspamd-ev LIBRARY DESTINATION ${RSPAMD_LIBDIR})
diff --git a/contrib/libev/Changes b/contrib/libev/Changes
new file mode 100644
index 0000000..04e58ba
--- /dev/null
+++ b/contrib/libev/Changes
@@ -0,0 +1,529 @@
+Revision history for libev, a high-performance and full-featured event loop.
+
+4.25 Fri Dec 21 07:49:20 CET 2018
+ - INCOMPATIBLE CHANGE: EV_THROW was renamed to EV_NOEXCEPT
+ (EV_THROW sitll provided) and now uses noexcept on C++11 or newer.
+ - move the darwin select workaround highe rin ev.c, as newer versions of
+ darwin managed to break their broken select even more.
+ - ANDROID => __ANDROID__ (reported by enh@google.com).
+ - disable epoll_create1 on android because it has broken header files
+ and google is unwilling to fix them (reported by enh@google.com).
+ - avoid a minor compilation warning on win32.
+ - c++: remove deprecated dynamic throw() specifications.
+ - c++: improve the (unsupported) bad_loop exception class.
+ - backport perl ev_periodic example to C, untested.
+ - update libecb, biggets change is to include a memory fence
+ in ECB_MEMORY_FENCE_RELEASE on x86/amd64.
+ - minor autoconf/automake modernisation.
+
+4.24 Wed Dec 28 05:19:55 CET 2016
+ - bump version to 4.24, as the release tarball inexplicably
+ didn't have the right version in ev.h, even though the cvs-tagged
+ version did have the right one (reported by Ales Teska).
+
+4.23 Wed Nov 16 18:23:41 CET 2016
+ - move some declarations at the beginning to help certain retarded
+ microsoft compilers, even though their documentation claims
+ otherwise (reported by Ruslan Osmanov).
+
+4.22 Sun Dec 20 22:11:50 CET 2015
+ - when epoll detects unremovable fds in the fd set, rebuild
+ only the epoll descriptor, not the signal pipe, to avoid
+ SIGPIPE in ev_async_send. This doesn't solve it on fork,
+ so document what needs to be done in ev_loop_fork
+ (analyzed by Benjamin Mahler).
+ - remove superfluous sys/timeb.h include on win32
+ (analyzed by Jason Madden).
+ - updated libecb.
+
+4.20 Sat Jun 20 13:01:43 CEST 2015
+ - prefer noexcept over throw () with C++ 11.
+ - update ecb.h due to incompatibilities with c11.
+ - fix a potential aliasing issue when reading and writing
+ watcher callbacks.
+
+4.19 Thu Sep 25 08:18:25 CEST 2014
+ - ev.h wasn't valid C++ anymore, which tripped compilers other than
+ clang, msvc or gcc (analyzed by Raphael 'kena' Poss). Unfortunately,
+ C++ doesn't support typedefs for function pointers fully, so the affected
+ declarations have to spell out the types each time.
+ - when not using autoconf, tighten the check for clock_gettime and related
+ functionality.
+
+4.18 Fri Sep 5 17:55:26 CEST 2014
+ - events on files were not always generated properly with the
+ epoll backend (testcase by Assaf Inbal).
+ - mark event pipe fd as cloexec after a fork (analyzed by Sami Farin).
+ - (ecb) support m68k, m88k and sh (patch by Miod Vallat).
+ - use a reasonable fallback for EV_NSIG instead of erroring out
+ when we can't detect the signal set size.
+ - in the absence of autoconf, do not use the clock syscall
+ on glibc >= 2.17 (avoids the syscall AND -lrt on systems
+ doing clock_gettime in userspace).
+ - ensure extern "C" function pointers are used for externally-visible
+ loop callbacks (not watcher callbacks yet).
+ - (ecb) work around memory barriers and volatile apparently both being
+ broken in visual studio 2008 and later (analysed and patch by Nicolas Noble).
+
+4.15 Fri Mar 1 12:04:50 CET 2013
+ - destroying a non-default loop would stop the global waitpid
+ watcher (Denis Bilenko).
+ - queueing pending watchers of higher priority from a watcher now invokes
+ them in a timely fashion (reported by Denis Bilenko).
+ - add throw() to all libev functions that cannot throw exceptions, for
+ further code size decrease when compiling for C++.
+ - add throw () to callbacks that must not throw exceptions (allocator,
+ syserr, loop acquire/release, periodic reschedule cbs).
+ - fix event_base_loop return code, add event_get_callback, event_base_new,
+ event_base_get_method calls to improve libevent 1.x emulation and add
+ some libevent 2.x functionality (based on a patch by Jeff Davey).
+ - add more memory fences to fix a bug reported by Jeff Davey. Better
+ be overfenced than underprotected.
+ - ev_run now returns a boolean status (true meaning watchers are
+ still active).
+ - ev_once: undef EV_ERROR in ev_kqueue.c, to avoid clashing with
+ libev's EV_ERROR (reported by 191919).
+ - (ecb) add memory fence support for xlC (Darin McBride).
+ - (ecb) add memory fence support for gcc-mips (Anton Kirilov).
+ - (ecb) add memory fence support for gcc-alpha (Christian Weisgerber).
+ - work around some kernels losing file descriptors by leaking
+ the kqueue descriptor in the child.
+ - work around linux inotify not reporting IN_ATTRIB changes for directories
+ in many cases.
+ - include sys/syscall.h instead of plain syscall.h.
+ - check for io watcher loops in ev_verify, check for the most
+ common reported usage bug in ev_io_start.
+ - choose socket vs. WSASocket at compiletime using EV_USE_WSASOCKET.
+ - always use WSASend/WSARecv directly on windows, hoping that this
+ works in all cases (unlike read/write/send/recv...).
+ - try to detect signals around a fork faster (test program by
+ Denis Bilenko).
+ - work around recent glibc versions that leak memory in realloc.
+ - rename ev::embed::set to ev::embed::set_embed to avoid clashing
+ the watcher base set (loop) method.
+ - rewrite the async/signal pipe logic to always keep a valid fd, which
+ simplifies (and hopefully correctifies :) the race checking
+ on fork, at the cost of one extra fd.
+ - add fat, msdos, jffs2, ramfs, ntfs and btrfs to the list of
+ inotify-supporting filesystems.
+ - move orig_CFLAGS assignment to after AC_INIT, as newer autoconf
+ versions ignore it before
+ (https://bugzilla.redhat.com/show_bug.cgi?id=908096).
+ - add some untested android support.
+ - enum expressions must be of type int (reported by Juan Pablo L).
+
+4.11 Sat Feb 4 19:52:39 CET 2012
+ - INCOMPATIBLE CHANGE: ev_timer_again now clears the pending status, as
+ was documented already, but not implemented in the repeating case.
+ - new compiletime symbols: EV_NO_SMP and EV_NO_THREADS.
+ - fix a race where the workaround against the epoll fork bugs
+ caused signals to not be handled anymore.
+ - correct backend_fudge for most backends, and implement a windows
+ specific workaround to avoid looping because we call both
+ select and Sleep, both with different time resolutions.
+ - document range and guarantees of ev_sleep.
+ - document reasonable ranges for periodics interval and offset.
+ - rename backend_fudge to backend_mintime to avoid future confusion :)
+ - change the default periodic reschedule function to hopefully be more
+ exact and correct even in corner cases or in the far future.
+ - do not rely on -lm anymore: use it when available but use our
+ own floor () if it is missing. This should make it easier to embed,
+ as no external libraries are required.
+ - strategically import macros from libecb and mark rarely-used functions
+ as cache-cold (saving almost 2k code size on typical amd64 setups).
+ - add Symbols.ev and Symbols.event files, that were missing.
+ - fix backend_mintime value for epoll (was 1/1024, is 1/1000 now).
+ - fix #3 "be smart about timeouts" to not "deadlock" when
+ timeout == now, also improve the section overall.
+ - avoid "AVOIDING FINISHING BEFORE RETURNING" idiom.
+ - support new EV_API_STATIC mode to make all libev symbols
+ static.
+ - supply default CFLAGS of -g -O3 with gcc when original CFLAGS
+ were empty.
+
+4.04 Wed Feb 16 09:01:51 CET 2011
+ - fix two problems in the native win32 backend, where reuse of fd's
+ with different underlying handles caused handles not to be removed
+ or added to the select set (analyzed and tested by Bert Belder).
+ - do no rely on ceil() in ev_e?poll.c.
+ - backport libev to HP-UX versions before 11 v3.
+ - configure did not detect nanosleep and clock_gettime properly when
+ they are available in the libc (as opposed to -lrt).
+
+4.03 Tue Jan 11 14:37:25 CET 2011
+ - officially support polling files with all backends.
+ - support files, /dev/zero etc. the same way as select in the epoll
+ backend, by generating events on our own.
+ - ports backend: work around solaris bug 6874410 and many related ones
+ (EINTR, maybe more), with no performance loss (note that the solaris
+ bug report is actually wrong, reality is far more bizarre and broken
+ than that).
+ - define EV_READ/EV_WRITE as macros in event.h, as some programs use
+ #ifdef to test for them.
+ - new (experimental) function: ev_feed_signal.
+ - new (to become default) EVFLAG_NOSIGMASK flag.
+ - new EVBACKEND_MASK symbol.
+ - updated COMMON IDIOMS SECTION.
+
+4.01 Fri Nov 5 21:51:29 CET 2010
+ - automake fucked it up, apparently, --add-missing -f is not quite enough
+ to make it update its files, so 4.00 didn't install ev++.h and
+ event.h on make install. grrr.
+ - ev_loop(count|depth) didn't return anything (Robin Haberkorn).
+ - change EV_UNDEF to 0xffffffff to silence some overzealous compilers.
+ - use "(libev) " prefix for all libev error messages now.
+
+4.00 Mon Oct 25 12:32:12 CEST 2010
+ - "PORTING FROM LIBEV 3.X TO 4.X" (in ev.pod) is recommended reading.
+ - ev_embed_stop did not correctly stop the watcher (very good
+ testcase by Vladimir Timofeev).
+ - ev_run will now always update the current loop time - it erroneously
+ didn't when idle watchers were active, causing timers not to fire.
+ - fix a bug where a timeout of zero caused the timer not to fire
+ in the libevent emulation (testcase by Péter Szabó).
+ - applied win32 fixes by Michael Lenaghan (also James Mansion).
+ - replace EV_MINIMAL by EV_FEATURES.
+ - prefer EPOLL_CTL_ADD over EPOLL_CTL_MOD in some more cases, as it
+ seems the former is *much* faster than the latter.
+ - linux kernel version detection (for inotify bug workarounds)
+ did not work properly.
+ - reduce the number of spurious wake-ups with the ports backend.
+ - remove dependency on sys/queue.h on freebsd (patch by Vanilla Hsu).
+ - do async init within ev_async_start, not ev_async_set, which avoids
+ an API quirk where the set function must be called in the C++ API
+ even when there is nothing to set.
+ - add (undocumented) EV_ENABLE when adding events with kqueue,
+ this might help with OS X, which seems to need it despite documenting
+ not to need it (helpfully pointed out by Tilghman Lesher).
+ - do not use poll by default on freebsd, it's broken (what isn't
+ on freebsd...).
+ - allow to embed epoll on kernels >= 2.6.32.
+ - configure now prepends -O3, not appends it, so one can still
+ override it.
+ - ev.pod: greatly expanded the portability section, added a porting
+ section, a description of watcher states and made lots of minor fixes.
+ - disable poll backend on AIX, the poll header spams the namespace
+ and it's not worth working around dead platforms (reported
+ and analyzed by Aivars Kalvans).
+ - improve header file compatibility of the standalone eventfd code
+ in an obscure case.
+ - implement EV_AVOID_STDIO option.
+ - do not use sscanf to parse linux version number (smaller, faster,
+ no sscanf dependency).
+ - new EV_CHILD_ENABLE and EV_SIGNAL_ENABLE configurable settings.
+ - update libev.m4 HAVE_CLOCK_SYSCALL test for newer glibcs.
+ - add section on accept() problems to the manpage.
+ - rename EV_TIMEOUT to EV_TIMER.
+ - rename ev_loop_count/depth/verify/loop/unloop.
+ - remove ev_default_destroy and ev_default_fork.
+ - switch to two-digit minor version.
+ - work around an apparent gentoo compiler bug.
+ - define _DARWIN_UNLIMITED_SELECT. just so.
+ - use enum instead of #define for most constants.
+ - improve compatibility to older C++ compilers.
+ - (experimental) ev_run/ev_default_loop/ev_break/ev_loop_new have now
+ default arguments when compiled as C++.
+ - enable automake dependency tracking.
+ - ev_loop_new no longer leaks memory when loop creation failed.
+ - new ev_cleanup watcher type.
+
+3.9 Thu Dec 31 07:59:59 CET 2009
+ - signalfd is no longer used by default and has to be requested
+ explicitly - this means that easy to catch bugs become hard to
+ catch race conditions, but the users have spoken.
+ - point out the unspecified signal mask in the documentation, and
+ that this is a race condition regardless of EV_SIGNALFD.
+ - backport inotify code to C89.
+ - inotify file descriptors could leak into child processes.
+ - ev_stat watchers could keep an erroneous extra ref on the loop,
+ preventing exit when unregistering all watchers (testcases
+ provided by ry@tinyclouds.org).
+ - implement EV_WIN32_HANDLE_TO_FD and EV_WIN32_CLOSE_FD configuration
+ symbols to make it easier for apps to do their own fd management.
+ - support EV_IDLE_ENABLE being disabled in ev++.h
+ (patch by Didier Spezia).
+ - take advantage of inotify_init1, if available, to set cloexec/nonblock
+ on fd creation, to avoid races.
+ - the signal handling pipe wasn't always initialised under windows
+ (analysed by lekma).
+ - changed minimum glibc requirement from glibc 2.9 to 2.7, for
+ signalfd.
+ - add missing string.h include (Denis F. Latypoff).
+ - only replace ev_stat.prev when we detect an actual difference,
+ so prev is (almost) always different to attr. this might
+ have caused the problems with 04_stat.t.
+ - add ev::timer->remaining () method to C++ API.
+
+3.8 Sun Aug 9 14:30:45 CEST 2009
+ - incompatible change: do not necessarily reset signal handler
+ to SIG_DFL when a sighandler is stopped.
+ - ev_default_destroy did not properly free or zero some members,
+ potentially causing crashes and memory corruption on repeated
+ ev_default_destroy/ev_default_loop calls.
+ - take advantage of signalfd on GNU/Linux systems.
+ - document that the signal mask might be in an unspecified
+ state when using libev's signal handling.
+ - take advantage of some GNU/Linux calls to set cloexec/nonblock
+ on fd creation, to avoid race conditions.
+
+3.7 Fri Jul 17 16:36:32 CEST 2009
+ - ev_unloop and ev_loop wrongly used a global variable to exit loops,
+ instead of using a per-loop variable (bug caught by accident...).
+ - the ev_set_io_collect_interval interpretation has changed.
+ - add new functionality: ev_set_userdata, ev_userdata,
+ ev_set_invoke_pending_cb, ev_set_loop_release_cb,
+ ev_invoke_pending, ev_pending_count, together with a long example
+ about thread locking.
+ - add ev_timer_remaining (as requested by Denis F. Latypoff).
+ - add ev_loop_depth.
+ - calling ev_unloop in fork/prepare watchers will no longer poll
+ for new events.
+ - Denis F. Latypoff corrected many typos in example code snippets.
+ - honor autoconf detection of EV_USE_CLOCK_SYSCALL, also double-
+ check that the syscall number is available before trying to
+ use it (reported by ry@tinyclouds).
+ - use GetSystemTimeAsFileTime instead of _timeb on windows, for
+ slightly higher accuracy.
+ - properly declare ev_loop_verify and ev_now_update even when
+ !EV_MULTIPLICITY.
+ - do not compile in any priority code when EV_MAXPRI == EV_MINPRI.
+ - support EV_MINIMAL==2 for a reduced API.
+ - actually 0-initialise struct sigaction when installing signals.
+ - add section on hibernate and stopped processes to ev_timer docs.
+
+3.6 Tue Apr 28 02:49:30 CEST 2009
+ - multiple timers becoming ready within an event loop iteration
+ will be invoked in the "correct" order now.
+ - do not leave the event loop early just because we have no active
+ watchers, fixing a problem when embedding a kqueue loop
+ that has active kernel events but no registered watchers
+ (reported by blacksand blacksand).
+ - correctly zero the idx values for arrays, so destroying and
+ reinitialising the default loop actually works (patch by
+ Malek Hadj-Ali).
+ - implement ev_suspend and ev_resume.
+ - new EV_CUSTOM revents flag for use by applications.
+ - add documentation section about priorities.
+ - add a glossary to the documentation.
+ - extend the ev_fork description slightly.
+ - optimize a jump out of call_pending.
+
+3.53 Sun Feb 15 02:38:20 CET 2009
+ - fix a bug in event pipe creation on win32 that would cause a
+ failed assertion on event loop creation (patch by Malek Hadj-Ali).
+ - probe for CLOCK_REALTIME support at runtime as well and fall
+ back to gettimeofday if there is an error, to support older
+ operating systems with newer header files/libraries.
+ - prefer gettimeofday over clock_gettime with USE_CLOCK_SYSCALL
+ (default most everywhere), otherwise not.
+
+3.52 Wed Jan 7 21:43:02 CET 2009
+ - fix compilation of select backend in fd_set mode when NFDBITS is
+ missing (to get it to compile on QNX, reported by Rodrigo Campos).
+ - better select-nfds handling when select backend is in fd_set mode.
+ - diagnose fd_set overruns when select backend is in fd_set mode.
+ - due to a thinko, instead of disabling everything but
+ select on the borked OS X platform, everything but select was
+ allowed (reported by Emanuele Giaquinta).
+ - actually verify that local and remote port are matching in
+ libev's socketpair emulation, which makes denial-of-service
+ attacks harder (but not impossible - it's windows). Make sure
+ it even works under vista, which thinks that getpeer/sockname
+ should return fantasy port numbers.
+ - include "libev" in all assertion messages for potentially
+ clearer diagnostics.
+ - event_get_version (libevent compatibility) returned
+ a useless string instead of the expected version string
+ (patch by W.C.A. Wijngaards).
+
+3.51 Wed Dec 24 23:00:11 CET 2008
+ - fix a bug where an inotify watcher was added twice, causing
+ freezes on hash collisions (reported and analysed by Graham Leggett).
+ - new config symbol, EV_USE_CLOCK_SYSCALL, to make libev use
+ a direct syscall - slower, but no dependency on librt et al.
+ - assume negative return values != -1 signals success of port_getn
+ (http://cvs.epicsol.org/cgi/viewcvs.cgi/epic5/source/newio.c?rev=1.52)
+ (no known failure reports, but it doesn't hurt).
+ - fork detection in ev_embed now stops and restarts the watcher
+ automatically.
+ - EXPERIMENTAL: default the method to operator () in ev++.h,
+ to make it nicer to use functors (requested by Benedek László).
+ - fixed const object callbacks in ev++.h.
+ - replaced loop_ref argument of watcher.set (loop) by a direct
+ ev_loop * in ev++.h, to avoid clashes with functor patch.
+ - do not try to watch the empty string via inotify.
+ - inotify watchers could be leaked under certain circumstances.
+ - OS X 10.5 is actually even more broken than earlier versions,
+ so fall back to select on that piece of garbage.
+ - fixed some weirdness in the ev_embed documentation.
+
+3.49 Wed Nov 19 11:26:53 CET 2008
+ - ev_stat watchers will now use inotify as a mere hint on
+ kernels <2.6.25, or if the filesystem is not in the
+ "known to be good" list.
+ - better mingw32 compatibility (it's not as borked as native win32)
+ (analysed by Roger Pack).
+ - include stdio.h in the example program, as too many people are
+ confused by the weird C language otherwise. I guess the next thing
+ I get told is that the "..." ellipses in the examples don't compile
+ with their C compiler.
+
+3.48 Thu Oct 30 09:02:37 CET 2008
+ - further optimise away the EPOLL_CTL_ADD/MOD combo in the epoll
+ backend by assuming the kernel event mask hasn't changed if
+ ADD fails with EEXIST.
+ - work around spurious event notification bugs in epoll by using
+ a 32-bit generation counter. recreate kernel state if we receive
+ spurious notifications or unwanted events. this is very costly,
+ but I didn't come up with this horrible design.
+ - use memset to initialise most arrays now and do away with the
+ init functions.
+ - expand time-out strategies into a "Be smart about timeouts" section.
+ - drop the "struct" from all ev_watcher declarations in the
+ documentation and did other clarifications (yeah, it was a mistake
+ to have a struct AND a function called ev_loop).
+ - fix a bug where ev_default would not initialise the default
+ loop again after it was destroyed with ev_default_destroy.
+ - rename syserr to ev_syserr to avoid name clashes when embedding,
+ do similar changes for event.c.
+
+3.45 Tue Oct 21 21:59:26 CEST 2008
+ - disable inotify usage on linux <2.6.25, as it is broken
+ (reported by Yoann Vandoorselaere).
+ - ev_stat erroneously would try to add inotify watchers
+ even when inotify wasn't available (this should only
+ have a performance impact).
+ - ev_once now passes both timeout and io to the callback if both
+ occur concurrently, instead of giving timeouts precedence.
+ - disable EV_USE_INOTIFY when sys/inotify.h is too old.
+
+3.44 Mon Sep 29 05:18:39 CEST 2008
+ - embed watchers now automatically invoke ev_loop_fork on the
+ embedded loop when the parent loop forks.
+ - new function: ev_now_update (loop).
+ - verify_watcher was not marked static.
+ - improve the "associating..." manpage section.
+ - documentation tweaks here and there.
+
+3.43 Sun Jul 6 05:34:41 CEST 2008
+ - include more include files on windows to get struct _stati64
+ (reported by Chris Hulbert, but doesn't quite fix his issue).
+ - add missing #include <io.h> in ev.c on windows (reported by
+ Matt Tolton).
+
+3.42 Tue Jun 17 12:12:07 CEST 2008
+ - work around yet another windows bug: FD_SET actually adds fd's
+ multiple times to the fd_*SET*, despite official MSN docs claiming
+ otherwise. Reported and well-analysed by Matt Tolton.
+ - define NFDBITS to 0 when EV_SELECT_IS_WINSOCKET to make it compile
+ (reported any analysed by Chris Hulbert).
+ - fix a bug in ev_ebadf (this function is only used to catch
+ programming errors in the libev user). reported by Matt Tolton.
+ - fix a bug in fd_intern on win32 (could lead to compile errors
+ under some circumstances, but would work correctly if it compiles).
+ reported by Matt Tolton.
+ - (try to) work around missing lstat on windows.
+ - pass in the write fd set as except fd set under windows. windows
+ is so uncontrollably lame that it requires this. this means that
+ switching off oobinline is not supported (but tcp/ip doesn't
+ have oob, so that would be stupid anyways.
+ - use posix module symbol to auto-detect monotonic clock presence
+ and some other default values.
+
+3.41 Fri May 23 18:42:54 CEST 2008
+ - work around an obscure bug in winsocket select: if you
+ provide only empty fd sets then select returns WSAEINVAL. how sucky.
+ - improve timer scheduling stability and reduce use of time_epsilon.
+ - use 1-based 2-heap for EV_MINIMAL, simplifies code, reduces
+ codesize and makes for better cache-efficiency.
+ - use 3-based 4-heap for !EV_MINIMAL. this makes better use
+ of cpu cache lines and gives better growth behaviour than
+ 2-based heaps.
+ - cache timestamp within heap for !EV_MINIMAL, to avoid random
+ memory accesses.
+ - document/add EV_USE_4HEAP and EV_HEAP_CACHE_AT.
+ - fix a potential aliasing issue in ev_timer_again.
+ - add/document ev_periodic_at, retract direct access to ->at.
+ - improve ev_stat docs.
+ - add portability requirements section.
+ - fix manpage headers etc.
+ - normalise WSA error codes to lower range on windows.
+ - add consistency check code that can be called automatically
+ or on demand to check for internal structures (ev_loop_verify).
+
+3.31 Wed Apr 16 20:45:04 CEST 2008
+ - added last minute fix for ev_poll.c by Brandon Black.
+
+3.3 Wed Apr 16 19:04:10 CEST 2008
+ - event_base_loopexit should return 0 on success
+ (W.C.A. Wijngaards).
+ - added linux eventfd support.
+ - try to autodetect epoll and inotify support
+ by libc header version if not using autoconf.
+ - new symbols: EV_DEFAULT_UC and EV_DEFAULT_UC_.
+ - declare functions defined in ev.h as inline if
+ C99 or gcc are available.
+ - enable inlining with gcc versions 2 and 3.
+ - work around broken poll implementations potentially
+ not clearing revents field in ev_poll (Brandon Black)
+ (no such systems are known at this time).
+ - work around a bug in realloc on openbsd and darwin,
+ also makes the erroneous valgrind complaints
+ go away (noted by various people).
+ - fix ev_async_pending, add c++ wrapper for ev_async
+ (based on patch sent by Johannes Deisenhofer).
+ - add sensible set method to ev::embed.
+ - made integer constants type int in ev.h.
+
+3.2 Wed Apr 2 17:11:19 CEST 2008
+ - fix a 64 bit overflow issue in the select backend,
+ by using fd_mask instead of int for the mask.
+ - rename internal sighandler to avoid clash with very old perls.
+ - entering ev_loop will not clear the ONESHOT or NONBLOCKING
+ flags of any outer loops anymore.
+ - add ev_async_pending.
+
+3.1 Thu Mar 13 13:45:22 CET 2008
+ - implement ev_async watchers.
+ - only initialise signal pipe on demand.
+ - make use of sig_atomic_t configurable.
+ - improved documentation.
+
+3.0 Mon Jan 28 13:14:47 CET 2008
+ - API/ABI bump to version 3.0.
+ - ev++.h includes "ev.h" by default now, not <ev.h>.
+ - slightly improved documentation.
+ - speed up signal detection after a fork.
+ - only optionally return trace status changed in ev_child
+ watchers.
+ - experimental (and undocumented) loop wrappers for ev++.h.
+
+2.01 Tue Dec 25 08:04:41 CET 2007
+ - separate Changes file.
+ - fix ev_path_set => ev_stat_set typo.
+ - remove event_compat.h from the libev tarball.
+ - change how include files are found.
+ - doc updates.
+ - update licenses, explicitly allow for GPL relicensing.
+
+2.0 Sat Dec 22 17:47:03 CET 2007
+ - new ev_sleep, ev_set_(io|timeout)_collect_interval.
+ - removed epoll from embeddable fd set.
+ - fix embed watchers.
+ - renamed ev_embed.loop to other.
+ - added exported Symbol tables.
+ - undefine member wrapper macros at the end of ev.c.
+ - respect EV_H in ev++.h.
+
+1.86 Tue Dec 18 02:36:57 CET 2007
+ - fix memleak on loop destroy (not relevant for perl).
+
+1.85 Fri Dec 14 20:32:40 CET 2007
+ - fix some aliasing issues w.r.t. timers and periodics
+ (not relevant for perl).
+
+(for historic versions refer to EV/Changes, found in the Perl interface)
+
+0.1 Wed Oct 31 21:31:48 CET 2007
+ - original version; hacked together in <24h.
+
diff --git a/contrib/libev/LICENSE b/contrib/libev/LICENSE
new file mode 100644
index 0000000..2fdabd4
--- /dev/null
+++ b/contrib/libev/LICENSE
@@ -0,0 +1,37 @@
+All files in libev are
+Copyright (c)2007,2008,2009,2010,2011,2012,2013 Marc Alexander Lehmann.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Alternatively, the contents of this package may be used under the terms
+of the GNU General Public License ("GPL") version 2 or any later version,
+in which case the provisions of the GPL are applicable instead of the
+above. If you wish to allow the use of your version of this package only
+under the terms of the GPL and not to allow others to use your version of
+this file under the BSD license, indicate your decision by deleting the
+provisions above and replace them with the notice and other provisions
+required by the GPL in this and the other files of this package. If you do
+not delete the provisions above, a recipient may use your version of this
+file under either the BSD or the GPL.
diff --git a/contrib/libev/config.h.in b/contrib/libev/config.h.in
new file mode 100644
index 0000000..686a379
--- /dev/null
+++ b/contrib/libev/config.h.in
@@ -0,0 +1,112 @@
+/* config.h.in. Generated from configure.ac by autoheader. */
+
+/* Define to 1 if you have the `clock_gettime' function. */
+#cmakedefine HAVE_CLOCK_GETTIME 1
+
+/* Define to 1 to use the syscall interface for clock_gettime */
+#cmakedefine HAVE_CLOCK_SYSCALL 1
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#cmakedefine HAVE_DLFCN_H 1
+
+/* Define to 1 if you have the `epoll_ctl' function. */
+#cmakedefine HAVE_EPOLL_CTL 1
+
+/* Define to 1 if you have the `eventfd' function. */
+#cmakedefine HAVE_EVENTFD 1
+
+/* Define to 1 if the floor function is available */
+#cmakedefine HAVE_FLOOR 1
+
+/* Define to 1 if you have the `inotify_init' function. */
+#cmakedefine HAVE_INOTIFY_INIT 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#cmakedefine HAVE_INTTYPES_H 1
+
+/* Define to 1 if linux/fs.h defined kernel_rwf_t */
+#cmakedefine HAVE_KERNEL_RWF_T 1
+
+/* Define to 1 if you have the `kqueue' function. */
+#cmakedefine HAVE_KQUEUE 1
+
+/* Define to 1 if you have the `rt' library (-lrt). */
+#cmakedefine HAVE_LIBRT 1
+
+/* Define to 1 if you have the <linux/aio_abi.h> header file. */
+#cmakedefine HAVE_LINUX_AIO_ABI_H 1
+
+/* Define to 1 if you have the <linux/fs.h> header file. */
+#cmakedefine HAVE_LINUX_FS_H 1
+
+/* Define to 1 if you have the <memory.h> header file. */
+#cmakedefine HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the `nanosleep' function. */
+#cmakedefine HAVE_NANOSLEEP 1
+
+/* Define to 1 if you have the `poll' function. */
+#cmakedefine HAVE_POLL 1
+
+/* Define to 1 if you have the <poll.h> header file. */
+#cmakedefine HAVE_POLL_H 1
+
+/* Define to 1 if you have the `port_create' function. */
+#cmakedefine HAVE_PORT_CREATE 1
+
+/* Define to 1 if you have the <port.h> header file. */
+#cmakedefine HAVE_PORT_H 1
+
+/* Define to 1 if you have the `select' function. */
+#cmakedefine HAVE_SELECT 1
+
+/* Define to 1 if you have the `signalfd' function. */
+#cmakedefine HAVE_SIGNALFD 1
+
+/* Define to 1 if you have the `timerfd_create' function. */
+#cmakedefine HAVE_TIMERFD 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#cmakedefine HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#cmakedefine HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#cmakedefine HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#cmakedefine HAVE_STRING_H 1
+
+/* Define to 1 if you have the <sys/epoll.h> header file. */
+#cmakedefine HAVE_SYS_EPOLL_H 1
+
+/* Define to 1 if you have the <sys/eventfd.h> header file. */
+#cmakedefine HAVE_SYS_EVENTFD_H 1
+
+/* Define to 1 if you have the <sys/event.h> header file. */
+#cmakedefine HAVE_SYS_EVENT_H 1
+
+/* Define to 1 if you have the <sys/inotify.h> header file. */
+#cmakedefine HAVE_SYS_INOTIFY_H 1
+
+/* Define to 1 if you have the <sys/select.h> header file. */
+#cmakedefine HAVE_SYS_SELECT_H 1
+
+/* Define to 1 if you have the <sys/signalfd.h> header file. */
+#cmakedefine HAVE_SYS_SIGNALFD_H 1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#cmakedefine HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/timerfd.h> header file. */
+#cmakedefine HAVE_SYS_TIMERFD_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#cmakedefine HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#cmakedefine HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
diff --git a/contrib/libev/ev++.h b/contrib/libev/ev++.h
new file mode 100644
index 0000000..0e1b60d
--- /dev/null
+++ b/contrib/libev/ev++.h
@@ -0,0 +1,816 @@
+/*
+ * libev simple C++ wrapper classes
+ *
+ * Copyright (c) 2007,2008,2010,2018 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#ifndef EVPP_H__
+#define EVPP_H__
+
+#ifdef EV_H
+# include EV_H
+#else
+# include "ev.h"
+#endif
+
+#ifndef EV_USE_STDEXCEPT
+# define EV_USE_STDEXCEPT 1
+#endif
+
+#if EV_USE_STDEXCEPT
+# include <stdexcept>
+#endif
+
+namespace ev {
+
+ typedef ev_tstamp tstamp;
+
+ enum {
+ UNDEF = EV_UNDEF,
+ NONE = EV_NONE,
+ READ = EV_READ,
+ WRITE = EV_WRITE,
+#if EV_COMPAT3
+ TIMEOUT = EV_TIMEOUT,
+#endif
+ TIMER = EV_TIMER,
+ PERIODIC = EV_PERIODIC,
+ SIGNAL = EV_SIGNAL,
+ CHILD = EV_CHILD,
+ STAT = EV_STAT,
+ IDLE = EV_IDLE,
+ CHECK = EV_CHECK,
+ PREPARE = EV_PREPARE,
+ FORK = EV_FORK,
+ ASYNC = EV_ASYNC,
+ EMBED = EV_EMBED,
+# undef ERROR // some systems stupidly #define ERROR
+ ERROR = EV_ERROR
+ };
+
+ enum
+ {
+ AUTO = EVFLAG_AUTO,
+ NOENV = EVFLAG_NOENV,
+ FORKCHECK = EVFLAG_FORKCHECK,
+
+ SELECT = EVBACKEND_SELECT,
+ POLL = EVBACKEND_POLL,
+ EPOLL = EVBACKEND_EPOLL,
+ KQUEUE = EVBACKEND_KQUEUE,
+ DEVPOLL = EVBACKEND_DEVPOLL,
+ PORT = EVBACKEND_PORT
+ };
+
+ enum
+ {
+#if EV_COMPAT3
+ NONBLOCK = EVLOOP_NONBLOCK,
+ ONESHOT = EVLOOP_ONESHOT,
+#endif
+ NOWAIT = EVRUN_NOWAIT,
+ ONCE = EVRUN_ONCE
+ };
+
+ enum how_t
+ {
+ ONE = EVBREAK_ONE,
+ ALL = EVBREAK_ALL
+ };
+
+ struct bad_loop
+#if EV_USE_STDEXCEPT
+ : std::exception
+#endif
+ {
+#if EV_USE_STDEXCEPT
+ const char *what () const EV_NOEXCEPT
+ {
+ return "libev event loop cannot be initialized, bad value of LIBEV_FLAGS?";
+ }
+#endif
+ };
+
+#ifdef EV_AX
+# undef EV_AX
+#endif
+
+#ifdef EV_AX_
+# undef EV_AX_
+#endif
+
+#if EV_MULTIPLICITY
+# define EV_AX raw_loop
+# define EV_AX_ raw_loop,
+#else
+# define EV_AX
+# define EV_AX_
+#endif
+
+ struct loop_ref
+ {
+ loop_ref (EV_P) EV_NOEXCEPT
+#if EV_MULTIPLICITY
+ : EV_AX (EV_A)
+#endif
+ {
+ }
+
+ bool operator == (const loop_ref &other) const EV_NOEXCEPT
+ {
+#if EV_MULTIPLICITY
+ return EV_AX == other.EV_AX;
+#else
+ return true;
+#endif
+ }
+
+ bool operator != (const loop_ref &other) const EV_NOEXCEPT
+ {
+#if EV_MULTIPLICITY
+ return ! (*this == other);
+#else
+ return false;
+#endif
+ }
+
+#if EV_MULTIPLICITY
+ bool operator == (const EV_P) const EV_NOEXCEPT
+ {
+ return this->EV_AX == EV_A;
+ }
+
+ bool operator != (const EV_P) const EV_NOEXCEPT
+ {
+ return ! (*this == EV_A);
+ }
+
+ operator struct ev_loop * () const EV_NOEXCEPT
+ {
+ return EV_AX;
+ }
+
+ operator const struct ev_loop * () const EV_NOEXCEPT
+ {
+ return EV_AX;
+ }
+
+ bool is_default () const EV_NOEXCEPT
+ {
+ return EV_AX == ev_default_loop (0);
+ }
+#endif
+
+#if EV_COMPAT3
+ void loop (int flags = 0)
+ {
+ ev_run (EV_AX_ flags);
+ }
+
+ void unloop (how_t how = ONE) EV_NOEXCEPT
+ {
+ ev_break (EV_AX_ how);
+ }
+#endif
+
+ void run (int flags = 0)
+ {
+ ev_run (EV_AX_ flags);
+ }
+
+ void break_loop (how_t how = ONE) EV_NOEXCEPT
+ {
+ ev_break (EV_AX_ how);
+ }
+
+ void post_fork () EV_NOEXCEPT
+ {
+ ev_loop_fork (EV_AX);
+ }
+
+ unsigned int backend () const EV_NOEXCEPT
+ {
+ return ev_backend (EV_AX);
+ }
+
+ tstamp now () const EV_NOEXCEPT
+ {
+ return ev_now (EV_AX);
+ }
+
+ void ref () EV_NOEXCEPT
+ {
+ ev_ref (EV_AX);
+ }
+
+ void unref () EV_NOEXCEPT
+ {
+ ev_unref (EV_AX);
+ }
+
+#if EV_FEATURE_API
+ unsigned int iteration () const EV_NOEXCEPT
+ {
+ return ev_iteration (EV_AX);
+ }
+
+ unsigned int depth () const EV_NOEXCEPT
+ {
+ return ev_depth (EV_AX);
+ }
+
+ void set_io_collect_interval (tstamp interval) EV_NOEXCEPT
+ {
+ ev_set_io_collect_interval (EV_AX_ interval);
+ }
+
+ void set_timeout_collect_interval (tstamp interval) EV_NOEXCEPT
+ {
+ ev_set_timeout_collect_interval (EV_AX_ interval);
+ }
+#endif
+
+ // function callback
+ void once (int fd, int events, tstamp timeout, void (*cb)(int, void *), void *arg = 0) EV_NOEXCEPT
+ {
+ ev_once (EV_AX_ fd, events, timeout, cb, arg);
+ }
+
+ // method callback
+ template<class K, void (K::*method)(int)>
+ void once (int fd, int events, tstamp timeout, K *object) EV_NOEXCEPT
+ {
+ once (fd, events, timeout, method_thunk<K, method>, object);
+ }
+
+ // default method == operator ()
+ template<class K>
+ void once (int fd, int events, tstamp timeout, K *object) EV_NOEXCEPT
+ {
+ once (fd, events, timeout, method_thunk<K, &K::operator ()>, object);
+ }
+
+ template<class K, void (K::*method)(int)>
+ static void method_thunk (int revents, void *arg)
+ {
+ (static_cast<K *>(arg)->*method)
+ (revents);
+ }
+
+ // no-argument method callback
+ template<class K, void (K::*method)()>
+ void once (int fd, int events, tstamp timeout, K *object) EV_NOEXCEPT
+ {
+ once (fd, events, timeout, method_noargs_thunk<K, method>, object);
+ }
+
+ template<class K, void (K::*method)()>
+ static void method_noargs_thunk (int revents, void *arg)
+ {
+ (static_cast<K *>(arg)->*method)
+ ();
+ }
+
+ // simpler function callback
+ template<void (*cb)(int)>
+ void once (int fd, int events, tstamp timeout) EV_NOEXCEPT
+ {
+ once (fd, events, timeout, simpler_func_thunk<cb>);
+ }
+
+ template<void (*cb)(int)>
+ static void simpler_func_thunk (int revents, void *arg)
+ {
+ (*cb)
+ (revents);
+ }
+
+ // simplest function callback
+ template<void (*cb)()>
+ void once (int fd, int events, tstamp timeout) EV_NOEXCEPT
+ {
+ once (fd, events, timeout, simplest_func_thunk<cb>);
+ }
+
+ template<void (*cb)()>
+ static void simplest_func_thunk (int revents, void *arg)
+ {
+ (*cb)
+ ();
+ }
+
+ void feed_fd_event (int fd, int revents) EV_NOEXCEPT
+ {
+ ev_feed_fd_event (EV_AX_ fd, revents);
+ }
+
+ void feed_signal_event (int signum) EV_NOEXCEPT
+ {
+ ev_feed_signal_event (EV_AX_ signum);
+ }
+
+#if EV_MULTIPLICITY
+ struct ev_loop* EV_AX;
+#endif
+
+ };
+
+#if EV_MULTIPLICITY
+ struct dynamic_loop : loop_ref
+ {
+
+ dynamic_loop (unsigned int flags = AUTO)
+ : loop_ref (ev_loop_new (flags))
+ {
+ if (!EV_AX)
+ throw bad_loop ();
+ }
+
+ ~dynamic_loop () EV_NOEXCEPT
+ {
+ ev_loop_destroy (EV_AX);
+ EV_AX = 0;
+ }
+
+ private:
+
+ dynamic_loop (const dynamic_loop &);
+
+ dynamic_loop & operator= (const dynamic_loop &);
+
+ };
+#endif
+
+ struct default_loop : loop_ref
+ {
+ default_loop (unsigned int flags = AUTO)
+#if EV_MULTIPLICITY
+ : loop_ref (ev_default_loop (flags))
+#endif
+ {
+ if (
+#if EV_MULTIPLICITY
+ !EV_AX
+#else
+ !ev_default_loop (flags)
+#endif
+ )
+ throw bad_loop ();
+ }
+
+ private:
+ default_loop (const default_loop &);
+ default_loop &operator = (const default_loop &);
+ };
+
+ inline loop_ref get_default_loop () EV_NOEXCEPT
+ {
+#if EV_MULTIPLICITY
+ return ev_default_loop (0);
+#else
+ return loop_ref ();
+#endif
+ }
+
+#undef EV_AX
+#undef EV_AX_
+
+#undef EV_PX
+#undef EV_PX_
+#if EV_MULTIPLICITY
+# define EV_PX loop_ref EV_A
+# define EV_PX_ loop_ref EV_A_
+#else
+# define EV_PX
+# define EV_PX_
+#endif
+
+ template<class ev_watcher, class watcher>
+ struct base : ev_watcher
+ {
+ #if EV_MULTIPLICITY
+ EV_PX;
+
+ // loop set
+ void set (EV_P) EV_NOEXCEPT
+ {
+ this->EV_A = EV_A;
+ }
+ #endif
+
+ base (EV_PX) EV_NOEXCEPT
+ #if EV_MULTIPLICITY
+ : EV_A (EV_A)
+ #endif
+ {
+ ev_init (this, 0);
+ }
+
+ void set_ (const void *data, void (*cb)(EV_P_ ev_watcher *w, int revents)) EV_NOEXCEPT
+ {
+ this->data = (void *)data;
+ ev_set_cb (static_cast<ev_watcher *>(this), cb);
+ }
+
+ // function callback
+ template<void (*function)(watcher &w, int)>
+ void set (void *data = 0) EV_NOEXCEPT
+ {
+ set_ (data, function_thunk<function>);
+ }
+
+ template<void (*function)(watcher &w, int)>
+ static void function_thunk (EV_P_ ev_watcher *w, int revents)
+ {
+ function
+ (*static_cast<watcher *>(w), revents);
+ }
+
+ // method callback
+ template<class K, void (K::*method)(watcher &w, int)>
+ void set (K *object) EV_NOEXCEPT
+ {
+ set_ (object, method_thunk<K, method>);
+ }
+
+ // default method == operator ()
+ template<class K>
+ void set (K *object) EV_NOEXCEPT
+ {
+ set_ (object, method_thunk<K, &K::operator ()>);
+ }
+
+ template<class K, void (K::*method)(watcher &w, int)>
+ static void method_thunk (EV_P_ ev_watcher *w, int revents)
+ {
+ (static_cast<K *>(w->data)->*method)
+ (*static_cast<watcher *>(w), revents);
+ }
+
+ // no-argument callback
+ template<class K, void (K::*method)()>
+ void set (K *object) EV_NOEXCEPT
+ {
+ set_ (object, method_noargs_thunk<K, method>);
+ }
+
+ template<class K, void (K::*method)()>
+ static void method_noargs_thunk (EV_P_ ev_watcher *w, int revents)
+ {
+ (static_cast<K *>(w->data)->*method)
+ ();
+ }
+
+ void operator ()(int events = EV_UNDEF)
+ {
+ return
+ ev_cb (static_cast<ev_watcher *>(this))
+ (static_cast<ev_watcher *>(this), events);
+ }
+
+ bool is_active () const EV_NOEXCEPT
+ {
+ return ev_is_active (static_cast<const ev_watcher *>(this));
+ }
+
+ bool is_pending () const EV_NOEXCEPT
+ {
+ return ev_is_pending (static_cast<const ev_watcher *>(this));
+ }
+
+ void feed_event (int revents) EV_NOEXCEPT
+ {
+ ev_feed_event (EV_A_ static_cast<ev_watcher *>(this), revents);
+ }
+ };
+
+ inline tstamp now (EV_P) EV_NOEXCEPT
+ {
+ return ev_now (EV_A);
+ }
+
+ inline void delay (tstamp interval) EV_NOEXCEPT
+ {
+ ev_sleep (interval);
+ }
+
+ inline int version_major () EV_NOEXCEPT
+ {
+ return ev_version_major ();
+ }
+
+ inline int version_minor () EV_NOEXCEPT
+ {
+ return ev_version_minor ();
+ }
+
+ inline unsigned int supported_backends () EV_NOEXCEPT
+ {
+ return ev_supported_backends ();
+ }
+
+ inline unsigned int recommended_backends () EV_NOEXCEPT
+ {
+ return ev_recommended_backends ();
+ }
+
+ inline unsigned int embeddable_backends () EV_NOEXCEPT
+ {
+ return ev_embeddable_backends ();
+ }
+
+ inline void set_allocator (void *(*cb)(void *ptr, long size) EV_NOEXCEPT) EV_NOEXCEPT
+ {
+ ev_set_allocator (cb);
+ }
+
+ inline void set_syserr_cb (void (*cb)(const char *msg) EV_NOEXCEPT) EV_NOEXCEPT
+ {
+ ev_set_syserr_cb (cb);
+ }
+
+ #if EV_MULTIPLICITY
+ #define EV_CONSTRUCT(cppstem,cstem) \
+ (EV_PX = get_default_loop ()) EV_NOEXCEPT \
+ : base<ev_ ## cstem, cppstem> (EV_A) \
+ { \
+ }
+ #else
+ #define EV_CONSTRUCT(cppstem,cstem) \
+ () EV_NOEXCEPT \
+ { \
+ }
+ #endif
+
+ /* using a template here would require quite a few more lines,
+ * so a macro solution was chosen */
+ #define EV_BEGIN_WATCHER(cppstem,cstem) \
+ \
+ struct cppstem : base<ev_ ## cstem, cppstem> \
+ { \
+ void start () EV_NOEXCEPT \
+ { \
+ ev_ ## cstem ## _start (EV_A_ static_cast<ev_ ## cstem *>(this)); \
+ } \
+ \
+ void stop () EV_NOEXCEPT \
+ { \
+ ev_ ## cstem ## _stop (EV_A_ static_cast<ev_ ## cstem *>(this)); \
+ } \
+ \
+ cppstem EV_CONSTRUCT(cppstem,cstem) \
+ \
+ ~cppstem () EV_NOEXCEPT \
+ { \
+ stop (); \
+ } \
+ \
+ using base<ev_ ## cstem, cppstem>::set; \
+ \
+ private: \
+ \
+ cppstem (const cppstem &o); \
+ \
+ cppstem &operator =(const cppstem &o); \
+ \
+ public:
+
+ #define EV_END_WATCHER(cppstem,cstem) \
+ };
+
+ EV_BEGIN_WATCHER (io, io)
+ void set (int fd, int events) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_io_set (static_cast<ev_io *>(this), fd, events);
+ if (active) start ();
+ }
+
+ void set (int events) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_io_set (static_cast<ev_io *>(this), fd, events);
+ if (active) start ();
+ }
+
+ void start (int fd, int events) EV_NOEXCEPT
+ {
+ set (fd, events);
+ start ();
+ }
+ EV_END_WATCHER (io, io)
+
+ EV_BEGIN_WATCHER (timer, timer)
+ void set (ev_tstamp after, ev_tstamp repeat = 0.) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_timer_set (static_cast<ev_timer *>(this), after, repeat);
+ if (active) start ();
+ }
+
+ void start (ev_tstamp after, ev_tstamp repeat = 0.) EV_NOEXCEPT
+ {
+ set (after, repeat);
+ start ();
+ }
+
+ void again () EV_NOEXCEPT
+ {
+ ev_timer_again (EV_A_ static_cast<ev_timer *>(this));
+ }
+
+ ev_tstamp remaining ()
+ {
+ return ev_timer_remaining (EV_A_ static_cast<ev_timer *>(this));
+ }
+ EV_END_WATCHER (timer, timer)
+
+ #if EV_PERIODIC_ENABLE
+ EV_BEGIN_WATCHER (periodic, periodic)
+ void set (ev_tstamp at, ev_tstamp interval = 0.) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_periodic_set (static_cast<ev_periodic *>(this), at, interval, 0);
+ if (active) start ();
+ }
+
+ void start (ev_tstamp at, ev_tstamp interval = 0.) EV_NOEXCEPT
+ {
+ set (at, interval);
+ start ();
+ }
+
+ void again () EV_NOEXCEPT
+ {
+ ev_periodic_again (EV_A_ static_cast<ev_periodic *>(this));
+ }
+ EV_END_WATCHER (periodic, periodic)
+ #endif
+
+ #if EV_SIGNAL_ENABLE
+ EV_BEGIN_WATCHER (sig, signal)
+ void set (int signum) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_signal_set (static_cast<ev_signal *>(this), signum);
+ if (active) start ();
+ }
+
+ void start (int signum) EV_NOEXCEPT
+ {
+ set (signum);
+ start ();
+ }
+ EV_END_WATCHER (sig, signal)
+ #endif
+
+ #if EV_CHILD_ENABLE
+ EV_BEGIN_WATCHER (child, child)
+ void set (int pid, int trace = 0) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_child_set (static_cast<ev_child *>(this), pid, trace);
+ if (active) start ();
+ }
+
+ void start (int pid, int trace = 0) EV_NOEXCEPT
+ {
+ set (pid, trace);
+ start ();
+ }
+ EV_END_WATCHER (child, child)
+ #endif
+
+ #if EV_STAT_ENABLE
+ EV_BEGIN_WATCHER (stat, stat)
+ void set (const char *path, ev_tstamp interval = 0.) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_stat_set (static_cast<ev_stat *>(this), path, interval);
+ if (active) start ();
+ }
+
+ void start (const char *path, ev_tstamp interval = 0.) EV_NOEXCEPT
+ {
+ stop ();
+ set (path, interval);
+ start ();
+ }
+
+ void update () EV_NOEXCEPT
+ {
+ ev_stat_stat (EV_A_ static_cast<ev_stat *>(this));
+ }
+ EV_END_WATCHER (stat, stat)
+ #endif
+
+ #if EV_IDLE_ENABLE
+ EV_BEGIN_WATCHER (idle, idle)
+ void set () EV_NOEXCEPT { }
+ EV_END_WATCHER (idle, idle)
+ #endif
+
+ #if EV_PREPARE_ENABLE
+ EV_BEGIN_WATCHER (prepare, prepare)
+ void set () EV_NOEXCEPT { }
+ EV_END_WATCHER (prepare, prepare)
+ #endif
+
+ #if EV_CHECK_ENABLE
+ EV_BEGIN_WATCHER (check, check)
+ void set () EV_NOEXCEPT { }
+ EV_END_WATCHER (check, check)
+ #endif
+
+ #if EV_EMBED_ENABLE
+ EV_BEGIN_WATCHER (embed, embed)
+ void set_embed (struct ev_loop *embedded_loop) EV_NOEXCEPT
+ {
+ int active = is_active ();
+ if (active) stop ();
+ ev_embed_set (static_cast<ev_embed *>(this), embedded_loop);
+ if (active) start ();
+ }
+
+ void start (struct ev_loop *embedded_loop) EV_NOEXCEPT
+ {
+ set (embedded_loop);
+ start ();
+ }
+
+ void sweep ()
+ {
+ ev_embed_sweep (EV_A_ static_cast<ev_embed *>(this));
+ }
+ EV_END_WATCHER (embed, embed)
+ #endif
+
+ #if EV_FORK_ENABLE
+ EV_BEGIN_WATCHER (fork, fork)
+ void set () EV_NOEXCEPT { }
+ EV_END_WATCHER (fork, fork)
+ #endif
+
+ #if EV_ASYNC_ENABLE
+ EV_BEGIN_WATCHER (async, async)
+ void send () EV_NOEXCEPT
+ {
+ ev_async_send (EV_A_ static_cast<ev_async *>(this));
+ }
+
+ bool async_pending () EV_NOEXCEPT
+ {
+ return ev_async_pending (static_cast<ev_async *>(this));
+ }
+ EV_END_WATCHER (async, async)
+ #endif
+
+ #undef EV_PX
+ #undef EV_PX_
+ #undef EV_CONSTRUCT
+ #undef EV_BEGIN_WATCHER
+ #undef EV_END_WATCHER
+}
+
+#endif
+
diff --git a/contrib/libev/ev.c b/contrib/libev/ev.c
new file mode 100644
index 0000000..230445d
--- /dev/null
+++ b/contrib/libev/ev.c
@@ -0,0 +1,5678 @@
+/*
+ * libev event processing core, watcher management
+ *
+ * Copyright (c) 2007-2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/* this big block deduces configuration from config.h */
+#ifndef EV_STANDALONE
+# ifdef EV_CONFIG_H
+# include EV_CONFIG_H
+# else
+# include "config.h"
+# endif
+
+#ifdef __GNUC__
+#pragma GCC diagnostic ignored "-Wunused-value"
+#endif
+
+# if HAVE_FLOOR
+# ifndef EV_USE_FLOOR
+# define EV_USE_FLOOR 1
+# endif
+# endif
+
+# if HAVE_CLOCK_SYSCALL
+# ifndef EV_USE_CLOCK_SYSCALL
+# define EV_USE_CLOCK_SYSCALL 1
+# ifndef EV_USE_REALTIME
+# define EV_USE_REALTIME 0
+# endif
+# ifndef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 1
+# endif
+# endif
+# elif !defined EV_USE_CLOCK_SYSCALL
+# define EV_USE_CLOCK_SYSCALL 0
+# endif
+
+# if HAVE_CLOCK_GETTIME
+# ifndef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 1
+# endif
+# ifndef EV_USE_REALTIME
+# define EV_USE_REALTIME 0
+# endif
+# else
+# ifndef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 0
+# endif
+# ifndef EV_USE_REALTIME
+# define EV_USE_REALTIME 0
+# endif
+# endif
+
+# if HAVE_NANOSLEEP
+# ifndef EV_USE_NANOSLEEP
+# define EV_USE_NANOSLEEP EV_FEATURE_OS
+# endif
+# else
+# undef EV_USE_NANOSLEEP
+# define EV_USE_NANOSLEEP 0
+# endif
+
+# if HAVE_SELECT && HAVE_SYS_SELECT_H
+# ifndef EV_USE_SELECT
+# define EV_USE_SELECT EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_SELECT
+# define EV_USE_SELECT 0
+# endif
+
+# if HAVE_POLL && HAVE_POLL_H
+# ifndef EV_USE_POLL
+# define EV_USE_POLL EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_POLL
+# define EV_USE_POLL 0
+# endif
+
+# if HAVE_EPOLL_CTL && HAVE_SYS_EPOLL_H
+# ifndef EV_USE_EPOLL
+# define EV_USE_EPOLL EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_EPOLL
+# define EV_USE_EPOLL 0
+# endif
+
+# if HAVE_LINUX_AIO_ABI_H
+# ifndef EV_USE_LINUXAIO
+# define EV_USE_LINUXAIO 0 /* was: EV_FEATURE_BACKENDS, always off by default */
+# endif
+# else
+# undef EV_USE_LINUXAIO
+# define EV_USE_LINUXAIO 0
+# endif
+
+# if HAVE_LINUX_FS_H && HAVE_SYS_TIMERFD_H && HAVE_KERNEL_RWF_T
+# ifndef EV_USE_IOURING
+# define EV_USE_IOURING EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_IOURING
+# define EV_USE_IOURING 0
+# endif
+
+# if HAVE_KQUEUE && HAVE_SYS_EVENT_H
+# ifndef EV_USE_KQUEUE
+# define EV_USE_KQUEUE EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_KQUEUE
+# define EV_USE_KQUEUE 0
+# endif
+
+# if HAVE_PORT_H && HAVE_PORT_CREATE
+# ifndef EV_USE_PORT
+# define EV_USE_PORT EV_FEATURE_BACKENDS
+# endif
+# else
+# undef EV_USE_PORT
+# define EV_USE_PORT 0
+# endif
+
+# if HAVE_INOTIFY_INIT && HAVE_SYS_INOTIFY_H
+# ifndef EV_USE_INOTIFY
+# define EV_USE_INOTIFY EV_FEATURE_OS
+# endif
+# else
+# undef EV_USE_INOTIFY
+# define EV_USE_INOTIFY 0
+# endif
+
+# if HAVE_SIGNALFD && HAVE_SYS_SIGNALFD_H
+# ifndef EV_USE_SIGNALFD
+# define EV_USE_SIGNALFD EV_FEATURE_OS
+# endif
+# else
+# undef EV_USE_SIGNALFD
+# define EV_USE_SIGNALFD 0
+# endif
+
+# if HAVE_EVENTFD
+# ifndef EV_USE_EVENTFD
+# define EV_USE_EVENTFD EV_FEATURE_OS
+# endif
+# else
+# undef EV_USE_EVENTFD
+# define EV_USE_EVENTFD 0
+# endif
+
+# if HAVE_SYS_TIMERFD_H && HAVE_TIMERFD
+# ifndef EV_USE_TIMERFD
+# define EV_USE_TIMERFD EV_FEATURE_OS
+# endif
+# else
+# undef EV_USE_TIMERFD
+# define EV_USE_TIMERFD 0
+# endif
+
+#endif
+
+/* OS X, in its infinite idiocy, actually HARDCODES
+ * a limit of 1024 into their select. Where people have brains,
+ * OS X engineers apparently have a vacuum. Or maybe they were
+ * ordered to have a vacuum, or they do anything for money.
+ * This might help. Or not.
+ * Note that this must be defined early, as other include files
+ * will rely on this define as well.
+ */
+#define _DARWIN_UNLIMITED_SELECT 1
+
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <stddef.h>
+
+#include <stdio.h>
+
+#include <assert.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <time.h>
+#include <limits.h>
+
+#include <signal.h>
+
+#ifdef EV_H
+# include EV_H
+#else
+# include "ev.h"
+#endif
+
+#if EV_NO_THREADS
+# undef EV_NO_SMP
+# define EV_NO_SMP 1
+# undef ECB_NO_THREADS
+# define ECB_NO_THREADS 1
+#endif
+#if EV_NO_SMP
+# undef EV_NO_SMP
+# define ECB_NO_SMP 1
+#endif
+
+#ifndef _WIN32
+# include <sys/time.h>
+# include <sys/wait.h>
+# include <unistd.h>
+#else
+# include <io.h>
+# define WIN32_LEAN_AND_MEAN
+# include <winsock2.h>
+# include <windows.h>
+# ifndef EV_SELECT_IS_WINSOCKET
+# define EV_SELECT_IS_WINSOCKET 1
+# endif
+# undef EV_AVOID_STDIO
+#endif
+
+/* this block tries to deduce configuration from header-defined symbols and defaults */
+
+/* try to deduce the maximum number of signals on this platform */
+#if defined EV_NSIG
+/* use what's provided */
+#elif defined NSIG
+# define EV_NSIG (NSIG)
+#elif defined _NSIG
+# define EV_NSIG (_NSIG)
+#elif defined SIGMAX
+# define EV_NSIG (SIGMAX+1)
+#elif defined SIG_MAX
+# define EV_NSIG (SIG_MAX+1)
+#elif defined _SIG_MAX
+# define EV_NSIG (_SIG_MAX+1)
+#elif defined MAXSIG
+# define EV_NSIG (MAXSIG+1)
+#elif defined MAX_SIG
+# define EV_NSIG (MAX_SIG+1)
+#elif defined SIGARRAYSIZE
+# define EV_NSIG (SIGARRAYSIZE) /* Assume ary[SIGARRAYSIZE] */
+#elif defined _sys_nsig
+# define EV_NSIG (_sys_nsig) /* Solaris 2.5 */
+#else
+# define EV_NSIG (8 * sizeof (sigset_t) + 1)
+#endif
+
+#ifndef EV_USE_FLOOR
+# define EV_USE_FLOOR 0
+#endif
+
+#ifndef EV_USE_CLOCK_SYSCALL
+# if __linux && __GLIBC__ == 2 && __GLIBC_MINOR__ < 17
+# define EV_USE_CLOCK_SYSCALL EV_FEATURE_OS
+# else
+# define EV_USE_CLOCK_SYSCALL 0
+# endif
+#endif
+
+#if !(_POSIX_TIMERS > 0)
+# ifndef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 0
+# endif
+# ifndef EV_USE_REALTIME
+# define EV_USE_REALTIME 0
+# endif
+#endif
+
+#ifndef EV_USE_MONOTONIC
+# if defined _POSIX_MONOTONIC_CLOCK && _POSIX_MONOTONIC_CLOCK >= 0
+# define EV_USE_MONOTONIC EV_FEATURE_OS
+# else
+# define EV_USE_MONOTONIC 0
+# endif
+#endif
+
+#ifndef EV_USE_REALTIME
+# define EV_USE_REALTIME !EV_USE_CLOCK_SYSCALL
+#endif
+
+#ifndef EV_USE_NANOSLEEP
+# if _POSIX_C_SOURCE >= 199309L
+# define EV_USE_NANOSLEEP EV_FEATURE_OS
+# else
+# define EV_USE_NANOSLEEP 0
+# endif
+#endif
+
+#ifndef EV_USE_SELECT
+# define EV_USE_SELECT EV_FEATURE_BACKENDS
+#endif
+
+#ifndef EV_USE_POLL
+# ifdef _WIN32
+# define EV_USE_POLL 0
+# else
+# define EV_USE_POLL EV_FEATURE_BACKENDS
+# endif
+#endif
+
+#ifndef EV_USE_EPOLL
+# if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 4))
+# define EV_USE_EPOLL EV_FEATURE_BACKENDS
+# else
+# define EV_USE_EPOLL 0
+# endif
+#endif
+
+#ifndef EV_USE_KQUEUE
+# define EV_USE_KQUEUE 0
+#endif
+
+#ifndef EV_USE_PORT
+# define EV_USE_PORT 0
+#endif
+
+#ifndef EV_USE_LINUXAIO
+# if __linux /* libev currently assumes linux/aio_abi.h is always available on linux */
+# define EV_USE_LINUXAIO 0 /* was: 1, always off by default */
+# else
+# define EV_USE_LINUXAIO 0
+# endif
+#endif
+
+#ifndef EV_USE_IOURING
+# if __linux /* later checks might disable again */
+# define EV_USE_IOURING 1
+# else
+# define EV_USE_IOURING 0
+# endif
+#endif
+
+#ifndef EV_USE_INOTIFY
+# if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 4))
+# define EV_USE_INOTIFY EV_FEATURE_OS
+# else
+# define EV_USE_INOTIFY 0
+# endif
+#endif
+
+#ifndef EV_PID_HASHSIZE
+# define EV_PID_HASHSIZE EV_FEATURE_DATA ? 16 : 1
+#endif
+
+#ifndef EV_INOTIFY_HASHSIZE
+# define EV_INOTIFY_HASHSIZE EV_FEATURE_DATA ? 16 : 1
+#endif
+
+#ifndef EV_USE_EVENTFD
+# if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 7))
+# define EV_USE_EVENTFD EV_FEATURE_OS
+# else
+# define EV_USE_EVENTFD 0
+# endif
+#endif
+
+#ifndef EV_USE_SIGNALFD
+# if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 7))
+# define EV_USE_SIGNALFD EV_FEATURE_OS
+# else
+# define EV_USE_SIGNALFD 0
+# endif
+#endif
+
+#ifndef EV_USE_TIMERFD
+# if __linux && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 8))
+# define EV_USE_TIMERFD EV_FEATURE_OS
+# else
+# define EV_USE_TIMERFD 0
+# endif
+#endif
+
+#if 0 /* debugging */
+# define EV_VERIFY 3
+# define EV_USE_4HEAP 1
+# define EV_HEAP_CACHE_AT 1
+#endif
+
+#ifndef EV_VERIFY
+# define EV_VERIFY (EV_FEATURE_API ? 1 : 0)
+#endif
+
+#ifndef EV_USE_4HEAP
+# define EV_USE_4HEAP EV_FEATURE_DATA
+#endif
+
+#ifndef EV_HEAP_CACHE_AT
+# define EV_HEAP_CACHE_AT EV_FEATURE_DATA
+#endif
+
+#ifdef __ANDROID__
+/* supposedly, android doesn't typedef fd_mask */
+# undef EV_USE_SELECT
+# define EV_USE_SELECT 0
+/* supposedly, we need to include syscall.h, not sys/syscall.h, so just disable */
+# undef EV_USE_CLOCK_SYSCALL
+# define EV_USE_CLOCK_SYSCALL 0
+#endif
+
+/* aix's poll.h seems to cause lots of trouble */
+#ifdef _AIX
+/* AIX has a completely broken poll.h header */
+# undef EV_USE_POLL
+# define EV_USE_POLL 0
+#endif
+
+/* on linux, we can use a (slow) syscall to avoid a dependency on pthread, */
+/* which makes programs even slower. might work on other unices, too. */
+#if EV_USE_CLOCK_SYSCALL
+# include <sys/syscall.h>
+# ifdef SYS_clock_gettime
+# define clock_gettime(id, ts) syscall (SYS_clock_gettime, (id), (ts))
+# undef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 1
+# define EV_NEED_SYSCALL 1
+# else
+# undef EV_USE_CLOCK_SYSCALL
+# define EV_USE_CLOCK_SYSCALL 0
+# endif
+#endif
+
+/* this block fixes any misconfiguration where we know we run into trouble otherwise */
+
+#ifndef CLOCK_MONOTONIC
+# undef EV_USE_MONOTONIC
+# define EV_USE_MONOTONIC 0
+#endif
+
+#ifndef CLOCK_REALTIME
+# undef EV_USE_REALTIME
+# define EV_USE_REALTIME 0
+#endif
+
+#if !EV_STAT_ENABLE
+# undef EV_USE_INOTIFY
+# define EV_USE_INOTIFY 0
+#endif
+
+#if __linux && EV_USE_IOURING
+# include <linux/version.h>
+# if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
+# undef EV_USE_IOURING
+# define EV_USE_IOURING 0
+# endif
+#endif
+
+#if !EV_USE_NANOSLEEP
+/* hp-ux has it in sys/time.h, which we unconditionally include above */
+# if !defined _WIN32 && !defined __hpux
+# include <sys/select.h>
+# endif
+#endif
+
+#if EV_USE_LINUXAIO
+# include <sys/syscall.h>
+# if SYS_io_getevents && EV_USE_EPOLL /* linuxaio backend requires epoll backend */
+# define EV_NEED_SYSCALL 1
+# else
+# undef EV_USE_LINUXAIO
+# define EV_USE_LINUXAIO 0
+# endif
+#endif
+
+#if EV_USE_IOURING
+# include <sys/syscall.h>
+# if !SYS_io_uring_register && __linux && !__alpha
+# define SYS_io_uring_setup 425
+# define SYS_io_uring_enter 426
+# define SYS_io_uring_register 427
+# endif
+# if SYS_io_uring_setup && EV_USE_EPOLL /* iouring backend requires epoll backend */
+# define EV_NEED_SYSCALL 1
+# else
+# undef EV_USE_IOURING
+# define EV_USE_IOURING 0
+# endif
+#endif
+
+#if EV_USE_INOTIFY
+# include <sys/statfs.h>
+# include <sys/inotify.h>
+/* some very old inotify.h headers don't have IN_DONT_FOLLOW */
+# ifndef IN_DONT_FOLLOW
+# undef EV_USE_INOTIFY
+# define EV_USE_INOTIFY 0
+# endif
+#endif
+
+#if EV_USE_EVENTFD
+/* our minimum requirement is glibc 2.7 which has the stub, but not the full header */
+# include <stdint.h>
+# ifndef EFD_NONBLOCK
+# define EFD_NONBLOCK O_NONBLOCK
+# endif
+# ifndef EFD_CLOEXEC
+# ifdef O_CLOEXEC
+# define EFD_CLOEXEC O_CLOEXEC
+# else
+# define EFD_CLOEXEC 02000000
+# endif
+# endif
+EV_CPP(extern "C") int (eventfd) (unsigned int initval, int flags);
+#endif
+
+#if EV_USE_SIGNALFD
+/* our minimum requirement is glibc 2.7 which has the stub, but not the full header */
+# include <stdint.h>
+# ifndef SFD_NONBLOCK
+# define SFD_NONBLOCK O_NONBLOCK
+# endif
+# ifndef SFD_CLOEXEC
+# ifdef O_CLOEXEC
+# define SFD_CLOEXEC O_CLOEXEC
+# else
+# define SFD_CLOEXEC 02000000
+# endif
+# endif
+EV_CPP (extern "C") int (signalfd) (int fd, const sigset_t *mask, int flags);
+
+struct signalfd_siginfo
+{
+ uint32_t ssi_signo;
+ char pad[128 - sizeof (uint32_t)];
+};
+#endif
+
+/* for timerfd, libev core requires TFD_TIMER_CANCEL_ON_SET &c */
+#if EV_USE_TIMERFD
+# include <sys/timerfd.h>
+/* timerfd is only used for periodics */
+# if !(defined (TFD_TIMER_CANCEL_ON_SET) && defined (TFD_CLOEXEC) && defined (TFD_NONBLOCK)) || !EV_PERIODIC_ENABLE
+# undef EV_USE_TIMERFD
+# define EV_USE_TIMERFD 0
+# endif
+#endif
+
+/*****************************************************************************/
+
+#if EV_VERIFY >= 3
+# define EV_FREQUENT_CHECK ev_verify (EV_A)
+#else
+# define EV_FREQUENT_CHECK do { } while (0)
+#endif
+
+/*
+ * This is used to work around floating point rounding problems.
+ * This value is good at least till the year 4000.
+ */
+#define MIN_INTERVAL 0.0001220703125 /* 1/2**13, good till 4000 */
+// #define MIN_INTERVAL 0.00000095367431640625 /* 1/2**20, good till 2200 */
+
+#define MIN_TIMEJUMP 1. /* minimum timejump that gets detected (if monotonic clock available) */
+#define MAX_BLOCKTIME 59.743 /* never wait longer than this time (to detect time jumps) */
+#define MAX_BLOCKTIME2 1500001.07 /* same, but when timerfd is used to detect jumps, also safe delay to not overflow */
+
+/* find a portable timestamp that is "always" in the future but fits into time_t.
+ * this is quite hard, and we are mostly guessing - we handle 32 bit signed/unsigned time_t,
+ * and sizes larger than 32 bit, and maybe the unlikely floating point time_t */
+#define EV_TSTAMP_HUGE \
+ (sizeof (time_t) >= 8 ? 10000000000000. \
+ : 0 < (time_t)4294967295 ? 4294967295. \
+ : 2147483647.) \
+
+#ifndef EV_TS_CONST
+# define EV_TS_CONST(nv) nv
+# define EV_TS_TO_MSEC(a) a * 1e3 + 0.9999
+# define EV_TS_FROM_USEC(us) us * 1e-6
+# define EV_TV_SET(tv,t) do { tv.tv_sec = (long)t; tv.tv_usec = (long)((t - tv.tv_sec) * 1e6); } while (0)
+# define EV_TS_SET(ts,t) do { ts.tv_sec = (long)t; ts.tv_nsec = (long)((t - ts.tv_sec) * 1e9); } while (0)
+# define EV_TV_GET(tv) ((tv).tv_sec + (tv).tv_usec * 1e-6)
+# define EV_TS_GET(ts) ((ts).tv_sec + (ts).tv_nsec * 1e-9)
+#endif
+
+/* the following is ecb.h embedded into libev - use update_ev_c to update from an external copy */
+/* ECB.H BEGIN */
+/*
+ * libecb - http://software.schmorp.de/pkg/libecb
+ *
+ * Copyright (©) 2009-2015,2018-2020 Marc Alexander Lehmann <libecb@schmorp.de>
+ * Copyright (©) 2011 Emanuele Giaquinta
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#ifndef ECB_H
+#define ECB_H
+
+/* 16 bits major, 16 bits minor */
+#define ECB_VERSION 0x00010008
+
+#include <string.h> /* for memcpy */
+
+#if defined (_WIN32) && !defined (__MINGW32__)
+ typedef signed char int8_t;
+ typedef unsigned char uint8_t;
+ typedef signed char int_fast8_t;
+ typedef unsigned char uint_fast8_t;
+ typedef signed short int16_t;
+ typedef unsigned short uint16_t;
+ typedef signed int int_fast16_t;
+ typedef unsigned int uint_fast16_t;
+ typedef signed int int32_t;
+ typedef unsigned int uint32_t;
+ typedef signed int int_fast32_t;
+ typedef unsigned int uint_fast32_t;
+ #if __GNUC__
+ typedef signed long long int64_t;
+ typedef unsigned long long uint64_t;
+ #else /* _MSC_VER || __BORLANDC__ */
+ typedef signed __int64 int64_t;
+ typedef unsigned __int64 uint64_t;
+ #endif
+ typedef int64_t int_fast64_t;
+ typedef uint64_t uint_fast64_t;
+ #ifdef _WIN64
+ #define ECB_PTRSIZE 8
+ typedef uint64_t uintptr_t;
+ typedef int64_t intptr_t;
+ #else
+ #define ECB_PTRSIZE 4
+ typedef uint32_t uintptr_t;
+ typedef int32_t intptr_t;
+ #endif
+#else
+ #include <inttypes.h>
+ #if (defined INTPTR_MAX ? INTPTR_MAX : ULONG_MAX) > 0xffffffffU
+ #define ECB_PTRSIZE 8
+ #else
+ #define ECB_PTRSIZE 4
+ #endif
+#endif
+
+#define ECB_GCC_AMD64 (__amd64 || __amd64__ || __x86_64 || __x86_64__)
+#define ECB_MSVC_AMD64 (_M_AMD64 || _M_X64)
+
+#ifndef ECB_OPTIMIZE_SIZE
+ #if __OPTIMIZE_SIZE__
+ #define ECB_OPTIMIZE_SIZE 1
+ #else
+ #define ECB_OPTIMIZE_SIZE 0
+ #endif
+#endif
+
+/* work around x32 idiocy by defining proper macros */
+#if ECB_GCC_AMD64 || ECB_MSVC_AMD64
+ #if _ILP32
+ #define ECB_AMD64_X32 1
+ #else
+ #define ECB_AMD64 1
+ #endif
+#endif
+
+/* many compilers define _GNUC_ to some versions but then only implement
+ * what their idiot authors think are the "more important" extensions,
+ * causing enormous grief in return for some better fake benchmark numbers.
+ * or so.
+ * we try to detect these and simply assume they are not gcc - if they have
+ * an issue with that they should have done it right in the first place.
+ */
+#if !defined __GNUC_MINOR__ || defined __INTEL_COMPILER || defined __SUNPRO_C || defined __SUNPRO_CC || defined __llvm__ || defined __clang__
+ #define ECB_GCC_VERSION(major,minor) 0
+#else
+ #define ECB_GCC_VERSION(major,minor) (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#endif
+
+#define ECB_CLANG_VERSION(major,minor) (__clang_major__ > (major) || (__clang_major__ == (major) && __clang_minor__ >= (minor)))
+
+#if __clang__ && defined __has_builtin
+ #define ECB_CLANG_BUILTIN(x) __has_builtin (x)
+#else
+ #define ECB_CLANG_BUILTIN(x) 0
+#endif
+
+#if __clang__ && defined __has_extension
+ #define ECB_CLANG_EXTENSION(x) __has_extension (x)
+#else
+ #define ECB_CLANG_EXTENSION(x) 0
+#endif
+
+#define ECB_CPP (__cplusplus+0)
+#define ECB_CPP11 (__cplusplus >= 201103L)
+#define ECB_CPP14 (__cplusplus >= 201402L)
+#define ECB_CPP17 (__cplusplus >= 201703L)
+
+#if ECB_CPP
+ #define ECB_C 0
+ #define ECB_STDC_VERSION 0
+#else
+ #define ECB_C 1
+ #define ECB_STDC_VERSION __STDC_VERSION__
+#endif
+
+#define ECB_C99 (ECB_STDC_VERSION >= 199901L)
+#define ECB_C11 (ECB_STDC_VERSION >= 201112L)
+#define ECB_C17 (ECB_STDC_VERSION >= 201710L)
+
+#if ECB_CPP
+ #define ECB_EXTERN_C extern "C"
+ #define ECB_EXTERN_C_BEG ECB_EXTERN_C {
+ #define ECB_EXTERN_C_END }
+#else
+ #define ECB_EXTERN_C extern
+ #define ECB_EXTERN_C_BEG
+ #define ECB_EXTERN_C_END
+#endif
+
+/*****************************************************************************/
+
+/* ECB_NO_THREADS - ecb is not used by multiple threads, ever */
+/* ECB_NO_SMP - ecb might be used in multiple threads, but only on a single cpu */
+
+#if ECB_NO_THREADS
+ #define ECB_NO_SMP 1
+#endif
+
+#if ECB_NO_SMP
+ #define ECB_MEMORY_FENCE do { } while (0)
+#endif
+
+/* http://www-01.ibm.com/support/knowledgecenter/SSGH3R_13.1.0/com.ibm.xlcpp131.aix.doc/compiler_ref/compiler_builtins.html */
+#if __xlC__ && ECB_CPP
+ #include <builtins.h>
+#endif
+
+#if 1400 <= _MSC_VER
+ #include <intrin.h> /* fence functions _ReadBarrier, also bit search functions _BitScanReverse */
+#endif
+
+#ifndef ECB_MEMORY_FENCE
+ #if ECB_GCC_VERSION(2,5) || defined __INTEL_COMPILER || (__llvm__ && __GNUC__) || __SUNPRO_C >= 0x5110 || __SUNPRO_CC >= 0x5110
+ #define ECB_MEMORY_FENCE_RELAXED __asm__ __volatile__ ("" : : : "memory")
+ #if __i386 || __i386__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("lock; orb $0, -1(%%esp)" : : : "memory")
+ #define ECB_MEMORY_FENCE_ACQUIRE __asm__ __volatile__ ("" : : : "memory")
+ #define ECB_MEMORY_FENCE_RELEASE __asm__ __volatile__ ("" : : : "memory")
+ #elif ECB_GCC_AMD64
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("mfence" : : : "memory")
+ #define ECB_MEMORY_FENCE_ACQUIRE __asm__ __volatile__ ("" : : : "memory")
+ #define ECB_MEMORY_FENCE_RELEASE __asm__ __volatile__ ("" : : : "memory")
+ #elif __powerpc__ || __ppc__ || __powerpc64__ || __ppc64__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("sync" : : : "memory")
+ #elif defined __ARM_ARCH_2__ \
+ || defined __ARM_ARCH_3__ || defined __ARM_ARCH_3M__ \
+ || defined __ARM_ARCH_4__ || defined __ARM_ARCH_4T__ \
+ || defined __ARM_ARCH_5__ || defined __ARM_ARCH_5E__ \
+ || defined __ARM_ARCH_5T__ || defined __ARM_ARCH_5TE__ \
+ || defined __ARM_ARCH_5TEJ__
+ /* should not need any, unless running old code on newer cpu - arm doesn't support that */
+ #elif defined __ARM_ARCH_6__ || defined __ARM_ARCH_6J__ \
+ || defined __ARM_ARCH_6K__ || defined __ARM_ARCH_6ZK__ \
+ || defined __ARM_ARCH_6T2__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("mcr p15,0,%0,c7,c10,5" : : "r" (0) : "memory")
+ #elif defined __ARM_ARCH_7__ || defined __ARM_ARCH_7A__ \
+ || defined __ARM_ARCH_7R__ || defined __ARM_ARCH_7M__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("dmb" : : : "memory")
+ #elif __aarch64__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("dmb ish" : : : "memory")
+ #elif (__sparc || __sparc__) && !(__sparc_v8__ || defined __sparcv8)
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("membar #LoadStore | #LoadLoad | #StoreStore | #StoreLoad" : : : "memory")
+ #define ECB_MEMORY_FENCE_ACQUIRE __asm__ __volatile__ ("membar #LoadStore | #LoadLoad" : : : "memory")
+ #define ECB_MEMORY_FENCE_RELEASE __asm__ __volatile__ ("membar #LoadStore | #StoreStore")
+ #elif defined __s390__ || defined __s390x__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("bcr 15,0" : : : "memory")
+ #elif defined __mips__
+ /* GNU/Linux emulates sync on mips1 architectures, so we force its use */
+ /* anybody else who still uses mips1 is supposed to send in their version, with detection code. */
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ (".set mips2; sync; .set mips0" : : : "memory")
+ #elif defined __alpha__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("mb" : : : "memory")
+ #elif defined __hppa__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("" : : : "memory")
+ #define ECB_MEMORY_FENCE_RELEASE __asm__ __volatile__ ("")
+ #elif defined __ia64__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("mf" : : : "memory")
+ #elif defined __m68k__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("" : : : "memory")
+ #elif defined __m88k__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("tb1 0,%%r0,128" : : : "memory")
+ #elif defined __sh__
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("" : : : "memory")
+ #elif defined __loongarch__ || __loongarch64
+ #define ECB_MEMORY_FENCE __asm__ __volatile__ ("dbar %0 ": : "I"(0) : "memory")
+ #endif
+ #endif
+#endif
+
+#ifndef ECB_MEMORY_FENCE
+ #if ECB_GCC_VERSION(4,7)
+ /* see comment below (stdatomic.h) about the C11 memory model. */
+ #define ECB_MEMORY_FENCE __atomic_thread_fence (__ATOMIC_SEQ_CST)
+ #define ECB_MEMORY_FENCE_ACQUIRE __atomic_thread_fence (__ATOMIC_ACQUIRE)
+ #define ECB_MEMORY_FENCE_RELEASE __atomic_thread_fence (__ATOMIC_RELEASE)
+ #define ECB_MEMORY_FENCE_RELAXED __atomic_thread_fence (__ATOMIC_RELAXED)
+
+ #elif ECB_CLANG_EXTENSION(c_atomic)
+ /* see comment below (stdatomic.h) about the C11 memory model. */
+ #define ECB_MEMORY_FENCE __c11_atomic_thread_fence (__ATOMIC_SEQ_CST)
+ #define ECB_MEMORY_FENCE_ACQUIRE __c11_atomic_thread_fence (__ATOMIC_ACQUIRE)
+ #define ECB_MEMORY_FENCE_RELEASE __c11_atomic_thread_fence (__ATOMIC_RELEASE)
+ #define ECB_MEMORY_FENCE_RELAXED __c11_atomic_thread_fence (__ATOMIC_RELAXED)
+
+ #elif ECB_GCC_VERSION(4,4) || defined __INTEL_COMPILER || defined __clang__
+ #define ECB_MEMORY_FENCE __sync_synchronize ()
+ #elif _MSC_VER >= 1500 /* VC++ 2008 */
+ /* apparently, microsoft broke all the memory barrier stuff in Visual Studio 2008... */
+ #pragma intrinsic(_ReadBarrier,_WriteBarrier,_ReadWriteBarrier)
+ #define ECB_MEMORY_FENCE _ReadWriteBarrier (); MemoryBarrier()
+ #define ECB_MEMORY_FENCE_ACQUIRE _ReadWriteBarrier (); MemoryBarrier() /* according to msdn, _ReadBarrier is not a load fence */
+ #define ECB_MEMORY_FENCE_RELEASE _WriteBarrier (); MemoryBarrier()
+ #elif _MSC_VER >= 1400 /* VC++ 2005 */
+ #pragma intrinsic(_ReadBarrier,_WriteBarrier,_ReadWriteBarrier)
+ #define ECB_MEMORY_FENCE _ReadWriteBarrier ()
+ #define ECB_MEMORY_FENCE_ACQUIRE _ReadWriteBarrier () /* according to msdn, _ReadBarrier is not a load fence */
+ #define ECB_MEMORY_FENCE_RELEASE _WriteBarrier ()
+ #elif defined _WIN32
+ #include <WinNT.h>
+ #define ECB_MEMORY_FENCE MemoryBarrier () /* actually just xchg on x86... scary */
+ #elif __SUNPRO_C >= 0x5110 || __SUNPRO_CC >= 0x5110
+ #include <mbarrier.h>
+ #define ECB_MEMORY_FENCE __machine_rw_barrier ()
+ #define ECB_MEMORY_FENCE_ACQUIRE __machine_acq_barrier ()
+ #define ECB_MEMORY_FENCE_RELEASE __machine_rel_barrier ()
+ #define ECB_MEMORY_FENCE_RELAXED __compiler_barrier ()
+ #elif __xlC__
+ #define ECB_MEMORY_FENCE __sync ()
+ #endif
+#endif
+
+#ifndef ECB_MEMORY_FENCE
+ #if ECB_C11 && !defined __STDC_NO_ATOMICS__
+ /* we assume that these memory fences work on all variables/all memory accesses, */
+ /* not just C11 atomics and atomic accesses */
+ #include <stdatomic.h>
+ #define ECB_MEMORY_FENCE atomic_thread_fence (memory_order_seq_cst)
+ #define ECB_MEMORY_FENCE_ACQUIRE atomic_thread_fence (memory_order_acquire)
+ #define ECB_MEMORY_FENCE_RELEASE atomic_thread_fence (memory_order_release)
+ #endif
+#endif
+
+#ifndef ECB_MEMORY_FENCE
+ #if !ECB_AVOID_PTHREADS
+ /*
+ * if you get undefined symbol references to pthread_mutex_lock,
+ * or failure to find pthread.h, then you should implement
+ * the ECB_MEMORY_FENCE operations for your cpu/compiler
+ * OR provide pthread.h and link against the posix thread library
+ * of your system.
+ */
+ #include <pthread.h>
+ #define ECB_NEEDS_PTHREADS 1
+ #define ECB_MEMORY_FENCE_NEEDS_PTHREADS 1
+
+ static pthread_mutex_t ecb_mf_lock = PTHREAD_MUTEX_INITIALIZER;
+ #define ECB_MEMORY_FENCE do { pthread_mutex_lock (&ecb_mf_lock); pthread_mutex_unlock (&ecb_mf_lock); } while (0)
+ #endif
+#endif
+
+#if !defined ECB_MEMORY_FENCE_ACQUIRE && defined ECB_MEMORY_FENCE
+ #define ECB_MEMORY_FENCE_ACQUIRE ECB_MEMORY_FENCE
+#endif
+
+#if !defined ECB_MEMORY_FENCE_RELEASE && defined ECB_MEMORY_FENCE
+ #define ECB_MEMORY_FENCE_RELEASE ECB_MEMORY_FENCE
+#endif
+
+#if !defined ECB_MEMORY_FENCE_RELAXED && defined ECB_MEMORY_FENCE
+ #define ECB_MEMORY_FENCE_RELAXED ECB_MEMORY_FENCE /* very heavy-handed */
+#endif
+
+/*****************************************************************************/
+
+#if ECB_CPP
+ #define ecb_inline static inline
+#elif ECB_GCC_VERSION(2,5)
+ #define ecb_inline static __inline__
+#elif ECB_C99
+ #define ecb_inline static inline
+#else
+ #define ecb_inline static
+#endif
+
+#if ECB_GCC_VERSION(3,3)
+ #define ecb_restrict __restrict__
+#elif ECB_C99
+ #define ecb_restrict restrict
+#else
+ #define ecb_restrict
+#endif
+
+typedef int ecb_bool;
+
+#define ECB_CONCAT_(a, b) a ## b
+#define ECB_CONCAT(a, b) ECB_CONCAT_(a, b)
+#define ECB_STRINGIFY_(a) # a
+#define ECB_STRINGIFY(a) ECB_STRINGIFY_(a)
+#define ECB_STRINGIFY_EXPR(expr) ((expr), ECB_STRINGIFY_ (expr))
+
+#define ecb_function_ ecb_inline
+
+#if ECB_GCC_VERSION(3,1) || ECB_CLANG_VERSION(2,8)
+ #define ecb_attribute(attrlist) __attribute__ (attrlist)
+#else
+ #define ecb_attribute(attrlist)
+#endif
+
+#if ECB_GCC_VERSION(3,1) || ECB_CLANG_BUILTIN(__builtin_constant_p)
+ #define ecb_is_constant(expr) __builtin_constant_p (expr)
+#else
+ /* possible C11 impl for integral types
+ typedef struct ecb_is_constant_struct ecb_is_constant_struct;
+ #define ecb_is_constant(expr) _Generic ((1 ? (struct ecb_is_constant_struct *)0 : (void *)((expr) - (expr)), ecb_is_constant_struct *: 0, default: 1)) */
+
+ #define ecb_is_constant(expr) 0
+#endif
+
+#if ECB_GCC_VERSION(3,1) || ECB_CLANG_BUILTIN(__builtin_expect)
+ #define ecb_expect(expr,value) __builtin_expect ((expr),(value))
+#else
+ #define ecb_expect(expr,value) (expr)
+#endif
+
+#if ECB_GCC_VERSION(3,1) || ECB_CLANG_BUILTIN(__builtin_prefetch)
+ #define ecb_prefetch(addr,rw,locality) __builtin_prefetch (addr, rw, locality)
+#else
+ #define ecb_prefetch(addr,rw,locality)
+#endif
+
+/* no emulation for ecb_decltype */
+#if ECB_CPP11
+ // older implementations might have problems with decltype(x)::type, work around it
+ template<class T> struct ecb_decltype_t { typedef T type; };
+ #define ecb_decltype(x) ecb_decltype_t<decltype (x)>::type
+#elif ECB_GCC_VERSION(3,0) || ECB_CLANG_VERSION(2,8)
+ #define ecb_decltype(x) __typeof__ (x)
+#endif
+
+#if _MSC_VER >= 1300
+ #define ecb_deprecated __declspec (deprecated)
+#else
+ #define ecb_deprecated ecb_attribute ((__deprecated__))
+#endif
+
+#if _MSC_VER >= 1500
+ #define ecb_deprecated_message(msg) __declspec (deprecated (msg))
+#elif ECB_GCC_VERSION(4,5)
+ #define ecb_deprecated_message(msg) ecb_attribute ((__deprecated__ (msg))
+#else
+ #define ecb_deprecated_message(msg) ecb_deprecated
+#endif
+
+#if _MSC_VER >= 1400
+ #define ecb_noinline __declspec (noinline)
+#else
+ #define ecb_noinline ecb_attribute ((__noinline__))
+#endif
+
+#define ecb_unused ecb_attribute ((__unused__))
+#define ecb_const ecb_attribute ((__const__))
+#define ecb_pure ecb_attribute ((__pure__))
+
+#if ECB_C11 || __IBMC_NORETURN
+ /* http://www-01.ibm.com/support/knowledgecenter/SSGH3R_13.1.0/com.ibm.xlcpp131.aix.doc/language_ref/noreturn.html */
+ #define ecb_noreturn _Noreturn
+#elif ECB_CPP11
+ #define ecb_noreturn [[noreturn]]
+#elif _MSC_VER >= 1200
+ /* http://msdn.microsoft.com/en-us/library/k6ktzx3s.aspx */
+ #define ecb_noreturn __declspec (noreturn)
+#else
+ #define ecb_noreturn ecb_attribute ((__noreturn__))
+#endif
+
+#if ECB_GCC_VERSION(4,3)
+ #define ecb_artificial ecb_attribute ((__artificial__))
+ #define ecb_hot ecb_attribute ((__hot__))
+ #define ecb_cold ecb_attribute ((__cold__))
+#else
+ #define ecb_artificial
+ #define ecb_hot
+ #define ecb_cold
+#endif
+
+/* put around conditional expressions if you are very sure that the */
+/* expression is mostly true or mostly false. note that these return */
+/* booleans, not the expression. */
+#define ecb_expect_false(expr) ecb_expect (!!(expr), 0)
+#define ecb_expect_true(expr) ecb_expect (!!(expr), 1)
+/* for compatibility to the rest of the world */
+#define ecb_likely(expr) ecb_expect_true (expr)
+#define ecb_unlikely(expr) ecb_expect_false (expr)
+
+/* count trailing zero bits and count # of one bits */
+#if ECB_GCC_VERSION(3,4) \
+ || (ECB_CLANG_BUILTIN(__builtin_clz) && ECB_CLANG_BUILTIN(__builtin_clzll) \
+ && ECB_CLANG_BUILTIN(__builtin_ctz) && ECB_CLANG_BUILTIN(__builtin_ctzll) \
+ && ECB_CLANG_BUILTIN(__builtin_popcount))
+ /* we assume int == 32 bit, long == 32 or 64 bit and long long == 64 bit */
+ #define ecb_ld32(x) (__builtin_clz (x) ^ 31)
+ #define ecb_ld64(x) (__builtin_clzll (x) ^ 63)
+ #define ecb_ctz32(x) __builtin_ctz (x)
+ #define ecb_ctz64(x) __builtin_ctzll (x)
+ #define ecb_popcount32(x) __builtin_popcount (x)
+ /* no popcountll */
+#else
+ ecb_function_ ecb_const int ecb_ctz32 (uint32_t x);
+ ecb_function_ ecb_const int
+ ecb_ctz32 (uint32_t x)
+ {
+#if 1400 <= _MSC_VER && (_M_IX86 || _M_X64 || _M_IA64 || _M_ARM)
+ unsigned long r;
+ _BitScanForward (&r, x);
+ return (int)r;
+#else
+ int r = 0;
+
+ x &= ~x + 1; /* this isolates the lowest bit */
+
+#if ECB_branchless_on_i386
+ r += !!(x & 0xaaaaaaaa) << 0;
+ r += !!(x & 0xcccccccc) << 1;
+ r += !!(x & 0xf0f0f0f0) << 2;
+ r += !!(x & 0xff00ff00) << 3;
+ r += !!(x & 0xffff0000) << 4;
+#else
+ if (x & 0xaaaaaaaa) r += 1;
+ if (x & 0xcccccccc) r += 2;
+ if (x & 0xf0f0f0f0) r += 4;
+ if (x & 0xff00ff00) r += 8;
+ if (x & 0xffff0000) r += 16;
+#endif
+
+ return r;
+#endif
+ }
+
+ ecb_function_ ecb_const int ecb_ctz64 (uint64_t x);
+ ecb_function_ ecb_const int
+ ecb_ctz64 (uint64_t x)
+ {
+#if 1400 <= _MSC_VER && (_M_X64 || _M_IA64 || _M_ARM)
+ unsigned long r;
+ _BitScanForward64 (&r, x);
+ return (int)r;
+#else
+ int shift = x & 0xffffffff ? 0 : 32;
+ return ecb_ctz32 (x >> shift) + shift;
+#endif
+ }
+
+ ecb_function_ ecb_const int ecb_popcount32 (uint32_t x);
+ ecb_function_ ecb_const int
+ ecb_popcount32 (uint32_t x)
+ {
+ x -= (x >> 1) & 0x55555555;
+ x = ((x >> 2) & 0x33333333) + (x & 0x33333333);
+ x = ((x >> 4) + x) & 0x0f0f0f0f;
+ x *= 0x01010101;
+
+ return x >> 24;
+ }
+
+ ecb_function_ ecb_const int ecb_ld32 (uint32_t x);
+ ecb_function_ ecb_const int ecb_ld32 (uint32_t x)
+ {
+#if 1400 <= _MSC_VER && (_M_IX86 || _M_X64 || _M_IA64 || _M_ARM)
+ unsigned long r;
+ _BitScanReverse (&r, x);
+ return (int)r;
+#else
+ int r = 0;
+
+ if (x >> 16) { x >>= 16; r += 16; }
+ if (x >> 8) { x >>= 8; r += 8; }
+ if (x >> 4) { x >>= 4; r += 4; }
+ if (x >> 2) { x >>= 2; r += 2; }
+ if (x >> 1) { r += 1; }
+
+ return r;
+#endif
+ }
+
+ ecb_function_ ecb_const int ecb_ld64 (uint64_t x);
+ ecb_function_ ecb_const int ecb_ld64 (uint64_t x)
+ {
+#if 1400 <= _MSC_VER && (_M_X64 || _M_IA64 || _M_ARM)
+ unsigned long r;
+ _BitScanReverse64 (&r, x);
+ return (int)r;
+#else
+ int r = 0;
+
+ if (x >> 32) { x >>= 32; r += 32; }
+
+ return r + ecb_ld32 (x);
+#endif
+ }
+#endif
+
+ecb_function_ ecb_const ecb_bool ecb_is_pot32 (uint32_t x);
+ecb_function_ ecb_const ecb_bool ecb_is_pot32 (uint32_t x) { return !(x & (x - 1)); }
+ecb_function_ ecb_const ecb_bool ecb_is_pot64 (uint64_t x);
+ecb_function_ ecb_const ecb_bool ecb_is_pot64 (uint64_t x) { return !(x & (x - 1)); }
+
+ecb_function_ ecb_const uint8_t ecb_bitrev8 (uint8_t x);
+ecb_function_ ecb_const uint8_t ecb_bitrev8 (uint8_t x)
+{
+ return ( (x * 0x0802U & 0x22110U)
+ | (x * 0x8020U & 0x88440U)) * 0x10101U >> 16;
+}
+
+ecb_function_ ecb_const uint16_t ecb_bitrev16 (uint16_t x);
+ecb_function_ ecb_const uint16_t ecb_bitrev16 (uint16_t x)
+{
+ x = ((x >> 1) & 0x5555) | ((x & 0x5555) << 1);
+ x = ((x >> 2) & 0x3333) | ((x & 0x3333) << 2);
+ x = ((x >> 4) & 0x0f0f) | ((x & 0x0f0f) << 4);
+ x = ( x >> 8 ) | ( x << 8);
+
+ return x;
+}
+
+ecb_function_ ecb_const uint32_t ecb_bitrev32 (uint32_t x);
+ecb_function_ ecb_const uint32_t ecb_bitrev32 (uint32_t x)
+{
+ x = ((x >> 1) & 0x55555555) | ((x & 0x55555555) << 1);
+ x = ((x >> 2) & 0x33333333) | ((x & 0x33333333) << 2);
+ x = ((x >> 4) & 0x0f0f0f0f) | ((x & 0x0f0f0f0f) << 4);
+ x = ((x >> 8) & 0x00ff00ff) | ((x & 0x00ff00ff) << 8);
+ x = ( x >> 16 ) | ( x << 16);
+
+ return x;
+}
+
+/* popcount64 is only available on 64 bit cpus as gcc builtin */
+/* so for this version we are lazy */
+ecb_function_ ecb_const int ecb_popcount64 (uint64_t x);
+ecb_function_ ecb_const int
+ecb_popcount64 (uint64_t x)
+{
+ return ecb_popcount32 (x) + ecb_popcount32 (x >> 32);
+}
+
+ecb_inline ecb_const uint8_t ecb_rotl8 (uint8_t x, unsigned int count);
+ecb_inline ecb_const uint8_t ecb_rotr8 (uint8_t x, unsigned int count);
+ecb_inline ecb_const uint16_t ecb_rotl16 (uint16_t x, unsigned int count);
+ecb_inline ecb_const uint16_t ecb_rotr16 (uint16_t x, unsigned int count);
+ecb_inline ecb_const uint32_t ecb_rotl32 (uint32_t x, unsigned int count);
+ecb_inline ecb_const uint32_t ecb_rotr32 (uint32_t x, unsigned int count);
+ecb_inline ecb_const uint64_t ecb_rotl64 (uint64_t x, unsigned int count);
+ecb_inline ecb_const uint64_t ecb_rotr64 (uint64_t x, unsigned int count);
+
+ecb_inline ecb_const uint8_t ecb_rotl8 (uint8_t x, unsigned int count) { return (x >> ( 8 - count)) | (x << count); }
+ecb_inline ecb_const uint8_t ecb_rotr8 (uint8_t x, unsigned int count) { return (x << ( 8 - count)) | (x >> count); }
+ecb_inline ecb_const uint16_t ecb_rotl16 (uint16_t x, unsigned int count) { return (x >> (16 - count)) | (x << count); }
+ecb_inline ecb_const uint16_t ecb_rotr16 (uint16_t x, unsigned int count) { return (x << (16 - count)) | (x >> count); }
+ecb_inline ecb_const uint32_t ecb_rotl32 (uint32_t x, unsigned int count) { return (x >> (32 - count)) | (x << count); }
+ecb_inline ecb_const uint32_t ecb_rotr32 (uint32_t x, unsigned int count) { return (x << (32 - count)) | (x >> count); }
+ecb_inline ecb_const uint64_t ecb_rotl64 (uint64_t x, unsigned int count) { return (x >> (64 - count)) | (x << count); }
+ecb_inline ecb_const uint64_t ecb_rotr64 (uint64_t x, unsigned int count) { return (x << (64 - count)) | (x >> count); }
+
+#if ECB_CPP
+
+inline uint8_t ecb_ctz (uint8_t v) { return ecb_ctz32 (v); }
+inline uint16_t ecb_ctz (uint16_t v) { return ecb_ctz32 (v); }
+inline uint32_t ecb_ctz (uint32_t v) { return ecb_ctz32 (v); }
+inline uint64_t ecb_ctz (uint64_t v) { return ecb_ctz64 (v); }
+
+inline bool ecb_is_pot (uint8_t v) { return ecb_is_pot32 (v); }
+inline bool ecb_is_pot (uint16_t v) { return ecb_is_pot32 (v); }
+inline bool ecb_is_pot (uint32_t v) { return ecb_is_pot32 (v); }
+inline bool ecb_is_pot (uint64_t v) { return ecb_is_pot64 (v); }
+
+inline int ecb_ld (uint8_t v) { return ecb_ld32 (v); }
+inline int ecb_ld (uint16_t v) { return ecb_ld32 (v); }
+inline int ecb_ld (uint32_t v) { return ecb_ld32 (v); }
+inline int ecb_ld (uint64_t v) { return ecb_ld64 (v); }
+
+inline int ecb_popcount (uint8_t v) { return ecb_popcount32 (v); }
+inline int ecb_popcount (uint16_t v) { return ecb_popcount32 (v); }
+inline int ecb_popcount (uint32_t v) { return ecb_popcount32 (v); }
+inline int ecb_popcount (uint64_t v) { return ecb_popcount64 (v); }
+
+inline uint8_t ecb_bitrev (uint8_t v) { return ecb_bitrev8 (v); }
+inline uint16_t ecb_bitrev (uint16_t v) { return ecb_bitrev16 (v); }
+inline uint32_t ecb_bitrev (uint32_t v) { return ecb_bitrev32 (v); }
+
+inline uint8_t ecb_rotl (uint8_t v, unsigned int count) { return ecb_rotl8 (v, count); }
+inline uint16_t ecb_rotl (uint16_t v, unsigned int count) { return ecb_rotl16 (v, count); }
+inline uint32_t ecb_rotl (uint32_t v, unsigned int count) { return ecb_rotl32 (v, count); }
+inline uint64_t ecb_rotl (uint64_t v, unsigned int count) { return ecb_rotl64 (v, count); }
+
+inline uint8_t ecb_rotr (uint8_t v, unsigned int count) { return ecb_rotr8 (v, count); }
+inline uint16_t ecb_rotr (uint16_t v, unsigned int count) { return ecb_rotr16 (v, count); }
+inline uint32_t ecb_rotr (uint32_t v, unsigned int count) { return ecb_rotr32 (v, count); }
+inline uint64_t ecb_rotr (uint64_t v, unsigned int count) { return ecb_rotr64 (v, count); }
+
+#endif
+
+#if ECB_GCC_VERSION(4,3) || (ECB_CLANG_BUILTIN(__builtin_bswap32) && ECB_CLANG_BUILTIN(__builtin_bswap64))
+ #if ECB_GCC_VERSION(4,8) || ECB_CLANG_BUILTIN(__builtin_bswap16)
+ #define ecb_bswap16(x) __builtin_bswap16 (x)
+ #else
+ #define ecb_bswap16(x) (__builtin_bswap32 (x) >> 16)
+ #endif
+ #define ecb_bswap32(x) __builtin_bswap32 (x)
+ #define ecb_bswap64(x) __builtin_bswap64 (x)
+#elif _MSC_VER
+ #include <stdlib.h>
+ #define ecb_bswap16(x) ((uint16_t)_byteswap_ushort ((uint16_t)(x)))
+ #define ecb_bswap32(x) ((uint32_t)_byteswap_ulong ((uint32_t)(x)))
+ #define ecb_bswap64(x) ((uint64_t)_byteswap_uint64 ((uint64_t)(x)))
+#else
+ ecb_function_ ecb_const uint16_t ecb_bswap16 (uint16_t x);
+ ecb_function_ ecb_const uint16_t
+ ecb_bswap16 (uint16_t x)
+ {
+ return ecb_rotl16 (x, 8);
+ }
+
+ ecb_function_ ecb_const uint32_t ecb_bswap32 (uint32_t x);
+ ecb_function_ ecb_const uint32_t
+ ecb_bswap32 (uint32_t x)
+ {
+ return (((uint32_t)ecb_bswap16 (x)) << 16) | ecb_bswap16 (x >> 16);
+ }
+
+ ecb_function_ ecb_const uint64_t ecb_bswap64 (uint64_t x);
+ ecb_function_ ecb_const uint64_t
+ ecb_bswap64 (uint64_t x)
+ {
+ return (((uint64_t)ecb_bswap32 (x)) << 32) | ecb_bswap32 (x >> 32);
+ }
+#endif
+
+#if ECB_GCC_VERSION(4,5) || ECB_CLANG_BUILTIN(__builtin_unreachable)
+ #define ecb_unreachable() __builtin_unreachable ()
+#else
+ /* this seems to work fine, but gcc always emits a warning for it :/ */
+ ecb_inline ecb_noreturn void ecb_unreachable (void);
+ ecb_inline ecb_noreturn void ecb_unreachable (void) { }
+#endif
+
+/* try to tell the compiler that some condition is definitely true */
+#define ecb_assume(cond) if (!(cond)) ecb_unreachable (); else 0
+
+ecb_inline ecb_const uint32_t ecb_byteorder_helper (void);
+ecb_inline ecb_const uint32_t
+ecb_byteorder_helper (void)
+{
+ /* the union code still generates code under pressure in gcc, */
+ /* but less than using pointers, and always seems to */
+ /* successfully return a constant. */
+ /* the reason why we have this horrible preprocessor mess */
+ /* is to avoid it in all cases, at least on common architectures */
+ /* or when using a recent enough gcc version (>= 4.6) */
+#if (defined __BYTE_ORDER__ && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
+ || ((__i386 || __i386__ || _M_IX86 || ECB_GCC_AMD64 || ECB_MSVC_AMD64) && !__VOS__)
+ #define ECB_LITTLE_ENDIAN 1
+ return 0x44332211;
+#elif (defined __BYTE_ORDER__ && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) \
+ || ((__AARCH64EB__ || __MIPSEB__ || __ARMEB__) && !__VOS__)
+ #define ECB_BIG_ENDIAN 1
+ return 0x11223344;
+#else
+ union
+ {
+ uint8_t c[4];
+ uint32_t u;
+ } u = { 0x11, 0x22, 0x33, 0x44 };
+ return u.u;
+#endif
+}
+
+ecb_inline ecb_const ecb_bool ecb_big_endian (void);
+ecb_inline ecb_const ecb_bool ecb_big_endian (void) { return ecb_byteorder_helper () == 0x11223344; }
+ecb_inline ecb_const ecb_bool ecb_little_endian (void);
+ecb_inline ecb_const ecb_bool ecb_little_endian (void) { return ecb_byteorder_helper () == 0x44332211; }
+
+/*****************************************************************************/
+/* unaligned load/store */
+
+ecb_inline uint_fast16_t ecb_be_u16_to_host (uint_fast16_t v) { return ecb_little_endian () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_be_u32_to_host (uint_fast32_t v) { return ecb_little_endian () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_be_u64_to_host (uint_fast64_t v) { return ecb_little_endian () ? ecb_bswap64 (v) : v; }
+
+ecb_inline uint_fast16_t ecb_le_u16_to_host (uint_fast16_t v) { return ecb_big_endian () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_le_u32_to_host (uint_fast32_t v) { return ecb_big_endian () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_le_u64_to_host (uint_fast64_t v) { return ecb_big_endian () ? ecb_bswap64 (v) : v; }
+
+ecb_inline uint_fast16_t ecb_peek_u16_u (const void *ptr) { uint16_t v; memcpy (&v, ptr, sizeof (v)); return v; }
+ecb_inline uint_fast32_t ecb_peek_u32_u (const void *ptr) { uint32_t v; memcpy (&v, ptr, sizeof (v)); return v; }
+ecb_inline uint_fast64_t ecb_peek_u64_u (const void *ptr) { uint64_t v; memcpy (&v, ptr, sizeof (v)); return v; }
+
+ecb_inline uint_fast16_t ecb_peek_be_u16_u (const void *ptr) { return ecb_be_u16_to_host (ecb_peek_u16_u (ptr)); }
+ecb_inline uint_fast32_t ecb_peek_be_u32_u (const void *ptr) { return ecb_be_u32_to_host (ecb_peek_u32_u (ptr)); }
+ecb_inline uint_fast64_t ecb_peek_be_u64_u (const void *ptr) { return ecb_be_u64_to_host (ecb_peek_u64_u (ptr)); }
+
+ecb_inline uint_fast16_t ecb_peek_le_u16_u (const void *ptr) { return ecb_le_u16_to_host (ecb_peek_u16_u (ptr)); }
+ecb_inline uint_fast32_t ecb_peek_le_u32_u (const void *ptr) { return ecb_le_u32_to_host (ecb_peek_u32_u (ptr)); }
+ecb_inline uint_fast64_t ecb_peek_le_u64_u (const void *ptr) { return ecb_le_u64_to_host (ecb_peek_u64_u (ptr)); }
+
+ecb_inline uint_fast16_t ecb_host_to_be_u16 (uint_fast16_t v) { return ecb_little_endian () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_host_to_be_u32 (uint_fast32_t v) { return ecb_little_endian () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_host_to_be_u64 (uint_fast64_t v) { return ecb_little_endian () ? ecb_bswap64 (v) : v; }
+
+ecb_inline uint_fast16_t ecb_host_to_le_u16 (uint_fast16_t v) { return ecb_big_endian () ? ecb_bswap16 (v) : v; }
+ecb_inline uint_fast32_t ecb_host_to_le_u32 (uint_fast32_t v) { return ecb_big_endian () ? ecb_bswap32 (v) : v; }
+ecb_inline uint_fast64_t ecb_host_to_le_u64 (uint_fast64_t v) { return ecb_big_endian () ? ecb_bswap64 (v) : v; }
+
+ecb_inline void ecb_poke_u16_u (void *ptr, uint16_t v) { memcpy (ptr, &v, sizeof (v)); }
+ecb_inline void ecb_poke_u32_u (void *ptr, uint32_t v) { memcpy (ptr, &v, sizeof (v)); }
+ecb_inline void ecb_poke_u64_u (void *ptr, uint64_t v) { memcpy (ptr, &v, sizeof (v)); }
+
+ecb_inline void ecb_poke_be_u16_u (void *ptr, uint_fast16_t v) { ecb_poke_u16_u (ptr, ecb_host_to_be_u16 (v)); }
+ecb_inline void ecb_poke_be_u32_u (void *ptr, uint_fast32_t v) { ecb_poke_u32_u (ptr, ecb_host_to_be_u32 (v)); }
+ecb_inline void ecb_poke_be_u64_u (void *ptr, uint_fast64_t v) { ecb_poke_u64_u (ptr, ecb_host_to_be_u64 (v)); }
+
+ecb_inline void ecb_poke_le_u16_u (void *ptr, uint_fast16_t v) { ecb_poke_u16_u (ptr, ecb_host_to_le_u16 (v)); }
+ecb_inline void ecb_poke_le_u32_u (void *ptr, uint_fast32_t v) { ecb_poke_u32_u (ptr, ecb_host_to_le_u32 (v)); }
+ecb_inline void ecb_poke_le_u64_u (void *ptr, uint_fast64_t v) { ecb_poke_u64_u (ptr, ecb_host_to_le_u64 (v)); }
+
+#if ECB_CPP
+
+inline uint8_t ecb_bswap (uint8_t v) { return v; }
+inline uint16_t ecb_bswap (uint16_t v) { return ecb_bswap16 (v); }
+inline uint32_t ecb_bswap (uint32_t v) { return ecb_bswap32 (v); }
+inline uint64_t ecb_bswap (uint64_t v) { return ecb_bswap64 (v); }
+
+template<typename T> inline T ecb_be_to_host (T v) { return ecb_little_endian () ? ecb_bswap (v) : v; }
+template<typename T> inline T ecb_le_to_host (T v) { return ecb_big_endian () ? ecb_bswap (v) : v; }
+template<typename T> inline T ecb_peek (const void *ptr) { return *(const T *)ptr; }
+template<typename T> inline T ecb_peek_be (const void *ptr) { return ecb_be_to_host (ecb_peek <T> (ptr)); }
+template<typename T> inline T ecb_peek_le (const void *ptr) { return ecb_le_to_host (ecb_peek <T> (ptr)); }
+template<typename T> inline T ecb_peek_u (const void *ptr) { T v; memcpy (&v, ptr, sizeof (v)); return v; }
+template<typename T> inline T ecb_peek_be_u (const void *ptr) { return ecb_be_to_host (ecb_peek_u<T> (ptr)); }
+template<typename T> inline T ecb_peek_le_u (const void *ptr) { return ecb_le_to_host (ecb_peek_u<T> (ptr)); }
+
+template<typename T> inline T ecb_host_to_be (T v) { return ecb_little_endian () ? ecb_bswap (v) : v; }
+template<typename T> inline T ecb_host_to_le (T v) { return ecb_big_endian () ? ecb_bswap (v) : v; }
+template<typename T> inline void ecb_poke (void *ptr, T v) { *(T *)ptr = v; }
+template<typename T> inline void ecb_poke_be (void *ptr, T v) { return ecb_poke <T> (ptr, ecb_host_to_be (v)); }
+template<typename T> inline void ecb_poke_le (void *ptr, T v) { return ecb_poke <T> (ptr, ecb_host_to_le (v)); }
+template<typename T> inline void ecb_poke_u (void *ptr, T v) { memcpy (ptr, &v, sizeof (v)); }
+template<typename T> inline void ecb_poke_be_u (void *ptr, T v) { return ecb_poke_u<T> (ptr, ecb_host_to_be (v)); }
+template<typename T> inline void ecb_poke_le_u (void *ptr, T v) { return ecb_poke_u<T> (ptr, ecb_host_to_le (v)); }
+
+#endif
+
+/*****************************************************************************/
+
+#if ECB_GCC_VERSION(3,0) || ECB_C99
+ #define ecb_mod(m,n) ((m) % (n) + ((m) % (n) < 0 ? (n) : 0))
+#else
+ #define ecb_mod(m,n) ((m) < 0 ? ((n) - 1 - ((-1 - (m)) % (n))) : ((m) % (n)))
+#endif
+
+#if ECB_CPP
+ template<typename T>
+ static inline T ecb_div_rd (T val, T div)
+ {
+ return val < 0 ? - ((-val + div - 1) / div) : (val ) / div;
+ }
+ template<typename T>
+ static inline T ecb_div_ru (T val, T div)
+ {
+ return val < 0 ? - ((-val ) / div) : (val + div - 1) / div;
+ }
+#else
+ #define ecb_div_rd(val,div) ((val) < 0 ? - ((-(val) + (div) - 1) / (div)) : ((val) ) / (div))
+ #define ecb_div_ru(val,div) ((val) < 0 ? - ((-(val) ) / (div)) : ((val) + (div) - 1) / (div))
+#endif
+
+#if ecb_cplusplus_does_not_suck
+ /* does not work for local types (http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2008/n2657.htm) */
+ template<typename T, int N>
+ static inline int ecb_array_length (const T (&arr)[N])
+ {
+ return N;
+ }
+#else
+ #define ecb_array_length(name) (sizeof (name) / sizeof (name [0]))
+#endif
+
+/*****************************************************************************/
+
+ecb_function_ ecb_const uint32_t ecb_binary16_to_binary32 (uint32_t x);
+ecb_function_ ecb_const uint32_t
+ecb_binary16_to_binary32 (uint32_t x)
+{
+ unsigned int s = (x & 0x8000) << (31 - 15);
+ int e = (x >> 10) & 0x001f;
+ unsigned int m = x & 0x03ff;
+
+ if (ecb_expect_false (e == 31))
+ /* infinity or NaN */
+ e = 255 - (127 - 15);
+ else if (ecb_expect_false (!e))
+ {
+ if (ecb_expect_true (!m))
+ /* zero, handled by code below by forcing e to 0 */
+ e = 0 - (127 - 15);
+ else
+ {
+ /* subnormal, renormalise */
+ unsigned int s = 10 - ecb_ld32 (m);
+
+ m = (m << s) & 0x3ff; /* mask implicit bit */
+ e -= s - 1;
+ }
+ }
+
+ /* e and m now are normalised, or zero, (or inf or nan) */
+ e += 127 - 15;
+
+ return s | (e << 23) | (m << (23 - 10));
+}
+
+ecb_function_ ecb_const uint16_t ecb_binary32_to_binary16 (uint32_t x);
+ecb_function_ ecb_const uint16_t
+ecb_binary32_to_binary16 (uint32_t x)
+{
+ unsigned int s = (x >> 16) & 0x00008000; /* sign bit, the easy part */
+ unsigned int e = ((x >> 23) & 0x000000ff) - (127 - 15); /* the desired exponent */
+ unsigned int m = x & 0x007fffff;
+
+ x &= 0x7fffffff;
+
+ /* if it's within range of binary16 normals, use fast path */
+ if (ecb_expect_true (0x38800000 <= x && x <= 0x477fefff))
+ {
+ /* mantissa round-to-even */
+ m += 0x00000fff + ((m >> (23 - 10)) & 1);
+
+ /* handle overflow */
+ if (ecb_expect_false (m >= 0x00800000))
+ {
+ m >>= 1;
+ e += 1;
+ }
+
+ return s | (e << 10) | (m >> (23 - 10));
+ }
+
+ /* handle large numbers and infinity */
+ if (ecb_expect_true (0x477fefff < x && x <= 0x7f800000))
+ return s | 0x7c00;
+
+ /* handle zero, subnormals and small numbers */
+ if (ecb_expect_true (x < 0x38800000))
+ {
+ /* zero */
+ if (ecb_expect_true (!x))
+ return s;
+
+ /* handle subnormals */
+
+ /* too small, will be zero */
+ if (e < (14 - 24)) /* might not be sharp, but is good enough */
+ return s;
+
+ m |= 0x00800000; /* make implicit bit explicit */
+
+ /* very tricky - we need to round to the nearest e (+10) bit value */
+ {
+ unsigned int bits = 14 - e;
+ unsigned int half = (1 << (bits - 1)) - 1;
+ unsigned int even = (m >> bits) & 1;
+
+ /* if this overflows, we will end up with a normalised number */
+ m = (m + half + even) >> bits;
+ }
+
+ return s | m;
+ }
+
+ /* handle NaNs, preserve leftmost nan bits, but make sure we don't turn them into infinities */
+ m >>= 13;
+
+ return s | 0x7c00 | m | !m;
+}
+
+/*******************************************************************************/
+/* floating point stuff, can be disabled by defining ECB_NO_LIBM */
+
+/* basically, everything uses "ieee pure-endian" floating point numbers */
+/* the only noteworthy exception is ancient armle, which uses order 43218765 */
+#if 0 \
+ || __i386 || __i386__ \
+ || ECB_GCC_AMD64 \
+ || __powerpc__ || __ppc__ || __powerpc64__ || __ppc64__ \
+ || defined __s390__ || defined __s390x__ \
+ || defined __mips__ \
+ || defined __alpha__ \
+ || defined __hppa__ \
+ || defined __ia64__ \
+ || defined __m68k__ \
+ || defined __m88k__ \
+ || defined __sh__ \
+ || defined _M_IX86 || defined ECB_MSVC_AMD64 || defined _M_IA64 \
+ || (defined __arm__ && (defined __ARM_EABI__ || defined __EABI__ || defined __VFP_FP__ || defined _WIN32_WCE || defined __ANDROID__)) \
+ || defined __aarch64__
+ #define ECB_STDFP 1
+#else
+ #define ECB_STDFP 0
+#endif
+
+#ifndef ECB_NO_LIBM
+
+ #include <math.h> /* for frexp*, ldexp*, INFINITY, NAN */
+
+ /* only the oldest of old doesn't have this one. solaris. */
+ #ifdef INFINITY
+ #define ECB_INFINITY INFINITY
+ #else
+ #define ECB_INFINITY HUGE_VAL
+ #endif
+
+ #ifdef NAN
+ #define ECB_NAN NAN
+ #else
+ #define ECB_NAN ECB_INFINITY
+ #endif
+
+ #if ECB_C99 || _XOPEN_VERSION >= 600 || _POSIX_VERSION >= 200112L
+ #define ecb_ldexpf(x,e) ldexpf ((x), (e))
+ #define ecb_frexpf(x,e) frexpf ((x), (e))
+ #else
+ #define ecb_ldexpf(x,e) (float) ldexp ((double) (x), (e))
+ #define ecb_frexpf(x,e) (float) frexp ((double) (x), (e))
+ #endif
+
+ /* convert a float to ieee single/binary32 */
+ ecb_function_ ecb_const uint32_t ecb_float_to_binary32 (float x);
+ ecb_function_ ecb_const uint32_t
+ ecb_float_to_binary32 (float x)
+ {
+ uint32_t r;
+
+ #if ECB_STDFP
+ memcpy (&r, &x, 4);
+ #else
+ /* slow emulation, works for anything but -0 */
+ uint32_t m;
+ int e;
+
+ if (x == 0e0f ) return 0x00000000U;
+ if (x > +3.40282346638528860e+38f) return 0x7f800000U;
+ if (x < -3.40282346638528860e+38f) return 0xff800000U;
+ if (x != x ) return 0x7fbfffffU;
+
+ m = ecb_frexpf (x, &e) * 0x1000000U;
+
+ r = m & 0x80000000U;
+
+ if (r)
+ m = -m;
+
+ if (e <= -126)
+ {
+ m &= 0xffffffU;
+ m >>= (-125 - e);
+ e = -126;
+ }
+
+ r |= (e + 126) << 23;
+ r |= m & 0x7fffffU;
+ #endif
+
+ return r;
+ }
+
+ /* converts an ieee single/binary32 to a float */
+ ecb_function_ ecb_const float ecb_binary32_to_float (uint32_t x);
+ ecb_function_ ecb_const float
+ ecb_binary32_to_float (uint32_t x)
+ {
+ float r;
+
+ #if ECB_STDFP
+ memcpy (&r, &x, 4);
+ #else
+ /* emulation, only works for normals and subnormals and +0 */
+ int neg = x >> 31;
+ int e = (x >> 23) & 0xffU;
+
+ x &= 0x7fffffU;
+
+ if (e)
+ x |= 0x800000U;
+ else
+ e = 1;
+
+ /* we distrust ldexpf a bit and do the 2**-24 scaling by an extra multiply */
+ r = ecb_ldexpf (x * (0.5f / 0x800000U), e - 126);
+
+ r = neg ? -r : r;
+ #endif
+
+ return r;
+ }
+
+ /* convert a double to ieee double/binary64 */
+ ecb_function_ ecb_const uint64_t ecb_double_to_binary64 (double x);
+ ecb_function_ ecb_const uint64_t
+ ecb_double_to_binary64 (double x)
+ {
+ uint64_t r;
+
+ #if ECB_STDFP
+ memcpy (&r, &x, 8);
+ #else
+ /* slow emulation, works for anything but -0 */
+ uint64_t m;
+ int e;
+
+ if (x == 0e0 ) return 0x0000000000000000U;
+ if (x > +1.79769313486231470e+308) return 0x7ff0000000000000U;
+ if (x < -1.79769313486231470e+308) return 0xfff0000000000000U;
+ if (x != x ) return 0X7ff7ffffffffffffU;
+
+ m = frexp (x, &e) * 0x20000000000000U;
+
+ r = m & 0x8000000000000000;;
+
+ if (r)
+ m = -m;
+
+ if (e <= -1022)
+ {
+ m &= 0x1fffffffffffffU;
+ m >>= (-1021 - e);
+ e = -1022;
+ }
+
+ r |= ((uint64_t)(e + 1022)) << 52;
+ r |= m & 0xfffffffffffffU;
+ #endif
+
+ return r;
+ }
+
+ /* converts an ieee double/binary64 to a double */
+ ecb_function_ ecb_const double ecb_binary64_to_double (uint64_t x);
+ ecb_function_ ecb_const double
+ ecb_binary64_to_double (uint64_t x)
+ {
+ double r;
+
+ #if ECB_STDFP
+ memcpy (&r, &x, 8);
+ #else
+ /* emulation, only works for normals and subnormals and +0 */
+ int neg = x >> 63;
+ int e = (x >> 52) & 0x7ffU;
+
+ x &= 0xfffffffffffffU;
+
+ if (e)
+ x |= 0x10000000000000U;
+ else
+ e = 1;
+
+ /* we distrust ldexp a bit and do the 2**-53 scaling by an extra multiply */
+ r = ldexp (x * (0.5 / 0x10000000000000U), e - 1022);
+
+ r = neg ? -r : r;
+ #endif
+
+ return r;
+ }
+
+ /* convert a float to ieee half/binary16 */
+ ecb_function_ ecb_const uint16_t ecb_float_to_binary16 (float x);
+ ecb_function_ ecb_const uint16_t
+ ecb_float_to_binary16 (float x)
+ {
+ return ecb_binary32_to_binary16 (ecb_float_to_binary32 (x));
+ }
+
+ /* convert an ieee half/binary16 to float */
+ ecb_function_ ecb_const float ecb_binary16_to_float (uint16_t x);
+ ecb_function_ ecb_const float
+ ecb_binary16_to_float (uint16_t x)
+ {
+ return ecb_binary32_to_float (ecb_binary16_to_binary32 (x));
+ }
+
+#endif
+
+#endif
+
+/* ECB.H END */
+
+#if ECB_MEMORY_FENCE_NEEDS_PTHREADS
+/* if your architecture doesn't need memory fences, e.g. because it is
+ * single-cpu/core, or if you use libev in a project that doesn't use libev
+ * from multiple threads, then you can define ECB_NO_THREADS when compiling
+ * libev, in which cases the memory fences become nops.
+ * alternatively, you can remove this #error and link against libpthread,
+ * which will then provide the memory fences.
+ */
+# error "memory fences not defined for your architecture, please report"
+#endif
+
+#ifndef ECB_MEMORY_FENCE
+# define ECB_MEMORY_FENCE do { } while (0)
+# define ECB_MEMORY_FENCE_ACQUIRE ECB_MEMORY_FENCE
+# define ECB_MEMORY_FENCE_RELEASE ECB_MEMORY_FENCE
+#endif
+
+#define inline_size ecb_inline
+
+#if EV_FEATURE_CODE
+# define inline_speed ecb_inline
+#else
+# define inline_speed ecb_noinline static
+#endif
+
+/*****************************************************************************/
+/* raw syscall wrappers */
+
+#if EV_NEED_SYSCALL
+
+#include <sys/syscall.h>
+
+/*
+ * define some syscall wrappers for common architectures
+ * this is mostly for nice looks during debugging, not performance.
+ * our syscalls return < 0, not == -1, on error. which is good
+ * enough for linux aio.
+ * TODO: arm is also common nowadays, maybe even mips and x86
+ * TODO: after implementing this, it suddenly looks like overkill, but its hard to remove...
+ */
+#if __GNUC__ && __linux && ECB_AMD64 && !EV_FEATURE_CODE
+ /* the costly errno access probably kills this for size optimisation */
+
+ #define ev_syscall(nr,narg,arg1,arg2,arg3,arg4,arg5,arg6) \
+ ({ \
+ long res; \
+ register unsigned long r6 __asm__ ("r9" ); \
+ register unsigned long r5 __asm__ ("r8" ); \
+ register unsigned long r4 __asm__ ("r10"); \
+ register unsigned long r3 __asm__ ("rdx"); \
+ register unsigned long r2 __asm__ ("rsi"); \
+ register unsigned long r1 __asm__ ("rdi"); \
+ if (narg >= 6) r6 = (unsigned long)(arg6); \
+ if (narg >= 5) r5 = (unsigned long)(arg5); \
+ if (narg >= 4) r4 = (unsigned long)(arg4); \
+ if (narg >= 3) r3 = (unsigned long)(arg3); \
+ if (narg >= 2) r2 = (unsigned long)(arg2); \
+ if (narg >= 1) r1 = (unsigned long)(arg1); \
+ __asm__ __volatile__ ( \
+ "syscall\n\t" \
+ : "=a" (res) \
+ : "0" (nr), "r" (r1), "r" (r2), "r" (r3), "r" (r4), "r" (r5) \
+ : "cc", "r11", "cx", "memory"); \
+ errno = -res; \
+ res; \
+ })
+
+#endif
+
+#ifdef ev_syscall
+ #define ev_syscall0(nr) ev_syscall (nr, 0, 0, 0, 0, 0, 0, 0)
+ #define ev_syscall1(nr,arg1) ev_syscall (nr, 1, arg1, 0, 0, 0, 0, 0)
+ #define ev_syscall2(nr,arg1,arg2) ev_syscall (nr, 2, arg1, arg2, 0, 0, 0, 0)
+ #define ev_syscall3(nr,arg1,arg2,arg3) ev_syscall (nr, 3, arg1, arg2, arg3, 0, 0, 0)
+ #define ev_syscall4(nr,arg1,arg2,arg3,arg4) ev_syscall (nr, 3, arg1, arg2, arg3, arg4, 0, 0)
+ #define ev_syscall5(nr,arg1,arg2,arg3,arg4,arg5) ev_syscall (nr, 5, arg1, arg2, arg3, arg4, arg5, 0)
+ #define ev_syscall6(nr,arg1,arg2,arg3,arg4,arg5,arg6) ev_syscall (nr, 6, arg1, arg2, arg3, arg4, arg5,arg6)
+#else
+ #define ev_syscall0(nr) syscall (nr)
+ #define ev_syscall1(nr,arg1) syscall (nr, arg1)
+ #define ev_syscall2(nr,arg1,arg2) syscall (nr, arg1, arg2)
+ #define ev_syscall3(nr,arg1,arg2,arg3) syscall (nr, arg1, arg2, arg3)
+ #define ev_syscall4(nr,arg1,arg2,arg3,arg4) syscall (nr, arg1, arg2, arg3, arg4)
+ #define ev_syscall5(nr,arg1,arg2,arg3,arg4,arg5) syscall (nr, arg1, arg2, arg3, arg4, arg5)
+ #define ev_syscall6(nr,arg1,arg2,arg3,arg4,arg5,arg6) syscall (nr, arg1, arg2, arg3, arg4, arg5,arg6)
+#endif
+
+#endif
+
+/*****************************************************************************/
+
+#define NUMPRI (EV_MAXPRI - EV_MINPRI + 1)
+
+#if EV_MINPRI == EV_MAXPRI
+# define ABSPRI(w) (((W)w), 0)
+#else
+# define ABSPRI(w) (((W)w)->priority - EV_MINPRI)
+#endif
+
+#define EMPTY /* required for microsofts broken pseudo-c compiler */
+
+typedef ev_watcher *W;
+typedef ev_watcher_list *WL;
+typedef ev_watcher_time *WT;
+
+#define ev_active(w) ((W)(w))->active
+#define ev_at(w) ((WT)(w))->at
+
+#if EV_USE_REALTIME
+/* sig_atomic_t is used to avoid per-thread variables or locking but still */
+/* giving it a reasonably high chance of working on typical architectures */
+static EV_ATOMIC_T have_realtime; /* did clock_gettime (CLOCK_REALTIME) work? */
+#endif
+
+#if EV_USE_MONOTONIC
+static EV_ATOMIC_T have_monotonic; /* did clock_gettime (CLOCK_MONOTONIC) work? */
+static EV_ATOMIC_T monotonic_clock_id;
+#endif
+static EV_ATOMIC_T have_cheap_timer = 0;
+
+#ifndef EV_FD_TO_WIN32_HANDLE
+# define EV_FD_TO_WIN32_HANDLE(fd) _get_osfhandle (fd)
+#endif
+#ifndef EV_WIN32_HANDLE_TO_FD
+# define EV_WIN32_HANDLE_TO_FD(handle) _open_osfhandle (handle, 0)
+#endif
+#ifndef EV_WIN32_CLOSE_FD
+# define EV_WIN32_CLOSE_FD(fd) close (fd)
+#endif
+
+#ifdef _WIN32
+# include "ev_win32.c"
+#endif
+
+/*****************************************************************************/
+
+#if EV_USE_LINUXAIO
+# include <linux/aio_abi.h> /* probably only needed for aio_context_t */
+#endif
+
+/* define a suitable floor function (only used by periodics atm) */
+
+#if EV_USE_FLOOR
+# include <math.h>
+# define ev_floor(v) floor (v)
+#else
+
+#include <float.h>
+
+/* a floor() replacement function, should be independent of ev_tstamp type */
+ecb_noinline
+static ev_tstamp
+ev_floor (ev_tstamp v)
+{
+ /* the choice of shift factor is not terribly important */
+#if FLT_RADIX != 2 /* assume FLT_RADIX == 10 */
+ const ev_tstamp shift = sizeof (unsigned long) >= 8 ? 10000000000000000000. : 1000000000.;
+#else
+ const ev_tstamp shift = sizeof (unsigned long) >= 8 ? 18446744073709551616. : 4294967296.;
+#endif
+
+ /* special treatment for negative arguments */
+ if (ecb_expect_false (v < 0.))
+ {
+ ev_tstamp f = -ev_floor (-v);
+
+ return f - (f == v ? 0 : 1);
+ }
+
+ /* argument too large for an unsigned long? then reduce it */
+ if (ecb_expect_false (v >= shift))
+ {
+ ev_tstamp f;
+
+ if (v == v - 1.)
+ return v; /* very large numbers are assumed to be integer */
+
+ f = shift * ev_floor (v * (1. / shift));
+ return f + ev_floor (v - f);
+ }
+
+ /* fits into an unsigned long */
+ return (unsigned long)v;
+}
+
+#endif
+
+/*****************************************************************************/
+
+#ifdef __linux
+# include <sys/utsname.h>
+#endif
+
+ecb_noinline ecb_cold
+static unsigned int
+ev_linux_version (void)
+{
+#ifdef __linux
+ unsigned int v = 0;
+ struct utsname buf;
+ int i;
+ char *p = buf.release;
+
+ if (uname (&buf))
+ return 0;
+
+ for (i = 3+1; --i; )
+ {
+ unsigned int c = 0;
+
+ for (;;)
+ {
+ if (*p >= '0' && *p <= '9')
+ c = c * 10 + *p++ - '0';
+ else
+ {
+ p += *p == '.';
+ break;
+ }
+ }
+
+ v = (v << 8) | c;
+ }
+
+ return v;
+#else
+ return 0;
+#endif
+}
+
+/*****************************************************************************/
+
+#if EV_AVOID_STDIO
+ecb_noinline ecb_cold
+static void
+ev_printerr (const char *msg)
+{
+ write (STDERR_FILENO, msg, strlen (msg));
+}
+#endif
+
+static void (*syserr_cb)(const char *msg) EV_NOEXCEPT;
+
+ecb_cold
+void
+ev_set_syserr_cb (void (*cb)(const char *msg) EV_NOEXCEPT) EV_NOEXCEPT
+{
+ syserr_cb = cb;
+}
+
+ecb_noinline ecb_cold
+static void
+ev_syserr (const char *msg)
+{
+ if (!msg)
+ msg = "(libev) system error";
+
+ if (syserr_cb)
+ syserr_cb (msg);
+ else
+ {
+#if EV_AVOID_STDIO
+ ev_printerr (msg);
+ ev_printerr (": ");
+ ev_printerr (strerror (errno));
+ ev_printerr ("\n");
+#else
+ perror (msg);
+#endif
+ abort ();
+ }
+}
+
+static void *
+ev_realloc_emul (void *ptr, long size) EV_NOEXCEPT
+{
+ /* some systems, notably openbsd and darwin, fail to properly
+ * implement realloc (x, 0) (as required by both ansi c-89 and
+ * the single unix specification, so work around them here.
+ * recently, also (at least) fedora and debian started breaking it,
+ * despite documenting it otherwise.
+ */
+
+ if (size)
+ return realloc (ptr, size);
+
+ free (ptr);
+ return 0;
+}
+
+static void *(*alloc)(void *ptr, long size) EV_NOEXCEPT = ev_realloc_emul;
+
+ecb_cold
+void
+ev_set_allocator (void *(*cb)(void *ptr, long size) EV_NOEXCEPT) EV_NOEXCEPT
+{
+ alloc = cb;
+}
+
+inline_speed void *
+ev_realloc (void *ptr, long size)
+{
+ ptr = alloc (ptr, size);
+
+ if (!ptr && size)
+ {
+#if EV_AVOID_STDIO
+ ev_printerr ("(libev) memory allocation failed, aborting.\n");
+#else
+ fprintf (stderr, "(libev) cannot allocate %ld bytes, aborting.", size);
+#endif
+ abort ();
+ }
+
+ return ptr;
+}
+
+#define ev_malloc(size) ev_realloc (0, (size))
+#define ev_free(ptr) ev_realloc ((ptr), 0)
+
+/*****************************************************************************/
+
+/* set in reify when reification needed */
+#define EV_ANFD_REIFY 1
+
+/* file descriptor info structure */
+typedef struct
+{
+ WL head;
+ unsigned char events; /* the events watched for */
+ unsigned char reify; /* flag set when this ANFD needs reification (EV_ANFD_REIFY, EV__IOFDSET) */
+ unsigned char emask; /* some backends store the actual kernel mask in here */
+ unsigned char eflags; /* flags field for use by backends */
+#if EV_USE_EPOLL
+ unsigned int egen; /* generation counter to counter epoll bugs */
+#endif
+#if EV_SELECT_IS_WINSOCKET || EV_USE_IOCP
+ SOCKET handle;
+#endif
+#if EV_USE_IOCP
+ OVERLAPPED or, ow;
+#endif
+} ANFD;
+
+/* stores the pending event set for a given watcher */
+typedef struct
+{
+ W w;
+ int events; /* the pending event set for the given watcher */
+} ANPENDING;
+
+#if EV_USE_INOTIFY
+/* hash table entry per inotify-id */
+typedef struct
+{
+ WL head;
+} ANFS;
+#endif
+
+/* Heap Entry */
+#if EV_HEAP_CACHE_AT
+ /* a heap element */
+ typedef struct {
+ ev_tstamp at;
+ WT w;
+ } ANHE;
+
+ #define ANHE_w(he) (he).w /* access watcher, read-write */
+ #define ANHE_at(he) (he).at /* access cached at, read-only */
+ #define ANHE_at_cache(he) (he).at = (he).w->at /* update at from watcher */
+#else
+ /* a heap element */
+ typedef WT ANHE;
+
+ #define ANHE_w(he) (he)
+ #define ANHE_at(he) (he)->at
+ #define ANHE_at_cache(he)
+#endif
+
+#if EV_MULTIPLICITY
+
+ struct ev_loop
+ {
+ ev_tstamp ev_rt_now;
+ #define ev_rt_now ((loop)->ev_rt_now)
+ #define VAR(name,decl) decl;
+ #include "ev_vars.h"
+ #undef VAR
+ };
+ #include "ev_wrap.h"
+
+ static struct ev_loop default_loop_struct;
+ EV_API_DECL struct ev_loop *ev_default_loop_ptr = 0; /* needs to be initialised to make it a definition despite extern */
+
+#else
+
+ EV_API_DECL ev_tstamp ev_rt_now = EV_TS_CONST (0.); /* needs to be initialised to make it a definition despite extern */
+ #define VAR(name,decl) static decl;
+ #include "ev_vars.h"
+ #undef VAR
+
+ static int ev_default_loop_ptr;
+
+#endif
+
+#if EV_FEATURE_API
+# define EV_RELEASE_CB if (ecb_expect_false (release_cb)) release_cb (EV_A)
+# define EV_ACQUIRE_CB if (ecb_expect_false (acquire_cb)) acquire_cb (EV_A)
+# define EV_INVOKE_PENDING invoke_cb (EV_A)
+#else
+# define EV_RELEASE_CB (void)0
+# define EV_ACQUIRE_CB (void)0
+# define EV_INVOKE_PENDING ev_invoke_pending (EV_A)
+#endif
+
+#define EVBREAK_RECURSE 0x80
+
+/*****************************************************************************/
+
+#ifndef EV_HAVE_EV_TIME
+ev_tstamp
+ev_time (void) EV_NOEXCEPT
+{
+#if EV_USE_REALTIME
+ if (ecb_expect_true (have_realtime))
+ {
+ struct timespec ts;
+ clock_gettime (CLOCK_REALTIME, &ts);
+ return EV_TS_GET (ts);
+ }
+#endif
+
+ {
+ struct timeval tv;
+ gettimeofday (&tv, 0);
+ return EV_TV_GET (tv);
+ }
+}
+#endif
+
+inline_size ev_tstamp
+get_clock (void)
+{
+#if EV_USE_MONOTONIC
+ if (ecb_expect_true (have_monotonic))
+ {
+ struct timespec ts;
+ clock_gettime (monotonic_clock_id, &ts);
+ return ((ev_tstamp)ts.tv_sec) + ts.tv_nsec * 1e-9;
+ }
+#endif
+
+ return ev_time ();
+}
+
+#if EV_MULTIPLICITY
+ev_tstamp
+ev_now (EV_P) EV_NOEXCEPT
+{
+ return ev_rt_now;
+}
+#endif
+
+void
+ev_sleep (ev_tstamp delay) EV_NOEXCEPT
+{
+ if (delay > EV_TS_CONST (0.))
+ {
+#if EV_USE_NANOSLEEP
+ struct timespec ts;
+
+ EV_TS_SET (ts, delay);
+ nanosleep (&ts, 0);
+#elif defined _WIN32
+ /* maybe this should round up, as ms is very low resolution */
+ /* compared to select (µs) or nanosleep (ns) */
+ Sleep ((unsigned long)(EV_TS_TO_MSEC (delay)));
+#else
+ struct timeval tv;
+
+ /* here we rely on sys/time.h + sys/types.h + unistd.h providing select */
+ /* something not guaranteed by newer posix versions, but guaranteed */
+ /* by older ones */
+ EV_TV_SET (tv, delay);
+ select (0, 0, 0, 0, &tv);
+#endif
+ }
+}
+
+/*****************************************************************************/
+
+#define MALLOC_ROUND 4096 /* prefer to allocate in chunks of this size, must be 2**n and >> 4 longs */
+
+/* find a suitable new size for the given array, */
+/* hopefully by rounding to a nice-to-malloc size */
+inline_size int
+array_nextsize (int elem, int cur, int cnt)
+{
+ int ncur = cur + 1;
+
+ do
+ ncur <<= 1;
+ while (cnt > ncur);
+
+ /* if size is large, round to MALLOC_ROUND - 4 * longs to accommodate malloc overhead */
+ if (elem * ncur > MALLOC_ROUND - sizeof (void *) * 4)
+ {
+ ncur *= elem;
+ ncur = (ncur + elem + (MALLOC_ROUND - 1) + sizeof (void *) * 4) & ~(MALLOC_ROUND - 1);
+ ncur = ncur - sizeof (void *) * 4;
+ ncur /= elem;
+ }
+
+ return ncur;
+}
+
+ecb_noinline ecb_cold
+static void *
+array_realloc (int elem, void *base, int *cur, int cnt)
+{
+ *cur = array_nextsize (elem, *cur, cnt);
+ return ev_realloc (base, elem * *cur);
+}
+
+#define array_needsize_noinit(base,offset,count)
+
+#define array_needsize_zerofill(base,offset,count) \
+ memset ((void *)(base + offset), 0, sizeof (*(base)) * (count))
+
+#define array_needsize(type,base,cur,cnt,init) \
+ if (ecb_expect_false ((cnt) > (cur))) \
+ { \
+ ecb_unused int ocur_ = (cur); \
+ (base) = (type *)array_realloc \
+ (sizeof (type), (base), &(cur), (cnt)); \
+ init ((base), ocur_, ((cur) - ocur_)); \
+ }
+
+#if 0
+#define array_slim(type,stem) \
+ if (stem ## max < array_roundsize (stem ## cnt >> 2)) \
+ { \
+ stem ## max = array_roundsize (stem ## cnt >> 1); \
+ base = (type *)ev_realloc (base, sizeof (type) * (stem ## max));\
+ fprintf (stderr, "slimmed down " # stem " to %d\n", stem ## max);/*D*/\
+ }
+#endif
+
+#define array_free(stem, idx) \
+ ev_free (stem ## s idx); stem ## cnt idx = stem ## max idx = 0; stem ## s idx = 0
+
+/*****************************************************************************/
+
+/* dummy callback for pending events */
+ecb_noinline
+static void
+pendingcb (EV_P_ ev_prepare *w, int revents)
+{
+}
+
+ecb_noinline
+void
+ev_feed_event (EV_P_ void *w, int revents) EV_NOEXCEPT
+{
+ W w_ = (W)w;
+ int pri = ABSPRI (w_);
+
+ if (ecb_expect_false (w_->pending))
+ pendings [pri][w_->pending - 1].events |= revents;
+ else
+ {
+ w_->pending = ++pendingcnt [pri];
+ array_needsize (ANPENDING, pendings [pri], pendingmax [pri], w_->pending, array_needsize_noinit);
+ pendings [pri][w_->pending - 1].w = w_;
+ pendings [pri][w_->pending - 1].events = revents;
+ }
+
+ pendingpri = NUMPRI - 1;
+}
+
+inline_speed void
+feed_reverse (EV_P_ W w)
+{
+ array_needsize (W, rfeeds, rfeedmax, rfeedcnt + 1, array_needsize_noinit);
+ rfeeds [rfeedcnt++] = w;
+}
+
+inline_size void
+feed_reverse_done (EV_P_ int revents)
+{
+ do
+ ev_feed_event (EV_A_ rfeeds [--rfeedcnt], revents);
+ while (rfeedcnt);
+}
+
+inline_speed void
+queue_events (EV_P_ W *events, int eventcnt, int type)
+{
+ int i;
+
+ for (i = 0; i < eventcnt; ++i)
+ ev_feed_event (EV_A_ events [i], type);
+}
+
+/*****************************************************************************/
+
+inline_speed void
+fd_event_nocheck (EV_P_ int fd, int revents)
+{
+ ANFD *anfd = anfds + fd;
+ ev_io *w;
+
+ for (w = (ev_io *)anfd->head; w; w = (ev_io *)((WL)w)->next)
+ {
+ int ev = w->events & revents;
+
+ if (ev)
+ ev_feed_event (EV_A_ (W)w, ev);
+ }
+}
+
+/* do not submit kernel events for fds that have reify set */
+/* because that means they changed while we were polling for new events */
+inline_speed void
+fd_event (EV_P_ int fd, int revents)
+{
+ ANFD *anfd = anfds + fd;
+
+ if (ecb_expect_true (!anfd->reify))
+ fd_event_nocheck (EV_A_ fd, revents);
+}
+
+void
+ev_feed_fd_event (EV_P_ int fd, int revents) EV_NOEXCEPT
+{
+ if (fd >= 0 && fd < anfdmax)
+ fd_event_nocheck (EV_A_ fd, revents);
+}
+
+/* make sure the external fd watch events are in-sync */
+/* with the kernel/libev internal state */
+inline_size void
+fd_reify (EV_P)
+{
+ int i;
+
+ /* most backends do not modify the fdchanges list in backend_modfiy.
+ * except io_uring, which has fixed-size buffers which might force us
+ * to handle events in backend_modify, causing fdchanges to be amended,
+ * which could result in an endless loop.
+ * to avoid this, we do not dynamically handle fds that were added
+ * during fd_reify. that means that for those backends, fdchangecnt
+ * might be non-zero during poll, which must cause them to not block.
+ * to not put too much of a burden on other backends, this detail
+ * needs to be handled in the backend.
+ */
+ int changecnt = fdchangecnt;
+
+#if EV_SELECT_IS_WINSOCKET || EV_USE_IOCP
+ for (i = 0; i < changecnt; ++i)
+ {
+ int fd = fdchanges [i];
+ ANFD *anfd = anfds + fd;
+
+ if (anfd->reify & EV__IOFDSET && anfd->head)
+ {
+ SOCKET handle = EV_FD_TO_WIN32_HANDLE (fd);
+
+ if (handle != anfd->handle)
+ {
+ unsigned long arg;
+
+ assert (("libev: only socket fds supported in this configuration", ioctlsocket (handle, FIONREAD, &arg) == 0));
+
+ /* handle changed, but fd didn't - we need to do it in two steps */
+ backend_modify (EV_A_ fd, anfd->events, 0);
+ anfd->events = 0;
+ anfd->handle = handle;
+ }
+ }
+ }
+#endif
+
+ for (i = 0; i < changecnt; ++i)
+ {
+ int fd = fdchanges [i];
+ ANFD *anfd = anfds + fd;
+ ev_io *w;
+
+ unsigned char o_events = anfd->events;
+ unsigned char o_reify = anfd->reify;
+
+ anfd->reify = 0;
+
+ /*if (ecb_expect_true (o_reify & EV_ANFD_REIFY)) probably a deoptimisation */
+ {
+ anfd->events = 0;
+
+ for (w = (ev_io *)anfd->head; w; w = (ev_io *)((WL)w)->next)
+ anfd->events |= (unsigned char)w->events;
+
+ if (o_events != anfd->events)
+ o_reify = EV__IOFDSET; /* actually |= */
+ }
+
+ if (o_reify & EV__IOFDSET)
+ backend_modify (EV_A_ fd, o_events, anfd->events);
+ }
+
+ /* normally, fdchangecnt hasn't changed. if it has, then new fds have been added.
+ * this is a rare case (see beginning comment in this function), so we copy them to the
+ * front and hope the backend handles this case.
+ */
+ if (ecb_expect_false (fdchangecnt != changecnt))
+ memmove (fdchanges, fdchanges + changecnt, (fdchangecnt - changecnt) * sizeof (*fdchanges));
+
+ fdchangecnt -= changecnt;
+}
+
+/* something about the given fd changed */
+inline_size
+void
+fd_change (EV_P_ int fd, int flags)
+{
+ unsigned char reify = anfds [fd].reify;
+ anfds [fd].reify = reify | flags;
+
+ if (ecb_expect_true (!reify))
+ {
+ ++fdchangecnt;
+ array_needsize (int, fdchanges, fdchangemax, fdchangecnt, array_needsize_noinit);
+ fdchanges [fdchangecnt - 1] = fd;
+ }
+}
+
+/* the given fd is invalid/unusable, so make sure it doesn't hurt us anymore */
+inline_speed ecb_cold void
+fd_kill (EV_P_ int fd)
+{
+ ev_io *w;
+
+ while ((w = (ev_io *)anfds [fd].head))
+ {
+ ev_io_stop (EV_A_ w);
+ ev_feed_event (EV_A_ (W)w, EV_ERROR | EV_READ | EV_WRITE);
+ }
+}
+
+/* check whether the given fd is actually valid, for error recovery */
+inline_size ecb_cold int
+fd_valid (int fd)
+{
+#ifdef _WIN32
+ return EV_FD_TO_WIN32_HANDLE (fd) != -1;
+#else
+ return fcntl (fd, F_GETFD) != -1;
+#endif
+}
+
+/* called on EBADF to verify fds */
+ecb_noinline ecb_cold
+static void
+fd_ebadf (EV_P)
+{
+ int fd;
+
+ for (fd = 0; fd < anfdmax; ++fd)
+ if (anfds [fd].events)
+ if (!fd_valid (fd) && errno == EBADF)
+ fd_kill (EV_A_ fd);
+}
+
+/* called on ENOMEM in select/poll to kill some fds and retry */
+ecb_noinline ecb_cold
+static void
+fd_enomem (EV_P)
+{
+ int fd;
+
+ for (fd = anfdmax; fd--; )
+ if (anfds [fd].events)
+ {
+ fd_kill (EV_A_ fd);
+ break;
+ }
+}
+
+/* usually called after fork if backend needs to re-arm all fds from scratch */
+ecb_noinline
+static void
+fd_rearm_all (EV_P)
+{
+ int fd;
+
+ for (fd = 0; fd < anfdmax; ++fd)
+ if (anfds [fd].events)
+ {
+ anfds [fd].events = 0;
+ anfds [fd].emask = 0;
+ fd_change (EV_A_ fd, EV__IOFDSET | EV_ANFD_REIFY);
+ }
+}
+
+/* used to prepare libev internal fd's */
+/* this is not fork-safe */
+inline_speed void
+fd_intern (int fd)
+{
+#ifdef _WIN32
+ unsigned long arg = 1;
+ ioctlsocket (EV_FD_TO_WIN32_HANDLE (fd), FIONBIO, &arg);
+#else
+ fcntl (fd, F_SETFD, FD_CLOEXEC);
+ fcntl (fd, F_SETFL, O_NONBLOCK);
+#endif
+}
+
+/*****************************************************************************/
+
+/*
+ * the heap functions want a real array index. array index 0 is guaranteed to not
+ * be in-use at any time. the first heap entry is at array [HEAP0]. DHEAP gives
+ * the branching factor of the d-tree.
+ */
+
+/*
+ * at the moment we allow libev the luxury of two heaps,
+ * a small-code-size 2-heap one and a ~1.5kb larger 4-heap
+ * which is more cache-efficient.
+ * the difference is about 5% with 50000+ watchers.
+ */
+#if EV_USE_4HEAP
+
+#define DHEAP 4
+#define HEAP0 (DHEAP - 1) /* index of first element in heap */
+#define HPARENT(k) ((((k) - HEAP0 - 1) / DHEAP) + HEAP0)
+#define UPHEAP_DONE(p,k) ((p) == (k))
+
+/* away from the root */
+inline_speed void
+downheap (ANHE *heap, int N, int k)
+{
+ ANHE he = heap [k];
+ ANHE *E = heap + N + HEAP0;
+
+ for (;;)
+ {
+ ev_tstamp minat;
+ ANHE *minpos;
+ ANHE *pos = heap + DHEAP * (k - HEAP0) + HEAP0 + 1;
+
+ /* find minimum child */
+ if (ecb_expect_true (pos + DHEAP - 1 < E))
+ {
+ /* fast path */ (minpos = pos + 0), (minat = ANHE_at (*minpos));
+ if ( minat > ANHE_at (pos [1])) (minpos = pos + 1), (minat = ANHE_at (*minpos));
+ if ( minat > ANHE_at (pos [2])) (minpos = pos + 2), (minat = ANHE_at (*minpos));
+ if ( minat > ANHE_at (pos [3])) (minpos = pos + 3), (minat = ANHE_at (*minpos));
+ }
+ else if (pos < E)
+ {
+ /* slow path */ (minpos = pos + 0), (minat = ANHE_at (*minpos));
+ if (pos + 1 < E && minat > ANHE_at (pos [1])) (minpos = pos + 1), (minat = ANHE_at (*minpos));
+ if (pos + 2 < E && minat > ANHE_at (pos [2])) (minpos = pos + 2), (minat = ANHE_at (*minpos));
+ if (pos + 3 < E && minat > ANHE_at (pos [3])) (minpos = pos + 3), (minat = ANHE_at (*minpos));
+ }
+ else
+ break;
+
+ if (ANHE_at (he) <= minat)
+ break;
+
+ heap [k] = *minpos;
+ ev_active (ANHE_w (*minpos)) = k;
+
+ k = minpos - heap;
+ }
+
+ heap [k] = he;
+ ev_active (ANHE_w (he)) = k;
+}
+
+#else /* not 4HEAP */
+
+#define HEAP0 1
+#define HPARENT(k) ((k) >> 1)
+#define UPHEAP_DONE(p,k) (!(p))
+
+/* away from the root */
+inline_speed void
+downheap (ANHE *heap, int N, int k)
+{
+ ANHE he = heap [k];
+
+ for (;;)
+ {
+ int c = k << 1;
+
+ if (c >= N + HEAP0)
+ break;
+
+ c += c + 1 < N + HEAP0 && ANHE_at (heap [c]) > ANHE_at (heap [c + 1])
+ ? 1 : 0;
+
+ if (ANHE_at (he) <= ANHE_at (heap [c]))
+ break;
+
+ heap [k] = heap [c];
+ ev_active (ANHE_w (heap [k])) = k;
+
+ k = c;
+ }
+
+ heap [k] = he;
+ ev_active (ANHE_w (he)) = k;
+}
+#endif
+
+/* towards the root */
+inline_speed void
+upheap (ANHE *heap, int k)
+{
+ ANHE he = heap [k];
+
+ for (;;)
+ {
+ int p = HPARENT (k);
+
+ if (UPHEAP_DONE (p, k) || ANHE_at (heap [p]) <= ANHE_at (he))
+ break;
+
+ heap [k] = heap [p];
+ ev_active (ANHE_w (heap [k])) = k;
+ k = p;
+ }
+
+ heap [k] = he;
+ ev_active (ANHE_w (he)) = k;
+}
+
+/* move an element suitably so it is in a correct place */
+inline_size void
+adjustheap (ANHE *heap, int N, int k)
+{
+ if (k > HEAP0 && ANHE_at (heap [k]) <= ANHE_at (heap [HPARENT (k)]))
+ upheap (heap, k);
+ else
+ downheap (heap, N, k);
+}
+
+/* rebuild the heap: this function is used only once and executed rarely */
+inline_size void
+reheap (ANHE *heap, int N)
+{
+ int i;
+
+ /* we don't use floyds algorithm, upheap is simpler and is more cache-efficient */
+ /* also, this is easy to implement and correct for both 2-heaps and 4-heaps */
+ for (i = 0; i < N; ++i)
+ upheap (heap, i + HEAP0);
+}
+
+/*****************************************************************************/
+
+/* associate signal watchers to a signal */
+typedef struct
+{
+ EV_ATOMIC_T pending;
+#if EV_MULTIPLICITY
+ EV_P;
+#endif
+ WL head;
+} ANSIG;
+
+static ANSIG signals [EV_NSIG - 1];
+
+/*****************************************************************************/
+
+#if EV_SIGNAL_ENABLE || EV_ASYNC_ENABLE
+
+ecb_noinline ecb_cold
+static void
+evpipe_init (EV_P)
+{
+ if (!ev_is_active (&pipe_w))
+ {
+ int fds [2];
+
+# if EV_USE_EVENTFD
+ fds [0] = -1;
+ fds [1] = eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC);
+ if (fds [1] < 0 && errno == EINVAL)
+ fds [1] = eventfd (0, 0);
+
+ if (fds [1] < 0)
+# endif
+ {
+ while (pipe (fds))
+ ev_syserr ("(libev) error creating signal/async pipe");
+
+ fd_intern (fds [0]);
+ }
+
+ evpipe [0] = fds [0];
+
+ if (evpipe [1] < 0)
+ evpipe [1] = fds [1]; /* first call, set write fd */
+ else
+ {
+ /* on subsequent calls, do not change evpipe [1] */
+ /* so that evpipe_write can always rely on its value. */
+ /* this branch does not do anything sensible on windows, */
+ /* so must not be executed on windows */
+
+ dup2 (fds [1], evpipe [1]);
+ close (fds [1]);
+ }
+
+ fd_intern (evpipe [1]);
+
+ ev_io_set (&pipe_w, evpipe [0] < 0 ? evpipe [1] : evpipe [0], EV_READ);
+ ev_io_start (EV_A_ &pipe_w);
+ ev_unref (EV_A); /* watcher should not keep loop alive */
+ }
+}
+
+inline_speed void
+evpipe_write (EV_P_ EV_ATOMIC_T *flag)
+{
+ ECB_MEMORY_FENCE; /* push out the write before this function was called, acquire flag */
+
+ if (ecb_expect_true (*flag))
+ return;
+
+ *flag = 1;
+ ECB_MEMORY_FENCE_RELEASE; /* make sure flag is visible before the wakeup */
+
+ pipe_write_skipped = 1;
+
+ ECB_MEMORY_FENCE; /* make sure pipe_write_skipped is visible before we check pipe_write_wanted */
+
+ if (pipe_write_wanted)
+ {
+ int old_errno;
+
+ pipe_write_skipped = 0;
+ ECB_MEMORY_FENCE_RELEASE;
+
+ old_errno = errno; /* save errno because write will clobber it */
+
+#if EV_USE_EVENTFD
+ if (evpipe [0] < 0)
+ {
+ uint64_t counter = 1;
+ (void) !write (evpipe [1], &counter, sizeof (uint64_t));
+ }
+ else
+#endif
+ {
+#ifdef _WIN32
+ WSABUF buf;
+ DWORD sent;
+ buf.buf = (char *)&buf;
+ buf.len = 1;
+ WSASend (EV_FD_TO_WIN32_HANDLE (evpipe [1]), &buf, 1, &sent, 0, 0, 0);
+#else
+ (void) !write (evpipe [1], &(evpipe [1]), 1);
+#endif
+ }
+
+ errno = old_errno;
+ }
+}
+
+/* called whenever the libev signal pipe */
+/* got some events (signal, async) */
+static void
+pipecb (EV_P_ ev_io *iow, int revents)
+{
+ int i;
+
+ if (revents & EV_READ)
+ {
+#if EV_USE_EVENTFD
+ if (evpipe [0] < 0)
+ {
+ uint64_t counter;
+ (void) !read (evpipe [1], &counter, sizeof (uint64_t));
+ }
+ else
+#endif
+ {
+ char dummy[4];
+#ifdef _WIN32
+ WSABUF buf;
+ DWORD recvd;
+ DWORD flags = 0;
+ buf.buf = dummy;
+ buf.len = sizeof (dummy);
+ WSARecv (EV_FD_TO_WIN32_HANDLE (evpipe [0]), &buf, 1, &recvd, &flags, 0, 0);
+#else
+ (void) !read (evpipe [0], &dummy, sizeof (dummy));
+#endif
+ }
+ }
+
+ pipe_write_skipped = 0;
+
+ ECB_MEMORY_FENCE; /* push out skipped, acquire flags */
+
+#if EV_SIGNAL_ENABLE
+ if (sig_pending)
+ {
+ sig_pending = 0;
+
+ ECB_MEMORY_FENCE;
+
+ for (i = EV_NSIG - 1; i--; )
+ if (ecb_expect_false (signals [i].pending))
+ ev_feed_signal_event (EV_A_ i + 1);
+ }
+#endif
+
+#if EV_ASYNC_ENABLE
+ if (async_pending)
+ {
+ async_pending = 0;
+
+ ECB_MEMORY_FENCE;
+
+ for (i = asynccnt; i--; )
+ if (asyncs [i]->sent)
+ {
+ asyncs [i]->sent = 0;
+ ECB_MEMORY_FENCE_RELEASE;
+ ev_feed_event (EV_A_ asyncs [i], EV_ASYNC);
+ }
+ }
+#endif
+}
+
+/*****************************************************************************/
+
+void
+ev_feed_signal (int signum) EV_NOEXCEPT
+{
+#if EV_MULTIPLICITY
+ EV_P;
+ ECB_MEMORY_FENCE_ACQUIRE;
+ EV_A = signals [signum - 1].loop;
+
+ if (!EV_A)
+ return;
+#endif
+
+ signals [signum - 1].pending = 1;
+ evpipe_write (EV_A_ &sig_pending);
+}
+
+static void
+ev_sighandler (int signum)
+{
+#ifdef _WIN32
+ signal (signum, ev_sighandler);
+#endif
+
+ ev_feed_signal (signum);
+}
+
+ecb_noinline
+void
+ev_feed_signal_event (EV_P_ int signum) EV_NOEXCEPT
+{
+ WL w;
+
+ if (ecb_expect_false (signum <= 0 || signum >= EV_NSIG))
+ return;
+
+ --signum;
+
+#if EV_MULTIPLICITY
+ /* it is permissible to try to feed a signal to the wrong loop */
+ /* or, likely more useful, feeding a signal nobody is waiting for */
+
+ if (ecb_expect_false (signals [signum].loop != EV_A))
+ return;
+#endif
+
+ signals [signum].pending = 0;
+ ECB_MEMORY_FENCE_RELEASE;
+
+ for (w = signals [signum].head; w; w = w->next)
+ ev_feed_event (EV_A_ (W)w, EV_SIGNAL);
+}
+
+#if EV_USE_SIGNALFD
+static void
+sigfdcb (EV_P_ ev_io *iow, int revents)
+{
+ struct signalfd_siginfo si[2], *sip; /* these structs are big */
+
+ for (;;)
+ {
+ ssize_t res = read (sigfd, si, sizeof (si));
+
+ /* not ISO-C, as res might be -1, but works with SuS */
+ for (sip = si; (char *)sip < (char *)si + res; ++sip)
+ ev_feed_signal_event (EV_A_ sip->ssi_signo);
+
+ if (res < (ssize_t)sizeof (si))
+ break;
+ }
+}
+#endif
+
+#endif
+
+/*****************************************************************************/
+
+#if EV_CHILD_ENABLE
+static WL childs [EV_PID_HASHSIZE];
+
+static ev_signal childev;
+
+#ifndef WIFCONTINUED
+# define WIFCONTINUED(status) 0
+#endif
+
+/* handle a single child status event */
+inline_speed void
+child_reap (EV_P_ int chain, int pid, int status)
+{
+ ev_child *w;
+ int traced = WIFSTOPPED (status) || WIFCONTINUED (status);
+
+ for (w = (ev_child *)childs [chain & ((EV_PID_HASHSIZE) - 1)]; w; w = (ev_child *)((WL)w)->next)
+ {
+ if ((w->pid == pid || !w->pid)
+ && (!traced || (w->flags & 1)))
+ {
+ ev_set_priority (w, EV_MAXPRI); /* need to do it *now*, this *must* be the same prio as the signal watcher itself */
+ w->rpid = pid;
+ w->rstatus = status;
+ ev_feed_event (EV_A_ (W)w, EV_CHILD);
+ }
+ }
+}
+
+#ifndef WCONTINUED
+# define WCONTINUED 0
+#endif
+
+/* called on sigchld etc., calls waitpid */
+static void
+childcb (EV_P_ ev_signal *sw, int revents)
+{
+ int pid, status;
+
+ /* some systems define WCONTINUED but then fail to support it (linux 2.4) */
+ if (0 >= (pid = waitpid (-1, &status, WNOHANG | WUNTRACED | WCONTINUED)))
+ if (!WCONTINUED
+ || errno != EINVAL
+ || 0 >= (pid = waitpid (-1, &status, WNOHANG | WUNTRACED)))
+ return;
+
+ /* make sure we are called again until all children have been reaped */
+ /* we need to do it this way so that the callback gets called before we continue */
+ ev_feed_event (EV_A_ (W)sw, EV_SIGNAL);
+
+ child_reap (EV_A_ pid, pid, status);
+ if ((EV_PID_HASHSIZE) > 1)
+ child_reap (EV_A_ 0, pid, status); /* this might trigger a watcher twice, but feed_event catches that */
+}
+
+#endif
+
+/*****************************************************************************/
+
+#if EV_USE_TIMERFD
+
+static void periodics_reschedule (EV_P);
+
+static void
+timerfdcb (EV_P_ ev_io *iow, int revents)
+{
+ struct itimerspec its = { 0 };
+
+ its.it_value.tv_sec = ev_rt_now + (int)MAX_BLOCKTIME2;
+ timerfd_settime (timerfd, TFD_TIMER_ABSTIME | TFD_TIMER_CANCEL_ON_SET, &its, 0);
+
+ ev_rt_now = ev_time ();
+ /* periodics_reschedule only needs ev_rt_now */
+ /* but maybe in the future we want the full treatment. */
+ /*
+ now_floor = EV_TS_CONST (0.);
+ time_update (EV_A_ EV_TSTAMP_HUGE);
+ */
+#if EV_PERIODIC_ENABLE
+ periodics_reschedule (EV_A);
+#endif
+}
+
+ecb_noinline ecb_cold
+static void
+evtimerfd_init (EV_P)
+{
+ if (!ev_is_active (&timerfd_w))
+ {
+ timerfd = timerfd_create (CLOCK_REALTIME, TFD_NONBLOCK | TFD_CLOEXEC);
+
+ if (timerfd >= 0)
+ {
+ fd_intern (timerfd); /* just to be sure */
+
+ ev_io_init (&timerfd_w, timerfdcb, timerfd, EV_READ);
+ ev_set_priority (&timerfd_w, EV_MINPRI);
+ ev_io_start (EV_A_ &timerfd_w);
+ ev_unref (EV_A); /* watcher should not keep loop alive */
+
+ /* (re-) arm timer */
+ timerfdcb (EV_A_ 0, 0);
+ }
+ }
+}
+
+#endif
+
+/*****************************************************************************/
+
+#if EV_USE_IOCP
+# include "ev_iocp.c"
+#endif
+#if EV_USE_PORT
+# include "ev_port.c"
+#endif
+#if EV_USE_KQUEUE
+# include "ev_kqueue.c"
+#endif
+#if EV_USE_EPOLL
+# include "ev_epoll.c"
+#endif
+#if EV_USE_LINUXAIO
+# include "ev_linuxaio.c"
+#endif
+#if EV_USE_IOURING
+# include "ev_iouring.c"
+#endif
+#if EV_USE_POLL
+# include "ev_poll.c"
+#endif
+#if EV_USE_SELECT
+# include "ev_select.c"
+#endif
+
+ecb_cold int
+ev_version_major (void) EV_NOEXCEPT
+{
+ return EV_VERSION_MAJOR;
+}
+
+ecb_cold int
+ev_version_minor (void) EV_NOEXCEPT
+{
+ return EV_VERSION_MINOR;
+}
+
+/* return true if we are running with elevated privileges and should ignore env variables */
+inline_size ecb_cold int
+enable_secure (void)
+{
+#ifdef _WIN32
+ return 0;
+#else
+ return getuid () != geteuid ()
+ || getgid () != getegid ();
+#endif
+}
+
+ecb_cold
+unsigned int
+ev_supported_backends (void) EV_NOEXCEPT
+{
+ unsigned int flags = 0;
+
+ if (EV_USE_PORT ) flags |= EVBACKEND_PORT;
+ if (EV_USE_KQUEUE ) flags |= EVBACKEND_KQUEUE;
+ if (EV_USE_EPOLL ) flags |= EVBACKEND_EPOLL;
+ if (EV_USE_LINUXAIO ) flags |= EVBACKEND_LINUXAIO;
+ if (EV_USE_IOURING && ev_linux_version () >= 0x050601) flags |= EVBACKEND_IOURING; /* 5.6.1+ */
+ if (EV_USE_POLL ) flags |= EVBACKEND_POLL;
+ if (EV_USE_SELECT ) flags |= EVBACKEND_SELECT;
+
+ return flags;
+}
+
+ecb_cold
+unsigned int
+ev_recommended_backends (void) EV_NOEXCEPT
+{
+ unsigned int flags = ev_supported_backends ();
+
+#ifndef __NetBSD__
+ /* kqueue is borked on everything but netbsd apparently */
+ /* it usually doesn't work correctly on anything but sockets and pipes */
+ flags &= ~EVBACKEND_KQUEUE;
+#endif
+#ifdef __APPLE__
+ /* only select works correctly on that "unix-certified" platform */
+ flags &= ~EVBACKEND_KQUEUE; /* horribly broken, even for sockets */
+ flags &= ~EVBACKEND_POLL; /* poll is based on kqueue from 10.5 onwards */
+#endif
+#ifdef __FreeBSD__
+ flags &= ~EVBACKEND_POLL; /* poll return value is unusable (http://forums.freebsd.org/archive/index.php/t-10270.html) */
+#endif
+
+ /* TODO: linuxaio is very experimental */
+#if !EV_RECOMMEND_LINUXAIO
+ flags &= ~EVBACKEND_LINUXAIO;
+#endif
+ /* TODO: linuxaio is super experimental */
+#if !EV_RECOMMEND_IOURING
+ flags &= ~EVBACKEND_IOURING;
+#endif
+
+ return flags;
+}
+
+ecb_cold
+unsigned int
+ev_embeddable_backends (void) EV_NOEXCEPT
+{
+ int flags = EVBACKEND_EPOLL | EVBACKEND_KQUEUE | EVBACKEND_PORT | EVBACKEND_IOURING;
+
+ /* epoll embeddability broken on all linux versions up to at least 2.6.23 */
+ if (ev_linux_version () < 0x020620) /* disable it on linux < 2.6.32 */
+ flags &= ~EVBACKEND_EPOLL;
+
+ /* EVBACKEND_LINUXAIO is theoretically embeddable, but suffers from a performance overhead */
+
+ return flags;
+}
+
+unsigned int
+ev_backend (EV_P) EV_NOEXCEPT
+{
+ return backend;
+}
+
+#if EV_FEATURE_API
+unsigned int
+ev_iteration (EV_P) EV_NOEXCEPT
+{
+ return loop_count;
+}
+
+unsigned int
+ev_depth (EV_P) EV_NOEXCEPT
+{
+ return loop_depth;
+}
+
+void
+ev_set_io_collect_interval (EV_P_ ev_tstamp interval) EV_NOEXCEPT
+{
+ io_blocktime = interval;
+}
+
+void
+ev_set_timeout_collect_interval (EV_P_ ev_tstamp interval) EV_NOEXCEPT
+{
+ timeout_blocktime = interval;
+}
+
+void
+ev_set_userdata (EV_P_ void *data) EV_NOEXCEPT
+{
+ userdata = data;
+}
+
+void *
+ev_userdata (EV_P) EV_NOEXCEPT
+{
+ return userdata;
+}
+
+void
+ev_set_invoke_pending_cb (EV_P_ ev_loop_callback invoke_pending_cb) EV_NOEXCEPT
+{
+ invoke_cb = invoke_pending_cb;
+}
+
+void
+ev_set_loop_release_cb (EV_P_ void (*release)(EV_P) EV_NOEXCEPT, void (*acquire)(EV_P) EV_NOEXCEPT) EV_NOEXCEPT
+{
+ release_cb = release;
+ acquire_cb = acquire;
+}
+#endif
+
+/* initialise a loop structure, must be zero-initialised */
+ecb_noinline ecb_cold
+static void
+loop_init (EV_P_ unsigned int flags) EV_NOEXCEPT
+{
+ if (!backend)
+ {
+ origflags = flags;
+
+#if EV_USE_REALTIME
+ if (!have_realtime)
+ {
+ struct timespec ts;
+
+ if (!clock_gettime (CLOCK_REALTIME, &ts))
+ have_realtime = 1;
+ }
+#endif
+
+#if EV_USE_MONOTONIC
+ if (!have_monotonic)
+ {
+ struct timespec ts;
+
+ if (!clock_gettime (CLOCK_MONOTONIC, &ts)) {
+ have_monotonic = 1;
+ monotonic_clock_id = CLOCK_MONOTONIC;
+#define CHECK_CLOCK_SOURCE(id) do { \
+ if (!clock_gettime ((id), &ts) && \
+ !clock_getres ((id), &ts)) { \
+ if (ts.tv_sec == 0 && ts.tv_nsec < 10ULL * 1000000) { \
+ monotonic_clock_id = (id); \
+ have_cheap_timer = 1; \
+ } \
+ } \
+} while(0)
+#ifdef CLOCK_MONOTONIC_COARSE
+ CHECK_CLOCK_SOURCE(CLOCK_MONOTONIC_COARSE);
+#elif defined(CLOCK_MONOTONIC_FAST) /* BSD stuff */
+ CHECK_CLOCK_SOURCE(CLOCK_MONOTONIC_FAST);
+#elif defined(CLOCK_MONOTONIC_RAW_APPROX) /* OSX stuff */
+ CHECK_CLOCK_SOURCE(CLOCK_MONOTONIC_RAW_APPROX);
+#endif
+#undef CHECK_CLOCK_SOURCE
+ }
+ }
+#endif
+
+ /* pid check not overridable via env */
+#ifndef _WIN32
+ if (flags & EVFLAG_FORKCHECK)
+ curpid = getpid ();
+#endif
+
+ if (!(flags & EVFLAG_NOENV)
+ && !enable_secure ()
+ && getenv ("LIBEV_FLAGS"))
+ flags = atoi (getenv ("LIBEV_FLAGS"));
+
+ ev_rt_now = ev_time ();
+ mn_now = get_clock ();
+ now_floor = mn_now;
+ rtmn_diff = ev_rt_now - mn_now;
+#if EV_FEATURE_API
+ invoke_cb = ev_invoke_pending;
+#endif
+
+ io_blocktime = 0.;
+ timeout_blocktime = 0.;
+ backend = 0;
+ backend_fd = -1;
+ sig_pending = 0;
+#if EV_ASYNC_ENABLE
+ async_pending = 0;
+#endif
+ pipe_write_skipped = 0;
+ pipe_write_wanted = 0;
+ evpipe [0] = -1;
+ evpipe [1] = -1;
+#if EV_USE_INOTIFY
+ fs_fd = flags & EVFLAG_NOINOTIFY ? -1 : -2;
+#endif
+#if EV_USE_SIGNALFD
+ sigfd = flags & EVFLAG_SIGNALFD ? -2 : -1;
+#endif
+#if EV_USE_TIMERFD
+ timerfd = flags & EVFLAG_NOTIMERFD ? -1 : -2;
+#endif
+
+ if (!(flags & EVBACKEND_MASK))
+ flags |= ev_recommended_backends ();
+
+#if EV_USE_IOCP
+ if (!backend && (flags & EVBACKEND_IOCP )) backend = iocp_init (EV_A_ flags);
+#endif
+#if EV_USE_PORT
+ if (!backend && (flags & EVBACKEND_PORT )) backend = port_init (EV_A_ flags);
+#endif
+#if EV_USE_KQUEUE
+ if (!backend && (flags & EVBACKEND_KQUEUE )) backend = kqueue_init (EV_A_ flags);
+#endif
+#if EV_USE_IOURING
+ if (!backend && (flags & EVBACKEND_IOURING )) backend = iouring_init (EV_A_ flags);
+#endif
+#if EV_USE_LINUXAIO
+ if (!backend && (flags & EVBACKEND_LINUXAIO)) backend = linuxaio_init (EV_A_ flags);
+#endif
+#if EV_USE_EPOLL
+ if (!backend && (flags & EVBACKEND_EPOLL )) backend = epoll_init (EV_A_ flags);
+#endif
+#if EV_USE_POLL
+ if (!backend && (flags & EVBACKEND_POLL )) backend = poll_init (EV_A_ flags);
+#endif
+#if EV_USE_SELECT
+ if (!backend && (flags & EVBACKEND_SELECT )) backend = select_init (EV_A_ flags);
+#endif
+
+ ev_prepare_init (&pending_w, pendingcb);
+
+#if EV_SIGNAL_ENABLE || EV_ASYNC_ENABLE
+ ev_init (&pipe_w, pipecb);
+ ev_set_priority (&pipe_w, EV_MAXPRI);
+#endif
+ }
+}
+
+EV_INLINE struct ev_loop *
+ev_default_loop_uc_ (void) EV_NOEXCEPT
+{
+ return ev_default_loop_ptr;
+}
+
+EV_INLINE int
+ev_is_default_loop (EV_P) EV_NOEXCEPT
+{
+ return EV_A == EV_DEFAULT_UC;
+}
+
+/* free up a loop structure */
+ecb_cold
+void
+ev_loop_destroy (EV_P)
+{
+ int i;
+
+#if EV_MULTIPLICITY
+ /* mimic free (0) */
+ if (!EV_A)
+ return;
+#endif
+
+#if EV_CLEANUP_ENABLE
+ /* queue cleanup watchers (and execute them) */
+ if (ecb_expect_false (cleanupcnt))
+ {
+ queue_events (EV_A_ (W *)cleanups, cleanupcnt, EV_CLEANUP);
+ EV_INVOKE_PENDING;
+ }
+#endif
+
+#if EV_CHILD_ENABLE
+ if (ev_is_default_loop (EV_A) && ev_is_active (&childev))
+ {
+ ev_ref (EV_A); /* child watcher */
+ ev_signal_stop (EV_A_ &childev);
+ }
+#endif
+
+ if (ev_is_active (&pipe_w))
+ {
+ /*ev_ref (EV_A);*/
+ /*ev_io_stop (EV_A_ &pipe_w);*/
+
+ if (evpipe [0] >= 0) EV_WIN32_CLOSE_FD (evpipe [0]);
+ if (evpipe [1] >= 0) EV_WIN32_CLOSE_FD (evpipe [1]);
+ }
+
+#if EV_USE_SIGNALFD
+ if (ev_is_active (&sigfd_w))
+ close (sigfd);
+#endif
+
+#if EV_USE_TIMERFD
+ if (ev_is_active (&timerfd_w))
+ close (timerfd);
+#endif
+
+#if EV_USE_INOTIFY
+ if (fs_fd >= 0)
+ close (fs_fd);
+#endif
+
+ if (backend_fd >= 0)
+ close (backend_fd);
+
+#if EV_USE_IOCP
+ if (backend == EVBACKEND_IOCP ) iocp_destroy (EV_A);
+#endif
+#if EV_USE_PORT
+ if (backend == EVBACKEND_PORT ) port_destroy (EV_A);
+#endif
+#if EV_USE_KQUEUE
+ if (backend == EVBACKEND_KQUEUE ) kqueue_destroy (EV_A);
+#endif
+#if EV_USE_IOURING
+ if (backend == EVBACKEND_IOURING ) iouring_destroy (EV_A);
+#endif
+#if EV_USE_LINUXAIO
+ if (backend == EVBACKEND_LINUXAIO) linuxaio_destroy (EV_A);
+#endif
+#if EV_USE_EPOLL
+ if (backend == EVBACKEND_EPOLL ) epoll_destroy (EV_A);
+#endif
+#if EV_USE_POLL
+ if (backend == EVBACKEND_POLL ) poll_destroy (EV_A);
+#endif
+#if EV_USE_SELECT
+ if (backend == EVBACKEND_SELECT ) select_destroy (EV_A);
+#endif
+
+ for (i = NUMPRI; i--; )
+ {
+ array_free (pending, [i]);
+#if EV_IDLE_ENABLE
+ array_free (idle, [i]);
+#endif
+ }
+
+ ev_free (anfds); anfds = 0; anfdmax = 0;
+
+ /* have to use the microsoft-never-gets-it-right macro */
+ array_free (rfeed, EMPTY);
+ array_free (fdchange, EMPTY);
+ array_free (timer, EMPTY);
+#if EV_PERIODIC_ENABLE
+ array_free (periodic, EMPTY);
+#endif
+#if EV_FORK_ENABLE
+ array_free (fork, EMPTY);
+#endif
+#if EV_CLEANUP_ENABLE
+ array_free (cleanup, EMPTY);
+#endif
+ array_free (prepare, EMPTY);
+ array_free (check, EMPTY);
+#if EV_ASYNC_ENABLE
+ array_free (async, EMPTY);
+#endif
+
+ backend = 0;
+
+#if EV_MULTIPLICITY
+ if (ev_is_default_loop (EV_A))
+#endif
+ ev_default_loop_ptr = 0;
+#if EV_MULTIPLICITY
+ else
+ ev_free (EV_A);
+#endif
+}
+
+#if EV_USE_INOTIFY
+inline_size void infy_fork (EV_P);
+#endif
+
+inline_size void
+loop_fork (EV_P)
+{
+#if EV_USE_PORT
+ if (backend == EVBACKEND_PORT ) port_fork (EV_A);
+#endif
+#if EV_USE_KQUEUE
+ if (backend == EVBACKEND_KQUEUE ) kqueue_fork (EV_A);
+#endif
+#if EV_USE_IOURING
+ if (backend == EVBACKEND_IOURING ) iouring_fork (EV_A);
+#endif
+#if EV_USE_LINUXAIO
+ if (backend == EVBACKEND_LINUXAIO) linuxaio_fork (EV_A);
+#endif
+#if EV_USE_EPOLL
+ if (backend == EVBACKEND_EPOLL ) epoll_fork (EV_A);
+#endif
+#if EV_USE_INOTIFY
+ infy_fork (EV_A);
+#endif
+
+ if (postfork != 2)
+ {
+ #if EV_USE_SIGNALFD
+ /* surprisingly, nothing needs to be done for signalfd, accoridng to docs, it does the right thing on fork */
+ #endif
+
+ #if EV_USE_TIMERFD
+ if (ev_is_active (&timerfd_w))
+ {
+ ev_ref (EV_A);
+ ev_io_stop (EV_A_ &timerfd_w);
+
+ close (timerfd);
+ timerfd = -2;
+
+ evtimerfd_init (EV_A);
+ /* reschedule periodics, in case we missed something */
+ ev_feed_event (EV_A_ &timerfd_w, EV_CUSTOM);
+ }
+ #endif
+
+ #if EV_SIGNAL_ENABLE || EV_ASYNC_ENABLE
+ if (ev_is_active (&pipe_w))
+ {
+ /* pipe_write_wanted must be false now, so modifying fd vars should be safe */
+
+ ev_ref (EV_A);
+ ev_io_stop (EV_A_ &pipe_w);
+
+ if (evpipe [0] >= 0)
+ EV_WIN32_CLOSE_FD (evpipe [0]);
+
+ evpipe_init (EV_A);
+ /* iterate over everything, in case we missed something before */
+ ev_feed_event (EV_A_ &pipe_w, EV_CUSTOM);
+ }
+ #endif
+ }
+
+ postfork = 0;
+}
+
+#if EV_MULTIPLICITY
+
+ecb_cold
+struct ev_loop *
+ev_loop_new (unsigned int flags) EV_NOEXCEPT
+{
+ EV_P = (struct ev_loop *)ev_malloc (sizeof (struct ev_loop));
+
+ memset (EV_A, 0, sizeof (struct ev_loop));
+ loop_init (EV_A_ flags);
+
+ if (ev_backend (EV_A))
+ return EV_A;
+
+ ev_free (EV_A);
+ return 0;
+}
+
+#endif /* multiplicity */
+
+#if EV_VERIFY
+ecb_noinline ecb_cold
+static void
+verify_watcher (EV_P_ W w)
+{
+ assert (("libev: watcher has invalid priority", ABSPRI (w) >= 0 && ABSPRI (w) < NUMPRI));
+
+ if (w->pending)
+ assert (("libev: pending watcher not on pending queue", pendings [ABSPRI (w)][w->pending - 1].w == w));
+}
+
+ecb_noinline ecb_cold
+static void
+verify_heap (EV_P_ ANHE *heap, int N)
+{
+ int i;
+
+ for (i = HEAP0; i < N + HEAP0; ++i)
+ {
+ assert (("libev: active index mismatch in heap", ev_active (ANHE_w (heap [i])) == i));
+ assert (("libev: heap condition violated", i == HEAP0 || ANHE_at (heap [HPARENT (i)]) <= ANHE_at (heap [i])));
+ assert (("libev: heap at cache mismatch", ANHE_at (heap [i]) == ev_at (ANHE_w (heap [i]))));
+
+ verify_watcher (EV_A_ (W)ANHE_w (heap [i]));
+ }
+}
+
+ecb_noinline ecb_cold
+static void
+array_verify (EV_P_ W *ws, int cnt)
+{
+ while (cnt--)
+ {
+ assert (("libev: active index mismatch", ev_active (ws [cnt]) == cnt + 1));
+ verify_watcher (EV_A_ ws [cnt]);
+ }
+}
+#endif
+
+#if EV_FEATURE_API
+void ecb_cold
+ev_verify (EV_P) EV_NOEXCEPT
+{
+#if EV_VERIFY
+ int i;
+ WL w, w2;
+
+ assert (activecnt >= -1);
+
+ assert (fdchangemax >= fdchangecnt);
+ for (i = 0; i < fdchangecnt; ++i)
+ assert (("libev: negative fd in fdchanges", fdchanges [i] >= 0));
+
+ assert (anfdmax >= 0);
+ for (i = 0; i < anfdmax; ++i)
+ {
+ int j = 0;
+
+ for (w = w2 = anfds [i].head; w; w = w->next)
+ {
+ verify_watcher (EV_A_ (W)w);
+
+ if (j++ & 1)
+ {
+ assert (("libev: io watcher list contains a loop", w != w2));
+ w2 = w2->next;
+ }
+
+ assert (("libev: inactive fd watcher on anfd list", ev_active (w) == 1));
+ assert (("libev: fd mismatch between watcher and anfd", ((ev_io *)w)->fd == i));
+ }
+ }
+
+ assert (timermax >= timercnt);
+ verify_heap (EV_A_ timers, timercnt);
+
+#if EV_PERIODIC_ENABLE
+ assert (periodicmax >= periodiccnt);
+ verify_heap (EV_A_ periodics, periodiccnt);
+#endif
+
+ for (i = NUMPRI; i--; )
+ {
+ assert (pendingmax [i] >= pendingcnt [i]);
+#if EV_IDLE_ENABLE
+ assert (idleall >= 0);
+ assert (idlemax [i] >= idlecnt [i]);
+ array_verify (EV_A_ (W *)idles [i], idlecnt [i]);
+#endif
+ }
+
+#if EV_FORK_ENABLE
+ assert (forkmax >= forkcnt);
+ array_verify (EV_A_ (W *)forks, forkcnt);
+#endif
+
+#if EV_CLEANUP_ENABLE
+ assert (cleanupmax >= cleanupcnt);
+ array_verify (EV_A_ (W *)cleanups, cleanupcnt);
+#endif
+
+#if EV_ASYNC_ENABLE
+ assert (asyncmax >= asynccnt);
+ array_verify (EV_A_ (W *)asyncs, asynccnt);
+#endif
+
+#if EV_PREPARE_ENABLE
+ assert (preparemax >= preparecnt);
+ array_verify (EV_A_ (W *)prepares, preparecnt);
+#endif
+
+#if EV_CHECK_ENABLE
+ assert (checkmax >= checkcnt);
+ array_verify (EV_A_ (W *)checks, checkcnt);
+#endif
+
+# if 0
+#if EV_CHILD_ENABLE
+ for (w = (ev_child *)childs [chain & ((EV_PID_HASHSIZE) - 1)]; w; w = (ev_child *)((WL)w)->next)
+ for (signum = EV_NSIG; signum--; ) if (signals [signum].pending)
+#endif
+# endif
+#endif
+}
+#endif
+
+#if EV_MULTIPLICITY
+ecb_cold
+struct ev_loop *
+#else
+int
+#endif
+ev_default_loop (unsigned int flags) EV_NOEXCEPT
+{
+ if (!ev_default_loop_ptr)
+ {
+#if EV_MULTIPLICITY
+ EV_P = ev_default_loop_ptr = &default_loop_struct;
+#else
+ ev_default_loop_ptr = 1;
+#endif
+
+ loop_init (EV_A_ flags);
+
+ if (ev_backend (EV_A))
+ {
+#if EV_CHILD_ENABLE
+ ev_signal_init (&childev, childcb, SIGCHLD);
+ ev_set_priority (&childev, EV_MAXPRI);
+ ev_signal_start (EV_A_ &childev);
+ ev_unref (EV_A); /* child watcher should not keep loop alive */
+#endif
+ }
+ else
+ ev_default_loop_ptr = 0;
+ }
+
+ return ev_default_loop_ptr;
+}
+
+void
+ev_loop_fork (EV_P) EV_NOEXCEPT
+{
+ postfork = 1;
+}
+
+/*****************************************************************************/
+
+void
+ev_invoke (EV_P_ void *w, int revents)
+{
+ EV_CB_INVOKE ((W)w, revents);
+}
+
+unsigned int
+ev_pending_count (EV_P) EV_NOEXCEPT
+{
+ int pri;
+ unsigned int count = 0;
+
+ for (pri = NUMPRI; pri--; )
+ count += pendingcnt [pri];
+
+ return count;
+}
+
+ecb_noinline
+void
+ev_invoke_pending (EV_P)
+{
+ pendingpri = NUMPRI;
+
+ do
+ {
+ --pendingpri;
+
+ /* pendingpri possibly gets modified in the inner loop */
+ while (pendingcnt [pendingpri])
+ {
+ ANPENDING *p = pendings [pendingpri] + --pendingcnt [pendingpri];
+
+ p->w->pending = 0;
+ EV_CB_INVOKE (p->w, p->events);
+ EV_FREQUENT_CHECK;
+ }
+ }
+ while (pendingpri);
+}
+
+#if EV_IDLE_ENABLE
+/* make idle watchers pending. this handles the "call-idle */
+/* only when higher priorities are idle" logic */
+inline_size void
+idle_reify (EV_P)
+{
+ if (ecb_expect_false (idleall))
+ {
+ int pri;
+
+ for (pri = NUMPRI; pri--; )
+ {
+ if (pendingcnt [pri])
+ break;
+
+ if (idlecnt [pri])
+ {
+ queue_events (EV_A_ (W *)idles [pri], idlecnt [pri], EV_IDLE);
+ break;
+ }
+ }
+ }
+}
+#endif
+
+/* make timers pending */
+inline_size void
+timers_reify (EV_P)
+{
+ EV_FREQUENT_CHECK;
+
+ if (timercnt && ANHE_at (timers [HEAP0]) < mn_now)
+ {
+ do
+ {
+ ev_timer *w = (ev_timer *)ANHE_w (timers [HEAP0]);
+
+ /*assert (("libev: inactive timer on timer heap detected", ev_is_active (w)));*/
+
+ /* first reschedule or stop timer */
+ if (w->repeat)
+ {
+ ev_at (w) += w->repeat;
+ if (ev_at (w) < mn_now)
+ ev_at (w) = mn_now;
+
+ assert (("libev: negative ev_timer repeat value found while processing timers", w->repeat > EV_TS_CONST (0.)));
+
+ ANHE_at_cache (timers [HEAP0]);
+ downheap (timers, timercnt, HEAP0);
+ }
+ else
+ ev_timer_stop (EV_A_ w); /* nonrepeating: stop timer */
+
+ EV_FREQUENT_CHECK;
+ feed_reverse (EV_A_ (W)w);
+ }
+ while (timercnt && ANHE_at (timers [HEAP0]) < mn_now);
+
+ feed_reverse_done (EV_A_ EV_TIMER);
+ }
+}
+
+#if EV_PERIODIC_ENABLE
+
+ecb_noinline
+static void
+periodic_recalc (EV_P_ ev_periodic *w)
+{
+ ev_tstamp interval = w->interval > MIN_INTERVAL ? w->interval : MIN_INTERVAL;
+ ev_tstamp at = w->offset + interval * ev_floor ((ev_rt_now - w->offset) / interval);
+
+ /* the above almost always errs on the low side */
+ while (at <= ev_rt_now)
+ {
+ ev_tstamp nat = at + w->interval;
+
+ /* when resolution fails us, we use ev_rt_now */
+ if (ecb_expect_false (nat == at))
+ {
+ at = ev_rt_now;
+ break;
+ }
+
+ at = nat;
+ }
+
+ ev_at (w) = at;
+}
+
+/* make periodics pending */
+inline_size void
+periodics_reify (EV_P)
+{
+ EV_FREQUENT_CHECK;
+
+ while (periodiccnt && ANHE_at (periodics [HEAP0]) < ev_rt_now)
+ {
+ do
+ {
+ ev_periodic *w = (ev_periodic *)ANHE_w (periodics [HEAP0]);
+
+ /*assert (("libev: inactive timer on periodic heap detected", ev_is_active (w)));*/
+
+ /* first reschedule or stop timer */
+ if (w->reschedule_cb)
+ {
+ ev_at (w) = w->reschedule_cb (w, ev_rt_now);
+
+ assert (("libev: ev_periodic reschedule callback returned time in the past", ev_at (w) >= ev_rt_now));
+
+ ANHE_at_cache (periodics [HEAP0]);
+ downheap (periodics, periodiccnt, HEAP0);
+ }
+ else if (w->interval)
+ {
+ periodic_recalc (EV_A_ w);
+ ANHE_at_cache (periodics [HEAP0]);
+ downheap (periodics, periodiccnt, HEAP0);
+ }
+ else
+ ev_periodic_stop (EV_A_ w); /* nonrepeating: stop timer */
+
+ EV_FREQUENT_CHECK;
+ feed_reverse (EV_A_ (W)w);
+ }
+ while (periodiccnt && ANHE_at (periodics [HEAP0]) < ev_rt_now);
+
+ feed_reverse_done (EV_A_ EV_PERIODIC);
+ }
+}
+
+/* simply recalculate all periodics */
+/* TODO: maybe ensure that at least one event happens when jumping forward? */
+ecb_noinline ecb_cold
+static void
+periodics_reschedule (EV_P)
+{
+ int i;
+
+ /* adjust periodics after time jump */
+ for (i = HEAP0; i < periodiccnt + HEAP0; ++i)
+ {
+ ev_periodic *w = (ev_periodic *)ANHE_w (periodics [i]);
+
+ if (w->reschedule_cb)
+ ev_at (w) = w->reschedule_cb (w, ev_rt_now);
+ else if (w->interval)
+ periodic_recalc (EV_A_ w);
+
+ ANHE_at_cache (periodics [i]);
+ }
+
+ reheap (periodics, periodiccnt);
+}
+#endif
+
+/* adjust all timers by a given offset */
+ecb_noinline ecb_cold
+static void
+timers_reschedule (EV_P_ ev_tstamp adjust)
+{
+ int i;
+
+ for (i = 0; i < timercnt; ++i)
+ {
+ ANHE *he = timers + i + HEAP0;
+ ANHE_w (*he)->at += adjust;
+ ANHE_at_cache (*he);
+ }
+}
+
+/* fetch new monotonic and realtime times from the kernel */
+/* also detect if there was a timejump, and act accordingly */
+inline_speed void
+time_update (EV_P_ ev_tstamp max_block)
+{
+#if EV_USE_MONOTONIC
+ if (ecb_expect_true (have_monotonic))
+ {
+ int i;
+ ev_tstamp odiff = rtmn_diff;
+
+ mn_now = get_clock ();
+
+ /* only fetch the realtime clock every 0.5*MIN_TIMEJUMP seconds */
+ /* interpolate in the meantime */
+ if (ecb_expect_true (mn_now - now_floor < EV_TS_CONST (MIN_TIMEJUMP * .5)))
+ {
+ ev_rt_now = rtmn_diff + mn_now;
+ return;
+ }
+
+ now_floor = mn_now;
+ ev_rt_now = ev_time ();
+
+ /* loop a few times, before making important decisions.
+ * on the choice of "4": one iteration isn't enough,
+ * in case we get preempted during the calls to
+ * ev_time and get_clock. a second call is almost guaranteed
+ * to succeed in that case, though. and looping a few more times
+ * doesn't hurt either as we only do this on time-jumps or
+ * in the unlikely event of having been preempted here.
+ */
+ for (i = 4; --i; )
+ {
+ ev_tstamp diff;
+ rtmn_diff = ev_rt_now - mn_now;
+
+ diff = odiff - rtmn_diff;
+
+ if (ecb_expect_true ((diff < EV_TS_CONST (0.) ? -diff : diff) < EV_TS_CONST (MIN_TIMEJUMP)))
+ return; /* all is well */
+
+ ev_rt_now = ev_time ();
+ mn_now = get_clock ();
+ now_floor = mn_now;
+ }
+
+ /* no timer adjustment, as the monotonic clock doesn't jump */
+ /* timers_reschedule (EV_A_ rtmn_diff - odiff) */
+# if EV_PERIODIC_ENABLE
+ periodics_reschedule (EV_A);
+# endif
+ }
+ else
+#endif
+ {
+ ev_rt_now = ev_time ();
+
+ if (ecb_expect_false (mn_now > ev_rt_now || ev_rt_now > mn_now + max_block + EV_TS_CONST (MIN_TIMEJUMP)))
+ {
+ /* adjust timers. this is easy, as the offset is the same for all of them */
+ timers_reschedule (EV_A_ ev_rt_now - mn_now);
+#if EV_PERIODIC_ENABLE
+ periodics_reschedule (EV_A);
+#endif
+ }
+
+ mn_now = ev_rt_now;
+ }
+}
+
+int
+ev_run (EV_P_ int flags)
+{
+#if EV_FEATURE_API
+ ++loop_depth;
+#endif
+
+ assert (("libev: ev_loop recursion during release detected", loop_done != EVBREAK_RECURSE));
+
+ loop_done = EVBREAK_CANCEL;
+
+ EV_INVOKE_PENDING; /* in case we recurse, ensure ordering stays nice and clean */
+
+ do
+ {
+#if EV_VERIFY >= 2
+ ev_verify (EV_A);
+#endif
+
+#ifndef _WIN32
+ if (ecb_expect_false (curpid)) /* penalise the forking check even more */
+ if (ecb_expect_false (getpid () != curpid))
+ {
+ curpid = getpid ();
+ postfork = 1;
+ }
+#endif
+
+#if EV_FORK_ENABLE
+ /* we might have forked, so queue fork handlers */
+ if (ecb_expect_false (postfork))
+ if (forkcnt)
+ {
+ queue_events (EV_A_ (W *)forks, forkcnt, EV_FORK);
+ EV_INVOKE_PENDING;
+ }
+#endif
+
+#if EV_PREPARE_ENABLE
+ /* queue prepare watchers (and execute them) */
+ if (ecb_expect_false (preparecnt))
+ {
+ queue_events (EV_A_ (W *)prepares, preparecnt, EV_PREPARE);
+ EV_INVOKE_PENDING;
+ }
+#endif
+
+ if (ecb_expect_false (loop_done))
+ break;
+
+ /* we might have forked, so reify kernel state if necessary */
+ if (ecb_expect_false (postfork))
+ loop_fork (EV_A);
+
+ /* update fd-related kernel structures */
+ fd_reify (EV_A);
+
+ /* calculate blocking time */
+ {
+ ev_tstamp waittime = 0.;
+ ev_tstamp sleeptime = 0.;
+
+ /* remember old timestamp for io_blocktime calculation */
+ ev_tstamp prev_mn_now = mn_now;
+
+ /* update time to cancel out callback processing overhead */
+ time_update (EV_A_ EV_TS_CONST (EV_TSTAMP_HUGE));
+
+ /* from now on, we want a pipe-wake-up */
+ pipe_write_wanted = 1;
+
+ ECB_MEMORY_FENCE; /* make sure pipe_write_wanted is visible before we check for potential skips */
+
+ if (ecb_expect_true (!(flags & EVRUN_NOWAIT || idleall || !activecnt || pipe_write_skipped)))
+ {
+ waittime = EV_TS_CONST (MAX_BLOCKTIME);
+#if EV_USE_MONOTONIC
+ if (ecb_expect_true (have_monotonic)) {
+#if EV_USE_TIMERFD
+ /* sleep a lot longer when we can reliably detect timejumps */
+ if (ecb_expect_true (timerfd != -1))
+ waittime = EV_TS_CONST (MAX_BLOCKTIME2);
+#endif
+#if !EV_PERIODIC_ENABLE
+ /* without periodics but with monotonic clock there is no need */
+ /* for any time jump detection, so sleep longer */
+
+ waittime = EV_TS_CONST (MAX_BLOCKTIME2);
+#endif
+ }
+#endif /* EV_USE_MONOTONIC */
+ if (timercnt)
+ {
+ ev_tstamp to = ANHE_at (timers [HEAP0]) - mn_now;
+ if (waittime > to) waittime = to;
+ }
+
+#if EV_PERIODIC_ENABLE
+ if (periodiccnt)
+ {
+ ev_tstamp to = ANHE_at (periodics [HEAP0]) - ev_rt_now;
+ if (waittime > to) waittime = to;
+ }
+#endif
+
+ /* don't let timeouts decrease the waittime below timeout_blocktime */
+ if (ecb_expect_false (waittime < timeout_blocktime))
+ waittime = timeout_blocktime;
+
+ /* now there are two more special cases left, either we have
+ * already-expired timers, so we should not sleep, or we have timers
+ * that expire very soon, in which case we need to wait for a minimum
+ * amount of time for some event loop backends.
+ */
+ if (ecb_expect_false (waittime < backend_mintime))
+ waittime = waittime <= EV_TS_CONST (0.)
+ ? EV_TS_CONST (0.)
+ : backend_mintime;
+
+ /* extra check because io_blocktime is commonly 0 */
+ if (ecb_expect_false (io_blocktime))
+ {
+ sleeptime = io_blocktime - (mn_now - prev_mn_now);
+
+ if (sleeptime > waittime - backend_mintime)
+ sleeptime = waittime - backend_mintime;
+
+ if (ecb_expect_true (sleeptime > EV_TS_CONST (0.)))
+ {
+ ev_sleep (sleeptime);
+ waittime -= sleeptime;
+ }
+ }
+ }
+
+#if EV_FEATURE_API
+ ++loop_count;
+#endif
+ assert ((loop_done = EVBREAK_RECURSE, 1)); /* assert for side effect */
+ backend_poll (EV_A_ waittime);
+ assert ((loop_done = EVBREAK_CANCEL, 1)); /* assert for side effect */
+
+ pipe_write_wanted = 0; /* just an optimisation, no fence needed */
+
+ ECB_MEMORY_FENCE_ACQUIRE;
+ if (pipe_write_skipped)
+ {
+ assert (("libev: pipe_w not active, but pipe not written", ev_is_active (&pipe_w)));
+ ev_feed_event (EV_A_ &pipe_w, EV_CUSTOM);
+ }
+
+ /* update ev_rt_now, do magic */
+ time_update (EV_A_ waittime + sleeptime);
+ }
+
+ /* queue pending timers and reschedule them */
+ timers_reify (EV_A); /* relative timers called last */
+#if EV_PERIODIC_ENABLE
+ periodics_reify (EV_A); /* absolute timers called first */
+#endif
+
+#if EV_IDLE_ENABLE
+ /* queue idle watchers unless other events are pending */
+ idle_reify (EV_A);
+#endif
+
+#if EV_CHECK_ENABLE
+ /* queue check watchers, to be executed first */
+ if (ecb_expect_false (checkcnt))
+ queue_events (EV_A_ (W *)checks, checkcnt, EV_CHECK);
+#endif
+
+ EV_INVOKE_PENDING;
+ }
+ while (ecb_expect_true (
+ activecnt
+ && !loop_done
+ && !(flags & (EVRUN_ONCE | EVRUN_NOWAIT))
+ ));
+
+ if (loop_done == EVBREAK_ONE)
+ loop_done = EVBREAK_CANCEL;
+
+#if EV_FEATURE_API
+ --loop_depth;
+#endif
+
+ return activecnt;
+}
+
+void
+ev_break (EV_P_ int how) EV_NOEXCEPT
+{
+ loop_done = how;
+}
+
+void
+ev_ref (EV_P) EV_NOEXCEPT
+{
+ ++activecnt;
+}
+
+void
+ev_unref (EV_P) EV_NOEXCEPT
+{
+ --activecnt;
+}
+
+void
+ev_now_update (EV_P) EV_NOEXCEPT
+{
+ time_update (EV_A_ EV_TSTAMP_HUGE);
+}
+
+void
+ev_suspend (EV_P) EV_NOEXCEPT
+{
+ ev_now_update (EV_A);
+}
+
+void
+ev_resume (EV_P) EV_NOEXCEPT
+{
+ ev_tstamp mn_prev = mn_now;
+
+ ev_now_update (EV_A);
+ timers_reschedule (EV_A_ mn_now - mn_prev);
+#if EV_PERIODIC_ENABLE
+ /* TODO: really do this? */
+ periodics_reschedule (EV_A);
+#endif
+}
+
+/*****************************************************************************/
+/* singly-linked list management, used when the expected list length is short */
+
+inline_size void
+wlist_add (WL *head, WL elem)
+{
+ elem->next = *head;
+ *head = elem;
+}
+
+inline_size void
+wlist_del (WL *head, WL elem)
+{
+ while (*head)
+ {
+ if (ecb_expect_true (*head == elem))
+ {
+ *head = elem->next;
+ break;
+ }
+
+ head = &(*head)->next;
+ }
+}
+
+/* internal, faster, version of ev_clear_pending */
+inline_speed void
+clear_pending (EV_P_ W w)
+{
+ if (w->pending)
+ {
+ pendings [ABSPRI (w)][w->pending - 1].w = (W)&pending_w;
+ w->pending = 0;
+ }
+}
+
+int
+ev_clear_pending (EV_P_ void *w) EV_NOEXCEPT
+{
+ W w_ = (W)w;
+ int pending = w_->pending;
+
+ if (ecb_expect_true (pending))
+ {
+ ANPENDING *p = pendings [ABSPRI (w_)] + pending - 1;
+ p->w = (W)&pending_w;
+ w_->pending = 0;
+ return p->events;
+ }
+ else
+ return 0;
+}
+
+inline_size void
+pri_adjust (EV_P_ W w)
+{
+ int pri = ev_priority (w);
+ pri = pri < EV_MINPRI ? EV_MINPRI : pri;
+ pri = pri > EV_MAXPRI ? EV_MAXPRI : pri;
+ ev_set_priority (w, pri);
+}
+
+inline_speed void
+ev_start (EV_P_ W w, int active)
+{
+ pri_adjust (EV_A_ w);
+ w->active = active;
+ ev_ref (EV_A);
+}
+
+inline_size void
+ev_stop (EV_P_ W w)
+{
+ ev_unref (EV_A);
+ w->active = 0;
+}
+
+/*****************************************************************************/
+
+ecb_noinline
+void
+ev_io_start (EV_P_ ev_io *w) EV_NOEXCEPT
+{
+ int fd = w->fd;
+
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ assert (("libev: ev_io_start called with negative fd", fd >= 0));
+ assert (("libev: ev_io_start called with illegal event mask", !(w->events & ~(EV__IOFDSET | EV_READ | EV_WRITE))));
+
+#if EV_VERIFY >= 2
+ assert (("libev: ev_io_start called on watcher with invalid fd", fd_valid (fd)));
+#endif
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, 1);
+ array_needsize (ANFD, anfds, anfdmax, fd + 1, array_needsize_zerofill);
+ wlist_add (&anfds[fd].head, (WL)w);
+
+ /* common bug, apparently */
+ assert (("libev: ev_io_start called with corrupted watcher", ((WL)w)->next != (WL)w));
+
+ fd_change (EV_A_ fd, (w->events & EV__IOFDSET) | EV_ANFD_REIFY);
+ w->events &= ~EV__IOFDSET;
+
+ EV_FREQUENT_CHECK;
+}
+
+ecb_noinline
+void
+ev_io_stop (EV_P_ ev_io *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ assert (("libev: ev_io_stop called with illegal fd (must stay constant after start!)", w->fd >= 0 && w->fd < anfdmax));
+
+#if EV_VERIFY >= 2
+ assert (("libev: ev_io_stop called on watcher with invalid fd", fd_valid (w->fd)));
+#endif
+ EV_FREQUENT_CHECK;
+
+ wlist_del (&anfds[w->fd].head, (WL)w);
+ ev_stop (EV_A_ (W)w);
+
+ fd_change (EV_A_ w->fd, EV_ANFD_REIFY);
+
+ EV_FREQUENT_CHECK;
+}
+
+ecb_noinline
+void
+ev_timer_start (EV_P_ ev_timer *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ ev_at (w) += mn_now;
+
+ assert (("libev: ev_timer_start called with negative timer repeat value", w->repeat >= 0.));
+
+ EV_FREQUENT_CHECK;
+
+ ++timercnt;
+ ev_start (EV_A_ (W)w, timercnt + HEAP0 - 1);
+ array_needsize (ANHE, timers, timermax, ev_active (w) + 1, array_needsize_noinit);
+ ANHE_w (timers [ev_active (w)]) = (WT)w;
+ ANHE_at_cache (timers [ev_active (w)]);
+ upheap (timers, ev_active (w));
+
+ EV_FREQUENT_CHECK;
+
+ /*assert (("libev: internal timer heap corruption", timers [ev_active (w)] == (WT)w));*/
+}
+
+ecb_noinline
+void
+ev_timer_stop (EV_P_ ev_timer *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ assert (("libev: internal timer heap corruption", ANHE_w (timers [active]) == (WT)w));
+
+ --timercnt;
+
+ if (ecb_expect_true (active < timercnt + HEAP0))
+ {
+ timers [active] = timers [timercnt + HEAP0];
+ adjustheap (timers, timercnt, active);
+ }
+ }
+
+ ev_at (w) -= mn_now;
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+
+ecb_noinline
+void
+ev_timer_again (EV_P_ ev_timer *w) EV_NOEXCEPT
+{
+ EV_FREQUENT_CHECK;
+
+ clear_pending (EV_A_ (W)w);
+
+ if (ev_is_active (w))
+ {
+ if (w->repeat)
+ {
+ ev_at (w) = mn_now + w->repeat;
+ ANHE_at_cache (timers [ev_active (w)]);
+ adjustheap (timers, timercnt, ev_active (w));
+ }
+ else
+ ev_timer_stop (EV_A_ w);
+ }
+ else if (w->repeat)
+ {
+ ev_at (w) = w->repeat;
+ ev_timer_start (EV_A_ w);
+ }
+
+ EV_FREQUENT_CHECK;
+}
+
+ev_tstamp
+ev_timer_remaining (EV_P_ ev_timer *w) EV_NOEXCEPT
+{
+ return ev_at (w) - (ev_is_active (w) ? mn_now : EV_TS_CONST (0.));
+}
+
+#if EV_PERIODIC_ENABLE
+ecb_noinline
+void
+ev_periodic_start (EV_P_ ev_periodic *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+#if EV_USE_TIMERFD
+ if (timerfd == -2)
+ evtimerfd_init (EV_A);
+#endif
+
+ if (w->reschedule_cb)
+ ev_at (w) = w->reschedule_cb (w, ev_rt_now);
+ else if (w->interval)
+ {
+ assert (("libev: ev_periodic_start called with negative interval value", w->interval >= 0.));
+ periodic_recalc (EV_A_ w);
+ }
+ else
+ ev_at (w) = w->offset;
+
+ EV_FREQUENT_CHECK;
+
+ ++periodiccnt;
+ ev_start (EV_A_ (W)w, periodiccnt + HEAP0 - 1);
+ array_needsize (ANHE, periodics, periodicmax, ev_active (w) + 1, array_needsize_noinit);
+ ANHE_w (periodics [ev_active (w)]) = (WT)w;
+ ANHE_at_cache (periodics [ev_active (w)]);
+ upheap (periodics, ev_active (w));
+
+ EV_FREQUENT_CHECK;
+
+ /*assert (("libev: internal periodic heap corruption", ANHE_w (periodics [ev_active (w)]) == (WT)w));*/
+}
+
+ecb_noinline
+void
+ev_periodic_stop (EV_P_ ev_periodic *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ assert (("libev: internal periodic heap corruption", ANHE_w (periodics [active]) == (WT)w));
+
+ --periodiccnt;
+
+ if (ecb_expect_true (active < periodiccnt + HEAP0))
+ {
+ periodics [active] = periodics [periodiccnt + HEAP0];
+ adjustheap (periodics, periodiccnt, active);
+ }
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+
+ecb_noinline
+void
+ev_periodic_again (EV_P_ ev_periodic *w) EV_NOEXCEPT
+{
+ /* TODO: use adjustheap and recalculation */
+ ev_periodic_stop (EV_A_ w);
+ ev_periodic_start (EV_A_ w);
+}
+#endif
+
+#ifndef SA_RESTART
+# define SA_RESTART 0
+#endif
+
+#if EV_SIGNAL_ENABLE
+
+ecb_noinline
+void
+ev_signal_start (EV_P_ ev_signal *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ assert (("libev: ev_signal_start called with illegal signal number", w->signum > 0 && w->signum < EV_NSIG));
+
+#if EV_MULTIPLICITY
+ assert (("libev: a signal must not be attached to two different loops",
+ !signals [w->signum - 1].loop || signals [w->signum - 1].loop == loop));
+
+ signals [w->signum - 1].loop = EV_A;
+ ECB_MEMORY_FENCE_RELEASE;
+#endif
+
+ EV_FREQUENT_CHECK;
+
+#if EV_USE_SIGNALFD
+ if (sigfd == -2)
+ {
+ sigfd = signalfd (-1, &sigfd_set, SFD_NONBLOCK | SFD_CLOEXEC);
+ if (sigfd < 0 && errno == EINVAL)
+ sigfd = signalfd (-1, &sigfd_set, 0); /* retry without flags */
+
+ if (sigfd >= 0)
+ {
+ fd_intern (sigfd); /* doing it twice will not hurt */
+
+ sigemptyset (&sigfd_set);
+
+ ev_io_init (&sigfd_w, sigfdcb, sigfd, EV_READ);
+ ev_set_priority (&sigfd_w, EV_MAXPRI);
+ ev_io_start (EV_A_ &sigfd_w);
+ ev_unref (EV_A); /* signalfd watcher should not keep loop alive */
+ }
+ }
+
+ if (sigfd >= 0)
+ {
+ /* TODO: check .head */
+ sigaddset (&sigfd_set, w->signum);
+ sigprocmask (SIG_BLOCK, &sigfd_set, 0);
+
+ signalfd (sigfd, &sigfd_set, 0);
+ }
+#endif
+
+ ev_start (EV_A_ (W)w, 1);
+ wlist_add (&signals [w->signum - 1].head, (WL)w);
+
+ if (!((WL)w)->next)
+# if EV_USE_SIGNALFD
+ if (sigfd < 0) /*TODO*/
+# endif
+ {
+# ifdef _WIN32
+ evpipe_init (EV_A);
+
+ signal (w->signum, ev_sighandler);
+# else
+ struct sigaction sa;
+
+ evpipe_init (EV_A);
+
+ sa.sa_handler = ev_sighandler;
+ sigfillset (&sa.sa_mask);
+ sa.sa_flags = SA_RESTART; /* if restarting works we save one iteration */
+ sigaction (w->signum, &sa, 0);
+
+ if (origflags & EVFLAG_NOSIGMASK)
+ {
+ sigemptyset (&sa.sa_mask);
+ sigaddset (&sa.sa_mask, w->signum);
+ sigprocmask (SIG_UNBLOCK, &sa.sa_mask, 0);
+ }
+#endif
+ }
+
+ EV_FREQUENT_CHECK;
+}
+
+ecb_noinline
+void
+ev_signal_stop (EV_P_ ev_signal *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ wlist_del (&signals [w->signum - 1].head, (WL)w);
+ ev_stop (EV_A_ (W)w);
+
+ if (!signals [w->signum - 1].head)
+ {
+#if EV_MULTIPLICITY
+ signals [w->signum - 1].loop = 0; /* unattach from signal */
+#endif
+#if EV_USE_SIGNALFD
+ if (sigfd >= 0)
+ {
+ sigset_t ss;
+
+ sigemptyset (&ss);
+ sigaddset (&ss, w->signum);
+ sigdelset (&sigfd_set, w->signum);
+
+ signalfd (sigfd, &sigfd_set, 0);
+ sigprocmask (SIG_UNBLOCK, &ss, 0);
+ }
+ else
+#endif
+ signal (w->signum, SIG_DFL);
+ }
+
+ EV_FREQUENT_CHECK;
+}
+
+#endif
+
+#if EV_CHILD_ENABLE
+
+void
+ev_child_start (EV_P_ ev_child *w) EV_NOEXCEPT
+{
+#if EV_MULTIPLICITY
+ assert (("libev: child watchers are only supported in the default loop", loop == ev_default_loop_ptr));
+#endif
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, 1);
+ wlist_add (&childs [w->pid & ((EV_PID_HASHSIZE) - 1)], (WL)w);
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_child_stop (EV_P_ ev_child *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ wlist_del (&childs [w->pid & ((EV_PID_HASHSIZE) - 1)], (WL)w);
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+
+#endif
+
+#if EV_STAT_ENABLE
+
+# ifdef _WIN32
+# undef lstat
+# define lstat(a,b) _stati64 (a,b)
+# endif
+
+#define DEF_STAT_INTERVAL 5.0074891
+#define NFS_STAT_INTERVAL 30.1074891 /* for filesystems potentially failing inotify */
+#define MIN_STAT_INTERVAL 0.1074891
+
+ecb_noinline static void stat_timer_cb (EV_P_ ev_timer *w_, int revents);
+
+#if EV_USE_INOTIFY
+
+/* the * 2 is to allow for alignment padding, which for some reason is >> 8 */
+# define EV_INOTIFY_BUFSIZE (sizeof (struct inotify_event) * 2 + NAME_MAX)
+
+ecb_noinline
+static void
+infy_add (EV_P_ ev_stat *w)
+{
+ w->wd = inotify_add_watch (fs_fd, w->path,
+ IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF | IN_MODIFY
+ | IN_CREATE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO
+ | IN_DONT_FOLLOW | IN_MASK_ADD);
+
+ if (w->wd >= 0)
+ {
+ struct statfs sfs;
+
+ /* now local changes will be tracked by inotify, but remote changes won't */
+ /* unless the filesystem is known to be local, we therefore still poll */
+ /* also do poll on <2.6.25, but with normal frequency */
+
+ if (!fs_2625)
+ w->timer.repeat = w->interval ? w->interval : DEF_STAT_INTERVAL;
+ else if (!statfs (w->path, &sfs)
+ && (sfs.f_type == 0x1373 /* devfs */
+ || sfs.f_type == 0x4006 /* fat */
+ || sfs.f_type == 0x4d44 /* msdos */
+ || sfs.f_type == 0xEF53 /* ext2/3 */
+ || sfs.f_type == 0x72b6 /* jffs2 */
+ || sfs.f_type == 0x858458f6 /* ramfs */
+ || sfs.f_type == 0x5346544e /* ntfs */
+ || sfs.f_type == 0x3153464a /* jfs */
+ || sfs.f_type == 0x9123683e /* btrfs */
+ || sfs.f_type == 0x52654973 /* reiser3 */
+ || sfs.f_type == 0x01021994 /* tmpfs */
+ || sfs.f_type == 0x58465342 /* xfs */))
+ w->timer.repeat = 0.; /* filesystem is local, kernel new enough */
+ else
+ w->timer.repeat = w->interval ? w->interval : NFS_STAT_INTERVAL; /* remote, use reduced frequency */
+ }
+ else
+ {
+ /* can't use inotify, continue to stat */
+ w->timer.repeat = w->interval ? w->interval : DEF_STAT_INTERVAL;
+
+ /* if path is not there, monitor some parent directory for speedup hints */
+ /* note that exceeding the hardcoded path limit is not a correctness issue, */
+ /* but an efficiency issue only */
+ if ((errno == ENOENT || errno == EACCES) && strlen (w->path) < 4096)
+ {
+ char path [4096];
+ strcpy (path, w->path);
+
+ do
+ {
+ int mask = IN_MASK_ADD | IN_DELETE_SELF | IN_MOVE_SELF
+ | (errno == EACCES ? IN_ATTRIB : IN_CREATE | IN_MOVED_TO);
+
+ char *pend = strrchr (path, '/');
+
+ if (!pend || pend == path)
+ break;
+
+ *pend = 0;
+ w->wd = inotify_add_watch (fs_fd, path, mask);
+ }
+ while (w->wd < 0 && (errno == ENOENT || errno == EACCES));
+ }
+ }
+
+ if (w->wd >= 0)
+ wlist_add (&fs_hash [w->wd & ((EV_INOTIFY_HASHSIZE) - 1)].head, (WL)w);
+
+ /* now re-arm timer, if required */
+ if (ev_is_active (&w->timer)) ev_ref (EV_A);
+ ev_timer_again (EV_A_ &w->timer);
+ if (ev_is_active (&w->timer)) ev_unref (EV_A);
+}
+
+ecb_noinline
+static void
+infy_del (EV_P_ ev_stat *w)
+{
+ int slot;
+ int wd = w->wd;
+
+ if (wd < 0)
+ return;
+
+ w->wd = -2;
+ slot = wd & ((EV_INOTIFY_HASHSIZE) - 1);
+ wlist_del (&fs_hash [slot].head, (WL)w);
+
+ /* remove this watcher, if others are watching it, they will rearm */
+ inotify_rm_watch (fs_fd, wd);
+}
+
+ecb_noinline
+static void
+infy_wd (EV_P_ int slot, int wd, struct inotify_event *ev)
+{
+ if (slot < 0)
+ /* overflow, need to check for all hash slots */
+ for (slot = 0; slot < (EV_INOTIFY_HASHSIZE); ++slot)
+ infy_wd (EV_A_ slot, wd, ev);
+ else
+ {
+ WL w_;
+
+ for (w_ = fs_hash [slot & ((EV_INOTIFY_HASHSIZE) - 1)].head; w_; )
+ {
+ ev_stat *w = (ev_stat *)w_;
+ w_ = w_->next; /* lets us remove this watcher and all before it */
+
+ if (w->wd == wd || wd == -1)
+ {
+ if (ev->mask & (IN_IGNORED | IN_UNMOUNT | IN_DELETE_SELF))
+ {
+ wlist_del (&fs_hash [slot & ((EV_INOTIFY_HASHSIZE) - 1)].head, (WL)w);
+ w->wd = -1;
+ infy_add (EV_A_ w); /* re-add, no matter what */
+ }
+
+ stat_timer_cb (EV_A_ &w->timer, 0);
+ }
+ }
+ }
+}
+
+static void
+infy_cb (EV_P_ ev_io *w, int revents)
+{
+ char buf [EV_INOTIFY_BUFSIZE];
+ int ofs;
+ int len = read (fs_fd, buf, sizeof (buf));
+
+ for (ofs = 0; ofs < len; )
+ {
+ struct inotify_event *ev = (struct inotify_event *)(buf + ofs);
+ infy_wd (EV_A_ ev->wd, ev->wd, ev);
+ ofs += sizeof (struct inotify_event) + ev->len;
+ }
+}
+
+inline_size ecb_cold
+void
+ev_check_2625 (EV_P)
+{
+ /* kernels < 2.6.25 are borked
+ * http://www.ussg.indiana.edu/hypermail/linux/kernel/0711.3/1208.html
+ */
+ if (ev_linux_version () < 0x020619)
+ return;
+
+ fs_2625 = 1;
+}
+
+inline_size int
+infy_newfd (void)
+{
+#if defined IN_CLOEXEC && defined IN_NONBLOCK
+ int fd = inotify_init1 (IN_CLOEXEC | IN_NONBLOCK);
+ if (fd >= 0)
+ return fd;
+#endif
+ return inotify_init ();
+}
+
+inline_size void
+infy_init (EV_P)
+{
+ if (fs_fd != -2)
+ return;
+
+ fs_fd = -1;
+
+ ev_check_2625 (EV_A);
+
+ fs_fd = infy_newfd ();
+
+ if (fs_fd >= 0)
+ {
+ fd_intern (fs_fd);
+ ev_io_init (&fs_w, infy_cb, fs_fd, EV_READ);
+ ev_set_priority (&fs_w, EV_MAXPRI);
+ ev_io_start (EV_A_ &fs_w);
+ ev_unref (EV_A);
+ }
+}
+
+inline_size void
+infy_fork (EV_P)
+{
+ int slot;
+
+ if (fs_fd < 0)
+ return;
+
+ ev_ref (EV_A);
+ ev_io_stop (EV_A_ &fs_w);
+ close (fs_fd);
+ fs_fd = infy_newfd ();
+
+ if (fs_fd >= 0)
+ {
+ fd_intern (fs_fd);
+ ev_io_set (&fs_w, fs_fd, EV_READ);
+ ev_io_start (EV_A_ &fs_w);
+ ev_unref (EV_A);
+ }
+
+ for (slot = 0; slot < (EV_INOTIFY_HASHSIZE); ++slot)
+ {
+ WL w_ = fs_hash [slot].head;
+ fs_hash [slot].head = 0;
+
+ while (w_)
+ {
+ ev_stat *w = (ev_stat *)w_;
+ w_ = w_->next; /* lets us add this watcher */
+
+ w->wd = -1;
+
+ if (fs_fd >= 0)
+ infy_add (EV_A_ w); /* re-add, no matter what */
+ else
+ {
+ w->timer.repeat = w->interval ? w->interval : DEF_STAT_INTERVAL;
+ if (ev_is_active (&w->timer)) ev_ref (EV_A);
+ ev_timer_again (EV_A_ &w->timer);
+ if (ev_is_active (&w->timer)) ev_unref (EV_A);
+ }
+ }
+ }
+}
+
+#endif
+
+#ifdef _WIN32
+# define EV_LSTAT(p,b) _stati64 (p, b)
+#else
+# define EV_LSTAT(p,b) lstat (p, b)
+#endif
+
+void
+ev_stat_stat (EV_P_ ev_stat *w) EV_NOEXCEPT
+{
+ if (lstat (w->path, &w->attr) < 0)
+ w->attr.st_nlink = 0;
+ else if (!w->attr.st_nlink)
+ w->attr.st_nlink = 1;
+}
+
+ecb_noinline
+static void
+stat_timer_cb (EV_P_ ev_timer *w_, int revents)
+{
+ ev_stat *w = (ev_stat *)(((char *)w_) - offsetof (ev_stat, timer));
+
+ ev_statdata prev = w->attr;
+ ev_stat_stat (EV_A_ w);
+
+ /* memcmp doesn't work on netbsd, they.... do stuff to their struct stat */
+ if (
+ prev.st_dev != w->attr.st_dev
+ || prev.st_ino != w->attr.st_ino
+ || prev.st_mode != w->attr.st_mode
+ || prev.st_nlink != w->attr.st_nlink
+ || prev.st_uid != w->attr.st_uid
+ || prev.st_gid != w->attr.st_gid
+ || prev.st_rdev != w->attr.st_rdev
+ || prev.st_size != w->attr.st_size
+ /* || prev.st_atime != w->attr.st_atime */ /* Rspamd: to avoid constant maps reload */
+ || prev.st_mtime != w->attr.st_mtime
+ || prev.st_ctime != w->attr.st_ctime
+ ) {
+ /* we only update w->prev on actual differences */
+ /* in case we test more often than invoke the callback, */
+ /* to ensure that prev is always different to attr */
+ w->prev = prev;
+
+ #if EV_USE_INOTIFY
+ if (fs_fd >= 0)
+ {
+ infy_del (EV_A_ w);
+ infy_add (EV_A_ w);
+ ev_stat_stat (EV_A_ w); /* avoid race... */
+ }
+ #endif
+
+ ev_feed_event (EV_A_ w, EV_STAT);
+ }
+}
+
+void
+ev_stat_start (EV_P_ ev_stat *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ ev_stat_stat (EV_A_ w);
+
+ if (w->interval < MIN_STAT_INTERVAL && w->interval)
+ w->interval = MIN_STAT_INTERVAL;
+
+ ev_timer_init (&w->timer, stat_timer_cb, 0., w->interval ? w->interval : DEF_STAT_INTERVAL);
+ ev_set_priority (&w->timer, ev_priority (w));
+
+#if EV_USE_INOTIFY
+ infy_init (EV_A);
+
+ if (fs_fd >= 0)
+ infy_add (EV_A_ w);
+ else
+#endif
+ {
+ ev_timer_again (EV_A_ &w->timer);
+ ev_unref (EV_A);
+ }
+
+ ev_start (EV_A_ (W)w, 1);
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_stat_stop (EV_P_ ev_stat *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+#if EV_USE_INOTIFY
+ infy_del (EV_A_ w);
+#endif
+
+ if (ev_is_active (&w->timer))
+ {
+ ev_ref (EV_A);
+ ev_timer_stop (EV_A_ &w->timer);
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_IDLE_ENABLE
+void
+ev_idle_start (EV_P_ ev_idle *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ pri_adjust (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ++idlecnt [ABSPRI (w)];
+
+ ++idleall;
+ ev_start (EV_A_ (W)w, active);
+
+ array_needsize (ev_idle *, idles [ABSPRI (w)], idlemax [ABSPRI (w)], active, array_needsize_noinit);
+ idles [ABSPRI (w)][active - 1] = w;
+ }
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_idle_stop (EV_P_ ev_idle *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ idles [ABSPRI (w)][active - 1] = idles [ABSPRI (w)][--idlecnt [ABSPRI (w)]];
+ ev_active (idles [ABSPRI (w)][active - 1]) = active;
+
+ ev_stop (EV_A_ (W)w);
+ --idleall;
+ }
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_PREPARE_ENABLE
+void
+ev_prepare_start (EV_P_ ev_prepare *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, ++preparecnt);
+ array_needsize (ev_prepare *, prepares, preparemax, preparecnt, array_needsize_noinit);
+ prepares [preparecnt - 1] = w;
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_prepare_stop (EV_P_ ev_prepare *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ prepares [active - 1] = prepares [--preparecnt];
+ ev_active (prepares [active - 1]) = active;
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_CHECK_ENABLE
+void
+ev_check_start (EV_P_ ev_check *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, ++checkcnt);
+ array_needsize (ev_check *, checks, checkmax, checkcnt, array_needsize_noinit);
+ checks [checkcnt - 1] = w;
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_check_stop (EV_P_ ev_check *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ checks [active - 1] = checks [--checkcnt];
+ ev_active (checks [active - 1]) = active;
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_EMBED_ENABLE
+ecb_noinline
+void
+ev_embed_sweep (EV_P_ ev_embed *w) EV_NOEXCEPT
+{
+ ev_run (w->other, EVRUN_NOWAIT);
+}
+
+static void
+embed_io_cb (EV_P_ ev_io *io, int revents)
+{
+ ev_embed *w = (ev_embed *)(((char *)io) - offsetof (ev_embed, io));
+
+ if (ev_cb (w))
+ ev_feed_event (EV_A_ (W)w, EV_EMBED);
+ else
+ ev_run (w->other, EVRUN_NOWAIT);
+}
+
+static void
+embed_prepare_cb (EV_P_ ev_prepare *prepare, int revents)
+{
+ ev_embed *w = (ev_embed *)(((char *)prepare) - offsetof (ev_embed, prepare));
+
+ {
+ EV_P = w->other;
+
+ while (fdchangecnt)
+ {
+ fd_reify (EV_A);
+ ev_run (EV_A_ EVRUN_NOWAIT);
+ }
+ }
+}
+
+#if EV_FORK_ENABLE
+static void
+embed_fork_cb (EV_P_ ev_fork *fork_w, int revents)
+{
+ ev_embed *w = (ev_embed *)(((char *)fork_w) - offsetof (ev_embed, fork));
+
+ ev_embed_stop (EV_A_ w);
+
+ {
+ EV_P = w->other;
+
+ ev_loop_fork (EV_A);
+ ev_run (EV_A_ EVRUN_NOWAIT);
+ }
+
+ ev_embed_start (EV_A_ w);
+}
+#endif
+
+#if 0
+static void
+embed_idle_cb (EV_P_ ev_idle *idle, int revents)
+{
+ ev_idle_stop (EV_A_ idle);
+}
+#endif
+
+void
+ev_embed_start (EV_P_ ev_embed *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ {
+ EV_P = w->other;
+ assert (("libev: loop to be embedded is not embeddable", backend & ev_embeddable_backends ()));
+ ev_io_init (&w->io, embed_io_cb, backend_fd, EV_READ);
+ }
+
+ EV_FREQUENT_CHECK;
+
+ ev_set_priority (&w->io, ev_priority (w));
+ ev_io_start (EV_A_ &w->io);
+
+ ev_prepare_init (&w->prepare, embed_prepare_cb);
+ ev_set_priority (&w->prepare, EV_MINPRI);
+ ev_prepare_start (EV_A_ &w->prepare);
+
+#if EV_FORK_ENABLE
+ ev_fork_init (&w->fork, embed_fork_cb);
+ ev_fork_start (EV_A_ &w->fork);
+#endif
+
+ /*ev_idle_init (&w->idle, e,bed_idle_cb);*/
+
+ ev_start (EV_A_ (W)w, 1);
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_embed_stop (EV_P_ ev_embed *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_io_stop (EV_A_ &w->io);
+ ev_prepare_stop (EV_A_ &w->prepare);
+#if EV_FORK_ENABLE
+ ev_fork_stop (EV_A_ &w->fork);
+#endif
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_FORK_ENABLE
+void
+ev_fork_start (EV_P_ ev_fork *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, ++forkcnt);
+ array_needsize (ev_fork *, forks, forkmax, forkcnt, array_needsize_noinit);
+ forks [forkcnt - 1] = w;
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_fork_stop (EV_P_ ev_fork *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ forks [active - 1] = forks [--forkcnt];
+ ev_active (forks [active - 1]) = active;
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_CLEANUP_ENABLE
+void
+ev_cleanup_start (EV_P_ ev_cleanup *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, ++cleanupcnt);
+ array_needsize (ev_cleanup *, cleanups, cleanupmax, cleanupcnt, array_needsize_noinit);
+ cleanups [cleanupcnt - 1] = w;
+
+ /* cleanup watchers should never keep a refcount on the loop */
+ ev_unref (EV_A);
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_cleanup_stop (EV_P_ ev_cleanup *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+ ev_ref (EV_A);
+
+ {
+ int active = ev_active (w);
+
+ cleanups [active - 1] = cleanups [--cleanupcnt];
+ ev_active (cleanups [active - 1]) = active;
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+#endif
+
+#if EV_ASYNC_ENABLE
+void
+ev_async_start (EV_P_ ev_async *w) EV_NOEXCEPT
+{
+ if (ecb_expect_false (ev_is_active (w)))
+ return;
+
+ w->sent = 0;
+
+ evpipe_init (EV_A);
+
+ EV_FREQUENT_CHECK;
+
+ ev_start (EV_A_ (W)w, ++asynccnt);
+ array_needsize (ev_async *, asyncs, asyncmax, asynccnt, array_needsize_noinit);
+ asyncs [asynccnt - 1] = w;
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_async_stop (EV_P_ ev_async *w) EV_NOEXCEPT
+{
+ clear_pending (EV_A_ (W)w);
+ if (ecb_expect_false (!ev_is_active (w)))
+ return;
+
+ EV_FREQUENT_CHECK;
+
+ {
+ int active = ev_active (w);
+
+ asyncs [active - 1] = asyncs [--asynccnt];
+ ev_active (asyncs [active - 1]) = active;
+ }
+
+ ev_stop (EV_A_ (W)w);
+
+ EV_FREQUENT_CHECK;
+}
+
+void
+ev_async_send (EV_P_ ev_async *w) EV_NOEXCEPT
+{
+ w->sent = 1;
+ evpipe_write (EV_A_ &async_pending);
+}
+#endif
+
+/*****************************************************************************/
+
+struct ev_once
+{
+ ev_io io;
+ ev_timer to;
+ void (*cb)(int revents, void *arg);
+ void *arg;
+};
+
+static void
+once_cb (EV_P_ struct ev_once *once, int revents)
+{
+ void (*cb)(int revents, void *arg) = once->cb;
+ void *arg = once->arg;
+
+ ev_io_stop (EV_A_ &once->io);
+ ev_timer_stop (EV_A_ &once->to);
+ ev_free (once);
+
+ cb (revents, arg);
+}
+
+static void
+once_cb_io (EV_P_ ev_io *w, int revents)
+{
+ struct ev_once *once = (struct ev_once *)(((char *)w) - offsetof (struct ev_once, io));
+
+ once_cb (EV_A_ once, revents | ev_clear_pending (EV_A_ &once->to));
+}
+
+static void
+once_cb_to (EV_P_ ev_timer *w, int revents)
+{
+ struct ev_once *once = (struct ev_once *)(((char *)w) - offsetof (struct ev_once, to));
+
+ once_cb (EV_A_ once, revents | ev_clear_pending (EV_A_ &once->io));
+}
+
+void
+ev_once (EV_P_ int fd, int events, ev_tstamp timeout, void (*cb)(int revents, void *arg), void *arg) EV_NOEXCEPT
+{
+ struct ev_once *once = (struct ev_once *)ev_malloc (sizeof (struct ev_once));
+
+ once->cb = cb;
+ once->arg = arg;
+
+ ev_init (&once->io, once_cb_io);
+ if (fd >= 0)
+ {
+ ev_io_set (&once->io, fd, events);
+ ev_io_start (EV_A_ &once->io);
+ }
+
+ ev_init (&once->to, once_cb_to);
+ if (timeout >= 0.)
+ {
+ ev_timer_set (&once->to, timeout, 0.);
+ ev_timer_start (EV_A_ &once->to);
+ }
+}
+/*****************************************************************************/
+
+#if EV_WALK_ENABLE
+ecb_cold
+void
+ev_walk (EV_P_ int types, void (*cb)(EV_P_ int type, void *w)) EV_NOEXCEPT
+{
+ int i, j;
+ ev_watcher_list *wl, *wn;
+
+ if (types & (EV_IO | EV_EMBED))
+ for (i = 0; i < anfdmax; ++i)
+ for (wl = anfds [i].head; wl; )
+ {
+ wn = wl->next;
+
+#if EV_EMBED_ENABLE
+ if (ev_cb ((ev_io *)wl) == embed_io_cb)
+ {
+ if (types & EV_EMBED)
+ cb (EV_A_ EV_EMBED, ((char *)wl) - offsetof (struct ev_embed, io));
+ }
+ else
+#endif
+#if EV_USE_INOTIFY
+ if (ev_cb ((ev_io *)wl) == infy_cb)
+ ;
+ else
+#endif
+ if ((ev_io *)wl != &pipe_w)
+ if (types & EV_IO)
+ cb (EV_A_ EV_IO, wl);
+
+ wl = wn;
+ }
+
+ if (types & (EV_TIMER | EV_STAT))
+ for (i = timercnt + HEAP0; i-- > HEAP0; )
+#if EV_STAT_ENABLE
+ /*TODO: timer is not always active*/
+ if (ev_cb ((ev_timer *)ANHE_w (timers [i])) == stat_timer_cb)
+ {
+ if (types & EV_STAT)
+ cb (EV_A_ EV_STAT, ((char *)ANHE_w (timers [i])) - offsetof (struct ev_stat, timer));
+ }
+ else
+#endif
+ if (types & EV_TIMER)
+ cb (EV_A_ EV_TIMER, ANHE_w (timers [i]));
+
+#if EV_PERIODIC_ENABLE
+ if (types & EV_PERIODIC)
+ for (i = periodiccnt + HEAP0; i-- > HEAP0; )
+ cb (EV_A_ EV_PERIODIC, ANHE_w (periodics [i]));
+#endif
+
+#if EV_IDLE_ENABLE
+ if (types & EV_IDLE)
+ for (j = NUMPRI; j--; )
+ for (i = idlecnt [j]; i--; )
+ cb (EV_A_ EV_IDLE, idles [j][i]);
+#endif
+
+#if EV_FORK_ENABLE
+ if (types & EV_FORK)
+ for (i = forkcnt; i--; )
+ if (ev_cb (forks [i]) != embed_fork_cb)
+ cb (EV_A_ EV_FORK, forks [i]);
+#endif
+
+#if EV_ASYNC_ENABLE
+ if (types & EV_ASYNC)
+ for (i = asynccnt; i--; )
+ cb (EV_A_ EV_ASYNC, asyncs [i]);
+#endif
+
+#if EV_PREPARE_ENABLE
+ if (types & EV_PREPARE)
+ for (i = preparecnt; i--; )
+# if EV_EMBED_ENABLE
+ if (ev_cb (prepares [i]) != embed_prepare_cb)
+# endif
+ cb (EV_A_ EV_PREPARE, prepares [i]);
+#endif
+
+#if EV_CHECK_ENABLE
+ if (types & EV_CHECK)
+ for (i = checkcnt; i--; )
+ cb (EV_A_ EV_CHECK, checks [i]);
+#endif
+
+#if EV_SIGNAL_ENABLE
+ if (types & EV_SIGNAL)
+ for (i = 0; i < EV_NSIG - 1; ++i)
+ for (wl = signals [i].head; wl; )
+ {
+ wn = wl->next;
+ cb (EV_A_ EV_SIGNAL, wl);
+ wl = wn;
+ }
+#endif
+
+#if EV_CHILD_ENABLE
+ if (types & EV_CHILD)
+ for (i = (EV_PID_HASHSIZE); i--; )
+ for (wl = childs [i]; wl; )
+ {
+ wn = wl->next;
+ cb (EV_A_ EV_CHILD, wl);
+ wl = wn;
+ }
+#endif
+// EV_STAT 0x00001000 /* stat data changed */
+// EV_EMBED 0x00010000 /* embedded event loop needs sweep */
+}
+#endif
+void
+ev_now_update_if_cheap (EV_P) EV_NOEXCEPT
+{
+ if (have_cheap_timer) time_update (EV_A_ 1e100);
+}
+
+int
+ev_active_cnt (EV_P) EV_NOEXCEPT
+{
+ return activecnt;
+}
+
+#if EV_MULTIPLICITY
+ #include "ev_wrap.h"
+#endif
+
diff --git a/contrib/libev/ev.h b/contrib/libev/ev.h
new file mode 100644
index 0000000..7135a08
--- /dev/null
+++ b/contrib/libev/ev.h
@@ -0,0 +1,849 @@
+/*
+ * libev native API header
+ *
+ * Copyright (c) 2007-2020 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#ifndef EV_H_
+#define EV_H_
+
+#ifdef __cplusplus
+# define EV_CPP(x) x
+# if __cplusplus >= 201103L
+# define EV_NOEXCEPT noexcept
+# else
+# define EV_NOEXCEPT
+# endif
+#else
+# define EV_CPP(x)
+# define EV_NOEXCEPT
+#endif
+#define EV_THROW EV_NOEXCEPT /* pre-4.25, do not use in new code */
+
+EV_CPP(extern "C" {)
+
+/*****************************************************************************/
+
+/* pre-4.0 compatibility */
+#ifndef EV_COMPAT3
+# define EV_COMPAT3 1
+#endif
+
+#ifndef EV_FEATURES
+# if defined __OPTIMIZE_SIZE__
+# define EV_FEATURES 0x7c
+# else
+# define EV_FEATURES 0x7f
+# endif
+#endif
+
+#define EV_FEATURE_CODE ((EV_FEATURES) & 1)
+#define EV_FEATURE_DATA ((EV_FEATURES) & 2)
+#define EV_FEATURE_CONFIG ((EV_FEATURES) & 4)
+#define EV_FEATURE_API ((EV_FEATURES) & 8)
+#define EV_FEATURE_WATCHERS ((EV_FEATURES) & 16)
+#define EV_FEATURE_BACKENDS ((EV_FEATURES) & 32)
+#define EV_FEATURE_OS ((EV_FEATURES) & 64)
+
+/* these priorities are inclusive, higher priorities will be invoked earlier */
+#ifndef EV_MINPRI
+# define EV_MINPRI (EV_FEATURE_CONFIG ? -2 : 0)
+#endif
+#ifndef EV_MAXPRI
+# define EV_MAXPRI (EV_FEATURE_CONFIG ? +2 : 0)
+#endif
+
+#ifndef EV_MULTIPLICITY
+# define EV_MULTIPLICITY EV_FEATURE_CONFIG
+#endif
+
+#ifndef EV_PERIODIC_ENABLE
+# define EV_PERIODIC_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_STAT_ENABLE
+# define EV_STAT_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_PREPARE_ENABLE
+# define EV_PREPARE_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_CHECK_ENABLE
+# define EV_CHECK_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_IDLE_ENABLE
+# define EV_IDLE_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_FORK_ENABLE
+# define EV_FORK_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_CLEANUP_ENABLE
+# define EV_CLEANUP_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_SIGNAL_ENABLE
+# define EV_SIGNAL_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_CHILD_ENABLE
+# ifdef _WIN32
+# define EV_CHILD_ENABLE 0
+# else
+# define EV_CHILD_ENABLE EV_FEATURE_WATCHERS
+#endif
+#endif
+
+#ifndef EV_ASYNC_ENABLE
+# define EV_ASYNC_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_EMBED_ENABLE
+# define EV_EMBED_ENABLE EV_FEATURE_WATCHERS
+#endif
+
+#ifndef EV_WALK_ENABLE
+# define EV_WALK_ENABLE 0 /* not yet */
+#endif
+
+/*****************************************************************************/
+
+#if EV_CHILD_ENABLE && !EV_SIGNAL_ENABLE
+# undef EV_SIGNAL_ENABLE
+# define EV_SIGNAL_ENABLE 1
+#endif
+
+/*****************************************************************************/
+
+#ifndef EV_TSTAMP_T
+# define EV_TSTAMP_T double
+#endif
+typedef EV_TSTAMP_T ev_tstamp;
+
+#include <string.h> /* for memmove */
+
+#ifndef EV_ATOMIC_T
+# include <signal.h>
+# define EV_ATOMIC_T sig_atomic_t volatile
+#endif
+
+#if EV_STAT_ENABLE
+# ifdef _WIN32
+# include <time.h>
+# include <sys/types.h>
+# endif
+# include <sys/stat.h>
+#endif
+
+/* support multiple event loops? */
+#if EV_MULTIPLICITY
+struct ev_loop;
+# define EV_P struct ev_loop *loop /* a loop as sole parameter in a declaration */
+# define EV_P_ EV_P, /* a loop as first of multiple parameters */
+# define EV_A loop /* a loop as sole argument to a function call */
+# define EV_A_ EV_A, /* a loop as first of multiple arguments */
+# define EV_DEFAULT_UC ev_default_loop_uc_ () /* the default loop, if initialised, as sole arg */
+# define EV_DEFAULT_UC_ EV_DEFAULT_UC, /* the default loop as first of multiple arguments */
+# define EV_DEFAULT ev_default_loop (0) /* the default loop as sole arg */
+# define EV_DEFAULT_ EV_DEFAULT, /* the default loop as first of multiple arguments */
+#else
+# define EV_P void
+# define EV_P_
+# define EV_A
+# define EV_A_
+# define EV_DEFAULT
+# define EV_DEFAULT_
+# define EV_DEFAULT_UC
+# define EV_DEFAULT_UC_
+# undef EV_EMBED_ENABLE
+#endif
+
+/* EV_INLINE is used for functions in header files */
+#if __STDC_VERSION__ >= 199901L || __GNUC__ >= 3
+# define EV_INLINE static inline
+#else
+# define EV_INLINE static
+#endif
+
+#ifdef EV_API_STATIC
+# define EV_API_DECL static
+#else
+# define EV_API_DECL extern
+#endif
+
+/* EV_PROTOTYPES can be used to switch of prototype declarations */
+#ifndef EV_PROTOTYPES
+# define EV_PROTOTYPES 1
+#endif
+
+/*****************************************************************************/
+
+#define EV_VERSION_MAJOR 4
+#define EV_VERSION_MINOR 33
+
+/* eventmask, revents, events... */
+enum {
+ EV_UNDEF = (int)0xFFFFFFFF, /* guaranteed to be invalid */
+ EV_NONE = 0x00, /* no events */
+ EV_READ = 0x01, /* ev_io detected read will not block */
+ EV_WRITE = 0x02, /* ev_io detected write will not block */
+ EV__IOFDSET = 0x80, /* internal use only */
+ EV_IO = EV_READ, /* alias for type-detection */
+ EV_TIMER = 0x00000100, /* timer timed out */
+#if EV_COMPAT3
+ EV_TIMEOUT = EV_TIMER, /* pre 4.0 API compatibility */
+#endif
+ EV_PERIODIC = 0x00000200, /* periodic timer timed out */
+ EV_SIGNAL = 0x00000400, /* signal was received */
+ EV_CHILD = 0x00000800, /* child/pid had status change */
+ EV_STAT = 0x00001000, /* stat data changed */
+ EV_IDLE = 0x00002000, /* event loop is idling */
+ EV_PREPARE = 0x00004000, /* event loop about to poll */
+ EV_CHECK = 0x00008000, /* event loop finished poll */
+ EV_EMBED = 0x00010000, /* embedded event loop needs sweep */
+ EV_FORK = 0x00020000, /* event loop resumed in child */
+ EV_CLEANUP = 0x00040000, /* event loop resumed in child */
+ EV_ASYNC = 0x00080000, /* async intra-loop signal */
+ EV_CUSTOM = 0x01000000, /* for use by user code */
+ EV_ERROR = (int)0x80000000 /* sent when an error occurs */
+};
+
+/* can be used to add custom fields to all watchers, while losing binary compatibility */
+#ifndef EV_COMMON
+# define EV_COMMON void *data;
+#endif
+
+#ifndef EV_CB_DECLARE
+# define EV_CB_DECLARE(type) void (*cb)(EV_P_ struct type *w, int revents);
+#endif
+#ifndef EV_CB_INVOKE
+# define EV_CB_INVOKE(watcher,revents) (watcher)->cb (EV_A_ (watcher), (revents))
+#endif
+
+/* not official, do not use */
+#define EV_CB(type,name) void name (EV_P_ struct ev_ ## type *w, int revents)
+
+/*
+ * struct member types:
+ * private: you may look at them, but not change them,
+ * and they might not mean anything to you.
+ * ro: can be read anytime, but only changed when the watcher isn't active.
+ * rw: can be read and modified anytime, even when the watcher is active.
+ *
+ * some internal details that might be helpful for debugging:
+ *
+ * active is either 0, which means the watcher is not active,
+ * or the array index of the watcher (periodics, timers)
+ * or the array index + 1 (most other watchers)
+ * or simply 1 for watchers that aren't in some array.
+ * pending is either 0, in which case the watcher isn't,
+ * or the array index + 1 in the pendings array.
+ */
+
+#if EV_MINPRI == EV_MAXPRI
+# define EV_DECL_PRIORITY
+#elif !defined (EV_DECL_PRIORITY)
+# define EV_DECL_PRIORITY int priority;
+#endif
+
+/* shared by all watchers */
+#define EV_WATCHER(type) \
+ int active; /* private */ \
+ int pending; /* private */ \
+ EV_DECL_PRIORITY /* private */ \
+ EV_COMMON /* rw */ \
+ EV_CB_DECLARE (type) /* private */
+
+#define EV_WATCHER_LIST(type) \
+ EV_WATCHER (type) \
+ struct ev_watcher_list *next; /* private */
+
+#define EV_WATCHER_TIME(type) \
+ EV_WATCHER (type) \
+ ev_tstamp at; /* private */
+
+/* base class, nothing to see here unless you subclass */
+typedef struct ev_watcher
+{
+ EV_WATCHER (ev_watcher)
+} ev_watcher;
+
+/* base class, nothing to see here unless you subclass */
+typedef struct ev_watcher_list
+{
+ EV_WATCHER_LIST (ev_watcher_list)
+} ev_watcher_list;
+
+/* base class, nothing to see here unless you subclass */
+typedef struct ev_watcher_time
+{
+ EV_WATCHER_TIME (ev_watcher_time)
+} ev_watcher_time;
+
+/* invoked when fd is either EV_READable or EV_WRITEable */
+/* revent EV_READ, EV_WRITE */
+typedef struct ev_io
+{
+ EV_WATCHER_LIST (ev_io)
+
+ int fd; /* ro */
+ int events; /* ro */
+} ev_io;
+
+/* invoked after a specific time, repeatable (based on monotonic clock) */
+/* revent EV_TIMEOUT */
+typedef struct ev_timer
+{
+ EV_WATCHER_TIME (ev_timer)
+
+ ev_tstamp repeat; /* rw */
+} ev_timer;
+
+/* invoked at some specific time, possibly repeating at regular intervals (based on UTC) */
+/* revent EV_PERIODIC */
+typedef struct ev_periodic
+{
+ EV_WATCHER_TIME (ev_periodic)
+
+ ev_tstamp offset; /* rw */
+ ev_tstamp interval; /* rw */
+ ev_tstamp (*reschedule_cb)(struct ev_periodic *w, ev_tstamp now) EV_NOEXCEPT; /* rw */
+} ev_periodic;
+
+/* invoked when the given signal has been received */
+/* revent EV_SIGNAL */
+typedef struct ev_signal
+{
+ EV_WATCHER_LIST (ev_signal)
+
+ int signum; /* ro */
+} ev_signal;
+
+/* invoked when sigchld is received and waitpid indicates the given pid */
+/* revent EV_CHILD */
+/* does not support priorities */
+typedef struct ev_child
+{
+ EV_WATCHER_LIST (ev_child)
+
+ int flags; /* private */
+ int pid; /* ro */
+ int rpid; /* rw, holds the received pid */
+ int rstatus; /* rw, holds the exit status, use the macros from sys/wait.h */
+} ev_child;
+
+#if EV_STAT_ENABLE
+/* st_nlink = 0 means missing file or other error */
+# ifdef _WIN32
+typedef struct _stati64 ev_statdata;
+# else
+typedef struct stat ev_statdata;
+# endif
+
+/* invoked each time the stat data changes for a given path */
+/* revent EV_STAT */
+typedef struct ev_stat
+{
+ EV_WATCHER_LIST (ev_stat)
+
+ ev_timer timer; /* private */
+ ev_tstamp interval; /* ro */
+ const char *path; /* ro */
+ ev_statdata prev; /* ro */
+ ev_statdata attr; /* ro */
+
+ int wd; /* wd for inotify, fd for kqueue */
+} ev_stat;
+#endif
+
+/* invoked when the nothing else needs to be done, keeps the process from blocking */
+/* revent EV_IDLE */
+typedef struct ev_idle
+{
+ EV_WATCHER (ev_idle)
+} ev_idle;
+
+/* invoked for each run of the mainloop, just before the blocking call */
+/* you can still change events in any way you like */
+/* revent EV_PREPARE */
+typedef struct ev_prepare
+{
+ EV_WATCHER (ev_prepare)
+} ev_prepare;
+
+/* invoked for each run of the mainloop, just after the blocking call */
+/* revent EV_CHECK */
+typedef struct ev_check
+{
+ EV_WATCHER (ev_check)
+} ev_check;
+
+/* the callback gets invoked before check in the child process when a fork was detected */
+/* revent EV_FORK */
+typedef struct ev_fork
+{
+ EV_WATCHER (ev_fork)
+} ev_fork;
+
+/* is invoked just before the loop gets destroyed */
+/* revent EV_CLEANUP */
+typedef struct ev_cleanup
+{
+ EV_WATCHER (ev_cleanup)
+} ev_cleanup;
+
+#if EV_EMBED_ENABLE
+/* used to embed an event loop inside another */
+/* the callback gets invoked when the event loop has handled events, and can be 0 */
+typedef struct ev_embed
+{
+ EV_WATCHER (ev_embed)
+
+ struct ev_loop *other; /* ro */
+#undef EV_IO_ENABLE
+#define EV_IO_ENABLE 1
+ ev_io io; /* private */
+#undef EV_PREPARE_ENABLE
+#define EV_PREPARE_ENABLE 1
+ ev_prepare prepare; /* private */
+ ev_check check; /* unused */
+ ev_timer timer; /* unused */
+ ev_periodic periodic; /* unused */
+ ev_idle idle; /* unused */
+ ev_fork fork; /* private */
+ ev_cleanup cleanup; /* unused */
+} ev_embed;
+#endif
+
+#if EV_ASYNC_ENABLE
+/* invoked when somebody calls ev_async_send on the watcher */
+/* revent EV_ASYNC */
+typedef struct ev_async
+{
+ EV_WATCHER (ev_async)
+
+ EV_ATOMIC_T sent; /* private */
+} ev_async;
+
+# define ev_async_pending(w) (+(w)->sent)
+#endif
+
+/* the presence of this union forces similar struct layout */
+union ev_any_watcher
+{
+ struct ev_watcher w;
+ struct ev_watcher_list wl;
+
+ struct ev_io io;
+ struct ev_timer timer;
+ struct ev_periodic periodic;
+ struct ev_signal signal;
+ struct ev_child child;
+#if EV_STAT_ENABLE
+ struct ev_stat stat;
+#endif
+#if EV_IDLE_ENABLE
+ struct ev_idle idle;
+#endif
+ struct ev_prepare prepare;
+ struct ev_check check;
+#if EV_FORK_ENABLE
+ struct ev_fork fork;
+#endif
+#if EV_CLEANUP_ENABLE
+ struct ev_cleanup cleanup;
+#endif
+#if EV_EMBED_ENABLE
+ struct ev_embed embed;
+#endif
+#if EV_ASYNC_ENABLE
+ struct ev_async async;
+#endif
+};
+
+/* flag bits for ev_default_loop and ev_loop_new */
+enum {
+ /* the default */
+ EVFLAG_AUTO = 0x00000000U, /* not quite a mask */
+ /* flag bits */
+ EVFLAG_NOENV = 0x01000000U, /* do NOT consult environment */
+ EVFLAG_FORKCHECK = 0x02000000U, /* check for a fork in each iteration */
+ /* debugging/feature disable */
+ EVFLAG_NOINOTIFY = 0x00100000U, /* do not attempt to use inotify */
+#if EV_COMPAT3
+ EVFLAG_NOSIGFD = 0, /* compatibility to pre-3.9 */
+#endif
+ EVFLAG_SIGNALFD = 0x00200000U, /* attempt to use signalfd */
+ EVFLAG_NOSIGMASK = 0x00400000U, /* avoid modifying the signal mask */
+ EVFLAG_NOTIMERFD = 0x00800000U /* avoid creating a timerfd */
+};
+
+/* method bits to be ored together */
+enum {
+ EVBACKEND_SELECT = 0x00000001U, /* available just about anywhere */
+ EVBACKEND_POLL = 0x00000002U, /* !win, !aix, broken on osx */
+ EVBACKEND_EPOLL = 0x00000004U, /* linux */
+ EVBACKEND_KQUEUE = 0x00000008U, /* bsd, broken on osx */
+ EVBACKEND_DEVPOLL = 0x00000010U, /* solaris 8 */ /* NYI */
+ EVBACKEND_PORT = 0x00000020U, /* solaris 10 */
+ EVBACKEND_LINUXAIO = 0x00000040U, /* linux AIO, 4.19+ */
+ EVBACKEND_IOURING = 0x00000080U, /* linux io_uring, 5.1+ */
+ EVBACKEND_ALL = 0x000000FFU, /* all known backends */
+ EVBACKEND_MASK = 0x0000FFFFU /* all future backends */
+};
+
+#if EV_PROTOTYPES
+EV_API_DECL int ev_version_major (void) EV_NOEXCEPT;
+EV_API_DECL int ev_version_minor (void) EV_NOEXCEPT;
+
+EV_API_DECL unsigned int ev_supported_backends (void) EV_NOEXCEPT;
+EV_API_DECL unsigned int ev_recommended_backends (void) EV_NOEXCEPT;
+EV_API_DECL unsigned int ev_embeddable_backends (void) EV_NOEXCEPT;
+
+EV_API_DECL ev_tstamp ev_time (void) EV_NOEXCEPT;
+EV_API_DECL void ev_sleep (ev_tstamp delay) EV_NOEXCEPT; /* sleep for a while */
+
+/* Sets the allocation function to use, works like realloc.
+ * It is used to allocate and free memory.
+ * If it returns zero when memory needs to be allocated, the library might abort
+ * or take some potentially destructive action.
+ * The default is your system realloc function.
+ */
+EV_API_DECL void ev_set_allocator (void *(*cb)(void *ptr, long size) EV_NOEXCEPT) EV_NOEXCEPT;
+
+/* set the callback function to call on a
+ * retryable syscall error
+ * (such as failed select, poll, epoll_wait)
+ */
+EV_API_DECL void ev_set_syserr_cb (void (*cb)(const char *msg) EV_NOEXCEPT) EV_NOEXCEPT;
+
+#if EV_MULTIPLICITY
+
+/* the default loop is the only one that handles signals and child watchers */
+/* you can call this as often as you like */
+EV_API_DECL struct ev_loop *ev_default_loop (unsigned int flags EV_CPP (= 0)) EV_NOEXCEPT;
+
+/* create and destroy alternative loops that don't handle signals */
+EV_API_DECL struct ev_loop *ev_loop_new (unsigned int flags EV_CPP (= 0)) EV_NOEXCEPT;
+
+EV_API_DECL ev_tstamp ev_now (EV_P) EV_NOEXCEPT; /* time w.r.t. timers and the eventloop, updated after each poll */
+
+#else
+
+EV_API_DECL int ev_default_loop (unsigned int flags EV_CPP (= 0)) EV_NOEXCEPT; /* returns true when successful */
+
+EV_API_DECL ev_tstamp ev_rt_now;
+
+EV_INLINE ev_tstamp
+ev_now (void) EV_NOEXCEPT
+{
+ return ev_rt_now;
+}
+
+/* looks weird, but ev_is_default_loop (EV_A) still works if this exists */
+EV_INLINE int
+ev_is_default_loop (void) EV_NOEXCEPT
+{
+ return 1;
+}
+
+#endif /* multiplicity */
+
+/* destroy event loops, also works for the default loop */
+EV_API_DECL void ev_loop_destroy (EV_P);
+
+/* this needs to be called after fork, to duplicate the loop */
+/* when you want to re-use it in the child */
+/* you can call it in either the parent or the child */
+/* you can actually call it at any time, anywhere :) */
+EV_API_DECL void ev_loop_fork (EV_P) EV_NOEXCEPT;
+
+EV_API_DECL unsigned int ev_backend (EV_P) EV_NOEXCEPT; /* backend in use by loop */
+
+EV_API_DECL void ev_now_update (EV_P) EV_NOEXCEPT; /* update event loop time */
+/*
+ * Same as ev_now_update, but will update time merely if cheap (coarse) timers
+ * are used in system.
+ */
+EV_API_DECL void ev_now_update_if_cheap (EV_P) EV_NOEXCEPT;
+
+#if EV_WALK_ENABLE
+/* walk (almost) all watchers in the loop of a given type, invoking the */
+/* callback on every such watcher. The callback might stop the watcher, */
+/* but do nothing else with the loop */
+EV_API_DECL void ev_walk (EV_P_ int types, void (*cb)(EV_P_ int type, void *w)) EV_NOEXCEPT;
+#endif
+
+#endif /* prototypes */
+
+/* ev_run flags values */
+enum {
+ EVRUN_NOWAIT = 1, /* do not block/wait */
+ EVRUN_ONCE = 2 /* block *once* only */
+};
+
+/* ev_break how values */
+enum {
+ EVBREAK_CANCEL = 0, /* undo unloop */
+ EVBREAK_ONE = 1, /* unloop once */
+ EVBREAK_ALL = 2 /* unloop all loops */
+};
+
+#if EV_PROTOTYPES
+EV_API_DECL int ev_run (EV_P_ int flags EV_CPP (= 0));
+EV_API_DECL void ev_break (EV_P_ int how EV_CPP (= EVBREAK_ONE)) EV_NOEXCEPT; /* break out of the loop */
+
+/*
+ * ref/unref can be used to add or remove a refcount on the mainloop. every watcher
+ * keeps one reference. if you have a long-running watcher you never unregister that
+ * should not keep ev_loop from running, unref() after starting, and ref() before stopping.
+ */
+EV_API_DECL void ev_ref (EV_P) EV_NOEXCEPT;
+EV_API_DECL void ev_unref (EV_P) EV_NOEXCEPT;
+EV_API_DECL int ev_active_cnt (EV_P) EV_NOEXCEPT;
+
+/*
+ * convenience function, wait for a single event, without registering an event watcher
+ * if timeout is < 0, do wait indefinitely
+ */
+EV_API_DECL void ev_once (EV_P_ int fd, int events, ev_tstamp timeout, void (*cb)(int revents, void *arg), void *arg) EV_NOEXCEPT;
+
+EV_API_DECL void ev_invoke_pending (EV_P); /* invoke all pending watchers */
+
+# if EV_FEATURE_API
+EV_API_DECL unsigned int ev_iteration (EV_P) EV_NOEXCEPT; /* number of loop iterations */
+EV_API_DECL unsigned int ev_depth (EV_P) EV_NOEXCEPT; /* #ev_loop enters - #ev_loop leaves */
+EV_API_DECL void ev_verify (EV_P) EV_NOEXCEPT; /* abort if loop data corrupted */
+
+EV_API_DECL void ev_set_io_collect_interval (EV_P_ ev_tstamp interval) EV_NOEXCEPT; /* sleep at least this time, default 0 */
+EV_API_DECL void ev_set_timeout_collect_interval (EV_P_ ev_tstamp interval) EV_NOEXCEPT; /* sleep at least this time, default 0 */
+
+/* advanced stuff for threading etc. support, see docs */
+EV_API_DECL void ev_set_userdata (EV_P_ void *data) EV_NOEXCEPT;
+EV_API_DECL void *ev_userdata (EV_P) EV_NOEXCEPT;
+typedef void (*ev_loop_callback)(EV_P);
+EV_API_DECL void ev_set_invoke_pending_cb (EV_P_ ev_loop_callback invoke_pending_cb) EV_NOEXCEPT;
+/* C++ doesn't allow the use of the ev_loop_callback typedef here, so we need to spell it out */
+EV_API_DECL void ev_set_loop_release_cb (EV_P_ void (*release)(EV_P) EV_NOEXCEPT, void (*acquire)(EV_P) EV_NOEXCEPT) EV_NOEXCEPT;
+
+EV_API_DECL unsigned int ev_pending_count (EV_P) EV_NOEXCEPT; /* number of pending events, if any */
+
+/*
+ * stop/start the timer handling.
+ */
+EV_API_DECL void ev_suspend (EV_P) EV_NOEXCEPT;
+EV_API_DECL void ev_resume (EV_P) EV_NOEXCEPT;
+#endif
+
+#endif
+
+/* these may evaluate ev multiple times, and the other arguments at most once */
+/* either use ev_init + ev_TYPE_set, or the ev_TYPE_init macro, below, to first initialise a watcher */
+#define ev_init(ev,cb_) do { \
+ ((ev_watcher *)(void *)(ev))->active = \
+ ((ev_watcher *)(void *)(ev))->pending = 0; \
+ ev_set_priority ((ev), 0); \
+ ev_set_cb ((ev), cb_); \
+} while (0)
+
+#define ev_io_modify(ev,events_) do { (ev)->events = (ev)->events & EV__IOFDSET | (events_); } while (0)
+#define ev_io_set(ev,fd_,events_) do { (ev)->fd = (fd_); (ev)->events = (events_) | EV__IOFDSET; } while (0)
+#define ev_timer_set(ev,after_,repeat_) do { ((ev_watcher_time *)(ev))->at = (after_); (ev)->repeat = (repeat_); } while (0)
+#define ev_periodic_set(ev,ofs_,ival_,rcb_) do { (ev)->offset = (ofs_); (ev)->interval = (ival_); (ev)->reschedule_cb = (rcb_); } while (0)
+#define ev_signal_set(ev,signum_) do { (ev)->signum = (signum_); } while (0)
+#define ev_child_set(ev,pid_,trace_) do { (ev)->pid = (pid_); (ev)->flags = !!(trace_); } while (0)
+#define ev_stat_set(ev,path_,interval_) do { (ev)->path = (path_); (ev)->interval = (interval_); (ev)->wd = -2; } while (0)
+#define ev_idle_set(ev) /* nop, yes, this is a serious in-joke */
+#define ev_prepare_set(ev) /* nop, yes, this is a serious in-joke */
+#define ev_check_set(ev) /* nop, yes, this is a serious in-joke */
+#define ev_embed_set(ev,other_) do { (ev)->other = (other_); } while (0)
+#define ev_fork_set(ev) /* nop, yes, this is a serious in-joke */
+#define ev_cleanup_set(ev) /* nop, yes, this is a serious in-joke */
+#define ev_async_set(ev) /* nop, yes, this is a serious in-joke */
+
+#define ev_io_init(ev,cb,fd,events) do { ev_init ((ev), (cb)); ev_io_set ((ev),(fd),(events)); } while (0)
+#define ev_timer_init(ev,cb,after,repeat) do { ev_init ((ev), (cb)); ev_timer_set ((ev),(after),(repeat)); } while (0)
+#define ev_periodic_init(ev,cb,ofs,ival,rcb) do { ev_init ((ev), (cb)); ev_periodic_set ((ev),(ofs),(ival),(rcb)); } while (0)
+#define ev_signal_init(ev,cb,signum) do { ev_init ((ev), (cb)); ev_signal_set ((ev), (signum)); } while (0)
+#define ev_child_init(ev,cb,pid,trace) do { ev_init ((ev), (cb)); ev_child_set ((ev),(pid),(trace)); } while (0)
+#define ev_stat_init(ev,cb,path,interval) do { ev_init ((ev), (cb)); ev_stat_set ((ev),(path),(interval)); } while (0)
+#define ev_idle_init(ev,cb) do { ev_init ((ev), (cb)); ev_idle_set ((ev)); } while (0)
+#define ev_prepare_init(ev,cb) do { ev_init ((ev), (cb)); ev_prepare_set ((ev)); } while (0)
+#define ev_check_init(ev,cb) do { ev_init ((ev), (cb)); ev_check_set ((ev)); } while (0)
+#define ev_embed_init(ev,cb,other) do { ev_init ((ev), (cb)); ev_embed_set ((ev),(other)); } while (0)
+#define ev_fork_init(ev,cb) do { ev_init ((ev), (cb)); ev_fork_set ((ev)); } while (0)
+#define ev_cleanup_init(ev,cb) do { ev_init ((ev), (cb)); ev_cleanup_set ((ev)); } while (0)
+#define ev_async_init(ev,cb) do { ev_init ((ev), (cb)); ev_async_set ((ev)); } while (0)
+
+#define ev_is_pending(ev) (0 + ((ev_watcher *)(void *)(ev))->pending) /* ro, true when watcher is waiting for callback invocation */
+#define ev_is_active(ev) (0 + ((ev_watcher *)(void *)(ev))->active) /* ro, true when the watcher has been started */
+#define ev_can_stop(ev) (ev_is_pending(ev) || ev_is_active(ev)) /* ro, true when the watcher has been started */
+
+#define ev_cb_(ev) (ev)->cb /* rw */
+#define ev_cb(ev) (memmove (&ev_cb_ (ev), &((ev_watcher *)(ev))->cb, sizeof (ev_cb_ (ev))), (ev)->cb)
+
+#if EV_MINPRI == EV_MAXPRI
+# define ev_priority(ev) ((ev), EV_MINPRI)
+# define ev_set_priority(ev,pri) ((ev), (pri))
+#else
+# define ev_priority(ev) (+(((ev_watcher *)(void *)(ev))->priority))
+# define ev_set_priority(ev,pri) ( (ev_watcher *)(void *)(ev))->priority = (pri)
+#endif
+
+#define ev_periodic_at(ev) (+((ev_watcher_time *)(ev))->at)
+
+#ifndef ev_set_cb
+/* memmove is used here to avoid strict aliasing violations, and hopefully is optimized out by any reasonable compiler */
+# define ev_set_cb(ev,cb_) (ev_cb_ (ev) = (cb_), memmove (&((ev_watcher *)(ev))->cb, &ev_cb_ (ev), sizeof (ev_cb_ (ev))))
+#endif
+
+/* stopping (enabling, adding) a watcher does nothing if it is already running */
+/* stopping (disabling, deleting) a watcher does nothing unless it's already running */
+#if EV_PROTOTYPES
+
+/* feeds an event into a watcher as if the event actually occurred */
+/* accepts any ev_watcher type */
+EV_API_DECL void ev_feed_event (EV_P_ void *w, int revents) EV_NOEXCEPT;
+EV_API_DECL void ev_feed_fd_event (EV_P_ int fd, int revents) EV_NOEXCEPT;
+#if EV_SIGNAL_ENABLE
+EV_API_DECL void ev_feed_signal (int signum) EV_NOEXCEPT;
+EV_API_DECL void ev_feed_signal_event (EV_P_ int signum) EV_NOEXCEPT;
+#endif
+EV_API_DECL void ev_invoke (EV_P_ void *w, int revents);
+EV_API_DECL int ev_clear_pending (EV_P_ void *w) EV_NOEXCEPT;
+
+EV_API_DECL void ev_io_start (EV_P_ ev_io *w) EV_NOEXCEPT;
+EV_API_DECL void ev_io_stop (EV_P_ ev_io *w) EV_NOEXCEPT;
+
+EV_API_DECL void ev_timer_start (EV_P_ ev_timer *w) EV_NOEXCEPT;
+EV_API_DECL void ev_timer_stop (EV_P_ ev_timer *w) EV_NOEXCEPT;
+/* stops if active and no repeat, restarts if active and repeating, starts if inactive and repeating */
+EV_API_DECL void ev_timer_again (EV_P_ ev_timer *w) EV_NOEXCEPT;
+/* return remaining time */
+EV_API_DECL ev_tstamp ev_timer_remaining (EV_P_ ev_timer *w) EV_NOEXCEPT;
+
+#if EV_PERIODIC_ENABLE
+EV_API_DECL void ev_periodic_start (EV_P_ ev_periodic *w) EV_NOEXCEPT;
+EV_API_DECL void ev_periodic_stop (EV_P_ ev_periodic *w) EV_NOEXCEPT;
+EV_API_DECL void ev_periodic_again (EV_P_ ev_periodic *w) EV_NOEXCEPT;
+#endif
+
+/* only supported in the default loop */
+#if EV_SIGNAL_ENABLE
+EV_API_DECL void ev_signal_start (EV_P_ ev_signal *w) EV_NOEXCEPT;
+EV_API_DECL void ev_signal_stop (EV_P_ ev_signal *w) EV_NOEXCEPT;
+#endif
+
+/* only supported in the default loop */
+# if EV_CHILD_ENABLE
+EV_API_DECL void ev_child_start (EV_P_ ev_child *w) EV_NOEXCEPT;
+EV_API_DECL void ev_child_stop (EV_P_ ev_child *w) EV_NOEXCEPT;
+# endif
+
+# if EV_STAT_ENABLE
+EV_API_DECL void ev_stat_start (EV_P_ ev_stat *w) EV_NOEXCEPT;
+EV_API_DECL void ev_stat_stop (EV_P_ ev_stat *w) EV_NOEXCEPT;
+EV_API_DECL void ev_stat_stat (EV_P_ ev_stat *w) EV_NOEXCEPT;
+# endif
+
+# if EV_IDLE_ENABLE
+EV_API_DECL void ev_idle_start (EV_P_ ev_idle *w) EV_NOEXCEPT;
+EV_API_DECL void ev_idle_stop (EV_P_ ev_idle *w) EV_NOEXCEPT;
+# endif
+
+#if EV_PREPARE_ENABLE
+EV_API_DECL void ev_prepare_start (EV_P_ ev_prepare *w) EV_NOEXCEPT;
+EV_API_DECL void ev_prepare_stop (EV_P_ ev_prepare *w) EV_NOEXCEPT;
+#endif
+
+#if EV_CHECK_ENABLE
+EV_API_DECL void ev_check_start (EV_P_ ev_check *w) EV_NOEXCEPT;
+EV_API_DECL void ev_check_stop (EV_P_ ev_check *w) EV_NOEXCEPT;
+#endif
+
+# if EV_FORK_ENABLE
+EV_API_DECL void ev_fork_start (EV_P_ ev_fork *w) EV_NOEXCEPT;
+EV_API_DECL void ev_fork_stop (EV_P_ ev_fork *w) EV_NOEXCEPT;
+# endif
+
+# if EV_CLEANUP_ENABLE
+EV_API_DECL void ev_cleanup_start (EV_P_ ev_cleanup *w) EV_NOEXCEPT;
+EV_API_DECL void ev_cleanup_stop (EV_P_ ev_cleanup *w) EV_NOEXCEPT;
+# endif
+
+# if EV_EMBED_ENABLE
+/* only supported when loop to be embedded is in fact embeddable */
+EV_API_DECL void ev_embed_start (EV_P_ ev_embed *w) EV_NOEXCEPT;
+EV_API_DECL void ev_embed_stop (EV_P_ ev_embed *w) EV_NOEXCEPT;
+EV_API_DECL void ev_embed_sweep (EV_P_ ev_embed *w) EV_NOEXCEPT;
+# endif
+
+# if EV_ASYNC_ENABLE
+EV_API_DECL void ev_async_start (EV_P_ ev_async *w) EV_NOEXCEPT;
+EV_API_DECL void ev_async_stop (EV_P_ ev_async *w) EV_NOEXCEPT;
+EV_API_DECL void ev_async_send (EV_P_ ev_async *w) EV_NOEXCEPT;
+# endif
+
+#if EV_COMPAT3
+ #define EVLOOP_NONBLOCK EVRUN_NOWAIT
+ #define EVLOOP_ONESHOT EVRUN_ONCE
+ #define EVUNLOOP_CANCEL EVBREAK_CANCEL
+ #define EVUNLOOP_ONE EVBREAK_ONE
+ #define EVUNLOOP_ALL EVBREAK_ALL
+ #if EV_PROTOTYPES
+ EV_INLINE void ev_loop (EV_P_ int flags) { ev_run (EV_A_ flags); }
+ EV_INLINE void ev_unloop (EV_P_ int how ) { ev_break (EV_A_ how ); }
+ EV_INLINE void ev_default_destroy (void) { ev_loop_destroy (EV_DEFAULT); }
+ EV_INLINE void ev_default_fork (void) { ev_loop_fork (EV_DEFAULT); }
+ #if EV_FEATURE_API
+ EV_INLINE unsigned int ev_loop_count (EV_P) { return ev_iteration (EV_A); }
+ EV_INLINE unsigned int ev_loop_depth (EV_P) { return ev_depth (EV_A); }
+ EV_INLINE void ev_loop_verify (EV_P) { ev_verify (EV_A); }
+ #endif
+ #endif
+#else
+ typedef struct ev_loop ev_loop;
+#endif
+
+#endif
+
+EV_CPP(})
+
+#endif
+
diff --git a/contrib/libev/ev_epoll.c b/contrib/libev/ev_epoll.c
new file mode 100644
index 0000000..58cfa68
--- /dev/null
+++ b/contrib/libev/ev_epoll.c
@@ -0,0 +1,298 @@
+/*
+ * libev epoll fd activity backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2016,2017,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/*
+ * general notes about epoll:
+ *
+ * a) epoll silently removes fds from the fd set. as nothing tells us
+ * that an fd has been removed otherwise, we have to continually
+ * "rearm" fds that we suspect *might* have changed (same
+ * problem with kqueue, but much less costly there).
+ * b) the fact that ADD != MOD creates a lot of extra syscalls due to a)
+ * and seems not to have any advantage.
+ * c) the inability to handle fork or file descriptors (think dup)
+ * limits the applicability over poll, so this is not a generic
+ * poll replacement.
+ * d) epoll doesn't work the same as select with many file descriptors
+ * (such as files). while not critical, no other advanced interface
+ * seems to share this (rather non-unixy) limitation.
+ * e) epoll claims to be embeddable, but in practise you never get
+ * a ready event for the epoll fd (broken: <=2.6.26, working: >=2.6.32).
+ * f) epoll_ctl returning EPERM means the fd is always ready.
+ *
+ * lots of "weird code" and complication handling in this file is due
+ * to these design problems with epoll, as we try very hard to avoid
+ * epoll_ctl syscalls for common usage patterns and handle the breakage
+ * ensuing from receiving events for closed and otherwise long gone
+ * file descriptors.
+ */
+
+#include <sys/epoll.h>
+
+#define EV_EMASK_EPERM 0x80
+
+static void
+epoll_modify (EV_P_ int fd, int oev, int nev)
+{
+ struct epoll_event ev;
+ unsigned char oldmask;
+
+ /*
+ * we handle EPOLL_CTL_DEL by ignoring it here
+ * on the assumption that the fd is gone anyways
+ * if that is wrong, we have to handle the spurious
+ * event in epoll_poll.
+ * if the fd is added again, we try to ADD it, and, if that
+ * fails, we assume it still has the same eventmask.
+ */
+ if (!nev)
+ return;
+
+ oldmask = anfds [fd].emask;
+ anfds [fd].emask = nev;
+
+ /* store the generation counter in the upper 32 bits, the fd in the lower 32 bits */
+ ev.data.u64 = (uint64_t)(uint32_t)fd
+ | ((uint64_t)(uint32_t)++anfds [fd].egen << 32);
+ ev.events = (nev & EV_READ ? EPOLLIN : 0)
+ | (nev & EV_WRITE ? EPOLLOUT : 0);
+
+ if (ecb_expect_true (!epoll_ctl (backend_fd, oev && oldmask != nev ? EPOLL_CTL_MOD : EPOLL_CTL_ADD, fd, &ev)))
+ return;
+
+ if (ecb_expect_true (errno == ENOENT))
+ {
+ /* if ENOENT then the fd went away, so try to do the right thing */
+ if (!nev)
+ goto dec_egen;
+
+ if (!epoll_ctl (backend_fd, EPOLL_CTL_ADD, fd, &ev))
+ return;
+ }
+ else if (ecb_expect_true (errno == EEXIST))
+ {
+ /* EEXIST means we ignored a previous DEL, but the fd is still active */
+ /* if the kernel mask is the same as the new mask, we assume it hasn't changed */
+ if (oldmask == nev)
+ goto dec_egen;
+
+ if (!epoll_ctl (backend_fd, EPOLL_CTL_MOD, fd, &ev))
+ return;
+ }
+ else if (ecb_expect_true (errno == EPERM))
+ {
+ /* EPERM means the fd is always ready, but epoll is too snobbish */
+ /* to handle it, unlike select or poll. */
+ anfds [fd].emask = EV_EMASK_EPERM;
+
+ /* add fd to epoll_eperms, if not already inside */
+ if (!(oldmask & EV_EMASK_EPERM))
+ {
+ array_needsize (int, epoll_eperms, epoll_epermmax, epoll_epermcnt + 1, array_needsize_noinit);
+ epoll_eperms [epoll_epermcnt++] = fd;
+ }
+
+ return;
+ }
+ else
+ assert (("libev: I/O watcher with invalid fd found in epoll_ctl", errno != EBADF && errno != ELOOP && errno != EINVAL));
+
+ fd_kill (EV_A_ fd);
+
+dec_egen:
+ /* we didn't successfully call epoll_ctl, so decrement the generation counter again */
+ --anfds [fd].egen;
+}
+
+static void
+epoll_poll (EV_P_ ev_tstamp timeout)
+{
+ int i;
+ int eventcnt;
+
+ if (ecb_expect_false (epoll_epermcnt))
+ timeout = EV_TS_CONST (0.);
+
+ /* epoll wait times cannot be larger than (LONG_MAX - 999UL) / HZ msecs, which is below */
+ /* the default libev max wait time, however. */
+ EV_RELEASE_CB;
+ eventcnt = epoll_wait (backend_fd, epoll_events, epoll_eventmax, EV_TS_TO_MSEC (timeout));
+ EV_ACQUIRE_CB;
+
+ if (ecb_expect_false (eventcnt < 0))
+ {
+ if (errno != EINTR)
+ ev_syserr ("(libev) epoll_wait");
+
+ return;
+ }
+
+ for (i = 0; i < eventcnt; ++i)
+ {
+ struct epoll_event *ev = epoll_events + i;
+
+ int fd = (uint32_t)ev->data.u64; /* mask out the lower 32 bits */
+ int want = anfds [fd].events;
+ int got = (ev->events & (EPOLLOUT | EPOLLERR | EPOLLHUP) ? EV_WRITE : 0)
+ | (ev->events & (EPOLLIN | EPOLLERR | EPOLLHUP) ? EV_READ : 0);
+
+ /*
+ * check for spurious notification.
+ * this only finds spurious notifications on egen updates
+ * other spurious notifications will be found by epoll_ctl, below
+ * we assume that fd is always in range, as we never shrink the anfds array
+ */
+ if (ecb_expect_false ((uint32_t)anfds [fd].egen != (uint32_t)(ev->data.u64 >> 32)))
+ {
+ /* recreate kernel state */
+ postfork |= 2;
+ continue;
+ }
+
+ if (ecb_expect_false (got & ~want))
+ {
+ anfds [fd].emask = want;
+
+ /*
+ * we received an event but are not interested in it, try mod or del
+ * this often happens because we optimistically do not unregister fds
+ * when we are no longer interested in them, but also when we get spurious
+ * notifications for fds from another process. this is partially handled
+ * above with the gencounter check (== our fd is not the event fd), and
+ * partially here, when epoll_ctl returns an error (== a child has the fd
+ * but we closed it).
+ * note: for events such as POLLHUP, where we can't know whether it refers
+ * to EV_READ or EV_WRITE, we might issue redundant EPOLL_CTL_MOD calls.
+ */
+ ev->events = (want & EV_READ ? EPOLLIN : 0)
+ | (want & EV_WRITE ? EPOLLOUT : 0);
+
+ /* pre-2.6.9 kernels require a non-null pointer with EPOLL_CTL_DEL, */
+ /* which is fortunately easy to do for us. */
+ if (epoll_ctl (backend_fd, want ? EPOLL_CTL_MOD : EPOLL_CTL_DEL, fd, ev))
+ {
+ postfork |= 2; /* an error occurred, recreate kernel state */
+ continue;
+ }
+ }
+
+ fd_event (EV_A_ fd, got);
+ }
+
+ /* if the receive array was full, increase its size */
+ if (ecb_expect_false (eventcnt == epoll_eventmax))
+ {
+ ev_free (epoll_events);
+ epoll_eventmax = array_nextsize (sizeof (struct epoll_event), epoll_eventmax, epoll_eventmax + 1);
+ epoll_events = (struct epoll_event *)ev_malloc (sizeof (struct epoll_event) * epoll_eventmax);
+ }
+
+ /* now synthesize events for all fds where epoll fails, while select works... */
+ for (i = epoll_epermcnt; i--; )
+ {
+ int fd = epoll_eperms [i];
+ unsigned char events = anfds [fd].events & (EV_READ | EV_WRITE);
+
+ if (anfds [fd].emask & EV_EMASK_EPERM && events)
+ fd_event (EV_A_ fd, events);
+ else
+ {
+ epoll_eperms [i] = epoll_eperms [--epoll_epermcnt];
+ anfds [fd].emask = 0;
+ }
+ }
+}
+
+static int
+epoll_epoll_create (void)
+{
+ int fd;
+
+#if defined EPOLL_CLOEXEC && !defined __ANDROID__
+ fd = epoll_create1 (EPOLL_CLOEXEC);
+
+ if (fd < 0 && (errno == EINVAL || errno == ENOSYS))
+#endif
+ {
+ fd = epoll_create (256);
+
+ if (fd >= 0)
+ fcntl (fd, F_SETFD, FD_CLOEXEC);
+ }
+
+ return fd;
+}
+
+inline_size
+int
+epoll_init (EV_P_ int flags)
+{
+ if ((backend_fd = epoll_epoll_create ()) < 0)
+ return 0;
+
+ backend_mintime = EV_TS_CONST (1e-3); /* epoll does sometimes return early, this is just to avoid the worst */
+ backend_modify = epoll_modify;
+ backend_poll = epoll_poll;
+
+ epoll_eventmax = 64; /* initial number of events receivable per poll */
+ epoll_events = (struct epoll_event *)ev_malloc (sizeof (struct epoll_event) * epoll_eventmax);
+
+ return EVBACKEND_EPOLL;
+}
+
+inline_size
+void
+epoll_destroy (EV_P)
+{
+ ev_free (epoll_events);
+ array_free (epoll_eperm, EMPTY);
+}
+
+ecb_cold
+static void
+epoll_fork (EV_P)
+{
+ close (backend_fd);
+
+ while ((backend_fd = epoll_epoll_create ()) < 0)
+ ev_syserr ("(libev) epoll_create");
+
+ fd_rearm_all (EV_A);
+}
+
diff --git a/contrib/libev/ev_iouring.c b/contrib/libev/ev_iouring.c
new file mode 100644
index 0000000..612391b
--- /dev/null
+++ b/contrib/libev/ev_iouring.c
@@ -0,0 +1,697 @@
+/*
+ * libev linux io_uring fd activity backend
+ *
+ * Copyright (c) 2019-2020 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/*
+ * general notes about linux io_uring:
+ *
+ * a) it's the best interface I have seen so far. on linux.
+ * b) best is not necessarily very good.
+ * c) it's better than the aio mess, doesn't suffer from the fork problems
+ * of linux aio or epoll and so on and so on. and you could do event stuff
+ * without any syscalls. what's not to like?
+ * d) ok, it's vastly more complex, but that's ok, really.
+ * e) why two mmaps instead of one? one would be more space-efficient,
+ * and I can't see what benefit two would have (other than being
+ * somehow resizable/relocatable, but that's apparently not possible).
+ * f) hmm, it's practically undebuggable (gdb can't access the memory, and
+ * the bizarre way structure offsets are communicated makes it hard to
+ * just print the ring buffer heads, even *iff* the memory were visible
+ * in gdb. but then, that's also ok, really.
+ * g) well, you cannot specify a timeout when waiting for events. no,
+ * seriously, the interface doesn't support a timeout. never seen _that_
+ * before. sure, you can use a timerfd, but that's another syscall
+ * you could have avoided. overall, this bizarre omission smells
+ * like a µ-optimisation by the io_uring author for his personal
+ * applications, to the detriment of everybody else who just wants
+ * an event loop. but, umm, ok, if that's all, it could be worse.
+ * (from what I gather from the author Jens Axboe, it simply didn't
+ * occur to him, and he made good on it by adding an unlimited nuber
+ * of timeouts later :).
+ * h) initially there was a hardcoded limit of 4096 outstanding events.
+ * later versions not only bump this to 32k, but also can handle
+ * an unlimited amount of events, so this only affects the batch size.
+ * i) unlike linux aio, you *can* register more then the limit
+ * of fd events. while early verisons of io_uring signalled an overflow
+ * and you ended up getting wet. 5.5+ does not do this anymore.
+ * j) but, oh my! it had exactly the same bugs as the linux aio backend,
+ * where some undocumented poll combinations just fail. fortunately,
+ * after finally reaching the author, he was more than willing to fix
+ * this probably in 5.6+.
+ * k) overall, the *API* itself is, I dare to say, not a total trainwreck.
+ * once the bugs ae fixed (probably in 5.6+), it will be without
+ * competition.
+ */
+
+/* TODO: use internal TIMEOUT */
+/* TODO: take advantage of single mmap, NODROP etc. */
+/* TODO: resize cq/sq size independently */
+
+#include <sys/timerfd.h>
+#include <sys/mman.h>
+#include <poll.h>
+#include <stdint.h>
+
+#define IOURING_INIT_ENTRIES 32
+
+/*****************************************************************************/
+/* syscall wrapdadoop - this section has the raw api/abi definitions */
+
+#include <linux/fs.h>
+#include <linux/types.h>
+
+/* mostly directly taken from the kernel or documentation */
+
+struct io_uring_sqe
+{
+ __u8 opcode;
+ __u8 flags;
+ __u16 ioprio;
+ __s32 fd;
+ union {
+ __u64 off;
+ __u64 addr2;
+ };
+ __u64 addr;
+ __u32 len;
+ union {
+ __kernel_rwf_t rw_flags;
+ __u32 fsync_flags;
+ __u16 poll_events;
+ __u32 sync_range_flags;
+ __u32 msg_flags;
+ __u32 timeout_flags;
+ __u32 accept_flags;
+ __u32 cancel_flags;
+ __u32 open_flags;
+ __u32 statx_flags;
+ };
+ __u64 user_data;
+ union {
+ __u16 buf_index;
+ __u64 __pad2[3];
+ };
+};
+
+struct io_uring_cqe
+{
+ __u64 user_data;
+ __s32 res;
+ __u32 flags;
+};
+
+struct io_sqring_offsets
+{
+ __u32 head;
+ __u32 tail;
+ __u32 ring_mask;
+ __u32 ring_entries;
+ __u32 flags;
+ __u32 dropped;
+ __u32 array;
+ __u32 resv1;
+ __u64 resv2;
+};
+
+struct io_cqring_offsets
+{
+ __u32 head;
+ __u32 tail;
+ __u32 ring_mask;
+ __u32 ring_entries;
+ __u32 overflow;
+ __u32 cqes;
+ __u64 resv[2];
+};
+
+struct io_uring_params
+{
+ __u32 sq_entries;
+ __u32 cq_entries;
+ __u32 flags;
+ __u32 sq_thread_cpu;
+ __u32 sq_thread_idle;
+ __u32 features;
+ __u32 resv[4];
+ struct io_sqring_offsets sq_off;
+ struct io_cqring_offsets cq_off;
+};
+
+#define IORING_SETUP_CQSIZE 0x00000008
+
+#define IORING_OP_POLL_ADD 6
+#define IORING_OP_POLL_REMOVE 7
+#define IORING_OP_TIMEOUT 11
+#define IORING_OP_TIMEOUT_REMOVE 12
+
+/* relative or absolute, reference clock is CLOCK_MONOTONIC */
+struct iouring_kernel_timespec
+{
+ int64_t tv_sec;
+ long long tv_nsec;
+};
+
+#define IORING_TIMEOUT_ABS 0x00000001
+
+#define IORING_ENTER_GETEVENTS 0x01
+
+#define IORING_OFF_SQ_RING 0x00000000ULL
+#define IORING_OFF_CQ_RING 0x08000000ULL
+#define IORING_OFF_SQES 0x10000000ULL
+
+#define IORING_FEAT_SINGLE_MMAP 0x00000001
+#define IORING_FEAT_NODROP 0x00000002
+#define IORING_FEAT_SUBMIT_STABLE 0x00000004
+
+inline_size
+int
+evsys_io_uring_setup (unsigned entries, struct io_uring_params *params)
+{
+ return ev_syscall2 (SYS_io_uring_setup, entries, params);
+}
+
+inline_size
+int
+evsys_io_uring_enter (int fd, unsigned to_submit, unsigned min_complete, unsigned flags, const sigset_t *sig, size_t sigsz)
+{
+ return ev_syscall6 (SYS_io_uring_enter, fd, to_submit, min_complete, flags, sig, sigsz);
+}
+
+/*****************************************************************************/
+/* actual backed implementation */
+
+/* we hope that volatile will make the compiler access this variables only once */
+#define EV_SQ_VAR(name) *(volatile unsigned *)((char *)iouring_sq_ring + iouring_sq_ ## name)
+#define EV_CQ_VAR(name) *(volatile unsigned *)((char *)iouring_cq_ring + iouring_cq_ ## name)
+
+/* the index array */
+#define EV_SQ_ARRAY ((unsigned *)((char *)iouring_sq_ring + iouring_sq_array))
+
+/* the submit/completion queue entries */
+#define EV_SQES ((struct io_uring_sqe *) iouring_sqes)
+#define EV_CQES ((struct io_uring_cqe *)((char *)iouring_cq_ring + iouring_cq_cqes))
+
+inline_speed
+int
+iouring_enter (EV_P_ ev_tstamp timeout)
+{
+ int res;
+
+ EV_RELEASE_CB;
+
+ res = evsys_io_uring_enter (iouring_fd, iouring_to_submit, 1,
+ timeout > EV_TS_CONST (0.) ? IORING_ENTER_GETEVENTS : 0, 0, 0);
+
+ assert (("libev: io_uring_enter did not consume all sqes", (res < 0 || res == iouring_to_submit)));
+
+ iouring_to_submit = 0;
+
+ EV_ACQUIRE_CB;
+
+ return res;
+}
+
+/* TODO: can we move things around so we don't need this forward-reference? */
+static void
+iouring_poll (EV_P_ ev_tstamp timeout);
+
+static
+struct io_uring_sqe *
+iouring_sqe_get (EV_P)
+{
+ unsigned tail;
+
+ for (;;)
+ {
+ tail = EV_SQ_VAR (tail);
+
+ if (ecb_expect_true (tail + 1 - EV_SQ_VAR (head) <= EV_SQ_VAR (ring_entries)))
+ break; /* whats the problem, we have free sqes */
+
+ /* queue full, need to flush and possibly handle some events */
+
+#if EV_FEATURE_CODE
+ /* first we ask the kernel nicely, most often this frees up some sqes */
+ int res = iouring_enter (EV_A_ EV_TS_CONST (0.));
+
+ ECB_MEMORY_FENCE_ACQUIRE; /* better safe than sorry */
+
+ if (res >= 0)
+ continue; /* yes, it worked, try again */
+#endif
+
+ /* some problem, possibly EBUSY - do the full poll and let it handle any issues */
+
+ iouring_poll (EV_A_ EV_TS_CONST (0.));
+ /* iouring_poll should have done ECB_MEMORY_FENCE_ACQUIRE for us */
+ }
+
+ /*assert (("libev: io_uring queue full after flush", tail + 1 - EV_SQ_VAR (head) <= EV_SQ_VAR (ring_entries)));*/
+
+ return EV_SQES + (tail & EV_SQ_VAR (ring_mask));
+}
+
+inline_size
+struct io_uring_sqe *
+iouring_sqe_submit (EV_P_ struct io_uring_sqe *sqe)
+{
+ unsigned idx = sqe - EV_SQES;
+
+ EV_SQ_ARRAY [idx] = idx;
+ ECB_MEMORY_FENCE_RELEASE;
+ ++EV_SQ_VAR (tail);
+ // ECB_MEMORY_FENCE_RELEASE; /* for the time being we assume this is not needed */
+ ++iouring_to_submit;
+ return sqe;
+}
+
+/*****************************************************************************/
+
+/* when the timerfd expires we simply note the fact,
+ * as the purpose of the timerfd is to wake us up, nothing else.
+ * the next iteration should re-set it.
+ */
+static void
+iouring_tfd_cb (EV_P_ struct ev_io *w, int revents)
+{
+ iouring_tfd_to = EV_TSTAMP_HUGE;
+}
+
+/* called for full and partial cleanup */
+ecb_cold
+static int
+iouring_internal_destroy (EV_P)
+{
+ close (iouring_tfd);
+ close (iouring_fd);
+
+ if (iouring_sq_ring != MAP_FAILED) munmap (iouring_sq_ring, iouring_sq_ring_size);
+ if (iouring_cq_ring != MAP_FAILED) munmap (iouring_cq_ring, iouring_cq_ring_size);
+ if (iouring_sqes != MAP_FAILED) munmap (iouring_sqes , iouring_sqes_size );
+
+ if (ev_is_active (&iouring_tfd_w))
+ {
+ ev_ref (EV_A);
+ ev_io_stop (EV_A_ &iouring_tfd_w);
+ }
+
+ return 0;
+}
+
+ecb_cold
+static int
+iouring_internal_init (EV_P)
+{
+ struct io_uring_params params = { 0 };
+
+ iouring_to_submit = 0;
+
+ iouring_tfd = -1;
+ iouring_sq_ring = MAP_FAILED;
+ iouring_cq_ring = MAP_FAILED;
+ iouring_sqes = MAP_FAILED;
+
+ if (!have_monotonic) /* cannot really happen, but what if11 */
+ return -1;
+
+ for (;;)
+ {
+ iouring_fd = evsys_io_uring_setup (iouring_entries, &params);
+
+ if (iouring_fd >= 0)
+ break; /* yippie */
+
+ if (errno != EINVAL)
+ return -1; /* we failed */
+
+#if TODO
+ if ((~params.features) & (IORING_FEAT_NODROP | IORING_FEATURE_SINGLE_MMAP | IORING_FEAT_SUBMIT_STABLE))
+ return -1; /* we require the above features */
+#endif
+
+ /* EINVAL: lots of possible reasons, but maybe
+ * it is because we hit the unqueryable hardcoded size limit
+ */
+
+ /* we hit the limit already, give up */
+ if (iouring_max_entries)
+ return -1;
+
+ /* first time we hit EINVAL? assume we hit the limit, so go back and retry */
+ iouring_entries >>= 1;
+ iouring_max_entries = iouring_entries;
+ }
+
+ iouring_sq_ring_size = params.sq_off.array + params.sq_entries * sizeof (unsigned);
+ iouring_cq_ring_size = params.cq_off.cqes + params.cq_entries * sizeof (struct io_uring_cqe);
+ iouring_sqes_size = params.sq_entries * sizeof (struct io_uring_sqe);
+
+ iouring_sq_ring = mmap (0, iouring_sq_ring_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, iouring_fd, IORING_OFF_SQ_RING);
+ iouring_cq_ring = mmap (0, iouring_cq_ring_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, iouring_fd, IORING_OFF_CQ_RING);
+ iouring_sqes = mmap (0, iouring_sqes_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, iouring_fd, IORING_OFF_SQES);
+
+ if (iouring_sq_ring == MAP_FAILED || iouring_cq_ring == MAP_FAILED || iouring_sqes == MAP_FAILED)
+ return -1;
+
+ iouring_sq_head = params.sq_off.head;
+ iouring_sq_tail = params.sq_off.tail;
+ iouring_sq_ring_mask = params.sq_off.ring_mask;
+ iouring_sq_ring_entries = params.sq_off.ring_entries;
+ iouring_sq_flags = params.sq_off.flags;
+ iouring_sq_dropped = params.sq_off.dropped;
+ iouring_sq_array = params.sq_off.array;
+
+ iouring_cq_head = params.cq_off.head;
+ iouring_cq_tail = params.cq_off.tail;
+ iouring_cq_ring_mask = params.cq_off.ring_mask;
+ iouring_cq_ring_entries = params.cq_off.ring_entries;
+ iouring_cq_overflow = params.cq_off.overflow;
+ iouring_cq_cqes = params.cq_off.cqes;
+
+ iouring_tfd = timerfd_create (CLOCK_MONOTONIC, TFD_CLOEXEC);
+
+ if (iouring_tfd < 0)
+ return iouring_tfd;
+
+ iouring_tfd_to = EV_TSTAMP_HUGE;
+
+ return 0;
+}
+
+ecb_cold
+static void
+iouring_fork (EV_P)
+{
+ iouring_internal_destroy (EV_A);
+
+ while (iouring_internal_init (EV_A) < 0)
+ ev_syserr ("(libev) io_uring_setup");
+
+ fd_rearm_all (EV_A);
+
+ ev_io_stop (EV_A_ &iouring_tfd_w);
+ ev_io_set (EV_A_ &iouring_tfd_w, iouring_tfd, EV_READ);
+ ev_io_start (EV_A_ &iouring_tfd_w);
+}
+
+/*****************************************************************************/
+
+static void
+iouring_modify (EV_P_ int fd, int oev, int nev)
+{
+ if (oev)
+ {
+ /* we assume the sqe's are all "properly" initialised */
+ struct io_uring_sqe *sqe = iouring_sqe_get (EV_A);
+ sqe->opcode = IORING_OP_POLL_REMOVE;
+ sqe->fd = fd;
+ /* Jens Axboe notified me that user_data is not what is documented, but is
+ * some kind of unique ID that has to match, otherwise the request cannot
+ * be removed. Since we don't *really* have that, we pass in the old
+ * generation counter - if that fails, too bad, it will hopefully be removed
+ * at close time and then be ignored. */
+ sqe->addr = (uint32_t)fd | ((__u64)(uint32_t)anfds [fd].egen << 32);
+ sqe->user_data = (uint64_t)-1;
+ iouring_sqe_submit (EV_A_ sqe);
+
+ /* increment generation counter to avoid handling old events */
+ ++anfds [fd].egen;
+ }
+
+ if (nev)
+ {
+ struct io_uring_sqe *sqe = iouring_sqe_get (EV_A);
+ sqe->opcode = IORING_OP_POLL_ADD;
+ sqe->fd = fd;
+ sqe->addr = 0;
+ sqe->user_data = (uint32_t)fd | ((__u64)(uint32_t)anfds [fd].egen << 32);
+ sqe->poll_events =
+ (nev & EV_READ ? POLLIN : 0)
+ | (nev & EV_WRITE ? POLLOUT : 0);
+ iouring_sqe_submit (EV_A_ sqe);
+ }
+}
+
+inline_size
+void
+iouring_tfd_update (EV_P_ ev_tstamp timeout)
+{
+ ev_tstamp tfd_to = mn_now + timeout;
+
+ /* we assume there will be many iterations per timer change, so
+ * we only re-set the timerfd when we have to because its expiry
+ * is too late.
+ */
+ if (ecb_expect_false (tfd_to < iouring_tfd_to))
+ {
+ struct itimerspec its;
+
+ iouring_tfd_to = tfd_to;
+ EV_TS_SET (its.it_interval, 0.);
+ EV_TS_SET (its.it_value, tfd_to);
+
+ if (timerfd_settime (iouring_tfd, TFD_TIMER_ABSTIME, &its, 0) < 0)
+ assert (("libev: iouring timerfd_settime failed", 0));
+ }
+}
+
+inline_size
+void
+iouring_process_cqe (EV_P_ struct io_uring_cqe *cqe)
+{
+ int fd = cqe->user_data & 0xffffffffU;
+ uint32_t gen = cqe->user_data >> 32;
+ int res = cqe->res;
+
+ /* user_data -1 is a remove that we are not atm. interested in */
+ if (cqe->user_data == (uint64_t)-1)
+ return;
+
+ assert (("libev: io_uring fd must be in-bounds", fd >= 0 && fd < anfdmax));
+
+ /* documentation lies, of course. the result value is NOT like
+ * normal syscalls, but like linux raw syscalls, i.e. negative
+ * error numbers. fortunate, as otherwise there would be no way
+ * to get error codes at all. still, why not document this?
+ */
+
+ /* ignore event if generation doesn't match */
+ /* other than skipping removal events, */
+ /* this should actually be very rare */
+ if (ecb_expect_false (gen != (uint32_t)anfds [fd].egen))
+ return;
+
+ if (ecb_expect_false (res < 0))
+ {
+ /*TODO: EINVAL handling (was something failed with this fd)*/
+
+ if (res == -EBADF)
+ {
+ assert (("libev: event loop rejected bad fd", res != -EBADF));
+ fd_kill (EV_A_ fd);
+ }
+ else
+ {
+ errno = -res;
+ ev_syserr ("(libev) IORING_OP_POLL_ADD");
+ }
+
+ return;
+ }
+
+ /* feed events, we do not expect or handle POLLNVAL */
+ fd_event (
+ EV_A_
+ fd,
+ (res & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
+ | (res & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
+ );
+
+ /* io_uring is oneshot, so we need to re-arm the fd next iteration */
+ /* this also means we usually have to do at least one syscall per iteration */
+ anfds [fd].events = 0;
+ fd_change (EV_A_ fd, EV_ANFD_REIFY);
+}
+
+/* called when the event queue overflows */
+ecb_cold
+static void
+iouring_overflow (EV_P)
+{
+ /* we have two options, resize the queue (by tearing down
+ * everything and recreating it, or living with it
+ * and polling.
+ * we implement this by resizing the queue, and, if that fails,
+ * we just recreate the state on every failure, which
+ * kind of is a very inefficient poll.
+ * one danger is, due to the bios toward lower fds,
+ * we will only really get events for those, so
+ * maybe we need a poll() fallback, after all.
+ */
+ /*EV_CQ_VAR (overflow) = 0;*/ /* need to do this if we keep the state and poll manually */
+
+ fd_rearm_all (EV_A);
+
+ /* we double the size until we hit the hard-to-probe maximum */
+ if (!iouring_max_entries)
+ {
+ iouring_entries <<= 1;
+ iouring_fork (EV_A);
+ }
+ else
+ {
+ /* we hit the kernel limit, we should fall back to something else.
+ * we can either poll() a few times and hope for the best,
+ * poll always, or switch to epoll.
+ * TODO: is this necessary with newer kernels?
+ */
+
+ iouring_internal_destroy (EV_A);
+
+ /* this should make it so that on return, we don't call any uring functions */
+ iouring_to_submit = 0;
+
+ for (;;)
+ {
+ backend = epoll_init (EV_A_ 0);
+
+ if (backend)
+ break;
+
+ ev_syserr ("(libev) iouring switch to epoll");
+ }
+ }
+}
+
+/* handle any events in the completion queue, return true if there were any */
+static int
+iouring_handle_cq (EV_P)
+{
+ unsigned head, tail, mask;
+
+ head = EV_CQ_VAR (head);
+ ECB_MEMORY_FENCE_ACQUIRE;
+ tail = EV_CQ_VAR (tail);
+
+ if (head == tail)
+ return 0;
+
+ /* it can only overflow if we have events, yes, yes? */
+ if (ecb_expect_false (EV_CQ_VAR (overflow)))
+ {
+ iouring_overflow (EV_A);
+ return 1;
+ }
+
+ mask = EV_CQ_VAR (ring_mask);
+
+ do
+ iouring_process_cqe (EV_A_ &EV_CQES [head++ & mask]);
+ while (head != tail);
+
+ EV_CQ_VAR (head) = head;
+ ECB_MEMORY_FENCE_RELEASE;
+
+ return 1;
+}
+
+static void
+iouring_poll (EV_P_ ev_tstamp timeout)
+{
+ /* if we have events, no need for extra syscalls, but we might have to queue events */
+ /* we also clar the timeout if there are outstanding fdchanges */
+ /* the latter should only happen if both the sq and cq are full, most likely */
+ /* because we have a lot of event sources that immediately complete */
+ /* TODO: fdchacngecnt is always 0 because fd_reify does not have two buffers yet */
+ if (iouring_handle_cq (EV_A) || fdchangecnt)
+ timeout = EV_TS_CONST (0.);
+ else
+ /* no events, so maybe wait for some */
+ iouring_tfd_update (EV_A_ timeout);
+
+ /* only enter the kernel if we have something to submit, or we need to wait */
+ if (timeout || iouring_to_submit)
+ {
+ int res = iouring_enter (EV_A_ timeout);
+
+ if (ecb_expect_false (res < 0))
+ if (errno == EINTR)
+ /* ignore */;
+ else if (errno == EBUSY)
+ /* cq full, cannot submit - should be rare because we flush the cq first, so simply ignore */;
+ else
+ ev_syserr ("(libev) iouring setup");
+ else
+ iouring_handle_cq (EV_A);
+ }
+}
+
+inline_size
+int
+iouring_init (EV_P_ int flags)
+{
+ iouring_entries = IOURING_INIT_ENTRIES;
+ iouring_max_entries = 0;
+
+ if (iouring_internal_init (EV_A) < 0)
+ {
+ iouring_internal_destroy (EV_A);
+ return 0;
+ }
+
+ ev_io_init (&iouring_tfd_w, iouring_tfd_cb, iouring_tfd, EV_READ);
+ ev_set_priority (&iouring_tfd_w, EV_MINPRI);
+ ev_io_start (EV_A_ &iouring_tfd_w);
+ ev_unref (EV_A); /* watcher should not keep loop alive */
+
+ backend_modify = iouring_modify;
+ backend_poll = iouring_poll;
+
+ return EVBACKEND_IOURING;
+}
+
+inline_size
+void
+iouring_destroy (EV_P)
+{
+ iouring_internal_destroy (EV_A);
+}
+
diff --git a/contrib/libev/ev_kqueue.c b/contrib/libev/ev_kqueue.c
new file mode 100644
index 0000000..69c5147
--- /dev/null
+++ b/contrib/libev/ev_kqueue.c
@@ -0,0 +1,224 @@
+/*
+ * libev kqueue backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2012,2013,2016,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/event.h>
+#include <string.h>
+#include <errno.h>
+
+inline_speed
+void
+kqueue_change (EV_P_ int fd, int filter, int flags, int fflags)
+{
+ ++kqueue_changecnt;
+ array_needsize (struct kevent, kqueue_changes, kqueue_changemax, kqueue_changecnt, array_needsize_noinit);
+
+ EV_SET (&kqueue_changes [kqueue_changecnt - 1], fd, filter, flags, fflags, 0, 0);
+}
+
+/* OS X at least needs this */
+#ifndef EV_ENABLE
+# define EV_ENABLE 0
+#endif
+#ifndef NOTE_EOF
+# define NOTE_EOF 0
+#endif
+
+static void
+kqueue_modify (EV_P_ int fd, int oev, int nev)
+{
+ if (oev != nev)
+ {
+ if (oev & EV_READ)
+ kqueue_change (EV_A_ fd, EVFILT_READ , EV_DELETE, 0);
+
+ if (oev & EV_WRITE)
+ kqueue_change (EV_A_ fd, EVFILT_WRITE, EV_DELETE, 0);
+ }
+
+ /* to detect close/reopen reliably, we have to re-add */
+ /* event requests even when oev == nev */
+
+ if (nev & EV_READ)
+ kqueue_change (EV_A_ fd, EVFILT_READ , EV_ADD | EV_ENABLE, NOTE_EOF);
+
+ if (nev & EV_WRITE)
+ kqueue_change (EV_A_ fd, EVFILT_WRITE, EV_ADD | EV_ENABLE, NOTE_EOF);
+}
+
+static void
+kqueue_poll (EV_P_ ev_tstamp timeout)
+{
+ int res, i;
+ struct timespec ts;
+
+ /* need to resize so there is enough space for errors */
+ if (kqueue_changecnt > kqueue_eventmax)
+ {
+ ev_free (kqueue_events);
+ kqueue_eventmax = array_nextsize (sizeof (struct kevent), kqueue_eventmax, kqueue_changecnt);
+ kqueue_events = (struct kevent *)ev_malloc (sizeof (struct kevent) * kqueue_eventmax);
+ }
+
+ EV_RELEASE_CB;
+ EV_TS_SET (ts, timeout);
+ res = kevent (backend_fd, kqueue_changes, kqueue_changecnt, kqueue_events, kqueue_eventmax, &ts);
+ EV_ACQUIRE_CB;
+ kqueue_changecnt = 0;
+
+ if (ecb_expect_false (res < 0))
+ {
+ if (errno != EINTR)
+ ev_syserr ("(libev) kqueue kevent");
+
+ return;
+ }
+
+ for (i = 0; i < res; ++i)
+ {
+ int fd = kqueue_events [i].ident;
+
+ if (ecb_expect_false (kqueue_events [i].flags & EV_ERROR))
+ {
+ int err = kqueue_events [i].data;
+
+ /* we are only interested in errors for fds that we are interested in :) */
+ if (anfds [fd].events)
+ {
+ if (err == ENOENT) /* resubmit changes on ENOENT */
+ kqueue_modify (EV_A_ fd, 0, anfds [fd].events);
+ else if (err == EBADF) /* on EBADF, we re-check the fd */
+ {
+ if (fd_valid (fd))
+ kqueue_modify (EV_A_ fd, 0, anfds [fd].events);
+ else
+ {
+ assert (("libev: kqueue found invalid fd", 0));
+ fd_kill (EV_A_ fd);
+ }
+ }
+ else /* on all other errors, we error out on the fd */
+ {
+ assert (("libev: kqueue found invalid fd", 0));
+ fd_kill (EV_A_ fd);
+ }
+ }
+ }
+ else
+ fd_event (
+ EV_A_
+ fd,
+ kqueue_events [i].filter == EVFILT_READ ? EV_READ
+ : kqueue_events [i].filter == EVFILT_WRITE ? EV_WRITE
+ : 0
+ );
+ }
+
+ if (ecb_expect_false (res == kqueue_eventmax))
+ {
+ ev_free (kqueue_events);
+ kqueue_eventmax = array_nextsize (sizeof (struct kevent), kqueue_eventmax, kqueue_eventmax + 1);
+ kqueue_events = (struct kevent *)ev_malloc (sizeof (struct kevent) * kqueue_eventmax);
+ }
+}
+
+inline_size
+int
+kqueue_init (EV_P_ int flags)
+{
+ /* initialize the kernel queue */
+ kqueue_fd_pid = getpid ();
+ if ((backend_fd = kqueue ()) < 0)
+ return 0;
+
+ fcntl (backend_fd, F_SETFD, FD_CLOEXEC); /* not sure if necessary, hopefully doesn't hurt */
+
+ backend_mintime = EV_TS_CONST (1e-9); /* apparently, they did the right thing in freebsd */
+ backend_modify = kqueue_modify;
+ backend_poll = kqueue_poll;
+
+ kqueue_eventmax = 64; /* initial number of events receivable per poll */
+ kqueue_events = (struct kevent *)ev_malloc (sizeof (struct kevent) * kqueue_eventmax);
+
+ kqueue_changes = 0;
+ kqueue_changemax = 0;
+ kqueue_changecnt = 0;
+
+ return EVBACKEND_KQUEUE;
+}
+
+inline_size
+void
+kqueue_destroy (EV_P)
+{
+ ev_free (kqueue_events);
+ ev_free (kqueue_changes);
+}
+
+inline_size
+void
+kqueue_fork (EV_P)
+{
+ /* some BSD kernels don't just destroy the kqueue itself,
+ * but also close the fd, which isn't documented, and
+ * impossible to support properly.
+ * we remember the pid of the kqueue call and only close
+ * the fd if the pid is still the same.
+ * this leaks fds on sane kernels, but BSD interfaces are
+ * notoriously buggy and rarely get fixed.
+ */
+ pid_t newpid = getpid ();
+
+ if (newpid == kqueue_fd_pid)
+ close (backend_fd);
+
+ kqueue_fd_pid = newpid;
+ while ((backend_fd = kqueue ()) < 0)
+ ev_syserr ("(libev) kqueue");
+
+ fcntl (backend_fd, F_SETFD, FD_CLOEXEC);
+
+ /* re-register interest in fds */
+ fd_rearm_all (EV_A);
+}
+
+/* sys/event.h defines EV_ERROR */
+#undef EV_ERROR
+
diff --git a/contrib/libev/ev_linuxaio.c b/contrib/libev/ev_linuxaio.c
new file mode 100644
index 0000000..4687a70
--- /dev/null
+++ b/contrib/libev/ev_linuxaio.c
@@ -0,0 +1,620 @@
+/*
+ * libev linux aio fd activity backend
+ *
+ * Copyright (c) 2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/*
+ * general notes about linux aio:
+ *
+ * a) at first, the linux aio IOCB_CMD_POLL functionality introduced in
+ * 4.18 looks too good to be true: both watchers and events can be
+ * batched, and events can even be handled in userspace using
+ * a ring buffer shared with the kernel. watchers can be canceled
+ * regardless of whether the fd has been closed. no problems with fork.
+ * ok, the ring buffer is 200% undocumented (there isn't even a
+ * header file), but otherwise, it's pure bliss!
+ * b) ok, watchers are one-shot, so you have to re-arm active ones
+ * on every iteration. so much for syscall-less event handling,
+ * but at least these re-arms can be batched, no big deal, right?
+ * c) well, linux as usual: the documentation lies to you: io_submit
+ * sometimes returns EINVAL because the kernel doesn't feel like
+ * handling your poll mask - ttys can be polled for POLLOUT,
+ * POLLOUT|POLLIN, but polling for POLLIN fails. just great,
+ * so we have to fall back to something else (hello, epoll),
+ * but at least the fallback can be slow, because these are
+ * exceptional cases, right?
+ * d) hmm, you have to tell the kernel the maximum number of watchers
+ * you want to queue when initialising the aio context. but of
+ * course the real limit is magically calculated in the kernel, and
+ * is often higher then we asked for. so we just have to destroy
+ * the aio context and re-create it a bit larger if we hit the limit.
+ * (starts to remind you of epoll? well, it's a bit more deterministic
+ * and less gambling, but still ugly as hell).
+ * e) that's when you find out you can also hit an arbitrary system-wide
+ * limit. or the kernel simply doesn't want to handle your watchers.
+ * what the fuck do we do then? you guessed it, in the middle
+ * of event handling we have to switch to 100% epoll polling. and
+ * that better is as fast as normal epoll polling, so you practically
+ * have to use the normal epoll backend with all its quirks.
+ * f) end result of this train wreck: it inherits all the disadvantages
+ * from epoll, while adding a number on its own. why even bother to use
+ * it? because if conditions are right and your fds are supported and you
+ * don't hit a limit, this backend is actually faster, doesn't gamble with
+ * your fds, batches watchers and events and doesn't require costly state
+ * recreates. well, until it does.
+ * g) all of this makes this backend use almost twice as much code as epoll.
+ * which in turn uses twice as much code as poll. and that#s not counting
+ * the fact that this backend also depends on the epoll backend, making
+ * it three times as much code as poll, or kqueue.
+ * h) bleah. why can't linux just do kqueue. sure kqueue is ugly, but by now
+ * it's clear that whatever linux comes up with is far, far, far worse.
+ */
+
+#include <sys/time.h> /* actually linux/time.h, but we must assume they are compatible */
+#include <poll.h>
+#include <linux/aio_abi.h>
+
+/*****************************************************************************/
+/* syscall wrapdadoop - this section has the raw api/abi definitions */
+
+#include <sys/syscall.h> /* no glibc wrappers */
+
+/* aio_abi.h is not versioned in any way, so we cannot test for its existance */
+#define IOCB_CMD_POLL 5
+
+/* taken from linux/fs/aio.c. yup, that's a .c file.
+ * not only is this totally undocumented, not even the source code
+ * can tell you what the future semantics of compat_features and
+ * incompat_features are, or what header_length actually is for.
+ */
+#define AIO_RING_MAGIC 0xa10a10a1
+#define EV_AIO_RING_INCOMPAT_FEATURES 0
+struct aio_ring
+{
+ unsigned id; /* kernel internal index number */
+ unsigned nr; /* number of io_events */
+ unsigned head; /* Written to by userland or by kernel. */
+ unsigned tail;
+
+ unsigned magic;
+ unsigned compat_features;
+ unsigned incompat_features;
+ unsigned header_length; /* size of aio_ring */
+
+ struct io_event io_events[0];
+};
+
+inline_size
+int
+evsys_io_setup (unsigned nr_events, aio_context_t *ctx_idp)
+{
+ return ev_syscall2 (SYS_io_setup, nr_events, ctx_idp);
+}
+
+inline_size
+int
+evsys_io_destroy (aio_context_t ctx_id)
+{
+ return ev_syscall1 (SYS_io_destroy, ctx_id);
+}
+
+inline_size
+int
+evsys_io_submit (aio_context_t ctx_id, long nr, struct iocb *cbp[])
+{
+ return ev_syscall3 (SYS_io_submit, ctx_id, nr, cbp);
+}
+
+inline_size
+int
+evsys_io_cancel (aio_context_t ctx_id, struct iocb *cbp, struct io_event *result)
+{
+ return ev_syscall3 (SYS_io_cancel, ctx_id, cbp, result);
+}
+
+inline_size
+int
+evsys_io_getevents (aio_context_t ctx_id, long min_nr, long nr, struct io_event *events, struct timespec *timeout)
+{
+ return ev_syscall5 (SYS_io_getevents, ctx_id, min_nr, nr, events, timeout);
+}
+
+/*****************************************************************************/
+/* actual backed implementation */
+
+ecb_cold
+static int
+linuxaio_nr_events (EV_P)
+{
+ /* we start with 16 iocbs and incraese from there
+ * that's tiny, but the kernel has a rather low system-wide
+ * limit that can be reached quickly, so let's be parsimonious
+ * with this resource.
+ * Rest assured, the kernel generously rounds up small and big numbers
+ * in different ways (but doesn't seem to charge you for it).
+ * The 15 here is because the kernel usually has a power of two as aio-max-nr,
+ * and this helps to take advantage of that limit.
+ */
+
+ /* we try to fill 4kB pages exactly.
+ * the ring buffer header is 32 bytes, every io event is 32 bytes.
+ * the kernel takes the io requests number, doubles it, adds 2
+ * and adds the ring buffer.
+ * the way we use this is by starting low, and then roughly doubling the
+ * size each time we hit a limit.
+ */
+
+ int requests = 15 << linuxaio_iteration;
+ int one_page = (4096
+ / sizeof (struct io_event) ) / 2; /* how many fit into one page */
+ int first_page = ((4096 - sizeof (struct aio_ring))
+ / sizeof (struct io_event) - 2) / 2; /* how many fit into the first page */
+
+ /* if everything fits into one page, use count exactly */
+ if (requests > first_page)
+ /* otherwise, round down to full pages and add the first page */
+ requests = requests / one_page * one_page + first_page;
+
+ return requests;
+}
+
+/* we use out own wrapper structure in case we ever want to do something "clever" */
+typedef struct aniocb
+{
+ struct iocb io;
+ /*int inuse;*/
+} *ANIOCBP;
+
+inline_size
+void
+linuxaio_array_needsize_iocbp (ANIOCBP *base, int offset, int count)
+{
+ while (count--)
+ {
+ /* TODO: quite the overhead to allocate every iocb separately, maybe use our own allocator? */
+ ANIOCBP iocb = (ANIOCBP)ev_malloc (sizeof (*iocb));
+
+ /* full zero initialise is probably not required at the moment, but
+ * this is not well documented, so we better do it.
+ */
+ memset (iocb, 0, sizeof (*iocb));
+
+ iocb->io.aio_lio_opcode = IOCB_CMD_POLL;
+ iocb->io.aio_fildes = offset;
+
+ base [offset++] = iocb;
+ }
+}
+
+ecb_cold
+static void
+linuxaio_free_iocbp (EV_P)
+{
+ while (linuxaio_iocbpmax--)
+ ev_free (linuxaio_iocbps [linuxaio_iocbpmax]);
+
+ linuxaio_iocbpmax = 0; /* next resize will completely reallocate the array, at some overhead */
+}
+
+static void
+linuxaio_modify (EV_P_ int fd, int oev, int nev)
+{
+ array_needsize (ANIOCBP, linuxaio_iocbps, linuxaio_iocbpmax, fd + 1, linuxaio_array_needsize_iocbp);
+ ANIOCBP iocb = linuxaio_iocbps [fd];
+ ANFD *anfd = &anfds [fd];
+
+ if (ecb_expect_false (iocb->io.aio_reqprio < 0))
+ {
+ /* we handed this fd over to epoll, so undo this first */
+ /* we do it manually because the optimisations on epoll_modify won't do us any good */
+ epoll_ctl (backend_fd, EPOLL_CTL_DEL, fd, 0);
+ anfd->emask = 0;
+ iocb->io.aio_reqprio = 0;
+ }
+ else if (ecb_expect_false (iocb->io.aio_buf))
+ {
+ /* iocb active, so cancel it first before resubmit */
+ /* this assumes we only ever get one call per fd per loop iteration */
+ for (;;)
+ {
+ /* on all relevant kernels, io_cancel fails with EINPROGRESS on "success" */
+ if (ecb_expect_false (evsys_io_cancel (linuxaio_ctx, &iocb->io, (struct io_event *)0) == 0))
+ break;
+
+ if (ecb_expect_true (errno == EINPROGRESS))
+ break;
+
+ /* the EINPROGRESS test is for nicer error message. clumsy. */
+ if (errno != EINTR)
+ {
+ assert (("libev: linuxaio unexpected io_cancel failed", errno != EINTR && errno != EINPROGRESS));
+ break;
+ }
+ }
+
+ /* increment generation counter to avoid handling old events */
+ ++anfd->egen;
+ }
+
+ iocb->io.aio_buf = (nev & EV_READ ? POLLIN : 0)
+ | (nev & EV_WRITE ? POLLOUT : 0);
+
+ if (nev)
+ {
+ iocb->io.aio_data = (uint32_t)fd | ((__u64)(uint32_t)anfd->egen << 32);
+
+ /* queue iocb up for io_submit */
+ /* this assumes we only ever get one call per fd per loop iteration */
+ ++linuxaio_submitcnt;
+ array_needsize (struct iocb *, linuxaio_submits, linuxaio_submitmax, linuxaio_submitcnt, array_needsize_noinit);
+ linuxaio_submits [linuxaio_submitcnt - 1] = &iocb->io;
+ }
+}
+
+static void
+linuxaio_epoll_cb (EV_P_ struct ev_io *w, int revents)
+{
+ epoll_poll (EV_A_ 0);
+}
+
+inline_speed
+void
+linuxaio_fd_rearm (EV_P_ int fd)
+{
+ anfds [fd].events = 0;
+ linuxaio_iocbps [fd]->io.aio_buf = 0;
+ fd_change (EV_A_ fd, EV_ANFD_REIFY);
+}
+
+static void
+linuxaio_parse_events (EV_P_ struct io_event *ev, int nr)
+{
+ while (nr)
+ {
+ int fd = ev->data & 0xffffffff;
+ uint32_t gen = ev->data >> 32;
+ int res = ev->res;
+
+ assert (("libev: iocb fd must be in-bounds", fd >= 0 && fd < anfdmax));
+
+ /* only accept events if generation counter matches */
+ if (ecb_expect_true (gen == (uint32_t)anfds [fd].egen))
+ {
+ /* feed events, we do not expect or handle POLLNVAL */
+ fd_event (
+ EV_A_
+ fd,
+ (res & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
+ | (res & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
+ );
+
+ /* linux aio is oneshot: rearm fd. TODO: this does more work than strictly needed */
+ linuxaio_fd_rearm (EV_A_ fd);
+ }
+
+ --nr;
+ ++ev;
+ }
+}
+
+/* get any events from ring buffer, return true if any were handled */
+static int
+linuxaio_get_events_from_ring (EV_P)
+{
+ struct aio_ring *ring = (struct aio_ring *)linuxaio_ctx;
+ unsigned head, tail;
+
+ /* the kernel reads and writes both of these variables, */
+ /* as a C extension, we assume that volatile use here */
+ /* both makes reads atomic and once-only */
+ head = *(volatile unsigned *)&ring->head;
+ ECB_MEMORY_FENCE_ACQUIRE;
+ tail = *(volatile unsigned *)&ring->tail;
+
+ if (head == tail)
+ return 0;
+
+ /* parse all available events, but only once, to avoid starvation */
+ if (ecb_expect_true (tail > head)) /* normal case around */
+ linuxaio_parse_events (EV_A_ ring->io_events + head, tail - head);
+ else /* wrapped around */
+ {
+ linuxaio_parse_events (EV_A_ ring->io_events + head, ring->nr - head);
+ linuxaio_parse_events (EV_A_ ring->io_events, tail);
+ }
+
+ ECB_MEMORY_FENCE_RELEASE;
+ /* as an extension to C, we hope that the volatile will make this atomic and once-only */
+ *(volatile unsigned *)&ring->head = tail;
+
+ return 1;
+}
+
+inline_size
+int
+linuxaio_ringbuf_valid (EV_P)
+{
+ struct aio_ring *ring = (struct aio_ring *)linuxaio_ctx;
+
+ return ecb_expect_true (ring->magic == AIO_RING_MAGIC)
+ && ring->incompat_features == EV_AIO_RING_INCOMPAT_FEATURES
+ && ring->header_length == sizeof (struct aio_ring); /* TODO: or use it to find io_event[0]? */
+}
+
+/* read at least one event from kernel, or timeout */
+inline_size
+void
+linuxaio_get_events (EV_P_ ev_tstamp timeout)
+{
+ struct timespec ts;
+ struct io_event ioev[8]; /* 256 octet stack space */
+ int want = 1; /* how many events to request */
+ int ringbuf_valid = linuxaio_ringbuf_valid (EV_A);
+
+ if (ecb_expect_true (ringbuf_valid))
+ {
+ /* if the ring buffer has any events, we don't wait or call the kernel at all */
+ if (linuxaio_get_events_from_ring (EV_A))
+ return;
+
+ /* if the ring buffer is empty, and we don't have a timeout, then don't call the kernel */
+ if (!timeout)
+ return;
+ }
+ else
+ /* no ringbuffer, request slightly larger batch */
+ want = sizeof (ioev) / sizeof (ioev [0]);
+
+ /* no events, so wait for some
+ * for fairness reasons, we do this in a loop, to fetch all events
+ */
+ for (;;)
+ {
+ int res;
+
+ EV_RELEASE_CB;
+
+ EV_TS_SET (ts, timeout);
+ res = evsys_io_getevents (linuxaio_ctx, 1, want, ioev, &ts);
+
+ EV_ACQUIRE_CB;
+
+ if (res < 0)
+ if (errno == EINTR)
+ /* ignored, retry */;
+ else
+ ev_syserr ("(libev) linuxaio io_getevents");
+ else if (res)
+ {
+ /* at least one event available, handle them */
+ linuxaio_parse_events (EV_A_ ioev, res);
+
+ if (ecb_expect_true (ringbuf_valid))
+ {
+ /* if we have a ring buffer, handle any remaining events in it */
+ linuxaio_get_events_from_ring (EV_A);
+
+ /* at this point, we should have handled all outstanding events */
+ break;
+ }
+ else if (res < want)
+ /* otherwise, if there were fewere events than we wanted, we assume there are no more */
+ break;
+ }
+ else
+ break; /* no events from the kernel, we are done */
+
+ timeout = EV_TS_CONST (0.); /* only wait in the first iteration */
+ }
+}
+
+inline_size
+int
+linuxaio_io_setup (EV_P)
+{
+ linuxaio_ctx = 0;
+ return evsys_io_setup (linuxaio_nr_events (EV_A), &linuxaio_ctx);
+}
+
+static void
+linuxaio_poll (EV_P_ ev_tstamp timeout)
+{
+ int submitted;
+
+ /* first phase: submit new iocbs */
+
+ /* io_submit might return less than the requested number of iocbs */
+ /* this is, afaics, only because of errors, but we go by the book and use a loop, */
+ /* which allows us to pinpoint the erroneous iocb */
+ for (submitted = 0; submitted < linuxaio_submitcnt; )
+ {
+ int res = evsys_io_submit (linuxaio_ctx, linuxaio_submitcnt - submitted, linuxaio_submits + submitted);
+
+ if (ecb_expect_false (res < 0))
+ if (errno == EINVAL)
+ {
+ /* This happens for unsupported fds, officially, but in my testing,
+ * also randomly happens for supported fds. We fall back to good old
+ * poll() here, under the assumption that this is a very rare case.
+ * See https://lore.kernel.org/patchwork/patch/1047453/ to see
+ * discussion about such a case (ttys) where polling for POLLIN
+ * fails but POLLIN|POLLOUT works.
+ */
+ struct iocb *iocb = linuxaio_submits [submitted];
+ epoll_modify (EV_A_ iocb->aio_fildes, 0, anfds [iocb->aio_fildes].events);
+ iocb->aio_reqprio = -1; /* mark iocb as epoll */
+
+ res = 1; /* skip this iocb - another iocb, another chance */
+ }
+ else if (errno == EAGAIN)
+ {
+ /* This happens when the ring buffer is full, or some other shit we
+ * don't know and isn't documented. Most likely because we have too
+ * many requests and linux aio can't be assed to handle them.
+ * In this case, we try to allocate a larger ring buffer, freeing
+ * ours first. This might fail, in which case we have to fall back to 100%
+ * epoll.
+ * God, how I hate linux not getting its act together. Ever.
+ */
+ evsys_io_destroy (linuxaio_ctx);
+ linuxaio_submitcnt = 0;
+
+ /* rearm all fds with active iocbs */
+ {
+ int fd;
+ for (fd = 0; fd < linuxaio_iocbpmax; ++fd)
+ if (linuxaio_iocbps [fd]->io.aio_buf)
+ linuxaio_fd_rearm (EV_A_ fd);
+ }
+
+ ++linuxaio_iteration;
+ if (linuxaio_io_setup (EV_A) < 0)
+ {
+ /* TODO: rearm all and recreate epoll backend from scratch */
+ /* TODO: might be more prudent? */
+
+ /* to bad, we can't get a new aio context, go 100% epoll */
+ linuxaio_free_iocbp (EV_A);
+ ev_io_stop (EV_A_ &linuxaio_epoll_w);
+ ev_ref (EV_A);
+ linuxaio_ctx = 0;
+
+ backend = EVBACKEND_EPOLL;
+ backend_modify = epoll_modify;
+ backend_poll = epoll_poll;
+ }
+
+ timeout = EV_TS_CONST (0.);
+ /* it's easiest to handle this mess in another iteration */
+ return;
+ }
+ else if (errno == EBADF)
+ {
+ assert (("libev: event loop rejected bad fd", errno != EBADF));
+ fd_kill (EV_A_ linuxaio_submits [submitted]->aio_fildes);
+
+ res = 1; /* skip this iocb */
+ }
+ else if (errno == EINTR) /* not seen in reality, not documented */
+ res = 0; /* silently ignore and retry */
+ else
+ {
+ ev_syserr ("(libev) linuxaio io_submit");
+ res = 0;
+ }
+
+ submitted += res;
+ }
+
+ linuxaio_submitcnt = 0;
+
+ /* second phase: fetch and parse events */
+
+ linuxaio_get_events (EV_A_ timeout);
+}
+
+inline_size
+int
+linuxaio_init (EV_P_ int flags)
+{
+ /* would be great to have a nice test for IOCB_CMD_POLL instead */
+ /* also: test some semi-common fd types, such as files and ttys in recommended_backends */
+ /* 4.18 introduced IOCB_CMD_POLL, 4.19 made epoll work, and we need that */
+ if (ev_linux_version () < 0x041300)
+ return 0;
+
+ if (!epoll_init (EV_A_ 0))
+ return 0;
+
+ linuxaio_iteration = 0;
+
+ if (linuxaio_io_setup (EV_A) < 0)
+ {
+ epoll_destroy (EV_A);
+ return 0;
+ }
+
+ ev_io_init (&linuxaio_epoll_w, linuxaio_epoll_cb, backend_fd, EV_READ);
+ ev_set_priority (&linuxaio_epoll_w, EV_MAXPRI);
+ ev_io_start (EV_A_ &linuxaio_epoll_w);
+ ev_unref (EV_A); /* watcher should not keep loop alive */
+
+ backend_modify = linuxaio_modify;
+ backend_poll = linuxaio_poll;
+
+ linuxaio_iocbpmax = 0;
+ linuxaio_iocbps = 0;
+
+ linuxaio_submits = 0;
+ linuxaio_submitmax = 0;
+ linuxaio_submitcnt = 0;
+
+ return EVBACKEND_LINUXAIO;
+}
+
+inline_size
+void
+linuxaio_destroy (EV_P)
+{
+ epoll_destroy (EV_A);
+ linuxaio_free_iocbp (EV_A);
+ evsys_io_destroy (linuxaio_ctx); /* fails in child, aio context is destroyed */
+}
+
+ecb_cold
+static void
+linuxaio_fork (EV_P)
+{
+ linuxaio_submitcnt = 0; /* all pointers were invalidated */
+ linuxaio_free_iocbp (EV_A); /* this frees all iocbs, which is very heavy-handed */
+ evsys_io_destroy (linuxaio_ctx); /* fails in child, aio context is destroyed */
+
+ linuxaio_iteration = 0; /* we start over in the child */
+
+ while (linuxaio_io_setup (EV_A) < 0)
+ ev_syserr ("(libev) linuxaio io_setup");
+
+ /* forking epoll should also effectively unregister all fds from the backend */
+ epoll_fork (EV_A);
+ /* epoll_fork already did this. hopefully */
+ /*fd_rearm_all (EV_A);*/
+
+ ev_io_stop (EV_A_ &linuxaio_epoll_w);
+ ev_io_set (EV_A_ &linuxaio_epoll_w, backend_fd, EV_READ);
+ ev_io_start (EV_A_ &linuxaio_epoll_w);
+}
+
diff --git a/contrib/libev/ev_poll.c b/contrib/libev/ev_poll.c
new file mode 100644
index 0000000..e5508dd
--- /dev/null
+++ b/contrib/libev/ev_poll.c
@@ -0,0 +1,156 @@
+/*
+ * libev poll fd activity backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2016,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#include <poll.h>
+
+inline_size
+void
+array_needsize_pollidx (int *base, int offset, int count)
+{
+ /* using memset (.., -1, ...) is tempting, we we try
+ * to be ultraportable
+ */
+ base += offset;
+ while (count--)
+ *base++ = -1;
+}
+
+static void
+poll_modify (EV_P_ int fd, int oev, int nev)
+{
+ int idx;
+
+ if (oev == nev)
+ return;
+
+ array_needsize (int, pollidxs, pollidxmax, fd + 1, array_needsize_pollidx);
+
+ idx = pollidxs [fd];
+
+ if (idx < 0) /* need to allocate a new pollfd */
+ {
+ pollidxs [fd] = idx = pollcnt++;
+ array_needsize (struct pollfd, polls, pollmax, pollcnt, array_needsize_noinit);
+ polls [idx].fd = fd;
+ }
+
+ assert (polls [idx].fd == fd);
+
+ if (nev)
+ polls [idx].events =
+ (nev & EV_READ ? POLLIN : 0)
+ | (nev & EV_WRITE ? POLLOUT : 0);
+ else /* remove pollfd */
+ {
+ pollidxs [fd] = -1;
+
+ if (ecb_expect_true (idx < --pollcnt))
+ {
+ polls [idx] = polls [pollcnt];
+ pollidxs [polls [idx].fd] = idx;
+ }
+ }
+}
+
+static void
+poll_poll (EV_P_ ev_tstamp timeout)
+{
+ struct pollfd *p;
+ int res;
+
+ EV_RELEASE_CB;
+ res = poll (polls, pollcnt, EV_TS_TO_MSEC (timeout));
+ EV_ACQUIRE_CB;
+
+ if (ecb_expect_false (res < 0))
+ {
+ if (errno == EBADF)
+ fd_ebadf (EV_A);
+ else if (errno == ENOMEM && !syserr_cb)
+ fd_enomem (EV_A);
+ else if (errno != EINTR)
+ ev_syserr ("(libev) poll");
+ }
+ else
+ for (p = polls; res; ++p)
+ {
+ assert (("libev: poll returned illegal result, broken BSD kernel?", p < polls + pollcnt));
+
+ if (ecb_expect_false (p->revents)) /* this expect is debatable */
+ {
+ --res;
+
+ if (ecb_expect_false (p->revents & POLLNVAL))
+ {
+ assert (("libev: poll found invalid fd in poll set", 0));
+ fd_kill (EV_A_ p->fd);
+ }
+ else
+ fd_event (
+ EV_A_
+ p->fd,
+ (p->revents & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
+ | (p->revents & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
+ );
+ }
+ }
+}
+
+inline_size
+int
+poll_init (EV_P_ int flags)
+{
+ backend_mintime = EV_TS_CONST (1e-3);
+ backend_modify = poll_modify;
+ backend_poll = poll_poll;
+
+ pollidxs = 0; pollidxmax = 0;
+ polls = 0; pollmax = 0; pollcnt = 0;
+
+ return EVBACKEND_POLL;
+}
+
+inline_size
+void
+poll_destroy (EV_P)
+{
+ ev_free (pollidxs);
+ ev_free (polls);
+}
+
diff --git a/contrib/libev/ev_port.c b/contrib/libev/ev_port.c
new file mode 100644
index 0000000..f4cd9d9
--- /dev/null
+++ b/contrib/libev/ev_port.c
@@ -0,0 +1,192 @@
+/*
+ * libev solaris event port backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+/* useful reading:
+ *
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6268715 (random results)
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6455223 (just totally broken)
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6873782 (manpage ETIME)
+ * http://bugs.opensolaris.org/view_bug.do?bug_id=6874410 (implementation ETIME)
+ * http://www.mail-archive.com/networking-discuss@opensolaris.org/msg11898.html ETIME vs. nget
+ * http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/lib/libc/port/gen/event_port.c (libc)
+ * http://cvs.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/uts/common/fs/portfs/port.c#1325 (kernel)
+ */
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <poll.h>
+#include <port.h>
+#include <string.h>
+#include <errno.h>
+
+inline_speed
+void
+port_associate_and_check (EV_P_ int fd, int ev)
+{
+ if (0 >
+ port_associate (
+ backend_fd, PORT_SOURCE_FD, fd,
+ (ev & EV_READ ? POLLIN : 0)
+ | (ev & EV_WRITE ? POLLOUT : 0),
+ 0
+ )
+ )
+ {
+ if (errno == EBADFD)
+ {
+ assert (("libev: port_associate found invalid fd", errno != EBADFD));
+ fd_kill (EV_A_ fd);
+ }
+ else
+ ev_syserr ("(libev) port_associate");
+ }
+}
+
+static void
+port_modify (EV_P_ int fd, int oev, int nev)
+{
+ /* we need to reassociate no matter what, as closes are
+ * once more silently being discarded.
+ */
+ if (!nev)
+ {
+ if (oev)
+ port_dissociate (backend_fd, PORT_SOURCE_FD, fd);
+ }
+ else
+ port_associate_and_check (EV_A_ fd, nev);
+}
+
+static void
+port_poll (EV_P_ ev_tstamp timeout)
+{
+ int res, i;
+ struct timespec ts;
+ uint_t nget = 1;
+
+ /* we initialise this to something we will skip in the loop, as */
+ /* port_getn can return with nget unchanged, but no indication */
+ /* whether it was the original value or has been updated :/ */
+ port_events [0].portev_source = 0;
+
+ EV_RELEASE_CB;
+ EV_TS_SET (ts, timeout);
+ res = port_getn (backend_fd, port_events, port_eventmax, &nget, &ts);
+ EV_ACQUIRE_CB;
+
+ /* port_getn may or may not set nget on error */
+ /* so we rely on port_events [0].portev_source not being updated */
+ if (res == -1 && errno != ETIME && errno != EINTR)
+ ev_syserr ("(libev) port_getn (see http://bugs.opensolaris.org/view_bug.do?bug_id=6268715, try LIBEV_FLAGS=3 env variable)");
+
+ for (i = 0; i < nget; ++i)
+ {
+ if (port_events [i].portev_source == PORT_SOURCE_FD)
+ {
+ int fd = port_events [i].portev_object;
+
+ fd_event (
+ EV_A_
+ fd,
+ (port_events [i].portev_events & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
+ | (port_events [i].portev_events & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
+ );
+
+ fd_change (EV_A_ fd, EV__IOFDSET);
+ }
+ }
+
+ if (ecb_expect_false (nget == port_eventmax))
+ {
+ ev_free (port_events);
+ port_eventmax = array_nextsize (sizeof (port_event_t), port_eventmax, port_eventmax + 1);
+ port_events = (port_event_t *)ev_malloc (sizeof (port_event_t) * port_eventmax);
+ }
+}
+
+inline_size
+int
+port_init (EV_P_ int flags)
+{
+ /* Initialize the kernel queue */
+ if ((backend_fd = port_create ()) < 0)
+ return 0;
+
+ assert (("libev: PORT_SOURCE_FD must not be zero", PORT_SOURCE_FD));
+
+ fcntl (backend_fd, F_SETFD, FD_CLOEXEC); /* not sure if necessary, hopefully doesn't hurt */
+
+ /* if my reading of the opensolaris kernel sources are correct, then
+ * opensolaris does something very stupid: it checks if the time has already
+ * elapsed and doesn't round up if that is the case, otherwise it DOES round
+ * up. Since we can't know what the case is, we need to guess by using a
+ * "large enough" timeout. Normally, 1e-9 would be correct.
+ */
+ backend_mintime = EV_TS_CONST (1e-3); /* needed to compensate for port_getn returning early */
+ backend_modify = port_modify;
+ backend_poll = port_poll;
+
+ port_eventmax = 64; /* initial number of events receivable per poll */
+ port_events = (port_event_t *)ev_malloc (sizeof (port_event_t) * port_eventmax);
+
+ return EVBACKEND_PORT;
+}
+
+inline_size
+void
+port_destroy (EV_P)
+{
+ ev_free (port_events);
+}
+
+inline_size
+void
+port_fork (EV_P)
+{
+ close (backend_fd);
+
+ while ((backend_fd = port_create ()) < 0)
+ ev_syserr ("(libev) port");
+
+ fcntl (backend_fd, F_SETFD, FD_CLOEXEC);
+
+ /* re-register interest in fds */
+ fd_rearm_all (EV_A);
+}
+
diff --git a/contrib/libev/ev_select.c b/contrib/libev/ev_select.c
new file mode 100644
index 0000000..b862c81
--- /dev/null
+++ b/contrib/libev/ev_select.c
@@ -0,0 +1,316 @@
+/*
+ * libev select fd activity backend
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#ifndef _WIN32
+/* for unix systems */
+# include <inttypes.h>
+# ifndef __hpux
+/* for REAL unix systems */
+# include <sys/select.h>
+# endif
+#endif
+
+#ifndef EV_SELECT_USE_FD_SET
+# ifdef NFDBITS
+# define EV_SELECT_USE_FD_SET 0
+# else
+# define EV_SELECT_USE_FD_SET 1
+# endif
+#endif
+
+#if EV_SELECT_IS_WINSOCKET
+# undef EV_SELECT_USE_FD_SET
+# define EV_SELECT_USE_FD_SET 1
+# undef NFDBITS
+# define NFDBITS 0
+#endif
+
+#if !EV_SELECT_USE_FD_SET
+# define NFDBYTES (NFDBITS / 8)
+#endif
+
+#include <string.h>
+
+static void
+select_modify (EV_P_ int fd, int oev, int nev)
+{
+ if (oev == nev)
+ return;
+
+ {
+#if EV_SELECT_USE_FD_SET
+
+ #if EV_SELECT_IS_WINSOCKET
+ SOCKET handle = anfds [fd].handle;
+ #else
+ int handle = fd;
+ #endif
+
+ assert (("libev: fd >= FD_SETSIZE passed to fd_set-based select backend", fd < FD_SETSIZE));
+
+ /* FD_SET is broken on windows (it adds the fd to a set twice or more,
+ * which eventually leads to overflows). Need to call it only on changes.
+ */
+ #if EV_SELECT_IS_WINSOCKET
+ if ((oev ^ nev) & EV_READ)
+ #endif
+ if (nev & EV_READ)
+ FD_SET (handle, (fd_set *)vec_ri);
+ else
+ FD_CLR (handle, (fd_set *)vec_ri);
+
+ #if EV_SELECT_IS_WINSOCKET
+ if ((oev ^ nev) & EV_WRITE)
+ #endif
+ if (nev & EV_WRITE)
+ FD_SET (handle, (fd_set *)vec_wi);
+ else
+ FD_CLR (handle, (fd_set *)vec_wi);
+
+#else
+
+ int word = fd / NFDBITS;
+ fd_mask mask = 1UL << (fd % NFDBITS);
+
+ if (ecb_expect_false (vec_max <= word))
+ {
+ int new_max = word + 1;
+
+ vec_ri = ev_realloc (vec_ri, new_max * NFDBYTES);
+ vec_ro = ev_realloc (vec_ro, new_max * NFDBYTES); /* could free/malloc */
+ vec_wi = ev_realloc (vec_wi, new_max * NFDBYTES);
+ vec_wo = ev_realloc (vec_wo, new_max * NFDBYTES); /* could free/malloc */
+ #ifdef _WIN32
+ vec_eo = ev_realloc (vec_eo, new_max * NFDBYTES); /* could free/malloc */
+ #endif
+
+ for (; vec_max < new_max; ++vec_max)
+ ((fd_mask *)vec_ri) [vec_max] =
+ ((fd_mask *)vec_wi) [vec_max] = 0;
+ }
+
+ ((fd_mask *)vec_ri) [word] |= mask;
+ if (!(nev & EV_READ))
+ ((fd_mask *)vec_ri) [word] &= ~mask;
+
+ ((fd_mask *)vec_wi) [word] |= mask;
+ if (!(nev & EV_WRITE))
+ ((fd_mask *)vec_wi) [word] &= ~mask;
+#endif
+ }
+}
+
+static void
+select_poll (EV_P_ ev_tstamp timeout)
+{
+ struct timeval tv;
+ int res;
+ int fd_setsize;
+
+ EV_RELEASE_CB;
+ EV_TV_SET (tv, timeout);
+
+#if EV_SELECT_USE_FD_SET
+ fd_setsize = sizeof (fd_set);
+#else
+ fd_setsize = vec_max * NFDBYTES;
+#endif
+
+ memcpy (vec_ro, vec_ri, fd_setsize);
+ memcpy (vec_wo, vec_wi, fd_setsize);
+
+#ifdef _WIN32
+ /* pass in the write set as except set.
+ * the idea behind this is to work around a windows bug that causes
+ * errors to be reported as an exception and not by setting
+ * the writable bit. this is so uncontrollably lame.
+ */
+ memcpy (vec_eo, vec_wi, fd_setsize);
+ res = select (vec_max * NFDBITS, (fd_set *)vec_ro, (fd_set *)vec_wo, (fd_set *)vec_eo, &tv);
+#elif EV_SELECT_USE_FD_SET
+ fd_setsize = anfdmax < FD_SETSIZE ? anfdmax : FD_SETSIZE;
+ res = select (fd_setsize, (fd_set *)vec_ro, (fd_set *)vec_wo, 0, &tv);
+#else
+ res = select (vec_max * NFDBITS, (fd_set *)vec_ro, (fd_set *)vec_wo, 0, &tv);
+#endif
+ EV_ACQUIRE_CB;
+
+ if (ecb_expect_false (res < 0))
+ {
+ #if EV_SELECT_IS_WINSOCKET
+ errno = WSAGetLastError ();
+ #endif
+ #ifdef WSABASEERR
+ /* on windows, select returns incompatible error codes, fix this */
+ if (errno >= WSABASEERR && errno < WSABASEERR + 1000)
+ if (errno == WSAENOTSOCK)
+ errno = EBADF;
+ else
+ errno -= WSABASEERR;
+ #endif
+
+ #ifdef _WIN32
+ /* select on windows erroneously returns EINVAL when no fd sets have been
+ * provided (this is documented). what microsoft doesn't tell you that this bug
+ * exists even when the fd sets _are_ provided, so we have to check for this bug
+ * here and emulate by sleeping manually.
+ * we also get EINVAL when the timeout is invalid, but we ignore this case here
+ * and assume that EINVAL always means: you have to wait manually.
+ */
+ if (errno == EINVAL)
+ {
+ if (timeout)
+ {
+ unsigned long ms = EV_TS_TO_MSEC (timeout);
+ Sleep (ms ? ms : 1);
+ }
+
+ return;
+ }
+ #endif
+
+ if (errno == EBADF)
+ fd_ebadf (EV_A);
+ else if (errno == ENOMEM && !syserr_cb)
+ fd_enomem (EV_A);
+ else if (errno != EINTR)
+ ev_syserr ("(libev) select");
+
+ return;
+ }
+
+#if EV_SELECT_USE_FD_SET
+
+ {
+ int fd;
+
+ for (fd = 0; fd < anfdmax; ++fd)
+ if (anfds [fd].events)
+ {
+ int events = 0;
+ #if EV_SELECT_IS_WINSOCKET
+ SOCKET handle = anfds [fd].handle;
+ #else
+ int handle = fd;
+ #endif
+
+ if (FD_ISSET (handle, (fd_set *)vec_ro)) events |= EV_READ;
+ if (FD_ISSET (handle, (fd_set *)vec_wo)) events |= EV_WRITE;
+ #ifdef _WIN32
+ if (FD_ISSET (handle, (fd_set *)vec_eo)) events |= EV_WRITE;
+ #endif
+
+ if (ecb_expect_true (events))
+ fd_event (EV_A_ fd, events);
+ }
+ }
+
+#else
+
+ {
+ int word, bit;
+ for (word = vec_max; word--; )
+ {
+ fd_mask word_r = ((fd_mask *)vec_ro) [word];
+ fd_mask word_w = ((fd_mask *)vec_wo) [word];
+ #ifdef _WIN32
+ word_w |= ((fd_mask *)vec_eo) [word];
+ #endif
+
+ if (word_r || word_w)
+ for (bit = NFDBITS; bit--; )
+ {
+ fd_mask mask = 1UL << bit;
+ int events = 0;
+
+ events |= word_r & mask ? EV_READ : 0;
+ events |= word_w & mask ? EV_WRITE : 0;
+
+ if (ecb_expect_true (events))
+ fd_event (EV_A_ word * NFDBITS + bit, events);
+ }
+ }
+ }
+
+#endif
+}
+
+inline_size
+int
+select_init (EV_P_ int flags)
+{
+ backend_mintime = EV_TS_CONST (1e-6);
+ backend_modify = select_modify;
+ backend_poll = select_poll;
+
+#if EV_SELECT_USE_FD_SET
+ vec_ri = ev_malloc (sizeof (fd_set)); FD_ZERO ((fd_set *)vec_ri);
+ vec_ro = ev_malloc (sizeof (fd_set));
+ vec_wi = ev_malloc (sizeof (fd_set)); FD_ZERO ((fd_set *)vec_wi);
+ vec_wo = ev_malloc (sizeof (fd_set));
+ #ifdef _WIN32
+ vec_eo = ev_malloc (sizeof (fd_set));
+ #endif
+#else
+ vec_max = 0;
+ vec_ri = 0;
+ vec_ro = 0;
+ vec_wi = 0;
+ vec_wo = 0;
+ #ifdef _WIN32
+ vec_eo = 0;
+ #endif
+#endif
+
+ return EVBACKEND_SELECT;
+}
+
+inline_size
+void
+select_destroy (EV_P)
+{
+ ev_free (vec_ri);
+ ev_free (vec_ro);
+ ev_free (vec_wi);
+ ev_free (vec_wo);
+ #ifdef _WIN32
+ ev_free (vec_eo);
+ #endif
+}
+
diff --git a/contrib/libev/ev_vars.h b/contrib/libev/ev_vars.h
new file mode 100644
index 0000000..fb0c583
--- /dev/null
+++ b/contrib/libev/ev_vars.h
@@ -0,0 +1,249 @@
+/*
+ * loop member variable declarations
+ *
+ * Copyright (c) 2007,2008,2009,2010,2011,2012,2013,2019 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#define VARx(type,name) VAR(name, type name)
+
+VARx(ev_tstamp, now_floor) /* last time we refreshed rt_time */
+VARx(ev_tstamp, mn_now) /* monotonic clock "now" */
+VARx(ev_tstamp, rtmn_diff) /* difference realtime - monotonic time */
+
+/* for reverse feeding of events */
+VARx(W *, rfeeds)
+VARx(int, rfeedmax)
+VARx(int, rfeedcnt)
+
+VAR (pendings, ANPENDING *pendings [NUMPRI])
+VAR (pendingmax, int pendingmax [NUMPRI])
+VAR (pendingcnt, int pendingcnt [NUMPRI])
+VARx(int, pendingpri) /* highest priority currently pending */
+VARx(ev_prepare, pending_w) /* dummy pending watcher */
+
+VARx(ev_tstamp, io_blocktime)
+VARx(ev_tstamp, timeout_blocktime)
+
+VARx(int, backend)
+VARx(int, activecnt) /* total number of active events ("refcount") */
+VARx(EV_ATOMIC_T, loop_done) /* signal by ev_break */
+
+VARx(int, backend_fd)
+VARx(ev_tstamp, backend_mintime) /* assumed typical timer resolution */
+VAR (backend_modify, void (*backend_modify)(EV_P_ int fd, int oev, int nev))
+VAR (backend_poll , void (*backend_poll)(EV_P_ ev_tstamp timeout))
+
+VARx(ANFD *, anfds)
+VARx(int, anfdmax)
+
+VAR (evpipe, int evpipe [2])
+VARx(ev_io, pipe_w)
+VARx(EV_ATOMIC_T, pipe_write_wanted)
+VARx(EV_ATOMIC_T, pipe_write_skipped)
+
+#if !defined(_WIN32) || EV_GENWRAP
+VARx(pid_t, curpid)
+#endif
+
+VARx(char, postfork) /* true if we need to recreate kernel state after fork */
+
+#if EV_USE_SELECT || EV_GENWRAP
+VARx(void *, vec_ri)
+VARx(void *, vec_ro)
+VARx(void *, vec_wi)
+VARx(void *, vec_wo)
+#if defined(_WIN32) || EV_GENWRAP
+VARx(void *, vec_eo)
+#endif
+VARx(int, vec_max)
+#endif
+
+#if EV_USE_POLL || EV_GENWRAP
+VARx(struct pollfd *, polls)
+VARx(int, pollmax)
+VARx(int, pollcnt)
+VARx(int *, pollidxs) /* maps fds into structure indices */
+VARx(int, pollidxmax)
+#endif
+
+#if EV_USE_EPOLL || EV_GENWRAP
+VARx(struct epoll_event *, epoll_events)
+VARx(int, epoll_eventmax)
+VARx(int *, epoll_eperms)
+VARx(int, epoll_epermcnt)
+VARx(int, epoll_epermmax)
+#endif
+
+#if EV_USE_LINUXAIO || EV_GENWRAP
+VARx(aio_context_t, linuxaio_ctx)
+VARx(int, linuxaio_iteration)
+VARx(struct aniocb **, linuxaio_iocbps)
+VARx(int, linuxaio_iocbpmax)
+VARx(struct iocb **, linuxaio_submits)
+VARx(int, linuxaio_submitcnt)
+VARx(int, linuxaio_submitmax)
+VARx(ev_io, linuxaio_epoll_w)
+#endif
+
+#if EV_USE_IOURING || EV_GENWRAP
+VARx(int, iouring_fd)
+VARx(unsigned, iouring_to_submit);
+VARx(int, iouring_entries)
+VARx(int, iouring_max_entries)
+VARx(void *, iouring_sq_ring)
+VARx(void *, iouring_cq_ring)
+VARx(void *, iouring_sqes)
+VARx(uint32_t, iouring_sq_ring_size)
+VARx(uint32_t, iouring_cq_ring_size)
+VARx(uint32_t, iouring_sqes_size)
+VARx(uint32_t, iouring_sq_head)
+VARx(uint32_t, iouring_sq_tail)
+VARx(uint32_t, iouring_sq_ring_mask)
+VARx(uint32_t, iouring_sq_ring_entries)
+VARx(uint32_t, iouring_sq_flags)
+VARx(uint32_t, iouring_sq_dropped)
+VARx(uint32_t, iouring_sq_array)
+VARx(uint32_t, iouring_cq_head)
+VARx(uint32_t, iouring_cq_tail)
+VARx(uint32_t, iouring_cq_ring_mask)
+VARx(uint32_t, iouring_cq_ring_entries)
+VARx(uint32_t, iouring_cq_overflow)
+VARx(uint32_t, iouring_cq_cqes)
+VARx(ev_tstamp, iouring_tfd_to)
+VARx(int, iouring_tfd)
+VARx(ev_io, iouring_tfd_w)
+#endif
+
+#if EV_USE_KQUEUE || EV_GENWRAP
+VARx(pid_t, kqueue_fd_pid)
+VARx(struct kevent *, kqueue_changes)
+VARx(int, kqueue_changemax)
+VARx(int, kqueue_changecnt)
+VARx(struct kevent *, kqueue_events)
+VARx(int, kqueue_eventmax)
+#endif
+
+#if EV_USE_PORT || EV_GENWRAP
+VARx(struct port_event *, port_events)
+VARx(int, port_eventmax)
+#endif
+
+#if EV_USE_IOCP || EV_GENWRAP
+VARx(HANDLE, iocp)
+#endif
+
+VARx(int *, fdchanges)
+VARx(int, fdchangemax)
+VARx(int, fdchangecnt)
+
+VARx(ANHE *, timers)
+VARx(int, timermax)
+VARx(int, timercnt)
+
+#if EV_PERIODIC_ENABLE || EV_GENWRAP
+VARx(ANHE *, periodics)
+VARx(int, periodicmax)
+VARx(int, periodiccnt)
+#endif
+
+#if EV_IDLE_ENABLE || EV_GENWRAP
+VAR (idles, ev_idle **idles [NUMPRI])
+VAR (idlemax, int idlemax [NUMPRI])
+VAR (idlecnt, int idlecnt [NUMPRI])
+#endif
+VARx(int, idleall) /* total number */
+
+VARx(struct ev_prepare **, prepares)
+VARx(int, preparemax)
+VARx(int, preparecnt)
+
+VARx(struct ev_check **, checks)
+VARx(int, checkmax)
+VARx(int, checkcnt)
+
+#if EV_FORK_ENABLE || EV_GENWRAP
+VARx(struct ev_fork **, forks)
+VARx(int, forkmax)
+VARx(int, forkcnt)
+#endif
+
+#if EV_CLEANUP_ENABLE || EV_GENWRAP
+VARx(struct ev_cleanup **, cleanups)
+VARx(int, cleanupmax)
+VARx(int, cleanupcnt)
+#endif
+
+#if EV_ASYNC_ENABLE || EV_GENWRAP
+VARx(EV_ATOMIC_T, async_pending)
+VARx(struct ev_async **, asyncs)
+VARx(int, asyncmax)
+VARx(int, asynccnt)
+#endif
+
+#if EV_USE_INOTIFY || EV_GENWRAP
+VARx(int, fs_fd)
+VARx(ev_io, fs_w)
+VARx(char, fs_2625) /* whether we are running in linux 2.6.25 or newer */
+VAR (fs_hash, ANFS fs_hash [EV_INOTIFY_HASHSIZE])
+#endif
+
+VARx(EV_ATOMIC_T, sig_pending)
+#if EV_USE_SIGNALFD || EV_GENWRAP
+VARx(int, sigfd)
+VARx(ev_io, sigfd_w)
+VARx(sigset_t, sigfd_set)
+#endif
+
+#if EV_USE_TIMERFD || EV_GENWRAP
+VARx(int, timerfd) /* timerfd for time jump detection */
+VARx(ev_io, timerfd_w)
+#endif
+
+VARx(unsigned int, origflags) /* original loop flags */
+
+#if EV_FEATURE_API || EV_GENWRAP
+VARx(unsigned int, loop_count) /* total number of loop iterations/blocks */
+VARx(unsigned int, loop_depth) /* #ev_run enters - #ev_run leaves */
+
+VARx(void *, userdata)
+/* C++ doesn't support the ev_loop_callback typedef here. stinks. */
+VAR (release_cb, void (*release_cb)(EV_P) EV_NOEXCEPT)
+VAR (acquire_cb, void (*acquire_cb)(EV_P) EV_NOEXCEPT)
+VAR (invoke_cb , ev_loop_callback invoke_cb)
+#endif
+
+#undef VARx
+
diff --git a/contrib/libev/ev_win32.c b/contrib/libev/ev_win32.c
new file mode 100644
index 0000000..97344c3
--- /dev/null
+++ b/contrib/libev/ev_win32.c
@@ -0,0 +1,162 @@
+/*
+ * libev win32 compatibility cruft (_not_ a backend)
+ *
+ * Copyright (c) 2007,2008,2009 Marc Alexander Lehmann <libev@schmorp.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modifica-
+ * tion, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
+ * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
+ * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
+ * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License ("GPL") version 2 or any later version,
+ * in which case the provisions of the GPL are applicable instead of
+ * the above. If you wish to allow the use of your version of this file
+ * only under the terms of the GPL and not to allow others to use your
+ * version of this file under the BSD license, indicate your decision
+ * by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL. If you do not delete the
+ * provisions above, a recipient may use your version of this file under
+ * either the BSD or the GPL.
+ */
+
+#ifdef _WIN32
+
+/* note: the comment below could not be substantiated, but what would I care */
+/* MSDN says this is required to handle SIGFPE */
+/* my wild guess would be that using something floating-pointy is required */
+/* for the crt to do something about it */
+volatile double SIGFPE_REQ = 0.0f;
+
+static SOCKET
+ev_tcp_socket (void)
+{
+#if EV_USE_WSASOCKET
+ return WSASocket (AF_INET, SOCK_STREAM, 0, 0, 0, 0);
+#else
+ return socket (AF_INET, SOCK_STREAM, 0);
+#endif
+}
+
+/* oh, the humanity! */
+static int
+ev_pipe (int filedes [2])
+{
+ struct sockaddr_in addr = { 0 };
+ int addr_size = sizeof (addr);
+ struct sockaddr_in adr2;
+ int adr2_size = sizeof (adr2);
+ SOCKET listener;
+ SOCKET sock [2] = { -1, -1 };
+
+ if ((listener = ev_tcp_socket ()) == INVALID_SOCKET)
+ return -1;
+
+ addr.sin_family = AF_INET;
+ addr.sin_addr.s_addr = htonl (INADDR_LOOPBACK);
+ addr.sin_port = 0;
+
+ if (bind (listener, (struct sockaddr *)&addr, addr_size))
+ goto fail;
+
+ if (getsockname (listener, (struct sockaddr *)&addr, &addr_size))
+ goto fail;
+
+ if (listen (listener, 1))
+ goto fail;
+
+ if ((sock [0] = ev_tcp_socket ()) == INVALID_SOCKET)
+ goto fail;
+
+ if (connect (sock [0], (struct sockaddr *)&addr, addr_size))
+ goto fail;
+
+ /* TODO: returns INVALID_SOCKET on winsock accept, not < 0. fix it */
+ /* when convenient, probably by just removing error checking altogether? */
+ if ((sock [1] = accept (listener, 0, 0)) < 0)
+ goto fail;
+
+ /* windows vista returns fantasy port numbers for sockets:
+ * example for two interconnected tcp sockets:
+ *
+ * (Socket::unpack_sockaddr_in getsockname $sock0)[0] == 53364
+ * (Socket::unpack_sockaddr_in getpeername $sock0)[0] == 53363
+ * (Socket::unpack_sockaddr_in getsockname $sock1)[0] == 53363
+ * (Socket::unpack_sockaddr_in getpeername $sock1)[0] == 53365
+ *
+ * wow! tridirectional sockets!
+ *
+ * this way of checking ports seems to work:
+ */
+ if (getpeername (sock [0], (struct sockaddr *)&addr, &addr_size))
+ goto fail;
+
+ if (getsockname (sock [1], (struct sockaddr *)&adr2, &adr2_size))
+ goto fail;
+
+ errno = WSAEINVAL;
+ if (addr_size != adr2_size
+ || addr.sin_addr.s_addr != adr2.sin_addr.s_addr /* just to be sure, I mean, it's windows */
+ || addr.sin_port != adr2.sin_port)
+ goto fail;
+
+ closesocket (listener);
+
+#if EV_SELECT_IS_WINSOCKET
+ filedes [0] = EV_WIN32_HANDLE_TO_FD (sock [0]);
+ filedes [1] = EV_WIN32_HANDLE_TO_FD (sock [1]);
+#else
+ /* when select isn't winsocket, we also expect socket, connect, accept etc.
+ * to work on fds */
+ filedes [0] = sock [0];
+ filedes [1] = sock [1];
+#endif
+
+ return 0;
+
+fail:
+ closesocket (listener);
+
+ if (sock [0] != INVALID_SOCKET) closesocket (sock [0]);
+ if (sock [1] != INVALID_SOCKET) closesocket (sock [1]);
+
+ return -1;
+}
+
+#undef pipe
+#define pipe(filedes) ev_pipe (filedes)
+
+#define EV_HAVE_EV_TIME 1
+ev_tstamp
+ev_time (void)
+{
+ FILETIME ft;
+ ULARGE_INTEGER ui;
+
+ GetSystemTimeAsFileTime (&ft);
+ ui.u.LowPart = ft.dwLowDateTime;
+ ui.u.HighPart = ft.dwHighDateTime;
+
+ /* also, msvc cannot convert ulonglong to double... yes, it is that sucky */
+ return EV_TS_FROM_USEC (((LONGLONG)(ui.QuadPart - 116444736000000000) * 1e-1));
+}
+
+#endif
+
diff --git a/contrib/libev/ev_wrap.h b/contrib/libev/ev_wrap.h
new file mode 100644
index 0000000..45d793c
--- /dev/null
+++ b/contrib/libev/ev_wrap.h
@@ -0,0 +1,272 @@
+/* DO NOT EDIT, automatically generated by update_ev_wrap */
+#ifndef EV_WRAP_H
+#define EV_WRAP_H
+#define acquire_cb ((loop)->acquire_cb)
+#define activecnt ((loop)->activecnt)
+#define anfdmax ((loop)->anfdmax)
+#define anfds ((loop)->anfds)
+#define async_pending ((loop)->async_pending)
+#define asynccnt ((loop)->asynccnt)
+#define asyncmax ((loop)->asyncmax)
+#define asyncs ((loop)->asyncs)
+#define backend ((loop)->backend)
+#define backend_fd ((loop)->backend_fd)
+#define backend_mintime ((loop)->backend_mintime)
+#define backend_modify ((loop)->backend_modify)
+#define backend_poll ((loop)->backend_poll)
+#define checkcnt ((loop)->checkcnt)
+#define checkmax ((loop)->checkmax)
+#define checks ((loop)->checks)
+#define cleanupcnt ((loop)->cleanupcnt)
+#define cleanupmax ((loop)->cleanupmax)
+#define cleanups ((loop)->cleanups)
+#define curpid ((loop)->curpid)
+#define epoll_epermcnt ((loop)->epoll_epermcnt)
+#define epoll_epermmax ((loop)->epoll_epermmax)
+#define epoll_eperms ((loop)->epoll_eperms)
+#define epoll_eventmax ((loop)->epoll_eventmax)
+#define epoll_events ((loop)->epoll_events)
+#define evpipe ((loop)->evpipe)
+#define fdchangecnt ((loop)->fdchangecnt)
+#define fdchangemax ((loop)->fdchangemax)
+#define fdchanges ((loop)->fdchanges)
+#define forkcnt ((loop)->forkcnt)
+#define forkmax ((loop)->forkmax)
+#define forks ((loop)->forks)
+#define fs_2625 ((loop)->fs_2625)
+#define fs_fd ((loop)->fs_fd)
+#define fs_hash ((loop)->fs_hash)
+#define fs_w ((loop)->fs_w)
+#define idleall ((loop)->idleall)
+#define idlecnt ((loop)->idlecnt)
+#define idlemax ((loop)->idlemax)
+#define idles ((loop)->idles)
+#define invoke_cb ((loop)->invoke_cb)
+#define io_blocktime ((loop)->io_blocktime)
+#define iocp ((loop)->iocp)
+#define iouring_cq_cqes ((loop)->iouring_cq_cqes)
+#define iouring_cq_head ((loop)->iouring_cq_head)
+#define iouring_cq_overflow ((loop)->iouring_cq_overflow)
+#define iouring_cq_ring ((loop)->iouring_cq_ring)
+#define iouring_cq_ring_entries ((loop)->iouring_cq_ring_entries)
+#define iouring_cq_ring_mask ((loop)->iouring_cq_ring_mask)
+#define iouring_cq_ring_size ((loop)->iouring_cq_ring_size)
+#define iouring_cq_tail ((loop)->iouring_cq_tail)
+#define iouring_entries ((loop)->iouring_entries)
+#define iouring_fd ((loop)->iouring_fd)
+#define iouring_max_entries ((loop)->iouring_max_entries)
+#define iouring_sq_array ((loop)->iouring_sq_array)
+#define iouring_sq_dropped ((loop)->iouring_sq_dropped)
+#define iouring_sq_flags ((loop)->iouring_sq_flags)
+#define iouring_sq_head ((loop)->iouring_sq_head)
+#define iouring_sq_ring ((loop)->iouring_sq_ring)
+#define iouring_sq_ring_entries ((loop)->iouring_sq_ring_entries)
+#define iouring_sq_ring_mask ((loop)->iouring_sq_ring_mask)
+#define iouring_sq_ring_size ((loop)->iouring_sq_ring_size)
+#define iouring_sq_tail ((loop)->iouring_sq_tail)
+#define iouring_sqes ((loop)->iouring_sqes)
+#define iouring_sqes_size ((loop)->iouring_sqes_size)
+#define iouring_tfd ((loop)->iouring_tfd)
+#define iouring_tfd_to ((loop)->iouring_tfd_to)
+#define iouring_tfd_w ((loop)->iouring_tfd_w)
+#define iouring_to_submit ((loop)->iouring_to_submit)
+#define kqueue_changecnt ((loop)->kqueue_changecnt)
+#define kqueue_changemax ((loop)->kqueue_changemax)
+#define kqueue_changes ((loop)->kqueue_changes)
+#define kqueue_eventmax ((loop)->kqueue_eventmax)
+#define kqueue_events ((loop)->kqueue_events)
+#define kqueue_fd_pid ((loop)->kqueue_fd_pid)
+#define linuxaio_ctx ((loop)->linuxaio_ctx)
+#define linuxaio_epoll_w ((loop)->linuxaio_epoll_w)
+#define linuxaio_iocbpmax ((loop)->linuxaio_iocbpmax)
+#define linuxaio_iocbps ((loop)->linuxaio_iocbps)
+#define linuxaio_iteration ((loop)->linuxaio_iteration)
+#define linuxaio_submitcnt ((loop)->linuxaio_submitcnt)
+#define linuxaio_submitmax ((loop)->linuxaio_submitmax)
+#define linuxaio_submits ((loop)->linuxaio_submits)
+#define loop_count ((loop)->loop_count)
+#define loop_depth ((loop)->loop_depth)
+#define loop_done ((loop)->loop_done)
+#define mn_now ((loop)->mn_now)
+#define now_floor ((loop)->now_floor)
+#define origflags ((loop)->origflags)
+#define pending_w ((loop)->pending_w)
+#define pendingcnt ((loop)->pendingcnt)
+#define pendingmax ((loop)->pendingmax)
+#define pendingpri ((loop)->pendingpri)
+#define pendings ((loop)->pendings)
+#define periodiccnt ((loop)->periodiccnt)
+#define periodicmax ((loop)->periodicmax)
+#define periodics ((loop)->periodics)
+#define pipe_w ((loop)->pipe_w)
+#define pipe_write_skipped ((loop)->pipe_write_skipped)
+#define pipe_write_wanted ((loop)->pipe_write_wanted)
+#define pollcnt ((loop)->pollcnt)
+#define pollidxmax ((loop)->pollidxmax)
+#define pollidxs ((loop)->pollidxs)
+#define pollmax ((loop)->pollmax)
+#define polls ((loop)->polls)
+#define port_eventmax ((loop)->port_eventmax)
+#define port_events ((loop)->port_events)
+#define postfork ((loop)->postfork)
+#define preparecnt ((loop)->preparecnt)
+#define preparemax ((loop)->preparemax)
+#define prepares ((loop)->prepares)
+#define release_cb ((loop)->release_cb)
+#define rfeedcnt ((loop)->rfeedcnt)
+#define rfeedmax ((loop)->rfeedmax)
+#define rfeeds ((loop)->rfeeds)
+#define rtmn_diff ((loop)->rtmn_diff)
+#define sig_pending ((loop)->sig_pending)
+#define sigfd ((loop)->sigfd)
+#define sigfd_set ((loop)->sigfd_set)
+#define sigfd_w ((loop)->sigfd_w)
+#define timeout_blocktime ((loop)->timeout_blocktime)
+#define timercnt ((loop)->timercnt)
+#define timerfd ((loop)->timerfd)
+#define timerfd_w ((loop)->timerfd_w)
+#define timermax ((loop)->timermax)
+#define timers ((loop)->timers)
+#define userdata ((loop)->userdata)
+#define vec_eo ((loop)->vec_eo)
+#define vec_max ((loop)->vec_max)
+#define vec_ri ((loop)->vec_ri)
+#define vec_ro ((loop)->vec_ro)
+#define vec_wi ((loop)->vec_wi)
+#define vec_wo ((loop)->vec_wo)
+#else
+#undef EV_WRAP_H
+#undef acquire_cb
+#undef activecnt
+#undef anfdmax
+#undef anfds
+#undef async_pending
+#undef asynccnt
+#undef asyncmax
+#undef asyncs
+#undef backend
+#undef backend_fd
+#undef backend_mintime
+#undef backend_modify
+#undef backend_poll
+#undef checkcnt
+#undef checkmax
+#undef checks
+#undef cleanupcnt
+#undef cleanupmax
+#undef cleanups
+#undef curpid
+#undef epoll_epermcnt
+#undef epoll_epermmax
+#undef epoll_eperms
+#undef epoll_eventmax
+#undef epoll_events
+#undef evpipe
+#undef fdchangecnt
+#undef fdchangemax
+#undef fdchanges
+#undef forkcnt
+#undef forkmax
+#undef forks
+#undef fs_2625
+#undef fs_fd
+#undef fs_hash
+#undef fs_w
+#undef idleall
+#undef idlecnt
+#undef idlemax
+#undef idles
+#undef invoke_cb
+#undef io_blocktime
+#undef iocp
+#undef iouring_cq_cqes
+#undef iouring_cq_head
+#undef iouring_cq_overflow
+#undef iouring_cq_ring
+#undef iouring_cq_ring_entries
+#undef iouring_cq_ring_mask
+#undef iouring_cq_ring_size
+#undef iouring_cq_tail
+#undef iouring_entries
+#undef iouring_fd
+#undef iouring_max_entries
+#undef iouring_sq_array
+#undef iouring_sq_dropped
+#undef iouring_sq_flags
+#undef iouring_sq_head
+#undef iouring_sq_ring
+#undef iouring_sq_ring_entries
+#undef iouring_sq_ring_mask
+#undef iouring_sq_ring_size
+#undef iouring_sq_tail
+#undef iouring_sqes
+#undef iouring_sqes_size
+#undef iouring_tfd
+#undef iouring_tfd_to
+#undef iouring_tfd_w
+#undef iouring_to_submit
+#undef kqueue_changecnt
+#undef kqueue_changemax
+#undef kqueue_changes
+#undef kqueue_eventmax
+#undef kqueue_events
+#undef kqueue_fd_pid
+#undef linuxaio_ctx
+#undef linuxaio_epoll_w
+#undef linuxaio_iocbpmax
+#undef linuxaio_iocbps
+#undef linuxaio_iteration
+#undef linuxaio_submitcnt
+#undef linuxaio_submitmax
+#undef linuxaio_submits
+#undef loop_count
+#undef loop_depth
+#undef loop_done
+#undef mn_now
+#undef now_floor
+#undef origflags
+#undef pending_w
+#undef pendingcnt
+#undef pendingmax
+#undef pendingpri
+#undef pendings
+#undef periodiccnt
+#undef periodicmax
+#undef periodics
+#undef pipe_w
+#undef pipe_write_skipped
+#undef pipe_write_wanted
+#undef pollcnt
+#undef pollidxmax
+#undef pollidxs
+#undef pollmax
+#undef polls
+#undef port_eventmax
+#undef port_events
+#undef postfork
+#undef preparecnt
+#undef preparemax
+#undef prepares
+#undef release_cb
+#undef rfeedcnt
+#undef rfeedmax
+#undef rfeeds
+#undef rtmn_diff
+#undef sig_pending
+#undef sigfd
+#undef sigfd_set
+#undef sigfd_w
+#undef timeout_blocktime
+#undef timercnt
+#undef timerfd
+#undef timerfd_w
+#undef timermax
+#undef timers
+#undef userdata
+#undef vec_eo
+#undef vec_max
+#undef vec_ri
+#undef vec_ro
+#undef vec_wi
+#undef vec_wo
+#endif
diff --git a/contrib/libottery/CMakeLists.txt b/contrib/libottery/CMakeLists.txt
new file mode 100644
index 0000000..b8536f2
--- /dev/null
+++ b/contrib/libottery/CMakeLists.txt
@@ -0,0 +1,11 @@
+SET(OTTERYSRC chacha_merged.c
+ ottery.c
+ ottery_cpuinfo.c
+ ottery_entropy.c
+ ottery_global.c
+ chacha_cryptobox.c
+ aes_cryptobox.c)
+ADD_LIBRARY(ottery STATIC ${OTTERYSRC})
+
+SET(OTTERY_CFLAGS "-DBUILD_RSPAMD -DOTTERY_NO_PID_CHECK -DOTTERY_NO_INIT_CHECK -DOTTERY_NO_WIPE_STACK")
+set_target_properties(ottery PROPERTIES COMPILE_FLAGS "${OTTERY_CFLAGS}") \ No newline at end of file
diff --git a/contrib/libottery/aes_cryptobox.c b/contrib/libottery/aes_cryptobox.c
new file mode 100644
index 0000000..ea86dc7
--- /dev/null
+++ b/contrib/libottery/aes_cryptobox.c
@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2017, Vsevolod Stakhov
+ * Copyright (c) 2017, Frank Denis
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "ottery-internal.h"
+#include "cryptobox.h"
+
+#if defined(__x86_64__) && defined(RSPAMD_HAS_TARGET_ATTR)
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC push_options
+#pragma GCC target("aes")
+#endif
+#ifndef __SSE2__
+#define __SSE2__
+#endif
+#ifndef __SSE__
+#define __SSE__
+#endif
+#ifndef __AES__
+#define __AES__
+#endif
+#include <immintrin.h>
+#define ROUNDS 10
+
+typedef struct RSPAMD_ALIGNED(16) aes_rng_state {
+ __m128i round_keys[ROUNDS + 1];
+ __m128i counter;
+} aes_stream_state;
+
+
+#define STATE_LEN sizeof(aes_stream_state)
+#define STATE_BYTES 16
+
+#define OUTPUT_LEN 1024
+
+static void
+aes_key_expand (__m128i round_keys[ROUNDS + 1], __m128i t) __attribute__((target("aes")));
+
+static void
+aes_key_expand (__m128i round_keys[ROUNDS + 1], __m128i t)
+{
+ __m128i t1;
+
+#define DO_ROUND_KEY(ROUND, RC) \
+ do { \
+ t1 = _mm_aeskeygenassist_si128(t, (RC)); \
+ round_keys[ROUND] = t; \
+ t = _mm_xor_si128(t, _mm_slli_si128(t, 4)); \
+ t = _mm_xor_si128(t, _mm_slli_si128(t, 8)); \
+ t = _mm_xor_si128(t, _mm_shuffle_epi32(t1, 0xff)); \
+ } while (0)
+
+ DO_ROUND_KEY(0, 1);
+ DO_ROUND_KEY(1, 2);
+ DO_ROUND_KEY(2, 4);
+ DO_ROUND_KEY(3, 8);
+ DO_ROUND_KEY(4, 16);
+ DO_ROUND_KEY(5, 32);
+ DO_ROUND_KEY(6, 64);
+ DO_ROUND_KEY(7, 128);
+ DO_ROUND_KEY(8, 27);
+ DO_ROUND_KEY(9, 54);
+ round_keys[10] = t;
+}
+
+/*
+ * Computes one 128 bytes block and refresh keys
+ */
+static void
+aes_round(unsigned char *buf, struct aes_rng_state *st) __attribute__((target("aes")));
+static void
+aes_round(unsigned char *buf, struct aes_rng_state *st)
+{
+ const __m128i one = _mm_set_epi64x(0, 1);
+ __m128i *round_keys = st->round_keys;
+ __m128i c0, c1, c2, c3, c4, c5, c6, c7;
+ __m128i r0, r1, r2, r3, r4, r5, r6, r7;
+ __m128i s0, s1, s2, s3, s4, s5, s6, s7;
+ size_t i;
+
+#define COMPUTE_ROUNDS(N) \
+ do { \
+ r##N = _mm_aesenc_si128( _mm_xor_si128(c##N, round_keys[0]), round_keys[1]); \
+ r##N = _mm_aesenc_si128(_mm_aesenc_si128(r##N, round_keys[2]), round_keys[3]); \
+ r##N = _mm_aesenc_si128(_mm_aesenc_si128(r##N, round_keys[4]), round_keys[5]); \
+ s##N = r##N; \
+ r##N = _mm_aesenc_si128(_mm_aesenc_si128(r##N, round_keys[6]), round_keys[7]); \
+ r##N = _mm_aesenc_si128(_mm_aesenc_si128(r##N, round_keys[8]), round_keys[9]); \
+ r##N = _mm_xor_si128(s##N, _mm_aesenclast_si128(r##N, round_keys[10])); \
+ } while (0)
+
+ c0 = st->counter;
+
+ for (i = 0; i < OUTPUT_LEN / 128; i ++) {
+ c1 = _mm_add_epi64 (c0, one);
+ c2 = _mm_add_epi64 (c1, one);
+ c3 = _mm_add_epi64 (c2, one);
+ c4 = _mm_add_epi64 (c3, one);
+ c5 = _mm_add_epi64 (c4, one);
+ c6 = _mm_add_epi64 (c5, one);
+ c7 = _mm_add_epi64 (c6, one);
+ COMPUTE_ROUNDS(0);
+ COMPUTE_ROUNDS(1);
+ COMPUTE_ROUNDS(2);
+ COMPUTE_ROUNDS(3);
+ COMPUTE_ROUNDS(4);
+ COMPUTE_ROUNDS(5);
+ COMPUTE_ROUNDS(6);
+ COMPUTE_ROUNDS(7);
+ c0 = _mm_add_epi64 (c7, one);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 0), r0);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 16), r1);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 32), r2);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 48), r3);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 64), r4);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 80), r5);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 96), r6);
+ _mm_storeu_si128 ((__m128i *) (void *) (buf + 112), r7);
+ buf += 128;
+ }
+
+ st->counter = c0;
+ c0 = _mm_setzero_si128();
+ COMPUTE_ROUNDS(0);
+ aes_key_expand(round_keys, r0);
+}
+
+
+static void
+aes_cryptobox_state_setup (void *state_, const uint8_t *bytes)
+{
+ struct aes_rng_state *x = state_;
+
+ aes_key_expand (x->round_keys,
+ _mm_loadu_si128((const __m128i *) (const void *)bytes));
+}
+
+static void
+aes_cryptobox_generate (void *state_, uint8_t *output, uint32_t idx)
+{
+ struct aes_rng_state *x = state_;
+
+ aes_round(output, x);
+}
+
+#define PRF_AES(r) { \
+ "AES-" #r, \
+ "AES-" #r "-NOSIMD", \
+ "AES-" #r "-NOSIMD-DEFAULT", \
+ STATE_LEN, \
+ STATE_BYTES, \
+ OUTPUT_LEN, \
+ OTTERY_CPUCAP_AES, \
+ aes_cryptobox_state_setup, \
+ aes_cryptobox_generate \
+}
+
+const struct ottery_prf ottery_prf_aes_cryptobox_ = PRF_AES(128);
+#endif /* x86_64 */
diff --git a/contrib/libottery/chacha_cryptobox.c b/contrib/libottery/chacha_cryptobox.c
new file mode 100644
index 0000000..4e9cdae
--- /dev/null
+++ b/contrib/libottery/chacha_cryptobox.c
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2015, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "ottery-internal.h"
+#include "libcryptobox/chacha20/chacha.h"
+
+#define STATE_LEN (sizeof(chacha_state))
+#define STATE_BYTES 40
+
+#define IDX_STEP 16
+#define OUTPUT_LEN (IDX_STEP * 64)
+
+static void
+chacha20_cryptobox_state_setup (void *state_, const uint8_t *bytes)
+{
+ chacha_state *x = state_;
+ chacha_init (x, (chacha_key *)bytes, (chacha_iv *)(bytes + 32), 20);
+}
+
+static void
+chacha20_cryptobox_generate (void *state_, uint8_t *output, uint32_t idx)
+{
+ chacha_state *x = state_;
+
+ memset (output, 0, OUTPUT_LEN);
+ memcpy (output, &idx, sizeof (idx));
+ chacha_update (x, output, output, OUTPUT_LEN);
+}
+
+#define PRF_CHACHA(r) { \
+ "CHACHA" #r "-CRYPTOBOX", \
+ "CHACHA" #r "-CRYPTOBOX", \
+ "CHACHA" #r "-CRYPTOBOX", \
+ STATE_LEN, \
+ STATE_BYTES, \
+ OUTPUT_LEN, \
+ 0, \
+ chacha ## r ## _cryptobox_state_setup, \
+ chacha ## r ## _cryptobox_generate \
+}
+
+const struct ottery_prf ottery_prf_chacha20_cryptobox_ = PRF_CHACHA(20);
diff --git a/contrib/libottery/chacha_merged.c b/contrib/libottery/chacha_merged.c
new file mode 100644
index 0000000..c31a8bb
--- /dev/null
+++ b/contrib/libottery/chacha_merged.c
@@ -0,0 +1,218 @@
+/*
+ * This code is based on Dan Bernstein's pure C "merged" ChaCha
+ * implementation; details below.
+ *
+ * Note that I've ripped out all of the code that wasn't suitable for doing
+ * block-oriented operation, all (residual) support for 128-bit ChaCha keys,
+ * all support for counter values over 32 bits, the ability to xor the stream
+ * with a plaintext, and so on.
+ *
+ * Future versions of this might remove bigendian conversions too. DO NOT use
+ * this code for your stream cipher: go back to the original source. (I got
+ * this copy from SUPERCOP).
+ */
+
+/*
+chacha-merged.c version 20080118
+D. J. Bernstein
+Public domain.
+*/
+#include <string.h>
+#include "ottery-internal.h"
+#define u8 uint8_t
+#define u32 uint32_t
+#include "chacha_merged_ecrypt.h"
+
+#define ROTATE(v,c) (ROTL32(v,c))
+#define XOR(v,w) ((v) ^ (w))
+#define PLUS(v,w) (U32V((v) + (w)))
+#define PLUSONE(v) (PLUS((v),1))
+
+#define QUARTERROUND(a,b,c,d) \
+ a = PLUS(a,b); d = ROTATE(XOR(d,a),16); \
+ c = PLUS(c,d); b = ROTATE(XOR(b,c),12); \
+ a = PLUS(a,b); d = ROTATE(XOR(d,a), 8); \
+ c = PLUS(c,d); b = ROTATE(XOR(b,c), 7);
+
+static const char sigma[16] = "expand 32-byte k";
+
+static void ECRYPT_keysetup(ECRYPT_ctx *x,const u8 *k,u32 ivbits)
+{
+ const char *constants;
+ (void)ivbits;
+
+ x->input[4] = U8TO32_LITTLE(k + 0);
+ x->input[5] = U8TO32_LITTLE(k + 4);
+ x->input[6] = U8TO32_LITTLE(k + 8);
+ x->input[7] = U8TO32_LITTLE(k + 12);
+ k += 16;
+ constants = sigma;
+ x->input[8] = U8TO32_LITTLE(k + 0);
+ x->input[9] = U8TO32_LITTLE(k + 4);
+ x->input[10] = U8TO32_LITTLE(k + 8);
+ x->input[11] = U8TO32_LITTLE(k + 12);
+ x->input[0] = U8TO32_LITTLE(constants + 0);
+ x->input[1] = U8TO32_LITTLE(constants + 4);
+ x->input[2] = U8TO32_LITTLE(constants + 8);
+ x->input[3] = U8TO32_LITTLE(constants + 12);
+}
+
+static void ECRYPT_ivsetup(ECRYPT_ctx *x,const u8 *iv)
+{
+ x->input[12] = 0;
+ x->input[13] = 0;
+ x->input[14] = U8TO32_LITTLE(iv + 0);
+ x->input[15] = U8TO32_LITTLE(iv + 4);
+}
+
+#define IDX_STEP 16
+#define OUTPUT_LEN (IDX_STEP * 64)
+
+static inline void chacha_merged_getblocks(const int chacha_rounds, ECRYPT_ctx *x,u8 *c) __attribute__((always_inline));
+
+/** Generate OUTPUT_LEN bytes of output using the key, nonce, and counter in x,
+ * and store them in c.
+ */
+static void chacha_merged_getblocks(const int chacha_rounds, ECRYPT_ctx *x,u8 *c)
+{
+ u32 x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
+ u32 j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15;
+ unsigned i, block;
+
+ j0 = x->input[0];
+ j1 = x->input[1];
+ j2 = x->input[2];
+ j3 = x->input[3];
+ j4 = x->input[4];
+ j5 = x->input[5];
+ j6 = x->input[6];
+ j7 = x->input[7];
+ j8 = x->input[8];
+ j9 = x->input[9];
+ j10 = x->input[10];
+ j11 = x->input[11];
+ j12 = x->input[12];
+ j13 = x->input[13];
+ j14 = x->input[14];
+ j15 = x->input[15];
+
+ for (block = 0; block < IDX_STEP; ++block) {
+ x0 = j0;
+ x1 = j1;
+ x2 = j2;
+ x3 = j3;
+ x4 = j4;
+ x5 = j5;
+ x6 = j6;
+ x7 = j7;
+ x8 = j8;
+ x9 = j9;
+ x10 = j10;
+ x11 = j11;
+ x12 = j12;
+ x13 = j13;
+ x14 = j14;
+ x15 = j15;
+ for (i = chacha_rounds;i > 0;i -= 2) {
+ QUARTERROUND( x0, x4, x8,x12)
+ QUARTERROUND( x1, x5, x9,x13)
+ QUARTERROUND( x2, x6,x10,x14)
+ QUARTERROUND( x3, x7,x11,x15)
+ QUARTERROUND( x0, x5,x10,x15)
+ QUARTERROUND( x1, x6,x11,x12)
+ QUARTERROUND( x2, x7, x8,x13)
+ QUARTERROUND( x3, x4, x9,x14)
+ }
+ x0 = PLUS(x0,j0);
+ x1 = PLUS(x1,j1);
+ x2 = PLUS(x2,j2);
+ x3 = PLUS(x3,j3);
+ x4 = PLUS(x4,j4);
+ x5 = PLUS(x5,j5);
+ x6 = PLUS(x6,j6);
+ x7 = PLUS(x7,j7);
+ x8 = PLUS(x8,j8);
+ x9 = PLUS(x9,j9);
+ x10 = PLUS(x10,j10);
+ x11 = PLUS(x11,j11);
+ x12 = PLUS(x12,j12);
+ x13 = PLUS(x13,j13);
+ x14 = PLUS(x14,j14);
+ x15 = PLUS(x15,j15);
+
+ j12 = PLUSONE(j12);
+ /* Ottery: j13 can never need to be incremented. */
+
+ U32TO8_LITTLE(c + 0,x0);
+ U32TO8_LITTLE(c + 4,x1);
+ U32TO8_LITTLE(c + 8,x2);
+ U32TO8_LITTLE(c + 12,x3);
+ U32TO8_LITTLE(c + 16,x4);
+ U32TO8_LITTLE(c + 20,x5);
+ U32TO8_LITTLE(c + 24,x6);
+
+ U32TO8_LITTLE(c + 28,x7);
+ U32TO8_LITTLE(c + 32,x8);
+ U32TO8_LITTLE(c + 36,x9);
+ U32TO8_LITTLE(c + 40,x10);
+ U32TO8_LITTLE(c + 44,x11);
+ U32TO8_LITTLE(c + 48,x12);
+ U32TO8_LITTLE(c + 52,x13);
+ U32TO8_LITTLE(c + 56,x14);
+ U32TO8_LITTLE(c + 60,x15);
+
+ c += 64;
+ }
+}
+
+#define STATE_LEN (sizeof(ECRYPT_ctx))
+#define STATE_BYTES 40
+
+static void
+chacha_merged_state_setup(void *state_, const uint8_t *bytes)
+{
+ ECRYPT_ctx *x = state_;
+ ECRYPT_keysetup(x, bytes, 0);
+ ECRYPT_ivsetup(x, bytes+32);
+}
+
+static void
+chacha8_merged_generate(void *state_, uint8_t *output, uint32_t idx)
+{
+ ECRYPT_ctx *x = state_;
+ x->input[12] = idx * IDX_STEP;
+ chacha_merged_getblocks(8, x, output);
+}
+
+static void
+chacha12_merged_generate(void *state_, uint8_t *output, uint32_t idx)
+{
+ ECRYPT_ctx *x = state_;
+ x->input[12] = idx * IDX_STEP;
+ chacha_merged_getblocks(12, x, output);
+}
+
+static void
+chacha20_merged_generate(void *state_, uint8_t *output, uint32_t idx)
+{
+ ECRYPT_ctx *x = state_;
+ x->input[12] = idx * IDX_STEP;
+ chacha_merged_getblocks(20, x, output);
+}
+
+#define PRF_CHACHA(r) { \
+ "CHACHA" #r, \
+ "CHACHA" #r "-NOSIMD", \
+ "CHACHA" #r "-NOSIMD-DEFAULT", \
+ STATE_LEN, \
+ STATE_BYTES, \
+ OUTPUT_LEN, \
+ 0, \
+ chacha_merged_state_setup, \
+ chacha ## r ## _merged_generate \
+}
+
+const struct ottery_prf ottery_prf_chacha8_merged_ = PRF_CHACHA(8);
+const struct ottery_prf ottery_prf_chacha12_merged_ = PRF_CHACHA(12);
+const struct ottery_prf ottery_prf_chacha20_merged_ = PRF_CHACHA(20);
+
diff --git a/contrib/libottery/chacha_merged_ecrypt.h b/contrib/libottery/chacha_merged_ecrypt.h
new file mode 100644
index 0000000..5cc94a9
--- /dev/null
+++ b/contrib/libottery/chacha_merged_ecrypt.h
@@ -0,0 +1,133 @@
+/* Definitions for types and macros used in chacha_merged.c. Taken from
+ * supercop.
+ */
+
+#include <limits.h>
+
+typedef struct
+{
+ u32 input[16]; /* could be compressed */
+ /*
+ * [edit]
+ *
+ * Put here all state variable needed during the encryption process.
+ */
+} ECRYPT_ctx;
+#if (UCHAR_MAX / 0xFFFFU > 0xFFFFU)
+#ifndef I32T
+#define I32T char
+#define U32C(v) (v##U)
+#endif
+#endif
+
+#if (USHRT_MAX / 0xFFFFU > 0xFFFFU)
+#ifndef I32T
+#define I32T short
+#define U32C(v) (v##U)
+#endif
+#endif
+
+#if (UINT_MAX / 0xFFFFU > 0xFFFFU)
+#ifndef I32T
+#define I32T int
+#define U32C(v) (v##U)
+#endif
+#endif
+
+#if (ULONG_MAX / 0xFFFFUL > 0xFFFFUL)
+#ifndef I32T
+#define I32T long
+#define U32C(v) (v##UL)
+#endif
+#endif
+
+#define U8C(v) (v ## U)
+#define U32V(v) ((u32)(v) & U32C(0xFFFFFFFF))
+#define U8V(v) ((u8)(v) & U8C(0xFF))
+
+#if (defined(WIN32) && defined(_MSC_VER))
+#include <stdlib.h>
+#pragma intrinsic(_lrotl) /* compile rotations "inline" */
+#define ROTL32(v, n) _lrotl(v, n)
+#else
+#define ROTL32(v, n) \
+ (U32V((v) << (n)) | ((v) >> (32 - (n))))
+#endif
+
+
+
+#if ECRYPT_LITTLE_ENDIAN
+#define U32TO32_LITTLE(v) (v)
+#endif
+#ifdef ECRYPT_BIG_ENDIAN
+#define SWAP32(v) \
+ ((ROTL32(v, 8) & U32C(0x00FF00FF)) | \
+ (ROTL32(v, 24) & U32C(0xFF00FF00)))
+
+#define U32TO32_LITTLE(v) SWAP32(v)
+#endif
+
+#ifdef U32TO32_LITTLE
+#define U8TO32_LITTLE(p) U32TO32_LITTLE(((u32*)(p))[0])
+#define U32TO8_LITTLE(p, v) (((u32*)(p))[0] = U32TO32_LITTLE(v))
+#else
+#define U8TO32_LITTLE(p) \
+ (((u32)((p)[0]) ) | \
+ ((u32)((p)[1]) << 8) | \
+ ((u32)((p)[2]) << 16) | \
+ ((u32)((p)[3]) << 24))
+#define U32TO8_LITTLE(p, v) \
+ do { \
+ (p)[0] = U8V((v) ); \
+ (p)[1] = U8V((v) >> 8); \
+ (p)[2] = U8V((v) >> 16); \
+ (p)[3] = U8V((v) >> 24); \
+ } while (0)
+#endif
+
+/*
+ * The LITTLE endian machines:
+ */
+#if defined(__ultrix) /* Older MIPS */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(__alpha) /* Alpha */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(i386) /* x86 (gcc) */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(__i386) /* x86 (gcc) */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(__x86_64) /* x86_64 (gcc) */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(_M_IX86) /* x86 (MSC, Borland) */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(_MSC_VER) /* x86 (surely MSC) */
+#define ECRYPT_LITTLE_ENDIAN
+#elif defined(__INTEL_COMPILER) /* x86 (surely Intel compiler icl.exe) */
+#define ECRYPT_LITTLE_ENDIAN
+
+/*
+ * The BIG endian machines:
+ */
+#elif defined(__sparc) /* Newer Sparc's */
+#define ECRYPT_BIG_ENDIAN
+#elif defined(__powerpc__) /* PowerPC */
+#define ECRYPT_BIG_ENDIAN
+#elif defined(__ppc__) /* PowerPC */
+#define ECRYPT_BIG_ENDIAN
+#elif defined(__hppa) /* HP-PA */
+#define ECRYPT_BIG_ENDIAN
+
+/*
+ * Finally machines with UNKNOWN endianness:
+ */
+#elif defined (_AIX) /* RS6000 */
+#define ECRYPT_UNKNOWN
+#elif defined(__aux) /* 68K */
+#define ECRYPT_UNKNOWN
+#elif defined(__dgux) /* 88K (but P6 in latest boxes) */
+#define ECRYPT_UNKNOWN
+#elif defined(__sgi) /* Newer MIPS */
+#define ECRYPT_UNKNOWN
+#else /* Any other processor */
+#define ECRYPT_UNKNOWN
+#endif
diff --git a/contrib/libottery/ottery-internal.h b/contrib/libottery/ottery-internal.h
new file mode 100644
index 0000000..cc047f8
--- /dev/null
+++ b/contrib/libottery/ottery-internal.h
@@ -0,0 +1,335 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_INTERNAL_H_HEADER_INCLUDED_
+#define OTTERY_INTERNAL_H_HEADER_INCLUDED_
+#include <stdint.h>
+#include <sys/types.h>
+
+#ifdef BUILD_RSPAMD
+#include "config.h"
+#endif
+
+#include "ottery-threading.h"
+
+
+/**
+ * Version number for Libottery. The first three bytes are the major number,
+ * minor number, and patch-level respectively. The final byte is 0 for a
+ * released version, and nonzero otherwise.
+ */
+#define OTTERY_VERSION 0x00000001
+/**
+ * Human-readable string representing the Libottery version.
+ */
+#define OTTERY_VERSION_STRING "0.0.0"
+
+/** Largest possible state_bytes value. */
+#define MAX_STATE_BYTES 64
+/** Largest possible state_len value. */
+#define MAX_STATE_LEN 256
+/** Largest possible output_len value. */
+#define MAX_OUTPUT_LEN 1024
+
+/**
+ * @brief Flags for external entropy sources.
+ *
+ * @{ */
+/** An RNG that probably provides strong entropy. */
+#define OTTERY_ENTROPY_FL_STRONG 0x000001
+/** An RNG that runs very quickly. */
+#define OTTERY_ENTROPY_FL_FAST 0x000002
+/** @} */
+
+/**
+ * @brief Identifying external entropy domains.
+ */
+/** An RNG provided by the operating system. */
+#define OTTERY_ENTROPY_DOM_OS 0x000100
+/** An RNG provided by the CPU. */
+#define OTTERY_ENTROPY_DOM_CPU 0x000200
+/** An EGD-style entropy source */
+#define OTTERY_ENTROPY_DOM_EGD 0x000400
+/** @} */
+
+#define OTTERY_ENTROPY_FLAG_MASK 0x000000ff
+#define OTTERY_ENTROPY_DOM_MASK 0x0000ff00
+#define OTTERY_ENTROPY_ALL_SOURCES 0x0fff0000
+
+struct sockaddr;
+
+/** Configuration for the strong RNG the we use for entropy. */
+struct ottery_entropy_config {
+ /** The filename to use as /dev/urandom. Ignored if this
+ * is not a unix-like operating system. If this is NULL, we use
+ * the default value. */
+ const char *urandom_fname;
+ /** An fd to use to access /dev/urandom. -1 if not set. Overrides
+ * urandom_fname. */
+ int urandom_fd;
+ /** True if urandom_fd has been set. */
+ unsigned urandom_fd_is_set;
+ /** Socket for egd */
+ const struct sockaddr *egd_sockaddr;
+ /** Socklen for egd_sockaddr. */
+ int egd_socklen;
+ /** Bitmask of sources to disable. */
+ uint32_t disabled_sources;
+ /** Bitmask of sources to consider weak. */
+ uint32_t weak_sources;
+
+ /** If true, we don't enforce that urandom_fname must be a device file.
+ * This is for testing, and is not exposed to user code.
+ */
+ unsigned allow_nondev_urandom;
+};
+
+struct ottery_entropy_state {
+ /* Cached value for the inode of the urandom device. If this value changes,
+ * we assume that somebody messed with the fd by accident. */
+ uint64_t urandom_fd_inode;
+};
+
+/**
+ * Return the buffer size to allocate when getting at least n bytes from each
+ * entropy source. We might not actually need so many. */
+size_t ottery_get_entropy_bufsize_(size_t n);
+
+/**
+ * Interface to underlying strong RNGs. If this were fast, we'd just use it
+ * for everything, and forget about having a userspace PRNG. Unfortunately,
+ * it typically isn't.
+ *
+ * @param config A correctly set-up ottery_entropy_config.
+ * @param state A correctly set-up ottery_entropy_state.
+ * @param require_flags Only run entropy sources with *all* of these
+ * OTTERY_ENTROPY_* flags set. Set this to 0 to use all the sources
+ * that work.
+ * @param bytes A buffer to receive random bytes.
+ * @param n The number of bytes to try to get from each entropy source.
+ * @param bufsize The number of bytes available in the buffer; modified
+ * to hold the number of bytes actually written.
+ * @param flags_out Set to a bitwise OR of all of the OTTERY_ENTROPY_* flags
+ * for sources in the result.
+ * @return Zero on success, or an error code on failure. On failure, it is not
+ * safe to treat the contents of the buffer as random at all.
+ */
+int ottery_get_entropy_(const struct ottery_entropy_config *config,
+ struct ottery_entropy_state *state,
+ uint32_t require_flags,
+ uint8_t *bytes, size_t n, size_t *bufsize,
+ uint32_t *flags_out);
+
+/**
+ * Clear all bytes stored in a structure. Unlike memset, the compiler is not
+ * going to optimize this out of existence because the target is about to go
+ * out of scope.
+ *
+ * @param mem Pointer to the memory to erase.
+ * @param len The number of bytes to erase.
+ */
+void ottery_memclear_(void *mem, size_t len);
+
+/**
+ * Information on a single pseudorandom function that we can use to generate
+ * a bytestream which (we hope) an observer can't distinguish from random
+ * bytes.
+ *
+ * Broadly speaking, every ottery_prf has an underlying function from an
+ * (state_bytes)-byte state and a 4 byte counter to an output_len-byte
+ * output block.
+ **/
+struct ottery_prf {
+ /** The name of this algorithm. */
+ const char *name;
+ /** The name of the implementation of this algorithm*/
+ const char *impl;
+ /** The name of the flavor of the implementation of this algorithm*/
+ const char *flav;
+ /** The length of the object that's used to hold the state (keys, nonces,
+ * subkeys as needed, etc) for this PRF. This can be longer than
+ * state_bytes because of key expansion or structure padding. It must be
+ * no greater than MAX_STATE_LEN. */
+ unsigned state_len;
+ /** The number of bytes used to generate a state object. It must be no
+ * greater than MAX_STATE_BYTES. It must be no grater than output_len. */
+ unsigned state_bytes;
+ /** The number of bytes generated by a single call to the generate
+ * function. It must be no larger than MAX_OUTPUT_LEN.
+ */
+ unsigned output_len;
+ /** Bitmask of CPU flags required to run this PRF. */
+ uint32_t required_cpucap;
+ /** Pointer to a function to initialize a state structure for the PRF.
+ *
+ * @param state An object of size at least (state_len) that will
+ * hold the state and any derived values. It must be aligned to
+ * a 16-byte boundary.
+ * @param bytes An array of (state_bytes) random bytes.
+ */
+ void (*setup)(void *state, const uint8_t *bytes);
+ /** Pointer to a function that calculates the PRF.
+ *
+ * @param state A state object previously initialized by the setup
+ * function.
+ * @param output An array of (output_len) bytes in which to store the
+ * result of the function
+ * @param idx A counter value for the function.
+ */
+ void (*generate)(void *state, uint8_t *output, uint32_t idx);
+};
+
+/**
+ * Evaluate the condition 'x', while hinting to the compiler that it is
+ * likely to be false.
+ */
+#ifdef __GNUC__
+#define UNLIKELY(x) __builtin_expect((x), 0)
+#else
+#define UNLIKELY(x) (x)
+#endif
+
+#ifdef OTTERY_INTERNAL
+struct ottery_config {
+ /** The PRF that we should use. If NULL, we use the default. */
+ const struct ottery_prf *impl;
+
+ /** Configuration for how we will set up our entropy sources. */
+ struct ottery_entropy_config entropy_config;
+};
+
+#define ottery_state_nolock ottery_state
+
+struct RSPAMD_ALIGNED(16) ottery_state {
+ /**
+ * Holds up to prf.output_len bytes that have been generated by the
+ * pseudorandom function. */
+ uint8_t buffer[MAX_OUTPUT_LEN] RSPAMD_ALIGNED(16);
+ /**
+ * Holds the state information (typically nonces and keys) used by the
+ * pseudorandom function. */
+
+ uint8_t state[MAX_STATE_LEN] RSPAMD_ALIGNED(16);
+ /**
+ * Parameters and function pointers for the cryptographic pseudorandom
+ * function that we're using. */
+ struct ottery_prf prf;
+ /**
+ * Index of the *next* block counter to use when generating random bytes
+ * with prf. When this equals or exceeds prf.stir_after, we should stir
+ * the PRNG. */
+ uint32_t block_counter;
+ /**
+ * Magic number; used to tell whether this state is initialized.
+ */
+ uint32_t magic;
+ /**
+ * Index of the next byte in (buffer) to yield to the user.
+ *
+ * Invariant: this is less than prf.output_len. */
+ uint16_t pos;
+ /**
+ * The pid of the process in which this PRF was most recently seeded
+ * from the OS. We use this to avoid use-after-fork problems; see
+ * ottery_st_rand_lock_and_check(). */
+ pid_t pid;
+ /**
+ * Combined flags_out results from all calls to the entropy source that
+ * have influenced our current state.
+ */
+ uint32_t entropy_src_flags;
+ /**
+ * flags_out result from our last call to the entropy source.
+ */
+ uint32_t last_entropy_flags;
+ /**
+ * Configuration for the entropy source.
+ */
+ struct ottery_entropy_config entropy_config;
+ /** State for the entropy source.
+ */
+ struct ottery_entropy_state entropy_state;
+ /**
+ * @brief Locks for this structure.
+ *
+ * This lock will not necessarily be recursive. It's probably a
+ * spinlock.
+ *
+ * @{
+ */
+DECL_LOCK(mutex)
+ /**@}*/
+};
+#endif
+
+struct ottery_config;
+/**
+ * For testing: manually supply a PRF.
+ */
+void ottery_config_set_manual_prf_(struct ottery_config *cfg,
+ const struct ottery_prf *prf);
+
+
+/** Called when a fatal error has occurred: Die horribly, or invoke
+ * ottery_fatal_handler. */
+void ottery_fatal_error_(int error);
+
+#define OTTERY_CPUCAP_SIMD (1<<0)
+#define OTTERY_CPUCAP_SSSE3 (1<<1)
+#define OTTERY_CPUCAP_AES (1<<2)
+#define OTTERY_CPUCAP_RAND (1<<3)
+
+/** Return a mask of OTTERY_CPUCAP_* for what the CPU will offer us. */
+uint32_t ottery_get_cpu_capabilities_(void);
+
+/** Tell ottery_get_cpu_capabilities to never report certain capabilities as
+ * present. */
+void ottery_disable_cpu_capabilities_(uint32_t disable);
+
+/**
+ * @brief pure-C portable ChaCha implementations.
+ *
+ * @{
+ */
+extern const struct ottery_prf ottery_prf_chacha8_merged_;
+extern const struct ottery_prf ottery_prf_chacha12_merged_;
+extern const struct ottery_prf ottery_prf_chacha20_merged_;
+
+#ifdef BUILD_RSPAMD
+#ifdef __x86_64__
+extern const struct ottery_prf ottery_prf_aes_cryptobox_;
+#endif
+extern const struct ottery_prf ottery_prf_chacha20_cryptobox_;
+#endif
+/**@}*/
+
+/**
+ * @brief SIMD-basd ChaCha implementations.
+ *
+ * These are much, much faster.
+ *
+ * @{ */
+#ifdef HAVE_SIMD_CHACHA
+extern const struct ottery_prf ottery_prf_chacha8_krovetz_1_;
+extern const struct ottery_prf ottery_prf_chacha12_krovetz_1_;
+extern const struct ottery_prf ottery_prf_chacha20_krovetz_1_;
+#endif
+
+#ifdef HAVE_SIMD_CHACHA_2
+extern const struct ottery_prf ottery_prf_chacha8_krovetz_2_;
+extern const struct ottery_prf ottery_prf_chacha12_krovetz_2_;
+extern const struct ottery_prf ottery_prf_chacha20_krovetz_2_;
+#endif
+/** @} */
+
+#endif
diff --git a/contrib/libottery/ottery-threading.h b/contrib/libottery/ottery-threading.h
new file mode 100644
index 0000000..c5427ad
--- /dev/null
+++ b/contrib/libottery/ottery-threading.h
@@ -0,0 +1,96 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_LOCKING_H_HEADER_INCLUDED_
+#define OTTERY_LOCKING_H_HEADER_INCLUDED_
+
+/* We don't need locks when building rspamd */
+#ifdef BUILD_RSPAMD
+#define OTTERY_NO_LOCKS
+#endif
+
+/* Locks */
+#ifdef OTTERY_NO_LOCKS
+/* Nothing here. */
+#elif defined(__APPLE__) && !defined(OTTERY_NO_SPINLOCKS)
+#define OTTERY_OSATOMIC_LOCKS
+#include <libkern/OSAtomic.h>
+#elif defined(_WIN32)
+#define OTTERY_CRITICAL_SECTION
+#include <windows.h>
+#elif defined(HAVE_PTHREAD)
+#define OTTERY_PTHREADS
+#include <pthread.h>
+#else
+#define OTTERY_NO_LOCKS
+#endif
+
+#ifdef OTTERY_NO_LOCKS
+#define DECL_LOCK(mutex)
+#elif defined(OTTERY_OSATOMIC_LOCKS)
+#define DECL_LOCK(mutex) OSSpinLock mutex;
+#elif defined(OTTERY_CRITICAL_SECTION)
+#define DECL_LOCK(mutex) CRITICAL_SECTION mutex;
+#elif defined(OTTERY_PTHREADS)
+#define DECL_LOCK(mutex) pthread_mutex_t mutex;
+#endif
+
+#if defined(OTTERY_PTHREADS)
+#define INIT_LOCK(mutex) \
+ (pthread_mutex_init((mutex), NULL) != 0)
+/** Acquire the lock for the state "st". */
+#define ACQUIRE_LOCK(mutex) do { \
+ pthread_mutex_lock(mutex); \
+ } while (0)
+/** Release the lock for the state "st". */
+#define RELEASE_LOCK(mutex) do { \
+ pthread_mutex_unlock(mutex); \
+ } while (0)
+#define DESTROY_LOCK(mutex) do { \
+ pthread_mutex_destroy(mutex); \
+ } while (0)
+
+#elif defined(OTTERY_CRITICAL_SECTION)
+#define INIT_LOCK(mutex) \
+ (InitializeCriticalSectionAndSpinCount((mutex), 3000) == 0)
+#define ACQUIRE_LOCK(mutex) do { \
+ EnterCriticalSection(mutex); \
+ } while (0)
+#define RELEASE_LOCK(mutex) do { \
+ LeaveCriticalSection(mutex); \
+ } while (0)
+#define DESTROY_LOCK(mutex) do { \
+ DeleteCriticalSection(mutex); \
+ } while (0)
+
+#elif defined(OTTERY_OSATOMIC_LOCKS)
+#define INIT_LOCK(mutex) \
+ ((*(mutex) = 0), 0)
+#define ACQUIRE_LOCK(mutex) do { \
+ OSSpinLockLock(mutex); \
+ } while (0)
+#define RELEASE_LOCK(mutex) do { \
+ OSSpinLockUnlock(mutex); \
+ } while (0)
+#define DESTROY_LOCK(mutex) ((void)0)
+
+#elif defined(OTTERY_NO_LOCKS)
+#define INIT_LOCK(mutex) (0)
+#define DESTROY_LOCK(mutex) ((void)0)
+#define ACQUIRE_LOCK(mutex) ((void)0)
+#define RELEASE_LOCK(mutex) ((void)0)
+#else
+#error How do I lock?
+#endif
+
+#endif
diff --git a/contrib/libottery/ottery.c b/contrib/libottery/ottery.c
new file mode 100644
index 0000000..c58a901
--- /dev/null
+++ b/contrib/libottery/ottery.c
@@ -0,0 +1,847 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#define OTTERY_INTERNAL
+#include "ottery-internal.h"
+#include "ottery.h"
+#include "ottery_st.h"
+#include "ottery_nolock.h"
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <limits.h>
+
+#include <stdio.h>
+
+/* I've added a few assertions to sanity-check for debugging, but they should
+ * never ever ever trigger. It's fine to build this code with NDEBUG. */
+#include <assert.h>
+
+#ifdef _WIN32
+/* On Windows, there is no fork(), so we don't need to worry about forking. */
+#define OTTERY_NO_PID_CHECK
+#endif
+
+#ifdef BUILD_RSPAMD
+#include "cryptobox.h"
+#endif
+
+/** Magic number for deciding whether an ottery_state is initialized. */
+#define MAGIC_BASIS 0x11b07734
+
+/** Macro: yield the correct magic number for an ottery_state, based on
+ * its position in RAM. */
+#define MAGIC(ptr) (((uint32_t)(uintptr_t)(ptr)) ^ MAGIC_BASIS)
+
+static inline int ottery_st_rand_lock_and_check(struct ottery_state *st)
+__attribute__((always_inline));
+static int ottery_st_reseed(struct ottery_state *state);
+static int ottery_st_add_seed_impl(struct ottery_state *st, const uint8_t *seed, size_t n, int locking, int check_magic);
+
+#ifndef OTTERY_NO_WIPE_STACK
+static void ottery_wipe_stack_(void) __attribute__((noinline));
+#endif
+
+#define LOCK(st) ACQUIRE_LOCK(&(st)->mutex)
+#define UNLOCK(st) RELEASE_LOCK(&(st)->mutex)
+
+size_t
+ottery_get_sizeof_config(void)
+{
+ return sizeof(struct ottery_config);
+}
+
+size_t
+ottery_get_sizeof_state(void)
+{
+ return sizeof(struct ottery_state);
+}
+
+size_t
+ottery_get_sizeof_state_nolock(void)
+{
+ return sizeof(struct ottery_state_nolock);
+}
+
+const char *
+ottery_get_version_string(void)
+{
+ return OTTERY_VERSION_STRING;
+}
+
+uint32_t
+ottery_get_version(void)
+{
+ return OTTERY_VERSION;
+}
+
+uint32_t
+ottery_get_build_flags(void)
+{
+ uint32_t result = 0;
+#ifdef OTTERY_NO_PID_CHECK
+ result |= OTTERY_BLDFLG_NO_PID_CHECK;
+#endif
+#ifdef OTTERY_NO_INIT_CHECK
+ result |= OTTERY_BLDFLG_NO_INIT_CHECK;
+#endif
+#ifdef OTTERY_NO_LOCKS
+ result |= OTTERY_BLDFLG_NO_LOCKING;
+#endif
+#ifdef OTTERY_NO_CLEAR_AFTER_YIELD
+ result |= OTTERY_BLDFLG_NO_CLEAR_AFTER_YIELD;
+#endif
+#ifdef OTTERY_NO_WIPE_STACK
+ result |= OTTERY_BLDFLG_NO_WIPE_STACK;
+#endif
+#ifdef OTTERY_NO_SIMD
+ result |= OTTERY_BLDFLG_NO_SIMD;
+#endif
+ return result;
+}
+
+#ifndef OTTERY_NO_CLEAR_AFTER_YIELD
+/** Used to zero out the contents of our buffer after we've just given a few
+ * to the user. */
+#define CLEARBUF(ptr,n) do { memset((ptr), 0, (n)); } while (0)
+#else
+#define CLEARBUF(ptr,n) ((void)0)
+#endif
+
+/**
+ * Volatile pointer to memset: we use this to keep the compiler from
+ * eliminating our call to memset. (Don't make this static.)
+ */
+void * (*volatile ottery_memset_volatile_)(void *, int, size_t) = memset;
+
+
+void
+ottery_memclear_(void *mem, size_t len)
+{
+ /* NOTE: whenever we change this, change test/test_memclear.c accordingly */
+ ottery_memset_volatile_(mem, 0, len);
+}
+
+#ifndef OTTERY_NO_WIPE_STACK
+
+/* Chosen more or less arbitrarily */
+#define WIPE_STACK_LEN 512
+
+/**
+ * Try to clear memory on the stack to clean up after our PRF. This can't
+ * easily be done in standard C, so we're doing an ugly hack in hopes that it
+ * actually helps.
+ *
+ * This should never be necessary in a correct program, but if your program is
+ * doing something stupid like leaking uninitialized stack, it might keep an
+ * attacker from exploiting that.
+ **/
+static void
+ottery_wipe_stack_(void)
+{
+ char buf[WIPE_STACK_LEN];
+ ottery_memset_volatile_(buf, 0, sizeof(buf));
+}
+#else
+#define ottery_wipe_stack_() ((void)0)
+#endif
+
+int
+ottery_config_init(struct ottery_config *cfg)
+{
+ cfg->impl = NULL;
+ cfg->entropy_config.urandom_fname = NULL;
+ cfg->entropy_config.urandom_fd = -1;
+ cfg->entropy_config.urandom_fd_is_set = 0;
+ cfg->entropy_config.disabled_sources = 0;
+ cfg->entropy_config.weak_sources = 0;
+ cfg->entropy_config.egd_sockaddr = NULL;
+ cfg->entropy_config.egd_socklen = 0;
+ cfg->entropy_config.allow_nondev_urandom = 0;
+ return 0;
+}
+
+static const struct ottery_prf *
+ottery_get_impl(const char *impl)
+{
+ int i;
+ const struct ottery_prf *ALL_PRFS[] = {
+#ifdef HAVE_SIMD_CHACHA_2
+ &ottery_prf_chacha20_krovetz_2_,
+ &ottery_prf_chacha12_krovetz_2_,
+ &ottery_prf_chacha8_krovetz_2_,
+#endif
+#ifdef HAVE_SIMD_CHACHA
+ &ottery_prf_chacha20_krovetz_1_,
+ &ottery_prf_chacha12_krovetz_1_,
+ &ottery_prf_chacha8_krovetz_1_,
+#endif
+
+#ifdef BUILD_RSPAMD
+#if defined(__x86_64__) && defined(RSPAMD_HAS_TARGET_ATTR)
+ &ottery_prf_aes_cryptobox_,
+#endif
+ &ottery_prf_chacha20_cryptobox_,
+#endif
+ &ottery_prf_chacha20_merged_,
+ &ottery_prf_chacha12_merged_,
+ &ottery_prf_chacha8_merged_,
+
+ NULL,
+ };
+ const uint32_t cap = ottery_get_cpu_capabilities_();
+
+ for (i = 0; ALL_PRFS[i]; ++i) {
+ const struct ottery_prf *prf = ALL_PRFS[i];
+ if ((prf->required_cpucap & cap) != prf->required_cpucap)
+ continue;
+ if (impl == NULL)
+ return prf;
+ if (!strcmp(impl, prf->name))
+ return prf;
+ if (!strcmp(impl, prf->impl))
+ return prf;
+ if (!strcmp(impl, prf->flav))
+ return prf;
+ }
+ return NULL;
+}
+
+int
+ottery_config_force_implementation(struct ottery_config *cfg,
+ const char *impl)
+{
+ const struct ottery_prf *prf = ottery_get_impl(impl);
+ if (prf) {
+ cfg->impl = prf;
+ return 0;
+ }
+ return OTTERY_ERR_INVALID_ARGUMENT;
+}
+
+void
+ottery_config_set_manual_prf_(struct ottery_config *cfg,
+ const struct ottery_prf *prf)
+{
+ cfg->impl = prf;
+}
+
+void
+ottery_config_set_urandom_device(struct ottery_config *cfg,
+ const char *fname)
+{
+ cfg->entropy_config.urandom_fname = fname;
+}
+
+void
+ottery_config_set_urandom_fd(struct ottery_config *cfg,
+ int fd)
+{
+ cfg->entropy_config.urandom_fd = fd;
+ cfg->entropy_config.urandom_fd_is_set = (fd >= 0);
+}
+
+void
+ottery_config_set_egd_socket(struct ottery_config *cfg,
+ const struct sockaddr *addr,
+ int len)
+{
+ cfg->entropy_config.egd_sockaddr = addr;
+ cfg->entropy_config.egd_socklen = len;
+}
+
+void
+ottery_config_disable_entropy_sources(struct ottery_config *cfg,
+ uint32_t disabled_sources)
+{
+ cfg->entropy_config.disabled_sources =
+ (disabled_sources & OTTERY_ENTROPY_ALL_SOURCES);
+}
+
+void
+ottery_config_mark_entropy_sources_weak(struct ottery_config *cfg,
+ uint32_t disabled_sources)
+{
+ cfg->entropy_config.weak_sources =
+ (disabled_sources & OTTERY_ENTROPY_ALL_SOURCES);
+}
+
+/**
+ * As ottery_st_nextblock_nolock(), but fill the entire block with
+ * entropy, and don't try to rekey the state.
+ */
+static void
+ottery_st_nextblock_nolock_norekey(struct ottery_state *st)
+{
+ st->prf.generate(st->state, st->buffer, st->block_counter);
+ ottery_wipe_stack_();
+ ++st->block_counter;
+}
+
+/**
+ * Generate (st->output_len) bytes of pseudorandom data from the PRF into
+ * (st->buffer). Use the first st->prf.state_bytes of those bytes to replace
+ * the PRF state and advance (st->pos) to point after them.
+ *
+ * This function does not acquire the lock on the state; use it within
+ * another function that does.
+ *
+ * @param st The state to use when generating the block.
+ */
+static void
+ottery_st_nextblock_nolock(struct ottery_state_nolock *st)
+{
+ ottery_st_nextblock_nolock_norekey(st);
+ st->prf.setup(st->state, st->buffer);
+ CLEARBUF(st->buffer, st->prf.state_bytes);
+ st->block_counter = 0;
+ st->pos = st->prf.state_bytes;
+}
+
+/**
+ * Initialize or reinitialize a PRNG state.
+ *
+ * @param st The state to initialize or reinitialize.
+ * @param prf The configuration to use. (Ignored for reinit)
+ * @return An OTTERY_ERR_* value (zero on success, nonzero on failure).
+ */
+static int
+ottery_st_initialize(struct ottery_state *st,
+ const struct ottery_config *config,
+ int locked)
+{
+ const struct ottery_prf *prf = NULL;
+ struct ottery_config cfg_tmp;
+ int err;
+ /* We really need our state to be aligned. If it isn't, let's give an
+ * error now, and not a crash when the SIMD instructions start to fail.
+ */
+ if (((uintptr_t)st) & 0xf)
+ return OTTERY_ERR_STATE_ALIGNMENT;
+
+ if (!config) {
+ ottery_config_init(&cfg_tmp);
+ config = &cfg_tmp;
+ }
+
+ prf = config->impl;
+
+ if (!prf)
+ prf = ottery_get_impl(NULL);
+
+ memset(st, 0, sizeof(*st));
+
+ if (locked) {
+ /* Now set up the spinlock or mutex or hybrid thing. */
+ if (INIT_LOCK(&st->mutex))
+ return OTTERY_ERR_LOCK_INIT;
+ }
+
+ /* Check invariants for PRF, in case we wrote some bad code. */
+ if ((prf->state_len > MAX_STATE_LEN) ||
+ (prf->state_bytes > MAX_STATE_BYTES) ||
+ (prf->state_bytes > prf->output_len) ||
+ (prf->output_len > MAX_OUTPUT_LEN))
+ return OTTERY_ERR_INTERNAL;
+
+ /* Check whether some of our structure size assumptions are right. */
+ if ((sizeof(struct ottery_state) > OTTERY_STATE_DUMMY_SIZE_) ||
+ (sizeof(struct ottery_config) > OTTERY_CONFIG_DUMMY_SIZE_))
+ return OTTERY_ERR_INTERNAL;
+
+ memcpy(&st->entropy_config, &config->entropy_config,
+ sizeof(struct ottery_entropy_config));
+
+ /* Copy the PRF into place. */
+ memcpy(&st->prf, prf, sizeof(*prf));
+
+ if ((err = ottery_st_reseed(st)))
+ return err;
+
+ /* Set the magic number last, or else we might look like we succeeded
+ * when we didn't */
+ st->magic = MAGIC(st);
+
+ st->pid = getpid();
+
+ return 0;
+}
+
+static int
+ottery_st_reseed(struct ottery_state *st)
+{
+ /* Now seed the PRF: Generate some random bytes from the OS, and use them
+ * as whatever keys/nonces/whatever the PRF wants to have. */
+ /* XXXX Add seed rather than starting from scratch? */
+ int err;
+ uint32_t flags=0;
+ size_t buflen = ottery_get_entropy_bufsize_(st->prf.state_bytes);
+ uint8_t *buf = alloca(buflen);
+ if (!buf)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+
+ if ((err = ottery_get_entropy_(&st->entropy_config, &st->entropy_state, 0,
+ buf, st->prf.state_bytes,
+ &buflen,
+ &flags)))
+ return err;
+ if (buflen < st->prf.state_bytes)
+ return OTTERY_ERR_ACCESS_STRONG_RNG;
+ /* The first state_bytes bytes become the initial key. */
+ st->prf.setup(st->state, buf);
+ /* If there are more bytes, we mix them into the key with add_seed */
+ if (buflen > st->prf.state_bytes)
+ ottery_st_add_seed_impl(st,
+ buf + st->prf.state_bytes,
+ buflen - st->prf.state_bytes,
+ 0,
+ 0);
+ ottery_memclear_(buf, buflen);
+ st->last_entropy_flags = flags;
+ st->entropy_src_flags = flags;
+
+ /* Generate the first block of output. */
+ st->block_counter = 0;
+ ottery_st_nextblock_nolock(st);
+
+ return 0;
+}
+
+int
+ottery_st_init(struct ottery_state *st, const struct ottery_config *cfg)
+{
+ return ottery_st_initialize(st, cfg, 1);
+}
+
+int
+ottery_st_init_nolock(struct ottery_state_nolock *st,
+ const struct ottery_config *cfg)
+{
+ return ottery_st_initialize(st, cfg, 0);
+}
+
+static int
+ottery_st_add_seed_impl(struct ottery_state *st, const uint8_t *seed, size_t n, int locking, int check_magic)
+{
+#ifndef OTTERY_NO_INIT_CHECK
+ if (check_magic && UNLIKELY(st->magic != MAGIC(st))) {
+ ottery_fatal_error_(OTTERY_ERR_STATE_INIT);
+ return OTTERY_ERR_STATE_INIT;
+ }
+#endif
+
+ /* If the user passed NULL, then we should reseed from the operating
+ * system. */
+ uint8_t *tmp_seed = NULL;
+ size_t tmp_seed_len = 0;
+ uint32_t flags = 0;
+
+ if (!seed || !n) {
+ int err;
+ tmp_seed_len = ottery_get_entropy_bufsize_(st->prf.state_bytes);
+ tmp_seed = alloca(tmp_seed_len);
+ if (!tmp_seed)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ n = tmp_seed_len;
+ if ((err = ottery_get_entropy_(&st->entropy_config, &st->entropy_state, 0,
+ tmp_seed, st->prf.state_bytes,
+ &n,
+ &flags)))
+ return err;
+ if (n < st->prf.state_bytes)
+ return OTTERY_ERR_ACCESS_STRONG_RNG;
+ seed = tmp_seed;
+ }
+
+ if (locking)
+ LOCK(st);
+ /* The algorithm here is really easy. We grab a block of output from the
+ * PRNG, that the first (state_bytes) bytes of that, XOR it with up to
+ * (state_bytes) bytes of our new seed data, and use that to set our new
+ * state. We do this over and over until we have no more seed data to add.
+ */
+ while (n) {
+ unsigned i;
+ size_t m = n > st->prf.state_bytes/2 ? st->prf.state_bytes/2 : n;
+ ottery_st_nextblock_nolock_norekey(st);
+ for (i = 0; i < m; ++i) {
+ st->buffer[i] ^= seed[i];
+ }
+ st->prf.setup(st->state, st->buffer);
+ st->block_counter = 0;
+ n -= m;
+ seed += m;
+ }
+
+ /* Now make sure that st->buffer is set up with the new state. */
+ ottery_st_nextblock_nolock(st);
+
+ st->entropy_src_flags |= flags;
+ st->last_entropy_flags = flags;
+
+ if (locking)
+ UNLOCK(st);
+
+ /* If we used stack-allocated seed material, wipe it. */
+ if (tmp_seed)
+ ottery_memclear_(tmp_seed, tmp_seed_len);
+
+ return 0;
+}
+
+int
+ottery_st_add_seed(struct ottery_state *st, const uint8_t *seed, size_t n)
+{
+ return ottery_st_add_seed_impl(st, seed, n, 1, 1);
+}
+int
+ottery_st_add_seed_nolock(struct ottery_state_nolock *st, const uint8_t *seed, size_t n)
+{
+ return ottery_st_add_seed_impl(st, seed, n, 0, 1);
+}
+
+
+void
+ottery_st_wipe(struct ottery_state *st)
+{
+ DESTROY_LOCK(&st->mutex);
+
+ ottery_st_wipe_nolock(st);
+}
+
+void
+ottery_st_wipe_nolock(struct ottery_state_nolock *st)
+{
+ ottery_memclear_(st, sizeof(struct ottery_state));
+}
+
+void
+ottery_st_prevent_backtracking_nolock(struct ottery_state_nolock *st)
+{
+#ifdef OTTERY_NO_CLEAR_AFTER_YIELD
+ memset(st->buffer, 0, st->pos);
+#else
+ (void)st;
+#endif
+}
+
+void
+ottery_st_prevent_backtracking(struct ottery_state *st)
+{
+ LOCK(st);
+ ottery_st_prevent_backtracking_nolock(st);
+ UNLOCK(st);
+}
+
+/** Function that's invoked on a fatal error. See
+ * ottery_set_fatal_handler() for more information. */
+static void (*ottery_fatal_handler)(int) = NULL;
+
+void
+ottery_fatal_error_(int error)
+{
+ if (ottery_fatal_handler)
+ ottery_fatal_handler(error);
+ else
+ abort();
+}
+
+void
+ottery_set_fatal_handler(void (*fn)(int))
+{
+ ottery_fatal_handler = fn;
+}
+
+/**
+ * Shared prologue for functions generating random bytes from an ottery_state.
+ * Make sure that the state is initialized.
+ */
+static inline int
+ottery_st_rand_check_init(struct ottery_state *st)
+{
+#ifndef OTTERY_NO_INIT_CHECK
+ if (UNLIKELY(st->magic != MAGIC(st))) {
+ ottery_fatal_error_(OTTERY_ERR_STATE_INIT);
+ return -1;
+ }
+#else
+ (void)st;
+#endif
+ return 0;
+}
+
+/* XXXX */
+static inline int
+ottery_st_rand_check_pid(struct ottery_state *st)
+{
+#ifndef OTTERY_NO_PID_CHECK
+ if (UNLIKELY(st->pid != getpid())) {
+ int err;
+ if ((err = ottery_st_reseed(st))) {
+ ottery_fatal_error_(OTTERY_ERR_FLAG_POSTFORK_RESEED|err);
+ return -1;
+ }
+ st->pid = getpid();
+ }
+#else
+ (void) st;
+#endif
+ return 0;
+}
+
+static inline int
+ottery_st_rand_lock_and_check(struct ottery_state *st)
+{
+ if (ottery_st_rand_check_init(st))
+ return -1;
+ LOCK(st);
+ if (ottery_st_rand_check_pid(st)) {
+ UNLOCK(st);
+ return -1;
+ }
+ return 0;
+}
+
+static inline int
+ottery_st_rand_check_nolock(struct ottery_state_nolock *st)
+{
+ if (ottery_st_rand_check_init(st))
+ return -1;
+ if (ottery_st_rand_check_pid(st))
+ return -1;
+ return 0;
+}
+
+/**
+ * Generate a small-ish number of bytes from an ottery_state, using
+ * buffered data. If there is insufficient data in the buffer right now,
+ * use what we have, and generate more.
+ *
+ * @param st The state to use.
+ * @param out A location to write to.
+ * @param n The number of bytes to write. Must not be greater than
+ * st->prf.output_len*2 - st->prf.state_bytes - st->pos - 1.
+ */
+static inline void
+ottery_st_rand_bytes_from_buf(struct ottery_state *st, uint8_t *out,
+ size_t n)
+{
+ if (n + st->pos < st->prf.output_len) {
+ memcpy(out, st->buffer+st->pos, n);
+ CLEARBUF(st->buffer+st->pos, n);
+ st->pos += n;
+ } else {
+ unsigned cpy = st->prf.output_len - st->pos;
+ memcpy(out, st->buffer+st->pos, cpy);
+ n -= cpy;
+ out += cpy;
+ ottery_st_nextblock_nolock(st);
+ memcpy(out, st->buffer+st->pos, n);
+ CLEARBUF(st->buffer, n);
+ st->pos += n;
+ assert(st->pos < st->prf.output_len);
+ }
+}
+
+static void
+ottery_st_rand_bytes_impl(struct ottery_state *st, void *out_,
+ size_t n)
+{
+ uint8_t *out = out_;
+ size_t cpy;
+
+ if (n + st->pos < st->prf.output_len * 2 - st->prf.state_bytes - 1) {
+ /* Fulfill it all from the buffer simply if possible. */
+ ottery_st_rand_bytes_from_buf(st, out, n);
+ return;
+ }
+
+ /* Okay. That's not going to happen. Well, take what we can... */
+ cpy = st->prf.output_len - st->pos;
+ memcpy(out, st->buffer + st->pos, cpy);
+ out += cpy;
+ n -= cpy;
+
+ /* Then take whole blocks so long as we need them, without stirring... */
+ while (n >= st->prf.output_len) {
+ /* (We could save a memcpy here if we generated the block directly at out
+ * rather than doing the memcpy here. First we'd need to make sure that we
+ * had gotten the block aligned to a 16-byte boundary, though, and we'd
+ * have some other tricky bookkeeping to do. Let's call this good enough
+ * for now.) */
+ ottery_st_nextblock_nolock_norekey(st);
+ memcpy(out, st->buffer, st->prf.output_len);
+ out += st->prf.output_len;
+ n -= st->prf.output_len;
+ }
+
+ /* Then stir for the last part. */
+ ottery_st_nextblock_nolock(st);
+ ottery_st_rand_bytes_from_buf(st, out, n);
+}
+
+void
+ottery_st_rand_bytes(struct ottery_state *st, void *out_, size_t n)
+{
+ if (ottery_st_rand_lock_and_check(st))
+ return;
+ ottery_st_rand_bytes_impl(st, out_, n);
+ UNLOCK(st);
+}
+
+void
+ottery_st_rand_bytes_nolock(struct ottery_state_nolock *st, void *out_, size_t n)
+{
+ if (ottery_st_rand_check_nolock(st))
+ return;
+ ottery_st_rand_bytes_impl(st, out_, n);
+}
+
+/**
+ * Assign an integer type from bytes at a possibly unaligned pointer.
+ *
+ * @param type the type of integer to assign.
+ * @param r the integer lvalue to write to.
+ * @param p a pointer to the bytes to read from.
+ **/
+#define INT_ASSIGN_PTR(type, r, p) do { \
+ memcpy(&r, p, sizeof(type)); \
+} while (0)
+
+/**
+ * Shared code for implementing rand_unsigned() and rand_uint64().
+ *
+ * @param st The state to use.
+ * @param inttype The type of integer to generate.
+ **/
+#define OTTERY_RETURN_RAND_INTTYPE_IMPL(st, inttype, unlock) do { \
+ inttype result; \
+ if (sizeof(inttype) + (st)->pos <= (st)->prf.output_len) { \
+ INT_ASSIGN_PTR(inttype, result, (st)->buffer + (st)->pos); \
+ CLEARBUF((st)->buffer + (st)->pos, sizeof(inttype)); \
+ (st)->pos += sizeof(inttype); \
+ if (st->pos == (st)->prf.output_len) { \
+ ottery_st_nextblock_nolock(st); \
+ } \
+ } else { \
+ /* Our handling of this case here is significantly simpler */ \
+ /* than that of ottery_st_rand_bytes_from_buf, at the expense */ \
+ /* of wasting up to sizeof(inttype)-1 bytes. Since inttype */ \
+ /* is at most 8 bytes long, that's not such a big deal. */ \
+ ottery_st_nextblock_nolock(st); \
+ INT_ASSIGN_PTR(inttype, result, (st)->buffer + (st)->pos); \
+ CLEARBUF((st)->buffer, sizeof(inttype)); \
+ (st)->pos += sizeof(inttype); \
+ } \
+ unlock; \
+ return result; \
+} while (0)
+
+#define OTTERY_RETURN_RAND_INTTYPE(st, inttype) do { \
+ if (ottery_st_rand_lock_and_check(st)) \
+ return (inttype)0; \
+ OTTERY_RETURN_RAND_INTTYPE_IMPL(st, inttype, UNLOCK(st)); \
+} while (0)
+
+#define OTTERY_RETURN_RAND_INTTYPE_NOLOCK(st, inttype) do { \
+ if (ottery_st_rand_check_nolock(st)) \
+ return (inttype)0; \
+ OTTERY_RETURN_RAND_INTTYPE_IMPL(st, inttype, ); \
+} while (0)
+
+unsigned
+ottery_st_rand_unsigned(struct ottery_state *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE(st, unsigned);
+}
+
+unsigned
+ottery_st_rand_unsigned_nolock(struct ottery_state_nolock *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE_NOLOCK(st, unsigned);
+}
+
+uint32_t
+ottery_st_rand_uint32(struct ottery_state *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE(st, uint32_t);
+}
+
+uint32_t
+ottery_st_rand_uint32_nolock(struct ottery_state_nolock *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE_NOLOCK(st, uint32_t);
+}
+
+uint64_t
+ottery_st_rand_uint64(struct ottery_state *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE(st, uint64_t);
+}
+
+uint64_t
+ottery_st_rand_uint64_nolock(struct ottery_state_nolock *st)
+{
+ OTTERY_RETURN_RAND_INTTYPE_NOLOCK(st, uint64_t);
+}
+
+unsigned
+ottery_st_rand_range_nolock(struct ottery_state_nolock *st, unsigned upper)
+{
+ unsigned lim = upper+1;
+ unsigned divisor = lim ? (UINT_MAX / lim) : 1;
+ unsigned n;
+ do {
+ n = (ottery_st_rand_unsigned_nolock(st) / divisor);
+ } while (n > upper);
+
+ return n;
+}
+
+uint64_t
+ottery_st_rand_range64_nolock(struct ottery_state_nolock *st, uint64_t upper)
+{
+ uint64_t lim = upper+1;
+ uint64_t divisor = lim ? (UINT64_MAX / lim) : 1;
+ uint64_t n;
+ do {
+ n = (ottery_st_rand_uint64_nolock(st) / divisor);
+ } while (n > upper);
+
+ return n;
+}
+
+unsigned
+ottery_st_rand_range(struct ottery_state *state, unsigned upper)
+{
+ unsigned n;
+ if (ottery_st_rand_check_init(state))
+ return 0;
+ LOCK(state);
+ n = ottery_st_rand_range_nolock(state, upper);
+ UNLOCK(state);
+ return n;
+}
+
+uint64_t
+ottery_st_rand_range64(struct ottery_state *state, uint64_t upper)
+{
+ uint64_t n;
+ if (ottery_st_rand_check_init(state))
+ return 0;
+ LOCK(state);
+ n = ottery_st_rand_range64_nolock(state, upper);
+ UNLOCK(state);
+ return n;
+}
diff --git a/contrib/libottery/ottery.h b/contrib/libottery/ottery.h
new file mode 100644
index 0000000..e2caac2
--- /dev/null
+++ b/contrib/libottery/ottery.h
@@ -0,0 +1,143 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_H_HEADER_INCLUDED_
+#define OTTERY_H_HEADER_INCLUDED_
+#include <stdint.h>
+#include <sys/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "ottery_common.h"
+
+/** @file */
+
+struct ottery_config;
+
+/* Functions that use an implicit global state */
+
+/**
+ * Fill a buffer with random bytes.
+ *
+ * @param buf The buffer to fill.
+ * @param n The number of bytes to write.
+ */
+void ottery_rand_bytes(void *buf, size_t n);
+/**
+ * Generate a random number of type unsigned.
+ *
+ * @return A random number between 0 and UINT_MAX included,
+ * chosen uniformly.
+ */
+unsigned ottery_rand_unsigned(void);
+/**
+ * Generate a random number of type uint32_t.
+ *
+ * @return A random number between 0 and UINT32_MAX included,
+ * chosen uniformly.
+ */
+uint32_t ottery_rand_uint32(void);
+/**
+ * Generate a random number of type uint64_t.
+ *
+ * @return A random number between 0 and UINT64_MAX included,
+ * chosen uniformly.
+ */
+uint64_t ottery_rand_uint64(void);
+/**
+ * Generate a random number of type unsigned in a given range.
+ *
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+unsigned ottery_rand_range(unsigned top);
+/**
+ * Generate a random number of type uint64_t in a given range.
+ *
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+uint64_t ottery_rand_range64(uint64_t top);
+
+/**
+ * Initialize the libottery global state.
+ *
+ * Most users should not need to use this function. If you use it, you must
+ * call it before any of: ottery_rand_bytes, ottery_rand_unsigned,
+ * ottery_rand_uint64, ottery_rand_range, ottery_rand_uint64_range,
+ * ottery_add_seed, ottery_wipe, ottery_stir.
+ *
+ * You would want to use this function if you want to select some non-default
+ * behavior using an ottery_config structure.
+ *
+ * @param cfg Either NULL, or an ottery_config structure that has been
+ * initialized with ottery_config_init().
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_init(const struct ottery_config *cfg);
+
+/**
+ * Add more entropy to the libottery global state.
+ *
+ * Calling this function should be needless, if you trust your operating
+ * system's random number generator and entropy extraction features. You
+ * would want to use this function if you think the operating system's random
+ * number generator might be inadequate, and you want to add more entropy from
+ * EGD or something.
+ *
+ * You might also want to call this function if your belief system says that
+ * it's useful to periodically add more raw entropy to a well-seeded
+ * cryptographically strong PRNG.
+ *
+ * @param seed Bytes to add to the state. If this value is NULL, we take
+ * more random bytes from the OS.
+ * @param n The number of bytes to add. If this value is 0, we take more
+ * random bytes from the OS, regardless of the value of seed.
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_add_seed(const uint8_t *seed, size_t n);
+
+/**
+ * Destroy the libottery global state and release any resources that it might
+ * hold.
+ *
+ * Ordinarily, you would only want to call this at exit, if at all.
+ */
+void ottery_wipe(void);
+
+/**
+ * Explicitly tell libottery to prevent backtracking attacks. (Usually
+ * needless.)
+ *
+ * Once this function has been called, an attacker who compromises the state
+ * later on will not be able to recover bytes that have previously been
+ * returned by any of the ottery_rand_* functions.
+ *
+ * You should not usually need to call this function: Libottery provides
+ * backtracking resistance by default, so unless you have manually recompiled
+ * with the OTTERY_NO_CLEAR_AFTER_YIELD option, this function isn't
+ * necessary and has no effect. Even *with* OTTERY_NO_CLEAR_AFTER_YIELD,
+ * this function isn't necessary in ordinary operation: the libottery state is
+ * implicitly "stirred" every 1k or so.
+ */
+void ottery_prevent_backtracking(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/libottery/ottery_common.h b/contrib/libottery/ottery_common.h
new file mode 100644
index 0000000..bac6f04
--- /dev/null
+++ b/contrib/libottery/ottery_common.h
@@ -0,0 +1,351 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_COMMON_H_HEADER_INCLUDED_
+#define OTTERY_COMMON_H_HEADER_INCLUDED_
+#include <stdint.h>
+#include <sys/types.h>
+
+/** @file */
+
+struct ottery_config;
+
+/* Error codes */
+
+/**
+ * @name libottery error codes and flags
+ *
+ * @{
+ */
+/** No error has occurred. */
+#define OTTERY_ERR_NONE 0x0000
+/** We failed to allocate or initialize a lock. */
+#define OTTERY_ERR_LOCK_INIT 0x0001
+/** An internal error occurrred. This is probably a programming mistake
+ * in libottery. */
+#define OTTERY_ERR_INTERNAL 0x0002
+/** We were unable to connect to the operating system's strong RNG. */
+#define OTTERY_ERR_INIT_STRONG_RNG 0x0003
+/** We were unable to retrieve sufficient random bytes from the
+ * operating system's strong RNG. */
+#define OTTERY_ERR_ACCESS_STRONG_RNG 0x0004
+/** At least one argument to the function was invalid. */
+#define OTTERY_ERR_INVALID_ARGUMENT 0x0005
+/** An ottery_state structure was not aligned to a 16-byte boundary. */
+#define OTTERY_ERR_STATE_ALIGNMENT 0x0006
+
+/** FATAL ERROR: An ottery_st function other than ottery_st_init() was
+ * called on and uninitialized state. */
+#define OTTERY_ERR_STATE_INIT 0x1000
+/** FLAG; FATAL ERROR: The error occurred while initializing the global
+ * state during the first call to an ottery_rand_* function. */
+#define OTTERY_ERR_FLAG_GLOBAL_PRNG_INIT 0x2000
+/** FLAG; FATAL ERROR: The error occurred while reinitializing a state
+ * after a fork(). (We need to do this, or else both processes would
+ * generate the same values, which could give dire results.)
+ */
+#define OTTERY_ERR_FLAG_POSTFORK_RESEED 0x4000
+
+/**
+ * Checks whether an OTTERY_ERR value is a fatal error.
+ *
+ * @param err an OTTERY_ERR_* valuer
+ * @return True if err is fatal; false if it is not fatal.
+ */
+#define OTTERY_ERR_IS_FATAL(err) \
+ (((err) & ~0xfff) != 0)
+
+/* Functions to interact with the library on a global level */
+
+/**
+ * Override the behavior of libottery on a fatal error.
+ *
+ * By default, libottery will call abort() in a few circumstances, in
+ * order to keep the program from operating insecurely. If you want,
+ * you can provide another function to call instead.
+ *
+ * If your function does not itself abort() or exit() the process, or throw an
+ * exception (assuming some C family that has exceptions), libottery will
+ * continue running insecurely -- it might return predictable random numbers,
+ * leak secrets, or just return 0 for everything -- so you should really be
+ * very careful here.
+ *
+ * (The alternative to fatal errors would have been having all the
+ * ottery_rand_* functions able to return an error, and requiring users
+ * to check those codes. But experience suggests that C programmers
+ * frequently do not check error codes.)
+ *
+ * @param fn A function to call in place of abort(). It will receive as
+ * its argument one of the OTTERY_ERR_* error codes.
+ */
+void ottery_set_fatal_handler(void (*fn)(int errorcode));
+
+/* Functions to manipulate parameters. */
+
+/**
+ * @name Names of prfs for use with ottery_config_force_implementation
+ *
+ * @{ */
+#define OTTERY_PRF_CHACHA "CHACHA"
+#define OTTERY_PRF_CHACHA8 "CHACHA8"
+#define OTTERY_PRF_CHACHA12 "CHACHA12"
+#define OTTERY_PRF_CHACHA20 "CHACHA20"
+#define OTTERY_PRF_CHACHA_SIMD "CHACHA-SIMD"
+#define OTTERY_PRF_CHACHA8_SIMD "CHACHA8-SIMD"
+#define OTTERY_PRF_CHACHA12_SIMD "CHACHA12-SIMD"
+#define OTTERY_PRF_CHACHA20_SIMD "CHACHA20-SIMD"
+#define OTTERY_PRF_CHACHA_NO_SIMD "CHACHA-NOSIMD"
+#define OTTERY_PRF_CHACHA8_NO_SIMD "CHACHA8-NOSIMD"
+#define OTTERY_PRF_CHACHA12_NO_SIMD "CHACHA12-NOSIMD"
+#define OTTERY_PRF_CHACHA20_NO_SIMD "CHACHA20-NOSIMD"
+/** @} */
+
+/**
+ * Initialize an ottery_config structure.
+ *
+ * You must call this function on any ottery_config structure before it
+ * can be passed to ottery_init() or ottery_st_init().
+ *
+ * @param cfg The configuration object to initialize.
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on
+ * failure.
+ */
+int ottery_config_init(struct ottery_config *cfg);
+
+/**
+ * Try to force the use of a particular pseudorandom function for a given
+ * libottery instance.
+ *
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * @param cfg The configuration structure to configure.
+ * @param impl The name of a pseudorandom function. One of the
+ * OTTERY_PRF_* values.
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on
+ * failure.
+ */
+int ottery_config_force_implementation(struct ottery_config *cfg,
+ const char *impl);
+
+/**
+ * Set a device file to use as a source of strong entropy.
+ *
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * By default, libottery will try /dev/urandom on Unix-like systems.
+ *
+ * @param cfg The configuration structure to configure.
+ * @param fname The name of the device to use instead of /dev/urandom. This
+ * pointer is copied around, and must not be freed while any libottery state
+ * configured using this structure is still in use.
+ *
+ */
+void ottery_config_set_urandom_device(struct ottery_config *cfg,
+ const char *fname);
+
+/**
+ * Set a device file to use as a source of strong entropy from the operating
+ * system.
+ *
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * This function overrides the default behavior, and overrides any
+ * setting in ottery_config_set_urandom_device.
+ *
+ * You MUST NOT change the the file descriptor while any libottery PRNG
+ * configured with it is still running. For example, don't close it, or use
+ * dup2 to make it refer to a different file, or anything like that.
+ *
+ * It is probably a good idea to open the file with the CLOEXEC flag set.
+ *
+ * @param cfg The configuration structure to configure.
+ * @param fd A file descriptor to use as an OS rng source.
+ */
+void ottery_config_set_urandom_fd(struct ottery_config *cfg,
+ int fd);
+
+struct sockaddr;
+
+/**
+ * Configure a socket at which to find a local copy of some service
+ * implementing the EGD (entropy-gathering daemon) protocol.
+ *
+ * Unless this function is called, EGD is not used by default.
+
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * TODO: This is not implemented for Windows yet.
+ *
+ * @param cfg The configuration structure to configure.
+ * @param addr The address of the daemon. Obviously, this should be
+ * some port on localhost, or a unix socket. This pointer is copied
+ * around, and must not be freed while any libottery state configured
+ * using this structure is still in use.
+ * @param len the length of the address.
+ *
+ */
+void ottery_config_set_egd_socket(struct ottery_config *cfg,
+ const struct sockaddr *addr,
+ int len);
+
+/**
+ * @brief External entropy sources.
+ *
+ * These can be passed as a bitmask to ottery_config_disable_entropy_sources.
+ *
+ * @{ */
+/** A unix-style /dev/urandom device. */
+#define OTTERY_ENTROPY_SRC_RANDOMDEV 0x0010000
+/** The Windows CryptGenRandom call. */
+#define OTTERY_ENTROPY_SRC_CRYPTGENRANDOM 0x0020000
+/** The Intel RDRAND instruction. */
+#define OTTERY_ENTROPY_SRC_RDRAND 0x0040000
+/** Some local server obeying the EGD protocol. Has no effect unless
+ * ottery_config_set_egd_socket was called. */
+#define OTTERY_ENTROPY_SRC_EGD 0x0080000
+/** @} */
+
+/**
+ * Disable the use of one or more entropy sources.
+ *
+ * Note that if enough entropy sources are disabled, the state will
+ * not be able to get initialized, and libottery might not work.
+ *
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * @param cfg A configuration in which to disable one or more entropy sources.
+ * @param disabled_sources a bitwise combination of one or more
+ * OTTERY_ENTROPY_SRC_* values to disable. This will replace
+ * any previous bitmask of disabled sources.
+ *
+ */
+void ottery_config_disable_entropy_sources(struct ottery_config *cfg,
+ uint32_t disabled_sources);
+
+/**
+ * Mark one or more entropy sources as "weak".
+ *
+ * Unlike a disabled source, we will still try to read entropy from
+ * a weak source -- but we will fail if _only_ weak sources are available.
+ *
+ * Note that if enough entropy sources are disabled and/or weak sources are
+ * failing, the state will not be able to get initialized, and libottery might
+ * not work.
+ *
+ * To use this function, you call it on an ottery_config structure after
+ * ottery_config_init(), and before passing that structure to
+ * ottery_st_init() or ottery_init().
+ *
+ * @param cfg A configuration in which to disable one or more entropy sources.
+ * @param weak_sources a bitwise combination of one or more
+ * OTTERY_ENTROPY_SRC_* values to mark as weak. This will replace
+ * any previous bitmask of weak sources.
+ */
+void ottery_config_mark_entropy_sources_weak(struct ottery_config *cfg,
+ uint32_t weak_source);
+
+/** Size reserved for struct ottery_config */
+#define OTTERY_CONFIG_DUMMY_SIZE_ 1024
+
+#ifndef OTTERY_INTERNAL
+/**
+ * A configuration object for setting up a libottery instance.
+ *
+ * An ottery_config structure is initialized with ottery_config_init,
+ * and passed to ottery_init() or ottery_st_init().
+ *
+ * The contents of this structure are opaque; The definition here is
+ * defined to be large enough so that programs that allocate it will get
+ * more than enough room.
+ */
+struct ottery_config {
+ /** Nothing to see here */
+ uint8_t dummy_[OTTERY_CONFIG_DUMMY_SIZE_];
+};
+#endif
+
+/**
+ * Get the minimal size for allocating an ottery_config.
+ *
+ * sizeof(ottery_config) will give an overestimate to allow binary
+ * compatibility with future versions of libottery. Use this function instead
+ * to get the minimal number of bytes to allocate.
+ *
+ * @return The minimal number of bytes to use when allocating an
+ * ottery_config structure.
+ */
+size_t ottery_get_sizeof_config(void);
+
+/**
+ * @name libottery build flag
+ *
+ * @see ottery_Get_build_flags()
+ *
+ * @{
+ */
+/** Set if libottery was built with PID checking disabled. If this option is
+ * present, fork()ing can be dangerous. */
+#define OTTERY_BLDFLG_NO_PID_CHECK 0x00000001
+/** Set if libottery was built with initialization checking disabled. If this
+ * option is present, libottery might use an uninitialized, unseeded PRNGs.
+ */
+#define OTTERY_BLDFLG_NO_INIT_CHECK 0x00000002
+/** Set if locking was disabled. If this option is present, no libottery
+ * state, including the global state, is thread-safe. */
+#define OTTERY_BLDFLG_NO_LOCKING 0x00000004
+/** Set if the clear-after-yield feature was disabled. If this option is
+ * present, backtracking-resistance is somewhat compromised. */
+#define OTTERY_BLDFLG_NO_CLEAR_AFTER_YIELD 0x00000008
+/** Set if the stack-wiping feature was disabled. If this option is
+ * present, programs which accidentally read uninitialized data from the
+ * stack may leak some cryptographic state. */
+#define OTTERY_BLDFLG_NO_WIPE_STACK 0x00000010
+/** Set if SIMD support was disabled. This will make libottery slower. */
+#define OTTERY_BLDFLG_NO_SIMD 0x00010000
+/** @} */
+
+/** A bitmask of any flags that might affect safe and secure program
+ * operation. */
+#define OTTERY_BLDFLG_MASK_SAFETY 0x0000ffff
+
+/**
+ * Return a bitmask of flags describing the compile-time options that this
+ * libottery instance was built with. Some of these flags might make the
+ * library less safe to use!
+ */
+uint32_t ottery_get_build_flags(void);
+
+/**
+ * Return a run-time version number for Libottery. The first three bytes are
+ * the major number, minor number, and patch-level respectively. The final
+ * byte is 0 for a released version, and nonzero otherwise.
+ */
+uint32_t ottery_get_version(void);
+/**
+ * Return a human-readable string representing the run-time Libottery version.
+ */
+const char *ottery_get_version_string(void);
+
+const char *ottery_get_impl_name(void);
+
+#endif
diff --git a/contrib/libottery/ottery_cpuinfo.c b/contrib/libottery/ottery_cpuinfo.c
new file mode 100644
index 0000000..2e8bdaa
--- /dev/null
+++ b/contrib/libottery/ottery_cpuinfo.c
@@ -0,0 +1,88 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#include "ottery-internal.h"
+#include <stdint.h>
+
+#if defined(i386) || \
+ defined(__i386) || \
+ defined(__M_IX86) || \
+ defined(_M_IX86)
+#define X86
+#elif defined(__x86_64) || \
+ defined(_M_AMD64)
+#define X86
+#define X86_64
+#endif
+
+#if defined(__arm__) || \
+ defined(_M_ARM)
+#define ARM
+#endif
+
+#if defined(X86)
+#ifdef _MSC_VER
+#include <intrin.h>
+#define cpuid(a,b) __cpuid((b), (a))
+#else
+static void
+cpuid(int index, int regs[4])
+{
+ unsigned int eax, ebx, ecx, edx;
+#ifdef X86_64
+ __asm("cpuid" : "=a"(eax), "=b" (ebx), "=c"(ecx), "=d"(edx)
+ : "0"(index));
+#else
+ __asm volatile(
+ "xchgl %%ebx, %1; cpuid; xchgl %%ebx, %1"
+ : "=a" (eax), "=r" (ebx), "=c" (ecx), "=d" (edx)
+ : "0" (index)
+ : "cc" );
+#endif
+
+ regs[0] = eax;
+ regs[1] = ebx;
+ regs[2] = ecx;
+ regs[3] = edx;
+}
+#endif
+#endif
+
+static uint32_t disabled_cpu_capabilities = 0;
+
+void
+ottery_disable_cpu_capabilities_(uint32_t disable)
+{
+ disabled_cpu_capabilities |= disable;
+}
+
+uint32_t
+ottery_get_cpu_capabilities_(void)
+{
+#ifdef X86
+ uint32_t cap = 0;
+ int res[4];
+ cpuid(1, res);
+ if (res[3] & (1<<26))
+ cap |= OTTERY_CPUCAP_SIMD;
+ if (res[2] & (1<<9))
+ cap |= OTTERY_CPUCAP_SSSE3;
+ if (res[2] & (1<<25))
+ cap |= OTTERY_CPUCAP_AES;
+ if (res[2] & (1<<30))
+ cap |= OTTERY_CPUCAP_RAND;
+#else
+ uint32_t cap = OTTERY_CPUCAP_SIMD;
+#endif
+ return cap & ~disabled_cpu_capabilities;
+}
diff --git a/contrib/libottery/ottery_entropy.c b/contrib/libottery/ottery_entropy.c
new file mode 100644
index 0000000..819a380
--- /dev/null
+++ b/contrib/libottery/ottery_entropy.c
@@ -0,0 +1,112 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#define OTTERY_INTERNAL
+#include "ottery-internal.h"
+#include "ottery.h"
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+
+#define SRC(x) OTTERY_ENTROPY_SRC_ ## x
+#define DOM(x) OTTERY_ENTROPY_DOM_ ## x
+#define FL(x) OTTERY_ENTROPY_FL_ ## x
+
+#include "ottery_entropy_cryptgenrandom.c"
+#include "ottery_entropy_urandom.c"
+#include "ottery_entropy_rdrand.c"
+#include "ottery_entropy_egd.c"
+
+/** Table of RNG functions and their properties. */
+static struct ottery_randbytes_source {
+ int (*fn)(const struct ottery_entropy_config *,
+ struct ottery_entropy_state *,
+ uint8_t *, size_t);
+ uint32_t flags;
+} RAND_SOURCES[] = {
+#ifdef ENTROPY_SOURCE_CRYPTGENRANDOM
+ ENTROPY_SOURCE_CRYPTGENRANDOM,
+#endif
+#ifdef ENTROPY_SOURCE_URANDOM
+ ENTROPY_SOURCE_URANDOM,
+#endif
+#ifdef ENTROPY_SOURCE_EGD
+ ENTROPY_SOURCE_EGD,
+#endif
+#ifdef ENTROPY_SOURCE_RDRAND
+ ENTROPY_SOURCE_RDRAND,
+#endif
+ { NULL, 0 }
+};
+
+size_t
+ottery_get_entropy_bufsize_(size_t n)
+{
+ return n * (sizeof(RAND_SOURCES)/sizeof(RAND_SOURCES[0]) - 1);
+}
+
+int
+ottery_get_entropy_(const struct ottery_entropy_config *config,
+ struct ottery_entropy_state *state,
+ uint32_t select_sources,
+ uint8_t *bytes, size_t n, size_t *buflen,
+ uint32_t *flags_out)
+{
+ ssize_t err = OTTERY_ERR_INIT_STRONG_RNG, last_err = 0;
+ int i;
+ uint32_t got = 0;
+ uint8_t *next;
+ const uint32_t disabled_sources = config ? config->disabled_sources : 0;
+
+ memset(bytes, 0, *buflen);
+ next = bytes;
+
+ *flags_out = 0;
+
+ for (i=0; RAND_SOURCES[i].fn; ++i) {
+ uint32_t flags = RAND_SOURCES[i].flags;
+ /* Don't use a disabled source. */
+ if (0 != (flags & disabled_sources))
+ continue;
+ /* If some flags must be set, only use those. */
+ if ((flags & select_sources) != select_sources)
+ continue;
+ /* If we already have input from a certain domain, we don't need more */
+ if ((flags & (got & OTTERY_ENTROPY_DOM_MASK)) != 0)
+ continue;
+ /* If we can't write these bytes, don't try. */
+ if (next + n > bytes + *buflen)
+ break;
+ err = RAND_SOURCES[i].fn(config, state, next, n);
+ if (err == 0) {
+ uint32_t flags = RAND_SOURCES[i].flags;
+ if (config && (flags & config->weak_sources))
+ flags &= ~OTTERY_ENTROPY_FL_STRONG;
+
+ got |= flags;
+ next += n;
+ } else {
+ last_err = err;
+ }
+ }
+
+ /* Do not report success unless at least one source was strong. */
+ if (0 == (got & OTTERY_ENTROPY_FL_STRONG))
+ return last_err ? last_err : OTTERY_ERR_INIT_STRONG_RNG;
+
+ *flags_out = got;
+ *buflen = next - bytes;
+
+ return 0;
+}
diff --git a/contrib/libottery/ottery_entropy_cryptgenrandom.c b/contrib/libottery/ottery_entropy_cryptgenrandom.c
new file mode 100644
index 0000000..d29d9d1
--- /dev/null
+++ b/contrib/libottery/ottery_entropy_cryptgenrandom.c
@@ -0,0 +1,51 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#define OTTERY_INTERNAL
+#include "ottery-internal.h"
+#include "ottery.h"
+
+#ifdef _WIN32
+
+/** Generate random bytes using the Windows CryptGenRandom operating-system
+ * RNG. */
+static int
+ottery_get_entropy_cryptgenrandom(const struct ottery_entropy_config *cfg,
+ struct ottery_entropy_state *state,
+ uint8_t *out, size_t outlen)
+{
+ /* On Windows, CryptGenRandom is supposed to be a well-seeded
+ * cryptographically strong random number generator. */
+ HCRYPTPROV provider;
+ int retval = 0;
+ (void) cfg;
+ (void) state;
+
+ if (0 == CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL,
+ CRYPT_VERIFYCONTEXT))
+ return OTTERY_ERR_INIT_STRONG_RNG;
+
+ if (0 == CryptGenRandom(provider, outlen, out))
+ retval = OTTERY_ERR_ACCESS_STRONG_RNG;
+
+ CryptReleaseContext(provider, 0);
+ return retval;
+}
+
+#define ENTROPY_SOURCE_CRYPTGENRANDOM \
+ { ottery_get_entropy_cryptgenrandom, \
+ SRC(CRYPTGENRANDOM)|DOM(OS)|FL(STRONG) }
+
+#endif
+
+
diff --git a/contrib/libottery/ottery_entropy_egd.c b/contrib/libottery/ottery_entropy_egd.c
new file mode 100644
index 0000000..7e9cb07
--- /dev/null
+++ b/contrib/libottery/ottery_entropy_egd.c
@@ -0,0 +1,75 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef _WIN32
+/* TODO: Support win32. */
+#include <sys/socket.h>
+
+/** Implement an entropy-source that uses the EGD protocol. The
+ * Entropy-Gathering Daemon is program (actually, one of several programs)
+ * that watches system events, periodically runs commands whose outputs have
+ * high variance, and so on. It communicates over a simple socket-based
+ * protocol, of which we use only a tiny piece. */
+static int
+ottery_get_entropy_egd(const struct ottery_entropy_config *cfg,
+ struct ottery_entropy_state *state,
+ uint8_t *out, size_t outlen)
+{
+ int sock, n, result;
+ unsigned char msg[2];
+ (void) state;
+
+ if (! cfg || ! cfg->egd_sockaddr || ! cfg->egd_socklen)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ if (outlen > 255)
+ return OTTERY_ERR_ACCESS_STRONG_RNG;
+
+ sock = socket(cfg->egd_sockaddr->sa_family, SOCK_STREAM, 0);
+ if (sock < 0)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+
+ if (connect(sock, cfg->egd_sockaddr, cfg->egd_socklen) < 0) {
+ result = OTTERY_ERR_INIT_STRONG_RNG;
+ goto out;
+ }
+
+ msg[0] = 1; /* nonblocking request */
+ msg[1] = (unsigned char) outlen; /* for outlen bytes */
+
+ if (write(sock, msg, 2) != 2 ||
+ read(sock, msg, 1) != 1) {
+ result = OTTERY_ERR_ACCESS_STRONG_RNG;
+ goto out;
+ }
+
+ if (msg[0] != outlen) {
+ /* TODO Use any bytes we get, even if they aren't as many as we wanted. */
+ result = OTTERY_ERR_ACCESS_STRONG_RNG;
+ goto out;
+ }
+
+ n = ottery_read_n_bytes_from_file_(sock, out, outlen);
+ if (n < 0 || (size_t)n != outlen) {
+ result = OTTERY_ERR_ACCESS_STRONG_RNG;
+ goto out;
+ }
+ result = 0;
+ out:
+ close(sock);
+ return result;
+}
+
+#define ENTROPY_SOURCE_EGD \
+ { ottery_get_entropy_egd, SRC(EGD)|DOM(EGD)|FL(STRONG) }
+
+#endif
diff --git a/contrib/libottery/ottery_entropy_rdrand.c b/contrib/libottery/ottery_entropy_rdrand.c
new file mode 100644
index 0000000..4f227d3
--- /dev/null
+++ b/contrib/libottery/ottery_entropy_rdrand.c
@@ -0,0 +1,58 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+
+#if defined(i386) || \
+ defined(__i386) || \
+ defined(__x86_64) || \
+ defined(__M_IX86) || \
+ defined(_M_IX86) || \
+ defined(__INTEL_COMPILER)
+
+extern int ottery_valgrind_;
+/** Helper: invoke the RDRAND instruction to get 4 random bytes in the output
+ * value. Return 1 on success, and 0 on failure. */
+#define rdrand32(x) ({ unsigned char err = 0; __asm volatile(".byte 0x0f; .byte 0xc7; .byte 0xf0; setc %1":"=a"(x), "=qm"(err) :"a"(0) :"cc"); err; })
+
+/** Generate bytes using the Intel RDRAND instruction. */
+static int
+ottery_get_entropy_rdrand(const struct ottery_entropy_config *cfg,
+ struct ottery_entropy_state *state,
+ uint8_t *out, size_t outlen)
+{
+ uint32_t up;
+ (void) cfg;
+ (void) state;
+ if (! (ottery_get_cpu_capabilities_() & OTTERY_CPUCAP_RAND) || ottery_valgrind_)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ while (outlen >= 4) {
+ if (rdrand32(up) != 1)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ memcpy (out, &up, sizeof (up));
+ out += sizeof (up);
+ outlen -= 4;
+ }
+
+ if (outlen) {
+ if (rdrand32(up) != 1)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ memcpy(out, &up, outlen);
+ }
+ return 0;
+}
+
+#define ENTROPY_SOURCE_RDRAND \
+ { ottery_get_entropy_rdrand, SRC(RDRAND)|DOM(CPU)|FL(FAST)|FL(STRONG) }
+
+#endif
+
diff --git a/contrib/libottery/ottery_entropy_urandom.c b/contrib/libottery/ottery_entropy_urandom.c
new file mode 100644
index 0000000..9eb761b
--- /dev/null
+++ b/contrib/libottery/ottery_entropy_urandom.c
@@ -0,0 +1,117 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+
+#ifndef _WIN32
+
+/**
+ * Read from a file into an n-byte buffer until the buffer is full or until
+ * we reach an error. Returns the number of bytes read. If the return
+ * value is less than n, an error occurred.
+ */
+static int
+ottery_read_n_bytes_from_file_(int fd, uint8_t *out, size_t n)
+{
+ ssize_t r;
+ uint8_t *outp = out;
+ while (n) {
+ r = read(fd, outp, n);
+ if (r <= 0 || (size_t)r > n)
+ return outp - out;
+ outp += r;
+ n -= r;
+ }
+ return outp - out;
+}
+
+
+/** Generate random bytes using the unix-style /dev/urandom RNG, or another
+ * such device as configured in the configuration. */
+static int
+ottery_get_entropy_urandom(const struct ottery_entropy_config *cfg,
+ struct ottery_entropy_state *state,
+ uint8_t *out, size_t outlen)
+{
+ /* On most unixes these days, you can get strong random numbers from
+ * /dev/urandom.
+ *
+ * That's assuming that /dev/urandom is seeded. For most applications,
+ * that won't be a problem. But for stuff that starts close to system
+ * startup, before the operating system has added any entropy to the pool,
+ * it can be pretty bad.
+ *
+ * You could use /dev/random instead, if you want, but that has another
+ * problem. It will block if the OS PRNG has received less entropy than
+ * it has emitted. If we assume that the OS PRNG isn't cryptographically
+ * weak, blocking in that case is simple overkill.
+ *
+ * It would be best if there were an alternative that blocked if the PRNG
+ * had _never_ been seeded. But most operating systems don't have that.
+ */
+ int fd;
+ ssize_t n;
+ int result = 0;
+ const char *urandom_fname;
+ struct stat st;
+ int own_fd = 0;
+ int check_device = !cfg || !cfg->allow_nondev_urandom;
+#ifndef O_CLOEXEC
+#define O_CLOEXEC 0
+#endif
+ if (cfg && cfg->urandom_fd_is_set && cfg->urandom_fd >= 0) {
+ fd = cfg->urandom_fd;
+ } else {
+ if (cfg && cfg->urandom_fname)
+ urandom_fname = cfg->urandom_fname;
+ else
+ urandom_fname = "/dev/urandom";
+
+ fd = open(urandom_fname, O_RDONLY|O_CLOEXEC);
+ own_fd = 1;
+ if (fd < 0)
+ return OTTERY_ERR_INIT_STRONG_RNG;
+ }
+ if (fstat(fd, &st) < 0) {
+ result = OTTERY_ERR_INIT_STRONG_RNG;
+ goto end;
+ }
+ if (check_device) {
+ if (0 == (st.st_mode & S_IFCHR)) {
+ result = OTTERY_ERR_INIT_STRONG_RNG;
+ goto end;
+ }
+
+ if (state) {
+ if (0 == state->urandom_fd_inode) {
+ state->urandom_fd_inode = (uint64_t) st.st_ino;
+ } else if ((uint64_t)st.st_ino != state->urandom_fd_inode) {
+ close(fd);
+ return OTTERY_ERR_ACCESS_STRONG_RNG;
+ }
+ }
+ }
+
+ n = ottery_read_n_bytes_from_file_(fd, out, outlen);
+ if (n < 0 || (size_t)n != outlen)
+ result = OTTERY_ERR_ACCESS_STRONG_RNG;
+
+ end:
+ if (own_fd)
+ close(fd);
+ return result;
+}
+
+#define ENTROPY_SOURCE_URANDOM \
+ { ottery_get_entropy_urandom, SRC(RANDOMDEV)|DOM(OS)|FL(STRONG) }
+
+#endif
diff --git a/contrib/libottery/ottery_global.c b/contrib/libottery/ottery_global.c
new file mode 100644
index 0000000..dd1efc5
--- /dev/null
+++ b/contrib/libottery/ottery_global.c
@@ -0,0 +1,116 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#define OTTERY_INTERNAL
+#include <stdlib.h>
+#include "ottery-internal.h"
+#include "ottery.h"
+#include "ottery_st.h"
+
+
+/** Flag: true iff ottery_global_state_ is initialized. */
+static int ottery_global_state_initialized_ = 0;
+int ottery_valgrind_ = 0;
+/** A global state to use for the ottery_* functions that don't take a
+ * state. */
+static struct ottery_state ottery_global_state_;
+
+/** Initialize ottery_global_state_ if it has not been initialize. */
+#define CHECK_INIT(rv) do { \
+ if (UNLIKELY(!ottery_global_state_initialized_)) { \
+ int err; \
+ if ((err = ottery_init(NULL))) { \
+ ottery_fatal_error_(OTTERY_ERR_FLAG_GLOBAL_PRNG_INIT|err); \
+ return rv; \
+ } \
+ } \
+} while (0)
+
+int
+ottery_init(const struct ottery_config *cfg)
+{
+ if (getenv("VALGRIND")) {
+ ottery_valgrind_ = 1;
+ }
+ int n = ottery_st_init(&ottery_global_state_, cfg);
+ if (n == 0)
+ ottery_global_state_initialized_ = 1;
+ return n;
+}
+
+int
+ottery_add_seed(const uint8_t *seed, size_t n)
+{
+ CHECK_INIT(0);
+ return ottery_st_add_seed(&ottery_global_state_, seed, n);
+}
+
+void
+ottery_wipe(void)
+{
+ if (ottery_global_state_initialized_) {
+ ottery_global_state_initialized_ = 0;
+ ottery_st_wipe(&ottery_global_state_);
+ }
+}
+
+void
+ottery_prevent_backtracking(void)
+{
+ CHECK_INIT();
+ ottery_st_prevent_backtracking(&ottery_global_state_);
+}
+
+void
+ottery_rand_bytes(void *out, size_t n)
+{
+ CHECK_INIT();
+ ottery_st_rand_bytes(&ottery_global_state_, out, n);
+}
+
+unsigned
+ottery_rand_unsigned(void)
+{
+ CHECK_INIT(0);
+ return ottery_st_rand_unsigned(&ottery_global_state_);
+}
+uint32_t
+ottery_rand_uint32(void)
+{
+ CHECK_INIT(0);
+ return ottery_st_rand_uint32(&ottery_global_state_);
+}
+uint64_t
+ottery_rand_uint64(void)
+{
+ CHECK_INIT(0);
+ return ottery_st_rand_uint64(&ottery_global_state_);
+}
+unsigned
+ottery_rand_range(unsigned top)
+{
+ CHECK_INIT(0);
+ return ottery_st_rand_range(&ottery_global_state_, top);
+}
+uint64_t
+ottery_rand_range64(uint64_t top)
+{
+ CHECK_INIT(0);
+ return ottery_st_rand_range64(&ottery_global_state_, top);
+}
+
+const char *ottery_get_impl_name(void)
+{
+ CHECK_INIT(0);
+ return ottery_global_state_.prf.name;
+} \ No newline at end of file
diff --git a/contrib/libottery/ottery_nolock.h b/contrib/libottery/ottery_nolock.h
new file mode 100644
index 0000000..b504e11
--- /dev/null
+++ b/contrib/libottery/ottery_nolock.h
@@ -0,0 +1,190 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_NOLOCK_H_HEADER_INCLUDED_
+#define OTTERY_NOLOCK_H_HEADER_INCLUDED_
+#include <stdint.h>
+#include <sys/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "ottery_common.h"
+
+/** @file */
+
+struct ottery_config;
+struct ottery_state_nolock;
+
+/** Size reserved for struct ottery_state_nolock */
+#define OTTERY_STATE_NOLOCK_DUMMY_SIZE_ 1536
+
+#ifndef OTTERY_INTERNAL
+/**
+ * The state for a non-thread-safe libottery PRNG
+ *
+ * Like struct ottery_state, but this structure (and its associated functions)
+ * are not thread safe. If you try to use this structure in more than one
+ * thread at a time, your program's behavior will be undefined. It might
+ * crash. It might insecurely give the same random sequence to multiple
+ * threads. It might fail in strange ways that you'd never predict.
+ *
+ * An ottery_state_nolock structure is constucted with ottery_st_init(). It
+ * MUST be aligned on a 16-byte boundary.
+ *
+ * You may not use an ottery_state_nolock structure with any other function
+ * before you have first initialized it with ottery_st_init_nolock().
+ *
+ * The contents of this structure are opaque; The definition here is
+ * defined to be large enough so that programs that allocate it will get
+ * more than enough room.
+ */
+struct __attribute__((aligned(16))) ottery_state_nolock {
+ /** Nothing to see here */
+ uint8_t dummy_[OTTERY_STATE_NOLOCK_DUMMY_SIZE_];
+};
+#endif
+
+/**
+ * Get the minimal size for allocating an ottery_state_nolock.
+ *
+ * sizeof(ottery_state_nolock) will give an overestimate to allow binary
+ * compatibility with future versions of libottery. Use this function instead
+ * to get the minimal number of bytes to allocate.
+ *
+ * @return The minimal number of bytes to use when allocating an
+ * ottery_state_nolock structure.
+ */
+size_t ottery_get_sizeof_state_nolock(void);
+
+/**
+ * Initialize an ottery_state_nolock structure.
+ *
+ * You must call this function on any ottery_state_nolock structure before
+ * calling any other functions on it.
+ *
+ * @param st The ottery_state_nolock to initialize.
+ * @param cfg Either NULL, or an ottery_config structure that has been
+ * initialized with ottery_config_init().
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_st_init_nolock(struct ottery_state_nolock *st, const struct ottery_config *cfg);
+
+/**
+ * Add more entropy to an ottery_state_nolock structure.
+ *
+ * Calling this function should be needless, if you trust your operating
+ * system's random number generator and entropy extraction features. You
+ * would want to use this function if you think the operating system's random
+ * number generator might be inadequate, and you want to add more entropy from
+ * EGD or something.
+ *
+ * You might also want to call this function if your belief system says that
+ * it's useful to periodically add more raw entropy to a well-seeded
+ * cryptographically strong PRNG.
+ *
+ * @param st The state which will receive more entropy.
+ * @param seed Bytes to add to the state.
+ * @param n The number of bytes to add.
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_st_add_seed_nolock(struct ottery_state_nolock *st, const uint8_t *seed, size_t n);
+
+/**
+ * Destroy an ottery_state_nolock structure and release any resources that it
+ * might hold.
+ *
+ * Ordinarily, you would want to call this at exit, or before freeing an
+ * ottery_state_nolock
+ *
+ * @param st The state to wipe.
+ */
+void ottery_st_wipe_nolock(struct ottery_state_nolock *st);
+
+/**
+ * Explicitly prevent backtracking attacks. (Usually needless).
+ *
+ * Once this function has been called, an attacker who compromises the state
+ * later on will not be able to recover bytes that have previously been
+ * returned by any of the ottery_st_rand_*_nolock functions.
+ *
+ * You should not usually need to call this function: Libottery provides
+ * backtracking resistance by default, so unless you have manually recompiled
+ * with the OTTERY_NO_CLEAR_AFTER_YIELD option, this function isn't
+ * necessary and has no effect. Even *with* OTTERY_NO_CLEAR_AFTER_YIELD,
+ * this function isn't necessary in ordinary operation: the libottery state is
+ * implicitly "stirred" every 1k or so.
+ *
+ * @param st The state to stir.
+ */
+void ottery_st_prevent_backtracking_nolock(struct ottery_state_nolock *st);
+
+/**
+ * Use an ottery_state_nolock structure to fill a buffer with random bytes.
+ *
+ * @param st The state structure to use.
+ * @param buf The buffer to fill.
+ * @param n The number of bytes to write.
+ */
+void ottery_st_rand_bytes_nolock(struct ottery_state_nolock *st, void *buf, size_t n);
+/**
+ * Use an ottery_state_nolock structure to generate a random number of type unsigned.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT_MAX included,
+ * chosen uniformly.
+ */
+unsigned ottery_st_rand_unsigned_nolock(struct ottery_state_nolock *st);
+/**
+ * Use an ottery_state_nolock structure to generate a random number of type uint32_t.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT32_MAX included,
+ * chosen uniformly.
+ */
+uint32_t ottery_st_rand_uint32_nolock(struct ottery_state_nolock *st);
+/**
+ * Use an ottery_state_nolock structure to generate a random number of type uint64_t.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT64_MAX included,
+ * chosen uniformly.
+ */
+uint64_t ottery_st_rand_uint64_nolock(struct ottery_state_nolock *st);
+/**
+ * Use an ottery_state_nolock structure to generate a random number of type unsigned
+ * in a given range.
+ *
+ * @param st The state structure to use.
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+unsigned ottery_st_rand_range_nolock(struct ottery_state_nolock *st, unsigned top);
+/**
+ * Use an ottery_state_nolock structure to generate a random number of type uint64_t
+ * in a given range.
+ *
+ * @param st The state structure to use.
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+uint64_t ottery_st_rand_range64_nolock(struct ottery_state_nolock *st, uint64_t top);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/libottery/ottery_st.h b/contrib/libottery/ottery_st.h
new file mode 100644
index 0000000..38955cb
--- /dev/null
+++ b/contrib/libottery/ottery_st.h
@@ -0,0 +1,184 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_ST_H_HEADER_INCLUDED_
+#define OTTERY_ST_H_HEADER_INCLUDED_
+#include <stdint.h>
+#include <sys/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "ottery_common.h"
+
+/** @file */
+
+struct ottery_config;
+struct ottery_state;
+
+/** Size reserved for struct ottery_state */
+#define OTTERY_STATE_DUMMY_SIZE_ 1536
+
+#ifndef OTTERY_INTERNAL
+/**
+ * The state for a libottery PRNG.
+ *
+ * An ottery_state structure is constucted with ottery_st_init(). It MUST be
+ * aligned on a 16-byte boundary.
+ *
+ * You may not use an ottery_state structure with any other function before
+ * you have first initialized it with ottery_st_init().
+ *
+ * The contents of this structure are opaque; The definition here is
+ * defined to be large enough so that programs that allocate it will get
+ * more than enough room.
+ */
+struct __attribute__((aligned(16))) ottery_state {
+ /** Nothing to see here */
+ uint8_t dummy_[OTTERY_STATE_DUMMY_SIZE_];
+};
+#endif
+
+/**
+ * Get the minimal size for allocating an ottery_state.
+ *
+ * sizeof(ottery_state) will give an overestimate to allow binary
+ * compatibility with future versions of libottery. Use this function instead
+ * to get the minimal number of bytes to allocate.
+ *
+ * @return The minimal number of bytes to use when allocating an
+ * ottery_state structure.
+ */
+size_t ottery_get_sizeof_state(void);
+
+/**
+ * Initialize an ottery_state structure.
+ *
+ * You must call this function on any ottery_state structure before
+ * calling any other functions on it.
+ *
+ * @param st The ottery_state to initialize.
+ * @param cfg Either NULL, or an ottery_config structure that has been
+ * initialized with ottery_config_init().
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_st_init(struct ottery_state *st, const struct ottery_config *cfg);
+
+/**
+ * Add more entropy to an ottery_state structure.
+ *
+ * Calling this function should be needless, if you trust your operating
+ * system's random number generator and entropy extraction features. You
+ * would want to use this function if you think the operating system's random
+ * number generator might be inadequate, and you want to add more entropy from
+ * EGD or something.
+ *
+ * You might also want to call this function if your belief system says that
+ * it's useful to periodically add more raw entropy to a well-seeded
+ * cryptographically strong PRNG.
+ *
+ * @param st The state which will receive more entropy.
+ * @param seed Bytes to add to the state.
+ * @param n The number of bytes to add.
+ * @return Zero on success, or one of the OTTERY_ERR_* error codes on failure.
+ */
+int ottery_st_add_seed(struct ottery_state *st, const uint8_t *seed, size_t n);
+
+/**
+ * Destroy an ottery_state structure and release any resources that it might
+ * hold.
+ *
+ * Ordinarily, you would want to call this at exit, or before freeing an
+ * ottery_state
+ *
+ * @param st The state to wipe.
+ */
+void ottery_st_wipe(struct ottery_state *st);
+
+/**
+ * Explicitly prevent backtracking attacks. (Usually needless).
+ *
+ * Once this function has been called, an attacker who compromises the state
+ * later on will not be able to recover bytes that have previously been
+ * returned by any of the ottery_st_rand_* functions.
+ *
+ * You should not usually need to call this function: Libottery provides
+ * backtracking resistance by default, so unless you have manually recompiled
+ * with the OTTERY_NO_CLEAR_AFTER_YIELD option, this function isn't
+ * necessary and has no effect. Even *with* OTTERY_NO_CLEAR_AFTER_YIELD,
+ * this function isn't necessary in ordinary operation: the libottery state is
+ * implicitly "stirred" every 1k or so.
+ *
+ * @param st The state to stir.
+ */
+void ottery_st_prevent_backtracking(struct ottery_state *st);
+
+/**
+ * Use an ottery_state structure to fill a buffer with random bytes.
+ *
+ * @param st The state structure to use.
+ * @param buf The buffer to fill.
+ * @param n The number of bytes to write.
+ */
+void ottery_st_rand_bytes(struct ottery_state *st, void *buf, size_t n);
+/**
+ * Use an ottery_state structure to generate a random number of type unsigned.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT_MAX included,
+ * chosen uniformly.
+ */
+unsigned ottery_st_rand_unsigned(struct ottery_state *st);
+/**
+ * Use an ottery_state structure to generate a random number of type uint32_t.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT32_MAX included,
+ * chosen uniformly.
+ */
+uint32_t ottery_st_rand_uint32(struct ottery_state *st);
+/**
+ * Use an ottery_state structure to generate a random number of type uint64_t.
+ *
+ * @param st The state structure to use.
+ * @return A random number between 0 and UINT64_MAX included,
+ * chosen uniformly.
+ */
+uint64_t ottery_st_rand_uint64(struct ottery_state *st);
+/**
+ * Use an ottery_state structure to generate a random number of type unsigned
+ * in a given range.
+ *
+ * @param st The state structure to use.
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+unsigned ottery_st_rand_range(struct ottery_state *st, unsigned top);
+/**
+ * Use an ottery_state structure to generate a random number of type uint64_t
+ * in a given range.
+ *
+ * @param st The state structure to use.
+ * @param top The upper bound of the range (inclusive).
+ * @return A random number no larger than top, and no less than 0,
+ * chosen uniformly.
+ */
+uint64_t ottery_st_rand_range64(struct ottery_state *st, uint64_t top);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/libottery/ottery_version.h.in b/contrib/libottery/ottery_version.h.in
new file mode 100644
index 0000000..1b6a244
--- /dev/null
+++ b/contrib/libottery/ottery_version.h.in
@@ -0,0 +1,28 @@
+/* Libottery by Nick Mathewson.
+
+ This software has been dedicated to the public domain under the CC0
+ public domain dedication.
+
+ To the extent possible under law, the person who associated CC0 with
+ libottery has waived all copyright and related or neighboring rights
+ to libottery.
+
+ You should have received a copy of the CC0 legalcode along with this
+ work in doc/cc0.txt. If not, see
+ <http://creativecommons.org/publicdomain/zero/1.0/>.
+ */
+#ifndef OTTERY_VERSION_H_HEADER_INCLUDED_
+#define OTTERY_VERSION_H_HEADER_INCLUDED_
+
+/**
+ * Version number for Libottery. The first three bytes are the major number,
+ * minor number, and patch-level respectively. The final byte is 0 for a
+ * released version, and nonzero otherwise.
+ */
+#undef OTTERY_VERSION
+/**
+ * Human-readable string representing the Libottery version.
+ */
+#undef OTTERY_VERSION_STRING
+
+#endif
diff --git a/contrib/librdns/CMakeLists.txt b/contrib/librdns/CMakeLists.txt
new file mode 100644
index 0000000..a5733e6
--- /dev/null
+++ b/contrib/librdns/CMakeLists.txt
@@ -0,0 +1,11 @@
+SET(LIBRDNSSRC util.c
+ logger.c
+ compression.c
+ punycode.c
+ curve.c
+ parse.c
+ packet.c
+ resolver.c)
+
+ADD_LIBRARY(rdns STATIC ${LIBRDNSSRC})
+SET_TARGET_PROPERTIES(rdns PROPERTIES COMPILE_FLAGS "-DUSE_RSPAMD_CRYPTOBOX") \ No newline at end of file
diff --git a/contrib/librdns/compression.c b/contrib/librdns/compression.c
new file mode 100644
index 0000000..895178e
--- /dev/null
+++ b/contrib/librdns/compression.c
@@ -0,0 +1,168 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "compression.h"
+#include "logger.h"
+#include "contrib/mumhash/mum.h"
+
+#define rdns_compression_hash(n) (mum_hash(n.suffix, n.suffix_len, 0xdeadbeef))
+#define rdns_compression_equal(n1, n2) ((n1).suffix_len == (n2).suffix_len && \
+ (memcmp((n1).suffix, (n2).suffix, (n1).suffix_len) == 0))
+__KHASH_IMPL(rdns_compression_hash, kh_inline, struct rdns_compression_name, char, 0, rdns_compression_hash,
+ rdns_compression_equal);
+
+static struct rdns_compression_name *
+rdns_can_compress (const char *pos, unsigned int len, khash_t(rdns_compression_hash) *comp)
+{
+ struct rdns_compression_name check;
+ khiter_t k;
+
+ if (comp == NULL) {
+ return NULL;
+ }
+
+ check.suffix_len = len;
+ check.suffix = pos;
+ k = kh_get(rdns_compression_hash, comp, check);
+
+ if (k != kh_end(comp)) {
+ return &kh_key(comp, k);
+ }
+
+ return NULL;
+}
+
+static unsigned int
+rdns_calculate_label_len (const char *pos, const char *end)
+{
+ const char *p = pos;
+ unsigned int res = 0;
+
+ while (p != end) {
+ if (*p == '.') {
+ break;
+ }
+ res ++;
+ p ++;
+ }
+ return res;
+}
+
+static void
+rdns_add_compressed (const char *pos, const char *end,
+ khash_t(rdns_compression_hash) *comp,
+ int offset)
+{
+ struct rdns_compression_name new_name;
+ int r;
+
+ if (comp != NULL) {
+
+ assert (offset >= 0);
+ new_name.suffix_len = end - pos;
+ new_name.suffix = pos;
+ new_name.offset = offset;
+
+ kh_put(rdns_compression_hash, comp, new_name, &r);
+ }
+}
+
+void
+rdns_compression_free (khash_t(rdns_compression_hash) *comp)
+{
+ if (comp != NULL) {
+ kh_destroy(rdns_compression_hash, comp);
+ }
+}
+
+bool
+rdns_write_name_compressed (struct rdns_request *req,
+ const char *name, unsigned int namelen,
+ khash_t(rdns_compression_hash) **comp)
+{
+ uint8_t *target = req->packet + req->pos;
+ const char *pos = name, *end = name + namelen;
+ unsigned int remain = req->packet_len - req->pos - 5, label_len;
+ struct rdns_resolver *resolver = req->resolver;
+ uint16_t pointer;
+
+ if (comp != NULL && *comp == NULL) {
+ *comp = kh_init(rdns_compression_hash);
+ }
+
+ while (pos < end && remain > 0) {
+ if (comp) {
+ struct rdns_compression_name *test = rdns_can_compress(pos, end - pos, *comp);
+ if (test != NULL) {
+ /* Can compress name */
+ if (remain < 2) {
+ rdns_info ("no buffer remain for constructing query");
+ return false;
+ }
+
+ pointer = htons ((uint16_t) test->offset) | DNS_COMPRESSION_BITS;
+ memcpy(target, &pointer, sizeof(pointer));
+ req->pos += 2;
+
+ return true;
+ }
+ }
+
+ label_len = rdns_calculate_label_len (pos, end);
+ if (label_len == 0) {
+ /* We have empty label it is allowed only if pos == end - 1 */
+ if (pos == end - 1) {
+ break;
+ }
+ else {
+ rdns_err ("double dots in the name requested");
+ return false;
+ }
+ }
+ else if (label_len > DNS_D_MAXLABEL) {
+ rdns_err ("too large label: %d", (int)label_len);
+ return false;
+ }
+
+ if (label_len + 1 > remain) {
+ rdns_info ("no buffer remain for constructing query, strip %d to %d",
+ (int)label_len, (int)remain);
+ label_len = remain - 1;
+ }
+
+ if (comp) {
+ rdns_add_compressed(pos, end, *comp, target - req->packet);
+ }
+ /* Write label as is */
+ *target++ = (uint8_t)label_len;
+ memcpy (target, pos, label_len);
+ target += label_len;
+ pos += label_len + 1;
+ }
+
+ /* Termination label */
+ *target++ = '\0';
+ req->pos = target - req->packet;
+
+ return true;
+}
diff --git a/contrib/librdns/compression.h b/contrib/librdns/compression.h
new file mode 100644
index 0000000..6056117
--- /dev/null
+++ b/contrib/librdns/compression.h
@@ -0,0 +1,48 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef COMPRESSION_H_
+#define COMPRESSION_H_
+
+#include "dns_private.h"
+#include "khash.h"
+
+struct rdns_compression_name {
+ const char *suffix; /**< suffix packed */
+ unsigned int suffix_len; /**< length of the suffix */
+ unsigned int offset; /**< offset in the packet */
+};
+
+
+KHASH_DECLARE(rdns_compression_hash, struct rdns_compression_name, char);
+
+/**
+ * Try to compress name passed or write it 'as is'
+ * @return
+ */
+bool rdns_write_name_compressed (struct rdns_request *req,
+ const char *name, unsigned int namelen,
+ khash_t(rdns_compression_hash) **comp);
+
+void rdns_compression_free (khash_t(rdns_compression_hash) *comp);
+
+#endif /* COMPRESSION_H_ */
diff --git a/contrib/librdns/curve.c b/contrib/librdns/curve.c
new file mode 100644
index 0000000..19ec250
--- /dev/null
+++ b/contrib/librdns/curve.c
@@ -0,0 +1,890 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include "rdns.h"
+#include "dns_private.h"
+#include "rdns_curve.h"
+#include "ottery.h"
+#include "ref.h"
+#include "logger.h"
+
+#ifdef TWEETNACL
+
+#include <tweetnacl.h>
+
+void
+randombytes(uint8_t *data, uint64_t len)
+{
+ ottery_rand_bytes (data, len);
+}
+void sodium_memzero (uint8_t *data, uint64_t len)
+{
+ volatile uint8_t *p = data;
+
+ while (len--) {
+ *p = '\0';
+ }
+}
+void sodium_init(void)
+{
+
+}
+
+ssize_t rdns_curve_send (struct rdns_request *req, void *plugin_data);
+ssize_t rdns_curve_recv (struct rdns_io_channel *ioc, void *buf, size_t len,
+ void *plugin_data, struct rdns_request **req_out);
+void rdns_curve_finish_request (struct rdns_request *req, void *plugin_data);
+void rdns_curve_dtor (struct rdns_resolver *resolver, void *plugin_data);
+
+struct rdns_curve_entry {
+ char *name;
+ unsigned char pk[crypto_box_PUBLICKEYBYTES];
+ UT_hash_handle hh;
+};
+
+struct rdns_curve_nm_entry {
+ unsigned char k[crypto_box_BEFORENMBYTES];
+ struct rdns_curve_entry *entry;
+ struct rdns_curve_nm_entry *prev, *next;
+};
+
+struct rdns_curve_client_key {
+ unsigned char pk[crypto_box_PUBLICKEYBYTES];
+ unsigned char sk[crypto_box_SECRETKEYBYTES];
+ struct rdns_curve_nm_entry *nms;
+ uint64_t counter;
+ unsigned int uses;
+ ref_entry_t ref;
+};
+
+struct rdns_curve_request {
+ struct rdns_request *req;
+ struct rdns_curve_client_key *key;
+ struct rdns_curve_entry *entry;
+ struct rdns_curve_nm_entry *nm;
+ unsigned char nonce[crypto_box_NONCEBYTES];
+ UT_hash_handle hh;
+};
+
+struct rdns_curve_ctx {
+ struct rdns_curve_entry *entries;
+ struct rdns_curve_client_key *cur_key;
+ struct rdns_curve_request *requests;
+ double key_refresh_interval;
+ void *key_refresh_event;
+ struct rdns_resolver *resolver;
+};
+
+static struct rdns_curve_client_key *
+rdns_curve_client_key_new (struct rdns_curve_ctx *ctx)
+{
+ struct rdns_curve_client_key *new;
+ struct rdns_curve_nm_entry *nm;
+ struct rdns_curve_entry *entry, *tmp;
+
+ new = calloc (1, sizeof (struct rdns_curve_client_key));
+ crypto_box_keypair (new->pk, new->sk);
+
+ HASH_ITER (hh, ctx->entries, entry, tmp) {
+ nm = calloc (1, sizeof (struct rdns_curve_nm_entry));
+ nm->entry = entry;
+ crypto_box_beforenm (nm->k, entry->pk, new->sk);
+
+ DL_APPEND (new->nms, nm);
+ }
+
+ new->counter = ottery_rand_uint64 ();
+
+ return new;
+}
+
+static struct rdns_curve_nm_entry *
+rdns_curve_find_nm (struct rdns_curve_client_key *key, struct rdns_curve_entry *entry)
+{
+ struct rdns_curve_nm_entry *nm;
+
+ DL_FOREACH (key->nms, nm) {
+ if (nm->entry == entry) {
+ return nm;
+ }
+ }
+
+ return NULL;
+}
+
+static void
+rdns_curve_client_key_free (struct rdns_curve_client_key *key)
+{
+ struct rdns_curve_nm_entry *nm, *tmp;
+
+ DL_FOREACH_SAFE (key->nms, nm, tmp) {
+ sodium_memzero (nm->k, sizeof (nm->k));
+ free (nm);
+ }
+ sodium_memzero (key->sk, sizeof (key->sk));
+ free (key);
+}
+
+struct rdns_curve_ctx*
+rdns_curve_ctx_new (double key_refresh_interval)
+{
+ struct rdns_curve_ctx *new;
+
+ new = calloc (1, sizeof (struct rdns_curve_ctx));
+ new->key_refresh_interval = key_refresh_interval;
+
+ return new;
+}
+
+void
+rdns_curve_ctx_add_key (struct rdns_curve_ctx *ctx,
+ const char *name, const unsigned char *pubkey)
+{
+ struct rdns_curve_entry *entry;
+ bool success = true;
+
+ entry = malloc (sizeof (struct rdns_curve_entry));
+ if (entry != NULL) {
+ entry->name = strdup (name);
+ if (entry->name == NULL) {
+ success = false;
+ }
+ memcpy (entry->pk, pubkey, sizeof (entry->pk));
+ if (success) {
+ HASH_ADD_KEYPTR (hh, ctx->entries, entry->name, strlen (entry->name), entry);
+ }
+ }
+}
+
+#define rdns_curve_write_hex(in, out, offset, base) do { \
+ *(out) |= ((in)[(offset)] - (base)) << ((1 - offset) * 4); \
+} while (0)
+
+static bool
+rdns_curve_hex_to_byte (const char *in, unsigned char *out)
+{
+ int i;
+
+ for (i = 0; i <= 1; i ++) {
+ if (in[i] >= '0' && in[i] <= '9') {
+ rdns_curve_write_hex (in, out, i, '0');
+ }
+ else if (in[i] >= 'a' && in[i] <= 'f') {
+ rdns_curve_write_hex (in, out, i, 'a' - 10);
+ }
+ else if (in[i] >= 'A' && in[i] <= 'F') {
+ rdns_curve_write_hex (in, out, i, 'A' - 10);
+ }
+ else {
+ return false;
+ }
+ }
+ return true;
+}
+
+#undef rdns_curve_write_hex
+
+unsigned char *
+rdns_curve_key_from_hex (const char *hex)
+{
+ unsigned int len = strlen (hex), i;
+ unsigned char *res = NULL;
+
+ if (len == crypto_box_PUBLICKEYBYTES * 2) {
+ res = calloc (1, crypto_box_PUBLICKEYBYTES);
+ for (i = 0; i < crypto_box_PUBLICKEYBYTES; i ++) {
+ if (!rdns_curve_hex_to_byte (&hex[i * 2], &res[i])) {
+ free (res);
+ return NULL;
+ }
+ }
+ }
+
+ return res;
+}
+
+void
+rdns_curve_ctx_destroy (struct rdns_curve_ctx *ctx)
+{
+ struct rdns_curve_entry *entry, *tmp;
+
+ HASH_ITER (hh, ctx->entries, entry, tmp) {
+ free (entry->name);
+ free (entry);
+ }
+
+ free (ctx);
+}
+
+static void
+rdns_curve_refresh_key_callback (void *user_data)
+{
+ struct rdns_curve_ctx *ctx = user_data;
+ struct rdns_resolver *resolver;
+
+ resolver = ctx->resolver;
+ rdns_info ("refresh dnscurve keys");
+ REF_RELEASE (ctx->cur_key);
+ ctx->cur_key = rdns_curve_client_key_new (ctx);
+ REF_INIT_RETAIN (ctx->cur_key, rdns_curve_client_key_free);
+}
+
+void
+rdns_curve_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_curve_ctx *ctx)
+{
+ struct rdns_plugin *plugin;
+
+ if (!resolver->async_binded) {
+ return;
+ }
+
+ plugin = calloc (1, sizeof (struct rdns_plugin));
+ if (plugin != NULL) {
+ plugin->data = ctx;
+ plugin->type = RDNS_PLUGIN_CURVE;
+ plugin->cb.curve_plugin.send_cb = rdns_curve_send;
+ plugin->cb.curve_plugin.recv_cb = rdns_curve_recv;
+ plugin->cb.curve_plugin.finish_cb = rdns_curve_finish_request;
+ plugin->dtor = rdns_curve_dtor;
+ sodium_init ();
+ ctx->cur_key = rdns_curve_client_key_new (ctx);
+ REF_INIT_RETAIN (ctx->cur_key, rdns_curve_client_key_free);
+
+ if (ctx->key_refresh_interval > 0) {
+ ctx->key_refresh_event = resolver->async->add_periodic (
+ resolver->async->data, ctx->key_refresh_interval,
+ rdns_curve_refresh_key_callback, ctx);
+ }
+ ctx->resolver = resolver;
+ rdns_resolver_register_plugin (resolver, plugin);
+ }
+}
+
+ssize_t
+rdns_curve_send (struct rdns_request *req, void *plugin_data)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ struct rdns_curve_entry *entry;
+ struct iovec iov[4];
+ unsigned char *m;
+ static const char qmagic[] = "Q6fnvWj8";
+ struct rdns_curve_request *creq;
+ struct rdns_curve_nm_entry *nm;
+ ssize_t ret, boxed_len;
+
+ /* Check for key */
+ HASH_FIND_STR (ctx->entries, req->io->srv->name, entry);
+ if (entry != NULL) {
+ nm = rdns_curve_find_nm (ctx->cur_key, entry);
+ creq = malloc (sizeof (struct rdns_curve_request));
+ if (creq == NULL) {
+ return -1;
+ }
+
+ boxed_len = req->pos + crypto_box_ZEROBYTES;
+ m = malloc (boxed_len);
+ if (m == NULL) {
+ return -1;
+ }
+
+ /* Ottery is faster than sodium native PRG that uses /dev/random only */
+ memcpy (creq->nonce, &ctx->cur_key->counter, sizeof (uint64_t));
+ ottery_rand_bytes (creq->nonce + sizeof (uint64_t), 12 - sizeof (uint64_t));
+ sodium_memzero (creq->nonce + 12, crypto_box_NONCEBYTES - 12);
+
+ sodium_memzero (m, crypto_box_ZEROBYTES);
+ memcpy (m + crypto_box_ZEROBYTES, req->packet, req->pos);
+
+ if (crypto_box_afternm (m, m, boxed_len,
+ creq->nonce, nm->k) == -1) {
+ sodium_memzero (m, boxed_len);
+ free (m);
+ return -1;
+ }
+
+ creq->key = ctx->cur_key;
+ REF_RETAIN (ctx->cur_key);
+ creq->entry = entry;
+ creq->req = req;
+ creq->nm = nm;
+ HASH_ADD_KEYPTR (hh, ctx->requests, creq->nonce, 12, creq);
+ req->curve_plugin_data = creq;
+
+ ctx->cur_key->counter ++;
+ ctx->cur_key->uses ++;
+
+ /* Now form a dnscurve packet */
+ iov[0].iov_base = (void *)qmagic;
+ iov[0].iov_len = sizeof (qmagic) - 1;
+ iov[1].iov_base = ctx->cur_key->pk;
+ iov[1].iov_len = sizeof (ctx->cur_key->pk);
+ iov[2].iov_base = creq->nonce;
+ iov[2].iov_len = 12;
+ iov[3].iov_base = m + crypto_box_BOXZEROBYTES;
+ iov[3].iov_len = boxed_len - crypto_box_BOXZEROBYTES;
+
+ ret = writev (req->io->sock, iov, sizeof (iov) / sizeof (iov[0]));
+ sodium_memzero (m, boxed_len);
+ free (m);
+ }
+ else {
+ ret = write (req->io->sock, req->packet, req->pos);
+ req->curve_plugin_data = NULL;
+ }
+
+ return ret;
+}
+
+ssize_t
+rdns_curve_recv (struct rdns_io_channel *ioc, void *buf, size_t len, void *plugin_data,
+ struct rdns_request **req_out)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ ssize_t ret, boxlen;
+ static const char rmagic[] = "R6fnvWJ8";
+ unsigned char *p, *box;
+ unsigned char enonce[crypto_box_NONCEBYTES];
+ struct rdns_curve_request *creq;
+ struct rdns_resolver *resolver;
+
+ resolver = ctx->resolver;
+ ret = read (ioc->sock, buf, len);
+
+ if (ret <= 0 || ret < 64) {
+ /* Definitely not a DNSCurve packet */
+ return ret;
+ }
+
+ if (memcmp (buf, rmagic, sizeof (rmagic) - 1) == 0) {
+ /* Likely DNSCurve packet */
+ p = ((unsigned char *)buf) + 8;
+ HASH_FIND (hh, ctx->requests, p, 12, creq);
+ if (creq == NULL) {
+ rdns_info ("unable to find nonce in the internal hash");
+ return ret;
+ }
+ memcpy (enonce, p, crypto_box_NONCEBYTES);
+ p += crypto_box_NONCEBYTES;
+ boxlen = ret - crypto_box_NONCEBYTES +
+ crypto_box_BOXZEROBYTES -
+ sizeof (rmagic) + 1;
+ if (boxlen < 0) {
+ return ret;
+ }
+ box = malloc (boxlen);
+ sodium_memzero (box, crypto_box_BOXZEROBYTES);
+ memcpy (box + crypto_box_BOXZEROBYTES, p,
+ boxlen - crypto_box_BOXZEROBYTES);
+
+ if (crypto_box_open_afternm (box, box, boxlen, enonce, creq->nm->k) != -1) {
+ memcpy (buf, box + crypto_box_ZEROBYTES,
+ boxlen - crypto_box_ZEROBYTES);
+ ret = boxlen - crypto_box_ZEROBYTES;
+ *req_out = creq->req;
+ }
+ else {
+ rdns_info ("unable open cryptobox of size %d", (int)boxlen);
+ }
+ free (box);
+ }
+
+ return ret;
+}
+
+void
+rdns_curve_finish_request (struct rdns_request *req, void *plugin_data)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ struct rdns_curve_request *creq = req->curve_plugin_data;
+
+ if (creq != NULL) {
+ REF_RELEASE (creq->key);
+ HASH_DELETE (hh, ctx->requests, creq);
+ }
+}
+
+void
+rdns_curve_dtor (struct rdns_resolver *resolver, void *plugin_data)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+
+ if (ctx->key_refresh_event != NULL) {
+ resolver->async->del_periodic (resolver->async->data,
+ ctx->key_refresh_event);
+ }
+ REF_RELEASE (ctx->cur_key);
+}
+#elif defined(USE_RSPAMD_CRYPTOBOX)
+
+#include "cryptobox.h"
+
+
+#ifndef crypto_box_ZEROBYTES
+#define crypto_box_ZEROBYTES 32
+#endif
+#ifndef crypto_box_BOXZEROBYTES
+#define crypto_box_BOXZEROBYTES 16
+#endif
+
+ssize_t rdns_curve_send (struct rdns_request *req, void *plugin_data,
+ struct sockaddr *saddr, socklen_t slen);
+ssize_t rdns_curve_recv (struct rdns_io_channel *ioc, void *buf, size_t len,
+ void *plugin_data, struct rdns_request **req_out,
+ struct sockaddr *saddr, socklen_t slen);
+void rdns_curve_finish_request (struct rdns_request *req, void *plugin_data);
+void rdns_curve_dtor (struct rdns_resolver *resolver, void *plugin_data);
+
+struct rdns_curve_entry {
+ char *name;
+ rspamd_pk_t pk;
+ UT_hash_handle hh;
+};
+
+struct rdns_curve_nm_entry {
+ rspamd_nm_t k;
+ struct rdns_curve_entry *entry;
+ struct rdns_curve_nm_entry *prev, *next;
+};
+
+struct rdns_curve_client_key {
+ rspamd_pk_t pk;
+ rspamd_sk_t sk;
+ struct rdns_curve_nm_entry *nms;
+ uint64_t counter;
+ unsigned int uses;
+ ref_entry_t ref;
+};
+
+struct rdns_curve_request {
+ struct rdns_request *req;
+ struct rdns_curve_client_key *key;
+ struct rdns_curve_entry *entry;
+ struct rdns_curve_nm_entry *nm;
+ rspamd_nonce_t nonce;
+ UT_hash_handle hh;
+};
+
+struct rdns_curve_ctx {
+ struct rdns_curve_entry *entries;
+ struct rdns_curve_client_key *cur_key;
+ struct rdns_curve_request *requests;
+ double key_refresh_interval;
+ void *key_refresh_event;
+ struct rdns_resolver *resolver;
+};
+
+static struct rdns_curve_client_key *
+rdns_curve_client_key_new (struct rdns_curve_ctx *ctx)
+{
+ struct rdns_curve_client_key *new;
+ struct rdns_curve_nm_entry *nm;
+ struct rdns_curve_entry *entry, *tmp;
+
+ new = calloc (1, sizeof (struct rdns_curve_client_key));
+ rspamd_cryptobox_keypair (new->pk, new->sk, RSPAMD_CRYPTOBOX_MODE_25519);
+
+ HASH_ITER (hh, ctx->entries, entry, tmp) {
+ nm = calloc (1, sizeof (struct rdns_curve_nm_entry));
+ nm->entry = entry;
+ rspamd_cryptobox_nm (nm->k, entry->pk, new->sk,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+
+ DL_APPEND (new->nms, nm);
+ }
+
+ new->counter = ottery_rand_uint64 ();
+
+ return new;
+}
+
+static struct rdns_curve_nm_entry *
+rdns_curve_find_nm (struct rdns_curve_client_key *key, struct rdns_curve_entry *entry)
+{
+ struct rdns_curve_nm_entry *nm;
+
+ DL_FOREACH (key->nms, nm) {
+ if (nm->entry == entry) {
+ return nm;
+ }
+ }
+
+ return NULL;
+}
+
+static void
+rdns_curve_client_key_free (struct rdns_curve_client_key *key)
+{
+ struct rdns_curve_nm_entry *nm, *tmp;
+
+ DL_FOREACH_SAFE (key->nms, nm, tmp) {
+ rspamd_explicit_memzero (nm->k, sizeof (nm->k));
+ free (nm);
+ }
+
+ rspamd_explicit_memzero (key->sk, sizeof (key->sk));
+ free (key);
+}
+
+struct rdns_curve_ctx*
+rdns_curve_ctx_new (double key_refresh_interval)
+{
+ struct rdns_curve_ctx *new;
+
+ new = calloc (1, sizeof (struct rdns_curve_ctx));
+ new->key_refresh_interval = key_refresh_interval;
+
+ return new;
+}
+
+void
+rdns_curve_ctx_add_key (struct rdns_curve_ctx *ctx,
+ const char *name, const unsigned char *pubkey)
+{
+ struct rdns_curve_entry *entry;
+ bool success = true;
+
+ entry = malloc (sizeof (struct rdns_curve_entry));
+ if (entry != NULL) {
+ entry->name = strdup (name);
+ if (entry->name == NULL) {
+ success = false;
+ }
+ memcpy (entry->pk, pubkey, sizeof (entry->pk));
+ if (success) {
+ HASH_ADD_KEYPTR (hh, ctx->entries, entry->name, strlen (entry->name), entry);
+ }
+ }
+}
+
+#define rdns_curve_write_hex(in, out, offset, base) do { \
+ *(out) |= ((in)[(offset)] - (base)) << ((1 - offset) * 4); \
+} while (0)
+
+static bool
+rdns_curve_hex_to_byte (const char *in, unsigned char *out)
+{
+ int i;
+
+ for (i = 0; i <= 1; i ++) {
+ if (in[i] >= '0' && in[i] <= '9') {
+ rdns_curve_write_hex (in, out, i, '0');
+ }
+ else if (in[i] >= 'a' && in[i] <= 'f') {
+ rdns_curve_write_hex (in, out, i, 'a' - 10);
+ }
+ else if (in[i] >= 'A' && in[i] <= 'F') {
+ rdns_curve_write_hex (in, out, i, 'A' - 10);
+ }
+ else {
+ return false;
+ }
+ }
+ return true;
+}
+
+#undef rdns_curve_write_hex
+
+unsigned char *
+rdns_curve_key_from_hex (const char *hex)
+{
+ unsigned int len = strlen (hex), i;
+ unsigned char *res = NULL;
+
+ if (len == rspamd_cryptobox_pk_bytes (RSPAMD_CRYPTOBOX_MODE_25519) * 2) {
+ res = calloc (1, rspamd_cryptobox_pk_bytes (RSPAMD_CRYPTOBOX_MODE_25519));
+ for (i = 0;
+ i < rspamd_cryptobox_pk_bytes (RSPAMD_CRYPTOBOX_MODE_25519);
+ i ++) {
+ if (!rdns_curve_hex_to_byte (&hex[i * 2], &res[i])) {
+ free (res);
+ return NULL;
+ }
+ }
+ }
+
+ return res;
+}
+
+void
+rdns_curve_ctx_destroy (struct rdns_curve_ctx *ctx)
+{
+ struct rdns_curve_entry *entry, *tmp;
+
+ HASH_ITER (hh, ctx->entries, entry, tmp) {
+ free (entry->name);
+ free (entry);
+ }
+
+ free (ctx);
+}
+
+static void
+rdns_curve_refresh_key_callback (void *user_data)
+{
+ struct rdns_curve_ctx *ctx = user_data;
+ struct rdns_resolver *resolver;
+
+ resolver = ctx->resolver;
+ rdns_info ("refresh dnscurve keys");
+ REF_RELEASE (ctx->cur_key);
+ ctx->cur_key = rdns_curve_client_key_new (ctx);
+ REF_INIT_RETAIN (ctx->cur_key, rdns_curve_client_key_free);
+}
+
+void
+rdns_curve_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_curve_ctx *ctx)
+{
+ struct rdns_plugin *plugin;
+
+ if (!resolver->async_binded) {
+ return;
+ }
+
+ plugin = calloc (1, sizeof (struct rdns_plugin));
+ if (plugin != NULL) {
+ plugin->data = ctx;
+ plugin->type = RDNS_PLUGIN_CURVE;
+ plugin->cb.curve_plugin.send_cb = rdns_curve_send;
+ plugin->cb.curve_plugin.recv_cb = rdns_curve_recv;
+ plugin->cb.curve_plugin.finish_cb = rdns_curve_finish_request;
+ plugin->dtor = rdns_curve_dtor;
+ ctx->cur_key = rdns_curve_client_key_new (ctx);
+ REF_INIT_RETAIN (ctx->cur_key, rdns_curve_client_key_free);
+
+ if (ctx->key_refresh_interval > 0) {
+ ctx->key_refresh_event = resolver->async->add_periodic (
+ resolver->async->data, ctx->key_refresh_interval,
+ rdns_curve_refresh_key_callback, ctx);
+ }
+ ctx->resolver = resolver;
+ rdns_resolver_register_plugin (resolver, plugin);
+ }
+}
+
+ssize_t
+rdns_curve_send (struct rdns_request *req, void *plugin_data,
+ struct sockaddr *saddr, socklen_t slen)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ struct rdns_curve_entry *entry;
+ struct iovec iov[4];
+ unsigned char *m;
+ static const char qmagic[] = "Q6fnvWj8";
+ struct rdns_curve_request *creq;
+ struct rdns_curve_nm_entry *nm;
+ ssize_t ret, boxed_len;
+
+ /* Check for key */
+ HASH_FIND_STR (ctx->entries, req->io->srv->name, entry);
+ if (entry != NULL) {
+ nm = rdns_curve_find_nm (ctx->cur_key, entry);
+ creq = malloc (sizeof (struct rdns_curve_request));
+ if (creq == NULL) {
+ return -1;
+ }
+
+ boxed_len = req->pos + crypto_box_ZEROBYTES;
+ m = malloc (boxed_len);
+ if (m == NULL) {
+ free(creq);
+ return -1;
+ }
+
+ /* Ottery is faster than sodium native PRG that uses /dev/random only */
+ memcpy (creq->nonce, &ctx->cur_key->counter, sizeof (uint64_t));
+ ottery_rand_bytes (creq->nonce + sizeof (uint64_t), 12 - sizeof (uint64_t));
+ rspamd_explicit_memzero (creq->nonce + 12,
+ rspamd_cryptobox_nonce_bytes (RSPAMD_CRYPTOBOX_MODE_25519) - 12);
+
+ rspamd_explicit_memzero (m, crypto_box_ZEROBYTES);
+ memcpy (m + crypto_box_ZEROBYTES, req->packet, req->pos);
+
+ rspamd_cryptobox_encrypt_nm_inplace (m + crypto_box_ZEROBYTES,
+ boxed_len,
+ creq->nonce,
+ nm->k,
+ m,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+
+ creq->key = ctx->cur_key;
+ REF_RETAIN (ctx->cur_key);
+ creq->entry = entry;
+ creq->req = req;
+ creq->nm = nm;
+ HASH_ADD_KEYPTR (hh, ctx->requests, creq->nonce, 12, creq);
+ req->curve_plugin_data = creq;
+
+ ctx->cur_key->counter ++;
+ ctx->cur_key->uses ++;
+
+ /* Now form a dnscurve packet */
+ iov[0].iov_base = (void *)qmagic;
+ iov[0].iov_len = sizeof (qmagic) - 1;
+ iov[1].iov_base = ctx->cur_key->pk;
+ iov[1].iov_len = sizeof (ctx->cur_key->pk);
+ iov[2].iov_base = creq->nonce;
+ iov[2].iov_len = 12;
+ iov[3].iov_base = m + crypto_box_BOXZEROBYTES;
+ iov[3].iov_len = boxed_len - crypto_box_BOXZEROBYTES;
+
+ struct msghdr msg;
+
+ memset (&msg, 0, sizeof (msg));
+ msg.msg_namelen = slen;
+ msg.msg_name = saddr;
+ msg.msg_iov = iov;
+ msg.msg_iovlen = sizeof (iov) / sizeof (iov[0]);
+ ret = sendmsg (req->io->sock, &msg, 0);
+ rspamd_explicit_memzero (m, boxed_len);
+ free (m);
+ }
+ else {
+ ret = sendto (req->io->sock, req->packet, req->pos, 0, saddr, slen);
+ req->curve_plugin_data = NULL;
+ }
+
+ return ret;
+}
+
+ssize_t
+rdns_curve_recv (struct rdns_io_channel *ioc, void *buf, size_t len, void *plugin_data,
+ struct rdns_request **req_out, struct sockaddr *saddr, socklen_t slen)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ ssize_t ret, boxlen;
+ static const char rmagic[] = "R6fnvWJ8";
+ unsigned char *p, *box;
+ unsigned char enonce[24];
+ struct rdns_curve_request *creq;
+ struct rdns_resolver *resolver;
+
+ resolver = ctx->resolver;
+ ret = recv (ioc->sock, buf, len, 0);
+
+ if (ret <= 0 || ret < 64) {
+ /* Definitely not a DNSCurve packet */
+ return ret;
+ }
+
+ if (memcmp (buf, rmagic, sizeof (rmagic) - 1) == 0) {
+ /* Likely DNSCurve packet */
+ p = ((unsigned char *)buf) + 8;
+ HASH_FIND (hh, ctx->requests, p, 12, creq);
+ if (creq == NULL) {
+ rdns_info ("unable to find nonce in the internal hash");
+ return ret;
+ }
+
+ memcpy (enonce, p, rspamd_cryptobox_nonce_bytes (RSPAMD_CRYPTOBOX_MODE_25519));
+ p += rspamd_cryptobox_nonce_bytes (RSPAMD_CRYPTOBOX_MODE_25519);
+ boxlen = ret - rspamd_cryptobox_nonce_bytes (RSPAMD_CRYPTOBOX_MODE_25519) +
+ crypto_box_BOXZEROBYTES -
+ sizeof (rmagic) + 1;
+ if (boxlen < 0) {
+ return ret;
+ }
+
+ box = malloc (boxlen);
+ rspamd_explicit_memzero (box, crypto_box_BOXZEROBYTES);
+ memcpy (box + crypto_box_BOXZEROBYTES, p,
+ boxlen - crypto_box_BOXZEROBYTES);
+
+ if (!rspamd_cryptobox_decrypt_nm_inplace (
+ box + rspamd_cryptobox_mac_bytes (RSPAMD_CRYPTOBOX_MODE_25519),
+ boxlen - rspamd_cryptobox_mac_bytes (RSPAMD_CRYPTOBOX_MODE_25519),
+ enonce, creq->nm->k, box, RSPAMD_CRYPTOBOX_MODE_25519)) {
+ memcpy (buf, box + crypto_box_ZEROBYTES,
+ boxlen - crypto_box_ZEROBYTES);
+ ret = boxlen - crypto_box_ZEROBYTES;
+ *req_out = creq->req;
+ }
+ else {
+ rdns_info ("unable open cryptobox of size %d", (int)boxlen);
+ }
+
+ free (box);
+ }
+
+ return ret;
+}
+
+void
+rdns_curve_finish_request (struct rdns_request *req, void *plugin_data)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+ struct rdns_curve_request *creq = req->curve_plugin_data;
+
+ if (creq != NULL) {
+ REF_RELEASE (creq->key);
+ HASH_DELETE (hh, ctx->requests, creq);
+ }
+}
+
+void
+rdns_curve_dtor (struct rdns_resolver *resolver, void *plugin_data)
+{
+ struct rdns_curve_ctx *ctx = (struct rdns_curve_ctx *)plugin_data;
+
+ if (ctx->key_refresh_event != NULL) {
+ resolver->async->del_periodic (resolver->async->data,
+ ctx->key_refresh_event);
+ }
+ REF_RELEASE (ctx->cur_key);
+}
+#else
+
+/* Fake functions */
+struct rdns_curve_ctx* rdns_curve_ctx_new (double rekey_interval)
+{
+ return NULL;
+}
+void rdns_curve_ctx_add_key (struct rdns_curve_ctx *ctx,
+ const char *name, const unsigned char *pubkey)
+{
+
+}
+void rdns_curve_ctx_destroy (struct rdns_curve_ctx *ctx)
+{
+
+}
+void rdns_curve_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_curve_ctx *ctx)
+{
+
+}
+
+unsigned char *
+rdns_curve_key_from_hex (const char *hex)
+{
+ return NULL;
+}
+#endif
diff --git a/contrib/librdns/dns_private.h b/contrib/librdns/dns_private.h
new file mode 100644
index 0000000..c240deb
--- /dev/null
+++ b/contrib/librdns/dns_private.h
@@ -0,0 +1,338 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef DNS_PRIVATE_H_
+#define DNS_PRIVATE_H_
+
+#include "config.h"
+#include "uthash.h"
+#include "utlist.h"
+#include "khash.h"
+#include "rdns.h"
+#include "upstream.h"
+#include "ref.h"
+
+static const int dns_port = 53;
+static const int default_io_cnt = 8;
+static const int default_tcp_io_cnt = 1;
+
+#define UDP_PACKET_SIZE (4096)
+
+#define DNS_COMPRESSION_BITS 0xC0
+
+#define DNS_D_MAXLABEL 63 /* + 1 '\0' */
+#define DNS_D_MAXNAME 255 /* + 1 '\0' */
+
+#define RESOLV_CONF "/etc/resolv.conf"
+
+struct dns_header {
+ unsigned int qid :16;
+
+#if BYTE_ORDER == BIG_ENDIAN
+ unsigned int qr:1;
+ unsigned int opcode:4;
+ unsigned int aa:1;
+ unsigned int tc:1;
+ unsigned int rd:1;
+
+ unsigned int ra:1;
+ unsigned int cd : 1;
+ unsigned int ad : 1;
+ unsigned int z : 1;
+ unsigned int rcode:4;
+#else
+ unsigned int rd :1;
+ unsigned int tc :1;
+ unsigned int aa :1;
+ unsigned int opcode :4;
+ unsigned int qr :1;
+
+ unsigned int rcode :4;
+ unsigned int z : 1;
+ unsigned int ad : 1;
+ unsigned int cd : 1;
+ unsigned int ra :1;
+#endif
+
+ unsigned int qdcount :16;
+ unsigned int ancount :16;
+ unsigned int nscount :16;
+ unsigned int arcount :16;
+};
+
+/**
+ * Represents DNS server
+ */
+struct rdns_server {
+ char *name;
+ unsigned int port;
+ unsigned int io_cnt;
+ unsigned int tcp_io_cnt;
+
+ struct rdns_io_channel **io_channels;
+ struct rdns_io_channel **tcp_io_channels;
+ void *ups_elt;
+ upstream_entry_t up;
+};
+
+enum rdns_request_state {
+ RDNS_REQUEST_NEW = 0,
+ RDNS_REQUEST_REGISTERED = 1,
+ RDNS_REQUEST_WAIT_SEND,
+ RDNS_REQUEST_WAIT_REPLY,
+ RDNS_REQUEST_REPLIED,
+ RDNS_REQUEST_FAKE,
+ RDNS_REQUEST_ERROR,
+ RDNS_REQUEST_TCP,
+};
+
+struct rdns_request {
+ struct rdns_resolver *resolver;
+ struct rdns_async_context *async;
+ struct rdns_io_channel *io;
+ struct rdns_reply *reply;
+ enum rdns_request_type type;
+
+ double timeout;
+ unsigned int retransmits;
+
+ int id;
+ struct rdns_request_name *requested_names;
+ unsigned int qcount;
+ enum rdns_request_state state;
+
+ uint8_t *packet;
+ off_t pos;
+ unsigned int packet_len;
+
+ dns_callback_type func;
+ void *arg;
+
+ void *async_event;
+
+#if defined(TWEETNACL) || defined(USE_RSPAMD_CRYPTOBOX)
+ void *curve_plugin_data;
+#endif
+
+ ref_entry_t ref;
+};
+
+
+enum rdns_io_channel_flags {
+ RDNS_CHANNEL_CONNECTED = 1u << 0u,
+ RDNS_CHANNEL_ACTIVE = 1u << 1u,
+ RDNS_CHANNEL_TCP = 1u << 2u,
+ RDNS_CHANNEL_TCP_CONNECTING = 1u << 3u,
+};
+
+#define IS_CHANNEL_CONNECTED(ioc) (((ioc)->flags & RDNS_CHANNEL_CONNECTED) != 0)
+#define IS_CHANNEL_ACTIVE(ioc) (((ioc)->flags & RDNS_CHANNEL_ACTIVE) != 0)
+#define IS_CHANNEL_TCP(ioc) (((ioc)->flags & RDNS_CHANNEL_TCP) != 0)
+
+/**
+ * Used to chain output DNS requests for a TCP connection
+ */
+struct rdns_tcp_output_chain {
+ uint16_t next_write_size; /* Network byte order! */
+ uint16_t cur_write; /* Cur bytes written including `next_write_size` */
+ unsigned char *write_buf;
+ struct rdns_tcp_output_chain *prev, *next;
+};
+
+/**
+ * Specific stuff for a TCP IO chain
+ */
+struct rdns_tcp_channel {
+ uint16_t next_read_size; /* Network byte order on read, then host byte order */
+ uint16_t cur_read; /* Cur bytes read including `next_read_size` */
+ unsigned char *cur_read_buf;
+ unsigned read_buf_allocated;
+
+ /* Chained set of the planned writes */
+ struct rdns_tcp_output_chain *output_chain;
+ unsigned cur_output_chains;
+
+ void *async_read; /** async read event */
+ void *async_write; /** async write event */
+};
+
+KHASH_DECLARE(rdns_requests_hash, int, struct rdns_request *);
+#define RDNS_IO_CHANNEL_TAG UINT64_C(0xe190a5ba12f094c8)
+/**
+ * IO channel for a specific DNS server
+ */
+struct rdns_io_channel {
+ uint64_t struct_magic; /**< tag for this structure */
+ struct rdns_server *srv;
+ struct rdns_resolver *resolver;
+ struct sockaddr *saddr;
+ socklen_t slen;
+ int sock; /**< persistent socket */
+ int flags; /**< see enum rdns_io_channel_flags */
+ void *async_io; /** async opaque ptr */
+ khash_t(rdns_requests_hash) *requests;
+ /*
+ * For DNS replies parsing we use per-channel structure
+ * which is used for two purposes:
+ * 1) We read the next DNS header
+ * 2) We find the corresponding request (if any)
+ * 3) We read the remaining packet (associated with a request or dangling)
+ * This structure is filled on each read-readiness for an IO channel
+ */
+ struct rdns_tcp_channel *tcp;
+ uint64_t uses;
+ ref_entry_t ref;
+};
+
+struct rdns_fake_reply_idx {
+ enum rdns_request_type type;
+ unsigned len;
+ char request[0];
+};
+
+struct rdns_fake_reply {
+ enum dns_rcode rcode;
+ struct rdns_reply_entry *result;
+ UT_hash_handle hh;
+ struct rdns_fake_reply_idx key;
+};
+
+
+struct rdns_resolver {
+ struct rdns_server *servers;
+ struct rdns_async_context *async; /** async callbacks */
+ void *periodic; /** periodic event for resolver */
+ struct rdns_upstream_context *ups;
+ struct rdns_plugin *curve_plugin;
+ struct rdns_fake_reply *fake_elts;
+
+#ifdef __GNUC__
+ __attribute__((format(printf, 4, 0)))
+#endif
+ rdns_log_function logger;
+ void *log_data;
+ enum rdns_log_level log_level;
+
+ uint64_t max_ioc_uses;
+ void *refresh_ioc_periodic;
+
+ bool async_binded;
+ bool initialized;
+ bool enable_dnssec;
+ int flags;
+ ref_entry_t ref;
+};
+
+struct dns_query;
+
+/* Internal DNS structs */
+
+enum dns_section {
+ DNS_S_QD = 0x01,
+#define DNS_S_QUESTION DNS_S_QD
+
+ DNS_S_AN = 0x02,
+#define DNS_S_ANSWER DNS_S_AN
+
+ DNS_S_NS = 0x04,
+#define DNS_S_AUTHORITY DNS_S_NS
+
+ DNS_S_AR = 0x08,
+#define DNS_S_ADDITIONAL DNS_S_AR
+
+ DNS_S_ALL = 0x0f
+};
+/* enum dns_section */
+
+enum dns_opcode {
+ DNS_OP_QUERY = 0,
+ DNS_OP_IQUERY = 1,
+ DNS_OP_STATUS = 2,
+ DNS_OP_NOTIFY = 4,
+ DNS_OP_UPDATE = 5,
+};
+/* dns_opcode */
+
+enum dns_class {
+ DNS_C_IN = 1,
+
+ DNS_C_ANY = 255
+};
+/* enum dns_class */
+
+struct dns_query {
+ char *qname;
+ unsigned int qtype :16;
+ unsigned int qclass :16;
+};
+
+enum dns_type {
+ DNS_T_A = RDNS_REQUEST_A,
+ DNS_T_NS = RDNS_REQUEST_NS,
+ DNS_T_CNAME = 5,
+ DNS_T_SOA = RDNS_REQUEST_SOA,
+ DNS_T_PTR = RDNS_REQUEST_PTR,
+ DNS_T_MX = RDNS_REQUEST_MX,
+ DNS_T_TXT = RDNS_REQUEST_TXT,
+ DNS_T_AAAA = RDNS_REQUEST_AAAA,
+ DNS_T_SRV = RDNS_REQUEST_SRV,
+ DNS_T_OPT = 41,
+ DNS_T_SSHFP = 44,
+ DNS_T_TLSA = RDNS_REQUEST_TLSA,
+ DNS_T_SPF = RDNS_REQUEST_SPF,
+ DNS_T_ALL = RDNS_REQUEST_ANY
+};
+/* enum dns_type */
+
+static const char dns_rcodes[][32] = {
+ [RDNS_RC_NOERROR] = "no error",
+ [RDNS_RC_FORMERR] = "query format error",
+ [RDNS_RC_SERVFAIL] = "server fail",
+ [RDNS_RC_NXDOMAIN] = "no records with this name",
+ [RDNS_RC_NOTIMP] = "not implemented",
+ [RDNS_RC_REFUSED] = "query refused",
+ [RDNS_RC_YXDOMAIN] = "YXDOMAIN",
+ [RDNS_RC_YXRRSET] = "YXRRSET",
+ [RDNS_RC_NXRRSET] = "NXRRSET",
+ [RDNS_RC_NOTAUTH] = "not authorized",
+ [RDNS_RC_NOTZONE] = "no such zone",
+ [RDNS_RC_TIMEOUT] = "query timed out",
+ [RDNS_RC_NETERR] = "network error",
+ [RDNS_RC_NOREC] = "requested record is not found"
+};
+
+static const char dns_types[][16] = {
+ [RDNS_REQUEST_A] = "A request",
+ [RDNS_REQUEST_NS] = "NS request",
+ [RDNS_REQUEST_PTR] = "PTR request",
+ [RDNS_REQUEST_MX] = "MX request",
+ [RDNS_REQUEST_TXT] = "TXT request",
+ [RDNS_REQUEST_SRV] = "SRV request",
+ [RDNS_REQUEST_SPF] = "SPF request",
+ [RDNS_REQUEST_AAAA] = "AAAA request",
+ [RDNS_REQUEST_TLSA] = "TLSA request",
+ [RDNS_REQUEST_ANY] = "ANY request"
+};
+
+
+#endif /* DNS_PRIVATE_H_ */
diff --git a/contrib/librdns/logger.c b/contrib/librdns/logger.c
new file mode 100644
index 0000000..c9ed2d9
--- /dev/null
+++ b/contrib/librdns/logger.c
@@ -0,0 +1,53 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include "dns_private.h"
+#include "logger.h"
+
+void
+rdns_logger_internal (void *log_data, enum rdns_log_level level,
+ const char *function, const char *format,
+ va_list args)
+{
+ struct rdns_resolver *resolver = log_data;
+
+ if (level <= resolver->log_level) {
+ fprintf (stderr, "rdns: %s: ", function);
+ vfprintf (stderr, format, args);
+ fprintf (stderr, "\n");
+ }
+}
+
+void rdns_logger_helper (struct rdns_resolver *resolver,
+ enum rdns_log_level level,
+ const char *function, const char *format, ...)
+{
+ va_list va;
+
+ if (resolver->logger != NULL) {
+ va_start (va, format);
+ resolver->logger (resolver->log_data, level, function, format, va);
+ va_end (va);
+ }
+}
diff --git a/contrib/librdns/logger.h b/contrib/librdns/logger.h
new file mode 100644
index 0000000..8072876
--- /dev/null
+++ b/contrib/librdns/logger.h
@@ -0,0 +1,48 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef LOGGER_H_
+#define LOGGER_H_
+
+#include <stdarg.h>
+#include "dns_private.h"
+
+#ifdef __GNUC__
+__attribute__((format(printf, 4, 0)))
+#endif
+void rdns_logger_internal (void *log_data, enum rdns_log_level level,
+ const char *function, const char *format,
+ va_list args);
+
+#ifdef __GNUC__
+__attribute__((format(printf, 4, 5)))
+#endif
+void rdns_logger_helper (struct rdns_resolver *resolver,
+ enum rdns_log_level level,
+ const char *function, const char *format, ...);
+
+#define rdns_err(...) do { rdns_logger_helper (resolver, RDNS_LOG_ERROR, __func__, __VA_ARGS__); } while (0)
+#define rdns_warn(...) do { rdns_logger_helper (resolver, RDNS_LOG_WARNING, __func__, __VA_ARGS__); } while (0)
+#define rdns_info(...) do { rdns_logger_helper (resolver, RDNS_LOG_INFO, __func__, __VA_ARGS__); } while (0)
+#define rdns_debug(...) do { rdns_logger_helper (resolver, RDNS_LOG_DEBUG, __func__, __VA_ARGS__); } while (0)
+
+#endif /* LOGGER_H_ */
diff --git a/contrib/librdns/packet.c b/contrib/librdns/packet.c
new file mode 100644
index 0000000..59a919e
--- /dev/null
+++ b/contrib/librdns/packet.c
@@ -0,0 +1,288 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "rdns.h"
+#include "dns_private.h"
+#include "punycode.h"
+#include "packet.h"
+#include "util.h"
+#include "logger.h"
+#include "compression.h"
+
+void
+rdns_allocate_packet (struct rdns_request* req, unsigned int namelen)
+{
+ namelen += 96 + 2 + 4 + 11; /* EDNS0 RR */
+ req->packet = malloc (namelen);
+ req->pos = 0;
+ req->packet_len = namelen;
+}
+
+
+void
+rdns_make_dns_header (struct rdns_request *req, unsigned int qcount)
+{
+ struct dns_header *header;
+
+ /* Set DNS header values */
+ header = (struct dns_header *)req->packet;
+ memset (header, 0 , sizeof (struct dns_header));
+ header->qid = rdns_permutor_generate_id ();
+ header->rd = 1;
+ header->qdcount = htons (qcount);
+ header->arcount = htons (1);
+ req->pos += sizeof (struct dns_header);
+ req->id = header->qid;
+}
+
+static bool
+rdns_maybe_punycode_label (const uint8_t *begin,
+ uint8_t const **dot, size_t *label_len)
+{
+ bool ret = false;
+ const uint8_t *p = begin;
+
+ *dot = NULL;
+
+ while (*p) {
+ if (*p == '.') {
+ *dot = p;
+ break;
+ }
+ else if ((*p) & 0x80) {
+ ret = true;
+ }
+ p ++;
+ }
+
+ if (label_len) {
+ *label_len = p - begin;
+ }
+
+ return ret;
+}
+
+bool
+rdns_format_dns_name (struct rdns_resolver *resolver, const char *in,
+ size_t inlen,
+ char **out, size_t *outlen)
+{
+ const uint8_t *dot;
+ const uint8_t *p = in, *end = in + inlen;
+ char *o;
+ int labels = 0;
+ size_t label_len, olen, remain;
+ uint32_t *uclabel = NULL;
+ size_t punylabel_len, uclabel_len;
+ char tmp_label[DNS_D_MAXLABEL];
+ bool need_encode = false;
+
+ if (inlen == 0) {
+ inlen = strlen (in);
+ }
+
+ /* Check for non-ascii characters */
+ if (!(resolver->flags & RDNS_RESOLVER_NOIDN)) {
+ while (p != end) {
+ if (*p >= 0x80) {
+ need_encode = true;
+ }
+ else if (*p == '.') {
+ labels++;
+ }
+ p++;
+ }
+ }
+
+ if (!need_encode) {
+ *out = malloc (inlen + 1);
+
+ if (*out == NULL) {
+ return false;
+ }
+
+ o = *out;
+ memcpy (o, in, inlen);
+ o[inlen] = '\0';
+ *outlen = inlen;
+
+ return true;
+ }
+
+ /* We need to encode */
+ p = in;
+ /* We allocate 4 times more memory as we cannot guarantee encoding bounds */
+ olen = inlen * sizeof (int32_t) + 1 + sizeof ("xn--") * labels;
+ *out = malloc (olen + 1);
+
+ if (*out == NULL) {
+ return false;
+ }
+
+ o = *out;
+ remain = olen;
+
+ while (p != end) {
+ /* Check label for unicode characters */
+ if (rdns_maybe_punycode_label (p, &dot, &label_len)) {
+ /* Convert to ucs4 */
+ if (rdns_utf8_to_ucs4 (p, label_len, &uclabel, &uclabel_len) == 0) {
+ punylabel_len = DNS_D_MAXLABEL;
+
+ rdns_punycode_label_toascii (uclabel, uclabel_len,
+ tmp_label, &punylabel_len);
+ if (remain >= punylabel_len + 1) {
+ memcpy (o, tmp_label, punylabel_len);
+ o += punylabel_len;
+ *o++ = '.';
+ remain -= punylabel_len + 1;
+ }
+ else {
+ rdns_info ("no buffer remain for punycoding query");
+ goto err;
+ }
+
+ free (uclabel);
+ uclabel = NULL;
+
+ if (dot) {
+ p = dot + 1;
+ }
+ else {
+ break;
+ }
+ }
+ else {
+ break;
+ }
+ }
+ else {
+ if (dot) {
+ if (label_len > DNS_D_MAXLABEL) {
+ rdns_info ("dns name component is longer than 63 bytes, should be stripped");
+ label_len = DNS_D_MAXLABEL;
+ }
+ if (remain < label_len + 1) {
+ rdns_info ("no buffer remain for punycoding query");
+ goto err;
+ }
+ if (label_len == 0) {
+ /* Two dots in order, skip this */
+ rdns_info ("name contains two or more dots in a row, replace it with one dot");
+ p = dot + 1;
+ continue;
+ }
+ memcpy (o, p, label_len);
+ o += label_len;
+ *o++ = '.';
+ remain -= label_len + 1;
+ p = dot + 1;
+ }
+ else {
+ if (label_len == 0) {
+ /* If name is ended with dot */
+ break;
+ }
+ if (label_len > DNS_D_MAXLABEL) {
+ rdns_info ("dns name component is longer than 63 bytes, should be stripped");
+ label_len = DNS_D_MAXLABEL;
+ }
+ if (remain < label_len + 1) {
+ rdns_info ("no buffer remain for punycoding query");
+ goto err;
+ }
+ memcpy (o, p, label_len);
+ o += label_len;
+ *o++ = '.';
+ remain -= label_len + 1;
+ p = dot + 1;
+ break;
+ }
+ }
+ if (remain == 0) {
+ rdns_info ("no buffer remain for punycoding query");
+ goto err;
+ }
+ }
+
+ *o = '\0';
+
+ *outlen = o - *out;
+
+ return true;
+
+err:
+ free (*out);
+ *out = NULL;
+ free (uclabel);
+
+ return false;
+}
+
+#define U16_TO_WIRE_ADVANCE(val, p8) \
+ *p8++ = (uint8_t)(((uint16_t)(val)) >> 8); \
+ *p8++ = (uint8_t)(((uint16_t)(val)) & 0xFF);
+
+bool
+rdns_add_rr (struct rdns_request *req, const char *name, unsigned int len,
+ enum dns_type type, struct kh_rdns_compression_hash_s **comp)
+{
+ uint8_t *p8;
+
+ if (!rdns_write_name_compressed (req, name, len, comp)) {
+ return false;
+ }
+
+ p8 = (req->packet + req->pos);
+ U16_TO_WIRE_ADVANCE (type, p8);
+ U16_TO_WIRE_ADVANCE (DNS_C_IN, p8);
+ req->pos += sizeof (uint16_t) * 2;
+
+ return true;
+}
+
+bool
+rdns_add_edns0 (struct rdns_request *req)
+{
+ uint8_t *p8;
+
+ p8 = (req->packet + req->pos);
+ *p8++ = '\0'; /* Name is root */
+ U16_TO_WIRE_ADVANCE (DNS_T_OPT, p8);
+ U16_TO_WIRE_ADVANCE (UDP_PACKET_SIZE, p8);
+ U16_TO_WIRE_ADVANCE (0, p8);
+
+ if (req->resolver->enable_dnssec) {
+ *p8++ = 0x80;
+ }
+ else {
+ *p8++ = 0x00;
+ }
+ *p8++ = 0;
+ /* Length */
+ U16_TO_WIRE_ADVANCE (0, p8);
+
+ req->pos += sizeof (uint8_t) + sizeof (uint16_t) * 5;
+
+ return true;
+}
diff --git a/contrib/librdns/packet.h b/contrib/librdns/packet.h
new file mode 100644
index 0000000..44e16a5
--- /dev/null
+++ b/contrib/librdns/packet.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef PACKET_H_
+#define PACKET_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+#include "dns_private.h"
+
+struct kh_rdns_compression_hash_s;
+
+/**
+ * Allocate dns packet suitable to handle up to `namelen` name
+ * @param req request
+ * @param namelen requested name
+ */
+void rdns_allocate_packet (struct rdns_request* req, unsigned int namelen);
+
+/**
+ * Add basic header to the dns packet
+ * @param req
+ */
+void rdns_make_dns_header (struct rdns_request *req, unsigned int qcount);
+
+
+/**
+ * Add a resource record to the DNS packet
+ * @param req request
+ * @param name requested name
+ * @param type type of resource record
+ */
+bool rdns_add_rr (struct rdns_request *req, const char *name, unsigned int len,
+ enum dns_type type, struct kh_rdns_compression_hash_s **comp);
+
+/**
+ * Add EDNS0 section
+ * @param req
+ */
+bool rdns_add_edns0 (struct rdns_request *req);
+
+#endif /* PACKET_H_ */
diff --git a/contrib/librdns/parse.c b/contrib/librdns/parse.c
new file mode 100644
index 0000000..f9025e1
--- /dev/null
+++ b/contrib/librdns/parse.c
@@ -0,0 +1,461 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "rdns.h"
+#include "dns_private.h"
+#include "parse.h"
+#include "logger.h"
+
+static uint8_t *
+rdns_decompress_label (uint8_t *begin, uint16_t *len, uint16_t max)
+{
+ uint16_t offset = (*len);
+
+ if (offset > max) {
+ return NULL;
+ }
+ *len = *(begin + offset);
+ return begin + offset;
+}
+
+#define UNCOMPRESS_DNS_OFFSET(p) (((*(p)) ^ DNS_COMPRESSION_BITS) << 8) + *((p) + 1)
+
+uint8_t *
+rdns_request_reply_cmp (struct rdns_request *req, uint8_t *in, int len)
+{
+ uint8_t *p, *c, *l1, *l2;
+ uint16_t len1, len2;
+ int decompressed = 0;
+ struct rdns_resolver *resolver = req->resolver;
+
+ /* QR format:
+ * labels - len:octets
+ * null label - 0
+ * class - 2 octets
+ * type - 2 octets
+ */
+
+ /* In p we would store current position in reply and in c - position in request */
+ p = in;
+ c = req->packet + req->pos;
+
+ for (;;) {
+ /* Get current label */
+ len1 = *p;
+ len2 = *c;
+ if (p - in > len) {
+ rdns_info ("invalid dns reply");
+ return NULL;
+ }
+ /* This may be compressed, so we need to decompress it */
+ if (len1 & DNS_COMPRESSION_BITS) {
+ len1 = UNCOMPRESS_DNS_OFFSET(p);
+ l1 = rdns_decompress_label (in, &len1, len);
+ if (l1 == NULL) {
+ return NULL;
+ }
+ decompressed ++;
+ l1 ++;
+ p += 2;
+ }
+ else {
+ l1 = ++p;
+ p += len1;
+ }
+ if (len2 & DNS_COMPRESSION_BITS) {
+ len2 = UNCOMPRESS_DNS_OFFSET(c);
+ l2 = rdns_decompress_label (c, &len2, len);
+ if (l2 == NULL) {
+ rdns_info ("invalid DNS pointer, cannot decompress");
+ return NULL;
+ }
+ decompressed ++;
+ l2 ++;
+ c += 2;
+ }
+ else {
+ l2 = ++c;
+ c += len2;
+ }
+ if (len1 != len2) {
+ return NULL;
+ }
+ if (len1 == 0) {
+ break;
+ }
+
+ if (memcmp (l1, l2, len1) != 0) {
+ return NULL;
+ }
+ if (decompressed == 2) {
+ break;
+ }
+ }
+
+ /* p now points to the end of QR section */
+ /* Compare class and type */
+ if (memcmp (p, c, sizeof (uint16_t) * 2) == 0) {
+ req->pos = c - req->packet + sizeof (uint16_t) * 2;
+ return p + sizeof (uint16_t) * 2;
+ }
+ return NULL;
+}
+
+#define MAX_RECURSION_LEVEL 10
+
+bool
+rdns_parse_labels (struct rdns_resolver *resolver,
+ uint8_t *in, char **target, uint8_t **pos, struct rdns_reply *rep,
+ int *remain, bool make_name)
+{
+ uint16_t namelen = 0;
+ uint8_t *p = *pos, *begin = *pos, *l, *t, *end = *pos + *remain, *new_pos = *pos;
+ uint16_t llen;
+ int length = *remain, new_remain = *remain;
+ int ptrs = 0, labels = 0;
+ bool got_compression = false;
+
+ /* First go through labels and calculate name length */
+ while (p - begin < length) {
+ if (ptrs > MAX_RECURSION_LEVEL) {
+ rdns_info ("dns pointers are nested too much");
+ return false;
+ }
+ llen = *p;
+ if (llen == 0) {
+ if (!got_compression) {
+ /* In case of compression we have already decremented the processing position */
+ new_remain -= sizeof (uint8_t);
+ new_pos += sizeof (uint8_t);
+ }
+ break;
+ }
+ else if ((llen & DNS_COMPRESSION_BITS)) {
+ if (end - p > 1) {
+ ptrs ++;
+ llen = UNCOMPRESS_DNS_OFFSET(p);
+ l = rdns_decompress_label (in, &llen, end - in);
+ if (l == NULL) {
+ rdns_info ("invalid DNS pointer");
+ return false;
+ }
+ if (!got_compression) {
+ /* Our label processing is finished actually */
+ new_remain -= sizeof (uint16_t);
+ new_pos += sizeof (uint16_t);
+ got_compression = true;
+ }
+ if (l < in || l > begin + length) {
+ rdns_info ("invalid pointer in DNS packet");
+ return false;
+ }
+ begin = l;
+ length = end - begin;
+ p = l + *l + 1;
+ namelen += *l;
+ labels ++;
+ }
+ else {
+ rdns_info ("DNS packet has incomplete compressed label, input length: %d bytes, remain: %d",
+ *remain, new_remain);
+ return false;
+ }
+ }
+ else {
+ namelen += llen;
+ p += llen + 1;
+ labels ++;
+ if (!got_compression) {
+ new_remain -= llen + 1;
+ new_pos += llen + 1;
+ }
+ }
+ }
+
+ if (!make_name) {
+ goto end;
+ }
+ *target = malloc (namelen + labels + 3);
+ t = (uint8_t *)*target;
+ p = *pos;
+ begin = *pos;
+ length = *remain;
+ /* Now copy labels to name */
+ while (p - begin < length) {
+ llen = *p;
+ if (llen == 0) {
+ break;
+ }
+ else if (llen & DNS_COMPRESSION_BITS) {
+ llen = UNCOMPRESS_DNS_OFFSET(p);
+ l = rdns_decompress_label (in, &llen, end - in);
+
+ if (l == NULL) {
+ goto end;
+ }
+
+ begin = l;
+ length = end - begin;
+ p = l + *l + 1;
+ memcpy (t, l + 1, *l);
+ t += *l;
+ *t ++ = '.';
+ }
+ else {
+ memcpy (t, p + 1, *p);
+ t += *p;
+ *t ++ = '.';
+ p += *p + 1;
+ }
+ }
+ if (t > (uint8_t *)*target) {
+ *(t - 1) = '\0';
+ }
+ else {
+ /* Handle empty labels */
+ **target = '\0';
+ }
+end:
+ *remain = new_remain;
+ *pos = new_pos;
+
+ return true;
+}
+
+#define GET8(x) do {(x) = ((*p)); p += sizeof (uint8_t); *remain -= sizeof (uint8_t); } while(0)
+#define GET16(x) do {(x) = ((*p) << 8) + *(p + 1); p += sizeof (uint16_t); *remain -= sizeof (uint16_t); } while(0)
+#define GET32(x) do {(x) = ((*p) << 24) + ((*(p + 1)) << 16) + ((*(p + 2)) << 8) + *(p + 3); p += sizeof (uint32_t); *remain -= sizeof (uint32_t); } while(0)
+#define SKIP(type) do { p += sizeof(type); *remain -= sizeof(type); } while (0)
+
+int
+rdns_parse_rr (struct rdns_resolver *resolver,
+ uint8_t *in, struct rdns_reply_entry *elt, uint8_t **pos,
+ struct rdns_reply *rep, int *remain)
+{
+ uint8_t *p = *pos, parts;
+ uint16_t type, datalen, txtlen, copied;
+ int32_t ttl;
+ bool parsed = false;
+
+ /* Skip the whole name */
+ if (!rdns_parse_labels (resolver, in, NULL, &p, rep, remain, false)) {
+ rdns_info ("bad RR name");
+ return -1;
+ }
+ if (*remain < (int)sizeof (uint16_t) * 6) {
+ rdns_info ("stripped dns reply: %d bytes remain; domain %s", *remain,
+ rep->requested_name);
+ return -1;
+ }
+ GET16 (type);
+ /* Skip class */
+ SKIP (uint16_t);
+ GET32 (ttl);
+ GET16 (datalen);
+ elt->type = type;
+ /* Now p points to RR data */
+ switch (type) {
+ case DNS_T_A:
+ if (!(datalen & 0x3) && datalen <= *remain) {
+ memcpy (&elt->content.a.addr, p, sizeof (struct in_addr));
+ p += datalen;
+ *remain -= datalen;
+ parsed = true;
+ }
+ else {
+ rdns_info ("corrupted A record; domain: %s", rep->requested_name);
+ return -1;
+ }
+ break;
+ case DNS_T_AAAA:
+ if (datalen == sizeof (struct in6_addr) && datalen <= *remain) {
+ memcpy (&elt->content.aaa.addr, p, sizeof (struct in6_addr));
+ p += datalen;
+ *remain -= datalen;
+ parsed = true;
+ }
+ else {
+ rdns_info ("corrupted AAAA record; domain %s", rep->requested_name);
+ return -1;
+ }
+ break;
+ case DNS_T_PTR:
+ if (! rdns_parse_labels (resolver, in, &elt->content.ptr.name, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in PTR record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ case DNS_T_NS:
+ if (! rdns_parse_labels (resolver, in, &elt->content.ns.name, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in NS record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ case DNS_T_SOA:
+ if (! rdns_parse_labels (resolver, in, &elt->content.soa.mname, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in SOA record; domain %s", rep->requested_name);
+ return -1;
+ }
+ if (! rdns_parse_labels (resolver, in, &elt->content.soa.admin, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in SOA record; domain %s", rep->requested_name);
+ return -1;
+ }
+ if (*remain >= sizeof(int32_t) * 5) {
+ GET32 (elt->content.soa.serial);
+ GET32 (elt->content.soa.refresh);
+ GET32 (elt->content.soa.retry);
+ GET32 (elt->content.soa.expire);
+ GET32 (elt->content.soa.minimum);
+ }
+ else {
+ rdns_info ("invalid data in SOA record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ case DNS_T_MX:
+ GET16 (elt->content.mx.priority);
+ if (! rdns_parse_labels (resolver, in, &elt->content.mx.name, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in MX record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ case DNS_T_TXT:
+ case DNS_T_SPF:
+ if (datalen <= *remain) {
+ elt->content.txt.data = malloc(datalen + 1);
+ if (elt->content.txt.data == NULL) {
+ rdns_err ("failed to allocate %d bytes for TXT record; domain %s",
+ (int) datalen + 1, rep->requested_name);
+ return -1;
+ }
+ /* Now we should compose data from parts */
+ copied = 0;
+ parts = 0;
+ while (copied + parts < datalen && *remain > 0) {
+ txtlen = *p;
+ if (txtlen + copied + parts <= datalen && *remain >= txtlen + 1) {
+ parts++;
+ memcpy (elt->content.txt.data + copied, p + 1, txtlen);
+ copied += txtlen;
+ p += txtlen + 1;
+ *remain -= txtlen + 1;
+ }
+ else {
+
+ if (txtlen + copied + parts > datalen) {
+ /* Incorrect datalen reported ! */
+ rdns_err ("incorrect txtlen (%d) > datalen (%d) reported; domain %s",
+ (txtlen + copied + parts), datalen,
+ rep->requested_name);
+ return -1;
+ }
+
+ /* Reported equal to the actual data copied */
+ break;
+ }
+ }
+ *(elt->content.txt.data + copied) = '\0';
+ parsed = true;
+ elt->type = RDNS_REQUEST_TXT;
+ }
+ else {
+ rdns_info ("stripped data in TXT record (%d bytes available, %d requested); "
+ "domain %s", (int)*remain, (int)datalen, rep->requested_name);
+ return -1;
+ }
+ break;
+ case DNS_T_SRV:
+ if (p - *pos > (int)(*remain - sizeof (uint16_t) * 3)) {
+ rdns_info ("stripped dns reply while reading SRV record; domain %s", rep->requested_name);
+ return -1;
+ }
+ GET16 (elt->content.srv.priority);
+ GET16 (elt->content.srv.weight);
+ GET16 (elt->content.srv.port);
+ if (! rdns_parse_labels (resolver, in, &elt->content.srv.target,
+ &p, rep, remain, true)) {
+ rdns_info ("invalid labels in SRV record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ case DNS_T_TLSA:
+ if (p - *pos > (int)(*remain - sizeof (uint8_t) * 3) || datalen <= 3) {
+ rdns_info ("stripped dns reply while reading TLSA record; domain %s", rep->requested_name);
+ return -1;
+ }
+
+ if (datalen > *remain) {
+ rdns_info ("too large datalen; domain %s", rep->requested_name);
+ return -1;
+ }
+
+ GET8 (elt->content.tlsa.usage);
+ GET8 (elt->content.tlsa.selector);
+ GET8 (elt->content.tlsa.match_type);
+ datalen -= 3;
+
+ elt->content.tlsa.data = malloc (datalen);
+ if (elt->content.tlsa.data == NULL) {
+ rdns_err ("failed to allocate %d bytes for TLSA record; domain %s",
+ (int)datalen + 1, rep->requested_name);
+ return -1;
+ }
+
+ elt->content.tlsa.datalen = datalen;
+ memcpy (elt->content.tlsa.data, p, datalen);
+ p += datalen;
+ *remain -= datalen;
+ parsed = true;
+ break;
+ case DNS_T_CNAME:
+ if (! rdns_parse_labels (resolver, in, &elt->content.cname.name, &p,
+ rep, remain, true)) {
+ rdns_info ("invalid labels in CNAME record; domain %s", rep->requested_name);
+ return -1;
+ }
+ parsed = true;
+ break;
+ default:
+ rdns_info ("unexpected RR type: %d; domain %s", type, rep->requested_name);
+ p += datalen;
+ *remain -= datalen;
+ break;
+ }
+ *pos = p;
+
+ if (parsed) {
+ elt->ttl = ttl;
+ return 1;
+ }
+ return 0;
+}
diff --git a/contrib/librdns/parse.h b/contrib/librdns/parse.h
new file mode 100644
index 0000000..ed8cd70
--- /dev/null
+++ b/contrib/librdns/parse.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef PARSE_H_
+#define PARSE_H_
+
+#include "dns_private.h"
+
+/**
+ * Compare request and reply checking names
+ * @param req request object
+ * @param in incoming packet
+ * @param len length of the incoming packet
+ * @return new position in the incoming packet or NULL if request is not equal to reply
+ */
+uint8_t * rdns_request_reply_cmp (struct rdns_request *req, uint8_t *in, int len);
+
+/**
+ * Parse labels in the packet
+ * @param in incoming packet
+ * @param target target to write the parsed label (out)
+ * @param pos output position in the packet (it/out)
+ * @param rep dns reply
+ * @param remain remaining bytes (in/out)
+ * @param make_name create name or just skip to the next label
+ * @return true if a label has been successfully parsed
+ */
+bool rdns_parse_labels (struct rdns_resolver *resolver,
+ uint8_t *in, char **target,
+ uint8_t **pos, struct rdns_reply *rep,
+ int *remain, bool make_name);
+
+/**
+ * Parse resource record
+ * @param in incoming packet
+ * @param elt new reply entry
+ * @param pos output position in the packet (it/out)
+ * @param rep dns reply
+ * @param remain remaining bytes (in/out)
+ * @return 1 if rr has been parsed, 0 if rr has been skipped and -1 if there was a parsing error
+ */
+int rdns_parse_rr (struct rdns_resolver *resolver,
+ uint8_t *in, struct rdns_reply_entry *elt, uint8_t **pos,
+ struct rdns_reply *rep, int *remain);
+
+#endif /* PARSE_H_ */
diff --git a/contrib/librdns/punycode.c b/contrib/librdns/punycode.c
new file mode 100644
index 0000000..61091b2
--- /dev/null
+++ b/contrib/librdns/punycode.c
@@ -0,0 +1,303 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ * Copyright (c) 2004, 2006, 2007, 2008 Kungliga Tekniska Högskolan
+ * (Royal Institute of Technology, Stockholm, Sweden).
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the Institute nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "dns_private.h"
+static const unsigned event_loop = 36;
+static const unsigned t_min = 1;
+static const unsigned t_max = 26;
+static const unsigned skew = 38;
+static const unsigned damp = 700;
+static const unsigned initial_n = 128;
+static const unsigned initial_bias = 72;
+/* Punycode utility */
+static unsigned int
+digit (unsigned n)
+{
+ static const char ascii[] = "abcdefghijklmnopqrstuvwxyz0123456789";
+ return ascii[n];
+}
+
+static unsigned int
+adapt (unsigned int delta, unsigned int numpoints, int first)
+{
+ unsigned int k;
+
+ if (first) {
+ delta = delta / damp;
+ }
+ else {
+ delta /= 2;
+ }
+ delta += delta / numpoints;
+ k = 0;
+ while (delta > ((event_loop - t_min) * t_max) / 2) {
+ delta /= event_loop - t_min;
+ k += event_loop;
+ }
+ return k + (((event_loop - t_min + 1) * delta) / (delta + skew));
+}
+
+/**
+ * Convert an UCS4 string to a puny-coded DNS label string suitable
+ * when combined with delimiters and other labels for DNS lookup.
+ *
+ * @param in an UCS4 string to convert
+ * @param in_len the length of in.
+ * @param out the resulting puny-coded string. The string is not NULL
+ * terminated.
+ * @param out_len before processing out_len should be the length of
+ * the out variable, after processing it will be the length of the out
+ * string.
+ *
+ * @return returns 0 on success, an wind error code otherwise
+ */
+
+bool
+rdns_punycode_label_toascii (const uint32_t *in, size_t in_len, char *out,
+ size_t *out_len)
+{
+ unsigned int n = initial_n;
+ unsigned int delta = 0;
+ unsigned int bias = initial_bias;
+ unsigned int h = 0;
+ unsigned int b;
+ unsigned int i;
+ unsigned int o = 0;
+ unsigned int m;
+
+ for (i = 0; i < in_len; ++i) {
+ if (in[i] < 0x80) {
+ ++h;
+ if (o >= *out_len) {
+ return false;
+ }
+ out[o++] = in[i];
+ }
+ }
+ b = h;
+ if (b > 0) {
+ if (o >= *out_len) {
+ return false;
+ }
+ out[o++] = 0x2D;
+ }
+ /* is this string punycoded */
+ if (h < in_len) {
+ if (o + 4 >= *out_len) {
+ return false;
+ }
+ memmove (out + 4, out, o);
+ memcpy (out, "xn--", 4);
+ o += 4;
+ }
+
+ while (h < in_len) {
+ m = (unsigned int) -1;
+ for (i = 0; i < in_len; ++i) {
+
+ if (in[i] < m && in[i] >= n) {
+ m = in[i];
+ }
+ }
+ delta += (m - n) * (h + 1);
+ n = m;
+ for (i = 0; i < in_len; ++i) {
+ if (in[i] < n) {
+ ++delta;
+ }
+ else if (in[i] == n) {
+ unsigned int q = delta;
+ unsigned int k;
+ for (k = event_loop;; k += event_loop) {
+ unsigned int t;
+ if (k <= bias) {
+ t = t_min;
+ }
+ else if (k >= bias + t_max) {
+ t = t_max;
+ }
+ else {
+ t = k - bias;
+ }
+ if (q < t) {
+ break;
+ }
+ if (o >= *out_len) {
+ return -1;
+ }
+ out[o++] = digit (t + ((q - t) % (event_loop - t)));
+ q = (q - t) / (event_loop - t);
+ }
+ if (o >= *out_len) {
+ return -1;
+ }
+ out[o++] = digit (q);
+ /* output */
+ bias = adapt (delta, h + 1, h == b);
+ delta = 0;
+ ++h;
+ }
+ }
+ ++delta;
+ ++n;
+ }
+
+ *out_len = o;
+ return true;
+}
+
+static int
+utf8toutf32 (const unsigned char **pp, uint32_t *out, size_t *remain)
+{
+ const unsigned char *p = *pp;
+ unsigned c = *p;
+ size_t reduce;
+
+ if (c & 0x80) {
+ if ((c & 0xE0) == 0xC0 && *remain >= 2) {
+ const unsigned c2 = *++p;
+ reduce = 2;
+ if ((c2 & 0xC0) == 0x80) {
+ *out = ((c & 0x1F) << 6) | (c2 & 0x3F);
+ }
+ else {
+ return -1;
+ }
+ }
+ else if ((c & 0xF0) == 0xE0 && *remain >= 3) {
+ const unsigned c2 = *++p;
+ if ((c2 & 0xC0) == 0x80) {
+ const unsigned c3 = *++p;
+ reduce = 3;
+ if ((c3 & 0xC0) == 0x80) {
+ *out = ((c & 0x0F) << 12) | ((c2 & 0x3F) << 6)
+ | (c3 & 0x3F);
+ }
+ else {
+ return -1;
+ }
+ }
+ else {
+ return -1;
+ }
+ }
+ else if ((c & 0xF8) == 0xF0 && *remain >= 4) {
+ const unsigned c2 = *++p;
+ if ((c2 & 0xC0) == 0x80) {
+ const unsigned c3 = *++p;
+ if ((c3 & 0xC0) == 0x80) {
+ const unsigned c4 = *++p;
+ reduce = 4;
+ if ((c4 & 0xC0) == 0x80) {
+ *out = ((c & 0x07) << 18) | ((c2 & 0x3F) << 12)
+ | ((c3 & 0x3F) << 6) | (c4 & 0x3F);
+ }
+ else {
+ return -1;
+ }
+ }
+ else {
+ return -1;
+ }
+ }
+ else {
+ return -1;
+ }
+ }
+ else {
+ return -1;
+ }
+ }
+ else {
+ *out = c;
+ reduce = 1;
+ }
+
+ *pp = ++p;
+ *remain -= reduce;
+
+ return 0;
+}
+
+/**
+ * Convert an UTF-8 string to an UCS4 string.
+ *
+ * @param in an UTF-8 string to convert.
+ * @param out the resulting UCS4 string
+ * @param out_len before processing out_len should be the length of
+ * the out variable, after processing it will be the length of the out
+ * string.
+ *
+ * @return returns 0 on success, an -1 otherwise
+ * @ingroup wind
+ */
+
+int
+rdns_utf8_to_ucs4 (const char *in, size_t in_len, uint32_t **out, size_t *out_len)
+{
+ const unsigned char *p;
+ size_t remain = in_len, olen = 0;
+ int ret;
+ uint32_t *res;
+
+ p = (const unsigned char *)in;
+ while (remain > 0) {
+ uint32_t u;
+
+ ret = utf8toutf32 (&p, &u, &remain);
+ if (ret != 0) {
+ return ret;
+ }
+
+ olen ++;
+ }
+ res = malloc (olen * sizeof (uint32_t));
+ if (res == NULL) {
+ return -1;
+ }
+
+ p = (const unsigned char *)in;
+ remain = in_len;
+ olen = 0;
+ while (remain > 0) {
+ uint32_t u;
+
+ (void)utf8toutf32 (&p, &u, &remain);
+ res[olen++] = u;
+ }
+
+ *out_len = olen;
+ *out = res;
+ return 0;
+}
diff --git a/contrib/librdns/punycode.h b/contrib/librdns/punycode.h
new file mode 100644
index 0000000..300fb92
--- /dev/null
+++ b/contrib/librdns/punycode.h
@@ -0,0 +1,59 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef PUNYCODE_H_
+#define PUNYCODE_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+
+/**
+ * Convert an UCS4 string to a puny-coded DNS label string suitable
+ * when combined with delimiters and other labels for DNS lookup.
+ *
+ * @param in an UCS4 string to convert
+ * @param in_len the length of in.
+ * @param out the resulting puny-coded string. The string is not NULL
+ * terminated.
+ * @param out_len before processing out_len should be the length of
+ * the out variable, after processing it will be the length of the out
+ * string.
+ *
+ * @return returns 0 on success, an wind error code otherwise
+ */
+bool rdns_punycode_label_toascii (const uint32_t *in, size_t in_len, char *out, size_t *out_len);
+/**
+ * Convert an UTF-8 string to an UCS4 string.
+ *
+ * @param in an UTF-8 string to convert.
+ * @param out the resulting UCS4 string
+ * @param out_len before processing out_len should be the length of
+ * the out variable, after processing it will be the length of the out
+ * string.
+ *
+ * @return returns 0 on success, an -1 otherwise
+ * @ingroup wind
+ */
+
+int rdns_utf8_to_ucs4 (const char *in, size_t in_len, uint32_t **out, size_t *out_len);
+
+#endif /* PUNYCODE_H_ */
diff --git a/contrib/librdns/rdns.h b/contrib/librdns/rdns.h
new file mode 100644
index 0000000..c0da5ed
--- /dev/null
+++ b/contrib/librdns/rdns.h
@@ -0,0 +1,493 @@
+/*
+ * Copyright (c) 2013-2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef RDNS_H
+#define RDNS_H
+
+#include <sys/types.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rdns_reply;
+struct rdns_request;
+struct rdns_io_channel;
+
+typedef void (*dns_callback_type) (struct rdns_reply *reply, void *arg);
+
+enum rdns_request_type {
+ RDNS_REQUEST_INVALID = -1,
+ RDNS_REQUEST_A = 1,
+ RDNS_REQUEST_NS = 2,
+ RDNS_REQUEST_CNAME = 5,
+ RDNS_REQUEST_SOA = 6,
+ RDNS_REQUEST_PTR = 12,
+ RDNS_REQUEST_MX = 15,
+ RDNS_REQUEST_TXT = 16,
+ RDNS_REQUEST_SRV = 33,
+ RDNS_REQUEST_SPF = 99,
+ RDNS_REQUEST_AAAA = 28,
+ RDNS_REQUEST_TLSA = 52,
+ RDNS_REQUEST_ANY = 255
+};
+
+union rdns_reply_element_un {
+ struct {
+ struct in_addr addr;
+ } a;
+ struct {
+ struct in6_addr addr;
+ } aaa;
+ struct {
+ char *name;
+ } ptr;
+ struct {
+ char *name;
+ } ns;
+ struct {
+ char *name;
+ uint16_t priority;
+ } mx;
+ struct {
+ char *data;
+ } txt;
+ struct {
+ uint16_t priority;
+ uint16_t weight;
+ uint16_t port;
+ char *target;
+ } srv;
+ struct {
+ char *mname;
+ char *admin;
+ uint32_t serial;
+ int32_t refresh;
+ int32_t retry;
+ int32_t expire;
+ uint32_t minimum;
+ } soa;
+ struct {
+ uint8_t usage;
+ uint8_t selector;
+ uint8_t match_type;
+ uint16_t datalen;
+ uint8_t *data;
+ } tlsa;
+ struct {
+ char *name;
+ } cname;
+};
+
+struct rdns_reply_entry {
+ union rdns_reply_element_un content;
+ enum rdns_request_type type;
+ int32_t ttl;
+ struct rdns_reply_entry *prev, *next;
+};
+
+
+enum dns_rcode {
+ RDNS_RC_INVALID = -1,
+ RDNS_RC_NOERROR = 0,
+ RDNS_RC_FORMERR = 1,
+ RDNS_RC_SERVFAIL = 2,
+ RDNS_RC_NXDOMAIN = 3,
+ RDNS_RC_NOTIMP = 4,
+ RDNS_RC_REFUSED = 5,
+ RDNS_RC_YXDOMAIN = 6,
+ RDNS_RC_YXRRSET = 7,
+ RDNS_RC_NXRRSET = 8,
+ RDNS_RC_NOTAUTH = 9,
+ RDNS_RC_NOTZONE = 10,
+ RDNS_RC_TIMEOUT = 11,
+ RDNS_RC_NETERR = 12,
+ RDNS_RC_NOREC = 13
+};
+
+enum dns_reply_flags {
+ RDNS_AUTH = (1u << 0u),
+ RDNS_TRUNCATED = (1u << 1u)
+};
+
+struct rdns_reply {
+ struct rdns_request *request;
+ struct rdns_resolver *resolver;
+ struct rdns_reply_entry *entries;
+ const char *requested_name;
+ enum dns_rcode code;
+ uint8_t flags; /* see enum dns_reply_flags */
+};
+
+typedef void (*rdns_periodic_callback)(void *user_data);
+
+struct rdns_async_context {
+ void *data;
+ void* (*add_read)(void *priv_data, int fd, void *user_data);
+ void (*del_read)(void *priv_data, void *ev_data);
+ void* (*add_write)(void *priv_data, int fd, void *user_data);
+ void (*del_write)(void *priv_data, void *ev_data);
+ void* (*add_timer)(void *priv_data, double after, void *user_data);
+ void (*repeat_timer)(void *priv_data, void *ev_data);
+ void (*del_timer)(void *priv_data, void *ev_data);
+ void* (*add_periodic)(void *priv_data, double after,
+ rdns_periodic_callback cb, void *user_data);
+ void (*del_periodic)(void *priv_data, void *ev_data);
+ void (*cleanup)(void *priv_data);
+};
+
+struct rdns_upstream_elt {
+ void *server;
+ void *lib_data;
+};
+
+struct rdns_upstream_context {
+ void *data;
+ struct rdns_upstream_elt* (*select)(const char *name,
+ size_t len, void *ups_data);
+ struct rdns_upstream_elt* (*select_retransmit)(const char *name, size_t len,
+ struct rdns_upstream_elt* prev_elt,
+ void *ups_data);
+ unsigned int (*count)(void *ups_data);
+ void (*ok)(struct rdns_upstream_elt *elt, void *ups_data);
+ void (*fail)(struct rdns_upstream_elt *elt, void *ups_data, const char *reason);
+};
+
+/**
+ * Type of rdns plugin
+ */
+enum rdns_plugin_type {
+ RDNS_PLUGIN_CURVE = 0
+};
+
+typedef ssize_t (*rdns_network_send_callback) (struct rdns_request *req, void *plugin_data,
+ struct sockaddr *saddr, socklen_t slen);
+typedef ssize_t (*rdns_network_recv_callback) (struct rdns_io_channel *ioc, void *buf,
+ size_t len, void *plugin_data,
+ struct rdns_request **req_out,
+ struct sockaddr *saddr, socklen_t slen);
+typedef void (*rdns_network_finish_callback) (struct rdns_request *req, void *plugin_data);
+typedef void (*rdns_plugin_dtor_callback) (struct rdns_resolver *resolver, void *plugin_data);
+
+struct rdns_plugin {
+ enum rdns_plugin_type type;
+ union {
+ struct {
+ rdns_network_send_callback send_cb;
+ rdns_network_recv_callback recv_cb;
+ rdns_network_finish_callback finish_cb;
+ } curve_plugin;
+ } cb;
+ rdns_plugin_dtor_callback dtor;
+ void *data;
+};
+
+/*
+ * RDNS logger types
+ */
+/*
+ * These types are somehow compatible with glib
+ */
+enum rdns_log_level {
+ RDNS_LOG_ERROR = 1 << 3,
+ RDNS_LOG_WARNING = 1 << 4,
+ RDNS_LOG_INFO = 1 << 6,
+ RDNS_LOG_DEBUG = 1 << 7
+};
+typedef void (*rdns_log_function) (
+ void *log_data, //!< opaque data pointer
+ enum rdns_log_level level, //!< level of message
+ const char *function, //!< calling function
+ const char *format, //!< format
+ va_list args //!< set of arguments
+ );
+
+struct rdns_request_name {
+ char *name;
+ enum rdns_request_type type;
+ unsigned int len;
+};
+
+#define MAX_FAKE_NAME 1000
+
+/*
+ * RDNS API
+ */
+
+enum rdns_resolver_flags {
+ RDNS_RESOLVER_DEFAULT,
+ RDNS_RESOLVER_NOIDN = (1u << 0u),
+};
+
+/**
+ * Create DNS resolver structure
+ */
+struct rdns_resolver *rdns_resolver_new (int flags);
+
+/**
+ * Bind resolver to specified async context
+ * @param ctx
+ */
+void rdns_resolver_async_bind (struct rdns_resolver *resolver,
+ struct rdns_async_context *ctx);
+
+/**
+ * Enable stub dnssec resolver
+ * @param resolver
+ */
+void rdns_resolver_set_dnssec (struct rdns_resolver *resolver, bool enabled);
+
+/**
+ * Add new DNS server definition to the resolver
+ * @param resolver resolver object
+ * @param name name of DNS server (should be ipv4 or ipv6 address)
+ * @param priority priority (can be 0 for fair round-robin)
+ * @param io_cnt a number of sockets that are simultaneously opened to this server
+ * @return opaque pointer that could be used to select upstream
+ */
+void* rdns_resolver_add_server (struct rdns_resolver *resolver,
+ const char *name, unsigned int port,
+ int priority, unsigned int io_cnt);
+
+
+/**
+ * Load nameservers definition from resolv.conf file
+ * @param resolver resolver object
+ * @param path path to resolv.conf file (/etc/resolv.conf typically)
+ * @return true if resolv.conf has been parsed
+ */
+bool rdns_resolver_parse_resolv_conf (struct rdns_resolver *resolver,
+ const char *path);
+
+typedef bool (*rdns_resolv_conf_cb) (struct rdns_resolver *resolver,
+ const char *name, unsigned int port,
+ int priority, unsigned int io_cnt, void *ud);
+/**
+ * Parse nameservers calling the specified callback for each nameserver
+ * @param resolve resolver object
+ * @param path path to resolv.conf file (/etc/resolv.conf typically)
+ * @param cb callback to call
+ * @param ud userdata for callback
+ * @return true if resolv.conf has been parsed
+ */
+bool rdns_resolver_parse_resolv_conf_cb (struct rdns_resolver *resolver,
+ const char *path, rdns_resolv_conf_cb cb, void *ud);
+
+/**
+ * Set an external logger function to log messages from the resolver
+ * @param resolver resolver object
+ * @param logger logger callback
+ * @param log_data opaque data
+ */
+void rdns_resolver_set_logger (struct rdns_resolver *resolver,
+ rdns_log_function logger, void *log_data);
+
+/**
+ * Set log level for an internal logger (stderr one)
+ * @param resolver resolver object
+ * @param level desired log level
+ */
+void rdns_resolver_set_log_level (struct rdns_resolver *resolver,
+ enum rdns_log_level level);
+
+/**
+ * Set upstream library for selecting DNS upstreams
+ * @param resolver resolver object
+ * @param ups_ctx upstream functions
+ * @param ups_data opaque data
+ */
+void rdns_resolver_set_upstream_lib (struct rdns_resolver *resolver,
+ struct rdns_upstream_context *ups_ctx,
+ void *ups_data);
+
+/**
+ * Set maximum number of dns requests to be sent to a socket to be refreshed
+ * @param resolver resolver object
+ * @param max_ioc_uses unsigned count of socket usage limit
+ * @param check_time specifies how often to check for sockets and refresh them
+ */
+void rdns_resolver_set_max_io_uses (struct rdns_resolver *resolver,
+ uint64_t max_ioc_uses, double check_time);
+
+/**
+ * Register new plugin for rdns resolver
+ * @param resolver
+ * @param plugin
+ */
+void rdns_resolver_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_plugin *plugin);
+
+/**
+ * Add a fake reply for a specified name
+ * @param resolver
+ * @param type
+ * @param name (must not be larger than MAX_FAKE_NAME)
+ * @param reply
+ */
+void rdns_resolver_set_fake_reply (struct rdns_resolver *resolver,
+ const char *name,
+ enum rdns_request_type type,
+ enum dns_rcode rcode,
+ struct rdns_reply_entry *reply);
+
+/**
+ * Init DNS resolver
+ * @param resolver
+ * @return
+ */
+bool rdns_resolver_init (struct rdns_resolver *resolver);
+
+/**
+ * Decrease refcount for a resolver and free it if refcount is 0
+ * @param resolver
+ */
+void rdns_resolver_release (struct rdns_resolver *resolver);
+
+/**
+ * Make a DNS request
+ * @param resolver resolver object
+ * @param cb callback to call on resolve completing
+ * @param ud user data for callback
+ * @param timeout timeout in seconds
+ * @param repeats how much time to retransmit query
+ * @param queries how much RR queries to send
+ * @param ... -> queries in format: <query_type>[,type_argument[,type_argument...]]
+ * @return opaque request object or NULL
+ */
+struct rdns_request* rdns_make_request_full (
+ struct rdns_resolver *resolver,
+ dns_callback_type cb,
+ void *cbdata,
+ double timeout,
+ unsigned int repeats,
+ unsigned int queries,
+ ...
+ );
+
+/**
+ * Get textual presentation of DNS error code
+ */
+const char *rdns_strerror (enum dns_rcode rcode);
+
+/**
+ * Get textual presentation of DNS request type
+ */
+const char *rdns_strtype (enum rdns_request_type type);
+
+/**
+ * Parse string and return request type
+ * @param str
+ * @return
+ */
+enum rdns_request_type rdns_type_fromstr (const char *str);
+
+/**
+ * Returns string representing request type
+ * @param rcode
+ * @return
+ */
+const char *
+rdns_str_from_type (enum rdns_request_type rcode);
+
+/**
+ * Parse string and return error code
+ * @param str
+ * @return
+ */
+enum dns_rcode rdns_rcode_fromstr (const char *str);
+
+/**
+ * Increase refcount for a request
+ * @param req
+ * @return
+ */
+struct rdns_request* rdns_request_retain (struct rdns_request *req);
+
+/**
+ * Decrease refcount for a request and free it if refcount is 0
+ * @param req
+ */
+void rdns_request_release (struct rdns_request *req);
+
+/**
+ * Check whether a request contains `type` request
+ * @param req request object
+ * @param type check for a specified type
+ * @return true if `type` has been requested
+ */
+bool rdns_request_has_type (struct rdns_request *req, enum rdns_request_type type);
+
+/**
+ * Return requested name for a request
+ * @param req request object
+ * @return requested name as it was passed to `rdns_make_request`
+ */
+const struct rdns_request_name* rdns_request_get_name (struct rdns_request *req,
+ unsigned int *count);
+
+/**
+ * Return a DNS server name associated with the request
+ * @param req request object
+ * @return name of a DNS server
+ */
+const char* rdns_request_get_server (struct rdns_request *req);
+
+
+/**
+ * Return PTR string for a request (ipv4 or ipv6) addresses
+ * @param str string representation of IP address
+ * @return name to resolve or NULL if `str` is not an IP address; caller must free result when it is unused
+ */
+char * rdns_generate_ptr_from_str (const char *str);
+
+/**
+ * Format DNS name of the packet punycoding if needed
+ * @param req request
+ * @param name name string
+ * @param namelen length of name
+ */
+bool rdns_format_dns_name (struct rdns_resolver *resolver,
+ const char *name, size_t namelen,
+ char **out, size_t *outlen);
+
+/*
+ * Private functions used by async libraries as callbacks
+ */
+
+void rdns_process_read (int fd, void *arg);
+void rdns_process_timer (void *arg);
+void rdns_process_write (int fd, void *arg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/contrib/librdns/rdns_curve.h b/contrib/librdns/rdns_curve.h
new file mode 100644
index 0000000..365e91b
--- /dev/null
+++ b/contrib/librdns/rdns_curve.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RDNS_CURVE_H_
+#define RDNS_CURVE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rdns_curve_ctx;
+
+/**
+ * Create new dnscurve ctx
+ * @return
+ */
+struct rdns_curve_ctx* rdns_curve_ctx_new (double rekey_interval);
+
+/**
+ * Add key for server `name`
+ * @param ctx curve context
+ * @param name name of server (ip address)
+ * @param pubkey pubkey bytes (must be `RDSN_CURVE_PUBKEY_LEN`)
+ */
+void rdns_curve_ctx_add_key (struct rdns_curve_ctx *ctx,
+ const char *name, const unsigned char *pubkey);
+
+/**
+ * Destroy curve context
+ * @param ctx
+ */
+void rdns_curve_ctx_destroy (struct rdns_curve_ctx *ctx);
+
+
+/**
+ * Register DNSCurve plugin (libsodium should be enabled for this)
+ * @param resolver
+ * @param ctx
+ */
+void rdns_curve_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_curve_ctx *ctx);
+
+/**
+ * Create DNSCurve key from the base16 encoded string
+ * @param hex input hex (must be NULL terminated)
+ * @return a key or NULL (not NULL terminated)
+ */
+unsigned char * rdns_curve_key_from_hex (const char *hex);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RDNS_CURVE_H_ */
diff --git a/contrib/librdns/rdns_ev.h b/contrib/librdns/rdns_ev.h
new file mode 100644
index 0000000..35f532a
--- /dev/null
+++ b/contrib/librdns/rdns_ev.h
@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RDNS_EV_H_
+#define RDNS_EV_H_
+
+#include "contrib/libev/ev.h"
+#include <stdlib.h>
+#include <string.h>
+#include "rdns.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static void* rdns_libev_add_read (void *priv_data, int fd, void *user_data);
+static void rdns_libev_del_read(void *priv_data, void *ev_data);
+static void* rdns_libev_add_write (void *priv_data, int fd, void *user_data);
+static void rdns_libev_del_write (void *priv_data, void *ev_data);
+static void* rdns_libev_add_timer (void *priv_data, double after, void *user_data);
+static void* rdns_libev_add_periodic (void *priv_data, double after,
+ rdns_periodic_callback cb, void *user_data);
+static void rdns_libev_del_periodic (void *priv_data, void *ev_data);
+static void rdns_libev_repeat_timer (void *priv_data, void *ev_data);
+static void rdns_libev_del_timer (void *priv_data, void *ev_data);
+
+struct rdns_ev_periodic_cbdata {
+ ev_timer *ev;
+ rdns_periodic_callback cb;
+ void *cbdata;
+};
+
+static void
+rdns_bind_libev (struct rdns_resolver *resolver, struct ev_loop *loop)
+{
+ static struct rdns_async_context ev_ctx = {
+ .data = NULL,
+ .add_read = rdns_libev_add_read,
+ .del_read = rdns_libev_del_read,
+ .add_write = rdns_libev_add_write,
+ .del_write = rdns_libev_del_write,
+ .add_timer = rdns_libev_add_timer,
+ .repeat_timer = rdns_libev_repeat_timer,
+ .del_timer = rdns_libev_del_timer,
+ .add_periodic = rdns_libev_add_periodic,
+ .del_periodic = rdns_libev_del_periodic,
+ .cleanup = NULL
+ }, *nctx;
+ void *ptr;
+
+ /* XXX: never got freed */
+ ptr = malloc (sizeof (struct rdns_async_context));
+ if (ptr != NULL) {
+ nctx = (struct rdns_async_context *)ptr;
+ memcpy (ptr, (void *)&ev_ctx, sizeof (struct rdns_async_context));
+ nctx->data = (void*)loop;
+ rdns_resolver_async_bind (resolver, nctx);
+ }
+}
+
+static void
+rdns_libev_read_event (struct ev_loop *loop, ev_io *ev, int revents)
+{
+ rdns_process_read (ev->fd, ev->data);
+}
+
+static void
+rdns_libev_write_event (struct ev_loop *loop, ev_io *ev, int revents)
+{
+ rdns_process_write(ev->fd, ev->data);
+}
+
+static void
+rdns_libev_timer_event (struct ev_loop *loop, ev_timer *ev, int revents)
+{
+ rdns_process_timer (ev->data);
+}
+
+static void
+rdns_libev_periodic_event (struct ev_loop *loop, ev_timer *ev, int revents)
+{
+ struct rdns_ev_periodic_cbdata *cbdata = (struct rdns_ev_periodic_cbdata *)
+ ev->data;
+ cbdata->cb (cbdata->cbdata);
+}
+
+static void*
+rdns_libev_add_read (void *priv_data, int fd, void *user_data)
+{
+ ev_io *ev;
+ void *ptr;
+
+ ptr = malloc (sizeof (ev_io));
+ if (ptr != NULL) {
+ ev = (ev_io *)ptr;
+ ev_io_init (ev, rdns_libev_read_event, fd, EV_READ);
+ ev->data = user_data;
+ ev_io_start ((struct ev_loop *)priv_data, ev);
+ }
+ return ptr;
+}
+
+static void
+rdns_libev_del_read (void *priv_data, void *ev_data)
+{
+ ev_io *ev = (ev_io*)ev_data;
+ if (ev != NULL) {
+ ev_io_stop ((struct ev_loop *)priv_data, ev);
+ free ((void *)ev);
+ }
+}
+static void*
+rdns_libev_add_write (void *priv_data, int fd, void *user_data)
+{
+ ev_io *ev;
+
+ ev = (ev_io *)malloc (sizeof (ev_io));
+ if (ev != NULL) {
+ ev_io_init (ev, rdns_libev_write_event, fd, EV_WRITE);
+ ev->data = user_data;
+ ev_io_start ((struct ev_loop *)priv_data, ev);
+ }
+ return (void *)ev;
+}
+
+static void
+rdns_libev_del_write (void *priv_data, void *ev_data)
+{
+ ev_io *ev = (ev_io *)ev_data;
+ if (ev != NULL) {
+ ev_io_stop ((struct ev_loop *)priv_data, ev);
+ free ((void *)ev);
+ }
+}
+
+static void*
+rdns_libev_add_timer (void *priv_data, double after, void *user_data)
+{
+ ev_timer *ev;
+ ev = (ev_timer *)malloc (sizeof (ev_timer));
+ if (ev != NULL) {
+ ev_timer_init (ev, rdns_libev_timer_event, after, after);
+ ev->data = user_data;
+ ev_now_update_if_cheap ((struct ev_loop *)priv_data);
+ ev_timer_start ((struct ev_loop *)priv_data, ev);
+ }
+ return (void *)ev;
+}
+
+static void*
+rdns_libev_add_periodic (void *priv_data, double after,
+ rdns_periodic_callback cb, void *user_data)
+{
+ ev_timer *ev;
+ struct rdns_ev_periodic_cbdata *cbdata = NULL;
+
+ ev = (ev_timer *)malloc (sizeof (ev_timer));
+ if (ev != NULL) {
+ cbdata = (struct rdns_ev_periodic_cbdata *)
+ malloc (sizeof (struct rdns_ev_periodic_cbdata));
+ if (cbdata != NULL) {
+ cbdata->cb = cb;
+ cbdata->cbdata = user_data;
+ cbdata->ev = ev;
+ ev_timer_init (ev, rdns_libev_periodic_event, after, after);
+ ev->data = cbdata;
+ ev_now_update_if_cheap ((struct ev_loop *)priv_data);
+ ev_timer_start ((struct ev_loop *)priv_data, ev);
+ }
+ else {
+ free ((void *)ev);
+ return NULL;
+ }
+ }
+ return (void *)cbdata;
+}
+
+static void
+rdns_libev_del_periodic (void *priv_data, void *ev_data)
+{
+ struct rdns_ev_periodic_cbdata *cbdata = (struct rdns_ev_periodic_cbdata *)
+ ev_data;
+ if (cbdata != NULL) {
+ ev_timer_stop ((struct ev_loop *)priv_data, (ev_timer *)cbdata->ev);
+ free ((void *)cbdata->ev);
+ free ((void *)cbdata);
+ }
+}
+
+static void
+rdns_libev_repeat_timer (void *priv_data, void *ev_data)
+{
+ ev_timer *ev = (ev_timer *)ev_data;
+ if (ev != NULL) {
+ ev_now_update_if_cheap ((struct ev_loop *)priv_data);
+ ev_timer_again ((struct ev_loop *)priv_data, ev);
+ }
+}
+
+static void
+rdns_libev_del_timer (void *priv_data, void *ev_data)
+{
+ ev_timer *ev = (ev_timer *)ev_data;
+ if (ev != NULL) {
+ ev_timer_stop ((struct ev_loop *)priv_data, ev);
+ free ((void *)ev);
+ }
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RDNS_EV_H_ */
diff --git a/contrib/librdns/rdns_event.h b/contrib/librdns/rdns_event.h
new file mode 100644
index 0000000..027181a
--- /dev/null
+++ b/contrib/librdns/rdns_event.h
@@ -0,0 +1,231 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RDNS_EVENT_H_
+#define RDNS_EVENT_H_
+
+#include <event.h>
+#include <stdlib.h>
+#include <string.h>
+#include "rdns.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static void* rdns_libevent_add_read (void *priv_data, int fd, void *user_data);
+static void rdns_libevent_del_read(void *priv_data, void *ev_data);
+static void* rdns_libevent_add_write (void *priv_data, int fd, void *user_data);
+static void rdns_libevent_del_write (void *priv_data, void *ev_data);
+static void* rdns_libevent_add_timer (void *priv_data, double after, void *user_data);
+static void* rdns_libevent_add_periodic (void *priv_data, double after,
+ rdns_periodic_callback cb, void *user_data);
+static void rdns_libevent_del_periodic (void *priv_data, void *ev_data);
+static void rdns_libevent_repeat_timer (void *priv_data, void *ev_data);
+static void rdns_libevent_del_timer (void *priv_data, void *ev_data);
+
+struct rdns_event_periodic_cbdata {
+ struct event *ev;
+ rdns_periodic_callback cb;
+ void *cbdata;
+};
+
+static void
+rdns_bind_libevent (struct rdns_resolver *resolver, struct event_base *ev_base)
+{
+ struct rdns_async_context ev_ctx = {
+ .add_read = rdns_libevent_add_read,
+ .del_read = rdns_libevent_del_read,
+ .add_write = rdns_libevent_add_write,
+ .del_write = rdns_libevent_del_write,
+ .add_timer = rdns_libevent_add_timer,
+ .add_periodic = rdns_libevent_add_periodic,
+ .del_periodic = rdns_libevent_del_periodic,
+ .repeat_timer = rdns_libevent_repeat_timer,
+ .del_timer = rdns_libevent_del_timer,
+ .cleanup = NULL
+ }, *nctx;
+
+ /* XXX: never got freed */
+ nctx = malloc (sizeof (struct rdns_async_context));
+ if (nctx != NULL) {
+ memcpy (nctx, &ev_ctx, sizeof (struct rdns_async_context));
+ nctx->data = ev_base;
+ }
+ rdns_resolver_async_bind (resolver, nctx);
+}
+
+static void
+rdns_libevent_read_event (int fd, short what, void *ud)
+{
+ rdns_process_read (fd, ud);
+}
+
+static void
+rdns_libevent_write_event (int fd, short what, void *ud)
+{
+ rdns_process_write (fd, ud);
+}
+
+static void
+rdns_libevent_timer_event (int fd, short what, void *ud)
+{
+ rdns_process_timer (ud);
+}
+
+static void
+rdns_libevent_periodic_event (int fd, short what, void *ud)
+{
+ struct rdns_event_periodic_cbdata *cbdata = ud;
+ cbdata->cb (cbdata->cbdata);
+}
+
+static void*
+rdns_libevent_add_read (void *priv_data, int fd, void *user_data)
+{
+ struct event *ev;
+ ev = malloc (sizeof (struct event));
+ if (ev != NULL) {
+ event_set (ev, fd, EV_READ | EV_PERSIST, rdns_libevent_read_event, user_data);
+ event_base_set (priv_data, ev);
+ event_add (ev, NULL);
+ }
+ return ev;
+}
+
+static void
+rdns_libevent_del_read(void *priv_data, void *ev_data)
+{
+ struct event *ev = ev_data;
+ if (ev != NULL) {
+ event_del (ev);
+ free (ev);
+ }
+}
+static void*
+rdns_libevent_add_write (void *priv_data, int fd, void *user_data)
+{
+ struct event *ev;
+ ev = malloc (sizeof (struct event));
+ if (ev != NULL) {
+ event_set (ev, fd, EV_WRITE | EV_PERSIST,
+ rdns_libevent_write_event, user_data);
+ event_base_set (priv_data, ev);
+ event_add (ev, NULL);
+ }
+ return ev;
+}
+
+static void
+rdns_libevent_del_write (void *priv_data, void *ev_data)
+{
+ struct event *ev = ev_data;
+ if (ev != NULL) {
+ event_del (ev);
+ free (ev);
+ }
+}
+
+#define rdns_event_double_to_tv(dbl, tv) do { \
+ (tv)->tv_sec = (int)(dbl); \
+ (tv)->tv_usec = ((dbl) - (int)(dbl))*1000*1000; \
+} while(0)
+
+static void*
+rdns_libevent_add_timer (void *priv_data, double after, void *user_data)
+{
+ struct event *ev;
+ struct timeval tv;
+ ev = malloc (sizeof (struct event));
+ if (ev != NULL) {
+ rdns_event_double_to_tv (after, &tv);
+ event_set (ev, -1, EV_TIMEOUT|EV_PERSIST, rdns_libevent_timer_event, user_data);
+ event_base_set (priv_data, ev);
+ event_add (ev, &tv);
+ }
+ return ev;
+}
+
+static void*
+rdns_libevent_add_periodic (void *priv_data, double after,
+ rdns_periodic_callback cb, void *user_data)
+{
+ struct event *ev;
+ struct timeval tv;
+ struct rdns_event_periodic_cbdata *cbdata = NULL;
+
+ ev = malloc (sizeof (struct event));
+ if (ev != NULL) {
+ cbdata = malloc (sizeof (struct rdns_event_periodic_cbdata));
+ if (cbdata != NULL) {
+ rdns_event_double_to_tv (after, &tv);
+ cbdata->cb = cb;
+ cbdata->cbdata = user_data;
+ cbdata->ev = ev;
+ event_set (ev, -1, EV_TIMEOUT|EV_PERSIST, rdns_libevent_periodic_event, cbdata);
+ event_base_set (priv_data, ev);
+ event_add (ev, &tv);
+ }
+ else {
+ free (ev);
+ return NULL;
+ }
+ }
+ return cbdata;
+}
+
+static void
+rdns_libevent_del_periodic (void *priv_data, void *ev_data)
+{
+ struct rdns_event_periodic_cbdata *cbdata = ev_data;
+ if (cbdata != NULL) {
+ event_del (cbdata->ev);
+ free (cbdata->ev);
+ free (cbdata);
+ }
+}
+
+static void
+rdns_libevent_repeat_timer (void *priv_data, void *ev_data)
+{
+ /* XXX: libevent hides timeval, so timeouts are persistent here */
+}
+
+#undef rdns_event_double_to_tv
+
+static void
+rdns_libevent_del_timer (void *priv_data, void *ev_data)
+{
+ struct event *ev = ev_data;
+ if (ev != NULL) {
+ event_del (ev);
+ free (ev);
+ }
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RDNS_EV_H_ */
diff --git a/contrib/librdns/ref.h b/contrib/librdns/ref.h
new file mode 100644
index 0000000..a8016b1
--- /dev/null
+++ b/contrib/librdns/ref.h
@@ -0,0 +1,71 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef REF_H_
+#define REF_H_
+
+/**
+ * @file ref.h
+ * A set of macros to handle refcounts
+ */
+
+typedef void (*ref_dtor_cb_t)(void *data);
+
+typedef struct ref_entry_s {
+ unsigned int refcount;
+ ref_dtor_cb_t dtor;
+} ref_entry_t;
+
+#define REF_INIT(obj, dtor_cb) do { \
+ (obj)->ref.refcount = 0; \
+ (obj)->ref.dtor = (ref_dtor_cb_t)(dtor_cb); \
+} while (0)
+
+#define REF_INIT_RETAIN(obj, dtor_cb) do { \
+ (obj)->ref.refcount = 1; \
+ (obj)->ref.dtor = (ref_dtor_cb_t)(dtor_cb); \
+} while (0)
+
+#ifdef HAVE_ATOMIC_BUILTINS
+#define REF_RETAIN(obj) do { \
+ __sync_add_and_fetch (&(obj)->ref.refcount, 1); \
+} while (0)
+
+#define REF_RELEASE(obj) do { \
+ unsigned int rc = __sync_sub_and_fetch (&(obj)->ref.refcount, 1); \
+ if (rc == 0 && (obj)->ref.dtor) { \
+ (obj)->ref.dtor (obj); \
+ } \
+} while (0)
+#else
+#define REF_RETAIN(obj) do { \
+ (obj)->ref.refcount ++; \
+} while (0)
+
+#define REF_RELEASE(obj) do { \
+ if (--(obj)->ref.refcount == 0 && (obj)->ref.dtor) { \
+ (obj)->ref.dtor (obj); \
+ } \
+} while (0)
+#endif
+
+#endif /* REF_H_ */
diff --git a/contrib/librdns/resolver.c b/contrib/librdns/resolver.c
new file mode 100644
index 0000000..bfcfd0a
--- /dev/null
+++ b/contrib/librdns/resolver.c
@@ -0,0 +1,1577 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <sys/uio.h>
+
+#include "rdns.h"
+#include "dns_private.h"
+#include "ottery.h"
+#include "util.h"
+#include "packet.h"
+#include "parse.h"
+#include "logger.h"
+#include "compression.h"
+
+__KHASH_IMPL(rdns_requests_hash, kh_inline, int, struct rdns_request *, true,
+ kh_int_hash_func, kh_int_hash_equal);
+
+static int
+rdns_send_request (struct rdns_request *req, int fd, bool new_req)
+{
+ ssize_t r;
+ struct rdns_server *serv = req->io->srv;
+ struct rdns_resolver *resolver = req->resolver;
+ struct dns_header *header;
+ const int max_id_cycles = 32;
+ khiter_t k;
+
+ /* Find ID collision */
+ if (new_req) {
+ r = 0;
+
+ for (;;) {
+ k = kh_get(rdns_requests_hash, req->io->requests, req->id);
+ if (k != kh_end(req->io->requests)) {
+ /* Check for unique id */
+ header = (struct dns_header *) req->packet;
+ header->qid = rdns_permutor_generate_id();
+ req->id = header->qid;
+ if (++r > max_id_cycles) {
+ return -1;
+ }
+ }
+ else {
+ break;
+ }
+ }
+ }
+
+ if (resolver->curve_plugin == NULL) {
+ if (!IS_CHANNEL_CONNECTED(req->io)) {
+ r = sendto (fd, req->packet, req->pos, 0,
+ req->io->saddr,
+ req->io->slen);
+ }
+ else {
+ r = send (fd, req->packet, req->pos, 0);
+ }
+ }
+ else {
+ if (!IS_CHANNEL_CONNECTED(req->io)) {
+ r = resolver->curve_plugin->cb.curve_plugin.send_cb (req,
+ resolver->curve_plugin->data,
+ req->io->saddr,
+ req->io->slen);
+ }
+ else {
+ r = resolver->curve_plugin->cb.curve_plugin.send_cb (req,
+ resolver->curve_plugin->data,
+ NULL,
+ 0);
+ }
+ }
+ if (r == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ if (new_req) {
+ /* Write when socket is ready */
+ int pr;
+
+ k = kh_put(rdns_requests_hash, req->io->requests, req->id, &pr);
+ kh_value(req->io->requests, k) = req;
+ req->async_event = resolver->async->add_write (resolver->async->data,
+ fd, req);
+ req->state = RDNS_REQUEST_WAIT_SEND;
+ }
+ /*
+ * If request is already processed then the calling function
+ * should take care about events processing
+ */
+ return 0;
+ }
+ else {
+ rdns_debug ("send failed: %s for server %s", strerror (errno), serv->name);
+ return -1;
+ }
+ }
+ else if (!IS_CHANNEL_CONNECTED(req->io)) {
+ /* Connect socket */
+ r = connect (fd, req->io->saddr, req->io->slen);
+
+ if (r == -1) {
+ rdns_err ("cannot connect after sending request: %s for server %s",
+ strerror (errno), serv->name);
+ }
+ else {
+ req->io->flags |= RDNS_CHANNEL_CONNECTED;
+ }
+ }
+
+ if (new_req) {
+ /* Add request to hash table */
+ int pr;
+ k = kh_put(rdns_requests_hash, req->io->requests, req->id, &pr);
+ kh_value(req->io->requests, k) = req;
+ /* Fill timeout */
+ req->async_event = resolver->async->add_timer (resolver->async->data,
+ req->timeout, req);
+ req->state = RDNS_REQUEST_WAIT_REPLY;
+ }
+
+ return 1;
+}
+
+
+static struct rdns_request *
+rdns_find_dns_request (uint8_t *in, struct rdns_io_channel *ioc)
+{
+ struct dns_header header;
+ int id;
+ struct rdns_resolver *resolver = ioc->resolver;
+
+ memcpy (&header, in, sizeof(header));
+ id = header.qid;
+ khiter_t k = kh_get(rdns_requests_hash, ioc->requests, id);
+
+ if (k == kh_end(ioc->requests)) {
+ /* No such requests found */
+ rdns_debug ("DNS request with id %d has not been found for IO channel", id);
+
+ return NULL;
+ }
+
+ return kh_value(ioc->requests, k);
+}
+
+static bool
+rdns_parse_reply (uint8_t *in, int r, struct rdns_request *req,
+ struct rdns_reply **_rep)
+{
+ struct dns_header *header = (struct dns_header *)in;
+ struct rdns_reply *rep;
+ struct rdns_reply_entry *elt;
+ uint8_t *pos, *npos;
+ struct rdns_resolver *resolver = req->resolver;
+ uint16_t qdcount;
+ int type;
+ bool found = false;
+
+ int i, t;
+
+ /* First check header fields */
+ if (header->qr == 0) {
+ rdns_info ("got request while waiting for reply");
+ return false;
+ }
+
+ qdcount = ntohs (header->qdcount);
+
+ if (qdcount != req->qcount) {
+ rdns_info ("request has %d queries, reply has %d queries", (int)req->qcount, (int)header->qdcount);
+ return false;
+ }
+
+ /*
+ * Now we have request and query data is now at the end of header, so compare
+ * request QR section and reply QR section
+ */
+ req->pos = sizeof (struct dns_header);
+ pos = in + sizeof (struct dns_header);
+ t = r - sizeof (struct dns_header);
+ for (i = 0; i < (int)qdcount; i ++) {
+ if ((npos = rdns_request_reply_cmp (req, pos,t)) == NULL) {
+ rdns_info ("DNS request with id %d is for different query, ignoring", (int)req->id);
+ return false;
+ }
+ t -= npos - pos;
+ pos = npos;
+ }
+ /*
+ * Now pos is in answer section, so we should extract data and form reply
+ */
+ rep = rdns_make_reply (req, header->rcode);
+
+ if (header->ad) {
+ rep->flags |= RDNS_AUTH;
+ }
+
+ if (header->tc) {
+ rep->flags |= RDNS_TRUNCATED;
+ }
+
+ if (rep == NULL) {
+ rdns_warn ("Cannot allocate memory for reply");
+ return false;
+ }
+
+ type = req->requested_names[0].type;
+
+ if (rep->code == RDNS_RC_NOERROR) {
+ r -= pos - in;
+ /* Extract RR records */
+ for (i = 0; i < ntohs (header->ancount); i ++) {
+ elt = malloc (sizeof (struct rdns_reply_entry));
+ t = rdns_parse_rr (resolver, in, elt, &pos, rep, &r);
+ if (t == -1) {
+ free (elt);
+ rdns_debug ("incomplete reply");
+ break;
+ }
+ else if (t == 1) {
+ DL_APPEND (rep->entries, elt);
+ if (elt->type == type) {
+ found = true;
+ }
+ }
+ else {
+ rdns_debug ("no matching reply for %s",
+ req->requested_names[0].name);
+ free (elt);
+ }
+ }
+ }
+
+ if (!found && type != RDNS_REQUEST_ANY) {
+ /* We have not found the requested RR type */
+ if (rep->code == RDNS_RC_NOERROR) {
+ rep->code = RDNS_RC_NOREC;
+ }
+ }
+
+ *_rep = rep;
+ return true;
+}
+
+static bool
+rdns_tcp_maybe_realloc_read_buf (struct rdns_io_channel *ioc)
+{
+ if (ioc->tcp->read_buf_allocated == 0 && ioc->tcp->next_read_size > 0) {
+ ioc->tcp->cur_read_buf = malloc(ioc->tcp->next_read_size);
+
+ if (ioc->tcp->cur_read_buf == NULL) {
+ return false;
+ }
+ ioc->tcp->read_buf_allocated = ioc->tcp->next_read_size;
+ }
+ else if (ioc->tcp->read_buf_allocated < ioc->tcp->next_read_size) {
+ /* Need to realloc */
+ unsigned next_shift = ioc->tcp->next_read_size;
+
+ if (next_shift < ioc->tcp->read_buf_allocated * 2) {
+ if (next_shift < UINT16_MAX && ioc->tcp->read_buf_allocated * 2 <= UINT16_MAX) {
+ next_shift = ioc->tcp->read_buf_allocated * 2;
+ }
+ }
+ void *next_buf = realloc(ioc->tcp->cur_read_buf, next_shift);
+
+ if (next_buf == NULL) {
+ free (ioc->tcp->cur_read_buf);
+ ioc->tcp->cur_read_buf = NULL;
+ return false;
+ }
+
+ ioc->tcp->cur_read_buf = next_buf;
+ }
+
+ return true;
+}
+
+static void
+rdns_process_tcp_read (int fd, struct rdns_io_channel *ioc)
+{
+ ssize_t r;
+ struct rdns_resolver *resolver = ioc->resolver;
+
+ if (ioc->tcp->cur_read == 0) {
+ /* We have to read size first */
+ r = read(fd, &ioc->tcp->next_read_size, sizeof(ioc->tcp->next_read_size));
+
+ if (r == -1 || r == 0) {
+ goto err;
+ }
+
+ ioc->tcp->cur_read += r;
+
+ if (r == sizeof(ioc->tcp->next_read_size)) {
+ ioc->tcp->next_read_size = ntohs(ioc->tcp->next_read_size);
+
+ /* We have read the size, so we can try read one more time */
+ if (!rdns_tcp_maybe_realloc_read_buf(ioc)) {
+ rdns_err("failed to allocate %d bytes: %s",
+ (int)ioc->tcp->next_read_size, strerror(errno));
+ r = -1;
+ goto err;
+ }
+ }
+ else {
+ /* We have read one byte, need to retry... */
+ return;
+ }
+ }
+ else if (ioc->tcp->cur_read == 1) {
+ r = read(fd, ((unsigned char *)&ioc->tcp->next_read_size) + 1, 1);
+
+ if (r == -1 || r == 0) {
+ goto err;
+ }
+
+ ioc->tcp->cur_read += r;
+ ioc->tcp->next_read_size = ntohs(ioc->tcp->next_read_size);
+
+ /* We have read the size, so we can try read one more time */
+ if (!rdns_tcp_maybe_realloc_read_buf(ioc)) {
+ rdns_err("failed to allocate %d bytes: %s",
+ (int)ioc->tcp->next_read_size, strerror(errno));
+ r = -1;
+ goto err;
+ }
+ }
+
+ if (ioc->tcp->next_read_size < sizeof(struct dns_header)) {
+ /* Truncated reply, reset channel */
+ rdns_err("got truncated size: %d on TCP read", ioc->tcp->next_read_size);
+ r = -1;
+ errno = EINVAL;
+ goto err;
+ }
+
+ /* Try to read the full packet if we can */
+ int to_read = ioc->tcp->next_read_size - (ioc->tcp->cur_read - 2);
+
+ if (to_read <= 0) {
+ /* Internal error */
+ rdns_err("internal buffer error on reading!");
+ r = -1;
+ errno = EINVAL;
+ goto err;
+ }
+
+ r = read(fd, ioc->tcp->cur_read_buf + (ioc->tcp->cur_read - 2), to_read);
+ ioc->tcp->cur_read += r;
+
+ if ((ioc->tcp->cur_read - 2) == ioc->tcp->next_read_size) {
+ /* We have a full packet ready, process it */
+ struct rdns_request *req = rdns_find_dns_request (ioc->tcp->cur_read_buf, ioc);
+
+ if (req != NULL) {
+ struct rdns_reply *rep;
+
+ if (rdns_parse_reply (ioc->tcp->cur_read_buf,
+ ioc->tcp->next_read_size, req, &rep)) {
+ UPSTREAM_OK (req->io->srv);
+
+ if (req->resolver->ups && req->io->srv->ups_elt) {
+ req->resolver->ups->ok (req->io->srv->ups_elt,
+ req->resolver->ups->data);
+ }
+
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+ }
+ }
+ else {
+ rdns_warn("unwanted DNS id received over TCP");
+ }
+
+ ioc->tcp->next_read_size = 0;
+ ioc->tcp->cur_read = 0;
+
+ /* Retry read the next packet to avoid unnecessary polling */
+ rdns_process_tcp_read (fd, ioc);
+ }
+
+ return;
+
+err:
+ if (r == 0) {
+ /* Got EOF, just close the socket */
+ rdns_debug ("closing TCP channel due to EOF");
+ rdns_ioc_tcp_reset (ioc);
+ }
+ else if (errno == EINTR || errno == EAGAIN) {
+ /* We just retry later as there is no real error */
+ return;
+ }
+ else {
+ rdns_debug ("closing TCP channel due to IO error: %s", strerror(errno));
+ rdns_ioc_tcp_reset (ioc);
+ }
+}
+
+static void
+rdns_process_tcp_connect (int fd, struct rdns_io_channel *ioc)
+{
+ ioc->flags |= RDNS_CHANNEL_CONNECTED|RDNS_CHANNEL_ACTIVE;
+ ioc->flags &= ~RDNS_CHANNEL_TCP_CONNECTING;
+
+ if (ioc->tcp->async_read == NULL) {
+ ioc->tcp->async_read = ioc->resolver->async->add_read(ioc->resolver->async->data,
+ ioc->sock, ioc);
+ }
+}
+
+static bool
+rdns_reschedule_req_over_tcp (struct rdns_request *req, struct rdns_server *serv)
+{
+ struct rdns_resolver *resolver;
+ struct rdns_io_channel *old_ioc = req->io,
+ *ioc = serv->tcp_io_channels[ottery_rand_uint32 () % serv->tcp_io_cnt];
+
+ resolver = req->resolver;
+
+ if (ioc != NULL) {
+ if (!IS_CHANNEL_CONNECTED(ioc)) {
+ if (!rdns_ioc_tcp_connect(ioc)) {
+ return false;
+ }
+ }
+
+ struct rdns_tcp_output_chain *oc;
+
+ oc = calloc(1, sizeof(*oc) + req->packet_len);
+
+ if (oc == NULL) {
+ rdns_err("failed to allocate output buffer for TCP ioc: %s",
+ strerror(errno));
+ return false;
+ }
+
+ oc->write_buf = ((unsigned char *)oc) + sizeof(*oc);
+ memcpy(oc->write_buf, req->packet, req->packet_len);
+ oc->next_write_size = htons(req->packet_len);
+
+ DL_APPEND(ioc->tcp->output_chain, oc);
+
+ if (ioc->tcp->async_write == NULL) {
+ ioc->tcp->async_write = resolver->async->add_write (
+ resolver->async->data,
+ ioc->sock, ioc);
+ }
+
+ req->state = RDNS_REQUEST_TCP;
+ /* Switch IO channel from UDP to TCP */
+ rdns_request_remove_from_hash (req);
+ req->io = ioc;
+
+ khiter_t k;
+ for (;;) {
+ int pr;
+ k = kh_put(rdns_requests_hash, ioc->requests, req->id, &pr);
+
+ if (pr == 0) {
+ /* We have already a request with this id, so we have to regenerate ID */
+ req->id = rdns_permutor_generate_id ();
+ /* Update packet as well */
+ uint16_t raw_id = req->id;
+ memcpy(req->packet, &raw_id, sizeof(raw_id));
+ }
+ else {
+ break;
+ }
+ }
+
+ req->async_event = resolver->async->add_timer (resolver->async->data,
+ req->timeout, req);
+
+ kh_value(req->io->requests, k) = req;
+ REF_RELEASE(old_ioc);
+ REF_RETAIN(ioc);
+
+ return true;
+ }
+
+ return false;
+}
+
+static void
+rdns_process_udp_read (int fd, struct rdns_io_channel *ioc)
+{
+ struct rdns_resolver *resolver;
+ struct rdns_request *req = NULL;
+ ssize_t r;
+ struct rdns_reply *rep;
+ uint8_t in[UDP_PACKET_SIZE];
+
+ resolver = ioc->resolver;
+
+ /* First read packet from socket */
+ if (resolver->curve_plugin == NULL) {
+ r = recv (fd, in, sizeof (in), 0);
+ if (r > (int)(sizeof (struct dns_header) + sizeof (struct dns_query))) {
+ req = rdns_find_dns_request (in, ioc);
+ }
+ }
+ else {
+ r = resolver->curve_plugin->cb.curve_plugin.recv_cb (ioc, in,
+ sizeof (in), resolver->curve_plugin->data, &req,
+ ioc->saddr, ioc->slen);
+ if (req == NULL &&
+ r > (int)(sizeof (struct dns_header) + sizeof (struct dns_query))) {
+ req = rdns_find_dns_request (in, ioc);
+ }
+ }
+
+ if (req != NULL) {
+ if (rdns_parse_reply (in, r, req, &rep)) {
+ UPSTREAM_OK (req->io->srv);
+
+ if (req->resolver->ups && req->io->srv->ups_elt) {
+ req->resolver->ups->ok (req->io->srv->ups_elt,
+ req->resolver->ups->data);
+ }
+
+ rdns_request_unschedule (req, true);
+
+ if (!(rep->flags & RDNS_TRUNCATED)) {
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func(rep, req->arg);
+ /* This will free reply as well */
+ REF_RELEASE (req);
+ }
+ else {
+ if (req->io->srv->tcp_io_cnt > 0) {
+ rdns_debug("truncated UDP reply for %s; schedule over TCP", req->requested_names[0].name);
+ /* Reschedule via TCP */
+ if (!rdns_reschedule_req_over_tcp (req, req->io->srv)) {
+ /* Use truncated reply as we have no other options */
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func(rep, req->arg);
+ REF_RELEASE (req);
+ }
+ else {
+ /* Remove and free the truncated reply, as we have rescheduled the reply */
+ req->reply = NULL;
+ rdns_reply_free(rep);
+ }
+ }
+ else {
+ /* No TCP channels available */
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func(rep, req->arg);
+ /* This will free reply as well */
+ REF_RELEASE (req);
+ }
+ }
+ }
+ }
+ else {
+ /* Still want to increase uses */
+ ioc->uses ++;
+ }
+}
+
+void
+rdns_process_read (int fd, void *arg)
+{
+ struct rdns_io_channel *ioc = (struct rdns_io_channel *)arg;
+ struct rdns_resolver *resolver;
+
+ resolver = ioc->resolver;
+
+ if (IS_CHANNEL_TCP(ioc)) {
+ if (IS_CHANNEL_CONNECTED(ioc)) {
+ rdns_process_tcp_read (fd, ioc);
+ }
+ else {
+ rdns_err ("read readiness on non connected TCP channel!");
+ }
+ }
+ else {
+ rdns_process_udp_read (fd, ioc);
+ }
+}
+
+void
+rdns_process_timer (void *arg)
+{
+ struct rdns_request *req = (struct rdns_request *)arg;
+ struct rdns_reply *rep;
+ int r;
+ bool renew = false;
+ struct rdns_resolver *resolver;
+ struct rdns_server *serv = NULL;
+ unsigned cnt;
+
+ req->retransmits --;
+ resolver = req->resolver;
+
+ if (req->resolver->ups && req->io->srv->ups_elt) {
+ req->resolver->ups->fail (req->io->srv->ups_elt,
+ req->resolver->ups->data, "timeout waiting reply");
+ }
+ else {
+ UPSTREAM_FAIL (req->io->srv, time (NULL));
+ }
+
+ if (req->state == RDNS_REQUEST_TCP) {
+ rep = rdns_make_reply (req, RDNS_RC_TIMEOUT);
+ rdns_request_unschedule (req, true);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+
+ return;
+ }
+
+ if (req->retransmits == 0) {
+
+ rep = rdns_make_reply (req, RDNS_RC_TIMEOUT);
+ rdns_request_unschedule (req, true);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+
+ return;
+ }
+
+ if (!IS_CHANNEL_ACTIVE(req->io) || req->retransmits == 1) {
+
+ if (resolver->ups) {
+ cnt = resolver->ups->count (resolver->ups->data);
+ }
+ else {
+ cnt = 0;
+ UPSTREAM_FOREACH (resolver->servers, serv) {
+ cnt ++;
+ }
+ }
+
+ if (!IS_CHANNEL_ACTIVE(req->io) || cnt > 1) {
+ /* Do not reschedule IO requests on inactive sockets */
+ rdns_debug ("reschedule request with id: %d", (int)req->id);
+ rdns_request_unschedule (req, true);
+ REF_RELEASE (req->io);
+
+ if (resolver->ups) {
+ struct rdns_upstream_elt *elt;
+
+ elt = resolver->ups->select_retransmit (
+ req->requested_names[0].name,
+ req->requested_names[0].len,
+ req->io->srv->ups_elt,
+ resolver->ups->data);
+
+ if (elt) {
+ serv = elt->server;
+ serv->ups_elt = elt;
+ }
+ else {
+ UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv);
+ }
+ }
+ else {
+ UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv);
+ }
+
+ if (serv == NULL) {
+ rdns_warn ("cannot find suitable server for request");
+ rep = rdns_make_reply (req, RDNS_RC_SERVFAIL);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+
+ return;
+ }
+
+ /* Select random IO channel */
+ req->io = serv->io_channels[ottery_rand_uint32 () % serv->io_cnt];
+ req->io->uses ++;
+ REF_RETAIN (req->io);
+ renew = true;
+ }
+ }
+
+ /*
+ * Note: when `renew` is true, then send_request deals with the
+ * timers and events itself
+ */
+ r = rdns_send_request (req, req->io->sock, renew);
+ if (r == 0) {
+ /* Retransmit one more time */
+ if (!renew) {
+ req->async->del_timer (req->async->data,
+ req->async_event);
+ req->async_event = req->async->add_write (req->async->data,
+ req->io->sock, req);
+ }
+
+ req->state = RDNS_REQUEST_WAIT_SEND;
+ }
+ else if (r == -1) {
+ if (req->resolver->ups && req->io->srv->ups_elt) {
+ req->resolver->ups->fail (req->io->srv->ups_elt,
+ req->resolver->ups->data, "cannot send retransmit after timeout");
+ }
+ else {
+ UPSTREAM_FAIL (req->io->srv, time (NULL));
+ }
+
+ if (!renew) {
+ req->async->del_timer (req->async->data,
+ req->async_event);
+ req->async_event = NULL;
+ rdns_request_remove_from_hash(req);
+ }
+
+ /* We have not scheduled timeout actually due to send error */
+ rep = rdns_make_reply (req, RDNS_RC_NETERR);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+ }
+ else {
+ req->async->repeat_timer (req->async->data, req->async_event);
+ req->state = RDNS_REQUEST_WAIT_REPLY;
+ }
+}
+
+static void
+rdns_process_periodic (void *arg)
+{
+ struct rdns_resolver *resolver = (struct rdns_resolver*)arg;
+ struct rdns_server *serv;
+
+ UPSTREAM_RESCAN (resolver->servers, time (NULL));
+
+ UPSTREAM_FOREACH (resolver->servers, serv) {
+ for (int i = 0; i < serv->tcp_io_cnt; i ++) {
+ if (IS_CHANNEL_CONNECTED(serv->tcp_io_channels[i])) {
+ /* Disconnect channels with no requests in flight */
+ if (kh_size(serv->tcp_io_channels[i]->requests) == 0) {
+ rdns_debug ("reset inactive TCP connection to %s", serv->name);
+ rdns_ioc_tcp_reset (serv->tcp_io_channels[i]);
+ }
+ }
+ }
+ }
+}
+
+static void
+rdns_process_ioc_refresh (void *arg)
+{
+ struct rdns_resolver *resolver = (struct rdns_resolver*)arg;
+ struct rdns_server *serv;
+ struct rdns_io_channel *ioc, *nioc;
+ unsigned int i;
+
+ if (resolver->max_ioc_uses > 0) {
+ UPSTREAM_FOREACH (resolver->servers, serv) {
+ for (i = 0; i < serv->io_cnt; i ++) {
+ ioc = serv->io_channels[i];
+ if (ioc->uses > resolver->max_ioc_uses) {
+ /* Schedule IOC removing */
+ nioc = rdns_ioc_new (serv, resolver, false);
+
+ if (nioc == NULL) {
+ rdns_err ("calloc fails to allocate rdns_io_channel");
+ continue;
+ }
+
+ serv->io_channels[i] = nioc;
+ rdns_debug ("scheduled io channel for server %s to be refreshed after "
+ "%lu usages", serv->name, (unsigned long)ioc->uses);
+ ioc->flags &= ~RDNS_CHANNEL_ACTIVE;
+ REF_RELEASE (ioc);
+ }
+ }
+ }
+ }
+}
+
+static void
+rdns_process_udp_retransmit (int fd, struct rdns_request *req)
+{
+ struct rdns_resolver *resolver;
+ struct rdns_reply *rep;
+ int r;
+
+ resolver = req->resolver;
+
+ resolver->async->del_write (resolver->async->data,
+ req->async_event);
+ req->async_event = NULL;
+
+ if (req->state == RDNS_REQUEST_FAKE) {
+ /* Reply is ready */
+ req->func (req->reply, req->arg);
+ REF_RELEASE (req);
+
+ return;
+ }
+
+ r = rdns_send_request (req, fd, false);
+
+ if (r == 0) {
+ /* Retransmit one more time */
+ req->async_event = req->async->add_write (req->async->data,
+ fd, req);
+ req->state = RDNS_REQUEST_WAIT_SEND;
+ }
+ else if (r == -1) {
+ if (req->resolver->ups && req->io->srv->ups_elt) {
+ req->resolver->ups->fail (req->io->srv->ups_elt,
+ req->resolver->ups->data, "retransmit send failed");
+ }
+ else {
+ UPSTREAM_FAIL (req->io->srv, time (NULL));
+ }
+
+ rep = rdns_make_reply (req, RDNS_RC_NETERR);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+ }
+ else {
+ req->async_event = req->async->add_timer (req->async->data,
+ req->timeout, req);
+ req->state = RDNS_REQUEST_WAIT_REPLY;
+ }
+}
+
+static ssize_t
+rdns_write_output_chain (struct rdns_io_channel *ioc, struct rdns_tcp_output_chain *oc)
+{
+ ssize_t r;
+ struct iovec iov[2];
+ int niov, already_written;
+ int packet_len = ntohs (oc->next_write_size);
+
+ switch (oc->cur_write) {
+ case 0:
+ /* Size + DNS request in full */
+ iov[0].iov_base = &oc->next_write_size;
+ iov[0].iov_len = sizeof (oc->next_write_size);
+ iov[1].iov_base = oc->write_buf;
+ iov[1].iov_len = packet_len;
+ niov = 2;
+ break;
+ case 1:
+ /* Partial Size + DNS request in full */
+ iov[0].iov_base = ((unsigned char *)&oc->next_write_size) + 1;
+ iov[0].iov_len = 1;
+ iov[1].iov_base = oc->write_buf;
+ iov[1].iov_len = packet_len;
+ niov = 2;
+ break;
+ default:
+ /* Merely DNS packet */
+ already_written = oc->cur_write - 2;
+ if (packet_len <= already_written) {
+ errno = EINVAL;
+ return -1;
+ }
+ iov[0].iov_base = oc->write_buf + already_written;
+ iov[0].iov_len = packet_len - already_written;
+ niov = 1;
+ break;
+ }
+
+ r = writev(ioc->sock, iov, niov);
+
+ if (r > 0) {
+ oc->cur_write += r;
+ }
+
+ return r;
+}
+
+static void
+rdns_process_tcp_write (int fd, struct rdns_io_channel *ioc)
+{
+ struct rdns_resolver *resolver = ioc->resolver;
+
+
+ /* Try to write as much as we can */
+ struct rdns_tcp_output_chain *oc, *tmp;
+ DL_FOREACH_SAFE(ioc->tcp->output_chain, oc, tmp) {
+ ssize_t r = rdns_write_output_chain (ioc, oc);
+
+ if (r == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ /* Write even is persistent */
+ return;
+ }
+ else {
+ rdns_err ("error when trying to write request to %s: %s",
+ ioc->srv->name, strerror (errno));
+ rdns_ioc_tcp_reset (ioc);
+ return;
+ }
+ }
+ else if (ntohs(oc->next_write_size) < oc->cur_write) {
+ /* Packet has been fully written, remove it */
+ DL_DELETE(ioc->tcp->output_chain, oc);
+ free (oc); /* It also frees write buf */
+ ioc->tcp->cur_output_chains --;
+ }
+ else {
+ /* Buffer is not yet processed, stop unless we can continue */
+ break;
+ }
+ }
+
+ if (ioc->tcp->cur_output_chains == 0) {
+ /* Unregister write event */
+ ioc->resolver->async->del_write (ioc->resolver->async->data,
+ ioc->tcp->async_write);
+ ioc->tcp->async_write = NULL;
+ }
+}
+
+void
+rdns_process_write (int fd, void *arg)
+{
+ /*
+ * We first need to dispatch *arg to understand what has caused the write
+ * readiness event.
+ * The one possibility is that it was a UDP retransmit request, so our
+ * arg will be struct rdns_request *
+ * Another possibility is that write event was triggered by some TCP related
+ * stuff. In this case the only possibility is that our arg is struct rdns_io_channel *
+ * To distinguish these two cases (due to flaws in the rdns architecture in the first
+ * place) we compare the first 8 bytes with RDNS_IO_CHANNEL_TAG
+ */
+ uint64_t tag;
+
+ memcpy (&tag, arg, sizeof(tag));
+
+ if (tag == RDNS_IO_CHANNEL_TAG) {
+ struct rdns_io_channel *ioc = (struct rdns_io_channel *) arg;
+
+ if (IS_CHANNEL_CONNECTED(ioc)) {
+ rdns_process_tcp_write(fd, ioc);
+ }
+ else {
+ rdns_process_tcp_connect(fd, ioc);
+ rdns_process_tcp_write(fd, ioc);
+ }
+ }
+ else {
+ struct rdns_request *req = (struct rdns_request *) arg;
+ rdns_process_udp_retransmit(fd, req);
+ }
+}
+
+struct rdns_server *
+rdns_select_request_upstream (struct rdns_resolver *resolver,
+ struct rdns_request *req,
+ bool is_retransmit,
+ struct rdns_server *prev_serv)
+{
+ struct rdns_server *serv = NULL;
+
+ if (resolver->ups) {
+ struct rdns_upstream_elt *elt;
+
+ if (is_retransmit && prev_serv) {
+ elt = resolver->ups->select_retransmit (req->requested_names[0].name,
+ req->requested_names[0].len,
+ prev_serv->ups_elt,
+ resolver->ups->data);
+ }
+ else {
+ elt = resolver->ups->select (req->requested_names[0].name,
+ req->requested_names[0].len, resolver->ups->data);
+ }
+
+ if (elt) {
+ serv = elt->server;
+ serv->ups_elt = elt;
+ }
+ else {
+ UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv);
+ }
+ }
+ else {
+ UPSTREAM_SELECT_ROUND_ROBIN (resolver->servers, serv);
+ }
+
+ return serv;
+}
+
+#define align_ptr(p, a) \
+ (guint8 *) (((uintptr_t) (p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1))
+
+struct rdns_request*
+rdns_make_request_full (
+ struct rdns_resolver *resolver,
+ dns_callback_type cb,
+ void *cbdata,
+ double timeout,
+ unsigned int repeats,
+ unsigned int queries,
+ ...
+ )
+{
+ va_list args;
+ struct rdns_request *req;
+ struct rdns_server *serv;
+ int r, type;
+ unsigned int i, tlen = 0, clen = 0, cur;
+ size_t olen;
+ const char *cur_name, *last_name = NULL;
+ khash_t(rdns_compression_hash) *comp = NULL;
+ struct rdns_fake_reply *fake_rep = NULL;
+ char fake_buf[MAX_FAKE_NAME + sizeof (struct rdns_fake_reply_idx) + 16];
+ struct rdns_fake_reply_idx *idx;
+
+ if (resolver == NULL || !resolver->initialized) {
+ if (resolver == NULL) {
+ return NULL;
+ }
+
+ rdns_err ("resolver is uninitialized");
+
+ return NULL;
+ }
+
+ req = malloc (sizeof (struct rdns_request));
+ if (req == NULL) {
+ rdns_err ("failed to allocate memory for request: %s",
+ strerror (errno));
+ return NULL;
+ }
+
+ req->resolver = resolver;
+ req->func = cb;
+ req->arg = cbdata;
+ req->reply = NULL;
+ req->qcount = queries;
+ req->io = NULL;
+ req->state = RDNS_REQUEST_NEW;
+ req->packet = NULL;
+ req->requested_names = calloc (queries, sizeof (struct rdns_request_name));
+ req->async_event = NULL;
+
+ if (req->requested_names == NULL) {
+ free (req);
+ rdns_err ("failed to allocate memory for request data: %s",
+ strerror (errno));
+
+ return NULL;
+ }
+
+ req->type = 0;
+#ifdef TWEETNACL
+ req->curve_plugin_data = NULL;
+#endif
+ REF_INIT_RETAIN (req, rdns_request_free);
+
+ /* Calculate packet's total length based on records count */
+ va_start (args, queries);
+ for (i = 0; i < queries * 2; i += 2) {
+ cur = i / 2;
+ cur_name = va_arg (args, const char *);
+ type = va_arg (args, int);
+
+ if (cur_name != NULL) {
+ clen = strlen (cur_name);
+
+ if (clen == 0) {
+ rdns_warn ("got empty name to resolve");
+ rdns_request_free (req);
+ return NULL;
+ }
+
+ if (cur_name[0] == '.') {
+ /* Skip dots at the begin */
+ unsigned int ndots = strspn (cur_name, ".");
+
+ cur_name += ndots;
+ clen -= ndots;
+
+ if (clen == 0) {
+ rdns_warn ("got empty name to resolve");
+ rdns_request_free (req);
+ return NULL;
+ }
+ }
+
+ if (cur_name[clen - 1] == '.') {
+ /* Skip trailing dots */
+ while (clen >= 1 && cur_name[clen - 1] == '.') {
+ clen --;
+ }
+
+ if (clen == 0) {
+ rdns_warn ("got empty name to resolve");
+ rdns_request_free (req);
+ return NULL;
+ }
+ }
+
+ if (last_name == NULL && queries == 1 && clen < MAX_FAKE_NAME) {
+ /* We allocate structure in the static space */
+ idx = (struct rdns_fake_reply_idx *)align_ptr (fake_buf, 16);
+ idx->type = type;
+ idx->len = clen;
+ memcpy (idx->request, cur_name, clen);
+ HASH_FIND (hh, resolver->fake_elts, idx, sizeof (*idx) + clen,
+ fake_rep);
+
+ if (fake_rep) {
+ /* We actually treat it as a short-circuit */
+ req->reply = rdns_make_reply (req, fake_rep->rcode);
+ req->reply->entries = fake_rep->result;
+ req->state = RDNS_REQUEST_FAKE;
+ }
+ }
+
+ last_name = cur_name;
+ tlen += clen;
+ }
+ else if (last_name == NULL) {
+ rdns_err ("got NULL as the first name to resolve");
+ rdns_request_free (req);
+ return NULL;
+ }
+
+ if (req->state != RDNS_REQUEST_FAKE) {
+ if (!rdns_format_dns_name (resolver, last_name, clen,
+ &req->requested_names[cur].name, &olen)) {
+ rdns_err ("cannot format %s", last_name);
+ rdns_request_free (req);
+ return NULL;
+ }
+
+ req->requested_names[cur].len = olen;
+ }
+ else {
+ req->requested_names[cur].len = clen;
+ }
+
+ req->requested_names[cur].type = type;
+ }
+
+ va_end (args);
+
+ if (req->state != RDNS_REQUEST_FAKE) {
+ rdns_allocate_packet (req, tlen);
+ rdns_make_dns_header (req, queries);
+
+ for (i = 0; i < queries; i++) {
+ cur_name = req->requested_names[i].name;
+ clen = req->requested_names[i].len;
+ type = req->requested_names[i].type;
+ if (queries > 1) {
+ if (!rdns_add_rr (req, cur_name, clen, type, &comp)) {
+ rdns_err ("cannot add rr");
+ REF_RELEASE (req);
+ rdns_compression_free(comp);
+ return NULL;
+ }
+ } else {
+ if (!rdns_add_rr (req, cur_name, clen, type, NULL)) {
+ rdns_err ("cannot add rr");
+ REF_RELEASE (req);
+ rdns_compression_free(comp);
+ return NULL;
+ }
+ }
+ }
+
+ rdns_compression_free(comp);
+
+ /* Add EDNS RR */
+ rdns_add_edns0 (req);
+
+ req->retransmits = repeats ? repeats : 1;
+ req->timeout = timeout;
+ req->state = RDNS_REQUEST_NEW;
+ }
+
+ req->async = resolver->async;
+
+ serv = rdns_select_request_upstream (resolver, req, false, NULL);
+
+ if (serv == NULL) {
+ rdns_warn ("cannot find suitable server for request");
+ REF_RELEASE (req);
+ return NULL;
+ }
+
+ /* Select random IO channel */
+ req->io = serv->io_channels[ottery_rand_uint32 () % serv->io_cnt];
+
+ if (req->state == RDNS_REQUEST_FAKE) {
+ req->async_event = resolver->async->add_write (resolver->async->data,
+ req->io->sock, req);
+ }
+ else {
+ /* Now send request to server */
+ do {
+ r = rdns_send_request (req, req->io->sock, true);
+
+ if (r == -1) {
+ req->retransmits --; /* It must be > 0 */
+
+ if (req->retransmits > 0) {
+ if (resolver->ups && serv->ups_elt) {
+ resolver->ups->fail (serv->ups_elt, resolver->ups->data,
+ "send IO error");
+ }
+ else {
+ UPSTREAM_FAIL (serv, time (NULL));
+ }
+
+ serv = rdns_select_request_upstream (resolver, req,
+ true, serv);
+
+ if (serv == NULL) {
+ rdns_warn ("cannot find suitable server for request");
+ REF_RELEASE (req);
+ return NULL;
+ }
+
+ req->io = serv->io_channels[ottery_rand_uint32 () % serv->io_cnt];
+ }
+ else {
+ rdns_info ("cannot send DNS request: %s", strerror (errno));
+ REF_RELEASE (req);
+
+ if (resolver->ups && serv->ups_elt) {
+ resolver->ups->fail (serv->ups_elt, resolver->ups->data,
+ "send IO error");
+ }
+ else {
+ UPSTREAM_FAIL (serv, time (NULL));
+ }
+
+ return NULL;
+ }
+ }
+ else {
+ /* All good */
+ req->io->uses++;
+ break;
+ }
+ } while (req->retransmits > 0);
+ }
+
+ REF_RETAIN (req->io);
+ REF_RETAIN (req->resolver);
+
+ return req;
+}
+
+bool
+rdns_resolver_init (struct rdns_resolver *resolver)
+{
+ unsigned int i;
+ struct rdns_server *serv;
+ struct rdns_io_channel *ioc;
+
+ if (!resolver->async_binded) {
+ rdns_err ("no async backend specified");
+ return false;
+ }
+
+ if (resolver->servers == NULL) {
+ rdns_err ("no DNS servers defined");
+ return false;
+ }
+
+ /* Now init io channels to all servers */
+ UPSTREAM_FOREACH (resolver->servers, serv) {
+ serv->io_channels = calloc (serv->io_cnt, sizeof (struct rdns_io_channel *));
+
+ if (serv->io_channels == NULL) {
+ rdns_err ("cannot allocate memory for the resolver IO channels");
+ return false;
+ }
+
+ for (i = 0; i < serv->io_cnt; i ++) {
+ ioc = rdns_ioc_new(serv, resolver, false);
+
+ if (ioc == NULL) {
+ rdns_err ("cannot allocate memory or init the IO channel");
+ return false;
+ }
+
+ serv->io_channels[i] = ioc;
+ }
+
+ int ntcp_channels = 0;
+
+ /*
+ * We are more forgiving for TCP IO channels: we can have zero of them
+ * if DNS is misconfigured and still be able to resolve stuff
+ */
+ serv->tcp_io_channels = calloc (serv->tcp_io_cnt, sizeof (struct rdns_io_channel *));
+ if (serv->tcp_io_channels == NULL) {
+ rdns_err ("cannot allocate memory for the resolver TCP IO channels");
+ return false;
+ }
+ for (i = 0; i < serv->tcp_io_cnt; i ++) {
+ ioc = rdns_ioc_new (serv, resolver, true);
+
+ if (ioc == NULL) {
+ rdns_err ("cannot allocate memory or init the TCP IO channel");
+ continue;
+ }
+
+ serv->tcp_io_channels[ntcp_channels++] = ioc;
+ }
+
+ serv->tcp_io_cnt = ntcp_channels;
+ }
+
+ if (resolver->async->add_periodic) {
+ resolver->periodic = resolver->async->add_periodic (resolver->async->data,
+ UPSTREAM_REVIVE_TIME, rdns_process_periodic, resolver);
+ }
+
+ resolver->initialized = true;
+
+ return true;
+}
+
+void
+rdns_resolver_register_plugin (struct rdns_resolver *resolver,
+ struct rdns_plugin *plugin)
+{
+ if (resolver != NULL && plugin != NULL) {
+ /* XXX: support only network plugin now, and only a single one */
+ if (plugin->type == RDNS_PLUGIN_CURVE) {
+ resolver->curve_plugin = plugin;
+ }
+ }
+}
+
+void *
+rdns_resolver_add_server (struct rdns_resolver *resolver,
+ const char *name, unsigned int port,
+ int priority, unsigned int io_cnt)
+{
+ struct rdns_server *serv;
+ union {
+ struct in_addr v4;
+ struct in6_addr v6;
+ } addr;
+
+ if (inet_pton (AF_INET, name, &addr) == 0 &&
+ inet_pton (AF_INET6, name, &addr) == 0) {
+ /* Invalid IP */
+ return NULL;
+ }
+
+ if (io_cnt == 0) {
+ return NULL;
+ }
+ if (port == 0 || port > UINT16_MAX) {
+ return NULL;
+ }
+
+ serv = calloc (1, sizeof (struct rdns_server));
+ if (serv == NULL) {
+ return NULL;
+ }
+ serv->name = strdup (name);
+ if (serv->name == NULL) {
+ free (serv);
+ return NULL;
+ }
+
+ serv->io_cnt = io_cnt;
+ /* TODO: make it configurable maybe? */
+ serv->tcp_io_cnt = default_tcp_io_cnt;
+ serv->port = port;
+
+ UPSTREAM_ADD (resolver->servers, serv, priority);
+
+ return serv;
+}
+
+void
+rdns_resolver_set_logger (struct rdns_resolver *resolver,
+ rdns_log_function logger, void *log_data)
+{
+ resolver->logger = logger;
+ resolver->log_data = log_data;
+}
+
+void
+rdns_resolver_set_log_level (struct rdns_resolver *resolver,
+ enum rdns_log_level level)
+{
+ resolver->log_level = level;
+}
+
+void
+rdns_resolver_set_upstream_lib (struct rdns_resolver *resolver,
+ struct rdns_upstream_context *ups_ctx,
+ void *ups_data)
+{
+ resolver->ups = ups_ctx;
+ resolver->ups->data = ups_data;
+}
+
+
+void
+rdns_resolver_set_max_io_uses (struct rdns_resolver *resolver,
+ uint64_t max_ioc_uses, double check_time)
+{
+ if (resolver->refresh_ioc_periodic != NULL) {
+ resolver->async->del_periodic (resolver->async->data,
+ resolver->refresh_ioc_periodic);
+ resolver->refresh_ioc_periodic = NULL;
+ }
+
+ resolver->max_ioc_uses = max_ioc_uses;
+ if (check_time > 0.0 && resolver->async->add_periodic) {
+ resolver->refresh_ioc_periodic =
+ resolver->async->add_periodic (resolver->async->data,
+ check_time, rdns_process_ioc_refresh, resolver);
+ }
+}
+
+static void
+rdns_resolver_free (struct rdns_resolver *resolver)
+{
+ struct rdns_server *serv, *stmp;
+ struct rdns_io_channel *ioc;
+ unsigned int i;
+
+ if (resolver->initialized) {
+ if (resolver->periodic != NULL) {
+ resolver->async->del_periodic (resolver->async->data, resolver->periodic);
+ }
+ if (resolver->refresh_ioc_periodic != NULL) {
+ resolver->async->del_periodic (resolver->async->data,
+ resolver->refresh_ioc_periodic);
+ }
+ if (resolver->curve_plugin != NULL && resolver->curve_plugin->dtor != NULL) {
+ resolver->curve_plugin->dtor (resolver, resolver->curve_plugin->data);
+ }
+ /* Stop IO watch on all IO channels */
+ UPSTREAM_FOREACH_SAFE (resolver->servers, serv, stmp) {
+ for (i = 0; i < serv->io_cnt; i ++) {
+ ioc = serv->io_channels[i];
+ REF_RELEASE (ioc);
+ }
+ for (i = 0; i < serv->tcp_io_cnt; i ++) {
+ ioc = serv->tcp_io_channels[i];
+ REF_RELEASE (ioc);
+ }
+ UPSTREAM_DEL (resolver->servers, serv);
+ free (serv->io_channels);
+ free (serv->tcp_io_channels);
+ free (serv->name);
+ free (serv);
+ }
+ }
+ free (resolver->async);
+ free (resolver);
+}
+
+
+struct rdns_resolver *
+rdns_resolver_new (int flags)
+{
+ struct rdns_resolver *new_resolver;
+
+ new_resolver = calloc (1, sizeof (struct rdns_resolver));
+
+ REF_INIT_RETAIN (new_resolver, rdns_resolver_free);
+
+ new_resolver->logger = rdns_logger_internal;
+ new_resolver->log_data = new_resolver;
+ new_resolver->flags = flags;
+
+ return new_resolver;
+}
+
+void
+rdns_resolver_async_bind (struct rdns_resolver *resolver,
+ struct rdns_async_context *ctx)
+{
+ if (resolver != NULL && ctx != NULL) {
+ resolver->async = ctx;
+ resolver->async_binded = true;
+ }
+}
+
+void
+rdns_resolver_set_dnssec (struct rdns_resolver *resolver, bool enabled)
+{
+ if (resolver) {
+ resolver->enable_dnssec = enabled;
+ }
+}
+
+
+void rdns_resolver_set_fake_reply (struct rdns_resolver *resolver,
+ const char *name,
+ enum rdns_request_type type,
+ enum dns_rcode rcode,
+ struct rdns_reply_entry *reply)
+{
+ struct rdns_fake_reply *fake_rep;
+ struct rdns_fake_reply_idx *srch;
+ unsigned len = strlen (name);
+
+ assert (len < MAX_FAKE_NAME);
+ srch = malloc (sizeof (*srch) + len);
+ srch->len = len;
+ srch->type = type;
+ memcpy (srch->request, name, len);
+
+ HASH_FIND (hh, resolver->fake_elts, srch, len + sizeof (*srch), fake_rep);
+
+ if (fake_rep) {
+ /* Append reply to the existing list */
+ fake_rep->rcode = rcode;
+
+ if (reply) {
+ DL_CONCAT (fake_rep->result, reply);
+ }
+ }
+ else {
+ fake_rep = calloc (1, sizeof (*fake_rep) + len);
+
+ if (fake_rep == NULL) {
+ abort ();
+ }
+
+ fake_rep->rcode = rcode;
+
+ memcpy (&fake_rep->key, srch, sizeof (*srch) + len);
+
+ if (reply) {
+ DL_CONCAT (fake_rep->result, reply);
+ }
+
+ HASH_ADD (hh, resolver->fake_elts, key, sizeof (*srch) + len, fake_rep);
+ }
+
+ free (srch);
+}
diff --git a/contrib/librdns/upstream.h b/contrib/librdns/upstream.h
new file mode 100644
index 0000000..a384155
--- /dev/null
+++ b/contrib/librdns/upstream.h
@@ -0,0 +1,282 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UPSTREAM_H_
+#define UPSTREAM_H_
+
+#include <time.h>
+#include <stdio.h>
+
+/**
+ * @file upstream.h
+ * The basic macros to define upstream objects
+ */
+
+#ifndef upstream_fatal
+#define upstream_fatal(msg) \
+ do { \
+ perror(msg); \
+ exit(-1); \
+ } while (0)
+#endif
+
+#ifndef upstream_malloc
+#define upstream_malloc(size) malloc(size)
+#endif
+
+#ifndef upstream_free
+#define upstream_free(size, ptr) free(ptr)
+#endif
+
+struct upstream_entry_s;
+struct upstream_common_data {
+ void **upstreams;
+ unsigned int allocated_nelts;
+ unsigned int nelts;
+ unsigned int alive;
+};
+
+typedef struct upstream_entry_s {
+ unsigned short errors; /**< errors for this upstream */
+ unsigned short dead;
+ unsigned short priority;
+ unsigned short weight;
+ time_t time; /**< time of marking */
+ void *parent; /**< parent object */
+ struct upstream_common_data *common; /**< common data */
+ void *next; /**< link to the next */
+} upstream_entry_t;
+
+/*
+ * Here we define some reasonable defaults:
+ * if an upstream has more than `UPSTREAM_MAX_ERRORS` in the period of time
+ * of `UPSTREAM_ERROR_TIME` then we shut it down for `UPSTREAM_REVIVE_TIME`.
+ * In this particular case times are 10 seconds for 10 errors and revive in
+ * 30 seconds.
+ */
+#ifndef UPSTREAM_REVIVE_TIME
+#define UPSTREAM_REVIVE_TIME 30
+#endif
+#ifndef UPSTREAM_ERROR_TIME
+#define UPSTREAM_ERROR_TIME 10
+#endif
+#ifndef UPSTREAM_MAX_ERRORS
+#define UPSTREAM_MAX_ERRORS 10
+#endif
+
+#define UPSTREAM_FAIL(u, now) \
+ do { \
+ if ((u)->up.time != 0) { \
+ if ((now) - (u)->up.time >= UPSTREAM_ERROR_TIME) { \
+ if ((u)->up.errors >= UPSTREAM_MAX_ERRORS) { \
+ (u)->up.dead = 1; \
+ (u)->up.time = now; \
+ (u)->up.common->alive--; \
+ } \
+ else { \
+ (u)->up.errors = 1; \
+ (u)->up.time = (now); \
+ } \
+ } \
+ else { \
+ (u)->up.errors++; \
+ } \
+ } \
+ else { \
+ (u)->up.errors++; \
+ (u)->up.time = (now); \
+ } \
+ } while (0)
+
+#define UPSTREAM_OK(u) \
+ do { \
+ (u)->up.errors = 0; \
+ (u)->up.time = 0; \
+ } while (0)
+
+#define UPSTREAM_ADD(head, u, priority) \
+ do { \
+ if (head == NULL) { \
+ struct upstream_common_data *cd; \
+ cd = upstream_malloc(sizeof(struct upstream_common_data)); \
+ if (cd == NULL) { \
+ upstream_fatal("malloc failed"); \
+ } \
+ cd->upstreams = upstream_malloc(sizeof(void *) * 8); \
+ if (cd == NULL) { \
+ upstream_fatal("malloc failed"); \
+ } \
+ cd->allocated_nelts = 8; \
+ cd->nelts = 1; \
+ cd->alive = 1; \
+ cd->upstreams[0] = (u); \
+ (u)->up.common = cd; \
+ } \
+ else { \
+ struct upstream_common_data *cd = (head)->up.common; \
+ (u)->up.common = cd; \
+ if (cd->nelts == cd->allocated_nelts) { \
+ void **nup; \
+ nup = upstream_malloc(sizeof(void *) * cd->nelts * 2); \
+ if (nup == NULL) { \
+ upstream_fatal("malloc failed"); \
+ } \
+ memcpy(nup, cd->upstreams, cd->nelts * sizeof(void *)); \
+ upstream_free(cd->nelts * sizeof(void *), cd->upstreams); \
+ cd->upstreams = nup; \
+ cd->allocated_nelts *= 2; \
+ } \
+ cd->upstreams[cd->nelts++] = (u); \
+ cd->alive++; \
+ } \
+ (u)->up.next = (head); \
+ (head) = (u); \
+ if (priority > 0) { \
+ (u)->up.priority = (u)->up.weight = (priority); \
+ } \
+ else { \
+ (u)->up.priority = (u)->up.weight = 65535; \
+ } \
+ (u)->up.time = 0; \
+ (u)->up.errors = 0; \
+ (u)->up.dead = 0; \
+ (u)->up.parent = (u); \
+ } while (0)
+
+#define UPSTREAM_DEL(head, u) \
+ do { \
+ if (head != NULL) { \
+ struct upstream_common_data *cd = (head)->up.common; \
+ if ((u)->up.next != NULL) { \
+ (head) = (u)->up.next; \
+ cd->nelts--; \
+ cd->alive--; \
+ } \
+ else { \
+ upstream_free(cd->allocated_nelts * sizeof(void *), \
+ cd->upstreams); \
+ upstream_free(sizeof(struct upstream_common_data), cd); \
+ (head) = NULL; \
+ } \
+ } \
+ } while (0)
+
+#define UPSTREAM_FOREACH(head, u) for ((u) = (head); (u) != NULL; (u) = (u)->up.next)
+#define UPSTREAM_FOREACH_SAFE(head, u, tmp) \
+ for ((u) = (head); \
+ (u) != NULL && ((tmp = (u)->up.next) || true); \
+ (u) = (tmp))
+
+#define UPSTREAM_REVIVE_ALL(head) \
+ do { \
+ __typeof(head) elt = (head); \
+ while (elt != NULL) { \
+ elt->up.dead = 0; \
+ elt->up.errors = 0; \
+ elt->up.time = 0; \
+ elt = elt->up.next; \
+ } \
+ (head)->up.common->alive = (head)->up.common->nelts; \
+ } while (0)
+
+#define UPSTREAM_RESCAN(head, now) \
+ do { \
+ __typeof(head) elt = (head); \
+ if ((head)->up.common->alive == 0) { \
+ UPSTREAM_REVIVE_ALL((head)); \
+ } \
+ else { \
+ while (elt != NULL) { \
+ if (elt->up.dead) { \
+ if ((now) -elt->up.time >= UPSTREAM_REVIVE_TIME) { \
+ elt->up.dead = 0; \
+ elt->up.errors = 0; \
+ elt->up.weight = elt->up.priority; \
+ (head)->up.common->alive++; \
+ } \
+ } \
+ else { \
+ if ((now) -elt->up.time >= UPSTREAM_ERROR_TIME && \
+ elt->up.errors >= UPSTREAM_MAX_ERRORS) { \
+ elt->up.dead = 1; \
+ elt->up.time = now; \
+ (head)->up.common->alive--; \
+ } \
+ } \
+ elt = elt->up.next; \
+ } \
+ } \
+ } while (0)
+
+#define UPSTREAM_SELECT_ROUND_ROBIN(head, selected) \
+ do { \
+ __typeof(head) elt = (head); \
+ (selected) = NULL; \
+ unsigned max_weight = 0; \
+ if ((head)->up.common->alive == 0) { \
+ UPSTREAM_REVIVE_ALL(head); \
+ } \
+ while (elt != NULL) { \
+ if (!elt->up.dead) { \
+ if (elt->up.weight > max_weight) { \
+ max_weight = elt->up.weight; \
+ (selected) = elt; \
+ } \
+ } \
+ elt = elt->up.next; \
+ } \
+ if (max_weight == 0) { \
+ elt = (head); \
+ while (elt != NULL) { \
+ elt->up.weight = elt->up.priority; \
+ if (!elt->up.dead) { \
+ if (elt->up.priority > max_weight) { \
+ max_weight = elt->up.priority; \
+ (selected) = elt; \
+ } \
+ } \
+ elt = elt->up.next; \
+ } \
+ } \
+ (selected)->up.weight--; \
+ } while (0)
+
+#endif /* UPSTREAM_H_ */
diff --git a/contrib/librdns/util.c b/contrib/librdns/util.c
new file mode 100644
index 0000000..0172ce0
--- /dev/null
+++ b/contrib/librdns/util.c
@@ -0,0 +1,1035 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <sys/un.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <netdb.h>
+#include <fcntl.h>
+#include <ctype.h>
+
+#include "ottery.h"
+#include "util.h"
+#include "logger.h"
+#include "rdns.h"
+
+inline void
+rdns_request_remove_from_hash (struct rdns_request *req)
+{
+ /* Remove from id hashes */
+ if (req->io) {
+ khiter_t k;
+
+ k = kh_get(rdns_requests_hash, req->io->requests, req->id);
+
+ if (k != kh_end(req->io->requests)) {
+ kh_del(rdns_requests_hash, req->io->requests, k);
+ }
+ }
+}
+
+static int
+rdns_make_socket_nonblocking (int fd)
+{
+ int ofl;
+
+ ofl = fcntl (fd, F_GETFL, 0);
+
+ if (fcntl (fd, F_SETFL, ofl | O_NONBLOCK) == -1) {
+ return -1;
+ }
+ return 0;
+}
+
+static int
+rdns_make_inet_socket (int type, struct addrinfo *addr, struct sockaddr **psockaddr,
+ socklen_t *psocklen)
+{
+ int fd = -1;
+ struct addrinfo *cur;
+
+ cur = addr;
+ while (cur) {
+ /* Create socket */
+ fd = socket (cur->ai_family, type, 0);
+ if (fd == -1) {
+ goto out;
+ }
+
+ if (rdns_make_socket_nonblocking (fd) < 0) {
+ goto out;
+ }
+
+ /* Set close on exec */
+ if (fcntl (fd, F_SETFD, FD_CLOEXEC) == -1) {
+ goto out;
+ }
+
+ if (psockaddr) {
+ *psockaddr = cur->ai_addr;
+ *psocklen = cur->ai_addrlen;
+ }
+ break;
+out:
+ if (fd != -1) {
+ close (fd);
+ }
+ fd = -1;
+ cur = cur->ai_next;
+ }
+
+ return (fd);
+}
+
+static int
+rdns_make_unix_socket (const char *path, struct sockaddr_un *addr, int type)
+{
+ int fd = -1, serrno;
+
+ if (path == NULL) {
+ return -1;
+ }
+
+ addr->sun_family = AF_UNIX;
+
+ memset (addr->sun_path, 0, sizeof (addr->sun_path));
+ memccpy (addr->sun_path, path, 0, sizeof (addr->sun_path) - 1);
+#ifdef FREEBSD
+ addr->sun_len = SUN_LEN (addr);
+#endif
+
+ fd = socket (PF_LOCAL, type, 0);
+
+ if (fd == -1) {
+ return -1;
+ }
+
+ if (rdns_make_socket_nonblocking (fd) < 0) {
+ goto out;
+ }
+
+ /* Set close on exec */
+ if (fcntl (fd, F_SETFD, FD_CLOEXEC) == -1) {
+ goto out;
+ }
+
+ return (fd);
+
+ out:
+ serrno = errno;
+ if (fd != -1) {
+ close (fd);
+ }
+ errno = serrno;
+ return (-1);
+}
+
+/**
+ * Make a universal socket
+ * @param credits host, ip or path to unix socket
+ * @param port port (used for network sockets)
+ * @param async make this socket asynced
+ * @param is_server make this socket as server socket
+ * @param try_resolve try name resolution for a socket (BLOCKING)
+ */
+int
+rdns_make_client_socket (const char *credits,
+ uint16_t port,
+ int type,
+ struct sockaddr **psockaddr,
+ socklen_t *psocklen)
+{
+ struct sockaddr_un un;
+ struct stat st;
+ struct addrinfo hints, *res;
+ int r;
+ char portbuf[8];
+
+ if (*credits == '/') {
+ r = stat (credits, &st);
+ if (r == -1) {
+ /* Unix socket doesn't exists it must be created first */
+ errno = ENOENT;
+ return -1;
+ }
+ else {
+ if ((st.st_mode & S_IFSOCK) == 0) {
+ /* Path is not valid socket */
+ errno = EINVAL;
+ return -1;
+ }
+ else {
+ r = rdns_make_unix_socket (credits, &un, type);
+
+ if (r != -1 && psockaddr) {
+ struct sockaddr *cpy;
+
+ cpy = calloc (1, sizeof (un));
+ *psocklen = sizeof (un);
+
+ if (cpy == NULL) {
+ close (r);
+
+ return -1;
+ }
+
+ memcpy (cpy, &un, *psocklen);
+ *psockaddr = cpy;
+ }
+
+ return r;
+ }
+ }
+ }
+ else {
+ /* TCP related part */
+ memset (&hints, 0, sizeof (hints));
+ hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
+ hints.ai_socktype = type; /* Type of the socket */
+ hints.ai_flags = 0;
+ hints.ai_protocol = 0; /* Any protocol */
+ hints.ai_canonname = NULL;
+ hints.ai_addr = NULL;
+ hints.ai_next = NULL;
+
+ hints.ai_flags |= AI_NUMERICHOST | AI_NUMERICSERV;
+
+ snprintf (portbuf, sizeof (portbuf), "%d", (int)port);
+ if (getaddrinfo (credits, portbuf, &hints, &res) == 0) {
+ r = rdns_make_inet_socket (type, res, psockaddr, psocklen);
+
+ if (r != -1 && psockaddr) {
+ struct sockaddr *cpy;
+
+ cpy = calloc (1, *psocklen);
+
+ if (cpy == NULL) {
+ close (r);
+ freeaddrinfo (res);
+
+ return -1;
+ }
+
+ memcpy (cpy, *psockaddr, *psocklen);
+ *psockaddr = cpy;
+ }
+
+ freeaddrinfo (res);
+ return r;
+ }
+ else {
+ return -1;
+ }
+ }
+
+ /* Not reached */
+ return -1;
+}
+
+const char *
+rdns_strerror (enum dns_rcode rcode)
+{
+ rcode &= 0xf;
+ static char numbuf[16];
+
+ if ('\0' == dns_rcodes[rcode][0]) {
+ snprintf (numbuf, sizeof (numbuf), "UNKNOWN: %d", (int)rcode);
+ return numbuf;
+ }
+ return dns_rcodes[rcode];
+}
+
+const char *
+rdns_strtype (enum rdns_request_type type)
+{
+ return dns_types[type];
+}
+
+enum rdns_request_type
+rdns_type_fromstr (const char *str)
+{
+ if (str) {
+ if (strcmp (str, "a") == 0) {
+ return RDNS_REQUEST_A;
+ }
+ else if (strcmp (str, "ns") == 0) {
+ return RDNS_REQUEST_NS;
+ }
+ else if (strcmp (str, "soa") == 0) {
+ return RDNS_REQUEST_SOA;
+ }
+ else if (strcmp (str, "ptr") == 0) {
+ return RDNS_REQUEST_PTR;
+ }
+ else if (strcmp (str, "mx") == 0) {
+ return RDNS_REQUEST_MX;
+ }
+ else if (strcmp (str, "srv") == 0) {
+ return RDNS_REQUEST_SRV;
+ }
+ else if (strcmp (str, "txt") == 0) {
+ return RDNS_REQUEST_TXT;
+ }
+ else if (strcmp (str, "spf") == 0) {
+ return RDNS_REQUEST_SPF;
+ }
+ else if (strcmp (str, "aaaa") == 0) {
+ return RDNS_REQUEST_AAAA;
+ }
+ else if (strcmp (str, "tlsa") == 0) {
+ return RDNS_REQUEST_TLSA;
+ }
+ else if (strcmp (str, "cname") == 0) {
+ return RDNS_REQUEST_CNAME;
+ }
+ else if (strcmp (str, "any") == 0) {
+ return RDNS_REQUEST_ANY;
+ }
+ }
+
+ return RDNS_REQUEST_INVALID;
+}
+
+const char *
+rdns_str_from_type (enum rdns_request_type rcode)
+{
+ switch (rcode) {
+ case RDNS_REQUEST_INVALID:
+ return "(invalid)";
+ case RDNS_REQUEST_A:
+ return "a";
+ case RDNS_REQUEST_NS:
+ return "ns";
+ case RDNS_REQUEST_SOA:
+ return "soa";
+ case RDNS_REQUEST_PTR:
+ return "ptr";
+ case RDNS_REQUEST_MX:
+ return "mx";
+ case RDNS_REQUEST_TXT:
+ return "txt";
+ case RDNS_REQUEST_SRV:
+ return "srv";
+ case RDNS_REQUEST_SPF:
+ return "spf";
+ case RDNS_REQUEST_AAAA:
+ return "aaaa";
+ case RDNS_REQUEST_TLSA:
+ return "tlsa";
+ case RDNS_REQUEST_CNAME:
+ return "cname";
+ case RDNS_REQUEST_ANY:
+ return "any";
+ default:
+ return "(unknown)";
+ }
+
+}
+
+enum dns_rcode
+rdns_rcode_fromstr (const char *str)
+{
+ if (str) {
+ if (strcmp (str, "noerror") == 0) {
+ return RDNS_RC_NOERROR;
+ }
+ else if (strcmp (str, "formerr") == 0) {
+ return RDNS_RC_FORMERR;
+ }
+ else if (strcmp (str, "servfail") == 0) {
+ return RDNS_RC_SERVFAIL;
+ }
+ else if (strcmp (str, "nxdomain") == 0) {
+ return RDNS_RC_NXDOMAIN;
+ }
+ else if (strcmp (str, "notimp") == 0) {
+ return RDNS_RC_NOTIMP;
+ }
+ else if (strcmp (str, "yxdomain") == 0) {
+ return RDNS_RC_YXDOMAIN;
+ }
+ else if (strcmp (str, "yxrrset") == 0) {
+ return RDNS_RC_YXRRSET;
+ }
+ else if (strcmp (str, "nxrrset") == 0) {
+ return RDNS_RC_NXRRSET;
+ }
+ else if (strcmp (str, "notauth") == 0) {
+ return RDNS_RC_NOTAUTH;
+ }
+ else if (strcmp (str, "notzone") == 0) {
+ return RDNS_RC_NOTZONE;
+ }
+ else if (strcmp (str, "timeout") == 0) {
+ return RDNS_RC_TIMEOUT;
+ }
+ else if (strcmp (str, "neterr") == 0) {
+ return RDNS_RC_NETERR;
+ }
+ else if (strcmp (str, "norec") == 0) {
+ return RDNS_RC_NOREC;
+ }
+ }
+
+ return RDNS_RC_INVALID;
+}
+
+uint16_t
+rdns_permutor_generate_id (void)
+{
+ uint16_t id;
+
+ id = ottery_rand_unsigned ();
+
+ return id;
+}
+
+struct rdns_reply *
+rdns_make_reply (struct rdns_request *req, enum dns_rcode rcode)
+{
+ struct rdns_reply *rep;
+
+ rep = malloc (sizeof (struct rdns_reply));
+ if (rep != NULL) {
+ rep->request = req;
+ rep->resolver = req->resolver;
+ rep->entries = NULL;
+ rep->code = rcode;
+ req->reply = rep;
+ rep->flags = 0;
+ rep->requested_name = req->requested_names[0].name;
+ }
+
+ return rep;
+}
+
+void
+rdns_reply_free (struct rdns_reply *rep)
+{
+ struct rdns_reply_entry *entry, *tmp;
+
+ /* We don't need to free data for faked replies */
+ if (!rep->request || rep->request->state != RDNS_REQUEST_FAKE) {
+ LL_FOREACH_SAFE (rep->entries, entry, tmp) {
+ switch (entry->type) {
+ case RDNS_REQUEST_PTR:
+ free (entry->content.ptr.name);
+ break;
+ case RDNS_REQUEST_NS:
+ free (entry->content.ns.name);
+ break;
+ case RDNS_REQUEST_MX:
+ free (entry->content.mx.name);
+ break;
+ case RDNS_REQUEST_TXT:
+ case RDNS_REQUEST_SPF:
+ free (entry->content.txt.data);
+ break;
+ case RDNS_REQUEST_SRV:
+ free (entry->content.srv.target);
+ break;
+ case RDNS_REQUEST_TLSA:
+ free (entry->content.tlsa.data);
+ break;
+ case RDNS_REQUEST_SOA:
+ free (entry->content.soa.mname);
+ free (entry->content.soa.admin);
+ break;
+ case RDNS_REQUEST_CNAME:
+ free(entry->content.cname.name);
+ break;
+ default:
+ break;
+ }
+ free (entry);
+ }
+ }
+
+ free (rep);
+}
+
+void
+rdns_request_free (struct rdns_request *req)
+{
+ unsigned int i;
+
+ if (req != NULL) {
+ if (req->packet != NULL) {
+ free (req->packet);
+ }
+ for (i = 0; i < req->qcount; i ++) {
+ free (req->requested_names[i].name);
+ }
+ if (req->requested_names != NULL) {
+ free (req->requested_names);
+ }
+ if (req->reply != NULL) {
+ rdns_reply_free (req->reply);
+ }
+ if (req->async_event) {
+ if (req->state == RDNS_REQUEST_WAIT_REPLY) {
+ /* Remove timer */
+ req->async->del_timer (req->async->data,
+ req->async_event);
+ rdns_request_remove_from_hash(req);
+ req->async_event = NULL;
+ }
+ else if (req->state == RDNS_REQUEST_WAIT_SEND) {
+ /* Remove retransmit event */
+ req->async->del_write (req->async->data,
+ req->async_event);
+ rdns_request_remove_from_hash(req);
+ req->async_event = NULL;
+ }
+ else if (req->state == RDNS_REQUEST_FAKE) {
+ req->async->del_write (req->async->data,
+ req->async_event);
+ req->async_event = NULL;
+ }
+ }
+ if (req->state == RDNS_REQUEST_TCP) {
+ if (req->async_event) {
+ req->async->del_timer (req->async->data,
+ req->async_event);
+ }
+
+ rdns_request_remove_from_hash(req);
+ }
+#ifdef TWEETNACL
+ if (req->curve_plugin_data != NULL) {
+ req->resolver->curve_plugin->cb.curve_plugin.finish_cb (
+ req, req->resolver->curve_plugin->data);
+ }
+#endif
+ if (req->io != NULL && req->state > RDNS_REQUEST_NEW) {
+ REF_RELEASE (req->io);
+ REF_RELEASE (req->resolver);
+ }
+
+ free (req);
+ }
+}
+
+void
+rdns_ioc_free (struct rdns_io_channel *ioc)
+{
+ struct rdns_request *req;
+
+ if (IS_CHANNEL_TCP(ioc)) {
+ rdns_ioc_tcp_reset(ioc);
+ }
+
+ kh_foreach_value(ioc->requests, req, {
+ REF_RELEASE (req);
+ });
+
+ if (ioc->async_io) {
+ ioc->resolver->async->del_read(ioc->resolver->async->data,
+ ioc->async_io);
+ }
+ kh_destroy(rdns_requests_hash, ioc->requests);
+
+ if (ioc->sock != -1) {
+ close(ioc->sock);
+ }
+
+ if (ioc->saddr != NULL) {
+ free(ioc->saddr);
+ }
+
+ free (ioc);
+}
+
+struct rdns_io_channel *
+rdns_ioc_new (struct rdns_server *serv,
+ struct rdns_resolver *resolver,
+ bool is_tcp)
+{
+ struct rdns_io_channel *nioc;
+
+ if (is_tcp) {
+ nioc = calloc (1, sizeof (struct rdns_io_channel)
+ + sizeof (struct rdns_tcp_channel));
+ }
+ else {
+ nioc = calloc (1, sizeof (struct rdns_io_channel));
+ }
+
+ if (nioc == NULL) {
+ rdns_err ("calloc fails to allocate rdns_io_channel");
+ return NULL;
+ }
+
+ nioc->struct_magic = RDNS_IO_CHANNEL_TAG;
+ nioc->srv = serv;
+ nioc->resolver = resolver;
+
+ nioc->sock = rdns_make_client_socket (serv->name, serv->port,
+ is_tcp ? SOCK_STREAM : SOCK_DGRAM, &nioc->saddr, &nioc->slen);
+ if (nioc->sock == -1) {
+ rdns_err ("cannot open socket to %s: %s", serv->name,
+ strerror (errno));
+ free (nioc);
+ return NULL;
+ }
+
+ if (is_tcp) {
+ /* We also need to connect a TCP channel and set a TCP buffer */
+ nioc->tcp = (struct rdns_tcp_channel *)(((unsigned char *)nioc) + sizeof(*nioc));
+
+ if (!rdns_ioc_tcp_connect(nioc)) {
+ rdns_err ("cannot connect TCP socket to %s: %s", serv->name,
+ strerror (errno));
+ close (nioc->sock);
+ free (nioc);
+
+ return NULL;
+ }
+
+ nioc->flags |= RDNS_CHANNEL_TCP;
+ }
+ else {
+ nioc->flags |= RDNS_CHANNEL_ACTIVE;
+ nioc->async_io = resolver->async->add_read(resolver->async->data,
+ nioc->sock, nioc);
+ }
+
+ nioc->requests = kh_init(rdns_requests_hash);
+ REF_INIT_RETAIN (nioc, rdns_ioc_free);
+
+ return nioc;
+}
+
+void
+rdns_resolver_release (struct rdns_resolver *resolver)
+{
+ REF_RELEASE (resolver);
+}
+
+struct rdns_request*
+rdns_request_retain (struct rdns_request *req)
+{
+ REF_RETAIN (req);
+ return req;
+}
+
+void
+rdns_request_unschedule (struct rdns_request *req, bool remove_from_hash)
+{
+ struct rdns_resolver *resolver = req->resolver;
+
+ switch (req->state) {
+ case RDNS_REQUEST_WAIT_REPLY:
+ /* We have a timer pending */
+ if (req->async_event) {
+ req->async->del_timer (req->async->data,
+ req->async_event);
+ if (remove_from_hash) {
+ rdns_request_remove_from_hash(req);
+ }
+ req->async_event = NULL;
+ }
+ break;
+ case RDNS_REQUEST_WAIT_SEND:
+ /* We have write request pending */
+ if (req->async_event) {
+ req->async->del_write (req->async->data,
+ req->async_event);
+ /* Remove from id hashes */
+ if (remove_from_hash) {
+ rdns_request_remove_from_hash(req);
+ }
+ req->async_event = NULL;
+ }
+ break;
+ case RDNS_REQUEST_TCP:
+ /* We also have a timer */
+ if (req->async_event) {
+ if (remove_from_hash) {
+ rdns_request_remove_from_hash(req);
+ }
+
+ req->async->del_timer(req->async->data,
+ req->async_event);
+
+ req->async_event = NULL;
+ }
+ default:
+ /* Nothing to unschedule, so blame if we have any event pending */
+ if (req->async_event) {
+ rdns_err("internal error: have unexpected pending async state on stage %d",
+ req->state);
+ }
+ break;
+ }
+}
+
+void
+rdns_request_release (struct rdns_request *req)
+{
+ rdns_request_unschedule (req, true);
+ REF_RELEASE (req);
+}
+
+void
+rdns_ioc_tcp_reset (struct rdns_io_channel *ioc)
+{
+ struct rdns_resolver *resolver = ioc->resolver;
+
+ if (IS_CHANNEL_CONNECTED(ioc)) {
+ if (ioc->tcp->async_write) {
+ resolver->async->del_write (resolver->async->data, ioc->tcp->async_write);
+ ioc->tcp->async_write = NULL;
+ }
+ if (ioc->tcp->async_read) {
+ resolver->async->del_read (resolver->async->data, ioc->tcp->async_read);
+ ioc->tcp->async_read = NULL;
+ }
+
+ /* Clean all buffers and temporaries */
+ if (ioc->tcp->cur_read_buf) {
+ free (ioc->tcp->cur_read_buf);
+ ioc->tcp->read_buf_allocated = 0;
+ ioc->tcp->next_read_size = 0;
+ ioc->tcp->cur_read = 0;
+ ioc->tcp->cur_read_buf = NULL;
+ }
+
+ struct rdns_tcp_output_chain *oc, *tmp;
+ DL_FOREACH_SAFE(ioc->tcp->output_chain, oc, tmp) {
+ DL_DELETE (ioc->tcp->output_chain, oc);
+ free (oc);
+ }
+
+ ioc->tcp->cur_output_chains = 0;
+ ioc->tcp->output_chain = NULL;
+
+ ioc->flags &= ~RDNS_CHANNEL_CONNECTED;
+ }
+
+ /* Remove all requests pending as we are unable to complete them */
+ struct rdns_request *req;
+ kh_foreach_value(ioc->requests, req, {
+ struct rdns_reply *rep = rdns_make_reply (req, RDNS_RC_NETERR);
+ /*
+ * Unschedule request explicitly as we set state to RDNS_REQUEST_REPLIED
+ * that will prevent timer from being removed on req dtor.
+ *
+ * We skip hash removal here, as the hash will be cleared as a single
+ * operation afterwards.
+ */
+ rdns_request_unschedule(req, false);
+ req->state = RDNS_REQUEST_REPLIED;
+ req->func (rep, req->arg);
+ REF_RELEASE (req);
+ });
+
+ if (ioc->sock != -1) {
+ close (ioc->sock);
+ ioc->sock = -1;
+ }
+ if (ioc->saddr) {
+ free (ioc->saddr);
+ ioc->saddr = NULL;
+ }
+
+ kh_clear(rdns_requests_hash, ioc->requests);
+}
+
+bool
+rdns_ioc_tcp_connect (struct rdns_io_channel *ioc)
+{
+ struct rdns_resolver *resolver = ioc->resolver;
+
+ if (IS_CHANNEL_CONNECTED(ioc)) {
+ rdns_err ("trying to connect already connected IO channel!");
+ return false;
+ }
+
+ if (ioc->flags & RDNS_CHANNEL_TCP_CONNECTING) {
+ /* Already connecting channel, ignore connect request */
+
+ return true;
+ }
+
+ if (ioc->sock == -1) {
+ ioc->sock = rdns_make_client_socket (ioc->srv->name, ioc->srv->port,
+ SOCK_STREAM, &ioc->saddr, &ioc->slen);
+ if (ioc->sock == -1) {
+ rdns_err ("cannot open socket to %s: %s", ioc->srv->name,
+ strerror (errno));
+
+ if (ioc->saddr) {
+ free (ioc->saddr);
+ ioc->saddr = NULL;
+ }
+
+ return false;
+ }
+ }
+
+ int r = connect (ioc->sock, ioc->saddr, ioc->slen);
+
+ if (r == -1) {
+ if (errno != EAGAIN && errno != EINTR && errno != EINPROGRESS) {
+ rdns_err ("cannot connect a TCP socket: %s for server %s",
+ strerror(errno), ioc->srv->name);
+ close (ioc->sock);
+
+ if (ioc->saddr) {
+ free (ioc->saddr);
+ ioc->saddr = NULL;
+ }
+
+ ioc->sock = -1;
+
+ return false;
+ }
+ else {
+ /* We need to wait for write readiness here */
+ if (ioc->tcp->async_write != NULL) {
+ rdns_err("internal rdns error: write event is already registered on connect");
+ }
+ else {
+ ioc->tcp->async_write = resolver->async->add_write(resolver->async->data,
+ ioc->sock, ioc);
+ }
+ /* Prevent double connect attempts */
+ ioc->flags |= RDNS_CHANNEL_TCP_CONNECTING;
+ }
+ }
+ else {
+ /* Always be ready to read from a TCP socket */
+ ioc->flags |= RDNS_CHANNEL_CONNECTED|RDNS_CHANNEL_ACTIVE;
+ ioc->flags &= ~RDNS_CHANNEL_TCP_CONNECTING;
+ ioc->tcp->async_read = resolver->async->add_read(resolver->async->data,
+ ioc->sock, ioc);
+ }
+
+ return true;
+}
+
+static bool
+rdns_resolver_conf_process_line (struct rdns_resolver *resolver,
+ const char *line, rdns_resolv_conf_cb cb, void *ud)
+{
+ const char *p, *c, *end;
+ bool has_obrace = false, ret;
+ unsigned int port = dns_port;
+ char *cpy_buf;
+
+ end = line + strlen (line);
+
+ if (end - line > sizeof ("nameserver") - 1 &&
+ strncmp (line, "nameserver", sizeof ("nameserver") - 1) == 0) {
+ p = line + sizeof ("nameserver") - 1;
+ /* Skip spaces */
+ while (isspace (*p)) {
+ p ++;
+ }
+
+ if (*p == '[') {
+ has_obrace = true;
+ p ++;
+ }
+
+ if (isxdigit (*p) || *p == ':') {
+ c = p;
+ while (isxdigit (*p) || *p == ':' || *p == '.') {
+ p ++;
+ }
+ if (has_obrace && *p != ']') {
+ return false;
+ }
+ else if (*p != '\0' && !isspace (*p) && *p != '#') {
+ return false;
+ }
+
+ if (has_obrace) {
+ p ++;
+ if (*p == ':') {
+ /* Maybe we have a port definition */
+ port = strtoul (p + 1, NULL, 10);
+ if (port == 0 || port > UINT16_MAX) {
+ return false;
+ }
+ }
+ }
+
+ cpy_buf = malloc (p - c + 1);
+ assert (cpy_buf != NULL);
+ memcpy (cpy_buf, c, p - c);
+ cpy_buf[p - c] = '\0';
+
+ if (cb == NULL) {
+ ret = rdns_resolver_add_server (resolver, cpy_buf, port, 0,
+ default_io_cnt) != NULL;
+ }
+ else {
+ ret = cb (resolver, cpy_buf, port, 0,
+ default_io_cnt, ud);
+ }
+
+ free (cpy_buf);
+
+ return ret;
+ }
+ else {
+ return false;
+ }
+ }
+ /* XXX: skip unknown resolv.conf lines */
+
+ return false;
+}
+
+bool
+rdns_resolver_parse_resolv_conf_cb (struct rdns_resolver *resolver,
+ const char *path, rdns_resolv_conf_cb cb, void *ud)
+{
+ FILE *in;
+ char buf[BUFSIZ];
+ char *p;
+ bool processed = false;
+
+ in = fopen (path, "r");
+
+ if (in == NULL) {
+ return false;
+ }
+
+ while (!feof (in)) {
+ if (fgets (buf, sizeof (buf) - 1, in) == NULL) {
+ break;
+ }
+
+ /* Strip trailing spaces */
+ p = buf + strlen (buf) - 1;
+ while (p > buf &&
+ (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n')) {
+ *p-- = '\0';
+ }
+
+ if (rdns_resolver_conf_process_line (resolver, buf, cb, ud)) {
+ processed = true;
+ }
+ }
+
+ fclose (in);
+
+ return processed;
+}
+
+bool
+rdns_resolver_parse_resolv_conf (struct rdns_resolver *resolver, const char *path)
+{
+ return rdns_resolver_parse_resolv_conf_cb (resolver, path, NULL, NULL);
+}
+
+bool
+rdns_request_has_type (struct rdns_request *req, enum rdns_request_type type)
+{
+ unsigned int i;
+
+ for (i = 0; i < req->qcount; i ++) {
+ if (req->requested_names[i].type == type) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+const struct rdns_request_name *
+rdns_request_get_name (struct rdns_request *req, unsigned int *count)
+{
+
+ if (count != NULL) {
+ *count = req->qcount;
+ }
+ return req->requested_names;
+}
+
+const char*
+rdns_request_get_server (struct rdns_request *req)
+{
+ if (req && req->io) {
+ return req->io->srv->name;
+ }
+
+ return NULL;
+}
+
+char *
+rdns_generate_ptr_from_str (const char *str)
+{
+ union {
+ struct in_addr v4;
+ struct in6_addr v6;
+ } addr;
+ char *res = NULL;
+ unsigned char *bytes;
+ size_t len;
+
+ if (inet_pton (AF_INET, str, &addr.v4) == 1) {
+ bytes = (unsigned char *)&addr.v4;
+
+ len = 4 * 4 + sizeof ("in-addr.arpa");
+ res = malloc (len);
+ if (res) {
+ snprintf (res, len, "%u.%u.%u.%u.in-addr.arpa",
+ (unsigned)bytes[3]&0xFF,
+ (unsigned)bytes[2]&0xFF,
+ (unsigned)bytes[1]&0xFF,
+ (unsigned)bytes[0]&0xFF);
+ }
+ }
+ else if (inet_pton (AF_INET6, str, &addr.v6) == 1) {
+ bytes = (unsigned char *)&addr.v6;
+
+ len = 2*32 + sizeof ("ip6.arpa");
+ res = malloc (len);
+ if (res) {
+ snprintf(res, len,
+ "%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x."
+ "%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.%x.ip6.arpa",
+ bytes[15]&0xF, bytes[15] >> 4, bytes[14]&0xF, bytes[14] >> 4,
+ bytes[13]&0xF, bytes[13] >> 4, bytes[12]&0xF, bytes[12] >> 4,
+ bytes[11]&0xF, bytes[11] >> 4, bytes[10]&0xF, bytes[10] >> 4,
+ bytes[9]&0xF, bytes[9] >> 4, bytes[8]&0xF, bytes[8] >> 4,
+ bytes[7]&0xF, bytes[7] >> 4, bytes[6]&0xF, bytes[6] >> 4,
+ bytes[5]&0xF, bytes[5] >> 4, bytes[4]&0xF, bytes[4] >> 4,
+ bytes[3]&0xF, bytes[3] >> 4, bytes[2]&0xF, bytes[2] >> 4,
+ bytes[1]&0xF, bytes[1] >> 4, bytes[0]&0xF, bytes[0] >> 4);
+ }
+ }
+
+ return res;
+}
diff --git a/contrib/librdns/util.h b/contrib/librdns/util.h
new file mode 100644
index 0000000..6f74d8b
--- /dev/null
+++ b/contrib/librdns/util.h
@@ -0,0 +1,99 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef UTIL_H_
+#define UTIL_H_
+
+#include "dns_private.h"
+
+/**
+ * Make a universal socket
+ * @param credits host, ip or path to unix socket
+ * @param port port (used for network sockets)
+ * @param type of socket (SOCK_STREAM or SOCK_DGRAM)
+ */
+int
+rdns_make_client_socket (const char *credits,
+ uint16_t port,
+ int type,
+ struct sockaddr **psockaddr,
+ socklen_t *psocklen);
+
+/**
+ * Generate new random DNS id
+ * @return dns id
+ */
+uint16_t rdns_permutor_generate_id (void);
+
+
+/**
+ * Free IO channel
+ */
+void rdns_ioc_free (struct rdns_io_channel *ioc);
+
+/**
+ * Creates a new IO channel
+ */
+struct rdns_io_channel * rdns_ioc_new (struct rdns_server *srv,
+ struct rdns_resolver *resolver,
+ bool is_tcp);
+
+/**
+ * Resets inactive/errored TCP chain as recommended by RFC
+ * @param ioc
+ */
+void rdns_ioc_tcp_reset (struct rdns_io_channel *ioc);
+
+/**
+ * Connect TCP IO channel to a server
+ * @param ioc
+ */
+bool rdns_ioc_tcp_connect (struct rdns_io_channel *ioc);
+
+/**
+ * Free request
+ * @param req
+ */
+void rdns_request_free (struct rdns_request *req);
+
+/**
+ * Removes request from a channel's hash (e.g. if needed to migrate to another channel)
+ * @param req
+ */
+void rdns_request_remove_from_hash (struct rdns_request *req);
+
+/**
+ * Creates a new reply
+ * @param req
+ * @param rcode
+ * @return
+ */
+struct rdns_reply * rdns_make_reply (struct rdns_request *req, enum dns_rcode rcode);
+/**
+ * Free reply
+ * @param rep
+ */
+void rdns_reply_free (struct rdns_reply *rep);
+
+void rdns_request_unschedule (struct rdns_request *req, bool remove_from_hash);
+
+#endif /* UTIL_H_ */
diff --git a/contrib/libucl/CMakeLists.txt b/contrib/libucl/CMakeLists.txt
new file mode 100644
index 0000000..eb78701
--- /dev/null
+++ b/contrib/libucl/CMakeLists.txt
@@ -0,0 +1,19 @@
+SET(UCLSRC ucl_util.c
+ ucl_parser.c
+ ucl_emitter.c
+ ucl_emitter_streamline.c
+ ucl_hash.c
+ ucl_schema.c
+ lua_ucl.c
+ ucl_msgpack.c
+ ucl_sexp.c)
+
+
+SET (LIB_TYPE STATIC)
+ADD_LIBRARY(ucl ${LIB_TYPE} ${UCLSRC})
+
+IF(ENABLE_URL_SIGN MATCHES "ON")
+ IF(OPENSSL_FOUND)
+ TARGET_LINK_LIBRARIES(ucl ${OPENSSL_LIBRARIES})
+ ENDIF(OPENSSL_FOUND)
+ENDIF(ENABLE_URL_SIGN MATCHES "ON")
diff --git a/contrib/libucl/README.md b/contrib/libucl/README.md
new file mode 100644
index 0000000..146143d
--- /dev/null
+++ b/contrib/libucl/README.md
@@ -0,0 +1,397 @@
+# LIBUCL
+
+[![CircleCI](https://circleci.com/gh/vstakhov/libucl.svg?style=svg)](https://circleci.com/gh/vstakhov/libucl)
+[![Coverity](https://scan.coverity.com/projects/4138/badge.svg)](https://scan.coverity.com/projects/4138)
+[![Coverage Status](https://coveralls.io/repos/github/vstakhov/libucl/badge.svg?branch=master)](https://coveralls.io/github/vstakhov/libucl?branch=master)
+
+**Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)*
+
+- [Introduction](#introduction)
+- [Basic structure](#basic-structure)
+- [Improvements to the json notation](#improvements-to-the-json-notation)
+ - [General syntax sugar](#general-syntax-sugar)
+ - [Automatic arrays creation](#automatic-arrays-creation)
+ - [Named keys hierarchy](#named-keys-hierarchy)
+ - [Convenient numbers and booleans](#convenient-numbers-and-booleans)
+- [General improvements](#general-improvements)
+ - [Comments](#comments)
+ - [Macros support](#macros-support)
+ - [Variables support](#variables-support)
+ - [Multiline strings](#multiline-strings)
+ - [Single quoted strings](#single-quoted-strings)
+- [Emitter](#emitter)
+- [Validation](#validation)
+- [Performance](#performance)
+- [Conclusion](#conclusion)
+
+## Introduction
+
+This document describes the main features and principles of the configuration
+language called `UCL` - universal configuration language.
+
+If you are looking for the libucl API documentation you can find it at [this page](doc/api.md).
+
+## Basic structure
+
+UCL is heavily infused by `nginx` configuration as the example of a convenient configuration
+system. However, UCL is fully compatible with `JSON` format and is able to parse json files.
+For example, you can write the same configuration in the following ways:
+
+* in nginx like:
+
+```nginx
+param = value;
+section {
+ param = value;
+ param1 = value1;
+ flag = true;
+ number = 10k;
+ time = 0.2s;
+ string = "something";
+ subsection {
+ host = {
+ host = "hostname";
+ port = 900;
+ }
+ host = {
+ host = "hostname";
+ port = 901;
+ }
+ }
+}
+```
+
+* or in JSON:
+
+```json
+{
+ "param": "value",
+ "param1": "value1",
+ "flag": true,
+ "subsection": {
+ "host": [
+ {
+ "host": "hostname",
+ "port": 900
+ },
+ {
+ "host": "hostname",
+ "port": 901
+ }
+ ]
+ }
+}
+```
+
+## Improvements to the json notation.
+
+There are various things that make ucl configuration more convenient for editing than strict json:
+
+### General syntax sugar
+
+* Braces are not necessary to enclose a top object: it is automatically treated as an object:
+
+```json
+"key": "value"
+```
+is equal to:
+```json
+{"key": "value"}
+```
+
+* There is no requirement of quotes for strings and keys, moreover, `:` may be replaced `=` or even be skipped for objects:
+
+```nginx
+key = value;
+section {
+ key = value;
+}
+```
+is equal to:
+```json
+{
+ "key": "value",
+ "section": {
+ "key": "value"
+ }
+}
+```
+
+* No commas mess: you can safely place a comma or semicolon for the last element in an array or an object:
+
+```json
+{
+ "key1": "value",
+ "key2": "value",
+}
+```
+### Automatic arrays creation
+
+* Non-unique keys in an object are allowed and are automatically converted to the arrays internally:
+
+```json
+{
+ "key": "value1",
+ "key": "value2"
+}
+```
+is converted to:
+```json
+{
+ "key": ["value1", "value2"]
+}
+```
+
+### Named keys hierarchy
+
+UCL accepts named keys and organize them into objects hierarchy internally. Here is an example of this process:
+```nginx
+section "blah" {
+ key = value;
+}
+section foo {
+ key = value;
+}
+```
+
+is converted to the following object:
+
+```nginx
+section {
+ blah {
+ key = value;
+ }
+ foo {
+ key = value;
+ }
+}
+```
+
+Plain definitions may be more complex and contain more than a single level of nested objects:
+
+```nginx
+section "blah" "foo" {
+ key = value;
+}
+```
+
+is presented as:
+
+```nginx
+section {
+ blah {
+ foo {
+ key = value;
+ }
+ }
+}
+```
+
+### Convenient numbers and booleans
+
+* Numbers can have suffixes to specify standard multipliers:
+ + `[kKmMgG]` - standard 10 base multipliers (so `1k` is translated to 1000)
+ + `[kKmMgG]b` - 2 power multipliers (so `1kb` is translated to 1024)
+ + `[s|min|d|w|y]` - time multipliers, all time values are translated to float number of seconds, for example `10min` is translated to 600.0 and `10ms` is translated to 0.01
+* Hexadecimal integers can be used by `0x` prefix, for example `key = 0xff`. However, floating point values can use decimal base only.
+* Booleans can be specified as `true` or `yes` or `on` and `false` or `no` or `off`.
+* It is still possible to treat numbers and booleans as strings by enclosing them in double quotes.
+
+## General improvements
+
+### Comments
+
+UCL supports different style of comments:
+
+* single line: `#`
+* multiline: `/* ... */`
+
+Multiline comments may be nested:
+```c
+# Sample single line comment
+/*
+ some comment
+ /* nested comment */
+ end of comment
+*/
+```
+
+### Macros support
+
+UCL supports external macros both multiline and single line ones:
+```nginx
+.macro_name "sometext";
+.macro_name {
+ Some long text
+ ....
+};
+```
+
+Moreover, each macro can accept an optional list of arguments in braces. These
+arguments themselves are the UCL object that is parsed and passed to a macro as
+options:
+
+```nginx
+.macro_name(param=value) "something";
+.macro_name(param={key=value}) "something";
+.macro_name(.include "params.conf") "something";
+.macro_name(#this is multiline macro
+param = [value1, value2]) "something";
+.macro_name(key="()") "something";
+```
+
+UCL also provide a convenient `include` macro to load content from another files
+to the current UCL object. This macro accepts either path to file:
+
+```nginx
+.include "/full/path.conf"
+.include "./relative/path.conf"
+.include "${CURDIR}/path.conf"
+```
+
+or URL (if ucl is built with url support provided by either `libcurl` or `libfetch`):
+
+ .include "http://example.com/file.conf"
+
+`.include` macro supports a set of options:
+
+* `try` (default: **false**) - if this option is `true` than UCL treats errors on loading of
+this file as non-fatal. For example, such a file can be absent but it won't stop the parsing
+of the top-level document.
+* `sign` (default: **false**) - if this option is `true` UCL loads and checks the signature for
+a file from path named `<FILEPATH>.sig`. Trusted public keys should be provided for UCL API after
+parser is created but before any configurations are parsed.
+* `glob` (default: **false**) - if this option is `true` UCL treats the filename as GLOB pattern and load
+all files that matches the specified pattern (normally the format of patterns is defined in `glob` manual page
+for your operating system). This option is meaningless for URL includes.
+* `url` (default: **true**) - allow URL includes.
+* `path` (default: empty) - A UCL_ARRAY of directories to search for the include file.
+Search ends after the first match, unless `glob` is true, then all matches are included.
+* `prefix` (default false) - Put included contents inside an object, instead
+of loading them into the root. If no `key` is provided, one is automatically generated based on each files basename()
+* `key` (default: <empty string>) - Key to load contents of include into. If
+the key already exists, it must be the correct type
+* `target` (default: object) - Specify if the `prefix` `key` should be an
+object or an array.
+* `priority` (default: 0) - specify priority for the include (see below).
+* `duplicate` (default: 'append') - specify policy of duplicates resolving:
+ - `append` - default strategy, if we have new object of higher priority then it replaces old one, if we have new object with less priority it is ignored completely, and if we have two duplicate objects with the same priority then we have a multi-value key (implicit array)
+ - `merge` - if we have object or array, then new keys are merged inside, if we have a plain object then an implicit array is formed (regardless of priorities)
+ - `error` - create error on duplicate keys and stop parsing
+ - `rewrite` - always rewrite an old value with new one (ignoring priorities)
+
+Priorities are used by UCL parser to manage the policy of objects rewriting during including other files
+as following:
+
+* If we have two objects with the same priority then we form an implicit array
+* If a new object has bigger priority then we overwrite an old one
+* If a new object has lower priority then we ignore it
+
+By default, the priority of top-level object is set to zero (lowest priority). Currently,
+you can define up to 16 priorities (from 0 to 15). Includes with bigger priorities will
+rewrite keys from the objects with lower priorities as specified by the policy.
+
+### Variables support
+
+UCL supports variables in input. Variables are registered by a user of the UCL parser and can be presented in the following forms:
+
+* `${VARIABLE}`
+* `$VARIABLE`
+
+UCL currently does not support nested variables. To escape variables one could use double dollar signs:
+
+* `$${VARIABLE}` is converted to `${VARIABLE}`
+* `$$VARIABLE` is converted to `$VARIABLE`
+
+However, if no valid variables are found in a string, no expansion will be performed (and `$$` thus remains unchanged). This may be a subject
+to change in future libucl releases.
+
+### Multiline strings
+
+UCL can handle multiline strings as well as single line ones. It uses shell/perl like notation for such objects:
+```
+key = <<EOD
+some text
+splitted to
+lines
+EOD
+```
+
+In this example `key` will be interpreted as the following string: `some text\nsplitted to\nlines`.
+Here are some rules for this syntax:
+
+* Multiline terminator must start just after `<<` symbols and it must consist of capital letters only (e.g. `<<eof` or `<< EOF` won't work);
+* Terminator must end with a single newline character (and no spaces are allowed between terminator and newline character);
+* To finish multiline string you need to include a terminator string just after newline and followed by a newline (no spaces or other characters are allowed as well);
+* The initial and the final newlines are not inserted to the resulting string, but you can still specify newlines at the beginning and at the end of a value, for example:
+
+```
+key <<EOD
+
+some
+text
+
+EOD
+```
+
+### Single quoted strings
+
+It is possible to use single quoted strings to simplify escaping rules. All values passed in single quoted strings are *NOT* escaped, with two exceptions: a single `'` character just before `\` character, and a newline character just after `\` character that is ignored.
+
+```
+key = 'value'; # Read as value
+key = 'value\n\'; # Read as value\n\
+key = 'value\''; # Read as value'
+key = 'value\
+bla'; # Read as valuebla
+```
+
+## Emitter
+
+Each UCL object can be serialized to one of the three supported formats:
+
+* `JSON` - canonic json notation (with spaces indented structure);
+* `Compacted JSON` - compact json notation (without spaces or newlines);
+* `Configuration` - nginx like notation;
+* `YAML` - yaml inlined notation.
+
+## Validation
+
+UCL allows validation of objects. It uses the same schema that is used for json: [json schema v4](http://json-schema.org). UCL supports the full set of json schema with the exception of remote references. This feature is unlikely useful for configuration objects. Of course, a schema definition can be in UCL format instead of JSON that simplifies schemas writing. Moreover, since UCL supports multiple values for keys in an object it is possible to specify generic integer constraints `maxValues` and `minValues` to define the limits of values count in a single key. UCL currently is not absolutely strict about validation schemas themselves, therefore UCL users should supply valid schemas (as it is defined in json-schema draft v4) to ensure that the input objects are validated properly.
+
+## Performance
+
+Are UCL parser and emitter fast enough? Well, there are some numbers.
+I got a 19Mb file that consist of ~700 thousand lines of json (obtained via
+http://www.json-generator.com/). Then I checked jansson library that performs json
+parsing and emitting and compared it with UCL. Here are results:
+
+```
+jansson: parsed json in 1.3899 seconds
+jansson: emitted object in 0.2609 seconds
+
+ucl: parsed input in 0.6649 seconds
+ucl: emitted config in 0.2423 seconds
+ucl: emitted json in 0.2329 seconds
+ucl: emitted compact json in 0.1811 seconds
+ucl: emitted yaml in 0.2489 seconds
+```
+
+So far, UCL seems to be significantly faster than jansson on parsing and slightly faster on emitting. Moreover,
+UCL compiled with optimizations (-O3) performs significantly faster:
+```
+ucl: parsed input in 0.3002 seconds
+ucl: emitted config in 0.1174 seconds
+ucl: emitted json in 0.1174 seconds
+ucl: emitted compact json in 0.0991 seconds
+ucl: emitted yaml in 0.1354 seconds
+```
+
+You can do your own benchmarks by running `make check` in libucl top directory.
+
+## Conclusion
+
+UCL has clear design that should be very convenient for reading and writing. At the same time it is compatible with
+JSON language and therefore can be used as a simple JSON parser. Macro logic provides an ability to extend configuration
+language (for example by including some lua code) and comments allow to disable or enable the parts of a configuration
+quickly.
diff --git a/contrib/libucl/khash.h b/contrib/libucl/khash.h
new file mode 100644
index 0000000..1499d75
--- /dev/null
+++ b/contrib/libucl/khash.h
@@ -0,0 +1,682 @@
+/* The MIT License
+
+ Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor@live.co.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+/*
+ An example:
+
+#include "khash.h"
+KHASH_MAP_INIT_INT(32, char)
+int main() {
+ int ret, is_missing;
+ khiter_t k;
+ khash_t(32) *h = kh_init(32);
+ k = kh_put(32, h, 5, &ret);
+ kh_value(h, k) = 10;
+ k = kh_get(32, h, 10);
+ is_missing = (k == kh_end(h));
+ k = kh_get(32, h, 5);
+ kh_del(32, h, k);
+ for (k = kh_begin(h); k != kh_end(h); ++k)
+ if (kh_exist(h, k)) kh_value(h, k) = 1;
+ kh_destroy(32, h);
+ return 0;
+}
+*/
+
+/*
+ 2013-05-02 (0.2.8):
+
+ * Use quadratic probing. When the capacity is power of 2, stepping function
+ i*(i+1)/2 guarantees to traverse each bucket. It is better than double
+ hashing on cache performance and is more robust than linear probing.
+
+ In theory, double hashing should be more robust than quadratic probing.
+ However, my implementation is probably not for large hash tables, because
+ the second hash function is closely tied to the first hash function,
+ which reduce the effectiveness of double hashing.
+
+ Reference: http://research.cs.vt.edu/AVresearch/hashing/quadratic.php
+
+ 2011-12-29 (0.2.7):
+
+ * Minor code clean up; no actual effect.
+
+ 2011-09-16 (0.2.6):
+
+ * The capacity is a power of 2. This seems to dramatically improve the
+ speed for simple keys. Thank Zilong Tan for the suggestion. Reference:
+
+ - http://code.google.com/p/ulib/
+ - http://nothings.org/computer/judy/
+
+ * Allow to optionally use linear probing which usually has better
+ performance for random input. Double hashing is still the default as it
+ is more robust to certain non-random input.
+
+ * Added Wang's integer hash function (not used by default). This hash
+ function is more robust to certain non-random input.
+
+ 2011-02-14 (0.2.5):
+
+ * Allow to declare global functions.
+
+ 2009-09-26 (0.2.4):
+
+ * Improve portability
+
+ 2008-09-19 (0.2.3):
+
+ * Corrected the example
+ * Improved interfaces
+
+ 2008-09-11 (0.2.2):
+
+ * Improved speed a little in kh_put()
+
+ 2008-09-10 (0.2.1):
+
+ * Added kh_clear()
+ * Fixed a compiling error
+
+ 2008-09-02 (0.2.0):
+
+ * Changed to token concatenation which increases flexibility.
+
+ 2008-08-31 (0.1.2):
+
+ * Fixed a bug in kh_get(), which has not been tested previously.
+
+ 2008-08-31 (0.1.1):
+
+ * Added destructor
+*/
+
+
+#ifndef __AC_KHASH_H
+#define __AC_KHASH_H
+
+/*!
+ @header
+
+ Generic hash table library.
+ */
+
+#define AC_VERSION_KHASH_H "0.2.8"
+
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+/* compiler specific configuration */
+
+#if UINT_MAX == 0xffffffffu
+typedef unsigned int khint32_t;
+#elif ULONG_MAX == 0xffffffffu
+typedef unsigned long khint32_t;
+#endif
+
+#if ULONG_MAX == ULLONG_MAX
+typedef unsigned long khint64_t;
+#else
+typedef unsigned long long khint64_t;
+#endif
+
+#ifndef kh_inline
+#ifdef _MSC_VER
+#define kh_inline __inline
+#else
+#define kh_inline inline
+#endif
+#endif /* kh_inline */
+
+#ifndef kh_unused
+# ifdef __GNUC__
+# define kh_unused(x) __attribute__((__unused__)) x
+# else
+# define kh_unused(x) x
+# endif
+#endif
+
+typedef khint32_t khint_t;
+typedef khint_t khiter_t;
+
+#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
+#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
+#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
+#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
+#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
+#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
+#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
+
+#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4)
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#ifndef kcalloc
+#define kcalloc(N,Z) calloc(N,Z)
+#endif
+#ifndef kmalloc
+#define kmalloc(Z) malloc(Z)
+#endif
+#ifndef krealloc
+#define krealloc(P,Z) realloc(P,Z)
+#endif
+#ifndef kfree
+#define kfree(P) free(P)
+#endif
+
+static const double __ac_HASH_UPPER = 0.77;
+
+#define __KHASH_TYPE(name, khkey_t, khval_t) \
+ typedef struct kh_##name##_s { \
+ khint_t n_buckets, size, n_occupied, upper_bound; \
+ khint32_t *flags; \
+ khkey_t *keys; \
+ khval_t *vals; \
+ } kh_##name##_t
+
+#define __KHASH_PROTOTYPES(name, khkey_t, khval_t) \
+ extern kh_##name##_t * kh_init_##name(void); \
+ extern void kh_static_init_##name(kh_##name##_t *); \
+ extern void kh_destroy_##name(kh_##name##_t *h); \
+ extern void kh_static_destroy_##name(kh_##name##_t *h); \
+ extern void kh_clear_##name(kh_##name##_t *h); \
+ extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \
+ extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
+ extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
+ extern void kh_del_##name(kh_##name##_t *h, khint_t x);
+
+#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+ SCOPE kh_##name##_t *kh_init_##name(void) { \
+ return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t)); \
+ } \
+ SCOPE void kh_unused(kh_static_init_##name)(kh_##name##_t *target) {\
+ memset(target, 0, sizeof(*target)); \
+ } \
+ SCOPE void kh_destroy_##name(kh_##name##_t *h) \
+ { \
+ if (h) { \
+ kfree((void *)h->keys); kfree(h->flags); \
+ kfree((void *)h->vals); \
+ kfree(h); \
+ } \
+ } \
+ SCOPE void kh_unused(kh_static_destroy_##name)(kh_##name##_t *h) { \
+ kfree((void *)h->keys); kfree(h->flags); \
+ kfree((void *)h->vals); \
+ } \
+ SCOPE void kh_unused(kh_clear_##name)(kh_##name##_t *h) \
+ { \
+ if (h && h->flags) { \
+ memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \
+ h->size = h->n_occupied = 0; \
+ } \
+ } \
+ SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \
+ { \
+ if (h->n_buckets) { \
+ khint_t k, i, last, mask, step = 0; \
+ mask = h->n_buckets - 1; \
+ k = __hash_func(key); i = k & mask; \
+ last = i; \
+ while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+ i = (i + (++step)) & mask; \
+ if (i == last) return h->n_buckets; \
+ } \
+ return __ac_iseither(h->flags, i)? h->n_buckets : i; \
+ } else return 0; \
+ } \
+ SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
+ { /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \
+ khint32_t *new_flags = 0; \
+ khint_t j = 1; \
+ { \
+ kroundup32(new_n_buckets); \
+ if (new_n_buckets < 4) new_n_buckets = 4; \
+ if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; /* requested size is too small */ \
+ else { /* hash table size to be changed (shrink or expand); rehash */ \
+ new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
+ if (!new_flags) return -1; \
+ memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
+ if (h->n_buckets < new_n_buckets) { /* expand */ \
+ khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+ if (!new_keys) { kfree(new_flags); return -1; } \
+ h->keys = new_keys; \
+ if (kh_is_map) { \
+ khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+ if (!new_vals) { kfree(new_flags); return -1; } \
+ h->vals = new_vals; \
+ } \
+ } /* otherwise shrink */ \
+ } \
+ } \
+ if (j) { /* rehashing is needed */ \
+ for (j = 0; j != h->n_buckets; ++j) { \
+ if (__ac_iseither(h->flags, j) == 0) { \
+ khkey_t key = h->keys[j]; \
+ khval_t val; \
+ khint_t new_mask; \
+ new_mask = new_n_buckets - 1; \
+ if (kh_is_map) val = h->vals[j]; \
+ __ac_set_isdel_true(h->flags, j); \
+ while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \
+ khint_t k, i, step = 0; \
+ k = __hash_func(key); \
+ i = k & new_mask; \
+ while (!__ac_isempty(new_flags, i)) i = (i + (++step)) & new_mask; \
+ __ac_set_isempty_false(new_flags, i); \
+ if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \
+ { khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
+ if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
+ __ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \
+ } else { /* write the element and jump out of the loop */ \
+ h->keys[i] = key; \
+ if (kh_is_map) h->vals[i] = val; \
+ break; \
+ } \
+ } \
+ } \
+ } \
+ if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \
+ h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+ if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+ } \
+ kfree(h->flags); /* free the working space */ \
+ h->flags = new_flags; \
+ h->n_buckets = new_n_buckets; \
+ h->n_occupied = h->size; \
+ h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
+ } \
+ return 0; \
+ } \
+ SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
+ { \
+ khint_t x; \
+ if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \
+ if (h->n_buckets > (h->size<<1)) { \
+ if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \
+ *ret = -1; return h->n_buckets; \
+ } \
+ } else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \
+ *ret = -1; return h->n_buckets; \
+ } \
+ } /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
+ { \
+ khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0; \
+ x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \
+ if (__ac_isempty(h->flags, i)) x = i; /* for speed up */ \
+ else { \
+ last = i; \
+ while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+ if (__ac_isdel(h->flags, i)) site = i; \
+ i = (i + (++step)) & mask; \
+ if (i == last) { x = site; break; } \
+ } \
+ if (x == h->n_buckets) { \
+ if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
+ else x = i; \
+ } \
+ } \
+ } \
+ if (__ac_isempty(h->flags, x)) { /* not present at all */ \
+ h->keys[x] = key; \
+ __ac_set_isboth_false(h->flags, x); \
+ ++h->size; ++h->n_occupied; \
+ *ret = 1; \
+ } else if (__ac_isdel(h->flags, x)) { /* deleted */ \
+ h->keys[x] = key; \
+ __ac_set_isboth_false(h->flags, x); \
+ ++h->size; \
+ *ret = 2; \
+ } else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \
+ return x; \
+ } \
+ SCOPE void kh_unused(kh_del_##name)(kh_##name##_t *h, khint_t x) \
+ { \
+ if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \
+ __ac_set_isdel_true(h->flags, x); \
+ --h->size; \
+ } \
+ }
+
+#define KHASH_DECLARE(name, khkey_t, khval_t) \
+ __KHASH_TYPE(name, khkey_t, khval_t); \
+ __KHASH_PROTOTYPES(name, khkey_t, khval_t)
+
+#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+ __KHASH_TYPE(name, khkey_t, khval_t); \
+ __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+ KHASH_INIT2(name, static kh_inline, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+/* --- BEGIN OF HASH FUNCTIONS --- */
+
+/*! @function
+ @abstract Integer hash function
+ @param key The integer [khint32_t]
+ @return The hash value [khint_t]
+ */
+#define kh_int_hash_func(key) (khint32_t)(key)
+/*! @function
+ @abstract Integer comparison function
+ */
+#define kh_int_hash_equal(a, b) ((a) == (b))
+/*! @function
+ @abstract 64-bit integer hash function
+ @param key The integer [khint64_t]
+ @return The hash value [khint_t]
+ */
+#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11)
+/*! @function
+ @abstract 64-bit integer comparison function
+ */
+#define kh_int64_hash_equal(a, b) ((a) == (b))
+/*! @function
+ @abstract const char* hash function
+ @param s Pointer to a null terminated string
+ @return The hash value
+ */
+static kh_inline khint_t __ac_X31_hash_string(const char *s)
+{
+ khint_t h = (khint_t)*s;
+ if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s;
+ return h;
+}
+
+/**
+ * Wyhash implementation from https://github.com/wangyi-fudan/wyhash
+ */
+static inline unsigned _wyr32(const uint8_t *p) { unsigned v; memcpy(&v, p, 4); return v;}
+static inline unsigned _wyr24(const uint8_t *p, unsigned k) { return (((unsigned)p[0])<<16)|(((unsigned)p[k>>1])<<8)|p[k-1];}
+static inline void _wymix32(unsigned *A, unsigned *B){
+ uint64_t c=*A^0x53c5ca59u; c*=*B^0x74743c1bu;
+ *A=(unsigned)c;
+ *B=(unsigned)(c>>32);
+}
+// This version is vulnerable when used with a few bad seeds, which should be skipped beforehand:
+// 0x429dacdd, 0xd637dbf3
+static inline unsigned _wyhash32(const void *key, uint64_t len, unsigned seed) {
+ const uint8_t *p=(const uint8_t *)key; uint64_t i=len;
+ unsigned see1=(unsigned)len; seed^=(unsigned)(len>>32); _wymix32(&seed, &see1);
+ for(;i>8;i-=8,p+=8){ seed^=_wyr32(p); see1^=_wyr32(p+4); _wymix32(&seed, &see1); }
+ if(i>=4){ seed^=_wyr32(p); see1^=_wyr32(p+i-4); } else if (i) seed^=_wyr24(p,(unsigned)i);
+ _wymix32(&seed, &see1); _wymix32(&seed, &see1); return seed^see1;
+}
+
+/*! @function
+ @abstract Another interface to const char* hash function
+ @param key Pointer to a null terminated string [const char*]
+ @return The hash value [khint_t]
+ */
+#define kh_str_hash_func(key) _wyhash32(key, strlen(key), 0)
+/*! @function
+ @abstract Const char* comparison function
+ */
+#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
+
+static kh_inline khint_t __ac_Wang_hash(khint_t key)
+{
+ key += ~(key << 15);
+ key ^= (key >> 10);
+ key += (key << 3);
+ key ^= (key >> 6);
+ key += ~(key << 11);
+ key ^= (key >> 16);
+ return key;
+}
+#define kh_int_hash_func2(k) __ac_Wang_hash((khint_t)key)
+
+/* --- END OF HASH FUNCTIONS --- */
+
+/* Other convenient macros... */
+
+/*!
+ @abstract Type of the hash table.
+ @param name Name of the hash table [symbol]
+ */
+#define khash_t(name) kh_##name##_t
+
+/*! @function
+ @abstract Initiate a hash table.
+ @param name Name of the hash table [symbol]
+ @return Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_init(name) kh_init_##name()
+#define kh_static_init(name, h) kh_static_init_##name(h)
+
+/*! @function
+ @abstract Destroy a hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_destroy(name, h) kh_destroy_##name(h)
+#define kh_static_destroy(name, h) kh_static_destroy_##name(h)
+
+/*! @function
+ @abstract Reset a hash table without deallocating memory.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_clear(name, h) kh_clear_##name(h)
+
+/*! @function
+ @abstract Resize a hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param s New size [khint_t]
+ */
+#define kh_resize(name, h, s) kh_resize_##name(h, s)
+
+/*! @function
+ @abstract Insert a key to the hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param k Key [type of keys]
+ @param r Extra return code: -1 if the operation failed;
+ 0 if the key is present in the hash table;
+ 1 if the bucket is empty (never used); 2 if the element in
+ the bucket has been deleted [int*]
+ @return Iterator to the inserted element [khint_t]
+ */
+#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
+
+/*! @function
+ @abstract Retrieve a key from the hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param k Key [type of keys]
+ @return Iterator to the found element, or kh_end(h) if the element is absent [khint_t]
+ */
+#define kh_get(name, h, k) kh_get_##name(h, k)
+
+/*! @function
+ @abstract Remove a key from the hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param k Iterator to the element to be deleted [khint_t]
+ */
+#define kh_del(name, h, k) kh_del_##name(h, k)
+
+/*! @function
+ @abstract Test whether a bucket contains data.
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param x Iterator to the bucket [khint_t]
+ @return 1 if containing data; 0 otherwise [int]
+ */
+#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
+
+/*! @function
+ @abstract Get key given an iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param x Iterator to the bucket [khint_t]
+ @return Key [type of keys]
+ */
+#define kh_key(h, x) ((h)->keys[x])
+
+/*! @function
+ @abstract Get value given an iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param x Iterator to the bucket [khint_t]
+ @return Value [type of values]
+ @discussion For hash sets, calling this results in segfault.
+ */
+#define kh_val(h, x) ((h)->vals[x])
+
+/*! @function
+ @abstract Alias of kh_val()
+ */
+#define kh_value(h, x) ((h)->vals[x])
+
+/*! @function
+ @abstract Get the start iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return The start iterator [khint_t]
+ */
+#define kh_begin(h) (khint_t)(0)
+
+/*! @function
+ @abstract Get the end iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return The end iterator [khint_t]
+ */
+#define kh_end(h) ((h)->n_buckets)
+
+/*! @function
+ @abstract Get the number of elements in the hash table
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return Number of elements in the hash table [khint_t]
+ */
+#define kh_size(h) ((h)->size)
+
+/*! @function
+ @abstract Get the number of buckets in the hash table
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return Number of buckets in the hash table [khint_t]
+ */
+#define kh_n_buckets(h) ((h)->n_buckets)
+
+/*! @function
+ @abstract Iterate over the entries in the hash table
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param kvar Variable to which key will be assigned
+ @param vvar Variable to which value will be assigned
+ @param code Block of code to execute
+ */
+#define kh_foreach(h, kvar, vvar, code) { khint_t __i; \
+ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
+ if (!kh_exist(h,__i)) continue; \
+ (kvar) = kh_key(h,__i); \
+ (vvar) = kh_val(h,__i); \
+ code; \
+ } }
+
+/*! @function
+ @abstract Iterate over the values in the hash table
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param vvar Variable to which value will be assigned
+ @param code Block of code to execute
+ */
+#define kh_foreach_value(h, vvar, code) { khint_t __i; \
+ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
+ if (!kh_exist(h,__i)) continue; \
+ (vvar) = kh_val(h,__i); \
+ code; \
+ } }
+
+#define kh_foreach_value_ptr(h, pvvar, code) { khint_t __i; \
+ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
+ if (!kh_exist(h,__i)) continue; \
+ (pvvar) = &kh_val(h,__i); \
+ code; \
+ } }
+
+#define kh_foreach_key_value_ptr(h, kvar, pvvar, code) { khint_t __i; \
+ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
+ if (!kh_exist(h,__i)) continue; \
+ (kvar) = kh_key(h,__i); \
+ (pvvar) = &kh_val(h,__i); \
+ code; \
+ } }
+
+#define kh_foreach_key(h, kvar, code) { \
+ khint_t __i; \
+ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
+ if (!kh_exist(h,__i)) continue; \
+ (kvar) = kh_key(h,__i); \
+ code; \
+ } }
+
+/* More conenient interfaces */
+
+/*! @function
+ @abstract Instantiate a hash set containing integer keys
+ @param name Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT(name) \
+ KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing integer keys
+ @param name Name of the hash table [symbol]
+ @param khval_t Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT(name, khval_t) \
+ KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing 64-bit integer keys
+ @param name Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT64(name) \
+ KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing 64-bit integer keys
+ @param name Name of the hash table [symbol]
+ @param khval_t Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT64(name, khval_t) \
+ KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
+
+typedef const char *kh_cstr_t;
+/*! @function
+ @abstract Instantiate a hash map containing const char* keys
+ @param name Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_STR(name) \
+ KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing const char* keys
+ @param name Name of the hash table [symbol]
+ @param khval_t Type of values [type]
+ */
+#define KHASH_MAP_INIT_STR(name, khval_t) \
+ KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
+
+#endif /* __AC_KHASH_H */
diff --git a/contrib/libucl/kvec.h b/contrib/libucl/kvec.h
new file mode 100644
index 0000000..ce6a536
--- /dev/null
+++ b/contrib/libucl/kvec.h
@@ -0,0 +1,161 @@
+/* The MIT License
+
+ Copyright (c) 2008, by Attractive Chaos <attractor@live.co.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+/*
+ An example:
+
+#include "kvec.h"
+int main() {
+ kvec_t(int) array;
+ kv_init(array);
+ kv_push_safe(int, array, 10, e0); // append
+ kv_a(int, array, 20) = 5; // dynamic
+ kv_A(array, 20) = 4; // static
+ kv_destroy(array);
+ return 0;
+e0:
+ return 1;
+}
+*/
+
+/*
+ 2008-09-22 (0.1.0):
+
+ * The initial version.
+
+*/
+
+#ifndef AC_KVEC_H
+#define AC_KVEC_H
+
+#include <stdlib.h>
+
+#define kv_roundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+
+#define kvec_t(type) struct { size_t n, m; type *a; }
+#define kv_init(v) ((v).n = (v).m = 0, (v).a = 0)
+#define kv_destroy(v) free((v).a)
+#define kv_A(v, i) ((v).a[(i)])
+#define kv_pop(v) ((v).a[--(v).n])
+#define kv_size(v) ((v).n)
+#define kv_max(v) ((v).m)
+
+#define kv_resize_safe(type, v, s, el) do { \
+ type *_tp = (type*)realloc((v).a, sizeof(type) * (s)); \
+ if (_tp == NULL) { \
+ goto el; \
+ } else { \
+ (v).a = _tp; \
+ (v).m = (s); \
+ } \
+ } while (0)
+
+#define kv_grow_factor 1.5
+#define kv_grow_safe(type, v, el) do { \
+ size_t _ts = ((v).m > 1 ? (v).m * kv_grow_factor : 2); \
+ type *_tp = (type*)realloc((v).a, sizeof(type) * _ts); \
+ if (_tp == NULL) { \
+ goto el; \
+ } else { \
+ (v).a = _tp; \
+ (v).m = _ts; \
+ } \
+ } while (0)
+
+#define kv_copy_safe(type, v1, v0, el) do { \
+ if ((v1).m < (v0).n) kv_resize_safe(type, v1, (v0).n, el); \
+ (v1).n = (v0).n; \
+ memcpy((v1).a, (v0).a, sizeof(type) * (v0).n); \
+ } while (0)
+
+#define kv_push_safe(type, v, x, el) do { \
+ if ((v).n == (v).m) { \
+ kv_grow_safe(type, v, el); \
+ } \
+ (v).a[(v).n++] = (x); \
+ } while (0)
+
+#define kv_prepend_safe(type, v, x, el) do { \
+ if ((v).n == (v).m) { \
+ kv_grow_safe(type, v, el); \
+ } \
+ memmove((v).a + 1, (v).a, sizeof(type) * (v).n); \
+ (v).a[0] = (x); \
+ (v).n ++; \
+ } while (0)
+
+#define kv_concat_safe(type, v1, v0, el) do { \
+ if ((v1).m < (v0).n + (v1).n) \
+ kv_resize_safe(type, v1, (v0).n + (v1).n, el); \
+ memcpy((v1).a + (v1).n, (v0).a, sizeof(type) * (v0).n); \
+ (v1).n = (v0).n + (v1).n; \
+ } while (0)
+
+#define kv_del(type, v, i) do { \
+ if ((i) < (v).n) { \
+ memmove((v).a + (i), (v).a + ((i) + 1), sizeof(type) * ((v).n - (i) - 1)); \
+ (v).n --; \
+ } \
+} while (0)
+
+/*
+ * Old (ENOMEM-unsafe) version of kv_xxx macros. Compat-only, not for use in
+ * the new library code.
+ */
+
+#define kv_resize(type, v, s) ((v).m = (s), (v).a = (type*)realloc((v).a, sizeof(type) * (v).m))
+
+#define kv_grow(type, v) ((v).m = ((v).m > 1 ? (v).m * kv_grow_factor : 2), \
+ (v).a = (type*)realloc((v).a, sizeof(type) * (v).m))
+
+#define kv_copy(type, v1, v0) do { \
+ if ((v1).m < (v0).n) kv_resize(type, v1, (v0).n); \
+ (v1).n = (v0).n; \
+ memcpy((v1).a, (v0).a, sizeof(type) * (v0).n); \
+ } while (0) \
+
+#define kv_push(type, v, x) do { \
+ if ((v).n == (v).m) { \
+ kv_grow(type, v); \
+ } \
+ (v).a[(v).n++] = (x); \
+ } while (0)
+
+#define kv_prepend(type, v, x) do { \
+ if ((v).n == (v).m) { \
+ kv_grow(type, v); \
+ } \
+ memmove((v).a + 1, (v).a, sizeof(type) * (v).n); \
+ (v).a[0] = (x); \
+ (v).n ++; \
+} while (0)
+
+#define kv_concat(type, v1, v0) do { \
+ if ((v1).m < (v0).n + (v1).n) kv_resize(type, v1, (v0).n + (v1).n); \
+ memcpy((v1).a + (v1).n, (v0).a, sizeof(type) * (v0).n); \
+ (v1).n = (v0).n + (v1).n; \
+ } while (0)
+
+#endif /* AC_KVEC_H */
diff --git a/contrib/libucl/lua_ucl.c b/contrib/libucl/lua_ucl.c
new file mode 100644
index 0000000..c2e39c4
--- /dev/null
+++ b/contrib/libucl/lua_ucl.c
@@ -0,0 +1,1574 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @file lua ucl bindings
+ */
+
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "lua_ucl.h"
+#include <strings.h>
+
+/***
+ * @module ucl
+ * This lua module allows to parse objects from strings and to store data into
+ * ucl objects. It uses `libucl` C library to parse and manipulate with ucl objects.
+ * @example
+local ucl = require("ucl")
+
+local parser = ucl.parser()
+local res,err = parser:parse_string('{key=value}')
+
+if not res then
+ print('parser error: ' .. err)
+else
+ local obj = parser:get_object()
+ local got = ucl.to_format(obj, 'json')
+endif
+
+local table = {
+ str = 'value',
+ num = 100500,
+ null = ucl.null,
+ func = function ()
+ return 'huh'
+ end
+}
+
+print(ucl.to_format(table, 'ucl'))
+-- Output:
+--[[
+num = 100500;
+str = "value";
+null = null;
+func = "huh";
+--]]
+ */
+
+#define PARSER_META "ucl.parser.meta"
+#define EMITTER_META "ucl.emitter.meta"
+#define NULL_META "ucl.null.meta"
+#define OBJECT_META "ucl.object.meta"
+#define UCL_OBJECT_TYPE_META "ucl.type.object"
+#define UCL_ARRAY_TYPE_META "ucl.type.array"
+#define UCL_IMPL_ARRAY_TYPE_META "ucl.type.impl_array"
+
+static int ucl_object_lua_push_array (lua_State *L, const ucl_object_t *obj, int flags);
+static int ucl_object_lua_push_scalar (lua_State *L, const ucl_object_t *obj, int flags);
+static int ucl_object_push_lua_common (lua_State *L, const ucl_object_t *obj, int flags);
+static ucl_object_t* ucl_object_lua_fromtable (lua_State *L, int idx, ucl_string_flags_t flags);
+static ucl_object_t* ucl_object_lua_fromelt (lua_State *L, int idx, ucl_string_flags_t flags);
+
+static void *ucl_null;
+
+struct _rspamd_lua_text {
+ const char *start;
+ unsigned int len;
+ unsigned int flags;
+};
+
+enum lua_ucl_push_flags {
+ LUA_UCL_DEFAULT_FLAGS = 0,
+ LUA_UCL_ALLOW_ARRAY = (1u << 0u),
+ LUA_UCL_CONVERT_NIL = (1u << 1u),
+};
+
+/**
+ * Push a single element of an object to lua
+ * @param L
+ * @param key
+ * @param obj
+ */
+static void
+ucl_object_lua_push_element (lua_State *L, const char *key,
+ const ucl_object_t *obj, int flags)
+{
+ lua_pushstring (L, key);
+ ucl_object_push_lua_common (L, obj, flags|LUA_UCL_ALLOW_ARRAY);
+ lua_settable (L, -3);
+}
+
+static void
+lua_ucl_userdata_dtor (void *ud)
+{
+ struct ucl_lua_funcdata *fd = (struct ucl_lua_funcdata *)ud;
+
+ luaL_unref (fd->L, LUA_REGISTRYINDEX, fd->idx);
+ if (fd->ret != NULL) {
+ free (fd->ret);
+ }
+ free (fd);
+}
+
+static const char *
+lua_ucl_userdata_emitter (void *ud)
+{
+ struct ucl_lua_funcdata *fd = (struct ucl_lua_funcdata *)ud;
+ const char *out = "";
+
+ lua_rawgeti (fd->L, LUA_REGISTRYINDEX, fd->idx);
+
+ lua_pcall (fd->L, 0, 1, 0);
+ out = lua_tostring (fd->L, -1);
+
+ if (out != NULL) {
+ /* We need to store temporary string in a more appropriate place */
+ if (fd->ret) {
+ free (fd->ret);
+ }
+ fd->ret = strdup (out);
+ }
+
+ lua_settop (fd->L, 0);
+
+ return fd->ret;
+}
+
+/**
+ * Push a single object to lua
+ * @param L
+ * @param obj
+ * @return
+ */
+static int
+ucl_object_lua_push_object (lua_State *L, const ucl_object_t *obj,
+ int flags)
+{
+ const ucl_object_t *cur;
+ ucl_object_iter_t it = NULL;
+
+ if ((flags & LUA_UCL_ALLOW_ARRAY) && obj->next != NULL) {
+ /* Actually we need to push this as an array */
+ return ucl_object_lua_push_array (L, obj, flags);
+ }
+
+ lua_createtable (L, 0, obj->len);
+ it = NULL;
+
+ while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) {
+ ucl_object_lua_push_element (L, ucl_object_key (cur), cur, flags);
+ }
+
+ luaL_getmetatable (L, UCL_OBJECT_TYPE_META);
+ lua_setmetatable (L, -2);
+
+ return 1;
+}
+
+/**
+ * Push an array to lua as table indexed by integers
+ * @param L
+ * @param obj
+ * @return
+ */
+static int
+ucl_object_lua_push_array (lua_State *L, const ucl_object_t *obj, int flags)
+{
+ const ucl_object_t *cur;
+ ucl_object_iter_t it;
+ int i = 1, nelt = 0;
+
+ if (obj->type == UCL_ARRAY) {
+ nelt = obj->len;
+ it = ucl_object_iterate_new (obj);
+ lua_createtable (L, nelt, 0);
+
+ while ((cur = ucl_object_iterate_safe (it, true))) {
+ ucl_object_push_lua (L, cur, (flags & ~LUA_UCL_ALLOW_ARRAY));
+ lua_rawseti (L, -2, i);
+ i ++;
+ }
+
+ luaL_getmetatable (L, UCL_ARRAY_TYPE_META);
+ lua_setmetatable (L, -2);
+
+ ucl_object_iterate_free (it);
+ }
+ else {
+ /* Optimize allocation by preallocation of table */
+ LL_FOREACH (obj, cur) {
+ nelt ++;
+ }
+
+ lua_createtable (L, nelt, 0);
+
+ LL_FOREACH (obj, cur) {
+ ucl_object_push_lua (L, cur, (flags & ~LUA_UCL_ALLOW_ARRAY));
+ lua_rawseti (L, -2, i);
+ i ++;
+ }
+
+ luaL_getmetatable (L, UCL_IMPL_ARRAY_TYPE_META);
+ lua_setmetatable (L, -2);
+ }
+
+ return 1;
+}
+
+/**
+ * Push a simple object to lua depending on its actual type
+ */
+static int
+ucl_object_lua_push_scalar (lua_State *L, const ucl_object_t *obj,
+ int flags)
+{
+ struct ucl_lua_funcdata *fd;
+
+ if ((flags & LUA_UCL_ALLOW_ARRAY) && obj->next != NULL) {
+ /* Actually we need to push this as an array */
+ return ucl_object_lua_push_array (L, obj, flags);
+ }
+
+ switch (obj->type) {
+ case UCL_BOOLEAN:
+ lua_pushboolean (L, ucl_obj_toboolean (obj));
+ break;
+ case UCL_STRING:
+ lua_pushlstring (L, ucl_obj_tostring (obj), obj->len);
+ break;
+ case UCL_INT:
+#if LUA_VERSION_NUM >= 501
+ lua_pushinteger (L, ucl_obj_toint (obj));
+#else
+ lua_pushnumber (L, ucl_obj_toint (obj));
+#endif
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ lua_pushnumber (L, ucl_obj_todouble (obj));
+ break;
+ case UCL_NULL:
+ if (flags & LUA_UCL_CONVERT_NIL) {
+ lua_pushboolean (L, false);
+ }
+ else {
+ lua_getfield (L, LUA_REGISTRYINDEX, "ucl.null");
+ }
+ break;
+ case UCL_USERDATA:
+ fd = (struct ucl_lua_funcdata *)obj->value.ud;
+ lua_rawgeti (L, LUA_REGISTRYINDEX, fd->idx);
+ break;
+ default:
+ lua_pushnil (L);
+ break;
+ }
+
+ return 1;
+}
+
+static int
+ucl_object_push_lua_common (lua_State *L, const ucl_object_t *obj, int flags)
+{
+ switch (obj->type) {
+ case UCL_OBJECT:
+ return ucl_object_lua_push_object (L, obj, flags);
+ case UCL_ARRAY:
+ return ucl_object_lua_push_array (L, obj, flags);
+ default:
+ return ucl_object_lua_push_scalar (L, obj, flags);
+ }
+}
+
+/***
+ * @function ucl_object_push_lua(L, obj, allow_array)
+ * This is a `C` function to push `UCL` object as lua variable. This function
+ * converts `obj` to lua representation using the following conversions:
+ *
+ * - *scalar* values are directly presented by lua objects
+ * - *userdata* values are converted to lua function objects using `LUA_REGISTRYINDEX`,
+ * this can be used to pass functions from lua to c and vice-versa
+ * - *arrays* are converted to lua tables with numeric indicies suitable for `ipairs` iterations
+ * - *objects* are converted to lua tables with string indicies
+ * @param {lua_State} L lua state pointer
+ * @param {ucl_object_t} obj object to push
+ * @param {bool} allow_array expand implicit arrays (should be true for all but partial arrays)
+ * @return {int} `1` if an object is pushed to lua
+ */
+int
+ucl_object_push_lua (lua_State *L, const ucl_object_t *obj, bool allow_array)
+{
+ return ucl_object_push_lua_common (L, obj,
+ allow_array ? LUA_UCL_ALLOW_ARRAY : LUA_UCL_DEFAULT_FLAGS);
+}
+
+int
+ucl_object_push_lua_filter_nil (lua_State *L, const ucl_object_t *obj, bool allow_array)
+{
+ return ucl_object_push_lua_common (L, obj,
+ allow_array ? (LUA_UCL_ALLOW_ARRAY|LUA_UCL_CONVERT_NIL) :
+ (LUA_UCL_DEFAULT_FLAGS|LUA_UCL_CONVERT_NIL));
+}
+
+/**
+ * Parse lua table into object top
+ * @param L
+ * @param top
+ * @param idx
+ */
+static ucl_object_t *
+ucl_object_lua_fromtable (lua_State *L, int idx, ucl_string_flags_t flags)
+{
+ ucl_object_t *obj, *top = NULL, *cur;
+ size_t keylen;
+ const char *k;
+ bool is_array = true, is_implicit = false, found_mt = false;
+ size_t max = 0, nelts = 0;
+
+ if (idx < 0) {
+ /* For negative indicies we want to invert them */
+ idx = lua_gettop (L) + idx + 1;
+ }
+
+ /* First, we check from metatable */
+ if (luaL_getmetafield (L, idx, "class") != 0) {
+
+ if (lua_type (L, -1) == LUA_TSTRING) {
+ const char *classname = lua_tostring (L, -1);
+
+ if (strcmp (classname, UCL_OBJECT_TYPE_META) == 0) {
+ is_array = false;
+ found_mt = true;
+ } else if (strcmp (classname, UCL_ARRAY_TYPE_META) == 0) {
+ is_array = true;
+ found_mt = true;
+#if LUA_VERSION_NUM >= 502
+ max = lua_rawlen (L, idx);
+#else
+ max = lua_objlen (L, idx);
+#endif
+ nelts = max;
+ } else if (strcmp (classname, UCL_IMPL_ARRAY_TYPE_META) == 0) {
+ is_array = true;
+ is_implicit = true;
+ found_mt = true;
+#if LUA_VERSION_NUM >= 502
+ max = lua_rawlen (L, idx);
+#else
+ max = lua_objlen (L, idx);
+#endif
+ nelts = max;
+ }
+ }
+
+ lua_pop (L, 1);
+ }
+
+ if (!found_mt) {
+ /* Check for array (it is all inefficient) */
+ lua_pushnil (L);
+
+ while (lua_next (L, idx) != 0) {
+ lua_pushvalue (L, -2);
+
+ if (lua_type (L, -1) == LUA_TNUMBER) {
+ double num = lua_tonumber (L, -1);
+ if (num == (int) num) {
+ if (num > max) {
+ max = num;
+ }
+ }
+ else {
+ /* Keys are not integer */
+ is_array = false;
+ }
+ }
+ else {
+ /* Keys are not numeric */
+ is_array = false;
+ }
+
+ lua_pop (L, 2);
+ nelts ++;
+ }
+ }
+
+ /* Table iterate */
+ if (is_array) {
+ int i;
+
+ if (!is_implicit) {
+ top = ucl_object_typed_new (UCL_ARRAY);
+ ucl_object_reserve (top, nelts);
+ }
+ else {
+ top = NULL;
+ }
+
+ for (i = 1; i <= max; i ++) {
+ lua_pushinteger (L, i);
+ lua_gettable (L, idx);
+
+ obj = ucl_object_lua_fromelt (L, lua_gettop (L), flags);
+
+ if (obj != NULL) {
+ if (is_implicit) {
+ DL_APPEND (top, obj);
+ }
+ else {
+ ucl_array_append (top, obj);
+ }
+ }
+ lua_pop (L, 1);
+ }
+ }
+ else {
+ lua_pushnil (L);
+ top = ucl_object_typed_new (UCL_OBJECT);
+ ucl_object_reserve (top, nelts);
+
+ while (lua_next (L, idx) != 0) {
+ /* copy key to avoid modifications */
+ lua_pushvalue (L, -2);
+ k = lua_tolstring (L, -1, &keylen);
+ obj = ucl_object_lua_fromelt (L, lua_gettop (L) - 1, flags);
+
+ if (obj != NULL) {
+ ucl_object_insert_key (top, obj, k, keylen, true);
+
+ DL_FOREACH (obj, cur) {
+ if (cur->keylen == 0) {
+ cur->keylen = obj->keylen;
+ cur->key = obj->key;
+ }
+ }
+ }
+ lua_pop (L, 2);
+ }
+ }
+
+ return top;
+}
+
+/**
+ * Get a single element from lua to object obj
+ * @param L
+ * @param obj
+ * @param idx
+ */
+static ucl_object_t *
+ucl_object_lua_fromelt (lua_State *L, int idx, ucl_string_flags_t flags)
+{
+ int type;
+ double num;
+ ucl_object_t *obj = NULL;
+ struct ucl_lua_funcdata *fd;
+ const char *str;
+ size_t sz;
+
+ type = lua_type (L, idx);
+
+ switch (type) {
+ case LUA_TSTRING:
+ str = lua_tolstring (L, idx, &sz);
+
+ if (str) {
+ /*
+ * ucl_object_fromstring_common has a `logic` to use strlen if sz is zero
+ * which is totally broken...
+ */
+ if (sz > 0) {
+ obj = ucl_object_fromstring_common(str, sz, flags);
+ }
+ else {
+ obj = ucl_object_fromstring_common("", sz, flags);
+ }
+ }
+ else {
+ obj = ucl_object_typed_new (UCL_NULL);
+ }
+ break;
+ case LUA_TNUMBER:
+ num = lua_tonumber (L, idx);
+ if (num == (int64_t)num) {
+ obj = ucl_object_fromint (num);
+ }
+ else {
+ obj = ucl_object_fromdouble (num);
+ }
+ break;
+ case LUA_TBOOLEAN:
+ obj = ucl_object_frombool (lua_toboolean (L, idx));
+ break;
+ case LUA_TUSERDATA:
+ if (lua_topointer (L, idx) == ucl_null) {
+ obj = ucl_object_typed_new (UCL_NULL);
+ }
+ else {
+ /* Assume it is a text like object */
+ struct _rspamd_lua_text *t = lua_touserdata (L, idx);
+
+ if (t) {
+ if (t->len >0) {
+ obj = ucl_object_fromstring_common(t->start, t->len, 0);
+ }
+ else {
+ obj = ucl_object_fromstring_common("", 0, 0);
+ }
+
+ /* Binary text */
+ if (t->flags & (1u << 5u)) {
+ obj->flags |= UCL_OBJECT_BINARY;
+ }
+ }
+ }
+ break;
+ case LUA_TTABLE:
+ case LUA_TFUNCTION:
+ case LUA_TTHREAD:
+ if (luaL_getmetafield (L, idx, "__gen_ucl")) {
+ if (lua_isfunction (L, -1)) {
+ lua_settop (L, 3); /* gen, obj, func */
+ lua_insert (L, 1); /* func, gen, obj */
+ lua_insert (L, 2); /* func, obj, gen */
+ lua_call(L, 2, 1);
+ obj = ucl_object_lua_fromelt (L, 1, flags);
+ }
+ lua_pop (L, 2);
+ }
+ else {
+ if (type == LUA_TTABLE) {
+ obj = ucl_object_lua_fromtable (L, idx, flags);
+ }
+ else if (type == LUA_TFUNCTION) {
+ fd = malloc (sizeof (*fd));
+ if (fd != NULL) {
+ lua_pushvalue (L, idx);
+ fd->L = L;
+ fd->ret = NULL;
+ fd->idx = luaL_ref (L, LUA_REGISTRYINDEX);
+
+ obj = ucl_object_new_userdata (lua_ucl_userdata_dtor,
+ lua_ucl_userdata_emitter, (void *)fd);
+ }
+ }
+ }
+ break;
+ }
+
+ return obj;
+}
+
+/**
+ * @function ucl_object_lua_import(L, idx)
+ * Extracts ucl object from lua variable at `idx` position,
+ * @see ucl_object_push_lua for conversion definitions
+ * @param {lua_state} L lua state machine pointer
+ * @param {int} idx index where the source variable is placed
+ * @return {ucl_object_t} new ucl object extracted from lua variable. Reference count of this object is 1,
+ * this object thus needs to be unref'ed after usage.
+ */
+ucl_object_t *
+ucl_object_lua_import (lua_State *L, int idx)
+{
+ ucl_object_t *obj;
+ int t;
+
+ t = lua_type (L, idx);
+ switch (t) {
+ case LUA_TTABLE:
+ obj = ucl_object_lua_fromtable (L, idx, UCL_STRING_RAW);
+ break;
+ default:
+ obj = ucl_object_lua_fromelt (L, idx, UCL_STRING_RAW);
+ break;
+ }
+
+ return obj;
+}
+
+/**
+ * @function ucl_object_lua_import_escape(L, idx)
+ * Extracts ucl object from lua variable at `idx` position escaping JSON strings
+ * @see ucl_object_push_lua for conversion definitions
+ * @param {lua_state} L lua state machine pointer
+ * @param {int} idx index where the source variable is placed
+ * @return {ucl_object_t} new ucl object extracted from lua variable. Reference count of this object is 1,
+ * this object thus needs to be unref'ed after usage.
+ */
+ucl_object_t *
+ucl_object_lua_import_escape (lua_State *L, int idx)
+{
+ ucl_object_t *obj;
+ int t;
+
+ t = lua_type (L, idx);
+ switch (t) {
+ case LUA_TTABLE:
+ obj = ucl_object_lua_fromtable (L, idx, UCL_STRING_ESCAPE);
+ break;
+ default:
+ obj = ucl_object_lua_fromelt (L, idx, UCL_STRING_ESCAPE);
+ break;
+ }
+
+ return obj;
+}
+
+static int
+lua_ucl_to_string (lua_State *L, const ucl_object_t *obj, enum ucl_emitter type)
+{
+ unsigned char *result;
+ size_t outlen;
+
+ result = ucl_object_emit_len (obj, type, &outlen);
+
+ if (result != NULL) {
+ lua_pushlstring (L, (const char *)result, outlen);
+ free (result);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+static int
+lua_ucl_parser_init (lua_State *L)
+{
+ struct ucl_parser *parser, **pparser;
+ int flags = UCL_PARSER_NO_FILEVARS;
+
+ if (lua_gettop (L) >= 1) {
+ flags = lua_tonumber (L, 1);
+ }
+
+ parser = ucl_parser_new (flags);
+ if (parser == NULL) {
+ lua_pushnil (L);
+ }
+
+ pparser = lua_newuserdata (L, sizeof (parser));
+ *pparser = parser;
+ luaL_getmetatable (L, PARSER_META);
+ lua_setmetatable (L, -2);
+
+ return 1;
+}
+
+static struct ucl_parser *
+lua_ucl_parser_get (lua_State *L, int index)
+{
+ return *((struct ucl_parser **) luaL_checkudata(L, index, PARSER_META));
+}
+
+static ucl_object_t *
+lua_ucl_object_get (lua_State *L, int index)
+{
+ return *((ucl_object_t **) luaL_checkudata(L, index, OBJECT_META));
+}
+
+static void
+lua_ucl_push_opaque (lua_State *L, ucl_object_t *obj)
+{
+ ucl_object_t **pobj;
+
+ pobj = lua_newuserdata (L, sizeof (*pobj));
+ *pobj = obj;
+ luaL_getmetatable (L, OBJECT_META);
+ lua_setmetatable (L, -2);
+}
+
+static inline enum ucl_parse_type
+lua_ucl_str_to_parse_type (const char *str)
+{
+ enum ucl_parse_type type = UCL_PARSE_UCL;
+
+ if (str != NULL) {
+ if (strcasecmp (str, "msgpack") == 0) {
+ type = UCL_PARSE_MSGPACK;
+ }
+ else if (strcasecmp (str, "sexp") == 0 ||
+ strcasecmp (str, "csexp") == 0) {
+ type = UCL_PARSE_CSEXP;
+ }
+ else if (strcasecmp (str, "auto") == 0) {
+ type = UCL_PARSE_AUTO;
+ }
+ }
+
+ return type;
+}
+
+/***
+ * @method parser:parse_file(name)
+ * Parse UCL object from file.
+ * @param {string} name filename to parse
+ * @return {bool[, string]} if res is `true` then file has been parsed successfully, otherwise an error string is also returned
+@example
+local parser = ucl.parser()
+local res,err = parser:parse_file('/some/file.conf')
+
+if not res then
+ print('parser error: ' .. err)
+else
+ -- Do something with object
+end
+ */
+static int
+lua_ucl_parser_parse_file (lua_State *L)
+{
+ struct ucl_parser *parser;
+ const char *file;
+ int ret = 2;
+
+ parser = lua_ucl_parser_get (L, 1);
+ file = luaL_checkstring (L, 2);
+
+ if (parser != NULL && file != NULL) {
+ if (ucl_parser_add_file (parser, file)) {
+ lua_pushboolean (L, true);
+ ret = 1;
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, ucl_parser_get_error (parser));
+ }
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "invalid arguments");
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:register_variable(name, value)
+ * Register parser variable
+ * @param {string} name name of variable
+ * @param {string} value value of variable
+ * @return {bool} success
+@example
+local parser = ucl.parser()
+local res = parser:register_variable('CONFDIR', '/etc/foo')
+ */
+static int
+lua_ucl_parser_register_variable (lua_State *L)
+{
+ struct ucl_parser *parser;
+ const char *name, *value;
+ int ret = 2;
+
+ parser = lua_ucl_parser_get (L, 1);
+ name = luaL_checkstring (L, 2);
+ value = luaL_checkstring (L, 3);
+
+ if (parser != NULL && name != NULL && value != NULL) {
+ ucl_parser_register_variable (parser, name, value);
+ lua_pushboolean (L, true);
+ ret = 1;
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:register_variables(vars)
+ * Register parser variables
+ * @param {table} vars names/values of variables
+ * @return {bool} success
+@example
+local parser = ucl.parser()
+local res = parser:register_variables({CONFDIR = '/etc/foo', VARDIR = '/var'})
+ */
+static int
+lua_ucl_parser_register_variables (lua_State *L)
+{
+ struct ucl_parser *parser;
+ const char *name, *value;
+ int ret = 2;
+
+ parser = lua_ucl_parser_get (L, 1);
+
+ if (parser != NULL && lua_type (L, 2) == LUA_TTABLE) {
+ for (lua_pushnil (L); lua_next (L, 2); lua_pop (L, 1)) {
+ lua_pushvalue (L, -2);
+ name = luaL_checkstring (L, -1);
+ value = luaL_checkstring (L, -2);
+ ucl_parser_register_variable (parser, name, value);
+ lua_pop (L, 1);
+ }
+
+ lua_pushboolean (L, true);
+ ret = 1;
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:parse_string(input)
+ * Parse UCL object from file.
+ * @param {string} input string to parse
+ * @return {bool[, string]} if res is `true` then file has been parsed successfully, otherwise an error string is also returned
+ */
+static int
+lua_ucl_parser_parse_string (lua_State *L)
+{
+ struct ucl_parser *parser;
+ const char *string;
+ size_t llen;
+ enum ucl_parse_type type = UCL_PARSE_UCL;
+ int ret = 2;
+
+ parser = lua_ucl_parser_get (L, 1);
+ string = luaL_checklstring (L, 2, &llen);
+
+ if (lua_type (L, 3) == LUA_TSTRING) {
+ type = lua_ucl_str_to_parse_type (lua_tostring (L, 3));
+ }
+
+ if (parser != NULL && string != NULL) {
+ if (ucl_parser_add_chunk_full (parser, (const unsigned char *)string,
+ llen, 0, UCL_DUPLICATE_APPEND, type)) {
+ lua_pushboolean (L, true);
+ ret = 1;
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, ucl_parser_get_error (parser));
+ }
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "invalid arguments");
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:parse_text(input)
+ * Parse UCL object from text object (Rspamd specific).
+ * @param {rspamd_text} input text to parse
+ * @return {bool[, string]} if res is `true` then file has been parsed successfully, otherwise an error string is also returned
+ */
+static int
+lua_ucl_parser_parse_text (lua_State *L)
+{
+ struct ucl_parser *parser;
+ struct _rspamd_lua_text *t;
+ enum ucl_parse_type type = UCL_PARSE_UCL;
+ int ret = 2;
+
+ parser = lua_ucl_parser_get (L, 1);
+
+ if (lua_type (L, 2) == LUA_TUSERDATA) {
+ t = lua_touserdata (L, 2);
+ }
+ else if (lua_type (L, 2) == LUA_TSTRING) {
+ const gchar *s;
+ gsize len;
+ static struct _rspamd_lua_text st_t;
+
+ s = lua_tolstring (L, 2, &len);
+ st_t.start = s;
+ st_t.len = len;
+
+ t = &st_t;
+ }
+ else {
+ return luaL_error(L, "invalid argument as input, expected userdata or a string");
+ }
+
+ if (lua_type (L, 3) == LUA_TSTRING) {
+ type = lua_ucl_str_to_parse_type (lua_tostring (L, 3));
+ }
+
+ if (parser != NULL && t != NULL) {
+ if (ucl_parser_add_chunk_full (parser, (const unsigned char *)t->start,
+ t->len, 0, UCL_DUPLICATE_APPEND, type)) {
+ lua_pushboolean (L, true);
+ ret = 1;
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, ucl_parser_get_error (parser));
+ }
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "invalid arguments");
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:get_object()
+ * Get top object from parser and export it to lua representation.
+ * @return {variant or nil} ucl object as lua native variable
+ */
+static int
+lua_ucl_parser_get_object (lua_State *L)
+{
+ struct ucl_parser *parser;
+ ucl_object_t *obj;
+ int ret = 1;
+
+ parser = lua_ucl_parser_get (L, 1);
+ obj = ucl_parser_get_object (parser);
+
+ if (obj != NULL) {
+ ret = ucl_object_push_lua (L, obj, false);
+ /* no need to keep reference */
+ ucl_object_unref (obj);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:get_object_wrapped()
+ * Get top object from parser and export it to userdata object without
+ * unwrapping to lua.
+ * @return {ucl.object or nil} ucl object wrapped variable
+ */
+static int
+lua_ucl_parser_get_object_wrapped (lua_State *L)
+{
+ struct ucl_parser *parser;
+ ucl_object_t *obj;
+ int ret = 1;
+
+ parser = lua_ucl_parser_get (L, 1);
+ obj = ucl_parser_get_object (parser);
+
+ if (obj != NULL) {
+ lua_ucl_push_opaque (L, obj);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return ret;
+}
+
+/***
+ * @method parser:validate(schema)
+ * Validates the top object in the parser against schema. Schema might be
+ * another object or a string that represents file to load schema from.
+ *
+ * @param {string/table} schema input schema
+ * @return {result,err} two values: boolean result and the corresponding error
+ *
+ */
+static int
+lua_ucl_parser_validate (lua_State *L)
+{
+ struct ucl_parser *parser, *schema_parser;
+ ucl_object_t *schema;
+ const char *schema_file;
+ struct ucl_schema_error err;
+
+ parser = lua_ucl_parser_get (L, 1);
+
+ if (parser && parser->top_obj) {
+ if (lua_type (L, 2) == LUA_TTABLE) {
+ schema = ucl_object_lua_import (L, 2);
+
+ if (schema == NULL) {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "cannot load schema from lua table");
+
+ return 2;
+ }
+ }
+ else if (lua_type (L, 2) == LUA_TSTRING) {
+ schema_parser = ucl_parser_new (0);
+ schema_file = luaL_checkstring (L, 2);
+
+ if (!ucl_parser_add_file (schema_parser, schema_file)) {
+ lua_pushboolean (L, false);
+ lua_pushfstring (L, "cannot parse schema file \"%s\": "
+ "%s", schema_file, ucl_parser_get_error (parser));
+ ucl_parser_free (schema_parser);
+
+ return 2;
+ }
+
+ schema = ucl_parser_get_object (schema_parser);
+ ucl_parser_free (schema_parser);
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "invalid schema argument");
+
+ return 2;
+ }
+
+ if (!ucl_object_validate (schema, parser->top_obj, &err)) {
+ lua_pushboolean (L, false);
+ lua_pushfstring (L, "validation error: "
+ "%s", err.msg);
+ }
+ else {
+ lua_pushboolean (L, true);
+ lua_pushnil (L);
+ }
+
+ ucl_object_unref (schema);
+ }
+ else {
+ lua_pushboolean (L, false);
+ lua_pushstring (L, "invalid parser or empty top object");
+ }
+
+ return 2;
+}
+
+static int
+lua_ucl_parser_gc (lua_State *L)
+{
+ struct ucl_parser *parser;
+
+ parser = lua_ucl_parser_get (L, 1);
+ ucl_parser_free (parser);
+
+ return 0;
+}
+
+/***
+ * @method object:unwrap()
+ * Unwraps opaque ucl object to the native lua object (performing copying)
+ * @return {variant} any lua object
+ */
+static int
+lua_ucl_object_unwrap (lua_State *L)
+{
+ ucl_object_t *obj;
+
+ obj = lua_ucl_object_get (L, 1);
+
+ if (obj) {
+ ucl_object_push_lua (L, obj, true);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+static inline enum ucl_emitter
+lua_ucl_str_to_emit_type (const char *strtype)
+{
+ enum ucl_emitter format = UCL_EMIT_JSON_COMPACT;
+
+ if (strcasecmp (strtype, "json") == 0) {
+ format = UCL_EMIT_JSON;
+ }
+ else if (strcasecmp (strtype, "json-compact") == 0) {
+ format = UCL_EMIT_JSON_COMPACT;
+ }
+ else if (strcasecmp (strtype, "yaml") == 0) {
+ format = UCL_EMIT_YAML;
+ }
+ else if (strcasecmp (strtype, "config") == 0 ||
+ strcasecmp (strtype, "ucl") == 0) {
+ format = UCL_EMIT_CONFIG;
+ }
+
+ return format;
+}
+
+/***
+ * @method object:tostring(type)
+ * Unwraps opaque ucl object to string (json by default). Optionally you can
+ * specify output format:
+ *
+ * - `json` - fine printed json
+ * - `json-compact` - compacted json
+ * - `config` - fine printed configuration
+ * - `ucl` - same as `config`
+ * - `yaml` - embedded yaml
+ * @param {string} type optional
+ * @return {string} string representation of the opaque ucl object
+ */
+static int
+lua_ucl_object_tostring (lua_State *L)
+{
+ ucl_object_t *obj;
+ enum ucl_emitter format = UCL_EMIT_JSON_COMPACT;
+
+ obj = lua_ucl_object_get (L, 1);
+
+ if (obj) {
+ if (lua_gettop (L) > 1) {
+ if (lua_type (L, 2) == LUA_TSTRING) {
+ const char *strtype = lua_tostring (L, 2);
+
+ format = lua_ucl_str_to_emit_type (strtype);
+ }
+ }
+
+ return lua_ucl_to_string (L, obj, format);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method object:validate(schema[, path[, ext_refs]])
+ * Validates the given ucl object using schema object represented as another
+ * opaque ucl object. You can also specify path in the form `#/path/def` to
+ * specify the specific schema element to perform validation.
+ *
+ * @param {ucl.object} schema schema object
+ * @param {string} path optional path for validation procedure
+ * @return {result,err} two values: boolean result and the corresponding
+ * error, if `ext_refs` are also specified, then they are returned as opaque
+ * ucl object as {result,err,ext_refs}
+ */
+static int
+lua_ucl_object_validate (lua_State *L)
+{
+ ucl_object_t *obj, *schema, *ext_refs = NULL;
+ const ucl_object_t *schema_elt;
+ bool res = false;
+ struct ucl_schema_error err;
+ const char *path = NULL;
+
+ obj = lua_ucl_object_get (L, 1);
+ schema = lua_ucl_object_get (L, 2);
+
+ if (schema && obj && ucl_object_type (schema) == UCL_OBJECT) {
+ if (lua_gettop (L) > 2) {
+ if (lua_type (L, 3) == LUA_TSTRING) {
+ path = lua_tostring (L, 3);
+ if (path[0] == '#') {
+ path++;
+ }
+ }
+ else if (lua_type (L, 3) == LUA_TUSERDATA || lua_type (L, 3) ==
+ LUA_TTABLE) {
+ /* External refs */
+ ext_refs = lua_ucl_object_get (L, 3);
+ }
+
+ if (lua_gettop (L) > 3) {
+ if (lua_type (L, 4) == LUA_TUSERDATA || lua_type (L, 4) ==
+ LUA_TTABLE) {
+ /* External refs */
+ ext_refs = lua_ucl_object_get (L, 4);
+ }
+ }
+ }
+
+ if (path) {
+ schema_elt = ucl_object_lookup_path_char (schema, path, '/');
+ }
+ else {
+ /* Use the top object */
+ schema_elt = schema;
+ }
+
+ if (schema_elt) {
+ res = ucl_object_validate_root_ext (schema_elt, obj, schema,
+ ext_refs, &err);
+
+ if (res) {
+ lua_pushboolean (L, res);
+ lua_pushnil (L);
+
+ if (ext_refs) {
+ lua_ucl_push_opaque (L, ext_refs);
+ }
+ }
+ else {
+ lua_pushboolean (L, res);
+ lua_pushfstring (L, "validation error: %s", err.msg);
+
+ if (ext_refs) {
+ lua_ucl_push_opaque (L, ext_refs);
+ }
+ }
+ }
+ else {
+ lua_pushboolean (L, res);
+
+ lua_pushfstring (L, "cannot find the requested path: %s", path);
+
+ if (ext_refs) {
+ lua_ucl_push_opaque (L, ext_refs);
+ }
+ }
+ }
+ else {
+ lua_pushboolean (L, res);
+ lua_pushstring (L, "invalid object or schema");
+ }
+
+ if (ext_refs) {
+ return 3;
+ }
+
+ return 2;
+}
+
+static int
+lua_ucl_object_gc (lua_State *L)
+{
+ ucl_object_t *obj;
+
+ obj = lua_ucl_object_get (L, 1);
+
+ ucl_object_unref (obj);
+
+ return 0;
+}
+
+static void
+lua_ucl_parser_mt (lua_State *L)
+{
+ luaL_newmetatable (L, PARSER_META);
+
+ lua_pushvalue(L, -1);
+ lua_setfield(L, -2, "__index");
+
+ lua_pushcfunction (L, lua_ucl_parser_gc);
+ lua_setfield (L, -2, "__gc");
+
+ lua_pushcfunction (L, lua_ucl_parser_parse_file);
+ lua_setfield (L, -2, "parse_file");
+
+ lua_pushcfunction (L, lua_ucl_parser_parse_string);
+ lua_setfield (L, -2, "parse_string");
+
+ lua_pushcfunction (L, lua_ucl_parser_parse_text);
+ lua_setfield (L, -2, "parse_text");
+
+ lua_pushcfunction (L, lua_ucl_parser_register_variable);
+ lua_setfield (L, -2, "register_variable");
+
+ lua_pushcfunction (L, lua_ucl_parser_register_variables);
+ lua_setfield (L, -2, "register_variables");
+
+ lua_pushcfunction (L, lua_ucl_parser_get_object);
+ lua_setfield (L, -2, "get_object");
+
+ lua_pushcfunction (L, lua_ucl_parser_get_object_wrapped);
+ lua_setfield (L, -2, "get_object_wrapped");
+
+ lua_pushcfunction (L, lua_ucl_parser_validate);
+ lua_setfield (L, -2, "validate");
+
+ lua_pop (L, 1);
+}
+
+static void
+lua_ucl_object_mt (lua_State *L)
+{
+ luaL_newmetatable (L, OBJECT_META);
+
+ lua_pushvalue(L, -1);
+ lua_setfield(L, -2, "__index");
+
+ lua_pushcfunction (L, lua_ucl_object_gc);
+ lua_setfield (L, -2, "__gc");
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "__tostring");
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "tostring");
+
+ lua_pushcfunction (L, lua_ucl_object_unwrap);
+ lua_setfield (L, -2, "unwrap");
+
+ lua_pushcfunction (L, lua_ucl_object_unwrap);
+ lua_setfield (L, -2, "tolua");
+
+ lua_pushcfunction (L, lua_ucl_object_validate);
+ lua_setfield (L, -2, "validate");
+
+ lua_pushstring (L, OBJECT_META);
+ lua_setfield (L, -2, "class");
+
+ lua_pop (L, 1);
+}
+
+static void
+lua_ucl_types_mt (lua_State *L)
+{
+ luaL_newmetatable (L, UCL_OBJECT_TYPE_META);
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "__tostring");
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "tostring");
+
+ lua_pushstring (L, UCL_OBJECT_TYPE_META);
+ lua_setfield (L, -2, "class");
+
+ lua_pop (L, 1);
+
+ luaL_newmetatable (L, UCL_ARRAY_TYPE_META);
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "__tostring");
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "tostring");
+
+ lua_pushstring (L, UCL_ARRAY_TYPE_META);
+ lua_setfield (L, -2, "class");
+
+ lua_pop (L, 1);
+
+ luaL_newmetatable (L, UCL_IMPL_ARRAY_TYPE_META);
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "__tostring");
+
+ lua_pushcfunction (L, lua_ucl_object_tostring);
+ lua_setfield (L, -2, "tostring");
+
+ lua_pushstring (L, UCL_IMPL_ARRAY_TYPE_META);
+ lua_setfield (L, -2, "class");
+
+ lua_pop (L, 1);
+}
+
+static int
+lua_ucl_to_json (lua_State *L)
+{
+ ucl_object_t *obj;
+ int format = UCL_EMIT_JSON;
+
+ if (lua_gettop (L) > 1) {
+ if (lua_toboolean (L, 2)) {
+ format = UCL_EMIT_JSON_COMPACT;
+ }
+ }
+
+ obj = ucl_object_lua_import (L, 1);
+ if (obj != NULL) {
+ lua_ucl_to_string (L, obj, format);
+ ucl_object_unref (obj);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+static int
+lua_ucl_to_config (lua_State *L)
+{
+ ucl_object_t *obj;
+
+ obj = ucl_object_lua_import (L, 1);
+ if (obj != NULL) {
+ lua_ucl_to_string (L, obj, UCL_EMIT_CONFIG);
+ ucl_object_unref (obj);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+/***
+ * @function ucl.to_format(var, format)
+ * Converts lua variable `var` to the specified `format`. Formats supported are:
+ *
+ * - `json` - fine printed json
+ * - `json-compact` - compacted json
+ * - `config` - fine printed configuration
+ * - `ucl` - same as `config`
+ * - `yaml` - embedded yaml
+ *
+ * If `var` contains function, they are called during output formatting and if
+ * they return string value, then this value is used for output.
+ * @param {variant} var any sort of lua variable (if userdata then metafield `__to_ucl` is searched for output)
+ * @param {string} format any available format
+ * @return {string} string representation of `var` in the specific `format`.
+ * @example
+local table = {
+ str = 'value',
+ num = 100500,
+ null = ucl.null,
+ func = function ()
+ return 'huh'
+ end
+}
+
+print(ucl.to_format(table, 'ucl'))
+-- Output:
+--[[
+num = 100500;
+str = "value";
+null = null;
+func = "huh";
+--]]
+ */
+static int
+lua_ucl_to_format (lua_State *L)
+{
+ ucl_object_t *obj;
+ int format = UCL_EMIT_JSON;
+ bool sort = false;
+
+ if (lua_gettop (L) > 1) {
+ if (lua_type (L, 2) == LUA_TNUMBER) {
+ format = lua_tonumber (L, 2);
+ if (format < 0 || format >= UCL_EMIT_YAML) {
+ lua_pushnil (L);
+ return 1;
+ }
+ }
+ else if (lua_type (L, 2) == LUA_TSTRING) {
+ const char *strtype = lua_tostring (L, 2);
+
+ if (strcasecmp (strtype, "json") == 0) {
+ format = UCL_EMIT_JSON;
+ }
+ else if (strcasecmp (strtype, "json-compact") == 0) {
+ format = UCL_EMIT_JSON_COMPACT;
+ }
+ else if (strcasecmp (strtype, "yaml") == 0) {
+ format = UCL_EMIT_YAML;
+ }
+ else if (strcasecmp (strtype, "config") == 0 ||
+ strcasecmp (strtype, "ucl") == 0) {
+ format = UCL_EMIT_CONFIG;
+ }
+ else if (strcasecmp (strtype, "msgpack") == 0 ||
+ strcasecmp (strtype, "messagepack") == 0) {
+ format = UCL_EMIT_MSGPACK;
+ }
+ }
+
+ if (lua_isboolean (L, 3)) {
+ sort = lua_toboolean (L, 3);
+ }
+ }
+
+ obj = ucl_object_lua_import (L, 1);
+
+ if (obj != NULL) {
+
+ if (sort) {
+ if (ucl_object_type (obj) == UCL_OBJECT) {
+ ucl_object_sort_keys (obj, UCL_SORT_KEYS_RECURSIVE);
+ }
+ }
+
+ lua_ucl_to_string (L, obj, format);
+ ucl_object_unref (obj);
+ }
+ else {
+ lua_pushnil (L);
+ }
+
+ return 1;
+}
+
+static int
+lua_ucl_null_tostring (lua_State* L)
+{
+ lua_pushstring (L, "null");
+ return 1;
+}
+
+static void
+lua_ucl_null_mt (lua_State *L)
+{
+ luaL_newmetatable (L, NULL_META);
+
+ lua_pushcfunction (L, lua_ucl_null_tostring);
+ lua_setfield (L, -2, "__tostring");
+
+ lua_pop (L, 1);
+}
+
+int
+luaopen_ucl (lua_State *L)
+{
+ lua_ucl_parser_mt (L);
+ lua_ucl_null_mt (L);
+ lua_ucl_object_mt (L);
+ lua_ucl_types_mt (L);
+
+ /* Create the refs weak table: */
+ lua_createtable (L, 0, 2);
+ lua_pushliteral (L, "v"); /* tbl, "v" */
+ lua_setfield (L, -2, "__mode");
+ lua_pushvalue (L, -1); /* tbl, tbl */
+ lua_setmetatable (L, -2); /* tbl */
+ lua_setfield (L, LUA_REGISTRYINDEX, "ucl.refs");
+
+ lua_newtable (L);
+
+ lua_pushcfunction (L, lua_ucl_parser_init);
+ lua_setfield (L, -2, "parser");
+
+ lua_pushcfunction (L, lua_ucl_to_json);
+ lua_setfield (L, -2, "to_json");
+
+ lua_pushcfunction (L, lua_ucl_to_config);
+ lua_setfield (L, -2, "to_config");
+
+ lua_pushcfunction (L, lua_ucl_to_format);
+ lua_setfield (L, -2, "to_format");
+
+ ucl_null = lua_newuserdata (L, 0);
+ luaL_getmetatable (L, NULL_META);
+ lua_setmetatable (L, -2);
+
+ lua_pushvalue (L, -1);
+ lua_setfield (L, LUA_REGISTRYINDEX, "ucl.null");
+
+ lua_setfield (L, -2, "null");
+
+ return 1;
+}
+
+struct ucl_lua_funcdata*
+ucl_object_toclosure (const ucl_object_t *obj)
+{
+ if (obj == NULL || obj->type != UCL_USERDATA) {
+ return NULL;
+ }
+
+ return (struct ucl_lua_funcdata*)obj->value.ud;
+}
diff --git a/contrib/libucl/lua_ucl.h b/contrib/libucl/lua_ucl.h
new file mode 100644
index 0000000..4a759e3
--- /dev/null
+++ b/contrib/libucl/lua_ucl.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef LUA_UCL_H_
+#define LUA_UCL_H_
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Include C++ guard as Lua headers miss one */
+#include <lua.h>
+#include <lauxlib.h>
+#include <lualib.h>
+
+/**
+ * Closure structure for lua function storing inside UCL
+ */
+struct ucl_lua_funcdata {
+ lua_State *L;
+ int idx;
+ char *ret;
+};
+
+/**
+ * Initialize lua UCL API
+ */
+UCL_EXTERN int luaopen_ucl(lua_State *L);
+
+/**
+ * Import UCL object from lua state
+ * @param L lua state
+ * @param idx index of object at the lua stack to convert to UCL
+ * @return new UCL object or NULL, the caller should unref object after using
+ */
+UCL_EXTERN ucl_object_t *ucl_object_lua_import(lua_State *L, int idx);
+
+/**
+ * Import UCL object from lua state, escaping JSON strings
+ * @param L lua state
+ * @param idx index of object at the lua stack to convert to UCL
+ * @return new UCL object or NULL, the caller should unref object after using
+ */
+UCL_EXTERN ucl_object_t *ucl_object_lua_import_escape(lua_State *L, int idx);
+
+/**
+ * Push an object to lua
+ * @param L lua state
+ * @param obj object to push
+ * @param allow_array traverse over implicit arrays
+ */
+UCL_EXTERN int ucl_object_push_lua(lua_State *L,
+ const ucl_object_t *obj, bool allow_array);
+/**
+ * Push an object to lua replacing all ucl.null with `false`
+ * @param L lua state
+ * @param obj object to push
+ * @param allow_array traverse over implicit arrays
+ */
+UCL_EXTERN int ucl_object_push_lua_filter_nil(lua_State *L,
+ const ucl_object_t *obj,
+ bool allow_array);
+
+UCL_EXTERN struct ucl_lua_funcdata *ucl_object_toclosure(const ucl_object_t *obj);
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* LUA_UCL_H_ */
diff --git a/contrib/libucl/tree.h b/contrib/libucl/tree.h
new file mode 100644
index 0000000..404b4a8
--- /dev/null
+++ b/contrib/libucl/tree.h
@@ -0,0 +1,219 @@
+/* tree.h -- AVL trees (in the spirit of BSD's 'queue.h') -*- C -*- */
+
+/* Copyright (c) 2005 Ian Piumarta
+ *
+ * All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the 'Software'), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * provided that the above copyright notice(s) and this permission notice appear
+ * in all copies of the Software and that both the above copyright notice(s) and
+ * this permission notice appear in supporting documentation.
+ *
+ * THE SOFTWARE IS PROVIDED 'AS IS'. USE ENTIRELY AT YOUR OWN RISK.
+ */
+
+/* This file defines an AVL balanced binary tree [Georgii M. Adelson-Velskii and
+ * Evgenii M. Landis, 'An algorithm for the organization of information',
+ * Doklady Akademii Nauk SSSR, 146:263-266, 1962 (Russian). Also in Myron
+ * J. Ricci (trans.), Soviet Math, 3:1259-1263, 1962 (English)].
+ *
+ * An AVL tree is headed by pointers to the root node and to a function defining
+ * the ordering relation between nodes. Each node contains an arbitrary payload
+ * plus three fields per tree entry: the depth of the subtree for which it forms
+ * the root and two pointers to child nodes (singly-linked for minimum space, at
+ * the expense of direct access to the parent node given a pointer to one of the
+ * children). The tree is rebalanced after every insertion or removal. The
+ * tree may be traversed in two directions: forward (in-order left-to-right) and
+ * reverse (in-order, right-to-left).
+ *
+ * Because of the recursive nature of many of the operations on trees it is
+ * necessary to define a number of helper functions for each type of tree node.
+ * The macro TREE_DEFINE(node_tag, entry_name) defines these functions with
+ * unique names according to the node_tag. This macro should be invoked,
+ * thereby defining the necessary functions, once per node tag in the program.
+ *
+ * For details on the use of these macros, see the tree(3) manual page.
+ */
+
+#ifndef __tree_h
+#define __tree_h
+
+
+#define TREE_DELTA_MAX 1
+#ifndef _HU_FUNCTION
+# if defined(__GNUC__) || defined(__clang__)
+# define _HU_FUNCTION(x) __attribute__((__unused__)) x
+# else
+# define _HU_FUNCTION(x) x
+# endif
+#endif
+
+#define TREE_ENTRY(type) \
+ struct { \
+ struct type *avl_left; \
+ struct type *avl_right; \
+ int avl_height; \
+ }
+
+#define TREE_HEAD(name, type) \
+ struct name { \
+ struct type *th_root; \
+ int (*th_cmp)(struct type *lhs, struct type *rhs); \
+ }
+
+#define TREE_INITIALIZER(cmp) { 0, cmp }
+
+#define TREE_DELTA(self, field) \
+ (( (((self)->field.avl_left) ? (self)->field.avl_left->field.avl_height : 0)) \
+ - (((self)->field.avl_right) ? (self)->field.avl_right->field.avl_height : 0))
+
+/* Recursion prevents the following from being defined as macros. */
+
+#define TREE_DEFINE(node, field) \
+ \
+ static struct node *_HU_FUNCTION(TREE_BALANCE_##node##_##field)(struct node *); \
+ \
+ static struct node *_HU_FUNCTION(TREE_ROTL_##node##_##field)(struct node *self) \
+ { \
+ struct node *r= self->field.avl_right; \
+ self->field.avl_right= r->field.avl_left; \
+ r->field.avl_left= TREE_BALANCE_##node##_##field(self); \
+ return TREE_BALANCE_##node##_##field(r); \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_ROTR_##node##_##field)(struct node *self) \
+ { \
+ struct node *l= self->field.avl_left; \
+ self->field.avl_left= l->field.avl_right; \
+ l->field.avl_right= TREE_BALANCE_##node##_##field(self); \
+ return TREE_BALANCE_##node##_##field(l); \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_BALANCE_##node##_##field)(struct node *self) \
+ { \
+ int delta= TREE_DELTA(self, field); \
+ \
+ if (delta < -TREE_DELTA_MAX) \
+ { \
+ if (TREE_DELTA(self->field.avl_right, field) > 0) \
+ self->field.avl_right= TREE_ROTR_##node##_##field(self->field.avl_right); \
+ return TREE_ROTL_##node##_##field(self); \
+ } \
+ else if (delta > TREE_DELTA_MAX) \
+ { \
+ if (TREE_DELTA(self->field.avl_left, field) < 0) \
+ self->field.avl_left= TREE_ROTL_##node##_##field(self->field.avl_left); \
+ return TREE_ROTR_##node##_##field(self); \
+ } \
+ self->field.avl_height= 0; \
+ if (self->field.avl_left && (self->field.avl_left->field.avl_height > self->field.avl_height)) \
+ self->field.avl_height= self->field.avl_left->field.avl_height; \
+ if (self->field.avl_right && (self->field.avl_right->field.avl_height > self->field.avl_height)) \
+ self->field.avl_height= self->field.avl_right->field.avl_height; \
+ self->field.avl_height += 1; \
+ return self; \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_INSERT_##node##_##field) \
+ (struct node *self, struct node *elm, int (*compare)(struct node *lhs, struct node *rhs)) \
+ { \
+ if (!self) \
+ return elm; \
+ if (compare(elm, self) < 0) \
+ self->field.avl_left= TREE_INSERT_##node##_##field(self->field.avl_left, elm, compare); \
+ else \
+ self->field.avl_right= TREE_INSERT_##node##_##field(self->field.avl_right, elm, compare); \
+ return TREE_BALANCE_##node##_##field(self); \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_FIND_##node##_##field) \
+ (struct node *self, struct node *elm, int (*compare)(struct node *lhs, struct node *rhs)) \
+ { \
+ if (!self) \
+ return 0; \
+ if (compare(elm, self) == 0) \
+ return self; \
+ if (compare(elm, self) < 0) \
+ return TREE_FIND_##node##_##field(self->field.avl_left, elm, compare); \
+ else \
+ return TREE_FIND_##node##_##field(self->field.avl_right, elm, compare); \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_MOVE_RIGHT)(struct node *self, struct node *rhs) \
+ { \
+ if (!self) \
+ return rhs; \
+ self->field.avl_right= TREE_MOVE_RIGHT(self->field.avl_right, rhs); \
+ return TREE_BALANCE_##node##_##field(self); \
+ } \
+ \
+ static struct node *_HU_FUNCTION(TREE_REMOVE_##node##_##field) \
+ (struct node *self, struct node *elm, int (*compare)(struct node *lhs, struct node *rhs)) \
+ { \
+ if (!self) return 0; \
+ \
+ if (compare(elm, self) == 0) \
+ { \
+ struct node *tmp= TREE_MOVE_RIGHT(self->field.avl_left, self->field.avl_right); \
+ self->field.avl_left= 0; \
+ self->field.avl_right= 0; \
+ return tmp; \
+ } \
+ if (compare(elm, self) < 0) \
+ self->field.avl_left= TREE_REMOVE_##node##_##field(self->field.avl_left, elm, compare); \
+ else \
+ self->field.avl_right= TREE_REMOVE_##node##_##field(self->field.avl_right, elm, compare); \
+ return TREE_BALANCE_##node##_##field(self); \
+ } \
+ \
+ static void _HU_FUNCTION(TREE_FORWARD_APPLY_ALL_##node##_##field) \
+ (struct node *self, void (*function)(struct node *node, void *data), void *data) \
+ { \
+ if (self) \
+ { \
+ TREE_FORWARD_APPLY_ALL_##node##_##field(self->field.avl_left, function, data); \
+ function(self, data); \
+ TREE_FORWARD_APPLY_ALL_##node##_##field(self->field.avl_right, function, data); \
+ } \
+ } \
+ \
+ static void _HU_FUNCTION(TREE_REVERSE_APPLY_ALL_##node##_##field) \
+ (struct node *self, void (*function)(struct node *node, void *data), void *data) \
+ { \
+ if (self) \
+ { \
+ TREE_REVERSE_APPLY_ALL_##node##_##field(self->field.avl_right, function, data); \
+ function(self, data); \
+ TREE_REVERSE_APPLY_ALL_##node##_##field(self->field.avl_left, function, data); \
+ } \
+ }
+
+#define TREE_INSERT(head, node, field, elm) \
+ ((head)->th_root= TREE_INSERT_##node##_##field((head)->th_root, (elm), (head)->th_cmp))
+
+#define TREE_FIND(head, node, field, elm) \
+ (TREE_FIND_##node##_##field((head)->th_root, (elm), (head)->th_cmp))
+
+#define TREE_REMOVE(head, node, field, elm) \
+ ((head)->th_root= TREE_REMOVE_##node##_##field((head)->th_root, (elm), (head)->th_cmp))
+
+#define TREE_DEPTH(head, field) \
+ ((head)->th_root->field.avl_height)
+
+#define TREE_FORWARD_APPLY(head, node, field, function, data) \
+ TREE_FORWARD_APPLY_ALL_##node##_##field((head)->th_root, function, data)
+
+#define TREE_REVERSE_APPLY(head, node, field, function, data) \
+ TREE_REVERSE_APPLY_ALL_##node##_##field((head)->th_root, function, data)
+
+#define TREE_INIT(head, cmp) do { \
+ (head)->th_root= 0; \
+ (head)->th_cmp= (cmp); \
+ } while (0)
+
+
+#endif /* __tree_h */
diff --git a/contrib/libucl/ucl.h b/contrib/libucl/ucl.h
new file mode 100644
index 0000000..b6b9f44
--- /dev/null
+++ b/contrib/libucl/ucl.h
@@ -0,0 +1,1650 @@
+/* Copyright (c) 2013-2015, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UCL_H_
+#define UCL_H_
+
+#include <string.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <stdio.h>
+
+#ifdef _WIN32
+# define UCL_EXTERN __declspec(dllexport)
+#else
+# define UCL_EXTERN
+#endif
+
+/**
+ * @mainpage
+ * This is a reference manual for UCL API. You may find the description of UCL format by following this
+ * [github repository](https://github.com/vstakhov/libucl).
+ *
+ * This manual has several main sections:
+ * - @ref structures
+ * - @ref utils
+ * - @ref parser
+ * - @ref emitter
+ */
+
+/**
+ * @file ucl.h
+ * @brief UCL parsing and emitting functions
+ *
+ * UCL is universal configuration language, which is a form of
+ * JSON with less strict rules that make it more comfortable for
+ * using as a configuration language
+ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Memory allocation utilities
+ * UCL_ALLOC(size) - allocate memory for UCL
+ * UCL_FREE(size, ptr) - free memory of specified size at ptr
+ * Default: malloc and free
+ */
+#ifndef UCL_ALLOC
+#define UCL_ALLOC(size) malloc(size)
+#endif
+#ifndef UCL_FREE
+#define UCL_FREE(size, ptr) free(ptr)
+#endif
+
+#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)
+#define UCL_WARN_UNUSED_RESULT \
+ __attribute__((warn_unused_result))
+#else
+#define UCL_WARN_UNUSED_RESULT
+#endif
+
+#ifdef __GNUC__
+#define UCL_DEPRECATED(func) func __attribute__ ((deprecated))
+#elif defined(_MSC_VER)
+#define UCL_DEPRECATED(func) __declspec(deprecated) func
+#else
+#define UCL_DEPRECATED(func) func
+#endif
+
+/**
+ * @defgroup structures Structures and types
+ * UCL defines several enumeration types used for error reporting or specifying flags and attributes.
+ *
+ * @{
+ */
+
+/**
+ * The common error codes returned by ucl parser
+ */
+typedef enum ucl_error {
+ UCL_EOK = 0, /**< No error */
+ UCL_ESYNTAX, /**< Syntax error occurred during parsing */
+ UCL_EIO, /**< IO error occurred during parsing */
+ UCL_ESTATE, /**< Invalid state machine state */
+ UCL_ENESTED, /**< Input has too many recursion levels */
+ UCL_EUNPAIRED, /**< Input has too many recursion levels */
+ UCL_EMACRO, /**< Error processing a macro */
+ UCL_EINTERNAL, /**< Internal unclassified error */
+ UCL_ESSL, /**< SSL error */
+ UCL_EMERGE /**< A merge error occurred */
+} ucl_error_t;
+
+/**
+ * #ucl_object_t may have one of specified types, some types are compatible with each other and some are not.
+ * For example, you can always convert #UCL_TIME to #UCL_FLOAT. Also you can convert #UCL_FLOAT to #UCL_INTEGER
+ * by loosing floating point. Every object may be converted to a string by #ucl_object_tostring_forced() function.
+ *
+ */
+typedef enum ucl_type {
+ UCL_OBJECT = 0, /**< UCL object - key/value pairs */
+ UCL_ARRAY, /**< UCL array */
+ UCL_INT, /**< Integer number */
+ UCL_FLOAT, /**< Floating point number */
+ UCL_STRING, /**< Null terminated string */
+ UCL_BOOLEAN, /**< Boolean value */
+ UCL_TIME, /**< Time value (floating point number of seconds) */
+ UCL_USERDATA, /**< Opaque userdata pointer (may be used in macros) */
+ UCL_NULL /**< Null value */
+} ucl_type_t;
+
+/**
+ * You can use one of these types to serialise #ucl_object_t by using ucl_object_emit().
+ */
+typedef enum ucl_emitter {
+ UCL_EMIT_JSON = 0, /**< Emit fine formatted JSON */
+ UCL_EMIT_JSON_COMPACT, /**< Emit compacted JSON */
+ UCL_EMIT_CONFIG, /**< Emit human readable config format */
+ UCL_EMIT_YAML, /**< Emit embedded YAML format */
+ UCL_EMIT_MSGPACK, /**< Emit msgpack output */
+ UCL_EMIT_MAX /**< Unsupported emitter type */
+} ucl_emitter_t;
+
+/**
+ * These flags defines parser behaviour. If you specify #UCL_PARSER_ZEROCOPY you must ensure
+ * that the input memory is not freed if an object is in use. Moreover, if you want to use
+ * zero-terminated keys and string values then you should not use zero-copy mode, as in this case
+ * UCL still has to perform copying implicitly.
+ */
+typedef enum ucl_parser_flags {
+ UCL_PARSER_DEFAULT = 0, /**< No special flags */
+ UCL_PARSER_KEY_LOWERCASE = (1 << 0), /**< Convert all keys to lower case */
+ UCL_PARSER_ZEROCOPY = (1 << 1), /**< Parse input in zero-copy mode if possible */
+ UCL_PARSER_NO_TIME = (1 << 2), /**< Do not parse time and treat time values as strings */
+ UCL_PARSER_NO_IMPLICIT_ARRAYS = (1 << 3), /** Create explicit arrays instead of implicit ones */
+ UCL_PARSER_SAVE_COMMENTS = (1 << 4), /** Save comments in the parser context */
+ UCL_PARSER_DISABLE_MACRO = (1 << 5), /** Treat macros as comments */
+ UCL_PARSER_NO_FILEVARS = (1 << 6) /** Do not set file vars */
+} ucl_parser_flags_t;
+
+/**
+ * String conversion flags, that are used in #ucl_object_fromstring_common function.
+ */
+typedef enum ucl_string_flags {
+ UCL_STRING_RAW = 0x0, /**< Treat string as is */
+ UCL_STRING_ESCAPE = (1 << 0), /**< Perform JSON escape */
+ UCL_STRING_TRIM = (1 << 1), /**< Trim leading and trailing whitespaces */
+ UCL_STRING_PARSE_BOOLEAN = (1 << 2), /**< Parse passed string and detect boolean */
+ UCL_STRING_PARSE_INT = (1 << 3), /**< Parse passed string and detect integer number */
+ UCL_STRING_PARSE_DOUBLE = (1 << 4), /**< Parse passed string and detect integer or float number */
+ UCL_STRING_PARSE_TIME = (1 << 5), /**< Parse time strings */
+ UCL_STRING_PARSE_NUMBER = UCL_STRING_PARSE_INT|UCL_STRING_PARSE_DOUBLE|UCL_STRING_PARSE_TIME, /**<
+ Parse passed string and detect number */
+ UCL_STRING_PARSE = UCL_STRING_PARSE_BOOLEAN|UCL_STRING_PARSE_NUMBER, /**<
+ Parse passed string (and detect booleans and numbers) */
+ UCL_STRING_PARSE_BYTES = (1 << 6) /**< Treat numbers as bytes */
+} ucl_string_flags_t;
+
+/**
+ * Basic flags for an object (can use up to 12 bits as higher 4 bits are used
+ * for priorities)
+ */
+typedef enum ucl_object_flags {
+ UCL_OBJECT_ALLOCATED_KEY = (1 << 0), /**< An object has key allocated internally */
+ UCL_OBJECT_ALLOCATED_VALUE = (1 << 1), /**< An object has a string value allocated internally */
+ UCL_OBJECT_NEED_KEY_ESCAPE = (1 << 2), /**< The key of an object need to be escaped on output */
+ UCL_OBJECT_EPHEMERAL = (1 << 3), /**< Temporary object that does not need to be freed really */
+ UCL_OBJECT_MULTILINE = (1 << 4), /**< String should be displayed as multiline string */
+ UCL_OBJECT_MULTIVALUE = (1 << 5), /**< Object is a key with multiple values */
+ UCL_OBJECT_INHERITED = (1 << 6), /**< Object has been inherited from another */
+ UCL_OBJECT_BINARY = (1 << 7), /**< Object contains raw binary data */
+ UCL_OBJECT_SQUOTED = (1 << 8) /**< Object has been enclosed in single quotes */
+} ucl_object_flags_t;
+
+/**
+ * Duplicate policy types
+ */
+enum ucl_duplicate_strategy {
+ UCL_DUPLICATE_APPEND = 0, /**< Default policy to merge based on priorities */
+ UCL_DUPLICATE_MERGE, /**< Merge new object with old one */
+ UCL_DUPLICATE_REWRITE, /**< Rewrite old keys */
+ UCL_DUPLICATE_ERROR /**< Stop parsing on duplicate found */
+};
+
+/**
+ * Input format type
+ */
+enum ucl_parse_type {
+ UCL_PARSE_UCL = 0, /**< Default ucl format */
+ UCL_PARSE_MSGPACK, /**< Message pack input format */
+ UCL_PARSE_CSEXP, /**< Canonical S-expressions */
+ UCL_PARSE_AUTO /**< Try to detect parse type */
+};
+
+/**
+ * UCL object structure. Please mention that the most of fields should not be touched by
+ * UCL users. In future, this structure may be converted to private one.
+ */
+typedef struct ucl_object_s {
+ /**
+ * Variant value type
+ */
+ union {
+ int64_t iv; /**< Int value of an object */
+ const char *sv; /**< String value of an object */
+ double dv; /**< Double value of an object */
+ void *av; /**< Array */
+ void *ov; /**< Object */
+ void* ud; /**< Opaque user data */
+ } value;
+ const char *key; /**< Key of an object */
+ struct ucl_object_s *next; /**< Array handle */
+ struct ucl_object_s *prev; /**< Array handle */
+ uint32_t keylen; /**< Length of a key */
+ uint32_t len; /**< Size of an object */
+ uint32_t ref; /**< Reference count */
+ uint16_t flags; /**< Object flags */
+ uint16_t type; /**< Real type */
+ unsigned char* trash_stack[2]; /**< Pointer to allocated chunks */
+} ucl_object_t;
+
+/**
+ * Destructor type for userdata objects
+ * @param ud user specified data pointer
+ */
+typedef void (*ucl_userdata_dtor)(void *ud);
+typedef const char* (*ucl_userdata_emitter)(void *ud);
+
+/** @} */
+
+/**
+ * @defgroup utils Utility functions
+ * A number of utility functions simplify handling of UCL objects
+ *
+ * @{
+ */
+/**
+ * Copy and return a key of an object, returned key is zero-terminated
+ * @param obj CL object
+ * @return zero terminated key
+ */
+UCL_EXTERN char* ucl_copy_key_trash (const ucl_object_t *obj);
+
+/**
+ * Copy and return a string value of an object, returned key is zero-terminated
+ * @param obj CL object
+ * @return zero terminated string representation of object value
+ */
+UCL_EXTERN char* ucl_copy_value_trash (const ucl_object_t *obj);
+
+/**
+ * Creates a new object
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_new (void) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create new object with type specified
+ * @param type type of a new object
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_typed_new (ucl_type_t type) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create new object with type and priority specified
+ * @param type type of a new object
+ * @param priority priority of an object
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_new_full (ucl_type_t type, unsigned priority)
+ UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create new object with userdata dtor
+ * @param dtor destructor function
+ * @param emitter emitter for userdata
+ * @param ptr opaque pointer
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_new_userdata (ucl_userdata_dtor dtor,
+ ucl_userdata_emitter emitter, void *ptr) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Perform deep copy of an object copying everything
+ * @param other object to copy
+ * @return new object with refcount equal to 1
+ */
+UCL_EXTERN ucl_object_t * ucl_object_copy (const ucl_object_t *other)
+ UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Return the type of an object
+ * @return the object type
+ */
+UCL_EXTERN ucl_type_t ucl_object_type (const ucl_object_t *obj);
+
+/**
+ * Converts ucl object type to its string representation
+ * @param type type of object
+ * @return constant string describing type
+ */
+UCL_EXTERN const char * ucl_object_type_to_string (ucl_type_t type);
+
+/**
+ * Converts string that represents ucl type to real ucl type enum
+ * @param input C string with name of type
+ * @param res resulting target
+ * @return true if `input` is a name of type stored in `res`
+ */
+UCL_EXTERN bool ucl_object_string_to_type (const char *input, ucl_type_t *res);
+
+/**
+ * Convert any string to an ucl object making the specified transformations
+ * @param str fixed size or NULL terminated string
+ * @param len length (if len is zero, than str is treated as NULL terminated)
+ * @param flags conversion flags
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t * ucl_object_fromstring_common (const char *str, size_t len,
+ enum ucl_string_flags flags) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create a UCL object from the specified string
+ * @param str NULL terminated string, will be json escaped
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t *ucl_object_fromstring (const char *str) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create a UCL object from the specified string
+ * @param str fixed size string, will be json escaped
+ * @param len length of a string
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t *ucl_object_fromlstring (const char *str,
+ size_t len) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create an object from an integer number
+ * @param iv number
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_fromint (int64_t iv) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create an object from a float number
+ * @param dv number
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_fromdouble (double dv) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Create an object from a boolean
+ * @param bv bool value
+ * @return new object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_frombool (bool bv) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Insert a object 'elt' to the hash 'top' and associate it with key 'key'
+ * @param top destination object (must be of type UCL_OBJECT)
+ * @param elt element to insert (must NOT be NULL)
+ * @param key key to associate with this object (either const or preallocated)
+ * @param keylen length of the key (or 0 for NULL terminated keys)
+ * @param copy_key make an internal copy of key
+ * @return true if key has been inserted
+ */
+UCL_EXTERN bool ucl_object_insert_key (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key);
+
+/**
+ * Replace a object 'elt' to the hash 'top' and associate it with key 'key', old object will be unrefed,
+ * if no object has been found this function works like ucl_object_insert_key()
+ * @param top destination object (must be of type UCL_OBJECT)
+ * @param elt element to insert (must NOT be NULL)
+ * @param key key to associate with this object (either const or preallocated)
+ * @param keylen length of the key (or 0 for NULL terminated keys)
+ * @param copy_key make an internal copy of key
+ * @return true if key has been inserted
+ */
+UCL_EXTERN bool ucl_object_replace_key (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key);
+
+/**
+ * Merge the keys from one object to another object. Overwrite on conflict
+ * @param top destination object (must be of type UCL_OBJECT)
+ * @param elt element to insert (must be of type UCL_OBJECT)
+ * @param copy copy rather than reference the elements
+ * @return true if all keys have been merged
+ */
+UCL_EXTERN bool ucl_object_merge (ucl_object_t *top, ucl_object_t *elt, bool copy);
+
+/**
+ * Delete a object associated with key 'key', old object will be unrefered,
+ * @param top object
+ * @param key key associated to the object to remove
+ * @param keylen length of the key (or 0 for NULL terminated keys)
+ */
+UCL_EXTERN bool ucl_object_delete_keyl (ucl_object_t *top,
+ const char *key, size_t keylen);
+
+/**
+ * Delete a object associated with key 'key', old object will be unrefered,
+ * @param top object
+ * @param key key associated to the object to remove
+ */
+UCL_EXTERN bool ucl_object_delete_key (ucl_object_t *top,
+ const char *key);
+
+
+/**
+ * Removes `key` from `top` object, returning the object that was removed. This
+ * object is not released, caller must unref the returned object when it is no
+ * longer needed.
+ * @param top object
+ * @param key key to remove
+ * @param keylen length of the key (or 0 for NULL terminated keys)
+ * @return removed object or NULL if object has not been found
+ */
+UCL_EXTERN ucl_object_t* ucl_object_pop_keyl (ucl_object_t *top, const char *key,
+ size_t keylen) UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Removes `key` from `top` object returning the object that was removed. This
+ * object is not released, caller must unref the returned object when it is no
+ * longer needed.
+ * @param top object
+ * @param key key to remove
+ * @return removed object or NULL if object has not been found
+ */
+UCL_EXTERN ucl_object_t* ucl_object_pop_key (ucl_object_t *top, const char *key)
+ UCL_WARN_UNUSED_RESULT;
+
+/**
+ * Insert a object 'elt' to the hash 'top' and associate it with key 'key', if
+ * the specified key exist, try to merge its content
+ * @param top destination object (must be of type UCL_OBJECT)
+ * @param elt element to insert (must NOT be NULL)
+ * @param key key to associate with this object (either const or preallocated)
+ * @param keylen length of the key (or 0 for NULL terminated keys)
+ * @param copy_key make an internal copy of key
+ * @return true if key has been inserted
+ */
+UCL_EXTERN bool ucl_object_insert_key_merged (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key);
+
+/**
+ * Reserve space in ucl array or object for `elt` elements
+ * @param obj object to reserve
+ * @param reserved size to reserve in an object
+ * @return 0 on success, -1 on failure (i.e. ENOMEM)
+ */
+UCL_EXTERN bool ucl_object_reserve (ucl_object_t *obj, size_t reserved);
+
+/**
+ * Append an element to the end of array object
+ * @param top destination object (must NOT be NULL)
+ * @param elt element to append (must NOT be NULL)
+ * @return true if value has been inserted
+ */
+UCL_EXTERN bool ucl_array_append (ucl_object_t *top,
+ ucl_object_t *elt);
+
+/**
+ * Append an element to the start of array object
+ * @param top destination object (must NOT be NULL)
+ * @param elt element to append (must NOT be NULL)
+ * @return true if value has been inserted
+ */
+UCL_EXTERN bool ucl_array_prepend (ucl_object_t *top,
+ ucl_object_t *elt);
+
+/**
+ * Merge all elements of second array into the first array
+ * @param top destination array (must be of type UCL_ARRAY)
+ * @param elt array to copy elements from (must be of type UCL_ARRAY)
+ * @param copy copy elements instead of referencing them
+ * @return true if arrays were merged
+ */
+UCL_EXTERN bool ucl_array_merge (ucl_object_t *top, ucl_object_t *elt,
+ bool copy);
+
+/**
+ * Removes an element `elt` from the array `top`, returning the object that was
+ * removed. This object is not released, caller must unref the returned object
+ * when it is no longer needed.
+ * @param top array ucl object
+ * @param elt element to remove
+ * @return removed element or NULL if `top` is NULL or not an array
+ */
+UCL_EXTERN ucl_object_t* ucl_array_delete (ucl_object_t *top,
+ ucl_object_t *elt);
+
+/**
+ * Returns the first element of the array `top`
+ * @param top array ucl object
+ * @return element or NULL if `top` is NULL or not an array
+ */
+UCL_EXTERN const ucl_object_t* ucl_array_head (const ucl_object_t *top);
+
+/**
+ * Returns the last element of the array `top`
+ * @param top array ucl object
+ * @return element or NULL if `top` is NULL or not an array
+ */
+UCL_EXTERN const ucl_object_t* ucl_array_tail (const ucl_object_t *top);
+
+/**
+ * Removes the last element from the array `top`, returning the object that was
+ * removed. This object is not released, caller must unref the returned object
+ * when it is no longer needed.
+ * @param top array ucl object
+ * @return removed element or NULL if `top` is NULL or not an array
+ */
+UCL_EXTERN ucl_object_t* ucl_array_pop_last (ucl_object_t *top);
+
+/**
+ * Removes the first element from the array `top`, returning the object that was
+ * removed. This object is not released, caller must unref the returned object
+ * when it is no longer needed.
+ * @param top array ucl object
+ * @return removed element or NULL if `top` is NULL or not an array
+ */
+UCL_EXTERN ucl_object_t* ucl_array_pop_first (ucl_object_t *top);
+
+/**
+ * Return size of the array `top`
+ * @param top object to get size from (must be of type UCL_ARRAY)
+ * @return size of the array
+ */
+UCL_EXTERN unsigned int ucl_array_size (const ucl_object_t *top);
+
+/**
+ * Return object identified by index of the array `top`
+ * @param top object to get a key from (must be of type UCL_ARRAY)
+ * @param index array index to return
+ * @return object at the specified index or NULL if index is not found
+ */
+UCL_EXTERN const ucl_object_t* ucl_array_find_index (const ucl_object_t *top,
+ unsigned int index);
+
+/**
+ * Return the index of `elt` in the array `top`
+ * @param top object to get a key from (must be of type UCL_ARRAY)
+ * @param elt element to find index of (must NOT be NULL)
+ * @return index of `elt` in the array `top or (unsigned int)-1 if `elt` is not found
+ */
+UCL_EXTERN unsigned int ucl_array_index_of (ucl_object_t *top,
+ ucl_object_t *elt);
+
+/**
+ * Replace an element in an array with a different element, returning the object
+ * that was replaced. This object is not released, caller must unref the
+ * returned object when it is no longer needed.
+ * @param top destination object (must be of type UCL_ARRAY)
+ * @param elt element to append (must NOT be NULL)
+ * @param index array index in destination to overwrite with elt
+ * @return object that was replaced or NULL if index is not found
+ */
+ucl_object_t *
+ucl_array_replace_index (ucl_object_t *top, ucl_object_t *elt,
+ unsigned int index);
+
+/**
+ * Append a element to another element forming an implicit array
+ * @param head head to append (may be NULL)
+ * @param elt new element
+ * @return the new implicit array
+ */
+UCL_EXTERN ucl_object_t * ucl_elt_append (ucl_object_t *head,
+ ucl_object_t *elt);
+
+/**
+ * Converts an object to double value
+ * @param obj CL object
+ * @param target target double variable
+ * @return true if conversion was successful
+ */
+UCL_EXTERN bool ucl_object_todouble_safe (const ucl_object_t *obj, double *target);
+
+/**
+ * Unsafe version of \ref ucl_obj_todouble_safe
+ * @param obj CL object
+ * @return double value
+ */
+UCL_EXTERN double ucl_object_todouble (const ucl_object_t *obj);
+
+/**
+ * Converts an object to integer value
+ * @param obj CL object
+ * @param target target integer variable
+ * @return true if conversion was successful
+ */
+UCL_EXTERN bool ucl_object_toint_safe (const ucl_object_t *obj, int64_t *target);
+
+/**
+ * Unsafe version of \ref ucl_obj_toint_safe
+ * @param obj CL object
+ * @return int value
+ */
+UCL_EXTERN int64_t ucl_object_toint (const ucl_object_t *obj);
+
+/**
+ * Converts an object to boolean value
+ * @param obj CL object
+ * @param target target boolean variable
+ * @return true if conversion was successful
+ */
+UCL_EXTERN bool ucl_object_toboolean_safe (const ucl_object_t *obj, bool *target);
+
+/**
+ * Unsafe version of \ref ucl_obj_toboolean_safe
+ * @param obj CL object
+ * @return boolean value
+ */
+UCL_EXTERN bool ucl_object_toboolean (const ucl_object_t *obj);
+
+/**
+ * Converts an object to string value
+ * @param obj CL object
+ * @param target target string variable, no need to free value
+ * @return true if conversion was successful
+ */
+UCL_EXTERN bool ucl_object_tostring_safe (const ucl_object_t *obj, const char **target);
+
+/**
+ * Unsafe version of \ref ucl_obj_tostring_safe
+ * @param obj CL object
+ * @return string value
+ */
+UCL_EXTERN const char* ucl_object_tostring (const ucl_object_t *obj);
+
+/**
+ * Convert any object to a string in JSON notation if needed
+ * @param obj CL object
+ * @return string value
+ */
+UCL_EXTERN const char* ucl_object_tostring_forced (const ucl_object_t *obj);
+
+/**
+ * Return string as char * and len, string may be not zero terminated, more efficient that \ref ucl_obj_tostring as it
+ * allows zero-copy (if #UCL_PARSER_ZEROCOPY has been used during parsing)
+ * @param obj CL object
+ * @param target target string variable, no need to free value
+ * @param tlen target length
+ * @return true if conversion was successful
+ */
+UCL_EXTERN bool ucl_object_tolstring_safe (const ucl_object_t *obj,
+ const char **target, size_t *tlen);
+
+/**
+ * Unsafe version of \ref ucl_obj_tolstring_safe
+ * @param obj CL object
+ * @return string value
+ */
+UCL_EXTERN const char* ucl_object_tolstring (const ucl_object_t *obj, size_t *tlen);
+
+/**
+ * Return object identified by a key in the specified object
+ * @param obj object to get a key from (must be of type UCL_OBJECT)
+ * @param key key to search
+ * @return object matching the specified key or NULL if key was not found
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_lookup (const ucl_object_t *obj,
+ const char *key);
+#define ucl_object_find_key ucl_object_lookup
+
+/**
+ * Return object identified by a key in the specified object, if the first key is
+ * not found then look for the next one. This process is repeated unless
+ * the next argument in the list is not NULL. So, `ucl_object_find_any_key(obj, key, NULL)`
+ * is equal to `ucl_object_find_key(obj, key)`
+ * @param obj object to get a key from (must be of type UCL_OBJECT)
+ * @param key key to search
+ * @param ... list of alternative keys to search (NULL terminated)
+ * @return object matching the specified key or NULL if key was not found
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_lookup_any (const ucl_object_t *obj,
+ const char *key, ...);
+#define ucl_object_find_any_key ucl_object_lookup_any
+
+/**
+ * Return object identified by a fixed size key in the specified object
+ * @param obj object to get a key from (must be of type UCL_OBJECT)
+ * @param key key to search
+ * @param klen length of a key
+ * @return object matching the specified key or NULL if key was not found
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_lookup_len (const ucl_object_t *obj,
+ const char *key, size_t klen);
+#define ucl_object_find_keyl ucl_object_lookup_len
+
+/**
+ * Return object identified by dot notation string
+ * @param obj object to search in
+ * @param path dot.notation.path to the path to lookup. May use numeric .index on arrays
+ * @return object matched the specified path or NULL if path is not found
+ */
+UCL_EXTERN const ucl_object_t *ucl_object_lookup_path (const ucl_object_t *obj,
+ const char *path);
+#define ucl_lookup_path ucl_object_lookup_path
+
+/**
+ * Return object identified by object notation string using arbitrary delimiter
+ * @param obj object to search in
+ * @param path dot.notation.path to the path to lookup. May use numeric .index on arrays
+ * @param sep the sepatorator to use in place of . (incase keys have . in them)
+ * @return object matched the specified path or NULL if path is not found
+ */
+UCL_EXTERN const ucl_object_t *ucl_object_lookup_path_char (const ucl_object_t *obj,
+ const char *path, char sep);
+#define ucl_lookup_path_char ucl_object_lookup_path_char
+
+/**
+ * Returns a key of an object as a NULL terminated string
+ * @param obj CL object
+ * @return key or NULL if there is no key
+ */
+UCL_EXTERN const char* ucl_object_key (const ucl_object_t *obj);
+
+/**
+ * Returns a key of an object as a fixed size string (may be more efficient)
+ * @param obj CL object
+ * @param len target key length
+ * @return key pointer
+ */
+UCL_EXTERN const char* ucl_object_keyl (const ucl_object_t *obj, size_t *len);
+
+/**
+ * Increase reference count for an object
+ * @param obj object to ref
+ * @return the referenced object
+ */
+UCL_EXTERN ucl_object_t* ucl_object_ref (const ucl_object_t *obj);
+
+/**
+ * Free ucl object
+ * @param obj ucl object to free
+ */
+UCL_DEPRECATED(UCL_EXTERN void ucl_object_free (ucl_object_t *obj));
+
+/**
+ * Decrease reference count for an object
+ * @param obj object to unref
+ */
+UCL_EXTERN void ucl_object_unref (ucl_object_t *obj);
+
+/**
+ * Compare objects `o1` and `o2`
+ * @param o1 the first object
+ * @param o2 the second object
+ * @return values >0, 0 and <0 if `o1` is more than, equal and less than `o2`.
+ * The order of comparison:
+ * 1) Type of objects
+ * 2) Size of objects
+ * 3) Content of objects
+ */
+UCL_EXTERN int ucl_object_compare (const ucl_object_t *o1,
+ const ucl_object_t *o2);
+
+/**
+ * Compare objects `o1` and `o2` useful for sorting
+ * @param o1 the first object
+ * @param o2 the second object
+ * @return values >0, 0 and <0 if `o1` is more than, equal and less than `o2`.
+ * The order of comparison:
+ * 1) Type of objects
+ * 2) Size of objects
+ * 3) Content of objects
+ */
+UCL_EXTERN int ucl_object_compare_qsort (const ucl_object_t **o1,
+ const ucl_object_t **o2);
+
+/**
+ * Sort UCL array using `cmp` compare function
+ * @param ar
+ * @param cmp
+ */
+UCL_EXTERN void ucl_object_array_sort (ucl_object_t *ar,
+ int (*cmp)(const ucl_object_t **o1, const ucl_object_t **o2));
+
+enum ucl_object_keys_sort_flags {
+ UCL_SORT_KEYS_DEFAULT = 0,
+ UCL_SORT_KEYS_ICASE = (1u << 0u),
+ UCL_SORT_KEYS_RECURSIVE = (1u << 1u),
+};
+/***
+ * Sorts keys in object in place
+ * @param obj
+ * @param how
+ */
+UCL_EXTERN void ucl_object_sort_keys (ucl_object_t *obj,
+ enum ucl_object_keys_sort_flags how);
+
+/**
+ * Get the priority for specific UCL object
+ * @param obj any ucl object
+ * @return priority of an object
+ */
+UCL_EXTERN unsigned int ucl_object_get_priority (const ucl_object_t *obj);
+
+/**
+ * Set explicit priority of an object.
+ * @param obj any ucl object
+ * @param priority new priroity value (only 4 least significant bits are considred)
+ */
+UCL_EXTERN void ucl_object_set_priority (ucl_object_t *obj,
+ unsigned int priority);
+
+/**
+ * Opaque iterator object
+ */
+typedef void* ucl_object_iter_t;
+
+/**
+ * Get next key from an object
+ * @param obj object to iterate
+ * @param iter opaque iterator, must be set to NULL on the first call:
+ * ucl_object_iter_t it = NULL;
+ * while ((cur = ucl_iterate_object (obj, &it)) != NULL) ...
+ * @param ep pointer record exception (such as ENOMEM), could be NULL
+ * @return the next object or NULL
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_iterate_with_error (const ucl_object_t *obj,
+ ucl_object_iter_t *iter, bool expand_values, int *ep);
+
+#define ucl_iterate_object ucl_object_iterate
+#define ucl_object_iterate(ob, it, ev) ucl_object_iterate_with_error((ob), (it), (ev), NULL)
+
+/**
+ * Create new safe iterator for the specified object
+ * @param obj object to iterate
+ * @return new iterator object that should be used with safe iterators API only
+ */
+UCL_EXTERN ucl_object_iter_t ucl_object_iterate_new (const ucl_object_t *obj)
+ UCL_WARN_UNUSED_RESULT;
+/**
+ * Check safe iterator object after performing some operations on it
+ * (such as ucl_object_iterate_safe()) to see if operation has encountered
+ * fatal exception while performing that operation (e.g. ENOMEM).
+ * @param iter opaque iterator
+ * @return true if exception has occured, false otherwise
+ */
+UCL_EXTERN bool ucl_object_iter_chk_excpn(ucl_object_iter_t *it);
+
+/**
+ * Reset initialized iterator to a new object
+ * @param obj new object to iterate
+ * @return modified iterator object
+ */
+UCL_EXTERN ucl_object_iter_t ucl_object_iterate_reset (ucl_object_iter_t it,
+ const ucl_object_t *obj);
+
+/**
+ * Get the next object from the `obj`. This function iterates over arrays, objects
+ * and implicit arrays
+ * @param iter safe iterator
+ * @param expand_values expand explicit arrays and objects
+ * @return the next object in sequence
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_iterate_safe (ucl_object_iter_t iter,
+ bool expand_values);
+/**
+ * Iteration type enumerator
+ */
+enum ucl_iterate_type {
+ UCL_ITERATE_EXPLICIT = 1 << 0, /**< Iterate just explicit arrays and objects */
+ UCL_ITERATE_IMPLICIT = 1 << 1, /**< Iterate just implicit arrays */
+ UCL_ITERATE_BOTH = (1 << 0) | (1 << 1), /**< Iterate both explicit and implicit arrays*/
+};
+
+/**
+ * Get the next object from the `obj`. This function iterates over arrays, objects
+ * and implicit arrays if needed
+ * @param iter safe iterator
+ * @param
+ * @return the next object in sequence
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_iterate_full (ucl_object_iter_t iter,
+ enum ucl_iterate_type type);
+
+/**
+ * Free memory associated with the safe iterator
+ * @param it safe iterator object
+ */
+UCL_EXTERN void ucl_object_iterate_free (ucl_object_iter_t it);
+
+/** @} */
+
+
+/**
+ * @defgroup parser Parsing functions
+ * These functions are used to parse UCL objects
+ *
+ * @{
+ */
+
+/**
+ * Macro handler for a parser
+ * @param data the content of macro
+ * @param len the length of content
+ * @param arguments arguments object
+ * @param ud opaque user data
+ * @param err error pointer
+ * @return true if macro has been parsed
+ */
+typedef bool (*ucl_macro_handler) (const unsigned char *data, size_t len,
+ const ucl_object_t *arguments,
+ void* ud);
+
+/**
+ * Context dependent macro handler for a parser
+ * @param data the content of macro
+ * @param len the length of content
+ * @param arguments arguments object
+ * @param context previously parsed context
+ * @param ud opaque user data
+ * @param err error pointer
+ * @return true if macro has been parsed
+ */
+typedef bool (*ucl_context_macro_handler) (const unsigned char *data, size_t len,
+ const ucl_object_t *arguments,
+ const ucl_object_t *context,
+ void* ud);
+
+/* Opaque parser */
+struct ucl_parser;
+
+/**
+ * Creates new parser object
+ * @param pool pool to allocate memory from
+ * @return new parser object
+ */
+UCL_EXTERN struct ucl_parser* ucl_parser_new (int flags);
+
+/**
+ * Sets the default priority for the parser applied to chunks that do not
+ * specify priority explicitly
+ * @param parser parser object
+ * @param prio default priority (0 .. 16)
+ * @return true if parser's default priority was set
+ */
+UCL_EXTERN bool ucl_parser_set_default_priority (struct ucl_parser *parser,
+ unsigned prio);
+/**
+ * Gets the default priority for the parser applied to chunks that do not
+ * specify priority explicitly
+ * @param parser parser object
+ * @return true default priority (0 .. 16), -1 for failure
+ */
+UCL_EXTERN int ucl_parser_get_default_priority (struct ucl_parser *parser);
+
+/**
+ * Register new handler for a macro
+ * @param parser parser object
+ * @param macro macro name (without leading dot)
+ * @param handler handler (it is called immediately after macro is parsed)
+ * @param ud opaque user data for a handler
+ * @return true on success, false on failure (i.e. ENOMEM)
+ */
+UCL_EXTERN bool ucl_parser_register_macro (struct ucl_parser *parser,
+ const char *macro,
+ ucl_macro_handler handler, void* ud);
+
+/**
+ * Register new context dependent handler for a macro
+ * @param parser parser object
+ * @param macro macro name (without leading dot)
+ * @param handler handler (it is called immediately after macro is parsed)
+ * @param ud opaque user data for a handler
+ * @return true on success, false on failure (i.e. ENOMEM)
+ */
+UCL_EXTERN bool ucl_parser_register_context_macro (struct ucl_parser *parser,
+ const char *macro,
+ ucl_context_macro_handler handler,
+ void* ud);
+
+/**
+ * Handler to detect unregistered variables
+ * @param data variable data
+ * @param len length of variable
+ * @param replace (out) replace value for variable
+ * @param replace_len (out) replace length for variable
+ * @param need_free (out) UCL will free `dest` after usage
+ * @param ud opaque userdata
+ * @return true if variable
+ */
+typedef bool (*ucl_variable_handler) (const unsigned char *data, size_t len,
+ unsigned char **replace, size_t *replace_len, bool *need_free, void* ud);
+
+/**
+ * Register new parser variable
+ * @param parser parser object
+ * @param var variable name
+ * @param value variable value
+ */
+UCL_EXTERN void ucl_parser_register_variable (struct ucl_parser *parser, const char *var,
+ const char *value);
+
+/**
+ * Set handler for unknown variables
+ * @param parser parser structure
+ * @param handler desired handler
+ * @param ud opaque data for the handler
+ */
+UCL_EXTERN void ucl_parser_set_variables_handler (struct ucl_parser *parser,
+ ucl_variable_handler handler, void *ud);
+
+/**
+ * Load new chunk to a parser
+ * @param parser parser structure
+ * @param data the pointer to the beginning of a chunk
+ * @param len the length of a chunk
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_chunk (struct ucl_parser *parser,
+ const unsigned char *data, size_t len);
+
+/**
+ * Load new chunk to a parser with the specified priority
+ * @param parser parser structure
+ * @param data the pointer to the beginning of a chunk
+ * @param len the length of a chunk
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_chunk_priority (struct ucl_parser *parser,
+ const unsigned char *data, size_t len, unsigned priority);
+
+/**
+ * Insert new chunk to a parser (must have previously processed data with an existing top object)
+ * @param parser parser structure
+ * @param data the pointer to the beginning of a chunk
+ * @param len the length of a chunk
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_insert_chunk (struct ucl_parser *parser,
+ const unsigned char *data, size_t len);
+
+/**
+ * Full version of ucl_add_chunk with priority and duplicate strategy
+ * @param parser parser structure
+ * @param data the pointer to the beginning of a chunk
+ * @param len the length of a chunk
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @param strat duplicates merging strategy
+ * @param parse_type input format
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_chunk_full (struct ucl_parser *parser,
+ const unsigned char *data, size_t len, unsigned priority,
+ enum ucl_duplicate_strategy strat, enum ucl_parse_type parse_type);
+
+/**
+ * Load ucl object from a string
+ * @param parser parser structure
+ * @param data the pointer to the string
+ * @param len the length of the string, if `len` is 0 then `data` must be zero-terminated string
+ * @return true if string has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_string (struct ucl_parser *parser,
+ const char *data, size_t len);
+
+/**
+ * Load ucl object from a string
+ * @param parser parser structure
+ * @param data the pointer to the string
+ * @param len the length of the string, if `len` is 0 then `data` must be zero-terminated string
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @return true if string has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_string_priority (struct ucl_parser *parser,
+ const char *data, size_t len, unsigned priority);
+
+/**
+ * Load and add data from a file
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param err if *err is NULL it is set to parser error
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_file (struct ucl_parser *parser,
+ const char *filename);
+
+/**
+ * Load and add data from a file
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param err if *err is NULL it is set to parser error
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_file_priority (struct ucl_parser *parser,
+ const char *filename, unsigned priority);
+
+/**
+ * Load and add data from a file
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @param strat Merge strategy to use while parsing this file
+ * @param parse_type Parser type to use while parsing this file
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_file_full (struct ucl_parser *parser, const char *filename,
+ unsigned priority, enum ucl_duplicate_strategy strat,
+ enum ucl_parse_type parse_type);
+
+/**
+ * Load and add data from a file descriptor
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param err if *err is NULL it is set to parser error
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_fd (struct ucl_parser *parser,
+ int fd);
+
+/**
+ * Load and add data from a file descriptor
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param err if *err is NULL it is set to parser error
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_fd_priority (struct ucl_parser *parser,
+ int fd, unsigned priority);
+
+/**
+ * Load and add data from a file descriptor
+ * @param parser parser structure
+ * @param filename the name of file
+ * @param err if *err is NULL it is set to parser error
+ * @param priority the desired priority of a chunk (only 4 least significant bits
+ * are considered for this parameter)
+ * @param strat Merge strategy to use while parsing this file
+ * @param parse_type Parser type to use while parsing this file
+ * @return true if chunk has been added and false in case of error
+ */
+UCL_EXTERN bool ucl_parser_add_fd_full (struct ucl_parser *parser, int fd,
+ unsigned priority, enum ucl_duplicate_strategy strat,
+ enum ucl_parse_type parse_type);
+
+/**
+ * Provide a UCL_ARRAY of paths to search for include files. The object is
+ * copied so caller must unref the object.
+ * @param parser parser structure
+ * @param paths UCL_ARRAY of paths to search
+ * @return true if the path search array was replaced in the parser
+ */
+UCL_EXTERN bool ucl_set_include_path (struct ucl_parser *parser,
+ ucl_object_t *paths);
+
+/**
+ * Get a top object for a parser (refcount is increased)
+ * @param parser parser structure
+ * @param err if *err is NULL it is set to parser error
+ * @return top parser object or NULL
+ */
+UCL_EXTERN ucl_object_t* ucl_parser_get_object (struct ucl_parser *parser);
+
+/**
+ * Get the current stack object as stack accessor function for use in macro
+ * functions (refcount is increased)
+ * @param parser parser object
+ * @param depth depth of stack to retrieve (top is 0)
+ * @return current stack object or NULL
+ */
+UCL_EXTERN ucl_object_t* ucl_parser_get_current_stack_object (struct ucl_parser *parser, unsigned int depth);
+
+/**
+ * Peek at the character at the current chunk position
+ * @param parser parser structure
+ * @return current chunk position character
+ */
+UCL_EXTERN unsigned char ucl_parser_chunk_peek (struct ucl_parser *parser);
+
+/**
+ * Skip the character at the current chunk position
+ * @param parser parser structure
+ * @return success boolean
+ */
+UCL_EXTERN bool ucl_parser_chunk_skip (struct ucl_parser *parser);
+
+/**
+ * Get the error string if parsing has been failed
+ * @param parser parser object
+ * @return error description
+ */
+UCL_EXTERN const char *ucl_parser_get_error (struct ucl_parser *parser);
+
+/**
+ * Get the code of the last error
+ * @param parser parser object
+ * @return error code
+ */
+UCL_EXTERN int ucl_parser_get_error_code (struct ucl_parser *parser);
+
+/**
+ * Get the current column number within parser
+ * @param parser parser object
+ * @return current column number
+ */
+UCL_EXTERN unsigned ucl_parser_get_column (struct ucl_parser *parser);
+
+/**
+ * Get the current line number within parser
+ * @param parser parser object
+ * @return current line number
+ */
+UCL_EXTERN unsigned ucl_parser_get_linenum (struct ucl_parser *parser);
+
+/**
+ * Clear the error in the parser
+ * @param parser parser object
+ */
+UCL_EXTERN void ucl_parser_clear_error (struct ucl_parser *parser);
+
+/**
+ * Free ucl parser object
+ * @param parser parser object
+ */
+UCL_EXTERN void ucl_parser_free (struct ucl_parser *parser);
+
+/**
+ * Get constant opaque pointer to comments structure for this parser. Increase
+ * refcount to prevent this object to be destroyed on parser's destruction
+ * @param parser parser structure
+ * @return ucl comments pointer or NULL
+ */
+UCL_EXTERN const ucl_object_t * ucl_parser_get_comments (struct ucl_parser *parser);
+
+/**
+ * Utility function to find a comment object for the specified object in the input
+ * @param comments comments object
+ * @param srch search object
+ * @return string comment enclosed in ucl_object_t
+ */
+UCL_EXTERN const ucl_object_t * ucl_comments_find (const ucl_object_t *comments,
+ const ucl_object_t *srch);
+
+/**
+ * Move comment from `from` object to `to` object
+ * @param comments comments object
+ * @param what source object
+ * @param with destination object
+ * @return `true` if `from` has comment and it has been moved to `to`
+ */
+UCL_EXTERN bool ucl_comments_move (ucl_object_t *comments,
+ const ucl_object_t *from, const ucl_object_t *to);
+
+/**
+ * Adds a new comment for an object
+ * @param comments comments object
+ * @param obj object to add comment to
+ * @param comment string representation of a comment
+ */
+UCL_EXTERN void ucl_comments_add (ucl_object_t *comments,
+ const ucl_object_t *obj, const char *comment);
+
+/**
+ * Add new public key to parser for signatures check
+ * @param parser parser object
+ * @param key PEM representation of a key
+ * @param len length of the key
+ * @param err if *err is NULL it is set to parser error
+ * @return true if a key has been successfully added
+ */
+UCL_EXTERN bool ucl_parser_pubkey_add (struct ucl_parser *parser,
+ const unsigned char *key, size_t len);
+
+/**
+ * Set FILENAME and CURDIR variables in parser
+ * @param parser parser object
+ * @param filename filename to set or NULL to set FILENAME to "undef" and CURDIR to getcwd()
+ * @param need_expand perform realpath() if this variable is true and filename is not NULL
+ * @return true if variables has been set
+ */
+UCL_EXTERN bool ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename,
+ bool need_expand);
+
+/**
+ * Returns current file for the parser
+ * @param parser parser object
+ * @return current file or NULL if parsing memory
+ */
+UCL_EXTERN const char *ucl_parser_get_cur_file (struct ucl_parser *parser);
+
+/**
+ * Defines special handler for certain types of data (identified by magic)
+ */
+typedef bool (*ucl_parser_special_handler_t) (struct ucl_parser *parser,
+ const unsigned char *source, size_t source_len,
+ unsigned char **destination, size_t *dest_len,
+ void *user_data);
+
+/**
+ * Special handler flags
+ */
+enum ucl_special_handler_flags {
+ UCL_SPECIAL_HANDLER_DEFAULT = 0,
+ UCL_SPECIAL_HANDLER_PREPROCESS_ALL = (1u << 0),
+};
+
+/**
+ * Special handler structure
+ */
+struct ucl_parser_special_handler {
+ const unsigned char *magic;
+ size_t magic_len;
+ enum ucl_special_handler_flags flags;
+ ucl_parser_special_handler_t handler;
+ void (*free_function) (unsigned char *data, size_t len, void *user_data);
+ void *user_data;
+ struct ucl_parser_special_handler *next; /* Used internally */
+};
+
+/**
+ * Add special handler for a parser, handles special sequences identified by magic
+ * @param parser parser structure
+ * @param handler handler structure
+ */
+UCL_EXTERN void ucl_parser_add_special_handler (struct ucl_parser *parser,
+ struct ucl_parser_special_handler *handler);
+
+/**
+ * Handler for include traces:
+ * @param parser parser object
+ * @param parent where include is done from
+ * @param args arguments to an include
+ * @param path path of the include
+ * @param pathlen length of the path
+ * @param user_data opaque userdata
+ */
+typedef void (ucl_include_trace_func_t) (struct ucl_parser *parser,
+ const ucl_object_t *parent,
+ const ucl_object_t *args,
+ const char *path,
+ size_t pathlen,
+ void *user_data);
+
+/**
+ * Register trace function for an include handler
+ * @param parser parser object
+ * @param func function to trace includes
+ * @param user_data opaque data
+ */
+UCL_EXTERN void ucl_parser_set_include_tracer (struct ucl_parser *parser,
+ ucl_include_trace_func_t func,
+ void *user_data);
+
+/** @} */
+
+/**
+ * @defgroup emitter Emitting functions
+ * These functions are used to serialise UCL objects to some string representation.
+ *
+ * @{
+ */
+
+struct ucl_emitter_context;
+/**
+ * Structure using for emitter callbacks
+ */
+struct ucl_emitter_functions {
+ /** Append a single character */
+ int (*ucl_emitter_append_character) (unsigned char c, size_t nchars, void *ud);
+ /** Append a string of a specified length */
+ int (*ucl_emitter_append_len) (unsigned const char *str, size_t len, void *ud);
+ /** Append a 64 bit integer */
+ int (*ucl_emitter_append_int) (int64_t elt, void *ud);
+ /** Append floating point element */
+ int (*ucl_emitter_append_double) (double elt, void *ud);
+ /** Free userdata */
+ void (*ucl_emitter_free_func)(void *ud);
+ /** Opaque userdata pointer */
+ void *ud;
+};
+
+struct ucl_emitter_operations {
+ /** Write a primitive element */
+ void (*ucl_emitter_write_elt) (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool first, bool print_key);
+ /** Start ucl object */
+ void (*ucl_emitter_start_object) (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key);
+ /** End ucl object */
+ void (*ucl_emitter_end_object) (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj);
+ /** Start ucl array */
+ void (*ucl_emitter_start_array) (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key);
+ void (*ucl_emitter_end_array) (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj);
+};
+
+/**
+ * Structure that defines emitter functions
+ */
+struct ucl_emitter_context {
+ /** Name of emitter (e.g. json, compact_json) */
+ const char *name;
+ /** Unique id (e.g. UCL_EMIT_JSON for standard emitters */
+ int id;
+ /** A set of output functions */
+ const struct ucl_emitter_functions *func;
+ /** A set of output operations */
+ const struct ucl_emitter_operations *ops;
+ /** Current amount of indent tabs */
+ unsigned int indent;
+ /** Top level object */
+ const ucl_object_t *top;
+ /** Optional comments */
+ const ucl_object_t *comments;
+};
+
+/**
+ * Emit object to a string
+ * @param obj object
+ * @param emit_type if type is #UCL_EMIT_JSON then emit json, if type is
+ * #UCL_EMIT_CONFIG then emit config like object
+ * @return dump of an object (must be freed after using) or NULL in case of error
+ */
+UCL_EXTERN unsigned char *ucl_object_emit (const ucl_object_t *obj,
+ enum ucl_emitter emit_type);
+
+/**
+ * Emit object to a string that can contain `\0` inside
+ * @param obj object
+ * @param emit_type if type is #UCL_EMIT_JSON then emit json, if type is
+ * #UCL_EMIT_CONFIG then emit config like object
+ * @param len the resulting length
+ * @return dump of an object (must be freed after using) or NULL in case of error
+ */
+UCL_EXTERN unsigned char *ucl_object_emit_len (const ucl_object_t *obj,
+ enum ucl_emitter emit_type, size_t *len);
+
+/**
+ * Emit object to a string
+ * @param obj object
+ * @param emit_type if type is #UCL_EMIT_JSON then emit json, if type is
+ * #UCL_EMIT_CONFIG then emit config like object
+ * @param emitter a set of emitter functions
+ * @param comments optional comments for the parser
+ * @return dump of an object (must be freed after using) or NULL in case of error
+ */
+UCL_EXTERN bool ucl_object_emit_full (const ucl_object_t *obj,
+ enum ucl_emitter emit_type,
+ struct ucl_emitter_functions *emitter,
+ const ucl_object_t *comments);
+
+/**
+ * Start streamlined UCL object emitter
+ * @param obj top UCL object
+ * @param emit_type emit type
+ * @param emitter a set of emitter functions
+ * @return new streamlined context that should be freed by
+ * `ucl_object_emit_streamline_finish`
+ */
+UCL_EXTERN struct ucl_emitter_context* ucl_object_emit_streamline_new (
+ const ucl_object_t *obj, enum ucl_emitter emit_type,
+ struct ucl_emitter_functions *emitter);
+
+/**
+ * Start object or array container for the streamlined output
+ * @param ctx streamlined context
+ * @param obj container object
+ */
+UCL_EXTERN void ucl_object_emit_streamline_start_container (
+ struct ucl_emitter_context *ctx, const ucl_object_t *obj);
+/**
+ * Add a complete UCL object to streamlined output
+ * @param ctx streamlined context
+ * @param obj object to output
+ */
+UCL_EXTERN void ucl_object_emit_streamline_add_object (
+ struct ucl_emitter_context *ctx, const ucl_object_t *obj);
+/**
+ * End previously added container
+ * @param ctx streamlined context
+ */
+UCL_EXTERN void ucl_object_emit_streamline_end_container (
+ struct ucl_emitter_context *ctx);
+/**
+ * Terminate streamlined container finishing all containers in it
+ * @param ctx streamlined context
+ */
+UCL_EXTERN void ucl_object_emit_streamline_finish (
+ struct ucl_emitter_context *ctx);
+
+/**
+ * Returns functions to emit object to memory
+ * @param pmem target pointer (should be freed by caller)
+ * @return emitter functions structure
+ */
+UCL_EXTERN struct ucl_emitter_functions* ucl_object_emit_memory_funcs (
+ void **pmem);
+
+/**
+ * Returns functions to emit object to FILE *
+ * @param fp FILE * object
+ * @return emitter functions structure
+ */
+UCL_EXTERN struct ucl_emitter_functions* ucl_object_emit_file_funcs (
+ FILE *fp);
+/**
+ * Returns functions to emit object to a file descriptor
+ * @param fd file descriptor
+ * @return emitter functions structure
+ */
+UCL_EXTERN struct ucl_emitter_functions* ucl_object_emit_fd_funcs (
+ int fd);
+
+/**
+ * Free emitter functions
+ * @param f pointer to functions
+ */
+UCL_EXTERN void ucl_object_emit_funcs_free (struct ucl_emitter_functions *f);
+
+/** @} */
+
+/**
+ * @defgroup schema Schema functions
+ * These functions are used to validate UCL objects using json schema format
+ *
+ * @{
+ */
+
+/**
+ * Used to define UCL schema error
+ */
+enum ucl_schema_error_code {
+ UCL_SCHEMA_OK = 0, /**< no error */
+ UCL_SCHEMA_TYPE_MISMATCH, /**< type of object is incorrect */
+ UCL_SCHEMA_INVALID_SCHEMA, /**< schema is invalid */
+ UCL_SCHEMA_MISSING_PROPERTY,/**< one or more missing properties */
+ UCL_SCHEMA_CONSTRAINT, /**< constraint found */
+ UCL_SCHEMA_MISSING_DEPENDENCY, /**< missing dependency */
+ UCL_SCHEMA_EXTERNAL_REF_MISSING, /**< cannot fetch external ref */
+ UCL_SCHEMA_EXTERNAL_REF_INVALID, /**< invalid external ref */
+ UCL_SCHEMA_INTERNAL_ERROR, /**< something bad happened */
+ UCL_SCHEMA_UNKNOWN /**< generic error */
+};
+
+/**
+ * Generic ucl schema error
+ */
+struct ucl_schema_error {
+ enum ucl_schema_error_code code; /**< error code */
+ char msg[128]; /**< error message */
+ const ucl_object_t *obj; /**< object where error occurred */
+};
+
+/**
+ * Validate object `obj` using schema object `schema`.
+ * @param schema schema object
+ * @param obj object to validate
+ * @param err error pointer, if this parameter is not NULL and error has been
+ * occurred, then `err` is filled with the exact error definition.
+ * @return true if `obj` is valid using `schema`
+ */
+UCL_EXTERN bool ucl_object_validate (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err);
+
+/**
+ * Validate object `obj` using schema object `schema` and root schema at `root`.
+ * @param schema schema object
+ * @param obj object to validate
+ * @param root root schema object
+ * @param err error pointer, if this parameter is not NULL and error has been
+ * occurred, then `err` is filled with the exact error definition.
+ * @return true if `obj` is valid using `schema`
+ */
+UCL_EXTERN bool ucl_object_validate_root (const ucl_object_t *schema,
+ const ucl_object_t *obj,
+ const ucl_object_t *root,
+ struct ucl_schema_error *err);
+
+/**
+ * Validate object `obj` using schema object `schema` and root schema at `root`
+ * using some external references provided.
+ * @param schema schema object
+ * @param obj object to validate
+ * @param root root schema object
+ * @param ext_refs external references (might be modified during validation)
+ * @param err error pointer, if this parameter is not NULL and error has been
+ * occurred, then `err` is filled with the exact error definition.
+ * @return true if `obj` is valid using `schema`
+ */
+UCL_EXTERN bool ucl_object_validate_root_ext (const ucl_object_t *schema,
+ const ucl_object_t *obj,
+ const ucl_object_t *root,
+ ucl_object_t *ext_refs,
+ struct ucl_schema_error *err);
+
+/** @} */
+
+#ifdef __cplusplus
+}
+#endif
+/*
+ * XXX: Poorly named API functions, need to replace them with the appropriate
+ * named function. All API functions *must* use naming ucl_object_*. Usage of
+ * ucl_obj* should be avoided.
+ */
+#define ucl_obj_todouble_safe ucl_object_todouble_safe
+#define ucl_obj_todouble ucl_object_todouble
+#define ucl_obj_tostring ucl_object_tostring
+#define ucl_obj_tostring_safe ucl_object_tostring_safe
+#define ucl_obj_tolstring ucl_object_tolstring
+#define ucl_obj_tolstring_safe ucl_object_tolstring_safe
+#define ucl_obj_toint ucl_object_toint
+#define ucl_obj_toint_safe ucl_object_toint_safe
+#define ucl_obj_toboolean ucl_object_toboolean
+#define ucl_obj_toboolean_safe ucl_object_toboolean_safe
+#define ucl_obj_get_key ucl_object_find_key
+#define ucl_obj_get_keyl ucl_object_find_keyl
+#define ucl_obj_unref ucl_object_unref
+#define ucl_obj_ref ucl_object_ref
+#define ucl_obj_free ucl_object_free
+
+#endif /* UCL_H_ */
diff --git a/contrib/libucl/ucl_chartable.h b/contrib/libucl/ucl_chartable.h
new file mode 100644
index 0000000..7571a1d
--- /dev/null
+++ b/contrib/libucl/ucl_chartable.h
@@ -0,0 +1,268 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UCL_CHARTABLE_H_
+#define UCL_CHARTABLE_H_
+
+#include "ucl_internal.h"
+
+static const unsigned int ucl_chartable[256] = {
+UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_VALUE_END|UCL_CHARACTER_UCL_UNSAFE, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE,
+UCL_CHARACTER_WHITESPACE|UCL_CHARACTER_WHITESPACE_UNSAFE|UCL_CHARACTER_KEY_SEP|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE,
+UCL_CHARACTER_WHITESPACE_UNSAFE|UCL_CHARACTER_VALUE_END|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE,
+UCL_CHARACTER_WHITESPACE_UNSAFE,
+UCL_CHARACTER_WHITESPACE_UNSAFE|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE,
+UCL_CHARACTER_WHITESPACE_UNSAFE|UCL_CHARACTER_VALUE_END|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_WHITESPACE|UCL_CHARACTER_WHITESPACE_UNSAFE|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_KEY_SEP|UCL_CHARACTER_UCL_UNSAFE /* */,
+UCL_CHARACTER_VALUE_STR /* ! */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_ESCAPE|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE /* " */,
+UCL_CHARACTER_VALUE_END /* # */, UCL_CHARACTER_VALUE_STR /* $ */,
+UCL_CHARACTER_VALUE_STR /* % */, UCL_CHARACTER_VALUE_STR /* & */,
+UCL_CHARACTER_VALUE_STR /* ' */, UCL_CHARACTER_VALUE_STR /* ( */,
+UCL_CHARACTER_VALUE_STR /* ) */, UCL_CHARACTER_VALUE_STR /* * */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_UCL_UNSAFE /* + */,
+UCL_CHARACTER_VALUE_END /* , */,
+UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* - */,
+UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* . */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_ESCAPE /* / */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 0 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 1 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 2 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 3 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 4 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 5 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 6 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 7 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 8 */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT_START|UCL_CHARACTER_VALUE_DIGIT /* 9 */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_KEY_SEP|UCL_CHARACTER_UCL_UNSAFE /* : */,
+UCL_CHARACTER_VALUE_END /* ; */, UCL_CHARACTER_VALUE_STR /* < */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_KEY_SEP|UCL_CHARACTER_UCL_UNSAFE /* = */,
+UCL_CHARACTER_VALUE_STR /* > */, UCL_CHARACTER_VALUE_STR /* ? */,
+UCL_CHARACTER_VALUE_STR /* @ */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* A */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* B */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* C */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* D */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* E */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* F */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* G */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* H */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* I */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* J */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* K */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* L */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* M */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* N */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* O */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* P */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* Q */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* R */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* S */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* T */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* U */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* V */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* W */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* X */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* Y */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* Z */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_UCL_UNSAFE /* [ */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_ESCAPE|UCL_CHARACTER_JSON_UNSAFE|UCL_CHARACTER_UCL_UNSAFE /* \ */,
+UCL_CHARACTER_VALUE_END /* ] */, UCL_CHARACTER_VALUE_STR /* ^ */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR /* _ */,
+UCL_CHARACTER_VALUE_STR /* ` */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* a */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* b */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* c */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* d */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* e */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* f */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* g */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* h */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* i */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* j */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* k */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* l */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* m */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* n */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* o */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* p */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* q */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* r */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* s */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* t */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT|UCL_CHARACTER_ESCAPE /* u */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* v */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* w */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* x */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* y */,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_VALUE_DIGIT /* z */,
+UCL_CHARACTER_VALUE_STR|UCL_CHARACTER_UCL_UNSAFE /* { */,
+UCL_CHARACTER_VALUE_STR /* | */, UCL_CHARACTER_VALUE_END /* } */,
+UCL_CHARACTER_VALUE_STR /* ~ */, UCL_CHARACTER_DENIED,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR,
+UCL_CHARACTER_KEY_START|UCL_CHARACTER_KEY|UCL_CHARACTER_VALUE_STR
+};
+
+static inline bool
+ucl_test_character (unsigned char c, int type_flags)
+{
+ return (ucl_chartable[c] & type_flags) != 0;
+}
+
+#endif /* UCL_CHARTABLE_H_ */
diff --git a/contrib/libucl/ucl_emitter.c b/contrib/libucl/ucl_emitter.c
new file mode 100644
index 0000000..777093a
--- /dev/null
+++ b/contrib/libucl/ucl_emitter.c
@@ -0,0 +1,1189 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "ucl_chartable.h"
+#ifdef HAVE_FLOAT_H
+#include <float.h>
+#endif
+#ifdef HAVE_MATH_H
+#include <math.h>
+#endif
+
+/**
+ * @file ucl_emitter.c
+ * Serialise UCL object to various of output formats
+ */
+
+static void ucl_emitter_common_elt (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool first, bool print_key, bool compact);
+
+#define UCL_EMIT_TYPE_OPS(type) \
+ static void ucl_emit_ ## type ## _elt (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool first, bool print_key); \
+ static void ucl_emit_ ## type ## _start_obj (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool print_key); \
+ static void ucl_emit_ ## type## _start_array (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool print_key); \
+ static void ucl_emit_ ##type## _end_object (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj); \
+ static void ucl_emit_ ##type## _end_array (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj)
+
+/*
+ * JSON format operations
+ */
+UCL_EMIT_TYPE_OPS(json);
+UCL_EMIT_TYPE_OPS(json_compact);
+UCL_EMIT_TYPE_OPS(config);
+UCL_EMIT_TYPE_OPS(yaml);
+UCL_EMIT_TYPE_OPS(msgpack);
+
+#define UCL_EMIT_TYPE_CONTENT(type) { \
+ .ucl_emitter_write_elt = ucl_emit_ ## type ## _elt, \
+ .ucl_emitter_start_object = ucl_emit_ ## type ##_start_obj, \
+ .ucl_emitter_start_array = ucl_emit_ ## type ##_start_array, \
+ .ucl_emitter_end_object = ucl_emit_ ## type ##_end_object, \
+ .ucl_emitter_end_array = ucl_emit_ ## type ##_end_array \
+}
+
+static const struct ucl_emitter_operations ucl_standart_emitter_ops[] = {
+ [UCL_EMIT_JSON] = UCL_EMIT_TYPE_CONTENT(json),
+ [UCL_EMIT_JSON_COMPACT] = UCL_EMIT_TYPE_CONTENT(json_compact),
+ [UCL_EMIT_CONFIG] = UCL_EMIT_TYPE_CONTENT(config),
+ [UCL_EMIT_YAML] = UCL_EMIT_TYPE_CONTENT(yaml),
+ [UCL_EMIT_MSGPACK] = UCL_EMIT_TYPE_CONTENT(msgpack)
+};
+
+/*
+ * Utility to check whether we need a top object
+ */
+#define UCL_EMIT_IDENT_TOP_OBJ(ctx, obj) ((ctx)->top != (obj) || \
+ ((ctx)->id == UCL_EMIT_JSON_COMPACT || (ctx)->id == UCL_EMIT_JSON))
+
+
+/**
+ * Add tabulation to the output buffer
+ * @param buf target buffer
+ * @param tabs number of tabs to add
+ */
+static inline void
+ucl_add_tabs (const struct ucl_emitter_functions *func, unsigned int tabs,
+ bool compact)
+{
+ if (!compact && tabs > 0) {
+ func->ucl_emitter_append_character (' ', tabs * 4, func->ud);
+ }
+}
+
+/**
+ * Print key for the element
+ * @param ctx
+ * @param obj
+ */
+static void
+ucl_emitter_print_key (bool print_key, struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool compact)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ if (!print_key) {
+ return;
+ }
+
+ if (ctx->id == UCL_EMIT_CONFIG) {
+ if (obj->flags & UCL_OBJECT_NEED_KEY_ESCAPE) {
+ ucl_elt_string_write_json (obj->key, obj->keylen, ctx);
+ }
+ else {
+ func->ucl_emitter_append_len (obj->key, obj->keylen, func->ud);
+ }
+
+ if (obj->type != UCL_OBJECT && obj->type != UCL_ARRAY) {
+ func->ucl_emitter_append_len (" = ", 3, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_character (' ', 1, func->ud);
+ }
+ }
+ else if (ctx->id == UCL_EMIT_YAML) {
+ if (obj->keylen > 0 && (obj->flags & UCL_OBJECT_NEED_KEY_ESCAPE)) {
+ ucl_elt_string_write_json (obj->key, obj->keylen, ctx);
+ }
+ else if (obj->keylen > 0) {
+ func->ucl_emitter_append_len (obj->key, obj->keylen, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len ("null", 4, func->ud);
+ }
+
+ func->ucl_emitter_append_len (": ", 2, func->ud);
+ }
+ else {
+ if (obj->keylen > 0) {
+ ucl_elt_string_write_json (obj->key, obj->keylen, ctx);
+ }
+ else {
+ func->ucl_emitter_append_len ("null", 4, func->ud);
+ }
+
+ if (compact) {
+ func->ucl_emitter_append_character (':', 1, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len (": ", 2, func->ud);
+ }
+ }
+}
+
+static void
+ucl_emitter_finish_object (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool compact, bool is_array)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ if (ctx->id == UCL_EMIT_CONFIG && obj != ctx->top) {
+ if (obj->type != UCL_OBJECT && obj->type != UCL_ARRAY) {
+ if (!is_array) {
+ /* Objects are split by ';' */
+ func->ucl_emitter_append_len (";\n", 2, func->ud);
+ }
+ else {
+ /* Use commas for arrays */
+ func->ucl_emitter_append_len (",\n", 2, func->ud);
+ }
+ }
+ else {
+ func->ucl_emitter_append_character ('\n', 1, func->ud);
+ }
+ }
+}
+
+/**
+ * End standard ucl object
+ * @param ctx emitter context
+ * @param compact compact flag
+ */
+static void
+ucl_emitter_common_end_object (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool compact)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ if (UCL_EMIT_IDENT_TOP_OBJ(ctx, obj)) {
+ ctx->indent --;
+ if (compact || obj->len == 0) {
+ func->ucl_emitter_append_character ('}', 1, func->ud);
+ }
+ else {
+ if (ctx->id != UCL_EMIT_CONFIG) {
+ /* newline is already added for this format */
+ func->ucl_emitter_append_character ('\n', 1, func->ud);
+ }
+ ucl_add_tabs (func, ctx->indent, compact);
+ func->ucl_emitter_append_character ('}', 1, func->ud);
+ }
+ }
+
+ ucl_emitter_finish_object (ctx, obj, compact, false);
+}
+
+/**
+ * End standard ucl array
+ * @param ctx emitter context
+ * @param compact compact flag
+ */
+static void
+ucl_emitter_common_end_array (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool compact)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ ctx->indent --;
+ if (compact || obj->len == 0) {
+ func->ucl_emitter_append_character (']', 1, func->ud);
+ }
+ else {
+ if (ctx->id != UCL_EMIT_CONFIG) {
+ /* newline is already added for this format */
+ func->ucl_emitter_append_character ('\n', 1, func->ud);
+ }
+ ucl_add_tabs (func, ctx->indent, compact);
+ func->ucl_emitter_append_character (']', 1, func->ud);
+ }
+
+ ucl_emitter_finish_object (ctx, obj, compact, true);
+}
+
+/**
+ * Start emit standard UCL array
+ * @param ctx emitter context
+ * @param obj object to write
+ * @param compact compact flag
+ */
+static void
+ucl_emitter_common_start_array (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key, bool compact)
+{
+ const ucl_object_t *cur;
+ ucl_object_iter_t iter = NULL;
+ const struct ucl_emitter_functions *func = ctx->func;
+ bool first = true;
+
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+
+ if (compact || obj->len == 0) {
+ func->ucl_emitter_append_character ('[', 1, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len ("[\n", 2, func->ud);
+ }
+
+ ctx->indent ++;
+
+ if (obj->type == UCL_ARRAY) {
+ /* explicit array */
+ while ((cur = ucl_object_iterate (obj, &iter, true)) != NULL) {
+ ucl_emitter_common_elt (ctx, cur, first, false, compact);
+ first = false;
+ }
+ }
+ else {
+ /* implicit array */
+ cur = obj;
+ while (cur) {
+ ucl_emitter_common_elt (ctx, cur, first, false, compact);
+ first = false;
+ cur = cur->next;
+ }
+ }
+}
+
+/**
+ * Start emit standard UCL object
+ * @param ctx emitter context
+ * @param obj object to write
+ * @param compact compact flag
+ */
+static void
+ucl_emitter_common_start_object (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key, bool compact)
+{
+ ucl_hash_iter_t it = NULL;
+ const ucl_object_t *cur, *elt;
+ const struct ucl_emitter_functions *func = ctx->func;
+ bool first = true;
+
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ /*
+ * Print <ident_level>{
+ * <ident_level + 1><object content>
+ */
+ if (UCL_EMIT_IDENT_TOP_OBJ(ctx, obj)) {
+ if (compact || obj->len == 0) {
+ func->ucl_emitter_append_character ('{', 1, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len ("{\n", 2, func->ud);
+ }
+ ctx->indent ++;
+ }
+
+ while ((cur = ucl_hash_iterate (obj->value.ov, &it))) {
+
+ if (ctx->id == UCL_EMIT_CONFIG) {
+ LL_FOREACH (cur, elt) {
+ ucl_emitter_common_elt (ctx, elt, first, true, compact);
+ }
+ }
+ else {
+ /* Expand implicit arrays */
+ if (cur->next != NULL) {
+ if (!first) {
+ if (compact) {
+ func->ucl_emitter_append_character (',', 1, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len (",\n", 2, func->ud);
+ }
+ }
+ ucl_add_tabs (func, ctx->indent, compact);
+ ucl_emitter_common_start_array (ctx, cur, true, compact);
+ ucl_emitter_common_end_array (ctx, cur, compact);
+ }
+ else {
+ ucl_emitter_common_elt (ctx, cur, first, true, compact);
+ }
+ }
+
+ first = false;
+ }
+}
+
+/**
+ * Common choice of object emitting
+ * @param ctx emitter context
+ * @param obj object to print
+ * @param first flag to mark the first element
+ * @param print_key print key of an object
+ * @param compact compact output
+ */
+static void
+ucl_emitter_common_elt (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool first, bool print_key, bool compact)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ bool flag;
+ struct ucl_object_userdata *ud;
+ const ucl_object_t *comment = NULL, *cur_comment;
+ const char *ud_out = "";
+
+ if (ctx->id != UCL_EMIT_CONFIG && !first) {
+ if (compact) {
+ func->ucl_emitter_append_character (',', 1, func->ud);
+ }
+ else {
+ if (ctx->id == UCL_EMIT_YAML && ctx->indent == 0) {
+ func->ucl_emitter_append_len ("\n", 1, func->ud);
+ } else {
+ func->ucl_emitter_append_len (",\n", 2, func->ud);
+ }
+ }
+ }
+
+ ucl_add_tabs (func, ctx->indent, compact);
+
+ if (ctx->comments && ctx->id == UCL_EMIT_CONFIG) {
+ comment = ucl_object_lookup_len (ctx->comments, (const char *)&obj,
+ sizeof (void *));
+
+ if (comment) {
+ if (!(comment->flags & UCL_OBJECT_INHERITED)) {
+ DL_FOREACH (comment, cur_comment) {
+ func->ucl_emitter_append_len (cur_comment->value.sv,
+ cur_comment->len,
+ func->ud);
+ func->ucl_emitter_append_character ('\n', 1, func->ud);
+ ucl_add_tabs (func, ctx->indent, compact);
+ }
+
+ comment = NULL;
+ }
+ }
+ }
+
+ switch (obj->type) {
+ case UCL_INT:
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ func->ucl_emitter_append_int (ucl_object_toint (obj), func->ud);
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ func->ucl_emitter_append_double (ucl_object_todouble (obj), func->ud);
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ case UCL_BOOLEAN:
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ flag = ucl_object_toboolean (obj);
+ if (flag) {
+ func->ucl_emitter_append_len ("true", 4, func->ud);
+ }
+ else {
+ func->ucl_emitter_append_len ("false", 5, func->ud);
+ }
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ case UCL_STRING:
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ if (ctx->id == UCL_EMIT_CONFIG) {
+ if (ucl_maybe_long_string (obj)) {
+ ucl_elt_string_write_multiline (obj->value.sv, obj->len, ctx);
+ } else {
+ if (obj->flags & UCL_OBJECT_SQUOTED) {
+ ucl_elt_string_write_squoted (obj->value.sv, obj->len, ctx);
+ } else {
+ ucl_elt_string_write_json (obj->value.sv, obj->len, ctx);
+ }
+ }
+ }
+ else {
+ ucl_elt_string_write_json (obj->value.sv, obj->len, ctx);
+ }
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ case UCL_NULL:
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ func->ucl_emitter_append_len ("null", 4, func->ud);
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ case UCL_OBJECT:
+ ucl_emitter_common_start_object (ctx, obj, print_key, compact);
+ ucl_emitter_common_end_object (ctx, obj, compact);
+ break;
+ case UCL_ARRAY:
+ ucl_emitter_common_start_array (ctx, obj, print_key, compact);
+ ucl_emitter_common_end_array (ctx, obj, compact);
+ break;
+ case UCL_USERDATA:
+ ud = (struct ucl_object_userdata *)obj;
+ ucl_emitter_print_key (print_key, ctx, obj, compact);
+ if (ud->emitter) {
+ ud_out = ud->emitter (obj->value.ud);
+ if (ud_out == NULL) {
+ ud_out = "null";
+ }
+ }
+ ucl_elt_string_write_json (ud_out, strlen (ud_out), ctx);
+ ucl_emitter_finish_object (ctx, obj, compact, !print_key);
+ break;
+ }
+
+ if (comment) {
+ DL_FOREACH (comment, cur_comment) {
+ func->ucl_emitter_append_len (cur_comment->value.sv,
+ cur_comment->len,
+ func->ud);
+ func->ucl_emitter_append_character ('\n', 1, func->ud);
+
+ if (cur_comment->next) {
+ ucl_add_tabs (func, ctx->indent, compact);
+ }
+ }
+ }
+}
+
+/*
+ * Specific standard implementations of the emitter functions
+ */
+#define UCL_EMIT_TYPE_IMPL(type, compact) \
+ static void ucl_emit_ ## type ## _elt (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool first, bool print_key) { \
+ ucl_emitter_common_elt (ctx, obj, first, print_key, (compact)); \
+ } \
+ static void ucl_emit_ ## type ## _start_obj (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool print_key) { \
+ ucl_emitter_common_start_object (ctx, obj, print_key, (compact)); \
+ } \
+ static void ucl_emit_ ## type## _start_array (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj, bool print_key) { \
+ ucl_emitter_common_start_array (ctx, obj, print_key, (compact)); \
+ } \
+ static void ucl_emit_ ##type## _end_object (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj) { \
+ ucl_emitter_common_end_object (ctx, obj, (compact)); \
+ } \
+ static void ucl_emit_ ##type## _end_array (struct ucl_emitter_context *ctx, \
+ const ucl_object_t *obj) { \
+ ucl_emitter_common_end_array (ctx, obj, (compact)); \
+ }
+
+UCL_EMIT_TYPE_IMPL(json, false)
+UCL_EMIT_TYPE_IMPL(json_compact, true)
+UCL_EMIT_TYPE_IMPL(config, false)
+UCL_EMIT_TYPE_IMPL(yaml, false)
+
+static void
+ucl_emit_msgpack_elt (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool first, bool print_key)
+{
+ ucl_object_iter_t it;
+ struct ucl_object_userdata *ud;
+ const char *ud_out;
+ const ucl_object_t *cur, *celt;
+
+ switch (obj->type) {
+ case UCL_INT:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emitter_print_int_msgpack (ctx, ucl_object_toint (obj));
+ break;
+
+ case UCL_FLOAT:
+ case UCL_TIME:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emitter_print_double_msgpack (ctx, ucl_object_todouble (obj));
+ break;
+
+ case UCL_BOOLEAN:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emitter_print_bool_msgpack (ctx, ucl_object_toboolean (obj));
+ break;
+
+ case UCL_STRING:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+
+ if (obj->flags & UCL_OBJECT_BINARY) {
+ ucl_emitter_print_binary_string_msgpack (ctx, obj->value.sv,
+ obj->len);
+ }
+ else {
+ ucl_emitter_print_string_msgpack (ctx, obj->value.sv, obj->len);
+ }
+ break;
+
+ case UCL_NULL:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emitter_print_null_msgpack (ctx);
+ break;
+
+ case UCL_OBJECT:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emit_msgpack_start_obj (ctx, obj, print_key);
+ it = NULL;
+
+ while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) {
+ LL_FOREACH (cur, celt) {
+ ucl_emit_msgpack_elt (ctx, celt, false, true);
+ /* XXX:
+ * in msgpack the length of objects is encoded within a single elt
+ * so in case of multi-value keys we are using merely the first
+ * element ignoring others
+ */
+ break;
+ }
+ }
+
+ break;
+
+ case UCL_ARRAY:
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+ ucl_emit_msgpack_start_array (ctx, obj, print_key);
+ it = NULL;
+
+ while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) {
+ ucl_emit_msgpack_elt (ctx, cur, false, false);
+ }
+
+ break;
+
+ case UCL_USERDATA:
+ ud = (struct ucl_object_userdata *)obj;
+ ucl_emitter_print_key_msgpack (print_key, ctx, obj);
+
+ if (ud->emitter) {
+ ud_out = ud->emitter (obj->value.ud);
+ if (ud_out == NULL) {
+ ud_out = "null";
+ }
+ }
+ ucl_emitter_print_string_msgpack (ctx, obj->value.sv, obj->len);
+ break;
+ }
+}
+
+static void
+ucl_emit_msgpack_start_obj (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key)
+{
+ ucl_emitter_print_object_msgpack (ctx, obj->len);
+}
+
+static void
+ucl_emit_msgpack_start_array (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj, bool print_key)
+{
+ ucl_emitter_print_array_msgpack (ctx, obj->len);
+}
+
+static void
+ucl_emit_msgpack_end_object (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj)
+{
+
+}
+
+static void
+ucl_emit_msgpack_end_array (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj)
+{
+
+}
+
+unsigned char *
+ucl_object_emit (const ucl_object_t *obj, enum ucl_emitter emit_type)
+{
+ return ucl_object_emit_len (obj, emit_type, NULL);
+}
+
+unsigned char *
+ucl_object_emit_len (const ucl_object_t *obj, enum ucl_emitter emit_type,
+ size_t *outlen)
+{
+ unsigned char *res = NULL;
+ struct ucl_emitter_functions *func;
+ UT_string *s;
+
+ if (obj == NULL) {
+ return NULL;
+ }
+
+ func = ucl_object_emit_memory_funcs ((void **)&res);
+
+ if (func != NULL) {
+ s = func->ud;
+ ucl_object_emit_full (obj, emit_type, func, NULL);
+
+ if (outlen != NULL) {
+ *outlen = s->i;
+ }
+
+ ucl_object_emit_funcs_free (func);
+ }
+
+ return res;
+}
+
+bool
+ucl_object_emit_full (const ucl_object_t *obj, enum ucl_emitter emit_type,
+ struct ucl_emitter_functions *emitter,
+ const ucl_object_t *comments)
+{
+ const struct ucl_emitter_context *ctx;
+ struct ucl_emitter_context my_ctx;
+ bool res = false;
+
+ ctx = ucl_emit_get_standard_context (emit_type);
+ if (ctx != NULL) {
+ memcpy (&my_ctx, ctx, sizeof (my_ctx));
+ my_ctx.func = emitter;
+ my_ctx.indent = 0;
+ my_ctx.top = obj;
+ my_ctx.comments = comments;
+
+ my_ctx.ops->ucl_emitter_write_elt (&my_ctx, obj, true, false);
+ res = true;
+ }
+
+ return res;
+}
+
+static const struct ucl_emitter_context ucl_standard_emitters[] = {
+ [UCL_EMIT_JSON] = {
+ .name = "json",
+ .id = UCL_EMIT_JSON,
+ .func = NULL,
+ .ops = &ucl_standart_emitter_ops[UCL_EMIT_JSON]
+ },
+ [UCL_EMIT_JSON_COMPACT] = {
+ .name = "json_compact",
+ .id = UCL_EMIT_JSON_COMPACT,
+ .func = NULL,
+ .ops = &ucl_standart_emitter_ops[UCL_EMIT_JSON_COMPACT]
+ },
+ [UCL_EMIT_CONFIG] = {
+ .name = "config",
+ .id = UCL_EMIT_CONFIG,
+ .func = NULL,
+ .ops = &ucl_standart_emitter_ops[UCL_EMIT_CONFIG]
+ },
+ [UCL_EMIT_YAML] = {
+ .name = "yaml",
+ .id = UCL_EMIT_YAML,
+ .func = NULL,
+ .ops = &ucl_standart_emitter_ops[UCL_EMIT_YAML]
+ },
+ [UCL_EMIT_MSGPACK] = {
+ .name = "msgpack",
+ .id = UCL_EMIT_MSGPACK,
+ .func = NULL,
+ .ops = &ucl_standart_emitter_ops[UCL_EMIT_MSGPACK]
+ }
+};
+
+/**
+ * Get standard emitter context for a specified emit_type
+ * @param emit_type type of emitter
+ * @return context or NULL if input is invalid
+ */
+const struct ucl_emitter_context *
+ucl_emit_get_standard_context (enum ucl_emitter emit_type)
+{
+ if (emit_type >= UCL_EMIT_JSON && emit_type < UCL_EMIT_MAX) {
+ return &ucl_standard_emitters[emit_type];
+ }
+
+ return NULL;
+}
+
+/**
+ * Serialise string
+ * @param str string to emit
+ * @param buf target buffer
+ */
+void
+ucl_elt_string_write_json (const char *str, size_t size,
+ struct ucl_emitter_context *ctx)
+{
+ const char *p = str, *c = str;
+ size_t len = 0;
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ func->ucl_emitter_append_character ('"', 1, func->ud);
+
+ while (size) {
+ if (ucl_test_character (*p, (UCL_CHARACTER_JSON_UNSAFE|
+ UCL_CHARACTER_DENIED|
+ UCL_CHARACTER_WHITESPACE_UNSAFE))) {
+ if (len > 0) {
+ func->ucl_emitter_append_len (c, len, func->ud);
+ }
+ switch (*p) {
+ case '\0':
+ func->ucl_emitter_append_len ("\\u0000", 6, func->ud);
+ break;
+ case '\n':
+ func->ucl_emitter_append_len ("\\n", 2, func->ud);
+ break;
+ case '\r':
+ func->ucl_emitter_append_len ("\\r", 2, func->ud);
+ break;
+ case '\b':
+ func->ucl_emitter_append_len ("\\b", 2, func->ud);
+ break;
+ case '\t':
+ func->ucl_emitter_append_len ("\\t", 2, func->ud);
+ break;
+ case '\f':
+ func->ucl_emitter_append_len ("\\f", 2, func->ud);
+ break;
+ case '\v':
+ func->ucl_emitter_append_len ("\\u000B", 6, func->ud);
+ break;
+ case '\\':
+ func->ucl_emitter_append_len ("\\\\", 2, func->ud);
+ break;
+ case ' ':
+ func->ucl_emitter_append_character (' ', 1, func->ud);
+ break;
+ case '"':
+ func->ucl_emitter_append_len ("\\\"", 2, func->ud);
+ break;
+ default:
+ /* Emit unicode unknown character */
+ func->ucl_emitter_append_len ("\\uFFFD", 6, func->ud);
+ break;
+ }
+ len = 0;
+ c = ++p;
+ }
+ else {
+ p ++;
+ len ++;
+ }
+ size --;
+ }
+
+ if (len > 0) {
+ func->ucl_emitter_append_len (c, len, func->ud);
+ }
+
+ func->ucl_emitter_append_character ('"', 1, func->ud);
+}
+
+void
+ucl_elt_string_write_squoted (const char *str, size_t size,
+ struct ucl_emitter_context *ctx)
+{
+ const char *p = str, *c = str;
+ size_t len = 0;
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ func->ucl_emitter_append_character ('\'', 1, func->ud);
+
+ while (size) {
+ if (*p == '\'') {
+ if (len > 0) {
+ func->ucl_emitter_append_len (c, len, func->ud);
+ }
+
+ len = 0;
+ c = ++p;
+ func->ucl_emitter_append_len ("\\\'", 2, func->ud);
+ }
+ else {
+ p ++;
+ len ++;
+ }
+ size --;
+ }
+
+ if (len > 0) {
+ func->ucl_emitter_append_len (c, len, func->ud);
+ }
+
+ func->ucl_emitter_append_character ('\'', 1, func->ud);
+}
+
+void
+ucl_elt_string_write_multiline (const char *str, size_t size,
+ struct ucl_emitter_context *ctx)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+
+ func->ucl_emitter_append_len ("<<EOD\n", sizeof ("<<EOD\n") - 1, func->ud);
+ func->ucl_emitter_append_len (str, size, func->ud);
+ func->ucl_emitter_append_len ("\nEOD", sizeof ("\nEOD") - 1, func->ud);
+}
+
+/*
+ * Generic utstring output
+ */
+static int
+ucl_utstring_append_character (unsigned char c, size_t len, void *ud)
+{
+ UT_string *buf = ud;
+
+ if (len == 1) {
+ utstring_append_c (buf, c);
+ }
+ else {
+ utstring_reserve (buf, len + 1);
+ memset (&buf->d[buf->i], c, len);
+ buf->i += len;
+ buf->d[buf->i] = '\0';
+ }
+
+ return 0;
+}
+
+static int
+ucl_utstring_append_len (const unsigned char *str, size_t len, void *ud)
+{
+ UT_string *buf = ud;
+
+ utstring_append_len (buf, str, len);
+
+ return 0;
+}
+
+static int
+ucl_utstring_append_int (int64_t val, void *ud)
+{
+ UT_string *buf = ud;
+
+ utstring_printf (buf, "%jd", (intmax_t)val);
+ return 0;
+}
+
+static int
+ucl_utstring_append_double (double val, void *ud)
+{
+ UT_string *buf = ud;
+ const double delta = 0.0000001;
+
+ if (val == (double)(int)val) {
+ utstring_printf (buf, "%.1lf", val);
+ }
+ else if (fabs (val - (double)(int)val) < delta) {
+ /* Write at maximum precision */
+ utstring_printf (buf, "%.*lg", DBL_DIG, val);
+ }
+ else {
+ utstring_printf (buf, "%lf", val);
+ }
+
+ return 0;
+}
+
+/*
+ * Generic file output
+ */
+static int
+ucl_file_append_character (unsigned char c, size_t len, void *ud)
+{
+ FILE *fp = ud;
+
+ while (len --) {
+ fputc (c, fp);
+ }
+
+ return 0;
+}
+
+static int
+ucl_file_append_len (const unsigned char *str, size_t len, void *ud)
+{
+ FILE *fp = ud;
+
+ fwrite (str, len, 1, fp);
+
+ return 0;
+}
+
+static int
+ucl_file_append_int (int64_t val, void *ud)
+{
+ FILE *fp = ud;
+
+ fprintf (fp, "%jd", (intmax_t)val);
+
+ return 0;
+}
+
+static int
+ucl_file_append_double (double val, void *ud)
+{
+ FILE *fp = ud;
+ const double delta = 0.0000001;
+
+ if (val == (double)(int)val) {
+ fprintf (fp, "%.1lf", val);
+ }
+ else if (fabs (val - (double)(int)val) < delta) {
+ /* Write at maximum precision */
+ fprintf (fp, "%.*lg", DBL_DIG, val);
+ }
+ else {
+ fprintf (fp, "%lf", val);
+ }
+
+ return 0;
+}
+
+/*
+ * Generic file descriptor writing functions
+ */
+static int
+ucl_fd_append_character (unsigned char c, size_t len, void *ud)
+{
+ int fd = *(int *)ud;
+ unsigned char *buf;
+
+ if (len == 1) {
+ return write (fd, &c, 1);
+ }
+ else {
+ buf = malloc (len);
+ if (buf == NULL) {
+ /* Fallback */
+ while (len --) {
+ if (write (fd, &c, 1) == -1) {
+ return -1;
+ }
+ }
+ }
+ else {
+ memset (buf, c, len);
+ if (write (fd, buf, len) == -1) {
+ free(buf);
+ return -1;
+ }
+ free (buf);
+ }
+ }
+
+ return 0;
+}
+
+static int
+ucl_fd_append_len (const unsigned char *str, size_t len, void *ud)
+{
+ int fd = *(int *)ud;
+
+ return write (fd, str, len);
+}
+
+static int
+ucl_fd_append_int (int64_t val, void *ud)
+{
+ int fd = *(int *)ud;
+ char intbuf[64];
+
+ snprintf (intbuf, sizeof (intbuf), "%jd", (intmax_t)val);
+ return write (fd, intbuf, strlen (intbuf));
+}
+
+static int
+ucl_fd_append_double (double val, void *ud)
+{
+ int fd = *(int *)ud;
+ const double delta = 0.0000001;
+ char nbuf[64];
+
+ if (val == (double)(int)val) {
+ snprintf (nbuf, sizeof (nbuf), "%.1lf", val);
+ }
+ else if (fabs (val - (double)(int)val) < delta) {
+ /* Write at maximum precision */
+ snprintf (nbuf, sizeof (nbuf), "%.*lg", DBL_DIG, val);
+ }
+ else {
+ snprintf (nbuf, sizeof (nbuf), "%lf", val);
+ }
+
+ return write (fd, nbuf, strlen (nbuf));
+}
+
+struct ucl_emitter_functions*
+ucl_object_emit_memory_funcs (void **pmem)
+{
+ struct ucl_emitter_functions *f;
+ UT_string *s;
+
+ f = calloc (1, sizeof (*f));
+
+ if (f != NULL) {
+ f->ucl_emitter_append_character = ucl_utstring_append_character;
+ f->ucl_emitter_append_double = ucl_utstring_append_double;
+ f->ucl_emitter_append_int = ucl_utstring_append_int;
+ f->ucl_emitter_append_len = ucl_utstring_append_len;
+ f->ucl_emitter_free_func = free;
+ utstring_new (s);
+ f->ud = s;
+ *pmem = s->d;
+ s->pd = pmem;
+ }
+
+ return f;
+}
+
+struct ucl_emitter_functions*
+ucl_object_emit_file_funcs (FILE *fp)
+{
+ struct ucl_emitter_functions *f;
+
+ f = calloc (1, sizeof (*f));
+
+ if (f != NULL) {
+ f->ucl_emitter_append_character = ucl_file_append_character;
+ f->ucl_emitter_append_double = ucl_file_append_double;
+ f->ucl_emitter_append_int = ucl_file_append_int;
+ f->ucl_emitter_append_len = ucl_file_append_len;
+ f->ucl_emitter_free_func = NULL;
+ f->ud = fp;
+ }
+
+ return f;
+}
+
+struct ucl_emitter_functions*
+ucl_object_emit_fd_funcs (int fd)
+{
+ struct ucl_emitter_functions *f;
+ int *ip;
+
+ f = calloc (1, sizeof (*f));
+
+ if (f != NULL) {
+ ip = malloc (sizeof (fd));
+ if (ip == NULL) {
+ free (f);
+ return NULL;
+ }
+
+ memcpy (ip, &fd, sizeof (fd));
+ f->ucl_emitter_append_character = ucl_fd_append_character;
+ f->ucl_emitter_append_double = ucl_fd_append_double;
+ f->ucl_emitter_append_int = ucl_fd_append_int;
+ f->ucl_emitter_append_len = ucl_fd_append_len;
+ f->ucl_emitter_free_func = free;
+ f->ud = ip;
+ }
+
+ return f;
+}
+
+void
+ucl_object_emit_funcs_free (struct ucl_emitter_functions *f)
+{
+ if (f != NULL) {
+ if (f->ucl_emitter_free_func != NULL) {
+ f->ucl_emitter_free_func (f->ud);
+ }
+ free (f);
+ }
+}
+
+
+unsigned char *
+ucl_object_emit_single_json (const ucl_object_t *obj)
+{
+ UT_string *buf = NULL;
+ unsigned char *res = NULL;
+
+ if (obj == NULL) {
+ return NULL;
+ }
+
+ utstring_new (buf);
+
+ if (buf != NULL) {
+ switch (obj->type) {
+ case UCL_OBJECT:
+ ucl_utstring_append_len ("object", 6, buf);
+ break;
+ case UCL_ARRAY:
+ ucl_utstring_append_len ("array", 5, buf);
+ break;
+ case UCL_INT:
+ ucl_utstring_append_int (obj->value.iv, buf);
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ ucl_utstring_append_double (obj->value.dv, buf);
+ break;
+ case UCL_NULL:
+ ucl_utstring_append_len ("null", 4, buf);
+ break;
+ case UCL_BOOLEAN:
+ if (obj->value.iv) {
+ ucl_utstring_append_len ("true", 4, buf);
+ }
+ else {
+ ucl_utstring_append_len ("false", 5, buf);
+ }
+ break;
+ case UCL_STRING:
+ ucl_utstring_append_len (obj->value.sv, obj->len, buf);
+ break;
+ case UCL_USERDATA:
+ ucl_utstring_append_len ("userdata", 8, buf);
+ break;
+ }
+ res = utstring_body (buf);
+ free (buf);
+ }
+
+ return res;
+}
+
+#define LONG_STRING_LIMIT 80
+
+bool
+ucl_maybe_long_string (const ucl_object_t *obj)
+{
+ if (obj->len > LONG_STRING_LIMIT || (obj->flags & UCL_OBJECT_MULTILINE)) {
+ /* String is long enough, so search for newline characters in it */
+ if (memchr (obj->value.sv, '\n', obj->len) != NULL) {
+ return true;
+ }
+ }
+
+ return false;
+}
diff --git a/contrib/libucl/ucl_emitter_streamline.c b/contrib/libucl/ucl_emitter_streamline.c
new file mode 100644
index 0000000..a7178c5
--- /dev/null
+++ b/contrib/libucl/ucl_emitter_streamline.c
@@ -0,0 +1,173 @@
+/* Copyright (c) 2014, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "ucl_chartable.h"
+
+struct ucl_emitter_streamline_stack {
+ bool is_array;
+ bool empty;
+ const ucl_object_t *obj;
+ struct ucl_emitter_streamline_stack *next;
+};
+
+struct ucl_emitter_context_streamline {
+ /* Inherited from the main context */
+ /** Name of emitter (e.g. json, compact_json) */
+ const char *name;
+ /** Unique id (e.g. UCL_EMIT_JSON for standard emitters */
+ int id;
+ /** A set of output functions */
+ const struct ucl_emitter_functions *func;
+ /** A set of output operations */
+ const struct ucl_emitter_operations *ops;
+ /** Current amount of indent tabs */
+ unsigned int indent;
+ /** Top level object */
+ const ucl_object_t *top;
+ /** Optional comments */
+ const ucl_object_t *comments;
+
+ /* Streamline specific fields */
+ struct ucl_emitter_streamline_stack *containers;
+};
+
+#define TO_STREAMLINE(ctx) (struct ucl_emitter_context_streamline *)(ctx)
+
+struct ucl_emitter_context*
+ucl_object_emit_streamline_new (const ucl_object_t *obj,
+ enum ucl_emitter emit_type,
+ struct ucl_emitter_functions *emitter)
+{
+ const struct ucl_emitter_context *ctx;
+ struct ucl_emitter_context_streamline *sctx;
+
+ ctx = ucl_emit_get_standard_context (emit_type);
+ if (ctx == NULL) {
+ return NULL;
+ }
+
+ sctx = calloc (1, sizeof (*sctx));
+ if (sctx == NULL) {
+ return NULL;
+ }
+
+ memcpy (sctx, ctx, sizeof (*ctx));
+ sctx->func = emitter;
+ sctx->top = obj;
+
+ ucl_object_emit_streamline_start_container ((struct ucl_emitter_context *)sctx,
+ obj);
+
+ return (struct ucl_emitter_context *)sctx;
+}
+
+void
+ucl_object_emit_streamline_start_container (struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj)
+{
+ struct ucl_emitter_context_streamline *sctx = TO_STREAMLINE(ctx);
+ struct ucl_emitter_streamline_stack *st, *top;
+ bool print_key = false;
+
+ /* Check top object presence */
+ if (sctx->top == NULL) {
+ sctx->top = obj;
+ }
+
+ top = sctx->containers;
+ st = malloc (sizeof (*st));
+ if (st != NULL) {
+ if (top != NULL && !top->is_array) {
+ print_key = true;
+ }
+ st->empty = true;
+ st->obj = obj;
+ if (obj != NULL && obj->type == UCL_ARRAY) {
+ st->is_array = true;
+ sctx->ops->ucl_emitter_start_array (ctx, obj, print_key);
+ }
+ else {
+ st->is_array = false;
+ sctx->ops->ucl_emitter_start_object (ctx, obj, print_key);
+ }
+ LL_PREPEND (sctx->containers, st);
+ }
+}
+
+void
+ucl_object_emit_streamline_add_object (
+ struct ucl_emitter_context *ctx, const ucl_object_t *obj)
+{
+ struct ucl_emitter_context_streamline *sctx = TO_STREAMLINE(ctx);
+ bool is_array = false, is_first = false;
+
+ if (sctx->containers != NULL) {
+ if (sctx->containers->is_array) {
+ is_array = true;
+ }
+ if (sctx->containers->empty) {
+ is_first = true;
+ sctx->containers->empty = false;
+ }
+ }
+
+ sctx->ops->ucl_emitter_write_elt (ctx, obj, is_first, !is_array);
+}
+
+void
+ucl_object_emit_streamline_end_container (struct ucl_emitter_context *ctx)
+{
+ struct ucl_emitter_context_streamline *sctx = TO_STREAMLINE(ctx);
+ struct ucl_emitter_streamline_stack *st;
+
+ if (sctx->containers != NULL) {
+ st = sctx->containers;
+
+ if (st->is_array) {
+ sctx->ops->ucl_emitter_end_array (ctx, st->obj);
+ }
+ else {
+ sctx->ops->ucl_emitter_end_object (ctx, st->obj);
+ }
+ sctx->containers = st->next;
+ free (st);
+ }
+}
+
+void
+ucl_object_emit_streamline_finish (struct ucl_emitter_context *ctx)
+{
+ struct ucl_emitter_context_streamline *sctx = TO_STREAMLINE(ctx);
+
+ while (sctx->containers != NULL) {
+ ucl_object_emit_streamline_end_container (ctx);
+ }
+
+ free (sctx);
+}
diff --git a/contrib/libucl/ucl_hash.c b/contrib/libucl/ucl_hash.c
new file mode 100644
index 0000000..a26c26f
--- /dev/null
+++ b/contrib/libucl/ucl_hash.c
@@ -0,0 +1,498 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ucl_internal.h"
+#include "ucl_hash.h"
+#include "khash.h"
+#include "utlist.h"
+
+#include "cryptobox.h"
+#include "libutil/str_util.h"
+#include "ucl.h"
+
+#include <time.h>
+#include <limits.h>
+
+struct ucl_hash_elt {
+ const ucl_object_t *obj;
+ struct ucl_hash_elt *prev, *next;
+};
+
+struct ucl_hash_struct {
+ void *hash;
+ struct ucl_hash_elt *head;
+ bool caseless;
+};
+
+static uint64_t
+ucl_hash_seed (void)
+{
+ static uint64_t seed;
+ if (seed == 0) {
+#ifdef UCL_RANDOM_FUNCTION
+ seed = UCL_RANDOM_FUNCTION;
+#else
+ /* Not very random but can be useful for our purposes */
+ seed = time (NULL);
+#endif
+ }
+
+ return seed;
+}
+
+extern const guchar lc_map[256];
+
+static inline uint32_t
+ucl_hash_func (const ucl_object_t *o)
+{
+ return (uint32_t)rspamd_cryptobox_fast_hash (o->key, o->keylen, 0xb9a1ef83c4561c95ULL);
+}
+
+static inline int
+ucl_hash_equal (const ucl_object_t *k1, const ucl_object_t *k2)
+{
+ if (k1->keylen == k2->keylen) {
+ return memcmp (k1->key, k2->key, k1->keylen) == 0;
+ }
+
+ return 0;
+}
+
+KHASH_INIT (ucl_hash_node, const ucl_object_t *, struct ucl_hash_elt *, 1,
+ ucl_hash_func, ucl_hash_equal)
+
+static inline uint32_t
+ucl_hash_caseless_func (const ucl_object_t *o)
+{
+ unsigned len = o->keylen;
+ unsigned leftover = o->keylen % 4;
+ unsigned fp, i;
+ const uint8_t* s = (const uint8_t*)o->key;
+ union {
+ struct {
+ unsigned char c1, c2, c3, c4;
+ } c;
+ uint32_t pp;
+ } u;
+ uint64_t h = 0xe5ae6ab1ef9f3b54ULL;
+ rspamd_cryptobox_fast_hash_state_t hst;
+
+ fp = len - leftover;
+ rspamd_cryptobox_fast_hash_init (&hst, h);
+
+ for (i = 0; i != fp; i += 4) {
+ u.c.c1 = s[i], u.c.c2 = s[i + 1], u.c.c3 = s[i + 2], u.c.c4 = s[i + 3];
+ u.c.c1 = lc_map[u.c.c1];
+ u.c.c2 = lc_map[u.c.c2];
+ u.c.c3 = lc_map[u.c.c3];
+ u.c.c4 = lc_map[u.c.c4];
+ rspamd_cryptobox_fast_hash_update (&hst, &u, sizeof (u));
+ }
+
+ u.pp = 0;
+ switch (leftover) {
+ case 3:
+ u.c.c3 = lc_map[(unsigned char)s[i++]];
+ case 2:
+ /* fallthrough */
+ u.c.c2 = lc_map[(unsigned char)s[i++]];
+ case 1:
+ /* fallthrough */
+ u.c.c1 = lc_map[(unsigned char)s[i]];
+ rspamd_cryptobox_fast_hash_update (&hst, &u, sizeof (u));
+ break;
+ }
+
+ return (uint32_t)rspamd_cryptobox_fast_hash_final (&hst);
+}
+
+
+static inline bool
+ucl_hash_caseless_equal (const ucl_object_t *k1, const ucl_object_t *k2)
+{
+ if (k1->keylen == k2->keylen) {
+ return rspamd_lc_cmp (k1->key, k2->key, k1->keylen) == 0;
+ }
+
+ return false;
+}
+
+KHASH_INIT (ucl_hash_caseless_node, const ucl_object_t *, struct ucl_hash_elt *, 1,
+ ucl_hash_caseless_func, ucl_hash_caseless_equal)
+
+ucl_hash_t*
+ucl_hash_create (bool ignore_case)
+{
+ ucl_hash_t *new;
+
+ new = UCL_ALLOC (sizeof (ucl_hash_t));
+ if (new != NULL) {
+ void *h;
+ new->head = NULL;
+ new->caseless = ignore_case;
+ if (ignore_case) {
+ h = (void *)kh_init (ucl_hash_caseless_node);
+ }
+ else {
+ h = (void *)kh_init (ucl_hash_node);
+ }
+ if (h == NULL) {
+ UCL_FREE (sizeof (ucl_hash_t), new);
+ return NULL;
+ }
+ new->hash = h;
+ }
+ return new;
+}
+
+void ucl_hash_destroy (ucl_hash_t* hashlin, ucl_hash_free_func func)
+{
+
+ if (hashlin == NULL) {
+ return;
+ }
+
+ if (func != NULL) {
+ /* Iterate over the hash first */
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ khiter_t k;
+ const ucl_object_t *cur, *tmp;
+
+ for (k = kh_begin (h); k != kh_end (h); ++k) {
+ if (kh_exist (h, k)) {
+ cur = (kh_value (h, k))->obj;
+ while (cur != NULL) {
+ tmp = cur->next;
+ func (__DECONST (ucl_object_t *, cur));
+ cur = tmp;
+ }
+ }
+ }
+ }
+
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(ucl_hash_caseless_node) *)
+ hashlin->hash;
+ kh_destroy (ucl_hash_caseless_node, h);
+ }
+ else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ kh_destroy (ucl_hash_node, h);
+ }
+
+ struct ucl_hash_elt *cur, *tmp;
+
+ DL_FOREACH_SAFE(hashlin->head, cur, tmp) {
+ UCL_FREE(sizeof(*cur), cur);
+ }
+
+ UCL_FREE (sizeof (*hashlin), hashlin);
+}
+
+bool
+ucl_hash_insert (ucl_hash_t* hashlin, const ucl_object_t *obj,
+ const char *key, unsigned keylen)
+{
+ khiter_t k;
+ int ret;
+ struct ucl_hash_elt **pelt, *elt;
+
+ if (hashlin == NULL) {
+ return false;
+ }
+
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(ucl_hash_caseless_node) *)
+ hashlin->hash;
+ k = kh_put (ucl_hash_caseless_node, h, obj, &ret);
+ if (ret > 0) {
+ elt = UCL_ALLOC(sizeof(*elt));
+ pelt = &kh_value (h, k);
+ *pelt = elt;
+ DL_APPEND(hashlin->head, elt);
+ elt->obj = obj;
+ }
+ else if (ret < 0) {
+ goto e0;
+ }
+ }
+ else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ k = kh_put (ucl_hash_node, h, obj, &ret);
+ if (ret > 0) {
+ elt = UCL_ALLOC(sizeof(*elt));
+ pelt = &kh_value (h, k);
+ *pelt = elt;
+ DL_APPEND(hashlin->head, elt);
+ elt->obj = obj;
+ } else if (ret < 0) {
+ goto e0;
+ }
+ }
+ return true;
+ e0:
+ return false;
+}
+
+void ucl_hash_replace (ucl_hash_t* hashlin, const ucl_object_t *old,
+ const ucl_object_t *new)
+{
+ khiter_t k;
+ int ret;
+ struct ucl_hash_elt *elt, *nelt;
+
+ if (hashlin == NULL) {
+ return;
+ }
+
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(ucl_hash_caseless_node) *)
+ hashlin->hash;
+ k = kh_put (ucl_hash_caseless_node, h, old, &ret);
+ if (ret == 0) {
+ elt = kh_value(h, k);
+ kh_del (ucl_hash_caseless_node, h, k);
+ k = kh_put (ucl_hash_caseless_node, h, new, &ret);
+ nelt = UCL_ALLOC(sizeof(*nelt));
+ nelt->obj = new;
+ kh_value(h, k) = nelt;
+ DL_REPLACE_ELEM(hashlin->head, elt, nelt);
+ UCL_FREE(sizeof(*elt), elt);
+ }
+ }
+ else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ k = kh_put (ucl_hash_node, h, old, &ret);
+ if (ret == 0) {
+ elt = kh_value (h, k);
+ kh_del (ucl_hash_node, h, k);
+ k = kh_put (ucl_hash_node, h, new, &ret);
+ nelt = UCL_ALLOC(sizeof(*nelt));
+ nelt->obj = new;
+ kh_value(h, k) = nelt;
+ DL_REPLACE_ELEM(hashlin->head, elt, nelt);
+ UCL_FREE(sizeof(*elt), elt);
+ }
+ }
+}
+
+struct ucl_hash_real_iter {
+ const struct ucl_hash_elt *cur;
+};
+
+#define UHI_SETERR(ep, ern) {if (ep != NULL) *ep = (ern);}
+
+const void*
+ucl_hash_iterate2 (ucl_hash_t *hashlin, ucl_hash_iter_t *iter, int *ep)
+{
+ struct ucl_hash_real_iter *it = (struct ucl_hash_real_iter *)(*iter);
+ const ucl_object_t *ret = NULL;
+
+ if (hashlin == NULL) {
+ UHI_SETERR(ep, EINVAL);
+ return NULL;
+ }
+
+ if (it == NULL) {
+ it = UCL_ALLOC (sizeof (*it));
+
+ if (it == NULL) {
+ UHI_SETERR(ep, ENOMEM);
+ return NULL;
+ }
+
+ it->cur = hashlin->head;
+ }
+
+ UHI_SETERR(ep, 0);
+ if (it->cur) {
+ ret = it->cur->obj;
+ it->cur = it->cur->next;
+ }
+ else {
+ UCL_FREE (sizeof (*it), it);
+ *iter = NULL;
+ return NULL;
+ }
+
+ *iter = it;
+
+ return ret;
+}
+
+bool
+ucl_hash_iter_has_next (ucl_hash_t *hashlin, ucl_hash_iter_t iter)
+{
+ struct ucl_hash_real_iter *it = (struct ucl_hash_real_iter *)(iter);
+
+ return it->cur != NULL;
+}
+
+
+const ucl_object_t*
+ucl_hash_search (ucl_hash_t* hashlin, const char *key, unsigned keylen)
+{
+ khiter_t k;
+ const ucl_object_t *ret = NULL;
+ ucl_object_t search;
+ struct ucl_hash_elt *elt;
+
+ search.key = key;
+ search.keylen = keylen;
+
+ if (hashlin == NULL) {
+ return NULL;
+ }
+
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(ucl_hash_caseless_node) *)
+ hashlin->hash;
+
+ k = kh_get (ucl_hash_caseless_node, h, &search);
+ if (k != kh_end (h)) {
+ elt = kh_value (h, k);
+ ret = elt->obj;
+ }
+ }
+ else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ k = kh_get (ucl_hash_node, h, &search);
+ if (k != kh_end (h)) {
+ elt = kh_value (h, k);
+ ret = elt->obj;
+ }
+ }
+
+ return ret;
+}
+
+void
+ucl_hash_delete (ucl_hash_t* hashlin, const ucl_object_t *obj)
+{
+ khiter_t k;
+ struct ucl_hash_elt *elt;
+
+ if (hashlin == NULL) {
+ return;
+ }
+
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(ucl_hash_caseless_node) *)
+ hashlin->hash;
+
+ k = kh_get (ucl_hash_caseless_node, h, obj);
+ if (k != kh_end (h)) {
+ elt = kh_value (h, k);
+ DL_DELETE(hashlin->head, elt);
+ kh_del (ucl_hash_caseless_node, h, k);
+ UCL_FREE(sizeof(*elt), elt);
+ }
+ }
+ else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ k = kh_get (ucl_hash_node, h, obj);
+ if (k != kh_end (h)) {
+ elt = kh_value (h, k);
+ DL_DELETE(hashlin->head, elt);
+ kh_del (ucl_hash_node, h, k);
+ UCL_FREE(sizeof(*elt), elt);
+ }
+ }
+}
+
+bool
+ucl_hash_reserve (ucl_hash_t *hashlin, size_t sz)
+{
+ if (hashlin == NULL) {
+ return false;
+ }
+
+ if (sz > kh_size((khash_t(ucl_hash_node) *)hashlin->hash)) {
+ if (hashlin->caseless) {
+ khash_t(ucl_hash_caseless_node) *h = (khash_t(
+ ucl_hash_caseless_node) *)
+ hashlin->hash;
+ kh_resize (ucl_hash_caseless_node, h, sz * 2);
+ } else {
+ khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+ hashlin->hash;
+ kh_resize (ucl_hash_node, h, sz * 2);
+ }
+ }
+
+ return true;
+}
+
+static int
+ucl_hash_cmp_icase (const void *a, const void *b)
+{
+ const struct ucl_hash_elt *oa = (const struct ucl_hash_elt *)a,
+ *ob = (const struct ucl_hash_elt *)b;
+
+ if (oa->obj->keylen == ob->obj->keylen) {
+ return rspamd_lc_cmp (oa->obj->key, ob->obj->key, oa->obj->keylen);
+ }
+
+ return ((int)(oa->obj->keylen)) - ob->obj->keylen;
+}
+
+static int
+ucl_hash_cmp_case_sens (const void *a, const void *b)
+{
+ const struct ucl_hash_elt *oa = (const struct ucl_hash_elt *)a,
+ *ob = (const struct ucl_hash_elt *)b;
+
+ if (oa->obj->keylen == ob->obj->keylen) {
+ return memcmp (oa->obj->key, ob->obj->key, oa->obj->keylen);
+ }
+
+ return ((int)(oa->obj->keylen)) - ob->obj->keylen;
+}
+
+void
+ucl_hash_sort (ucl_hash_t *hashlin, enum ucl_object_keys_sort_flags fl)
+{
+
+ if (fl & UCL_SORT_KEYS_ICASE) {
+ DL_SORT(hashlin->head, ucl_hash_cmp_icase);
+ }
+ else {
+ DL_SORT(hashlin->head, ucl_hash_cmp_case_sens);
+ }
+
+ if (fl & UCL_SORT_KEYS_RECURSIVE) {
+ struct ucl_hash_elt *elt;
+
+ DL_FOREACH(hashlin->head, elt) {
+ if (ucl_object_type (elt->obj) == UCL_OBJECT) {
+ ucl_hash_sort (elt->obj->value.ov, fl);
+ }
+ }
+ }
+} \ No newline at end of file
diff --git a/contrib/libucl/ucl_hash.h b/contrib/libucl/ucl_hash.h
new file mode 100644
index 0000000..c2d5517
--- /dev/null
+++ b/contrib/libucl/ucl_hash.h
@@ -0,0 +1,114 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __UCL_HASH_H
+#define __UCL_HASH_H
+
+#include "ucl.h"
+
+/******************************************************************************/
+
+struct ucl_hash_node_s;
+typedef struct ucl_hash_node_s ucl_hash_node_t;
+
+typedef int (*ucl_hash_cmp_func) (const void* void_a, const void* void_b);
+typedef void (*ucl_hash_free_func) (void *ptr);
+typedef void* ucl_hash_iter_t;
+
+
+/**
+ * Linear chained hashtable.
+ */
+struct ucl_hash_struct;
+typedef struct ucl_hash_struct ucl_hash_t;
+
+
+/**
+ * Initializes the hashtable.
+ */
+ucl_hash_t* ucl_hash_create (bool ignore_case);
+
+/**
+ * Deinitializes the hashtable.
+ */
+void ucl_hash_destroy (ucl_hash_t* hashlin, ucl_hash_free_func func);
+
+/**
+ * Inserts an element in the the hashtable.
+ * @return true on success, false on failure (i.e. ENOMEM)
+ */
+bool ucl_hash_insert (ucl_hash_t* hashlin, const ucl_object_t *obj, const char *key,
+ unsigned keylen);
+
+/**
+ * Replace element in the hash
+ */
+void ucl_hash_replace (ucl_hash_t* hashlin, const ucl_object_t *old,
+ const ucl_object_t *new);
+
+/**
+ * Delete an element from the the hashtable.
+ */
+void ucl_hash_delete (ucl_hash_t* hashlin, const ucl_object_t *obj);
+
+/**
+ * Searches an element in the hashtable.
+ */
+const ucl_object_t* ucl_hash_search (ucl_hash_t* hashlin, const char *key,
+ unsigned keylen);
+
+
+/**
+ * Iterate over hash table
+ * @param hashlin hash
+ * @param iter iterator (must be NULL on first iteration)
+ * @param ep pointer record exception (such as ENOMEM), could be NULL
+ * @return the next object
+ */
+const void* ucl_hash_iterate2 (ucl_hash_t *hashlin, ucl_hash_iter_t *iter, int *ep);
+
+/**
+ * Helper macro to support older code
+ */
+#define ucl_hash_iterate(hl, ip) ucl_hash_iterate2((hl), (ip), NULL)
+
+/**
+ * Check whether an iterator has next element
+ */
+bool ucl_hash_iter_has_next (ucl_hash_t *hashlin, ucl_hash_iter_t iter);
+
+/**
+ * Reserves space in hash
+ * @return true on sucess, false on failure (e.g. ENOMEM)
+ * @param hashlin
+ */
+bool ucl_hash_reserve (ucl_hash_t *hashlin, size_t sz);
+
+/**
+ * Sorts keys in a hash
+ * @param hashlin
+ * @param fl
+ */
+void ucl_hash_sort (ucl_hash_t *hashlin, enum ucl_object_keys_sort_flags fl);
+
+#endif
diff --git a/contrib/libucl/ucl_internal.h b/contrib/libucl/ucl_internal.h
new file mode 100644
index 0000000..a7dd6ee
--- /dev/null
+++ b/contrib/libucl/ucl_internal.h
@@ -0,0 +1,666 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef UCL_INTERNAL_H_
+#define UCL_INTERNAL_H_
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#else
+/* Help embedded builds */
+#define HAVE_SYS_TYPES_H
+#define HAVE_SYS_MMAN_H
+#define HAVE_SYS_STAT_H
+#define HAVE_SYS_PARAM_H
+#define HAVE_LIMITS_H
+#define HAVE_FCNTL_H
+#define HAVE_ERRNO_H
+#define HAVE_UNISTD_H
+#define HAVE_CTYPE_H
+#define HAVE_STDIO_H
+#define HAVE_STRING_H
+#define HAVE_FLOAT_H
+#define HAVE_LIBGEN_H
+#define HAVE_MATH_H
+#define HAVE_STDBOOL_H
+#define HAVE_STDINT_H
+#define HAVE_STDARG_H
+#ifndef _WIN32
+# define HAVE_REGEX_H
+#endif
+#endif
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+
+#ifdef HAVE_SYS_MMAN_H
+# ifndef _WIN32
+# include <sys/mman.h>
+# endif
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_SYS_PARAM_H
+# ifndef _WIN32
+# include <sys/param.h>
+# endif
+#endif
+
+#ifdef HAVE_LIMITS_H
+#include <limits.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+#ifdef HAVE_UNISTD_H
+# ifndef _WIN32
+# include <unistd.h>
+# endif
+#endif
+#ifdef HAVE_CTYPE_H
+#include <ctype.h>
+#endif
+#ifdef HAVE_STDIO_H
+#include <stdio.h>
+#endif
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+#ifdef HAVE_STRINGS_H
+#include <strings.h>
+#endif
+
+#if defined(_MSC_VER)
+/* Windows hacks */
+#include <BaseTsd.h>
+typedef SSIZE_T ssize_t;
+#define strdup _strdup
+#define snprintf _snprintf
+#define vsnprintf _vsnprintf
+#define strcasecmp _stricmp
+#define strncasecmp _strnicmp
+#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
+#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
+#if _MSC_VER >= 1900
+#include <../ucrt/stdlib.h>
+#else
+#include <../include/stdlib.h>
+#endif
+#ifndef PATH_MAX
+#define PATH_MAX _MAX_PATH
+#endif
+
+/* Dirname, basename implementations */
+
+
+#endif
+
+#include "utlist.h"
+#include "utstring.h"
+#include "uthash.h"
+#include "ucl.h"
+#include "ucl_hash.h"
+
+#ifdef HAVE_OPENSSL
+#include <openssl/evp.h>
+#endif
+
+#ifndef __DECONST
+#define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
+#endif
+
+/**
+ * @file rcl_internal.h
+ * Internal structures and functions of UCL library
+ */
+
+#define UCL_MAX_RECURSION 16
+#define UCL_TRASH_KEY 0
+#define UCL_TRASH_VALUE 1
+
+enum ucl_parser_state {
+ UCL_STATE_INIT = 0,
+ UCL_STATE_OBJECT,
+ UCL_STATE_ARRAY,
+ UCL_STATE_KEY,
+ UCL_STATE_KEY_OBRACE,
+ UCL_STATE_VALUE,
+ UCL_STATE_AFTER_VALUE,
+ UCL_STATE_ARRAY_VALUE,
+ UCL_STATE_SCOMMENT,
+ UCL_STATE_MCOMMENT,
+ UCL_STATE_MACRO_NAME,
+ UCL_STATE_MACRO,
+ UCL_STATE_ERROR
+};
+
+enum ucl_character_type {
+ UCL_CHARACTER_DENIED = (1 << 0),
+ UCL_CHARACTER_KEY = (1 << 1),
+ UCL_CHARACTER_KEY_START = (1 << 2),
+ UCL_CHARACTER_WHITESPACE = (1 << 3),
+ UCL_CHARACTER_WHITESPACE_UNSAFE = (1 << 4),
+ UCL_CHARACTER_VALUE_END = (1 << 5),
+ UCL_CHARACTER_VALUE_STR = (1 << 6),
+ UCL_CHARACTER_VALUE_DIGIT = (1 << 7),
+ UCL_CHARACTER_VALUE_DIGIT_START = (1 << 8),
+ UCL_CHARACTER_ESCAPE = (1 << 9),
+ UCL_CHARACTER_KEY_SEP = (1 << 10),
+ UCL_CHARACTER_JSON_UNSAFE = (1 << 11),
+ UCL_CHARACTER_UCL_UNSAFE = (1 << 12)
+};
+
+struct ucl_macro {
+ char *name;
+ union _ucl_macro {
+ ucl_macro_handler handler;
+ ucl_context_macro_handler context_handler;
+ } h;
+ void* ud;
+ bool is_context;
+ UT_hash_handle hh;
+};
+
+enum ucl_stack_flags {
+ UCL_STACK_HAS_OBRACE = (1u << 0),
+ UCL_STACK_MAX = (1u << 1),
+};
+
+struct ucl_stack {
+ ucl_object_t *obj;
+ struct ucl_stack *next;
+ union {
+ struct {
+ uint16_t level;
+ uint16_t flags;
+ uint32_t line;
+ } params;
+ uint64_t len;
+ } e;
+ struct ucl_chunk *chunk;
+};
+
+struct ucl_parser_special_handler_chain {
+ unsigned char *begin;
+ size_t len;
+ struct ucl_parser_special_handler *special_handler;
+ struct ucl_parser_special_handler_chain *next;
+};
+
+struct ucl_chunk {
+ const unsigned char *begin;
+ const unsigned char *end;
+ const unsigned char *pos;
+ char *fname;
+ size_t remain;
+ unsigned int line;
+ unsigned int column;
+ unsigned priority;
+ enum ucl_duplicate_strategy strategy;
+ enum ucl_parse_type parse_type;
+ struct ucl_parser_special_handler_chain *special_handlers;
+ struct ucl_chunk *next;
+};
+
+#ifdef HAVE_OPENSSL
+struct ucl_pubkey {
+ EVP_PKEY *key;
+ struct ucl_pubkey *next;
+};
+#else
+struct ucl_pubkey {
+ struct ucl_pubkey *next;
+};
+#endif
+
+struct ucl_variable {
+ char *var;
+ char *value;
+ size_t var_len;
+ size_t value_len;
+ struct ucl_variable *prev, *next;
+};
+
+struct ucl_parser {
+ enum ucl_parser_state state;
+ enum ucl_parser_state prev_state;
+ unsigned int recursion;
+ int flags;
+ unsigned default_priority;
+ int err_code;
+ ucl_object_t *top_obj;
+ ucl_object_t *cur_obj;
+ ucl_object_t *trash_objs;
+ ucl_object_t *includepaths;
+ char *cur_file;
+ struct ucl_macro *macroes;
+ struct ucl_stack *stack;
+ struct ucl_chunk *chunks;
+ struct ucl_pubkey *keys;
+ struct ucl_parser_special_handler *special_handlers;
+ ucl_include_trace_func_t *include_trace_func;
+ void *include_trace_ud;
+ struct ucl_variable *variables;
+ ucl_variable_handler var_handler;
+ void *var_data;
+ ucl_object_t *comments;
+ ucl_object_t *last_comment;
+ UT_string *err;
+};
+
+struct ucl_object_userdata {
+ ucl_object_t obj;
+ ucl_userdata_dtor dtor;
+ ucl_userdata_emitter emitter;
+};
+
+/**
+ * Unescape json string inplace
+ * @param str
+ */
+size_t ucl_unescape_json_string (char *str, size_t len);
+
+
+/**
+ * Unescape single quoted string inplace
+ * @param str
+ */
+size_t ucl_unescape_squoted_string (char *str, size_t len);
+
+/**
+ * Handle include macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool ucl_include_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud);
+
+/**
+ * Handle tryinclude macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool ucl_try_include_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud);
+
+/**
+ * Handle includes macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool ucl_includes_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud);
+
+/**
+ * Handle priority macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool ucl_priority_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud);
+
+/**
+ * Handle load macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool ucl_load_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud);
+/**
+ * Handle inherit macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ctx the current context object
+ * @param ud user data
+ * @return
+ */
+bool ucl_inherit_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, const ucl_object_t *ctx, void* ud);
+
+size_t ucl_strlcpy (char *dst, const char *src, size_t siz);
+size_t ucl_strlcpy_unsafe (char *dst, const char *src, size_t siz);
+size_t ucl_strlcpy_tolower (char *dst, const char *src, size_t siz);
+
+char *ucl_strnstr (const char *s, const char *find, int len);
+char *ucl_strncasestr (const char *s, const char *find, int len);
+
+#ifdef __GNUC__
+static inline void
+ucl_create_err (UT_string **err, const char *fmt, ...)
+__attribute__ (( format( printf, 2, 3) ));
+#endif
+
+#undef UCL_FATAL_ERRORS
+
+static inline void
+ucl_create_err (UT_string **err, const char *fmt, ...)
+{
+ if (*err == NULL) {
+ utstring_new (*err);
+ va_list ap;
+ va_start (ap, fmt);
+ utstring_printf_va (*err, fmt, ap);
+ va_end (ap);
+ }
+
+#ifdef UCL_FATAL_ERRORS
+ assert (0);
+#endif
+}
+
+/**
+ * Check whether a given string contains a boolean value
+ * @param obj object to set
+ * @param start start of a string
+ * @param len length of a string
+ * @return true if a string is a boolean value
+ */
+static inline bool
+ucl_maybe_parse_boolean (ucl_object_t *obj, const unsigned char *start, size_t len)
+{
+ const char *p = (const char *)start;
+ bool ret = false, val = false;
+
+ if (len == 5) {
+ if ((p[0] == 'f' || p[0] == 'F') && strncasecmp (p, "false", 5) == 0) {
+ ret = true;
+ val = false;
+ }
+ }
+ else if (len == 4) {
+ if ((p[0] == 't' || p[0] == 'T') && strncasecmp (p, "true", 4) == 0) {
+ ret = true;
+ val = true;
+ }
+ }
+ else if (len == 3) {
+ if ((p[0] == 'y' || p[0] == 'Y') && strncasecmp (p, "yes", 3) == 0) {
+ ret = true;
+ val = true;
+ }
+ else if ((p[0] == 'o' || p[0] == 'O') && strncasecmp (p, "off", 3) == 0) {
+ ret = true;
+ val = false;
+ }
+ }
+ else if (len == 2) {
+ if ((p[0] == 'n' || p[0] == 'N') && strncasecmp (p, "no", 2) == 0) {
+ ret = true;
+ val = false;
+ }
+ else if ((p[0] == 'o' || p[0] == 'O') && strncasecmp (p, "on", 2) == 0) {
+ ret = true;
+ val = true;
+ }
+ }
+
+ if (ret && obj != NULL) {
+ obj->type = UCL_BOOLEAN;
+ obj->value.iv = val;
+ }
+
+ return ret;
+}
+
+/**
+ * Check numeric string
+ * @param obj object to set if a string is numeric
+ * @param start start of string
+ * @param end end of string
+ * @param pos position where parsing has stopped
+ * @param allow_double allow parsing of floating point values
+ * @return 0 if string is numeric and error code (EINVAL or ERANGE) in case of conversion error
+ */
+int ucl_maybe_parse_number (ucl_object_t *obj,
+ const char *start, const char *end, const char **pos,
+ bool allow_double, bool number_bytes, bool allow_time);
+
+
+static inline const ucl_object_t *
+ucl_hash_search_obj (ucl_hash_t* hashlin, ucl_object_t *obj)
+{
+ return (const ucl_object_t *)ucl_hash_search (hashlin, obj->key, obj->keylen);
+}
+
+static inline ucl_hash_t * ucl_hash_insert_object (ucl_hash_t *hashlin,
+ const ucl_object_t *obj,
+ bool ignore_case) UCL_WARN_UNUSED_RESULT;
+
+static inline ucl_hash_t *
+ucl_hash_insert_object (ucl_hash_t *hashlin,
+ const ucl_object_t *obj,
+ bool ignore_case)
+{
+ ucl_hash_t *nhp;
+
+ if (hashlin == NULL) {
+ nhp = ucl_hash_create (ignore_case);
+ if (nhp == NULL) {
+ return NULL;
+ }
+ } else {
+ nhp = hashlin;
+ }
+ if (!ucl_hash_insert (nhp, obj, obj->key, obj->keylen)) {
+ if (nhp != hashlin) {
+ ucl_hash_destroy(nhp, NULL);
+ }
+ return NULL;
+ }
+
+ return nhp;
+}
+
+/**
+ * Get standard emitter context for a specified emit_type
+ * @param emit_type type of emitter
+ * @return context or NULL if input is invalid
+ */
+const struct ucl_emitter_context *
+ucl_emit_get_standard_context (enum ucl_emitter emit_type);
+
+/**
+ * Serialize string as JSON string
+ * @param str string to emit
+ * @param buf target buffer
+ */
+void ucl_elt_string_write_json (const char *str, size_t size,
+ struct ucl_emitter_context *ctx);
+
+
+/**
+ * Serialize string as single quoted string
+ * @param str string to emit
+ * @param buf target buffer
+ */
+void
+ucl_elt_string_write_squoted (const char *str, size_t size,
+ struct ucl_emitter_context *ctx);
+
+/**
+ * Write multiline string using `EOD` as string terminator
+ * @param str
+ * @param size
+ * @param ctx
+ */
+void ucl_elt_string_write_multiline (const char *str, size_t size,
+ struct ucl_emitter_context *ctx);
+
+/**
+ * Emit a single object to string
+ * @param obj
+ * @return
+ */
+unsigned char * ucl_object_emit_single_json (const ucl_object_t *obj);
+
+/**
+ * Check whether a specified string is long and should be likely printed in
+ * multiline mode
+ * @param obj
+ * @return
+ */
+bool ucl_maybe_long_string (const ucl_object_t *obj);
+
+/**
+ * Print integer to the msgpack output
+ * @param ctx
+ * @param val
+ */
+void ucl_emitter_print_int_msgpack (struct ucl_emitter_context *ctx,
+ int64_t val);
+/**
+ * Print integer to the msgpack output
+ * @param ctx
+ * @param val
+ */
+void ucl_emitter_print_double_msgpack (struct ucl_emitter_context *ctx,
+ double val);
+/**
+ * Print double to the msgpack output
+ * @param ctx
+ * @param val
+ */
+void ucl_emitter_print_bool_msgpack (struct ucl_emitter_context *ctx,
+ bool val);
+/**
+ * Print string to the msgpack output
+ * @param ctx
+ * @param s
+ * @param len
+ */
+void ucl_emitter_print_string_msgpack (struct ucl_emitter_context *ctx,
+ const char *s, size_t len);
+
+/**
+ * Print binary string to the msgpack output
+ * @param ctx
+ * @param s
+ * @param len
+ */
+void ucl_emitter_print_binary_string_msgpack (struct ucl_emitter_context *ctx,
+ const char *s, size_t len);
+
+/**
+ * Print array preamble for msgpack
+ * @param ctx
+ * @param len
+ */
+void ucl_emitter_print_array_msgpack (struct ucl_emitter_context *ctx,
+ size_t len);
+
+/**
+ * Print object preamble for msgpack
+ * @param ctx
+ * @param len
+ */
+void ucl_emitter_print_object_msgpack (struct ucl_emitter_context *ctx,
+ size_t len);
+/**
+ * Print NULL to the msgpack output
+ * @param ctx
+ */
+void ucl_emitter_print_null_msgpack (struct ucl_emitter_context *ctx);
+/**
+ * Print object's key if needed to the msgpack output
+ * @param print_key
+ * @param ctx
+ * @param obj
+ */
+void ucl_emitter_print_key_msgpack (bool print_key,
+ struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj);
+
+/**
+ * Fetch URL into a buffer
+ * @param url url to fetch
+ * @param buf pointer to buffer (must be freed by callee)
+ * @param buflen pointer to buffer length
+ * @param err pointer to error argument
+ * @param must_exist fail if cannot find a url
+ */
+bool ucl_fetch_url (const unsigned char *url,
+ unsigned char **buf,
+ size_t *buflen,
+ UT_string **err,
+ bool must_exist);
+
+/**
+ * Fetch a file and save results to the memory buffer
+ * @param filename filename to fetch
+ * @param len length of filename
+ * @param buf target buffer
+ * @param buflen target length
+ * @return
+ */
+bool ucl_fetch_file (const unsigned char *filename,
+ unsigned char **buf,
+ size_t *buflen,
+ UT_string **err,
+ bool must_exist);
+
+/**
+ * Add new element to an object using the current merge strategy and priority
+ * @param parser
+ * @param nobj
+ * @return
+ */
+bool ucl_parser_process_object_element (struct ucl_parser *parser,
+ ucl_object_t *nobj);
+
+/**
+ * Parse msgpack chunk
+ * @param parser
+ * @return
+ */
+bool ucl_parse_msgpack (struct ucl_parser *parser);
+
+bool ucl_parse_csexp (struct ucl_parser *parser);
+
+/**
+ * Free ucl chunk
+ * @param chunk
+ */
+void ucl_chunk_free (struct ucl_chunk *chunk);
+
+#endif /* UCL_INTERNAL_H_ */
diff --git a/contrib/libucl/ucl_msgpack.c b/contrib/libucl/ucl_msgpack.c
new file mode 100644
index 0000000..1fcdcc8
--- /dev/null
+++ b/contrib/libucl/ucl_msgpack.c
@@ -0,0 +1,1615 @@
+/*
+ * Copyright (c) 2015, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ucl.h"
+#include "ucl_internal.h"
+
+#ifdef HAVE_ENDIAN_H
+#include <endian.h>
+#elif defined(HAVE_SYS_ENDIAN_H)
+#include <sys/endian.h>
+#elif defined(HAVE_MACHINE_ENDIAN_H)
+#include <machine/endian.h>
+#endif
+
+#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+ #if __BYTE_ORDER == __LITTLE_ENDIAN
+ #define __LITTLE_ENDIAN__
+ #elif __BYTE_ORDER == __BIG_ENDIAN
+ #define __BIG_ENDIAN__
+ #elif _WIN32
+ #define __LITTLE_ENDIAN__
+ #endif
+#endif
+
+#define SWAP_LE_BE16(val) ((uint16_t) ( \
+ (uint16_t) ((uint16_t) (val) >> 8) | \
+ (uint16_t) ((uint16_t) (val) << 8)))
+
+#if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined (__GNUC_MINOR__) && __GNUC_MINOR__ >= 3)
+# define SWAP_LE_BE32(val) ((uint32_t)__builtin_bswap32 ((uint32_t)(val)))
+# define SWAP_LE_BE64(val) ((uint64_t)__builtin_bswap64 ((uint64_t)(val)))
+#else
+ #define SWAP_LE_BE32(val) ((uint32_t)( \
+ (((uint32_t)(val) & (uint32_t)0x000000ffU) << 24) | \
+ (((uint32_t)(val) & (uint32_t)0x0000ff00U) << 8) | \
+ (((uint32_t)(val) & (uint32_t)0x00ff0000U) >> 8) | \
+ (((uint32_t)(val) & (uint32_t)0xff000000U) >> 24)))
+
+ #define SWAP_LE_BE64(val) ((uint64_t)( \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x00000000000000ffULL)) << 56) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x000000000000ff00ULL)) << 40) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x0000000000ff0000ULL)) << 24) | \
+ (((uint64_t)(val) & \
+ (uint64_t) (0x00000000ff000000ULL)) << 8) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x000000ff00000000ULL)) >> 8) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x0000ff0000000000ULL)) >> 24) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0x00ff000000000000ULL)) >> 40) | \
+ (((uint64_t)(val) & \
+ (uint64_t)(0xff00000000000000ULL)) >> 56)))
+#endif
+
+#ifdef __LITTLE_ENDIAN__
+#define TO_BE16 SWAP_LE_BE16
+#define TO_BE32 SWAP_LE_BE32
+#define TO_BE64 SWAP_LE_BE64
+#define FROM_BE16 SWAP_LE_BE16
+#define FROM_BE32 SWAP_LE_BE32
+#define FROM_BE64 SWAP_LE_BE64
+#else
+#define TO_BE16(val) (uint16_t)(val)
+#define TO_BE32(val) (uint32_t)(val)
+#define TO_BE64(val) (uint64_t)(val)
+#define FROM_BE16(val) (uint16_t)(val)
+#define FROM_BE32(val) (uint32_t)(val)
+#define FROM_BE64(val) (uint64_t)(val)
+#endif
+
+void
+ucl_emitter_print_int_msgpack (struct ucl_emitter_context *ctx, int64_t val)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ unsigned char buf[sizeof(uint64_t) + 1];
+ const unsigned char mask_positive = 0x7f, mask_negative = 0xe0,
+ uint8_ch = 0xcc, uint16_ch = 0xcd, uint32_ch = 0xce, uint64_ch = 0xcf,
+ int8_ch = 0xd0, int16_ch = 0xd1, int32_ch = 0xd2, int64_ch = 0xd3;
+ unsigned len;
+
+ if (val >= 0) {
+ if (val <= 0x7f) {
+ /* Fixed num 7 bits */
+ len = 1;
+ buf[0] = mask_positive & val;
+ }
+ else if (val <= UINT8_MAX) {
+ len = 2;
+ buf[0] = uint8_ch;
+ buf[1] = val & 0xff;
+ }
+ else if (val <= UINT16_MAX) {
+ uint16_t v = TO_BE16 (val);
+
+ len = 3;
+ buf[0] = uint16_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ else if (val <= UINT32_MAX) {
+ uint32_t v = TO_BE32 (val);
+
+ len = 5;
+ buf[0] = uint32_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ else {
+ uint64_t v = TO_BE64 (val);
+
+ len = 9;
+ buf[0] = uint64_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ }
+ else {
+ uint64_t uval;
+ /* Bithack abs */
+ uval = ((val ^ (val >> 63)) - (val >> 63));
+
+ if (val > -(1 << 5)) {
+ len = 1;
+ buf[0] = (mask_negative | uval) & 0xff;
+ }
+ else if (uval <= INT8_MAX) {
+ uint8_t v = (uint8_t)val;
+ len = 2;
+ buf[0] = int8_ch;
+ buf[1] = v;
+ }
+ else if (uval <= INT16_MAX) {
+ uint16_t v = TO_BE16 (val);
+
+ len = 3;
+ buf[0] = int16_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ else if (uval <= INT32_MAX) {
+ uint32_t v = TO_BE32 (val);
+
+ len = 5;
+ buf[0] = int32_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ else {
+ uint64_t v = TO_BE64 (val);
+
+ len = 9;
+ buf[0] = int64_ch;
+ memcpy (&buf[1], &v, sizeof (v));
+ }
+ }
+
+ func->ucl_emitter_append_len (buf, len, func->ud);
+}
+
+void
+ucl_emitter_print_double_msgpack (struct ucl_emitter_context *ctx, double val)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ union {
+ double d;
+ uint64_t i;
+ } u;
+ const unsigned char dbl_ch = 0xcb;
+ unsigned char buf[sizeof(double) + 1];
+
+ /* Convert to big endian */
+ u.d = val;
+ u.i = TO_BE64 (u.i);
+
+ buf[0] = dbl_ch;
+ memcpy (&buf[1], &u.d, sizeof (double));
+ func->ucl_emitter_append_len (buf, sizeof (buf), func->ud);
+}
+
+void
+ucl_emitter_print_bool_msgpack (struct ucl_emitter_context *ctx, bool val)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char true_ch = 0xc3, false_ch = 0xc2;
+
+ func->ucl_emitter_append_character (val ? true_ch : false_ch, 1, func->ud);
+}
+
+void
+ucl_emitter_print_string_msgpack (struct ucl_emitter_context *ctx,
+ const char *s, size_t len)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char fix_mask = 0xA0, l8_ch = 0xd9, l16_ch = 0xda, l32_ch = 0xdb;
+ unsigned char buf[5];
+ unsigned blen;
+
+ if (len <= 0x1F) {
+ blen = 1;
+ buf[0] = (len | fix_mask) & 0xff;
+ }
+ else if (len <= 0xff) {
+ blen = 2;
+ buf[0] = l8_ch;
+ buf[1] = len & 0xff;
+ }
+ else if (len <= 0xffff) {
+ uint16_t bl = TO_BE16 (len);
+
+ blen = 3;
+ buf[0] = l16_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+ else {
+ uint32_t bl = TO_BE32 (len);
+
+ blen = 5;
+ buf[0] = l32_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+
+ func->ucl_emitter_append_len (buf, blen, func->ud);
+ func->ucl_emitter_append_len (s, len, func->ud);
+}
+
+void
+ucl_emitter_print_binary_string_msgpack (struct ucl_emitter_context *ctx,
+ const char *s, size_t len)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char l8_ch = 0xc4, l16_ch = 0xc5, l32_ch = 0xc6;
+ unsigned char buf[5];
+ unsigned blen;
+
+ if (len <= 0xff) {
+ blen = 2;
+ buf[0] = l8_ch;
+ buf[1] = len & 0xff;
+ }
+ else if (len <= 0xffff) {
+ uint16_t bl = TO_BE16 (len);
+
+ blen = 3;
+ buf[0] = l16_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+ else {
+ uint32_t bl = TO_BE32 (len);
+
+ blen = 5;
+ buf[0] = l32_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+
+ func->ucl_emitter_append_len (buf, blen, func->ud);
+ func->ucl_emitter_append_len (s, len, func->ud);
+}
+
+void
+ucl_emitter_print_null_msgpack (struct ucl_emitter_context *ctx)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char nil = 0xc0;
+
+ func->ucl_emitter_append_character (nil, 1, func->ud);
+}
+
+void
+ucl_emitter_print_key_msgpack (bool print_key, struct ucl_emitter_context *ctx,
+ const ucl_object_t *obj)
+{
+ if (print_key) {
+ ucl_emitter_print_string_msgpack (ctx, obj->key, obj->keylen);
+ }
+}
+
+void
+ucl_emitter_print_array_msgpack (struct ucl_emitter_context *ctx, size_t len)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char fix_mask = 0x90, l16_ch = 0xdc, l32_ch = 0xdd;
+ unsigned char buf[5];
+ unsigned blen;
+
+ if (len <= 0xF) {
+ blen = 1;
+ buf[0] = (len | fix_mask) & 0xff;
+ }
+ else if (len <= 0xffff) {
+ uint16_t bl = TO_BE16 (len);
+
+ blen = 3;
+ buf[0] = l16_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+ else {
+ uint32_t bl = TO_BE32 (len);
+
+ blen = 5;
+ buf[0] = l32_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+
+ func->ucl_emitter_append_len (buf, blen, func->ud);
+}
+
+void
+ucl_emitter_print_object_msgpack (struct ucl_emitter_context *ctx, size_t len)
+{
+ const struct ucl_emitter_functions *func = ctx->func;
+ const unsigned char fix_mask = 0x80, l16_ch = 0xde, l32_ch = 0xdf;
+ unsigned char buf[5];
+ unsigned blen;
+
+ if (len <= 0xF) {
+ blen = 1;
+ buf[0] = (len | fix_mask) & 0xff;
+ }
+ else if (len <= 0xffff) {
+ uint16_t bl = TO_BE16 (len);
+
+ blen = 3;
+ buf[0] = l16_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+ else {
+ uint32_t bl = TO_BE32 (len);
+
+ blen = 5;
+ buf[0] = l32_ch;
+ memcpy (&buf[1], &bl, sizeof (bl));
+ }
+
+ func->ucl_emitter_append_len (buf, blen, func->ud);
+}
+
+
+enum ucl_msgpack_format {
+ msgpack_positive_fixint = 0,
+ msgpack_fixmap,
+ msgpack_fixarray,
+ msgpack_fixstr,
+ msgpack_nil,
+ msgpack_false,
+ msgpack_true,
+ msgpack_bin8,
+ msgpack_bin16,
+ msgpack_bin32,
+ msgpack_ext8,
+ msgpack_ext16,
+ msgpack_ext32,
+ msgpack_float32,
+ msgpack_float64,
+ msgpack_uint8,
+ msgpack_uint16,
+ msgpack_uint32,
+ msgpack_uint64,
+ msgpack_int8,
+ msgpack_int16,
+ msgpack_int32,
+ msgpack_int64,
+ msgpack_fixext1,
+ msgpack_fixext2,
+ msgpack_fixext4,
+ msgpack_fixext8,
+ msgpack_fixext16,
+ msgpack_str8,
+ msgpack_str16,
+ msgpack_str32,
+ msgpack_array16,
+ msgpack_array32,
+ msgpack_map16,
+ msgpack_map32,
+ msgpack_negative_fixint,
+ msgpack_invalid
+};
+
+typedef ssize_t (*ucl_msgpack_parse_function)(struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+
+static ssize_t ucl_msgpack_parse_map (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_array (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_string (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_int (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_float (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_bool (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_null (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+static ssize_t ucl_msgpack_parse_ignore (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain);
+
+#define MSGPACK_FLAG_FIXED (1 << 0)
+#define MSGPACK_FLAG_CONTAINER (1 << 1)
+#define MSGPACK_FLAG_TYPEVALUE (1 << 2)
+#define MSGPACK_FLAG_EXT (1 << 3)
+#define MSGPACK_FLAG_ASSOC (1 << 4)
+#define MSGPACK_FLAG_KEY (1 << 5)
+
+/*
+ * Search tree packed in array
+ */
+static struct ucl_msgpack_parser {
+ uint8_t prefix; /* Prefix byte */
+ uint8_t prefixlen; /* Length of prefix in bits */
+ uint8_t fmt; /* The desired format */
+ uint8_t len; /* Length of the object
+ (either length bytes
+ or length of value in case
+ of fixed objects */
+ uint8_t flags; /* Flags of the specified type */
+ ucl_msgpack_parse_function func; /* Parser function */
+} parsers[] = {
+ {
+ 0xa0,
+ 3,
+ msgpack_fixstr,
+ 0,
+ MSGPACK_FLAG_FIXED|MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0x0,
+ 1,
+ msgpack_positive_fixint,
+ 0,
+ MSGPACK_FLAG_FIXED|MSGPACK_FLAG_TYPEVALUE,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xe0,
+ 3,
+ msgpack_negative_fixint,
+ 0,
+ MSGPACK_FLAG_FIXED|MSGPACK_FLAG_TYPEVALUE,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0x80,
+ 4,
+ msgpack_fixmap,
+ 0,
+ MSGPACK_FLAG_FIXED|MSGPACK_FLAG_CONTAINER|MSGPACK_FLAG_ASSOC,
+ ucl_msgpack_parse_map
+ },
+ {
+ 0x90,
+ 4,
+ msgpack_fixarray,
+ 0,
+ MSGPACK_FLAG_FIXED|MSGPACK_FLAG_CONTAINER,
+ ucl_msgpack_parse_array
+ },
+ {
+ 0xd9,
+ 8,
+ msgpack_str8,
+ 1,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xc4,
+ 8,
+ msgpack_bin8,
+ 1,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xcf,
+ 8,
+ msgpack_uint64,
+ 8,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xd3,
+ 8,
+ msgpack_int64,
+ 8,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xce,
+ 8,
+ msgpack_uint32,
+ 4,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xd2,
+ 8,
+ msgpack_int32,
+ 4,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xcb,
+ 8,
+ msgpack_float64,
+ 8,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_float
+ },
+ {
+ 0xca,
+ 8,
+ msgpack_float32,
+ 4,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_float
+ },
+ {
+ 0xc2,
+ 8,
+ msgpack_false,
+ 1,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_TYPEVALUE,
+ ucl_msgpack_parse_bool
+ },
+ {
+ 0xc3,
+ 8,
+ msgpack_true,
+ 1,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_TYPEVALUE,
+ ucl_msgpack_parse_bool
+ },
+ {
+ 0xcc,
+ 8,
+ msgpack_uint8,
+ 1,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xcd,
+ 8,
+ msgpack_uint16,
+ 2,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xd0,
+ 8,
+ msgpack_int8,
+ 1,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xd1,
+ 8,
+ msgpack_int16,
+ 2,
+ MSGPACK_FLAG_FIXED,
+ ucl_msgpack_parse_int
+ },
+ {
+ 0xc0,
+ 8,
+ msgpack_nil,
+ 0,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_TYPEVALUE,
+ ucl_msgpack_parse_null
+ },
+ {
+ 0xda,
+ 8,
+ msgpack_str16,
+ 2,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xdb,
+ 8,
+ msgpack_str32,
+ 4,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xc5,
+ 8,
+ msgpack_bin16,
+ 2,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xc6,
+ 8,
+ msgpack_bin32,
+ 4,
+ MSGPACK_FLAG_KEY,
+ ucl_msgpack_parse_string
+ },
+ {
+ 0xdc,
+ 8,
+ msgpack_array16,
+ 2,
+ MSGPACK_FLAG_CONTAINER,
+ ucl_msgpack_parse_array
+ },
+ {
+ 0xdd,
+ 8,
+ msgpack_array32,
+ 4,
+ MSGPACK_FLAG_CONTAINER,
+ ucl_msgpack_parse_array
+ },
+ {
+ 0xde,
+ 8,
+ msgpack_map16,
+ 2,
+ MSGPACK_FLAG_CONTAINER|MSGPACK_FLAG_ASSOC,
+ ucl_msgpack_parse_map
+ },
+ {
+ 0xdf,
+ 8,
+ msgpack_map32,
+ 4,
+ MSGPACK_FLAG_CONTAINER|MSGPACK_FLAG_ASSOC,
+ ucl_msgpack_parse_map
+ },
+ {
+ 0xc7,
+ 8,
+ msgpack_ext8,
+ 1,
+ MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xc8,
+ 8,
+ msgpack_ext16,
+ 2,
+ MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xc9,
+ 8,
+ msgpack_ext32,
+ 4,
+ MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xd4,
+ 8,
+ msgpack_fixext1,
+ 1,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xd5,
+ 8,
+ msgpack_fixext2,
+ 2,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xd6,
+ 8,
+ msgpack_fixext4,
+ 4,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xd7,
+ 8,
+ msgpack_fixext8,
+ 8,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ },
+ {
+ 0xd8,
+ 8,
+ msgpack_fixext16,
+ 16,
+ MSGPACK_FLAG_FIXED | MSGPACK_FLAG_EXT,
+ ucl_msgpack_parse_ignore
+ }
+};
+
+#undef MSGPACK_DEBUG_PARSER
+
+static inline struct ucl_msgpack_parser *
+ucl_msgpack_get_parser_from_type (unsigned char t)
+{
+ unsigned int i, shift, mask;
+
+ for (i = 0; i < sizeof (parsers) / sizeof (parsers[0]); i ++) {
+ shift = CHAR_BIT - parsers[i].prefixlen;
+ mask = parsers[i].prefix >> shift;
+
+ if (mask == (((unsigned int)t) >> shift)) {
+ return &parsers[i];
+ }
+ }
+
+ return NULL;
+}
+
+static inline struct ucl_stack *
+ucl_msgpack_get_container (struct ucl_parser *parser,
+ struct ucl_msgpack_parser *obj_parser, uint64_t len)
+{
+ struct ucl_stack *stack;
+
+ assert (obj_parser != NULL);
+
+ if (obj_parser->flags & MSGPACK_FLAG_CONTAINER) {
+ /*
+ * Insert new container to the stack
+ */
+ if (parser->stack == NULL) {
+ parser->stack = calloc (1, sizeof (struct ucl_stack));
+
+ if (parser->stack == NULL) {
+ ucl_create_err (&parser->err, "no memory");
+ return NULL;
+ }
+
+ parser->stack->chunk = parser->chunks;
+ }
+ else {
+ stack = calloc (1, sizeof (struct ucl_stack));
+
+ if (stack == NULL) {
+ ucl_create_err (&parser->err, "no memory");
+ return NULL;
+ }
+
+ stack->chunk = parser->chunks;
+ stack->next = parser->stack;
+ parser->stack = stack;
+ }
+
+ parser->stack->e.len = len;
+
+#ifdef MSGPACK_DEBUG_PARSER
+ stack = parser->stack;
+ while (stack) {
+ fprintf(stderr, "+");
+ stack = stack->next;
+ }
+
+ fprintf(stderr, "%s -> %d\n", obj_parser->flags & MSGPACK_FLAG_ASSOC ? "object" : "array", (int)len);
+#endif
+ }
+ else {
+ /*
+ * Get the current stack top
+ */
+ if (parser->stack) {
+ return parser->stack;
+ }
+ else {
+ ucl_create_err (&parser->err, "bad top level object for msgpack");
+ return NULL;
+ }
+ }
+
+ return parser->stack;
+}
+
+static bool
+ucl_msgpack_is_container_finished (struct ucl_stack *container)
+{
+ assert (container != NULL);
+
+
+ if (container->e.len == 0) {
+ return true;
+ }
+
+ return false;
+}
+
+static bool
+ucl_msgpack_insert_object (struct ucl_parser *parser,
+ const unsigned char *key,
+ size_t keylen, ucl_object_t *obj)
+{
+ struct ucl_stack *container;
+
+ container = parser->stack;
+ assert (container != NULL);
+ assert (container->e.len > 0);
+ assert (obj != NULL);
+ assert (container->obj != NULL);
+
+ if (container->obj->type == UCL_ARRAY) {
+ ucl_array_append (container->obj, obj);
+ }
+ else if (container->obj->type == UCL_OBJECT) {
+ if (key == NULL || keylen == 0) {
+ ucl_create_err (&parser->err, "cannot insert object with no key");
+ return false;
+ }
+
+ obj->key = key;
+ obj->keylen = keylen;
+
+ if (!(parser->flags & UCL_PARSER_ZEROCOPY)) {
+ ucl_copy_key_trash (obj);
+ }
+
+ ucl_parser_process_object_element (parser, obj);
+ }
+ else {
+ ucl_create_err (&parser->err, "bad container type");
+ return false;
+ }
+
+ container->e.len--;
+
+ return true;
+}
+
+static struct ucl_stack *
+ucl_msgpack_get_next_container (struct ucl_parser *parser)
+{
+ struct ucl_stack *cur = NULL;
+ uint64_t len;
+
+ cur = parser->stack;
+
+ if (cur == NULL) {
+ return NULL;
+ }
+
+ len = cur->e.len;
+
+ if (len == 0) {
+ /* We need to switch to the previous container */
+ parser->stack = cur->next;
+ parser->cur_obj = cur->obj;
+ free (cur);
+
+#ifdef MSGPACK_DEBUG_PARSER
+ cur = parser->stack;
+ while (cur) {
+ fprintf(stderr, "-");
+ cur = cur->next;
+ }
+ fprintf(stderr, "-%s -> %d\n", parser->cur_obj->type == UCL_OBJECT ? "object" : "array", (int)parser->cur_obj->len);
+#endif
+
+ return ucl_msgpack_get_next_container (parser);
+ }
+
+ /*
+ * For UCL containers we don't know length, so we just insert the whole
+ * message pack blob into the top level container
+ */
+
+ assert (cur->obj != NULL);
+
+ return cur;
+}
+
+#define CONSUME_RET do { \
+ if (ret != -1) { \
+ p += ret; \
+ remain -= ret; \
+ obj_parser = NULL; \
+ assert (remain >= 0); \
+ } \
+ else { \
+ ucl_create_err (&parser->err, \
+ "cannot parse type %d of len %u", \
+ (int)obj_parser->fmt, \
+ (unsigned)len); \
+ return false; \
+ } \
+} while(0)
+
+#define GET_NEXT_STATE do { \
+ container = ucl_msgpack_get_next_container (parser); \
+ if (container == NULL) { \
+ ucl_create_err (&parser->err, \
+ "empty container"); \
+ return false; \
+ } \
+ next_state = container->obj->type == UCL_OBJECT ? \
+ read_assoc_key : read_array_value; \
+} while(0)
+
+static bool
+ucl_msgpack_consume (struct ucl_parser *parser)
+{
+ const unsigned char *p, *end, *key = NULL;
+ struct ucl_stack *container;
+ enum e_msgpack_parser_state {
+ read_type,
+ start_assoc,
+ start_array,
+ read_assoc_key,
+ read_assoc_value,
+ finish_assoc_value,
+ read_array_value,
+ finish_array_value,
+ error_state
+ } state = read_type, next_state = error_state;
+ struct ucl_msgpack_parser *obj_parser = NULL;
+ uint64_t len = 0;
+ ssize_t ret, remain, keylen = 0;
+#ifdef MSGPACK_DEBUG_PARSER
+ uint64_t i;
+ enum e_msgpack_parser_state hist[256];
+#endif
+
+ p = parser->chunks->begin;
+ remain = parser->chunks->remain;
+ end = p + remain;
+
+
+ while (p < end) {
+#ifdef MSGPACK_DEBUG_PARSER
+ hist[i++ % 256] = state;
+#endif
+ switch (state) {
+ case read_type:
+ obj_parser = ucl_msgpack_get_parser_from_type (*p);
+
+ if (obj_parser == NULL) {
+ ucl_create_err (&parser->err, "unknown msgpack format: %x",
+ (unsigned int)*p);
+
+ return false;
+ }
+ /* Now check length sanity */
+ if (obj_parser->flags & MSGPACK_FLAG_FIXED) {
+ if (obj_parser->len == 0) {
+ /* We have an embedded size */
+ len = *p & ~obj_parser->prefix;
+ }
+ else {
+ if (remain < obj_parser->len) {
+ ucl_create_err (&parser->err, "not enough data remain to "
+ "read object's length: %u remain, %u needed",
+ (unsigned)remain, obj_parser->len);
+
+ return false;
+ }
+
+ len = obj_parser->len;
+ }
+
+ if (!(obj_parser->flags & MSGPACK_FLAG_TYPEVALUE)) {
+ /* We must pass value as the second byte */
+ if (remain > 0) {
+ p ++;
+ remain --;
+ }
+ }
+ else {
+ /* Len is irrelevant now */
+ len = 0;
+ }
+ }
+ else {
+ /* Length is not embedded */
+ if (remain < obj_parser->len) {
+ ucl_create_err (&parser->err, "not enough data remain to "
+ "read object's length: %u remain, %u needed",
+ (unsigned)remain, obj_parser->len);
+
+ return false;
+ }
+
+ p ++;
+ remain --;
+
+ switch (obj_parser->len) {
+ case 1:
+ len = *p;
+ break;
+ case 2:
+ len = FROM_BE16 (*(uint16_t *)p);
+ break;
+ case 4:
+ len = FROM_BE32 (*(uint32_t *)p);
+ break;
+ case 8:
+ len = FROM_BE64 (*(uint64_t *)p);
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ p += obj_parser->len;
+ remain -= obj_parser->len;
+ }
+
+ if (obj_parser->flags & MSGPACK_FLAG_ASSOC) {
+ /* We have just read the new associative map */
+ state = start_assoc;
+ }
+ else if (obj_parser->flags & MSGPACK_FLAG_CONTAINER){
+ state = start_array;
+ }
+ else {
+ state = next_state;
+ }
+
+ break;
+ case start_assoc:
+ parser->cur_obj = ucl_object_new_full (UCL_OBJECT,
+ parser->chunks->priority);
+ /* Insert to the previous level container */
+ if (parser->stack && !ucl_msgpack_insert_object (parser,
+ key, keylen, parser->cur_obj)) {
+ return false;
+ }
+ /* Get new container */
+ container = ucl_msgpack_get_container (parser, obj_parser, len);
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ CONSUME_RET;
+ key = NULL;
+ keylen = 0;
+
+ if (len > 0) {
+ state = read_type;
+ next_state = read_assoc_key;
+ }
+ else {
+ /* Empty object */
+ state = finish_assoc_value;
+ }
+ break;
+
+ case start_array:
+ parser->cur_obj = ucl_object_new_full (UCL_ARRAY,
+ parser->chunks->priority);
+ /* Insert to the previous level container */
+ if (parser->stack && !ucl_msgpack_insert_object (parser,
+ key, keylen, parser->cur_obj)) {
+ return false;
+ }
+ /* Get new container */
+ container = ucl_msgpack_get_container (parser, obj_parser, len);
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ CONSUME_RET;
+
+ if (len > 0) {
+ state = read_type;
+ next_state = read_array_value;
+ }
+ else {
+ /* Empty array */
+ state = finish_array_value;
+ }
+ break;
+
+ case read_array_value:
+ /*
+ * p is now at the value start, len now contains length read and
+ * obj_parser contains the corresponding specific parser
+ */
+ container = parser->stack;
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ CONSUME_RET;
+
+
+ /* Insert value to the container and check if we have finished array */
+ if (!ucl_msgpack_insert_object (parser, NULL, 0,
+ parser->cur_obj)) {
+ return false;
+ }
+
+ if (ucl_msgpack_is_container_finished (container)) {
+ state = finish_array_value;
+ }
+ else {
+ /* Read more elements */
+ state = read_type;
+ next_state = read_array_value;
+ }
+
+ break;
+
+ case read_assoc_key:
+ /*
+ * Keys must have string type for ucl msgpack
+ */
+ if (!(obj_parser->flags & MSGPACK_FLAG_KEY)) {
+ ucl_create_err (&parser->err, "bad type for key: %u, expected "
+ "string", (unsigned)obj_parser->fmt);
+
+ return false;
+ }
+
+ key = p;
+ keylen = len;
+
+ if (keylen > remain || keylen == 0) {
+ ucl_create_err (&parser->err, "too long or empty key");
+ return false;
+ }
+
+ p += len;
+ remain -= len;
+
+ state = read_type;
+ next_state = read_assoc_value;
+ break;
+
+ case read_assoc_value:
+ /*
+ * p is now at the value start, len now contains length read and
+ * obj_parser contains the corresponding specific parser
+ */
+ container = parser->stack;
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ CONSUME_RET;
+
+ assert (key != NULL && keylen > 0);
+
+ if (!ucl_msgpack_insert_object (parser, key, keylen,
+ parser->cur_obj)) {
+ return false;
+ }
+
+ key = NULL;
+ keylen = 0;
+
+ if (ucl_msgpack_is_container_finished (container)) {
+ state = finish_assoc_value;
+ }
+ else {
+ /* Read more elements */
+ state = read_type;
+ next_state = read_assoc_key;
+ }
+ break;
+
+ case finish_array_value:
+ case finish_assoc_value:
+ GET_NEXT_STATE;
+ state = read_type;
+ break;
+
+ case error_state:
+ ucl_create_err (&parser->err, "invalid state machine state");
+
+ return false;
+ }
+ }
+
+ /* Check the finishing state */
+ switch (state) {
+ case start_array:
+ case start_assoc:
+ /* Empty container at the end */
+ if (len != 0) {
+ ucl_create_err (&parser->err, "invalid non-empty container at the end");
+
+ return false;
+ }
+
+ parser->cur_obj = ucl_object_new_full (
+ state == start_array ? UCL_ARRAY : UCL_OBJECT,
+ parser->chunks->priority);
+ /* Insert to the previous level container */
+ if (!ucl_msgpack_insert_object (parser,
+ key, keylen, parser->cur_obj)) {
+ return false;
+ }
+ /* Get new container */
+ container = ucl_msgpack_get_container (parser, obj_parser, len);
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ break;
+
+ case read_array_value:
+ case read_assoc_value:
+ if (len != 0) {
+ ucl_create_err (&parser->err, "unfinished value at the end");
+
+ return false;
+ }
+
+ container = parser->stack;
+
+ if (container == NULL) {
+ return false;
+ }
+
+ ret = obj_parser->func (parser, container, len, obj_parser->fmt,
+ p, remain);
+ CONSUME_RET;
+
+
+ /* Insert value to the container and check if we have finished array */
+ if (!ucl_msgpack_insert_object (parser, NULL, 0,
+ parser->cur_obj)) {
+ return false;
+ }
+ break;
+ case finish_array_value:
+ case finish_assoc_value:
+ case read_type:
+ /* Valid finishing state */
+ break;
+ default:
+ /* Invalid finishing state */
+ ucl_create_err (&parser->err, "invalid state machine finishing state: %d",
+ state);
+
+ return false;
+ }
+
+ /* Rewind to the top level container */
+ ucl_msgpack_get_next_container (parser);
+ assert (parser->stack == NULL);
+
+ return true;
+}
+
+bool
+ucl_parse_msgpack (struct ucl_parser *parser)
+{
+ ucl_object_t *container = NULL;
+ const unsigned char *p;
+ bool ret;
+
+ assert (parser != NULL);
+ assert (parser->chunks != NULL);
+ assert (parser->chunks->begin != NULL);
+ assert (parser->chunks->remain != 0);
+
+ p = parser->chunks->begin;
+
+ if (parser->stack) {
+ container = parser->stack->obj;
+ }
+
+ /*
+ * When we start parsing message pack chunk, we must ensure that we
+ * have either a valid container or the top object inside message pack is
+ * of container type
+ */
+ if (container == NULL) {
+ if ((*p & 0x80) != 0x80 && !(*p >= 0xdc && *p <= 0xdf)) {
+ ucl_create_err (&parser->err, "bad top level object for msgpack");
+ return false;
+ }
+ }
+
+ ret = ucl_msgpack_consume (parser);
+
+ if (ret && parser->top_obj == NULL) {
+ parser->top_obj = parser->cur_obj;
+ }
+
+ return ret;
+}
+
+static ssize_t
+ucl_msgpack_parse_map (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ container->obj = parser->cur_obj;
+
+ return 0;
+}
+
+static ssize_t
+ucl_msgpack_parse_array (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ container->obj = parser->cur_obj;
+
+ return 0;
+}
+
+static ssize_t
+ucl_msgpack_parse_string (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ ucl_object_t *obj;
+
+ if (len > remain) {
+ return -1;
+ }
+
+ obj = ucl_object_new_full (UCL_STRING, parser->chunks->priority);
+ obj->value.sv = pos;
+ obj->len = len;
+
+ if (fmt >= msgpack_bin8 && fmt <= msgpack_bin32) {
+ obj->flags |= UCL_OBJECT_BINARY;
+ }
+
+ if (!(parser->flags & UCL_PARSER_ZEROCOPY)) {
+ if (obj->flags & UCL_OBJECT_BINARY) {
+ obj->trash_stack[UCL_TRASH_VALUE] = malloc (len);
+
+ if (obj->trash_stack[UCL_TRASH_VALUE] != NULL) {
+ memcpy (obj->trash_stack[UCL_TRASH_VALUE], pos, len);
+ }
+ }
+ else {
+ ucl_copy_value_trash (obj);
+ }
+ }
+
+ parser->cur_obj = obj;
+
+ return len;
+}
+
+static ssize_t
+ucl_msgpack_parse_int (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ ucl_object_t *obj;
+ int8_t iv8;
+ int16_t iv16;
+ int32_t iv32;
+ int64_t iv64;
+ uint16_t uiv16;
+ uint32_t uiv32;
+ uint64_t uiv64;
+
+
+ if (len > remain) {
+ return -1;
+ }
+
+ obj = ucl_object_new_full (UCL_INT, parser->chunks->priority);
+
+ switch (fmt) {
+ case msgpack_positive_fixint:
+ obj->value.iv = (*pos & 0x7f);
+ len = 1;
+ break;
+ case msgpack_negative_fixint:
+ obj->value.iv = - (*pos & 0x1f);
+ len = 1;
+ break;
+ case msgpack_uint8:
+ obj->value.iv = (unsigned char)*pos;
+ len = 1;
+ break;
+ case msgpack_int8:
+ memcpy (&iv8, pos, sizeof (iv8));
+ obj->value.iv = iv8;
+ len = 1;
+ break;
+ case msgpack_int16:
+ memcpy (&iv16, pos, sizeof (iv16));
+ iv16 = FROM_BE16 (iv16);
+ obj->value.iv = iv16;
+ len = 2;
+ break;
+ case msgpack_uint16:
+ memcpy (&uiv16, pos, sizeof (uiv16));
+ uiv16 = FROM_BE16 (uiv16);
+ obj->value.iv = uiv16;
+ len = 2;
+ break;
+ case msgpack_int32:
+ memcpy (&iv32, pos, sizeof (iv32));
+ iv32 = FROM_BE32 (iv32);
+ obj->value.iv = iv32;
+ len = 4;
+ break;
+ case msgpack_uint32:
+ memcpy(&uiv32, pos, sizeof(uiv32));
+ uiv32 = FROM_BE32(uiv32);
+ obj->value.iv = uiv32;
+ len = 4;
+ break;
+ case msgpack_int64:
+ memcpy (&iv64, pos, sizeof (iv64));
+ iv64 = FROM_BE64 (iv64);
+ obj->value.iv = iv64;
+ len = 8;
+ break;
+ case msgpack_uint64:
+ memcpy(&uiv64, pos, sizeof(uiv64));
+ uiv64 = FROM_BE64(uiv64);
+ obj->value.iv = uiv64;
+ len = 8;
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ parser->cur_obj = obj;
+
+ return len;
+}
+
+static ssize_t
+ucl_msgpack_parse_float (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ ucl_object_t *obj;
+ union {
+ uint32_t i;
+ float f;
+ } d;
+ uint64_t uiv64;
+
+ if (len > remain) {
+ return -1;
+ }
+
+ obj = ucl_object_new_full (UCL_FLOAT, parser->chunks->priority);
+
+ switch (fmt) {
+ case msgpack_float32:
+ memcpy(&d.i, pos, sizeof(d.i));
+ d.i = FROM_BE32(d.i);
+ /* XXX: can be slow */
+ obj->value.dv = d.f;
+ len = 4;
+ break;
+ case msgpack_float64:
+ memcpy(&uiv64, pos, sizeof(uiv64));
+ uiv64 = FROM_BE64(uiv64);
+ obj->value.iv = uiv64;
+ len = 8;
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ parser->cur_obj = obj;
+
+ return len;
+}
+
+static ssize_t
+ucl_msgpack_parse_bool (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ ucl_object_t *obj;
+
+ if (len > remain) {
+ return -1;
+ }
+
+ obj = ucl_object_new_full (UCL_BOOLEAN, parser->chunks->priority);
+
+ switch (fmt) {
+ case msgpack_true:
+ obj->value.iv = true;
+ break;
+ case msgpack_false:
+ obj->value.iv = false;
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ parser->cur_obj = obj;
+
+ return 1;
+}
+
+static ssize_t
+ucl_msgpack_parse_null (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ ucl_object_t *obj;
+
+ if (len > remain) {
+ return -1;
+ }
+
+ obj = ucl_object_new_full (UCL_NULL, parser->chunks->priority);
+ parser->cur_obj = obj;
+
+ return 1;
+}
+
+static ssize_t
+ucl_msgpack_parse_ignore (struct ucl_parser *parser,
+ struct ucl_stack *container, size_t len, enum ucl_msgpack_format fmt,
+ const unsigned char *pos, size_t remain)
+{
+ if (len > remain) {
+ return -1;
+ }
+
+ switch (fmt) {
+ case msgpack_fixext1:
+ len = 2;
+ break;
+ case msgpack_fixext2:
+ len = 3;
+ break;
+ case msgpack_fixext4:
+ len = 5;
+ break;
+ case msgpack_fixext8:
+ len = 9;
+ break;
+ case msgpack_fixext16:
+ len = 17;
+ break;
+ case msgpack_ext8:
+ case msgpack_ext16:
+ case msgpack_ext32:
+ len = len + 1;
+ break;
+ default:
+ ucl_create_err (&parser->err, "bad type: %x", (unsigned)fmt);
+ return -1;
+ }
+
+ return len;
+}
diff --git a/contrib/libucl/ucl_parser.c b/contrib/libucl/ucl_parser.c
new file mode 100644
index 0000000..354bfe8
--- /dev/null
+++ b/contrib/libucl/ucl_parser.c
@@ -0,0 +1,3212 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <math.h>
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "ucl_chartable.h"
+
+/**
+ * @file ucl_parser.c
+ * The implementation of ucl parser
+ */
+
+struct ucl_parser_saved_state {
+ unsigned int line;
+ unsigned int column;
+ size_t remain;
+ const unsigned char *pos;
+};
+
+/**
+ * Move up to len characters
+ * @param parser
+ * @param begin
+ * @param len
+ * @return new position in chunk
+ */
+#define ucl_chunk_skipc(chunk, p) \
+do { \
+ if (p == chunk->end) { \
+ break; \
+ } \
+ if (*(p) == '\n') { \
+ (chunk)->line ++; \
+ (chunk)->column = 0; \
+ } \
+ else (chunk)->column ++; \
+ (p++); \
+ (chunk)->pos ++; \
+ (chunk)->remain --; \
+} while (0)
+
+static inline void
+ucl_set_err (struct ucl_parser *parser, int code, const char *str, UT_string **err)
+{
+ const char *fmt_string, *filename;
+ struct ucl_chunk *chunk = parser->chunks;
+
+ if (parser->cur_file) {
+ filename = parser->cur_file;
+ }
+ else {
+ filename = "<unknown>";
+ }
+
+ if (chunk->pos < chunk->end) {
+ if (isgraph (*chunk->pos)) {
+ fmt_string = "error while parsing %s: "
+ "line: %d, column: %d - '%s', character: '%c'";
+ }
+ else {
+ fmt_string = "error while parsing %s: "
+ "line: %d, column: %d - '%s', character: '0x%02x'";
+ }
+ ucl_create_err (err, fmt_string,
+ filename, chunk->line, chunk->column,
+ str, *chunk->pos);
+ }
+ else {
+ ucl_create_err (err, "error while parsing %s: at the end of chunk: %s",
+ filename, str);
+ }
+
+ parser->err_code = code;
+ parser->state = UCL_STATE_ERROR;
+}
+
+static void
+ucl_save_comment (struct ucl_parser *parser, const char *begin, size_t len)
+{
+ ucl_object_t *nobj;
+
+ if (len > 0 && begin != NULL) {
+ nobj = ucl_object_fromstring_common (begin, len, 0);
+
+ if (parser->last_comment) {
+ /* We need to append data to an existing object */
+ DL_APPEND (parser->last_comment, nobj);
+ }
+ else {
+ parser->last_comment = nobj;
+ }
+ }
+}
+
+static void
+ucl_attach_comment (struct ucl_parser *parser, ucl_object_t *obj, bool before)
+{
+ if (parser->last_comment) {
+ ucl_object_insert_key (parser->comments, parser->last_comment,
+ (const char *)&obj, sizeof (void *), true);
+
+ if (before) {
+ parser->last_comment->flags |= UCL_OBJECT_INHERITED;
+ }
+
+ parser->last_comment = NULL;
+ }
+}
+
+/**
+ * Skip all comments from the current pos resolving nested and multiline comments
+ * @param parser
+ * @return
+ */
+static bool
+ucl_skip_comments (struct ucl_parser *parser)
+{
+ struct ucl_chunk *chunk = parser->chunks;
+ const unsigned char *p, *beg = NULL;
+ int comments_nested = 0;
+ bool quoted = false;
+
+ p = chunk->pos;
+
+start:
+ if (chunk->remain > 0 && *p == '#') {
+ if (parser->state != UCL_STATE_SCOMMENT &&
+ parser->state != UCL_STATE_MCOMMENT) {
+ beg = p;
+
+ while (p < chunk->end) {
+ if (*p == '\n') {
+ if (parser->flags & UCL_PARSER_SAVE_COMMENTS) {
+ ucl_save_comment (parser, beg, p - beg);
+ beg = NULL;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+
+ goto start;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ }
+ }
+ else if (chunk->remain >= 2 && *p == '/') {
+ if (p[1] == '*') {
+ beg = p;
+ ucl_chunk_skipc (chunk, p);
+ comments_nested ++;
+ ucl_chunk_skipc (chunk, p);
+
+ while (p < chunk->end) {
+ if (*p == '"' && *(p - 1) != '\\') {
+ quoted = !quoted;
+ }
+
+ if (!quoted) {
+ if (*p == '*') {
+ ucl_chunk_skipc (chunk, p);
+ if (chunk->remain > 0 && *p == '/') {
+ comments_nested --;
+ if (comments_nested == 0) {
+ if (parser->flags & UCL_PARSER_SAVE_COMMENTS) {
+ ucl_save_comment (parser, beg, p - beg + 1);
+ beg = NULL;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ goto start;
+ }
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (p[0] == '/' && chunk->remain >= 2 && p[1] == '*') {
+ comments_nested ++;
+ ucl_chunk_skipc (chunk, p);
+ ucl_chunk_skipc (chunk, p);
+ continue;
+ }
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ }
+ if (comments_nested != 0) {
+ ucl_set_err (parser, UCL_ENESTED,
+ "unfinished multiline comment", &parser->err);
+ return false;
+ }
+ }
+ }
+
+ if (beg && p > beg && (parser->flags & UCL_PARSER_SAVE_COMMENTS)) {
+ ucl_save_comment (parser, beg, p - beg);
+ }
+
+ return true;
+}
+
+/**
+ * Return multiplier for a character
+ * @param c multiplier character
+ * @param is_bytes if true use 1024 multiplier
+ * @return multiplier
+ */
+static inline unsigned long
+ucl_lex_num_multiplier (const unsigned char c, bool is_bytes) {
+ const struct {
+ char c;
+ long mult_normal;
+ long mult_bytes;
+ } multipliers[] = {
+ {'m', 1000 * 1000, 1024 * 1024},
+ {'k', 1000, 1024},
+ {'g', 1000 * 1000 * 1000, 1024 * 1024 * 1024}
+ };
+ int i;
+
+ for (i = 0; i < 3; i ++) {
+ if (tolower (c) == multipliers[i].c) {
+ if (is_bytes) {
+ return multipliers[i].mult_bytes;
+ }
+ return multipliers[i].mult_normal;
+ }
+ }
+
+ return 1;
+}
+
+
+/**
+ * Return multiplier for time scaling
+ * @param c
+ * @return
+ */
+static inline double
+ucl_lex_time_multiplier (const unsigned char c) {
+ const struct {
+ char c;
+ double mult;
+ } multipliers[] = {
+ {'m', 60},
+ {'h', 60 * 60},
+ {'d', 60 * 60 * 24},
+ {'w', 60 * 60 * 24 * 7},
+ {'y', 60 * 60 * 24 * 365}
+ };
+ int i;
+
+ for (i = 0; i < 5; i ++) {
+ if (tolower (c) == multipliers[i].c) {
+ return multipliers[i].mult;
+ }
+ }
+
+ return 1;
+}
+
+/**
+ * Return true if a character is a end of an atom
+ * @param c
+ * @return
+ */
+static inline bool
+ucl_lex_is_atom_end (const unsigned char c)
+{
+ return ucl_test_character (c, UCL_CHARACTER_VALUE_END);
+}
+
+static inline bool
+ucl_lex_is_comment (const unsigned char c1, const unsigned char c2)
+{
+ if (c1 == '/') {
+ if (c2 == '*') {
+ return true;
+ }
+ }
+ else if (c1 == '#') {
+ return true;
+ }
+ return false;
+}
+
+/**
+ * Check variable found
+ * @param parser
+ * @param ptr
+ * @param remain
+ * @param out_len
+ * @param strict
+ * @param found
+ * @return
+ */
+static inline const char *
+ucl_check_variable_safe (struct ucl_parser *parser, const char *ptr, size_t remain,
+ size_t *out_len, bool strict, bool *found)
+{
+ struct ucl_variable *var;
+ unsigned char *dst;
+ size_t dstlen;
+ bool need_free = false;
+
+ LL_FOREACH (parser->variables, var) {
+ if (strict) {
+ if (remain == var->var_len) {
+ if (memcmp (ptr, var->var, var->var_len) == 0) {
+ *out_len += var->value_len;
+ *found = true;
+ return (ptr + var->var_len);
+ }
+ }
+ }
+ else {
+ if (remain >= var->var_len) {
+ if (memcmp (ptr, var->var, var->var_len) == 0) {
+ *out_len += var->value_len;
+ *found = true;
+ return (ptr + var->var_len);
+ }
+ }
+ }
+ }
+
+ /* XXX: can only handle ${VAR} */
+ if (!(*found) && parser->var_handler != NULL && strict) {
+ /* Call generic handler */
+ if (parser->var_handler (ptr, remain, &dst, &dstlen, &need_free,
+ parser->var_data)) {
+ *found = true;
+ if (need_free) {
+ free (dst);
+ }
+ return (ptr + remain);
+ }
+ }
+
+ return ptr;
+}
+
+/**
+ * Check for a variable in a given string
+ * @param parser
+ * @param ptr
+ * @param remain
+ * @param out_len
+ * @param vars_found
+ * @return
+ */
+static const char *
+ucl_check_variable (struct ucl_parser *parser, const char *ptr,
+ size_t remain, size_t *out_len, bool *vars_found)
+{
+ const char *p, *end, *ret = ptr;
+ bool found = false;
+
+ if (*ptr == '{') {
+ /* We need to match the variable enclosed in braces */
+ p = ptr + 1;
+ end = ptr + remain;
+ while (p < end) {
+ if (*p == '}') {
+ ret = ucl_check_variable_safe (parser, ptr + 1, p - ptr - 1,
+ out_len, true, &found);
+ if (found) {
+ /* {} must be excluded actually */
+ ret ++;
+ if (!*vars_found) {
+ *vars_found = true;
+ }
+ }
+ else {
+ *out_len += 2;
+ }
+ break;
+ }
+ p ++;
+ }
+ if(p == end) {
+ (*out_len) ++;
+ }
+ }
+ else if (*ptr != '$') {
+ /* Not count escaped dollar sign */
+ ret = ucl_check_variable_safe (parser, ptr, remain, out_len, false, &found);
+ if (found && !*vars_found) {
+ *vars_found = true;
+ }
+ if (!found) {
+ (*out_len) ++;
+ }
+ }
+ else {
+ ret ++;
+ (*out_len) ++;
+ }
+
+ return ret;
+}
+
+/**
+ * Expand a single variable
+ * @param parser
+ * @param ptr
+ * @param in_len
+ * @param dest
+ * @param out_len
+ * @return
+ */
+static const char *
+ucl_expand_single_variable (struct ucl_parser *parser, const char *ptr,
+ size_t in_len, unsigned char **dest, size_t out_len)
+{
+ unsigned char *d = *dest, *dst;
+ const char *p = ptr + 1, *ret;
+ struct ucl_variable *var;
+ size_t dstlen;
+ bool need_free = false;
+ bool found = false;
+ bool strict = false;
+
+ ret = ptr + 1;
+ /* For the $ sign */
+ in_len --;
+
+ if (*p == '$') {
+ *d++ = *p++;
+ *dest = d;
+ return p;
+ }
+ else if (*p == '{') {
+ p ++;
+ in_len --;
+ strict = true;
+ ret += 2;
+ }
+
+ LL_FOREACH (parser->variables, var) {
+ if (out_len >= var->value_len && in_len >= (var->var_len + (strict ? 1 : 0))) {
+ if (memcmp (p, var->var, var->var_len) == 0) {
+ if (!strict || p[var->var_len] == '}') {
+ memcpy (d, var->value, var->value_len);
+ ret += var->var_len;
+ d += var->value_len;
+ found = true;
+ break;
+ }
+ }
+ }
+ }
+
+ if (!found) {
+ if (strict && parser->var_handler != NULL) {
+ dstlen = out_len;
+
+ if (parser->var_handler (p, in_len, &dst, &dstlen, &need_free,
+ parser->var_data)) {
+ if (dstlen > out_len) {
+ /* We do not have enough space! */
+ if (need_free) {
+ free (dst);
+ }
+ }
+ else {
+ memcpy(d, dst, dstlen);
+ ret += in_len;
+ d += dstlen;
+ found = true;
+
+ if (need_free) {
+ free(dst);
+ }
+ }
+ }
+ }
+
+ /* Leave variable as is, in this case we use dest */
+ if (!found) {
+ if (strict && out_len >= 2) {
+ /* Copy '${' */
+ memcpy (d, ptr, 2);
+ d += 2;
+ ret --;
+ }
+ else {
+ memcpy (d, ptr, 1);
+ d ++;
+ }
+ }
+ }
+
+ *dest = d;
+ return ret;
+}
+
+/**
+ * Expand variables in string
+ * @param parser
+ * @param dst
+ * @param src
+ * @param in_len
+ * @return
+ */
+static ssize_t
+ucl_expand_variable (struct ucl_parser *parser, unsigned char **dst,
+ const char *src, size_t in_len)
+{
+ const char *p, *end = src + in_len;
+ unsigned char *d, *d_end;
+ size_t out_len = 0;
+ bool vars_found = false;
+
+ if (parser->flags & UCL_PARSER_DISABLE_MACRO) {
+ *dst = NULL;
+ return in_len;
+ }
+
+ p = src;
+ while (p != end) {
+ if (*p == '$' && p + 1 != end) {
+ p = ucl_check_variable (parser, p + 1, end - p - 1, &out_len, &vars_found);
+ }
+ else {
+ p ++;
+ out_len ++;
+ }
+ }
+
+ if (!vars_found) {
+ /* Trivial case */
+ *dst = NULL;
+ return in_len;
+ }
+
+ *dst = UCL_ALLOC (out_len + 1);
+ if (*dst == NULL) {
+ return in_len;
+ }
+
+ d = *dst;
+ d_end = d + out_len;
+ p = src;
+ while (p != end && d != d_end) {
+ if (*p == '$' && p + 1 != end) {
+ p = ucl_expand_single_variable (parser, p, end - p, &d, d_end - d);
+ }
+ else {
+ *d++ = *p++;
+ }
+ }
+
+ *d = '\0';
+
+ return out_len;
+}
+
+/**
+ * Store or copy pointer to the trash stack
+ * @param parser parser object
+ * @param src src string
+ * @param dst destination buffer (trash stack pointer)
+ * @param dst_const const destination pointer (e.g. value of object)
+ * @param in_len input length
+ * @param need_unescape need to unescape source (and copy it)
+ * @param need_lowercase need to lowercase value (and copy)
+ * @param need_expand need to expand variables (and copy as well)
+ * @param unescape_squote unescape single quoted string
+ * @return output length (excluding \0 symbol)
+ */
+static inline ssize_t
+ucl_copy_or_store_ptr (struct ucl_parser *parser,
+ const unsigned char *src, unsigned char **dst,
+ const char **dst_const, size_t in_len,
+ bool need_unescape, bool need_lowercase, bool need_expand,
+ bool unescape_squote)
+{
+ ssize_t ret = -1, tret;
+ unsigned char *tmp;
+
+ if (need_unescape || need_lowercase ||
+ (need_expand && parser->variables != NULL) ||
+ !(parser->flags & UCL_PARSER_ZEROCOPY)) {
+ /* Copy string */
+ *dst = UCL_ALLOC (in_len + 1);
+ if (*dst == NULL) {
+ ucl_set_err (parser, UCL_EINTERNAL, "cannot allocate memory for a string",
+ &parser->err);
+ return false;
+ }
+ if (need_lowercase) {
+ ret = ucl_strlcpy_tolower (*dst, src, in_len + 1);
+ }
+ else {
+ ret = ucl_strlcpy_unsafe (*dst, src, in_len + 1);
+ }
+
+ if (need_unescape) {
+ if (!unescape_squote) {
+ ret = ucl_unescape_json_string (*dst, ret);
+ }
+ else {
+ ret = ucl_unescape_squoted_string (*dst, ret);
+ }
+ }
+
+ if (need_expand) {
+ tmp = *dst;
+ tret = ret;
+ ret = ucl_expand_variable (parser, dst, tmp, ret);
+ if (*dst == NULL) {
+ /* Nothing to expand */
+ *dst = tmp;
+ ret = tret;
+ }
+ else {
+ /* Free unexpanded value */
+ UCL_FREE (in_len + 1, tmp);
+ }
+ }
+ *dst_const = *dst;
+ }
+ else {
+ *dst_const = src;
+ ret = in_len;
+ }
+
+ return ret;
+}
+
+/**
+ * Create and append an object at the specified level
+ * @param parser
+ * @param is_array
+ * @param level
+ * @return
+ */
+static inline ucl_object_t *
+ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser,
+ bool is_array, uint32_t level, bool has_obrace)
+{
+ struct ucl_stack *st;
+ ucl_object_t *nobj;
+
+ if (obj == NULL) {
+ nobj = ucl_object_new_full (is_array ? UCL_ARRAY : UCL_OBJECT, parser->chunks->priority);
+ if (nobj == NULL) {
+ goto enomem0;
+ }
+ } else {
+ if (obj->type == (is_array ? UCL_OBJECT : UCL_ARRAY)) {
+ /* Bad combination for merge: array and object */
+ ucl_set_err (parser, UCL_EMERGE,
+ "cannot merge an object with an array",
+ &parser->err);
+
+ return NULL;
+ }
+ nobj = obj;
+ nobj->type = is_array ? UCL_ARRAY : UCL_OBJECT;
+ }
+
+ if (!is_array) {
+ if (nobj->value.ov == NULL) {
+ nobj->value.ov = ucl_hash_create (parser->flags & UCL_PARSER_KEY_LOWERCASE);
+ if (nobj->value.ov == NULL) {
+ goto enomem1;
+ }
+ }
+ parser->state = UCL_STATE_KEY;
+ } else {
+ parser->state = UCL_STATE_VALUE;
+ }
+
+ st = UCL_ALLOC (sizeof (struct ucl_stack));
+
+ if (st == NULL) {
+ goto enomem1;
+ }
+
+ st->obj = nobj;
+
+ if (level >= UINT16_MAX) {
+ ucl_set_err (parser, UCL_ENESTED,
+ "objects are nesting too deep (over 65535 limit)",
+ &parser->err);
+ if (nobj != obj) {
+ ucl_object_unref (obj);
+ }
+
+ UCL_FREE(sizeof (struct ucl_stack), st);
+
+ return NULL;
+ }
+
+
+ st->e.params.level = level;
+ st->e.params.line = parser->chunks->line;
+ st->chunk = parser->chunks;
+
+ if (has_obrace) {
+ st->e.params.flags = UCL_STACK_HAS_OBRACE;
+ }
+ else {
+ st->e.params.flags = 0;
+ }
+
+ LL_PREPEND (parser->stack, st);
+ parser->cur_obj = nobj;
+
+ return nobj;
+enomem1:
+ if (nobj != obj)
+ ucl_object_unref (nobj);
+enomem0:
+ ucl_set_err (parser, UCL_EINTERNAL, "cannot allocate memory for an object",
+ &parser->err);
+ return NULL;
+}
+
+int
+ucl_maybe_parse_number (ucl_object_t *obj,
+ const char *start, const char *end, const char **pos,
+ bool allow_double, bool number_bytes, bool allow_time)
+{
+ const char *p = start, *c = start;
+ char *endptr;
+ bool got_dot = false, got_exp = false, need_double = false,
+ is_time = false, valid_start = false, is_hex = false;
+ int is_neg = 0;
+ double dv = 0;
+ int64_t lv = 0;
+
+ if (*p == '-') {
+ is_neg = 1;
+ c ++;
+ p ++;
+ }
+ while (p < end) {
+ if (is_hex && isxdigit (*p)) {
+ p ++;
+ }
+ else if (isdigit (*p)) {
+ valid_start = true;
+ p ++;
+ }
+ else if (!is_hex && (*p == 'x' || *p == 'X')) {
+ is_hex = true;
+ allow_double = false;
+ c = p + 1;
+ p ++;
+ }
+ else if (allow_double) {
+ if (p == c) {
+ /* Empty digits sequence, not a number */
+ *pos = start;
+ return EINVAL;
+ }
+ else if (*p == '.') {
+ if (got_dot) {
+ /* Double dots, not a number */
+ *pos = start;
+ return EINVAL;
+ }
+ else {
+ got_dot = true;
+ need_double = true;
+ p ++;
+ }
+ }
+ else if (*p == 'e' || *p == 'E') {
+ if (got_exp) {
+ /* Double exp, not a number */
+ *pos = start;
+ return EINVAL;
+ }
+ else {
+ got_exp = true;
+ need_double = true;
+ p ++;
+ if (p >= end) {
+ *pos = start;
+ return EINVAL;
+ }
+ if (!isdigit (*p) && *p != '+' && *p != '-') {
+ /* Wrong exponent sign */
+ *pos = start;
+ return EINVAL;
+ }
+ else {
+ p ++;
+ }
+ }
+ }
+ else {
+ /* Got the end of the number, need to check */
+ break;
+ }
+ }
+ else if (!allow_double && *p == '.') {
+ /* Unexpected dot */
+ *pos = start;
+ return EINVAL;
+ }
+ else {
+ break;
+ }
+ }
+
+ if (!valid_start || p == c) {
+ *pos = start;
+ return EINVAL;
+ }
+
+ char numbuf[128];
+
+ if ((size_t)(p - c + 1) >= sizeof(numbuf)) {
+ *pos = start;
+ return EINVAL;
+ }
+
+ if (is_neg) {
+ numbuf[0] = '-';
+ ucl_strlcpy (&numbuf[1], c, p - c + 1);
+ }
+ else {
+ ucl_strlcpy (numbuf, c, p - c + 1);
+ }
+
+ errno = 0;
+ if (need_double) {
+ dv = strtod (numbuf, &endptr);
+ }
+ else {
+ if (is_hex) {
+ lv = strtoimax (numbuf, &endptr, 16);
+ }
+ else {
+ lv = strtoimax (numbuf, &endptr, 10);
+ }
+ }
+ if (errno == ERANGE) {
+ *pos = start;
+ return ERANGE;
+ }
+
+ /* Now check endptr and move it from numbuf to the real ending */
+ if (endptr != NULL) {
+ long shift = endptr - numbuf - is_neg;
+ endptr = (char *)c + shift;
+ }
+ if (endptr >= end) {
+ p = end;
+ goto set_obj;
+ }
+ if (endptr == NULL || ucl_lex_is_atom_end (*endptr) || *endptr == '\0') {
+ p = endptr;
+ goto set_obj;
+ }
+
+ if (endptr < end && endptr != start) {
+ switch (*p) {
+ case 'm':
+ case 'M':
+ case 'g':
+ case 'G':
+ case 'k':
+ case 'K':
+ if (end - p >= 2) {
+ if (p[1] == 's' || p[1] == 'S') {
+ /* Milliseconds */
+ if (!need_double) {
+ need_double = true;
+ dv = lv;
+ }
+ is_time = true;
+ if (p[0] == 'm' || p[0] == 'M') {
+ dv /= 1000.;
+ }
+ else {
+ dv *= ucl_lex_num_multiplier (*p, false);
+ }
+ p += 2;
+ if (end - p > 0 && !ucl_lex_is_atom_end (*p)) {
+ *pos = start;
+ return EINVAL;
+ }
+ goto set_obj;
+ }
+ else if (number_bytes || (p[1] == 'b' || p[1] == 'B')) {
+ /* Bytes */
+ if (need_double) {
+ need_double = false;
+ lv = dv;
+ }
+ lv *= ucl_lex_num_multiplier (*p, true);
+ p += 2;
+ if (end - p > 0 && !ucl_lex_is_atom_end (*p)) {
+ *pos = start;
+ return EINVAL;
+ }
+ goto set_obj;
+ }
+ else if (ucl_lex_is_atom_end (p[1])) {
+ if (need_double) {
+ dv *= ucl_lex_num_multiplier (*p, false);
+ }
+ else {
+ lv *= ucl_lex_num_multiplier (*p, number_bytes);
+ }
+ p ++;
+ goto set_obj;
+ }
+ else if (allow_time && end - p >= 3) {
+ if (tolower (p[0]) == 'm' &&
+ tolower (p[1]) == 'i' &&
+ tolower (p[2]) == 'n') {
+ /* Minutes */
+ if (!need_double) {
+ need_double = true;
+ dv = lv;
+ }
+ is_time = true;
+ dv *= 60.;
+ p += 3;
+ if (end - p > 0 && !ucl_lex_is_atom_end (*p)) {
+ *pos = start;
+ return EINVAL;
+ }
+ goto set_obj;
+ }
+ }
+ }
+ else {
+ if (need_double) {
+ dv *= ucl_lex_num_multiplier (*p, false);
+ }
+ else {
+ lv *= ucl_lex_num_multiplier (*p, number_bytes);
+ }
+ p ++;
+ if (end - p > 0 && !ucl_lex_is_atom_end (*p)) {
+ *pos = start;
+ return EINVAL;
+ }
+ goto set_obj;
+ }
+ break;
+ case 'S':
+ case 's':
+ if (allow_time &&
+ (p == end - 1 || ucl_lex_is_atom_end (p[1]))) {
+ if (!need_double) {
+ need_double = true;
+ dv = lv;
+ }
+ p ++;
+ is_time = true;
+ goto set_obj;
+ }
+ break;
+ case 'h':
+ case 'H':
+ case 'd':
+ case 'D':
+ case 'w':
+ case 'W':
+ case 'Y':
+ case 'y':
+ if (allow_time &&
+ (p == end - 1 || ucl_lex_is_atom_end (p[1]))) {
+ if (!need_double) {
+ need_double = true;
+ dv = lv;
+ }
+ is_time = true;
+ dv *= ucl_lex_time_multiplier (*p);
+ p ++;
+ goto set_obj;
+ }
+ break;
+ case '\t':
+ case ' ':
+ while (p < end && ucl_test_character(*p, UCL_CHARACTER_WHITESPACE)) {
+ p++;
+ }
+ if (ucl_lex_is_atom_end(*p))
+ goto set_obj;
+ break;
+ }
+ }
+ else if (endptr == end) {
+ /* Just a number at the end of chunk */
+ p = end;
+ goto set_obj;
+ }
+
+ *pos = c;
+ return EINVAL;
+
+set_obj:
+ if (obj != NULL) {
+ if (allow_double && (need_double || is_time)) {
+ if (!is_time) {
+ obj->type = UCL_FLOAT;
+ }
+ else {
+ obj->type = UCL_TIME;
+ }
+ obj->value.dv = dv;
+ }
+ else {
+ obj->type = UCL_INT;
+ obj->value.iv = lv;
+ }
+ }
+ *pos = p;
+ return 0;
+}
+
+/**
+ * Parse possible number
+ * @param parser
+ * @param chunk
+ * @param obj
+ * @return true if a number has been parsed
+ */
+static bool
+ucl_lex_number (struct ucl_parser *parser,
+ struct ucl_chunk *chunk, ucl_object_t *obj)
+{
+ const unsigned char *pos;
+ int ret;
+
+ ret = ucl_maybe_parse_number (obj, chunk->pos, chunk->end, (const char **)&pos,
+ true, false, ((parser->flags & UCL_PARSER_NO_TIME) == 0));
+
+ if (ret == 0) {
+ chunk->remain -= pos - chunk->pos;
+ chunk->column += pos - chunk->pos;
+ chunk->pos = pos;
+ return true;
+ }
+ else if (ret == ERANGE) {
+ ucl_set_err (parser, UCL_ESYNTAX, "numeric value out of range",
+ &parser->err);
+ }
+
+ return false;
+}
+
+/**
+ * Parse quoted string with possible escapes
+ * @param parser
+ * @param chunk
+ * @param need_unescape
+ * @param ucl_escape
+ * @param var_expand
+ * @return true if a string has been parsed
+ */
+static bool
+ucl_lex_json_string (struct ucl_parser *parser,
+ struct ucl_chunk *chunk,
+ bool *need_unescape,
+ bool *ucl_escape,
+ bool *var_expand)
+{
+ const unsigned char *p = chunk->pos;
+ unsigned char c;
+ int i;
+
+ while (p < chunk->end) {
+ c = *p;
+ if (c < 0x1F) {
+ /* Unmasked control character */
+ if (c == '\n') {
+ ucl_set_err (parser, UCL_ESYNTAX, "unexpected newline",
+ &parser->err);
+ }
+ else {
+ ucl_set_err (parser, UCL_ESYNTAX, "unexpected control character",
+ &parser->err);
+ }
+ return false;
+ }
+ else if (c == '\\') {
+ ucl_chunk_skipc (chunk, p);
+ if (p >= chunk->end) {
+ ucl_set_err (parser, UCL_ESYNTAX, "unfinished escape character",
+ &parser->err);
+ return false;
+ }
+ c = *p;
+ if (ucl_test_character (c, UCL_CHARACTER_ESCAPE)) {
+ if (c == 'u') {
+ ucl_chunk_skipc (chunk, p);
+ for (i = 0; i < 4 && p < chunk->end; i ++) {
+ if (!isxdigit (*p)) {
+ ucl_set_err (parser, UCL_ESYNTAX, "invalid utf escape",
+ &parser->err);
+ return false;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ if (p >= chunk->end) {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "unfinished escape character",
+ &parser->err);
+ return false;
+ }
+ }
+ else {
+ ucl_chunk_skipc (chunk, p);
+ }
+ }
+ *need_unescape = true;
+ *ucl_escape = true;
+ continue;
+ }
+ else if (c == '"') {
+ ucl_chunk_skipc (chunk, p);
+ return true;
+ }
+ else if (ucl_test_character (c, UCL_CHARACTER_UCL_UNSAFE)) {
+ *ucl_escape = true;
+ }
+ else if (c == '$') {
+ *var_expand = true;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "no quote at the end of json string",
+ &parser->err);
+ return false;
+}
+
+/**
+ * Process single quoted string
+ * @param parser
+ * @param chunk
+ * @param need_unescape
+ * @return
+ */
+static bool
+ucl_lex_squoted_string (struct ucl_parser *parser,
+ struct ucl_chunk *chunk, bool *need_unescape)
+{
+ const unsigned char *p = chunk->pos;
+ unsigned char c;
+
+ while (p < chunk->end) {
+ c = *p;
+ if (c == '\\') {
+ ucl_chunk_skipc (chunk, p);
+
+ if (p >= chunk->end) {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "unfinished escape character",
+ &parser->err);
+ return false;
+ }
+ else {
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ *need_unescape = true;
+ continue;
+ }
+ else if (c == '\'') {
+ ucl_chunk_skipc (chunk, p);
+ return true;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "no quote at the end of single quoted string",
+ &parser->err);
+ return false;
+}
+
+static void
+ucl_parser_append_elt (struct ucl_parser *parser, ucl_hash_t *cont,
+ ucl_object_t *top,
+ ucl_object_t *elt)
+{
+ ucl_object_t *nobj;
+
+ if ((parser->flags & UCL_PARSER_NO_IMPLICIT_ARRAYS) == 0) {
+ /* Implicit array */
+ top->flags |= UCL_OBJECT_MULTIVALUE;
+ DL_APPEND (top, elt);
+ parser->stack->obj->len ++;
+ }
+ else {
+ if ((top->flags & UCL_OBJECT_MULTIVALUE) != 0) {
+ /* Just add to the explicit array */
+ ucl_array_append (top, elt);
+ }
+ else {
+ /* Convert to an array */
+ nobj = ucl_object_typed_new (UCL_ARRAY);
+ nobj->key = top->key;
+ nobj->keylen = top->keylen;
+ nobj->flags |= UCL_OBJECT_MULTIVALUE;
+ ucl_array_append (nobj, top);
+ ucl_array_append (nobj, elt);
+ ucl_hash_replace (cont, top, nobj);
+ }
+ }
+}
+
+bool
+ucl_parser_process_object_element (struct ucl_parser *parser, ucl_object_t *nobj)
+{
+ ucl_hash_t *container;
+ ucl_object_t *tobj = NULL, *cur;
+ char errmsg[256];
+
+ container = parser->stack->obj->value.ov;
+
+ DL_FOREACH (parser->stack->obj, cur) {
+ tobj = __DECONST (ucl_object_t *, ucl_hash_search_obj (cur->value.ov, nobj));
+
+ if (tobj != NULL) {
+ break;
+ }
+ }
+
+
+ if (tobj == NULL) {
+ container = ucl_hash_insert_object (container, nobj,
+ parser->flags & UCL_PARSER_KEY_LOWERCASE);
+ if (container == NULL) {
+ return false;
+ }
+ nobj->prev = nobj;
+ nobj->next = NULL;
+ parser->stack->obj->len ++;
+ }
+ else {
+ unsigned priold = ucl_object_get_priority (tobj),
+ prinew = ucl_object_get_priority (nobj);
+ switch (parser->chunks->strategy) {
+
+ case UCL_DUPLICATE_APPEND:
+ /*
+ * The logic here is the following:
+ *
+ * - if we have two objects with the same priority, then we form an
+ * implicit or explicit array
+ * - if a new object has bigger priority, then we overwrite an old one
+ * - if a new object has lower priority, then we ignore it
+ */
+ /* Special case for inherited objects */
+ if (tobj->flags & UCL_OBJECT_INHERITED) {
+ prinew = priold + 1;
+ }
+
+ if (priold == prinew) {
+ ucl_parser_append_elt (parser, container, tobj, nobj);
+ }
+ else if (priold > prinew) {
+ /*
+ * We add this new object to a list of trash objects just to ensure
+ * that it won't come to any real object
+ * XXX: rather inefficient approach
+ */
+ DL_APPEND (parser->trash_objs, nobj);
+ }
+ else {
+ ucl_hash_replace (container, tobj, nobj);
+ ucl_object_unref (tobj);
+ }
+
+ break;
+
+ case UCL_DUPLICATE_REWRITE:
+ /* We just rewrite old values regardless of priority */
+ ucl_hash_replace (container, tobj, nobj);
+ ucl_object_unref (tobj);
+
+ break;
+
+ case UCL_DUPLICATE_ERROR:
+ snprintf(errmsg, sizeof(errmsg),
+ "duplicate element for key '%s' found",
+ nobj->key);
+ ucl_set_err (parser, UCL_EMERGE, errmsg, &parser->err);
+ return false;
+
+ case UCL_DUPLICATE_MERGE:
+ /*
+ * Here we do have some old object so we just push it on top of objects stack
+ * Check priority and then perform the merge on the remaining objects
+ */
+ if (tobj->type == UCL_OBJECT || tobj->type == UCL_ARRAY) {
+ ucl_object_unref (nobj);
+ nobj = tobj;
+ }
+ else if (priold == prinew) {
+ ucl_parser_append_elt (parser, container, tobj, nobj);
+ }
+ else if (priold > prinew) {
+ /*
+ * We add this new object to a list of trash objects just to ensure
+ * that it won't come to any real object
+ * XXX: rather inefficient approach
+ */
+ DL_APPEND (parser->trash_objs, nobj);
+ }
+ else {
+ ucl_hash_replace (container, tobj, nobj);
+ ucl_object_unref (tobj);
+ }
+ break;
+ }
+ }
+
+ parser->stack->obj->value.ov = container;
+ parser->cur_obj = nobj;
+ ucl_attach_comment (parser, nobj, false);
+
+ return true;
+}
+
+/**
+ * Parse a key in an object
+ * @param parser
+ * @param chunk
+ * @param next_key
+ * @param end_of_object
+ * @return true if a key has been parsed
+ */
+static bool
+ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk,
+ bool *next_key, bool *end_of_object)
+{
+ const unsigned char *p, *c = NULL, *end, *t;
+ const char *key = NULL;
+ bool got_quote = false, got_eq = false, got_semicolon = false,
+ need_unescape = false, ucl_escape = false, var_expand = false,
+ got_content = false, got_sep = false;
+ ucl_object_t *nobj;
+ ssize_t keylen;
+
+ p = chunk->pos;
+
+ if (*p == '.') {
+ /* It is macro actually */
+ if (!(parser->flags & UCL_PARSER_DISABLE_MACRO)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_MACRO_NAME;
+ *end_of_object = false;
+ return true;
+ }
+ while (p < chunk->end) {
+ /*
+ * A key must start with alpha, number, '/' or '_' and end with space character
+ */
+ if (c == NULL) {
+ if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) {
+ if (!ucl_skip_comments (parser)) {
+ return false;
+ }
+ p = chunk->pos;
+ }
+ else if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (ucl_test_character (*p, UCL_CHARACTER_KEY_START)) {
+ /* The first symbol */
+ c = p;
+ ucl_chunk_skipc (chunk, p);
+ got_content = true;
+ }
+ else if (*p == '"') {
+ /* JSON style key */
+ c = p + 1;
+ got_quote = true;
+ got_content = true;
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (*p == '}') {
+ /* We have actually end of an object */
+ *end_of_object = true;
+ return true;
+ }
+ else if (*p == '.') {
+ ucl_chunk_skipc (chunk, p);
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_MACRO_NAME;
+ return true;
+ }
+ else {
+ /* Invalid identifier */
+ ucl_set_err (parser, UCL_ESYNTAX, "key must begin with a letter",
+ &parser->err);
+ return false;
+ }
+ }
+ else {
+ /* Parse the body of a key */
+ if (!got_quote) {
+ if (ucl_test_character (*p, UCL_CHARACTER_KEY)) {
+ got_content = true;
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (ucl_test_character (*p, UCL_CHARACTER_KEY_SEP)) {
+ end = p;
+ break;
+ }
+ else {
+ ucl_set_err (parser, UCL_ESYNTAX, "invalid character in a key",
+ &parser->err);
+ return false;
+ }
+ }
+ else {
+ /* We need to parse json like quoted string */
+ if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape, &var_expand)) {
+ return false;
+ }
+ /* Always escape keys obtained via json */
+ end = chunk->pos - 1;
+ p = chunk->pos;
+ break;
+ }
+ }
+ }
+
+ if (p >= chunk->end && got_content) {
+ ucl_set_err (parser, UCL_ESYNTAX, "unfinished key", &parser->err);
+ return false;
+ }
+ else if (!got_content) {
+ return true;
+ }
+ *end_of_object = false;
+ /* We are now at the end of the key, need to parse the rest */
+ while (p < chunk->end) {
+ if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (*p == '=') {
+ if (!got_eq && !got_semicolon) {
+ ucl_chunk_skipc (chunk, p);
+ got_eq = true;
+ }
+ else {
+ ucl_set_err (parser, UCL_ESYNTAX, "unexpected '=' character",
+ &parser->err);
+ return false;
+ }
+ }
+ else if (*p == ':') {
+ if (!got_eq && !got_semicolon) {
+ ucl_chunk_skipc (chunk, p);
+ got_semicolon = true;
+ }
+ else {
+ ucl_set_err (parser, UCL_ESYNTAX, "unexpected ':' character",
+ &parser->err);
+ return false;
+ }
+ }
+ else if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) {
+ /* Check for comment */
+ if (!ucl_skip_comments (parser)) {
+ return false;
+ }
+ p = chunk->pos;
+ }
+ else {
+ /* Start value */
+ break;
+ }
+ }
+
+ if (p >= chunk->end && got_content) {
+ ucl_set_err (parser, UCL_ESYNTAX, "unfinished key", &parser->err);
+ return false;
+ }
+
+ got_sep = got_semicolon || got_eq;
+
+ if (!got_sep) {
+ /*
+ * Maybe we have more keys nested, so search for termination character.
+ * Possible choices:
+ * 1) key1 key2 ... keyN [:=] value <- we treat that as error
+ * 2) key1 ... keyN {} or [] <- we treat that as nested objects
+ * 3) key1 value[;,\n] <- we treat that as linear object
+ */
+ t = p;
+ *next_key = false;
+ while (ucl_test_character (*t, UCL_CHARACTER_WHITESPACE)) {
+ t ++;
+ }
+ /* Check first non-space character after a key */
+ if (*t != '{' && *t != '[') {
+ while (t < chunk->end) {
+ if (*t == ',' || *t == ';' || *t == '\n' || *t == '\r') {
+ break;
+ }
+ else if (*t == '{' || *t == '[') {
+ *next_key = true;
+ break;
+ }
+ t ++;
+ }
+ }
+ }
+
+ /* Create a new object */
+ nobj = ucl_object_new_full (UCL_NULL, parser->chunks->priority);
+ if (nobj == NULL) {
+ return false;
+ }
+ keylen = ucl_copy_or_store_ptr (parser, c, &nobj->trash_stack[UCL_TRASH_KEY],
+ &key, end - c, need_unescape, parser->flags & UCL_PARSER_KEY_LOWERCASE,
+ false, false);
+ if (keylen == -1) {
+ ucl_object_unref (nobj);
+ return false;
+ }
+ else if (keylen == 0) {
+ ucl_set_err (parser, UCL_ESYNTAX, "empty keys are not allowed", &parser->err);
+ ucl_object_unref (nobj);
+ return false;
+ }
+
+ nobj->key = key;
+ nobj->keylen = keylen;
+
+ if (!ucl_parser_process_object_element (parser, nobj)) {
+ return false;
+ }
+
+ if (ucl_escape) {
+ nobj->flags |= UCL_OBJECT_NEED_KEY_ESCAPE;
+ }
+
+
+ return true;
+}
+
+/**
+ * Parse a cl string
+ * @param parser
+ * @param chunk
+ * @param var_expand
+ * @param need_unescape
+ * @return true if a key has been parsed
+ */
+static bool
+ucl_parse_string_value (struct ucl_parser *parser,
+ struct ucl_chunk *chunk, bool *var_expand, bool *need_unescape)
+{
+ const unsigned char *p;
+ enum {
+ UCL_BRACE_ROUND = 0,
+ UCL_BRACE_SQUARE,
+ UCL_BRACE_FIGURE
+ };
+ int braces[3][2] = {{0, 0}, {0, 0}, {0, 0}};
+
+ p = chunk->pos;
+
+ while (p < chunk->end) {
+
+ /* Skip pairs of figure braces */
+ if (*p == '{') {
+ braces[UCL_BRACE_FIGURE][0] ++;
+ }
+ else if (*p == '}') {
+ braces[UCL_BRACE_FIGURE][1] ++;
+ if (braces[UCL_BRACE_FIGURE][1] <= braces[UCL_BRACE_FIGURE][0]) {
+ /* This is not a termination symbol, continue */
+ ucl_chunk_skipc (chunk, p);
+ continue;
+ }
+ }
+ /* Skip pairs of square braces */
+ else if (*p == '[') {
+ braces[UCL_BRACE_SQUARE][0] ++;
+ }
+ else if (*p == ']') {
+ braces[UCL_BRACE_SQUARE][1] ++;
+ if (braces[UCL_BRACE_SQUARE][1] <= braces[UCL_BRACE_SQUARE][0]) {
+ /* This is not a termination symbol, continue */
+ ucl_chunk_skipc (chunk, p);
+ continue;
+ }
+ }
+ else if (*p == '$') {
+ *var_expand = true;
+ }
+ else if (*p == '\\') {
+ *need_unescape = true;
+ ucl_chunk_skipc (chunk, p);
+ if (p < chunk->end) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ continue;
+ }
+
+ if (ucl_lex_is_atom_end (*p) || (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1]))) {
+ break;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ return true;
+}
+
+/**
+ * Parse multiline string ending with \n{term}\n
+ * @param parser
+ * @param chunk
+ * @param term
+ * @param term_len
+ * @param beg
+ * @param var_expand
+ * @return size of multiline string or 0 in case of error
+ */
+static int
+ucl_parse_multiline_string (struct ucl_parser *parser,
+ struct ucl_chunk *chunk, const unsigned char *term,
+ int term_len, unsigned char const **beg,
+ bool *var_expand)
+{
+ const unsigned char *p, *c, *tend;
+ bool newline = false;
+ int len = 0;
+
+ p = chunk->pos;
+
+ c = p;
+
+ while (p < chunk->end) {
+ if (newline) {
+ if (chunk->end - p < term_len) {
+ return 0;
+ }
+ else if (memcmp (p, term, term_len) == 0) {
+ tend = p + term_len;
+ if (*tend != '\n' && *tend != ';' && *tend != ',') {
+ /* Incomplete terminator */
+ ucl_chunk_skipc (chunk, p);
+ continue;
+ }
+ len = p - c;
+ chunk->remain -= term_len;
+ chunk->pos = p + term_len;
+ chunk->column = term_len;
+ *beg = c;
+ break;
+ }
+ }
+ if (*p == '\n') {
+ newline = true;
+ }
+ else {
+ if (*p == '$') {
+ *var_expand = true;
+ }
+ newline = false;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ return len;
+}
+
+static inline ucl_object_t*
+ucl_parser_get_container (struct ucl_parser *parser)
+{
+ ucl_object_t *t, *obj = NULL;
+
+ if (parser == NULL || parser->stack == NULL || parser->stack->obj == NULL) {
+ return NULL;
+ }
+
+ if (parser->stack->obj->type == UCL_ARRAY) {
+ /* Object must be allocated */
+ obj = ucl_object_new_full (UCL_NULL, parser->chunks->priority);
+ t = parser->stack->obj;
+
+ if (!ucl_array_append (t, obj)) {
+ ucl_object_unref (obj);
+ return NULL;
+ }
+
+ parser->cur_obj = obj;
+ ucl_attach_comment (parser, obj, false);
+ }
+ else {
+ /* Object has been already allocated */
+ obj = parser->cur_obj;
+ }
+
+ return obj;
+}
+
+/**
+ * Handle value data
+ * @param parser
+ * @param chunk
+ * @return
+ */
+static bool
+ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk)
+{
+ const unsigned char *p, *c;
+ ucl_object_t *obj = NULL;
+ unsigned int stripped_spaces;
+ ssize_t str_len;
+ bool need_unescape = false, ucl_escape = false, var_expand = false;
+
+ p = chunk->pos;
+
+ /* Skip any spaces and comments */
+ if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) ||
+ (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1]))) {
+ while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ if (!ucl_skip_comments (parser)) {
+ return false;
+ }
+ p = chunk->pos;
+ }
+
+ while (p < chunk->end) {
+ c = p;
+ switch (*p) {
+ case '"':
+ ucl_chunk_skipc (chunk, p);
+
+ if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape,
+ &var_expand)) {
+ return false;
+ }
+
+ obj = ucl_parser_get_container (parser);
+ if (!obj) {
+ return false;
+ }
+
+ str_len = chunk->pos - c - 2;
+ obj->type = UCL_STRING;
+ if ((str_len = ucl_copy_or_store_ptr (parser, c + 1,
+ &obj->trash_stack[UCL_TRASH_VALUE],
+ &obj->value.sv, str_len, need_unescape, false,
+ var_expand, false)) == -1) {
+ return false;
+ }
+
+ obj->len = str_len;
+ parser->state = UCL_STATE_AFTER_VALUE;
+
+ return true;
+ break;
+ case '\'':
+ ucl_chunk_skipc (chunk, p);
+
+ if (!ucl_lex_squoted_string (parser, chunk, &need_unescape)) {
+ return false;
+ }
+
+ obj = ucl_parser_get_container (parser);
+ if (!obj) {
+ return false;
+ }
+
+ str_len = chunk->pos - c - 2;
+ obj->type = UCL_STRING;
+ obj->flags |= UCL_OBJECT_SQUOTED;
+
+ if ((str_len = ucl_copy_or_store_ptr (parser, c + 1,
+ &obj->trash_stack[UCL_TRASH_VALUE],
+ &obj->value.sv, str_len, need_unescape, false,
+ var_expand, true)) == -1) {
+ return false;
+ }
+
+ obj->len = str_len;
+
+ parser->state = UCL_STATE_AFTER_VALUE;
+
+ return true;
+ break;
+ case '{':
+ obj = ucl_parser_get_container (parser);
+ if (obj == NULL) {
+ return false;
+ }
+ /* We have a new object */
+ if (parser->stack) {
+ obj = ucl_parser_add_container (obj, parser, false,
+ parser->stack->e.params.level, true);
+ }
+ else {
+ return false;
+ }
+ if (obj == NULL) {
+ return false;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+
+ return true;
+ break;
+ case '[':
+ obj = ucl_parser_get_container (parser);
+ if (obj == NULL) {
+ return false;
+ }
+ /* We have a new array */
+ if (parser->stack) {
+ obj = ucl_parser_add_container (obj, parser, true,
+ parser->stack->e.params.level, true);
+ }
+ else {
+ return false;
+ }
+
+ if (obj == NULL) {
+ return false;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+
+ return true;
+ break;
+ case ']':
+ /* We have the array ending */
+ if (parser->stack && parser->stack->obj->type == UCL_ARRAY) {
+ parser->state = UCL_STATE_AFTER_VALUE;
+ return true;
+ }
+ else {
+ goto parse_string;
+ }
+ break;
+ case '<':
+ obj = ucl_parser_get_container (parser);
+ /* We have something like multiline value, which must be <<[A-Z]+\n */
+ if (chunk->end - p > 3) {
+ if (memcmp (p, "<<", 2) == 0) {
+ p += 2;
+ /* We allow only uppercase characters in multiline definitions */
+ while (p < chunk->end && *p >= 'A' && *p <= 'Z') {
+ p ++;
+ }
+ if(p == chunk->end) {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "unterminated multiline value", &parser->err);
+ return false;
+ }
+ if (*p =='\n') {
+ /* Set chunk positions and start multiline parsing */
+ chunk->remain -= p - c + 1;
+ c += 2;
+ chunk->pos = p + 1;
+ chunk->column = 0;
+ chunk->line ++;
+ if ((str_len = ucl_parse_multiline_string (parser, chunk, c,
+ p - c, &c, &var_expand)) == 0) {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "unterminated multiline value", &parser->err);
+ return false;
+ }
+
+ obj->type = UCL_STRING;
+ obj->flags |= UCL_OBJECT_MULTILINE;
+ if ((str_len = ucl_copy_or_store_ptr (parser, c,
+ &obj->trash_stack[UCL_TRASH_VALUE],
+ &obj->value.sv, str_len - 1, false,
+ false, var_expand, false)) == -1) {
+ return false;
+ }
+ obj->len = str_len;
+
+ parser->state = UCL_STATE_AFTER_VALUE;
+
+ return true;
+ }
+ }
+ }
+ /* Fallback to ordinary strings */
+ /* FALLTHRU */
+ default:
+parse_string:
+ if (obj == NULL) {
+ obj = ucl_parser_get_container (parser);
+ }
+
+ /* Parse atom */
+ if (ucl_test_character (*p, UCL_CHARACTER_VALUE_DIGIT_START)) {
+ if (!ucl_lex_number (parser, chunk, obj)) {
+ if (parser->state == UCL_STATE_ERROR) {
+ return false;
+ }
+ }
+ else {
+ parser->state = UCL_STATE_AFTER_VALUE;
+ return true;
+ }
+ /* Fallback to normal string */
+ }
+
+ if (!ucl_parse_string_value (parser, chunk, &var_expand,
+ &need_unescape)) {
+ return false;
+ }
+ /* Cut trailing spaces */
+ stripped_spaces = 0;
+ while (ucl_test_character (*(chunk->pos - 1 - stripped_spaces),
+ UCL_CHARACTER_WHITESPACE)) {
+ stripped_spaces ++;
+ }
+ str_len = chunk->pos - c - stripped_spaces;
+ if (str_len <= 0) {
+ ucl_set_err (parser, UCL_ESYNTAX, "string value must not be empty",
+ &parser->err);
+ return false;
+ }
+ else if (str_len == 4 && memcmp (c, "null", 4) == 0) {
+ obj->len = 0;
+ obj->type = UCL_NULL;
+ }
+ else if (str_len == 3 && memcmp (c, "nan", 3) == 0) {
+ obj->len = 0;
+ obj->type = UCL_FLOAT;
+ obj->value.dv = NAN;
+ }
+ else if (str_len == 3 && memcmp (c, "inf", 3) == 0) {
+ obj->len = 0;
+ obj->type = UCL_FLOAT;
+ obj->value.dv = INFINITY;
+ }
+ else if (!ucl_maybe_parse_boolean (obj, c, str_len)) {
+ obj->type = UCL_STRING;
+ if ((str_len = ucl_copy_or_store_ptr (parser, c,
+ &obj->trash_stack[UCL_TRASH_VALUE],
+ &obj->value.sv, str_len, need_unescape,
+ false, var_expand, false)) == -1) {
+ return false;
+ }
+ obj->len = str_len;
+ }
+
+ parser->state = UCL_STATE_AFTER_VALUE;
+
+ return true;
+ break;
+ }
+ }
+
+ return true;
+}
+
+/**
+ * Handle after value data
+ * @param parser
+ * @param chunk
+ * @return
+ */
+static bool
+ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk)
+{
+ const unsigned char *p;
+ bool got_sep = false;
+ struct ucl_stack *st;
+
+ p = chunk->pos;
+
+ while (p < chunk->end) {
+ if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) {
+ /* Skip whitespaces */
+ ucl_chunk_skipc (chunk, p);
+ }
+ else if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) {
+ /* Skip comment */
+ if (!ucl_skip_comments (parser)) {
+ return false;
+ }
+ /* Treat comment as a separator */
+ got_sep = true;
+ p = chunk->pos;
+ }
+ else if (ucl_test_character (*p, UCL_CHARACTER_VALUE_END)) {
+ if (*p == '}' || *p == ']') {
+ if (parser->stack == NULL) {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "end of array or object detected without corresponding start",
+ &parser->err);
+ return false;
+ }
+ if ((*p == '}' && parser->stack->obj->type == UCL_OBJECT) ||
+ (*p == ']' && parser->stack->obj->type == UCL_ARRAY)) {
+
+ /* Pop all nested objects from a stack */
+ st = parser->stack;
+
+ if (!(st->e.params.flags & UCL_STACK_HAS_OBRACE)) {
+ parser->err_code = UCL_EUNPAIRED;
+ ucl_create_err (&parser->err,
+ "%s:%d object closed with } is not opened with { at line %d",
+ chunk->fname ? chunk->fname : "memory",
+ parser->chunks->line, st->e.params.line);
+
+ return false;
+ }
+
+ parser->stack = st->next;
+ UCL_FREE (sizeof (struct ucl_stack), st);
+
+ if (parser->cur_obj) {
+ ucl_attach_comment (parser, parser->cur_obj, true);
+ }
+
+ while (parser->stack != NULL) {
+ st = parser->stack;
+
+ if (st->next == NULL) {
+ break;
+ }
+ else if (st->next->e.params.level == st->e.params.level) {
+ break;
+ }
+
+
+ parser->stack = st->next;
+ parser->cur_obj = st->obj;
+ UCL_FREE (sizeof (struct ucl_stack), st);
+ }
+ }
+ else {
+ ucl_set_err (parser, UCL_ESYNTAX,
+ "unexpected terminating symbol detected",
+ &parser->err);
+ return false;
+ }
+
+ if (parser->stack == NULL) {
+ /* Ignore everything after a top object */
+ return true;
+ }
+ else {
+ ucl_chunk_skipc (chunk, p);
+ }
+ got_sep = true;
+ }
+ else {
+ /* Got a separator */
+ got_sep = true;
+ ucl_chunk_skipc (chunk, p);
+ }
+ }
+ else {
+ /* Anything else */
+ if (!got_sep) {
+ ucl_set_err (parser, UCL_ESYNTAX, "delimiter is missing",
+ &parser->err);
+ return false;
+ }
+ return true;
+ }
+ }
+
+ return true;
+}
+
+static bool
+ucl_skip_macro_as_comment (struct ucl_parser *parser,
+ struct ucl_chunk *chunk)
+{
+ const unsigned char *p, *c;
+ enum {
+ macro_skip_start = 0,
+ macro_has_symbols,
+ macro_has_obrace,
+ macro_has_quote,
+ macro_has_backslash,
+ macro_has_sqbrace,
+ macro_save
+ } state = macro_skip_start, prev_state = macro_skip_start;
+
+ p = chunk->pos;
+ c = chunk->pos;
+
+ while (p < chunk->end) {
+ switch (state) {
+ case macro_skip_start:
+ if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) {
+ state = macro_has_symbols;
+ }
+ else if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ state = macro_save;
+ continue;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_has_symbols:
+ if (*p == '{') {
+ state = macro_has_sqbrace;
+ }
+ else if (*p == '(') {
+ state = macro_has_obrace;
+ }
+ else if (*p == '"') {
+ state = macro_has_quote;
+ }
+ else if (*p == '\n') {
+ state = macro_save;
+ continue;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_has_obrace:
+ if (*p == '\\') {
+ prev_state = state;
+ state = macro_has_backslash;
+ }
+ else if (*p == ')') {
+ state = macro_has_symbols;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_has_sqbrace:
+ if (*p == '\\') {
+ prev_state = state;
+ state = macro_has_backslash;
+ }
+ else if (*p == '}') {
+ state = macro_save;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_has_quote:
+ if (*p == '\\') {
+ prev_state = state;
+ state = macro_has_backslash;
+ }
+ else if (*p == '"') {
+ state = macro_save;
+ }
+
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_has_backslash:
+ state = prev_state;
+ ucl_chunk_skipc (chunk, p);
+ break;
+
+ case macro_save:
+ if (parser->flags & UCL_PARSER_SAVE_COMMENTS) {
+ ucl_save_comment (parser, c, p - c);
+ }
+
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/**
+ * Handle macro data
+ * @param parser
+ * @param chunk
+ * @param marco
+ * @param macro_start
+ * @param macro_len
+ * @return
+ */
+static bool
+ucl_parse_macro_value (struct ucl_parser *parser,
+ struct ucl_chunk *chunk, struct ucl_macro *macro,
+ unsigned char const **macro_start, size_t *macro_len)
+{
+ const unsigned char *p, *c;
+ bool need_unescape = false, ucl_escape = false, var_expand = false;
+
+ p = chunk->pos;
+
+ switch (*p) {
+ case '"':
+ /* We have macro value encoded in quotes */
+ c = p;
+ ucl_chunk_skipc (chunk, p);
+ if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape, &var_expand)) {
+ return false;
+ }
+
+ *macro_start = c + 1;
+ *macro_len = chunk->pos - c - 2;
+ p = chunk->pos;
+ break;
+ case '{':
+ /* We got a multiline macro body */
+ ucl_chunk_skipc (chunk, p);
+ /* Skip spaces at the beginning */
+ while (p < chunk->end) {
+ if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ else {
+ break;
+ }
+ }
+ c = p;
+ while (p < chunk->end) {
+ if (*p == '}') {
+ break;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ *macro_start = c;
+ *macro_len = p - c;
+ ucl_chunk_skipc (chunk, p);
+ break;
+ default:
+ /* Macro is not enclosed in quotes or braces */
+ c = p;
+ while (p < chunk->end) {
+ if (ucl_lex_is_atom_end (*p)) {
+ break;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ *macro_start = c;
+ *macro_len = p - c;
+ break;
+ }
+
+ /* We are at the end of a macro */
+ /* Skip ';' and space characters and return to previous state */
+ while (p < chunk->end) {
+ if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) && *p != ';') {
+ break;
+ }
+ ucl_chunk_skipc (chunk, p);
+ }
+ return true;
+}
+
+/**
+ * Parse macro arguments as UCL object
+ * @param parser parser structure
+ * @param chunk the current data chunk
+ * @return
+ */
+static ucl_object_t *
+ucl_parse_macro_arguments (struct ucl_parser *parser,
+ struct ucl_chunk *chunk)
+{
+ ucl_object_t *res = NULL;
+ struct ucl_parser *params_parser;
+ int obraces = 1, ebraces = 0, state = 0;
+ const unsigned char *p, *c;
+ size_t args_len = 0;
+ struct ucl_parser_saved_state saved;
+
+ saved.column = chunk->column;
+ saved.line = chunk->line;
+ saved.pos = chunk->pos;
+ saved.remain = chunk->remain;
+ p = chunk->pos;
+
+ if (*p != '(' || chunk->remain < 2) {
+ return NULL;
+ }
+
+ /* Set begin and start */
+ ucl_chunk_skipc (chunk, p);
+ c = p;
+
+ while ((p) < (chunk)->end) {
+ switch (state) {
+ case 0:
+ /* Parse symbols and check for '(', ')' and '"' */
+ if (*p == '(') {
+ obraces ++;
+ }
+ else if (*p == ')') {
+ ebraces ++;
+ }
+ else if (*p == '"') {
+ state = 1;
+ }
+ /* Check pairing */
+ if (obraces == ebraces) {
+ state = 99;
+ }
+ else {
+ args_len ++;
+ }
+ /* Check overflow */
+ if (chunk->remain == 0) {
+ goto restore_chunk;
+ }
+ ucl_chunk_skipc (chunk, p);
+ break;
+ case 1:
+ /* We have quote character, so skip all but quotes */
+ if (*p == '"' && *(p - 1) != '\\') {
+ state = 0;
+ }
+ if (chunk->remain == 0) {
+ goto restore_chunk;
+ }
+ args_len ++;
+ ucl_chunk_skipc (chunk, p);
+ break;
+ case 99:
+ /*
+ * We have read the full body of arguments, so we need to parse and set
+ * object from that
+ */
+ params_parser = ucl_parser_new (parser->flags);
+ if (!ucl_parser_add_chunk (params_parser, c, args_len)) {
+ ucl_set_err (parser, UCL_ESYNTAX, "macro arguments parsing error",
+ &parser->err);
+ }
+ else {
+ res = ucl_parser_get_object (params_parser);
+ }
+ ucl_parser_free (params_parser);
+
+ return res;
+
+ break;
+ }
+ }
+
+ return res;
+
+restore_chunk:
+ chunk->column = saved.column;
+ chunk->line = saved.line;
+ chunk->pos = saved.pos;
+ chunk->remain = saved.remain;
+
+ return NULL;
+}
+
+#define SKIP_SPACES_COMMENTS(parser, chunk, p) do { \
+ while ((p) < (chunk)->end) { \
+ if (!ucl_test_character (*(p), UCL_CHARACTER_WHITESPACE_UNSAFE)) { \
+ if ((chunk)->remain >= 2 && ucl_lex_is_comment ((p)[0], (p)[1])) { \
+ if (!ucl_skip_comments (parser)) { \
+ return false; \
+ } \
+ p = (chunk)->pos; \
+ } \
+ break; \
+ } \
+ ucl_chunk_skipc (chunk, p); \
+ } \
+} while(0)
+
+/**
+ * Handle the main states of rcl parser
+ * @param parser parser structure
+ * @return true if chunk has been parsed and false in case of error
+ */
+static bool
+ucl_state_machine (struct ucl_parser *parser)
+{
+ ucl_object_t *obj, *macro_args;
+ struct ucl_chunk *chunk = parser->chunks;
+ const unsigned char *p, *c = NULL, *macro_start = NULL;
+ unsigned char *macro_escaped;
+ size_t macro_len = 0;
+ struct ucl_macro *macro = NULL;
+ bool next_key = false, end_of_object = false, ret;
+
+ if (parser->top_obj == NULL) {
+ parser->state = UCL_STATE_INIT;
+ }
+
+ p = chunk->pos;
+ while (chunk->pos < chunk->end) {
+ switch (parser->state) {
+ case UCL_STATE_INIT:
+ /*
+ * At the init state we can either go to the parse array or object
+ * if we got [ or { correspondingly or can just treat new data as
+ * a key of newly created object
+ */
+ if (!ucl_skip_comments (parser)) {
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ else {
+ bool seen_obrace = false;
+
+ /* Skip any spaces */
+ while (p < chunk->end && ucl_test_character (*p,
+ UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+
+ p = chunk->pos;
+
+ if (p < chunk->end) {
+ if (*p == '[') {
+ parser->state = UCL_STATE_VALUE;
+ ucl_chunk_skipc (chunk, p);
+ seen_obrace = true;
+ }
+ else {
+
+ if (*p == '{') {
+ ucl_chunk_skipc (chunk, p);
+ parser->state = UCL_STATE_KEY_OBRACE;
+ seen_obrace = true;
+ }
+ else {
+ parser->state = UCL_STATE_KEY;
+ }
+ }
+ }
+
+ if (parser->top_obj == NULL) {
+ if (parser->state == UCL_STATE_VALUE) {
+ obj = ucl_parser_add_container (NULL, parser, true, 0,
+ seen_obrace);
+ }
+ else {
+ obj = ucl_parser_add_container (NULL, parser, false, 0,
+ seen_obrace);
+ }
+
+ if (obj == NULL) {
+ return false;
+ }
+
+ parser->top_obj = obj;
+ parser->cur_obj = obj;
+ }
+
+ }
+ break;
+ case UCL_STATE_KEY:
+ case UCL_STATE_KEY_OBRACE:
+ /* Skip any spaces */
+ while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ ucl_chunk_skipc (chunk, p);
+ }
+ if (p == chunk->end || *p == '}') {
+ /* We have the end of an object */
+ parser->state = UCL_STATE_AFTER_VALUE;
+ continue;
+ }
+ if (parser->stack == NULL) {
+ /* No objects are on stack, but we want to parse a key */
+ ucl_set_err (parser, UCL_ESYNTAX, "top object is finished but the parser "
+ "expects a key", &parser->err);
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ if (!ucl_parse_key (parser, chunk, &next_key, &end_of_object)) {
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+
+ if (end_of_object) {
+ p = chunk->pos;
+ parser->state = UCL_STATE_AFTER_VALUE;
+ continue;
+ }
+ else if (parser->state != UCL_STATE_MACRO_NAME) {
+ if (next_key && parser->stack->obj->type == UCL_OBJECT) {
+ /* Parse more keys and nest objects accordingly */
+ obj = ucl_parser_add_container (parser->cur_obj,
+ parser,
+ false,
+ parser->stack->e.params.level + 1,
+ parser->state == UCL_STATE_KEY_OBRACE);
+ if (obj == NULL) {
+ return false;
+ }
+ }
+ else {
+ parser->state = UCL_STATE_VALUE;
+ }
+ }
+ else {
+ c = chunk->pos;
+ }
+ p = chunk->pos;
+ break;
+ case UCL_STATE_VALUE:
+ /* We need to check what we do have */
+ if (!parser->cur_obj || !ucl_parse_value (parser, chunk)) {
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ /* State is set in ucl_parse_value call */
+ p = chunk->pos;
+ break;
+ case UCL_STATE_AFTER_VALUE:
+ if (!ucl_parse_after_value (parser, chunk)) {
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+
+ if (parser->stack != NULL) {
+ if (parser->stack->obj->type == UCL_OBJECT) {
+ parser->state = UCL_STATE_KEY;
+ }
+ else {
+ /* Array */
+ parser->state = UCL_STATE_VALUE;
+ }
+ }
+ else {
+ /* Skip everything at the end */
+ return true;
+ }
+
+ p = chunk->pos;
+ break;
+ case UCL_STATE_MACRO_NAME:
+ if (parser->flags & UCL_PARSER_DISABLE_MACRO) {
+ if (!ucl_skip_macro_as_comment (parser, chunk)) {
+ /* We have invalid macro */
+ ucl_create_err (&parser->err,
+ "error at %s:%d at column %d: invalid macro",
+ chunk->fname ? chunk->fname : "memory",
+ chunk->line,
+ chunk->column);
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ else {
+ p = chunk->pos;
+ parser->state = parser->prev_state;
+ }
+ }
+ else {
+ if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) &&
+ *p != '(') {
+ ucl_chunk_skipc (chunk, p);
+ }
+ else {
+ if (c != NULL && p - c > 0) {
+ /* We got macro name */
+ macro_len = (size_t) (p - c);
+ HASH_FIND (hh, parser->macroes, c, macro_len, macro);
+ if (macro == NULL) {
+ ucl_create_err (&parser->err,
+ "error at %s:%d at column %d: "
+ "unknown macro: '%.*s', character: '%c'",
+ chunk->fname ? chunk->fname : "memory",
+ chunk->line,
+ chunk->column,
+ (int) (p - c),
+ c,
+ *chunk->pos);
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ /* Now we need to skip all spaces */
+ SKIP_SPACES_COMMENTS(parser, chunk, p);
+ parser->state = UCL_STATE_MACRO;
+ }
+ else {
+ /* We have invalid macro name */
+ ucl_create_err (&parser->err,
+ "error at %s:%d at column %d: invalid macro name",
+ chunk->fname ? chunk->fname : "memory",
+ chunk->line,
+ chunk->column);
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ }
+ }
+ break;
+ case UCL_STATE_MACRO:
+ if (*chunk->pos == '(') {
+ macro_args = ucl_parse_macro_arguments (parser, chunk);
+ p = chunk->pos;
+ if (macro_args) {
+ SKIP_SPACES_COMMENTS(parser, chunk, p);
+ }
+ }
+ else {
+ macro_args = NULL;
+ }
+ if (!ucl_parse_macro_value (parser, chunk, macro,
+ &macro_start, &macro_len)) {
+ parser->prev_state = parser->state;
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ macro_len = ucl_expand_variable (parser, &macro_escaped,
+ macro_start, macro_len);
+ parser->state = parser->prev_state;
+
+ if (macro_escaped == NULL && macro != NULL) {
+ if (macro->is_context) {
+ ret = macro->h.context_handler (macro_start, macro_len,
+ macro_args,
+ parser->top_obj,
+ macro->ud);
+ }
+ else {
+ ret = macro->h.handler (macro_start, macro_len, macro_args,
+ macro->ud);
+ }
+ }
+ else if (macro != NULL) {
+ if (macro->is_context) {
+ ret = macro->h.context_handler (macro_escaped, macro_len,
+ macro_args,
+ parser->top_obj,
+ macro->ud);
+ }
+ else {
+ ret = macro->h.handler (macro_escaped, macro_len, macro_args,
+ macro->ud);
+ }
+
+ UCL_FREE (macro_len + 1, macro_escaped);
+ }
+ else {
+ ret = false;
+ ucl_set_err (parser, UCL_EINTERNAL,
+ "internal error: parser has macro undefined", &parser->err);
+ }
+
+ /*
+ * Chunk can be modified within macro handler
+ */
+ chunk = parser->chunks;
+ p = chunk->pos;
+
+ if (macro_args) {
+ ucl_object_unref (macro_args);
+ }
+
+ if (!ret) {
+ return false;
+ }
+ break;
+ default:
+ ucl_set_err (parser, UCL_EINTERNAL,
+ "internal error: parser is in an unknown state", &parser->err);
+ parser->state = UCL_STATE_ERROR;
+ return false;
+ }
+ }
+
+ if (parser->last_comment) {
+ if (parser->cur_obj) {
+ ucl_attach_comment (parser, parser->cur_obj, true);
+ }
+ else if (parser->stack && parser->stack->obj) {
+ ucl_attach_comment (parser, parser->stack->obj, true);
+ }
+ else if (parser->top_obj) {
+ ucl_attach_comment (parser, parser->top_obj, true);
+ }
+ else {
+ ucl_object_unref (parser->last_comment);
+ }
+ }
+
+ if (parser->stack != NULL && parser->state != UCL_STATE_ERROR) {
+ struct ucl_stack *st;
+ bool has_error = false;
+
+ LL_FOREACH (parser->stack, st) {
+ if (st->chunk != parser->chunks) {
+ break; /* Not our chunk, give up */
+ }
+ if (st->e.params.flags & UCL_STACK_HAS_OBRACE) {
+ if (parser->err == NULL) {
+ utstring_new (parser->err);
+ }
+
+ utstring_printf (parser->err, "%s:%d unmatched open brace at %d; ",
+ chunk->fname ? chunk->fname : "memory",
+ parser->chunks->line,
+ st->e.params.line);
+
+ has_error = true;
+ }
+ }
+
+ if (has_error) {
+ parser->err_code = UCL_EUNPAIRED;
+
+ return false;
+ }
+ }
+
+ return true;
+}
+
+#define UPRM_SAFE(fn, a, b, c, el) do { \
+ if (!fn(a, b, c, a)) \
+ goto el; \
+ } while (0)
+
+struct ucl_parser*
+ucl_parser_new (int flags)
+{
+ struct ucl_parser *parser;
+
+ parser = UCL_ALLOC (sizeof (struct ucl_parser));
+ if (parser == NULL) {
+ return NULL;
+ }
+
+ memset (parser, 0, sizeof (struct ucl_parser));
+
+ UPRM_SAFE(ucl_parser_register_macro, parser, "include", ucl_include_handler, e0);
+ UPRM_SAFE(ucl_parser_register_macro, parser, "try_include", ucl_try_include_handler, e0);
+ UPRM_SAFE(ucl_parser_register_macro, parser, "includes", ucl_includes_handler, e0);
+ UPRM_SAFE(ucl_parser_register_macro, parser, "priority", ucl_priority_handler, e0);
+ UPRM_SAFE(ucl_parser_register_macro, parser, "load", ucl_load_handler, e0);
+ UPRM_SAFE(ucl_parser_register_context_macro, parser, "inherit", ucl_inherit_handler, e0);
+
+ parser->flags = flags;
+ parser->includepaths = NULL;
+
+ if (flags & UCL_PARSER_SAVE_COMMENTS) {
+ parser->comments = ucl_object_typed_new (UCL_OBJECT);
+ }
+
+ if (!(flags & UCL_PARSER_NO_FILEVARS)) {
+ /* Initial assumption about filevars */
+ ucl_parser_set_filevars (parser, NULL, false);
+ }
+
+ return parser;
+e0:
+ ucl_parser_free(parser);
+ return NULL;
+}
+
+bool
+ucl_parser_set_default_priority (struct ucl_parser *parser, unsigned prio)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ parser->default_priority = prio;
+
+ return true;
+}
+
+int
+ucl_parser_get_default_priority (struct ucl_parser *parser)
+{
+ if (parser == NULL) {
+ return -1;
+ }
+
+ return parser->default_priority;
+}
+
+bool
+ucl_parser_register_macro (struct ucl_parser *parser, const char *macro,
+ ucl_macro_handler handler, void* ud)
+{
+ struct ucl_macro *new;
+
+ if (macro == NULL || handler == NULL) {
+ return false;
+ }
+
+ new = UCL_ALLOC (sizeof (struct ucl_macro));
+ if (new == NULL) {
+ return false;
+ }
+
+ memset (new, 0, sizeof (struct ucl_macro));
+ new->h.handler = handler;
+ new->name = strdup (macro);
+ if (new->name == NULL) {
+ UCL_FREE (sizeof (struct ucl_macro), new);
+ return false;
+ }
+ new->ud = ud;
+ HASH_ADD_KEYPTR (hh, parser->macroes, new->name, strlen (new->name), new);
+ return true;
+}
+
+bool
+ucl_parser_register_context_macro (struct ucl_parser *parser, const char *macro,
+ ucl_context_macro_handler handler, void* ud)
+{
+ struct ucl_macro *new;
+
+ if (macro == NULL || handler == NULL) {
+ return false;
+ }
+
+ new = UCL_ALLOC (sizeof (struct ucl_macro));
+ if (new == NULL) {
+ return false;
+ }
+
+ memset (new, 0, sizeof (struct ucl_macro));
+ new->h.context_handler = handler;
+ new->name = strdup (macro);
+ if (new->name == NULL) {
+ UCL_FREE (sizeof (struct ucl_macro), new);
+ return false;
+ }
+ new->ud = ud;
+ new->is_context = true;
+ HASH_ADD_KEYPTR (hh, parser->macroes, new->name, strlen (new->name), new);
+ return true;
+}
+
+void
+ucl_parser_register_variable (struct ucl_parser *parser, const char *var,
+ const char *value)
+{
+ struct ucl_variable *new = NULL, *cur;
+
+ if (var == NULL) {
+ return;
+ }
+
+ /* Find whether a variable already exists */
+ LL_FOREACH (parser->variables, cur) {
+ if (strcmp (cur->var, var) == 0) {
+ new = cur;
+ break;
+ }
+ }
+
+ if (value == NULL) {
+
+ if (new != NULL) {
+ /* Remove variable */
+ DL_DELETE (parser->variables, new);
+ free (new->var);
+ free (new->value);
+ UCL_FREE (sizeof (struct ucl_variable), new);
+ }
+ else {
+ /* Do nothing */
+ return;
+ }
+ }
+ else {
+ if (new == NULL) {
+ new = UCL_ALLOC (sizeof (struct ucl_variable));
+ if (new == NULL) {
+ return;
+ }
+ memset (new, 0, sizeof (struct ucl_variable));
+ new->var = strdup (var);
+ new->var_len = strlen (var);
+ new->value = strdup (value);
+ new->value_len = strlen (value);
+
+ DL_APPEND (parser->variables, new);
+ }
+ else {
+ free (new->value);
+ new->value = strdup (value);
+ new->value_len = strlen (value);
+ }
+ }
+}
+
+void
+ucl_parser_set_variables_handler (struct ucl_parser *parser,
+ ucl_variable_handler handler, void *ud)
+{
+ parser->var_handler = handler;
+ parser->var_data = ud;
+}
+
+bool
+ucl_parser_add_chunk_full (struct ucl_parser *parser, const unsigned char *data,
+ size_t len, unsigned priority, enum ucl_duplicate_strategy strat,
+ enum ucl_parse_type parse_type)
+{
+ struct ucl_chunk *chunk;
+ struct ucl_parser_special_handler *special_handler;
+
+ if (parser == NULL) {
+ return false;
+ }
+
+ if (data == NULL && len != 0) {
+ ucl_create_err (&parser->err, "invalid chunk added");
+ return false;
+ }
+
+ if (parser->state != UCL_STATE_ERROR) {
+ chunk = UCL_ALLOC (sizeof (struct ucl_chunk));
+ if (chunk == NULL) {
+ ucl_create_err (&parser->err, "cannot allocate chunk structure");
+ return false;
+ }
+
+ memset (chunk, 0, sizeof (*chunk));
+
+ /* Apply all matching handlers from the first to the last */
+ LL_FOREACH (parser->special_handlers, special_handler) {
+ if ((special_handler->flags & UCL_SPECIAL_HANDLER_PREPROCESS_ALL) ||
+ (len >= special_handler->magic_len &&
+ memcmp (data, special_handler->magic, special_handler->magic_len) == 0)) {
+ unsigned char *ndata = NULL;
+ size_t nlen = 0;
+
+ if (!special_handler->handler (parser, data, len, &ndata, &nlen,
+ special_handler->user_data)) {
+ UCL_FREE(sizeof (struct ucl_chunk), chunk);
+ ucl_create_err (&parser->err, "call for external handler failed");
+
+ return false;
+ }
+
+ struct ucl_parser_special_handler_chain *nchain;
+ nchain = UCL_ALLOC (sizeof (*nchain));
+ nchain->begin = ndata;
+ nchain->len = nlen;
+ nchain->special_handler = special_handler;
+
+ /* Free order is reversed */
+ LL_PREPEND (chunk->special_handlers, nchain);
+
+ data = ndata;
+ len = nlen;
+ }
+ }
+
+ if (parse_type == UCL_PARSE_AUTO && len > 0) {
+ /* We need to detect parse type by the first symbol */
+ if ((*data & 0x80) == 0x80 && (*data >= 0xdc && *data <= 0xdf)) {
+ parse_type = UCL_PARSE_MSGPACK;
+ }
+ else if (*data == '(') {
+ parse_type = UCL_PARSE_CSEXP;
+ }
+ else {
+ parse_type = UCL_PARSE_UCL;
+ }
+ }
+
+ chunk->begin = data;
+ chunk->remain = len;
+ chunk->pos = chunk->begin;
+ chunk->end = chunk->begin + len;
+ chunk->line = 1;
+ chunk->column = 0;
+ chunk->priority = priority;
+ chunk->strategy = strat;
+ chunk->parse_type = parse_type;
+
+ if (parser->cur_file) {
+ chunk->fname = strdup (parser->cur_file);
+ }
+
+ LL_PREPEND (parser->chunks, chunk);
+ parser->recursion ++;
+
+ if (parser->recursion > UCL_MAX_RECURSION) {
+ ucl_create_err (&parser->err, "maximum include nesting limit is reached: %d",
+ parser->recursion);
+ return false;
+ }
+
+ if (len > 0) {
+ /* Need to parse something */
+ switch (parse_type) {
+ default:
+ case UCL_PARSE_UCL:
+ return ucl_state_machine (parser);
+ case UCL_PARSE_MSGPACK:
+ return ucl_parse_msgpack (parser);
+ case UCL_PARSE_CSEXP:
+ return ucl_parse_csexp (parser);
+ }
+ }
+ else {
+ /* Just add empty chunk and go forward */
+ if (parser->top_obj == NULL) {
+ /*
+ * In case of empty object, create one to indicate that we've
+ * read something
+ */
+ parser->top_obj = ucl_object_new_full (UCL_OBJECT, priority);
+ }
+
+ return true;
+ }
+ }
+
+ ucl_create_err (&parser->err, "a parser is in an invalid state");
+
+ return false;
+}
+
+bool
+ucl_parser_add_chunk_priority (struct ucl_parser *parser,
+ const unsigned char *data, size_t len, unsigned priority)
+{
+ /* We dereference parser, so this check is essential */
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_chunk_full (parser, data, len,
+ priority, UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
+}
+
+bool
+ucl_parser_add_chunk (struct ucl_parser *parser, const unsigned char *data,
+ size_t len)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_chunk_full (parser, data, len,
+ parser->default_priority, UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
+}
+
+bool
+ucl_parser_insert_chunk (struct ucl_parser *parser, const unsigned char *data,
+ size_t len)
+{
+ if (parser == NULL || parser->top_obj == NULL) {
+ return false;
+ }
+
+ bool res;
+ struct ucl_chunk *chunk;
+
+ int state = parser->state;
+ parser->state = UCL_STATE_INIT;
+
+ /* Prevent inserted chunks from unintentionally closing the current object */
+ if (parser->stack != NULL && parser->stack->next != NULL) {
+ parser->stack->e.params.level = parser->stack->next->e.params.level;
+ }
+
+ res = ucl_parser_add_chunk_full (parser, data, len, parser->chunks->priority,
+ parser->chunks->strategy, parser->chunks->parse_type);
+
+ /* Remove chunk from the stack */
+ chunk = parser->chunks;
+ if (chunk != NULL) {
+ parser->chunks = chunk->next;
+ ucl_chunk_free (chunk);
+ parser->recursion --;
+ }
+
+ parser->state = state;
+
+ return res;
+}
+
+bool
+ucl_parser_add_string_priority (struct ucl_parser *parser, const char *data,
+ size_t len, unsigned priority)
+{
+ if (data == NULL) {
+ ucl_create_err (&parser->err, "invalid string added");
+ return false;
+ }
+ if (len == 0) {
+ len = strlen (data);
+ }
+
+ return ucl_parser_add_chunk_priority (parser,
+ (const unsigned char *)data, len, priority);
+}
+
+bool
+ucl_parser_add_string (struct ucl_parser *parser, const char *data,
+ size_t len)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_string_priority (parser,
+ (const unsigned char *)data, len, parser->default_priority);
+}
+
+bool
+ucl_set_include_path (struct ucl_parser *parser, ucl_object_t *paths)
+{
+ if (parser == NULL || paths == NULL) {
+ return false;
+ }
+
+ if (parser->includepaths == NULL) {
+ parser->includepaths = ucl_object_copy (paths);
+ }
+ else {
+ ucl_object_unref (parser->includepaths);
+ parser->includepaths = ucl_object_copy (paths);
+ }
+
+ if (parser->includepaths == NULL) {
+ return false;
+ }
+
+ return true;
+}
+
+unsigned char ucl_parser_chunk_peek (struct ucl_parser *parser)
+{
+ if (parser == NULL || parser->chunks == NULL || parser->chunks->pos == NULL || parser->chunks->end == NULL ||
+ parser->chunks->pos == parser->chunks->end) {
+ return 0;
+ }
+
+ return( *parser->chunks->pos );
+}
+
+bool ucl_parser_chunk_skip (struct ucl_parser *parser)
+{
+ if (parser == NULL || parser->chunks == NULL || parser->chunks->pos == NULL || parser->chunks->end == NULL ||
+ parser->chunks->pos == parser->chunks->end) {
+ return false;
+ }
+
+ const unsigned char *p = parser->chunks->pos;
+ ucl_chunk_skipc( parser->chunks, p );
+ if( parser->chunks->pos != NULL ) return true;
+ return false;
+}
+
+ucl_object_t*
+ucl_parser_get_current_stack_object (struct ucl_parser *parser, unsigned int depth)
+{
+ ucl_object_t *obj;
+
+ if (parser == NULL || parser->stack == NULL) {
+ return NULL;
+ }
+
+ struct ucl_stack *stack = parser->stack;
+ if(stack == NULL || stack->obj == NULL || ucl_object_type (stack->obj) != UCL_OBJECT)
+ {
+ return NULL;
+ }
+
+ for( unsigned int i = 0; i < depth; ++i )
+ {
+ stack = stack->next;
+ if(stack == NULL || stack->obj == NULL || ucl_object_type (stack->obj) != UCL_OBJECT)
+ {
+ return NULL;
+ }
+ }
+
+ obj = ucl_object_ref (stack->obj);
+ return obj;
+}
+
diff --git a/contrib/libucl/ucl_schema.c b/contrib/libucl/ucl_schema.c
new file mode 100644
index 0000000..68f0118
--- /dev/null
+++ b/contrib/libucl/ucl_schema.c
@@ -0,0 +1,1104 @@
+/*
+ * Copyright (c) 2014, Vsevolod Stakhov
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "tree.h"
+#include "utlist.h"
+#ifdef HAVE_STDARG_H
+#include <stdarg.h>
+#endif
+#ifdef HAVE_STDIO_H
+#include <stdio.h>
+#endif
+#ifdef HAVE_REGEX_H
+#include <regex.h>
+#endif
+#ifdef HAVE_MATH_H
+#include <math.h>
+#endif
+
+static bool ucl_schema_validate (const ucl_object_t *schema,
+ const ucl_object_t *obj, bool try_array,
+ struct ucl_schema_error *err,
+ const ucl_object_t *root,
+ ucl_object_t *ext_ref);
+
+/*
+ * Create validation error
+ */
+
+#ifdef __GNUC__
+static inline void
+ucl_schema_create_error (struct ucl_schema_error *err,
+ enum ucl_schema_error_code code, const ucl_object_t *obj,
+ const char *fmt, ...)
+__attribute__ (( format( printf, 4, 5) ));
+#endif
+
+static inline void
+ucl_schema_create_error (struct ucl_schema_error *err,
+ enum ucl_schema_error_code code, const ucl_object_t *obj,
+ const char *fmt, ...)
+{
+ va_list va;
+
+ if (err != NULL) {
+ err->code = code;
+ err->obj = obj;
+ va_start (va, fmt);
+ vsnprintf (err->msg, sizeof (err->msg), fmt, va);
+ va_end (va);
+ }
+}
+
+/*
+ * Check whether we have a pattern specified
+ */
+static const ucl_object_t *
+ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern, bool recursive)
+{
+ const ucl_object_t *res = NULL;
+#ifdef HAVE_REGEX_H
+ regex_t reg;
+ const ucl_object_t *elt;
+ ucl_object_iter_t iter = NULL;
+
+ if (regcomp (&reg, pattern, REG_EXTENDED | REG_NOSUB) == 0) {
+ if (recursive) {
+ while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) {
+ if (regexec (&reg, ucl_object_key (elt), 0, NULL, 0) == 0) {
+ res = elt;
+ break;
+ }
+ }
+ } else {
+ if (regexec (&reg, ucl_object_key (obj), 0, NULL, 0) == 0)
+ res = obj;
+ }
+ regfree (&reg);
+ }
+#endif
+ return res;
+}
+
+/*
+ * Check dependencies for an object
+ */
+static bool
+ucl_schema_validate_dependencies (const ucl_object_t *deps,
+ const ucl_object_t *obj, struct ucl_schema_error *err,
+ const ucl_object_t *root,
+ ucl_object_t *ext_ref)
+{
+ const ucl_object_t *elt, *cur, *cur_dep;
+ ucl_object_iter_t iter = NULL, piter;
+ bool ret = true;
+
+ while (ret && (cur = ucl_object_iterate (deps, &iter, true)) != NULL) {
+ elt = ucl_object_lookup (obj, ucl_object_key (cur));
+ if (elt != NULL) {
+ /* Need to check dependencies */
+ if (cur->type == UCL_ARRAY) {
+ piter = NULL;
+ while (ret && (cur_dep = ucl_object_iterate (cur, &piter, true)) != NULL) {
+ if (ucl_object_lookup (obj, ucl_object_tostring (cur_dep)) == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_MISSING_DEPENDENCY, elt,
+ "dependency %s is missing for key %s",
+ ucl_object_tostring (cur_dep), ucl_object_key (cur));
+ ret = false;
+ break;
+ }
+ }
+ }
+ else if (cur->type == UCL_OBJECT) {
+ ret = ucl_schema_validate (cur, obj, true, err, root, ext_ref);
+ }
+ }
+ }
+
+ return ret;
+}
+
+/*
+ * Validate object
+ */
+static bool
+ucl_schema_validate_object (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err,
+ const ucl_object_t *root,
+ ucl_object_t *ext_ref)
+{
+ const ucl_object_t *elt, *prop, *found, *additional_schema = NULL,
+ *required = NULL, *pat, *pelt;
+ ucl_object_iter_t iter = NULL, piter = NULL;
+ bool ret = true, allow_additional = true;
+ int64_t minmax;
+
+ while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) {
+ if (elt->type == UCL_OBJECT &&
+ strcmp (ucl_object_key (elt), "properties") == 0) {
+ piter = NULL;
+ while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) {
+ found = ucl_object_lookup (obj, ucl_object_key (prop));
+ if (found) {
+ ret = ucl_schema_validate (prop, found, true, err, root,
+ ext_ref);
+ }
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "additionalProperties") == 0) {
+ if (elt->type == UCL_BOOLEAN) {
+ if (!ucl_object_toboolean (elt)) {
+ /* Deny additional fields completely */
+ allow_additional = false;
+ }
+ }
+ else if (elt->type == UCL_OBJECT) {
+ /* Define validator for additional fields */
+ additional_schema = elt;
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "additionalProperties attribute is invalid in schema");
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "required") == 0) {
+ if (elt->type == UCL_ARRAY) {
+ required = elt;
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "required attribute is invalid in schema");
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "minProperties") == 0
+ && ucl_object_toint_safe (elt, &minmax)) {
+ if (obj->len < minmax) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object has not enough properties: %u, minimum is: %u",
+ obj->len, (unsigned)minmax);
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "maxProperties") == 0
+ && ucl_object_toint_safe (elt, &minmax)) {
+ if (obj->len > minmax) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object has too many properties: %u, maximum is: %u",
+ obj->len, (unsigned)minmax);
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "patternProperties") == 0) {
+ const ucl_object_t *vobj;
+ ucl_object_iter_t viter;
+ piter = NULL;
+ while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) {
+ viter = NULL;
+ while (ret && (vobj = ucl_object_iterate (obj, &viter, true)) != NULL) {
+ found = ucl_schema_test_pattern (vobj, ucl_object_key (prop), false);
+ if (found) {
+ ret = ucl_schema_validate (prop, found, true, err, root,
+ ext_ref);
+ }
+ }
+ }
+ }
+ else if (elt->type == UCL_OBJECT &&
+ strcmp (ucl_object_key (elt), "dependencies") == 0) {
+ ret = ucl_schema_validate_dependencies (elt, obj, err, root,
+ ext_ref);
+ }
+ }
+
+ if (ret) {
+ /* Additional properties */
+ if (!allow_additional || additional_schema != NULL) {
+ /* Check if we have exactly the same properties in schema and object */
+ iter = ucl_object_iterate_new (obj);
+ prop = ucl_object_lookup (schema, "properties");
+ while ((elt = ucl_object_iterate_safe (iter, true)) != NULL) {
+ found = ucl_object_lookup (prop, ucl_object_key (elt));
+ if (found == NULL) {
+ /* Try patternProperties */
+ pat = ucl_object_lookup (schema, "patternProperties");
+ piter = ucl_object_iterate_new (pat);
+ while ((pelt = ucl_object_iterate_safe (piter, true)) != NULL) {
+ found = ucl_schema_test_pattern (obj, ucl_object_key (pelt), true);
+ if (found != NULL) {
+ break;
+ }
+ }
+ ucl_object_iterate_free (piter);
+ piter = NULL;
+ }
+ if (found == NULL) {
+ if (!allow_additional) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object has non-allowed property %s",
+ ucl_object_key (elt));
+ ret = false;
+ break;
+ }
+ else if (additional_schema != NULL) {
+ if (!ucl_schema_validate (additional_schema, elt,
+ true, err, root, ext_ref)) {
+ ret = false;
+ break;
+ }
+ }
+ }
+ }
+ ucl_object_iterate_free (iter);
+ iter = NULL;
+ }
+ /* Required properties */
+ if (required != NULL) {
+ iter = NULL;
+ while ((elt = ucl_object_iterate (required, &iter, true)) != NULL) {
+ if (ucl_object_lookup (obj, ucl_object_tostring (elt)) == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_MISSING_PROPERTY, obj,
+ "object has missing property %s",
+ ucl_object_tostring (elt));
+ ret = false;
+ break;
+ }
+ }
+ }
+ }
+
+
+ return ret;
+}
+
+static bool
+ucl_schema_validate_number (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err)
+{
+ const ucl_object_t *elt, *test;
+ ucl_object_iter_t iter = NULL;
+ bool ret = true, exclusive = false;
+ double constraint, val;
+ const double alpha = 1e-16;
+
+ while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) {
+ if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
+ strcmp (ucl_object_key (elt), "multipleOf") == 0) {
+ constraint = ucl_object_todouble (elt);
+ if (constraint <= 0) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "multipleOf must be greater than zero");
+ ret = false;
+ break;
+ }
+ val = ucl_object_todouble (obj);
+ if (fabs (remainder (val, constraint)) > alpha) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "number %.4f is not multiple of %.4f, remainder is %.7f",
+ val, constraint, remainder (val, constraint));
+ ret = false;
+ break;
+ }
+ }
+ else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
+ strcmp (ucl_object_key (elt), "maximum") == 0) {
+ constraint = ucl_object_todouble (elt);
+ test = ucl_object_lookup (schema, "exclusiveMaximum");
+ if (test && test->type == UCL_BOOLEAN) {
+ exclusive = ucl_object_toboolean (test);
+ }
+ val = ucl_object_todouble (obj);
+ if (val > constraint || (exclusive && val >= constraint)) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "number is too big: %.3f, maximum is: %.3f",
+ val, constraint);
+ ret = false;
+ break;
+ }
+ }
+ else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
+ strcmp (ucl_object_key (elt), "minimum") == 0) {
+ constraint = ucl_object_todouble (elt);
+ test = ucl_object_lookup (schema, "exclusiveMinimum");
+ if (test && test->type == UCL_BOOLEAN) {
+ exclusive = ucl_object_toboolean (test);
+ }
+ val = ucl_object_todouble (obj);
+ if (val < constraint || (exclusive && val <= constraint)) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "number is too small: %.3f, minimum is: %.3f",
+ val, constraint);
+ ret = false;
+ break;
+ }
+ }
+ }
+
+ return ret;
+}
+
+static bool
+ucl_schema_validate_string (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err)
+{
+ const ucl_object_t *elt;
+ ucl_object_iter_t iter = NULL;
+ bool ret = true;
+ int64_t constraint;
+#ifdef HAVE_REGEX_H
+ regex_t re;
+#endif
+
+ while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) {
+ if (elt->type == UCL_INT &&
+ strcmp (ucl_object_key (elt), "maxLength") == 0) {
+ constraint = ucl_object_toint (elt);
+ if (obj->len > constraint) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "string is too big: %u, maximum is: %" PRId64,
+ obj->len, constraint);
+ ret = false;
+ break;
+ }
+ }
+ else if (elt->type == UCL_INT &&
+ strcmp (ucl_object_key (elt), "minLength") == 0) {
+ constraint = ucl_object_toint (elt);
+ if (obj->len < constraint) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "string is too short: %u, minimum is: %" PRId64,
+ obj->len, constraint);
+ ret = false;
+ break;
+ }
+ }
+#ifdef HAVE_REGEX_H
+ else if (elt->type == UCL_STRING &&
+ strcmp (ucl_object_key (elt), "pattern") == 0) {
+ if (regcomp (&re, ucl_object_tostring (elt),
+ REG_EXTENDED | REG_NOSUB) != 0) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "cannot compile pattern %s", ucl_object_tostring (elt));
+ ret = false;
+ break;
+ }
+ if (regexec (&re, ucl_object_tostring (obj), 0, NULL, 0) != 0) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "string doesn't match regexp %s",
+ ucl_object_tostring (elt));
+ ret = false;
+ }
+ regfree (&re);
+ }
+#endif
+ }
+
+ return ret;
+}
+
+struct ucl_compare_node {
+ const ucl_object_t *obj;
+ TREE_ENTRY(ucl_compare_node) link;
+ struct ucl_compare_node *next;
+};
+
+typedef TREE_HEAD(_tree, ucl_compare_node) ucl_compare_tree_t;
+
+TREE_DEFINE(ucl_compare_node, link)
+
+static int
+ucl_schema_elt_compare (struct ucl_compare_node *n1, struct ucl_compare_node *n2)
+{
+ const ucl_object_t *o1 = n1->obj, *o2 = n2->obj;
+
+ return ucl_object_compare (o1, o2);
+}
+
+static bool
+ucl_schema_array_is_unique (const ucl_object_t *obj, struct ucl_schema_error *err)
+{
+ ucl_compare_tree_t tree = TREE_INITIALIZER (ucl_schema_elt_compare);
+ ucl_object_iter_t iter = NULL;
+ const ucl_object_t *elt;
+ struct ucl_compare_node *node, test, *nodes = NULL, *tmp;
+ bool ret = true;
+
+ while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) {
+ test.obj = elt;
+ node = TREE_FIND (&tree, ucl_compare_node, link, &test);
+ if (node != NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, elt,
+ "duplicate values detected while uniqueItems is true");
+ ret = false;
+ break;
+ }
+ node = calloc (1, sizeof (*node));
+ if (node == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_UNKNOWN, elt,
+ "cannot allocate tree node");
+ ret = false;
+ break;
+ }
+ node->obj = elt;
+ TREE_INSERT (&tree, ucl_compare_node, link, node);
+ LL_PREPEND (nodes, node);
+ }
+
+ LL_FOREACH_SAFE (nodes, node, tmp) {
+ free (node);
+ }
+
+ return ret;
+}
+
+static bool
+ucl_schema_validate_array (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err,
+ const ucl_object_t *root,
+ ucl_object_t *ext_ref)
+{
+ const ucl_object_t *elt, *it, *found, *additional_schema = NULL,
+ *first_unvalidated = NULL;
+ ucl_object_iter_t iter = NULL, piter = NULL;
+ bool ret = true, allow_additional = true, need_unique = false;
+ int64_t minmax;
+ unsigned int idx = 0;
+
+ while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) {
+ if (strcmp (ucl_object_key (elt), "items") == 0) {
+ if (elt->type == UCL_ARRAY) {
+ found = ucl_array_head (obj);
+ while (ret && (it = ucl_object_iterate (elt, &piter, true)) != NULL) {
+ if (found) {
+ ret = ucl_schema_validate (it, found, false, err,
+ root, ext_ref);
+ found = ucl_array_find_index (obj, ++idx);
+ }
+ }
+ if (found != NULL) {
+ /* The first element that is not validated */
+ first_unvalidated = found;
+ }
+ }
+ else if (elt->type == UCL_OBJECT) {
+ /* Validate all items using the specified schema */
+ while (ret && (it = ucl_object_iterate (obj, &piter, true)) != NULL) {
+ ret = ucl_schema_validate (elt, it, false, err, root,
+ ext_ref);
+ }
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "items attribute is invalid in schema");
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "additionalItems") == 0) {
+ if (elt->type == UCL_BOOLEAN) {
+ if (!ucl_object_toboolean (elt)) {
+ /* Deny additional fields completely */
+ allow_additional = false;
+ }
+ }
+ else if (elt->type == UCL_OBJECT) {
+ /* Define validator for additional fields */
+ additional_schema = elt;
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
+ "additionalItems attribute is invalid in schema");
+ ret = false;
+ break;
+ }
+ }
+ else if (elt->type == UCL_BOOLEAN &&
+ strcmp (ucl_object_key (elt), "uniqueItems") == 0) {
+ need_unique = ucl_object_toboolean (elt);
+ }
+ else if (strcmp (ucl_object_key (elt), "minItems") == 0
+ && ucl_object_toint_safe (elt, &minmax)) {
+ if (obj->len < minmax) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "array has not enough items: %u, minimum is: %u",
+ obj->len, (unsigned)minmax);
+ ret = false;
+ break;
+ }
+ }
+ else if (strcmp (ucl_object_key (elt), "maxItems") == 0
+ && ucl_object_toint_safe (elt, &minmax)) {
+ if (obj->len > minmax) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "array has too many items: %u, maximum is: %u",
+ obj->len, (unsigned)minmax);
+ ret = false;
+ break;
+ }
+ }
+ }
+
+ if (ret) {
+ /* Additional properties */
+ if (!allow_additional || additional_schema != NULL) {
+ if (first_unvalidated != NULL) {
+ if (!allow_additional) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "array has undefined item");
+ ret = false;
+ }
+ else if (additional_schema != NULL) {
+ elt = ucl_array_find_index (obj, idx);
+ while (elt) {
+ if (!ucl_schema_validate (additional_schema, elt, false,
+ err, root, ext_ref)) {
+ ret = false;
+ break;
+ }
+ elt = ucl_array_find_index (obj, idx ++);
+ }
+ }
+ }
+ }
+ /* Required properties */
+ if (ret && need_unique) {
+ ret = ucl_schema_array_is_unique (obj, err);
+ }
+ }
+
+ return ret;
+}
+
+/*
+ * Returns whether this object is allowed for this type
+ */
+static bool
+ucl_schema_type_is_allowed (const ucl_object_t *type, const ucl_object_t *obj,
+ struct ucl_schema_error *err)
+{
+ ucl_object_iter_t iter = NULL;
+ const ucl_object_t *elt;
+ const char *type_str;
+ ucl_type_t t;
+
+ if (type == NULL) {
+ /* Any type is allowed */
+ return true;
+ }
+
+ if (type->type == UCL_ARRAY) {
+ /* One of allowed types */
+ while ((elt = ucl_object_iterate (type, &iter, true)) != NULL) {
+ if (ucl_schema_type_is_allowed (elt, obj, err)) {
+ return true;
+ }
+ }
+ }
+ else if (type->type == UCL_STRING) {
+ type_str = ucl_object_tostring (type);
+ if (!ucl_object_string_to_type (type_str, &t)) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, type,
+ "Type attribute is invalid in schema");
+ return false;
+ }
+ if (obj->type != t) {
+ /* Some types are actually compatible */
+ if (obj->type == UCL_TIME && t == UCL_FLOAT) {
+ return true;
+ }
+ else if (obj->type == UCL_INT && t == UCL_FLOAT) {
+ return true;
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_TYPE_MISMATCH, obj,
+ "Invalid type of %s, expected %s",
+ ucl_object_type_to_string (obj->type),
+ ucl_object_type_to_string (t));
+ }
+ }
+ else {
+ /* Types are equal */
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/*
+ * Check if object is equal to one of elements of enum
+ */
+static bool
+ucl_schema_validate_enum (const ucl_object_t *en, const ucl_object_t *obj,
+ struct ucl_schema_error *err)
+{
+ ucl_object_iter_t iter = NULL;
+ const ucl_object_t *elt;
+ bool ret = false;
+
+ while ((elt = ucl_object_iterate (en, &iter, true)) != NULL) {
+ if (ucl_object_compare (elt, obj) == 0) {
+ ret = true;
+ break;
+ }
+ }
+
+ if (!ret) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object is not one of enumerated patterns");
+ }
+
+ return ret;
+}
+
+
+/*
+ * Check a single ref component
+ */
+static const ucl_object_t *
+ucl_schema_resolve_ref_component (const ucl_object_t *cur,
+ const char *refc, int len,
+ struct ucl_schema_error *err)
+{
+ const ucl_object_t *res = NULL;
+ char *err_str;
+ int num, i;
+
+ if (cur->type == UCL_OBJECT) {
+ /* Find a key inside an object */
+ res = ucl_object_lookup_len (cur, refc, len);
+ if (res == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, cur,
+ "reference %s is invalid, missing path component", refc);
+ return NULL;
+ }
+ }
+ else if (cur->type == UCL_ARRAY) {
+ /* We must figure out a number inside array */
+ num = strtoul (refc, &err_str, 10);
+ if (err_str != NULL && *err_str != '/' && *err_str != '\0') {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, cur,
+ "reference %s is invalid, invalid item number", refc);
+ return NULL;
+ }
+ res = ucl_array_head (cur);
+ i = 0;
+ while (res != NULL) {
+ if (i == num) {
+ break;
+ }
+ res = res->next;
+ }
+ if (res == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, cur,
+ "reference %s is invalid, item number %d does not exist",
+ refc, num);
+ return NULL;
+ }
+ }
+ else {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, res,
+ "reference %s is invalid, contains primitive object in the path",
+ refc);
+ return NULL;
+ }
+
+ return res;
+}
+/*
+ * Find reference schema
+ */
+static const ucl_object_t *
+ucl_schema_resolve_ref (const ucl_object_t *root, const char *ref,
+ struct ucl_schema_error *err, ucl_object_t *ext_ref,
+ ucl_object_t const ** nroot)
+{
+ UT_string *url_err = NULL;
+ struct ucl_parser *parser;
+ const ucl_object_t *res = NULL, *ext_obj = NULL;
+ ucl_object_t *url_obj;
+ const char *p, *c, *hash_ptr = NULL;
+ char *url_copy = NULL;
+ unsigned char *url_buf;
+ size_t url_buflen;
+
+ if (ref[0] != '#') {
+ hash_ptr = strrchr (ref, '#');
+
+ if (hash_ptr) {
+ url_copy = malloc (hash_ptr - ref + 1);
+
+ if (url_copy == NULL) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INTERNAL_ERROR, root,
+ "cannot allocate memory");
+ return NULL;
+ }
+
+ ucl_strlcpy (url_copy, ref, hash_ptr - ref + 1);
+ p = url_copy;
+ }
+ else {
+ /* Full URL */
+ p = ref;
+ }
+
+ ext_obj = ucl_object_lookup (ext_ref, p);
+
+ if (ext_obj == NULL) {
+ if (ucl_strnstr (p, "://", strlen (p)) != NULL) {
+ if (!ucl_fetch_url (p, &url_buf, &url_buflen, &url_err, true)) {
+
+ ucl_schema_create_error (err,
+ UCL_SCHEMA_INVALID_SCHEMA,
+ root,
+ "cannot fetch reference %s: %s",
+ p,
+ url_err != NULL ? utstring_body (url_err)
+ : "unknown");
+ free (url_copy);
+
+ return NULL;
+ }
+ }
+ else {
+ if (!ucl_fetch_file (p, &url_buf, &url_buflen, &url_err,
+ true)) {
+ ucl_schema_create_error (err,
+ UCL_SCHEMA_INVALID_SCHEMA,
+ root,
+ "cannot fetch reference %s: %s",
+ p,
+ url_err != NULL ? utstring_body (url_err)
+ : "unknown");
+ free (url_copy);
+
+ return NULL;
+ }
+ }
+
+ parser = ucl_parser_new (0);
+
+ if (!ucl_parser_add_chunk (parser, url_buf, url_buflen)) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, root,
+ "cannot fetch reference %s: %s", p,
+ ucl_parser_get_error (parser));
+ ucl_parser_free (parser);
+ free (url_copy);
+
+ return NULL;
+ }
+
+ url_obj = ucl_parser_get_object (parser);
+ ext_obj = url_obj;
+ ucl_object_insert_key (ext_ref, url_obj, p, 0, true);
+ free (url_buf);
+ }
+
+ free (url_copy);
+
+ if (hash_ptr) {
+ p = hash_ptr + 1;
+ }
+ else {
+ p = "";
+ }
+ }
+ else {
+ p = ref + 1;
+ }
+
+ res = ext_obj != NULL ? ext_obj : root;
+ *nroot = res;
+
+ if (*p == '/') {
+ p++;
+ }
+ else if (*p == '\0') {
+ return res;
+ }
+
+ c = p;
+
+ while (*p != '\0') {
+ if (*p == '/') {
+ if (p - c == 0) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, res,
+ "reference %s is invalid, empty path component", ref);
+ return NULL;
+ }
+ /* Now we have some url part, so we need to figure out where we are */
+ res = ucl_schema_resolve_ref_component (res, c, p - c, err);
+ if (res == NULL) {
+ return NULL;
+ }
+ c = p + 1;
+ }
+ p ++;
+ }
+
+ if (p - c != 0) {
+ res = ucl_schema_resolve_ref_component (res, c, p - c, err);
+ }
+
+ if (res == NULL || res->type != UCL_OBJECT) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, res,
+ "reference %s is invalid, cannot find specified object",
+ ref);
+ return NULL;
+ }
+
+ return res;
+}
+
+static bool
+ucl_schema_validate_values (const ucl_object_t *schema, const ucl_object_t *obj,
+ struct ucl_schema_error *err)
+{
+ const ucl_object_t *elt, *cur;
+ int64_t constraint, i;
+
+ elt = ucl_object_lookup (schema, "maxValues");
+ if (elt != NULL && elt->type == UCL_INT) {
+ constraint = ucl_object_toint (elt);
+ cur = obj;
+ i = 0;
+ while (cur) {
+ if (i > constraint) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object has more values than defined: %ld",
+ (long int)constraint);
+ return false;
+ }
+ i ++;
+ cur = cur->next;
+ }
+ }
+ elt = ucl_object_lookup (schema, "minValues");
+ if (elt != NULL && elt->type == UCL_INT) {
+ constraint = ucl_object_toint (elt);
+ cur = obj;
+ i = 0;
+ while (cur) {
+ if (i >= constraint) {
+ break;
+ }
+ i ++;
+ cur = cur->next;
+ }
+ if (i < constraint) {
+ ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
+ "object has less values than defined: %ld",
+ (long int)constraint);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool
+ucl_schema_validate (const ucl_object_t *schema,
+ const ucl_object_t *obj, bool try_array,
+ struct ucl_schema_error *err,
+ const ucl_object_t *root,
+ ucl_object_t *external_refs)
+{
+ const ucl_object_t *elt, *cur, *ref_root;
+ ucl_object_iter_t iter = NULL;
+ bool ret;
+
+ if (schema->type != UCL_OBJECT) {
+ ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, schema,
+ "schema is %s instead of object",
+ ucl_object_type_to_string (schema->type));
+ return false;
+ }
+
+ if (try_array) {
+ /*
+ * Special case for multiple values
+ */
+ if (!ucl_schema_validate_values (schema, obj, err)) {
+ return false;
+ }
+ LL_FOREACH (obj, cur) {
+ if (!ucl_schema_validate (schema, cur, false, err, root, external_refs)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ elt = ucl_object_lookup (schema, "enum");
+ if (elt != NULL && elt->type == UCL_ARRAY) {
+ if (!ucl_schema_validate_enum (elt, obj, err)) {
+ return false;
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "allOf");
+ if (elt != NULL && elt->type == UCL_ARRAY) {
+ iter = NULL;
+ while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) {
+ ret = ucl_schema_validate (cur, obj, true, err, root, external_refs);
+ if (!ret) {
+ return false;
+ }
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "anyOf");
+ if (elt != NULL && elt->type == UCL_ARRAY) {
+ iter = NULL;
+ while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) {
+ ret = ucl_schema_validate (cur, obj, true, err, root, external_refs);
+ if (ret) {
+ break;
+ }
+ }
+ if (!ret) {
+ return false;
+ }
+ else {
+ /* Reset error */
+ err->code = UCL_SCHEMA_OK;
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "oneOf");
+ if (elt != NULL && elt->type == UCL_ARRAY) {
+ iter = NULL;
+ ret = false;
+ while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) {
+ if (!ret) {
+ ret = ucl_schema_validate (cur, obj, true, err, root, external_refs);
+ }
+ else if (ucl_schema_validate (cur, obj, true, err, root, external_refs)) {
+ ret = false;
+ break;
+ }
+ }
+ if (!ret) {
+ return false;
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "not");
+ if (elt != NULL && elt->type == UCL_OBJECT) {
+ if (ucl_schema_validate (elt, obj, true, err, root, external_refs)) {
+ return false;
+ }
+ else {
+ /* Reset error */
+ err->code = UCL_SCHEMA_OK;
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "$ref");
+ if (elt != NULL) {
+ ref_root = root;
+ cur = ucl_schema_resolve_ref (root, ucl_object_tostring (elt),
+ err, external_refs, &ref_root);
+
+ if (cur == NULL) {
+ return false;
+ }
+ if (!ucl_schema_validate (cur, obj, try_array, err, ref_root,
+ external_refs)) {
+ return false;
+ }
+ }
+
+ elt = ucl_object_lookup (schema, "type");
+ if (!ucl_schema_type_is_allowed (elt, obj, err)) {
+ return false;
+ }
+
+ switch (obj->type) {
+ case UCL_OBJECT:
+ return ucl_schema_validate_object (schema, obj, err, root, external_refs);
+ break;
+ case UCL_ARRAY:
+ return ucl_schema_validate_array (schema, obj, err, root, external_refs);
+ break;
+ case UCL_INT:
+ case UCL_FLOAT:
+ return ucl_schema_validate_number (schema, obj, err);
+ break;
+ case UCL_STRING:
+ return ucl_schema_validate_string (schema, obj, err);
+ break;
+ default:
+ break;
+ }
+
+ return true;
+}
+
+bool
+ucl_object_validate (const ucl_object_t *schema,
+ const ucl_object_t *obj, struct ucl_schema_error *err)
+{
+ return ucl_object_validate_root_ext (schema, obj, schema, NULL, err);
+}
+
+bool
+ucl_object_validate_root (const ucl_object_t *schema,
+ const ucl_object_t *obj,
+ const ucl_object_t *root,
+ struct ucl_schema_error *err)
+{
+ return ucl_object_validate_root_ext (schema, obj, root, NULL, err);
+}
+
+bool
+ucl_object_validate_root_ext (const ucl_object_t *schema,
+ const ucl_object_t *obj,
+ const ucl_object_t *root,
+ ucl_object_t *ext_refs,
+ struct ucl_schema_error *err)
+{
+ bool ret, need_unref = false;
+
+ if (ext_refs == NULL) {
+ ext_refs = ucl_object_typed_new (UCL_OBJECT);
+ need_unref = true;
+ }
+
+ ret = ucl_schema_validate (schema, obj, true, err, root, ext_refs);
+
+ if (need_unref) {
+ ucl_object_unref (ext_refs);
+ }
+
+ return ret;
+}
diff --git a/contrib/libucl/ucl_sexp.c b/contrib/libucl/ucl_sexp.c
new file mode 100644
index 0000000..1ad93d2
--- /dev/null
+++ b/contrib/libucl/ucl_sexp.c
@@ -0,0 +1,226 @@
+/*
+ * Copyright (c) 2015, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <ucl.h>
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "utlist.h"
+
+#define NEXT_STATE do { \
+if (p >= end) { \
+ if (state != read_ebrace) { \
+ ucl_create_err (&parser->err,\
+ "extra data");\
+ state = parse_err; \
+ } \
+} \
+else { \
+switch (*p) { \
+ case '(': \
+ state = read_obrace; \
+ break; \
+ case ')': \
+ state = read_ebrace; \
+ break; \
+ default: \
+ len = 0; \
+ mult = 1; \
+ state = read_length; \
+ break; \
+ } \
+} \
+} while(0)
+
+bool
+ucl_parse_csexp (struct ucl_parser *parser)
+{
+ const unsigned char *p, *end;
+ ucl_object_t *obj;
+ struct ucl_stack *st;
+ uint64_t len = 0, mult = 1;
+ enum {
+ start_parse,
+ read_obrace,
+ read_length,
+ read_value,
+ read_ebrace,
+ parse_err
+ } state = start_parse;
+
+ assert (parser != NULL);
+ assert (parser->chunks != NULL);
+ assert (parser->chunks->begin != NULL);
+ assert (parser->chunks->remain != 0);
+
+ p = parser->chunks->begin;
+ end = p + parser->chunks->remain;
+
+ while (p < end) {
+ switch (state) {
+ case start_parse:
+ /* At this point we expect open brace */
+ if (*p == '(') {
+ state = read_obrace;
+ }
+ else {
+ ucl_create_err (&parser->err, "bad starting character for "
+ "sexp block: %x", (int)*p);
+ state = parse_err;
+ }
+ break;
+
+ case read_obrace:
+ st = calloc (1, sizeof (*st));
+
+ if (st == NULL) {
+ ucl_create_err (&parser->err, "no memory");
+ state = parse_err;
+ continue;
+ }
+
+ st->obj = ucl_object_typed_new (UCL_ARRAY);
+
+ if (st->obj == NULL) {
+ ucl_create_err (&parser->err, "no memory");
+ state = parse_err;
+ free (st);
+ continue;
+ }
+
+ if (parser->stack == NULL) {
+ /* We have no stack */
+ parser->stack = st;
+
+ if (parser->top_obj == NULL) {
+ parser->top_obj = st->obj;
+ }
+ }
+ else {
+ /* Prepend new element to the stack */
+ LL_PREPEND (parser->stack, st);
+ }
+
+ p ++;
+ NEXT_STATE;
+
+ break;
+
+ case read_length:
+ if (*p == ':') {
+ if (len == 0) {
+ ucl_create_err (&parser->err, "zero length element");
+ state = parse_err;
+ continue;
+ }
+
+ state = read_value;
+ }
+ else if (*p >= '0' && *p <= '9') {
+ len += (*p - '0') * mult;
+ mult *= 10;
+
+ if (len > UINT32_MAX) {
+ ucl_create_err (&parser->err, "too big length of an "
+ "element");
+ state = parse_err;
+ continue;
+ }
+ }
+ else {
+ ucl_create_err (&parser->err, "bad length character: %x",
+ (int)*p);
+ state = parse_err;
+ continue;
+ }
+
+ p ++;
+ break;
+
+ case read_value:
+ if ((uint64_t)(end - p) > len || len == 0) {
+ ucl_create_err (&parser->err, "invalid length: %llu, %ld "
+ "remain", (long long unsigned)len, (long)(end - p));
+ state = parse_err;
+ continue;
+ }
+ obj = ucl_object_typed_new (UCL_STRING);
+
+ obj->value.sv = (const char*)p;
+ obj->len = len;
+ obj->flags |= UCL_OBJECT_BINARY;
+
+ if (!(parser->flags & UCL_PARSER_ZEROCOPY)) {
+ ucl_copy_value_trash (obj);
+ }
+
+ ucl_array_append (parser->stack->obj, obj);
+ p += len;
+ NEXT_STATE;
+ break;
+
+ case read_ebrace:
+ if (parser->stack == NULL) {
+ /* We have an extra end brace */
+ ucl_create_err (&parser->err, "invalid length: %llu, %ld "
+ "remain", (long long unsigned)len, (long)(end - p));
+ state = parse_err;
+ continue;
+ }
+ /* Pop the container */
+ st = parser->stack;
+ parser->stack = st->next;
+
+ if (parser->stack->obj->type == UCL_ARRAY) {
+ ucl_array_append (parser->stack->obj, st->obj);
+ }
+ else {
+ ucl_create_err (&parser->err, "bad container object, array "
+ "expected");
+ state = parse_err;
+ continue;
+ }
+
+ free (st);
+ st = NULL;
+ p++;
+ NEXT_STATE;
+ break;
+
+ case parse_err:
+ default:
+ return false;
+ }
+ }
+
+ if (state != read_ebrace) {
+ ucl_create_err (&parser->err, "invalid finishing state: %d", state);
+ return false;
+ }
+
+ return true;
+}
diff --git a/contrib/libucl/ucl_util.c b/contrib/libucl/ucl_util.c
new file mode 100644
index 0000000..3f2483c
--- /dev/null
+++ b/contrib/libucl/ucl_util.c
@@ -0,0 +1,3952 @@
+/* Copyright (c) 2013, Vsevolod Stakhov
+ * Copyright (c) 2015 Allan Jude <allanjude@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ucl.h"
+#include "ucl_internal.h"
+#include "ucl_chartable.h"
+#include "kvec.h"
+#include <limits.h>
+#include <stdarg.h>
+#include <stdio.h> /* for snprintf */
+
+#ifndef _WIN32
+#include <glob.h>
+#include <sys/param.h>
+#else
+#ifndef NBBY
+#define NBBY 8
+#endif
+#endif
+
+#ifdef HAVE_LIBGEN_H
+#ifndef _WIN32
+# include <libgen.h> /* For dirname */
+#endif
+#endif
+
+typedef kvec_t(ucl_object_t *) ucl_array_t;
+
+#define UCL_ARRAY_GET(ar, obj) ucl_array_t *ar = \
+ (ucl_array_t *)((obj) != NULL ? (obj)->value.av : NULL)
+
+#ifdef HAVE_OPENSSL
+#include <openssl/err.h>
+#include <openssl/sha.h>
+#include <openssl/rsa.h>
+#include <openssl/ssl.h>
+#include <openssl/evp.h>
+#endif
+
+#ifdef CURL_FOUND
+/* Seems to be broken */
+#define CURL_DISABLE_TYPECHECK 1
+#include <curl/curl.h>
+#endif
+#ifdef HAVE_FETCH_H
+#include <fetch.h>
+#endif
+
+#if defined(_MSC_VER)
+#include <windows.h>
+#include <io.h>
+#include <direct.h>
+
+#ifndef PROT_READ
+#define PROT_READ 1
+#endif
+#ifndef PROT_WRITE
+#define PROT_WRITE 2
+#endif
+#ifndef PROT_READWRITE
+#define PROT_READWRITE 3
+#endif
+#ifndef MAP_SHARED
+#define MAP_SHARED 1
+#endif
+#ifndef MAP_PRIVATE
+#define MAP_PRIVATE 2
+#endif
+#ifndef MAP_FAILED
+#define MAP_FAILED ((void *) -1)
+#endif
+
+#define getcwd _getcwd
+#define open _open
+#define close _close
+
+static void *ucl_mmap(char *addr, size_t length, int prot, int access, int fd, off_t offset)
+{
+ void *map = NULL;
+ HANDLE handle = INVALID_HANDLE_VALUE;
+
+ switch (prot) {
+ default:
+ case PROT_READ:
+ {
+ handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READONLY, 0, length, 0);
+ if (!handle) break;
+ map = (void *) MapViewOfFile(handle, FILE_MAP_READ, 0, 0, length);
+ CloseHandle(handle);
+ break;
+ }
+ case PROT_WRITE:
+ {
+ handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0);
+ if (!handle) break;
+ map = (void *) MapViewOfFile(handle, FILE_MAP_WRITE, 0, 0, length);
+ CloseHandle(handle);
+ break;
+ }
+ case PROT_READWRITE:
+ {
+ handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0);
+ if (!handle) break;
+ map = (void *) MapViewOfFile(handle, FILE_MAP_ALL_ACCESS, 0, 0, length);
+ CloseHandle(handle);
+ break;
+ }
+ }
+ if (map == (void *) NULL) {
+ return (void *) MAP_FAILED;
+ }
+ return (void *) ((char *) map + offset);
+}
+
+static int ucl_munmap(void *map,size_t length)
+{
+ if (!UnmapViewOfFile(map)) {
+ return(-1);
+ }
+ return(0);
+}
+
+static char* ucl_realpath(const char *path, char *resolved_path)
+{
+ char *p;
+ char tmp[MAX_PATH + 1];
+ strncpy(tmp, path, sizeof(tmp)-1);
+ p = tmp;
+ while(*p) {
+ if (*p == '/') *p = '\\';
+ p++;
+ }
+ return _fullpath(resolved_path, tmp, MAX_PATH);
+}
+
+
+char *dirname(char *path)
+{
+ static char path_buffer[_MAX_PATH];
+ char drive[_MAX_DRIVE];
+ char dir[_MAX_DIR];
+ char fname[_MAX_FNAME];
+ char ext[_MAX_EXT];
+
+ _splitpath (path, drive, dir, fname, ext);
+ _makepath(path_buffer, drive, dir, NULL, NULL);
+
+ return path_buffer;
+}
+
+char *basename(char *path)
+{
+ static char path_buffer[_MAX_PATH];
+ char drive[_MAX_DRIVE];
+ char dir[_MAX_DIR];
+ char fname[_MAX_FNAME];
+ char ext[_MAX_EXT];
+
+ _splitpath(path, drive, dir, fname, ext);
+ _makepath(path_buffer, NULL, NULL, fname, ext);
+
+ return path_buffer;
+}
+#else
+#define ucl_mmap mmap
+#define ucl_munmap munmap
+#define ucl_realpath realpath
+#endif
+
+typedef void (*ucl_object_dtor) (ucl_object_t *obj);
+static void ucl_object_free_internal (ucl_object_t *obj, bool allow_rec,
+ ucl_object_dtor dtor);
+static void ucl_object_dtor_unref (ucl_object_t *obj);
+
+static void
+ucl_object_dtor_free (ucl_object_t *obj)
+{
+ if (obj->trash_stack[UCL_TRASH_KEY] != NULL) {
+ UCL_FREE (obj->hh.keylen, obj->trash_stack[UCL_TRASH_KEY]);
+ }
+ if (obj->trash_stack[UCL_TRASH_VALUE] != NULL) {
+ UCL_FREE (obj->len, obj->trash_stack[UCL_TRASH_VALUE]);
+ }
+ /* Do not free ephemeral objects */
+ if ((obj->flags & UCL_OBJECT_EPHEMERAL) == 0) {
+ if (obj->type != UCL_USERDATA) {
+ UCL_FREE (sizeof (ucl_object_t), obj);
+ }
+ else {
+ struct ucl_object_userdata *ud = (struct ucl_object_userdata *)obj;
+ if (ud->dtor) {
+ ud->dtor (obj->value.ud);
+ }
+ UCL_FREE (sizeof (*ud), obj);
+ }
+ }
+}
+
+/*
+ * This is a helper function that performs exactly the same as
+ * `ucl_object_unref` but it doesn't iterate over elements allowing
+ * to use it for individual elements of arrays and multiple values
+ */
+static void
+ucl_object_dtor_unref_single (ucl_object_t *obj)
+{
+ if (obj != NULL) {
+#ifdef HAVE_ATOMIC_BUILTINS
+ unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1);
+ if (rc == 0) {
+#else
+ if (--obj->ref == 0) {
+#endif
+ ucl_object_free_internal (obj, false, ucl_object_dtor_unref);
+ }
+ }
+}
+
+static void
+ucl_object_dtor_unref (ucl_object_t *obj)
+{
+ if (obj->ref == 0) {
+ ucl_object_dtor_free (obj);
+ }
+ else {
+ /* This may cause dtor unref being called one more time */
+ ucl_object_dtor_unref_single (obj);
+ }
+}
+
+static void
+ucl_object_free_internal (ucl_object_t *obj, bool allow_rec, ucl_object_dtor dtor)
+{
+ ucl_object_t *tmp, *sub;
+
+ while (obj != NULL) {
+ if (obj->type == UCL_ARRAY) {
+ UCL_ARRAY_GET (vec, obj);
+ unsigned int i;
+
+ if (vec != NULL) {
+ for (i = 0; i < vec->n; i ++) {
+ sub = kv_A (*vec, i);
+ if (sub != NULL) {
+ tmp = sub;
+ while (sub) {
+ tmp = sub->next;
+ dtor (sub);
+ sub = tmp;
+ }
+ }
+ }
+ kv_destroy (*vec);
+ UCL_FREE (sizeof (*vec), vec);
+ }
+ obj->value.av = NULL;
+ }
+ else if (obj->type == UCL_OBJECT) {
+ if (obj->value.ov != NULL) {
+ ucl_hash_destroy (obj->value.ov, (ucl_hash_free_func)dtor);
+ }
+ obj->value.ov = NULL;
+ }
+ tmp = obj->next;
+ dtor (obj);
+ obj = tmp;
+
+ if (!allow_rec) {
+ break;
+ }
+ }
+}
+
+void
+ucl_object_free (ucl_object_t *obj)
+{
+ ucl_object_free_internal (obj, true, ucl_object_dtor_free);
+}
+
+size_t
+ucl_unescape_json_string (char *str, size_t len)
+{
+ char *t = str, *h = str;
+ int i, uval;
+
+ if (len <= 1) {
+ return len;
+ }
+ /* t is target (tortoise), h is source (hare) */
+
+ while (len) {
+ if (*h == '\\') {
+ h ++;
+
+ if (len == 1) {
+ /*
+ * If \ is last, then do not try to go further
+ * Issue: #74
+ */
+ len --;
+ *t++ = '\\';
+ continue;
+ }
+
+ switch (*h) {
+ case 'n':
+ *t++ = '\n';
+ break;
+ case 'r':
+ *t++ = '\r';
+ break;
+ case 'b':
+ *t++ = '\b';
+ break;
+ case 't':
+ *t++ = '\t';
+ break;
+ case 'f':
+ *t++ = '\f';
+ break;
+ case '\\':
+ *t++ = '\\';
+ break;
+ case '"':
+ *t++ = '"';
+ break;
+ case 'u':
+ /* Unicode escape */
+ uval = 0;
+ h ++; /* u character */
+ len --;
+
+ if (len > 3) {
+ for (i = 0; i < 4; i++) {
+ uval <<= 4;
+ if (isdigit (h[i])) {
+ uval += h[i] - '0';
+ }
+ else if (h[i] >= 'a' && h[i] <= 'f') {
+ uval += h[i] - 'a' + 10;
+ }
+ else if (h[i] >= 'A' && h[i] <= 'F') {
+ uval += h[i] - 'A' + 10;
+ }
+ else {
+ break;
+ }
+ }
+
+ /* Encode */
+ if(uval < 0x80) {
+ t[0] = (char)uval;
+ t ++;
+ }
+ else if(uval < 0x800) {
+ t[0] = 0xC0 + ((uval & 0x7C0) >> 6);
+ t[1] = 0x80 + ((uval & 0x03F));
+ t += 2;
+ }
+ else if(uval < 0x10000) {
+ t[0] = 0xE0 + ((uval & 0xF000) >> 12);
+ t[1] = 0x80 + ((uval & 0x0FC0) >> 6);
+ t[2] = 0x80 + ((uval & 0x003F));
+ t += 3;
+ }
+#if 0
+ /* It's not actually supported now */
+ else if(uval <= 0x10FFFF) {
+ t[0] = 0xF0 + ((uval & 0x1C0000) >> 18);
+ t[1] = 0x80 + ((uval & 0x03F000) >> 12);
+ t[2] = 0x80 + ((uval & 0x000FC0) >> 6);
+ t[3] = 0x80 + ((uval & 0x00003F));
+ t += 4;
+ }
+#endif
+ else {
+ *t++ = '?';
+ }
+
+ /* Consume 4 characters of source */
+ h += 4;
+ len -= 4;
+
+ if (len > 0) {
+ len --; /* for '\' character */
+ }
+ continue;
+ }
+ else {
+ *t++ = 'u';
+ }
+ break;
+ default:
+ *t++ = *h;
+ break;
+ }
+ h ++;
+ len --;
+ }
+ else {
+ *t++ = *h++;
+ }
+
+ if (len > 0) {
+ len --;
+ }
+ }
+ *t = '\0';
+
+ return (t - str);
+}
+
+size_t
+ucl_unescape_squoted_string (char *str, size_t len)
+{
+ char *t = str, *h = str;
+
+ if (len <= 1) {
+ return len;
+ }
+
+ /* t is target (tortoise), h is source (hare) */
+
+ while (len) {
+ if (*h == '\\') {
+ h ++;
+
+ if (len == 1) {
+ /*
+ * If \ is last, then do not try to go further
+ * Issue: #74
+ */
+ len --;
+ *t++ = '\\';
+ continue;
+ }
+
+ switch (*h) {
+ case '\'':
+ *t++ = '\'';
+ break;
+ case '\n':
+ /* Ignore \<newline> style stuff */
+ break;
+ case '\r':
+ /* Ignore \r and the following \n if needed */
+ if (len > 1 && h[1] == '\n') {
+ h ++;
+ len --;
+ }
+ break;
+ default:
+ /* Ignore \ */
+ *t++ = '\\';
+ *t++ = *h;
+ break;
+ }
+
+ h ++;
+ len --;
+ }
+ else {
+ *t++ = *h++;
+ }
+
+ if (len > 0) {
+ len --;
+ }
+ }
+
+ *t = '\0';
+
+ return (t - str);
+}
+
+char *
+ucl_copy_key_trash (const ucl_object_t *obj)
+{
+ ucl_object_t *deconst;
+
+ if (obj == NULL) {
+ return NULL;
+ }
+ if (obj->trash_stack[UCL_TRASH_KEY] == NULL && obj->key != NULL) {
+ deconst = __DECONST (ucl_object_t *, obj);
+ deconst->trash_stack[UCL_TRASH_KEY] = malloc (obj->keylen + 1);
+ if (deconst->trash_stack[UCL_TRASH_KEY] != NULL) {
+ memcpy (deconst->trash_stack[UCL_TRASH_KEY], obj->key, obj->keylen);
+ deconst->trash_stack[UCL_TRASH_KEY][obj->keylen] = '\0';
+ }
+ deconst->key = obj->trash_stack[UCL_TRASH_KEY];
+ deconst->flags |= UCL_OBJECT_ALLOCATED_KEY;
+ }
+
+ return obj->trash_stack[UCL_TRASH_KEY];
+}
+
+void
+ucl_chunk_free (struct ucl_chunk *chunk)
+{
+ if (chunk) {
+ struct ucl_parser_special_handler_chain *chain, *tmp;
+
+ LL_FOREACH_SAFE (chunk->special_handlers, chain, tmp) {
+ if (chain->special_handler->free_function) {
+ chain->special_handler->free_function (
+ chain->begin,
+ chain->len,
+ chain->special_handler->user_data);
+ } else {
+ UCL_FREE (chain->len, chain->begin);
+ }
+
+ UCL_FREE (sizeof (*chain), chain);
+ }
+
+ chunk->special_handlers = NULL;
+
+ if (chunk->fname) {
+ free (chunk->fname);
+ }
+
+ UCL_FREE (sizeof (*chunk), chunk);
+ }
+}
+
+char *
+ucl_copy_value_trash (const ucl_object_t *obj)
+{
+ ucl_object_t *deconst;
+
+ if (obj == NULL) {
+ return NULL;
+ }
+ if (obj->trash_stack[UCL_TRASH_VALUE] == NULL) {
+ deconst = __DECONST (ucl_object_t *, obj);
+ if (obj->type == UCL_STRING) {
+
+ /* Special case for strings */
+ if (obj->flags & UCL_OBJECT_BINARY) {
+ deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len);
+ if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
+ memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
+ obj->value.sv,
+ obj->len);
+ deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
+ }
+ }
+ else {
+ deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len + 1);
+ if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
+ memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
+ obj->value.sv,
+ obj->len);
+ deconst->trash_stack[UCL_TRASH_VALUE][obj->len] = '\0';
+ deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
+ }
+ }
+ }
+ else {
+ /* Just emit value in json notation */
+ deconst->trash_stack[UCL_TRASH_VALUE] = ucl_object_emit_single_json (obj);
+ deconst->len = strlen (obj->trash_stack[UCL_TRASH_VALUE]);
+ }
+ deconst->flags |= UCL_OBJECT_ALLOCATED_VALUE;
+ }
+
+ return obj->trash_stack[UCL_TRASH_VALUE];
+}
+
+ucl_object_t*
+ucl_parser_get_object (struct ucl_parser *parser)
+{
+ if (parser->state != UCL_STATE_ERROR && parser->top_obj != NULL) {
+ return ucl_object_ref (parser->top_obj);
+ }
+
+ return NULL;
+}
+
+void
+ucl_parser_free (struct ucl_parser *parser)
+{
+ struct ucl_stack *stack, *stmp;
+ struct ucl_macro *macro, *mtmp;
+ struct ucl_chunk *chunk, *ctmp;
+ struct ucl_pubkey *key, *ktmp;
+ struct ucl_variable *var, *vtmp;
+ ucl_object_t *tr, *trtmp;
+
+ if (parser == NULL) {
+ return;
+ }
+
+ if (parser->top_obj != NULL) {
+ ucl_object_unref (parser->top_obj);
+ }
+
+ if (parser->includepaths != NULL) {
+ ucl_object_unref (parser->includepaths);
+ }
+
+ LL_FOREACH_SAFE (parser->stack, stack, stmp) {
+ free (stack);
+ }
+ HASH_ITER (hh, parser->macroes, macro, mtmp) {
+ free (macro->name);
+ HASH_DEL (parser->macroes, macro);
+ UCL_FREE (sizeof (struct ucl_macro), macro);
+ }
+ LL_FOREACH_SAFE (parser->chunks, chunk, ctmp) {
+ ucl_chunk_free (chunk);
+ }
+ LL_FOREACH_SAFE (parser->keys, key, ktmp) {
+ UCL_FREE (sizeof (struct ucl_pubkey), key);
+ }
+ LL_FOREACH_SAFE (parser->variables, var, vtmp) {
+ free (var->value);
+ free (var->var);
+ UCL_FREE (sizeof (struct ucl_variable), var);
+ }
+ LL_FOREACH_SAFE (parser->trash_objs, tr, trtmp) {
+ ucl_object_free_internal (tr, false, ucl_object_dtor_free);
+ }
+
+ if (parser->err != NULL) {
+ utstring_free (parser->err);
+ }
+
+ if (parser->cur_file) {
+ free (parser->cur_file);
+ }
+
+ if (parser->comments) {
+ ucl_object_unref (parser->comments);
+ }
+
+ UCL_FREE (sizeof (struct ucl_parser), parser);
+}
+
+const char *
+ucl_parser_get_error(struct ucl_parser *parser)
+{
+ if (parser == NULL) {
+ return NULL;
+ }
+
+ if (parser->err == NULL) {
+ return NULL;
+ }
+
+ return utstring_body (parser->err);
+}
+
+int
+ucl_parser_get_error_code(struct ucl_parser *parser)
+{
+ if (parser == NULL) {
+ return 0;
+ }
+
+ return parser->err_code;
+}
+
+unsigned
+ucl_parser_get_column(struct ucl_parser *parser)
+{
+ if (parser == NULL || parser->chunks == NULL) {
+ return 0;
+ }
+
+ return parser->chunks->column;
+}
+
+unsigned
+ucl_parser_get_linenum(struct ucl_parser *parser)
+{
+ if (parser == NULL || parser->chunks == NULL) {
+ return 0;
+ }
+
+ return parser->chunks->line;
+}
+
+void
+ucl_parser_clear_error(struct ucl_parser *parser)
+{
+ if (parser != NULL && parser->err != NULL) {
+ utstring_free(parser->err);
+ parser->err = NULL;
+ parser->err_code = 0;
+ }
+}
+
+bool
+ucl_pubkey_add (struct ucl_parser *parser, const unsigned char *key, size_t len)
+{
+#ifndef HAVE_OPENSSL
+ ucl_create_err (&parser->err, "cannot check signatures without openssl");
+ return false;
+#else
+# if (OPENSSL_VERSION_NUMBER < 0x10000000L)
+ ucl_create_err (&parser->err, "cannot check signatures, openssl version is unsupported");
+ return EXIT_FAILURE;
+# else
+ struct ucl_pubkey *nkey;
+ BIO *mem;
+
+ mem = BIO_new_mem_buf ((void *)key, len);
+ nkey = UCL_ALLOC (sizeof (struct ucl_pubkey));
+ if (nkey == NULL) {
+ ucl_create_err (&parser->err, "cannot allocate memory for key");
+ return false;
+ }
+ nkey->key = PEM_read_bio_PUBKEY (mem, &nkey->key, NULL, NULL);
+ BIO_free (mem);
+ if (nkey->key == NULL) {
+ UCL_FREE (sizeof (struct ucl_pubkey), nkey);
+ ucl_create_err (&parser->err, "%s",
+ ERR_error_string (ERR_get_error (), NULL));
+ return false;
+ }
+ LL_PREPEND (parser->keys, nkey);
+# endif
+#endif
+ return true;
+}
+
+void ucl_parser_add_special_handler (struct ucl_parser *parser,
+ struct ucl_parser_special_handler *handler)
+{
+ LL_APPEND (parser->special_handlers, handler);
+}
+
+#ifdef CURL_FOUND
+struct ucl_curl_cbdata {
+ unsigned char *buf;
+ size_t buflen;
+};
+
+static size_t
+ucl_curl_write_callback (void* contents, size_t size, size_t nmemb, void* ud)
+{
+ struct ucl_curl_cbdata *cbdata = ud;
+ size_t realsize = size * nmemb;
+
+ cbdata->buf = realloc (cbdata->buf, cbdata->buflen + realsize + 1);
+ if (cbdata->buf == NULL) {
+ return 0;
+ }
+
+ memcpy (&(cbdata->buf[cbdata->buflen]), contents, realsize);
+ cbdata->buflen += realsize;
+ cbdata->buf[cbdata->buflen] = 0;
+
+ return realsize;
+}
+#endif
+
+/**
+ * Fetch a url and save results to the memory buffer
+ * @param url url to fetch
+ * @param len length of url
+ * @param buf target buffer
+ * @param buflen target length
+ * @return
+ */
+bool
+ucl_fetch_url (const unsigned char *url, unsigned char **buf, size_t *buflen,
+ UT_string **err, bool must_exist)
+{
+
+#ifdef HAVE_FETCH_H
+ struct url *fetch_url;
+ struct url_stat us;
+ FILE *in;
+
+ fetch_url = fetchParseURL (url);
+ if (fetch_url == NULL) {
+ ucl_create_err (err, "invalid URL %s: %s",
+ url, strerror (errno));
+ return false;
+ }
+ if ((in = fetchXGet (fetch_url, &us, "")) == NULL) {
+ if (!must_exist) {
+ ucl_create_err (err, "cannot fetch URL %s: %s",
+ url, strerror (errno));
+ }
+ fetchFreeURL (fetch_url);
+ return false;
+ }
+
+ *buflen = us.size;
+ *buf = malloc (*buflen);
+ if (*buf == NULL) {
+ ucl_create_err (err, "cannot allocate buffer for URL %s: %s",
+ url, strerror (errno));
+ fclose (in);
+ fetchFreeURL (fetch_url);
+ return false;
+ }
+
+ if (fread (*buf, *buflen, 1, in) != 1) {
+ ucl_create_err (err, "cannot read URL %s: %s",
+ url, strerror (errno));
+ fclose (in);
+ fetchFreeURL (fetch_url);
+ return false;
+ }
+
+ fetchFreeURL (fetch_url);
+ return true;
+#elif defined(CURL_FOUND)
+ CURL *curl;
+ int r;
+ struct ucl_curl_cbdata cbdata;
+
+ curl = curl_easy_init ();
+ if (curl == NULL) {
+ ucl_create_err (err, "CURL interface is broken");
+ return false;
+ }
+ if ((r = curl_easy_setopt (curl, CURLOPT_URL, url)) != CURLE_OK) {
+ ucl_create_err (err, "invalid URL %s: %s",
+ url, curl_easy_strerror (r));
+ curl_easy_cleanup (curl);
+ return false;
+ }
+ curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, ucl_curl_write_callback);
+ cbdata.buf = NULL;
+ cbdata.buflen = 0;
+ curl_easy_setopt (curl, CURLOPT_WRITEDATA, &cbdata);
+
+ if ((r = curl_easy_perform (curl)) != CURLE_OK) {
+ if (!must_exist) {
+ ucl_create_err (err, "error fetching URL %s: %s",
+ url, curl_easy_strerror (r));
+ }
+ curl_easy_cleanup (curl);
+ if (cbdata.buf) {
+ free (cbdata.buf);
+ }
+ return false;
+ }
+ *buf = cbdata.buf;
+ *buflen = cbdata.buflen;
+
+ return true;
+#else
+ ucl_create_err (err, "URL support is disabled");
+ return false;
+#endif
+}
+
+/**
+ * Fetch a file and save results to the memory buffer
+ * @param filename filename to fetch
+ * @param len length of filename
+ * @param buf target buffer
+ * @param buflen target length
+ * @return
+ */
+bool
+ucl_fetch_file (const unsigned char *filename, unsigned char **buf, size_t *buflen,
+ UT_string **err, bool must_exist)
+{
+ int fd;
+ struct stat st;
+ if ((fd = open (filename, O_RDONLY)) == -1) {
+ ucl_create_err (err, "cannot open file %s: %s",
+ filename, strerror (errno));
+ return false;
+ }
+
+ if (fstat (fd, &st) == -1) {
+ if (must_exist || errno == EPERM) {
+ ucl_create_err (err, "cannot stat file %s: %s",
+ filename, strerror (errno));
+ }
+ close (fd);
+
+ return false;
+ }
+ if (!S_ISREG (st.st_mode)) {
+ if (must_exist) {
+ ucl_create_err (err, "file %s is not a regular file", filename);
+ }
+ close (fd);
+
+ return false;
+ }
+
+ if (st.st_size == 0) {
+ /* Do not map empty files */
+ *buf = NULL;
+ *buflen = 0;
+ }
+ else {
+ if ((*buf = ucl_mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
+ close(fd);
+ ucl_create_err(err, "cannot mmap file %s: %s",
+ filename, strerror(errno));
+ *buf = NULL;
+
+ return false;
+ }
+ *buflen = st.st_size;
+ }
+
+ close (fd);
+
+ return true;
+}
+
+
+#if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
+static inline bool
+ucl_sig_check (const unsigned char *data, size_t datalen,
+ const unsigned char *sig, size_t siglen, struct ucl_parser *parser)
+{
+ struct ucl_pubkey *key;
+ char dig[EVP_MAX_MD_SIZE];
+ unsigned int diglen;
+ EVP_PKEY_CTX *key_ctx;
+ EVP_MD_CTX *sign_ctx = NULL;
+
+ sign_ctx = EVP_MD_CTX_create ();
+
+ LL_FOREACH (parser->keys, key) {
+ key_ctx = EVP_PKEY_CTX_new (key->key, NULL);
+ if (key_ctx != NULL) {
+ if (EVP_PKEY_verify_init (key_ctx) <= 0) {
+ EVP_PKEY_CTX_free (key_ctx);
+ continue;
+ }
+ if (EVP_PKEY_CTX_set_rsa_padding (key_ctx, RSA_PKCS1_PADDING) <= 0) {
+ EVP_PKEY_CTX_free (key_ctx);
+ continue;
+ }
+ if (EVP_PKEY_CTX_set_signature_md (key_ctx, EVP_sha256 ()) <= 0) {
+ EVP_PKEY_CTX_free (key_ctx);
+ continue;
+ }
+ EVP_DigestInit (sign_ctx, EVP_sha256 ());
+ EVP_DigestUpdate (sign_ctx, data, datalen);
+ EVP_DigestFinal (sign_ctx, dig, &diglen);
+
+ if (EVP_PKEY_verify (key_ctx, sig, siglen, dig, diglen) == 1) {
+ EVP_MD_CTX_destroy (sign_ctx);
+ EVP_PKEY_CTX_free (key_ctx);
+ return true;
+ }
+
+ EVP_PKEY_CTX_free (key_ctx);
+ }
+ }
+
+ EVP_MD_CTX_destroy (sign_ctx);
+
+ return false;
+}
+#endif
+
+struct ucl_include_params {
+ bool check_signature;
+ bool must_exist;
+ bool use_glob;
+ bool use_prefix;
+ bool soft_fail;
+ bool allow_glob;
+ unsigned priority;
+ enum ucl_duplicate_strategy strat;
+ enum ucl_parse_type parse_type;
+ const char *prefix;
+ const char *target;
+};
+
+/**
+ * Include an url to configuration
+ * @param data
+ * @param len
+ * @param parser
+ * @param err
+ * @return
+ */
+static bool
+ucl_include_url (const unsigned char *data, size_t len,
+ struct ucl_parser *parser,
+ struct ucl_include_params *params)
+{
+
+ bool res;
+ unsigned char *buf = NULL;
+ size_t buflen = 0;
+ struct ucl_chunk *chunk;
+ char urlbuf[PATH_MAX];
+ int prev_state;
+
+ snprintf (urlbuf, sizeof (urlbuf), "%.*s", (int)len, data);
+
+ if (!ucl_fetch_url (urlbuf, &buf, &buflen, &parser->err, params->must_exist)) {
+ return !params->must_exist;
+ }
+
+ if (params->check_signature) {
+#if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
+ unsigned char *sigbuf = NULL;
+ size_t siglen = 0;
+ /* We need to check signature first */
+ snprintf (urlbuf, sizeof (urlbuf), "%.*s.sig", (int)len, data);
+ if (!ucl_fetch_url (urlbuf, &sigbuf, &siglen, &parser->err, true)) {
+ return false;
+ }
+ if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) {
+ ucl_create_err (&parser->err, "cannot verify url %s: %s",
+ urlbuf,
+ ERR_error_string (ERR_get_error (), NULL));
+ if (siglen > 0) {
+ ucl_munmap (sigbuf, siglen);
+ }
+ return false;
+ }
+ if (siglen > 0) {
+ ucl_munmap (sigbuf, siglen);
+ }
+#endif
+ }
+
+ prev_state = parser->state;
+ parser->state = UCL_STATE_INIT;
+
+ res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
+ params->strat, params->parse_type);
+ if (res == true) {
+ /* Remove chunk from the stack */
+ chunk = parser->chunks;
+ if (chunk != NULL) {
+ parser->chunks = chunk->next;
+ ucl_chunk_free (chunk);
+ }
+ }
+
+ parser->state = prev_state;
+ free (buf);
+
+ return res;
+}
+
+/**
+ * Include a single file to the parser
+ * @param data
+ * @param len
+ * @param parser
+ * @param check_signature
+ * @param must_exist
+ * @param allow_glob
+ * @param priority
+ * @return
+ */
+static bool
+ucl_include_file_single (const unsigned char *data, size_t len,
+ struct ucl_parser *parser, struct ucl_include_params *params)
+{
+ bool res;
+ struct ucl_chunk *chunk;
+ unsigned char *buf = NULL;
+ char *old_curfile, *ext;
+ size_t buflen = 0;
+ char filebuf[PATH_MAX], realbuf[PATH_MAX];
+ int prev_state;
+ struct ucl_variable *cur_var, *tmp_var, *old_curdir = NULL,
+ *old_filename = NULL;
+ ucl_object_t *nest_obj = NULL, *old_obj = NULL, *new_obj = NULL;
+ ucl_hash_t *container = NULL;
+ struct ucl_stack *st = NULL;
+
+ snprintf (filebuf, sizeof (filebuf), "%.*s", (int)len, data);
+ if (ucl_realpath (filebuf, realbuf) == NULL) {
+ if (params->soft_fail) {
+ return false;
+ }
+ if (!params->must_exist && errno != EPERM) {
+ return true;
+ }
+
+ ucl_create_err (&parser->err, "cannot open file %s: %s",
+ filebuf,
+ strerror (errno));
+ return false;
+ }
+
+ if (parser->cur_file && strcmp (realbuf, parser->cur_file) == 0) {
+ /* We are likely including the file itself */
+ if (params->soft_fail) {
+ return false;
+ }
+
+ ucl_create_err (&parser->err, "trying to include the file %s from itself",
+ realbuf);
+ return false;
+ }
+
+ if (!ucl_fetch_file (realbuf, &buf, &buflen, &parser->err, params->must_exist)) {
+ if (params->soft_fail) {
+ return false;
+ }
+
+ if (params->must_exist || parser->err != NULL) {
+ /* The case of fatal errors */
+ return false;
+ }
+
+ return true;
+ }
+
+ if (params->check_signature) {
+#if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
+ unsigned char *sigbuf = NULL;
+ size_t siglen = 0;
+ /* We need to check signature first */
+ snprintf (filebuf, sizeof (filebuf), "%s.sig", realbuf);
+ if (!ucl_fetch_file (filebuf, &sigbuf, &siglen, &parser->err, true)) {
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+ if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) {
+ ucl_create_err (&parser->err, "cannot verify file %s: %s",
+ filebuf,
+ ERR_error_string (ERR_get_error (), NULL));
+ if (sigbuf) {
+ ucl_munmap (sigbuf, siglen);
+ }
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+
+ if (sigbuf) {
+ ucl_munmap (sigbuf, siglen);
+ }
+#endif
+ }
+
+ old_curfile = parser->cur_file;
+ parser->cur_file = NULL;
+
+ /* Store old file vars */
+ DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) {
+ if (strcmp (cur_var->var, "CURDIR") == 0) {
+ old_curdir = cur_var;
+ DL_DELETE (parser->variables, cur_var);
+ }
+ else if (strcmp (cur_var->var, "FILENAME") == 0) {
+ old_filename = cur_var;
+ DL_DELETE (parser->variables, cur_var);
+ }
+ }
+
+ ucl_parser_set_filevars (parser, realbuf, false);
+
+ prev_state = parser->state;
+ parser->state = UCL_STATE_INIT;
+
+ if (params->use_prefix && params->prefix == NULL) {
+ /* Auto generate a key name based on the included filename */
+ params->prefix = basename (realbuf);
+ ext = strrchr (params->prefix, '.');
+ if (ext != NULL && (strcmp (ext, ".conf") == 0 || strcmp (ext, ".ucl") == 0)) {
+ /* Strip off .conf or .ucl */
+ *ext = '\0';
+ }
+ }
+ if (params->prefix != NULL) {
+ /* This is a prefixed include */
+ container = parser->stack->obj->value.ov;
+
+ old_obj = __DECONST (ucl_object_t *, ucl_hash_search (container,
+ params->prefix, strlen (params->prefix)));
+
+ if (strcasecmp (params->target, "array") == 0) {
+ if (old_obj == NULL) {
+ /* Create an array with key: prefix */
+ old_obj = ucl_object_new_full (UCL_ARRAY, params->priority);
+ old_obj->key = params->prefix;
+ old_obj->keylen = strlen (params->prefix);
+ ucl_copy_key_trash (old_obj);
+ old_obj->prev = old_obj;
+ old_obj->next = NULL;
+
+ container = ucl_hash_insert_object (container, old_obj,
+ parser->flags & UCL_PARSER_KEY_LOWERCASE);
+ parser->stack->obj->len++;
+
+ nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
+ nest_obj->prev = nest_obj;
+ nest_obj->next = NULL;
+
+ ucl_array_append (old_obj, nest_obj);
+ }
+ else {
+ if (ucl_object_type (old_obj) == UCL_ARRAY) {
+ /* Append to the existing array */
+ nest_obj = ucl_object_new_full (UCL_OBJECT,
+ params->priority);
+ if (nest_obj == NULL) {
+ ucl_create_err (&parser->err,
+ "cannot allocate memory for an object");
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+ nest_obj->prev = nest_obj;
+ nest_obj->next = NULL;
+
+ ucl_array_append (old_obj, nest_obj);
+ }
+ else {
+ /* Convert the object to an array */
+ new_obj = ucl_object_typed_new (UCL_ARRAY);
+ if (new_obj == NULL) {
+ ucl_create_err (&parser->err,
+ "cannot allocate memory for an object");
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+ new_obj->key = old_obj->key;
+ new_obj->keylen = old_obj->keylen;
+ new_obj->flags |= UCL_OBJECT_MULTIVALUE;
+ new_obj->prev = new_obj;
+ new_obj->next = NULL;
+
+ nest_obj = ucl_object_new_full (UCL_OBJECT,
+ params->priority);
+ if (nest_obj == NULL) {
+ ucl_create_err (&parser->err,
+ "cannot allocate memory for an object");
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ ucl_object_unref (new_obj);
+
+ return false;
+ }
+ nest_obj->prev = nest_obj;
+ nest_obj->next = NULL;
+
+ ucl_array_append (new_obj, old_obj);
+ ucl_array_append (new_obj, nest_obj);
+ ucl_hash_replace (container, old_obj, new_obj);
+ }
+ }
+ }
+ else {
+ /* Case of object */
+ if (old_obj == NULL) {
+ /* Create an object with key: prefix */
+ nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
+
+ if (nest_obj == NULL) {
+ ucl_create_err (&parser->err, "cannot allocate memory for an object");
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+
+ nest_obj->key = params->prefix;
+ nest_obj->keylen = strlen (params->prefix);
+ ucl_copy_key_trash(nest_obj);
+ nest_obj->prev = nest_obj;
+ nest_obj->next = NULL;
+
+ container = ucl_hash_insert_object (container, nest_obj,
+ parser->flags & UCL_PARSER_KEY_LOWERCASE);
+ parser->stack->obj->len ++;
+ }
+ else {
+ if (ucl_object_type (old_obj) == UCL_OBJECT) {
+ /* Append to existing Object*/
+ nest_obj = old_obj;
+ }
+ else {
+ /* The key is not an object */
+ ucl_create_err (&parser->err,
+ "Conflicting type for key: %s, asked %s, has %s",
+ params->prefix, params->target,
+ ucl_object_type_to_string (ucl_object_type (old_obj)));
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+ }
+ }
+
+
+ /* Put all of the content of the include inside that object */
+ parser->stack->obj->value.ov = container;
+
+ st = UCL_ALLOC (sizeof (struct ucl_stack));
+ if (st == NULL) {
+ ucl_create_err (&parser->err, "cannot allocate memory for an object");
+ ucl_object_unref (nest_obj);
+
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+ st->obj = nest_obj;
+ st->e.params.level = parser->stack->e.params.level;
+ st->e.params.flags = parser->stack->e.params.flags;
+ st->e.params.line = parser->stack->e.params.line;
+ st->chunk = parser->chunks;
+ LL_PREPEND (parser->stack, st);
+ parser->cur_obj = nest_obj;
+ }
+
+ res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
+ params->strat, params->parse_type);
+
+ if (res) {
+ /* Stop nesting the include, take 1 level off the stack */
+ if (params->prefix != NULL && nest_obj != NULL) {
+ parser->stack = st->next;
+ UCL_FREE (sizeof (struct ucl_stack), st);
+ }
+
+ /* Remove chunk from the stack */
+ chunk = parser->chunks;
+ if (chunk != NULL) {
+ parser->chunks = chunk->next;
+ ucl_chunk_free (chunk);
+ parser->recursion--;
+ }
+
+ /* Restore old file vars */
+ if (parser->cur_file) {
+ free (parser->cur_file);
+ }
+
+ parser->cur_file = old_curfile;
+ DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) {
+ if (strcmp (cur_var->var, "CURDIR") == 0 && old_curdir) {
+ DL_DELETE (parser->variables, cur_var);
+ free (cur_var->var);
+ free (cur_var->value);
+ UCL_FREE (sizeof (struct ucl_variable), cur_var);
+ } else if (strcmp (cur_var->var, "FILENAME") == 0 && old_filename) {
+ DL_DELETE (parser->variables, cur_var);
+ free (cur_var->var);
+ free (cur_var->value);
+ UCL_FREE (sizeof (struct ucl_variable), cur_var);
+ }
+ }
+ if (old_filename) {
+ DL_APPEND (parser->variables, old_filename);
+ }
+ if (old_curdir) {
+ DL_APPEND (parser->variables, old_curdir);
+ }
+
+ parser->state = prev_state;
+ }
+
+ if (buflen > 0) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return res;
+}
+
+/**
+ * Include a file to configuration
+ * @param data
+ * @param len
+ * @param parser
+ * @param err
+ * @return
+ */
+static bool
+ucl_include_file (const unsigned char *data, size_t len,
+ struct ucl_parser *parser,
+ struct ucl_include_params *params,
+ const ucl_object_t *args)
+{
+ const unsigned char *p = data, *end = data + len;
+ bool need_glob = false;
+ int cnt = 0;
+ char glob_pattern[PATH_MAX];
+ size_t i;
+
+#ifndef _WIN32
+ if (!params->allow_glob) {
+ return ucl_include_file_single (data, len, parser, params);
+ }
+ else {
+ /* Check for special symbols in a filename */
+ while (p != end) {
+ if (*p == '*' || *p == '?') {
+ need_glob = true;
+ break;
+ }
+ p ++;
+ }
+ if (need_glob) {
+ glob_t globbuf;
+ memset (&globbuf, 0, sizeof (globbuf));
+ ucl_strlcpy (glob_pattern, (const char *)data,
+ (len + 1 < sizeof (glob_pattern) ? len + 1 : sizeof (glob_pattern)));
+ if (glob (glob_pattern, 0, NULL, &globbuf) != 0) {
+ return (!params->must_exist || false);
+ }
+ for (i = 0; i < globbuf.gl_pathc; i ++) {
+
+ if (parser->include_trace_func) {
+ const ucl_object_t *parent = NULL;
+
+ if (parser->stack) {
+ parent = parser->stack->obj;
+ }
+
+ parser->include_trace_func (parser, parent, NULL,
+ globbuf.gl_pathv[i],
+ strlen (globbuf.gl_pathv[i]),
+ parser->include_trace_ud);
+ }
+
+ if (!ucl_include_file_single ((unsigned char *)globbuf.gl_pathv[i],
+ strlen (globbuf.gl_pathv[i]), parser, params)) {
+ if (params->soft_fail) {
+ continue;
+ }
+ globfree (&globbuf);
+ return false;
+ }
+ cnt ++;
+ }
+ globfree (&globbuf);
+
+ if (cnt == 0 && params->must_exist) {
+ ucl_create_err (&parser->err, "cannot match any files for pattern %s",
+ glob_pattern);
+ return false;
+ }
+ }
+ else {
+ return ucl_include_file_single (data, len, parser, params);
+ }
+ }
+#else
+ /* Win32 compilers do not support globbing. Therefore, for Win32,
+ treat allow_glob/need_glob as a NOOP and just return */
+ return ucl_include_file_single (data, len, parser, params);
+#endif
+
+ return true;
+}
+
+/**
+ * Common function to handle .*include* macros
+ * @param data
+ * @param len
+ * @param args
+ * @param parser
+ * @param default_try
+ * @param default_sign
+ * @return
+ */
+static bool
+ucl_include_common (const unsigned char *data, size_t len,
+ const ucl_object_t *args, struct ucl_parser *parser,
+ bool default_try,
+ bool default_sign)
+{
+ bool allow_url = false, search = false;
+ const char *duplicate;
+ const ucl_object_t *param;
+ ucl_object_iter_t it = NULL, ip = NULL;
+ char ipath[PATH_MAX];
+ struct ucl_include_params params;
+
+ /* Default values */
+ params.soft_fail = default_try;
+ params.allow_glob = false;
+ params.check_signature = default_sign;
+ params.use_prefix = false;
+ params.target = "object";
+ params.prefix = NULL;
+ params.priority = 0;
+ params.parse_type = UCL_PARSE_UCL;
+ params.strat = UCL_DUPLICATE_APPEND;
+ params.must_exist = !default_try;
+
+ if (parser->include_trace_func) {
+ const ucl_object_t *parent = NULL;
+
+ if (parser->stack) {
+ parent = parser->stack->obj;
+ }
+
+ parser->include_trace_func (parser, parent, args,
+ data, len, parser->include_trace_ud);
+ }
+
+ /* Process arguments */
+ if (args != NULL && args->type == UCL_OBJECT) {
+ while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
+ if (param->type == UCL_BOOLEAN) {
+ if (strncmp (param->key, "try", param->keylen) == 0) {
+ params.must_exist = !ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "sign", param->keylen) == 0) {
+ params.check_signature = ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "glob", param->keylen) == 0) {
+ params.allow_glob = ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "url", param->keylen) == 0) {
+ allow_url = ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "prefix", param->keylen) == 0) {
+ params.use_prefix = ucl_object_toboolean (param);
+ }
+ }
+ else if (param->type == UCL_STRING) {
+ if (strncmp (param->key, "key", param->keylen) == 0) {
+ params.prefix = ucl_object_tostring (param);
+ }
+ else if (strncmp (param->key, "target", param->keylen) == 0) {
+ params.target = ucl_object_tostring (param);
+ }
+ else if (strncmp (param->key, "duplicate", param->keylen) == 0) {
+ duplicate = ucl_object_tostring (param);
+
+ if (strcmp (duplicate, "append") == 0) {
+ params.strat = UCL_DUPLICATE_APPEND;
+ }
+ else if (strcmp (duplicate, "merge") == 0) {
+ params.strat = UCL_DUPLICATE_MERGE;
+ }
+ else if (strcmp (duplicate, "rewrite") == 0) {
+ params.strat = UCL_DUPLICATE_REWRITE;
+ }
+ else if (strcmp (duplicate, "error") == 0) {
+ params.strat = UCL_DUPLICATE_ERROR;
+ }
+ }
+ }
+ else if (param->type == UCL_ARRAY) {
+ if (strncmp (param->key, "path", param->keylen) == 0) {
+ ucl_set_include_path (parser, __DECONST(ucl_object_t *, param));
+ }
+ }
+ else if (param->type == UCL_INT) {
+ if (strncmp (param->key, "priority", param->keylen) == 0) {
+ params.priority = ucl_object_toint (param);
+ }
+ }
+ }
+ }
+
+ if (parser->includepaths == NULL) {
+ if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
+ /* Globbing is not used for URL's */
+ return ucl_include_url (data, len, parser, &params);
+ }
+ else if (data != NULL) {
+ /* Try to load a file */
+ return ucl_include_file (data, len, parser, &params, args);
+ }
+ }
+ else {
+ if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
+ /* Globbing is not used for URL's */
+ return ucl_include_url (data, len, parser, &params);
+ }
+
+ ip = ucl_object_iterate_new (parser->includepaths);
+ while ((param = ucl_object_iterate_safe (ip, true)) != NULL) {
+ if (ucl_object_type(param) == UCL_STRING) {
+ snprintf (ipath, sizeof (ipath), "%s/%.*s", ucl_object_tostring(param),
+ (int)len, data);
+ if ((search = ucl_include_file (ipath, strlen (ipath),
+ parser, &params, args))) {
+ if (!params.allow_glob) {
+ break;
+ }
+ }
+ }
+ }
+ ucl_object_iterate_free (ip);
+ if (search == true) {
+ return true;
+ }
+ else {
+ ucl_create_err (&parser->err,
+ "cannot find file: %.*s in search path",
+ (int)len, data);
+ return false;
+ }
+ }
+
+ return false;
+}
+
+/**
+ * Handle include macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool
+ucl_include_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud)
+{
+ struct ucl_parser *parser = ud;
+
+ return ucl_include_common (data, len, args, parser, false, false);
+}
+
+/**
+ * Handle includes macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool
+ucl_includes_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud)
+{
+ struct ucl_parser *parser = ud;
+
+ return ucl_include_common (data, len, args, parser, false, true);
+}
+
+/**
+ * Handle tryinclude macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool
+ucl_try_include_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud)
+{
+ struct ucl_parser *parser = ud;
+
+ return ucl_include_common (data, len, args, parser, true, false);
+}
+
+/**
+ * Handle priority macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool
+ucl_priority_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud)
+{
+ struct ucl_parser *parser = ud;
+ unsigned priority = 255;
+ const ucl_object_t *param;
+ bool found = false;
+ char *value = NULL, *leftover = NULL;
+ ucl_object_iter_t it = NULL;
+
+ if (parser == NULL) {
+ return false;
+ }
+
+ /* Process arguments */
+ if (args != NULL && args->type == UCL_OBJECT) {
+ while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
+ if (param->type == UCL_INT) {
+ if (strncmp (param->key, "priority", param->keylen) == 0) {
+ priority = ucl_object_toint (param);
+ found = true;
+ }
+ }
+ }
+ }
+
+ if (len > 0) {
+ value = malloc(len + 1);
+ ucl_strlcpy(value, (const char *)data, len + 1);
+ priority = strtol(value, &leftover, 10);
+ if (*leftover != '\0') {
+ ucl_create_err (&parser->err, "Invalid priority value in macro: %s",
+ value);
+ free(value);
+ return false;
+ }
+ free(value);
+ found = true;
+ }
+
+ if (found == true) {
+ parser->chunks->priority = priority;
+ return true;
+ }
+
+ ucl_create_err (&parser->err, "Unable to parse priority macro");
+ return false;
+}
+
+/**
+ * Handle load macro
+ * @param data include data
+ * @param len length of data
+ * @param args UCL object representing arguments to the macro
+ * @param ud user data
+ * @return
+ */
+bool
+ucl_load_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, void* ud)
+{
+ struct ucl_parser *parser = ud;
+ const ucl_object_t *param;
+ ucl_object_t *obj, *old_obj;
+ ucl_object_iter_t it = NULL;
+ bool try_load, multiline, test;
+ const char *target, *prefix;
+ char *load_file, *tmp;
+ unsigned char *buf;
+ size_t buflen;
+ unsigned priority;
+ int64_t iv;
+ ucl_object_t *container = NULL;
+ enum ucl_string_flags flags;
+
+ /* Default values */
+ try_load = false;
+ multiline = false;
+ test = false;
+ target = "string";
+ prefix = NULL;
+ load_file = NULL;
+ buf = NULL;
+ buflen = 0;
+ priority = 0;
+ obj = NULL;
+ old_obj = NULL;
+ flags = 0;
+
+ if (parser == NULL) {
+ return false;
+ }
+
+ /* Process arguments */
+ if (args != NULL && args->type == UCL_OBJECT) {
+ while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
+ if (param->type == UCL_BOOLEAN) {
+ if (strncmp (param->key, "try", param->keylen) == 0) {
+ try_load = ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "multiline", param->keylen) == 0) {
+ multiline = ucl_object_toboolean (param);
+ }
+ else if (strncmp (param->key, "escape", param->keylen) == 0) {
+ test = ucl_object_toboolean (param);
+ if (test) {
+ flags |= UCL_STRING_ESCAPE;
+ }
+ }
+ else if (strncmp (param->key, "trim", param->keylen) == 0) {
+ test = ucl_object_toboolean (param);
+ if (test) {
+ flags |= UCL_STRING_TRIM;
+ }
+ }
+ }
+ else if (param->type == UCL_STRING) {
+ if (strncmp (param->key, "key", param->keylen) == 0) {
+ prefix = ucl_object_tostring (param);
+ }
+ else if (strncmp (param->key, "target", param->keylen) == 0) {
+ target = ucl_object_tostring (param);
+ }
+ }
+ else if (param->type == UCL_INT) {
+ if (strncmp (param->key, "priority", param->keylen) == 0) {
+ priority = ucl_object_toint (param);
+ }
+ }
+ }
+ }
+
+ if (prefix == NULL || strlen (prefix) == 0) {
+ ucl_create_err (&parser->err, "No Key specified in load macro");
+ return false;
+ }
+
+ if (len > 0) {
+ load_file = malloc (len + 1);
+ if (!load_file) {
+ ucl_create_err (&parser->err, "cannot allocate memory for suffix");
+
+ return false;
+ }
+
+ snprintf (load_file, len + 1, "%.*s", (int)len, data);
+
+ if (!ucl_fetch_file (load_file, &buf, &buflen, &parser->err,
+ !try_load)) {
+ free (load_file);
+
+ return (try_load || false);
+ }
+
+ free (load_file);
+ container = parser->stack->obj;
+ old_obj = __DECONST (ucl_object_t *, ucl_object_lookup (container,
+ prefix));
+
+ if (old_obj != NULL) {
+ ucl_create_err (&parser->err, "Key %s already exists", prefix);
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+
+ if (strcasecmp (target, "string") == 0) {
+ obj = ucl_object_fromstring_common (buf, buflen, flags);
+ ucl_copy_value_trash (obj);
+ if (multiline) {
+ obj->flags |= UCL_OBJECT_MULTILINE;
+ }
+ }
+ else if (strcasecmp (target, "int") == 0) {
+ tmp = malloc (buflen + 1);
+
+ if (tmp == NULL) {
+ ucl_create_err (&parser->err, "Memory allocation failed");
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ return false;
+ }
+
+ snprintf (tmp, buflen + 1, "%.*s", (int)buflen, buf);
+ iv = strtoll (tmp, NULL, 10);
+ obj = ucl_object_fromint (iv);
+ free (tmp);
+ }
+
+ if (buf) {
+ ucl_munmap (buf, buflen);
+ }
+
+ if (obj != NULL) {
+ obj->key = prefix;
+ obj->keylen = strlen (prefix);
+ ucl_copy_key_trash (obj);
+ obj->prev = obj;
+ obj->next = NULL;
+ ucl_object_set_priority (obj, priority);
+ ucl_object_insert_key (container, obj, obj->key, obj->keylen, false);
+ }
+
+ return true;
+ }
+
+ ucl_create_err (&parser->err, "Unable to parse load macro");
+ return false;
+}
+
+bool
+ucl_inherit_handler (const unsigned char *data, size_t len,
+ const ucl_object_t *args, const ucl_object_t *ctx, void* ud)
+{
+ const ucl_object_t *parent, *cur;
+ ucl_object_t *target, *copy;
+ ucl_object_iter_t it = NULL;
+ bool replace = false;
+ struct ucl_parser *parser = ud;
+
+ parent = ucl_object_lookup_len (ctx, data, len);
+
+ /* Some sanity checks */
+ if (parent == NULL || ucl_object_type (parent) != UCL_OBJECT) {
+ ucl_create_err (&parser->err, "Unable to find inherited object %.*s",
+ (int)len, data);
+ return false;
+ }
+
+ if (parser->stack == NULL || parser->stack->obj == NULL ||
+ ucl_object_type (parser->stack->obj) != UCL_OBJECT) {
+ ucl_create_err (&parser->err, "Invalid inherit context");
+ return false;
+ }
+
+ target = parser->stack->obj;
+
+ if (args && (cur = ucl_object_lookup (args, "replace")) != NULL) {
+ replace = ucl_object_toboolean (cur);
+ }
+
+ while ((cur = ucl_object_iterate (parent, &it, true))) {
+ /* We do not replace existing keys */
+ if (!replace && ucl_object_lookup_len (target, cur->key, cur->keylen)) {
+ continue;
+ }
+
+ copy = ucl_object_copy (cur);
+
+ if (!replace) {
+ copy->flags |= UCL_OBJECT_INHERITED;
+ }
+
+ ucl_object_insert_key (target, copy, copy->key,
+ copy->keylen, false);
+ }
+
+ return true;
+}
+
+bool
+ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename, bool need_expand)
+{
+ char realbuf[PATH_MAX], *curdir;
+
+ if (filename != NULL) {
+ if (need_expand) {
+ if (ucl_realpath (filename, realbuf) == NULL) {
+ return false;
+ }
+ }
+ else {
+ ucl_strlcpy (realbuf, filename, sizeof (realbuf));
+ }
+
+ if (parser->cur_file) {
+ free (parser->cur_file);
+ }
+
+ parser->cur_file = strdup (realbuf);
+
+ /* Define variables */
+ ucl_parser_register_variable (parser, "FILENAME", realbuf);
+ curdir = dirname (realbuf);
+ ucl_parser_register_variable (parser, "CURDIR", curdir);
+ }
+ else {
+ /* Set everything from the current dir */
+ curdir = getcwd (realbuf, sizeof (realbuf));
+ ucl_parser_register_variable (parser, "FILENAME", "undef");
+ ucl_parser_register_variable (parser, "CURDIR", curdir);
+ }
+
+ return true;
+}
+
+bool
+ucl_parser_add_file_full (struct ucl_parser *parser, const char *filename,
+ unsigned priority, enum ucl_duplicate_strategy strat,
+ enum ucl_parse_type parse_type)
+{
+ unsigned char *buf;
+ size_t len;
+ bool ret;
+ char realbuf[PATH_MAX];
+
+ if (ucl_realpath (filename, realbuf) == NULL) {
+ ucl_create_err (&parser->err, "cannot open file %s: %s",
+ filename,
+ strerror (errno));
+ return false;
+ }
+
+ if (!ucl_fetch_file (realbuf, &buf, &len, &parser->err, true)) {
+ return false;
+ }
+
+ ucl_parser_set_filevars (parser, realbuf, false);
+ ret = ucl_parser_add_chunk_full (parser, buf, len, priority, strat,
+ parse_type);
+
+ if (len > 0) {
+ ucl_munmap (buf, len);
+ }
+
+ return ret;
+}
+
+bool
+ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename,
+ unsigned priority)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_file_full(parser, filename, priority,
+ UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
+}
+
+bool
+ucl_parser_add_file (struct ucl_parser *parser, const char *filename)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_file_full(parser, filename,
+ parser->default_priority, UCL_DUPLICATE_APPEND,
+ UCL_PARSE_UCL);
+}
+
+
+bool
+ucl_parser_add_fd_full (struct ucl_parser *parser, int fd,
+ unsigned priority, enum ucl_duplicate_strategy strat,
+ enum ucl_parse_type parse_type)
+{
+ unsigned char *buf;
+ size_t len;
+ bool ret;
+ struct stat st;
+
+ if (fstat (fd, &st) == -1) {
+ ucl_create_err (&parser->err, "cannot stat fd %d: %s",
+ fd, strerror (errno));
+ return false;
+ }
+ if (st.st_size == 0) {
+ return true;
+ }
+ if ((buf = ucl_mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
+ ucl_create_err (&parser->err, "cannot mmap fd %d: %s",
+ fd, strerror (errno));
+ return false;
+ }
+
+ if (parser->cur_file) {
+ free (parser->cur_file);
+ }
+ parser->cur_file = NULL;
+ len = st.st_size;
+ ret = ucl_parser_add_chunk_full (parser, buf, len, priority, strat,
+ parse_type);
+
+ if (len > 0) {
+ ucl_munmap (buf, len);
+ }
+
+ return ret;
+}
+
+bool
+ucl_parser_add_fd_priority (struct ucl_parser *parser, int fd,
+ unsigned priority)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_fd_full(parser, fd, parser->default_priority,
+ UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
+}
+
+bool
+ucl_parser_add_fd (struct ucl_parser *parser, int fd)
+{
+ if (parser == NULL) {
+ return false;
+ }
+
+ return ucl_parser_add_fd_priority(parser, fd, parser->default_priority);
+}
+
+size_t
+ucl_strlcpy (char *dst, const char *src, size_t siz)
+{
+ char *d = dst;
+ const char *s = src;
+ size_t n = siz;
+
+ /* Copy as many bytes as will fit */
+ if (n != 0) {
+ while (--n != 0) {
+ if ((*d++ = *s++) == '\0') {
+ break;
+ }
+ }
+ }
+
+ if (n == 0 && siz != 0) {
+ *d = '\0';
+ }
+
+ return (s - src - 1); /* count does not include NUL */
+}
+
+size_t
+ucl_strlcpy_unsafe (char *dst, const char *src, size_t siz)
+{
+ memcpy (dst, src, siz - 1);
+ dst[siz - 1] = '\0';
+
+ return siz - 1;
+}
+
+size_t
+ucl_strlcpy_tolower (char *dst, const char *src, size_t siz)
+{
+ char *d = dst;
+ const char *s = src;
+ size_t n = siz;
+
+ /* Copy as many bytes as will fit */
+ if (n != 0) {
+ while (--n != 0) {
+ if ((*d++ = tolower (*s++)) == '\0') {
+ break;
+ }
+ }
+ }
+
+ if (n == 0 && siz != 0) {
+ *d = '\0';
+ }
+
+ return (s - src); /* count does not include NUL */
+}
+
+/*
+ * Find the first occurrence of find in s
+ */
+char *
+ucl_strnstr (const char *s, const char *find, int len)
+{
+ char c, sc;
+ int mlen;
+
+ if ((c = *find++) != 0) {
+ mlen = strlen (find);
+ do {
+ do {
+ if ((sc = *s++) == 0 || len-- < mlen)
+ return (NULL);
+ } while (sc != c);
+ } while (strncmp (s, find, mlen) != 0);
+ s--;
+ }
+ return ((char *)s);
+}
+
+/*
+ * Find the first occurrence of find in s, ignore case.
+ */
+char *
+ucl_strncasestr (const char *s, const char *find, int len)
+{
+ char c, sc;
+ int mlen;
+
+ if ((c = *find++) != 0) {
+ c = tolower (c);
+ mlen = strlen (find);
+ do {
+ do {
+ if ((sc = *s++) == 0 || len-- == 0)
+ return (NULL);
+ } while (tolower (sc) != c);
+ } while (strncasecmp (s, find, mlen) != 0);
+ s--;
+ }
+ return ((char *)s);
+}
+
+ucl_object_t *
+ucl_object_fromstring_common (const char *str, size_t len, enum ucl_string_flags flags)
+{
+ ucl_object_t *obj;
+ const char *start, *end, *p, *pos;
+ char *dst, *d;
+ size_t escaped_len;
+
+ if (str == NULL) {
+ return NULL;
+ }
+
+ obj = ucl_object_new ();
+ if (obj) {
+ if (len == 0) {
+ len = strlen (str);
+ }
+ if (flags & UCL_STRING_TRIM) {
+ /* Skip leading spaces */
+ for (start = str; (size_t)(start - str) < len; start ++) {
+ if (!ucl_test_character (*start, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ break;
+ }
+ }
+ /* Skip trailing spaces */
+ for (end = str + len - 1; end > start; end --) {
+ if (!ucl_test_character (*end, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ break;
+ }
+ }
+ end ++;
+ }
+ else {
+ start = str;
+ end = str + len;
+ }
+
+ obj->type = UCL_STRING;
+ if (flags & UCL_STRING_ESCAPE) {
+ for (p = start, escaped_len = 0; p < end; p ++, escaped_len ++) {
+ if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE | UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ switch (*p) {
+ case '\v':
+ case '\0':
+ escaped_len += 5;
+ break;
+ case ' ':
+ break;
+ default:
+ escaped_len ++;
+ break;
+ }
+ }
+ }
+ dst = malloc (escaped_len + 1);
+ if (dst != NULL) {
+ for (p = start, d = dst; p < end; p ++, d ++) {
+ if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE | UCL_CHARACTER_WHITESPACE_UNSAFE)) {
+ switch (*p) {
+ case '\n':
+ *d++ = '\\';
+ *d = 'n';
+ break;
+ case '\r':
+ *d++ = '\\';
+ *d = 'r';
+ break;
+ case '\b':
+ *d++ = '\\';
+ *d = 'b';
+ break;
+ case '\t':
+ *d++ = '\\';
+ *d = 't';
+ break;
+ case '\f':
+ *d++ = '\\';
+ *d = 'f';
+ break;
+ case '\0':
+ *d++ = '\\';
+ *d++ = 'u';
+ *d++ = '0';
+ *d++ = '0';
+ *d++ = '0';
+ *d = '0';
+ break;
+ case '\v':
+ *d++ = '\\';
+ *d++ = 'u';
+ *d++ = '0';
+ *d++ = '0';
+ *d++ = '0';
+ *d = 'B';
+ break;
+ case '\\':
+ *d++ = '\\';
+ *d = '\\';
+ break;
+ case ' ':
+ *d = ' ';
+ break;
+ case '"':
+ *d++ = '\\';
+ *d = '"';
+ break;
+ }
+ }
+ else {
+ *d = *p;
+ }
+ }
+ *d = '\0';
+ obj->value.sv = dst;
+ obj->trash_stack[UCL_TRASH_VALUE] = dst;
+ obj->len = escaped_len;
+ }
+ }
+ else {
+ dst = malloc (end - start + 1);
+ if (dst != NULL) {
+ ucl_strlcpy_unsafe (dst, start, end - start + 1);
+ obj->value.sv = dst;
+ obj->trash_stack[UCL_TRASH_VALUE] = dst;
+ obj->len = end - start;
+ }
+ }
+ if ((flags & UCL_STRING_PARSE) && dst != NULL) {
+ /* Parse what we have */
+ if (flags & UCL_STRING_PARSE_BOOLEAN) {
+ if (!ucl_maybe_parse_boolean (obj, dst, obj->len) && (flags & UCL_STRING_PARSE_NUMBER)) {
+ ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos,
+ flags & UCL_STRING_PARSE_DOUBLE,
+ flags & UCL_STRING_PARSE_BYTES,
+ flags & UCL_STRING_PARSE_TIME);
+ }
+ }
+ else {
+ ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos,
+ flags & UCL_STRING_PARSE_DOUBLE,
+ flags & UCL_STRING_PARSE_BYTES,
+ flags & UCL_STRING_PARSE_TIME);
+ }
+ }
+ }
+
+ return obj;
+}
+
+static bool
+ucl_object_insert_key_common (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key, bool merge, bool replace)
+{
+ ucl_object_t *found, *tmp;
+ const ucl_object_t *cur;
+ ucl_object_iter_t it = NULL;
+ const char *p;
+ int ret = true;
+
+ if (elt == NULL || key == NULL) {
+ return false;
+ }
+
+ if (top == NULL) {
+ return false;
+ }
+
+ if (top->type != UCL_OBJECT) {
+ /* It is possible to convert NULL type to an object */
+ if (top->type == UCL_NULL) {
+ top->type = UCL_OBJECT;
+ }
+ else {
+ /* Refuse converting of other object types */
+ return false;
+ }
+ }
+
+ if (top->value.ov == NULL) {
+ top->value.ov = ucl_hash_create (false);
+ }
+
+ if (keylen == 0) {
+ keylen = strlen (key);
+ }
+
+ for (p = key; p < key + keylen; p ++) {
+ if (ucl_test_character (*p, UCL_CHARACTER_UCL_UNSAFE)) {
+ elt->flags |= UCL_OBJECT_NEED_KEY_ESCAPE;
+ break;
+ }
+ }
+
+ /* workaround for some use cases */
+ if (elt->trash_stack[UCL_TRASH_KEY] != NULL &&
+ key != (const char *)elt->trash_stack[UCL_TRASH_KEY]) {
+ /* Remove copied key */
+ free (elt->trash_stack[UCL_TRASH_KEY]);
+ elt->trash_stack[UCL_TRASH_KEY] = NULL;
+ elt->flags &= ~UCL_OBJECT_ALLOCATED_KEY;
+ }
+
+ elt->key = key;
+ elt->keylen = keylen;
+
+ if (copy_key) {
+ ucl_copy_key_trash (elt);
+ }
+
+ found = __DECONST (ucl_object_t *, ucl_hash_search_obj (top->value.ov, elt));
+
+ if (found == NULL) {
+ top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false);
+ top->len ++;
+ if (replace) {
+ ret = false;
+ }
+ }
+ else {
+ if (replace) {
+ ucl_hash_replace (top->value.ov, found, elt);
+ ucl_object_unref (found);
+ }
+ else if (merge) {
+ if (found->type != UCL_OBJECT && elt->type == UCL_OBJECT) {
+ /* Insert old elt to new one */
+ ucl_object_insert_key_common (elt, found, found->key,
+ found->keylen, copy_key, false, false);
+ ucl_hash_delete (top->value.ov, found);
+ top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false);
+ }
+ else if (found->type == UCL_OBJECT && elt->type != UCL_OBJECT) {
+ /* Insert new to old */
+ ucl_object_insert_key_common (found, elt, elt->key,
+ elt->keylen, copy_key, false, false);
+ }
+ else if (found->type == UCL_OBJECT && elt->type == UCL_OBJECT) {
+ /* Mix two hashes */
+ while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) {
+ tmp = ucl_object_ref (cur);
+ ucl_object_insert_key_common (found, tmp, cur->key,
+ cur->keylen, copy_key, true, false);
+ }
+ ucl_object_unref (elt);
+ }
+ else {
+ /* Just make a list of scalars */
+ DL_CONCAT (found, elt);
+ }
+ }
+ else {
+ DL_CONCAT (found, elt);
+ }
+ }
+
+ return ret;
+}
+
+bool
+ucl_object_delete_keyl (ucl_object_t *top, const char *key, size_t keylen)
+{
+ ucl_object_t *found;
+
+ if (top == NULL || key == NULL) {
+ return false;
+ }
+
+ found = __DECONST (ucl_object_t *, ucl_object_lookup_len (top, key, keylen));
+
+ if (found == NULL) {
+ return false;
+ }
+
+ ucl_hash_delete (top->value.ov, found);
+ ucl_object_unref (found);
+ top->len --;
+
+ return true;
+}
+
+bool
+ucl_object_delete_key (ucl_object_t *top, const char *key)
+{
+ return ucl_object_delete_keyl (top, key, strlen (key));
+}
+
+ucl_object_t*
+ucl_object_pop_keyl (ucl_object_t *top, const char *key, size_t keylen)
+{
+ const ucl_object_t *found;
+
+ if (top == NULL || key == NULL) {
+ return false;
+ }
+ found = ucl_object_lookup_len (top, key, keylen);
+
+ if (found == NULL) {
+ return NULL;
+ }
+ ucl_hash_delete (top->value.ov, found);
+ top->len --;
+
+ return __DECONST (ucl_object_t *, found);
+}
+
+ucl_object_t*
+ucl_object_pop_key (ucl_object_t *top, const char *key)
+{
+ return ucl_object_pop_keyl (top, key, strlen (key));
+}
+
+bool
+ucl_object_insert_key (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key)
+{
+ return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, false);
+}
+
+bool
+ucl_object_insert_key_merged (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key)
+{
+ return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, true, false);
+}
+
+bool
+ucl_object_replace_key (ucl_object_t *top, ucl_object_t *elt,
+ const char *key, size_t keylen, bool copy_key)
+{
+ return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, true);
+}
+
+bool
+ucl_object_merge (ucl_object_t *top, ucl_object_t *elt, bool copy)
+{
+ ucl_object_t *cur = NULL, *cp = NULL, *found = NULL;
+ ucl_object_iter_t iter = NULL;
+
+ if (top == NULL || elt == NULL) {
+ return false;
+ }
+
+ if (top->type == UCL_ARRAY) {
+ if (elt->type == UCL_ARRAY) {
+ /* Merge two arrays */
+ return ucl_array_merge (top, elt, copy);
+ }
+ else {
+ if (copy) {
+ ucl_array_append (top, ucl_object_copy (elt));
+
+ return true;
+ }
+ else {
+ ucl_array_append (top, ucl_object_ref (elt));
+
+ return true;
+ }
+ }
+ }
+ else if (top->type == UCL_OBJECT) {
+ if (elt->type == UCL_OBJECT) {
+ /* Mix two hashes */
+ while ((cur = (ucl_object_t *) ucl_hash_iterate (elt->value.ov,
+ &iter))) {
+
+ if (copy) {
+ cp = ucl_object_copy (cur);
+ } else {
+ cp = ucl_object_ref (cur);
+ }
+
+ found = __DECONST(ucl_object_t *,
+ ucl_hash_search (top->value.ov, cp->key, cp->keylen));
+
+ if (found == NULL) {
+ /* The key does not exist */
+ top->value.ov = ucl_hash_insert_object (top->value.ov, cp,
+ false);
+ top->len++;
+ }
+ else {
+ /* The key already exists, merge it recursively */
+ if (found->type == UCL_OBJECT || found->type == UCL_ARRAY) {
+ if (!ucl_object_merge (found, cp, copy)) {
+ return false;
+ }
+ ucl_object_unref (cp);
+ }
+ else {
+ ucl_hash_replace (top->value.ov, found, cp);
+ ucl_object_unref (found);
+ }
+ }
+ }
+ }
+ else {
+ if (copy) {
+ cp = ucl_object_copy (elt);
+ }
+ else {
+ cp = ucl_object_ref (elt);
+ }
+
+ found = __DECONST(ucl_object_t *,
+ ucl_hash_search (top->value.ov, cp->key, cp->keylen));
+
+ if (found == NULL) {
+ /* The key does not exist */
+ top->value.ov = ucl_hash_insert_object (top->value.ov, cp,
+ false);
+ top->len++;
+ }
+ else {
+ /* The key already exists, merge it recursively */
+ if (found->type == UCL_OBJECT || found->type == UCL_ARRAY) {
+ if (!ucl_object_merge (found, cp, copy)) {
+ return false;
+ }
+ ucl_object_unref (cp);
+ }
+ else {
+ ucl_hash_replace (top->value.ov, found, cp);
+ ucl_object_unref (found);
+ }
+ }
+ }
+ }
+ else {
+ /* Cannot merge trivial objects */
+ return false;
+ }
+
+ return true;
+}
+
+const ucl_object_t *
+ucl_object_lookup_len (const ucl_object_t *obj, const char *key, size_t klen)
+{
+ const ucl_object_t *ret;
+ ucl_object_t srch;
+
+ if (obj == NULL || obj->type != UCL_OBJECT || key == NULL) {
+ return NULL;
+ }
+
+ srch.key = key;
+ srch.keylen = klen;
+ ret = ucl_hash_search_obj (obj->value.ov, &srch);
+
+ return ret;
+}
+
+const ucl_object_t *
+ucl_object_lookup (const ucl_object_t *obj, const char *key)
+{
+ if (key == NULL) {
+ return NULL;
+ }
+
+ return ucl_object_lookup_len (obj, key, strlen (key));
+}
+
+const ucl_object_t*
+ucl_object_lookup_any (const ucl_object_t *obj,
+ const char *key, ...)
+{
+ va_list ap;
+ const ucl_object_t *ret = NULL;
+ const char *nk = NULL;
+
+ if (obj == NULL || key == NULL) {
+ return NULL;
+ }
+
+ ret = ucl_object_lookup_len (obj, key, strlen (key));
+
+ if (ret == NULL) {
+ va_start (ap, key);
+
+ while (ret == NULL) {
+ nk = va_arg (ap, const char *);
+
+ if (nk == NULL) {
+ break;
+ }
+ else {
+ ret = ucl_object_lookup_len (obj, nk, strlen (nk));
+ }
+ }
+
+ va_end (ap);
+ }
+
+ return ret;
+}
+
+const ucl_object_t*
+ucl_object_iterate_with_error (const ucl_object_t *obj, ucl_object_iter_t *iter, bool expand_values,
+ int *ep)
+{
+ const ucl_object_t *elt = NULL;
+
+ if (obj == NULL || iter == NULL) {
+ return NULL;
+ }
+
+ if (expand_values) {
+ switch (obj->type) {
+ case UCL_OBJECT:
+ return (const ucl_object_t*)ucl_hash_iterate2 (obj->value.ov, iter, ep);
+ break;
+ case UCL_ARRAY: {
+ unsigned int idx;
+ UCL_ARRAY_GET (vec, obj);
+ idx = (unsigned int)(uintptr_t)(*iter);
+
+ if (vec != NULL) {
+ while (idx < kv_size (*vec)) {
+ if ((elt = kv_A (*vec, idx)) != NULL) {
+ idx ++;
+ break;
+ }
+ idx ++;
+ }
+ *iter = (void *)(uintptr_t)idx;
+ }
+
+ return elt;
+ break;
+ }
+ default:
+ /* Go to linear iteration */
+ break;
+ }
+ }
+ /* Treat everything as a linear list */
+ elt = *iter;
+ if (elt == NULL) {
+ elt = obj;
+ }
+ else if (elt == obj) {
+ return NULL;
+ }
+ *iter = __DECONST (void *, elt->next ? elt->next : obj);
+ return elt;
+
+ /* Not reached */
+ return NULL;
+}
+
+enum ucl_safe_iter_flags {
+ UCL_ITERATE_FLAG_UNDEFINED = 0,
+ UCL_ITERATE_FLAG_INSIDE_ARRAY,
+ UCL_ITERATE_FLAG_INSIDE_OBJECT,
+ UCL_ITERATE_FLAG_IMPLICIT,
+ UCL_ITERATE_FLAG_EXCEPTION
+};
+
+static const char safe_iter_magic[4] = {'u', 'i', 't', 'e'};
+struct ucl_object_safe_iter {
+ char magic[4]; /* safety check */
+ uint32_t flags;
+ const ucl_object_t *impl_it; /* implicit object iteration */
+ ucl_object_iter_t expl_it; /* explicit iteration */
+};
+
+#define UCL_SAFE_ITER(ptr) (struct ucl_object_safe_iter *)(ptr)
+#define UCL_SAFE_ITER_CHECK(it) do { \
+ assert (it != NULL); \
+ assert (memcmp (it->magic, safe_iter_magic, sizeof (it->magic)) == 0); \
+ } while (0)
+
+ucl_object_iter_t
+ucl_object_iterate_new (const ucl_object_t *obj)
+{
+ struct ucl_object_safe_iter *it;
+
+ it = UCL_ALLOC (sizeof (*it));
+ if (it != NULL) {
+ memcpy (it->magic, safe_iter_magic, sizeof (it->magic));
+ it->flags = UCL_ITERATE_FLAG_UNDEFINED;
+ it->expl_it = NULL;
+ it->impl_it = obj;
+ }
+
+ return (ucl_object_iter_t)it;
+}
+
+bool
+ucl_object_iter_chk_excpn(ucl_object_iter_t *it)
+{
+ struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
+
+ UCL_SAFE_ITER_CHECK (rit);
+
+ return (rit->flags == UCL_ITERATE_FLAG_EXCEPTION);
+}
+
+ucl_object_iter_t
+ucl_object_iterate_reset (ucl_object_iter_t it, const ucl_object_t *obj)
+{
+ struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
+
+ UCL_SAFE_ITER_CHECK (rit);
+
+ if (rit->expl_it != NULL) {
+ if (rit->flags == UCL_ITERATE_FLAG_INSIDE_OBJECT) {
+ UCL_FREE (sizeof (*rit->expl_it), rit->expl_it);
+ }
+ }
+
+ rit->impl_it = obj;
+ rit->expl_it = NULL;
+ rit->flags = UCL_ITERATE_FLAG_UNDEFINED;
+
+ return it;
+}
+
+const ucl_object_t*
+ucl_object_iterate_safe (ucl_object_iter_t it, bool expand_values)
+{
+ return ucl_object_iterate_full (it, expand_values ? UCL_ITERATE_BOTH :
+ UCL_ITERATE_IMPLICIT);
+}
+
+const ucl_object_t*
+ucl_object_iterate_full (ucl_object_iter_t it, enum ucl_iterate_type type)
+{
+ struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
+ const ucl_object_t *ret = NULL;
+ int ern;
+
+ UCL_SAFE_ITER_CHECK (rit);
+
+ if (rit->impl_it == NULL) {
+ return NULL;
+ }
+
+ if (rit->impl_it->type == UCL_OBJECT) {
+ rit->flags = UCL_ITERATE_FLAG_INSIDE_OBJECT;
+ ret = ucl_object_iterate_with_error (rit->impl_it, &rit->expl_it, true, &ern);
+
+ if (ret == NULL && ern != 0) {
+ rit->flags = UCL_ITERATE_FLAG_EXCEPTION;
+ return NULL;
+ }
+
+ if (ret == NULL && (type & UCL_ITERATE_IMPLICIT)) {
+ /* Need to switch to another implicit object in chain */
+ rit->impl_it = rit->impl_it->next;
+ rit->expl_it = NULL;
+
+ return ucl_object_iterate_safe (it, type);
+ }
+ }
+ else if (rit->impl_it->type == UCL_ARRAY) {
+ rit->flags = UCL_ITERATE_FLAG_INSIDE_ARRAY;
+ ret = ucl_object_iterate (rit->impl_it, &rit->expl_it, true);
+
+ if (ret == NULL && (type & UCL_ITERATE_IMPLICIT)) {
+ /* Need to switch to another implicit object in chain */
+ rit->impl_it = rit->impl_it->next;
+ rit->expl_it = NULL;
+
+ return ucl_object_iterate_safe (it, type);
+ }
+ }
+ else {
+ /* Just iterate over the implicit array */
+ rit->flags = UCL_ITERATE_FLAG_IMPLICIT;
+ ret = rit->impl_it;
+ rit->impl_it = rit->impl_it->next;
+
+ if (type & UCL_ITERATE_EXPLICIT) {
+ /* We flatten objects if need to expand values */
+ if (ret->type == UCL_OBJECT || ret->type == UCL_ARRAY) {
+ return ucl_object_iterate_safe (it, type);
+ }
+ }
+ }
+
+ return ret;
+}
+
+void
+ucl_object_iterate_free (ucl_object_iter_t it)
+{
+ struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
+
+ UCL_SAFE_ITER_CHECK (rit);
+
+ if (rit->expl_it != NULL) {
+ if (rit->flags == UCL_ITERATE_FLAG_INSIDE_OBJECT) {
+ UCL_FREE (sizeof (*rit->expl_it), rit->expl_it);
+ }
+ }
+
+ UCL_FREE (sizeof (*rit), it);
+}
+
+const ucl_object_t *
+ucl_object_lookup_path (const ucl_object_t *top, const char *path_in) {
+ return ucl_object_lookup_path_char (top, path_in, '.');
+}
+
+
+const ucl_object_t *
+ucl_object_lookup_path_char (const ucl_object_t *top, const char *path_in, const char sep) {
+ const ucl_object_t *o = NULL, *found;
+ const char *p, *c;
+ char *err_str;
+ unsigned index;
+
+ if (path_in == NULL || top == NULL) {
+ return NULL;
+ }
+
+ found = NULL;
+ p = path_in;
+
+ /* Skip leading dots */
+ while (*p == sep) {
+ p ++;
+ }
+
+ c = p;
+ while (*p != '\0') {
+ p ++;
+ if (*p == sep || *p == '\0') {
+ if (p > c) {
+ switch (top->type) {
+ case UCL_ARRAY:
+ /* Key should be an int */
+ index = strtoul (c, &err_str, 10);
+ if (err_str != NULL && (*err_str != sep && *err_str != '\0')) {
+ return NULL;
+ }
+ o = ucl_array_find_index (top, index);
+ break;
+ default:
+ o = ucl_object_lookup_len (top, c, p - c);
+ break;
+ }
+ if (o == NULL) {
+ return NULL;
+ }
+ top = o;
+ }
+ if (*p != '\0') {
+ c = p + 1;
+ }
+ }
+ }
+ found = o;
+
+ return found;
+}
+
+
+ucl_object_t *
+ucl_object_new (void)
+{
+ return ucl_object_typed_new (UCL_NULL);
+}
+
+ucl_object_t *
+ucl_object_typed_new (ucl_type_t type)
+{
+ return ucl_object_new_full (type, 0);
+}
+
+ucl_object_t *
+ucl_object_new_full (ucl_type_t type, unsigned priority)
+{
+ ucl_object_t *new;
+
+ if (type != UCL_USERDATA) {
+ new = UCL_ALLOC (sizeof (ucl_object_t));
+ if (new != NULL) {
+ memset (new, 0, sizeof (ucl_object_t));
+ new->ref = 1;
+ new->type = (type <= UCL_NULL ? type : UCL_NULL);
+ new->next = NULL;
+ new->prev = new;
+ ucl_object_set_priority (new, priority);
+
+ if (type == UCL_ARRAY) {
+ new->value.av = UCL_ALLOC (sizeof (ucl_array_t));
+ if (new->value.av) {
+ memset (new->value.av, 0, sizeof (ucl_array_t));
+ UCL_ARRAY_GET (vec, new);
+
+ /* Preallocate some space for arrays */
+ kv_resize_safe (ucl_object_t *, *vec, 8, enomem);
+ }
+ }
+ }
+ }
+ else {
+ new = ucl_object_new_userdata (NULL, NULL, NULL);
+ ucl_object_set_priority (new, priority);
+ }
+enomem:
+ return new;
+}
+
+bool ucl_object_reserve (ucl_object_t *obj, size_t reserved)
+{
+ if (obj->type == UCL_ARRAY) {
+ UCL_ARRAY_GET (vec, obj);
+
+ if (vec->m < reserved) {
+ /* Preallocate some space for arrays */
+ kv_resize_safe (ucl_object_t *, *vec, reserved, e0);
+ }
+ }
+ else if (obj->type == UCL_OBJECT) {
+ ucl_hash_reserve (obj->value.ov, reserved);
+ }
+ return true;
+e0:
+ return false;
+}
+
+ucl_object_t*
+ucl_object_new_userdata (ucl_userdata_dtor dtor,
+ ucl_userdata_emitter emitter,
+ void *ptr)
+{
+ struct ucl_object_userdata *new;
+ size_t nsize = sizeof (*new);
+
+ new = UCL_ALLOC (nsize);
+ if (new != NULL) {
+ memset (new, 0, nsize);
+ new->obj.ref = 1;
+ new->obj.type = UCL_USERDATA;
+ new->obj.next = NULL;
+ new->obj.prev = (ucl_object_t *)new;
+ new->dtor = dtor;
+ new->emitter = emitter;
+ new->obj.value.ud = ptr;
+ }
+
+ return (ucl_object_t *)new;
+}
+
+ucl_type_t
+ucl_object_type (const ucl_object_t *obj)
+{
+ if (obj == NULL) {
+ return UCL_NULL;
+ }
+
+ return obj->type;
+}
+
+ucl_object_t*
+ucl_object_fromstring (const char *str)
+{
+ return ucl_object_fromstring_common (str, 0, UCL_STRING_ESCAPE);
+}
+
+ucl_object_t *
+ucl_object_fromlstring (const char *str, size_t len)
+{
+ return ucl_object_fromstring_common (str, len, UCL_STRING_ESCAPE);
+}
+
+ucl_object_t *
+ucl_object_fromint (int64_t iv)
+{
+ ucl_object_t *obj;
+
+ obj = ucl_object_new ();
+ if (obj != NULL) {
+ obj->type = UCL_INT;
+ obj->value.iv = iv;
+ }
+
+ return obj;
+}
+
+ucl_object_t *
+ucl_object_fromdouble (double dv)
+{
+ ucl_object_t *obj;
+
+ obj = ucl_object_new ();
+ if (obj != NULL) {
+ obj->type = UCL_FLOAT;
+ obj->value.dv = dv;
+ }
+
+ return obj;
+}
+
+ucl_object_t*
+ucl_object_frombool (bool bv)
+{
+ ucl_object_t *obj;
+
+ obj = ucl_object_new ();
+ if (obj != NULL) {
+ obj->type = UCL_BOOLEAN;
+ obj->value.iv = bv;
+ }
+
+ return obj;
+}
+
+bool
+ucl_array_append (ucl_object_t *top, ucl_object_t *elt)
+{
+ UCL_ARRAY_GET (vec, top);
+
+ if (elt == NULL || top == NULL) {
+ return false;
+ }
+
+ if (vec == NULL) {
+ vec = UCL_ALLOC (sizeof (*vec));
+
+ if (vec == NULL) {
+ return false;
+ }
+
+ kv_init (*vec);
+ top->value.av = (void *)vec;
+ }
+
+ kv_push_safe (ucl_object_t *, *vec, elt, e0);
+
+ top->len ++;
+
+ return true;
+e0:
+ return false;
+}
+
+bool
+ucl_array_prepend (ucl_object_t *top, ucl_object_t *elt)
+{
+ UCL_ARRAY_GET (vec, top);
+
+ if (elt == NULL || top == NULL) {
+ return false;
+ }
+
+ if (vec == NULL) {
+ vec = UCL_ALLOC (sizeof (*vec));
+ kv_init (*vec);
+ top->value.av = (void *)vec;
+ kv_push_safe (ucl_object_t *, *vec, elt, e0);
+ }
+ else {
+ /* Slow O(n) algorithm */
+ kv_prepend_safe (ucl_object_t *, *vec, elt, e0);
+ }
+
+ top->len ++;
+
+ return true;
+e0:
+ return false;
+}
+
+bool
+ucl_array_merge (ucl_object_t *top, ucl_object_t *elt, bool copy)
+{
+ unsigned i;
+ ucl_object_t *cp = NULL;
+ ucl_object_t **obj;
+
+ if (elt == NULL || top == NULL || top->type != UCL_ARRAY || elt->type != UCL_ARRAY) {
+ return false;
+ }
+
+ if (copy) {
+ cp = ucl_object_copy (elt);
+ }
+ else {
+ cp = ucl_object_ref (elt);
+ }
+
+ UCL_ARRAY_GET (v1, top);
+ UCL_ARRAY_GET (v2, cp);
+
+ if (v1 && v2) {
+ kv_concat_safe (ucl_object_t *, *v1, *v2, e0);
+
+ for (i = v2->n; i < v1->n; i ++) {
+ obj = &kv_A (*v1, i);
+ if (*obj == NULL) {
+ continue;
+ }
+ top->len ++;
+ }
+ }
+
+ return true;
+e0:
+ return false;
+}
+
+ucl_object_t *
+ucl_array_delete (ucl_object_t *top, ucl_object_t *elt)
+{
+ UCL_ARRAY_GET (vec, top);
+ ucl_object_t *ret = NULL;
+ unsigned i;
+
+ if (vec == NULL) {
+ return NULL;
+ }
+
+ for (i = 0; i < vec->n; i ++) {
+ if (kv_A (*vec, i) == elt) {
+ kv_del (ucl_object_t *, *vec, i);
+ ret = elt;
+ top->len --;
+ break;
+ }
+ }
+
+ return ret;
+}
+
+const ucl_object_t *
+ucl_array_head (const ucl_object_t *top)
+{
+ UCL_ARRAY_GET (vec, top);
+
+ if (vec == NULL || top == NULL || top->type != UCL_ARRAY ||
+ top->value.av == NULL) {
+ return NULL;
+ }
+
+ return (vec->n > 0 ? vec->a[0] : NULL);
+}
+
+const ucl_object_t *
+ucl_array_tail (const ucl_object_t *top)
+{
+ UCL_ARRAY_GET (vec, top);
+
+ if (top == NULL || top->type != UCL_ARRAY || top->value.av == NULL) {
+ return NULL;
+ }
+
+ return (vec->n > 0 ? vec->a[vec->n - 1] : NULL);
+}
+
+ucl_object_t *
+ucl_array_pop_last (ucl_object_t *top)
+{
+ UCL_ARRAY_GET (vec, top);
+ ucl_object_t **obj, *ret = NULL;
+
+ if (vec != NULL && vec->n > 0) {
+ obj = &kv_A (*vec, vec->n - 1);
+ ret = *obj;
+ kv_del (ucl_object_t *, *vec, vec->n - 1);
+ top->len --;
+ }
+
+ return ret;
+}
+
+ucl_object_t *
+ucl_array_pop_first (ucl_object_t *top)
+{
+ UCL_ARRAY_GET (vec, top);
+ ucl_object_t **obj, *ret = NULL;
+
+ if (vec != NULL && vec->n > 0) {
+ obj = &kv_A (*vec, 0);
+ ret = *obj;
+ kv_del (ucl_object_t *, *vec, 0);
+ top->len --;
+ }
+
+ return ret;
+}
+
+unsigned int
+ucl_array_size (const ucl_object_t *top)
+{
+ if (top == NULL || top->type != UCL_ARRAY) {
+ return 0;
+ }
+
+ UCL_ARRAY_GET (vec, top);
+
+ if (vec != NULL) {
+ return kv_size(*vec);
+ }
+
+ return 0;
+}
+
+const ucl_object_t *
+ucl_array_find_index (const ucl_object_t *top, unsigned int index)
+{
+ UCL_ARRAY_GET (vec, top);
+
+ if (vec != NULL && vec->n > 0 && index < vec->n) {
+ return kv_A (*vec, index);
+ }
+
+ return NULL;
+}
+
+unsigned int
+ucl_array_index_of (ucl_object_t *top, ucl_object_t *elt)
+{
+ UCL_ARRAY_GET (vec, top);
+ unsigned i;
+
+ if (vec == NULL) {
+ return (unsigned int)(-1);
+ }
+
+ for (i = 0; i < vec->n; i ++) {
+ if (kv_A (*vec, i) == elt) {
+ return i;
+ }
+ }
+
+ return (unsigned int)(-1);
+}
+
+ucl_object_t *
+ucl_array_replace_index (ucl_object_t *top, ucl_object_t *elt,
+ unsigned int index)
+{
+ UCL_ARRAY_GET (vec, top);
+ ucl_object_t *ret = NULL;
+
+ if (vec != NULL && vec->n > 0 && index < vec->n) {
+ ret = kv_A (*vec, index);
+ kv_A (*vec, index) = elt;
+ }
+
+ return ret;
+}
+
+ucl_object_t *
+ucl_elt_append (ucl_object_t *head, ucl_object_t *elt)
+{
+
+ if (head == NULL) {
+ elt->next = NULL;
+ elt->prev = elt;
+ head = elt;
+ }
+ else {
+ elt->prev = head->prev;
+ head->prev->next = elt;
+ head->prev = elt;
+ elt->next = NULL;
+ }
+
+ return head;
+}
+
+bool
+ucl_object_todouble_safe (const ucl_object_t *obj, double *target)
+{
+ if (obj == NULL || target == NULL) {
+ return false;
+ }
+ switch (obj->type) {
+ case UCL_INT:
+ *target = obj->value.iv; /* Probably could cause overflow */
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ *target = obj->value.dv;
+ break;
+ default:
+ return false;
+ }
+
+ return true;
+}
+
+double
+ucl_object_todouble (const ucl_object_t *obj)
+{
+ double result = 0.;
+
+ ucl_object_todouble_safe (obj, &result);
+ return result;
+}
+
+bool
+ucl_object_toint_safe (const ucl_object_t *obj, int64_t *target)
+{
+ if (obj == NULL || target == NULL) {
+ return false;
+ }
+ switch (obj->type) {
+ case UCL_INT:
+ *target = obj->value.iv;
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ *target = obj->value.dv; /* Losing of decimal points */
+ break;
+ default:
+ return false;
+ }
+
+ return true;
+}
+
+int64_t
+ucl_object_toint (const ucl_object_t *obj)
+{
+ int64_t result = 0;
+
+ ucl_object_toint_safe (obj, &result);
+ return result;
+}
+
+bool
+ucl_object_toboolean_safe (const ucl_object_t *obj, bool *target)
+{
+ if (obj == NULL || target == NULL) {
+ return false;
+ }
+ switch (obj->type) {
+ case UCL_BOOLEAN:
+ *target = (obj->value.iv == true);
+ break;
+ default:
+ return false;
+ }
+
+ return true;
+}
+
+bool
+ucl_object_toboolean (const ucl_object_t *obj)
+{
+ bool result = false;
+
+ ucl_object_toboolean_safe (obj, &result);
+ return result;
+}
+
+bool
+ucl_object_tostring_safe (const ucl_object_t *obj, const char **target)
+{
+ if (obj == NULL || target == NULL) {
+ return false;
+ }
+
+ switch (obj->type) {
+ case UCL_STRING:
+ if (!(obj->flags & UCL_OBJECT_BINARY)) {
+ *target = ucl_copy_value_trash (obj);
+ }
+ break;
+ default:
+ return false;
+ }
+
+ return true;
+}
+
+const char *
+ucl_object_tostring (const ucl_object_t *obj)
+{
+ const char *result = NULL;
+
+ ucl_object_tostring_safe (obj, &result);
+ return result;
+}
+
+const char *
+ucl_object_tostring_forced (const ucl_object_t *obj)
+{
+ /* TODO: For binary strings we might encode string here */
+ if (!(obj->flags & UCL_OBJECT_BINARY)) {
+ return ucl_copy_value_trash (obj);
+ }
+
+ return NULL;
+}
+
+bool
+ucl_object_tolstring_safe (const ucl_object_t *obj, const char **target, size_t *tlen)
+{
+ if (obj == NULL || target == NULL) {
+ return false;
+ }
+ switch (obj->type) {
+ case UCL_STRING:
+ *target = obj->value.sv;
+ if (tlen != NULL) {
+ *tlen = obj->len;
+ }
+ break;
+ default:
+ return false;
+ }
+
+ return true;
+}
+
+const char *
+ucl_object_tolstring (const ucl_object_t *obj, size_t *tlen)
+{
+ const char *result = NULL;
+
+ ucl_object_tolstring_safe (obj, &result, tlen);
+ return result;
+}
+
+const char *
+ucl_object_key (const ucl_object_t *obj)
+{
+ return ucl_copy_key_trash (obj);
+}
+
+const char *
+ucl_object_keyl (const ucl_object_t *obj, size_t *len)
+{
+ if (len == NULL || obj == NULL) {
+ return NULL;
+ }
+ *len = obj->keylen;
+ return obj->key;
+}
+
+ucl_object_t *
+ucl_object_ref (const ucl_object_t *obj)
+{
+ ucl_object_t *res = NULL;
+
+ if (obj != NULL) {
+ if (obj->flags & UCL_OBJECT_EPHEMERAL) {
+ /*
+ * Use deep copy for ephemeral objects, note that its refcount
+ * is NOT increased, since ephemeral objects does not need refcount
+ * at all
+ */
+ res = ucl_object_copy (obj);
+ }
+ else {
+ res = __DECONST (ucl_object_t *, obj);
+#ifdef HAVE_ATOMIC_BUILTINS
+ (void)__sync_add_and_fetch (&res->ref, 1);
+#else
+ res->ref ++;
+#endif
+ }
+ }
+ return res;
+}
+
+static ucl_object_t *
+ucl_object_copy_internal (const ucl_object_t *other, bool allow_array)
+{
+
+ ucl_object_t *new;
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur;
+
+ new = malloc (sizeof (*new));
+
+ if (new != NULL) {
+ memcpy (new, other, sizeof (*new));
+ if (other->flags & UCL_OBJECT_EPHEMERAL) {
+ /* Copied object is always non ephemeral */
+ new->flags &= ~UCL_OBJECT_EPHEMERAL;
+ }
+ new->ref = 1;
+ /* Unlink from others */
+ new->next = NULL;
+ new->prev = new;
+
+ /* deep copy of values stored */
+ if (other->trash_stack[UCL_TRASH_KEY] != NULL) {
+ new->trash_stack[UCL_TRASH_KEY] = NULL;
+ if (other->key == (const char *)other->trash_stack[UCL_TRASH_KEY]) {
+ new->trash_stack[UCL_TRASH_KEY] = malloc(other->keylen + 1);
+ memcpy(new->trash_stack[UCL_TRASH_KEY], other->trash_stack[UCL_TRASH_KEY], other->keylen);
+ new->trash_stack[UCL_TRASH_KEY][other->keylen] = '\0';
+ new->key = new->trash_stack[UCL_TRASH_KEY];
+ }
+ }
+ if (other->trash_stack[UCL_TRASH_VALUE] != NULL) {
+ new->trash_stack[UCL_TRASH_VALUE] =
+ strdup (other->trash_stack[UCL_TRASH_VALUE]);
+ if (new->type == UCL_STRING) {
+ new->value.sv = new->trash_stack[UCL_TRASH_VALUE];
+ }
+ }
+
+ if (other->type == UCL_ARRAY || other->type == UCL_OBJECT) {
+ /* reset old value */
+ memset (&new->value, 0, sizeof (new->value));
+
+ while ((cur = ucl_object_iterate (other, &it, true)) != NULL) {
+ if (other->type == UCL_ARRAY) {
+ ucl_array_append (new, ucl_object_copy_internal (cur, false));
+ }
+ else {
+ ucl_object_t *cp = ucl_object_copy_internal (cur, true);
+ if (cp != NULL) {
+ ucl_object_insert_key (new, cp, cp->key, cp->keylen,
+ false);
+ }
+ }
+ }
+ }
+ else if (allow_array && other->next != NULL) {
+ LL_FOREACH (other->next, cur) {
+ ucl_object_t *cp = ucl_object_copy_internal (cur, false);
+ if (cp != NULL) {
+ DL_APPEND (new, cp);
+ }
+ }
+ }
+ }
+
+ return new;
+}
+
+ucl_object_t *
+ucl_object_copy (const ucl_object_t *other)
+{
+ return ucl_object_copy_internal (other, true);
+}
+
+void
+ucl_object_unref (ucl_object_t *obj)
+{
+ if (obj != NULL) {
+#ifdef HAVE_ATOMIC_BUILTINS
+ unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1);
+ if (rc == 0) {
+#else
+ if (--obj->ref == 0) {
+#endif
+ ucl_object_free_internal (obj, true, ucl_object_dtor_unref);
+ }
+ }
+}
+
+int
+ucl_object_compare (const ucl_object_t *o1, const ucl_object_t *o2)
+{
+ const ucl_object_t *it1, *it2;
+ ucl_object_iter_t iter = NULL;
+ int ret = 0;
+
+ if (o1->type != o2->type) {
+ return (o1->type) - (o2->type);
+ }
+
+ switch (o1->type) {
+ case UCL_STRING:
+ if (o1->len == o2->len && o1->len > 0) {
+ ret = strcmp (ucl_object_tostring(o1), ucl_object_tostring(o2));
+ }
+ else {
+ ret = o1->len - o2->len;
+ }
+ break;
+ case UCL_FLOAT:
+ case UCL_INT:
+ case UCL_TIME:
+ ret = ucl_object_todouble (o1) - ucl_object_todouble (o2);
+ break;
+ case UCL_BOOLEAN:
+ ret = ucl_object_toboolean (o1) - ucl_object_toboolean (o2);
+ break;
+ case UCL_ARRAY:
+ if (o1->len == o2->len && o1->len > 0) {
+ UCL_ARRAY_GET (vec1, o1);
+ UCL_ARRAY_GET (vec2, o2);
+ unsigned i;
+
+ /* Compare all elements in both arrays */
+ for (i = 0; i < vec1->n; i ++) {
+ it1 = kv_A (*vec1, i);
+ it2 = kv_A (*vec2, i);
+
+ if (it1 == NULL && it2 != NULL) {
+ return -1;
+ }
+ else if (it2 == NULL && it1 != NULL) {
+ return 1;
+ }
+ else if (it1 != NULL && it2 != NULL) {
+ ret = ucl_object_compare (it1, it2);
+ if (ret != 0) {
+ break;
+ }
+ }
+ }
+ }
+ else {
+ ret = o1->len - o2->len;
+ }
+ break;
+ case UCL_OBJECT:
+ if (o1->len == o2->len && o1->len > 0) {
+ while ((it1 = ucl_object_iterate (o1, &iter, true)) != NULL) {
+ it2 = ucl_object_lookup (o2, ucl_object_key (it1));
+ if (it2 == NULL) {
+ ret = 1;
+ break;
+ }
+ ret = ucl_object_compare (it1, it2);
+ if (ret != 0) {
+ break;
+ }
+ }
+ }
+ else {
+ ret = o1->len - o2->len;
+ }
+ break;
+ default:
+ ret = 0;
+ break;
+ }
+
+ return ret;
+}
+
+int
+ucl_object_compare_qsort (const ucl_object_t **o1,
+ const ucl_object_t **o2)
+{
+ return ucl_object_compare (*o1, *o2);
+}
+
+void
+ucl_object_array_sort (ucl_object_t *ar,
+ int (*cmp)(const ucl_object_t **o1, const ucl_object_t **o2))
+{
+ UCL_ARRAY_GET (vec, ar);
+
+ if (cmp == NULL || ar == NULL || ar->type != UCL_ARRAY) {
+ return;
+ }
+
+ qsort (vec->a, vec->n, sizeof (ucl_object_t *),
+ (int (*)(const void *, const void *))cmp);
+}
+
+void ucl_object_sort_keys (ucl_object_t *obj,
+ enum ucl_object_keys_sort_flags how)
+{
+ if (obj != NULL && obj->type == UCL_OBJECT) {
+ ucl_hash_sort (obj->value.ov, how);
+ }
+}
+
+#define PRIOBITS 4
+
+unsigned int
+ucl_object_get_priority (const ucl_object_t *obj)
+{
+ if (obj == NULL) {
+ return 0;
+ }
+
+ return (obj->flags >> ((sizeof (obj->flags) * NBBY) - PRIOBITS));
+}
+
+void
+ucl_object_set_priority (ucl_object_t *obj,
+ unsigned int priority)
+{
+ if (obj != NULL) {
+ priority &= (0x1 << PRIOBITS) - 1;
+ priority <<= ((sizeof (obj->flags) * NBBY) - PRIOBITS);
+ priority |= obj->flags & ((1 << ((sizeof (obj->flags) * NBBY) -
+ PRIOBITS)) - 1);
+ obj->flags = priority;
+ }
+}
+
+bool
+ucl_object_string_to_type (const char *input, ucl_type_t *res)
+{
+ if (strcasecmp (input, "object") == 0) {
+ *res = UCL_OBJECT;
+ }
+ else if (strcasecmp (input, "array") == 0) {
+ *res = UCL_ARRAY;
+ }
+ else if (strcasecmp (input, "integer") == 0) {
+ *res = UCL_INT;
+ }
+ else if (strcasecmp (input, "number") == 0) {
+ *res = UCL_FLOAT;
+ }
+ else if (strcasecmp (input, "string") == 0) {
+ *res = UCL_STRING;
+ }
+ else if (strcasecmp (input, "boolean") == 0) {
+ *res = UCL_BOOLEAN;
+ }
+ else if (strcasecmp (input, "null") == 0) {
+ *res = UCL_NULL;
+ }
+ else if (strcasecmp (input, "userdata") == 0) {
+ *res = UCL_USERDATA;
+ }
+ else {
+ return false;
+ }
+
+ return true;
+}
+
+const char *
+ucl_object_type_to_string (ucl_type_t type)
+{
+ const char *res = "unknown";
+
+ switch (type) {
+ case UCL_OBJECT:
+ res = "object";
+ break;
+ case UCL_ARRAY:
+ res = "array";
+ break;
+ case UCL_INT:
+ res = "integer";
+ break;
+ case UCL_FLOAT:
+ case UCL_TIME:
+ res = "number";
+ break;
+ case UCL_STRING:
+ res = "string";
+ break;
+ case UCL_BOOLEAN:
+ res = "boolean";
+ break;
+ case UCL_USERDATA:
+ res = "userdata";
+ break;
+ case UCL_NULL:
+ res = "null";
+ break;
+ }
+
+ return res;
+}
+
+const ucl_object_t *
+ucl_parser_get_comments (struct ucl_parser *parser)
+{
+ if (parser && parser->comments) {
+ return parser->comments;
+ }
+
+ return NULL;
+}
+
+const ucl_object_t *
+ucl_comments_find (const ucl_object_t *comments,
+ const ucl_object_t *srch)
+{
+ if (comments && srch) {
+ return ucl_object_lookup_len (comments, (const char *)&srch,
+ sizeof (void *));
+ }
+
+ return NULL;
+}
+
+bool
+ucl_comments_move (ucl_object_t *comments,
+ const ucl_object_t *from, const ucl_object_t *to)
+{
+ const ucl_object_t *found;
+ ucl_object_t *obj;
+
+ if (comments && from && to) {
+ found = ucl_object_lookup_len (comments,
+ (const char *)&from, sizeof (void *));
+
+ if (found) {
+ /* Replace key */
+ obj = ucl_object_ref (found);
+ ucl_object_delete_keyl (comments, (const char *)&from,
+ sizeof (void *));
+ ucl_object_insert_key (comments, obj, (const char *)&to,
+ sizeof (void *), true);
+
+ return true;
+ }
+ }
+
+ return false;
+}
+
+void
+ucl_comments_add (ucl_object_t *comments, const ucl_object_t *obj,
+ const char *comment)
+{
+ if (comments && obj && comment) {
+ ucl_object_insert_key (comments, ucl_object_fromstring (comment),
+ (const char *)&obj, sizeof (void *), true);
+ }
+}
+
+void
+ucl_parser_set_include_tracer (struct ucl_parser *parser,
+ ucl_include_trace_func_t func,
+ void *user_data)
+{
+ parser->include_trace_func = func;
+ parser->include_trace_ud = user_data;
+}
+
+const char *
+ucl_parser_get_cur_file (struct ucl_parser *parser)
+{
+ return parser->cur_file;
+} \ No newline at end of file
diff --git a/contrib/lua-argparse/LICENSE b/contrib/lua-argparse/LICENSE
new file mode 100644
index 0000000..87579ac
--- /dev/null
+++ b/contrib/lua-argparse/LICENSE
@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 - 2018 Peter Melnichenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/contrib/lua-argparse/argparse.lua b/contrib/lua-argparse/argparse.lua
new file mode 100644
index 0000000..6b52962
--- /dev/null
+++ b/contrib/lua-argparse/argparse.lua
@@ -0,0 +1,2100 @@
+-- The MIT License (MIT)
+
+-- Copyright (c) 2013 - 2018 Peter Melnichenko
+-- 2019 Paul Ouellette
+
+-- Permission is hereby granted, free of charge, to any person obtaining a copy of
+-- this software and associated documentation files (the "Software"), to deal in
+-- the Software without restriction, including without limitation the rights to
+-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+-- the Software, and to permit persons to whom the Software is furnished to do so,
+-- subject to the following conditions:
+
+-- The above copyright notice and this permission notice shall be included in all
+-- copies or substantial portions of the Software.
+
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+-- FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+-- COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+-- IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+-- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+local function deep_update(t1, t2)
+ for k, v in pairs(t2) do
+ if type(v) == "table" then
+ v = deep_update({}, v)
+ end
+
+ t1[k] = v
+ end
+
+ return t1
+end
+
+-- A property is a tuple {name, callback}.
+-- properties.args is number of properties that can be set as arguments
+-- when calling an object.
+local function class(prototype, properties, parent)
+ -- Class is the metatable of its instances.
+ local cl = {}
+ cl.__index = cl
+
+ if parent then
+ cl.__prototype = deep_update(deep_update({}, parent.__prototype), prototype)
+ else
+ cl.__prototype = prototype
+ end
+
+ if properties then
+ local names = {}
+
+ -- Create setter methods and fill set of property names.
+ for _, property in ipairs(properties) do
+ local name, callback = property[1], property[2]
+
+ cl[name] = function(self, value)
+ if not callback(self, value) then
+ self["_" .. name] = value
+ end
+
+ return self
+ end
+
+ names[name] = true
+ end
+
+ function cl.__call(self, ...)
+ -- When calling an object, if the first argument is a table,
+ -- interpret keys as property names, else delegate arguments
+ -- to corresponding setters in order.
+ if type((...)) == "table" then
+ for name, value in pairs((...)) do
+ if names[name] then
+ self[name](self, value)
+ end
+ end
+ else
+ local nargs = select("#", ...)
+
+ for i, property in ipairs(properties) do
+ if i > nargs or i > properties.args then
+ break
+ end
+
+ local arg = select(i, ...)
+
+ if arg ~= nil then
+ self[property[1]](self, arg)
+ end
+ end
+ end
+
+ return self
+ end
+ end
+
+ -- If indexing class fails, fallback to its parent.
+ local class_metatable = {}
+ class_metatable.__index = parent
+
+ function class_metatable.__call(self, ...)
+ -- Calling a class returns its instance.
+ -- Arguments are delegated to the instance.
+ local object = deep_update({}, self.__prototype)
+ setmetatable(object, self)
+ return object(...)
+ end
+
+ return setmetatable(cl, class_metatable)
+end
+
+local function typecheck(name, types, value)
+ for _, type_ in ipairs(types) do
+ if type(value) == type_ then
+ return true
+ end
+ end
+
+ error(("bad property '%s' (%s expected, got %s)"):format(name, table.concat(types, " or "), type(value)))
+end
+
+local function typechecked(name, ...)
+ local types = {...}
+ return {name, function(_, value) typecheck(name, types, value) end}
+end
+
+local multiname = {"name", function(self, value)
+ typecheck("name", {"string"}, value)
+
+ for alias in value:gmatch("%S+") do
+ self._name = self._name or alias
+ table.insert(self._aliases, alias)
+ table.insert(self._public_aliases, alias)
+ -- If alias contains '_', accept '-' also.
+ if alias:find("_", 1, true) then
+ table.insert(self._aliases, (alias:gsub("_", "-")))
+ end
+ end
+
+ -- Do not set _name as with other properties.
+ return true
+end}
+
+local multiname_hidden = {"hidden_name", function(self, value)
+ typecheck("hidden_name", {"string"}, value)
+
+ for alias in value:gmatch("%S+") do
+ table.insert(self._aliases, alias)
+ if alias:find("_", 1, true) then
+ table.insert(self._aliases, (alias:gsub("_", "-")))
+ end
+ end
+
+ return true
+end}
+
+local function parse_boundaries(str)
+ if tonumber(str) then
+ return tonumber(str), tonumber(str)
+ end
+
+ if str == "*" then
+ return 0, math.huge
+ end
+
+ if str == "+" then
+ return 1, math.huge
+ end
+
+ if str == "?" then
+ return 0, 1
+ end
+
+ if str:match "^%d+%-%d+$" then
+ local min, max = str:match "^(%d+)%-(%d+)$"
+ return tonumber(min), tonumber(max)
+ end
+
+ if str:match "^%d+%+$" then
+ local min = str:match "^(%d+)%+$"
+ return tonumber(min), math.huge
+ end
+end
+
+local function boundaries(name)
+ return {name, function(self, value)
+ typecheck(name, {"number", "string"}, value)
+
+ local min, max = parse_boundaries(value)
+
+ if not min then
+ error(("bad property '%s'"):format(name))
+ end
+
+ self["_min" .. name], self["_max" .. name] = min, max
+ end}
+end
+
+local actions = {}
+
+local option_action = {"action", function(_, value)
+ typecheck("action", {"function", "string"}, value)
+
+ if type(value) == "string" and not actions[value] then
+ error(("unknown action '%s'"):format(value))
+ end
+end}
+
+local option_init = {"init", function(self)
+ self._has_init = true
+end}
+
+local option_default = {"default", function(self, value)
+ if type(value) ~= "string" then
+ self._init = value
+ self._has_init = true
+ return true
+ end
+end}
+
+local add_help = {"add_help", function(self, value)
+ typecheck("add_help", {"boolean", "string", "table"}, value)
+
+ if self._help_option_idx then
+ table.remove(self._options, self._help_option_idx)
+ self._help_option_idx = nil
+ end
+
+ if value then
+ local help = self:flag()
+ :description "Show this help message and exit."
+ :action(function()
+ print(self:get_help())
+ os.exit(0)
+ end)
+
+ if value ~= true then
+ help = help(value)
+ end
+
+ if not help._name then
+ help "-h" "--help"
+ end
+
+ self._help_option_idx = #self._options
+ end
+end}
+
+local Parser = class({
+ _arguments = {},
+ _options = {},
+ _commands = {},
+ _mutexes = {},
+ _groups = {},
+ _require_command = true,
+ _handle_options = true
+}, {
+ args = 3,
+ typechecked("name", "string"),
+ typechecked("description", "string"),
+ typechecked("epilog", "string"),
+ typechecked("usage", "string"),
+ typechecked("help", "string"),
+ typechecked("require_command", "boolean"),
+ typechecked("handle_options", "boolean"),
+ typechecked("action", "function"),
+ typechecked("command_target", "string"),
+ typechecked("help_vertical_space", "number"),
+ typechecked("usage_margin", "number"),
+ typechecked("usage_max_width", "number"),
+ typechecked("help_usage_margin", "number"),
+ typechecked("help_description_margin", "number"),
+ typechecked("help_max_width", "number"),
+ add_help
+})
+
+local Command = class({
+ _aliases = {},
+ _public_aliases = {}
+}, {
+ args = 3,
+ multiname,
+ typechecked("description", "string"),
+ typechecked("epilog", "string"),
+ multiname_hidden,
+ typechecked("summary", "string"),
+ typechecked("target", "string"),
+ typechecked("usage", "string"),
+ typechecked("help", "string"),
+ typechecked("require_command", "boolean"),
+ typechecked("handle_options", "boolean"),
+ typechecked("action", "function"),
+ typechecked("command_target", "string"),
+ typechecked("help_vertical_space", "number"),
+ typechecked("usage_margin", "number"),
+ typechecked("usage_max_width", "number"),
+ typechecked("help_usage_margin", "number"),
+ typechecked("help_description_margin", "number"),
+ typechecked("help_max_width", "number"),
+ typechecked("hidden", "boolean"),
+ add_help
+}, Parser)
+
+local Argument = class({
+ _minargs = 1,
+ _maxargs = 1,
+ _mincount = 1,
+ _maxcount = 1,
+ _defmode = "unused",
+ _show_default = true
+}, {
+ args = 5,
+ typechecked("name", "string"),
+ typechecked("description", "string"),
+ option_default,
+ typechecked("convert", "function", "table"),
+ boundaries("args"),
+ typechecked("target", "string"),
+ typechecked("defmode", "string"),
+ typechecked("show_default", "boolean"),
+ typechecked("argname", "string", "table"),
+ typechecked("choices", "table"),
+ typechecked("hidden", "boolean"),
+ option_action,
+ option_init
+})
+
+local Option = class({
+ _aliases = {},
+ _public_aliases = {},
+ _mincount = 0,
+ _overwrite = true
+}, {
+ args = 6,
+ multiname,
+ typechecked("description", "string"),
+ option_default,
+ typechecked("convert", "function", "table"),
+ boundaries("args"),
+ boundaries("count"),
+ multiname_hidden,
+ typechecked("target", "string"),
+ typechecked("defmode", "string"),
+ typechecked("show_default", "boolean"),
+ typechecked("overwrite", "boolean"),
+ typechecked("argname", "string", "table"),
+ typechecked("choices", "table"),
+ typechecked("hidden", "boolean"),
+ option_action,
+ option_init
+}, Argument)
+
+function Parser:_inherit_property(name, default)
+ local element = self
+
+ while true do
+ local value = element["_" .. name]
+
+ if value ~= nil then
+ return value
+ end
+
+ if not element._parent then
+ return default
+ end
+
+ element = element._parent
+ end
+end
+
+function Argument:_get_argument_list()
+ local buf = {}
+ local i = 1
+
+ while i <= math.min(self._minargs, 3) do
+ local argname = self:_get_argname(i)
+
+ if self._default and self._defmode:find "a" then
+ argname = "[" .. argname .. "]"
+ end
+
+ table.insert(buf, argname)
+ i = i+1
+ end
+
+ while i <= math.min(self._maxargs, 3) do
+ table.insert(buf, "[" .. self:_get_argname(i) .. "]")
+ i = i+1
+
+ if self._maxargs == math.huge then
+ break
+ end
+ end
+
+ if i < self._maxargs then
+ table.insert(buf, "...")
+ end
+
+ return buf
+end
+
+function Argument:_get_usage()
+ local usage = table.concat(self:_get_argument_list(), " ")
+
+ if self._default and self._defmode:find "u" then
+ if self._maxargs > 1 or (self._minargs == 1 and not self._defmode:find "a") then
+ usage = "[" .. usage .. "]"
+ end
+ end
+
+ return usage
+end
+
+function actions.store_true(result, target)
+ result[target] = true
+end
+
+function actions.store_false(result, target)
+ result[target] = false
+end
+
+function actions.store(result, target, argument)
+ result[target] = argument
+end
+
+function actions.count(result, target, _, overwrite)
+ if not overwrite then
+ result[target] = result[target] + 1
+ end
+end
+
+function actions.append(result, target, argument, overwrite)
+ result[target] = result[target] or {}
+ table.insert(result[target], argument)
+
+ if overwrite then
+ table.remove(result[target], 1)
+ end
+end
+
+function actions.concat(result, target, arguments, overwrite)
+ if overwrite then
+ error("'concat' action can't handle too many invocations")
+ end
+
+ result[target] = result[target] or {}
+
+ for _, argument in ipairs(arguments) do
+ table.insert(result[target], argument)
+ end
+end
+
+function Argument:_get_action()
+ local action, init
+
+ if self._maxcount == 1 then
+ if self._maxargs == 0 then
+ action, init = "store_true", nil
+ else
+ action, init = "store", nil
+ end
+ else
+ if self._maxargs == 0 then
+ action, init = "count", 0
+ else
+ action, init = "append", {}
+ end
+ end
+
+ if self._action then
+ action = self._action
+ end
+
+ if self._has_init then
+ init = self._init
+ end
+
+ if type(action) == "string" then
+ action = actions[action]
+ end
+
+ return action, init
+end
+
+-- Returns placeholder for `narg`-th argument.
+function Argument:_get_argname(narg)
+ local argname = self._argname or self:_get_default_argname()
+
+ if type(argname) == "table" then
+ return argname[narg]
+ else
+ return argname
+ end
+end
+
+function Argument:_get_choices_list()
+ return "{" .. table.concat(self._choices, ",") .. "}"
+end
+
+function Argument:_get_default_argname()
+ if self._choices then
+ return self:_get_choices_list()
+ else
+ return "<" .. self._name .. ">"
+ end
+end
+
+function Option:_get_default_argname()
+ if self._choices then
+ return self:_get_choices_list()
+ else
+ return "<" .. self:_get_default_target() .. ">"
+ end
+end
+
+-- Returns labels to be shown in the help message.
+function Argument:_get_label_lines()
+ if self._choices then
+ return {self:_get_choices_list()}
+ else
+ return {self._name}
+ end
+end
+
+function Option:_get_label_lines()
+ local argument_list = self:_get_argument_list()
+
+ if #argument_list == 0 then
+ -- Don't put aliases for simple flags like `-h` on different lines.
+ return {table.concat(self._public_aliases, ", ")}
+ end
+
+ local longest_alias_length = -1
+
+ for _, alias in ipairs(self._public_aliases) do
+ longest_alias_length = math.max(longest_alias_length, #alias)
+ end
+
+ local argument_list_repr = table.concat(argument_list, " ")
+ local lines = {}
+
+ for i, alias in ipairs(self._public_aliases) do
+ local line = (" "):rep(longest_alias_length - #alias) .. alias .. " " .. argument_list_repr
+
+ if i ~= #self._public_aliases then
+ line = line .. ","
+ end
+
+ table.insert(lines, line)
+ end
+
+ return lines
+end
+
+function Command:_get_label_lines()
+ return {table.concat(self._public_aliases, ", ")}
+end
+
+function Argument:_get_description()
+ if self._default and self._show_default then
+ if self._description then
+ return ("%s (default: %s)"):format(self._description, self._default)
+ else
+ return ("default: %s"):format(self._default)
+ end
+ else
+ return self._description or ""
+ end
+end
+
+function Command:_get_description()
+ return self._summary or self._description or ""
+end
+
+function Option:_get_usage()
+ local usage = self:_get_argument_list()
+ table.insert(usage, 1, self._name)
+ usage = table.concat(usage, " ")
+
+ if self._mincount == 0 or self._default then
+ usage = "[" .. usage .. "]"
+ end
+
+ return usage
+end
+
+function Argument:_get_default_target()
+ return self._name
+end
+
+function Option:_get_default_target()
+ local res
+
+ for _, alias in ipairs(self._public_aliases) do
+ if alias:sub(1, 1) == alias:sub(2, 2) then
+ res = alias:sub(3)
+ break
+ end
+ end
+
+ res = res or self._name:sub(2)
+ return (res:gsub("-", "_"))
+end
+
+function Option:_is_vararg()
+ return self._maxargs ~= self._minargs
+end
+
+function Parser:_get_fullname(exclude_root)
+ local parent = self._parent
+ if exclude_root and not parent then
+ return ""
+ end
+ local buf = {self._name}
+
+ while parent do
+ if not exclude_root or parent._parent then
+ table.insert(buf, 1, parent._name)
+ end
+ parent = parent._parent
+ end
+
+ return table.concat(buf, " ")
+end
+
+function Parser:_update_charset(charset)
+ charset = charset or {}
+
+ for _, command in ipairs(self._commands) do
+ command:_update_charset(charset)
+ end
+
+ for _, option in ipairs(self._options) do
+ for _, alias in ipairs(option._aliases) do
+ charset[alias:sub(1, 1)] = true
+ end
+ end
+
+ return charset
+end
+
+function Parser:argument(...)
+ local argument = Argument(...)
+ table.insert(self._arguments, argument)
+ return argument
+end
+
+function Parser:option(...)
+ local option = Option(...)
+ table.insert(self._options, option)
+ return option
+end
+
+function Parser:flag(...)
+ return self:option():args(0)(...)
+end
+
+function Parser:command(...)
+ local command = Command():add_help(true)(...)
+ command._parent = self
+ table.insert(self._commands, command)
+ return command
+end
+
+function Parser:mutex(...)
+ local elements = {...}
+
+ for i, element in ipairs(elements) do
+ local mt = getmetatable(element)
+ assert(mt == Option or mt == Argument, ("bad argument #%d to 'mutex' (Option or Argument expected)"):format(i))
+ end
+
+ table.insert(self._mutexes, elements)
+ return self
+end
+
+function Parser:group(name, ...)
+ assert(type(name) == "string", ("bad argument #1 to 'group' (string expected, got %s)"):format(type(name)))
+
+ local group = {name = name, ...}
+
+ for i, element in ipairs(group) do
+ local mt = getmetatable(element)
+ assert(mt == Option or mt == Argument or mt == Command,
+ ("bad argument #%d to 'group' (Option or Argument or Command expected)"):format(i + 1))
+ end
+
+ table.insert(self._groups, group)
+ return self
+end
+
+local usage_welcome = "Usage: "
+
+function Parser:get_usage()
+ if self._usage then
+ return self._usage
+ end
+
+ local usage_margin = self:_inherit_property("usage_margin", #usage_welcome)
+ local max_usage_width = self:_inherit_property("usage_max_width", 70)
+ local lines = {usage_welcome .. self:_get_fullname()}
+
+ local function add(s)
+ if #lines[#lines]+1+#s <= max_usage_width then
+ lines[#lines] = lines[#lines] .. " " .. s
+ else
+ lines[#lines+1] = (" "):rep(usage_margin) .. s
+ end
+ end
+
+ -- Normally options are before positional arguments in usage messages.
+ -- However, vararg options should be after, because they can't be reliable used
+ -- before a positional argument.
+ -- Mutexes come into play, too, and are shown as soon as possible.
+ -- Overall, output usages in the following order:
+ -- 1. Mutexes that don't have positional arguments or vararg options.
+ -- 2. Options that are not in any mutexes and are not vararg.
+ -- 3. Positional arguments - on their own or as a part of a mutex.
+ -- 4. Remaining mutexes.
+ -- 5. Remaining options.
+
+ local elements_in_mutexes = {}
+ local added_elements = {}
+ local added_mutexes = {}
+ local argument_to_mutexes = {}
+
+ local function add_mutex(mutex, main_argument)
+ if added_mutexes[mutex] then
+ return
+ end
+
+ added_mutexes[mutex] = true
+ local buf = {}
+
+ for _, element in ipairs(mutex) do
+ if not element._hidden and not added_elements[element] then
+ if getmetatable(element) == Option or element == main_argument then
+ table.insert(buf, element:_get_usage())
+ added_elements[element] = true
+ end
+ end
+ end
+
+ if #buf == 1 then
+ add(buf[1])
+ elseif #buf > 1 then
+ add("(" .. table.concat(buf, " | ") .. ")")
+ end
+ end
+
+ local function add_element(element)
+ if not element._hidden and not added_elements[element] then
+ add(element:_get_usage())
+ added_elements[element] = true
+ end
+ end
+
+ for _, mutex in ipairs(self._mutexes) do
+ local is_vararg = false
+ local has_argument = false
+
+ for _, element in ipairs(mutex) do
+ if getmetatable(element) == Option then
+ if element:_is_vararg() then
+ is_vararg = true
+ end
+ else
+ has_argument = true
+ argument_to_mutexes[element] = argument_to_mutexes[element] or {}
+ table.insert(argument_to_mutexes[element], mutex)
+ end
+
+ elements_in_mutexes[element] = true
+ end
+
+ if not is_vararg and not has_argument then
+ add_mutex(mutex)
+ end
+ end
+
+ for _, option in ipairs(self._options) do
+ if not elements_in_mutexes[option] and not option:_is_vararg() then
+ add_element(option)
+ end
+ end
+
+ -- Add usages for positional arguments, together with one mutex containing them, if they are in a mutex.
+ for _, argument in ipairs(self._arguments) do
+ -- Pick a mutex as a part of which to show this argument, take the first one that's still available.
+ local mutex
+
+ if elements_in_mutexes[argument] then
+ for _, argument_mutex in ipairs(argument_to_mutexes[argument]) do
+ if not added_mutexes[argument_mutex] then
+ mutex = argument_mutex
+ end
+ end
+ end
+
+ if mutex then
+ add_mutex(mutex, argument)
+ else
+ add_element(argument)
+ end
+ end
+
+ for _, mutex in ipairs(self._mutexes) do
+ add_mutex(mutex)
+ end
+
+ for _, option in ipairs(self._options) do
+ add_element(option)
+ end
+
+ if #self._commands > 0 then
+ if self._require_command then
+ add("<command>")
+ else
+ add("[<command>]")
+ end
+
+ add("...")
+ end
+
+ return table.concat(lines, "\n")
+end
+
+local function split_lines(s)
+ if s == "" then
+ return {}
+ end
+
+ local lines = {}
+
+ if s:sub(-1) ~= "\n" then
+ s = s .. "\n"
+ end
+
+ for line in s:gmatch("([^\n]*)\n") do
+ table.insert(lines, line)
+ end
+
+ return lines
+end
+
+local function autowrap_line(line, max_length)
+ -- Algorithm for splitting lines is simple and greedy.
+ local result_lines = {}
+
+ -- Preserve original indentation of the line, put this at the beginning of each result line.
+ -- If the first word looks like a list marker ('*', '+', or '-'), add spaces so that starts
+ -- of the second and the following lines vertically align with the start of the second word.
+ local indentation = line:match("^ *")
+
+ if line:find("^ *[%*%+%-]") then
+ indentation = indentation .. " " .. line:match("^ *[%*%+%-]( *)")
+ end
+
+ -- Parts of the last line being assembled.
+ local line_parts = {}
+
+ -- Length of the current line.
+ local line_length = 0
+
+ -- Index of the next character to consider.
+ local index = 1
+
+ while true do
+ local word_start, word_finish, word = line:find("([^ ]+)", index)
+
+ if not word_start then
+ -- Ignore trailing spaces, if any.
+ break
+ end
+
+ local preceding_spaces = line:sub(index, word_start - 1)
+ index = word_finish + 1
+
+ if (#line_parts == 0) or (line_length + #preceding_spaces + #word <= max_length) then
+ -- Either this is the very first word or it fits as an addition to the current line, add it.
+ table.insert(line_parts, preceding_spaces) -- For the very first word this adds the indentation.
+ table.insert(line_parts, word)
+ line_length = line_length + #preceding_spaces + #word
+ else
+ -- Does not fit, finish current line and put the word into a new one.
+ table.insert(result_lines, table.concat(line_parts))
+ line_parts = {indentation, word}
+ line_length = #indentation + #word
+ end
+ end
+
+ if #line_parts > 0 then
+ table.insert(result_lines, table.concat(line_parts))
+ end
+
+ if #result_lines == 0 then
+ -- Preserve empty lines.
+ result_lines[1] = ""
+ end
+
+ return result_lines
+end
+
+-- Automatically wraps lines within given array,
+-- attempting to limit line length to `max_length`.
+-- Existing line splits are preserved.
+local function autowrap(lines, max_length)
+ local result_lines = {}
+
+ for _, line in ipairs(lines) do
+ local autowrapped_lines = autowrap_line(line, max_length)
+
+ for _, autowrapped_line in ipairs(autowrapped_lines) do
+ table.insert(result_lines, autowrapped_line)
+ end
+ end
+
+ return result_lines
+end
+
+function Parser:_get_element_help(element)
+ local label_lines = element:_get_label_lines()
+ local description_lines = split_lines(element:_get_description())
+
+ local result_lines = {}
+
+ -- All label lines should have the same length (except the last one, it has no comma).
+ -- If too long, start description after all the label lines.
+ -- Otherwise, combine label and description lines.
+
+ local usage_margin_len = self:_inherit_property("help_usage_margin", 3)
+ local usage_margin = (" "):rep(usage_margin_len)
+ local description_margin_len = self:_inherit_property("help_description_margin", 25)
+ local description_margin = (" "):rep(description_margin_len)
+
+ local help_max_width = self:_inherit_property("help_max_width")
+
+ if help_max_width then
+ local description_max_width = math.max(help_max_width - description_margin_len, 10)
+ description_lines = autowrap(description_lines, description_max_width)
+ end
+
+ if #label_lines[1] >= (description_margin_len - usage_margin_len) then
+ for _, label_line in ipairs(label_lines) do
+ table.insert(result_lines, usage_margin .. label_line)
+ end
+
+ for _, description_line in ipairs(description_lines) do
+ table.insert(result_lines, description_margin .. description_line)
+ end
+ else
+ for i = 1, math.max(#label_lines, #description_lines) do
+ local label_line = label_lines[i]
+ local description_line = description_lines[i]
+
+ local line = ""
+
+ if label_line then
+ line = usage_margin .. label_line
+ end
+
+ if description_line and description_line ~= "" then
+ line = line .. (" "):rep(description_margin_len - #line) .. description_line
+ end
+
+ table.insert(result_lines, line)
+ end
+ end
+
+ return table.concat(result_lines, "\n")
+end
+
+local function get_group_types(group)
+ local types = {}
+
+ for _, element in ipairs(group) do
+ types[getmetatable(element)] = true
+ end
+
+ return types
+end
+
+function Parser:_add_group_help(blocks, added_elements, label, elements)
+ local buf = {label}
+
+ for _, element in ipairs(elements) do
+ if not element._hidden and not added_elements[element] then
+ added_elements[element] = true
+ table.insert(buf, self:_get_element_help(element))
+ end
+ end
+
+ if #buf > 1 then
+ table.insert(blocks, table.concat(buf, ("\n"):rep(self:_inherit_property("help_vertical_space", 0) + 1)))
+ end
+end
+
+function Parser:get_help()
+ if self._help then
+ return self._help
+ end
+
+ local blocks = {self:get_usage()}
+
+ local help_max_width = self:_inherit_property("help_max_width")
+
+ if self._description then
+ local description = self._description
+
+ if help_max_width then
+ description = table.concat(autowrap(split_lines(description), help_max_width), "\n")
+ end
+
+ table.insert(blocks, description)
+ end
+
+ -- 1. Put groups containing arguments first, then other arguments.
+ -- 2. Put remaining groups containing options, then other options.
+ -- 3. Put remaining groups containing commands, then other commands.
+ -- Assume that an element can't be in several groups.
+ local groups_by_type = {
+ [Argument] = {},
+ [Option] = {},
+ [Command] = {}
+ }
+
+ for _, group in ipairs(self._groups) do
+ local group_types = get_group_types(group)
+
+ for _, mt in ipairs({Argument, Option, Command}) do
+ if group_types[mt] then
+ table.insert(groups_by_type[mt], group)
+ break
+ end
+ end
+ end
+
+ local default_groups = {
+ {name = "Arguments", type = Argument, elements = self._arguments},
+ {name = "Options", type = Option, elements = self._options},
+ {name = "Commands", type = Command, elements = self._commands}
+ }
+
+ local added_elements = {}
+
+ for _, default_group in ipairs(default_groups) do
+ local type_groups = groups_by_type[default_group.type]
+
+ for _, group in ipairs(type_groups) do
+ self:_add_group_help(blocks, added_elements, group.name .. ":", group)
+ end
+
+ local default_label = default_group.name .. ":"
+
+ if #type_groups > 0 then
+ default_label = "Other " .. default_label:gsub("^.", string.lower)
+ end
+
+ self:_add_group_help(blocks, added_elements, default_label, default_group.elements)
+ end
+
+ if self._epilog then
+ local epilog = self._epilog
+
+ if help_max_width then
+ epilog = table.concat(autowrap(split_lines(epilog), help_max_width), "\n")
+ end
+
+ table.insert(blocks, epilog)
+ end
+
+ return table.concat(blocks, "\n\n")
+end
+
+function Parser:add_help_command(value)
+ if value then
+ assert(type(value) == "string" or type(value) == "table",
+ ("bad argument #1 to 'add_help_command' (string or table expected, got %s)"):format(type(value)))
+ end
+
+ local help = self:command()
+ :description "Show help for commands."
+ help:argument "command"
+ :description "The command to show help for."
+ :args "?"
+ :action(function(_, _, cmd)
+ if not cmd then
+ print(self:get_help())
+ os.exit(0)
+ else
+ for _, command in ipairs(self._commands) do
+ for _, alias in ipairs(command._aliases) do
+ if alias == cmd then
+ print(command:get_help())
+ os.exit(0)
+ end
+ end
+ end
+ end
+ help:error(("unknown command '%s'"):format(cmd))
+ end)
+
+ if value then
+ help = help(value)
+ end
+
+ if not help._name then
+ help "help"
+ end
+
+ help._is_help_command = true
+ return self
+end
+
+function Parser:_is_shell_safe()
+ if self._basename then
+ if self._basename:find("[^%w_%-%+%.]") then
+ return false
+ end
+ else
+ for _, alias in ipairs(self._aliases) do
+ if alias:find("[^%w_%-%+%.]") then
+ return false
+ end
+ end
+ end
+ for _, option in ipairs(self._options) do
+ for _, alias in ipairs(option._aliases) do
+ if alias:find("[^%w_%-%+%.]") then
+ return false
+ end
+ end
+ if option._choices then
+ for _, choice in ipairs(option._choices) do
+ if choice:find("[%s'\"]") then
+ return false
+ end
+ end
+ end
+ end
+ for _, argument in ipairs(self._arguments) do
+ if argument._choices then
+ for _, choice in ipairs(argument._choices) do
+ if choice:find("[%s'\"]") then
+ return false
+ end
+ end
+ end
+ end
+ for _, command in ipairs(self._commands) do
+ if not command:_is_shell_safe() then
+ return false
+ end
+ end
+ return true
+end
+
+function Parser:add_complete(value)
+ if value then
+ assert(type(value) == "string" or type(value) == "table",
+ ("bad argument #1 to 'add_complete' (string or table expected, got %s)"):format(type(value)))
+ end
+
+ local complete = self:option()
+ :description "Output a shell completion script for the specified shell."
+ :args(1)
+ :choices {"bash", "zsh", "fish"}
+ :action(function(_, _, shell)
+ io.write(self["get_" .. shell .. "_complete"](self))
+ os.exit(0)
+ end)
+
+ if value then
+ complete = complete(value)
+ end
+
+ if not complete._name then
+ complete "--completion"
+ end
+
+ return self
+end
+
+function Parser:add_complete_command(value)
+ if value then
+ assert(type(value) == "string" or type(value) == "table",
+ ("bad argument #1 to 'add_complete_command' (string or table expected, got %s)"):format(type(value)))
+ end
+
+ local complete = self:command()
+ :description "Output a shell completion script."
+ complete:argument "shell"
+ :description "The shell to output a completion script for."
+ :choices {"bash", "zsh", "fish"}
+ :action(function(_, _, shell)
+ io.write(self["get_" .. shell .. "_complete"](self))
+ os.exit(0)
+ end)
+
+ if value then
+ complete = complete(value)
+ end
+
+ if not complete._name then
+ complete "completion"
+ end
+
+ return self
+end
+
+local function base_name(pathname)
+ return pathname:gsub("[/\\]*$", ""):match(".*[/\\]([^/\\]*)") or pathname
+end
+
+local function get_short_description(element)
+ local short = element:_get_description():match("^(.-)%.%s")
+ return short or element:_get_description():match("^(.-)%.?$")
+end
+
+function Parser:_get_options()
+ local options = {}
+ for _, option in ipairs(self._options) do
+ for _, alias in ipairs(option._aliases) do
+ table.insert(options, alias)
+ end
+ end
+ return table.concat(options, " ")
+end
+
+function Parser:_get_commands()
+ local commands = {}
+ for _, command in ipairs(self._commands) do
+ for _, alias in ipairs(command._aliases) do
+ table.insert(commands, alias)
+ end
+ end
+ return table.concat(commands, " ")
+end
+
+function Parser:_bash_option_args(buf, indent)
+ local opts = {}
+ for _, option in ipairs(self._options) do
+ if option._choices or option._minargs > 0 then
+ local compreply
+ if option._choices then
+ compreply = 'COMPREPLY=($(compgen -W "' .. table.concat(option._choices, " ") .. '" -- "$cur"))'
+ else
+ compreply = 'COMPREPLY=($(compgen -f -- "$cur"))'
+ end
+ table.insert(opts, (" "):rep(indent + 4) .. table.concat(option._aliases, "|") .. ")")
+ table.insert(opts, (" "):rep(indent + 8) .. compreply)
+ table.insert(opts, (" "):rep(indent + 8) .. "return 0")
+ table.insert(opts, (" "):rep(indent + 8) .. ";;")
+ end
+ end
+
+ if #opts > 0 then
+ table.insert(buf, (" "):rep(indent) .. 'case "$prev" in')
+ table.insert(buf, table.concat(opts, "\n"))
+ table.insert(buf, (" "):rep(indent) .. "esac\n")
+ end
+end
+
+function Parser:_bash_get_cmd(buf, indent)
+ if #self._commands == 0 then
+ return
+ end
+
+ table.insert(buf, (" "):rep(indent) .. 'args=("${args[@]:1}")')
+ table.insert(buf, (" "):rep(indent) .. 'for arg in "${args[@]}"; do')
+ table.insert(buf, (" "):rep(indent + 4) .. 'case "$arg" in')
+
+ for _, command in ipairs(self._commands) do
+ table.insert(buf, (" "):rep(indent + 8) .. table.concat(command._aliases, "|") .. ")")
+ if self._parent then
+ table.insert(buf, (" "):rep(indent + 12) .. 'cmd="$cmd ' .. command._name .. '"')
+ else
+ table.insert(buf, (" "):rep(indent + 12) .. 'cmd="' .. command._name .. '"')
+ end
+ table.insert(buf, (" "):rep(indent + 12) .. 'opts="$opts ' .. command:_get_options() .. '"')
+ command:_bash_get_cmd(buf, indent + 12)
+ table.insert(buf, (" "):rep(indent + 12) .. "break")
+ table.insert(buf, (" "):rep(indent + 12) .. ";;")
+ end
+
+ table.insert(buf, (" "):rep(indent + 4) .. "esac")
+ table.insert(buf, (" "):rep(indent) .. "done")
+end
+
+function Parser:_bash_cmd_completions(buf)
+ local cmd_buf = {}
+ if self._parent then
+ self:_bash_option_args(cmd_buf, 12)
+ end
+ if #self._commands > 0 then
+ table.insert(cmd_buf, (" "):rep(12) .. 'COMPREPLY=($(compgen -W "' .. self:_get_commands() .. '" -- "$cur"))')
+ elseif self._is_help_command then
+ table.insert(cmd_buf, (" "):rep(12)
+ .. 'COMPREPLY=($(compgen -W "'
+ .. self._parent:_get_commands()
+ .. '" -- "$cur"))')
+ end
+ if #cmd_buf > 0 then
+ table.insert(buf, (" "):rep(8) .. "'" .. self:_get_fullname(true) .. "')")
+ table.insert(buf, table.concat(cmd_buf, "\n"))
+ table.insert(buf, (" "):rep(12) .. ";;")
+ end
+
+ for _, command in ipairs(self._commands) do
+ command:_bash_cmd_completions(buf)
+ end
+end
+
+function Parser:get_bash_complete()
+ self._basename = base_name(self._name)
+ assert(self:_is_shell_safe())
+ local buf = {([[
+_%s() {
+ local IFS=$' \t\n'
+ local args cur prev cmd opts arg
+ args=("${COMP_WORDS[@]}")
+ cur="${COMP_WORDS[COMP_CWORD]}"
+ prev="${COMP_WORDS[COMP_CWORD-1]}"
+ opts="%s"
+]]):format(self._basename, self:_get_options())}
+
+ self:_bash_option_args(buf, 4)
+ self:_bash_get_cmd(buf, 4)
+ if #self._commands > 0 then
+ table.insert(buf, "")
+ table.insert(buf, (" "):rep(4) .. 'case "$cmd" in')
+ self:_bash_cmd_completions(buf)
+ table.insert(buf, (" "):rep(4) .. "esac\n")
+ end
+
+ table.insert(buf, ([=[
+ if [[ "$cur" = -* ]]; then
+ COMPREPLY=($(compgen -W "$opts" -- "$cur"))
+ fi
+}
+
+complete -F _%s -o bashdefault -o default %s
+]=]):format(self._basename, self._basename))
+
+ return table.concat(buf, "\n")
+end
+
+function Parser:_zsh_arguments(buf, cmd_name, indent)
+ if self._parent then
+ table.insert(buf, (" "):rep(indent) .. "options=(")
+ table.insert(buf, (" "):rep(indent + 2) .. "$options")
+ else
+ table.insert(buf, (" "):rep(indent) .. "local -a options=(")
+ end
+
+ for _, option in ipairs(self._options) do
+ local line = {}
+ if #option._aliases > 1 then
+ if option._maxcount > 1 then
+ table.insert(line, '"*"')
+ end
+ table.insert(line, "{" .. table.concat(option._aliases, ",") .. '}"')
+ else
+ table.insert(line, '"')
+ if option._maxcount > 1 then
+ table.insert(line, "*")
+ end
+ table.insert(line, option._name)
+ end
+ if option._description then
+ local description = get_short_description(option):gsub('["%]:`$]', "\\%0")
+ table.insert(line, "[" .. description .. "]")
+ end
+ if option._maxargs == math.huge then
+ table.insert(line, ":*")
+ end
+ if option._choices then
+ table.insert(line, ": :(" .. table.concat(option._choices, " ") .. ")")
+ elseif option._maxargs > 0 then
+ table.insert(line, ": :_files")
+ end
+ table.insert(line, '"')
+ table.insert(buf, (" "):rep(indent + 2) .. table.concat(line))
+ end
+
+ table.insert(buf, (" "):rep(indent) .. ")")
+ table.insert(buf, (" "):rep(indent) .. "_arguments -s -S \\")
+ table.insert(buf, (" "):rep(indent + 2) .. "$options \\")
+
+ if self._is_help_command then
+ table.insert(buf, (" "):rep(indent + 2) .. '": :(' .. self._parent:_get_commands() .. ')" \\')
+ else
+ for _, argument in ipairs(self._arguments) do
+ local spec
+ if argument._choices then
+ spec = ": :(" .. table.concat(argument._choices, " ") .. ")"
+ else
+ spec = ": :_files"
+ end
+ if argument._maxargs == math.huge then
+ table.insert(buf, (" "):rep(indent + 2) .. '"*' .. spec .. '" \\')
+ break
+ end
+ for _ = 1, argument._maxargs do
+ table.insert(buf, (" "):rep(indent + 2) .. '"' .. spec .. '" \\')
+ end
+ end
+
+ if #self._commands > 0 then
+ table.insert(buf, (" "):rep(indent + 2) .. '": :_' .. cmd_name .. '_cmds" \\')
+ table.insert(buf, (" "):rep(indent + 2) .. '"*:: :->args" \\')
+ end
+ end
+
+ table.insert(buf, (" "):rep(indent + 2) .. "&& return 0")
+end
+
+function Parser:_zsh_cmds(buf, cmd_name)
+ table.insert(buf, "\n_" .. cmd_name .. "_cmds() {")
+ table.insert(buf, " local -a commands=(")
+
+ for _, command in ipairs(self._commands) do
+ local line = {}
+ if #command._aliases > 1 then
+ table.insert(line, "{" .. table.concat(command._aliases, ",") .. '}"')
+ else
+ table.insert(line, '"' .. command._name)
+ end
+ if command._description then
+ table.insert(line, ":" .. get_short_description(command):gsub('["`$]', "\\%0"))
+ end
+ table.insert(buf, " " .. table.concat(line) .. '"')
+ end
+
+ table.insert(buf, ' )\n _describe "command" commands\n}')
+end
+
+function Parser:_zsh_complete_help(buf, cmds_buf, cmd_name, indent)
+ if #self._commands == 0 then
+ return
+ end
+
+ self:_zsh_cmds(cmds_buf, cmd_name)
+ table.insert(buf, "\n" .. (" "):rep(indent) .. "case $words[1] in")
+
+ for _, command in ipairs(self._commands) do
+ local name = cmd_name .. "_" .. command._name
+ table.insert(buf, (" "):rep(indent + 2) .. table.concat(command._aliases, "|") .. ")")
+ command:_zsh_arguments(buf, name, indent + 4)
+ command:_zsh_complete_help(buf, cmds_buf, name, indent + 4)
+ table.insert(buf, (" "):rep(indent + 4) .. ";;\n")
+ end
+
+ table.insert(buf, (" "):rep(indent) .. "esac")
+end
+
+function Parser:get_zsh_complete()
+ self._basename = base_name(self._name)
+ assert(self:_is_shell_safe())
+ local buf = {("#compdef %s\n"):format(self._basename)}
+ local cmds_buf = {}
+ table.insert(buf, "_" .. self._basename .. "() {")
+ if #self._commands > 0 then
+ table.insert(buf, " local context state state_descr line")
+ table.insert(buf, " typeset -A opt_args\n")
+ end
+ self:_zsh_arguments(buf, self._basename, 2)
+ self:_zsh_complete_help(buf, cmds_buf, self._basename, 2)
+ table.insert(buf, "\n return 1")
+ table.insert(buf, "}")
+
+ local result = table.concat(buf, "\n")
+ if #cmds_buf > 0 then
+ result = result .. "\n" .. table.concat(cmds_buf, "\n")
+ end
+ return result .. "\n\n_" .. self._basename .. "\n"
+end
+
+local function fish_escape(string)
+ return string:gsub("[\\']", "\\%0")
+end
+
+function Parser:_fish_get_cmd(buf, indent)
+ if #self._commands == 0 then
+ return
+ end
+
+ table.insert(buf, (" "):rep(indent) .. "set -e cmdline[1]")
+ table.insert(buf, (" "):rep(indent) .. "for arg in $cmdline")
+ table.insert(buf, (" "):rep(indent + 4) .. "switch $arg")
+
+ for _, command in ipairs(self._commands) do
+ table.insert(buf, (" "):rep(indent + 8) .. "case " .. table.concat(command._aliases, " "))
+ table.insert(buf, (" "):rep(indent + 12) .. "set cmd $cmd " .. command._name)
+ command:_fish_get_cmd(buf, indent + 12)
+ table.insert(buf, (" "):rep(indent + 12) .. "break")
+ end
+
+ table.insert(buf, (" "):rep(indent + 4) .. "end")
+ table.insert(buf, (" "):rep(indent) .. "end")
+end
+
+function Parser:_fish_complete_help(buf, basename)
+ local prefix = "complete -c " .. basename
+ table.insert(buf, "")
+
+ for _, command in ipairs(self._commands) do
+ local aliases = table.concat(command._aliases, " ")
+ local line
+ if self._parent then
+ line = ("%s -n '__fish_%s_using_command %s' -xa '%s'")
+ :format(prefix, basename, self:_get_fullname(true), aliases)
+ else
+ line = ("%s -n '__fish_%s_using_command' -xa '%s'"):format(prefix, basename, aliases)
+ end
+ if command._description then
+ line = ("%s -d '%s'"):format(line, fish_escape(get_short_description(command)))
+ end
+ table.insert(buf, line)
+ end
+
+ if self._is_help_command then
+ local line = ("%s -n '__fish_%s_using_command %s' -xa '%s'")
+ :format(prefix, basename, self:_get_fullname(true), self._parent:_get_commands())
+ table.insert(buf, line)
+ end
+
+ for _, option in ipairs(self._options) do
+ local parts = {prefix}
+
+ if self._parent then
+ table.insert(parts, "-n '__fish_" .. basename .. "_seen_command " .. self:_get_fullname(true) .. "'")
+ end
+
+ for _, alias in ipairs(option._aliases) do
+ if alias:match("^%-.$") then
+ table.insert(parts, "-s " .. alias:sub(2))
+ elseif alias:match("^%-%-.+") then
+ table.insert(parts, "-l " .. alias:sub(3))
+ end
+ end
+
+ if option._choices then
+ table.insert(parts, "-xa '" .. table.concat(option._choices, " ") .. "'")
+ elseif option._minargs > 0 then
+ table.insert(parts, "-r")
+ end
+
+ if option._description then
+ table.insert(parts, "-d '" .. fish_escape(get_short_description(option)) .. "'")
+ end
+
+ table.insert(buf, table.concat(parts, " "))
+ end
+
+ for _, command in ipairs(self._commands) do
+ command:_fish_complete_help(buf, basename)
+ end
+end
+
+function Parser:get_fish_complete()
+ self._basename = base_name(self._name)
+ assert(self:_is_shell_safe())
+ local buf = {}
+
+ if #self._commands > 0 then
+ table.insert(buf, ([[
+function __fish_%s_print_command
+ set -l cmdline (commandline -poc)
+ set -l cmd]]):format(self._basename))
+ self:_fish_get_cmd(buf, 4)
+ table.insert(buf, ([[
+ echo "$cmd"
+end
+
+function __fish_%s_using_command
+ test (__fish_%s_print_command) = "$argv"
+ and return 0
+ or return 1
+end
+
+function __fish_%s_seen_command
+ string match -q "$argv*" (__fish_%s_print_command)
+ and return 0
+ or return 1
+end]]):format(self._basename, self._basename, self._basename, self._basename))
+ end
+
+ self:_fish_complete_help(buf, self._basename)
+ return table.concat(buf, "\n") .. "\n"
+end
+
+local function get_tip(context, wrong_name)
+ local context_pool = {}
+ local possible_name
+ local possible_names = {}
+
+ for name in pairs(context) do
+ if type(name) == "string" then
+ for i = 1, #name do
+ possible_name = name:sub(1, i - 1) .. name:sub(i + 1)
+
+ if not context_pool[possible_name] then
+ context_pool[possible_name] = {}
+ end
+
+ table.insert(context_pool[possible_name], name)
+ end
+ end
+ end
+
+ for i = 1, #wrong_name + 1 do
+ possible_name = wrong_name:sub(1, i - 1) .. wrong_name:sub(i + 1)
+
+ if context[possible_name] then
+ possible_names[possible_name] = true
+ elseif context_pool[possible_name] then
+ for _, name in ipairs(context_pool[possible_name]) do
+ possible_names[name] = true
+ end
+ end
+ end
+
+ local first = next(possible_names)
+
+ if first then
+ if next(possible_names, first) then
+ local possible_names_arr = {}
+
+ for name in pairs(possible_names) do
+ table.insert(possible_names_arr, "'" .. name .. "'")
+ end
+
+ table.sort(possible_names_arr)
+ return "\nDid you mean one of these: " .. table.concat(possible_names_arr, " ") .. "?"
+ else
+ return "\nDid you mean '" .. first .. "'?"
+ end
+ else
+ return ""
+ end
+end
+
+local ElementState = class({
+ invocations = 0
+})
+
+function ElementState:__call(state, element)
+ self.state = state
+ self.result = state.result
+ self.element = element
+ self.target = element._target or element:_get_default_target()
+ self.action, self.result[self.target] = element:_get_action()
+ return self
+end
+
+function ElementState:error(fmt, ...)
+ self.state:error(fmt, ...)
+end
+
+function ElementState:convert(argument, index)
+ local converter = self.element._convert
+
+ if converter then
+ local ok, err
+
+ if type(converter) == "function" then
+ ok, err = converter(argument)
+ elseif type(converter[index]) == "function" then
+ ok, err = converter[index](argument)
+ else
+ ok = converter[argument]
+ end
+
+ if ok == nil then
+ self:error(err and "%s" or "malformed argument '%s'", err or argument)
+ end
+
+ argument = ok
+ end
+
+ return argument
+end
+
+function ElementState:default(mode)
+ return self.element._defmode:find(mode) and self.element._default
+end
+
+local function bound(noun, min, max, is_max)
+ local res = ""
+
+ if min ~= max then
+ res = "at " .. (is_max and "most" or "least") .. " "
+ end
+
+ local number = is_max and max or min
+ return res .. tostring(number) .. " " .. noun .. (number == 1 and "" or "s")
+end
+
+function ElementState:set_name(alias)
+ self.name = ("%s '%s'"):format(alias and "option" or "argument", alias or self.element._name)
+end
+
+function ElementState:invoke()
+ self.open = true
+ self.overwrite = false
+
+ if self.invocations >= self.element._maxcount then
+ if self.element._overwrite then
+ self.overwrite = true
+ else
+ local num_times_repr = bound("time", self.element._mincount, self.element._maxcount, true)
+ self:error("%s must be used %s", self.name, num_times_repr)
+ end
+ else
+ self.invocations = self.invocations + 1
+ end
+
+ self.args = {}
+
+ if self.element._maxargs <= 0 then
+ self:close()
+ end
+
+ return self.open
+end
+
+function ElementState:check_choices(argument)
+ if self.element._choices then
+ for _, choice in ipairs(self.element._choices) do
+ if argument == choice then
+ return
+ end
+ end
+ local choices_list = "'" .. table.concat(self.element._choices, "', '") .. "'"
+ local is_option = getmetatable(self.element) == Option
+ self:error("%s%s must be one of %s", is_option and "argument for " or "", self.name, choices_list)
+ end
+end
+
+function ElementState:pass(argument)
+ self:check_choices(argument)
+ argument = self:convert(argument, #self.args + 1)
+ table.insert(self.args, argument)
+
+ if #self.args >= self.element._maxargs then
+ self:close()
+ end
+
+ return self.open
+end
+
+function ElementState:complete_invocation()
+ while #self.args < self.element._minargs do
+ self:pass(self.element._default)
+ end
+end
+
+function ElementState:close()
+ if self.open then
+ self.open = false
+
+ if #self.args < self.element._minargs then
+ if self:default("a") then
+ self:complete_invocation()
+ else
+ if #self.args == 0 then
+ if getmetatable(self.element) == Argument then
+ self:error("missing %s", self.name)
+ elseif self.element._maxargs == 1 then
+ self:error("%s requires an argument", self.name)
+ end
+ end
+
+ self:error("%s requires %s", self.name, bound("argument", self.element._minargs, self.element._maxargs))
+ end
+ end
+
+ local args
+
+ if self.element._maxargs == 0 then
+ args = self.args[1]
+ elseif self.element._maxargs == 1 then
+ if self.element._minargs == 0 and self.element._mincount ~= self.element._maxcount then
+ args = self.args
+ else
+ args = self.args[1]
+ end
+ else
+ args = self.args
+ end
+
+ self.action(self.result, self.target, args, self.overwrite)
+ end
+end
+
+local ParseState = class({
+ result = {},
+ options = {},
+ arguments = {},
+ argument_i = 1,
+ element_to_mutexes = {},
+ mutex_to_element_state = {},
+ command_actions = {}
+})
+
+function ParseState:__call(parser, error_handler)
+ self.parser = parser
+ self.error_handler = error_handler
+ self.charset = parser:_update_charset()
+ self:switch(parser)
+ return self
+end
+
+function ParseState:error(fmt, ...)
+ self.error_handler(self.parser, fmt:format(...))
+end
+
+function ParseState:switch(parser)
+ self.parser = parser
+
+ if parser._action then
+ table.insert(self.command_actions, {action = parser._action, name = parser._name})
+ end
+
+ for _, option in ipairs(parser._options) do
+ option = ElementState(self, option)
+ table.insert(self.options, option)
+
+ for _, alias in ipairs(option.element._aliases) do
+ self.options[alias] = option
+ end
+ end
+
+ for _, mutex in ipairs(parser._mutexes) do
+ for _, element in ipairs(mutex) do
+ if not self.element_to_mutexes[element] then
+ self.element_to_mutexes[element] = {}
+ end
+
+ table.insert(self.element_to_mutexes[element], mutex)
+ end
+ end
+
+ for _, argument in ipairs(parser._arguments) do
+ argument = ElementState(self, argument)
+ table.insert(self.arguments, argument)
+ argument:set_name()
+ argument:invoke()
+ end
+
+ self.handle_options = parser._handle_options
+ self.argument = self.arguments[self.argument_i]
+ self.commands = parser._commands
+
+ for _, command in ipairs(self.commands) do
+ for _, alias in ipairs(command._aliases) do
+ self.commands[alias] = command
+ end
+ end
+end
+
+function ParseState:get_option(name)
+ local option = self.options[name]
+
+ if not option then
+ self:error("unknown option '%s'%s", name, get_tip(self.options, name))
+ else
+ return option
+ end
+end
+
+function ParseState:get_command(name)
+ local command = self.commands[name]
+
+ if not command then
+ if #self.commands > 0 then
+ self:error("unknown command '%s'%s", name, get_tip(self.commands, name))
+ else
+ self:error("too many arguments")
+ end
+ else
+ return command
+ end
+end
+
+function ParseState:check_mutexes(element_state)
+ if self.element_to_mutexes[element_state.element] then
+ for _, mutex in ipairs(self.element_to_mutexes[element_state.element]) do
+ local used_element_state = self.mutex_to_element_state[mutex]
+
+ if used_element_state and used_element_state ~= element_state then
+ self:error("%s can not be used together with %s", element_state.name, used_element_state.name)
+ else
+ self.mutex_to_element_state[mutex] = element_state
+ end
+ end
+ end
+end
+
+function ParseState:invoke(option, name)
+ self:close()
+ option:set_name(name)
+ self:check_mutexes(option, name)
+
+ if option:invoke() then
+ self.option = option
+ end
+end
+
+function ParseState:pass(arg)
+ if self.option then
+ if not self.option:pass(arg) then
+ self.option = nil
+ end
+ elseif self.argument then
+ self:check_mutexes(self.argument)
+
+ if not self.argument:pass(arg) then
+ self.argument_i = self.argument_i + 1
+ self.argument = self.arguments[self.argument_i]
+ end
+ else
+ local command = self:get_command(arg)
+ self.result[command._target or command._name] = true
+
+ if self.parser._command_target then
+ self.result[self.parser._command_target] = command._name
+ end
+
+ self:switch(command)
+ end
+end
+
+function ParseState:close()
+ if self.option then
+ self.option:close()
+ self.option = nil
+ end
+end
+
+function ParseState:finalize()
+ self:close()
+
+ for i = self.argument_i, #self.arguments do
+ local argument = self.arguments[i]
+ if #argument.args == 0 and argument:default("u") then
+ argument:complete_invocation()
+ else
+ argument:close()
+ end
+ end
+
+ if self.parser._require_command and #self.commands > 0 then
+ self:error("a command is required")
+ end
+
+ for _, option in ipairs(self.options) do
+ option.name = option.name or ("option '%s'"):format(option.element._name)
+
+ if option.invocations == 0 then
+ if option:default("u") then
+ option:invoke()
+ option:complete_invocation()
+ option:close()
+ end
+ end
+
+ local mincount = option.element._mincount
+
+ if option.invocations < mincount then
+ if option:default("a") then
+ while option.invocations < mincount do
+ option:invoke()
+ option:close()
+ end
+ elseif option.invocations == 0 then
+ self:error("missing %s", option.name)
+ else
+ self:error("%s must be used %s", option.name, bound("time", mincount, option.element._maxcount))
+ end
+ end
+ end
+
+ for i = #self.command_actions, 1, -1 do
+ self.command_actions[i].action(self.result, self.command_actions[i].name)
+ end
+end
+
+function ParseState:parse(args)
+ for _, arg in ipairs(args) do
+ local plain = true
+
+ if self.handle_options then
+ local first = arg:sub(1, 1)
+
+ if self.charset[first] then
+ if #arg > 1 then
+ plain = false
+
+ if arg:sub(2, 2) == first then
+ if #arg == 2 then
+ if self.options[arg] then
+ local option = self:get_option(arg)
+ self:invoke(option, arg)
+ else
+ self:close()
+ end
+
+ self.handle_options = false
+ else
+ local equals = arg:find "="
+ if equals then
+ local name = arg:sub(1, equals - 1)
+ local option = self:get_option(name)
+
+ if option.element._maxargs <= 0 then
+ self:error("option '%s' does not take arguments", name)
+ end
+
+ self:invoke(option, name)
+ self:pass(arg:sub(equals + 1))
+ else
+ local option = self:get_option(arg)
+ self:invoke(option, arg)
+ end
+ end
+ else
+ for i = 2, #arg do
+ local name = first .. arg:sub(i, i)
+ local option = self:get_option(name)
+ self:invoke(option, name)
+
+ if i ~= #arg and option.element._maxargs > 0 then
+ self:pass(arg:sub(i + 1))
+ break
+ end
+ end
+ end
+ end
+ end
+ end
+
+ if plain then
+ self:pass(arg)
+ end
+ end
+
+ self:finalize()
+ return self.result
+end
+
+function Parser:error(msg)
+ io.stderr:write(("%s\n\nError: %s\n"):format(self:get_usage(), msg))
+ os.exit(1)
+end
+
+-- Compatibility with strict.lua and other checkers:
+local default_cmdline = rawget(_G, "arg") or {}
+
+function Parser:_parse(args, error_handler)
+ return ParseState(self, error_handler):parse(args or default_cmdline)
+end
+
+function Parser:parse(args)
+ return self:_parse(args, self.error)
+end
+
+local function xpcall_error_handler(err)
+ return tostring(err) .. "\noriginal " .. debug.traceback("", 2):sub(2)
+end
+
+function Parser:pparse(args)
+ local parse_error
+
+ local ok, result = xpcall(function()
+ return self:_parse(args, function(_, err)
+ parse_error = err
+ error(err, 0)
+ end)
+ end, xpcall_error_handler)
+
+ if ok then
+ return true, result
+ elseif not parse_error then
+ error(result, 0)
+ else
+ return false, parse_error
+ end
+end
+
+local argparse = {}
+
+argparse.version = "0.7.1"
+
+setmetatable(argparse, {__call = function(_, ...)
+ return Parser(default_cmdline[0]):add_help(true)(...)
+end})
+
+return argparse
diff --git a/contrib/lua-bit/CMakeLists.txt b/contrib/lua-bit/CMakeLists.txt
new file mode 100644
index 0000000..8fac490
--- /dev/null
+++ b/contrib/lua-bit/CMakeLists.txt
@@ -0,0 +1,4 @@
+SET(BITSRC bit.c)
+
+SET(LIB_TYPE STATIC)
+ADD_LIBRARY(rspamd-bit ${LIB_TYPE} ${BITSRC})
diff --git a/contrib/lua-bit/bit.c b/contrib/lua-bit/bit.c
new file mode 100644
index 0000000..01326c9
--- /dev/null
+++ b/contrib/lua-bit/bit.c
@@ -0,0 +1,198 @@
+/*
+** Lua BitOp -- a bit operations library for Lua 5.1/5.2.
+** http://bitop.luajit.org/
+**
+** Copyright (C) 2008-2012 Mike Pall. All rights reserved.
+**
+** Permission is hereby granted, free of charge, to any person obtaining
+** a copy of this software and associated documentation files (the
+** "Software"), to deal in the Software without restriction, including
+** without limitation the rights to use, copy, modify, merge, publish,
+** distribute, sublicense, and/or sell copies of the Software, and to
+** permit persons to whom the Software is furnished to do so, subject to
+** the following conditions:
+**
+** The above copyright notice and this permission notice shall be
+** included in all copies or substantial portions of the Software.
+**
+** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+** SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+**
+** [ MIT license: http://www.opensource.org/licenses/mit-license.php ]
+*/
+
+#define LUA_BITOP_VERSION "1.0.2"
+
+#define LUA_LIB
+#include "lua.h"
+#include "lauxlib.h"
+
+#ifdef _MSC_VER
+/* MSVC is stuck in the last century and doesn't have C99's stdint.h. */
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
+#else
+#include <stdint.h>
+#endif
+
+typedef int32_t SBits;
+typedef uint32_t UBits;
+
+typedef union {
+ lua_Number n;
+#if defined(LUA_NUMBER_DOUBLE) || defined(LUA_FLOAT_DOUBLE)
+ uint64_t b;
+#else
+ UBits b;
+#endif
+} BitNum;
+
+/* Convert argument to bit type. */
+static UBits barg(lua_State *L, int idx)
+{
+ BitNum bn;
+ UBits b;
+#if LUA_VERSION_NUM < 502
+ bn.n = lua_tonumber(L, idx);
+#else
+ bn.n = luaL_checknumber(L, idx);
+#endif
+#if defined(LUA_NUMBER_DOUBLE) || defined(LUA_FLOAT_DOUBLE)
+ bn.n += 6755399441055744.0; /* 2^52+2^51 */
+#ifdef SWAPPED_DOUBLE
+ b = (UBits)(bn.b >> 32);
+#else
+ b = (UBits)bn.b;
+#endif
+#elif defined(LUA_NUMBER_INT) || defined(LUA_INT_INT) || \
+ defined(LUA_NUMBER_LONG) || defined(LUA_INT_LONG) || \
+ defined(LUA_NUMBER_LONGLONG) || defined(LUA_INT_LONGLONG) || \
+ defined(LUA_NUMBER_LONG_LONG) || defined(LUA_NUMBER_LLONG)
+ if (sizeof(UBits) == sizeof(lua_Number))
+ b = bn.b;
+ else
+ b = (UBits)(SBits)bn.n;
+#elif defined(LUA_NUMBER_FLOAT) || defined(LUA_FLOAT_FLOAT)
+#error "A 'float' lua_Number type is incompatible with this library"
+#else
+#error "Unknown number type, check LUA_NUMBER_*, LUA_FLOAT_*, LUA_INT_* in luaconf.h"
+#endif
+#if LUA_VERSION_NUM < 502
+ if (b == 0 && !lua_isnumber(L, idx)) {
+ luaL_typerror(L, idx, "number");
+ }
+#endif
+ return b;
+}
+
+/* Return bit type. */
+#if LUA_VERSION_NUM < 503
+#define BRET(b) lua_pushnumber(L, (lua_Number)(SBits)(b)); return 1;
+#else
+#define BRET(b) lua_pushinteger(L, (lua_Integer)(SBits)(b)); return 1;
+#endif
+
+static int bit_tobit(lua_State *L) { BRET(barg(L, 1)) }
+static int bit_bnot(lua_State *L) { BRET(~barg(L, 1)) }
+
+#define BIT_OP(func, opr) \
+ static int func(lua_State *L) { int i; UBits b = barg(L, 1); \
+ for (i = lua_gettop(L); i > 1; i--) b opr barg(L, i); BRET(b) }
+BIT_OP(bit_band, &=)
+BIT_OP(bit_bor, |=)
+BIT_OP(bit_bxor, ^=)
+
+#define bshl(b, n) (b << n)
+#define bshr(b, n) (b >> n)
+#define bsar(b, n) ((SBits)b >> n)
+#define brol(b, n) ((b << n) | (b >> (32-n)))
+#define bror(b, n) ((b << (32-n)) | (b >> n))
+#define BIT_SH(func, fn) \
+ static int func(lua_State *L) { \
+ UBits b = barg(L, 1); UBits n = barg(L, 2) & 31; BRET(fn(b, n)) }
+BIT_SH(bit_lshift, bshl)
+BIT_SH(bit_rshift, bshr)
+BIT_SH(bit_arshift, bsar)
+BIT_SH(bit_rol, brol)
+BIT_SH(bit_ror, bror)
+
+static int bit_bswap(lua_State *L)
+{
+ UBits b = barg(L, 1);
+ b = (b >> 24) | ((b >> 8) & 0xff00) | ((b & 0xff00) << 8) | (b << 24);
+ BRET(b)
+}
+
+static int bit_tohex(lua_State *L)
+{
+ UBits b = barg(L, 1);
+ SBits n = lua_isnone(L, 2) ? 8 : (SBits)barg(L, 2);
+ const char *hexdigits = "0123456789abcdef";
+ char buf[8];
+ int i;
+ if (n < 0) { n = -n; hexdigits = "0123456789ABCDEF"; }
+ if (n > 8) n = 8;
+ for (i = (int)n; --i >= 0; ) { buf[i] = hexdigits[b & 15]; b >>= 4; }
+ lua_pushlstring(L, buf, (size_t)n);
+ return 1;
+}
+
+static const struct luaL_Reg bit_funcs[] = {
+ { "tobit", bit_tobit },
+ { "bnot", bit_bnot },
+ { "band", bit_band },
+ { "bor", bit_bor },
+ { "bxor", bit_bxor },
+ { "lshift", bit_lshift },
+ { "rshift", bit_rshift },
+ { "arshift", bit_arshift },
+ { "rol", bit_rol },
+ { "ror", bit_ror },
+ { "bswap", bit_bswap },
+ { "tohex", bit_tohex },
+ { NULL, NULL }
+};
+
+/* Signed right-shifts are implementation-defined per C89/C99.
+** But the de facto standard are arithmetic right-shifts on two's
+** complement CPUs. This behaviour is required here, so test for it.
+*/
+#define BAD_SAR (bsar(-8, 2) != (SBits)-2)
+
+LUALIB_API int luaopen_bit(lua_State *L)
+{
+ UBits b;
+#if LUA_VERSION_NUM < 503
+ lua_pushnumber(L, (lua_Number)1437217655L);
+#else
+ lua_pushinteger(L, (lua_Integer)1437217655L);
+#endif
+ b = barg(L, -1);
+ if (b != (UBits)1437217655L || BAD_SAR) { /* Perform a simple self-test. */
+ const char *msg = "compiled with incompatible luaconf.h";
+#if defined(LUA_NUMBER_DOUBLE) || defined(LUA_FLOAT_DOUBLE)
+#ifdef _WIN32
+ if (b == (UBits)1610612736L)
+ msg = "use D3DCREATE_FPU_PRESERVE with DirectX";
+#endif
+ if (b == (UBits)1127743488L)
+ msg = "not compiled with SWAPPED_DOUBLE";
+#endif
+ if (BAD_SAR)
+ msg = "arithmetic right-shift broken";
+ luaL_error(L, "bit library self-test failed (%s)", msg);
+ }
+#if LUA_VERSION_NUM < 502
+ luaL_register(L, "bit", bit_funcs);
+#else
+ luaL_newlib(L, bit_funcs);
+#endif
+ return 1;
+}
+
diff --git a/contrib/lua-fun/COPYING.md b/contrib/lua-fun/COPYING.md
new file mode 100644
index 0000000..42ee231
--- /dev/null
+++ b/contrib/lua-fun/COPYING.md
@@ -0,0 +1,27 @@
+Copying
+=======
+
+**Lua Fun** source codes, logo and documentation are distributed under the
+**[MIT/X11 License]** - same as Lua and LuaJIT.
+
+Copyright (c) 2013 Roman Tsisyk <roman@tsisyk.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+[MIT/X11 License]: http://www.opensource.org/licenses/mit-license.php
diff --git a/contrib/lua-fun/fun.lua b/contrib/lua-fun/fun.lua
new file mode 100644
index 0000000..2c712d3
--- /dev/null
+++ b/contrib/lua-fun/fun.lua
@@ -0,0 +1,1076 @@
+---
+--- Lua Fun - a high-performance functional programming library for LuaJIT
+---
+--- Copyright (c) 2013-2017 Roman Tsisyk <roman@tsisyk.com>
+---
+--- Distributed under the MIT/X11 License. See COPYING.md for more details.
+---
+
+local exports = {}
+local methods = {}
+
+-- Rspamd specific
+local rspamd_text = require "rspamd_text"
+local text_cookie = rspamd_text.cookie
+
+-- compatibility with Lua 5.1/5.2
+local unpack = rawget(table, "unpack") or unpack
+
+--------------------------------------------------------------------------------
+-- Tools
+--------------------------------------------------------------------------------
+
+local return_if_not_empty = function(state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ return ...
+end
+
+local call_if_not_empty = function(fun, state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ return state_x, fun(...)
+end
+
+local function deepcopy(orig) -- used by cycle()
+ local orig_type = type(orig)
+ local copy
+ if orig_type == 'table' then
+ copy = {}
+ for orig_key, orig_value in next, orig, nil do
+ copy[deepcopy(orig_key)] = deepcopy(orig_value)
+ end
+ else
+ copy = orig
+ end
+ return copy
+end
+
+local iterator_mt = {
+ -- usually called by for-in loop
+ __call = function(self, param, state)
+ return self.gen(param, state)
+ end;
+ __tostring = function(self)
+ return '<generator>'
+ end;
+ -- add all exported methods
+ __index = methods;
+}
+
+local wrap = function(gen, param, state)
+ return setmetatable({
+ gen = gen,
+ param = param,
+ state = state
+ }, iterator_mt), param, state
+end
+exports.wrap = wrap
+
+local unwrap = function(self)
+ return self.gen, self.param, self.state
+end
+methods.unwrap = unwrap
+
+--------------------------------------------------------------------------------
+-- Basic Functions
+--------------------------------------------------------------------------------
+
+local nil_gen = function(_param, _state)
+ return nil
+end
+
+local string_gen = function(param, state)
+ local state = state + 1
+ if state > #param then
+ return nil
+ end
+ local r = string.sub(param, state, state)
+ return state, r
+end
+
+local text_gen = function(param, state)
+ local state = state + 1
+ if state > #param then
+ return nil
+ end
+ local r = string.char(param:byte(state))
+ return state, r
+end
+
+local ipairs_gen = ipairs({}) -- get the generating function from ipairs
+
+local pairs_gen = pairs({ a = 0 }) -- get the generating function from pairs
+local map_gen = function(tab, key)
+ local value
+ local key, value = pairs_gen(tab, key)
+ return key, key, value
+end
+
+local rawiter = function(obj, param, state)
+ assert(obj ~= nil, "invalid iterator")
+ if type(obj) == "table" then
+ local mt = getmetatable(obj);
+ if mt ~= nil then
+ if mt == iterator_mt then
+ return obj.gen, obj.param, obj.state
+ elseif mt.__ipairs ~= nil then
+ return mt.__ipairs(obj)
+ elseif mt.__pairs ~= nil then
+ return mt.__pairs(obj)
+ end
+ end
+ if #obj > 0 then
+ -- array
+ return ipairs(obj)
+ else
+ -- hash
+ return map_gen, obj, nil
+ end
+ elseif (type(obj) == "function") then
+ return obj, param, state
+ elseif (type(obj) == "string") then
+ if #obj == 0 then
+ return nil_gen, nil, nil
+ end
+ return string_gen, obj, 0
+ elseif (type(obj) == "userdata" and obj.cookie == text_cookie) then
+ if #obj == 0 then
+ return nil_gen, nil, nil
+ end
+ return text_gen, obj, 0
+ end
+ error(string.format('object %s of type "%s" is not iterable',
+ obj, type(obj)))
+end
+
+local iter = function(obj, param, state)
+ return wrap(rawiter(obj, param, state))
+end
+exports.iter = iter
+
+local method0 = function(fun)
+ return function(self)
+ return fun(self.gen, self.param, self.state)
+ end
+end
+
+local method1 = function(fun)
+ return function(self, arg1)
+ return fun(arg1, self.gen, self.param, self.state)
+ end
+end
+
+local method2 = function(fun)
+ return function(self, arg1, arg2)
+ return fun(arg1, arg2, self.gen, self.param, self.state)
+ end
+end
+
+local export0 = function(fun)
+ return function(gen, param, state)
+ return fun(rawiter(gen, param, state))
+ end
+end
+
+local export1 = function(fun)
+ return function(arg1, gen, param, state)
+ return fun(arg1, rawiter(gen, param, state))
+ end
+end
+
+local export2 = function(fun)
+ return function(arg1, arg2, gen, param, state)
+ return fun(arg1, arg2, rawiter(gen, param, state))
+ end
+end
+
+local each = function(fun, gen, param, state)
+ repeat
+ state = call_if_not_empty(fun, gen(param, state))
+ until state == nil
+end
+methods.each = method1(each)
+exports.each = export1(each)
+methods.for_each = methods.each
+exports.for_each = exports.each
+methods.foreach = methods.each
+exports.foreach = exports.each
+
+--------------------------------------------------------------------------------
+-- Generators
+--------------------------------------------------------------------------------
+
+local range_gen = function(param, state)
+ local stop, step = param[1], param[2]
+ local state = state + step
+ if state > stop then
+ return nil
+ end
+ return state, state
+end
+
+local range_rev_gen = function(param, state)
+ local stop, step = param[1], param[2]
+ local state = state + step
+ if state < stop then
+ return nil
+ end
+ return state, state
+end
+
+local range = function(start, stop, step)
+ if step == nil then
+ if stop == nil then
+ if start == 0 then
+ return nil_gen, nil, nil
+ end
+ stop = start
+ start = stop > 0 and 1 or -1
+ end
+ step = start <= stop and 1 or -1
+ end
+
+ assert(type(start) == "number", "start must be a number")
+ assert(type(stop) == "number", "stop must be a number")
+ assert(type(step) == "number", "step must be a number")
+ assert(step ~= 0, "step must not be zero")
+
+ if (step > 0) then
+ return wrap(range_gen, {stop, step}, start - step)
+ elseif (step < 0) then
+ return wrap(range_rev_gen, {stop, step}, start - step)
+ end
+end
+exports.range = range
+
+local duplicate_table_gen = function(param_x, state_x)
+ return state_x + 1, unpack(param_x)
+end
+
+local duplicate_fun_gen = function(param_x, state_x)
+ return state_x + 1, param_x(state_x)
+end
+
+local duplicate_gen = function(param_x, state_x)
+ return state_x + 1, param_x
+end
+
+local duplicate = function(...)
+ if select('#', ...) <= 1 then
+ return wrap(duplicate_gen, select(1, ...), 0)
+ else
+ return wrap(duplicate_table_gen, {...}, 0)
+ end
+end
+exports.duplicate = duplicate
+exports.replicate = duplicate
+exports.xrepeat = duplicate
+
+local tabulate = function(fun)
+ assert(type(fun) == "function")
+ return wrap(duplicate_fun_gen, fun, 0)
+end
+exports.tabulate = tabulate
+
+local zeros = function()
+ return wrap(duplicate_gen, 0, 0)
+end
+exports.zeros = zeros
+
+local ones = function()
+ return wrap(duplicate_gen, 1, 0)
+end
+exports.ones = ones
+
+local rands_gen = function(param_x, _state_x)
+ return 0, math.random(param_x[1], param_x[2])
+end
+
+local rands_nil_gen = function(_param_x, _state_x)
+ return 0, math.random()
+end
+
+local rands = function(n, m)
+ if n == nil and m == nil then
+ return wrap(rands_nil_gen, 0, 0)
+ end
+ assert(type(n) == "number", "invalid first arg to rands")
+ if m == nil then
+ m = n
+ n = 0
+ else
+ assert(type(m) == "number", "invalid second arg to rands")
+ end
+ assert(n < m, "empty interval")
+ return wrap(rands_gen, {n, m - 1}, 0)
+end
+exports.rands = rands
+
+--------------------------------------------------------------------------------
+-- Slicing
+--------------------------------------------------------------------------------
+
+local nth = function(n, gen_x, param_x, state_x)
+ assert(n > 0, "invalid first argument to nth")
+ -- An optimization for arrays and strings
+ if gen_x == ipairs_gen then
+ return param_x[n]
+ elseif gen_x == string_gen then
+ if n <= #param_x then
+ return string.sub(param_x, n, n)
+ else
+ return nil
+ end
+ end
+ for i=1,n-1,1 do
+ state_x = gen_x(param_x, state_x)
+ if state_x == nil then
+ return nil
+ end
+ end
+ return return_if_not_empty(gen_x(param_x, state_x))
+end
+methods.nth = method1(nth)
+exports.nth = export1(nth)
+
+local head_call = function(state, ...)
+ if state == nil then
+ error("head: iterator is empty")
+ end
+ return ...
+end
+
+local head = function(gen, param, state)
+ return head_call(gen(param, state))
+end
+methods.head = method0(head)
+exports.head = export0(head)
+exports.car = exports.head
+methods.car = methods.head
+
+local tail = function(gen, param, state)
+ state = gen(param, state)
+ if state == nil then
+ return wrap(nil_gen, nil, nil)
+ end
+ return wrap(gen, param, state)
+end
+methods.tail = method0(tail)
+exports.tail = export0(tail)
+exports.cdr = exports.tail
+methods.cdr = methods.tail
+
+local take_n_gen_x = function(i, state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ return {i, state_x}, ...
+end
+
+local take_n_gen = function(param, state)
+ local n, gen_x, param_x = param[1], param[2], param[3]
+ local i, state_x = state[1], state[2]
+ if i >= n then
+ return nil
+ end
+ return take_n_gen_x(i + 1, gen_x(param_x, state_x))
+end
+
+local take_n = function(n, gen, param, state)
+ assert(n >= 0, "invalid first argument to take_n")
+ return wrap(take_n_gen, {n, gen, param}, {0, state})
+end
+methods.take_n = method1(take_n)
+exports.take_n = export1(take_n)
+
+local take_while_gen_x = function(fun, state_x, ...)
+ if state_x == nil or not fun(...) then
+ return nil
+ end
+ return state_x, ...
+end
+
+local take_while_gen = function(param, state_x)
+ local fun, gen_x, param_x = param[1], param[2], param[3]
+ return take_while_gen_x(fun, gen_x(param_x, state_x))
+end
+
+local take_while = function(fun, gen, param, state)
+ assert(type(fun) == "function", "invalid first argument to take_while")
+ return wrap(take_while_gen, {fun, gen, param}, state)
+end
+methods.take_while = method1(take_while)
+exports.take_while = export1(take_while)
+
+local take = function(n_or_fun, gen, param, state)
+ if type(n_or_fun) == "number" then
+ return take_n(n_or_fun, gen, param, state)
+ else
+ return take_while(n_or_fun, gen, param, state)
+ end
+end
+methods.take = method1(take)
+exports.take = export1(take)
+
+local drop_n = function(n, gen, param, state)
+ assert(n >= 0, "invalid first argument to drop_n")
+ local i
+ for i=1,n,1 do
+ state = gen(param, state)
+ if state == nil then
+ return wrap(nil_gen, nil, nil)
+ end
+ end
+ return wrap(gen, param, state)
+end
+methods.drop_n = method1(drop_n)
+exports.drop_n = export1(drop_n)
+
+local drop_while_x = function(fun, state_x, ...)
+ if state_x == nil or not fun(...) then
+ return state_x, false
+ end
+ return state_x, true, ...
+end
+
+local drop_while = function(fun, gen_x, param_x, state_x)
+ assert(type(fun) == "function", "invalid first argument to drop_while")
+ local cont, state_x_prev
+ repeat
+ state_x_prev = deepcopy(state_x)
+ state_x, cont = drop_while_x(fun, gen_x(param_x, state_x))
+ until not cont
+ if state_x == nil then
+ return wrap(nil_gen, nil, nil)
+ end
+ return wrap(gen_x, param_x, state_x_prev)
+end
+methods.drop_while = method1(drop_while)
+exports.drop_while = export1(drop_while)
+
+local drop = function(n_or_fun, gen_x, param_x, state_x)
+ if type(n_or_fun) == "number" then
+ return drop_n(n_or_fun, gen_x, param_x, state_x)
+ else
+ return drop_while(n_or_fun, gen_x, param_x, state_x)
+ end
+end
+methods.drop = method1(drop)
+exports.drop = export1(drop)
+
+local split = function(n_or_fun, gen_x, param_x, state_x)
+ return take(n_or_fun, gen_x, param_x, state_x),
+ drop(n_or_fun, gen_x, param_x, state_x)
+end
+methods.split = method1(split)
+exports.split = export1(split)
+methods.split_at = methods.split
+exports.split_at = exports.split
+methods.span = methods.split
+exports.span = exports.split
+
+--------------------------------------------------------------------------------
+-- Indexing
+--------------------------------------------------------------------------------
+
+local index = function(x, gen, param, state)
+ local i = 1
+ for _k, r in gen, param, state do
+ if r == x then
+ return i
+ end
+ i = i + 1
+ end
+ return nil
+end
+methods.index = method1(index)
+exports.index = export1(index)
+methods.index_of = methods.index
+exports.index_of = exports.index
+methods.elem_index = methods.index
+exports.elem_index = exports.index
+
+local indexes_gen = function(param, state)
+ local x, gen_x, param_x = param[1], param[2], param[3]
+ local i, state_x = state[1], state[2]
+ local r
+ while true do
+ state_x, r = gen_x(param_x, state_x)
+ if state_x == nil then
+ return nil
+ end
+ i = i + 1
+ if r == x then
+ return {i, state_x}, i
+ end
+ end
+end
+
+local indexes = function(x, gen, param, state)
+ return wrap(indexes_gen, {x, gen, param}, {0, state})
+end
+methods.indexes = method1(indexes)
+exports.indexes = export1(indexes)
+methods.elem_indexes = methods.indexes
+exports.elem_indexes = exports.indexes
+methods.indices = methods.indexes
+exports.indices = exports.indexes
+methods.elem_indices = methods.indexes
+exports.elem_indices = exports.indexes
+
+--------------------------------------------------------------------------------
+-- Filtering
+--------------------------------------------------------------------------------
+
+local filter1_gen = function(fun, gen_x, param_x, state_x, a)
+ while true do
+ if state_x == nil or fun(a) then break; end
+ state_x, a = gen_x(param_x, state_x)
+ end
+ return state_x, a
+end
+
+-- call each other
+local filterm_gen
+local filterm_gen_shrink = function(fun, gen_x, param_x, state_x)
+ return filterm_gen(fun, gen_x, param_x, gen_x(param_x, state_x))
+end
+
+filterm_gen = function(fun, gen_x, param_x, state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ if fun(...) then
+ return state_x, ...
+ end
+ return filterm_gen_shrink(fun, gen_x, param_x, state_x)
+end
+
+local filter_detect = function(fun, gen_x, param_x, state_x, ...)
+ if select('#', ...) < 2 then
+ return filter1_gen(fun, gen_x, param_x, state_x, ...)
+ else
+ return filterm_gen(fun, gen_x, param_x, state_x, ...)
+ end
+end
+
+local filter_gen = function(param, state_x)
+ local fun, gen_x, param_x = param[1], param[2], param[3]
+ return filter_detect(fun, gen_x, param_x, gen_x(param_x, state_x))
+end
+
+local filter = function(fun, gen, param, state)
+ return wrap(filter_gen, {fun, gen, param}, state)
+end
+methods.filter = method1(filter)
+exports.filter = export1(filter)
+methods.remove_if = methods.filter
+exports.remove_if = exports.filter
+
+local grep = function(fun_or_regexp, gen, param, state)
+ local fun = fun_or_regexp
+ if type(fun_or_regexp) == "string" then
+ fun = function(x) return string.find(x, fun_or_regexp) ~= nil end
+ end
+ return filter(fun, gen, param, state)
+end
+methods.grep = method1(grep)
+exports.grep = export1(grep)
+
+local partition = function(fun, gen, param, state)
+ local neg_fun = function(...)
+ return not fun(...)
+ end
+ return filter(fun, gen, param, state),
+ filter(neg_fun, gen, param, state)
+end
+methods.partition = method1(partition)
+exports.partition = export1(partition)
+
+--------------------------------------------------------------------------------
+-- Reducing
+--------------------------------------------------------------------------------
+
+local foldl_call = function(fun, start, state, ...)
+ if state == nil then
+ return nil, start
+ end
+ return state, fun(start, ...)
+end
+
+local foldl = function(fun, start, gen_x, param_x, state_x)
+ while true do
+ state_x, start = foldl_call(fun, start, gen_x(param_x, state_x))
+ if state_x == nil then
+ break;
+ end
+ end
+ return start
+end
+methods.foldl = method2(foldl)
+exports.foldl = export2(foldl)
+methods.reduce = methods.foldl
+exports.reduce = exports.foldl
+
+local length = function(gen, param, state)
+ if gen == ipairs_gen or gen == string_gen then
+ return #param
+ end
+ local len = 0
+ repeat
+ state = gen(param, state)
+ len = len + 1
+ until state == nil
+ return len - 1
+end
+methods.length = method0(length)
+exports.length = export0(length)
+
+local is_null = function(gen, param, state)
+ return gen(param, deepcopy(state)) == nil
+end
+methods.is_null = method0(is_null)
+exports.is_null = export0(is_null)
+
+local is_prefix_of = function(iter_x, iter_y)
+ local gen_x, param_x, state_x = iter(iter_x)
+ local gen_y, param_y, state_y = iter(iter_y)
+
+ local r_x, r_y
+ for i=1,10,1 do
+ state_x, r_x = gen_x(param_x, state_x)
+ state_y, r_y = gen_y(param_y, state_y)
+ if state_x == nil then
+ return true
+ end
+ if state_y == nil or r_x ~= r_y then
+ return false
+ end
+ end
+end
+methods.is_prefix_of = is_prefix_of
+exports.is_prefix_of = is_prefix_of
+
+local all = function(fun, gen_x, param_x, state_x)
+ local r
+ repeat
+ state_x, r = call_if_not_empty(fun, gen_x(param_x, state_x))
+ until state_x == nil or not r
+ return state_x == nil
+end
+methods.all = method1(all)
+exports.all = export1(all)
+methods.every = methods.all
+exports.every = exports.all
+
+local any = function(fun, gen_x, param_x, state_x)
+ local r
+ repeat
+ state_x, r = call_if_not_empty(fun, gen_x(param_x, state_x))
+ until state_x == nil or r
+ return not not r
+end
+methods.any = method1(any)
+exports.any = export1(any)
+methods.some = methods.any
+exports.some = exports.any
+
+local sum = function(gen, param, state)
+ local s = 0
+ local r = 0
+ repeat
+ s = s + r
+ state, r = gen(param, state)
+ until state == nil
+ return s
+end
+methods.sum = method0(sum)
+exports.sum = export0(sum)
+
+local product = function(gen, param, state)
+ local p = 1
+ local r = 1
+ repeat
+ p = p * r
+ state, r = gen(param, state)
+ until state == nil
+ return p
+end
+methods.product = method0(product)
+exports.product = export0(product)
+
+local min_cmp = function(m, n)
+ if n < m then return n else return m end
+end
+
+local max_cmp = function(m, n)
+ if n > m then return n else return m end
+end
+
+local min = function(gen, param, state)
+ local state, m = gen(param, state)
+ if state == nil then
+ error("min: iterator is empty")
+ end
+
+ local cmp
+ if type(m) == "number" then
+ -- An optimization: use math.min for numbers
+ cmp = math.min
+ else
+ cmp = min_cmp
+ end
+
+ for _, r in gen, param, state do
+ m = cmp(m, r)
+ end
+ return m
+end
+methods.min = method0(min)
+exports.min = export0(min)
+methods.minimum = methods.min
+exports.minimum = exports.min
+
+local min_by = function(cmp, gen_x, param_x, state_x)
+ local state_x, m = gen_x(param_x, state_x)
+ if state_x == nil then
+ error("min: iterator is empty")
+ end
+
+ for _, r in gen_x, param_x, state_x do
+ m = cmp(m, r)
+ end
+ return m
+end
+methods.min_by = method1(min_by)
+exports.min_by = export1(min_by)
+methods.minimum_by = methods.min_by
+exports.minimum_by = exports.min_by
+
+local max = function(gen_x, param_x, state_x)
+ local state_x, m = gen_x(param_x, state_x)
+ if state_x == nil then
+ error("max: iterator is empty")
+ end
+
+ local cmp
+ if type(m) == "number" then
+ -- An optimization: use math.max for numbers
+ cmp = math.max
+ else
+ cmp = max_cmp
+ end
+
+ for _, r in gen_x, param_x, state_x do
+ m = cmp(m, r)
+ end
+ return m
+end
+methods.max = method0(max)
+exports.max = export0(max)
+methods.maximum = methods.max
+exports.maximum = exports.max
+
+local max_by = function(cmp, gen_x, param_x, state_x)
+ local state_x, m = gen_x(param_x, state_x)
+ if state_x == nil then
+ error("max: iterator is empty")
+ end
+
+ for _, r in gen_x, param_x, state_x do
+ m = cmp(m, r)
+ end
+ return m
+end
+methods.max_by = method1(max_by)
+exports.max_by = export1(max_by)
+methods.maximum_by = methods.maximum_by
+exports.maximum_by = exports.maximum_by
+
+local totable = function(gen_x, param_x, state_x)
+ local tab, key, val = {}
+ while true do
+ state_x, val = gen_x(param_x, state_x)
+ if state_x == nil then
+ break
+ end
+ table.insert(tab, val)
+ end
+ return tab
+end
+methods.totable = method0(totable)
+exports.totable = export0(totable)
+
+local tomap = function(gen_x, param_x, state_x)
+ local tab, key, val = {}
+ while true do
+ state_x, key, val = gen_x(param_x, state_x)
+ if state_x == nil then
+ break
+ end
+ tab[key] = val
+ end
+ return tab
+end
+methods.tomap = method0(tomap)
+exports.tomap = export0(tomap)
+
+--------------------------------------------------------------------------------
+-- Transformations
+--------------------------------------------------------------------------------
+
+local map_gen = function(param, state)
+ local gen_x, param_x, fun = param[1], param[2], param[3]
+ return call_if_not_empty(fun, gen_x(param_x, state))
+end
+
+local map = function(fun, gen, param, state)
+ return wrap(map_gen, {gen, param, fun}, state)
+end
+methods.map = method1(map)
+exports.map = export1(map)
+
+local enumerate_gen_call = function(state, i, state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ return {i + 1, state_x}, i, ...
+end
+
+local enumerate_gen = function(param, state)
+ local gen_x, param_x = param[1], param[2]
+ local i, state_x = state[1], state[2]
+ return enumerate_gen_call(state, i, gen_x(param_x, state_x))
+end
+
+local enumerate = function(gen, param, state)
+ return wrap(enumerate_gen, {gen, param}, {1, state})
+end
+methods.enumerate = method0(enumerate)
+exports.enumerate = export0(enumerate)
+
+local intersperse_call = function(i, state_x, ...)
+ if state_x == nil then
+ return nil
+ end
+ return {i + 1, state_x}, ...
+end
+
+local intersperse_gen = function(param, state)
+ local x, gen_x, param_x = param[1], param[2], param[3]
+ local i, state_x = state[1], state[2]
+ if i % 2 == 1 then
+ return {i + 1, state_x}, x
+ else
+ return intersperse_call(i, gen_x(param_x, state_x))
+ end
+end
+
+-- TODO: interperse must not add x to the tail
+local intersperse = function(x, gen, param, state)
+ return wrap(intersperse_gen, {x, gen, param}, {0, state})
+end
+methods.intersperse = method1(intersperse)
+exports.intersperse = export1(intersperse)
+
+--------------------------------------------------------------------------------
+-- Compositions
+--------------------------------------------------------------------------------
+
+local function zip_gen_r(param, state, state_new, ...)
+ if #state_new == #param / 2 then
+ return state_new, ...
+ end
+
+ local i = #state_new + 1
+ local gen_x, param_x = param[2 * i - 1], param[2 * i]
+ local state_x, r = gen_x(param_x, state[i])
+ if state_x == nil then
+ return nil
+ end
+ table.insert(state_new, state_x)
+ return zip_gen_r(param, state, state_new, r, ...)
+end
+
+local zip_gen = function(param, state)
+ return zip_gen_r(param, state, {})
+end
+
+-- A special hack for zip/chain to skip last two state, if a wrapped iterator
+-- has been passed
+local numargs = function(...)
+ local n = select('#', ...)
+ if n >= 3 then
+ -- Fix last argument
+ local it = select(n - 2, ...)
+ if type(it) == 'table' and getmetatable(it) == iterator_mt and
+ it.param == select(n - 1, ...) and it.state == select(n, ...) then
+ return n - 2
+ end
+ end
+ return n
+end
+
+local zip = function(...)
+ local n = numargs(...)
+ if n == 0 then
+ return wrap(nil_gen, nil, nil)
+ end
+ local param = { [2 * n] = 0 }
+ local state = { [n] = 0 }
+
+ local i, gen_x, param_x, state_x
+ for i=1,n,1 do
+ local it = select(n - i + 1, ...)
+ gen_x, param_x, state_x = rawiter(it)
+ param[2 * i - 1] = gen_x
+ param[2 * i] = param_x
+ state[i] = state_x
+ end
+
+ return wrap(zip_gen, param, state)
+end
+methods.zip = zip
+exports.zip = zip
+
+local cycle_gen_call = function(param, state_x, ...)
+ if state_x == nil then
+ local gen_x, param_x, state_x0 = param[1], param[2], param[3]
+ return gen_x(param_x, deepcopy(state_x0))
+ end
+ return state_x, ...
+end
+
+local cycle_gen = function(param, state_x)
+ local gen_x, param_x, state_x0 = param[1], param[2], param[3]
+ return cycle_gen_call(param, gen_x(param_x, state_x))
+end
+
+local cycle = function(gen, param, state)
+ return wrap(cycle_gen, {gen, param, state}, deepcopy(state))
+end
+methods.cycle = method0(cycle)
+exports.cycle = export0(cycle)
+
+-- call each other
+local chain_gen_r1
+local chain_gen_r2 = function(param, state, state_x, ...)
+ if state_x == nil then
+ local i = state[1]
+ i = i + 1
+ if param[3 * i - 1] == nil then
+ return nil
+ end
+ local state_x = param[3 * i]
+ return chain_gen_r1(param, {i, state_x})
+ end
+ return {state[1], state_x}, ...
+end
+
+chain_gen_r1 = function(param, state)
+ local i, state_x = state[1], state[2]
+ local gen_x, param_x = param[3 * i - 2], param[3 * i - 1]
+ return chain_gen_r2(param, state, gen_x(param_x, state[2]))
+end
+
+local chain = function(...)
+ local n = numargs(...)
+ if n == 0 then
+ return wrap(nil_gen, nil, nil)
+ end
+
+ local param = { [3 * n] = 0 }
+ local i, gen_x, param_x, state_x
+ for i=1,n,1 do
+ local elem = select(i, ...)
+ gen_x, param_x, state_x = iter(elem)
+ param[3 * i - 2] = gen_x
+ param[3 * i - 1] = param_x
+ param[3 * i] = state_x
+ end
+
+ return wrap(chain_gen_r1, param, {1, param[3]})
+end
+methods.chain = chain
+exports.chain = chain
+
+--------------------------------------------------------------------------------
+-- Operators
+--------------------------------------------------------------------------------
+
+local operator = {
+ ----------------------------------------------------------------------------
+ -- Comparison operators
+ ----------------------------------------------------------------------------
+ lt = function(a, b) return a < b end,
+ le = function(a, b) return a <= b end,
+ eq = function(a, b) return a == b end,
+ ne = function(a, b) return a ~= b end,
+ ge = function(a, b) return a >= b end,
+ gt = function(a, b) return a > b end,
+
+ ----------------------------------------------------------------------------
+ -- Arithmetic operators
+ ----------------------------------------------------------------------------
+ add = function(a, b) return a + b end,
+ div = function(a, b) return a / b end,
+ floordiv = function(a, b) return math.floor(a/b) end,
+ intdiv = function(a, b)
+ local q = a / b
+ if a >= 0 then return math.floor(q) else return math.ceil(q) end
+ end,
+ mod = function(a, b) return a % b end,
+ mul = function(a, b) return a * b end,
+ neq = function(a) return -a end,
+ unm = function(a) return -a end, -- an alias
+ pow = function(a, b) return a ^ b end,
+ sub = function(a, b) return a - b end,
+ truediv = function(a, b) return a / b end,
+
+ ----------------------------------------------------------------------------
+ -- String operators
+ ----------------------------------------------------------------------------
+ concat = function(a, b) return a..b end,
+ len = function(a) return #a end,
+ length = function(a) return #a end, -- an alias
+
+ ----------------------------------------------------------------------------
+ -- Logical operators
+ ----------------------------------------------------------------------------
+ land = function(a, b) return a and b end,
+ lor = function(a, b) return a or b end,
+ lnot = function(a) return not a end,
+ truth = function(a) return not not a end,
+}
+exports.operator = operator
+methods.operator = operator
+exports.op = operator
+methods.op = operator
+
+--------------------------------------------------------------------------------
+-- module definitions
+--------------------------------------------------------------------------------
+
+-- a special syntax sugar to export all functions to the global table
+setmetatable(exports, {
+ __call = function(t, override)
+ for k, v in pairs(t) do
+ if _G[k] ~= nil then
+ local msg = 'function ' .. k .. ' already exists in global scope.'
+ if override then
+ _G[k] = v
+ print('WARNING: ' .. msg .. ' Overwritten.')
+ else
+ print('NOTICE: ' .. msg .. ' Skipped.')
+ end
+ else
+ _G[k] = v
+ end
+ end
+ end,
+})
+
+return exports
diff --git a/contrib/lua-lpeg/CMakeLists.txt b/contrib/lua-lpeg/CMakeLists.txt
new file mode 100644
index 0000000..92dd018
--- /dev/null
+++ b/contrib/lua-lpeg/CMakeLists.txt
@@ -0,0 +1,9 @@
+SET(LPEGSRC lpcap.c
+ lpcode.c
+ lpprint.c
+ lptree.c
+ lpvm.c)
+
+SET(LIB_TYPE STATIC)
+ADD_LIBRARY(rspamd-lpeg ${LIB_TYPE} ${LPEGSRC})
+set_target_properties(rspamd-lpeg PROPERTIES COMPILE_FLAGS "${LPEG_CFLAGS}")
diff --git a/contrib/lua-lpeg/LICENSE b/contrib/lua-lpeg/LICENSE
new file mode 100644
index 0000000..cea2d8b
--- /dev/null
+++ b/contrib/lua-lpeg/LICENSE
@@ -0,0 +1,19 @@
+Copyright © 2007-2015 Lua.org, PUC-Rio.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/contrib/lua-lpeg/lpcap.c b/contrib/lua-lpeg/lpcap.c
new file mode 100644
index 0000000..b332fde
--- /dev/null
+++ b/contrib/lua-lpeg/lpcap.c
@@ -0,0 +1,555 @@
+/*
+** $Id: lpcap.c $
+** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+*/
+
+#include "lua.h"
+#include "lauxlib.h"
+
+#include "lpcap.h"
+#include "lptypes.h"
+
+
+#define captype(cap) ((cap)->kind)
+
+#define isclosecap(cap) (captype(cap) == Cclose)
+
+#define closeaddr(c) ((c)->s + (c)->siz - 1)
+
+#define isfullcap(cap) ((cap)->siz != 0)
+
+#define getfromktable(cs,v) lua_rawgeti((cs)->L, ktableidx((cs)->ptop), v)
+
+#define pushluaval(cs) getfromktable(cs, (cs)->cap->idx)
+
+
+
+/*
+** Put at the cache for Lua values the value indexed by 'v' in ktable
+** of the running pattern (if it is not there yet); returns its index.
+*/
+static int updatecache (CapState *cs, int v) {
+ int idx = cs->ptop + 1; /* stack index of cache for Lua values */
+ if (v != cs->valuecached) { /* not there? */
+ getfromktable(cs, v); /* get value from 'ktable' */
+ lua_replace(cs->L, idx); /* put it at reserved stack position */
+ cs->valuecached = v; /* keep track of what is there */
+ }
+ return idx;
+}
+
+
+static int pushcapture (CapState *cs);
+
+
+/*
+** Goes back in a list of captures looking for an open capture
+** corresponding to a close
+*/
+static Capture *findopen (Capture *cap) {
+ int n = 0; /* number of closes waiting an open */
+ for (;;) {
+ cap--;
+ if (isclosecap(cap)) n++; /* one more open to skip */
+ else if (!isfullcap(cap))
+ if (n-- == 0) return cap;
+ }
+}
+
+
+/*
+** Go to the next capture
+*/
+static void nextcap (CapState *cs) {
+ Capture *cap = cs->cap;
+ if (!isfullcap(cap)) { /* not a single capture? */
+ int n = 0; /* number of opens waiting a close */
+ for (;;) { /* look for corresponding close */
+ cap++;
+ if (isclosecap(cap)) {
+ if (n-- == 0) break;
+ }
+ else if (!isfullcap(cap)) n++;
+ }
+ }
+ cs->cap = cap + 1; /* + 1 to skip last close (or entire single capture) */
+}
+
+
+/*
+** Push on the Lua stack all values generated by nested captures inside
+** the current capture. Returns number of values pushed. 'addextra'
+** makes it push the entire match after all captured values. The
+** entire match is pushed also if there are no other nested values,
+** so the function never returns zero.
+*/
+static int pushnestedvalues (CapState *cs, int addextra) {
+ Capture *co = cs->cap;
+ if (isfullcap(cs->cap++)) { /* no nested captures? */
+ lua_pushlstring(cs->L, co->s, co->siz - 1); /* push whole match */
+ return 1; /* that is it */
+ }
+ else {
+ int n = 0;
+ while (!isclosecap(cs->cap)) /* repeat for all nested patterns */
+ n += pushcapture(cs);
+ if (addextra || n == 0) { /* need extra? */
+ lua_pushlstring(cs->L, co->s, cs->cap->s - co->s); /* push whole match */
+ n++;
+ }
+ cs->cap++; /* skip close entry */
+ return n;
+ }
+}
+
+
+/*
+** Push only the first value generated by nested captures
+*/
+static void pushonenestedvalue (CapState *cs) {
+ int n = pushnestedvalues(cs, 0);
+ if (n > 1)
+ lua_pop(cs->L, n - 1); /* pop extra values */
+}
+
+
+/*
+** Try to find a named group capture with the name given at the top of
+** the stack; goes backward from 'cap'.
+*/
+static Capture *findback (CapState *cs, Capture *cap) {
+ lua_State *L = cs->L;
+ while (cap-- > cs->ocap) { /* repeat until end of list */
+ if (isclosecap(cap))
+ cap = findopen(cap); /* skip nested captures */
+ else if (!isfullcap(cap))
+ continue; /* opening an enclosing capture: skip and get previous */
+ if (captype(cap) == Cgroup) {
+ getfromktable(cs, cap->idx); /* get group name */
+ if (lp_equal(L, -2, -1)) { /* right group? */
+ lua_pop(L, 2); /* remove reference name and group name */
+ return cap;
+ }
+ else lua_pop(L, 1); /* remove group name */
+ }
+ }
+ luaL_error(L, "back reference '%s' not found", lua_tostring(L, -1));
+ return NULL; /* to avoid warnings */
+}
+
+
+/*
+** Back-reference capture. Return number of values pushed.
+*/
+static int backrefcap (CapState *cs) {
+ int n;
+ Capture *curr = cs->cap;
+ pushluaval(cs); /* reference name */
+ cs->cap = findback(cs, curr); /* find corresponding group */
+ n = pushnestedvalues(cs, 0); /* push group's values */
+ cs->cap = curr + 1;
+ return n;
+}
+
+
+/*
+** Table capture: creates a new table and populates it with nested
+** captures.
+*/
+static int tablecap (CapState *cs) {
+ lua_State *L = cs->L;
+ int n = 0;
+ lua_newtable(L);
+ if (isfullcap(cs->cap++))
+ return 1; /* table is empty */
+ while (!isclosecap(cs->cap)) {
+ if (captype(cs->cap) == Cgroup && cs->cap->idx != 0) { /* named group? */
+ pushluaval(cs); /* push group name */
+ pushonenestedvalue(cs);
+ lua_settable(L, -3);
+ }
+ else { /* not a named group */
+ int i;
+ int k = pushcapture(cs);
+ for (i = k; i > 0; i--) /* store all values into table */
+ lua_rawseti(L, -(i + 1), n + i);
+ n += k;
+ }
+ }
+ cs->cap++; /* skip close entry */
+ return 1; /* number of values pushed (only the table) */
+}
+
+
+/*
+** Table-query capture
+*/
+static int querycap (CapState *cs) {
+ int idx = cs->cap->idx;
+ pushonenestedvalue(cs); /* get nested capture */
+ lua_gettable(cs->L, updatecache(cs, idx)); /* query cap. value at table */
+ if (!lua_isnil(cs->L, -1))
+ return 1;
+ else { /* no value */
+ lua_pop(cs->L, 1); /* remove nil */
+ return 0;
+ }
+}
+
+
+/*
+** Fold capture
+*/
+static int foldcap (CapState *cs) {
+ int n;
+ lua_State *L = cs->L;
+ int idx = cs->cap->idx;
+ if (isfullcap(cs->cap++) || /* no nested captures? */
+ isclosecap(cs->cap) || /* no nested captures (large subject)? */
+ (n = pushcapture(cs)) == 0) /* nested captures with no values? */
+ return luaL_error(L, "no initial value for fold capture");
+ if (n > 1)
+ lua_pop(L, n - 1); /* leave only one result for accumulator */
+ while (!isclosecap(cs->cap)) {
+ lua_pushvalue(L, updatecache(cs, idx)); /* get folding function */
+ lua_insert(L, -2); /* put it before accumulator */
+ n = pushcapture(cs); /* get next capture's values */
+ lua_call(L, n + 1, 1); /* call folding function */
+ }
+ cs->cap++; /* skip close entry */
+ return 1; /* only accumulator left on the stack */
+}
+
+
+/*
+** Function capture
+*/
+static int functioncap (CapState *cs) {
+ int n;
+ int top = lua_gettop(cs->L);
+ pushluaval(cs); /* push function */
+ n = pushnestedvalues(cs, 0); /* push nested captures */
+ lua_call(cs->L, n, LUA_MULTRET); /* call function */
+ return lua_gettop(cs->L) - top; /* return function's results */
+}
+
+
+/*
+** Select capture
+*/
+static int numcap (CapState *cs) {
+ int idx = cs->cap->idx; /* value to select */
+ if (idx == 0) { /* no values? */
+ nextcap(cs); /* skip entire capture */
+ return 0; /* no value produced */
+ }
+ else {
+ int n = pushnestedvalues(cs, 0);
+ if (n < idx) /* invalid index? */
+ return luaL_error(cs->L, "no capture '%d'", idx);
+ else {
+ lua_pushvalue(cs->L, -(n - idx + 1)); /* get selected capture */
+ lua_replace(cs->L, -(n + 1)); /* put it in place of 1st capture */
+ lua_pop(cs->L, n - 1); /* remove other captures */
+ return 1;
+ }
+ }
+}
+
+
+/*
+** Return the stack index of the first runtime capture in the given
+** list of captures (or zero if no runtime captures)
+*/
+int finddyncap (Capture *cap, Capture *last) {
+ for (; cap < last; cap++) {
+ if (cap->kind == Cruntime)
+ return cap->idx; /* stack position of first capture */
+ }
+ return 0; /* no dynamic captures in this segment */
+}
+
+
+/*
+** Calls a runtime capture. Returns number of captures "removed" by the
+** call, that is, those inside the group capture. Captures to be added
+** are on the Lua stack.
+*/
+int runtimecap (CapState *cs, Capture *close, const char *s, int *rem) {
+ int n, id;
+ lua_State *L = cs->L;
+ int otop = lua_gettop(L);
+ Capture *open = findopen(close); /* get open group capture */
+ assert(captype(open) == Cgroup);
+ id = finddyncap(open, close); /* get first dynamic capture argument */
+ close->kind = Cclose; /* closes the group */
+ close->s = s;
+ cs->cap = open; cs->valuecached = 0; /* prepare capture state */
+ luaL_checkstack(L, 4, "too many runtime captures");
+ pushluaval(cs); /* push function to be called */
+ lua_pushvalue(L, SUBJIDX); /* push original subject */
+ lua_pushinteger(L, s - cs->s + 1); /* push current position */
+ n = pushnestedvalues(cs, 0); /* push nested captures */
+ lua_call(L, n + 2, LUA_MULTRET); /* call dynamic function */
+ if (id > 0) { /* are there old dynamic captures to be removed? */
+ int i;
+ for (i = id; i <= otop; i++)
+ lua_remove(L, id); /* remove old dynamic captures */
+ *rem = otop - id + 1; /* total number of dynamic captures removed */
+ }
+ else
+ *rem = 0; /* no dynamic captures removed */
+ return close - open - 1; /* number of captures to be removed */
+}
+
+
+/*
+** Auxiliary structure for substitution and string captures: keep
+** information about nested captures for future use, avoiding to push
+** string results into Lua
+*/
+typedef struct StrAux {
+ int isstring; /* whether capture is a string */
+ union {
+ Capture *cp; /* if not a string, respective capture */
+ struct { /* if it is a string... */
+ const char *s; /* ... starts here */
+ const char *e; /* ... ends here */
+ } s;
+ } u;
+} StrAux;
+
+#define MAXSTRCAPS 10
+
+/*
+** Collect values from current capture into array 'cps'. Current
+** capture must be Cstring (first call) or Csimple (recursive calls).
+** (In first call, fills %0 with whole match for Cstring.)
+** Returns number of elements in the array that were filled.
+*/
+static int getstrcaps (CapState *cs, StrAux *cps, int n) {
+ int k = n++;
+ cps[k].isstring = 1; /* get string value */
+ cps[k].u.s.s = cs->cap->s; /* starts here */
+ if (!isfullcap(cs->cap++)) { /* nested captures? */
+ while (!isclosecap(cs->cap)) { /* traverse them */
+ if (n >= MAXSTRCAPS) /* too many captures? */
+ nextcap(cs); /* skip extra captures (will not need them) */
+ else if (captype(cs->cap) == Csimple) /* string? */
+ n = getstrcaps(cs, cps, n); /* put info. into array */
+ else {
+ cps[n].isstring = 0; /* not a string */
+ cps[n].u.cp = cs->cap; /* keep original capture */
+ nextcap(cs);
+ n++;
+ }
+ }
+ cs->cap++; /* skip close */
+ }
+ cps[k].u.s.e = closeaddr(cs->cap - 1); /* ends here */
+ return n;
+}
+
+
+/*
+** add next capture value (which should be a string) to buffer 'b'
+*/
+static int addonestring (luaL_Buffer *b, CapState *cs, const char *what);
+
+
+/*
+** String capture: add result to buffer 'b' (instead of pushing
+** it into the stack)
+*/
+static void stringcap (luaL_Buffer *b, CapState *cs) {
+ StrAux cps[MAXSTRCAPS];
+ int n;
+ size_t len, i;
+ const char *fmt; /* format string */
+ fmt = lua_tolstring(cs->L, updatecache(cs, cs->cap->idx), &len);
+ n = getstrcaps(cs, cps, 0) - 1; /* collect nested captures */
+ for (i = 0; i < len; i++) { /* traverse them */
+ if (fmt[i] != '%') /* not an escape? */
+ luaL_addchar(b, fmt[i]); /* add it to buffer */
+ else if (fmt[++i] < '0' || fmt[i] > '9') /* not followed by a digit? */
+ luaL_addchar(b, fmt[i]); /* add to buffer */
+ else {
+ int l = fmt[i] - '0'; /* capture index */
+ if (l > n)
+ luaL_error(cs->L, "invalid capture index (%d)", l);
+ else if (cps[l].isstring)
+ luaL_addlstring(b, cps[l].u.s.s, cps[l].u.s.e - cps[l].u.s.s);
+ else {
+ Capture *curr = cs->cap;
+ cs->cap = cps[l].u.cp; /* go back to evaluate that nested capture */
+ if (!addonestring(b, cs, "capture"))
+ luaL_error(cs->L, "no values in capture index %d", l);
+ cs->cap = curr; /* continue from where it stopped */
+ }
+ }
+ }
+}
+
+
+/*
+** Substitution capture: add result to buffer 'b'
+*/
+static void substcap (luaL_Buffer *b, CapState *cs) {
+ const char *curr = cs->cap->s;
+ if (isfullcap(cs->cap)) /* no nested captures? */
+ luaL_addlstring(b, curr, cs->cap->siz - 1); /* keep original text */
+ else {
+ cs->cap++; /* skip open entry */
+ while (!isclosecap(cs->cap)) { /* traverse nested captures */
+ const char *next = cs->cap->s;
+ luaL_addlstring(b, curr, next - curr); /* add text up to capture */
+ if (addonestring(b, cs, "replacement"))
+ curr = closeaddr(cs->cap - 1); /* continue after match */
+ else /* no capture value */
+ curr = next; /* keep original text in final result */
+ }
+ luaL_addlstring(b, curr, cs->cap->s - curr); /* add last piece of text */
+ }
+ cs->cap++; /* go to next capture */
+}
+
+
+/*
+** Evaluates a capture and adds its first value to buffer 'b'; returns
+** whether there was a value
+*/
+static int addonestring (luaL_Buffer *b, CapState *cs, const char *what) {
+ switch (captype(cs->cap)) {
+ case Cstring:
+ stringcap(b, cs); /* add capture directly to buffer */
+ return 1;
+ case Csubst:
+ substcap(b, cs); /* add capture directly to buffer */
+ return 1;
+ default: {
+ lua_State *L = cs->L;
+ int n = pushcapture(cs);
+ if (n > 0) {
+ if (n > 1) lua_pop(L, n - 1); /* only one result */
+ if (!lua_isstring(L, -1))
+ luaL_error(L, "invalid %s value (a %s)", what, luaL_typename(L, -1));
+ luaL_addvalue(b);
+ }
+ return n;
+ }
+ }
+}
+
+
+#if !defined(MAXRECLEVEL)
+#define MAXRECLEVEL 200
+#endif
+
+
+/*
+** Push all values of the current capture into the stack; returns
+** number of values pushed
+*/
+static int pushcapture (CapState *cs) {
+ lua_State *L = cs->L;
+ int res;
+ luaL_checkstack(L, 4, "too many captures");
+ if (cs->reclevel++ > MAXRECLEVEL)
+ return luaL_error(L, "subcapture nesting too deep");
+ switch (captype(cs->cap)) {
+ case Cposition: {
+ lua_pushinteger(L, cs->cap->s - cs->s + 1);
+ cs->cap++;
+ res = 1;
+ break;
+ }
+ case Cconst: {
+ pushluaval(cs);
+ cs->cap++;
+ res = 1;
+ break;
+ }
+ case Carg: {
+ int arg = (cs->cap++)->idx;
+ if (arg + FIXEDARGS > cs->ptop)
+ return luaL_error(L, "reference to absent extra argument #%d", arg);
+ lua_pushvalue(L, arg + FIXEDARGS);
+ res = 1;
+ break;
+ }
+ case Csimple: {
+ int k = pushnestedvalues(cs, 1);
+ lua_insert(L, -k); /* make whole match be first result */
+ res = k;
+ break;
+ }
+ case Cruntime: {
+ lua_pushvalue(L, (cs->cap++)->idx); /* value is in the stack */
+ res = 1;
+ break;
+ }
+ case Cstring: {
+ luaL_Buffer b;
+ luaL_buffinit(L, &b);
+ stringcap(&b, cs);
+ luaL_pushresult(&b);
+ res = 1;
+ break;
+ }
+ case Csubst: {
+ luaL_Buffer b;
+ luaL_buffinit(L, &b);
+ substcap(&b, cs);
+ luaL_pushresult(&b);
+ res = 1;
+ break;
+ }
+ case Cgroup: {
+ if (cs->cap->idx == 0) /* anonymous group? */
+ res = pushnestedvalues(cs, 0); /* add all nested values */
+ else { /* named group: add no values */
+ nextcap(cs); /* skip capture */
+ res = 0;
+ }
+ break;
+ }
+ case Cbackref: res = backrefcap(cs); break;
+ case Ctable: res = tablecap(cs); break;
+ case Cfunction: res = functioncap(cs); break;
+ case Cnum: res = numcap(cs); break;
+ case Cquery: res = querycap(cs); break;
+ case Cfold: res = foldcap(cs); break;
+ default: assert(0); res = 0;
+ }
+ cs->reclevel--;
+ return res;
+}
+
+
+/*
+** Prepare a CapState structure and traverse the entire list of
+** captures in the stack pushing its results. 's' is the subject
+** string, 'r' is the final position of the match, and 'ptop'
+** the index in the stack where some useful values were pushed.
+** Returns the number of results pushed. (If the list produces no
+** results, push the final position of the match.)
+*/
+int getcaptures (lua_State *L, const char *s, const char *r, int ptop) {
+ Capture *capture = (Capture *)lua_touserdata(L, caplistidx(ptop));
+ int n = 0;
+ if (!isclosecap(capture)) { /* is there any capture? */
+ CapState cs;
+ cs.ocap = cs.cap = capture; cs.L = L; cs.reclevel = 0;
+ cs.s = s; cs.valuecached = 0; cs.ptop = ptop;
+ do { /* collect their values */
+ n += pushcapture(&cs);
+ } while (!isclosecap(cs.cap));
+ }
+ if (n == 0) { /* no capture values? */
+ lua_pushinteger(L, r - s + 1); /* return only end position */
+ n = 1;
+ }
+ return n;
+}
+
+
diff --git a/contrib/lua-lpeg/lpcap.h b/contrib/lua-lpeg/lpcap.h
new file mode 100644
index 0000000..dc10d69
--- /dev/null
+++ b/contrib/lua-lpeg/lpcap.h
@@ -0,0 +1,57 @@
+/*
+** $Id: lpcap.h $
+*/
+
+#if !defined(lpcap_h)
+#define lpcap_h
+
+
+#include "lptypes.h"
+
+
+/* kinds of captures */
+typedef enum CapKind {
+ Cclose, /* not used in trees */
+ Cposition,
+ Cconst, /* ktable[key] is Lua constant */
+ Cbackref, /* ktable[key] is "name" of group to get capture */
+ Carg, /* 'key' is arg's number */
+ Csimple, /* next node is pattern */
+ Ctable, /* next node is pattern */
+ Cfunction, /* ktable[key] is function; next node is pattern */
+ Cquery, /* ktable[key] is table; next node is pattern */
+ Cstring, /* ktable[key] is string; next node is pattern */
+ Cnum, /* numbered capture; 'key' is number of value to return */
+ Csubst, /* substitution capture; next node is pattern */
+ Cfold, /* ktable[key] is function; next node is pattern */
+ Cruntime, /* not used in trees (is uses another type for tree) */
+ Cgroup /* ktable[key] is group's "name" */
+} CapKind;
+
+
+typedef struct Capture {
+ const char *s; /* subject position */
+ unsigned short idx; /* extra info (group name, arg index, etc.) */
+ byte kind; /* kind of capture */
+ byte siz; /* size of full capture + 1 (0 = not a full capture) */
+} Capture;
+
+
+typedef struct CapState {
+ Capture *cap; /* current capture */
+ Capture *ocap; /* (original) capture list */
+ lua_State *L;
+ int ptop; /* index of last argument to 'match' */
+ const char *s; /* original string */
+ int valuecached; /* value stored in cache slot */
+ int reclevel; /* recursion level */
+} CapState;
+
+
+int runtimecap (CapState *cs, Capture *close, const char *s, int *rem);
+int getcaptures (lua_State *L, const char *s, const char *r, int ptop);
+int finddyncap (Capture *cap, Capture *last);
+
+#endif
+
+
diff --git a/contrib/lua-lpeg/lpcode.c b/contrib/lua-lpeg/lpcode.c
new file mode 100644
index 0000000..3923459
--- /dev/null
+++ b/contrib/lua-lpeg/lpcode.c
@@ -0,0 +1,1014 @@
+/*
+** $Id: lpcode.c $
+** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+*/
+
+#include <limits.h>
+
+
+#include "lua.h"
+#include "lauxlib.h"
+
+#include "lptypes.h"
+#include "lpcode.h"
+
+
+/* signals a "no-instruction */
+#define NOINST -1
+
+
+
+static const Charset fullset_ =
+ {{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}};
+
+static const Charset *fullset = &fullset_;
+
+/*
+** {======================================================
+** Analysis and some optimizations
+** =======================================================
+*/
+
+/*
+** Check whether a charset is empty (returns IFail), singleton (IChar),
+** full (IAny), or none of those (ISet). When singleton, '*c' returns
+** which character it is. (When generic set, the set was the input,
+** so there is no need to return it.)
+*/
+static Opcode charsettype (const byte *cs, int *c) {
+ int count = 0; /* number of characters in the set */
+ int i;
+ int candidate = -1; /* candidate position for the singleton char */
+ for (i = 0; i < CHARSETSIZE; i++) { /* for each byte */
+ int b = cs[i];
+ if (b == 0) { /* is byte empty? */
+ if (count > 1) /* was set neither empty nor singleton? */
+ return ISet; /* neither full nor empty nor singleton */
+ /* else set is still empty or singleton */
+ }
+ else if (b == 0xFF) { /* is byte full? */
+ if (count < (i * BITSPERCHAR)) /* was set not full? */
+ return ISet; /* neither full nor empty nor singleton */
+ else count += BITSPERCHAR; /* set is still full */
+ }
+ else if ((b & (b - 1)) == 0) { /* has byte only one bit? */
+ if (count > 0) /* was set not empty? */
+ return ISet; /* neither full nor empty nor singleton */
+ else { /* set has only one char till now; track it */
+ count++;
+ candidate = i;
+ }
+ }
+ else return ISet; /* byte is neither empty, full, nor singleton */
+ }
+ switch (count) {
+ case 0: return IFail; /* empty set */
+ case 1: { /* singleton; find character bit inside byte */
+ int b = cs[candidate];
+ *c = candidate * BITSPERCHAR;
+ if ((b & 0xF0) != 0) { *c += 4; b >>= 4; }
+ if ((b & 0x0C) != 0) { *c += 2; b >>= 2; }
+ if ((b & 0x02) != 0) { *c += 1; }
+ return IChar;
+ }
+ default: {
+ assert(count == CHARSETSIZE * BITSPERCHAR); /* full set */
+ return IAny;
+ }
+ }
+}
+
+
+/*
+** A few basic operations on Charsets
+*/
+static void cs_complement (Charset *cs) {
+ loopset(i, cs->cs[i] = ~cs->cs[i]);
+}
+
+static int cs_equal (const byte *cs1, const byte *cs2) {
+ loopset(i, if (cs1[i] != cs2[i]) return 0);
+ return 1;
+}
+
+static int cs_disjoint (const Charset *cs1, const Charset *cs2) {
+ loopset(i, if ((cs1->cs[i] & cs2->cs[i]) != 0) return 0;)
+ return 1;
+}
+
+
+/*
+** If 'tree' is a 'char' pattern (TSet, TChar, TAny), convert it into a
+** charset and return 1; else return 0.
+*/
+int tocharset (TTree *tree, Charset *cs) {
+ switch (tree->tag) {
+ case TSet: { /* copy set */
+ loopset(i, cs->cs[i] = treebuffer(tree)[i]);
+ return 1;
+ }
+ case TChar: { /* only one char */
+ assert(0 <= tree->u.n && tree->u.n <= UCHAR_MAX);
+ loopset(i, cs->cs[i] = 0); /* erase all chars */
+ setchar(cs->cs, tree->u.n); /* add that one */
+ return 1;
+ }
+ case TAny: {
+ loopset(i, cs->cs[i] = 0xFF); /* add all characters to the set */
+ return 1;
+ }
+ default: return 0;
+ }
+}
+
+
+/*
+** Visit a TCall node taking care to stop recursion. If node not yet
+** visited, return 'f(sib2(tree))', otherwise return 'def' (default
+** value)
+*/
+static int callrecursive (TTree *tree, int f (TTree *t), int def) {
+ int key = tree->key;
+ assert(tree->tag == TCall);
+ assert(sib2(tree)->tag == TRule);
+ if (key == 0) /* node already visited? */
+ return def; /* return default value */
+ else { /* first visit */
+ int result;
+ tree->key = 0; /* mark call as already visited */
+ result = f(sib2(tree)); /* go to called rule */
+ tree->key = key; /* restore tree */
+ return result;
+ }
+}
+
+
+/*
+** Check whether a pattern tree has captures
+*/
+int hascaptures (TTree *tree) {
+ tailcall:
+ switch (tree->tag) {
+ case TCapture: case TRunTime:
+ return 1;
+ case TCall:
+ return callrecursive(tree, hascaptures, 0);
+ case TRule: /* do not follow siblings */
+ tree = sib1(tree); goto tailcall;
+ case TOpenCall: assert(0);
+ default: {
+ switch (numsiblings[tree->tag]) {
+ case 1: /* return hascaptures(sib1(tree)); */
+ tree = sib1(tree); goto tailcall;
+ case 2:
+ if (hascaptures(sib1(tree)))
+ return 1;
+ /* else return hascaptures(sib2(tree)); */
+ tree = sib2(tree); goto tailcall;
+ default: assert(numsiblings[tree->tag] == 0); return 0;
+ }
+ }
+ }
+}
+
+
+/*
+** Checks how a pattern behaves regarding the empty string,
+** in one of two different ways:
+** A pattern is *nullable* if it can match without consuming any character;
+** A pattern is *nofail* if it never fails for any string
+** (including the empty string).
+** The difference is only for predicates and run-time captures;
+** for other patterns, the two properties are equivalent.
+** (With predicates, &'a' is nullable but not nofail. Of course,
+** nofail => nullable.)
+** These functions are all convervative in the following way:
+** p is nullable => nullable(p)
+** nofail(p) => p cannot fail
+** The function assumes that TOpenCall is not nullable;
+** this will be checked again when the grammar is fixed.
+** Run-time captures can do whatever they want, so the result
+** is conservative.
+*/
+int checkaux (TTree *tree, int pred) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny:
+ case TFalse: case TOpenCall:
+ return 0; /* not nullable */
+ case TRep: case TTrue:
+ return 1; /* no fail */
+ case TNot: case TBehind: /* can match empty, but can fail */
+ if (pred == PEnofail) return 0;
+ else return 1; /* PEnullable */
+ case TAnd: /* can match empty; fail iff body does */
+ if (pred == PEnullable) return 1;
+ /* else return checkaux(sib1(tree), pred); */
+ tree = sib1(tree); goto tailcall;
+ case TRunTime: /* can fail; match empty iff body does */
+ if (pred == PEnofail) return 0;
+ /* else return checkaux(sib1(tree), pred); */
+ tree = sib1(tree); goto tailcall;
+ case TSeq:
+ if (!checkaux(sib1(tree), pred)) return 0;
+ /* else return checkaux(sib2(tree), pred); */
+ tree = sib2(tree); goto tailcall;
+ case TChoice:
+ if (checkaux(sib2(tree), pred)) return 1;
+ /* else return checkaux(sib1(tree), pred); */
+ tree = sib1(tree); goto tailcall;
+ case TCapture: case TGrammar: case TRule:
+ /* return checkaux(sib1(tree), pred); */
+ tree = sib1(tree); goto tailcall;
+ case TCall: /* return checkaux(sib2(tree), pred); */
+ tree = sib2(tree); goto tailcall;
+ default: assert(0); return 0;
+ }
+}
+
+
+/*
+** number of characters to match a pattern (or -1 if variable)
+*/
+int fixedlen (TTree *tree) {
+ int len = 0; /* to accumulate in tail calls */
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny:
+ return len + 1;
+ case TFalse: case TTrue: case TNot: case TAnd: case TBehind:
+ return len;
+ case TRep: case TRunTime: case TOpenCall:
+ return -1;
+ case TCapture: case TRule: case TGrammar:
+ /* return fixedlen(sib1(tree)); */
+ tree = sib1(tree); goto tailcall;
+ case TCall: {
+ int n1 = callrecursive(tree, fixedlen, -1);
+ if (n1 < 0)
+ return -1;
+ else
+ return len + n1;
+ }
+ case TSeq: {
+ int n1 = fixedlen(sib1(tree));
+ if (n1 < 0)
+ return -1;
+ /* else return fixedlen(sib2(tree)) + len; */
+ len += n1; tree = sib2(tree); goto tailcall;
+ }
+ case TChoice: {
+ int n1 = fixedlen(sib1(tree));
+ int n2 = fixedlen(sib2(tree));
+ if (n1 != n2 || n1 < 0)
+ return -1;
+ else
+ return len + n1;
+ }
+ default: assert(0); return 0;
+ };
+}
+
+
+/*
+** Computes the 'first set' of a pattern.
+** The result is a conservative aproximation:
+** match p ax -> x (for some x) ==> a belongs to first(p)
+** or
+** a not in first(p) ==> match p ax -> fail (for all x)
+**
+** The set 'follow' is the first set of what follows the
+** pattern (full set if nothing follows it).
+**
+** The function returns 0 when this resulting set can be used for
+** test instructions that avoid the pattern altogether.
+** A non-zero return can happen for two reasons:
+** 1) match p '' -> '' ==> return has bit 1 set
+** (tests cannot be used because they would always fail for an empty input);
+** 2) there is a match-time capture ==> return has bit 2 set
+** (optimizations should not bypass match-time captures).
+*/
+static int getfirst (TTree *tree, const Charset *follow, Charset *firstset) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny: {
+ tocharset(tree, firstset);
+ return 0;
+ }
+ case TTrue: {
+ loopset(i, firstset->cs[i] = follow->cs[i]);
+ return 1; /* accepts the empty string */
+ }
+ case TFalse: {
+ loopset(i, firstset->cs[i] = 0);
+ return 0;
+ }
+ case TChoice: {
+ Charset csaux;
+ int e1 = getfirst(sib1(tree), follow, firstset);
+ int e2 = getfirst(sib2(tree), follow, &csaux);
+ loopset(i, firstset->cs[i] |= csaux.cs[i]);
+ return e1 | e2;
+ }
+ case TSeq: {
+ if (!nullable(sib1(tree))) {
+ /* when p1 is not nullable, p2 has nothing to contribute;
+ return getfirst(sib1(tree), fullset, firstset); */
+ tree = sib1(tree); follow = fullset; goto tailcall;
+ }
+ else { /* FIRST(p1 p2, fl) = FIRST(p1, FIRST(p2, fl)) */
+ Charset csaux;
+ int e2 = getfirst(sib2(tree), follow, &csaux);
+ int e1 = getfirst(sib1(tree), &csaux, firstset);
+ if (e1 == 0) return 0; /* 'e1' ensures that first can be used */
+ else if ((e1 | e2) & 2) /* one of the children has a matchtime? */
+ return 2; /* pattern has a matchtime capture */
+ else return e2; /* else depends on 'e2' */
+ }
+ }
+ case TRep: {
+ getfirst(sib1(tree), follow, firstset);
+ loopset(i, firstset->cs[i] |= follow->cs[i]);
+ return 1; /* accept the empty string */
+ }
+ case TCapture: case TGrammar: case TRule: {
+ /* return getfirst(sib1(tree), follow, firstset); */
+ tree = sib1(tree); goto tailcall;
+ }
+ case TRunTime: { /* function invalidates any follow info. */
+ int e = getfirst(sib1(tree), fullset, firstset);
+ if (e) return 2; /* function is not "protected"? */
+ else return 0; /* pattern inside capture ensures first can be used */
+ }
+ case TCall: {
+ /* return getfirst(sib2(tree), follow, firstset); */
+ tree = sib2(tree); goto tailcall;
+ }
+ case TAnd: {
+ int e = getfirst(sib1(tree), follow, firstset);
+ loopset(i, firstset->cs[i] &= follow->cs[i]);
+ return e;
+ }
+ case TNot: {
+ if (tocharset(sib1(tree), firstset)) {
+ cs_complement(firstset);
+ return 1;
+ }
+ /* else go through */
+ }
+ case TBehind: { /* instruction gives no new information */
+ /* call 'getfirst' only to check for math-time captures */
+ int e = getfirst(sib1(tree), follow, firstset);
+ loopset(i, firstset->cs[i] = follow->cs[i]); /* uses follow */
+ return e | 1; /* always can accept the empty string */
+ }
+ default: assert(0); return 0;
+ }
+}
+
+
+/*
+** If 'headfail(tree)' true, then 'tree' can fail only depending on the
+** next character of the subject.
+*/
+static int headfail (TTree *tree) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny: case TFalse:
+ return 1;
+ case TTrue: case TRep: case TRunTime: case TNot:
+ case TBehind:
+ return 0;
+ case TCapture: case TGrammar: case TRule: case TAnd:
+ tree = sib1(tree); goto tailcall; /* return headfail(sib1(tree)); */
+ case TCall:
+ tree = sib2(tree); goto tailcall; /* return headfail(sib2(tree)); */
+ case TSeq:
+ if (!nofail(sib2(tree))) return 0;
+ /* else return headfail(sib1(tree)); */
+ tree = sib1(tree); goto tailcall;
+ case TChoice:
+ if (!headfail(sib1(tree))) return 0;
+ /* else return headfail(sib2(tree)); */
+ tree = sib2(tree); goto tailcall;
+ default: assert(0); return 0;
+ }
+}
+
+
+/*
+** Check whether the code generation for the given tree can benefit
+** from a follow set (to avoid computing the follow set when it is
+** not needed)
+*/
+static int needfollow (TTree *tree) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny:
+ case TFalse: case TTrue: case TAnd: case TNot:
+ case TRunTime: case TGrammar: case TCall: case TBehind:
+ return 0;
+ case TChoice: case TRep:
+ return 1;
+ case TCapture:
+ tree = sib1(tree); goto tailcall;
+ case TSeq:
+ tree = sib2(tree); goto tailcall;
+ default: assert(0); return 0;
+ }
+}
+
+/* }====================================================== */
+
+
+
+/*
+** {======================================================
+** Code generation
+** =======================================================
+*/
+
+
+/*
+** size of an instruction
+*/
+int sizei (const Instruction *i) {
+ switch((Opcode)i->i.code) {
+ case ISet: case ISpan: return CHARSETINSTSIZE;
+ case ITestSet: return CHARSETINSTSIZE + 1;
+ case ITestChar: case ITestAny: case IChoice: case IJmp: case ICall:
+ case IOpenCall: case ICommit: case IPartialCommit: case IBackCommit:
+ return 2;
+ default: return 1;
+ }
+}
+
+
+/*
+** state for the compiler
+*/
+typedef struct CompileState {
+ Pattern *p; /* pattern being compiled */
+ int ncode; /* next position in p->code to be filled */
+ lua_State *L;
+} CompileState;
+
+
+/*
+** code generation is recursive; 'opt' indicates that the code is being
+** generated as the last thing inside an optional pattern (so, if that
+** code is optional too, it can reuse the 'IChoice' already in place for
+** the outer pattern). 'tt' points to a previous test protecting this
+** code (or NOINST). 'fl' is the follow set of the pattern.
+*/
+static void codegen (CompileState *compst, TTree *tree, int opt, int tt,
+ const Charset *fl);
+
+
+void realloccode (lua_State *L, Pattern *p, int nsize) {
+ void *ud;
+ lua_Alloc f = lua_getallocf(L, &ud);
+ void *newblock = f(ud, p->code, p->codesize * sizeof(Instruction),
+ nsize * sizeof(Instruction));
+ if (newblock == NULL && nsize > 0)
+ luaL_error(L, "not enough memory");
+ p->code = (Instruction *)newblock;
+ p->codesize = nsize;
+}
+
+
+static int nextinstruction (CompileState *compst) {
+ int size = compst->p->codesize;
+ if (compst->ncode >= size)
+ realloccode(compst->L, compst->p, size * 2);
+ return compst->ncode++;
+}
+
+
+#define getinstr(cs,i) ((cs)->p->code[i])
+
+
+static int addinstruction (CompileState *compst, Opcode op, int aux) {
+ int i = nextinstruction(compst);
+ getinstr(compst, i).i.code = op;
+ getinstr(compst, i).i.aux = aux;
+ return i;
+}
+
+
+/*
+** Add an instruction followed by space for an offset (to be set later)
+*/
+static int addoffsetinst (CompileState *compst, Opcode op) {
+ int i = addinstruction(compst, op, 0); /* instruction */
+ addinstruction(compst, (Opcode)0, 0); /* open space for offset */
+ assert(op == ITestSet || sizei(&getinstr(compst, i)) == 2);
+ return i;
+}
+
+
+/*
+** Set the offset of an instruction
+*/
+static void setoffset (CompileState *compst, int instruction, int offset) {
+ getinstr(compst, instruction + 1).offset = offset;
+}
+
+
+/*
+** Add a capture instruction:
+** 'op' is the capture instruction; 'cap' the capture kind;
+** 'key' the key into ktable; 'aux' is the optional capture offset
+**
+*/
+static int addinstcap (CompileState *compst, Opcode op, int cap, int key,
+ int aux) {
+ int i = addinstruction(compst, op, joinkindoff(cap, aux));
+ getinstr(compst, i).i.key = key;
+ return i;
+}
+
+
+#define gethere(compst) ((compst)->ncode)
+
+#define target(code,i) ((i) + code[i + 1].offset)
+
+
+/*
+** Patch 'instruction' to jump to 'target'
+*/
+static void jumptothere (CompileState *compst, int instruction, int target) {
+ if (instruction >= 0)
+ setoffset(compst, instruction, target - instruction);
+}
+
+
+/*
+** Patch 'instruction' to jump to current position
+*/
+static void jumptohere (CompileState *compst, int instruction) {
+ jumptothere(compst, instruction, gethere(compst));
+}
+
+
+/*
+** Code an IChar instruction, or IAny if there is an equivalent
+** test dominating it
+*/
+static void codechar (CompileState *compst, int c, int tt) {
+ if (tt >= 0 && getinstr(compst, tt).i.code == ITestChar &&
+ getinstr(compst, tt).i.aux == c)
+ addinstruction(compst, IAny, 0);
+ else
+ addinstruction(compst, IChar, c);
+}
+
+
+/*
+** Add a charset posfix to an instruction
+*/
+static void addcharset (CompileState *compst, const byte *cs) {
+ int p = gethere(compst);
+ int i;
+ for (i = 0; i < (int)CHARSETINSTSIZE - 1; i++)
+ nextinstruction(compst); /* space for buffer */
+ /* fill buffer with charset */
+ loopset(j, getinstr(compst, p).buff[j] = cs[j]);
+}
+
+
+/*
+** code a char set, optimizing unit sets for IChar, "complete"
+** sets for IAny, and empty sets for IFail; also use an IAny
+** when instruction is dominated by an equivalent test.
+*/
+static void codecharset (CompileState *compst, const byte *cs, int tt) {
+ int c = 0; /* (=) to avoid warnings */
+ Opcode op = charsettype(cs, &c);
+ switch (op) {
+ case IChar: codechar(compst, c, tt); break;
+ case ISet: { /* non-trivial set? */
+ if (tt >= 0 && getinstr(compst, tt).i.code == ITestSet &&
+ cs_equal(cs, getinstr(compst, tt + 2).buff))
+ addinstruction(compst, IAny, 0);
+ else {
+ addinstruction(compst, ISet, 0);
+ addcharset(compst, cs);
+ }
+ break;
+ }
+ default: addinstruction(compst, op, c); break;
+ }
+}
+
+
+/*
+** code a test set, optimizing unit sets for ITestChar, "complete"
+** sets for ITestAny, and empty sets for IJmp (always fails).
+** 'e' is true iff test should accept the empty string. (Test
+** instructions in the current VM never accept the empty string.)
+*/
+static int codetestset (CompileState *compst, Charset *cs, int e) {
+ if (e) return NOINST; /* no test */
+ else {
+ int c = 0;
+ Opcode op = charsettype(cs->cs, &c);
+ switch (op) {
+ case IFail: return addoffsetinst(compst, IJmp); /* always jump */
+ case IAny: return addoffsetinst(compst, ITestAny);
+ case IChar: {
+ int i = addoffsetinst(compst, ITestChar);
+ getinstr(compst, i).i.aux = c;
+ return i;
+ }
+ case ISet: {
+ int i = addoffsetinst(compst, ITestSet);
+ addcharset(compst, cs->cs);
+ return i;
+ }
+ default: assert(0); return 0;
+ }
+ }
+}
+
+
+/*
+** Find the final destination of a sequence of jumps
+*/
+static int finaltarget (Instruction *code, int i) {
+ while (code[i].i.code == IJmp)
+ i = target(code, i);
+ return i;
+}
+
+
+/*
+** final label (after traversing any jumps)
+*/
+static int finallabel (Instruction *code, int i) {
+ return finaltarget(code, target(code, i));
+}
+
+
+/*
+** <behind(p)> == behind n; <p> (where n = fixedlen(p))
+*/
+static void codebehind (CompileState *compst, TTree *tree) {
+ if (tree->u.n > 0)
+ addinstruction(compst, IBehind, tree->u.n);
+ codegen(compst, sib1(tree), 0, NOINST, fullset);
+}
+
+
+/*
+** Choice; optimizations:
+** - when p1 is headfail or
+** when first(p1) and first(p2) are disjoint, than
+** a character not in first(p1) cannot go to p1, and a character
+** in first(p1) cannot go to p2 (at it is not in first(p2)).
+** (The optimization is not valid if p1 accepts the empty string,
+** as then there is no character at all...)
+** - when p2 is empty and opt is true; a IPartialCommit can reuse
+** the Choice already active in the stack.
+*/
+static void codechoice (CompileState *compst, TTree *p1, TTree *p2, int opt,
+ const Charset *fl) {
+ int emptyp2 = (p2->tag == TTrue);
+ Charset cs1, cs2;
+ int e1 = getfirst(p1, fullset, &cs1);
+ if (headfail(p1) ||
+ (!e1 && (getfirst(p2, fl, &cs2), cs_disjoint(&cs1, &cs2)))) {
+ /* <p1 / p2> == test (fail(p1)) -> L1 ; p1 ; jmp L2; L1: p2; L2: */
+ int test = codetestset(compst, &cs1, 0);
+ int jmp = NOINST;
+ codegen(compst, p1, 0, test, fl);
+ if (!emptyp2)
+ jmp = addoffsetinst(compst, IJmp);
+ jumptohere(compst, test);
+ codegen(compst, p2, opt, NOINST, fl);
+ jumptohere(compst, jmp);
+ }
+ else if (opt && emptyp2) {
+ /* p1? == IPartialCommit; p1 */
+ jumptohere(compst, addoffsetinst(compst, IPartialCommit));
+ codegen(compst, p1, 1, NOINST, fullset);
+ }
+ else {
+ /* <p1 / p2> ==
+ test(first(p1)) -> L1; choice L1; <p1>; commit L2; L1: <p2>; L2: */
+ int pcommit;
+ int test = codetestset(compst, &cs1, e1);
+ int pchoice = addoffsetinst(compst, IChoice);
+ codegen(compst, p1, emptyp2, test, fullset);
+ pcommit = addoffsetinst(compst, ICommit);
+ jumptohere(compst, pchoice);
+ jumptohere(compst, test);
+ codegen(compst, p2, opt, NOINST, fl);
+ jumptohere(compst, pcommit);
+ }
+}
+
+
+/*
+** And predicate
+** optimization: fixedlen(p) = n ==> <&p> == <p>; behind n
+** (valid only when 'p' has no captures)
+*/
+static void codeand (CompileState *compst, TTree *tree, int tt) {
+ int n = fixedlen(tree);
+ if (n >= 0 && n <= MAXBEHIND && !hascaptures(tree)) {
+ codegen(compst, tree, 0, tt, fullset);
+ if (n > 0)
+ addinstruction(compst, IBehind, n);
+ }
+ else { /* default: Choice L1; p1; BackCommit L2; L1: Fail; L2: */
+ int pcommit;
+ int pchoice = addoffsetinst(compst, IChoice);
+ codegen(compst, tree, 0, tt, fullset);
+ pcommit = addoffsetinst(compst, IBackCommit);
+ jumptohere(compst, pchoice);
+ addinstruction(compst, IFail, 0);
+ jumptohere(compst, pcommit);
+ }
+}
+
+
+/*
+** Captures: if pattern has fixed (and not too big) length, and it
+** has no nested captures, use a single IFullCapture instruction
+** after the match; otherwise, enclose the pattern with OpenCapture -
+** CloseCapture.
+*/
+static void codecapture (CompileState *compst, TTree *tree, int tt,
+ const Charset *fl) {
+ int len = fixedlen(sib1(tree));
+ if (len >= 0 && len <= MAXOFF && !hascaptures(sib1(tree))) {
+ codegen(compst, sib1(tree), 0, tt, fl);
+ addinstcap(compst, IFullCapture, tree->cap, tree->key, len);
+ }
+ else {
+ addinstcap(compst, IOpenCapture, tree->cap, tree->key, 0);
+ codegen(compst, sib1(tree), 0, tt, fl);
+ addinstcap(compst, ICloseCapture, Cclose, 0, 0);
+ }
+}
+
+
+static void coderuntime (CompileState *compst, TTree *tree, int tt) {
+ addinstcap(compst, IOpenCapture, Cgroup, tree->key, 0);
+ codegen(compst, sib1(tree), 0, tt, fullset);
+ addinstcap(compst, ICloseRunTime, Cclose, 0, 0);
+}
+
+
+/*
+** Repetion; optimizations:
+** When pattern is a charset, can use special instruction ISpan.
+** When pattern is head fail, or if it starts with characters that
+** are disjoint from what follows the repetions, a simple test
+** is enough (a fail inside the repetition would backtrack to fail
+** again in the following pattern, so there is no need for a choice).
+** When 'opt' is true, the repetion can reuse the Choice already
+** active in the stack.
+*/
+static void coderep (CompileState *compst, TTree *tree, int opt,
+ const Charset *fl) {
+ Charset st;
+ if (tocharset(tree, &st)) {
+ addinstruction(compst, ISpan, 0);
+ addcharset(compst, st.cs);
+ }
+ else {
+ int e1 = getfirst(tree, fullset, &st);
+ if (headfail(tree) || (!e1 && cs_disjoint(&st, fl))) {
+ /* L1: test (fail(p1)) -> L2; <p>; jmp L1; L2: */
+ int jmp;
+ int test = codetestset(compst, &st, 0);
+ codegen(compst, tree, 0, test, fullset);
+ jmp = addoffsetinst(compst, IJmp);
+ jumptohere(compst, test);
+ jumptothere(compst, jmp, test);
+ }
+ else {
+ /* test(fail(p1)) -> L2; choice L2; L1: <p>; partialcommit L1; L2: */
+ /* or (if 'opt'): partialcommit L1; L1: <p>; partialcommit L1; */
+ int commit, l2;
+ int test = codetestset(compst, &st, e1);
+ int pchoice = NOINST;
+ if (opt)
+ jumptohere(compst, addoffsetinst(compst, IPartialCommit));
+ else
+ pchoice = addoffsetinst(compst, IChoice);
+ l2 = gethere(compst);
+ codegen(compst, tree, 0, NOINST, fullset);
+ commit = addoffsetinst(compst, IPartialCommit);
+ jumptothere(compst, commit, l2);
+ jumptohere(compst, pchoice);
+ jumptohere(compst, test);
+ }
+ }
+}
+
+
+/*
+** Not predicate; optimizations:
+** In any case, if first test fails, 'not' succeeds, so it can jump to
+** the end. If pattern is headfail, that is all (it cannot fail
+** in other parts); this case includes 'not' of simple sets. Otherwise,
+** use the default code (a choice plus a failtwice).
+*/
+static void codenot (CompileState *compst, TTree *tree) {
+ Charset st;
+ int e = getfirst(tree, fullset, &st);
+ int test = codetestset(compst, &st, e);
+ if (headfail(tree)) /* test (fail(p1)) -> L1; fail; L1: */
+ addinstruction(compst, IFail, 0);
+ else {
+ /* test(fail(p))-> L1; choice L1; <p>; failtwice; L1: */
+ int pchoice = addoffsetinst(compst, IChoice);
+ codegen(compst, tree, 0, NOINST, fullset);
+ addinstruction(compst, IFailTwice, 0);
+ jumptohere(compst, pchoice);
+ }
+ jumptohere(compst, test);
+}
+
+
+/*
+** change open calls to calls, using list 'positions' to find
+** correct offsets; also optimize tail calls
+*/
+static void correctcalls (CompileState *compst, int *positions,
+ int from, int to) {
+ int i;
+ Instruction *code = compst->p->code;
+ for (i = from; i < to; i += sizei(&code[i])) {
+ if (code[i].i.code == IOpenCall) {
+ int n = code[i].i.key; /* rule number */
+ int rule = positions[n]; /* rule position */
+ assert(rule == from || code[rule - 1].i.code == IRet);
+ if (code[finaltarget(code, i + 2)].i.code == IRet) /* call; ret ? */
+ code[i].i.code = IJmp; /* tail call */
+ else
+ code[i].i.code = ICall;
+ jumptothere(compst, i, rule); /* call jumps to respective rule */
+ }
+ }
+ assert(i == to);
+}
+
+
+/*
+** Code for a grammar:
+** call L1; jmp L2; L1: rule 1; ret; rule 2; ret; ...; L2:
+*/
+static void codegrammar (CompileState *compst, TTree *grammar) {
+ int positions[MAXRULES];
+ int rulenumber = 0;
+ TTree *rule;
+ int firstcall = addoffsetinst(compst, ICall); /* call initial rule */
+ int jumptoend = addoffsetinst(compst, IJmp); /* jump to the end */
+ int start = gethere(compst); /* here starts the initial rule */
+ jumptohere(compst, firstcall);
+ for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) {
+ positions[rulenumber++] = gethere(compst); /* save rule position */
+ codegen(compst, sib1(rule), 0, NOINST, fullset); /* code rule */
+ addinstruction(compst, IRet, 0);
+ }
+ assert(rule->tag == TTrue);
+ jumptohere(compst, jumptoend);
+ correctcalls(compst, positions, start, gethere(compst));
+}
+
+
+static void codecall (CompileState *compst, TTree *call) {
+ int c = addoffsetinst(compst, IOpenCall); /* to be corrected later */
+ getinstr(compst, c).i.key = sib2(call)->cap; /* rule number */
+ assert(sib2(call)->tag == TRule);
+}
+
+
+/*
+** Code first child of a sequence
+** (second child is called in-place to allow tail call)
+** Return 'tt' for second child
+*/
+static int codeseq1 (CompileState *compst, TTree *p1, TTree *p2,
+ int tt, const Charset *fl) {
+ if (needfollow(p1)) {
+ Charset fl1;
+ getfirst(p2, fl, &fl1); /* p1 follow is p2 first */
+ codegen(compst, p1, 0, tt, &fl1);
+ }
+ else /* use 'fullset' as follow */
+ codegen(compst, p1, 0, tt, fullset);
+ if (fixedlen(p1) != 0) /* can 'p1' consume anything? */
+ return NOINST; /* invalidate test */
+ else return tt; /* else 'tt' still protects sib2 */
+}
+
+
+/*
+** Main code-generation function: dispatch to auxiliar functions
+** according to kind of tree. ('needfollow' should return true
+** only for consructions that use 'fl'.)
+*/
+static void codegen (CompileState *compst, TTree *tree, int opt, int tt,
+ const Charset *fl) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: codechar(compst, tree->u.n, tt); break;
+ case TAny: addinstruction(compst, IAny, 0); break;
+ case TSet: codecharset(compst, treebuffer(tree), tt); break;
+ case TTrue: break;
+ case TFalse: addinstruction(compst, IFail, 0); break;
+ case TChoice: codechoice(compst, sib1(tree), sib2(tree), opt, fl); break;
+ case TRep: coderep(compst, sib1(tree), opt, fl); break;
+ case TBehind: codebehind(compst, tree); break;
+ case TNot: codenot(compst, sib1(tree)); break;
+ case TAnd: codeand(compst, sib1(tree), tt); break;
+ case TCapture: codecapture(compst, tree, tt, fl); break;
+ case TRunTime: coderuntime(compst, tree, tt); break;
+ case TGrammar: codegrammar(compst, tree); break;
+ case TCall: codecall(compst, tree); break;
+ case TSeq: {
+ tt = codeseq1(compst, sib1(tree), sib2(tree), tt, fl); /* code 'p1' */
+ /* codegen(compst, p2, opt, tt, fl); */
+ tree = sib2(tree); goto tailcall;
+ }
+ default: assert(0);
+ }
+}
+
+
+/*
+** Optimize jumps and other jump-like instructions.
+** * Update labels of instructions with labels to their final
+** destinations (e.g., choice L1; ... L1: jmp L2: becomes
+** choice L2)
+** * Jumps to other instructions that do jumps become those
+** instructions (e.g., jump to return becomes a return; jump
+** to commit becomes a commit)
+*/
+static void peephole (CompileState *compst) {
+ Instruction *code = compst->p->code;
+ int i;
+ for (i = 0; i < compst->ncode; i += sizei(&code[i])) {
+ redo:
+ switch (code[i].i.code) {
+ case IChoice: case ICall: case ICommit: case IPartialCommit:
+ case IBackCommit: case ITestChar: case ITestSet:
+ case ITestAny: { /* instructions with labels */
+ jumptothere(compst, i, finallabel(code, i)); /* optimize label */
+ break;
+ }
+ case IJmp: {
+ int ft = finaltarget(code, i);
+ switch (code[ft].i.code) { /* jumping to what? */
+ case IRet: case IFail: case IFailTwice:
+ case IEnd: { /* instructions with unconditional implicit jumps */
+ code[i] = code[ft]; /* jump becomes that instruction */
+ code[i + 1].i.code = IAny; /* 'no-op' for target position */
+ break;
+ }
+ case ICommit: case IPartialCommit:
+ case IBackCommit: { /* inst. with unconditional explicit jumps */
+ int fft = finallabel(code, ft);
+ code[i] = code[ft]; /* jump becomes that instruction... */
+ jumptothere(compst, i, fft); /* but must correct its offset */
+ goto redo; /* reoptimize its label */
+ }
+ default: {
+ jumptothere(compst, i, ft); /* optimize label */
+ break;
+ }
+ }
+ break;
+ }
+ default: break;
+ }
+ }
+ assert(code[i - 1].i.code == IEnd);
+}
+
+
+/*
+** Compile a pattern
+*/
+Instruction *compile (lua_State *L, Pattern *p) {
+ CompileState compst;
+ compst.p = p; compst.ncode = 0; compst.L = L;
+ realloccode(L, p, 2); /* minimum initial size */
+ codegen(&compst, p->tree, 0, NOINST, fullset);
+ addinstruction(&compst, IEnd, 0);
+ realloccode(L, p, compst.ncode); /* set final size */
+ peephole(&compst);
+ return p->code;
+}
+
+
+/* }====================================================== */
+
diff --git a/contrib/lua-lpeg/lpcode.h b/contrib/lua-lpeg/lpcode.h
new file mode 100644
index 0000000..34ee276
--- /dev/null
+++ b/contrib/lua-lpeg/lpcode.h
@@ -0,0 +1,40 @@
+/*
+** $Id: lpcode.h $
+*/
+
+#if !defined(lpcode_h)
+#define lpcode_h
+
+#include "lua.h"
+
+#include "lptypes.h"
+#include "lptree.h"
+#include "lpvm.h"
+
+int tocharset (TTree *tree, Charset *cs);
+int checkaux (TTree *tree, int pred);
+int fixedlen (TTree *tree);
+int hascaptures (TTree *tree);
+int lp_gc (lua_State *L);
+Instruction *compile (lua_State *L, Pattern *p);
+void realloccode (lua_State *L, Pattern *p, int nsize);
+int sizei (const Instruction *i);
+
+
+#define PEnullable 0
+#define PEnofail 1
+
+/*
+** nofail(t) implies that 't' cannot fail with any input
+*/
+#define nofail(t) checkaux(t, PEnofail)
+
+/*
+** (not nullable(t)) implies 't' cannot match without consuming
+** something
+*/
+#define nullable(t) checkaux(t, PEnullable)
+
+
+
+#endif
diff --git a/contrib/lua-lpeg/lpegre.lua b/contrib/lua-lpeg/lpegre.lua
new file mode 100644
index 0000000..3bb8af7
--- /dev/null
+++ b/contrib/lua-lpeg/lpegre.lua
@@ -0,0 +1,267 @@
+-- $Id: re.lua $
+
+-- imported functions and modules
+local tonumber, type, print, error = tonumber, type, print, error
+local setmetatable = setmetatable
+local m = require"lpeg"
+
+-- 'm' will be used to parse expressions, and 'mm' will be used to
+-- create expressions; that is, 're' runs on 'm', creating patterns
+-- on 'mm'
+local mm = m
+
+-- pattern's metatable
+local mt = getmetatable(mm.P(0))
+
+
+
+-- No more global accesses after this point
+local version = _VERSION
+if version == "Lua 5.2" then _ENV = nil end
+
+
+local any = m.P(1)
+
+
+-- Pre-defined names
+local Predef = { nl = m.P"\n" }
+
+
+local mem
+local fmem
+local gmem
+
+
+local function updatelocale ()
+ mm.locale(Predef)
+ Predef.a = Predef.alpha
+ Predef.c = Predef.cntrl
+ Predef.d = Predef.digit
+ Predef.g = Predef.graph
+ Predef.l = Predef.lower
+ Predef.p = Predef.punct
+ Predef.s = Predef.space
+ Predef.u = Predef.upper
+ Predef.w = Predef.alnum
+ Predef.x = Predef.xdigit
+ Predef.A = any - Predef.a
+ Predef.C = any - Predef.c
+ Predef.D = any - Predef.d
+ Predef.G = any - Predef.g
+ Predef.L = any - Predef.l
+ Predef.P = any - Predef.p
+ Predef.S = any - Predef.s
+ Predef.U = any - Predef.u
+ Predef.W = any - Predef.w
+ Predef.X = any - Predef.x
+ mem = {} -- restart memoization
+ fmem = {}
+ gmem = {}
+ local mt = {__mode = "v"}
+ setmetatable(mem, mt)
+ setmetatable(fmem, mt)
+ setmetatable(gmem, mt)
+end
+
+
+updatelocale()
+
+
+
+local I = m.P(function (s,i) print(i, s:sub(1, i-1)); return i end)
+
+
+local function patt_error (s, i)
+ local msg = (#s < i + 20) and s:sub(i)
+ or s:sub(i,i+20) .. "..."
+ msg = ("pattern error near '%s'"):format(msg)
+ error(msg, 2)
+end
+
+local function mult (p, n)
+ local np = mm.P(true)
+ while n >= 1 do
+ if n%2 >= 1 then np = np * p end
+ p = p * p
+ n = n/2
+ end
+ return np
+end
+
+local function equalcap (s, i, c)
+ if type(c) ~= "string" then return nil end
+ local e = #c + i
+ if s:sub(i, e - 1) == c then return e else return nil end
+end
+
+
+local S = (Predef.space + "--" * (any - Predef.nl)^0)^0
+
+local name = m.R("AZ", "az", "__") * m.R("AZ", "az", "__", "09")^0
+
+local arrow = S * "<-"
+
+local seq_follow = m.P"/" + ")" + "}" + ":}" + "~}" + "|}" + (name * arrow) + -1
+
+name = m.C(name)
+
+
+-- a defined name only have meaning in a given environment
+local Def = name * m.Carg(1)
+
+
+local function getdef (id, defs)
+ local c = defs and defs[id]
+ if not c then error("undefined name: " .. id) end
+ return c
+end
+
+-- match a name and return a group of its corresponding definition
+-- and 'f' (to be folded in 'Suffix')
+local function defwithfunc (f)
+ return m.Cg(Def / getdef * m.Cc(f))
+end
+
+
+local num = m.C(m.R"09"^1) * S / tonumber
+
+local String = "'" * m.C((any - "'")^0) * "'" +
+ '"' * m.C((any - '"')^0) * '"'
+
+
+local defined = "%" * Def / function (c,Defs)
+ local cat = Defs and Defs[c] or Predef[c]
+ if not cat then error ("name '" .. c .. "' undefined") end
+ return cat
+end
+
+local Range = m.Cs(any * (m.P"-"/"") * (any - "]")) / mm.R
+
+local item = (defined + Range + m.C(any)) / m.P
+
+local Class =
+ "["
+ * (m.C(m.P"^"^-1)) -- optional complement symbol
+ * m.Cf(item * (item - "]")^0, mt.__add) /
+ function (c, p) return c == "^" and any - p or p end
+ * "]"
+
+local function adddef (t, k, exp)
+ if t[k] then
+ error("'"..k.."' already defined as a rule")
+ else
+ t[k] = exp
+ end
+ return t
+end
+
+local function firstdef (n, r) return adddef({n}, n, r) end
+
+
+local function NT (n, b)
+ if not b then
+ error("rule '"..n.."' used outside a grammar")
+ else return mm.V(n)
+ end
+end
+
+
+local exp = m.P{ "Exp",
+ Exp = S * ( m.V"Grammar"
+ + m.Cf(m.V"Seq" * ("/" * S * m.V"Seq")^0, mt.__add) );
+ Seq = m.Cf(m.Cc(m.P"") * m.V"Prefix"^0 , mt.__mul)
+ * (#seq_follow + patt_error);
+ Prefix = "&" * S * m.V"Prefix" / mt.__len
+ + "!" * S * m.V"Prefix" / mt.__unm
+ + m.V"Suffix";
+ Suffix = m.Cf(m.V"Primary" * S *
+ ( ( m.P"+" * m.Cc(1, mt.__pow)
+ + m.P"*" * m.Cc(0, mt.__pow)
+ + m.P"?" * m.Cc(-1, mt.__pow)
+ + "^" * ( m.Cg(num * m.Cc(mult))
+ + m.Cg(m.C(m.S"+-" * m.R"09"^1) * m.Cc(mt.__pow))
+ )
+ + "->" * S * ( m.Cg((String + num) * m.Cc(mt.__div))
+ + m.P"{}" * m.Cc(nil, m.Ct)
+ + defwithfunc(mt.__div)
+ )
+ + "=>" * S * defwithfunc(m.Cmt)
+ + "~>" * S * defwithfunc(m.Cf)
+ ) * S
+ )^0, function (a,b,f) return f(a,b) end );
+ Primary = "(" * m.V"Exp" * ")"
+ + String / mm.P
+ + Class
+ + defined
+ + "{:" * (name * ":" + m.Cc(nil)) * m.V"Exp" * ":}" /
+ function (n, p) return mm.Cg(p, n) end
+ + "=" * name / function (n) return mm.Cmt(mm.Cb(n), equalcap) end
+ + m.P"{}" / mm.Cp
+ + "{~" * m.V"Exp" * "~}" / mm.Cs
+ + "{|" * m.V"Exp" * "|}" / mm.Ct
+ + "{" * m.V"Exp" * "}" / mm.C
+ + m.P"." * m.Cc(any)
+ + (name * -arrow + "<" * name * ">") * m.Cb("G") / NT;
+ Definition = name * arrow * m.V"Exp";
+ Grammar = m.Cg(m.Cc(true), "G") *
+ m.Cf(m.V"Definition" / firstdef * m.Cg(m.V"Definition")^0,
+ adddef) / mm.P
+}
+
+local pattern = S * m.Cg(m.Cc(false), "G") * exp / mm.P * (-any + patt_error)
+
+
+local function compile (p, defs)
+ if mm.type(p) == "pattern" then return p end -- already compiled
+ local cp = pattern:match(p, 1, defs)
+ if not cp then error("incorrect pattern", 3) end
+ return cp
+end
+
+local function match (s, p, i)
+ local cp = mem[p]
+ if not cp then
+ cp = compile(p)
+ mem[p] = cp
+ end
+ return cp:match(s, i or 1)
+end
+
+local function find (s, p, i)
+ local cp = fmem[p]
+ if not cp then
+ cp = compile(p) / 0
+ cp = mm.P{ mm.Cp() * cp * mm.Cp() + 1 * mm.V(1) }
+ fmem[p] = cp
+ end
+ local i, e = cp:match(s, i or 1)
+ if i then return i, e - 1
+ else return i
+ end
+end
+
+local function gsub (s, p, rep)
+ local g = gmem[p] or {} -- ensure gmem[p] is not collected while here
+ gmem[p] = g
+ local cp = g[rep]
+ if not cp then
+ cp = compile(p)
+ cp = mm.Cs((cp / rep + 1)^0)
+ g[rep] = cp
+ end
+ return cp:match(s)
+end
+
+
+-- exported names
+local re = {
+ compile = compile,
+ match = match,
+ find = find,
+ gsub = gsub,
+ updatelocale = updatelocale,
+}
+
+if version == "Lua 5.1" then _G.re = re end
+
+return re
diff --git a/contrib/lua-lpeg/lpprint.c b/contrib/lua-lpeg/lpprint.c
new file mode 100644
index 0000000..174d168
--- /dev/null
+++ b/contrib/lua-lpeg/lpprint.c
@@ -0,0 +1,244 @@
+/*
+** $Id: lpprint.c,v 1.9 2015/06/15 16:09:57 roberto Exp $
+** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+*/
+
+#include <ctype.h>
+#include <limits.h>
+#include <stdio.h>
+
+
+#include "lptypes.h"
+#include "lpprint.h"
+#include "lpcode.h"
+
+
+#if defined(LPEG_DEBUG)
+
+/*
+** {======================================================
+** Printing patterns (for debugging)
+** =======================================================
+*/
+
+
+void printcharset (const byte *st) {
+ int i;
+ printf("[");
+ for (i = 0; i <= UCHAR_MAX; i++) {
+ int first = i;
+ while (testchar(st, i) && i <= UCHAR_MAX) i++;
+ if (i - 1 == first) /* unary range? */
+ printf("(%02x)", first);
+ else if (i - 1 > first) /* non-empty range? */
+ printf("(%02x-%02x)", first, i - 1);
+ }
+ printf("]");
+}
+
+
+static void printcapkind (int kind) {
+ const char *const modes[] = {
+ "close", "position", "constant", "backref",
+ "argument", "simple", "table", "function",
+ "query", "string", "num", "substitution", "fold",
+ "runtime", "group"};
+ printf("%s", modes[kind]);
+}
+
+
+static void printjmp (const Instruction *op, const Instruction *p) {
+ printf("-> %d", (int)(p + (p + 1)->offset - op));
+}
+
+
+void printinst (const Instruction *op, const Instruction *p) {
+ const char *const names[] = {
+ "any", "char", "set",
+ "testany", "testchar", "testset",
+ "span", "behind",
+ "ret", "end",
+ "choice", "jmp", "call", "open_call",
+ "commit", "partial_commit", "back_commit", "failtwice", "fail", "giveup",
+ "fullcapture", "opencapture", "closecapture", "closeruntime"
+ };
+ printf("%02ld: %s ", (long)(p - op), names[p->i.code]);
+ switch ((Opcode)p->i.code) {
+ case IChar: {
+ printf("'%c'", p->i.aux);
+ break;
+ }
+ case ITestChar: {
+ printf("'%c'", p->i.aux); printjmp(op, p);
+ break;
+ }
+ case IFullCapture: {
+ printcapkind(getkind(p));
+ printf(" (size = %d) (idx = %d)", getoff(p), p->i.key);
+ break;
+ }
+ case IOpenCapture: {
+ printcapkind(getkind(p));
+ printf(" (idx = %d)", p->i.key);
+ break;
+ }
+ case ISet: {
+ printcharset((p+1)->buff);
+ break;
+ }
+ case ITestSet: {
+ printcharset((p+2)->buff); printjmp(op, p);
+ break;
+ }
+ case ISpan: {
+ printcharset((p+1)->buff);
+ break;
+ }
+ case IOpenCall: {
+ printf("-> %d", (p + 1)->offset);
+ break;
+ }
+ case IBehind: {
+ printf("%d", p->i.aux);
+ break;
+ }
+ case IJmp: case ICall: case ICommit: case IChoice:
+ case IPartialCommit: case IBackCommit: case ITestAny: {
+ printjmp(op, p);
+ break;
+ }
+ default: break;
+ }
+ printf("\n");
+}
+
+
+void printpatt (Instruction *p, int n) {
+ Instruction *op = p;
+ while (p < op + n) {
+ printinst(op, p);
+ p += sizei(p);
+ }
+}
+
+
+#if defined(LPEG_DEBUG)
+static void printcap (Capture *cap) {
+ printcapkind(cap->kind);
+ printf(" (idx: %d - size: %d) -> %p\n", cap->idx, cap->siz, cap->s);
+}
+
+
+void printcaplist (Capture *cap, Capture *limit) {
+ printf(">======\n");
+ for (; cap->s && (limit == NULL || cap < limit); cap++)
+ printcap(cap);
+ printf("=======\n");
+}
+#endif
+
+/* }====================================================== */
+
+
+/*
+** {======================================================
+** Printing trees (for debugging)
+** =======================================================
+*/
+
+static const char *tagnames[] = {
+ "char", "set", "any",
+ "true", "false",
+ "rep",
+ "seq", "choice",
+ "not", "and",
+ "call", "opencall", "rule", "grammar",
+ "behind",
+ "capture", "run-time"
+};
+
+
+void printtree (TTree *tree, int ident) {
+ int i;
+ for (i = 0; i < ident; i++) printf(" ");
+ printf("%s", tagnames[tree->tag]);
+ switch (tree->tag) {
+ case TChar: {
+ int c = tree->u.n;
+ if (isprint(c))
+ printf(" '%c'\n", c);
+ else
+ printf(" (%02X)\n", c);
+ break;
+ }
+ case TSet: {
+ printcharset(treebuffer(tree));
+ printf("\n");
+ break;
+ }
+ case TOpenCall: case TCall: {
+ printf(" key: %d\n", tree->key);
+ break;
+ }
+ case TBehind: {
+ printf(" %d\n", tree->u.n);
+ printtree(sib1(tree), ident + 2);
+ break;
+ }
+ case TCapture: {
+ printf(" cap: %d key: %d n: %d\n", tree->cap, tree->key, tree->u.n);
+ printtree(sib1(tree), ident + 2);
+ break;
+ }
+ case TRule: {
+ printf(" n: %d key: %d\n", tree->cap, tree->key);
+ printtree(sib1(tree), ident + 2);
+ break; /* do not print next rule as a sibling */
+ }
+ case TGrammar: {
+ TTree *rule = sib1(tree);
+ printf(" %d\n", tree->u.n); /* number of rules */
+ for (i = 0; i < tree->u.n; i++) {
+ printtree(rule, ident + 2);
+ rule = sib2(rule);
+ }
+ assert(rule->tag == TTrue); /* sentinel */
+ break;
+ }
+ default: {
+ int sibs = numsiblings[tree->tag];
+ printf("\n");
+ if (sibs >= 1) {
+ printtree(sib1(tree), ident + 2);
+ if (sibs >= 2)
+ printtree(sib2(tree), ident + 2);
+ }
+ break;
+ }
+ }
+}
+
+
+void printktable (lua_State *L, int idx) {
+ int n, i;
+ lua_getuservalue(L, idx);
+ if (lua_isnil(L, -1)) /* no ktable? */
+ return;
+ n = lua_rawlen(L, -1);
+ printf("[");
+ for (i = 1; i <= n; i++) {
+ printf("%d = ", i);
+ lua_rawgeti(L, -1, i);
+ if (lua_isstring(L, -1))
+ printf("%s ", lua_tostring(L, -1));
+ else
+ printf("%s ", lua_typename(L, lua_type(L, -1)));
+ lua_pop(L, 1);
+ }
+ printf("]\n");
+ /* leave ktable at the stack */
+}
+
+/* }====================================================== */
+
+#endif
diff --git a/contrib/lua-lpeg/lpprint.h b/contrib/lua-lpeg/lpprint.h
new file mode 100644
index 0000000..6329760
--- /dev/null
+++ b/contrib/lua-lpeg/lpprint.h
@@ -0,0 +1,36 @@
+/*
+** $Id: lpprint.h,v 1.2 2015/06/12 18:18:08 roberto Exp $
+*/
+
+
+#if !defined(lpprint_h)
+#define lpprint_h
+
+
+#include "lptree.h"
+#include "lpvm.h"
+
+
+#if defined(LPEG_DEBUG)
+
+void printpatt (Instruction *p, int n);
+void printtree (TTree *tree, int ident);
+void printktable (lua_State *L, int idx);
+void printcharset (const byte *st);
+void printcaplist (Capture *cap, Capture *limit);
+void printinst (const Instruction *op, const Instruction *p);
+
+#else
+
+#define printktable(L,idx) \
+ luaL_error(L, "function only implemented in debug mode")
+#define printtree(tree,i) \
+ luaL_error(L, "function only implemented in debug mode")
+#define printpatt(p,n) \
+ luaL_error(L, "function only implemented in debug mode")
+
+#endif
+
+
+#endif
+
diff --git a/contrib/lua-lpeg/lptree.c b/contrib/lua-lpeg/lptree.c
new file mode 100644
index 0000000..df24e3c
--- /dev/null
+++ b/contrib/lua-lpeg/lptree.c
@@ -0,0 +1,1319 @@
+/*
+** $Id: lptree.c,v 1.21 2015/09/28 17:01:25 roberto Exp $
+** Copyright 2013, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+*/
+
+#include <ctype.h>
+#include <limits.h>
+#include <string.h>
+#include <src/lua/lua_common.h>
+
+
+#include "lua.h"
+#include "lauxlib.h"
+
+#include "lptypes.h"
+#include "lpcap.h"
+#include "lpcode.h"
+#include "lpprint.h"
+#include "lptree.h"
+
+
+/* number of siblings for each tree */
+const byte numsiblings[] = {
+ 0, 0, 0, /* char, set, any */
+ 0, 0, /* true, false */
+ 1, /* rep */
+ 2, 2, /* seq, choice */
+ 1, 1, /* not, and */
+ 0, 0, 2, 1, /* call, opencall, rule, grammar */
+ 1, /* behind */
+ 1, 1 /* capture, runtime capture */
+};
+
+
+static TTree *newgrammar (lua_State *L, int arg);
+
+
+/*
+** returns a reasonable name for value at index 'idx' on the stack
+*/
+static const char *val2str (lua_State *L, int idx) {
+ const char *k = lua_tostring(L, idx);
+ if (k != NULL)
+ return lua_pushfstring(L, "%s", k);
+ else
+ return lua_pushfstring(L, "(a %s)", luaL_typename(L, idx));
+}
+
+
+/*
+** Fix a TOpenCall into a TCall node, using table 'postable' to
+** translate a key to its rule address in the tree. Raises an
+** error if key does not exist.
+*/
+static void fixonecall (lua_State *L, int postable, TTree *g, TTree *t) {
+ int n;
+ lua_rawgeti(L, -1, t->key); /* get rule's name */
+ lua_gettable(L, postable); /* query name in position table */
+ n = lua_tonumber(L, -1); /* get (absolute) position */
+ lua_pop(L, 1); /* remove position */
+ if (n == 0) { /* no position? */
+ lua_rawgeti(L, -1, t->key); /* get rule's name again */
+ luaL_error(L, "rule '%s' undefined in given grammar", val2str(L, -1));
+ }
+ t->tag = TCall;
+ t->u.ps = n - (t - g); /* position relative to node */
+ assert(sib2(t)->tag == TRule);
+ sib2(t)->key = t->key;
+}
+
+
+/*
+** Transform left associative constructions into right
+** associative ones, for sequence and choice; that is:
+** (t11 + t12) + t2 => t11 + (t12 + t2)
+** (t11 * t12) * t2 => t11 * (t12 * t2)
+** (that is, Op (Op t11 t12) t2 => Op t11 (Op t12 t2))
+*/
+static void correctassociativity (TTree *tree) {
+ TTree *t1 = sib1(tree);
+ assert(tree->tag == TChoice || tree->tag == TSeq);
+ while (t1->tag == tree->tag) {
+ int n1size = tree->u.ps - 1; /* t1 == Op t11 t12 */
+ int n11size = t1->u.ps - 1;
+ int n12size = n1size - n11size - 1;
+ memmove(sib1(tree), sib1(t1), n11size * sizeof(TTree)); /* move t11 */
+ tree->u.ps = n11size + 1;
+ sib2(tree)->tag = tree->tag;
+ sib2(tree)->u.ps = n12size + 1;
+ }
+}
+
+
+/*
+** Make final adjustments in a tree. Fix open calls in tree 't',
+** making them refer to their respective rules or raising appropriate
+** errors (if not inside a grammar). Correct associativity of associative
+** constructions (making them right associative). Assume that tree's
+** ktable is at the top of the stack (for error messages).
+*/
+static void finalfix (lua_State *L, int postable, TTree *g, TTree *t) {
+ tailcall:
+ switch (t->tag) {
+ case TGrammar: /* subgrammars were already fixed */
+ return;
+ case TOpenCall: {
+ if (g != NULL) /* inside a grammar? */
+ fixonecall(L, postable, g, t);
+ else { /* open call outside grammar */
+ lua_rawgeti(L, -1, t->key);
+ luaL_error(L, "rule '%s' used outside a grammar", val2str(L, -1));
+ }
+ break;
+ }
+ case TSeq: case TChoice:
+ correctassociativity(t);
+ break;
+ }
+ switch (numsiblings[t->tag]) {
+ case 1: /* finalfix(L, postable, g, sib1(t)); */
+ t = sib1(t); goto tailcall;
+ case 2:
+ finalfix(L, postable, g, sib1(t));
+ t = sib2(t); goto tailcall; /* finalfix(L, postable, g, sib2(t)); */
+ default: assert(numsiblings[t->tag] == 0); break;
+ }
+}
+
+
+
+/*
+** {===================================================================
+** KTable manipulation
+**
+** - The ktable of a pattern 'p' can be shared by other patterns that
+** contain 'p' and no other constants. Because of this sharing, we
+** should not add elements to a 'ktable' unless it was freshly created
+** for the new pattern.
+**
+** - The maximum index in a ktable is USHRT_MAX, because trees and
+** patterns use unsigned shorts to store those indices.
+** ====================================================================
+*/
+
+/*
+** Create a new 'ktable' to the pattern at the top of the stack.
+*/
+static void newktable (lua_State *L, int n) {
+ lua_createtable(L, n, 0); /* create a fresh table */
+ lua_setuservalue(L, -2); /* set it as 'ktable' for pattern */
+}
+
+
+/*
+** Add element 'idx' to 'ktable' of pattern at the top of the stack;
+** Return index of new element.
+** If new element is nil, does not add it to table (as it would be
+** useless) and returns 0, as ktable[0] is always nil.
+*/
+static int addtoktable (lua_State *L, int idx) {
+ if (lua_isnil(L, idx)) /* nil value? */
+ return 0;
+ else {
+ int n;
+ lua_getuservalue(L, -1); /* get ktable from pattern */
+ n = lua_rawlen(L, -1);
+ if (n >= USHRT_MAX)
+ luaL_error(L, "too many Lua values in pattern");
+ lua_pushvalue(L, idx); /* element to be added */
+ lua_rawseti(L, -2, ++n);
+ lua_pop(L, 1); /* remove 'ktable' */
+ return n;
+ }
+}
+
+
+/*
+** Return the number of elements in the ktable at 'idx'.
+** In Lua 5.2/5.3, default "environment" for patterns is nil, not
+** a table. Treat it as an empty table. In Lua 5.1, assumes that
+** the environment has no numeric indices (len == 0)
+*/
+static int ktablelen (lua_State *L, int idx) {
+ if (!lua_istable(L, idx)) return 0;
+ else return lua_rawlen(L, idx);
+}
+
+
+/*
+** Concatenate the contents of table 'idx1' into table 'idx2'.
+** (Assume that both indices are negative.)
+** Return the original length of table 'idx2' (or 0, if no
+** element was added, as there is no need to correct any index).
+*/
+static int concattable (lua_State *L, int idx1, int idx2) {
+ int i;
+ int n1 = ktablelen(L, idx1);
+ int n2 = ktablelen(L, idx2);
+ if (n1 + n2 > USHRT_MAX)
+ luaL_error(L, "too many Lua values in pattern");
+ if (n1 == 0) return 0; /* nothing to correct */
+ for (i = 1; i <= n1; i++) {
+ lua_rawgeti(L, idx1, i);
+ lua_rawseti(L, idx2 - 1, n2 + i); /* correct 'idx2' */
+ }
+ return n2;
+}
+
+
+/*
+** When joining 'ktables', constants from one of the subpatterns must
+** be renumbered; 'correctkeys' corrects their indices (adding 'n'
+** to each of them)
+*/
+static void correctkeys (TTree *tree, int n) {
+ if (n == 0) return; /* no correction? */
+ tailcall:
+ switch (tree->tag) {
+ case TOpenCall: case TCall: case TRunTime: case TRule: {
+ if (tree->key > 0)
+ tree->key += n;
+ break;
+ }
+ case TCapture: {
+ if (tree->key > 0 && tree->cap != Carg && tree->cap != Cnum)
+ tree->key += n;
+ break;
+ }
+ default: break;
+ }
+ switch (numsiblings[tree->tag]) {
+ case 1: /* correctkeys(sib1(tree), n); */
+ tree = sib1(tree); goto tailcall;
+ case 2:
+ correctkeys(sib1(tree), n);
+ tree = sib2(tree); goto tailcall; /* correctkeys(sib2(tree), n); */
+ default: assert(numsiblings[tree->tag] == 0); break;
+ }
+}
+
+
+/*
+** Join the ktables from p1 and p2 the ktable for the new pattern at the
+** top of the stack, reusing them when possible.
+*/
+static void joinktables (lua_State *L, int p1, TTree *t2, int p2) {
+ int n1, n2;
+ lua_getuservalue(L, p1); /* get ktables */
+ lua_getuservalue(L, p2);
+ n1 = ktablelen(L, -2);
+ n2 = ktablelen(L, -1);
+ if (n1 == 0 && n2 == 0) /* are both tables empty? */
+ lua_pop(L, 2); /* nothing to be done; pop tables */
+ else if (n2 == 0 || lp_equal(L, -2, -1)) { /* 2nd table empty or equal? */
+ lua_pop(L, 1); /* pop 2nd table */
+ lua_setuservalue(L, -2); /* set 1st ktable into new pattern */
+ }
+ else if (n1 == 0) { /* first table is empty? */
+ lua_setuservalue(L, -3); /* set 2nd table into new pattern */
+ lua_pop(L, 1); /* pop 1st table */
+ }
+ else {
+ lua_createtable(L, n1 + n2, 0); /* create ktable for new pattern */
+ /* stack: new p; ktable p1; ktable p2; new ktable */
+ concattable(L, -3, -1); /* from p1 into new ktable */
+ concattable(L, -2, -1); /* from p2 into new ktable */
+ lua_setuservalue(L, -4); /* new ktable becomes 'p' environment */
+ lua_pop(L, 2); /* pop other ktables */
+ correctkeys(t2, n1); /* correction for indices from p2 */
+ }
+}
+
+
+/*
+** copy 'ktable' of element 'idx' to new tree (on top of stack)
+*/
+static void copyktable (lua_State *L, int idx) {
+ lua_getuservalue(L, idx);
+ lua_setuservalue(L, -2);
+}
+
+
+/*
+** merge 'ktable' from 'stree' at stack index 'idx' into 'ktable'
+** from tree at the top of the stack, and correct corresponding
+** tree.
+*/
+static void mergektable (lua_State *L, int idx, TTree *stree) {
+ int n;
+ lua_getuservalue(L, -1); /* get ktables */
+ lua_getuservalue(L, idx);
+ n = concattable(L, -1, -2);
+ lua_pop(L, 2); /* remove both ktables */
+ correctkeys(stree, n);
+}
+
+
+/*
+** Create a new 'ktable' to the pattern at the top of the stack, adding
+** all elements from pattern 'p' (if not 0) plus element 'idx' to it.
+** Return index of new element.
+*/
+static int addtonewktable (lua_State *L, int p, int idx) {
+ newktable(L, 1);
+ if (p)
+ mergektable(L, p, NULL);
+ return addtoktable(L, idx);
+}
+
+/* }====================================================== */
+
+
+/*
+** {======================================================
+** Tree generation
+** =======================================================
+*/
+
+/*
+** In 5.2, could use 'luaL_testudata'...
+*/
+static int testpattern (lua_State *L, int idx) {
+ if (lua_touserdata(L, idx)) { /* value is a userdata? */
+ if (lua_getmetatable(L, idx)) { /* does it have a metatable? */
+ luaL_getmetatable(L, PATTERN_T);
+ if (lua_rawequal(L, -1, -2)) { /* does it have the correct mt? */
+ lua_pop(L, 2); /* remove both metatables */
+ return 1;
+ }
+ }
+ }
+ return 0;
+}
+
+
+static Pattern *getpattern (lua_State *L, int idx) {
+ return (Pattern *)luaL_checkudata(L, idx, PATTERN_T);
+}
+
+
+static int getsize (lua_State *L, int idx) {
+ return (lua_rawlen(L, idx) - sizeof(Pattern)) / sizeof(TTree) + 1;
+}
+
+
+static TTree *gettree (lua_State *L, int idx, int *len) {
+ Pattern *p = getpattern(L, idx);
+ if (len)
+ *len = getsize(L, idx);
+ return p->tree;
+}
+
+
+/*
+** create a pattern. Set its uservalue (the 'ktable') equal to its
+** metatable. (It could be any empty sequence; the metatable is at
+** hand here, so we use it.)
+*/
+static TTree *newtree (lua_State *L, int len) {
+ size_t size = (len - 1) * sizeof(TTree) + sizeof(Pattern);
+ Pattern *p = (Pattern *)lua_newuserdata(L, size);
+ memset(p, 0, size);
+ luaL_getmetatable(L, PATTERN_T);
+ lua_pushvalue(L, -1);
+ lua_setuservalue(L, -3);
+ lua_setmetatable(L, -2);
+ p->code = NULL; p->codesize = 0;
+ return p->tree;
+}
+
+
+static TTree *newleaf (lua_State *L, int tag) {
+ TTree *tree = newtree(L, 1);
+ tree->tag = tag;
+ return tree;
+}
+
+
+static TTree *newcharset (lua_State *L) {
+ TTree *tree = newtree(L, bytes2slots(CHARSETSIZE) + 1);
+ tree->tag = TSet;
+ loopset(i, treebuffer(tree)[i] = 0);
+ return tree;
+}
+
+
+/*
+** add to tree a sequence where first sibling is 'sib' (with size
+** 'sibsize'); returns position for second sibling
+*/
+static TTree *seqaux (TTree *tree, TTree *sib, int sibsize) {
+ tree->tag = TSeq; tree->u.ps = sibsize + 1;
+ memcpy(sib1(tree), sib, sibsize * sizeof(TTree));
+ return sib2(tree);
+}
+
+
+/*
+** Build a sequence of 'n' nodes, each with tag 'tag' and 'u.n' got
+** from the array 's' (or 0 if array is NULL). (TSeq is binary, so it
+** must build a sequence of sequence of sequence...)
+*/
+static void fillseq (TTree *tree, int tag, int n, const char *s) {
+ int i;
+ for (i = 0; i < n - 1; i++) { /* initial n-1 copies of Seq tag; Seq ... */
+ tree->tag = TSeq; tree->u.ps = 2;
+ sib1(tree)->tag = tag;
+ sib1(tree)->u.n = s ? (byte)s[i] : 0;
+ tree = sib2(tree);
+ }
+ tree->tag = tag; /* last one does not need TSeq */
+ tree->u.n = s ? (byte)s[i] : 0;
+}
+
+
+/*
+** Numbers as patterns:
+** 0 == true (always match); n == TAny repeated 'n' times;
+** -n == not (TAny repeated 'n' times)
+*/
+static TTree *numtree (lua_State *L, int n) {
+ if (n == 0)
+ return newleaf(L, TTrue);
+ else {
+ TTree *tree, *nd;
+ if (n > 0)
+ tree = nd = newtree(L, 2 * n - 1);
+ else { /* negative: code it as !(-n) */
+ n = -n;
+ tree = newtree(L, 2 * n);
+ tree->tag = TNot;
+ nd = sib1(tree);
+ }
+ fillseq(nd, TAny, n, NULL); /* sequence of 'n' any's */
+ return tree;
+ }
+}
+
+
+/*
+** Convert value at index 'idx' to a pattern
+*/
+static TTree *getpatt (lua_State *L, int idx, int *len) {
+ TTree *tree;
+ switch (lua_type(L, idx)) {
+ case LUA_TSTRING: {
+ size_t slen;
+ const char *s = lua_tolstring(L, idx, &slen); /* get string */
+ if (slen == 0) /* empty? */
+ tree = newleaf(L, TTrue); /* always match */
+ else {
+ tree = newtree(L, 2 * (slen - 1) + 1);
+ fillseq(tree, TChar, slen, s); /* sequence of 'slen' chars */
+ }
+ break;
+ }
+ case LUA_TNUMBER: {
+ int n = lua_tointeger(L, idx);
+ tree = numtree(L, n);
+ break;
+ }
+ case LUA_TBOOLEAN: {
+ tree = (lua_toboolean(L, idx) ? newleaf(L, TTrue) : newleaf(L, TFalse));
+ break;
+ }
+ case LUA_TTABLE: {
+ tree = newgrammar(L, idx);
+ break;
+ }
+ case LUA_TFUNCTION: {
+ tree = newtree(L, 2);
+ tree->tag = TRunTime;
+ tree->key = addtonewktable(L, 0, idx);
+ sib1(tree)->tag = TTrue;
+ break;
+ }
+ default: {
+ return gettree(L, idx, len);
+ }
+ }
+ lua_replace(L, idx); /* put new tree into 'idx' slot */
+ if (len)
+ *len = getsize(L, idx);
+ return tree;
+}
+
+
+/*
+** create a new tree, with a new root and one sibling.
+** Sibling must be on the Lua stack, at index 1.
+*/
+static TTree *newroot1sib (lua_State *L, int tag) {
+ int s1;
+ TTree *tree1 = getpatt(L, 1, &s1);
+ TTree *tree = newtree(L, 1 + s1); /* create new tree */
+ tree->tag = tag;
+ memcpy(sib1(tree), tree1, s1 * sizeof(TTree));
+ copyktable(L, 1);
+ return tree;
+}
+
+
+/*
+** create a new tree, with a new root and 2 siblings.
+** Siblings must be on the Lua stack, first one at index 1.
+*/
+static TTree *newroot2sib (lua_State *L, int tag) {
+ int s1, s2;
+ TTree *tree1 = getpatt(L, 1, &s1);
+ TTree *tree2 = getpatt(L, 2, &s2);
+ TTree *tree = newtree(L, 1 + s1 + s2); /* create new tree */
+ tree->tag = tag;
+ tree->u.ps = 1 + s1;
+ memcpy(sib1(tree), tree1, s1 * sizeof(TTree));
+ memcpy(sib2(tree), tree2, s2 * sizeof(TTree));
+ joinktables(L, 1, sib2(tree), 2);
+ return tree;
+}
+
+
+static int lp_P (lua_State *L) {
+ luaL_checkany(L, 1);
+ getpatt(L, 1, NULL);
+ lua_settop(L, 1);
+ return 1;
+}
+
+
+/*
+** sequence operator; optimizations:
+** false x => false, x true => x, true x => x
+** (cannot do x . false => false because x may have runtime captures)
+*/
+static int lp_seq (lua_State *L) {
+ TTree *tree1 = getpatt(L, 1, NULL);
+ TTree *tree2 = getpatt(L, 2, NULL);
+ if (tree1->tag == TFalse || tree2->tag == TTrue)
+ lua_pushvalue(L, 1); /* false . x == false, x . true = x */
+ else if (tree1->tag == TTrue)
+ lua_pushvalue(L, 2); /* true . x = x */
+ else
+ newroot2sib(L, TSeq);
+ return 1;
+}
+
+
+/*
+** choice operator; optimizations:
+** charset / charset => charset
+** true / x => true, x / false => x, false / x => x
+** (x / true is not equivalent to true)
+*/
+static int lp_choice (lua_State *L) {
+ Charset st1, st2;
+ TTree *t1 = getpatt(L, 1, NULL);
+ TTree *t2 = getpatt(L, 2, NULL);
+ if (tocharset(t1, &st1) && tocharset(t2, &st2)) {
+ TTree *t = newcharset(L);
+ loopset(i, treebuffer(t)[i] = st1.cs[i] | st2.cs[i]);
+ }
+ else if (nofail(t1) || t2->tag == TFalse)
+ lua_pushvalue(L, 1); /* true / x => true, x / false => x */
+ else if (t1->tag == TFalse)
+ lua_pushvalue(L, 2); /* false / x => x */
+ else
+ newroot2sib(L, TChoice);
+ return 1;
+}
+
+
+/*
+** p^n
+*/
+static int lp_star (lua_State *L) {
+ int size1;
+ int n = (int)luaL_checkinteger(L, 2);
+ TTree *tree1 = getpatt(L, 1, &size1);
+ if (n >= 0) { /* seq tree1 (seq tree1 ... (seq tree1 (rep tree1))) */
+ TTree *tree = newtree(L, (n + 1) * (size1 + 1));
+ if (nullable(tree1))
+ luaL_error(L, "loop body may accept empty string");
+ while (n--) /* repeat 'n' times */
+ tree = seqaux(tree, tree1, size1);
+ tree->tag = TRep;
+ memcpy(sib1(tree), tree1, size1 * sizeof(TTree));
+ }
+ else { /* choice (seq tree1 ... choice tree1 true ...) true */
+ TTree *tree;
+ n = -n;
+ /* size = (choice + seq + tree1 + true) * n, but the last has no seq */
+ tree = newtree(L, n * (size1 + 3) - 1);
+ for (; n > 1; n--) { /* repeat (n - 1) times */
+ tree->tag = TChoice; tree->u.ps = n * (size1 + 3) - 2;
+ sib2(tree)->tag = TTrue;
+ tree = sib1(tree);
+ tree = seqaux(tree, tree1, size1);
+ }
+ tree->tag = TChoice; tree->u.ps = size1 + 1;
+ sib2(tree)->tag = TTrue;
+ memcpy(sib1(tree), tree1, size1 * sizeof(TTree));
+ }
+ copyktable(L, 1);
+ return 1;
+}
+
+
+/*
+** #p == &p
+*/
+static int lp_and (lua_State *L) {
+ newroot1sib(L, TAnd);
+ return 1;
+}
+
+
+/*
+** -p == !p
+*/
+static int lp_not (lua_State *L) {
+ newroot1sib(L, TNot);
+ return 1;
+}
+
+
+/*
+** [t1 - t2] == Seq (Not t2) t1
+** If t1 and t2 are charsets, make their difference.
+*/
+static int lp_sub (lua_State *L) {
+ Charset st1, st2;
+ int s1, s2;
+ TTree *t1 = getpatt(L, 1, &s1);
+ TTree *t2 = getpatt(L, 2, &s2);
+ if (tocharset(t1, &st1) && tocharset(t2, &st2)) {
+ TTree *t = newcharset(L);
+ loopset(i, treebuffer(t)[i] = st1.cs[i] & ~st2.cs[i]);
+ }
+ else {
+ TTree *tree = newtree(L, 2 + s1 + s2);
+ tree->tag = TSeq; /* sequence of... */
+ tree->u.ps = 2 + s2;
+ sib1(tree)->tag = TNot; /* ...not... */
+ memcpy(sib1(sib1(tree)), t2, s2 * sizeof(TTree)); /* ...t2 */
+ memcpy(sib2(tree), t1, s1 * sizeof(TTree)); /* ... and t1 */
+ joinktables(L, 1, sib1(tree), 2);
+ }
+ return 1;
+}
+
+
+static int lp_set (lua_State *L) {
+ size_t l;
+ const char *s = luaL_checklstring(L, 1, &l);
+ TTree *tree = newcharset(L);
+ while (l--) {
+ setchar(treebuffer(tree), (byte)(*s));
+ s++;
+ }
+ return 1;
+}
+
+
+static int lp_range (lua_State *L) {
+ int arg;
+ int top = lua_gettop(L);
+ TTree *tree = newcharset(L);
+ for (arg = 1; arg <= top; arg++) {
+ int c;
+ size_t l;
+ const char *r = luaL_checklstring(L, arg, &l);
+ luaL_argcheck(L, l == 2, arg, "range must have two characters");
+ for (c = (byte)r[0]; c <= (byte)r[1]; c++)
+ setchar(treebuffer(tree), c);
+ }
+ return 1;
+}
+
+
+/*
+** Look-behind predicate
+*/
+static int lp_behind (lua_State *L) {
+ TTree *tree;
+ TTree *tree1 = getpatt(L, 1, NULL);
+ int n = fixedlen(tree1);
+ luaL_argcheck(L, n >= 0, 1, "pattern may not have fixed length");
+ luaL_argcheck(L, !hascaptures(tree1), 1, "pattern have captures");
+ luaL_argcheck(L, n <= MAXBEHIND, 1, "pattern too long to look behind");
+ tree = newroot1sib(L, TBehind);
+ tree->u.n = n;
+ return 1;
+}
+
+
+/*
+** Create a non-terminal
+*/
+static int lp_V (lua_State *L) {
+ TTree *tree = newleaf(L, TOpenCall);
+ luaL_argcheck(L, !lua_isnoneornil(L, 1), 1, "non-nil value expected");
+ tree->key = addtonewktable(L, 0, 1);
+ return 1;
+}
+
+
+/*
+** Create a tree for a non-empty capture, with a body and
+** optionally with an associated Lua value (at index 'labelidx' in the
+** stack)
+*/
+static int capture_aux (lua_State *L, int cap, int labelidx) {
+ TTree *tree = newroot1sib(L, TCapture);
+ tree->cap = cap;
+ tree->key = (labelidx == 0) ? 0 : addtonewktable(L, 1, labelidx);
+ return 1;
+}
+
+
+/*
+** Fill a tree with an empty capture, using an empty (TTrue) sibling.
+*/
+static TTree *auxemptycap (TTree *tree, int cap) {
+ tree->tag = TCapture;
+ tree->cap = cap;
+ sib1(tree)->tag = TTrue;
+ return tree;
+}
+
+
+/*
+** Create a tree for an empty capture
+*/
+static TTree *newemptycap (lua_State *L, int cap) {
+ return auxemptycap(newtree(L, 2), cap);
+}
+
+
+/*
+** Create a tree for an empty capture with an associated Lua value
+*/
+static TTree *newemptycapkey (lua_State *L, int cap, int idx) {
+ TTree *tree = auxemptycap(newtree(L, 2), cap);
+ tree->key = addtonewktable(L, 0, idx);
+ return tree;
+}
+
+
+/*
+** Captures with syntax p / v
+** (function capture, query capture, string capture, or number capture)
+*/
+static int lp_divcapture (lua_State *L) {
+ switch (lua_type(L, 2)) {
+ case LUA_TFUNCTION: return capture_aux(L, Cfunction, 2);
+ case LUA_TTABLE: return capture_aux(L, Cquery, 2);
+ case LUA_TSTRING: return capture_aux(L, Cstring, 2);
+ case LUA_TNUMBER: {
+ int n = lua_tointeger(L, 2);
+ TTree *tree = newroot1sib(L, TCapture);
+ luaL_argcheck(L, 0 <= n && n <= SHRT_MAX, 1, "invalid number");
+ tree->cap = Cnum;
+ tree->key = n;
+ return 1;
+ }
+ default: return luaL_argerror(L, 2, "invalid replacement value");
+ }
+}
+
+
+static int lp_substcapture (lua_State *L) {
+ return capture_aux(L, Csubst, 0);
+}
+
+
+static int lp_tablecapture (lua_State *L) {
+ return capture_aux(L, Ctable, 0);
+}
+
+
+static int lp_groupcapture (lua_State *L) {
+ if (lua_isnoneornil(L, 2))
+ return capture_aux(L, Cgroup, 0);
+ else
+ return capture_aux(L, Cgroup, 2);
+}
+
+
+static int lp_foldcapture (lua_State *L) {
+ luaL_checktype(L, 2, LUA_TFUNCTION);
+ return capture_aux(L, Cfold, 2);
+}
+
+
+static int lp_simplecapture (lua_State *L) {
+ return capture_aux(L, Csimple, 0);
+}
+
+
+static int lp_poscapture (lua_State *L) {
+ newemptycap(L, Cposition);
+ return 1;
+}
+
+
+static int lp_argcapture (lua_State *L) {
+ int n = (int)luaL_checkinteger(L, 1);
+ TTree *tree = newemptycap(L, Carg);
+ tree->key = n;
+ luaL_argcheck(L, 0 < n && n <= SHRT_MAX, 1, "invalid argument index");
+ return 1;
+}
+
+
+static int lp_backref (lua_State *L) {
+ luaL_checkany(L, 1);
+ newemptycapkey(L, Cbackref, 1);
+ return 1;
+}
+
+
+/*
+** Constant capture
+*/
+static int lp_constcapture (lua_State *L) {
+ int i;
+ int n = lua_gettop(L); /* number of values */
+ if (n == 0) /* no values? */
+ newleaf(L, TTrue); /* no capture */
+ else if (n == 1)
+ newemptycapkey(L, Cconst, 1); /* single constant capture */
+ else { /* create a group capture with all values */
+ TTree *tree = newtree(L, 1 + 3 * (n - 1) + 2);
+ newktable(L, n); /* create a 'ktable' for new tree */
+ tree->tag = TCapture;
+ tree->cap = Cgroup;
+ tree->key = 0;
+ tree = sib1(tree);
+ for (i = 1; i <= n - 1; i++) {
+ tree->tag = TSeq;
+ tree->u.ps = 3; /* skip TCapture and its sibling */
+ auxemptycap(sib1(tree), Cconst);
+ sib1(tree)->key = addtoktable(L, i);
+ tree = sib2(tree);
+ }
+ auxemptycap(tree, Cconst);
+ tree->key = addtoktable(L, i);
+ }
+ return 1;
+}
+
+
+static int lp_matchtime (lua_State *L) {
+ TTree *tree;
+ luaL_checktype(L, 2, LUA_TFUNCTION);
+ tree = newroot1sib(L, TRunTime);
+ tree->key = addtonewktable(L, 1, 2);
+ return 1;
+}
+
+/* }====================================================== */
+
+
+/*
+** {======================================================
+** Grammar - Tree generation
+** =======================================================
+*/
+
+/*
+** push on the stack the index and the pattern for the
+** initial rule of grammar at index 'arg' in the stack;
+** also add that index into position table.
+*/
+static void getfirstrule (lua_State *L, int arg, int postab) {
+ lua_rawgeti(L, arg, 1); /* access first element */
+ if (lua_isstring(L, -1)) { /* is it the name of initial rule? */
+ lua_pushvalue(L, -1); /* duplicate it to use as key */
+ lua_gettable(L, arg); /* get associated rule */
+ }
+ else {
+ lua_pushinteger(L, 1); /* key for initial rule */
+ lua_insert(L, -2); /* put it before rule */
+ }
+ if (!testpattern(L, -1)) { /* initial rule not a pattern? */
+ if (lua_isnil(L, -1))
+ luaL_error(L, "grammar has no initial rule");
+ else
+ luaL_error(L, "initial rule '%s' is not a pattern", lua_tostring(L, -2));
+ }
+ lua_pushvalue(L, -2); /* push key */
+ lua_pushinteger(L, 1); /* push rule position (after TGrammar) */
+ lua_settable(L, postab); /* insert pair at position table */
+}
+
+/*
+** traverse grammar at index 'arg', pushing all its keys and patterns
+** into the stack. Create a new table (before all pairs key-pattern) to
+** collect all keys and their associated positions in the final tree
+** (the "position table").
+** Return the number of rules and (in 'totalsize') the total size
+** for the new tree.
+*/
+static int collectrules (lua_State *L, int arg, int *totalsize) {
+ int n = 1; /* to count number of rules */
+ int postab = lua_gettop(L) + 1; /* index of position table */
+ int size; /* accumulator for total size */
+ lua_newtable(L); /* create position table */
+ getfirstrule(L, arg, postab);
+ size = 2 + getsize(L, postab + 2); /* TGrammar + TRule + rule */
+ lua_pushnil(L); /* prepare to traverse grammar table */
+ while (lua_next(L, arg) != 0) {
+ if (lua_tonumber(L, -2) == 1 ||
+ lp_equal(L, -2, postab + 1)) { /* initial rule? */
+ lua_pop(L, 1); /* remove value (keep key for lua_next) */
+ continue;
+ }
+ if (!testpattern(L, -1)) /* value is not a pattern? */
+ luaL_error(L, "rule '%s' is not a pattern", val2str(L, -2));
+ luaL_checkstack(L, LUA_MINSTACK, "grammar has too many rules");
+ lua_pushvalue(L, -2); /* push key (to insert into position table) */
+ lua_pushinteger(L, size);
+ lua_settable(L, postab);
+ size += 1 + getsize(L, -1); /* update size */
+ lua_pushvalue(L, -2); /* push key (for next lua_next) */
+ n++;
+ }
+ *totalsize = size + 1; /* TTrue to finish list of rules */
+ return n;
+}
+
+
+static void buildgrammar (lua_State *L, TTree *grammar, int frule, int n) {
+ int i;
+ TTree *nd = sib1(grammar); /* auxiliary pointer to traverse the tree */
+ for (i = 0; i < n; i++) { /* add each rule into new tree */
+ int ridx = frule + 2*i + 1; /* index of i-th rule */
+ int rulesize;
+ TTree *rn = gettree(L, ridx, &rulesize);
+ nd->tag = TRule;
+ nd->key = 0;
+ nd->cap = i; /* rule number */
+ nd->u.ps = rulesize + 1; /* point to next rule */
+ memcpy(sib1(nd), rn, rulesize * sizeof(TTree)); /* copy rule */
+ mergektable(L, ridx, sib1(nd)); /* merge its ktable into new one */
+ nd = sib2(nd); /* move to next rule */
+ }
+ nd->tag = TTrue; /* finish list of rules */
+}
+
+
+/*
+** Check whether a tree has potential infinite loops
+*/
+static int checkloops (TTree *tree) {
+ tailcall:
+ if (tree->tag == TRep && nullable(sib1(tree)))
+ return 1;
+ else if (tree->tag == TGrammar)
+ return 0; /* sub-grammars already checked */
+ else {
+ switch (numsiblings[tree->tag]) {
+ case 1: /* return checkloops(sib1(tree)); */
+ tree = sib1(tree); goto tailcall;
+ case 2:
+ if (checkloops(sib1(tree))) return 1;
+ /* else return checkloops(sib2(tree)); */
+ tree = sib2(tree); goto tailcall;
+ default: assert(numsiblings[tree->tag] == 0); return 0;
+ }
+ }
+}
+
+
+static int verifyerror (lua_State *L, int *passed, int npassed) {
+ int i, j;
+ for (i = npassed - 1; i >= 0; i--) { /* search for a repetition */
+ for (j = i - 1; j >= 0; j--) {
+ if (passed[i] == passed[j]) {
+ lua_rawgeti(L, -1, passed[i]); /* get rule's key */
+ return luaL_error(L, "rule '%s' may be left recursive", val2str(L, -1));
+ }
+ }
+ }
+ return luaL_error(L, "too many left calls in grammar");
+}
+
+
+/*
+** Check whether a rule can be left recursive; raise an error in that
+** case; otherwise return 1 iff pattern is nullable.
+** The return value is used to check sequences, where the second pattern
+** is only relevant if the first is nullable.
+** Parameter 'nb' works as an accumulator, to allow tail calls in
+** choices. ('nb' true makes function returns true.)
+** Assume ktable at the top of the stack.
+*/
+static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed,
+ int nb) {
+ tailcall:
+ switch (tree->tag) {
+ case TChar: case TSet: case TAny:
+ case TFalse:
+ return nb; /* cannot pass from here */
+ case TTrue:
+ case TBehind: /* look-behind cannot have calls */
+ return 1;
+ case TNot: case TAnd: case TRep:
+ /* return verifyrule(L, sib1(tree), passed, npassed, 1); */
+ tree = sib1(tree); nb = 1; goto tailcall;
+ case TCapture: case TRunTime:
+ /* return verifyrule(L, sib1(tree), passed, npassed, nb); */
+ tree = sib1(tree); goto tailcall;
+ case TCall:
+ /* return verifyrule(L, sib2(tree), passed, npassed, nb); */
+ tree = sib2(tree); goto tailcall;
+ case TSeq: /* only check 2nd child if first is nb */
+ if (!verifyrule(L, sib1(tree), passed, npassed, 0))
+ return nb;
+ /* else return verifyrule(L, sib2(tree), passed, npassed, nb); */
+ tree = sib2(tree); goto tailcall;
+ case TChoice: /* must check both children */
+ nb = verifyrule(L, sib1(tree), passed, npassed, nb);
+ /* return verifyrule(L, sib2(tree), passed, npassed, nb); */
+ tree = sib2(tree); goto tailcall;
+ case TRule:
+ if (npassed >= MAXRULES)
+ return verifyerror(L, passed, npassed);
+ else {
+ passed[npassed++] = tree->key;
+ /* return verifyrule(L, sib1(tree), passed, npassed); */
+ tree = sib1(tree); goto tailcall;
+ }
+ case TGrammar:
+ return nullable(tree); /* sub-grammar cannot be left recursive */
+ default: assert(0); return 0;
+ }
+}
+
+
+static void verifygrammar (lua_State *L, TTree *grammar) {
+ int passed[MAXRULES];
+ TTree *rule;
+ /* check left-recursive rules */
+ for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) {
+ if (rule->key == 0) continue; /* unused rule */
+ verifyrule(L, sib1(rule), passed, 0, 0);
+ }
+ assert(rule->tag == TTrue);
+ /* check infinite loops inside rules */
+ for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) {
+ if (rule->key == 0) continue; /* unused rule */
+ if (checkloops(sib1(rule))) {
+ lua_rawgeti(L, -1, rule->key); /* get rule's key */
+ luaL_error(L, "empty loop in rule '%s'", val2str(L, -1));
+ }
+ }
+ assert(rule->tag == TTrue);
+}
+
+
+/*
+** Give a name for the initial rule if it is not referenced
+*/
+static void initialrulename (lua_State *L, TTree *grammar, int frule) {
+ if (sib1(grammar)->key == 0) { /* initial rule is not referenced? */
+ int n = lua_rawlen(L, -1) + 1; /* index for name */
+ lua_pushvalue(L, frule); /* rule's name */
+ lua_rawseti(L, -2, n); /* ktable was on the top of the stack */
+ sib1(grammar)->key = n;
+ }
+}
+
+
+static TTree *newgrammar (lua_State *L, int arg) {
+ int treesize;
+ int frule = lua_gettop(L) + 2; /* position of first rule's key */
+ int n = collectrules(L, arg, &treesize);
+ TTree *g = newtree(L, treesize);
+ luaL_argcheck(L, n <= MAXRULES, arg, "grammar has too many rules");
+ g->tag = TGrammar; g->u.n = n;
+ lua_newtable(L); /* create 'ktable' */
+ lua_setuservalue(L, -2);
+ buildgrammar(L, g, frule, n);
+ lua_getuservalue(L, -1); /* get 'ktable' for new tree */
+ finalfix(L, frule - 1, g, sib1(g));
+ initialrulename(L, g, frule);
+ verifygrammar(L, g);
+ lua_pop(L, 1); /* remove 'ktable' */
+ lua_insert(L, -(n * 2 + 2)); /* move new table to proper position */
+ lua_pop(L, n * 2 + 1); /* remove position table + rule pairs */
+ return g; /* new table at the top of the stack */
+}
+
+/* }====================================================== */
+
+
+static Instruction *prepcompile (lua_State *L, Pattern *p, int idx) {
+ lua_getuservalue(L, idx); /* push 'ktable' (may be used by 'finalfix') */
+ finalfix(L, 0, NULL, p->tree);
+ lua_pop(L, 1); /* remove 'ktable' */
+ return compile(L, p);
+}
+
+
+static int lp_printtree (lua_State *L) {
+ TTree *tree = getpatt(L, 1, NULL);
+ int c = lua_toboolean(L, 2);
+ if (c) {
+ lua_getuservalue(L, 1); /* push 'ktable' (may be used by 'finalfix') */
+ finalfix(L, 0, NULL, tree);
+ lua_pop(L, 1); /* remove 'ktable' */
+ }
+ printktable(L, 1);
+ printtree(tree, 0);
+ return 0;
+}
+
+
+static int lp_printcode (lua_State *L) {
+ Pattern *p = getpattern(L, 1);
+ printktable(L, 1);
+ if (p->code == NULL) /* not compiled yet? */
+ prepcompile(L, p, 1);
+ printpatt(p->code, p->codesize);
+ return 0;
+}
+
+
+/*
+** Get the initial position for the match, interpreting negative
+** values from the end of the subject
+*/
+static size_t initposition (lua_State *L, size_t len) {
+ lua_Integer ii = luaL_optinteger(L, 3, 1);
+ if (ii > 0) { /* positive index? */
+ if ((size_t)ii <= len) /* inside the string? */
+ return (size_t)ii - 1; /* return it (corrected to 0-base) */
+ else return len; /* crop at the end */
+ }
+ else { /* negative index */
+ if ((size_t)(-ii) <= len) /* inside the string? */
+ return len - ((size_t)(-ii)); /* return position from the end */
+ else return 0; /* crop at the beginning */
+ }
+}
+
+
+/*
+** Main match function
+*/
+static int lp_match (lua_State *L) {
+ Capture capture[INITCAPSIZE];
+ const char *r;
+ size_t l;
+ const char *s;
+
+ Pattern *p = (getpatt(L, 1, NULL), getpattern(L, 1));
+ Instruction *code = (p->code != NULL) ? p->code : prepcompile(L, p, 1);
+
+ if (lua_type (L, SUBJIDX) == LUA_TSTRING) {
+ s = luaL_checklstring (L, SUBJIDX, &l);
+ }
+ else if (lua_type (L, SUBJIDX) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text (L, SUBJIDX);
+ if (!t) {
+ return luaL_error (L, "invalid argument (not a text)");
+ }
+ s = t->start;
+ l = t->len;
+
+ if (s == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+ }
+ else {
+ return luaL_error (L, "invalid argument: %s",
+ lua_typename (L, lua_type (L, SUBJIDX)));
+ }
+ size_t i = initposition(L, l);
+ int ptop = lua_gettop(L), rs;
+ lua_pushnil(L); /* initialize subscache */
+ lua_pushlightuserdata(L, capture); /* initialize caplistidx */
+ lua_getuservalue(L, 1); /* initialize penvidx */
+ r = match(L, s, s + i, s + l, code, capture, ptop);
+ if (r == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+ rs = getcaptures(L, s, r, ptop);
+ return rs;
+}
+
+
+
+/*
+** {======================================================
+** Library creation and functions not related to matching
+** =======================================================
+*/
+
+/* maximum limit for stack size */
+#define MAXLIM (INT_MAX / 100)
+
+static int lp_setmax (lua_State *L) {
+ lua_Integer lim = luaL_checkinteger(L, 1);
+ luaL_argcheck(L, 0 < lim && lim <= MAXLIM, 1, "out of range");
+ lua_settop(L, 1);
+ lua_setfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX);
+ return 0;
+}
+
+
+static int lp_version (lua_State *L) {
+ lua_pushstring(L, VERSION);
+ return 1;
+}
+
+
+static int lp_type (lua_State *L) {
+ if (testpattern(L, 1))
+ lua_pushliteral(L, "pattern");
+ else
+ lua_pushnil(L);
+ return 1;
+}
+
+
+int lp_gc (lua_State *L) {
+ Pattern *p = getpattern(L, 1);
+ realloccode(L, p, 0); /* delete code block */
+ return 0;
+}
+
+
+static void createcat (lua_State *L, const char *catname, int (catf) (int)) {
+ TTree *t = newcharset(L);
+ int i;
+ for (i = 0; i <= UCHAR_MAX; i++)
+ if (catf(i)) setchar(treebuffer(t), i);
+ lua_setfield(L, -2, catname);
+}
+
+
+static int lp_locale (lua_State *L) {
+ if (lua_isnoneornil(L, 1)) {
+ lua_settop(L, 0);
+ lua_createtable(L, 0, 12);
+ }
+ else {
+ luaL_checktype(L, 1, LUA_TTABLE);
+ lua_settop(L, 1);
+ }
+ createcat(L, "alnum", isalnum);
+ createcat(L, "alpha", isalpha);
+ createcat(L, "cntrl", iscntrl);
+ createcat(L, "digit", isdigit);
+ createcat(L, "graph", isgraph);
+ createcat(L, "lower", islower);
+ createcat(L, "print", isprint);
+ createcat(L, "punct", ispunct);
+ createcat(L, "space", isspace);
+ createcat(L, "upper", isupper);
+ createcat(L, "xdigit", isxdigit);
+ return 1;
+}
+
+
+static struct luaL_Reg pattreg[] = {
+ {"ptree", lp_printtree},
+ {"pcode", lp_printcode},
+ {"match", lp_match},
+ {"B", lp_behind},
+ {"V", lp_V},
+ {"C", lp_simplecapture},
+ {"Cc", lp_constcapture},
+ {"Cmt", lp_matchtime},
+ {"Cb", lp_backref},
+ {"Carg", lp_argcapture},
+ {"Cp", lp_poscapture},
+ {"Cs", lp_substcapture},
+ {"Ct", lp_tablecapture},
+ {"Cf", lp_foldcapture},
+ {"Cg", lp_groupcapture},
+ {"P", lp_P},
+ {"S", lp_set},
+ {"R", lp_range},
+ {"locale", lp_locale},
+ {"version", lp_version},
+ {"setmaxstack", lp_setmax},
+ {"type", lp_type},
+ {NULL, NULL}
+};
+
+
+static struct luaL_Reg metareg[] = {
+ {"__mul", lp_seq},
+ {"__add", lp_choice},
+ {"__pow", lp_star},
+ {"__gc", lp_gc},
+ {"__len", lp_and},
+ {"__div", lp_divcapture},
+ {"__unm", lp_not},
+ {"__sub", lp_sub},
+ {NULL, NULL}
+};
+
+int luaopen_lpeg (lua_State *L) {
+ luaL_newmetatable(L, PATTERN_T);
+ lua_pushnumber(L, MAXBACK); /* initialize maximum backtracking */
+ lua_setfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX);
+ luaL_setfuncs(L, metareg, 0);
+ luaL_newlib(L, pattreg);
+ lua_pushvalue(L, -1);
+ lua_setfield(L, -3, "__index");
+ return 1;
+}
+
+/* }====================================================== */
diff --git a/contrib/lua-lpeg/lptree.h b/contrib/lua-lpeg/lptree.h
new file mode 100644
index 0000000..38a668e
--- /dev/null
+++ b/contrib/lua-lpeg/lptree.h
@@ -0,0 +1,77 @@
+/*
+** $Id: lptree.h,v 1.2 2013/03/24 13:51:12 roberto Exp $
+*/
+
+#if !defined(lptree_h)
+#define lptree_h
+
+
+#include "lptypes.h"
+
+
+/*
+** types of trees
+*/
+typedef enum TTag {
+ TChar = 0, TSet, TAny, /* standard PEG elements */
+ TTrue, TFalse,
+ TRep,
+ TSeq, TChoice,
+ TNot, TAnd,
+ TCall,
+ TOpenCall,
+ TRule, /* sib1 is rule's pattern, sib2 is 'next' rule */
+ TGrammar, /* sib1 is initial (and first) rule */
+ TBehind, /* match behind */
+ TCapture, /* regular capture */
+ TRunTime /* run-time capture */
+} TTag;
+
+/* number of siblings for each tree */
+extern const byte numsiblings[];
+
+
+/*
+** Tree trees
+** The first sibling of a tree (if there is one) is immediately after
+** the tree. A reference to a second sibling (ps) is its position
+** relative to the position of the tree itself. A key in ktable
+** uses the (unique) address of the original tree that created that
+** entry. NULL means no data.
+*/
+typedef struct TTree {
+ byte tag;
+ byte cap; /* kind of capture (if it is a capture) */
+ unsigned short key; /* key in ktable for Lua data (0 if no key) */
+ union {
+ int ps; /* occasional second sibling */
+ int n; /* occasional counter */
+ } u;
+} TTree;
+
+
+/*
+** A complete pattern has its tree plus, if already compiled,
+** its corresponding code
+*/
+typedef struct Pattern {
+ union Instruction *code;
+ int codesize;
+ TTree tree[1];
+} Pattern;
+
+
+/* number of siblings for each tree */
+extern const byte numsiblings[];
+
+/* access to siblings */
+#define sib1(t) ((t) + 1)
+#define sib2(t) ((t) + (t)->u.ps)
+
+
+int luaopen_lpeg (lua_State *L);
+
+
+
+#endif
+
diff --git a/contrib/lua-lpeg/lptypes.h b/contrib/lua-lpeg/lptypes.h
new file mode 100644
index 0000000..f541c7a
--- /dev/null
+++ b/contrib/lua-lpeg/lptypes.h
@@ -0,0 +1,154 @@
+/*
+** $Id: lptypes.h,v 1.14 2015/09/28 17:17:41 roberto Exp $
+** LPeg - PEG pattern matching for Lua
+** Copyright 2007-2015, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+** written by Roberto Ierusalimschy
+*/
+
+#if !defined(lptypes_h)
+#define lptypes_h
+
+
+#if !defined(LPEG_DEBUG) && !defined(NDEBUG)
+#define NDEBUG
+#endif
+
+#include <assert.h>
+#include <limits.h>
+
+#include "lua.h"
+
+
+#define VERSION "1.0.0"
+
+
+#define PATTERN_T "lpeg-pattern"
+#define MAXSTACKIDX "lpeg-maxstack"
+
+
+/*
+** compatibility with Lua 5.1
+*/
+#if (LUA_VERSION_NUM == 501)
+
+#define lp_equal lua_equal
+
+#define lua_getuservalue lua_getfenv
+#define lua_setuservalue lua_setfenv
+
+#ifndef lua_rawlen
+#define lua_rawlen lua_objlen
+#endif
+
+#ifndef luaL_setfuncs
+#define luaL_setfuncs(L,f,n) luaL_register(L,NULL,f)
+#endif
+#ifndef luaL_newlib
+#define luaL_newlib(L,f) luaL_register(L,"lpeg",f)
+#endif
+#endif
+
+
+#if !defined(lp_equal)
+#define lp_equal(L,idx1,idx2) lua_compare(L,(idx1),(idx2),LUA_OPEQ)
+#endif
+
+
+/* default maximum size for call/backtrack stack */
+#if !defined(MAXBACK)
+#define MAXBACK 400
+#endif
+
+
+/* maximum number of rules in a grammar */
+#if !defined(MAXRULES)
+#define MAXRULES 1000
+#endif
+
+
+
+/* initial size for capture's list */
+#define INITCAPSIZE 32
+
+
+/* index, on Lua stack, for subject */
+#define SUBJIDX 2
+
+/* number of fixed arguments to 'match' (before capture arguments) */
+#define FIXEDARGS 3
+
+/* index, on Lua stack, for capture list */
+#define caplistidx(ptop) ((ptop) + 2)
+
+/* index, on Lua stack, for pattern's ktable */
+#define ktableidx(ptop) ((ptop) + 3)
+
+/* index, on Lua stack, for backtracking stack */
+#define stackidx(ptop) ((ptop) + 4)
+
+
+
+typedef unsigned char byte;
+
+
+#define BITSPERCHAR 8
+
+#define CHARSETSIZE ((UCHAR_MAX/BITSPERCHAR) + 1)
+
+
+
+typedef struct Charset {
+ byte cs[CHARSETSIZE];
+} Charset;
+
+
+
+#define loopset(v,b) { int v; for (v = 0; v < CHARSETSIZE; v++) {b;} }
+
+/* access to charset */
+#define treebuffer(t) ((byte *)((t) + 1))
+
+/* number of slots needed for 'n' bytes */
+#define bytes2slots(n) (((n) - 1) / sizeof(TTree) + 1)
+
+/* set 'b' bit in charset 'cs' */
+#define setchar(cs,b) ((cs)[(b) >> 3] |= (1 << ((b) & 7)))
+
+
+/*
+** in capture instructions, 'kind' of capture and its offset are
+** packed in field 'aux', 4 bits for each
+*/
+#define getkind(op) ((op)->i.aux & 0xF)
+#define getoff(op) (((op)->i.aux >> 4) & 0xF)
+#define joinkindoff(k,o) ((k) | ((o) << 4))
+
+#define MAXOFF 0xF
+#define MAXAUX 0xFF
+
+
+/* maximum number of bytes to look behind */
+#define MAXBEHIND MAXAUX
+
+
+/* maximum size (in elements) for a pattern */
+#define MAXPATTSIZE (SHRT_MAX - 10)
+
+
+/* size (in elements) for an instruction plus extra l bytes */
+#define instsize(l) (((l) + sizeof(Instruction) - 1)/sizeof(Instruction) + 1)
+
+
+/* size (in elements) for a ISet instruction */
+#define CHARSETINSTSIZE instsize(CHARSETSIZE)
+
+/* size (in elements) for a IFunc instruction */
+#define funcinstsize(p) ((p)->i.aux + 2)
+
+
+
+#define testchar(st,c) (((int)(st)[((c) >> 3)] & (1 << ((c) & 7))))
+
+
+#endif
+
diff --git a/contrib/lua-lpeg/lpvm.c b/contrib/lua-lpeg/lpvm.c
new file mode 100644
index 0000000..0dfca18
--- /dev/null
+++ b/contrib/lua-lpeg/lpvm.c
@@ -0,0 +1,383 @@
+/*
+** $Id: lpvm.c,v 1.6 2015/09/28 17:01:25 roberto Exp $
+** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
+*/
+
+#include "config.h"
+
+#include <limits.h>
+#include <string.h>
+
+
+#include "lua.h"
+#include "lauxlib.h"
+
+#include "lpcap.h"
+#include "lptypes.h"
+#include "lpvm.h"
+#include "lpprint.h"
+
+/* initial size for call/backtrack stack */
+#if !defined(INITBACK)
+#define INITBACK MAXBACK
+#endif
+
+
+#define getoffset(p) (((p) + 1)->offset)
+
+static const Instruction giveup = {{IGiveup, 0, 0}};
+
+
+/*
+** {======================================================
+** Virtual Machine
+** =======================================================
+*/
+
+
+typedef struct Stack {
+ const char *s; /* saved position (or NULL for calls) */
+ const Instruction *p; /* next instruction */
+ int caplevel;
+} Stack;
+
+
+#define getstackbase(L, ptop) ((Stack *)lua_touserdata(L, stackidx(ptop)))
+
+
+/*
+** Ensures the size of array 'capture' (with size '*capsize' and
+** 'captop' elements being used) is enough to accomodate 'n' extra
+** elements plus one. (Because several opcodes add stuff to the capture
+** array, it is simpler to ensure the array always has at least one free
+** slot upfront and check its size later.)
+*/
+static Capture *growcap (lua_State *L, Capture *capture, int *capsize,
+ int captop, int n, int ptop) {
+ if (*capsize - captop > n)
+ return capture; /* no need to grow array */
+ else { /* must grow */
+ Capture *newc;
+ int newsize = captop + n + 1; /* minimum size needed */
+ if (newsize < INT_MAX/((int)sizeof(Capture) * 2))
+ newsize *= 2; /* twice that size, if not too big */
+ else if (newsize >= INT_MAX/((int)sizeof(Capture)))
+ luaL_error(L, "too many captures");
+ newc = (Capture *)lua_newuserdata(L, newsize * sizeof(Capture));
+ memcpy(newc, capture, captop * sizeof(Capture));
+ *capsize = newsize;
+ lua_replace(L, caplistidx(ptop));
+ return newc;
+ }
+}
+
+
+/*
+** Double the size of the stack
+*/
+static Stack *doublestack (lua_State *L, Stack **stacklimit, int ptop) {
+ Stack *stack = getstackbase(L, ptop);
+ Stack *newstack;
+ int n = *stacklimit - stack; /* current stack size */
+ int max, newn;
+ lua_getfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX);
+ max = lua_tointeger(L, -1); /* maximum allowed size */
+ lua_pop(L, 1);
+ if (n >= max) /* already at maximum size? */
+ luaL_error(L, "backtrack stack overflow (current limit is %d)", max);
+ newn = 2 * n; /* new size */
+ if (newn > max) newn = max;
+ newstack = (Stack *)lua_newuserdata(L, newn * sizeof(Stack));
+ memcpy(newstack, stack, n * sizeof(Stack));
+ lua_replace(L, stackidx(ptop));
+ *stacklimit = newstack + newn;
+ return newstack + n; /* return next position */
+}
+
+
+/*
+** Interpret the result of a dynamic capture: false -> fail;
+** true -> keep current position; number -> next position.
+** Return new subject position. 'fr' is stack index where
+** is the result; 'curr' is current subject position; 'limit'
+** is subject's size.
+*/
+static int resdyncaptures (lua_State *L, int fr, int curr, int limit) {
+ lua_Integer res;
+ if (!lua_toboolean(L, fr)) { /* false value? */
+ lua_settop(L, fr - 1); /* remove results */
+ return -1; /* and fail */
+ }
+ else if (lua_isboolean(L, fr)) /* true? */
+ res = curr; /* keep current position */
+ else {
+ res = lua_tointeger(L, fr) - 1; /* new position */
+ if (res < curr || res > limit)
+ luaL_error(L, "invalid position returned by match-time capture");
+ }
+ lua_remove(L, fr); /* remove first result (offset) */
+ return res;
+}
+
+
+/*
+** Add capture values returned by a dynamic capture to the list
+** 'capture', nested inside a group. 'fd' indexes the first capture
+** value, 'n' is the number of values (at least 1). The open group
+** capture is already in 'capture', before the place for the new entries.
+*/
+static void adddyncaptures (const char *s, Capture *capture, int n, int fd) {
+ int i;
+ assert(capture[-1].kind == Cgroup && capture[-1].siz == 0);
+ capture[-1].idx = 0; /* make group capture an anonymous group */
+ for (i = 0; i < n; i++) { /* add runtime captures */
+ capture[i].kind = Cruntime;
+ capture[i].siz = 1; /* mark it as closed */
+ capture[i].idx = fd + i; /* stack index of capture value */
+ capture[i].s = s;
+ }
+ capture[n].kind = Cclose; /* close group */
+ capture[n].siz = 1;
+ capture[n].s = s;
+}
+
+
+/*
+** Remove dynamic captures from the Lua stack (called in case of failure)
+*/
+static int removedyncap (lua_State *L, Capture *capture,
+ int level, int last) {
+ int id = finddyncap(capture + level, capture + last); /* index of 1st cap. */
+ int top = lua_gettop(L);
+ if (id == 0) return 0; /* no dynamic captures? */
+ lua_settop(L, id - 1); /* remove captures */
+ return top - id + 1; /* number of values removed */
+}
+
+
+/*
+** Opcode interpreter
+*/
+const char *match (lua_State *L, const char *o, const char *s, const char *e,
+ Instruction *op, Capture *capture, int ptop) {
+ Stack stackbase[INITBACK];
+ Stack *stacklimit = stackbase + INITBACK;
+ Stack *stack = stackbase; /* point to first empty slot in stack */
+ int capsize = INITCAPSIZE;
+ int captop = 0; /* point to first empty slot in captures */
+ int ndyncap = 0; /* number of dynamic captures (in Lua stack) */
+ const Instruction *p = op; /* current instruction */
+ stack->p = &giveup; stack->s = s; stack->caplevel = 0; stack++;
+ lua_pushlightuserdata(L, stackbase);
+ for (;;) {
+#if defined(DEBUG)
+ printf("s: |%s| stck:%d, dyncaps:%d, caps:%d ",
+ s, (int)(stack - getstackbase(L, ptop)), ndyncap, captop);
+ printinst(op, p);
+#endif
+ assert(stackidx(ptop) + ndyncap == lua_gettop(L) && ndyncap <= captop);
+ switch ((Opcode)p->i.code) {
+ case IEnd: {
+ assert(stack == getstackbase(L, ptop) + 1);
+ capture[captop].kind = Cclose;
+ capture[captop].s = NULL;
+ return s;
+ }
+ case IGiveup: {
+ assert(stack == getstackbase(L, ptop));
+ return NULL;
+ }
+ case IRet: {
+ assert(stack > getstackbase(L, ptop) && (stack - 1)->s == NULL);
+ p = (--stack)->p;
+ continue;
+ }
+ case IAny: {
+ if (s < e) { p++; s++; }
+ else goto fail;
+ continue;
+ }
+ case ITestAny: {
+ if (s < e) p += 2;
+ else p += getoffset(p);
+ continue;
+ }
+ case IChar: {
+ if (s < e && (byte)*s == p->i.aux) { p++; s++; }
+ else goto fail;
+ continue;
+ }
+ case ITestChar: {
+ if (s < e && (byte)*s == p->i.aux) p += 2;
+ else p += getoffset(p);
+ continue;
+ }
+ case ISet: {
+ if (s < e) {
+ int c = (byte) *s;
+ if (testchar((p + 1)->buff, c)) {
+ p += CHARSETINSTSIZE;
+ s++;
+ }
+ else goto fail;
+ }
+ else {
+ goto fail;
+ }
+ continue;
+ }
+ case ITestSet: {
+ if (s < e) {
+ int c = (byte) *s;
+ if (testchar((p + 2)->buff, c))
+ p += 1 + CHARSETINSTSIZE;
+ else p += getoffset(p);
+ }
+ else {
+ p += getoffset(p);
+ }
+ continue;
+ }
+ case IBehind: {
+ int n = p->i.aux;
+ if (n > s - o) goto fail;
+ s -= n; p++;
+ continue;
+ }
+ case ISpan: {
+ for (; s < e; s++) {
+ int c = (byte)*s;
+ if (!testchar((p+1)->buff, c)) break;
+ }
+ p += CHARSETINSTSIZE;
+ continue;
+ }
+ case IJmp: {
+ p += getoffset(p);
+ continue;
+ }
+ case IChoice: {
+ if (stack == stacklimit)
+ stack = doublestack(L, &stacklimit, ptop);
+ stack->p = p + getoffset(p);
+ stack->s = s;
+ stack->caplevel = captop;
+ stack++;
+ p += 2;
+ continue;
+ }
+ case ICall: {
+ if (stack == stacklimit)
+ stack = doublestack(L, &stacklimit, ptop);
+ stack->s = NULL;
+ stack->p = p + 2; /* save return address */
+ stack++;
+ p += getoffset(p);
+ continue;
+ }
+ case ICommit: {
+ assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL);
+ stack--;
+ p += getoffset(p);
+ continue;
+ }
+ case IPartialCommit: {
+ assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL);
+ (stack - 1)->s = s;
+ (stack - 1)->caplevel = captop;
+ p += getoffset(p);
+ continue;
+ }
+ case IBackCommit: {
+ assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL);
+ s = (--stack)->s;
+ captop = stack->caplevel;
+ p += getoffset(p);
+ continue;
+ }
+ case IFailTwice:
+ assert(stack > getstackbase(L, ptop));
+ stack--;
+ /* go through */
+ case IFail:
+ fail: { /* pattern failed: try to backtrack */
+ do { /* remove pending calls */
+ assert(stack > getstackbase(L, ptop));
+ s = (--stack)->s;
+ } while (s == NULL);
+ if (ndyncap > 0) /* is there matchtime captures? */
+ ndyncap -= removedyncap(L, capture, stack->caplevel, captop);
+ captop = stack->caplevel;
+ p = stack->p;
+ continue;
+ }
+ case ICloseRunTime: {
+ CapState cs;
+ int rem, res, n;
+ int fr = lua_gettop(L) + 1; /* stack index of first result */
+ cs.reclevel = 0; cs.L = L;
+ cs.s = o; cs.ocap = capture; cs.ptop = ptop;
+ n = runtimecap(&cs, capture + captop, s, &rem); /* call function */
+ captop -= n; /* remove nested captures */
+ ndyncap -= rem; /* update number of dynamic captures */
+ fr -= rem; /* 'rem' items were popped from Lua stack */
+ res = resdyncaptures(L, fr, s - o, e - o); /* get result */
+ if (res == -1) /* fail? */
+ goto fail;
+ s = o + res; /* else update current position */
+ n = lua_gettop(L) - fr + 1; /* number of new captures */
+ ndyncap += n; /* update number of dynamic captures */
+ if (n == 0) /* no new captures? */
+ captop--; /* remove open group */
+ else { /* new captures; keep original open group */
+ if (fr + n >= SHRT_MAX)
+ luaL_error(L, "too many results in match-time capture");
+ /* add new captures + close group to 'capture' list */
+ capture = growcap(L, capture, &capsize, captop, n + 1, ptop);
+ adddyncaptures(s, capture + captop, n, fr);
+ captop += n + 1; /* new captures + close group */
+ }
+ p++;
+ continue;
+ }
+ case ICloseCapture: {
+ const char *s1 = s;
+ assert(captop > 0);
+ /* if possible, turn capture into a full capture */
+ if (capture[captop - 1].siz == 0 &&
+ s1 - capture[captop - 1].s < UCHAR_MAX) {
+ capture[captop - 1].siz = s1 - capture[captop - 1].s + 1;
+ p++;
+ continue;
+ }
+ else {
+ capture[captop].siz = 1; /* mark entry as closed */
+ capture[captop].s = s;
+ goto pushcapture;
+ }
+ }
+ case IOpenCapture:
+ capture[captop].siz = 0; /* mark entry as open */
+ capture[captop].s = s;
+ goto pushcapture;
+ case IFullCapture:
+ capture[captop].siz = getoff(p) + 1; /* save capture size */
+ capture[captop].s = s - getoff(p);
+ /* goto pushcapture; */
+ pushcapture: {
+ capture[captop].idx = p->i.key;
+ capture[captop].kind = getkind(p);
+ captop++;
+ capture = growcap(L, capture, &capsize, captop, 0, ptop);
+ p++;
+ continue;
+ }
+ default: assert(0); return NULL;
+ }
+ }
+
+}
+
+/* }====================================================== */
+
+
diff --git a/contrib/lua-lpeg/lpvm.h b/contrib/lua-lpeg/lpvm.h
new file mode 100644
index 0000000..757b9e1
--- /dev/null
+++ b/contrib/lua-lpeg/lpvm.h
@@ -0,0 +1,58 @@
+/*
+** $Id: lpvm.h,v 1.3 2014/02/21 13:06:41 roberto Exp $
+*/
+
+#if !defined(lpvm_h)
+#define lpvm_h
+
+#include "lpcap.h"
+
+
+/* Virtual Machine's instructions */
+typedef enum Opcode {
+ IAny, /* if no char, fail */
+ IChar, /* if char != aux, fail */
+ ISet, /* if char not in buff, fail */
+ ITestAny, /* in no char, jump to 'offset' */
+ ITestChar, /* if char != aux, jump to 'offset' */
+ ITestSet, /* if char not in buff, jump to 'offset' */
+ ISpan, /* read a span of chars in buff */
+ IBehind, /* walk back 'aux' characters (fail if not possible) */
+ IRet, /* return from a rule */
+ IEnd, /* end of pattern */
+ IChoice, /* stack a choice; next fail will jump to 'offset' */
+ IJmp, /* jump to 'offset' */
+ ICall, /* call rule at 'offset' */
+ IOpenCall, /* call rule number 'key' (must be closed to a ICall) */
+ ICommit, /* pop choice and jump to 'offset' */
+ IPartialCommit, /* update top choice to current position and jump */
+ IBackCommit, /* "fails" but jump to its own 'offset' */
+ IFailTwice, /* pop one choice and then fail */
+ IFail, /* go back to saved state on choice and jump to saved offset */
+ IGiveup, /* internal use */
+ IFullCapture, /* complete capture of last 'off' chars */
+ IOpenCapture, /* start a capture */
+ ICloseCapture,
+ ICloseRunTime
+} Opcode;
+
+
+
+typedef union Instruction {
+ struct Inst {
+ byte code;
+ byte aux;
+ short key;
+ } i;
+ int offset;
+ byte buff[1];
+} Instruction;
+
+
+void printpatt (Instruction *p, int n);
+const char *match (lua_State *L, const char *o, const char *s, const char *e,
+ Instruction *op, Capture *capture, int ptop);
+
+
+#endif
+
diff --git a/contrib/lua-lupa/LICENSE b/contrib/lua-lupa/LICENSE
new file mode 100644
index 0000000..66c1141
--- /dev/null
+++ b/contrib/lua-lupa/LICENSE
@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2015-2018 Mitchell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/contrib/lua-lupa/README.md b/contrib/lua-lupa/README.md
new file mode 100644
index 0000000..edf6dce
--- /dev/null
+++ b/contrib/lua-lupa/README.md
@@ -0,0 +1,179 @@
+# Lupa
+
+## Introduction
+
+Lupa is a [Jinja2][] template engine implementation written in Lua and supports
+Lua syntax within tags and variables.
+
+Lupa was sponsored by the [Library of the University of Antwerp][].
+
+[Jinja2]: http://jinja.pocoo.org
+[Library of the University of Antwerp]: http://www.uantwerpen.be/
+
+## Requirements
+
+Lupa has the following requirements:
+
+* [Lua][] 5.1, 5.2, or 5.3.
+* The [LPeg][] library.
+
+[Lua]: http://www.lua.org
+[LPeg]: http://www.inf.puc-rio.br/~roberto/lpeg/
+
+## Download
+
+Download Lupa from the project’s [download page][].
+
+[download page]: download
+
+## Installation
+
+Unzip Lupa and place the "lupa.lua" file in your Lua installation's
+`package.path`. This location depends on your version of Lua. Typical locations
+are listed below.
+
+* Lua 5.1: */usr/local/share/lua/5.1/* or */usr/local/share/lua/5.1/*
+* Lua 5.2: */usr/local/share/lua/5.2/* or */usr/local/share/lua/5.2/*
+* Lua 5.3: */usr/local/share/lua/5.3/* or */usr/local/share/lua/5.3/*
+
+You can also place the "lupa.lua" file wherever you'd like and add it to Lua's
+`package.path` manually in your program. For example, if Lupa was placed in a
+*/home/user/lua/* directory, it can be used as follows:
+
+ package.path = package.path..';/home/user/lua/?.lua'
+
+## Usage
+
+Lupa is simply a Lua library. Its `lupa.expand()` and `lupa.expand_file()`
+functions may called to process templates. For example:
+
+ lupa = require('lupa')
+ lupa.expand("hello {{ s }}!", {s = "world"}) --> "hello world!"
+ lupa.expand("{% for i in {1, 2, 3} %}{{ i }}{% endfor %}") --> 123
+
+By default, Lupa loads templates relative to the current working directory. This
+can be changed by reconfiguring Lupa:
+
+ lupa.expand_file('name') --> expands template "./name"
+ lupa.configure{loader = lupa.loaders.filesystem('path/to/templates')}
+ lupa.expand_file('name') --> expands template "path/to/templates/name"
+
+See Lupa's [API documentation][] for more information.
+
+[API documentation]: api.html
+
+## Syntax
+
+Please refer to Jinja2's extensive [template documentation][]. Any
+incompatibilities are listed in the sections below.
+
+[template documentation]: http://jinja.pocoo.org/docs/dev/templates/
+
+## Comparison with Jinja2
+
+While Lua and Python (Jinja2's implementation language) share some similarities,
+the languages themselves are fundamentally different. Nevertheless, a
+significant effort was made to support a vast majority of Jinja2's Python-style
+syntax. As a result, Lupa passes Jinja2's test suite with only a handful of
+modifications. The comprehensive list of differences between Lupa and Jinja2 is
+described in the following sections.
+
+### Fundamental Differences
+
+* Expressions use Lua's syntax instead of Python's, so many of Python's
+ syntactic constructs are not valid. However, the following constructs
+ *are valid*, despite being invalid in pure Lua:
+
+ + Iterating over table literals or table variables directly in a "for" loop:
+
+ {% for i in {1, 2, 3} %}...{% endfor %}
+
+ + Conditional loops via an "if" expression suffix:
+
+ {% for x in range(10) if is_odd(x) %}...{% endfor %}
+
+ + Table unpacking for list elements when iterating through a list of lists:
+
+ {% for a, b, c in {{1, 2, 3}, {4, 5, 6}} %}...{% endfor %}
+
+ + Default values for macro arguments:
+
+ {% macro m(a, b, c='c', d='d') %}...{% endmacro %}
+
+* Strings do not have unicode escapes nor is unicode interpreted in any way.
+
+### Syntactic Differences
+
+* Line statements are not supported due to parsing complexity.
+* In `{% for ... %}` loops, the `loop.length`, `loop.revindex`,
+ `loop.revindex0`, and `loop.last` variables only apply to sequences, where
+ Lua's `'#'` operator applies.
+* The `{% continue %}` and `{% break %}` loop controls are not supported due to
+ complexity.
+* Loops may be used recursively by default, so the `recursive` loop modifier is
+ not supported.
+* The `is` operator is not supported by Lua, so tests of the form `{{ x is y }}`
+ should be written `{{ is_y(x) }}` (e.g. `{{ is_number(42) }}`).
+* Filters cannot occur after tokens within an expression (e.g.
+ `{{ "foo"|upper .. "bar"|upper }}`), but can only occur at the end of an
+ expression (e.g. `{{ "foo".."bar"|upper }}`).
+* Blocks always have access to scoped variables, so the `scoped` block modifier
+ is not supported.
+* Named block end tags are not supported since the parser cannot easily keep
+ track of that state information.
+* Any `{% block ... %}` tags within a "false" block (e.g. `{% if a %}` where `a`
+ evaluates to `false`) are never read and stored due to the parser
+ implementation.
+* Inline "if" expressions (e.g. `{% extends b if a else c %}`) are not
+ supported. Instead, use a Lua conditional expression
+ (e.g. `{% extends a and b or c %}`).
+* Any `{% extends ... %}` tags within a sub-scope are not effective outside that
+ scope (e.g. `{% if a %}{% extends a %}{% else %}{% extends b %}{% endif %}`).
+ Instead, use a Lua conditional expression (e.g. `{% extends a or b %}`).
+* Macros are simply Lua functions and have no metadata attributes.
+* Macros do not have access to a `kwargs` variable since Lua does not support
+ keyword arguments.
+* `{% from x import y %}` tags are not supported. Instead, you must use either
+ `{% import x %}`, which imports all globals in `x` into the current
+ environment, or use `{% import x as z %}`, which imports all globals in `x`
+ into the variable `z`.
+* `{% set ... %}` does not support multiple assignment. Use `{% do ...%}`
+ instead. The catch is that `{% do ... %}` does not support filters.
+* The `{% trans %}` and `{% endtrans %}` tags, `{% with %}` and `{% endwith %}`
+ tags, and `{% autoescape %}` and `{% endautoescape %}` tags are not supported
+ since they are outside the scope of this implementation.
+
+### Filter Differences
+
+* Only the `batch`, `groupby`, and `slice` filters return generators which
+ produce one item at a time when looping. All other filters that produce
+ iterable results generate all items at once.
+* The `float` filter only works in Lua 5.3 since that version of Lua has a
+ distinction between floats and integers.
+* The `safe` filter must appear at the end of a filter chain since its output
+ cannot be passed to any other filter.
+
+### Function Differences
+
+* The global `range(n)` function returns a sequence from 1 to `n`, inclusive,
+ since lists start at 1 in Lua.
+* No `lipsum()`, `dict()`, or `joiner()` functions for the sake of simplicity.
+
+### API Differences
+
+* Lupa has a much simpler API consisting of just four functions and three
+ fields:
+
+ + `lupa.expand()`: Expands a string template subject to an environment.
+ + `lupa.expand_file()`: Expands a file template subject to an environment.
+ + `lupa.configure()` Configures delimiters and template options.
+ + `lupa.reset()`: Resets delimiters and options to their defaults.
+ + `lupa.env`: The default environment for templates.
+ + `lupa.filters`: The set of available filters (`escape`, `join`, etc.).
+ + `lupa.tests`: The set of available tests (`is_odd`, `is_defined`, etc.).
+
+* There is no bytecode caching.
+* Lupa has no extension mechanism. Instead, modify `lupa.env`, `lupa.filters`,
+ and `lupa.tests` directly. However, the parser cannot be extended.
+* Sandboxing is not supported, although `lupa.env` is safe by default (`io`,
+ `os.execute`, `os.remove`, etc. are not available).
diff --git a/contrib/lua-lupa/lupa.lua b/contrib/lua-lupa/lupa.lua
new file mode 100644
index 0000000..adaf419
--- /dev/null
+++ b/contrib/lua-lupa/lupa.lua
@@ -0,0 +1,1810 @@
+-- Copyright 2015-2020 Mitchell. See LICENSE.
+-- Sponsored by the Library of the University of Antwerp.
+-- Contributions from Ana Balan.
+-- Lupa templating engine.
+
+--[[ This comment is for LuaDoc.
+---
+-- Lupa is a Jinja2 template engine implementation written in Lua and supports
+-- Lua syntax within tags and variables.
+module('lupa')]]
+local M = {}
+
+local lpeg = require('lpeg')
+lpeg.locale(lpeg)
+local space, newline = lpeg.space, lpeg.P('\r')^-1 * '\n'
+local P, S, V = lpeg.P, lpeg.S, lpeg.V
+local C, Cc, Cg, Cp, Ct = lpeg.C, lpeg.Cc, lpeg.Cg, lpeg.Cp, lpeg.Ct
+
+---
+-- Lupa's expression filters.
+-- @class table
+-- @name filters
+M.filters = {}
+
+---
+-- Lupa's value tests.
+-- @class table
+-- @name tests
+M.tests = {}
+
+---
+-- Lupa's template loaders.
+-- @class table
+-- @name loaders
+M.loaders = {}
+
+-- Lua version compatibility.
+if _VERSION == 'Lua 5.1' then
+ function load(ld, source, mode, env)
+ local f, err = loadstring(ld)
+ if f and env then return setfenv(f, env) end
+ return f, err
+ end
+ table.unpack = unpack
+end
+
+local newline_sequence, keep_trailing_newline, autoescape = '\n', false, false
+local loader
+
+-- Creates and returns a token pattern with token name *name* and pattern
+-- *patt*.
+-- The returned pattern captures three values: the token's position and name,
+-- and either a string value or table of capture values.
+-- Tokens are used to construct an Abstract Syntax Tree (AST) for a template.
+-- @param name The name of the token.
+-- @param patt The pattern to match. It must contain only one capture: either a
+-- string or table of captures.
+-- @see evaluate
+local function token(name, patt) return Cp() * Cc(name) * patt end
+
+-- Returns an LPeg pattern that immediately raises an error with message
+-- *errmsg* for invalid syntax when parsing a template.
+-- @param errmsg The error message to raise an error with.
+local function lpeg_error(errmsg)
+ return P(function(input, index)
+ input = input:sub(1, index)
+ local _, line_num = input:gsub('\n', '')
+ local col_num = #input:match('[^\n]*$')
+ error(string.format('Parse Error in file "%s" on line %d, column %d: %s',
+ M._FILENAME, line_num + 1, col_num, errmsg), 0)
+ end)
+end
+
+---
+-- Configures the basic delimiters and options for templates.
+-- This function then regenerates the grammar for parsing templates.
+-- Note: this function cannot be used iteratively to configure Lupa options.
+-- Any options not provided are reset to their default values.
+-- @param ts The tag start delimiter. The default value is '{%'.
+-- @param te The tag end delimiter. The default value is '%}'.
+-- @param vs The variable start delimiter. The default value is '{{'.
+-- @param ve The variable end delimiter. The default value is '}}'.
+-- @param cs The comment start delimiter. The default value is '{#'.
+-- @param ce The comment end delimiter. The default value is '#}'.
+-- @param options Optional set of options for templates:
+--
+-- * `trim_blocks`: Trim the first newline after blocks.
+-- * `lstrip_blocks`: Strip line-leading whitespace in front of tags.
+-- * `newline_sequence`: The end-of-line character to use.
+-- * `keep_trailing_newline`: Whether or not to keep a newline at the end of
+-- a template.
+-- * `autoescape`: Whether or not to autoescape HTML entities. May be a
+-- function that accepts the template's filename as an argument and returns
+-- a boolean.
+-- * `loader`: Function that receives a template name to load and returns the
+-- path to that template.
+-- @name configure
+function M.configure(ts, te, vs, ve, cs, ce, options)
+ if type(ts) == 'table' then options, ts = ts, nil end
+ if not ts then ts = '{%' end
+ if not te then te = '%}' end
+ if not vs then vs = '{{' end
+ if not ve then ve = '}}' end
+ if not cs then cs = '{#' end
+ if not ce then ce = '#}' end
+
+ -- Tokens for whitespace control.
+ local lstrip = token('lstrip', C('-')) + '+' -- '+' is handled by grammar
+ local rstrip = token('rstrip', -(P(te) + ve + ce) * C('-'))
+
+ -- Configure delimiters, including whitespace control.
+ local tag_start = P(ts) * lstrip^-1 * space^0
+ local tag_end = space^0 * rstrip^-1 * P(te)
+ local variable_start = P(vs) * lstrip^-1 * space^0
+ local variable_end = space^0 * rstrip^-1 * P(ve)
+ local comment_start = P(cs) * lstrip^-1 * space^0
+ local comment_end = space^0 * rstrip^-1 * P(ce)
+ if options and options.trim_blocks then
+ -- Consider whitespace, including a newline, immediately following a tag as
+ -- part of that tag so it is not captured as plain text. Basically, strip
+ -- the trailing newline from tags.
+ tag_end = tag_end * S(' \t')^0 * newline^-1
+ comment_end = comment_end * S(' \t')^0 * newline^-1
+ end
+
+ -- Error messages.
+ local variable_end_error = lpeg_error('"'..ve..'" expected')
+ local comment_end_error = lpeg_error('"'..ce..'" expected')
+ local tag_end_error = lpeg_error('"'..te..'" expected')
+ local endraw_error = lpeg_error('additional tag or "'..ts..' endraw '..te..
+ '" expected')
+ local expr_error = lpeg_error('expression expected')
+ local endblock_error = lpeg_error('additional tag or "'..ts..' endblock '..
+ te..'" expected')
+ local endfor_error = lpeg_error('additional tag or "'..ts..' endfor '..te..
+ '" expected')
+ local endif_error = lpeg_error('additional tag or "'..ts..' endif '..te..
+ '" expected')
+ local endmacro_error = lpeg_error('additional tag or "'..ts..' endmacro '..
+ te..'" expected')
+ local endcall_error = lpeg_error('additional tag or "'..ts..' endcall '..te..
+ '" expected')
+ local endfilter_error = lpeg_error('additional tag or "'..ts..' endfilter '..
+ te..'" expected')
+ local tag_error = lpeg_error('unknown or unexpected tag')
+ local main_error = lpeg_error('unexpected character; text or tag expected')
+
+ -- Grammar.
+ M.grammar = Ct(P{
+ -- Utility patterns used by tokens.
+ entity_start = tag_start + variable_start + comment_start,
+ any_text = (1 - V('entity_start'))^1,
+ -- Allow '{{' by default in expression text since it is valid in Lua.
+ expr_text = (1 - tag_end - tag_start - comment_start)^1,
+ -- When `options.lstrip_blocks` is enabled, ignore leading whitespace
+ -- immediately followed by a tag (as long as '+' is not present) so that
+ -- whitespace not captured as plain text. Basically, strip leading spaces
+ -- from tags.
+ line_text = (1 - newline - V('entity_start'))^1,
+ lstrip_entity_start = -P(vs) * (P(ts) + cs) * -P('+'),
+ lstrip_space = S(' \t')^1 * #V('lstrip_entity_start'),
+ text_lines = V('line_text') * (newline * -(S(' \t')^0 * V('lstrip_entity_start')) * V('line_text'))^0 * newline^-1 + newline,
+
+ -- Plain text.
+ text = (not options or not options.lstrip_blocks) and
+ token('text', C(V('any_text'))) or
+ V('lstrip_space') + token('text', C(V('text_lines'))),
+
+ -- Variables: {{ expr }}.
+ lua_table = '{' * ((1 - S('{}')) + V('lua_table'))^0 * '}',
+ variable = variable_start *
+ token('variable', C((V('lua_table') + (1 - variable_end))^0)) *
+ (variable_end + variable_end_error),
+
+ -- Filters: handled in variable evaluation.
+
+ -- Tests: handled in control structure expression evaluation.
+
+ -- Comments: {# comment #}.
+ comment = comment_start * (1 - comment_end)^0 * (comment_end + comment_end_error),
+
+ -- Whitespace control: handled in tag/variable/comment start/end.
+
+ -- Escaping: {% raw %} body {% endraw %}.
+ raw_block = tag_start * 'raw' * (tag_end + tag_end_error) *
+ token('text', C((1 - (tag_start * 'endraw' * tag_end))^0)) *
+ (tag_start * 'endraw' * tag_end + endraw_error),
+
+ -- Note: line statements are not supported since this grammer cannot parse
+ -- Lua itself.
+
+ -- Template inheritence.
+ -- {% block ... %} body {% endblock %}
+ block_block = tag_start * 'block' * space^1 * token('block', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1)) *
+ (tag_start * 'endblock' * tag_end + endblock_error),
+ -- {% extends ... %}
+ extends_tag = tag_start * 'extends' * space^1 * token('extends', C(V('expr_text')) + expr_error) * (tag_end + tag_end_error),
+ -- Super blocks are handled in variables.
+ -- Note: named block end tags are not supported since keeping track of that
+ -- state information is difficult.
+ -- Note: block nesting and scope is not applicable since blocks always have
+ -- access to scoped variables in this implementation.
+
+ -- Control Structures.
+ -- {% for expr %} body {% else %} body {% endfor %}
+ for_block = tag_start * 'for' * space^1 * token('for', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1 *
+ Cg(Ct(tag_start * 'else' * tag_end *
+ V('body')^-1), 'else')^-1)) *
+ (tag_start * 'endfor' * tag_end + endfor_error),
+ -- {% if expr %} body {% elseif expr %} body {% else %} body {% endif %}
+ if_block = tag_start * 'if' * space^1 * token('if', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1 *
+ Cg(Ct(Ct(tag_start * 'elseif' * space^1 * (Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1)^1), 'elseif')^-1 *
+ Cg(Ct(tag_start * 'else' * tag_end *
+ V('body')^-1), 'else')^-1)) *
+ (tag_start * 'endif' * tag_end + endif_error),
+ -- {% macro expr %} body {% endmacro %}
+ macro_block = tag_start * 'macro' * space^1 * token('macro', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1)) *
+ (tag_start * 'endmacro' * tag_end + endmacro_error),
+ -- {% call expr %} body {% endcall %}
+ call_block = tag_start * 'call' * (space^1 + #P('(')) * token('call', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1)) *
+ (tag_start * 'endcall' * tag_end + endcall_error),
+ -- {% filter expr %} body {% endfilter %}
+ filter_block = tag_start * 'filter' * space^1 * token('filter', Ct((Cg(V('expr_text'), 'expression') + expr_error) * (tag_end + tag_end_error) *
+ V('body')^-1)) *
+ (tag_start * 'endfilter' * tag_end + endfilter_error),
+ -- {% set ... %}
+ set_tag = tag_start * 'set' * space^1 * token('set', C(V('expr_text')) + expr_error) * (tag_end + tag_end_error),
+ -- {% include ... %}
+ include_tag = tag_start * 'include' * space^1 * token('include', C(V('expr_text')) + expr_error) * (tag_end + tag_end_error),
+ -- {% import ... %}
+ import_tag = tag_start * 'import' * space^1 * token('import', C(V('expr_text')) + expr_error) * (tag_end + tag_end_error),
+
+ -- Note: i18n is not supported since it is out of scope for this
+ -- implementation.
+
+ -- Expression statement: {% do ... %}.
+ do_tag = tag_start * 'do' * space^1 * token('do', C(V('expr_text')) + expr_error) * (tag_end + tag_end_error),
+
+ -- Note: loop controls are not supported since that would require jumping
+ -- between "scopes" (e.g. from within an "if" block to outside that "if"
+ -- block's parent "for" block when coming across a {% break %} tag).
+
+ -- Note: with statement is not supported since it is out of scope for this
+ -- implementation.
+
+ -- Note: autoescape is not supported since it is out of scope for this
+ -- implementation.
+
+ -- Any valid blocks of text or tags.
+ body = (V('text') + V('variable') + V('comment') + V('raw_block') +
+ V('block_block') + V('extends_tag') + V('for_block') +
+ V('if_block') + V('macro_block') + V('call_block') +
+ V('filter_block') + V('set_tag') + V('include_tag') +
+ V('import_tag') + V('do_tag'))^0,
+
+ -- Main pattern.
+ V('body') * (-1 + tag_start * tag_error + main_error),
+ })
+
+ -- Other options.
+ if options and options.newline_sequence then
+ assert(options.newline_sequence:find('^\r?\n$'),
+ 'options.newline_sequence must be "\r\n" or "\n"')
+ newline_sequence = options.newline_sequence
+ else
+ newline_sequence = '\n'
+ end
+ if options and options.keep_trailing_newline then
+ keep_trailing_newline = options.keep_trailing_newline
+ else
+ keep_trailing_newline = false
+ end
+ if options and options.autoescape then
+ autoescape = options.autoescape
+ else
+ autoescape = false
+ end
+ if options and options.loader then
+ assert(type(options.loader) == 'function',
+ 'options.loader must be a function that returns a filename')
+ loader = options.loader
+ else
+ loader = M.loaders.filesystem()
+ end
+end
+
+-- Wraps Lua's `assert()` in template environment *env* such that, when called
+-- in conjunction with another Lua function that produces an error message (e.g.
+-- `load()` and `pcall()`), that error message's context (source and line
+-- number) is replaced by the template's context.
+-- This results in Lua's error messages pointing to a template position rather
+-- than this library's source code.
+-- @param env The environment for the currently running template. It must have
+-- a `_SOURCE` field with the template's source text and a `_POSITION` field
+-- with the current position of expansion.
+-- @param ... Arguments to Lua's `assert()`.
+local function env_assert(env, ...)
+ if not select(1, ...) then
+ local input = env._LUPASOURCE:sub(1, env._LUPAPOSITION)
+ local _, line_num = input:gsub('\n', '')
+ local col_num = #input:match('[^\n]*$')
+ local errmsg = select(2, ...)
+ errmsg = errmsg:match(':%d+: (.*)$') or errmsg -- reformat if necessary
+ error(string.format('Runtime Error in file "%s" on line %d, column %d: %s',
+ env._LUPAFILENAME, line_num + 1, col_num, errmsg), 0)
+ end
+ return ...
+end
+
+-- Returns a generator that returns the position and filter in a list of
+-- filters, taking into account '|'s that may be within filter arguments.
+-- @usage for pos, filter in each_filter('foo|join("|")|bar') do ... end
+local function each_filter(s)
+ local init = 1
+ return function(s)
+ local pos, filter, e = s:match('^%s*()([^|(]+%b()[^|]*)|?()', init)
+ if not pos then pos, filter, e = s:match('()([^|]+)|?()', init) end
+ init = e
+ return pos, filter
+ end, s
+end
+
+-- Evaluates template variable *expression* subject to template environment
+-- *env*, applying any filters given in *expression*.
+-- @param expression The string expression to evaluate.
+-- @param env The environment to evaluate the expression in.
+local function eval(expression, env)
+ local expr, pos, filters = expression:match('^([^|]*)|?()(.-)$')
+ -- Evaluate base expression.
+ local f = env_assert(env, load('return '..expr, nil, nil, env))
+ local result = select(2, env_assert(env, pcall(f)))
+ -- Apply any filters.
+ local results, multiple_results = nil, false
+ local p = env._LUPAPOSITION + pos - 1 -- mark position at first filter
+ for pos, filter in each_filter(filters) do
+ env._LUPAPOSITION = p + pos - 1 -- update position for error messages
+ local name, params = filter:match('^%s*([%w_]+)%(?(.-)%)?%s*$')
+ f = M.filters[name]
+ env_assert(env, f, 'unknown filter "'..name..'"')
+ local args = env_assert(env, load('return {'..params..'}', nil, nil, env),
+ 'invalid filter parameter(s) for "'..name..'"')()
+ if not multiple_results then
+ results = {select(2,
+ env_assert(env, pcall(f, result, table.unpack(args))))}
+ else
+ for i = 1, #results do table.insert(args, i, results[i]) end
+ results = {select(2, env_assert(env, pcall(f, table.unpack(args))))}
+ end
+ result, multiple_results = results[1], #results > 1
+ end
+ if multiple_results then return table.unpack(results) end
+ return result
+end
+
+local iterate
+
+-- Iterates over *ast*, a collection of tokens from a portion of a template's
+-- Abstract Syntax Tree (AST), evaluating any expressions in template
+-- environment *env*, and returns a concatenation of the results.
+-- @param ast A template's AST or portion of its AST (e.g. portion inside a
+-- 'for' control structure).
+-- @param env Environment to evaluate any expressions in.
+local function evaluate(ast, env)
+ local chunks = {}
+ local extends -- text of a parent template
+ local rstrip -- flag for stripping leading whitespace of next token
+ for i = 1, #ast, 3 do
+ local pos, token, block = ast[i], ast[i + 1], ast[i + 2]
+ env._LUPAPOSITION = pos
+ if token == 'text' then
+ chunks[#chunks + 1] = block
+ elseif token == 'variable' then
+ local value = eval(block, env)
+ if autoescape then
+ local escape = autoescape
+ if type(autoescape) == 'function' then
+ escape = autoescape(env._LUPAFILENAME) -- TODO: test
+ end
+ if escape and type(value) == 'string' then
+ value = M.filters.escape(value)
+ end
+ end
+ chunks[#chunks + 1] = value ~= nil and tostring(value) or ''
+ elseif token == 'extends' then
+ env_assert(env, not extends,
+ 'cannot have multiple "extends" in the same scope')
+ local file = eval(block, env) -- covers strings and variables
+ extends = file
+ env._LUPAEXTENDED = true -- used by parent templates
+ elseif token == 'block' then
+ local name = block.expression:match('^[%w_]+$')
+ env_assert(env, name, 'invalid block name')
+ -- Store the block for potential use by the parent template if this
+ -- template is a child template, or for use by `self`.
+ if not env._LUPABLOCKS then env._LUPABLOCKS = {} end
+ if not env._LUPABLOCKS[name] then env._LUPABLOCKS[name] = {} end
+ table.insert(env._LUPABLOCKS[name], 1, block)
+ -- Handle the block properly.
+ if not extends then
+ if not env._LUPAEXTENDED then
+ -- Evaluate the block normally.
+ chunks[#chunks + 1] = evaluate(block, env)
+ else
+ -- A child template is overriding this parent's named block. Evaluate
+ -- the child's block and use it instead of the parent's.
+ local blocks = env._LUPABLOCKS[name]
+ local super_env = setmetatable({super = function()
+ -- Loop through the chain of defined blocks, evaluating from top to
+ -- bottom, and return the bottom block. In each sub-block, the
+ -- 'super' variable needs to point to the next-highest block's
+ -- evaluated result.
+ local super = evaluate(block, env) -- start with parent block
+ local sub_env = setmetatable({super = function() return super end},
+ {__index = env})
+ for i = 1, #blocks - 1 do super = evaluate(blocks[i], sub_env) end
+ return super
+ end}, {__index = env})
+ chunks[#chunks + 1] = evaluate(blocks[#blocks], super_env)
+ end
+ end
+ elseif token == 'for' then
+ local expr = block.expression
+ local p = env._LUPAPOSITION -- mark position at beginning of expression
+ -- Extract variable list and generator.
+ local patt = '^([%w_,%s]+)%s+in%s+()(.+)%s+if%s+(.+)$'
+ local var_list, pos, generator, if_expr = expr:match(patt)
+ if not var_list then
+ var_list, pos, generator = expr:match('^([%w_,%s]+)%s+in%s+()(.+)$')
+ end
+ env_assert(env, var_list and generator, 'invalid for expression')
+ -- Store variable names in a list for loop assignment.
+ local variables = {}
+ for variable, pos in var_list:gmatch('([^,%s]+)()') do
+ env._LUPAPOSITION = p + pos - 1 -- update position for error messages
+ env_assert(env, variable:find('^[%a_]') and variable ~= 'loop',
+ 'invalid variable name')
+ variables[#variables + 1] = variable
+ end
+ -- Evaluate the generator and perform the iteration.
+ env._LUPAPOSITION = p + pos - 1 -- update position to generator
+ if not generator:find('|') then
+ generator = env_assert(env, load('return '..generator, nil, nil, env))
+ else
+ local generator_expr = generator
+ generator = function() return eval(generator_expr, env) end
+ end
+ local new_env = setmetatable({}, {__index = env})
+ chunks[#chunks + 1] = iterate(generator, variables, if_expr, block,
+ new_env, 1, ast[i + 4] == 'lstrip')
+ elseif token == 'if' then
+ if eval(block.expression, env) then
+ chunks[#chunks + 1] = evaluate(block, env)
+ else
+ local evaluate_else = true
+ local elseifs = block['elseif']
+ if elseifs then
+ for j = 1, #elseifs do
+ if eval(elseifs[j].expression, env) then
+ chunks[#chunks + 1] = evaluate(elseifs[j], env)
+ evaluate_else = false
+ break
+ end
+ end
+ end
+ if evaluate_else and block['else'] then
+ chunks[#chunks + 1] = evaluate(block['else'], env)
+ end
+ end
+ elseif token == 'macro' then
+ -- Parse the macro's name and parameter list.
+ local signature = block.expression
+ local name, param_list = signature:match('^([%w_]+)(%b())')
+ env_assert(env, name and param_list, 'invalid macro expression')
+ param_list = param_list:sub(2, -2)
+ local p = env._LUPAPOSITION + #name + 1 -- mark pos at beginning of args
+ local params, defaults = {}, {}
+ for param, pos, default in param_list:gmatch('([%w_]+)=?()([^,]*)') do
+ params[#params + 1] = param
+ if default ~= '' then
+ env._LUPAPOSITION = p + pos - 1 -- update position for error messages
+ local f = env_assert(env, load('return '..default))
+ defaults[param] = select(2, env_assert(env, pcall(f)))
+ end
+ end
+ -- Create the function associated with the macro such that when the
+ -- function is called (from within {{ ... }}), the macro's body is
+ -- evaluated subject to an environment where parameter names are variables
+ -- whose values are the ones passed to the macro itself.
+ env[name] = function(...)
+ local new_env = setmetatable({}, {__index = function(_, k)
+ if k == 'caller' and type(env[k]) ~= 'function' then return nil end
+ return env[k]
+ end})
+ local args = {...}
+ -- Assign the given parameter values.
+ for i = 1, #args do
+ if i > #params then break end
+ new_env[params[i]] = args[i]
+ end
+ -- Clear all other unspecified parameter values or set them to their
+ -- defined defaults.
+ for i = #args + 1, #params do
+ new_env[params[i]] = defaults[params[i]]
+ end
+ -- Store extra parameters in "varargs" variable.
+ new_env.varargs = {}
+ for i = #params + 1, #args do
+ new_env.varargs[#new_env.varargs + 1] = args[i]
+ end
+ local chunk = evaluate(block, new_env)
+ if ast[i + 4] == 'lstrip' then chunk = chunk:gsub('%s*$', '') end
+ return chunk
+ end
+ elseif token == 'call' then
+ -- Parse the call block's parameter list (if any) and determine the macro
+ -- to call.
+ local param_list = block.expression:match('^(%b())')
+ local params = {}
+ if param_list then
+ for param in param_list:gmatch('[%w_]+') do
+ params[#params + 1] = param
+ end
+ end
+ local macro = block.expression:match('^%b()(.+)$') or block.expression
+ -- Evaluate the given macro, subject to a "caller" function that returns
+ -- the contents of this call block. Any arguments passed to the caller
+ -- function are used as values of this parameters parsed earlier.
+ local old_caller = M.env.caller -- save
+ M.env.caller = function(...)
+ local new_env = setmetatable({}, {__index = env})
+ local args = {...}
+ -- Assign the given parameter values (if any).
+ for i = 1, #args do new_env[params[i]] = args[i] end
+ local chunk = evaluate(block, new_env)
+ if ast[i + 4] == 'lstrip' then chunk = chunk:gsub('%s*$', '') end
+ return chunk
+ end
+ chunks[#chunks + 1] = eval(macro, env)
+ M.env.caller = old_caller -- restore
+ elseif token == 'filter' then
+ local text = evaluate(block, env)
+ local p = env._LUPAPOSITION -- mark position at beginning of expression
+ for pos, filter in each_filter(block.expression) do
+ env._LUPAPOSITION = p + pos - 1 -- update position for error messages
+ local name, params = filter:match('^%s*([%w_]+)%(?(.-)%)?%s*$')
+ local f = M.filters[name]
+ env_assert(env, f, 'unknown filter "'..name..'"')
+ local args = env_assert(env, load('return {'..params..'}'),
+ 'invalid filter parameter(s) for "'..name..
+ '"')()
+ text = select(2, env_assert(env, pcall(f, text, table.unpack(args))))
+ end
+ chunks[#chunks + 1] = text
+ elseif token == 'set' then
+ local var, expr = block:match('^([%a_][%w_]*)%s*=%s*(.+)$')
+ env_assert(env, var and expr, 'invalid variable name or expression')
+ env[var] = eval(expr, env)
+ elseif token == 'do' then
+ env_assert(env, pcall(env_assert(env, load(block, nil, nil, env))))
+ elseif token == 'include' then
+ -- Parse the include block for flags.
+ local without_context = block:find('without%s+context%s*')
+ local ignore_missing = block:find('ignore%s+missing%s*')
+ block = block:gsub('witho?u?t?%s+context%s*', '')
+ :gsub('ignore%s+missing%s*', '')
+ -- Evaluate the include expression in order to determine the file to
+ -- include. If the result is a table, use the first file that exists.
+ local file = eval(block, env) -- covers strings and variables
+ if type(file) == 'table' then
+ local files = file
+ for i = 1, #files do
+ file = loader(files[i], env)
+ if file then break end
+ end
+ if type(file) == 'table' then file = nil end
+ elseif type(file) == 'string' then
+ file = loader(file, env)
+ else
+ error('"include" requires a string or table of files')
+ end
+ -- If the file exists, include it. Otherwise throw an error unless the
+ -- "ignore missing" flag was given.
+ env_assert(env, file or ignore_missing, 'no file(s) found to include')
+ if file then
+ chunks[#chunks + 1] = M.expand_file(file, not without_context and env or
+ M.env)
+ end
+ elseif token == 'import' then
+ local file, global = block:match('^%s*(.+)%s+as%s+([%a][%w_]*)%s*')
+ local new_env = setmetatable({}, {
+ __index = block:find('with%s+context%s*$') and env or M.env
+ })
+ M.expand_file(eval(file or block, env), new_env)
+ -- Copy any defined macros and variables over into the proper namespace.
+ if global then env[global] = {} end
+ local namespace = global and env[global] or env
+ for k, v in pairs(new_env) do if not env[k] then namespace[k] = v end end
+ elseif token == 'lstrip' and chunks[#chunks] then
+ chunks[#chunks] = chunks[#chunks]:gsub('%s*$', '')
+ elseif token == 'rstrip' then
+ rstrip = true -- can only strip after determining the next chunk
+ end
+ if rstrip and token ~= 'rstrip' then
+ chunks[#chunks] = chunks[#chunks]:gsub('^%s*', '')
+ rstrip = false
+ end
+ end
+ return not extends and table.concat(chunks) or M.expand_file(extends, env)
+end
+
+local pairs_gen, ipairs_gen = pairs({}), ipairs({})
+
+-- Iterates over the generator *generator* subject to string "if" expression
+-- *if_expr*, assigns that generator's returned values to the variable names
+-- listed in *variables* within template environment *env*, evaluates any
+-- expressions in *block* (a portion of a template's AST), and returns a
+-- concatenation of the results.
+-- @param generator Either a function that returns a generator function, or a
+-- table to iterate over. In the latter case, `ipairs()` is used as the
+-- generator function.
+-- @param variables List of variable names to assign values returned by
+-- *generator* to.
+-- @param if_expr A conditional expression that when `false`, skips the current
+-- loop item.
+-- @param block The portion inside the 'for' structure of a template's AST to
+-- iterate with.
+-- @param env The environment iteration variables are defined in and where
+-- expressions are evaluated in.
+-- @param depth The current recursion depth. Recursion is performed by calling
+-- `loop(t)` with a table to iterate over.
+-- @param lstrip Whether or not the "endfor" block strips whitespace on the
+-- left. When `true`, all blocks produced by iteration are left-stripped.
+iterate = function(generator, variables, if_expr, block, env, depth, lstrip)
+ local chunks = {}
+ local orig_variables = {} -- used to store original loop variables' values
+ for i = 1, #variables do orig_variables[variables[i]] = env[variables[i]] end
+ local i, n = 1 -- used for loop variables
+ local _, s, v -- state variables
+ if type(generator) == 'function' then
+ _, generator, s, v = env_assert(env, pcall(generator))
+ -- In practice, a generator's state variable is normally unused and hidden.
+ -- This is not the case for 'pairs()' and 'ipairs', though.
+ if variables[1] ~= '_index' and generator ~= pairs_gen and
+ generator ~= ipairs_gen then
+ table.insert(variables, 1, '_index')
+ end
+ end
+ if type(generator) == 'table' then
+ n = #generator
+ generator, s, v = ipairs(generator)
+ -- "for x in y" translates to "for _, x in ipairs(y)"; hide _ state variable
+ if variables[1] ~= '_index' then table.insert(variables, 1, '_index') end
+ end
+ if generator then
+ local first_results -- for preventing infinite loop from invalid generator
+ while true do
+ local results = {generator(s, v)}
+ if results[1] == nil then break end
+ -- If the results from the generator look like results returned by a
+ -- generator itself (function, state, initial variable), verify last two
+ -- results are different. If they are the same, then the original
+ -- generator is invalid and will loop infinitely.
+ if first_results == nil then
+ first_results = #results == 3 and type(results[1]) == 'function' and
+ results
+ elseif first_results then
+ env_assert(env, results[3] ~= first_results[3] or
+ results[2] ~= first_results[2],
+ 'invalid generator (infinite loop)')
+ end
+ -- Assign context variables and evaluate the body of the loop.
+ -- As long as the result (ignoring the _index variable) is not a single
+ -- table and there is only one loop variable defined (again, ignoring
+ -- _index variable), assignment occurs as normal in Lua. Otherwise,
+ -- unpacking on the table is done (like assignment to ...).
+ if not (type(results[2]) == 'table' and #results == 2 and
+ #variables > 2) then
+ for j = 1, #variables do env[variables[j]] = results[j] end
+ else
+ for j = 2, #variables do env[variables[j]] = results[2][j - 1] end
+ end
+ if not if_expr or eval(if_expr, env) then
+ env.loop = setmetatable({
+ index = i, index0 = i - 1,
+ revindex = n and n - (i - 1), revindex0 = n and n - i,
+ first = i == 1, last = i == n, length = n,
+ cycle = function(...)
+ return select((i - 1) % select('#', ...) + 1, ...)
+ end,
+ depth = depth, depth0 = depth - 1
+ }, {__call = function(_, t)
+ return iterate(t, variables, if_expr, block, env, depth + 1, lstrip)
+ end})
+ chunks[#chunks + 1] = evaluate(block, env)
+ if lstrip then chunks[#chunks] = chunks[#chunks]:gsub('%s*$', '') end
+ i = i + 1
+ end
+ -- Prepare for next iteration.
+ v = results[1]
+ end
+ end
+ if i == 1 and block['else'] then
+ chunks[#chunks + 1] = evaluate(block['else'], env)
+ end
+ for i = 1, #variables do env[variables[i]] = orig_variables[variables[i]] end
+ return table.concat(chunks)
+end
+
+-- Expands string template *template* from source *source*, subject to template
+-- environment *env*, and returns the result.
+-- @param template String template to expand.
+-- @param env Environment for the given template.
+-- @param source Filename or identifier the template comes from for error
+-- messages and debugging.
+local function expand(template, env, source)
+ template = template:gsub('\r?\n', newline_sequence) -- normalize
+ if not keep_trailing_newline then template = template:gsub('\r?\n$', '') end
+ -- Set up environment.
+ if not env then env = {} end
+ if not getmetatable(env) then env = setmetatable(env, {__index = M.env}) end
+ env.self = setmetatable({}, {__index = function(_, k)
+ env_assert(env, env._LUPABLOCKS and env._LUPABLOCKS[k],
+ 'undefined block "'..k..'"')
+ return function() return evaluate(env._LUPABLOCKS[k][1], env) end
+ end})
+ -- Set context variables and expand the template.
+ env._LUPASOURCE, env._LUPAFILENAME = template, source
+ M._FILENAME = source -- for lpeg errors only
+ local ast = assert(lpeg.match(M.grammar, template), "internal error")
+ local result = evaluate(ast, env)
+ return result
+end
+
+---
+-- Expands the string template *template*, subject to template environment
+-- *env*, and returns the result.
+-- @param template String template to expand.
+-- @param env Optional environment for the given template.
+-- @name expand
+function M.expand(template, env) return expand(template, env, '<string>') end
+
+---
+-- Expands the template within file *filename*, subject to template environment
+-- *env*, and returns the result.
+-- @param filename Filename containing the template to expand.
+-- @param env Optional environment for the template to expand.
+-- @name expand_file
+function M.expand_file(filename, env)
+ filename = loader(filename, env) or filename
+ local f = (not env or not env._LUPASOURCE) and assert(io.open(filename)) or
+ env_assert(env, io.open(filename))
+ local template = f:read('*a')
+ f:close()
+ return expand(template, env, filename)
+end
+
+---
+-- Returns a loader for templates that uses the filesystem starting at directory
+-- *directory*.
+-- When looking up the template for a given filename, the loader considers the
+-- following: if no template is being expanded, the loader assumes the given
+-- filename is relative to *directory* and returns the full path; otherwise the
+-- loader assumes the given filename is relative to the current template's
+-- directory and returns the full path.
+-- The returned path may be passed to `io.open()`.
+-- @param directory Optional the template root directory. The default value is
+-- ".", which is the current working directory.
+-- @name loaders.filesystem
+-- @see configure
+function M.loaders.filesystem(directory)
+ return function(filename, env)
+ if not filename then return nil end
+ local current_dir = env and env._LUPAFILENAME and
+ env._LUPAFILENAME:match('^(.+)[/\\]')
+ if not filename:find('^/') and not filename:find('^%a:[/\\]') then
+ filename = (current_dir or directory or '.')..'/'..filename
+ end
+ local f = io.open(filename)
+ if not f then return nil end
+ f:close()
+ return filename
+ end
+end
+
+-- Globally defined functions.
+
+---
+-- Returns a sequence of integers from *start* to *stop*, inclusive, in
+-- increments of *step*.
+-- The complete sequence is generated at once -- no generator is returned.
+-- @param start Optional number to start at. The default value is `1`.
+-- @param stop Number to stop at.
+-- @param step Optional increment between sequence elements. The default value
+-- is `1`.
+-- @name _G.range
+function range(start, stop, step)
+ if not stop and not step then stop, start = start, 1 end
+ if not step then step = 1 end
+ local t = {}
+ for i = start, stop, step do t[#t + 1] = i end
+ return t
+end
+
+---
+-- Returns an object that cycles through the given values by calls to its
+-- `next()` function.
+-- A `current` field contains the cycler's current value and a `reset()`
+-- function resets the cycler to its beginning.
+-- @param ... Values to cycle through.
+-- @usage c = cycler(1, 2, 3)
+-- @usage c:next(), c:next() --> 1, 2
+-- @usage c:reset() --> c.current == 1
+-- @name _G.cycler
+function cycler(...)
+ local c = {...}
+ c.n, c.i, c.current = #c, 1, c[1]
+ function c:next()
+ local current = self.current
+ self.i = self.i + 1
+ if self.i > self.n then self.i = 1 end
+ self.current = self[self.i]
+ return current
+ end
+ function c:reset() self.i, self.current = 1, self[1] end
+ return c
+end
+
+-- Create the default sandbox environment for templates.
+local safe = {
+ -- Lua globals.
+ '_VERSION', 'ipairs', 'math', 'pairs', 'select', 'tonumber', 'tostring',
+ 'type', 'bit32', 'os.date', 'os.time', 'string', 'table', 'utf8',
+ -- Lupa globals.
+ 'range', 'cycler'
+}
+local sandbox_env = setmetatable({}, {__index = M.tests})
+for i = 1, #safe do
+ local v = safe[i]
+ if not v:find('%.') then
+ sandbox_env[v] = _G[v]
+ else
+ local mod, func = v:match('^([^.]+)%.(.+)$')
+ if not sandbox_env[mod] then sandbox_env[mod] = {} end
+ sandbox_env[mod][func] = _G[mod][func]
+ end
+end
+sandbox_env._G = sandbox_env
+
+---
+-- Resets Lupa's default delimiters, options, and environments to their
+-- original default values.
+-- @name reset
+function M.reset()
+ M.configure('{%', '%}', '{{', '}}', '{#', '#}')
+ M.env = setmetatable({}, {__index = sandbox_env})
+end
+M.reset()
+
+---
+-- The default template environment.
+-- @class table
+-- @name env
+local env
+
+-- Lupa filters.
+
+---
+-- Returns the absolute value of number *n*.
+-- @param n The number to compute the absolute value of.
+-- @name filters.abs
+M.filters.abs = math.abs
+
+-- Returns a table that, when indexed with an integer, indexes table *t* with
+-- that integer along with string *attribute*.
+-- This is used by filters that operate on particular attributes of table
+-- elements.
+-- @param t The table to index.
+-- @param attribute The additional attribute to index with.
+local function attr_accessor(t, attribute)
+ return setmetatable({}, {__index = function(_, i)
+ local value = t[i]
+ attribute = tonumber(attribute) or attribute
+ if type(attribute) == 'number' then return value[attribute] end
+ for k in attribute:gmatch('[^.]+') do value = value[k] end
+ return value
+ end})
+end
+
+---
+-- Returns a generator that produces all of the items in table *t* in batches
+-- of size *size*, filling any empty spaces with value *fill*.
+-- Combine this with the "list" filter to produce a list.
+-- @param t The table to split into batches.
+-- @param size The batch size.
+-- @param fill The value to use when filling in any empty space in the last
+-- batch.
+-- @usage expand('{% for i in {1, 2, 3}|batch(2, 0) %}{{ i|string }}
+-- {% endfor %}') --> {1, 2} {3, 0}
+-- @see filters.list
+-- @name filters.batch
+function M.filters.batch(t, size, fill)
+ assert(t, 'input to filter "batch" was nil instead of a table')
+ local n = #t
+ return function(t, i)
+ if i > n then return nil end
+ local batch = {}
+ for j = i, i + size - 1 do batch[j - i + 1] = t[j] end
+ if i + size > n and fill then
+ for j = n + 1, i + size - 1 do batch[#batch + 1] = fill end
+ end
+ return i + size, batch
+ end, t, 1
+end
+
+---
+-- Capitalizes string *s*.
+-- The first character will be uppercased, the others lowercased.
+-- @param s The string to capitalize.
+-- @usage expand('{{ "foo bar"|capitalize }}') --> Foo bar
+-- @name filters.capitalize
+function M.filters.capitalize(s)
+ assert(s, 'input to filter "capitalize" was nil instead of a string')
+ local first, rest = s:match('^(.)(.*)$')
+ return first and first:upper()..rest:lower() or s
+end
+
+---
+-- Centers string *s* within a string of length *width*.
+-- @param s The string to center.
+-- @param width The length of the centered string.
+-- @usage expand('{{ "foo"|center(9) }}') --> " foo "
+-- @name filters.center
+function M.filters.center(s, width)
+ assert(s, 'input to filter "center" was nil instead of a string')
+ local padding = (width or 80) - #s
+ local left, right = math.ceil(padding / 2), math.floor(padding / 2)
+ return ("%s%s%s"):format((' '):rep(left), s, (' '):rep(right))
+end
+
+---
+-- Returns value *value* or value *default*, depending on whether or not *value*
+-- is "true" and whether or not boolean *false_defaults* is `true`.
+-- @param value The value return if "true" or if `false` and *false_defaults*
+-- is `true`.
+-- @param default The value to return if *value* is `nil` or `false` (the latter
+-- applies only if *false_defaults* is `true`).
+-- @param false_defaults Optional flag indicating whether or not to return
+-- *default* if *value* is `false`. The default value is `false`.
+-- @usage expand('{{ false|default("no") }}') --> false
+-- @usage expand('{{ false|default("no", true) }') --> no
+-- @name filters.default
+function M.filters.default(value, default, false_defaults)
+ if value == nil or false_defaults and not value then return default end
+ return value
+end
+
+---
+-- Returns a table constructed from table *t* such that each element is a list
+-- that contains a single key-value pair and all elements are sorted according
+-- to string *by* (which is either "key" or "value") and boolean
+-- *case_sensitive*.
+-- @param value The table to sort.
+-- @param case_sensitive Optional flag indicating whether or not to consider
+-- case when sorting string values. The default value is `false`.
+-- @param by Optional string that specifies which of the key-value to sort by,
+-- either "key" or "value". The default value is `"key"`.
+-- @usage expand('{{ {b = 1, a = 2}|dictsort|string }}') --> {{"a", 2},
+-- {"b", 1}}
+-- @name filters.dictsort
+function M.filters.dictsort(t, case_sensitive, by)
+ assert(t, 'input to filter "dictsort" was nil instead of a table')
+ assert(not by or by == 'key' or by == 'value',
+ 'filter "dictsort" can only sort tables by "key" or "value"')
+ local i = (not by or by == 'key') and 1 or 2
+ local items = {}
+ for k, v in pairs(t) do items[#items + 1] = {k, v} end
+ table.sort(items, function(a, b)
+ a, b = a[i], b[i]
+ if not case_sensitive then
+ if type(a) == 'string' then a = a:lower() end
+ if type(b) == 'string' then b = b:lower() end
+ end
+ return a < b
+ end)
+ return items
+end
+
+---
+-- Returns an HTML-safe copy of string *s*.
+-- @param s String to ensure is HTML-safe.
+-- @usage expand([[{{ '<">&'|e}}]]) --> &lt;&#34;&gt;&amp;
+-- @name filters.escape
+function M.filters.escape(s)
+ assert(s, 'input to filter "escape" was nil instead of a string')
+ return s:gsub('[<>"\'&]', {
+ ['<'] = '&lt;', ['>'] = '&gt;', ['"'] = '&#34;', ["'"] = '&#39;',
+ ['&'] = '&amp;'
+ })
+end
+
+---
+-- Returns an HTML-safe copy of string *s*.
+-- @param s String to ensure is HTML-safe.
+-- @usage expand([[{{ '<">&'|escape}}]]) --> &lt;&#34;&gt;&amp;
+-- @name filters.e
+function M.filters.e(s)
+ assert(s, 'input to filter "e" was nil instead of a string')
+ return M.filters.escape(s)
+end
+
+---
+-- Returns a human-readable, decimal (or binary, depending on boolean *binary*)
+-- file size for *bytes* number of bytes.
+-- @param bytes The number of bytes to return the size for.
+-- @param binary Flag indicating whether or not to report binary file size
+-- as opposed to decimal file size. The default value is `false`.
+-- @usage expand('{{ 1000|filesizeformat }}') --> 1.0 kB
+-- @name filters.filesizeformat
+function M.filters.filesizeformat(bytes, binary)
+ assert(bytes, 'input to filter "filesizeformat" was nil instead of a number')
+ local base = binary and 1024 or 1000
+ local units = {
+ binary and 'KiB' or 'kB', binary and 'MiB' or 'MB',
+ binary and 'GiB' or 'GB', binary and 'TiB' or 'TB',
+ binary and 'PiB' or 'PB', binary and 'EiB' or 'EB',
+ binary and 'ZiB' or 'ZB', binary and 'YiB' or 'YB'
+ }
+ if bytes < base then
+ return string.format('%d Byte%s', bytes, bytes > 1 and 's' or '')
+ else
+ local limit, unit
+ for i = 1, #units do
+ limit, unit = base^(i + 1), units[i]
+ if bytes < limit then break end
+ end
+ return string.format('%.1f %s', (base * bytes / limit), unit)
+ end
+end
+
+---
+-- Returns the first element in table *t*.
+-- @param t The table to get the first element of.
+-- @usage expand('{{ range(10)|first }}') --> 1
+-- @name filters.first
+function M.filters.first(t)
+ assert(t, 'input to filter "first" was nil instead of a table')
+ return t[1]
+end
+
+---
+-- Returns value *value* as a float.
+-- This filter only works in Lua 5.3, which has a distinction between floats and
+-- integers.
+-- @param value The value to interpret as a float.
+-- @usage expand('{{ 42|float }}') --> 42.0
+-- @name filters.float
+function M.filters.float(value)
+ assert(value, 'input to filter "float" was nil instead of a number')
+ return (tonumber(value) or 0) * 1.0
+end
+
+---
+-- Returns an HTML-safe copy of value *value*, even if *value* was returned by
+-- the "safe" filter.
+-- @param value Value to ensure is HTML-safe.
+-- @usage expand('{% set x = "<div />"|safe %}{{ x|forceescape }}') -->
+-- &lt;div /&gt;
+-- @name filters.forceescape
+function M.filters.forceescape(value)
+ assert(value, 'input to filter "forceescape" was nil instead of a string')
+ return M.filters.escape(tostring(value))
+end
+
+---
+-- Returns the given arguments formatted according to string *s*.
+-- See Lua's `string.format()` for more information.
+-- @param s The string to format subsequent arguments according to.
+-- @param ... Arguments to format.
+-- @usage expand('{{ "%s,%s"|format("a", "b") }}') --> a,b
+-- @name filters.format
+function M.filters.format(s, ...)
+ assert(s, 'input to filter "format" was nil instead of a string')
+ return string.format(s, ...)
+end
+
+---
+-- Returns a generator that produces lists of items in table *t* grouped by
+-- string attribute *attribute*.
+-- @param t The table to group items from.
+-- @param attribute The attribute of items in the table to group by. This may
+-- be nested (e.g. "foo.bar" groups by t[i].foo.bar for all i).
+-- @usage expand('{% for age, group in people|groupby("age") %}...{% endfor %}')
+-- @name filters.groupby
+function M.filters.groupby(t, attribute)
+ assert(t, 'input to filter "groupby" was nil instead of a table')
+ local n = #t
+ local seen = {} -- keep track of groupers in order to avoid duplicates
+ return function(t, i)
+ if i > n then return nil end
+ local ta = attr_accessor(t, attribute)
+ -- Determine the next grouper.
+ local grouper = ta[i]
+ while seen[grouper] do
+ i = i + 1
+ if i > n then return nil end
+ grouper = ta[i]
+ end
+ seen[grouper] = true
+ -- Create and return the group.
+ local group = {}
+ for j = i, #t do if ta[j] == grouper then group[#group + 1] = t[j] end end
+ return i + 1, grouper, group
+ end, t, 1
+end
+
+---
+-- Returns a copy of string *s* with all lines after the first indented by
+-- *width* number of spaces.
+-- If boolean *first_line* is `true`, indents the first line as well.
+-- @param s The string to indent lines of.
+-- @param width The number of spaces to indent lines with.
+-- @param first_line Optional flag indicating whether or not to indent the
+-- first line of text. The default value is `false`.
+-- @usage expand('{{ "foo\nbar"|indent(2) }}') --> "foo\n bar"
+-- @name filters.indent
+function M.filters.indent(s, width, first_line)
+ assert(s, 'input to filter "indent" was nil instead of a string')
+ local indent = (' '):rep(width)
+ return (first_line and indent or '')..s:gsub('([\r\n]+)', '%1'..indent)
+end
+
+---
+-- Returns value *value* as an integer.
+-- @param value The value to interpret as an integer.
+-- @usage expand('{{ 32.32|int }}') --> 32
+-- @name filters.int
+function M.filters.int(value)
+ assert(value, 'input to filter "int" was nil instead of a number')
+ return math.floor(tonumber(value) or 0)
+end
+
+---
+-- Returns a string that contains all the elements in table *t* (or all the
+-- attributes named *attribute* in *t*) separated by string *sep*.
+-- @param t The table to join.
+-- @param sep The string to separate table elements with.
+-- @param attribute Optional attribute of elements to use for joining instead
+-- of the elements themselves. This may be nested (e.g. "foo.bar" joins
+-- `t[i].foo.bar` for all i).
+-- @usage expand('{{ {1, 2, 3}|join("|") }}') --> 1|2|3
+-- @name filters.join
+function M.filters.join(t, sep, attribute)
+ assert(t, 'input to filter "join" was nil instead of a table')
+ if not attribute then
+ local strings = {}
+ for i = 1, #t do strings[#strings + 1] = tostring(t[i]) end
+ return table.concat(strings, sep)
+ end
+ local ta = attr_accessor(t, attribute)
+ local attributes = {}
+ for i = 1, #t do attributes[#attributes + 1] = ta[i] end
+ return table.concat(attributes, sep)
+end
+
+---
+-- Returns the last element in table *t*.
+-- @param t The table to get the last element of.
+-- @usage expand('{{ range(10)|last }}') --> 10
+-- @name filters.last
+function M.filters.last(t)
+ assert(t, 'input to filter "last" was nil instead of a table')
+ return t[#t]
+end
+
+---
+-- Returns the length of string or table *value*.
+-- @param value The value to get the length of.
+-- @usage expand('{{ "hello world"|length }}') --> 11
+-- @name filters.length
+function M.filters.length(value)
+ assert(value, 'input to filter "length" was nil instead of a table or string')
+ return #value
+end
+
+---
+-- Returns the list of items produced by generator *generator*, subject to
+-- initial state *s* and initial iterator variable *i*.
+-- This filter should only be used after a filter that returns a generator.
+-- @param generator Generator function that produces an item.
+-- @param s Initial state for the generator.
+-- @param i Initial iterator variable for the generator.
+-- @usage expand('{{ range(4)|batch(2)|list|string }}') --> {{1, 2}, {3, 4}}
+-- @see filters.batch
+-- @see filters.groupby
+-- @see filters.slice
+-- @name filters.list
+function M.filters.list(generator, s, i)
+ assert(type(generator) == 'function',
+ 'input to filter "list" must be a generator')
+ local list = {}
+ for _, v in generator, s, i do list[#list + 1] = v end
+ return list
+end
+
+---
+-- Returns a copy of string *s* with all lowercase characters.
+-- @param s The string to lowercase.
+-- @usage expand('{{ "FOO"|lower }}') --> foo
+-- @name filters.lower
+function M.filters.lower(s)
+ assert(s, 'input to filter "lower" was nil instead of a string')
+ return string.lower(s)
+end
+
+---
+-- Maps each element of table *t* to a value produced by filter name *filter*
+-- and returns the resultant table.
+-- @param t The table of elements to map.
+-- @param filter The name of the filter to pass table elements through.
+-- @param ... Any arguments for the filter.
+-- @usage expand('{{ {"1", "2", "3"}|map("int")|sum }}') --> 6
+-- @name filters.map
+function M.filters.map(t, filter, ...)
+ assert(t, 'input to filter "map" was nil instead of a table')
+ local f = M.filters[filter]
+ assert(f, 'unknown filter "'..filter..'"')
+ local map = {}
+ for i = 1, #t do map[i] = f(t[i], ...) end
+ return map
+end
+
+---
+-- Maps the value of each element's string *attribute* in table *t* to the
+-- value produced by filter name *filter* and returns the resultant table.
+-- @param t The table of elements with attributes to map.
+-- @param attribute The attribute of elements in the table to filter. This may
+-- be nested (e.g. "foo.bar" maps t[i].foo.bar for all i).
+-- @param filter The name of the filter to pass table elements through.
+-- @param ... Any arguments for the filter.
+-- @usage expand('{{ users|mapattr("name")|join("|") }}')
+-- @name filters.mapattr
+function M.filters.mapattr(t, attribute, filter, ...)
+ assert(t, 'input to filter "mapattr" was nil instead of a table')
+ local ta = attr_accessor(t, attribute)
+ local f = M.filters[filter]
+ if filter then
+ assert(f, 'unknown filter "'..filter..'" given to filter "mapattr"')
+ end
+ local map = {}
+ for i = 1, #t do map[i] = filter and f(ta[i], ...) or ta[i] end
+ return map
+end
+
+---
+-- Returns a random element from table *t*.
+-- @param t The table to get a random element from.
+-- @usage expand('{{ range(100)|random }}')
+-- @name filters.random
+function M.filters.random(t)
+ assert(t, 'input to filter "random" was nil instead of a table')
+ math.randomseed(os.time())
+ return t[math.random(#t)]
+end
+
+---
+-- Returns a list of elements in table *t* that fail test name *test*.
+-- @param t The table of elements to reject from.
+-- @param test The name of the test to use on table elements.
+-- @param ... Any arguments for the test.
+-- @usage expand('{{ range(5)|reject(is_odd)|join("|") }}') --> 2|4
+-- @name filters.reject
+function M.filters.reject(t, test, ...)
+ assert(t, 'input to filter "reject" was nil instead of a table')
+ local f = test or function(value) return not not value end
+ local items = {}
+ for i = 1, #t do if not f(t[i], ...) then items[#items + 1] = t[i] end end
+ return items
+end
+
+---
+-- Returns a list of elements in table *t* whose string attribute *attribute*
+-- fails test name *test*.
+-- @param t The table of elements to reject from.
+-- @param attribute The attribute of items in the table to reject from. This
+-- may be nested (e.g. "foo.bar" tests t[i].foo.bar for all i).
+-- @param test The name of the test to use on table elements.
+-- @param ... Any arguments for the test.
+-- @usage expand('{{ users|rejectattr("offline")|mapattr("name")|join(",") }}')
+-- @name filters.rejectattr
+function M.filters.rejectattr(t, attribute, test, ...)
+ assert(t, 'input to filter "rejectattr" was nil instead of a table')
+ local ta = attr_accessor(t, attribute)
+ local f = test or function(value) return not not value end
+ local items = {}
+ for i = 1, #t do if not f(ta[i], ...) then items[#items + 1] = t[i] end end
+ return items
+end
+
+---
+-- Returns a copy of string *s* with all (or up to *n*) occurrences of string
+-- *old* replaced by string *new*.
+-- Identical to Lua's `string.gsub()` and handles Lua patterns.
+-- @param s The subject string.
+-- @param pattern The string or Lua pattern to replace.
+-- @param repl The replacement text (may contain Lua captures).
+-- @param n Optional number indicating the maximum number of replacements to
+-- make. The default value is `nil`, which is unlimited.
+-- @usage expand('{% filter upper|replace("FOO", "foo") %}foobar
+-- {% endfilter %}') --> fooBAR
+-- @name filters.replace
+function M.filters.replace(s, pattern, repl, n)
+ assert(s, 'input to filter "replace" was nil instead of a string')
+ return string.gsub(s, pattern, repl, n)
+end
+
+---
+-- Returns a copy of the given string or table *value* in reverse order.
+-- @param value The value to reverse.
+-- @usage expand('{{ {1, 2, 3}|reverse|string }}') --> {3, 2, 1}
+-- @name filters.reverse
+function M.filters.reverse(value)
+ assert(type(value) == 'table' or type(value) == 'string',
+ 'input to filter "reverse" was nil instead of a table or string')
+ if type(value) == 'string' then return value:reverse() end
+ local t = {}
+ for i = 1, #value do t[i] = value[#value - i + 1] end
+ return t
+end
+
+---
+-- Returns number *value* rounded to *precision* decimal places based on string
+-- *method* (if given).
+-- @param value The number to round.
+-- @param precision Optional precision to round the number to. The default
+-- value is `0`.
+-- @param method Optional string rounding method, either `"ceil"` or
+-- `"floor"`. The default value is `nil`, which uses the common rounding
+-- method (if a number's fractional part is 0.5 or greater, rounds up;
+-- otherwise rounds down).
+-- @usage expand('{{ 2.1236|round(3, "floor") }}') --> 2.123
+-- @name filters.round
+function M.filters.round(value, precision, method)
+ assert(value, 'input to filter "round" was nil instead of a number')
+ assert(not method or method == 'ceil' or method == 'floor',
+ 'rounding method given to filter "round" must be "ceil" or "floor"')
+ precision = precision or 0
+ method = method or (select(2, math.modf(value)) >= 0.5 and 'ceil' or 'floor')
+ local s = string.format('%.'..(precision >= 0 and precision or 0)..'f',
+ math[method](value * 10^precision) / 10^precision)
+ return tonumber(s)
+end
+
+---
+-- Marks string *s* as HTML-safe, preventing Lupa from modifying it when
+-- configured to autoescape HTML entities.
+-- This filter must be used at the end of a filter chain unless it is
+-- immediately proceeded by the "forceescape" filter.
+-- @param s The string to mark as HTML-safe.
+-- @usage lupa.configure{autoescape = true}
+-- @usage expand('{{ "<div>foo</div>"|safe }}') --> <div>foo</div>
+-- @name filters.safe
+function M.filters.safe(s)
+ assert(s, 'input to filter "safe" was nil instead of a string')
+ return setmetatable({}, {__tostring = function() return s end})
+end
+
+---
+-- Returns a list of the elements in table *t* that pass test name *test*.
+-- @param t The table of elements to select from.
+-- @param test The name of the test to use on table elements.
+-- @param ... Any arguments for the test.
+-- @usage expand('{{ range(5)|select(is_odd)|join("|") }}') --> 1|3|5
+-- @name filters.select
+function M.filters.select(t, test, ...)
+ assert(t, 'input to filter "select" was nil instead of a table')
+ local f = test or function(value) return not not value end
+ local items = {}
+ for i = 1, #t do if f(t[i], ...) then items[#items + 1] = t[i] end end
+ return items
+end
+
+---
+-- Returns a list of elements in table *t* whose string attribute *attribute*
+-- passes test name *test*.
+-- @param t The table of elements to select from.
+-- @param attribute The attribute of items in the table to select from. This
+-- may be nested (e.g. "foo.bar" tests t[i].foo.bar for all i).
+-- @param test The name of the test to use on table elements.
+-- @param ... Any arguments for the test.
+-- @usage expand('{{ users|selectattr("online")|mapattr("name")|join("|") }}')
+-- @name filters.selectattr
+function M.filters.selectattr(t, attribute, test, ...)
+ assert(t, 'input to filter "selectattr" was nil instead of a table')
+ local ta = attr_accessor(t, attribute)
+ local f = test or function(value) return not not value end
+ local items = {}
+ for i = 1, #t do if f(ta[i], ...) then items[#items + 1] = t[i] end end
+ return items
+end
+
+---
+-- Returns a generator that produces all of the items in table *t* in *slices*
+-- number of iterations, filling any empty spaces with value *fill*.
+-- Combine this with the "list" filter to produce a list.
+-- @param t The table to slice.
+-- @param slices The number of slices to produce.
+-- @param fill The value to use when filling in any empty space in the last
+-- slice.
+-- @usage expand('{% for i in {1, 2, 3}|slice(2, 0) %}{{ i|string }}
+-- {% endfor %}') --> {1, 2} {3, 0}
+-- @see filters.list
+-- @name filters.slice
+function M.filters.slice(t, slices, fill)
+ assert(t, 'input to filter "slice" was nil instead of a table')
+ local size, slices_with_extra = math.floor(#t / slices), #t % slices
+ return function(t, i)
+ if i > slices then return nil end
+ local slice = {}
+ local s = (i - 1) * size + math.min(i, slices_with_extra + 1)
+ local e = i * size + math.min(i, slices_with_extra)
+ for j = s, e do slice[j - s + 1] = t[j] end
+ if slices_with_extra > 0 and i > slices_with_extra and fill then
+ slice[#slice + 1] = fill
+ end
+ return i + 1, slice
+ end, t, 1
+end
+
+---
+-- Returns a copy of table or string *value* in sorted order by value (or by
+-- an attribute named *attribute*), depending on booleans *reverse* and
+-- *case_sensitive*.
+-- @param value The table or string to sort.
+-- @param reverse Optional flag indicating whether or not to sort in reverse
+-- (descending) order. The default value is `false`, which sorts in ascending
+-- order.
+-- @param case_sensitive Optional flag indicating whether or not to consider
+-- case when sorting string values. The default value is `false`.
+-- @param attribute Optional attribute of elements to sort by instead of the
+-- elements themselves.
+-- @usage expand('{{ {2, 3, 1}|sort|string }}') --> {1, 2, 3}
+-- @name filters.sort
+function M.filters.sort(value, reverse, case_sensitive, attribute)
+ assert(value, 'input to filter "sort" was nil instead of a table or string')
+ assert(not attribute or type(attribute) == 'string' or
+ type(attribute) == 'number',
+ 'attribute to filter "sort" must be a string or number')
+ local t = {}
+ local sort_string = type(value) == 'string'
+ if not sort_string then
+ for i = 1, #value do t[#t + 1] = value[i] end
+ else
+ for char in value:gmatch('.') do t[#t + 1] = char end -- chars in string
+ end
+ table.sort(t, function(a, b)
+ if attribute then
+ if type(attribute) == 'number' then
+ a, b = a[attribute], b[attribute]
+ else
+ for k in attribute:gmatch('[^.]+') do a, b = a[k], b[k] end
+ end
+ end
+ if not case_sensitive then
+ if type(a) == 'string' then a = a:lower() end
+ if type(b) == 'string' then b = b:lower() end
+ end
+ if not reverse then
+ return a < b
+ else
+ return a > b
+ end
+ end)
+ return not sort_string and t or table.concat(t)
+end
+
+---
+-- Returns the string representation of value *value*, handling lists properly.
+-- @param value Value to return the string representation of.
+-- @usage expand('{{ {1 * 1, 2 * 2, 3 * 3}|string }}') --> {1, 4, 9}
+-- @name filters.string
+function M.filters.string(value)
+ if type(value) ~= 'table' then return tostring(value) end
+ local t = {}
+ for i = 1, #value do
+ local item = value[i]
+ t[i] = type(item) == 'string' and '"'..item..'"' or M.filters.string(item)
+ end
+ return '{'..table.concat(t, ', ')..'}'
+end
+
+---
+-- Returns a copy of string *s* with any HTML tags stripped.
+-- Also cleans up whitespace.
+-- @param s String to strip HTML tags from.
+-- @usage expand('{{ "<div>foo</div>"|striptags }}') --> foo
+-- @name filters.striptags
+function M.filters.striptags(s)
+ assert(s, 'input to filter "striptags" was nil instead of a string')
+ return s:gsub('%b<>', ''):gsub('%s+', ' '):match('^%s*(.-)%s*$')
+end
+
+---
+-- Returns the numeric sum of the elements in table *t* or the sum of all
+-- attributes named *attribute* in *t*.
+-- @param t The table to calculate the sum of.
+-- @param attribute Optional attribute of elements to use for summing instead
+-- of the elements themselves. This may be nested (e.g. "foo.bar" sums
+-- `t[i].foo.bar` for all i).
+-- @usage expand('{{ range(6)|sum }}') --> 21
+-- @name filters.sum
+function M.filters.sum(t, attribute)
+ assert(t, 'input to filter "sum" was nil instead of a table')
+ local ta = attribute and attr_accessor(t, attribute) or t
+ local sum = 0
+ for i = 1, #t do sum = sum + ta[i] end
+ return sum
+end
+
+---
+-- Returns a copy of all words in string *s* in titlecase.
+-- @param s The string to titlecase.
+-- @usage expand('{{ "foo bar"|title }}') --> Foo Bar
+-- @name filters.title
+function M.filters.title(s)
+ assert(s, 'input to filter "title" was nil instead of a string')
+ return s:gsub('[^-%s]+', M.filters.capitalize)
+end
+
+---
+-- Returns a copy of string *s* truncated to *length* number of characters.
+-- Truncated strings end with '...' or string *delimiter*. If boolean
+-- *partial_words* is `false`, truncation will only happen at word boundaries.
+-- @param s The string to truncate.
+-- @param length The length to truncate the string to.
+-- @param partial_words Optional flag indicating whether or not to allow
+-- truncation within word boundaries. The default value is `false`.
+-- @param delimiter Optional delimiter text. The default value is '...'.
+-- @usage expand('{{ "foo bar"|truncate(4) }}') --> "foo ..."
+-- @name filters.truncate
+function M.filters.truncate(s, length, partial_words, delimiter)
+ assert(s, 'input to filter "truncate" was nil instead of a string')
+ if #s <= length then return s end
+ local truncated = s:sub(1, length)
+ if s:find('[%w_]', length) and not partial_words then
+ truncated = truncated:match('^(.-)[%w_]*$') -- drop partial word
+ end
+ return truncated..(delimiter or '...')
+end
+
+---
+-- Returns a copy of string *s* with all uppercase characters.
+-- @param s The string to uppercase.
+-- @usage expand('{{ "foo"|upper }}') --> FOO
+-- @name filters.upper
+function M.filters.upper(s)
+ assert(s, 'input to filter "upper" was nil instead of a string')
+ return string.upper(s)
+end
+
+---
+-- Returns a string suitably encoded to be used in a URL from value *value*.
+-- *value* may be a string, table of key-value query parameters, or table of
+-- lists of key-value query parameters (for order).
+-- @param value Value to URL-encode.
+-- @usage expand('{{ {{'f', 1}, {'z', 2}}|urlencode }}') --> f=1&z=2
+-- @name filters.urlencode
+function M.filters.urlencode(value)
+ assert(value,
+ 'input to filter "urlencode" was nil instead of a string or table')
+ if type(value) ~= 'table' then
+ return tostring(value):gsub('[^%w.-]', function(c)
+ return string.format('%%%X', string.byte(c))
+ end)
+ end
+ local params = {}
+ if #value > 0 then
+ for i = 1, #value do
+ local k = M.filters.urlencode(value[i][1])
+ local v = M.filters.urlencode(value[i][2])
+ params[#params + 1] = k..'='..v
+ end
+ else
+ for k, v in pairs(value) do
+ params[#params + 1] = M.filters.urlencode(k)..'='..M.filters.urlencode(v)
+ end
+ end
+ return table.concat(params, '&')
+end
+
+---
+-- Replaces any URLs in string *s* with HTML links, limiting link text to
+-- *length* characters.
+-- @param s The string to replace URLs with HTML links in.
+-- @param length Optional maximum number of characters to include in link text.
+-- The default value is `nil`, which imposes no limit.
+-- @param nofollow Optional flag indicating whether or not HTML links will get a
+-- "nofollow" attribute.
+-- @usage expand('{{ "example.com"|urlize }}') -->
+-- <a href="http://example.com">example.com</a>
+-- @name filters.urlize
+function M.filters.urlize(s, length, nofollow)
+ assert(s, 'input to filter "urlize" was nil instead of a string')
+ -- Trims the given url.
+ local function trim_url(url)
+ return length and s:sub(1, length)..(#s > length and '...' or '') or url
+ end
+ local nofollow_attr = nofollow and ' rel="nofollow"' or ''
+ local lead, trail = C((S('(<') + '&lt;')^0), C((S('.,)>\n') + '&gt;')^0) * -1
+ local middle = C((1 - trail)^0)
+ local patt = lpeg.Cs(lead * middle * trail / function(lead, middle, trail)
+ local linked
+ if middle:find('^www%.') or (not middle:find('@') and
+ not middle:find('^https?://') and
+ #middle > 0 and middle:find('^%w') and (
+ middle:find('%.com$') or
+ middle:find('%.net$') or
+ middle:find('%.org$')
+ )) then
+ middle, linked = string.format('<a href="http://%s"%s>%s</a>', middle,
+ nofollow_attr, trim_url(middle)), true
+ end
+ if middle:find('^https?://') then
+ middle, linked = string.format('<a href="%s"%s>%s</a>', middle,
+ nofollow_attr, trim_url(middle)), true
+ end
+ if middle:find('@') and not middle:find('^www%.') and
+ not middle:find(':') and middle:find('^%S+@[%w._-]+%.[%w._-]+$') then
+ middle, linked = string.format('<a href="mailto:%s">%s</a>', middle,
+ middle), true
+ end
+ if linked then return lead..middle..trail end
+ end)
+ return M.filters.escape(s):gsub('%S+', function(word)
+ return lpeg.match(patt, word)
+ end)
+end
+
+---
+-- Returns the number of words in string *s*.
+-- A word is a sequence of non-space characters.
+-- @param s The string to count words in.
+-- @usage expand('{{ "foo bar baz"|wordcount }}') --> 3
+-- @name filters.wordcount
+function M.filters.wordcount(s)
+ assert(s, 'input to filter "wordcount" was nil instead of a string')
+ return select(2, s:gsub('%S+', ''))
+end
+
+---
+-- Interprets table *t* as a list of XML attribute-value pairs, returning them
+-- as a properly formatted, space-separated string.
+-- @param t The table of XML attribute-value pairs.
+-- @usage expand('<data {{ {foo = 42, bar = 23}|xmlattr }} />')
+-- @name filters.xmlattr
+function M.filters.xmlattr(t)
+ assert(t, 'input to filter "xmlattr" was nil instead of a table')
+ local attributes = {}
+ for k, v in pairs(t) do
+ attributes[#attributes + 1] = string.format('%s="%s"', k,
+ M.filters.escape(tostring(v)))
+ end
+ return table.concat(attributes, ' ')
+end
+
+-- Lupa tests.
+
+---
+-- Returns whether or not number *n* is odd.
+-- @param n The number to test.
+-- @usage expand('{% for x in range(10) if is_odd(x) %}...{% endif %}')
+-- @name tests.is_odd
+function M.tests.is_odd(n) return n % 2 == 1 end
+
+---
+-- Returns whether or not number *n* is even.
+-- @param n The number to test.
+-- @usage expand('{% for x in range(10) if is_even(x) %}...{% endif %}')
+-- @name tests.is_even
+function M.tests.is_even(n) return n % 2 == 0 end
+
+---
+-- Returns whether or not number *n* is evenly divisible by number *num*.
+-- @param n The dividend to test.
+-- @param num The divisor to use.
+-- @usage expand('{% if is_divisibleby(x, y) %}...{% endif %}')
+-- @name tests.is_divisibleby
+function M.tests.is_divisibleby(n, num) return n % num == 0 end
+
+---
+-- Returns whether or not value *value* is non-nil, and thus defined.
+-- @param value The value to test.
+-- @usage expand('{% if is_defined(x) %}...{% endif %}')
+-- @name tests.is_defined
+function M.tests.is_defined(value) return value ~= nil end
+
+---
+-- Returns whether or not value *value* is nil, and thus effectively undefined.
+-- @param value The value to test.
+-- @usage expand('{% if is_undefined(x) %}...{% endif %}')
+-- @name tests.is_undefined
+function M.tests.is_undefined(value) return value == nil end
+
+---
+-- Returns whether or not value *value* is nil.
+-- @param value The value to test.
+-- @usage expand('{% if is_none(x) %}...{% endif %}')
+-- @name tests.is_none
+function M.tests.is_none(value) return value == nil end
+
+---
+-- Returns whether or not value *value* is nil.
+-- @param value The value to test.
+-- @usage expand('{% if is_nil(x) %}...{% endif %}')
+-- @name tests.is_nil
+function M.tests.is_nil(value) return value == nil end
+
+---
+-- Returns whether or not string *s* is in all lower-case characters.
+-- @param s The string to test.
+-- @usage expand('{% if is_lower(s) %}...{% endif %}')
+-- @name tests.is_lower
+function M.tests.is_lower(s) return s:lower() == s end
+
+---
+-- Returns whether or not string *s* is in all upper-case characters.
+-- @param s The string to test.
+-- @usage expand('{% if is_upper(s) %}...{% endif %}')
+-- @name tests.is_upper
+function M.tests.is_upper(s) return s:upper() == s end
+
+---
+-- Returns whether or not value *value* is a string.
+-- @param value The value to test.
+-- @usage expand('{% if is_string(x) %}...{% endif %}')
+-- @name tests.is_string
+function M.tests.is_string(value) return type(value) == 'string' end
+
+---
+-- Returns whether or not value *value* is a table.
+-- @param value The value to test.
+-- @usage expand('{% if is_mapping(x) %}...{% endif %}')
+-- @name tests.is_mapping
+function M.tests.is_mapping(value) return type(value) == 'table' end
+
+---
+-- Returns whether or not value *value* is a table.
+-- @param value The value to test.
+-- @usage expand('{% if is_table(x) %}...{% endif %}')
+-- @name tests.is_table
+function M.tests.is_table(value) return type(value) == 'table' end
+
+---
+-- Returns whether or not value *value* is a number.
+-- @param value The value to test.
+-- @usage expand('{% if is_number(x) %}...{% endif %}')
+-- @name tests.is_number
+function M.tests.is_number(value) return type(value) == 'number' end
+
+---
+-- Returns whether or not value *value* is a sequence, namely a table with
+-- non-zero length.
+-- @param value The value to test.
+-- @usage expand('{% if is_sequence(x) %}...{% endif %}')
+-- @name tests.is_sequence
+function M.tests.is_sequence(value)
+ return type(value) == 'table' and #value > 0
+end
+
+---
+-- Returns whether or not value *value* is a sequence (a table with non-zero
+-- length) or a generator.
+-- At the moment, all functions are considered generators.
+-- @param value The value to test.
+-- @usage expand('{% if is_iterable(x) %}...{% endif %}')
+-- @name tests.is_iterable
+function M.tests.is_iterable(value)
+ return M.tests.is_sequence(value) or type(value) == 'function'
+end
+
+---
+-- Returns whether or not value *value* is a function.
+-- @param value The value to test.
+-- @usage expand('{% if is_callable(x) %}...{% endif %}')
+-- @name tests.is_callable
+function M.tests.is_callable(value) return type(value) == 'function' end
+
+---
+-- Returns whether or not value *value* is the same as value *other*.
+-- @param value The value to test.
+-- @param other The value to compare with.
+-- @usage expand('{% if is_sameas(x, y) %}...{% endif %}')
+-- @name tests.is_sameas
+function M.tests.is_sameas(value, other) return value == other end
+
+---
+-- Returns whether or not value *value* is HTML-safe.
+-- @param value The value to test.
+-- @usage expand('{% if is_escaped(x) %}...{% endif %}')
+-- @name tests.is_escaped
+function M.tests.is_escaped(value)
+ return getmetatable(value) and getmetatable(value).__tostring ~= nil
+end
+
+return M
diff --git a/contrib/lua-tableshape/LICENSE b/contrib/lua-tableshape/LICENSE
new file mode 100644
index 0000000..38ba5fc
--- /dev/null
+++ b/contrib/lua-tableshape/LICENSE
@@ -0,0 +1,19 @@
+Copyright (C) 2018 by Leaf Corcoran
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/contrib/lua-tableshape/tableshape.lua b/contrib/lua-tableshape/tableshape.lua
new file mode 100644
index 0000000..ccc7355
--- /dev/null
+++ b/contrib/lua-tableshape/tableshape.lua
@@ -0,0 +1,2324 @@
+local OptionalType, TaggedType, types, is_type
+local BaseType, TransformNode, SequenceNode, FirstOfNode, DescribeNode, NotType, Literal
+local FailedTransform = { }
+local unpack = unpack or table.unpack
+local clone_state
+clone_state = function(state_obj)
+ if type(state_obj) ~= "table" then
+ return { }
+ end
+ local out
+ do
+ local _tbl_0 = { }
+ for k, v in pairs(state_obj) do
+ _tbl_0[k] = v
+ end
+ out = _tbl_0
+ end
+ do
+ local mt = getmetatable(state_obj)
+ if mt then
+ setmetatable(out, mt)
+ end
+ end
+ return out
+end
+local describe_type
+describe_type = function(val)
+ if type(val) == "string" then
+ if not val:match('"') then
+ return "\"" .. tostring(val) .. "\""
+ elseif not val:match("'") then
+ return "'" .. tostring(val) .. "'"
+ else
+ return "`" .. tostring(val) .. "`"
+ end
+ elseif BaseType:is_base_type(val) then
+ return val:_describe()
+ else
+ return tostring(val)
+ end
+end
+local coerce_literal
+coerce_literal = function(value)
+ local _exp_0 = type(value)
+ if "string" == _exp_0 or "number" == _exp_0 or "boolean" == _exp_0 then
+ return Literal(value)
+ elseif "table" == _exp_0 then
+ if BaseType:is_base_type(value) then
+ return value
+ end
+ end
+ return nil, "failed to coerce literal into type, use types.literal() to test for literal value"
+end
+local join_names
+join_names = function(items, sep, last_sep)
+ if sep == nil then
+ sep = ", "
+ end
+ local count = #items
+ local chunks = { }
+ for idx, name in ipairs(items) do
+ if idx > 1 then
+ local current_sep
+ if idx == count then
+ current_sep = last_sep or sep
+ else
+ current_sep = sep
+ end
+ table.insert(chunks, current_sep)
+ end
+ table.insert(chunks, name)
+ end
+ return table.concat(chunks)
+end
+do
+ local _class_0
+ local _base_0 = {
+ __div = function(self, fn)
+ return TransformNode(self, fn)
+ end,
+ __mod = function(self, fn)
+ do
+ local _with_0 = TransformNode(self, fn)
+ _with_0.with_state = true
+ return _with_0
+ end
+ end,
+ __mul = function(_left, _right)
+ local left, err = coerce_literal(_left)
+ if not (left) then
+ error("left hand side of multiplication: " .. tostring(_left) .. ": " .. tostring(err))
+ end
+ local right
+ right, err = coerce_literal(_right)
+ if not (right) then
+ error("right hand side of multiplication: " .. tostring(_right) .. ": " .. tostring(err))
+ end
+ return SequenceNode(left, right)
+ end,
+ __add = function(_left, _right)
+ local left, err = coerce_literal(_left)
+ if not (left) then
+ error("left hand side of addition: " .. tostring(_left) .. ": " .. tostring(err))
+ end
+ local right
+ right, err = coerce_literal(_right)
+ if not (right) then
+ error("right hand side of addition: " .. tostring(_right) .. ": " .. tostring(err))
+ end
+ if left.__class == FirstOfNode then
+ local options = {
+ unpack(left.options)
+ }
+ table.insert(options, right)
+ return FirstOfNode(unpack(options))
+ elseif right.__class == FirstOfNode then
+ return FirstOfNode(left, unpack(right.options))
+ else
+ return FirstOfNode(left, right)
+ end
+ end,
+ __unm = function(self, right)
+ return NotType(right)
+ end,
+ __tostring = function(self)
+ return self:_describe()
+ end,
+ _describe = function(self)
+ return error("Node missing _describe: " .. tostring(self.__class.__name))
+ end,
+ check_value = function(self, ...)
+ local value, state_or_err = self:_transform(...)
+ if value == FailedTransform then
+ return nil, state_or_err
+ end
+ if type(state_or_err) == "table" then
+ return state_or_err
+ else
+ return true
+ end
+ end,
+ transform = function(self, ...)
+ local value, state_or_err = self:_transform(...)
+ if value == FailedTransform then
+ return nil, state_or_err
+ end
+ if type(state_or_err) == "table" then
+ return value, state_or_err
+ else
+ return value
+ end
+ end,
+ repair = function(self, ...)
+ return self:transform(...)
+ end,
+ on_repair = function(self, fn)
+ return (self + types.any / fn * self):describe(function()
+ return self:_describe()
+ end)
+ end,
+ is_optional = function(self)
+ return OptionalType(self)
+ end,
+ describe = function(self, ...)
+ return DescribeNode(self, ...)
+ end,
+ tag = function(self, name)
+ return TaggedType(self, {
+ tag = name
+ })
+ end,
+ clone_opts = function(self)
+ return error("clone_opts is not longer supported")
+ end,
+ __call = function(self, ...)
+ return self:check_value(...)
+ end
+ }
+ _base_0.__index = _base_0
+ _class_0 = setmetatable({
+ __init = function(self, opts) end,
+ __base = _base_0,
+ __name = "BaseType"
+ }, {
+ __index = _base_0,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ local self = _class_0
+ self.is_base_type = function(self, val)
+ do
+ local mt = type(val) == "table" and getmetatable(val)
+ if mt then
+ if mt.__class then
+ return mt.__class.is_base_type == BaseType.is_base_type
+ end
+ end
+ end
+ return false
+ end
+ self.__inherited = function(self, cls)
+ cls.__base.__call = cls.__call
+ cls.__base.__div = self.__div
+ cls.__base.__mod = self.__mod
+ cls.__base.__mul = self.__mul
+ cls.__base.__add = self.__add
+ cls.__base.__unm = self.__unm
+ cls.__base.__tostring = self.__tostring
+ end
+ BaseType = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return self.node:_describe()
+ end,
+ _transform = function(self, value, state)
+ local state_or_err
+ value, state_or_err = self.node:_transform(value, state)
+ if value == FailedTransform then
+ return FailedTransform, state_or_err
+ else
+ local out
+ local _exp_0 = type(self.t_fn)
+ if "function" == _exp_0 then
+ if self.with_state then
+ out = self.t_fn(value, state_or_err)
+ else
+ out = self.t_fn(value)
+ end
+ else
+ out = self.t_fn
+ end
+ return out, state_or_err
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, node, t_fn)
+ self.node, self.t_fn = node, t_fn
+ return assert(self.node, "missing node for transform")
+ end,
+ __base = _base_0,
+ __name = "TransformNode",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ TransformNode = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ local item_names
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _list_0 = self.sequence
+ for _index_0 = 1, #_list_0 do
+ local i = _list_0[_index_0]
+ _accum_0[_len_0] = describe_type(i)
+ _len_0 = _len_0 + 1
+ end
+ item_names = _accum_0
+ end
+ return join_names(item_names, " then ")
+ end,
+ _transform = function(self, value, state)
+ local _list_0 = self.sequence
+ for _index_0 = 1, #_list_0 do
+ local node = _list_0[_index_0]
+ value, state = node:_transform(value, state)
+ if value == FailedTransform then
+ break
+ end
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ self.sequence = {
+ ...
+ }
+ end,
+ __base = _base_0,
+ __name = "SequenceNode",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ SequenceNode = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ local item_names
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _list_0 = self.options
+ for _index_0 = 1, #_list_0 do
+ local i = _list_0[_index_0]
+ _accum_0[_len_0] = describe_type(i)
+ _len_0 = _len_0 + 1
+ end
+ item_names = _accum_0
+ end
+ return join_names(item_names, ", ", ", or ")
+ end,
+ _transform = function(self, value, state)
+ if not (self.options[1]) then
+ return FailedTransform, "no options for node"
+ end
+ local _list_0 = self.options
+ for _index_0 = 1, #_list_0 do
+ local node = _list_0[_index_0]
+ local new_val, new_state = node:_transform(value, state)
+ if not (new_val == FailedTransform) then
+ return new_val, new_state
+ end
+ end
+ return FailedTransform, "expected " .. tostring(self:_describe())
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ self.options = {
+ ...
+ }
+ end,
+ __base = _base_0,
+ __name = "FirstOfNode",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ FirstOfNode = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, input, ...)
+ local value, state = self.node:_transform(input, ...)
+ if value == FailedTransform then
+ local err
+ if self.err_handler then
+ err = self.err_handler(input, state)
+ else
+ err = "expected " .. tostring(self:_describe())
+ end
+ return FailedTransform, err
+ end
+ return value, state
+ end,
+ describe = function(self, ...)
+ return DescribeNode(self.node, ...)
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, node, describe)
+ self.node = node
+ local err_message
+ if type(describe) == "table" then
+ describe, err_message = describe.type, describe.error
+ end
+ if type(describe) == "string" then
+ self._describe = function()
+ return describe
+ end
+ else
+ self._describe = describe
+ end
+ if err_message then
+ if type(err_message) == "string" then
+ self.err_handler = function()
+ return err_message
+ end
+ else
+ self.err_handler = err_message
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "DescribeNode",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ DescribeNode = _class_0
+end
+local AnnotateNode
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ format_error = function(self, value, err)
+ return tostring(tostring(value)) .. ": " .. tostring(err)
+ end,
+ _transform = function(self, value, state)
+ local new_value, state_or_err = self.base_type:_transform(value, state)
+ if new_value == FailedTransform then
+ return FailedTransform, self:format_error(value, state_or_err)
+ else
+ return new_value, state_or_err
+ end
+ end,
+ _describe = function(self)
+ if self.base_type._describe then
+ return self.base_type:_describe()
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type, opts)
+ self.base_type = assert(coerce_literal(base_type))
+ if opts then
+ if opts.format_error then
+ self.format_error = assert(types.func:transform(opts.format_error))
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "AnnotateNode",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ AnnotateNode = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ update_state = function(self, state, value, ...)
+ local out = clone_state(state)
+ if self.tag_type == "function" then
+ if select("#", ...) > 0 then
+ self.tag_name(out, ..., value)
+ else
+ self.tag_name(out, value)
+ end
+ else
+ if self.tag_array then
+ local existing = out[self.tag_name]
+ if type(existing) == "table" then
+ local copy
+ do
+ local _tbl_0 = { }
+ for k, v in pairs(existing) do
+ _tbl_0[k] = v
+ end
+ copy = _tbl_0
+ end
+ table.insert(copy, value)
+ out[self.tag_name] = copy
+ else
+ out[self.tag_name] = {
+ value
+ }
+ end
+ else
+ out[self.tag_name] = value
+ end
+ end
+ return out
+ end,
+ _transform = function(self, value, state)
+ value, state = self.base_type:_transform(value, state)
+ if value == FailedTransform then
+ return FailedTransform, state
+ end
+ state = self:update_state(state, value)
+ return value, state
+ end,
+ _describe = function(self)
+ local base_description = self.base_type:_describe()
+ return tostring(base_description) .. " tagged " .. tostring(describe_type(self.tag_name))
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type, opts)
+ if opts == nil then
+ opts = { }
+ end
+ self.base_type = base_type
+ self.tag_name = assert(opts.tag, "tagged type missing tag")
+ self.tag_type = type(self.tag_name)
+ if self.tag_type == "string" then
+ if self.tag_name:match("%[%]$") then
+ self.tag_name = self.tag_name:sub(1, -3)
+ self.tag_array = true
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "TaggedType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ TaggedType = _class_0
+end
+local TagScopeType
+do
+ local _class_0
+ local _parent_0 = TaggedType
+ local _base_0 = {
+ create_scope_state = function(self, state)
+ return nil
+ end,
+ _transform = function(self, value, state)
+ local scope
+ value, scope = self.base_type:_transform(value, self:create_scope_state(state))
+ if value == FailedTransform then
+ return FailedTransform, scope
+ end
+ if self.tag_name then
+ state = self:update_state(state, scope, value)
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type, opts)
+ if opts then
+ return _class_0.__parent.__init(self, base_type, opts)
+ else
+ self.base_type = base_type
+ end
+ end,
+ __base = _base_0,
+ __name = "TagScopeType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ TagScopeType = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, value, state)
+ if value == nil then
+ return value, state
+ end
+ return self.base_type:_transform(value, state)
+ end,
+ is_optional = function(self)
+ return self
+ end,
+ _describe = function(self)
+ if self.base_type._describe then
+ local base_description = self.base_type:_describe()
+ return "optional " .. tostring(base_description)
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type)
+ self.base_type = base_type
+ return assert(BaseType:is_base_type(self.base_type), "expected a type checker")
+ end,
+ __base = _base_0,
+ __name = "OptionalType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ OptionalType = _class_0
+end
+local AnyType
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, v, state)
+ return v, state
+ end,
+ _describe = function(self)
+ return "anything"
+ end,
+ is_optional = function(self)
+ return self
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ return _class_0.__parent.__init(self, ...)
+ end,
+ __base = _base_0,
+ __name = "AnyType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ AnyType = _class_0
+end
+local Type
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, value, state)
+ local got = type(value)
+ if self.t ~= got then
+ return FailedTransform, "expected type " .. tostring(describe_type(self.t)) .. ", got " .. tostring(describe_type(got))
+ end
+ if self.length_type then
+ local len = #value
+ local res
+ res, state = self.length_type:_transform(len, state)
+ if res == FailedTransform then
+ return FailedTransform, tostring(self.t) .. " length " .. tostring(state) .. ", got " .. tostring(len)
+ end
+ end
+ return value, state
+ end,
+ length = function(self, left, right)
+ local l
+ if BaseType:is_base_type(left) then
+ l = left
+ else
+ l = types.range(left, right)
+ end
+ return Type(self.t, {
+ length = l
+ })
+ end,
+ _describe = function(self)
+ local t = "type " .. tostring(describe_type(self.t))
+ if self.length_type then
+ t = t .. " length_type " .. tostring(self.length_type:_describe())
+ end
+ return t
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, t, opts)
+ self.t = t
+ if opts then
+ if opts.length then
+ self.length_type = assert(coerce_literal(opts.length))
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "Type",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Type = _class_0
+end
+local ArrayType
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "an array"
+ end,
+ _transform = function(self, value, state)
+ if not (type(value) == "table") then
+ return FailedTransform, "expecting table"
+ end
+ local k = 1
+ for i, v in pairs(value) do
+ if not (type(i) == "number") then
+ return FailedTransform, "non number field: " .. tostring(i)
+ end
+ if not (i == k) then
+ return FailedTransform, "non array index, got " .. tostring(describe_type(i)) .. " but expected " .. tostring(describe_type(k))
+ end
+ k = k + 1
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ return _class_0.__parent.__init(self, ...)
+ end,
+ __base = _base_0,
+ __name = "ArrayType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ ArrayType = _class_0
+end
+local OneOf
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ local item_names
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _list_0 = self.options
+ for _index_0 = 1, #_list_0 do
+ local i = _list_0[_index_0]
+ if type(i) == "table" and i._describe then
+ _accum_0[_len_0] = i:_describe()
+ else
+ _accum_0[_len_0] = describe_type(i)
+ end
+ _len_0 = _len_0 + 1
+ end
+ item_names = _accum_0
+ end
+ return tostring(join_names(item_names, ", ", ", or "))
+ end,
+ _transform = function(self, value, state)
+ if self.options_hash then
+ if self.options_hash[value] then
+ return value, state
+ end
+ else
+ local _list_0 = self.options
+ for _index_0 = 1, #_list_0 do
+ local _continue_0 = false
+ repeat
+ local item = _list_0[_index_0]
+ if item == value then
+ return value, state
+ end
+ if BaseType:is_base_type(item) then
+ local new_value, new_state = item:_transform(value, state)
+ if new_value == FailedTransform then
+ _continue_0 = true
+ break
+ end
+ return new_value, new_state
+ end
+ _continue_0 = true
+ until true
+ if not _continue_0 then
+ break
+ end
+ end
+ end
+ return FailedTransform, "expected " .. tostring(self:_describe())
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, options)
+ self.options = options
+ assert(type(self.options) == "table", "expected table for options in one_of")
+ local fast_opts = types.array_of(types.number + types.string)
+ if fast_opts(self.options) then
+ do
+ local _tbl_0 = { }
+ local _list_0 = self.options
+ for _index_0 = 1, #_list_0 do
+ local v = _list_0[_index_0]
+ _tbl_0[v] = true
+ end
+ self.options_hash = _tbl_0
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "OneOf",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ OneOf = _class_0
+end
+local AllOf
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ local item_names
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _list_0 = self.types
+ for _index_0 = 1, #_list_0 do
+ local i = _list_0[_index_0]
+ _accum_0[_len_0] = describe_type(i)
+ _len_0 = _len_0 + 1
+ end
+ item_names = _accum_0
+ end
+ return join_names(item_names, " and ")
+ end,
+ _transform = function(self, value, state)
+ local _list_0 = self.types
+ for _index_0 = 1, #_list_0 do
+ local t = _list_0[_index_0]
+ value, state = t:_transform(value, state)
+ if value == FailedTransform then
+ return FailedTransform, state
+ end
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, types)
+ self.types = types
+ assert(type(self.types) == "table", "expected table for first argument")
+ local _list_0 = self.types
+ for _index_0 = 1, #_list_0 do
+ local checker = _list_0[_index_0]
+ assert(BaseType:is_base_type(checker), "all_of expects all type checkers")
+ end
+ end,
+ __base = _base_0,
+ __name = "AllOf",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ AllOf = _class_0
+end
+local ArrayOf
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "array of " .. tostring(describe_type(self.expected))
+ end,
+ _transform = function(self, value, state)
+ local pass, err = types.table(value)
+ if not (pass) then
+ return FailedTransform, err
+ end
+ if self.length_type then
+ local len = #value
+ local res
+ res, state = self.length_type:_transform(len, state)
+ if res == FailedTransform then
+ return FailedTransform, "array length " .. tostring(state) .. ", got " .. tostring(len)
+ end
+ end
+ local is_literal = not BaseType:is_base_type(self.expected)
+ local copy, k
+ for idx, item in ipairs(value) do
+ local skip_item = false
+ local transformed_item
+ if is_literal then
+ if self.expected ~= item then
+ return FailedTransform, "array item " .. tostring(idx) .. ": expected " .. tostring(describe_type(self.expected))
+ else
+ transformed_item = item
+ end
+ else
+ local item_val
+ item_val, state = self.expected:_transform(item, state)
+ if item_val == FailedTransform then
+ return FailedTransform, "array item " .. tostring(idx) .. ": " .. tostring(state)
+ end
+ if item_val == nil and not self.keep_nils then
+ skip_item = true
+ else
+ transformed_item = item_val
+ end
+ end
+ if transformed_item ~= item or skip_item then
+ if not (copy) then
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _max_0 = idx - 1
+ for _index_0 = 1, _max_0 < 0 and #value + _max_0 or _max_0 do
+ local i = value[_index_0]
+ _accum_0[_len_0] = i
+ _len_0 = _len_0 + 1
+ end
+ copy = _accum_0
+ end
+ k = idx
+ end
+ end
+ if copy and not skip_item then
+ copy[k] = transformed_item
+ k = k + 1
+ end
+ end
+ return copy or value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, expected, opts)
+ self.expected = expected
+ if opts then
+ self.keep_nils = opts.keep_nils and true
+ if opts.length then
+ self.length_type = assert(coerce_literal(opts.length))
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "ArrayOf",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ local self = _class_0
+ self.type_err_message = "expecting table"
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ ArrayOf = _class_0
+end
+local ArrayContains
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ short_circuit = true,
+ keep_nils = false,
+ _describe = function(self)
+ return "array containing " .. tostring(describe_type(self.contains))
+ end,
+ _transform = function(self, value, state)
+ local pass, err = types.table(value)
+ if not (pass) then
+ return FailedTransform, err
+ end
+ local is_literal = not BaseType:is_base_type(self.contains)
+ local contains = false
+ local copy, k
+ for idx, item in ipairs(value) do
+ local skip_item = false
+ local transformed_item
+ if is_literal then
+ if self.contains == item then
+ contains = true
+ end
+ transformed_item = item
+ else
+ local item_val, new_state = self.contains:_transform(item, state)
+ if item_val == FailedTransform then
+ transformed_item = item
+ else
+ state = new_state
+ contains = true
+ if item_val == nil and not self.keep_nils then
+ skip_item = true
+ else
+ transformed_item = item_val
+ end
+ end
+ end
+ if transformed_item ~= item or skip_item then
+ if not (copy) then
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ local _max_0 = idx - 1
+ for _index_0 = 1, _max_0 < 0 and #value + _max_0 or _max_0 do
+ local i = value[_index_0]
+ _accum_0[_len_0] = i
+ _len_0 = _len_0 + 1
+ end
+ copy = _accum_0
+ end
+ k = idx
+ end
+ end
+ if copy and not skip_item then
+ copy[k] = transformed_item
+ k = k + 1
+ end
+ if contains and self.short_circuit then
+ if copy then
+ for kdx = idx + 1, #value do
+ copy[k] = value[kdx]
+ k = k + 1
+ end
+ end
+ break
+ end
+ end
+ if not (contains) then
+ return FailedTransform, "expected " .. tostring(self:_describe())
+ end
+ return copy or value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, contains, opts)
+ self.contains = contains
+ assert(self.contains, "missing contains")
+ if opts then
+ self.short_circuit = opts.short_circuit and true
+ self.keep_nils = opts.keep_nils and true
+ end
+ end,
+ __base = _base_0,
+ __name = "ArrayContains",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ local self = _class_0
+ self.type_err_message = "expecting table"
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ ArrayContains = _class_0
+end
+local MapOf
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "map of " .. tostring(self.expected_key:_describe()) .. " -> " .. tostring(self.expected_value:_describe())
+ end,
+ _transform = function(self, value, state)
+ local pass, err = types.table(value)
+ if not (pass) then
+ return FailedTransform, err
+ end
+ local key_literal = not BaseType:is_base_type(self.expected_key)
+ local value_literal = not BaseType:is_base_type(self.expected_value)
+ local transformed = false
+ local out = { }
+ for k, v in pairs(value) do
+ local _continue_0 = false
+ repeat
+ local new_k = k
+ local new_v = v
+ if key_literal then
+ if k ~= self.expected_key then
+ return FailedTransform, "map key expected " .. tostring(describe_type(self.expected_key))
+ end
+ else
+ new_k, state = self.expected_key:_transform(k, state)
+ if new_k == FailedTransform then
+ return FailedTransform, "map key " .. tostring(state)
+ end
+ end
+ if value_literal then
+ if v ~= self.expected_value then
+ return FailedTransform, "map value expected " .. tostring(describe_type(self.expected_value))
+ end
+ else
+ new_v, state = self.expected_value:_transform(v, state)
+ if new_v == FailedTransform then
+ return FailedTransform, "map value " .. tostring(state)
+ end
+ end
+ if new_k ~= k or new_v ~= v then
+ transformed = true
+ end
+ if new_k == nil then
+ _continue_0 = true
+ break
+ end
+ out[new_k] = new_v
+ _continue_0 = true
+ until true
+ if not _continue_0 then
+ break
+ end
+ end
+ return transformed and out or value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, expected_key, expected_value)
+ self.expected_key = coerce_literal(expected_key)
+ self.expected_value = coerce_literal(expected_value)
+ end,
+ __base = _base_0,
+ __name = "MapOf",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ MapOf = _class_0
+end
+local Shape
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ open = false,
+ check_all = false,
+ is_open = function(self)
+ return Shape(self.shape, {
+ open = true,
+ check_all = self.check_all or nil
+ })
+ end,
+ _describe = function(self)
+ local parts
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ for k, v in pairs(self.shape) do
+ _accum_0[_len_0] = tostring(describe_type(k)) .. " = " .. tostring(describe_type(v))
+ _len_0 = _len_0 + 1
+ end
+ parts = _accum_0
+ end
+ return "{ " .. tostring(table.concat(parts, ", ")) .. " }"
+ end,
+ _transform = function(self, value, state)
+ local pass, err = types.table(value)
+ if not (pass) then
+ return FailedTransform, err
+ end
+ local check_all = self.check_all
+ local remaining_keys
+ do
+ local _tbl_0 = { }
+ for key in pairs(value) do
+ _tbl_0[key] = true
+ end
+ remaining_keys = _tbl_0
+ end
+ local errors
+ local dirty = false
+ local out = { }
+ for shape_key, shape_val in pairs(self.shape) do
+ local item_value = value[shape_key]
+ if remaining_keys then
+ remaining_keys[shape_key] = nil
+ end
+ local new_val
+ if BaseType:is_base_type(shape_val) then
+ new_val, state = shape_val:_transform(item_value, state)
+ else
+ if shape_val == item_value then
+ new_val, state = item_value, state
+ else
+ new_val, state = FailedTransform, "expected " .. tostring(describe_type(shape_val))
+ end
+ end
+ if new_val == FailedTransform then
+ err = "field " .. tostring(describe_type(shape_key)) .. ": " .. tostring(state)
+ if check_all then
+ if errors then
+ table.insert(errors, err)
+ else
+ errors = {
+ err
+ }
+ end
+ else
+ return FailedTransform, err
+ end
+ else
+ if new_val ~= item_value then
+ dirty = true
+ end
+ out[shape_key] = new_val
+ end
+ end
+ if remaining_keys and next(remaining_keys) then
+ if self.open then
+ for k in pairs(remaining_keys) do
+ out[k] = value[k]
+ end
+ elseif self.extra_fields_type then
+ for k in pairs(remaining_keys) do
+ local item_value = value[k]
+ local tuple
+ tuple, state = self.extra_fields_type:_transform({
+ [k] = item_value
+ }, state)
+ if tuple == FailedTransform then
+ err = "field " .. tostring(describe_type(k)) .. ": " .. tostring(state)
+ if check_all then
+ if errors then
+ table.insert(errors, err)
+ else
+ errors = {
+ err
+ }
+ end
+ else
+ return FailedTransform, err
+ end
+ else
+ do
+ local nk = tuple and next(tuple)
+ if nk then
+ if nk ~= k then
+ dirty = true
+ elseif tuple[nk] ~= item_value then
+ dirty = true
+ end
+ out[nk] = tuple[nk]
+ else
+ dirty = true
+ end
+ end
+ end
+ end
+ else
+ local names
+ do
+ local _accum_0 = { }
+ local _len_0 = 1
+ for key in pairs(remaining_keys) do
+ _accum_0[_len_0] = describe_type(key)
+ _len_0 = _len_0 + 1
+ end
+ names = _accum_0
+ end
+ err = "extra fields: " .. tostring(table.concat(names, ", "))
+ if check_all then
+ if errors then
+ table.insert(errors, err)
+ else
+ errors = {
+ err
+ }
+ end
+ else
+ return FailedTransform, err
+ end
+ end
+ end
+ if errors and next(errors) then
+ return FailedTransform, table.concat(errors, "; ")
+ end
+ return dirty and out or value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, shape, opts)
+ self.shape = shape
+ assert(type(self.shape) == "table", "expected table for shape")
+ if opts then
+ if opts.extra_fields then
+ assert(BaseType:is_base_type(opts.extra_fields), "extra_fields_type must be type checker")
+ self.extra_fields_type = opts.extra_fields
+ end
+ self.open = opts.open and true
+ self.check_all = opts.check_all and true
+ if self.open then
+ assert(not self.extra_fields_type, "open can not be combined with extra_fields")
+ end
+ if self.extra_fields_type then
+ return assert(not self.open, "extra_fields can not be combined with open")
+ end
+ end
+ end,
+ __base = _base_0,
+ __name = "Shape",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ local self = _class_0
+ self.type_err_message = "expecting table"
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Shape = _class_0
+end
+local Partial
+do
+ local _class_0
+ local _parent_0 = Shape
+ local _base_0 = {
+ open = true,
+ is_open = function(self)
+ return error("is_open has no effect on Partial")
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ return _class_0.__parent.__init(self, ...)
+ end,
+ __base = _base_0,
+ __name = "Partial",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Partial = _class_0
+end
+local Pattern
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "pattern " .. tostring(describe_type(self.pattern))
+ end,
+ _transform = function(self, value, state)
+ local test_value
+ if self.coerce then
+ if BaseType:is_base_type(self.coerce) then
+ local c_res, err = self.coerce:_transform(value)
+ if c_res == FailedTransform then
+ return FailedTransform, err
+ end
+ test_value = c_res
+ else
+ test_value = tostring(value)
+ end
+ else
+ test_value = value
+ end
+ local t_res, err = types.string(test_value)
+ if not (t_res) then
+ return FailedTransform, err
+ end
+ if test_value:match(self.pattern) then
+ return value, state
+ else
+ return FailedTransform, "doesn't match " .. tostring(self:_describe())
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, pattern, opts)
+ self.pattern = pattern
+ assert(type(self.pattern) == "string", "Pattern must be a string")
+ if opts then
+ self.coerce = opts.coerce
+ return assert(opts.initial_type == nil, "initial_type has been removed from types.pattern (got: " .. tostring(opts.initial_type) .. ")")
+ end
+ end,
+ __base = _base_0,
+ __name = "Pattern",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Pattern = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return describe_type(self.value)
+ end,
+ _transform = function(self, value, state)
+ if self.value ~= value then
+ return FailedTransform, "expected " .. tostring(self:_describe())
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, value)
+ self.value = value
+ end,
+ __base = _base_0,
+ __name = "Literal",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Literal = _class_0
+end
+local Custom
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "custom checker " .. tostring(self.fn)
+ end,
+ _transform = function(self, value, state)
+ local pass, err = self.fn(value, state)
+ if not (pass) then
+ return FailedTransform, err or "failed custom check"
+ end
+ return value, state
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, fn)
+ self.fn = fn
+ return assert(type(self.fn) == "function", "custom checker must be a function")
+ end,
+ __base = _base_0,
+ __name = "Custom",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Custom = _class_0
+end
+local Equivalent
+do
+ local _class_0
+ local values_equivalent
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _describe = function(self)
+ return "equivalent to " .. tostring(describe_type(self.val))
+ end,
+ _transform = function(self, value, state)
+ if values_equivalent(self.val, value) then
+ return value, state
+ else
+ return FailedTransform, "not equivalent to " .. tostring(self.val)
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, val)
+ self.val = val
+ end,
+ __base = _base_0,
+ __name = "Equivalent",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ local self = _class_0
+ values_equivalent = function(a, b)
+ if a == b then
+ return true
+ end
+ if type(a) == "table" and type(b) == "table" then
+ local seen_keys = { }
+ for k, v in pairs(a) do
+ seen_keys[k] = true
+ if not (values_equivalent(v, b[k])) then
+ return false
+ end
+ end
+ for k, v in pairs(b) do
+ local _continue_0 = false
+ repeat
+ if seen_keys[k] then
+ _continue_0 = true
+ break
+ end
+ if not (values_equivalent(v, a[k])) then
+ return false
+ end
+ _continue_0 = true
+ until true
+ if not _continue_0 then
+ break
+ end
+ end
+ return true
+ else
+ return false
+ end
+ end
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Equivalent = _class_0
+end
+local Range
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, value, state)
+ local res
+ res, state = self.value_type:_transform(value, state)
+ if res == FailedTransform then
+ return FailedTransform, "range " .. tostring(state)
+ end
+ if value < self.left then
+ return FailedTransform, "not in " .. tostring(self:_describe())
+ end
+ if value > self.right then
+ return FailedTransform, "not in " .. tostring(self:_describe())
+ end
+ return value, state
+ end,
+ _describe = function(self)
+ return "range from " .. tostring(self.left) .. " to " .. tostring(self.right)
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, left, right)
+ self.left, self.right = left, right
+ assert(self.left <= self.right, "left range value should be less than right range value")
+ self.value_type = assert(types[type(self.left)], "couldn't figure out type of range boundary")
+ end,
+ __base = _base_0,
+ __name = "Range",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Range = _class_0
+end
+local Proxy
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, ...)
+ return assert(self.fn(), "proxy missing transformer"):_transform(...)
+ end,
+ _describe = function(self, ...)
+ return assert(self.fn(), "proxy missing transformer"):_describe(...)
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, fn)
+ self.fn = fn
+ end,
+ __base = _base_0,
+ __name = "Proxy",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ Proxy = _class_0
+end
+local AssertType
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ assert = assert,
+ _transform = function(self, value, state)
+ local state_or_err
+ value, state_or_err = self.base_type:_transform(value, state)
+ self.assert(value ~= FailedTransform, state_or_err)
+ return value, state_or_err
+ end,
+ _describe = function(self)
+ if self.base_type._describe then
+ local base_description = self.base_type:_describe()
+ return "assert " .. tostring(base_description)
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type)
+ self.base_type = base_type
+ return assert(BaseType:is_base_type(self.base_type), "expected a type checker")
+ end,
+ __base = _base_0,
+ __name = "AssertType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ AssertType = _class_0
+end
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, value, state)
+ local out, _ = self.base_type:_transform(value, state)
+ if out == FailedTransform then
+ return value, state
+ else
+ return FailedTransform, "expected " .. tostring(self:_describe())
+ end
+ end,
+ _describe = function(self)
+ if self.base_type._describe then
+ local base_description = self.base_type:_describe()
+ return "not " .. tostring(base_description)
+ end
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, base_type)
+ self.base_type = base_type
+ return assert(BaseType:is_base_type(self.base_type), "expected a type checker")
+ end,
+ __base = _base_0,
+ __name = "NotType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ NotType = _class_0
+end
+local CloneType
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ _transform = function(self, value, state)
+ local _exp_0 = type(value)
+ if "nil" == _exp_0 or "string" == _exp_0 or "number" == _exp_0 or "boolean" == _exp_0 then
+ return value, state
+ elseif "table" == _exp_0 then
+ local clone_value
+ do
+ local _tbl_0 = { }
+ for k, v in pairs(value) do
+ _tbl_0[k] = v
+ end
+ clone_value = _tbl_0
+ end
+ do
+ local mt = getmetatable(value)
+ if mt then
+ setmetatable(clone_value, mt)
+ end
+ end
+ return clone_value, state
+ else
+ return FailedTransform, tostring(describe_type(value)) .. " is not cloneable"
+ end
+ end,
+ _describe = function(self)
+ return "cloneable value"
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, ...)
+ return _class_0.__parent.__init(self, ...)
+ end,
+ __base = _base_0,
+ __name = "CloneType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ CloneType = _class_0
+end
+local MetatableIsType
+do
+ local _class_0
+ local _parent_0 = BaseType
+ local _base_0 = {
+ allow_metatable_update = false,
+ _transform = function(self, value, state)
+ local state_or_err
+ value, state_or_err = types.table:_transform(value, state)
+ if value == FailedTransform then
+ return FailedTransform, state_or_err
+ end
+ local mt = getmetatable(value)
+ local new_mt
+ new_mt, state_or_err = self.metatable_type:_transform(mt, state_or_err)
+ if new_mt == FailedTransform then
+ return FailedTransform, "metatable expected: " .. tostring(state_or_err)
+ end
+ if new_mt ~= mt then
+ if self.allow_metatable_update then
+ setmetatable(value, new_mt)
+ else
+ return FailedTransform, "metatable was modified by a type but { allow_metatable_update = true } is not enabled"
+ end
+ end
+ return value, state_or_err
+ end,
+ _describe = function(self)
+ return "has metatable " .. tostring(describe_type(self.metatable_type))
+ end
+ }
+ _base_0.__index = _base_0
+ setmetatable(_base_0, _parent_0.__base)
+ _class_0 = setmetatable({
+ __init = function(self, metatable_type, opts)
+ if BaseType:is_base_type(metatable_type) then
+ self.metatable_type = metatable_type
+ else
+ self.metatable_type = Literal(metatable_type)
+ end
+ if opts then
+ self.allow_metatable_update = opts.allow_metatable_update and true
+ end
+ end,
+ __base = _base_0,
+ __name = "MetatableIsType",
+ __parent = _parent_0
+ }, {
+ __index = function(cls, name)
+ local val = rawget(_base_0, name)
+ if val == nil then
+ local parent = rawget(cls, "__parent")
+ if parent then
+ return parent[name]
+ end
+ else
+ return val
+ end
+ end,
+ __call = function(cls, ...)
+ local _self_0 = setmetatable({}, _base_0)
+ cls.__init(_self_0, ...)
+ return _self_0
+ end
+ })
+ _base_0.__class = _class_0
+ if _parent_0.__inherited then
+ _parent_0.__inherited(_parent_0, _class_0)
+ end
+ MetatableIsType = _class_0
+end
+local type_nil = Type("nil")
+local type_function = Type("function")
+local type_number = Type("number")
+types = setmetatable({
+ any = AnyType(),
+ string = Type("string"),
+ number = type_number,
+ ["function"] = type_function,
+ func = type_function,
+ boolean = Type("boolean"),
+ userdata = Type("userdata"),
+ ["nil"] = type_nil,
+ null = type_nil,
+ table = Type("table"),
+ array = ArrayType(),
+ clone = CloneType(),
+ integer = Pattern("^%d+$", {
+ coerce = type_number / tostring
+ }),
+ one_of = OneOf,
+ all_of = AllOf,
+ shape = Shape,
+ partial = Partial,
+ pattern = Pattern,
+ array_of = ArrayOf,
+ array_contains = ArrayContains,
+ map_of = MapOf,
+ literal = Literal,
+ range = Range,
+ equivalent = Equivalent,
+ custom = Custom,
+ scope = TagScopeType,
+ proxy = Proxy,
+ assert = AssertType,
+ annotate = AnnotateNode,
+ metatable_is = MetatableIsType
+}, {
+ __index = function(self, fn_name)
+ return error("Type checker does not exist: `" .. tostring(fn_name) .. "`")
+ end
+})
+local check_shape
+check_shape = function(value, shape)
+ assert(shape.check_value, "missing check_value method from shape")
+ return shape:check_value(value)
+end
+is_type = function(val)
+ return BaseType:is_base_type(val)
+end
+return {
+ check_shape = check_shape,
+ types = types,
+ is_type = is_type,
+ BaseType = BaseType,
+ FailedTransform = FailedTransform,
+ VERSION = "2.6.0"
+}
diff --git a/contrib/mumhash/mum.h b/contrib/mumhash/mum.h
new file mode 100644
index 0000000..52b7845
--- /dev/null
+++ b/contrib/mumhash/mum.h
@@ -0,0 +1,392 @@
+/* Copyright (c) 2016 Vladimir Makarov <vmakarov@gcc.gnu.org>
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use, copy,
+ modify, merge, publish, distribute, sublicense, and/or sell copies
+ of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+/* This file implements MUM (MUltiply and Mix) hashing. We randomize
+ input data by 64x64-bit multiplication and mixing hi- and low-parts
+ of the multiplication result by using an addition and then mix it
+ into the current state. We use prime numbers randomly generated
+ with the equal probability of their bit values for the
+ multiplication. When all primes are used once, the state is
+ randomized and the same prime numbers are used again for data
+ randomization.
+
+ The MUM hashing passes all SMHasher tests. Pseudo Random Number
+ Generator based on MUM also passes NIST Statistical Test Suite for
+ Random and Pseudorandom Number Generators for Cryptographic
+ Applications (version 2.2.1) with 1000 bitstreams each containing
+ 1M bits. MUM hashing is also faster Spooky64 and City64 on small
+ strings (at least up to 512-bit) on Haswell and Power7. The MUM bulk
+ speed (speed on very long data) is bigger than Spooky and City on
+ Power7. On Haswell the bulk speed is bigger than Spooky one and
+ close to City speed. */
+
+#ifndef __MUM_HASH__
+#define __MUM_HASH__
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+#ifdef _MSC_VER
+typedef unsigned __int16 uint16_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
+#else
+#include <stdint.h>
+#endif
+
+/* Macro saying to use 128-bit integers implemented by GCC for some
+ targets. */
+#ifndef _MUM_USE_INT128
+/* In GCC uint128_t is defined if HOST_BITS_PER_WIDE_INT >= 64.
+ HOST_WIDE_INT is long if HOST_BITS_PER_LONG > HOST_BITS_PER_INT,
+ otherwise int. */
+#ifdef __SIZEOF_INT128__
+#define _MUM_USE_INT128 1
+#else
+#define _MUM_USE_INT128 0
+#endif
+#endif
+
+#if defined(__GNUC__) && ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 9) || (__GNUC__ > 4))
+#define _MUM_FRESH_GCC
+#endif
+
+#if !defined(__llvm__) && defined(_MUM_FRESH_GCC)
+#define _MUM_ATTRIBUTE_UNUSED __attribute__((unused))
+#define _MUM_OPTIMIZE(opts) __attribute__((__optimize__ (opts)))
+#define _MUM_TARGET(opts) __attribute__((__target__ (opts)))
+#define _MUM_INLINE __attribute__((always_inline))
+#else
+#define _MUM_ATTRIBUTE_UNUSED
+#define _MUM_OPTIMIZE(opts)
+#define _MUM_TARGET(opts)
+#define _MUM_INLINE
+#endif
+
+
+/* Here are different primes randomly generated with the equal
+ probability of their bit values. They are used to randomize input
+ values. */
+static uint64_t _mum_hash_step_prime = 0x2e0bb864e9ea7df5ULL;
+static uint64_t _mum_key_step_prime = 0xcdb32970830fcaa1ULL;
+static uint64_t _mum_block_start_prime = 0xc42b5e2e6480b23bULL;
+static uint64_t _mum_unroll_prime = 0x7b51ec3d22f7096fULL;
+static uint64_t _mum_tail_prime = 0xaf47d47c99b1461bULL;
+static uint64_t _mum_finish_prime1 = 0xa9a7ae7ceff79f3fULL;
+static uint64_t _mum_finish_prime2 = 0xaf47d47c99b1461bULL;
+
+static uint64_t _mum_primes [] = {
+ 0X9ebdcae10d981691, 0X32b9b9b97a27ac7d, 0X29b5584d83d35bbd, 0X4b04e0e61401255f,
+ 0X25e8f7b1f1c9d027, 0X80d4c8c000f3e881, 0Xbd1255431904b9dd, 0X8a3bd4485eee6d81,
+ 0X3bc721b2aad05197, 0X71b1a19b907d6e33, 0X525e6c1084a8534b, 0X9e4c2cd340c1299f,
+ 0Xde3add92e94caa37, 0X7e14eadb1f65311d, 0X3f5aa40f89812853, 0X33b15a3b587d15c9,
+};
+
+/* Multiply 64-bit V and P and return sum of high and low parts of the
+ result. */
+static inline uint64_t _MUM_INLINE
+_mum (uint64_t v, uint64_t p) {
+ uint64_t hi, lo;
+#if _MUM_USE_INT128
+#if defined(__aarch64__)
+ /* AARCH64 needs 2 insns to calculate 128-bit result of the
+ multiplication. If we use a generic code we actually call a
+ function doing 128x128->128 bit multiplication. The function is
+ very slow. */
+ lo = v * p;
+ __asm__ ("umulh %0, %1, %2" : "=r" (hi) : "r" (v), "r" (p));
+#else
+ __uint128_t r = (__uint128_t) v * (__uint128_t) p;
+ hi = (uint64_t) (r >> 64);
+ lo = (uint64_t) r;
+#endif
+#else
+ /* Implementation of 64x64->128-bit multiplication by four 32x32->64
+ bit multiplication. */
+ uint64_t hv = v >> 32, hp = p >> 32;
+ uint64_t lv = (uint32_t) v, lp = (uint32_t) p;
+ uint64_t rh = hv * hp;
+ uint64_t rm_0 = hv * lp;
+ uint64_t rm_1 = hp * lv;
+ uint64_t rl = lv * lp;
+ uint64_t t, carry = 0;
+
+ /* We could ignore a carry bit here if we did not care about the
+ same hash for 32-bit and 64-bit targets. */
+ t = rl + (rm_0 << 32);
+#ifdef MUM_TARGET_INDEPENDENT_HASH
+ carry = t < rl;
+#endif
+ lo = t + (rm_1 << 32);
+#ifdef MUM_TARGET_INDEPENDENT_HASH
+ carry += lo < t;
+#endif
+ hi = rh + (rm_0 >> 32) + (rm_1 >> 32) + carry;
+#endif
+ /* We could use XOR here too but, for some reasons, on Haswell and
+ Power7 using an addition improves hashing performance by 10% for
+ small strings. */
+ return hi + lo;
+}
+
+#if defined(_MSC_VER)
+#define _mum_bswap_32(x) _byteswap_uint32_t (x)
+#define _mum_bswap_64(x) _byteswap_uint64_t (x)
+#elif defined(__APPLE__)
+#include <libkern/OSByteOrder.h>
+#define _mum_bswap_32(x) OSSwapInt32 (x)
+#define _mum_bswap_64(x) OSSwapInt64 (x)
+#elif defined(__GNUC__)
+#define _mum_bswap32(x) __builtin_bswap32 (x)
+#define _mum_bswap64(x) __builtin_bswap64 (x)
+#else
+#include <byteswap.h>
+#define _mum_bswap32(x) bswap32 (x)
+#define _mum_bswap64(x) bswap64 (x)
+#endif
+
+static inline uint64_t _MUM_INLINE
+_mum_le (uint64_t v) {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ || !defined(MUM_TARGET_INDEPENDENT_HASH)
+ return v;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return _mum_bswap64 (v);
+#else
+#error "Unknown endianness"
+#endif
+}
+
+static inline uint32_t _MUM_INLINE
+_mum_le32 (uint32_t v) {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ || !defined(MUM_TARGET_INDEPENDENT_HASH)
+ return v;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return _mum_bswap32 (v);
+#else
+#error "Unknown endianness"
+#endif
+}
+
+/* Macro defining how many times the most nested loop in
+ _mum_hash_aligned will be unrolled by the compiler (although it can
+ make an own decision:). Use only a constant here to help a
+ compiler to unroll a major loop.
+
+ The macro value affects the result hash for strings > 128 bit. The
+ unroll factor greatly affects the hashing speed. We prefer the
+ speed. */
+#ifndef _MUM_UNROLL_FACTOR_POWER
+#if defined(__PPC64__) && !defined(MUM_TARGET_INDEPENDENT_HASH)
+#define _MUM_UNROLL_FACTOR_POWER 3
+#elif defined(__aarch64__) && !defined(MUM_TARGET_INDEPENDENT_HASH)
+#define _MUM_UNROLL_FACTOR_POWER 4
+#else
+#define _MUM_UNROLL_FACTOR_POWER 2
+#endif
+#endif
+
+#if _MUM_UNROLL_FACTOR_POWER < 1
+#error "too small unroll factor"
+#elif _MUM_UNROLL_FACTOR_POWER > 4
+#error "We have not enough primes for such unroll factor"
+#endif
+
+#define _MUM_UNROLL_FACTOR (1 << _MUM_UNROLL_FACTOR_POWER)
+
+static inline uint64_t _MUM_OPTIMIZE("unroll-loops") _MUM_INLINE
+_mum_hash_aligned (uint64_t start, const void *key, size_t len) {
+ uint64_t result = start;
+ const unsigned char *str = (const unsigned char *) key;
+ uint64_t u64;
+ int i;
+ size_t n;
+
+ result = _mum (result, _mum_block_start_prime);
+ while (len > _MUM_UNROLL_FACTOR * sizeof (uint64_t)) {
+ /* This loop could be vectorized when we have vector insns for
+ 64x64->128-bit multiplication. AVX2 currently only have a
+ vector insn for 4 32x32->64-bit multiplication. */
+ for (i = 0; i < _MUM_UNROLL_FACTOR; i++)
+ result ^= _mum (_mum_le (((uint64_t *) str)[i]), _mum_primes[i]);
+ len -= _MUM_UNROLL_FACTOR * sizeof (uint64_t);
+ str += _MUM_UNROLL_FACTOR * sizeof (uint64_t);
+ /* We will use the same prime numbers on the next iterations --
+ randomize the state. */
+ result = _mum (result, _mum_unroll_prime);
+ }
+ n = len / sizeof (uint64_t);
+ for (i = 0; i < (int)n; i++)
+ result ^= _mum (_mum_le (((uint64_t *) str)[i]), _mum_primes[i]);
+ len -= n * sizeof (uint64_t); str += n * sizeof (uint64_t);
+ switch (len) {
+ case 7:
+ u64 = _mum_le32 (*(uint32_t *) str);
+ u64 |= (uint64_t) str[4] << 32;
+ u64 |= (uint64_t) str[5] << 40;
+ u64 |= (uint64_t) str[6] << 48;
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 6:
+ u64 = _mum_le32 (*(uint32_t *) str);
+ u64 |= (uint64_t) str[4] << 32;
+ u64 |= (uint64_t) str[5] << 40;
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 5:
+ u64 = _mum_le32 (*(uint32_t *) str);
+ u64 |= (uint64_t) str[4] << 32;
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 4:
+ u64 = _mum_le32 (*(uint32_t *) str);
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 3:
+ u64 = str[0];
+ u64 |= (uint64_t) str[1] << 8;
+ u64 |= (uint64_t) str[2] << 16;
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 2:
+ u64 = str[0];
+ u64 |= (uint64_t) str[1] << 8;
+ return result ^ _mum (u64, _mum_tail_prime);
+ case 1:
+ u64 = str[0];
+ return result ^ _mum (u64, _mum_tail_prime);
+ }
+ return result;
+}
+
+/* Final randomization of H. */
+static inline uint64_t
+_mum_final (uint64_t h) {
+ h ^= _mum (h, _mum_finish_prime1);
+ h ^= _mum (h, _mum_finish_prime2);
+ return h;
+}
+
+#ifndef _MUM_UNALIGNED_ACCESS
+#if defined(__x86_64__) || defined(__i386__) || defined(__PPC64__) \
+ || defined(__s390__) || defined(__m32c__) || defined(cris) \
+ || defined(__CR16__) || defined(__vax__) || defined(__m68k__) \
+ || defined(__aarch64__)
+#define _MUM_UNALIGNED_ACCESS 1
+#else
+#define _MUM_UNALIGNED_ACCESS 0
+#endif
+#endif
+
+/* When we need an aligned access to data being hashed we move part of
+ the unaligned data to an aligned block of given size and then
+ process it, repeating processing the data by the block. */
+#ifndef _MUM_BLOCK_LEN
+#define _MUM_BLOCK_LEN 1024
+#endif
+
+#if _MUM_BLOCK_LEN < 8
+#error "too small block length"
+#endif
+
+static inline uint64_t _MUM_INLINE
+_mum_hash_default (const void *key, size_t len, uint64_t seed) {
+ uint64_t result;
+ const unsigned char *str = (const unsigned char *) key;
+ size_t block_len;
+ uint64_t buf[_MUM_BLOCK_LEN / sizeof (uint64_t)];
+
+ result = seed + len;
+ if (_MUM_UNALIGNED_ACCESS || ((size_t) str & 0x7) == 0)
+ result = _mum_hash_aligned (result, key, len);
+ else {
+ while (len != 0) {
+ block_len = len < _MUM_BLOCK_LEN ? len : _MUM_BLOCK_LEN;
+ memmove (buf, str, block_len);
+ result = _mum_hash_aligned (result, buf, block_len);
+ len -= block_len;
+ str += block_len;
+ }
+ }
+ return _mum_final (result);
+}
+
+static inline uint64_t _MUM_INLINE
+_mum_next_factor (void) {
+ uint64_t start = 0;
+ int i;
+
+ for (i = 0; i < 8; i++)
+ start = (start << 8) | rand() % 256;
+ return start;
+}
+
+/* ++++++++++++++++++++++++++ Interface functions: +++++++++++++++++++ */
+
+/* Set random multiplicators depending on SEED. */
+static inline void
+mum_hash_randomize (uint64_t seed) {
+ int i;
+
+ srand (seed);
+ _mum_hash_step_prime = _mum_next_factor ();
+ _mum_key_step_prime = _mum_next_factor ();
+ _mum_finish_prime1 = _mum_next_factor ();
+ _mum_finish_prime2 = _mum_next_factor ();
+ _mum_block_start_prime = _mum_next_factor ();
+ _mum_unroll_prime = _mum_next_factor ();
+ _mum_tail_prime = _mum_next_factor ();
+ for (i = 0; i < (int)(sizeof (_mum_primes) / sizeof (uint64_t)); i++)
+ _mum_primes[i] = _mum_next_factor ();
+}
+
+/* Start hashing data with SEED. Return the state. */
+static inline uint64_t
+mum_hash_init (uint64_t seed) {
+ return seed;
+}
+
+/* Process data KEY with the state H and return the updated state. */
+static inline uint64_t
+mum_hash_step (uint64_t h, uint64_t key)
+{
+ return _mum (h, _mum_hash_step_prime) ^ _mum (key, _mum_key_step_prime);
+}
+
+/* Return the result of hashing using the current state H. */
+static inline uint64_t
+mum_hash_finish (uint64_t h) {
+ return _mum_final (h);
+}
+
+/* Fast hashing of KEY with SEED. The hash is always the same for the
+ same key on any target. */
+static inline size_t
+mum_hash64 (uint64_t key, uint64_t seed) {
+ return mum_hash_finish (mum_hash_step (mum_hash_init (seed), key));
+}
+
+/* Hash data KEY of length LEN and SEED. The hash depends on the
+ target endianness and the unroll factor. */
+static inline uint64_t _MUM_INLINE
+mum_hash (const void *key, size_t len, uint64_t seed) {
+ return _mum_hash_default (key, len, seed);
+}
+
+#endif
diff --git a/contrib/publicsuffix/effective_tld_names.dat b/contrib/publicsuffix/effective_tld_names.dat
new file mode 100644
index 0000000..c4b3c98
--- /dev/null
+++ b/contrib/publicsuffix/effective_tld_names.dat
@@ -0,0 +1,14206 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Please pull this list from, and only from https://publicsuffix.org/list/public_suffix_list.dat,
+// rather than any other VCS sites. Pulling from any other URL is not guaranteed to be supported.
+
+// Instructions on pulling and using this list can be found at https://publicsuffix.org/list/.
+
+// ===BEGIN ICANN DOMAINS===
+
+// ac : https://en.wikipedia.org/wiki/.ac
+ac
+com.ac
+edu.ac
+gov.ac
+net.ac
+mil.ac
+org.ac
+
+// ad : https://en.wikipedia.org/wiki/.ad
+ad
+nom.ad
+
+// ae : https://en.wikipedia.org/wiki/.ae
+// see also: "Domain Name Eligibility Policy" at http://www.aeda.ae/eng/aepolicy.php
+ae
+co.ae
+net.ae
+org.ae
+sch.ae
+ac.ae
+gov.ae
+mil.ae
+
+// aero : see https://www.information.aero/index.php?id=66
+aero
+accident-investigation.aero
+accident-prevention.aero
+aerobatic.aero
+aeroclub.aero
+aerodrome.aero
+agents.aero
+aircraft.aero
+airline.aero
+airport.aero
+air-surveillance.aero
+airtraffic.aero
+air-traffic-control.aero
+ambulance.aero
+amusement.aero
+association.aero
+author.aero
+ballooning.aero
+broker.aero
+caa.aero
+cargo.aero
+catering.aero
+certification.aero
+championship.aero
+charter.aero
+civilaviation.aero
+club.aero
+conference.aero
+consultant.aero
+consulting.aero
+control.aero
+council.aero
+crew.aero
+design.aero
+dgca.aero
+educator.aero
+emergency.aero
+engine.aero
+engineer.aero
+entertainment.aero
+equipment.aero
+exchange.aero
+express.aero
+federation.aero
+flight.aero
+fuel.aero
+gliding.aero
+government.aero
+groundhandling.aero
+group.aero
+hanggliding.aero
+homebuilt.aero
+insurance.aero
+journal.aero
+journalist.aero
+leasing.aero
+logistics.aero
+magazine.aero
+maintenance.aero
+media.aero
+microlight.aero
+modelling.aero
+navigation.aero
+parachuting.aero
+paragliding.aero
+passenger-association.aero
+pilot.aero
+press.aero
+production.aero
+recreation.aero
+repbody.aero
+res.aero
+research.aero
+rotorcraft.aero
+safety.aero
+scientist.aero
+services.aero
+show.aero
+skydiving.aero
+software.aero
+student.aero
+trader.aero
+trading.aero
+trainer.aero
+union.aero
+workinggroup.aero
+works.aero
+
+// af : http://www.nic.af/help.jsp
+af
+gov.af
+com.af
+org.af
+net.af
+edu.af
+
+// ag : http://www.nic.ag/prices.htm
+ag
+com.ag
+org.ag
+net.ag
+co.ag
+nom.ag
+
+// ai : http://nic.com.ai/
+ai
+off.ai
+com.ai
+net.ai
+org.ai
+
+// al : http://www.ert.gov.al/ert_alb/faq_det.html?Id=31
+al
+com.al
+edu.al
+gov.al
+mil.al
+net.al
+org.al
+
+// am : https://www.amnic.net/policy/en/Policy_EN.pdf
+am
+co.am
+com.am
+commune.am
+net.am
+org.am
+
+// ao : https://en.wikipedia.org/wiki/.ao
+// http://www.dns.ao/REGISTR.DOC
+ao
+ed.ao
+gv.ao
+og.ao
+co.ao
+pb.ao
+it.ao
+
+// aq : https://en.wikipedia.org/wiki/.aq
+aq
+
+// ar : https://nic.ar/nic-argentina/normativa-vigente
+ar
+com.ar
+edu.ar
+gob.ar
+gov.ar
+int.ar
+mil.ar
+musica.ar
+net.ar
+org.ar
+tur.ar
+
+// arpa : https://en.wikipedia.org/wiki/.arpa
+// Confirmed by registry <iana-questions@icann.org> 2008-06-18
+arpa
+e164.arpa
+in-addr.arpa
+ip6.arpa
+iris.arpa
+uri.arpa
+urn.arpa
+
+// as : https://en.wikipedia.org/wiki/.as
+as
+gov.as
+
+// asia : https://en.wikipedia.org/wiki/.asia
+asia
+
+// at : https://en.wikipedia.org/wiki/.at
+// Confirmed by registry <it@nic.at> 2008-06-17
+at
+ac.at
+co.at
+gv.at
+or.at
+sth.ac.at
+
+// au : https://en.wikipedia.org/wiki/.au
+// http://www.auda.org.au/
+au
+// 2LDs
+com.au
+net.au
+org.au
+edu.au
+gov.au
+asn.au
+id.au
+// Historic 2LDs (closed to new registration, but sites still exist)
+info.au
+conf.au
+oz.au
+// CGDNs - http://www.cgdn.org.au/
+act.au
+nsw.au
+nt.au
+qld.au
+sa.au
+tas.au
+vic.au
+wa.au
+// 3LDs
+act.edu.au
+catholic.edu.au
+// eq.edu.au - Removed at the request of the Queensland Department of Education
+nsw.edu.au
+nt.edu.au
+qld.edu.au
+sa.edu.au
+tas.edu.au
+vic.edu.au
+wa.edu.au
+// act.gov.au Bug 984824 - Removed at request of Greg Tankard
+// nsw.gov.au Bug 547985 - Removed at request of <Shae.Donelan@services.nsw.gov.au>
+// nt.gov.au Bug 940478 - Removed at request of Greg Connors <Greg.Connors@nt.gov.au>
+qld.gov.au
+sa.gov.au
+tas.gov.au
+vic.gov.au
+wa.gov.au
+// 4LDs
+// education.tas.edu.au - Removed at the request of the Department of Education Tasmania
+schools.nsw.edu.au
+
+// aw : https://en.wikipedia.org/wiki/.aw
+aw
+com.aw
+
+// ax : https://en.wikipedia.org/wiki/.ax
+ax
+
+// az : https://en.wikipedia.org/wiki/.az
+az
+com.az
+net.az
+int.az
+gov.az
+org.az
+edu.az
+info.az
+pp.az
+mil.az
+name.az
+pro.az
+biz.az
+
+// ba : http://nic.ba/users_data/files/pravilnik_o_registraciji.pdf
+ba
+com.ba
+edu.ba
+gov.ba
+mil.ba
+net.ba
+org.ba
+
+// bb : https://en.wikipedia.org/wiki/.bb
+bb
+biz.bb
+co.bb
+com.bb
+edu.bb
+gov.bb
+info.bb
+net.bb
+org.bb
+store.bb
+tv.bb
+
+// bd : https://en.wikipedia.org/wiki/.bd
+*.bd
+
+// be : https://en.wikipedia.org/wiki/.be
+// Confirmed by registry <tech@dns.be> 2008-06-08
+be
+ac.be
+
+// bf : https://en.wikipedia.org/wiki/.bf
+bf
+gov.bf
+
+// bg : https://en.wikipedia.org/wiki/.bg
+// https://www.register.bg/user/static/rules/en/index.html
+bg
+a.bg
+b.bg
+c.bg
+d.bg
+e.bg
+f.bg
+g.bg
+h.bg
+i.bg
+j.bg
+k.bg
+l.bg
+m.bg
+n.bg
+o.bg
+p.bg
+q.bg
+r.bg
+s.bg
+t.bg
+u.bg
+v.bg
+w.bg
+x.bg
+y.bg
+z.bg
+0.bg
+1.bg
+2.bg
+3.bg
+4.bg
+5.bg
+6.bg
+7.bg
+8.bg
+9.bg
+
+// bh : https://en.wikipedia.org/wiki/.bh
+bh
+com.bh
+edu.bh
+net.bh
+org.bh
+gov.bh
+
+// bi : https://en.wikipedia.org/wiki/.bi
+// http://whois.nic.bi/
+bi
+co.bi
+com.bi
+edu.bi
+or.bi
+org.bi
+
+// biz : https://en.wikipedia.org/wiki/.biz
+biz
+
+// bj : https://en.wikipedia.org/wiki/.bj
+bj
+asso.bj
+barreau.bj
+gouv.bj
+
+// bm : http://www.bermudanic.bm/dnr-text.txt
+bm
+com.bm
+edu.bm
+gov.bm
+net.bm
+org.bm
+
+// bn : http://www.bnnic.bn/faqs
+bn
+com.bn
+edu.bn
+gov.bn
+net.bn
+org.bn
+
+// bo : https://nic.bo/delegacion2015.php#h-1.10
+bo
+com.bo
+edu.bo
+gob.bo
+int.bo
+org.bo
+net.bo
+mil.bo
+tv.bo
+web.bo
+// Social Domains
+academia.bo
+agro.bo
+arte.bo
+blog.bo
+bolivia.bo
+ciencia.bo
+cooperativa.bo
+democracia.bo
+deporte.bo
+ecologia.bo
+economia.bo
+empresa.bo
+indigena.bo
+industria.bo
+info.bo
+medicina.bo
+movimiento.bo
+musica.bo
+natural.bo
+nombre.bo
+noticias.bo
+patria.bo
+politica.bo
+profesional.bo
+plurinacional.bo
+pueblo.bo
+revista.bo
+salud.bo
+tecnologia.bo
+tksat.bo
+transporte.bo
+wiki.bo
+
+// br : http://registro.br/dominio/categoria.html
+// Submitted by registry <fneves@registro.br>
+br
+9guacu.br
+abc.br
+adm.br
+adv.br
+agr.br
+aju.br
+am.br
+anani.br
+aparecida.br
+app.br
+arq.br
+art.br
+ato.br
+b.br
+barueri.br
+belem.br
+bhz.br
+bib.br
+bio.br
+blog.br
+bmd.br
+boavista.br
+bsb.br
+campinagrande.br
+campinas.br
+caxias.br
+cim.br
+cng.br
+cnt.br
+com.br
+contagem.br
+coop.br
+coz.br
+cri.br
+cuiaba.br
+curitiba.br
+def.br
+des.br
+det.br
+dev.br
+ecn.br
+eco.br
+edu.br
+emp.br
+enf.br
+eng.br
+esp.br
+etc.br
+eti.br
+far.br
+feira.br
+flog.br
+floripa.br
+fm.br
+fnd.br
+fortal.br
+fot.br
+foz.br
+fst.br
+g12.br
+geo.br
+ggf.br
+goiania.br
+gov.br
+// gov.br 26 states + df https://en.wikipedia.org/wiki/States_of_Brazil
+ac.gov.br
+al.gov.br
+am.gov.br
+ap.gov.br
+ba.gov.br
+ce.gov.br
+df.gov.br
+es.gov.br
+go.gov.br
+ma.gov.br
+mg.gov.br
+ms.gov.br
+mt.gov.br
+pa.gov.br
+pb.gov.br
+pe.gov.br
+pi.gov.br
+pr.gov.br
+rj.gov.br
+rn.gov.br
+ro.gov.br
+rr.gov.br
+rs.gov.br
+sc.gov.br
+se.gov.br
+sp.gov.br
+to.gov.br
+gru.br
+imb.br
+ind.br
+inf.br
+jab.br
+jampa.br
+jdf.br
+joinville.br
+jor.br
+jus.br
+leg.br
+lel.br
+log.br
+londrina.br
+macapa.br
+maceio.br
+manaus.br
+maringa.br
+mat.br
+med.br
+mil.br
+morena.br
+mp.br
+mus.br
+natal.br
+net.br
+niteroi.br
+*.nom.br
+not.br
+ntr.br
+odo.br
+ong.br
+org.br
+osasco.br
+palmas.br
+poa.br
+ppg.br
+pro.br
+psc.br
+psi.br
+pvh.br
+qsl.br
+radio.br
+rec.br
+recife.br
+rep.br
+ribeirao.br
+rio.br
+riobranco.br
+riopreto.br
+salvador.br
+sampa.br
+santamaria.br
+santoandre.br
+saobernardo.br
+saogonca.br
+seg.br
+sjc.br
+slg.br
+slz.br
+sorocaba.br
+srv.br
+taxi.br
+tc.br
+tec.br
+teo.br
+the.br
+tmp.br
+trd.br
+tur.br
+tv.br
+udi.br
+vet.br
+vix.br
+vlog.br
+wiki.br
+zlg.br
+
+// bs : http://www.nic.bs/rules.html
+bs
+com.bs
+net.bs
+org.bs
+edu.bs
+gov.bs
+
+// bt : https://en.wikipedia.org/wiki/.bt
+bt
+com.bt
+edu.bt
+gov.bt
+net.bt
+org.bt
+
+// bv : No registrations at this time.
+// Submitted by registry <jarle@uninett.no>
+bv
+
+// bw : https://en.wikipedia.org/wiki/.bw
+// http://www.gobin.info/domainname/bw.doc
+// list of other 2nd level tlds ?
+bw
+co.bw
+org.bw
+
+// by : https://en.wikipedia.org/wiki/.by
+// http://tld.by/rules_2006_en.html
+// list of other 2nd level tlds ?
+by
+gov.by
+mil.by
+// Official information does not indicate that com.by is a reserved
+// second-level domain, but it's being used as one (see www.google.com.by and
+// www.yahoo.com.by, for example), so we list it here for safety's sake.
+com.by
+
+// http://hoster.by/
+of.by
+
+// bz : https://en.wikipedia.org/wiki/.bz
+// http://www.belizenic.bz/
+bz
+com.bz
+net.bz
+org.bz
+edu.bz
+gov.bz
+
+// ca : https://en.wikipedia.org/wiki/.ca
+ca
+// ca geographical names
+ab.ca
+bc.ca
+mb.ca
+nb.ca
+nf.ca
+nl.ca
+ns.ca
+nt.ca
+nu.ca
+on.ca
+pe.ca
+qc.ca
+sk.ca
+yk.ca
+// gc.ca: https://en.wikipedia.org/wiki/.gc.ca
+// see also: http://registry.gc.ca/en/SubdomainFAQ
+gc.ca
+
+// cat : https://en.wikipedia.org/wiki/.cat
+cat
+
+// cc : https://en.wikipedia.org/wiki/.cc
+cc
+
+// cd : https://en.wikipedia.org/wiki/.cd
+// see also: https://www.nic.cd/domain/insertDomain_2.jsp?act=1
+cd
+gov.cd
+
+// cf : https://en.wikipedia.org/wiki/.cf
+cf
+
+// cg : https://en.wikipedia.org/wiki/.cg
+cg
+
+// ch : https://en.wikipedia.org/wiki/.ch
+ch
+
+// ci : https://en.wikipedia.org/wiki/.ci
+// http://www.nic.ci/index.php?page=charte
+ci
+org.ci
+or.ci
+com.ci
+co.ci
+edu.ci
+ed.ci
+ac.ci
+net.ci
+go.ci
+asso.ci
+xn--aroport-bya.ci
+aéroport.ci
+int.ci
+presse.ci
+md.ci
+gouv.ci
+
+// ck : https://en.wikipedia.org/wiki/.ck
+*.ck
+!www.ck
+
+// cl : https://www.nic.cl
+// Confirmed by .CL registry <hsalgado@nic.cl>
+cl
+co.cl
+gob.cl
+gov.cl
+mil.cl
+
+// cm : https://en.wikipedia.org/wiki/.cm plus bug 981927
+cm
+co.cm
+com.cm
+gov.cm
+net.cm
+
+// cn : https://en.wikipedia.org/wiki/.cn
+// Submitted by registry <tanyaling@cnnic.cn>
+cn
+ac.cn
+com.cn
+edu.cn
+gov.cn
+net.cn
+org.cn
+mil.cn
+xn--55qx5d.cn
+å…¬å¸.cn
+xn--io0a7i.cn
+网络.cn
+xn--od0alg.cn
+網絡.cn
+// cn geographic names
+ah.cn
+bj.cn
+cq.cn
+fj.cn
+gd.cn
+gs.cn
+gz.cn
+gx.cn
+ha.cn
+hb.cn
+he.cn
+hi.cn
+hl.cn
+hn.cn
+jl.cn
+js.cn
+jx.cn
+ln.cn
+nm.cn
+nx.cn
+qh.cn
+sc.cn
+sd.cn
+sh.cn
+sn.cn
+sx.cn
+tj.cn
+xj.cn
+xz.cn
+yn.cn
+zj.cn
+hk.cn
+mo.cn
+tw.cn
+
+// co : https://en.wikipedia.org/wiki/.co
+// Submitted by registry <tecnico@uniandes.edu.co>
+co
+arts.co
+com.co
+edu.co
+firm.co
+gov.co
+info.co
+int.co
+mil.co
+net.co
+nom.co
+org.co
+rec.co
+web.co
+
+// com : https://en.wikipedia.org/wiki/.com
+com
+
+// coop : https://en.wikipedia.org/wiki/.coop
+coop
+
+// cr : http://www.nic.cr/niccr_publico/showRegistroDominiosScreen.do
+cr
+ac.cr
+co.cr
+ed.cr
+fi.cr
+go.cr
+or.cr
+sa.cr
+
+// cu : https://en.wikipedia.org/wiki/.cu
+cu
+com.cu
+edu.cu
+org.cu
+net.cu
+gov.cu
+inf.cu
+
+// cv : https://en.wikipedia.org/wiki/.cv
+cv
+
+// cw : http://www.una.cw/cw_registry/
+// Confirmed by registry <registry@una.net> 2013-03-26
+cw
+com.cw
+edu.cw
+net.cw
+org.cw
+
+// cx : https://en.wikipedia.org/wiki/.cx
+// list of other 2nd level tlds ?
+cx
+gov.cx
+
+// cy : http://www.nic.cy/
+// Submitted by registry Panayiotou Fotia <cydns@ucy.ac.cy>
+cy
+ac.cy
+biz.cy
+com.cy
+ekloges.cy
+gov.cy
+ltd.cy
+name.cy
+net.cy
+org.cy
+parliament.cy
+press.cy
+pro.cy
+tm.cy
+
+// cz : https://en.wikipedia.org/wiki/.cz
+cz
+
+// de : https://en.wikipedia.org/wiki/.de
+// Confirmed by registry <ops@denic.de> (with technical
+// reservations) 2008-07-01
+de
+
+// dj : https://en.wikipedia.org/wiki/.dj
+dj
+
+// dk : https://en.wikipedia.org/wiki/.dk
+// Confirmed by registry <robert@dk-hostmaster.dk> 2008-06-17
+dk
+
+// dm : https://en.wikipedia.org/wiki/.dm
+dm
+com.dm
+net.dm
+org.dm
+edu.dm
+gov.dm
+
+// do : https://en.wikipedia.org/wiki/.do
+do
+art.do
+com.do
+edu.do
+gob.do
+gov.do
+mil.do
+net.do
+org.do
+sld.do
+web.do
+
+// dz : http://www.nic.dz/images/pdf_nic/charte.pdf
+dz
+art.dz
+asso.dz
+com.dz
+edu.dz
+gov.dz
+org.dz
+net.dz
+pol.dz
+soc.dz
+tm.dz
+
+// ec : http://www.nic.ec/reg/paso1.asp
+// Submitted by registry <vabboud@nic.ec>
+ec
+com.ec
+info.ec
+net.ec
+fin.ec
+k12.ec
+med.ec
+pro.ec
+org.ec
+edu.ec
+gov.ec
+gob.ec
+mil.ec
+
+// edu : https://en.wikipedia.org/wiki/.edu
+edu
+
+// ee : http://www.eenet.ee/EENet/dom_reeglid.html#lisa_B
+ee
+edu.ee
+gov.ee
+riik.ee
+lib.ee
+med.ee
+com.ee
+pri.ee
+aip.ee
+org.ee
+fie.ee
+
+// eg : https://en.wikipedia.org/wiki/.eg
+eg
+com.eg
+edu.eg
+eun.eg
+gov.eg
+mil.eg
+name.eg
+net.eg
+org.eg
+sci.eg
+
+// er : https://en.wikipedia.org/wiki/.er
+*.er
+
+// es : https://www.nic.es/site_ingles/ingles/dominios/index.html
+es
+com.es
+nom.es
+org.es
+gob.es
+edu.es
+
+// et : https://en.wikipedia.org/wiki/.et
+et
+com.et
+gov.et
+org.et
+edu.et
+biz.et
+name.et
+info.et
+net.et
+
+// eu : https://en.wikipedia.org/wiki/.eu
+eu
+
+// fi : https://en.wikipedia.org/wiki/.fi
+fi
+// aland.fi : https://en.wikipedia.org/wiki/.ax
+// This domain is being phased out in favor of .ax. As there are still many
+// domains under aland.fi, we still keep it on the list until aland.fi is
+// completely removed.
+// TODO: Check for updates (expected to be phased out around Q1/2009)
+aland.fi
+
+// fj : http://domains.fj/
+// Submitted by registry <garth.miller@cocca.org.nz> 2020-02-11
+fj
+ac.fj
+biz.fj
+com.fj
+gov.fj
+info.fj
+mil.fj
+name.fj
+net.fj
+org.fj
+pro.fj
+
+// fk : https://en.wikipedia.org/wiki/.fk
+*.fk
+
+// fm : https://en.wikipedia.org/wiki/.fm
+com.fm
+edu.fm
+net.fm
+org.fm
+fm
+
+// fo : https://en.wikipedia.org/wiki/.fo
+fo
+
+// fr : http://www.afnic.fr/
+// domaines descriptifs : https://www.afnic.fr/medias/documents/Cadre_legal/Afnic_Naming_Policy_12122016_VEN.pdf
+fr
+asso.fr
+com.fr
+gouv.fr
+nom.fr
+prd.fr
+tm.fr
+// domaines sectoriels : https://www.afnic.fr/en/products-and-services/the-fr-tld/sector-based-fr-domains-4.html
+aeroport.fr
+avocat.fr
+avoues.fr
+cci.fr
+chambagri.fr
+chirurgiens-dentistes.fr
+experts-comptables.fr
+geometre-expert.fr
+greta.fr
+huissier-justice.fr
+medecin.fr
+notaires.fr
+pharmacien.fr
+port.fr
+veterinaire.fr
+
+// ga : https://en.wikipedia.org/wiki/.ga
+ga
+
+// gb : This registry is effectively dormant
+// Submitted by registry <Damien.Shaw@ja.net>
+gb
+
+// gd : https://en.wikipedia.org/wiki/.gd
+edu.gd
+gov.gd
+gd
+
+// ge : http://www.nic.net.ge/policy_en.pdf
+ge
+com.ge
+edu.ge
+gov.ge
+org.ge
+mil.ge
+net.ge
+pvt.ge
+
+// gf : https://en.wikipedia.org/wiki/.gf
+gf
+
+// gg : http://www.channelisles.net/register-domains/
+// Confirmed by registry <nigel@channelisles.net> 2013-11-28
+gg
+co.gg
+net.gg
+org.gg
+
+// gh : https://en.wikipedia.org/wiki/.gh
+// see also: http://www.nic.gh/reg_now.php
+// Although domains directly at second level are not possible at the moment,
+// they have been possible for some time and may come back.
+gh
+com.gh
+edu.gh
+gov.gh
+org.gh
+mil.gh
+
+// gi : http://www.nic.gi/rules.html
+gi
+com.gi
+ltd.gi
+gov.gi
+mod.gi
+edu.gi
+org.gi
+
+// gl : https://en.wikipedia.org/wiki/.gl
+// http://nic.gl
+gl
+co.gl
+com.gl
+edu.gl
+net.gl
+org.gl
+
+// gm : http://www.nic.gm/htmlpages%5Cgm-policy.htm
+gm
+
+// gn : http://psg.com/dns/gn/gn.txt
+// Submitted by registry <randy@psg.com>
+gn
+ac.gn
+com.gn
+edu.gn
+gov.gn
+org.gn
+net.gn
+
+// gov : https://en.wikipedia.org/wiki/.gov
+gov
+
+// gp : http://www.nic.gp/index.php?lang=en
+gp
+com.gp
+net.gp
+mobi.gp
+edu.gp
+org.gp
+asso.gp
+
+// gq : https://en.wikipedia.org/wiki/.gq
+gq
+
+// gr : https://grweb.ics.forth.gr/english/1617-B-2005.html
+// Submitted by registry <segred@ics.forth.gr>
+gr
+com.gr
+edu.gr
+net.gr
+org.gr
+gov.gr
+
+// gs : https://en.wikipedia.org/wiki/.gs
+gs
+
+// gt : https://www.gt/sitio/registration_policy.php?lang=en
+gt
+com.gt
+edu.gt
+gob.gt
+ind.gt
+mil.gt
+net.gt
+org.gt
+
+// gu : http://gadao.gov.gu/register.html
+// University of Guam : https://www.uog.edu
+// Submitted by uognoc@triton.uog.edu
+gu
+com.gu
+edu.gu
+gov.gu
+guam.gu
+info.gu
+net.gu
+org.gu
+web.gu
+
+// gw : https://en.wikipedia.org/wiki/.gw
+gw
+
+// gy : https://en.wikipedia.org/wiki/.gy
+// http://registry.gy/
+gy
+co.gy
+com.gy
+edu.gy
+gov.gy
+net.gy
+org.gy
+
+// hk : https://www.hkirc.hk
+// Submitted by registry <hk.tech@hkirc.hk>
+hk
+com.hk
+edu.hk
+gov.hk
+idv.hk
+net.hk
+org.hk
+xn--55qx5d.hk
+å…¬å¸.hk
+xn--wcvs22d.hk
+教育.hk
+xn--lcvr32d.hk
+敎育.hk
+xn--mxtq1m.hk
+政府.hk
+xn--gmqw5a.hk
+個人.hk
+xn--ciqpn.hk
+个人.hk
+xn--gmq050i.hk
+箇人.hk
+xn--zf0avx.hk
+網络.hk
+xn--io0a7i.hk
+网络.hk
+xn--mk0axi.hk
+组織.hk
+xn--od0alg.hk
+網絡.hk
+xn--od0aq3b.hk
+网絡.hk
+xn--tn0ag.hk
+组织.hk
+xn--uc0atv.hk
+組織.hk
+xn--uc0ay4a.hk
+組织.hk
+
+// hm : https://en.wikipedia.org/wiki/.hm
+hm
+
+// hn : http://www.nic.hn/politicas/ps02,,05.html
+hn
+com.hn
+edu.hn
+org.hn
+net.hn
+mil.hn
+gob.hn
+
+// hr : http://www.dns.hr/documents/pdf/HRTLD-regulations.pdf
+hr
+iz.hr
+from.hr
+name.hr
+com.hr
+
+// ht : http://www.nic.ht/info/charte.cfm
+ht
+com.ht
+shop.ht
+firm.ht
+info.ht
+adult.ht
+net.ht
+pro.ht
+org.ht
+med.ht
+art.ht
+coop.ht
+pol.ht
+asso.ht
+edu.ht
+rel.ht
+gouv.ht
+perso.ht
+
+// hu : http://www.domain.hu/domain/English/sld.html
+// Confirmed by registry <pasztor@iszt.hu> 2008-06-12
+hu
+co.hu
+info.hu
+org.hu
+priv.hu
+sport.hu
+tm.hu
+2000.hu
+agrar.hu
+bolt.hu
+casino.hu
+city.hu
+erotica.hu
+erotika.hu
+film.hu
+forum.hu
+games.hu
+hotel.hu
+ingatlan.hu
+jogasz.hu
+konyvelo.hu
+lakas.hu
+media.hu
+news.hu
+reklam.hu
+sex.hu
+shop.hu
+suli.hu
+szex.hu
+tozsde.hu
+utazas.hu
+video.hu
+
+// id : https://pandi.id/en/domain/registration-requirements/
+id
+ac.id
+biz.id
+co.id
+desa.id
+go.id
+mil.id
+my.id
+net.id
+or.id
+ponpes.id
+sch.id
+web.id
+
+// ie : https://en.wikipedia.org/wiki/.ie
+ie
+gov.ie
+
+// il : http://www.isoc.org.il/domains/
+il
+ac.il
+co.il
+gov.il
+idf.il
+k12.il
+muni.il
+net.il
+org.il
+
+// im : https://www.nic.im/
+// Submitted by registry <info@nic.im>
+im
+ac.im
+co.im
+com.im
+ltd.co.im
+net.im
+org.im
+plc.co.im
+tt.im
+tv.im
+
+// in : https://en.wikipedia.org/wiki/.in
+// see also: https://registry.in/Policies
+// Please note, that nic.in is not an official eTLD, but used by most
+// government institutions.
+in
+co.in
+firm.in
+net.in
+org.in
+gen.in
+ind.in
+nic.in
+ac.in
+edu.in
+res.in
+gov.in
+mil.in
+
+// info : https://en.wikipedia.org/wiki/.info
+info
+
+// int : https://en.wikipedia.org/wiki/.int
+// Confirmed by registry <iana-questions@icann.org> 2008-06-18
+int
+eu.int
+
+// io : http://www.nic.io/rules.html
+// list of other 2nd level tlds ?
+io
+com.io
+
+// iq : http://www.cmc.iq/english/iq/iqregister1.htm
+iq
+gov.iq
+edu.iq
+mil.iq
+com.iq
+org.iq
+net.iq
+
+// ir : http://www.nic.ir/Terms_and_Conditions_ir,_Appendix_1_Domain_Rules
+// Also see http://www.nic.ir/Internationalized_Domain_Names
+// Two <iran>.ir entries added at request of <tech-team@nic.ir>, 2010-04-16
+ir
+ac.ir
+co.ir
+gov.ir
+id.ir
+net.ir
+org.ir
+sch.ir
+// xn--mgba3a4f16a.ir (<iran>.ir, Persian YEH)
+xn--mgba3a4f16a.ir
+ایران.ir
+// xn--mgba3a4fra.ir (<iran>.ir, Arabic YEH)
+xn--mgba3a4fra.ir
+ايران.ir
+
+// is : http://www.isnic.is/domain/rules.php
+// Confirmed by registry <marius@isgate.is> 2008-12-06
+is
+net.is
+com.is
+edu.is
+gov.is
+org.is
+int.is
+
+// it : https://en.wikipedia.org/wiki/.it
+it
+gov.it
+edu.it
+// Reserved geo-names (regions and provinces):
+// https://www.nic.it/sites/default/files/archivio/docs/Regulation_assignation_v7.1.pdf
+// Regions
+abr.it
+abruzzo.it
+aosta-valley.it
+aostavalley.it
+bas.it
+basilicata.it
+cal.it
+calabria.it
+cam.it
+campania.it
+emilia-romagna.it
+emiliaromagna.it
+emr.it
+friuli-v-giulia.it
+friuli-ve-giulia.it
+friuli-vegiulia.it
+friuli-venezia-giulia.it
+friuli-veneziagiulia.it
+friuli-vgiulia.it
+friuliv-giulia.it
+friulive-giulia.it
+friulivegiulia.it
+friulivenezia-giulia.it
+friuliveneziagiulia.it
+friulivgiulia.it
+fvg.it
+laz.it
+lazio.it
+lig.it
+liguria.it
+lom.it
+lombardia.it
+lombardy.it
+lucania.it
+mar.it
+marche.it
+mol.it
+molise.it
+piedmont.it
+piemonte.it
+pmn.it
+pug.it
+puglia.it
+sar.it
+sardegna.it
+sardinia.it
+sic.it
+sicilia.it
+sicily.it
+taa.it
+tos.it
+toscana.it
+trentin-sud-tirol.it
+xn--trentin-sd-tirol-rzb.it
+trentin-süd-tirol.it
+trentin-sudtirol.it
+xn--trentin-sdtirol-7vb.it
+trentin-südtirol.it
+trentin-sued-tirol.it
+trentin-suedtirol.it
+trentino-a-adige.it
+trentino-aadige.it
+trentino-alto-adige.it
+trentino-altoadige.it
+trentino-s-tirol.it
+trentino-stirol.it
+trentino-sud-tirol.it
+xn--trentino-sd-tirol-c3b.it
+trentino-süd-tirol.it
+trentino-sudtirol.it
+xn--trentino-sdtirol-szb.it
+trentino-südtirol.it
+trentino-sued-tirol.it
+trentino-suedtirol.it
+trentino.it
+trentinoa-adige.it
+trentinoaadige.it
+trentinoalto-adige.it
+trentinoaltoadige.it
+trentinos-tirol.it
+trentinostirol.it
+trentinosud-tirol.it
+xn--trentinosd-tirol-rzb.it
+trentinosüd-tirol.it
+trentinosudtirol.it
+xn--trentinosdtirol-7vb.it
+trentinosüdtirol.it
+trentinosued-tirol.it
+trentinosuedtirol.it
+trentinsud-tirol.it
+xn--trentinsd-tirol-6vb.it
+trentinsüd-tirol.it
+trentinsudtirol.it
+xn--trentinsdtirol-nsb.it
+trentinsüdtirol.it
+trentinsued-tirol.it
+trentinsuedtirol.it
+tuscany.it
+umb.it
+umbria.it
+val-d-aosta.it
+val-daosta.it
+vald-aosta.it
+valdaosta.it
+valle-aosta.it
+valle-d-aosta.it
+valle-daosta.it
+valleaosta.it
+valled-aosta.it
+valledaosta.it
+vallee-aoste.it
+xn--valle-aoste-ebb.it
+vallée-aoste.it
+vallee-d-aoste.it
+xn--valle-d-aoste-ehb.it
+vallée-d-aoste.it
+valleeaoste.it
+xn--valleaoste-e7a.it
+valléeaoste.it
+valleedaoste.it
+xn--valledaoste-ebb.it
+valléedaoste.it
+vao.it
+vda.it
+ven.it
+veneto.it
+// Provinces
+ag.it
+agrigento.it
+al.it
+alessandria.it
+alto-adige.it
+altoadige.it
+an.it
+ancona.it
+andria-barletta-trani.it
+andria-trani-barletta.it
+andriabarlettatrani.it
+andriatranibarletta.it
+ao.it
+aosta.it
+aoste.it
+ap.it
+aq.it
+aquila.it
+ar.it
+arezzo.it
+ascoli-piceno.it
+ascolipiceno.it
+asti.it
+at.it
+av.it
+avellino.it
+ba.it
+balsan-sudtirol.it
+xn--balsan-sdtirol-nsb.it
+balsan-südtirol.it
+balsan-suedtirol.it
+balsan.it
+bari.it
+barletta-trani-andria.it
+barlettatraniandria.it
+belluno.it
+benevento.it
+bergamo.it
+bg.it
+bi.it
+biella.it
+bl.it
+bn.it
+bo.it
+bologna.it
+bolzano-altoadige.it
+bolzano.it
+bozen-sudtirol.it
+xn--bozen-sdtirol-2ob.it
+bozen-südtirol.it
+bozen-suedtirol.it
+bozen.it
+br.it
+brescia.it
+brindisi.it
+bs.it
+bt.it
+bulsan-sudtirol.it
+xn--bulsan-sdtirol-nsb.it
+bulsan-südtirol.it
+bulsan-suedtirol.it
+bulsan.it
+bz.it
+ca.it
+cagliari.it
+caltanissetta.it
+campidano-medio.it
+campidanomedio.it
+campobasso.it
+carbonia-iglesias.it
+carboniaiglesias.it
+carrara-massa.it
+carraramassa.it
+caserta.it
+catania.it
+catanzaro.it
+cb.it
+ce.it
+cesena-forli.it
+xn--cesena-forl-mcb.it
+cesena-forlì.it
+cesenaforli.it
+xn--cesenaforl-i8a.it
+cesenaforlì.it
+ch.it
+chieti.it
+ci.it
+cl.it
+cn.it
+co.it
+como.it
+cosenza.it
+cr.it
+cremona.it
+crotone.it
+cs.it
+ct.it
+cuneo.it
+cz.it
+dell-ogliastra.it
+dellogliastra.it
+en.it
+enna.it
+fc.it
+fe.it
+fermo.it
+ferrara.it
+fg.it
+fi.it
+firenze.it
+florence.it
+fm.it
+foggia.it
+forli-cesena.it
+xn--forl-cesena-fcb.it
+forlì-cesena.it
+forlicesena.it
+xn--forlcesena-c8a.it
+forlìcesena.it
+fr.it
+frosinone.it
+ge.it
+genoa.it
+genova.it
+go.it
+gorizia.it
+gr.it
+grosseto.it
+iglesias-carbonia.it
+iglesiascarbonia.it
+im.it
+imperia.it
+is.it
+isernia.it
+kr.it
+la-spezia.it
+laquila.it
+laspezia.it
+latina.it
+lc.it
+le.it
+lecce.it
+lecco.it
+li.it
+livorno.it
+lo.it
+lodi.it
+lt.it
+lu.it
+lucca.it
+macerata.it
+mantova.it
+massa-carrara.it
+massacarrara.it
+matera.it
+mb.it
+mc.it
+me.it
+medio-campidano.it
+mediocampidano.it
+messina.it
+mi.it
+milan.it
+milano.it
+mn.it
+mo.it
+modena.it
+monza-brianza.it
+monza-e-della-brianza.it
+monza.it
+monzabrianza.it
+monzaebrianza.it
+monzaedellabrianza.it
+ms.it
+mt.it
+na.it
+naples.it
+napoli.it
+no.it
+novara.it
+nu.it
+nuoro.it
+og.it
+ogliastra.it
+olbia-tempio.it
+olbiatempio.it
+or.it
+oristano.it
+ot.it
+pa.it
+padova.it
+padua.it
+palermo.it
+parma.it
+pavia.it
+pc.it
+pd.it
+pe.it
+perugia.it
+pesaro-urbino.it
+pesarourbino.it
+pescara.it
+pg.it
+pi.it
+piacenza.it
+pisa.it
+pistoia.it
+pn.it
+po.it
+pordenone.it
+potenza.it
+pr.it
+prato.it
+pt.it
+pu.it
+pv.it
+pz.it
+ra.it
+ragusa.it
+ravenna.it
+rc.it
+re.it
+reggio-calabria.it
+reggio-emilia.it
+reggiocalabria.it
+reggioemilia.it
+rg.it
+ri.it
+rieti.it
+rimini.it
+rm.it
+rn.it
+ro.it
+roma.it
+rome.it
+rovigo.it
+sa.it
+salerno.it
+sassari.it
+savona.it
+si.it
+siena.it
+siracusa.it
+so.it
+sondrio.it
+sp.it
+sr.it
+ss.it
+suedtirol.it
+xn--sdtirol-n2a.it
+südtirol.it
+sv.it
+ta.it
+taranto.it
+te.it
+tempio-olbia.it
+tempioolbia.it
+teramo.it
+terni.it
+tn.it
+to.it
+torino.it
+tp.it
+tr.it
+trani-andria-barletta.it
+trani-barletta-andria.it
+traniandriabarletta.it
+tranibarlettaandria.it
+trapani.it
+trento.it
+treviso.it
+trieste.it
+ts.it
+turin.it
+tv.it
+ud.it
+udine.it
+urbino-pesaro.it
+urbinopesaro.it
+va.it
+varese.it
+vb.it
+vc.it
+ve.it
+venezia.it
+venice.it
+verbania.it
+vercelli.it
+verona.it
+vi.it
+vibo-valentia.it
+vibovalentia.it
+vicenza.it
+viterbo.it
+vr.it
+vs.it
+vt.it
+vv.it
+
+// je : http://www.channelisles.net/register-domains/
+// Confirmed by registry <nigel@channelisles.net> 2013-11-28
+je
+co.je
+net.je
+org.je
+
+// jm : http://www.com.jm/register.html
+*.jm
+
+// jo : http://www.dns.jo/Registration_policy.aspx
+jo
+com.jo
+org.jo
+net.jo
+edu.jo
+sch.jo
+gov.jo
+mil.jo
+name.jo
+
+// jobs : https://en.wikipedia.org/wiki/.jobs
+jobs
+
+// jp : https://en.wikipedia.org/wiki/.jp
+// http://jprs.co.jp/en/jpdomain.html
+// Submitted by registry <info@jprs.jp>
+jp
+// jp organizational type names
+ac.jp
+ad.jp
+co.jp
+ed.jp
+go.jp
+gr.jp
+lg.jp
+ne.jp
+or.jp
+// jp prefecture type names
+aichi.jp
+akita.jp
+aomori.jp
+chiba.jp
+ehime.jp
+fukui.jp
+fukuoka.jp
+fukushima.jp
+gifu.jp
+gunma.jp
+hiroshima.jp
+hokkaido.jp
+hyogo.jp
+ibaraki.jp
+ishikawa.jp
+iwate.jp
+kagawa.jp
+kagoshima.jp
+kanagawa.jp
+kochi.jp
+kumamoto.jp
+kyoto.jp
+mie.jp
+miyagi.jp
+miyazaki.jp
+nagano.jp
+nagasaki.jp
+nara.jp
+niigata.jp
+oita.jp
+okayama.jp
+okinawa.jp
+osaka.jp
+saga.jp
+saitama.jp
+shiga.jp
+shimane.jp
+shizuoka.jp
+tochigi.jp
+tokushima.jp
+tokyo.jp
+tottori.jp
+toyama.jp
+wakayama.jp
+yamagata.jp
+yamaguchi.jp
+yamanashi.jp
+xn--4pvxs.jp
+栃木.jp
+xn--vgu402c.jp
+愛知.jp
+xn--c3s14m.jp
+愛媛.jp
+xn--f6qx53a.jp
+兵庫.jp
+xn--8pvr4u.jp
+熊本.jp
+xn--uist22h.jp
+茨城.jp
+xn--djrs72d6uy.jp
+北海é“.jp
+xn--mkru45i.jp
+åƒè‘‰.jp
+xn--0trq7p7nn.jp
+和歌山.jp
+xn--8ltr62k.jp
+é•·å´Ž.jp
+xn--2m4a15e.jp
+長野.jp
+xn--efvn9s.jp
+新潟.jp
+xn--32vp30h.jp
+é’森.jp
+xn--4it797k.jp
+é™å²¡.jp
+xn--1lqs71d.jp
+æ±äº¬.jp
+xn--5rtp49c.jp
+石å·.jp
+xn--5js045d.jp
+埼玉.jp
+xn--ehqz56n.jp
+三é‡.jp
+xn--1lqs03n.jp
+京都.jp
+xn--qqqt11m.jp
+ä½è³€.jp
+xn--kbrq7o.jp
+大分.jp
+xn--pssu33l.jp
+大阪.jp
+xn--ntsq17g.jp
+奈良.jp
+xn--uisz3g.jp
+宮城.jp
+xn--6btw5a.jp
+宮崎.jp
+xn--1ctwo.jp
+富山.jp
+xn--6orx2r.jp
+å±±å£.jp
+xn--rht61e.jp
+山形.jp
+xn--rht27z.jp
+山梨.jp
+xn--djty4k.jp
+岩手.jp
+xn--nit225k.jp
+å²é˜œ.jp
+xn--rht3d.jp
+岡山.jp
+xn--klty5x.jp
+島根.jp
+xn--kltx9a.jp
+広島.jp
+xn--kltp7d.jp
+徳島.jp
+xn--uuwu58a.jp
+沖縄.jp
+xn--zbx025d.jp
+滋賀.jp
+xn--ntso0iqx3a.jp
+神奈å·.jp
+xn--elqq16h.jp
+ç¦äº•.jp
+xn--4it168d.jp
+ç¦å²¡.jp
+xn--klt787d.jp
+ç¦å³¶.jp
+xn--rny31h.jp
+秋田.jp
+xn--7t0a264c.jp
+群馬.jp
+xn--5rtq34k.jp
+香å·.jp
+xn--k7yn95e.jp
+高知.jp
+xn--tor131o.jp
+é³¥å–.jp
+xn--d5qv7z876c.jp
+鹿å…島.jp
+// jp geographic type names
+// http://jprs.jp/doc/rule/saisoku-1.html
+*.kawasaki.jp
+*.kitakyushu.jp
+*.kobe.jp
+*.nagoya.jp
+*.sapporo.jp
+*.sendai.jp
+*.yokohama.jp
+!city.kawasaki.jp
+!city.kitakyushu.jp
+!city.kobe.jp
+!city.nagoya.jp
+!city.sapporo.jp
+!city.sendai.jp
+!city.yokohama.jp
+// 4th level registration
+aisai.aichi.jp
+ama.aichi.jp
+anjo.aichi.jp
+asuke.aichi.jp
+chiryu.aichi.jp
+chita.aichi.jp
+fuso.aichi.jp
+gamagori.aichi.jp
+handa.aichi.jp
+hazu.aichi.jp
+hekinan.aichi.jp
+higashiura.aichi.jp
+ichinomiya.aichi.jp
+inazawa.aichi.jp
+inuyama.aichi.jp
+isshiki.aichi.jp
+iwakura.aichi.jp
+kanie.aichi.jp
+kariya.aichi.jp
+kasugai.aichi.jp
+kira.aichi.jp
+kiyosu.aichi.jp
+komaki.aichi.jp
+konan.aichi.jp
+kota.aichi.jp
+mihama.aichi.jp
+miyoshi.aichi.jp
+nishio.aichi.jp
+nisshin.aichi.jp
+obu.aichi.jp
+oguchi.aichi.jp
+oharu.aichi.jp
+okazaki.aichi.jp
+owariasahi.aichi.jp
+seto.aichi.jp
+shikatsu.aichi.jp
+shinshiro.aichi.jp
+shitara.aichi.jp
+tahara.aichi.jp
+takahama.aichi.jp
+tobishima.aichi.jp
+toei.aichi.jp
+togo.aichi.jp
+tokai.aichi.jp
+tokoname.aichi.jp
+toyoake.aichi.jp
+toyohashi.aichi.jp
+toyokawa.aichi.jp
+toyone.aichi.jp
+toyota.aichi.jp
+tsushima.aichi.jp
+yatomi.aichi.jp
+akita.akita.jp
+daisen.akita.jp
+fujisato.akita.jp
+gojome.akita.jp
+hachirogata.akita.jp
+happou.akita.jp
+higashinaruse.akita.jp
+honjo.akita.jp
+honjyo.akita.jp
+ikawa.akita.jp
+kamikoani.akita.jp
+kamioka.akita.jp
+katagami.akita.jp
+kazuno.akita.jp
+kitaakita.akita.jp
+kosaka.akita.jp
+kyowa.akita.jp
+misato.akita.jp
+mitane.akita.jp
+moriyoshi.akita.jp
+nikaho.akita.jp
+noshiro.akita.jp
+odate.akita.jp
+oga.akita.jp
+ogata.akita.jp
+semboku.akita.jp
+yokote.akita.jp
+yurihonjo.akita.jp
+aomori.aomori.jp
+gonohe.aomori.jp
+hachinohe.aomori.jp
+hashikami.aomori.jp
+hiranai.aomori.jp
+hirosaki.aomori.jp
+itayanagi.aomori.jp
+kuroishi.aomori.jp
+misawa.aomori.jp
+mutsu.aomori.jp
+nakadomari.aomori.jp
+noheji.aomori.jp
+oirase.aomori.jp
+owani.aomori.jp
+rokunohe.aomori.jp
+sannohe.aomori.jp
+shichinohe.aomori.jp
+shingo.aomori.jp
+takko.aomori.jp
+towada.aomori.jp
+tsugaru.aomori.jp
+tsuruta.aomori.jp
+abiko.chiba.jp
+asahi.chiba.jp
+chonan.chiba.jp
+chosei.chiba.jp
+choshi.chiba.jp
+chuo.chiba.jp
+funabashi.chiba.jp
+futtsu.chiba.jp
+hanamigawa.chiba.jp
+ichihara.chiba.jp
+ichikawa.chiba.jp
+ichinomiya.chiba.jp
+inzai.chiba.jp
+isumi.chiba.jp
+kamagaya.chiba.jp
+kamogawa.chiba.jp
+kashiwa.chiba.jp
+katori.chiba.jp
+katsuura.chiba.jp
+kimitsu.chiba.jp
+kisarazu.chiba.jp
+kozaki.chiba.jp
+kujukuri.chiba.jp
+kyonan.chiba.jp
+matsudo.chiba.jp
+midori.chiba.jp
+mihama.chiba.jp
+minamiboso.chiba.jp
+mobara.chiba.jp
+mutsuzawa.chiba.jp
+nagara.chiba.jp
+nagareyama.chiba.jp
+narashino.chiba.jp
+narita.chiba.jp
+noda.chiba.jp
+oamishirasato.chiba.jp
+omigawa.chiba.jp
+onjuku.chiba.jp
+otaki.chiba.jp
+sakae.chiba.jp
+sakura.chiba.jp
+shimofusa.chiba.jp
+shirako.chiba.jp
+shiroi.chiba.jp
+shisui.chiba.jp
+sodegaura.chiba.jp
+sosa.chiba.jp
+tako.chiba.jp
+tateyama.chiba.jp
+togane.chiba.jp
+tohnosho.chiba.jp
+tomisato.chiba.jp
+urayasu.chiba.jp
+yachimata.chiba.jp
+yachiyo.chiba.jp
+yokaichiba.chiba.jp
+yokoshibahikari.chiba.jp
+yotsukaido.chiba.jp
+ainan.ehime.jp
+honai.ehime.jp
+ikata.ehime.jp
+imabari.ehime.jp
+iyo.ehime.jp
+kamijima.ehime.jp
+kihoku.ehime.jp
+kumakogen.ehime.jp
+masaki.ehime.jp
+matsuno.ehime.jp
+matsuyama.ehime.jp
+namikata.ehime.jp
+niihama.ehime.jp
+ozu.ehime.jp
+saijo.ehime.jp
+seiyo.ehime.jp
+shikokuchuo.ehime.jp
+tobe.ehime.jp
+toon.ehime.jp
+uchiko.ehime.jp
+uwajima.ehime.jp
+yawatahama.ehime.jp
+echizen.fukui.jp
+eiheiji.fukui.jp
+fukui.fukui.jp
+ikeda.fukui.jp
+katsuyama.fukui.jp
+mihama.fukui.jp
+minamiechizen.fukui.jp
+obama.fukui.jp
+ohi.fukui.jp
+ono.fukui.jp
+sabae.fukui.jp
+sakai.fukui.jp
+takahama.fukui.jp
+tsuruga.fukui.jp
+wakasa.fukui.jp
+ashiya.fukuoka.jp
+buzen.fukuoka.jp
+chikugo.fukuoka.jp
+chikuho.fukuoka.jp
+chikujo.fukuoka.jp
+chikushino.fukuoka.jp
+chikuzen.fukuoka.jp
+chuo.fukuoka.jp
+dazaifu.fukuoka.jp
+fukuchi.fukuoka.jp
+hakata.fukuoka.jp
+higashi.fukuoka.jp
+hirokawa.fukuoka.jp
+hisayama.fukuoka.jp
+iizuka.fukuoka.jp
+inatsuki.fukuoka.jp
+kaho.fukuoka.jp
+kasuga.fukuoka.jp
+kasuya.fukuoka.jp
+kawara.fukuoka.jp
+keisen.fukuoka.jp
+koga.fukuoka.jp
+kurate.fukuoka.jp
+kurogi.fukuoka.jp
+kurume.fukuoka.jp
+minami.fukuoka.jp
+miyako.fukuoka.jp
+miyama.fukuoka.jp
+miyawaka.fukuoka.jp
+mizumaki.fukuoka.jp
+munakata.fukuoka.jp
+nakagawa.fukuoka.jp
+nakama.fukuoka.jp
+nishi.fukuoka.jp
+nogata.fukuoka.jp
+ogori.fukuoka.jp
+okagaki.fukuoka.jp
+okawa.fukuoka.jp
+oki.fukuoka.jp
+omuta.fukuoka.jp
+onga.fukuoka.jp
+onojo.fukuoka.jp
+oto.fukuoka.jp
+saigawa.fukuoka.jp
+sasaguri.fukuoka.jp
+shingu.fukuoka.jp
+shinyoshitomi.fukuoka.jp
+shonai.fukuoka.jp
+soeda.fukuoka.jp
+sue.fukuoka.jp
+tachiarai.fukuoka.jp
+tagawa.fukuoka.jp
+takata.fukuoka.jp
+toho.fukuoka.jp
+toyotsu.fukuoka.jp
+tsuiki.fukuoka.jp
+ukiha.fukuoka.jp
+umi.fukuoka.jp
+usui.fukuoka.jp
+yamada.fukuoka.jp
+yame.fukuoka.jp
+yanagawa.fukuoka.jp
+yukuhashi.fukuoka.jp
+aizubange.fukushima.jp
+aizumisato.fukushima.jp
+aizuwakamatsu.fukushima.jp
+asakawa.fukushima.jp
+bandai.fukushima.jp
+date.fukushima.jp
+fukushima.fukushima.jp
+furudono.fukushima.jp
+futaba.fukushima.jp
+hanawa.fukushima.jp
+higashi.fukushima.jp
+hirata.fukushima.jp
+hirono.fukushima.jp
+iitate.fukushima.jp
+inawashiro.fukushima.jp
+ishikawa.fukushima.jp
+iwaki.fukushima.jp
+izumizaki.fukushima.jp
+kagamiishi.fukushima.jp
+kaneyama.fukushima.jp
+kawamata.fukushima.jp
+kitakata.fukushima.jp
+kitashiobara.fukushima.jp
+koori.fukushima.jp
+koriyama.fukushima.jp
+kunimi.fukushima.jp
+miharu.fukushima.jp
+mishima.fukushima.jp
+namie.fukushima.jp
+nango.fukushima.jp
+nishiaizu.fukushima.jp
+nishigo.fukushima.jp
+okuma.fukushima.jp
+omotego.fukushima.jp
+ono.fukushima.jp
+otama.fukushima.jp
+samegawa.fukushima.jp
+shimogo.fukushima.jp
+shirakawa.fukushima.jp
+showa.fukushima.jp
+soma.fukushima.jp
+sukagawa.fukushima.jp
+taishin.fukushima.jp
+tamakawa.fukushima.jp
+tanagura.fukushima.jp
+tenei.fukushima.jp
+yabuki.fukushima.jp
+yamato.fukushima.jp
+yamatsuri.fukushima.jp
+yanaizu.fukushima.jp
+yugawa.fukushima.jp
+anpachi.gifu.jp
+ena.gifu.jp
+gifu.gifu.jp
+ginan.gifu.jp
+godo.gifu.jp
+gujo.gifu.jp
+hashima.gifu.jp
+hichiso.gifu.jp
+hida.gifu.jp
+higashishirakawa.gifu.jp
+ibigawa.gifu.jp
+ikeda.gifu.jp
+kakamigahara.gifu.jp
+kani.gifu.jp
+kasahara.gifu.jp
+kasamatsu.gifu.jp
+kawaue.gifu.jp
+kitagata.gifu.jp
+mino.gifu.jp
+minokamo.gifu.jp
+mitake.gifu.jp
+mizunami.gifu.jp
+motosu.gifu.jp
+nakatsugawa.gifu.jp
+ogaki.gifu.jp
+sakahogi.gifu.jp
+seki.gifu.jp
+sekigahara.gifu.jp
+shirakawa.gifu.jp
+tajimi.gifu.jp
+takayama.gifu.jp
+tarui.gifu.jp
+toki.gifu.jp
+tomika.gifu.jp
+wanouchi.gifu.jp
+yamagata.gifu.jp
+yaotsu.gifu.jp
+yoro.gifu.jp
+annaka.gunma.jp
+chiyoda.gunma.jp
+fujioka.gunma.jp
+higashiagatsuma.gunma.jp
+isesaki.gunma.jp
+itakura.gunma.jp
+kanna.gunma.jp
+kanra.gunma.jp
+katashina.gunma.jp
+kawaba.gunma.jp
+kiryu.gunma.jp
+kusatsu.gunma.jp
+maebashi.gunma.jp
+meiwa.gunma.jp
+midori.gunma.jp
+minakami.gunma.jp
+naganohara.gunma.jp
+nakanojo.gunma.jp
+nanmoku.gunma.jp
+numata.gunma.jp
+oizumi.gunma.jp
+ora.gunma.jp
+ota.gunma.jp
+shibukawa.gunma.jp
+shimonita.gunma.jp
+shinto.gunma.jp
+showa.gunma.jp
+takasaki.gunma.jp
+takayama.gunma.jp
+tamamura.gunma.jp
+tatebayashi.gunma.jp
+tomioka.gunma.jp
+tsukiyono.gunma.jp
+tsumagoi.gunma.jp
+ueno.gunma.jp
+yoshioka.gunma.jp
+asaminami.hiroshima.jp
+daiwa.hiroshima.jp
+etajima.hiroshima.jp
+fuchu.hiroshima.jp
+fukuyama.hiroshima.jp
+hatsukaichi.hiroshima.jp
+higashihiroshima.hiroshima.jp
+hongo.hiroshima.jp
+jinsekikogen.hiroshima.jp
+kaita.hiroshima.jp
+kui.hiroshima.jp
+kumano.hiroshima.jp
+kure.hiroshima.jp
+mihara.hiroshima.jp
+miyoshi.hiroshima.jp
+naka.hiroshima.jp
+onomichi.hiroshima.jp
+osakikamijima.hiroshima.jp
+otake.hiroshima.jp
+saka.hiroshima.jp
+sera.hiroshima.jp
+seranishi.hiroshima.jp
+shinichi.hiroshima.jp
+shobara.hiroshima.jp
+takehara.hiroshima.jp
+abashiri.hokkaido.jp
+abira.hokkaido.jp
+aibetsu.hokkaido.jp
+akabira.hokkaido.jp
+akkeshi.hokkaido.jp
+asahikawa.hokkaido.jp
+ashibetsu.hokkaido.jp
+ashoro.hokkaido.jp
+assabu.hokkaido.jp
+atsuma.hokkaido.jp
+bibai.hokkaido.jp
+biei.hokkaido.jp
+bifuka.hokkaido.jp
+bihoro.hokkaido.jp
+biratori.hokkaido.jp
+chippubetsu.hokkaido.jp
+chitose.hokkaido.jp
+date.hokkaido.jp
+ebetsu.hokkaido.jp
+embetsu.hokkaido.jp
+eniwa.hokkaido.jp
+erimo.hokkaido.jp
+esan.hokkaido.jp
+esashi.hokkaido.jp
+fukagawa.hokkaido.jp
+fukushima.hokkaido.jp
+furano.hokkaido.jp
+furubira.hokkaido.jp
+haboro.hokkaido.jp
+hakodate.hokkaido.jp
+hamatonbetsu.hokkaido.jp
+hidaka.hokkaido.jp
+higashikagura.hokkaido.jp
+higashikawa.hokkaido.jp
+hiroo.hokkaido.jp
+hokuryu.hokkaido.jp
+hokuto.hokkaido.jp
+honbetsu.hokkaido.jp
+horokanai.hokkaido.jp
+horonobe.hokkaido.jp
+ikeda.hokkaido.jp
+imakane.hokkaido.jp
+ishikari.hokkaido.jp
+iwamizawa.hokkaido.jp
+iwanai.hokkaido.jp
+kamifurano.hokkaido.jp
+kamikawa.hokkaido.jp
+kamishihoro.hokkaido.jp
+kamisunagawa.hokkaido.jp
+kamoenai.hokkaido.jp
+kayabe.hokkaido.jp
+kembuchi.hokkaido.jp
+kikonai.hokkaido.jp
+kimobetsu.hokkaido.jp
+kitahiroshima.hokkaido.jp
+kitami.hokkaido.jp
+kiyosato.hokkaido.jp
+koshimizu.hokkaido.jp
+kunneppu.hokkaido.jp
+kuriyama.hokkaido.jp
+kuromatsunai.hokkaido.jp
+kushiro.hokkaido.jp
+kutchan.hokkaido.jp
+kyowa.hokkaido.jp
+mashike.hokkaido.jp
+matsumae.hokkaido.jp
+mikasa.hokkaido.jp
+minamifurano.hokkaido.jp
+mombetsu.hokkaido.jp
+moseushi.hokkaido.jp
+mukawa.hokkaido.jp
+muroran.hokkaido.jp
+naie.hokkaido.jp
+nakagawa.hokkaido.jp
+nakasatsunai.hokkaido.jp
+nakatombetsu.hokkaido.jp
+nanae.hokkaido.jp
+nanporo.hokkaido.jp
+nayoro.hokkaido.jp
+nemuro.hokkaido.jp
+niikappu.hokkaido.jp
+niki.hokkaido.jp
+nishiokoppe.hokkaido.jp
+noboribetsu.hokkaido.jp
+numata.hokkaido.jp
+obihiro.hokkaido.jp
+obira.hokkaido.jp
+oketo.hokkaido.jp
+okoppe.hokkaido.jp
+otaru.hokkaido.jp
+otobe.hokkaido.jp
+otofuke.hokkaido.jp
+otoineppu.hokkaido.jp
+oumu.hokkaido.jp
+ozora.hokkaido.jp
+pippu.hokkaido.jp
+rankoshi.hokkaido.jp
+rebun.hokkaido.jp
+rikubetsu.hokkaido.jp
+rishiri.hokkaido.jp
+rishirifuji.hokkaido.jp
+saroma.hokkaido.jp
+sarufutsu.hokkaido.jp
+shakotan.hokkaido.jp
+shari.hokkaido.jp
+shibecha.hokkaido.jp
+shibetsu.hokkaido.jp
+shikabe.hokkaido.jp
+shikaoi.hokkaido.jp
+shimamaki.hokkaido.jp
+shimizu.hokkaido.jp
+shimokawa.hokkaido.jp
+shinshinotsu.hokkaido.jp
+shintoku.hokkaido.jp
+shiranuka.hokkaido.jp
+shiraoi.hokkaido.jp
+shiriuchi.hokkaido.jp
+sobetsu.hokkaido.jp
+sunagawa.hokkaido.jp
+taiki.hokkaido.jp
+takasu.hokkaido.jp
+takikawa.hokkaido.jp
+takinoue.hokkaido.jp
+teshikaga.hokkaido.jp
+tobetsu.hokkaido.jp
+tohma.hokkaido.jp
+tomakomai.hokkaido.jp
+tomari.hokkaido.jp
+toya.hokkaido.jp
+toyako.hokkaido.jp
+toyotomi.hokkaido.jp
+toyoura.hokkaido.jp
+tsubetsu.hokkaido.jp
+tsukigata.hokkaido.jp
+urakawa.hokkaido.jp
+urausu.hokkaido.jp
+uryu.hokkaido.jp
+utashinai.hokkaido.jp
+wakkanai.hokkaido.jp
+wassamu.hokkaido.jp
+yakumo.hokkaido.jp
+yoichi.hokkaido.jp
+aioi.hyogo.jp
+akashi.hyogo.jp
+ako.hyogo.jp
+amagasaki.hyogo.jp
+aogaki.hyogo.jp
+asago.hyogo.jp
+ashiya.hyogo.jp
+awaji.hyogo.jp
+fukusaki.hyogo.jp
+goshiki.hyogo.jp
+harima.hyogo.jp
+himeji.hyogo.jp
+ichikawa.hyogo.jp
+inagawa.hyogo.jp
+itami.hyogo.jp
+kakogawa.hyogo.jp
+kamigori.hyogo.jp
+kamikawa.hyogo.jp
+kasai.hyogo.jp
+kasuga.hyogo.jp
+kawanishi.hyogo.jp
+miki.hyogo.jp
+minamiawaji.hyogo.jp
+nishinomiya.hyogo.jp
+nishiwaki.hyogo.jp
+ono.hyogo.jp
+sanda.hyogo.jp
+sannan.hyogo.jp
+sasayama.hyogo.jp
+sayo.hyogo.jp
+shingu.hyogo.jp
+shinonsen.hyogo.jp
+shiso.hyogo.jp
+sumoto.hyogo.jp
+taishi.hyogo.jp
+taka.hyogo.jp
+takarazuka.hyogo.jp
+takasago.hyogo.jp
+takino.hyogo.jp
+tamba.hyogo.jp
+tatsuno.hyogo.jp
+toyooka.hyogo.jp
+yabu.hyogo.jp
+yashiro.hyogo.jp
+yoka.hyogo.jp
+yokawa.hyogo.jp
+ami.ibaraki.jp
+asahi.ibaraki.jp
+bando.ibaraki.jp
+chikusei.ibaraki.jp
+daigo.ibaraki.jp
+fujishiro.ibaraki.jp
+hitachi.ibaraki.jp
+hitachinaka.ibaraki.jp
+hitachiomiya.ibaraki.jp
+hitachiota.ibaraki.jp
+ibaraki.ibaraki.jp
+ina.ibaraki.jp
+inashiki.ibaraki.jp
+itako.ibaraki.jp
+iwama.ibaraki.jp
+joso.ibaraki.jp
+kamisu.ibaraki.jp
+kasama.ibaraki.jp
+kashima.ibaraki.jp
+kasumigaura.ibaraki.jp
+koga.ibaraki.jp
+miho.ibaraki.jp
+mito.ibaraki.jp
+moriya.ibaraki.jp
+naka.ibaraki.jp
+namegata.ibaraki.jp
+oarai.ibaraki.jp
+ogawa.ibaraki.jp
+omitama.ibaraki.jp
+ryugasaki.ibaraki.jp
+sakai.ibaraki.jp
+sakuragawa.ibaraki.jp
+shimodate.ibaraki.jp
+shimotsuma.ibaraki.jp
+shirosato.ibaraki.jp
+sowa.ibaraki.jp
+suifu.ibaraki.jp
+takahagi.ibaraki.jp
+tamatsukuri.ibaraki.jp
+tokai.ibaraki.jp
+tomobe.ibaraki.jp
+tone.ibaraki.jp
+toride.ibaraki.jp
+tsuchiura.ibaraki.jp
+tsukuba.ibaraki.jp
+uchihara.ibaraki.jp
+ushiku.ibaraki.jp
+yachiyo.ibaraki.jp
+yamagata.ibaraki.jp
+yawara.ibaraki.jp
+yuki.ibaraki.jp
+anamizu.ishikawa.jp
+hakui.ishikawa.jp
+hakusan.ishikawa.jp
+kaga.ishikawa.jp
+kahoku.ishikawa.jp
+kanazawa.ishikawa.jp
+kawakita.ishikawa.jp
+komatsu.ishikawa.jp
+nakanoto.ishikawa.jp
+nanao.ishikawa.jp
+nomi.ishikawa.jp
+nonoichi.ishikawa.jp
+noto.ishikawa.jp
+shika.ishikawa.jp
+suzu.ishikawa.jp
+tsubata.ishikawa.jp
+tsurugi.ishikawa.jp
+uchinada.ishikawa.jp
+wajima.ishikawa.jp
+fudai.iwate.jp
+fujisawa.iwate.jp
+hanamaki.iwate.jp
+hiraizumi.iwate.jp
+hirono.iwate.jp
+ichinohe.iwate.jp
+ichinoseki.iwate.jp
+iwaizumi.iwate.jp
+iwate.iwate.jp
+joboji.iwate.jp
+kamaishi.iwate.jp
+kanegasaki.iwate.jp
+karumai.iwate.jp
+kawai.iwate.jp
+kitakami.iwate.jp
+kuji.iwate.jp
+kunohe.iwate.jp
+kuzumaki.iwate.jp
+miyako.iwate.jp
+mizusawa.iwate.jp
+morioka.iwate.jp
+ninohe.iwate.jp
+noda.iwate.jp
+ofunato.iwate.jp
+oshu.iwate.jp
+otsuchi.iwate.jp
+rikuzentakata.iwate.jp
+shiwa.iwate.jp
+shizukuishi.iwate.jp
+sumita.iwate.jp
+tanohata.iwate.jp
+tono.iwate.jp
+yahaba.iwate.jp
+yamada.iwate.jp
+ayagawa.kagawa.jp
+higashikagawa.kagawa.jp
+kanonji.kagawa.jp
+kotohira.kagawa.jp
+manno.kagawa.jp
+marugame.kagawa.jp
+mitoyo.kagawa.jp
+naoshima.kagawa.jp
+sanuki.kagawa.jp
+tadotsu.kagawa.jp
+takamatsu.kagawa.jp
+tonosho.kagawa.jp
+uchinomi.kagawa.jp
+utazu.kagawa.jp
+zentsuji.kagawa.jp
+akune.kagoshima.jp
+amami.kagoshima.jp
+hioki.kagoshima.jp
+isa.kagoshima.jp
+isen.kagoshima.jp
+izumi.kagoshima.jp
+kagoshima.kagoshima.jp
+kanoya.kagoshima.jp
+kawanabe.kagoshima.jp
+kinko.kagoshima.jp
+kouyama.kagoshima.jp
+makurazaki.kagoshima.jp
+matsumoto.kagoshima.jp
+minamitane.kagoshima.jp
+nakatane.kagoshima.jp
+nishinoomote.kagoshima.jp
+satsumasendai.kagoshima.jp
+soo.kagoshima.jp
+tarumizu.kagoshima.jp
+yusui.kagoshima.jp
+aikawa.kanagawa.jp
+atsugi.kanagawa.jp
+ayase.kanagawa.jp
+chigasaki.kanagawa.jp
+ebina.kanagawa.jp
+fujisawa.kanagawa.jp
+hadano.kanagawa.jp
+hakone.kanagawa.jp
+hiratsuka.kanagawa.jp
+isehara.kanagawa.jp
+kaisei.kanagawa.jp
+kamakura.kanagawa.jp
+kiyokawa.kanagawa.jp
+matsuda.kanagawa.jp
+minamiashigara.kanagawa.jp
+miura.kanagawa.jp
+nakai.kanagawa.jp
+ninomiya.kanagawa.jp
+odawara.kanagawa.jp
+oi.kanagawa.jp
+oiso.kanagawa.jp
+sagamihara.kanagawa.jp
+samukawa.kanagawa.jp
+tsukui.kanagawa.jp
+yamakita.kanagawa.jp
+yamato.kanagawa.jp
+yokosuka.kanagawa.jp
+yugawara.kanagawa.jp
+zama.kanagawa.jp
+zushi.kanagawa.jp
+aki.kochi.jp
+geisei.kochi.jp
+hidaka.kochi.jp
+higashitsuno.kochi.jp
+ino.kochi.jp
+kagami.kochi.jp
+kami.kochi.jp
+kitagawa.kochi.jp
+kochi.kochi.jp
+mihara.kochi.jp
+motoyama.kochi.jp
+muroto.kochi.jp
+nahari.kochi.jp
+nakamura.kochi.jp
+nankoku.kochi.jp
+nishitosa.kochi.jp
+niyodogawa.kochi.jp
+ochi.kochi.jp
+okawa.kochi.jp
+otoyo.kochi.jp
+otsuki.kochi.jp
+sakawa.kochi.jp
+sukumo.kochi.jp
+susaki.kochi.jp
+tosa.kochi.jp
+tosashimizu.kochi.jp
+toyo.kochi.jp
+tsuno.kochi.jp
+umaji.kochi.jp
+yasuda.kochi.jp
+yusuhara.kochi.jp
+amakusa.kumamoto.jp
+arao.kumamoto.jp
+aso.kumamoto.jp
+choyo.kumamoto.jp
+gyokuto.kumamoto.jp
+kamiamakusa.kumamoto.jp
+kikuchi.kumamoto.jp
+kumamoto.kumamoto.jp
+mashiki.kumamoto.jp
+mifune.kumamoto.jp
+minamata.kumamoto.jp
+minamioguni.kumamoto.jp
+nagasu.kumamoto.jp
+nishihara.kumamoto.jp
+oguni.kumamoto.jp
+ozu.kumamoto.jp
+sumoto.kumamoto.jp
+takamori.kumamoto.jp
+uki.kumamoto.jp
+uto.kumamoto.jp
+yamaga.kumamoto.jp
+yamato.kumamoto.jp
+yatsushiro.kumamoto.jp
+ayabe.kyoto.jp
+fukuchiyama.kyoto.jp
+higashiyama.kyoto.jp
+ide.kyoto.jp
+ine.kyoto.jp
+joyo.kyoto.jp
+kameoka.kyoto.jp
+kamo.kyoto.jp
+kita.kyoto.jp
+kizu.kyoto.jp
+kumiyama.kyoto.jp
+kyotamba.kyoto.jp
+kyotanabe.kyoto.jp
+kyotango.kyoto.jp
+maizuru.kyoto.jp
+minami.kyoto.jp
+minamiyamashiro.kyoto.jp
+miyazu.kyoto.jp
+muko.kyoto.jp
+nagaokakyo.kyoto.jp
+nakagyo.kyoto.jp
+nantan.kyoto.jp
+oyamazaki.kyoto.jp
+sakyo.kyoto.jp
+seika.kyoto.jp
+tanabe.kyoto.jp
+uji.kyoto.jp
+ujitawara.kyoto.jp
+wazuka.kyoto.jp
+yamashina.kyoto.jp
+yawata.kyoto.jp
+asahi.mie.jp
+inabe.mie.jp
+ise.mie.jp
+kameyama.mie.jp
+kawagoe.mie.jp
+kiho.mie.jp
+kisosaki.mie.jp
+kiwa.mie.jp
+komono.mie.jp
+kumano.mie.jp
+kuwana.mie.jp
+matsusaka.mie.jp
+meiwa.mie.jp
+mihama.mie.jp
+minamiise.mie.jp
+misugi.mie.jp
+miyama.mie.jp
+nabari.mie.jp
+shima.mie.jp
+suzuka.mie.jp
+tado.mie.jp
+taiki.mie.jp
+taki.mie.jp
+tamaki.mie.jp
+toba.mie.jp
+tsu.mie.jp
+udono.mie.jp
+ureshino.mie.jp
+watarai.mie.jp
+yokkaichi.mie.jp
+furukawa.miyagi.jp
+higashimatsushima.miyagi.jp
+ishinomaki.miyagi.jp
+iwanuma.miyagi.jp
+kakuda.miyagi.jp
+kami.miyagi.jp
+kawasaki.miyagi.jp
+marumori.miyagi.jp
+matsushima.miyagi.jp
+minamisanriku.miyagi.jp
+misato.miyagi.jp
+murata.miyagi.jp
+natori.miyagi.jp
+ogawara.miyagi.jp
+ohira.miyagi.jp
+onagawa.miyagi.jp
+osaki.miyagi.jp
+rifu.miyagi.jp
+semine.miyagi.jp
+shibata.miyagi.jp
+shichikashuku.miyagi.jp
+shikama.miyagi.jp
+shiogama.miyagi.jp
+shiroishi.miyagi.jp
+tagajo.miyagi.jp
+taiwa.miyagi.jp
+tome.miyagi.jp
+tomiya.miyagi.jp
+wakuya.miyagi.jp
+watari.miyagi.jp
+yamamoto.miyagi.jp
+zao.miyagi.jp
+aya.miyazaki.jp
+ebino.miyazaki.jp
+gokase.miyazaki.jp
+hyuga.miyazaki.jp
+kadogawa.miyazaki.jp
+kawaminami.miyazaki.jp
+kijo.miyazaki.jp
+kitagawa.miyazaki.jp
+kitakata.miyazaki.jp
+kitaura.miyazaki.jp
+kobayashi.miyazaki.jp
+kunitomi.miyazaki.jp
+kushima.miyazaki.jp
+mimata.miyazaki.jp
+miyakonojo.miyazaki.jp
+miyazaki.miyazaki.jp
+morotsuka.miyazaki.jp
+nichinan.miyazaki.jp
+nishimera.miyazaki.jp
+nobeoka.miyazaki.jp
+saito.miyazaki.jp
+shiiba.miyazaki.jp
+shintomi.miyazaki.jp
+takaharu.miyazaki.jp
+takanabe.miyazaki.jp
+takazaki.miyazaki.jp
+tsuno.miyazaki.jp
+achi.nagano.jp
+agematsu.nagano.jp
+anan.nagano.jp
+aoki.nagano.jp
+asahi.nagano.jp
+azumino.nagano.jp
+chikuhoku.nagano.jp
+chikuma.nagano.jp
+chino.nagano.jp
+fujimi.nagano.jp
+hakuba.nagano.jp
+hara.nagano.jp
+hiraya.nagano.jp
+iida.nagano.jp
+iijima.nagano.jp
+iiyama.nagano.jp
+iizuna.nagano.jp
+ikeda.nagano.jp
+ikusaka.nagano.jp
+ina.nagano.jp
+karuizawa.nagano.jp
+kawakami.nagano.jp
+kiso.nagano.jp
+kisofukushima.nagano.jp
+kitaaiki.nagano.jp
+komagane.nagano.jp
+komoro.nagano.jp
+matsukawa.nagano.jp
+matsumoto.nagano.jp
+miasa.nagano.jp
+minamiaiki.nagano.jp
+minamimaki.nagano.jp
+minamiminowa.nagano.jp
+minowa.nagano.jp
+miyada.nagano.jp
+miyota.nagano.jp
+mochizuki.nagano.jp
+nagano.nagano.jp
+nagawa.nagano.jp
+nagiso.nagano.jp
+nakagawa.nagano.jp
+nakano.nagano.jp
+nozawaonsen.nagano.jp
+obuse.nagano.jp
+ogawa.nagano.jp
+okaya.nagano.jp
+omachi.nagano.jp
+omi.nagano.jp
+ookuwa.nagano.jp
+ooshika.nagano.jp
+otaki.nagano.jp
+otari.nagano.jp
+sakae.nagano.jp
+sakaki.nagano.jp
+saku.nagano.jp
+sakuho.nagano.jp
+shimosuwa.nagano.jp
+shinanomachi.nagano.jp
+shiojiri.nagano.jp
+suwa.nagano.jp
+suzaka.nagano.jp
+takagi.nagano.jp
+takamori.nagano.jp
+takayama.nagano.jp
+tateshina.nagano.jp
+tatsuno.nagano.jp
+togakushi.nagano.jp
+togura.nagano.jp
+tomi.nagano.jp
+ueda.nagano.jp
+wada.nagano.jp
+yamagata.nagano.jp
+yamanouchi.nagano.jp
+yasaka.nagano.jp
+yasuoka.nagano.jp
+chijiwa.nagasaki.jp
+futsu.nagasaki.jp
+goto.nagasaki.jp
+hasami.nagasaki.jp
+hirado.nagasaki.jp
+iki.nagasaki.jp
+isahaya.nagasaki.jp
+kawatana.nagasaki.jp
+kuchinotsu.nagasaki.jp
+matsuura.nagasaki.jp
+nagasaki.nagasaki.jp
+obama.nagasaki.jp
+omura.nagasaki.jp
+oseto.nagasaki.jp
+saikai.nagasaki.jp
+sasebo.nagasaki.jp
+seihi.nagasaki.jp
+shimabara.nagasaki.jp
+shinkamigoto.nagasaki.jp
+togitsu.nagasaki.jp
+tsushima.nagasaki.jp
+unzen.nagasaki.jp
+ando.nara.jp
+gose.nara.jp
+heguri.nara.jp
+higashiyoshino.nara.jp
+ikaruga.nara.jp
+ikoma.nara.jp
+kamikitayama.nara.jp
+kanmaki.nara.jp
+kashiba.nara.jp
+kashihara.nara.jp
+katsuragi.nara.jp
+kawai.nara.jp
+kawakami.nara.jp
+kawanishi.nara.jp
+koryo.nara.jp
+kurotaki.nara.jp
+mitsue.nara.jp
+miyake.nara.jp
+nara.nara.jp
+nosegawa.nara.jp
+oji.nara.jp
+ouda.nara.jp
+oyodo.nara.jp
+sakurai.nara.jp
+sango.nara.jp
+shimoichi.nara.jp
+shimokitayama.nara.jp
+shinjo.nara.jp
+soni.nara.jp
+takatori.nara.jp
+tawaramoto.nara.jp
+tenkawa.nara.jp
+tenri.nara.jp
+uda.nara.jp
+yamatokoriyama.nara.jp
+yamatotakada.nara.jp
+yamazoe.nara.jp
+yoshino.nara.jp
+aga.niigata.jp
+agano.niigata.jp
+gosen.niigata.jp
+itoigawa.niigata.jp
+izumozaki.niigata.jp
+joetsu.niigata.jp
+kamo.niigata.jp
+kariwa.niigata.jp
+kashiwazaki.niigata.jp
+minamiuonuma.niigata.jp
+mitsuke.niigata.jp
+muika.niigata.jp
+murakami.niigata.jp
+myoko.niigata.jp
+nagaoka.niigata.jp
+niigata.niigata.jp
+ojiya.niigata.jp
+omi.niigata.jp
+sado.niigata.jp
+sanjo.niigata.jp
+seiro.niigata.jp
+seirou.niigata.jp
+sekikawa.niigata.jp
+shibata.niigata.jp
+tagami.niigata.jp
+tainai.niigata.jp
+tochio.niigata.jp
+tokamachi.niigata.jp
+tsubame.niigata.jp
+tsunan.niigata.jp
+uonuma.niigata.jp
+yahiko.niigata.jp
+yoita.niigata.jp
+yuzawa.niigata.jp
+beppu.oita.jp
+bungoono.oita.jp
+bungotakada.oita.jp
+hasama.oita.jp
+hiji.oita.jp
+himeshima.oita.jp
+hita.oita.jp
+kamitsue.oita.jp
+kokonoe.oita.jp
+kuju.oita.jp
+kunisaki.oita.jp
+kusu.oita.jp
+oita.oita.jp
+saiki.oita.jp
+taketa.oita.jp
+tsukumi.oita.jp
+usa.oita.jp
+usuki.oita.jp
+yufu.oita.jp
+akaiwa.okayama.jp
+asakuchi.okayama.jp
+bizen.okayama.jp
+hayashima.okayama.jp
+ibara.okayama.jp
+kagamino.okayama.jp
+kasaoka.okayama.jp
+kibichuo.okayama.jp
+kumenan.okayama.jp
+kurashiki.okayama.jp
+maniwa.okayama.jp
+misaki.okayama.jp
+nagi.okayama.jp
+niimi.okayama.jp
+nishiawakura.okayama.jp
+okayama.okayama.jp
+satosho.okayama.jp
+setouchi.okayama.jp
+shinjo.okayama.jp
+shoo.okayama.jp
+soja.okayama.jp
+takahashi.okayama.jp
+tamano.okayama.jp
+tsuyama.okayama.jp
+wake.okayama.jp
+yakage.okayama.jp
+aguni.okinawa.jp
+ginowan.okinawa.jp
+ginoza.okinawa.jp
+gushikami.okinawa.jp
+haebaru.okinawa.jp
+higashi.okinawa.jp
+hirara.okinawa.jp
+iheya.okinawa.jp
+ishigaki.okinawa.jp
+ishikawa.okinawa.jp
+itoman.okinawa.jp
+izena.okinawa.jp
+kadena.okinawa.jp
+kin.okinawa.jp
+kitadaito.okinawa.jp
+kitanakagusuku.okinawa.jp
+kumejima.okinawa.jp
+kunigami.okinawa.jp
+minamidaito.okinawa.jp
+motobu.okinawa.jp
+nago.okinawa.jp
+naha.okinawa.jp
+nakagusuku.okinawa.jp
+nakijin.okinawa.jp
+nanjo.okinawa.jp
+nishihara.okinawa.jp
+ogimi.okinawa.jp
+okinawa.okinawa.jp
+onna.okinawa.jp
+shimoji.okinawa.jp
+taketomi.okinawa.jp
+tarama.okinawa.jp
+tokashiki.okinawa.jp
+tomigusuku.okinawa.jp
+tonaki.okinawa.jp
+urasoe.okinawa.jp
+uruma.okinawa.jp
+yaese.okinawa.jp
+yomitan.okinawa.jp
+yonabaru.okinawa.jp
+yonaguni.okinawa.jp
+zamami.okinawa.jp
+abeno.osaka.jp
+chihayaakasaka.osaka.jp
+chuo.osaka.jp
+daito.osaka.jp
+fujiidera.osaka.jp
+habikino.osaka.jp
+hannan.osaka.jp
+higashiosaka.osaka.jp
+higashisumiyoshi.osaka.jp
+higashiyodogawa.osaka.jp
+hirakata.osaka.jp
+ibaraki.osaka.jp
+ikeda.osaka.jp
+izumi.osaka.jp
+izumiotsu.osaka.jp
+izumisano.osaka.jp
+kadoma.osaka.jp
+kaizuka.osaka.jp
+kanan.osaka.jp
+kashiwara.osaka.jp
+katano.osaka.jp
+kawachinagano.osaka.jp
+kishiwada.osaka.jp
+kita.osaka.jp
+kumatori.osaka.jp
+matsubara.osaka.jp
+minato.osaka.jp
+minoh.osaka.jp
+misaki.osaka.jp
+moriguchi.osaka.jp
+neyagawa.osaka.jp
+nishi.osaka.jp
+nose.osaka.jp
+osakasayama.osaka.jp
+sakai.osaka.jp
+sayama.osaka.jp
+sennan.osaka.jp
+settsu.osaka.jp
+shijonawate.osaka.jp
+shimamoto.osaka.jp
+suita.osaka.jp
+tadaoka.osaka.jp
+taishi.osaka.jp
+tajiri.osaka.jp
+takaishi.osaka.jp
+takatsuki.osaka.jp
+tondabayashi.osaka.jp
+toyonaka.osaka.jp
+toyono.osaka.jp
+yao.osaka.jp
+ariake.saga.jp
+arita.saga.jp
+fukudomi.saga.jp
+genkai.saga.jp
+hamatama.saga.jp
+hizen.saga.jp
+imari.saga.jp
+kamimine.saga.jp
+kanzaki.saga.jp
+karatsu.saga.jp
+kashima.saga.jp
+kitagata.saga.jp
+kitahata.saga.jp
+kiyama.saga.jp
+kouhoku.saga.jp
+kyuragi.saga.jp
+nishiarita.saga.jp
+ogi.saga.jp
+omachi.saga.jp
+ouchi.saga.jp
+saga.saga.jp
+shiroishi.saga.jp
+taku.saga.jp
+tara.saga.jp
+tosu.saga.jp
+yoshinogari.saga.jp
+arakawa.saitama.jp
+asaka.saitama.jp
+chichibu.saitama.jp
+fujimi.saitama.jp
+fujimino.saitama.jp
+fukaya.saitama.jp
+hanno.saitama.jp
+hanyu.saitama.jp
+hasuda.saitama.jp
+hatogaya.saitama.jp
+hatoyama.saitama.jp
+hidaka.saitama.jp
+higashichichibu.saitama.jp
+higashimatsuyama.saitama.jp
+honjo.saitama.jp
+ina.saitama.jp
+iruma.saitama.jp
+iwatsuki.saitama.jp
+kamiizumi.saitama.jp
+kamikawa.saitama.jp
+kamisato.saitama.jp
+kasukabe.saitama.jp
+kawagoe.saitama.jp
+kawaguchi.saitama.jp
+kawajima.saitama.jp
+kazo.saitama.jp
+kitamoto.saitama.jp
+koshigaya.saitama.jp
+kounosu.saitama.jp
+kuki.saitama.jp
+kumagaya.saitama.jp
+matsubushi.saitama.jp
+minano.saitama.jp
+misato.saitama.jp
+miyashiro.saitama.jp
+miyoshi.saitama.jp
+moroyama.saitama.jp
+nagatoro.saitama.jp
+namegawa.saitama.jp
+niiza.saitama.jp
+ogano.saitama.jp
+ogawa.saitama.jp
+ogose.saitama.jp
+okegawa.saitama.jp
+omiya.saitama.jp
+otaki.saitama.jp
+ranzan.saitama.jp
+ryokami.saitama.jp
+saitama.saitama.jp
+sakado.saitama.jp
+satte.saitama.jp
+sayama.saitama.jp
+shiki.saitama.jp
+shiraoka.saitama.jp
+soka.saitama.jp
+sugito.saitama.jp
+toda.saitama.jp
+tokigawa.saitama.jp
+tokorozawa.saitama.jp
+tsurugashima.saitama.jp
+urawa.saitama.jp
+warabi.saitama.jp
+yashio.saitama.jp
+yokoze.saitama.jp
+yono.saitama.jp
+yorii.saitama.jp
+yoshida.saitama.jp
+yoshikawa.saitama.jp
+yoshimi.saitama.jp
+aisho.shiga.jp
+gamo.shiga.jp
+higashiomi.shiga.jp
+hikone.shiga.jp
+koka.shiga.jp
+konan.shiga.jp
+kosei.shiga.jp
+koto.shiga.jp
+kusatsu.shiga.jp
+maibara.shiga.jp
+moriyama.shiga.jp
+nagahama.shiga.jp
+nishiazai.shiga.jp
+notogawa.shiga.jp
+omihachiman.shiga.jp
+otsu.shiga.jp
+ritto.shiga.jp
+ryuoh.shiga.jp
+takashima.shiga.jp
+takatsuki.shiga.jp
+torahime.shiga.jp
+toyosato.shiga.jp
+yasu.shiga.jp
+akagi.shimane.jp
+ama.shimane.jp
+gotsu.shimane.jp
+hamada.shimane.jp
+higashiizumo.shimane.jp
+hikawa.shimane.jp
+hikimi.shimane.jp
+izumo.shimane.jp
+kakinoki.shimane.jp
+masuda.shimane.jp
+matsue.shimane.jp
+misato.shimane.jp
+nishinoshima.shimane.jp
+ohda.shimane.jp
+okinoshima.shimane.jp
+okuizumo.shimane.jp
+shimane.shimane.jp
+tamayu.shimane.jp
+tsuwano.shimane.jp
+unnan.shimane.jp
+yakumo.shimane.jp
+yasugi.shimane.jp
+yatsuka.shimane.jp
+arai.shizuoka.jp
+atami.shizuoka.jp
+fuji.shizuoka.jp
+fujieda.shizuoka.jp
+fujikawa.shizuoka.jp
+fujinomiya.shizuoka.jp
+fukuroi.shizuoka.jp
+gotemba.shizuoka.jp
+haibara.shizuoka.jp
+hamamatsu.shizuoka.jp
+higashiizu.shizuoka.jp
+ito.shizuoka.jp
+iwata.shizuoka.jp
+izu.shizuoka.jp
+izunokuni.shizuoka.jp
+kakegawa.shizuoka.jp
+kannami.shizuoka.jp
+kawanehon.shizuoka.jp
+kawazu.shizuoka.jp
+kikugawa.shizuoka.jp
+kosai.shizuoka.jp
+makinohara.shizuoka.jp
+matsuzaki.shizuoka.jp
+minamiizu.shizuoka.jp
+mishima.shizuoka.jp
+morimachi.shizuoka.jp
+nishiizu.shizuoka.jp
+numazu.shizuoka.jp
+omaezaki.shizuoka.jp
+shimada.shizuoka.jp
+shimizu.shizuoka.jp
+shimoda.shizuoka.jp
+shizuoka.shizuoka.jp
+susono.shizuoka.jp
+yaizu.shizuoka.jp
+yoshida.shizuoka.jp
+ashikaga.tochigi.jp
+bato.tochigi.jp
+haga.tochigi.jp
+ichikai.tochigi.jp
+iwafune.tochigi.jp
+kaminokawa.tochigi.jp
+kanuma.tochigi.jp
+karasuyama.tochigi.jp
+kuroiso.tochigi.jp
+mashiko.tochigi.jp
+mibu.tochigi.jp
+moka.tochigi.jp
+motegi.tochigi.jp
+nasu.tochigi.jp
+nasushiobara.tochigi.jp
+nikko.tochigi.jp
+nishikata.tochigi.jp
+nogi.tochigi.jp
+ohira.tochigi.jp
+ohtawara.tochigi.jp
+oyama.tochigi.jp
+sakura.tochigi.jp
+sano.tochigi.jp
+shimotsuke.tochigi.jp
+shioya.tochigi.jp
+takanezawa.tochigi.jp
+tochigi.tochigi.jp
+tsuga.tochigi.jp
+ujiie.tochigi.jp
+utsunomiya.tochigi.jp
+yaita.tochigi.jp
+aizumi.tokushima.jp
+anan.tokushima.jp
+ichiba.tokushima.jp
+itano.tokushima.jp
+kainan.tokushima.jp
+komatsushima.tokushima.jp
+matsushige.tokushima.jp
+mima.tokushima.jp
+minami.tokushima.jp
+miyoshi.tokushima.jp
+mugi.tokushima.jp
+nakagawa.tokushima.jp
+naruto.tokushima.jp
+sanagochi.tokushima.jp
+shishikui.tokushima.jp
+tokushima.tokushima.jp
+wajiki.tokushima.jp
+adachi.tokyo.jp
+akiruno.tokyo.jp
+akishima.tokyo.jp
+aogashima.tokyo.jp
+arakawa.tokyo.jp
+bunkyo.tokyo.jp
+chiyoda.tokyo.jp
+chofu.tokyo.jp
+chuo.tokyo.jp
+edogawa.tokyo.jp
+fuchu.tokyo.jp
+fussa.tokyo.jp
+hachijo.tokyo.jp
+hachioji.tokyo.jp
+hamura.tokyo.jp
+higashikurume.tokyo.jp
+higashimurayama.tokyo.jp
+higashiyamato.tokyo.jp
+hino.tokyo.jp
+hinode.tokyo.jp
+hinohara.tokyo.jp
+inagi.tokyo.jp
+itabashi.tokyo.jp
+katsushika.tokyo.jp
+kita.tokyo.jp
+kiyose.tokyo.jp
+kodaira.tokyo.jp
+koganei.tokyo.jp
+kokubunji.tokyo.jp
+komae.tokyo.jp
+koto.tokyo.jp
+kouzushima.tokyo.jp
+kunitachi.tokyo.jp
+machida.tokyo.jp
+meguro.tokyo.jp
+minato.tokyo.jp
+mitaka.tokyo.jp
+mizuho.tokyo.jp
+musashimurayama.tokyo.jp
+musashino.tokyo.jp
+nakano.tokyo.jp
+nerima.tokyo.jp
+ogasawara.tokyo.jp
+okutama.tokyo.jp
+ome.tokyo.jp
+oshima.tokyo.jp
+ota.tokyo.jp
+setagaya.tokyo.jp
+shibuya.tokyo.jp
+shinagawa.tokyo.jp
+shinjuku.tokyo.jp
+suginami.tokyo.jp
+sumida.tokyo.jp
+tachikawa.tokyo.jp
+taito.tokyo.jp
+tama.tokyo.jp
+toshima.tokyo.jp
+chizu.tottori.jp
+hino.tottori.jp
+kawahara.tottori.jp
+koge.tottori.jp
+kotoura.tottori.jp
+misasa.tottori.jp
+nanbu.tottori.jp
+nichinan.tottori.jp
+sakaiminato.tottori.jp
+tottori.tottori.jp
+wakasa.tottori.jp
+yazu.tottori.jp
+yonago.tottori.jp
+asahi.toyama.jp
+fuchu.toyama.jp
+fukumitsu.toyama.jp
+funahashi.toyama.jp
+himi.toyama.jp
+imizu.toyama.jp
+inami.toyama.jp
+johana.toyama.jp
+kamiichi.toyama.jp
+kurobe.toyama.jp
+nakaniikawa.toyama.jp
+namerikawa.toyama.jp
+nanto.toyama.jp
+nyuzen.toyama.jp
+oyabe.toyama.jp
+taira.toyama.jp
+takaoka.toyama.jp
+tateyama.toyama.jp
+toga.toyama.jp
+tonami.toyama.jp
+toyama.toyama.jp
+unazuki.toyama.jp
+uozu.toyama.jp
+yamada.toyama.jp
+arida.wakayama.jp
+aridagawa.wakayama.jp
+gobo.wakayama.jp
+hashimoto.wakayama.jp
+hidaka.wakayama.jp
+hirogawa.wakayama.jp
+inami.wakayama.jp
+iwade.wakayama.jp
+kainan.wakayama.jp
+kamitonda.wakayama.jp
+katsuragi.wakayama.jp
+kimino.wakayama.jp
+kinokawa.wakayama.jp
+kitayama.wakayama.jp
+koya.wakayama.jp
+koza.wakayama.jp
+kozagawa.wakayama.jp
+kudoyama.wakayama.jp
+kushimoto.wakayama.jp
+mihama.wakayama.jp
+misato.wakayama.jp
+nachikatsuura.wakayama.jp
+shingu.wakayama.jp
+shirahama.wakayama.jp
+taiji.wakayama.jp
+tanabe.wakayama.jp
+wakayama.wakayama.jp
+yuasa.wakayama.jp
+yura.wakayama.jp
+asahi.yamagata.jp
+funagata.yamagata.jp
+higashine.yamagata.jp
+iide.yamagata.jp
+kahoku.yamagata.jp
+kaminoyama.yamagata.jp
+kaneyama.yamagata.jp
+kawanishi.yamagata.jp
+mamurogawa.yamagata.jp
+mikawa.yamagata.jp
+murayama.yamagata.jp
+nagai.yamagata.jp
+nakayama.yamagata.jp
+nanyo.yamagata.jp
+nishikawa.yamagata.jp
+obanazawa.yamagata.jp
+oe.yamagata.jp
+oguni.yamagata.jp
+ohkura.yamagata.jp
+oishida.yamagata.jp
+sagae.yamagata.jp
+sakata.yamagata.jp
+sakegawa.yamagata.jp
+shinjo.yamagata.jp
+shirataka.yamagata.jp
+shonai.yamagata.jp
+takahata.yamagata.jp
+tendo.yamagata.jp
+tozawa.yamagata.jp
+tsuruoka.yamagata.jp
+yamagata.yamagata.jp
+yamanobe.yamagata.jp
+yonezawa.yamagata.jp
+yuza.yamagata.jp
+abu.yamaguchi.jp
+hagi.yamaguchi.jp
+hikari.yamaguchi.jp
+hofu.yamaguchi.jp
+iwakuni.yamaguchi.jp
+kudamatsu.yamaguchi.jp
+mitou.yamaguchi.jp
+nagato.yamaguchi.jp
+oshima.yamaguchi.jp
+shimonoseki.yamaguchi.jp
+shunan.yamaguchi.jp
+tabuse.yamaguchi.jp
+tokuyama.yamaguchi.jp
+toyota.yamaguchi.jp
+ube.yamaguchi.jp
+yuu.yamaguchi.jp
+chuo.yamanashi.jp
+doshi.yamanashi.jp
+fuefuki.yamanashi.jp
+fujikawa.yamanashi.jp
+fujikawaguchiko.yamanashi.jp
+fujiyoshida.yamanashi.jp
+hayakawa.yamanashi.jp
+hokuto.yamanashi.jp
+ichikawamisato.yamanashi.jp
+kai.yamanashi.jp
+kofu.yamanashi.jp
+koshu.yamanashi.jp
+kosuge.yamanashi.jp
+minami-alps.yamanashi.jp
+minobu.yamanashi.jp
+nakamichi.yamanashi.jp
+nanbu.yamanashi.jp
+narusawa.yamanashi.jp
+nirasaki.yamanashi.jp
+nishikatsura.yamanashi.jp
+oshino.yamanashi.jp
+otsuki.yamanashi.jp
+showa.yamanashi.jp
+tabayama.yamanashi.jp
+tsuru.yamanashi.jp
+uenohara.yamanashi.jp
+yamanakako.yamanashi.jp
+yamanashi.yamanashi.jp
+
+// ke : http://www.kenic.or.ke/index.php/en/ke-domains/ke-domains
+ke
+ac.ke
+co.ke
+go.ke
+info.ke
+me.ke
+mobi.ke
+ne.ke
+or.ke
+sc.ke
+
+// kg : http://www.domain.kg/dmn_n.html
+kg
+org.kg
+net.kg
+com.kg
+edu.kg
+gov.kg
+mil.kg
+
+// kh : http://www.mptc.gov.kh/dns_registration.htm
+*.kh
+
+// ki : http://www.ki/dns/index.html
+ki
+edu.ki
+biz.ki
+net.ki
+org.ki
+gov.ki
+info.ki
+com.ki
+
+// km : https://en.wikipedia.org/wiki/.km
+// http://www.domaine.km/documents/charte.doc
+km
+org.km
+nom.km
+gov.km
+prd.km
+tm.km
+edu.km
+mil.km
+ass.km
+com.km
+// These are only mentioned as proposed suggestions at domaine.km, but
+// https://en.wikipedia.org/wiki/.km says they're available for registration:
+coop.km
+asso.km
+presse.km
+medecin.km
+notaires.km
+pharmaciens.km
+veterinaire.km
+gouv.km
+
+// kn : https://en.wikipedia.org/wiki/.kn
+// http://www.dot.kn/domainRules.html
+kn
+net.kn
+org.kn
+edu.kn
+gov.kn
+
+// kp : http://www.kcce.kp/en_index.php
+kp
+com.kp
+edu.kp
+gov.kp
+org.kp
+rep.kp
+tra.kp
+
+// kr : https://en.wikipedia.org/wiki/.kr
+// see also: http://domain.nida.or.kr/eng/registration.jsp
+kr
+ac.kr
+co.kr
+es.kr
+go.kr
+hs.kr
+kg.kr
+mil.kr
+ms.kr
+ne.kr
+or.kr
+pe.kr
+re.kr
+sc.kr
+// kr geographical names
+busan.kr
+chungbuk.kr
+chungnam.kr
+daegu.kr
+daejeon.kr
+gangwon.kr
+gwangju.kr
+gyeongbuk.kr
+gyeonggi.kr
+gyeongnam.kr
+incheon.kr
+jeju.kr
+jeonbuk.kr
+jeonnam.kr
+seoul.kr
+ulsan.kr
+
+// kw : https://www.nic.kw/policies/
+// Confirmed by registry <nic.tech@citra.gov.kw>
+kw
+com.kw
+edu.kw
+emb.kw
+gov.kw
+ind.kw
+net.kw
+org.kw
+
+// ky : http://www.icta.ky/da_ky_reg_dom.php
+// Confirmed by registry <kysupport@perimeterusa.com> 2008-06-17
+ky
+edu.ky
+gov.ky
+com.ky
+org.ky
+net.ky
+
+// kz : https://en.wikipedia.org/wiki/.kz
+// see also: http://www.nic.kz/rules/index.jsp
+kz
+org.kz
+edu.kz
+net.kz
+gov.kz
+mil.kz
+com.kz
+
+// la : https://en.wikipedia.org/wiki/.la
+// Submitted by registry <gavin.brown@nic.la>
+la
+int.la
+net.la
+info.la
+edu.la
+gov.la
+per.la
+com.la
+org.la
+
+// lb : https://en.wikipedia.org/wiki/.lb
+// Submitted by registry <randy@psg.com>
+lb
+com.lb
+edu.lb
+gov.lb
+net.lb
+org.lb
+
+// lc : https://en.wikipedia.org/wiki/.lc
+// see also: http://www.nic.lc/rules.htm
+lc
+com.lc
+net.lc
+co.lc
+org.lc
+edu.lc
+gov.lc
+
+// li : https://en.wikipedia.org/wiki/.li
+li
+
+// lk : https://www.nic.lk/index.php/domain-registration/lk-domain-naming-structure
+lk
+gov.lk
+sch.lk
+net.lk
+int.lk
+com.lk
+org.lk
+edu.lk
+ngo.lk
+soc.lk
+web.lk
+ltd.lk
+assn.lk
+grp.lk
+hotel.lk
+ac.lk
+
+// lr : http://psg.com/dns/lr/lr.txt
+// Submitted by registry <randy@psg.com>
+lr
+com.lr
+edu.lr
+gov.lr
+org.lr
+net.lr
+
+// ls : http://www.nic.ls/
+// Confirmed by registry <lsadmin@nic.ls>
+ls
+ac.ls
+biz.ls
+co.ls
+edu.ls
+gov.ls
+info.ls
+net.ls
+org.ls
+sc.ls
+
+// lt : https://en.wikipedia.org/wiki/.lt
+lt
+// gov.lt : http://www.gov.lt/index_en.php
+gov.lt
+
+// lu : http://www.dns.lu/en/
+lu
+
+// lv : http://www.nic.lv/DNS/En/generic.php
+lv
+com.lv
+edu.lv
+gov.lv
+org.lv
+mil.lv
+id.lv
+net.lv
+asn.lv
+conf.lv
+
+// ly : http://www.nic.ly/regulations.php
+ly
+com.ly
+net.ly
+gov.ly
+plc.ly
+edu.ly
+sch.ly
+med.ly
+org.ly
+id.ly
+
+// ma : https://en.wikipedia.org/wiki/.ma
+// http://www.anrt.ma/fr/admin/download/upload/file_fr782.pdf
+ma
+co.ma
+net.ma
+gov.ma
+org.ma
+ac.ma
+press.ma
+
+// mc : http://www.nic.mc/
+mc
+tm.mc
+asso.mc
+
+// md : https://en.wikipedia.org/wiki/.md
+md
+
+// me : https://en.wikipedia.org/wiki/.me
+me
+co.me
+net.me
+org.me
+edu.me
+ac.me
+gov.me
+its.me
+priv.me
+
+// mg : http://nic.mg/nicmg/?page_id=39
+mg
+org.mg
+nom.mg
+gov.mg
+prd.mg
+tm.mg
+edu.mg
+mil.mg
+com.mg
+co.mg
+
+// mh : https://en.wikipedia.org/wiki/.mh
+mh
+
+// mil : https://en.wikipedia.org/wiki/.mil
+mil
+
+// mk : https://en.wikipedia.org/wiki/.mk
+// see also: http://dns.marnet.net.mk/postapka.php
+mk
+com.mk
+org.mk
+net.mk
+edu.mk
+gov.mk
+inf.mk
+name.mk
+
+// ml : http://www.gobin.info/domainname/ml-template.doc
+// see also: https://en.wikipedia.org/wiki/.ml
+ml
+com.ml
+edu.ml
+gouv.ml
+gov.ml
+net.ml
+org.ml
+presse.ml
+
+// mm : https://en.wikipedia.org/wiki/.mm
+*.mm
+
+// mn : https://en.wikipedia.org/wiki/.mn
+mn
+gov.mn
+edu.mn
+org.mn
+
+// mo : http://www.monic.net.mo/
+mo
+com.mo
+net.mo
+org.mo
+edu.mo
+gov.mo
+
+// mobi : https://en.wikipedia.org/wiki/.mobi
+mobi
+
+// mp : http://www.dot.mp/
+// Confirmed by registry <dcamacho@saipan.com> 2008-06-17
+mp
+
+// mq : https://en.wikipedia.org/wiki/.mq
+mq
+
+// mr : https://en.wikipedia.org/wiki/.mr
+mr
+gov.mr
+
+// ms : http://www.nic.ms/pdf/MS_Domain_Name_Rules.pdf
+ms
+com.ms
+edu.ms
+gov.ms
+net.ms
+org.ms
+
+// mt : https://www.nic.org.mt/go/policy
+// Submitted by registry <help@nic.org.mt>
+mt
+com.mt
+edu.mt
+net.mt
+org.mt
+
+// mu : https://en.wikipedia.org/wiki/.mu
+mu
+com.mu
+net.mu
+org.mu
+gov.mu
+ac.mu
+co.mu
+or.mu
+
+// museum : http://about.museum/naming/
+// http://index.museum/
+museum
+academy.museum
+agriculture.museum
+air.museum
+airguard.museum
+alabama.museum
+alaska.museum
+amber.museum
+ambulance.museum
+american.museum
+americana.museum
+americanantiques.museum
+americanart.museum
+amsterdam.museum
+and.museum
+annefrank.museum
+anthro.museum
+anthropology.museum
+antiques.museum
+aquarium.museum
+arboretum.museum
+archaeological.museum
+archaeology.museum
+architecture.museum
+art.museum
+artanddesign.museum
+artcenter.museum
+artdeco.museum
+arteducation.museum
+artgallery.museum
+arts.museum
+artsandcrafts.museum
+asmatart.museum
+assassination.museum
+assisi.museum
+association.museum
+astronomy.museum
+atlanta.museum
+austin.museum
+australia.museum
+automotive.museum
+aviation.museum
+axis.museum
+badajoz.museum
+baghdad.museum
+bahn.museum
+bale.museum
+baltimore.museum
+barcelona.museum
+baseball.museum
+basel.museum
+baths.museum
+bauern.museum
+beauxarts.museum
+beeldengeluid.museum
+bellevue.museum
+bergbau.museum
+berkeley.museum
+berlin.museum
+bern.museum
+bible.museum
+bilbao.museum
+bill.museum
+birdart.museum
+birthplace.museum
+bonn.museum
+boston.museum
+botanical.museum
+botanicalgarden.museum
+botanicgarden.museum
+botany.museum
+brandywinevalley.museum
+brasil.museum
+bristol.museum
+british.museum
+britishcolumbia.museum
+broadcast.museum
+brunel.museum
+brussel.museum
+brussels.museum
+bruxelles.museum
+building.museum
+burghof.museum
+bus.museum
+bushey.museum
+cadaques.museum
+california.museum
+cambridge.museum
+can.museum
+canada.museum
+capebreton.museum
+carrier.museum
+cartoonart.museum
+casadelamoneda.museum
+castle.museum
+castres.museum
+celtic.museum
+center.museum
+chattanooga.museum
+cheltenham.museum
+chesapeakebay.museum
+chicago.museum
+children.museum
+childrens.museum
+childrensgarden.museum
+chiropractic.museum
+chocolate.museum
+christiansburg.museum
+cincinnati.museum
+cinema.museum
+circus.museum
+civilisation.museum
+civilization.museum
+civilwar.museum
+clinton.museum
+clock.museum
+coal.museum
+coastaldefence.museum
+cody.museum
+coldwar.museum
+collection.museum
+colonialwilliamsburg.museum
+coloradoplateau.museum
+columbia.museum
+columbus.museum
+communication.museum
+communications.museum
+community.museum
+computer.museum
+computerhistory.museum
+xn--comunicaes-v6a2o.museum
+comunicações.museum
+contemporary.museum
+contemporaryart.museum
+convent.museum
+copenhagen.museum
+corporation.museum
+xn--correios-e-telecomunicaes-ghc29a.museum
+correios-e-telecomunicações.museum
+corvette.museum
+costume.museum
+countryestate.museum
+county.museum
+crafts.museum
+cranbrook.museum
+creation.museum
+cultural.museum
+culturalcenter.museum
+culture.museum
+cyber.museum
+cymru.museum
+dali.museum
+dallas.museum
+database.museum
+ddr.museum
+decorativearts.museum
+delaware.museum
+delmenhorst.museum
+denmark.museum
+depot.museum
+design.museum
+detroit.museum
+dinosaur.museum
+discovery.museum
+dolls.museum
+donostia.museum
+durham.museum
+eastafrica.museum
+eastcoast.museum
+education.museum
+educational.museum
+egyptian.museum
+eisenbahn.museum
+elburg.museum
+elvendrell.museum
+embroidery.museum
+encyclopedic.museum
+england.museum
+entomology.museum
+environment.museum
+environmentalconservation.museum
+epilepsy.museum
+essex.museum
+estate.museum
+ethnology.museum
+exeter.museum
+exhibition.museum
+family.museum
+farm.museum
+farmequipment.museum
+farmers.museum
+farmstead.museum
+field.museum
+figueres.museum
+filatelia.museum
+film.museum
+fineart.museum
+finearts.museum
+finland.museum
+flanders.museum
+florida.museum
+force.museum
+fortmissoula.museum
+fortworth.museum
+foundation.museum
+francaise.museum
+frankfurt.museum
+franziskaner.museum
+freemasonry.museum
+freiburg.museum
+fribourg.museum
+frog.museum
+fundacio.museum
+furniture.museum
+gallery.museum
+garden.museum
+gateway.museum
+geelvinck.museum
+gemological.museum
+geology.museum
+georgia.museum
+giessen.museum
+glas.museum
+glass.museum
+gorge.museum
+grandrapids.museum
+graz.museum
+guernsey.museum
+halloffame.museum
+hamburg.museum
+handson.museum
+harvestcelebration.museum
+hawaii.museum
+health.museum
+heimatunduhren.museum
+hellas.museum
+helsinki.museum
+hembygdsforbund.museum
+heritage.museum
+histoire.museum
+historical.museum
+historicalsociety.museum
+historichouses.museum
+historisch.museum
+historisches.museum
+history.museum
+historyofscience.museum
+horology.museum
+house.museum
+humanities.museum
+illustration.museum
+imageandsound.museum
+indian.museum
+indiana.museum
+indianapolis.museum
+indianmarket.museum
+intelligence.museum
+interactive.museum
+iraq.museum
+iron.museum
+isleofman.museum
+jamison.museum
+jefferson.museum
+jerusalem.museum
+jewelry.museum
+jewish.museum
+jewishart.museum
+jfk.museum
+journalism.museum
+judaica.museum
+judygarland.museum
+juedisches.museum
+juif.museum
+karate.museum
+karikatur.museum
+kids.museum
+koebenhavn.museum
+koeln.museum
+kunst.museum
+kunstsammlung.museum
+kunstunddesign.museum
+labor.museum
+labour.museum
+lajolla.museum
+lancashire.museum
+landes.museum
+lans.museum
+xn--lns-qla.museum
+läns.museum
+larsson.museum
+lewismiller.museum
+lincoln.museum
+linz.museum
+living.museum
+livinghistory.museum
+localhistory.museum
+london.museum
+losangeles.museum
+louvre.museum
+loyalist.museum
+lucerne.museum
+luxembourg.museum
+luzern.museum
+mad.museum
+madrid.museum
+mallorca.museum
+manchester.museum
+mansion.museum
+mansions.museum
+manx.museum
+marburg.museum
+maritime.museum
+maritimo.museum
+maryland.museum
+marylhurst.museum
+media.museum
+medical.museum
+medizinhistorisches.museum
+meeres.museum
+memorial.museum
+mesaverde.museum
+michigan.museum
+midatlantic.museum
+military.museum
+mill.museum
+miners.museum
+mining.museum
+minnesota.museum
+missile.museum
+missoula.museum
+modern.museum
+moma.museum
+money.museum
+monmouth.museum
+monticello.museum
+montreal.museum
+moscow.museum
+motorcycle.museum
+muenchen.museum
+muenster.museum
+mulhouse.museum
+muncie.museum
+museet.museum
+museumcenter.museum
+museumvereniging.museum
+music.museum
+national.museum
+nationalfirearms.museum
+nationalheritage.museum
+nativeamerican.museum
+naturalhistory.museum
+naturalhistorymuseum.museum
+naturalsciences.museum
+nature.museum
+naturhistorisches.museum
+natuurwetenschappen.museum
+naumburg.museum
+naval.museum
+nebraska.museum
+neues.museum
+newhampshire.museum
+newjersey.museum
+newmexico.museum
+newport.museum
+newspaper.museum
+newyork.museum
+niepce.museum
+norfolk.museum
+north.museum
+nrw.museum
+nyc.museum
+nyny.museum
+oceanographic.museum
+oceanographique.museum
+omaha.museum
+online.museum
+ontario.museum
+openair.museum
+oregon.museum
+oregontrail.museum
+otago.museum
+oxford.museum
+pacific.museum
+paderborn.museum
+palace.museum
+paleo.museum
+palmsprings.museum
+panama.museum
+paris.museum
+pasadena.museum
+pharmacy.museum
+philadelphia.museum
+philadelphiaarea.museum
+philately.museum
+phoenix.museum
+photography.museum
+pilots.museum
+pittsburgh.museum
+planetarium.museum
+plantation.museum
+plants.museum
+plaza.museum
+portal.museum
+portland.museum
+portlligat.museum
+posts-and-telecommunications.museum
+preservation.museum
+presidio.museum
+press.museum
+project.museum
+public.museum
+pubol.museum
+quebec.museum
+railroad.museum
+railway.museum
+research.museum
+resistance.museum
+riodejaneiro.museum
+rochester.museum
+rockart.museum
+roma.museum
+russia.museum
+saintlouis.museum
+salem.museum
+salvadordali.museum
+salzburg.museum
+sandiego.museum
+sanfrancisco.museum
+santabarbara.museum
+santacruz.museum
+santafe.museum
+saskatchewan.museum
+satx.museum
+savannahga.museum
+schlesisches.museum
+schoenbrunn.museum
+schokoladen.museum
+school.museum
+schweiz.museum
+science.museum
+scienceandhistory.museum
+scienceandindustry.museum
+sciencecenter.museum
+sciencecenters.museum
+science-fiction.museum
+sciencehistory.museum
+sciences.museum
+sciencesnaturelles.museum
+scotland.museum
+seaport.museum
+settlement.museum
+settlers.museum
+shell.museum
+sherbrooke.museum
+sibenik.museum
+silk.museum
+ski.museum
+skole.museum
+society.museum
+sologne.museum
+soundandvision.museum
+southcarolina.museum
+southwest.museum
+space.museum
+spy.museum
+square.museum
+stadt.museum
+stalbans.museum
+starnberg.museum
+state.museum
+stateofdelaware.museum
+station.museum
+steam.museum
+steiermark.museum
+stjohn.museum
+stockholm.museum
+stpetersburg.museum
+stuttgart.museum
+suisse.museum
+surgeonshall.museum
+surrey.museum
+svizzera.museum
+sweden.museum
+sydney.museum
+tank.museum
+tcm.museum
+technology.museum
+telekommunikation.museum
+television.museum
+texas.museum
+textile.museum
+theater.museum
+time.museum
+timekeeping.museum
+topology.museum
+torino.museum
+touch.museum
+town.museum
+transport.museum
+tree.museum
+trolley.museum
+trust.museum
+trustee.museum
+uhren.museum
+ulm.museum
+undersea.museum
+university.museum
+usa.museum
+usantiques.museum
+usarts.museum
+uscountryestate.museum
+usculture.museum
+usdecorativearts.museum
+usgarden.museum
+ushistory.museum
+ushuaia.museum
+uslivinghistory.museum
+utah.museum
+uvic.museum
+valley.museum
+vantaa.museum
+versailles.museum
+viking.museum
+village.museum
+virginia.museum
+virtual.museum
+virtuel.museum
+vlaanderen.museum
+volkenkunde.museum
+wales.museum
+wallonie.museum
+war.museum
+washingtondc.museum
+watchandclock.museum
+watch-and-clock.museum
+western.museum
+westfalen.museum
+whaling.museum
+wildlife.museum
+williamsburg.museum
+windmill.museum
+workshop.museum
+york.museum
+yorkshire.museum
+yosemite.museum
+youth.museum
+zoological.museum
+zoology.museum
+xn--9dbhblg6di.museum
+ירושלי×.museum
+xn--h1aegh.museum
+иком.museum
+
+// mv : https://en.wikipedia.org/wiki/.mv
+// "mv" included because, contra Wikipedia, google.mv exists.
+mv
+aero.mv
+biz.mv
+com.mv
+coop.mv
+edu.mv
+gov.mv
+info.mv
+int.mv
+mil.mv
+museum.mv
+name.mv
+net.mv
+org.mv
+pro.mv
+
+// mw : http://www.registrar.mw/
+mw
+ac.mw
+biz.mw
+co.mw
+com.mw
+coop.mw
+edu.mw
+gov.mw
+int.mw
+museum.mw
+net.mw
+org.mw
+
+// mx : http://www.nic.mx/
+// Submitted by registry <farias@nic.mx>
+mx
+com.mx
+org.mx
+gob.mx
+edu.mx
+net.mx
+
+// my : http://www.mynic.my/
+// Available strings: https://mynic.my/resources/domains/buying-a-domain/
+my
+biz.my
+com.my
+edu.my
+gov.my
+mil.my
+name.my
+net.my
+org.my
+
+// mz : http://www.uem.mz/
+// Submitted by registry <antonio@uem.mz>
+mz
+ac.mz
+adv.mz
+co.mz
+edu.mz
+gov.mz
+mil.mz
+net.mz
+org.mz
+
+// na : http://www.na-nic.com.na/
+// http://www.info.na/domain/
+na
+info.na
+pro.na
+name.na
+school.na
+or.na
+dr.na
+us.na
+mx.na
+ca.na
+in.na
+cc.na
+tv.na
+ws.na
+mobi.na
+co.na
+com.na
+org.na
+
+// name : has 2nd-level tlds, but there's no list of them
+name
+
+// nc : http://www.cctld.nc/
+nc
+asso.nc
+nom.nc
+
+// ne : https://en.wikipedia.org/wiki/.ne
+ne
+
+// net : https://en.wikipedia.org/wiki/.net
+net
+
+// nf : https://en.wikipedia.org/wiki/.nf
+nf
+com.nf
+net.nf
+per.nf
+rec.nf
+web.nf
+arts.nf
+firm.nf
+info.nf
+other.nf
+store.nf
+
+// ng : http://www.nira.org.ng/index.php/join-us/register-ng-domain/189-nira-slds
+ng
+com.ng
+edu.ng
+gov.ng
+i.ng
+mil.ng
+mobi.ng
+name.ng
+net.ng
+org.ng
+sch.ng
+
+// ni : http://www.nic.ni/
+ni
+ac.ni
+biz.ni
+co.ni
+com.ni
+edu.ni
+gob.ni
+in.ni
+info.ni
+int.ni
+mil.ni
+net.ni
+nom.ni
+org.ni
+web.ni
+
+// nl : https://en.wikipedia.org/wiki/.nl
+// https://www.sidn.nl/
+// ccTLD for the Netherlands
+nl
+
+// no : https://www.norid.no/en/om-domenenavn/regelverk-for-no/
+// Norid geographical second level domains : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-b/
+// Norid category second level domains : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-c/
+// Norid category second-level domains managed by parties other than Norid : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-d/
+// RSS feed: https://teknisk.norid.no/en/feed/
+no
+// Norid category second level domains : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-c/
+fhs.no
+vgs.no
+fylkesbibl.no
+folkebibl.no
+museum.no
+idrett.no
+priv.no
+// Norid category second-level domains managed by parties other than Norid : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-d/
+mil.no
+stat.no
+dep.no
+kommune.no
+herad.no
+// Norid geographical second level domains : https://www.norid.no/en/om-domenenavn/regelverk-for-no/vedlegg-b/
+// counties
+aa.no
+ah.no
+bu.no
+fm.no
+hl.no
+hm.no
+jan-mayen.no
+mr.no
+nl.no
+nt.no
+of.no
+ol.no
+oslo.no
+rl.no
+sf.no
+st.no
+svalbard.no
+tm.no
+tr.no
+va.no
+vf.no
+// primary and lower secondary schools per county
+gs.aa.no
+gs.ah.no
+gs.bu.no
+gs.fm.no
+gs.hl.no
+gs.hm.no
+gs.jan-mayen.no
+gs.mr.no
+gs.nl.no
+gs.nt.no
+gs.of.no
+gs.ol.no
+gs.oslo.no
+gs.rl.no
+gs.sf.no
+gs.st.no
+gs.svalbard.no
+gs.tm.no
+gs.tr.no
+gs.va.no
+gs.vf.no
+// cities
+akrehamn.no
+Ã¥krehamn.no
+algard.no
+xn--lgrd-poac.no
+ålgård.no
+arna.no
+brumunddal.no
+bryne.no
+bronnoysund.no
+xn--brnnysund-m8ac.no
+brønnøysund.no
+drobak.no
+xn--drbak-wua.no
+drøbak.no
+egersund.no
+fetsund.no
+floro.no
+xn--flor-jra.no
+florø.no
+fredrikstad.no
+hokksund.no
+honefoss.no
+xn--hnefoss-q1a.no
+hønefoss.no
+jessheim.no
+jorpeland.no
+xn--jrpeland-54a.no
+jørpeland.no
+kirkenes.no
+kopervik.no
+krokstadelva.no
+langevag.no
+xn--langevg-jxa.no
+langevåg.no
+leirvik.no
+mjondalen.no
+xn--mjndalen-64a.no
+mjøndalen.no
+mo-i-rana.no
+mosjoen.no
+xn--mosjen-eya.no
+mosjøen.no
+nesoddtangen.no
+orkanger.no
+osoyro.no
+xn--osyro-wua.no
+osøyro.no
+raholt.no
+xn--rholt-mra.no
+råholt.no
+sandnessjoen.no
+xn--sandnessjen-ogb.no
+sandnessjøen.no
+skedsmokorset.no
+slattum.no
+spjelkavik.no
+stathelle.no
+stavern.no
+stjordalshalsen.no
+xn--stjrdalshalsen-sqb.no
+stjørdalshalsen.no
+tananger.no
+tranby.no
+vossevangen.no
+// communities
+afjord.no
+Ã¥fjord.no
+agdenes.no
+al.no
+Ã¥l.no
+alesund.no
+Ã¥lesund.no
+alstahaug.no
+alta.no
+xn--lt-liac.no
+áltá.no
+alaheadju.no
+álaheadju.no
+alvdal.no
+amli.no
+Ã¥mli.no
+amot.no
+Ã¥mot.no
+andebu.no
+andoy.no
+xn--andy-ira.no
+andøy.no
+andasuolo.no
+ardal.no
+Ã¥rdal.no
+aremark.no
+arendal.no
+Ã¥s.no
+aseral.no
+Ã¥seral.no
+asker.no
+askim.no
+askvoll.no
+askoy.no
+xn--asky-ira.no
+askøy.no
+asnes.no
+Ã¥snes.no
+audnedaln.no
+aukra.no
+aure.no
+aurland.no
+aurskog-holand.no
+xn--aurskog-hland-jnb.no
+aurskog-høland.no
+austevoll.no
+austrheim.no
+averoy.no
+xn--avery-yua.no
+averøy.no
+balestrand.no
+ballangen.no
+balat.no
+xn--blt-elab.no
+bálát.no
+balsfjord.no
+bahccavuotna.no
+xn--bhccavuotna-k7a.no
+báhccavuotna.no
+bamble.no
+bardu.no
+beardu.no
+beiarn.no
+bajddar.no
+xn--bjddar-pta.no
+bájddar.no
+baidar.no
+xn--bidr-5nac.no
+báidár.no
+berg.no
+bergen.no
+berlevag.no
+xn--berlevg-jxa.no
+berlevåg.no
+bearalvahki.no
+xn--bearalvhki-y4a.no
+bearalváhki.no
+bindal.no
+birkenes.no
+bjarkoy.no
+xn--bjarky-fya.no
+bjarkøy.no
+bjerkreim.no
+bjugn.no
+bodo.no
+xn--bod-2na.no
+bodø.no
+badaddja.no
+xn--bdddj-mrabd.no
+bådåddjå.no
+budejju.no
+bokn.no
+bremanger.no
+bronnoy.no
+xn--brnny-wuac.no
+brønnøy.no
+bygland.no
+bykle.no
+barum.no
+xn--brum-voa.no
+bærum.no
+bo.telemark.no
+xn--b-5ga.telemark.no
+bø.telemark.no
+bo.nordland.no
+xn--b-5ga.nordland.no
+bø.nordland.no
+bievat.no
+xn--bievt-0qa.no
+bievát.no
+bomlo.no
+xn--bmlo-gra.no
+bømlo.no
+batsfjord.no
+xn--btsfjord-9za.no
+båtsfjord.no
+bahcavuotna.no
+xn--bhcavuotna-s4a.no
+báhcavuotna.no
+dovre.no
+drammen.no
+drangedal.no
+dyroy.no
+xn--dyry-ira.no
+dyrøy.no
+donna.no
+xn--dnna-gra.no
+dønna.no
+eid.no
+eidfjord.no
+eidsberg.no
+eidskog.no
+eidsvoll.no
+eigersund.no
+elverum.no
+enebakk.no
+engerdal.no
+etne.no
+etnedal.no
+evenes.no
+evenassi.no
+xn--eveni-0qa01ga.no
+evenášši.no
+evje-og-hornnes.no
+farsund.no
+fauske.no
+fuossko.no
+fuoisku.no
+fedje.no
+fet.no
+finnoy.no
+xn--finny-yua.no
+finnøy.no
+fitjar.no
+fjaler.no
+fjell.no
+flakstad.no
+flatanger.no
+flekkefjord.no
+flesberg.no
+flora.no
+fla.no
+xn--fl-zia.no
+flå.no
+folldal.no
+forsand.no
+fosnes.no
+frei.no
+frogn.no
+froland.no
+frosta.no
+frana.no
+xn--frna-woa.no
+fræna.no
+froya.no
+xn--frya-hra.no
+frøya.no
+fusa.no
+fyresdal.no
+forde.no
+xn--frde-gra.no
+førde.no
+gamvik.no
+gangaviika.no
+xn--ggaviika-8ya47h.no
+gáŋgaviika.no
+gaular.no
+gausdal.no
+gildeskal.no
+xn--gildeskl-g0a.no
+gildeskål.no
+giske.no
+gjemnes.no
+gjerdrum.no
+gjerstad.no
+gjesdal.no
+gjovik.no
+xn--gjvik-wua.no
+gjøvik.no
+gloppen.no
+gol.no
+gran.no
+grane.no
+granvin.no
+gratangen.no
+grimstad.no
+grong.no
+kraanghke.no
+xn--kranghke-b0a.no
+kråanghke.no
+grue.no
+gulen.no
+hadsel.no
+halden.no
+halsa.no
+hamar.no
+hamaroy.no
+habmer.no
+xn--hbmer-xqa.no
+hábmer.no
+hapmir.no
+xn--hpmir-xqa.no
+hápmir.no
+hammerfest.no
+hammarfeasta.no
+xn--hmmrfeasta-s4ac.no
+hámmárfeasta.no
+haram.no
+hareid.no
+harstad.no
+hasvik.no
+aknoluokta.no
+xn--koluokta-7ya57h.no
+ákŋoluokta.no
+hattfjelldal.no
+aarborte.no
+haugesund.no
+hemne.no
+hemnes.no
+hemsedal.no
+heroy.more-og-romsdal.no
+xn--hery-ira.xn--mre-og-romsdal-qqb.no
+herøy.møre-og-romsdal.no
+heroy.nordland.no
+xn--hery-ira.nordland.no
+herøy.nordland.no
+hitra.no
+hjartdal.no
+hjelmeland.no
+hobol.no
+xn--hobl-ira.no
+hobøl.no
+hof.no
+hol.no
+hole.no
+holmestrand.no
+holtalen.no
+xn--holtlen-hxa.no
+holtålen.no
+hornindal.no
+horten.no
+hurdal.no
+hurum.no
+hvaler.no
+hyllestad.no
+hagebostad.no
+xn--hgebostad-g3a.no
+hægebostad.no
+hoyanger.no
+xn--hyanger-q1a.no
+høyanger.no
+hoylandet.no
+xn--hylandet-54a.no
+høylandet.no
+ha.no
+xn--h-2fa.no
+hå.no
+ibestad.no
+inderoy.no
+xn--indery-fya.no
+inderøy.no
+iveland.no
+jevnaker.no
+jondal.no
+jolster.no
+xn--jlster-bya.no
+jølster.no
+karasjok.no
+karasjohka.no
+xn--krjohka-hwab49j.no
+kárášjohka.no
+karlsoy.no
+galsa.no
+xn--gls-elac.no
+gálsá.no
+karmoy.no
+xn--karmy-yua.no
+karmøy.no
+kautokeino.no
+guovdageaidnu.no
+klepp.no
+klabu.no
+xn--klbu-woa.no
+klæbu.no
+kongsberg.no
+kongsvinger.no
+kragero.no
+xn--krager-gya.no
+kragerø.no
+kristiansand.no
+kristiansund.no
+krodsherad.no
+xn--krdsherad-m8a.no
+krødsherad.no
+kvalsund.no
+rahkkeravju.no
+xn--rhkkervju-01af.no
+ráhkkerávju.no
+kvam.no
+kvinesdal.no
+kvinnherad.no
+kviteseid.no
+kvitsoy.no
+xn--kvitsy-fya.no
+kvitsøy.no
+kvafjord.no
+xn--kvfjord-nxa.no
+kvæfjord.no
+giehtavuoatna.no
+kvanangen.no
+xn--kvnangen-k0a.no
+kvænangen.no
+navuotna.no
+xn--nvuotna-hwa.no
+návuotna.no
+kafjord.no
+xn--kfjord-iua.no
+kåfjord.no
+gaivuotna.no
+xn--givuotna-8ya.no
+gáivuotna.no
+larvik.no
+lavangen.no
+lavagis.no
+loabat.no
+xn--loabt-0qa.no
+loabát.no
+lebesby.no
+davvesiida.no
+leikanger.no
+leirfjord.no
+leka.no
+leksvik.no
+lenvik.no
+leangaviika.no
+xn--leagaviika-52b.no
+leaŋgaviika.no
+lesja.no
+levanger.no
+lier.no
+lierne.no
+lillehammer.no
+lillesand.no
+lindesnes.no
+lindas.no
+xn--linds-pra.no
+lindås.no
+lom.no
+loppa.no
+lahppi.no
+xn--lhppi-xqa.no
+láhppi.no
+lund.no
+lunner.no
+luroy.no
+xn--lury-ira.no
+lurøy.no
+luster.no
+lyngdal.no
+lyngen.no
+ivgu.no
+lardal.no
+lerdal.no
+xn--lrdal-sra.no
+lærdal.no
+lodingen.no
+xn--ldingen-q1a.no
+lødingen.no
+lorenskog.no
+xn--lrenskog-54a.no
+lørenskog.no
+loten.no
+xn--lten-gra.no
+løten.no
+malvik.no
+masoy.no
+xn--msy-ula0h.no
+måsøy.no
+muosat.no
+xn--muost-0qa.no
+muosát.no
+mandal.no
+marker.no
+marnardal.no
+masfjorden.no
+meland.no
+meldal.no
+melhus.no
+meloy.no
+xn--mely-ira.no
+meløy.no
+meraker.no
+xn--merker-kua.no
+meråker.no
+moareke.no
+xn--moreke-jua.no
+moåreke.no
+midsund.no
+midtre-gauldal.no
+modalen.no
+modum.no
+molde.no
+moskenes.no
+moss.no
+mosvik.no
+malselv.no
+xn--mlselv-iua.no
+målselv.no
+malatvuopmi.no
+xn--mlatvuopmi-s4a.no
+málatvuopmi.no
+namdalseid.no
+aejrie.no
+namsos.no
+namsskogan.no
+naamesjevuemie.no
+xn--nmesjevuemie-tcba.no
+nååmesjevuemie.no
+laakesvuemie.no
+nannestad.no
+narvik.no
+narviika.no
+naustdal.no
+nedre-eiker.no
+nes.akershus.no
+nes.buskerud.no
+nesna.no
+nesodden.no
+nesseby.no
+unjarga.no
+xn--unjrga-rta.no
+unjárga.no
+nesset.no
+nissedal.no
+nittedal.no
+nord-aurdal.no
+nord-fron.no
+nord-odal.no
+norddal.no
+nordkapp.no
+davvenjarga.no
+xn--davvenjrga-y4a.no
+davvenjárga.no
+nordre-land.no
+nordreisa.no
+raisa.no
+xn--risa-5na.no
+ráisa.no
+nore-og-uvdal.no
+notodden.no
+naroy.no
+xn--nry-yla5g.no
+nærøy.no
+notteroy.no
+xn--nttery-byae.no
+nøtterøy.no
+odda.no
+oksnes.no
+øksnes.no
+oppdal.no
+oppegard.no
+xn--oppegrd-ixa.no
+oppegård.no
+orkdal.no
+orland.no
+ørland.no
+orskog.no
+ørskog.no
+orsta.no
+ørsta.no
+os.hedmark.no
+os.hordaland.no
+osen.no
+osteroy.no
+xn--ostery-fya.no
+osterøy.no
+ostre-toten.no
+østre-toten.no
+overhalla.no
+ovre-eiker.no
+øvre-eiker.no
+oyer.no
+øyer.no
+oygarden.no
+øygarden.no
+oystre-slidre.no
+øystre-slidre.no
+porsanger.no
+porsangu.no
+xn--porsgu-sta26f.no
+porsáŋgu.no
+porsgrunn.no
+radoy.no
+xn--rady-ira.no
+radøy.no
+rakkestad.no
+rana.no
+ruovat.no
+randaberg.no
+rauma.no
+rendalen.no
+rennebu.no
+rennesoy.no
+xn--rennesy-v1a.no
+rennesøy.no
+rindal.no
+ringebu.no
+ringerike.no
+ringsaker.no
+rissa.no
+risor.no
+xn--risr-ira.no
+risør.no
+roan.no
+rollag.no
+rygge.no
+ralingen.no
+xn--rlingen-mxa.no
+rælingen.no
+rodoy.no
+xn--rdy-0nab.no
+rødøy.no
+romskog.no
+xn--rmskog-bya.no
+rømskog.no
+roros.no
+xn--rros-gra.no
+røros.no
+rost.no
+xn--rst-0na.no
+røst.no
+royken.no
+xn--ryken-vua.no
+røyken.no
+royrvik.no
+xn--ryrvik-bya.no
+røyrvik.no
+rade.no
+xn--rde-ula.no
+råde.no
+salangen.no
+siellak.no
+saltdal.no
+salat.no
+xn--slt-elab.no
+sálát.no
+xn--slat-5na.no
+sálat.no
+samnanger.no
+sande.more-og-romsdal.no
+sande.xn--mre-og-romsdal-qqb.no
+sande.møre-og-romsdal.no
+sande.vestfold.no
+sandefjord.no
+sandnes.no
+sandoy.no
+xn--sandy-yua.no
+sandøy.no
+sarpsborg.no
+sauda.no
+sauherad.no
+sel.no
+selbu.no
+selje.no
+seljord.no
+sigdal.no
+siljan.no
+sirdal.no
+skaun.no
+skedsmo.no
+ski.no
+skien.no
+skiptvet.no
+skjervoy.no
+xn--skjervy-v1a.no
+skjervøy.no
+skierva.no
+xn--skierv-uta.no
+skiervá.no
+skjak.no
+xn--skjk-soa.no
+skjåk.no
+skodje.no
+skanland.no
+xn--sknland-fxa.no
+skånland.no
+skanit.no
+xn--sknit-yqa.no
+skánit.no
+smola.no
+xn--smla-hra.no
+smøla.no
+snillfjord.no
+snasa.no
+xn--snsa-roa.no
+snåsa.no
+snoasa.no
+snaase.no
+xn--snase-nra.no
+snåase.no
+sogndal.no
+sokndal.no
+sola.no
+solund.no
+songdalen.no
+sortland.no
+spydeberg.no
+stange.no
+stavanger.no
+steigen.no
+steinkjer.no
+stjordal.no
+xn--stjrdal-s1a.no
+stjørdal.no
+stokke.no
+stor-elvdal.no
+stord.no
+stordal.no
+storfjord.no
+omasvuotna.no
+strand.no
+stranda.no
+stryn.no
+sula.no
+suldal.no
+sund.no
+sunndal.no
+surnadal.no
+sveio.no
+svelvik.no
+sykkylven.no
+sogne.no
+xn--sgne-gra.no
+søgne.no
+somna.no
+xn--smna-gra.no
+sømna.no
+sondre-land.no
+xn--sndre-land-0cb.no
+søndre-land.no
+sor-aurdal.no
+xn--sr-aurdal-l8a.no
+sør-aurdal.no
+sor-fron.no
+xn--sr-fron-q1a.no
+sør-fron.no
+sor-odal.no
+xn--sr-odal-q1a.no
+sør-odal.no
+sor-varanger.no
+xn--sr-varanger-ggb.no
+sør-varanger.no
+matta-varjjat.no
+xn--mtta-vrjjat-k7af.no
+mátta-várjjat.no
+sorfold.no
+xn--srfold-bya.no
+sørfold.no
+sorreisa.no
+xn--srreisa-q1a.no
+sørreisa.no
+sorum.no
+xn--srum-gra.no
+sørum.no
+tana.no
+deatnu.no
+time.no
+tingvoll.no
+tinn.no
+tjeldsund.no
+dielddanuorri.no
+tjome.no
+xn--tjme-hra.no
+tjøme.no
+tokke.no
+tolga.no
+torsken.no
+tranoy.no
+xn--trany-yua.no
+tranøy.no
+tromso.no
+xn--troms-zua.no
+tromsø.no
+tromsa.no
+romsa.no
+trondheim.no
+troandin.no
+trysil.no
+trana.no
+xn--trna-woa.no
+træna.no
+trogstad.no
+xn--trgstad-r1a.no
+trøgstad.no
+tvedestrand.no
+tydal.no
+tynset.no
+tysfjord.no
+divtasvuodna.no
+divttasvuotna.no
+tysnes.no
+tysvar.no
+xn--tysvr-vra.no
+tysvær.no
+tonsberg.no
+xn--tnsberg-q1a.no
+tønsberg.no
+ullensaker.no
+ullensvang.no
+ulvik.no
+utsira.no
+vadso.no
+xn--vads-jra.no
+vadsø.no
+cahcesuolo.no
+xn--hcesuolo-7ya35b.no
+Äáhcesuolo.no
+vaksdal.no
+valle.no
+vang.no
+vanylven.no
+vardo.no
+xn--vard-jra.no
+vardø.no
+varggat.no
+xn--vrggt-xqad.no
+várggát.no
+vefsn.no
+vaapste.no
+vega.no
+vegarshei.no
+xn--vegrshei-c0a.no
+vegårshei.no
+vennesla.no
+verdal.no
+verran.no
+vestby.no
+vestnes.no
+vestre-slidre.no
+vestre-toten.no
+vestvagoy.no
+xn--vestvgy-ixa6o.no
+vestvågøy.no
+vevelstad.no
+vik.no
+vikna.no
+vindafjord.no
+volda.no
+voss.no
+varoy.no
+xn--vry-yla5g.no
+værøy.no
+vagan.no
+xn--vgan-qoa.no
+vågan.no
+voagat.no
+vagsoy.no
+xn--vgsy-qoa0j.no
+vågsøy.no
+vaga.no
+xn--vg-yiab.no
+vågå.no
+valer.ostfold.no
+xn--vler-qoa.xn--stfold-9xa.no
+våler.østfold.no
+valer.hedmark.no
+xn--vler-qoa.hedmark.no
+våler.hedmark.no
+
+// np : http://www.mos.com.np/register.html
+*.np
+
+// nr : http://cenpac.net.nr/dns/index.html
+// Submitted by registry <technician@cenpac.net.nr>
+nr
+biz.nr
+info.nr
+gov.nr
+edu.nr
+org.nr
+net.nr
+com.nr
+
+// nu : https://en.wikipedia.org/wiki/.nu
+nu
+
+// nz : https://en.wikipedia.org/wiki/.nz
+// Submitted by registry <jay@nzrs.net.nz>
+nz
+ac.nz
+co.nz
+cri.nz
+geek.nz
+gen.nz
+govt.nz
+health.nz
+iwi.nz
+kiwi.nz
+maori.nz
+mil.nz
+xn--mori-qsa.nz
+mÄori.nz
+net.nz
+org.nz
+parliament.nz
+school.nz
+
+// om : https://en.wikipedia.org/wiki/.om
+om
+co.om
+com.om
+edu.om
+gov.om
+med.om
+museum.om
+net.om
+org.om
+pro.om
+
+// onion : https://tools.ietf.org/html/rfc7686
+onion
+
+// org : https://en.wikipedia.org/wiki/.org
+org
+
+// pa : http://www.nic.pa/
+// Some additional second level "domains" resolve directly as hostnames, such as
+// pannet.pa, so we add a rule for "pa".
+pa
+ac.pa
+gob.pa
+com.pa
+org.pa
+sld.pa
+edu.pa
+net.pa
+ing.pa
+abo.pa
+med.pa
+nom.pa
+
+// pe : https://www.nic.pe/InformeFinalComision.pdf
+pe
+edu.pe
+gob.pe
+nom.pe
+mil.pe
+org.pe
+com.pe
+net.pe
+
+// pf : http://www.gobin.info/domainname/formulaire-pf.pdf
+pf
+com.pf
+org.pf
+edu.pf
+
+// pg : https://en.wikipedia.org/wiki/.pg
+*.pg
+
+// ph : http://www.domains.ph/FAQ2.asp
+// Submitted by registry <jed@email.com.ph>
+ph
+com.ph
+net.ph
+org.ph
+gov.ph
+edu.ph
+ngo.ph
+mil.ph
+i.ph
+
+// pk : http://pk5.pknic.net.pk/pk5/msgNamepk.PK
+pk
+com.pk
+net.pk
+edu.pk
+org.pk
+fam.pk
+biz.pk
+web.pk
+gov.pk
+gob.pk
+gok.pk
+gon.pk
+gop.pk
+gos.pk
+info.pk
+
+// pl http://www.dns.pl/english/index.html
+// Submitted by registry
+pl
+com.pl
+net.pl
+org.pl
+// pl functional domains (http://www.dns.pl/english/index.html)
+aid.pl
+agro.pl
+atm.pl
+auto.pl
+biz.pl
+edu.pl
+gmina.pl
+gsm.pl
+info.pl
+mail.pl
+miasta.pl
+media.pl
+mil.pl
+nieruchomosci.pl
+nom.pl
+pc.pl
+powiat.pl
+priv.pl
+realestate.pl
+rel.pl
+sex.pl
+shop.pl
+sklep.pl
+sos.pl
+szkola.pl
+targi.pl
+tm.pl
+tourism.pl
+travel.pl
+turystyka.pl
+// Government domains
+gov.pl
+ap.gov.pl
+ic.gov.pl
+is.gov.pl
+us.gov.pl
+kmpsp.gov.pl
+kppsp.gov.pl
+kwpsp.gov.pl
+psp.gov.pl
+wskr.gov.pl
+kwp.gov.pl
+mw.gov.pl
+ug.gov.pl
+um.gov.pl
+umig.gov.pl
+ugim.gov.pl
+upow.gov.pl
+uw.gov.pl
+starostwo.gov.pl
+pa.gov.pl
+po.gov.pl
+psse.gov.pl
+pup.gov.pl
+rzgw.gov.pl
+sa.gov.pl
+so.gov.pl
+sr.gov.pl
+wsa.gov.pl
+sko.gov.pl
+uzs.gov.pl
+wiih.gov.pl
+winb.gov.pl
+pinb.gov.pl
+wios.gov.pl
+witd.gov.pl
+wzmiuw.gov.pl
+piw.gov.pl
+wiw.gov.pl
+griw.gov.pl
+wif.gov.pl
+oum.gov.pl
+sdn.gov.pl
+zp.gov.pl
+uppo.gov.pl
+mup.gov.pl
+wuoz.gov.pl
+konsulat.gov.pl
+oirm.gov.pl
+// pl regional domains (http://www.dns.pl/english/index.html)
+augustow.pl
+babia-gora.pl
+bedzin.pl
+beskidy.pl
+bialowieza.pl
+bialystok.pl
+bielawa.pl
+bieszczady.pl
+boleslawiec.pl
+bydgoszcz.pl
+bytom.pl
+cieszyn.pl
+czeladz.pl
+czest.pl
+dlugoleka.pl
+elblag.pl
+elk.pl
+glogow.pl
+gniezno.pl
+gorlice.pl
+grajewo.pl
+ilawa.pl
+jaworzno.pl
+jelenia-gora.pl
+jgora.pl
+kalisz.pl
+kazimierz-dolny.pl
+karpacz.pl
+kartuzy.pl
+kaszuby.pl
+katowice.pl
+kepno.pl
+ketrzyn.pl
+klodzko.pl
+kobierzyce.pl
+kolobrzeg.pl
+konin.pl
+konskowola.pl
+kutno.pl
+lapy.pl
+lebork.pl
+legnica.pl
+lezajsk.pl
+limanowa.pl
+lomza.pl
+lowicz.pl
+lubin.pl
+lukow.pl
+malbork.pl
+malopolska.pl
+mazowsze.pl
+mazury.pl
+mielec.pl
+mielno.pl
+mragowo.pl
+naklo.pl
+nowaruda.pl
+nysa.pl
+olawa.pl
+olecko.pl
+olkusz.pl
+olsztyn.pl
+opoczno.pl
+opole.pl
+ostroda.pl
+ostroleka.pl
+ostrowiec.pl
+ostrowwlkp.pl
+pila.pl
+pisz.pl
+podhale.pl
+podlasie.pl
+polkowice.pl
+pomorze.pl
+pomorskie.pl
+prochowice.pl
+pruszkow.pl
+przeworsk.pl
+pulawy.pl
+radom.pl
+rawa-maz.pl
+rybnik.pl
+rzeszow.pl
+sanok.pl
+sejny.pl
+slask.pl
+slupsk.pl
+sosnowiec.pl
+stalowa-wola.pl
+skoczow.pl
+starachowice.pl
+stargard.pl
+suwalki.pl
+swidnica.pl
+swiebodzin.pl
+swinoujscie.pl
+szczecin.pl
+szczytno.pl
+tarnobrzeg.pl
+tgory.pl
+turek.pl
+tychy.pl
+ustka.pl
+walbrzych.pl
+warmia.pl
+warszawa.pl
+waw.pl
+wegrow.pl
+wielun.pl
+wlocl.pl
+wloclawek.pl
+wodzislaw.pl
+wolomin.pl
+wroclaw.pl
+zachpomor.pl
+zagan.pl
+zarow.pl
+zgora.pl
+zgorzelec.pl
+
+// pm : http://www.afnic.fr/medias/documents/AFNIC-naming-policy2012.pdf
+pm
+
+// pn : http://www.government.pn/PnRegistry/policies.htm
+pn
+gov.pn
+co.pn
+org.pn
+edu.pn
+net.pn
+
+// post : https://en.wikipedia.org/wiki/.post
+post
+
+// pr : http://www.nic.pr/index.asp?f=1
+pr
+com.pr
+net.pr
+org.pr
+gov.pr
+edu.pr
+isla.pr
+pro.pr
+biz.pr
+info.pr
+name.pr
+// these aren't mentioned on nic.pr, but on https://en.wikipedia.org/wiki/.pr
+est.pr
+prof.pr
+ac.pr
+
+// pro : http://registry.pro/get-pro
+pro
+aaa.pro
+aca.pro
+acct.pro
+avocat.pro
+bar.pro
+cpa.pro
+eng.pro
+jur.pro
+law.pro
+med.pro
+recht.pro
+
+// ps : https://en.wikipedia.org/wiki/.ps
+// http://www.nic.ps/registration/policy.html#reg
+ps
+edu.ps
+gov.ps
+sec.ps
+plo.ps
+com.ps
+org.ps
+net.ps
+
+// pt : http://online.dns.pt/dns/start_dns
+pt
+net.pt
+gov.pt
+org.pt
+edu.pt
+int.pt
+publ.pt
+com.pt
+nome.pt
+
+// pw : https://en.wikipedia.org/wiki/.pw
+pw
+co.pw
+ne.pw
+or.pw
+ed.pw
+go.pw
+belau.pw
+
+// py : http://www.nic.py/pautas.html#seccion_9
+// Submitted by registry
+py
+com.py
+coop.py
+edu.py
+gov.py
+mil.py
+net.py
+org.py
+
+// qa : http://domains.qa/en/
+qa
+com.qa
+edu.qa
+gov.qa
+mil.qa
+name.qa
+net.qa
+org.qa
+sch.qa
+
+// re : http://www.afnic.re/obtenir/chartes/nommage-re/annexe-descriptifs
+re
+asso.re
+com.re
+nom.re
+
+// ro : http://www.rotld.ro/
+ro
+arts.ro
+com.ro
+firm.ro
+info.ro
+nom.ro
+nt.ro
+org.ro
+rec.ro
+store.ro
+tm.ro
+www.ro
+
+// rs : https://www.rnids.rs/en/domains/national-domains
+rs
+ac.rs
+co.rs
+edu.rs
+gov.rs
+in.rs
+org.rs
+
+// ru : https://cctld.ru/files/pdf/docs/en/rules_ru-rf.pdf
+// Submitted by George Georgievsky <gug@cctld.ru>
+ru
+
+// rw : https://www.ricta.org.rw/sites/default/files/resources/registry_registrar_contract_0.pdf
+rw
+ac.rw
+co.rw
+coop.rw
+gov.rw
+mil.rw
+net.rw
+org.rw
+
+// sa : http://www.nic.net.sa/
+sa
+com.sa
+net.sa
+org.sa
+gov.sa
+med.sa
+pub.sa
+edu.sa
+sch.sa
+
+// sb : http://www.sbnic.net.sb/
+// Submitted by registry <lee.humphries@telekom.com.sb>
+sb
+com.sb
+edu.sb
+gov.sb
+net.sb
+org.sb
+
+// sc : http://www.nic.sc/
+sc
+com.sc
+gov.sc
+net.sc
+org.sc
+edu.sc
+
+// sd : http://www.isoc.sd/sudanic.isoc.sd/billing_pricing.htm
+// Submitted by registry <admin@isoc.sd>
+sd
+com.sd
+net.sd
+org.sd
+edu.sd
+med.sd
+tv.sd
+gov.sd
+info.sd
+
+// se : https://en.wikipedia.org/wiki/.se
+// Submitted by registry <patrik.wallstrom@iis.se>
+se
+a.se
+ac.se
+b.se
+bd.se
+brand.se
+c.se
+d.se
+e.se
+f.se
+fh.se
+fhsk.se
+fhv.se
+g.se
+h.se
+i.se
+k.se
+komforb.se
+kommunalforbund.se
+komvux.se
+l.se
+lanbib.se
+m.se
+n.se
+naturbruksgymn.se
+o.se
+org.se
+p.se
+parti.se
+pp.se
+press.se
+r.se
+s.se
+t.se
+tm.se
+u.se
+w.se
+x.se
+y.se
+z.se
+
+// sg : http://www.nic.net.sg/page/registration-policies-procedures-and-guidelines
+sg
+com.sg
+net.sg
+org.sg
+gov.sg
+edu.sg
+per.sg
+
+// sh : http://www.nic.sh/registrar.html
+sh
+com.sh
+net.sh
+gov.sh
+org.sh
+mil.sh
+
+// si : https://en.wikipedia.org/wiki/.si
+si
+
+// sj : No registrations at this time.
+// Submitted by registry <jarle@uninett.no>
+sj
+
+// sk : https://en.wikipedia.org/wiki/.sk
+// list of 2nd level domains ?
+sk
+
+// sl : http://www.nic.sl
+// Submitted by registry <adam@neoip.com>
+sl
+com.sl
+net.sl
+edu.sl
+gov.sl
+org.sl
+
+// sm : https://en.wikipedia.org/wiki/.sm
+sm
+
+// sn : https://en.wikipedia.org/wiki/.sn
+sn
+art.sn
+com.sn
+edu.sn
+gouv.sn
+org.sn
+perso.sn
+univ.sn
+
+// so : http://sonic.so/policies/
+so
+com.so
+edu.so
+gov.so
+me.so
+net.so
+org.so
+
+// sr : https://en.wikipedia.org/wiki/.sr
+sr
+
+// ss : https://registry.nic.ss/
+// Submitted by registry <technical@nic.ss>
+ss
+biz.ss
+com.ss
+edu.ss
+gov.ss
+me.ss
+net.ss
+org.ss
+sch.ss
+
+// st : http://www.nic.st/html/policyrules/
+st
+co.st
+com.st
+consulado.st
+edu.st
+embaixada.st
+mil.st
+net.st
+org.st
+principe.st
+saotome.st
+store.st
+
+// su : https://en.wikipedia.org/wiki/.su
+su
+
+// sv : http://www.svnet.org.sv/niveldos.pdf
+sv
+com.sv
+edu.sv
+gob.sv
+org.sv
+red.sv
+
+// sx : https://en.wikipedia.org/wiki/.sx
+// Submitted by registry <jcvignes@openregistry.com>
+sx
+gov.sx
+
+// sy : https://en.wikipedia.org/wiki/.sy
+// see also: http://www.gobin.info/domainname/sy.doc
+sy
+edu.sy
+gov.sy
+net.sy
+mil.sy
+com.sy
+org.sy
+
+// sz : https://en.wikipedia.org/wiki/.sz
+// http://www.sispa.org.sz/
+sz
+co.sz
+ac.sz
+org.sz
+
+// tc : https://en.wikipedia.org/wiki/.tc
+tc
+
+// td : https://en.wikipedia.org/wiki/.td
+td
+
+// tel: https://en.wikipedia.org/wiki/.tel
+// http://www.telnic.org/
+tel
+
+// tf : https://en.wikipedia.org/wiki/.tf
+tf
+
+// tg : https://en.wikipedia.org/wiki/.tg
+// http://www.nic.tg/
+tg
+
+// th : https://en.wikipedia.org/wiki/.th
+// Submitted by registry <krit@thains.co.th>
+th
+ac.th
+co.th
+go.th
+in.th
+mi.th
+net.th
+or.th
+
+// tj : http://www.nic.tj/policy.html
+tj
+ac.tj
+biz.tj
+co.tj
+com.tj
+edu.tj
+go.tj
+gov.tj
+int.tj
+mil.tj
+name.tj
+net.tj
+nic.tj
+org.tj
+test.tj
+web.tj
+
+// tk : https://en.wikipedia.org/wiki/.tk
+tk
+
+// tl : https://en.wikipedia.org/wiki/.tl
+tl
+gov.tl
+
+// tm : http://www.nic.tm/local.html
+tm
+com.tm
+co.tm
+org.tm
+net.tm
+nom.tm
+gov.tm
+mil.tm
+edu.tm
+
+// tn : https://en.wikipedia.org/wiki/.tn
+// http://whois.ati.tn/
+tn
+com.tn
+ens.tn
+fin.tn
+gov.tn
+ind.tn
+intl.tn
+nat.tn
+net.tn
+org.tn
+info.tn
+perso.tn
+tourism.tn
+edunet.tn
+rnrt.tn
+rns.tn
+rnu.tn
+mincom.tn
+agrinet.tn
+defense.tn
+turen.tn
+
+// to : https://en.wikipedia.org/wiki/.to
+// Submitted by registry <egullich@colo.to>
+to
+com.to
+gov.to
+net.to
+org.to
+edu.to
+mil.to
+
+// tr : https://nic.tr/
+// https://nic.tr/forms/eng/policies.pdf
+// https://nic.tr/index.php?USRACTN=PRICELST
+tr
+av.tr
+bbs.tr
+bel.tr
+biz.tr
+com.tr
+dr.tr
+edu.tr
+gen.tr
+gov.tr
+info.tr
+mil.tr
+k12.tr
+kep.tr
+name.tr
+net.tr
+org.tr
+pol.tr
+tel.tr
+tsk.tr
+tv.tr
+web.tr
+// Used by Northern Cyprus
+nc.tr
+// Used by government agencies of Northern Cyprus
+gov.nc.tr
+
+// tt : http://www.nic.tt/
+tt
+co.tt
+com.tt
+org.tt
+net.tt
+biz.tt
+info.tt
+pro.tt
+int.tt
+coop.tt
+jobs.tt
+mobi.tt
+travel.tt
+museum.tt
+aero.tt
+name.tt
+gov.tt
+edu.tt
+
+// tv : https://en.wikipedia.org/wiki/.tv
+// Not listing any 2LDs as reserved since none seem to exist in practice,
+// Wikipedia notwithstanding.
+tv
+
+// tw : https://en.wikipedia.org/wiki/.tw
+tw
+edu.tw
+gov.tw
+mil.tw
+com.tw
+net.tw
+org.tw
+idv.tw
+game.tw
+ebiz.tw
+club.tw
+xn--zf0ao64a.tw
+網路.tw
+xn--uc0atv.tw
+組織.tw
+xn--czrw28b.tw
+商業.tw
+
+// tz : http://www.tznic.or.tz/index.php/domains
+// Submitted by registry <manager@tznic.or.tz>
+tz
+ac.tz
+co.tz
+go.tz
+hotel.tz
+info.tz
+me.tz
+mil.tz
+mobi.tz
+ne.tz
+or.tz
+sc.tz
+tv.tz
+
+// ua : https://hostmaster.ua/policy/?ua
+// Submitted by registry <dk@cctld.ua>
+ua
+// ua 2LD
+com.ua
+edu.ua
+gov.ua
+in.ua
+net.ua
+org.ua
+// ua geographic names
+// https://hostmaster.ua/2ld/
+cherkassy.ua
+cherkasy.ua
+chernigov.ua
+chernihiv.ua
+chernivtsi.ua
+chernovtsy.ua
+ck.ua
+cn.ua
+cr.ua
+crimea.ua
+cv.ua
+dn.ua
+dnepropetrovsk.ua
+dnipropetrovsk.ua
+donetsk.ua
+dp.ua
+if.ua
+ivano-frankivsk.ua
+kh.ua
+kharkiv.ua
+kharkov.ua
+kherson.ua
+khmelnitskiy.ua
+khmelnytskyi.ua
+kiev.ua
+kirovograd.ua
+km.ua
+kr.ua
+krym.ua
+ks.ua
+kv.ua
+kyiv.ua
+lg.ua
+lt.ua
+lugansk.ua
+lutsk.ua
+lv.ua
+lviv.ua
+mk.ua
+mykolaiv.ua
+nikolaev.ua
+od.ua
+odesa.ua
+odessa.ua
+pl.ua
+poltava.ua
+rivne.ua
+rovno.ua
+rv.ua
+sb.ua
+sebastopol.ua
+sevastopol.ua
+sm.ua
+sumy.ua
+te.ua
+ternopil.ua
+uz.ua
+uzhgorod.ua
+vinnica.ua
+vinnytsia.ua
+vn.ua
+volyn.ua
+yalta.ua
+zaporizhzhe.ua
+zaporizhzhia.ua
+zhitomir.ua
+zhytomyr.ua
+zp.ua
+zt.ua
+
+// ug : https://www.registry.co.ug/
+ug
+co.ug
+or.ug
+ac.ug
+sc.ug
+go.ug
+ne.ug
+com.ug
+org.ug
+
+// uk : https://en.wikipedia.org/wiki/.uk
+// Submitted by registry <Michael.Daly@nominet.org.uk>
+uk
+ac.uk
+co.uk
+gov.uk
+ltd.uk
+me.uk
+net.uk
+nhs.uk
+org.uk
+plc.uk
+police.uk
+*.sch.uk
+
+// us : https://en.wikipedia.org/wiki/.us
+us
+dni.us
+fed.us
+isa.us
+kids.us
+nsn.us
+// us geographic names
+ak.us
+al.us
+ar.us
+as.us
+az.us
+ca.us
+co.us
+ct.us
+dc.us
+de.us
+fl.us
+ga.us
+gu.us
+hi.us
+ia.us
+id.us
+il.us
+in.us
+ks.us
+ky.us
+la.us
+ma.us
+md.us
+me.us
+mi.us
+mn.us
+mo.us
+ms.us
+mt.us
+nc.us
+nd.us
+ne.us
+nh.us
+nj.us
+nm.us
+nv.us
+ny.us
+oh.us
+ok.us
+or.us
+pa.us
+pr.us
+ri.us
+sc.us
+sd.us
+tn.us
+tx.us
+ut.us
+vi.us
+vt.us
+va.us
+wa.us
+wi.us
+wv.us
+wy.us
+// The registrar notes several more specific domains available in each state,
+// such as state.*.us, dst.*.us, etc., but resolution of these is somewhat
+// haphazard; in some states these domains resolve as addresses, while in others
+// only subdomains are available, or even nothing at all. We include the
+// most common ones where it's clear that different sites are different
+// entities.
+k12.ak.us
+k12.al.us
+k12.ar.us
+k12.as.us
+k12.az.us
+k12.ca.us
+k12.co.us
+k12.ct.us
+k12.dc.us
+k12.de.us
+k12.fl.us
+k12.ga.us
+k12.gu.us
+// k12.hi.us Bug 614565 - Hawaii has a state-wide DOE login
+k12.ia.us
+k12.id.us
+k12.il.us
+k12.in.us
+k12.ks.us
+k12.ky.us
+k12.la.us
+k12.ma.us
+k12.md.us
+k12.me.us
+k12.mi.us
+k12.mn.us
+k12.mo.us
+k12.ms.us
+k12.mt.us
+k12.nc.us
+// k12.nd.us Bug 1028347 - Removed at request of Travis Rosso <trossow@nd.gov>
+k12.ne.us
+k12.nh.us
+k12.nj.us
+k12.nm.us
+k12.nv.us
+k12.ny.us
+k12.oh.us
+k12.ok.us
+k12.or.us
+k12.pa.us
+k12.pr.us
+// k12.ri.us Removed at request of Kim Cournoyer <netsupport@staff.ri.net>
+k12.sc.us
+// k12.sd.us Bug 934131 - Removed at request of James Booze <James.Booze@k12.sd.us>
+k12.tn.us
+k12.tx.us
+k12.ut.us
+k12.vi.us
+k12.vt.us
+k12.va.us
+k12.wa.us
+k12.wi.us
+// k12.wv.us Bug 947705 - Removed at request of Verne Britton <verne@wvnet.edu>
+k12.wy.us
+cc.ak.us
+cc.al.us
+cc.ar.us
+cc.as.us
+cc.az.us
+cc.ca.us
+cc.co.us
+cc.ct.us
+cc.dc.us
+cc.de.us
+cc.fl.us
+cc.ga.us
+cc.gu.us
+cc.hi.us
+cc.ia.us
+cc.id.us
+cc.il.us
+cc.in.us
+cc.ks.us
+cc.ky.us
+cc.la.us
+cc.ma.us
+cc.md.us
+cc.me.us
+cc.mi.us
+cc.mn.us
+cc.mo.us
+cc.ms.us
+cc.mt.us
+cc.nc.us
+cc.nd.us
+cc.ne.us
+cc.nh.us
+cc.nj.us
+cc.nm.us
+cc.nv.us
+cc.ny.us
+cc.oh.us
+cc.ok.us
+cc.or.us
+cc.pa.us
+cc.pr.us
+cc.ri.us
+cc.sc.us
+cc.sd.us
+cc.tn.us
+cc.tx.us
+cc.ut.us
+cc.vi.us
+cc.vt.us
+cc.va.us
+cc.wa.us
+cc.wi.us
+cc.wv.us
+cc.wy.us
+lib.ak.us
+lib.al.us
+lib.ar.us
+lib.as.us
+lib.az.us
+lib.ca.us
+lib.co.us
+lib.ct.us
+lib.dc.us
+// lib.de.us Issue #243 - Moved to Private section at request of Ed Moore <Ed.Moore@lib.de.us>
+lib.fl.us
+lib.ga.us
+lib.gu.us
+lib.hi.us
+lib.ia.us
+lib.id.us
+lib.il.us
+lib.in.us
+lib.ks.us
+lib.ky.us
+lib.la.us
+lib.ma.us
+lib.md.us
+lib.me.us
+lib.mi.us
+lib.mn.us
+lib.mo.us
+lib.ms.us
+lib.mt.us
+lib.nc.us
+lib.nd.us
+lib.ne.us
+lib.nh.us
+lib.nj.us
+lib.nm.us
+lib.nv.us
+lib.ny.us
+lib.oh.us
+lib.ok.us
+lib.or.us
+lib.pa.us
+lib.pr.us
+lib.ri.us
+lib.sc.us
+lib.sd.us
+lib.tn.us
+lib.tx.us
+lib.ut.us
+lib.vi.us
+lib.vt.us
+lib.va.us
+lib.wa.us
+lib.wi.us
+// lib.wv.us Bug 941670 - Removed at request of Larry W Arnold <arnold@wvlc.lib.wv.us>
+lib.wy.us
+// k12.ma.us contains school districts in Massachusetts. The 4LDs are
+// managed independently except for private (PVT), charter (CHTR) and
+// parochial (PAROCH) schools. Those are delegated directly to the
+// 5LD operators. <k12-ma-hostmaster _ at _ rsuc.gweep.net>
+pvt.k12.ma.us
+chtr.k12.ma.us
+paroch.k12.ma.us
+// Merit Network, Inc. maintains the registry for =~ /(k12|cc|lib).mi.us/ and the following
+// see also: http://domreg.merit.edu
+// see also: whois -h whois.domreg.merit.edu help
+ann-arbor.mi.us
+cog.mi.us
+dst.mi.us
+eaton.mi.us
+gen.mi.us
+mus.mi.us
+tec.mi.us
+washtenaw.mi.us
+
+// uy : http://www.nic.org.uy/
+uy
+com.uy
+edu.uy
+gub.uy
+mil.uy
+net.uy
+org.uy
+
+// uz : http://www.reg.uz/
+uz
+co.uz
+com.uz
+net.uz
+org.uz
+
+// va : https://en.wikipedia.org/wiki/.va
+va
+
+// vc : https://en.wikipedia.org/wiki/.vc
+// Submitted by registry <kshah@ca.afilias.info>
+vc
+com.vc
+net.vc
+org.vc
+gov.vc
+mil.vc
+edu.vc
+
+// ve : https://registro.nic.ve/
+// Submitted by registry
+ve
+arts.ve
+co.ve
+com.ve
+e12.ve
+edu.ve
+firm.ve
+gob.ve
+gov.ve
+info.ve
+int.ve
+mil.ve
+net.ve
+org.ve
+rec.ve
+store.ve
+tec.ve
+web.ve
+
+// vg : https://en.wikipedia.org/wiki/.vg
+vg
+
+// vi : http://www.nic.vi/newdomainform.htm
+// http://www.nic.vi/Domain_Rules/body_domain_rules.html indicates some other
+// TLDs are "reserved", such as edu.vi and gov.vi, but doesn't actually say they
+// are available for registration (which they do not seem to be).
+vi
+co.vi
+com.vi
+k12.vi
+net.vi
+org.vi
+
+// vn : https://www.dot.vn/vnnic/vnnic/domainregistration.jsp
+vn
+com.vn
+net.vn
+org.vn
+edu.vn
+gov.vn
+int.vn
+ac.vn
+biz.vn
+info.vn
+name.vn
+pro.vn
+health.vn
+
+// vu : https://en.wikipedia.org/wiki/.vu
+// http://www.vunic.vu/
+vu
+com.vu
+edu.vu
+net.vu
+org.vu
+
+// wf : http://www.afnic.fr/medias/documents/AFNIC-naming-policy2012.pdf
+wf
+
+// ws : https://en.wikipedia.org/wiki/.ws
+// http://samoanic.ws/index.dhtml
+ws
+com.ws
+net.ws
+org.ws
+gov.ws
+edu.ws
+
+// yt : http://www.afnic.fr/medias/documents/AFNIC-naming-policy2012.pdf
+yt
+
+// IDN ccTLDs
+// When submitting patches, please maintain a sort by ISO 3166 ccTLD, then
+// U-label, and follow this format:
+// // A-Label ("<Latin renderings>", <language name>[, variant info]) : <ISO 3166 ccTLD>
+// // [sponsoring org]
+// U-Label
+
+// xn--mgbaam7a8h ("Emerat", Arabic) : AE
+// http://nic.ae/english/arabicdomain/rules.jsp
+xn--mgbaam7a8h
+امارات
+
+// xn--y9a3aq ("hye", Armenian) : AM
+// ISOC AM (operated by .am Registry)
+xn--y9a3aq
+Õ°Õ¡Õµ
+
+// xn--54b7fta0cc ("Bangla", Bangla) : BD
+xn--54b7fta0cc
+বাংলা
+
+// xn--90ae ("bg", Bulgarian) : BG
+xn--90ae
+бг
+
+// xn--mgbcpq6gpa1a ("albahrain", Arabic) : BH
+xn--mgbcpq6gpa1a
+البحرين
+
+// xn--90ais ("bel", Belarusian/Russian Cyrillic) : BY
+// Operated by .by registry
+xn--90ais
+бел
+
+// xn--fiqs8s ("Zhongguo/China", Chinese, Simplified) : CN
+// CNNIC
+// http://cnnic.cn/html/Dir/2005/10/11/3218.htm
+xn--fiqs8s
+中国
+
+// xn--fiqz9s ("Zhongguo/China", Chinese, Traditional) : CN
+// CNNIC
+// http://cnnic.cn/html/Dir/2005/10/11/3218.htm
+xn--fiqz9s
+中國
+
+// xn--lgbbat1ad8j ("Algeria/Al Jazair", Arabic) : DZ
+xn--lgbbat1ad8j
+الجزائر
+
+// xn--wgbh1c ("Egypt/Masr", Arabic) : EG
+// http://www.dotmasr.eg/
+xn--wgbh1c
+مصر
+
+// xn--e1a4c ("eu", Cyrillic) : EU
+// https://eurid.eu
+xn--e1a4c
+ею
+
+// xn--qxa6a ("eu", Greek) : EU
+// https://eurid.eu
+xn--qxa6a
+ευ
+
+// xn--mgbah1a3hjkrd ("Mauritania", Arabic) : MR
+xn--mgbah1a3hjkrd
+موريتانيا
+
+// xn--node ("ge", Georgian Mkhedruli) : GE
+xn--node
+გე
+
+// xn--qxam ("el", Greek) : GR
+// Hellenic Ministry of Infrastructure, Transport, and Networks
+xn--qxam
+ελ
+
+// xn--j6w193g ("Hong Kong", Chinese) : HK
+// https://www.hkirc.hk
+// Submitted by registry <hk.tech@hkirc.hk>
+// https://www.hkirc.hk/content.jsp?id=30#!/34
+xn--j6w193g
+香港
+xn--55qx5d.xn--j6w193g
+å…¬å¸.香港
+xn--wcvs22d.xn--j6w193g
+教育.香港
+xn--mxtq1m.xn--j6w193g
+政府.香港
+xn--gmqw5a.xn--j6w193g
+個人.香港
+xn--od0alg.xn--j6w193g
+網絡.香港
+xn--uc0atv.xn--j6w193g
+組織.香港
+
+// xn--2scrj9c ("Bharat", Kannada) : IN
+// India
+xn--2scrj9c
+ಭಾರತ
+
+// xn--3hcrj9c ("Bharat", Oriya) : IN
+// India
+xn--3hcrj9c
+ଭାରତ
+
+// xn--45br5cyl ("Bharatam", Assamese) : IN
+// India
+xn--45br5cyl
+ভাৰত
+
+// xn--h2breg3eve ("Bharatam", Sanskrit) : IN
+// India
+xn--h2breg3eve
+भारतमà¥
+
+// xn--h2brj9c8c ("Bharot", Santali) : IN
+// India
+xn--h2brj9c8c
+भारोत
+
+// xn--mgbgu82a ("Bharat", Sindhi) : IN
+// India
+xn--mgbgu82a
+ڀارت
+
+// xn--rvc1e0am3e ("Bharatam", Malayalam) : IN
+// India
+xn--rvc1e0am3e
+ഭാരതം
+
+// xn--h2brj9c ("Bharat", Devanagari) : IN
+// India
+xn--h2brj9c
+भारत
+
+// xn--mgbbh1a ("Bharat", Kashmiri) : IN
+// India
+xn--mgbbh1a
+بارت
+
+// xn--mgbbh1a71e ("Bharat", Arabic) : IN
+// India
+xn--mgbbh1a71e
+بھارت
+
+// xn--fpcrj9c3d ("Bharat", Telugu) : IN
+// India
+xn--fpcrj9c3d
+భారతà±
+
+// xn--gecrj9c ("Bharat", Gujarati) : IN
+// India
+xn--gecrj9c
+ભારત
+
+// xn--s9brj9c ("Bharat", Gurmukhi) : IN
+// India
+xn--s9brj9c
+ਭਾਰਤ
+
+// xn--45brj9c ("Bharat", Bengali) : IN
+// India
+xn--45brj9c
+ভারত
+
+// xn--xkc2dl3a5ee0h ("India", Tamil) : IN
+// India
+xn--xkc2dl3a5ee0h
+இநà¯à®¤à®¿à®¯à®¾
+
+// xn--mgba3a4f16a ("Iran", Persian) : IR
+xn--mgba3a4f16a
+ایران
+
+// xn--mgba3a4fra ("Iran", Arabic) : IR
+xn--mgba3a4fra
+ايران
+
+// xn--mgbtx2b ("Iraq", Arabic) : IQ
+// Communications and Media Commission
+xn--mgbtx2b
+عراق
+
+// xn--mgbayh7gpa ("al-Ordon", Arabic) : JO
+// National Information Technology Center (NITC)
+// Royal Scientific Society, Al-Jubeiha
+xn--mgbayh7gpa
+الاردن
+
+// xn--3e0b707e ("Republic of Korea", Hangul) : KR
+xn--3e0b707e
+한국
+
+// xn--80ao21a ("Kaz", Kazakh) : KZ
+xn--80ao21a
+қаз
+
+// xn--q7ce6a ("Lao", Lao) : LA
+xn--q7ce6a
+ລາວ
+
+// xn--fzc2c9e2c ("Lanka", Sinhalese-Sinhala) : LK
+// https://nic.lk
+xn--fzc2c9e2c
+ලංකà·
+
+// xn--xkc2al3hye2a ("Ilangai", Tamil) : LK
+// https://nic.lk
+xn--xkc2al3hye2a
+இலஙà¯à®•à¯ˆ
+
+// xn--mgbc0a9azcg ("Morocco/al-Maghrib", Arabic) : MA
+xn--mgbc0a9azcg
+المغرب
+
+// xn--d1alf ("mkd", Macedonian) : MK
+// MARnet
+xn--d1alf
+мкд
+
+// xn--l1acc ("mon", Mongolian) : MN
+xn--l1acc
+мон
+
+// xn--mix891f ("Macao", Chinese, Traditional) : MO
+// MONIC / HNET Asia (Registry Operator for .mo)
+xn--mix891f
+澳門
+
+// xn--mix082f ("Macao", Chinese, Simplified) : MO
+xn--mix082f
+澳门
+
+// xn--mgbx4cd0ab ("Malaysia", Malay) : MY
+xn--mgbx4cd0ab
+مليسيا
+
+// xn--mgb9awbf ("Oman", Arabic) : OM
+xn--mgb9awbf
+عمان
+
+// xn--mgbai9azgqp6j ("Pakistan", Urdu/Arabic) : PK
+xn--mgbai9azgqp6j
+پاکستان
+
+// xn--mgbai9a5eva00b ("Pakistan", Urdu/Arabic, variant) : PK
+xn--mgbai9a5eva00b
+پاكستان
+
+// xn--ygbi2ammx ("Falasteen", Arabic) : PS
+// The Palestinian National Internet Naming Authority (PNINA)
+// http://www.pnina.ps
+xn--ygbi2ammx
+Ùلسطين
+
+// xn--90a3ac ("srb", Cyrillic) : RS
+// https://www.rnids.rs/en/domains/national-domains
+xn--90a3ac
+Ñрб
+xn--o1ac.xn--90a3ac
+пр.Ñрб
+xn--c1avg.xn--90a3ac
+орг.Ñрб
+xn--90azh.xn--90a3ac
+обр.Ñрб
+xn--d1at.xn--90a3ac
+од.Ñрб
+xn--o1ach.xn--90a3ac
+упр.Ñрб
+xn--80au.xn--90a3ac
+ак.Ñрб
+
+// xn--p1ai ("rf", Russian-Cyrillic) : RU
+// https://cctld.ru/files/pdf/docs/en/rules_ru-rf.pdf
+// Submitted by George Georgievsky <gug@cctld.ru>
+xn--p1ai
+рф
+
+// xn--wgbl6a ("Qatar", Arabic) : QA
+// http://www.ict.gov.qa/
+xn--wgbl6a
+قطر
+
+// xn--mgberp4a5d4ar ("AlSaudiah", Arabic) : SA
+// http://www.nic.net.sa/
+xn--mgberp4a5d4ar
+السعودية
+
+// xn--mgberp4a5d4a87g ("AlSaudiah", Arabic, variant) : SA
+xn--mgberp4a5d4a87g
+السعودیة
+
+// xn--mgbqly7c0a67fbc ("AlSaudiah", Arabic, variant) : SA
+xn--mgbqly7c0a67fbc
+السعودیۃ
+
+// xn--mgbqly7cvafr ("AlSaudiah", Arabic, variant) : SA
+xn--mgbqly7cvafr
+السعوديه
+
+// xn--mgbpl2fh ("sudan", Arabic) : SD
+// Operated by .sd registry
+xn--mgbpl2fh
+سودان
+
+// xn--yfro4i67o Singapore ("Singapore", Chinese) : SG
+xn--yfro4i67o
+新加å¡
+
+// xn--clchc0ea0b2g2a9gcd ("Singapore", Tamil) : SG
+xn--clchc0ea0b2g2a9gcd
+சிஙà¯à®•à®ªà¯à®ªà¯‚à®°à¯
+
+// xn--ogbpf8fl ("Syria", Arabic) : SY
+xn--ogbpf8fl
+سورية
+
+// xn--mgbtf8fl ("Syria", Arabic, variant) : SY
+xn--mgbtf8fl
+سوريا
+
+// xn--o3cw4h ("Thai", Thai) : TH
+// http://www.thnic.co.th
+xn--o3cw4h
+ไทย
+xn--12c1fe0br.xn--o3cw4h
+ศึà¸à¸©à¸².ไทย
+xn--12co0c3b4eva.xn--o3cw4h
+ธุรà¸à¸´à¸ˆ.ไทย
+xn--h3cuzk1di.xn--o3cw4h
+รัà¸à¸šà¸²à¸¥.ไทย
+xn--o3cyx2a.xn--o3cw4h
+ทหาร.ไทย
+xn--m3ch0j3a.xn--o3cw4h
+เน็ต.ไทย
+xn--12cfi8ixb8l.xn--o3cw4h
+องค์à¸à¸£.ไทย
+
+// xn--pgbs0dh ("Tunisia", Arabic) : TN
+// http://nic.tn
+xn--pgbs0dh
+تونس
+
+// xn--kpry57d ("Taiwan", Chinese, Traditional) : TW
+// http://www.twnic.net/english/dn/dn_07a.htm
+xn--kpry57d
+å°ç£
+
+// xn--kprw13d ("Taiwan", Chinese, Simplified) : TW
+// http://www.twnic.net/english/dn/dn_07a.htm
+xn--kprw13d
+å°æ¹¾
+
+// xn--nnx388a ("Taiwan", Chinese, variant) : TW
+xn--nnx388a
+臺ç£
+
+// xn--j1amh ("ukr", Cyrillic) : UA
+xn--j1amh
+укр
+
+// xn--mgb2ddes ("AlYemen", Arabic) : YE
+xn--mgb2ddes
+اليمن
+
+// xxx : http://icmregistry.com
+xxx
+
+// ye : http://www.y.net.ye/services/domain_name.htm
+ye
+com.ye
+edu.ye
+gov.ye
+net.ye
+mil.ye
+org.ye
+
+// za : https://www.zadna.org.za/content/page/domain-information/
+ac.za
+agric.za
+alt.za
+co.za
+edu.za
+gov.za
+grondar.za
+law.za
+mil.za
+net.za
+ngo.za
+nic.za
+nis.za
+nom.za
+org.za
+school.za
+tm.za
+web.za
+
+// zm : https://zicta.zm/
+// Submitted by registry <info@zicta.zm>
+zm
+ac.zm
+biz.zm
+co.zm
+com.zm
+edu.zm
+gov.zm
+info.zm
+mil.zm
+net.zm
+org.zm
+sch.zm
+
+// zw : https://www.potraz.gov.zw/
+// Confirmed by registry <bmtengwa@potraz.gov.zw> 2017-01-25
+zw
+ac.zw
+co.zw
+gov.zw
+mil.zw
+org.zw
+
+
+// newGTLDs
+
+// List of new gTLDs imported from https://www.icann.org/resources/registries/gtlds/v2/gtlds.json on 2021-08-05T15:14:20Z
+// This list is auto-generated, don't edit it manually.
+// aaa : 2015-02-26 American Automobile Association, Inc.
+aaa
+
+// aarp : 2015-05-21 AARP
+aarp
+
+// abarth : 2015-07-30 Fiat Chrysler Automobiles N.V.
+abarth
+
+// abb : 2014-10-24 ABB Ltd
+abb
+
+// abbott : 2014-07-24 Abbott Laboratories, Inc.
+abbott
+
+// abbvie : 2015-07-30 AbbVie Inc.
+abbvie
+
+// abc : 2015-07-30 Disney Enterprises, Inc.
+abc
+
+// able : 2015-06-25 Able Inc.
+able
+
+// abogado : 2014-04-24 Registry Services, LLC
+abogado
+
+// abudhabi : 2015-07-30 Abu Dhabi Systems and Information Centre
+abudhabi
+
+// academy : 2013-11-07 Binky Moon, LLC
+academy
+
+// accenture : 2014-08-15 Accenture plc
+accenture
+
+// accountant : 2014-11-20 dot Accountant Limited
+accountant
+
+// accountants : 2014-03-20 Binky Moon, LLC
+accountants
+
+// aco : 2015-01-08 ACO Severin Ahlmann GmbH & Co. KG
+aco
+
+// actor : 2013-12-12 Dog Beach, LLC
+actor
+
+// adac : 2015-07-16 Allgemeiner Deutscher Automobil-Club e.V. (ADAC)
+adac
+
+// ads : 2014-12-04 Charleston Road Registry Inc.
+ads
+
+// adult : 2014-10-16 ICM Registry AD LLC
+adult
+
+// aeg : 2015-03-19 Aktiebolaget Electrolux
+aeg
+
+// aetna : 2015-05-21 Aetna Life Insurance Company
+aetna
+
+// afamilycompany : 2015-07-23 Johnson Shareholdings, Inc.
+afamilycompany
+
+// afl : 2014-10-02 Australian Football League
+afl
+
+// africa : 2014-03-24 ZA Central Registry NPC trading as Registry.Africa
+africa
+
+// agakhan : 2015-04-23 Fondation Aga Khan (Aga Khan Foundation)
+agakhan
+
+// agency : 2013-11-14 Binky Moon, LLC
+agency
+
+// aig : 2014-12-18 American International Group, Inc.
+aig
+
+// airbus : 2015-07-30 Airbus S.A.S.
+airbus
+
+// airforce : 2014-03-06 Dog Beach, LLC
+airforce
+
+// airtel : 2014-10-24 Bharti Airtel Limited
+airtel
+
+// akdn : 2015-04-23 Fondation Aga Khan (Aga Khan Foundation)
+akdn
+
+// alfaromeo : 2015-07-31 Fiat Chrysler Automobiles N.V.
+alfaromeo
+
+// alibaba : 2015-01-15 Alibaba Group Holding Limited
+alibaba
+
+// alipay : 2015-01-15 Alibaba Group Holding Limited
+alipay
+
+// allfinanz : 2014-07-03 Allfinanz Deutsche Vermögensberatung Aktiengesellschaft
+allfinanz
+
+// allstate : 2015-07-31 Allstate Fire and Casualty Insurance Company
+allstate
+
+// ally : 2015-06-18 Ally Financial Inc.
+ally
+
+// alsace : 2014-07-02 Region Grand Est
+alsace
+
+// alstom : 2015-07-30 ALSTOM
+alstom
+
+// amazon : 2019-12-19 Amazon Registry Services, Inc.
+amazon
+
+// americanexpress : 2015-07-31 American Express Travel Related Services Company, Inc.
+americanexpress
+
+// americanfamily : 2015-07-23 AmFam, Inc.
+americanfamily
+
+// amex : 2015-07-31 American Express Travel Related Services Company, Inc.
+amex
+
+// amfam : 2015-07-23 AmFam, Inc.
+amfam
+
+// amica : 2015-05-28 Amica Mutual Insurance Company
+amica
+
+// amsterdam : 2014-07-24 Gemeente Amsterdam
+amsterdam
+
+// analytics : 2014-12-18 Campus IP LLC
+analytics
+
+// android : 2014-08-07 Charleston Road Registry Inc.
+android
+
+// anquan : 2015-01-08 Beijing Qihu Keji Co., Ltd.
+anquan
+
+// anz : 2015-07-31 Australia and New Zealand Banking Group Limited
+anz
+
+// aol : 2015-09-17 Oath Inc.
+aol
+
+// apartments : 2014-12-11 Binky Moon, LLC
+apartments
+
+// app : 2015-05-14 Charleston Road Registry Inc.
+app
+
+// apple : 2015-05-14 Apple Inc.
+apple
+
+// aquarelle : 2014-07-24 Aquarelle.com
+aquarelle
+
+// arab : 2015-11-12 League of Arab States
+arab
+
+// aramco : 2014-11-20 Aramco Services Company
+aramco
+
+// archi : 2014-02-06 Afilias Limited
+archi
+
+// army : 2014-03-06 Dog Beach, LLC
+army
+
+// art : 2016-03-24 UK Creative Ideas Limited
+art
+
+// arte : 2014-12-11 Association Relative à la Télévision Européenne G.E.I.E.
+arte
+
+// asda : 2015-07-31 Wal-Mart Stores, Inc.
+asda
+
+// associates : 2014-03-06 Binky Moon, LLC
+associates
+
+// athleta : 2015-07-30 The Gap, Inc.
+athleta
+
+// attorney : 2014-03-20 Dog Beach, LLC
+attorney
+
+// auction : 2014-03-20 Dog Beach, LLC
+auction
+
+// audi : 2015-05-21 AUDI Aktiengesellschaft
+audi
+
+// audible : 2015-06-25 Amazon Registry Services, Inc.
+audible
+
+// audio : 2014-03-20 UNR Corp.
+audio
+
+// auspost : 2015-08-13 Australian Postal Corporation
+auspost
+
+// author : 2014-12-18 Amazon Registry Services, Inc.
+author
+
+// auto : 2014-11-13 XYZ.COM LLC
+auto
+
+// autos : 2014-01-09 XYZ.COM LLC
+autos
+
+// avianca : 2015-01-08 Avianca Holdings S.A.
+avianca
+
+// aws : 2015-06-25 AWS Registry LLC
+aws
+
+// axa : 2013-12-19 AXA Group Operations SAS
+axa
+
+// azure : 2014-12-18 Microsoft Corporation
+azure
+
+// baby : 2015-04-09 XYZ.COM LLC
+baby
+
+// baidu : 2015-01-08 Baidu, Inc.
+baidu
+
+// banamex : 2015-07-30 Citigroup Inc.
+banamex
+
+// bananarepublic : 2015-07-31 The Gap, Inc.
+bananarepublic
+
+// band : 2014-06-12 Dog Beach, LLC
+band
+
+// bank : 2014-09-25 fTLD Registry Services LLC
+bank
+
+// bar : 2013-12-12 Punto 2012 Sociedad Anonima Promotora de Inversion de Capital Variable
+bar
+
+// barcelona : 2014-07-24 Municipi de Barcelona
+barcelona
+
+// barclaycard : 2014-11-20 Barclays Bank PLC
+barclaycard
+
+// barclays : 2014-11-20 Barclays Bank PLC
+barclays
+
+// barefoot : 2015-06-11 Gallo Vineyards, Inc.
+barefoot
+
+// bargains : 2013-11-14 Binky Moon, LLC
+bargains
+
+// baseball : 2015-10-29 MLB Advanced Media DH, LLC
+baseball
+
+// basketball : 2015-08-20 Fédération Internationale de Basketball (FIBA)
+basketball
+
+// bauhaus : 2014-04-17 Werkhaus GmbH
+bauhaus
+
+// bayern : 2014-01-23 Bayern Connect GmbH
+bayern
+
+// bbc : 2014-12-18 British Broadcasting Corporation
+bbc
+
+// bbt : 2015-07-23 BB&T Corporation
+bbt
+
+// bbva : 2014-10-02 BANCO BILBAO VIZCAYA ARGENTARIA, S.A.
+bbva
+
+// bcg : 2015-04-02 The Boston Consulting Group, Inc.
+bcg
+
+// bcn : 2014-07-24 Municipi de Barcelona
+bcn
+
+// beats : 2015-05-14 Beats Electronics, LLC
+beats
+
+// beauty : 2015-12-03 XYZ.COM LLC
+beauty
+
+// beer : 2014-01-09 Registry Services, LLC
+beer
+
+// bentley : 2014-12-18 Bentley Motors Limited
+bentley
+
+// berlin : 2013-10-31 dotBERLIN GmbH & Co. KG
+berlin
+
+// best : 2013-12-19 BestTLD Pty Ltd
+best
+
+// bestbuy : 2015-07-31 BBY Solutions, Inc.
+bestbuy
+
+// bet : 2015-05-07 Afilias Limited
+bet
+
+// bharti : 2014-01-09 Bharti Enterprises (Holding) Private Limited
+bharti
+
+// bible : 2014-06-19 American Bible Society
+bible
+
+// bid : 2013-12-19 dot Bid Limited
+bid
+
+// bike : 2013-08-27 Binky Moon, LLC
+bike
+
+// bing : 2014-12-18 Microsoft Corporation
+bing
+
+// bingo : 2014-12-04 Binky Moon, LLC
+bingo
+
+// bio : 2014-03-06 Afilias Limited
+bio
+
+// black : 2014-01-16 Afilias Limited
+black
+
+// blackfriday : 2014-01-16 UNR Corp.
+blackfriday
+
+// blockbuster : 2015-07-30 Dish DBS Corporation
+blockbuster
+
+// blog : 2015-05-14 Knock Knock WHOIS There, LLC
+blog
+
+// bloomberg : 2014-07-17 Bloomberg IP Holdings LLC
+bloomberg
+
+// blue : 2013-11-07 Afilias Limited
+blue
+
+// bms : 2014-10-30 Bristol-Myers Squibb Company
+bms
+
+// bmw : 2014-01-09 Bayerische Motoren Werke Aktiengesellschaft
+bmw
+
+// bnpparibas : 2014-05-29 BNP Paribas
+bnpparibas
+
+// boats : 2014-12-04 XYZ.COM LLC
+boats
+
+// boehringer : 2015-07-09 Boehringer Ingelheim International GmbH
+boehringer
+
+// bofa : 2015-07-31 Bank of America Corporation
+bofa
+
+// bom : 2014-10-16 Núcleo de Informação e Coordenação do Ponto BR - NIC.br
+bom
+
+// bond : 2014-06-05 ShortDot SA
+bond
+
+// boo : 2014-01-30 Charleston Road Registry Inc.
+boo
+
+// book : 2015-08-27 Amazon Registry Services, Inc.
+book
+
+// booking : 2015-07-16 Booking.com B.V.
+booking
+
+// bosch : 2015-06-18 Robert Bosch GMBH
+bosch
+
+// bostik : 2015-05-28 Bostik SA
+bostik
+
+// boston : 2015-12-10 Boston TLD Management, LLC
+boston
+
+// bot : 2014-12-18 Amazon Registry Services, Inc.
+bot
+
+// boutique : 2013-11-14 Binky Moon, LLC
+boutique
+
+// box : 2015-11-12 Intercap Registry Inc.
+box
+
+// bradesco : 2014-12-18 Banco Bradesco S.A.
+bradesco
+
+// bridgestone : 2014-12-18 Bridgestone Corporation
+bridgestone
+
+// broadway : 2014-12-22 Celebrate Broadway, Inc.
+broadway
+
+// broker : 2014-12-11 Dog Beach, LLC
+broker
+
+// brother : 2015-01-29 Brother Industries, Ltd.
+brother
+
+// brussels : 2014-02-06 DNS.be vzw
+brussels
+
+// budapest : 2013-11-21 Minds + Machines Group Limited
+budapest
+
+// bugatti : 2015-07-23 Bugatti International SA
+bugatti
+
+// build : 2013-11-07 Plan Bee LLC
+build
+
+// builders : 2013-11-07 Binky Moon, LLC
+builders
+
+// business : 2013-11-07 Binky Moon, LLC
+business
+
+// buy : 2014-12-18 Amazon Registry Services, Inc.
+buy
+
+// buzz : 2013-10-02 DOTSTRATEGY CO.
+buzz
+
+// bzh : 2014-02-27 Association www.bzh
+bzh
+
+// cab : 2013-10-24 Binky Moon, LLC
+cab
+
+// cafe : 2015-02-11 Binky Moon, LLC
+cafe
+
+// cal : 2014-07-24 Charleston Road Registry Inc.
+cal
+
+// call : 2014-12-18 Amazon Registry Services, Inc.
+call
+
+// calvinklein : 2015-07-30 PVH gTLD Holdings LLC
+calvinklein
+
+// cam : 2016-04-21 AC Webconnecting Holding B.V.
+cam
+
+// camera : 2013-08-27 Binky Moon, LLC
+camera
+
+// camp : 2013-11-07 Binky Moon, LLC
+camp
+
+// cancerresearch : 2014-05-15 Australian Cancer Research Foundation
+cancerresearch
+
+// canon : 2014-09-12 Canon Inc.
+canon
+
+// capetown : 2014-03-24 ZA Central Registry NPC trading as ZA Central Registry
+capetown
+
+// capital : 2014-03-06 Binky Moon, LLC
+capital
+
+// capitalone : 2015-08-06 Capital One Financial Corporation
+capitalone
+
+// car : 2015-01-22 XYZ.COM LLC
+car
+
+// caravan : 2013-12-12 Caravan International, Inc.
+caravan
+
+// cards : 2013-12-05 Binky Moon, LLC
+cards
+
+// care : 2014-03-06 Binky Moon, LLC
+care
+
+// career : 2013-10-09 dotCareer LLC
+career
+
+// careers : 2013-10-02 Binky Moon, LLC
+careers
+
+// cars : 2014-11-13 XYZ.COM LLC
+cars
+
+// casa : 2013-11-21 Registry Services, LLC
+casa
+
+// case : 2015-09-03 CNH Industrial N.V.
+case
+
+// cash : 2014-03-06 Binky Moon, LLC
+cash
+
+// casino : 2014-12-18 Binky Moon, LLC
+casino
+
+// catering : 2013-12-05 Binky Moon, LLC
+catering
+
+// catholic : 2015-10-21 Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)
+catholic
+
+// cba : 2014-06-26 COMMONWEALTH BANK OF AUSTRALIA
+cba
+
+// cbn : 2014-08-22 The Christian Broadcasting Network, Inc.
+cbn
+
+// cbre : 2015-07-02 CBRE, Inc.
+cbre
+
+// cbs : 2015-08-06 CBS Domains Inc.
+cbs
+
+// center : 2013-11-07 Binky Moon, LLC
+center
+
+// ceo : 2013-11-07 CEOTLD Pty Ltd
+ceo
+
+// cern : 2014-06-05 European Organization for Nuclear Research ("CERN")
+cern
+
+// cfa : 2014-08-28 CFA Institute
+cfa
+
+// cfd : 2014-12-11 ShortDot SA
+cfd
+
+// chanel : 2015-04-09 Chanel International B.V.
+chanel
+
+// channel : 2014-05-08 Charleston Road Registry Inc.
+channel
+
+// charity : 2018-04-11 Binky Moon, LLC
+charity
+
+// chase : 2015-04-30 JPMorgan Chase Bank, National Association
+chase
+
+// chat : 2014-12-04 Binky Moon, LLC
+chat
+
+// cheap : 2013-11-14 Binky Moon, LLC
+cheap
+
+// chintai : 2015-06-11 CHINTAI Corporation
+chintai
+
+// christmas : 2013-11-21 UNR Corp.
+christmas
+
+// chrome : 2014-07-24 Charleston Road Registry Inc.
+chrome
+
+// church : 2014-02-06 Binky Moon, LLC
+church
+
+// cipriani : 2015-02-19 Hotel Cipriani Srl
+cipriani
+
+// circle : 2014-12-18 Amazon Registry Services, Inc.
+circle
+
+// cisco : 2014-12-22 Cisco Technology, Inc.
+cisco
+
+// citadel : 2015-07-23 Citadel Domain LLC
+citadel
+
+// citi : 2015-07-30 Citigroup Inc.
+citi
+
+// citic : 2014-01-09 CITIC Group Corporation
+citic
+
+// city : 2014-05-29 Binky Moon, LLC
+city
+
+// cityeats : 2014-12-11 Lifestyle Domain Holdings, Inc.
+cityeats
+
+// claims : 2014-03-20 Binky Moon, LLC
+claims
+
+// cleaning : 2013-12-05 Binky Moon, LLC
+cleaning
+
+// click : 2014-06-05 UNR Corp.
+click
+
+// clinic : 2014-03-20 Binky Moon, LLC
+clinic
+
+// clinique : 2015-10-01 The Estée Lauder Companies Inc.
+clinique
+
+// clothing : 2013-08-27 Binky Moon, LLC
+clothing
+
+// cloud : 2015-04-16 Aruba PEC S.p.A.
+cloud
+
+// club : 2013-11-08 Registry Services, LLC
+club
+
+// clubmed : 2015-06-25 Club Méditerranée S.A.
+clubmed
+
+// coach : 2014-10-09 Binky Moon, LLC
+coach
+
+// codes : 2013-10-31 Binky Moon, LLC
+codes
+
+// coffee : 2013-10-17 Binky Moon, LLC
+coffee
+
+// college : 2014-01-16 XYZ.COM LLC
+college
+
+// cologne : 2014-02-05 dotKoeln GmbH
+cologne
+
+// comcast : 2015-07-23 Comcast IP Holdings I, LLC
+comcast
+
+// commbank : 2014-06-26 COMMONWEALTH BANK OF AUSTRALIA
+commbank
+
+// community : 2013-12-05 Binky Moon, LLC
+community
+
+// company : 2013-11-07 Binky Moon, LLC
+company
+
+// compare : 2015-10-08 Registry Services, LLC
+compare
+
+// computer : 2013-10-24 Binky Moon, LLC
+computer
+
+// comsec : 2015-01-08 VeriSign, Inc.
+comsec
+
+// condos : 2013-12-05 Binky Moon, LLC
+condos
+
+// construction : 2013-09-16 Binky Moon, LLC
+construction
+
+// consulting : 2013-12-05 Dog Beach, LLC
+consulting
+
+// contact : 2015-01-08 Dog Beach, LLC
+contact
+
+// contractors : 2013-09-10 Binky Moon, LLC
+contractors
+
+// cooking : 2013-11-21 Registry Services, LLC
+cooking
+
+// cookingchannel : 2015-07-02 Lifestyle Domain Holdings, Inc.
+cookingchannel
+
+// cool : 2013-11-14 Binky Moon, LLC
+cool
+
+// corsica : 2014-09-25 Collectivité de Corse
+corsica
+
+// country : 2013-12-19 DotCountry LLC
+country
+
+// coupon : 2015-02-26 Amazon Registry Services, Inc.
+coupon
+
+// coupons : 2015-03-26 Binky Moon, LLC
+coupons
+
+// courses : 2014-12-04 OPEN UNIVERSITIES AUSTRALIA PTY LTD
+courses
+
+// cpa : 2019-06-10 American Institute of Certified Public Accountants
+cpa
+
+// credit : 2014-03-20 Binky Moon, LLC
+credit
+
+// creditcard : 2014-03-20 Binky Moon, LLC
+creditcard
+
+// creditunion : 2015-01-22 DotCooperation LLC
+creditunion
+
+// cricket : 2014-10-09 dot Cricket Limited
+cricket
+
+// crown : 2014-10-24 Crown Equipment Corporation
+crown
+
+// crs : 2014-04-03 Federated Co-operatives Limited
+crs
+
+// cruise : 2015-12-10 Viking River Cruises (Bermuda) Ltd.
+cruise
+
+// cruises : 2013-12-05 Binky Moon, LLC
+cruises
+
+// csc : 2014-09-25 Alliance-One Services, Inc.
+csc
+
+// cuisinella : 2014-04-03 SCHMIDT GROUPE S.A.S.
+cuisinella
+
+// cymru : 2014-05-08 Nominet UK
+cymru
+
+// cyou : 2015-01-22 ShortDot SA
+cyou
+
+// dabur : 2014-02-06 Dabur India Limited
+dabur
+
+// dad : 2014-01-23 Charleston Road Registry Inc.
+dad
+
+// dance : 2013-10-24 Dog Beach, LLC
+dance
+
+// data : 2016-06-02 Dish DBS Corporation
+data
+
+// date : 2014-11-20 dot Date Limited
+date
+
+// dating : 2013-12-05 Binky Moon, LLC
+dating
+
+// datsun : 2014-03-27 NISSAN MOTOR CO., LTD.
+datsun
+
+// day : 2014-01-30 Charleston Road Registry Inc.
+day
+
+// dclk : 2014-11-20 Charleston Road Registry Inc.
+dclk
+
+// dds : 2015-05-07 Registry Services, LLC
+dds
+
+// deal : 2015-06-25 Amazon Registry Services, Inc.
+deal
+
+// dealer : 2014-12-22 Intercap Registry Inc.
+dealer
+
+// deals : 2014-05-22 Binky Moon, LLC
+deals
+
+// degree : 2014-03-06 Dog Beach, LLC
+degree
+
+// delivery : 2014-09-11 Binky Moon, LLC
+delivery
+
+// dell : 2014-10-24 Dell Inc.
+dell
+
+// deloitte : 2015-07-31 Deloitte Touche Tohmatsu
+deloitte
+
+// delta : 2015-02-19 Delta Air Lines, Inc.
+delta
+
+// democrat : 2013-10-24 Dog Beach, LLC
+democrat
+
+// dental : 2014-03-20 Binky Moon, LLC
+dental
+
+// dentist : 2014-03-20 Dog Beach, LLC
+dentist
+
+// desi : 2013-11-14 Desi Networks LLC
+desi
+
+// design : 2014-11-07 Registry Services, LLC
+design
+
+// dev : 2014-10-16 Charleston Road Registry Inc.
+dev
+
+// dhl : 2015-07-23 Deutsche Post AG
+dhl
+
+// diamonds : 2013-09-22 Binky Moon, LLC
+diamonds
+
+// diet : 2014-06-26 UNR Corp.
+diet
+
+// digital : 2014-03-06 Binky Moon, LLC
+digital
+
+// direct : 2014-04-10 Binky Moon, LLC
+direct
+
+// directory : 2013-09-20 Binky Moon, LLC
+directory
+
+// discount : 2014-03-06 Binky Moon, LLC
+discount
+
+// discover : 2015-07-23 Discover Financial Services
+discover
+
+// dish : 2015-07-30 Dish DBS Corporation
+dish
+
+// diy : 2015-11-05 Lifestyle Domain Holdings, Inc.
+diy
+
+// dnp : 2013-12-13 Dai Nippon Printing Co., Ltd.
+dnp
+
+// docs : 2014-10-16 Charleston Road Registry Inc.
+docs
+
+// doctor : 2016-06-02 Binky Moon, LLC
+doctor
+
+// dog : 2014-12-04 Binky Moon, LLC
+dog
+
+// domains : 2013-10-17 Binky Moon, LLC
+domains
+
+// dot : 2015-05-21 Dish DBS Corporation
+dot
+
+// download : 2014-11-20 dot Support Limited
+download
+
+// drive : 2015-03-05 Charleston Road Registry Inc.
+drive
+
+// dtv : 2015-06-04 Dish DBS Corporation
+dtv
+
+// dubai : 2015-01-01 Dubai Smart Government Department
+dubai
+
+// duck : 2015-07-23 Johnson Shareholdings, Inc.
+duck
+
+// dunlop : 2015-07-02 The Goodyear Tire & Rubber Company
+dunlop
+
+// dupont : 2015-06-25 E. I. du Pont de Nemours and Company
+dupont
+
+// durban : 2014-03-24 ZA Central Registry NPC trading as ZA Central Registry
+durban
+
+// dvag : 2014-06-23 Deutsche Vermögensberatung Aktiengesellschaft DVAG
+dvag
+
+// dvr : 2016-05-26 DISH Technologies L.L.C.
+dvr
+
+// earth : 2014-12-04 Interlink Co., Ltd.
+earth
+
+// eat : 2014-01-23 Charleston Road Registry Inc.
+eat
+
+// eco : 2016-07-08 Big Room Inc.
+eco
+
+// edeka : 2014-12-18 EDEKA Verband kaufmännischer Genossenschaften e.V.
+edeka
+
+// education : 2013-11-07 Binky Moon, LLC
+education
+
+// email : 2013-10-31 Binky Moon, LLC
+email
+
+// emerck : 2014-04-03 Merck KGaA
+emerck
+
+// energy : 2014-09-11 Binky Moon, LLC
+energy
+
+// engineer : 2014-03-06 Dog Beach, LLC
+engineer
+
+// engineering : 2014-03-06 Binky Moon, LLC
+engineering
+
+// enterprises : 2013-09-20 Binky Moon, LLC
+enterprises
+
+// epson : 2014-12-04 Seiko Epson Corporation
+epson
+
+// equipment : 2013-08-27 Binky Moon, LLC
+equipment
+
+// ericsson : 2015-07-09 Telefonaktiebolaget L M Ericsson
+ericsson
+
+// erni : 2014-04-03 ERNI Group Holding AG
+erni
+
+// esq : 2014-05-08 Charleston Road Registry Inc.
+esq
+
+// estate : 2013-08-27 Binky Moon, LLC
+estate
+
+// etisalat : 2015-09-03 Emirates Telecommunications Corporation (trading as Etisalat)
+etisalat
+
+// eurovision : 2014-04-24 European Broadcasting Union (EBU)
+eurovision
+
+// eus : 2013-12-12 Puntueus Fundazioa
+eus
+
+// events : 2013-12-05 Binky Moon, LLC
+events
+
+// exchange : 2014-03-06 Binky Moon, LLC
+exchange
+
+// expert : 2013-11-21 Binky Moon, LLC
+expert
+
+// exposed : 2013-12-05 Binky Moon, LLC
+exposed
+
+// express : 2015-02-11 Binky Moon, LLC
+express
+
+// extraspace : 2015-05-14 Extra Space Storage LLC
+extraspace
+
+// fage : 2014-12-18 Fage International S.A.
+fage
+
+// fail : 2014-03-06 Binky Moon, LLC
+fail
+
+// fairwinds : 2014-11-13 FairWinds Partners, LLC
+fairwinds
+
+// faith : 2014-11-20 dot Faith Limited
+faith
+
+// family : 2015-04-02 Dog Beach, LLC
+family
+
+// fan : 2014-03-06 Dog Beach, LLC
+fan
+
+// fans : 2014-11-07 ZDNS International Limited
+fans
+
+// farm : 2013-11-07 Binky Moon, LLC
+farm
+
+// farmers : 2015-07-09 Farmers Insurance Exchange
+farmers
+
+// fashion : 2014-07-03 Registry Services, LLC
+fashion
+
+// fast : 2014-12-18 Amazon Registry Services, Inc.
+fast
+
+// fedex : 2015-08-06 Federal Express Corporation
+fedex
+
+// feedback : 2013-12-19 Top Level Spectrum, Inc.
+feedback
+
+// ferrari : 2015-07-31 Fiat Chrysler Automobiles N.V.
+ferrari
+
+// ferrero : 2014-12-18 Ferrero Trading Lux S.A.
+ferrero
+
+// fiat : 2015-07-31 Fiat Chrysler Automobiles N.V.
+fiat
+
+// fidelity : 2015-07-30 Fidelity Brokerage Services LLC
+fidelity
+
+// fido : 2015-08-06 Rogers Communications Canada Inc.
+fido
+
+// film : 2015-01-08 Motion Picture Domain Registry Pty Ltd
+film
+
+// final : 2014-10-16 Núcleo de Informação e Coordenação do Ponto BR - NIC.br
+final
+
+// finance : 2014-03-20 Binky Moon, LLC
+finance
+
+// financial : 2014-03-06 Binky Moon, LLC
+financial
+
+// fire : 2015-06-25 Amazon Registry Services, Inc.
+fire
+
+// firestone : 2014-12-18 Bridgestone Licensing Services, Inc
+firestone
+
+// firmdale : 2014-03-27 Firmdale Holdings Limited
+firmdale
+
+// fish : 2013-12-12 Binky Moon, LLC
+fish
+
+// fishing : 2013-11-21 Registry Services, LLC
+fishing
+
+// fit : 2014-11-07 Registry Services, LLC
+fit
+
+// fitness : 2014-03-06 Binky Moon, LLC
+fitness
+
+// flickr : 2015-04-02 Flickr, Inc.
+flickr
+
+// flights : 2013-12-05 Binky Moon, LLC
+flights
+
+// flir : 2015-07-23 FLIR Systems, Inc.
+flir
+
+// florist : 2013-11-07 Binky Moon, LLC
+florist
+
+// flowers : 2014-10-09 UNR Corp.
+flowers
+
+// fly : 2014-05-08 Charleston Road Registry Inc.
+fly
+
+// foo : 2014-01-23 Charleston Road Registry Inc.
+foo
+
+// food : 2016-04-21 Lifestyle Domain Holdings, Inc.
+food
+
+// foodnetwork : 2015-07-02 Lifestyle Domain Holdings, Inc.
+foodnetwork
+
+// football : 2014-12-18 Binky Moon, LLC
+football
+
+// ford : 2014-11-13 Ford Motor Company
+ford
+
+// forex : 2014-12-11 Dog Beach, LLC
+forex
+
+// forsale : 2014-05-22 Dog Beach, LLC
+forsale
+
+// forum : 2015-04-02 Fegistry, LLC
+forum
+
+// foundation : 2013-12-05 Binky Moon, LLC
+foundation
+
+// fox : 2015-09-11 FOX Registry, LLC
+fox
+
+// free : 2015-12-10 Amazon Registry Services, Inc.
+free
+
+// fresenius : 2015-07-30 Fresenius Immobilien-Verwaltungs-GmbH
+fresenius
+
+// frl : 2014-05-15 FRLregistry B.V.
+frl
+
+// frogans : 2013-12-19 OP3FT
+frogans
+
+// frontdoor : 2015-07-02 Lifestyle Domain Holdings, Inc.
+frontdoor
+
+// frontier : 2015-02-05 Frontier Communications Corporation
+frontier
+
+// ftr : 2015-07-16 Frontier Communications Corporation
+ftr
+
+// fujitsu : 2015-07-30 Fujitsu Limited
+fujitsu
+
+// fun : 2016-01-14 Radix FZC
+fun
+
+// fund : 2014-03-20 Binky Moon, LLC
+fund
+
+// furniture : 2014-03-20 Binky Moon, LLC
+furniture
+
+// futbol : 2013-09-20 Dog Beach, LLC
+futbol
+
+// fyi : 2015-04-02 Binky Moon, LLC
+fyi
+
+// gal : 2013-11-07 Asociación puntoGAL
+gal
+
+// gallery : 2013-09-13 Binky Moon, LLC
+gallery
+
+// gallo : 2015-06-11 Gallo Vineyards, Inc.
+gallo
+
+// gallup : 2015-02-19 Gallup, Inc.
+gallup
+
+// game : 2015-05-28 UNR Corp.
+game
+
+// games : 2015-05-28 Dog Beach, LLC
+games
+
+// gap : 2015-07-31 The Gap, Inc.
+gap
+
+// garden : 2014-06-26 Registry Services, LLC
+garden
+
+// gay : 2019-05-23 Top Level Design, LLC
+gay
+
+// gbiz : 2014-07-17 Charleston Road Registry Inc.
+gbiz
+
+// gdn : 2014-07-31 Joint Stock Company "Navigation-information systems"
+gdn
+
+// gea : 2014-12-04 GEA Group Aktiengesellschaft
+gea
+
+// gent : 2014-01-23 COMBELL NV
+gent
+
+// genting : 2015-03-12 Resorts World Inc Pte. Ltd.
+genting
+
+// george : 2015-07-31 Wal-Mart Stores, Inc.
+george
+
+// ggee : 2014-01-09 GMO Internet, Inc.
+ggee
+
+// gift : 2013-10-17 DotGift, LLC
+gift
+
+// gifts : 2014-07-03 Binky Moon, LLC
+gifts
+
+// gives : 2014-03-06 Dog Beach, LLC
+gives
+
+// giving : 2014-11-13 Giving Limited
+giving
+
+// glade : 2015-07-23 Johnson Shareholdings, Inc.
+glade
+
+// glass : 2013-11-07 Binky Moon, LLC
+glass
+
+// gle : 2014-07-24 Charleston Road Registry Inc.
+gle
+
+// global : 2014-04-17 Dot Global Domain Registry Limited
+global
+
+// globo : 2013-12-19 Globo Comunicação e Participações S.A
+globo
+
+// gmail : 2014-05-01 Charleston Road Registry Inc.
+gmail
+
+// gmbh : 2016-01-29 Binky Moon, LLC
+gmbh
+
+// gmo : 2014-01-09 GMO Internet, Inc.
+gmo
+
+// gmx : 2014-04-24 1&1 Mail & Media GmbH
+gmx
+
+// godaddy : 2015-07-23 Go Daddy East, LLC
+godaddy
+
+// gold : 2015-01-22 Binky Moon, LLC
+gold
+
+// goldpoint : 2014-11-20 YODOBASHI CAMERA CO.,LTD.
+goldpoint
+
+// golf : 2014-12-18 Binky Moon, LLC
+golf
+
+// goo : 2014-12-18 NTT Resonant Inc.
+goo
+
+// goodyear : 2015-07-02 The Goodyear Tire & Rubber Company
+goodyear
+
+// goog : 2014-11-20 Charleston Road Registry Inc.
+goog
+
+// google : 2014-07-24 Charleston Road Registry Inc.
+google
+
+// gop : 2014-01-16 Republican State Leadership Committee, Inc.
+gop
+
+// got : 2014-12-18 Amazon Registry Services, Inc.
+got
+
+// grainger : 2015-05-07 Grainger Registry Services, LLC
+grainger
+
+// graphics : 2013-09-13 Binky Moon, LLC
+graphics
+
+// gratis : 2014-03-20 Binky Moon, LLC
+gratis
+
+// green : 2014-05-08 Afilias Limited
+green
+
+// gripe : 2014-03-06 Binky Moon, LLC
+gripe
+
+// grocery : 2016-06-16 Wal-Mart Stores, Inc.
+grocery
+
+// group : 2014-08-15 Binky Moon, LLC
+group
+
+// guardian : 2015-07-30 The Guardian Life Insurance Company of America
+guardian
+
+// gucci : 2014-11-13 Guccio Gucci S.p.a.
+gucci
+
+// guge : 2014-08-28 Charleston Road Registry Inc.
+guge
+
+// guide : 2013-09-13 Binky Moon, LLC
+guide
+
+// guitars : 2013-11-14 UNR Corp.
+guitars
+
+// guru : 2013-08-27 Binky Moon, LLC
+guru
+
+// hair : 2015-12-03 XYZ.COM LLC
+hair
+
+// hamburg : 2014-02-20 Hamburg Top-Level-Domain GmbH
+hamburg
+
+// hangout : 2014-11-13 Charleston Road Registry Inc.
+hangout
+
+// haus : 2013-12-05 Dog Beach, LLC
+haus
+
+// hbo : 2015-07-30 HBO Registry Services, Inc.
+hbo
+
+// hdfc : 2015-07-30 HOUSING DEVELOPMENT FINANCE CORPORATION LIMITED
+hdfc
+
+// hdfcbank : 2015-02-12 HDFC Bank Limited
+hdfcbank
+
+// health : 2015-02-11 DotHealth, LLC
+health
+
+// healthcare : 2014-06-12 Binky Moon, LLC
+healthcare
+
+// help : 2014-06-26 UNR Corp.
+help
+
+// helsinki : 2015-02-05 City of Helsinki
+helsinki
+
+// here : 2014-02-06 Charleston Road Registry Inc.
+here
+
+// hermes : 2014-07-10 HERMES INTERNATIONAL
+hermes
+
+// hgtv : 2015-07-02 Lifestyle Domain Holdings, Inc.
+hgtv
+
+// hiphop : 2014-03-06 UNR Corp.
+hiphop
+
+// hisamitsu : 2015-07-16 Hisamitsu Pharmaceutical Co.,Inc.
+hisamitsu
+
+// hitachi : 2014-10-31 Hitachi, Ltd.
+hitachi
+
+// hiv : 2014-03-13 UNR Corp.
+hiv
+
+// hkt : 2015-05-14 PCCW-HKT DataCom Services Limited
+hkt
+
+// hockey : 2015-03-19 Binky Moon, LLC
+hockey
+
+// holdings : 2013-08-27 Binky Moon, LLC
+holdings
+
+// holiday : 2013-11-07 Binky Moon, LLC
+holiday
+
+// homedepot : 2015-04-02 Home Depot Product Authority, LLC
+homedepot
+
+// homegoods : 2015-07-16 The TJX Companies, Inc.
+homegoods
+
+// homes : 2014-01-09 XYZ.COM LLC
+homes
+
+// homesense : 2015-07-16 The TJX Companies, Inc.
+homesense
+
+// honda : 2014-12-18 Honda Motor Co., Ltd.
+honda
+
+// horse : 2013-11-21 Registry Services, LLC
+horse
+
+// hospital : 2016-10-20 Binky Moon, LLC
+hospital
+
+// host : 2014-04-17 Radix FZC
+host
+
+// hosting : 2014-05-29 UNR Corp.
+hosting
+
+// hot : 2015-08-27 Amazon Registry Services, Inc.
+hot
+
+// hoteles : 2015-03-05 Travel Reservations SRL
+hoteles
+
+// hotels : 2016-04-07 Booking.com B.V.
+hotels
+
+// hotmail : 2014-12-18 Microsoft Corporation
+hotmail
+
+// house : 2013-11-07 Binky Moon, LLC
+house
+
+// how : 2014-01-23 Charleston Road Registry Inc.
+how
+
+// hsbc : 2014-10-24 HSBC Global Services (UK) Limited
+hsbc
+
+// hughes : 2015-07-30 Hughes Satellite Systems Corporation
+hughes
+
+// hyatt : 2015-07-30 Hyatt GTLD, L.L.C.
+hyatt
+
+// hyundai : 2015-07-09 Hyundai Motor Company
+hyundai
+
+// ibm : 2014-07-31 International Business Machines Corporation
+ibm
+
+// icbc : 2015-02-19 Industrial and Commercial Bank of China Limited
+icbc
+
+// ice : 2014-10-30 IntercontinentalExchange, Inc.
+ice
+
+// icu : 2015-01-08 ShortDot SA
+icu
+
+// ieee : 2015-07-23 IEEE Global LLC
+ieee
+
+// ifm : 2014-01-30 ifm electronic gmbh
+ifm
+
+// ikano : 2015-07-09 Ikano S.A.
+ikano
+
+// imamat : 2015-08-06 Fondation Aga Khan (Aga Khan Foundation)
+imamat
+
+// imdb : 2015-06-25 Amazon Registry Services, Inc.
+imdb
+
+// immo : 2014-07-10 Binky Moon, LLC
+immo
+
+// immobilien : 2013-11-07 Dog Beach, LLC
+immobilien
+
+// inc : 2018-03-10 Intercap Registry Inc.
+inc
+
+// industries : 2013-12-05 Binky Moon, LLC
+industries
+
+// infiniti : 2014-03-27 NISSAN MOTOR CO., LTD.
+infiniti
+
+// ing : 2014-01-23 Charleston Road Registry Inc.
+ing
+
+// ink : 2013-12-05 Top Level Design, LLC
+ink
+
+// institute : 2013-11-07 Binky Moon, LLC
+institute
+
+// insurance : 2015-02-19 fTLD Registry Services LLC
+insurance
+
+// insure : 2014-03-20 Binky Moon, LLC
+insure
+
+// international : 2013-11-07 Binky Moon, LLC
+international
+
+// intuit : 2015-07-30 Intuit Administrative Services, Inc.
+intuit
+
+// investments : 2014-03-20 Binky Moon, LLC
+investments
+
+// ipiranga : 2014-08-28 Ipiranga Produtos de Petroleo S.A.
+ipiranga
+
+// irish : 2014-08-07 Binky Moon, LLC
+irish
+
+// ismaili : 2015-08-06 Fondation Aga Khan (Aga Khan Foundation)
+ismaili
+
+// ist : 2014-08-28 Istanbul Metropolitan Municipality
+ist
+
+// istanbul : 2014-08-28 Istanbul Metropolitan Municipality
+istanbul
+
+// itau : 2014-10-02 Itau Unibanco Holding S.A.
+itau
+
+// itv : 2015-07-09 ITV Services Limited
+itv
+
+// jaguar : 2014-11-13 Jaguar Land Rover Ltd
+jaguar
+
+// java : 2014-06-19 Oracle Corporation
+java
+
+// jcb : 2014-11-20 JCB Co., Ltd.
+jcb
+
+// jeep : 2015-07-30 FCA US LLC.
+jeep
+
+// jetzt : 2014-01-09 Binky Moon, LLC
+jetzt
+
+// jewelry : 2015-03-05 Binky Moon, LLC
+jewelry
+
+// jio : 2015-04-02 Reliance Industries Limited
+jio
+
+// jll : 2015-04-02 Jones Lang LaSalle Incorporated
+jll
+
+// jmp : 2015-03-26 Matrix IP LLC
+jmp
+
+// jnj : 2015-06-18 Johnson & Johnson Services, Inc.
+jnj
+
+// joburg : 2014-03-24 ZA Central Registry NPC trading as ZA Central Registry
+joburg
+
+// jot : 2014-12-18 Amazon Registry Services, Inc.
+jot
+
+// joy : 2014-12-18 Amazon Registry Services, Inc.
+joy
+
+// jpmorgan : 2015-04-30 JPMorgan Chase Bank, National Association
+jpmorgan
+
+// jprs : 2014-09-18 Japan Registry Services Co., Ltd.
+jprs
+
+// juegos : 2014-03-20 UNR Corp.
+juegos
+
+// juniper : 2015-07-30 JUNIPER NETWORKS, INC.
+juniper
+
+// kaufen : 2013-11-07 Dog Beach, LLC
+kaufen
+
+// kddi : 2014-09-12 KDDI CORPORATION
+kddi
+
+// kerryhotels : 2015-04-30 Kerry Trading Co. Limited
+kerryhotels
+
+// kerrylogistics : 2015-04-09 Kerry Trading Co. Limited
+kerrylogistics
+
+// kerryproperties : 2015-04-09 Kerry Trading Co. Limited
+kerryproperties
+
+// kfh : 2014-12-04 Kuwait Finance House
+kfh
+
+// kia : 2015-07-09 KIA MOTORS CORPORATION
+kia
+
+// kim : 2013-09-23 Afilias Limited
+kim
+
+// kinder : 2014-11-07 Ferrero Trading Lux S.A.
+kinder
+
+// kindle : 2015-06-25 Amazon Registry Services, Inc.
+kindle
+
+// kitchen : 2013-09-20 Binky Moon, LLC
+kitchen
+
+// kiwi : 2013-09-20 DOT KIWI LIMITED
+kiwi
+
+// koeln : 2014-01-09 dotKoeln GmbH
+koeln
+
+// komatsu : 2015-01-08 Komatsu Ltd.
+komatsu
+
+// kosher : 2015-08-20 Kosher Marketing Assets LLC
+kosher
+
+// kpmg : 2015-04-23 KPMG International Cooperative (KPMG International Genossenschaft)
+kpmg
+
+// kpn : 2015-01-08 Koninklijke KPN N.V.
+kpn
+
+// krd : 2013-12-05 KRG Department of Information Technology
+krd
+
+// kred : 2013-12-19 KredTLD Pty Ltd
+kred
+
+// kuokgroup : 2015-04-09 Kerry Trading Co. Limited
+kuokgroup
+
+// kyoto : 2014-11-07 Academic Institution: Kyoto Jyoho Gakuen
+kyoto
+
+// lacaixa : 2014-01-09 Fundación Bancaria Caixa d’Estalvis i Pensions de Barcelona, “la Caixaâ€
+lacaixa
+
+// lamborghini : 2015-06-04 Automobili Lamborghini S.p.A.
+lamborghini
+
+// lamer : 2015-10-01 The Estée Lauder Companies Inc.
+lamer
+
+// lancaster : 2015-02-12 LANCASTER
+lancaster
+
+// lancia : 2015-07-31 Fiat Chrysler Automobiles N.V.
+lancia
+
+// land : 2013-09-10 Binky Moon, LLC
+land
+
+// landrover : 2014-11-13 Jaguar Land Rover Ltd
+landrover
+
+// lanxess : 2015-07-30 LANXESS Corporation
+lanxess
+
+// lasalle : 2015-04-02 Jones Lang LaSalle Incorporated
+lasalle
+
+// lat : 2014-10-16 ECOM-LAC Federaciòn de Latinoamèrica y el Caribe para Internet y el Comercio Electrònico
+lat
+
+// latino : 2015-07-30 Dish DBS Corporation
+latino
+
+// latrobe : 2014-06-16 La Trobe University
+latrobe
+
+// law : 2015-01-22 Registry Services, LLC
+law
+
+// lawyer : 2014-03-20 Dog Beach, LLC
+lawyer
+
+// lds : 2014-03-20 IRI Domain Management, LLC
+lds
+
+// lease : 2014-03-06 Binky Moon, LLC
+lease
+
+// leclerc : 2014-08-07 A.C.D. LEC Association des Centres Distributeurs Edouard Leclerc
+leclerc
+
+// lefrak : 2015-07-16 LeFrak Organization, Inc.
+lefrak
+
+// legal : 2014-10-16 Binky Moon, LLC
+legal
+
+// lego : 2015-07-16 LEGO Juris A/S
+lego
+
+// lexus : 2015-04-23 TOYOTA MOTOR CORPORATION
+lexus
+
+// lgbt : 2014-05-08 Afilias Limited
+lgbt
+
+// lidl : 2014-09-18 Schwarz Domains und Services GmbH & Co. KG
+lidl
+
+// life : 2014-02-06 Binky Moon, LLC
+life
+
+// lifeinsurance : 2015-01-15 American Council of Life Insurers
+lifeinsurance
+
+// lifestyle : 2014-12-11 Lifestyle Domain Holdings, Inc.
+lifestyle
+
+// lighting : 2013-08-27 Binky Moon, LLC
+lighting
+
+// like : 2014-12-18 Amazon Registry Services, Inc.
+like
+
+// lilly : 2015-07-31 Eli Lilly and Company
+lilly
+
+// limited : 2014-03-06 Binky Moon, LLC
+limited
+
+// limo : 2013-10-17 Binky Moon, LLC
+limo
+
+// lincoln : 2014-11-13 Ford Motor Company
+lincoln
+
+// linde : 2014-12-04 Linde Aktiengesellschaft
+linde
+
+// link : 2013-11-14 UNR Corp.
+link
+
+// lipsy : 2015-06-25 Lipsy Ltd
+lipsy
+
+// live : 2014-12-04 Dog Beach, LLC
+live
+
+// living : 2015-07-30 Lifestyle Domain Holdings, Inc.
+living
+
+// lixil : 2015-03-19 LIXIL Group Corporation
+lixil
+
+// llc : 2017-12-14 Afilias Limited
+llc
+
+// llp : 2019-08-26 UNR Corp.
+llp
+
+// loan : 2014-11-20 dot Loan Limited
+loan
+
+// loans : 2014-03-20 Binky Moon, LLC
+loans
+
+// locker : 2015-06-04 Dish DBS Corporation
+locker
+
+// locus : 2015-06-25 Locus Analytics LLC
+locus
+
+// loft : 2015-07-30 Annco, Inc.
+loft
+
+// lol : 2015-01-30 UNR Corp.
+lol
+
+// london : 2013-11-14 Dot London Domains Limited
+london
+
+// lotte : 2014-11-07 Lotte Holdings Co., Ltd.
+lotte
+
+// lotto : 2014-04-10 Afilias Limited
+lotto
+
+// love : 2014-12-22 Merchant Law Group LLP
+love
+
+// lpl : 2015-07-30 LPL Holdings, Inc.
+lpl
+
+// lplfinancial : 2015-07-30 LPL Holdings, Inc.
+lplfinancial
+
+// ltd : 2014-09-25 Binky Moon, LLC
+ltd
+
+// ltda : 2014-04-17 InterNetX, Corp
+ltda
+
+// lundbeck : 2015-08-06 H. Lundbeck A/S
+lundbeck
+
+// luxe : 2014-01-09 Registry Services, LLC
+luxe
+
+// luxury : 2013-10-17 Luxury Partners, LLC
+luxury
+
+// macys : 2015-07-31 Macys, Inc.
+macys
+
+// madrid : 2014-05-01 Comunidad de Madrid
+madrid
+
+// maif : 2014-10-02 Mutuelle Assurance Instituteur France (MAIF)
+maif
+
+// maison : 2013-12-05 Binky Moon, LLC
+maison
+
+// makeup : 2015-01-15 XYZ.COM LLC
+makeup
+
+// man : 2014-12-04 MAN SE
+man
+
+// management : 2013-11-07 Binky Moon, LLC
+management
+
+// mango : 2013-10-24 PUNTO FA S.L.
+mango
+
+// map : 2016-06-09 Charleston Road Registry Inc.
+map
+
+// market : 2014-03-06 Dog Beach, LLC
+market
+
+// marketing : 2013-11-07 Binky Moon, LLC
+marketing
+
+// markets : 2014-12-11 Dog Beach, LLC
+markets
+
+// marriott : 2014-10-09 Marriott Worldwide Corporation
+marriott
+
+// marshalls : 2015-07-16 The TJX Companies, Inc.
+marshalls
+
+// maserati : 2015-07-31 Fiat Chrysler Automobiles N.V.
+maserati
+
+// mattel : 2015-08-06 Mattel Sites, Inc.
+mattel
+
+// mba : 2015-04-02 Binky Moon, LLC
+mba
+
+// mckinsey : 2015-07-31 McKinsey Holdings, Inc.
+mckinsey
+
+// med : 2015-08-06 Medistry LLC
+med
+
+// media : 2014-03-06 Binky Moon, LLC
+media
+
+// meet : 2014-01-16 Charleston Road Registry Inc.
+meet
+
+// melbourne : 2014-05-29 The Crown in right of the State of Victoria, represented by its Department of State Development, Business and Innovation
+melbourne
+
+// meme : 2014-01-30 Charleston Road Registry Inc.
+meme
+
+// memorial : 2014-10-16 Dog Beach, LLC
+memorial
+
+// men : 2015-02-26 Exclusive Registry Limited
+men
+
+// menu : 2013-09-11 Dot Menu Registry, LLC
+menu
+
+// merckmsd : 2016-07-14 MSD Registry Holdings, Inc.
+merckmsd
+
+// miami : 2013-12-19 Minds + Machines Group Limited
+miami
+
+// microsoft : 2014-12-18 Microsoft Corporation
+microsoft
+
+// mini : 2014-01-09 Bayerische Motoren Werke Aktiengesellschaft
+mini
+
+// mint : 2015-07-30 Intuit Administrative Services, Inc.
+mint
+
+// mit : 2015-07-02 Massachusetts Institute of Technology
+mit
+
+// mitsubishi : 2015-07-23 Mitsubishi Corporation
+mitsubishi
+
+// mlb : 2015-05-21 MLB Advanced Media DH, LLC
+mlb
+
+// mls : 2015-04-23 The Canadian Real Estate Association
+mls
+
+// mma : 2014-11-07 MMA IARD
+mma
+
+// mobile : 2016-06-02 Dish DBS Corporation
+mobile
+
+// moda : 2013-11-07 Dog Beach, LLC
+moda
+
+// moe : 2013-11-13 Interlink Co., Ltd.
+moe
+
+// moi : 2014-12-18 Amazon Registry Services, Inc.
+moi
+
+// mom : 2015-04-16 UNR Corp.
+mom
+
+// monash : 2013-09-30 Monash University
+monash
+
+// money : 2014-10-16 Binky Moon, LLC
+money
+
+// monster : 2015-09-11 XYZ.COM LLC
+monster
+
+// mormon : 2013-12-05 IRI Domain Management, LLC
+mormon
+
+// mortgage : 2014-03-20 Dog Beach, LLC
+mortgage
+
+// moscow : 2013-12-19 Foundation for Assistance for Internet Technologies and Infrastructure Development (FAITID)
+moscow
+
+// moto : 2015-06-04 Motorola Trademark Holdings, LLC
+moto
+
+// motorcycles : 2014-01-09 XYZ.COM LLC
+motorcycles
+
+// mov : 2014-01-30 Charleston Road Registry Inc.
+mov
+
+// movie : 2015-02-05 Binky Moon, LLC
+movie
+
+// msd : 2015-07-23 MSD Registry Holdings, Inc.
+msd
+
+// mtn : 2014-12-04 MTN Dubai Limited
+mtn
+
+// mtr : 2015-03-12 MTR Corporation Limited
+mtr
+
+// music : 2021-05-04 DotMusic Limited
+music
+
+// mutual : 2015-04-02 Northwestern Mutual MU TLD Registry, LLC
+mutual
+
+// nab : 2015-08-20 National Australia Bank Limited
+nab
+
+// nagoya : 2013-10-24 GMO Registry, Inc.
+nagoya
+
+// natura : 2015-03-12 NATURA COSMÉTICOS S.A.
+natura
+
+// navy : 2014-03-06 Dog Beach, LLC
+navy
+
+// nba : 2015-07-31 NBA REGISTRY, LLC
+nba
+
+// nec : 2015-01-08 NEC Corporation
+nec
+
+// netbank : 2014-06-26 COMMONWEALTH BANK OF AUSTRALIA
+netbank
+
+// netflix : 2015-06-18 Netflix, Inc.
+netflix
+
+// network : 2013-11-14 Binky Moon, LLC
+network
+
+// neustar : 2013-12-05 NeuStar, Inc.
+neustar
+
+// new : 2014-01-30 Charleston Road Registry Inc.
+new
+
+// news : 2014-12-18 Dog Beach, LLC
+news
+
+// next : 2015-06-18 Next plc
+next
+
+// nextdirect : 2015-06-18 Next plc
+nextdirect
+
+// nexus : 2014-07-24 Charleston Road Registry Inc.
+nexus
+
+// nfl : 2015-07-23 NFL Reg Ops LLC
+nfl
+
+// ngo : 2014-03-06 Public Interest Registry
+ngo
+
+// nhk : 2014-02-13 Japan Broadcasting Corporation (NHK)
+nhk
+
+// nico : 2014-12-04 DWANGO Co., Ltd.
+nico
+
+// nike : 2015-07-23 NIKE, Inc.
+nike
+
+// nikon : 2015-05-21 NIKON CORPORATION
+nikon
+
+// ninja : 2013-11-07 Dog Beach, LLC
+ninja
+
+// nissan : 2014-03-27 NISSAN MOTOR CO., LTD.
+nissan
+
+// nissay : 2015-10-29 Nippon Life Insurance Company
+nissay
+
+// nokia : 2015-01-08 Nokia Corporation
+nokia
+
+// northwesternmutual : 2015-06-18 Northwestern Mutual Registry, LLC
+northwesternmutual
+
+// norton : 2014-12-04 NortonLifeLock Inc.
+norton
+
+// now : 2015-06-25 Amazon Registry Services, Inc.
+now
+
+// nowruz : 2014-09-04 Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti.
+nowruz
+
+// nowtv : 2015-05-14 Starbucks (HK) Limited
+nowtv
+
+// nra : 2014-05-22 NRA Holdings Company, INC.
+nra
+
+// nrw : 2013-11-21 Minds + Machines GmbH
+nrw
+
+// ntt : 2014-10-31 NIPPON TELEGRAPH AND TELEPHONE CORPORATION
+ntt
+
+// nyc : 2014-01-23 The City of New York by and through the New York City Department of Information Technology & Telecommunications
+nyc
+
+// obi : 2014-09-25 OBI Group Holding SE & Co. KGaA
+obi
+
+// observer : 2015-04-30 Dog Beach, LLC
+observer
+
+// off : 2015-07-23 Johnson Shareholdings, Inc.
+off
+
+// office : 2015-03-12 Microsoft Corporation
+office
+
+// okinawa : 2013-12-05 BRregistry, Inc.
+okinawa
+
+// olayan : 2015-05-14 Crescent Holding GmbH
+olayan
+
+// olayangroup : 2015-05-14 Crescent Holding GmbH
+olayangroup
+
+// oldnavy : 2015-07-31 The Gap, Inc.
+oldnavy
+
+// ollo : 2015-06-04 Dish DBS Corporation
+ollo
+
+// omega : 2015-01-08 The Swatch Group Ltd
+omega
+
+// one : 2014-11-07 One.com A/S
+one
+
+// ong : 2014-03-06 Public Interest Registry
+ong
+
+// onl : 2013-09-16 iRegistry GmbH
+onl
+
+// online : 2015-01-15 Radix FZC
+online
+
+// ooo : 2014-01-09 INFIBEAM AVENUES LIMITED
+ooo
+
+// open : 2015-07-31 American Express Travel Related Services Company, Inc.
+open
+
+// oracle : 2014-06-19 Oracle Corporation
+oracle
+
+// orange : 2015-03-12 Orange Brand Services Limited
+orange
+
+// organic : 2014-03-27 Afilias Limited
+organic
+
+// origins : 2015-10-01 The Estée Lauder Companies Inc.
+origins
+
+// osaka : 2014-09-04 Osaka Registry Co., Ltd.
+osaka
+
+// otsuka : 2013-10-11 Otsuka Holdings Co., Ltd.
+otsuka
+
+// ott : 2015-06-04 Dish DBS Corporation
+ott
+
+// ovh : 2014-01-16 MédiaBC
+ovh
+
+// page : 2014-12-04 Charleston Road Registry Inc.
+page
+
+// panasonic : 2015-07-30 Panasonic Corporation
+panasonic
+
+// paris : 2014-01-30 City of Paris
+paris
+
+// pars : 2014-09-04 Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti.
+pars
+
+// partners : 2013-12-05 Binky Moon, LLC
+partners
+
+// parts : 2013-12-05 Binky Moon, LLC
+parts
+
+// party : 2014-09-11 Blue Sky Registry Limited
+party
+
+// passagens : 2015-03-05 Travel Reservations SRL
+passagens
+
+// pay : 2015-08-27 Amazon Registry Services, Inc.
+pay
+
+// pccw : 2015-05-14 PCCW Enterprises Limited
+pccw
+
+// pet : 2015-05-07 Afilias Limited
+pet
+
+// pfizer : 2015-09-11 Pfizer Inc.
+pfizer
+
+// pharmacy : 2014-06-19 National Association of Boards of Pharmacy
+pharmacy
+
+// phd : 2016-07-28 Charleston Road Registry Inc.
+phd
+
+// philips : 2014-11-07 Koninklijke Philips N.V.
+philips
+
+// phone : 2016-06-02 Dish DBS Corporation
+phone
+
+// photo : 2013-11-14 UNR Corp.
+photo
+
+// photography : 2013-09-20 Binky Moon, LLC
+photography
+
+// photos : 2013-10-17 Binky Moon, LLC
+photos
+
+// physio : 2014-05-01 PhysBiz Pty Ltd
+physio
+
+// pics : 2013-11-14 UNR Corp.
+pics
+
+// pictet : 2014-06-26 Pictet Europe S.A.
+pictet
+
+// pictures : 2014-03-06 Binky Moon, LLC
+pictures
+
+// pid : 2015-01-08 Top Level Spectrum, Inc.
+pid
+
+// pin : 2014-12-18 Amazon Registry Services, Inc.
+pin
+
+// ping : 2015-06-11 Ping Registry Provider, Inc.
+ping
+
+// pink : 2013-10-01 Afilias Limited
+pink
+
+// pioneer : 2015-07-16 Pioneer Corporation
+pioneer
+
+// pizza : 2014-06-26 Binky Moon, LLC
+pizza
+
+// place : 2014-04-24 Binky Moon, LLC
+place
+
+// play : 2015-03-05 Charleston Road Registry Inc.
+play
+
+// playstation : 2015-07-02 Sony Interactive Entertainment Inc.
+playstation
+
+// plumbing : 2013-09-10 Binky Moon, LLC
+plumbing
+
+// plus : 2015-02-05 Binky Moon, LLC
+plus
+
+// pnc : 2015-07-02 PNC Domain Co., LLC
+pnc
+
+// pohl : 2014-06-23 Deutsche Vermögensberatung Aktiengesellschaft DVAG
+pohl
+
+// poker : 2014-07-03 Afilias Limited
+poker
+
+// politie : 2015-08-20 Politie Nederland
+politie
+
+// porn : 2014-10-16 ICM Registry PN LLC
+porn
+
+// pramerica : 2015-07-30 Prudential Financial, Inc.
+pramerica
+
+// praxi : 2013-12-05 Praxi S.p.A.
+praxi
+
+// press : 2014-04-03 Radix FZC
+press
+
+// prime : 2015-06-25 Amazon Registry Services, Inc.
+prime
+
+// prod : 2014-01-23 Charleston Road Registry Inc.
+prod
+
+// productions : 2013-12-05 Binky Moon, LLC
+productions
+
+// prof : 2014-07-24 Charleston Road Registry Inc.
+prof
+
+// progressive : 2015-07-23 Progressive Casualty Insurance Company
+progressive
+
+// promo : 2014-12-18 Afilias Limited
+promo
+
+// properties : 2013-12-05 Binky Moon, LLC
+properties
+
+// property : 2014-05-22 UNR Corp.
+property
+
+// protection : 2015-04-23 XYZ.COM LLC
+protection
+
+// pru : 2015-07-30 Prudential Financial, Inc.
+pru
+
+// prudential : 2015-07-30 Prudential Financial, Inc.
+prudential
+
+// pub : 2013-12-12 Dog Beach, LLC
+pub
+
+// pwc : 2015-10-29 PricewaterhouseCoopers LLP
+pwc
+
+// qpon : 2013-11-14 dotCOOL, Inc.
+qpon
+
+// quebec : 2013-12-19 PointQuébec Inc
+quebec
+
+// quest : 2015-03-26 XYZ.COM LLC
+quest
+
+// qvc : 2015-07-30 QVC, Inc.
+qvc
+
+// racing : 2014-12-04 Premier Registry Limited
+racing
+
+// radio : 2016-07-21 European Broadcasting Union (EBU)
+radio
+
+// raid : 2015-07-23 Johnson Shareholdings, Inc.
+raid
+
+// read : 2014-12-18 Amazon Registry Services, Inc.
+read
+
+// realestate : 2015-09-11 dotRealEstate LLC
+realestate
+
+// realtor : 2014-05-29 Real Estate Domains LLC
+realtor
+
+// realty : 2015-03-19 Dog Beach, LLC
+realty
+
+// recipes : 2013-10-17 Binky Moon, LLC
+recipes
+
+// red : 2013-11-07 Afilias Limited
+red
+
+// redstone : 2014-10-31 Redstone Haute Couture Co., Ltd.
+redstone
+
+// redumbrella : 2015-03-26 Travelers TLD, LLC
+redumbrella
+
+// rehab : 2014-03-06 Dog Beach, LLC
+rehab
+
+// reise : 2014-03-13 Binky Moon, LLC
+reise
+
+// reisen : 2014-03-06 Binky Moon, LLC
+reisen
+
+// reit : 2014-09-04 National Association of Real Estate Investment Trusts, Inc.
+reit
+
+// reliance : 2015-04-02 Reliance Industries Limited
+reliance
+
+// ren : 2013-12-12 ZDNS International Limited
+ren
+
+// rent : 2014-12-04 XYZ.COM LLC
+rent
+
+// rentals : 2013-12-05 Binky Moon, LLC
+rentals
+
+// repair : 2013-11-07 Binky Moon, LLC
+repair
+
+// report : 2013-12-05 Binky Moon, LLC
+report
+
+// republican : 2014-03-20 Dog Beach, LLC
+republican
+
+// rest : 2013-12-19 Punto 2012 Sociedad Anonima Promotora de Inversion de Capital Variable
+rest
+
+// restaurant : 2014-07-03 Binky Moon, LLC
+restaurant
+
+// review : 2014-11-20 dot Review Limited
+review
+
+// reviews : 2013-09-13 Dog Beach, LLC
+reviews
+
+// rexroth : 2015-06-18 Robert Bosch GMBH
+rexroth
+
+// rich : 2013-11-21 iRegistry GmbH
+rich
+
+// richardli : 2015-05-14 Pacific Century Asset Management (HK) Limited
+richardli
+
+// ricoh : 2014-11-20 Ricoh Company, Ltd.
+ricoh
+
+// ril : 2015-04-02 Reliance Industries Limited
+ril
+
+// rio : 2014-02-27 Empresa Municipal de Informática SA - IPLANRIO
+rio
+
+// rip : 2014-07-10 Dog Beach, LLC
+rip
+
+// rmit : 2015-11-19 Royal Melbourne Institute of Technology
+rmit
+
+// rocher : 2014-12-18 Ferrero Trading Lux S.A.
+rocher
+
+// rocks : 2013-11-14 Dog Beach, LLC
+rocks
+
+// rodeo : 2013-12-19 Registry Services, LLC
+rodeo
+
+// rogers : 2015-08-06 Rogers Communications Canada Inc.
+rogers
+
+// room : 2014-12-18 Amazon Registry Services, Inc.
+room
+
+// rsvp : 2014-05-08 Charleston Road Registry Inc.
+rsvp
+
+// rugby : 2016-12-15 World Rugby Strategic Developments Limited
+rugby
+
+// ruhr : 2013-10-02 regiodot GmbH & Co. KG
+ruhr
+
+// run : 2015-03-19 Binky Moon, LLC
+run
+
+// rwe : 2015-04-02 RWE AG
+rwe
+
+// ryukyu : 2014-01-09 BRregistry, Inc.
+ryukyu
+
+// saarland : 2013-12-12 dotSaarland GmbH
+saarland
+
+// safe : 2014-12-18 Amazon Registry Services, Inc.
+safe
+
+// safety : 2015-01-08 Safety Registry Services, LLC.
+safety
+
+// sakura : 2014-12-18 SAKURA Internet Inc.
+sakura
+
+// sale : 2014-10-16 Dog Beach, LLC
+sale
+
+// salon : 2014-12-11 Binky Moon, LLC
+salon
+
+// samsclub : 2015-07-31 Wal-Mart Stores, Inc.
+samsclub
+
+// samsung : 2014-04-03 SAMSUNG SDS CO., LTD
+samsung
+
+// sandvik : 2014-11-13 Sandvik AB
+sandvik
+
+// sandvikcoromant : 2014-11-07 Sandvik AB
+sandvikcoromant
+
+// sanofi : 2014-10-09 Sanofi
+sanofi
+
+// sap : 2014-03-27 SAP AG
+sap
+
+// sarl : 2014-07-03 Binky Moon, LLC
+sarl
+
+// sas : 2015-04-02 Research IP LLC
+sas
+
+// save : 2015-06-25 Amazon Registry Services, Inc.
+save
+
+// saxo : 2014-10-31 Saxo Bank A/S
+saxo
+
+// sbi : 2015-03-12 STATE BANK OF INDIA
+sbi
+
+// sbs : 2014-11-07 ShortDot SA
+sbs
+
+// sca : 2014-03-13 SVENSKA CELLULOSA AKTIEBOLAGET SCA (publ)
+sca
+
+// scb : 2014-02-20 The Siam Commercial Bank Public Company Limited ("SCB")
+scb
+
+// schaeffler : 2015-08-06 Schaeffler Technologies AG & Co. KG
+schaeffler
+
+// schmidt : 2014-04-03 SCHMIDT GROUPE S.A.S.
+schmidt
+
+// scholarships : 2014-04-24 Scholarships.com, LLC
+scholarships
+
+// school : 2014-12-18 Binky Moon, LLC
+school
+
+// schule : 2014-03-06 Binky Moon, LLC
+schule
+
+// schwarz : 2014-09-18 Schwarz Domains und Services GmbH & Co. KG
+schwarz
+
+// science : 2014-09-11 dot Science Limited
+science
+
+// scjohnson : 2015-07-23 Johnson Shareholdings, Inc.
+scjohnson
+
+// scot : 2014-01-23 Dot Scot Registry Limited
+scot
+
+// search : 2016-06-09 Charleston Road Registry Inc.
+search
+
+// seat : 2014-05-22 SEAT, S.A. (Sociedad Unipersonal)
+seat
+
+// secure : 2015-08-27 Amazon Registry Services, Inc.
+secure
+
+// security : 2015-05-14 XYZ.COM LLC
+security
+
+// seek : 2014-12-04 Seek Limited
+seek
+
+// select : 2015-10-08 Registry Services, LLC
+select
+
+// sener : 2014-10-24 Sener Ingeniería y Sistemas, S.A.
+sener
+
+// services : 2014-02-27 Binky Moon, LLC
+services
+
+// ses : 2015-07-23 SES
+ses
+
+// seven : 2015-08-06 Seven West Media Ltd
+seven
+
+// sew : 2014-07-17 SEW-EURODRIVE GmbH & Co KG
+sew
+
+// sex : 2014-11-13 ICM Registry SX LLC
+sex
+
+// sexy : 2013-09-11 UNR Corp.
+sexy
+
+// sfr : 2015-08-13 Societe Francaise du Radiotelephone - SFR
+sfr
+
+// shangrila : 2015-09-03 Shangriâ€La International Hotel Management Limited
+shangrila
+
+// sharp : 2014-05-01 Sharp Corporation
+sharp
+
+// shaw : 2015-04-23 Shaw Cablesystems G.P.
+shaw
+
+// shell : 2015-07-30 Shell Information Technology International Inc
+shell
+
+// shia : 2014-09-04 Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti.
+shia
+
+// shiksha : 2013-11-14 Afilias Limited
+shiksha
+
+// shoes : 2013-10-02 Binky Moon, LLC
+shoes
+
+// shop : 2016-04-08 GMO Registry, Inc.
+shop
+
+// shopping : 2016-03-31 Binky Moon, LLC
+shopping
+
+// shouji : 2015-01-08 Beijing Qihu Keji Co., Ltd.
+shouji
+
+// show : 2015-03-05 Binky Moon, LLC
+show
+
+// showtime : 2015-08-06 CBS Domains Inc.
+showtime
+
+// silk : 2015-06-25 Amazon Registry Services, Inc.
+silk
+
+// sina : 2015-03-12 Sina Corporation
+sina
+
+// singles : 2013-08-27 Binky Moon, LLC
+singles
+
+// site : 2015-01-15 Radix FZC
+site
+
+// ski : 2015-04-09 Afilias Limited
+ski
+
+// skin : 2015-01-15 XYZ.COM LLC
+skin
+
+// sky : 2014-06-19 Sky International AG
+sky
+
+// skype : 2014-12-18 Microsoft Corporation
+skype
+
+// sling : 2015-07-30 DISH Technologies L.L.C.
+sling
+
+// smart : 2015-07-09 Smart Communications, Inc. (SMART)
+smart
+
+// smile : 2014-12-18 Amazon Registry Services, Inc.
+smile
+
+// sncf : 2015-02-19 Société Nationale des Chemins de fer Francais S N C F
+sncf
+
+// soccer : 2015-03-26 Binky Moon, LLC
+soccer
+
+// social : 2013-11-07 Dog Beach, LLC
+social
+
+// softbank : 2015-07-02 SoftBank Group Corp.
+softbank
+
+// software : 2014-03-20 Dog Beach, LLC
+software
+
+// sohu : 2013-12-19 Sohu.com Limited
+sohu
+
+// solar : 2013-11-07 Binky Moon, LLC
+solar
+
+// solutions : 2013-11-07 Binky Moon, LLC
+solutions
+
+// song : 2015-02-26 Amazon Registry Services, Inc.
+song
+
+// sony : 2015-01-08 Sony Corporation
+sony
+
+// soy : 2014-01-23 Charleston Road Registry Inc.
+soy
+
+// spa : 2019-09-19 Asia Spa and Wellness Promotion Council Limited
+spa
+
+// space : 2014-04-03 Radix FZC
+space
+
+// sport : 2017-11-16 Global Association of International Sports Federations (GAISF)
+sport
+
+// spot : 2015-02-26 Amazon Registry Services, Inc.
+spot
+
+// srl : 2015-05-07 InterNetX, Corp
+srl
+
+// stada : 2014-11-13 STADA Arzneimittel AG
+stada
+
+// staples : 2015-07-30 Staples, Inc.
+staples
+
+// star : 2015-01-08 Star India Private Limited
+star
+
+// statebank : 2015-03-12 STATE BANK OF INDIA
+statebank
+
+// statefarm : 2015-07-30 State Farm Mutual Automobile Insurance Company
+statefarm
+
+// stc : 2014-10-09 Saudi Telecom Company
+stc
+
+// stcgroup : 2014-10-09 Saudi Telecom Company
+stcgroup
+
+// stockholm : 2014-12-18 Stockholms kommun
+stockholm
+
+// storage : 2014-12-22 XYZ.COM LLC
+storage
+
+// store : 2015-04-09 Radix FZC
+store
+
+// stream : 2016-01-08 dot Stream Limited
+stream
+
+// studio : 2015-02-11 Dog Beach, LLC
+studio
+
+// study : 2014-12-11 OPEN UNIVERSITIES AUSTRALIA PTY LTD
+study
+
+// style : 2014-12-04 Binky Moon, LLC
+style
+
+// sucks : 2014-12-22 Vox Populi Registry Ltd.
+sucks
+
+// supplies : 2013-12-19 Binky Moon, LLC
+supplies
+
+// supply : 2013-12-19 Binky Moon, LLC
+supply
+
+// support : 2013-10-24 Binky Moon, LLC
+support
+
+// surf : 2014-01-09 Registry Services, LLC
+surf
+
+// surgery : 2014-03-20 Binky Moon, LLC
+surgery
+
+// suzuki : 2014-02-20 SUZUKI MOTOR CORPORATION
+suzuki
+
+// swatch : 2015-01-08 The Swatch Group Ltd
+swatch
+
+// swiftcover : 2015-07-23 Swiftcover Insurance Services Limited
+swiftcover
+
+// swiss : 2014-10-16 Swiss Confederation
+swiss
+
+// sydney : 2014-09-18 State of New South Wales, Department of Premier and Cabinet
+sydney
+
+// systems : 2013-11-07 Binky Moon, LLC
+systems
+
+// tab : 2014-12-04 Tabcorp Holdings Limited
+tab
+
+// taipei : 2014-07-10 Taipei City Government
+taipei
+
+// talk : 2015-04-09 Amazon Registry Services, Inc.
+talk
+
+// taobao : 2015-01-15 Alibaba Group Holding Limited
+taobao
+
+// target : 2015-07-31 Target Domain Holdings, LLC
+target
+
+// tatamotors : 2015-03-12 Tata Motors Ltd
+tatamotors
+
+// tatar : 2014-04-24 Limited Liability Company "Coordination Center of Regional Domain of Tatarstan Republic"
+tatar
+
+// tattoo : 2013-08-30 UNR Corp.
+tattoo
+
+// tax : 2014-03-20 Binky Moon, LLC
+tax
+
+// taxi : 2015-03-19 Binky Moon, LLC
+taxi
+
+// tci : 2014-09-12 Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti.
+tci
+
+// tdk : 2015-06-11 TDK Corporation
+tdk
+
+// team : 2015-03-05 Binky Moon, LLC
+team
+
+// tech : 2015-01-30 Radix FZC
+tech
+
+// technology : 2013-09-13 Binky Moon, LLC
+technology
+
+// temasek : 2014-08-07 Temasek Holdings (Private) Limited
+temasek
+
+// tennis : 2014-12-04 Binky Moon, LLC
+tennis
+
+// teva : 2015-07-02 Teva Pharmaceutical Industries Limited
+teva
+
+// thd : 2015-04-02 Home Depot Product Authority, LLC
+thd
+
+// theater : 2015-03-19 Binky Moon, LLC
+theater
+
+// theatre : 2015-05-07 XYZ.COM LLC
+theatre
+
+// tiaa : 2015-07-23 Teachers Insurance and Annuity Association of America
+tiaa
+
+// tickets : 2015-02-05 XYZ.COM LLC
+tickets
+
+// tienda : 2013-11-14 Binky Moon, LLC
+tienda
+
+// tiffany : 2015-01-30 Tiffany and Company
+tiffany
+
+// tips : 2013-09-20 Binky Moon, LLC
+tips
+
+// tires : 2014-11-07 Binky Moon, LLC
+tires
+
+// tirol : 2014-04-24 punkt Tirol GmbH
+tirol
+
+// tjmaxx : 2015-07-16 The TJX Companies, Inc.
+tjmaxx
+
+// tjx : 2015-07-16 The TJX Companies, Inc.
+tjx
+
+// tkmaxx : 2015-07-16 The TJX Companies, Inc.
+tkmaxx
+
+// tmall : 2015-01-15 Alibaba Group Holding Limited
+tmall
+
+// today : 2013-09-20 Binky Moon, LLC
+today
+
+// tokyo : 2013-11-13 GMO Registry, Inc.
+tokyo
+
+// tools : 2013-11-21 Binky Moon, LLC
+tools
+
+// top : 2014-03-20 .TOP Registry
+top
+
+// toray : 2014-12-18 Toray Industries, Inc.
+toray
+
+// toshiba : 2014-04-10 TOSHIBA Corporation
+toshiba
+
+// total : 2015-08-06 Total SA
+total
+
+// tours : 2015-01-22 Binky Moon, LLC
+tours
+
+// town : 2014-03-06 Binky Moon, LLC
+town
+
+// toyota : 2015-04-23 TOYOTA MOTOR CORPORATION
+toyota
+
+// toys : 2014-03-06 Binky Moon, LLC
+toys
+
+// trade : 2014-01-23 Elite Registry Limited
+trade
+
+// trading : 2014-12-11 Dog Beach, LLC
+trading
+
+// training : 2013-11-07 Binky Moon, LLC
+training
+
+// travel : 2015-10-09 Dog Beach, LLC
+travel
+
+// travelchannel : 2015-07-02 Lifestyle Domain Holdings, Inc.
+travelchannel
+
+// travelers : 2015-03-26 Travelers TLD, LLC
+travelers
+
+// travelersinsurance : 2015-03-26 Travelers TLD, LLC
+travelersinsurance
+
+// trust : 2014-10-16 UNR Corp.
+trust
+
+// trv : 2015-03-26 Travelers TLD, LLC
+trv
+
+// tube : 2015-06-11 Latin American Telecom LLC
+tube
+
+// tui : 2014-07-03 TUI AG
+tui
+
+// tunes : 2015-02-26 Amazon Registry Services, Inc.
+tunes
+
+// tushu : 2014-12-18 Amazon Registry Services, Inc.
+tushu
+
+// tvs : 2015-02-19 T V SUNDRAM IYENGAR & SONS LIMITED
+tvs
+
+// ubank : 2015-08-20 National Australia Bank Limited
+ubank
+
+// ubs : 2014-12-11 UBS AG
+ubs
+
+// unicom : 2015-10-15 China United Network Communications Corporation Limited
+unicom
+
+// university : 2014-03-06 Binky Moon, LLC
+university
+
+// uno : 2013-09-11 Radix FZC
+uno
+
+// uol : 2014-05-01 UBN INTERNET LTDA.
+uol
+
+// ups : 2015-06-25 UPS Market Driver, Inc.
+ups
+
+// vacations : 2013-12-05 Binky Moon, LLC
+vacations
+
+// vana : 2014-12-11 Lifestyle Domain Holdings, Inc.
+vana
+
+// vanguard : 2015-09-03 The Vanguard Group, Inc.
+vanguard
+
+// vegas : 2014-01-16 Dot Vegas, Inc.
+vegas
+
+// ventures : 2013-08-27 Binky Moon, LLC
+ventures
+
+// verisign : 2015-08-13 VeriSign, Inc.
+verisign
+
+// versicherung : 2014-03-20 tldbox GmbH
+versicherung
+
+// vet : 2014-03-06 Dog Beach, LLC
+vet
+
+// viajes : 2013-10-17 Binky Moon, LLC
+viajes
+
+// video : 2014-10-16 Dog Beach, LLC
+video
+
+// vig : 2015-05-14 VIENNA INSURANCE GROUP AG Wiener Versicherung Gruppe
+vig
+
+// viking : 2015-04-02 Viking River Cruises (Bermuda) Ltd.
+viking
+
+// villas : 2013-12-05 Binky Moon, LLC
+villas
+
+// vin : 2015-06-18 Binky Moon, LLC
+vin
+
+// vip : 2015-01-22 Registry Services, LLC
+vip
+
+// virgin : 2014-09-25 Virgin Enterprises Limited
+virgin
+
+// visa : 2015-07-30 Visa Worldwide Pte. Limited
+visa
+
+// vision : 2013-12-05 Binky Moon, LLC
+vision
+
+// viva : 2014-11-07 Saudi Telecom Company
+viva
+
+// vivo : 2015-07-31 Telefonica Brasil S.A.
+vivo
+
+// vlaanderen : 2014-02-06 DNS.be vzw
+vlaanderen
+
+// vodka : 2013-12-19 Registry Services, LLC
+vodka
+
+// volkswagen : 2015-05-14 Volkswagen Group of America Inc.
+volkswagen
+
+// volvo : 2015-11-12 Volvo Holding Sverige Aktiebolag
+volvo
+
+// vote : 2013-11-21 Monolith Registry LLC
+vote
+
+// voting : 2013-11-13 Valuetainment Corp.
+voting
+
+// voto : 2013-11-21 Monolith Registry LLC
+voto
+
+// voyage : 2013-08-27 Binky Moon, LLC
+voyage
+
+// vuelos : 2015-03-05 Travel Reservations SRL
+vuelos
+
+// wales : 2014-05-08 Nominet UK
+wales
+
+// walmart : 2015-07-31 Wal-Mart Stores, Inc.
+walmart
+
+// walter : 2014-11-13 Sandvik AB
+walter
+
+// wang : 2013-10-24 Zodiac Wang Limited
+wang
+
+// wanggou : 2014-12-18 Amazon Registry Services, Inc.
+wanggou
+
+// watch : 2013-11-14 Binky Moon, LLC
+watch
+
+// watches : 2014-12-22 Afilias Limited
+watches
+
+// weather : 2015-01-08 International Business Machines Corporation
+weather
+
+// weatherchannel : 2015-03-12 International Business Machines Corporation
+weatherchannel
+
+// webcam : 2014-01-23 dot Webcam Limited
+webcam
+
+// weber : 2015-06-04 Saint-Gobain Weber SA
+weber
+
+// website : 2014-04-03 Radix FZC
+website
+
+// wedding : 2014-04-24 Registry Services, LLC
+wedding
+
+// weibo : 2015-03-05 Sina Corporation
+weibo
+
+// weir : 2015-01-29 Weir Group IP Limited
+weir
+
+// whoswho : 2014-02-20 Who's Who Registry
+whoswho
+
+// wien : 2013-10-28 punkt.wien GmbH
+wien
+
+// wiki : 2013-11-07 Top Level Design, LLC
+wiki
+
+// williamhill : 2014-03-13 William Hill Organization Limited
+williamhill
+
+// win : 2014-11-20 First Registry Limited
+win
+
+// windows : 2014-12-18 Microsoft Corporation
+windows
+
+// wine : 2015-06-18 Binky Moon, LLC
+wine
+
+// winners : 2015-07-16 The TJX Companies, Inc.
+winners
+
+// wme : 2014-02-13 William Morris Endeavor Entertainment, LLC
+wme
+
+// wolterskluwer : 2015-08-06 Wolters Kluwer N.V.
+wolterskluwer
+
+// woodside : 2015-07-09 Woodside Petroleum Limited
+woodside
+
+// work : 2013-12-19 Registry Services, LLC
+work
+
+// works : 2013-11-14 Binky Moon, LLC
+works
+
+// world : 2014-06-12 Binky Moon, LLC
+world
+
+// wow : 2015-10-08 Amazon Registry Services, Inc.
+wow
+
+// wtc : 2013-12-19 World Trade Centers Association, Inc.
+wtc
+
+// wtf : 2014-03-06 Binky Moon, LLC
+wtf
+
+// xbox : 2014-12-18 Microsoft Corporation
+xbox
+
+// xerox : 2014-10-24 Xerox DNHC LLC
+xerox
+
+// xfinity : 2015-07-09 Comcast IP Holdings I, LLC
+xfinity
+
+// xihuan : 2015-01-08 Beijing Qihu Keji Co., Ltd.
+xihuan
+
+// xin : 2014-12-11 Elegant Leader Limited
+xin
+
+// xn--11b4c3d : 2015-01-15 VeriSign Sarl
+xn--11b4c3d
+कॉम
+
+// xn--1ck2e1b : 2015-02-26 Amazon Registry Services, Inc.
+xn--1ck2e1b
+セール
+
+// xn--1qqw23a : 2014-01-09 Guangzhou YU Wei Information Technology Co., Ltd.
+xn--1qqw23a
+佛山
+
+// xn--30rr7y : 2014-06-12 Excellent First Limited
+xn--30rr7y
+慈善
+
+// xn--3bst00m : 2013-09-13 Eagle Horizon Limited
+xn--3bst00m
+集团
+
+// xn--3ds443g : 2013-09-08 TLD REGISTRY LIMITED OY
+xn--3ds443g
+在线
+
+// xn--3oq18vl8pn36a : 2015-07-02 Volkswagen (China) Investment Co., Ltd.
+xn--3oq18vl8pn36a
+大众汽车
+
+// xn--3pxu8k : 2015-01-15 VeriSign Sarl
+xn--3pxu8k
+点看
+
+// xn--42c2d9a : 2015-01-15 VeriSign Sarl
+xn--42c2d9a
+คอม
+
+// xn--45q11c : 2013-11-21 Zodiac Gemini Ltd
+xn--45q11c
+å…«å¦
+
+// xn--4gbrim : 2013-10-04 Fans TLD Limited
+xn--4gbrim
+موقع
+
+// xn--55qw42g : 2013-11-08 China Organizational Name Administration Center
+xn--55qw42g
+公益
+
+// xn--55qx5d : 2013-11-14 China Internet Network Information Center (CNNIC)
+xn--55qx5d
+å…¬å¸
+
+// xn--5su34j936bgsg : 2015-09-03 Shangriâ€La International Hotel Management Limited
+xn--5su34j936bgsg
+香格里拉
+
+// xn--5tzm5g : 2014-12-22 Global Website TLD Asia Limited
+xn--5tzm5g
+网站
+
+// xn--6frz82g : 2013-09-23 Afilias Limited
+xn--6frz82g
+移动
+
+// xn--6qq986b3xl : 2013-09-13 Tycoon Treasure Limited
+xn--6qq986b3xl
+我爱你
+
+// xn--80adxhks : 2013-12-19 Foundation for Assistance for Internet Technologies and Infrastructure Development (FAITID)
+xn--80adxhks
+моÑква
+
+// xn--80aqecdr1a : 2015-10-21 Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)
+xn--80aqecdr1a
+католик
+
+// xn--80asehdb : 2013-07-14 CORE Association
+xn--80asehdb
+онлайн
+
+// xn--80aswg : 2013-07-14 CORE Association
+xn--80aswg
+Ñайт
+
+// xn--8y0a063a : 2015-03-26 China United Network Communications Corporation Limited
+xn--8y0a063a
+è”通
+
+// xn--9dbq2a : 2015-01-15 VeriSign Sarl
+xn--9dbq2a
+קו×
+
+// xn--9et52u : 2014-06-12 RISE VICTORY LIMITED
+xn--9et52u
+时尚
+
+// xn--9krt00a : 2015-03-12 Sina Corporation
+xn--9krt00a
+å¾®åš
+
+// xn--b4w605ferd : 2014-08-07 Temasek Holdings (Private) Limited
+xn--b4w605ferd
+淡马锡
+
+// xn--bck1b9a5dre4c : 2015-02-26 Amazon Registry Services, Inc.
+xn--bck1b9a5dre4c
+ファッション
+
+// xn--c1avg : 2013-11-14 Public Interest Registry
+xn--c1avg
+орг
+
+// xn--c2br7g : 2015-01-15 VeriSign Sarl
+xn--c2br7g
+नेट
+
+// xn--cck2b3b : 2015-02-26 Amazon Registry Services, Inc.
+xn--cck2b3b
+ストア
+
+// xn--cckwcxetd : 2019-12-19 Amazon Registry Services, Inc.
+xn--cckwcxetd
+アマゾン
+
+// xn--cg4bki : 2013-09-27 SAMSUNG SDS CO., LTD
+xn--cg4bki
+삼성
+
+// xn--czr694b : 2014-01-16 Internet DotTrademark Organisation Limited
+xn--czr694b
+商标
+
+// xn--czrs0t : 2013-12-19 Binky Moon, LLC
+xn--czrs0t
+商店
+
+// xn--czru2d : 2013-11-21 Zodiac Aquarius Limited
+xn--czru2d
+商城
+
+// xn--d1acj3b : 2013-11-20 The Foundation for Network Initiatives “The Smart Internetâ€
+xn--d1acj3b
+дети
+
+// xn--eckvdtc9d : 2014-12-18 Amazon Registry Services, Inc.
+xn--eckvdtc9d
+ãƒã‚¤ãƒ³ãƒˆ
+
+// xn--efvy88h : 2014-08-22 Guangzhou YU Wei Information Technology Co., Ltd.
+xn--efvy88h
+æ–°é—»
+
+// xn--fct429k : 2015-04-09 Amazon Registry Services, Inc.
+xn--fct429k
+家電
+
+// xn--fhbei : 2015-01-15 VeriSign Sarl
+xn--fhbei
+كوم
+
+// xn--fiq228c5hs : 2013-09-08 TLD REGISTRY LIMITED OY
+xn--fiq228c5hs
+中文网
+
+// xn--fiq64b : 2013-10-14 CITIC Group Corporation
+xn--fiq64b
+中信
+
+// xn--fjq720a : 2014-05-22 Binky Moon, LLC
+xn--fjq720a
+娱ä¹
+
+// xn--flw351e : 2014-07-31 Charleston Road Registry Inc.
+xn--flw351e
+谷歌
+
+// xn--fzys8d69uvgm : 2015-05-14 PCCW Enterprises Limited
+xn--fzys8d69uvgm
+電訊盈科
+
+// xn--g2xx48c : 2015-01-30 Nawang Heli(Xiamen) Network Service Co., LTD.
+xn--g2xx48c
+购物
+
+// xn--gckr3f0f : 2015-02-26 Amazon Registry Services, Inc.
+xn--gckr3f0f
+クラウド
+
+// xn--gk3at1e : 2015-10-08 Amazon Registry Services, Inc.
+xn--gk3at1e
+通販
+
+// xn--hxt814e : 2014-05-15 Zodiac Taurus Limited
+xn--hxt814e
+网店
+
+// xn--i1b6b1a6a2e : 2013-11-14 Public Interest Registry
+xn--i1b6b1a6a2e
+संगठन
+
+// xn--imr513n : 2014-12-11 Internet DotTrademark Organisation Limited
+xn--imr513n
+é¤åŽ…
+
+// xn--io0a7i : 2013-11-14 China Internet Network Information Center (CNNIC)
+xn--io0a7i
+网络
+
+// xn--j1aef : 2015-01-15 VeriSign Sarl
+xn--j1aef
+ком
+
+// xn--jlq480n2rg : 2019-12-19 Amazon Registry Services, Inc.
+xn--jlq480n2rg
+亚马逊
+
+// xn--jlq61u9w7b : 2015-01-08 Nokia Corporation
+xn--jlq61u9w7b
+诺基亚
+
+// xn--jvr189m : 2015-02-26 Amazon Registry Services, Inc.
+xn--jvr189m
+食å“
+
+// xn--kcrx77d1x4a : 2014-11-07 Koninklijke Philips N.V.
+xn--kcrx77d1x4a
+飞利浦
+
+// xn--kput3i : 2014-02-13 Beijing RITT-Net Technology Development Co., Ltd
+xn--kput3i
+手机
+
+// xn--mgba3a3ejt : 2014-11-20 Aramco Services Company
+xn--mgba3a3ejt
+ارامكو
+
+// xn--mgba7c0bbn0a : 2015-05-14 Crescent Holding GmbH
+xn--mgba7c0bbn0a
+العليان
+
+// xn--mgbaakc7dvf : 2015-09-03 Emirates Telecommunications Corporation (trading as Etisalat)
+xn--mgbaakc7dvf
+اتصالات
+
+// xn--mgbab2bd : 2013-10-31 CORE Association
+xn--mgbab2bd
+بازار
+
+// xn--mgbca7dzdo : 2015-07-30 Abu Dhabi Systems and Information Centre
+xn--mgbca7dzdo
+ابوظبي
+
+// xn--mgbi4ecexp : 2015-10-21 Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)
+xn--mgbi4ecexp
+كاثوليك
+
+// xn--mgbt3dhd : 2014-09-04 Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti.
+xn--mgbt3dhd
+همراه
+
+// xn--mk1bu44c : 2015-01-15 VeriSign Sarl
+xn--mk1bu44c
+ë‹·ì»´
+
+// xn--mxtq1m : 2014-03-06 Net-Chinese Co., Ltd.
+xn--mxtq1m
+政府
+
+// xn--ngbc5azd : 2013-07-13 International Domain Registry Pty. Ltd.
+xn--ngbc5azd
+شبكة
+
+// xn--ngbe9e0a : 2014-12-04 Kuwait Finance House
+xn--ngbe9e0a
+بيتك
+
+// xn--ngbrx : 2015-11-12 League of Arab States
+xn--ngbrx
+عرب
+
+// xn--nqv7f : 2013-11-14 Public Interest Registry
+xn--nqv7f
+机构
+
+// xn--nqv7fs00ema : 2013-11-14 Public Interest Registry
+xn--nqv7fs00ema
+组织机构
+
+// xn--nyqy26a : 2014-11-07 Stable Tone Limited
+xn--nyqy26a
+å¥åº·
+
+// xn--otu796d : 2017-08-06 Jiang Yu Liang Cai Technology Company Limited
+xn--otu796d
+æ‹›è˜
+
+// xn--p1acf : 2013-12-12 Rusnames Limited
+xn--p1acf
+руÑ
+
+// xn--pssy2u : 2015-01-15 VeriSign Sarl
+xn--pssy2u
+大拿
+
+// xn--q9jyb4c : 2013-09-17 Charleston Road Registry Inc.
+xn--q9jyb4c
+ã¿ã‚“ãª
+
+// xn--qcka1pmc : 2014-07-31 Charleston Road Registry Inc.
+xn--qcka1pmc
+グーグル
+
+// xn--rhqv96g : 2013-09-11 Stable Tone Limited
+xn--rhqv96g
+世界
+
+// xn--rovu88b : 2015-02-26 Amazon Registry Services, Inc.
+xn--rovu88b
+書ç±
+
+// xn--ses554g : 2014-01-16 KNET Co., Ltd.
+xn--ses554g
+网å€
+
+// xn--t60b56a : 2015-01-15 VeriSign Sarl
+xn--t60b56a
+ë‹·ë„·
+
+// xn--tckwe : 2015-01-15 VeriSign Sarl
+xn--tckwe
+コム
+
+// xn--tiq49xqyj : 2015-10-21 Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)
+xn--tiq49xqyj
+天主教
+
+// xn--unup4y : 2013-07-14 Binky Moon, LLC
+xn--unup4y
+游æˆ
+
+// xn--vermgensberater-ctb : 2014-06-23 Deutsche Vermögensberatung Aktiengesellschaft DVAG
+xn--vermgensberater-ctb
+vermögensberater
+
+// xn--vermgensberatung-pwb : 2014-06-23 Deutsche Vermögensberatung Aktiengesellschaft DVAG
+xn--vermgensberatung-pwb
+vermögensberatung
+
+// xn--vhquv : 2013-08-27 Binky Moon, LLC
+xn--vhquv
+ä¼ä¸š
+
+// xn--vuq861b : 2014-10-16 Beijing Tele-info Network Technology Co., Ltd.
+xn--vuq861b
+ä¿¡æ¯
+
+// xn--w4r85el8fhu5dnra : 2015-04-30 Kerry Trading Co. Limited
+xn--w4r85el8fhu5dnra
+嘉里大酒店
+
+// xn--w4rs40l : 2015-07-30 Kerry Trading Co. Limited
+xn--w4rs40l
+嘉里
+
+// xn--xhq521b : 2013-11-14 Guangzhou YU Wei Information Technology Co., Ltd.
+xn--xhq521b
+广东
+
+// xn--zfr164b : 2013-11-08 China Organizational Name Administration Center
+xn--zfr164b
+政务
+
+// xyz : 2013-12-05 XYZ.COM LLC
+xyz
+
+// yachts : 2014-01-09 XYZ.COM LLC
+yachts
+
+// yahoo : 2015-04-02 Oath Inc.
+yahoo
+
+// yamaxun : 2014-12-18 Amazon Registry Services, Inc.
+yamaxun
+
+// yandex : 2014-04-10 Yandex Europe B.V.
+yandex
+
+// yodobashi : 2014-11-20 YODOBASHI CAMERA CO.,LTD.
+yodobashi
+
+// yoga : 2014-05-29 Registry Services, LLC
+yoga
+
+// yokohama : 2013-12-12 GMO Registry, Inc.
+yokohama
+
+// you : 2015-04-09 Amazon Registry Services, Inc.
+you
+
+// youtube : 2014-05-01 Charleston Road Registry Inc.
+youtube
+
+// yun : 2015-01-08 Beijing Qihu Keji Co., Ltd.
+yun
+
+// zappos : 2015-06-25 Amazon Registry Services, Inc.
+zappos
+
+// zara : 2014-11-07 Industria de Diseño Textil, S.A. (INDITEX, S.A.)
+zara
+
+// zero : 2014-12-18 Amazon Registry Services, Inc.
+zero
+
+// zip : 2014-05-08 Charleston Road Registry Inc.
+zip
+
+// zone : 2013-11-14 Binky Moon, LLC
+zone
+
+// zuerich : 2014-11-07 Kanton Zürich (Canton of Zurich)
+zuerich
+
+
+// ===END ICANN DOMAINS===
+// ===BEGIN PRIVATE DOMAINS===
+// (Note: these are in alphabetical order by company name)
+
+// 1GB LLC : https://www.1gb.ua/
+// Submitted by 1GB LLC <noc@1gb.com.ua>
+cc.ua
+inf.ua
+ltd.ua
+
+// 611coin : https://611project.org/
+611.to
+
+// Aaron Marais' Gitlab pages: https://lab.aaronleem.co.za
+// Submitted by Aaron Marais <its_me@aaronleem.co.za>
+graphox.us
+
+// accesso Technology Group, plc. : https://accesso.com/
+// Submitted by accesso Team <accessoecommerce@accesso.com>
+*.devcdnaccesso.com
+
+// Adobe : https://www.adobe.com/
+// Submitted by Ian Boston <boston@adobe.com> and Lars Trieloff <trieloff@adobe.com>
+adobeaemcloud.com
+*.dev.adobeaemcloud.com
+hlx.live
+adobeaemcloud.net
+hlx.page
+hlx3.page
+
+// Agnat sp. z o.o. : https://domena.pl
+// Submitted by Przemyslaw Plewa <it-admin@domena.pl>
+beep.pl
+
+// alboto.ca : http://alboto.ca
+// Submitted by Anton Avramov <avramov@alboto.ca>
+barsy.ca
+
+// Alces Software Ltd : http://alces-software.com
+// Submitted by Mark J. Titorenko <mark.titorenko@alces-software.com>
+*.compute.estate
+*.alces.network
+
+// all-inkl.com : https://all-inkl.com
+// Submitted by Werner Kaltofen <wk@all-inkl.com>
+kasserver.com
+
+// Altervista: https://www.altervista.org
+// Submitted by Carlo Cannas <tech_staff@altervista.it>
+altervista.org
+
+// alwaysdata : https://www.alwaysdata.com
+// Submitted by Cyril <admin@alwaysdata.com>
+alwaysdata.net
+
+// Amazon CloudFront : https://aws.amazon.com/cloudfront/
+// Submitted by Donavan Miller <donavanm@amazon.com>
+cloudfront.net
+
+// Amazon Elastic Compute Cloud : https://aws.amazon.com/ec2/
+// Submitted by Luke Wells <psl-maintainers@amazon.com>
+*.compute.amazonaws.com
+*.compute-1.amazonaws.com
+*.compute.amazonaws.com.cn
+us-east-1.amazonaws.com
+
+// Amazon Elastic Beanstalk : https://aws.amazon.com/elasticbeanstalk/
+// Submitted by Luke Wells <psl-maintainers@amazon.com>
+cn-north-1.eb.amazonaws.com.cn
+cn-northwest-1.eb.amazonaws.com.cn
+elasticbeanstalk.com
+ap-northeast-1.elasticbeanstalk.com
+ap-northeast-2.elasticbeanstalk.com
+ap-northeast-3.elasticbeanstalk.com
+ap-south-1.elasticbeanstalk.com
+ap-southeast-1.elasticbeanstalk.com
+ap-southeast-2.elasticbeanstalk.com
+ca-central-1.elasticbeanstalk.com
+eu-central-1.elasticbeanstalk.com
+eu-west-1.elasticbeanstalk.com
+eu-west-2.elasticbeanstalk.com
+eu-west-3.elasticbeanstalk.com
+sa-east-1.elasticbeanstalk.com
+us-east-1.elasticbeanstalk.com
+us-east-2.elasticbeanstalk.com
+us-gov-west-1.elasticbeanstalk.com
+us-west-1.elasticbeanstalk.com
+us-west-2.elasticbeanstalk.com
+
+// Amazon Elastic Load Balancing : https://aws.amazon.com/elasticloadbalancing/
+// Submitted by Luke Wells <psl-maintainers@amazon.com>
+*.elb.amazonaws.com
+*.elb.amazonaws.com.cn
+
+// Amazon Global Accelerator : https://aws.amazon.com/global-accelerator/
+// Submitted by Daniel Massaguer <psl-maintainers@amazon.com>
+awsglobalaccelerator.com
+
+// Amazon S3 : https://aws.amazon.com/s3/
+// Submitted by Luke Wells <psl-maintainers@amazon.com>
+s3.amazonaws.com
+s3-ap-northeast-1.amazonaws.com
+s3-ap-northeast-2.amazonaws.com
+s3-ap-south-1.amazonaws.com
+s3-ap-southeast-1.amazonaws.com
+s3-ap-southeast-2.amazonaws.com
+s3-ca-central-1.amazonaws.com
+s3-eu-central-1.amazonaws.com
+s3-eu-west-1.amazonaws.com
+s3-eu-west-2.amazonaws.com
+s3-eu-west-3.amazonaws.com
+s3-external-1.amazonaws.com
+s3-fips-us-gov-west-1.amazonaws.com
+s3-sa-east-1.amazonaws.com
+s3-us-gov-west-1.amazonaws.com
+s3-us-east-2.amazonaws.com
+s3-us-west-1.amazonaws.com
+s3-us-west-2.amazonaws.com
+s3.ap-northeast-2.amazonaws.com
+s3.ap-south-1.amazonaws.com
+s3.cn-north-1.amazonaws.com.cn
+s3.ca-central-1.amazonaws.com
+s3.eu-central-1.amazonaws.com
+s3.eu-west-2.amazonaws.com
+s3.eu-west-3.amazonaws.com
+s3.us-east-2.amazonaws.com
+s3.dualstack.ap-northeast-1.amazonaws.com
+s3.dualstack.ap-northeast-2.amazonaws.com
+s3.dualstack.ap-south-1.amazonaws.com
+s3.dualstack.ap-southeast-1.amazonaws.com
+s3.dualstack.ap-southeast-2.amazonaws.com
+s3.dualstack.ca-central-1.amazonaws.com
+s3.dualstack.eu-central-1.amazonaws.com
+s3.dualstack.eu-west-1.amazonaws.com
+s3.dualstack.eu-west-2.amazonaws.com
+s3.dualstack.eu-west-3.amazonaws.com
+s3.dualstack.sa-east-1.amazonaws.com
+s3.dualstack.us-east-1.amazonaws.com
+s3.dualstack.us-east-2.amazonaws.com
+s3-website-us-east-1.amazonaws.com
+s3-website-us-west-1.amazonaws.com
+s3-website-us-west-2.amazonaws.com
+s3-website-ap-northeast-1.amazonaws.com
+s3-website-ap-southeast-1.amazonaws.com
+s3-website-ap-southeast-2.amazonaws.com
+s3-website-eu-west-1.amazonaws.com
+s3-website-sa-east-1.amazonaws.com
+s3-website.ap-northeast-2.amazonaws.com
+s3-website.ap-south-1.amazonaws.com
+s3-website.ca-central-1.amazonaws.com
+s3-website.eu-central-1.amazonaws.com
+s3-website.eu-west-2.amazonaws.com
+s3-website.eu-west-3.amazonaws.com
+s3-website.us-east-2.amazonaws.com
+
+// Amsterdam Wireless: https://www.amsterdamwireless.nl/
+// Submitted by Imre Jonk <hostmaster@amsterdamwireless.nl>
+amsw.nl
+
+// Amune : https://amune.org/
+// Submitted by Team Amune <cert@amune.org>
+t3l3p0rt.net
+tele.amune.org
+
+// Apigee : https://apigee.com/
+// Submitted by Apigee Security Team <security@apigee.com>
+apigee.io
+
+// Appspace : https://www.appspace.com
+// Submitted by Appspace Security Team <security@appspace.com>
+appspacehosted.com
+appspaceusercontent.com
+
+// Appudo UG (haftungsbeschränkt) : https://www.appudo.com
+// Submitted by Alexander Hochbaum <admin@appudo.com>
+appudo.net
+
+// Aptible : https://www.aptible.com/
+// Submitted by Thomas Orozco <thomas@aptible.com>
+on-aptible.com
+
+// ASEINet : https://www.aseinet.com/
+// Submitted by Asei SEKIGUCHI <mail@aseinet.com>
+user.aseinet.ne.jp
+gv.vc
+d.gv.vc
+
+// Asociación Amigos de la Informática "Euskalamiga" : http://encounter.eus/
+// Submitted by Hector Martin <marcan@euskalencounter.org>
+user.party.eus
+
+// Association potager.org : https://potager.org/
+// Submitted by Lunar <jardiniers@potager.org>
+pimienta.org
+poivron.org
+potager.org
+sweetpepper.org
+
+// ASUSTOR Inc. : http://www.asustor.com
+// Submitted by Vincent Tseng <vincenttseng@asustor.com>
+myasustor.com
+
+// Atlassian : https://atlassian.com
+// Submitted by Sam Smyth <devloop@atlassian.com>
+cdn.prod.atlassian-dev.net
+
+// AVM : https://avm.de
+// Submitted by Andreas Weise <a.weise@avm.de>
+myfritz.net
+
+// AVStack Pte. Ltd. : https://avstack.io
+// Submitted by Jasper Hugo <jasper@avstack.io>
+onavstack.net
+
+// AW AdvisorWebsites.com Software Inc : https://advisorwebsites.com
+// Submitted by James Kennedy <domains@advisorwebsites.com>
+*.awdev.ca
+*.advisor.ws
+
+// b-data GmbH : https://www.b-data.io
+// Submitted by Olivier Benz <olivier.benz@b-data.ch>
+b-data.io
+
+// backplane : https://www.backplane.io
+// Submitted by Anthony Voutas <anthony@backplane.io>
+backplaneapp.io
+
+// Balena : https://www.balena.io
+// Submitted by Petros Angelatos <petrosagg@balena.io>
+balena-devices.com
+
+// University of Banja Luka : https://unibl.org
+// Domains for Republic of Srpska administrative entity.
+// Submitted by Marko Ivanovic <kormang@hotmail.rs>
+rs.ba
+
+// Banzai Cloud
+// Submitted by Janos Matyas <info@banzaicloud.com>
+*.banzai.cloud
+app.banzaicloud.io
+*.backyards.banzaicloud.io
+
+
+// BetaInABox
+// Submitted by Adrian <adrian@betainabox.com>
+betainabox.com
+
+// BinaryLane : http://www.binarylane.com
+// Submitted by Nathan O'Sullivan <nathan@mammoth.com.au>
+bnr.la
+
+// Bitbucket : http://bitbucket.org
+// Submitted by Andy Ortlieb <aortlieb@atlassian.com>
+bitbucket.io
+
+// Blackbaud, Inc. : https://www.blackbaud.com
+// Submitted by Paul Crowder <paul.crowder@blackbaud.com>
+blackbaudcdn.net
+
+// Blatech : http://www.blatech.net
+// Submitted by Luke Bratch <luke@bratch.co.uk>
+of.je
+
+// Blue Bite, LLC : https://bluebite.com
+// Submitted by Joshua Weiss <admin.engineering@bluebite.com>
+bluebite.io
+
+// Boomla : https://boomla.com
+// Submitted by Tibor Halter <thalter@boomla.com>
+boomla.net
+
+// Boutir : https://www.boutir.com
+// Submitted by Eric Ng Ka Ka <ngkaka@boutir.com>
+boutir.com
+
+// Boxfuse : https://boxfuse.com
+// Submitted by Axel Fontaine <axel@boxfuse.com>
+boxfuse.io
+
+// bplaced : https://www.bplaced.net/
+// Submitted by Miroslav Bozic <security@bplaced.net>
+square7.ch
+bplaced.com
+bplaced.de
+square7.de
+bplaced.net
+square7.net
+
+// Brendly : https://brendly.rs
+// Submitted by Dusan Radovanovic <dusan.radovanovic@brendly.rs>
+shop.brendly.rs
+
+// BrowserSafetyMark
+// Submitted by Dave Tharp <browsersafetymark.io@quicinc.com>
+browsersafetymark.io
+
+// Bytemark Hosting : https://www.bytemark.co.uk
+// Submitted by Paul Cammish <paul.cammish@bytemark.co.uk>
+uk0.bigv.io
+dh.bytemark.co.uk
+vm.bytemark.co.uk
+
+// Caf.js Labs LLC : https://www.cafjs.com
+// Submitted by Antonio Lain <antlai@cafjs.com>
+cafjs.com
+
+// callidomus : https://www.callidomus.com/
+// Submitted by Marcus Popp <admin@callidomus.com>
+mycd.eu
+
+// Carrd : https://carrd.co
+// Submitted by AJ <aj@carrd.co>
+drr.ac
+uwu.ai
+carrd.co
+crd.co
+ju.mp
+
+// CentralNic : http://www.centralnic.com/names/domains
+// Submitted by registry <gavin.brown@centralnic.com>
+ae.org
+br.com
+cn.com
+com.de
+com.se
+de.com
+eu.com
+gb.net
+hu.net
+jp.net
+jpn.com
+mex.com
+ru.com
+sa.com
+se.net
+uk.com
+uk.net
+us.com
+za.bz
+za.com
+
+// No longer operated by CentralNic, these entries should be adopted and/or removed by current operators
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+ar.com
+gb.com
+hu.com
+kr.com
+no.com
+qc.com
+uy.com
+
+// Africa.com Web Solutions Ltd : https://registry.africa.com
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+africa.com
+
+// iDOT Services Limited : http://www.domain.gr.com
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+gr.com
+
+// Radix FZC : http://domains.in.net
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+in.net
+web.in
+
+// US REGISTRY LLC : http://us.org
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+us.org
+
+// co.com Registry, LLC : https://registry.co.com
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+co.com
+
+// Roar Domains LLC : https://roar.basketball/
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+aus.basketball
+nz.basketball
+
+// BRS Media : https://brsmedia.com/
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+radio.am
+radio.fm
+
+// c.la : http://www.c.la/
+c.la
+
+// certmgr.org : https://certmgr.org
+// Submitted by B. Blechschmidt <hostmaster@certmgr.org>
+certmgr.org
+
+// Cityhost LLC : https://cityhost.ua
+// Submitted by Maksym Rivtin <support@cityhost.net.ua>
+cx.ua
+
+// Civilized Discourse Construction Kit, Inc. : https://www.discourse.org/
+// Submitted by Rishabh Nambiar & Michael Brown <team@discourse.org>
+discourse.group
+discourse.team
+
+// ClearVox : http://www.clearvox.nl/
+// Submitted by Leon Rowland <leon@clearvox.nl>
+virtueeldomein.nl
+
+// Clever Cloud : https://www.clever-cloud.com/
+// Submitted by Quentin Adam <noc@clever-cloud.com>
+cleverapps.io
+
+// Clerk : https://www.clerk.dev
+// Submitted by Colin Sidoti <systems@clerk.dev>
+*.lcl.dev
+*.lclstage.dev
+*.stg.dev
+*.stgstage.dev
+
+// Clic2000 : https://clic2000.fr
+// Submitted by Mathilde Blanchemanche <mathilde@clic2000.fr>
+clic2000.net
+
+// ClickRising : https://clickrising.com/
+// Submitted by Umut Gumeli <infrastructure-publicsuffixlist@clickrising.com>
+clickrising.net
+
+// Cloud66 : https://www.cloud66.com/
+// Submitted by Khash Sajadi <khash@cloud66.com>
+c66.me
+cloud66.ws
+cloud66.zone
+
+// CloudAccess.net : https://www.cloudaccess.net/
+// Submitted by Pawel Panek <noc@cloudaccess.net>
+jdevcloud.com
+wpdevcloud.com
+cloudaccess.host
+freesite.host
+cloudaccess.net
+
+// cloudControl : https://www.cloudcontrol.com/
+// Submitted by Tobias Wilken <tw@cloudcontrol.com>
+cloudcontrolled.com
+cloudcontrolapp.com
+
+// Cloudera, Inc. : https://www.cloudera.com/
+// Submitted by Philip Langdale <security@cloudera.com>
+cloudera.site
+
+// Cloudflare, Inc. : https://www.cloudflare.com/
+// Submitted by Cloudflare Team <publicsuffixlist@cloudflare.com>
+pages.dev
+trycloudflare.com
+workers.dev
+
+// Clovyr : https://clovyr.io
+// Submitted by Patrick Nielsen <patrick@clovyr.io>
+wnext.app
+
+// co.ca : http://registry.co.ca/
+co.ca
+
+// Co & Co : https://co-co.nl/
+// Submitted by Govert Versluis <govert@co-co.nl>
+*.otap.co
+
+// i-registry s.r.o. : http://www.i-registry.cz/
+// Submitted by Martin Semrad <semrad@i-registry.cz>
+co.cz
+
+// CDN77.com : http://www.cdn77.com
+// Submitted by Jan Krpes <jan.krpes@cdn77.com>
+c.cdn77.org
+cdn77-ssl.net
+r.cdn77.net
+rsc.cdn77.org
+ssl.origin.cdn77-secure.org
+
+// Cloud DNS Ltd : http://www.cloudns.net
+// Submitted by Aleksander Hristov <noc@cloudns.net>
+cloudns.asia
+cloudns.biz
+cloudns.club
+cloudns.cc
+cloudns.eu
+cloudns.in
+cloudns.info
+cloudns.org
+cloudns.pro
+cloudns.pw
+cloudns.us
+
+// CNPY : https://cnpy.gdn
+// Submitted by Angelo Gladding <angelo@lahacker.net>
+cnpy.gdn
+
+// CoDNS B.V.
+co.nl
+co.no
+
+// Combell.com : https://www.combell.com
+// Submitted by Thomas Wouters <thomas.wouters@combellgroup.com>
+webhosting.be
+hosting-cluster.nl
+
+// Coordination Center for TLD RU and XN--P1AI : https://cctld.ru/en/domains/domens_ru/reserved/
+// Submitted by George Georgievsky <gug@cctld.ru>
+ac.ru
+edu.ru
+gov.ru
+int.ru
+mil.ru
+test.ru
+
+// COSIMO GmbH : http://www.cosimo.de
+// Submitted by Rene Marticke <rmarticke@cosimo.de>
+dyn.cosidns.de
+dynamisches-dns.de
+dnsupdater.de
+internet-dns.de
+l-o-g-i-n.de
+dynamic-dns.info
+feste-ip.net
+knx-server.net
+static-access.net
+
+// Craynic, s.r.o. : http://www.craynic.com/
+// Submitted by Ales Krajnik <ales.krajnik@craynic.com>
+realm.cz
+
+// Cryptonomic : https://cryptonomic.net/
+// Submitted by Andrew Cady <public-suffix-list@cryptonomic.net>
+*.cryptonomic.net
+
+// Cupcake : https://cupcake.io/
+// Submitted by Jonathan Rudenberg <jonathan@cupcake.io>
+cupcake.is
+
+// Curv UG : https://curv-labs.de/
+// Submitted by Marvin Wiesner <Marvin@curv-labs.de>
+curv.dev
+
+// Customer OCI - Oracle Dyn https://cloud.oracle.com/home https://dyn.com/dns/
+// Submitted by Gregory Drake <support@dyn.com>
+// Note: This is intended to also include customer-oci.com due to wildcards implicitly including the current label
+*.customer-oci.com
+*.oci.customer-oci.com
+*.ocp.customer-oci.com
+*.ocs.customer-oci.com
+
+// cyon GmbH : https://www.cyon.ch/
+// Submitted by Dominic Luechinger <dol@cyon.ch>
+cyon.link
+cyon.site
+
+// Danger Science Group: https://dangerscience.com/
+// Submitted by Skylar MacDonald <skylar@dangerscience.com>
+fnwk.site
+folionetwork.site
+platform0.app
+
+// Daplie, Inc : https://daplie.com
+// Submitted by AJ ONeal <aj@daplie.com>
+daplie.me
+localhost.daplie.me
+
+// Datto, Inc. : https://www.datto.com/
+// Submitted by Philipp Heckel <ph@datto.com>
+dattolocal.com
+dattorelay.com
+dattoweb.com
+mydatto.com
+dattolocal.net
+mydatto.net
+
+// Dansk.net : http://www.dansk.net/
+// Submitted by Anani Voule <digital@digital.co.dk>
+biz.dk
+co.dk
+firm.dk
+reg.dk
+store.dk
+
+// dappnode.io : https://dappnode.io/
+// Submitted by Abel Boldu / DAppNode Team <community@dappnode.io>
+dyndns.dappnode.io
+
+// dapps.earth : https://dapps.earth/
+// Submitted by Daniil Burdakov <icqkill@gmail.com>
+*.dapps.earth
+*.bzz.dapps.earth
+
+// Dark, Inc. : https://darklang.com
+// Submitted by Paul Biggar <ops@darklang.com>
+builtwithdark.com
+
+// DataDetect, LLC. : https://datadetect.com
+// Submitted by Andrew Banchich <abanchich@sceven.com>
+demo.datadetect.com
+instance.datadetect.com
+
+// Datawire, Inc : https://www.datawire.io
+// Submitted by Richard Li <secalert@datawire.io>
+edgestack.me
+
+// DDNS5 : https://ddns5.com
+// Submitted by Cameron Elliott <cameron@cameronelliott.com>
+ddns5.com
+
+// Debian : https://www.debian.org/
+// Submitted by Peter Palfrader / Debian Sysadmin Team <dsa-publicsuffixlist@debian.org>
+debian.net
+
+// Deno Land Inc : https://deno.com/
+// Submitted by Luca Casonato <hostmaster@deno.com>
+deno.dev
+deno-staging.dev
+
+// deSEC : https://desec.io/
+// Submitted by Peter Thomassen <peter@desec.io>
+dedyn.io
+
+// DNS Africa Ltd https://dns.business
+// Submitted by Calvin Browne <calvin@dns.business>
+jozi.biz
+
+// DNShome : https://www.dnshome.de/
+// Submitted by Norbert Auler <mail@dnshome.de>
+dnshome.de
+
+// DotArai : https://www.dotarai.com/
+// Submitted by Atsadawat Netcharadsang <atsadawat@dotarai.co.th>
+online.th
+shop.th
+
+// DrayTek Corp. : https://www.draytek.com/
+// Submitted by Paul Fang <mis@draytek.com>
+drayddns.com
+
+// DreamCommerce : https://shoper.pl/
+// Submitted by Konrad Kotarba <konrad.kotarba@dreamcommerce.com>
+shoparena.pl
+
+// DreamHost : http://www.dreamhost.com/
+// Submitted by Andrew Farmer <andrew.farmer@dreamhost.com>
+dreamhosters.com
+
+// Drobo : http://www.drobo.com/
+// Submitted by Ricardo Padilha <rpadilha@drobo.com>
+mydrobo.com
+
+// Drud Holdings, LLC. : https://www.drud.com/
+// Submitted by Kevin Bridges <kevin@drud.com>
+drud.io
+drud.us
+
+// DuckDNS : http://www.duckdns.org/
+// Submitted by Richard Harper <richard@duckdns.org>
+duckdns.org
+
+// Bip : https://bip.sh
+// Submitted by Joel Kennedy <joel@bip.sh>
+bip.sh
+
+// bitbridge.net : Submitted by Craig Welch, abeliidev@gmail.com
+bitbridge.net
+
+// dy.fi : http://dy.fi/
+// Submitted by Heikki Hannikainen <hessu@hes.iki.fi>
+dy.fi
+tunk.org
+
+// DynDNS.com : http://www.dyndns.com/services/dns/dyndns/
+dyndns-at-home.com
+dyndns-at-work.com
+dyndns-blog.com
+dyndns-free.com
+dyndns-home.com
+dyndns-ip.com
+dyndns-mail.com
+dyndns-office.com
+dyndns-pics.com
+dyndns-remote.com
+dyndns-server.com
+dyndns-web.com
+dyndns-wiki.com
+dyndns-work.com
+dyndns.biz
+dyndns.info
+dyndns.org
+dyndns.tv
+at-band-camp.net
+ath.cx
+barrel-of-knowledge.info
+barrell-of-knowledge.info
+better-than.tv
+blogdns.com
+blogdns.net
+blogdns.org
+blogsite.org
+boldlygoingnowhere.org
+broke-it.net
+buyshouses.net
+cechire.com
+dnsalias.com
+dnsalias.net
+dnsalias.org
+dnsdojo.com
+dnsdojo.net
+dnsdojo.org
+does-it.net
+doesntexist.com
+doesntexist.org
+dontexist.com
+dontexist.net
+dontexist.org
+doomdns.com
+doomdns.org
+dvrdns.org
+dyn-o-saur.com
+dynalias.com
+dynalias.net
+dynalias.org
+dynathome.net
+dyndns.ws
+endofinternet.net
+endofinternet.org
+endoftheinternet.org
+est-a-la-maison.com
+est-a-la-masion.com
+est-le-patron.com
+est-mon-blogueur.com
+for-better.biz
+for-more.biz
+for-our.info
+for-some.biz
+for-the.biz
+forgot.her.name
+forgot.his.name
+from-ak.com
+from-al.com
+from-ar.com
+from-az.net
+from-ca.com
+from-co.net
+from-ct.com
+from-dc.com
+from-de.com
+from-fl.com
+from-ga.com
+from-hi.com
+from-ia.com
+from-id.com
+from-il.com
+from-in.com
+from-ks.com
+from-ky.com
+from-la.net
+from-ma.com
+from-md.com
+from-me.org
+from-mi.com
+from-mn.com
+from-mo.com
+from-ms.com
+from-mt.com
+from-nc.com
+from-nd.com
+from-ne.com
+from-nh.com
+from-nj.com
+from-nm.com
+from-nv.com
+from-ny.net
+from-oh.com
+from-ok.com
+from-or.com
+from-pa.com
+from-pr.com
+from-ri.com
+from-sc.com
+from-sd.com
+from-tn.com
+from-tx.com
+from-ut.com
+from-va.com
+from-vt.com
+from-wa.com
+from-wi.com
+from-wv.com
+from-wy.com
+ftpaccess.cc
+fuettertdasnetz.de
+game-host.org
+game-server.cc
+getmyip.com
+gets-it.net
+go.dyndns.org
+gotdns.com
+gotdns.org
+groks-the.info
+groks-this.info
+ham-radio-op.net
+here-for-more.info
+hobby-site.com
+hobby-site.org
+home.dyndns.org
+homedns.org
+homeftp.net
+homeftp.org
+homeip.net
+homelinux.com
+homelinux.net
+homelinux.org
+homeunix.com
+homeunix.net
+homeunix.org
+iamallama.com
+in-the-band.net
+is-a-anarchist.com
+is-a-blogger.com
+is-a-bookkeeper.com
+is-a-bruinsfan.org
+is-a-bulls-fan.com
+is-a-candidate.org
+is-a-caterer.com
+is-a-celticsfan.org
+is-a-chef.com
+is-a-chef.net
+is-a-chef.org
+is-a-conservative.com
+is-a-cpa.com
+is-a-cubicle-slave.com
+is-a-democrat.com
+is-a-designer.com
+is-a-doctor.com
+is-a-financialadvisor.com
+is-a-geek.com
+is-a-geek.net
+is-a-geek.org
+is-a-green.com
+is-a-guru.com
+is-a-hard-worker.com
+is-a-hunter.com
+is-a-knight.org
+is-a-landscaper.com
+is-a-lawyer.com
+is-a-liberal.com
+is-a-libertarian.com
+is-a-linux-user.org
+is-a-llama.com
+is-a-musician.com
+is-a-nascarfan.com
+is-a-nurse.com
+is-a-painter.com
+is-a-patsfan.org
+is-a-personaltrainer.com
+is-a-photographer.com
+is-a-player.com
+is-a-republican.com
+is-a-rockstar.com
+is-a-socialist.com
+is-a-soxfan.org
+is-a-student.com
+is-a-teacher.com
+is-a-techie.com
+is-a-therapist.com
+is-an-accountant.com
+is-an-actor.com
+is-an-actress.com
+is-an-anarchist.com
+is-an-artist.com
+is-an-engineer.com
+is-an-entertainer.com
+is-by.us
+is-certified.com
+is-found.org
+is-gone.com
+is-into-anime.com
+is-into-cars.com
+is-into-cartoons.com
+is-into-games.com
+is-leet.com
+is-lost.org
+is-not-certified.com
+is-saved.org
+is-slick.com
+is-uberleet.com
+is-very-bad.org
+is-very-evil.org
+is-very-good.org
+is-very-nice.org
+is-very-sweet.org
+is-with-theband.com
+isa-geek.com
+isa-geek.net
+isa-geek.org
+isa-hockeynut.com
+issmarterthanyou.com
+isteingeek.de
+istmein.de
+kicks-ass.net
+kicks-ass.org
+knowsitall.info
+land-4-sale.us
+lebtimnetz.de
+leitungsen.de
+likes-pie.com
+likescandy.com
+merseine.nu
+mine.nu
+misconfused.org
+mypets.ws
+myphotos.cc
+neat-url.com
+office-on-the.net
+on-the-web.tv
+podzone.net
+podzone.org
+readmyblog.org
+saves-the-whales.com
+scrapper-site.net
+scrapping.cc
+selfip.biz
+selfip.com
+selfip.info
+selfip.net
+selfip.org
+sells-for-less.com
+sells-for-u.com
+sells-it.net
+sellsyourhome.org
+servebbs.com
+servebbs.net
+servebbs.org
+serveftp.net
+serveftp.org
+servegame.org
+shacknet.nu
+simple-url.com
+space-to-rent.com
+stuff-4-sale.org
+stuff-4-sale.us
+teaches-yoga.com
+thruhere.net
+traeumtgerade.de
+webhop.biz
+webhop.info
+webhop.net
+webhop.org
+worse-than.tv
+writesthisblog.com
+
+// ddnss.de : https://www.ddnss.de/
+// Submitted by Robert Niedziela <webmaster@ddnss.de>
+ddnss.de
+dyn.ddnss.de
+dyndns.ddnss.de
+dyndns1.de
+dyn-ip24.de
+home-webserver.de
+dyn.home-webserver.de
+myhome-server.de
+ddnss.org
+
+// Definima : http://www.definima.com/
+// Submitted by Maxence Bitterli <maxence@definima.com>
+definima.net
+definima.io
+
+// DigitalOcean : https://digitalocean.com/
+// Submitted by Braxton Huggins <bhuggins@digitalocean.com>
+ondigitalocean.app
+
+// dnstrace.pro : https://dnstrace.pro/
+// Submitted by Chris Partridge <chris@partridge.tech>
+bci.dnstrace.pro
+
+// Dynu.com : https://www.dynu.com/
+// Submitted by Sue Ye <sue@dynu.com>
+ddnsfree.com
+ddnsgeek.com
+giize.com
+gleeze.com
+kozow.com
+loseyourip.com
+ooguy.com
+theworkpc.com
+casacam.net
+dynu.net
+accesscam.org
+camdvr.org
+freeddns.org
+mywire.org
+webredirect.org
+myddns.rocks
+blogsite.xyz
+
+// dynv6 : https://dynv6.com
+// Submitted by Dominik Menke <dom@digineo.de>
+dynv6.net
+
+// E4YOU spol. s.r.o. : https://e4you.cz/
+// Submitted by Vladimir Dudr <info@e4you.cz>
+e4.cz
+
+// eero : https://eero.com/
+// Submitted by Yue Kang <eero-dynamic-dns@amazon.com>
+eero.online
+eero-stage.online
+
+// Elementor : Elementor Ltd.
+// Submitted by Anton Barkan <antonb@elementor.com>
+elementor.cloud
+elementor.cool
+
+// En root‽ : https://en-root.org
+// Submitted by Emmanuel Raviart <emmanuel@raviart.com>
+en-root.fr
+
+// Enalean SAS: https://www.enalean.com
+// Submitted by Thomas Cottier <thomas.cottier@enalean.com>
+mytuleap.com
+tuleap-partners.com
+
+// ECG Robotics, Inc: https://ecgrobotics.org
+// Submitted by <frc1533@ecgrobotics.org>
+onred.one
+staging.onred.one
+
+// One.com: https://www.one.com/
+// Submitted by Jacob Bunk Nielsen <jbn@one.com>
+service.one
+
+// Enonic : http://enonic.com/
+// Submitted by Erik Kaareng-Sunde <esu@enonic.com>
+enonic.io
+customer.enonic.io
+
+// EU.org https://eu.org/
+// Submitted by Pierre Beyssac <hostmaster@eu.org>
+eu.org
+al.eu.org
+asso.eu.org
+at.eu.org
+au.eu.org
+be.eu.org
+bg.eu.org
+ca.eu.org
+cd.eu.org
+ch.eu.org
+cn.eu.org
+cy.eu.org
+cz.eu.org
+de.eu.org
+dk.eu.org
+edu.eu.org
+ee.eu.org
+es.eu.org
+fi.eu.org
+fr.eu.org
+gr.eu.org
+hr.eu.org
+hu.eu.org
+ie.eu.org
+il.eu.org
+in.eu.org
+int.eu.org
+is.eu.org
+it.eu.org
+jp.eu.org
+kr.eu.org
+lt.eu.org
+lu.eu.org
+lv.eu.org
+mc.eu.org
+me.eu.org
+mk.eu.org
+mt.eu.org
+my.eu.org
+net.eu.org
+ng.eu.org
+nl.eu.org
+no.eu.org
+nz.eu.org
+paris.eu.org
+pl.eu.org
+pt.eu.org
+q-a.eu.org
+ro.eu.org
+ru.eu.org
+se.eu.org
+si.eu.org
+sk.eu.org
+tr.eu.org
+uk.eu.org
+us.eu.org
+
+// Eurobyte : https://eurobyte.ru
+// Submitted by Evgeniy Subbotin <e.subbotin@eurobyte.ru>
+eurodir.ru
+
+// Evennode : http://www.evennode.com/
+// Submitted by Michal Kralik <support@evennode.com>
+eu-1.evennode.com
+eu-2.evennode.com
+eu-3.evennode.com
+eu-4.evennode.com
+us-1.evennode.com
+us-2.evennode.com
+us-3.evennode.com
+us-4.evennode.com
+
+// eDirect Corp. : https://hosting.url.com.tw/
+// Submitted by C.S. chang <cschang@corp.url.com.tw>
+twmail.cc
+twmail.net
+twmail.org
+mymailer.com.tw
+url.tw
+
+// Fabrica Technologies, Inc. : https://www.fabrica.dev/
+// Submitted by Eric Jiang <eric@fabrica.dev>
+onfabrica.com
+
+// Facebook, Inc.
+// Submitted by Peter Ruibal <public-suffix@fb.com>
+apps.fbsbx.com
+
+// FAITID : https://faitid.org/
+// Submitted by Maxim Alzoba <tech.contact@faitid.org>
+// https://www.flexireg.net/stat_info
+ru.net
+adygeya.ru
+bashkiria.ru
+bir.ru
+cbg.ru
+com.ru
+dagestan.ru
+grozny.ru
+kalmykia.ru
+kustanai.ru
+marine.ru
+mordovia.ru
+msk.ru
+mytis.ru
+nalchik.ru
+nov.ru
+pyatigorsk.ru
+spb.ru
+vladikavkaz.ru
+vladimir.ru
+abkhazia.su
+adygeya.su
+aktyubinsk.su
+arkhangelsk.su
+armenia.su
+ashgabad.su
+azerbaijan.su
+balashov.su
+bashkiria.su
+bryansk.su
+bukhara.su
+chimkent.su
+dagestan.su
+east-kazakhstan.su
+exnet.su
+georgia.su
+grozny.su
+ivanovo.su
+jambyl.su
+kalmykia.su
+kaluga.su
+karacol.su
+karaganda.su
+karelia.su
+khakassia.su
+krasnodar.su
+kurgan.su
+kustanai.su
+lenug.su
+mangyshlak.su
+mordovia.su
+msk.su
+murmansk.su
+nalchik.su
+navoi.su
+north-kazakhstan.su
+nov.su
+obninsk.su
+penza.su
+pokrovsk.su
+sochi.su
+spb.su
+tashkent.su
+termez.su
+togliatti.su
+troitsk.su
+tselinograd.su
+tula.su
+tuva.su
+vladikavkaz.su
+vladimir.su
+vologda.su
+
+// Fancy Bits, LLC : http://getchannels.com
+// Submitted by Aman Gupta <aman@getchannels.com>
+channelsdvr.net
+u.channelsdvr.net
+
+// Fastly Inc. : http://www.fastly.com/
+// Submitted by Fastly Security <security@fastly.com>
+edgecompute.app
+fastly-terrarium.com
+fastlylb.net
+map.fastlylb.net
+freetls.fastly.net
+map.fastly.net
+a.prod.fastly.net
+global.prod.fastly.net
+a.ssl.fastly.net
+b.ssl.fastly.net
+global.ssl.fastly.net
+
+// FASTVPS EESTI OU : https://fastvps.ru/
+// Submitted by Likhachev Vasiliy <lihachev@fastvps.ru>
+fastvps-server.com
+fastvps.host
+myfast.host
+fastvps.site
+myfast.space
+
+// Fedora : https://fedoraproject.org/
+// submitted by Patrick Uiterwijk <puiterwijk@fedoraproject.org>
+fedorainfracloud.org
+fedorapeople.org
+cloud.fedoraproject.org
+app.os.fedoraproject.org
+app.os.stg.fedoraproject.org
+
+// FearWorks Media Ltd. : https://fearworksmedia.co.uk
+// submitted by Keith Fairley <domains@fearworksmedia.co.uk>
+couk.me
+ukco.me
+conn.uk
+copro.uk
+hosp.uk
+
+// Fermax : https://fermax.com/
+// submitted by Koen Van Isterdael <k.vanisterdael@fermax.be>
+mydobiss.com
+
+// FH Muenster : https://www.fh-muenster.de
+// Submitted by Robin Naundorf <r.naundorf@fh-muenster.de>
+fh-muenster.io
+
+// Filegear Inc. : https://www.filegear.com
+// Submitted by Jason Zhu <jason@owtware.com>
+filegear.me
+filegear-au.me
+filegear-de.me
+filegear-gb.me
+filegear-ie.me
+filegear-jp.me
+filegear-sg.me
+
+// Firebase, Inc.
+// Submitted by Chris Raynor <chris@firebase.com>
+firebaseapp.com
+
+// Firewebkit : https://www.firewebkit.com
+// Submitted by Majid Qureshi <mqureshi@amrayn.com>
+fireweb.app
+
+// FLAP : https://www.flap.cloud
+// Submitted by Louis Chemineau <louis@chmn.me>
+flap.id
+
+// fly.io: https://fly.io
+// Submitted by Kurt Mackey <kurt@fly.io>
+fly.dev
+edgeapp.net
+shw.io
+
+// Flynn : https://flynn.io
+// Submitted by Jonathan Rudenberg <jonathan@flynn.io>
+flynnhosting.net
+
+// Forgerock : https://www.forgerock.com
+// Submitted by Roderick Parr <roderick.parr@forgerock.com>
+forgeblocks.com
+id.forgerock.io
+
+// Framer : https://www.framer.com
+// Submitted by Koen Rouwhorst <koenrh@framer.com>
+framer.app
+framercanvas.com
+
+// Frusky MEDIA&PR : https://www.frusky.de
+// Submitted by Victor Pupynin <hallo@frusky.de>
+*.frusky.de
+
+// RavPage : https://www.ravpage.co.il
+// Submitted by Roni Horowitz <roni@responder.co.il>
+ravpage.co.il
+
+// Frederik Braun https://frederik-braun.com
+// Submitted by Frederik Braun <fb@frederik-braun.com>
+0e.vc
+
+// Freebox : http://www.freebox.fr
+// Submitted by Romain Fliedel <rfliedel@freebox.fr>
+freebox-os.com
+freeboxos.com
+fbx-os.fr
+fbxos.fr
+freebox-os.fr
+freeboxos.fr
+
+// freedesktop.org : https://www.freedesktop.org
+// Submitted by Daniel Stone <daniel@fooishbar.org>
+freedesktop.org
+
+// freemyip.com : https://freemyip.com
+// Submitted by Cadence <contact@freemyip.com>
+freemyip.com
+
+// FunkFeuer - Verein zur Förderung freier Netze : https://www.funkfeuer.at
+// Submitted by Daniel A. Maierhofer <vorstand@funkfeuer.at>
+wien.funkfeuer.at
+
+// Futureweb OG : http://www.futureweb.at
+// Submitted by Andreas Schnederle-Wagner <schnederle@futureweb.at>
+*.futurecms.at
+*.ex.futurecms.at
+*.in.futurecms.at
+futurehosting.at
+futuremailing.at
+*.ex.ortsinfo.at
+*.kunden.ortsinfo.at
+*.statics.cloud
+
+// GDS : https://www.gov.uk/service-manual/operations/operating-servicegovuk-subdomains
+// Submitted by David Illsley <david.illsley@digital.cabinet-office.gov.uk>
+service.gov.uk
+
+// Gehirn Inc. : https://www.gehirn.co.jp/
+// Submitted by Kohei YOSHIDA <tech@gehirn.co.jp>
+gehirn.ne.jp
+usercontent.jp
+
+// Gentlent, Inc. : https://www.gentlent.com
+// Submitted by Tom Klein <tom@gentlent.com>
+gentapps.com
+gentlentapis.com
+lab.ms
+cdn-edges.net
+
+// Ghost Foundation : https://ghost.org
+// Submitted by Matt Hanley <security@ghost.org>
+ghost.io
+
+// GignoSystemJapan: http://gsj.bz
+// Submitted by GignoSystemJapan <kakutou-ec@gsj.bz>
+gsj.bz
+
+// GitHub, Inc.
+// Submitted by Patrick Toomey <security@github.com>
+githubusercontent.com
+githubpreview.dev
+github.io
+
+// GitLab, Inc.
+// Submitted by Alex Hanselka <alex@gitlab.com>
+gitlab.io
+
+// Gitplac.si - https://gitplac.si
+// Submitted by Aljaž Starc <me@aljaxus.eu>
+gitapp.si
+gitpage.si
+
+// Glitch, Inc : https://glitch.com
+// Submitted by Mads Hartmann <mads@glitch.com>
+glitch.me
+
+// Global NOG Alliance : https://nogalliance.org/
+// Submitted by Sander Steffann <sander@nogalliance.org>
+nog.community
+
+// Globe Hosting SRL : https://www.globehosting.com/
+// Submitted by Gavin Brown <gavin.brown@centralnic.com>
+co.ro
+shop.ro
+
+// GMO Pepabo, Inc. : https://pepabo.com/
+// Submitted by dojineko <admin@pepabo.com>
+lolipop.io
+
+// GOV.UK Platform as a Service : https://www.cloud.service.gov.uk/
+// Submitted by Tom Whitwell <gov-uk-paas-support@digital.cabinet-office.gov.uk>
+cloudapps.digital
+london.cloudapps.digital
+
+// GOV.UK Pay : https://www.payments.service.gov.uk/
+// Submitted by Richard Baker <richard.baker@digital.cabinet-office.gov.uk>
+pymnt.uk
+
+// UKHomeOffice : https://www.gov.uk/government/organisations/home-office
+// Submitted by Jon Shanks <jon.shanks@digital.homeoffice.gov.uk>
+homeoffice.gov.uk
+
+// GlobeHosting, Inc.
+// Submitted by Zoltan Egresi <egresi@globehosting.com>
+ro.im
+
+// GoIP DNS Services : http://www.goip.de
+// Submitted by Christian Poulter <milchstrasse@goip.de>
+goip.de
+
+// Google, Inc.
+// Submitted by Eduardo Vela <evn@google.com>
+run.app
+a.run.app
+web.app
+*.0emm.com
+appspot.com
+*.r.appspot.com
+codespot.com
+googleapis.com
+googlecode.com
+pagespeedmobilizer.com
+publishproxy.com
+withgoogle.com
+withyoutube.com
+*.gateway.dev
+cloud.goog
+translate.goog
+cloudfunctions.net
+blogspot.ae
+blogspot.al
+blogspot.am
+blogspot.ba
+blogspot.be
+blogspot.bg
+blogspot.bj
+blogspot.ca
+blogspot.cf
+blogspot.ch
+blogspot.cl
+blogspot.co.at
+blogspot.co.id
+blogspot.co.il
+blogspot.co.ke
+blogspot.co.nz
+blogspot.co.uk
+blogspot.co.za
+blogspot.com
+blogspot.com.ar
+blogspot.com.au
+blogspot.com.br
+blogspot.com.by
+blogspot.com.co
+blogspot.com.cy
+blogspot.com.ee
+blogspot.com.eg
+blogspot.com.es
+blogspot.com.mt
+blogspot.com.ng
+blogspot.com.tr
+blogspot.com.uy
+blogspot.cv
+blogspot.cz
+blogspot.de
+blogspot.dk
+blogspot.fi
+blogspot.fr
+blogspot.gr
+blogspot.hk
+blogspot.hr
+blogspot.hu
+blogspot.ie
+blogspot.in
+blogspot.is
+blogspot.it
+blogspot.jp
+blogspot.kr
+blogspot.li
+blogspot.lt
+blogspot.lu
+blogspot.md
+blogspot.mk
+blogspot.mr
+blogspot.mx
+blogspot.my
+blogspot.nl
+blogspot.no
+blogspot.pe
+blogspot.pt
+blogspot.qa
+blogspot.re
+blogspot.ro
+blogspot.rs
+blogspot.ru
+blogspot.se
+blogspot.sg
+blogspot.si
+blogspot.sk
+blogspot.sn
+blogspot.td
+blogspot.tw
+blogspot.ug
+blogspot.vn
+
+// Goupile : https://goupile.fr
+// Submitted by Niels Martignene <hello@goupile.fr>
+goupile.fr
+
+// Group 53, LLC : https://www.group53.com
+// Submitted by Tyler Todd <noc@nova53.net>
+awsmppl.com
+
+// GünstigBestellen : https://günstigbestellen.de
+// Submitted by Furkan Akkoc <info@hendelzon.de>
+xn--gnstigbestellen-zvb.de
+günstigbestellen.de
+xn--gnstigliefern-wob.de
+günstigliefern.de
+
+// Hakaran group: http://hakaran.cz
+// Submited by Arseniy Sokolov <security@hakaran.cz>
+fin.ci
+free.hr
+caa.li
+ua.rs
+conf.se
+
+// Handshake : https://handshake.org
+// Submitted by Mike Damm <md@md.vc>
+hs.zone
+hs.run
+
+// Hashbang : https://hashbang.sh
+hashbang.sh
+
+// Hasura : https://hasura.io
+// Submitted by Shahidh K Muhammed <shahidh@hasura.io>
+hasura.app
+hasura-app.io
+
+// Hepforge : https://www.hepforge.org
+// Submitted by David Grellscheid <admin@hepforge.org>
+hepforge.org
+
+// Heroku : https://www.heroku.com/
+// Submitted by Tom Maher <tmaher@heroku.com>
+herokuapp.com
+herokussl.com
+
+// Hibernating Rhinos
+// Submitted by Oren Eini <oren@ravendb.net>
+myravendb.com
+ravendb.community
+ravendb.me
+development.run
+ravendb.run
+
+// Hong Kong Productivity Council: https://www.hkpc.org/
+// Submitted by SECaaS Team <summchan@hkpc.org>
+secaas.hk
+
+// HOSTBIP REGISTRY : https://www.hostbip.com/
+// Submitted by Atanunu Igbunuroghene <publicsuffixlist@hostbip.com>
+orx.biz
+biz.gl
+col.ng
+firm.ng
+gen.ng
+ltd.ng
+ngo.ng
+edu.scot
+sch.so
+org.yt
+
+// HostyHosting (hostyhosting.com)
+hostyhosting.io
+
+// Häkkinen.fi
+// Submitted by Eero Häkkinen <Eero+psl@Häkkinen.fi>
+xn--hkkinen-5wa.fi
+häkkinen.fi
+
+// Ici la Lune : http://www.icilalune.com/
+// Submitted by Simon Morvan <simon@icilalune.com>
+*.moonscale.io
+moonscale.net
+
+// iki.fi
+// Submitted by Hannu Aronsson <haa@iki.fi>
+iki.fi
+
+// Impertrix Solutions : <https://impertrixcdn.com>
+// Submitted by Zhixiang Zhao <csuite@impertrix.com>
+impertrixcdn.com
+impertrix.com
+
+// Incsub, LLC: https://incsub.com/
+// Submitted by Aaron Edwards <sysadmins@incsub.com>
+smushcdn.com
+wphostedmail.com
+wpmucdn.com
+tempurl.host
+wpmudev.host
+
+// Individual Network Berlin e.V. : https://www.in-berlin.de/
+// Submitted by Christian Seitz <chris@in-berlin.de>
+dyn-berlin.de
+in-berlin.de
+in-brb.de
+in-butter.de
+in-dsl.de
+in-dsl.net
+in-dsl.org
+in-vpn.de
+in-vpn.net
+in-vpn.org
+
+// info.at : http://www.info.at/
+biz.at
+info.at
+
+// info.cx : http://info.cx
+// Submitted by Jacob Slater <whois@igloo.to>
+info.cx
+
+// Interlegis : http://www.interlegis.leg.br
+// Submitted by Gabriel Ferreira <registrobr@interlegis.leg.br>
+ac.leg.br
+al.leg.br
+am.leg.br
+ap.leg.br
+ba.leg.br
+ce.leg.br
+df.leg.br
+es.leg.br
+go.leg.br
+ma.leg.br
+mg.leg.br
+ms.leg.br
+mt.leg.br
+pa.leg.br
+pb.leg.br
+pe.leg.br
+pi.leg.br
+pr.leg.br
+rj.leg.br
+rn.leg.br
+ro.leg.br
+rr.leg.br
+rs.leg.br
+sc.leg.br
+se.leg.br
+sp.leg.br
+to.leg.br
+
+// intermetrics GmbH : https://pixolino.com/
+// Submitted by Wolfgang Schwarz <admin@intermetrics.de>
+pixolino.com
+
+// Internet-Pro, LLP: https://netangels.ru/
+// Submited by Vasiliy Sheredeko <piphon@gmail.com>
+na4u.ru
+
+// iopsys software solutions AB : https://iopsys.eu/
+// Submitted by Roman Azarenko <roman.azarenko@iopsys.eu>
+iopsys.se
+
+// IPiFony Systems, Inc. : https://www.ipifony.com/
+// Submitted by Matthew Hardeman <mhardeman@ipifony.com>
+ipifony.net
+
+// IServ GmbH : https://iserv.eu
+// Submitted by Kim-Alexander Brodowski <info@iserv.eu>
+mein-iserv.de
+schulserver.de
+test-iserv.de
+iserv.dev
+
+// I-O DATA DEVICE, INC. : http://www.iodata.com/
+// Submitted by Yuji Minagawa <domains-admin@iodata.jp>
+iobb.net
+
+// Jelastic, Inc. : https://jelastic.com/
+// Submited by Ihor Kolodyuk <ik@jelastic.com>
+mel.cloudlets.com.au
+cloud.interhostsolutions.be
+users.scale.virtualcloud.com.br
+mycloud.by
+alp1.ae.flow.ch
+appengine.flow.ch
+es-1.axarnet.cloud
+diadem.cloud
+vip.jelastic.cloud
+jele.cloud
+it1.eur.aruba.jenv-aruba.cloud
+it1.jenv-aruba.cloud
+keliweb.cloud
+cs.keliweb.cloud
+oxa.cloud
+tn.oxa.cloud
+uk.oxa.cloud
+primetel.cloud
+uk.primetel.cloud
+ca.reclaim.cloud
+uk.reclaim.cloud
+us.reclaim.cloud
+ch.trendhosting.cloud
+de.trendhosting.cloud
+jele.club
+amscompute.com
+clicketcloud.com
+dopaas.com
+hidora.com
+paas.hosted-by-previder.com
+rag-cloud.hosteur.com
+rag-cloud-ch.hosteur.com
+jcloud.ik-server.com
+jcloud-ver-jpc.ik-server.com
+demo.jelastic.com
+kilatiron.com
+paas.massivegrid.com
+jed.wafaicloud.com
+lon.wafaicloud.com
+ryd.wafaicloud.com
+j.scaleforce.com.cy
+jelastic.dogado.eu
+fi.cloudplatform.fi
+demo.datacenter.fi
+paas.datacenter.fi
+jele.host
+mircloud.host
+paas.beebyte.io
+sekd1.beebyteapp.io
+jele.io
+cloud-fr1.unispace.io
+jc.neen.it
+cloud.jelastic.open.tim.it
+jcloud.kz
+upaas.kazteleport.kz
+cloudjiffy.net
+fra1-de.cloudjiffy.net
+west1-us.cloudjiffy.net
+jls-sto1.elastx.net
+jls-sto2.elastx.net
+jls-sto3.elastx.net
+faststacks.net
+fr-1.paas.massivegrid.net
+lon-1.paas.massivegrid.net
+lon-2.paas.massivegrid.net
+ny-1.paas.massivegrid.net
+ny-2.paas.massivegrid.net
+sg-1.paas.massivegrid.net
+jelastic.saveincloud.net
+nordeste-idc.saveincloud.net
+j.scaleforce.net
+jelastic.tsukaeru.net
+sdscloud.pl
+unicloud.pl
+mircloud.ru
+jelastic.regruhosting.ru
+enscaled.sg
+jele.site
+jelastic.team
+orangecloud.tn
+j.layershift.co.uk
+phx.enscaled.us
+mircloud.us
+
+// Jino : https://www.jino.ru
+// Submitted by Sergey Ulyashin <ulyashin@jino.ru>
+myjino.ru
+*.hosting.myjino.ru
+*.landing.myjino.ru
+*.spectrum.myjino.ru
+*.vps.myjino.ru
+
+// Jotelulu S.L. : https://jotelulu.com
+// Submitted by Daniel Fariña <ingenieria@jotelulu.com>
+jotelulu.cloud
+
+// Joyent : https://www.joyent.com/
+// Submitted by Brian Bennett <brian.bennett@joyent.com>
+*.triton.zone
+*.cns.joyent.com
+
+// JS.ORG : http://dns.js.org
+// Submitted by Stefan Keim <admin@js.org>
+js.org
+
+// KaasHosting : http://www.kaashosting.nl/
+// Submitted by Wouter Bakker <hostmaster@kaashosting.nl>
+kaas.gg
+khplay.nl
+
+// Keyweb AG : https://www.keyweb.de
+// Submitted by Martin Dannehl <postmaster@keymachine.de>
+keymachine.de
+
+// KingHost : https://king.host
+// Submitted by Felipe Keller Braz <felipebraz@kinghost.com.br>
+kinghost.net
+uni5.net
+
+// KnightPoint Systems, LLC : http://www.knightpoint.com/
+// Submitted by Roy Keene <rkeene@knightpoint.com>
+knightpoint.systems
+
+// KUROKU LTD : https://kuroku.ltd/
+// Submitted by DisposaBoy <security@oya.to>
+oya.to
+
+// Katholieke Universiteit Leuven: https://www.kuleuven.be
+// Submitted by Abuse KU Leuven <abuse@kuleuven.be>
+kuleuven.cloud
+ezproxy.kuleuven.be
+
+// .KRD : http://nic.krd/data/krd/Registration%20Policy.pdf
+co.krd
+edu.krd
+
+// Krellian Ltd. : https://krellian.com
+// Submitted by Ben Francis <ben@krellian.com>
+krellian.net
+webthings.io
+
+// LCube - Professional hosting e.K. : https://www.lcube-webhosting.de
+// Submitted by Lars Laehn <info@lcube.de>
+git-repos.de
+lcube-server.de
+svn-repos.de
+
+// Leadpages : https://www.leadpages.net
+// Submitted by Greg Dallavalle <domains@leadpages.net>
+leadpages.co
+lpages.co
+lpusercontent.com
+
+// Lelux.fi : https://lelux.fi/
+// Submitted by Lelux Admin <publisuffix@lelux.site>
+lelux.site
+
+// Lifetime Hosting : https://Lifetime.Hosting/
+// Submitted by Mike Fillator <support@lifetime.hosting>
+co.business
+co.education
+co.events
+co.financial
+co.network
+co.place
+co.technology
+
+// Lightmaker Property Manager, Inc. : https://app.lmpm.com/
+// Submitted by Greg Holland <greg.holland@lmpm.com>
+app.lmpm.com
+
+// linkyard ldt: https://www.linkyard.ch/
+// Submitted by Mario Siegenthaler <mario.siegenthaler@linkyard.ch>
+linkyard.cloud
+linkyard-cloud.ch
+
+// Linode : https://linode.com
+// Submitted by <security@linode.com>
+members.linode.com
+*.nodebalancer.linode.com
+*.linodeobjects.com
+
+// LiquidNet Ltd : http://www.liquidnetlimited.com/
+// Submitted by Victor Velchev <admin@liquidnetlimited.com>
+we.bs
+
+// localzone.xyz
+// Submitted by Kenny Niehage <hello@yahe.sh>
+localzone.xyz
+
+// Log'in Line : https://www.loginline.com/
+// Submitted by Rémi Mach <remi.mach@loginline.com>
+loginline.app
+loginline.dev
+loginline.io
+loginline.services
+loginline.site
+
+// Lõhmus Family, The
+// Submitted by Heiki Lõhmus <hostmaster at lohmus dot me>
+lohmus.me
+
+// LubMAN UMCS Sp. z o.o : https://lubman.pl/
+// Submitted by Ireneusz Maliszewski <ireneusz.maliszewski@lubman.pl>
+krasnik.pl
+leczna.pl
+lubartow.pl
+lublin.pl
+poniatowa.pl
+swidnik.pl
+
+// Lug.org.uk : https://lug.org.uk
+// Submitted by Jon Spriggs <admin@lug.org.uk>
+glug.org.uk
+lug.org.uk
+lugs.org.uk
+
+// Lukanet Ltd : https://lukanet.com
+// Submitted by Anton Avramov <register@lukanet.com>
+barsy.bg
+barsy.co.uk
+barsyonline.co.uk
+barsycenter.com
+barsyonline.com
+barsy.club
+barsy.de
+barsy.eu
+barsy.in
+barsy.info
+barsy.io
+barsy.me
+barsy.menu
+barsy.mobi
+barsy.net
+barsy.online
+barsy.org
+barsy.pro
+barsy.pub
+barsy.shop
+barsy.site
+barsy.support
+barsy.uk
+
+// Magento Commerce
+// Submitted by Damien Tournoud <dtournoud@magento.cloud>
+*.magentosite.cloud
+
+// May First - People Link : https://mayfirst.org/
+// Submitted by Jamie McClelland <info@mayfirst.org>
+mayfirst.info
+mayfirst.org
+
+// Mail.Ru Group : https://hb.cldmail.ru
+// Submitted by Ilya Zaretskiy <zaretskiy@corp.mail.ru>
+hb.cldmail.ru
+
+// Mail Transfer Platform : https://www.neupeer.com
+// Submitted by Li Hui <lihui@neupeer.com>
+cn.vu
+
+// Maze Play: https://www.mazeplay.com
+// Submitted by Adam Humpherys <adam@mws.dev>
+mazeplay.com
+
+// mcpe.me : https://mcpe.me
+// Submitted by Noa Heyl <hi@noa.dev>
+mcpe.me
+
+// McHost : https://mchost.ru
+// Submitted by Evgeniy Subbotin <e.subbotin@mchost.ru>
+mcdir.me
+mcdir.ru
+mcpre.ru
+vps.mcdir.ru
+
+// Mediatech : https://mediatech.by
+// Submitted by Evgeniy Kozhuhovskiy <ugenk@mediatech.by>
+mediatech.by
+mediatech.dev
+
+// Medicom Health : https://medicomhealth.com
+// Submitted by Michael Olson <molson@medicomhealth.com>
+hra.health
+
+// Memset hosting : https://www.memset.com
+// Submitted by Tom Whitwell <domains@memset.com>
+miniserver.com
+memset.net
+
+// MetaCentrum, CESNET z.s.p.o. : https://www.metacentrum.cz/en/
+// Submitted by Zdeněk Šustr <zdenek.sustr@cesnet.cz>
+*.cloud.metacentrum.cz
+custom.metacentrum.cz
+
+// MetaCentrum, CESNET z.s.p.o. : https://www.metacentrum.cz/en/
+// Submitted by Radim JanÄa <janca@cesnet.cz>
+flt.cloud.muni.cz
+usr.cloud.muni.cz
+
+// Meteor Development Group : https://www.meteor.com/hosting
+// Submitted by Pierre Carrier <pierre@meteor.com>
+meteorapp.com
+eu.meteorapp.com
+
+// Michau Enterprises Limited : http://www.co.pl/
+co.pl
+
+// Microsoft Corporation : http://microsoft.com
+// Submitted by Mitch Webster <miwebst@microsoft.com>
+*.azurecontainer.io
+azurewebsites.net
+azure-mobile.net
+cloudapp.net
+azurestaticapps.net
+centralus.azurestaticapps.net
+eastasia.azurestaticapps.net
+eastus2.azurestaticapps.net
+westeurope.azurestaticapps.net
+westus2.azurestaticapps.net
+
+// minion.systems : http://minion.systems
+// Submitted by Robert Böttinger <r@minion.systems>
+csx.cc
+
+// Mintere : https://mintere.com/
+// Submitted by Ben Aubin <security@mintere.com>
+mintere.site
+
+// MobileEducation, LLC : https://joinforte.com
+// Submitted by Grayson Martin <grayson.martin@mobileeducation.us>
+forte.id
+
+// Mozilla Corporation : https://mozilla.com
+// Submitted by Ben Francis <bfrancis@mozilla.com>
+mozilla-iot.org
+
+// Mozilla Foundation : https://mozilla.org/
+// Submitted by glob <glob@mozilla.com>
+bmoattachments.org
+
+// MSK-IX : https://www.msk-ix.ru/
+// Submitted by Khannanov Roman <r.khannanov@msk-ix.ru>
+net.ru
+org.ru
+pp.ru
+
+// Mythic Beasts : https://www.mythic-beasts.com
+// Submitted by Paul Cammish <kelduum@mythic-beasts.com>
+hostedpi.com
+customer.mythic-beasts.com
+caracal.mythic-beasts.com
+fentiger.mythic-beasts.com
+lynx.mythic-beasts.com
+ocelot.mythic-beasts.com
+oncilla.mythic-beasts.com
+onza.mythic-beasts.com
+sphinx.mythic-beasts.com
+vs.mythic-beasts.com
+x.mythic-beasts.com
+yali.mythic-beasts.com
+cust.retrosnub.co.uk
+
+// Nabu Casa : https://www.nabucasa.com
+// Submitted by Paulus Schoutsen <infra@nabucasa.com>
+ui.nabu.casa
+
+// Names.of.London : https://names.of.london/
+// Submitted by James Stevens <registry[at]names.of.london> or <publiclist[at]jrcs.net>
+pony.club
+of.fashion
+in.london
+of.london
+from.marketing
+with.marketing
+for.men
+repair.men
+and.mom
+for.mom
+for.one
+under.one
+for.sale
+that.win
+from.work
+to.work
+
+// NCTU.ME : https://nctu.me/
+// Submitted by Tocknicsu <admin@nctu.me>
+nctu.me
+
+// Netlify : https://www.netlify.com
+// Submitted by Jessica Parsons <jessica@netlify.com>
+netlify.app
+
+// Neustar Inc.
+// Submitted by Trung Tran <Trung.Tran@neustar.biz>
+4u.com
+
+// ngrok : https://ngrok.com/
+// Submitted by Alan Shreve <alan@ngrok.com>
+ngrok.io
+
+// Nimbus Hosting Ltd. : https://www.nimbushosting.co.uk/
+// Submitted by Nicholas Ford <nick@nimbushosting.co.uk>
+nh-serv.co.uk
+
+// NFSN, Inc. : https://www.NearlyFreeSpeech.NET/
+// Submitted by Jeff Wheelhouse <support@nearlyfreespeech.net>
+nfshost.com
+
+// Noop : https://noop.app
+// Submitted by Nathaniel Schweinberg <noop@rearc.io>
+*.developer.app
+noop.app
+
+// Northflank Ltd. : https://northflank.com/
+// Submitted by Marco Suter <marco@northflank.com>
+*.northflank.app
+*.code.run
+
+// Noticeable : https://noticeable.io
+// Submitted by Laurent Pellegrino <security@noticeable.io>
+noticeable.news
+
+// Now-DNS : https://now-dns.com
+// Submitted by Steve Russell <steve@now-dns.com>
+dnsking.ch
+mypi.co
+n4t.co
+001www.com
+ddnslive.com
+myiphost.com
+forumz.info
+16-b.it
+32-b.it
+64-b.it
+soundcast.me
+tcp4.me
+dnsup.net
+hicam.net
+now-dns.net
+ownip.net
+vpndns.net
+dynserv.org
+now-dns.org
+x443.pw
+now-dns.top
+ntdll.top
+freeddns.us
+crafting.xyz
+zapto.xyz
+
+// nsupdate.info : https://www.nsupdate.info/
+// Submitted by Thomas Waldmann <info@nsupdate.info>
+nsupdate.info
+nerdpol.ovh
+
+// No-IP.com : https://noip.com/
+// Submitted by Deven Reza <publicsuffixlist@noip.com>
+blogsyte.com
+brasilia.me
+cable-modem.org
+ciscofreak.com
+collegefan.org
+couchpotatofries.org
+damnserver.com
+ddns.me
+ditchyourip.com
+dnsfor.me
+dnsiskinky.com
+dvrcam.info
+dynns.com
+eating-organic.net
+fantasyleague.cc
+geekgalaxy.com
+golffan.us
+health-carereform.com
+homesecuritymac.com
+homesecuritypc.com
+hopto.me
+ilovecollege.info
+loginto.me
+mlbfan.org
+mmafan.biz
+myactivedirectory.com
+mydissent.net
+myeffect.net
+mymediapc.net
+mypsx.net
+mysecuritycamera.com
+mysecuritycamera.net
+mysecuritycamera.org
+net-freaks.com
+nflfan.org
+nhlfan.net
+no-ip.ca
+no-ip.co.uk
+no-ip.net
+noip.us
+onthewifi.com
+pgafan.net
+point2this.com
+pointto.us
+privatizehealthinsurance.net
+quicksytes.com
+read-books.org
+securitytactics.com
+serveexchange.com
+servehumour.com
+servep2p.com
+servesarcasm.com
+stufftoread.com
+ufcfan.org
+unusualperson.com
+workisboring.com
+3utilities.com
+bounceme.net
+ddns.net
+ddnsking.com
+gotdns.ch
+hopto.org
+myftp.biz
+myftp.org
+myvnc.com
+no-ip.biz
+no-ip.info
+no-ip.org
+noip.me
+redirectme.net
+servebeer.com
+serveblog.net
+servecounterstrike.com
+serveftp.com
+servegame.com
+servehalflife.com
+servehttp.com
+serveirc.com
+serveminecraft.net
+servemp3.com
+servepics.com
+servequake.com
+sytes.net
+webhop.me
+zapto.org
+
+// NodeArt : https://nodeart.io
+// Submitted by Konstantin Nosov <Nosov@nodeart.io>
+stage.nodeart.io
+
+// Nodum B.V. : https://nodum.io/
+// Submitted by Wietse Wind <hello+publicsuffixlist@nodum.io>
+nodum.co
+nodum.io
+
+// Nucleos Inc. : https://nucleos.com
+// Submitted by Piotr Zduniak <piotr@nucleos.com>
+pcloud.host
+
+// NYC.mn : http://www.information.nyc.mn
+// Submitted by Matthew Brown <mattbrown@nyc.mn>
+nyc.mn
+
+// Observable, Inc. : https://observablehq.com
+// Submitted by Mike Bostock <dns@observablehq.com>
+static.observableusercontent.com
+
+// Octopodal Solutions, LLC. : https://ulterius.io/
+// Submitted by Andrew Sampson <andrew@ulterius.io>
+cya.gg
+
+// OMG.LOL : <https://omg.lol>
+// Submitted by Adam Newbold <adam@omg.lol>
+omg.lol
+
+// Omnibond Systems, LLC. : https://www.omnibond.com
+// Submitted by Cole Estep <cole@omnibond.com>
+cloudycluster.net
+
+// OmniWe Limited: https://omniwe.com
+// Submitted by Vicary Archangel <vicary@omniwe.com>
+omniwe.site
+
+// One Fold Media : http://www.onefoldmedia.com/
+// Submitted by Eddie Jones <eddie@onefoldmedia.com>
+nid.io
+
+// Open Social : https://www.getopensocial.com/
+// Submitted by Alexander Varwijk <security@getopensocial.com>
+opensocial.site
+
+// OpenCraft GmbH : http://opencraft.com/
+// Submitted by Sven Marnach <sven@opencraft.com>
+opencraft.hosting
+
+// OpenResearch GmbH: https://openresearch.com/
+// Submitted by Philipp Schmid <ops@openresearch.com>
+orsites.com
+
+// Opera Software, A.S.A.
+// Submitted by Yngve Pettersen <yngve@opera.com>
+operaunite.com
+
+// Oursky Limited : https://authgear.com/, https://skygear.io/
+// Submited by Authgear Team <hello@authgear.com>, Skygear Developer <hello@skygear.io>
+authgear-staging.com
+authgearapps.com
+skygearapp.com
+
+// OutSystems
+// Submitted by Duarte Santos <domain-admin@outsystemscloud.com>
+outsystemscloud.com
+
+// OVHcloud: https://ovhcloud.com
+// Submitted by Vincent Cassé <vincent.casse@ovhcloud.com>
+*.webpaas.ovh.net
+*.hosting.ovh.net
+
+// OwnProvider GmbH: http://www.ownprovider.com
+// Submitted by Jan Moennich <jan.moennich@ownprovider.com>
+ownprovider.com
+own.pm
+
+// OwO : https://whats-th.is/
+// Submitted by Dean Sheather <dean@deansheather.com>
+*.owo.codes
+
+// OX : http://www.ox.rs
+// Submitted by Adam Grand <webmaster@mail.ox.rs>
+ox.rs
+
+// oy.lc
+// Submitted by Charly Coste <changaco@changaco.oy.lc>
+oy.lc
+
+// Pagefog : https://pagefog.com/
+// Submitted by Derek Myers <derek@pagefog.com>
+pgfog.com
+
+// Pagefront : https://www.pagefronthq.com/
+// Submitted by Jason Kriss <jason@pagefronthq.com>
+pagefrontapp.com
+
+// PageXL : https://pagexl.com
+// Submitted by Yann Guichard <yann@pagexl.com>
+pagexl.com
+
+// Paywhirl, Inc : https://paywhirl.com/
+// Submitted by Daniel Netzer <dan@paywhirl.com>
+*.paywhirl.com
+
+// pcarrier.ca Software Inc: https://pcarrier.ca/
+// Submitted by Pierre Carrier <pc@rrier.ca>
+bar0.net
+bar1.net
+bar2.net
+rdv.to
+
+// .pl domains (grandfathered)
+art.pl
+gliwice.pl
+krakow.pl
+poznan.pl
+wroc.pl
+zakopane.pl
+
+// Pantheon Systems, Inc. : https://pantheon.io/
+// Submitted by Gary Dylina <gary@pantheon.io>
+pantheonsite.io
+gotpantheon.com
+
+// Peplink | Pepwave : http://peplink.com/
+// Submitted by Steve Leung <steveleung@peplink.com>
+mypep.link
+
+// Perspecta : https://perspecta.com/
+// Submitted by Kenneth Van Alstyne <kvanalstyne@perspecta.com>
+perspecta.cloud
+
+// PE Ulyanov Kirill Sergeevich : https://airy.host
+// Submitted by Kirill Ulyanov <k.ulyanov@airy.host>
+lk3.ru
+
+// Planet-Work : https://www.planet-work.com/
+// Submitted by Frédéric VANNIÈRE <f.vanniere@planet-work.com>
+on-web.fr
+
+// Platform.sh : https://platform.sh
+// Submitted by Nikola Kotur <nikola@platform.sh>
+bc.platform.sh
+ent.platform.sh
+eu.platform.sh
+us.platform.sh
+*.platformsh.site
+*.tst.site
+
+// Platter: https://platter.dev
+// Submitted by Patrick Flor <patrick@platter.dev>
+platter-app.com
+platter-app.dev
+platterp.us
+
+// Plesk : https://www.plesk.com/
+// Submitted by Anton Akhtyamov <program-managers@plesk.com>
+pdns.page
+plesk.page
+pleskns.com
+
+// Port53 : https://port53.io/
+// Submitted by Maximilian Schieder <maxi@zeug.co>
+dyn53.io
+
+// Positive Codes Technology Company : http://co.bn/faq.html
+// Submitted by Zulfais <pc@co.bn>
+co.bn
+
+// Postman, Inc : https://postman.com
+// Submitted by Rahul Dhawan <security@postman.com>
+postman-echo.com
+pstmn.io
+mock.pstmn.io
+httpbin.org
+
+// prgmr.com : https://prgmr.com/
+// Submitted by Sarah Newman <owner@prgmr.com>
+xen.prgmr.com
+
+// priv.at : http://www.nic.priv.at/
+// Submitted by registry <lendl@nic.at>
+priv.at
+
+// privacytools.io : https://www.privacytools.io/
+// Submitted by Jonah Aragon <jonah@privacytools.io>
+prvcy.page
+
+// Protocol Labs : https://protocol.ai/
+// Submitted by Michael Burns <noc@protocol.ai>
+*.dweb.link
+
+// Protonet GmbH : http://protonet.io
+// Submitted by Martin Meier <admin@protonet.io>
+protonet.io
+
+// Publication Presse Communication SARL : https://ppcom.fr
+// Submitted by Yaacov Akiba Slama <admin@chirurgiens-dentistes-en-france.fr>
+chirurgiens-dentistes-en-france.fr
+byen.site
+
+// pubtls.org: https://www.pubtls.org
+// Submitted by Kor Nielsen <kor@pubtls.org>
+pubtls.org
+
+// PythonAnywhere LLP: https://www.pythonanywhere.com
+// Submitted by Giles Thomas <giles@pythonanywhere.com>
+pythonanywhere.com
+eu.pythonanywhere.com
+
+// QOTO, Org.
+// Submitted by Jeffrey Phillips Freeman <jeffrey.freeman@qoto.org>
+qoto.io
+
+// Qualifio : https://qualifio.com/
+// Submitted by Xavier De Cock <xdecock@gmail.com>
+qualifioapp.com
+
+// QuickBackend: https://www.quickbackend.com
+// Submitted by Dani Biro <dani@pymet.com>
+qbuser.com
+
+// Rad Web Hosting: https://radwebhosting.com
+// Submitted by Scott Claeys <s.claeys@radwebhosting.com>
+cloudsite.builders
+
+// Redstar Consultants : https://www.redstarconsultants.com/
+// Submitted by Jons Slemmer <jons@redstarconsultants.com>
+instantcloud.cn
+
+// Russian Academy of Sciences
+// Submitted by Tech Support <support@rasnet.ru>
+ras.ru
+
+// QA2
+// Submitted by Daniel Dent (https://www.danieldent.com/)
+qa2.com
+
+// QCX
+// Submitted by Cassandra Beelen <cassandra@beelen.one>
+qcx.io
+*.sys.qcx.io
+
+// QNAP System Inc : https://www.qnap.com
+// Submitted by Nick Chang <nickchang@qnap.com>
+dev-myqnapcloud.com
+alpha-myqnapcloud.com
+myqnapcloud.com
+
+// Quip : https://quip.com
+// Submitted by Patrick Linehan <plinehan@quip.com>
+*.quipelements.com
+
+// Qutheory LLC : http://qutheory.io
+// Submitted by Jonas Schwartz <jonas@qutheory.io>
+vapor.cloud
+vaporcloud.io
+
+// Rackmaze LLC : https://www.rackmaze.com
+// Submitted by Kirill Pertsev <kika@rackmaze.com>
+rackmaze.com
+rackmaze.net
+
+// Rakuten Games, Inc : https://dev.viberplay.io
+// Submitted by Joshua Zhang <public-suffix@rgames.jp>
+g.vbrplsbx.io
+
+// Rancher Labs, Inc : https://rancher.com
+// Submitted by Vincent Fiduccia <domains@rancher.com>
+*.on-k3s.io
+*.on-rancher.cloud
+*.on-rio.io
+
+// Read The Docs, Inc : https://www.readthedocs.org
+// Submitted by David Fischer <team@readthedocs.org>
+readthedocs.io
+
+// Red Hat, Inc. OpenShift : https://openshift.redhat.com/
+// Submitted by Tim Kramer <tkramer@rhcloud.com>
+rhcloud.com
+
+// Render : https://render.com
+// Submitted by Anurag Goel <dev@render.com>
+app.render.com
+onrender.com
+
+// Repl.it : https://repl.it
+// Submitted by Mason Clayton <mason@repl.it>
+repl.co
+id.repl.co
+repl.run
+
+// Resin.io : https://resin.io
+// Submitted by Tim Perry <tim@resin.io>
+resindevice.io
+devices.resinstaging.io
+
+// RethinkDB : https://www.rethinkdb.com/
+// Submitted by Chris Kastorff <info@rethinkdb.com>
+hzc.io
+
+// Revitalised Limited : http://www.revitalised.co.uk
+// Submitted by Jack Price <jack@revitalised.co.uk>
+wellbeingzone.eu
+wellbeingzone.co.uk
+
+// Rico Developments Limited : https://adimo.co
+// Submitted by Colin Brown <hello@adimo.co>
+adimo.co.uk
+
+// Riseup Networks : https://riseup.net
+// Submitted by Micah Anderson <micah@riseup.net>
+itcouldbewor.se
+
+// Rochester Institute of Technology : http://www.rit.edu/
+// Submitted by Jennifer Herting <jchits@rit.edu>
+git-pages.rit.edu
+
+// Rusnames Limited: http://rusnames.ru/
+// Submitted by Sergey Zotov <admin@rusnames.ru>
+xn--90amc.xn--p1acf
+биз.руÑ
+xn--j1aef.xn--p1acf
+ком.руÑ
+xn--j1ael8b.xn--p1acf
+крым.руÑ
+xn--h1ahn.xn--p1acf
+мир.руÑ
+xn--j1adp.xn--p1acf
+мÑк.руÑ
+xn--c1avg.xn--p1acf
+орг.руÑ
+xn--80aaa0cvac.xn--p1acf
+Ñамара.руÑ
+xn--h1aliz.xn--p1acf
+Ñочи.руÑ
+xn--90a1af.xn--p1acf
+Ñпб.руÑ
+xn--41a.xn--p1acf
+Ñ.руÑ
+
+// Sandstorm Development Group, Inc. : https://sandcats.io/
+// Submitted by Asheesh Laroia <asheesh@sandstorm.io>
+sandcats.io
+
+// SBE network solutions GmbH : https://www.sbe.de/
+// Submitted by Norman Meilick <nm@sbe.de>
+logoip.de
+logoip.com
+
+// schokokeks.org GbR : https://schokokeks.org/
+// Submitted by Hanno Böck <hanno@schokokeks.org>
+schokokeks.net
+
+// Scottish Government: https://www.gov.scot
+// Submitted by Martin Ellis <martin.ellis@gov.scot>
+gov.scot
+service.gov.scot
+
+// Scry Security : http://www.scrysec.com
+// Submitted by Shante Adam <shante@skyhat.io>
+scrysec.com
+
+// Securepoint GmbH : https://www.securepoint.de
+// Submitted by Erik Anders <erik.anders@securepoint.de>
+firewall-gateway.com
+firewall-gateway.de
+my-gateway.de
+my-router.de
+spdns.de
+spdns.eu
+firewall-gateway.net
+my-firewall.org
+myfirewall.org
+spdns.org
+
+// Seidat : https://www.seidat.com
+// Submitted by Artem Kondratev <accounts@seidat.com>
+seidat.net
+
+// Sellfy : https://sellfy.com
+// Submitted by Yuriy Romadin <contact@sellfy.com>
+sellfy.store
+
+// Senseering GmbH : https://www.senseering.de
+// Submitted by Felix Mönckemeyer <f.moenckemeyer@senseering.de>
+senseering.net
+
+// Sendmsg: https://www.sendmsg.co.il
+// Submitted by Assaf Stern <domains@comstar.co.il>
+minisite.ms
+
+// Service Magnet : https://myservicemagnet.com
+// Submitted by Dave Sanders <dave@myservicemagnet.com>
+magnet.page
+
+// Service Online LLC : http://drs.ua/
+// Submitted by Serhii Bulakh <support@drs.ua>
+biz.ua
+co.ua
+pp.ua
+
+// Shift Crypto AG : https://shiftcrypto.ch
+// Submitted by alex <alex@shiftcrypto.ch>
+shiftcrypto.dev
+shiftcrypto.io
+
+// ShiftEdit : https://shiftedit.net/
+// Submitted by Adam Jimenez <adam@shiftcreate.com>
+shiftedit.io
+
+// Shopblocks : http://www.shopblocks.com/
+// Submitted by Alex Bowers <alex@shopblocks.com>
+myshopblocks.com
+
+// Shopify : https://www.shopify.com
+// Submitted by Alex Richter <alex.richter@shopify.com>
+myshopify.com
+
+// Shopit : https://www.shopitcommerce.com/
+// Submitted by Craig McMahon <craig@shopitcommerce.com>
+shopitsite.com
+
+// shopware AG : https://shopware.com
+// Submitted by Jens Küper <cloud@shopware.com>
+shopware.store
+
+// Siemens Mobility GmbH
+// Submitted by Oliver Graebner <security@mo-siemens.io>
+mo-siemens.io
+
+// SinaAppEngine : http://sae.sina.com.cn/
+// Submitted by SinaAppEngine <saesupport@sinacloud.com>
+1kapp.com
+appchizi.com
+applinzi.com
+sinaapp.com
+vipsinaapp.com
+
+// Siteleaf : https://www.siteleaf.com/
+// Submitted by Skylar Challand <support@siteleaf.com>
+siteleaf.net
+
+// Skyhat : http://www.skyhat.io
+// Submitted by Shante Adam <shante@skyhat.io>
+bounty-full.com
+alpha.bounty-full.com
+beta.bounty-full.com
+
+// Small Technology Foundation : https://small-tech.org
+// Submitted by Aral Balkan <aral@small-tech.org>
+small-web.org
+
+// Snowplow Analytics : https://snowplowanalytics.com/
+// Submitted by Ian Streeter <ian@snowplowanalytics.com>
+try-snowplow.com
+
+// SourceHut : https://sourcehut.org
+// Submitted by Drew DeVault <sir@cmpwn.com>
+srht.site
+
+// Stackhero : https://www.stackhero.io
+// Submitted by Adrien Gillon <adrien+public-suffix-list@stackhero.io>
+stackhero-network.com
+
+// Staclar : https://staclar.com
+// Submitted by Matthias Merkel <matthias.merkel@staclar.com>
+novecore.site
+
+// staticland : https://static.land
+// Submitted by Seth Vincent <sethvincent@gmail.com>
+static.land
+dev.static.land
+sites.static.land
+
+// Storebase : https://www.storebase.io
+// Submitted by Tony Schirmer <tony@storebase.io>
+storebase.store
+
+// Strategic System Consulting (eApps Hosting): https://www.eapps.com/
+// Submitted by Alex Oancea <aoancea@cloudscale365.com>
+vps-host.net
+atl.jelastic.vps-host.net
+njs.jelastic.vps-host.net
+ric.jelastic.vps-host.net
+
+// Sony Interactive Entertainment LLC : https://sie.com/
+// Submitted by David Coles <david.coles@sony.com>
+playstation-cloud.com
+
+// SourceLair PC : https://www.sourcelair.com
+// Submitted by Antonis Kalipetis <akalipetis@sourcelair.com>
+apps.lair.io
+*.stolos.io
+
+// SpaceKit : https://www.spacekit.io/
+// Submitted by Reza Akhavan <spacekit.io@gmail.com>
+spacekit.io
+
+// SpeedPartner GmbH: https://www.speedpartner.de/
+// Submitted by Stefan Neufeind <info@speedpartner.de>
+customer.speedpartner.de
+
+// Spreadshop (sprd.net AG) : https://www.spreadshop.com/
+// Submitted by Martin Breest <security@spreadshop.com>
+myspreadshop.at
+myspreadshop.com.au
+myspreadshop.be
+myspreadshop.ca
+myspreadshop.ch
+myspreadshop.com
+myspreadshop.de
+myspreadshop.dk
+myspreadshop.es
+myspreadshop.fi
+myspreadshop.fr
+myspreadshop.ie
+myspreadshop.it
+myspreadshop.net
+myspreadshop.nl
+myspreadshop.no
+myspreadshop.pl
+myspreadshop.se
+myspreadshop.co.uk
+
+// Standard Library : https://stdlib.com
+// Submitted by Jacob Lee <jacob@stdlib.com>
+api.stdlib.com
+
+// Storj Labs Inc. : https://storj.io/
+// Submitted by Philip Hutchins <hostmaster@storj.io>
+storj.farm
+
+// Studenten Net Twente : http://www.snt.utwente.nl/
+// Submitted by Silke Hofstra <syscom@snt.utwente.nl>
+utwente.io
+
+// Student-Run Computing Facility : https://www.srcf.net/
+// Submitted by Edwin Balani <sysadmins@srcf.net>
+soc.srcf.net
+user.srcf.net
+
+// Sub 6 Limited: http://www.sub6.com
+// Submitted by Dan Miller <dm@sub6.com>
+temp-dns.com
+
+// Supabase : https://supabase.io
+// Submitted by Inian Parameshwaran <security@supabase.io>
+supabase.co
+supabase.in
+supabase.net
+su.paba.se
+
+// Symfony, SAS : https://symfony.com/
+// Submitted by Fabien Potencier <fabien@symfony.com>
+*.s5y.io
+*.sensiosite.cloud
+
+// Syncloud : https://syncloud.org
+// Submitted by Boris Rybalkin <syncloud@syncloud.it>
+syncloud.it
+
+// Synology, Inc. : https://www.synology.com/
+// Submitted by Rony Weng <ronyweng@synology.com>
+diskstation.me
+dscloud.biz
+dscloud.me
+dscloud.mobi
+dsmynas.com
+dsmynas.net
+dsmynas.org
+familyds.com
+familyds.net
+familyds.org
+i234.me
+myds.me
+synology.me
+vpnplus.to
+direct.quickconnect.to
+
+// TAIFUN Software AG : http://taifun-software.de
+// Submitted by Bjoern Henke <dev-server@taifun-software.de>
+taifun-dns.de
+
+// TASK geographical domains (www.task.gda.pl/uslugi/dns)
+gda.pl
+gdansk.pl
+gdynia.pl
+med.pl
+sopot.pl
+
+// Teckids e.V. : https://www.teckids.org
+// Submitted by Dominik George <dominik.george@teckids.org>
+edugit.org
+
+// Telebit : https://telebit.cloud
+// Submitted by AJ ONeal <aj@telebit.cloud>
+telebit.app
+telebit.io
+*.telebit.xyz
+
+// The Gwiddle Foundation : https://gwiddlefoundation.org.uk
+// Submitted by Joshua Bayfield <joshua.bayfield@gwiddlefoundation.org.uk>
+gwiddle.co.uk
+
+// Thingdust AG : https://thingdust.com/
+// Submitted by Adrian Imboden <adi@thingdust.com>
+*.firenet.ch
+*.svc.firenet.ch
+reservd.com
+thingdustdata.com
+cust.dev.thingdust.io
+cust.disrec.thingdust.io
+cust.prod.thingdust.io
+cust.testing.thingdust.io
+reservd.dev.thingdust.io
+reservd.disrec.thingdust.io
+reservd.testing.thingdust.io
+
+// Tlon.io : https://tlon.io
+// Submitted by Mark Staarink <mark@tlon.io>
+arvo.network
+azimuth.network
+tlon.network
+
+// Tor Project, Inc. : https://torproject.org
+// Submitted by Antoine Beaupré <anarcat@torproject.org
+torproject.net
+pages.torproject.net
+
+// TownNews.com : http://www.townnews.com
+// Submitted by Dustin Ward <dward@townnews.com>
+bloxcms.com
+townnews-staging.com
+
+// TradableBits: https://tradablebits.com
+// Submitted by Dmitry Khrisanov dmitry@tradablebits.com
+tbits.me
+
+// TrafficPlex GmbH : https://www.trafficplex.de/
+// Submitted by Phillipp Röll <phillipp.roell@trafficplex.de>
+12hp.at
+2ix.at
+4lima.at
+lima-city.at
+12hp.ch
+2ix.ch
+4lima.ch
+lima-city.ch
+trafficplex.cloud
+de.cool
+12hp.de
+2ix.de
+4lima.de
+lima-city.de
+1337.pictures
+clan.rip
+lima-city.rocks
+webspace.rocks
+lima.zone
+
+// TransIP : https://www.transip.nl
+// Submitted by Rory Breuk <rbreuk@transip.nl>
+*.transurl.be
+*.transurl.eu
+*.transurl.nl
+
+// TuxFamily : http://tuxfamily.org
+// Submitted by TuxFamily administrators <adm@staff.tuxfamily.org>
+tuxfamily.org
+
+// TwoDNS : https://www.twodns.de/
+// Submitted by TwoDNS-Support <support@two-dns.de>
+dd-dns.de
+diskstation.eu
+diskstation.org
+dray-dns.de
+draydns.de
+dyn-vpn.de
+dynvpn.de
+mein-vigor.de
+my-vigor.de
+my-wan.de
+syno-ds.de
+synology-diskstation.de
+synology-ds.de
+
+// Uberspace : https://uberspace.de
+// Submitted by Moritz Werner <mwerner@jonaspasche.com>
+uber.space
+*.uberspace.de
+
+// UDR Limited : http://www.udr.hk.com
+// Submitted by registry <hostmaster@udr.hk.com>
+hk.com
+hk.org
+ltd.hk
+inc.hk
+
+// United Gameserver GmbH : https://united-gameserver.de
+// Submitted by Stefan Schwarz <sysadm@united-gameserver.de>
+virtualuser.de
+virtual-user.de
+
+// urown.net : https://urown.net
+// Submitted by Hostmaster <hostmaster@urown.net>
+urown.cloud
+dnsupdate.info
+
+// .US
+// Submitted by Ed Moore <Ed.Moore@lib.de.us>
+lib.de.us
+
+// VeryPositive SIA : http://very.lv
+// Submitted by Danko Aleksejevs <danko@very.lv>
+2038.io
+
+// Vercel, Inc : https://vercel.com/
+// Submitted by Connor Davis <security@vercel.com>
+vercel.app
+vercel.dev
+now.sh
+
+// Viprinet Europe GmbH : http://www.viprinet.com
+// Submitted by Simon Kissel <hostmaster@viprinet.com>
+router.management
+
+// Virtual-Info : https://www.virtual-info.info/
+// Submitted by Adnan RIHAN <hostmaster@v-info.info>
+v-info.info
+
+// Voorloper.com: https://voorloper.com
+// Submitted by Nathan van Bakel <info@voorloper.com>
+voorloper.cloud
+
+// Voxel.sh DNS : https://voxel.sh/dns/
+// Submitted by Mia Rehlinger <dns@voxel.sh>
+neko.am
+nyaa.am
+be.ax
+cat.ax
+es.ax
+eu.ax
+gg.ax
+mc.ax
+us.ax
+xy.ax
+nl.ci
+xx.gl
+app.gp
+blog.gt
+de.gt
+to.gt
+be.gy
+cc.hn
+blog.kg
+io.kg
+jp.kg
+tv.kg
+uk.kg
+us.kg
+de.ls
+at.md
+de.md
+jp.md
+to.md
+indie.porn
+vxl.sh
+ch.tc
+me.tc
+we.tc
+nyan.to
+at.vg
+blog.vu
+dev.vu
+me.vu
+
+// V.UA Domain Administrator : https://domain.v.ua/
+// Submitted by Serhii Rostilo <sergey@rostilo.kiev.ua>
+v.ua
+
+// Waffle Computer Inc., Ltd. : https://docs.waffleinfo.com
+// Submitted by Masayuki Note <masa@blade.wafflecell.com>
+wafflecell.com
+
+// WapBlog.ID : https://www.wapblog.id
+// Submitted by Fajar Sodik <official@wapblog.id>
+idnblogger.com
+indowapblog.com
+bloger.id
+wblog.id
+wbq.me
+fastblog.net
+
+// WebHare bv: https://www.webhare.com/
+// Submitted by Arnold Hendriks <info@webhare.com>
+*.webhare.dev
+
+// WebHotelier Technologies Ltd: https://www.webhotelier.net/
+// Submitted by Apostolos Tsakpinis <apostolos.tsakpinis@gmail.com>
+reserve-online.net
+reserve-online.com
+bookonline.app
+hotelwithflight.com
+
+// WeDeploy by Liferay, Inc. : https://www.wedeploy.com
+// Submitted by Henrique Vicente <security@wedeploy.com>
+wedeploy.io
+wedeploy.me
+wedeploy.sh
+
+// Western Digital Technologies, Inc : https://www.wdc.com
+// Submitted by Jung Jin <jungseok.jin@wdc.com>
+remotewd.com
+
+// WIARD Enterprises : https://wiardweb.com
+// Submitted by Kidd Hustle <kiddhustle@wiardweb.com>
+pages.wiardweb.com
+
+// Wikimedia Labs : https://wikitech.wikimedia.org
+// Submitted by Arturo Borrero Gonzalez <aborrero@wikimedia.org>
+wmflabs.org
+toolforge.org
+wmcloud.org
+
+// WISP : https://wisp.gg
+// Submitted by Stepan Fedotov <stepan@wisp.gg>
+panel.gg
+daemon.panel.gg
+
+// WoltLab GmbH : https://www.woltlab.com
+// Submitted by Tim Düsterhus <security@woltlab.cloud>
+woltlab-demo.com
+myforum.community
+community-pro.de
+diskussionsbereich.de
+community-pro.net
+meinforum.net
+
+// WP Engine : https://wpengine.com/
+// Submitted by Michael Smith <michael.smith@wpengine.com>
+// Submitted by Brandon DuRette <brandon.durette@wpengine.com>
+wpenginepowered.com
+js.wpenginepowered.com
+
+// Wix.com, Inc. : https://www.wix.com
+// Submitted by Shahar Talmi <shahart@wix.com>
+wixsite.com
+editorx.io
+
+// XenonCloud GbR: https://xenoncloud.net
+// Submitted by Julian Uphoff <publicsuffixlist@xenoncloud.net>
+half.host
+
+// XnBay Technology : http://www.xnbay.com/
+// Submitted by XnBay Developer <developer.xncloud@gmail.com>
+xnbay.com
+u2.xnbay.com
+u2-local.xnbay.com
+
+// XS4ALL Internet bv : https://www.xs4all.nl/
+// Submitted by Daniel Mostertman <unixbeheer+publicsuffix@xs4all.net>
+cistron.nl
+demon.nl
+xs4all.space
+
+// Yandex.Cloud LLC: https://cloud.yandex.com
+// Submitted by Alexander Lodin <security+psl@yandex-team.ru>
+yandexcloud.net
+storage.yandexcloud.net
+website.yandexcloud.net
+
+// YesCourse Pty Ltd : https://yescourse.com
+// Submitted by Atul Bhouraskar <atul@yescourse.com>
+official.academy
+
+// Yola : https://www.yola.com/
+// Submitted by Stefano Rivera <stefano@yola.com>
+yolasite.com
+
+// Yombo : https://yombo.net
+// Submitted by Mitch Schwenk <mitch@yombo.net>
+ybo.faith
+yombo.me
+homelink.one
+ybo.party
+ybo.review
+ybo.science
+ybo.trade
+
+// Yunohost : https://yunohost.org
+// Submitted by Valentin Grimaud <security@yunohost.org>
+ynh.fr
+nohost.me
+noho.st
+
+// ZaNiC : http://www.za.net/
+// Submitted by registry <hostmaster@nic.za.net>
+za.net
+za.org
+
+// Zine EOOD : https://zine.bg/
+// Submitted by Martin Angelov <martin@zine.bg>
+bss.design
+
+// Zitcom A/S : https://www.zitcom.dk
+// Submitted by Emil Stahl <esp@zitcom.dk>
+basicserver.io
+virtualserver.io
+enterprisecloud.nu
+
+// ===END PRIVATE DOMAINS===
diff --git a/contrib/publicsuffix/idn.pl b/contrib/publicsuffix/idn.pl
new file mode 100644
index 0000000..dd46d65
--- /dev/null
+++ b/contrib/publicsuffix/idn.pl
@@ -0,0 +1,20 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+use Net::IDN::Encode ':all';
+use Unicode::Normalize;
+
+binmode(STDOUT, ":utf8");
+binmode(STDIN, ":utf8");
+
+while (<>) {
+ $_ = NFC($_);
+ if (/^[^\/].*[^\x00-\x7F]+.*/) {
+ chomp;
+ printf "%s\n", domain_to_ascii($_);
+ $_ .= "\n";
+ }
+} continue {
+ print $_;
+} \ No newline at end of file
diff --git a/contrib/replxx/CMakeLists.txt b/contrib/replxx/CMakeLists.txt
new file mode 100644
index 0000000..f48a068
--- /dev/null
+++ b/contrib/replxx/CMakeLists.txt
@@ -0,0 +1,82 @@
+# -*- mode: CMAKE; -*-
+project( replxx VERSION 0.0.2 LANGUAGES CXX C )
+
+# INFO
+set(REPLXX_DISPLAY_NAME "replxx")
+set(REPLXX_URL_INFO_ABOUT "https://github.com/AmokHuginnsson/replxx")
+set(REPLXX_CONTACT "amok@codestation.org")
+set(REPLXX_FRIENDLY_STRING "replxx - Read Evaluate Print Loop library")
+
+# compiler options
+if(CMAKE_COMPILER_IS_GNUCXX)
+ message(STATUS "Compiler type GNU: ${CMAKE_CXX_COMPILER}")
+ set(BASE_COMPILER_OPTIONS "-std=c++11 -Wall -D_GNU_SOURCE -pthread")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${BASE_COMPILER_OPTIONS}")
+ set(CMAKE_CXX_FLAGS_COVERAGE "${CMAKE_CXX_FLAGS} ${BASE_COMPILER_OPTIONS} -O0 --coverage -fno-inline -fno-default-inline -fno-inline-small-functions")
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${BASE_COMPILER_OPTIONS} -O0 -g -ggdb -g3 -ggdb3")
+ set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} ${BASE_COMPILER_OPTIONS} -Os")
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${BASE_COMPILER_OPTIONS} -O3 -fomit-frame-pointer")
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${BASE_COMPILER_OPTIONS} -O3 -g")
+ set(CMAKE_C_FLAGS "-std=c99")
+elseif(CMAKE_COMPILER_IS_CLANGCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+ # using regular Clang or AppleClang
+ message(STATUS "Compiler type CLANG: ${CMAKE_CXX_COMPILER}")
+ set(BASE_COMPILER_OPTIONS "-std=c++11 -Wall -Wextra -D_GNU_SOURCE -pthread")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${BASE_COMPILER_OPTIONS}")
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${BASE_COMPILER_OPTIONS} -O0 -g")
+ set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} ${BASE_COMPILER_OPTIONS} -Os")
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${BASE_COMPILER_OPTIONS} -O3 -fomit-frame-pointer")
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${BASE_COMPILER_OPTIONS} -O3 -g")
+ set(CMAKE_C_FLAGS "-std=c99")
+elseif(MSVC)
+ message(STATUS "Compiler type MSVC: ${CMAKE_CXX_COMPILER}")
+ add_definitions("-D_CRT_SECURE_NO_WARNINGS=1")
+
+ set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} /INCREMENTAL:NO /SUBSYSTEM:CONSOLE /LTCG /ignore:4099")
+ set(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL "${CMAKE_EXE_LINKER_FLAGS_MINSIZEREL} /SUBSYSTEM:CONSOLE /ignore:4099")
+ set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /SUBSYSTEM:CONSOLE /ignore:4099")
+ set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} /SUBSYSTEM:CONSOLE /ignore:4099")
+else()
+ # unknown compiler
+ message(STATUS "Compiler type UNKNOWN: ${CMAKE_CXX_COMPILER}")
+ set(BASE_COMPILER_OPTIONS "-std=c++11 -Wall -pthread")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${BASE_COMPILER_OPTIONS}")
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${BASE_COMPILER_OPTIONS} -O0 -g")
+ set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} ${BASE_COMPILER_OPTIONS} -Os")
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${BASE_COMPILER_OPTIONS} -O3 -fomit-frame-pointer")
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${BASE_COMPILER_OPTIONS} -O3 -g")
+ set(CMAKE_C_FLAGS "-std=c99")
+endif()
+
+# build libreplxx
+set(
+ REPLXX_SOURCES
+ src/conversion.cxx
+ src/escape.cxx
+ src/history.cxx
+ src/replxx_impl.cxx
+ src/prompt.cxx
+ src/replxx.cxx
+ src/util.cxx
+ src/wcwidth.cpp
+ src/terminal.cxx
+ src/windows.cxx
+)
+
+set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+if(ENABLE_STATIC MATCHES "ON")
+ add_library(rspamd-replxx STATIC ${REPLXX_SOURCES})
+else()
+ add_library(rspamd-replxx SHARED ${REPLXX_SOURCES})
+endif()
+
+target_include_directories(
+ rspamd-replxx
+ PUBLIC ${PROJECT_SOURCE_DIR}/include
+ PRIVATE ${PROJECT_SOURCE_DIR}/src
+)
+set( TARGETS ${TARGETS} rspamd-replxx )
+target_compile_definitions(rspamd-replxx PRIVATE REPLXX_BUILDING_DLL)
+target_link_libraries(rspamd-replxx "${RSPAMD_REQUIRED_LIBRARIES}")
+
+install( TARGETS ${TARGETS} LIBRARY DESTINATION ${RSPAMD_LIBDIR}) \ No newline at end of file
diff --git a/contrib/replxx/LICENSE.md b/contrib/replxx/LICENSE.md
new file mode 100644
index 0000000..c73c3f2
--- /dev/null
+++ b/contrib/replxx/LICENSE.md
@@ -0,0 +1,63 @@
+Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
+Copyright (c) 2010, Salvatore Sanfilippo (antirez at gmail dot com)
+Copyright (c) 2010, Pieter Noordhuis (pcnoordhuis at gmail dot com)
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of Redis nor the names of its contributors may be used
+ to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+
+wcwidth.cpp
+===========
+
+Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+
+Permission to use, copy, modify, and distribute this software
+for any purpose and without fee is hereby granted. The author
+disclaims all warranties with regard to this software.
+
+
+ConvertUTF.cpp
+==============
+
+Copyright 2001-2004 Unicode, Inc.
+
+Disclaimer
+
+This source code is provided as is by Unicode, Inc. No claims are
+made as to fitness for any particular purpose. No warranties of any
+kind are expressed or implied. The recipient agrees to determine
+applicability of information provided. If this file has been
+purchased on magnetic or optical media from Unicode, Inc., the
+sole remedy for any claim will be exchange of defective media
+within 90 days of receipt.
+
+Limitations on Rights to Redistribute This Code
+
+Unicode, Inc. hereby grants the right to freely use the information
+supplied in this file in the creation of products supporting the
+Unicode Standard, and to make copies of this file in any form
+for internal or external distribution as long as this notice
+remains attached.
diff --git a/contrib/replxx/README.md b/contrib/replxx/README.md
new file mode 100644
index 0000000..21c907a
--- /dev/null
+++ b/contrib/replxx/README.md
@@ -0,0 +1,119 @@
+# Read Evaluate Print Loop ++
+
+![demo](https://drive.google.com/uc?export=download&id=0B53g2Y3z7rWNT2dCRGVVNldaRnc)
+
+[![Build Status](https://travis-ci.org/AmokHuginnsson/replxx.svg?branch=master)](https://travis-ci.org/AmokHuginnsson/replxx)
+
+A small, portable GNU readline replacement for Linux, Windows and
+MacOS which is capable of handling UTF-8 characters. Unlike GNU
+readline, which is GPL, this library uses a BSD license and can be
+used in any kind of program.
+
+## Origin
+
+This replxx implementation is based on the work by
+[ArangoDB Team](https://github.com/arangodb/linenoise-ng) and
+[Salvatore Sanfilippo](https://github.com/antirez/linenoise) and
+10gen Inc. The goal is to create a zero-config, BSD
+licensed, readline replacement usable in Apache2 or BSD licensed
+programs.
+
+## Features
+
+* single-line and multi-line editing mode with the usual key bindings implemented
+* history handling
+* completion
+* syntax highlighting
+* hints
+* BSD license source code
+* Only uses a subset of VT100 escapes (ANSI.SYS compatible)
+* UTF8 aware
+* support for Linux, MacOS and Windows
+
+It deviates from Salvatore's original goal to have a minimal readline
+replacement for the sake of supporting UTF8 and Windows. It deviates
+from 10gen Inc.'s goal to create a C++ interface to linenoise. This
+library uses C++ internally, but to the user it provides a pure C
+interface that is compatible with the original linenoise API.
+C interface.
+
+## Requirements
+
+To build this library, you will need a C++11-enabled compiler and
+some recent version of CMake.
+
+## Build instructions
+
+### *nix
+
+1. Create a build directory
+
+```bash
+mkdir -p build && cd build
+```
+
+2. Build the library
+
+```bash
+cmake -DCMAKE_BUILD_TYPE=Release .. && make
+```
+
+3. Install the library at the default target location
+
+```bash
+sudo make install
+```
+
+The default installation location can be adjusted by setting the `DESTDIR`
+variable when invoking `make install`:
+
+```bash
+make DESTDIR=/tmp install
+```
+
+### Windows
+
+1. Create a build directory in MS-DOS command prompt
+
+```
+md build
+cd build
+```
+
+2. Generate Visual Studio solution file with cmake
+
+* 32 bit:
+```bash
+cmake -G "Visual Studio 12 2013" -DCMAKE_BUILD_TYPE=Release ..`
+```
+* 64 bit:
+```bash
+`cmake -G "Visual Studio 12 2013 Win64" -DCMAKE_BUILD_TYPE=Release ..`
+```
+
+3. Open the generated file `replxx.sln` in the `build` subdirectory with Visual Studio.
+
+## Tested with...
+
+ * Linux text only console ($TERM = linux)
+ * Linux KDE terminal application ($TERM = xterm)
+ * Linux xterm ($TERM = xterm)
+ * Linux Buildroot ($TERM = vt100)
+ * Mac OS X iTerm ($TERM = xterm)
+ * Mac OS X default Terminal.app ($TERM = xterm)
+ * OpenBSD 4.5 through an OSX Terminal.app ($TERM = screen)
+ * IBM AIX 6.1
+ * FreeBSD xterm ($TERM = xterm)
+ * ANSI.SYS
+ * Emacs comint mode ($TERM = dumb)
+ * Windows
+
+Please test it everywhere you can and report back!
+
+## Let's push this forward!
+
+Patches should be provided in the respect of linenoise sensibility for
+small and easy to understand code that and the license
+restrictions. Extensions must be submitted under a BSD license-style.
+A contributor license is required for contributions.
+
diff --git a/contrib/replxx/include/replxx.h b/contrib/replxx/include/replxx.h
new file mode 100644
index 0000000..5127ac2
--- /dev/null
+++ b/contrib/replxx/include/replxx.h
@@ -0,0 +1,575 @@
+/* linenoise.h -- guerrilla line editing library against the idea that a
+ * line editing lib needs to be 20,000 lines of C code.
+ *
+ * See linenoise.c for more information.
+ *
+ * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __REPLXX_H
+#define __REPLXX_H
+
+#define REPLXX_VERSION "0.0.2"
+#define REPLXX_VERSION_MAJOR 0
+#define REPLXX_VERSION_MINOR 0
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * For use in Windows DLLs:
+ *
+ * If you are building replxx into a DLL,
+ * unless you are using supplied CMake based build,
+ * ensure that 'REPLXX_BUILDING_DLL' is defined when
+ * building the DLL so that proper symbols are exported.
+ */
+#if defined( _WIN32 ) && ! defined( REPLXX_STATIC )
+# ifdef REPLXX_BUILDING_DLL
+# define REPLXX_IMPEXP __declspec( dllexport )
+# else
+# define REPLXX_IMPEXP __declspec( dllimport )
+# endif
+#else
+# define REPLXX_IMPEXP /**/
+#endif
+
+/*! \brief Color definitions to use in highlighter callbacks.
+ */
+typedef enum {
+ REPLXX_COLOR_BLACK = 0,
+ REPLXX_COLOR_RED = 1,
+ REPLXX_COLOR_GREEN = 2,
+ REPLXX_COLOR_BROWN = 3,
+ REPLXX_COLOR_BLUE = 4,
+ REPLXX_COLOR_MAGENTA = 5,
+ REPLXX_COLOR_CYAN = 6,
+ REPLXX_COLOR_LIGHTGRAY = 7,
+ REPLXX_COLOR_GRAY = 8,
+ REPLXX_COLOR_BRIGHTRED = 9,
+ REPLXX_COLOR_BRIGHTGREEN = 10,
+ REPLXX_COLOR_YELLOW = 11,
+ REPLXX_COLOR_BRIGHTBLUE = 12,
+ REPLXX_COLOR_BRIGHTMAGENTA = 13,
+ REPLXX_COLOR_BRIGHTCYAN = 14,
+ REPLXX_COLOR_WHITE = 15,
+ REPLXX_COLOR_NORMAL = REPLXX_COLOR_LIGHTGRAY,
+ REPLXX_COLOR_DEFAULT = -1,
+ REPLXX_COLOR_ERROR = -2
+} ReplxxColor;
+
+enum { REPLXX_KEY_BASE = 0x0010ffff + 1 };
+enum { REPLXX_KEY_BASE_SHIFT = 0x01000000 };
+enum { REPLXX_KEY_BASE_CONTROL = 0x02000000 };
+enum { REPLXX_KEY_BASE_META = 0x04000000 };
+enum { REPLXX_KEY_ESCAPE = 27 };
+enum { REPLXX_KEY_PAGE_UP = REPLXX_KEY_BASE + 1 };
+enum { REPLXX_KEY_PAGE_DOWN = REPLXX_KEY_PAGE_UP + 1 };
+enum { REPLXX_KEY_DOWN = REPLXX_KEY_PAGE_DOWN + 1 };
+enum { REPLXX_KEY_UP = REPLXX_KEY_DOWN + 1 };
+enum { REPLXX_KEY_LEFT = REPLXX_KEY_UP + 1 };
+enum { REPLXX_KEY_RIGHT = REPLXX_KEY_LEFT + 1 };
+enum { REPLXX_KEY_HOME = REPLXX_KEY_RIGHT + 1 };
+enum { REPLXX_KEY_END = REPLXX_KEY_HOME + 1 };
+enum { REPLXX_KEY_DELETE = REPLXX_KEY_END + 1 };
+enum { REPLXX_KEY_INSERT = REPLXX_KEY_DELETE + 1 };
+enum { REPLXX_KEY_F1 = REPLXX_KEY_INSERT + 1 };
+enum { REPLXX_KEY_F2 = REPLXX_KEY_F1 + 1 };
+enum { REPLXX_KEY_F3 = REPLXX_KEY_F2 + 1 };
+enum { REPLXX_KEY_F4 = REPLXX_KEY_F3 + 1 };
+enum { REPLXX_KEY_F5 = REPLXX_KEY_F4 + 1 };
+enum { REPLXX_KEY_F6 = REPLXX_KEY_F5 + 1 };
+enum { REPLXX_KEY_F7 = REPLXX_KEY_F6 + 1 };
+enum { REPLXX_KEY_F8 = REPLXX_KEY_F7 + 1 };
+enum { REPLXX_KEY_F9 = REPLXX_KEY_F8 + 1 };
+enum { REPLXX_KEY_F10 = REPLXX_KEY_F9 + 1 };
+enum { REPLXX_KEY_F11 = REPLXX_KEY_F10 + 1 };
+enum { REPLXX_KEY_F12 = REPLXX_KEY_F11 + 1 };
+enum { REPLXX_KEY_F13 = REPLXX_KEY_F12 + 1 };
+enum { REPLXX_KEY_F14 = REPLXX_KEY_F13 + 1 };
+enum { REPLXX_KEY_F15 = REPLXX_KEY_F14 + 1 };
+enum { REPLXX_KEY_F16 = REPLXX_KEY_F15 + 1 };
+enum { REPLXX_KEY_F17 = REPLXX_KEY_F16 + 1 };
+enum { REPLXX_KEY_F18 = REPLXX_KEY_F17 + 1 };
+enum { REPLXX_KEY_F19 = REPLXX_KEY_F18 + 1 };
+enum { REPLXX_KEY_F20 = REPLXX_KEY_F19 + 1 };
+enum { REPLXX_KEY_F21 = REPLXX_KEY_F20 + 1 };
+enum { REPLXX_KEY_F22 = REPLXX_KEY_F21 + 1 };
+enum { REPLXX_KEY_F23 = REPLXX_KEY_F22 + 1 };
+enum { REPLXX_KEY_F24 = REPLXX_KEY_F23 + 1 };
+enum { REPLXX_KEY_MOUSE = REPLXX_KEY_F24 + 1 };
+enum { REPLXX_KEY_PASTE_START = REPLXX_KEY_MOUSE + 1 };
+enum { REPLXX_KEY_PASTE_FINISH = REPLXX_KEY_PASTE_START + 1 };
+
+#define REPLXX_KEY_SHIFT( key ) ( ( key ) | REPLXX_KEY_BASE_SHIFT )
+#define REPLXX_KEY_CONTROL( key ) ( ( key ) | REPLXX_KEY_BASE_CONTROL )
+#define REPLXX_KEY_META( key ) ( ( key ) | REPLXX_KEY_BASE_META )
+
+enum { REPLXX_KEY_BACKSPACE = REPLXX_KEY_CONTROL( 'H' ) };
+enum { REPLXX_KEY_TAB = REPLXX_KEY_CONTROL( 'I' ) };
+enum { REPLXX_KEY_ENTER = REPLXX_KEY_CONTROL( 'M' ) };
+
+/*! \brief List of built-in actions that act upon user input.
+ */
+typedef enum {
+ REPLXX_ACTION_INSERT_CHARACTER,
+ REPLXX_ACTION_NEW_LINE,
+ REPLXX_ACTION_DELETE_CHARACTER_UNDER_CURSOR,
+ REPLXX_ACTION_DELETE_CHARACTER_LEFT_OF_CURSOR,
+ REPLXX_ACTION_KILL_TO_END_OF_LINE,
+ REPLXX_ACTION_KILL_TO_BEGINING_OF_LINE,
+ REPLXX_ACTION_KILL_TO_END_OF_WORD,
+ REPLXX_ACTION_KILL_TO_BEGINING_OF_WORD,
+ REPLXX_ACTION_KILL_TO_END_OF_SUBWORD,
+ REPLXX_ACTION_KILL_TO_BEGINING_OF_SUBWORD,
+ REPLXX_ACTION_KILL_TO_WHITESPACE_ON_LEFT,
+ REPLXX_ACTION_YANK,
+ REPLXX_ACTION_YANK_CYCLE,
+ REPLXX_ACTION_YANK_LAST_ARG,
+ REPLXX_ACTION_MOVE_CURSOR_TO_BEGINING_OF_LINE,
+ REPLXX_ACTION_MOVE_CURSOR_TO_END_OF_LINE,
+ REPLXX_ACTION_MOVE_CURSOR_ONE_WORD_LEFT,
+ REPLXX_ACTION_MOVE_CURSOR_ONE_WORD_RIGHT,
+ REPLXX_ACTION_MOVE_CURSOR_ONE_SUBWORD_LEFT,
+ REPLXX_ACTION_MOVE_CURSOR_ONE_SUBWORD_RIGHT,
+ REPLXX_ACTION_MOVE_CURSOR_LEFT,
+ REPLXX_ACTION_MOVE_CURSOR_RIGHT,
+ REPLXX_ACTION_HISTORY_NEXT,
+ REPLXX_ACTION_HISTORY_PREVIOUS,
+ REPLXX_ACTION_HISTORY_FIRST,
+ REPLXX_ACTION_HISTORY_LAST,
+ REPLXX_ACTION_HISTORY_INCREMENTAL_SEARCH,
+ REPLXX_ACTION_HISTORY_COMMON_PREFIX_SEARCH,
+ REPLXX_ACTION_HINT_NEXT,
+ REPLXX_ACTION_HINT_PREVIOUS,
+ REPLXX_ACTION_CAPITALIZE_WORD,
+ REPLXX_ACTION_LOWERCASE_WORD,
+ REPLXX_ACTION_UPPERCASE_WORD,
+ REPLXX_ACTION_CAPITALIZE_SUBWORD,
+ REPLXX_ACTION_LOWERCASE_SUBWORD,
+ REPLXX_ACTION_UPPERCASE_SUBWORD,
+ REPLXX_ACTION_TRANSPOSE_CHARACTERS,
+ REPLXX_ACTION_TOGGLE_OVERWRITE_MODE,
+#ifndef _WIN32
+ REPLXX_ACTION_VERBATIM_INSERT,
+ REPLXX_ACTION_SUSPEND,
+#endif
+ REPLXX_ACTION_BRACKETED_PASTE,
+ REPLXX_ACTION_CLEAR_SCREEN,
+ REPLXX_ACTION_CLEAR_SELF,
+ REPLXX_ACTION_REPAINT,
+ REPLXX_ACTION_COMPLETE_LINE,
+ REPLXX_ACTION_COMPLETE_NEXT,
+ REPLXX_ACTION_COMPLETE_PREVIOUS,
+ REPLXX_ACTION_COMMIT_LINE,
+ REPLXX_ACTION_ABORT_LINE,
+ REPLXX_ACTION_SEND_EOF
+} ReplxxAction;
+
+/*! \brief Possible results of key-press handler actions.
+ */
+typedef enum {
+ REPLXX_ACTION_RESULT_CONTINUE, /*!< Continue processing user input. */
+ REPLXX_ACTION_RESULT_RETURN, /*!< Return user input entered so far. */
+ REPLXX_ACTION_RESULT_BAIL /*!< Stop processing user input, returns nullptr from the \e input() call. */
+} ReplxxActionResult;
+
+typedef struct ReplxxStateTag {
+ char const* text;
+ int cursorPosition;
+} ReplxxState;
+
+typedef struct Replxx Replxx;
+typedef struct ReplxxHistoryScan ReplxxHistoryScan;
+typedef struct ReplxxHistoryEntryTag {
+ char const* timestamp;
+ char const* text;
+} ReplxxHistoryEntry;
+
+/*! \brief Create Replxx library resource holder.
+ *
+ * Use replxx_end() to free resources acquired with this function.
+ *
+ * \return Replxx library resource holder.
+ */
+REPLXX_IMPEXP Replxx* replxx_init( void );
+
+/*! \brief Cleanup resources used by Replxx library.
+ *
+ * \param replxx - a Replxx library resource holder.
+ */
+REPLXX_IMPEXP void replxx_end( Replxx* replxx );
+
+/*! \brief Line modification callback type definition.
+ *
+ * User can observe and modify line contents (and cursor position)
+ * in response to changes to both introduced by the user through
+ * normal interactions.
+ *
+ * When callback returns Replxx updates current line content
+ * and current cursor position to the ones updated by the callback.
+ *
+ * \param line[in,out] - a R/W reference to an UTF-8 encoded input entered by the user so far.
+ * \param cursorPosition[in,out] - a R/W reference to current cursor position.
+ * \param userData - pointer to opaque user data block.
+ */
+typedef void (replxx_modify_callback_t)(char** input, int* contextLen, void* userData);
+
+/*! \brief Register modify callback.
+ *
+ * \param fn - user defined callback function.
+ * \param userData - pointer to opaque user data block to be passed into each invocation of the callback.
+ */
+REPLXX_IMPEXP void replxx_set_modify_callback( Replxx*, replxx_modify_callback_t* fn, void* userData );
+
+/*! \brief Highlighter callback type definition.
+ *
+ * If user want to have colorful input she must simply install highlighter callback.
+ * The callback would be invoked by the library after each change to the input done by
+ * the user. After callback returns library uses data from colors buffer to colorize
+ * displayed user input.
+ *
+ * \e size of \e colors buffer is equal to number of code points in user \e input
+ * which will be different from simple `strlen( input )`!
+ *
+ * \param input - an UTF-8 encoded input entered by the user so far.
+ * \param colors - output buffer for color information.
+ * \param size - size of output buffer for color information.
+ * \param userData - pointer to opaque user data block.
+ */
+typedef void (replxx_highlighter_callback_t)(char const* input, ReplxxColor* colors, int size, void* userData);
+
+/*! \brief Register highlighter callback.
+ *
+ * \param fn - user defined callback function.
+ * \param userData - pointer to opaque user data block to be passed into each invocation of the callback.
+ */
+REPLXX_IMPEXP void replxx_set_highlighter_callback( Replxx*, replxx_highlighter_callback_t* fn, void* userData );
+
+typedef struct replxx_completions replxx_completions;
+
+/*! \brief Completions callback type definition.
+ *
+ * \e contextLen is counted in Unicode code points (not in bytes!).
+ *
+ * For user input:
+ * if ( obj.me
+ *
+ * input == "if ( obj.me"
+ * contextLen == 2 (depending on \e replxx_set_word_break_characters())
+ *
+ * Client application is free to update \e contextLen to be 6 (or any other non-negative
+ * number not greater than the number of code points in input) if it makes better sense
+ * for given client application semantics.
+ *
+ * \param input - UTF-8 encoded input entered by the user until current cursor position.
+ * \param completions - pointer to opaque list of user completions.
+ * \param contextLen[in,out] - length of the additional context to provide while displaying completions.
+ * \param userData - pointer to opaque user data block.
+ */
+typedef void(replxx_completion_callback_t)(const char* input, replxx_completions* completions, int* contextLen, void* userData);
+
+/*! \brief Register completion callback.
+ *
+ * \param fn - user defined callback function.
+ * \param userData - pointer to opaque user data block to be passed into each invocation of the callback.
+ */
+REPLXX_IMPEXP void replxx_set_completion_callback( Replxx*, replxx_completion_callback_t* fn, void* userData );
+
+/*! \brief Add another possible completion for current user input.
+ *
+ * \param completions - pointer to opaque list of user completions.
+ * \param str - UTF-8 encoded completion string.
+ */
+REPLXX_IMPEXP void replxx_add_completion( replxx_completions* completions, const char* str );
+
+/*! \brief Add another possible completion for current user input.
+ *
+ * \param completions - pointer to opaque list of user completions.
+ * \param str - UTF-8 encoded completion string.
+ * \param color - a color for the completion.
+ */
+REPLXX_IMPEXP void replxx_add_color_completion( replxx_completions* completions, const char* str, ReplxxColor color );
+
+typedef struct replxx_hints replxx_hints;
+
+/*! \brief Hints callback type definition.
+ *
+ * \e contextLen is counted in Unicode code points (not in bytes!).
+ *
+ * For user input:
+ * if ( obj.me
+ *
+ * input == "if ( obj.me"
+ * contextLen == 2 (depending on \e replxx_set_word_break_characters())
+ *
+ * Client application is free to update \e contextLen to be 6 (or any other non-negative
+ * number not greater than the number of code points in input) if it makes better sense
+ * for given client application semantics.
+ *
+ * \param input - UTF-8 encoded input entered by the user until current cursor position.
+ * \param hints - pointer to opaque list of possible hints.
+ * \param contextLen[in,out] - length of the additional context to provide while displaying hints.
+ * \param color - a color used for displaying hints.
+ * \param userData - pointer to opaque user data block.
+ */
+typedef void(replxx_hint_callback_t)(const char* input, replxx_hints* hints, int* contextLen, ReplxxColor* color, void* userData);
+
+/*! \brief Register hints callback.
+ *
+ * \param fn - user defined callback function.
+ * \param userData - pointer to opaque user data block to be passed into each invocation of the callback.
+ */
+REPLXX_IMPEXP void replxx_set_hint_callback( Replxx*, replxx_hint_callback_t* fn, void* userData );
+
+/*! \brief Key press handler type definition.
+ *
+ * \param code - the key code replxx got from terminal.
+ * \return Decision on how should input() behave after this key handler returns.
+ */
+typedef ReplxxActionResult (key_press_handler_t)( int code, void* userData );
+
+/*! \brief Add another possible hint for current user input.
+ *
+ * \param hints - pointer to opaque list of hints.
+ * \param str - UTF-8 encoded hint string.
+ */
+REPLXX_IMPEXP void replxx_add_hint( replxx_hints* hints, const char* str );
+
+/*! \brief Read line of user input.
+ *
+ * Returned pointer is managed by the library and is not to be freed in the client.
+ *
+ * \param prompt - prompt to be displayed before getting user input.
+ * \return An UTF-8 encoded input given by the user (or nullptr on EOF).
+ */
+REPLXX_IMPEXP char const* replxx_input( Replxx*, const char* prompt );
+
+/*! \brief Get current state data.
+ *
+ * This call is intended to be used in handlers.
+ *
+ * \param state - buffer for current state of the model.
+ */
+REPLXX_IMPEXP void replxx_get_state( Replxx*, ReplxxState* state );
+
+/*! \brief Set new state data.
+ *
+ * This call is intended to be used in handlers.
+ *
+ * \param state - new state of the model.
+ */
+REPLXX_IMPEXP void replxx_set_state( Replxx*, ReplxxState* state );
+
+/*! \brief Print formatted string to standard output.
+ *
+ * This function ensures proper handling of ANSI escape sequences
+ * contained in printed data, which is especially useful on Windows
+ * since Unixes handle them correctly out of the box.
+ *
+ * \param fmt - printf style format.
+ */
+REPLXX_IMPEXP int replxx_print( Replxx*, char const* fmt, ... );
+
+/*! \brief Prints a char array with the given length to standard output.
+ *
+ * \copydetails print
+ *
+ * \param str - The char array to print.
+ * \param length - The length of the array.
+ */
+REPLXX_IMPEXP int replxx_write( Replxx*, char const* str, int length );
+
+/*! \brief Schedule an emulated key press event.
+ *
+ * \param code - key press code to be emulated.
+ */
+REPLXX_IMPEXP void replxx_emulate_key_press( Replxx*, int unsigned code );
+
+/*! \brief Invoke built-in action handler.
+ *
+ * \pre This function can be called only from key-press handler.
+ *
+ * \param action - a built-in action to invoke.
+ * \param code - a supplementary key-code to consume by built-in action handler.
+ * \return The action result informing the replxx what shall happen next.
+ */
+REPLXX_IMPEXP ReplxxActionResult replxx_invoke( Replxx*, ReplxxAction action, int unsigned code );
+
+/*! \brief Bind user defined action to handle given key-press event.
+ *
+ * \param code - handle this key-press event with following handler.
+ * \param handler - use this handler to handle key-press event.
+ * \param userData - supplementary user data passed to invoked handlers.
+ */
+REPLXX_IMPEXP void replxx_bind_key( Replxx*, int code, key_press_handler_t handler, void* userData );
+
+/*! \brief Bind internal `replxx` action (by name) to handle given key-press event.
+ *
+ * Action names are the same as unique part of names of ReplxxAction enumerations
+ * but in lower case, e.g.: an action for recalling previous history line
+ * is \e REPLXX_ACTION_HISTORY_PREVIOUS so action name to be used in this
+ * interface for the same effect is "history_previous".
+ *
+ * \param code - handle this key-press event with following handler.
+ * \param actionName - name of internal action to be invoked on key press.
+ * \return -1 if invalid action name was used, 0 otherwise.
+ */
+int replxx_bind_key_internal( Replxx*, int code, char const* actionName );
+
+REPLXX_IMPEXP void replxx_set_preload_buffer( Replxx*, const char* preloadText );
+
+REPLXX_IMPEXP void replxx_history_add( Replxx*, const char* line );
+REPLXX_IMPEXP int replxx_history_size( Replxx* );
+
+/*! \brief Set set of word break characters.
+ *
+ * This setting influences word based cursor movement and line editing capabilities.
+ *
+ * \param wordBreakers - 7-bit ASCII set of word breaking characters.
+ */
+REPLXX_IMPEXP void replxx_set_word_break_characters( Replxx*, char const* wordBreakers );
+
+/*! \brief How many completions should trigger pagination.
+ */
+REPLXX_IMPEXP void replxx_set_completion_count_cutoff( Replxx*, int count );
+
+/*! \brief Set maximum number of displayed hint rows.
+ */
+REPLXX_IMPEXP void replxx_set_max_hint_rows( Replxx*, int count );
+
+/*! \brief Set a delay before hint are shown after user stopped typing..
+ *
+ * \param milliseconds - a number of milliseconds to wait before showing hints.
+ */
+REPLXX_IMPEXP void replxx_set_hint_delay( Replxx*, int milliseconds );
+
+/*! \brief Set tab completion behavior.
+ *
+ * \param val - use double tab to invoke completions (if != 0).
+ */
+REPLXX_IMPEXP void replxx_set_double_tab_completion( Replxx*, int val );
+
+/*! \brief Set tab completion behavior.
+ *
+ * \param val - invoke completion even if user input is empty (if != 0).
+ */
+REPLXX_IMPEXP void replxx_set_complete_on_empty( Replxx*, int val );
+
+/*! \brief Set tab completion behavior.
+ *
+ * \param val - beep if completion is ambiguous (if != 0).
+ */
+REPLXX_IMPEXP void replxx_set_beep_on_ambiguous_completion( Replxx*, int val );
+
+/*! \brief Set complete next/complete previous behavior.
+ *
+ * COMPLETE_NEXT/COMPLETE_PREVIOUS actions have two modes of operations,
+ * in case when a partial completion is possible complete only partial part (`false` setting)
+ * or complete first proposed completion fully (`true` setting).
+ * The default is to complete fully (a `true` setting - complete immediately).
+ *
+ * \param val - complete immediately.
+ */
+REPLXX_IMPEXP void replxx_set_immediate_completion( Replxx*, int val );
+
+/*! \brief Set history duplicate entries behaviour.
+ *
+ * \param val - should history contain only unique entries?
+ */
+REPLXX_IMPEXP void replxx_set_unique_history( Replxx*, int val );
+
+/*! \brief Disable output coloring.
+ *
+ * \param val - if set to non-zero disable output colors.
+ */
+REPLXX_IMPEXP void replxx_set_no_color( Replxx*, int val );
+
+/*! \brief Set maximum number of entries in history list.
+ */
+REPLXX_IMPEXP void replxx_set_max_history_size( Replxx*, int len );
+REPLXX_IMPEXP ReplxxHistoryScan* replxx_history_scan_start( Replxx* );
+REPLXX_IMPEXP void replxx_history_scan_stop( Replxx*, ReplxxHistoryScan* );
+REPLXX_IMPEXP int replxx_history_scan_next( Replxx*, ReplxxHistoryScan*, ReplxxHistoryEntry* );
+
+/*! \brief Synchronize REPL's history with given file.
+ *
+ * Synchronizing means loading existing history from given file,
+ * merging it with current history sorted by timestamps,
+ * saving merged version to given file,
+ * keeping merged version as current REPL's history.
+ *
+ * This call is an equivalent of calling:
+ * replxx_history_save( rx, "some-file" );
+ * replxx_history_load( rx, "some-file" );
+ *
+ * \param filename - a path to the file with which REPL's current history should be synchronized.
+ * \return 0 iff history file was successfully created, -1 otherwise.
+ */
+REPLXX_IMPEXP int replxx_history_sync( Replxx*, const char* filename );
+
+/*! \brief Save REPL's history into given file.
+ *
+ * Saving means loading existing history from given file,
+ * merging it with current history sorted by timestamps,
+ * saving merged version to given file,
+ * keeping original (NOT merged) version as current REPL's history.
+ *
+ * \param filename - a path to the file where REPL's history should be saved.
+ * \return 0 iff history file was successfully created, -1 otherwise.
+ */
+REPLXX_IMPEXP int replxx_history_save( Replxx*, const char* filename );
+
+/*! \brief Load REPL's history from given file.
+ *
+ * \param filename - a path to the file which contains REPL's history that should be loaded.
+ * \return 0 iff history file was successfully opened, -1 otherwise.
+ */
+REPLXX_IMPEXP int replxx_history_load( Replxx*, const char* filename );
+
+/*! \brief Clear REPL's in-memory history.
+ */
+REPLXX_IMPEXP void replxx_history_clear( Replxx* );
+REPLXX_IMPEXP void replxx_clear_screen( Replxx* );
+#ifdef __REPLXX_DEBUG__
+void replxx_debug_dump_print_codes(void);
+#endif
+/* the following is extension to the original linenoise API */
+REPLXX_IMPEXP int replxx_install_window_change_handler( Replxx* );
+REPLXX_IMPEXP void replxx_enable_bracketed_paste( Replxx* );
+REPLXX_IMPEXP void replxx_disable_bracketed_paste( Replxx* );
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __REPLXX_H */
+
diff --git a/contrib/replxx/include/replxx.hxx b/contrib/replxx/include/replxx.hxx
new file mode 100644
index 0000000..5362312
--- /dev/null
+++ b/contrib/replxx/include/replxx.hxx
@@ -0,0 +1,615 @@
+/*
+ * Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HAVE_REPLXX_HXX_INCLUDED
+#define HAVE_REPLXX_HXX_INCLUDED 1
+
+#include <memory>
+#include <vector>
+#include <string>
+#include <functional>
+
+/*
+ * For use in Windows DLLs:
+ *
+ * If you are building replxx into a DLL,
+ * unless you are using supplied CMake based build,
+ * ensure that 'REPLXX_BUILDING_DLL' is defined when
+ * building the DLL so that proper symbols are exported.
+ */
+#if defined( _WIN32 ) && ! defined( REPLXX_STATIC )
+# ifdef REPLXX_BUILDING_DLL
+# define REPLXX_IMPEXP __declspec( dllexport )
+# else
+# define REPLXX_IMPEXP __declspec( dllimport )
+# endif
+#else
+# define REPLXX_IMPEXP /**/
+#endif
+
+#ifdef ERROR
+enum { ERROR_BB1CA97EC761FC37101737BA0AA2E7C5 = ERROR };
+#undef ERROR
+enum { ERROR = ERROR_BB1CA97EC761FC37101737BA0AA2E7C5 };
+#endif
+#ifdef DELETE
+enum { DELETE_32F68A60CEF40FAEDBC6AF20298C1A1E = DELETE };
+#undef DELETE
+enum { DELETE = DELETE_32F68A60CEF40FAEDBC6AF20298C1A1E };
+#endif
+
+namespace replxx {
+
+class REPLXX_IMPEXP Replxx {
+public:
+ enum class Color {
+ BLACK = 0,
+ RED = 1,
+ GREEN = 2,
+ BROWN = 3,
+ BLUE = 4,
+ MAGENTA = 5,
+ CYAN = 6,
+ LIGHTGRAY = 7,
+ GRAY = 8,
+ BRIGHTRED = 9,
+ BRIGHTGREEN = 10,
+ YELLOW = 11,
+ BRIGHTBLUE = 12,
+ BRIGHTMAGENTA = 13,
+ BRIGHTCYAN = 14,
+ WHITE = 15,
+ NORMAL = LIGHTGRAY,
+ DEFAULT = -1,
+ ERROR = -2
+ };
+ struct KEY {
+ static char32_t const BASE = 0x0010ffff + 1;
+ static char32_t const BASE_SHIFT = 0x01000000;
+ static char32_t const BASE_CONTROL = 0x02000000;
+ static char32_t const BASE_META = 0x04000000;
+ static char32_t const ESCAPE = 27;
+ static char32_t const PAGE_UP = BASE + 1;
+ static char32_t const PAGE_DOWN = PAGE_UP + 1;
+ static char32_t const DOWN = PAGE_DOWN + 1;
+ static char32_t const UP = DOWN + 1;
+ static char32_t const LEFT = UP + 1;
+ static char32_t const RIGHT = LEFT + 1;
+ static char32_t const HOME = RIGHT + 1;
+ static char32_t const END = HOME + 1;
+ static char32_t const DELETE = END + 1;
+ static char32_t const INSERT = DELETE + 1;
+ static char32_t const F1 = INSERT + 1;
+ static char32_t const F2 = F1 + 1;
+ static char32_t const F3 = F2 + 1;
+ static char32_t const F4 = F3 + 1;
+ static char32_t const F5 = F4 + 1;
+ static char32_t const F6 = F5 + 1;
+ static char32_t const F7 = F6 + 1;
+ static char32_t const F8 = F7 + 1;
+ static char32_t const F9 = F8 + 1;
+ static char32_t const F10 = F9 + 1;
+ static char32_t const F11 = F10 + 1;
+ static char32_t const F12 = F11 + 1;
+ static char32_t const F13 = F12 + 1;
+ static char32_t const F14 = F13 + 1;
+ static char32_t const F15 = F14 + 1;
+ static char32_t const F16 = F15 + 1;
+ static char32_t const F17 = F16 + 1;
+ static char32_t const F18 = F17 + 1;
+ static char32_t const F19 = F18 + 1;
+ static char32_t const F20 = F19 + 1;
+ static char32_t const F21 = F20 + 1;
+ static char32_t const F22 = F21 + 1;
+ static char32_t const F23 = F22 + 1;
+ static char32_t const F24 = F23 + 1;
+ static char32_t const MOUSE = F24 + 1;
+ static char32_t const PASTE_START = MOUSE + 1;
+ static char32_t const PASTE_FINISH = PASTE_START + 1;
+ static constexpr char32_t shift( char32_t key_ ) {
+ return ( key_ | BASE_SHIFT );
+ }
+ static constexpr char32_t control( char32_t key_ ) {
+ return ( key_ | BASE_CONTROL );
+ }
+ static constexpr char32_t meta( char32_t key_ ) {
+ return ( key_ | BASE_META );
+ }
+ static char32_t const BACKSPACE = 'H' | BASE_CONTROL;
+ static char32_t const TAB = 'I' | BASE_CONTROL;
+ static char32_t const ENTER = 'M' | BASE_CONTROL;
+ };
+ /*! \brief List of built-in actions that act upon user input.
+ */
+ enum class ACTION {
+ INSERT_CHARACTER,
+ NEW_LINE,
+ DELETE_CHARACTER_UNDER_CURSOR,
+ DELETE_CHARACTER_LEFT_OF_CURSOR,
+ KILL_TO_END_OF_LINE,
+ KILL_TO_BEGINING_OF_LINE,
+ KILL_TO_END_OF_WORD,
+ KILL_TO_BEGINING_OF_WORD,
+ KILL_TO_END_OF_SUBWORD,
+ KILL_TO_BEGINING_OF_SUBWORD,
+ KILL_TO_WHITESPACE_ON_LEFT,
+ YANK,
+ YANK_CYCLE,
+ YANK_LAST_ARG,
+ MOVE_CURSOR_TO_BEGINING_OF_LINE,
+ MOVE_CURSOR_TO_END_OF_LINE,
+ MOVE_CURSOR_ONE_WORD_LEFT,
+ MOVE_CURSOR_ONE_WORD_RIGHT,
+ MOVE_CURSOR_ONE_SUBWORD_LEFT,
+ MOVE_CURSOR_ONE_SUBWORD_RIGHT,
+ MOVE_CURSOR_LEFT,
+ MOVE_CURSOR_RIGHT,
+ HISTORY_NEXT,
+ HISTORY_PREVIOUS,
+ HISTORY_FIRST,
+ HISTORY_LAST,
+ HISTORY_INCREMENTAL_SEARCH,
+ HISTORY_COMMON_PREFIX_SEARCH,
+ HINT_NEXT,
+ HINT_PREVIOUS,
+ CAPITALIZE_WORD,
+ LOWERCASE_WORD,
+ UPPERCASE_WORD,
+ CAPITALIZE_SUBWORD,
+ LOWERCASE_SUBWORD,
+ UPPERCASE_SUBWORD,
+ TRANSPOSE_CHARACTERS,
+ TOGGLE_OVERWRITE_MODE,
+#ifndef _WIN32
+ VERBATIM_INSERT,
+ SUSPEND,
+#endif
+ BRACKETED_PASTE,
+ CLEAR_SCREEN,
+ CLEAR_SELF,
+ REPAINT,
+ COMPLETE_LINE,
+ COMPLETE_NEXT,
+ COMPLETE_PREVIOUS,
+ COMMIT_LINE,
+ ABORT_LINE,
+ SEND_EOF
+ };
+ /*! \brief Possible results of key-press handler actions.
+ */
+ enum class ACTION_RESULT {
+ CONTINUE, /*!< Continue processing user input. */
+ RETURN, /*!< Return user input entered so far. */
+ BAIL /*!< Stop processing user input, returns nullptr from the \e input() call. */
+ };
+ typedef std::vector<Color> colors_t;
+ class Completion {
+ std::string _text;
+ Color _color;
+ public:
+ Completion( char const* text_ )
+ : _text( text_ )
+ , _color( Color::DEFAULT ) {
+ }
+ Completion( std::string const& text_ )
+ : _text( text_ )
+ , _color( Color::DEFAULT ) {
+ }
+ Completion( std::string const& text_, Color color_ )
+ : _text( text_ )
+ , _color( color_ ) {
+ }
+ std::string const& text( void ) const {
+ return ( _text );
+ }
+ Color color( void ) const {
+ return ( _color );
+ }
+ };
+ typedef std::vector<Completion> completions_t;
+ class HistoryEntry {
+ std::string _timestamp;
+ std::string _text;
+ public:
+ HistoryEntry( std::string const& timestamp_, std::string const& text_ )
+ : _timestamp( timestamp_ )
+ , _text( text_ ) {
+ }
+ std::string const& timestamp( void ) const {
+ return ( _timestamp );
+ }
+ std::string const& text( void ) const {
+ return ( _text );
+ }
+ };
+ class HistoryScanImpl;
+ class HistoryScan {
+ public:
+ typedef std::unique_ptr<HistoryScanImpl, void (*)( HistoryScanImpl* )> impl_t;
+ private:
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable:4251)
+#endif
+ impl_t _impl;
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+ public:
+ HistoryScan( impl_t );
+ HistoryScan( HistoryScan&& ) = default;
+ HistoryScan& operator = ( HistoryScan&& ) = default;
+ bool next( void );
+ HistoryEntry const& get( void ) const;
+ private:
+ HistoryScan( HistoryScan const& ) = delete;
+ HistoryScan& operator = ( HistoryScan const& ) = delete;
+ };
+ typedef std::vector<std::string> hints_t;
+
+ /*! \brief Line modification callback type definition.
+ *
+ * User can observe and modify line contents (and cursor position)
+ * in response to changes to both introduced by the user through
+ * normal interactions.
+ *
+ * When callback returns Replxx updates current line content
+ * and current cursor position to the ones updated by the callback.
+ *
+ * \param line[in,out] - a R/W reference to an UTF-8 encoded input entered by the user so far.
+ * \param cursorPosition[in,out] - a R/W reference to current cursor position.
+ */
+ typedef std::function<void ( std::string& line, int& cursorPosition )> modify_callback_t;
+
+ /*! \brief Completions callback type definition.
+ *
+ * \e contextLen is counted in Unicode code points (not in bytes!).
+ *
+ * For user input:
+ * if ( obj.me
+ *
+ * input == "if ( obj.me"
+ * contextLen == 2 (depending on \e set_word_break_characters())
+ *
+ * Client application is free to update \e contextLen to be 6 (or any other non-negative
+ * number not greater than the number of code points in input) if it makes better sense
+ * for given client application semantics.
+ *
+ * \param input - UTF-8 encoded input entered by the user until current cursor position.
+ * \param[in,out] contextLen - length of the additional context to provide while displaying completions.
+ * \return A list of user completions.
+ */
+ typedef std::function<completions_t ( std::string const& input, int& contextLen )> completion_callback_t;
+
+ /*! \brief Highlighter callback type definition.
+ *
+ * If user want to have colorful input she must simply install highlighter callback.
+ * The callback would be invoked by the library after each change to the input done by
+ * the user. After callback returns library uses data from colors buffer to colorize
+ * displayed user input.
+ *
+ * Size of \e colors buffer is equal to number of code points in user \e input
+ * which will be different from simple `input.length()`!
+ *
+ * \param input - an UTF-8 encoded input entered by the user so far.
+ * \param colors - output buffer for color information.
+ */
+ typedef std::function<void ( std::string const& input, colors_t& colors )> highlighter_callback_t;
+
+ /*! \brief Hints callback type definition.
+ *
+ * \e contextLen is counted in Unicode code points (not in bytes!).
+ *
+ * For user input:
+ * if ( obj.me
+ *
+ * input == "if ( obj.me"
+ * contextLen == 2 (depending on \e set_word_break_characters())
+ *
+ * Client application is free to update \e contextLen to be 6 (or any other non-negative
+ * number not greater than the number of code points in input) if it makes better sense
+ * for given client application semantics.
+ *
+ * \param input - UTF-8 encoded input entered by the user until current cursor position.
+ * \param contextLen[in,out] - length of the additional context to provide while displaying hints.
+ * \param color - a color used for displaying hints.
+ * \return A list of possible hints.
+ */
+ typedef std::function<hints_t ( std::string const& input, int& contextLen, Color& color )> hint_callback_t;
+
+ /*! \brief Key press handler type definition.
+ *
+ * \param code - the key code replxx got from terminal.
+ * \return Decision on how should input() behave after this key handler returns.
+ */
+ typedef std::function<ACTION_RESULT ( char32_t code )> key_press_handler_t;
+
+ struct State {
+ char const* _text;
+ int _cursorPosition;
+ State( char const* text_, int cursorPosition_ = -1 )
+ : _text( text_ )
+ , _cursorPosition( cursorPosition_ ) {
+ }
+ State( State const& ) = default;
+ State& operator = ( State const& ) = default;
+ char const* text( void ) const {
+ return ( _text );
+ }
+ int cursor_position( void ) const {
+ return ( _cursorPosition );
+ }
+ };
+
+ class ReplxxImpl;
+private:
+ typedef std::unique_ptr<ReplxxImpl, void (*)( ReplxxImpl* )> impl_t;
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable:4251)
+#endif
+ impl_t _impl;
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+public:
+ Replxx( void );
+ Replxx( Replxx&& ) = default;
+ Replxx& operator = ( Replxx&& ) = default;
+
+ /*! \brief Register modify callback.
+ *
+ * \param fn - user defined callback function.
+ */
+ void set_modify_callback( modify_callback_t const& fn );
+
+ /*! \brief Register completion callback.
+ *
+ * \param fn - user defined callback function.
+ */
+ void set_completion_callback( completion_callback_t const& fn );
+
+ /*! \brief Register highlighter callback.
+ *
+ * \param fn - user defined callback function.
+ */
+ void set_highlighter_callback( highlighter_callback_t const& fn );
+
+ /*! \brief Register hints callback.
+ *
+ * \param fn - user defined callback function.
+ */
+ void set_hint_callback( hint_callback_t const& fn );
+
+ /*! \brief Read line of user input.
+ *
+ * Returned pointer is managed by the library and is not to be freed in the client.
+ *
+ * \param prompt - prompt to be displayed before getting user input.
+ * \return An UTF-8 encoded input given by the user (or nullptr on EOF).
+ */
+ char const* input( std::string const& prompt );
+
+ /*! \brief Get current state data.
+ *
+ * This call is intended to be used in handlers.
+ *
+ * \return Current state of the model.
+ */
+ State get_state( void ) const;
+
+ /*! \brief Set new state data.
+ *
+ * This call is intended to be used in handlers.
+ *
+ * \param state - new state of the model.
+ */
+ void set_state( State const& state );
+
+ /*! \brief Print formatted string to standard output.
+ *
+ * This function ensures proper handling of ANSI escape sequences
+ * contained in printed data, which is especially useful on Windows
+ * since Unixes handle them correctly out of the box.
+ *
+ * \param fmt - printf style format.
+ */
+ void print( char const* fmt, ... );
+
+ /*! \brief Prints a char array with the given length to standard output.
+ *
+ * \copydetails print
+ *
+ * \param str - The char array to print.
+ * \param length - The length of the array.
+ */
+ void write( char const* str, int length );
+
+ /*! \brief Schedule an emulated key press event.
+ *
+ * \param code - key press code to be emulated.
+ */
+ void emulate_key_press( char32_t code );
+
+ /*! \brief Invoke built-in action handler.
+ *
+ * \pre This method can be called only from key-press handler.
+ *
+ * \param action - a built-in action to invoke.
+ * \param code - a supplementary key-code to consume by built-in action handler.
+ * \return The action result informing the replxx what shall happen next.
+ */
+ ACTION_RESULT invoke( ACTION action, char32_t code );
+
+ /*! \brief Bind user defined action to handle given key-press event.
+ *
+ * \param code - handle this key-press event with following handler.
+ * \param handle - use this handler to handle key-press event.
+ */
+ void bind_key( char32_t code, key_press_handler_t handler );
+
+ /*! \brief Bind internal `replxx` action (by name) to handle given key-press event.
+ *
+ * Action names are the same as names of Replxx::ACTION enumerations
+ * but in lower case, e.g.: an action for recalling previous history line
+ * is \e Replxx::ACTION::HISTORY_PREVIOUS so action name to be used in this
+ * interface for the same effect is "history_previous".
+ *
+ * \param code - handle this key-press event with following handler.
+ * \param actionName - name of internal action to be invoked on key press.
+ */
+ void bind_key_internal( char32_t code, char const* actionName );
+
+ void history_add( std::string const& line );
+
+ /*! \brief Synchronize REPL's history with given file.
+ *
+ * Synchronizing means loading existing history from given file,
+ * merging it with current history sorted by timestamps,
+ * saving merged version to given file,
+ * keeping merged version as current REPL's history.
+ *
+ * This call is an equivalent of calling:
+ * history_save( "some-file" );
+ * history_load( "some-file" );
+ *
+ * \param filename - a path to the file with which REPL's current history should be synchronized.
+ * \return True iff history file was successfully created.
+ */
+ bool history_sync( std::string const& filename );
+
+ /*! \brief Save REPL's history into given file.
+ *
+ * Saving means loading existing history from given file,
+ * merging it with current history sorted by timestamps,
+ * saving merged version to given file,
+ * keeping original (NOT merged) version as current REPL's history.
+ *
+ * \param filename - a path to the file where REPL's history should be saved.
+ * \return True iff history file was successfully created.
+ */
+ bool history_save( std::string const& filename );
+
+ /*! \brief Load REPL's history from given file.
+ *
+ * \param filename - a path to the file which contains REPL's history that should be loaded.
+ * \return True iff history file was successfully opened.
+ */
+ bool history_load( std::string const& filename );
+
+ /*! \brief Clear REPL's in-memory history.
+ */
+ void history_clear( void );
+ int history_size( void ) const;
+ HistoryScan history_scan( void ) const;
+
+ void set_preload_buffer( std::string const& preloadText );
+
+ /*! \brief Set set of word break characters.
+ *
+ * This setting influences word based cursor movement and line editing capabilities.
+ *
+ * \param wordBreakers - 7-bit ASCII set of word breaking characters.
+ */
+ void set_word_break_characters( char const* wordBreakers );
+
+ /*! \brief How many completions should trigger pagination.
+ */
+ void set_completion_count_cutoff( int count );
+
+ /*! \brief Set maximum number of displayed hint rows.
+ */
+ void set_max_hint_rows( int count );
+
+ /*! \brief Set a delay before hint are shown after user stopped typing..
+ *
+ * \param milliseconds - a number of milliseconds to wait before showing hints.
+ */
+ void set_hint_delay( int milliseconds );
+
+ /*! \brief Set tab completion behavior.
+ *
+ * \param val - use double tab to invoke completions.
+ */
+ void set_double_tab_completion( bool val );
+
+ /*! \brief Set tab completion behavior.
+ *
+ * \param val - invoke completion even if user input is empty.
+ */
+ void set_complete_on_empty( bool val );
+
+ /*! \brief Set tab completion behavior.
+ *
+ * \param val - beep if completion is ambiguous.
+ */
+ void set_beep_on_ambiguous_completion( bool val );
+
+ /*! \brief Set complete next/complete previous behavior.
+ *
+ * COMPLETE_NEXT/COMPLETE_PREVIOUS actions have two modes of operations,
+ * in case when a partial completion is possible complete only partial part (`false` setting)
+ * or complete first proposed completion fully (`true` setting).
+ * The default is to complete fully (a `true` setting - complete immediately).
+ *
+ * \param val - complete immediately.
+ */
+ void set_immediate_completion( bool val );
+
+ /*! \brief Set history duplicate entries behaviour.
+ *
+ * \param val - should history contain only unique entries?
+ */
+ void set_unique_history( bool val );
+
+ /*! \brief Disable output coloring.
+ *
+ * \param val - if set to non-zero disable output colors.
+ */
+ void set_no_color( bool val );
+
+ /*! \brief Set maximum number of entries in history list.
+ */
+ void set_max_history_size( int len );
+ void clear_screen( void );
+ int install_window_change_handler( void );
+ void enable_bracketed_paste( void );
+ void disable_bracketed_paste( void );
+
+private:
+ Replxx( Replxx const& ) = delete;
+ Replxx& operator = ( Replxx const& ) = delete;
+};
+
+}
+
+#endif /* HAVE_REPLXX_HXX_INCLUDED */
+
diff --git a/contrib/replxx/src/conversion.cxx b/contrib/replxx/src/conversion.cxx
new file mode 100644
index 0000000..f629f91
--- /dev/null
+++ b/contrib/replxx/src/conversion.cxx
@@ -0,0 +1,134 @@
+#include <algorithm>
+#include <string>
+#include <cstring>
+#include <cctype>
+#include <clocale>
+
+#include "unicode/utf8.h"
+#include "conversion.hxx"
+
+#ifdef _WIN32
+#define strdup _strdup
+#endif
+
+using namespace std;
+
+namespace replxx {
+
+namespace locale {
+
+void to_lower( std::string& s_ ) {
+ transform( s_.begin(), s_.end(), s_.begin(), static_cast<int(*)(int)>( &tolower ) );
+}
+
+bool is_8bit_encoding( void ) {
+ bool is8BitEncoding( false );
+ string origLC( setlocale( LC_CTYPE, nullptr ) );
+ string lc( origLC );
+ to_lower( lc );
+ if ( lc == "c" ) {
+ setlocale( LC_CTYPE, "" );
+ }
+ lc = setlocale( LC_CTYPE, nullptr );
+ setlocale( LC_CTYPE, origLC.c_str() );
+ to_lower( lc );
+ if ( lc.find( "8859" ) != std::string::npos ) {
+ is8BitEncoding = true;
+ }
+ return ( is8BitEncoding );
+}
+
+bool is8BitEncoding( is_8bit_encoding() );
+
+}
+
+ConversionResult copyString8to32(char32_t* dst, int dstSize, int& dstCount, const char* src) {
+ ConversionResult res = ConversionResult::conversionOK;
+ if ( ! locale::is8BitEncoding ) {
+ auto sourceStart = reinterpret_cast<const unsigned char*>(src);
+ auto slen = strlen(src);
+ auto targetStart = reinterpret_cast<UChar32*>(dst);
+ int i = 0, j = 0;
+
+ while (i < slen && j < dstSize) {
+ UChar32 uc;
+ auto prev_i = i;
+ U8_NEXT (sourceStart, i, slen, uc);
+
+ if (uc <= 0) {
+ if (U8_IS_LEAD (sourceStart[prev_i])) {
+ auto lead_byte = sourceStart[prev_i];
+ auto trailing_bytes = (((uint8_t)(lead_byte)>=0xc2)+
+ ((uint8_t)(lead_byte)>=0xe0)+
+ ((uint8_t)(lead_byte)>=0xf0));
+
+ if (trailing_bytes + i > slen) {
+ return ConversionResult::sourceExhausted;
+ }
+ }
+
+ /* Replace with 0xFFFD */
+ uc = 0x0000FFFD;
+ }
+ targetStart[j++] = uc;
+ }
+
+ dstCount = j;
+
+ if (j < dstSize) {
+ targetStart[j] = 0;
+ }
+ } else {
+ for ( dstCount = 0; ( dstCount < dstSize ) && src[dstCount]; ++ dstCount ) {
+ dst[dstCount] = src[dstCount];
+ }
+ }
+ return res;
+}
+
+ConversionResult copyString8to32(char32_t* dst, int dstSize, int& dstCount, const char8_t* src) {
+ return copyString8to32(
+ dst, dstSize, dstCount, reinterpret_cast<const char*>(src)
+ );
+}
+
+int copyString32to8(
+ char* dst, int dstSize, const char32_t* src, int srcSize
+) {
+ int resCount = 0;
+
+ if ( ! locale::is8BitEncoding ) {
+ int j = 0;
+ UBool is_error = 0;
+
+ for (auto i = 0; i < srcSize; i ++) {
+ U8_APPEND ((uint8_t *)dst, j, dstSize, src[i], is_error);
+
+ if (is_error) {
+ break;
+ }
+ }
+
+ if (!is_error) {
+ resCount = j;
+
+ if (j < dstSize) {
+ dst[j] = '\0';
+ }
+ }
+ } else {
+ int i( 0 );
+ for ( i = 0; ( i < dstSize ) && ( i < srcSize ) && src[i]; ++ i ) {
+ dst[i] = static_cast<char>( src[i] );
+ }
+ resCount = i;
+ if ( i < dstSize ) {
+ dst[i] = 0;
+ }
+ }
+
+ return resCount;
+}
+
+}
+
diff --git a/contrib/replxx/src/conversion.hxx b/contrib/replxx/src/conversion.hxx
new file mode 100644
index 0000000..05ea64f
--- /dev/null
+++ b/contrib/replxx/src/conversion.hxx
@@ -0,0 +1,35 @@
+#ifndef REPLXX_CONVERSION_HXX_INCLUDED
+#define REPLXX_CONVERSION_HXX_INCLUDED 1
+
+#ifdef __has_include
+#if __has_include( <version> )
+#include <version>
+#endif
+#endif
+
+typedef enum {
+ conversionOK, /* conversion successful */
+ sourceExhausted, /* partial character in source, but hit end */
+ targetExhausted, /* insuff. room in target for conversion */
+ sourceIllegal /* source sequence is illegal/malformed */
+} ConversionResult;
+
+#if ! ( defined( __cpp_lib_char8_t ) || ( defined( __clang_major__ ) && ( __clang_major__ >= 8 ) && ( __cplusplus > 201703L ) ) )
+namespace replxx {
+typedef unsigned char char8_t;
+}
+#endif
+
+namespace replxx {
+
+ConversionResult copyString8to32( char32_t* dst, int dstSize, int& dstCount, char const* src );
+ConversionResult copyString8to32( char32_t* dst, int dstSize, int& dstCount, char8_t const* src );
+int copyString32to8( char* dst, int dstSize, char32_t const* src, int srcSize );
+
+namespace locale {
+extern bool is8BitEncoding;
+}
+
+}
+
+#endif
diff --git a/contrib/replxx/src/escape.cxx b/contrib/replxx/src/escape.cxx
new file mode 100644
index 0000000..dda1ab0
--- /dev/null
+++ b/contrib/replxx/src/escape.cxx
@@ -0,0 +1,890 @@
+#include "escape.hxx"
+#include "terminal.hxx"
+#include "replxx.hxx"
+
+#ifndef _WIN32
+
+namespace replxx {
+
+namespace EscapeSequenceProcessing { // move these out of global namespace
+
+// This chunk of code does parsing of the escape sequences sent by various Linux
+// terminals.
+//
+// It handles arrow keys, Home, End and Delete keys by interpreting the
+// sequences sent by
+// gnome terminal, xterm, rxvt, konsole, aterm and yakuake including the Alt and
+// Ctrl key
+// combinations that are understood by replxx.
+//
+// The parsing uses tables, a bunch of intermediate dispatch routines and a
+// doDispatch
+// loop that reads the tables and sends control to "deeper" routines to continue
+// the
+// parsing. The starting call to doDispatch( c, initialDispatch ) will
+// eventually return
+// either a character (with optional CTRL and META bits set), or -1 if parsing
+// fails, or
+// zero if an attempt to read from the keyboard fails.
+//
+// This is rather sloppy escape sequence processing, since we're not paying
+// attention to what the
+// actual TERM is set to and are processing all key sequences for all terminals,
+// but it works with
+// the most common keystrokes on the most common terminals. It's intricate, but
+// the nested 'if'
+// statements required to do it directly would be worse. This way has the
+// advantage of allowing
+// changes and extensions without having to touch a lot of code.
+
+
+static char32_t thisKeyMetaCtrl = 0; // holds pre-set Meta and/or Ctrl modifiers
+
+// This dispatch routine is given a dispatch table and then farms work out to
+// routines
+// listed in the table based on the character it is called with. The dispatch
+// routines can
+// read more input characters to decide what should eventually be returned.
+// Eventually,
+// a called routine returns either a character or -1 to indicate parsing
+// failure.
+//
+char32_t doDispatch(char32_t c, CharacterDispatch& dispatchTable) {
+ for (unsigned int i = 0; i < dispatchTable.len; ++i) {
+ if (static_cast<unsigned char>(dispatchTable.chars[i]) == c) {
+ return dispatchTable.dispatch[i](c);
+ }
+ }
+ return dispatchTable.dispatch[dispatchTable.len](c);
+}
+
+// Final dispatch routines -- return something
+//
+static char32_t normalKeyRoutine(char32_t c) { return thisKeyMetaCtrl | c; }
+static char32_t upArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::UP;;
+}
+static char32_t downArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::DOWN;
+}
+static char32_t rightArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::RIGHT;
+}
+static char32_t leftArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::LEFT;
+}
+static char32_t homeKeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::HOME; }
+static char32_t endKeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::END; }
+static char32_t shiftTabRoutine(char32_t) { return Replxx::KEY::BASE_SHIFT | Replxx::KEY::TAB; }
+static char32_t f1KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F1; }
+static char32_t f2KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F2; }
+static char32_t f3KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F3; }
+static char32_t f4KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F4; }
+static char32_t f5KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F5; }
+static char32_t f6KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F6; }
+static char32_t f7KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F7; }
+static char32_t f8KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F8; }
+static char32_t f9KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F9; }
+static char32_t f10KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F10; }
+static char32_t f11KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F11; }
+static char32_t f12KeyRoutine(char32_t) { return thisKeyMetaCtrl | Replxx::KEY::F12; }
+static char32_t pageUpKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::PAGE_UP;
+}
+static char32_t pageDownKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::PAGE_DOWN;
+}
+static char32_t deleteCharRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::BACKSPACE;
+} // key labeled Backspace
+static char32_t insertKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::INSERT;
+} // key labeled Delete
+static char32_t deleteKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::DELETE;
+} // key labeled Delete
+static char32_t ctrlUpArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::BASE_CONTROL | Replxx::KEY::UP;
+}
+static char32_t ctrlDownArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::BASE_CONTROL | Replxx::KEY::DOWN;
+}
+static char32_t ctrlRightArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::BASE_CONTROL | Replxx::KEY::RIGHT;
+}
+static char32_t ctrlLeftArrowKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::BASE_CONTROL | Replxx::KEY::LEFT;
+}
+static char32_t bracketPasteStartKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::PASTE_START;
+}
+static char32_t bracketPasteFinishKeyRoutine(char32_t) {
+ return thisKeyMetaCtrl | Replxx::KEY::PASTE_FINISH;
+}
+static char32_t escFailureRoutine(char32_t) {
+ beep();
+ return -1;
+}
+
+// Handle ESC [ 1 ; 2 or 3 (or 5) <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket1Semicolon2or3or5Routines[] = {
+ upArrowKeyRoutine,
+ downArrowKeyRoutine,
+ rightArrowKeyRoutine,
+ leftArrowKeyRoutine,
+ homeKeyRoutine,
+ endKeyRoutine,
+ f1KeyRoutine,
+ f2KeyRoutine,
+ f3KeyRoutine,
+ f4KeyRoutine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket1Semicolon2or3or5Dispatch = {
+ 10, "ABCDHFPQRS", escLeftBracket1Semicolon2or3or5Routines
+};
+
+// Handle ESC [ 1 ; <more stuff> escape sequences
+//
+static char32_t escLeftBracket1Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket1Semicolon2or3or5Dispatch);
+}
+static char32_t escLeftBracket1Semicolon3Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_META;
+ return doDispatch(c, escLeftBracket1Semicolon2or3or5Dispatch);
+}
+static char32_t escLeftBracket1Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket1Semicolon2or3or5Dispatch);
+}
+static CharacterDispatchRoutine escLeftBracket1SemicolonRoutines[] = {
+ escLeftBracket1Semicolon2Routine,
+ escLeftBracket1Semicolon3Routine,
+ escLeftBracket1Semicolon5Routine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket1SemicolonDispatch = {
+ 3, "235", escLeftBracket1SemicolonRoutines
+};
+
+// Handle ESC [ 1 ; <more stuff> escape sequences
+//
+static char32_t escLeftBracket1SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket1SemicolonDispatch);
+}
+
+// (S)-F5
+static CharacterDispatchRoutine escLeftBracket15Semicolon2Routines[] = {
+ f5KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket15Semicolon2Dispatch = {
+ 1, "~", escLeftBracket15Semicolon2Routines
+};
+static char32_t escLeftBracket15Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket15Semicolon2Dispatch);
+}
+
+// (C)-F5
+static CharacterDispatchRoutine escLeftBracket15Semicolon5Routines[] = {
+ f5KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket15Semicolon5Dispatch = {
+ 1, "~", escLeftBracket15Semicolon5Routines
+};
+static char32_t escLeftBracket15Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket15Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket15SemicolonRoutines[] = {
+ escLeftBracket15Semicolon2Routine, escLeftBracket15Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket15SemicolonDispatch = {
+ 2, "25", escLeftBracket15SemicolonRoutines
+};
+static char32_t escLeftBracket15SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket15SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket15Routines[] = {
+ f5KeyRoutine, escLeftBracket15SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket15Dispatch = {
+ 2, "~;", escLeftBracket15Routines
+};
+static char32_t escLeftBracket15Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket15Dispatch);
+}
+
+// (S)-F6
+static CharacterDispatchRoutine escLeftBracket17Semicolon2Routines[] = {
+ f6KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket17Semicolon2Dispatch = {
+ 1, "~", escLeftBracket17Semicolon2Routines
+};
+static char32_t escLeftBracket17Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket17Semicolon2Dispatch);
+}
+
+// (C)-F6
+static CharacterDispatchRoutine escLeftBracket17Semicolon5Routines[] = {
+ f6KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket17Semicolon5Dispatch = {
+ 1, "~", escLeftBracket17Semicolon5Routines
+};
+static char32_t escLeftBracket17Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket17Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket17SemicolonRoutines[] = {
+ escLeftBracket17Semicolon2Routine, escLeftBracket17Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket17SemicolonDispatch = {
+ 2, "25", escLeftBracket17SemicolonRoutines
+};
+static char32_t escLeftBracket17SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket17SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket17Routines[] = {
+ f6KeyRoutine, escLeftBracket17SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket17Dispatch = {
+ 2, "~;", escLeftBracket17Routines
+};
+static char32_t escLeftBracket17Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket17Dispatch);
+}
+
+// (S)-F7
+static CharacterDispatchRoutine escLeftBracket18Semicolon2Routines[] = {
+ f7KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket18Semicolon2Dispatch = {
+ 1, "~", escLeftBracket18Semicolon2Routines
+};
+static char32_t escLeftBracket18Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket18Semicolon2Dispatch);
+}
+
+// (C)-F7
+static CharacterDispatchRoutine escLeftBracket18Semicolon5Routines[] = {
+ f7KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket18Semicolon5Dispatch = {
+ 1, "~", escLeftBracket18Semicolon5Routines
+};
+static char32_t escLeftBracket18Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket18Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket18SemicolonRoutines[] = {
+ escLeftBracket18Semicolon2Routine, escLeftBracket18Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket18SemicolonDispatch = {
+ 2, "25", escLeftBracket18SemicolonRoutines
+};
+static char32_t escLeftBracket18SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket18SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket18Routines[] = {
+ f7KeyRoutine, escLeftBracket18SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket18Dispatch = {
+ 2, "~;", escLeftBracket18Routines
+};
+static char32_t escLeftBracket18Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket18Dispatch);
+}
+
+// (S)-F8
+static CharacterDispatchRoutine escLeftBracket19Semicolon2Routines[] = {
+ f8KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket19Semicolon2Dispatch = {
+ 1, "~", escLeftBracket19Semicolon2Routines
+};
+static char32_t escLeftBracket19Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket19Semicolon2Dispatch);
+}
+
+// (C)-F8
+static CharacterDispatchRoutine escLeftBracket19Semicolon5Routines[] = {
+ f8KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket19Semicolon5Dispatch = {
+ 1, "~", escLeftBracket19Semicolon5Routines
+};
+static char32_t escLeftBracket19Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket19Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket19SemicolonRoutines[] = {
+ escLeftBracket19Semicolon2Routine, escLeftBracket19Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket19SemicolonDispatch = {
+ 2, "25", escLeftBracket19SemicolonRoutines
+};
+static char32_t escLeftBracket19SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket19SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket19Routines[] = {
+ f8KeyRoutine, escLeftBracket19SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket19Dispatch = {
+ 2, "~;", escLeftBracket19Routines
+};
+static char32_t escLeftBracket19Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket19Dispatch);
+}
+
+// Handle ESC [ 1 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket1Routines[] = {
+ homeKeyRoutine, escLeftBracket1SemicolonRoutine,
+ escLeftBracket15Routine,
+ escLeftBracket17Routine,
+ escLeftBracket18Routine,
+ escLeftBracket19Routine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket1Dispatch = {
+ 6, "~;5789", escLeftBracket1Routines
+};
+
+// Handle ESC [ 2 <more stuff> escape sequences
+//
+
+// (S)-F9
+static CharacterDispatchRoutine escLeftBracket20Semicolon2Routines[] = {
+ f9KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket20Semicolon2Dispatch = {
+ 1, "~", escLeftBracket20Semicolon2Routines
+};
+static char32_t escLeftBracket20Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket20Semicolon2Dispatch);
+}
+
+// (C)-F9
+static CharacterDispatchRoutine escLeftBracket20Semicolon5Routines[] = {
+ f9KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket20Semicolon5Dispatch = {
+ 1, "~", escLeftBracket20Semicolon5Routines
+};
+static char32_t escLeftBracket20Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket20Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket20SemicolonRoutines[] = {
+ escLeftBracket20Semicolon2Routine, escLeftBracket20Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket20SemicolonDispatch = {
+ 2, "25", escLeftBracket20SemicolonRoutines
+};
+static char32_t escLeftBracket20SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket20SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket200Routines[] = {
+ bracketPasteStartKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket200Dispatch = {
+ 1, "~", escLeftBracket200Routines
+};
+static char32_t escLeftBracket200Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket200Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket201Routines[] = {
+ bracketPasteFinishKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket201Dispatch = {
+ 1, "~", escLeftBracket201Routines
+};
+static char32_t escLeftBracket201Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket201Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket20Routines[] = {
+ f9KeyRoutine, escLeftBracket20SemicolonRoutine, escLeftBracket200Routine, escLeftBracket201Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket20Dispatch = {
+ 4, "~;01", escLeftBracket20Routines
+};
+static char32_t escLeftBracket20Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket20Dispatch);
+}
+
+// (S)-F10
+static CharacterDispatchRoutine escLeftBracket21Semicolon2Routines[] = {
+ f10KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket21Semicolon2Dispatch = {
+ 1, "~", escLeftBracket21Semicolon2Routines
+};
+static char32_t escLeftBracket21Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket21Semicolon2Dispatch);
+}
+
+// (C)-F10
+static CharacterDispatchRoutine escLeftBracket21Semicolon5Routines[] = {
+ f10KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket21Semicolon5Dispatch = {
+ 1, "~", escLeftBracket21Semicolon5Routines
+};
+static char32_t escLeftBracket21Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket21Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket21SemicolonRoutines[] = {
+ escLeftBracket21Semicolon2Routine, escLeftBracket21Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket21SemicolonDispatch = {
+ 2, "25", escLeftBracket21SemicolonRoutines
+};
+static char32_t escLeftBracket21SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket21SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket21Routines[] = {
+ f10KeyRoutine, escLeftBracket21SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket21Dispatch = {
+ 2, "~;", escLeftBracket21Routines
+};
+static char32_t escLeftBracket21Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket21Dispatch);
+}
+
+// (S)-F11
+static CharacterDispatchRoutine escLeftBracket23Semicolon2Routines[] = {
+ f11KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket23Semicolon2Dispatch = {
+ 1, "~", escLeftBracket23Semicolon2Routines
+};
+static char32_t escLeftBracket23Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket23Semicolon2Dispatch);
+}
+
+// (C)-F11
+static CharacterDispatchRoutine escLeftBracket23Semicolon5Routines[] = {
+ f11KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket23Semicolon5Dispatch = {
+ 1, "~", escLeftBracket23Semicolon5Routines
+};
+static char32_t escLeftBracket23Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket23Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket23SemicolonRoutines[] = {
+ escLeftBracket23Semicolon2Routine, escLeftBracket23Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket23SemicolonDispatch = {
+ 2, "25", escLeftBracket23SemicolonRoutines
+};
+static char32_t escLeftBracket23SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket23SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket23Routines[] = {
+ f11KeyRoutine, escLeftBracket23SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket23Dispatch = {
+ 2, "~;", escLeftBracket23Routines
+};
+static char32_t escLeftBracket23Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket23Dispatch);
+}
+
+// (S)-F12
+static CharacterDispatchRoutine escLeftBracket24Semicolon2Routines[] = {
+ f12KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket24Semicolon2Dispatch = {
+ 1, "~", escLeftBracket24Semicolon2Routines
+};
+static char32_t escLeftBracket24Semicolon2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_SHIFT;
+ return doDispatch(c, escLeftBracket24Semicolon2Dispatch);
+}
+
+// (C)-F12
+static CharacterDispatchRoutine escLeftBracket24Semicolon5Routines[] = {
+ f12KeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket24Semicolon5Dispatch = {
+ 1, "~", escLeftBracket24Semicolon5Routines
+};
+static char32_t escLeftBracket24Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket24Semicolon5Dispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket24SemicolonRoutines[] = {
+ escLeftBracket24Semicolon2Routine, escLeftBracket24Semicolon5Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket24SemicolonDispatch = {
+ 2, "25", escLeftBracket24SemicolonRoutines
+};
+static char32_t escLeftBracket24SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket24SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket24Routines[] = {
+ f12KeyRoutine, escLeftBracket24SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket24Dispatch = {
+ 2, "~;", escLeftBracket24Routines
+};
+static char32_t escLeftBracket24Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket24Dispatch);
+}
+
+// Handle ESC [ 2 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket2Routines[] = {
+ insertKeyRoutine,
+ escLeftBracket20Routine,
+ escLeftBracket21Routine,
+ escLeftBracket23Routine,
+ escLeftBracket24Routine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket2Dispatch = {
+ 5, "~0134", escLeftBracket2Routines
+};
+
+// Handle ESC [ 3 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket3Routines[] = {
+ deleteKeyRoutine, escFailureRoutine
+};
+
+static CharacterDispatch escLeftBracket3Dispatch = {
+ 1, "~", escLeftBracket3Routines
+};
+
+// Handle ESC [ 4 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket4Routines[] = {
+ endKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket4Dispatch = {
+ 1, "~", escLeftBracket4Routines
+};
+
+// Handle ESC [ 5 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket5Semicolon5Routines[] = {
+ pageUpKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket5Semicolon5Dispatch = {
+ 1, "~", escLeftBracket5Semicolon5Routines
+};
+static char32_t escLeftBracket5Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket5Semicolon5Dispatch);
+}
+static CharacterDispatchRoutine escLeftBracket5SemicolonRoutines[] = {
+ escLeftBracket5Semicolon5Routine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket5SemicolonDispatch = {
+ 1, "5", escLeftBracket5SemicolonRoutines
+};
+static char32_t escLeftBracket5SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket5SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket5Routines[] = {
+ pageUpKeyRoutine, escLeftBracket5SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket5Dispatch = {
+ 2, "~;", escLeftBracket5Routines
+};
+
+// Handle ESC [ 6 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket6Semicolon5Routines[] = {
+ pageDownKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket6Semicolon5Dispatch = {
+ 1, "~", escLeftBracket6Semicolon5Routines
+};
+static char32_t escLeftBracket6Semicolon5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ thisKeyMetaCtrl |= Replxx::KEY::BASE_CONTROL;
+ return doDispatch(c, escLeftBracket6Semicolon5Dispatch);
+}
+static CharacterDispatchRoutine escLeftBracket6SemicolonRoutines[] = {
+ escLeftBracket6Semicolon5Routine,
+ escFailureRoutine
+};
+static CharacterDispatch escLeftBracket6SemicolonDispatch = {
+ 1, "5", escLeftBracket6SemicolonRoutines
+};
+static char32_t escLeftBracket6SemicolonRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket6SemicolonDispatch);
+}
+
+static CharacterDispatchRoutine escLeftBracket6Routines[] = {
+ pageDownKeyRoutine, escLeftBracket6SemicolonRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket6Dispatch = {
+ 2, "~;", escLeftBracket6Routines
+};
+
+// Handle ESC [ 7 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket7Routines[] = {
+ homeKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket7Dispatch = {
+ 1, "~", escLeftBracket7Routines
+};
+
+// Handle ESC [ 8 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket8Routines[] = {
+ endKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracket8Dispatch = {
+ 1, "~", escLeftBracket8Routines
+};
+
+// Handle ESC [ <digit> escape sequences
+//
+static char32_t escLeftBracket0Routine(char32_t c) {
+ return escFailureRoutine(c);
+}
+static char32_t escLeftBracket1Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket1Dispatch);
+}
+static char32_t escLeftBracket2Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket2Dispatch);
+}
+static char32_t escLeftBracket3Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket3Dispatch);
+}
+static char32_t escLeftBracket4Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket4Dispatch);
+}
+static char32_t escLeftBracket5Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket5Dispatch);
+}
+static char32_t escLeftBracket6Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket6Dispatch);
+}
+static char32_t escLeftBracket7Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket7Dispatch);
+}
+static char32_t escLeftBracket8Routine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracket8Dispatch);
+}
+static char32_t escLeftBracket9Routine(char32_t c) {
+ return escFailureRoutine(c);
+}
+
+// Handle ESC [ <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracketRoutines[] = {
+ upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine,
+ leftArrowKeyRoutine, homeKeyRoutine, endKeyRoutine,
+ shiftTabRoutine,
+ escLeftBracket0Routine, escLeftBracket1Routine, escLeftBracket2Routine,
+ escLeftBracket3Routine, escLeftBracket4Routine, escLeftBracket5Routine,
+ escLeftBracket6Routine, escLeftBracket7Routine, escLeftBracket8Routine,
+ escLeftBracket9Routine, escFailureRoutine
+};
+static CharacterDispatch escLeftBracketDispatch = {17, "ABCDHFZ0123456789",
+ escLeftBracketRoutines};
+
+// Handle ESC O <char> escape sequences
+//
+static CharacterDispatchRoutine escORoutines[] = {
+ upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine,
+ leftArrowKeyRoutine, homeKeyRoutine, endKeyRoutine,
+ f1KeyRoutine, f2KeyRoutine, f3KeyRoutine,
+ f4KeyRoutine,
+ ctrlUpArrowKeyRoutine, ctrlDownArrowKeyRoutine, ctrlRightArrowKeyRoutine,
+ ctrlLeftArrowKeyRoutine, escFailureRoutine
+};
+static CharacterDispatch escODispatch = {14, "ABCDHFPQRSabcd", escORoutines};
+
+// Initial ESC dispatch -- could be a Meta prefix or the start of an escape
+// sequence
+//
+static char32_t escLeftBracketRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escLeftBracketDispatch);
+}
+static char32_t escORoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escODispatch);
+}
+static char32_t setMetaRoutine(char32_t c); // need forward reference
+static CharacterDispatchRoutine escRoutines[] = {
+ escLeftBracketRoutine, escORoutine, setMetaRoutine
+};
+static CharacterDispatch escDispatch = {2, "[O", escRoutines};
+
+// Initial dispatch -- we are not in the middle of anything yet
+//
+static char32_t escRoutine(char32_t c) {
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escDispatch);
+}
+static CharacterDispatchRoutine initialRoutines[] = {
+ escRoutine, deleteCharRoutine, normalKeyRoutine
+};
+static CharacterDispatch initialDispatch = {2, "\x1B\x7F", initialRoutines};
+
+// Special handling for the ESC key because it does double duty
+//
+static char32_t setMetaRoutine(char32_t c) {
+ thisKeyMetaCtrl = Replxx::KEY::BASE_META;
+ if (c == 0x1B) { // another ESC, stay in ESC processing mode
+ c = read_unicode_character();
+ if (c == 0) return 0;
+ return doDispatch(c, escDispatch);
+ }
+ return doDispatch(c, initialDispatch);
+}
+
+char32_t doDispatch(char32_t c) {
+ EscapeSequenceProcessing::thisKeyMetaCtrl = 0; // no modifiers yet at initialDispatch
+ return doDispatch(c, initialDispatch);
+}
+
+} // namespace EscapeSequenceProcessing // move these out of global namespace
+
+}
+
+#endif /* #ifndef _WIN32 */
+
diff --git a/contrib/replxx/src/escape.hxx b/contrib/replxx/src/escape.hxx
new file mode 100644
index 0000000..6597395
--- /dev/null
+++ b/contrib/replxx/src/escape.hxx
@@ -0,0 +1,37 @@
+#ifndef REPLXX_ESCAPE_HXX_INCLUDED
+#define REPLXX_ESCAPE_HXX_INCLUDED 1
+
+namespace replxx {
+
+namespace EscapeSequenceProcessing {
+
+// This is a typedef for the routine called by doDispatch(). It takes the
+// current character
+// as input, does any required processing including reading more characters and
+// calling other
+// dispatch routines, then eventually returns the final (possibly extended or
+// special) character.
+//
+typedef char32_t (*CharacterDispatchRoutine)(char32_t);
+
+// This structure is used by doDispatch() to hold a list of characters to test
+// for and
+// a list of routines to call if the character matches. The dispatch routine
+// list is one
+// longer than the character list; the final entry is used if no character
+// matches.
+//
+struct CharacterDispatch {
+ unsigned int len; // length of the chars list
+ const char* chars; // chars to test
+ CharacterDispatchRoutine* dispatch; // array of routines to call
+};
+
+char32_t doDispatch(char32_t c);
+
+}
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/history.cxx b/contrib/replxx/src/history.cxx
new file mode 100644
index 0000000..fe691df
--- /dev/null
+++ b/contrib/replxx/src/history.cxx
@@ -0,0 +1,402 @@
+#include <algorithm>
+#include <memory>
+#include <fstream>
+#include <cstring>
+
+#ifndef _WIN32
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+#endif /* _WIN32 */
+
+#include "replxx.hxx"
+#include "history.hxx"
+
+using namespace std;
+
+namespace replxx {
+
+namespace {
+void delete_ReplxxHistoryScanImpl( Replxx::HistoryScanImpl* impl_ ) {
+ delete impl_;
+}
+}
+
+static int const REPLXX_DEFAULT_HISTORY_MAX_LEN( 1000 );
+
+Replxx::HistoryScan::HistoryScan( impl_t impl_ )
+ : _impl( std::move( impl_ ) ) {
+}
+
+bool Replxx::HistoryScan::next( void ) {
+ return ( _impl->next() );
+}
+
+Replxx::HistoryScanImpl::HistoryScanImpl( History::entries_t const& entries_ )
+ : _entries( entries_ )
+ , _it( _entries.end() )
+ , _utf8Cache()
+ , _entryCache( std::string(), std::string() )
+ , _cacheValid( false ) {
+}
+
+Replxx::HistoryEntry const& Replxx::HistoryScan::get( void ) const {
+ return ( _impl->get() );
+}
+
+bool Replxx::HistoryScanImpl::next( void ) {
+ if ( _it == _entries.end() ) {
+ _it = _entries.begin();
+ } else {
+ ++ _it;
+ }
+ _cacheValid = false;
+ return ( _it != _entries.end() );
+}
+
+Replxx::HistoryEntry const& Replxx::HistoryScanImpl::get( void ) const {
+ if ( _cacheValid ) {
+ return ( _entryCache );
+ }
+ _utf8Cache.assign( _it->text() );
+ _entryCache = Replxx::HistoryEntry( _it->timestamp(), _utf8Cache.get() );
+ _cacheValid = true;
+ return ( _entryCache );
+}
+
+Replxx::HistoryScan::impl_t History::scan( void ) const {
+ return ( Replxx::HistoryScan::impl_t( new Replxx::HistoryScanImpl( _entries ), delete_ReplxxHistoryScanImpl ) );
+}
+
+History::History( void )
+ : _entries()
+ , _maxSize( REPLXX_DEFAULT_HISTORY_MAX_LEN )
+ , _current( _entries.begin() )
+ , _yankPos( _entries.end() )
+ , _previous( _entries.begin() )
+ , _recallMostRecent( false )
+ , _unique( true ) {
+}
+
+void History::add( UnicodeString const& line, std::string const& when ) {
+ if ( _maxSize <= 0 ) {
+ return;
+ }
+ if ( ! _entries.empty() && ( line == _entries.back().text() ) ) {
+ _entries.back() = Entry( now_ms_str(), line );
+ return;
+ }
+ remove_duplicate( line );
+ trim_to_max_size();
+ _entries.emplace_back( when, line );
+ _locations.insert( make_pair( line, last() ) );
+ if ( _current == _entries.end() ) {
+ _current = last();
+ }
+ _yankPos = _entries.end();
+}
+
+#ifndef _WIN32
+class FileLock {
+ std::string _path;
+ int _lockFd;
+public:
+ FileLock( std::string const& name_ )
+ : _path( name_ + ".lock" )
+ , _lockFd( ::open( _path.c_str(), O_CREAT | O_RDWR, 0600 ) ) {
+ static_cast<void>( ::lockf( _lockFd, F_LOCK, 0 ) == 0 );
+ }
+ ~FileLock( void ) {
+ static_cast<void>( ::lockf( _lockFd, F_ULOCK, 0 ) == 0 );
+ ::close( _lockFd );
+ ::unlink( _path.c_str() );
+ return;
+ }
+};
+#endif
+
+bool History::save( std::string const& filename, bool sync_ ) {
+#ifndef _WIN32
+ mode_t old_umask = umask( S_IXUSR | S_IRWXG | S_IRWXO );
+ FileLock fileLock( filename );
+#endif
+ entries_t entries;
+ locations_t locations;
+ if ( ! sync_ ) {
+ entries.swap( _entries );
+ locations.swap( _locations );
+ _entries = entries;
+ reset_iters();
+ }
+ do_load( filename );
+ sort();
+ remove_duplicates();
+ trim_to_max_size();
+ ofstream histFile( filename );
+ if ( ! histFile ) {
+ return ( false );
+ }
+#ifndef _WIN32
+ umask( old_umask );
+ chmod( filename.c_str(), S_IRUSR | S_IWUSR );
+#endif
+ Utf8String utf8;
+ for ( Entry const& h : _entries ) {
+ if ( ! h.text().is_empty() ) {
+ utf8.assign( h.text() );
+ histFile << "### " << h.timestamp() << "\n" << utf8.get() << endl;
+ }
+ }
+ if ( ! sync_ ) {
+ _entries = std::move( entries );
+ _locations = std::move( locations );
+ }
+ reset_iters();
+ return ( true );
+}
+
+namespace {
+
+bool is_timestamp( std::string const& s ) {
+ static char const TIMESTAMP_PATTERN[] = "### dddd-dd-dd dd:dd:dd.ddd";
+ static int const TIMESTAMP_LENGTH( sizeof ( TIMESTAMP_PATTERN ) - 1 );
+ if ( s.length() != TIMESTAMP_LENGTH ) {
+ return ( false );
+ }
+ for ( int i( 0 ); i < TIMESTAMP_LENGTH; ++ i ) {
+ if ( TIMESTAMP_PATTERN[i] == 'd' ) {
+ if ( ! isdigit( s[i] ) ) {
+ return ( false );
+ }
+ } else if ( s[i] != TIMESTAMP_PATTERN[i] ) {
+ return ( false );
+ }
+ }
+ return ( true );
+}
+
+}
+
+bool History::do_load( std::string const& filename ) {
+ ifstream histFile( filename );
+ if ( ! histFile ) {
+ return ( false );
+ }
+ string line;
+ string when( "0000-00-00 00:00:00.000" );
+ while ( getline( histFile, line ).good() ) {
+ string::size_type eol( line.find_first_of( "\r\n" ) );
+ if ( eol != string::npos ) {
+ line.erase( eol );
+ }
+ if ( is_timestamp( line ) ) {
+ when.assign( line, 4, std::string::npos );
+ continue;
+ }
+ if ( ! line.empty() ) {
+ _entries.emplace_back( when, UnicodeString( line ) );
+ }
+ }
+ return ( true );
+}
+
+bool History::load( std::string const& filename ) {
+ clear();
+ bool success( do_load( filename ) );
+ sort();
+ remove_duplicates();
+ trim_to_max_size();
+ _previous = _current = last();
+ _yankPos = _entries.end();
+ return ( success );
+}
+
+void History::sort( void ) {
+ typedef std::vector<Entry> sortable_entries_t;
+ _locations.clear();
+ sortable_entries_t sortableEntries( _entries.begin(), _entries.end() );
+ std::stable_sort( sortableEntries.begin(), sortableEntries.end() );
+ _entries.clear();
+ _entries.insert( _entries.begin(), sortableEntries.begin(), sortableEntries.end() );
+}
+
+void History::clear( void ) {
+ _locations.clear();
+ _entries.clear();
+ _current = _entries.begin();
+ _recallMostRecent = false;
+}
+
+void History::set_max_size( int size_ ) {
+ if ( size_ >= 0 ) {
+ _maxSize = size_;
+ trim_to_max_size();
+ }
+}
+
+void History::reset_yank_iterator( void ) {
+ _yankPos = _entries.end();
+}
+
+bool History::next_yank_position( void ) {
+ bool resetYankSize( false );
+ if ( _yankPos == _entries.end() ) {
+ resetYankSize = true;
+ }
+ if ( ( _yankPos != _entries.begin() ) && ( _yankPos != _entries.end() ) ) {
+ -- _yankPos;
+ } else {
+ _yankPos = moved( _entries.end(), -2 );
+ }
+ return ( resetYankSize );
+}
+
+bool History::move( bool up_ ) {
+ bool doRecall( _recallMostRecent && ! up_ );
+ if ( doRecall ) {
+ _current = _previous; // emulate Windows down-arrow
+ }
+ _recallMostRecent = false;
+ return ( doRecall || move( _current, up_ ? -1 : 1 ) );
+}
+
+void History::jump( bool start_, bool reset_ ) {
+ if ( start_ ) {
+ _current = _entries.begin();
+ } else {
+ _current = last();
+ }
+ if ( reset_ ) {
+ _recallMostRecent = false;
+ }
+}
+
+void History::save_pos( void ) {
+ _previous = _current;
+}
+
+void History::restore_pos( void ) {
+ _current = _previous;
+}
+
+bool History::common_prefix_search( UnicodeString const& prefix_, int prefixSize_, bool back_ ) {
+ int step( back_ ? -1 : 1 );
+ entries_t::const_iterator it( moved( _current, step, true ) );
+ while ( it != _current ) {
+ if ( it->text().starts_with( prefix_.begin(), prefix_.begin() + prefixSize_ ) ) {
+ _current = it;
+ commit_index();
+ return ( true );
+ }
+ move( it, step, true );
+ }
+ return ( false );
+}
+
+bool History::move( entries_t::const_iterator& it_, int by_, bool wrapped_ ) const {
+ if ( by_ > 0 ) {
+ for ( int i( 0 ); i < by_; ++ i ) {
+ ++ it_;
+ if ( it_ != _entries.end() ) {
+ } else if ( wrapped_ ) {
+ it_ = _entries.begin();
+ } else {
+ -- it_;
+ return ( false );
+ }
+ }
+ } else {
+ for ( int i( 0 ); i > by_; -- i ) {
+ if ( it_ != _entries.begin() ) {
+ -- it_;
+ } else if ( wrapped_ ) {
+ it_ = last();
+ } else {
+ return ( false );
+ }
+ }
+ }
+ return ( true );
+}
+
+History::entries_t::const_iterator History::moved( entries_t::const_iterator it_, int by_, bool wrapped_ ) const {
+ move( it_, by_, wrapped_ );
+ return ( it_ );
+}
+
+void History::erase( entries_t::const_iterator it_ ) {
+ bool invalidated( it_ == _current );
+ _locations.erase( it_->text() );
+ it_ = _entries.erase( it_ );
+ if ( invalidated ) {
+ _current = it_;
+ }
+ if ( ( _current == _entries.end() ) && ! _entries.empty() ) {
+ -- _current;
+ }
+ _yankPos = _entries.end();
+ _previous = _current;
+}
+
+void History::trim_to_max_size( void ) {
+ while ( size() > _maxSize ) {
+ erase( _entries.begin() );
+ }
+}
+
+void History::remove_duplicate( UnicodeString const& line_ ) {
+ if ( ! _unique ) {
+ return;
+ }
+ locations_t::iterator it( _locations.find( line_ ) );
+ if ( it == _locations.end() ) {
+ return;
+ }
+ erase( it->second );
+}
+
+void History::remove_duplicates( void ) {
+ if ( ! _unique ) {
+ return;
+ }
+ _locations.clear();
+ typedef std::pair<locations_t::iterator, bool> locations_insertion_result_t;
+ for ( entries_t::iterator it( _entries.begin() ), end( _entries.end() ); it != end; ++ it ) {
+ locations_insertion_result_t locationsInsertionResult( _locations.insert( make_pair( it->text(), it ) ) );
+ if ( ! locationsInsertionResult.second ) {
+ _entries.erase( locationsInsertionResult.first->second );
+ locationsInsertionResult.first->second = it;
+ }
+ }
+}
+
+void History::update_last( UnicodeString const& line_ ) {
+ if ( _unique ) {
+ _locations.erase( _entries.back().text() );
+ remove_duplicate( line_ );
+ _locations.insert( make_pair( line_, last() ) );
+ }
+ _entries.back() = Entry( now_ms_str(), line_ );
+}
+
+void History::drop_last( void ) {
+ erase( last() );
+}
+
+bool History::is_last( void ) const {
+ return ( _current == last() );
+}
+
+History::entries_t::const_iterator History::last( void ) const {
+ return ( moved( _entries.end(), -1 ) );
+}
+
+void History::reset_iters( void ) {
+ _previous = _current = last();
+ _yankPos = _entries.end();
+}
+
+}
+
diff --git a/contrib/replxx/src/history.hxx b/contrib/replxx/src/history.hxx
new file mode 100644
index 0000000..4e72c03
--- /dev/null
+++ b/contrib/replxx/src/history.hxx
@@ -0,0 +1,141 @@
+#ifndef REPLXX_HISTORY_HXX_INCLUDED
+#define REPLXX_HISTORY_HXX_INCLUDED 1
+
+#include <list>
+#include <unordered_map>
+
+#include "unicodestring.hxx"
+#include "utf8string.hxx"
+#include "conversion.hxx"
+#include "util.hxx"
+
+namespace std {
+template<>
+struct hash<replxx::UnicodeString> {
+ std::size_t operator()( replxx::UnicodeString const& us_ ) const {
+ std::size_t h( 0 );
+ char32_t const* p( us_.get() );
+ char32_t const* e( p + us_.length() );
+ while ( p != e ) {
+ h *= 31;
+ h += *p;
+ ++ p;
+ }
+ return ( h );
+ }
+};
+}
+
+namespace replxx {
+
+class History {
+public:
+ class Entry {
+ std::string _timestamp;
+ UnicodeString _text;
+ public:
+ Entry( std::string const& timestamp_, UnicodeString const& text_ )
+ : _timestamp( timestamp_ )
+ , _text( text_ ) {
+ }
+ std::string const& timestamp( void ) const {
+ return ( _timestamp );
+ }
+ UnicodeString const& text( void ) const {
+ return ( _text );
+ }
+ bool operator < ( Entry const& other_ ) const {
+ return ( _timestamp < other_._timestamp );
+ }
+ };
+ typedef std::list<Entry> entries_t;
+ typedef std::unordered_map<UnicodeString, entries_t::const_iterator> locations_t;
+private:
+ entries_t _entries;
+ locations_t _locations;
+ int _maxSize;
+ entries_t::const_iterator _current;
+ entries_t::const_iterator _yankPos;
+ /*
+ * _previous and _recallMostRecent are used to allow
+ * HISTORY_NEXT action (a down-arrow key) to have a special meaning
+ * if invoked after a line from history was accepted without
+ * any modification.
+ * Special meaning is: a down arrow shall jump to the line one
+ * after previously accepted from history.
+ */
+ entries_t::const_iterator _previous;
+ bool _recallMostRecent;
+ bool _unique;
+public:
+ History( void );
+ void add( UnicodeString const& line, std::string const& when = now_ms_str() );
+ bool save( std::string const& filename, bool );
+ bool load( std::string const& filename );
+ void clear( void );
+ void set_max_size( int len );
+ void set_unique( bool unique_ ) {
+ _unique = unique_;
+ remove_duplicates();
+ }
+ void reset_yank_iterator();
+ bool next_yank_position( void );
+ void reset_recall_most_recent( void ) {
+ _recallMostRecent = false;
+ }
+ void commit_index( void ) {
+ _previous = _current;
+ _recallMostRecent = true;
+ }
+ bool is_empty( void ) const {
+ return ( _entries.empty() );
+ }
+ void update_last( UnicodeString const& );
+ void drop_last( void );
+ bool is_last( void ) const;
+ bool move( bool );
+ UnicodeString const& current( void ) const {
+ return ( _current->text() );
+ }
+ UnicodeString const& yank_line( void ) const {
+ return ( _yankPos->text() );
+ }
+ void jump( bool, bool = true );
+ bool common_prefix_search( UnicodeString const&, int, bool );
+ int size( void ) const {
+ return ( static_cast<int>( _entries.size() ) );
+ }
+ Replxx::HistoryScan::impl_t scan( void ) const;
+ void save_pos( void );
+ void restore_pos( void );
+private:
+ History( History const& ) = delete;
+ History& operator = ( History const& ) = delete;
+ bool move( entries_t::const_iterator&, int, bool = false ) const;
+ entries_t::const_iterator moved( entries_t::const_iterator, int, bool = false ) const;
+ void erase( entries_t::const_iterator );
+ void trim_to_max_size( void );
+ void remove_duplicate( UnicodeString const& );
+ void remove_duplicates( void );
+ bool do_load( std::string const& );
+ entries_t::const_iterator last( void ) const;
+ void sort( void );
+ void reset_iters( void );
+};
+
+class Replxx::HistoryScanImpl {
+ History::entries_t const& _entries;
+ History::entries_t::const_iterator _it;
+ mutable Utf8String _utf8Cache;
+ mutable Replxx::HistoryEntry _entryCache;
+ mutable bool _cacheValid;
+public:
+ HistoryScanImpl( History::entries_t const& );
+ bool next( void );
+ Replxx::HistoryEntry const& get( void ) const;
+};
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/killring.hxx b/contrib/replxx/src/killring.hxx
new file mode 100644
index 0000000..0baf108
--- /dev/null
+++ b/contrib/replxx/src/killring.hxx
@@ -0,0 +1,78 @@
+#ifndef REPLXX_KILLRING_HXX_INCLUDED
+#define REPLXX_KILLRING_HXX_INCLUDED 1
+
+#include <vector>
+
+#include "unicodestring.hxx"
+
+namespace replxx {
+
+class KillRing {
+ static const int capacity = 10;
+ int size;
+ int index;
+ char indexToSlot[10];
+ std::vector<UnicodeString> theRing;
+
+public:
+ enum action { actionOther, actionKill, actionYank };
+ action lastAction;
+
+ KillRing()
+ : size(0)
+ , index(0)
+ , lastAction(actionOther) {
+ theRing.reserve(capacity);
+ }
+
+ void kill(const char32_t* text, int textLen, bool forward) {
+ if (textLen == 0) {
+ return;
+ }
+ UnicodeString killedText(text, textLen);
+ if (lastAction == actionKill && size > 0) {
+ int slot = indexToSlot[0];
+ int currentLen = static_cast<int>(theRing[slot].length());
+ UnicodeString temp;
+ if ( forward ) {
+ temp.append( theRing[slot].get(), currentLen ).append( killedText.get(), textLen );
+ } else {
+ temp.append( killedText.get(), textLen ).append( theRing[slot].get(), currentLen );
+ }
+ theRing[slot] = temp;
+ } else {
+ if (size < capacity) {
+ if (size > 0) {
+ memmove(&indexToSlot[1], &indexToSlot[0], size);
+ }
+ indexToSlot[0] = size;
+ size++;
+ theRing.push_back(killedText);
+ } else {
+ int slot = indexToSlot[capacity - 1];
+ theRing[slot] = killedText;
+ memmove(&indexToSlot[1], &indexToSlot[0], capacity - 1);
+ indexToSlot[0] = slot;
+ }
+ index = 0;
+ }
+ }
+
+ UnicodeString* yank() { return (size > 0) ? &theRing[indexToSlot[index]] : 0; }
+
+ UnicodeString* yankPop() {
+ if (size == 0) {
+ return 0;
+ }
+ ++index;
+ if (index == size) {
+ index = 0;
+ }
+ return &theRing[indexToSlot[index]];
+ }
+};
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/prompt.cxx b/contrib/replxx/src/prompt.cxx
new file mode 100644
index 0000000..c13ea80
--- /dev/null
+++ b/contrib/replxx/src/prompt.cxx
@@ -0,0 +1,144 @@
+#ifdef _WIN32
+
+#include <conio.h>
+#include <windows.h>
+#include <io.h>
+#if _MSC_VER < 1900 && defined (_MSC_VER)
+#define snprintf _snprintf // Microsoft headers use underscores in some names
+#endif
+#define strcasecmp _stricmp
+#define strdup _strdup
+#define write _write
+#define STDIN_FILENO 0
+
+#else /* _WIN32 */
+
+#include <unistd.h>
+
+#endif /* _WIN32 */
+
+#include "prompt.hxx"
+#include "util.hxx"
+
+namespace replxx {
+
+Prompt::Prompt( Terminal& terminal_ )
+ : _extraLines( 0 )
+ , _lastLinePosition( 0 )
+ , _cursorRowOffset( 0 )
+ , _screenColumns( 0 )
+ , _terminal( terminal_ ) {
+}
+
+void Prompt::write() {
+ _terminal.write32( _text.get(), _text.length() );
+}
+
+void Prompt::update_screen_columns( void ) {
+ _screenColumns = _terminal.get_screen_columns();
+}
+
+void Prompt::set_text( UnicodeString const& text_ ) {
+ _text = text_;
+ update_state();
+}
+
+void Prompt::update_state() {
+ _cursorRowOffset -= _extraLines;
+ _extraLines = 0;
+ _lastLinePosition = 0;
+ _screenColumns = 0;
+ update_screen_columns();
+ // strip control characters from the prompt -- we do allow newline
+ UnicodeString::const_iterator in( _text.begin() );
+ UnicodeString::iterator out( _text.begin() );
+
+ int visibleCount = 0;
+ int x = 0;
+
+ bool const strip = !tty::out;
+
+ while (in != _text.end()) {
+ char32_t c = *in;
+ if ('\n' == c || !is_control_code(c)) {
+ *out = c;
+ ++out;
+ ++in;
+ ++visibleCount;
+ if ('\n' == c || ++x >= _screenColumns) {
+ x = 0;
+ ++_extraLines;
+ _lastLinePosition = visibleCount;
+ }
+ } else if (c == '\x1b') {
+ if ( strip ) {
+ // jump over control chars
+ ++in;
+ if (*in == '[') {
+ ++in;
+ while ( ( in != _text.end() ) && ( ( *in == ';' ) || ( ( ( *in >= '0' ) && ( *in <= '9' ) ) ) ) ) {
+ ++in;
+ }
+ if (*in == 'm') {
+ ++in;
+ }
+ }
+ } else {
+ // copy control chars
+ *out = *in;
+ ++out;
+ ++in;
+ if (*in == '[') {
+ *out = *in;
+ ++out;
+ ++in;
+ while ( ( in != _text.end() ) && ( ( *in == ';' ) || ( ( ( *in >= '0' ) && ( *in <= '9' ) ) ) ) ) {
+ *out = *in;
+ ++out;
+ ++in;
+ }
+ if (*in == 'm') {
+ *out = *in;
+ ++out;
+ ++in;
+ }
+ }
+ }
+ } else {
+ ++in;
+ }
+ }
+ _characterCount = visibleCount;
+ int charCount( static_cast<int>( out - _text.begin() ) );
+ _text.erase( charCount, _text.length() - charCount );
+
+ _cursorRowOffset += _extraLines;
+}
+
+int Prompt::indentation() const {
+ return _characterCount - _lastLinePosition;
+}
+
+// Used with DynamicPrompt (history search)
+//
+const UnicodeString forwardSearchBasePrompt("(i-search)`");
+const UnicodeString reverseSearchBasePrompt("(reverse-i-search)`");
+const UnicodeString endSearchBasePrompt("': ");
+
+DynamicPrompt::DynamicPrompt( Terminal& terminal_, int initialDirection )
+ : Prompt( terminal_ )
+ , _searchText()
+ , _direction( initialDirection ) {
+ updateSearchPrompt();
+}
+
+void DynamicPrompt::updateSearchPrompt(void) {
+ update_screen_columns();
+ const UnicodeString* basePrompt =
+ (_direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt;
+ _text.assign( *basePrompt ).append( _searchText ).append( endSearchBasePrompt );
+ update_state();
+}
+
+}
+
diff --git a/contrib/replxx/src/prompt.hxx b/contrib/replxx/src/prompt.hxx
new file mode 100644
index 0000000..9ed3f5f
--- /dev/null
+++ b/contrib/replxx/src/prompt.hxx
@@ -0,0 +1,45 @@
+#ifndef REPLXX_PROMPT_HXX_INCLUDED
+#define REPLXX_PROMPT_HXX_INCLUDED 1
+
+#include <cstdlib>
+
+#include "unicodestring.hxx"
+#include "terminal.hxx"
+
+namespace replxx {
+
+class Prompt { // a convenience struct for grouping prompt info
+public:
+ UnicodeString _text; // our copy of the prompt text, edited
+ int _characterCount; // visible characters in _text
+ int _extraLines; // extra lines (beyond 1) occupied by prompt
+ int _lastLinePosition; // index into _text where last line begins
+ int _cursorRowOffset; // where the cursor is relative to the start of the prompt
+private:
+ int _screenColumns; // width of screen in columns [cache]
+ Terminal& _terminal;
+public:
+ Prompt( Terminal& );
+ void set_text( UnicodeString const& textPtr );
+ void update_state();
+ void update_screen_columns( void );
+ int screen_columns() const {
+ return ( _screenColumns );
+ }
+ void write();
+ int indentation() const;
+};
+
+// changing prompt for "(reverse-i-search)`text':" etc.
+//
+struct DynamicPrompt : public Prompt {
+ UnicodeString _searchText; // text we are searching for
+ int _direction; // current search _direction, 1=forward, -1=reverse
+
+ DynamicPrompt( Terminal&, int initialDirection );
+ void updateSearchPrompt(void);
+};
+
+}
+
+#endif
diff --git a/contrib/replxx/src/replxx.cxx b/contrib/replxx/src/replxx.cxx
new file mode 100644
index 0000000..29d35a2
--- /dev/null
+++ b/contrib/replxx/src/replxx.cxx
@@ -0,0 +1,648 @@
+/*
+ * Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
+ * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * line editing lib needs to be 20,000 lines of C code.
+ *
+ * You can find the latest source code at:
+ *
+ * http://github.com/antirez/linenoise
+ *
+ * Does a number of crazy assumptions that happen to be true in 99.9999% of
+ * the 2010 UNIX computers around.
+ *
+ * References:
+ * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
+ * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html
+ *
+ * Todo list:
+ * - Switch to gets() if $TERM is something we can't support.
+ * - Filter bogus Ctrl+<char> combinations.
+ * - Win32 support
+ *
+ * Bloat:
+ * - Completion?
+ * - History search like Ctrl+r in readline?
+ *
+ * List of escape sequences used by this program, we do everything just
+ * with three sequences. In order to be so cheap we may have some
+ * flickering effect with some slow terminal, but the lesser sequences
+ * the more compatible.
+ *
+ * CHA (Cursor Horizontal Absolute)
+ * Sequence: ESC [ n G
+ * Effect: moves cursor to column n (1 based)
+ *
+ * EL (Erase Line)
+ * Sequence: ESC [ n K
+ * Effect: if n is 0 or missing, clear from cursor to end of line
+ * Effect: if n is 1, clear from beginning of line to cursor
+ * Effect: if n is 2, clear entire line
+ *
+ * CUF (Cursor Forward)
+ * Sequence: ESC [ n C
+ * Effect: moves cursor forward of n chars
+ *
+ * The following are used to clear the screen: ESC [ H ESC [ 2 J
+ * This is actually composed of two sequences:
+ *
+ * cursorhome
+ * Sequence: ESC [ H
+ * Effect: moves the cursor to upper left corner
+ *
+ * ED2 (Clear entire screen)
+ * Sequence: ESC [ 2 J
+ * Effect: clear the whole screen
+ *
+ */
+
+#include <algorithm>
+#include <cstdarg>
+
+#ifdef _WIN32
+
+#include <io.h>
+#define STDIN_FILENO 0
+
+#else /* _WIN32 */
+
+#include <signal.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+#endif /* _WIN32 */
+
+#include "replxx.h"
+#include "replxx.hxx"
+#include "replxx_impl.hxx"
+#include "history.hxx"
+
+static_assert(
+ static_cast<int>( replxx::Replxx::ACTION::SEND_EOF ) == static_cast<int>( REPLXX_ACTION_SEND_EOF ),
+ "C and C++ `ACTION` APIs are missaligned!"
+);
+
+static_assert(
+ static_cast<int>( replxx::Replxx::KEY::PASTE_FINISH ) == static_cast<int>( REPLXX_KEY_PASTE_FINISH ),
+ "C and C++ `KEY` APIs are missaligned!"
+);
+
+using namespace std;
+using namespace std::placeholders;
+using namespace replxx;
+
+namespace replxx {
+
+namespace {
+void delete_ReplxxImpl( Replxx::ReplxxImpl* impl_ ) {
+ delete impl_;
+}
+}
+
+Replxx::Replxx( void )
+ : _impl( new Replxx::ReplxxImpl( nullptr, nullptr, nullptr ), delete_ReplxxImpl ) {
+}
+
+void Replxx::set_completion_callback( completion_callback_t const& fn ) {
+ _impl->set_completion_callback( fn );
+}
+
+void Replxx::set_modify_callback( modify_callback_t const& fn ) {
+ _impl->set_modify_callback( fn );
+}
+
+void Replxx::set_highlighter_callback( highlighter_callback_t const& fn ) {
+ _impl->set_highlighter_callback( fn );
+}
+
+void Replxx::set_hint_callback( hint_callback_t const& fn ) {
+ _impl->set_hint_callback( fn );
+}
+
+char const* Replxx::input( std::string const& prompt ) {
+ return ( _impl->input( prompt ) );
+}
+
+void Replxx::history_add( std::string const& line ) {
+ _impl->history_add( line );
+}
+
+bool Replxx::history_sync( std::string const& filename ) {
+ return ( _impl->history_sync( filename ) );
+}
+
+bool Replxx::history_save( std::string const& filename ) {
+ return ( _impl->history_save( filename ) );
+}
+
+bool Replxx::history_load( std::string const& filename ) {
+ return ( _impl->history_load( filename ) );
+}
+
+void Replxx::history_clear( void ) {
+ _impl->history_clear();
+}
+
+int Replxx::history_size( void ) const {
+ return ( _impl->history_size() );
+}
+
+Replxx::HistoryScan Replxx::history_scan( void ) const {
+ return ( _impl->history_scan() );
+}
+
+void Replxx::set_preload_buffer( std::string const& preloadText ) {
+ _impl->set_preload_buffer( preloadText );
+}
+
+void Replxx::set_word_break_characters( char const* wordBreakers ) {
+ _impl->set_word_break_characters( wordBreakers );
+}
+
+void Replxx::set_max_hint_rows( int count ) {
+ _impl->set_max_hint_rows( count );
+}
+
+void Replxx::set_hint_delay( int milliseconds ) {
+ _impl->set_hint_delay( milliseconds );
+}
+
+void Replxx::set_completion_count_cutoff( int count ) {
+ _impl->set_completion_count_cutoff( count );
+}
+
+void Replxx::set_double_tab_completion( bool val ) {
+ _impl->set_double_tab_completion( val );
+}
+
+void Replxx::set_complete_on_empty( bool val ) {
+ _impl->set_complete_on_empty( val );
+}
+
+void Replxx::set_beep_on_ambiguous_completion( bool val ) {
+ _impl->set_beep_on_ambiguous_completion( val );
+}
+
+void Replxx::set_immediate_completion( bool val ) {
+ _impl->set_immediate_completion( val );
+}
+
+void Replxx::set_unique_history( bool val ) {
+ _impl->set_unique_history( val );
+}
+
+void Replxx::set_no_color( bool val ) {
+ _impl->set_no_color( val );
+}
+
+void Replxx::set_max_history_size( int len ) {
+ _impl->set_max_history_size( len );
+}
+
+void Replxx::clear_screen( void ) {
+ _impl->clear_screen( 0 );
+}
+
+void Replxx::emulate_key_press( char32_t keyPress_ ) {
+ _impl->emulate_key_press( keyPress_ );
+}
+
+Replxx::ACTION_RESULT Replxx::invoke( ACTION action_, char32_t keyPress_ ) {
+ return ( _impl->invoke( action_, keyPress_ ) );
+}
+
+void Replxx::bind_key( char32_t keyPress_, key_press_handler_t handler_ ) {
+ _impl->bind_key( keyPress_, handler_ );
+}
+
+void Replxx::bind_key_internal( char32_t keyPress_, char const* actionName_ ) {
+ _impl->bind_key_internal( keyPress_, actionName_ );
+}
+
+Replxx::State Replxx::get_state( void ) const {
+ return ( _impl->get_state() );
+}
+
+void Replxx::set_state( Replxx::State const& state_ ) {
+ _impl->set_state( state_ );
+}
+
+int Replxx::install_window_change_handler( void ) {
+ return ( _impl->install_window_change_handler() );
+}
+
+void Replxx::enable_bracketed_paste( void ) {
+ _impl->enable_bracketed_paste();
+}
+
+void Replxx::disable_bracketed_paste( void ) {
+ _impl->disable_bracketed_paste();
+}
+
+void Replxx::print( char const* format_, ... ) {
+ ::std::va_list ap;
+ va_start( ap, format_ );
+ int size = static_cast<int>( vsnprintf( nullptr, 0, format_, ap ) );
+ va_end( ap );
+ va_start( ap, format_ );
+ unique_ptr<char[]> buf( new char[size + 1] );
+ vsnprintf( buf.get(), static_cast<size_t>( size + 1 ), format_, ap );
+ va_end( ap );
+ return ( _impl->print( buf.get(), size ) );
+}
+
+void Replxx::write( char const* str, int length ) {
+ return ( _impl->print( str, length ) );
+}
+
+}
+
+::Replxx* replxx_init() {
+ typedef ::Replxx* replxx_data_t;
+ return ( reinterpret_cast<replxx_data_t>( new replxx::Replxx::ReplxxImpl( nullptr, nullptr, nullptr ) ) );
+}
+
+void replxx_end( ::Replxx* replxx_ ) {
+ delete reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ );
+}
+
+void replxx_clear_screen( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->clear_screen( 0 );
+}
+
+void replxx_emulate_key_press( ::Replxx* replxx_, int unsigned keyPress_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->emulate_key_press( keyPress_ );
+}
+
+ReplxxActionResult replxx_invoke( ::Replxx* replxx_, ReplxxAction action_, int unsigned keyPress_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( static_cast<ReplxxActionResult>( replxx->invoke( static_cast<replxx::Replxx::ACTION>( action_ ), keyPress_ ) ) );
+}
+
+replxx::Replxx::ACTION_RESULT key_press_handler_forwarder( key_press_handler_t handler_, char32_t code_, void* userData_ ) {
+ return ( static_cast<replxx::Replxx::ACTION_RESULT>( handler_( code_, userData_ ) ) );
+}
+
+void replxx_bind_key( ::Replxx* replxx_, int code_, key_press_handler_t handler_, void* userData_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->bind_key( code_, std::bind( key_press_handler_forwarder, handler_, _1, userData_ ) );
+}
+
+int replxx_bind_key_internal( ::Replxx* replxx_, int code_, char const* actionName_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ try {
+ replxx->bind_key_internal( code_, actionName_ );
+ } catch ( ... ) {
+ return ( -1 );
+ }
+ return ( 0 );
+}
+
+void replxx_get_state( ::Replxx* replxx_, ReplxxState* state ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx::Replxx::State s( replxx->get_state() );
+ state->text = s.text();
+ state->cursorPosition = s.cursor_position();
+}
+
+void replxx_set_state( ::Replxx* replxx_, ReplxxState* state ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_state( replxx::Replxx::State( state->text, state->cursorPosition ) );
+}
+
+/**
+ * replxx_set_preload_buffer provides text to be inserted into the command buffer
+ *
+ * the provided text will be processed to be usable and will be used to preload
+ * the input buffer on the next call to replxx_input()
+ *
+ * @param preloadText text to begin with on the next call to replxx_input()
+ */
+void replxx_set_preload_buffer(::Replxx* replxx_, const char* preloadText) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_preload_buffer( preloadText ? preloadText : "" );
+}
+
+/**
+ * replxx_input is a readline replacement.
+ *
+ * call it with a prompt to display and it will return a line of input from the
+ * user
+ *
+ * @param prompt text of prompt to display to the user
+ * @return the returned string is managed by replxx library
+ * and it must NOT be freed in the client.
+ */
+char const* replxx_input( ::Replxx* replxx_, const char* prompt ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->input( prompt ) );
+}
+
+int replxx_print( ::Replxx* replxx_, char const* format_, ... ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ ::std::va_list ap;
+ va_start( ap, format_ );
+ int size = static_cast<int>( vsnprintf( nullptr, 0, format_, ap ) );
+ va_end( ap );
+ va_start( ap, format_ );
+ unique_ptr<char[]> buf( new char[size + 1] );
+ vsnprintf( buf.get(), static_cast<size_t>( size + 1 ), format_, ap );
+ va_end( ap );
+ try {
+ replxx->print( buf.get(), size );
+ } catch ( ... ) {
+ return ( -1 );
+ }
+ return ( size );
+}
+
+int replxx_write( ::Replxx* replxx_, char const* str, int length ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ try {
+ replxx->print( str, length );
+ } catch ( ... ) {
+ return ( -1 );
+ }
+ return static_cast<int>( length );
+}
+
+struct replxx_completions {
+ replxx::Replxx::completions_t data;
+};
+
+struct replxx_hints {
+ replxx::Replxx::hints_t data;
+};
+
+void modify_fwd( replxx_modify_callback_t fn, std::string& line_, int& cursorPosition_, void* userData_ ) {
+#ifdef _WIN32
+#define strdup _strdup
+#endif
+ char* s( strdup( line_.c_str() ) );
+#undef strdup
+ fn( &s, &cursorPosition_, userData_ );
+ line_ = s;
+ free( s );
+ return;
+}
+
+void replxx_set_modify_callback(::Replxx* replxx_, replxx_modify_callback_t* fn, void* userData) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_modify_callback( std::bind( &modify_fwd, fn, _1, _2, userData ) );
+}
+
+replxx::Replxx::completions_t completions_fwd( replxx_completion_callback_t fn, std::string const& input_, int& contextLen_, void* userData ) {
+ replxx_completions completions;
+ fn( input_.c_str(), &completions, &contextLen_, userData );
+ return ( completions.data );
+}
+
+/* Register a callback function to be called for tab-completion. */
+void replxx_set_completion_callback(::Replxx* replxx_, replxx_completion_callback_t* fn, void* userData) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_completion_callback( std::bind( &completions_fwd, fn, _1, _2, userData ) );
+}
+
+void highlighter_fwd( replxx_highlighter_callback_t fn, std::string const& input, replxx::Replxx::colors_t& colors, void* userData ) {
+ std::vector<ReplxxColor> colorsTmp( colors.size() );
+ std::transform(
+ colors.begin(),
+ colors.end(),
+ colorsTmp.begin(),
+ []( replxx::Replxx::Color c ) {
+ return ( static_cast<ReplxxColor>( c ) );
+ }
+ );
+ fn( input.c_str(), colorsTmp.data(), static_cast<int>( colors.size() ), userData );
+ std::transform(
+ colorsTmp.begin(),
+ colorsTmp.end(),
+ colors.begin(),
+ []( ReplxxColor c ) {
+ return ( static_cast<replxx::Replxx::Color>( c ) );
+ }
+ );
+}
+
+void replxx_set_highlighter_callback( ::Replxx* replxx_, replxx_highlighter_callback_t* fn, void* userData ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_highlighter_callback( std::bind( &highlighter_fwd, fn, _1, _2, userData ) );
+}
+
+replxx::Replxx::hints_t hints_fwd( replxx_hint_callback_t fn, std::string const& input_, int& contextLen_, replxx::Replxx::Color& color_, void* userData ) {
+ replxx_hints hints;
+ ReplxxColor c( static_cast<ReplxxColor>( color_ ) );
+ fn( input_.c_str(), &hints, &contextLen_, &c, userData );
+ return ( hints.data );
+}
+
+void replxx_set_hint_callback( ::Replxx* replxx_, replxx_hint_callback_t* fn, void* userData ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_hint_callback( std::bind( &hints_fwd, fn, _1, _2, _3, userData ) );
+}
+
+void replxx_add_hint(replxx_hints* lh, const char* str) {
+ lh->data.emplace_back(str);
+}
+
+void replxx_add_completion( replxx_completions* lc, const char* str ) {
+ lc->data.emplace_back( str );
+}
+
+void replxx_add_color_completion( replxx_completions* lc, const char* str, ReplxxColor color ) {
+ lc->data.emplace_back( str, static_cast<replxx::Replxx::Color>( color ) );
+}
+
+void replxx_history_add( ::Replxx* replxx_, const char* line ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->history_add( line );
+}
+
+void replxx_set_max_history_size( ::Replxx* replxx_, int len ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_max_history_size( len );
+}
+
+void replxx_set_max_hint_rows( ::Replxx* replxx_, int count ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_max_hint_rows( count );
+}
+
+void replxx_set_hint_delay( ::Replxx* replxx_, int milliseconds ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_hint_delay( milliseconds );
+}
+
+void replxx_set_completion_count_cutoff( ::Replxx* replxx_, int count ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_completion_count_cutoff( count );
+}
+
+void replxx_set_word_break_characters( ::Replxx* replxx_, char const* breakChars_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_word_break_characters( breakChars_ );
+}
+
+void replxx_set_double_tab_completion( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_double_tab_completion( val ? true : false );
+}
+
+void replxx_set_complete_on_empty( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_complete_on_empty( val ? true : false );
+}
+
+void replxx_set_no_color( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_no_color( val ? true : false );
+}
+
+void replxx_set_beep_on_ambiguous_completion( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_beep_on_ambiguous_completion( val ? true : false );
+}
+
+void replxx_set_immediate_completion( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_immediate_completion( val ? true : false );
+}
+
+void replxx_set_unique_history( ::Replxx* replxx_, int val ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->set_unique_history( val ? true : false );
+}
+
+void replxx_enable_bracketed_paste( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->enable_bracketed_paste();
+}
+
+void replxx_disable_bracketed_paste( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->disable_bracketed_paste();
+}
+
+ReplxxHistoryScan* replxx_history_scan_start( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( reinterpret_cast<ReplxxHistoryScan*>( replxx->history_scan().release() ) );
+}
+
+void replxx_history_scan_stop( ::Replxx*, ReplxxHistoryScan* historyScan_ ) {
+ delete reinterpret_cast<replxx::Replxx::HistoryScanImpl*>( historyScan_ );
+}
+
+int replxx_history_scan_next( ::Replxx*, ReplxxHistoryScan* historyScan_, ReplxxHistoryEntry* historyEntry_ ) {
+ replxx::Replxx::HistoryScanImpl* historyScan( reinterpret_cast<replxx::Replxx::HistoryScanImpl*>( historyScan_ ) );
+ bool hasNext( historyScan->next() );
+ if ( hasNext ) {
+ replxx::Replxx::HistoryEntry const& historyEntry( historyScan->get() );
+ historyEntry_->timestamp = historyEntry.timestamp().c_str();
+ historyEntry_->text = historyEntry.text().c_str();
+ }
+ return ( hasNext ? 0 : -1 );
+}
+
+/* Save the history in the specified file. On success 0 is returned
+ * otherwise -1 is returned. */
+int replxx_history_sync( ::Replxx* replxx_, const char* filename ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->history_sync( filename ) ? 0 : -1 );
+}
+
+/* Save the history in the specified file. On success 0 is returned
+ * otherwise -1 is returned. */
+int replxx_history_save( ::Replxx* replxx_, const char* filename ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->history_save( filename ) ? 0 : -1 );
+}
+
+/* Load the history from the specified file. If the file does not exist
+ * zero is returned and no operation is performed.
+ *
+ * If the file exists and the operation succeeded 0 is returned, otherwise
+ * on error -1 is returned. */
+int replxx_history_load( ::Replxx* replxx_, const char* filename ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->history_load( filename ) ? 0 : -1 );
+}
+
+void replxx_history_clear( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ replxx->history_clear();
+}
+
+int replxx_history_size( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->history_size() );
+}
+
+/* This special mode is used by replxx in order to print scan codes
+ * on screen for debugging / development purposes. It is implemented
+ * by the replxx-c-api-example program using the --keycodes option. */
+#ifdef __REPLXX_DEBUG__
+void replxx_debug_dump_print_codes(void) {
+ char quit[4];
+
+ printf(
+ "replxx key codes debugging mode.\n"
+ "Press keys to see scan codes. Type 'quit' at any time to exit.\n");
+ if (enableRawMode() == -1) return;
+ memset(quit, ' ', 4);
+ while (1) {
+ char c;
+ int nread;
+
+#if _WIN32
+ nread = _read(STDIN_FILENO, &c, 1);
+#else
+ nread = read(STDIN_FILENO, &c, 1);
+#endif
+ if (nread <= 0) continue;
+ memmove(quit, quit + 1, sizeof(quit) - 1); /* shift string to left. */
+ quit[sizeof(quit) - 1] = c; /* Insert current char on the right. */
+ if (memcmp(quit, "quit", sizeof(quit)) == 0) break;
+
+ printf("'%c' %02x (%d) (type quit to exit)\n", isprint(c) ? c : '?', (int)c,
+ (int)c);
+ printf("\r"); /* Go left edge manually, we are in raw mode. */
+ fflush(stdout);
+ }
+ disableRawMode();
+}
+#endif // __REPLXX_DEBUG__
+
+int replxx_install_window_change_handler( ::Replxx* replxx_ ) {
+ replxx::Replxx::ReplxxImpl* replxx( reinterpret_cast<replxx::Replxx::ReplxxImpl*>( replxx_ ) );
+ return ( replxx->install_window_change_handler() );
+}
+
diff --git a/contrib/replxx/src/replxx_impl.cxx b/contrib/replxx/src/replxx_impl.cxx
new file mode 100644
index 0000000..28152d5
--- /dev/null
+++ b/contrib/replxx/src/replxx_impl.cxx
@@ -0,0 +1,2248 @@
+#include <algorithm>
+#include <memory>
+#include <cerrno>
+#include <iostream>
+#include <chrono>
+
+#ifdef _WIN32
+
+#include <windows.h>
+#include <io.h>
+#if _MSC_VER < 1900
+#define snprintf _snprintf // Microsoft headers use underscores in some names
+#endif
+#define strcasecmp _stricmp
+#define write _write
+#define STDIN_FILENO 0
+
+#else /* _WIN32 */
+
+#include <unistd.h>
+#include <signal.h>
+
+#endif /* _WIN32 */
+
+#ifdef _WIN32
+#include "windows.hxx"
+#endif
+
+#include "replxx_impl.hxx"
+#include "utf8string.hxx"
+#include "prompt.hxx"
+#include "util.hxx"
+#include "terminal.hxx"
+#include "history.hxx"
+#include "replxx.hxx"
+
+using namespace std;
+
+namespace replxx {
+
+namespace {
+
+namespace action_names {
+
+char const INSERT_CHARACTER[] = "insert_character";
+char const NEW_LINE[] = "new_line";
+char const MOVE_CURSOR_TO_BEGINING_OF_LINE[] = "move_cursor_to_begining_of_line";
+char const MOVE_CURSOR_TO_END_OF_LINE[] = "move_cursor_to_end_of_line";
+char const MOVE_CURSOR_LEFT[] = "move_cursor_left";
+char const MOVE_CURSOR_RIGHT[] = "move_cursor_right";
+char const MOVE_CURSOR_ONE_WORD_LEFT[] = "move_cursor_one_word_left";
+char const MOVE_CURSOR_ONE_WORD_RIGHT[] = "move_cursor_one_word_right";
+char const MOVE_CURSOR_ONE_SUBWORD_LEFT[] = "move_cursor_one_subword_left";
+char const MOVE_CURSOR_ONE_SUBWORD_RIGHT[] = "move_cursor_one_subword_right";
+char const KILL_TO_WHITESPACE_ON_LEFT[] = "kill_to_whitespace_on_left";
+char const KILL_TO_END_OF_WORD[] = "kill_to_end_of_word";
+char const KILL_TO_END_OF_SUBWORD[] = "kill_to_end_of_subword";
+char const KILL_TO_BEGINING_OF_WORD[] = "kill_to_begining_of_word";
+char const KILL_TO_BEGINING_OF_SUBWORD[] = "kill_to_begining_of_subword";
+char const KILL_TO_BEGINING_OF_LINE[] = "kill_to_begining_of_line";
+char const KILL_TO_END_OF_LINE[] = "kill_to_end_of_line";
+char const YANK[] = "yank";
+char const YANK_CYCLE[] = "yank_cycle";
+char const YANK_LAST_ARG[] = "yank_last_arg";
+char const CAPITALIZE_WORD[] = "capitalize_word";
+char const LOWERCASE_WORD[] = "lowercase_word";
+char const UPPERCASE_WORD[] = "uppercase_word";
+char const CAPITALIZE_SUBWORD[] = "capitalize_subword";
+char const LOWERCASE_SUBWORD[] = "lowercase_subword";
+char const UPPERCASE_SUBWORD[] = "uppercase_subword";
+char const TRANSPOSE_CHARACTERS[] = "transpose_characters";
+char const ABORT_LINE[] = "abort_line";
+char const SEND_EOF[] = "send_eof";
+char const TOGGLE_OVERWRITE_MODE[] = "toggle_overwrite_mode";
+char const DELETE_CHARACTER_UNDER_CURSOR[] = "delete_character_under_cursor";
+char const DELETE_CHARACTER_LEFT_OF_CURSOR[] = "delete_character_left_of_cursor";
+char const COMMIT_LINE[] = "commit_line";
+char const CLEAR_SCREEN[] = "clear_screen";
+char const COMPLETE_NEXT[] = "complete_next";
+char const COMPLETE_PREVIOUS[] = "complete_previous";
+char const HISTORY_NEXT[] = "history_next";
+char const HISTORY_PREVIOUS[] = "history_previous";
+char const HISTORY_LAST[] = "history_last";
+char const HISTORY_FIRST[] = "history_first";
+char const HINT_PREVIOUS[] = "hint_previous";
+char const HINT_NEXT[] = "hint_next";
+char const VERBATIM_INSERT[] = "verbatim_insert";
+char const SUSPEND[] = "suspend";
+char const COMPLETE_LINE[] = "complete_line";
+char const HISTORY_INCREMENTAL_SEARCH[] = "history_incremental_search";
+char const HISTORY_COMMON_PREFIX_SEARCH[] = "history_common_prefix_search";
+}
+
+static int const REPLXX_MAX_HINT_ROWS( 4 );
+/*
+ * All whitespaces and all non-alphanumerical characters from ASCII range
+ * with an exception of an underscore ('_').
+ */
+char const defaultWordBreakChars[] = " \t\v\f\a\b\r\n`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?";
+/*
+ * All whitespaces and all non-alphanumerical characters from ASCII range
+ */
+char const defaultSubwordBreakChars[] = " \t\v\f\a\b\r\n`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?_";
+static const char* unsupported_term[] = {"dumb", "cons25", "emacs", NULL};
+
+static bool isUnsupportedTerm(void) {
+ char* term = getenv("TERM");
+ if (term == NULL) {
+ return false;
+ }
+ for (int j = 0; unsupported_term[j]; ++j) {
+ if (!strcasecmp(term, unsupported_term[j])) {
+ return true;
+ }
+ }
+ return false;
+}
+
+int long long RAPID_REFRESH_MS = 1;
+int long long RAPID_REFRESH_US = RAPID_REFRESH_MS * 1000;
+
+inline int long long now_us( void ) {
+ return ( std::chrono::duration_cast<std::chrono::microseconds>( std::chrono::high_resolution_clock::now().time_since_epoch() ).count() );
+}
+
+class IOModeGuard {
+ Terminal& _terminal;
+public:
+ IOModeGuard( Terminal& terminal_ )
+ : _terminal( terminal_ ) {
+ _terminal.disable_raw_mode();
+ }
+ ~IOModeGuard( void ) {
+ try {
+ _terminal.enable_raw_mode();
+ } catch ( ... ) {
+ }
+ }
+};
+
+}
+
+Replxx::ReplxxImpl::ReplxxImpl( FILE*, FILE*, FILE* )
+ : _utf8Buffer()
+ , _data()
+ , _pos( 0 )
+ , _display()
+ , _displayInputLength( 0 )
+ , _hint()
+ , _prefix( 0 )
+ , _hintSelection( -1 )
+ , _history()
+ , _killRing()
+ , _lastRefreshTime( now_us() )
+ , _refreshSkipped( false )
+ , _lastYankSize( 0 )
+ , _maxHintRows( REPLXX_MAX_HINT_ROWS )
+ , _hintDelay( 0 )
+ , _wordBreakChars( defaultWordBreakChars )
+ , _subwordBreakChars( defaultSubwordBreakChars )
+ , _completionCountCutoff( 100 )
+ , _overwrite( false )
+ , _doubleTabCompletion( false )
+ , _completeOnEmpty( true )
+ , _beepOnAmbiguousCompletion( false )
+ , _immediateCompletion( true )
+ , _bracketedPaste( false )
+ , _noColor( false )
+ , _namedActions()
+ , _keyPressHandlers()
+ , _terminal()
+ , _currentThread()
+ , _prompt( _terminal )
+ , _completionCallback( nullptr )
+ , _highlighterCallback( nullptr )
+ , _hintCallback( nullptr )
+ , _keyPresses()
+ , _messages()
+ , _completions()
+ , _completionContextLength( 0 )
+ , _completionSelection( -1 )
+ , _preloadedBuffer()
+ , _errorMessage()
+ , _previousSearchText()
+ , _modifiedState( false )
+ , _hintColor( Replxx::Color::GRAY )
+ , _hintsCache()
+ , _hintContextLenght( -1 )
+ , _hintSeed()
+ , _mutex() {
+ using namespace std::placeholders;
+ _namedActions[action_names::INSERT_CHARACTER] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::INSERT_CHARACTER, _1 );
+ _namedActions[action_names::NEW_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::NEW_LINE, _1 );
+ _namedActions[action_names::MOVE_CURSOR_TO_BEGINING_OF_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_TO_BEGINING_OF_LINE, _1 );
+ _namedActions[action_names::MOVE_CURSOR_TO_END_OF_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_TO_END_OF_LINE, _1 );
+ _namedActions[action_names::MOVE_CURSOR_LEFT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_LEFT, _1 );
+ _namedActions[action_names::MOVE_CURSOR_RIGHT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_RIGHT, _1 );
+ _namedActions[action_names::MOVE_CURSOR_ONE_WORD_LEFT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_ONE_WORD_LEFT, _1 );
+ _namedActions[action_names::MOVE_CURSOR_ONE_WORD_RIGHT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_ONE_WORD_RIGHT, _1 );
+ _namedActions[action_names::MOVE_CURSOR_ONE_SUBWORD_LEFT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_ONE_SUBWORD_LEFT, _1 );
+ _namedActions[action_names::MOVE_CURSOR_ONE_SUBWORD_RIGHT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::MOVE_CURSOR_ONE_SUBWORD_RIGHT, _1 );
+ _namedActions[action_names::KILL_TO_WHITESPACE_ON_LEFT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_WHITESPACE_ON_LEFT, _1 );
+ _namedActions[action_names::KILL_TO_END_OF_WORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_END_OF_WORD, _1 );
+ _namedActions[action_names::KILL_TO_BEGINING_OF_WORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_BEGINING_OF_WORD, _1 );
+ _namedActions[action_names::KILL_TO_END_OF_SUBWORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_END_OF_SUBWORD, _1 );
+ _namedActions[action_names::KILL_TO_BEGINING_OF_SUBWORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_BEGINING_OF_SUBWORD, _1 );
+ _namedActions[action_names::KILL_TO_BEGINING_OF_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_BEGINING_OF_LINE, _1 );
+ _namedActions[action_names::KILL_TO_END_OF_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::KILL_TO_END_OF_LINE, _1 );
+ _namedActions[action_names::YANK] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::YANK, _1 );
+ _namedActions[action_names::YANK_CYCLE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::YANK_CYCLE, _1 );
+ _namedActions[action_names::YANK_LAST_ARG] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::YANK_LAST_ARG, _1 );
+ _namedActions[action_names::CAPITALIZE_WORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::CAPITALIZE_WORD, _1 );
+ _namedActions[action_names::LOWERCASE_WORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::LOWERCASE_WORD, _1 );
+ _namedActions[action_names::UPPERCASE_WORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::UPPERCASE_WORD, _1 );
+ _namedActions[action_names::CAPITALIZE_SUBWORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::CAPITALIZE_SUBWORD, _1 );
+ _namedActions[action_names::LOWERCASE_SUBWORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::LOWERCASE_SUBWORD, _1 );
+ _namedActions[action_names::UPPERCASE_SUBWORD] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::UPPERCASE_SUBWORD, _1 );
+ _namedActions[action_names::TRANSPOSE_CHARACTERS] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::TRANSPOSE_CHARACTERS, _1 );
+ _namedActions[action_names::ABORT_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::ABORT_LINE, _1 );
+ _namedActions[action_names::SEND_EOF] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::SEND_EOF, _1 );
+ _namedActions[action_names::TOGGLE_OVERWRITE_MODE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::TOGGLE_OVERWRITE_MODE, _1 );
+ _namedActions[action_names::DELETE_CHARACTER_UNDER_CURSOR] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::DELETE_CHARACTER_UNDER_CURSOR, _1 );
+ _namedActions[action_names::DELETE_CHARACTER_LEFT_OF_CURSOR] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::DELETE_CHARACTER_LEFT_OF_CURSOR, _1 );
+ _namedActions[action_names::COMMIT_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::COMMIT_LINE, _1 );
+ _namedActions[action_names::CLEAR_SCREEN] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::CLEAR_SCREEN, _1 );
+ _namedActions[action_names::COMPLETE_NEXT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::COMPLETE_NEXT, _1 );
+ _namedActions[action_names::COMPLETE_PREVIOUS] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::COMPLETE_PREVIOUS, _1 );
+ _namedActions[action_names::HISTORY_NEXT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_NEXT, _1 );
+ _namedActions[action_names::HISTORY_PREVIOUS] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_PREVIOUS, _1 );
+ _namedActions[action_names::HISTORY_LAST] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_LAST, _1 );
+ _namedActions[action_names::HISTORY_FIRST] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_FIRST, _1 );
+ _namedActions[action_names::HINT_PREVIOUS] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HINT_PREVIOUS, _1 );
+ _namedActions[action_names::HINT_NEXT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HINT_NEXT, _1 );
+#ifndef _WIN32
+ _namedActions[action_names::VERBATIM_INSERT] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::VERBATIM_INSERT, _1 );
+ _namedActions[action_names::SUSPEND] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::SUSPEND, _1 );
+#else
+ _namedActions[action_names::VERBATIM_INSERT] = _namedActions[action_names::SUSPEND] = Replxx::key_press_handler_t();
+#endif
+ _namedActions[action_names::COMPLETE_LINE] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::COMPLETE_LINE, _1 );
+ _namedActions[action_names::HISTORY_INCREMENTAL_SEARCH] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, _1 );
+ _namedActions[action_names::HISTORY_COMMON_PREFIX_SEARCH] = std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::HISTORY_COMMON_PREFIX_SEARCH, _1 );
+
+ bind_key( Replxx::KEY::control( 'A' ), _namedActions.at( action_names::MOVE_CURSOR_TO_BEGINING_OF_LINE ) );
+ bind_key( Replxx::KEY::HOME + 0, _namedActions.at( action_names::MOVE_CURSOR_TO_BEGINING_OF_LINE ) );
+ bind_key( Replxx::KEY::control( 'E' ), _namedActions.at( action_names::MOVE_CURSOR_TO_END_OF_LINE ) );
+ bind_key( Replxx::KEY::END + 0, _namedActions.at( action_names::MOVE_CURSOR_TO_END_OF_LINE ) );
+ bind_key( Replxx::KEY::control( 'B' ), _namedActions.at( action_names::MOVE_CURSOR_LEFT ) );
+ bind_key( Replxx::KEY::LEFT + 0, _namedActions.at( action_names::MOVE_CURSOR_LEFT ) );
+ bind_key( Replxx::KEY::control( 'F' ), _namedActions.at( action_names::MOVE_CURSOR_RIGHT ) );
+ bind_key( Replxx::KEY::RIGHT + 0, _namedActions.at( action_names::MOVE_CURSOR_RIGHT ) );
+ bind_key( Replxx::KEY::meta( 'b' ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_LEFT ) );
+ bind_key( Replxx::KEY::meta( 'B' ), _namedActions.at( action_names::MOVE_CURSOR_ONE_SUBWORD_LEFT ) );
+ bind_key( Replxx::KEY::control( Replxx::KEY::LEFT ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_LEFT ) );
+ bind_key( Replxx::KEY::meta( Replxx::KEY::LEFT ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_LEFT ) ); // Emacs allows Meta, readline don't
+ bind_key( Replxx::KEY::meta( 'f' ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_RIGHT ) );
+ bind_key( Replxx::KEY::meta( 'F' ), _namedActions.at( action_names::MOVE_CURSOR_ONE_SUBWORD_RIGHT ) );
+ bind_key( Replxx::KEY::control( Replxx::KEY::RIGHT ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_RIGHT ) );
+ bind_key( Replxx::KEY::meta( Replxx::KEY::RIGHT ), _namedActions.at( action_names::MOVE_CURSOR_ONE_WORD_RIGHT ) ); // Emacs allows Meta, readline don't
+ bind_key( Replxx::KEY::meta( Replxx::KEY::BACKSPACE ), _namedActions.at( action_names::KILL_TO_WHITESPACE_ON_LEFT ) );
+ bind_key( Replxx::KEY::meta( 'd' ), _namedActions.at( action_names::KILL_TO_END_OF_WORD ) );
+ bind_key( Replxx::KEY::meta( 'D' ), _namedActions.at( action_names::KILL_TO_END_OF_SUBWORD ) );
+ bind_key( Replxx::KEY::control( 'W' ), _namedActions.at( action_names::KILL_TO_BEGINING_OF_WORD ) );
+ bind_key( Replxx::KEY::meta( 'W' ), _namedActions.at( action_names::KILL_TO_BEGINING_OF_SUBWORD ) );
+ bind_key( Replxx::KEY::control( 'U' ), _namedActions.at( action_names::KILL_TO_BEGINING_OF_LINE ) );
+ bind_key( Replxx::KEY::control( 'K' ), _namedActions.at( action_names::KILL_TO_END_OF_LINE ) );
+ bind_key( Replxx::KEY::control( 'Y' ), _namedActions.at( action_names::YANK ) );
+ bind_key( Replxx::KEY::meta( 'y' ), _namedActions.at( action_names::YANK_CYCLE ) );
+ bind_key( Replxx::KEY::meta( 'Y' ), _namedActions.at( action_names::YANK_CYCLE ) );
+ bind_key( Replxx::KEY::meta( '.' ), _namedActions.at( action_names::YANK_LAST_ARG ) );
+ bind_key( Replxx::KEY::meta( 'c' ), _namedActions.at( action_names::CAPITALIZE_WORD ) );
+ bind_key( Replxx::KEY::meta( 'C' ), _namedActions.at( action_names::CAPITALIZE_SUBWORD ) );
+ bind_key( Replxx::KEY::meta( 'l' ), _namedActions.at( action_names::LOWERCASE_WORD ) );
+ bind_key( Replxx::KEY::meta( 'L' ), _namedActions.at( action_names::LOWERCASE_SUBWORD ) );
+ bind_key( Replxx::KEY::meta( 'u' ), _namedActions.at( action_names::UPPERCASE_WORD ) );
+ bind_key( Replxx::KEY::meta( 'U' ), _namedActions.at( action_names::UPPERCASE_SUBWORD ) );
+ bind_key( Replxx::KEY::control( 'T' ), _namedActions.at( action_names::TRANSPOSE_CHARACTERS ) );
+ bind_key( Replxx::KEY::control( 'C' ), _namedActions.at( action_names::ABORT_LINE ) );
+ bind_key( Replxx::KEY::control( 'D' ), _namedActions.at( action_names::SEND_EOF ) );
+ bind_key( Replxx::KEY::INSERT + 0, _namedActions.at( action_names::TOGGLE_OVERWRITE_MODE ) );
+ bind_key( 127, _namedActions.at( action_names::DELETE_CHARACTER_UNDER_CURSOR ) );
+ bind_key( Replxx::KEY::DELETE + 0, _namedActions.at( action_names::DELETE_CHARACTER_UNDER_CURSOR ) );
+ bind_key( Replxx::KEY::BACKSPACE + 0, _namedActions.at( action_names::DELETE_CHARACTER_LEFT_OF_CURSOR ) );
+ bind_key( Replxx::KEY::control( 'J' ), _namedActions.at( action_names::NEW_LINE ) );
+ bind_key( Replxx::KEY::ENTER + 0, _namedActions.at( action_names::COMMIT_LINE ) );
+ bind_key( Replxx::KEY::control( 'L' ), _namedActions.at( action_names::CLEAR_SCREEN ) );
+ bind_key( Replxx::KEY::control( 'N' ), _namedActions.at( action_names::COMPLETE_NEXT ) );
+ bind_key( Replxx::KEY::control( 'P' ), _namedActions.at( action_names::COMPLETE_PREVIOUS ) );
+ bind_key( Replxx::KEY::DOWN + 0, _namedActions.at( action_names::HISTORY_NEXT ) );
+ bind_key( Replxx::KEY::UP + 0, _namedActions.at( action_names::HISTORY_PREVIOUS ) );
+ bind_key( Replxx::KEY::meta( '<' ), _namedActions.at( action_names::HISTORY_FIRST ) );
+ bind_key( Replxx::KEY::PAGE_UP + 0, _namedActions.at( action_names::HISTORY_FIRST ) );
+ bind_key( Replxx::KEY::meta( '>' ), _namedActions.at( action_names::HISTORY_LAST ) );
+ bind_key( Replxx::KEY::PAGE_DOWN + 0, _namedActions.at( action_names::HISTORY_LAST ) );
+ bind_key( Replxx::KEY::control( Replxx::KEY::UP ), _namedActions.at( action_names::HINT_PREVIOUS ) );
+ bind_key( Replxx::KEY::control( Replxx::KEY::DOWN ), _namedActions.at( action_names::HINT_NEXT ) );
+#ifndef _WIN32
+ bind_key( Replxx::KEY::control( 'V' ), _namedActions.at( action_names::VERBATIM_INSERT ) );
+ bind_key( Replxx::KEY::control( 'Z' ), _namedActions.at( action_names::SUSPEND ) );
+#endif
+ bind_key( Replxx::KEY::TAB + 0, _namedActions.at( action_names::COMPLETE_LINE ) );
+ bind_key( Replxx::KEY::control( 'R' ), _namedActions.at( action_names::HISTORY_INCREMENTAL_SEARCH ) );
+ bind_key( Replxx::KEY::control( 'S' ), _namedActions.at( action_names::HISTORY_INCREMENTAL_SEARCH ) );
+ bind_key( Replxx::KEY::meta( 'p' ), _namedActions.at( action_names::HISTORY_COMMON_PREFIX_SEARCH ) );
+ bind_key( Replxx::KEY::meta( 'P' ), _namedActions.at( action_names::HISTORY_COMMON_PREFIX_SEARCH ) );
+ bind_key( Replxx::KEY::meta( 'n' ), _namedActions.at( action_names::HISTORY_COMMON_PREFIX_SEARCH ) );
+ bind_key( Replxx::KEY::meta( 'N' ), _namedActions.at( action_names::HISTORY_COMMON_PREFIX_SEARCH ) );
+ bind_key( Replxx::KEY::PASTE_START, std::bind( &ReplxxImpl::invoke, this, Replxx::ACTION::BRACKETED_PASTE, _1 ) );
+}
+
+Replxx::ReplxxImpl::~ReplxxImpl( void ) {
+ disable_bracketed_paste();
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::invoke( Replxx::ACTION action_, char32_t code ) {
+ switch ( action_ ) {
+ case ( Replxx::ACTION::INSERT_CHARACTER ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::insert_character, code ) );
+ case ( Replxx::ACTION::NEW_LINE ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::new_line, code ) );
+ case ( Replxx::ACTION::DELETE_CHARACTER_UNDER_CURSOR ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::delete_character, code ) );
+ case ( Replxx::ACTION::DELETE_CHARACTER_LEFT_OF_CURSOR ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::backspace_character, code ) );
+ case ( Replxx::ACTION::KILL_TO_END_OF_LINE ): return ( action( WANT_REFRESH | SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_to_end_of_line, code ) );
+ case ( Replxx::ACTION::KILL_TO_BEGINING_OF_LINE ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_to_begining_of_line, code ) );
+ case ( Replxx::ACTION::KILL_TO_END_OF_WORD ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_word_to_right<false>, code ) );
+ case ( Replxx::ACTION::KILL_TO_BEGINING_OF_WORD ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_word_to_left<false>, code ) );
+ case ( Replxx::ACTION::KILL_TO_END_OF_SUBWORD ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_word_to_right<true>, code ) );
+ case ( Replxx::ACTION::KILL_TO_BEGINING_OF_SUBWORD ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_word_to_left<true>, code ) );
+ case ( Replxx::ACTION::KILL_TO_WHITESPACE_ON_LEFT ): return ( action( SET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::kill_to_whitespace_to_left, code ) );
+ case ( Replxx::ACTION::YANK ): return ( action( HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::yank, code ) );
+ case ( Replxx::ACTION::YANK_CYCLE ): return ( action( HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::yank_cycle, code ) );
+ case ( Replxx::ACTION::YANK_LAST_ARG ): return ( action( HISTORY_RECALL_MOST_RECENT | DONT_RESET_HIST_YANK_INDEX, &Replxx::ReplxxImpl::yank_last_arg, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_TO_BEGINING_OF_LINE ): return ( action( WANT_REFRESH, &Replxx::ReplxxImpl::go_to_begining_of_line, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_TO_END_OF_LINE ): return ( action( WANT_REFRESH, &Replxx::ReplxxImpl::go_to_end_of_line, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_ONE_WORD_LEFT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_word_left<false>, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_ONE_WORD_RIGHT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_word_right<false>, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_ONE_SUBWORD_LEFT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_word_left<true>, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_ONE_SUBWORD_RIGHT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_word_right<true>, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_LEFT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_char_left, code ) );
+ case ( Replxx::ACTION::MOVE_CURSOR_RIGHT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::move_one_char_right, code ) );
+ case ( Replxx::ACTION::HISTORY_NEXT ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::history_next, code ) );
+ case ( Replxx::ACTION::HISTORY_PREVIOUS ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::history_previous, code ) );
+ case ( Replxx::ACTION::HISTORY_FIRST ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::history_first, code ) );
+ case ( Replxx::ACTION::HISTORY_LAST ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::history_last, code ) );
+ case ( Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH ): return ( action( NOOP, &Replxx::ReplxxImpl::incremental_history_search, code ) );
+ case ( Replxx::ACTION::HISTORY_COMMON_PREFIX_SEARCH ): return ( action( RESET_KILL_ACTION | DONT_RESET_PREFIX, &Replxx::ReplxxImpl::common_prefix_search, code ) );
+ case ( Replxx::ACTION::HINT_NEXT ): return ( action( NOOP, &Replxx::ReplxxImpl::hint_next, code ) );
+ case ( Replxx::ACTION::HINT_PREVIOUS ): return ( action( NOOP, &Replxx::ReplxxImpl::hint_previous, code ) );
+ case ( Replxx::ACTION::CAPITALIZE_WORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::capitalize_word<false>, code ) );
+ case ( Replxx::ACTION::LOWERCASE_WORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::lowercase_word<false>, code ) );
+ case ( Replxx::ACTION::UPPERCASE_WORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::uppercase_word<false>, code ) );
+ case ( Replxx::ACTION::CAPITALIZE_SUBWORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::capitalize_word<true>, code ) );
+ case ( Replxx::ACTION::LOWERCASE_SUBWORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::lowercase_word<true>, code ) );
+ case ( Replxx::ACTION::UPPERCASE_SUBWORD ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::uppercase_word<true>, code ) );
+ case ( Replxx::ACTION::TRANSPOSE_CHARACTERS ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::transpose_characters, code ) );
+ case ( Replxx::ACTION::TOGGLE_OVERWRITE_MODE ): return ( action( NOOP, &Replxx::ReplxxImpl::toggle_overwrite_mode, code ) );
+#ifndef _WIN32
+ case ( Replxx::ACTION::VERBATIM_INSERT ): return ( action( WANT_REFRESH | RESET_KILL_ACTION, &Replxx::ReplxxImpl::verbatim_insert, code ) );
+ case ( Replxx::ACTION::SUSPEND ): return ( action( WANT_REFRESH, &Replxx::ReplxxImpl::suspend, code ) );
+#endif
+ case ( Replxx::ACTION::CLEAR_SCREEN ): return ( action( NOOP, &Replxx::ReplxxImpl::clear_screen, code ) );
+ case ( Replxx::ACTION::CLEAR_SELF ): clear_self_to_end_of_screen(); return ( Replxx::ACTION_RESULT::CONTINUE );
+ case ( Replxx::ACTION::REPAINT ): repaint(); return ( Replxx::ACTION_RESULT::CONTINUE );
+ case ( Replxx::ACTION::COMPLETE_LINE ): return ( action( HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::complete_line, code ) );
+ case ( Replxx::ACTION::COMPLETE_NEXT ): return ( action( RESET_KILL_ACTION | DONT_RESET_COMPLETIONS | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::complete_next, code ) );
+ case ( Replxx::ACTION::COMPLETE_PREVIOUS ): return ( action( RESET_KILL_ACTION | DONT_RESET_COMPLETIONS | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::complete_previous, code ) );
+ case ( Replxx::ACTION::COMMIT_LINE ): return ( action( RESET_KILL_ACTION, &Replxx::ReplxxImpl::commit_line, code ) );
+ case ( Replxx::ACTION::ABORT_LINE ): return ( action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::abort_line, code ) );
+ case ( Replxx::ACTION::SEND_EOF ): return ( action( HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::send_eof, code ) );
+ case ( Replxx::ACTION::BRACKETED_PASTE ): return ( action( WANT_REFRESH | RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::bracketed_paste, code ) );
+ }
+ return ( Replxx::ACTION_RESULT::BAIL );
+}
+
+void Replxx::ReplxxImpl::bind_key( char32_t code_, Replxx::key_press_handler_t handler_ ) {
+ _keyPressHandlers[code_] = handler_;
+}
+
+void Replxx::ReplxxImpl::bind_key_internal( char32_t code_, char const* actionName_ ) {
+ named_actions_t::const_iterator it( _namedActions.find( actionName_ ) );
+ if ( it == _namedActions.end() ) {
+ throw std::runtime_error( std::string( "replxx: Unknown action name: " ).append( actionName_ ) );
+ }
+ if ( !! it->second ) {
+ bind_key( code_, it->second );
+ }
+}
+
+Replxx::State Replxx::ReplxxImpl::get_state( void ) const {
+ _utf8Buffer.assign( _data );
+ return ( Replxx::State( _utf8Buffer.get(), _pos ) );
+}
+
+void Replxx::ReplxxImpl::set_state( Replxx::State const& state_ ) {
+ _data.assign( state_.text() );
+ if ( state_.cursor_position() >= 0 ) {
+ _pos = min( state_.cursor_position(), _data.length() );
+ }
+ _modifiedState = true;
+}
+
+char32_t Replxx::ReplxxImpl::read_char( HINT_ACTION hintAction_ ) {
+ /* try scheduled key presses */ {
+ std::lock_guard<std::mutex> l( _mutex );
+ if ( !_keyPresses.empty() ) {
+ char32_t keyPress( _keyPresses.front() );
+ _keyPresses.pop_front();
+ return ( keyPress );
+ }
+ }
+ int hintDelay(
+ _refreshSkipped
+ ? static_cast<int>( RAPID_REFRESH_MS * 2 )
+ : ( hintAction_ != HINT_ACTION::SKIP ? _hintDelay : 0 )
+ );
+ while ( true ) {
+ Terminal::EVENT_TYPE eventType( _terminal.wait_for_input( hintDelay ) );
+ if ( eventType == Terminal::EVENT_TYPE::TIMEOUT ) {
+ refresh_line( _refreshSkipped ? HINT_ACTION::REGENERATE : HINT_ACTION::REPAINT );
+ hintDelay = 0;
+ _refreshSkipped = false;
+ continue;
+ }
+ if ( eventType == Terminal::EVENT_TYPE::KEY_PRESS ) {
+ break;
+ }
+ if ( eventType == Terminal::EVENT_TYPE::RESIZE ) {
+ // caught a window resize event
+ // now redraw the prompt and line
+ _prompt.update_screen_columns();
+ // redraw the original prompt with current input
+ refresh_line( HINT_ACTION::REPAINT );
+ continue;
+ }
+ std::lock_guard<std::mutex> l( _mutex );
+ clear_self_to_end_of_screen();
+ while ( ! _messages.empty() ) {
+ string const& message( _messages.front() );
+ _terminal.write8( message.data(), static_cast<int>( message.length() ) );
+ _messages.pop_front();
+ }
+ repaint();
+ }
+ /* try scheduled key presses */ {
+ std::lock_guard<std::mutex> l( _mutex );
+ if ( !_keyPresses.empty() ) {
+ char32_t keyPress( _keyPresses.front() );
+ _keyPresses.pop_front();
+ return ( keyPress );
+ }
+ }
+ return ( _terminal.read_char() );
+}
+
+void Replxx::ReplxxImpl::clear( void ) {
+ _pos = 0;
+ _prefix = 0;
+ _completions.clear();
+ _completionContextLength = 0;
+ _completionSelection = -1;
+ _data.clear();
+ _hintSelection = -1;
+ _hint = UnicodeString();
+ _display.clear();
+ _displayInputLength = 0;
+}
+
+void Replxx::ReplxxImpl::call_modify_callback( void ) {
+ if ( ! _modifyCallback ) {
+ return;
+ }
+ _utf8Buffer.assign( _data );
+ std::string origLine( _utf8Buffer.get() );
+ int pos( _pos );
+ std::string line( origLine );
+ /* IOModeGuard scope */ {
+ IOModeGuard ioModeGuard( _terminal );
+ _modifyCallback( line, pos );
+ }
+ if ( ( pos != _pos ) || ( line != origLine ) ) {
+ _data.assign( line.c_str() );
+ _pos = min( pos, _data.length() );
+ _modifiedState = true;
+ }
+}
+
+Replxx::ReplxxImpl::completions_t Replxx::ReplxxImpl::call_completer( std::string const& input, int& contextLen_ ) const {
+ Replxx::completions_t completionsIntermediary(
+ !! _completionCallback
+ ? _completionCallback( input, contextLen_ )
+ : Replxx::completions_t()
+ );
+ completions_t completions;
+ completions.reserve( completionsIntermediary.size() );
+ for ( Replxx::Completion const& c : completionsIntermediary ) {
+ completions.emplace_back( c );
+ }
+ return ( completions );
+}
+
+Replxx::ReplxxImpl::hints_t Replxx::ReplxxImpl::call_hinter( std::string const& input, int& contextLen, Replxx::Color& color ) const {
+ Replxx::hints_t hintsIntermediary(
+ !! _hintCallback
+ ? _hintCallback( input, contextLen, color )
+ : Replxx::hints_t()
+ );
+ hints_t hints;
+ hints.reserve( hintsIntermediary.size() );
+ for ( std::string const& h : hintsIntermediary ) {
+ hints.emplace_back( h.c_str() );
+ }
+ return ( hints );
+}
+
+void Replxx::ReplxxImpl::set_preload_buffer( std::string const& preloadText ) {
+ _preloadedBuffer = preloadText;
+ // remove characters that won't display correctly
+ bool controlsStripped = false;
+ int whitespaceSeen( 0 );
+ for ( std::string::iterator it( _preloadedBuffer.begin() ); it != _preloadedBuffer.end(); ) {
+ unsigned char c = *it;
+ if ( '\r' == c ) { // silently skip CR
+ _preloadedBuffer.erase( it, it + 1 );
+ continue;
+ }
+ if ( ( '\n' == c ) || ( '\t' == c ) ) { // note newline or tab
+ ++ whitespaceSeen;
+ ++ it;
+ continue;
+ }
+ if ( whitespaceSeen > 0 ) {
+ it -= whitespaceSeen;
+ *it = ' ';
+ _preloadedBuffer.erase( it + 1, it + whitespaceSeen - 1 );
+ }
+ if ( is_control_code( c ) ) { // remove other control characters, flag for message
+ controlsStripped = true;
+ if ( whitespaceSeen > 0 ) {
+ _preloadedBuffer.erase( it, it + 1 );
+ -- it;
+ } else {
+ *it = ' ';
+ }
+ }
+ whitespaceSeen = 0;
+ ++ it;
+ }
+ if ( whitespaceSeen > 0 ) {
+ std::string::iterator it = _preloadedBuffer.end() - whitespaceSeen;
+ *it = ' ';
+ if ( whitespaceSeen > 1 ) {
+ _preloadedBuffer.erase( it + 1, _preloadedBuffer.end() );
+ }
+ }
+ _errorMessage.clear();
+ if ( controlsStripped ) {
+ _errorMessage.assign( " [Edited line: control characters were converted to spaces]\n" );
+ }
+}
+
+char const* Replxx::ReplxxImpl::read_from_stdin( void ) {
+ if ( _preloadedBuffer.empty() ) {
+ getline( cin, _preloadedBuffer );
+ if ( ! cin.good() ) {
+ return nullptr;
+ }
+ }
+ while ( ! _preloadedBuffer.empty() && ( ( _preloadedBuffer.back() == '\r' ) || ( _preloadedBuffer.back() == '\n' ) ) ) {
+ _preloadedBuffer.pop_back();
+ }
+ _utf8Buffer.assign( _preloadedBuffer );
+ _preloadedBuffer.clear();
+ return _utf8Buffer.get();
+}
+
+void Replxx::ReplxxImpl::emulate_key_press( char32_t keyCode_ ) {
+ std::lock_guard<std::mutex> l( _mutex );
+ _keyPresses.push_back( keyCode_ );
+ if ( ( _currentThread != std::thread::id() ) && ( _currentThread != std::this_thread::get_id() ) ) {
+ _terminal.notify_event( Terminal::EVENT_TYPE::KEY_PRESS );
+ }
+}
+
+char const* Replxx::ReplxxImpl::input( std::string const& prompt ) {
+ try {
+ errno = 0;
+ if ( ! tty::in ) { // input not from a terminal, we should work with piped input, i.e. redirected stdin
+ return ( read_from_stdin() );
+ }
+ if (!_errorMessage.empty()) {
+ printf("%s", _errorMessage.c_str());
+ fflush(stdout);
+ _errorMessage.clear();
+ }
+ if ( isUnsupportedTerm() ) {
+ cout << prompt << flush;
+ fflush(stdout);
+ return ( read_from_stdin() );
+ }
+ if (_terminal.enable_raw_mode() == -1) {
+ return nullptr;
+ }
+ _prompt.set_text( UnicodeString( prompt ) );
+ _currentThread = std::this_thread::get_id();
+ clear();
+ if (!_preloadedBuffer.empty()) {
+ preload_puffer(_preloadedBuffer.c_str());
+ _preloadedBuffer.clear();
+ }
+ if ( get_input_line() == -1 ) {
+ return ( finalize_input( nullptr ) );
+ }
+ _terminal.write8( "\n", 1 );
+ _utf8Buffer.assign( _data );
+ return ( finalize_input( _utf8Buffer.get() ) );
+ } catch ( std::exception const& ) {
+ return ( finalize_input( nullptr ) );
+ }
+}
+
+char const* Replxx::ReplxxImpl::finalize_input( char const* retVal_ ) {
+ _currentThread = std::thread::id();
+ _terminal.disable_raw_mode();
+ return ( retVal_ );
+}
+
+int Replxx::ReplxxImpl::install_window_change_handler( void ) {
+#ifndef _WIN32
+ return ( _terminal.install_window_change_handler() );
+#else
+ return 0;
+#endif
+}
+
+void Replxx::ReplxxImpl::enable_bracketed_paste( void ) {
+ if ( _bracketedPaste ) {
+ return;
+ }
+ _terminal.enable_bracketed_paste();
+ _bracketedPaste = true;
+}
+
+void Replxx::ReplxxImpl::disable_bracketed_paste( void ) {
+ if ( ! _bracketedPaste ) {
+ return;
+ }
+ _terminal.disable_bracketed_paste();
+ _bracketedPaste = false;
+}
+
+void Replxx::ReplxxImpl::print( char const* str_, int size_ ) {
+ if ( ( _currentThread == std::thread::id() ) || ( _currentThread == std::this_thread::get_id() ) ) {
+ _terminal.write8( str_, size_ );
+ } else {
+ std::lock_guard<std::mutex> l( _mutex );
+ _messages.emplace_back( str_, size_ );
+ _terminal.notify_event( Terminal::EVENT_TYPE::MESSAGE );
+ }
+ return;
+}
+
+void Replxx::ReplxxImpl::preload_puffer(const char* preloadText) {
+ _data.assign( preloadText );
+ _prefix = _pos = _data.length();
+}
+
+void Replxx::ReplxxImpl::set_color( Replxx::Color color_ ) {
+ char const* code( ansi_color( color_ ) );
+ while ( *code ) {
+ _display.push_back( *code );
+ ++ code;
+ }
+}
+
+void Replxx::ReplxxImpl::render( char32_t ch ) {
+ if ( ch == Replxx::KEY::ESCAPE ) {
+ _display.push_back( '^' );
+ _display.push_back( '[' );
+ } else if ( is_control_code( ch ) && ( ch != '\n' ) ) {
+ _display.push_back( '^' );
+ _display.push_back( control_to_human( ch ) );
+ } else {
+ _display.push_back( ch );
+ }
+ return;
+}
+
+void Replxx::ReplxxImpl::render( HINT_ACTION hintAction_ ) {
+ if ( hintAction_ == HINT_ACTION::TRIM ) {
+ _display.erase( _display.begin() + _displayInputLength, _display.end() );
+ _modifiedState = false;
+ return;
+ }
+ if ( hintAction_ == HINT_ACTION::SKIP ) {
+ return;
+ }
+ _display.clear();
+ if ( _noColor ) {
+ for ( char32_t ch : _data ) {
+ render( ch );
+ }
+ _displayInputLength = static_cast<int>( _display.size() );
+ _modifiedState = false;
+ return;
+ }
+ Replxx::colors_t colors( _data.length(), Replxx::Color::DEFAULT );
+ _utf8Buffer.assign( _data );
+ if ( !! _highlighterCallback ) {
+ IOModeGuard ioModeGuard( _terminal );
+ _highlighterCallback( _utf8Buffer.get(), colors );
+ }
+ paren_info_t pi( matching_paren() );
+ if ( pi.index != -1 ) {
+ colors[pi.index] = pi.error ? Replxx::Color::ERROR : Replxx::Color::BRIGHTRED;
+ }
+ Replxx::Color c( Replxx::Color::DEFAULT );
+ for ( int i( 0 ); i < _data.length(); ++ i ) {
+ if ( colors[i] != c ) {
+ c = colors[i];
+ set_color( c );
+ }
+ render( _data[i] );
+ }
+ set_color( Replxx::Color::DEFAULT );
+ _displayInputLength = static_cast<int>( _display.size() );
+ _modifiedState = false;
+ return;
+}
+
+int Replxx::ReplxxImpl::handle_hints( HINT_ACTION hintAction_ ) {
+ if ( _noColor ) {
+ return ( 0 );
+ }
+ if ( ! _hintCallback ) {
+ return ( 0 );
+ }
+ if ( ( _hintDelay > 0 ) && ( hintAction_ != HINT_ACTION::REPAINT ) ) {
+ _hintSelection = -1;
+ return ( 0 );
+ }
+ if ( ( hintAction_ == HINT_ACTION::SKIP ) || ( hintAction_ == HINT_ACTION::TRIM ) ) {
+ return ( 0 );
+ }
+ if ( _pos != _data.length() ) {
+ return ( 0 );
+ }
+ _hint = UnicodeString();
+ int len( 0 );
+ if ( hintAction_ == HINT_ACTION::REGENERATE ) {
+ _hintSelection = -1;
+ }
+ _utf8Buffer.assign( _data, _pos );
+ if ( ( _utf8Buffer != _hintSeed ) || ( _hintContextLenght < 0 ) ) {
+ _hintSeed.assign( _utf8Buffer );
+ _hintContextLenght = context_length();
+ _hintColor = Replxx::Color::GRAY;
+ IOModeGuard ioModeGuard( _terminal );
+ _hintsCache = call_hinter( _utf8Buffer.get(), _hintContextLenght, _hintColor );
+ }
+ int hintCount( static_cast<int>( _hintsCache.size() ) );
+ if ( hintCount == 1 ) {
+ _hint = _hintsCache.front();
+ len = _hint.length() - _hintContextLenght;
+ if ( len > 0 ) {
+ set_color( _hintColor );
+ for ( int i( 0 ); i < len; ++ i ) {
+ _display.push_back( _hint[i + _hintContextLenght] );
+ }
+ set_color( Replxx::Color::DEFAULT );
+ }
+ } else if ( ( _maxHintRows > 0 ) && ( hintCount > 0 ) ) {
+ int startCol( _prompt.indentation() + _pos );
+ int maxCol( _prompt.screen_columns() );
+#ifdef _WIN32
+ -- maxCol;
+#endif
+ if ( _hintSelection < -1 ) {
+ _hintSelection = hintCount - 1;
+ } else if ( _hintSelection >= hintCount ) {
+ _hintSelection = -1;
+ }
+ if ( _hintSelection != -1 ) {
+ _hint = _hintsCache[_hintSelection];
+ len = min<int>( _hint.length(), maxCol - startCol );
+ if ( _hintContextLenght < len ) {
+ set_color( _hintColor );
+ for ( int i( _hintContextLenght ); i < len; ++ i ) {
+ _display.push_back( _hint[i] );
+ }
+ set_color( Replxx::Color::DEFAULT );
+ }
+ }
+ startCol -= _hintContextLenght;
+ for ( int hintRow( 0 ); hintRow < min( hintCount, _maxHintRows ); ++ hintRow ) {
+#ifdef _WIN32
+ _display.push_back( '\r' );
+#endif
+ _display.push_back( '\n' );
+ int col( 0 );
+ for ( int i( 0 ); ( i < startCol ) && ( col < maxCol ); ++ i, ++ col ) {
+ _display.push_back( ' ' );
+ }
+ set_color( _hintColor );
+ for ( int i( _pos - _hintContextLenght ); ( i < _pos ) && ( col < maxCol ); ++ i, ++ col ) {
+ _display.push_back( _data[i] );
+ }
+ int hintNo( hintRow + _hintSelection + 1 );
+ if ( hintNo == hintCount ) {
+ continue;
+ } else if ( hintNo > hintCount ) {
+ -- hintNo;
+ }
+ UnicodeString const& h( _hintsCache[hintNo % hintCount] );
+ for ( int i( _hintContextLenght ); ( i < h.length() ) && ( col < maxCol ); ++ i, ++ col ) {
+ _display.push_back( h[i] );
+ }
+ set_color( Replxx::Color::DEFAULT );
+ }
+ }
+ return ( len );
+}
+
+Replxx::ReplxxImpl::paren_info_t Replxx::ReplxxImpl::matching_paren( void ) {
+ if (_pos >= _data.length()) {
+ return ( paren_info_t{ -1, false } );
+ }
+ /* this scans for a brace matching _data[_pos] to highlight */
+ unsigned char part1, part2;
+ int scanDirection = 0;
+ if ( strchr("}])", _data[_pos]) ) {
+ scanDirection = -1; /* backwards */
+ if (_data[_pos] == '}') {
+ part1 = '}'; part2 = '{';
+ } else if (_data[_pos] == ']') {
+ part1 = ']'; part2 = '[';
+ } else {
+ part1 = ')'; part2 = '(';
+ }
+ } else if ( strchr("{[(", _data[_pos]) ) {
+ scanDirection = 1; /* forwards */
+ if (_data[_pos] == '{') {
+ //part1 = '{'; part2 = '}';
+ part1 = '}'; part2 = '{';
+ } else if (_data[_pos] == '[') {
+ //part1 = '['; part2 = ']';
+ part1 = ']'; part2 = '[';
+ } else {
+ //part1 = '('; part2 = ')';
+ part1 = ')'; part2 = '(';
+ }
+ } else {
+ return ( paren_info_t{ -1, false } );
+ }
+ int highlightIdx = -1;
+ bool indicateError = false;
+ int unmatched = scanDirection;
+ int unmatchedOther = 0;
+ for (int i = _pos + scanDirection; i >= 0 && i < _data.length(); i += scanDirection) {
+ /* TODO: the right thing when inside a string */
+ if (strchr("}])", _data[i])) {
+ if (_data[i] == part1) {
+ --unmatched;
+ } else {
+ --unmatchedOther;
+ }
+ } else if (strchr("{[(", _data[i])) {
+ if (_data[i] == part2) {
+ ++unmatched;
+ } else {
+ ++unmatchedOther;
+ }
+ }
+
+ if (unmatched == 0) {
+ highlightIdx = i;
+ indicateError = (unmatchedOther != 0);
+ break;
+ }
+ }
+ return ( paren_info_t{ highlightIdx, indicateError } );
+}
+
+/**
+ * Refresh the user's input line: the prompt is already onscreen and is not
+ * redrawn here screen position
+ */
+void Replxx::ReplxxImpl::refresh_line( HINT_ACTION hintAction_ ) {
+ int long long now( now_us() );
+ int long long duration( now - _lastRefreshTime );
+ if ( duration < RAPID_REFRESH_US ) {
+ _lastRefreshTime = now;
+ _refreshSkipped = true;
+ return;
+ }
+ _refreshSkipped = false;
+ // check for a matching brace/bracket/paren, remember its position if found
+ render( hintAction_ );
+ int hintLen( handle_hints( hintAction_ ) );
+ // calculate the position of the end of the input line
+ int xEndOfInput( 0 ), yEndOfInput( 0 );
+ calculate_screen_position(
+ _prompt.indentation(), 0, _prompt.screen_columns(),
+ calculate_displayed_length( _data.get(), _data.length() ) + hintLen,
+ xEndOfInput, yEndOfInput
+ );
+ yEndOfInput += static_cast<int>( count( _display.begin(), _display.end(), '\n' ) );
+
+ // calculate the desired position of the cursor
+ int xCursorPos( 0 ), yCursorPos( 0 );
+ calculate_screen_position(
+ _prompt.indentation(), 0, _prompt.screen_columns(),
+ calculate_displayed_length( _data.get(), _pos ),
+ xCursorPos, yCursorPos
+ );
+
+ // position at the end of the prompt, clear to end of previous input
+ _terminal.set_cursor_visible( false );
+ _terminal.jump_cursor(
+ _prompt.indentation(), // 0-based on Win32
+ -( _prompt._cursorRowOffset - _prompt._extraLines )
+ );
+ // display the input line
+ _terminal.write32( _display.data(), _displayInputLength );
+ _terminal.clear_screen( Terminal::CLEAR_SCREEN::TO_END );
+ _terminal.write32( _display.data() + _displayInputLength, static_cast<int>( _display.size() ) - _displayInputLength );
+#ifndef _WIN32
+ // we have to generate our own newline on line wrap
+ if ( ( xEndOfInput == 0 ) && ( yEndOfInput > 0 ) ) {
+ _terminal.write8( "\n", 1 );
+ }
+#endif
+ // position the cursor
+ _terminal.jump_cursor( xCursorPos, -( yEndOfInput - yCursorPos ) );
+ _terminal.set_cursor_visible( true );
+ _prompt._cursorRowOffset = _prompt._extraLines + yCursorPos; // remember row for next pass
+ _lastRefreshTime = now_us();
+}
+
+int Replxx::ReplxxImpl::context_length() {
+ int prefixLength = _pos;
+ while ( prefixLength > 0 ) {
+ if ( is_word_break_character<false>( _data[prefixLength - 1] ) ) {
+ break;
+ }
+ -- prefixLength;
+ }
+ return ( _pos - prefixLength );
+}
+
+void Replxx::ReplxxImpl::repaint( void ) {
+ _prompt.write();
+ for ( int i( _prompt._extraLines ); i < _prompt._cursorRowOffset; ++ i ) {
+ _terminal.write8( "\n", 1 );
+ }
+ refresh_line( HINT_ACTION::SKIP );
+}
+
+void Replxx::ReplxxImpl::clear_self_to_end_of_screen( Prompt const* prompt_ ) {
+ // position at the start of the prompt, clear to end of previous input
+ _terminal.jump_cursor( 0, prompt_ ? -prompt_->_cursorRowOffset : -_prompt._cursorRowOffset );
+ _terminal.clear_screen( Terminal::CLEAR_SCREEN::TO_END );
+ return;
+}
+
+namespace {
+int longest_common_prefix( Replxx::ReplxxImpl::completions_t const& completions ) {
+ int completionsCount( static_cast<int>( completions.size() ) );
+ if ( completionsCount < 1 ) {
+ return ( 0 );
+ }
+ int longestCommonPrefix( 0 );
+ UnicodeString const& sample( completions.front().text() );
+ while ( true ) {
+ if ( longestCommonPrefix >= sample.length() ) {
+ return ( longestCommonPrefix );
+ }
+ char32_t sc( sample[longestCommonPrefix] );
+ for ( int i( 1 ); i < completionsCount; ++ i ) {
+ UnicodeString const& candidate( completions[i].text() );
+ if ( longestCommonPrefix >= candidate.length() ) {
+ return ( longestCommonPrefix );
+ }
+ char32_t cc( candidate[longestCommonPrefix] );
+ if ( cc != sc ) {
+ return ( longestCommonPrefix );
+ }
+ }
+ ++ longestCommonPrefix;
+ }
+}
+}
+
+/**
+ * Handle command completion, using a completionCallback() routine to provide
+ * possible substitutions
+ * This routine handles the mechanics of updating the user's input buffer with
+ * possible replacement of text as the user selects a proposed completion string,
+ * or cancels the completion attempt.
+ * @param pi - Prompt struct holding information about the prompt and our
+ * screen position
+ */
+char32_t Replxx::ReplxxImpl::do_complete_line( bool showCompletions_ ) {
+ char32_t c = 0;
+
+ // completionCallback() expects a parsable entity, so find the previous break
+ // character and
+ // extract a copy to parse. we also handle the case where tab is hit while
+ // not at end-of-line.
+
+ _utf8Buffer.assign( _data, _pos );
+ // get a list of completions
+ _completionSelection = -1;
+ _completionContextLength = context_length();
+ /* IOModeGuard scope */ {
+ IOModeGuard ioModeGuard( _terminal );
+ _completions = call_completer( _utf8Buffer.get(), _completionContextLength );
+ }
+
+ // if no completions, we are done
+ if ( _completions.empty() ) {
+ beep();
+ return 0;
+ }
+
+ // at least one completion
+ int longestCommonPrefix = 0;
+ int completionsCount( static_cast<int>( _completions.size() ) );
+ int selectedCompletion( 0 );
+ if ( _hintSelection != -1 ) {
+ selectedCompletion = _hintSelection;
+ completionsCount = 1;
+ }
+ if ( completionsCount == 1 ) {
+ longestCommonPrefix = static_cast<int>( _completions[selectedCompletion].text().length() );
+ } else {
+ longestCommonPrefix = longest_common_prefix( _completions );
+ }
+ if ( _beepOnAmbiguousCompletion && ( completionsCount != 1 ) ) { // beep if ambiguous
+ beep();
+ }
+
+ // if we can extend the item, extend it and return to main loop
+ if ( ( longestCommonPrefix > _completionContextLength ) || ( completionsCount == 1 ) ) {
+ _pos -= _completionContextLength;
+ _data.erase( _pos, _completionContextLength );
+ _data.insert( _pos, _completions[selectedCompletion].text(), 0, longestCommonPrefix );
+ _pos = _pos + longestCommonPrefix;
+ _completionContextLength = longestCommonPrefix;
+ refresh_line();
+ return 0;
+ }
+
+ if ( ! showCompletions_ ) {
+ return ( 0 );
+ }
+
+ if ( _doubleTabCompletion ) {
+ // we can't complete any further, wait for second tab
+ do {
+ c = read_char();
+ } while ( c == static_cast<char32_t>( -1 ) );
+
+ // if any character other than tab, pass it to the main loop
+ if ( c != Replxx::KEY::TAB ) {
+ return c;
+ }
+ }
+
+ // we got a second tab, maybe show list of possible completions
+ bool showCompletions = true;
+ bool onNewLine = false;
+ if ( static_cast<int>( _completions.size() ) > _completionCountCutoff ) {
+ int savePos = _pos; // move cursor to EOL to avoid overwriting the command line
+ _pos = _data.length();
+ refresh_line();
+ _pos = savePos;
+ printf( "\nDisplay all %u possibilities? (y or n)", static_cast<unsigned int>( _completions.size() ) );
+ fflush(stdout);
+ onNewLine = true;
+ while (c != 'y' && c != 'Y' && c != 'n' && c != 'N' && c != Replxx::KEY::control('C')) {
+ do {
+ c = read_char();
+ } while (c == static_cast<char32_t>(-1));
+ }
+ switch (c) {
+ case 'n':
+ case 'N':
+ showCompletions = false;
+ break;
+ case Replxx::KEY::control('C'):
+ showCompletions = false;
+ // Display the ^C we got
+ _terminal.write8( "^C", 2 );
+ c = 0;
+ break;
+ }
+ }
+
+ // if showing the list, do it the way readline does it
+ bool stopList( false );
+ if ( showCompletions ) {
+ int longestCompletion( 0 );
+ for ( size_t j( 0 ); j < _completions.size(); ++ j ) {
+ int itemLength( static_cast<int>( _completions[j].text().length() ) );
+ if ( itemLength > longestCompletion ) {
+ longestCompletion = itemLength;
+ }
+ }
+ longestCompletion += 2;
+ int columnCount = _prompt.screen_columns() / longestCompletion;
+ if ( columnCount < 1 ) {
+ columnCount = 1;
+ }
+ if ( ! onNewLine ) { // skip this if we showed "Display all %d possibilities?"
+ int savePos = _pos; // move cursor to EOL to avoid overwriting the command line
+ _pos = _data.length();
+ refresh_line( HINT_ACTION::TRIM );
+ _pos = savePos;
+ } else {
+ _terminal.clear_screen( Terminal::CLEAR_SCREEN::TO_END );
+ }
+ size_t pauseRow = _terminal.get_screen_rows() - 1;
+ size_t rowCount = (_completions.size() + columnCount - 1) / columnCount;
+ for (size_t row = 0; row < rowCount; ++row) {
+ if (row == pauseRow) {
+ printf("\n--More--");
+ fflush(stdout);
+ c = 0;
+ bool doBeep = false;
+ while (c != ' ' && c != Replxx::KEY::ENTER && c != 'y' && c != 'Y' &&
+ c != 'n' && c != 'N' && c != 'q' && c != 'Q' &&
+ c != Replxx::KEY::control('C')) {
+ if (doBeep) {
+ beep();
+ }
+ doBeep = true;
+ do {
+ c = read_char();
+ } while (c == static_cast<char32_t>(-1));
+ }
+ switch (c) {
+ case ' ':
+ case 'y':
+ case 'Y':
+ printf("\r \r");
+ pauseRow += _terminal.get_screen_rows() - 1;
+ break;
+ case Replxx::KEY::ENTER:
+ printf("\r \r");
+ ++pauseRow;
+ break;
+ case 'n':
+ case 'N':
+ case 'q':
+ case 'Q':
+ printf("\r \r");
+ stopList = true;
+ break;
+ case Replxx::KEY::control('C'):
+ // Display the ^C we got
+ _terminal.write8( "^C", 2 );
+ stopList = true;
+ break;
+ }
+ } else {
+ _terminal.write8( "\n", 1 );
+ }
+ if (stopList) {
+ break;
+ }
+ static UnicodeString const res( ansi_color( Replxx::Color::DEFAULT ) );
+ for (int column = 0; column < columnCount; ++column) {
+ size_t index = (column * rowCount) + row;
+ if ( index < _completions.size() ) {
+ Completion const& c( _completions[index] );
+ int itemLength = static_cast<int>(c.text().length());
+ fflush(stdout);
+
+ if ( longestCommonPrefix > 0 ) {
+ static UnicodeString const col( ansi_color( Replxx::Color::BRIGHTMAGENTA ) );
+ if (!_noColor) {
+ _terminal.write32(col.get(), col.length());
+ }
+ _terminal.write32(&_data[_pos - _completionContextLength], longestCommonPrefix);
+ if (!_noColor) {
+ _terminal.write32(res.get(), res.length());
+ }
+ }
+
+ if ( !_noColor && ( c.color() != Replxx::Color::DEFAULT ) ) {
+ UnicodeString ac( ansi_color( c.color() ) );
+ _terminal.write32( ac.get(), ac.length() );
+ }
+ _terminal.write32( c.text().get() + longestCommonPrefix, itemLength - longestCommonPrefix );
+ if ( !_noColor && ( c.color() != Replxx::Color::DEFAULT ) ) {
+ _terminal.write32( res.get(), res.length() );
+ }
+
+ if ( ((column + 1) * rowCount) + row < _completions.size() ) {
+ for ( int k( itemLength ); k < longestCompletion; ++k ) {
+ printf( " " );
+ }
+ }
+ }
+ }
+ }
+ fflush(stdout);
+ }
+
+ // display the prompt on a new line, then redisplay the input buffer
+ if (!stopList || c == Replxx::KEY::control('C')) {
+ _terminal.write8( "\n", 1 );
+ }
+ _prompt.write();
+ _prompt._cursorRowOffset = _prompt._extraLines;
+ refresh_line();
+ return 0;
+}
+
+int Replxx::ReplxxImpl::get_input_line( void ) {
+ // The latest history entry is always our current buffer
+ if ( _data.length() > 0 ) {
+ _history.add( _data );
+ } else {
+ _history.add( UnicodeString() );
+ }
+ _history.jump( false, false );
+
+ // display the prompt
+ _prompt.write();
+
+ // the cursor starts out at the end of the prompt
+ _prompt._cursorRowOffset = _prompt._extraLines;
+
+ // kill and yank start in "other" mode
+ _killRing.lastAction = KillRing::actionOther;
+
+ // if there is already text in the buffer, display it first
+ if (_data.length() > 0) {
+ refresh_line();
+ }
+
+ // loop collecting characters, respond to line editing characters
+ Replxx::ACTION_RESULT next( Replxx::ACTION_RESULT::CONTINUE );
+ while ( next == Replxx::ACTION_RESULT::CONTINUE ) {
+ int c( read_char( HINT_ACTION::REPAINT ) ); // get a new keystroke
+
+ if (c == 0) {
+ return _data.length();
+ }
+
+ if (c == -1) {
+ refresh_line();
+ continue;
+ }
+
+ if (c == -2) {
+ _prompt.write();
+ refresh_line();
+ continue;
+ }
+
+ key_press_handlers_t::iterator it( _keyPressHandlers.find( c ) );
+ if ( it != _keyPressHandlers.end() ) {
+ next = it->second( c );
+ if ( _modifiedState ) {
+ refresh_line();
+ }
+ } else {
+ next = action( RESET_KILL_ACTION | HISTORY_RECALL_MOST_RECENT, &Replxx::ReplxxImpl::insert_character, c );
+ }
+ }
+ return ( next == Replxx::ACTION_RESULT::RETURN ? _data.length() : -1 );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::action( action_trait_t actionTrait_, key_press_handler_raw_t const& handler_, char32_t code_ ) {
+ Replxx::ACTION_RESULT res( ( this->*handler_ )( code_ ) );
+ call_modify_callback();
+ if ( actionTrait_ & HISTORY_RECALL_MOST_RECENT ) {
+ _history.reset_recall_most_recent();
+ }
+ if ( actionTrait_ & RESET_KILL_ACTION ) {
+ _killRing.lastAction = KillRing::actionOther;
+ }
+ if ( actionTrait_ & SET_KILL_ACTION ) {
+ _killRing.lastAction = KillRing::actionKill;
+ }
+ if ( ! ( actionTrait_ & DONT_RESET_PREFIX ) ) {
+ _prefix = _pos;
+ }
+ if ( ! ( actionTrait_ & DONT_RESET_COMPLETIONS ) ) {
+ _completions.clear();
+ _completionSelection = -1;
+ _completionContextLength = 0;
+ }
+ if ( ! ( actionTrait_ & DONT_RESET_HIST_YANK_INDEX ) ) {
+ _history.reset_yank_iterator();
+ }
+ if ( actionTrait_ & WANT_REFRESH ) {
+ _modifiedState = true;
+ }
+ return ( res );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::insert_character( char32_t c ) {
+ /*
+ * beep on unknown Ctrl and/or Meta keys
+ * don't insert control characters
+ */
+ if ( ( c >= static_cast<int>( Replxx::KEY::BASE ) ) || ( is_control_code( c ) && ( c != '\n' ) ) ) {
+ beep();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ if ( ! _overwrite || ( _pos >= _data.length() ) ) {
+ _data.insert( _pos, c );
+ } else {
+ _data[_pos] = c;
+ }
+ ++ _pos;
+ call_modify_callback();
+ int long long now( now_us() );
+ int long long duration( now - _lastRefreshTime );
+ if ( duration < RAPID_REFRESH_US ) {
+ _lastRefreshTime = now;
+ _refreshSkipped = true;
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ int inputLen = calculate_displayed_length( _data.get(), _data.length() );
+ if (
+ ( _pos == _data.length() )
+ && ! _modifiedState
+ && ( _noColor || ! ( !! _highlighterCallback || !! _hintCallback ) )
+ && ( _prompt.indentation() + inputLen < _prompt.screen_columns() )
+ ) {
+ /* Avoid a full assign of the line in the
+ * trivial case. */
+ render( c );
+ _displayInputLength = static_cast<int>( _display.size() );
+ _terminal.write32( reinterpret_cast<char32_t*>( &c ), 1 );
+ } else {
+ refresh_line();
+ }
+ _lastRefreshTime = now_us();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-J/linefeed/newline
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::new_line( char32_t ) {
+ return ( insert_character( '\n' ) );
+}
+
+// ctrl-A, HOME: move cursor to start of line
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::go_to_begining_of_line( char32_t ) {
+ _pos = 0;
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::go_to_end_of_line( char32_t ) {
+ _pos = _data.length();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-B, move cursor left by one character
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::move_one_char_left( char32_t ) {
+ if (_pos > 0) {
+ --_pos;
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-F, move cursor right by one character
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::move_one_char_right( char32_t ) {
+ if ( _pos < _data.length() ) {
+ ++_pos;
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-B, move cursor left by one word
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::move_one_word_left( char32_t ) {
+ if (_pos > 0) {
+ while (_pos > 0 && is_word_break_character<subword>( _data[_pos - 1] ) ) {
+ --_pos;
+ }
+ while (_pos > 0 && !is_word_break_character<subword>( _data[_pos - 1] ) ) {
+ --_pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-f, move cursor right by one word
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::move_one_word_right( char32_t ) {
+ if ( _pos < _data.length() ) {
+ while ( _pos < _data.length() && is_word_break_character<subword>( _data[_pos] ) ) {
+ ++_pos;
+ }
+ while ( _pos < _data.length() && !is_word_break_character<subword>( _data[_pos] ) ) {
+ ++_pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-Backspace, kill word to left of cursor
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::kill_word_to_left( char32_t ) {
+ if ( _pos > 0 ) {
+ int startingPos = _pos;
+ while ( _pos > 0 && is_word_break_character<subword>( _data[_pos - 1] ) ) {
+ -- _pos;
+ }
+ while ( _pos > 0 && !is_word_break_character<subword>( _data[_pos - 1] ) ) {
+ -- _pos;
+ }
+ _killRing.kill( _data.get() + _pos, startingPos - _pos, false);
+ _data.erase( _pos, startingPos - _pos );
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-D, kill word to right of cursor
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::kill_word_to_right( char32_t ) {
+ if ( _pos < _data.length() ) {
+ int endingPos = _pos;
+ while ( endingPos < _data.length() && is_word_break_character<subword>( _data[endingPos] ) ) {
+ ++ endingPos;
+ }
+ while ( endingPos < _data.length() && !is_word_break_character<subword>( _data[endingPos] ) ) {
+ ++ endingPos;
+ }
+ _killRing.kill( _data.get() + _pos, endingPos - _pos, true );
+ _data.erase( _pos, endingPos - _pos );
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-W, kill to whitespace (not word) to left of cursor
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::kill_to_whitespace_to_left( char32_t ) {
+ if ( _pos > 0 ) {
+ int startingPos = _pos;
+ while ( ( _pos > 0 ) && isspace( _data[_pos - 1] ) ) {
+ --_pos;
+ }
+ while ( ( _pos > 0 ) && ! isspace( _data[_pos - 1] ) ) {
+ -- _pos;
+ }
+ _killRing.kill( _data.get() + _pos, startingPos - _pos, false );
+ _data.erase( _pos, startingPos - _pos );
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-K, kill from cursor to end of line
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::kill_to_end_of_line( char32_t ) {
+ _killRing.kill( _data.get() + _pos, _data.length() - _pos, true );
+ _data.erase( _pos, _data.length() - _pos );
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-U, kill all characters to the left of the cursor
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::kill_to_begining_of_line( char32_t ) {
+ if (_pos > 0) {
+ _killRing.kill( _data.get(), _pos, false );
+ _data.erase( 0, _pos );
+ _pos = 0;
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-Y, yank killed text
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::yank( char32_t ) {
+ UnicodeString* restoredText( _killRing.yank() );
+ if ( restoredText ) {
+ _data.insert( _pos, *restoredText, 0, restoredText->length() );
+ _pos += restoredText->length();
+ refresh_line();
+ _killRing.lastAction = KillRing::actionYank;
+ _lastYankSize = restoredText->length();
+ } else {
+ beep();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-Y, "yank-pop", rotate popped text
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::yank_cycle( char32_t ) {
+ if ( _killRing.lastAction != KillRing::actionYank ) {
+ beep();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ UnicodeString* restoredText = _killRing.yankPop();
+ if ( !restoredText ) {
+ beep();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ _pos -= _lastYankSize;
+ _data.erase( _pos, _lastYankSize );
+ _data.insert( _pos, *restoredText, 0, restoredText->length() );
+ _pos += restoredText->length();
+ _lastYankSize = restoredText->length();
+ refresh_line();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-., "yank-last-arg", on consecutive uses move back in history for popped text
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::yank_last_arg( char32_t ) {
+ if ( _history.size() < 2 ) {
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ if ( _history.next_yank_position() ) {
+ _lastYankSize = 0;
+ }
+ UnicodeString const& histLine( _history.yank_line() );
+ int endPos( histLine.length() );
+ while ( ( endPos > 0 ) && isspace( histLine[endPos - 1] ) ) {
+ -- endPos;
+ }
+ int startPos( endPos );
+ while ( ( startPos > 0 ) && ! isspace( histLine[startPos - 1] ) ) {
+ -- startPos;
+ }
+ _pos -= _lastYankSize;
+ _data.erase( _pos, _lastYankSize );
+ _lastYankSize = endPos - startPos;
+ _data.insert( _pos, histLine, startPos, _lastYankSize );
+ _pos += _lastYankSize;
+ refresh_line();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-C, give word initial Cap
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::capitalize_word( char32_t ) {
+ if (_pos < _data.length()) {
+ while ( _pos < _data.length() && is_word_break_character<subword>( _data[_pos] ) ) {
+ ++_pos;
+ }
+ if (_pos < _data.length() && !is_word_break_character<subword>( _data[_pos] ) ) {
+ if ( _data[_pos] >= 'a' && _data[_pos] <= 'z' ) {
+ _data[_pos] += 'A' - 'a';
+ }
+ ++_pos;
+ }
+ while (_pos < _data.length() && !is_word_break_character<subword>( _data[_pos] ) ) {
+ if ( _data[_pos] >= 'A' && _data[_pos] <= 'Z' ) {
+ _data[_pos] += 'a' - 'A';
+ }
+ ++_pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-L, lowercase word
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::lowercase_word( char32_t ) {
+ if (_pos < _data.length()) {
+ while ( _pos < _data.length() && is_word_break_character<subword>( _data[_pos] ) ) {
+ ++ _pos;
+ }
+ while (_pos < _data.length() && !is_word_break_character<subword>( _data[_pos] ) ) {
+ if ( _data[_pos] >= 'A' && _data[_pos] <= 'Z' ) {
+ _data[_pos] += 'a' - 'A';
+ }
+ ++ _pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-U, uppercase word
+template <bool subword>
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::uppercase_word( char32_t ) {
+ if (_pos < _data.length()) {
+ while ( _pos < _data.length() && is_word_break_character<subword>( _data[_pos] ) ) {
+ ++ _pos;
+ }
+ while ( _pos < _data.length() && !is_word_break_character<subword>( _data[_pos] ) ) {
+ if ( _data[_pos] >= 'a' && _data[_pos] <= 'z') {
+ _data[_pos] += 'A' - 'a';
+ }
+ ++ _pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-T, transpose characters
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::transpose_characters( char32_t ) {
+ if ( _pos > 0 && _data.length() > 1 ) {
+ size_t leftCharPos = ( _pos == _data.length() ) ? _pos - 2 : _pos - 1;
+ char32_t aux = _data[leftCharPos];
+ _data[leftCharPos] = _data[leftCharPos + 1];
+ _data[leftCharPos + 1] = aux;
+ if ( _pos != _data.length() ) {
+ ++_pos;
+ }
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-C, abort this line
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::abort_line( char32_t ) {
+ errno = EAGAIN;
+ _history.drop_last();
+ // we need one last refresh with the cursor at the end of the line
+ // so we don't display the next prompt over the previous input line
+ _pos = _data.length(); // pass _data.length() as _pos for EOL
+ _lastRefreshTime = 0;
+ refresh_line( _refreshSkipped ? HINT_ACTION::REGENERATE : HINT_ACTION::TRIM );
+ _terminal.write8( "^C\r\n", 4 );
+ return ( Replxx::ACTION_RESULT::BAIL );
+}
+
+// DEL, delete the character under the cursor
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::delete_character( char32_t ) {
+ if ( ( _data.length() > 0 ) && ( _pos < _data.length() ) ) {
+ _data.erase( _pos );
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-D, delete the character under the cursor
+// on an empty line, exit the shell
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::send_eof( char32_t key_ ) {
+ if ( _data.length() == 0 ) {
+ _history.drop_last();
+ return ( Replxx::ACTION_RESULT::BAIL );
+ }
+ return ( delete_character( key_ ) );
+}
+
+// backspace/ctrl-H, delete char to left of cursor
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::backspace_character( char32_t ) {
+ if ( _pos > 0 ) {
+ -- _pos;
+ _data.erase( _pos );
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-M/return/enter, accept line
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::commit_line( char32_t ) {
+ // we need one last refresh with the cursor at the end of the line
+ // so we don't display the next prompt over the previous input line
+ _pos = _data.length(); // pass _data.length() as _pos for EOL
+ _lastRefreshTime = 0;
+ refresh_line( _refreshSkipped ? HINT_ACTION::REGENERATE : HINT_ACTION::TRIM );
+ _history.commit_index();
+ _history.drop_last();
+ return ( Replxx::ACTION_RESULT::RETURN );
+}
+
+// Down, recall next line in history
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_next( char32_t ) {
+ return ( history_move( false ) );
+}
+
+// Up, recall previous line in history
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_previous( char32_t ) {
+ return ( history_move( true ) );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_move( bool previous_ ) {
+ // if not already recalling, add the current line to the history list so
+ // we don't
+ // have to special case it
+ if ( _history.is_last() ) {
+ _history.update_last( _data );
+ }
+ if ( _history.is_empty() ) {
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ if ( ! _history.move( previous_ ) ) {
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ _data.assign( _history.current() );
+ _pos = _data.length();
+ refresh_line();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// meta-<, beginning of history
+// Page Up, beginning of history
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_first( char32_t ) {
+ return ( history_jump( true ) );
+}
+
+// meta->, end of history
+// Page Down, end of history
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_last( char32_t ) {
+ return ( history_jump( false ) );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::history_jump( bool back_ ) {
+ // if not already recalling, add the current line to the history list so
+ // we don't
+ // have to special case it
+ if ( _history.is_last() ) {
+ _history.update_last( _data );
+ }
+ if ( ! _history.is_empty() ) {
+ _history.jump( back_ );
+ _data.assign( _history.current() );
+ _pos = _data.length();
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::hint_next( char32_t ) {
+ return ( hint_move( false ) );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::hint_previous( char32_t ) {
+ return ( hint_move( true ) );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::hint_move( bool previous_ ) {
+ if ( ! _noColor ) {
+ _killRing.lastAction = KillRing::actionOther;
+ if ( previous_ ) {
+ -- _hintSelection;
+ } else {
+ ++ _hintSelection;
+ }
+ refresh_line( HINT_ACTION::REPAINT );
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::toggle_overwrite_mode( char32_t ) {
+ _overwrite = ! _overwrite;
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+#ifndef _WIN32
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::verbatim_insert( char32_t ) {
+ static int const MAX_ESC_SEQ( 32 );
+ char32_t buf[MAX_ESC_SEQ];
+ int len( _terminal.read_verbatim( buf, MAX_ESC_SEQ ) );
+ _data.insert( _pos, UnicodeString( buf, len ), 0, len );
+ _pos += len;
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-Z, job control
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::suspend( char32_t ) {
+ /* IOModeGuard scope */ {
+ IOModeGuard ioModeGuard( _terminal );
+ raise( SIGSTOP ); // Break out in mid-line
+ }
+ // Redraw prompt
+ _prompt.write();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+#endif
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::complete_line( char32_t c ) {
+ if ( !! _completionCallback && ( _completeOnEmpty || ( _pos > 0 ) ) ) {
+ // complete_line does the actual completion and replacement
+ c = do_complete_line( c != 0 );
+
+ if ( static_cast<int>( c ) < 0 ) {
+ return ( Replxx::ACTION_RESULT::BAIL );
+ }
+ if ( c != 0 ) {
+ emulate_key_press( c );
+ }
+ } else {
+ insert_character( c );
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::complete( bool previous_ ) {
+ if ( _completions.empty() ) {
+ bool first( _completions.empty() );
+ int dataLen( _data.length() );
+ complete_line( 0 );
+ if ( ! _immediateCompletion && first && ( _data.length() > dataLen ) ) {
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+ }
+ }
+ int newSelection( _completionSelection + ( previous_ ? -1 : 1 ) );
+ if ( newSelection >= static_cast<int>( _completions.size() ) ) {
+ newSelection = -1;
+ } else if ( newSelection == -2 ) {
+ newSelection = static_cast<int>( _completions.size() ) - 1;
+ }
+ if ( _completionSelection != -1 ) {
+ int oldCompletionLength( max( _completions[_completionSelection].text().length() - _completionContextLength, 0 ) );
+ _pos -= oldCompletionLength;
+ _data.erase( _pos, oldCompletionLength );
+ }
+ if ( newSelection != -1 ) {
+ int newCompletionLength( max( _completions[newSelection].text().length() - _completionContextLength, 0 ) );
+ _data.insert( _pos, _completions[newSelection].text(), _completionContextLength, newCompletionLength );
+ _pos += newCompletionLength;
+ }
+ _completionSelection = newSelection;
+ refresh_line(); // Refresh the line
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::complete_next( char32_t ) {
+ return ( complete( false ) );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::complete_previous( char32_t ) {
+ return ( complete( true ) );
+}
+
+// Alt-P, reverse history search for prefix
+// Alt-P, reverse history search for prefix
+// Alt-N, forward history search for prefix
+// Alt-N, forward history search for prefix
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::common_prefix_search( char32_t startChar ) {
+ int prefixSize( calculate_displayed_length( _data.get(), _prefix ) );
+ if (
+ _history.common_prefix_search(
+ _data, prefixSize, ( startChar == ( Replxx::KEY::meta( 'p' ) ) ) || ( startChar == ( Replxx::KEY::meta( 'P' ) ) )
+ )
+ ) {
+ _data.assign( _history.current() );
+ _pos = _data.length();
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-R, reverse history search
+// ctrl-S, forward history search
+/**
+ * Incremental history search -- take over the prompt and keyboard as the user
+ * types a search string, deletes characters from it, changes _direction,
+ * and either accepts the found line (for execution orediting) or cancels.
+ * @param startChar - the character that began the search, used to set the initial
+ * _direction
+ */
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::incremental_history_search( char32_t startChar ) {
+ // if not already recalling, add the current line to the history list so we
+ // don't have to special case it
+ if ( _history.is_last() ) {
+ _history.update_last( _data );
+ }
+ _history.save_pos();
+ int historyLinePosition( _pos );
+ clear_self_to_end_of_screen();
+
+ DynamicPrompt dp( _terminal, (startChar == Replxx::KEY::control('R')) ? -1 : 1 );
+
+ // draw user's text with our prompt
+ dynamicRefresh(_prompt, dp, _data.get(), _data.length(), historyLinePosition);
+
+ // loop until we get an exit character
+ char32_t c( 0 );
+ bool keepLooping = true;
+ bool useSearchedLine = true;
+ bool searchAgain = false;
+ UnicodeString activeHistoryLine;
+ while ( keepLooping ) {
+ c = read_char();
+
+ switch (c) {
+ // these characters keep the selected text but do not execute it
+ case Replxx::KEY::control('A'): // ctrl-A, move cursor to start of line
+ case Replxx::KEY::HOME:
+ case Replxx::KEY::control('B'): // ctrl-B, move cursor left by one character
+ case Replxx::KEY::LEFT:
+ case Replxx::KEY::meta( 'b' ): // meta-B, move cursor left by one word
+ case Replxx::KEY::meta( 'B' ):
+ case Replxx::KEY::control( Replxx::KEY::LEFT ):
+ case Replxx::KEY::meta( Replxx::KEY::LEFT ): // Emacs allows Meta, bash & readline don't
+ case Replxx::KEY::control('D'):
+ case Replxx::KEY::meta( 'd' ): // meta-D, kill word to right of cursor
+ case Replxx::KEY::meta( 'D' ):
+ case Replxx::KEY::control('E'): // ctrl-E, move cursor to end of line
+ case Replxx::KEY::END:
+ case Replxx::KEY::control('F'): // ctrl-F, move cursor right by one character
+ case Replxx::KEY::RIGHT:
+ case Replxx::KEY::meta( 'f' ): // meta-F, move cursor right by one word
+ case Replxx::KEY::meta( 'F' ):
+ case Replxx::KEY::control( Replxx::KEY::RIGHT ):
+ case Replxx::KEY::meta( Replxx::KEY::RIGHT ): // Emacs allows Meta, bash & readline don't
+ case Replxx::KEY::meta( Replxx::KEY::BACKSPACE ):
+ case Replxx::KEY::control('J'):
+ case Replxx::KEY::control('K'): // ctrl-K, kill from cursor to end of line
+ case Replxx::KEY::ENTER:
+ case Replxx::KEY::control('N'): // ctrl-N, recall next line in history
+ case Replxx::KEY::control('P'): // ctrl-P, recall previous line in history
+ case Replxx::KEY::DOWN:
+ case Replxx::KEY::UP:
+ case Replxx::KEY::control('T'): // ctrl-T, transpose characters
+ case Replxx::KEY::control('U'): // ctrl-U, kill all characters to the left of the cursor
+ case Replxx::KEY::control('W'):
+ case Replxx::KEY::meta( 'y' ): // meta-Y, "yank-pop", rotate popped text
+ case Replxx::KEY::meta( 'Y' ):
+ case 127:
+ case Replxx::KEY::DELETE:
+ case Replxx::KEY::meta( '<' ): // start of history
+ case Replxx::KEY::PAGE_UP:
+ case Replxx::KEY::meta( '>' ): // end of history
+ case Replxx::KEY::PAGE_DOWN: {
+ keepLooping = false;
+ } break;
+
+ // these characters revert the input line to its previous state
+ case Replxx::KEY::control('C'): // ctrl-C, abort this line
+ case Replxx::KEY::control('G'):
+ case Replxx::KEY::control('L'): { // ctrl-L, clear screen and redisplay line
+ keepLooping = false;
+ useSearchedLine = false;
+ if (c != Replxx::KEY::control('L')) {
+ c = -1; // ctrl-C and ctrl-G just abort the search and do nothing else
+ }
+ } break;
+
+ // these characters stay in search mode and assign the display
+ case Replxx::KEY::control('S'):
+ case Replxx::KEY::control('R'): {
+ if ( dp._searchText.length() == 0 ) { // if no current search text, recall previous text
+ if ( _previousSearchText.length() > 0 ) {
+ dp._searchText = _previousSearchText;
+ }
+ }
+ if (
+ ( ( dp._direction == 1 ) && ( c == Replxx::KEY::control( 'R' ) ) )
+ || ( ( dp._direction == -1 ) && ( c == Replxx::KEY::control( 'S' ) ) )
+ ) {
+ dp._direction = 0 - dp._direction; // reverse direction
+ dp.updateSearchPrompt(); // change the prompt
+ } else {
+ searchAgain = true; // same direction, search again
+ }
+ } break;
+
+// job control is its own thing
+#ifndef _WIN32
+ case Replxx::KEY::control('Z'): { // ctrl-Z, job control
+ /* IOModeGuard scope */ {
+ IOModeGuard ioModeGuard( _terminal );
+ // Returning to Linux (whatever) shell, leave raw mode
+ // Break out in mid-line
+ // Back from Linux shell, re-enter raw mode
+ raise( SIGSTOP );
+ }
+ dynamicRefresh( dp, dp, activeHistoryLine.get(), activeHistoryLine.length(), historyLinePosition );
+ continue;
+ } break;
+#endif
+
+ // these characters assign the search string, and hence the selected input line
+ case Replxx::KEY::BACKSPACE: { // backspace/ctrl-H, delete char to left of cursor
+ if ( dp._searchText.length() > 0 ) {
+ dp._searchText.erase( dp._searchText.length() - 1 );
+ dp.updateSearchPrompt();
+ _history.restore_pos();
+ historyLinePosition = _pos;
+ } else {
+ beep();
+ }
+ } break;
+
+ case Replxx::KEY::control('Y'): { // ctrl-Y, yank killed text
+ } break;
+
+ default: {
+ if ( ! is_control_code( c ) && ( c < static_cast<int>( Replxx::KEY::BASE ) ) ) { // not an action character
+ dp._searchText.insert( dp._searchText.length(), c );
+ dp.updateSearchPrompt();
+ } else {
+ beep();
+ }
+ }
+ } // switch
+
+ // if we are staying in search mode, search now
+ if ( ! keepLooping ) {
+ break;
+ }
+ activeHistoryLine.assign( _history.current() );
+ if ( dp._searchText.length() > 0 ) {
+ bool found = false;
+ int lineSearchPos = historyLinePosition;
+ if ( searchAgain ) {
+ lineSearchPos += dp._direction;
+ }
+ searchAgain = false;
+ while ( true ) {
+ while (
+ dp._direction < 0
+ ? ( lineSearchPos >= 0 )
+ : ( ( lineSearchPos + dp._searchText.length() ) <= activeHistoryLine.length() )
+ ) {
+ if (
+ ( lineSearchPos >= 0 )
+ && ( ( lineSearchPos + dp._searchText.length() ) <= activeHistoryLine.length() )
+ && std::equal( dp._searchText.begin(), dp._searchText.end(), activeHistoryLine.begin() + lineSearchPos )
+ ) {
+ found = true;
+ break;
+ }
+ lineSearchPos += dp._direction;
+ }
+ if ( found ) {
+ historyLinePosition = lineSearchPos;
+ break;
+ } else if ( _history.move( dp._direction < 0 ) ) {
+ activeHistoryLine.assign( _history.current() );
+ lineSearchPos = ( dp._direction > 0 ) ? 0 : ( activeHistoryLine.length() - dp._searchText.length() );
+ } else {
+ historyLinePosition = _pos;
+ beep();
+ break;
+ }
+ } // while
+ if ( ! found ) {
+ _history.restore_pos();
+ }
+ } else {
+ _history.restore_pos();
+ historyLinePosition = _pos;
+ }
+ activeHistoryLine.assign( _history.current() );
+ dynamicRefresh( dp, dp, activeHistoryLine.get(), activeHistoryLine.length(), historyLinePosition ); // draw user's text with our prompt
+ } // while
+
+ // leaving history search, restore previous prompt, maybe make searched line
+ // current
+ Prompt pb( _terminal );
+ UnicodeString tempUnicode( &_prompt._text[_prompt._lastLinePosition], _prompt._text.length() - _prompt._lastLinePosition );
+ pb.set_text( tempUnicode );
+ pb.update_screen_columns();
+ if ( useSearchedLine && ( activeHistoryLine.length() > 0 ) ) {
+ _history.commit_index();
+ _data.assign( activeHistoryLine );
+ _pos = historyLinePosition;
+ _modifiedState = true;
+ } else if ( ! useSearchedLine ) {
+ _history.restore_pos();
+ }
+ dynamicRefresh(pb, _prompt, _data.get(), _data.length(), _pos); // redraw the original prompt with current input
+ _previousSearchText = dp._searchText; // save search text for possible reuse on ctrl-R ctrl-R
+ emulate_key_press( c ); // pass a character or -1 back to main loop
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+// ctrl-L, clear screen and redisplay line
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::clear_screen( char32_t c ) {
+ _terminal.clear_screen( Terminal::CLEAR_SCREEN::WHOLE );
+ if ( c ) {
+ _prompt.write();
+ _prompt._cursorRowOffset = _prompt._extraLines;
+ refresh_line();
+ }
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+Replxx::ACTION_RESULT Replxx::ReplxxImpl::bracketed_paste( char32_t ) {
+ UnicodeString buf;
+ while ( char32_t c = _terminal.read_char() ) {
+ if ( c == KEY::PASTE_FINISH ) {
+ break;
+ }
+ if ( ( c == '\r' ) || ( c == KEY::control( 'M' ) ) ) {
+ c = '\n';
+ }
+ buf.push_back( c );
+ }
+ _data.insert( _pos, buf, 0, buf.length() );
+ _pos += buf.length();
+ return ( Replxx::ACTION_RESULT::CONTINUE );
+}
+
+template <bool subword>
+bool Replxx::ReplxxImpl::is_word_break_character( char32_t char_ ) const {
+ bool wbc( false );
+ if ( char_ < 128 ) {
+ wbc = strchr( subword ? _subwordBreakChars.c_str() : _wordBreakChars.c_str(), static_cast<char>( char_ ) ) != nullptr;
+ }
+ return ( wbc );
+}
+
+void Replxx::ReplxxImpl::history_add( std::string const& line ) {
+ _history.add( UnicodeString( line ) );
+}
+
+bool Replxx::ReplxxImpl::history_save( std::string const& filename ) {
+ return ( _history.save( filename, false ) );
+}
+
+bool Replxx::ReplxxImpl::history_sync( std::string const& filename ) {
+ return ( _history.save( filename, true ) );
+}
+
+bool Replxx::ReplxxImpl::history_load( std::string const& filename ) {
+ return ( _history.load( filename ) );
+}
+
+void Replxx::ReplxxImpl::history_clear( void ) {
+ _history.clear();
+}
+
+int Replxx::ReplxxImpl::history_size( void ) const {
+ return ( _history.size() );
+}
+
+Replxx::HistoryScan::impl_t Replxx::ReplxxImpl::history_scan( void ) const {
+ return ( _history.scan() );
+}
+
+void Replxx::ReplxxImpl::set_modify_callback( Replxx::modify_callback_t const& fn ) {
+ _modifyCallback = fn;
+}
+
+void Replxx::ReplxxImpl::set_completion_callback( Replxx::completion_callback_t const& fn ) {
+ _completionCallback = fn;
+}
+
+void Replxx::ReplxxImpl::set_highlighter_callback( Replxx::highlighter_callback_t const& fn ) {
+ _highlighterCallback = fn;
+}
+
+void Replxx::ReplxxImpl::set_hint_callback( Replxx::hint_callback_t const& fn ) {
+ _hintCallback = fn;
+}
+
+void Replxx::ReplxxImpl::set_max_history_size( int len ) {
+ _history.set_max_size( len );
+}
+
+void Replxx::ReplxxImpl::set_completion_count_cutoff( int count ) {
+ _completionCountCutoff = count;
+}
+
+void Replxx::ReplxxImpl::set_max_hint_rows( int count ) {
+ _maxHintRows = count;
+}
+
+void Replxx::ReplxxImpl::set_hint_delay( int hintDelay_ ) {
+ _hintDelay = hintDelay_;
+}
+
+void Replxx::ReplxxImpl::set_word_break_characters( char const* wordBreakers ) {
+ _wordBreakChars = wordBreakers;
+}
+
+void Replxx::ReplxxImpl::set_subword_break_characters( char const* subwordBreakers ) {
+ _subwordBreakChars = subwordBreakers;
+}
+
+void Replxx::ReplxxImpl::set_double_tab_completion( bool val ) {
+ _doubleTabCompletion = val;
+}
+
+void Replxx::ReplxxImpl::set_complete_on_empty( bool val ) {
+ _completeOnEmpty = val;
+}
+
+void Replxx::ReplxxImpl::set_beep_on_ambiguous_completion( bool val ) {
+ _beepOnAmbiguousCompletion = val;
+}
+
+void Replxx::ReplxxImpl::set_immediate_completion( bool val ) {
+ _immediateCompletion = val;
+}
+
+void Replxx::ReplxxImpl::set_unique_history( bool val ) {
+ _history.set_unique( val );
+}
+
+void Replxx::ReplxxImpl::set_no_color( bool val ) {
+ _noColor = val;
+}
+
+/**
+ * Display the dynamic incremental search prompt and the current user input
+ * line.
+ * @param pi Prompt struct holding information about the prompt and our
+ * screen position
+ * @param buf32 input buffer to be displayed
+ * @param len count of characters in the buffer
+ * @param pos current cursor position within the buffer (0 <= pos <= len)
+ */
+void Replxx::ReplxxImpl::dynamicRefresh(Prompt& oldPrompt, Prompt& newPrompt, char32_t* buf32, int len, int pos) {
+ clear_self_to_end_of_screen( &oldPrompt );
+ // calculate the position of the end of the prompt
+ int xEndOfPrompt, yEndOfPrompt;
+ calculate_screen_position(
+ 0, 0, newPrompt.screen_columns(), newPrompt._characterCount,
+ xEndOfPrompt, yEndOfPrompt
+ );
+
+ // calculate the position of the end of the input line
+ int xEndOfInput, yEndOfInput;
+ calculate_screen_position(
+ xEndOfPrompt, yEndOfPrompt, newPrompt.screen_columns(),
+ calculate_displayed_length(buf32, len), xEndOfInput,
+ yEndOfInput
+ );
+
+ // calculate the desired position of the cursor
+ int xCursorPos, yCursorPos;
+ calculate_screen_position(
+ xEndOfPrompt, yEndOfPrompt, newPrompt.screen_columns(),
+ calculate_displayed_length(buf32, pos), xCursorPos,
+ yCursorPos
+ );
+
+ // display the prompt
+ newPrompt.write();
+
+ // display the input line
+ _terminal.write32( buf32, len );
+
+#ifndef _WIN32
+ // we have to generate our own newline on line wrap
+ if (xEndOfInput == 0 && yEndOfInput > 0) {
+ _terminal.write8( "\n", 1 );
+ }
+#endif
+ // position the cursor
+ _terminal.jump_cursor(
+ xCursorPos, // 0-based on Win32
+ -( yEndOfInput - yCursorPos )
+ );
+ newPrompt._cursorRowOffset = newPrompt._extraLines + yCursorPos; // remember row for next pass
+}
+
+}
+
diff --git a/contrib/replxx/src/replxx_impl.hxx b/contrib/replxx/src/replxx_impl.hxx
new file mode 100644
index 0000000..bec9383
--- /dev/null
+++ b/contrib/replxx/src/replxx_impl.hxx
@@ -0,0 +1,280 @@
+/*
+ * Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HAVE_REPLXX_REPLXX_IMPL_HXX_INCLUDED
+#define HAVE_REPLXX_REPLXX_IMPL_HXX_INCLUDED 1
+
+#include <vector>
+#include <deque>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <thread>
+#include <mutex>
+#include <chrono>
+
+#include "replxx.hxx"
+#include "history.hxx"
+#include "killring.hxx"
+#include "utf8string.hxx"
+#include "prompt.hxx"
+
+namespace replxx {
+
+class Replxx::ReplxxImpl {
+public:
+ class Completion {
+ UnicodeString _text;
+ Replxx::Color _color;
+ public:
+ Completion( UnicodeString const& text_, Replxx::Color color_ )
+ : _text( text_ )
+ , _color( color_ ) {
+ }
+ Completion( Replxx::Completion const& completion_ )
+ : _text( completion_.text() )
+ , _color( completion_.color() ) {
+ }
+ Completion( Completion const& ) = default;
+ Completion& operator = ( Completion const& ) = default;
+ Completion( Completion&& ) = default;
+ Completion& operator = ( Completion&& ) = default;
+ UnicodeString const& text( void ) const {
+ return ( _text );
+ }
+ Replxx::Color color( void ) const {
+ return ( _color );
+ }
+ };
+ typedef std::vector<Completion> completions_t;
+ typedef std::vector<UnicodeString> data_t;
+ typedef std::vector<UnicodeString> hints_t;
+ typedef std::unique_ptr<char[]> utf8_buffer_t;
+ typedef std::unique_ptr<char32_t[]> input_buffer_t;
+ typedef std::vector<char32_t> display_t;
+ typedef std::deque<char32_t> key_presses_t;
+ typedef std::deque<std::string> messages_t;
+ enum class HINT_ACTION {
+ REGENERATE,
+ REPAINT,
+ TRIM,
+ SKIP
+ };
+ typedef std::unordered_map<std::string, Replxx::key_press_handler_t> named_actions_t;
+ typedef Replxx::ACTION_RESULT ( ReplxxImpl::* key_press_handler_raw_t )( char32_t );
+ typedef std::unordered_map<int, Replxx::key_press_handler_t> key_press_handlers_t;
+private:
+ typedef int long long unsigned action_trait_t;
+ static action_trait_t const NOOP = 0;
+ static action_trait_t const WANT_REFRESH = 1;
+ static action_trait_t const RESET_KILL_ACTION = 2;
+ static action_trait_t const SET_KILL_ACTION = 4;
+ static action_trait_t const DONT_RESET_PREFIX = 8;
+ static action_trait_t const DONT_RESET_COMPLETIONS = 16;
+ static action_trait_t const HISTORY_RECALL_MOST_RECENT = 32;
+ static action_trait_t const DONT_RESET_HIST_YANK_INDEX = 64;
+private:
+ mutable Utf8String _utf8Buffer;
+ UnicodeString _data;
+ int _pos; // character position in buffer ( 0 <= _pos <= _data[_line].length() )
+ display_t _display;
+ int _displayInputLength;
+ UnicodeString _hint;
+ int _prefix; // prefix length used in common prefix search
+ int _hintSelection; // Currently selected hint.
+ History _history;
+ KillRing _killRing;
+ int long long _lastRefreshTime;
+ bool _refreshSkipped;
+ int _lastYankSize;
+ int _maxHintRows;
+ int _hintDelay;
+ std::string _wordBreakChars;
+ std::string _subwordBreakChars;
+ int _completionCountCutoff;
+ bool _overwrite;
+ bool _doubleTabCompletion;
+ bool _completeOnEmpty;
+ bool _beepOnAmbiguousCompletion;
+ bool _immediateCompletion;
+ bool _bracketedPaste;
+ bool _noColor;
+ named_actions_t _namedActions;
+ key_press_handlers_t _keyPressHandlers;
+ Terminal _terminal;
+ std::thread::id _currentThread;
+ Prompt _prompt;
+ Replxx::modify_callback_t _modifyCallback;
+ Replxx::completion_callback_t _completionCallback;
+ Replxx::highlighter_callback_t _highlighterCallback;
+ Replxx::hint_callback_t _hintCallback;
+ key_presses_t _keyPresses;
+ messages_t _messages;
+ completions_t _completions;
+ int _completionContextLength;
+ int _completionSelection;
+ std::string _preloadedBuffer; // used with set_preload_buffer
+ std::string _errorMessage;
+ UnicodeString _previousSearchText; // remembered across invocations of replxx_input()
+ bool _modifiedState;
+ Replxx::Color _hintColor;
+ hints_t _hintsCache;
+ int _hintContextLenght;
+ Utf8String _hintSeed;
+ mutable std::mutex _mutex;
+public:
+ ReplxxImpl( FILE*, FILE*, FILE* );
+ virtual ~ReplxxImpl( void );
+ void set_modify_callback( Replxx::modify_callback_t const& fn );
+ void set_completion_callback( Replxx::completion_callback_t const& fn );
+ void set_highlighter_callback( Replxx::highlighter_callback_t const& fn );
+ void set_hint_callback( Replxx::hint_callback_t const& fn );
+ char const* input( std::string const& prompt );
+ void history_add( std::string const& line );
+ bool history_sync( std::string const& filename );
+ bool history_save( std::string const& filename );
+ bool history_load( std::string const& filename );
+ void history_clear( void );
+ Replxx::HistoryScan::impl_t history_scan( void ) const;
+ int history_size( void ) const;
+ void set_preload_buffer(std::string const& preloadText);
+ void set_word_break_characters( char const* wordBreakers );
+ void set_subword_break_characters( char const* subwordBreakers );
+ void set_max_hint_rows( int count );
+ void set_hint_delay( int milliseconds );
+ void set_double_tab_completion( bool val );
+ void set_complete_on_empty( bool val );
+ void set_beep_on_ambiguous_completion( bool val );
+ void set_immediate_completion( bool val );
+ void set_unique_history( bool );
+ void set_no_color( bool val );
+ void set_max_history_size( int len );
+ void set_completion_count_cutoff( int len );
+ int install_window_change_handler( void );
+ void enable_bracketed_paste( void );
+ void disable_bracketed_paste( void );
+ void print( char const*, int );
+ Replxx::ACTION_RESULT clear_screen( char32_t );
+ void emulate_key_press( char32_t );
+ Replxx::ACTION_RESULT invoke( Replxx::ACTION, char32_t );
+ void bind_key( char32_t, Replxx::key_press_handler_t );
+ void bind_key_internal( char32_t, char const* );
+ Replxx::State get_state( void ) const;
+ void set_state( Replxx::State const& );
+private:
+ ReplxxImpl( ReplxxImpl const& ) = delete;
+ ReplxxImpl& operator = ( ReplxxImpl const& ) = delete;
+private:
+ void preload_puffer( char const* preloadText );
+ int get_input_line( void );
+ Replxx::ACTION_RESULT action( action_trait_t, key_press_handler_raw_t const&, char32_t );
+ Replxx::ACTION_RESULT insert_character( char32_t );
+ Replxx::ACTION_RESULT new_line( char32_t );
+ Replxx::ACTION_RESULT go_to_begining_of_line( char32_t );
+ Replxx::ACTION_RESULT go_to_end_of_line( char32_t );
+ Replxx::ACTION_RESULT move_one_char_left( char32_t );
+ Replxx::ACTION_RESULT move_one_char_right( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT move_one_word_left( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT move_one_word_right( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT kill_word_to_left( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT kill_word_to_right( char32_t );
+ Replxx::ACTION_RESULT kill_to_whitespace_to_left( char32_t );
+ Replxx::ACTION_RESULT kill_to_begining_of_line( char32_t );
+ Replxx::ACTION_RESULT kill_to_end_of_line( char32_t );
+ Replxx::ACTION_RESULT yank( char32_t );
+ Replxx::ACTION_RESULT yank_cycle( char32_t );
+ Replxx::ACTION_RESULT yank_last_arg( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT capitalize_word( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT lowercase_word( char32_t );
+ template <bool subword>
+ Replxx::ACTION_RESULT uppercase_word( char32_t );
+ Replxx::ACTION_RESULT transpose_characters( char32_t );
+ Replxx::ACTION_RESULT abort_line( char32_t );
+ Replxx::ACTION_RESULT send_eof( char32_t );
+ Replxx::ACTION_RESULT delete_character( char32_t );
+ Replxx::ACTION_RESULT backspace_character( char32_t );
+ Replxx::ACTION_RESULT commit_line( char32_t );
+ Replxx::ACTION_RESULT history_next( char32_t );
+ Replxx::ACTION_RESULT history_previous( char32_t );
+ Replxx::ACTION_RESULT history_move( bool );
+ Replxx::ACTION_RESULT history_first( char32_t );
+ Replxx::ACTION_RESULT history_last( char32_t );
+ Replxx::ACTION_RESULT history_jump( bool );
+ Replxx::ACTION_RESULT hint_next( char32_t );
+ Replxx::ACTION_RESULT hint_previous( char32_t );
+ Replxx::ACTION_RESULT hint_move( bool );
+ Replxx::ACTION_RESULT toggle_overwrite_mode( char32_t );
+#ifndef _WIN32
+ Replxx::ACTION_RESULT verbatim_insert( char32_t );
+ Replxx::ACTION_RESULT suspend( char32_t );
+#endif
+ Replxx::ACTION_RESULT complete_line( char32_t );
+ Replxx::ACTION_RESULT complete_next( char32_t );
+ Replxx::ACTION_RESULT complete_previous( char32_t );
+ Replxx::ACTION_RESULT complete( bool );
+ Replxx::ACTION_RESULT incremental_history_search( char32_t startChar );
+ Replxx::ACTION_RESULT common_prefix_search( char32_t startChar );
+ Replxx::ACTION_RESULT bracketed_paste( char32_t startChar );
+ char32_t read_char( HINT_ACTION = HINT_ACTION::SKIP );
+ char const* read_from_stdin( void );
+ char32_t do_complete_line( bool );
+ void call_modify_callback( void );
+ completions_t call_completer( std::string const& input, int& ) const;
+ hints_t call_hinter( std::string const& input, int&, Replxx::Color& color ) const;
+ void refresh_line( HINT_ACTION = HINT_ACTION::REGENERATE );
+ void render( char32_t );
+ void render( HINT_ACTION );
+ int handle_hints( HINT_ACTION );
+ void set_color( Replxx::Color );
+ int context_length( void );
+ void clear( void );
+ void repaint( void );
+ template <bool subword>
+ bool is_word_break_character( char32_t ) const;
+ void dynamicRefresh(Prompt& oldPrompt, Prompt& newPrompt, char32_t* buf32, int len, int pos);
+ char const* finalize_input( char const* );
+ void clear_self_to_end_of_screen( Prompt const* = nullptr );
+ typedef struct {
+ int index;
+ bool error;
+ } paren_info_t;
+ paren_info_t matching_paren( void );
+};
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/terminal.cxx b/contrib/replxx/src/terminal.cxx
new file mode 100644
index 0000000..e618219
--- /dev/null
+++ b/contrib/replxx/src/terminal.cxx
@@ -0,0 +1,742 @@
+#include <memory>
+#include <cerrno>
+#include <cstdlib>
+#include <cstring>
+#include <array>
+#include <stdexcept>
+
+#ifdef _WIN32
+
+#include <conio.h>
+#include <windows.h>
+#include <io.h>
+#define isatty _isatty
+#define strcasecmp _stricmp
+#define strdup _strdup
+#define write _write
+#define STDIN_FILENO 0
+
+#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
+static DWORD const ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4;
+#endif
+
+#include "windows.hxx"
+
+#else /* _WIN32 */
+
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/select.h>
+#include <fcntl.h>
+#include <signal.h>
+
+#endif /* _WIN32 */
+
+#include "terminal.hxx"
+#include "conversion.hxx"
+#include "escape.hxx"
+#include "replxx.hxx"
+#include "util.hxx"
+
+using namespace std;
+
+namespace replxx {
+
+namespace tty {
+
+bool is_a_tty( int fd_ ) {
+ bool aTTY( isatty( fd_ ) != 0 );
+#ifdef _WIN32
+ do {
+ if ( aTTY ) {
+ break;
+ }
+ HANDLE h( (HANDLE)_get_osfhandle( fd_ ) );
+ if ( h == INVALID_HANDLE_VALUE ) {
+ break;
+ }
+ DWORD st( 0 );
+ if ( ! GetConsoleMode( h, &st ) ) {
+ break;
+ }
+ aTTY = true;
+ } while ( false );
+#endif
+ return ( aTTY );
+}
+
+bool in( is_a_tty( 0 ) );
+bool out( is_a_tty( 1 ) );
+
+}
+
+#ifndef _WIN32
+Terminal* _terminal_ = nullptr;
+static void WindowSizeChanged( int ) {
+ if ( ! _terminal_ ) {
+ return;
+ }
+ _terminal_->notify_event( Terminal::EVENT_TYPE::RESIZE );
+}
+#endif
+
+
+Terminal::Terminal( void )
+#ifdef _WIN32
+ : _consoleOut( INVALID_HANDLE_VALUE )
+ , _consoleIn( INVALID_HANDLE_VALUE )
+ , _origOutMode()
+ , _origInMode()
+ , _oldDisplayAttribute()
+ , _inputCodePage( GetConsoleCP() )
+ , _outputCodePage( GetConsoleOutputCP() )
+ , _interrupt( INVALID_HANDLE_VALUE )
+ , _events()
+ , _empty()
+#else
+ : _origTermios()
+ , _interrupt()
+#endif
+ , _rawMode( false )
+ , _utf8() {
+#ifdef _WIN32
+ _interrupt = CreateEvent( nullptr, true, false, TEXT( "replxx_interrupt_event" ) );
+#else
+ static_cast<void>( ::pipe( _interrupt ) == 0 );
+#endif
+}
+
+Terminal::~Terminal( void ) {
+ if ( _rawMode ) {
+ disable_raw_mode();
+ }
+#ifdef _WIN32
+ CloseHandle( _interrupt );
+#else
+ static_cast<void>( ::close( _interrupt[0] ) == 0 );
+ static_cast<void>( ::close( _interrupt[1] ) == 0 );
+#endif
+}
+
+void Terminal::write32( char32_t const* text32, int len32 ) {
+ _utf8.assign( text32, len32 );
+ write8( _utf8.get(), _utf8.size() );
+ return;
+}
+
+void Terminal::write8( char const* data_, int size_ ) {
+#ifdef _WIN32
+ if ( ! _rawMode ) {
+ enable_out();
+ }
+ int nWritten( win_write( _consoleOut, _autoEscape, data_, size_ ) );
+ if ( ! _rawMode ) {
+ disable_out();
+ }
+#else
+ int nWritten( write( 1, data_, size_ ) );
+#endif
+ if ( nWritten != size_ ) {
+ throw std::runtime_error( "write failed" );
+ }
+ return;
+}
+
+int Terminal::get_screen_columns( void ) {
+ int cols( 0 );
+#ifdef _WIN32
+ CONSOLE_SCREEN_BUFFER_INFO inf;
+ GetConsoleScreenBufferInfo( _consoleOut, &inf );
+ cols = inf.dwSize.X;
+#else
+ struct winsize ws;
+ cols = ( ioctl( 1, TIOCGWINSZ, &ws ) == -1 ) ? 80 : ws.ws_col;
+#endif
+ // cols is 0 in certain circumstances like inside debugger, which creates
+ // further issues
+ return ( cols > 0 ) ? cols : 80;
+}
+
+int Terminal::get_screen_rows( void ) {
+ int rows;
+#ifdef _WIN32
+ CONSOLE_SCREEN_BUFFER_INFO inf;
+ GetConsoleScreenBufferInfo( _consoleOut, &inf );
+ rows = 1 + inf.srWindow.Bottom - inf.srWindow.Top;
+#else
+ struct winsize ws;
+ rows = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 24 : ws.ws_row;
+#endif
+ return (rows > 0) ? rows : 24;
+}
+
+namespace {
+inline int notty( void ) {
+ errno = ENOTTY;
+ return ( -1 );
+}
+}
+
+void Terminal::enable_out( void ) {
+#ifdef _WIN32
+ _consoleOut = GetStdHandle( STD_OUTPUT_HANDLE );
+ SetConsoleOutputCP( 65001 );
+ GetConsoleMode( _consoleOut, &_origOutMode );
+ _autoEscape = SetConsoleMode( _consoleOut, _origOutMode | ENABLE_VIRTUAL_TERMINAL_PROCESSING ) != 0;
+#endif
+}
+
+void Terminal::disable_out( void ) {
+#ifdef _WIN32
+ SetConsoleMode( _consoleOut, _origOutMode );
+ SetConsoleOutputCP( _outputCodePage );
+ _consoleOut = INVALID_HANDLE_VALUE;
+ _autoEscape = false;
+#endif
+}
+
+void Terminal::enable_bracketed_paste( void ) {
+ static char const BRACK_PASTE_INIT[] = "\033[?2004h";
+ write8( BRACK_PASTE_INIT, sizeof ( BRACK_PASTE_INIT ) - 1 );
+}
+
+void Terminal::disable_bracketed_paste( void ) {
+ static char const BRACK_PASTE_DISABLE[] = "\033[?2004l";
+ write8( BRACK_PASTE_DISABLE, sizeof ( BRACK_PASTE_DISABLE ) - 1 );
+}
+
+int Terminal::enable_raw_mode( void ) {
+ if ( ! _rawMode ) {
+#ifdef _WIN32
+ _consoleIn = GetStdHandle( STD_INPUT_HANDLE );
+ SetConsoleCP( 65001 );
+ GetConsoleMode( _consoleIn, &_origInMode );
+ SetConsoleMode(
+ _consoleIn,
+ _origInMode & ~( ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT )
+ );
+ enable_out();
+#else
+ struct termios raw;
+
+ if ( ! tty::in ) {
+ return ( notty() );
+ }
+ if ( tcgetattr( 0, &_origTermios ) == -1 ) {
+ return ( notty() );
+ }
+
+ raw = _origTermios; /* modify the original mode */
+ /* input modes: no break, no CR to NL, no parity check, no strip char,
+ * no start/stop output control. */
+ raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
+ /* output modes - disable post processing */
+ // this is wrong, we don't want raw output, it turns newlines into straight
+ // linefeeds
+ // raw.c_oflag &= ~(OPOST);
+ /* control modes - set 8 bit chars */
+ raw.c_cflag |= (CS8);
+ /* local modes - echoing off, canonical off, no extended functions,
+ * no signal chars (^Z,^C) */
+ raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
+ /* control chars - set return condition: min number of bytes and timer.
+ * We want read to return every single byte, without timeout. */
+ raw.c_cc[VMIN] = 1;
+ raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
+
+ /* put terminal in raw mode after flushing */
+ if ( tcsetattr(0, TCSADRAIN, &raw) < 0 ) {
+ return ( notty() );
+ }
+ _terminal_ = this;
+#endif
+ _rawMode = true;
+ }
+ return ( 0 );
+}
+
+void Terminal::disable_raw_mode(void) {
+ if ( _rawMode ) {
+#ifdef _WIN32
+ disable_out();
+ SetConsoleMode( _consoleIn, _origInMode );
+ SetConsoleCP( _inputCodePage );
+ _consoleIn = INVALID_HANDLE_VALUE;
+#else
+ _terminal_ = nullptr;
+ if ( tcsetattr( 0, TCSADRAIN, &_origTermios ) == -1 ) {
+ return;
+ }
+#endif
+ _rawMode = false;
+ }
+}
+
+#ifndef _WIN32
+
+/**
+ * Read a UTF-8 sequence from the non-Windows keyboard and return the Unicode
+ * (char32_t) character it encodes
+ *
+ * @return char32_t Unicode character
+ */
+char32_t read_unicode_character(void) {
+ static char8_t utf8String[5];
+ static size_t utf8Count = 0;
+ while (true) {
+ char8_t c;
+
+ /* Continue reading if interrupted by signal. */
+ ssize_t nread;
+ do {
+ nread = read( STDIN_FILENO, &c, 1 );
+ } while ((nread == -1) && (errno == EINTR));
+
+ if (nread <= 0) return 0;
+ if (c <= 0x7F || locale::is8BitEncoding) { // short circuit ASCII
+ utf8Count = 0;
+ return c;
+ } else if (utf8Count < sizeof(utf8String) - 1) {
+ utf8String[utf8Count++] = c;
+ utf8String[utf8Count] = 0;
+ char32_t unicodeChar[2];
+ int ucharCount( 0 );
+ ConversionResult res = copyString8to32(unicodeChar, 2, ucharCount, utf8String);
+ if (res == conversionOK && ucharCount) {
+ utf8Count = 0;
+ return unicodeChar[0];
+ }
+ } else {
+ utf8Count = 0; // this shouldn't happen: got four bytes but no UTF-8 character
+ }
+ }
+}
+
+#endif // #ifndef _WIN32
+
+void beep() {
+ fprintf(stderr, "\x7"); // ctrl-G == bell/beep
+ fflush(stderr);
+}
+
+// replxx_read_char -- read a keystroke or keychord from the keyboard, and translate it
+// into an encoded "keystroke". When convenient, extended keys are translated into their
+// simpler Emacs keystrokes, so an unmodified "left arrow" becomes Ctrl-B.
+//
+// A return value of zero means "no input available", and a return value of -1
+// means "invalid key".
+//
+char32_t Terminal::read_char( void ) {
+ char32_t c( 0 );
+#ifdef _WIN32
+ INPUT_RECORD rec;
+ DWORD count;
+ char32_t modifierKeys = 0;
+ bool escSeen = false;
+ int highSurrogate( 0 );
+ while (true) {
+ ReadConsoleInputW( _consoleIn, &rec, 1, &count );
+#if __REPLXX_DEBUG__ // helper for debugging keystrokes, display info in the debug "Output"
+ // window in the debugger
+ {
+ if ( rec.EventType == KEY_EVENT ) {
+ //if ( rec.Event.KeyEvent.uChar.UnicodeChar ) {
+ char buf[1024];
+ sprintf(
+ buf,
+ "Unicode character 0x%04X, repeat count %d, virtual keycode 0x%04X, "
+ "virtual scancode 0x%04X, key %s%s%s%s%s\n",
+ rec.Event.KeyEvent.uChar.UnicodeChar,
+ rec.Event.KeyEvent.wRepeatCount,
+ rec.Event.KeyEvent.wVirtualKeyCode,
+ rec.Event.KeyEvent.wVirtualScanCode,
+ rec.Event.KeyEvent.bKeyDown ? "down" : "up",
+ (rec.Event.KeyEvent.dwControlKeyState & LEFT_CTRL_PRESSED) ? " L-Ctrl" : "",
+ (rec.Event.KeyEvent.dwControlKeyState & RIGHT_CTRL_PRESSED) ? " R-Ctrl" : "",
+ (rec.Event.KeyEvent.dwControlKeyState & LEFT_ALT_PRESSED) ? " L-Alt" : "",
+ (rec.Event.KeyEvent.dwControlKeyState & RIGHT_ALT_PRESSED) ? " R-Alt" : ""
+ );
+ OutputDebugStringA( buf );
+ //}
+ }
+ }
+#endif
+ if ( rec.EventType != KEY_EVENT ) {
+ continue;
+ }
+ // Windows provides for entry of characters that are not on your keyboard by sending the
+ // Unicode characters as a "key up" with virtual keycode 0x12 (VK_MENU == Alt key) ...
+ // accept these characters, otherwise only process characters on "key down"
+ if ( !rec.Event.KeyEvent.bKeyDown && ( rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU ) ) {
+ continue;
+ }
+ modifierKeys = 0;
+ // AltGr is encoded as ( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED ), so don't treat this
+ // combination as either CTRL or META we just turn off those two bits, so it is still
+ // possible to combine CTRL and/or META with an AltGr key by using right-Ctrl and/or
+ // left-Alt
+ DWORD const AltGr( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED );
+ if ( ( rec.Event.KeyEvent.dwControlKeyState & AltGr ) == AltGr ) {
+ rec.Event.KeyEvent.dwControlKeyState &= ~( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED );
+ }
+ if ( rec.Event.KeyEvent.dwControlKeyState & ( RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED ) ) {
+ modifierKeys |= Replxx::KEY::BASE_CONTROL;
+ }
+ if ( rec.Event.KeyEvent.dwControlKeyState & ( RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED ) ) {
+ modifierKeys |= Replxx::KEY::BASE_META;
+ }
+ if ( escSeen ) {
+ modifierKeys |= Replxx::KEY::BASE_META;
+ }
+ int key( rec.Event.KeyEvent.uChar.UnicodeChar );
+ if ( key == 0 ) {
+ switch (rec.Event.KeyEvent.wVirtualKeyCode) {
+ case VK_LEFT:
+ return modifierKeys | Replxx::KEY::LEFT;
+ case VK_RIGHT:
+ return modifierKeys | Replxx::KEY::RIGHT;
+ case VK_UP:
+ return modifierKeys | Replxx::KEY::UP;
+ case VK_DOWN:
+ return modifierKeys | Replxx::KEY::DOWN;
+ case VK_DELETE:
+ return modifierKeys | Replxx::KEY::DELETE;
+ case VK_HOME:
+ return modifierKeys | Replxx::KEY::HOME;
+ case VK_END:
+ return modifierKeys | Replxx::KEY::END;
+ case VK_PRIOR:
+ return modifierKeys | Replxx::KEY::PAGE_UP;
+ case VK_NEXT:
+ return modifierKeys | Replxx::KEY::PAGE_DOWN;
+ case VK_F1:
+ return modifierKeys | Replxx::KEY::F1;
+ case VK_F2:
+ return modifierKeys | Replxx::KEY::F2;
+ case VK_F3:
+ return modifierKeys | Replxx::KEY::F3;
+ case VK_F4:
+ return modifierKeys | Replxx::KEY::F4;
+ case VK_F5:
+ return modifierKeys | Replxx::KEY::F5;
+ case VK_F6:
+ return modifierKeys | Replxx::KEY::F6;
+ case VK_F7:
+ return modifierKeys | Replxx::KEY::F7;
+ case VK_F8:
+ return modifierKeys | Replxx::KEY::F8;
+ case VK_F9:
+ return modifierKeys | Replxx::KEY::F9;
+ case VK_F10:
+ return modifierKeys | Replxx::KEY::F10;
+ case VK_F11:
+ return modifierKeys | Replxx::KEY::F11;
+ case VK_F12:
+ return modifierKeys | Replxx::KEY::F12;
+ default:
+ continue; // in raw mode, ReadConsoleInput shows shift, ctrl - ignore them
+ }
+ } else if ( key == Replxx::KEY::ESCAPE ) { // ESC, set flag for later
+ escSeen = true;
+ continue;
+ } else if ( ( key >= 0xD800 ) && ( key <= 0xDBFF ) ) {
+ highSurrogate = key - 0xD800;
+ continue;
+ } else {
+ // we got a real character, return it
+ if ( ( key >= 0xDC00 ) && ( key <= 0xDFFF ) ) {
+ key -= 0xDC00;
+ key |= ( highSurrogate << 10 );
+ key += 0x10000;
+ }
+ if ( is_control_code( key ) ) {
+ key = control_to_human( key );
+ modifierKeys |= Replxx::KEY::BASE_CONTROL;
+ }
+ key |= modifierKeys;
+ highSurrogate = 0;
+ c = key;
+ break;
+ }
+ }
+
+#else
+ c = read_unicode_character();
+ if (c == 0) {
+ return 0;
+ }
+
+// If _DEBUG_LINUX_KEYBOARD is set, then ctrl-^ puts us into a keyboard
+// debugging mode
+// where we print out decimal and decoded values for whatever the "terminal"
+// program
+// gives us on different keystrokes. Hit ctrl-C to exit this mode.
+//
+#ifdef __REPLXX_DEBUG__
+ if (c == ctrlChar('^')) { // ctrl-^, special debug mode, prints all keys hit,
+ // ctrl-C to get out
+ printf(
+ "\nEntering keyboard debugging mode (on ctrl-^), press ctrl-C to exit "
+ "this mode\n");
+ while (true) {
+ unsigned char keys[10];
+ int ret = read(0, keys, 10);
+
+ if (ret <= 0) {
+ printf("\nret: %d\n", ret);
+ }
+ for (int i = 0; i < ret; ++i) {
+ char32_t key = static_cast<char32_t>(keys[i]);
+ char* friendlyTextPtr;
+ char friendlyTextBuf[10];
+ const char* prefixText = (key < 0x80) ? "" : "0x80+";
+ char32_t keyCopy = (key < 0x80) ? key : key - 0x80;
+ if (keyCopy >= '!' && keyCopy <= '~') { // printable
+ friendlyTextBuf[0] = '\'';
+ friendlyTextBuf[1] = keyCopy;
+ friendlyTextBuf[2] = '\'';
+ friendlyTextBuf[3] = 0;
+ friendlyTextPtr = friendlyTextBuf;
+ } else if (keyCopy == ' ') {
+ friendlyTextPtr = const_cast<char*>("space");
+ } else if (keyCopy == 27) {
+ friendlyTextPtr = const_cast<char*>("ESC");
+ } else if (keyCopy == 0) {
+ friendlyTextPtr = const_cast<char*>("NUL");
+ } else if (keyCopy == 127) {
+ friendlyTextPtr = const_cast<char*>("DEL");
+ } else {
+ friendlyTextBuf[0] = '^';
+ friendlyTextBuf[1] = control_to_human( keyCopy );
+ friendlyTextBuf[2] = 0;
+ friendlyTextPtr = friendlyTextBuf;
+ }
+ printf("%d x%02X (%s%s) ", key, key, prefixText, friendlyTextPtr);
+ }
+ printf("\x1b[1G\n"); // go to first column of new line
+
+ // drop out of this loop on ctrl-C
+ if (keys[0] == ctrlChar('C')) {
+ printf("Leaving keyboard debugging mode (on ctrl-C)\n");
+ fflush(stdout);
+ return -2;
+ }
+ }
+ }
+#endif // __REPLXX_DEBUG__
+
+ c = EscapeSequenceProcessing::doDispatch(c);
+ if ( is_control_code( c ) ) {
+ c = Replxx::KEY::control( control_to_human( c ) );
+ }
+#endif // #_WIN32
+ return ( c );
+}
+
+Terminal::EVENT_TYPE Terminal::wait_for_input( int long timeout_ ) {
+#ifdef _WIN32
+ std::array<HANDLE,2> handles = { _consoleIn, _interrupt };
+ while ( true ) {
+ DWORD event( WaitForMultipleObjects( static_cast<DWORD>( handles.size() ), handles.data(), false, timeout_ > 0 ? timeout_ : INFINITE ) );
+ switch ( event ) {
+ case ( WAIT_OBJECT_0 + 0 ): {
+ // peek events that will be skipped
+ INPUT_RECORD rec;
+ DWORD count;
+ PeekConsoleInputW( _consoleIn, &rec, 1, &count );
+
+ if (
+ ( rec.EventType != KEY_EVENT )
+ || ( !rec.Event.KeyEvent.bKeyDown && ( rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU ) )
+ ) {
+ // read the event to unsignal the handle
+ ReadConsoleInputW( _consoleIn, &rec, 1, &count );
+ continue;
+ } else if (rec.EventType == KEY_EVENT) {
+ int key(rec.Event.KeyEvent.uChar.UnicodeChar);
+ if (key == 0) {
+ switch (rec.Event.KeyEvent.wVirtualKeyCode) {
+ case VK_LEFT:
+ case VK_RIGHT:
+ case VK_UP:
+ case VK_DOWN:
+ case VK_DELETE:
+ case VK_HOME:
+ case VK_END:
+ case VK_PRIOR:
+ case VK_NEXT:
+ break;
+ default:
+ ReadConsoleInputW(_consoleIn, &rec, 1, &count);
+ continue; // in raw mode, ReadConsoleInput shows shift, ctrl - ignore them
+ }
+ }
+ }
+
+ return ( EVENT_TYPE::KEY_PRESS );
+ }
+ case ( WAIT_OBJECT_0 + 1 ): {
+ ResetEvent( _interrupt );
+ if ( _events.empty() ) {
+ continue;
+ }
+ EVENT_TYPE eventType( _events.front() );
+ _events.pop_front();
+ return ( eventType );
+ }
+ case ( WAIT_TIMEOUT ): {
+ return ( EVENT_TYPE::TIMEOUT );
+ }
+ }
+ }
+#else
+ fd_set fdSet;
+ int nfds( max( _interrupt[0], _interrupt[1] ) + 1 );
+ while ( true ) {
+ FD_ZERO( &fdSet );
+ FD_SET( 0, &fdSet );
+ FD_SET( _interrupt[0], &fdSet );
+ timeval tv{ timeout_ / 1000, static_cast<suseconds_t>( ( timeout_ % 1000 ) * 1000 ) };
+ int err( select( nfds, &fdSet, nullptr, nullptr, timeout_ > 0 ? &tv : nullptr ) );
+ if ( ( err == -1 ) && ( errno == EINTR ) ) {
+ continue;
+ }
+ if ( err == 0 ) {
+ return ( EVENT_TYPE::TIMEOUT );
+ }
+ if ( FD_ISSET( _interrupt[0], &fdSet ) ) {
+ char data( 0 );
+ static_cast<void>( read( _interrupt[0], &data, 1 ) == 1 );
+ if ( data == 'k' ) {
+ return ( EVENT_TYPE::KEY_PRESS );
+ }
+ if ( data == 'm' ) {
+ return ( EVENT_TYPE::MESSAGE );
+ }
+ if ( data == 'r' ) {
+ return ( EVENT_TYPE::RESIZE );
+ }
+ }
+ if ( FD_ISSET( 0, &fdSet ) ) {
+ return ( EVENT_TYPE::KEY_PRESS );
+ }
+ }
+#endif
+}
+
+void Terminal::notify_event( EVENT_TYPE eventType_ ) {
+#ifdef _WIN32
+ _events.push_back( eventType_ );
+ SetEvent( _interrupt );
+#else
+ char data( ( eventType_ == EVENT_TYPE::KEY_PRESS ) ? 'k' : ( eventType_ == EVENT_TYPE::MESSAGE ? 'm' : 'r' ) );
+ static_cast<void>( write( _interrupt[1], &data, 1 ) == 1 );
+#endif
+}
+
+/**
+ * Clear the screen ONLY (no redisplay of anything)
+ */
+void Terminal::clear_screen( CLEAR_SCREEN clearScreen_ ) {
+#ifdef _WIN32
+ if ( _autoEscape ) {
+#endif
+ if ( clearScreen_ == CLEAR_SCREEN::WHOLE ) {
+ char const clearCode[] = "\033c\033[H\033[2J\033[0m";
+ static_cast<void>( write(1, clearCode, sizeof ( clearCode ) - 1) >= 0 );
+ } else {
+ char const clearCode[] = "\033[J";
+ static_cast<void>( write(1, clearCode, sizeof ( clearCode ) - 1) >= 0 );
+ }
+ return;
+#ifdef _WIN32
+ }
+ COORD coord = { 0, 0 };
+ CONSOLE_SCREEN_BUFFER_INFO inf;
+ HANDLE consoleOut( _consoleOut != INVALID_HANDLE_VALUE ? _consoleOut : GetStdHandle( STD_OUTPUT_HANDLE ) );
+ GetConsoleScreenBufferInfo( consoleOut, &inf );
+ if ( clearScreen_ == CLEAR_SCREEN::TO_END ) {
+ coord = inf.dwCursorPosition;
+ DWORD nWritten( 0 );
+ SHORT height( inf.srWindow.Bottom - inf.srWindow.Top );
+ DWORD yPos( inf.dwCursorPosition.Y - inf.srWindow.Top );
+ DWORD toWrite( ( height + 1 - yPos ) * inf.dwSize.X - inf.dwCursorPosition.X );
+// FillConsoleOutputCharacterA( consoleOut, ' ', toWrite, coord, &nWritten );
+ _empty.resize( toWrite - 1, ' ' );
+ WriteConsoleA( consoleOut, _empty.data(), toWrite - 1, &nWritten, nullptr );
+ } else {
+ COORD scrollTarget = { 0, -inf.dwSize.Y };
+ CHAR_INFO fill{ TEXT( ' ' ), inf.wAttributes };
+ SMALL_RECT scrollRect = { 0, 0, inf.dwSize.X, inf.dwSize.Y };
+ ScrollConsoleScreenBuffer( consoleOut, &scrollRect, nullptr, scrollTarget, &fill );
+ }
+ SetConsoleCursorPosition( consoleOut, coord );
+#endif
+}
+
+void Terminal::jump_cursor( int xPos_, int yOffset_ ) {
+#ifdef _WIN32
+ CONSOLE_SCREEN_BUFFER_INFO inf;
+ GetConsoleScreenBufferInfo( _consoleOut, &inf );
+ inf.dwCursorPosition.X = xPos_;
+ inf.dwCursorPosition.Y += yOffset_;
+ SetConsoleCursorPosition( _consoleOut, inf.dwCursorPosition );
+#else
+ char seq[64];
+ if ( yOffset_ != 0 ) { // move the cursor up as required
+ snprintf( seq, sizeof seq, "\033[%d%c", abs( yOffset_ ), yOffset_ > 0 ? 'B' : 'A' );
+ write8( seq, strlen( seq ) );
+ }
+ // position at the end of the prompt, clear to end of screen
+ snprintf(
+ seq, sizeof seq, "\033[%dG",
+ xPos_ + 1 /* 1-based on VT100 */
+ );
+ write8( seq, strlen( seq ) );
+#endif
+}
+
+#ifdef _WIN32
+void Terminal::set_cursor_visible( bool visible_ ) {
+ CONSOLE_CURSOR_INFO cursorInfo;
+ GetConsoleCursorInfo( _consoleOut, &cursorInfo );
+ cursorInfo.bVisible = visible_;
+ SetConsoleCursorInfo( _consoleOut, &cursorInfo );
+ return;
+}
+#else
+void Terminal::set_cursor_visible( bool ) {}
+#endif
+
+#ifndef _WIN32
+int Terminal::read_verbatim( char32_t* buffer_, int size_ ) {
+ int len( 0 );
+ buffer_[len ++] = read_unicode_character();
+ int statusFlags( ::fcntl( STDIN_FILENO, F_GETFL, 0 ) );
+ ::fcntl( STDIN_FILENO, F_SETFL, statusFlags | O_NONBLOCK );
+ while ( len < size_ ) {
+ char32_t c( read_unicode_character() );
+ if ( c == 0 ) {
+ break;
+ }
+ buffer_[len ++] = c;
+ }
+ ::fcntl( STDIN_FILENO, F_SETFL, statusFlags );
+ return ( len );
+}
+
+int Terminal::install_window_change_handler( void ) {
+ struct sigaction sa;
+ sigemptyset(&sa.sa_mask);
+ sa.sa_flags = 0;
+ sa.sa_handler = &WindowSizeChanged;
+
+ if (sigaction(SIGWINCH, &sa, nullptr) == -1) {
+ return errno;
+ }
+ return 0;
+}
+#endif
+
+}
+
diff --git a/contrib/replxx/src/terminal.hxx b/contrib/replxx/src/terminal.hxx
new file mode 100644
index 0000000..e6a2578
--- /dev/null
+++ b/contrib/replxx/src/terminal.hxx
@@ -0,0 +1,94 @@
+#ifndef REPLXX_IO_HXX_INCLUDED
+#define REPLXX_IO_HXX_INCLUDED 1
+
+#include <deque>
+
+#ifdef _WIN32
+#include <vector>
+#include <windows.h>
+#else
+#include <termios.h>
+#endif
+
+#include "utf8string.hxx"
+
+namespace replxx {
+
+class Terminal {
+public:
+ enum class EVENT_TYPE {
+ KEY_PRESS,
+ MESSAGE,
+ TIMEOUT,
+ RESIZE
+ };
+private:
+#ifdef _WIN32
+ HANDLE _consoleOut;
+ HANDLE _consoleIn;
+ DWORD _origOutMode;
+ DWORD _origInMode;
+ bool _autoEscape;
+ WORD _oldDisplayAttribute;
+ UINT const _inputCodePage;
+ UINT const _outputCodePage;
+ HANDLE _interrupt;
+ typedef std::deque<EVENT_TYPE> events_t;
+ events_t _events;
+ std::vector<char> _empty;
+#else
+ struct termios _origTermios; /* in order to restore at exit */
+ int _interrupt[2];
+#endif
+ bool _rawMode; /* for destructor to check if restore is needed */
+ Utf8String _utf8;
+public:
+ enum class CLEAR_SCREEN {
+ WHOLE,
+ TO_END
+ };
+public:
+ Terminal( void );
+ ~Terminal( void );
+ void write32( char32_t const*, int );
+ void write8( char const*, int );
+ int get_screen_columns(void);
+ int get_screen_rows(void);
+ void enable_bracketed_paste( void );
+ void disable_bracketed_paste( void );
+ int enable_raw_mode(void);
+ void disable_raw_mode(void);
+ char32_t read_char(void);
+ void clear_screen( CLEAR_SCREEN );
+ EVENT_TYPE wait_for_input( int long = 0 );
+ void notify_event( EVENT_TYPE );
+ void jump_cursor( int, int );
+ void set_cursor_visible( bool );
+#ifndef _WIN32
+ int read_verbatim( char32_t*, int );
+ int install_window_change_handler( void );
+#endif
+private:
+ void enable_out( void );
+ void disable_out( void );
+private:
+ Terminal( Terminal const& ) = delete;
+ Terminal& operator = ( Terminal const& ) = delete;
+ Terminal( Terminal&& ) = delete;
+ Terminal& operator = ( Terminal&& ) = delete;
+};
+
+void beep();
+char32_t read_unicode_character(void);
+
+namespace tty {
+
+extern bool in;
+extern bool out;
+
+}
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/unicodestring.hxx b/contrib/replxx/src/unicodestring.hxx
new file mode 100644
index 0000000..8ff98a7
--- /dev/null
+++ b/contrib/replxx/src/unicodestring.hxx
@@ -0,0 +1,201 @@
+#ifndef REPLXX_UNICODESTRING_HXX_INCLUDED
+#define REPLXX_UNICODESTRING_HXX_INCLUDED
+
+#include <vector>
+#include <cstring>
+#include <string>
+
+#include "conversion.hxx"
+
+namespace replxx {
+
+class UnicodeString {
+public:
+ typedef std::vector<char32_t> data_buffer_t;
+ typedef data_buffer_t::const_iterator const_iterator;
+ typedef data_buffer_t::iterator iterator;
+private:
+ data_buffer_t _data;
+public:
+ UnicodeString()
+ : _data() {
+ }
+
+ explicit UnicodeString( std::string const& src )
+ : _data() {
+ assign( src );
+ }
+
+ explicit UnicodeString( UnicodeString const& other, int offset, int len = -1 )
+ : _data() {
+ _data.insert(
+ _data.end(),
+ other._data.begin() + offset,
+ len > 0 ? other._data.begin() + offset + len : other._data.end()
+ );
+ }
+
+ explicit UnicodeString( char const* src )
+ : _data() {
+ assign( src );
+ }
+
+ explicit UnicodeString( char8_t const* src )
+ : UnicodeString( reinterpret_cast<const char*>( src ) ) {
+ }
+
+ explicit UnicodeString( char32_t const* src )
+ : _data() {
+ int len( 0 );
+ while ( src[len] != 0 ) {
+ ++ len;
+ }
+ _data.assign( src, src + len );
+ }
+
+ explicit UnicodeString( char32_t const* src, int len )
+ : _data() {
+ _data.assign( src, src + len );
+ }
+
+ explicit UnicodeString( int len )
+ : _data() {
+ _data.resize( len );
+ }
+
+ UnicodeString& assign( std::string const& str_ ) {
+ _data.resize( static_cast<int>( str_.length() ) );
+ int len( 0 );
+ copyString8to32( _data.data(), static_cast<int>( str_.length() ), len, str_.c_str() );
+ _data.resize( len );
+ return *this;
+ }
+
+ UnicodeString& assign( char const* str_ ) {
+ int byteCount( static_cast<int>( strlen( str_ ) ) );
+ _data.resize( byteCount );
+ int len( 0 );
+ copyString8to32( _data.data(), byteCount, len, str_ );
+ _data.resize( len );
+ return *this;
+ }
+
+ UnicodeString& assign( UnicodeString const& other_ ) {
+ _data = other_._data;
+ return *this;
+ }
+
+ explicit UnicodeString( UnicodeString const& ) = default;
+ UnicodeString& operator = ( UnicodeString const& ) = default;
+ UnicodeString( UnicodeString&& ) = default;
+ UnicodeString& operator = ( UnicodeString&& ) = default;
+ bool operator == ( UnicodeString const& other_ ) const {
+ return ( _data == other_._data );
+ }
+
+ bool operator != ( UnicodeString const& other_ ) const {
+ return ( _data != other_._data );
+ }
+
+ UnicodeString& append( UnicodeString const& other ) {
+ _data.insert( _data.end(), other._data.begin(), other._data.end() );
+ return *this;
+ }
+
+ void push_back( char32_t c_ ) {
+ _data.push_back( c_ );
+ }
+
+ UnicodeString& append( char32_t const* src, int len ) {
+ _data.insert( _data.end(), src, src + len );
+ return *this;
+ }
+
+ UnicodeString& insert( int pos_, UnicodeString const& str_, int offset_, int len_ ) {
+ _data.insert( _data.begin() + pos_, str_._data.begin() + offset_, str_._data.begin() + offset_ + len_ );
+ return *this;
+ }
+
+ UnicodeString& insert( int pos_, char32_t c_ ) {
+ _data.insert( _data.begin() + pos_, c_ );
+ return *this;
+ }
+
+ UnicodeString& erase( int pos_ ) {
+ _data.erase( _data.begin() + pos_ );
+ return *this;
+ }
+
+ UnicodeString& erase( int pos_, int len_ ) {
+ _data.erase( _data.begin() + pos_, _data.begin() + pos_ + len_ );
+ return *this;
+ }
+
+ char32_t const* get() const {
+ return _data.data();
+ }
+
+ char32_t* get() {
+ return _data.data();
+ }
+
+ int length() const {
+ return static_cast<int>( _data.size() );
+ }
+
+ void clear( void ) {
+ _data.clear();
+ }
+
+ const char32_t& operator[]( size_t pos ) const {
+ return _data[pos];
+ }
+
+ char32_t& operator[]( size_t pos ) {
+ return _data[pos];
+ }
+
+ bool starts_with( data_buffer_t::const_iterator first_, data_buffer_t::const_iterator last_ ) const {
+ return (
+ ( std::distance( first_, last_ ) <= length() )
+ && ( std::equal( first_, last_, _data.begin() ) )
+ );
+ }
+
+ bool ends_with( data_buffer_t::const_iterator first_, data_buffer_t::const_iterator last_ ) const {
+ int len( static_cast<int>( std::distance( first_, last_ ) ) );
+ return (
+ ( len <= length() )
+ && ( std::equal( first_, last_, _data.end() - len ) )
+ );
+ }
+
+ bool is_empty( void ) const {
+ return ( _data.size() == 0 );
+ }
+
+ void swap( UnicodeString& other_ ) {
+ _data.swap( other_._data );
+ }
+
+ const_iterator begin( void ) const {
+ return ( _data.begin() );
+ }
+
+ const_iterator end( void ) const {
+ return ( _data.end() );
+ }
+
+ iterator begin( void ) {
+ return ( _data.begin() );
+ }
+
+ iterator end( void ) {
+ return ( _data.end() );
+ }
+};
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/utf8string.hxx b/contrib/replxx/src/utf8string.hxx
new file mode 100644
index 0000000..29effa2
--- /dev/null
+++ b/contrib/replxx/src/utf8string.hxx
@@ -0,0 +1,94 @@
+#ifndef REPLXX_UTF8STRING_HXX_INCLUDED
+#define REPLXX_UTF8STRING_HXX_INCLUDED
+
+#include <memory>
+
+#include "unicodestring.hxx"
+
+namespace replxx {
+
+class Utf8String {
+private:
+ typedef std::unique_ptr<char[]> buffer_t;
+ buffer_t _data;
+ int _bufSize;
+ int _len;
+public:
+ Utf8String( void )
+ : _data()
+ , _bufSize( 0 )
+ , _len( 0 ) {
+ }
+ explicit Utf8String( UnicodeString const& src )
+ : _data()
+ , _bufSize( 0 )
+ , _len( 0 ) {
+ assign( src, src.length() );
+ }
+
+ Utf8String( UnicodeString const& src_, int len_ )
+ : _data()
+ , _bufSize( 0 )
+ , _len( 0 ) {
+ assign( src_, len_ );
+ }
+
+ void assign( UnicodeString const& str_ ) {
+ assign( str_, str_.length() );
+ }
+
+ void assign( UnicodeString const& str_, int len_ ) {
+ assign( str_.get(), len_ );
+ }
+
+ void assign( char32_t const* str_, int len_ ) {
+ int len( len_ * 4 );
+ realloc( len );
+ _len = copyString32to8( _data.get(), len, str_, len_ );
+ }
+
+ void assign( std::string const& str_ ) {
+ realloc( static_cast<int>( str_.length() ) );
+ strncpy( _data.get(), str_.c_str(), str_.length() );
+ _len = static_cast<int>( str_.length() );
+ }
+
+ void assign( Utf8String const& other_ ) {
+ realloc( other_._len );
+ strncpy( _data.get(), other_._data.get(), other_._len );
+ _len = other_._len;
+ }
+
+ char const* get() const {
+ return _data.get();
+ }
+
+ int size( void ) const {
+ return ( _len );
+ }
+
+ bool operator != ( Utf8String const& other_ ) {
+ return ( ( other_._len != _len ) || ( memcmp( other_._data.get(), _data.get(), _len ) != 0 ) );
+ }
+
+private:
+ void realloc( int reqLen ) {
+ if ( ( reqLen + 1 ) > _bufSize ) {
+ _bufSize = 1;
+ while ( ( reqLen + 1 ) > _bufSize ) {
+ _bufSize *= 2;
+ }
+ _data.reset( new char[_bufSize] );
+ memset( _data.get(), 0, _bufSize );
+ }
+ _data[reqLen] = 0;
+ return;
+ }
+ Utf8String(const Utf8String&) = delete;
+ Utf8String& operator=(const Utf8String&) = delete;
+};
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/util.cxx b/contrib/replxx/src/util.cxx
new file mode 100644
index 0000000..719d707
--- /dev/null
+++ b/contrib/replxx/src/util.cxx
@@ -0,0 +1,158 @@
+#include <chrono>
+#include <cstdlib>
+#include <cstring>
+#include <ctime>
+#include <wctype.h>
+
+#include "util.hxx"
+
+namespace replxx {
+
+int mk_wcwidth( char32_t );
+
+/**
+ * Calculate a new screen position given a starting position, screen width and
+ * character count
+ * @param x - initial x position (zero-based)
+ * @param y - initial y position (zero-based)
+ * @param screenColumns - screen column count
+ * @param charCount - character positions to advance
+ * @param xOut - returned x position (zero-based)
+ * @param yOut - returned y position (zero-based)
+ */
+void calculate_screen_position(
+ int x, int y, int screenColumns,
+ int charCount, int& xOut, int& yOut
+) {
+ xOut = x;
+ yOut = y;
+ int charsRemaining = charCount;
+ while ( charsRemaining > 0 ) {
+ int charsThisRow = ( ( x + charsRemaining ) < screenColumns )
+ ? charsRemaining
+ : screenColumns - x;
+ xOut = x + charsThisRow;
+ yOut = y;
+ charsRemaining -= charsThisRow;
+ x = 0;
+ ++ y;
+ }
+ if ( xOut == screenColumns ) { // we have to special-case line wrap
+ xOut = 0;
+ ++ yOut;
+ }
+}
+
+/**
+ * Calculate a column width using mk_wcswidth()
+ * @param buf32 - text to calculate
+ * @param len - length of text to calculate
+ */
+int calculate_displayed_length( char32_t const* buf32_, int size_ ) {
+ int len( 0 );
+ for ( int i( 0 ); i < size_; ++ i ) {
+ char32_t c( buf32_[i] );
+ if ( c == '\033' ) {
+ int escStart( i );
+ ++ i;
+ if ( ( i < size_ ) && ( buf32_[i] != '[' ) ) {
+ i = escStart;
+ ++ len;
+ continue;
+ }
+ ++ i;
+ for ( ; i < size_; ++ i ) {
+ c = buf32_[i];
+ if ( ( c != ';' ) && ( ( c < '0' ) || ( c > '9' ) ) ) {
+ break;
+ }
+ }
+ if ( ( i < size_ ) && ( buf32_[i] == 'm' ) ) {
+ continue;
+ }
+ i = escStart;
+ len += 2;
+ } else if ( is_control_code( c ) ) {
+ len += 2;
+ } else {
+ int wcw( mk_wcwidth( c ) );
+ if ( wcw < 0 ) {
+ len = -1;
+ break;
+ }
+ len += wcw;
+ }
+ }
+ return ( len );
+}
+
+char const* ansi_color( Replxx::Color color_ ) {
+ static char const reset[] = "\033[0m";
+ static char const black[] = "\033[0;22;30m";
+ static char const red[] = "\033[0;22;31m";
+ static char const green[] = "\033[0;22;32m";
+ static char const brown[] = "\033[0;22;33m";
+ static char const blue[] = "\033[0;22;34m";
+ static char const magenta[] = "\033[0;22;35m";
+ static char const cyan[] = "\033[0;22;36m";
+ static char const lightgray[] = "\033[0;22;37m";
+
+#ifdef _WIN32
+ static bool const has256colorDefault( true );
+#else
+ static bool const has256colorDefault( false );
+#endif
+ static char const* TERM( getenv( "TERM" ) );
+ static bool const has256color( TERM ? ( strstr( TERM, "256" ) != nullptr ) : has256colorDefault );
+ static char const* gray = has256color ? "\033[0;1;90m" : "\033[0;1;30m";
+ static char const* brightred = has256color ? "\033[0;1;91m" : "\033[0;1;31m";
+ static char const* brightgreen = has256color ? "\033[0;1;92m" : "\033[0;1;32m";
+ static char const* yellow = has256color ? "\033[0;1;93m" : "\033[0;1;33m";
+ static char const* brightblue = has256color ? "\033[0;1;94m" : "\033[0;1;34m";
+ static char const* brightmagenta = has256color ? "\033[0;1;95m" : "\033[0;1;35m";
+ static char const* brightcyan = has256color ? "\033[0;1;96m" : "\033[0;1;36m";
+ static char const* white = has256color ? "\033[0;1;97m" : "\033[0;1;37m";
+ static char const error[] = "\033[101;1;33m";
+
+ char const* code( reset );
+ switch ( color_ ) {
+ case Replxx::Color::BLACK: code = black; break;
+ case Replxx::Color::RED: code = red; break;
+ case Replxx::Color::GREEN: code = green; break;
+ case Replxx::Color::BROWN: code = brown; break;
+ case Replxx::Color::BLUE: code = blue; break;
+ case Replxx::Color::MAGENTA: code = magenta; break;
+ case Replxx::Color::CYAN: code = cyan; break;
+ case Replxx::Color::LIGHTGRAY: code = lightgray; break;
+ case Replxx::Color::GRAY: code = gray; break;
+ case Replxx::Color::BRIGHTRED: code = brightred; break;
+ case Replxx::Color::BRIGHTGREEN: code = brightgreen; break;
+ case Replxx::Color::YELLOW: code = yellow; break;
+ case Replxx::Color::BRIGHTBLUE: code = brightblue; break;
+ case Replxx::Color::BRIGHTMAGENTA: code = brightmagenta; break;
+ case Replxx::Color::BRIGHTCYAN: code = brightcyan; break;
+ case Replxx::Color::WHITE: code = white; break;
+ case Replxx::Color::ERROR: code = error; break;
+ case Replxx::Color::DEFAULT: code = reset; break;
+ }
+ return ( code );
+}
+
+std::string now_ms_str( void ) {
+ std::chrono::milliseconds ms( std::chrono::duration_cast<std::chrono::milliseconds>( std::chrono::system_clock::now().time_since_epoch() ) );
+ time_t t( ms.count() / 1000 );
+ tm broken;
+#ifdef _WIN32
+#define localtime_r( t, b ) localtime_s( ( b ), ( t ) )
+#endif
+ localtime_r( &t, &broken );
+#undef localtime_r
+ static int const BUFF_SIZE( 32 );
+ char str[BUFF_SIZE];
+ strftime( str, BUFF_SIZE, "%Y-%m-%d %H:%M:%S.", &broken );
+ snprintf( str + sizeof ( "YYYY-mm-dd HH:MM:SS" ), 5, "%03d", static_cast<int>( ms.count() % 1000 ) );
+ return ( str );
+}
+
+}
+
diff --git a/contrib/replxx/src/util.hxx b/contrib/replxx/src/util.hxx
new file mode 100644
index 0000000..17c1086
--- /dev/null
+++ b/contrib/replxx/src/util.hxx
@@ -0,0 +1,25 @@
+#ifndef REPLXX_UTIL_HXX_INCLUDED
+#define REPLXX_UTIL_HXX_INCLUDED 1
+
+#include "replxx.hxx"
+
+namespace replxx {
+
+inline bool is_control_code(char32_t testChar) {
+ return (testChar < ' ') || // C0 controls
+ (testChar >= 0x7F && testChar <= 0x9F); // DEL and C1 controls
+}
+
+inline char32_t control_to_human( char32_t key ) {
+ return ( key < 27 ? ( key + 0x40 ) : ( key + 0x18 ) );
+}
+
+void calculate_screen_position( int x, int y, int screenColumns, int charCount, int& xOut, int& yOut );
+int calculate_displayed_length( char32_t const* buf32, int size );
+char const* ansi_color( Replxx::Color );
+std::string now_ms_str( void );
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/wcwidth.cpp b/contrib/replxx/src/wcwidth.cpp
new file mode 100644
index 0000000..c6c05fa
--- /dev/null
+++ b/contrib/replxx/src/wcwidth.cpp
@@ -0,0 +1,296 @@
+/*
+ * This is an implementation of wcwidth() and wcswidth() (defined in
+ * IEEE Std 1002.1-2001) for Unicode.
+ *
+ * http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
+ * http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
+ *
+ * In fixed-width output devices, Latin characters all occupy a single
+ * "cell" position of equal width, whereas ideographic CJK characters
+ * occupy two such cells. Interoperability between terminal-line
+ * applications and (teletype-style) character terminals using the
+ * UTF-8 encoding requires agreement on which character should advance
+ * the cursor by how many cell positions. No established formal
+ * standards exist at present on which Unicode character shall occupy
+ * how many cell positions on character terminals. These routines are
+ * a first attempt of defining such behavior based on simple rules
+ * applied to data provided by the Unicode Consortium.
+ *
+ * For some graphical characters, the Unicode standard explicitly
+ * defines a character-cell width via the definition of the East Asian
+ * FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes.
+ * In all these cases, there is no ambiguity about which width a
+ * terminal shall use. For characters in the East Asian Ambiguous (A)
+ * class, the width choice depends purely on a preference of backward
+ * compatibility with either historic CJK or Western practice.
+ * Choosing single-width for these characters is easy to justify as
+ * the appropriate long-term solution, as the CJK practice of
+ * displaying these characters as double-width comes from historic
+ * implementation simplicity (8-bit encoded characters were displayed
+ * single-width and 16-bit ones double-width, even for Greek,
+ * Cyrillic, etc.) and not any typographic considerations.
+ *
+ * Much less clear is the choice of width for the Not East Asian
+ * (Neutral) class. Existing practice does not dictate a width for any
+ * of these characters. It would nevertheless make sense
+ * typographically to allocate two character cells to characters such
+ * as for instance EM SPACE or VOLUME INTEGRAL, which cannot be
+ * represented adequately with a single-width glyph. The following
+ * routines at present merely assign a single-cell width to all
+ * neutral characters, in the interest of simplicity. This is not
+ * entirely satisfactory and should be reconsidered before
+ * establishing a formal standard in this area. At the moment, the
+ * decision which Not East Asian (Neutral) characters should be
+ * represented by double-width glyphs cannot yet be answered by
+ * applying a simple rule from the Unicode database content. Setting
+ * up a proper standard for the behavior of UTF-8 character terminals
+ * will require a careful analysis not only of each Unicode character,
+ * but also of each presentation form, something the author of these
+ * routines has avoided to do so far.
+ *
+ * http://www.unicode.org/unicode/reports/tr11/
+ *
+ * Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+ *
+ * Permission to use, copy, modify, and distribute this software
+ * for any purpose and without fee is hereby granted. The author
+ * disclaims all warranties with regard to this software.
+ *
+ * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
+ */
+
+#include <wchar.h>
+#include <string>
+#include <memory>
+
+namespace replxx {
+
+struct interval {
+ char32_t first;
+ char32_t last;
+};
+
+/* auxiliary function for binary search in interval table */
+static int bisearch(char32_t ucs, const struct interval *table, int max) {
+ int min = 0;
+ int mid;
+
+ if (ucs < table[0].first || ucs > table[max].last)
+ return 0;
+ while (max >= min) {
+ mid = (min + max) / 2;
+ if (ucs > table[mid].last)
+ min = mid + 1;
+ else if (ucs < table[mid].first)
+ max = mid - 1;
+ else
+ return 1;
+ }
+
+ return 0;
+}
+
+
+/* The following two functions define the column width of an ISO 10646
+ * character as follows:
+ *
+ * - The null character (U+0000) has a column width of 0.
+ *
+ * - Other C0/C1 control characters and DEL will lead to a return
+ * value of -1.
+ *
+ * - Non-spacing and enclosing combining characters (general
+ * category code Mn or Me in the Unicode database) have a
+ * column width of 0.
+ *
+ * - SOFT HYPHEN (U+00AD) has a column width of 1.
+ *
+ * - Other format characters (general category code Cf in the Unicode
+ * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
+ *
+ * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
+ * have a column width of 0.
+ *
+ * - Spacing characters in the East Asian Wide (W) or East Asian
+ * Full-width (F) category as defined in Unicode Technical
+ * Report #11 have a column width of 2.
+ *
+ * - All remaining characters (including all printable
+ * ISO 8859-1 and WGL4 characters, Unicode control characters,
+ * etc.) have a column width of 1.
+ *
+ * This implementation assumes that wchar_t characters are encoded
+ * in ISO 10646.
+ */
+
+int mk_is_wide_char(char32_t ucs) {
+ static const struct interval wide[] = {
+ {0x1100, 0x115f}, {0x231a, 0x231b}, {0x2329, 0x232a},
+ {0x23e9, 0x23ec}, {0x23f0, 0x23f0}, {0x23f3, 0x23f3},
+ {0x25fd, 0x25fe}, {0x2614, 0x2615}, {0x2648, 0x2653},
+ {0x267f, 0x267f}, {0x2693, 0x2693}, {0x26a1, 0x26a1},
+ {0x26aa, 0x26ab}, {0x26bd, 0x26be}, {0x26c4, 0x26c5},
+ {0x26ce, 0x26ce}, {0x26d4, 0x26d4}, {0x26ea, 0x26ea},
+ {0x26f2, 0x26f3}, {0x26f5, 0x26f5}, {0x26fa, 0x26fa},
+ {0x26fd, 0x26fd}, {0x2705, 0x2705}, {0x270a, 0x270b},
+ {0x2728, 0x2728}, {0x274c, 0x274c}, {0x274e, 0x274e},
+ {0x2753, 0x2755}, {0x2757, 0x2757}, {0x2795, 0x2797},
+ {0x27b0, 0x27b0}, {0x27bf, 0x27bf}, {0x2b1b, 0x2b1c},
+ {0x2b50, 0x2b50}, {0x2b55, 0x2b55}, {0x2e80, 0x2fdf},
+ {0x2ff0, 0x303e}, {0x3040, 0x3247}, {0x3250, 0x4dbf},
+ {0x4e00, 0xa4cf}, {0xa960, 0xa97f}, {0xac00, 0xd7a3},
+ {0xf900, 0xfaff}, {0xfe10, 0xfe19}, {0xfe30, 0xfe6f},
+ {0xff01, 0xff60}, {0xffe0, 0xffe6}, {0x16fe0, 0x16fe1},
+ {0x17000, 0x18aff}, {0x1b000, 0x1b12f}, {0x1b170, 0x1b2ff},
+ {0x1f004, 0x1f004}, {0x1f0cf, 0x1f0cf}, {0x1f18e, 0x1f18e},
+ {0x1f191, 0x1f19a}, {0x1f200, 0x1f202}, {0x1f210, 0x1f23b},
+ {0x1f240, 0x1f248}, {0x1f250, 0x1f251}, {0x1f260, 0x1f265},
+ {0x1f300, 0x1f320}, {0x1f32d, 0x1f335}, {0x1f337, 0x1f37c},
+ {0x1f37e, 0x1f393}, {0x1f3a0, 0x1f3ca}, {0x1f3cf, 0x1f3d3},
+ {0x1f3e0, 0x1f3f0}, {0x1f3f4, 0x1f3f4}, {0x1f3f8, 0x1f43e},
+ {0x1f440, 0x1f440}, {0x1f442, 0x1f4fc}, {0x1f4ff, 0x1f53d},
+ {0x1f54b, 0x1f54e}, {0x1f550, 0x1f567}, {0x1f57a, 0x1f57a},
+ {0x1f595, 0x1f596}, {0x1f5a4, 0x1f5a4}, {0x1f5fb, 0x1f64f},
+ {0x1f680, 0x1f6c5}, {0x1f6cc, 0x1f6cc}, {0x1f6d0, 0x1f6d2},
+ {0x1f6eb, 0x1f6ec}, {0x1f6f4, 0x1f6f8}, {0x1f910, 0x1f93e},
+ {0x1f940, 0x1f94c}, {0x1f950, 0x1f96b}, {0x1f980, 0x1f997},
+ {0x1f9c0, 0x1f9c0}, {0x1f9d0, 0x1f9e6}, {0x20000, 0x2fffd},
+ {0x30000, 0x3fffd},
+ };
+
+ if ( bisearch(ucs, wide, sizeof(wide) / sizeof(struct interval) - 1) ) {
+ return 1;
+ }
+
+ return 0;
+}
+
+int mk_wcwidth(char32_t ucs) {
+ /* sorted list of non-overlapping intervals of non-spacing characters */
+ /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */
+ static const struct interval combining[] = {
+ {0x00ad, 0x00ad}, {0x0300, 0x036f}, {0x0483, 0x0489},
+ {0x0591, 0x05bd}, {0x05bf, 0x05bf}, {0x05c1, 0x05c2},
+ {0x05c4, 0x05c5}, {0x05c7, 0x05c7}, {0x0610, 0x061a},
+ {0x061c, 0x061c}, {0x064b, 0x065f}, {0x0670, 0x0670},
+ {0x06d6, 0x06dc}, {0x06df, 0x06e4}, {0x06e7, 0x06e8},
+ {0x06ea, 0x06ed}, {0x0711, 0x0711}, {0x0730, 0x074a},
+ {0x07a6, 0x07b0}, {0x07eb, 0x07f3}, {0x0816, 0x0819},
+ {0x081b, 0x0823}, {0x0825, 0x0827}, {0x0829, 0x082d},
+ {0x0859, 0x085b}, {0x08d4, 0x08e1}, {0x08e3, 0x0902},
+ {0x093a, 0x093a}, {0x093c, 0x093c}, {0x0941, 0x0948},
+ {0x094d, 0x094d}, {0x0951, 0x0957}, {0x0962, 0x0963},
+ {0x0981, 0x0981}, {0x09bc, 0x09bc}, {0x09c1, 0x09c4},
+ {0x09cd, 0x09cd}, {0x09e2, 0x09e3}, {0x0a01, 0x0a02},
+ {0x0a3c, 0x0a3c}, {0x0a41, 0x0a42}, {0x0a47, 0x0a48},
+ {0x0a4b, 0x0a4d}, {0x0a51, 0x0a51}, {0x0a70, 0x0a71},
+ {0x0a75, 0x0a75}, {0x0a81, 0x0a82}, {0x0abc, 0x0abc},
+ {0x0ac1, 0x0ac5}, {0x0ac7, 0x0ac8}, {0x0acd, 0x0acd},
+ {0x0ae2, 0x0ae3}, {0x0afa, 0x0aff}, {0x0b01, 0x0b01},
+ {0x0b3c, 0x0b3c}, {0x0b3f, 0x0b3f}, {0x0b41, 0x0b44},
+ {0x0b4d, 0x0b4d}, {0x0b56, 0x0b56}, {0x0b62, 0x0b63},
+ {0x0b82, 0x0b82}, {0x0bc0, 0x0bc0}, {0x0bcd, 0x0bcd},
+ {0x0c00, 0x0c00}, {0x0c3e, 0x0c40}, {0x0c46, 0x0c48},
+ {0x0c4a, 0x0c4d}, {0x0c55, 0x0c56}, {0x0c62, 0x0c63},
+ {0x0c81, 0x0c81}, {0x0cbc, 0x0cbc}, {0x0cbf, 0x0cbf},
+ {0x0cc6, 0x0cc6}, {0x0ccc, 0x0ccd}, {0x0ce2, 0x0ce3},
+ {0x0d00, 0x0d01}, {0x0d3b, 0x0d3c}, {0x0d41, 0x0d44},
+ {0x0d4d, 0x0d4d}, {0x0d62, 0x0d63}, {0x0dca, 0x0dca},
+ {0x0dd2, 0x0dd4}, {0x0dd6, 0x0dd6}, {0x0e31, 0x0e31},
+ {0x0e34, 0x0e3a}, {0x0e47, 0x0e4e}, {0x0eb1, 0x0eb1},
+ {0x0eb4, 0x0eb9}, {0x0ebb, 0x0ebc}, {0x0ec8, 0x0ecd},
+ {0x0f18, 0x0f19}, {0x0f35, 0x0f35}, {0x0f37, 0x0f37},
+ {0x0f39, 0x0f39}, {0x0f71, 0x0f7e}, {0x0f80, 0x0f84},
+ {0x0f86, 0x0f87}, {0x0f8d, 0x0f97}, {0x0f99, 0x0fbc},
+ {0x0fc6, 0x0fc6}, {0x102d, 0x1030}, {0x1032, 0x1037},
+ {0x1039, 0x103a}, {0x103d, 0x103e}, {0x1058, 0x1059},
+ {0x105e, 0x1060}, {0x1071, 0x1074}, {0x1082, 0x1082},
+ {0x1085, 0x1086}, {0x108d, 0x108d}, {0x109d, 0x109d},
+ {0x1160, 0x11ff}, {0x135d, 0x135f}, {0x1712, 0x1714},
+ {0x1732, 0x1734}, {0x1752, 0x1753}, {0x1772, 0x1773},
+ {0x17b4, 0x17b5}, {0x17b7, 0x17bd}, {0x17c6, 0x17c6},
+ {0x17c9, 0x17d3}, {0x17dd, 0x17dd}, {0x180b, 0x180e},
+ {0x1885, 0x1886}, {0x18a9, 0x18a9}, {0x1920, 0x1922},
+ {0x1927, 0x1928}, {0x1932, 0x1932}, {0x1939, 0x193b},
+ {0x1a17, 0x1a18}, {0x1a1b, 0x1a1b}, {0x1a56, 0x1a56},
+ {0x1a58, 0x1a5e}, {0x1a60, 0x1a60}, {0x1a62, 0x1a62},
+ {0x1a65, 0x1a6c}, {0x1a73, 0x1a7c}, {0x1a7f, 0x1a7f},
+ {0x1ab0, 0x1abe}, {0x1b00, 0x1b03}, {0x1b34, 0x1b34},
+ {0x1b36, 0x1b3a}, {0x1b3c, 0x1b3c}, {0x1b42, 0x1b42},
+ {0x1b6b, 0x1b73}, {0x1b80, 0x1b81}, {0x1ba2, 0x1ba5},
+ {0x1ba8, 0x1ba9}, {0x1bab, 0x1bad}, {0x1be6, 0x1be6},
+ {0x1be8, 0x1be9}, {0x1bed, 0x1bed}, {0x1bef, 0x1bf1},
+ {0x1c2c, 0x1c33}, {0x1c36, 0x1c37}, {0x1cd0, 0x1cd2},
+ {0x1cd4, 0x1ce0}, {0x1ce2, 0x1ce8}, {0x1ced, 0x1ced},
+ {0x1cf4, 0x1cf4}, {0x1cf8, 0x1cf9}, {0x1dc0, 0x1df9},
+ {0x1dfb, 0x1dff}, {0x200b, 0x200f}, {0x202a, 0x202e},
+ {0x2060, 0x2064}, {0x2066, 0x206f}, {0x20d0, 0x20f0},
+ {0x2cef, 0x2cf1}, {0x2d7f, 0x2d7f}, {0x2de0, 0x2dff},
+ {0x302a, 0x302d}, {0x3099, 0x309a}, {0xa66f, 0xa672},
+ {0xa674, 0xa67d}, {0xa69e, 0xa69f}, {0xa6f0, 0xa6f1},
+ {0xa802, 0xa802}, {0xa806, 0xa806}, {0xa80b, 0xa80b},
+ {0xa825, 0xa826}, {0xa8c4, 0xa8c5}, {0xa8e0, 0xa8f1},
+ {0xa926, 0xa92d}, {0xa947, 0xa951}, {0xa980, 0xa982},
+ {0xa9b3, 0xa9b3}, {0xa9b6, 0xa9b9}, {0xa9bc, 0xa9bc},
+ {0xa9e5, 0xa9e5}, {0xaa29, 0xaa2e}, {0xaa31, 0xaa32},
+ {0xaa35, 0xaa36}, {0xaa43, 0xaa43}, {0xaa4c, 0xaa4c},
+ {0xaa7c, 0xaa7c}, {0xaab0, 0xaab0}, {0xaab2, 0xaab4},
+ {0xaab7, 0xaab8}, {0xaabe, 0xaabf}, {0xaac1, 0xaac1},
+ {0xaaec, 0xaaed}, {0xaaf6, 0xaaf6}, {0xabe5, 0xabe5},
+ {0xabe8, 0xabe8}, {0xabed, 0xabed}, {0xfb1e, 0xfb1e},
+ {0xfe00, 0xfe0f}, {0xfe20, 0xfe2f}, {0xfeff, 0xfeff},
+ {0xfff9, 0xfffb}, {0x101fd, 0x101fd}, {0x102e0, 0x102e0},
+ {0x10376, 0x1037a}, {0x10a01, 0x10a03}, {0x10a05, 0x10a06},
+ {0x10a0c, 0x10a0f}, {0x10a38, 0x10a3a}, {0x10a3f, 0x10a3f},
+ {0x10ae5, 0x10ae6}, {0x11001, 0x11001}, {0x11038, 0x11046},
+ {0x1107f, 0x11081}, {0x110b3, 0x110b6}, {0x110b9, 0x110ba},
+ {0x11100, 0x11102}, {0x11127, 0x1112b}, {0x1112d, 0x11134},
+ {0x11173, 0x11173}, {0x11180, 0x11181}, {0x111b6, 0x111be},
+ {0x111ca, 0x111cc}, {0x1122f, 0x11231}, {0x11234, 0x11234},
+ {0x11236, 0x11237}, {0x1123e, 0x1123e}, {0x112df, 0x112df},
+ {0x112e3, 0x112ea}, {0x11300, 0x11301}, {0x1133c, 0x1133c},
+ {0x11340, 0x11340}, {0x11366, 0x1136c}, {0x11370, 0x11374},
+ {0x11438, 0x1143f}, {0x11442, 0x11444}, {0x11446, 0x11446},
+ {0x114b3, 0x114b8}, {0x114ba, 0x114ba}, {0x114bf, 0x114c0},
+ {0x114c2, 0x114c3}, {0x115b2, 0x115b5}, {0x115bc, 0x115bd},
+ {0x115bf, 0x115c0}, {0x115dc, 0x115dd}, {0x11633, 0x1163a},
+ {0x1163d, 0x1163d}, {0x1163f, 0x11640}, {0x116ab, 0x116ab},
+ {0x116ad, 0x116ad}, {0x116b0, 0x116b5}, {0x116b7, 0x116b7},
+ {0x1171d, 0x1171f}, {0x11722, 0x11725}, {0x11727, 0x1172b},
+ {0x11a01, 0x11a06}, {0x11a09, 0x11a0a}, {0x11a33, 0x11a38},
+ {0x11a3b, 0x11a3e}, {0x11a47, 0x11a47}, {0x11a51, 0x11a56},
+ {0x11a59, 0x11a5b}, {0x11a8a, 0x11a96}, {0x11a98, 0x11a99},
+ {0x11c30, 0x11c36}, {0x11c38, 0x11c3d}, {0x11c3f, 0x11c3f},
+ {0x11c92, 0x11ca7}, {0x11caa, 0x11cb0}, {0x11cb2, 0x11cb3},
+ {0x11cb5, 0x11cb6}, {0x11d31, 0x11d36}, {0x11d3a, 0x11d3a},
+ {0x11d3c, 0x11d3d}, {0x11d3f, 0x11d45}, {0x11d47, 0x11d47},
+ {0x16af0, 0x16af4}, {0x16b30, 0x16b36}, {0x16f8f, 0x16f92},
+ {0x1bc9d, 0x1bc9e}, {0x1bca0, 0x1bca3}, {0x1d167, 0x1d169},
+ {0x1d173, 0x1d182}, {0x1d185, 0x1d18b}, {0x1d1aa, 0x1d1ad},
+ {0x1d242, 0x1d244}, {0x1da00, 0x1da36}, {0x1da3b, 0x1da6c},
+ {0x1da75, 0x1da75}, {0x1da84, 0x1da84}, {0x1da9b, 0x1da9f},
+ {0x1daa1, 0x1daaf}, {0x1e000, 0x1e006}, {0x1e008, 0x1e018},
+ {0x1e01b, 0x1e021}, {0x1e023, 0x1e024}, {0x1e026, 0x1e02a},
+ {0x1e8d0, 0x1e8d6}, {0x1e944, 0x1e94a}, {0xe0001, 0xe0001},
+ {0xe0020, 0xe007f}, {0xe0100, 0xe01ef},
+ };
+
+ /* test for 8-bit control characters */
+ if ( ucs == 0 ) {
+ return 0;
+ }
+ if ( ( ucs < 32 ) || ( ( ucs >= 0x7f ) && ( ucs < 0xa0 ) ) ) {
+ return -1;
+ }
+
+ /* binary search in table of non-spacing characters */
+ if ( bisearch( ucs, combining, sizeof( combining ) / sizeof( struct interval ) - 1 ) ) {
+ return 0;
+ }
+
+ /* if we arrive here, ucs is not a combining or C0/C1 control character */
+ return ( mk_is_wide_char( ucs ) ? 2 : 1 );
+}
+
+}
+
diff --git a/contrib/replxx/src/windows.cxx b/contrib/replxx/src/windows.cxx
new file mode 100644
index 0000000..715292c
--- /dev/null
+++ b/contrib/replxx/src/windows.cxx
@@ -0,0 +1,144 @@
+#ifdef _WIN32
+
+#include <iostream>
+
+#include "windows.hxx"
+#include "conversion.hxx"
+#include "terminal.hxx"
+
+using namespace std;
+
+namespace replxx {
+
+WinAttributes WIN_ATTR;
+
+template<typename T>
+T* HandleEsc(HANDLE out_, T* p, T* end) {
+ if (*p == '[') {
+ int code = 0;
+
+ int thisBackground( WIN_ATTR._defaultBackground );
+ for (++p; p < end; ++p) {
+ char32_t c = *p;
+
+ if ('0' <= c && c <= '9') {
+ code = code * 10 + (c - '0');
+ } else if (c == 'm' || c == ';') {
+ switch (code) {
+ case 0:
+ WIN_ATTR._consoleAttribute = WIN_ATTR._defaultAttribute;
+ WIN_ATTR._consoleColor = WIN_ATTR._defaultColor | thisBackground;
+ break;
+ case 1: // BOLD
+ case 5: // BLINK
+ WIN_ATTR._consoleAttribute = (WIN_ATTR._defaultAttribute ^ FOREGROUND_INTENSITY) & INTENSITY;
+ break;
+ case 22:
+ WIN_ATTR._consoleAttribute = WIN_ATTR._defaultAttribute;
+ break;
+ case 30:
+ case 90:
+ WIN_ATTR._consoleColor = thisBackground;
+ break;
+ case 31:
+ case 91:
+ WIN_ATTR._consoleColor = FOREGROUND_RED | thisBackground;
+ break;
+ case 32:
+ case 92:
+ WIN_ATTR._consoleColor = FOREGROUND_GREEN | thisBackground;
+ break;
+ case 33:
+ case 93:
+ WIN_ATTR._consoleColor = FOREGROUND_RED | FOREGROUND_GREEN | thisBackground;
+ break;
+ case 34:
+ case 94:
+ WIN_ATTR._consoleColor = FOREGROUND_BLUE | thisBackground;
+ break;
+ case 35:
+ case 95:
+ WIN_ATTR._consoleColor = FOREGROUND_BLUE | FOREGROUND_RED | thisBackground;
+ break;
+ case 36:
+ case 96:
+ WIN_ATTR._consoleColor = FOREGROUND_BLUE | FOREGROUND_GREEN | thisBackground;
+ break;
+ case 37:
+ case 97:
+ WIN_ATTR._consoleColor = FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE | thisBackground;
+ break;
+ case 101:
+ thisBackground = BACKGROUND_RED;
+ break;
+ }
+
+ if ( ( code >= 90 ) && ( code <= 97 ) ) {
+ WIN_ATTR._consoleAttribute = (WIN_ATTR._defaultAttribute ^ FOREGROUND_INTENSITY) & INTENSITY;
+ }
+
+ code = 0;
+ }
+
+ if (*p == 'm') {
+ ++p;
+ break;
+ }
+ }
+ } else {
+ ++p;
+ }
+
+ SetConsoleTextAttribute(
+ out_,
+ WIN_ATTR._consoleAttribute | WIN_ATTR._consoleColor
+ );
+
+ return p;
+}
+
+int win_write( HANDLE out_, bool autoEscape_, char const* str_, int size_ ) {
+ int count( 0 );
+ if ( tty::out ) {
+ DWORD nWritten( 0 );
+ if ( autoEscape_ ) {
+ WriteConsoleA( out_, str_, size_, &nWritten, nullptr );
+ count = nWritten;
+ } else {
+ char const* s( str_ );
+ char const* e( str_ + size_ );
+ while ( str_ < e ) {
+ if ( *str_ == 27 ) {
+ if ( s < str_ ) {
+ int toWrite( static_cast<int>( str_ - s ) );
+ WriteConsoleA( out_, s, static_cast<DWORD>( toWrite ), &nWritten, nullptr );
+ count += nWritten;
+ if ( nWritten != toWrite ) {
+ s = str_ = nullptr;
+ break;
+ }
+ }
+ s = HandleEsc( out_, str_ + 1, e );
+ int escaped( static_cast<int>( s - str_ ) );
+ count += escaped;
+ str_ = s;
+ } else {
+ ++ str_;
+ }
+ }
+
+ if ( s < str_ ) {
+ WriteConsoleA( out_, s, static_cast<DWORD>( str_ - s ), &nWritten, nullptr );
+ count += nWritten;
+ }
+ }
+ } else {
+ count = _write( 1, str_, size_ );
+ }
+ return ( count );
+}
+
+}
+
+#endif
+
diff --git a/contrib/replxx/src/windows.hxx b/contrib/replxx/src/windows.hxx
new file mode 100644
index 0000000..243f41c
--- /dev/null
+++ b/contrib/replxx/src/windows.hxx
@@ -0,0 +1,44 @@
+#ifndef REPLXX_WINDOWS_HXX_INCLUDED
+#define REPLXX_WINDOWS_HXX_INCLUDED 1
+
+#include <conio.h>
+#include <windows.h>
+#include <io.h>
+
+namespace replxx {
+
+static const int FOREGROUND_WHITE =
+ FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE;
+static const int BACKGROUND_WHITE =
+ BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE;
+static const int INTENSITY = FOREGROUND_INTENSITY | BACKGROUND_INTENSITY;
+
+class WinAttributes {
+ public:
+ WinAttributes() {
+ CONSOLE_SCREEN_BUFFER_INFO info;
+ GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info);
+ _defaultAttribute = info.wAttributes & INTENSITY;
+ _defaultColor = info.wAttributes & FOREGROUND_WHITE;
+ _defaultBackground = info.wAttributes & BACKGROUND_WHITE;
+
+ _consoleAttribute = _defaultAttribute;
+ _consoleColor = _defaultColor | _defaultBackground;
+ }
+
+ public:
+ int _defaultAttribute;
+ int _defaultColor;
+ int _defaultBackground;
+
+ int _consoleAttribute;
+ int _consoleColor;
+};
+
+int win_write( HANDLE, bool, char const*, int );
+
+extern WinAttributes WIN_ATTR;
+
+}
+
+#endif
diff --git a/contrib/snowball/.gitignore b/contrib/snowball/.gitignore
new file mode 100644
index 0000000..2147da8
--- /dev/null
+++ b/contrib/snowball/.gitignore
@@ -0,0 +1,5 @@
+*.o
+/libstemmer
+/snowball
+/src_c
+/stemwords
diff --git a/contrib/snowball/.travis.yml b/contrib/snowball/.travis.yml
new file mode 100644
index 0000000..e576233
--- /dev/null
+++ b/contrib/snowball/.travis.yml
@@ -0,0 +1,4 @@
+language: c
+compiler: gcc
+before_script: git clone https://github.com/snowballstem/snowball-data ../data
+script: make check
diff --git a/contrib/snowball/AUTHORS b/contrib/snowball/AUTHORS
new file mode 100644
index 0000000..60eae6f
--- /dev/null
+++ b/contrib/snowball/AUTHORS
@@ -0,0 +1,27 @@
+Authors
+=======
+
+Martin Porter
+-------------
+
+ - Designed the snowball language.
+ - Implemented the snowball to C compiler.
+ - Implemented the stemming algorithms in C.
+ - Wrote the documentation.
+
+Richard Boulton
+---------------
+
+ - Implemented Java backend of the snowball compiler.
+ - Developed build system.
+ - Assisted with website maintenance.
+
+
+Assistance from
+---------------
+
+Olivier Bornet - fixes to java packaging and build system.
+Andreas Jung - useful bug reports on the libstemmer library.
+Olly Betts - several patches, bug reports, and performance improvements.
+Sebastiano Vigna and Oerd Cukalla - patches for the Java stemming algorithms.
+Ralf Junker - fix a potential memory leak in sb_stemmer_new().
diff --git a/contrib/snowball/CMakeLists.txt b/contrib/snowball/CMakeLists.txt
new file mode 100644
index 0000000..7ee961e
--- /dev/null
+++ b/contrib/snowball/CMakeLists.txt
@@ -0,0 +1,70 @@
+# End of configuration
+SET(LIBSTEM_ALGORITHMS arabic danish dutch english finnish french german greek hindi hungarian
+ indonesian italian lithuanian nepali norwegian porter portuguese romanian
+ russian serbian spanish swedish tamil turkish)
+SET(ALL_ALGORITHMS ${LIBSTEM_ALGORITHMS})
+
+SET(COMPILER_SOURCES compiler/space.c
+ compiler/tokeniser.c
+ compiler/analyser.c
+ compiler/generator.c
+ compiler/driver.c)
+
+SET(SNOWBALL_RUNTIME runtime/api.c
+ runtime/utilities.c)
+SET(LIBSTEMMER_SOURCES libstemmer/libstemmer.c)
+SET(LIBSTEMMER_UTF8_SOURCES libstemmer/libstemmer_utf8.c)
+#LIBSTEMMER_UTF8_SOURCES = libstemmer/libstemmer_utf8.c
+#LIBSTEMMER_HEADERS = include/libstemmer.h libstemmer/modules.h libstemmer/modules_utf8.h
+#LIBSTEMMER_EXTRA = libstemmer/modules.txt libstemmer/modules_utf8.txt libstemmer/libstemmer_c.in
+
+SET(MODULES_H "modules.h")
+CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/libstemmer/libstemmer_c.in ${CMAKE_CURRENT_BINARY_DIR}/libstemmer/libstemmer.c @ONLY)
+ADD_DEFINITIONS("-DDISABLE_JS")
+ADD_DEFINITIONS("-DDISABLE_GO")
+ADD_DEFINITIONS("-DDISABLE_JAVA")
+ADD_DEFINITIONS("-DDISABLE_PYTHON")
+ADD_DEFINITIONS("-DDISABLE_CSHARP")
+ADD_DEFINITIONS("-DDISABLE_PASCAL")
+ADD_DEFINITIONS("-DDISABLE_RUST")
+
+MACRO(gen_stem IN ENCODING)
+ FOREACH(_it ${IN})
+ SET(_base "${CMAKE_CURRENT_BINARY_DIR}/libstemmer/stem_${ENCODING}_${_it}")
+ SET(_header "${_base}.h")
+ SET(_source "${_base}.c")
+ STRING(REPLACE "UTF_8" "Unicode" _in_enc "${ENCODING}")
+ SET(_input "${CMAKE_CURRENT_SOURCE_DIR}/algorithms/${_it}.sbl")
+ IF(${_in_enc} STREQUAL "Unicode" AND NOT EXISTS ${_input})
+ ADD_CUSTOM_COMMAND(OUTPUT ${_source}
+ COMMAND env "ASAN_OPTIONS=detect_leaks=0" ${CMAKE_CURRENT_BINARY_DIR}/snowball "${CMAKE_CURRENT_SOURCE_DIR}/algorithms/${_it}/stem_ISO_8859_1.sbl" -o ${_base} -eprefix ${_it}_${ENCODING}_ -r ../runtime -u
+ DEPENDS snowball)
+ LIST(APPEND STEMMER_SOURCES ${_source})
+
+ ELSE()
+ IF(EXISTS "${_input}")
+ ADD_CUSTOM_COMMAND(OUTPUT ${_source}
+ COMMAND env "ASAN_OPTIONS=detect_leaks=0" ${CMAKE_CURRENT_BINARY_DIR}/snowball ${_input} -o ${_base} -eprefix ${_it}_${ENCODING}_ -r ../runtime -u
+ DEPENDS snowball)
+ LIST(APPEND STEMMER_SOURCES ${_source})
+ ENDIF()
+ ENDIF()
+ ENDFOREACH()
+ENDMACRO()
+
+INCLUDE_DIRECTORIES("include")
+
+ADD_EXECUTABLE(snowball ${COMPILER_SOURCES})
+
+ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libstemmer/modules.h
+ COMMAND ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/libstemmer/mkmodules.pl ${CMAKE_CURRENT_BINARY_DIR}/libstemmer/modules.h libstemmer ${CMAKE_CURRENT_SOURCE_DIR}/libstemmer/modules.txt ${CMAKE_CURRENT_BINARY_DIR}/libstemmer/mkinc.mak)
+ADD_CUSTOM_TARGET(modules DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/libstemmer/modules.h")
+
+SET(STEMMER_SOURCES "${CMAKE_CURRENT_BINARY_DIR}/libstemmer/libstemmer.c")
+ADD_CUSTOM_TARGET(stemmer_deps ALL)
+ADD_DEPENDENCIES(stemmer_deps modules)
+
+gen_stem("${LIBSTEM_ALGORITHMS}" "UTF_8")
+
+ADD_LIBRARY(stemmer ${LINK_TYPE} ${SNOWBALL_RUNTIME} ${STEMMER_SOURCES})
+ADD_DEPENDENCIES(stemmer stemmer_deps)
diff --git a/contrib/snowball/NEWS b/contrib/snowball/NEWS
new file mode 100644
index 0000000..c71c12d
--- /dev/null
+++ b/contrib/snowball/NEWS
@@ -0,0 +1,407 @@
+Snowball 2.0.0 (2019-10-02)
+===========================
+
+C/C++
+-----
+
+* Fully handle 4-byte UTF-8 sequences. Previously `hop` and `next` handled
+ sequences of any length, but commands which look at the character value only
+ handled sequences up to length 3. Fixes #89.
+
+* Fix handling of a 3-byte UTF-8 sequence in a grouping in `backwardmode`.
+
+Java
+----
+
+* TestApp.java:
+
+ - Always use UTF-8 for I/O. Patch from David Corbett (#80).
+
+ - Allow reading input from stdin.
+
+ - Remove rather pointless "stem n times" feature.
+
+ - Only lower case ASCII to match stemwords.c.
+
+ - Stem empty lines too to match stemwords.c.
+
+Code Quality Improvements
+-------------------------
+
+* Fix various warnings from newer compilers.
+
+* Improve use of `const`.
+
+* Share common functions between compiler backends rather than having multiple
+ copies of the same code.
+
+* Assorted code clean-up.
+
+* Initialise line_labelled member of struct generator to 0. Previously we were
+ invoking undefined behaviour, though in practice it'll be zero initialised on
+ most platforms.
+
+New Code Generators
+-------------------
+
+* Add Python generator (#24). Originally written by Yoshiki Shibukawa, with
+ additional updates by Dmitry Shachnev.
+
+* Add Javascript generator. Based on JSX generator (#26) written by Yoshiki
+ Shibukawa.
+
+* Add Rust generator from Jakob Demler (#51).
+
+* Add Go generator from Marty Schoch (#57).
+
+* Add C# generator. Based on patch from Cesar Souza (#16, #17).
+
+* Add Pascal generator. Based on Delphi backend from stemming.zip file on old
+ website (#75).
+
+New Language Features
+---------------------
+
+* Add `len` and `lenof` to measure Unicode length. These are similar to `size`
+ and `sizeof` (respectively), but `size` and `sizeof` return the length in
+ bytes under `-utf8`, whereas these new commands give the same result whether
+ using `-utf8`, `-widechars` or neither (but under `-utf8` they are O(n) in
+ the length of the string). For compatibility with existing code which might
+ use these as variable or function names, they stop being treated as tokens if
+ declared to be a variable or function.
+
+* New `{U+1234}` stringdef notation for Unicode codepoints.
+
+* More versatile integer tests. Now you can compare any two arithmetic
+ expressions with a relational operator in parentheses after the `$`, so for
+ example `$(len > 3)` can now be used when previously a temporary variable was
+ required: `$tmp = len $tmp > 3`
+
+Code generation improvements
+----------------------------
+
+* General:
+
+ + Avoid unnecessarily saving and restoring of the cursor for more commands -
+ `atlimit`, `do`, `set` and `unset` all leave the cursor alone or always
+ restore its value, and for C `booltest` (which other languages already
+ handled).
+
+ + Special case handling for `setlimit tomark AE`. All uses of setlimit in
+ the current stemmers we ship follow this pattern, and by special-casing we
+ can avoid having to save and restore the cursor (#74).
+
+ + Merge duplicate actions in the same `among`. This reduces the size of the
+ switch/if-chain in the generated code which dispatch the among for many of
+ the stemmers.
+
+ + Generate simpler code for `among`. We always check for a zero return value
+ when we call the among, so there's no point also checking for that in the
+ switch/if-chain. We can also avoid the switch/if-chain entirely when
+ there's only one possible outcome (besides the zero return).
+
+ + Optimise code generated for `do <function call>`. This speeds up "make
+ check_python" by about 2%, and should speed up other interpreted languages
+ too (#110).
+
+ + Generate more and better comments referencing snowball source.
+
+ + Add homepage URL and compiler version as comments in generated files.
+
+* C/C++:
+
+ + Fix `size` and `sizeof` to not report one too high (reported by Assem
+ Chelli in #32).
+
+ + If signal `f` from a function call would lead to return from the current
+ function then handle this and bailing out on an error together with a
+ simple `if (ret <= 0) return ret;`
+
+ + Inline testing for a single character literals.
+
+ + Avoiding generating `|| 0` in corner case - this can result in a compiler
+ warning when building the generated code.
+
+ + Implement `insert_v()` in terms of `insert_s()`.
+
+ + Add conditional `extern "C"` so `runtime/api.h` can be included from C++
+ code. Closes #90, reported by vvarma.
+
+* Java:
+
+ + Fix functions in `among` to work in Java. We seem to need to make the
+ methods called from among `public` instead of `private`, and to call them
+ on `this` instead of the `methodObject` (which is cleaner anyway). No
+ revision in version control seems to generate working code for this case,
+ but Richard says it definitely used to work - possibly older JVMs failed to
+ correctly enforce the access controls when methods were invoked by
+ reflection.
+
+ + Code after handling `f` by returning from the current function is
+ unreachable too.
+
+ + Previously we incorrectly decided that code after an `or` was
+ unreachable in certain cases. None of the current stemmers in the
+ distribution triggered this, but Martin Porter's snowball version
+ of the Schinke Latin stemmer does. Fixes #58, reported by Alexander
+ Myltsev.
+
+ + The reachability logic was failing to consider reachability from
+ the final command in an `or`. Fixes #82, reported by David Corbett.
+
+ + Fix `maxint` and `minint`. Patch from David Corbett in #31.
+
+ + Fix `$` on strings. The previous generated code was just wrong. This
+ doesn't affect any of the included algorithms, but for example breaks
+ Martin Porter's snowball implementation of Schinke's Latin Stemmer.
+ Issue noted by Jakob Demler while working on the Rust backend in #51,
+ and reported in the Schinke's Latin Stemmer by Alexander Myltsev
+ in #58.
+
+ + Make SnowballProgram objects serializable. Patch from Oleg Smirnov in #43.
+
+ + Eliminate range-check implementation for groupings. This was removed from
+ the C generator 10 years earlier, isn't used for any of the existing
+ algorithms, and it doesn't seem likely it would be - the grouping would
+ have to consist entirely of a contiguous block of Unicode code-points.
+
+ + Simplify code generated for `repeat` and `atleast`.
+
+ + Eliminate unused return values and variables from runtime functions.
+
+ + Only import the `among` and `SnowballProgram` classes if they're actually
+ used.
+
+ + Only generate `copy_from()` method if it's used.
+
+ + Merge runtime functions `eq_s` and `eq_v` functions.
+
+ + Java arrays know their own length so stop storing it separately.
+
+ + Escape char 127 (DEL) in generated Java code. It's unlikely that this
+ character would actually be used in a real stemmer, so this was more of a
+ theoretical bug.
+
+ + Drop unused import of InvocationTargetException from SnowballStemmer.
+ Reported by GerritDeMeulder in #72.
+
+ + Fix lint check issues in generated Java code. The stemmer classes are only
+ referenced in the example app via reflection, so add
+ @SuppressWarnings("unused") for them. The stemmer classes override
+ equals() and hashCode() methods from the standard java Object class, so
+ mark these with @Override. Both suggested by GerritDeMeulder in #72.
+
+ + Declare Java variables at point of use in generated code. Putting all
+ declarations at the top of the function was adding unnecessary complexity
+ to the Java generator code for no benefit.
+
+ + Improve formatting of generated code.
+
+New stemming algorithms
+-----------------------
+
+* Add Tamil stemmer from Damodharan Rajalingam (#2, #3).
+
+* Add Arabic stemmer from Assem Chelli (#32, #50).
+
+* Add Irish stemmer Jim O'Regan (#48).
+
+* Add Nepali stemmer from Arthur Zakirov (#70).
+
+* Add Indonesian stemmer from Olly Betts (#71).
+
+* Add Hindi stemmer from Olly Betts (#73). Thanks to David Corbett for review.
+
+* Add Lithuanian stemmer from Dainius Jocas (#22, #76).
+
+* Add Greek stemmer from Oleg Smirnov (#44).
+
+* Add Catalan and Basque stemmers from Israel Olalla (#104).
+
+Behavioural changes to existing algorithms
+------------------------------------------
+
+* Portuguese:
+
+ + Replace incorrect Spanish suffixes by Portuguese suffixes (#1).
+
+* French:
+
+ + The MSDOS CP850 version of the French algorithm was missing changes present
+ in the ISO8859-1 and Unicode versions. There's now a single version of
+ each algorithm which was based on the Unicode version.
+
+ + Recognize French suffixes even when they begin with diaereses. Patch from
+ David Corbett in #78.
+
+* Russian:
+
+ + We now normalise 'ё' to 'е' before stemming. The documentation has long
+ said "we assume ['ё'] is mapped into ['е']" but it's more convenient for
+ the stemmer to actually perform this normalisation. This change has no
+ effect if the caller is already normalising as we recommend. It's a change
+ in behaviour they aren't, but 'Ñ‘' occurs rarely (there are currently no
+ instances in our test vocabulary) and this improves behaviour when it does
+ occur. Patch from Eugene Mirotin (#65, #68).
+
+* Finish:
+
+ + Adjust the Finnish algorithm not to mangle numbers. This change also
+ means it tends to leave foreign words alone. Fixes #66.
+
+* Danish:
+
+ + Adjust Danish algorithm not to mangle alphanumeric codes. In particular
+ alphanumeric codes ending in a double digit (e.g. 0x0e00, hal9000,
+ space1999) are no longer mangled. See #81.
+
+Optimisations to existing algorithms
+------------------------------------
+
+* Turkish:
+
+ + Simplify uses of `test` in stemmer code.
+
+ + Check for 'ad' or 'soyad' more efficiently, and without needing the
+ strlen variable. This speeds up "make check_utf8_turkish" by 11%
+ on x86 Linux.
+
+* Kraaij-Pohlmann:
+
+ + Eliminate variable x `$p1 <= cursor` is simpler and a little more efficient
+ than `setmark x $x >= p1`.
+
+Code clarity improvements to existing algorithms
+------------------------------------------------
+
+* Turkish:
+
+ + Use , for cedilla to match the conventions used in other stemmers.
+
+* Kraaij-Pohlmann:
+
+ + Avoid cryptic `[among ( (])` ... `)` construct - instead use the same
+ `[substring] among (` ... `)` construct we do in other stemmers.
+
+Compiler
+--------
+
+* Support conventional --help and --version options.
+
+* Warn if -r or -ep used with backend other than C/C++.
+
+* Warn if encoding command line options are specified when generating code in a
+ language with a fixed encoding.
+
+* The default classname is now set based on the output filename, so `-n` is now
+ often no longer needed. Fixes #64.
+
+* Avoid potential one byte buffer over-read when parsing snowball code.
+
+* Avoid comparing with uninitialised array element during compilation.
+
+* Improve `-syntax` output for `setlimit L for C`.
+
+* Optimise away double negation so generators don't have to worry about
+ generating `--` (decrement operator in many languages). Fixes #52, reported
+ by David Corbett.
+
+* Improved compiler error and warning messages:
+
+ - We now report FILE:LINE: before each diagnostic message.
+
+ - Improve warnings for unused declarations/definitions.
+
+ - Warn for variables which are used, but either never initialised
+ or never read.
+
+ - Flag non-ASCII literal strings. This is an error for wide Unicode, but
+ only a warning for single-byte and UTF-8 which work so long as the source
+ encoding matches the encoding used in the generated stemmer code.
+
+ - Improve error recovery after an undeclared `define`. We now sniff the
+ token after the identifier and if it is `as` we parse as a routine,
+ otherwise we parse as a grouping. Previously we always just assumed it was
+ a routine, which gave a confusing second error if it was a grouping.
+
+ - Improve error recovery after an unexpected token in `among`. Previously
+ we acted as if the unexpected token closed the `among` (this probably
+ wasn't intended but just a missing `break;` in a switch statement). Now we
+ issue an error and try the next token.
+
+* Report error instead of silently truncating character values (e.g. `hex 123`
+ previously silently became byte 0x23 which is `#` rather than a
+ g-with-cedilla).
+
+* Enlarge the initial input buffer size to 8192 bytes and double each time we
+ hit the end. Snowball programs are typically a few KB in size (with the
+ current largest we ship being the Greek stemmer at 27KB) so the previous
+ approach of starting with a 10 byte input buffer and increasing its size by
+ 50% plus 40 bytes each time it filled was inefficient, needing up to 15
+ reallocations to load greek.sbl.
+
+* Identify variables only used by one `routine`/`external`. This information
+ isn't yet used, but such variables which are also always written to before
+ being read can be emitted as local variables in most target languages.
+
+* We now allow multiple source files on command line, and allow them to be
+ after (or even interspersed) with options to better match modern Unix
+ conventions. Support for multiple source files allows specifying a single
+ byte character set mapping via a source file of `stringdef`.
+
+* Avoid infinite recursion in compiler when optimising a recursive snowball
+ function. Recursive functions aren't typical in snowball programs, but
+ the compiler shouldn't crash for any input, especially not a valid one.
+ We now simply limit on how deep the compiler will recurse and make the
+ pessimistic assumption in the unlikely event we hit this limit.
+
+Build system:
+
+* `make clean` in C libstemmer_c distribution now removes `examples/*.o`.
+ (#59)
+
+* Fix all the places which previously had to have a list of stemmers to work
+ dynamically or be generated, so now only modules.txt needs updating to add
+ a new stemmer.
+
+* Add check_java make target which runs tests for java.
+
+* Support gzipped test data (the uncompressed arabic test data is too big for
+ github).
+
+* GNUmakefile: Drop useless `-eprefix` and `-r` options from snowball
+ invocations for Java - these are only meaningful when generating C code.
+
+* Pass CFLAGS when linking which matches convention (e.g. automake does it) and
+ facilitates use of tools such as ASan. Fixes #84, reported by Thomas
+ Pointhuber.
+
+* Add CI builds with -std=c90 to check compiler and generated code are C90
+ (#54)
+
+libstemmer stuff:
+
+* Split out CPPFLAGS from CFLAGS and use CFLAGS when linking stemwords.
+
+* Add -O2 to CFLAGS.
+
+* Make generated tables of encodings and modules const.
+
+* Fix clang static analyzer memory leak warning (in practice this code path
+ can never actually be taken). Patch from Patrick O. Perry (#56)
+
+documentation
+
+* Added copyright and licensing details (#10).
+
+* Document that libstemmer supports ISO_8859_2 encoding. Currently hungarian
+ and romanian are available in ISO_8859_2.
+
+* Remove documentation falsely claiming that libstemmer supports CP850
+ encoding.
+
+* CONTRIBUTING.rst: Add guidance for contributing new stemming algorithms and
+ new language backends.
+
+* Overhaul libstemmer_python_README. Most notably, replace the benchmark data
+ which was very out of date.
diff --git a/contrib/snowball/README b/contrib/snowball/README
new file mode 100644
index 0000000..afb51b3
--- /dev/null
+++ b/contrib/snowball/README
@@ -0,0 +1,5 @@
+This contains the source code for the snowball compiler and the stemming
+algorithms on the website.
+
+See http://snowball.tartarus.org/ for more details.
+
diff --git a/contrib/snowball/algorithms/arabic.sbl b/contrib/snowball/algorithms/arabic.sbl
new file mode 100644
index 0000000..d827ee7
--- /dev/null
+++ b/contrib/snowball/algorithms/arabic.sbl
@@ -0,0 +1,561 @@
+/*
+ * Authors:
+ * - Assem Chelli, < assem [dot] ch [at] gmail >
+ * - Abdelkrim Aries <ab [underscore] aries [at] esi [dot] dz>
+ *
+*/
+
+stringescapes { }
+
+/* the Arabic letters in Unicode */
+// Hamza
+stringdef o '{U+0621}' // Hamza
+stringdef ao '{U+0623}' // Hamza above Alef
+stringdef ao_ '{U+0625}' // Hamza below Alef
+stringdef a~ '{U+0622}' // Alef madda
+stringdef wo '{U+0624}' // Hamza above waw
+stringdef yo '{U+0626}' // Hamza above yeh
+
+// Letters
+stringdef a '{U+0627}' // Alef
+stringdef a_ '{U+0649}' // Alef Maksura
+stringdef b '{U+0628}' // Beh
+stringdef t_ '{U+0629}' // Teh_Marbuta
+stringdef t '{U+062A}' // Teh
+stringdef th '{U+062B}' // Theh
+stringdef j '{U+062C}' // Jeem
+stringdef h '{U+062D}' // Hah
+stringdef x '{U+062E}' // Khah
+stringdef d '{U+062F}' // Dal
+stringdef dz '{U+0630}' // Thal
+stringdef r '{U+0631}' // Reh
+stringdef z '{U+0632}' // Zain
+stringdef s '{U+0633}' // Seen
+stringdef sh '{U+0634}' // Sheen
+stringdef c '{U+0635}' // Sad
+stringdef dh '{U+0636}' // Dad
+stringdef tt '{U+0637}' // Tah
+stringdef zh '{U+0638}' // Zah
+stringdef i '{U+0639}' // Ain
+stringdef gh '{U+063A}' // Ghain
+stringdef f '{U+0641}' // Feh
+stringdef q '{U+0642}' // Qaf
+stringdef k '{U+0643}' // Kaf
+stringdef l '{U+0644}' // Lam
+stringdef m '{U+0645}' // Meem
+stringdef n '{U+0646}' // Noon
+stringdef e '{U+0647}' // Heh
+stringdef w '{U+0648}' // Waw
+stringdef y '{U+064A}' // Yeh
+
+// Diacritics
+stringdef aan '{U+064B}' // FatHatan
+stringdef uun '{U+064C}' // Dammatan
+stringdef iin '{U+064D}' // Kasratan
+stringdef aa '{U+064E}' // FatHa
+stringdef uu '{U+064F}' // Damma
+stringdef ii '{U+0650}' // Kasra
+stringdef oo '{U+0652}' // Sukun
+stringdef ~ '{U+0651}' // Shadda
+
+// Hindu–Arabic numerals
+stringdef 0 '{U+0660}'
+stringdef 1 '{U+0661}'
+stringdef 2 '{U+0662}'
+stringdef 3 '{U+0663}'
+stringdef 4 '{U+0664}'
+stringdef 5 '{U+0665}'
+stringdef 6 '{U+0666}'
+stringdef 7 '{U+0667}'
+stringdef 8 '{U+0668}'
+stringdef 9 '{U+0669}'
+
+
+// Kasheeda
+stringdef _ '{U+0640}' // Kasheeda, Tatweel
+
+// Shaped forms
+stringdef o1 '{U+FE80}' // HAMZA
+stringdef ao1 '{U+FE83}' // ALEF_HAMZA_ABOVE
+stringdef ao2 '{U+FE84}' // ALEF_HAMZA_ABOVE
+stringdef ao_1 '{U+FE87}' // ALEF_HAMZA_BELOW
+stringdef ao_2 '{U+FE88}' // ALEF_HAMZA_BELOW
+stringdef yo1 '{U+FE8B}' // YEH_HAMZA
+stringdef yo2 '{U+FE8C}' // YEH_HAMZA
+stringdef yo3 '{U+FE89}' // YEH_HAMZA
+stringdef yo4 '{U+FE8A}' // YEH_HAMZA
+stringdef a~1 '{U+FE81}' // ALEF_MADDA
+stringdef a~2 '{U+FE82}' // ALEF_MADDA
+stringdef wo1 '{U+FE85}' // WAW_HAMZA
+stringdef wo2 '{U+FE86}' // WAW_HAMZA
+stringdef a1 '{U+FE8D}' // ALEF
+stringdef a2 '{U+FE8E}' // ALEF
+stringdef b1 '{U+FE8F}' // BEH
+stringdef b2 '{U+FE90}' // BEH
+stringdef b3 '{U+FE91}' // BEH
+stringdef b4 '{U+FE92}' // BEH
+stringdef t_1 '{U+FE93}' // TEH_MARBUTA
+stringdef t_2 '{U+FE94}' // TEH_MARBUTA
+stringdef t1 '{U+FE97}' // TEH
+stringdef t2 '{U+FE98}' // TEH
+stringdef t3 '{U+FE95}' // TEH
+stringdef t4 '{U+FE96}' // TEH
+stringdef th1 '{U+FE9B}' // THEH
+stringdef th2 '{U+FE9C}' // THEH
+stringdef th3 '{U+FE9A}' // THEH
+stringdef th4 '{U+FE99}' // THEH
+stringdef j1 '{U+FE9F}' // JEEM
+stringdef j2 '{U+FEA0}' // JEEM
+stringdef j3 '{U+FE9D}' // JEEM
+stringdef j4 '{U+FE9E}' // JEEM
+stringdef h1 '{U+FEA3}' // HAH
+stringdef h2 '{U+FEA4}' // HAH
+stringdef h3 '{U+FEA1}' // HAH
+stringdef h4 '{U+FEA2}' // HAH
+stringdef x1 '{U+FEA7}' // KHAH
+stringdef x2 '{U+FEA8}' // KHAH
+stringdef x3 '{U+FEA5}' // KHAH
+stringdef x4 '{U+FEA6}' // KHAH
+stringdef d1 '{U+FEA9}' // DAL
+stringdef d2 '{U+FEAA}' // DAL
+stringdef dz1 '{U+FEAB}' // THAL
+stringdef dz2 '{U+FEAC}' // THAL
+stringdef r1 '{U+FEAD}' // REH
+stringdef r2 '{U+FEAE}' // REH
+stringdef z1 '{U+FEAF}' // ZAIN
+stringdef z2 '{U+FEB0}' // ZAIN
+stringdef s1 '{U+FEB3}' // SEEN
+stringdef s2 '{U+FEB4}' // SEEN
+stringdef s3 '{U+FEB1}' // SEEN
+stringdef s4 '{U+FEB2}' // SEEN
+stringdef sh1 '{U+FEB7}' // SHEEN
+stringdef sh2 '{U+FEB8}' // SHEEN
+stringdef sh3 '{U+FEB5}' // SHEEN
+stringdef sh4 '{U+FEB6}' // SHEEN
+stringdef c1 '{U+FEBB}' // SAD
+stringdef c2 '{U+FEBC}' // SAD
+stringdef c3 '{U+FEB9}' // SAD
+stringdef c4 '{U+FEBA}' // SAD
+stringdef dh1 '{U+FEBF}' // DAD
+stringdef dh2 '{U+FEC0}' // DAD
+stringdef dh3 '{U+FEBD}' // DAD
+stringdef dh4 '{U+FEBE}' // DAD
+stringdef tt1 '{U+FEC3}' // TAH
+stringdef tt2 '{U+FEC4}' // TAH
+stringdef tt3 '{U+FEC1}' // TAH
+stringdef tt4 '{U+FEC2}' // TAH
+stringdef zh1 '{U+FEC7}' // ZAH
+stringdef zh2 '{U+FEC8}' // ZAH
+stringdef zh3 '{U+FEC5}' // ZAH
+stringdef zh4 '{U+FEC6}' // ZAH
+stringdef i1 '{U+FECB}' // AIN
+stringdef i2 '{U+FECC}' // AIN
+stringdef i3 '{U+FEC9}' // AIN
+stringdef i4 '{U+FECA}' // AIN
+stringdef gh1 '{U+FECF}' // GHAIN
+stringdef gh2 '{U+FED0}' // GHAIN
+stringdef gh3 '{U+FECD}' // GHAIN
+stringdef gh4 '{U+FECE}' // GHAIN
+stringdef f1 '{U+FED3}' // FEH
+stringdef f2 '{U+FED4}' // FEH
+stringdef f3 '{U+FED1}' // FEH
+stringdef f4 '{U+FED2}' // FEH
+stringdef q1 '{U+FED7}' // QAF
+stringdef q2 '{U+FED8}' // QAF
+stringdef q3 '{U+FED5}' // QAF
+stringdef q4 '{U+FED6}' // QAF
+stringdef k1 '{U+FEDB}' // KAF
+stringdef k2 '{U+FEDC}' // KAF
+stringdef k3 '{U+FED9}' // KAF
+stringdef k4 '{U+FEDA}' // KAF
+stringdef l1 '{U+FEDF}' // LAM
+stringdef l2 '{U+FEE0}' // LAM
+stringdef l3 '{U+FEDD}' // LAM
+stringdef l4 '{U+FEDE}' // LAM
+stringdef m1 '{U+FEE3}' // MEEM
+stringdef m2 '{U+FEE4}' // MEEM
+stringdef m3 '{U+FEE1}' // MEEM
+stringdef m4 '{U+FEE2}' // MEEM
+stringdef n1 '{U+FEE7}' // NOON
+stringdef n2 '{U+FEE8}' // NOON
+stringdef n3 '{U+FEE5}' // NOON
+stringdef n4 '{U+FEE6}' // NOON
+stringdef e1 '{U+FEEB}' // HEH
+stringdef e2 '{U+FEEC}' // HEH
+stringdef e3 '{U+FEE9}' // HEH
+stringdef e4 '{U+FEEA}' // HEH
+stringdef w1 '{U+FEED}' // WAW
+stringdef w2 '{U+FEEE}' // WAW
+stringdef a_1 '{U+FEEF}' // ALEF_MAKSURA
+stringdef a_2 '{U+FEF0}' // ALEF_MAKSURA
+stringdef y1 '{U+FEF3}' // YEH
+stringdef y2 '{U+FEF4}' // YEH
+stringdef y3 '{U+FEF1}' // YEH
+stringdef y4 '{U+FEF2}' // YEH
+
+// Ligatures Lam-Alef
+stringdef la '{U+FEFB}' // LAM_ALEF
+stringdef la2 '{U+FEFC}' // LAM_ALEF
+stringdef lao '{U+FEF7}' // LAM_ALEF_HAMZA_ABOVE
+stringdef lao2 '{U+FEF8}' // LAM_ALEF_HAMZA_ABOVE
+stringdef lao_ '{U+FEF9}' // LAM_ALEF_HAMZA_BELOW
+stringdef lao_2 '{U+FEFA}' // LAM_ALEF_HAMZA_BELOW
+stringdef la~ '{U+FEF5}' // LAM_ALEF_MADDA_ABOVE
+stringdef la~2 '{U+FEF6}' // LAM_ALEF_MADDA_ABOVE
+
+
+booleans (
+ is_noun
+ is_verb
+ is_defined
+ )
+
+routines (
+ Prefix_Step1
+ Prefix_Step2
+ Prefix_Step3a_Noun
+ Prefix_Step3b_Noun
+ Prefix_Step3_Verb
+ Prefix_Step4_Verb
+
+ Suffix_All_alef_maqsura
+ Suffix_Noun_Step1a
+ Suffix_Noun_Step1b
+ Suffix_Noun_Step2a
+ Suffix_Noun_Step2b
+ Suffix_Noun_Step2c1
+ Suffix_Noun_Step2c2
+ Suffix_Noun_Step3
+ Suffix_Verb_Step1
+ Suffix_Verb_Step2a
+ Suffix_Verb_Step2b
+ Suffix_Verb_Step2c
+
+ Normalize_post
+ Normalize_pre
+
+ Checks1
+)
+
+externals ( stem )
+
+groupings ( )
+
+
+// Normalizations
+define Normalize_pre as (
+ do repeat (
+ (
+ [substring] among (
+ '{aan}' '{uun}' '{iin}' '{aa}' '{uu}' '{ii}' '{oo}' '{~}'( delete ) // strip vocalization
+ '{_}' ( delete ) // strip kasheeda
+
+ // Hindu–Arabic numerals
+ '{0}' ( <- '0')
+ '{1}' ( <- '1')
+ '{2}' ( <- '2')
+ '{3}' ( <- '3')
+ '{4}' ( <- '4')
+ '{5}' ( <- '5')
+ '{6}' ( <- '6')
+ '{7}' ( <- '7')
+ '{8}' ( <- '8')
+ '{9}' ( <- '9')
+
+ // Shaped forms
+ '{o1}' ( <- '{o}' ) // HAMZA
+ '{ao1}' '{ao2}' ( <- '{ao}' ) // ALEF_HAMZA_ABOVE
+ '{ao_1}' '{ao_2}' ( <- '{ao_}' ) // ALEF_HAMZA_BELOW
+ '{yo1}' '{yo2}' '{yo3}' '{yo4}' ( <- '{yo}' ) // YEH_HAMZA
+ '{a~1}' '{a~2}'( <- '{a~}' ) // ALEF_MADDA
+ '{wo1}' '{wo2}'( <- '{wo}' ) // WAW_HAMZA
+ '{a1}' '{a2}' ( <- '{a}' ) // ALEF
+ '{b1}' '{b2}' '{b3}' '{b4}' ( <- '{b}' ) // BEH
+ '{t_1}' '{t_2}' ( <- '{t_}' ) // TEH_MARBUTA
+ '{t1}' '{t2}' '{t3}' '{t4}' ( <- '{t}' ) // TEH
+ '{th1}' '{th2}' '{th3}' '{th4}' ( <- '{th}' ) // THEH
+ '{j1}' '{j2}' '{j3}' '{j4}'( <- '{j}' ) // JEEM
+ '{h1}' '{h2}' '{h3}' '{h4}' ( <- '{h}' ) // HAH
+ '{x1}' '{x2}' '{x3}' '{x4}'( <- '{x}' ) // KHAH
+ '{d1}' '{d2}' ( <- '{d}' ) // DAL
+ '{dz1}''{dz2}' ( <- '{dz}' ) // THAL
+ '{r1}' '{r2}'( <- '{r}' ) // REH
+ '{z1}' '{z2}' ( <- '{z}' ) // ZAIN
+ '{s1}' '{s2}' '{s3}' '{s4}'( <- '{s}' ) // SEEN
+ '{sh1}' '{sh2}' '{sh3}' '{sh4}' ( <- '{sh}' ) // SHEEN
+ '{c1}' '{c2}' '{c3}' '{c4}'( <- '{c}' ) // SAD
+ '{dh1}' '{dh2}' '{dh3}' '{dh4}'( <- '{dh}' ) // DAD
+ '{tt1}' '{tt2}' '{tt3}' '{tt4}' ( <- '{tt}' ) // TAH
+ '{zh1}' '{zh2}' '{zh3}' '{zh4}'( <- '{zh}' ) // ZAH
+ '{i1}' '{i2}' '{i3}' '{i4}'( <- '{i}' ) // AIN
+ '{gh1}' '{gh2}' '{gh3}' '{gh4}'( <- '{gh}' ) // GHAIN
+ '{f1}' '{f2}' '{f3}' '{f4}' ( <- '{f}' ) // FEH
+ '{q1}' '{q2}' '{q3}' '{q4}' ( <- '{q}' ) // QAF
+ '{k1}' '{k2}' '{k3}' '{k4}'( <- '{k}' ) // KAF
+ '{l1}' '{l2}' '{l3}' '{l4}'( <- '{l}' ) // LAM
+ '{m1}' '{m2}' '{m3}' '{m4}' ( <- '{m}' ) // MEEM
+ '{n1}' '{n2}' '{n3}' '{n4}'( <- '{n}' ) // NOON
+ '{e1}' '{e2}' '{e3}' '{e4}' ( <- '{e}' ) // HEH
+ '{w1}' '{w2}' ( <- '{w}' ) // WAW
+ '{a_1}' '{a_2}' ( <- '{a_}' ) // ALEF_MAKSURA
+ '{y1}' '{y2}' '{y3}' '{y4}' ( <- '{y}' ) // YEH
+
+ // Ligatures Lam-Alef
+ '{la}' '{la2}' (<- '{l}{a}')
+ '{lao}' '{lao2}' (<- '{l}{ao}')
+ '{lao_}' '{lao_2}' (<- '{l}{ao_}')
+ '{la~}' '{la~2}' (<- '{l}{a~}')
+
+ )
+ )
+ or
+ next
+ )
+)
+
+define Normalize_post as (
+
+ do (
+ // normalize last hamza
+ backwards (
+ [substring] among (
+ '{ao}''{ao_}' '{a~}' ( <- '{o}')
+ '{wo}' ( <- '{o}')
+ '{yo}' ( <- '{o}')
+ )
+ )
+ )
+
+ do repeat (
+ (
+ // normalize other hamza's
+ [substring] among (
+ '{ao}''{ao_}' '{a~}' ( <- '{a}')
+ '{wo}' ( <- '{w}')
+ '{yo}' ( <- '{y}')
+ )
+ )
+ or
+ next
+ )
+)
+
+// Checks
+define Checks1 as (
+ [substring] among (
+ '{b}{a}{l}' '{k}{a}{l}' ($(len > 4) set is_noun unset is_verb set is_defined)
+ '{l}{l}' '{a}{l}' ($(len > 3) set is_noun unset is_verb set is_defined)
+ )
+)
+
+
+//prefixes
+define Prefix_Step1 as (
+ [substring] among (
+ '{ao}{ao}' ($(len > 3) <- '{ao}' )
+ '{ao}{a~}' ($(len > 3) <- '{a~}' )
+ '{ao}{wo}' ($(len > 3) <- '{ao}' )
+ '{ao}{a}' ($(len > 3) <- '{a}' )
+ '{ao}{ao_}' ($(len > 3) <- '{ao_}' )
+ // '{ao}' ($(len > 3) delete) //rare case
+ )
+)
+
+define Prefix_Step2 as (
+ not '{f}{a}'
+ not '{w}{a}'
+ [substring] among (
+ '{f}' ($(len > 3) delete)
+ '{w}' ($(len > 3) delete)
+ )
+)
+
+define Prefix_Step3a_Noun as ( // it is noun and defined
+ [substring] among (
+ '{b}{a}{l}' '{k}{a}{l}' ($(len > 5) delete)
+ '{l}{l}' '{a}{l}' ($(len > 4) delete)
+ )
+)
+
+define Prefix_Step3b_Noun as ( // probably noun and defined
+ not '{b}{a}' // exception
+ [substring] among (
+ '{b}' ($(len > 3) delete)
+ // '{k}' '{l}' ($(len > 3) delete) // BUG: cause confusion
+ '{b}{b}' ($(len > 3) <- '{b}' )
+ '{k}{k}' ($(len > 3) <- '{k}' )
+ )
+
+)
+
+define Prefix_Step3_Verb as (
+ [substring] among (
+ //'{s}' ($(len > 4) delete)// BUG: cause confusion
+ '{s}{y}' ($(len > 4) <- '{y}' )
+ '{s}{t}' ($(len > 4) <- '{t}')
+ '{s}{n}' ($(len > 4) <- '{n}')
+ '{s}{ao}' ($(len > 4) <- '{ao}')
+ )
+)
+
+define Prefix_Step4_Verb as (
+ [substring] among (
+ '{y}{s}{t}' '{n}{s}{t}' '{t}{s}{t}' ($(len > 4) set is_verb unset is_noun <- '{a}{s}{t}' )
+ )
+)
+
+// suffixes
+backwardmode (
+
+ define Suffix_Noun_Step1a as (
+ [substring] among (
+ '{y}' '{k}' '{e}' ($(len >= 4) delete)
+ '{n}{a}' '{k}{m}' '{e}{a}' '{e}{n}' '{e}{m}' ($(len >= 5) delete)
+ '{k}{m}{a}' '{e}{m}{a}' ($(len >= 6) delete)
+ )
+ )
+ define Suffix_Noun_Step1b as (
+ [substring] among (
+ '{n}' ($(len > 5) delete)
+ )
+ )
+
+ define Suffix_Noun_Step2a as (
+ [substring] among (
+ '{a}' '{y}' '{w}' ($(len > 4) delete)
+ )
+ )
+
+ define Suffix_Noun_Step2b as (
+ [substring] among (
+ '{a}{t}' ($(len >= 5) delete)
+ )
+ )
+
+ define Suffix_Noun_Step2c1 as (
+ [substring] among (
+ '{t}' ($(len >= 4) delete)
+ )
+ )
+ define Suffix_Noun_Step2c2 as ( // feminine t_
+ [substring] among (
+ '{t_}' ($(len >= 4) delete)
+ )
+ )
+ define Suffix_Noun_Step3 as ( // ya' nisbiya
+ [substring] among (
+ '{y}' ($(len >= 3) delete)
+ )
+ )
+
+ define Suffix_Verb_Step1 as (
+ [substring] among (
+ '{e}' '{k}' ($(len >= 4) delete)
+ '{n}{y}' '{n}{a}' '{e}{a}' '{e}{m}' '{e}{n}' '{k}{m}' '{k}{n}' ($(len >= 5) delete)
+ '{e}{m}{a}' '{k}{m}{a}' '{k}{m}{w}'($(len >= 6) delete)
+ )
+ )
+ define Suffix_Verb_Step2a as (
+ [substring] among (
+ '{t}' ($(len >= 4) delete)
+ '{a}' '{n}' '{y}' ($(len >= 4) delete)
+ '{n}{a}' '{t}{a}' '{t}{n}' ($(len >= 5) delete)// past
+ '{a}{n}' '{w}{n}' '{y}{n}' ($(len > 5) delete) // present
+ '{t}{m}{a}' ($(len >= 6) delete)
+ )
+ )
+
+ define Suffix_Verb_Step2b as (
+ [substring] among (
+ '{w}{a}' '{t}{m}' ($(len >= 5) delete)
+ )
+ )
+
+
+ define Suffix_Verb_Step2c as (
+ [substring] among (
+ '{w}' ($(len >= 4) delete)
+ '{t}{m}{w}' ($(len >= 6) delete)
+ )
+ )
+
+ define Suffix_All_alef_maqsura as (
+ [substring] among (
+ '{a_}' ( <- '{y}' ) // spell error
+ // '{a_}' ( delete ) // if noun > 3
+ // '{a_}' ( <- '{a}') // if verb
+ )
+ )
+)
+
+define stem as (
+ // set initial values
+ set is_noun
+ set is_verb
+ unset is_defined
+
+ // guess type and properties
+ do Checks1
+
+ // normalization pre-stemming
+ do Normalize_pre
+
+
+ backwards (
+
+ do (
+ //Suffixes for verbs
+ (
+ is_verb
+ (
+ (
+ (atleast 1 Suffix_Verb_Step1)
+ ( Suffix_Verb_Step2a or Suffix_Verb_Step2c or next)
+ )
+ or Suffix_Verb_Step2b
+ or Suffix_Verb_Step2a
+ )
+ )
+ //Suffixes for nouns
+ or (
+ is_noun
+ (
+
+ try (
+ Suffix_Noun_Step2c2
+ or (not is_defined Suffix_Noun_Step1a (
+ Suffix_Noun_Step2a
+ or Suffix_Noun_Step2b
+ or Suffix_Noun_Step2c1
+ or next))
+ or (Suffix_Noun_Step1b (
+ Suffix_Noun_Step2a
+ or Suffix_Noun_Step2b
+ or Suffix_Noun_Step2c1))
+ or (not is_defined Suffix_Noun_Step2a)
+ or (Suffix_Noun_Step2b)
+ )
+ Suffix_Noun_Step3
+ )
+
+ )
+
+ // Suffixes for alef maqsura
+ or Suffix_All_alef_maqsura
+ )
+ )
+
+ //Prefixes
+ do (
+ try Prefix_Step1
+ try Prefix_Step2
+ ( Prefix_Step3a_Noun
+ or (is_noun Prefix_Step3b_Noun)
+ or (is_verb try Prefix_Step3_Verb Prefix_Step4_Verb)
+ )
+ )
+
+ // normalization post-stemming
+ do Normalize_post
+
+)
diff --git a/contrib/snowball/algorithms/basque.sbl b/contrib/snowball/algorithms/basque.sbl
new file mode 100644
index 0000000..267abc7
--- /dev/null
+++ b/contrib/snowball/algorithms/basque.sbl
@@ -0,0 +1,149 @@
+routines (
+ aditzak
+ izenak
+ adjetiboak
+ mark_regions
+ RV R2 R1
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef n~ '{U+00F1}'
+
+define v 'aeiou'
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v (non-v gopast v) or (v gopast non-v) )
+ or
+ ( non-v (non-v gopast v) or (v next) )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R2 as $p2 <= cursor
+ define R1 as $p1 <= cursor
+
+ define aditzak as (
+ [substring] among(
+ 'le' 'la' 'tzaile' 'aldatu' 'atu' 'tzailea' 'taile' 'tailea' 'pera' 'gale' 'galea'
+ 'gura' 'kura' 'kor' 'korra' 'or' 'orra' 'tun' 'tuna' 'gaitz' 'gaitza'
+ 'kaitz' 'kaitza' 'ezin' 'ezina' 'tezin' 'tezina' 'errez' 'erreza'
+ 'karri' 'karria' 'tzaga' 'tzaka' 'tzake' 'tzeke' 'ez' 'eza' 'tzez'
+ 'keta' 'eta' 'etan' 'pen' 'pena' 'tze' 'atze' 'kuntza' 'kunde' 'kundea'
+ 'kune' 'kunea' 'kuna' 'kera' 'era' 'kizun' 'kizuna' 'dura' 'tura' 'men' 'mena'
+ 'go' 'ago' 'tio' 'taldi' 'taldia' 'aldi' 'aldia' 'gune' 'gunea' 'bide' 'bidea'
+ 'pide' 'pidea' 'gai' 'gaia' 'ki' 'kin' 'rekin' 'kina' 'kari' 'karia' 'ari' 'tari' 'etari'
+ 'gailu' 'gailua' 'kide' 'kidea' 'ide' 'idea' 'du' 'ka' 'kan' 'an' 'ean' 'tu' 'lari' 'tatu'
+ 'rean' 'tarazi' 'arazi' 'tzat' 'bera' 'dako'
+ ( RV delete )
+ 'garri' 'garria' 'tza'
+ (R2 delete)
+ 'atseden'
+ (<- 'atseden')
+ 'arabera'
+ (<- 'arabera')
+ 'baditu'
+ (<- 'baditu')
+
+ )
+ )
+
+ define izenak as (
+ [substring] among(
+ 'ari' 'aria' 'bizia' 'kari' 'karia' 'lari' 'laria' 'tari' 'taria' 'zain' 'zaina'
+ 'tzain' 'tzaina' 'zale' 'zalea' 'tzale' 'tzalea' 'aizun' 'orde' 'ordea'
+ 'burua' 'ohi' 'ohia' 'kintza' 'gintzo' 'gintzu' 'tzu' 'tzua'
+ 'tzo' 'tzoa' 'kuntza' 'talde' 'taldea' 'eria' 'keria' 'teria' 'di'
+ 'za' 'ada' 'tara' 'etara' 'tra' 'ta' 'tegi' 'tegia' 'keta' 'z' 'zko' 'zkoa'
+ 'ti' 'tia' 'tsu' 'tsua' 'zu' 'zua' 'bera' 'pera' 'zto' 'ztoa' 'asi' 'asia'
+ 'gile' 'gilea' 'estu' 'estua' 'larri' 'larria' 'nahi' 'nahia'
+ 'koi' 'koia' 'oi' 'oia' 'goi' 'min' 'mina' 'dun' 'duna' 'duru' 'durua'
+ 'duri' 'duria' 'os' 'osa' 'oso' 'osoa' 'ar' 'ara' 'tar' 'dar' 'dara'
+ 'tiar' 'tiara' 'liar' 'liara' 'gabe' 'gabea' 'kabe' 'kabea' 'ga' 'ge'
+ 'kada' 'tasun' 'tasuna' 'asun' 'asuna' 'go' 'mendu' 'mendua' 'mentu' 'mentua'
+ 'mendi' 'mendia' 'zio' 'zioa' 'zino' 'zinoa' 'zione' 'zionea' 'ezia'
+ 'degi' 'degia' 'egi' 'egia' 'toki' 'tokia' 'leku' 'lekua' 'gintza' 'alde'
+ 'aldea' 'kalde' 'kaldea' 'gune' 'gunea' 'une' 'unea' 'una' 'pe' 'pea'
+ 'gibel' 'gibela' 'ondo' 'ondoa' 'arte' 'artea' 'aurre' 'aurrea'
+ 'etxe' 'etxea' 'ola' 'ontzi' 'ontzia' 'gela' 'denda' 'taldi' 'taldia'
+ 'aldi' 'aldia' 'te' 'tea' 'zaro' 'zaroa' 'taro' 'taroa' 'oro' 'oroa'
+ 'aro' 'aroa' 'ero' 'eroa' 'eroz' 'eroza' 'ka' 'kan' 'kana' 'tako' 'etako' 'takoa'
+ 'kote' 'kotea' 'tzar' 'tzarra' 'handi' 'handia' 'kondo' 'kondoa' 'skila'
+ 'no' 'noa' '{n~}o' '{n~}oa' 'ska' 'xka' 'zka' 'tila' 'to' 'toa' 'tto' 'ttoa'
+ 'txo' 'txoa' 'txu' 'txua' 'anda' 'anga' 'urren' 'urrena' 'gai' 'gaia'
+ 'gei' 'geia' 'eme' 'emea' 'kume' 'kumea' 'sa' 'ko' 'eko' 'koa' 'ena'
+ 'enea' 'ne' 'nea' 'kor' 'korra' 'ez' 'eza' 'eta' 'etan'
+ 'ki' 'kia' 'kin' 'kina' 'tu' 'tua' 'du' 'dua' 'ek'
+ 'tarik' 'tariko' 'tan' 'ordu' 'ordua' 'oste' 'ostea' 'tzara'
+ 'ra' 'antza' 'behar' 'ro' 'giro' 'ak' 'zp' 'ket'
+ 'kail' 'kaila' 'ail' 'kirri' 'kirria' 'ngo' 'ngoa' '{n~}i' 'sko'
+ 'sta' 'koitz' 'koitza' 'na' 'garren' 'garrena' 'kera'
+ 'gerren' 'gerrena' 'garna' 'kide' 'tz' 'tuko'
+ ( RV delete )
+ 'ora' 'garri' 'garria' 'or' 'buru' 'ren' 'tza'
+ ( R2 delete )
+ 'joka'
+ (<- 'jok')
+ 'tzen' 'ten' 'en' 'tatu'
+ (R1 delete)
+ 'trako'
+ (<- 'tra')
+ 'minutuko'
+ (<- 'minutu')
+ 'zehar'
+ (<- 'zehar')
+ 'geldi'
+ (<- 'geldi')
+ 'igaro'
+ (<- 'igaro')
+ 'aurka'
+ (<- 'aurka')
+ )
+ )
+
+ define adjetiboak as (
+ [substring] among(
+ 'era' 'ero' 'go' 'tate' 'tade' 'date' 'dade' 'keria'
+ 'ki' 'to' 'ro' 'la' 'gi' 'larik' 'lanik' 'ik' 'ztik' 'rik'
+ ( RV delete )
+ 'zlea'
+ (<- 'z')
+ )
+ )
+
+)
+
+define stem as (
+ do mark_regions
+ backwards (
+ repeat aditzak
+ repeat izenak
+ do adjetiboak
+ )
+
+)
+
+/*
+ Note 1: additions of 21 Jul 2010
+*/
diff --git a/contrib/snowball/algorithms/catalan.sbl b/contrib/snowball/algorithms/catalan.sbl
new file mode 100644
index 0000000..0a1e3a5
--- /dev/null
+++ b/contrib/snowball/algorithms/catalan.sbl
@@ -0,0 +1,202 @@
+routines (
+ cleaning mark_regions
+ R1 R2
+ attached_pronoun
+ standard_suffix
+ verb_suffix
+ residual_suffix
+)
+
+externals ( stem )
+
+integers ( p1 p2 )
+
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a' '{U+00E1}' // a-acute
+stringdef a` '{U+00E0}' // a-grave
+stringdef c, '{U+00E7}' // c-cedilla
+stringdef e' '{U+00E9}' // e-acute
+stringdef e` '{U+00E8}' // e-grave
+stringdef i' '{U+00ED}' // i-acute
+stringdef i` '{U+00EC}' // i-grave
+stringdef i" '{U+00EF}' // i-diaeresis
+stringdef o' '{U+00F3}' // o-acute
+stringdef o` '{U+00F2}' // o-grave
+stringdef u' '{U+00FA}' // u-acute
+stringdef u" '{U+00FC}' // u-diaeresis
+stringdef . '{U+00B7}' // - per l aggeminades
+
+define v 'aeiou{a'}{a`}{e'}{e`}{i'}{i"}{o'}{o`}{u'}{u"}'
+
+define mark_regions as (
+
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define cleaning as repeat (
+ [substring] among(
+ '{a'}' (<- 'a')
+ '{a`}' (<- 'a')
+ '{e'}' (<- 'e')
+ '{e`}' (<- 'e')
+ '{i'}' (<- 'i')
+ '{i`}' (<- 'i')
+ '{o'}' (<- 'o')
+ '{o`}' (<- 'o')
+ '{u'}' (<- 'u')
+ '{u"}' (<- 'u')
+ '{i"}' (<- 'i')
+ '{.}' (<- '.')
+ '' (next)
+ )
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define attached_pronoun as (
+ [substring] among (
+ '{'}s' '{'}hi' '{'}ho' '{'}l' '{'}ls'
+ '-ls' '-la' '-les' '-li'
+ 'vos' 'se' 'nos' '-nos' '-us' 'us'
+ '{'}n' '{'}ns' '-n' '-ns'
+ '{'}m' '-me' '-m'
+ '-te' '{'}t'
+ 'li' 'lo' 'los'
+ 'me' 'sela' 'selo' 'selas' 'selos' 'le'
+ 'la' 'las' 'les' 'ens' 'ho' 'hi'
+ (R1 delete)
+ )
+ )
+
+ define standard_suffix as (
+ [substring] among(
+ 'ar' 'atge' 'formes' 'icte' 'ictes'
+ 'ell' 'ells' 'ella' '{e'}s' '{e`}s' 'esc' 'essa' 'et' 'ets' 'eta'
+ 'eres' 'eries' 'ers' 'ina' 'ines' 'able' 'ls'
+ 'i{o'}' 'itat' 'itats' 'itzar' 'iva' 'ives' 'ivisme' 'ius'
+ 'fer' 'ment' 'amen' 'ament' 'aments' 'ments' 'ot' 'sfera' 'al' 'als' 'era' 'ana' 'iste'
+ 'aire' 'eria' 'esa' 'eses' 'esos' 'or' '{i'}cia' '{i'}cies' 'icis' 'ici' '{i'}ci' '{i'}cis'
+ '{a`}ria' '{a`}ries' 'alla' 'ci{o'}' 'cions' 'n{c,}a' 'nces' '{o'}' 'dor' 'all'
+ 'il' '{i'}stic' 'enc' 'enca' '{i'}s' 'issa' 'issos' '{i'}ssem' '{i'}ssiu' 'issem' 'isseu' '{i'}sseu'
+ '{o'}s' 'osa' 'dora' 'dores' 'dors' 'adura' 'ble' 'bles' '{i'}vol' '{i'}vola' 'd{i'}s' 'egar' 'ejar' 'ificar'
+ 'itar' 'ables' 'adors' 'idores' 'idors'
+ 'adora' 'aci{o'}' 'doras' 'dur' 'dures' 'alleng{u"}es'
+ 'ant' 'ants' 'ancia' 'ancies' 'at{o`}ria' 'at{o`}ries' 'tori' 'toris'
+ 'ats' 'ions' 'ota' 'isam' 'ors' 'ora' 'ores' 'isament'
+ 'bilitat' 'bilitats' 'ivitat' 'ivitats' 'ari' 'aris' 'ionisme' 'ionista' 'ionistes'
+ 'ialista' 'ialistes' 'ialisme' 'ialismes' 'ud' 'uts' 'uds' 'encia' 'encies' '{e`}ncia' '{e`}ncies'
+ '{i"}tat' '{i"}tats' 'atiu' 'atius' 'atives' 'ativa' 'ativitat' 'ativitats' 'ible' 'ibles'
+ 'assa' 'asses' 'assos'
+ 'ent' 'ents'
+ '{i'}ssim' '{i'}ssima' '{i'}ssims' '{i'}ssimes' '{i`}ssem' '{i`}sseu' '{i`}ssin'
+ 'ims' 'ima' 'imes'
+ 'isme' 'ista' 'ismes' 'istes'
+ 'inia' 'inies' '{i'}inia' '{i'}nies' 'ita' 'ites' 'triu' 'trius'
+ 'oses' 'osos' 'ient' 'otes' 'ots'
+ (R1 delete)
+ 'acions' 'ada' 'ades'
+ (R2 delete)
+ 'log{i'}a' 'log{i'}es''logia' 'logies' 'logi' 'logis' 'l{o'}gica' 'l{o'}gics' 'l{o'}giques'
+ (R2 <- 'log')
+ 'ic' 'ica' 'ics' 'iques'
+ (R2 <- 'ic')
+ 'qu{i'}ssim' 'qu{i'}ssims' 'qu{i'}ssimes' 'qu{i'}ssima'
+ (R1 <- 'c')
+ )
+ )
+
+ define verb_suffix as (
+ [substring] among(
+ 'ador' 'adora' 'adors' 'adores' 're' 'ie'
+ 'ent' 'ents' 'udes' 'ar{a`}' 'eren'
+ 'ar{a'}' 'ar{i'}an' 'ar{i'}as' 'ar{a'}n' 'ar{a'}s' 'ar{i'}ais'
+ 'aria' 'arian' 'arien' 'aries' 'ar{a`}s'
+ 'ar{i'}a' 'ar{e'}is' 'ar{i'}amos' 'aremos' 'ara'
+ 'ar{e'}' 'ar{e'}s'
+ 'er{i'}an' 'er{i'}as' 'er{a'}n' 'er{a'}s' 'er{i'}ais'
+ 'er{i'}a' 'er{e'}is' 'er{i'}amos' 'eremos' 'er{a'}'
+ 'er{e'}' 'er' 'erau' 'erass'
+ 'ir{i'}an' 'ir{i'}as' 'ir{a'}n' 'ir{a'}s' 'ir{i'}ais'
+ 'ir{i'}a' 'ir{e'}is' 'ir{i'}amos' 'iremos' 'ir{a'}'
+ 'ir{e'}' '{i'}rem' '{i'}reu' '{i'}eu'
+ 'ia' 'ies' '{i'}em' '{i`}eu' 'ien'
+ 'at' 'ut' 'uda' 'ava' 'aves' 'avem' '{a'}vem' '{a`}vem' '{a`}veu' '{a'}veu' 'aven' 'au' 'ats'
+ 'asseu' 'esseu' 'eresseu' '{a`}sseu' '{a`}ssem' '{a`}ssim' '{a`}ssiu'
+ 'essen' 'esses' 'assen' 'asses' 'assim' 'assiu'
+ '{e'}ssen' '{e'}sseu' '{e'}ssim' '{e'}ssiu' '{e'}ssem'
+ '{i'}' 'ares' '{a`}rem' '{a`}reu' '{a`}ren'
+ 'ar{i'}em' 'ar{i'}eu'
+ 'areu' 'aren' 'ant' '{i"}m' '{i"}u'
+ '{e'}s' '{i"}en' 'en' 'es' 'em' 'am' 'ams' '{i"}a' '{i"}es'
+ 'dre' 'eix' 'eixer' 'tzar' 'eixes' 'ides' '{i"}des' 'it' '{i"}t' '{i"}da'
+ 'aba' 'ada' 'ades' 'ida' '{i'}a' 'iera' 'ad' 'ed' 'its'
+ 'id' 'ids' 'ase' 'iese' 'aste' 'iste' 'an' 'aban' '{i'}an'
+ 'aran' 'ieran' 'asen' 'iesen' 'aron' 'ieron' 'ado'
+ 'ido' 'iendo' 'i{o'}' 'ar' 'ir' 'as'
+ 'ieu' 'ii' 'io' 'i{a`}'
+ 'ess' 'essin' 'essis' 'ass' 'assin' 'assis' 'essim' '{e`}ssim' '{e`}ssiu'
+ 'abas' 'adas' 'idas' '{i'}as' 'aras' 'ieras' 'ases'
+ 'ieses' '{i'}s' '{a'}is' 'abais' '{i'}ais' 'arais'
+ 'ierais' 'aseis' 'ieseis' 'asteis' 'isteis' 'ados'
+ 'idos' 'amos' '{a'}bamos' '{i'}amos' 'imos' 'ques'
+ '{a'}ramos' 'i{e'}ramos' 'i{e'}semos' '{a'}semos'
+ 'ira' 'iran' 'irem' 'iren' 'ires' 'ireu' 'iria' 'irien'
+ 'iries' 'ir{a`}' 'ir{a`}s' 'ir{e`}' 'ir{i`}em' 'ir{i`}eu'
+ 'isquen' 'iguem' 'igueu' 'esqui' 'esquin' 'esquis' 'eixi' 'eixin' 'eixis'
+ 'eixen' 'eixo' 'isin' 'isis' 'esques' 'sis' 'sin'
+ 'int' 'ir{i'}em' 'ir{i'}eu' 'isc' 'atges' 'esca' 'esquen'
+ 'issen' 'isses' 'issin' 'issis' 'isca' 'issiu' 'issim'
+ '{i"}sc' '{i"}sca' '{i"}ssin' '{i'}ssiu' '{i'}ssim' '{i"}ssis' '{i"}guem' '{i"}gueu'
+ '{i"}ra' '{i"}ren' '{i"}res'
+ '{i"}squen' '{i"}sques' '{i"}ssen' '{i"}sses' '{i"}xo' '{i"}xen' '{i"}xes' '{i"}x'
+ 'ixo' 'ixen' 'ixes' 'ix' 'ixa' 'inin' 'inis' 'ini' 'ineu' 'itza' 'itzi' 'itzeu' 'itzis'
+ 'itzo' 'itz' 'itz{a`}' 'arem' 'in' '{a`}s' 'i{i"}' 'i{i"}n' 'i{i"}s'
+ (R1 delete)
+ 'ando'
+ (R2 delete)
+ )
+ )
+
+ define residual_suffix as (
+ [substring] among(
+ 'os' 'a' 'o' '{a'}' '{a`}' '{i'}' '{o'}' 'e' '{e'}' 'eu' 'iu'
+ 'is' 'i' 'ir' 's' '{i`}' 'itz' '{i"}' '{i"}n' '{i"}s' 'it'
+ (R1 delete)
+ 'iqu'
+ (R1 <- 'ic')
+ )
+ )
+)
+
+define stem as (
+ do mark_regions
+ backwards (
+ do attached_pronoun
+ do ( standard_suffix or
+ verb_suffix
+ )
+ do residual_suffix
+ )
+ do cleaning
+)
+
+/*
+ First works 2010/07/19
+ First Grammatical Reviews: https://ca.wikipedia.org/wiki/Gram%C3%A0tica_del_catal%C3%A0
+ Suffix list: https://ca.wikipedia.org/wiki/Llista_de_sufixos
+ Irregular Verbs: https://ca.wikipedia.org/wiki/Flexi%C3%B3_verbal_del_catal%C3%A0
+*/
diff --git a/contrib/snowball/algorithms/danish.sbl b/contrib/snowball/algorithms/danish.sbl
new file mode 100644
index 0000000..761270f
--- /dev/null
+++ b/contrib/snowball/algorithms/danish.sbl
@@ -0,0 +1,93 @@
+routines (
+ mark_regions
+ main_suffix
+ consonant_pair
+ other_suffix
+ undouble
+)
+
+externals ( stem )
+
+strings ( ch )
+
+integers ( p1 x )
+
+groupings ( c v s_ending )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef ae '{U+00E6}'
+stringdef ao '{U+00E5}'
+stringdef o/ '{U+00F8}'
+
+define c 'bcdfghjklmnpqrstvwxz'
+
+define v 'aeiouy{ae}{ao}{o/}'
+
+define s_ending 'abcdfghjklmnoprtvyz{ao}'
+
+define mark_regions as (
+
+ $p1 = limit
+
+ test ( hop 3 setmark x )
+ goto v gopast non-v setmark p1
+ try ( $p1 < x $p1 = x )
+)
+
+backwardmode (
+
+ define main_suffix as (
+ setlimit tomark p1 for ([substring])
+ among(
+
+ 'hed' 'ethed' 'ered' 'e' 'erede' 'ende' 'erende' 'ene' 'erne' 'ere'
+ 'en' 'heden' 'eren' 'er' 'heder' 'erer' 'heds' 'es' 'endes'
+ 'erendes' 'enes' 'ernes' 'eres' 'ens' 'hedens' 'erens' 'ers' 'ets'
+ 'erets' 'et' 'eret'
+ (delete)
+ 's'
+ (s_ending delete)
+ )
+ )
+
+ define consonant_pair as (
+ test (
+ setlimit tomark p1 for ([substring])
+ among(
+ 'gd' // significant in the call from other_suffix
+ 'dt' 'gt' 'kt'
+ )
+ )
+ next] delete
+ )
+
+ define other_suffix as (
+ do ( ['st'] 'ig' delete )
+ setlimit tomark p1 for ([substring])
+ among(
+ 'ig' 'lig' 'elig' 'els'
+ (delete do consonant_pair)
+ 'l{o/}st'
+ (<-'l{o/}s')
+ )
+ )
+ define undouble as (
+ setlimit tomark p1 for ([c] ->ch)
+ ch
+ delete
+ )
+)
+
+define stem as (
+
+ do mark_regions
+ backwards (
+ do main_suffix
+ do consonant_pair
+ do other_suffix
+ do undouble
+ )
+)
diff --git a/contrib/snowball/algorithms/dutch.sbl b/contrib/snowball/algorithms/dutch.sbl
new file mode 100644
index 0000000..f24c82d
--- /dev/null
+++ b/contrib/snowball/algorithms/dutch.sbl
@@ -0,0 +1,164 @@
+routines (
+ prelude postlude
+ e_ending
+ en_ending
+ mark_regions
+ R1 R2
+ undouble
+ standard_suffix
+)
+
+externals ( stem )
+
+booleans ( e_found )
+
+integers ( p1 p2 )
+
+groupings ( v v_I v_j )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a" '{U+00E4}'
+stringdef e" '{U+00EB}'
+stringdef i" '{U+00EF}'
+stringdef o" '{U+00F6}'
+stringdef u" '{U+00FC}'
+
+stringdef a' '{U+00E1}'
+stringdef e' '{U+00E9}'
+stringdef i' '{U+00ED}'
+stringdef o' '{U+00F3}'
+stringdef u' '{U+00FA}'
+
+stringdef e` '{U+00E8}'
+
+define v 'aeiouy{e`}'
+define v_I v + 'I'
+define v_j v + 'j'
+
+define prelude as (
+ test repeat (
+ [substring] among(
+ '{a"}' '{a'}'
+ (<- 'a')
+ '{e"}' '{e'}'
+ (<- 'e')
+ '{i"}' '{i'}'
+ (<- 'i')
+ '{o"}' '{o'}'
+ (<- 'o')
+ '{u"}' '{u'}'
+ (<- 'u')
+ '' (next)
+ ) //or next
+ )
+ try(['y'] <- 'Y')
+ repeat goto (
+ v [('i'] v <- 'I') or
+ ('y'] <- 'Y')
+ )
+)
+
+define mark_regions as (
+
+ $p1 = limit
+ $p2 = limit
+
+ gopast v gopast non-v setmark p1
+ try($p1 < 3 $p1 = 3) // at least 3
+ gopast v gopast non-v setmark p2
+
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'Y' (<- 'y')
+ 'I' (<- 'i')
+ '' (next)
+ ) //or next
+
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define undouble as (
+ test among('kk' 'dd' 'tt') [next] delete
+ )
+
+ define e_ending as (
+ unset e_found
+ ['e'] R1 test non-v delete
+ set e_found
+ undouble
+ )
+
+ define en_ending as (
+ R1 non-v and not 'gem' delete
+ undouble
+ )
+
+ define standard_suffix as (
+ do (
+ [substring] among(
+ 'heden'
+ ( R1 <- 'heid'
+ )
+ 'en' 'ene'
+ ( en_ending
+ )
+ 's' 'se'
+ ( R1 non-v_j delete
+ )
+ )
+ )
+ do e_ending
+
+ do ( ['heid'] R2 not 'c' delete
+ ['en'] en_ending
+ )
+
+ do (
+ [substring] among(
+ 'end' 'ing'
+ ( R2 delete
+ (['ig'] R2 not 'e' delete) or undouble
+ )
+ 'ig'
+ ( R2 not 'e' delete
+ )
+ 'lijk'
+ ( R2 delete e_ending
+ )
+ 'baar'
+ ( R2 delete
+ )
+ 'bar'
+ ( R2 e_found delete
+ )
+ )
+ )
+ do (
+ non-v_I
+ test (
+ among ('aa' 'ee' 'oo' 'uu')
+ non-v
+ )
+ [next] delete
+ )
+ )
+)
+
+define stem as (
+
+ do prelude
+ do mark_regions
+ backwards
+ do standard_suffix
+ do postlude
+)
diff --git a/contrib/snowball/algorithms/english.sbl b/contrib/snowball/algorithms/english.sbl
new file mode 100644
index 0000000..fe18d7a
--- /dev/null
+++ b/contrib/snowball/algorithms/english.sbl
@@ -0,0 +1,229 @@
+integers ( p1 p2 )
+booleans ( Y_found )
+
+routines (
+ prelude postlude
+ mark_regions
+ shortv
+ R1 R2
+ Step_1a Step_1b Step_1c Step_2 Step_3 Step_4 Step_5
+ exception1
+ exception2
+)
+
+externals ( stem )
+
+groupings ( v v_WXY valid_LI )
+
+stringescapes {}
+
+define v 'aeiouy'
+define v_WXY v + 'wxY'
+
+define valid_LI 'cdeghkmnrt'
+
+define prelude as (
+ unset Y_found
+ do ( ['{'}'] delete)
+ do ( ['y'] <-'Y' set Y_found)
+ do repeat(goto (v ['y']) <-'Y' set Y_found)
+)
+
+define mark_regions as (
+ $p1 = limit
+ $p2 = limit
+ do(
+ among (
+ 'gener'
+ 'commun' // added May 2005
+ 'arsen' // added Nov 2006 (arsenic/arsenal)
+ // ... extensions possible here ...
+ ) or (gopast v gopast non-v)
+ setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+backwardmode (
+
+ define shortv as (
+ ( non-v_WXY v non-v )
+ or
+ ( non-v v atlimit )
+ )
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define Step_1a as (
+ try (
+ [substring] among (
+ '{'}' '{'}s' '{'}s{'}'
+ (delete)
+ )
+ )
+ [substring] among (
+ 'sses' (<-'ss')
+ 'ied' 'ies'
+ ((hop 2 <-'i') or <-'ie')
+ 's' (next gopast v delete)
+ 'us' 'ss'
+ )
+ )
+
+ define Step_1b as (
+ [substring] among (
+ 'eed' 'eedly'
+ (R1 <-'ee')
+ 'ed' 'edly' 'ing' 'ingly'
+ (
+ test gopast v delete
+ test substring among(
+ 'at' 'bl' 'iz'
+ (<+ 'e')
+ 'bb' 'dd' 'ff' 'gg' 'mm' 'nn' 'pp' 'rr' 'tt'
+ // ignoring double c, h, j, k, q, v, w, and x
+ ([next] delete)
+ '' (atmark p1 test shortv <+ 'e')
+ )
+ )
+ )
+ )
+
+ define Step_1c as (
+ ['y' or 'Y']
+ non-v not atlimit
+ <-'i'
+ )
+
+ define Step_2 as (
+ [substring] R1 among (
+ 'tional' (<-'tion')
+ 'enci' (<-'ence')
+ 'anci' (<-'ance')
+ 'abli' (<-'able')
+ 'entli' (<-'ent')
+ 'izer' 'ization'
+ (<-'ize')
+ 'ational' 'ation' 'ator'
+ (<-'ate')
+ 'alism' 'aliti' 'alli'
+ (<-'al')
+ 'fulness' (<-'ful')
+ 'ousli' 'ousness'
+ (<-'ous')
+ 'iveness' 'iviti'
+ (<-'ive')
+ 'biliti' 'bli'
+ (<-'ble')
+ 'ogi' ('l' <-'og')
+ 'fulli' (<-'ful')
+ 'lessli' (<-'less')
+ 'li' (valid_LI delete)
+ )
+ )
+
+ define Step_3 as (
+ [substring] R1 among (
+ 'tional' (<- 'tion')
+ 'ational' (<- 'ate')
+ 'alize' (<-'al')
+ 'icate' 'iciti' 'ical'
+ (<-'ic')
+ 'ful' 'ness'
+ (delete)
+ 'ative'
+ (R2 delete) // 'R2' added Dec 2001
+ )
+ )
+
+ define Step_4 as (
+ [substring] R2 among (
+ 'al' 'ance' 'ence' 'er' 'ic' 'able' 'ible' 'ant' 'ement'
+ 'ment' 'ent' 'ism' 'ate' 'iti' 'ous' 'ive' 'ize'
+ (delete)
+ 'ion' ('s' or 't' delete)
+ )
+ )
+
+ define Step_5 as (
+ [substring] among (
+ 'e' (R2 or (R1 not shortv) delete)
+ 'l' (R2 'l' delete)
+ )
+ )
+
+ define exception2 as (
+
+ [substring] atlimit among(
+ 'inning' 'outing' 'canning' 'herring' 'earring'
+ 'proceed' 'exceed' 'succeed'
+
+ // ... extensions possible here ...
+
+ )
+ )
+)
+
+define exception1 as (
+
+ [substring] atlimit among(
+
+ /* special changes: */
+
+ 'skis' (<-'ski')
+ 'skies' (<-'sky')
+ 'dying' (<-'die')
+ 'lying' (<-'lie')
+ 'tying' (<-'tie')
+
+ /* special -LY cases */
+
+ 'idly' (<-'idl')
+ 'gently' (<-'gentl')
+ 'ugly' (<-'ugli')
+ 'early' (<-'earli')
+ 'only' (<-'onli')
+ 'singly' (<-'singl')
+
+ // ... extensions possible here ...
+
+ /* invariant forms: */
+
+ 'sky'
+ 'news'
+ 'howe'
+
+ 'atlas' 'cosmos' 'bias' 'andes' // not plural forms
+
+ // ... extensions possible here ...
+ )
+)
+
+define postlude as (Y_found repeat(goto (['Y']) <-'y'))
+
+define stem as (
+
+ exception1 or
+ not hop 3 or (
+ do prelude
+ do mark_regions
+ backwards (
+
+ do Step_1a
+
+ exception2 or (
+
+ do Step_1b
+ do Step_1c
+
+ do Step_2
+ do Step_3
+ do Step_4
+
+ do Step_5
+ )
+ )
+ do postlude
+ )
+)
diff --git a/contrib/snowball/algorithms/finnish.sbl b/contrib/snowball/algorithms/finnish.sbl
new file mode 100644
index 0000000..3891d22
--- /dev/null
+++ b/contrib/snowball/algorithms/finnish.sbl
@@ -0,0 +1,197 @@
+
+/* Finnish stemmer.
+
+ Numbers in square brackets refer to the sections in
+ Fred Karlsson, Finnish: An Essential Grammar. Routledge, 1999
+ ISBN 0-415-20705-3
+
+*/
+
+routines (
+ mark_regions
+ R2
+ particle_etc possessive
+ LONG VI
+ case_ending
+ i_plural
+ t_plural
+ other_endings
+ tidy
+)
+
+externals ( stem )
+
+integers ( p1 p2 )
+strings ( x )
+booleans ( ending_removed )
+groupings ( AEI C V1 V2 particle_end )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a" '{U+00E4}'
+stringdef o" '{U+00F6}'
+
+define AEI 'a{a"}ei'
+define C 'bcdfghjklmnpqrstvwxz'
+define V1 'aeiouy{a"}{o"}'
+define V2 'aeiou{a"}{o"}'
+define particle_end V1 + 'nt'
+
+define mark_regions as (
+
+ $p1 = limit
+ $p2 = limit
+
+ goto V1 gopast non-V1 setmark p1
+ goto V1 gopast non-V1 setmark p2
+)
+
+backwardmode (
+
+ define R2 as $p2 <= cursor
+
+ define particle_etc as (
+ setlimit tomark p1 for ([substring])
+ among(
+ 'kin'
+ 'kaan' 'k{a"}{a"}n'
+ 'ko' 'k{o"}'
+ 'han' 'h{a"}n'
+ 'pa' 'p{a"}' // Particles [91]
+ (particle_end)
+ 'sti' // Adverb [87]
+ (R2)
+ )
+ delete
+ )
+ define possessive as ( // [36]
+ setlimit tomark p1 for ([substring])
+ among(
+ 'si'
+ (not 'k' delete) // take 'ksi' as the Comitative case
+ 'ni'
+ (delete ['kse'] <- 'ksi') // kseni = ksi + ni
+ 'nsa' 'ns{a"}'
+ 'mme'
+ 'nne'
+ (delete)
+ /* Now for Vn possessives after case endings: [36] */
+ 'an'
+ (among('ta' 'ssa' 'sta' 'lla' 'lta' 'na') delete)
+ '{a"}n'
+ (among('t{a"}' 'ss{a"}' 'st{a"}'
+ 'll{a"}' 'lt{a"}' 'n{a"}') delete)
+ 'en'
+ (among('lle' 'ine') delete)
+ )
+ )
+
+ define LONG as
+ among('aa' 'ee' 'ii' 'oo' 'uu' '{a"}{a"}' '{o"}{o"}')
+
+ define VI as ('i' V2)
+
+ define case_ending as (
+ setlimit tomark p1 for ([substring])
+ among(
+ 'han' ('a') //-.
+ 'hen' ('e') // |
+ 'hin' ('i') // |
+ 'hon' ('o') // |
+ 'h{a"}n' ('{a"}') // Illative [43]
+ 'h{o"}n' ('{o"}') // |
+ 'siin' VI // |
+ 'seen' LONG //-'
+
+ 'den' VI
+ 'tten' VI // Genitive plurals [34]
+ ()
+ 'n' // Genitive or Illative
+ ( try ( LONG // Illative
+ or 'ie' // Genitive
+ and next ]
+ )
+ /* otherwise Genitive */
+ )
+
+ 'a' '{a"}' //-.
+ (V1 C) // |
+ 'tta' 'tt{a"}' // Partitive [32]
+ ('e') // |
+ 'ta' 't{a"}' //-'
+
+ 'ssa' 'ss{a"}' // Inessive [41]
+ 'sta' 'st{a"}' // Elative [42]
+
+ 'lla' 'll{a"}' // Adessive [44]
+ 'lta' 'lt{a"}' // Ablative [51]
+ 'lle' // Allative [46]
+ 'na' 'n{a"}' // Essive [49]
+ 'ksi' // Translative[50]
+ 'ine' // Comitative [51]
+
+ /* Abessive and Instructive are too rare for
+ inclusion [51] */
+
+ )
+ delete
+ set ending_removed
+ )
+ define other_endings as (
+ setlimit tomark p2 for ([substring])
+ among(
+ 'mpi' 'mpa' 'mp{a"}'
+ 'mmi' 'mma' 'mm{a"}' // Comparative forms [85]
+ (not 'po') //-improves things
+ 'impi' 'impa' 'imp{a"}'
+ 'immi' 'imma' 'imm{a"}' // Superlative forms [86]
+ 'eja' 'ej{a"}' // indicates agent [93.1B]
+ )
+ delete
+ )
+ define i_plural as ( // [26]
+ setlimit tomark p1 for ([substring])
+ among(
+ 'i' 'j'
+ )
+ delete
+ )
+ define t_plural as ( // [26]
+ setlimit tomark p1 for (
+ ['t'] test V1
+ delete
+ )
+ setlimit tomark p2 for ([substring])
+ among(
+ 'mma' (not 'po') //-mmat endings
+ 'imma' //-immat endings
+ )
+ delete
+ )
+ define tidy as (
+ setlimit tomark p1 for (
+ do ( LONG and ([next] delete ) ) // undouble vowel
+ do ( [AEI] C delete ) // remove trailing a, a", e, i
+ do ( ['j'] 'o' or 'u' delete )
+ do ( ['o'] 'j' delete )
+ )
+ goto non-V1 [C] -> x x delete // undouble consonant
+ )
+)
+
+define stem as (
+
+ do mark_regions
+ unset ending_removed
+ backwards (
+ do particle_etc
+ do possessive
+ do case_ending
+ do other_endings
+ (ending_removed do i_plural) or do t_plural
+ do tidy
+ )
+)
+
diff --git a/contrib/snowball/algorithms/french.sbl b/contrib/snowball/algorithms/french.sbl
new file mode 100644
index 0000000..5c4f32d
--- /dev/null
+++ b/contrib/snowball/algorithms/french.sbl
@@ -0,0 +1,254 @@
+routines (
+ prelude postlude mark_regions
+ RV R1 R2
+ standard_suffix
+ i_verb_suffix
+ verb_suffix
+ residual_suffix
+ un_double
+ un_accent
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v keep_with_s )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a^ '{U+00E2}' // a-circumflex
+stringdef a` '{U+00E0}' // a-grave
+stringdef c, '{U+00E7}' // c-cedilla
+
+stringdef e" '{U+00EB}' // e-diaeresis (rare)
+stringdef e' '{U+00E9}' // e-acute
+stringdef e^ '{U+00EA}' // e-circumflex
+stringdef e` '{U+00E8}' // e-grave
+stringdef i" '{U+00EF}' // i-diaeresis
+stringdef i^ '{U+00EE}' // i-circumflex
+stringdef o^ '{U+00F4}' // o-circumflex
+stringdef u^ '{U+00FB}' // u-circumflex
+stringdef u` '{U+00F9}' // u-grave
+
+define v 'aeiouy{a^}{a`}{e"}{e'}{e^}{e`}{i"}{i^}{o^}{u^}{u`}'
+
+define prelude as repeat goto (
+
+ ( v [ ('u' ] v <- 'U') or
+ ('i' ] v <- 'I') or
+ ('y' ] <- 'Y')
+ )
+ or
+ ( [ '{e"}' ] <- 'He' )
+ or
+ ( [ '{i"}' ] <- 'Hi' )
+ or
+ ( ['y'] v <- 'Y' )
+ or
+ ( 'q' ['u'] <- 'U' )
+)
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v v next )
+ or
+ among ( // this exception list begun Nov 2006
+ 'par' // paris, parie, pari
+ 'col' // colis
+ 'tap' // tapis
+ // extensions possible here
+ )
+ or
+ ( next gopast v )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'I' (<- 'i')
+ 'U' (<- 'u')
+ 'Y' (<- 'y')
+ 'He' (<- '{e"}')
+ 'Hi' (<- '{i"}')
+ 'H' (delete)
+ '' (next)
+ )
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define standard_suffix as (
+ [substring] among(
+
+ 'ance' 'iqUe' 'isme' 'able' 'iste' 'eux'
+ 'ances' 'iqUes' 'ismes' 'ables' 'istes'
+ ( R2 delete )
+ 'atrice' 'ateur' 'ation'
+ 'atrices' 'ateurs' 'ations'
+ ( R2 delete
+ try ( ['ic'] (R2 delete) or <-'iqU' )
+ )
+ 'logie'
+ 'logies'
+ ( R2 <- 'log' )
+ 'usion' 'ution'
+ 'usions' 'utions'
+ ( R2 <- 'u' )
+ 'ence'
+ 'ences'
+ ( R2 <- 'ent' )
+ 'ement'
+ 'ements'
+ (
+ RV delete
+ try (
+ [substring] among(
+ 'iv' (R2 delete ['at'] R2 delete)
+ 'eus' ((R2 delete) or (R1<-'eux'))
+ 'abl' 'iqU'
+ (R2 delete)
+ 'i{e`}r' 'I{e`}r' //)
+ (RV <-'i') //)--new 2 Sept 02
+ )
+ )
+ )
+ 'it{e'}'
+ 'it{e'}s'
+ (
+ R2 delete
+ try (
+ [substring] among(
+ 'abil' ((R2 delete) or <-'abl')
+ 'ic' ((R2 delete) or <-'iqU')
+ 'iv' (R2 delete)
+ )
+ )
+ )
+ 'if' 'ive'
+ 'ifs' 'ives'
+ (
+ R2 delete
+ try ( ['at'] R2 delete ['ic'] (R2 delete) or <-'iqU' )
+ )
+ 'eaux' (<- 'eau')
+ 'aux' (R1 <- 'al')
+ 'euse'
+ 'euses'((R2 delete) or (R1<-'eux'))
+
+ 'issement'
+ 'issements'(R1 non-v delete) // verbal
+
+ // fail(...) below forces entry to verb_suffix. -ment typically
+ // follows the p.p., e.g 'confus{e'}ment'.
+
+ 'amment' (RV fail(<- 'ant'))
+ 'emment' (RV fail(<- 'ent'))
+ 'ment'
+ 'ments' (test(v RV) fail(delete))
+ // v is e,i,u,{e'},I or U
+ )
+ )
+
+ define i_verb_suffix as setlimit tomark pV for (
+ [substring] among (
+ '{i^}mes' '{i^}t' '{i^}tes' 'i' 'ie' 'ies' 'ir' 'ira' 'irai'
+ 'iraIent' 'irais' 'irait' 'iras' 'irent' 'irez' 'iriez'
+ 'irions' 'irons' 'iront' 'is' 'issaIent' 'issais' 'issait'
+ 'issant' 'issante' 'issantes' 'issants' 'isse' 'issent' 'isses'
+ 'issez' 'issiez' 'issions' 'issons' 'it'
+ (not 'H' non-v delete)
+ )
+ )
+
+ define verb_suffix as setlimit tomark pV for (
+ [substring] among (
+ 'ions'
+ (R2 delete)
+
+ '{e'}' '{e'}e' '{e'}es' '{e'}s' '{e`}rent' 'er' 'era' 'erai'
+ 'eraIent' 'erais' 'erait' 'eras' 'erez' 'eriez' 'erions'
+ 'erons' 'eront' 'ez' 'iez'
+
+ // 'ons' //-best omitted
+
+ (delete)
+
+ '{a^}mes' '{a^}t' '{a^}tes' 'a' 'ai' 'aIent' 'ais' 'ait' 'ant'
+ 'ante' 'antes' 'ants' 'as' 'asse' 'assent' 'asses' 'assiez'
+ 'assions'
+ (delete
+ try(['e'] delete)
+ )
+ )
+ )
+
+ define keep_with_s 'aiou{e`}s'
+
+ define residual_suffix as (
+ try(['s'] test ('Hi' or non-keep_with_s) delete)
+ setlimit tomark pV for (
+ [substring] among(
+ 'ion' (R2 's' or 't' delete)
+ 'ier' 'i{e`}re'
+ 'Ier' 'I{e`}re' (<-'i')
+ 'e' (delete)
+ )
+ )
+ )
+
+ define un_double as (
+ test among('enn' 'onn' 'ett' 'ell' 'eill') [next] delete
+ )
+
+ define un_accent as (
+ atleast 1 non-v
+ [ '{e'}' or '{e`}' ] <-'e'
+ )
+)
+
+define stem as (
+
+ do prelude
+ do mark_regions
+ backwards (
+
+ do (
+ (
+ ( standard_suffix or
+ i_verb_suffix or
+ verb_suffix
+ )
+ and
+ try( [ ('Y' ] <- 'i' ) or
+ ('{c,}'] <- 'c' )
+ )
+ ) or
+ residual_suffix
+ )
+
+ // try(['ent'] RV delete) // is best omitted
+
+ do un_double
+ do un_accent
+ )
+ do postlude
+)
+
diff --git a/contrib/snowball/algorithms/german.sbl b/contrib/snowball/algorithms/german.sbl
new file mode 100644
index 0000000..61f24ef
--- /dev/null
+++ b/contrib/snowball/algorithms/german.sbl
@@ -0,0 +1,139 @@
+
+/*
+ Extra rule for -nisse ending added 11 Dec 2009
+*/
+
+routines (
+ prelude postlude
+ mark_regions
+ R1 R2
+ standard_suffix
+)
+
+externals ( stem )
+
+integers ( p1 p2 x )
+
+groupings ( v s_ending st_ending )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a" '{U+00E4}'
+stringdef o" '{U+00F6}'
+stringdef u" '{U+00FC}'
+stringdef ss '{U+00DF}'
+
+define v 'aeiouy{a"}{o"}{u"}'
+
+define s_ending 'bdfghklmnrt'
+define st_ending s_ending - 'r'
+
+define prelude as (
+
+ test repeat (
+ (
+ ['{ss}'] <- 'ss'
+ ) or next
+ )
+
+ repeat goto (
+ v [('u'] v <- 'U') or
+ ('y'] v <- 'Y')
+ )
+)
+
+define mark_regions as (
+
+ $p1 = limit
+ $p2 = limit
+
+ test(hop 3 setmark x)
+
+ gopast v gopast non-v setmark p1
+ try($p1 < x $p1 = x) // at least 3
+ gopast v gopast non-v setmark p2
+
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'Y' (<- 'y')
+ 'U' (<- 'u')
+ '{a"}' (<- 'a')
+ '{o"}' (<- 'o')
+ '{u"}' (<- 'u')
+ '' (next)
+ )
+
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define standard_suffix as (
+ do (
+ [substring] R1 among(
+ 'em' 'ern' 'er'
+ ( delete
+ )
+ 'e' 'en' 'es'
+ ( delete
+ try (['s'] 'nis' delete)
+ )
+ 's'
+ ( s_ending delete
+ )
+ )
+ )
+ do (
+ [substring] R1 among(
+ 'en' 'er' 'est'
+ ( delete
+ )
+ 'st'
+ ( st_ending hop 3 delete
+ )
+ )
+ )
+ do (
+ [substring] R2 among(
+ 'end' 'ung'
+ ( delete
+ try (['ig'] not 'e' R2 delete)
+ )
+ 'ig' 'ik' 'isch'
+ ( not 'e' delete
+ )
+ 'lich' 'heit'
+ ( delete
+ try (
+ ['er' or 'en'] R1 delete
+ )
+ )
+ 'keit'
+ ( delete
+ try (
+ [substring] R2 among(
+ 'lich' 'ig'
+ ( delete
+ )
+ )
+ )
+ )
+ )
+ )
+ )
+)
+
+define stem as (
+ do prelude
+ do mark_regions
+ backwards
+ do standard_suffix
+ do postlude
+)
diff --git a/contrib/snowball/algorithms/german2.sbl b/contrib/snowball/algorithms/german2.sbl
new file mode 100644
index 0000000..47ff61e
--- /dev/null
+++ b/contrib/snowball/algorithms/german2.sbl
@@ -0,0 +1,145 @@
+
+/*
+ Extra rule for -nisse ending added 11 Dec 2009
+*/
+
+routines (
+ prelude postlude
+ mark_regions
+ R1 R2
+ standard_suffix
+)
+
+externals ( stem )
+
+integers ( p1 p2 x )
+
+groupings ( v s_ending st_ending )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a" '{U+00E4}'
+stringdef o" '{U+00F6}'
+stringdef u" '{U+00FC}'
+stringdef ss '{U+00DF}'
+
+define v 'aeiouy{a"}{o"}{u"}'
+
+define s_ending 'bdfghklmnrt'
+define st_ending s_ending - 'r'
+
+define prelude as (
+
+ test repeat goto (
+ v [('u'] v <- 'U') or
+ ('y'] v <- 'Y')
+ )
+
+ repeat (
+ [substring] among(
+ '{ss}' (<- 'ss')
+ 'ae' (<- '{a"}')
+ 'oe' (<- '{o"}')
+ 'ue' (<- '{u"}')
+ 'qu' (hop 2)
+ '' (next)
+ )
+ )
+
+)
+
+define mark_regions as (
+
+ $p1 = limit
+ $p2 = limit
+
+ test(hop 3 setmark x)
+
+ gopast v gopast non-v setmark p1
+ try($p1 < x $p1 = x) // at least 3
+ gopast v gopast non-v setmark p2
+
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'Y' (<- 'y')
+ 'U' (<- 'u')
+ '{a"}' (<- 'a')
+ '{o"}' (<- 'o')
+ '{u"}' (<- 'u')
+ '' (next)
+ )
+
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define standard_suffix as (
+ do (
+ [substring] R1 among(
+ 'em' 'ern' 'er'
+ ( delete
+ )
+ 'e' 'en' 'es'
+ ( delete
+ try (['s'] 'nis' delete)
+ )
+ 's'
+ ( s_ending delete
+ )
+ )
+ )
+ do (
+ [substring] R1 among(
+ 'en' 'er' 'est'
+ ( delete
+ )
+ 'st'
+ ( st_ending hop 3 delete
+ )
+ )
+ )
+ do (
+ [substring] R2 among(
+ 'end' 'ung'
+ ( delete
+ try (['ig'] not 'e' R2 delete)
+ )
+ 'ig' 'ik' 'isch'
+ ( not 'e' delete
+ )
+ 'lich' 'heit'
+ ( delete
+ try (
+ ['er' or 'en'] R1 delete
+ )
+ )
+ 'keit'
+ ( delete
+ try (
+ [substring] R2 among(
+ 'lich' 'ig'
+ ( delete
+ )
+ )
+ )
+ )
+ )
+ )
+ )
+)
+
+define stem as (
+ do prelude
+ do mark_regions
+ backwards
+ do standard_suffix
+ do postlude
+)
diff --git a/contrib/snowball/algorithms/greek.sbl b/contrib/snowball/algorithms/greek.sbl
new file mode 100644
index 0000000..02df6c3
--- /dev/null
+++ b/contrib/snowball/algorithms/greek.sbl
@@ -0,0 +1,706 @@
+// A stemmer for Modern Greek language, based on:
+//
+// Ntais, Georgios. Development of a Stemmer for the Greek
+// Language. Diss. Royal Institute of Technology, 2006.
+// https://sais.se/mthprize/2007/ntais2007.pdf
+//
+// Saroukos, Spyridon. Enhancing a Greek language stemmer.
+// University of Tampere, 2008.
+// https://tampub.uta.fi/bitstream/handle/10024/80480/gradu03463.pdf
+
+stringescapes {}
+
+stringdef a '{U+03B1}' // alpha
+stringdef v '{U+03B2}' // beta
+stringdef g '{U+03B3}' // gamma
+stringdef d '{U+03B4}' // delta
+stringdef e '{U+03B5}' // epsilon
+stringdef z '{U+03B6}' // zeta
+stringdef i '{U+03B7}' // eta
+stringdef th '{U+03B8}' // theta
+stringdef y '{U+03B9}' // iota
+stringdef k '{U+03BA}' // kappa
+stringdef l '{U+03BB}' // lamda
+stringdef m '{U+03BC}' // mu
+stringdef n '{U+03BD}' // nu
+stringdef x '{U+03BE}' // xi
+stringdef o '{U+03BF}' // omicron
+stringdef p '{U+03C0}' // pi
+stringdef r '{U+03C1}' // rho
+stringdef ss '{U+03C2}' // sigma final
+stringdef s '{U+03C3}' // sigma
+stringdef t '{U+03C4}' // tau
+stringdef u '{U+03C5}' // upsilon
+stringdef f '{U+03C6}' // phi
+stringdef ch '{U+03C7}' // chi
+stringdef ps '{U+03C8}' // psi
+stringdef oo '{U+03C9}' // omega
+
+stringdef A '{U+0391}' // Alpha
+stringdef V '{U+0392}' // Beta
+stringdef G '{U+0393}' // Gamma
+stringdef D '{U+0394}' // Delta
+stringdef E '{U+0395}' // Epsilon
+stringdef Z '{U+0396}' // Zeta
+stringdef I '{U+0397}' // Eta
+stringdef Th '{U+0398}' // Theta
+stringdef Y '{U+0399}' // Iota
+stringdef K '{U+039A}' // Kappa
+stringdef L '{U+039B}' // Lamda
+stringdef M '{U+039C}' // Mu
+stringdef N '{U+039D}' // Nu
+stringdef X '{U+039E}' // Xi
+stringdef O '{U+039F}' // Omicron
+stringdef P '{U+03A0}' // Pi
+stringdef R '{U+03A1}' // Rho
+stringdef S '{U+03A3}' // Sigma
+stringdef T '{U+03A4}' // Tau
+stringdef U '{U+03A5}' // Upsilon
+stringdef F '{U+03A6}' // Phi
+stringdef Ch '{U+03A7}' // Chi
+stringdef Ps '{U+03A8}' // Psi
+stringdef Oo '{U+03A9}' // Omega
+
+stringdef Y: '{U+03AA}' // Iota with dialytika
+stringdef U: '{U+03AB}' // Upsilon with dialytika
+
+stringdef a' '{U+03AC}' // alpha with tonos
+stringdef e' '{U+03AD}' // epsilon with tonos
+stringdef i' '{U+03AE}' // eta with tonos
+stringdef y' '{U+03AF}' // iota with tonos
+stringdef o' '{U+03CC}' // omicron with tonos
+stringdef u' '{U+03CD}' // upsilon with tonos
+stringdef oo' '{U+03CE}' // omega with tonos
+
+stringdef i:' '{U+0390}' // iota with dialytika and tonos
+stringdef u:' '{U+03B0}' // upsilon with dialytika and tonos
+
+stringdef i: '{U+03CA}' // iota with dialytika
+stringdef u: '{U+03CB}' // upsilon with dialytika
+
+stringdef A' '{U+0386}' // Alpha with tonos
+stringdef E' '{U+0388}' // Epsilon with tonos
+stringdef I' '{U+0389}' // Eta with tonos
+stringdef Y' '{U+038A}' // Iota with tonos
+stringdef O' '{U+038C}' // Omicron with tonos
+stringdef U' '{U+038E}' // Upsilon with tonos
+stringdef OO' '{U+038F}' // Omega with tonos
+
+externals ( stem )
+
+booleans ( test1 )
+
+groupings ( v v2 )
+
+routines ( tolower has_min_length
+ steps1 steps2 steps3 steps4 steps5 steps6 steps7
+ steps8 steps9 steps10
+ step1 step2a step2b step2c step2d step3 step4
+ step5a step5b step5c step5d step5e step5f
+ step5g step5h step5i
+ step5j step5k step5l step5m
+ step6 step7 )
+
+define v '{a}{e}{i}{y}{o}{u}{oo}'
+define v2 '{a}{e}{i}{y}{o}{oo}'
+
+backwardmode (
+ define has_min_length as (
+ $(len >= 3)
+ )
+
+ define tolower as (
+ repeat (
+ [substring] among (
+ '{A}' (<- '{a}')
+ '{V}' (<- '{v}')
+ '{G}' (<- '{g}')
+ '{D}' (<- '{d}')
+ '{E}' (<- '{e}')
+ '{Z}' (<- '{z}')
+ '{I}' (<- '{i}')
+ '{Th}' (<- '{th}')
+ '{Y}' (<- '{y}')
+ '{K}' (<- '{k}')
+ '{L}' (<- '{l}')
+ '{M}' (<- '{m}')
+ '{N}' (<- '{n}')
+ '{X}' (<- '{x}')
+ '{O}' (<- '{o}')
+ '{P}' (<- '{p}')
+ '{R}' (<- '{r}')
+ '{S}' (<- '{s}')
+ '{T}' (<- '{t}')
+ '{U}' (<- '{u}')
+ '{F}' (<- '{f}')
+ '{Ch}' (<- '{ch}')
+ '{Ps}' (<- '{ps}')
+ '{Oo}' (<- '{oo}')
+ '{Y:}' (<- '{y}')
+ '{U:}' (<- '{u}')
+ '{a'}' (<- '{a}')
+ '{e'}' (<- '{e}')
+ '{i'}' (<- '{i}')
+ '{y'}' (<- '{y}')
+ '{o'}' (<- '{o}')
+ '{u'}' (<- '{u}')
+ '{oo'}' (<- '{oo}')
+ '{i:'}' (<- '{i}')
+ '{u:'}' (<- '{u}')
+ '{i:}' (<- '{i}')
+ '{u:}' (<- '{u}')
+ '{A'}' (<- '{a}')
+ '{E'}' (<- '{e}')
+ '{I'}' (<- '{i}')
+ '{Y'}' (<- '{y}')
+ '{O'}' (<- '{o}')
+ '{U'}' (<- '{u}')
+ '{OO'}' (<- '{oo}')
+ '{ss}' (<- '{s}')
+ '' (next)
+ )
+ )
+ )
+
+ define step1 as (
+ [substring] among (
+ '{f}{a}{g}{y}{a}' '{f}{a}{g}{y}{o}{u}' '{f}{a}{g}{y}{oo}{n}' (<- '{f}{a}')
+ '{s}{k}{a}{g}{y}{a}' '{s}{k}{a}{g}{y}{o}{u}' '{s}{k}{a}{g}{y}{oo}{n}' (<- '{s}{k}{a}')
+ '{o}{l}{o}{g}{y}{o}{u}' '{o}{l}{o}{g}{y}{a}' '{o}{l}{o}{g}{y}{oo}{n}' (<- '{o}{l}{o}')
+ '{s}{o}{g}{y}{o}{u}' '{s}{o}{g}{y}{a}' '{s}{o}{g}{y}{oo}{n}' (<- '{s}{o}')
+ '{t}{a}{t}{o}{g}{y}{a}' '{t}{a}{t}{o}{g}{y}{o}{u}' '{t}{a}{t}{o}{g}{y}{oo}{n}' (<- '{t}{a}{t}{o}')
+ '{k}{r}{e}{a}{s}' '{k}{r}{e}{a}{t}{o}{s}' '{k}{r}{e}{a}{t}{a}' '{k}{r}{e}{a}{t}{oo}{n}' (<- '{k}{r}{e}')
+ '{p}{e}{r}{a}{s}' '{p}{e}{r}{a}{t}{o}{s}' '{p}{e}{r}{a}{t}{i}' '{p}{e}{r}{a}{t}{a}' '{p}{e}{r}{a}{t}{oo}{n}' (<- '{p}{e}{r}')
+ '{t}{e}{r}{a}{s}' '{t}{e}{r}{a}{t}{o}{s}' '{t}{e}{r}{a}{t}{a}' '{t}{e}{r}{a}{t}{oo}{n}' (<- '{t}{e}{r}')
+ '{f}{oo}{s}' '{f}{oo}{t}{o}{s}' '{f}{oo}{t}{a}' '{f}{oo}{t}{oo}{n}' (<- '{f}{oo}')
+ '{k}{a}{th}{e}{s}{t}{oo}{s}' '{k}{a}{th}{e}{s}{t}{oo}{t}{o}{s}' '{k}{a}{th}{e}{s}{t}{oo}{t}{a}' '{k}{a}{th}{e}{s}{t}{oo}{t}{oo}{n}' (<- '{k}{a}{th}{e}{s}{t}')
+ '{g}{e}{g}{o}{n}{o}{s}' '{g}{e}{g}{o}{n}{o}{t}{o}{s}' '{g}{e}{g}{o}{n}{o}{t}{a}' '{g}{e}{g}{o}{n}{o}{t}{oo}{n}' (<- '{g}{e}{g}{o}{n}')
+ )
+ unset test1
+ )
+
+ define steps1 as (
+ [substring] among (
+ '{y}{z}{a}' '{y}{z}{e}{s}' '{y}{z}{e}' '{y}{z}{a}{m}{e}' '{y}{z}{a}{t}{e}' '{y}{z}{a}{n}' '{y}{z}{a}{n}{e}' '{y}{z}{oo}' '{y}{z}{e}{y}{s}' '{y}{z}{e}{y}'
+ '{y}{z}{o}{u}{m}{e}' '{y}{z}{e}{t}{e}' '{y}{z}{o}{u}{n}' '{y}{z}{o}{u}{n}{e}' (
+ delete
+ unset test1
+ ([] substring atlimit among (
+ '{a}{n}{a}{m}{p}{a}' '{e}{m}{p}{a}' '{e}{p}{a}' '{x}{a}{n}{a}{p}{a}' '{p}{a}' '{p}{e}{r}{y}{p}{a}' '{a}{th}{r}{o}' '{s}{u}{n}{a}{th}{r}{o}' '{d}{a}{n}{e}'
+ (<- '{y}')
+ )) or
+ ([] substring atlimit among (
+ '{m}{a}{r}{k}' '{k}{o}{r}{n}' '{a}{m}{p}{a}{r}' '{a}{r}{r}' '{v}{a}{th}{u}{r}{y}' '{v}{a}{r}{k}' '{v}' '{v}{o}{l}{v}{o}{r}' '{g}{k}{r}'
+ '{g}{l}{u}{k}{o}{r}' '{g}{l}{u}{k}{u}{r}' '{y}{m}{p}' '{l}' '{l}{o}{u}' '{m}{a}{r}' '{m}' '{p}{r}' '{m}{p}{r}' '{p}{o}{l}{u}{r}' '{p}'
+ '{r}' '{p}{y}{p}{e}{r}{o}{r}'
+ (<- '{y}{z}')
+ ))
+ )
+ )
+ )
+
+ define steps2 as (
+ [substring] among (
+ '{oo}{th}{i}{k}{a}' '{oo}{th}{i}{k}{e}{s}' '{oo}{th}{i}{k}{e}' '{oo}{th}{i}{k}{a}{m}{e}' '{oo}{th}{i}{k}{a}{t}{e}' '{oo}{th}{i}{k}{a}{n}' '{oo}{th}{i}{k}{a}{n}{e}' (
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{a}{l}' '{v}{y}' '{e}{n}' '{u}{ps}' '{l}{y}' '{z}{oo}' '{s}' '{ch}' (<- '{oo}{n}')
+ )
+ )
+ )
+ )
+
+ define steps3 as (
+ [substring] among (
+ '{y}{s}{a}' '{y}{s}{e}{s}' '{y}{s}{e}' '{y}{s}{a}{m}{e}' '{y}{s}{a}{t}{e}' '{y}{s}{a}{n}' '{y}{s}{a}{n}{e}' (
+ delete
+ unset test1
+ ('{y}{s}{a}' atlimit <- '{y}{s}') or
+ ([] substring atlimit among (
+ '{a}{n}{a}{m}{p}{a}' '{a}{th}{r}{o}' '{e}{m}{p}{a}' '{e}{s}{e}' '{e}{s}{oo}{k}{l}{e}' '{e}{p}{a}' '{x}{a}{n}{a}{p}{a}' '{e}{p}{e}' '{p}{e}{r}{y}{p}{a}'
+ '{s}{u}{n}{a}{th}{r}{o}' '{d}{a}{n}{e}' '{k}{l}{e}' '{ch}{a}{r}{t}{o}{p}{a}' '{e}{x}{a}{r}{ch}{a}' '{m}{e}{t}{e}{p}{e}' '{a}{p}{o}{k}{l}{e}'
+ '{a}{p}{e}{k}{l}{e}' '{e}{k}{l}{e}' '{p}{e}'
+ (<- '{y}')
+ )) or
+ ([] substring atlimit among (
+ '{a}{n}' '{a}{f}' '{g}{e}' '{g}{y}{g}{a}{n}{t}{o}{a}{f}' '{g}{k}{e}' '{d}{i}{m}{o}{k}{r}{a}{t}' '{k}{o}{m}' '{g}{k}' '{m}' '{p}'
+ '{p}{o}{u}{k}{a}{m}' '{o}{l}{o}' '{l}{a}{r}'
+ (<- '{y}{s}')
+ ))
+ )
+ )
+ )
+
+ define steps4 as (
+ [substring] among (
+ '{y}{s}{oo}' '{y}{s}{e}{y}{s}' '{y}{s}{e}{y}' '{y}{s}{o}{u}{m}{e}' '{y}{s}{e}{t}{e}' '{y}{s}{o}{u}{n}' '{y}{s}{o}{u}{n}{e}' (
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{a}{n}{a}{m}{p}{a}' '{e}{m}{p}{a}' '{e}{s}{e}' '{e}{s}{oo}{k}{l}{e}' '{e}{p}{a}' '{x}{a}{n}{a}{p}{a}' '{e}{p}{e}' '{p}{e}{r}{y}{p}{a}' '{a}{th}{r}{o}'
+ '{s}{u}{n}{a}{th}{r}{o}' '{d}{a}{n}{e}' '{k}{l}{e}' '{ch}{a}{r}{t}{o}{p}{a}' '{e}{x}{a}{r}{ch}{a}' '{m}{e}{t}{e}{p}{e}' '{a}{p}{o}{k}{l}{e}' '{a}{p}{e}{k}{l}{e}'
+ '{e}{k}{l}{e}' '{p}{e}'
+ (<- '{y}')
+ )
+ )
+ )
+ )
+
+ define steps5 as (
+ [substring] among (
+ '{y}{s}{t}{o}{s}' '{y}{s}{t}{o}{u}' '{y}{s}{t}{o}' '{y}{s}{t}{e}' '{y}{s}{t}{o}{y}' '{y}{s}{t}{oo}{n}' '{y}{s}{t}{o}{u}{s}' '{y}{s}{t}{i}' '{y}{s}{t}{i}{s}'
+ '{y}{s}{t}{a}' '{y}{s}{t}{e}{s}' (
+ delete
+ unset test1
+ ([] substring atlimit among (
+ '{d}{a}{n}{e}' '{s}{u}{n}{a}{th}{r}{o}' '{k}{l}{e}' '{s}{e}' '{e}{s}{oo}{k}{l}{e}' '{a}{s}{e}' '{p}{l}{e}'
+ (<- '{y}')
+ )) or
+ ([] substring atlimit among (
+ '{m}' '{p}' '{a}{p}' '{a}{r}' '{i}{d}' '{k}{t}' '{s}{k}' '{s}{ch}' '{u}{ps}' '{f}{a}' '{ch}{r}' '{ch}{t}' '{a}{k}{t}'
+ '{a}{o}{r}' '{a}{s}{ch}' '{a}{t}{a}' '{a}{ch}{n}' '{a}{ch}{t}' '{g}{e}{m}' '{g}{u}{r}' '{e}{m}{p}' '{e}{u}{p}' '{e}{ch}{th}' '{i}{f}{a}'
+ '{k}{a}{th}' '{k}{a}{k}' '{k}{u}{l}' '{l}{u}{g}' '{m}{a}{k}' '{m}{e}{g}' '{t}{a}{ch}' '{f}{y}{l}' '{ch}{oo}{r}'
+ (<- '{y}{s}{t}')
+ ))
+ )
+ )
+ )
+
+ define steps6 as (
+ [substring] among (
+ '{y}{s}{m}{o}' '{y}{s}{m}{o}{y}' '{y}{s}{m}{o}{s}' '{y}{s}{m}{o}{u}' '{y}{s}{m}{o}{u}{s}' '{y}{s}{m}{oo}{n}' (
+ delete
+ unset test1
+ ([] substring atlimit among (
+ '{s}{e}' '{m}{e}{t}{a}{s}{e}' '{m}{y}{k}{r}{o}{s}{e}' '{e}{g}{k}{l}{e}' '{a}{p}{o}{k}{l}{e}'
+ (<- '{y}{s}{m}')
+ )) or
+ ([] substring atlimit among (
+ '{d}{a}{n}{e}' '{a}{n}{t}{y}{d}{a}{n}{e}'
+ (<- '{y}')
+ )) or
+ ([substring] among (
+ '{a}{g}{n}{oo}{s}{t}{y}{k}' (<- '{a}{g}{n}{oo}{s}{t}')
+ '{a}{t}{o}{m}{y}{k}' (<- '{a}{t}{o}{m}')
+ '{g}{n}{oo}{s}{t}{y}{k}' (<- '{g}{n}{oo}{s}{t}')
+ '{e}{th}{n}{y}{k}' (<- '{e}{th}{n}')
+ '{e}{k}{l}{e}{k}{t}{y}{k}' (<- '{e}{k}{l}{e}{k}{t}')
+ '{s}{k}{e}{p}{t}{y}{k}' (<- '{s}{k}{e}{p}{t}')
+ '{t}{o}{p}{y}{k}' (<- '{t}{o}{p}')
+ '{a}{l}{e}{x}{a}{n}{d}{r}{y}{n}' (<- '{a}{l}{e}{x}{a}{n}{d}{r}')
+ '{v}{u}{z}{a}{n}{t}{y}{n}' (<- '{v}{u}{z}{a}{n}{t}')
+ '{th}{e}{a}{t}{r}{y}{n}' (<- '{th}{e}{a}{t}{r}')
+ ))
+ )
+ )
+ )
+
+ define steps7 as (
+ [substring] among (
+ '{a}{r}{a}{k}{y}' '{a}{r}{a}{k}{y}{a}' '{o}{u}{d}{a}{k}{y}' '{o}{u}{d}{a}{k}{y}{a}' (
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{s}' '{ch}'
+ (<- '{a}{r}{a}{k}')
+ )
+ )
+ )
+ )
+
+ define steps8 as (
+ [substring] among (
+ '{a}{k}{y}' '{a}{k}{y}{a}' '{y}{t}{s}{a}' '{y}{t}{s}{a}{s}' '{y}{t}{s}{e}{s}' '{y}{t}{s}{oo}{n}' '{a}{r}{a}{k}{y}' '{a}{r}{a}{k}{y}{a}' (
+ delete
+ unset test1
+ ([] substring atlimit among (
+ '{v}{a}{m}{v}' '{v}{r}' '{k}{a}{y}{m}' '{k}{o}{n}' '{k}{o}{r}' '{l}{a}{v}{r}' '{l}{o}{u}{l}' '{m}{e}{r}' '{m}{o}{u}{s}{t}'
+ '{n}{a}{g}{k}{a}{s}' '{p}{l}' '{r}' '{r}{u}' '{s}' '{s}{k}' '{s}{o}{k}' '{s}{p}{a}{n}' '{t}{z}' '{f}{a}{r}{m}' '{ch}' '{k}{a}{p}{a}{k}'
+ '{a}{l}{y}{s}{f}' '{a}{m}{v}{r}' '{a}{n}{th}{r}' '{k}' '{f}{u}{l}' '{k}{a}{t}{r}{a}{p}' '{k}{l}{y}{m}' '{m}{a}{l}' '{s}{l}{o}{v}' '{f}'
+ '{s}{f}' '{t}{s}{e}{ch}{o}{s}{l}{o}{v}'
+ (<- '{a}{k}')
+ )) or
+ ([] substring atlimit among (
+ '{v}' '{v}{a}{l}' '{g}{y}{a}{n}' '{g}{l}' '{z}' '{i}{g}{o}{u}{m}{e}{n}' '{k}{a}{r}{d}' '{k}{o}{n}' '{m}{a}{k}{r}{u}{n}' '{n}{u}{f}'
+ '{p}{a}{t}{e}{r}' '{p}' '{s}{k}' '{t}{o}{s}' '{t}{r}{y}{p}{o}{l}'
+ (<- '{y}{t}{s}')
+ )) or
+ ([] '{k}{o}{r}' <- '{y}{t}{s}')
+ )
+ )
+ )
+
+ define steps9 as (
+ [substring] among (
+ '{y}{d}{y}{o}' '{y}{d}{y}{a}' '{y}{d}{y}{oo}{n}' (
+ delete
+ unset test1
+ ([] substring atlimit among (
+ '{a}{y}{f}{n}' '{y}{r}' '{o}{l}{o}' '{ps}{a}{l}' (<- '{y}{d}')
+ )) or
+ ([] substring among (
+ '{e}' '{p}{a}{y}{ch}{n}' (<- '{y}{d}')
+ ))
+ )
+ )
+ )
+
+ define steps10 as (
+ [substring] among (
+ '{y}{s}{k}{o}{s}' '{y}{s}{k}{o}{u}' '{y}{s}{k}{o}' '{y}{s}{k}{e}' (
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{d}' '{y}{v}' '{m}{i}{n}' '{r}' '{f}{r}{a}{g}{k}' '{l}{u}{k}' '{o}{v}{e}{l}'
+ (<- '{y}{s}{k}')
+ )
+ )
+ )
+ )
+
+ define step2a as (
+ [substring] among (
+ '{a}{d}{e}{s}' '{a}{d}{oo}{n}' (delete)
+ )
+ not ([substring] among (
+ '{o}{k}' '{m}{a}{m}' '{m}{a}{n}' '{m}{p}{a}{m}{p}' '{p}{a}{t}{e}{r}' '{g}{y}{a}{g}{y}' '{n}{t}{a}{n}{t}' '{k}{u}{r}' '{th}{e}{y}' '{p}{e}{th}{e}{r}'
+ ))
+ insert '{a}{d}'
+ )
+
+ define step2b as (
+ [substring] among (
+ '{e}{d}{e}{s}' '{e}{d}{oo}{n}' (delete)
+ )
+ [] substring among (
+ '{o}{p}' '{y}{p}' '{e}{m}{p}' '{u}{p}' '{g}{i}{p}' '{d}{a}{p}' '{k}{r}{a}{s}{p}' '{m}{y}{l}' (<- '{e}{d}')
+ )
+ )
+
+ define step2c as (
+ [substring] among (
+ '{o}{u}{d}{e}{s}' '{o}{u}{d}{oo}{n}' (delete)
+ )
+ [] substring among (
+ '{a}{r}{k}' '{k}{a}{l}{y}{a}{k}' '{p}{e}{t}{a}{l}' '{l}{y}{ch}' '{p}{l}{e}{x}' '{s}{k}' '{s}' '{f}{l}' '{f}{r}' '{v}{e}{l}' '{l}{o}{u}{l}' '{ch}{n}'
+ '{s}{p}' '{t}{r}{a}{g}' '{f}{e}' (<- '{o}{u}{d}')
+ )
+ )
+
+ define step2d as (
+ [substring] among (
+ '{e}{oo}{s}' '{e}{oo}{n}' (delete unset test1)
+ )
+ [] substring atlimit among (
+ '{th}' '{d}' '{e}{l}' '{g}{a}{l}' '{n}' '{p}' '{y}{d}' '{p}{a}{r}' (<- '{e}')
+ )
+ )
+
+ define step3 as (
+ [substring] among (
+ '{y}{a}' '{y}{o}{u}' '{y}{oo}{n}' (delete unset test1)
+ )
+ ([] v <- '{y}')
+ )
+
+ define step4 as (
+ [substring] among (
+ '{y}{k}{a}' '{y}{k}{o}' '{y}{k}{o}{u}' '{y}{k}{oo}{n}' (delete unset test1)
+ )
+ ([] v <- '{y}{k}') or
+ [] substring atlimit among (
+ '{a}{l}' '{a}{d}' '{e}{n}{d}' '{a}{m}{a}{n}' '{a}{m}{m}{o}{ch}{a}{l}' '{i}{th}' '{a}{n}{i}{th}' '{a}{n}{t}{y}{d}' '{f}{u}{s}' '{v}{r}{oo}{m}' '{g}{e}{r}'
+ '{e}{x}{oo}{d}' '{k}{a}{l}{p}' '{k}{a}{l}{l}{y}{n}' '{k}{a}{t}{a}{d}' '{m}{o}{u}{l}' '{m}{p}{a}{n}' '{m}{p}{a}{g}{y}{a}{t}' '{m}{p}{o}{l}' '{m}{p}{o}{s}'
+ '{n}{y}{t}' '{x}{y}{k}' '{s}{u}{n}{o}{m}{i}{l}' '{p}{e}{t}{s}' '{p}{y}{t}{s}' '{p}{y}{k}{a}{n}{t}' '{p}{l}{y}{a}{t}{s}' '{p}{o}{s}{t}{e}{l}{n}' '{p}{r}{oo}{t}{o}{d}'
+ '{s}{e}{r}{t}' '{s}{u}{n}{a}{d}' '{t}{s}{a}{m}' '{u}{p}{o}{d}' '{f}{y}{l}{o}{n}' '{f}{u}{l}{o}{d}' '{ch}{a}{s}'
+ (<- '{y}{k}')
+ )
+ )
+
+ define step5a as (
+ do ('{a}{g}{a}{m}{e}' atlimit <- '{a}{g}{a}{m}')
+ do (
+ [substring] among (
+ '{a}{g}{a}{m}{e}' '{i}{s}{a}{m}{e}' '{o}{u}{s}{a}{m}{e}' '{i}{k}{a}{m}{e}' '{i}{th}{i}{k}{a}{m}{e}' (delete unset test1)
+ )
+ )
+ ['{a}{m}{e}']
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{a}{n}{a}{p}' '{a}{p}{o}{th}' '{a}{p}{o}{k}' '{a}{p}{o}{s}{t}' '{v}{o}{u}{v}' '{x}{e}{th}' '{o}{u}{l}' '{p}{e}{th}' '{p}{y}{k}{r}' '{p}{o}{t}' '{s}{y}{ch}' '{ch}'
+ (<- '{a}{m}')
+ )
+ )
+
+ define step5b as (
+ do (
+ [substring] among (
+ '{a}{g}{a}{n}{e}' '{i}{s}{a}{n}{e}' '{o}{u}{s}{a}{n}{e}' '{y}{o}{n}{t}{a}{n}{e}' '{y}{o}{t}{a}{n}{e}' '{y}{o}{u}{n}{t}{a}{n}{e}' '{o}{n}{t}{a}{n}{e}' '{o}{t}{a}{n}{e}'
+ '{o}{u}{n}{t}{a}{n}{e}' '{i}{k}{a}{n}{e}' '{i}{th}{i}{k}{a}{n}{e}' (
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{t}{r}' '{t}{s}' (<- '{a}{g}{a}{n}')
+ )
+ )
+ )
+ )
+ ['{a}{n}{e}']
+ delete
+ unset test1
+ ([] v2 <- '{a}{n}') or
+ [] substring atlimit among (
+ '{v}{e}{t}{e}{r}' '{v}{o}{u}{l}{k}' '{v}{r}{a}{ch}{m}' '{g}' '{d}{r}{a}{d}{o}{u}{m}'
+ '{th}' '{k}{a}{l}{p}{o}{u}{z}' '{k}{a}{s}{t}{e}{l}' '{k}{o}{r}{m}{o}{r}' '{l}{a}{o}{p}{l}' '{m}{oo}{a}{m}{e}{th}'
+ '{m}' '{m}{o}{u}{s}{o}{u}{l}{m}' '{n}' '{o}{u}{l}' '{p}' '{p}{e}{l}{e}{k}' '{p}{l}' '{p}{o}{l}{y}{s}'
+ '{p}{o}{r}{t}{o}{l}' '{s}{a}{r}{a}{k}{a}{t}{s}' '{s}{o}{u}{l}{t}' '{t}{s}{a}{r}{l}{a}{t}' '{o}{r}{f}'
+ '{t}{s}{y}{g}{g}' '{t}{s}{o}{p}' '{f}{oo}{t}{o}{s}{t}{e}{f}' '{ch}' '{ps}{u}{ch}{o}{p}{l}' '{a}{g}'
+ '{g}{a}{l}' '{g}{e}{r}' '{d}{e}{k}' '{d}{y}{p}{l}' '{a}{m}{e}{r}{y}{k}{a}{n}' '{o}{u}{r}' '{p}{y}{th}'
+ '{p}{o}{u}{r}{y}{t}' '{s}' '{z}{oo}{n}{t}' '{y}{k}' '{k}{a}{s}{t}' '{k}{o}{p}' '{l}{y}{ch}'
+ '{l}{o}{u}{th}{i}{r}' '{m}{a}{y}{n}{t}' '{m}{e}{l}' '{s}{y}{g}' '{s}{p}' '{s}{t}{e}{g}' '{t}{r}{a}{g}'
+ '{t}{s}{a}{g}' '{f}' '{e}{r}' '{a}{d}{a}{p}' '{a}{th}{y}{g}{g}' '{a}{m}{i}{ch}' '{a}{n}{y}{k}'
+ '{a}{n}{o}{r}{g}' '{a}{p}{i}{g}' '{a}{p}{y}{th}' '{a}{t}{s}{y}{g}{g}' '{v}{a}{s}' '{v}{a}{s}{k}'
+ '{v}{a}{th}{u}{g}{a}{l}' '{v}{y}{o}{m}{i}{ch}' '{v}{r}{a}{ch}{u}{k}' '{d}{y}{a}{t}' '{d}{y}{a}{f}' '{e}{n}{o}{r}{g}'
+ '{th}{u}{s}' '{k}{a}{p}{n}{o}{v}{y}{o}{m}{i}{ch}' '{k}{a}{t}{a}{g}{a}{l}' '{k}{l}{y}{v}' '{k}{o}{y}{l}{a}{r}{f}'
+ '{l}{y}{v}' '{m}{e}{g}{l}{o}{v}{y}{o}{m}{i}{ch}' '{m}{y}{k}{r}{o}{v}{y}{o}{m}{i}{ch}' '{n}{t}{a}{v}'
+ '{x}{i}{r}{o}{k}{l}{y}{v}' '{o}{l}{y}{g}{o}{d}{a}{m}' '{o}{l}{o}{g}{a}{l}' '{p}{e}{n}{t}{a}{r}{f}' '{p}{e}{r}{i}{f}'
+ '{p}{e}{r}{y}{t}{r}' '{p}{l}{a}{t}' '{p}{o}{l}{u}{d}{a}{p}' '{p}{o}{l}{u}{m}{i}{ch}' '{s}{t}{e}{f}' '{t}{a}{v}'
+ '{t}{e}{t}' '{u}{p}{e}{r}{i}{f}' '{u}{p}{o}{k}{o}{p}' '{ch}{a}{m}{i}{l}{o}{d}{a}{p}' '{ps}{i}{l}{o}{t}{a}{v}'
+ (<- '{a}{n}')
+ )
+ )
+
+ define step5c as (
+ do (
+ [substring] among (
+ '{i}{s}{e}{t}{e}' (delete unset test1)
+ )
+ )
+ ['{e}{t}{e}']
+ delete
+ unset test1
+ ([] v2 <- '{e}{t}') or
+ ([] substring among (
+ '{o}{d}' '{a}{y}{r}' '{f}{o}{r}' '{t}{a}{th}' '{d}{y}{a}{th}' '{s}{ch}' '{e}{n}{d}' '{e}{u}{r}' '{t}{y}{th}' '{u}{p}{e}{r}{th}'
+ '{r}{a}{th}' '{e}{n}{th}' '{r}{o}{th}' '{s}{th}' '{p}{u}{r}' '{a}{y}{n}' '{s}{u}{n}{d}' '{s}{u}{n}' '{s}{u}{n}{th}' '{ch}{oo}{r}'
+ '{p}{o}{n}' '{v}{r}' '{k}{a}{th}' '{e}{u}{th}' '{e}{k}{th}' '{n}{e}{t}' '{r}{o}{n}' '{a}{r}{k}' '{v}{a}{r}' '{v}{o}{l}' '{oo}{f}{e}{l}'
+ (<- '{e}{t}')
+ )) or
+ [] substring atlimit among (
+ '{a}{v}{a}{r}' '{v}{e}{n}' '{e}{n}{a}{r}' '{a}{v}{r}' '{a}{d}' '{a}{th}' '{a}{n}' '{a}{p}{l}' '{v}{a}{r}{o}{n}' '{n}{t}{r}' '{s}{k}' '{k}{o}{p}'
+ '{m}{p}{o}{r}' '{n}{y}{f}' '{p}{a}{g}' '{p}{a}{r}{a}{k}{a}{l}' '{s}{e}{r}{p}' '{s}{k}{e}{l}' '{s}{u}{r}{f}' '{t}{o}{k}' '{u}' '{d}' '{e}{m}'
+ '{th}{a}{r}{r}' '{th}'
+ (<- '{e}{t}')
+ )
+ )
+
+ define step5d as (
+ [substring] among (
+ '{o}{n}{t}{a}{s}' '{oo}{n}{t}{a}{s}' (
+ delete
+ unset test1
+ ([] '{a}{r}{ch}' atlimit <- '{o}{n}{t}') or
+ ([] '{k}{r}{e}' <- '{oo}{n}{t}')
+ )
+ )
+ )
+
+ define step5e as (
+ [substring] among (
+ '{o}{m}{a}{s}{t}{e}' '{y}{o}{m}{a}{s}{t}{e}' (
+ delete
+ unset test1
+ ([] '{o}{n}' atlimit <- '{o}{m}{a}{s}{t}')
+ )
+ )
+ )
+
+ define step5f as (
+ do (
+ ['{y}{e}{s}{t}{e}']
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{p}' '{a}{p}' '{s}{u}{m}{p}' '{a}{s}{u}{m}{p}' '{a}{k}{a}{t}{a}{p}' '{a}{m}{e}{t}{a}{m}{f}' (<- '{y}{e}{s}{t}')
+ )
+ )
+ ['{e}{s}{t}{e}']
+ delete
+ unset test1
+ [] substring atlimit among (
+ '{a}{l}' '{a}{r}' '{e}{k}{t}{e}{l}' '{z}' '{m}' '{x}' '{p}{a}{r}{a}{k}{a}{l}' '{p}{r}{o}' '{n}{y}{s}'
+ (<- '{y}{e}{s}{t}')
+ )
+ )
+
+ define step5g as (
+ do (
+ [substring] among (
+ '{i}{th}{i}{k}{a}' '{i}{th}{i}{k}{e}{s}' '{i}{th}{i}{k}{e}' (delete unset test1)
+ )
+ )
+ [substring] among (
+ '{i}{k}{a}' '{i}{k}{e}{s}' '{i}{k}{e}' (
+ delete
+ unset test1
+ ([] substring among (
+ '{s}{k}{oo}{l}' '{s}{k}{o}{u}{l}' '{n}{a}{r}{th}' '{s}{f}' '{o}{th}' '{p}{y}{th}' (<- '{i}{k}')
+ )) or
+ ([] substring atlimit among (
+ '{d}{y}{a}{th}' '{th}' '{p}{a}{r}{a}{k}{a}{t}{a}{th}' '{p}{r}{o}{s}{th}' '{s}{u}{n}{th}' (<- '{i}{k}')
+ ))
+ )
+ )
+ )
+
+ define step5h as (
+ [substring] among (
+ '{o}{u}{s}{a}' '{o}{u}{s}{e}{s}' '{o}{u}{s}{e}' (
+ delete
+ unset test1
+ ([] substring among (
+ '{p}{o}{d}{a}{r}' '{v}{l}{e}{p}' '{p}{a}{n}{t}{a}{ch}' '{f}{r}{u}{d}' '{m}{a}{n}{t}{y}{l}' '{m}{a}{l}{l}' '{k}{u}{m}{a}{t}' '{l}{a}{ch}' '{l}{i}{g}'
+ '{f}{a}{g}' '{o}{m}' '{p}{r}{oo}{t}' (<- '{o}{u}{s}')
+
+ )) or
+ ([] substring atlimit among (
+ '{f}{a}{r}{m}{a}{k}' '{ch}{a}{d}' '{a}{g}{k}' '{a}{n}{a}{r}{r}' '{v}{r}{o}{m}' '{e}{k}{l}{y}{p}' '{l}{a}{m}{p}{y}{d}' '{l}{e}{ch}' '{m}' '{p}{a}{t}'
+ '{r}' '{l}' '{m}{e}{d}' '{m}{e}{s}{a}{z}' '{u}{p}{o}{t}{e}{y}{n}' '{a}{m}' '{a}{y}{th}' '{a}{n}{i}{k}' '{d}{e}{s}{p}{o}{z}'
+ '{e}{n}{d}{y}{a}{f}{e}{r}' '{d}{e}' '{d}{e}{u}{t}{e}{r}{e}{u}' '{k}{a}{th}{a}{r}{e}{u}' '{p}{l}{e}' '{t}{s}{a}'
+ (<- '{o}{u}{s}')
+ ))
+ )
+ )
+ )
+
+ define step5i as (
+ [substring] among (
+ '{a}{g}{a}' '{a}{g}{e}{s}' '{a}{g}{e}' (
+ delete
+ unset test1
+ ([] '{k}{o}{l}{l}' <- '{a}{g}') or (
+ not ([substring] among ('{ps}{o}{f}' '{n}{a}{u}{l}{o}{ch}'))
+ ([] substring among (
+ '{o}{f}' '{p}{e}{l}' '{ch}{o}{r}{t}' '{l}{l}' '{s}{f}' '{r}{p}' '{f}{r}' '{p}{r}' '{l}{o}{ch}' '{s}{m}{i}{n}'
+ (<- '{a}{g}')
+ )) or
+ ([] substring atlimit among (
+ '{a}{v}{a}{s}{t}' '{p}{o}{l}{u}{f}' '{a}{d}{i}{f}' '{p}{a}{m}{f}' '{r}' '{a}{s}{p}' '{a}{f}' '{a}{m}{a}{l}' '{a}{m}{a}{l}{l}{y}'
+ '{a}{n}{u}{s}{t}' '{a}{p}{e}{r}' '{a}{s}{p}{a}{r}' '{a}{ch}{a}{r}' '{d}{e}{r}{v}{e}{n}' '{d}{r}{o}{s}{o}{p}' '{x}{e}{f}' '{n}{e}{o}{p}'
+ '{n}{o}{m}{o}{t}' '{o}{l}{o}{p}' '{o}{m}{o}{t}' '{p}{r}{o}{s}{t}' '{p}{r}{o}{s}{oo}{p}{o}{p}' '{s}{u}{m}{p}' '{s}{u}{n}{t}' '{t}' '{u}{p}{o}{t}'
+ '{ch}{a}{r}' '{a}{e}{y}{p}' '{a}{y}{m}{o}{s}{t}' '{a}{n}{u}{p}' '{a}{p}{o}{t}' '{a}{r}{t}{y}{p}' '{d}{y}{a}{t}' '{e}{n}' '{e}{p}{y}{t}'
+ '{k}{r}{o}{k}{a}{l}{o}{p}' '{s}{y}{d}{i}{r}{o}{p}' '{l}' '{n}{a}{u}' '{o}{u}{l}{a}{m}' '{o}{u}{r}' '{p}' '{t}{r}' '{m}'
+ (<- '{a}{g}')
+ ))
+ )
+ )
+ )
+ )
+
+ define step5j as (
+ [substring] among (
+ '{i}{s}{e}' '{i}{s}{o}{u}' '{i}{s}{a}' (delete unset test1)
+ )
+ [] substring atlimit among (
+ '{n}' '{ch}{e}{r}{s}{o}{n}' '{d}{oo}{d}{e}{k}{a}{n}' '{e}{r}{i}{m}{o}{n}' '{m}{e}{g}{a}{l}{o}{n}' '{e}{p}{t}{a}{n}' (<- '{i}{s}')
+ )
+ )
+
+ define step5k as (
+ [substring] among (
+ '{i}{s}{t}{e}' (delete unset test1)
+ )
+ [] substring atlimit among (
+ '{a}{s}{v}' '{s}{v}' '{a}{ch}{r}' '{ch}{r}' '{a}{p}{l}' '{a}{e}{y}{m}{n}' '{d}{u}{s}{ch}{r}' '{e}{u}{ch}{r}' '{k}{o}{y}{n}{o}{ch}{r}' '{p}{a}{l}{y}{m}{ps}'
+ (<- '{i}{s}{t}')
+ )
+ )
+
+ define step5l as (
+ [substring] among (
+ '{o}{u}{n}{e}' '{i}{s}{o}{u}{n}{e}' '{i}{th}{o}{u}{n}{e}' (delete unset test1)
+ )
+ [] substring atlimit among (
+ '{n}' '{r}' '{s}{p}{y}' '{s}{t}{r}{a}{v}{o}{m}{o}{u}{t}{s}' '{k}{a}{k}{o}{m}{o}{u}{t}{s}' '{e}{x}{oo}{n}' (<- '{o}{u}{n}')
+ )
+ )
+
+ define step5m as (
+ [substring] among (
+ '{o}{u}{m}{e}' '{i}{s}{o}{u}{m}{e}' '{i}{th}{o}{u}{m}{e}' (delete unset test1)
+ )
+ [] substring atlimit among (
+ '{p}{a}{r}{a}{s}{o}{u}{s}' '{f}' '{ch}' '{oo}{r}{y}{o}{p}{l}' '{a}{z}' '{a}{l}{l}{o}{s}{o}{u}{s}' '{a}{s}{o}{u}{s}'
+ (<- '{o}{u}{m}')
+ )
+ )
+
+ define step6 as (
+ do (
+ [substring] among (
+ '{m}{a}{t}{a}' '{m}{a}{t}{oo}{n}' '{m}{a}{t}{o}{s}' (<- '{m}{a}')
+ )
+ )
+ test1
+ [substring] among (
+ '{a}' '{a}{g}{a}{t}{e}' '{a}{g}{a}{n}' '{a}{e}{y}' '{a}{m}{a}{y}' '{a}{n}' '{a}{s}' '{a}{s}{a}{y}' '{a}{t}{a}{y}' '{a}{oo}' '{e}' '{e}{y}'
+ '{e}{y}{s}' '{e}{y}{t}{e}' '{e}{s}{a}{y}' '{e}{s}' '{e}{t}{a}{y}' '{y}' '{y}{e}{m}{a}{y}' '{y}{e}{m}{a}{s}{t}{e}' '{y}{e}{t}{a}{y}' '{y}{e}{s}{a}{y}'
+ '{y}{e}{s}{a}{s}{t}{e}' '{y}{o}{m}{a}{s}{t}{a}{n}' '{y}{o}{m}{o}{u}{n}' '{y}{o}{m}{o}{u}{n}{a}' '{y}{o}{n}{t}{a}{n}' '{y}{o}{n}{t}{o}{u}{s}{a}{n}' '{y}{o}{s}{a}{s}{t}{a}{n}'
+ '{y}{o}{s}{a}{s}{t}{e}' '{y}{o}{s}{o}{u}{n}' '{y}{o}{s}{o}{u}{n}{a}' '{y}{o}{t}{a}{n}' '{y}{o}{u}{m}{a}' '{y}{o}{u}{m}{a}{s}{t}{e}' '{y}{o}{u}{n}{t}{a}{y}'
+ '{y}{o}{u}{n}{t}{a}{n}' '{i}' '{i}{d}{e}{s}' '{i}{d}{oo}{n}' '{i}{th}{e}{y}' '{i}{th}{e}{y}{s}' '{i}{th}{e}{y}{t}{e}' '{i}{th}{i}{k}{a}{t}{e}' '{i}{th}{i}{k}{a}{n}'
+ '{i}{th}{o}{u}{n}' '{i}{th}{oo}' '{i}{k}{a}{t}{e}' '{i}{k}{a}{n}' '{i}{s}' '{i}{s}{a}{n}' '{i}{s}{a}{t}{e}' '{i}{s}{e}{y}' '{i}{s}{e}{s}' '{i}{s}{o}{u}{n}'
+ '{i}{s}{oo}' '{o}' '{o}{y}' '{o}{m}{a}{y}' '{o}{m}{a}{s}{t}{a}{n}' '{o}{m}{o}{u}{n}' '{o}{m}{o}{u}{n}{a}' '{o}{n}{t}{a}{y}' '{o}{n}{t}{a}{n}'
+ '{o}{n}{t}{o}{u}{s}{a}{n}' '{o}{s}' '{o}{s}{a}{s}{t}{a}{n}' '{o}{s}{a}{s}{t}{e}' '{o}{s}{o}{u}{n}' '{o}{s}{o}{u}{n}{a}' '{o}{t}{a}{n}' '{o}{u}' '{o}{u}{m}{a}{y}'
+ '{o}{u}{m}{a}{s}{t}{e}' '{o}{u}{n}' '{o}{u}{n}{t}{a}{y}' '{o}{u}{n}{t}{a}{n}' '{o}{u}{s}' '{o}{u}{s}{a}{n}' '{o}{u}{s}{a}{t}{e}' '{u}' '{u}{s}' '{oo}'
+ '{oo}{n}' (delete)
+ )
+ )
+
+ define step7 as (
+ [substring] among (
+ '{e}{s}{t}{e}{r}' '{e}{s}{t}{a}{t}' '{o}{t}{e}{r}' '{o}{t}{a}{t}' '{u}{t}{e}{r}' '{u}{t}{a}{t}' '{oo}{t}{e}{r}' '{oo}{t}{a}{t}' (delete)
+ )
+ )
+)
+
+define stem as (
+ backwards (
+ do tolower
+ has_min_length
+ set test1
+ do step1
+ do steps1
+ do steps2
+ do steps3
+ do steps4
+ do steps5
+ do steps6
+ do steps7
+ do steps8
+ do steps9
+ do steps10
+ do step2a
+ do step2b
+ do step2c
+ do step2d
+ do step3
+ do step4
+ do step5a
+ do step5b
+ do step5c
+ do step5d
+ do step5e
+ do step5f
+ do step5g
+ do step5h
+ do step5j
+ do step5i
+ do step5k
+ do step5l
+ do step5m
+ do step6
+ do step7
+ )
+)
diff --git a/contrib/snowball/algorithms/hindi.sbl b/contrib/snowball/algorithms/hindi.sbl
new file mode 100644
index 0000000..bfdfac0
--- /dev/null
+++ b/contrib/snowball/algorithms/hindi.sbl
@@ -0,0 +1,323 @@
+// An implementation of "A Lightweight Stemmer for Hindi":
+// http://www.kbcs.in/downloads/papers/StmmerHindi.pdf
+
+externals ( stem )
+
+stringescapes {}
+
+// The transliteration scheme used for our stringdefs matches that used in the
+// paper, as documented in the appendix. It appears to match the WX notation
+// (https://en.wikipedia.org/wiki/WX_notation) except that WX apparently
+// uses 'z' for Anunasika whereas the paper uses Mh.
+//
+// We discriminate dependent vowels by adding a leading "_" to their stringdef
+// names (mnemonic: the _ signifies removing the implicit a from the preceding
+// character).
+
+// Vowels and sonorants:
+stringdef a '{U+0905}'
+stringdef A '{U+0906}'
+stringdef i '{U+0907}'
+stringdef I '{U+0908}'
+stringdef u '{U+0909}'
+stringdef U '{U+090A}'
+stringdef q '{U+090B}'
+stringdef e '{U+090F}'
+stringdef E '{U+0910}'
+stringdef o '{U+0913}'
+stringdef O '{U+0914}'
+
+// Vowel signs:
+stringdef _A '{U+093E}'
+stringdef _i '{U+093F}'
+stringdef _I '{U+0940}'
+stringdef _u '{U+0941}'
+stringdef _U '{U+0942}'
+stringdef _q '{U+0943}'
+stringdef _e '{U+0947}'
+stringdef _E '{U+0948}'
+stringdef _o '{U+094B}'
+stringdef _O '{U+094C}'
+
+// Diacritics:
+stringdef M '{U+0902}'
+stringdef H '{U+0903}'
+stringdef Mh '{U+0901}'
+stringdef Z '{U+093C}' // Nukta
+stringdef virama '{U+094D}'
+
+// Velar consonants:
+stringdef k '{U+0915}'
+stringdef K '{U+0916}'
+stringdef g '{U+0917}'
+stringdef G '{U+0918}'
+stringdef f '{U+0919}'
+
+// Palatal consonants:
+stringdef c '{U+091A}'
+stringdef C '{U+091B}'
+stringdef j '{U+091C}'
+stringdef J '{U+091D}'
+stringdef F '{U+091E}'
+
+// Retroflex consonants:
+stringdef t '{U+091F}'
+stringdef T '{U+0920}'
+stringdef d '{U+0921}'
+stringdef D '{U+0922}'
+stringdef N '{U+0923}'
+
+// Dental consonants:
+stringdef w '{U+0924}'
+stringdef W '{U+0925}'
+stringdef x '{U+0926}'
+stringdef X '{U+0927}'
+stringdef n '{U+0928}'
+
+// Labial consonants:
+stringdef p '{U+092A}'
+stringdef P '{U+092B}'
+stringdef b '{U+092C}'
+stringdef B '{U+092D}'
+stringdef m '{U+092E}'
+
+// Semi-vowels:
+stringdef y '{U+092F}'
+stringdef r '{U+0930}'
+stringdef l '{U+0932}'
+stringdef v '{U+0935}'
+
+// Fricatives:
+stringdef S '{U+0936}'
+stringdef R '{U+0937}'
+stringdef s '{U+0938}'
+stringdef h '{U+0939}'
+
+stringdef lY '{U+0933}'
+
+// Precomposed characters - letters + nukta:
+stringdef nZ '{U+0929}' // ≡ {n}{Z}
+stringdef rZ '{U+0931}' // ≡ {r}{Z}
+stringdef lYZ '{U+0934}' // ≡ {lY}{Z}
+stringdef kZ '{U+0958}' // ≡ {k}{Z}
+stringdef KZ '{U+0959}' // ≡ {K}{Z}
+stringdef gZ '{U+095A}' // ≡ {g}{Z}
+stringdef jZ '{U+095B}' // ≡ {j}{Z}
+stringdef dZ '{U+095C}' // ≡ {d}{Z}
+stringdef DZ '{U+095D}' // ≡ {D}{Z}
+stringdef PZ '{U+095E}' // ≡ {P}{Z}
+stringdef yZ '{U+095F}' // ≡ {y}{Z}
+
+integers ( p )
+
+groupings ( consonant )
+
+routines ( CONSONANT )
+
+define consonant '{k}{K}{g}{G}{f}' +
+ '{c}{C}{j}{J}{F}' +
+ '{t}{T}{d}{D}{N}' +
+ '{w}{W}{x}{X}{n}' +
+ '{p}{P}{b}{B}{m}' +
+ '{y}{r}{l}{v}' +
+ '{S}{R}{s}{h}' +
+ '{lY}' +
+ '{Z}' + // Nukta
+ // Precomposed characters - letter and nukta:
+ '{nZ}{rZ}{lYZ}{kZ}{KZ}{gZ}{jZ}{dZ}{DZ}{PZ}{yZ}'
+
+backwardmode ( define CONSONANT as ( consonant ) )
+
+define stem as (
+ test ( next setmark p )
+ backwards (
+ // We assume in this implementation that the whole word doesn't count
+ // as a valid suffix to remove, so we remove the longest suffix from
+ // the list which leaves at least one character. This change affects
+ // 47 words out of the 65,140 in the sample vocabulary from Hindi
+ // wikipedia.
+ setlimit tomark p for ([substring])
+ among (
+ // The list below is derived from figure 3 in the paper.
+ //
+ // We perform the stemming on the Devanagari characters rather than
+ // transliterating to Latin, so we have adapted the list below to
+ // reflect this by converting suffixes back to Devanagari as
+ // follows:
+ //
+ // * within the suffixes, "a" after a consonant is dropped since
+ // consonants have an implicit "a".
+ //
+ // * within the suffixes, a vowel other than "a" after a consonant
+ // is a dependent vowel (vowel sign); a vowel (including "a")
+ // after a non-consonant is an independent vowel.
+ //
+ // * to allow the vowel at the start of each suffix being dependent
+ // or independent, we include each suffix twice. For the
+ // dependent version, a leading "a" is dropped and we check that
+ // the suffix is preceded by a consonant (which will have an
+ // implicit "a").
+ //
+ // * we add '{a}', which is needed for the example given right at
+ // the end of section 5 to work (conflating BarawIya and
+ // BarawIyawA), and which 3.1 a.v strongly suggests should be in
+ // the list:
+ //
+ // Thus, the following suffix deletions (longest possible
+ // match) are required to reduce inflected forms of masculine
+ // nouns to a common stem:
+ // a A i [...]
+ //
+ // Adding '{a}' only affect 2 words out of the 65,140 in the
+ // sample vocabulary.
+ //
+ // * The transliterations of our stems would end with "a" when our
+ // stems end in a consonant, so we also include {virama} in the
+ // list of suffixes to remove (this affects 222 words from the
+ // sample vocabulary).
+ //
+ // We've also assumed that Mh in the suffix list always means {Mh}
+ // and never {M}{h}{virama}. Only one of the 65,140 words in the
+ // sample vocabulary stems differently due to this (and that word
+ // seems to be a typo).
+
+ '{virama}'
+
+ '{a}'
+ '{A}'
+ '{i}'
+ '{I}'
+ '{u}'
+ '{U}'
+ '{e}'
+ '{o}'
+ '{e}{M}'
+ '{o}{M}'
+ '{A}{M}'
+ '{u}{A}{M}'
+ '{u}{e}{M}'
+ '{u}{o}{M}'
+ '{A}{e}{M}'
+ '{A}{o}{M}'
+ '{i}{y}{_A}{M}'
+ '{i}{y}{_o}{M}'
+ '{A}{i}{y}{_A}{M}'
+ '{A}{i}{y}{_o}{M}'
+ '{A}{Mh}'
+ '{i}{y}{_A}{Mh}'
+ '{A}{i}{y}{_A}{Mh}'
+ '{a}{w}{_A}{e}{M}'
+ '{a}{w}{_A}{o}{M}'
+ '{a}{n}{_A}{e}{M}'
+ '{a}{n}{_A}{o}{M}'
+ '{a}{w}{_A}'
+ '{a}{w}{_I}'
+ '{I}{M}'
+ '{a}{w}{_I}{M}'
+ '{a}{w}{_e}'
+ '{A}{w}{_A}'
+ '{A}{w}{_I}'
+ '{A}{w}{_I}{M}'
+ '{A}{w}{_e}'
+ '{a}{n}{_A}'
+ '{a}{n}{_I}'
+ '{a}{n}{_e}'
+ '{A}{n}{_A}'
+ '{A}{n}{_e}'
+ '{U}{M}{g}{_A}'
+ '{U}{M}{g}{_I}'
+ '{A}{U}{M}{g}{_A}'
+ '{A}{U}{M}{g}{_I}'
+ '{e}{M}{g}{_e}'
+ '{e}{M}{g}{_I}'
+ '{A}{e}{M}{g}{_e}'
+ '{A}{e}{M}{g}{_I}'
+ '{o}{g}{_e}'
+ '{o}{g}{_I}'
+ '{A}{o}{g}{_e}'
+ '{A}{o}{g}{_I}'
+ '{e}{g}{_A}'
+ '{e}{g}{_I}'
+ '{A}{e}{g}{_A}'
+ '{A}{e}{g}{_I}'
+ '{A}{y}{_A}'
+ '{A}{e}'
+ '{A}{I}'
+ '{A}{I}{M}'
+ '{i}{e}'
+ '{A}{o}'
+ '{A}{i}{e}'
+ '{a}{k}{r}'
+ '{A}{k}{r}'
+
+ '{_A}'
+ '{_i}'
+ '{_I}'
+ '{_u}'
+ '{_U}'
+ '{_e}'
+ '{_o}'
+ '{_e}{M}'
+ '{_o}{M}'
+ '{_A}{M}'
+ '{_u}{A}{M}'
+ '{_u}{e}{M}'
+ '{_u}{o}{M}'
+ '{_A}{e}{M}'
+ '{_A}{o}{M}'
+ '{_i}{y}{_A}{M}'
+ '{_i}{y}{_o}{M}'
+ '{_A}{i}{y}{_A}{M}'
+ '{_A}{i}{y}{_o}{M}'
+ '{_A}{Mh}'
+ '{_i}{y}{_A}{Mh}'
+ '{_A}{i}{y}{_A}{Mh}'
+ '{_I}{M}'
+ '{_A}{w}{_A}'
+ '{_A}{w}{_I}'
+ '{_A}{w}{_I}{M}'
+ '{_A}{w}{_e}'
+ '{_A}{n}{_A}'
+ '{_A}{n}{_e}'
+ '{_U}{M}{g}{_A}'
+ '{_U}{M}{g}{_I}'
+ '{_A}{U}{M}{g}{_A}'
+ '{_A}{U}{M}{g}{_I}'
+ '{_e}{M}{g}{_e}'
+ '{_e}{M}{g}{_I}'
+ '{_A}{e}{M}{g}{_e}'
+ '{_A}{e}{M}{g}{_I}'
+ '{_o}{g}{_e}'
+ '{_o}{g}{_I}'
+ '{_A}{o}{g}{_e}'
+ '{_A}{o}{g}{_I}'
+ '{_e}{g}{_A}'
+ '{_e}{g}{_I}'
+ '{_A}{e}{g}{_A}'
+ '{_A}{e}{g}{_I}'
+ '{_A}{y}{_A}'
+ '{_A}{e}'
+ '{_A}{I}'
+ '{_A}{I}{M}'
+ '{_i}{e}'
+ '{_A}{o}'
+ '{_A}{i}{e}'
+ '{_A}{k}{r}'
+
+ /* Suffixes with a leading implicit a: */
+ '{w}{_A}{e}{M}' CONSONANT
+ '{w}{_A}{o}{M}' CONSONANT
+ '{n}{_A}{e}{M}' CONSONANT
+ '{n}{_A}{o}{M}' CONSONANT
+ '{w}{_A}' CONSONANT
+ '{w}{_I}' CONSONANT
+ '{w}{_I}{M}' CONSONANT
+ '{w}{_e}' CONSONANT
+ '{n}{_A}' CONSONANT
+ '{n}{_I}' CONSONANT
+ '{n}{_e}' CONSONANT
+ '{k}{r}' CONSONANT
+ )
+ delete
+ )
+)
diff --git a/contrib/snowball/algorithms/hungarian.sbl b/contrib/snowball/algorithms/hungarian.sbl
new file mode 100644
index 0000000..2d7885c
--- /dev/null
+++ b/contrib/snowball/algorithms/hungarian.sbl
@@ -0,0 +1,241 @@
+/*
+Hungarian Stemmer
+Removes noun inflections
+*/
+
+routines (
+ mark_regions
+ R1
+ v_ending
+ case
+ case_special
+ case_other
+ plural
+ owned
+ sing_owner
+ plur_owner
+ instrum
+ factive
+ undouble
+ double
+)
+
+externals ( stem )
+
+integers ( p1 )
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a' '{U+00E1}' //a-acute
+stringdef e' '{U+00E9}' //e-acute
+stringdef i' '{U+00ED}' //i-acute
+stringdef o' '{U+00F3}' //o-acute
+stringdef o" '{U+00F6}' //o-umlaut
+stringdef oq '{U+0151}' //o-double acute
+stringdef u' '{U+00FA}' //u-acute
+stringdef u" '{U+00FC}' //u-umlaut
+stringdef uq '{U+0171}' //u-double acute
+
+define v 'aeiou{a'}{e'}{i'}{o'}{o"}{oq}{u'}{u"}{uq}'
+
+define mark_regions as (
+
+ $p1 = limit
+
+ (v goto non-v
+ among('cs' 'gy' 'ly' 'ny' 'sz' 'ty' 'zs' 'dzs') or next
+ setmark p1)
+ or
+
+ (non-v gopast v setmark p1)
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+
+ define v_ending as (
+ [substring] R1 among(
+ '{a'}' (<- 'a')
+ '{e'}' (<- 'e')
+ )
+ )
+
+ define double as (
+ test among('bb' 'cc' 'ccs' 'dd' 'ff' 'gg' 'ggy' 'jj' 'kk' 'll' 'lly' 'mm'
+ 'nn' 'nny' 'pp' 'rr' 'ss' 'ssz' 'tt' 'tty' 'vv' 'zz' 'zzs')
+ )
+
+ define undouble as (
+ next [hop 1] delete
+ )
+
+ define instrum as(
+ [substring] R1 among(
+ 'al' (double)
+ 'el' (double)
+ )
+ delete
+ undouble
+ )
+
+
+ define case as (
+ [substring] R1 among(
+ 'ban' 'ben'
+ 'ba' 'be'
+ 'ra' 're'
+ 'nak' 'nek'
+ 'val' 'vel'
+ 't{o'}l' 't{oq}l'
+ 'r{o'}l' 'r{oq}l'
+ 'b{o'}l' 'b{oq}l'
+ 'hoz' 'hez' 'h{o"}z'
+ 'n{a'}l' 'n{e'}l'
+ 'ig'
+ 'at' 'et' 'ot' '{o"}t'
+ '{e'}rt'
+ 'k{e'}pp' 'k{e'}ppen'
+ 'kor'
+ 'ul' '{u"}l'
+ 'v{a'}' 'v{e'}'
+ 'onk{e'}nt' 'enk{e'}nt' 'ank{e'}nt'
+ 'k{e'}nt'
+ 'en' 'on' 'an' '{o"}n'
+ 'n'
+ 't'
+ )
+ delete
+ v_ending
+ )
+
+ define case_special as(
+ [substring] R1 among(
+ '{e'}n' (<- 'e')
+ '{a'}n' (<- 'a')
+ '{a'}nk{e'}nt' (<- 'a')
+ )
+ )
+
+ define case_other as(
+ [substring] R1 among(
+ 'astul' 'est{u"}l' (delete)
+ 'stul' 'st{u"}l' (delete)
+ '{a'}stul' (<- 'a')
+ '{e'}st{u"}l' (<- 'e')
+ )
+ )
+
+ define factive as(
+ [substring] R1 among(
+ '{a'}' (double)
+ '{e'}' (double)
+ )
+ delete
+ undouble
+ )
+
+ define plural as (
+ [substring] R1 among(
+ '{a'}k' (<- 'a')
+ '{e'}k' (<- 'e')
+ '{o"}k' (delete)
+ 'ak' (delete)
+ 'ok' (delete)
+ 'ek' (delete)
+ 'k' (delete)
+ )
+ )
+
+ define owned as (
+ [substring] R1 among (
+ 'ok{e'}' '{o"}k{e'}' 'ak{e'}' 'ek{e'}' (delete)
+ '{e'}k{e'}' (<- 'e')
+ '{a'}k{e'}' (<- 'a')
+ 'k{e'}' (delete)
+ '{e'}{e'}i' (<- 'e')
+ '{a'}{e'}i' (<- 'a')
+ '{e'}i' (delete)
+ '{e'}{e'}' (<- 'e')
+ '{e'}' (delete)
+ )
+ )
+
+ define sing_owner as (
+ [substring] R1 among(
+ '{u"}nk' 'unk' (delete)
+ '{a'}nk' (<- 'a')
+ '{e'}nk' (<- 'e')
+ 'nk' (delete)
+ '{a'}juk' (<- 'a')
+ '{e'}j{u"}k' (<- 'e')
+ 'juk' 'j{u"}k' (delete)
+ 'uk' '{u"}k' (delete)
+ 'em' 'om' 'am' (delete)
+ '{a'}m' (<- 'a')
+ '{e'}m' (<- 'e')
+ 'm' (delete)
+ 'od' 'ed' 'ad' '{o"}d' (delete)
+ '{a'}d' (<- 'a')
+ '{e'}d' (<- 'e')
+ 'd' (delete)
+ 'ja' 'je' (delete)
+ 'a' 'e' 'o' (delete)
+ '{a'}' (<- 'a')
+ '{e'}' (<- 'e')
+ )
+ )
+
+ define plur_owner as (
+ [substring] R1 among(
+ 'jaim' 'jeim' (delete)
+ '{a'}im' (<- 'a')
+ '{e'}im' (<- 'e')
+ 'aim' 'eim' (delete)
+ 'im' (delete)
+ 'jaid' 'jeid' (delete)
+ '{a'}id' (<- 'a')
+ '{e'}id' (<- 'e')
+ 'aid' 'eid' (delete)
+ 'id' (delete)
+ 'jai' 'jei' (delete)
+ '{a'}i' (<- 'a')
+ '{e'}i' (<- 'e')
+ 'ai' 'ei' (delete)
+ 'i' (delete)
+ 'jaink' 'jeink' (delete)
+ 'eink' 'aink' (delete)
+ '{a'}ink' (<- 'a')
+ '{e'}ink' (<- 'e')
+ 'ink'
+ 'jaitok' 'jeitek' (delete)
+ 'aitok' 'eitek' (delete)
+ '{a'}itok' (<- 'a')
+ '{e'}itek' (<- 'e')
+ 'itek' (delete)
+ 'jeik' 'jaik' (delete)
+ 'aik' 'eik' (delete)
+ '{a'}ik' (<- 'a')
+ '{e'}ik' (<- 'e')
+ 'ik' (delete)
+ )
+ )
+)
+
+define stem as (
+ do mark_regions
+ backwards (
+ do instrum
+ do case
+ do case_special
+ do case_other
+ do factive
+ do owned
+ do sing_owner
+ do plur_owner
+ do plural
+ )
+)
diff --git a/contrib/snowball/algorithms/indonesian.sbl b/contrib/snowball/algorithms/indonesian.sbl
new file mode 100644
index 0000000..ac0ee36
--- /dev/null
+++ b/contrib/snowball/algorithms/indonesian.sbl
@@ -0,0 +1,192 @@
+// An implementation of the "Porter Stemmer for Bahasa Indonesia" from:
+// http://www.illc.uva.nl/Research/Publications/Reports/MoL-2003-02.text.pdf
+
+integers (
+ // The paper defines measure as the number of vowels in the word. We
+ // count this initially, then adjust the count each time we remove a
+ // prefix or suffix.
+ measure
+
+ // Numeric code for the type of prefix removed:
+ //
+ // 0 other/none
+ // 1 'di' or 'meng' or 'ter'
+ // 2 'per'
+ // 3 'ke' or 'peng'
+ // 4 'ber'
+ //
+ // Some of these have variant forms, so e.g. "meng" includes "men", "me",
+ // "meny", "mem".
+ //
+ // Note that the value of prefix is only used in remove_suffix (and
+ // routines it calls) so we don't need to worry about
+ // remove_second_order_prefix overwriting a value of prefix set by
+ // remove_first_order_prefix since remove_suffix gets called between
+ // the two.
+ prefix
+)
+
+groupings ( vowel )
+
+routines (
+ remove_particle
+ remove_possessive_pronoun
+ remove_first_order_prefix
+ remove_second_order_prefix
+ remove_suffix
+ KER
+ SUFFIX_KAN_OK
+ SUFFIX_AN_OK
+ SUFFIX_I_OK
+ VOWEL
+)
+
+externals ( stem )
+
+stringescapes {}
+
+backwardmode (
+
+ define remove_particle as (
+ [substring] among (
+ 'kah' 'lah' 'pun' (delete $measure-=1)
+ )
+ )
+
+ define remove_possessive_pronoun as (
+ [substring] among (
+ 'ku' 'mu' 'nya' (delete $measure-=1)
+ )
+ )
+
+ // prefix not in {ke, peng, per}
+ define SUFFIX_KAN_OK as (
+ // On page 29, the example "kompas Q.31" says "Both Nazief and Porter
+ // stemmer converted the word peledakan (blast, explotion) to ledak (to
+ // blast, to explode)". However, the algorithm as described doesn't
+ // behave in this way - grammatically the prefix pe- occurs as a
+ // variation of both the first-order derivational prefix peng- and the
+ // second-order derivational prefix per-, but table 2.5 doesn't include
+ // "pe", only table 2.6 does, so "peledakan" is handled (incorrectly)
+ // as having prefix "per" not "peng", and so we remove derivational
+ // suffix "kan" rather than "an" to give stem leda. (Porter-style
+ // stemmers remove the longest suffix they can amongst those available,
+ // which this paper notes in the last paragraph on page 15).
+ //
+ // We resolve this by amending the condition on suffix "kan" to
+ // "prefix ∉ {ke, peng, per}", which seems to make the stemmer's
+ // behaviour match all the examples in the paper except for one:
+ // "perbaikan" is shown in table 3.4 as stemming to "bai", but with
+ // this change it now stems to "baik". The table notes that "baik" is
+ // the actual root so this deviation is an improvement. In a sample
+ // vocabulary derived from the most common words in id.wikipedia.org,
+ // this change only affects 0.12% of words (76 out of 64,587, including
+ // "peledakan" and "perbaikan").
+ $prefix != 3 and $prefix != 2
+ )
+
+ // prefix not in {di, meng, ter}
+ define SUFFIX_AN_OK as ( $prefix != 1 )
+
+ define SUFFIX_I_OK as (
+ // prefix not in {ke, peng, ber}
+ $prefix <= 2
+
+ // The rest of the condition from the paper is:
+ // V|K...câ‚câ‚, c₠≠ s, câ‚‚ ≠ i
+ //
+ // The meaning of this is unclear in several ways, and none of the
+ // examples given of the stemmer's behaviour in the paper help to
+ // resolve these issues.
+ //
+ // Notice that câ‚‚ isn't actually used - the most obvious explanation
+ // seems to be that "câ‚câ‚" should read "câ‚câ‚‚", or maybe "câ‚‚câ‚".
+ //
+ // Elsewhere the paper defines V... as meaning "the stem starts with
+ // a vowel" and K... as meaning "the stem starts with a consonant".
+ //
+ // In other places where it says X|Y... it seems the | binds more
+ // tightly, so it's (V|K)...cáµ¢câ±¼ not V|(K...cáµ¢câ±¼). That seems a bit
+ // odd as the first letter must be either a vowel or a consonant, so
+ // that really just means "ends cáµ¢câ±¼". However, nowhere in the paper
+ // uses or defines a notation such as ...X, which may explain this
+ // seemingly redundant way of specifying this.
+ //
+ // The conditions elsewhere on prefix removal (e.g. V...) are clearly
+ // on the stem left after the prefix is removed. None of the other
+ // rules for suffix removal have conditions on the stem, but for
+ // consistency with the prefix rules we might expect that the cáµ¢câ±¼
+ // test is on what's left *after* removing the "i" suffix.
+ //
+ // However, studying Indonesian wordlists and discussion with a native
+ // speaker leads us to conclude that the purpose of this check is to
+ // protect words of foreign origin (e.g. "televisi", "organisasi",
+ // "komunikasi") from stemming, and the common feature of these is
+ // that the word ends "-si", so we conclude that the condition here
+ // should be read as "word does not end -si", and this is what we
+ // have implemented.
+ not 's'
+ )
+
+ define remove_suffix as (
+ [substring] among (
+ 'kan' SUFFIX_KAN_OK 'an' SUFFIX_AN_OK 'i' SUFFIX_I_OK
+ (delete $measure-=1)
+ )
+ )
+)
+
+define vowel 'aeiou'
+
+define VOWEL as ( vowel )
+
+define KER as ( non-vowel 'er' )
+
+define remove_first_order_prefix as (
+ [substring] among (
+ 'di' 'meng' 'men' 'me' 'ter' (delete $prefix=1 $measure-=1)
+ 'ke' 'peng' 'pen' (delete $prefix=3 $measure-=1)
+ 'meny' VOWEL ($prefix=1 <-'s' $measure-=1)
+ 'peny' VOWEL ($prefix=3 <-'s' $measure-=1)
+ 'mem' ($prefix=1 $measure-=1 vowel and <-'p' or delete)
+ 'pem' ($prefix=3 $measure-=1 vowel and <-'p' or delete)
+ )
+)
+
+define remove_second_order_prefix as (
+ // The paper has the condition on removal of prefix "bel" and "pel" as
+ // just "ajar" not "ajar..." but it seems that the latter must be what
+ // is intended so that e.g. "pelajaran" stems to "ajar" not "lajar".
+ // This change only affects a very small number of words (11 out of
+ // 64,587) and only for the better.
+ [substring] among (
+ 'per' 'pe' (delete $prefix=2 $measure-=1)
+ 'pelajar' (<-'ajar' $measure-=1)
+ 'ber' (delete $prefix=4 $measure-=1)
+ 'belajar' (<-'ajar' $prefix=4 $measure-=1)
+ 'be' KER (delete $prefix=4 $measure-=1)
+ )
+)
+
+define stem as (
+ $measure = 0
+ do ( repeat ( gopast vowel $measure+=1 ) )
+ $measure > 2
+ $prefix = 0
+ backwards (
+ do remove_particle
+ $measure > 2
+ do remove_possessive_pronoun
+ )
+ $measure > 2
+ test (
+ remove_first_order_prefix
+ do (
+ test ($measure > 2 backwards remove_suffix)
+ $measure > 2 remove_second_order_prefix
+ )
+ ) or (
+ do remove_second_order_prefix
+ do ($measure > 2 backwards remove_suffix)
+ )
+)
diff --git a/contrib/snowball/algorithms/irish.sbl b/contrib/snowball/algorithms/irish.sbl
new file mode 100644
index 0000000..0b1288a
--- /dev/null
+++ b/contrib/snowball/algorithms/irish.sbl
@@ -0,0 +1,151 @@
+routines (
+ R1 R2 RV
+ initial_morph
+ mark_regions
+ noun_sfx
+ deriv
+ verb_sfx
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+stringescapes {}
+
+/* Accented characters */
+
+stringdef a' '{U+00E1}' // a-acute
+stringdef e' '{U+00E9}' // e-acute
+stringdef i' '{U+00ED}' // i-acute
+stringdef o' '{U+00F3}' // o-acute
+stringdef u' '{U+00FA}' // u-acute
+
+define v 'aeiou{a'}{e'}{i'}{o'}{u'}'
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ gopast v setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define initial_morph as (
+ [substring] among (
+ 'h-' 'n-' 't-' //nAthair -> n-athair, but alone are problematic
+ (delete)
+
+ // verbs
+ 'd{'}'
+ (delete)
+ 'd{'}fh'
+ (<- 'f')
+ // other contractions
+ 'm{'}' 'b{'}'
+ (delete)
+
+ 'sh'
+ (<- 's')
+
+ 'mb'
+ (<- 'b')
+ 'gc'
+ (<- 'c')
+ 'nd'
+ (<- 'd')
+ 'bhf'
+ (<- 'f')
+ 'ng'
+ (<- 'g')
+ 'bp'
+ (<- 'p')
+ 'ts'
+ (<- 's')
+ 'dt'
+ (<- 't')
+
+ // Lenition
+ 'bh'
+ (<- 'b')
+ 'ch'
+ (<- 'c')
+ 'dh'
+ (<- 'd')
+ 'fh'
+ (<- 'f')
+ 'gh'
+ (<- 'g')
+ 'mh'
+ (<- 'm')
+ 'ph'
+ (<- 'p')
+ 'th'
+ (<- 't')
+ )
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define noun_sfx as (
+ [substring] among (
+ 'amh' 'eamh' 'abh' 'eabh'
+ 'aibh' 'ibh' 'aimh' 'imh'
+ 'a{i'}ocht' '{i'}ocht' 'a{i'}ochta' '{i'}ochta'
+ (R1 delete)
+ 'ire' 'ir{i'}' 'aire' 'air{i'}'
+ (R2 delete)
+ )
+ )
+ define deriv as (
+ [substring] among (
+ 'acht' 'eacht' 'ach' 'each' 'eacht{u'}il' 'eachta' 'acht{u'}il' 'achta'
+ (R2 delete) //siopadóireacht -> siopadóir but not poblacht -> pobl
+ 'arcacht' 'arcachta{i'}' 'arcachta'
+ (<- 'arc') // monarcacht -> monarc
+ 'gineach' 'gineas' 'ginis'
+ (<- 'gin')
+ 'grafa{i'}och' 'grafa{i'}ocht' 'grafa{i'}ochta' 'grafa{i'}ochta{i'}'
+ (<- 'graf')
+ 'paite' 'patach' 'pataigh' 'patacha'
+ (<- 'paite')
+ '{o'}ideach' '{o'}ideacha' '{o'}idigh'
+ (<- '{o'}id')
+ )
+ )
+ define verb_sfx as (
+ [substring] among (
+ 'imid' 'aimid' '{i'}mid' 'a{i'}mid'
+ 'faidh' 'fidh'
+ (RV delete)
+ 'ain'
+ 'eadh' 'adh'
+ '{a'}il'
+ 'tear' 'tar'
+ (R1 delete)
+ )
+ )
+)
+
+define stem as (
+ do initial_morph
+ do mark_regions
+ backwards (
+ do noun_sfx
+ do deriv
+ do verb_sfx
+ )
+)
diff --git a/contrib/snowball/algorithms/italian.sbl b/contrib/snowball/algorithms/italian.sbl
new file mode 100644
index 0000000..bf0c161
--- /dev/null
+++ b/contrib/snowball/algorithms/italian.sbl
@@ -0,0 +1,195 @@
+
+routines (
+ prelude postlude mark_regions
+ RV R1 R2
+ attached_pronoun
+ standard_suffix
+ verb_suffix
+ vowel_suffix
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v AEIO CG )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a' '{U+00E1}'
+stringdef a` '{U+00E0}'
+stringdef e' '{U+00E9}'
+stringdef e` '{U+00E8}'
+stringdef i' '{U+00ED}'
+stringdef i` '{U+00EC}'
+stringdef o' '{U+00F3}'
+stringdef o` '{U+00F2}'
+stringdef u' '{U+00FA}'
+stringdef u` '{U+00F9}'
+
+define v 'aeiou{a`}{e`}{i`}{o`}{u`}'
+
+define prelude as (
+ test repeat (
+ [substring] among(
+ '{a'}' (<- '{a`}')
+ '{e'}' (<- '{e`}')
+ '{i'}' (<- '{i`}')
+ '{o'}' (<- '{o`}')
+ '{u'}' (<- '{u`}')
+ 'qu' (<- 'qU')
+ '' (next)
+ )
+ )
+ repeat goto (
+ v [ ('u' ] v <- 'U') or
+ ('i' ] v <- 'I')
+ )
+)
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v (non-v gopast v) or (v gopast non-v) )
+ or
+ ( non-v (non-v gopast v) or (v next) )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'I' (<- 'i')
+ 'U' (<- 'u')
+ '' (next)
+ )
+
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define attached_pronoun as (
+ [substring] among(
+ 'ci' 'gli' 'la' 'le' 'li' 'lo'
+ 'mi' 'ne' 'si' 'ti' 'vi'
+ // the compound forms are:
+ 'sene' 'gliela' 'gliele' 'glieli' 'glielo' 'gliene'
+ 'mela' 'mele' 'meli' 'melo' 'mene'
+ 'tela' 'tele' 'teli' 'telo' 'tene'
+ 'cela' 'cele' 'celi' 'celo' 'cene'
+ 'vela' 'vele' 'veli' 'velo' 'vene'
+ )
+ among( (RV)
+ 'ando' 'endo' (delete)
+ 'ar' 'er' 'ir' (<- 'e')
+ )
+ )
+
+ define standard_suffix as (
+ [substring] among(
+
+ 'anza' 'anze' 'ico' 'ici' 'ica' 'ice' 'iche' 'ichi' 'ismo'
+ 'ismi' 'abile' 'abili' 'ibile' 'ibili' 'ista' 'iste' 'isti'
+ 'ist{a`}' 'ist{e`}' 'ist{i`}' 'oso' 'osi' 'osa' 'ose' 'mente'
+ 'atrice' 'atrici'
+ 'ante' 'anti' // Note 1
+ ( R2 delete )
+ 'azione' 'azioni' 'atore' 'atori'
+ ( R2 delete
+ try ( ['ic'] R2 delete )
+ )
+ 'logia' 'logie'
+ ( R2 <- 'log' )
+ 'uzione' 'uzioni' 'usione' 'usioni'
+ ( R2 <- 'u' )
+ 'enza' 'enze'
+ ( R2 <- 'ente' )
+ 'amento' 'amenti' 'imento' 'imenti'
+ ( RV delete )
+ 'amente' (
+ R1 delete
+ try (
+ [substring] R2 delete among(
+ 'iv' ( ['at'] R2 delete )
+ 'os' 'ic' 'abil'
+ )
+ )
+ )
+ 'it{a`}' (
+ R2 delete
+ try (
+ [substring] among(
+ 'abil' 'ic' 'iv' (R2 delete)
+ )
+ )
+ )
+ 'ivo' 'ivi' 'iva' 'ive' (
+ R2 delete
+ try ( ['at'] R2 delete ['ic'] R2 delete )
+ )
+ )
+ )
+
+ define verb_suffix as setlimit tomark pV for (
+ [substring] among(
+ 'ammo' 'ando' 'ano' 'are' 'arono' 'asse' 'assero' 'assi'
+ 'assimo' 'ata' 'ate' 'ati' 'ato' 'ava' 'avamo' 'avano' 'avate'
+ 'avi' 'avo' 'emmo' 'enda' 'ende' 'endi' 'endo' 'er{a`}' 'erai'
+ 'eranno' 'ere' 'erebbe' 'erebbero' 'erei' 'eremmo' 'eremo'
+ 'ereste' 'eresti' 'erete' 'er{o`}' 'erono' 'essero' 'ete'
+ 'eva' 'evamo' 'evano' 'evate' 'evi' 'evo' 'Yamo' 'iamo' 'immo'
+ 'ir{a`}' 'irai' 'iranno' 'ire' 'irebbe' 'irebbero' 'irei'
+ 'iremmo' 'iremo' 'ireste' 'iresti' 'irete' 'ir{o`}' 'irono'
+ 'isca' 'iscano' 'isce' 'isci' 'isco' 'iscono' 'issero' 'ita'
+ 'ite' 'iti' 'ito' 'iva' 'ivamo' 'ivano' 'ivate' 'ivi' 'ivo'
+ 'ono' 'uta' 'ute' 'uti' 'uto'
+
+ 'ar' 'ir' // but 'er' is problematical
+ (delete)
+ )
+ )
+
+ define AEIO 'aeio{a`}{e`}{i`}{o`}'
+ define CG 'cg'
+
+ define vowel_suffix as (
+ try (
+ [AEIO] RV delete
+ ['i'] RV delete
+ )
+ try (
+ ['h'] CG RV delete
+ )
+ )
+)
+
+define stem as (
+ do prelude
+ do mark_regions
+ backwards (
+ do attached_pronoun
+ do (standard_suffix or verb_suffix)
+ do vowel_suffix
+ )
+ do postlude
+)
+
+/*
+ Note 1: additions of 15 Jun 2005
+*/
+
diff --git a/contrib/snowball/algorithms/kraaij_pohlmann.sbl b/contrib/snowball/algorithms/kraaij_pohlmann.sbl
new file mode 100644
index 0000000..409bf0f
--- /dev/null
+++ b/contrib/snowball/algorithms/kraaij_pohlmann.sbl
@@ -0,0 +1,240 @@
+strings ( ch )
+integers ( p1 p2 )
+booleans ( Y_found stemmed GE_removed )
+
+routines (
+
+ R1 R2
+ C V VX
+ lengthen_V
+ Step_1 Step_2 Step_3 Step_4 Step_7
+ Step_6 Step_1c
+ Lose_prefix
+ Lose_infix
+ measure
+)
+
+externals ( stem )
+
+groupings ( v v_WX AOU AIOU )
+
+stringescapes {}
+
+define v 'aeiouy'
+define v_WX v + 'wx'
+define AOU 'aou'
+define AIOU 'aiou'
+
+backwardmode (
+
+ define R1 as ($p1 <= cursor)
+ define R2 as ($p2 <= cursor)
+
+ define V as test (v or 'ij')
+ define VX as test (next v or 'ij')
+ define C as test (not 'ij' non-v)
+
+ define lengthen_V as do (
+ non-v_WX [ (AOU] test (non-v or atlimit)) or
+ ('e'] test (non-v or atlimit
+ not AIOU
+ not (next AIOU non-v)))
+ ->ch insert ch
+ )
+
+ define Step_1 as
+ (
+ [substring] among (
+
+ '{'}s' (delete)
+ 's' (R1 not ('t' R1) C delete)
+ 'ies' (R1 <-'ie')
+ 'es'
+ (('ar' R1 C ] delete lengthen_V) or
+ ('er' R1 C ] delete) or
+ (R1 C <-'e'))
+
+ 'aus' (R1 V <-'au')
+ 'en' (('hed' R1 ] <-'heid') or
+ ('nd' delete) or
+ ('d' R1 C ] delete) or
+ ('i' or 'j' V delete) or
+ (R1 C delete lengthen_V))
+ 'nde' (<-'nd')
+ )
+ )
+
+ define Step_2 as
+ (
+ [substring] among (
+ 'je' (('{'}t' ] delete) or
+ ('et' ] R1 C delete) or
+ ('rnt' ] <-'rn') or
+ ('t' ] R1 VX delete) or
+ ('ink' ] <-'ing') or
+ ('mp' ] <-'m') or
+ ('{'}' ] R1 delete) or
+ (] R1 C delete))
+ 'ge' (R1 <-'g')
+ 'lijke'(R1 <-'lijk')
+ 'ische'(R1 <-'isch')
+ 'de' (R1 C delete)
+ 'te' (R1 <-'t')
+ 'se' (R1 <-'s')
+ 're' (R1 <-'r')
+ 'le' (R1 delete attach 'l' lengthen_V)
+ 'ene' (R1 C delete attach 'en' lengthen_V)
+ 'ieve' (R1 C <-'ief')
+ )
+ )
+
+ define Step_3 as
+ (
+ [substring] among (
+ 'atie' (R1 <-'eer')
+ 'iteit' (R1 delete lengthen_V)
+ 'heid'
+ 'sel'
+ 'ster' (R1 delete)
+ 'rder' (<-'r')
+ 'ing'
+ 'isme'
+ 'erij' (R1 delete lengthen_V)
+ 'arij' (R1 C <-'aar')
+ 'fie' (R2 delete attach 'f' lengthen_V)
+ 'gie' (R2 delete attach 'g' lengthen_V)
+ 'tst' (R1 C <-'t')
+ 'dst' (R1 C <-'d')
+ )
+ )
+
+ define Step_4 as
+ (
+ ( [substring] among (
+ 'ioneel' (R1 <-'ie')
+ 'atief' (R1 <-'eer')
+ 'baar' (R1 delete)
+ 'naar' (R1 V <-'n')
+ 'laar' (R1 V <-'l')
+ 'raar' (R1 V <-'r')
+ 'tant' (R1 <-'teer')
+ 'lijker'
+ 'lijkst' (R1 <-'lijk')
+ 'achtig'
+ 'achtiger'
+ 'achtigst'(R1 delete)
+ 'eriger'
+ 'erigst'
+ 'erig'
+ 'end' (R1 C delete lengthen_V)
+ )
+ )
+ or
+ ( [substring] among (
+ 'iger'
+ 'igst'
+ 'ig' (R1 C delete lengthen_V)
+ )
+ )
+ )
+
+ define Step_7 as
+ (
+ [substring] among (
+ 'kt' (<-'k')
+ 'ft' (<-'f')
+ 'pt' (<-'p')
+ )
+ )
+
+ define Step_6 as
+ (
+ [substring] among (
+ 'bb' (<-'b')
+ 'cc' (<-'c')
+ 'dd' (<-'d')
+ 'ff' (<-'f')
+ 'gg' (<-'g')
+ 'hh' (<-'h')
+ 'jj' (<-'j')
+ 'kk' (<-'k')
+ 'll' (<-'l')
+ 'mm' (<-'m')
+ 'nn' (<-'n')
+ 'pp' (<-'p')
+ 'qq' (<-'q')
+ 'rr' (<-'r')
+ 'ss' (<-'s')
+ 'tt' (<-'t')
+ 'vv' (<-'v')
+ 'ww' (<-'w')
+ 'xx' (<-'x')
+ 'zz' (<-'z')
+ 'v' (<-'f')
+ 'z' (<-'s')
+ )
+ )
+
+ define Step_1c as
+ (
+ [substring] among ( (R1 C)
+ 'd' (not ('n' R1) delete)
+ 't' (not ('h' R1) delete)
+ )
+ )
+)
+
+define Lose_prefix as (
+ ['ge'] test hop 3 (goto v goto non-v)
+ set GE_removed
+ delete
+)
+
+define Lose_infix as (
+ next
+ gopast (['ge']) test hop 3 (goto v goto non-v)
+ set GE_removed
+ delete
+)
+
+define measure as (
+ $p1 = limit
+ $p2 = limit
+ do(
+ repeat non-v atleast 1 ('ij' or v) non-v setmark p1
+ repeat non-v atleast 1 ('ij' or v) non-v setmark p2
+ )
+
+)
+define stem as (
+
+ unset Y_found
+ unset stemmed
+ do ( ['y'] <-'Y' set Y_found )
+ do repeat(goto (v ['y'])<-'Y' set Y_found )
+
+ measure
+
+ backwards (
+ do (Step_1 set stemmed )
+ do (Step_2 set stemmed )
+ do (Step_3 set stemmed )
+ do (Step_4 set stemmed )
+ )
+ unset GE_removed
+ do (Lose_prefix and measure)
+ backwards (
+ do (GE_removed Step_1c)
+ )
+ unset GE_removed
+ do (Lose_infix and measure)
+ backwards (
+ do (GE_removed Step_1c)
+ )
+ backwards (
+ do (Step_7 set stemmed )
+ do (stemmed or GE_removed Step_6)
+ )
+ do(Y_found repeat(goto (['Y']) <-'y'))
+)
+
diff --git a/contrib/snowball/algorithms/lithuanian.sbl b/contrib/snowball/algorithms/lithuanian.sbl
new file mode 100644
index 0000000..ff7fdb9
--- /dev/null
+++ b/contrib/snowball/algorithms/lithuanian.sbl
@@ -0,0 +1,373 @@
+externals ( stem )
+
+// escape symbols for substituting lithuanian characters
+stringescapes { }
+
+/* Special characters in Unicode Latin Extended-A */
+// ' nosine
+stringdef a' '{U+0105}' // Ä… a + ogonek
+stringdef e' '{U+0119}' // Ä™ e + ogonek
+stringdef i' '{U+012F}' // į i + ogonek
+stringdef u' '{U+0173}' // ų u + ogonek
+
+// . taskas
+stringdef e. '{U+0117}' // Ä— e + dot
+
+// - ilgoji
+stringdef u- '{U+016B}' // Å« u + macron
+
+// * varnele
+stringdef c* '{U+010D}' // Ä c + caron (haÄek)
+stringdef s* '{U+0161}' // Å¡ s + caron (haÄek)
+stringdef z* '{U+017E}' // ž z + caron (haÄek)
+
+// [C](VC)^m[V|C]
+// definitions of variables for
+// p1 - position of m = 0
+integers ( p1 )
+
+// groupings
+// v - lithuanian vowels
+groupings ( v )
+
+// v - all lithuanian vowels
+define v 'aeiyou{a'}{e'}{i'}{u'}{e.}{u-}'
+
+// all lithuanian stemmer routines: 4 steps
+routines (
+ step2 R1 step1 fix_chdz fix_gd fix_conflicts
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define step1 as (
+ setlimit tomark p1 for ([substring]) R1 among (
+ // Daiktavardžiai (Nouns)
+ // I linksniuotÄ— (declension I)
+ 'as' 'ias' 'is' 'ys' // vyras, kelias, brolis, gaidys
+ 'o' 'io' // vyro, kelio
+ 'ui' 'iui' // vyrui, keliui
+ '{a'}' 'i{a'}' '{i'}' // vyrą, kelią, brolį
+ 'u' 'iu' // vyru, keliu
+ 'e' 'yje' // vyre, kelyje
+ 'y' 'au' 'i' // kely, brolau, broli,
+ 'an' // nusižengiman
+
+ 'ai' 'iai' // vyrai, keliai
+ '{u'}' 'i{u'}' // vyrų, kelių
+ 'ams' 'am' // vyrams, vyram
+ 'iams' 'iam' // broliams, broliam
+ 'us' 'ius' // vyrus, brolius
+ 'ais' 'iais' // vyrais, keliais
+ 'uose' 'iuose' 'uos' 'iuos' // vyruose, keliuose, vyruos, keliuos
+ 'uosna' 'iuosna' // vyruosna, keliuosna
+ 'ysna' // žutysna
+
+ 'asis' 'aisi' // sukimasis, sukimaisi
+ 'osi' '{u'}si' // sukimosi, sukimųsi
+ 'uisi' // sukimuisi
+ '{a'}si' // sukimÄ…si
+ 'usi' // sukimusi
+ 'esi' // sukimesi
+
+ 'uo' // mÄ—nuo
+
+
+ // II linksniuote (declension II)
+ 'a' 'ia' // galva, vysnios
+ 'os' 'ios' // galvos, vysnios
+ 'oj' 'oje' 'ioje' // galvoje, vysnioje
+ 'osna' 'iosna' // galvosna, vyšniosna
+ 'om' 'oms' 'ioms' // galvoms, vysnioms
+ 'omis' 'iomis' // galvomis, vysniomis
+ 'ose' 'iose' // galvose, vysniose
+ 'on' 'ion' // galvon, vyšnion
+
+
+ // III linksniuote (declension III)
+ '{e.}' // gervÄ—
+ '{e.}s' // gervÄ—s
+ 'ei' // gervei
+ '{e'}' // gervÄ™
+ '{e.}j' '{e.}je' // gervÄ—j, gervÄ—je
+ '{e.}ms' // gervÄ—ms
+ 'es' // gerves
+ '{e.}mis' // gervÄ—mis
+ '{e.}se' // gervÄ—se
+ '{e.}sna' // gervÄ—sna
+ '{e.}n' // žydaitėn
+
+
+ // IV linksniuote (declension IV)
+ 'aus' 'iaus' // sÅ«naus, skaiÄiaus
+ 'umi' 'iumi' // sÅ«numi, skaiÄiumi
+ 'uje' 'iuje' // sÅ«nuje, skaiÄiuje
+ 'iau' // skaiÄiau
+
+ '{u-}s' // sūnūs
+ 'ums' // sūnums
+ 'umis' // sūnumis
+ 'un' 'iun' // sūnun, administratoriun
+
+
+ // V linksniuote (declension V)
+ 'ies' 'ens' 'enio' 'ers' // avies, vandens, sesers
+ 'eniui' 'eriai' // vandeniui, eriai
+ 'en{i'}' 'er{i'}' // vandenį, seserį
+ 'imi' 'eniu' 'erimi' 'eria' // avimi, vandeniu, seserimi, seseria
+ 'enyje' 'eryje' // vandenyje, seseryje
+ 'ie' 'enie' 'erie' // avie, vandenie, seserie
+
+ 'enys' 'erys' // vandenys, seserys
+ // 'en{u'}' konfliktas su 'žandenų' 'antenų'
+ 'er{u'}' // seserų
+ 'ims' 'enims' 'erims' // avims, vandemins, seserims
+ 'enis' // vandenis
+ 'imis' // žebenkštimis
+ 'enimis' // vandenimis
+ 'yse' 'enyse' 'eryse' // avyse, vandenyse, seseryse
+
+
+ // Būdvardžiai (Adjectives)
+ // (i)a linksniuotÄ—
+ 'iem' 'iems' // geriem, geriems
+ 'ame' 'iame' // naujame, mediniame
+
+
+ // Veiksmažodžiai (Verbs)
+ // TiesioginÄ— nuosaka (indicative mood)
+ // esamasis laikas (present tense)
+ // (i)a asmenuotÄ— (declension (i)a)
+ 'uosi' 'iuosi' // dirbuosi, traukiuosi
+ 'iesi' // dirbiesi
+ 'asi' 'iasi' // dirbasi, traukiasi
+ 'am{e.}s' 'iam{e.}s' // dirbamÄ—s, traukiamÄ—s
+ 'at' 'ate' 'iat' 'iate' // dirbat, dirbate, ariat, traukiate
+ 'at{e.}s' 'iat{e.}s' // dirbatÄ—s, traukiatÄ—s
+
+ // i asmenuotÄ— (declension i)
+ 'isi' // tikisi
+ 'im' // mylim
+ // 'ime' konfliktassu daiktavardžiu vietininku, pvz. 'gėrime'
+ 'im{e.}s' // tikimÄ—s
+ 'it' 'ite' // mylit, mylite, tikitÄ—s
+ // 'it{e.}s' konfliktas su priesaga ir dgs. vardininko galūne -ait-ės pvz. žydaitės
+
+ // o asmenuotÄ— (declension o)
+ 'ome' // mokome
+ 'ot' 'ote' // mokot, mokote
+
+ // būtasis laikas
+ // o asmenuotÄ— (declension o)
+ '{e.}jo' '{e.}josi' // tikÄ—jo, tikÄ—josi
+ 'ot{e.}s' // tikÄ—jotÄ—s/bijotÄ—s
+
+ // Ä— asmenuotÄ— (declension Ä—)
+ 'eisi' // mokeisi
+ '{e.}si' // mokÄ—si
+ '{e.}m' '{e.}me' // mokÄ—m, mokÄ—me
+ '{e.}m{e.}s' // mokÄ—mÄ—s
+ '{e.}t' '{e.}te' // mokÄ—t, mokÄ—te
+ '{e.}t{e.}s' // mokÄ—tÄ—s
+
+ // būtasis dažninis laikas (frequentative past tense)
+ 'ausi' // mokydavausi
+ 'om{e.}s' // mokydavomÄ—s/bijomÄ—s
+
+
+ // būsimasis laikas (future tense)
+ 'siu' 'siuosi' // dirbsiu, mokysiuosi
+ 'si' 'siesi' // dirbsi, dirbsiesi
+ 's' 'ysis' // dirbs, mokysis
+ 'sim' 'sime' // dirbsim, dirbsime
+ 'sit' 'site' // gersit, gersite
+
+ // tariamoji nuosaka (subjunctive mood)
+ '{c*}iau' '{c*}iausi' // dirbÄiau
+ 'tum' 'tumei' // dirbtum, dirbtumei
+ 'tumeis' 'tumeisi' // mokytumeis, mokytumeisi
+ // 't{u'}' nes blogai batutų -> batų
+ 't{u'}si' // mokytųsi
+ // 'tume' konfliktas su 'Å¡ventume'
+ 'tum{e.}m' // dirbtumÄ—m
+ 'tum{e.}me' // dirbtumÄ—me
+ 'tum{e.}m{e.}s' // mokytumÄ—mÄ—s
+ 'tute' 'tum{e.}t' // dirbtute, dirbtumÄ—t
+ 'tum{e.}te' // dirbtumÄ—te
+ 'tum{e.}t{e.}s' // mokytumÄ—tÄ—s
+
+ // liepiamoji nuosaka (imperative mood)
+ 'k' 'ki' // dirbk, dirbki, mokykis
+ // 'kis' konfliktas viln-išk-is
+ // 'kime' konfliktas, nes pirkime
+ 'kim{e.}s' // mokykimÄ—s
+
+ // bendratis (infinitive)
+ 'uoti' 'iuoti' // meluoti, dygsniuoti
+ 'auti' 'iauti' // draugauti, girtuokliauti
+ 'oti' 'ioti' // dovanoti, meškerioti
+ '{e.}ti' // auklÄ—ti
+ 'yti' // akyti
+ 'inti' // auginti
+ 'in{e.}ti' // blusinÄ—ti
+ 'enti' // gyventi
+ 'tel{e.}ti' // bumbtelÄ—ti
+ 'ter{e.}ti' // bumbterÄ—ti
+
+ 'ti' // skalbti
+ // 'tis' konfliktas, nes rytme-tis -> rytme
+
+ // dalyviai (participles)
+ '{a'}s' 'i{a'}s' '{i'}s' // dirbąs, žaidžiąs, gulįs
+ 't{u'}s' // suktųs -> suk
+ 'sim{e.}s' // suksimÄ—s
+ 'sit{e.}s' // suksitÄ—s
+ 'kite' // supkite
+ )
+
+ delete
+ )
+
+ define step2 as repeat (
+ setlimit tomark p1 for ([substring]) among (
+ // daiktavardziu priesagos (Noun suffixes)
+
+ // budvardziu priesagos (Adjective suffixes)
+ // 'in' // konfliktas su 'augintinis' ir 'akiniais' // lauk-in-is
+ 'ing' // tvark-ing-as
+ 'i{s*}k' // lenk-išk-as
+ '{e.}t' // dem-Ä—t-as
+ 'ot' // garban-ot-as
+ 'uot' 'iuot' // lang-uot-as, akin-iuot-as
+ // 'tin', nes augintinis // dirb-tin-is
+ // 'ut', nes batutas, degutas etc. // maž-ut-is
+ 'yt' // maž-yt-is
+ 'iuk' // maž-iuk-as
+ 'iul' // maž-ul-is
+ '{e.}l' // maž-ėl-is
+ 'yl' // maž-yl-is
+ 'u{c*}iuk' // maž-uÄiuk-as
+ 'uliuk' // maž-uliuk-as
+ 'ut{e.}ait' // maž-utėlait-is
+ 'ok' // did-ok-as
+ 'iok' // viÅ¡Ä-iok-as
+ 'sv' '{s*}v' 'zgan' // sal-sv-as, pilk-Å¡v-as, bal-zgan-as
+ 'op' 'iop' // dvej-op-as, viener-iop-as
+ 'ain' // apval-ain-as
+ 'yk{s*}t' 'yk{s*}{c*}' // ten-ykÅ¡t-is, vakar-ykÅ¡Ä-ias
+
+ // laisniai
+ 'esn' // did-esn-is
+ 'aus' 'iaus' // nauj-aus-ias, ger-iaus-ias
+
+ // ivardziuotiniai budvardziai (Pronominal adjectives)
+ // vyriska gimine (Male gender)
+ 'ias' // žaliasis
+ 'oj' 'ioj' // gerojo, žaliojo
+ 'aj' 'iaj' // gerajam, žaliajam
+ '{a'}j' 'i{a'}j' // garąjį, žaliąjį
+ 'uoj' 'iuoj' // geruoju, žaliuoju
+ 'iej' // gerieji
+ '{u'}j' 'i{u'}j' // gerųjų, žaliųjų
+ 'ies' // geriesiems
+ 'uos' 'iuos' // geruosius, žaliuosius
+ 'ais' 'iais' // geraisiais, žaliaisiais
+
+ // moteriska gimine (Female gender)
+ 'os' 'ios' // gerosios, žaliosios
+ '{a'}s' 'i{a'}s' // gerąsios, žaliąsias
+
+ // būtasis dažninis laikas (frequentative past tense)
+ 'dav' // ei-dav-o
+
+ // dalyvių priesagos (particple suffix)
+ 'ant' 'iant'
+ 'int' // tur-int-is
+ '{e.}j' // tur-Ä—j-o
+ '{e'}' //
+ '{e.}j{e'}'
+ '{e'}s' // dirb-ęs-is
+
+ 'siant' // dirb-siant
+
+ // pusdalyviai (participle)
+ 'dam' // bÄ—g-dam-as
+
+ 'auj' // Å«kinink-auj-a
+ 'jam'
+ 'iau'
+ 'am' // baiminim-ams-i
+ )
+
+ delete
+ )
+
+ define fix_conflicts as (
+ [substring] among (
+ // 'lietuvaite' -> 'lietuvaitÄ—', konfliktas su 'myl-ite'
+ 'aite' (<-'ait{e.}')
+ // 'lietuvaitÄ—s' -> 'lietuvaitÄ—', konfliktas su 'myl-itÄ—s'
+ 'ait{e.}s' (<-'ait{e.}')
+
+ // ''Å«s-uotÄ—s' -> 'Å«s-uotÄ—', konfliktas 'mokotÄ—s'
+ 'uot{e.}s' (<-'uot{e.}')
+ // ''Å«s-uote' -> 'Å«s-uotÄ—', konfliktas 'mokote'
+ 'uote' (<-'uot{e.}')
+
+ // 'žerėjime' -> 'žėrėjimas', konfliktas su 'žais-ime'
+ '{e.}jime' (<-'{e.}jimas')
+
+ // 'žvilgesiu' -> 'žvilgesys', konfliktas su 'dirb-siu'
+ 'esiu' (<-'esys')
+ // 'duobkasiu' -> 'duobkasys', konfliktas su 'pakasiu'
+ 'asius' (<-'asys')
+
+ // 'žioravime' -> 'žioravimas', konfliktas su 'myl-ime'
+ 'avime' (<-'avimas')
+ 'ojime' (<-'ojimas')
+
+ // 'advokatÄ—s' -> 'advokatÄ—', konfliktas su 'dirb-atÄ—s'
+ 'okat{e.}s' (<-'okat{e.}')
+ // 'advokate' -> 'advokatÄ—', konfliktas su 'dirb-ate'
+ 'okate' (<-'okat{e.}')
+ )
+ )
+
+ define fix_chdz as (
+ [substring] among (
+ '{c*}' (<-'t')
+ 'd{z*}' (<-'d')
+ )
+ )
+
+ define fix_gd as (
+ [substring] among (
+ 'gd' (<-'g')
+ // '{e.}k' (<-'{e.}g')
+ )
+ )
+
+)
+
+define stem as (
+
+ $p1 = limit
+
+ do (
+ // priešdėlis 'a' ilgeniuose nei 6 raidės žodžiuose, pvz. 'a-liejus'.
+ try (test 'a' $(len > 6) hop 1)
+
+ gopast v gopast non-v setmark p1
+ )
+
+ backwards (
+ do fix_conflicts
+ do step1
+ do fix_chdz
+ do step2
+ do fix_chdz
+ do fix_gd
+ )
+
+)
diff --git a/contrib/snowball/algorithms/lovins.sbl b/contrib/snowball/algorithms/lovins.sbl
new file mode 100644
index 0000000..3f69f15
--- /dev/null
+++ b/contrib/snowball/algorithms/lovins.sbl
@@ -0,0 +1,208 @@
+
+stringescapes {}
+
+routines (
+ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z AA BB CC
+
+ endings
+
+ undouble respell
+)
+
+externals ( stem )
+
+backwardmode (
+
+ /* Lovins' conditions A, B ... CC, as given in her Appendix B, where
+ a test for a two letter prefix ('test hop 2') is implicitly
+ assumed. Note that 'e' next 'u' corresponds to her u*e because
+ Snowball is scanning backwards. */
+
+ define A as ( hop 2 )
+ define B as ( hop 3 )
+ define C as ( hop 4 )
+ define D as ( hop 5 )
+ define E as ( test hop 2 not 'e' )
+ define F as ( test hop 3 not 'e' )
+ define G as ( test hop 3 'f' )
+ define H as ( test hop 2 't' or 'll' )
+ define I as ( test hop 2 not 'o' not 'e' )
+ define J as ( test hop 2 not 'a' not 'e' )
+ define K as ( test hop 3 'l' or 'i' or ('e' next 'u') )
+ define L as ( test hop 2 not 'u' not 'x' not ('s' not 'o') )
+ define M as ( test hop 2 not 'a' not 'c' not 'e' not 'm' )
+ define N as ( test hop 3 ( hop 2 not 's' or hop 2 ) )
+ define O as ( test hop 2 'l' or 'i' )
+ define P as ( test hop 2 not 'c' )
+ define Q as ( test hop 2 test hop 3 not 'l' not 'n' )
+ define R as ( test hop 2 'n' or 'r' )
+ define S as ( test hop 2 'dr' or ('t' not 't') )
+ define T as ( test hop 2 's' or ('t' not 'o') )
+ define U as ( test hop 2 'l' or 'm' or 'n' or 'r' )
+ define V as ( test hop 2 'c' )
+ define W as ( test hop 2 not 's' not 'u' )
+ define X as ( test hop 2 'l' or 'i' or ('e' next 'u') )
+ define Y as ( test hop 2 'in' )
+ define Z as ( test hop 2 not 'f' )
+ define AA as ( test hop 2 among ( 'd' 'f' 'ph' 'th' 'l' 'er' 'or'
+ 'es' 't' ) )
+ define BB as ( test hop 3 not 'met' not 'ryst' )
+ define CC as ( test hop 2 'l' )
+
+
+ /* The system of endings, as given in Appendix A. */
+
+ define endings as (
+ [substring] among(
+ 'alistically' B 'arizability' A 'izationally' B
+
+ 'antialness' A 'arisations' A 'arizations' A 'entialness' A
+
+ 'allically' C 'antaneous' A 'antiality' A 'arisation' A
+ 'arization' A 'ationally' B 'ativeness' A 'eableness' E
+ 'entations' A 'entiality' A 'entialize' A 'entiation' A
+ 'ionalness' A 'istically' A 'itousness' A 'izability' A
+ 'izational' A
+
+ 'ableness' A 'arizable' A 'entation' A 'entially' A
+ 'eousness' A 'ibleness' A 'icalness' A 'ionalism' A
+ 'ionality' A 'ionalize' A 'iousness' A 'izations' A
+ 'lessness' A
+
+ 'ability' A 'aically' A 'alistic' B 'alities' A
+ 'ariness' E 'aristic' A 'arizing' A 'ateness' A
+ 'atingly' A 'ational' B 'atively' A 'ativism' A
+ 'elihood' E 'encible' A 'entally' A 'entials' A
+ 'entiate' A 'entness' A 'fulness' A 'ibility' A
+ 'icalism' A 'icalist' A 'icality' A 'icalize' A
+ 'ication' G 'icianry' A 'ination' A 'ingness' A
+ 'ionally' A 'isation' A 'ishness' A 'istical' A
+ 'iteness' A 'iveness' A 'ivistic' A 'ivities' A
+ 'ization' F 'izement' A 'oidally' A 'ousness' A
+
+ 'aceous' A 'acious' B 'action' G 'alness' A
+ 'ancial' A 'ancies' A 'ancing' B 'ariser' A
+ 'arized' A 'arizer' A 'atable' A 'ations' B
+ 'atives' A 'eature' Z 'efully' A 'encies' A
+ 'encing' A 'ential' A 'enting' C 'entist' A
+ 'eously' A 'ialist' A 'iality' A 'ialize' A
+ 'ically' A 'icance' A 'icians' A 'icists' A
+ 'ifully' A 'ionals' A 'ionate' D 'ioning' A
+ 'ionist' A 'iously' A 'istics' A 'izable' E
+ 'lessly' A 'nesses' A 'oidism' A
+
+ 'acies' A 'acity' A 'aging' B 'aical' A
+ 'alist' A 'alism' B 'ality' A 'alize' A
+ 'allic'BB 'anced' B 'ances' B 'antic' C
+ 'arial' A 'aries' A 'arily' A 'arity' B
+ 'arize' A 'aroid' A 'ately' A 'ating' I
+ 'ation' B 'ative' A 'ators' A 'atory' A
+ 'ature' E 'early' Y 'ehood' A 'eless' A
+ 'elity' A 'ement' A 'enced' A 'ences' A
+ 'eness' E 'ening' E 'ental' A 'ented' C
+ 'ently' A 'fully' A 'ially' A 'icant' A
+ 'ician' A 'icide' A 'icism' A 'icist' A
+ 'icity' A 'idine' I 'iedly' A 'ihood' A
+ 'inate' A 'iness' A 'ingly' B 'inism' J
+ 'inity'CC 'ional' A 'ioned' A 'ished' A
+ 'istic' A 'ities' A 'itous' A 'ively' A
+ 'ivity' A 'izers' F 'izing' F 'oidal' A
+ 'oides' A 'otide' A 'ously' A
+
+ 'able' A 'ably' A 'ages' B 'ally' B
+ 'ance' B 'ancy' B 'ants' B 'aric' A
+ 'arly' K 'ated' I 'ates' A 'atic' B
+ 'ator' A 'ealy' Y 'edly' E 'eful' A
+ 'eity' A 'ence' A 'ency' A 'ened' E
+ 'enly' E 'eous' A 'hood' A 'ials' A
+ 'ians' A 'ible' A 'ibly' A 'ical' A
+ 'ides' L 'iers' A 'iful' A 'ines' M
+ 'ings' N 'ions' B 'ious' A 'isms' B
+ 'ists' A 'itic' H 'ized' F 'izer' F
+ 'less' A 'lily' A 'ness' A 'ogen' A
+ 'ward' A 'wise' A 'ying' B 'yish' A
+
+ 'acy' A 'age' B 'aic' A 'als'BB
+ 'ant' B 'ars' O 'ary' F 'ata' A
+ 'ate' A 'eal' Y 'ear' Y 'ely' E
+ 'ene' E 'ent' C 'ery' E 'ese' A
+ 'ful' A 'ial' A 'ian' A 'ics' A
+ 'ide' L 'ied' A 'ier' A 'ies' P
+ 'ily' A 'ine' M 'ing' N 'ion' Q
+ 'ish' C 'ism' B 'ist' A 'ite'AA
+ 'ity' A 'ium' A 'ive' A 'ize' F
+ 'oid' A 'one' R 'ous' A
+
+ 'ae' A 'al'BB 'ar' X 'as' B
+ 'ed' E 'en' F 'es' E 'ia' A
+ 'ic' A 'is' A 'ly' B 'on' S
+ 'or' T 'um' U 'us' V 'yl' R
+ '{'}s' A 's{'}' A
+
+ 'a' A 'e' A 'i' A 'o' A
+ 's' W 'y' B
+
+ (delete)
+ )
+ )
+
+ /* Undoubling is rule 1 of appendix C. */
+
+ define undouble as (
+ test substring among ('bb' 'dd' 'gg' 'll' 'mm' 'nn' 'pp' 'rr' 'ss'
+ 'tt')
+ [next] delete
+ )
+
+ /* The other appendix C rules can be done together. */
+
+ define respell as (
+ [substring] among (
+ 'iev' (<-'ief')
+ 'uct' (<-'uc')
+ 'umpt' (<-'um')
+ 'rpt' (<-'rb')
+ 'urs' (<-'ur')
+ 'istr' (<-'ister')
+ 'metr' (<-'meter')
+ 'olv' (<-'olut')
+ 'ul' (not 'a' not 'i' not 'o' <-'l')
+ 'bex' (<-'bic')
+ 'dex' (<-'dic')
+ 'pex' (<-'pic')
+ 'tex' (<-'tic')
+ 'ax' (<-'ac')
+ 'ex' (<-'ec')
+ 'ix' (<-'ic')
+ 'lux' (<-'luc')
+ 'uad' (<-'uas')
+ 'vad' (<-'vas')
+ 'cid' (<-'cis')
+ 'lid' (<-'lis')
+ 'erid' (<-'eris')
+ 'pand' (<-'pans')
+ 'end' (not 's' <-'ens')
+ 'ond' (<-'ons')
+ 'lud' (<-'lus')
+ 'rud' (<-'rus')
+ 'her' (not 'p' not 't' <-'hes')
+ 'mit' (<-'mis')
+ 'ent' (not 'm' <-'ens')
+ /* 'ent' was 'end' in the 1968 paper - a typo. */
+ 'ert' (<-'ers')
+ 'et' (not 'n' <-'es')
+ 'yt' (<-'ys')
+ 'yz' (<-'ys')
+ )
+ )
+)
+
+define stem as (
+
+ backwards (
+ do endings
+ do undouble
+ do respell
+ )
+)
+
diff --git a/contrib/snowball/algorithms/nepali.sbl b/contrib/snowball/algorithms/nepali.sbl
new file mode 100644
index 0000000..d388748
--- /dev/null
+++ b/contrib/snowball/algorithms/nepali.sbl
@@ -0,0 +1,92 @@
+/*
+ * Authors:
+ * - Ingroj Shrestha <ing.stha@gmail.com>, Nepali NLP Group
+ * - Oleg Bartunov <obartunov@gmail.com>, Postgres Professional Ltd.
+ * - Shreeya Singh Dhakal, Nepali NLP Group
+ */
+
+routines (
+ remove_category_1
+ check_category_2
+ remove_category_2
+ remove_category_3
+)
+
+stringescapes {}
+
+stringdef dsc '{U+0901}' // DEVANAGARI_SIGN_CANDRABINDU
+stringdef dsa '{U+0902}' // DEVANAGARI_SIGN_ANUSVARA
+stringdef dli '{U+0907}' // DEVANAGARI_LETTER_I
+stringdef dlii '{U+0908}' // DEVANAGARI_LETTER_II
+stringdef dle '{U+090F}' // DEVANAGARI_LETTER_E
+stringdef dlka '{U+0915}' // DEVANAGARI_LETTER_KA
+stringdef dlkha '{U+0916}' // DEVANAGARI_LETTER_KHA
+stringdef dlg '{U+0917}' // DEVANAGARI_LETTER_GA
+stringdef dlc '{U+091B}' // DEVANAGARI_LETTER_CHA
+stringdef dlta '{U+0924}' // DEVANAGARI_LETTER_TA
+stringdef dltha '{U+0925}' // DEVANAGARI_LETTER_THA
+stringdef dld '{U+0926}' // DEVANAGARI_LETTER_DA
+stringdef dln '{U+0928}' // DEVANAGARI_LETTER_NA
+stringdef dlpa '{U+092A}' // DEVANAGARI_LETTER_PA
+stringdef dlpha '{U+092B}' // DEVANAGARI_LETTER_PHA
+stringdef dlb '{U+092D}' // DEVANAGARI_LETTER_BHA
+stringdef dlm '{U+092E}' // DEVANAGARI_LETTER_MA
+stringdef dly '{U+092F}' // DEVANAGARI_LETTER_YA
+stringdef dlr '{U+0930}' // DEVANAGARI_LETTER_RA
+stringdef dll '{U+0932}' // DEVANAGARI_LETTER_LA
+stringdef dlv '{U+0935}' // DEVANAGARI_LETTER_VA
+stringdef dls '{U+0938}' // DEVANAGARI_LETTER_SA
+stringdef dlh '{U+0939}' // DEVANAGARI_LETTER_HA
+stringdef dvsaa '{U+093E}' // DEVANAGARI_VOWEL_SIGN_AA
+stringdef dvsi '{U+093F}' // DEVANAGARI_VOWEL_SIGN_I
+stringdef dvsii '{U+0940}' // DEVANAGARI_VOWEL_SIGN_II
+stringdef dvsu '{U+0941}' // DEVANAGARI_VOWEL_SIGN_U
+stringdef dvsuu '{U+0942}' // DEVANAGARI_VOWEL_SIGN_UU
+stringdef dvse '{U+0947}' // DEVANAGARI_VOWEL_SIGN_E
+stringdef dvsai '{U+0948}' // DEVANAGARI_VOWEL_SIGN_AI
+stringdef dvso '{U+094B}' // DEVANAGARI_VOWEL_SIGN_O
+stringdef dvsau '{U+094C}' // DEVANAGARI_VOWEL_SIGN_AU
+stringdef dsv '{U+094D}' // DEVANAGARI_SIGN_VIRAMA
+
+externals ( stem )
+backwardmode (
+ define remove_category_1 as(
+ [substring] among (
+ '{dlm}{dvsaa}{dlr}{dsv}{dlpha}{dlta}' '{dld}{dsv}{dlv}{dvsaa}{dlr}{dvsaa}' '{dls}{dsc}{dlg}{dvsai}' '{dls}{dsa}{dlg}'
+ '{dls}{dsc}{dlg}' '{dll}{dvsaa}{dli}' '{dll}{dvsaa}{dlii}' '{dlpa}{dlc}{dvsi}'
+ '{dll}{dvse}' '{dlr}{dlta}' '{dlm}{dvsai}' '{dlm}{dvsaa}'
+ (delete)
+ '{dlka}{dvso}' '{dlka}{dvsaa}' '{dlka}{dvsi}' '{dlka}{dvsii}' '{dlka}{dvsai}'(('{dle}' or '{dvse}' ()) or delete)
+ )
+ )
+
+ define check_category_2 as(
+ [substring] among(
+ '{dsc}' '{dsa}' '{dvsai}'
+ )
+ )
+
+ define remove_category_2 as (
+ [substring] among(
+ '{dsc}' '{dsa}' ('{dly}{dvsau}' or '{dlc}{dvsau}' or '{dln}{dvsau}' or '{dltha}{dvse}' delete)
+ '{dvsai}' ('{dlta}{dsv}{dlr}' delete)
+ )
+ )
+
+ define remove_category_3 as(
+ [substring] among(
+ '{dltha}{dvsi}{dli}{dls}{dsv}' '{dlh}{dvsu}{dln}{dvse}{dlc}' '{dlh}{dvsu}{dln}{dsv}{dlc}' '{dln}{dvse}{dlc}{dls}{dsv}' '{dln}{dvse}{dlc}{dln}{dsv}' '{dli}{dle}{dlka}{dvsii}' '{dli}{dle}{dlka}{dvsaa}' '{dli}{dle}{dlka}{dvso}' '{dvsi}{dle}{dlka}{dvsii}' '{dvsi}{dle}{dlka}{dvsaa}' '{dvsi}{dle}{dlka}{dvso}' '{dli}{dlc}{dln}{dsv}' '{dvsi}{dlc}{dln}{dsv}' '{dli}{dlc}{dls}{dsv}' '{dvsi}{dlc}{dls}{dsv}' '{dle}{dlc}{dln}{dsv}' '{dvse}{dlc}{dln}{dsv}' '{dle}{dlc}{dls}{dsv}' '{dvse}{dlc}{dls}{dsv}' '{dlc}{dvsi}{dln}{dsv}' '{dlc}{dvse}{dls}{dsv}' '{dlc}{dsv}{dly}{dvsau}' '{dltha}{dvsi}{dln}{dsv}' '{dltha}{dvsi}{dly}{dvso}' '{dltha}{dvsi}{dly}{dvsau}' '{dltha}{dvsi}{dls}{dsv}' '{dltha}{dsv}{dly}{dvso}' '{dltha}{dsv}{dly}{dvsau}' '{dld}{dvsi}{dly}{dvso}' '{dld}{dvse}{dlkha}{dvsi}' '{dld}{dvse}{dlkha}{dvsii}' '{dll}{dvsaa}{dln}{dsv}' '{dlm}{dvsaa}{dltha}{dvsi}' '{dln}{dvse}{dlka}{dvsai}' '{dln}{dvse}{dlka}{dvsaa}' '{dln}{dvse}{dlka}{dvso}' '{dln}{dvse}{dlc}{dvsau}' '{dlh}{dvso}{dls}{dsv}' '{dli}{dln}{dsv}{dlc}' '{dvsi}{dln}{dsv}{dlc}' '{dln}{dvse}{dlc}{dvsu}' '{dli}{dlc}{dvsau}' '{dvsi}{dlc}{dvsau}' '{dli}{dls}{dsv}' '{dvsi}{dls}{dsv}' '{dvsi}{dly}{dvso}' '{dli}{dly}{dvso}' '{dle}{dlka}{dvsaa}' '{dvse}{dlka}{dvsaa}' '{dle}{dlka}{dvsii}' '{dvse}{dlka}{dvsii}' '{dle}{dlka}{dvsai}' '{dvse}{dlka}{dvsai}' '{dle}{dlka}{dvso}' '{dvse}{dlka}{dvso}' '{dle}{dlc}{dvsu}' '{dvse}{dlc}{dvsu}' '{dle}{dlc}{dvsau}' '{dvse}{dlc}{dvsau}' '{dlc}{dln}{dsv}' '{dlc}{dls}{dsv}' '{dltha}{dvsi}{dle}' '{dlpa}{dlr}{dsv}' '{dlb}{dly}{dvso}' '{dlh}{dlr}{dvsu}' '{dlh}{dlr}{dvsuu}' '{dvsi}{dld}{dvsaa}' '{dli}{dld}{dvsaa}' '{dvsi}{dld}{dvso}' '{dli}{dld}{dvso}' '{dvsi}{dld}{dvsai}' '{dli}{dld}{dvsai}' '{dln}{dvse}{dlc}' '{dli}{dlc}' '{dvsi}{dlc}' '{dle}{dlc}' '{dvse}{dlc}' '{dlc}{dvsu}' '{dlc}{dvse}' '{dlc}{dvsau}' '{dltha}{dvsii}' '{dltha}{dvse}' '{dld}{dvsaa}' '{dld}{dvsii}' '{dld}{dvsai}' '{dld}{dvso}' '{dln}{dvsu}' '{dln}{dvse}' '{dly}{dvso}' '{dly}{dvsau}' '{dlc}'
+ (delete)
+ )
+ )
+
+)
+
+define stem as (
+ backwards (
+ do remove_category_1
+ do (
+ repeat (do (check_category_2 and remove_category_2) remove_category_3)
+ )
+ )
+)
diff --git a/contrib/snowball/algorithms/norwegian.sbl b/contrib/snowball/algorithms/norwegian.sbl
new file mode 100644
index 0000000..39f4aff
--- /dev/null
+++ b/contrib/snowball/algorithms/norwegian.sbl
@@ -0,0 +1,80 @@
+routines (
+ mark_regions
+ main_suffix
+ consonant_pair
+ other_suffix
+)
+
+externals ( stem )
+
+integers ( p1 x )
+
+groupings ( v s_ending )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef ae '{U+00E6}'
+stringdef ao '{U+00E5}'
+stringdef o/ '{U+00F8}'
+
+define v 'aeiouy{ae}{ao}{o/}'
+
+define s_ending 'bcdfghjlmnoprtvyz'
+
+define mark_regions as (
+
+ $p1 = limit
+
+ test ( hop 3 setmark x )
+ goto v gopast non-v setmark p1
+ try ( $p1 < x $p1 = x )
+)
+
+backwardmode (
+
+ define main_suffix as (
+ setlimit tomark p1 for ([substring])
+ among(
+
+ 'a' 'e' 'ede' 'ande' 'ende' 'ane' 'ene' 'hetene' 'en' 'heten' 'ar'
+ 'er' 'heter' 'as' 'es' 'edes' 'endes' 'enes' 'hetenes' 'ens'
+ 'hetens' 'ers' 'ets' 'et' 'het' 'ast'
+ (delete)
+ 's'
+ (s_ending or ('k' non-v) delete)
+ 'erte' 'ert'
+ (<-'er')
+ )
+ )
+
+ define consonant_pair as (
+ test (
+ setlimit tomark p1 for ([substring])
+ among(
+ 'dt' 'vt'
+ )
+ )
+ next] delete
+ )
+
+ define other_suffix as (
+ setlimit tomark p1 for ([substring])
+ among(
+ 'leg' 'eleg' 'ig' 'eig' 'lig' 'elig' 'els' 'lov' 'elov' 'slov'
+ 'hetslov'
+ (delete)
+ )
+ )
+)
+
+define stem as (
+
+ do mark_regions
+ backwards (
+ do main_suffix
+ do consonant_pair
+ do other_suffix
+ )
+)
diff --git a/contrib/snowball/algorithms/porter.sbl b/contrib/snowball/algorithms/porter.sbl
new file mode 100644
index 0000000..9533b79
--- /dev/null
+++ b/contrib/snowball/algorithms/porter.sbl
@@ -0,0 +1,139 @@
+integers ( p1 p2 )
+booleans ( Y_found )
+
+routines (
+ shortv
+ R1 R2
+ Step_1a Step_1b Step_1c Step_2 Step_3 Step_4 Step_5a Step_5b
+)
+
+externals ( stem )
+
+groupings ( v v_WXY )
+
+define v 'aeiouy'
+define v_WXY v + 'wxY'
+
+backwardmode (
+
+ define shortv as ( non-v_WXY v non-v )
+
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define Step_1a as (
+ [substring] among (
+ 'sses' (<-'ss')
+ 'ies' (<-'i')
+ 'ss' ()
+ 's' (delete)
+ )
+ )
+
+ define Step_1b as (
+ [substring] among (
+ 'eed' (R1 <-'ee')
+ 'ed'
+ 'ing' (
+ test gopast v delete
+ test substring among(
+ 'at' 'bl' 'iz'
+ (<+ 'e')
+ 'bb' 'dd' 'ff' 'gg' 'mm' 'nn' 'pp' 'rr' 'tt'
+ // ignoring double c, h, j, k, q, v, w, and x
+ ([next] delete)
+ '' (atmark p1 test shortv <+ 'e')
+ )
+ )
+ )
+ )
+
+ define Step_1c as (
+ ['y' or 'Y']
+ gopast v
+ <-'i'
+ )
+
+ define Step_2 as (
+ [substring] R1 among (
+ 'tional' (<-'tion')
+ 'enci' (<-'ence')
+ 'anci' (<-'ance')
+ 'abli' (<-'able')
+ 'entli' (<-'ent')
+ 'eli' (<-'e')
+ 'izer' 'ization'
+ (<-'ize')
+ 'ational' 'ation' 'ator'
+ (<-'ate')
+ 'alli' (<-'al')
+ 'alism' 'aliti'
+ (<-'al')
+ 'fulness' (<-'ful')
+ 'ousli' 'ousness'
+ (<-'ous')
+ 'iveness' 'iviti'
+ (<-'ive')
+ 'biliti' (<-'ble')
+ )
+ )
+
+ define Step_3 as (
+ [substring] R1 among (
+ 'alize' (<-'al')
+ 'icate' 'iciti' 'ical'
+ (<-'ic')
+ 'ative' 'ful' 'ness'
+ (delete)
+ )
+ )
+
+ define Step_4 as (
+ [substring] R2 among (
+ 'al' 'ance' 'ence' 'er' 'ic' 'able' 'ible' 'ant' 'ement'
+ 'ment' 'ent' 'ou' 'ism' 'ate' 'iti' 'ous' 'ive' 'ize'
+ (delete)
+ 'ion' ('s' or 't' delete)
+ )
+ )
+
+ define Step_5a as (
+ ['e']
+ R2 or (R1 not shortv)
+ delete
+ )
+
+ define Step_5b as (
+ ['l']
+ R2 'l'
+ delete
+ )
+)
+
+define stem as (
+
+ unset Y_found
+ do ( ['y'] <-'Y' set Y_found)
+ do repeat(goto (v ['y']) <-'Y' set Y_found)
+
+ $p1 = limit
+ $p2 = limit
+ do(
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+
+ backwards (
+ do Step_1a
+ do Step_1b
+ do Step_1c
+ do Step_2
+ do Step_3
+ do Step_4
+ do Step_5a
+ do Step_5b
+ )
+
+ do(Y_found repeat(goto (['Y']) <-'y'))
+
+)
diff --git a/contrib/snowball/algorithms/portuguese.sbl b/contrib/snowball/algorithms/portuguese.sbl
new file mode 100644
index 0000000..3fb14f1
--- /dev/null
+++ b/contrib/snowball/algorithms/portuguese.sbl
@@ -0,0 +1,218 @@
+routines (
+ prelude postlude mark_regions
+ RV R1 R2
+ standard_suffix
+ verb_suffix
+ residual_suffix
+ residual_form
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a' '{U+00E1}' // a-acute
+stringdef a^ '{U+00E2}' // a-circumflex e.g. 'bota^nico
+stringdef e' '{U+00E9}' // e-acute
+stringdef e^ '{U+00EA}' // e-circumflex
+stringdef i' '{U+00ED}' // i-acute
+stringdef o^ '{U+00F4}' // o-circumflex
+stringdef o' '{U+00F3}' // o-acute
+stringdef u' '{U+00FA}' // u-acute
+stringdef c, '{U+00E7}' // c-cedilla
+
+stringdef a~ '{U+00E3}' // a-tilde
+stringdef o~ '{U+00F5}' // o-tilde
+
+
+define v 'aeiou{a'}{e'}{i'}{o'}{u'}{a^}{e^}{o^}'
+
+define prelude as repeat (
+ [substring] among(
+ '{a~}' (<- 'a~')
+ '{o~}' (<- 'o~')
+ '' (next)
+ ) //or next
+)
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v (non-v gopast v) or (v gopast non-v) )
+ or
+ ( non-v (non-v gopast v) or (v next) )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define postlude as repeat (
+ [substring] among(
+ 'a~' (<- '{a~}')
+ 'o~' (<- '{o~}')
+ '' (next)
+ ) //or next
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define standard_suffix as (
+ [substring] among(
+
+ 'eza' 'ezas'
+ 'ico' 'ica' 'icos' 'icas'
+ 'ismo' 'ismos'
+ '{a'}vel'
+ '{i'}vel'
+ 'ista' 'istas'
+ 'oso' 'osa' 'osos' 'osas'
+ 'amento' 'amentos'
+ 'imento' 'imentos'
+
+ 'adora' 'ador' 'a{c,}a~o'
+ 'adoras' 'adores' 'a{c,}o~es' // no -ic test
+ 'ante' 'antes' '{a^}ncia' // Note 1
+ (
+ R2 delete
+ )
+ 'logia'
+ 'logias'
+ (
+ R2 <- 'log'
+ )
+ 'u{c,}a~o' 'u{c,}o~es'
+ (
+ R2 <- 'u'
+ )
+ '{e^}ncia' '{e^}ncias'
+ (
+ R2 <- 'ente'
+ )
+ 'amente'
+ (
+ R1 delete
+ try (
+ [substring] R2 delete among(
+ 'iv' (['at'] R2 delete)
+ 'os'
+ 'ic'
+ 'ad'
+ )
+ )
+ )
+ 'mente'
+ (
+ R2 delete
+ try (
+ [substring] among(
+ 'ante' // Note 1
+ 'avel'
+ '{i'}vel' (R2 delete)
+ )
+ )
+ )
+ 'idade'
+ 'idades'
+ (
+ R2 delete
+ try (
+ [substring] among(
+ 'abil'
+ 'ic'
+ 'iv' (R2 delete)
+ )
+ )
+ )
+ 'iva' 'ivo'
+ 'ivas' 'ivos'
+ (
+ R2 delete
+ try (
+ ['at'] R2 delete // but not a further ['ic'] R2 delete
+ )
+ )
+ 'ira' 'iras'
+ (
+ RV 'e' // -eira -eiras usually non-verbal
+ <- 'ir'
+ )
+ )
+ )
+
+ define verb_suffix as setlimit tomark pV for (
+ [substring] among(
+ 'ada' 'ida' 'ia' 'aria' 'eria' 'iria' 'ar{a'}' 'ara' 'er{a'}'
+ 'era' 'ir{a'}' 'ava' 'asse' 'esse' 'isse' 'aste' 'este' 'iste'
+ 'ei' 'arei' 'erei' 'irei' 'am' 'iam' 'ariam' 'eriam' 'iriam'
+ 'aram' 'eram' 'iram' 'avam' 'em' 'arem' 'erem' 'irem' 'assem'
+ 'essem' 'issem' 'ado' 'ido' 'ando' 'endo' 'indo' 'ara~o'
+ 'era~o' 'ira~o' 'ar' 'er' 'ir' 'as' 'adas' 'idas' 'ias'
+ 'arias' 'erias' 'irias' 'ar{a'}s' 'aras' 'er{a'}s' 'eras'
+ 'ir{a'}s' 'avas' 'es' 'ardes' 'erdes' 'irdes' 'ares' 'eres'
+ 'ires' 'asses' 'esses' 'isses' 'astes' 'estes' 'istes' 'is'
+ 'ais' 'eis' '{i'}eis' 'ar{i'}eis' 'er{i'}eis' 'ir{i'}eis'
+ '{a'}reis' 'areis' '{e'}reis' 'ereis' '{i'}reis' 'ireis'
+ '{a'}sseis' '{e'}sseis' '{i'}sseis' '{a'}veis' 'ados' 'idos'
+ '{a'}mos' 'amos' '{i'}amos' 'ar{i'}amos' 'er{i'}amos'
+ 'ir{i'}amos' '{a'}ramos' '{e'}ramos' '{i'}ramos' '{a'}vamos'
+ 'emos' 'aremos' 'eremos' 'iremos' '{a'}ssemos' '{e^}ssemos'
+ '{i'}ssemos' 'imos' 'armos' 'ermos' 'irmos' 'eu' 'iu' 'ou'
+
+ 'ira' 'iras'
+ (delete)
+ )
+ )
+
+ define residual_suffix as (
+ [substring] among(
+ 'os'
+ 'a' 'i' 'o' '{a'}' '{i'}' '{o'}'
+ ( RV delete )
+ )
+ )
+
+ define residual_form as (
+ [substring] among(
+ 'e' '{e'}' '{e^}'
+ ( RV delete [('u'] test 'g') or
+ ('i'] test 'c') RV delete )
+ '{c,}' (<-'c')
+ )
+ )
+)
+
+define stem as (
+ do prelude
+ do mark_regions
+ backwards (
+ do (
+ ( ( standard_suffix or verb_suffix )
+ and do ( ['i'] test 'c' RV delete )
+ )
+ or residual_suffix
+ )
+ do residual_form
+ )
+ do postlude
+)
+
+/*
+ Note 1: additions of 15 Jun 2005
+*/
diff --git a/contrib/snowball/algorithms/romanian.sbl b/contrib/snowball/algorithms/romanian.sbl
new file mode 100644
index 0000000..7db9e0a
--- /dev/null
+++ b/contrib/snowball/algorithms/romanian.sbl
@@ -0,0 +1,236 @@
+
+routines (
+ prelude postlude mark_regions
+ RV R1 R2
+ step_0
+ standard_suffix combo_suffix
+ verb_suffix
+ vowel_suffix
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+booleans ( standard_suffix_removed )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a^ '{U+00E2}' // a circumflex
+stringdef i^ '{U+00EE}' // i circumflex
+stringdef a+ '{U+0103}' // a breve
+stringdef s, '{U+015F}' // s cedilla
+stringdef t, '{U+0163}' // t cedilla
+
+define v 'aeiou{a^}{i^}{a+}'
+
+define prelude as (
+ repeat goto (
+ v [ ('u' ] v <- 'U') or
+ ('i' ] v <- 'I')
+ )
+)
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v (non-v gopast v) or (v gopast non-v) )
+ or
+ ( non-v (non-v gopast v) or (v next) )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define postlude as repeat (
+
+ [substring] among(
+ 'I' (<- 'i')
+ 'U' (<- 'u')
+ '' (next)
+ )
+
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define step_0 as (
+ [substring] R1 among(
+ 'ul' 'ului'
+ ( delete )
+ 'aua'
+ ( <-'a' )
+ 'ea' 'ele' 'elor'
+ ( <-'e' )
+ 'ii' 'iua' 'iei' 'iile' 'iilor' 'ilor'
+ ( <-'i')
+ 'ile'
+ ( not 'ab' <- 'i' )
+ 'atei'
+ ( <- 'at' )
+ 'a{t,}ie' 'a{t,}ia'
+ ( <- 'a{t,}i' )
+ )
+ )
+
+ define combo_suffix as test (
+ [substring] R1 (
+ among(
+ /* 'IST'. alternative: include the following
+ 'alism' 'alisme'
+ 'alist' 'alista' 'aliste' 'alisti' 'alist{a+}' 'ali{s,}ti' (
+ <- 'al'
+ )
+ */
+ 'abilitate' 'abilitati' 'abilit{a+}i' 'abilit{a+}{t,}i' (
+ <- 'abil'
+ )
+ 'ibilitate' (
+ <- 'ibil'
+ )
+ 'ivitate' 'ivitati' 'ivit{a+}i' 'ivit{a+}{t,}i' (
+ <- 'iv'
+ )
+ 'icitate' 'icitati' 'icit{a+}i' 'icit{a+}{t,}i'
+ 'icator' 'icatori'
+ 'iciv' 'iciva' 'icive' 'icivi' 'iciv{a+}'
+ 'ical' 'icala' 'icale' 'icali' 'ical{a+}' (
+ <- 'ic'
+ )
+ 'ativ' 'ativa' 'ative' 'ativi' 'ativ{a+}' 'a{t,}iune'
+ 'atoare' 'ator' 'atori'
+ '{a+}toare' '{a+}tor' '{a+}tori' (
+ <- 'at'
+ )
+ 'itiv' 'itiva' 'itive' 'itivi' 'itiv{a+}' 'i{t,}iune'
+ 'itoare' 'itor' 'itori' (
+ <- 'it'
+ )
+ )
+ set standard_suffix_removed
+ )
+ )
+
+ define standard_suffix as (
+ unset standard_suffix_removed
+ repeat combo_suffix
+ [substring] R2 (
+ among(
+
+ // past participle is treated here, rather than
+ // as a verb ending:
+ 'at' 'ata' 'at{a+}' 'ati' 'ate'
+ 'ut' 'uta' 'ut{a+}' 'uti' 'ute'
+ 'it' 'ita' 'it{a+}' 'iti' 'ite'
+
+ 'ic' 'ica' 'ice' 'ici' 'ic{a+}'
+ 'abil' 'abila' 'abile' 'abili' 'abil{a+}'
+ 'ibil' 'ibila' 'ibile' 'ibili' 'ibil{a+}'
+ 'oasa' 'oas{a+}' 'oase' 'os' 'osi' 'o{s,}i'
+ 'ant' 'anta' 'ante' 'anti' 'ant{a+}'
+ 'ator' 'atori'
+ 'itate' 'itati' 'it{a+}i' 'it{a+}{t,}i'
+ 'iv' 'iva' 'ive' 'ivi' 'iv{a+}' (
+ delete
+ )
+ 'iune' 'iuni' (
+ '{t,}'] <- 't'
+ )
+ 'ism' 'isme'
+ 'ist' 'ista' 'iste' 'isti' 'ist{a+}' 'i{s,}ti' (
+ <- 'ist'
+ /* 'IST'. alternative: remove with <- '' */
+ )
+ )
+ set standard_suffix_removed
+ )
+ )
+
+ define verb_suffix as setlimit tomark pV for (
+ [substring] among(
+ // 'long' infinitive:
+ 'are' 'ere' 'ire' '{a^}re'
+
+ // gerund:
+ 'ind' '{a^}nd'
+ 'indu' '{a^}ndu'
+
+ 'eze'
+ 'easc{a+}'
+ // present:
+ 'ez' 'ezi' 'eaz{a+}' 'esc' 'e{s,}ti'
+ 'e{s,}te'
+ '{a+}sc' '{a+}{s,}ti'
+ '{a+}{s,}te'
+
+ // imperfect:
+ 'am' 'ai' 'au'
+ 'eam' 'eai' 'ea' 'ea{t,}i' 'eau'
+ 'iam' 'iai' 'ia' 'ia{t,}i' 'iau'
+
+ // past: // (not 'ii')
+ 'ui'
+ 'a{s,}i' 'ar{a+}m' 'ar{a+}{t,}i' 'ar{a+}'
+ 'u{s,}i' 'ur{a+}m' 'ur{a+}{t,}i' 'ur{a+}'
+ 'i{s,}i' 'ir{a+}m' 'ir{a+}{t,}i' 'ir{a+}'
+ '{a^}i' '{a^}{s,}i' '{a^}r{a+}m' '{a^}r{a+}{t,}i' '{a^}r{a+}'
+
+ // pluferfect:
+ 'asem' 'ase{s,}i' 'ase' 'aser{a+}m' 'aser{a+}{t,}i' 'aser{a+}'
+ 'isem' 'ise{s,}i' 'ise' 'iser{a+}m' 'iser{a+}{t,}i' 'iser{a+}'
+ '{a^}sem' '{a^}se{s,}i' '{a^}se' '{a^}ser{a+}m' '{a^}ser{a+}{t,}i'
+ '{a^}ser{a+}'
+ 'usem' 'use{s,}i' 'use' 'user{a+}m' 'user{a+}{t,}i' 'user{a+}'
+
+ ( non-v or 'u' delete )
+
+ // present:
+ '{a+}m' 'a{t,}i'
+ 'em' 'e{t,}i'
+ 'im' 'i{t,}i'
+ '{a^}m' '{a^}{t,}i'
+
+ // past:
+ 'se{s,}i' 'ser{a+}m' 'ser{a+}{t,}i' 'ser{a+}'
+ 'sei' 'se'
+
+ // pluperfect:
+ 'sesem' 'sese{s,}i' 'sese' 'seser{a+}m' 'seser{a+}{t,}i' 'seser{a+}'
+ (delete)
+ )
+ )
+
+ define vowel_suffix as (
+ [substring] RV among (
+ 'a' 'e' 'i' 'ie' '{a+}' ( delete )
+ )
+ )
+)
+
+define stem as (
+ do prelude
+ do mark_regions
+ backwards (
+ do step_0
+ do standard_suffix
+ do ( standard_suffix_removed or verb_suffix )
+ do vowel_suffix
+ )
+ do postlude
+)
+
diff --git a/contrib/snowball/algorithms/russian.sbl b/contrib/snowball/algorithms/russian.sbl
new file mode 100644
index 0000000..20de639
--- /dev/null
+++ b/contrib/snowball/algorithms/russian.sbl
@@ -0,0 +1,221 @@
+stringescapes {}
+
+/* the 33 Cyrillic letters represented in ASCII characters following the
+ * conventions of the standard Library of Congress transliteration: */
+
+stringdef a '{U+0430}'
+stringdef b '{U+0431}'
+stringdef v '{U+0432}'
+stringdef g '{U+0433}'
+stringdef d '{U+0434}'
+stringdef e '{U+0435}'
+stringdef e" '{U+0451}'
+stringdef zh '{U+0436}'
+stringdef z '{U+0437}'
+stringdef i '{U+0438}'
+stringdef i` '{U+0439}'
+stringdef k '{U+043A}'
+stringdef l '{U+043B}'
+stringdef m '{U+043C}'
+stringdef n '{U+043D}'
+stringdef o '{U+043E}'
+stringdef p '{U+043F}'
+stringdef r '{U+0440}'
+stringdef s '{U+0441}'
+stringdef t '{U+0442}'
+stringdef u '{U+0443}'
+stringdef f '{U+0444}'
+stringdef kh '{U+0445}'
+stringdef ts '{U+0446}'
+stringdef ch '{U+0447}'
+stringdef sh '{U+0448}'
+stringdef shch '{U+0449}'
+stringdef " '{U+044A}'
+stringdef y '{U+044B}'
+stringdef ' '{U+044C}'
+stringdef e` '{U+044D}'
+stringdef iu '{U+044E}'
+stringdef ia '{U+044F}'
+
+routines ( mark_regions R2
+ perfective_gerund
+ adjective
+ adjectival
+ reflexive
+ verb
+ noun
+ derivational
+ tidy_up
+)
+
+externals ( stem )
+
+integers ( pV p2 )
+
+groupings ( v )
+
+define v '{a}{e}{i}{o}{u}{y}{e`}{iu}{ia}'
+
+define mark_regions as (
+
+ $pV = limit
+ $p2 = limit
+ do (
+ gopast v setmark pV gopast non-v
+ gopast v gopast non-v setmark p2
+ )
+)
+
+backwardmode (
+
+ define R2 as $p2 <= cursor
+
+ define perfective_gerund as (
+ [substring] among (
+ '{v}'
+ '{v}{sh}{i}'
+ '{v}{sh}{i}{s}{'}'
+ ('{a}' or '{ia}' delete)
+ '{i}{v}'
+ '{i}{v}{sh}{i}'
+ '{i}{v}{sh}{i}{s}{'}'
+ '{y}{v}'
+ '{y}{v}{sh}{i}'
+ '{y}{v}{sh}{i}{s}{'}'
+ (delete)
+ )
+ )
+
+ define adjective as (
+ [substring] among (
+ '{e}{e}' '{i}{e}' '{y}{e}' '{o}{e}' '{i}{m}{i}' '{y}{m}{i}'
+ '{e}{i`}' '{i}{i`}' '{y}{i`}' '{o}{i`}' '{e}{m}' '{i}{m}'
+ '{y}{m}' '{o}{m}' '{e}{g}{o}' '{o}{g}{o}' '{e}{m}{u}'
+ '{o}{m}{u}' '{i}{kh}' '{y}{kh}' '{u}{iu}' '{iu}{iu}' '{a}{ia}'
+ '{ia}{ia}'
+ // and -
+ '{o}{iu}' // - which is somewhat archaic
+ '{e}{iu}' // - soft form of {o}{iu}
+ (delete)
+ )
+ )
+
+ define adjectival as (
+ adjective
+
+ /* of the participle forms, em, vsh, ivsh, yvsh are readily removable.
+ nn, {iu}shch, shch, u{iu}shch can be removed, with a small proportion of
+ errors. Removing im, uem, enn creates too many errors.
+ */
+
+ try (
+ [substring] among (
+ '{e}{m}' // present passive participle
+ '{n}{n}' // adjective from past passive participle
+ '{v}{sh}' // past active participle
+ '{iu}{shch}' '{shch}' // present active participle
+ ('{a}' or '{ia}' delete)
+
+ //but not '{i}{m}' '{u}{e}{m}' // present passive participle
+ //or '{e}{n}{n}' // adjective from past passive participle
+
+ '{i}{v}{sh}' '{y}{v}{sh}'// past active participle
+ '{u}{iu}{shch}' // present active participle
+ (delete)
+ )
+ )
+
+ )
+
+ define reflexive as (
+ [substring] among (
+ '{s}{ia}'
+ '{s}{'}'
+ (delete)
+ )
+ )
+
+ define verb as (
+ [substring] among (
+ '{l}{a}' '{n}{a}' '{e}{t}{e}' '{i`}{t}{e}' '{l}{i}' '{i`}'
+ '{l}' '{e}{m}' '{n}' '{l}{o}' '{n}{o}' '{e}{t}' '{iu}{t}'
+ '{n}{y}' '{t}{'}' '{e}{sh}{'}'
+
+ '{n}{n}{o}'
+ ('{a}' or '{ia}' delete)
+
+ '{i}{l}{a}' '{y}{l}{a}' '{e}{n}{a}' '{e}{i`}{t}{e}'
+ '{u}{i`}{t}{e}' '{i}{t}{e}' '{i}{l}{i}' '{y}{l}{i}' '{e}{i`}'
+ '{u}{i`}' '{i}{l}' '{y}{l}' '{i}{m}' '{y}{m}' '{e}{n}'
+ '{i}{l}{o}' '{y}{l}{o}' '{e}{n}{o}' '{ia}{t}' '{u}{e}{t}'
+ '{u}{iu}{t}' '{i}{t}' '{y}{t}' '{e}{n}{y}' '{i}{t}{'}'
+ '{y}{t}{'}' '{i}{sh}{'}' '{u}{iu}' '{iu}'
+ (delete)
+ /* note the short passive participle tests:
+ '{n}{a}' '{n}' '{n}{o}' '{n}{y}'
+ '{e}{n}{a}' '{e}{n}' '{e}{n}{o}' '{e}{n}{y}'
+ */
+ )
+ )
+
+ define noun as (
+ [substring] among (
+ '{a}' '{e}{v}' '{o}{v}' '{i}{e}' '{'}{e}' '{e}'
+ '{i}{ia}{m}{i}' '{ia}{m}{i}' '{a}{m}{i}' '{e}{i}' '{i}{i}'
+ '{i}' '{i}{e}{i`}' '{e}{i`}' '{o}{i`}' '{i}{i`}' '{i`}'
+ '{i}{ia}{m}' '{ia}{m}' '{i}{e}{m}' '{e}{m}' '{a}{m}' '{o}{m}'
+ '{o}' '{u}' '{a}{kh}' '{i}{ia}{kh}' '{ia}{kh}' '{y}' '{'}'
+ '{i}{iu}' '{'}{iu}' '{iu}' '{i}{ia}' '{'}{ia}' '{ia}'
+ (delete)
+ /* the small class of neuter forms '{e}{n}{i}' '{e}{n}{e}{m}'
+ '{e}{n}{a}' '{e}{n}' '{e}{n}{a}{m}' '{e}{n}{a}{m}{i}' '{e}{n}{a}{x}'
+ omitted - they only occur on 12 words.
+ */
+ )
+ )
+
+ define derivational as (
+ [substring] R2 among (
+ '{o}{s}{t}'
+ '{o}{s}{t}{'}'
+ (delete)
+ )
+ )
+
+ define tidy_up as (
+ [substring] among (
+
+ '{e}{i`}{sh}'
+ '{e}{i`}{sh}{e}' // superlative forms
+ (delete
+ ['{n}'] '{n}' delete
+ )
+ '{n}'
+ ('{n}' delete) // e.g. -nno endings
+ '{'}'
+ (delete) // with some slight false conflations
+ )
+ )
+)
+
+define stem as (
+
+ // Normalise {e"} to {e}. The documentation has long suggested the user
+ // should do this before calling the stemmer - we now do it for them.
+ do repeat ( goto (['{e"}']) <- '{e}' )
+
+ do mark_regions
+ backwards setlimit tomark pV for (
+ do (
+ perfective_gerund or
+ ( try reflexive
+ adjectival or verb or noun
+ )
+ )
+ try([ '{i}' ] delete)
+ // because noun ending -i{iu} is being treated as verb ending -{iu}
+
+ do derivational
+ do tidy_up
+ )
+)
diff --git a/contrib/snowball/algorithms/serbian.sbl b/contrib/snowball/algorithms/serbian.sbl
new file mode 100644
index 0000000..bddf76b
--- /dev/null
+++ b/contrib/snowball/algorithms/serbian.sbl
@@ -0,0 +1,2378 @@
+/* Stemmer for Serbian language, based on:
+ *
+ * Ljubesic, Nikola. Pandzic, Ivan. Stemmer for Croatian
+ * http://nlp.ffzg.hr/resources/tools/stemmer-for-croatian/
+ *
+ * authors: Stefan Petkovic and Dragan Ivanovic
+ * emails: petkovic8 at gmail.com and dragan.ivanovic at uns.ac.rs
+ * version: 1.0 (20.04.2019)
+*/
+
+routines (
+ cyr_to_lat
+ prelude
+ mark_regions
+ R1 R2
+ Step_1
+ Step_2
+ Step_3
+)
+
+externals ( stem )
+
+integers ( p1 p2 p3 )
+
+groupings ( v ca sa rg )
+
+stringescapes {}
+
+/* special characters - Unicode codepoints */
+
+/* serbian cyrillic */
+
+stringdef cyrA '{U+0430}'
+stringdef cyrB '{U+0431}'
+stringdef cyrV '{U+0432}'
+stringdef cyrG '{U+0433}'
+stringdef cyrD '{U+0434}'
+stringdef cyrDx '{U+0452}'
+stringdef cyrE '{U+0435}'
+stringdef cyrZh '{U+0436}'
+stringdef cyrZ '{U+0437}'
+stringdef cyrI '{U+0438}'
+stringdef cyrJ '{U+0458}'
+stringdef cyrK '{U+043A}'
+stringdef cyrL '{U+043B}'
+stringdef cyrLJ '{U+0459}'
+stringdef cyrM '{U+043C}'
+stringdef cyrN '{U+043D}'
+stringdef cyrNJ '{U+045A}'
+stringdef cyrO '{U+043E}'
+stringdef cyrP '{U+043F}'
+stringdef cyrR '{U+0440}'
+stringdef cyrS '{U+0441}'
+stringdef cyrT '{U+0442}'
+stringdef cyrCy '{U+045B}'
+stringdef cyrU '{U+0443}'
+stringdef cyrF '{U+0444}'
+stringdef cyrH '{U+0445}'
+stringdef cyrC '{U+0446}'
+stringdef cyrCx '{U+0447}'
+stringdef cyrDzx '{U+045F}'
+stringdef cyrSx '{U+0448}'
+
+/* serbian latin with diacritics */
+
+stringdef cx '{U+010D}' // small c with caron
+stringdef cy '{U+0107}' // small c with acute
+stringdef zx '{U+017E}' // small z with caron
+stringdef sx '{U+0161}' // small s with caron
+stringdef dx '{U+0111}' // small d with stroke
+
+define v 'aeiou'
+define sa '{cx}{cy}{zx}{sx}{dx}'
+define ca 'bvgdzjklmnprstfhc' + sa
+define rg 'r'
+
+
+define cyr_to_lat as (
+
+ do repeat goto (
+ [substring] among (
+ '{cyrA}' (<- 'a')
+ '{cyrB}' (<- 'b')
+ '{cyrV}' (<- 'v')
+ '{cyrG}' (<- 'g')
+ '{cyrD}' (<- 'd')
+ '{cyrDx}' (<- '{dx}')
+ '{cyrE}' (<- 'e')
+ '{cyrZh}' (<- '{zx}')
+ '{cyrZ}' (<- 'z')
+ '{cyrI}' (<- 'i')
+ '{cyrJ}' (<- 'j')
+ '{cyrK}' (<- 'k')
+ '{cyrL}' (<- 'l')
+ '{cyrLJ}' (<- 'lj')
+ '{cyrM}' (<- 'm')
+ '{cyrN}' (<- 'n')
+ '{cyrNJ}' (<- 'nj')
+ '{cyrO}' (<- 'o')
+ '{cyrP}' (<- 'p')
+ '{cyrR}' (<- 'r')
+ '{cyrS}' (<- 's')
+ '{cyrT}' (<- 't')
+ '{cyrCy}' (<- '{cy}')
+ '{cyrU}' (<- 'u')
+ '{cyrF}' (<- 'f')
+ '{cyrH}' (<- 'h')
+ '{cyrC}' (<- 'c')
+ '{cyrCx}' (<- '{cx}')
+ '{cyrDzx}' (<- 'd{zx}')
+ '{cyrSx}' (<- '{sx}')
+ )
+ )
+
+)
+
+define prelude as (
+
+ do repeat goto (
+ ca ['ije'] ca <- 'e'
+ )
+
+ do repeat goto (
+ ca ['je'] ca <- 'e'
+ )
+
+ do repeat goto (
+ ['dj'] <- '{dx}'
+ )
+
+)
+
+define mark_regions as (
+
+ $p3 = 0
+
+ do (
+ gopast sa setmark p3
+ )
+
+ $p1 = limit
+ $p2 = 0
+
+ do (
+ gopast v setmark p1
+ )
+ do (
+ gopast 'r' setmark p2
+ $(p1 - p2 > 1) ($p1 = p2)
+ )
+ ($p1 < 2) (
+ ($p1 == p2 gopast 'r' gopast non-rg) or ($p1 != p2 gopast v gopast non-v)
+ setmark p1
+ )
+
+)
+
+backwardmode (
+
+ define R1 as $p1 <= cursor
+ define R2 as $p3 == 0
+
+ define Step_1 as (
+ [substring] among (
+ 'lozi'
+ 'lozima' (<-'loga')
+ 'pesi'
+ 'pesima' (<-'peh')
+ 'vojci' (<-'vojka')
+ 'bojci' (<-'bojka')
+ 'jaci'
+ 'jacima' (<-'jak')
+ '{cx}ajan' (<-'{cx}ajni')
+ 'cajan' (R2 <-'cajni')
+ 'eran' (<-'erni')
+ 'laran' (<-'larni')
+ 'esan' (<-'esni')
+ 'anjac' (<-'anjca')
+ 'ajac'
+ 'ajaca' (<-'ajca')
+ 'ljaca'
+ 'ljac' (<-'ljca')
+ 'ejac'
+ 'ejaca' (<-'ejca')
+ 'ojac'
+ 'ojaca' (<-'ojca')
+ 'ajaka' (<-'ajka')
+ 'ojaka' (<-'ojka')
+ '{sx}aca'
+ '{sx}ac' (<-'{sx}ca')
+ 'inzima'
+ 'inzi' (<-'ing')
+ 'tvenici' (<-'tvenik')
+ 'tetici'
+ 'teticima' (<-'tetika')
+ 'nstava' (<-'nstva')
+ 'nicima' (<-'nik')
+ 'ticima' (<-'tik')
+ 'zicima' (<-'zik')
+ 'snici' (<-'snik')
+ 'kuse' (<-'kusi')
+ 'kusan' (<-'kusni')
+ 'kustava' (<-'kustva')
+ 'du{sx}an' (<-'du{sx}ni')
+ 'dusan' (R2 <-'dusni')
+ 'antan' (<-'antni')
+ 'bilan' (<-'bilni')
+ 'tilan' (<-'tilni')
+ 'avilan' (<-'avilni')
+ 'silan' (<-'silni')
+ 'gilan' (<-'gilni')
+ 'rilan' (<-'rilni')
+ 'nilan' (<-'nilni')
+ 'alan' (<-'alni')
+ 'ozan' (<-'ozni')
+ 'rave' (<-'ravi')
+ 'stavan' (<-'stavni')
+ 'pravan' (<-'pravni')
+ 'tivan' (<-'tivni')
+ 'sivan' (<-'sivni')
+ 'atan' (<-'atni')
+ 'enat' (<-'enta')
+ 'tetan' (<-'tetni')
+ 'pletan' (<-'pletni')
+ '{sx}ave' (<-'{sx}avi')
+ 'save' (R2 <-'savi')
+ 'anata' (<-'anta')
+ 'a{cx}ak'
+ 'a{cx}aka' (<-'a{cx}ka')
+ 'acak'
+ 'acaka' (R2 <-'acka')
+ 'u{sx}ak' (<-'u{sx}ka')
+ 'usak' (R2 <-'uska')
+ 'atak'
+ 'ataka'
+ 'atci'
+ 'atcima' (<-'atka')
+ 'etak'
+ 'etaka' (<-'etka')
+ 'itak'
+ 'itaka'
+ 'itci' (<-'itka')
+ 'otak'
+ 'otaka' (<-'otka')
+ 'utak'
+ 'utaka'
+ 'utci'
+ 'utcima' (<-'utka')
+ 'eskan' (<-'eskna')
+ 'ti{cx}an' (<-'ti{cx}ni')
+ 'tican' (R2 <-'ticni')
+ 'ojsci' (<-'ojska')
+ 'esama' (<-'esma')
+ 'metar'
+ 'metara' (<-'metra')
+ 'centar'
+ 'centara' (<-'centra')
+ 'istar'
+ 'istara' (<-'istra')
+ 'o{sx}{cy}u' (<-'osti')
+ 'oscu' (R2 <-'osti')
+ 'daba' (<-'dba')
+ '{cx}cima'
+ '{cx}ci' (<-'{cx}ka')
+ 'mac'
+ 'maca' (<-'mca')
+ 'naca'
+ 'nac' (<-'nca')
+ 'voljan' (<-'voljni')
+ 'anaka' (<-'anki')
+ 'vac'
+ 'vaca' (<-'vca')
+ 'saca'
+ 'sac' (<-'sca')
+ 'raca'
+ 'rac' (<-'rca')
+ 'aoca'
+ 'alaca'
+ 'alac' (<-'alca')
+ 'elaca'
+ 'elac' (<-'elca')
+ 'olaca'
+ 'olac'
+ 'olce' (<-'olca')
+ 'njac'
+ 'njaca' (<-'njca')
+ 'ekata'
+ 'ekat' (<-'ekta')
+ 'izam'
+ 'izama' (<-'izma')
+ 'jebe' (<-'jebi')
+ 'baci' (<-'baci')
+ 'a{sx}an' (<-'a{sx}ni')
+ 'asan' (R2 <-'asni')
+ )
+ )
+
+ define Step_2 as (
+ [substring] R1 among (
+ 'skijima'
+ 'skijega'
+ 'skijemu'
+ 'skijem'
+ 'skega'
+ 'skemu'
+ 'skem'
+ 'skijim'
+ 'skijih'
+ 'skijoj'
+ 'skijeg'
+ 'skiji'
+ 'skije'
+ 'skija'
+ 'skoga'
+ 'skome'
+ 'skomu'
+ 'skima'
+ 'skog'
+ 'skom'
+ 'skim'
+ 'skih'
+ 'skoj'
+ 'ski'
+ 'ske'
+ 'sko'
+ 'ska'
+ 'sku' (<-'sk')
+ '{sx}kijima'
+ '{sx}kijega'
+ '{sx}kijemu'
+ '{sx}kijem'
+ '{sx}kega'
+ '{sx}kemu'
+ '{sx}kem'
+ '{sx}kijim'
+ '{sx}kijih'
+ '{sx}kijoj'
+ '{sx}kijeg'
+ '{sx}kiji'
+ '{sx}kije'
+ '{sx}kija'
+ '{sx}koga'
+ '{sx}kome'
+ '{sx}komu'
+ '{sx}kima'
+ '{sx}kog'
+ '{sx}kom'
+ '{sx}kim'
+ '{sx}kih'
+ '{sx}koj'
+ '{sx}ki'
+ '{sx}ke'
+ '{sx}ko'
+ '{sx}ka'
+ '{sx}ku' (<-'{sx}k')
+ 'stvima'
+ 'stvom'
+ 'stvo'
+ 'stva'
+ 'stvu' (<-'stv')
+ '{sx}tvima'
+ '{sx}tvom'
+ '{sx}tvo'
+ '{sx}tva'
+ '{sx}tvu' (<-'{sx}tv')
+ 'tanijama'
+ 'tanijima'
+ 'tanijom'
+ 'tanija'
+ 'taniju'
+ 'tanije'
+ 'taniji' (<-'tanij')
+ 'manijama'
+ 'manijima'
+ 'manijom'
+ 'manija'
+ 'maniju'
+ 'manije'
+ 'maniji' (<-'manij')
+ 'panijama'
+ 'panijima'
+ 'panijom'
+ 'panija'
+ 'paniju'
+ 'panije'
+ 'paniji' (<-'panij')
+ 'ranijama'
+ 'ranijima'
+ 'ranijom'
+ 'ranija'
+ 'raniju'
+ 'ranije'
+ 'raniji' (<-'ranij')
+ 'ganijama'
+ 'ganijima'
+ 'ganijom'
+ 'ganija'
+ 'ganiju'
+ 'ganije'
+ 'ganiji' (<-'ganij')
+ 'aninom'
+ 'anina'
+ 'aninu'
+ 'anine'
+ 'anima'
+ 'anin'
+ 'anom'
+ 'anu'
+ 'ani'
+ 'ana'
+ 'ane' (<-'an')
+ 'inima'
+ 'inama'
+ 'inom'
+ 'ina'
+ 'ine'
+ 'ini'
+ 'inu'
+ 'ino' (<-'in')
+ 'onovima'
+ 'onova'
+ 'onove'
+ 'onovi'
+ 'onima'
+ 'onom'
+ 'ona'
+ 'one'
+ 'oni'
+ 'onu' (<-'on')
+ 'nijima'
+ 'nijega'
+ 'nijemu'
+ 'nijeg'
+ 'nijem'
+ 'nega'
+ 'nemu'
+ 'neg'
+ 'nem'
+ 'nijim'
+ 'nijih'
+ 'nijoj'
+ 'niji'
+ 'nije'
+ 'nija'
+ 'niju'
+ 'nima'
+ 'nome'
+ 'nomu'
+ 'noga'
+ 'noj'
+ 'nom'
+ 'nih'
+ 'nim'
+ 'nog'
+ 'no'
+ 'ne'
+ 'na'
+ 'nu'
+ 'ni' (<-'n')
+ 'a{cy}oga'
+ 'a{cy}ome'
+ 'a{cy}omu'
+ 'a{cy}ega'
+ 'a{cy}emu'
+ 'a{cy}ima'
+ 'a{cy}oj'
+ 'a{cy}ih'
+ 'a{cy}om'
+ 'a{cy}eg'
+ 'a{cy}em'
+ 'a{cy}og'
+ 'a{cy}uh'
+ 'a{cy}im'
+ 'a{cy}e'
+ 'a{cy}a' (<-'a{cy}')
+ 'e{cy}oga'
+ 'e{cy}ome'
+ 'e{cy}omu'
+ 'e{cy}ega'
+ 'e{cy}emu'
+ 'e{cy}ima'
+ 'e{cy}oj'
+ 'e{cy}ih'
+ 'e{cy}om'
+ 'e{cy}eg'
+ 'e{cy}em'
+ 'e{cy}og'
+ 'e{cy}uh'
+ 'e{cy}im'
+ 'e{cy}e'
+ 'e{cy}a' (<-'e{cy}')
+ 'u{cy}oga'
+ 'u{cy}ome'
+ 'u{cy}omu'
+ 'u{cy}ega'
+ 'u{cy}emu'
+ 'u{cy}ima'
+ 'u{cy}oj'
+ 'u{cy}ih'
+ 'u{cy}om'
+ 'u{cy}eg'
+ 'u{cy}em'
+ 'u{cy}og'
+ 'u{cy}uh'
+ 'u{cy}im'
+ 'u{cy}e'
+ 'u{cy}a' (<-'u{cy}')
+ 'ugovima'
+ 'ugovi'
+ 'ugove'
+ 'ugova' (<-'ugov')
+ 'ugama'
+ 'ugom'
+ 'uga'
+ 'uge'
+ 'ugi'
+ 'ugu'
+ 'ugo' (<-'ug')
+ 'logama'
+ 'logom'
+ 'loga'
+ 'logu'
+ 'loge' (<-'log')
+ 'govima'
+ 'gama'
+ 'govi'
+ 'gove'
+ 'gova'
+ 'gom'
+ 'ga'
+ 'ge'
+ 'gi'
+ 'gu'
+ 'go' (<-'g')
+ 'rarijem'
+ 'rarija'
+ 'rariju'
+ 'rario' (<-'rari')
+ 'otijem'
+ 'otija'
+ 'otiju'
+ 'otio' (<-'oti')
+ 'sijem'
+ 'sija'
+ 'siju'
+ 'sio' (<-'si')
+ 'lijem'
+ 'lija'
+ 'liju'
+ 'lio' (<-'li')
+ 'uju{cy}i'
+ 'ujemo'
+ 'ujete'
+ 'ujmo'
+ 'ujem'
+ 'uje{sx}'
+ 'uje'
+ 'uju' (<-'uj')
+ 'cajevima'
+ 'cajevi'
+ 'cajeva'
+ 'cajeve'
+ 'cajama'
+ 'cajima'
+ 'cajem'
+ 'caja'
+ 'caje'
+ 'caji'
+ 'caju' (<-'caj')
+ '{cx}ajevima'
+ '{cx}ajevi'
+ '{cx}ajeva'
+ '{cx}ajeve'
+ '{cx}ajama'
+ '{cx}ajima'
+ '{cx}ajem'
+ '{cx}aja'
+ '{cx}aje'
+ '{cx}aji'
+ '{cx}aju' (<-'{cx}aj')
+ '{cy}ajevima'
+ '{cy}ajevi'
+ '{cy}ajeva'
+ '{cy}ajeve'
+ '{cy}ajama'
+ '{cy}ajima'
+ '{cy}ajem'
+ '{cy}aja'
+ '{cy}aje'
+ '{cy}aji'
+ '{cy}aju' (<-'{cy}aj')
+ '{dx}ajevima'
+ '{dx}ajevi'
+ '{dx}ajeva'
+ '{dx}ajeve'
+ '{dx}ajama'
+ '{dx}ajima'
+ '{dx}ajem'
+ '{dx}aja'
+ '{dx}aje'
+ '{dx}aji'
+ '{dx}aju' (<-'{dx}aj')
+ 'lajevima'
+ 'lajevi'
+ 'lajeva'
+ 'lajeve'
+ 'lajama'
+ 'lajima'
+ 'lajem'
+ 'laja'
+ 'laje'
+ 'laji'
+ 'laju' (<-'laj')
+ 'rajevima'
+ 'rajevi'
+ 'rajeva'
+ 'rajeve'
+ 'rajama'
+ 'rajima'
+ 'rajem'
+ 'raja'
+ 'raje'
+ 'raji'
+ 'raju' (<-'raj')
+ 'bijima'
+ 'bijama'
+ 'bijom'
+ 'bija'
+ 'bije'
+ 'biji'
+ 'biju'
+ 'bijo' (<-'bij')
+ 'cijima'
+ 'cijama'
+ 'cijom'
+ 'cija'
+ 'cije'
+ 'ciji'
+ 'ciju'
+ 'cijo' (<-'cij')
+ 'dijima'
+ 'dijama'
+ 'dijom'
+ 'dija'
+ 'dije'
+ 'diji'
+ 'diju'
+ 'dijo' (<-'dij')
+ 'lijima'
+ 'lijama'
+ 'lijom'
+ 'lije'
+ 'liji'
+ 'lijo' (<-'lij')
+ 'nijama'
+ 'nijom'
+ 'nijo' (<-'nij')
+ 'mijima'
+ 'mijama'
+ 'mijom'
+ 'mija'
+ 'mije'
+ 'miji'
+ 'miju'
+ 'mijo' (<-'mij')
+ '{zx}ijima'
+ '{zx}ijama'
+ '{zx}ijom'
+ '{zx}ija'
+ '{zx}ije'
+ '{zx}iji'
+ '{zx}iju'
+ '{zx}ijo' (<-'{zx}ij')
+ 'gijima'
+ 'gijama'
+ 'gijom'
+ 'gija'
+ 'gije'
+ 'giji'
+ 'giju'
+ 'gijo' (<-'gij')
+ 'fijima'
+ 'fijama'
+ 'fijom'
+ 'fija'
+ 'fije'
+ 'fiji'
+ 'fiju'
+ 'fijo' (<-'fij')
+ 'pijima'
+ 'pijama'
+ 'pijom'
+ 'pija'
+ 'pije'
+ 'piji'
+ 'piju'
+ 'pijo' (<-'pij')
+ 'rijima'
+ 'rijama'
+ 'rijom'
+ 'rija'
+ 'rije'
+ 'riji'
+ 'riju'
+ 'rijo' (<-'rij')
+ 'sijima'
+ 'sijama'
+ 'sijom'
+ 'sije'
+ 'siji'
+ 'sijo' (<-'sij')
+ 'tijima'
+ 'tijama'
+ 'tijom'
+ 'tija'
+ 'tije'
+ 'tiji'
+ 'tiju'
+ 'tijo' (<-'tij')
+ 'zijima'
+ 'zijama'
+ 'zijom'
+ 'zija'
+ 'zije'
+ 'ziji'
+ 'ziju'
+ 'zijo' (<-'zij')
+ 'nalima'
+ 'nalama'
+ 'nalom'
+ 'nala'
+ 'nale'
+ 'nali'
+ 'nalu'
+ 'nalo' (<-'nal')
+ 'ijalima'
+ 'ijalama'
+ 'ijalom'
+ 'ijala'
+ 'ijale'
+ 'ijali'
+ 'ijalu'
+ 'ijalo' (<-'ijal')
+ 'ozilima'
+ 'ozilom'
+ 'ozila'
+ 'ozile'
+ 'ozilu'
+ 'ozili' (<-'ozil')
+ 'olovima'
+ 'olovi'
+ 'olova'
+ 'olove' (<-'olov')
+ 'olima'
+ 'olom'
+ 'ola'
+ 'olu'
+ 'ole'
+ 'oli' (<-'ol')
+ 'lemama'
+ 'lemima'
+ 'lemom'
+ 'lema'
+ 'leme'
+ 'lemi'
+ 'lemu'
+ 'lemo' (<-'lem')
+ 'ramama'
+ 'ramom'
+ 'rama'
+ 'rame'
+ 'rami'
+ 'ramu'
+ 'ramo' (<-'ram')
+ 'arama'
+ 'arima'
+ 'arom'
+ 'aru'
+ 'ara'
+ 'are'
+ 'ari' (<-'ar')
+ 'drama'
+ 'drima'
+ 'drom'
+ 'dru'
+ 'dra'
+ 'dre'
+ 'dri' (<-'dr')
+ 'erama'
+ 'erima'
+ 'erom'
+ 'eru'
+ 'era'
+ 'ere'
+ 'eri' (<-'er')
+ 'orama'
+ 'orima'
+ 'orom'
+ 'oru'
+ 'ora'
+ 'ore'
+ 'ori' (<-'or')
+ 'esima'
+ 'esom'
+ 'ese'
+ 'esa'
+ 'esu' (<-'es')
+ 'isima'
+ 'isom'
+ 'ise'
+ 'isa'
+ 'isu' (<-'is')
+ 'ta{sx}ama'
+ 'ta{sx}ima'
+ 'ta{sx}om'
+ 'ta{sx}em'
+ 'ta{sx}a'
+ 'ta{sx}u'
+ 'ta{sx}i'
+ 'ta{sx}e' (<-'ta{sx}')
+ 'na{sx}ama'
+ 'na{sx}ima'
+ 'na{sx}om'
+ 'na{sx}em'
+ 'na{sx}a'
+ 'na{sx}u'
+ 'na{sx}i'
+ 'na{sx}e' (<-'na{sx}')
+ 'ja{sx}ama'
+ 'ja{sx}ima'
+ 'ja{sx}om'
+ 'ja{sx}em'
+ 'ja{sx}a'
+ 'ja{sx}u'
+ 'ja{sx}i'
+ 'ja{sx}e' (<-'ja{sx}')
+ 'ka{sx}ama'
+ 'ka{sx}ima'
+ 'ka{sx}om'
+ 'ka{sx}em'
+ 'ka{sx}a'
+ 'ka{sx}u'
+ 'ka{sx}i'
+ 'ka{sx}e' (<-'ka{sx}')
+ 'ba{sx}ama'
+ 'ba{sx}ima'
+ 'ba{sx}om'
+ 'ba{sx}em'
+ 'ba{sx}a'
+ 'ba{sx}u'
+ 'ba{sx}i'
+ 'ba{sx}e' (<-'ba{sx}')
+ 'ga{sx}ama'
+ 'ga{sx}ima'
+ 'ga{sx}om'
+ 'ga{sx}em'
+ 'ga{sx}a'
+ 'ga{sx}u'
+ 'ga{sx}i'
+ 'ga{sx}e' (<-'ga{sx}')
+ 'va{sx}ama'
+ 'va{sx}ima'
+ 'va{sx}om'
+ 'va{sx}em'
+ 'va{sx}a'
+ 'va{sx}u'
+ 'va{sx}i'
+ 'va{sx}e' (<-'va{sx}')
+ 'e{sx}ima'
+ 'e{sx}ama'
+ 'e{sx}om'
+ 'e{sx}em'
+ 'e{sx}i'
+ 'e{sx}e'
+ 'e{sx}a'
+ 'e{sx}u' (<-'e{sx}')
+ 'i{sx}ima'
+ 'i{sx}ama'
+ 'i{sx}om'
+ 'i{sx}em'
+ 'i{sx}i'
+ 'i{sx}e'
+ 'i{sx}a'
+ 'i{sx}u' (<-'i{sx}')
+ 'ikatima'
+ 'ikatom'
+ 'ikata'
+ 'ikate'
+ 'ikati'
+ 'ikatu'
+ 'ikato' (<-'ikat')
+ 'latima'
+ 'latom'
+ 'lata'
+ 'late'
+ 'lati'
+ 'latu'
+ 'lato' (<-'lat')
+ 'etama'
+ 'etima'
+ 'etom'
+ 'eta'
+ 'ete'
+ 'eti'
+ 'etu'
+ 'eto' (<-'et')
+ 'estima'
+ 'estama'
+ 'estom'
+ 'esta'
+ 'este'
+ 'esti'
+ 'estu'
+ 'esto' (<-'est')
+ 'istima'
+ 'istama'
+ 'istom'
+ 'ista'
+ 'iste'
+ 'isti'
+ 'istu'
+ 'isto' (<-'ist')
+ 'kstima'
+ 'kstama'
+ 'kstom'
+ 'ksta'
+ 'kste'
+ 'ksti'
+ 'kstu'
+ 'ksto' (<-'kst')
+ 'ostima'
+ 'ostama'
+ 'ostom'
+ 'osta'
+ 'oste'
+ 'osti'
+ 'ostu'
+ 'osto' (<-'ost')
+ 'i{sx}tima'
+ 'i{sx}tem'
+ 'i{sx}ta'
+ 'i{sx}te'
+ 'i{sx}tu' (<-'i{sx}t')
+ 'ovasmo'
+ 'ovaste'
+ 'ovahu'
+ 'ovati'
+ 'ova{sx}e'
+ 'ovali'
+ 'ovala'
+ 'ovale'
+ 'ovalo'
+ 'ovat'
+ 'ovah'
+ 'ovao' (<-'ova')
+ 'avijemu'
+ 'avijima'
+ 'avijega'
+ 'avijeg'
+ 'avijem'
+ 'avemu'
+ 'avega'
+ 'aveg'
+ 'avem'
+ 'avijim'
+ 'avijih'
+ 'avijoj'
+ 'avoga'
+ 'avome'
+ 'avomu'
+ 'avima'
+ 'avama'
+ 'aviji'
+ 'avije'
+ 'avija'
+ 'aviju'
+ 'avim'
+ 'avih'
+ 'avoj'
+ 'avom'
+ 'avog'
+ 'avi'
+ 'ava'
+ 'avu'
+ 'ave'
+ 'avo' (<-'av')
+ 'evijemu'
+ 'evijima'
+ 'evijega'
+ 'evijeg'
+ 'evijem'
+ 'evemu'
+ 'evega'
+ 'eveg'
+ 'evem'
+ 'evijim'
+ 'evijih'
+ 'evijoj'
+ 'evoga'
+ 'evome'
+ 'evomu'
+ 'evima'
+ 'evama'
+ 'eviji'
+ 'evije'
+ 'evija'
+ 'eviju'
+ 'evim'
+ 'evih'
+ 'evoj'
+ 'evom'
+ 'evog'
+ 'evi'
+ 'eva'
+ 'evu'
+ 'eve'
+ 'evo' (<-'ev')
+ 'ivijemu'
+ 'ivijima'
+ 'ivijega'
+ 'ivijeg'
+ 'ivijem'
+ 'ivemu'
+ 'ivega'
+ 'iveg'
+ 'ivem'
+ 'ivijim'
+ 'ivijih'
+ 'ivijoj'
+ 'ivoga'
+ 'ivome'
+ 'ivomu'
+ 'ivima'
+ 'ivama'
+ 'iviji'
+ 'ivije'
+ 'ivija'
+ 'iviju'
+ 'ivim'
+ 'ivih'
+ 'ivoj'
+ 'ivom'
+ 'ivog'
+ 'ivi'
+ 'iva'
+ 'ivu'
+ 'ive'
+ 'ivo' (<-'iv')
+ 'ovijemu'
+ 'ovijima'
+ 'ovijega'
+ 'ovijeg'
+ 'ovijem'
+ 'ovemu'
+ 'ovega'
+ 'oveg'
+ 'ovijim'
+ 'ovijih'
+ 'ovijoj'
+ 'ovoga'
+ 'ovome'
+ 'ovomu'
+ 'ovima'
+ 'oviji'
+ 'ovije'
+ 'ovija'
+ 'oviju'
+ 'ovim'
+ 'ovih'
+ 'ovoj'
+ 'ovom'
+ 'ovog'
+ 'ovi'
+ 'ova'
+ 'ovu'
+ 'ove'
+ 'ovo' (<-'ov')
+ 'movima'
+ 'movom'
+ 'mova'
+ 'movu'
+ 'move'
+ 'movi' (<-'mov')
+ 'lovima'
+ 'lovom'
+ 'lova'
+ 'lovu'
+ 'love'
+ 'lovi' (<-'lov')
+ 'elijemu'
+ 'elijima'
+ 'elijega'
+ 'elijeg'
+ 'elijem'
+ 'elemu'
+ 'elega'
+ 'eleg'
+ 'elem'
+ 'elijim'
+ 'elijih'
+ 'elijoj'
+ 'eloga'
+ 'elome'
+ 'elomu'
+ 'elima'
+ 'eliji'
+ 'elije'
+ 'elija'
+ 'eliju'
+ 'elim'
+ 'elih'
+ 'eloj'
+ 'elom'
+ 'elog'
+ 'eli'
+ 'ela'
+ 'elu'
+ 'ele'
+ 'elo' (<-'el')
+ 'anjijemu'
+ 'anjijima'
+ 'anjijega'
+ 'anjijeg'
+ 'anjijem'
+ 'anjemu'
+ 'anjega'
+ 'anjeg'
+ 'anjem'
+ 'anjijim'
+ 'anjijih'
+ 'anjijoj'
+ 'anjoga'
+ 'anjome'
+ 'anjomu'
+ 'anjima'
+ 'anjiji'
+ 'anjije'
+ 'anjija'
+ 'anjiju'
+ 'anjim'
+ 'anjih'
+ 'anjoj'
+ 'anjom'
+ 'anjog'
+ 'anja'
+ 'anje'
+ 'anji'
+ 'anjo'
+ 'anju' (<-'anj')
+ 'enjijemu'
+ 'enjijima'
+ 'enjijega'
+ 'enjijeg'
+ 'enjijem'
+ 'enjemu'
+ 'enjega'
+ 'enjeg'
+ 'enjem'
+ 'enjijim'
+ 'enjijih'
+ 'enjijoj'
+ 'enjoga'
+ 'enjome'
+ 'enjomu'
+ 'enjima'
+ 'enjiji'
+ 'enjije'
+ 'enjija'
+ 'enjiju'
+ 'enjim'
+ 'enjih'
+ 'enjoj'
+ 'enjom'
+ 'enjog'
+ 'enja'
+ 'enje'
+ 'enji'
+ 'enjo'
+ 'enju' (<-'enj')
+ '{sx}njijemu'
+ '{sx}njijima'
+ '{sx}njijega'
+ '{sx}njijeg'
+ '{sx}njijem'
+ '{sx}njemu'
+ '{sx}njega'
+ '{sx}njeg'
+ '{sx}njem'
+ '{sx}njijim'
+ '{sx}njijih'
+ '{sx}njijoj'
+ '{sx}njoga'
+ '{sx}njome'
+ '{sx}njomu'
+ '{sx}njima'
+ '{sx}njiji'
+ '{sx}njije'
+ '{sx}njija'
+ '{sx}njiju'
+ '{sx}njim'
+ '{sx}njih'
+ '{sx}njoj'
+ '{sx}njom'
+ '{sx}njog'
+ '{sx}nja'
+ '{sx}nje'
+ '{sx}nji'
+ '{sx}njo'
+ '{sx}nju' (<-'{sx}nj')
+ 'anemu'
+ 'anega'
+ 'aneg'
+ 'anem' (<-'an')
+ 'enemu'
+ 'enega'
+ 'eneg'
+ 'enem' (<-'en')
+ '{sx}nemu'
+ '{sx}nega'
+ '{sx}neg'
+ '{sx}nem' (<-'{sx}n')
+ '{cx}inama'
+ '{cx}inome'
+ '{cx}inomu'
+ '{cx}inoga'
+ '{cx}inima'
+ '{cx}inog'
+ '{cx}inom'
+ '{cx}inim'
+ '{cx}inih'
+ '{cx}inoj'
+ '{cx}ina'
+ '{cx}inu'
+ '{cx}ini'
+ '{cx}ino'
+ '{cx}ine' (<-'{cx}in')
+ 'ro{sx}iv{sx}i'
+ 'ro{sx}ismo'
+ 'ro{sx}iste'
+ 'ro{sx}i{sx}e'
+ 'ro{sx}imo'
+ 'ro{sx}ite'
+ 'ro{sx}iti'
+ 'ro{sx}ili'
+ 'ro{sx}ila'
+ 'ro{sx}ilo'
+ 'ro{sx}ile'
+ 'ro{sx}im'
+ 'ro{sx}i{sx}'
+ 'ro{sx}it'
+ 'ro{sx}ih'
+ 'ro{sx}io' (<-'ro{sx}i')
+ 'o{sx}ijemu'
+ 'o{sx}ijima'
+ 'o{sx}ijega'
+ 'o{sx}ijeg'
+ 'o{sx}ijem'
+ 'o{sx}emu'
+ 'o{sx}ega'
+ 'o{sx}eg'
+ 'o{sx}em'
+ 'o{sx}ijim'
+ 'o{sx}ijih'
+ 'o{sx}ijoj'
+ 'o{sx}oga'
+ 'o{sx}ome'
+ 'o{sx}omu'
+ 'o{sx}ima'
+ 'o{sx}iji'
+ 'o{sx}ije'
+ 'o{sx}ija'
+ 'o{sx}iju'
+ 'o{sx}im'
+ 'o{sx}ih'
+ 'o{sx}oj'
+ 'o{sx}om'
+ 'o{sx}og'
+ 'o{sx}i'
+ 'o{sx}a'
+ 'o{sx}u'
+ 'o{sx}e' (<-'o{sx}')
+ 'evitijima'
+ 'evitijega'
+ 'evitijemu'
+ 'evitijem'
+ 'evitega'
+ 'evitemu'
+ 'evitem'
+ 'evitijim'
+ 'evitijih'
+ 'evitijoj'
+ 'evitijeg'
+ 'evitiji'
+ 'evitije'
+ 'evitija'
+ 'evitoga'
+ 'evitome'
+ 'evitomu'
+ 'evitima'
+ 'evitog'
+ 'evitom'
+ 'evitim'
+ 'evitih'
+ 'evitoj'
+ 'eviti'
+ 'evite'
+ 'evito'
+ 'evita'
+ 'evitu' (<-'evit')
+ 'ovitijima'
+ 'ovitijega'
+ 'ovitijemu'
+ 'ovitijem'
+ 'ovitega'
+ 'ovitemu'
+ 'ovitem'
+ 'ovitijim'
+ 'ovitijih'
+ 'ovitijoj'
+ 'ovitijeg'
+ 'ovitiji'
+ 'ovitije'
+ 'ovitija'
+ 'ovitoga'
+ 'ovitome'
+ 'ovitomu'
+ 'ovitima'
+ 'ovitog'
+ 'ovitom'
+ 'ovitim'
+ 'ovitih'
+ 'ovitoj'
+ 'oviti'
+ 'ovite'
+ 'ovito'
+ 'ovita'
+ 'ovitu' (<-'ovit')
+ 'astijima'
+ 'astijega'
+ 'astijemu'
+ 'astijem'
+ 'astega'
+ 'astemu'
+ 'astem'
+ 'astijim'
+ 'astijih'
+ 'astijoj'
+ 'astijeg'
+ 'astiji'
+ 'astije'
+ 'astija'
+ 'astoga'
+ 'astome'
+ 'astomu'
+ 'astima'
+ 'astog'
+ 'astom'
+ 'astim'
+ 'astih'
+ 'astoj'
+ 'asti'
+ 'aste'
+ 'asto'
+ 'asta'
+ 'astu' (<-'ast')
+ 'kijemu'
+ 'kijima'
+ 'kijega'
+ 'kijeg'
+ 'kijem'
+ 'kemu'
+ 'kega'
+ 'keg'
+ 'kem'
+ 'kijim'
+ 'kijih'
+ 'kijoj'
+ 'koga'
+ 'kome'
+ 'komu'
+ 'kima'
+ 'kiji'
+ 'kije'
+ 'kija'
+ 'kiju'
+ 'kim'
+ 'kih'
+ 'koj'
+ 'kom'
+ 'kog'
+ 'kov'
+ 'ki'
+ 'ka'
+ 'ku'
+ 'ke'
+ 'ko' (<-'k')
+ 'evaju{cy}i'
+ 'evasmo'
+ 'evaste'
+ 'evajmo'
+ 'evajte'
+ 'evaju'
+ 'evala'
+ 'evale'
+ 'evali'
+ 'evalo'
+ 'evamo'
+ 'evana'
+ 'evane'
+ 'evani'
+ 'evano'
+ 'evate'
+ 'evati'
+ 'eva{sx}e'
+ 'evahu'
+ 'evah'
+ 'evaj'
+ 'evam'
+ 'evan'
+ 'evao'
+ 'evat'
+ 'evav'
+ 'eva{sx}' (<-'eva')
+ 'avaju{cy}i'
+ 'avasmo'
+ 'avaste'
+ 'avajmo'
+ 'avajte'
+ 'avaju'
+ 'avala'
+ 'avale'
+ 'avali'
+ 'avalo'
+ 'avamo'
+ 'avana'
+ 'avane'
+ 'avani'
+ 'avano'
+ 'avate'
+ 'avati'
+ 'ava{sx}e'
+ 'avahu'
+ 'avah'
+ 'avaj'
+ 'avam'
+ 'avan'
+ 'avao'
+ 'avat'
+ 'avav'
+ 'ava{sx}' (<-'ava')
+ 'ivaju{cy}i'
+ 'ivasmo'
+ 'ivaste'
+ 'ivajmo'
+ 'ivajte'
+ 'ivaju'
+ 'ivala'
+ 'ivale'
+ 'ivali'
+ 'ivalo'
+ 'ivamo'
+ 'ivana'
+ 'ivane'
+ 'ivani'
+ 'ivano'
+ 'ivate'
+ 'ivati'
+ 'iva{sx}e'
+ 'ivahu'
+ 'ivah'
+ 'ivaj'
+ 'ivam'
+ 'ivan'
+ 'ivao'
+ 'ivat'
+ 'ivav'
+ 'iva{sx}' (<-'iva')
+ 'uvaju{cy}i'
+ 'uvasmo'
+ 'uvaste'
+ 'uvajmo'
+ 'uvajte'
+ 'uvaju'
+ 'uvala'
+ 'uvale'
+ 'uvali'
+ 'uvalo'
+ 'uvamo'
+ 'uvana'
+ 'uvane'
+ 'uvani'
+ 'uvano'
+ 'uvate'
+ 'uvati'
+ 'uva{sx}e'
+ 'uvahu'
+ 'uvah'
+ 'uvaj'
+ 'uvam'
+ 'uvan'
+ 'uvao'
+ 'uvat'
+ 'uvav'
+ 'uva{sx}' (<-'uva')
+ 'irujemo'
+ 'irujete'
+ 'iruju{cy}i'
+ 'iraju{cy}i'
+ 'irivat'
+ 'irujem'
+ 'iruje{sx}'
+ 'irujmo'
+ 'irujte'
+ 'irav{sx}i'
+ 'irasmo'
+ 'iraste'
+ 'irati'
+ 'iramo'
+ 'irate'
+ 'iraju'
+ 'ira{sx}e'
+ 'irahu'
+ 'irala'
+ 'iralo'
+ 'irali'
+ 'irale'
+ 'iruje'
+ 'iruju'
+ 'iruj'
+ 'iral'
+ 'iran'
+ 'iram'
+ 'ira{sx}'
+ 'irat'
+ 'irah'
+ 'irao' (<-'ir')
+ 'a{cx}ismo'
+ 'a{cx}iste'
+ 'a{cx}iti'
+ 'a{cx}imo'
+ 'a{cx}ite'
+ 'a{cx}i{sx}e'
+ 'a{cx}e{cy}i'
+ 'a{cx}ila'
+ 'a{cx}ilo'
+ 'a{cx}ili'
+ 'a{cx}ile'
+ 'a{cx}ena'
+ 'a{cx}eno'
+ 'a{cx}eni'
+ 'a{cx}ene'
+ 'a{cx}io'
+ 'a{cx}im'
+ 'a{cx}i{sx}'
+ 'a{cx}it'
+ 'a{cx}ih'
+ 'a{cx}en'
+ 'a{cx}i'
+ 'a{cx}e' (<-'a{cx}')
+ 'a{cx}av{sx}i'
+ 'a{cx}asmo'
+ 'a{cx}aste'
+ 'a{cx}ahu'
+ 'a{cx}ati'
+ 'a{cx}amo'
+ 'a{cx}ate'
+ 'a{cx}a{sx}e'
+ 'a{cx}ala'
+ 'a{cx}alo'
+ 'a{cx}ali'
+ 'a{cx}ale'
+ 'a{cx}aju'
+ 'a{cx}ana'
+ 'a{cx}ano'
+ 'a{cx}ani'
+ 'a{cx}ane'
+ 'a{cx}ao'
+ 'a{cx}am'
+ 'a{cx}a{sx}'
+ 'a{cx}at'
+ 'a{cx}ah'
+ 'a{cx}an' (<-'a{cx}a')
+ 'nuv{sx}i'
+ 'nusmo'
+ 'nuste'
+ 'nu{cy}i'
+ 'nimo'
+ 'nite'
+ 'nemo'
+ 'nete'
+ 'nula'
+ 'nulo'
+ 'nule'
+ 'nuli'
+ 'nuto'
+ 'nuti'
+ 'nuta'
+ 'ne{sx}'
+ 'nuo'
+ 'nut' (<-'n')
+ 'niv{sx}i'
+ 'nismo'
+ 'niste'
+ 'niti'
+ 'nila'
+ 'nilo'
+ 'nile'
+ 'nili'
+ 'ni{sx}'
+ 'nio' (<-'ni')
+ 'aju{cy}i'
+ 'av{sx}i'
+ 'asmo'
+ 'ajmo'
+ 'ajte'
+ 'ajem'
+ 'aloj'
+ 'amo'
+ 'ate'
+ 'aje'
+ 'aju'
+ 'ati'
+ 'a{sx}e'
+ 'ahu'
+ 'ala'
+ 'ali'
+ 'ale'
+ 'alo'
+ 'ano'
+ 'at'
+ 'ah'
+ 'ao'
+ 'aj'
+ 'an'
+ 'am'
+ 'a{sx}' (<-'a')
+ 'uraju{cy}i'
+ 'urasmo'
+ 'uraste'
+ 'urajmo'
+ 'urajte'
+ 'uramo'
+ 'urate'
+ 'uraju'
+ 'urati'
+ 'ura{sx}e'
+ 'urahu'
+ 'urala'
+ 'urali'
+ 'urale'
+ 'uralo'
+ 'urana'
+ 'urano'
+ 'urani'
+ 'urane'
+ 'ural'
+ 'urat'
+ 'urah'
+ 'urao'
+ 'uraj'
+ 'uran'
+ 'uram'
+ 'ura{sx}' (<-'ur')
+ 'astajasmo'
+ 'astajaste'
+ 'astajahu'
+ 'astajati'
+ 'astajemo'
+ 'astajete'
+ 'astaja{sx}e'
+ 'astajali'
+ 'astaju{cy}i'
+ 'astajala'
+ 'astajalo'
+ 'astajale'
+ 'astajmo'
+ 'astajao'
+ 'astajem'
+ 'astaje{sx}'
+ 'astajat'
+ 'astajah'
+ 'astajte'
+ 'astaje'
+ 'astaju' (<-'astaj')
+ 'istajasmo'
+ 'istajaste'
+ 'istajahu'
+ 'istajati'
+ 'istajemo'
+ 'istajete'
+ 'istaja{sx}e'
+ 'istajali'
+ 'istaju{cy}i'
+ 'istajala'
+ 'istajalo'
+ 'istajale'
+ 'istajmo'
+ 'istajao'
+ 'istajem'
+ 'istaje{sx}'
+ 'istajat'
+ 'istajah'
+ 'istajte'
+ 'istaje'
+ 'istaju' (<-'istaj')
+ 'ostajasmo'
+ 'ostajaste'
+ 'ostajahu'
+ 'ostajati'
+ 'ostajemo'
+ 'ostajete'
+ 'ostaja{sx}e'
+ 'ostajali'
+ 'ostaju{cy}i'
+ 'ostajala'
+ 'ostajalo'
+ 'ostajale'
+ 'ostajmo'
+ 'ostajao'
+ 'ostajem'
+ 'ostaje{sx}'
+ 'ostajat'
+ 'ostajah'
+ 'ostajte'
+ 'ostaje'
+ 'ostaju' (<-'ostaj')
+ 'alama'
+ 'alima'
+ 'alom'
+ 'alu'
+ 'al' (<-'a')
+ 'ajevima'
+ 'ajevi'
+ 'ajeva'
+ 'ajeve'
+ 'ajama'
+ 'ajima'
+ 'aja'
+ 'aji' (<-'aj')
+ 'astadosmo'
+ 'astadoste'
+ 'astado{sx}e'
+ 'astanemo'
+ 'astademo'
+ 'astanete'
+ 'astadete'
+ 'astanimo'
+ 'astanite'
+ 'astanila'
+ 'astav{sx}i'
+ 'astanem'
+ 'astadem'
+ 'astane{sx}'
+ 'astade{sx}'
+ 'astadoh'
+ 'astade'
+ 'astati'
+ 'astane'
+ 'astanu'
+ 'astadu'
+ 'astala'
+ 'astali'
+ 'astalo'
+ 'astale'
+ 'astat'
+ 'astao' (<-'asta')
+ 'istadosmo'
+ 'istadoste'
+ 'istado{sx}e'
+ 'istanemo'
+ 'istademo'
+ 'istanete'
+ 'istadete'
+ 'istanimo'
+ 'istanite'
+ 'istanila'
+ 'istav{sx}i'
+ 'istanem'
+ 'istadem'
+ 'istane{sx}'
+ 'istade{sx}'
+ 'istadoh'
+ 'istade'
+ 'istati'
+ 'istane'
+ 'istanu'
+ 'istadu'
+ 'istala'
+ 'istali'
+ 'istalo'
+ 'istale'
+ 'istat'
+ 'istao' (<-'ista')
+ 'ostadosmo'
+ 'ostadoste'
+ 'ostado{sx}e'
+ 'ostanemo'
+ 'ostademo'
+ 'ostanete'
+ 'ostadete'
+ 'ostanimo'
+ 'ostanite'
+ 'ostanila'
+ 'ostav{sx}i'
+ 'ostanem'
+ 'ostadem'
+ 'ostane{sx}'
+ 'ostade{sx}'
+ 'ostadoh'
+ 'ostade'
+ 'ostati'
+ 'ostane'
+ 'ostanu'
+ 'ostadu'
+ 'ostala'
+ 'ostali'
+ 'ostalo'
+ 'ostale'
+ 'ostat'
+ 'ostao' (<-'osta')
+ 'tasmo'
+ 'taste'
+ 'tajmo'
+ 'tajte'
+ 'tav{sx}i'
+ 'tati'
+ 'tamo'
+ 'tate'
+ 'taju'
+ 'tala'
+ 'talo'
+ 'tale'
+ 'tali'
+ 'tana'
+ 'tano'
+ 'tani'
+ 'tane'
+ 'tan'
+ 'taj'
+ 'tao'
+ 'tam'
+ 'ta{sx}'
+ 'tat'
+ 'tah' (<-'ta')
+ 'injasmo'
+ 'injaste'
+ 'injati'
+ 'injemo'
+ 'injete'
+ 'injali'
+ 'injala'
+ 'injalo'
+ 'injale'
+ 'inja{sx}e'
+ 'injahu'
+ 'injem'
+ 'inje{sx}'
+ 'injat'
+ 'injah'
+ 'injao' (<-'inj')
+ 'astemo'
+ 'astete'
+ 'astimo'
+ 'astite'
+ 'astu{cy}i'
+ 'aste{sx}'
+ 'asli'
+ 'asla'
+ 'aslo'
+ 'asle' (<-'as')
+ 'iv{sx}i'
+ 'ie{cy}i'
+ 'ismo'
+ 'imo'
+ 'ite'
+ 'iti'
+ 'ili'
+ 'ila'
+ 'ilo'
+ 'ile'
+ 'im'
+ 'i{sx}'
+ 'it'
+ 'ih'
+ 'io' (<-'i')
+ 'ijemo'
+ 'ijete'
+ 'ijem'
+ 'ije{sx}'
+ 'ijmo'
+ 'ijte'
+ 'iju'
+ 'ije'
+ 'ij'
+ 'ilu' (<-'i')
+ 'lu{cx}ujete'
+ 'lu{cx}uju{cy}i'
+ 'lu{cx}ujemo'
+ 'lu{cx}ujem'
+ 'lu{cx}uje{sx}'
+ 'lu{cx}ismo'
+ 'lu{cx}iste'
+ 'lu{cx}ujmo'
+ 'lu{cx}ujte'
+ 'lu{cx}uje'
+ 'lu{cx}uju'
+ 'lu{cx}i{sx}e'
+ 'lu{cx}iti'
+ 'lu{cx}imo'
+ 'lu{cx}ite'
+ 'lu{cx}ila'
+ 'lu{cx}ilo'
+ 'lu{cx}ili'
+ 'lu{cx}ile'
+ 'lu{cx}ena'
+ 'lu{cx}eno'
+ 'lu{cx}eni'
+ 'lu{cx}ene'
+ 'lu{cx}uj'
+ 'lu{cx}io'
+ 'lu{cx}en'
+ 'lu{cx}im'
+ 'lu{cx}i{sx}'
+ 'lu{cx}it'
+ 'lu{cx}ih'
+ 'lu{cx}e'
+ 'lu{cx}i' (<-'lu{cx}')
+ 'jetismo'
+ 'jetiste'
+ 'jeti{sx}e'
+ 'jetimo'
+ 'jetite'
+ 'jetiti'
+ 'jetili'
+ 'jetila'
+ 'jetilo'
+ 'jetile'
+ 'jetim'
+ 'jeti{sx}'
+ 'jetit'
+ 'jetih'
+ 'jetio' (<-'jeti')
+ 'emo'
+ 'em'
+ 'e{sx}'
+ 'elama'
+ 'el' (<-'e')
+ 'ilama'
+ 'ilima'
+ 'ilom'
+ 'il' (<-'i')
+ 'atijega'
+ 'atijemu'
+ 'atijima'
+ 'atijeg'
+ 'atijem'
+ 'atega'
+ 'atemu'
+ 'ateg'
+ 'atem'
+ 'atijih'
+ 'atijim'
+ 'atima'
+ 'atoga'
+ 'atome'
+ 'atomu'
+ 'atiji'
+ 'atije'
+ 'atija'
+ 'atiju'
+ 'atoj'
+ 'atog'
+ 'atom'
+ 'atim'
+ 'atih'
+ 'ata'
+ 'atu'
+ 'ato' (<-'at')
+ 'etav{sx}i'
+ 'etu{cy}i'
+ 'etemo'
+ 'etimo'
+ 'etem'
+ 'ete{sx}' (<-'et')
+ 'lucujuci'
+ 'lucujemo'
+ 'lucujete'
+ 'lucujem'
+ 'lucujes'
+ 'lucujmo'
+ 'lucujte'
+ 'lucismo'
+ 'luciste'
+ 'luciti'
+ 'lucite'
+ 'lucise'
+ 'lucuje'
+ 'lucuju'
+ 'lucila'
+ 'lucile'
+ 'lucili'
+ 'lucilo'
+ 'lucena'
+ 'luceni'
+ 'lucene'
+ 'luceno'
+ 'lucimo'
+ 'lucim'
+ 'lucis'
+ 'lucih'
+ 'lucit'
+ 'lucio'
+ 'lucuj'
+ 'lucen'
+ 'luce'
+ 'luci' (R2 <-'luc')
+ 'snjijima'
+ 'snjijemu'
+ 'snjijega'
+ 'snjijim'
+ 'snjijih'
+ 'snjijeg'
+ 'snjijoj'
+ 'snjiji'
+ 'snjija'
+ 'snjije'
+ 'snjiju'
+ 'snjima'
+ 'snjemu'
+ 'snjomu'
+ 'snjome'
+ 'snjega'
+ 'snjoga'
+ 'snjih'
+ 'snjim'
+ 'snjem'
+ 'snjom'
+ 'snjeg'
+ 'snjog'
+ 'snjoj'
+ 'snja'
+ 'snje'
+ 'snji'
+ 'snjo'
+ 'snju' (R2 <-'snj')
+ 'osijima'
+ 'osijemu'
+ 'osijega'
+ 'snjijem'
+ 'osijih'
+ 'osijim'
+ 'osijem'
+ 'osijeg'
+ 'osijoj'
+ 'osima'
+ 'osemu'
+ 'osomu'
+ 'osome'
+ 'osega'
+ 'osoga'
+ 'osija'
+ 'osije'
+ 'osiji'
+ 'osiju'
+ 'osih'
+ 'osim'
+ 'osem'
+ 'osom'
+ 'oseg'
+ 'osog'
+ 'osoj'
+ 'osa'
+ 'ose'
+ 'osi'
+ 'osu' (R2 <-'os')
+ 'acismo'
+ 'aciste'
+ 'acima'
+ 'acimo'
+ 'acome'
+ 'acomu'
+ 'acite'
+ 'aciti'
+ 'acise'
+ 'acila'
+ 'acile'
+ 'acili'
+ 'acilo'
+ 'acega'
+ 'acene'
+ 'aceci'
+ 'aceni'
+ 'acemu'
+ 'acena'
+ 'aceno'
+ 'acoga'
+ 'acoj'
+ 'acih'
+ 'acem'
+ 'acom'
+ 'acen'
+ 'acog'
+ 'acit'
+ 'acio'
+ 'aceg'
+ 'acim'
+ 'acuh'
+ 'acis'
+ 'ace'
+ 'aca'
+ 'aci' (R2 <-'ac')
+ 'ecome'
+ 'ecoga'
+ 'ecemu'
+ 'ecima'
+ 'ecega'
+ 'ecomu'
+ 'ecoj'
+ 'ecuh'
+ 'ecom'
+ 'ecog'
+ 'eceg'
+ 'ecih'
+ 'ecem'
+ 'ecim'
+ 'eca'
+ 'ece' (R2 <-'ec')
+ 'ucomu'
+ 'ucome'
+ 'ucima'
+ 'ucoga'
+ 'ucega'
+ 'ucemu'
+ 'ucih'
+ 'ucog'
+ 'uceg'
+ 'ucom'
+ 'ucem'
+ 'ucim'
+ 'ucuh'
+ 'ucoj'
+ 'uca'
+ 'uce' (R2 <-'uc')
+ 'rosismo'
+ 'rosivsi'
+ 'rosiste'
+ 'rositi'
+ 'rosili'
+ 'rosise'
+ 'rosite'
+ 'rosilo'
+ 'rosimo'
+ 'rosile'
+ 'rosila'
+ 'rosit'
+ 'rosis'
+ 'rosio'
+ 'rosim'
+ 'rosih' (R2 <-'rosi')
+ 'acavsi'
+ 'acaste'
+ 'acasmo'
+ 'acaju'
+ 'acane'
+ 'acate'
+ 'acali'
+ 'acani'
+ 'acati'
+ 'acale'
+ 'acahu'
+ 'acase'
+ 'acano'
+ 'acamo'
+ 'acalo'
+ 'acana'
+ 'acala'
+ 'acam'
+ 'acan'
+ 'acao'
+ 'acas'
+ 'acat'
+ 'acah' (R2 <-'aca')
+ 'jasima'
+ 'jasama'
+ 'jasem'
+ 'jasom'
+ 'jase'
+ 'jasi'
+ 'jasa'
+ 'jasu' (R2 <-'jas')
+ 'tasima'
+ 'tasama'
+ 'tasem'
+ 'tasom'
+ 'tase'
+ 'tasa'
+ 'tasu'
+ 'tasi' (R2 <-'tas')
+ 'gasima'
+ 'gasama'
+ 'gasem'
+ 'gasom'
+ 'gasi'
+ 'gasu'
+ 'gase'
+ 'gasa' (R2 <-'gas')
+ 'nasama'
+ 'nasima'
+ 'nasem'
+ 'nasom'
+ 'nasu'
+ 'nasi'
+ 'nase'
+ 'nasa' (R2 <-'nas')
+ 'kasama'
+ 'kasima'
+ 'kasom'
+ 'kasem'
+ 'kasi'
+ 'kasu'
+ 'kase'
+ 'kasa' (R2 <-'kas')
+ 'vasama'
+ 'vasima'
+ 'vasom'
+ 'vasem'
+ 'vasi'
+ 'vase'
+ 'vasa'
+ 'vasu' (R2 <-'vas')
+ 'basama'
+ 'basima'
+ 'basom'
+ 'basem'
+ 'basi'
+ 'base'
+ 'basu'
+ 'basa' (R2 <-'bas')
+ 'astuci'
+ 'astes' (R2 <-'as')
+ 'cinima'
+ 'cinome'
+ 'cinama'
+ 'cinomu'
+ 'cinoga'
+ 'cinom'
+ 'cinih'
+ 'cinim'
+ 'cinog'
+ 'cinoj'
+ 'cino'
+ 'cini'
+ 'cinu'
+ 'cine'
+ 'cina' (R2 <-'cin')
+ 'astajase'
+ 'astajuci'
+ 'astajes' (R2 <-'astaj')
+ 'istajase'
+ 'istajuci'
+ 'istajes' (R2 <-'istaj')
+ 'ostajase'
+ 'ostajuci'
+ 'ostajes' (R2 <-'ostaj')
+ 'astadose'
+ 'astades'
+ 'astanes'
+ 'astavsi' (R2 <-'asta')
+ 'istadose'
+ 'istades'
+ 'istanes'
+ 'istavsi' (R2 <-'ista')
+ 'ostadose'
+ 'ostades'
+ 'ostanes'
+ 'ostavsi' (R2 <-'osta')
+ 'avajuci'
+ 'avase'
+ 'avas' (R2 <-'ava')
+ 'evajuci'
+ 'evase'
+ 'evas' (R2 <-'eva')
+ 'ivajuci'
+ 'ivase'
+ 'ivas' (R2 <-'iva')
+ 'uvajuci'
+ 'uvase'
+ 'uvas' (R2 <-'uva')
+ 'ovase' (R2 <-'ova')
+ 'jetise'
+ 'jetis' (R2 <-'jeti')
+ 'injase'
+ 'injes' (R2 <-'inj')
+ 'istem' (R2 <-'ist')
+ 'esama'
+ 'esem'
+ 'esi' (R2 <-'es')
+ 'etavsi'
+ 'etuci'
+ 'etes' (R2 <-'et')
+ 'isama'
+ 'isem'
+ 'isi' (R2 <-'is')
+ 'irajuci'
+ 'irujuci'
+ 'irujes'
+ 'iravsi'
+ 'irase'
+ 'iras' (R2 <-'ir')
+ 'urajuci'
+ 'urase'
+ 'uras' (R2 <-'ur')
+ 'ujuci'
+ 'ujes' (R2 <-'uj')
+ 'nivsi'
+ 'nis' (R2 <-'ni')
+ 'snega'
+ 'snemu'
+ 'snem'
+ 'sneg' (R2 <-'sn')
+ 'tavsi'
+ 'tas' (R2 <-'ta')
+ 'ajuci'
+ 'avsi'
+ 'ase'
+ 'as' (R2 <-'a')
+ 'ijes'
+ 'ivsi'
+ 'ieci'
+ 'is' (R2 <-'i')
+ 'es' (R2 <-'e')
+ 'nuvsi'
+ 'nuci'
+ 'nes' (R2 <-'n')
+ )
+ )
+
+ define Step_3 as (
+ [substring] R1 among (
+ 'enom'
+ 'enoj'
+ 'enog'
+ 'enim'
+ 'enih'
+ 'anoj'
+ 'anog'
+ 'anim'
+ 'anih'
+ 'ost'
+ 'eno'
+ 'eni'
+ 'oga'
+ 'ima'
+ 'enu'
+ 'ena'
+ 'ama'
+ 'ano'
+ 'ani'
+ 'om'
+ 'og'
+ 'u'
+ 'o'
+ 'i'
+ 'e'
+ 'a' (<-'')
+ )
+ )
+)
+
+define stem as (
+ do cyr_to_lat
+ do prelude
+ do mark_regions
+ backwards (
+ do Step_1
+ do (Step_2 or Step_3)
+ )
+)
diff --git a/contrib/snowball/algorithms/spanish.sbl b/contrib/snowball/algorithms/spanish.sbl
new file mode 100644
index 0000000..6638f5f
--- /dev/null
+++ b/contrib/snowball/algorithms/spanish.sbl
@@ -0,0 +1,230 @@
+routines (
+ postlude mark_regions
+ RV R1 R2
+ attached_pronoun
+ standard_suffix
+ y_verb_suffix
+ verb_suffix
+ residual_suffix
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a' '{U+00E1}' // a-acute
+stringdef e' '{U+00E9}' // e-acute
+stringdef i' '{U+00ED}' // i-acute
+stringdef o' '{U+00F3}' // o-acute
+stringdef u' '{U+00FA}' // u-acute
+stringdef u" '{U+00FC}' // u-diaeresis
+stringdef n~ '{U+00F1}' // n-tilde
+
+define v 'aeiou{a'}{e'}{i'}{o'}{u'}{u"}'
+
+define mark_regions as (
+
+ $pV = limit
+ $p1 = limit
+ $p2 = limit // defaults
+
+ do (
+ ( v (non-v gopast v) or (v gopast non-v) )
+ or
+ ( non-v (non-v gopast v) or (v next) )
+ setmark pV
+ )
+ do (
+ gopast v gopast non-v setmark p1
+ gopast v gopast non-v setmark p2
+ )
+)
+
+define postlude as repeat (
+ [substring] among(
+ '{a'}' (<- 'a')
+ '{e'}' (<- 'e')
+ '{i'}' (<- 'i')
+ '{o'}' (<- 'o')
+ '{u'}' (<- 'u')
+ // and possibly {u"}->u here, or in prelude
+ '' (next)
+ ) //or next
+)
+
+backwardmode (
+
+ define RV as $pV <= cursor
+ define R1 as $p1 <= cursor
+ define R2 as $p2 <= cursor
+
+ define attached_pronoun as (
+ [substring] among(
+ 'me' 'se' 'sela' 'selo' 'selas' 'selos' 'la' 'le' 'lo'
+ 'las' 'les' 'los' 'nos'
+ )
+ substring RV among(
+ 'i{e'}ndo' (] <- 'iendo')
+ '{a'}ndo' (] <- 'ando')
+ '{a'}r' (] <- 'ar')
+ '{e'}r' (] <- 'er')
+ '{i'}r' (] <- 'ir')
+ 'ando'
+ 'iendo'
+ 'ar' 'er' 'ir'
+ (delete)
+ 'yendo' ('u' delete)
+ )
+ )
+
+ define standard_suffix as (
+ [substring] among(
+
+ 'anza' 'anzas'
+ 'ico' 'ica' 'icos' 'icas'
+ 'ismo' 'ismos'
+ 'able' 'ables'
+ 'ible' 'ibles'
+ 'ista' 'istas'
+ 'oso' 'osa' 'osos' 'osas'
+ 'amiento' 'amientos'
+ 'imiento' 'imientos'
+ (
+ R2 delete
+ )
+ 'adora' 'ador' 'aci{o'}n'
+ 'adoras' 'adores' 'aciones'
+ 'ante' 'antes' 'ancia' 'ancias'// Note 1
+ (
+ R2 delete
+ try ( ['ic'] R2 delete )
+ )
+ 'log{i'}a'
+ 'log{i'}as'
+ (
+ R2 <- 'log'
+ )
+ 'uci{o'}n' 'uciones'
+ (
+ R2 <- 'u'
+ )
+ 'encia' 'encias'
+ (
+ R2 <- 'ente'
+ )
+ 'amente'
+ (
+ R1 delete
+ try (
+ [substring] R2 delete among(
+ 'iv' (['at'] R2 delete)
+ 'os'
+ 'ic'
+ 'ad'
+ )
+ )
+ )
+ 'mente'
+ (
+ R2 delete
+ try (
+ [substring] among(
+ 'ante' // Note 1
+ 'able'
+ 'ible' (R2 delete)
+ )
+ )
+ )
+ 'idad'
+ 'idades'
+ (
+ R2 delete
+ try (
+ [substring] among(
+ 'abil'
+ 'ic'
+ 'iv' (R2 delete)
+ )
+ )
+ )
+ 'iva' 'ivo'
+ 'ivas' 'ivos'
+ (
+ R2 delete
+ try (
+ ['at'] R2 delete // but not a further ['ic'] R2 delete
+ )
+ )
+ )
+ )
+
+ define y_verb_suffix as (
+ setlimit tomark pV for ([substring]) among(
+ 'ya' 'ye' 'yan' 'yen' 'yeron' 'yendo' 'yo' 'y{o'}'
+ 'yas' 'yes' 'yais' 'yamos'
+ ('u' delete)
+ )
+ )
+
+ define verb_suffix as (
+ setlimit tomark pV for ([substring]) among(
+
+ 'en' 'es' '{e'}is' 'emos'
+ (try ('u' test 'g') ] delete)
+
+ 'ar{i'}an' 'ar{i'}as' 'ar{a'}n' 'ar{a'}s' 'ar{i'}ais'
+ 'ar{i'}a' 'ar{e'}is' 'ar{i'}amos' 'aremos' 'ar{a'}'
+ 'ar{e'}'
+ 'er{i'}an' 'er{i'}as' 'er{a'}n' 'er{a'}s' 'er{i'}ais'
+ 'er{i'}a' 'er{e'}is' 'er{i'}amos' 'eremos' 'er{a'}'
+ 'er{e'}'
+ 'ir{i'}an' 'ir{i'}as' 'ir{a'}n' 'ir{a'}s' 'ir{i'}ais'
+ 'ir{i'}a' 'ir{e'}is' 'ir{i'}amos' 'iremos' 'ir{a'}'
+ 'ir{e'}'
+
+ 'aba' 'ada' 'ida' '{i'}a' 'ara' 'iera' 'ad' 'ed'
+ 'id' 'ase' 'iese' 'aste' 'iste' 'an' 'aban' '{i'}an'
+ 'aran' 'ieran' 'asen' 'iesen' 'aron' 'ieron' 'ado'
+ 'ido' 'ando' 'iendo' 'i{o'}' 'ar' 'er' 'ir' 'as'
+ 'abas' 'adas' 'idas' '{i'}as' 'aras' 'ieras' 'ases'
+ 'ieses' '{i'}s' '{a'}is' 'abais' '{i'}ais' 'arais'
+ 'ierais' 'aseis' 'ieseis' 'asteis' 'isteis' 'ados'
+ 'idos' 'amos' '{a'}bamos' '{i'}amos' 'imos'
+ '{a'}ramos' 'i{e'}ramos' 'i{e'}semos' '{a'}semos'
+ (delete)
+ )
+ )
+
+ define residual_suffix as (
+ [substring] among(
+ 'os'
+ 'a' 'o' '{a'}' '{i'}' '{o'}'
+ ( RV delete )
+ 'e' '{e'}'
+ ( RV delete try( ['u'] test 'g' RV delete ) )
+ )
+ )
+)
+
+define stem as (
+ do mark_regions
+ backwards (
+ do attached_pronoun
+ do ( standard_suffix or
+ y_verb_suffix or
+ verb_suffix
+ )
+ do residual_suffix
+ )
+ do postlude
+)
+
+/*
+ Note 1: additions of 15 Jun 2005
+*/
diff --git a/contrib/snowball/algorithms/swedish.sbl b/contrib/snowball/algorithms/swedish.sbl
new file mode 100644
index 0000000..2cbb885
--- /dev/null
+++ b/contrib/snowball/algorithms/swedish.sbl
@@ -0,0 +1,72 @@
+routines (
+ mark_regions
+ main_suffix
+ consonant_pair
+ other_suffix
+)
+
+externals ( stem )
+
+integers ( p1 x )
+
+groupings ( v s_ending )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a" '{U+00E4}'
+stringdef ao '{U+00E5}'
+stringdef o" '{U+00F6}'
+
+define v 'aeiouy{a"}{ao}{o"}'
+
+define s_ending 'bcdfghjklmnoprtvy'
+
+define mark_regions as (
+
+ $p1 = limit
+ test ( hop 3 setmark x )
+ goto v gopast non-v setmark p1
+ try ( $p1 < x $p1 = x )
+)
+
+backwardmode (
+
+ define main_suffix as (
+ setlimit tomark p1 for ([substring])
+ among(
+
+ 'a' 'arna' 'erna' 'heterna' 'orna' 'ad' 'e' 'ade' 'ande' 'arne'
+ 'are' 'aste' 'en' 'anden' 'aren' 'heten' 'ern' 'ar' 'er' 'heter'
+ 'or' 'as' 'arnas' 'ernas' 'ornas' 'es' 'ades' 'andes' 'ens' 'arens'
+ 'hetens' 'erns' 'at' 'andet' 'het' 'ast'
+ (delete)
+ 's'
+ (s_ending delete)
+ )
+ )
+
+ define consonant_pair as setlimit tomark p1 for (
+ among('dd' 'gd' 'nn' 'dt' 'gt' 'kt' 'tt')
+ and ([next] delete)
+ )
+
+ define other_suffix as setlimit tomark p1 for (
+ [substring] among(
+ 'lig' 'ig' 'els' (delete)
+ 'l{o"}st' (<-'l{o"}s')
+ 'fullt' (<-'full')
+ )
+ )
+)
+
+define stem as (
+
+ do mark_regions
+ backwards (
+ do main_suffix
+ do consonant_pair
+ do other_suffix
+ )
+)
diff --git a/contrib/snowball/algorithms/tamil.sbl b/contrib/snowball/algorithms/tamil.sbl
new file mode 100644
index 0000000..9635777
--- /dev/null
+++ b/contrib/snowball/algorithms/tamil.sbl
@@ -0,0 +1,405 @@
+/*
+* Affix stripping stemming algorithm for Tamil
+* By Damodharan Rajalingam
+*/
+
+stringescapes {}
+
+/* Aytham */
+stringdef aytham '{U+0B83}'
+
+/* Uyir - independent vowels */
+stringdef a '{U+0B85}'
+stringdef aa '{U+0B86}'
+stringdef i '{U+0B87}'
+stringdef ii '{U+0B88}'
+stringdef u '{U+0B89}'
+stringdef uu '{U+0B8A}'
+stringdef e '{U+0B8E}'
+stringdef ee '{U+0B8F}'
+stringdef ai '{U+0B90}'
+stringdef o '{U+0B92}'
+stringdef oo '{U+0B93}'
+stringdef au '{U+0B94}'
+
+/* Consonants */
+stringdef ka '{U+0B95}'
+stringdef nga '{U+0B99}'
+stringdef ca '{U+0B9A}'
+stringdef ja '{U+0B9C}'
+stringdef nya '{U+0B9E}'
+stringdef tta '{U+0B9F}'
+stringdef nna '{U+0BA3}'
+stringdef ta '{U+0BA4}'
+stringdef tha '{U+0BA4}'
+stringdef na '{U+0BA8}'
+stringdef nnna '{U+0BA9}'
+stringdef pa '{U+0BAA}'
+stringdef ma '{U+0BAE}'
+stringdef ya '{U+0BAF}'
+stringdef ra '{U+0BB0}'
+stringdef rra '{U+0BB1}'
+stringdef la '{U+0BB2}'
+stringdef lla '{U+0BB3}'
+stringdef llla '{U+0BB4}'
+stringdef zha '{U+0BB4}'
+stringdef va '{U+0BB5}'
+
+/* Vatamozi - borrowed */
+stringdef sha '{U+0BB6}'
+stringdef ssa '{U+0BB7}'
+stringdef sa '{U+0BB8}'
+stringdef ha '{U+0BB9}'
+
+
+/* Dependent vowel signs (kombu etc.) */
+stringdef vs_aa '{U+0BBE}'
+stringdef vs_i '{U+0BBF}'
+stringdef vs_ii '{U+0BC0}'
+stringdef vs_u '{U+0BC1}'
+stringdef vs_uu '{U+0BC2}'
+stringdef vs_e '{U+0BC6}'
+stringdef vs_ee '{U+0BC7}'
+stringdef vs_ai '{U+0BC8}'
+stringdef vs_o '{U+0BCA}'
+stringdef vs_oo '{U+0BCB}'
+stringdef vs_au '{U+0BCC}'
+
+/* Pulli */
+stringdef pulli '{U+0BCD}'
+
+/* AU length markk */
+stringdef au_lmark '{U+0BD7}'
+
+
+routines (
+ remove_plural_suffix
+ remove_question_suffixes
+ remove_question_prefixes
+ remove_pronoun_prefixes
+ remove_command_suffixes
+ remove_um
+ remove_vetrumai_urupukal
+ fix_va_start
+ fix_ending
+ fix_endings
+ remove_tense_suffix
+ remove_tense_suffixes
+ remove_common_word_endings
+ has_min_length
+)
+
+externals ( stem )
+
+booleans (
+ found_a_match
+ found_vetrumai_urupu
+)
+
+define has_min_length as (
+ $(len > 4)
+)
+
+define fix_va_start as (
+ (try '{va}{vs_oo}' and [ '{va}{vs_oo}' ] <- '{oo}' ) or
+ (try '{va}{vs_o}' and [ '{va}{vs_o}' ] <- '{o}' ) or
+ (try '{va}{vs_u}' and [ '{va}{vs_u}' ] <- '{u}' ) or
+ (try '{va}{vs_uu}' and [ '{va}{vs_uu}' ] <- '{uu}' )
+)
+
+define fix_endings as (
+ do repeat fix_ending
+)
+
+define remove_question_prefixes as (
+ [ ('{e}' ) among('{ka}' '{ca}' '{tha}' '{va}' '{na}' '{pa}' '{ma}' '{ya}' '{nga}' '{nya}') '{pulli}' ] delete
+ do fix_va_start
+)
+
+// Gives signal t if an ending was fixed, signal f otherwise.
+define fix_ending as (
+ $(len > 3)
+ backwards (
+ ( [among('{na}{pulli}' '{na}{pulli}{ta}' '{na}{pulli}{ta}{pulli}') ] delete )
+ or
+ ( ['{ya}{pulli}' test among('{vs_ai}' '{vs_i}' '{vs_ii}') ] delete )
+ or
+ ( [ '{tta}{pulli}{pa}{pulli}' or '{tta}{pulli}{ka}{pulli}' ] <- '{lla}{pulli}' )
+ or
+ ( [ '{nnna}{pulli}{rra}{pulli}' ] <- '{la}{pulli}' )
+ or
+// ( [ '{rra}{pulli}{ka}{pulli}' or '{nnna}{pulli}{nnna}{pulli}' ] <- '{la}{pulli}' )
+ ( [ '{rra}{pulli}{ka}{pulli}' ] <- '{la}{pulli}' )
+ or
+ ( [ '{tta}{pulli}{tta}{pulli}' ] <- '{tta}{vs_u}' )
+ or
+ ( found_vetrumai_urupu [ '{ta}{pulli}{ta}{pulli}' (test not '{vs_ai}') ] <- '{ma}{pulli}' ] )
+ or
+ ( [ '{vs_u}{ka}{pulli}' or '{vs_u}{ka}{pulli}{ka}{pulli}' ] <- '{pulli}' )
+ or
+ ( [ '{pulli}' among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') '{pulli}' among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') ] delete )
+ or
+ ( [ '{vs_u}{ka}{pulli}' ] <- '{pulli}' )
+ or
+ ( [ '{pulli}' among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') ] delete )
+ or
+ ( [ '{pulli}' (among('{ya}' '{ra}' '{la}' '{va}' '{zha}' '{lla}') or among('{nga}' '{nya}' '{nna}' '{na}' '{ma}' '{nnna}')) '{pulli}' ] <- '{pulli}' )
+ or
+ ( [ among('{va}' '{ya}' '{va}{pulli}') ] delete )
+ or
+ ( [ '{nnna}{vs_u}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}')) ] delete )
+ or
+ ( [ '{nga}{pulli}' (test not '{vs_ai}')] <- '{ma}{pulli}' )
+ or
+ ( [ '{nga}{pulli}' ] delete )
+ or
+ ( [ '{pulli}' (test (among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}') or '{pulli}')) ] delete )
+ )
+)
+
+define remove_pronoun_prefixes as (
+ unset found_a_match
+ [ among('{a}' '{i}' '{u}') among('{ka}' '{ca}' '{tha}' '{va}' '{na}' '{pa}' '{ma}' '{ya}' '{nga}' '{nya}') '{pulli}' ] delete
+ (set found_a_match)
+ do fix_va_start
+)
+
+define remove_plural_suffix as (
+ unset found_a_match
+ backwards (
+ ( [ '{vs_u}{nga}{pulli}{ka}{lla}{pulli}' (test not among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}')) ] <- '{pulli}' ) or
+ ( [ '{rra}{pulli}{ka}{lla}{pulli}' ] <- '{la}{pulli}' ) or
+ ( [ '{tta}{pulli}{ka}{lla}{pulli}' ] <- '{lla}{pulli}' ) or
+ ( [ '{ka}{lla}{pulli}' ] delete )
+ (set found_a_match)
+ )
+)
+
+define remove_question_suffixes as (
+ has_min_length
+ unset found_a_match
+ backwards (
+ do (
+ [ among('{vs_oo}' '{vs_ee}' '{vs_aa}') ] <- '{pulli}'
+ (set found_a_match)
+ )
+ )
+ do fix_endings
+)
+
+define remove_command_suffixes as (
+ has_min_length
+ unset found_a_match
+ backwards (
+ [ among('{pa}{vs_i}' '{va}{vs_i}') ] delete
+ (set found_a_match)
+ )
+)
+
+define remove_um as (
+ unset found_a_match
+ has_min_length
+ backwards ( [ '{vs_u}{ma}{pulli}' ] <- '{pulli}'
+ (set found_a_match)
+ )
+ do fix_ending
+)
+
+define remove_common_word_endings as (
+ // These are not suffixes actually but are
+ // some words that are attached to other words
+ // but can be removed for stemming
+ unset found_a_match
+ has_min_length
+ backwards (
+ test ( [ '{vs_u}{tta}{nnna}{pulli}' or
+ '{vs_i}{la}{pulli}{la}{vs_ai}' or
+ '{vs_i}{tta}{ma}{pulli}' or
+ '{vs_i}{nnna}{pulli}{rra}{vs_i}' or
+ '{vs_aa}{ka}{vs_i}' or
+ '{vs_aa}{ka}{vs_i}{ya}' or
+ '{vs_e}{nnna}{pulli}{rra}{vs_u}' or
+ '{vs_u}{lla}{pulli}{lla}' or
+ '{vs_u}{tta}{vs_ai}{ya}' or
+ '{vs_u}{tta}{vs_ai}' or
+ '{vs_e}{nnna}{vs_u}{ma}{pulli}' or
+ ('{la}{pulli}{la}' test (not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}'))) or
+ '{vs_e}{nnna}' or
+ '{vs_aa}{ka}{vs_i}' ] <- '{pulli}'
+ (set found_a_match)
+ )
+ or
+ test ( [ among('{pa}{tta}{vs_u}'
+ '{pa}{tta}{pulli}{tta}'
+ '{pa}{tta}{pulli}{tta}{vs_u}'
+ '{pa}{tta}{pulli}{tta}{ta}{vs_u}'
+ '{pa}{tta}{pulli}{tta}{nna}'
+ '{ka}{vs_u}{ra}{vs_i}{ya}'
+ '{pa}{rra}{pulli}{rra}{vs_i}'
+ '{va}{vs_i}{tta}{vs_u}'
+ '{va}{vs_i}{tta}{pulli}{tta}{vs_u}'
+ '{pa}{tta}{vs_i}{ta}{vs_aa}{nnna}'
+ '{pa}{tta}{vs_i}'
+ '{ta}{vs_aa}{nnna}'
+ '{vs_e}{la}{pulli}{la}{vs_aa}{ma}{pulli}')
+ ] delete
+ (set found_a_match)
+ )
+ )
+ do fix_endings
+)
+
+define remove_vetrumai_urupukal as (
+ unset found_a_match
+ unset found_vetrumai_urupu
+ has_min_length
+ backwards (
+ (
+ test ( ['{nnna}{vs_ai}'] delete )
+ or
+ test ([ ( '{vs_i}{nnna}{vs_ai}' or
+ '{vs_ai}' (test not among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}'))) or
+ ( '{vs_ai}' (test (among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') '{pulli}')))
+ ] <- '{pulli}'
+ )
+ or
+ test ( [
+ '{vs_o}{tta}{vs_u}' or
+ '{vs_oo}{tta}{vs_u}' or
+ '{vs_i}{la}{pulli}' or
+ '{vs_i}{rra}{pulli}' or
+ ('{vs_i}{nnna}{pulli}' (test not '{ma}')) or
+ '{vs_i}{nnna}{pulli}{rra}{vs_u}' or
+ '{vs_i}{ra}{vs_u}{na}{pulli}{ta}{vs_u}' or
+ '{va}{vs_i}{tta}' or
+ ($(len >= 7) '{vs_i}{tta}{ma}{pulli}') or
+ '{vs_aa}{la}{pulli}' or
+ '{vs_u}{tta}{vs_ai}' or
+ '{vs_aa}{ma}{la}{pulli}' or
+ ('{la}{pulli}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}'))) or
+ '{vs_u}{lla}{pulli}'
+ ] <- '{pulli}'
+ )
+ or
+ test ( [
+ '{ka}{nna}{pulli}' or
+ '{ma}{vs_u}{nnna}{pulli}' or
+ '{ma}{vs_ee}{la}{pulli}' or
+ '{ma}{vs_ee}{rra}{pulli}' or
+ '{ka}{vs_ii}{llla}{pulli}' or
+ '{pa}{vs_i}{nnna}{pulli}' or
+ ('{ta}{vs_u}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}')))
+ ] delete
+ )
+ or
+ test ([ '{vs_ii}' ] <- '{vs_i}')
+ )
+ (set found_a_match)
+ (set found_vetrumai_urupu)
+ do ( [ '{vs_i}{nnna}{pulli}' ] <- '{pulli}' )
+ )
+ do fix_endings
+)
+
+define remove_tense_suffixes as (
+ set found_a_match
+ repeat ( found_a_match (do remove_tense_suffix) )
+)
+
+define remove_tense_suffix as (
+ unset found_a_match
+ has_min_length
+ backwards (
+ do (
+ test ( [among(
+ '{ka}{vs_o}{nna}{pulli}{tta}{vs_i}{ra}{pulli}'
+ '{pa}{tta}{vs_u}'
+ )] delete
+ (set found_a_match)
+ )
+ or
+ test ( [
+ '{ma}{vs_aa}{ra}{pulli}' or
+ '{ma}{vs_i}{nnna}{pulli}' or
+ '{nnna}{nnna}{pulli}' or
+ '{nnna}{vs_aa}{nnna}{pulli}' or
+ '{nnna}{vs_aa}{lla}{pulli}' or
+ '{nnna}{vs_aa}{ra}{pulli}' or
+ ('{va}{nnna}{pulli}' test (not among('{a}' '{aa}' '{i}' '{ii}' '{u}' '{uu}' '{e}' '{ee}' '{ai}' '{o}' '{oo}' '{au}')) ) or
+ '{nnna}{lla}{pulli}' or
+ '{va}{lla}{pulli}' or
+ '{nnna}{ra}{pulli}' or
+ '{va}{ra}{pulli}' or
+ '{nnna}' or '{pa}' or '{ka}' or '{ta}' or '{ya}' or
+ '{pa}{nnna}{pulli}' or
+ '{pa}{lla}{pulli}' or
+ '{pa}{ra}{pulli}' or
+ ('{ta}{vs_u}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}'))) or
+ '{vs_i}{rra}{pulli}{rra}{vs_u}' or
+ '{pa}{ma}{pulli}' or
+ '{nnna}{ma}{pulli}' or
+ '{ta}{vs_u}{ma}{pulli}' or
+ '{rra}{vs_u}{ma}{pulli}' or
+ '{ka}{vs_u}{ma}{pulli}' or
+ '{nnna}{vs_e}{nnna}{pulli}' or
+ '{nnna}{vs_ai}' or
+ '{va}{vs_ai}'
+ ] delete
+ (set found_a_match)
+ )
+ or
+ test ( [
+ ('{vs_aa}{nnna}{pulli}' test (not '{ca}')) or
+ '{vs_aa}{lla}{pulli}' or
+ '{vs_aa}{ra}{pulli}' or
+ '{vs_ee}{nnna}{pulli}' or
+ '{vs_aa}' or
+ '{vs_aa}{ma}{pulli}' or
+ '{vs_e}{ma}{pulli}' or
+ '{vs_ee}{ma}{pulli}' or
+ '{vs_oo}{ma}{pulli}' or
+ '{ka}{vs_u}{ma}{pulli}' or
+ '{ta}{vs_u}{ma}{pulli}' or
+ '{tta}{vs_u}{ma}{pulli}' or
+ '{rra}{vs_u}{ma}{pulli}' or
+ '{vs_aa}{ya}{pulli}' or
+ '{nnna}{vs_e}{nnna}{pulli}' or
+ '{nnna}{vs_i}{ra}{pulli}' or
+ '{vs_ii}{ra}{pulli}' or
+ '{vs_ii}{ya}{ra}{pulli}'
+ ] <- '{pulli}'
+ (set found_a_match)
+ )
+ or
+ test ( ([ '{ka}{vs_u}' or '{ta}{vs_u}' ) (test '{pulli}') ] delete
+ (set found_a_match)
+ )
+ )
+ do ([among(
+ '{vs_aa}{na}{vs_i}{nnna}{pulli}{rra}'
+ '{vs_aa}{na}{vs_i}{nnna}{pulli}{rra}{pulli}'
+ '{ka}{vs_i}{nnna}{pulli}{rra}'
+ '{ka}{vs_i}{nnna}{pulli}{rra}{pulli}'
+ '{ka}{vs_i}{rra}'
+ '{ka}{vs_i}{rra}{pulli}'
+ )] delete
+ (set found_a_match)
+ )
+ )
+ do fix_endings
+)
+
+define stem as (
+ unset found_vetrumai_urupu
+ do fix_ending
+ has_min_length
+ do remove_question_prefixes
+ do remove_pronoun_prefixes
+ do remove_question_suffixes
+ do remove_um
+ do remove_common_word_endings
+ do remove_vetrumai_urupukal
+ do remove_plural_suffix
+ do remove_command_suffixes
+ do remove_tense_suffixes
+)
diff --git a/contrib/snowball/algorithms/turkish.sbl b/contrib/snowball/algorithms/turkish.sbl
new file mode 100644
index 0000000..eadd61d
--- /dev/null
+++ b/contrib/snowball/algorithms/turkish.sbl
@@ -0,0 +1,470 @@
+/* Stemmer for Turkish
+ * author: Evren (Kapusuz) Çilden
+ * email: evren.kapusuz at gmail.com
+ * version: 1.0 (15.01.2007)
+
+
+ * stems nominal verb suffixes
+ * stems nominal inflections
+ * more than one syllable word check
+ * (y,n,s,U) context check
+ * vowel harmony check
+ * last consonant check and conversion (b, c, d, ğ to p, ç, t, k)
+
+ * The stemming algorithm is based on the paper "An Affix Stripping
+ * Morphological Analyzer for Turkish" by Gülşen Eryiğit and
+ * Eşref Adalı (Proceedings of the IAESTED International Conference
+ * ARTIFICIAL INTELLIGENCE AND APPLICATIONS, February 16-18,2004,
+ * Innsbruck, Austria
+
+ * Turkish is an agglutinative language and has a very rich morphological
+ * structure. In Turkish, you can form many different words from a single stem
+ * by appending a sequence of suffixes. Eg. The word "doktoruymuÅŸsunuz" means
+ * "You had been the doctor of him". The stem of the word is "doktor" and it
+ * takes three different suffixes -sU, -ymUs, and -sUnUz. The rules about
+ * the append order of suffixes can be clearly described as FSMs.
+ * The paper referenced above defines some FSMs for right to left
+ * morphological analysis. I generated a method for constructing snowball
+ * expressions from right to left FSMs for stemming suffixes.
+*/
+
+routines (
+ append_U_to_stems_ending_with_d_or_g // for preventing some overstemmings
+ check_vowel_harmony // tests vowel harmony for suffixes
+ is_reserved_word // tests whether current string is a reserved word ('ad','soyad')
+ mark_cAsInA // nominal verb suffix
+ mark_DA // noun suffix
+ mark_DAn // noun suffix
+ mark_DUr // nominal verb suffix
+ mark_ki // noun suffix
+ mark_lAr // noun suffix, nominal verb suffix
+ mark_lArI // noun suffix
+ mark_nA // noun suffix
+ mark_ncA // noun suffix
+ mark_ndA // noun suffix
+ mark_ndAn // noun suffix
+ mark_nU // noun suffix
+ mark_nUn // noun suffix
+ mark_nUz // nominal verb suffix
+ mark_sU // noun suffix
+ mark_sUn // nominal verb suffix
+ mark_sUnUz // nominal verb suffix
+ mark_possessives // -(U)m,-(U)n,-(U)mUz,-(U)nUz,
+ mark_yA // noun suffix
+ mark_ylA // noun suffix
+ mark_yU // noun suffix
+ mark_yUm // nominal verb suffix
+ mark_yUz // nominal verb suffix
+ mark_yDU // nominal verb suffix
+ mark_yken // nominal verb suffix
+ mark_ymUs_ // nominal verb suffix
+ mark_ysA // nominal verb suffix
+
+ mark_suffix_with_optional_y_consonant
+ mark_suffix_with_optional_U_vowel
+ mark_suffix_with_optional_n_consonant
+ mark_suffix_with_optional_s_consonant
+
+ more_than_one_syllable_word
+
+ post_process_last_consonants
+ postlude
+
+ stem_nominal_verb_suffixes
+ stem_noun_suffixes
+ stem_suffix_chain_before_ki
+)
+
+stringescapes { }
+
+/* Special characters in Unicode Latin-1 and Latin Extended-A */
+stringdef c, '{U+00E7}' // LATIN SMALL LETTER C WITH CEDILLA
+stringdef g~ '{U+011F}' // LATIN SMALL LETTER G WITH BREVE
+stringdef i' '{U+0131}' // LATIN SMALL LETTER I WITHOUT DOT
+stringdef o" '{U+00F6}' // LATIN SMALL LETTER O WITH DIAERESIS
+stringdef s, '{U+015F}' // LATIN SMALL LETTER S WITH CEDILLA
+stringdef u" '{U+00FC}' // LATIN SMALL LETTER U WITH DIAERESIS
+
+booleans ( continue_stemming_noun_suffixes )
+
+groupings ( vowel U vowel1 vowel2 vowel3 vowel4 vowel5 vowel6)
+
+define vowel 'ae{i'}io{o"}u{u"}'
+define U '{i'}iu{u"}'
+
+// the vowel grouping definitions below are used for checking vowel harmony
+define vowel1 'a{i'}ou' // vowels that can end with suffixes containing 'a'
+define vowel2 'ei{o"}{u"}' // vowels that can end with suffixes containing 'e'
+define vowel3 'a{i'}' // vowels that can end with suffixes containing 'i''
+define vowel4 'ei' // vowels that can end with suffixes containing 'i'
+define vowel5 'ou' // vowels that can end with suffixes containing 'o' or 'u'
+define vowel6 '{o"}{u"}' // vowels that can end with suffixes containing 'o"' or 'u"'
+
+externals ( stem )
+
+backwardmode (
+ // checks vowel harmony for possible suffixes,
+ // helps to detect whether the candidate for suffix applies to vowel harmony
+ // this rule is added to prevent over stemming
+ define check_vowel_harmony as (
+ test
+ (
+ (goto vowel) // if there is a vowel
+ (
+ ('a' goto vowel1) or
+ ('e' goto vowel2) or
+ ('{i'}' goto vowel3) or
+ ('i' goto vowel4) or
+ ('o' goto vowel5) or
+ ('{o"}' goto vowel6) or
+ ('u' goto vowel5) or
+ ('{u"}' goto vowel6)
+ )
+ )
+ )
+
+ // if the last consonant before suffix is vowel and n then advance and delete
+ // if the last consonant before suffix is non vowel and n do nothing
+ // if the last consonant before suffix is not n then only delete the suffix
+ // assumption: slice beginning is set correctly
+ define mark_suffix_with_optional_n_consonant as (
+ ('n' (test vowel))
+ or
+ ((not(test 'n')) test(next vowel))
+
+ )
+
+ // if the last consonant before suffix is vowel and s then advance and delete
+ // if the last consonant before suffix is non vowel and s do nothing
+ // if the last consonant before suffix is not s then only delete the suffix
+ // assumption: slice beginning is set correctly
+ define mark_suffix_with_optional_s_consonant as (
+ ('s' (test vowel))
+ or
+ ((not(test 's')) test(next vowel))
+ )
+
+ // if the last consonant before suffix is vowel and y then advance and delete
+ // if the last consonant before suffix is non vowel and y do nothing
+ // if the last consonant before suffix is not y then only delete the suffix
+ // assumption: slice beginning is set correctly
+ define mark_suffix_with_optional_y_consonant as (
+ ('y' (test vowel))
+ or
+ ((not(test 'y')) test(next vowel))
+ )
+
+ define mark_suffix_with_optional_U_vowel as (
+ (U (test non-vowel))
+ or
+ ((not(test U)) test(next non-vowel))
+
+ )
+
+ define mark_possessives as (
+ among ('m{i'}z' 'miz' 'muz' 'm{u"}z'
+ 'n{i'}z' 'niz' 'nuz' 'n{u"}z' 'm' 'n')
+ (mark_suffix_with_optional_U_vowel)
+ )
+
+ define mark_sU as (
+ check_vowel_harmony
+ U
+ (mark_suffix_with_optional_s_consonant)
+ )
+
+ define mark_lArI as (
+ among ('leri' 'lar{i'}')
+ )
+
+ define mark_yU as (
+ check_vowel_harmony
+ U
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_nU as (
+ check_vowel_harmony
+ among ('n{i'}' 'ni' 'nu' 'n{u"}')
+ )
+
+ define mark_nUn as (
+ check_vowel_harmony
+ among ('{i'}n' 'in' 'un' '{u"}n')
+ (mark_suffix_with_optional_n_consonant)
+ )
+
+ define mark_yA as (
+ check_vowel_harmony
+ among('a' 'e')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_nA as (
+ check_vowel_harmony
+ among('na' 'ne')
+ )
+
+ define mark_DA as (
+ check_vowel_harmony
+ among('da' 'de' 'ta' 'te')
+ )
+
+ define mark_ndA as (
+ check_vowel_harmony
+ among('nda' 'nde')
+ )
+
+ define mark_DAn as (
+ check_vowel_harmony
+ among('dan' 'den' 'tan' 'ten')
+ )
+
+ define mark_ndAn as (
+ check_vowel_harmony
+ among('ndan' 'nden')
+ )
+
+ define mark_ylA as (
+ check_vowel_harmony
+ among('la' 'le')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_ki as (
+ 'ki'
+ )
+
+ define mark_ncA as (
+ check_vowel_harmony
+ among('ca' 'ce')
+ (mark_suffix_with_optional_n_consonant)
+ )
+
+ define mark_yUm as (
+ check_vowel_harmony
+ among ('{i'}m' 'im' 'um' '{u"}m')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_sUn as (
+ check_vowel_harmony
+ among ('s{i'}n' 'sin' 'sun' 's{u"}n' )
+ )
+
+ define mark_yUz as (
+ check_vowel_harmony
+ among ('{i'}z' 'iz' 'uz' '{u"}z')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_sUnUz as (
+ among ('s{i'}n{i'}z' 'siniz' 'sunuz' 's{u"}n{u"}z')
+ )
+
+ define mark_lAr as (
+ check_vowel_harmony
+ among ('ler' 'lar')
+ )
+
+ define mark_nUz as (
+ check_vowel_harmony
+ among ('n{i'}z' 'niz' 'nuz' 'n{u"}z')
+ )
+
+ define mark_DUr as (
+ check_vowel_harmony
+ among ('t{i'}r' 'tir' 'tur' 't{u"}r' 'd{i'}r' 'dir' 'dur' 'd{u"}r')
+ )
+
+ define mark_cAsInA as (
+ among ('cas{i'}na' 'cesine')
+ )
+
+ define mark_yDU as (
+ check_vowel_harmony
+ among ('t{i'}m' 'tim' 'tum' 't{u"}m' 'd{i'}m' 'dim' 'dum' 'd{u"}m'
+ 't{i'}n' 'tin' 'tun' 't{u"}n' 'd{i'}n' 'din' 'dun' 'd{u"}n'
+ 't{i'}k' 'tik' 'tuk' 't{u"}k' 'd{i'}k' 'dik' 'duk' 'd{u"}k'
+ 't{i'}' 'ti' 'tu' 't{u"}' 'd{i'}' 'di' 'du' 'd{u"}')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ // does not fully obey vowel harmony
+ define mark_ysA as (
+ among ('sam' 'san' 'sak' 'sem' 'sen' 'sek' 'sa' 'se')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_ymUs_ as (
+ check_vowel_harmony
+ among ('m{i'}{s,}' 'mi{s,}' 'mu{s,}' 'm{u"}{s,}')
+ (mark_suffix_with_optional_y_consonant)
+ )
+
+ define mark_yken as (
+ 'ken' (mark_suffix_with_optional_y_consonant)
+ )
+
+ define stem_nominal_verb_suffixes as (
+ [
+ set continue_stemming_noun_suffixes
+ (mark_ymUs_ or mark_yDU or mark_ysA or mark_yken)
+ or
+ (mark_cAsInA (mark_sUnUz or mark_lAr or mark_yUm or mark_sUn or mark_yUz or true) mark_ymUs_)
+ or
+ (
+ mark_lAr ] delete try([(mark_DUr or mark_yDU or mark_ysA or mark_ymUs_))
+ unset continue_stemming_noun_suffixes
+ )
+ or
+ (mark_nUz (mark_yDU or mark_ysA))
+ or
+ ((mark_sUnUz or mark_yUz or mark_sUn or mark_yUm) ] delete try([ mark_ymUs_))
+ or
+ (mark_DUr ] delete try([ (mark_sUnUz or mark_lAr or mark_yUm or mark_sUn or mark_yUz or true) mark_ymUs_))
+ ]delete
+ )
+
+ // stems noun suffix chains ending with -ki
+ define stem_suffix_chain_before_ki as (
+ [
+ mark_ki
+ (
+ (mark_DA] delete try([
+ (mark_lAr] delete try(stem_suffix_chain_before_ki))
+ or
+ (mark_possessives] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+
+ ))
+ or
+ (mark_nUn] delete try([
+ (mark_lArI] delete)
+ or
+ ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ or
+ (stem_suffix_chain_before_ki)
+ ))
+ or
+ (mark_ndA (
+ (mark_lArI] delete)
+ or
+ ((mark_sU] delete try([mark_lAr]delete stem_suffix_chain_before_ki)))
+ or
+ (stem_suffix_chain_before_ki)
+ ))
+ )
+ )
+
+ define stem_noun_suffixes as (
+ ([mark_lAr] delete try(stem_suffix_chain_before_ki))
+ or
+ ([mark_ncA] delete
+ try(
+ ([mark_lArI] delete)
+ or
+ ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ or
+ ([mark_lAr] delete stem_suffix_chain_before_ki)
+ )
+ )
+ or
+ ([(mark_ndA or mark_nA)
+ (
+ (mark_lArI] delete)
+ or
+ (mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ or
+ (stem_suffix_chain_before_ki)
+ )
+ )
+ or
+ ([(mark_ndAn or mark_nU) ((mark_sU ] delete try([mark_lAr] delete stem_suffix_chain_before_ki)) or (mark_lArI)))
+ or
+ ( [mark_DAn] delete try ([
+ (
+ (mark_possessives ] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ or
+ (mark_lAr] delete try(stem_suffix_chain_before_ki))
+ or
+ (stem_suffix_chain_before_ki)
+ ))
+ )
+ or
+ ([mark_nUn or mark_ylA] delete
+ try(
+ ([mark_lAr] delete stem_suffix_chain_before_ki)
+ or
+ ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ or
+ stem_suffix_chain_before_ki
+ )
+ )
+ or
+ ([mark_lArI] delete)
+ or
+ (stem_suffix_chain_before_ki)
+ or
+ ([mark_DA or mark_yU or mark_yA] delete try([((mark_possessives] delete try([mark_lAr)) or mark_lAr) ] delete [ stem_suffix_chain_before_ki))
+ or
+ ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki))
+ )
+
+ define post_process_last_consonants as (
+ [substring] among (
+ 'b' (<- 'p')
+ 'c' (<- '{c,}')
+ 'd' (<- 't')
+ '{g~}' (<- 'k')
+ )
+ )
+
+ // after stemming if the word ends with 'd' or 'g' most probably last U is overstemmed
+ // like in 'kedim' -> 'ked'
+ // Turkish words don't usually end with 'd' or 'g'
+ // some very well known words are ignored (like 'ad' 'soyad'
+ // appends U to stems ending with d or g, decides which vowel to add
+ // based on the last vowel in the stem
+ define append_U_to_stems_ending_with_d_or_g as (
+ test('d' or 'g')
+ (test((goto vowel) 'a' or '{i'}') <+ '{i'}')
+ or
+ (test((goto vowel) 'e' or 'i') <+ 'i')
+ or
+ (test((goto vowel) 'o' or 'u') <+ 'u')
+ or
+ (test((goto vowel) '{o"}' or '{u"}') <+ '{u"}')
+ )
+
+ define is_reserved_word as (
+ 'ad' try 'soy' atlimit
+ )
+)
+
+// Tests if there are more than one syllables
+// In Turkish each vowel indicates a distinct syllable
+define more_than_one_syllable_word as (
+ test (atleast 2 (gopast vowel))
+)
+
+define postlude as (
+ backwards (
+ not(is_reserved_word)
+ do append_U_to_stems_ending_with_d_or_g
+ do post_process_last_consonants
+
+ )
+)
+
+define stem as (
+ (more_than_one_syllable_word)
+ (
+ backwards (
+ do stem_nominal_verb_suffixes
+ continue_stemming_noun_suffixes
+ do stem_noun_suffixes
+ )
+
+ postlude
+ )
+)
diff --git a/contrib/snowball/charsets/ISO-8859-2.sbl b/contrib/snowball/charsets/ISO-8859-2.sbl
new file mode 100644
index 0000000..5829ea8
--- /dev/null
+++ b/contrib/snowball/charsets/ISO-8859-2.sbl
@@ -0,0 +1,98 @@
+// ISO-8859-2 character mappings.
+
+stringdef U+00A0 hex 'A0'
+stringdef U+0104 hex 'A1'
+stringdef U+02D8 hex 'A2'
+stringdef U+0141 hex 'A3'
+stringdef U+00A4 hex 'A4'
+stringdef U+013D hex 'A5'
+stringdef U+015A hex 'A6'
+stringdef U+00A7 hex 'A7'
+stringdef U+00A8 hex 'A8'
+stringdef U+0160 hex 'A9'
+stringdef U+015E hex 'AA'
+stringdef U+0164 hex 'AB'
+stringdef U+0179 hex 'AC'
+stringdef U+00AD hex 'AD'
+stringdef U+017D hex 'AE'
+stringdef U+017B hex 'AF'
+stringdef U+00B0 hex 'B0'
+stringdef U+0105 hex 'B1'
+stringdef U+02DB hex 'B2'
+stringdef U+0142 hex 'B3'
+stringdef U+00B4 hex 'B4'
+stringdef U+013E hex 'B5'
+stringdef U+015B hex 'B6'
+stringdef U+02C7 hex 'B7'
+stringdef U+00B8 hex 'B8'
+stringdef U+0161 hex 'B9'
+stringdef U+015F hex 'BA'
+stringdef U+0165 hex 'BB'
+stringdef U+017A hex 'BC'
+stringdef U+02DD hex 'BD'
+stringdef U+017E hex 'BE'
+stringdef U+017C hex 'BF'
+stringdef U+0154 hex 'C0'
+stringdef U+00C1 hex 'C1'
+stringdef U+00C2 hex 'C2'
+stringdef U+0102 hex 'C3'
+stringdef U+00C4 hex 'C4'
+stringdef U+0139 hex 'C5'
+stringdef U+0106 hex 'C6'
+stringdef U+00C7 hex 'C7'
+stringdef U+010C hex 'C8'
+stringdef U+00C9 hex 'C9'
+stringdef U+0118 hex 'CA'
+stringdef U+00CB hex 'CB'
+stringdef U+011A hex 'CC'
+stringdef U+00CD hex 'CD'
+stringdef U+00CE hex 'CE'
+stringdef U+010E hex 'CF'
+stringdef U+0110 hex 'D0'
+stringdef U+0143 hex 'D1'
+stringdef U+0147 hex 'D2'
+stringdef U+00D3 hex 'D3'
+stringdef U+00D4 hex 'D4'
+stringdef U+0150 hex 'D5'
+stringdef U+00D6 hex 'D6'
+stringdef U+00D7 hex 'D7'
+stringdef U+0158 hex 'D8'
+stringdef U+016E hex 'D9'
+stringdef U+00DA hex 'DA'
+stringdef U+0170 hex 'DB'
+stringdef U+00DC hex 'DC'
+stringdef U+00DD hex 'DD'
+stringdef U+0162 hex 'DE'
+stringdef U+00DF hex 'DF'
+stringdef U+0155 hex 'E0'
+stringdef U+00E1 hex 'E1'
+stringdef U+00E2 hex 'E2'
+stringdef U+0103 hex 'E3'
+stringdef U+00E4 hex 'E4'
+stringdef U+013A hex 'E5'
+stringdef U+0107 hex 'E6'
+stringdef U+00E7 hex 'E7'
+stringdef U+010D hex 'E8'
+stringdef U+00E9 hex 'E9'
+stringdef U+0119 hex 'EA'
+stringdef U+00EB hex 'EB'
+stringdef U+011B hex 'EC'
+stringdef U+00ED hex 'ED'
+stringdef U+00EE hex 'EE'
+stringdef U+010F hex 'EF'
+stringdef U+0111 hex 'F0'
+stringdef U+0144 hex 'F1'
+stringdef U+0148 hex 'F2'
+stringdef U+00F3 hex 'F3'
+stringdef U+00F4 hex 'F4'
+stringdef U+0151 hex 'F5'
+stringdef U+00F6 hex 'F6'
+stringdef U+00F7 hex 'F7'
+stringdef U+0159 hex 'F8'
+stringdef U+016F hex 'F9'
+stringdef U+00FA hex 'FA'
+stringdef U+0171 hex 'FB'
+stringdef U+00FC hex 'FC'
+stringdef U+00FD hex 'FD'
+stringdef U+0163 hex 'FE'
+stringdef U+02D9 hex 'FF'
diff --git a/contrib/snowball/charsets/KOI8-R.sbl b/contrib/snowball/charsets/KOI8-R.sbl
new file mode 100644
index 0000000..46854e8
--- /dev/null
+++ b/contrib/snowball/charsets/KOI8-R.sbl
@@ -0,0 +1,74 @@
+// KOI8-R character mappings.
+
+stringdef U+00A0 hex '9A'
+stringdef U+00A9 hex 'BF'
+stringdef U+00B0 hex '9C'
+stringdef U+00B2 hex '9D'
+stringdef U+00B7 hex '9E'
+stringdef U+00F7 hex '9F'
+stringdef U+0401 hex 'B3'
+stringdef U+0410 hex 'E1'
+stringdef U+0411 hex 'E2'
+stringdef U+0412 hex 'F7'
+stringdef U+0413 hex 'E7'
+stringdef U+0414 hex 'E4'
+stringdef U+0415 hex 'E5'
+stringdef U+0416 hex 'F6'
+stringdef U+0417 hex 'FA'
+stringdef U+0418 hex 'E9'
+stringdef U+0419 hex 'EA'
+stringdef U+041A hex 'EB'
+stringdef U+041B hex 'EC'
+stringdef U+041C hex 'ED'
+stringdef U+041D hex 'EE'
+stringdef U+041E hex 'EF'
+stringdef U+041F hex 'F0'
+stringdef U+0420 hex 'F2'
+stringdef U+0421 hex 'F3'
+stringdef U+0422 hex 'F4'
+stringdef U+0423 hex 'F5'
+stringdef U+0424 hex 'E6'
+stringdef U+0425 hex 'E8'
+stringdef U+0426 hex 'E3'
+stringdef U+0427 hex 'FE'
+stringdef U+0428 hex 'FB'
+stringdef U+0429 hex 'FD'
+stringdef U+042A hex 'FF'
+stringdef U+042B hex 'F9'
+stringdef U+042C hex 'F8'
+stringdef U+042D hex 'FC'
+stringdef U+042E hex 'E0'
+stringdef U+042F hex 'F1'
+stringdef U+0430 hex 'C1'
+stringdef U+0431 hex 'C2'
+stringdef U+0432 hex 'D7'
+stringdef U+0433 hex 'C7'
+stringdef U+0434 hex 'C4'
+stringdef U+0435 hex 'C5'
+stringdef U+0436 hex 'D6'
+stringdef U+0437 hex 'DA'
+stringdef U+0438 hex 'C9'
+stringdef U+0439 hex 'CA'
+stringdef U+043A hex 'CB'
+stringdef U+043B hex 'CC'
+stringdef U+043C hex 'CD'
+stringdef U+043D hex 'CE'
+stringdef U+043E hex 'CF'
+stringdef U+043F hex 'D0'
+stringdef U+0440 hex 'D2'
+stringdef U+0441 hex 'D3'
+stringdef U+0442 hex 'D4'
+stringdef U+0443 hex 'D5'
+stringdef U+0444 hex 'C6'
+stringdef U+0445 hex 'C8'
+stringdef U+0446 hex 'C3'
+stringdef U+0447 hex 'DE'
+stringdef U+0448 hex 'DB'
+stringdef U+0449 hex 'DD'
+stringdef U+044A hex 'DF'
+stringdef U+044B hex 'D9'
+stringdef U+044C hex 'D8'
+stringdef U+044D hex 'DC'
+stringdef U+044E hex 'C0'
+stringdef U+044F hex 'D1'
+stringdef U+0451 hex 'A3'
diff --git a/contrib/snowball/charsets/cp850.sbl b/contrib/snowball/charsets/cp850.sbl
new file mode 100644
index 0000000..b780220
--- /dev/null
+++ b/contrib/snowball/charsets/cp850.sbl
@@ -0,0 +1,130 @@
+// Code page 850 (MSDOS Latin 1) character mappings.
+
+stringdef U+00A0 hex 'FF'
+stringdef U+00A1 hex 'AD'
+stringdef U+00A2 hex 'BD'
+stringdef U+00A3 hex '9C'
+stringdef U+00A4 hex 'CF'
+stringdef U+00A5 hex 'BE'
+stringdef U+00A6 hex 'DD'
+stringdef U+00A7 hex 'F5'
+stringdef U+00A8 hex 'F9'
+stringdef U+00A9 hex 'B8'
+stringdef U+00AA hex 'A6'
+stringdef U+00AB hex 'AE'
+stringdef U+00AC hex 'AA'
+stringdef U+00AD hex 'F0'
+stringdef U+00AE hex 'A9'
+stringdef U+00AF hex 'EE'
+stringdef U+00B0 hex 'F8'
+stringdef U+00B1 hex 'F1'
+stringdef U+00B2 hex 'FD'
+stringdef U+00B3 hex 'FC'
+stringdef U+00B4 hex 'EF'
+stringdef U+00B5 hex 'E6'
+stringdef U+00B6 hex 'F4'
+stringdef U+00B7 hex 'FA'
+stringdef U+00B8 hex 'F7'
+stringdef U+00B9 hex 'FB'
+stringdef U+00BA hex 'A7'
+stringdef U+00BB hex 'AF'
+stringdef U+00BC hex 'AC'
+stringdef U+00BD hex 'AB'
+stringdef U+00BE hex 'F3'
+stringdef U+00BF hex 'A8'
+stringdef U+00C0 hex 'B7'
+stringdef U+00C1 hex 'B5'
+stringdef U+00C2 hex 'B6'
+stringdef U+00C3 hex 'C7'
+stringdef U+00C4 hex '8E'
+stringdef U+00C5 hex '8F'
+stringdef U+00C6 hex '92'
+stringdef U+00C7 hex '80'
+stringdef U+00C8 hex 'D4'
+stringdef U+00C9 hex '90'
+stringdef U+00CA hex 'D2'
+stringdef U+00CB hex 'D3'
+stringdef U+00CC hex 'DE'
+stringdef U+00CD hex 'D6'
+stringdef U+00CE hex 'D7'
+stringdef U+00CF hex 'D8'
+stringdef U+00D0 hex 'D1'
+stringdef U+00D1 hex 'A5'
+stringdef U+00D2 hex 'E3'
+stringdef U+00D3 hex 'E0'
+stringdef U+00D4 hex 'E2'
+stringdef U+00D5 hex 'E5'
+stringdef U+00D6 hex '99'
+stringdef U+00D7 hex '9E'
+stringdef U+00D8 hex '9D'
+stringdef U+00D9 hex 'EB'
+stringdef U+00DA hex 'E9'
+stringdef U+00DB hex 'EA'
+stringdef U+00DC hex '9A'
+stringdef U+00DD hex 'ED'
+stringdef U+00DE hex 'E8'
+stringdef U+00DF hex 'E1'
+stringdef U+00E0 hex '85'
+stringdef U+00E1 hex 'A0'
+stringdef U+00E2 hex '83'
+stringdef U+00E3 hex 'C6'
+stringdef U+00E4 hex '84'
+stringdef U+00E5 hex '86'
+stringdef U+00E6 hex '91'
+stringdef U+00E7 hex '87'
+stringdef U+00E8 hex '8A'
+stringdef U+00E9 hex '82'
+stringdef U+00EA hex '88'
+stringdef U+00EB hex '89'
+stringdef U+00EC hex '8D'
+stringdef U+00ED hex 'A1'
+stringdef U+00EE hex '8C'
+stringdef U+00EF hex '8B'
+stringdef U+00F0 hex 'D0'
+stringdef U+00F1 hex 'A4'
+stringdef U+00F2 hex '95'
+stringdef U+00F3 hex 'A2'
+stringdef U+00F4 hex '93'
+stringdef U+00F5 hex 'E4'
+stringdef U+00F6 hex '94'
+stringdef U+00F7 hex 'F6'
+stringdef U+00F8 hex '9B'
+stringdef U+00F9 hex '97'
+stringdef U+00FA hex 'A3'
+stringdef U+00FB hex '96'
+stringdef U+00FC hex '81'
+stringdef U+00FD hex 'EC'
+stringdef U+00FE hex 'E7'
+stringdef U+00FF hex '98'
+stringdef U+0131 hex 'D5'
+stringdef U+0192 hex '9F'
+stringdef U+2017 hex 'F2'
+stringdef U+2500 hex 'C4'
+stringdef U+2502 hex 'B3'
+stringdef U+250C hex 'DA'
+stringdef U+2510 hex 'BF'
+stringdef U+2514 hex 'C0'
+stringdef U+2518 hex 'D9'
+stringdef U+251C hex 'C3'
+stringdef U+2524 hex 'B4'
+stringdef U+252C hex 'C2'
+stringdef U+2534 hex 'C1'
+stringdef U+253C hex 'C5'
+stringdef U+2550 hex 'CD'
+stringdef U+2551 hex 'BA'
+stringdef U+2554 hex 'C9'
+stringdef U+2557 hex 'BB'
+stringdef U+255A hex 'C8'
+stringdef U+255D hex 'BC'
+stringdef U+2560 hex 'CC'
+stringdef U+2563 hex 'B9'
+stringdef U+2566 hex 'CB'
+stringdef U+2569 hex 'CA'
+stringdef U+256C hex 'CE'
+stringdef U+2580 hex 'DF'
+stringdef U+2584 hex 'DC'
+stringdef U+2588 hex 'DB'
+stringdef U+2591 hex 'B0'
+stringdef U+2592 hex 'B1'
+stringdef U+2593 hex 'B2'
+stringdef U+25A0 hex 'FE'
diff --git a/contrib/snowball/compiler/analyser.c b/contrib/snowball/compiler/analyser.c
new file mode 100644
index 0000000..dffa555
--- /dev/null
+++ b/contrib/snowball/compiler/analyser.c
@@ -0,0 +1,1380 @@
+
+#include <stdio.h> /* printf etc */
+#include <stdlib.h> /* exit */
+#include <string.h> /* memmove */
+#include "header.h"
+
+typedef enum {
+ e_token_omitted = 0,
+ e_unexpected_token = 1,
+ e_string_omitted = 2,
+ e_unexpected_token_in_among = 3,
+ /* For codes above here, report "after " t->previous_token after the error. */
+ e_unresolved_substring = 14,
+ e_not_allowed_inside_reverse = 15,
+ e_empty_grouping = 16,
+ e_already_backwards = 17,
+ e_empty_among = 18,
+ e_adjacent_bracketed_in_among = 19,
+ e_substring_preceded_by_substring = 20,
+ /* For codes below here, tokeniser->b is printed before the error. */
+ e_redeclared = 30,
+ e_undeclared = 31,
+ e_declared_as_different_mode = 32,
+ e_not_of_type_x = 33,
+ e_not_of_type_string_or_integer = 34,
+ e_misplaced = 35,
+ e_redefined = 36,
+ e_misused = 37
+} error_code;
+
+/* recursive usage: */
+
+static void read_program_(struct analyser * a, int terminator);
+static struct node * read_C(struct analyser * a);
+static struct node * C_style(struct analyser * a, const char * s, int token);
+
+
+static void print_node_(struct node * p, int n, const char * s) {
+
+ int i;
+ for (i = 0; i < n; i++) fputs(i == n - 1 ? s : " ", stdout);
+ printf("%s ", name_of_token(p->type));
+ if (p->name) report_b(stdout, p->name->b);
+ if (p->literalstring) {
+ printf("'");
+ report_b(stdout, p->literalstring);
+ printf("'");
+ }
+ printf("\n");
+ if (p->AE) print_node_(p->AE, n+1, "# ");
+ if (p->left) print_node_(p->left, n+1, " ");
+ if (p->aux) print_node_(p->aux, n+1, "@ ");
+ if (p->right) print_node_(p->right, n, " ");
+}
+
+extern void print_program(struct analyser * a) {
+ print_node_(a->program, 0, " ");
+}
+
+static struct node * new_node(struct analyser * a, int type) {
+ NEW(node, p);
+ p->next = a->nodes; a->nodes = p;
+ p->left = 0;
+ p->right = 0;
+ p->aux = 0;
+ p->AE = 0;
+ p->name = 0;
+ p->literalstring = 0;
+ p->mode = a->mode;
+ p->line_number = a->tokeniser->line_number;
+ p->type = type;
+ return p;
+}
+
+static const char * name_of_mode(int n) {
+ switch (n) {
+ case m_backward: return "string backward";
+ case m_forward: return "string forward";
+ /* case m_integer: return "integer"; */
+ }
+ fprintf(stderr, "Invalid mode %d in name_of_mode()\n", n);
+ exit(1);
+}
+
+static const char * name_of_type(int n) {
+ switch (n) {
+ case 's': return "string";
+ case 'i': return "integer";
+ case 'r': return "routine";
+ case 'R': return "routine or grouping";
+ case 'g': return "grouping";
+ }
+ fprintf(stderr, "Invalid type %d in name_of_type()\n", n);
+ exit(1);
+}
+
+static const char * name_of_name_type(int code) {
+ switch (code) {
+ case t_string: return "string";
+ case t_boolean: return "boolean";
+ case t_integer: return "integer";
+ case t_routine: return "routine";
+ case t_external: return "external";
+ case t_grouping: return "grouping";
+ }
+ fprintf(stderr, "Invalid type code %d in name_of_name_type()\n", code);
+ exit(1);
+}
+
+static void count_error(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ if (t->error_count >= 20) { fprintf(stderr, "... etc\n"); exit(1); }
+ t->error_count++;
+}
+
+static void error2(struct analyser * a, error_code n, int x) {
+ struct tokeniser * t = a->tokeniser;
+ count_error(a);
+ fprintf(stderr, "%s:%d: ", t->file, t->line_number);
+ if ((int)n >= (int)e_redeclared) report_b(stderr, t->b);
+ switch (n) {
+ case e_token_omitted:
+ fprintf(stderr, "%s omitted", name_of_token(t->omission)); break;
+ case e_unexpected_token_in_among:
+ fprintf(stderr, "in among(...), ");
+ /* fall through */
+ case e_unexpected_token:
+ fprintf(stderr, "unexpected %s", name_of_token(t->token));
+ if (t->token == c_number) fprintf(stderr, " %d", t->number);
+ if (t->token == c_name) {
+ fprintf(stderr, " ");
+ report_b(stderr, t->b);
+ } break;
+ case e_string_omitted:
+ fprintf(stderr, "string omitted"); break;
+
+ case e_unresolved_substring:
+ fprintf(stderr, "unresolved substring on line %d", x); break;
+ case e_not_allowed_inside_reverse:
+ fprintf(stderr, "%s not allowed inside reverse(...)", name_of_token(t->token)); break;
+ case e_empty_grouping:
+ fprintf(stderr, "empty grouping"); break;
+ case e_already_backwards:
+ fprintf(stderr, "backwards used when already in this mode"); break;
+ case e_empty_among:
+ fprintf(stderr, "empty among(...)"); break;
+ case e_adjacent_bracketed_in_among:
+ fprintf(stderr, "two adjacent bracketed expressions in among(...)"); break;
+ case e_substring_preceded_by_substring:
+ fprintf(stderr, "substring preceded by another substring on line %d", x); break;
+
+ case e_redeclared:
+ fprintf(stderr, " re-declared"); break;
+ case e_undeclared:
+ fprintf(stderr, " undeclared"); break;
+ case e_declared_as_different_mode:
+ fprintf(stderr, " declared as %s mode; used as %s mode",
+ name_of_mode(a->mode), name_of_mode(x)); break;
+ case e_not_of_type_x:
+ fprintf(stderr, " not of type %s", name_of_type(x)); break;
+ case e_not_of_type_string_or_integer:
+ fprintf(stderr, " not of type string or integer"); break;
+ case e_misplaced:
+ fprintf(stderr, " misplaced"); break;
+ case e_redefined:
+ fprintf(stderr, " redefined"); break;
+ case e_misused:
+ fprintf(stderr, " mis-used as %s mode",
+ name_of_mode(x)); break;
+ }
+ if ((int)n < (int)e_unresolved_substring && t->previous_token > 0)
+ fprintf(stderr, " after %s", name_of_token(t->previous_token));
+ fprintf(stderr, "\n");
+}
+
+static void error(struct analyser * a, error_code n) { error2(a, n, 0); }
+
+static void error4(struct analyser * a, struct name * q) {
+ count_error(a);
+ fprintf(stderr, "%s:%d: ", a->tokeniser->file, q->used->line_number);
+ report_b(stderr, q->b);
+ fprintf(stderr, " undefined\n");
+}
+
+static void omission_error(struct analyser * a, int n) {
+ a->tokeniser->omission = n;
+ error(a, e_token_omitted);
+}
+
+static int check_token(struct analyser * a, int code) {
+ struct tokeniser * t = a->tokeniser;
+ if (t->token != code) { omission_error(a, code); return false; }
+ return true;
+}
+
+static int get_token(struct analyser * a, int code) {
+ struct tokeniser * t = a->tokeniser;
+ read_token(t);
+ {
+ int x = check_token(a, code);
+ if (!x) t->token_held = true;
+ return x;
+ }
+}
+
+static struct name * look_for_name(struct analyser * a) {
+ symbol * q = a->tokeniser->b;
+ struct name * p;
+ for (p = a->names; p; p = p->next) {
+ symbol * b = p->b;
+ int n = SIZE(b);
+ if (n == SIZE(q) && memcmp(q, b, n * sizeof(symbol)) == 0) {
+ p->referenced = true;
+ return p;
+ }
+ }
+ return 0;
+}
+
+static struct name * find_name(struct analyser * a) {
+ struct name * p = look_for_name(a);
+ if (p == 0) error(a, e_undeclared);
+ return p;
+}
+
+static void check_routine_mode(struct analyser * a, struct name * p, int mode) {
+ if (p->mode < 0) p->mode = mode; else
+ if (p->mode != mode) error2(a, e_misused, mode);
+}
+
+static void check_name_type(struct analyser * a, struct name * p, int type) {
+ switch (type) {
+ case 's':
+ if (p->type == t_string) return;
+ break;
+ case 'i':
+ if (p->type == t_integer) return;
+ break;
+ case 'b':
+ if (p->type == t_boolean) return;
+ break;
+ case 'R':
+ if (p->type == t_grouping) return;
+ /* FALLTHRU */
+ case 'r':
+ if (p->type == t_routine || p->type == t_external) return;
+ break;
+ case 'g':
+ if (p->type == t_grouping) return;
+ break;
+ }
+ error2(a, e_not_of_type_x, type);
+}
+
+static void read_names(struct analyser * a, int type) {
+ struct tokeniser * t = a->tokeniser;
+ if (!get_token(a, c_bra)) return;
+ while (true) {
+ int token = read_token(t);
+ switch (token) {
+ case c_len: {
+ /* Context-sensitive token - once declared as a name, it loses
+ * its special meaning, for compatibility with older versions
+ * of snowball.
+ */
+ static const symbol c_len_lit[] = {
+ 'l', 'e', 'n'
+ };
+ MOVE_TO_B(t->b, c_len_lit);
+ goto handle_as_name;
+ }
+ case c_lenof: {
+ /* Context-sensitive token - once declared as a name, it loses
+ * its special meaning, for compatibility with older versions
+ * of snowball.
+ */
+ static const symbol c_lenof_lit[] = {
+ 'l', 'e', 'n', 'o', 'f'
+ };
+ MOVE_TO_B(t->b, c_lenof_lit);
+ goto handle_as_name;
+ }
+ case c_name:
+handle_as_name:
+ if (look_for_name(a) != 0) error(a, e_redeclared); else {
+ NEW(name, p);
+ p->b = copy_b(t->b);
+ p->type = type;
+ p->mode = -1; /* routines, externals */
+ /* We defer assigning counts until after we've eliminated
+ * variables whose values are never used. */
+ p->count = -1;
+ p->referenced = false;
+ p->used_in_among = false;
+ p->used = 0;
+ p->value_used = false;
+ p->initialised = false;
+ p->used_in_definition = false;
+ p->local_to = 0;
+ p->grouping = 0;
+ p->definition = 0;
+ p->declaration_line_number = t->line_number;
+ p->next = a->names;
+ a->names = p;
+ if (token != c_name) {
+ disable_token(t, token);
+ }
+ }
+ break;
+ default:
+ if (!check_token(a, c_ket)) t->token_held = true;
+ return;
+ }
+ }
+}
+
+static symbol * new_literalstring(struct analyser * a) {
+ NEW(literalstring, p);
+ p->b = copy_b(a->tokeniser->b);
+ p->next = a->literalstrings;
+ a->literalstrings = p;
+ return p->b;
+}
+
+static int read_AE_test(struct analyser * a) {
+
+ struct tokeniser * t = a->tokeniser;
+ switch (read_token(t)) {
+ case c_assign: return c_mathassign;
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le: return t->token;
+ default: error(a, e_unexpected_token); t->token_held = true; return c_eq;
+ }
+}
+
+static int binding(int t) {
+ switch (t) {
+ case c_plus: case c_minus: return 1;
+ case c_multiply: case c_divide: return 2;
+ default: return -2;
+ }
+}
+
+static void mark_used_in(struct analyser * a, struct name * q, struct node * p) {
+ if (!q->used) {
+ q->used = p;
+ q->local_to = a->program_end->name;
+ } else if (q->local_to) {
+ if (q->local_to != a->program_end->name) {
+ /* Used in more than one routine/external. */
+ q->local_to = NULL;
+ }
+ }
+}
+
+static void name_to_node(struct analyser * a, struct node * p, int type) {
+ struct name * q = find_name(a);
+ if (q) {
+ check_name_type(a, q, type);
+ mark_used_in(a, q, p);
+ }
+ p->name = q;
+}
+
+static struct node * read_AE(struct analyser * a, int B) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p;
+ struct node * q;
+ switch (read_token(t)) {
+ case c_minus: /* monadic */
+ q = read_AE(a, 100);
+ if (q->type == c_neg) {
+ /* Optimise away double negation, which avoids generators
+ * having to worry about generating "--" (decrement operator
+ * in many languages).
+ */
+ p = q->right;
+ /* Don't free q, it's in the linked list a->nodes. */
+ break;
+ }
+ p = new_node(a, c_neg);
+ p->right = q;
+ break;
+ case c_bra:
+ p = read_AE(a, 0);
+ get_token(a, c_ket);
+ break;
+ case c_name:
+ p = new_node(a, c_name);
+ name_to_node(a, p, 'i');
+ if (p->name) p->name->value_used = true;
+ break;
+ case c_maxint:
+ case c_minint:
+ a->int_limits_used = true;
+ /* fall through */
+ case c_cursor:
+ case c_limit:
+ case c_len:
+ case c_size:
+ p = new_node(a, t->token);
+ break;
+ case c_number:
+ p = new_node(a, c_number);
+ p->number = t->number;
+ break;
+ case c_lenof:
+ case c_sizeof:
+ p = C_style(a, "s", t->token);
+ break;
+ default:
+ error(a, e_unexpected_token);
+ t->token_held = true;
+ return 0;
+ }
+ while (true) {
+ int token = read_token(t);
+ int b = binding(token);
+ if (binding(token) <= B) {
+ t->token_held = true;
+ return p;
+ }
+ q = new_node(a, token);
+ q->left = p;
+ q->right = read_AE(a, b);
+ p = q;
+ }
+}
+
+static struct node * read_C_connection(struct analyser * a, struct node * q, int op) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p = new_node(a, op);
+ struct node * p_end = q;
+ p->left = q;
+ do {
+ q = read_C(a);
+ p_end->right = q; p_end = q;
+ } while (read_token(t) == op);
+ t->token_held = true;
+ return p;
+}
+
+static struct node * read_C_list(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p = new_node(a, c_bra);
+ struct node * p_end = 0;
+ while (true) {
+ int token = read_token(t);
+ if (token == c_ket) return p;
+ if (token < 0) { omission_error(a, c_ket); return p; }
+ t->token_held = true;
+ {
+ struct node * q = read_C(a);
+ while (true) {
+ token = read_token(t);
+ if (token != c_and && token != c_or) {
+ t->token_held = true;
+ break;
+ }
+ q = read_C_connection(a, q, token);
+ }
+ if (p_end == 0) p->left = q; else p_end->right = q;
+ p_end = q;
+ }
+ }
+}
+
+static struct node * C_style(struct analyser * a, const char * s, int token) {
+ int i;
+ struct node * p = new_node(a, token);
+ for (i = 0; s[i] != 0; i++) switch (s[i]) {
+ case 'C':
+ p->left = read_C(a); continue;
+ case 'D':
+ p->aux = read_C(a); continue;
+ case 'A':
+ p->AE = read_AE(a, 0); continue;
+ case 'f':
+ get_token(a, c_for); continue;
+ case 'S':
+ {
+ int str_token = read_token(a->tokeniser);
+ if (str_token == c_name) name_to_node(a, p, 's'); else
+ if (str_token == c_literalstring) p->literalstring = new_literalstring(a);
+ else error(a, e_string_omitted);
+ }
+ continue;
+ case 'b':
+ case 's':
+ case 'i':
+ if (get_token(a, c_name)) name_to_node(a, p, s[i]);
+ continue;
+ }
+ return p;
+}
+
+static struct node * read_literalstring(struct analyser * a) {
+ struct node * p = new_node(a, c_literalstring);
+ p->literalstring = new_literalstring(a);
+ return p;
+}
+
+static void reverse_b(symbol * b) {
+ int i = 0; int j = SIZE(b) - 1;
+ while (i < j) {
+ int ch1 = b[i]; int ch2 = b[j];
+ b[i++] = ch2; b[j--] = ch1;
+ }
+}
+
+static int compare_amongvec(const void *pv, const void *qv) {
+ const struct amongvec * p = (const struct amongvec*)pv;
+ const struct amongvec * q = (const struct amongvec*)qv;
+ symbol * b_p = p->b; int p_size = p->size;
+ symbol * b_q = q->b; int q_size = q->size;
+ int smaller_size = p_size < q_size ? p_size : q_size;
+ int i;
+ for (i = 0; i < smaller_size; i++)
+ if (b_p[i] != b_q[i]) return b_p[i] - b_q[i];
+ if (p_size - q_size)
+ return p_size - q_size;
+ return p->line_number - q->line_number;
+}
+
+#define PTR_NULL_CHECK(P, Q) do {\
+ if ((Q) == NULL) {\
+ if ((P) != NULL) return 1;\
+ } else {\
+ if ((P) == NULL) return -1;\
+ }\
+ } while (0)
+
+static int compare_node(const struct node *p, const struct node *q) {
+ PTR_NULL_CHECK(p, q);
+ if (q == NULL) {
+ /* p must be NULL too. */
+ return 0;
+ }
+
+ if (p->type != q->type) return p->type > q->type ? 1 : -1;
+ if (p->mode != q->mode) return p->mode > q->mode ? 1 : -1;
+ if (p->type == c_number) {
+ if (p->number != q->number)
+ return p->number > q->number ? 1 : -1;
+ }
+
+ PTR_NULL_CHECK(p->left, q->left);
+ if (p->left) {
+ int r = compare_node(p->left, q->left);
+ if (r != 0) return r;
+ }
+
+ PTR_NULL_CHECK(p->AE, q->AE);
+ if (p->AE) {
+ int r = compare_node(p->AE, q->AE);
+ if (r != 0) return r;
+ }
+
+ PTR_NULL_CHECK(p->aux, q->aux);
+ if (p->aux) {
+ int r = compare_node(p->aux, q->aux);
+ if (r != 0) return r;
+ }
+
+ PTR_NULL_CHECK(p->name, q->name);
+ if (p->name) {
+ int r;
+ if (SIZE(p->name->b) != SIZE(q->name->b)) {
+ return SIZE(p->name->b) - SIZE(q->name->b);
+ }
+ r = memcmp(p->name->b, q->name->b,
+ SIZE(p->name->b) * sizeof(symbol));
+ if (r != 0) return r;
+ }
+
+ PTR_NULL_CHECK(p->literalstring, q->literalstring);
+ if (p->literalstring) {
+ int r;
+ if (SIZE(p->literalstring) != SIZE(q->literalstring)) {
+ return SIZE(p->literalstring) - SIZE(q->literalstring);
+ }
+ r = memcmp(p->literalstring, q->literalstring,
+ SIZE(p->literalstring) * sizeof(symbol));
+ if (r != 0) return r;
+ }
+
+ return compare_node(p->right, q->right);
+}
+
+static void make_among(struct analyser * a, struct node * p, struct node * substring) {
+
+ NEW(among, x);
+ NEWVEC(amongvec, v, p->number);
+ struct node * q = p->left;
+ struct amongvec * w0 = v;
+ struct amongvec * w1 = v;
+ int result = 1;
+
+ int direction = substring != 0 ? substring->mode : p->mode;
+ int backward = direction == m_backward;
+
+ if (a->amongs == 0) a->amongs = x; else a->amongs_end->next = x;
+ a->amongs_end = x;
+ x->next = 0;
+ x->b = v;
+ x->number = a->among_count++;
+ x->function_count = 0;
+ x->starter = 0;
+ x->nocommand_count = 0;
+ x->amongvar_needed = false;
+
+ if (q->type == c_bra) { x->starter = q; q = q->right; }
+
+ while (q) {
+ if (q->type == c_literalstring) {
+ symbol * b = q->literalstring;
+ w1->b = b; /* pointer to case string */
+ w1->action = NULL; /* action gets filled in below */
+ w1->line_number = q->line_number;
+ w1->size = SIZE(b); /* number of characters in string */
+ w1->i = -1; /* index of longest substring */
+ w1->result = -1; /* number of corresponding case expression */
+ if (q->left) {
+ struct name * function = q->left->name;
+ w1->function = function;
+ function->used_in_among = true;
+ check_routine_mode(a, function, direction);
+ x->function_count++;
+ } else {
+ w1->function = 0;
+ }
+ w1++;
+ } else if (q->left == 0) {
+ /* empty command: () */
+ w0 = w1;
+ } else {
+ /* Check for previous action which is the same as this one and use
+ * the same action code if we find one.
+ */
+ int among_result = -1;
+ struct amongvec * w;
+ for (w = v; w < w0; ++w) {
+ if (w->action && compare_node(w->action->left, q->left) == 0) {
+ if (w->result <= 0) {
+ printf("Among code %d isn't positive\n", w->result);
+ exit(1);
+ }
+ among_result = w->result;
+ break;
+ }
+ }
+ if (among_result < 0) {
+ among_result = result++;
+ }
+
+ while (w0 != w1) {
+ w0->action = q;
+ w0->result = among_result;
+ w0++;
+ }
+ }
+ q = q->right;
+ }
+ if (w1-v != p->number) { fprintf(stderr, "oh! %d %d\n", (int)(w1-v), p->number); exit(1); }
+ x->command_count = result - 1;
+ {
+ NEWVEC(node*, commands, x->command_count);
+ memset(commands, 0, x->command_count * sizeof(struct node*));
+ for (w0 = v; w0 < w1; w0++) {
+ if (w0->result > 0) {
+ /* result == -1 when there's no command. */
+ if (w0->result > x->command_count) {
+ fprintf(stderr, "More among codes than expected\n");
+ exit(1);
+ }
+ if (!commands[w0->result - 1])
+ commands[w0->result - 1] = w0->action;
+ } else {
+ ++x->nocommand_count;
+ }
+ if (backward) reverse_b(w0->b);
+ }
+ x->commands = commands;
+ }
+ qsort(v, w1 - v, sizeof(struct amongvec), compare_amongvec);
+
+ /* the following loop is O(n squared) */
+ for (w0 = w1 - 1; w0 >= v; w0--) {
+ symbol * b = w0->b;
+ int size = w0->size;
+ struct amongvec * w;
+
+ for (w = w0 - 1; w >= v; w--) {
+ if (w->size < size && memcmp(w->b, b, w->size * sizeof(symbol)) == 0) {
+ w0->i = w - v; /* fill in index of longest substring */
+ break;
+ }
+ }
+ }
+ if (backward) for (w0 = v; w0 < w1; w0++) reverse_b(w0->b);
+
+ for (w0 = v; w0 < w1 - 1; w0++)
+ if (w0->size == (w0 + 1)->size &&
+ memcmp(w0->b, (w0 + 1)->b, w0->size * sizeof(symbol)) == 0) {
+ count_error(a);
+ fprintf(stderr, "%s:%d: among(...) has repeated string '",
+ a->tokeniser->file, (w0 + 1)->line_number);
+ report_b(stderr, (w0 + 1)->b);
+ fprintf(stderr, "'\n");
+ count_error(a);
+ fprintf(stderr, "%s:%d: previously seen here\n",
+ a->tokeniser->file, w0->line_number);
+ }
+
+ x->literalstring_count = p->number;
+ p->among = x;
+
+ x->substring = substring;
+ if (substring != 0) substring->among = x;
+ if (x->command_count > 1 ||
+ (x->command_count == 1 && x->nocommand_count > 0) ||
+ x->starter != 0) {
+ /* We need to set among_var rather than just checking if find_among*()
+ * returns zero or not.
+ */
+ x->amongvar_needed = a->amongvar_needed = true;
+ }
+}
+
+static struct node * read_among(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p = new_node(a, c_among);
+ struct node * p_end = 0;
+ int previous_token = -1;
+ struct node * substring = a->substring;
+
+ a->substring = 0;
+ p->number = 0; /* counts the number of literals */
+ if (!get_token(a, c_bra)) return p;
+ while (true) {
+ struct node * q;
+ int token = read_token(t);
+ switch (token) {
+ case c_literalstring:
+ q = read_literalstring(a);
+ if (read_token(t) == c_name) {
+ struct node * r = new_node(a, c_name);
+ name_to_node(a, r, 'r');
+ q->left = r;
+ }
+ else t->token_held = true;
+ p->number++; break;
+ case c_bra:
+ if (previous_token == c_bra) error(a, e_adjacent_bracketed_in_among);
+ q = read_C_list(a); break;
+ default:
+ error(a, e_unexpected_token_in_among);
+ previous_token = token;
+ continue;
+ case c_ket:
+ if (p->number == 0) error(a, e_empty_among);
+ if (t->error_count == 0) make_among(a, p, substring);
+ return p;
+ }
+ previous_token = token;
+ if (p_end == 0) p->left = q; else p_end->right = q;
+ p_end = q;
+ }
+}
+
+static struct node * read_substring(struct analyser * a) {
+
+ struct node * p = new_node(a, c_substring);
+ if (a->substring != 0) error2(a, e_substring_preceded_by_substring, a->substring->line_number);
+ a->substring = p;
+ return p;
+}
+
+static void check_modifyable(struct analyser * a) {
+ if (!a->modifyable) error(a, e_not_allowed_inside_reverse);
+}
+
+static struct node * read_C(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ int token = read_token(t);
+ switch (token) {
+ case c_bra:
+ return read_C_list(a);
+ case c_backwards:
+ {
+ int mode = a->mode;
+ if (a->mode == m_backward) error(a, e_already_backwards); else a->mode = m_backward;
+ { struct node * p = C_style(a, "C", token);
+ a->mode = mode;
+ return p;
+ }
+ }
+ case c_reverse:
+ {
+ int mode = a->mode;
+ int modifyable = a->modifyable;
+ a->modifyable = false;
+ a->mode = mode == m_forward ? m_backward : m_forward;
+ {
+ struct node * p = C_style(a, "C", token);
+ a->mode = mode;
+ a->modifyable = modifyable;
+ return p;
+ }
+ }
+ case c_not:
+ case c_try:
+ case c_fail:
+ case c_test:
+ case c_do:
+ case c_goto:
+ case c_gopast:
+ case c_repeat:
+ return C_style(a, "C", token);
+ case c_loop:
+ case c_atleast:
+ return C_style(a, "AC", token);
+ case c_setmark: {
+ struct node * n = C_style(a, "i", token);
+ if (n->name) n->name->initialised = true;
+ return n;
+ }
+ case c_tomark:
+ case c_atmark:
+ case c_hop:
+ return C_style(a, "A", token);
+ case c_delete:
+ check_modifyable(a);
+ /* fall through */
+ case c_next:
+ case c_tolimit:
+ case c_atlimit:
+ case c_leftslice:
+ case c_rightslice:
+ case c_true:
+ case c_false:
+ case c_debug:
+ return new_node(a, token);
+ case c_assignto:
+ case c_sliceto: {
+ struct node *n;
+ check_modifyable(a);
+ n = C_style(a, "s", token);
+ if (n->name) n->name->initialised = true;
+ return n;
+ }
+ case c_assign:
+ case c_insert:
+ case c_attach:
+ case c_slicefrom: {
+ struct node *n;
+ check_modifyable(a);
+ n = C_style(a, "S", token);
+ if (n->name) n->name->value_used = true;
+ return n;
+ }
+ case c_setlimit:
+ return C_style(a, "CfD", token);
+ case c_set:
+ case c_unset: {
+ struct node * n = C_style(a, "b", token);
+ if (n->name) n->name->initialised = true;
+ return n;
+ }
+ case c_dollar: {
+ struct tokeniser * t = a->tokeniser;
+ read_token(t);
+ if (t->token == c_bra) {
+ /* Handle newer $(AE REL_OP AE) syntax. */
+ struct node * n = read_AE(a, 0);
+ read_token(t);
+ switch (t->token) {
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le: {
+ struct node * lhs = n;
+ n = new_node(a, t->token);
+ n->left = lhs;
+ n->AE = read_AE(a, 0);
+ get_token(a, c_ket);
+ break;
+ }
+ default:
+ error(a, e_unexpected_token);
+ t->token_held = true;
+ break;
+ }
+ return n;
+ }
+
+ if (t->token == c_name) {
+ struct node * p;
+ struct name * q = find_name(a);
+ int mode = a->mode;
+ int modifyable = a->modifyable;
+ if (q && q->type == t_string) {
+ /* Assume for now that $ on string both initialises and
+ * uses the string variable. FIXME: Can we do better?
+ */
+ q->initialised = true;
+ q->value_used = true;
+ a->mode = m_forward;
+ a->modifyable = true;
+ p = new_node(a, c_dollar);
+ p->left = read_C(a);
+ p->name = q;
+ } else {
+ if (q && q->type != t_integer) {
+ /* If $ is used on an unknown name or a name which
+ * isn't a string or an integer then we assume the
+ * unknown name is an integer as $ is used more often
+ * on integers than strings, so hopefully this it less
+ * likely to cause an error avalanche.
+ *
+ * For an unknown name, we'll already have reported an
+ * error.
+ */
+ error(a, e_not_of_type_string_or_integer);
+ q = NULL;
+ }
+ p = new_node(a, read_AE_test(a));
+ p->AE = read_AE(a, 0);
+
+ if (q) {
+ switch (p->type) {
+ case c_mathassign:
+ q->initialised = true;
+ p->name = q;
+ break;
+ default:
+ /* +=, etc don't "initialise" as they only
+ * amend an existing value. Similarly, they
+ * don't count as using the value.
+ */
+ p->name = q;
+ break;
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ p->left = new_node(a, c_name);
+ p->left->name = q;
+ q->value_used = true;
+ break;
+ }
+ }
+ }
+ if (q) mark_used_in(a, q, p);
+ a->mode = mode;
+ a->modifyable = modifyable;
+ return p;
+ }
+
+ error(a, e_unexpected_token);
+ t->token_held = true;
+ return new_node(a, c_dollar);
+ }
+ case c_name:
+ {
+ struct name * q = find_name(a);
+ struct node * p = new_node(a, c_name);
+ if (q) {
+ mark_used_in(a, q, p);
+ switch (q->type) {
+ case t_boolean:
+ p->type = c_booltest;
+ q->value_used = true;
+ break;
+ case t_integer:
+ error(a, e_misplaced); /* integer name misplaced */
+ break;
+ case t_string:
+ q->value_used = true;
+ break;
+ case t_routine:
+ case t_external:
+ p->type = c_call;
+ check_routine_mode(a, q, a->mode);
+ break;
+ case t_grouping:
+ p->type = c_grouping; break;
+ }
+ }
+ p->name = q;
+ return p;
+ }
+ case c_non:
+ {
+ struct node * p = new_node(a, token);
+ read_token(t);
+ if (t->token == c_minus) read_token(t);
+ if (!check_token(a, c_name)) { omission_error(a, c_name); return p; }
+ name_to_node(a, p, 'g');
+ return p;
+ }
+ case c_literalstring:
+ return read_literalstring(a);
+ case c_among: return read_among(a);
+ case c_substring: return read_substring(a);
+ default: error(a, e_unexpected_token); return 0;
+ }
+}
+
+static int next_symbol(symbol * p, symbol * W, int utf8) {
+ if (utf8) {
+ int ch;
+ int j = get_utf8(p, & ch);
+ W[0] = ch; return j;
+ } else {
+ W[0] = p[0]; return 1;
+ }
+}
+
+static symbol * alter_grouping(symbol * p, symbol * q, int style, int utf8) {
+ int j = 0;
+ symbol W[1];
+ int width;
+ if (style == c_plus) {
+ while (j < SIZE(q)) {
+ width = next_symbol(q + j, W, utf8);
+ p = add_to_b(p, 1, W);
+ j += width;
+ }
+ } else {
+ while (j < SIZE(q)) {
+ int i;
+ width = next_symbol(q + j, W, utf8);
+ for (i = 0; i < SIZE(p); i++) {
+ if (p[i] == W[0]) {
+ memmove(p + i, p + i + 1, (SIZE(p) - i - 1) * sizeof(symbol));
+ SIZE(p)--;
+ }
+ }
+ j += width;
+ }
+ }
+ return p;
+}
+
+static void read_define_grouping(struct analyser * a, struct name * q) {
+ struct tokeniser * t = a->tokeniser;
+ int style = c_plus;
+ {
+ NEW(grouping, p);
+ if (a->groupings == 0) a->groupings = p; else a->groupings_end->next = p;
+ a->groupings_end = p;
+ if (q) q->grouping = p;
+ p->next = 0;
+ p->name = q;
+ p->line_number = a->tokeniser->line_number;
+ p->b = create_b(0);
+ while (true) {
+ switch (read_token(t)) {
+ case c_name:
+ {
+ struct name * r = find_name(a);
+ if (r) {
+ check_name_type(a, r, 'g');
+ p->b = alter_grouping(p->b, r->grouping->b, style, false);
+ r->used_in_definition = true;
+ }
+ }
+ break;
+ case c_literalstring:
+ p->b = alter_grouping(p->b, t->b, style, (a->encoding == ENC_UTF8));
+ break;
+ default: error(a, e_unexpected_token); return;
+ }
+ switch (read_token(t)) {
+ case c_plus:
+ case c_minus: style = t->token; break;
+ default: goto label0;
+ }
+ }
+ label0:
+ {
+ int i;
+ int max = 0;
+ int min = 1<<16;
+ for (i = 0; i < SIZE(p->b); i++) {
+ if (p->b[i] > max) max = p->b[i];
+ if (p->b[i] < min) min = p->b[i];
+ }
+ p->largest_ch = max;
+ p->smallest_ch = min;
+ if (min == 1<<16) error(a, e_empty_grouping);
+ }
+ t->token_held = true; return;
+ }
+}
+
+static void read_define_routine(struct analyser * a, struct name * q) {
+ struct node * p = new_node(a, c_define);
+ a->amongvar_needed = false;
+ if (q) {
+ check_name_type(a, q, 'R');
+ if (q->definition != 0) error(a, e_redefined);
+ if (q->mode < 0) q->mode = a->mode; else
+ if (q->mode != a->mode) error2(a, e_declared_as_different_mode, q->mode);
+ }
+ p->name = q;
+ if (a->program == 0) a->program = p; else a->program_end->right = p;
+ a->program_end = p;
+ get_token(a, c_as);
+ p->left = read_C(a);
+ if (q) q->definition = p->left;
+
+ if (a->substring != 0) {
+ error2(a, e_unresolved_substring, a->substring->line_number);
+ a->substring = 0;
+ }
+ p->amongvar_needed = a->amongvar_needed;
+}
+
+static void read_define(struct analyser * a) {
+ if (get_token(a, c_name)) {
+ struct name * q = find_name(a);
+ int type;
+ if (q) {
+ type = q->type;
+ } else {
+ /* No declaration, so sniff next token - if it is 'as' then parse
+ * as a routine, otherwise as a grouping.
+ */
+ if (read_token(a->tokeniser) == c_as) {
+ type = t_routine;
+ } else {
+ type = t_grouping;
+ }
+ a->tokeniser->token_held = true;
+ }
+
+ if (type == t_grouping) {
+ read_define_grouping(a, q);
+ } else {
+ read_define_routine(a, q);
+ }
+ }
+}
+
+static void read_backwardmode(struct analyser * a) {
+ int mode = a->mode;
+ a->mode = m_backward;
+ if (get_token(a, c_bra)) {
+ read_program_(a, c_ket);
+ check_token(a, c_ket);
+ }
+ a->mode = mode;
+}
+
+static void read_program_(struct analyser * a, int terminator) {
+ struct tokeniser * t = a->tokeniser;
+ while (true) {
+ switch (read_token(t)) {
+ case c_strings: read_names(a, t_string); break;
+ case c_booleans: read_names(a, t_boolean); break;
+ case c_integers: read_names(a, t_integer); break;
+ case c_routines: read_names(a, t_routine); break;
+ case c_externals: read_names(a, t_external); break;
+ case c_groupings: read_names(a, t_grouping); break;
+ case c_define: read_define(a); break;
+ case c_backwardmode:read_backwardmode(a); break;
+ case c_ket:
+ if (terminator == c_ket) return;
+ /* fall through */
+ default:
+ error(a, e_unexpected_token); break;
+ case -1:
+ if (terminator >= 0) omission_error(a, c_ket);
+ return;
+ }
+ }
+}
+
+static void remove_dead_assignments(struct node * p, struct name * q) {
+ if (p->name == q) {
+ switch (p->type) {
+ case c_assignto:
+ case c_sliceto:
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_setmark:
+ case c_set:
+ case c_unset:
+ case c_dollar:
+ /* c_true is a no-op. */
+ p->type = c_true;
+ break;
+ default:
+ /* There are no read accesses to this variable, so any
+ * references must be assignments.
+ */
+ fprintf(stderr, "Unhandled type of dead assignment via %s\n",
+ name_of_token(p->type));
+ exit(1);
+ }
+ }
+ if (p->AE) remove_dead_assignments(p->AE, q);
+ if (p->left) remove_dead_assignments(p->left, q);
+ if (p->aux) remove_dead_assignments(p->aux, q);
+ if (p->right) remove_dead_assignments(p->right, q);
+}
+
+extern void read_program(struct analyser * a) {
+ read_program_(a, -1);
+ {
+ struct name * q = a->names;
+ while (q) {
+ switch (q->type) {
+ case t_external: case t_routine:
+ if (q->used && q->definition == 0) error4(a, q);
+ break;
+ case t_grouping:
+ if (q->used && q->grouping == 0) error4(a, q);
+ break;
+ }
+ q = q->next;
+ }
+ }
+
+ if (a->tokeniser->error_count == 0) {
+ struct name * q = a->names;
+ struct name ** ptr = &(a->names);
+ while (q) {
+ if (!q->referenced) {
+ fprintf(stderr, "%s:%d: warning: %s '",
+ a->tokeniser->file,
+ q->declaration_line_number,
+ name_of_name_type(q->type));
+ report_b(stderr, q->b);
+ if (q->type == t_routine ||
+ q->type == t_external ||
+ q->type == t_grouping) {
+ fprintf(stderr, "' declared but not defined\n");
+ } else {
+ fprintf(stderr, "' defined but not used\n");
+ q = q->next;
+ *ptr = q;
+ continue;
+ }
+ } else if (q->type == t_routine || q->type == t_grouping) {
+ /* It's OK to define a grouping but only use it to define other
+ * groupings.
+ */
+ if (!q->used && !q->used_in_definition) {
+ int line_num;
+ if (q->type == t_routine) {
+ line_num = q->definition->line_number;
+ } else {
+ line_num = q->grouping->line_number;
+ }
+ fprintf(stderr, "%s:%d: warning: %s '",
+ a->tokeniser->file,
+ line_num,
+ name_of_name_type(q->type));
+ report_b(stderr, q->b);
+ fprintf(stderr, "' defined but not used\n");
+ }
+ } else if (q->type == t_external) {
+ /* Unused is OK. */
+ } else if (!q->initialised) {
+ fprintf(stderr, "%s:%d: warning: %s '",
+ a->tokeniser->file,
+ q->declaration_line_number,
+ name_of_name_type(q->type));
+ report_b(stderr, q->b);
+ fprintf(stderr, "' is never initialised\n");
+ } else if (!q->value_used) {
+ fprintf(stderr, "%s:%d: warning: %s '",
+ a->tokeniser->file,
+ q->declaration_line_number,
+ name_of_name_type(q->type));
+ report_b(stderr, q->b);
+ fprintf(stderr, "' is set but never used\n");
+ remove_dead_assignments(a->program, q);
+ q = q->next;
+ *ptr = q;
+ continue;
+ }
+ ptr = &(q->next);
+ q = q->next;
+ }
+
+ {
+ /* Now we've eliminated variables whose values are never used we
+ * can number the variables, which is used by some generators.
+ */
+ int * name_count = a->name_count;
+ struct name * n;
+ for (n = a->names; n; n = n->next) {
+ n->count = name_count[n->type]++;
+ }
+ }
+ }
+}
+
+extern struct analyser * create_analyser(struct tokeniser * t) {
+ NEW(analyser, a);
+ a->tokeniser = t;
+ a->nodes = 0;
+ a->names = 0;
+ a->literalstrings = 0;
+ a->program = 0;
+ a->amongs = 0;
+ a->among_count = 0;
+ a->groupings = 0;
+ a->mode = m_forward;
+ a->modifyable = true;
+ { int i; for (i = 0; i < t_size; i++) a->name_count[i] = 0; }
+ a->substring = 0;
+ a->int_limits_used = false;
+ return a;
+}
+
+extern void close_analyser(struct analyser * a) {
+ {
+ struct node * q = a->nodes;
+ while (q) {
+ struct node * q_next = q->next;
+ FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct name * q = a->names;
+ while (q) {
+ struct name * q_next = q->next;
+ lose_b(q->b); FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct literalstring * q = a->literalstrings;
+ while (q) {
+ struct literalstring * q_next = q->next;
+ lose_b(q->b); FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct among * q = a->amongs;
+ while (q) {
+ struct among * q_next = q->next;
+ FREE(q->b);
+ FREE(q->commands);
+ FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct grouping * q = a->groupings;
+ while (q) {
+ struct grouping * q_next = q->next;
+ lose_b(q->b); FREE(q);
+ q = q_next;
+ }
+ }
+ FREE(a);
+}
diff --git a/contrib/snowball/compiler/driver.c b/contrib/snowball/compiler/driver.c
new file mode 100644
index 0000000..587028f
--- /dev/null
+++ b/contrib/snowball/compiler/driver.c
@@ -0,0 +1,574 @@
+#include <ctype.h> /* for toupper etc */
+#include <stdio.h> /* for fprintf etc */
+#include <stdlib.h> /* for free etc */
+#include <string.h> /* for strcmp */
+#include "header.h"
+
+#define DEFAULT_JAVA_PACKAGE "org.tartarus.snowball.ext"
+#define DEFAULT_JAVA_BASE_CLASS "org.tartarus.snowball.SnowballProgram"
+#define DEFAULT_JAVA_AMONG_CLASS "org.tartarus.snowball.Among"
+#define DEFAULT_JAVA_STRING_CLASS "java.lang.StringBuilder"
+
+#define DEFAULT_GO_PACKAGE "snowball"
+#define DEFAULT_GO_SNOWBALL_RUNTIME "github.com/snowballstem/snowball/go"
+
+#define DEFAULT_CS_NAMESPACE "Snowball"
+#define DEFAULT_CS_BASE_CLASS "Stemmer"
+#define DEFAULT_CS_AMONG_CLASS "Among"
+#define DEFAULT_CS_STRING_CLASS "StringBuilder"
+
+#define DEFAULT_JS_BASE_CLASS "BaseStemmer"
+
+#define DEFAULT_PYTHON_BASE_CLASS "BaseStemmer"
+
+static int eq(const char * s1, const char * s2) {
+ return strcmp(s1, s2) == 0;
+}
+
+__attribute__((noreturn))
+static void print_arglist(int exit_code) {
+ FILE * f = exit_code ? stderr : stdout;
+ fprintf(f, "Usage: snowball SOURCE_FILE... [OPTIONS]\n\n"
+ "Supported options:\n"
+ " -o[utput] file\n"
+ " -s[yntax]\n"
+ " -comments\n"
+#ifndef DISABLE_JAVA
+ " -j[ava]\n"
+#endif
+#ifndef DISABLE_CSHARP
+ " -cs[harp]\n"
+#endif
+ " -c++\n"
+#ifndef DISABLE_PASCAL
+ " -pascal\n"
+#endif
+#ifndef DISABLE_PYTHON
+ " -py[thon]\n"
+#endif
+#ifndef DISABLE_JS
+ " -js\n"
+#endif
+#ifndef DISABLE_RUST
+ " -rust\n"
+#endif
+#ifndef DISABLE_GO
+ " -go\n"
+#endif
+ " -w[idechars]\n"
+ " -u[tf8]\n"
+ " -n[ame] class name\n"
+ " -ep[refix] string\n"
+ " -vp[refix] string\n"
+ " -i[nclude] directory\n"
+ " -r[untime] path to runtime headers\n"
+ " -p[arentclassname] fully qualified parent class name\n"
+#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP)
+ " -P[ackage] package name for stemmers\n"
+ " -S[tringclass] StringBuffer-compatible class\n"
+ " -a[mongclass] fully qualified name of the Among class\n"
+#endif
+#ifndef DISABLE_GO
+ " -gop[ackage] Go package name for stemmers\n"
+ " -gor[untime] Go snowball runtime package\n"
+#endif
+ " --help display this help and exit\n"
+ " --version output version information and exit\n"
+ );
+ exit(exit_code);
+}
+
+static void check_lim(int i, int argc) {
+ if (i >= argc) {
+ fprintf(stderr, "argument list is one short\n");
+ print_arglist(1);
+ }
+}
+
+static FILE * get_output(symbol * b) {
+ char * s = b_to_s(b);
+ FILE * output = fopen(s, "w");
+ if (output == 0) {
+ fprintf(stderr, "Can't open output %s\n", s);
+ exit(1);
+ }
+ free(s);
+ return output;
+}
+
+static int read_options(struct options * o, int argc, char * argv[]) {
+ char * s;
+ int i = 1;
+ int new_argc = 1;
+ /* Note down the last option used to specify an explicit encoding so
+ * we can warn we ignored it for languages with a fixed encoding.
+ */
+ const char * encoding_opt = NULL;
+
+ /* set defaults: */
+
+ o->output_file = 0;
+ o->syntax_tree = false;
+ o->comments = false;
+ o->externals_prefix = NULL;
+ o->variables_prefix = 0;
+ o->runtime_path = 0;
+ o->parent_class_name = NULL;
+ o->string_class = NULL;
+ o->among_class = NULL;
+ o->package = NULL;
+ o->go_snowball_runtime = DEFAULT_GO_SNOWBALL_RUNTIME;
+ o->name = NULL;
+ o->make_lang = LANG_C;
+ o->includes = 0;
+ o->includes_end = 0;
+ o->encoding = ENC_SINGLEBYTE;
+
+ /* read options: */
+
+ while (i < argc) {
+ s = argv[i++];
+ if (s[0] != '-') {
+ /* Non-option argument - shuffle down. */
+ argv[new_argc++] = s;
+ continue;
+ }
+
+ {
+ if (eq(s, "-o") || eq(s, "-output")) {
+ check_lim(i, argc);
+ o->output_file = argv[i++];
+ continue;
+ }
+ if (eq(s, "-n") || eq(s, "-name")) {
+ check_lim(i, argc);
+ o->name = argv[i++];
+ continue;
+ }
+#ifndef DISABLE_JS
+ if (eq(s, "-js")) {
+ o->make_lang = LANG_JAVASCRIPT;
+ continue;
+ }
+#endif
+#ifndef DISABLE_RUST
+ if (eq(s, "-rust")) {
+ o->make_lang = LANG_RUST;
+ continue;
+ }
+#endif
+#ifndef DISABLE_GO
+ if (eq(s, "-go")) {
+ o->make_lang = LANG_GO;
+ continue;
+ }
+#endif
+#ifndef DISABLE_JAVA
+ if (eq(s, "-j") || eq(s, "-java")) {
+ o->make_lang = LANG_JAVA;
+ continue;
+ }
+#endif
+#ifndef DISABLE_CSHARP
+ if (eq(s, "-cs") || eq(s, "-csharp")) {
+ o->make_lang = LANG_CSHARP;
+ continue;
+ }
+#endif
+ if (eq(s, "-c++")) {
+ o->make_lang = LANG_CPLUSPLUS;
+ continue;
+ }
+#ifndef DISABLE_PASCAL
+ if (eq(s, "-pascal")) {
+ o->make_lang = LANG_PASCAL;
+ continue;
+ }
+#endif
+#ifndef DISABLE_PYTHON
+ if (eq(s, "-py") || eq(s, "-python")) {
+ o->make_lang = LANG_PYTHON;
+ continue;
+ }
+#endif
+ if (eq(s, "-w") || eq(s, "-widechars")) {
+ encoding_opt = s;
+ o->encoding = ENC_WIDECHARS;
+ continue;
+ }
+ if (eq(s, "-s") || eq(s, "-syntax")) {
+ o->syntax_tree = true;
+ continue;
+ }
+ if (eq(s, "-comments")) {
+ o->comments = true;
+ continue;
+ }
+ if (eq(s, "-ep") || eq(s, "-eprefix")) {
+ check_lim(i, argc);
+ o->externals_prefix = argv[i++];
+ continue;
+ }
+ if (eq(s, "-vp") || eq(s, "-vprefix")) {
+ check_lim(i, argc);
+ o->variables_prefix = argv[i++];
+ continue;
+ }
+ if (eq(s, "-i") || eq(s, "-include")) {
+ check_lim(i, argc);
+
+ {
+ NEW(include, p);
+ symbol * b = add_s_to_b(0, argv[i++]);
+ b = add_s_to_b(b, "/");
+ p->next = 0; p->b = b;
+
+ if (o->includes == 0) o->includes = p; else
+ o->includes_end->next = p;
+ o->includes_end = p;
+ }
+ continue;
+ }
+ if (eq(s, "-r") || eq(s, "-runtime")) {
+ check_lim(i, argc);
+ o->runtime_path = argv[i++];
+ continue;
+ }
+ if (eq(s, "-u") || eq(s, "-utf8")) {
+ encoding_opt = s;
+ o->encoding = ENC_UTF8;
+ continue;
+ }
+ if (eq(s, "-p") || eq(s, "-parentclassname")) {
+ check_lim(i, argc);
+ o->parent_class_name = argv[i++];
+ continue;
+ }
+#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP)
+ if (eq(s, "-P") || eq(s, "-Package")) {
+ check_lim(i, argc);
+ o->package = argv[i++];
+ continue;
+ }
+ if (eq(s, "-S") || eq(s, "-stringclass")) {
+ check_lim(i, argc);
+ o->string_class = argv[i++];
+ continue;
+ }
+ if (eq(s, "-a") || eq(s, "-amongclass")) {
+ check_lim(i, argc);
+ o->among_class = argv[i++];
+ continue;
+ }
+#endif
+#ifndef DISABLE_GO
+ if (eq(s, "-gop") || eq(s, "-gopackage")) {
+ check_lim(i, argc);
+ o->package = argv[i++];
+ continue;
+ }
+ if (eq(s, "-gor") || eq(s, "-goruntime")) {
+ check_lim(i, argc);
+ o->go_snowball_runtime = argv[i++];
+ continue;
+ }
+#endif
+ if (eq(s, "--help")) {
+ print_arglist(0);
+ }
+
+ if (eq(s, "--version")) {
+ printf("Snowball compiler version " SNOWBALL_VERSION "\n");
+ exit(0);
+ }
+
+ fprintf(stderr, "'%s' misplaced\n", s);
+ print_arglist(1);
+ }
+ }
+ if (new_argc == 1) {
+ fprintf(stderr, "no source files specified\n");
+ print_arglist(1);
+ }
+ argv[new_argc] = NULL;
+
+ /* Set language-dependent defaults. */
+ switch (o->make_lang) {
+ case LANG_C:
+ case LANG_CPLUSPLUS:
+ encoding_opt = NULL;
+ break;
+ case LANG_CSHARP:
+ o->encoding = ENC_WIDECHARS;
+ if (!o->parent_class_name)
+ o->parent_class_name = DEFAULT_CS_BASE_CLASS;
+ if (!o->string_class)
+ o->string_class = DEFAULT_CS_STRING_CLASS;
+ if (!o->among_class)
+ o->among_class = DEFAULT_CS_AMONG_CLASS;
+ if (!o->package)
+ o->package = DEFAULT_CS_NAMESPACE;
+ break;
+ case LANG_GO:
+ o->encoding = ENC_UTF8;
+ if (!o->package)
+ o->package = DEFAULT_GO_PACKAGE;
+ break;
+ case LANG_JAVA:
+ o->encoding = ENC_WIDECHARS;
+ if (!o->parent_class_name)
+ o->parent_class_name = DEFAULT_JAVA_BASE_CLASS;
+ if (!o->string_class)
+ o->string_class = DEFAULT_JAVA_STRING_CLASS;
+ if (!o->among_class)
+ o->among_class = DEFAULT_JAVA_AMONG_CLASS;
+ if (!o->package)
+ o->package = DEFAULT_JAVA_PACKAGE;
+ break;
+ case LANG_JAVASCRIPT:
+ o->encoding = ENC_WIDECHARS;
+ if (!o->parent_class_name)
+ o->parent_class_name = DEFAULT_JS_BASE_CLASS;
+ break;
+ case LANG_PYTHON:
+ o->encoding = ENC_WIDECHARS;
+ if (!o->parent_class_name)
+ o->parent_class_name = DEFAULT_PYTHON_BASE_CLASS;
+ break;
+ case LANG_RUST:
+ o->encoding = ENC_UTF8;
+ break;
+ default:
+ break;
+ }
+
+ if (encoding_opt) {
+ fprintf(stderr, "warning: %s only meaningful for C and C++\n",
+ encoding_opt);
+ }
+
+ if (o->make_lang != LANG_C && o->make_lang != LANG_CPLUSPLUS) {
+ if (o->runtime_path) {
+ fprintf(stderr, "warning: -r/-runtime only meaningful for C and C++\n");
+ }
+ if (o->externals_prefix) {
+ fprintf(stderr, "warning: -ep/-eprefix only meaningful for C and C++\n");
+ }
+ }
+ if (!o->externals_prefix) o->externals_prefix = "";
+
+ if (!o->name && o->output_file) {
+ /* Default class name to basename of output_file - this is the standard
+ * convention for at least Java and C#.
+ */
+ const char * slash = strrchr(o->output_file, '/');
+ size_t len;
+ const char * leaf = (slash == NULL) ? o->output_file : slash + 1;
+
+ slash = strrchr(leaf, '\\');
+ if (slash != NULL) leaf = slash + 1;
+
+ {
+ const char * dot = strchr(leaf, '.');
+ len = (dot == NULL) ? strlen(leaf) : (size_t)(dot - leaf);
+ }
+
+ {
+ char * new_name = malloc(len + 1);
+ switch (o->make_lang) {
+ case LANG_CSHARP:
+ case LANG_PASCAL:
+ /* Upper case initial letter. */
+ memcpy(new_name, leaf, len);
+ new_name[0] = toupper(new_name[0]);
+ break;
+ case LANG_JAVASCRIPT:
+ case LANG_PYTHON: {
+ /* Upper case initial letter and change each
+ * underscore+letter or hyphen+letter to an upper case
+ * letter.
+ */
+ size_t i, j = 0;
+ int uc_next = true;
+ for (i = 0; i != len; ++i) {
+ unsigned char ch = leaf[i];
+ if (ch == '_' || ch == '-') {
+ uc_next = true;
+ } else {
+ if (uc_next) {
+ new_name[j] = toupper(ch);
+ uc_next = false;
+ } else {
+ new_name[j] = ch;
+ }
+ ++j;
+ }
+ }
+ len = j;
+ break;
+ }
+ default:
+ /* Just copy. */
+ memcpy(new_name, leaf, len);
+ break;
+ }
+ new_name[len] = '\0';
+ o->name = new_name;
+ }
+ }
+
+ return new_argc;
+}
+
+extern int main(int argc, char * argv[]) {
+
+ int i;
+ NEW(options, o);
+ argc = read_options(o, argc, argv);
+ {
+ char * file = argv[1];
+ symbol * u = get_input(file);
+ if (u == 0) {
+ fprintf(stderr, "Can't open input %s\n", file);
+ exit(1);
+ }
+ {
+ struct tokeniser * t = create_tokeniser(u, file);
+ struct analyser * a = create_analyser(t);
+ struct input ** next_input_ptr = &(t->next);
+ a->encoding = t->encoding = o->encoding;
+ t->includes = o->includes;
+ /* If multiple source files are specified, set up the others to be
+ * read after the first in order, using the same mechanism as
+ * 'get' uses. */
+ for (i = 2; i != argc; ++i) {
+ NEW(input, q);
+ file = argv[i];
+ u = get_input(file);
+ if (u == 0) {
+ fprintf(stderr, "Can't open input %s\n", file);
+ exit(1);
+ }
+ q->p = u;
+ q->c = 0;
+ q->file = file;
+ q->file_needs_freeing = false;
+ q->line_number = 1;
+ *next_input_ptr = q;
+ next_input_ptr = &(q->next);
+ }
+ *next_input_ptr = NULL;
+ read_program(a);
+ if (t->error_count > 0) exit(1);
+ if (o->syntax_tree) print_program(a);
+ close_tokeniser(t);
+ if (!o->syntax_tree) {
+ struct generator * g;
+
+ const char * s = o->output_file;
+ if (!s) {
+ fprintf(stderr, "Please include the -o option\n");
+ print_arglist(1);
+ }
+ g = create_generator(a, o);
+ if (o->make_lang == LANG_C || o->make_lang == LANG_CPLUSPLUS) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".h");
+ o->output_h = get_output(b);
+ b[SIZE(b) - 1] = 'c';
+ if (o->make_lang == LANG_CPLUSPLUS) {
+ b = add_s_to_b(b, "c");
+ }
+ o->output_src = get_output(b);
+ lose_b(b);
+
+ generate_program_c(g);
+ fclose(o->output_src);
+ fclose(o->output_h);
+ }
+#ifndef DISABLE_JAVA
+ if (o->make_lang == LANG_JAVA) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".java");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_java(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_PASCAL
+ if (o->make_lang == LANG_PASCAL) {
+ symbol *b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".pas");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_pascal(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_PYTHON
+ if (o->make_lang == LANG_PYTHON) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".py");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_python(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_JS
+ if (o->make_lang == LANG_JAVASCRIPT) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".js");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_js(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_CSHARP
+ if (o->make_lang == LANG_CSHARP) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".cs");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_csharp(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_RUST
+ if (o->make_lang == LANG_RUST) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".rs");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_rust(g);
+ fclose(o->output_src);
+ }
+#endif
+#ifndef DISABLE_GO
+ if (o->make_lang == LANG_GO) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".go");
+ o->output_src = get_output(b);
+ lose_b(b);
+ generate_program_go(g);
+ fclose(o->output_src);
+ }
+#endif
+ close_generator(g);
+ }
+ close_analyser(a);
+ }
+ lose_b(u);
+ }
+ { struct include * p = o->includes;
+ while (p) {
+ struct include * q = p->next;
+ lose_b(p->b); FREE(p); p = q;
+ }
+ }
+ FREE(o);
+ if (space_count) fprintf(stderr, "%d blocks unfreed\n", space_count);
+ return 0;
+}
diff --git a/contrib/snowball/compiler/generator.c b/contrib/snowball/compiler/generator.c
new file mode 100644
index 0000000..eed86c1
--- /dev/null
+++ b/contrib/snowball/compiler/generator.c
@@ -0,0 +1,1725 @@
+
+#include <limits.h> /* for INT_MAX */
+#include <stdio.h> /* for fprintf etc */
+#include <stdlib.h> /* for free etc */
+#include <string.h> /* for strlen */
+#include "header.h"
+
+/* Define this to get warning messages when optimisations can't be used. */
+/* #define OPTIMISATION_WARNINGS */
+
+/* recursive use: */
+
+static void generate(struct generator * g, struct node * p);
+
+static int new_label(struct generator * g) {
+ return g->next_label++;
+}
+
+/* Write routines for simple entities */
+
+/* Write a space if the preceding character was not whitespace */
+static void ws_opt_space(struct generator * g, const char * s) {
+ int ch = str_back(g->outbuf);
+ if (ch != ' ' && ch != '\n' && ch != '\t' && ch != -1)
+ write_char(g, ' ');
+ write_string(g, s);
+}
+
+static void wi3(struct generator * g, int i) {
+ if (i < 100) write_char(g, ' ');
+ if (i < 10) write_char(g, ' ');
+ write_int(g, i); /* integer (width 3) */
+}
+
+
+/* Write routines for items from the syntax tree */
+
+static void write_varname(struct generator * g, struct name * p) {
+
+ int ch = "SIIrxg"[p->type];
+ switch (p->type) {
+ case t_external:
+ write_string(g, g->options->externals_prefix); break;
+ case t_string:
+ case t_boolean:
+ case t_integer: {
+ int count = p->count;
+ if (count < 0) {
+ fprintf(stderr, "Reference to optimised out variable ");
+ report_b(stderr, p->b);
+ fprintf(stderr, " attempted\n");
+ exit(1);
+ }
+ if (p->type == t_boolean) {
+ /* We use a single array for booleans and integers, with the
+ * integers first.
+ */
+ count += g->analyser->name_count[t_integer];
+ }
+ write_char(g, ch);
+ write_char(g, '[');
+ write_int(g, count);
+ write_char(g, ']');
+ return;
+ }
+ default:
+ write_char(g, ch); write_char(g, '_');
+ }
+ write_b(g, p->b);
+}
+
+static void write_varref(struct generator * g, struct name * p) { /* reference to variable */
+ if (p->type < t_routine) write_string(g, "z->");
+ write_varname(g, p);
+}
+
+static void write_hexdigit(struct generator * g, int i) {
+ str_append_ch(g->outbuf, "0123456789ABCDEF"[i & 0xF]); /* hexchar */
+}
+
+static void write_hex(struct generator * g, int i) {
+ if (i >> 4) write_hex(g, i >> 4);
+ write_hexdigit(g, i); /* hex integer */
+}
+
+/* write character literal */
+static void wlitch(struct generator * g, int ch) {
+ if (32 <= ch && ch < 127) {
+ write_char(g, '\'');
+ if (ch == '\'' || ch == '\\') {
+ write_char(g, '\\');
+ }
+ write_char(g, ch);
+ write_char(g, '\'');
+ } else {
+ write_string(g, "0x"); write_hex(g, ch);
+ }
+}
+
+static void wlitarray(struct generator * g, symbol * p) { /* write literal array */
+
+ write_string(g, "{ ");
+ {
+ int i;
+ for (i = 0; i < SIZE(p); i++) {
+ wlitch(g, p[i]);
+ if (i < SIZE(p) - 1) write_string(g, ", ");
+ }
+ }
+ write_string(g, " }");
+}
+
+static void wlitref(struct generator * g, symbol * p) { /* write ref to literal array */
+
+ if (SIZE(p) == 0) {
+ write_char(g, '0');
+ } else {
+ struct str * s = g->outbuf;
+ g->outbuf = g->declarations;
+ write_string(g, "static const symbol s_"); write_int(g, g->literalstring_count); write_string(g, "[] = ");
+ wlitarray(g, p);
+ write_string(g, ";\n");
+ g->outbuf = s;
+ write_string(g, "s_"); write_int(g, g->literalstring_count);
+ g->literalstring_count++;
+ }
+}
+
+static void write_margin(struct generator * g) {
+ int i;
+ for (i = 0; i < g->margin; i++) write_string(g, " ");
+}
+
+void write_comment_content(struct generator * g, struct node * p) {
+ switch (p->type) {
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ if (p->name) {
+ write_char(g, '$');
+ write_b(g, p->name->b);
+ write_char(g, ' ');
+ }
+ write_string(g, name_of_token(p->type));
+ write_string(g, " <integer expression>");
+ break;
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ write_string(g, "$(<integer expression> ");
+ write_string(g, name_of_token(p->type));
+ write_string(g, " <integer expression>)");
+ break;
+ default:
+ write_string(g, name_of_token(p->type));
+ if (p->name) {
+ write_char(g, ' ');
+ write_b(g, p->name->b);
+ }
+ }
+ write_string(g, ", line ");
+ write_int(g, p->line_number);
+}
+
+static void write_comment(struct generator * g, struct node * p) {
+ if (g->options->comments) {
+ ws_opt_space(g, "/* ");
+ write_comment_content(g, p);
+ write_string(g, " */");
+ }
+ write_newline(g);
+}
+
+static void wms(struct generator * g, const char * s) {
+ write_margin(g); write_string(g, s); } /* margin + string */
+
+static void write_block_start(struct generator * g) { /* block start */
+ wms(g, "{ ");
+ g->margin++;
+}
+
+static void write_block_end(struct generator * g) { /* block end */
+
+ if (g->line_labelled == g->line_count) { wms(g, ";"); write_newline(g); }
+ g->margin--;
+ wms(g, "}"); write_newline(g);
+}
+
+static void w(struct generator * g, const char * s);
+
+/* keep c */
+static void wk(struct generator * g, struct node * p, int keep_limit) {
+ ++g->keep_count;
+ if (p->mode == m_forward) {
+ write_string(g, "int c");
+ write_int(g, g->keep_count);
+ write_string(g, " = z->c");
+ if (keep_limit) {
+ write_string(g, ", mlimit");
+ write_int(g, g->keep_count);
+ }
+ write_char(g, ';');
+ } else {
+ write_string(g, "int m");
+ write_int(g, g->keep_count);
+ write_string(g, " = z->l - z->c");
+ if (keep_limit) {
+ write_string(g, ", mlimit");
+ write_int(g, g->keep_count);
+ }
+ write_string(g, "; (void)m");
+ write_int(g, g->keep_count);
+ write_char(g, ';');
+ }
+}
+
+static void wrestore(struct generator * g, struct node * p, int keep_token) { /* restore c */
+ if (p->mode == m_forward) {
+ write_string(g, "z->c = c");
+ } else {
+ write_string(g, "z->c = z->l - m");
+ }
+ write_int(g, keep_token); write_char(g, ';');
+}
+
+static void wrestorelimit(struct generator * g, struct node * p, int keep_token) { /* restore limit */
+ if (p->mode == m_forward) {
+ w(g, "z->l += mlimit");
+ } else {
+ w(g, "z->lb = mlimit");
+ }
+ write_int(g, keep_token); write_string(g, ";");
+}
+
+static void winc(struct generator * g, struct node * p) { /* increment c */
+ write_string(g, p->mode == m_forward ? "z->c++;" :
+ "z->c--;");
+}
+
+static void wsetl(struct generator * g, int n) {
+
+ g->margin--;
+ wms(g, "lab"); write_int(g, n); write_char(g, ':'); write_newline(g);
+ g->line_labelled = g->line_count;
+ g->margin++;
+}
+
+static void wgotol(struct generator * g, int n) {
+ wms(g, "goto lab"); write_int(g, n); write_char(g, ';'); write_newline(g);
+}
+
+static void write_failure(struct generator * g, struct node * p) { /* fail */
+ if (g->failure_keep_count != 0) {
+ write_string(g, "{ ");
+ if (g->failure_keep_count > 0) {
+ wrestore(g, p, g->failure_keep_count);
+ } else {
+ wrestorelimit(g, p, -g->failure_keep_count);
+ }
+ write_char(g, ' ');
+ }
+ switch (g->failure_label) {
+ case x_return:
+ write_string(g, "return 0;");
+ break;
+ default:
+ write_string(g, "goto lab");
+ write_int(g, g->failure_label);
+ write_char(g, ';');
+ g->label_used = 1;
+ }
+ if (g->failure_keep_count != 0) write_string(g, " }");
+}
+
+
+/* if at limit fail */
+static void write_check_limit(struct generator * g, struct node * p) {
+
+ write_string(g, p->mode == m_forward ? "if (z->c >= z->l) " :
+ "if (z->c <= z->lb) ");
+ write_failure(g, p);
+}
+
+static void write_data_address(struct generator * g, struct node * p) {
+ symbol * b = p->literalstring;
+ if (b != 0) {
+ write_int(g, SIZE(b)); w(g, ", ");
+ wlitref(g, b);
+ } else {
+ write_varref(g, p->name);
+ }
+}
+
+/* Formatted write. */
+static void writef(struct generator * g, const char * input, struct node * p) {
+ int i = 0;
+ int l = strlen(input);
+
+ while (i < l) {
+ int ch = input[i++];
+ if (ch != '~') {
+ write_char(g, ch);
+ continue;
+ }
+ switch (input[i++]) {
+ default: write_char(g, input[i - 1]); continue;
+ case 'C': write_comment(g, p); continue;
+ case 'k': wk(g, p, false); continue;
+ case 'K': wk(g, p, true); continue;
+ case 'i': winc(g, p); continue;
+ case 'l': write_check_limit(g, p); continue;
+ case 'f': write_failure(g, p); continue;
+ case 'M': write_margin(g); continue;
+ case 'N': write_newline(g); continue;
+ case '{': write_block_start(g); continue;
+ case '}': write_block_end(g); continue;
+ case 'S': write_string(g, g->S[input[i++] - '0']); continue;
+ case 'I': write_int(g, g->I[input[i++] - '0']); continue;
+ case 'J': wi3(g, g->I[input[i++] - '0']); continue;
+ case 'V': write_varref(g, g->V[input[i++] - '0']); continue;
+ case 'W': write_varname(g, g->V[input[i++] - '0']); continue;
+ case 'L': wlitref(g, g->L[input[i++] - '0']); continue;
+ case 'A': wlitarray(g, g->L[input[i++] - '0']); continue;
+ case 'c': wlitch(g, g->I[input[i++] - '0']); continue;
+ case 'a': write_data_address(g, p); continue;
+ case '+': g->margin++; continue;
+ case '-': g->margin--; continue;
+ case '$': /* insert_s, insert_v etc */
+ write_char(g, p->literalstring == 0 ? 'v' : 's');
+ continue;
+ case 'p': write_string(g, g->options->externals_prefix); continue;
+ }
+ }
+}
+
+static void w(struct generator * g, const char * s) {
+ writef(g, s, 0);
+}
+
+static void generate_AE(struct generator * g, struct node * p) {
+ const char * s;
+ switch (p->type) {
+ case c_name:
+ write_varref(g, p->name); break;
+ case c_number:
+ write_int(g, p->number); break;
+ case c_maxint:
+ write_string(g, "MAXINT"); break;
+ case c_minint:
+ write_string(g, "MININT"); break;
+ case c_neg:
+ write_char(g, '-'); generate_AE(g, p->right); break;
+ case c_multiply:
+ s = " * "; goto label0;
+ case c_plus:
+ s = " + "; goto label0;
+ case c_minus:
+ s = " - "; goto label0;
+ case c_divide:
+ s = " / ";
+ label0:
+ write_char(g, '('); generate_AE(g, p->left);
+ write_string(g, s); generate_AE(g, p->right); write_char(g, ')'); break;
+ case c_cursor:
+ w(g, "z->c"); break;
+ case c_limit:
+ w(g, p->mode == m_forward ? "z->l" : "z->lb"); break;
+ case c_len:
+ if (g->options->encoding == ENC_UTF8) {
+ w(g, "len_utf8(z->p)");
+ break;
+ }
+ /* FALLTHRU */
+ case c_size:
+ w(g, "SIZE(z->p)");
+ break;
+ case c_lenof:
+ if (g->options->encoding == ENC_UTF8) {
+ g->V[0] = p->name;
+ w(g, "len_utf8(~V0)");
+ break;
+ }
+ /* FALLTHRU */
+ case c_sizeof:
+ g->V[0] = p->name;
+ w(g, "SIZE(~V0)");
+ break;
+ }
+}
+
+/* K_needed() tests to see if we really need to keep c. Not true when the
+ command does not touch the cursor. This and repeat_score() could be
+ elaborated almost indefinitely.
+*/
+
+static int K_needed_(struct generator * g, struct node * p, int call_depth) {
+ while (p) {
+ switch (p->type) {
+ case c_atlimit:
+ case c_do:
+ case c_dollar:
+ case c_leftslice:
+ case c_rightslice:
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ case c_sliceto:
+ case c_booltest:
+ case c_set:
+ case c_unset:
+ case c_true:
+ case c_false:
+ case c_debug:
+ break;
+
+ case c_call:
+ /* Recursive functions aren't typical in snowball programs, so
+ * make the pessimistic assumption that keep is needed if we
+ * hit a generous limit on recursion. It's not likely to make
+ * a difference to any real world program, but means we won't
+ * recurse until we run out of stack for pathological cases.
+ */
+ if (call_depth >= 100) return true;
+ if (K_needed_(g, p->name->definition, call_depth + 1))
+ return true;
+ break;
+
+ case c_bra:
+ if (K_needed_(g, p->left, call_depth)) return true;
+ break;
+
+ default: return true;
+ }
+ p = p->right;
+ }
+ return false;
+}
+
+extern int K_needed(struct generator * g, struct node * p) {
+ return K_needed_(g, p, 0);
+}
+
+static int repeat_score(struct generator * g, struct node * p, int call_depth) {
+ int score = 0;
+ while (p) {
+ switch (p->type) {
+ case c_dollar:
+ case c_leftslice:
+ case c_rightslice:
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ case c_sliceto: /* case c_not: must not be included here! */
+ case c_debug:
+ break;
+
+ case c_call:
+ /* Recursive functions aren't typical in snowball programs, so
+ * make the pessimistic assumption that repeat requires cursor
+ * reinstatement if we hit a generous limit on recursion. It's
+ * not likely to make a difference to any real world program,
+ * but means we won't recurse until we run out of stack for
+ * pathological cases.
+ */
+ if (call_depth >= 100) {
+ return 2;
+ }
+ score += repeat_score(g, p->name->definition, call_depth + 1);
+ if (score >= 2)
+ return score;
+ break;
+
+ case c_bra:
+ score += repeat_score(g, p->left, call_depth);
+ if (score >= 2)
+ return score;
+ break;
+
+ case c_name:
+ case c_literalstring:
+ case c_next:
+ case c_grouping:
+ case c_non:
+ case c_hop:
+ if (++score >= 2)
+ return score;
+ break;
+
+ default:
+ return 2;
+ }
+ p = p->right;
+ }
+ return score;
+}
+
+/* tests if an expression requires cursor reinstatement in a repeat */
+
+extern int repeat_restore(struct generator * g, struct node * p) {
+ return repeat_score(g, p, 0) >= 2;
+}
+
+static void generate_bra(struct generator * g, struct node * p) {
+ p = p->left;
+ while (p) {
+ generate(g, p);
+ p = p->right;
+ }
+}
+
+static void generate_and(struct generator * g, struct node * p) {
+ int keep_c = 0;
+ if (K_needed(g, p->left)) {
+ writef(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ writef(g, "~M~C", p);
+ }
+ p = p->left;
+ while (p) {
+ generate(g, p);
+ if (keep_c && p->right != 0) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+ p = p->right;
+ }
+ if (keep_c) w(g, "~}");
+}
+
+static void generate_or(struct generator * g, struct node * p) {
+ int keep_c = 0;
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+
+ int out_lab = new_label(g);
+
+ if (K_needed(g, p->left)) {
+ writef(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ writef(g, "~M~C", p);
+ }
+ p = p->left;
+ g->failure_keep_count = 0;
+ while (p->right) {
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ generate(g, p);
+ wgotol(g, out_lab);
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+ p = p->right;
+ }
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+
+ generate(g, p);
+ if (keep_c) w(g, "~}");
+ wsetl(g, out_lab);
+}
+
+static void generate_backwards(struct generator * g, struct node * p) {
+
+ writef(g, "~Mz->lb = z->c; z->c = z->l;~C~N", p);
+ generate(g, p->left);
+ w(g, "~Mz->c = z->lb;~N");
+}
+
+
+static void generate_not(struct generator * g, struct node * p) {
+ int keep_c = 0;
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+
+ if (K_needed(g, p->left)) {
+ writef(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ writef(g, "~M~C", p);
+ }
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_keep_count = 0;
+ generate(g, p->left);
+
+ {
+ int l = g->failure_label;
+ int u = g->label_used;
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+
+ writef(g, "~M~f~N", p);
+ if (u)
+ wsetl(g, l);
+ }
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N~}");
+ }
+}
+
+
+static void generate_try(struct generator * g, struct node * p) {
+ int keep_c = 0;
+ if (K_needed(g, p->left)) {
+ writef(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ writef(g, "~M~C", p);
+ }
+ g->failure_keep_count = keep_c;
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ generate(g, p->left);
+
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+
+ if (keep_c) w(g, "~}");
+}
+
+static void generate_set(struct generator * g, struct node * p) {
+ g->V[0] = p->name; writef(g, "~M~V0 = 1;~C", p);
+}
+
+static void generate_unset(struct generator * g, struct node * p) {
+ g->V[0] = p->name; writef(g, "~M~V0 = 0;~C", p);
+}
+
+static void generate_fail(struct generator * g, struct node * p) {
+ generate(g, p->left);
+ writef(g, "~M~f~C", p);
+}
+
+/* generate_test() also implements 'reverse' */
+
+static void generate_test(struct generator * g, struct node * p) {
+ int keep_c = 0;
+ if (K_needed(g, p->left)) {
+ keep_c = ++g->keep_count;
+ w(g, p->mode == m_forward ? "~{int c_test" :
+ "~{int m_test");
+ write_int(g, keep_c);
+ w(g, p->mode == m_forward ? " = z->c;" :
+ " = z->l - z->c;");
+ writef(g, "~C", p);
+ } else writef(g, "~M~C", p);
+
+ generate(g, p->left);
+
+ if (keep_c) {
+ w(g, p->mode == m_forward ? "~Mz->c = c_test" :
+ "~Mz->c = z->l - m_test");
+ write_int(g, keep_c);
+ writef(g, ";~N~}", p);
+ }
+}
+
+static void generate_do(struct generator * g, struct node * p) {
+ int keep_c = 0;
+ if (K_needed(g, p->left)) {
+ writef(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ writef(g, "~M~C", p);
+ }
+
+ if (p->left->type == c_call) {
+ /* Optimise do <call> */
+ g->V[0] = p->left->name;
+ writef(g, "~{int ret = ~V0(z);~C", p->left);
+ w(g, "~Mif (ret < 0) return ret;~N~}");
+ } else {
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_keep_count = 0;
+ generate(g, p->left);
+
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ }
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c);
+ w(g, "~N~}");
+ }
+}
+
+static void generate_next(struct generator * g, struct node * p) {
+ if (g->options->encoding == ENC_UTF8) {
+ if (p->mode == m_forward)
+ w(g, "~{int ret = skip_utf8(z->p, z->c, 0, z->l, 1");
+ else
+ w(g, "~{int ret = skip_utf8(z->p, z->c, z->lb, 0, -1");
+ writef(g, ");~N"
+ "~Mif (ret < 0) ~f~N"
+ "~Mz->c = ret;~C"
+ "~}", p);
+ } else
+ writef(g, "~M~l~N"
+ "~M~i~C", p);
+}
+
+static void generate_GO_grouping(struct generator * g, struct node * p, int is_goto, int complement) {
+
+ struct grouping * q = p->name->grouping;
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->S[1] = complement ? "in" : "out";
+ g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : "";
+ g->V[0] = p->name;
+ g->I[0] = q->smallest_ch;
+ g->I[1] = q->largest_ch;
+ if (is_goto) {
+ writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1) < 0) ~f~C", p);
+ } else {
+ writef(g, "~{~C"
+ "~Mint ret = ~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1);~N"
+ "~Mif (ret < 0) ~f~N", p);
+ if (p->mode == m_forward)
+ w(g, "~Mz->c += ret;~N");
+ else
+ w(g, "~Mz->c -= ret;~N");
+ w(g, "~}");
+ }
+}
+
+static void generate_GO(struct generator * g, struct node * p, int style) {
+ int keep_c = 0;
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+
+ if (p->left->type == c_grouping || p->left->type == c_non) {
+ /* Special case for "goto" or "gopast" when used on a grouping or an
+ * inverted grouping - the movement of c by the matching action is
+ * exactly what we want! */
+#ifdef OPTIMISATION_WARNINGS
+ printf("Optimising %s %s\n", style ? "goto" : "gopast", p->left->type == c_non ? "non" : "grouping");
+#endif
+ if (g->options->comments) {
+ writef(g, "~M~C", p);
+ }
+ generate_GO_grouping(g, p->left, style, p->left->type == c_non);
+ return;
+ }
+
+ w(g, "~Mwhile(1) {"); writef(g, "~C~+", p);
+
+ if (style == 1 || repeat_restore(g, p->left)) {
+ writef(g, "~M~k~N", p);
+ keep_c = g->keep_count;
+ }
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ generate(g, p->left);
+
+ if (style == 1) {
+ /* include for goto; omit for gopast */
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+ w(g, "~Mbreak;~N");
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+
+/* writef(g, "~M~l~N"
+ "~M~i~N", p); */
+ generate_next(g, p);
+ w(g, "~}");
+}
+
+static void generate_loop(struct generator * g, struct node * p) {
+ w(g, "~{int i; for (i = "); generate_AE(g, p->AE); writef(g, "; i > 0; i--)~C"
+ "~{", p);
+
+ generate(g, p->left);
+
+ w(g, "~}"
+ "~}");
+}
+
+static void generate_repeat_or_atleast(struct generator * g, struct node * p, int atleast_case) {
+ int keep_c = 0;
+ if (atleast_case) {
+ writef(g, "~Mwhile(1) {~+~N", p);
+ } else {
+ writef(g, "~Mwhile(1) {~+~C", p);
+ }
+
+ if (repeat_restore(g, p->left)) {
+ writef(g, "~M~k~N", p);
+ keep_c = g->keep_count;
+ }
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_keep_count = 0;
+ generate(g, p->left);
+
+ if (atleast_case) w(g, "~Mi--;~N");
+
+ w(g, "~Mcontinue;~N");
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+
+ w(g, "~Mbreak;~N"
+ "~}");
+}
+
+static void generate_repeat(struct generator * g, struct node * p) {
+ generate_repeat_or_atleast(g, p, false);
+}
+
+static void generate_atleast(struct generator * g, struct node * p) {
+ w(g, "~{int i = "); generate_AE(g, p->AE); w(g, ";~C");
+ {
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+
+ generate_repeat_or_atleast(g, p, true);
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+ }
+ writef(g, "~Mif (i > 0) ~f~N"
+ "~}", p);
+}
+
+static void generate_setmark(struct generator * g, struct node * p) {
+ g->V[0] = p->name;
+ writef(g, "~M~V0 = z->c;~C", p);
+}
+
+static void generate_tomark(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? ">" : "<";
+
+ w(g, "~Mif (z->c ~S0 "); generate_AE(g, p->AE); writef(g, ") ~f~N", p);
+ w(g, "~Mz->c = "); generate_AE(g, p->AE); writef(g, ";~C", p);
+}
+
+static void generate_atmark(struct generator * g, struct node * p) {
+
+ w(g, "~Mif (z->c != "); generate_AE(g, p->AE); writef(g, ") ~f~C", p);
+}
+
+static void generate_hop(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "+" : "-";
+ g->S[1] = p->mode == m_forward ? "0" : "z->lb";
+ if (g->options->encoding == ENC_UTF8) {
+ w(g, "~{int ret = skip_utf8(z->p, z->c, ~S1, z->l, ~S0 ");
+ generate_AE(g, p->AE); writef(g, ");~C", p);
+ writef(g, "~Mif (ret < 0) ~f~N", p);
+ } else {
+ w(g, "~{int ret = z->c ~S0 ");
+ generate_AE(g, p->AE); writef(g, ";~C", p);
+ writef(g, "~Mif (~S1 > ret || ret > z->l) ~f~N", p);
+ }
+ writef(g, "~Mz->c = ret;~N"
+ "~}", p);
+}
+
+static void generate_delete(struct generator * g, struct node * p) {
+ writef(g, "~{int ret = slice_del(z);~C", p);
+ writef(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+static void generate_tolimit(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "" : "b";
+ writef(g, "~Mz->c = z->l~S0;~C", p);
+}
+
+static void generate_atlimit(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "" : "b";
+ g->S[1] = p->mode == m_forward ? "<" : ">";
+ writef(g, "~Mif (z->c ~S1 z->l~S0) ~f~C", p);
+}
+
+static void generate_leftslice(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "bra" : "ket";
+ writef(g, "~Mz->~S0 = z->c;~C", p);
+}
+
+static void generate_rightslice(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "ket" : "bra";
+ writef(g, "~Mz->~S0 = z->c;~C", p);
+}
+
+static void generate_assignto(struct generator * g, struct node * p) {
+ g->V[0] = p->name;
+ writef(g, "~M~V0 = assign_to(z, ~V0);~C"
+ "~Mif (~V0 == 0) return -1;~C", p);
+}
+
+static void generate_sliceto(struct generator * g, struct node * p) {
+ g->V[0] = p->name;
+ writef(g, "~M~V0 = slice_to(z, ~V0);~C"
+ "~Mif (~V0 == 0) return -1;~N", p);
+}
+
+static void generate_insert(struct generator * g, struct node * p, int style) {
+
+ int keep_c = style == c_attach;
+ if (p->mode == m_backward) keep_c = !keep_c;
+ writef(g, "~{int ret;~N", p);
+ if (keep_c) w(g, "~{int saved_c = z->c;~N");
+ writef(g, "~Mret = insert_~$(z, z->c, z->c, ~a);~C", p);
+ if (keep_c) w(g, "~Mz->c = saved_c;~N~}");
+ writef(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+static void generate_assignfrom(struct generator * g, struct node * p) {
+
+ int keep_c = p->mode == m_forward; /* like 'attach' */
+ writef(g, "~{int ret;~N", p);
+ if (keep_c) writef(g, "~{int saved_c = z->c;~N", p);
+ w(g, "~Mret = ");
+ writef(g, keep_c ? "insert_~$(z, z->c, z->l, ~a);~C" : "insert_~$(z, z->lb, z->c, ~a);~C", p);
+ if (keep_c) w(g, "~Mz->c = saved_c;~N~}");
+ writef(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+/* bugs marked <======= fixed 22/7/02. Similar fixes required for Java */
+
+static void generate_slicefrom(struct generator * g, struct node * p) {
+
+/* w(g, "~Mslice_from_s(z, "); <============= bug! should be: */
+ writef(g, "~{int ret = slice_from_~$(z, ~a);~C", p);
+ writef(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+static void generate_setlimit(struct generator * g, struct node * p) {
+ int keep_c;
+ if (p->left && p->left->type == c_tomark) {
+ /* Special case for:
+ *
+ * setlimit tomark AE for C
+ *
+ * All uses of setlimit in the current stemmers we ship follow this
+ * pattern, and by special-casing we can avoid having to save and
+ * restore c.
+ */
+ struct node * q = p->left;
+
+ ++g->keep_count;
+ writef(g, "~N~{int mlimit", p);
+ write_int(g, g->keep_count);
+ writef(g, ";~C", p);
+ keep_c = g->keep_count;
+
+ g->S[0] = q->mode == m_forward ? ">" : "<";
+
+ w(g, "~Mif (z->c ~S0 "); generate_AE(g, q->AE); writef(g, ") ~f~N", q);
+ w(g, "~Mmlimit");
+ write_int(g, keep_c);
+ if (p->mode == m_forward) {
+ w(g, " = z->l - z->c; z->l = ");
+ } else {
+ w(g, " = z->lb; z->lb = ");
+ }
+ generate_AE(g, q->AE);
+ w(g, ";~N");
+ } else {
+ writef(g, "~{~K~C", p);
+ keep_c = g->keep_count;
+ generate(g, p->left);
+
+ w(g, "~Mmlimit");
+ write_int(g, keep_c);
+ if (p->mode == m_forward)
+ w(g, " = z->l - z->c; z->l = z->c;~N");
+ else
+ w(g, " = z->lb; z->lb = z->c;~N");
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+
+ g->failure_keep_count = -keep_c;
+ generate(g, p->aux);
+ w(g, "~M");
+ wrestorelimit(g, p, -g->failure_keep_count);
+ w(g, "~N"
+ "~}");
+}
+
+/* dollar sets snowball up to operate on a string variable as if it were the
+ * current string */
+static void generate_dollar(struct generator * g, struct node * p) {
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+ int keep_token;
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_keep_count = 0;
+
+ keep_token = ++g->keep_count;
+ g->I[0] = keep_token;
+ writef(g, "~{struct SN_env env~I0 = * z;~C", p);
+ g->V[0] = p->name;
+ /* Assume failure. */
+ writef(g, "~Mint failure = 1;~N"
+ "~Mz->p = ~V0;~N"
+ "~Mz->lb = z->c = 0;~N"
+ "~Mz->l = SIZE(z->p);~N", p);
+ generate(g, p->left);
+ /* Mark success. */
+ w(g, "~Mfailure = 0;~N");
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ g->V[0] = p->name; /* necessary */
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+
+ g->I[0] = keep_token;
+ writef(g, "~M~V0 = z->p;~N"
+ "~M* z = env~I0;~N"
+ "~Mif (failure) ~f~N~}", p);
+}
+
+static void generate_integer_assign(struct generator * g, struct node * p, char * s) {
+
+ g->V[0] = p->name;
+ g->S[0] = s;
+ w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); writef(g, ";~C", p);
+}
+
+static void generate_integer_test(struct generator * g, struct node * p, char * s) {
+
+ w(g, "~Mif (!(");
+ generate_AE(g, p->left);
+ write_char(g, ' ');
+ write_string(g, s);
+ write_char(g, ' ');
+ generate_AE(g, p->AE);
+ writef(g, ")) ~f~C", p);
+}
+
+static void generate_call(struct generator * g, struct node * p) {
+
+ g->V[0] = p->name;
+ writef(g, "~{int ret = ~V0(z);~C", p);
+ if (g->failure_keep_count == 0 && g->failure_label == x_return) {
+ /* Combine the two tests in this special case for better optimisation
+ * and clearer generated code. */
+ writef(g, "~Mif (ret <= 0) return ret;~N~}", p);
+ } else {
+ writef(g, "~Mif (ret == 0) ~f~N"
+ "~Mif (ret < 0) return ret;~N~}", p);
+ }
+}
+
+static void generate_grouping(struct generator * g, struct node * p, int complement) {
+
+ struct grouping * q = p->name->grouping;
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->S[1] = complement ? "out" : "in";
+ g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : "";
+ g->V[0] = p->name;
+ g->I[0] = q->smallest_ch;
+ g->I[1] = q->largest_ch;
+ writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 0)) ~f~C", p);
+}
+
+static void generate_namedstring(struct generator * g, struct node * p) {
+
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->V[0] = p->name;
+ writef(g, "~Mif (!(eq_v~S0(z, ~V0))) ~f~C", p);
+}
+
+static void generate_literalstring(struct generator * g, struct node * p) {
+ symbol * b = p->literalstring;
+ if (SIZE(b) == 1) {
+ /* It's quite common to compare with a single character literal string,
+ * so just inline the simpler code for this case rather than making a
+ * function call. In UTF-8 mode, only do this for the ASCII subset,
+ * since multi-byte characters are more complex to test against.
+ */
+ if (g->options->encoding == ENC_UTF8 && *b >= 128) {
+ printf("single byte %d\n", *b);
+ exit(1);
+ }
+ g->I[0] = *b;
+ if (p->mode == m_forward) {
+ writef(g, "~Mif (z->c == z->l || z->p[z->c] != ~c0) ~f~C"
+ "~Mz->c++;~N", p);
+ } else {
+ writef(g, "~Mif (z->c <= z->lb || z->p[z->c - 1] != ~c0) ~f~C"
+ "~Mz->c--;~N", p);
+ }
+ } else {
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->I[0] = SIZE(b);
+ g->L[0] = b;
+
+ writef(g, "~Mif (!(eq_s~S0(z, ~I0, ~L0))) ~f~C", p);
+ }
+}
+
+static void generate_define(struct generator * g, struct node * p) {
+ struct name * q = p->name;
+ g->next_label = 0;
+
+ g->S[0] = q->type == t_routine ? "static" : "extern";
+ g->V[0] = q;
+
+ w(g, "~N~S0 int ~V0(struct SN_env * z) {");
+ if (g->options->comments) {
+ write_string(g, p->mode == m_forward ? " /* forwardmode */" : " /* backwardmode */");
+ }
+ w(g, "~N~+");
+ if (p->amongvar_needed) w(g, "~Mint among_var;~N");
+ g->failure_keep_count = 0;
+ g->failure_label = x_return;
+ g->label_used = 0;
+ g->keep_count = 0;
+ generate(g, p->left);
+ w(g, "~Mreturn 1;~N~}");
+}
+
+static void generate_substring(struct generator * g, struct node * p) {
+
+ struct among * x = p->among;
+ int block = -1;
+ unsigned int bitmap = 0;
+ struct amongvec * among_cases = x->b;
+ int c;
+ int empty_case = -1;
+ int n_cases = 0;
+ symbol cases[2];
+ int shortest_size = INT_MAX;
+ int shown_comment = 0;
+
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->I[0] = x->number;
+ g->I[1] = x->literalstring_count;
+
+ /* In forward mode with non-ASCII UTF-8 characters, the first character
+ * of the string will often be the same, so instead look at the last
+ * common character position.
+ *
+ * In backward mode, we can't match if there are fewer characters before
+ * the current position than the minimum length.
+ */
+ for (c = 0; c < x->literalstring_count; ++c) {
+ int size = among_cases[c].size;
+ if (size != 0 && size < shortest_size) {
+ shortest_size = size;
+ }
+ }
+
+ for (c = 0; c < x->literalstring_count; ++c) {
+ symbol ch;
+ if (among_cases[c].size == 0) {
+ empty_case = c;
+ continue;
+ }
+ if (p->mode == m_forward) {
+ ch = among_cases[c].b[shortest_size - 1];
+ } else {
+ ch = among_cases[c].b[among_cases[c].size - 1];
+ }
+ if (n_cases == 0) {
+ block = ch >> 5;
+ } else if (ch >> 5 != block) {
+ block = -1;
+ if (n_cases > 2) break;
+ }
+ if (block == -1) {
+ if (n_cases > 0 && ch == cases[0]) continue;
+ if (n_cases < 2) {
+ cases[n_cases++] = ch;
+ } else if (ch != cases[1]) {
+ ++n_cases;
+ break;
+ }
+ } else {
+ if ((bitmap & (1u << (ch & 0x1f))) == 0) {
+ bitmap |= 1u << (ch & 0x1f);
+ if (n_cases < 2)
+ cases[n_cases] = ch;
+ ++n_cases;
+ }
+ }
+ }
+
+ if (block != -1 || n_cases <= 2) {
+ char buf[64];
+ g->I[2] = block;
+ g->I[3] = bitmap;
+ g->I[4] = shortest_size - 1;
+ if (p->mode == m_forward) {
+ sprintf(buf, "z->p[z->c + %d]", shortest_size - 1);
+ g->S[1] = buf;
+ if (shortest_size == 1) {
+ writef(g, "~Mif (z->c >= z->l", p);
+ } else {
+ writef(g, "~Mif (z->c + ~I4 >= z->l", p);
+ }
+ } else {
+ g->S[1] = "z->p[z->c - 1]";
+ if (shortest_size == 1) {
+ writef(g, "~Mif (z->c <= z->lb", p);
+ } else {
+ writef(g, "~Mif (z->c - ~I4 <= z->lb", p);
+ }
+ }
+ if (n_cases == 0) {
+ /* We get this for the degenerate case: among { '' }
+ * This doesn't seem to be a useful construct, but it is
+ * syntactically valid.
+ */
+ } else if (n_cases == 1) {
+ g->I[4] = cases[0];
+ writef(g, " || ~S1 != ~I4", p);
+ } else if (n_cases == 2) {
+ g->I[4] = cases[0];
+ g->I[5] = cases[1];
+ writef(g, " || (~S1 != ~I4 && ~S1 != ~I5)", p);
+ } else {
+ writef(g, " || ~S1 >> 5 != ~I2 || !((~I3 >> (~S1 & 0x1f)) & 1)", p);
+ }
+ write_string(g, ") ");
+ if (empty_case != -1) {
+ /* If the among includes the empty string, it can never fail
+ * so not matching the bitmap means we match the empty string.
+ */
+ g->I[4] = among_cases[empty_case].result;
+ writef(g, "among_var = ~I4; else~C", p);
+ } else {
+ writef(g, "~f~C", p);
+ }
+ shown_comment = 1;
+ } else {
+#ifdef OPTIMISATION_WARNINGS
+ printf("Couldn't shortcut among %d\n", x->number);
+#endif
+ }
+
+ if (!x->amongvar_needed) {
+ writef(g, "~Mif (!(find_among~S0(z, a_~I0, ~I1))) ~f", p);
+ writef(g, shown_comment ? "~N" : "~C", p);
+ } else {
+ writef(g, "~Mamong_var = find_among~S0(z, a_~I0, ~I1);", p);
+ writef(g, shown_comment ? "~N" : "~C", p);
+ writef(g, "~Mif (!(among_var)) ~f~N", p);
+ }
+}
+
+static void generate_among(struct generator * g, struct node * p) {
+
+ struct among * x = p->among;
+
+ if (x->substring == 0) generate_substring(g, p);
+
+ if (x->starter != 0) generate(g, x->starter);
+
+ if (x->command_count == 1 && x->nocommand_count == 0) {
+ /* Only one outcome ("no match" already handled). */
+ generate(g, x->commands[0]);
+ } else if (x->command_count > 0) {
+ int i;
+ writef(g, "~Mswitch (among_var) {~C~+", p);
+ for (i = 1; i <= x->command_count; i++) {
+ g->I[0] = i;
+ w(g, "~Mcase ~I0:~N~+");
+ generate(g, x->commands[i - 1]);
+ w(g, "~Mbreak;~N~-");
+ }
+ w(g, "~}");
+ }
+}
+
+static void generate_booltest(struct generator * g, struct node * p) {
+
+ g->V[0] = p->name;
+ writef(g, "~Mif (!(~V0)) ~f~C", p);
+}
+
+static void generate_false(struct generator * g, struct node * p) {
+
+ writef(g, "~M~f~C", p);
+}
+
+static void generate_debug(struct generator * g, struct node * p) {
+
+ g->I[0] = g->debug_count++;
+ g->I[1] = p->line_number;
+ writef(g, "~Mdebug(z, ~I0, ~I1);~C", p);
+
+}
+
+static void generate(struct generator * g, struct node * p) {
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ int a1 = g->failure_keep_count;
+
+ switch (p->type) {
+ case c_define: generate_define(g, p); break;
+ case c_bra: generate_bra(g, p); break;
+ case c_and: generate_and(g, p); break;
+ case c_or: generate_or(g, p); break;
+ case c_backwards: generate_backwards(g, p); break;
+ case c_not: generate_not(g, p); break;
+ case c_set: generate_set(g, p); break;
+ case c_unset: generate_unset(g, p); break;
+ case c_try: generate_try(g, p); break;
+ case c_fail: generate_fail(g, p); break;
+ case c_reverse:
+ case c_test: generate_test(g, p); break;
+ case c_do: generate_do(g, p); break;
+ case c_goto: generate_GO(g, p, 1); break;
+ case c_gopast: generate_GO(g, p, 0); break;
+ case c_repeat: generate_repeat(g, p); break;
+ case c_loop: generate_loop(g, p); break;
+ case c_atleast: generate_atleast(g, p); break;
+ case c_setmark: generate_setmark(g, p); break;
+ case c_tomark: generate_tomark(g, p); break;
+ case c_atmark: generate_atmark(g, p); break;
+ case c_hop: generate_hop(g, p); break;
+ case c_delete: generate_delete(g, p); break;
+ case c_next: generate_next(g, p); break;
+ case c_tolimit: generate_tolimit(g, p); break;
+ case c_atlimit: generate_atlimit(g, p); break;
+ case c_leftslice: generate_leftslice(g, p); break;
+ case c_rightslice: generate_rightslice(g, p); break;
+ case c_assignto: generate_assignto(g, p); break;
+ case c_sliceto: generate_sliceto(g, p); break;
+ case c_assign: generate_assignfrom(g, p); break;
+ case c_insert:
+ case c_attach: generate_insert(g, p, p->type); break;
+ case c_slicefrom: generate_slicefrom(g, p); break;
+ case c_setlimit: generate_setlimit(g, p); break;
+ case c_dollar: generate_dollar(g, p); break;
+ case c_mathassign: generate_integer_assign(g, p, "="); break;
+ case c_plusassign: generate_integer_assign(g, p, "+="); break;
+ case c_minusassign: generate_integer_assign(g, p, "-="); break;
+ case c_multiplyassign:generate_integer_assign(g, p, "*="); break;
+ case c_divideassign: generate_integer_assign(g, p, "/="); break;
+ case c_eq: generate_integer_test(g, p, "=="); break;
+ case c_ne: generate_integer_test(g, p, "!="); break;
+ case c_gr: generate_integer_test(g, p, ">"); break;
+ case c_ge: generate_integer_test(g, p, ">="); break;
+ case c_ls: generate_integer_test(g, p, "<"); break;
+ case c_le: generate_integer_test(g, p, "<="); break;
+ case c_call: generate_call(g, p); break;
+ case c_grouping: generate_grouping(g, p, false); break;
+ case c_non: generate_grouping(g, p, true); break;
+ case c_name: generate_namedstring(g, p); break;
+ case c_literalstring: generate_literalstring(g, p); break;
+ case c_among: generate_among(g, p); break;
+ case c_substring: generate_substring(g, p); break;
+ case c_booltest: generate_booltest(g, p); break;
+ case c_false: generate_false(g, p); break;
+ case c_true: break;
+ case c_debug: generate_debug(g, p); break;
+ default: fprintf(stderr, "%d encountered\n", p->type);
+ exit(1);
+ }
+
+ if (g->failure_label != a0)
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_keep_count = a1;
+}
+
+void write_generated_comment_content(struct generator * g) {
+ w(g, "Generated by Snowball " SNOWBALL_VERSION
+ " - https://snowballstem.org/");
+}
+
+void write_start_comment(struct generator * g,
+ const char * comment_start,
+ const char * comment_end) {
+ write_margin(g);
+ w(g, comment_start);
+ write_generated_comment_content(g);
+ if (comment_end) {
+ w(g, comment_end);
+ }
+ w(g, "~N~N");
+}
+
+static void generate_head(struct generator * g) {
+
+ w(g, "#include \"");
+ if (g->options->runtime_path) {
+ write_string(g, g->options->runtime_path);
+ if (g->options->runtime_path[strlen(g->options->runtime_path) - 1] != '/')
+ write_char(g, '/');
+ }
+ w(g, "header.h\"~N~N");
+}
+
+static void generate_routine_headers(struct generator * g) {
+ struct name * q;
+ for (q = g->analyser->names; q; q = q->next) {
+ g->V[0] = q;
+ switch (q->type) {
+ case t_routine:
+ w(g, "static int ~W0(struct SN_env * z);~N");
+ break;
+ case t_external:
+ w(g,
+ "#ifdef __cplusplus~N"
+ "extern \"C\" {~N"
+ "#endif~N"
+ "extern int ~W0(struct SN_env * z);~N"
+ "#ifdef __cplusplus~N"
+ "}~N"
+ "#endif~N"
+ );
+ break;
+ }
+ }
+}
+
+static void generate_among_table(struct generator * g, struct among * x) {
+
+ struct amongvec * v = x->b;
+
+ g->I[0] = x->number;
+ {
+ int i;
+ for (i = 0; i < x->literalstring_count; i++) {
+ g->I[1] = i;
+ g->I[2] = v->size;
+ g->L[0] = v->b;
+ if (v->size)
+ w(g, "static const symbol s_~I0_~I1[~I2] = ~A0;~N");
+ v++;
+ }
+ }
+
+ g->I[1] = x->literalstring_count;
+ w(g, "~N~Mstatic const struct among a_~I0[~I1] =~N{~N");
+
+ v = x->b;
+ {
+ int i;
+ for (i = 0; i < x->literalstring_count; i++) {
+ g->I[1] = i;
+ g->I[2] = v->size;
+ g->I[3] = v->i;
+ g->I[4] = v->result;
+ g->S[0] = i < x->literalstring_count - 1 ? "," : "";
+
+ if (g->options->comments) {
+ w(g, "/*~J1 */ ");
+ }
+ w(g, "{ ~I2, ");
+ if (v->size == 0) {
+ w(g, "0,");
+ } else {
+ w(g, "s_~I0_~I1,");
+ }
+ w(g, " ~I3, ~I4, ");
+ if (v->function == 0) {
+ write_char(g, '0');
+ } else {
+ write_varname(g, v->function);
+ }
+ w(g, "}~S0~N");
+ v++;
+ }
+ }
+ w(g, "};~N~N");
+}
+
+static void generate_amongs(struct generator * g) {
+ struct among * x;
+ for (x = g->analyser->amongs; x; x = x->next) {
+ generate_among_table(g, x);
+ }
+}
+
+static void set_bit(symbol * b, int i) { b[i/8] |= 1 << i%8; }
+
+static void generate_grouping_table(struct generator * g, struct grouping * q) {
+
+ int range = q->largest_ch - q->smallest_ch + 1;
+ int size = (range + 7)/ 8; /* assume 8 bits per symbol */
+ symbol * b = q->b;
+ symbol * map = create_b(size);
+ int i;
+ for (i = 0; i < size; i++) map[i] = 0;
+
+ for (i = 0; i < SIZE(b); i++) set_bit(map, b[i] - q->smallest_ch);
+
+ g->V[0] = q->name;
+
+ w(g, "static const unsigned char ~V0[] = { ");
+ for (i = 0; i < size; i++) {
+ write_int(g, map[i]);
+ if (i < size - 1) w(g, ", ");
+ }
+ w(g, " };~N~N");
+ lose_b(map);
+}
+
+static void generate_groupings(struct generator * g) {
+ struct grouping * q;
+ for (q = g->analyser->groupings; q; q = q->next) {
+ if (q->name->used)
+ generate_grouping_table(g, q);
+ }
+}
+
+static void generate_create(struct generator * g) {
+
+ int * p = g->analyser->name_count;
+ g->I[0] = p[t_string];
+ g->I[1] = p[t_integer] + p[t_boolean];
+ w(g, "~N"
+ "extern struct SN_env * ~pcreate_env(void) { return SN_create_env(~I0, ~I1); }"
+ "~N");
+}
+
+static void generate_close(struct generator * g) {
+
+ int * p = g->analyser->name_count;
+ g->I[0] = p[t_string];
+ w(g, "~Nextern void ~pclose_env(struct SN_env * z) { SN_close_env(z, ~I0); }~N~N");
+}
+
+static void generate_create_and_close_templates(struct generator * g) {
+ w(g, "~N"
+ "extern struct SN_env * ~pcreate_env(void);~N"
+ "extern void ~pclose_env(struct SN_env * z);~N"
+ "~N");
+}
+
+static void generate_header_file(struct generator * g) {
+
+ struct name * q;
+ const char * vp = g->options->variables_prefix;
+ g->S[0] = vp;
+
+ w(g, "#ifdef __cplusplus~N"
+ "extern \"C\" {~N"
+ "#endif~N"); /* for C++ */
+
+ generate_create_and_close_templates(g);
+ for (q = g->analyser->names; q; q = q->next) {
+ g->V[0] = q;
+ switch (q->type) {
+ case t_external:
+ w(g, "extern int ~W0(struct SN_env * z);~N");
+ break;
+ case t_string:
+ case t_integer:
+ case t_boolean:
+ if (vp) {
+ int count = q->count;
+ if (count < 0) {
+ /* Unused variables should get removed from `names`. */
+ fprintf(stderr, "Optimised out variable ");
+ report_b(stderr, q->b);
+ fprintf(stderr, " still in names list\n");
+ exit(1);
+ }
+ if (q->type == t_boolean) {
+ /* We use a single array for booleans and integers,
+ * with the integers first.
+ */
+ count += g->analyser->name_count[t_integer];
+ }
+ g->I[0] = count;
+ g->I[1] = "SIIrxg"[q->type];
+ w(g, "#define ~S0");
+ write_b(g, q->b);
+ w(g, " (~c1[~I0])~N");
+ }
+ break;
+ }
+ }
+
+ w(g, "~N"
+ "#ifdef __cplusplus~N"
+ "}~N"
+ "#endif~N"); /* for C++ */
+
+ w(g, "~N");
+}
+
+extern void generate_program_c(struct generator * g) {
+
+ g->outbuf = str_new();
+ write_start_comment(g, "/* ", " */");
+ generate_head(g);
+ generate_routine_headers(g);
+ w(g, "#ifdef __cplusplus~N"
+ "extern \"C\" {~N"
+ "#endif~N"
+ "~N");
+ generate_create_and_close_templates(g);
+ w(g, "~N"
+ "#ifdef __cplusplus~N"
+ "}~N"
+ "#endif~N");
+ generate_amongs(g);
+ generate_groupings(g);
+ g->declarations = g->outbuf;
+ g->outbuf = str_new();
+ g->literalstring_count = 0;
+ {
+ struct node * p = g->analyser->program;
+ while (p) { generate(g, p); p = p->right; }
+ }
+ generate_create(g);
+ generate_close(g);
+ output_str(g->options->output_src, g->declarations);
+ str_delete(g->declarations);
+ output_str(g->options->output_src, g->outbuf);
+ str_clear(g->outbuf);
+
+ write_start_comment(g, "/* ", " */");
+ generate_header_file(g);
+ output_str(g->options->output_h, g->outbuf);
+ str_delete(g->outbuf);
+}
+
+/* Generator functions common to multiple languages. */
+
+extern struct generator * create_generator(struct analyser * a, struct options * o) {
+ NEW(generator, g);
+ g->analyser = a;
+ g->options = o;
+ g->margin = 0;
+ g->debug_count = 0;
+ g->copy_from_count = 0;
+ g->line_count = 0;
+ g->line_labelled = 0;
+ g->failure_label = -1;
+ g->unreachable = false;
+#ifndef DISABLE_PYTHON
+ g->max_label = 0;
+#endif
+ return g;
+}
+
+extern void close_generator(struct generator * g) {
+ FREE(g);
+}
+
+/* Write routines for simple entities */
+
+extern void write_char(struct generator * g, int ch) {
+ str_append_ch(g->outbuf, ch); /* character */
+}
+
+extern void write_newline(struct generator * g) {
+ str_append_ch(g->outbuf, '\n'); /* newline */
+ g->line_count++;
+}
+
+extern void write_string(struct generator * g, const char * s) {
+ str_append_string(g->outbuf, s);
+}
+
+extern void write_int(struct generator * g, int i) {
+ str_append_int(g->outbuf, i);
+}
+
+extern void write_b(struct generator * g, symbol * b) {
+
+ str_append_b(g->outbuf, b);
+}
+
+extern void write_str(struct generator * g, struct str * str) {
+
+ str_append(g->outbuf, str);
+}
diff --git a/contrib/snowball/compiler/header.h b/contrib/snowball/compiler/header.h
new file mode 100644
index 0000000..dadbee3
--- /dev/null
+++ b/contrib/snowball/compiler/header.h
@@ -0,0 +1,411 @@
+#include <stdio.h>
+
+#define SNOWBALL_VERSION "2.0.0"
+
+typedef unsigned char byte;
+typedef unsigned short symbol;
+
+#define true 1
+#define false 0
+
+#define MALLOC check_malloc
+#define FREE check_free
+
+#define NEW(type, p) struct type * p = (struct type *) MALLOC(sizeof(struct type))
+#define NEWVEC(type, p, n) struct type * p = (struct type *) MALLOC(sizeof(struct type) * (n))
+
+#define SIZE(p) ((int *)(p))[-1]
+#define CAPACITY(p) ((int *)(p))[-2]
+
+extern symbol * create_b(int n);
+extern void report_b(FILE * out, const symbol * p);
+extern void lose_b(symbol * p);
+extern symbol * increase_capacity(symbol * p, int n);
+extern symbol * move_to_b(symbol * p, int n, const symbol * q);
+extern symbol * add_to_b(symbol * p, int n, const symbol * q);
+extern symbol * copy_b(const symbol * p);
+extern char * b_to_s(const symbol * p);
+extern symbol * add_s_to_b(symbol * p, const char * s);
+
+#define MOVE_TO_B(B, LIT) \
+ move_to_b(B, sizeof(LIT) / sizeof(LIT[0]), LIT)
+
+struct str; /* defined in space.c */
+
+extern struct str * str_new(void);
+extern void str_delete(struct str * str);
+extern void str_append(struct str * str, const struct str * add);
+extern void str_append_ch(struct str * str, char add);
+extern void str_append_b(struct str * str, const symbol * q);
+extern void str_append_b_tail(struct str * str, const symbol * q, int skip);
+extern void str_append_string(struct str * str, const char * s);
+extern void str_append_int(struct str * str, int i);
+extern void str_clear(struct str * str);
+extern void str_assign(struct str * str, const char * s);
+extern struct str * str_copy(const struct str * old);
+extern symbol * str_data(const struct str * str);
+extern int str_len(const struct str * str);
+extern int str_back(const struct str *str);
+extern int get_utf8(const symbol * p, int * slot);
+extern int put_utf8(int ch, symbol * p);
+extern void output_str(FILE * outfile, struct str * str);
+
+typedef enum { ENC_SINGLEBYTE, ENC_UTF8, ENC_WIDECHARS } enc;
+
+struct m_pair {
+
+ struct m_pair * next;
+ symbol * name;
+ symbol * value;
+
+};
+
+/* struct input must be a prefix of struct tokeniser. */
+struct input {
+
+ struct input * next;
+ symbol * p;
+ int c;
+ char * file;
+ int file_needs_freeing;
+ int line_number;
+
+};
+
+struct include {
+
+ struct include * next;
+ symbol * b;
+
+};
+
+enum token_codes {
+
+#include "syswords2.h"
+
+ c_mathassign,
+ c_name,
+ c_number,
+ c_literalstring,
+ c_neg,
+ c_call,
+ c_grouping,
+ c_booltest,
+
+ NUM_TOKEN_CODES
+};
+
+enum uplus_modes {
+ UPLUS_NONE,
+ UPLUS_DEFINED,
+ UPLUS_UNICODE
+};
+
+/* struct input must be a prefix of struct tokeniser. */
+struct tokeniser {
+
+ struct input * next;
+ symbol * p;
+ int c;
+ char * file;
+ int file_needs_freeing;
+ int line_number;
+ symbol * b;
+ symbol * b2;
+ int number;
+ int m_start;
+ int m_end;
+ struct m_pair * m_pairs;
+ int get_depth;
+ int error_count;
+ int token;
+ int previous_token;
+ byte token_held;
+ enc encoding;
+
+ int omission;
+ struct include * includes;
+
+ /* Mode in which U+ has been used:
+ * UPLUS_NONE - not used yet
+ * UPLUS_DEFINED - stringdef U+xxxx ....
+ * UPLUS_UNICODE - {U+xxxx} used with implicit meaning
+ */
+ int uplusmode;
+
+ char token_disabled[NUM_TOKEN_CODES];
+};
+
+extern symbol * get_input(const char * filename);
+extern struct tokeniser * create_tokeniser(symbol * b, char * file);
+extern int read_token(struct tokeniser * t);
+extern const char * name_of_token(int code);
+extern void disable_token(struct tokeniser * t, int code);
+extern void close_tokeniser(struct tokeniser * t);
+
+extern int space_count;
+extern void * check_malloc(int n);
+extern void check_free(void * p);
+
+struct node;
+
+struct name {
+
+ struct name * next;
+ symbol * b;
+ int type; /* t_string etc */
+ int mode; /* )_ for routines, externals */
+ struct node * definition; /* ) */
+ int count; /* 0, 1, 2 for each type */
+ struct grouping * grouping; /* for grouping names */
+ byte referenced;
+ byte used_in_among; /* Function used in among? */
+ byte value_used; /* (For variables) is its value ever used? */
+ byte initialised; /* (For variables) is it ever initialised? */
+ byte used_in_definition; /* (grouping) used in grouping definition? */
+ struct node * used; /* First use, or NULL if not used */
+ struct name * local_to; /* Local to one routine/external */
+ int declaration_line_number;/* Line number of declaration */
+
+};
+
+struct literalstring {
+
+ struct literalstring * next;
+ symbol * b;
+
+};
+
+struct amongvec {
+
+ symbol * b; /* the string giving the case */
+ int size; /* - and its size */
+ struct node * action; /* the corresponding action */
+ int i; /* the amongvec index of the longest substring of b */
+ int result; /* the numeric result for the case */
+ int line_number; /* for diagnostics and stable sorting */
+ struct name * function;
+
+};
+
+struct among {
+
+ struct among * next;
+ struct amongvec * b; /* pointer to the amongvec */
+ int number; /* amongs are numbered 0, 1, 2 ... */
+ int literalstring_count; /* in this among */
+ int command_count; /* in this among */
+ int nocommand_count; /* number of "no command" entries in this among */
+ int function_count; /* in this among */
+ int amongvar_needed; /* do we need to set among_var? */
+ struct node * starter; /* i.e. among( (starter) 'string' ... ) */
+ struct node * substring; /* i.e. substring ... among ( ... ) */
+ struct node ** commands; /* array with command_count entries */
+};
+
+struct grouping {
+
+ struct grouping * next;
+ symbol * b; /* the characters of this group */
+ int largest_ch; /* character with max code */
+ int smallest_ch; /* character with min code */
+ struct name * name; /* so g->name->grouping == g */
+ int line_number;
+};
+
+struct node {
+
+ struct node * next;
+ struct node * left;
+ struct node * aux; /* used in setlimit */
+ struct among * among; /* used in among */
+ struct node * right;
+ int type;
+ int mode;
+ struct node * AE;
+ struct name * name;
+ symbol * literalstring;
+ int number;
+ int line_number;
+ int amongvar_needed; /* used in routine definitions */
+};
+
+enum name_types {
+
+ t_size = 6,
+
+ t_string = 0, t_boolean = 1, t_integer = 2, t_routine = 3, t_external = 4,
+ t_grouping = 5
+
+/* If this list is extended, adjust wvn in generator.c */
+};
+
+/* In name_count[i] below, remember that
+ type is
+ ----+----
+ 0 | string
+ 1 | boolean
+ 2 | integer
+ 3 | routine
+ 4 | external
+ 5 | grouping
+*/
+
+struct analyser {
+
+ struct tokeniser * tokeniser;
+ struct node * nodes;
+ struct name * names;
+ struct literalstring * literalstrings;
+ int mode;
+ byte modifyable; /* false inside reverse(...) */
+ struct node * program;
+ struct node * program_end;
+ int name_count[t_size]; /* name_count[i] counts the number of names of type i */
+ struct among * amongs;
+ struct among * amongs_end;
+ int among_count;
+ int amongvar_needed; /* used in reading routine definitions */
+ struct grouping * groupings;
+ struct grouping * groupings_end;
+ struct node * substring; /* pending 'substring' in current routine definition */
+ enc encoding;
+ byte int_limits_used; /* are maxint or minint used? */
+};
+
+enum analyser_modes {
+
+ m_forward = 0, m_backward /*, m_integer */
+
+};
+
+extern void print_program(struct analyser * a);
+extern struct analyser * create_analyser(struct tokeniser * t);
+extern void close_analyser(struct analyser * a);
+
+extern void read_program(struct analyser * a);
+
+struct generator {
+
+ struct analyser * analyser;
+ struct options * options;
+ int unreachable; /* 0 if code can be reached, 1 if current code
+ * is unreachable. */
+ int var_number; /* Number of next variable to use. */
+ struct str * outbuf; /* temporary str to store output */
+ struct str * declarations; /* str storing variable declarations */
+ int next_label;
+#ifndef DISABLE_PYTHON
+ int max_label;
+#endif
+ int margin;
+
+ /* if > 0, keep_count to restore in case of a failure;
+ * if < 0, the negated keep_count for the limit to restore in case of
+ * failure. */
+ int failure_keep_count;
+#if !defined(DISABLE_JAVA) && !defined(DISABLE_JS) && !defined(DISABLE_PYTHON) && !defined(DISABLE_CSHARP)
+ struct str * failure_str; /* This is used by some generators instead of failure_keep_count */
+#endif
+
+ int label_used; /* Keep track of whether the failure label is used. */
+ int failure_label;
+ int debug_count;
+ int copy_from_count; /* count of calls to copy_from() */
+
+ const char * S[10]; /* strings */
+ symbol * B[10]; /* blocks */
+ int I[10]; /* integers */
+ struct name * V[5]; /* variables */
+ symbol * L[5]; /* literals, used in formatted write */
+
+ int line_count; /* counts number of lines output */
+ int line_labelled; /* in ISO C, will need extra ';' if it is a block end */
+ int literalstring_count;
+ int keep_count; /* used to number keep/restore pairs to avoid compiler warnings
+ about shadowed variables */
+};
+
+/* Special values for failure_label in struct generator. */
+enum special_labels {
+ x_return = -1
+};
+
+struct options {
+
+ /* for the command line: */
+
+ const char * output_file;
+ const char * name;
+ FILE * output_src;
+ FILE * output_h;
+ byte syntax_tree;
+ byte comments;
+ enc encoding;
+ enum { LANG_JAVA, LANG_C, LANG_CPLUSPLUS, LANG_CSHARP, LANG_PASCAL, LANG_PYTHON, LANG_JAVASCRIPT, LANG_RUST, LANG_GO } make_lang;
+ const char * externals_prefix;
+ const char * variables_prefix;
+ const char * runtime_path;
+ const char * parent_class_name;
+ const char * package;
+ const char * go_snowball_runtime;
+ const char * string_class;
+ const char * among_class;
+ struct include * includes;
+ struct include * includes_end;
+};
+
+/* Generator functions common to several backends. */
+
+extern struct generator * create_generator(struct analyser * a, struct options * o);
+extern void close_generator(struct generator * g);
+
+extern void write_char(struct generator * g, int ch);
+extern void write_newline(struct generator * g);
+extern void write_string(struct generator * g, const char * s);
+extern void write_int(struct generator * g, int i);
+extern void write_b(struct generator * g, symbol * b);
+extern void write_str(struct generator * g, struct str * str);
+
+extern void write_comment_content(struct generator * g, struct node * p);
+extern void write_generated_comment_content(struct generator * g);
+extern void write_start_comment(struct generator * g,
+ const char * comment_start,
+ const char * comment_end);
+
+extern int K_needed(struct generator * g, struct node * p);
+extern int repeat_restore(struct generator * g, struct node * p);
+
+/* Generator for C code. */
+extern void generate_program_c(struct generator * g);
+
+#ifndef DISABLE_JAVA
+/* Generator for Java code. */
+extern void generate_program_java(struct generator * g);
+#endif
+
+#ifndef DISABLE_CSHARP
+/* Generator for C# code. */
+extern void generate_program_csharp(struct generator * g);
+#endif
+
+#ifndef DISABLE_PASCAL
+extern void generate_program_pascal(struct generator * g);
+#endif
+
+#ifndef DISABLE_PYTHON
+/* Generator for Python code. */
+extern void generate_program_python(struct generator * g);
+#endif
+
+#ifndef DISABLE_JS
+extern void generate_program_js(struct generator * g);
+#endif
+
+#ifndef DISABLE_RUST
+extern void generate_program_rust(struct generator * g);
+#endif
+
+#ifndef DISABLE_GO
+extern void generate_program_go(struct generator * g);
+#endif
diff --git a/contrib/snowball/compiler/space.c b/contrib/snowball/compiler/space.c
new file mode 100644
index 0000000..5b05876
--- /dev/null
+++ b/contrib/snowball/compiler/space.c
@@ -0,0 +1,287 @@
+
+#include <stdio.h> /* for printf */
+#include <stdlib.h> /* malloc, free */
+#include <string.h> /* memmove */
+
+#include "header.h"
+
+#define HEAD 2*sizeof(int)
+#define EXTENDER 40
+
+
+/* This modules provides a simple mechanism for arbitrary length writable
+ strings, called 'blocks'. They are 'symbol *' items rather than 'char *'
+ items however.
+
+ The calls are:
+
+ symbol * b = create_b(n);
+ - create an empty block b with room for n symbols
+ b = increase_capacity(b, n);
+ - increase the capacity of block b by n symbols (b may change)
+ b2 = copy_b(b)
+ - copy block b into b2
+ lose_b(b);
+ - lose block b
+ b = move_to_b(b, n, p);
+ - set the data in b to be the n symbols at address p
+ b = add_to_b(b, n, p);
+ - add the n symbols at address p to the end of the data in b
+ SIZE(b)
+ - is the number of symbols in b
+ For example:
+
+ symbol * b = create_b(0);
+ { int i;
+ char p[10];
+ for (i = 0; i < 100; i++) {
+ sprintf(p, " %d", i);
+ add_s_to_b(b, p);
+ }
+ }
+
+ and b contains " 0 1 2 ... 99" spaced out as symbols.
+*/
+
+/* For a block b, SIZE(b) is the number of symbols so far written into it,
+ CAPACITY(b) the total number it can contain, so SIZE(b) <= CAPACITY(b).
+ In fact blocks have 1 extra character over the promised capacity so
+ they can be zero terminated by 'b[SIZE(b)] = 0;' without fear of
+ overwriting.
+*/
+
+extern symbol * create_b(int n) {
+ symbol * p = (symbol *) (HEAD + (char *) MALLOC(HEAD + (n + 1) * sizeof(symbol)));
+ CAPACITY(p) = n;
+ SIZE(p) = 0;
+ return p;
+}
+
+extern void report_b(FILE * out, const symbol * p) {
+ int i;
+ for (i = 0; i < SIZE(p); i++) {
+ if (p[i] > 255) {
+ printf("In report_b, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]);
+ exit(1);
+ }
+ putc(p[i], out);
+ }
+}
+
+extern void output_str(FILE * outfile, struct str * str) {
+ report_b(outfile, str_data(str));
+}
+
+extern void lose_b(symbol * p) {
+ if (p == 0) return;
+ FREE((char *) p - HEAD);
+}
+
+extern symbol * increase_capacity(symbol * p, int n) {
+ symbol * q = create_b(CAPACITY(p) + n + EXTENDER);
+ memmove(q, p, CAPACITY(p) * sizeof(symbol));
+ SIZE(q) = SIZE(p);
+ lose_b(p); return q;
+}
+
+extern symbol * move_to_b(symbol * p, int n, const symbol * q) {
+ int x = n - CAPACITY(p);
+ if (x > 0) p = increase_capacity(p, x);
+ memmove(p, q, n * sizeof(symbol)); SIZE(p) = n; return p;
+}
+
+extern symbol * add_to_b(symbol * p, int n, const symbol * q) {
+ int x = SIZE(p) + n - CAPACITY(p);
+ if (x > 0) p = increase_capacity(p, x);
+ memmove(p + SIZE(p), q, n * sizeof(symbol)); SIZE(p) += n; return p;
+}
+
+extern symbol * copy_b(const symbol * p) {
+ int n = SIZE(p);
+ symbol * q = create_b(n);
+ move_to_b(q, n, p);
+ return q;
+}
+
+int space_count = 0;
+
+extern void * check_malloc(int n) {
+ space_count++;
+ return malloc(n);
+}
+
+extern void check_free(void * p) {
+ space_count--;
+ free(p);
+}
+
+/* To convert a block to a zero terminated string: */
+
+extern char * b_to_s(const symbol * p) {
+ int n = SIZE(p);
+ char * s = (char *)malloc(n + 1);
+ {
+ int i;
+ for (i = 0; i < n; i++) {
+ if (p[i] > 255) {
+ printf("In b_to_s, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]);
+ exit(1);
+ }
+ s[i] = (char)p[i];
+ }
+ }
+ s[n] = 0;
+ return s;
+}
+
+/* To add a zero terminated string to a block. If p = 0 the
+ block is created. */
+
+extern symbol * add_s_to_b(symbol * p, const char * s) {
+ int n = strlen(s);
+ int k;
+ if (p == 0) p = create_b(n);
+ k = SIZE(p);
+ {
+ int x = k + n - CAPACITY(p);
+ if (x > 0) p = increase_capacity(p, x);
+ }
+ {
+ int i;
+ for (i = 0; i < n; i++) p[i + k] = s[i];
+ }
+ SIZE(p) += n;
+ return p;
+}
+
+/* The next section defines string handling capabilities in terms
+ of the lower level block handling capabilities of space.c */
+/* -------------------------------------------------------------*/
+
+struct str {
+ symbol * data;
+};
+
+/* Create a new string. */
+extern struct str * str_new(void) {
+
+ struct str * output = (struct str *) malloc(sizeof(struct str));
+ output->data = create_b(0);
+ return output;
+}
+
+/* Delete a string. */
+extern void str_delete(struct str * str) {
+
+ lose_b(str->data);
+ free(str);
+}
+
+/* Append a str to this str. */
+extern void str_append(struct str * str, const struct str * add) {
+
+ symbol * q = add->data;
+ str->data = add_to_b(str->data, SIZE(q), q);
+}
+
+/* Append a character to this str. */
+extern void str_append_ch(struct str * str, char add) {
+
+ symbol q[1];
+ q[0] = add;
+ str->data = add_to_b(str->data, 1, q);
+}
+
+/* Append a low level block to a str. */
+extern void str_append_b(struct str * str, const symbol * q) {
+
+ str->data = add_to_b(str->data, SIZE(q), q);
+}
+
+/* Append the tail of a low level block to a str. */
+extern void str_append_b_tail(struct str * str, const symbol * q, int skip) {
+ if (skip < 0 || skip >= SIZE(q)) return;
+
+ str->data = add_to_b(str->data, SIZE(q) - skip, q + skip);
+}
+
+/* Append a (char *, null terminated) string to a str. */
+extern void str_append_string(struct str * str, const char * s) {
+
+ str->data = add_s_to_b(str->data, s);
+}
+
+/* Append an integer to a str. */
+extern void str_append_int(struct str * str, int i) {
+
+ char s[30];
+ sprintf(s, "%d", i);
+ str_append_string(str, s);
+}
+
+/* Clear a string */
+extern void str_clear(struct str * str) {
+
+ SIZE(str->data) = 0;
+}
+
+/* Set a string */
+extern void str_assign(struct str * str, const char * s) {
+
+ str_clear(str);
+ str_append_string(str, s);
+}
+
+/* Copy a string. */
+extern struct str * str_copy(const struct str * old) {
+
+ struct str * newstr = str_new();
+ str_append(newstr, old);
+ return newstr;
+}
+
+/* Get the data stored in this str. */
+extern symbol * str_data(const struct str * str) {
+
+ return str->data;
+}
+
+/* Get the length of the str. */
+extern int str_len(const struct str * str) {
+
+ return SIZE(str->data);
+}
+
+/* Get the last character of the str.
+ *
+ * Or -1 if the string is empty.
+ */
+extern int str_back(const struct str *str) {
+ return SIZE(str->data) ? str->data[SIZE(str->data) - 1] : -1;
+}
+
+extern int get_utf8(const symbol * p, int * slot) {
+ int b0, b1;
+ b0 = *p++;
+ if (b0 < 0xC0) { /* 1100 0000 */
+ * slot = b0; return 1;
+ }
+ b1 = *p++;
+ if (b0 < 0xE0) { /* 1110 0000 */
+ * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2;
+ }
+ * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (*p & 0x3F); return 3;
+}
+
+extern int put_utf8(int ch, symbol * p) {
+ if (ch < 0x80) {
+ p[0] = ch; return 1;
+ }
+ if (ch < 0x800) {
+ p[0] = (ch >> 6) | 0xC0;
+ p[1] = (ch & 0x3F) | 0x80; return 2;
+ }
+ p[0] = (ch >> 12) | 0xE0;
+ p[1] = ((ch >> 6) & 0x3F) | 0x80;
+ p[2] = (ch & 0x3F) | 0x80; return 3;
+}
diff --git a/contrib/snowball/compiler/syswords.h b/contrib/snowball/compiler/syswords.h
new file mode 100644
index 0000000..c401d3e
--- /dev/null
+++ b/contrib/snowball/compiler/syswords.h
@@ -0,0 +1,86 @@
+static const struct system_word vocab[82+1] = {
+ { 0, (const byte *)"", 82+1},
+
+ { 1, (const byte *)"$", c_dollar },
+ { 1, (const byte *)"(", c_bra },
+ { 1, (const byte *)")", c_ket },
+ { 1, (const byte *)"*", c_multiply },
+ { 1, (const byte *)"+", c_plus },
+ { 1, (const byte *)"-", c_minus },
+ { 1, (const byte *)"/", c_divide },
+ { 1, (const byte *)"<", c_ls },
+ { 1, (const byte *)"=", c_assign },
+ { 1, (const byte *)">", c_gr },
+ { 1, (const byte *)"?", c_debug },
+ { 1, (const byte *)"[", c_leftslice },
+ { 1, (const byte *)"]", c_rightslice },
+ { 2, (const byte *)"!=", c_ne },
+ { 2, (const byte *)"*=", c_multiplyassign },
+ { 2, (const byte *)"+=", c_plusassign },
+ { 2, (const byte *)"-=", c_minusassign },
+ { 2, (const byte *)"->", c_sliceto },
+ { 2, (const byte *)"/*", c_comment2 },
+ { 2, (const byte *)"//", c_comment1 },
+ { 2, (const byte *)"/=", c_divideassign },
+ { 2, (const byte *)"<+", c_insert },
+ { 2, (const byte *)"<-", c_slicefrom },
+ { 2, (const byte *)"<=", c_le },
+ { 2, (const byte *)"==", c_eq },
+ { 2, (const byte *)"=>", c_assignto },
+ { 2, (const byte *)">=", c_ge },
+ { 2, (const byte *)"as", c_as },
+ { 2, (const byte *)"do", c_do },
+ { 2, (const byte *)"or", c_or },
+ { 3, (const byte *)"and", c_and },
+ { 3, (const byte *)"for", c_for },
+ { 3, (const byte *)"get", c_get },
+ { 3, (const byte *)"hex", c_hex },
+ { 3, (const byte *)"hop", c_hop },
+ { 3, (const byte *)"len", c_len },
+ { 3, (const byte *)"non", c_non },
+ { 3, (const byte *)"not", c_not },
+ { 3, (const byte *)"set", c_set },
+ { 3, (const byte *)"try", c_try },
+ { 4, (const byte *)"fail", c_fail },
+ { 4, (const byte *)"goto", c_goto },
+ { 4, (const byte *)"loop", c_loop },
+ { 4, (const byte *)"next", c_next },
+ { 4, (const byte *)"size", c_size },
+ { 4, (const byte *)"test", c_test },
+ { 4, (const byte *)"true", c_true },
+ { 5, (const byte *)"among", c_among },
+ { 5, (const byte *)"false", c_false },
+ { 5, (const byte *)"lenof", c_lenof },
+ { 5, (const byte *)"limit", c_limit },
+ { 5, (const byte *)"unset", c_unset },
+ { 6, (const byte *)"atmark", c_atmark },
+ { 6, (const byte *)"attach", c_attach },
+ { 6, (const byte *)"cursor", c_cursor },
+ { 6, (const byte *)"define", c_define },
+ { 6, (const byte *)"delete", c_delete },
+ { 6, (const byte *)"gopast", c_gopast },
+ { 6, (const byte *)"insert", c_insert },
+ { 6, (const byte *)"maxint", c_maxint },
+ { 6, (const byte *)"minint", c_minint },
+ { 6, (const byte *)"repeat", c_repeat },
+ { 6, (const byte *)"sizeof", c_sizeof },
+ { 6, (const byte *)"tomark", c_tomark },
+ { 7, (const byte *)"atleast", c_atleast },
+ { 7, (const byte *)"atlimit", c_atlimit },
+ { 7, (const byte *)"decimal", c_decimal },
+ { 7, (const byte *)"reverse", c_reverse },
+ { 7, (const byte *)"setmark", c_setmark },
+ { 7, (const byte *)"strings", c_strings },
+ { 7, (const byte *)"tolimit", c_tolimit },
+ { 8, (const byte *)"booleans", c_booleans },
+ { 8, (const byte *)"integers", c_integers },
+ { 8, (const byte *)"routines", c_routines },
+ { 8, (const byte *)"setlimit", c_setlimit },
+ { 9, (const byte *)"backwards", c_backwards },
+ { 9, (const byte *)"externals", c_externals },
+ { 9, (const byte *)"groupings", c_groupings },
+ { 9, (const byte *)"stringdef", c_stringdef },
+ { 9, (const byte *)"substring", c_substring },
+ { 12, (const byte *)"backwardmode", c_backwardmode },
+ { 13, (const byte *)"stringescapes", c_stringescapes }
+};
diff --git a/contrib/snowball/compiler/syswords2.h b/contrib/snowball/compiler/syswords2.h
new file mode 100644
index 0000000..e853e32
--- /dev/null
+++ b/contrib/snowball/compiler/syswords2.h
@@ -0,0 +1,13 @@
+ c_among = 4, c_and, c_as, c_assign, c_assignto, c_atleast,
+ c_atlimit, c_atmark, c_attach, c_backwardmode, c_backwards,
+ c_booleans, c_bra, c_comment1, c_comment2, c_cursor, c_debug,
+ c_decimal, c_define, c_delete, c_divide, c_divideassign, c_do,
+ c_dollar, c_eq, c_externals, c_fail, c_false, c_for, c_ge, c_get,
+ c_gopast, c_goto, c_gr, c_groupings, c_hex, c_hop, c_insert,
+ c_integers, c_ket, c_le, c_leftslice, c_len, c_lenof, c_limit, c_loop,
+ c_ls, c_maxint, c_minint, c_minus, c_minusassign, c_multiply,
+ c_multiplyassign, c_ne, c_next, c_non, c_not, c_or, c_plus,
+ c_plusassign, c_repeat, c_reverse, c_rightslice, c_routines,
+ c_set, c_setlimit, c_setmark, c_size, c_sizeof, c_slicefrom,
+ c_sliceto, c_stringdef, c_stringescapes, c_strings, c_substring,
+ c_test, c_tolimit, c_tomark, c_true, c_try, c_unset,
diff --git a/contrib/snowball/compiler/tokeniser.c b/contrib/snowball/compiler/tokeniser.c
new file mode 100644
index 0000000..e6c6386
--- /dev/null
+++ b/contrib/snowball/compiler/tokeniser.c
@@ -0,0 +1,567 @@
+
+#include <stdio.h> /* stderr etc */
+#include <stdlib.h> /* malloc free */
+#include <string.h> /* strlen */
+#include <ctype.h> /* isalpha etc */
+#include "header.h"
+
+struct system_word {
+ int s_size; /* size of system word */
+ const byte * s; /* pointer to the system word */
+ int code; /* its internal code */
+};
+
+
+/* ASCII collating assumed in syswords.c */
+
+#include "syswords.h"
+
+#define INITIAL_INPUT_BUFFER_SIZE 8192
+
+static int hex_to_num(int ch);
+
+static int smaller(int a, int b) { return a < b ? a : b; }
+
+extern symbol * get_input(const char * filename) {
+ FILE * input = fopen(filename, "r");
+ if (input == 0) { return 0; }
+ {
+ symbol * u = create_b(INITIAL_INPUT_BUFFER_SIZE);
+ int size = 0;
+ while (true) {
+ int ch = getc(input);
+ if (ch == EOF) break;
+ if (size >= CAPACITY(u)) u = increase_capacity(u, size);
+ u[size++] = ch;
+ }
+ fclose(input);
+ SIZE(u) = size;
+ return u;
+ }
+}
+
+static void error(struct tokeniser * t, const char * s1, int n, symbol * p, const char * s2) {
+ if (t->error_count == 20) { fprintf(stderr, "... etc\n"); exit(1); }
+ fprintf(stderr, "%s:%d: ", t->file, t->line_number);
+ if (s1) fprintf(stderr, "%s", s1);
+ if (p) {
+ int i;
+ for (i = 0; i < n; i++) fprintf(stderr, "%c", p[i]);
+ }
+ if (s2) fprintf(stderr, "%s", s2);
+ fprintf(stderr, "\n");
+ t->error_count++;
+}
+
+static void error1(struct tokeniser * t, const char * s) {
+ error(t, s, 0,0, 0);
+}
+
+static void error2(struct tokeniser * t, const char * s) {
+ error(t, "unexpected end of text after ", 0,0, s);
+}
+
+static int compare_words(int m, symbol * p, int n, const byte * q) {
+ if (m != n) return m - n;
+ {
+ int i; for (i = 0; i < n; i++) {
+ int diff = p[i] - q[i];
+ if (diff) return diff;
+ }
+ }
+ return 0;
+}
+
+static int find_word(int n, symbol * p) {
+ int i = 0; int j = vocab->code;
+ do {
+ int k = i + (j - i)/2;
+ const struct system_word * w = vocab + k;
+ int diff = compare_words(n, p, w->s_size, w->s);
+ if (diff == 0) return w->code;
+ if (diff < 0) j = k; else i = k;
+ } while (j - i != 1);
+ return -1;
+}
+
+static int get_number(int n, symbol * p) {
+ int x = 0;
+ int i; for (i = 0; i < n; i++) x = 10*x + p[i] - '0';
+ return x;
+}
+
+static int eq_s(struct tokeniser * t, const char * s) {
+ int l = strlen(s);
+ if (SIZE(t->p) - t->c < l) return false;
+ {
+ int i;
+ for (i = 0; i < l; i++) if (t->p[t->c + i] != s[i]) return false;
+ }
+ t->c += l; return true;
+}
+
+static int white_space(struct tokeniser * t, int ch) {
+ switch (ch) {
+ case '\n':
+ t->line_number++;
+ /* fall through */
+ case '\r':
+ case '\t':
+ case ' ':
+ return true;
+ }
+ return false;
+}
+
+static symbol * find_in_m(struct tokeniser * t, int n, symbol * p) {
+ struct m_pair * q;
+ for (q = t->m_pairs; q; q = q->next) {
+ symbol * name = q->name;
+ if (n == SIZE(name) && memcmp(name, p, n * sizeof(symbol)) == 0) return q->value;
+ }
+ return 0;
+}
+
+static int read_literal_string(struct tokeniser * t, int c) {
+ symbol * p = t->p;
+ int ch;
+ SIZE(t->b) = 0;
+ while (true) {
+ if (c >= SIZE(p)) { error2(t, "'"); return c; }
+ ch = p[c];
+ if (ch == '\n') { error1(t, "string not terminated"); return c; }
+ c++;
+ if (ch == t->m_start) {
+ /* Inside insert characters. */
+ int c0 = c;
+ int newlines = false; /* no newlines as yet */
+ int black_found = false; /* no printing chars as yet */
+ while (true) {
+ if (c >= SIZE(p)) { error2(t, "'"); return c; }
+ ch = p[c]; c++;
+ if (ch == t->m_end) break;
+ if (!white_space(t, ch)) black_found = true;
+ if (ch == '\n') newlines = true;
+ if (newlines && black_found) {
+ error1(t, "string not terminated");
+ return c;
+ }
+ }
+ if (!newlines) {
+ int n = c - c0 - 1; /* macro size */
+ int firstch = p[c0];
+ symbol * q = find_in_m(t, n, p + c0);
+ if (q == 0) {
+ if (n == 1 && (firstch == '\'' || firstch == t->m_start))
+ t->b = add_to_b(t->b, 1, p + c0);
+ else if (n >= 3 && firstch == 'U' && p[c0 + 1] == '+') {
+ int codepoint = 0;
+ int x;
+ if (t->uplusmode == UPLUS_DEFINED) {
+ /* See if found with xxxx upper-cased. */
+ symbol * uc = create_b(n);
+ int i;
+ for (i = 0; i != n; ++i) {
+ uc[i] = toupper(p[c0 + i]);
+ }
+ q = find_in_m(t, n, uc);
+ lose_b(uc);
+ if (q != 0) {
+ t->b = add_to_b(t->b, SIZE(q), q);
+ continue;
+ }
+ error1(t, "Some U+xxxx stringdefs seen but not this one");
+ } else {
+ t->uplusmode = UPLUS_UNICODE;
+ }
+ for (x = c0 + 2; x != c - 1; ++x) {
+ int hex = hex_to_num(p[x]);
+ if (hex < 0) {
+ error1(t, "Bad hex digit following U+");
+ break;
+ }
+ codepoint = (codepoint << 4) | hex;
+ }
+ if (t->encoding == ENC_UTF8) {
+ if (codepoint < 0 || codepoint > 0x01ffff) {
+ error1(t, "character values exceed 0x01ffff");
+ }
+ /* Ensure there's enough space for a max length
+ * UTF-8 sequence. */
+ if (CAPACITY(t->b) < SIZE(t->b) + 3) {
+ t->b = increase_capacity(t->b, 3);
+ }
+ SIZE(t->b) += put_utf8(codepoint, t->b + SIZE(t->b));
+ } else {
+ symbol sym;
+ if (t->encoding == ENC_SINGLEBYTE) {
+ /* Only ISO-8859-1 is handled this way - for
+ * other single-byte character sets you need
+ * stringdef all the U+xxxx codes you use
+ * like - e.g.:
+ *
+ * stringdef U+0171 hex 'FB'
+ */
+ if (codepoint < 0 || codepoint > 0xff) {
+ error1(t, "character values exceed 256");
+ }
+ } else {
+ if (codepoint < 0 || codepoint > 0xffff) {
+ error1(t, "character values exceed 64K");
+ }
+ }
+ sym = codepoint;
+ t->b = add_to_b(t->b, 1, &sym);
+ }
+ } else
+ error(t, "string macro '", n, p + c0, "' undeclared");
+ } else
+ t->b = add_to_b(t->b, SIZE(q), q);
+ }
+ } else {
+ if (ch == '\'') return c;
+ if (ch < 0 || ch >= 0x80) {
+ if (t->encoding != ENC_WIDECHARS) {
+ /* We don't really want people using non-ASCII literal
+ * strings, but historically it's worked for single-byte
+ * and UTF-8 if the source encoding matches what the
+ * generated stemmer works in and it seems unfair to just
+ * suddenly make this a hard error.`
+ */
+ fprintf(stderr,
+ "%s:%d: warning: Non-ASCII literal strings aren't "
+ "portable - use stringdef instead\n",
+ t->file, t->line_number);
+ } else {
+ error1(t, "Non-ASCII literal strings aren't "
+ "portable - use stringdef instead");
+ }
+ }
+ t->b = add_to_b(t->b, 1, p + c - 1);
+ }
+ }
+}
+
+static int next_token(struct tokeniser * t) {
+ symbol * p = t->p;
+ int c = t->c;
+ int ch;
+ int code = -1;
+ while (true) {
+ if (c >= SIZE(p)) { t->c = c; return -1; }
+ ch = p[c];
+ if (white_space(t, ch)) { c++; continue; }
+ if (isalpha(ch)) {
+ int c0 = c;
+ while (c < SIZE(p) && (isalnum(p[c]) || p[c] == '_')) c++;
+ code = find_word(c - c0, p + c0);
+ if (code < 0 || t->token_disabled[code]) {
+ t->b = move_to_b(t->b, c - c0, p + c0);
+ code = c_name;
+ }
+ } else
+ if (isdigit(ch)) {
+ int c0 = c;
+ while (c < SIZE(p) && isdigit(p[c])) c++;
+ t->number = get_number(c - c0, p + c0);
+ code = c_number;
+ } else
+ if (ch == '\'') {
+ c = read_literal_string(t, c + 1);
+ code = c_literalstring;
+ } else
+ {
+ int lim = smaller(2, SIZE(p) - c);
+ int i;
+ for (i = lim; i > 0; i--) {
+ code = find_word(i, p + c);
+ if (code >= 0) { c += i; break; }
+ }
+ }
+ if (code >= 0) {
+ t->c = c;
+ return code;
+ }
+ error(t, "'", 1, p + c, "' unknown");
+ c++;
+ continue;
+ }
+}
+
+static int next_char(struct tokeniser * t) {
+ if (t->c >= SIZE(t->p)) return -1;
+ return t->p[t->c++];
+}
+
+static int next_real_char(struct tokeniser * t) {
+ while (true) {
+ int ch = next_char(t);
+ if (!white_space(t, ch)) return ch;
+ }
+}
+
+static void read_chars(struct tokeniser * t) {
+ int ch = next_real_char(t);
+ if (ch < 0) { error2(t, "stringdef"); return; }
+ {
+ int c0 = t->c-1;
+ while (true) {
+ ch = next_char(t);
+ if (white_space(t, ch) || ch < 0) break;
+ }
+ t->b2 = move_to_b(t->b2, t->c - c0 - 1, t->p + c0);
+ }
+}
+
+static int decimal_to_num(int ch) {
+ if ('0' <= ch && ch <= '9') return ch - '0';
+ return -1;
+}
+
+static int hex_to_num(int ch) {
+ if ('0' <= ch && ch <= '9') return ch - '0';
+ if ('a' <= ch && ch <= 'f') return ch - 'a' + 10;
+ if ('A' <= ch && ch <= 'F') return ch - 'A' + 10;
+ return -1;
+}
+
+static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) {
+ int c = 0; int d = 0;
+ while (true) {
+ while (c < SIZE(p) && p[c] == ' ') c++;
+ if (c == SIZE(p)) break;
+ {
+ int number = 0;
+ while (c != SIZE(p)) {
+ int ch = p[c];
+ if (ch == ' ') break;
+ if (base == 10) {
+ ch = decimal_to_num(ch);
+ if (ch < 0) {
+ error1(t, "decimal string contains non-digits");
+ return;
+ }
+ } else {
+ ch = hex_to_num(ch);
+ if (ch < 0) {
+ error1(t, "hex string contains non-hex characters");
+ return;
+ }
+ }
+ number = base * number + ch;
+ c++;
+ }
+ if (t->encoding == ENC_SINGLEBYTE) {
+ if (number < 0 || number > 0xff) {
+ error1(t, "character values exceed 256");
+ return;
+ }
+ } else {
+ if (number < 0 || number > 0xffff) {
+ error1(t, "character values exceed 64K");
+ return;
+ }
+ }
+ if (t->encoding == ENC_UTF8)
+ d += put_utf8(number, p + d);
+ else
+ p[d++] = number;
+ }
+ }
+ SIZE(p) = d;
+}
+
+extern int read_token(struct tokeniser * t) {
+ symbol * p = t->p;
+ int held = t->token_held;
+ t->token_held = false;
+ if (held) return t->token;
+ while (true) {
+ int code = next_token(t);
+ switch (code) {
+ case c_comment1: /* slash-slash comment */
+ while (t->c < SIZE(p) && p[t->c] != '\n') t->c++;
+ continue;
+ case c_comment2: /* slash-star comment */
+ while (true) {
+ if (t->c >= SIZE(p)) {
+ error1(t, "/* comment not terminated");
+ t->token = -1;
+ return -1;
+ }
+ if (p[t->c] == '\n') t->line_number++;
+ if (eq_s(t, "*/")) break;
+ t->c++;
+ }
+ continue;
+ case c_stringescapes: {
+ int ch1 = next_real_char(t);
+ int ch2 = next_real_char(t);
+ if (ch2 < 0) {
+ error2(t, "stringescapes");
+ continue;
+ }
+ if (ch1 == '\'') {
+ error1(t, "first stringescape cannot be '");
+ continue;
+ }
+ t->m_start = ch1;
+ t->m_end = ch2;
+ continue;
+ }
+ case c_stringdef: {
+ int base = 0;
+ read_chars(t);
+ code = read_token(t);
+ if (code == c_hex) { base = 16; code = read_token(t); } else
+ if (code == c_decimal) { base = 10; code = read_token(t); }
+ if (code != c_literalstring) {
+ error1(t, "string omitted after stringdef");
+ continue;
+ }
+ if (base > 0) convert_numeric_string(t, t->b, base);
+ { NEW(m_pair, q);
+ q->next = t->m_pairs;
+ q->name = copy_b(t->b2);
+ q->value = copy_b(t->b);
+ t->m_pairs = q;
+ if (t->uplusmode != UPLUS_DEFINED &&
+ (SIZE(t->b2) >= 3 && t->b2[0] == 'U' && t->b2[1] == '+')) {
+ if (t->uplusmode == UPLUS_UNICODE) {
+ error1(t, "U+xxxx already used with implicit meaning");
+ } else {
+ t->uplusmode = UPLUS_DEFINED;
+ }
+ }
+ }
+ continue;
+ }
+ case c_get:
+ code = read_token(t);
+ if (code != c_literalstring) {
+ error1(t, "string omitted after get"); continue;
+ }
+ t->get_depth++;
+ if (t->get_depth > 10) {
+ fprintf(stderr, "get directives go 10 deep. Looping?\n");
+ exit(1);
+ }
+ {
+ NEW(input, q);
+ char * file = b_to_s(t->b);
+ symbol * u = get_input(file);
+ if (u == 0) {
+ struct include * r;
+ for (r = t->includes; r; r = r->next) {
+ symbol * b = copy_b(r->b);
+ b = add_to_b(b, SIZE(t->b), t->b);
+ free(file);
+ file = b_to_s(b);
+ u = get_input(file);
+ lose_b(b);
+ if (u != 0) break;
+ }
+ }
+ if (u == 0) {
+ error(t, "Can't get '", SIZE(t->b), t->b, "'");
+ exit(1);
+ }
+ memmove(q, t, sizeof(struct input));
+ t->next = q;
+ t->p = u;
+ t->c = 0;
+ t->file = file;
+ t->file_needs_freeing = true;
+ t->line_number = 1;
+ }
+ p = t->p;
+ continue;
+ case -1:
+ if (t->next) {
+ lose_b(p);
+ {
+ struct input * q = t->next;
+ memmove(t, q, sizeof(struct input)); p = t->p;
+ FREE(q);
+ }
+ t->get_depth--;
+ continue;
+ }
+ /* fall through */
+ default:
+ t->previous_token = t->token;
+ t->token = code;
+ return code;
+ }
+ }
+}
+
+extern const char * name_of_token(int code) {
+ int i;
+ for (i = 1; i < vocab->code; i++)
+ if ((vocab + i)->code == code) return (const char *)(vocab + i)->s;
+ switch (code) {
+ case c_mathassign: return "=";
+ case c_name: return "name";
+ case c_number: return "number";
+ case c_literalstring:return "literal";
+ case c_neg: return "neg";
+ case c_grouping: return "grouping";
+ case c_call: return "call";
+ case c_booltest: return "Boolean test";
+ case -2: return "start of text";
+ case -1: return "end of text";
+ default: return "?";
+ }
+}
+
+extern void disable_token(struct tokeniser * t, int code) {
+ t->token_disabled[code] = 1;
+}
+
+extern struct tokeniser * create_tokeniser(symbol * p, char * file) {
+ NEW(tokeniser, t);
+ t->next = 0;
+ t->p = p;
+ t->c = 0;
+ t->file = file;
+ t->file_needs_freeing = false;
+ t->line_number = 1;
+ t->b = create_b(0);
+ t->b2 = create_b(0);
+ t->m_start = -1;
+ t->m_pairs = 0;
+ t->get_depth = 0;
+ t->error_count = 0;
+ t->token_held = false;
+ t->token = -2;
+ t->previous_token = -2;
+ t->uplusmode = UPLUS_NONE;
+ memset(t->token_disabled, 0, sizeof(t->token_disabled));
+ return t;
+}
+
+extern void close_tokeniser(struct tokeniser * t) {
+ lose_b(t->b);
+ lose_b(t->b2);
+ {
+ struct m_pair * q = t->m_pairs;
+ while (q) {
+ struct m_pair * q_next = q->next;
+ lose_b(q->name);
+ lose_b(q->value);
+ FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct input * q = t->next;
+ while (q) {
+ struct input * q_next = q->next;
+ FREE(q);
+ q = q_next;
+ }
+ }
+ if (t->file_needs_freeing) free(t->file);
+ FREE(t);
+}
diff --git a/contrib/snowball/doc/TODO b/contrib/snowball/doc/TODO
new file mode 100644
index 0000000..0cfa1b1
--- /dev/null
+++ b/contrib/snowball/doc/TODO
@@ -0,0 +1,15 @@
+Things to do:
+
+ - Write documentation for how to use libstemmer (as opposed to how stemming
+ algorithms themselves work).
+ Currently, the documentation in the include/libstemmer.h header file is
+ pretty clear and comprehensive, but an overview document wouldn't go amiss.
+
+Things that would be nice to include at some point.
+
+ - Add version numbers to each stemming algorithm, and allow the interface to
+ request a specific version of the stemming algorithms. Default to providing
+ the latest version of the algorithm.
+ - Make mkmodules.pl generate the build system, instead of being called from it.
+ This would allow it to generate the list of modules to be built, so that it's
+ not necessary to change things in more than one place to add a new algorithm.
diff --git a/contrib/snowball/doc/libstemmer_c_README b/contrib/snowball/doc/libstemmer_c_README
new file mode 100644
index 0000000..9d3af8b
--- /dev/null
+++ b/contrib/snowball/doc/libstemmer_c_README
@@ -0,0 +1,125 @@
+libstemmer_c
+============
+
+This document pertains to the C version of the libstemmer distribution,
+available for download from:
+
+http://snowball.tartarus.org/dist/libstemmer_c.tgz
+
+
+Compiling the library
+=====================
+
+A simple makefile is provided for Unix style systems. On such systems, it
+should be possible simply to run "make", and the file "libstemmer.o"
+and the example program "stemwords" will be generated.
+
+If this doesn't work on your system, you need to write your own build
+system (or call the compiler directly). The files to compile are
+all contained in the "libstemmer", "runtime" and "src_c" directories,
+and the public header file is contained in the "include" directory.
+
+The library comes in two flavours; UTF-8 only, and UTF-8 plus other character
+sets. To use the utf-8 only flavour, compile "libstemmer_utf8.c" instead of
+"libstemmer.c".
+
+For convenience "mkinc.mak" is a makefile fragment listing the source files and
+header files used to compile the standard version of the library.
+"mkinc_utf8.mak" is a comparable makefile fragment listing just the source
+files for the UTF-8 only version of the library.
+
+
+Using the library
+=================
+
+The library provides a simple C API. Essentially, a new stemmer can
+be obtained by using "sb_stemmer_new". "sb_stemmer_stem" is then
+used to stem a word, "sb_stemmer_length" returns the stemmed
+length of the last word processed, and "sb_stemmer_delete" is
+used to delete a stemmer.
+
+Creating a stemmer is a relatively expensive operation - the expected
+usage pattern is that a new stemmer is created when needed, used
+to stem many words, and deleted after some time.
+
+Stemmers are re-entrant, but not threadsafe. In other words, if
+you wish to access the same stemmer object from multiple threads,
+you must ensure that all access is protected by a mutex or similar
+device.
+
+libstemmer does not currently incorporate any mechanism for caching the results
+of stemming operations. Such caching can greatly increase the performance of a
+stemmer under certain situations, so suitable patches will be considered for
+inclusion.
+
+The standard libstemmer sources contain an algorithm for each of the supported
+languages. The algorithm may be selected using the english name of the
+language, or using the 2 or 3 letter ISO 639 language codes. In addition,
+the traditional "Porter" stemming algorithm for english is included for
+backwards compatibility purposes, but we recommend use of the "English"
+stemmer in preference for new projects.
+
+(Some minor algorithms which are included only as curiosities in the snowball
+website, such as the Lovins stemmer and the Kraaij Pohlmann stemmer, are not
+included in the standard libstemmer sources. These are not really supported by
+the snowball project, but it would be possible to compile a modified libstemmer
+library containing these if desired.)
+
+
+The stemwords example
+=====================
+
+The stemwords example program allows you to run any of the stemmers
+compiled into the libstemmer library on a sample vocabulary. For
+details on how to use it, run it with the "-h" command line option.
+
+
+Using the library in a larger system
+====================================
+
+If you are incorporating the library into the build system of a larger
+program, I recommend copying the unpacked tarball without modification into
+a subdirectory of the sources of your program. Future versions of the
+library are intended to keep the same structure, so this will keep the
+work required to move to a new version of the library to a minimum.
+
+As an additional convenience, the list of source and header files used
+in the library is detailed in mkinc.mak - a file which is in a suitable
+format for inclusion by a Makefile. By including this file in your build
+system, you can link the snowball system into your program with a few
+extra rules.
+
+Using the library in a system using GNU autotools
+=================================================
+
+The libstemmer_c library can be integrated into a larger system which uses the
+GNU autotool framework (and in particular, automake and autoconf) as follows:
+
+1) Unpack libstemmer_c.tgz in the top level project directory so that there is
+ a libstemmer_c subdirectory of the top level directory of the project.
+
+2) Add a file "Makefile.am" to the unpacked libstemmer_c folder, containing:
+
+noinst_LTLIBRARIES = libstemmer.la
+include $(srcdir)/mkinc.mak
+noinst_HEADERS = $(snowball_headers)
+libstemmer_la_SOURCES = $(snowball_sources)
+
+(You may also need to add other lines to this, for example, if you are using
+compiler options which are not compatible with compiling the libstemmer
+library.)
+
+3) Add libstemmer_c to the AC_CONFIG_FILES declaration in the project's
+ configure.ac file.
+
+4) Add to the top level makefile the following lines (or modify existing
+ assignments to these variables appropriately):
+
+AUTOMAKE_OPTIONS = subdir-objects
+AM_CPPFLAGS = -I$(top_srcdir)/libstemmer_c/include
+SUBDIRS=libstemmer_c
+<name>_LIBADD = libstemmer_c/libstemmer.la
+
+(Where <name> is the name of the library or executable which links against
+libstemmer.)
+
diff --git a/contrib/snowball/doc/libstemmer_java_README b/contrib/snowball/doc/libstemmer_java_README
new file mode 100644
index 0000000..38b1af6
--- /dev/null
+++ b/contrib/snowball/doc/libstemmer_java_README
@@ -0,0 +1,40 @@
+libstemmer_java
+===============
+
+This document pertains to the Java version of the libstemmer distribution,
+available for download from:
+
+http://snowball.tartarus.org/dist/libstemmer_java.tgz
+
+
+Compiling the library
+=====================
+
+Simply run the java compiler on all the java source files under the java
+directory. For example, this can be done under unix by changing directory into
+the java directory, and running:
+
+ javac org/tartarus/snowball/*.java org/tartarus/snowball/ext/*.java
+
+This will compile the library and also an example program "TestApp" which
+provides a command line interface to the library.
+
+
+Using the library
+=================
+
+There is currently no formal documentation on the use of the Java version
+of the library. Additionally, its interface is not guaranteed to be
+stable.
+
+The best documentation of the library is the source of the TestApp example
+program.
+
+
+The TestApp example
+===================
+
+The TestApp example program allows you to run any of the stemmers
+compiled into the libstemmer library on a sample vocabulary. For
+details on how to use it, run it with no command line parameters.
+
diff --git a/contrib/snowball/include/libstemmer.h b/contrib/snowball/include/libstemmer.h
new file mode 100644
index 0000000..98051e1
--- /dev/null
+++ b/contrib/snowball/include/libstemmer.h
@@ -0,0 +1,78 @@
+
+/* Make header file work when included from C++ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct sb_stemmer;
+typedef unsigned char sb_symbol;
+
+/* FIXME - should be able to get a version number for each stemming
+ * algorithm (which will be incremented each time the output changes). */
+
+/** Returns an array of the names of the available stemming algorithms.
+ * Note that these are the canonical names - aliases (ie, other names for
+ * the same algorithm) will not be included in the list.
+ * The list is terminated with a null pointer.
+ *
+ * The list must not be modified in any way.
+ */
+const char ** sb_stemmer_list(void);
+
+/** Create a new stemmer object, using the specified algorithm, for the
+ * specified character encoding.
+ *
+ * All algorithms will usually be available in UTF-8, but may also be
+ * available in other character encodings.
+ *
+ * @param algorithm The algorithm name. This is either the english
+ * name of the algorithm, or the 2 or 3 letter ISO 639 codes for the
+ * language. Note that case is significant in this parameter - the
+ * value should be supplied in lower case.
+ *
+ * @param charenc The character encoding. NULL may be passed as
+ * this value, in which case UTF-8 encoding will be assumed. Otherwise,
+ * the argument may be one of "UTF_8", "ISO_8859_1" (i.e. Latin 1),
+ * "ISO_8859_2" (i.e. Latin 2) or "KOI8_R" (Russian). Note that case is
+ * significant in this parameter.
+ *
+ * @return NULL if the specified algorithm is not recognised, or the
+ * algorithm is not available for the requested encoding. Otherwise,
+ * returns a pointer to a newly created stemmer for the requested algorithm.
+ * The returned pointer must be deleted by calling sb_stemmer_delete().
+ *
+ * @note NULL will also be returned if an out of memory error occurs.
+ */
+struct sb_stemmer * sb_stemmer_new(const char * algorithm, const char * charenc);
+
+/** Delete a stemmer object.
+ *
+ * This frees all resources allocated for the stemmer. After calling
+ * this function, the supplied stemmer may no longer be used in any way.
+ *
+ * It is safe to pass a null pointer to this function - this will have
+ * no effect.
+ */
+void sb_stemmer_delete(struct sb_stemmer * stemmer);
+
+/** Stem a word.
+ *
+ * The return value is owned by the stemmer - it must not be freed or
+ * modified, and it will become invalid when the stemmer is called again,
+ * or if the stemmer is freed.
+ *
+ * The length of the return value can be obtained using sb_stemmer_length().
+ *
+ * If an out-of-memory error occurs, this will return NULL.
+ */
+const sb_symbol * sb_stemmer_stem(struct sb_stemmer * stemmer,
+ const sb_symbol * word, int size);
+
+/** Get the length of the result of the last stemmed word.
+ * This should not be called before sb_stemmer_stem() has been called.
+ */
+int sb_stemmer_length(struct sb_stemmer * stemmer);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/contrib/snowball/libstemmer/libstemmer_c.in b/contrib/snowball/libstemmer/libstemmer_c.in
new file mode 100644
index 0000000..2aa918d
--- /dev/null
+++ b/contrib/snowball/libstemmer/libstemmer_c.in
@@ -0,0 +1,96 @@
+
+#include <stdlib.h>
+#include <string.h>
+#include "../include/libstemmer.h"
+#include "../runtime/api.h"
+#include "@MODULES_H@"
+
+struct sb_stemmer {
+ struct SN_env * (*create)(void);
+ void (*close)(struct SN_env *);
+ int (*stem)(struct SN_env *);
+
+ struct SN_env * env;
+};
+
+extern const char **
+sb_stemmer_list(void)
+{
+ return algorithm_names;
+}
+
+static stemmer_encoding_t
+sb_getenc(const char * charenc)
+{
+ const struct stemmer_encoding * encoding;
+ if (charenc == NULL) return ENC_UTF_8;
+ for (encoding = encodings; encoding->name != 0; encoding++) {
+ if (strcmp(encoding->name, charenc) == 0) break;
+ }
+ if (encoding->name == NULL) return ENC_UNKNOWN;
+ return encoding->enc;
+}
+
+extern struct sb_stemmer *
+sb_stemmer_new(const char * algorithm, const char * charenc)
+{
+ stemmer_encoding_t enc;
+ const struct stemmer_modules * module;
+ struct sb_stemmer * stemmer;
+
+ enc = sb_getenc(charenc);
+ if (enc == ENC_UNKNOWN) return NULL;
+
+ for (module = modules; module->name != 0; module++) {
+ if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
+ }
+ if (module->name == NULL) return NULL;
+
+ stemmer = (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
+ if (stemmer == NULL) return NULL;
+
+ stemmer->create = module->create;
+ stemmer->close = module->close;
+ stemmer->stem = module->stem;
+
+ stemmer->env = stemmer->create();
+ if (stemmer->env == NULL)
+ {
+ sb_stemmer_delete(stemmer);
+ return NULL;
+ }
+
+ return stemmer;
+}
+
+void
+sb_stemmer_delete(struct sb_stemmer * stemmer)
+{
+ if (stemmer == 0) return;
+ if (stemmer->close) {
+ stemmer->close(stemmer->env);
+ stemmer->close = 0;
+ }
+ free(stemmer);
+}
+
+const sb_symbol *
+sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
+{
+ int ret;
+ if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
+ {
+ stemmer->env->l = 0;
+ return NULL;
+ }
+ ret = stemmer->stem(stemmer->env);
+ if (ret < 0) return NULL;
+ stemmer->env->p[stemmer->env->l] = 0;
+ return (const sb_symbol *)(stemmer->env->p);
+}
+
+int
+sb_stemmer_length(struct sb_stemmer * stemmer)
+{
+ return stemmer->env->l;
+}
diff --git a/contrib/snowball/libstemmer/mkmodules.pl b/contrib/snowball/libstemmer/mkmodules.pl
new file mode 100755
index 0000000..dd66787
--- /dev/null
+++ b/contrib/snowball/libstemmer/mkmodules.pl
@@ -0,0 +1,267 @@
+#!/usr/bin/env perl
+use strict;
+use 5.006;
+use warnings;
+
+my $progname = $0;
+
+if (scalar @ARGV < 4 || scalar @ARGV > 5) {
+ print "Usage: $progname <outfile> <C source directory> <modules description file> <source list file> [<enc>]\n";
+ exit 1;
+}
+
+my $outname = shift(@ARGV);
+my $c_src_dir = shift(@ARGV);
+my $descfile = shift(@ARGV);
+my $srclistfile = shift(@ARGV);
+my $enc_only;
+my $extn = '';
+if (@ARGV) {
+ $enc_only = shift(@ARGV);
+ $extn = '_'.$enc_only;
+}
+
+my %aliases = ();
+my %algorithms = ();
+my %algorithm_encs = ();
+
+my %encs = ();
+
+sub addalgenc($$) {
+ my $alg = shift();
+ my $enc = shift();
+
+ if (defined $enc_only) {
+ my $norm_enc = lc $enc;
+ $norm_enc =~ s/_//g;
+ if ($norm_enc ne $enc_only) {
+ return;
+ }
+ }
+
+ if (defined $algorithm_encs{$alg}) {
+ my $hashref = $algorithm_encs{$alg};
+ $$hashref{$enc}=1;
+ } else {
+ my %newhash = ($enc => 1);
+ $algorithm_encs{$alg}=\%newhash;
+ }
+
+ $encs{$enc} = 1;
+}
+
+sub readinput()
+{
+ open DESCFILE, $descfile;
+ my $line;
+ while ($line = <DESCFILE>)
+ {
+ next if $line =~ m/^\s*#/;
+ next if $line =~ m/^\s*$/;
+ my ($alg,$encstr,$aliases) = split(/\s+/, $line);
+ my $enc;
+ my $alias;
+
+ $algorithms{$alg} = 1;
+ foreach $alias (split(/,/, $aliases)) {
+ foreach $enc (split(/,/, $encstr)) {
+ # print "$alias, $enc\n";
+ $aliases{$alias} = $alg;
+ addalgenc($alg, $enc);
+ }
+ }
+ }
+}
+
+sub printoutput()
+{
+ open (OUT, ">$outname") or die "Can't open output file `$outname': $!\n";
+
+ print OUT <<EOS;
+/* $outname: List of stemming modules.
+ *
+ * This file is generated by mkmodules.pl from a list of module names.
+ * Do not edit manually.
+ *
+EOS
+
+ my $line = " * Modules included by this file are: ";
+ print OUT $line;
+ my $linelen = length($line);
+
+ my $need_sep = 0;
+ my $lang;
+ my $enc;
+ my @algorithms = sort keys(%algorithms);
+ foreach $lang (@algorithms) {
+ if ($need_sep) {
+ if (($linelen + 2 + length($lang)) > 77) {
+ print OUT ",\n * ";
+ $linelen = 3;
+ } else {
+ print OUT ', ';
+ $linelen += 2;
+ }
+ }
+ print OUT $lang;
+ $linelen += length($lang);
+ $need_sep = 1;
+ }
+ print OUT "\n */\n\n";
+
+ foreach $lang (@algorithms) {
+ my $hashref = $algorithm_encs{$lang};
+ foreach $enc (sort keys (%$hashref)) {
+ print OUT "#include \"../$c_src_dir/stem_${enc}_$lang.h\"\n";
+ }
+ }
+
+ print OUT <<EOS;
+
+typedef enum {
+ ENC_UNKNOWN=0,
+EOS
+ my $neednl = 0;
+ for $enc (sort keys %encs) {
+ print OUT ",\n" if $neednl;
+ print OUT " ENC_${enc}";
+ $neednl = 1;
+ }
+ print OUT <<EOS;
+
+} stemmer_encoding_t;
+
+struct stemmer_encoding {
+ const char * name;
+ stemmer_encoding_t enc;
+};
+static const struct stemmer_encoding encodings[] = {
+EOS
+ for $enc (sort keys %encs) {
+ print OUT " {\"${enc}\", ENC_${enc}},\n";
+ }
+ print OUT <<EOS;
+ {0,ENC_UNKNOWN}
+};
+
+struct stemmer_modules {
+ const char * name;
+ stemmer_encoding_t enc;
+ struct SN_env * (*create)(void);
+ void (*close)(struct SN_env *);
+ int (*stem)(struct SN_env *);
+};
+static const struct stemmer_modules modules[] = {
+EOS
+
+ for $lang (sort keys %aliases) {
+ my $l = $aliases{$lang};
+ my $hashref = $algorithm_encs{$l};
+ my $enc;
+ foreach $enc (sort keys (%$hashref)) {
+ my $p = "${l}_${enc}";
+ print OUT " {\"$lang\", ENC_$enc, ${p}_create_env, ${p}_close_env, ${p}_stem},\n";
+ }
+ }
+
+ print OUT <<EOS;
+ {0,ENC_UNKNOWN,0,0,0}
+};
+EOS
+
+ print OUT <<EOS;
+static const char * algorithm_names[] = {
+EOS
+
+ for $lang (@algorithms) {
+ print OUT " \"$lang\", \n";
+ }
+
+ print OUT <<EOS;
+ 0
+};
+EOS
+ close OUT or die "Can't close ${outname}: $!\n";
+}
+
+sub printsrclist()
+{
+ open (OUT, ">$srclistfile") or die "Can't open output file `$srclistfile': $!\n";
+
+ print OUT <<EOS;
+# $srclistfile: List of stemming module source files
+#
+# This file is generated by mkmodules.pl from a list of module names.
+# Do not edit manually.
+#
+EOS
+
+ my $line = "# Modules included by this file are: ";
+ print OUT $line;
+ my $linelen = length($line);
+
+ my $need_sep = 0;
+ my $lang;
+ my $srcfile;
+ my $enc;
+ my @algorithms = sort keys(%algorithms);
+ foreach $lang (@algorithms) {
+ if ($need_sep) {
+ if (($linelen + 2 + length($lang)) > 77) {
+ print OUT ",\n# ";
+ $linelen = 3;
+ } else {
+ print OUT ', ';
+ $linelen += 2;
+ }
+ }
+ print OUT $lang;
+ $linelen += length($lang);
+ $need_sep = 1;
+ }
+
+ print OUT "\n\nsnowball_sources= \\\n";
+ for $lang (sort keys %aliases) {
+ my $hashref = $algorithm_encs{$lang};
+ my $enc;
+ foreach $enc (sort keys (%$hashref)) {
+ print OUT " src_c/stem_${enc}_${lang}.c \\\n";
+ }
+ }
+
+ $need_sep = 0;
+ for $srcfile ('runtime/api.c',
+ 'runtime/utilities.c',
+ "libstemmer/libstemmer${extn}.c") {
+ print OUT " \\\n" if $need_sep;
+ print OUT " $srcfile";
+ $need_sep = 1;
+ }
+
+ print OUT "\n\nsnowball_headers= \\\n";
+ for $lang (sort keys %aliases) {
+ my $hashref = $algorithm_encs{$lang};
+ my $enc;
+ foreach $enc (sort keys (%$hashref)) {
+ my $p = "${lang}_${enc}";
+ print OUT " src_c/stem_${enc}_${lang}.h \\\n";
+ }
+ }
+
+ $need_sep = 0;
+ for $srcfile ('include/libstemmer.h',
+ "libstemmer/modules${extn}.h",
+ 'runtime/api.h',
+ 'runtime/header.h') {
+ print OUT " \\\n" if $need_sep;
+ print OUT " $srcfile";
+ $need_sep = 1;
+ }
+
+ print OUT "\n\n";
+ close OUT or die "Can't close ${srclistfile}: $!\n";
+}
+
+readinput();
+printoutput();
+printsrclist();
diff --git a/contrib/snowball/libstemmer/modules.txt b/contrib/snowball/libstemmer/modules.txt
new file mode 100644
index 0000000..f6dcc7e
--- /dev/null
+++ b/contrib/snowball/libstemmer/modules.txt
@@ -0,0 +1,58 @@
+# This file contains a list of stemmers to include in the distribution.
+# The format is a set of space separated lines - on each line:
+# First item is name of stemmer.
+# Second item is comma separated list of character sets.
+# Third item is comma separated list of names to refer to the stemmer by.
+#
+# Lines starting with a #, or blank lines, are ignored.
+
+# List all the main algorithms for each language, in UTF-8, and also with
+# the most commonly used encoding.
+
+arabic UTF_8 arabic,ar,ara
+danish UTF_8 danish,da,dan
+dutch UTF_8 dutch,nl,dut,nld
+english UTF_8 english,en,eng
+finnish UTF_8 finnish,fi,fin
+french UTF_8 french,fr,fre,fra
+german UTF_8 german,de,ger,deu
+greek UTF_8 greek,el,gre,ell
+hindi UTF_8 hindi,hi,hin
+hungarian UTF_8 hungarian,hu,hun
+indonesian UTF_8 indonesian,id,ind
+italian UTF_8 italian,it,ita
+lithuanian UTF_8 lithuanian,lt,lit
+nepali UTF_8 nepali,ne,nep
+norwegian UTF_8 norwegian,no,nor
+portuguese UTF_8 portuguese,pt,por
+romanian UTF_8 romanian,ro,rum,ron
+russian UTF_8 russian,ru,rus
+serbian UTF_8 serbian,sr,srp
+spanish UTF_8 spanish,es,esl,spa
+swedish UTF_8 swedish,sv,swe
+tamil UTF_8 tamil,ta,tam
+turkish UTF_8 turkish,tr,tur
+
+# Also include the traditional porter algorithm for english.
+# The porter algorithm is included in the libstemmer distribution to assist
+# with backwards compatibility, but for new systems the english algorithm
+# should be used in preference.
+porter UTF_8 porter english
+
+# Some other stemmers in the snowball project are not included in the standard
+# distribution. To compile a libstemmer with them in, add them to this list,
+# and regenerate the distribution. (You will need a full source checkout for
+# this.) They are included in the snowball website as curiosities, but are not
+# intended for general use, and use of them is is not fully supported. These
+# algorithms are:
+#
+# german2 - This is a slight modification of the german stemmer.
+#german2 UTF_8,ISO_8859_1 german2 german
+#
+# kraaij_pohlmann - This is a different dutch stemmer.
+#kraaij_pohlmann UTF_8,ISO_8859_1 kraaij_pohlmann dutch
+#
+# lovins - This is an english stemmer, but fairly outdated, and
+# only really applicable to a restricted type of input text
+# (keywords in academic publications).
+#lovins UTF_8,ISO_8859_1 lovins english
diff --git a/contrib/snowball/libstemmer/modules_utf8.txt b/contrib/snowball/libstemmer/modules_utf8.txt
new file mode 100644
index 0000000..60a0e1d
--- /dev/null
+++ b/contrib/snowball/libstemmer/modules_utf8.txt
@@ -0,0 +1,49 @@
+# This file contains a list of stemmers to include in the distribution.
+# The format is a set of space separated lines - on each line:
+# First item is name of stemmer.
+# Second item is comma separated list of character sets.
+# Third item is comma separated list of names to refer to the stemmer by.
+#
+# Lines starting with a #, or blank lines, are ignored.
+
+# List all the main algorithms for each language, in UTF-8.
+
+danish UTF_8 danish,da,dan
+dutch UTF_8 dutch,nl,dut,nld
+english UTF_8 english,en,eng
+finnish UTF_8 finnish,fi,fin
+french UTF_8 french,fr,fre,fra
+german UTF_8 german,de,ger,deu
+hungarian UTF_8 hungarian,hu,hun
+italian UTF_8 italian,it,ita
+norwegian UTF_8 norwegian,no,nor
+portuguese UTF_8 portuguese,pt,por
+romanian UTF_8 romanian,ro,rum,ron
+russian UTF_8 russian,ru,rus
+spanish UTF_8 spanish,es,esl,spa
+swedish UTF_8 swedish,sv,swe
+turkish UTF_8 turkish,tr,tur
+
+# Also include the traditional porter algorithm for english.
+# The porter algorithm is included in the libstemmer distribution to assist
+# with backwards compatibility, but for new systems the english algorithm
+# should be used in preference.
+porter UTF_8 porter
+
+# Some other stemmers in the snowball project are not included in the standard
+# distribution. To compile a libstemmer with them in, add them to this list,
+# and regenerate the distribution. (You will need a full source checkout for
+# this.) They are included in the snowball website as curiosities, but are not
+# intended for general use, and use of them is is not fully supported. These
+# algorithms are:
+#
+# german2 - This is a slight modification of the german stemmer.
+#german2 UTF_8 german2
+#
+# kraaij_pohlmann - This is a different dutch stemmer.
+#kraaij_pohlmann UTF_8 kraaij_pohlmann
+#
+# lovins - This is an english stemmer, but fairly outdated, and
+# only really applicable to a restricted type of input text
+# (keywords in academic publications).
+#lovins UTF_8 lovins
diff --git a/contrib/snowball/runtime/api.c b/contrib/snowball/runtime/api.c
new file mode 100644
index 0000000..21ea5f2
--- /dev/null
+++ b/contrib/snowball/runtime/api.c
@@ -0,0 +1,58 @@
+
+#include <stdlib.h> /* for calloc, free */
+#include "header.h"
+
+extern struct SN_env * SN_create_env(int S_size, int I_size)
+{
+ struct SN_env * z = (struct SN_env *) calloc(1, sizeof(struct SN_env));
+ if (z == NULL) return NULL;
+ z->p = create_s();
+ if (z->p == NULL) goto error;
+ if (S_size)
+ {
+ int i;
+ z->S = (symbol * *) calloc(S_size, sizeof(symbol *));
+ if (z->S == NULL) goto error;
+
+ for (i = 0; i < S_size; i++)
+ {
+ z->S[i] = create_s();
+ if (z->S[i] == NULL) goto error;
+ }
+ }
+
+ if (I_size)
+ {
+ z->I = (int *) calloc(I_size, sizeof(int));
+ if (z->I == NULL) goto error;
+ }
+
+ return z;
+error:
+ SN_close_env(z, S_size);
+ return NULL;
+}
+
+extern void SN_close_env(struct SN_env * z, int S_size)
+{
+ if (z == NULL) return;
+ if (S_size)
+ {
+ int i;
+ for (i = 0; i < S_size; i++)
+ {
+ lose_s(z->S[i]);
+ }
+ free(z->S);
+ }
+ free(z->I);
+ if (z->p) lose_s(z->p);
+ free(z);
+}
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
+{
+ int err = replace_s(z, 0, z->l, size, s, NULL);
+ z->c = 0;
+ return err;
+}
diff --git a/contrib/snowball/runtime/api.h b/contrib/snowball/runtime/api.h
new file mode 100644
index 0000000..ba9d1c1
--- /dev/null
+++ b/contrib/snowball/runtime/api.h
@@ -0,0 +1,32 @@
+
+typedef unsigned char symbol;
+
+/* Or replace 'char' above with 'short' for 16 bit characters.
+
+ More precisely, replace 'char' with whatever type guarantees the
+ character width you need. Note however that sizeof(symbol) should divide
+ HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
+ there is an alignment problem. In the unlikely event of a problem here,
+ consult Martin Porter.
+
+*/
+
+struct SN_env {
+ symbol * p;
+ int c; int l; int lb; int bra; int ket;
+ symbol * * S;
+ int * I;
+};
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * SN_create_env(int S_size, int I_size);
+extern void SN_close_env(struct SN_env * z, int S_size);
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/contrib/snowball/runtime/header.h b/contrib/snowball/runtime/header.h
new file mode 100644
index 0000000..85a42fd
--- /dev/null
+++ b/contrib/snowball/runtime/header.h
@@ -0,0 +1,59 @@
+
+#include <limits.h>
+
+#include "api.h"
+
+#define MAXINT INT_MAX
+#define MININT INT_MIN
+
+#define HEAD 2*sizeof(int)
+
+#define SIZE(p) ((int *)(p))[-1]
+#define SET_SIZE(p, n) ((int *)(p))[-1] = n
+#define CAPACITY(p) ((int *)(p))[-2]
+
+struct among
+{ int s_size; /* number of chars in string */
+ const symbol * s; /* search string */
+ int substring_i;/* index to longest matching substring */
+ int result; /* result of the lookup */
+ int (* function)(struct SN_env *);
+};
+
+extern symbol * create_s(void);
+extern void lose_s(symbol * p);
+
+extern int skip_utf8(const symbol * p, int c, int lb, int l, int n);
+
+extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+
+extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+
+extern int eq_s(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_v(struct SN_env * z, const symbol * p);
+extern int eq_v_b(struct SN_env * z, const symbol * p);
+
+extern int find_among(struct SN_env * z, const struct among * v, int v_size);
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size);
+
+extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjustment);
+extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s);
+extern int slice_from_v(struct SN_env * z, const symbol * p);
+extern int slice_del(struct SN_env * z);
+
+extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s);
+extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p);
+
+extern symbol * slice_to(struct SN_env * z, symbol * p);
+extern symbol * assign_to(struct SN_env * z, symbol * p);
+
+extern int len_utf8(const symbol * p);
+
+extern void debug(struct SN_env * z, int number, int line_count);
diff --git a/contrib/snowball/runtime/utilities.c b/contrib/snowball/runtime/utilities.c
new file mode 100644
index 0000000..1cfd1a1
--- /dev/null
+++ b/contrib/snowball/runtime/utilities.c
@@ -0,0 +1,503 @@
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "header.h"
+
+#define CREATE_SIZE 1
+
+extern symbol * create_s(void) {
+ symbol * p;
+ void * mem = malloc(HEAD + (CREATE_SIZE + 1) * sizeof(symbol));
+ if (mem == NULL) return NULL;
+ p = (symbol *) (HEAD + (char *) mem);
+ CAPACITY(p) = CREATE_SIZE;
+ SET_SIZE(p, 0);
+ return p;
+}
+
+extern void lose_s(symbol * p) {
+ if (p == NULL) return;
+ free((char *) p - HEAD);
+}
+
+/*
+ new_p = skip_utf8(p, c, lb, l, n); skips n characters forwards from p + c
+ if n +ve, or n characters backwards from p + c - 1 if n -ve. new_p is the new
+ position, or -1 on failure.
+
+ -- used to implement hop and next in the utf8 case.
+*/
+
+extern int skip_utf8(const symbol * p, int c, int lb, int l, int n) {
+ int b;
+ if (n >= 0) {
+ for (; n > 0; n--) {
+ if (c >= l) return -1;
+ b = p[c++];
+ if (b >= 0xC0) { /* 1100 0000 */
+ while (c < l) {
+ b = p[c];
+ if (b >= 0xC0 || b < 0x80) break;
+ /* break unless b is 10------ */
+ c++;
+ }
+ }
+ }
+ } else {
+ for (; n < 0; n++) {
+ if (c <= lb) return -1;
+ b = p[--c];
+ if (b >= 0x80) { /* 1000 0000 */
+ while (c > lb) {
+ b = p[c];
+ if (b >= 0xC0) break; /* 1100 0000 */
+ c--;
+ }
+ }
+ }
+ }
+ return c;
+}
+
+/* Code for character groupings: utf8 cases */
+
+static int get_utf8(const symbol * p, int c, int l, int * slot) {
+ int b0, b1, b2;
+ if (c >= l) return 0;
+ b0 = p[c++];
+ if (b0 < 0xC0 || c == l) { /* 1100 0000 */
+ *slot = b0;
+ return 1;
+ }
+ b1 = p[c++] & 0x3F;
+ if (b0 < 0xE0 || c == l) { /* 1110 0000 */
+ *slot = (b0 & 0x1F) << 6 | b1;
+ return 2;
+ }
+ b2 = p[c++] & 0x3F;
+ if (b0 < 0xF0 || c == l) { /* 1111 0000 */
+ *slot = (b0 & 0xF) << 12 | b1 << 6 | b2;
+ return 3;
+ }
+ *slot = (b0 & 0xE) << 18 | b1 << 12 | b2 << 6 | (p[c] & 0x3F);
+ return 4;
+}
+
+static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
+ int a, b;
+ if (c <= lb) return 0;
+ b = p[--c];
+ if (b < 0x80 || c == lb) { /* 1000 0000 */
+ *slot = b;
+ return 1;
+ }
+ a = b & 0x3F;
+ b = p[--c];
+ if (b >= 0xC0 || c == lb) { /* 1100 0000 */
+ *slot = (b & 0x1F) << 6 | a;
+ return 2;
+ }
+ a |= (b & 0x3F) << 6;
+ b = p[--c];
+ if (b >= 0xE0 || c == lb) { /* 1110 0000 */
+ *slot = (b & 0xF) << 12 | a;
+ return 3;
+ }
+ *slot = (p[--c] & 0xE) << 18 | (b & 0x3F) << 12 | a;
+ return 4;
+}
+
+extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_utf8(z->p, z->c, z->l, & ch);
+ if (!w) return -1;
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return w;
+ z->c += w;
+ } while (repeat);
+ return 0;
+}
+
+extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
+ if (!w) return -1;
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return w;
+ z->c -= w;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_utf8(z->p, z->c, z->l, & ch);
+ if (!w) return -1;
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+ return w;
+ z->c += w;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
+ if (!w) return -1;
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+ return w;
+ z->c -= w;
+ } while (repeat);
+ return 0;
+}
+
+/* Code for character groupings: non-utf8 cases */
+
+extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c >= z->l) return -1;
+ ch = z->p[z->c];
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return 1;
+ z->c++;
+ } while (repeat);
+ return 0;
+}
+
+extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c <= z->lb) return -1;
+ ch = z->p[z->c - 1];
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return 1;
+ z->c--;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c >= z->l) return -1;
+ ch = z->p[z->c];
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+ return 1;
+ z->c++;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c <= z->lb) return -1;
+ ch = z->p[z->c - 1];
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+ return 1;
+ z->c--;
+ } while (repeat);
+ return 0;
+}
+
+extern int eq_s(struct SN_env * z, int s_size, const symbol * s) {
+ if (z->l - z->c < s_size || memcmp(z->p + z->c, s, s_size * sizeof(symbol)) != 0) return 0;
+ z->c += s_size; return 1;
+}
+
+extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s) {
+ if (z->c - z->lb < s_size || memcmp(z->p + z->c - s_size, s, s_size * sizeof(symbol)) != 0) return 0;
+ z->c -= s_size; return 1;
+}
+
+extern int eq_v(struct SN_env * z, const symbol * p) {
+ return eq_s(z, SIZE(p), p);
+}
+
+extern int eq_v_b(struct SN_env * z, const symbol * p) {
+ return eq_s_b(z, SIZE(p), p);
+}
+
+extern int find_among(struct SN_env * z, const struct among * v, int v_size) {
+
+ int i = 0;
+ int j = v_size;
+
+ int c = z->c; int l = z->l;
+ const symbol * q = z->p + c;
+
+ const struct among * w;
+
+ int common_i = 0;
+ int common_j = 0;
+
+ int first_key_inspected = 0;
+
+ while (1) {
+ int k = i + ((j - i) >> 1);
+ int diff = 0;
+ int common = common_i < common_j ? common_i : common_j; /* smaller */
+ w = v + k;
+ {
+ int i2; for (i2 = common; i2 < w->s_size; i2++) {
+ if (c + common == l) { diff = -1; break; }
+ diff = q[common] - w->s[i2];
+ if (diff != 0) break;
+ common++;
+ }
+ }
+ if (diff < 0) {
+ j = k;
+ common_j = common;
+ } else {
+ i = k;
+ common_i = common;
+ }
+ if (j - i <= 1) {
+ if (i > 0) break; /* v->s has been inspected */
+ if (j == i) break; /* only one item in v */
+
+ /* - but now we need to go round once more to get
+ v->s inspected. This looks messy, but is actually
+ the optimal approach. */
+
+ if (first_key_inspected) break;
+ first_key_inspected = 1;
+ }
+ }
+ while (1) {
+ w = v + i;
+ if (common_i >= w->s_size) {
+ z->c = c + w->s_size;
+ if (w->function == 0) return w->result;
+ {
+ int res = w->function(z);
+ z->c = c + w->s_size;
+ if (res) return w->result;
+ }
+ }
+ i = w->substring_i;
+ if (i < 0) return 0;
+ }
+}
+
+/* find_among_b is for backwards processing. Same comments apply */
+
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
+
+ int i = 0;
+ int j = v_size;
+
+ int c = z->c; int lb = z->lb;
+ const symbol * q = z->p + c - 1;
+
+ const struct among * w;
+
+ int common_i = 0;
+ int common_j = 0;
+
+ int first_key_inspected = 0;
+
+ while (1) {
+ int k = i + ((j - i) >> 1);
+ int diff = 0;
+ int common = common_i < common_j ? common_i : common_j;
+ w = v + k;
+ {
+ int i2; for (i2 = w->s_size - 1 - common; i2 >= 0; i2--) {
+ if (c - common == lb) { diff = -1; break; }
+ diff = q[- common] - w->s[i2];
+ if (diff != 0) break;
+ common++;
+ }
+ }
+ if (diff < 0) { j = k; common_j = common; }
+ else { i = k; common_i = common; }
+ if (j - i <= 1) {
+ if (i > 0) break;
+ if (j == i) break;
+ if (first_key_inspected) break;
+ first_key_inspected = 1;
+ }
+ }
+ while (1) {
+ w = v + i;
+ if (common_i >= w->s_size) {
+ z->c = c - w->s_size;
+ if (w->function == 0) return w->result;
+ {
+ int res = w->function(z);
+ z->c = c - w->s_size;
+ if (res) return w->result;
+ }
+ }
+ i = w->substring_i;
+ if (i < 0) return 0;
+ }
+}
+
+
+/* Increase the size of the buffer pointed to by p to at least n symbols.
+ * If insufficient memory, returns NULL and frees the old buffer.
+ */
+static symbol * increase_size(symbol * p, int n) {
+ symbol * q;
+ int new_size = n + 20;
+ void * mem = realloc((char *) p - HEAD,
+ HEAD + (new_size + 1) * sizeof(symbol));
+ if (mem == NULL) {
+ lose_s(p);
+ return NULL;
+ }
+ q = (symbol *) (HEAD + (char *)mem);
+ CAPACITY(q) = new_size;
+ return q;
+}
+
+/* to replace symbols between c_bra and c_ket in z->p by the
+ s_size symbols at s.
+ Returns 0 on success, -1 on error.
+ Also, frees z->p (and sets it to NULL) on error.
+*/
+extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjptr)
+{
+ int adjustment;
+ int len;
+ if (z->p == NULL) {
+ z->p = create_s();
+ if (z->p == NULL) return -1;
+ }
+ adjustment = s_size - (c_ket - c_bra);
+ len = SIZE(z->p);
+ if (adjustment != 0) {
+ if (adjustment + len > CAPACITY(z->p)) {
+ z->p = increase_size(z->p, adjustment + len);
+ if (z->p == NULL) return -1;
+ }
+ memmove(z->p + c_ket + adjustment,
+ z->p + c_ket,
+ (len - c_ket) * sizeof(symbol));
+ SET_SIZE(z->p, adjustment + len);
+ z->l += adjustment;
+ if (z->c >= c_ket)
+ z->c += adjustment;
+ else if (z->c > c_bra)
+ z->c = c_bra;
+ }
+ if (s_size) memmove(z->p + c_bra, s, s_size * sizeof(symbol));
+ if (adjptr != NULL)
+ *adjptr = adjustment;
+ return 0;
+}
+
+static int slice_check(struct SN_env * z) {
+
+ if (z->bra < 0 ||
+ z->bra > z->ket ||
+ z->ket > z->l ||
+ z->p == NULL ||
+ z->l > SIZE(z->p)) /* this line could be removed */
+ {
+#if 0
+ fprintf(stderr, "faulty slice operation:\n");
+ debug(z, -1, 0);
+#endif
+ return -1;
+ }
+ return 0;
+}
+
+extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s) {
+ if (slice_check(z)) return -1;
+ return replace_s(z, z->bra, z->ket, s_size, s, NULL);
+}
+
+extern int slice_from_v(struct SN_env * z, const symbol * p) {
+ return slice_from_s(z, SIZE(p), p);
+}
+
+extern int slice_del(struct SN_env * z) {
+ return slice_from_s(z, 0, 0);
+}
+
+extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s) {
+ int adjustment;
+ if (replace_s(z, bra, ket, s_size, s, &adjustment))
+ return -1;
+ if (bra <= z->bra) z->bra += adjustment;
+ if (bra <= z->ket) z->ket += adjustment;
+ return 0;
+}
+
+extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p) {
+ return insert_s(z, bra, ket, SIZE(p), p);
+}
+
+extern symbol * slice_to(struct SN_env * z, symbol * p) {
+ if (slice_check(z)) {
+ lose_s(p);
+ return NULL;
+ }
+ {
+ int len = z->ket - z->bra;
+ if (CAPACITY(p) < len) {
+ p = increase_size(p, len);
+ if (p == NULL)
+ return NULL;
+ }
+ memmove(p, z->p + z->bra, len * sizeof(symbol));
+ SET_SIZE(p, len);
+ }
+ return p;
+}
+
+extern symbol * assign_to(struct SN_env * z, symbol * p) {
+ int len = z->l;
+ if (CAPACITY(p) < len) {
+ p = increase_size(p, len);
+ if (p == NULL)
+ return NULL;
+ }
+ memmove(p, z->p, len * sizeof(symbol));
+ SET_SIZE(p, len);
+ return p;
+}
+
+extern int len_utf8(const symbol * p) {
+ int size = SIZE(p);
+ int len = 0;
+ while (size--) {
+ symbol b = *p++;
+ if (b >= 0xC0 || b < 0x80) ++len;
+ }
+ return len;
+}
+
+#if 0
+extern void debug(struct SN_env * z, int number, int line_count) {
+ int i;
+ int limit = SIZE(z->p);
+ /*if (number >= 0) printf("%3d (line %4d): '", number, line_count);*/
+ if (number >= 0) printf("%3d (line %4d): [%d]'", number, line_count,limit);
+ for (i = 0; i <= limit; i++) {
+ if (z->lb == i) printf("{");
+ if (z->bra == i) printf("[");
+ if (z->c == i) printf("|");
+ if (z->ket == i) printf("]");
+ if (z->l == i) printf("}");
+ if (i < limit)
+ { int ch = z->p[i];
+ if (ch == 0) ch = '#';
+ printf("%c", ch);
+ }
+ }
+ printf("'\n");
+}
+#endif
diff --git a/contrib/t1ha/CMakeLists.txt b/contrib/t1ha/CMakeLists.txt
new file mode 100644
index 0000000..c8de483
--- /dev/null
+++ b/contrib/t1ha/CMakeLists.txt
@@ -0,0 +1,6 @@
+SET(T1HASRC t1ha1.c
+ t1ha2.c)
+
+ADD_LIBRARY(rspamd-t1ha STATIC ${T1HASRC})
+SET_TARGET_PROPERTIES(rspamd-t1ha PROPERTIES VERSION ${RSPAMD_VERSION})
+ADD_DEFINITIONS("-DT1HA_USE_FAST_ONESHOT_READ=0")
diff --git a/contrib/t1ha/LICENSE b/contrib/t1ha/LICENSE
new file mode 100644
index 0000000..d02db65
--- /dev/null
+++ b/contrib/t1ha/LICENSE
@@ -0,0 +1,21 @@
+ Copyright (c) 2016-2018 Positive Technologies, https://www.ptsecurity.com,
+ Fast Positive Hash.
+
+ Portions Copyright (c) 2010-2013 Leonid Yuriev <leo@yuriev.ru>,
+ The 1Hippeus project (t1h).
+
+ This software is provided 'as-is', without any express or implied
+ warranty. In no event will the authors be held liable for any damages
+ arising from the use of this software.
+
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely, subject to the following restrictions:
+
+ 1. The origin of this software must not be misrepresented; you must not
+ claim that you wrote the original software. If you use this software
+ in a product, an acknowledgement in the product documentation would be
+ appreciated but is not required.
+ 2. Altered source versions must be plainly marked as such, and must not be
+ misrepresented as being the original software.
+ 3. This notice may not be removed or altered from any source distribution.
diff --git a/contrib/t1ha/t1ha.h b/contrib/t1ha/t1ha.h
new file mode 100644
index 0000000..e5f2dad
--- /dev/null
+++ b/contrib/t1ha/t1ha.h
@@ -0,0 +1,321 @@
+/*
+ * Copyright (c) 2016-2018 Positive Technologies, https://www.ptsecurity.com,
+ * Fast Positive Hash.
+ *
+ * Portions Copyright (c) 2010-2018 Leonid Yuriev <leo@yuriev.ru>,
+ * The 1Hippeus project (t1h).
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty. In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ * claim that you wrote the original software. If you use this software
+ * in a product, an acknowledgement in the product documentation would be
+ * appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ * misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * t1ha = { Fast Positive Hash, aka "Позитивный Ð¥Ñш" }
+ * by [Positive Technologies](https://www.ptsecurity.ru)
+ *
+ * Briefly, it is a 64-bit Hash Function:
+ * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
+ * but portable and without penalties it can run on any 64-bit CPU.
+ * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
+ * and all others portable hash-functions (which do not use specific
+ * hardware tricks).
+ * 3. Not suitable for cryptography.
+ *
+ * The Future will Positive. Ð’ÑÑ‘ будет хорошо.
+ *
+ * ACKNOWLEDGEMENT:
+ * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
+ * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
+ */
+
+#pragma once
+
+#ifndef __has_attribute
+#define __has_attribute(x) (0)
+#endif
+
+#ifndef __has_include
+#define __has_include(x) (0)
+#endif
+
+#ifndef __GNUC_PREREQ
+#if defined(__GNUC__) && defined(__GNUC_MINOR__)
+#define __GNUC_PREREQ(maj, min) \
+ ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+#else
+#define __GNUC_PREREQ(maj, min) 0
+#endif
+#endif /* __GNUC_PREREQ */
+
+#ifndef __CLANG_PREREQ
+#ifdef __clang__
+#define __CLANG_PREREQ(maj, min) \
+ ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
+#else
+#define __CLANG_PREREQ(maj, min) (0)
+#endif
+#endif /* __CLANG_PREREQ */
+
+/*****************************************************************************/
+
+#ifdef _MSC_VER
+/* Avoid '16' bytes padding added after data member 't1ha_context::total'
+ * and other warnings from std-headers if warning-level > 3. */
+#pragma warning(push, 3)
+#endif
+
+#if defined(__cplusplus) && __cplusplus >= 201103L
+#include <climits>
+#include <cstddef>
+#include <cstdint>
+#else
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#endif
+
+/*****************************************************************************/
+
+#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
+ defined(i486) || defined(__i486) || defined(__i486__) || \
+ defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
+ defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
+ defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
+ defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
+ defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
+#ifndef __ia32__
+/* LY: define neutral __ia32__ for x86 and x86-64 archs */
+#define __ia32__ 1
+#endif /* __ia32__ */
+#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64) || defined(_M_X64))
+/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
+#define __amd64__ 1
+#endif /* __amd64__ */
+#endif /* all x86 */
+
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
+ !defined(__ORDER_BIG_ENDIAN__)
+
+/* *INDENT-OFF* */
+/* clang-format off */
+
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) || \
+ defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
+#include <endian.h>
+#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
+ defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
+#include <machine/endian.h>
+#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
+#include <sys/isa_defs.h>
+#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
+ (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
+#include <sys/endian.h>
+#include <sys/types.h>
+#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
+ defined(__NETBSD__) || defined(__NetBSD__) || \
+ defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif /* OS */
+
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
+#define __BYTE_ORDER__ __BYTE_ORDER
+#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
+#define __BYTE_ORDER__ _BYTE_ORDER
+#else
+#define __ORDER_LITTLE_ENDIAN__ 1234
+#define __ORDER_BIG_ENDIAN__ 4321
+
+#if defined(__LITTLE_ENDIAN__) || \
+ (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
+ defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
+ defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
+ defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
+ defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
+ defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
+ defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
+ defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
+ defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
+ defined(__WINDOWS__)
+#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+
+#elif defined(__BIG_ENDIAN__) || \
+ (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
+ defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
+ defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
+ defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
+ defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
+ defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
+ defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
+#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
+
+#else
+#error __BYTE_ORDER__ should be defined.
+#endif /* Arch */
+
+#endif
+#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
+
+/*****************************************************************************/
+
+#ifndef __dll_export
+#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__)
+#if defined(__GNUC__) || __has_attribute(dllexport)
+#define __dll_export __attribute__((dllexport))
+#elif defined(_MSC_VER)
+#define __dll_export __declspec(dllexport)
+#else
+#define __dll_export
+#endif
+#elif defined(__GNUC__) || __has_attribute(visibility)
+#define __dll_export __attribute__((visibility("default")))
+#else
+#define __dll_export
+#endif
+#endif /* __dll_export */
+
+#ifndef __dll_import
+#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__)
+#if defined(__GNUC__) || __has_attribute(dllimport)
+#define __dll_import __attribute__((dllimport))
+#elif defined(_MSC_VER)
+#define __dll_import __declspec(dllimport)
+#else
+#define __dll_import
+#endif
+#else
+#define __dll_import
+#endif
+#endif /* __dll_import */
+
+#if defined(t1ha_EXPORTS)
+#define T1HA_API __dll_export
+#elif defined(t1ha_IMPORTS)
+#define T1HA_API __dll_import
+#else
+#define T1HA_API
+#endif /* T1HA_API */
+
+#define T1HA_ALIGN_PREFIX
+#define T1HA_ALIGN_SUFFIX
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef union T1HA_ALIGN_PREFIX t1ha_state256 {
+ uint8_t bytes[32];
+ uint32_t u32[8];
+ uint64_t u64[4];
+ struct {
+ uint64_t a, b, c, d;
+ } n;
+} t1ha_state256_t T1HA_ALIGN_SUFFIX;
+
+typedef struct t1ha_context {
+ t1ha_state256_t state;
+ t1ha_state256_t buffer;
+ size_t partial;
+ uint64_t total;
+} t1ha_context_t;
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+/******************************************************************************
+ *
+ * t1ha2 = 64 and 128-bit, SLIGHTLY MORE ATTENTION FOR QUALITY AND STRENGTH.
+ *
+ * - The recommended version of "Fast Positive Hash" with good quality
+ * for checksum, hash tables and fingerprinting.
+ * - Portable and extremely efficiency on modern 64-bit CPUs.
+ * Designed for 64-bit little-endian platforms,
+ * in other cases will runs slowly.
+ * - Great quality of hashing and still faster than other non-t1ha hashes.
+ * Provides streaming mode and 128-bit result.
+ *
+ * Note: Due performance reason 64- and 128-bit results are completely
+ * different each other, i.e. 64-bit result is NOT any part of 128-bit.
+ */
+
+/* The at-once variant with 64-bit result */
+T1HA_API uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed);
+
+/* The at-once variant with 128-bit result.
+ * Argument `extra_result` is NOT optional and MUST be valid.
+ * The high 64-bit part of 128-bit hash will be always unconditionally
+ * stored to the address given by `extra_result` argument. */
+T1HA_API uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result,
+ const void *__restrict data, size_t length,
+ uint64_t seed);
+
+/* The init/update/final trinity for streaming.
+ * Return 64 or 128-bit result depentently from `extra_result` argument. */
+T1HA_API void t1ha2_init(t1ha_context_t *ctx, uint64_t seed_x, uint64_t seed_y);
+T1HA_API void t1ha2_update(t1ha_context_t *__restrict ctx,
+ const void *__restrict data, size_t length);
+
+/* Argument `extra_result` is optional and MAY be NULL.
+ * - If `extra_result` is NOT NULL then the 128-bit hash will be calculated,
+ * and high 64-bit part of it will be stored to the address given
+ * by `extra_result` argument.
+ * - Otherwise the 64-bit hash will be calculated
+ * and returned from function directly.
+ *
+ * Note: Due performance reason 64- and 128-bit results are completely
+ * different each other, i.e. 64-bit result is NOT any part of 128-bit. */
+T1HA_API uint64_t t1ha2_final(t1ha_context_t *__restrict ctx,
+ uint64_t *__restrict extra_result /* optional */);
+
+/******************************************************************************
+ *
+ * t1ha1 = 64-bit, BASELINE FAST PORTABLE HASH:
+ *
+ * - Runs faster on 64-bit platforms in other cases may runs slowly.
+ * - Portable and stable, returns same 64-bit result
+ * on all architectures and CPUs.
+ * - Unfortunately it fails the "strict avalanche criteria",
+ * see test results at https://github.com/demerphq/smhasher.
+ *
+ * This flaw is insignificant for the t1ha1() purposes and imperceptible
+ * from a practical point of view.
+ * However, nowadays this issue has resolved in the next t1ha2(),
+ * that was initially planned to providing a bit more quality.
+ */
+
+/* The little-endian variant. */
+T1HA_API uint64_t t1ha1_le(const void *data, size_t length, uint64_t seed);
+
+/* The big-endian variant. */
+T1HA_API uint64_t t1ha1_be(const void *data, size_t length, uint64_t seed);
+
+/* The historical nicname for generic little-endian variant. */
+static __inline uint64_t t1ha(const void *data, size_t length, uint64_t seed) {
+ return t1ha1_le(data, length, seed);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/contrib/t1ha/t1ha1.c b/contrib/t1ha/t1ha1.c
new file mode 100644
index 0000000..6e25d37
--- /dev/null
+++ b/contrib/t1ha/t1ha1.c
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2016-2018 Positive Technologies, https://www.ptsecurity.com,
+ * Fast Positive Hash.
+ *
+ * Portions Copyright (c) 2010-2018 Leonid Yuriev <leo@yuriev.ru>,
+ * The 1Hippeus project (t1h).
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty. In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ * claim that you wrote the original software. If you use this software
+ * in a product, an acknowledgement in the product documentation would be
+ * appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ * misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * t1ha = { Fast Positive Hash, aka "Позитивный Ð¥Ñш" }
+ * by [Positive Technologies](https://www.ptsecurity.ru)
+ *
+ * Briefly, it is a 64-bit Hash Function:
+ * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
+ * but portable and without penalties it can run on any 64-bit CPU.
+ * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
+ * and all others portable hash-functions (which do not use specific
+ * hardware tricks).
+ * 3. Not suitable for cryptography.
+ *
+ * The Future will Positive. Ð’ÑÑ‘ будет хорошо.
+ *
+ * ACKNOWLEDGEMENT:
+ * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
+ * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
+ */
+
+#include "config.h"
+#include "t1ha_bits.h"
+
+/* xor-mul-xor mixer */
+static __inline uint64_t mix64(uint64_t v, uint64_t p) {
+ v *= p;
+ return v ^ rot64(v, 41);
+}
+
+static __inline uint64_t final_weak_avalanche(uint64_t a, uint64_t b) {
+ /* LY: for performance reason on a some not high-end CPUs
+ * I replaced the second mux64() operation by mix64().
+ * Unfortunately this approach fails the "strict avalanche criteria",
+ * see test results at https://github.com/demerphq/smhasher. */
+ return mux64(rot64(a + b, 17), prime_4) + mix64(a ^ b, prime_0);
+}
+
+/* TODO: C++ template in the next version */
+#define T1HA1_BODY(ENDIANNES, ALIGNESS) \
+ const uint64_t *v = (const uint64_t *)data; \
+ if (unlikely(len > 32)) { \
+ uint64_t c = rot64(len, 17) + seed; \
+ uint64_t d = len ^ rot64(seed, 17); \
+ const uint64_t *detent = \
+ (const uint64_t *)((const uint8_t *)data + len - 31); \
+ do { \
+ const uint64_t w0 = fetch64_##ENDIANNES##_##ALIGNESS(v + 0); \
+ const uint64_t w1 = fetch64_##ENDIANNES##_##ALIGNESS(v + 1); \
+ const uint64_t w2 = fetch64_##ENDIANNES##_##ALIGNESS(v + 2); \
+ const uint64_t w3 = fetch64_##ENDIANNES##_##ALIGNESS(v + 3); \
+ v += 4; \
+ prefetch(v); \
+ \
+ const uint64_t d02 = w0 ^ rot64(w2 + d, 17); \
+ const uint64_t c13 = w1 ^ rot64(w3 + c, 17); \
+ c += a ^ rot64(w0, 41); \
+ d -= b ^ rot64(w1, 31); \
+ a ^= prime_1 * (d02 + w3); \
+ b ^= prime_0 * (c13 + w2); \
+ } while (likely(v < detent)); \
+ \
+ a ^= prime_6 * (rot64(c, 17) + d); \
+ b ^= prime_5 * (c + rot64(d, 17)); \
+ len &= 31; \
+ } \
+ \
+ switch (len) { \
+ default: \
+ b += mux64(fetch64_##ENDIANNES##_##ALIGNESS(v++), prime_4); \
+ /* fall through */ \
+ case 24: \
+ case 23: \
+ case 22: \
+ case 21: \
+ case 20: \
+ case 19: \
+ case 18: \
+ case 17: \
+ a += mux64(fetch64_##ENDIANNES##_##ALIGNESS(v++), prime_3); \
+ /* fall through */ \
+ case 16: \
+ case 15: \
+ case 14: \
+ case 13: \
+ case 12: \
+ case 11: \
+ case 10: \
+ case 9: \
+ b += mux64(fetch64_##ENDIANNES##_##ALIGNESS(v++), prime_2); \
+ /* fall through */ \
+ case 8: \
+ case 7: \
+ case 6: \
+ case 5: \
+ case 4: \
+ case 3: \
+ case 2: \
+ case 1: \
+ a += mux64(tail64_##ENDIANNES##_##ALIGNESS(v, len), prime_1); \
+ /* fall through */ \
+ case 0: \
+ return final_weak_avalanche(a, b); \
+ }
+
+uint64_t t1ha1_le(const void *data, size_t len, uint64_t seed) {
+ uint64_t a = seed;
+ uint64_t b = len;
+
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ T1HA1_BODY(le, unaligned);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ T1HA1_BODY(le, unaligned);
+ } else {
+ T1HA1_BODY(le, aligned);
+ }
+#endif
+}
+
+uint64_t t1ha1_be(const void *data, size_t len, uint64_t seed) {
+ uint64_t a = seed;
+ uint64_t b = len;
+
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ T1HA1_BODY(be, unaligned);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ T1HA1_BODY(be, unaligned);
+ } else {
+ T1HA1_BODY(be, aligned);
+ }
+#endif
+}
diff --git a/contrib/t1ha/t1ha2.c b/contrib/t1ha/t1ha2.c
new file mode 100644
index 0000000..4cb5281
--- /dev/null
+++ b/contrib/t1ha/t1ha2.c
@@ -0,0 +1,326 @@
+/*
+ * Copyright (c) 2016-2018 Positive Technologies, https://www.ptsecurity.com,
+ * Fast Positive Hash.
+ *
+ * Portions Copyright (c) 2010-2018 Leonid Yuriev <leo@yuriev.ru>,
+ * The 1Hippeus project (t1h).
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty. In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ * claim that you wrote the original software. If you use this software
+ * in a product, an acknowledgement in the product documentation would be
+ * appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ * misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * t1ha = { Fast Positive Hash, aka "Позитивный Ð¥Ñш" }
+ * by [Positive Technologies](https://www.ptsecurity.ru)
+ *
+ * Briefly, it is a 64-bit Hash Function:
+ * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
+ * but portable and without penalties it can run on any 64-bit CPU.
+ * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
+ * and all others portable hash-functions (which do not use specific
+ * hardware tricks).
+ * 3. Not suitable for cryptography.
+ *
+ * The Future will Positive. Ð’ÑÑ‘ будет хорошо.
+ *
+ * ACKNOWLEDGEMENT:
+ * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
+ * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
+ */
+
+#include "config.h"
+#include "t1ha_bits.h"
+
+static __always_inline void init_ab(t1ha_state256_t *s, uint64_t x,
+ uint64_t y) {
+ s->n.a = x;
+ s->n.b = y;
+}
+
+static __always_inline void init_cd(t1ha_state256_t *s, uint64_t x,
+ uint64_t y) {
+ s->n.c = rot64(y, 23) + ~x;
+ s->n.d = ~y + rot64(x, 19);
+}
+
+/* TODO: C++ template in the next version */
+#define T1HA2_UPDATE(ENDIANNES, ALIGNESS, state, v) \
+ do { \
+ t1ha_state256_t *const s = state; \
+ const uint64_t w0 = fetch64_##ENDIANNES##_##ALIGNESS(v + 0); \
+ const uint64_t w1 = fetch64_##ENDIANNES##_##ALIGNESS(v + 1); \
+ const uint64_t w2 = fetch64_##ENDIANNES##_##ALIGNESS(v + 2); \
+ const uint64_t w3 = fetch64_##ENDIANNES##_##ALIGNESS(v + 3); \
+ \
+ const uint64_t d02 = w0 + rot64(w2 + s->n.d, 56); \
+ const uint64_t c13 = w1 + rot64(w3 + s->n.c, 19); \
+ s->n.c ^= s->n.a + rot64(w0, 57); \
+ s->n.d ^= s->n.b + rot64(w1, 38); \
+ s->n.b ^= prime_6 * (c13 + w2); \
+ s->n.a ^= prime_5 * (d02 + w3); \
+ } while (0)
+
+static __always_inline void squash(t1ha_state256_t *s) {
+ s->n.a ^= prime_6 * (s->n.c + rot64(s->n.d, 23));
+ s->n.b ^= prime_5 * (rot64(s->n.c, 19) + s->n.d);
+}
+
+/* TODO: C++ template in the next version */
+#define T1HA2_LOOP(ENDIANNES, ALIGNESS, state, data, len) \
+ do { \
+ const void *detent = (const uint8_t *)data + len - 31; \
+ do { \
+ const uint64_t *v = (const uint64_t *)data; \
+ data = (const uint64_t *)data + 4; \
+ prefetch(data); \
+ T1HA2_UPDATE(le, ALIGNESS, state, v); \
+ } while (likely(data < detent)); \
+ } while (0)
+
+/* TODO: C++ template in the next version */
+#define T1HA2_TAIL_AB(ENDIANNES, ALIGNESS, state, data, len) \
+ do { \
+ t1ha_state256_t *const s = state; \
+ const uint64_t *v = (const uint64_t *)data; \
+ switch (len) { \
+ default: \
+ mixup64(&s->n.a, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_4); \
+ /* fall through */ \
+ case 24: \
+ case 23: \
+ case 22: \
+ case 21: \
+ case 20: \
+ case 19: \
+ case 18: \
+ case 17: \
+ mixup64(&s->n.b, &s->n.a, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_3); \
+ /* fall through */ \
+ case 16: \
+ case 15: \
+ case 14: \
+ case 13: \
+ case 12: \
+ case 11: \
+ case 10: \
+ case 9: \
+ mixup64(&s->n.a, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_2); \
+ /* fall through */ \
+ case 8: \
+ case 7: \
+ case 6: \
+ case 5: \
+ case 4: \
+ case 3: \
+ case 2: \
+ case 1: \
+ mixup64(&s->n.b, &s->n.a, tail64_##ENDIANNES##_##ALIGNESS(v, len), \
+ prime_1); \
+ /* fall through */ \
+ case 0: \
+ return final64(s->n.a, s->n.b); \
+ } \
+ } while (0)
+
+/* TODO: C++ template in the next version */
+#define T1HA2_TAIL_ABCD(ENDIANNES, ALIGNESS, state, data, len) \
+ do { \
+ t1ha_state256_t *const s = state; \
+ const uint64_t *v = (const uint64_t *)data; \
+ switch (len) { \
+ default: \
+ mixup64(&s->n.a, &s->n.d, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_4); \
+ /* fall through */ \
+ case 24: \
+ case 23: \
+ case 22: \
+ case 21: \
+ case 20: \
+ case 19: \
+ case 18: \
+ case 17: \
+ mixup64(&s->n.b, &s->n.a, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_3); \
+ /* fall through */ \
+ case 16: \
+ case 15: \
+ case 14: \
+ case 13: \
+ case 12: \
+ case 11: \
+ case 10: \
+ case 9: \
+ mixup64(&s->n.c, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
+ prime_2); \
+ /* fall through */ \
+ case 8: \
+ case 7: \
+ case 6: \
+ case 5: \
+ case 4: \
+ case 3: \
+ case 2: \
+ case 1: \
+ mixup64(&s->n.d, &s->n.c, tail64_##ENDIANNES##_##ALIGNESS(v, len), \
+ prime_1); \
+ /* fall through */ \
+ case 0: \
+ return final128(s->n.a, s->n.b, s->n.c, s->n.d, extra_result); \
+ } \
+ } while (0)
+
+static __always_inline uint64_t final128(uint64_t a, uint64_t b, uint64_t c,
+ uint64_t d, uint64_t *h) {
+ mixup64(&a, &b, rot64(c, 41) ^ d, prime_0);
+ mixup64(&b, &c, rot64(d, 23) ^ a, prime_6);
+ mixup64(&c, &d, rot64(a, 19) ^ b, prime_5);
+ mixup64(&d, &a, rot64(b, 31) ^ c, prime_4);
+ *h = c + d;
+ return a ^ b;
+}
+
+//------------------------------------------------------------------------------
+
+uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed) {
+ t1ha_state256_t state;
+ init_ab(&state, seed, length);
+
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ if (unlikely(length > 32)) {
+ init_cd(&state, seed, length);
+ T1HA2_LOOP(le, unaligned, &state, data, length);
+ squash(&state);
+ length &= 31;
+ }
+ T1HA2_TAIL_AB(le, unaligned, &state, data, length);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ if (unlikely(length > 32)) {
+ init_cd(&state, seed, length);
+ T1HA2_LOOP(le, unaligned, &state, data, length);
+ squash(&state);
+ length &= 31;
+ }
+ T1HA2_TAIL_AB(le, unaligned, &state, data, length);
+ } else {
+ if (unlikely(length > 32)) {
+ init_cd(&state, seed, length);
+ T1HA2_LOOP(le, aligned, &state, data, length);
+ squash(&state);
+ length &= 31;
+ }
+ T1HA2_TAIL_AB(le, aligned, &state, data, length);
+ }
+#endif
+}
+
+uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result,
+ const void *__restrict data, size_t length,
+ uint64_t seed) {
+ t1ha_state256_t state;
+ init_ab(&state, seed, length);
+ init_cd(&state, seed, length);
+
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ if (unlikely(length > 32)) {
+ T1HA2_LOOP(le, unaligned, &state, data, length);
+ length &= 31;
+ }
+ T1HA2_TAIL_ABCD(le, unaligned, &state, data, length);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ if (unlikely(length > 32)) {
+ T1HA2_LOOP(le, unaligned, &state, data, length);
+ length &= 31;
+ }
+ T1HA2_TAIL_ABCD(le, unaligned, &state, data, length);
+ } else {
+ if (unlikely(length > 32)) {
+ T1HA2_LOOP(le, aligned, &state, data, length);
+ length &= 31;
+ }
+ T1HA2_TAIL_ABCD(le, aligned, &state, data, length);
+ }
+#endif
+}
+
+//------------------------------------------------------------------------------
+
+void t1ha2_init(t1ha_context_t *ctx, uint64_t seed_x, uint64_t seed_y) {
+ init_ab(&ctx->state, seed_x, seed_y);
+ init_cd(&ctx->state, seed_x, seed_y);
+ ctx->partial = 0;
+ ctx->total = 0;
+}
+
+void t1ha2_update(t1ha_context_t *__restrict ctx, const void *__restrict data,
+ size_t length) {
+ ctx->total += length;
+
+ if (ctx->partial) {
+ const size_t left = 32 - ctx->partial;
+ const size_t chunk = (length >= left) ? left : length;
+ memcpy(ctx->buffer.bytes + ctx->partial, data, chunk);
+ ctx->partial += chunk;
+ if (ctx->partial < 32) {
+ assert(left >= length);
+ return;
+ }
+ ctx->partial = 0;
+ data = (const uint8_t *)data + chunk;
+ length -= chunk;
+ T1HA2_UPDATE(le, aligned, &ctx->state, ctx->buffer.u64);
+ }
+
+ if (length >= 32) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+ T1HA2_LOOP(le, unaligned, &ctx->state, data, length);
+#else
+ const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+ if (misaligned) {
+ T1HA2_LOOP(le, unaligned, &ctx->state, data, length);
+ } else {
+ T1HA2_LOOP(le, aligned, &ctx->state, data, length);
+ }
+#endif
+ length &= 31;
+ }
+
+ if (length)
+ memcpy(ctx->buffer.bytes, data, ctx->partial = length);
+}
+
+uint64_t t1ha2_final(t1ha_context_t *__restrict ctx,
+ uint64_t *__restrict extra_result) {
+ uint64_t bits = (ctx->total << 3) ^ (UINT64_C(1) << 63);
+#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
+ bits = bswap64(bits);
+#endif
+ t1ha2_update(ctx, &bits, 8);
+
+ if (likely(!extra_result)) {
+ squash(&ctx->state);
+ T1HA2_TAIL_AB(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial);
+ }
+
+ T1HA2_TAIL_ABCD(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial);
+}
diff --git a/contrib/t1ha/t1ha_bits.h b/contrib/t1ha/t1ha_bits.h
new file mode 100644
index 0000000..5710d2d
--- /dev/null
+++ b/contrib/t1ha/t1ha_bits.h
@@ -0,0 +1,1171 @@
+/*
+ * Copyright (c) 2016-2018 Positive Technologies, https://www.ptsecurity.com,
+ * Fast Positive Hash.
+ *
+ * Portions Copyright (c) 2010-2018 Leonid Yuriev <leo@yuriev.ru>,
+ * The 1Hippeus project (t1h).
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty. In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ * claim that you wrote the original software. If you use this software
+ * in a product, an acknowledgement in the product documentation would be
+ * appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ * misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * t1ha = { Fast Positive Hash, aka "Позитивный Ð¥Ñш" }
+ * by [Positive Technologies](https://www.ptsecurity.ru)
+ *
+ * Briefly, it is a 64-bit Hash Function:
+ * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
+ * but portable and without penalties it can run on any 64-bit CPU.
+ * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
+ * and all others portable hash-functions (which do not use specific
+ * hardware tricks).
+ * 3. Not suitable for cryptography.
+ *
+ * The Future will Positive. Ð’ÑÑ‘ будет хорошо.
+ *
+ * ACKNOWLEDGEMENT:
+ * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
+ * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
+ */
+
+#pragma once
+
+#if defined(_MSC_VER)
+#pragma warning(disable : 4201) /* nameless struct/union */
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif /* 1800 */
+#endif /* MSVC */
+
+#include "config.h"
+#include "t1ha.h"
+
+#ifndef T1HA_USE_FAST_ONESHOT_READ
+/* Define it to 1 for little bit faster code.
+ * Unfortunately this may triggering a false-positive alarms from Valgrind,
+ * AddressSanitizer and other similar tool.
+ * So, define it to 0 for calmness if doubt. */
+#define T1HA_USE_FAST_ONESHOT_READ 1
+#endif /* T1HA_USE_FAST_ONESHOT_READ */
+
+/*****************************************************************************/
+
+#include <assert.h> /* for assert() */
+#include <stdbool.h> /* for bool */
+#include <string.h> /* for memcpy() */
+
+#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \
+ __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__
+#error Unsupported byte order.
+#endif
+
+#define T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE 0
+#define T1HA_CONFIG_UNALIGNED_ACCESS__SLOW 1
+#define T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT 2
+
+#ifndef T1HA_CONFIG_UNALIGNED_ACCESS
+#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+#elif defined(__ia32__)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+#elif defined(__e2k__)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__SLOW
+#elif defined(__ARM_FEATURE_UNALIGNED)
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT
+#else
+#define T1HA_CONFIG_UNALIGNED_ACCESS T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+#endif
+#endif /* T1HA_CONFIG_UNALIGNED_ACCESS */
+
+#define ALIGNMENT_16 2
+#define ALIGNMENT_32 4
+#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
+#define ALIGNMENT_64 8
+#else
+#define ALIGNMENT_64 4
+#endif
+
+#ifndef PAGESIZE
+#define PAGESIZE 4096
+#endif /* PAGESIZE */
+
+/***************************************************************************/
+
+#ifndef __has_builtin
+#define __has_builtin(x) (0)
+#endif
+
+#ifndef __has_warning
+#define __has_warning(x) (0)
+#endif
+
+#ifndef __has_feature
+#define __has_feature(x) (0)
+#endif
+
+#ifndef __has_extension
+#define __has_extension(x) (0)
+#endif
+
+#if __GNUC_PREREQ(4, 4) || defined(__clang__)
+
+#if defined(__ia32__) || defined(__e2k__)
+#include <x86intrin.h>
+#endif
+
+#if defined(__ia32__) && !defined(__cpuid_count)
+#include <cpuid.h>
+#endif
+
+#if defined(__e2k__)
+#include <e2kbuiltin.h>
+#endif
+
+#ifndef likely
+#define likely(cond) __builtin_expect(!!(cond), 1)
+#endif
+
+#ifndef unlikely
+#define unlikely(cond) __builtin_expect(!!(cond), 0)
+#endif
+
+#if __GNUC_PREREQ(4, 5) || __has_builtin(__builtin_unreachable)
+#define unreachable() __builtin_unreachable()
+#endif
+
+#define bswap64(v) __builtin_bswap64(v)
+#define bswap32(v) __builtin_bswap32(v)
+#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)
+#define bswap16(v) __builtin_bswap16(v)
+#endif
+
+#if !defined(__maybe_unused) && (__GNUC_PREREQ(4, 3) || __has_attribute(unused))
+#define __maybe_unused __attribute__((unused))
+#endif
+
+#if !defined(__always_inline) && \
+ (__GNUC_PREREQ(3, 2) || __has_attribute(always_inline))
+#define __always_inline __inline __attribute__((always_inline))
+#endif
+
+#if defined(__e2k__)
+
+#if __iset__ >= 3
+#define mul_64x64_high(a, b) __builtin_e2k_umulhd(a, b)
+#endif /* __iset__ >= 3 */
+
+#if __iset__ >= 5
+static __maybe_unused __always_inline unsigned
+e2k_add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) {
+ *sum = base + addend;
+ return (unsigned)__builtin_e2k_addcd_c(base, addend, 0);
+}
+#define add64carry_first(base, addend, sum) \
+ e2k_add64carry_first(base, addend, sum)
+
+static __maybe_unused __always_inline unsigned
+e2k_add64carry_next(unsigned carry, uint64_t base, uint64_t addend,
+ uint64_t *sum) {
+ *sum = __builtin_e2k_addcd(base, addend, carry);
+ return (unsigned)__builtin_e2k_addcd_c(base, addend, carry);
+}
+#define add64carry_next(carry, base, addend, sum) \
+ e2k_add64carry_next(carry, base, addend, sum)
+
+static __maybe_unused __always_inline void e2k_add64carry_last(unsigned carry,
+ uint64_t base,
+ uint64_t addend,
+ uint64_t *sum) {
+ *sum = __builtin_e2k_addcd(base, addend, carry);
+}
+#define add64carry_last(carry, base, addend, sum) \
+ e2k_add64carry_last(carry, base, addend, sum)
+#endif /* __iset__ >= 5 */
+
+#define fetch64_be_aligned(ptr) ((uint64_t)__builtin_e2k_ld_64s_be(ptr))
+#define fetch32_be_aligned(ptr) ((uint32_t)__builtin_e2k_ld_32u_be(ptr))
+
+#endif /* __e2k__ Elbrus */
+
+#elif defined(_MSC_VER)
+
+#if _MSC_FULL_VER < 190024218 && defined(_M_IX86)
+#pragma message( \
+ "For AES-NI at least \"Microsoft C/C++ Compiler\" version 19.00.24218 (Visual Studio 2015 Update 5) is required.")
+#endif
+#if _MSC_FULL_VER < 191025019
+#pragma message( \
+ "It is recommended to use \"Microsoft C/C++ Compiler\" version 19.10.25019 (Visual Studio 2017) or newer.")
+#endif
+#if _MSC_FULL_VER < 180040629
+#error At least "Microsoft C/C++ Compiler" version 18.00.40629 (Visual Studio 2013 Update 5) is required.
+#endif
+
+#pragma warning(push, 1)
+
+#include <intrin.h>
+#include <stdlib.h>
+#define likely(cond) (cond)
+#define unlikely(cond) (cond)
+#define unreachable() __assume(0)
+#define bswap64(v) _byteswap_uint64(v)
+#define bswap32(v) _byteswap_ulong(v)
+#define bswap16(v) _byteswap_ushort(v)
+#define rot64(v, s) _rotr64(v, s)
+#define rot32(v, s) _rotr(v, s)
+#define __always_inline __forceinline
+
+#if defined(_M_X64) || defined(_M_IA64)
+#pragma intrinsic(_umul128)
+#define mul_64x64_128(a, b, ph) _umul128(a, b, ph)
+#pragma intrinsic(_addcarry_u64)
+#define add64carry_first(base, addend, sum) _addcarry_u64(0, base, addend, sum)
+#define add64carry_next(carry, base, addend, sum) \
+ _addcarry_u64(carry, base, addend, sum)
+#define add64carry_last(carry, base, addend, sum) \
+ (void)_addcarry_u64(carry, base, addend, sum)
+#endif
+
+#if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
+#pragma intrinsic(__umulh)
+#define mul_64x64_high(a, b) __umulh(a, b)
+#endif
+
+#if defined(_M_IX86)
+#pragma intrinsic(__emulu)
+#define mul_32x32_64(a, b) __emulu(a, b)
+
+#if _MSC_FULL_VER >= 190024231 /* LY: workaround for optimizer bug */
+#pragma intrinsic(_addcarry_u32)
+#define add32carry_first(base, addend, sum) _addcarry_u32(0, base, addend, sum)
+#define add32carry_next(carry, base, addend, sum) \
+ _addcarry_u32(carry, base, addend, sum)
+#define add32carry_last(carry, base, addend, sum) \
+ (void)_addcarry_u32(carry, base, addend, sum)
+
+static __forceinline char
+msvc32_add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) {
+ uint32_t *const sum32 = (uint32_t *)sum;
+ const uint32_t base_32l = (uint32_t)base;
+ const uint32_t base_32h = (uint32_t)(base >> 32);
+ const uint32_t addend_32l = (uint32_t)addend;
+ const uint32_t addend_32h = (uint32_t)(addend >> 32);
+ return add32carry_next(add32carry_first(base_32l, addend_32l, sum32),
+ base_32h, addend_32h, sum32 + 1);
+}
+#define add64carry_first(base, addend, sum) \
+ msvc32_add64carry_first(base, addend, sum)
+
+static __forceinline char msvc32_add64carry_next(char carry, uint64_t base,
+ uint64_t addend,
+ uint64_t *sum) {
+ uint32_t *const sum32 = (uint32_t *)sum;
+ const uint32_t base_32l = (uint32_t)base;
+ const uint32_t base_32h = (uint32_t)(base >> 32);
+ const uint32_t addend_32l = (uint32_t)addend;
+ const uint32_t addend_32h = (uint32_t)(addend >> 32);
+ return add32carry_next(add32carry_next(carry, base_32l, addend_32l, sum32),
+ base_32h, addend_32h, sum32 + 1);
+}
+#define add64carry_next(carry, base, addend, sum) \
+ msvc32_add64carry_next(carry, base, addend, sum)
+
+static __forceinline void msvc32_add64carry_last(char carry, uint64_t base,
+ uint64_t addend,
+ uint64_t *sum) {
+ uint32_t *const sum32 = (uint32_t *)sum;
+ const uint32_t base_32l = (uint32_t)base;
+ const uint32_t base_32h = (uint32_t)(base >> 32);
+ const uint32_t addend_32l = (uint32_t)addend;
+ const uint32_t addend_32h = (uint32_t)(addend >> 32);
+ add32carry_last(add32carry_next(carry, base_32l, addend_32l, sum32), base_32h,
+ addend_32h, sum32 + 1);
+}
+#define add64carry_last(carry, base, addend, sum) \
+ msvc32_add64carry_last(carry, base, addend, sum)
+#endif /* _MSC_FULL_VER >= 190024231 */
+
+#elif defined(_M_ARM)
+#define mul_32x32_64(a, b) _arm_umull(a, b)
+#endif
+
+#pragma warning(pop)
+#pragma warning(disable : 4514) /* 'xyz': unreferenced inline function \
+ has been removed */
+#pragma warning(disable : 4710) /* 'xyz': function not inlined */
+#pragma warning(disable : 4711) /* function 'xyz' selected for \
+ automatic inline expansion */
+#pragma warning(disable : 4127) /* conditional expression is constant */
+#pragma warning(disable : 4702) /* unreachable code */
+#endif /* Compiler */
+
+#ifndef likely
+#define likely(cond) (cond)
+#endif
+#ifndef unlikely
+#define unlikely(cond) (cond)
+#endif
+#ifndef __maybe_unused
+#define __maybe_unused
+#endif
+#ifndef __always_inline
+#define __always_inline __inline
+#endif
+#ifndef unreachable
+#define unreachable() \
+ do { \
+ } while (1)
+#endif
+
+#ifndef bswap64
+#if defined(bswap_64)
+#define bswap64 bswap_64
+#elif defined(__bswap_64)
+#define bswap64 __bswap_64
+#else
+static __always_inline uint64_t bswap64(uint64_t v) {
+ return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) |
+ ((v << 24) & UINT64_C(0x0000ff0000000000)) |
+ ((v << 8) & UINT64_C(0x000000ff00000000)) |
+ ((v >> 8) & UINT64_C(0x00000000ff000000)) |
+ ((v >> 24) & UINT64_C(0x0000000000ff0000)) |
+ ((v >> 40) & UINT64_C(0x000000000000ff00));
+}
+#endif
+#endif /* bswap64 */
+
+#ifndef bswap32
+#if defined(bswap_32)
+#define bswap32 bswap_32
+#elif defined(__bswap_32)
+#define bswap32 __bswap_32
+#else
+static __always_inline uint32_t bswap32(uint32_t v) {
+ return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) |
+ ((v >> 8) & UINT32_C(0x0000ff00));
+}
+#endif
+#endif /* bswap32 */
+
+#ifndef bswap16
+#if defined(bswap_16)
+#define bswap16 bswap_16
+#elif defined(__bswap_16)
+#define bswap16 __bswap_16
+#else
+static __always_inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; }
+#endif
+#endif /* bswap16 */
+
+#ifndef read_unaligned
+#if defined(__GNUC__) || __has_attribute(packed)
+typedef struct {
+ uint8_t unaligned_8;
+ uint16_t unaligned_16;
+ uint32_t unaligned_32;
+ uint64_t unaligned_64;
+} __attribute__((packed)) t1ha_unaligned_proxy;
+#define read_unaligned(ptr, bits) \
+ (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \
+ t1ha_unaligned_proxy, unaligned_##bits))) \
+ ->unaligned_##bits)
+#elif defined(_MSC_VER)
+#pragma warning( \
+ disable : 4235) /* nonstandard extension used: '__unaligned' \
+ * keyword not supported on this architecture */
+#define read_unaligned(ptr, bits) (*(const __unaligned uint##bits##_t *)(ptr))
+#else
+#pragma pack(push, 1)
+typedef struct {
+ uint8_t unaligned_8;
+ uint16_t unaligned_16;
+ uint32_t unaligned_32;
+ uint64_t unaligned_64;
+} t1ha_unaligned_proxy;
+#pragma pack(pop)
+#define read_unaligned(ptr, bits) \
+ (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \
+ t1ha_unaligned_proxy, unaligned_##bits))) \
+ ->unaligned_##bits)
+#endif
+#endif /* read_unaligned */
+
+#ifndef read_aligned
+#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_assume_aligned)
+#define read_aligned(ptr, bits) \
+ (*(const uint##bits##_t *)__builtin_assume_aligned(ptr, ALIGNMENT_##bits))
+#elif (__GNUC_PREREQ(3, 3) || __has_attribute(aligned)) && !defined(__clang__)
+#define read_aligned(ptr, bits) \
+ (*(const uint##bits##_t __attribute__((aligned(ALIGNMENT_##bits))) *)(ptr))
+#elif __has_attribute(assume_aligned)
+
+static __always_inline const
+ uint16_t *__attribute__((assume_aligned(ALIGNMENT_16)))
+ cast_aligned_16(const void *ptr) {
+ return (const uint16_t *)ptr;
+}
+static __always_inline const
+ uint32_t *__attribute__((assume_aligned(ALIGNMENT_32)))
+ cast_aligned_32(const void *ptr) {
+ return (const uint32_t *)ptr;
+}
+static __always_inline const
+ uint64_t *__attribute__((assume_aligned(ALIGNMENT_64)))
+ cast_aligned_64(const void *ptr) {
+ return (const uint64_t *)ptr;
+}
+
+#define read_aligned(ptr, bits) (*cast_aligned_##bits(ptr))
+
+#elif defined(_MSC_VER)
+#define read_aligned(ptr, bits) \
+ (*(const __declspec(align(ALIGNMENT_##bits)) uint##bits##_t *)(ptr))
+#else
+#define read_aligned(ptr, bits) (*(const uint##bits##_t *)(ptr))
+#endif
+#endif /* read_aligned */
+
+#ifndef prefetch
+#if (__GNUC_PREREQ(4, 0) || __has_builtin(__builtin_prefetch)) && \
+ !defined(__ia32__)
+#define prefetch(ptr) __builtin_prefetch(ptr)
+#elif defined(_M_ARM64) || defined(_M_ARM)
+#define prefetch(ptr) __prefetch(ptr)
+#else
+#define prefetch(ptr) \
+ do { \
+ (void)(ptr); \
+ } while (0)
+#endif
+#endif /* prefetch */
+
+#if __has_warning("-Wconstant-logical-operand")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wconstant-logical-operand"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wconstant-logical-operand"
+#else
+#pragma warning disable "constant-logical-operand"
+#endif
+#endif /* -Wconstant-logical-operand */
+
+#if __has_warning("-Wtautological-pointer-compare")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wtautological-pointer-compare"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wtautological-pointer-compare"
+#else
+#pragma warning disable "tautological-pointer-compare"
+#endif
+#endif /* -Wtautological-pointer-compare */
+
+/***************************************************************************/
+
+/*---------------------------------------------------------- Little Endian */
+
+#ifndef fetch16_le_aligned
+static __always_inline uint16_t fetch16_le_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_16 == 0);
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_aligned(v, 16);
+#else
+ return bswap16(read_aligned(v, 16));
+#endif
+}
+#endif /* fetch16_le_aligned */
+
+#ifndef fetch16_le_unaligned
+static __always_inline uint16_t fetch16_le_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ const uint8_t *p = (const uint8_t *)v;
+ return p[0] | (uint16_t)p[1] << 8;
+#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_unaligned(v, 16);
+#else
+ return bswap16(read_unaligned(v, 16));
+#endif
+}
+#endif /* fetch16_le_unaligned */
+
+#ifndef fetch32_le_aligned
+static __always_inline uint32_t fetch32_le_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_32 == 0);
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_aligned(v, 32);
+#else
+ return bswap32(read_aligned(v, 32));
+#endif
+}
+#endif /* fetch32_le_aligned */
+
+#ifndef fetch32_le_unaligned
+static __always_inline uint32_t fetch32_le_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return fetch16_le_unaligned(v) |
+ (uint32_t)fetch16_le_unaligned((const uint8_t *)v + 2) << 16;
+#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_unaligned(v, 32);
+#else
+ return bswap32(read_unaligned(v, 32));
+#endif
+}
+#endif /* fetch32_le_unaligned */
+
+#ifndef fetch64_le_aligned
+static __always_inline uint64_t fetch64_le_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_64 == 0);
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_aligned(v, 64);
+#else
+ return bswap64(read_aligned(v, 64));
+#endif
+}
+#endif /* fetch64_le_aligned */
+
+#ifndef fetch64_le_unaligned
+static __always_inline uint64_t fetch64_le_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return fetch32_le_unaligned(v) |
+ (uint64_t)fetch32_le_unaligned((const uint8_t *)v + 4) << 32;
+#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ return read_unaligned(v, 64);
+#else
+ return bswap64(read_unaligned(v, 64));
+#endif
+}
+#endif /* fetch64_le_unaligned */
+
+static __always_inline uint64_t tail64_le_aligned(const void *v, size_t tail) {
+ const uint8_t *const p = (const uint8_t *)v;
+#if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__)
+ /* We can perform a 'oneshot' read, which is little bit faster. */
+ const unsigned shift = ((8 - tail) & 7) << 3;
+ return fetch64_le_aligned(p) & ((~UINT64_C(0)) >> shift);
+#else
+ uint64_t r = 0;
+ switch (tail & 7) {
+ default:
+ unreachable();
+/* fall through */
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ /* For most CPUs this code is better when not needed byte reordering. */
+ case 0:
+ return fetch64_le_aligned(p);
+ case 7:
+ r = (uint64_t)p[6] << 8;
+ /* fall through */
+ case 6:
+ r += p[5];
+ r <<= 8;
+ /* fall through */
+ case 5:
+ r += p[4];
+ r <<= 32;
+ /* fall through */
+ case 4:
+ return r + fetch32_le_aligned(p);
+ case 3:
+ r = (uint64_t)p[2] << 16;
+ /* fall through */
+ case 2:
+ return r + fetch16_le_aligned(p);
+ case 1:
+ return p[0];
+#else
+ case 0:
+ r = p[7] << 8;
+ /* fall through */
+ case 7:
+ r += p[6];
+ r <<= 8;
+ /* fall through */
+ case 6:
+ r += p[5];
+ r <<= 8;
+ /* fall through */
+ case 5:
+ r += p[4];
+ r <<= 8;
+ /* fall through */
+ case 4:
+ r += p[3];
+ r <<= 8;
+ /* fall through */
+ case 3:
+ r += p[2];
+ r <<= 8;
+ /* fall through */
+ case 2:
+ r += p[1];
+ r <<= 8;
+ /* fall through */
+ case 1:
+ return r + p[0];
+#endif
+ }
+#endif /* T1HA_USE_FAST_ONESHOT_READ */
+}
+
+#if T1HA_USE_FAST_ONESHOT_READ && \
+ T1HA_CONFIG_UNALIGNED_ACCESS != T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE && \
+ defined(PAGESIZE) && !defined(__sun) && !defined(__SANITIZE_ADDRESS__)
+#define can_read_underside(ptr, size) \
+ ((size) <= sizeof(uintptr_t) && ((PAGESIZE - (size)) & (uintptr_t)(ptr)) != 0)
+#endif /* T1HA_USE_FAST_ONESHOT_READ */
+
+static __always_inline uint64_t tail64_le_unaligned(const void *v,
+ size_t tail) {
+ const uint8_t *p = (const uint8_t *)v;
+#ifdef can_read_underside
+ /* On some systems (e.g. x86) we can perform a 'oneshot' read, which
+ * is little bit faster. Thanks Marcin Żukowski <marcin.zukowski@gmail.com>
+ * for the reminder. */
+ const unsigned offset = (8 - tail) & 7;
+ const unsigned shift = offset << 3;
+ if (likely(can_read_underside(p, 8))) {
+ p -= offset;
+ return fetch64_le_unaligned(p) >> shift;
+ }
+ return fetch64_le_unaligned(p) & ((~UINT64_C(0)) >> shift);
+#else
+ uint64_t r = 0;
+ switch (tail & 7) {
+ default:
+ unreachable();
+/* fall through */
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT && \
+ __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ /* For most CPUs this code is better when not needed
+ * copying for alignment or byte reordering. */
+ case 0:
+ return fetch64_le_unaligned(p);
+ case 7:
+ r = (uint64_t)p[6] << 8;
+ /* fall through */
+ case 6:
+ r += p[5];
+ r <<= 8;
+ /* fall through */
+ case 5:
+ r += p[4];
+ r <<= 32;
+ /* fall through */
+ case 4:
+ return r + fetch32_le_unaligned(p);
+ case 3:
+ r = (uint64_t)p[2] << 16;
+ /* fall through */
+ case 2:
+ return r + fetch16_le_unaligned(p);
+ case 1:
+ return p[0];
+#else
+ /* For most CPUs this code is better than a
+ * copying for alignment and/or byte reordering. */
+ case 0:
+ r = p[7] << 8;
+ /* fall through */
+ case 7:
+ r += p[6];
+ r <<= 8;
+ /* fall through */
+ case 6:
+ r += p[5];
+ r <<= 8;
+ /* fall through */
+ case 5:
+ r += p[4];
+ r <<= 8;
+ /* fall through */
+ case 4:
+ r += p[3];
+ r <<= 8;
+ /* fall through */
+ case 3:
+ r += p[2];
+ r <<= 8;
+ /* fall through */
+ case 2:
+ r += p[1];
+ r <<= 8;
+ /* fall through */
+ case 1:
+ return r + p[0];
+#endif
+ }
+#endif /* can_read_underside */
+}
+
+/*------------------------------------------------------------- Big Endian */
+
+#ifndef fetch16_be_aligned
+static __maybe_unused __always_inline uint16_t
+fetch16_be_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_16 == 0);
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_aligned(v, 16);
+#else
+ return bswap16(read_aligned(v, 16));
+#endif
+}
+#endif /* fetch16_be_aligned */
+
+#ifndef fetch16_be_unaligned
+static __maybe_unused __always_inline uint16_t
+fetch16_be_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ const uint8_t *p = (const uint8_t *)v;
+ return (uint16_t)p[0] << 8 | p[1];
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_unaligned(v, 16);
+#else
+ return bswap16(read_unaligned(v, 16));
+#endif
+}
+#endif /* fetch16_be_unaligned */
+
+#ifndef fetch32_be_aligned
+static __maybe_unused __always_inline uint32_t
+fetch32_be_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_32 == 0);
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_aligned(v, 32);
+#else
+ return bswap32(read_aligned(v, 32));
+#endif
+}
+#endif /* fetch32_be_aligned */
+
+#ifndef fetch32_be_unaligned
+static __maybe_unused __always_inline uint32_t
+fetch32_be_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return (uint32_t)fetch16_be_unaligned(v) << 16 |
+ fetch16_be_unaligned((const uint8_t *)v + 2);
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_unaligned(v, 32);
+#else
+ return bswap32(read_unaligned(v, 32));
+#endif
+}
+#endif /* fetch32_be_unaligned */
+
+#ifndef fetch64_be_aligned
+static __maybe_unused __always_inline uint64_t
+fetch64_be_aligned(const void *v) {
+ assert(((uintptr_t)v) % ALIGNMENT_64 == 0);
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_aligned(v, 64);
+#else
+ return bswap64(read_aligned(v, 64));
+#endif
+}
+#endif /* fetch64_be_aligned */
+
+#ifndef fetch64_be_unaligned
+static __maybe_unused __always_inline uint64_t
+fetch64_be_unaligned(const void *v) {
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__UNABLE
+ return (uint64_t)fetch32_be_unaligned(v) << 32 |
+ fetch32_be_unaligned((const uint8_t *)v + 4);
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ return read_unaligned(v, 64);
+#else
+ return bswap64(read_unaligned(v, 64));
+#endif
+}
+#endif /* fetch64_be_unaligned */
+
+static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v,
+ size_t tail) {
+ const uint8_t *const p = (const uint8_t *)v;
+#if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__)
+ /* We can perform a 'oneshot' read, which is little bit faster. */
+ const unsigned shift = ((8 - tail) & 7) << 3;
+ return fetch64_be_aligned(p) >> shift;
+#else
+ switch (tail & 7) {
+ default:
+ unreachable();
+/* fall through */
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ /* For most CPUs this code is better when not byte reordering. */
+ case 1:
+ return p[0];
+ case 2:
+ return fetch16_be_aligned(p);
+ case 3:
+ return (uint32_t)fetch16_be_aligned(p) << 8 | p[2];
+ case 4:
+ return fetch32_be_aligned(p);
+ case 5:
+ return (uint64_t)fetch32_be_aligned(p) << 8 | p[4];
+ case 6:
+ return (uint64_t)fetch32_be_aligned(p) << 16 | fetch16_be_aligned(p + 4);
+ case 7:
+ return (uint64_t)fetch32_be_aligned(p) << 24 |
+ (uint32_t)fetch16_be_aligned(p + 4) << 8 | p[6];
+ case 0:
+ return fetch64_be_aligned(p);
+#else
+ case 1:
+ return p[0];
+ case 2:
+ return p[1] | (uint32_t)p[0] << 8;
+ case 3:
+ return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16;
+ case 4:
+ return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 |
+ (uint32_t)p[0] << 24;
+ case 5:
+ return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 |
+ (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32;
+ case 6:
+ return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 |
+ (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40;
+ case 7:
+ return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 |
+ (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | (uint64_t)p[1] << 40 |
+ (uint64_t)p[0] << 48;
+ case 0:
+ return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 |
+ (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | (uint64_t)p[2] << 40 |
+ (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56;
+#endif
+ }
+#endif /* T1HA_USE_FAST_ONESHOT_READ */
+}
+
+static __maybe_unused __always_inline uint64_t
+tail64_be_unaligned(const void *v, size_t tail) {
+ const uint8_t *p = (const uint8_t *)v;
+#ifdef can_read_underside
+ /* On some systems we can perform a 'oneshot' read, which is little bit
+ * faster. Thanks Marcin Żukowski <marcin.zukowski@gmail.com> for the
+ * reminder. */
+ const unsigned offset = (8 - tail) & 7;
+ const unsigned shift = offset << 3;
+ if (likely(can_read_underside(p, 8))) {
+ p -= offset;
+ return fetch64_be_unaligned(p) & ((~UINT64_C(0)) >> shift);
+ }
+ return fetch64_be_unaligned(p) >> shift;
+#else
+ switch (tail & 7) {
+ default:
+ unreachable();
+/* fall through */
+#if T1HA_CONFIG_UNALIGNED_ACCESS == T1HA_CONFIG_UNALIGNED_ACCESS__EFFICIENT && \
+ __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ /* For most CPUs this code is better when not needed
+ * copying for alignment or byte reordering. */
+ case 1:
+ return p[0];
+ case 2:
+ return fetch16_be_unaligned(p);
+ case 3:
+ return (uint32_t)fetch16_be_unaligned(p) << 8 | p[2];
+ case 4:
+ return fetch32_be(p);
+ case 5:
+ return (uint64_t)fetch32_be_unaligned(p) << 8 | p[4];
+ case 6:
+ return (uint64_t)fetch32_be_unaligned(p) << 16 |
+ fetch16_be_unaligned(p + 4);
+ case 7:
+ return (uint64_t)fetch32_be_unaligned(p) << 24 |
+ (uint32_t)fetch16_be_unaligned(p + 4) << 8 | p[6];
+ case 0:
+ return fetch64_be_unaligned(p);
+#else
+ /* For most CPUs this code is better than a
+ * copying for alignment and/or byte reordering. */
+ case 1:
+ return p[0];
+ case 2:
+ return p[1] | (uint32_t)p[0] << 8;
+ case 3:
+ return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16;
+ case 4:
+ return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 |
+ (uint32_t)p[0] << 24;
+ case 5:
+ return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 |
+ (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32;
+ case 6:
+ return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 |
+ (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40;
+ case 7:
+ return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 |
+ (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | (uint64_t)p[1] << 40 |
+ (uint64_t)p[0] << 48;
+ case 0:
+ return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 |
+ (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | (uint64_t)p[2] << 40 |
+ (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56;
+#endif
+ }
+#endif /* can_read_underside */
+}
+
+/***************************************************************************/
+
+#ifndef rot64
+static __always_inline uint64_t rot64(uint64_t v, unsigned s) {
+ return (v >> s) | (v << (64 - s));
+}
+#endif /* rot64 */
+
+#ifndef mul_32x32_64
+static __always_inline uint64_t mul_32x32_64(uint32_t a, uint32_t b) {
+ return a * (uint64_t)b;
+}
+#endif /* mul_32x32_64 */
+
+#ifndef add64carry_first
+static __maybe_unused __always_inline unsigned
+add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) {
+#if __has_builtin(__builtin_addcll)
+ unsigned long long carryout;
+ *sum = __builtin_addcll(base, addend, 0, &carryout);
+ return (unsigned)carryout;
+#else
+ *sum = base + addend;
+ return *sum < addend;
+#endif /* __has_builtin(__builtin_addcll) */
+}
+#endif /* add64carry_fist */
+
+#ifndef add64carry_next
+static __maybe_unused __always_inline unsigned
+add64carry_next(unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) {
+#if __has_builtin(__builtin_addcll)
+ unsigned long long carryout;
+ *sum = __builtin_addcll(base, addend, carry, &carryout);
+ return (unsigned)carryout;
+#else
+ *sum = base + addend + carry;
+ return *sum < addend || (carry && *sum == addend);
+#endif /* __has_builtin(__builtin_addcll) */
+}
+#endif /* add64carry_next */
+
+#ifndef add64carry_last
+static __maybe_unused __always_inline void
+add64carry_last(unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) {
+#if __has_builtin(__builtin_addcll)
+ unsigned long long carryout;
+ *sum = __builtin_addcll(base, addend, carry, &carryout);
+ (void)carryout;
+#else
+ *sum = base + addend + carry;
+#endif /* __has_builtin(__builtin_addcll) */
+}
+#endif /* add64carry_last */
+
+#ifndef mul_64x64_128
+static __maybe_unused __always_inline uint64_t mul_64x64_128(uint64_t a,
+ uint64_t b,
+ uint64_t *h) {
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ __uint128_t r = (__uint128_t)a * (__uint128_t)b;
+ /* modern GCC could nicely optimize this */
+ *h = (uint64_t)(r >> 64);
+ return (uint64_t)r;
+#elif defined(mul_64x64_high)
+ *h = mul_64x64_high(a, b);
+ return a * b;
+#else
+ /* performs 64x64 to 128 bit multiplication */
+ const uint64_t ll = mul_32x32_64((uint32_t)a, (uint32_t)b);
+ const uint64_t lh = mul_32x32_64(a >> 32, (uint32_t)b);
+ const uint64_t hl = mul_32x32_64((uint32_t)a, b >> 32);
+ const uint64_t hh = mul_32x32_64(a >> 32, b >> 32);
+
+ /* Few simplification are possible here for 32-bit architectures,
+ * but thus we would lost compatibility with the original 64-bit
+ * version. Think is very bad idea, because then 32-bit t1ha will
+ * still (relatively) very slowly and well yet not compatible. */
+ uint64_t l;
+ add64carry_last(add64carry_first(ll, lh << 32, &l), hh, lh >> 32, h);
+ add64carry_last(add64carry_first(l, hl << 32, &l), *h, hl >> 32, h);
+ return l;
+#endif
+}
+#endif /* mul_64x64_128() */
+
+#ifndef mul_64x64_high
+static __maybe_unused __always_inline uint64_t mul_64x64_high(uint64_t a,
+ uint64_t b) {
+ uint64_t h;
+ mul_64x64_128(a, b, &h);
+ return h;
+}
+#endif /* mul_64x64_high */
+
+/***************************************************************************/
+
+/* 'magic' primes */
+static const uint64_t prime_0 = UINT64_C(0xEC99BF0D8372CAAB);
+static const uint64_t prime_1 = UINT64_C(0x82434FE90EDCEF39);
+static const uint64_t prime_2 = UINT64_C(0xD4F06DB99D67BE4B);
+static const uint64_t prime_3 = UINT64_C(0xBD9CACC22C6E9571);
+static const uint64_t prime_4 = UINT64_C(0x9C06FAF4D023E3AB);
+static const uint64_t prime_5 = UINT64_C(0xC060724A8424F345);
+static const uint64_t prime_6 = UINT64_C(0xCB5AF53AE3AAAC31);
+
+/* xor high and low parts of full 128-bit product */
+static __maybe_unused __always_inline uint64_t mux64(uint64_t v,
+ uint64_t prime) {
+ uint64_t l, h;
+ l = mul_64x64_128(v, prime, &h);
+ return l ^ h;
+}
+
+static __always_inline uint64_t final64(uint64_t a, uint64_t b) {
+ uint64_t x = (a + rot64(b, 41)) * prime_0;
+ uint64_t y = (rot64(a, 23) + b) * prime_6;
+ return mux64(x ^ y, prime_5);
+}
+
+static __always_inline void mixup64(uint64_t *__restrict a,
+ uint64_t *__restrict b, uint64_t v,
+ uint64_t prime) {
+ uint64_t h;
+ *a ^= mul_64x64_128(*b + v, prime, &h);
+ *b += h;
+}
+
+/***************************************************************************/
+
+typedef union t1ha_uint128 {
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ __uint128_t v;
+#endif
+ struct {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ uint64_t l;
+ uint64_t h;
+#else
+ uint64_t h;
+ uint64_t l;
+#endif
+ } p;
+} t1ha_uint128_t;
+
+static __always_inline t1ha_uint128_t not128(const t1ha_uint128_t v) {
+ t1ha_uint128_t r;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = ~v.v;
+#else
+ r.p.l = ~v.p.l;
+ r.p.h = ~v.p.h;
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t left128(const t1ha_uint128_t v,
+ unsigned s) {
+ t1ha_uint128_t r;
+ assert(s < 128);
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = v.v << s;
+#else
+ r.p.l = (s < 64) ? v.p.l << s : 0;
+ r.p.h = (s < 64) ? (v.p.h << s) | (s ? v.p.l >> (64 - s) : 0) : v.p.l << (s - 64);
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t right128(const t1ha_uint128_t v,
+ unsigned s) {
+ t1ha_uint128_t r;
+ assert(s < 128);
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = v.v >> s;
+#else
+ r.p.l = (s < 64) ? (s ? v.p.h << (64 - s) : 0) | (v.p.l >> s) : v.p.h >> (s - 64);
+ r.p.h = (s < 64) ? v.p.h >> s : 0;
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t or128(t1ha_uint128_t x,
+ t1ha_uint128_t y) {
+ t1ha_uint128_t r;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = x.v | y.v;
+#else
+ r.p.l = x.p.l | y.p.l;
+ r.p.h = x.p.h | y.p.h;
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t xor128(t1ha_uint128_t x,
+ t1ha_uint128_t y) {
+ t1ha_uint128_t r;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = x.v ^ y.v;
+#else
+ r.p.l = x.p.l ^ y.p.l;
+ r.p.h = x.p.h ^ y.p.h;
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t rot128(t1ha_uint128_t v, unsigned s) {
+ s &= 127;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ v.v = (v.v << (128 - s)) | (v.v >> s);
+ return v;
+#else
+ return s ? or128(left128(v, 128 - s), right128(v, s)) : v;
+#endif
+}
+
+static __always_inline t1ha_uint128_t add128(t1ha_uint128_t x,
+ t1ha_uint128_t y) {
+ t1ha_uint128_t r;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = x.v + y.v;
+#else
+ add64carry_last(add64carry_first(x.p.l, y.p.l, &r.p.l), x.p.h, y.p.h, &r.p.h);
+#endif
+ return r;
+}
+
+static __always_inline t1ha_uint128_t mul128(t1ha_uint128_t x,
+ t1ha_uint128_t y) {
+ t1ha_uint128_t r;
+#if defined(__SIZEOF_INT128__) || \
+ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+ r.v = x.v * y.v;
+#else
+ r.p.l = mul_64x64_128(x.p.l, y.p.l, &r.p.h);
+ r.p.h += x.p.l * y.p.h + y.p.l * x.p.h;
+#endif
+ return r;
+}
diff --git a/contrib/uthash/uthash.h b/contrib/uthash/uthash.h
new file mode 100644
index 0000000..1547d30
--- /dev/null
+++ b/contrib/uthash/uthash.h
@@ -0,0 +1,951 @@
+/*
+Copyright (c) 2003-2013, Troy D. Hanson http://troydhanson.github.com/uthash/
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef UTHASH_H
+#define UTHASH_H
+
+#include <string.h> /* memcmp,strlen */
+#include <stddef.h> /* ptrdiff_t */
+#include <stdlib.h> /* exit() */
+
+/* These macros use decltype or the earlier __typeof GNU extension.
+ As decltype is only available in newer compilers (VS2010 or gcc 4.3+
+ when compiling c++ source) this code uses whatever method is needed
+ or, for VS2008 where neither is available, uses casting workarounds. */
+#ifdef _MSC_VER /* MS compiler */
+#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */
+#define DECLTYPE(x) (decltype(x))
+#else /* VS2008 or older (or VS2010 in C mode) */
+#define NO_DECLTYPE
+#define DECLTYPE(x)
+#endif
+#else /* GNU, Sun and other compilers */
+#define DECLTYPE(x) (__typeof(x))
+#endif
+
+#ifdef NO_DECLTYPE
+#define DECLTYPE_ASSIGN(dst,src) \
+do { \
+ char **_da_dst = (char**)(&(dst)); \
+ *_da_dst = (char*)(src); \
+} while(0)
+#else
+#define DECLTYPE_ASSIGN(dst,src) \
+do { \
+ (dst) = DECLTYPE(dst)(src); \
+} while(0)
+#endif
+
+/* a number of the hash function use uint32_t which isn't defined on win32 */
+#ifdef _MSC_VER
+typedef unsigned int uint32_t;
+typedef unsigned char uint8_t;
+#else
+#include <inttypes.h> /* uint32_t */
+#endif
+
+#define UTHASH_VERSION 1.9.8
+
+#ifndef uthash_fatal
+#define uthash_fatal(msg) exit(-1) /* fatal error (out of memory,etc) */
+#endif
+#ifndef uthash_malloc
+#define uthash_malloc(sz) malloc(sz) /* malloc fcn */
+#endif
+#ifndef uthash_free
+#define uthash_free(ptr,sz) free(ptr) /* free fcn */
+#endif
+
+#ifndef uthash_noexpand_fyi
+#define uthash_noexpand_fyi(tbl) /* can be defined to log noexpand */
+#endif
+#ifndef uthash_expand_fyi
+#define uthash_expand_fyi(tbl) /* can be defined to log expands */
+#endif
+
+/* initial number of buckets */
+#define HASH_INITIAL_NUM_BUCKETS 32 /* initial number of buckets */
+#define HASH_INITIAL_NUM_BUCKETS_LOG2 5 /* lg2 of initial number of buckets */
+#define HASH_BKT_CAPACITY_THRESH 10 /* expand when bucket count reaches */
+
+/* calculate the element whose hash handle address is hhe */
+#define ELMT_FROM_HH(tbl,hhp) ((void*)(((char*)(hhp)) - ((tbl)->hho)))
+
+#define HASH_FIND(hh,head,keyptr,keylen,out) \
+do { \
+ unsigned _hf_bkt,_hf_hashv; \
+ out=NULL; \
+ if (head) { \
+ HASH_FCN(keyptr,keylen, (head)->hh.tbl->num_buckets, _hf_hashv, _hf_bkt); \
+ if (HASH_BLOOM_TEST((head)->hh.tbl, _hf_hashv)) { \
+ HASH_FIND_IN_BKT((head)->hh.tbl, hh, (head)->hh.tbl->buckets[ _hf_bkt ], \
+ keyptr,keylen,out); \
+ } \
+ } \
+} while (0)
+
+#ifdef HASH_BLOOM
+#define HASH_BLOOM_BITLEN (1ULL << HASH_BLOOM)
+#define HASH_BLOOM_BYTELEN (HASH_BLOOM_BITLEN/8) + ((HASH_BLOOM_BITLEN%8) ? 1:0)
+#define HASH_BLOOM_MAKE(tbl) \
+do { \
+ (tbl)->bloom_nbits = HASH_BLOOM; \
+ (tbl)->bloom_bv = (uint8_t*)uthash_malloc(HASH_BLOOM_BYTELEN); \
+ if (!((tbl)->bloom_bv)) { uthash_fatal( "out of memory"); } \
+ memset((tbl)->bloom_bv, 0, HASH_BLOOM_BYTELEN); \
+ (tbl)->bloom_sig = HASH_BLOOM_SIGNATURE; \
+} while (0)
+
+#define HASH_BLOOM_FREE(tbl) \
+do { \
+ uthash_free((tbl)->bloom_bv, HASH_BLOOM_BYTELEN); \
+} while (0)
+
+#define HASH_BLOOM_BITSET(bv,idx) (bv[(idx)/8] |= (1U << ((idx)%8)))
+#define HASH_BLOOM_BITTEST(bv,idx) (bv[(idx)/8] & (1U << ((idx)%8)))
+
+#define HASH_BLOOM_ADD(tbl,hashv) \
+ HASH_BLOOM_BITSET((tbl)->bloom_bv, (hashv & (uint32_t)((1ULL << (tbl)->bloom_nbits) - 1)))
+
+#define HASH_BLOOM_TEST(tbl,hashv) \
+ HASH_BLOOM_BITTEST((tbl)->bloom_bv, (hashv & (uint32_t)((1ULL << (tbl)->bloom_nbits) - 1)))
+
+#else
+#define HASH_BLOOM_MAKE(tbl)
+#define HASH_BLOOM_FREE(tbl)
+#define HASH_BLOOM_ADD(tbl,hashv)
+#define HASH_BLOOM_TEST(tbl,hashv) (1)
+#define HASH_BLOOM_BYTELEN 0
+#endif
+
+#define HASH_MAKE_TABLE(hh,head) \
+do { \
+ (head)->hh.tbl = (UT_hash_table*)uthash_malloc( \
+ sizeof(UT_hash_table)); \
+ if (!((head)->hh.tbl)) { uthash_fatal( "out of memory"); } \
+ memset((head)->hh.tbl, 0, sizeof(UT_hash_table)); \
+ (head)->hh.tbl->tail = &((head)->hh); \
+ (head)->hh.tbl->num_buckets = HASH_INITIAL_NUM_BUCKETS; \
+ (head)->hh.tbl->log2_num_buckets = HASH_INITIAL_NUM_BUCKETS_LOG2; \
+ (head)->hh.tbl->hho = (char*)(&(head)->hh) - (char*)(head); \
+ (head)->hh.tbl->buckets = (UT_hash_bucket*)uthash_malloc( \
+ HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \
+ if (! (head)->hh.tbl->buckets) { uthash_fatal( "out of memory"); } \
+ memset((head)->hh.tbl->buckets, 0, \
+ HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \
+ HASH_BLOOM_MAKE((head)->hh.tbl); \
+ (head)->hh.tbl->signature = HASH_SIGNATURE; \
+} while(0)
+
+#define HASH_ADD(hh,head,fieldname,keylen_in,add) \
+ HASH_ADD_KEYPTR(hh,head,&((add)->fieldname),keylen_in,add)
+
+#define HASH_REPLACE(hh,head,fieldname,keylen_in,add,replaced) \
+do { \
+ replaced=NULL; \
+ HASH_FIND(hh,head,&((add)->fieldname),keylen_in,replaced); \
+ if (replaced!=NULL) { \
+ HASH_DELETE(hh,head,replaced); \
+ }; \
+ HASH_ADD(hh,head,fieldname,keylen_in,add); \
+} while(0)
+
+#define HASH_ADD_KEYPTR(hh,head,keyptr,keylen_in,add) \
+do { \
+ unsigned _ha_bkt; \
+ (add)->hh.next = NULL; \
+ (add)->hh.key = (const char*)keyptr; \
+ (add)->hh.keylen = (unsigned)keylen_in; \
+ if (!(head)) { \
+ head = (add); \
+ (head)->hh.prev = NULL; \
+ HASH_MAKE_TABLE(hh,head); \
+ } else { \
+ (head)->hh.tbl->tail->next = (add); \
+ (add)->hh.prev = ELMT_FROM_HH((head)->hh.tbl, (head)->hh.tbl->tail); \
+ (head)->hh.tbl->tail = &((add)->hh); \
+ } \
+ (head)->hh.tbl->num_items++; \
+ (add)->hh.tbl = (head)->hh.tbl; \
+ HASH_FCN(keyptr,keylen_in, (head)->hh.tbl->num_buckets, \
+ (add)->hh.hashv, _ha_bkt); \
+ HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt],&(add)->hh); \
+ HASH_BLOOM_ADD((head)->hh.tbl,(add)->hh.hashv); \
+ HASH_EMIT_KEY(hh,head,keyptr,keylen_in); \
+ HASH_FSCK(hh,head); \
+} while(0)
+
+#define HASH_TO_BKT( hashv, num_bkts, bkt ) \
+do { \
+ bkt = ((hashv) & ((num_bkts) - 1)); \
+} while(0)
+
+/* delete "delptr" from the hash table.
+ * "the usual" patch-up process for the app-order doubly-linked-list.
+ * The use of _hd_hh_del below deserves special explanation.
+ * These used to be expressed using (delptr) but that led to a bug
+ * if someone used the same symbol for the head and deletee, like
+ * HASH_DELETE(hh,users,users);
+ * We want that to work, but by changing the head (users) below
+ * we were forfeiting our ability to further refer to the deletee (users)
+ * in the patch-up process. Solution: use scratch space to
+ * copy the deletee pointer, then the latter references are via that
+ * scratch pointer rather than through the repointed (users) symbol.
+ */
+#define HASH_DELETE(hh,head,delptr) \
+do { \
+ unsigned _hd_bkt; \
+ struct UT_hash_handle *_hd_hh_del; \
+ if ( ((delptr)->hh.prev == NULL) && ((delptr)->hh.next == NULL) ) { \
+ uthash_free((head)->hh.tbl->buckets, \
+ (head)->hh.tbl->num_buckets*sizeof(struct UT_hash_bucket) ); \
+ HASH_BLOOM_FREE((head)->hh.tbl); \
+ uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \
+ head = NULL; \
+ } else { \
+ _hd_hh_del = &((delptr)->hh); \
+ if ((delptr) == ELMT_FROM_HH((head)->hh.tbl,(head)->hh.tbl->tail)) { \
+ (head)->hh.tbl->tail = \
+ (UT_hash_handle*)((ptrdiff_t)((delptr)->hh.prev) + \
+ (head)->hh.tbl->hho); \
+ } \
+ if ((delptr)->hh.prev) { \
+ ((UT_hash_handle*)((ptrdiff_t)((delptr)->hh.prev) + \
+ (head)->hh.tbl->hho))->next = (delptr)->hh.next; \
+ } else { \
+ DECLTYPE_ASSIGN(head,(delptr)->hh.next); \
+ } \
+ if (_hd_hh_del->next) { \
+ ((UT_hash_handle*)((ptrdiff_t)_hd_hh_del->next + \
+ (head)->hh.tbl->hho))->prev = \
+ _hd_hh_del->prev; \
+ } \
+ HASH_TO_BKT( _hd_hh_del->hashv, (head)->hh.tbl->num_buckets, _hd_bkt); \
+ HASH_DEL_IN_BKT(hh,(head)->hh.tbl->buckets[_hd_bkt], _hd_hh_del); \
+ (head)->hh.tbl->num_items--; \
+ } \
+ HASH_FSCK(hh,head); \
+} while (0)
+
+
+/* convenience forms of HASH_FIND/HASH_ADD/HASH_DEL */
+#define HASH_FIND_STR(head,findstr,out) \
+ HASH_FIND(hh,head,findstr,strlen(findstr),out)
+#define HASH_ADD_STR(head,strfield,add) \
+ HASH_ADD(hh,head,strfield,strlen(add->strfield),add)
+#define HASH_REPLACE_STR(head,strfield,add,replaced) \
+ HASH_REPLACE(hh,head,strfield,strlen(add->strfield),add,replaced)
+#define HASH_FIND_INT(head,findint,out) \
+ HASH_FIND(hh,head,findint,sizeof(int),out)
+#define HASH_ADD_INT(head,intfield,add) \
+ HASH_ADD(hh,head,intfield,sizeof(int),add)
+#define HASH_REPLACE_INT(head,intfield,add,replaced) \
+ HASH_REPLACE(hh,head,intfield,sizeof(int),add,replaced)
+#define HASH_FIND_PTR(head,findptr,out) \
+ HASH_FIND(hh,head,findptr,sizeof(void *),out)
+#define HASH_ADD_PTR(head,ptrfield,add) \
+ HASH_ADD(hh,head,ptrfield,sizeof(void *),add)
+#define HASH_REPLACE_PTR(head,ptrfield,add) \
+ HASH_REPLACE(hh,head,ptrfield,sizeof(void *),add,replaced)
+#define HASH_DEL(head,delptr) \
+ HASH_DELETE(hh,head,delptr)
+
+/* HASH_FSCK checks hash integrity on every add/delete when HASH_DEBUG is defined.
+ * This is for uthash developer only; it compiles away if HASH_DEBUG isn't defined.
+ */
+#ifdef HASH_DEBUG
+#define HASH_OOPS(...) do { fprintf(stderr,__VA_ARGS__); exit(-1); } while (0)
+#define HASH_FSCK(hh,head) \
+do { \
+ unsigned _bkt_i; \
+ unsigned _count, _bkt_count; \
+ char *_prev; \
+ struct UT_hash_handle *_thh; \
+ if (head) { \
+ _count = 0; \
+ for( _bkt_i = 0; _bkt_i < (head)->hh.tbl->num_buckets; _bkt_i++) { \
+ _bkt_count = 0; \
+ _thh = (head)->hh.tbl->buckets[_bkt_i].hh_head; \
+ _prev = NULL; \
+ while (_thh) { \
+ if (_prev != (char*)(_thh->hh_prev)) { \
+ HASH_OOPS("invalid hh_prev %p, actual %p\n", \
+ _thh->hh_prev, _prev ); \
+ } \
+ _bkt_count++; \
+ _prev = (char*)(_thh); \
+ _thh = _thh->hh_next; \
+ } \
+ _count += _bkt_count; \
+ if ((head)->hh.tbl->buckets[_bkt_i].count != _bkt_count) { \
+ HASH_OOPS("invalid bucket count %d, actual %d\n", \
+ (head)->hh.tbl->buckets[_bkt_i].count, _bkt_count); \
+ } \
+ } \
+ if (_count != (head)->hh.tbl->num_items) { \
+ HASH_OOPS("invalid hh item count %d, actual %d\n", \
+ (head)->hh.tbl->num_items, _count ); \
+ } \
+ /* traverse hh in app order; check next/prev integrity, count */ \
+ _count = 0; \
+ _prev = NULL; \
+ _thh = &(head)->hh; \
+ while (_thh) { \
+ _count++; \
+ if (_prev !=(char*)(_thh->prev)) { \
+ HASH_OOPS("invalid prev %p, actual %p\n", \
+ _thh->prev, _prev ); \
+ } \
+ _prev = (char*)ELMT_FROM_HH((head)->hh.tbl, _thh); \
+ _thh = ( _thh->next ? (UT_hash_handle*)((char*)(_thh->next) + \
+ (head)->hh.tbl->hho) : NULL ); \
+ } \
+ if (_count != (head)->hh.tbl->num_items) { \
+ HASH_OOPS("invalid app item count %d, actual %d\n", \
+ (head)->hh.tbl->num_items, _count ); \
+ } \
+ } \
+} while (0)
+#else
+#define HASH_FSCK(hh,head)
+#endif
+
+/* When compiled with -DHASH_EMIT_KEYS, length-prefixed keys are emitted to
+ * the descriptor to which this macro is defined for tuning the hash function.
+ * The app can #include <unistd.h> to get the prototype for write(2). */
+#ifdef HASH_EMIT_KEYS
+#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen) \
+do { \
+ unsigned _klen = fieldlen; \
+ write(HASH_EMIT_KEYS, &_klen, sizeof(_klen)); \
+ write(HASH_EMIT_KEYS, keyptr, fieldlen); \
+} while (0)
+#else
+#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen)
+#endif
+
+/* default to Jenkin's hash unless overridden e.g. DHASH_FUNCTION=HASH_SAX */
+#ifdef HASH_FUNCTION
+#define HASH_FCN HASH_FUNCTION
+#else
+#define HASH_FCN HASH_JEN
+#endif
+
+/* The Bernstein hash function, used in Perl prior to v5.6 */
+#define HASH_BER(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned _hb_keylen=keylen; \
+ char *_hb_key=(char*)(key); \
+ (hashv) = 0; \
+ while (_hb_keylen--) { (hashv) = ((hashv) * 33) + *_hb_key++; } \
+ bkt = (hashv) & (num_bkts-1); \
+} while (0)
+
+
+/* SAX/FNV/OAT/JEN hash functions are macro variants of those listed at
+ * http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx */
+#define HASH_SAX(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned _sx_i; \
+ char *_hs_key=(char*)(key); \
+ hashv = 0; \
+ for(_sx_i=0; _sx_i < keylen; _sx_i++) \
+ hashv ^= (hashv << 5) + (hashv >> 2) + _hs_key[_sx_i]; \
+ bkt = hashv & (num_bkts-1); \
+} while (0)
+
+#define HASH_FNV(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned _fn_i; \
+ char *_hf_key=(char*)(key); \
+ hashv = 2166136261UL; \
+ for(_fn_i=0; _fn_i < keylen; _fn_i++) \
+ hashv = (hashv * 16777619) ^ _hf_key[_fn_i]; \
+ bkt = hashv & (num_bkts-1); \
+} while(0)
+
+#define HASH_OAT(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned _ho_i; \
+ char *_ho_key=(char*)(key); \
+ hashv = 0; \
+ for(_ho_i=0; _ho_i < keylen; _ho_i++) { \
+ hashv += _ho_key[_ho_i]; \
+ hashv += (hashv << 10); \
+ hashv ^= (hashv >> 6); \
+ } \
+ hashv += (hashv << 3); \
+ hashv ^= (hashv >> 11); \
+ hashv += (hashv << 15); \
+ bkt = hashv & (num_bkts-1); \
+} while(0)
+
+#define HASH_JEN_MIX(a,b,c) \
+do { \
+ a -= b; a -= c; a ^= ( c >> 13 ); \
+ b -= c; b -= a; b ^= ( a << 8 ); \
+ c -= a; c -= b; c ^= ( b >> 13 ); \
+ a -= b; a -= c; a ^= ( c >> 12 ); \
+ b -= c; b -= a; b ^= ( a << 16 ); \
+ c -= a; c -= b; c ^= ( b >> 5 ); \
+ a -= b; a -= c; a ^= ( c >> 3 ); \
+ b -= c; b -= a; b ^= ( a << 10 ); \
+ c -= a; c -= b; c ^= ( b >> 15 ); \
+} while (0)
+
+#define HASH_JEN(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned _hj_i,_hj_j,_hj_k; \
+ unsigned const char *_hj_key=(unsigned const char*)(key); \
+ hashv = 0xfeedbeef; \
+ _hj_i = _hj_j = 0x9e3779b9; \
+ _hj_k = (unsigned)keylen; \
+ while (_hj_k >= 12) { \
+ _hj_i += (_hj_key[0] + ( (unsigned)_hj_key[1] << 8 ) \
+ + ( (unsigned)_hj_key[2] << 16 ) \
+ + ( (unsigned)_hj_key[3] << 24 ) ); \
+ _hj_j += (_hj_key[4] + ( (unsigned)_hj_key[5] << 8 ) \
+ + ( (unsigned)_hj_key[6] << 16 ) \
+ + ( (unsigned)_hj_key[7] << 24 ) ); \
+ hashv += (_hj_key[8] + ( (unsigned)_hj_key[9] << 8 ) \
+ + ( (unsigned)_hj_key[10] << 16 ) \
+ + ( (unsigned)_hj_key[11] << 24 ) ); \
+ \
+ HASH_JEN_MIX(_hj_i, _hj_j, hashv); \
+ \
+ _hj_key += 12; \
+ _hj_k -= 12; \
+ } \
+ hashv += keylen; \
+ switch ( _hj_k ) { \
+ case 11: hashv += ( (unsigned)_hj_key[10] << 24 ); \
+ case 10: hashv += ( (unsigned)_hj_key[9] << 16 ); \
+ case 9: hashv += ( (unsigned)_hj_key[8] << 8 ); \
+ case 8: _hj_j += ( (unsigned)_hj_key[7] << 24 ); \
+ case 7: _hj_j += ( (unsigned)_hj_key[6] << 16 ); \
+ case 6: _hj_j += ( (unsigned)_hj_key[5] << 8 ); \
+ case 5: _hj_j += _hj_key[4]; \
+ case 4: _hj_i += ( (unsigned)_hj_key[3] << 24 ); \
+ case 3: _hj_i += ( (unsigned)_hj_key[2] << 16 ); \
+ case 2: _hj_i += ( (unsigned)_hj_key[1] << 8 ); \
+ case 1: _hj_i += _hj_key[0]; \
+ } \
+ HASH_JEN_MIX(_hj_i, _hj_j, hashv); \
+ bkt = hashv & (num_bkts-1); \
+} while(0)
+
+/* The Paul Hsieh hash function */
+#undef get16bits
+#if (defined(__GNUC__) && defined(__i386__)) || defined(__WATCOMC__) \
+ || defined(_MSC_VER) || defined (__BORLANDC__) || defined (__TURBOC__)
+#define get16bits(d) (*((const uint16_t *) (d)))
+#endif
+
+#if !defined (get16bits)
+#define get16bits(d) ((((uint32_t)(((const uint8_t *)(d))[1])) << 8) \
+ +(uint32_t)(((const uint8_t *)(d))[0]) )
+#endif
+#define HASH_SFH(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ unsigned const char *_sfh_key=(unsigned const char*)(key); \
+ uint32_t _sfh_tmp, _sfh_len = keylen; \
+ \
+ int _sfh_rem = _sfh_len & 3; \
+ _sfh_len >>= 2; \
+ hashv = 0xcafebabe; \
+ \
+ /* Main loop */ \
+ for (;_sfh_len > 0; _sfh_len--) { \
+ hashv += get16bits (_sfh_key); \
+ _sfh_tmp = (uint32_t)(get16bits (_sfh_key+2)) << 11 ^ hashv; \
+ hashv = (hashv << 16) ^ _sfh_tmp; \
+ _sfh_key += 2*sizeof (uint16_t); \
+ hashv += hashv >> 11; \
+ } \
+ \
+ /* Handle end cases */ \
+ switch (_sfh_rem) { \
+ case 3: hashv += get16bits (_sfh_key); \
+ hashv ^= hashv << 16; \
+ hashv ^= (uint32_t)(_sfh_key[sizeof (uint16_t)] << 18); \
+ hashv += hashv >> 11; \
+ break; \
+ case 2: hashv += get16bits (_sfh_key); \
+ hashv ^= hashv << 11; \
+ hashv += hashv >> 17; \
+ break; \
+ case 1: hashv += *_sfh_key; \
+ hashv ^= hashv << 10; \
+ hashv += hashv >> 1; \
+ } \
+ \
+ /* Force "avalanching" of final 127 bits */ \
+ hashv ^= hashv << 3; \
+ hashv += hashv >> 5; \
+ hashv ^= hashv << 4; \
+ hashv += hashv >> 17; \
+ hashv ^= hashv << 25; \
+ hashv += hashv >> 6; \
+ bkt = hashv & (num_bkts-1); \
+} while(0)
+
+#ifdef HASH_USING_NO_STRICT_ALIASING
+/* The MurmurHash exploits some CPU's (x86,x86_64) tolerance for unaligned reads.
+ * For other types of CPU's (e.g. Sparc) an unaligned read causes a bus error.
+ * MurmurHash uses the faster approach only on CPU's where we know it's safe.
+ *
+ * Note the preprocessor built-in defines can be emitted using:
+ *
+ * gcc -m64 -dM -E - < /dev/null (on gcc)
+ * cc -## a.c (where a.c is a simple test file) (Sun Studio)
+ */
+#if (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86))
+#define MUR_GETBLOCK(p,i) p[i]
+#else /* non intel */
+#define MUR_PLUS0_ALIGNED(p) (((unsigned long)p & 0x3) == 0)
+#define MUR_PLUS1_ALIGNED(p) (((unsigned long)p & 0x3) == 1)
+#define MUR_PLUS2_ALIGNED(p) (((unsigned long)p & 0x3) == 2)
+#define MUR_PLUS3_ALIGNED(p) (((unsigned long)p & 0x3) == 3)
+#define WP(p) ((uint32_t*)((unsigned long)(p) & ~3UL))
+#if (defined(__BIG_ENDIAN__) || defined(SPARC) || defined(__ppc__) || defined(__ppc64__))
+#define MUR_THREE_ONE(p) ((((*WP(p))&0x00ffffff) << 8) | (((*(WP(p)+1))&0xff000000) >> 24))
+#define MUR_TWO_TWO(p) ((((*WP(p))&0x0000ffff) <<16) | (((*(WP(p)+1))&0xffff0000) >> 16))
+#define MUR_ONE_THREE(p) ((((*WP(p))&0x000000ff) <<24) | (((*(WP(p)+1))&0xffffff00) >> 8))
+#else /* assume little endian non-intel */
+#define MUR_THREE_ONE(p) ((((*WP(p))&0xffffff00) >> 8) | (((*(WP(p)+1))&0x000000ff) << 24))
+#define MUR_TWO_TWO(p) ((((*WP(p))&0xffff0000) >>16) | (((*(WP(p)+1))&0x0000ffff) << 16))
+#define MUR_ONE_THREE(p) ((((*WP(p))&0xff000000) >>24) | (((*(WP(p)+1))&0x00ffffff) << 8))
+#endif
+#define MUR_GETBLOCK(p,i) (MUR_PLUS0_ALIGNED(p) ? ((p)[i]) : \
+ (MUR_PLUS1_ALIGNED(p) ? MUR_THREE_ONE(p) : \
+ (MUR_PLUS2_ALIGNED(p) ? MUR_TWO_TWO(p) : \
+ MUR_ONE_THREE(p))))
+#endif
+#define MUR_ROTL32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
+#define MUR_FMIX(_h) \
+do { \
+ _h ^= _h >> 16; \
+ _h *= 0x85ebca6b; \
+ _h ^= _h >> 13; \
+ _h *= 0xc2b2ae35l; \
+ _h ^= _h >> 16; \
+} while(0)
+
+#define HASH_MUR(key,keylen,num_bkts,hashv,bkt) \
+do { \
+ const uint8_t *_mur_data = (const uint8_t*)(key); \
+ const int _mur_nblocks = (keylen) / 4; \
+ uint32_t _mur_h1 = 0xf88D5353; \
+ uint32_t _mur_c1 = 0xcc9e2d51; \
+ uint32_t _mur_c2 = 0x1b873593; \
+ uint32_t _mur_k1 = 0; \
+ const uint8_t *_mur_tail; \
+ const uint32_t *_mur_blocks = (const uint32_t*)(_mur_data+_mur_nblocks*4); \
+ int _mur_i; \
+ for(_mur_i = -_mur_nblocks; _mur_i; _mur_i++) { \
+ _mur_k1 = MUR_GETBLOCK(_mur_blocks,_mur_i); \
+ _mur_k1 *= _mur_c1; \
+ _mur_k1 = MUR_ROTL32(_mur_k1,15); \
+ _mur_k1 *= _mur_c2; \
+ \
+ _mur_h1 ^= _mur_k1; \
+ _mur_h1 = MUR_ROTL32(_mur_h1,13); \
+ _mur_h1 = _mur_h1*5+0xe6546b64; \
+ } \
+ _mur_tail = (const uint8_t*)(_mur_data + _mur_nblocks*4); \
+ _mur_k1=0; \
+ switch((keylen) & 3) { \
+ case 3: _mur_k1 ^= _mur_tail[2] << 16; \
+ case 2: _mur_k1 ^= _mur_tail[1] << 8; \
+ case 1: _mur_k1 ^= _mur_tail[0]; \
+ _mur_k1 *= _mur_c1; \
+ _mur_k1 = MUR_ROTL32(_mur_k1,15); \
+ _mur_k1 *= _mur_c2; \
+ _mur_h1 ^= _mur_k1; \
+ case 0: break; \
+ } \
+ _mur_h1 ^= (keylen); \
+ MUR_FMIX(_mur_h1); \
+ hashv = _mur_h1; \
+ bkt = hashv & (num_bkts-1); \
+} while(0)
+#endif /* HASH_USING_NO_STRICT_ALIASING */
+
+/* key comparison function; return 0 if keys equal */
+#ifndef HASH_KEYCMP
+#define HASH_KEYCMP(a,b,len) memcmp(a,b,len)
+#endif
+
+/* iterate over items in a known bucket to find desired item */
+#define HASH_FIND_IN_BKT(tbl,hh,head,keyptr,keylen_in,out) \
+do { \
+ if (head.hh_head) DECLTYPE_ASSIGN(out,ELMT_FROM_HH(tbl,head.hh_head)); \
+ else out=NULL; \
+ while (out) { \
+ if ((out)->hh.keylen == keylen_in) { \
+ if ((HASH_KEYCMP((out)->hh.key,keyptr,keylen_in)) == 0) break; \
+ } \
+ if ((out)->hh.hh_next) DECLTYPE_ASSIGN(out,ELMT_FROM_HH(tbl,(out)->hh.hh_next)); \
+ else out = NULL; \
+ } \
+} while(0)
+
+/* add an item to a bucket */
+#define HASH_ADD_TO_BKT(head,addhh) \
+do { \
+ head.count++; \
+ (addhh)->hh_next = head.hh_head; \
+ (addhh)->hh_prev = NULL; \
+ if (head.hh_head) { (head).hh_head->hh_prev = (addhh); } \
+ (head).hh_head=addhh; \
+ if (head.count >= ((head.expand_mult+1) * HASH_BKT_CAPACITY_THRESH) \
+ && (addhh)->tbl->noexpand != 1) { \
+ HASH_EXPAND_BUCKETS((addhh)->tbl); \
+ } \
+} while(0)
+
+/* remove an item from a given bucket */
+#define HASH_DEL_IN_BKT(hh,head,hh_del) \
+ (head).count--; \
+ if ((head).hh_head == hh_del) { \
+ (head).hh_head = hh_del->hh_next; \
+ } \
+ if (hh_del->hh_prev) { \
+ hh_del->hh_prev->hh_next = hh_del->hh_next; \
+ } \
+ if (hh_del->hh_next) { \
+ hh_del->hh_next->hh_prev = hh_del->hh_prev; \
+ }
+
+/* Bucket expansion has the effect of doubling the number of buckets
+ * and redistributing the items into the new buckets. Ideally the
+ * items will distribute more or less evenly into the new buckets
+ * (the extent to which this is true is a measure of the quality of
+ * the hash function as it applies to the key domain).
+ *
+ * With the items distributed into more buckets, the chain length
+ * (item count) in each bucket is reduced. Thus by expanding buckets
+ * the hash keeps a bound on the chain length. This bounded chain
+ * length is the essence of how a hash provides constant time lookup.
+ *
+ * The calculation of tbl->ideal_chain_maxlen below deserves some
+ * explanation. First, keep in mind that we're calculating the ideal
+ * maximum chain length based on the *new* (doubled) bucket count.
+ * In fractions this is just n/b (n=number of items,b=new num buckets).
+ * Since the ideal chain length is an integer, we want to calculate
+ * ceil(n/b). We don't depend on floating point arithmetic in this
+ * hash, so to calculate ceil(n/b) with integers we could write
+ *
+ * ceil(n/b) = (n/b) + ((n%b)?1:0)
+ *
+ * and in fact a previous version of this hash did just that.
+ * But now we have improved things a bit by recognizing that b is
+ * always a power of two. We keep its base 2 log handy (call it lb),
+ * so now we can write this with a bit shift and logical AND:
+ *
+ * ceil(n/b) = (n>>lb) + ( (n & (b-1)) ? 1:0)
+ *
+ */
+#define HASH_EXPAND_BUCKETS(tbl) \
+do { \
+ unsigned _he_bkt; \
+ unsigned _he_bkt_i; \
+ struct UT_hash_handle *_he_thh, *_he_hh_nxt; \
+ UT_hash_bucket *_he_new_buckets, *_he_newbkt; \
+ _he_new_buckets = (UT_hash_bucket*)uthash_malloc( \
+ 2 * tbl->num_buckets * sizeof(struct UT_hash_bucket)); \
+ if (!_he_new_buckets) { uthash_fatal( "out of memory"); } \
+ memset(_he_new_buckets, 0, \
+ 2 * tbl->num_buckets * sizeof(struct UT_hash_bucket)); \
+ tbl->ideal_chain_maxlen = \
+ (tbl->num_items >> (tbl->log2_num_buckets+1)) + \
+ ((tbl->num_items & ((tbl->num_buckets*2)-1)) ? 1 : 0); \
+ tbl->nonideal_items = 0; \
+ for(_he_bkt_i = 0; _he_bkt_i < tbl->num_buckets; _he_bkt_i++) \
+ { \
+ _he_thh = tbl->buckets[ _he_bkt_i ].hh_head; \
+ while (_he_thh) { \
+ _he_hh_nxt = _he_thh->hh_next; \
+ HASH_TO_BKT( _he_thh->hashv, tbl->num_buckets*2, _he_bkt); \
+ _he_newbkt = &(_he_new_buckets[ _he_bkt ]); \
+ if (++(_he_newbkt->count) > tbl->ideal_chain_maxlen) { \
+ tbl->nonideal_items++; \
+ _he_newbkt->expand_mult = _he_newbkt->count / \
+ tbl->ideal_chain_maxlen; \
+ } \
+ _he_thh->hh_prev = NULL; \
+ _he_thh->hh_next = _he_newbkt->hh_head; \
+ if (_he_newbkt->hh_head) _he_newbkt->hh_head->hh_prev = \
+ _he_thh; \
+ _he_newbkt->hh_head = _he_thh; \
+ _he_thh = _he_hh_nxt; \
+ } \
+ } \
+ uthash_free( tbl->buckets, tbl->num_buckets*sizeof(struct UT_hash_bucket) ); \
+ tbl->num_buckets *= 2; \
+ tbl->log2_num_buckets++; \
+ tbl->buckets = _he_new_buckets; \
+ tbl->ineff_expands = (tbl->nonideal_items > (tbl->num_items >> 1)) ? \
+ (tbl->ineff_expands+1) : 0; \
+ if (tbl->ineff_expands > 1) { \
+ tbl->noexpand=1; \
+ uthash_noexpand_fyi(tbl); \
+ } \
+ uthash_expand_fyi(tbl); \
+} while(0)
+
+
+/* This is an adaptation of Simon Tatham's O(n log(n)) mergesort */
+/* Note that HASH_SORT assumes the hash handle name to be hh.
+ * HASH_SRT was added to allow the hash handle name to be passed in. */
+#define HASH_SORT(head,cmpfcn) HASH_SRT(hh,head,cmpfcn)
+#define HASH_SRT(hh,head,cmpfcn) \
+do { \
+ unsigned _hs_i; \
+ unsigned _hs_looping,_hs_nmerges,_hs_insize,_hs_psize,_hs_qsize; \
+ struct UT_hash_handle *_hs_p, *_hs_q, *_hs_e, *_hs_list, *_hs_tail; \
+ if (head) { \
+ _hs_insize = 1; \
+ _hs_looping = 1; \
+ _hs_list = &((head)->hh); \
+ while (_hs_looping) { \
+ _hs_p = _hs_list; \
+ _hs_list = NULL; \
+ _hs_tail = NULL; \
+ _hs_nmerges = 0; \
+ while (_hs_p) { \
+ _hs_nmerges++; \
+ _hs_q = _hs_p; \
+ _hs_psize = 0; \
+ for ( _hs_i = 0; _hs_i < _hs_insize; _hs_i++ ) { \
+ _hs_psize++; \
+ _hs_q = (UT_hash_handle*)((_hs_q->next) ? \
+ ((void*)((char*)(_hs_q->next) + \
+ (head)->hh.tbl->hho)) : NULL); \
+ if (! (_hs_q) ) break; \
+ } \
+ _hs_qsize = _hs_insize; \
+ while ((_hs_psize > 0) || ((_hs_qsize > 0) && _hs_q )) { \
+ if (_hs_psize == 0) { \
+ _hs_e = _hs_q; \
+ _hs_q = (UT_hash_handle*)((_hs_q->next) ? \
+ ((void*)((char*)(_hs_q->next) + \
+ (head)->hh.tbl->hho)) : NULL); \
+ _hs_qsize--; \
+ } else if ( (_hs_qsize == 0) || !(_hs_q) ) { \
+ _hs_e = _hs_p; \
+ if (_hs_p){ \
+ _hs_p = (UT_hash_handle*)((_hs_p->next) ? \
+ ((void*)((char*)(_hs_p->next) + \
+ (head)->hh.tbl->hho)) : NULL); \
+ } \
+ _hs_psize--; \
+ } else if (( \
+ cmpfcn(DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_p)), \
+ DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_q))) \
+ ) <= 0) { \
+ _hs_e = _hs_p; \
+ if (_hs_p){ \
+ _hs_p = (UT_hash_handle*)((_hs_p->next) ? \
+ ((void*)((char*)(_hs_p->next) + \
+ (head)->hh.tbl->hho)) : NULL); \
+ } \
+ _hs_psize--; \
+ } else { \
+ _hs_e = _hs_q; \
+ _hs_q = (UT_hash_handle*)((_hs_q->next) ? \
+ ((void*)((char*)(_hs_q->next) + \
+ (head)->hh.tbl->hho)) : NULL); \
+ _hs_qsize--; \
+ } \
+ if ( _hs_tail ) { \
+ _hs_tail->next = ((_hs_e) ? \
+ ELMT_FROM_HH((head)->hh.tbl,_hs_e) : NULL); \
+ } else { \
+ _hs_list = _hs_e; \
+ } \
+ if (_hs_e) { \
+ _hs_e->prev = ((_hs_tail) ? \
+ ELMT_FROM_HH((head)->hh.tbl,_hs_tail) : NULL); \
+ } \
+ _hs_tail = _hs_e; \
+ } \
+ _hs_p = _hs_q; \
+ } \
+ if (_hs_tail){ \
+ _hs_tail->next = NULL; \
+ } \
+ if ( _hs_nmerges <= 1 ) { \
+ _hs_looping=0; \
+ (head)->hh.tbl->tail = _hs_tail; \
+ DECLTYPE_ASSIGN(head,ELMT_FROM_HH((head)->hh.tbl, _hs_list)); \
+ } \
+ _hs_insize *= 2; \
+ } \
+ HASH_FSCK(hh,head); \
+ } \
+} while (0)
+
+/* This function selects items from one hash into another hash.
+ * The end result is that the selected items have dual presence
+ * in both hashes. There is no copy of the items made; rather
+ * they are added into the new hash through a secondary hash
+ * hash handle that must be present in the structure. */
+#define HASH_SELECT(hh_dst, dst, hh_src, src, cond) \
+do { \
+ unsigned _src_bkt, _dst_bkt; \
+ void *_last_elt=NULL, *_elt; \
+ UT_hash_handle *_src_hh, *_dst_hh, *_last_elt_hh=NULL; \
+ ptrdiff_t _dst_hho = ((char*)(&(dst)->hh_dst) - (char*)(dst)); \
+ if (src) { \
+ for(_src_bkt=0; _src_bkt < (src)->hh_src.tbl->num_buckets; _src_bkt++) { \
+ for(_src_hh = (src)->hh_src.tbl->buckets[_src_bkt].hh_head; \
+ _src_hh; \
+ _src_hh = _src_hh->hh_next) { \
+ _elt = ELMT_FROM_HH((src)->hh_src.tbl, _src_hh); \
+ if (cond(_elt)) { \
+ _dst_hh = (UT_hash_handle*)(((char*)_elt) + _dst_hho); \
+ _dst_hh->key = _src_hh->key; \
+ _dst_hh->keylen = _src_hh->keylen; \
+ _dst_hh->hashv = _src_hh->hashv; \
+ _dst_hh->prev = _last_elt; \
+ _dst_hh->next = NULL; \
+ if (_last_elt_hh) { _last_elt_hh->next = _elt; } \
+ if (!dst) { \
+ DECLTYPE_ASSIGN(dst,_elt); \
+ HASH_MAKE_TABLE(hh_dst,dst); \
+ } else { \
+ _dst_hh->tbl = (dst)->hh_dst.tbl; \
+ } \
+ HASH_TO_BKT(_dst_hh->hashv, _dst_hh->tbl->num_buckets, _dst_bkt); \
+ HASH_ADD_TO_BKT(_dst_hh->tbl->buckets[_dst_bkt],_dst_hh); \
+ (dst)->hh_dst.tbl->num_items++; \
+ _last_elt = _elt; \
+ _last_elt_hh = _dst_hh; \
+ } \
+ } \
+ } \
+ } \
+ HASH_FSCK(hh_dst,dst); \
+} while (0)
+
+#define HASH_CLEAR(hh,head) \
+do { \
+ if (head) { \
+ uthash_free((head)->hh.tbl->buckets, \
+ (head)->hh.tbl->num_buckets*sizeof(struct UT_hash_bucket)); \
+ HASH_BLOOM_FREE((head)->hh.tbl); \
+ uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \
+ (head)=NULL; \
+ } \
+} while(0)
+
+#define HASH_OVERHEAD(hh,head) \
+ (size_t)((((head)->hh.tbl->num_items * sizeof(UT_hash_handle)) + \
+ ((head)->hh.tbl->num_buckets * sizeof(UT_hash_bucket)) + \
+ (sizeof(UT_hash_table)) + \
+ (HASH_BLOOM_BYTELEN)))
+
+#ifdef NO_DECLTYPE
+#define HASH_ITER(hh,head,el,tmp) \
+for((el)=(head), (*(char**)(&(tmp)))=(char*)((head)?(head)->hh.next:NULL); \
+ el; (el)=(tmp),(*(char**)(&(tmp)))=(char*)((tmp)?(tmp)->hh.next:NULL))
+#else
+#define HASH_ITER(hh,head,el,tmp) \
+for((el)=(head),(tmp)=DECLTYPE(el)((head)?(head)->hh.next:NULL); \
+ el; (el)=(tmp),(tmp)=DECLTYPE(el)((tmp)?(tmp)->hh.next:NULL))
+#endif
+
+/* obtain a count of items in the hash */
+#define HASH_COUNT(head) HASH_CNT(hh,head)
+#define HASH_CNT(hh,head) ((head)?((head)->hh.tbl->num_items):0)
+
+typedef struct UT_hash_bucket {
+ struct UT_hash_handle *hh_head;
+ unsigned count;
+
+ /* expand_mult is normally set to 0. In this situation, the max chain length
+ * threshold is enforced at its default value, HASH_BKT_CAPACITY_THRESH. (If
+ * the bucket's chain exceeds this length, bucket expansion is triggered).
+ * However, setting expand_mult to a non-zero value delays bucket expansion
+ * (that would be triggered by additions to this particular bucket)
+ * until its chain length reaches a *multiple* of HASH_BKT_CAPACITY_THRESH.
+ * (The multiplier is simply expand_mult+1). The whole idea of this
+ * multiplier is to reduce bucket expansions, since they are expensive, in
+ * situations where we know that a particular bucket tends to be overused.
+ * It is better to let its chain length grow to a longer yet-still-bounded
+ * value, than to do an O(n) bucket expansion too often.
+ */
+ unsigned expand_mult;
+
+} UT_hash_bucket;
+
+/* random signature used only to find hash tables in external analysis */
+#define HASH_SIGNATURE 0xa0111fe1
+#define HASH_BLOOM_SIGNATURE 0xb12220f2
+
+typedef struct UT_hash_table {
+ UT_hash_bucket *buckets;
+ unsigned num_buckets, log2_num_buckets;
+ unsigned num_items;
+ struct UT_hash_handle *tail; /* tail hh in app order, for fast append */
+ ptrdiff_t hho; /* hash handle offset (byte pos of hash handle in element */
+
+ /* in an ideal situation (all buckets used equally), no bucket would have
+ * more than ceil(#items/#buckets) items. that's the ideal chain length. */
+ unsigned ideal_chain_maxlen;
+
+ /* nonideal_items is the number of items in the hash whose chain position
+ * exceeds the ideal chain maxlen. these items pay the penalty for an uneven
+ * hash distribution; reaching them in a chain traversal takes >ideal steps */
+ unsigned nonideal_items;
+
+ /* ineffective expands occur when a bucket doubling was performed, but
+ * afterward, more than half the items in the hash had nonideal chain
+ * positions. If this happens on two consecutive expansions we inhibit any
+ * further expansion, as it's not helping; this happens when the hash
+ * function isn't a good fit for the key domain. When expansion is inhibited
+ * the hash will still work, albeit no longer in constant time. */
+ unsigned ineff_expands, noexpand;
+
+ uint32_t signature; /* used only to find hash tables in external analysis */
+#ifdef HASH_BLOOM
+ uint32_t bloom_sig; /* used only to test bloom exists in external analysis */
+ uint8_t *bloom_bv;
+ char bloom_nbits;
+#endif
+
+} UT_hash_table;
+
+typedef struct UT_hash_handle {
+ struct UT_hash_table *tbl;
+ void *prev; /* prev element in app order */
+ void *next; /* next element in app order */
+ struct UT_hash_handle *hh_prev; /* previous hh in bucket order */
+ struct UT_hash_handle *hh_next; /* next hh in bucket order */
+ const void *key; /* ptr to enclosing struct's key */
+ unsigned keylen; /* enclosing struct's key len */
+ unsigned hashv; /* result of hash-fcn(key) */
+} UT_hash_handle;
+
+#endif /* UTHASH_H */
diff --git a/contrib/uthash/utlist.h b/contrib/uthash/utlist.h
new file mode 100644
index 0000000..6c72b9f
--- /dev/null
+++ b/contrib/uthash/utlist.h
@@ -0,0 +1,766 @@
+/*
+Copyright (c) 2007-2013, Troy D. Hanson http://troydhanson.github.com/uthash/
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef UTLIST_H
+#define UTLIST_H
+
+#define UTLIST_VERSION 1.9.8
+
+#include <assert.h>
+
+/*
+ * This file contains macros to manipulate singly and doubly-linked lists.
+ *
+ * 1. LL_ macros: singly-linked lists.
+ * 2. DL_ macros: doubly-linked lists.
+ * 3. CDL_ macros: circular doubly-linked lists.
+ *
+ * To use singly-linked lists, your structure must have a "next" pointer.
+ * To use doubly-linked lists, your structure must "prev" and "next" pointers.
+ * Either way, the pointer to the head of the list must be initialized to NULL.
+ *
+ * ----------------.EXAMPLE -------------------------
+ * struct item {
+ * int id;
+ * struct item *prev, *next;
+ * }
+ *
+ * struct item *list = NULL:
+ *
+ * int main() {
+ * struct item *item;
+ * ... allocate and populate item ...
+ * DL_APPEND(list, item);
+ * }
+ * --------------------------------------------------
+ *
+ * For doubly-linked lists, the append and delete macros are O(1)
+ * For singly-linked lists, append and delete are O(n) but prepend is O(1)
+ * The sort macro is O(n log(n)) for all types of single/double/circular lists.
+ */
+
+/* These macros use decltype or the earlier __typeof GNU extension.
+ As decltype is only available in newer compilers (VS2010 or gcc 4.3+
+ when compiling c++ code), this code uses whatever method is needed
+ or, for VS2008 where neither is available, uses casting workarounds. */
+#ifdef _MSC_VER /* MS compiler */
+#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */
+#define LDECLTYPE(x) decltype(x)
+#else /* VS2008 or older (or VS2010 in C mode) */
+#define NO_DECLTYPE
+#define LDECLTYPE(x) char*
+#endif
+#elif defined(__ICCARM__)
+#define NO_DECLTYPE
+#define LDECLTYPE(x) char*
+#else /* GNU, Sun and other compilers */
+#define LDECLTYPE(x) __typeof(x)
+#endif
+
+/* for VS2008 we use some workarounds to get around the lack of decltype,
+ * namely, we always reassign our tmp variable to the list head if we need
+ * to dereference its prev/next pointers, and save/restore the real head.*/
+#ifdef NO_DECLTYPE
+#define _SV(elt,list) _tmp = (char*)(list); {char **_alias = (char**)&(list); *_alias = (elt); }
+#define _NEXT(elt,list,next) ((char*)((list)->next))
+#define _NEXTASGN(elt,list,to,next) { char **_alias = (char**)&((list)->next); *_alias=(char*)(to); }
+/* #define _PREV(elt,list,prev) ((char*)((list)->prev)) */
+#define _PREVASGN(elt,list,to,prev) { char **_alias = (char**)&((list)->prev); *_alias=(char*)(to); }
+#define _RS(list) { char **_alias = (char**)&(list); *_alias=_tmp; }
+#define _CASTASGN(a,b) { char **_alias = (char**)&(a); *_alias=(char*)(b); }
+#else
+#define _SV(elt,list)
+#define _NEXT(elt,list,next) ((elt)->next)
+#define _NEXTASGN(elt,list,to,next) ((elt)->next)=(to)
+/* #define _PREV(elt,list,prev) ((elt)->prev) */
+#define _PREVASGN(elt,list,to,prev) ((elt)->prev)=(to)
+#define _RS(list)
+#define _CASTASGN(a,b) (a)=(b)
+#endif
+
+/******************************************************************************
+ * The sort macro is an adaptation of Simon Tatham's O(n log(n)) mergesort *
+ * Unwieldy variable names used here to avoid shadowing passed-in variables. *
+ *****************************************************************************/
+#define LL_SORT(list, cmp) \
+ LL_SORT2(list, cmp, next)
+
+#define LL_SORT2(list, cmp, next) \
+do { \
+ LDECLTYPE(list) _ls_p; \
+ LDECLTYPE(list) _ls_q; \
+ LDECLTYPE(list) _ls_e; \
+ LDECLTYPE(list) _ls_tail; \
+ int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \
+ if (list) { \
+ _ls_insize = 1; \
+ _ls_looping = 1; \
+ while (_ls_looping) { \
+ _CASTASGN(_ls_p,list); \
+ list = NULL; \
+ _ls_tail = NULL; \
+ _ls_nmerges = 0; \
+ while (_ls_p) { \
+ _ls_nmerges++; \
+ _ls_q = _ls_p; \
+ _ls_psize = 0; \
+ for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \
+ _ls_psize++; \
+ _SV(_ls_q,list); _ls_q = _NEXT(_ls_q,list,next); _RS(list); \
+ if (!_ls_q) break; \
+ } \
+ _ls_qsize = _ls_insize; \
+ while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \
+ if (_ls_psize == 0) { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ } else if (_ls_qsize == 0 || !_ls_q) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ } else if (cmp(_ls_p,_ls_q) <= 0) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ } else { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ } \
+ if (_ls_tail) { \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \
+ } else { \
+ _CASTASGN(list,_ls_e); \
+ } \
+ _ls_tail = _ls_e; \
+ } \
+ _ls_p = _ls_q; \
+ } \
+ if (_ls_tail) { \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,NULL,next); _RS(list); \
+ } \
+ if (_ls_nmerges <= 1) { \
+ _ls_looping=0; \
+ } \
+ _ls_insize *= 2; \
+ } \
+ } \
+} while (0)
+
+
+#define DL_SORT(list, cmp) \
+ DL_SORT2(list, cmp, prev, next)
+
+#define DL_SORT2(list, cmp, prev, next) \
+do { \
+ LDECLTYPE(list) _ls_p; \
+ LDECLTYPE(list) _ls_q; \
+ LDECLTYPE(list) _ls_e; \
+ LDECLTYPE(list) _ls_tail; \
+ int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \
+ if (list) { \
+ _ls_insize = 1; \
+ _ls_looping = 1; \
+ while (_ls_looping) { \
+ _CASTASGN(_ls_p,list); \
+ list = NULL; \
+ _ls_tail = NULL; \
+ _ls_nmerges = 0; \
+ while (_ls_p) { \
+ _ls_nmerges++; \
+ _ls_q = _ls_p; \
+ _ls_psize = 0; \
+ for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \
+ _ls_psize++; \
+ _SV(_ls_q,list); _ls_q = _NEXT(_ls_q,list,next); _RS(list); \
+ if (!_ls_q) break; \
+ } \
+ _ls_qsize = _ls_insize; \
+ while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \
+ if (_ls_psize == 0) { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ } else if (_ls_qsize == 0 || !_ls_q) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ } else if (cmp(_ls_p,_ls_q) <= 0) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ } else { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ } \
+ if (_ls_tail) { \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \
+ } else { \
+ _CASTASGN(list,_ls_e); \
+ } \
+ _SV(_ls_e,list); _PREVASGN(_ls_e,list,_ls_tail,prev); _RS(list); \
+ _ls_tail = _ls_e; \
+ } \
+ _ls_p = _ls_q; \
+ } \
+ _CASTASGN(list->prev, _ls_tail); \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,NULL,next); _RS(list); \
+ if (_ls_nmerges <= 1) { \
+ _ls_looping=0; \
+ } \
+ _ls_insize *= 2; \
+ } \
+ } \
+} while (0)
+
+#define CDL_SORT(list, cmp) \
+ CDL_SORT2(list, cmp, prev, next)
+
+#define CDL_SORT2(list, cmp, prev, next) \
+do { \
+ LDECLTYPE(list) _ls_p; \
+ LDECLTYPE(list) _ls_q; \
+ LDECLTYPE(list) _ls_e; \
+ LDECLTYPE(list) _ls_tail; \
+ LDECLTYPE(list) _ls_oldhead; \
+ LDECLTYPE(list) _tmp; \
+ int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \
+ if (list) { \
+ _ls_insize = 1; \
+ _ls_looping = 1; \
+ while (_ls_looping) { \
+ _CASTASGN(_ls_p,list); \
+ _CASTASGN(_ls_oldhead,list); \
+ list = NULL; \
+ _ls_tail = NULL; \
+ _ls_nmerges = 0; \
+ while (_ls_p) { \
+ _ls_nmerges++; \
+ _ls_q = _ls_p; \
+ _ls_psize = 0; \
+ for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \
+ _ls_psize++; \
+ _SV(_ls_q,list); \
+ if (_NEXT(_ls_q,list,next) == _ls_oldhead) { \
+ _ls_q = NULL; \
+ } else { \
+ _ls_q = _NEXT(_ls_q,list,next); \
+ } \
+ _RS(list); \
+ if (!_ls_q) break; \
+ } \
+ _ls_qsize = _ls_insize; \
+ while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \
+ if (_ls_psize == 0) { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \
+ } else if (_ls_qsize == 0 || !_ls_q) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \
+ } else if (cmp(_ls_p,_ls_q) <= 0) { \
+ _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
+ _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
+ if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \
+ } else { \
+ _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
+ _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
+ if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \
+ } \
+ if (_ls_tail) { \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \
+ } else { \
+ _CASTASGN(list,_ls_e); \
+ } \
+ _SV(_ls_e,list); _PREVASGN(_ls_e,list,_ls_tail,prev); _RS(list); \
+ _ls_tail = _ls_e; \
+ } \
+ _ls_p = _ls_q; \
+ } \
+ _CASTASGN(list->prev,_ls_tail); \
+ _CASTASGN(_tmp,list); \
+ _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_tmp,next); _RS(list); \
+ if (_ls_nmerges <= 1) { \
+ _ls_looping=0; \
+ } \
+ _ls_insize *= 2; \
+ } \
+ } \
+} while (0)
+
+/******************************************************************************
+ * singly linked list macros (non-circular) *
+ *****************************************************************************/
+#define LL_PREPEND(head,add) \
+ LL_PREPEND2(head,add,next)
+
+#define LL_PREPEND2(head,add,next) \
+do { \
+ (add)->next = head; \
+ (head) = (add); \
+} while (0)
+
+#define LL_CONCAT(head1,head2) \
+ LL_CONCAT2(head1,head2,next)
+
+#define LL_CONCAT2(head1,head2,next) \
+do { \
+ LDECLTYPE(head1) _tmp; \
+ if (head1) { \
+ _tmp = head1; \
+ while (_tmp->next) { _tmp = _tmp->next; } \
+ _tmp->next=(head2); \
+ } else { \
+ (head1)=(head2); \
+ } \
+} while (0)
+
+#define LL_APPEND(head,add) \
+ LL_APPEND2(head,add,next)
+
+#define LL_APPEND2(head,add,next) \
+do { \
+ LDECLTYPE(head) _tmp; \
+ (add)->next=NULL; \
+ if (head) { \
+ _tmp = head; \
+ while (_tmp->next) { _tmp = _tmp->next; } \
+ _tmp->next=(add); \
+ } else { \
+ (head)=(add); \
+ } \
+} while (0)
+
+#define LL_DELETE(head,del) \
+ LL_DELETE2(head,del,next)
+
+#define LL_DELETE2(head,del,next) \
+do { \
+ LDECLTYPE(head) _tmp; \
+ if ((head) == (del)) { \
+ (head)=(head)->next; \
+ } else { \
+ _tmp = head; \
+ while (_tmp->next && (_tmp->next != (del))) { \
+ _tmp = _tmp->next; \
+ } \
+ if (_tmp->next) { \
+ _tmp->next = ((del)->next); \
+ } \
+ } \
+} while (0)
+
+#define LL_REVERSE2(head,next) do { \
+ LDECLTYPE(head) _cur = (head), _p = NULL, _n = NULL; \
+ while(_cur != NULL) { _n = _cur->next; _cur->next = _p; _p = _cur; _cur = _n; } \
+ (head) = _p; \
+} while (0)
+
+#define LL_REVERSE(head) \
+ LL_REVERSE2(head,next)
+
+/* Here are VS2008 replacements for LL_APPEND and LL_DELETE */
+#define LL_APPEND_VS2008(head,add) \
+ LL_APPEND2_VS2008(head,add,next)
+
+#define LL_APPEND2_VS2008(head,add,next) \
+do { \
+ if (head) { \
+ (add)->next = head; /* use add->next as a temp variable */ \
+ while ((add)->next->next) { (add)->next = (add)->next->next; } \
+ (add)->next->next=(add); \
+ } else { \
+ (head)=(add); \
+ } \
+ (add)->next=NULL; \
+} while (0)
+
+#define LL_DELETE_VS2008(head,del) \
+ LL_DELETE2_VS2008(head,del,next)
+
+#define LL_DELETE2_VS2008(head,del,next) \
+do { \
+ if ((head) == (del)) { \
+ (head)=(head)->next; \
+ } else { \
+ char *_tmp = (char*)(head); \
+ while ((head)->next && ((head)->next != (del))) { \
+ head = (head)->next; \
+ } \
+ if ((head)->next) { \
+ (head)->next = ((del)->next); \
+ } \
+ { \
+ char **_head_alias = (char**)&(head); \
+ *_head_alias = _tmp; \
+ } \
+ } \
+} while (0)
+#ifdef NO_DECLTYPE
+#undef LL_APPEND
+#define LL_APPEND LL_APPEND_VS2008
+#undef LL_DELETE
+#define LL_DELETE LL_DELETE_VS2008
+#undef LL_DELETE2
+#define LL_DELETE2 LL_DELETE2_VS2008
+#undef LL_APPEND2
+#define LL_APPEND2 LL_APPEND2_VS2008
+#undef LL_CONCAT /* no LL_CONCAT_VS2008 */
+#undef DL_CONCAT /* no DL_CONCAT_VS2008 */
+#endif
+/* end VS2008 replacements */
+
+#define LL_COUNT(head,el,counter) \
+ LL_COUNT2(head,el,counter,next) \
+
+#define LL_COUNT2(head,el,counter,next) \
+{ \
+ counter = 0; \
+ LL_FOREACH2(head,el,next){ ++counter; } \
+}
+
+#define LL_FOREACH(head,el) \
+ LL_FOREACH2(head,el,next)
+
+#define LL_FOREACH2(head,el,next) \
+ for(el=head;el;el=(el)->next)
+
+#define LL_FOREACH_SAFE(head,el,tmp) \
+ LL_FOREACH_SAFE2(head,el,tmp,next)
+
+#define LL_FOREACH_SAFE2(head,el,tmp,next) \
+ for((el)=(head);(el) && (tmp = (el)->next, 1); (el) = tmp)
+
+#define LL_SEARCH_SCALAR(head,out,field,val) \
+ LL_SEARCH_SCALAR2(head,out,field,val,next)
+
+#define LL_SEARCH_SCALAR2(head,out,field,val,next) \
+do { \
+ LL_FOREACH2(head,out,next) { \
+ if ((out)->field == (val)) break; \
+ } \
+} while(0)
+
+#define LL_SEARCH(head,out,elt,cmp) \
+ LL_SEARCH2(head,out,elt,cmp,next)
+
+#define LL_SEARCH2(head,out,elt,cmp,next) \
+do { \
+ LL_FOREACH2(head,out,next) { \
+ if ((cmp(out,elt))==0) break; \
+ } \
+} while(0)
+
+#define LL_REPLACE_ELEM(head, el, add) \
+do { \
+ LDECLTYPE(head) _tmp; \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ (add)->next = (el)->next; \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ } else { \
+ _tmp = head; \
+ while (_tmp->next && (_tmp->next != (el))) { \
+ _tmp = _tmp->next; \
+ } \
+ if (_tmp->next) { \
+ _tmp->next = (add); \
+ } \
+ } \
+} while (0)
+
+#define LL_PREPEND_ELEM(head, el, add) \
+do { \
+ LDECLTYPE(head) _tmp; \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ (add)->next = (el); \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ } else { \
+ _tmp = head; \
+ while (_tmp->next && (_tmp->next != (el))) { \
+ _tmp = _tmp->next; \
+ } \
+ if (_tmp->next) { \
+ _tmp->next = (add); \
+ } \
+ } \
+} while (0) \
+
+
+/******************************************************************************
+ * doubly linked list macros (non-circular) *
+ *****************************************************************************/
+#define DL_PREPEND(head,add) \
+ DL_PREPEND2(head,add,prev,next)
+
+#define DL_PREPEND2(head,add,prev,next) \
+do { \
+ (add)->next = head; \
+ if (head) { \
+ (add)->prev = (head)->prev; \
+ (head)->prev = (add); \
+ } else { \
+ (add)->prev = (add); \
+ } \
+ (head) = (add); \
+} while (0)
+
+#define DL_APPEND(head,add) \
+ DL_APPEND2(head,add,prev,next)
+
+#define DL_APPEND2(head,add,prev,next) \
+do { \
+ if (head) { \
+ (add)->prev = (head)->prev; \
+ (head)->prev->next = (add); \
+ (head)->prev = (add); \
+ (add)->next = NULL; \
+ } else { \
+ (head)=(add); \
+ (head)->prev = (head); \
+ (head)->next = NULL; \
+ } \
+} while (0)
+
+#define DL_CONCAT(head1,head2) \
+ DL_CONCAT2(head1,head2,prev,next)
+
+#define DL_CONCAT2(head1,head2,prev,next) \
+do { \
+ LDECLTYPE(head1) _tmp; \
+ if (head2) { \
+ if (head1) { \
+ _tmp = (head2)->prev; \
+ (head2)->prev = (head1)->prev; \
+ (head1)->prev->next = (head2); \
+ (head1)->prev = _tmp; \
+ } else { \
+ (head1)=(head2); \
+ } \
+ } \
+} while (0)
+
+#define DL_DELETE(head,del) \
+ DL_DELETE2(head,del,prev,next)
+
+#define DL_DELETE2(head,del,prev,next) \
+do { \
+ assert((del)->prev != NULL); \
+ if ((del)->prev == (del)) { \
+ (head)=NULL; \
+ } else if ((del)==(head)) { \
+ (del)->next->prev = (del)->prev; \
+ (head) = (del)->next; \
+ } else { \
+ (del)->prev->next = (del)->next; \
+ if ((del)->next) { \
+ (del)->next->prev = (del)->prev; \
+ } else { \
+ (head)->prev = (del)->prev; \
+ } \
+ } \
+} while (0)
+
+#define DL_COUNT(head,el,counter) \
+ DL_COUNT2(head,el,counter,next) \
+
+#define DL_COUNT2(head,el,counter,next) \
+{ \
+ counter = 0; \
+ DL_FOREACH2(head,el,next){ ++counter; } \
+}
+
+#define DL_FOREACH(head,el) \
+ DL_FOREACH2(head,el,next)
+
+#define DL_FOREACH2(head,el,next) \
+ for(el=head;el;el=(el)->next)
+
+/* this version is safe for deleting the elements during iteration */
+#define DL_FOREACH_SAFE(head,el,tmp) \
+ DL_FOREACH_SAFE2(head,el,tmp,next)
+
+#define DL_FOREACH_SAFE2(head,el,tmp,next) \
+ for((el)=(head);(el) && (tmp = (el)->next, 1); (el) = tmp)
+
+/* these are identical to their singly-linked list counterparts */
+#define DL_SEARCH_SCALAR LL_SEARCH_SCALAR
+#define DL_SEARCH LL_SEARCH
+#define DL_SEARCH_SCALAR2 LL_SEARCH_SCALAR2
+#define DL_SEARCH2 LL_SEARCH2
+
+#define DL_REPLACE_ELEM(head, el, add) \
+do { \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ (add)->next = (el)->next; \
+ if ((el)->next == NULL) { \
+ (add)->prev = (add); \
+ } else { \
+ (add)->prev = (el)->prev; \
+ (add)->next->prev = (add); \
+ } \
+ } else { \
+ (add)->next = (el)->next; \
+ (add)->prev = (el)->prev; \
+ (add)->prev->next = (add); \
+ if ((el)->next == NULL) { \
+ (head)->prev = (add); \
+ } else { \
+ (add)->next->prev = (add); \
+ } \
+ } \
+} while (0)
+
+#define DL_PREPEND_ELEM(head, el, add) \
+do { \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ (add)->next = (el); \
+ (add)->prev = (el)->prev; \
+ (el)->prev = (add); \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ } else { \
+ (add)->prev->next = (add); \
+ } \
+} while (0) \
+
+
+/******************************************************************************
+ * circular doubly linked list macros *
+ *****************************************************************************/
+#define CDL_PREPEND(head,add) \
+ CDL_PREPEND2(head,add,prev,next)
+
+#define CDL_PREPEND2(head,add,prev,next) \
+do { \
+ if (head) { \
+ (add)->prev = (head)->prev; \
+ (add)->next = (head); \
+ (head)->prev = (add); \
+ (add)->prev->next = (add); \
+ } else { \
+ (add)->prev = (add); \
+ (add)->next = (add); \
+ } \
+(head)=(add); \
+} while (0)
+
+#define CDL_DELETE(head,del) \
+ CDL_DELETE2(head,del,prev,next)
+
+#define CDL_DELETE2(head,del,prev,next) \
+do { \
+ if ( ((head)==(del)) && ((head)->next == (head))) { \
+ (head) = 0L; \
+ } else { \
+ (del)->next->prev = (del)->prev; \
+ (del)->prev->next = (del)->next; \
+ if ((del) == (head)) (head)=(del)->next; \
+ } \
+} while (0)
+
+#define CDL_COUNT(head,el,counter) \
+ CDL_COUNT2(head,el,counter,next) \
+
+#define CDL_COUNT2(head, el, counter,next) \
+{ \
+ counter = 0; \
+ CDL_FOREACH2(head,el,next){ ++counter; } \
+}
+
+#define CDL_FOREACH(head,el) \
+ CDL_FOREACH2(head,el,next)
+
+#define CDL_FOREACH2(head,el,next) \
+ for(el=head;el;el=((el)->next==head ? 0L : (el)->next))
+
+#define CDL_FOREACH_SAFE(head,el,tmp1,tmp2) \
+ CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next)
+
+#define CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next) \
+ for((el)=(head), ((tmp1)=(head)?((head)->prev):NULL); \
+ (el) && ((tmp2)=(el)->next, 1); \
+ ((el) = (((el)==(tmp1)) ? 0L : (tmp2))))
+
+#define CDL_SEARCH_SCALAR(head,out,field,val) \
+ CDL_SEARCH_SCALAR2(head,out,field,val,next)
+
+#define CDL_SEARCH_SCALAR2(head,out,field,val,next) \
+do { \
+ CDL_FOREACH2(head,out,next) { \
+ if ((out)->field == (val)) break; \
+ } \
+} while(0)
+
+#define CDL_SEARCH(head,out,elt,cmp) \
+ CDL_SEARCH2(head,out,elt,cmp,next)
+
+#define CDL_SEARCH2(head,out,elt,cmp,next) \
+do { \
+ CDL_FOREACH2(head,out,next) { \
+ if ((cmp(out,elt))==0) break; \
+ } \
+} while(0)
+
+#define CDL_REPLACE_ELEM(head, el, add) \
+do { \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ if ((el)->next == (el)) { \
+ (add)->next = (add); \
+ (add)->prev = (add); \
+ (head) = (add); \
+ } else { \
+ (add)->next = (el)->next; \
+ (add)->prev = (el)->prev; \
+ (add)->next->prev = (add); \
+ (add)->prev->next = (add); \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ } \
+ } \
+} while (0)
+
+#define CDL_PREPEND_ELEM(head, el, add) \
+do { \
+ assert(head != NULL); \
+ assert(el != NULL); \
+ assert(add != NULL); \
+ (add)->next = (el); \
+ (add)->prev = (el)->prev; \
+ (el)->prev = (add); \
+ (add)->prev->next = (add); \
+ if ((head) == (el)) { \
+ (head) = (add); \
+ } \
+} while (0) \
+
+#endif /* UTLIST_H */
+
diff --git a/contrib/uthash/utstring.h b/contrib/uthash/utstring.h
new file mode 100644
index 0000000..6130fac
--- /dev/null
+++ b/contrib/uthash/utstring.h
@@ -0,0 +1,415 @@
+/*
+Copyright (c) 2008-2013, Troy D. Hanson http://troydhanson.github.com/uthash/
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/* a dynamic string implementation using macros
+ */
+#ifndef UTSTRING_H
+#define UTSTRING_H
+
+#define UTSTRING_VERSION 1.9.8
+
+#ifdef __GNUC__
+#define _UNUSED_ __attribute__ ((__unused__))
+#else
+#define _UNUSED_
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+
+#ifndef oom
+#define oom() exit(-1)
+#endif
+
+typedef struct {
+ char *d;
+ void **pd;
+ size_t n; /* allocd size */
+ size_t i; /* index of first unused byte */
+} UT_string;
+
+#define utstring_reserve(s,amt) \
+do { \
+ if (((s)->n - (s)->i) < (size_t)(amt)) { \
+ (s)->d = (char*)realloc((s)->d, (s)->n + amt); \
+ if ((s)->d == NULL) oom(); \
+ (s)->n += amt; \
+ if ((s)->pd) *((s)->pd) = (s)->d; \
+ } \
+} while(0)
+
+#define utstring_init(s) \
+do { \
+ (s)->n = 0; (s)->i = 0; (s)->d = NULL; \
+ utstring_reserve(s,128); \
+ (s)->d[0] = '\0'; \
+} while(0)
+
+#define utstring_done(s) \
+do { \
+ if ((s)->d != NULL) free((s)->d); \
+ (s)->n = 0; \
+} while(0)
+
+#define utstring_free(s) \
+do { \
+ utstring_done(s); \
+ free(s); \
+} while(0)
+
+#define utstring_new(s) \
+do { \
+ s = (UT_string*)calloc(1, sizeof(UT_string)); \
+ if (!s) oom(); \
+ utstring_init(s); \
+} while(0)
+
+#define utstring_renew(s) \
+do { \
+ if (s) { \
+ utstring_clear(s); \
+ } else { \
+ utstring_new(s); \
+ } \
+} while(0)
+
+#define utstring_clear(s) \
+do { \
+ (s)->i = 0; \
+ (s)->d[0] = '\0'; \
+} while(0)
+
+#define utstring_bincpy(s,b,l) \
+do { \
+ utstring_reserve((s),(l)+1); \
+ if (l) memcpy(&(s)->d[(s)->i], b, l); \
+ (s)->i += l; \
+ (s)->d[(s)->i]='\0'; \
+} while(0)
+
+#define utstring_concat(dst,src) \
+do { \
+ utstring_reserve((dst),((src)->i)+1); \
+ if ((src)->i) memcpy(&(dst)->d[(dst)->i], (src)->d, (src)->i); \
+ (dst)->i += (src)->i; \
+ (dst)->d[(dst)->i]='\0'; \
+} while(0)
+
+#define utstring_len(s) ((unsigned)((s)->i))
+
+#define utstring_body(s) ((s)->d)
+
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 0)))
+#endif
+_UNUSED_ static void utstring_printf_va(UT_string *s, const char *fmt, va_list ap) {
+ int n;
+ va_list cp;
+ while (1) {
+#ifdef _WIN32
+ cp = ap;
+#else
+ va_copy(cp, ap);
+#endif
+ n = vsnprintf (&s->d[s->i], s->n-s->i, fmt, cp);
+ va_end(cp);
+
+ if ((n > -1) && (n < (int)(s->n-s->i))) {
+ s->i += n;
+ return;
+ }
+
+ /* Else try again with more space. */
+ if (n > -1) utstring_reserve(s,n+1); /* exact */
+ else utstring_reserve(s,(s->n)*2); /* 2x */
+ }
+}
+#ifdef __GNUC__
+/* support printf format checking (2=the format string, 3=start of varargs) */
+static void utstring_printf(UT_string *s, const char *fmt, ...)
+ __attribute__ (( format( printf, 2, 3) ));
+#endif
+_UNUSED_ static void utstring_printf(UT_string *s, const char *fmt, ...) {
+ va_list ap;
+ va_start(ap,fmt);
+ utstring_printf_va(s,fmt,ap);
+ va_end(ap);
+}
+
+#define utstring_append_len(dst, src, len) \
+do { \
+ while ((dst)->n-(dst)->i <= (len)) utstring_reserve((dst),((dst)->n)*2); \
+ memcpy(&(dst)->d[(dst)->i], (src), (len)); \
+ (dst)->i+=(len); \
+ (dst)->d[(dst)->i]='\0'; \
+} while(0)
+
+#define utstring_append_c(dst, c) \
+do { \
+ if ((dst)->n-(dst)->i < 2) utstring_reserve((dst),((dst)->n)*2); \
+ (dst)->d[(dst)->i++] = (c); \
+ (dst)->d[(dst)->i]='\0'; \
+} while(0)
+
+/*******************************************************************************
+ * begin substring search functions *
+ ******************************************************************************/
+/* Build KMP table from left to right. */
+_UNUSED_ static void _utstring_BuildTable(
+ const char *P_Needle,
+ ssize_t P_NeedleLen,
+ long *P_KMP_Table)
+{
+ long i, j;
+
+ i = 0;
+ j = i - 1;
+ P_KMP_Table[i] = j;
+ while (i < P_NeedleLen)
+ {
+ while ( (j > -1) && (P_Needle[i] != P_Needle[j]) )
+ {
+ j = P_KMP_Table[j];
+ }
+ i++;
+ j++;
+ if (i < P_NeedleLen)
+ {
+ if (P_Needle[i] == P_Needle[j])
+ {
+ P_KMP_Table[i] = P_KMP_Table[j];
+ }
+ else
+ {
+ P_KMP_Table[i] = j;
+ }
+ }
+ else
+ {
+ P_KMP_Table[i] = j;
+ }
+ }
+
+ return;
+}
+
+
+/* Build KMP table from right to left. */
+_UNUSED_ static void _utstring_BuildTableR(
+ const char *P_Needle,
+ ssize_t P_NeedleLen,
+ long *P_KMP_Table)
+{
+ long i, j;
+
+ i = P_NeedleLen - 1;
+ j = i + 1;
+ P_KMP_Table[i + 1] = j;
+ while (i >= 0)
+ {
+ while ( (j < P_NeedleLen) && (P_Needle[i] != P_Needle[j]) )
+ {
+ j = P_KMP_Table[j + 1];
+ }
+ i--;
+ j--;
+ if (i >= 0)
+ {
+ if (P_Needle[i] == P_Needle[j])
+ {
+ P_KMP_Table[i + 1] = P_KMP_Table[j + 1];
+ }
+ else
+ {
+ P_KMP_Table[i + 1] = j;
+ }
+ }
+ else
+ {
+ P_KMP_Table[i + 1] = j;
+ }
+ }
+
+ return;
+}
+
+
+/* Search data from left to right. ( Multiple search mode. ) */
+_UNUSED_ static long _utstring_find(
+ const char *P_Haystack,
+ size_t P_HaystackLen,
+ const char *P_Needle,
+ size_t P_NeedleLen,
+ long *P_KMP_Table)
+{
+ long i, j;
+ long V_FindPosition = -1;
+
+ /* Search from left to right. */
+ i = j = 0;
+ while ( (j < (int)P_HaystackLen) && (((P_HaystackLen - j) + i) >= P_NeedleLen) )
+ {
+ while ( (i > -1) && (P_Needle[i] != P_Haystack[j]) )
+ {
+ i = P_KMP_Table[i];
+ }
+ i++;
+ j++;
+ if (i >= (int)P_NeedleLen)
+ {
+ /* Found. */
+ V_FindPosition = j - i;
+ break;
+ }
+ }
+
+ return V_FindPosition;
+}
+
+
+/* Search data from right to left. ( Multiple search mode. ) */
+_UNUSED_ static long _utstring_findR(
+ const char *P_Haystack,
+ size_t P_HaystackLen,
+ const char *P_Needle,
+ size_t P_NeedleLen,
+ long *P_KMP_Table)
+{
+ long i, j;
+ long V_FindPosition = -1;
+
+ /* Search from right to left. */
+ j = (P_HaystackLen - 1);
+ i = (P_NeedleLen - 1);
+ while ( (j >= 0) && (j >= i) )
+ {
+ while ( (i < (int)P_NeedleLen) && (P_Needle[i] != P_Haystack[j]) )
+ {
+ i = P_KMP_Table[i + 1];
+ }
+ i--;
+ j--;
+ if (i < 0)
+ {
+ /* Found. */
+ V_FindPosition = j + 1;
+ break;
+ }
+ }
+
+ return V_FindPosition;
+}
+
+
+/* Search data from left to right. ( One time search mode. ) */
+_UNUSED_ static long utstring_find(
+ UT_string *s,
+ long P_StartPosition, /* Start from 0. -1 means last position. */
+ const char *P_Needle,
+ ssize_t P_NeedleLen)
+{
+ long V_StartPosition;
+ long V_HaystackLen;
+ long *V_KMP_Table;
+ long V_FindPosition = -1;
+
+ if (P_StartPosition < 0)
+ {
+ V_StartPosition = s->i + P_StartPosition;
+ }
+ else
+ {
+ V_StartPosition = P_StartPosition;
+ }
+ V_HaystackLen = s->i - V_StartPosition;
+ if ( (V_HaystackLen >= P_NeedleLen) && (P_NeedleLen > 0) )
+ {
+ V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1));
+ if (V_KMP_Table != NULL)
+ {
+ _utstring_BuildTable(P_Needle, P_NeedleLen, V_KMP_Table);
+
+ V_FindPosition = _utstring_find(s->d + V_StartPosition,
+ V_HaystackLen,
+ P_Needle,
+ P_NeedleLen,
+ V_KMP_Table);
+ if (V_FindPosition >= 0)
+ {
+ V_FindPosition += V_StartPosition;
+ }
+
+ free(V_KMP_Table);
+ }
+ }
+
+ return V_FindPosition;
+}
+
+
+/* Search data from right to left. ( One time search mode. ) */
+_UNUSED_ static long utstring_findR(
+ UT_string *s,
+ long P_StartPosition, /* Start from 0. -1 means last position. */
+ const char *P_Needle,
+ ssize_t P_NeedleLen)
+{
+ long V_StartPosition;
+ long V_HaystackLen;
+ long *V_KMP_Table;
+ long V_FindPosition = -1;
+
+ if (P_StartPosition < 0)
+ {
+ V_StartPosition = s->i + P_StartPosition;
+ }
+ else
+ {
+ V_StartPosition = P_StartPosition;
+ }
+ V_HaystackLen = V_StartPosition + 1;
+ if ( (V_HaystackLen >= P_NeedleLen) && (P_NeedleLen > 0) )
+ {
+ V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1));
+ if (V_KMP_Table != NULL)
+ {
+ _utstring_BuildTableR(P_Needle, P_NeedleLen, V_KMP_Table);
+
+ V_FindPosition = _utstring_findR(s->d,
+ V_HaystackLen,
+ P_Needle,
+ P_NeedleLen,
+ V_KMP_Table);
+
+ free(V_KMP_Table);
+ }
+ }
+
+ return V_FindPosition;
+}
+/*******************************************************************************
+ * end substring search functions *
+ ******************************************************************************/
+
+#endif /* UTSTRING_H */
diff --git a/contrib/xxhash/CMakeLists.txt b/contrib/xxhash/CMakeLists.txt
new file mode 100644
index 0000000..fa69efb
--- /dev/null
+++ b/contrib/xxhash/CMakeLists.txt
@@ -0,0 +1,13 @@
+SET(XXHASHSRC xxhash.c)
+
+ADD_LIBRARY(xxhash STATIC ${XXHASHSRC})
+
+IF(ENABLE_FULL_DEBUG MATCHES "OFF")
+if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+ SET_TARGET_PROPERTIES(xxhash PROPERTIES COMPILE_FLAGS "-O3")
+endif ()
+else()
+ADD_DEFINITIONS(-DXXH_NO_INLINE_HINTS=1)
+ENDIF()
+
+set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DXXH_NO_INLINE_HINTS=1" PARENT_SCOPE)
diff --git a/contrib/xxhash/LICENSE b/contrib/xxhash/LICENSE
new file mode 100644
index 0000000..7de801e
--- /dev/null
+++ b/contrib/xxhash/LICENSE
@@ -0,0 +1,24 @@
+xxHash Library
+Copyright (c) 2012-2014, Yann Collet
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+ list of conditions and the following disclaimer in the documentation and/or
+ other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/contrib/xxhash/xxh3.h b/contrib/xxhash/xxh3.h
new file mode 100644
index 0000000..f7dc195
--- /dev/null
+++ b/contrib/xxhash/xxh3.h
@@ -0,0 +1,55 @@
+/*
+ * xxHash - Extremely Fast Hash algorithm
+ * Development source file for `xxh3`
+ * Copyright (C) 2019-2020 Yann Collet
+ *
+ * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://www.xxhash.com
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+
+/*
+ * Note: This file used to host the source code of XXH3_* variants.
+ * during the development period.
+ * The source code is now properly integrated within xxhash.h.
+ *
+ * xxh3.h is no longer useful,
+ * but it is still provided for compatibility with source code
+ * which used to include it directly.
+ *
+ * Programs are now highly discouraged to include xxh3.h.
+ * Include `xxhash.h` instead, which is the officially supported interface.
+ *
+ * In the future, xxh3.h will start to generate warnings, then errors,
+ * then it will be removed from source package and from include directory.
+ */
+
+/* Simulate the same impact as including the old xxh3.h source file */
+
+#define XXH_INLINE_ALL
+#include "xxhash.h"
diff --git a/contrib/xxhash/xxhash.c b/contrib/xxhash/xxhash.c
new file mode 100644
index 0000000..0fae88c
--- /dev/null
+++ b/contrib/xxhash/xxhash.c
@@ -0,0 +1,43 @@
+/*
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2012-2020 Yann Collet
+ *
+ * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://www.xxhash.com
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+
+
+/*
+ * xxhash.c instantiates functions defined in xxhash.h
+ */
+
+#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
+#define XXH_IMPLEMENTATION /* access definitions */
+
+#include "xxhash.h"
diff --git a/contrib/xxhash/xxhash.h b/contrib/xxhash/xxhash.h
new file mode 100644
index 0000000..08ab794
--- /dev/null
+++ b/contrib/xxhash/xxhash.h
@@ -0,0 +1,5580 @@
+/*
+ * xxHash - Extremely Fast Hash algorithm
+ * Header File
+ * Copyright (C) 2012-2020 Yann Collet
+ *
+ * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://www.xxhash.com
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+/*!
+ * @mainpage xxHash
+ *
+ * @file xxhash.h
+ * xxHash prototypes and implementation
+ */
+/* TODO: update */
+/* Notice extracted from xxHash homepage:
+
+xxHash is an extremely fast hash algorithm, running at RAM speed limits.
+It also successfully passes all tests from the SMHasher suite.
+
+Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
+
+Name Speed Q.Score Author
+xxHash 5.4 GB/s 10
+CrapWow 3.2 GB/s 2 Andrew
+MurmurHash 3a 2.7 GB/s 10 Austin Appleby
+SpookyHash 2.0 GB/s 10 Bob Jenkins
+SBox 1.4 GB/s 9 Bret Mulvey
+Lookup3 1.2 GB/s 9 Bob Jenkins
+SuperFastHash 1.2 GB/s 1 Paul Hsieh
+CityHash64 1.05 GB/s 10 Pike & Alakuijala
+FNV 0.55 GB/s 5 Fowler, Noll, Vo
+CRC32 0.43 GB/s 9
+MD5-32 0.33 GB/s 10 Ronald L. Rivest
+SHA1-32 0.28 GB/s 10
+
+Q.Score is a measure of quality of the hash function.
+It depends on successfully passing SMHasher test set.
+10 is a perfect score.
+
+Note: SMHasher's CRC32 implementation is not the fastest one.
+Other speed-oriented implementations can be faster,
+especially in combination with PCLMUL instruction:
+https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735
+
+A 64-bit version, named XXH64, is available since r35.
+It offers much better speed, but for 64-bit applications only.
+Name Speed on 64 bits Speed on 32 bits
+XXH64 13.8 GB/s 1.9 GB/s
+XXH32 6.8 GB/s 6.0 GB/s
+*/
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* ****************************
+ * INLINE mode
+ ******************************/
+/*!
+ * XXH_INLINE_ALL (and XXH_PRIVATE_API)
+ * Use these build macros to inline xxhash into the target unit.
+ * Inlining improves performance on small inputs, especially when the length is
+ * expressed as a compile-time constant:
+ *
+ * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html
+ *
+ * It also keeps xxHash symbols private to the unit, so they are not exported.
+ *
+ * Usage:
+ * #define XXH_INLINE_ALL
+ * #include "xxhash.h"
+ *
+ * Do not compile and link xxhash.o as a separate object, as it is not useful.
+ */
+#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \
+ && !defined(XXH_INLINE_ALL_31684351384)
+ /* this section should be traversed only once */
+# define XXH_INLINE_ALL_31684351384
+ /* give access to the advanced API, required to compile implementations */
+# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */
+# define XXH_STATIC_LINKING_ONLY
+ /* make all functions private */
+# undef XXH_PUBLIC_API
+# if defined(__GNUC__)
+# define XXH_PUBLIC_API static __inline __attribute__((unused))
+# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+# define XXH_PUBLIC_API static inline
+# elif defined(_MSC_VER)
+# define XXH_PUBLIC_API static __inline
+# else
+ /* note: this version may generate warnings for unused static functions */
+# define XXH_PUBLIC_API static
+# endif
+
+ /*
+ * This part deals with the special case where a unit wants to inline xxHash,
+ * but "xxhash.h" has previously been included without XXH_INLINE_ALL,
+ * such as part of some previously included *.h header file.
+ * Without further action, the new include would just be ignored,
+ * and functions would effectively _not_ be inlined (silent failure).
+ * The following macros solve this situation by prefixing all inlined names,
+ * avoiding naming collision with previous inclusions.
+ */
+ /* Before that, we unconditionally #undef all symbols,
+ * in case they were already defined with XXH_NAMESPACE.
+ * They will then be redefined for XXH_INLINE_ALL
+ */
+# undef XXH_versionNumber
+ /* XXH32 */
+# undef XXH32
+# undef XXH32_createState
+# undef XXH32_freeState
+# undef XXH32_reset
+# undef XXH32_update
+# undef XXH32_digest
+# undef XXH32_copyState
+# undef XXH32_canonicalFromHash
+# undef XXH32_hashFromCanonical
+ /* XXH64 */
+# undef XXH64
+# undef XXH64_createState
+# undef XXH64_freeState
+# undef XXH64_reset
+# undef XXH64_update
+# undef XXH64_digest
+# undef XXH64_copyState
+# undef XXH64_canonicalFromHash
+# undef XXH64_hashFromCanonical
+ /* XXH3_64bits */
+# undef XXH3_64bits
+# undef XXH3_64bits_withSecret
+# undef XXH3_64bits_withSeed
+# undef XXH3_64bits_withSecretandSeed
+# undef XXH3_createState
+# undef XXH3_freeState
+# undef XXH3_copyState
+# undef XXH3_64bits_reset
+# undef XXH3_64bits_reset_withSeed
+# undef XXH3_64bits_reset_withSecret
+# undef XXH3_64bits_update
+# undef XXH3_64bits_digest
+# undef XXH3_generateSecret
+ /* XXH3_128bits */
+# undef XXH128
+# undef XXH3_128bits
+# undef XXH3_128bits_withSeed
+# undef XXH3_128bits_withSecret
+# undef XXH3_128bits_reset
+# undef XXH3_128bits_reset_withSeed
+# undef XXH3_128bits_reset_withSecret
+# undef XXH3_128bits_reset_withSecretandSeed
+# undef XXH3_128bits_update
+# undef XXH3_128bits_digest
+# undef XXH128_isEqual
+# undef XXH128_cmp
+# undef XXH128_canonicalFromHash
+# undef XXH128_hashFromCanonical
+ /* Finally, free the namespace itself */
+# undef XXH_NAMESPACE
+
+ /* employ the namespace for XXH_INLINE_ALL */
+# define XXH_NAMESPACE XXH_INLINE_
+ /*
+ * Some identifiers (enums, type names) are not symbols,
+ * but they must nonetheless be renamed to avoid redeclaration.
+ * Alternative solution: do not redeclare them.
+ * However, this requires some #ifdefs, and has a more dispersed impact.
+ * Meanwhile, renaming can be achieved in a single place.
+ */
+# define XXH_IPREF(Id) XXH_NAMESPACE ## Id
+# define XXH_OK XXH_IPREF(XXH_OK)
+# define XXH_ERROR XXH_IPREF(XXH_ERROR)
+# define XXH_errorcode XXH_IPREF(XXH_errorcode)
+# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t)
+# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t)
+# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t)
+# define XXH32_state_s XXH_IPREF(XXH32_state_s)
+# define XXH32_state_t XXH_IPREF(XXH32_state_t)
+# define XXH64_state_s XXH_IPREF(XXH64_state_s)
+# define XXH64_state_t XXH_IPREF(XXH64_state_t)
+# define XXH3_state_s XXH_IPREF(XXH3_state_s)
+# define XXH3_state_t XXH_IPREF(XXH3_state_t)
+# define XXH128_hash_t XXH_IPREF(XXH128_hash_t)
+ /* Ensure the header is parsed again, even if it was previously included */
+# undef XXHASH_H_5627135585666179
+# undef XXHASH_H_STATIC_13879238742
+#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
+
+
+
+/* ****************************************************************
+ * Stable API
+ *****************************************************************/
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
+
+
+/*!
+ * @defgroup public Public API
+ * Contains details on the public xxHash functions.
+ * @{
+ */
+/* specific declaration modes for Windows */
+#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
+# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
+# ifdef XXH_EXPORT
+# define XXH_PUBLIC_API __declspec(dllexport)
+# elif XXH_IMPORT
+# define XXH_PUBLIC_API __declspec(dllimport)
+# endif
+# else
+# define XXH_PUBLIC_API /* do nothing */
+# endif
+#endif
+
+#ifdef XXH_DOXYGEN
+/*!
+ * @brief Emulate a namespace by transparently prefixing all symbols.
+ *
+ * If you want to include _and expose_ xxHash functions from within your own
+ * library, but also want to avoid symbol collisions with other libraries which
+ * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix
+ * any public symbol from xxhash library with the value of XXH_NAMESPACE
+ * (therefore, avoid empty or numeric values).
+ *
+ * Note that no change is required within the calling program as long as it
+ * includes `xxhash.h`: Regular symbol names will be automatically translated
+ * by this header.
+ */
+# define XXH_NAMESPACE /* YOUR NAME HERE */
+# undef XXH_NAMESPACE
+#endif
+
+#ifdef XXH_NAMESPACE
+# define XXH_CAT(A,B) A##B
+# define XXH_NAME2(A,B) XXH_CAT(A,B)
+# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
+/* XXH32 */
+# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
+# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
+# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
+# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
+# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
+# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
+# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
+# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
+# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+/* XXH64 */
+# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
+# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
+# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
+# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
+# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
+# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
+# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
+# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
+/* XXH3_64bits */
+# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
+# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
+# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
+# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed)
+# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
+# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
+# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
+# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
+# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
+# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
+# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed)
+# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
+# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
+# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
+# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed)
+/* XXH3_128bits */
+# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
+# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
+# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
+# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
+# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed)
+# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
+# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
+# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
+# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed)
+# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
+# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
+# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
+# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
+# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
+# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
+#endif
+
+
+/* *************************************
+* Version
+***************************************/
+#define XXH_VERSION_MAJOR 0
+#define XXH_VERSION_MINOR 8
+#define XXH_VERSION_RELEASE 1
+#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
+
+/*!
+ * @brief Obtains the xxHash version.
+ *
+ * This is mostly useful when xxHash is compiled as a shared library,
+ * since the returned value comes from the library, as opposed to header file.
+ *
+ * @return `XXH_VERSION_NUMBER` of the invoked library.
+ */
+XXH_PUBLIC_API unsigned XXH_versionNumber (void);
+
+
+/* ****************************
+* Common basic types
+******************************/
+#include <stddef.h> /* size_t */
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
+/*-**********************************************************************
+* 32-bit hash
+************************************************************************/
+#if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */
+/*!
+ * @brief An unsigned 32-bit integer.
+ *
+ * Not necessarily defined to `uint32_t` but functionally equivalent.
+ */
+typedef uint32_t XXH32_hash_t;
+
+#elif !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+ typedef uint32_t XXH32_hash_t;
+
+#else
+# include <limits.h>
+# if UINT_MAX == 0xFFFFFFFFUL
+ typedef unsigned int XXH32_hash_t;
+# else
+# if ULONG_MAX == 0xFFFFFFFFUL
+ typedef unsigned long XXH32_hash_t;
+# else
+# error "unsupported platform: need a 32-bit type"
+# endif
+# endif
+#endif
+
+/*!
+ * @}
+ *
+ * @defgroup xxh32_family XXH32 family
+ * @ingroup public
+ * Contains functions used in the classic 32-bit xxHash algorithm.
+ *
+ * @note
+ * XXH32 is useful for older platforms, with no or poor 64-bit performance.
+ * Note that @ref xxh3_family provides competitive speed
+ * for both 32-bit and 64-bit systems, and offers true 64/128 bit hash results.
+ *
+ * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families
+ * @see @ref xxh32_impl for implementation details
+ * @{
+ */
+
+/*!
+ * @brief Calculates the 32-bit hash of @p input using xxHash32.
+ *
+ * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s
+ *
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ * @param seed The 32-bit seed to alter the hash's output predictably.
+ *
+ * @pre
+ * The memory between @p input and @p input + @p length must be valid,
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 32-bit hash value.
+ *
+ * @see
+ * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
+ * Direct equivalents for the other variants of xxHash.
+ * @see
+ * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version.
+ */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);
+
+/*!
+ * Streaming functions generate the xxHash value from an incremental input.
+ * This method is slower than single-call functions, due to state management.
+ * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
+ *
+ * An XXH state must first be allocated using `XXH*_createState()`.
+ *
+ * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
+ *
+ * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
+ *
+ * The function returns an error code, with 0 meaning OK, and any other value
+ * meaning there is an error.
+ *
+ * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
+ * This function returns the nn-bits hash as an int or long long.
+ *
+ * It's still possible to continue inserting input into the hash state after a
+ * digest, and generate new hash values later on by invoking `XXH*_digest()`.
+ *
+ * When done, release the state using `XXH*_freeState()`.
+ *
+ * Example code for incrementally hashing a file:
+ * @code{.c}
+ * #include <stdio.h>
+ * #include <xxhash.h>
+ * #define BUFFER_SIZE 256
+ *
+ * // Note: XXH64 and XXH3 use the same interface.
+ * XXH32_hash_t
+ * hashFile(FILE* stream)
+ * {
+ * XXH32_state_t* state;
+ * unsigned char buf[BUFFER_SIZE];
+ * size_t amt;
+ * XXH32_hash_t hash;
+ *
+ * state = XXH32_createState(); // Create a state
+ * assert(state != NULL); // Error check here
+ * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed
+ * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) {
+ * XXH32_update(state, buf, amt); // Hash the file in chunks
+ * }
+ * hash = XXH32_digest(state); // Finalize the hash
+ * XXH32_freeState(state); // Clean up
+ * return hash;
+ * }
+ * @endcode
+ */
+
+/*!
+ * @typedef struct XXH32_state_s XXH32_state_t
+ * @brief The opaque state struct for the XXH32 streaming API.
+ *
+ * @see XXH32_state_s for details.
+ */
+typedef struct XXH32_state_s XXH32_state_t;
+
+/*!
+ * @brief Allocates an @ref XXH32_state_t.
+ *
+ * Must be freed with XXH32_freeState().
+ * @return An allocated XXH32_state_t on success, `NULL` on failure.
+ */
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
+/*!
+ * @brief Frees an @ref XXH32_state_t.
+ *
+ * Must be allocated with XXH32_createState().
+ * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState().
+ * @return XXH_OK.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
+/*!
+ * @brief Copies one @ref XXH32_state_t to another.
+ *
+ * @param dst_state The state to copy to.
+ * @param src_state The state to copy from.
+ * @pre
+ * @p dst_state and @p src_state must not be `NULL` and must not overlap.
+ */
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
+
+/*!
+ * @brief Resets an @ref XXH32_state_t to begin a new hash.
+ *
+ * This function resets and seeds a state. Call it before @ref XXH32_update().
+ *
+ * @param statePtr The state struct to reset.
+ * @param seed The 32-bit seed to alter the hash result predictably.
+ *
+ * @pre
+ * @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed);
+
+/*!
+ * @brief Consumes a block of @p input to an @ref XXH32_state_t.
+ *
+ * Call this to incrementally consume blocks of data.
+ *
+ * @param statePtr The state struct to update.
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ *
+ * @pre
+ * @p statePtr must not be `NULL`.
+ * @pre
+ * The memory between @p input and @p input + @p length must be valid,
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
+
+/*!
+ * @brief Returns the calculated hash value from an @ref XXH32_state_t.
+ *
+ * @note
+ * Calling XXH32_digest() will not affect @p statePtr, so you can update,
+ * digest, and update again.
+ *
+ * @param statePtr The state struct to calculate the hash from.
+ *
+ * @pre
+ * @p statePtr must not be `NULL`.
+ *
+ * @return The calculated xxHash32 value from that state.
+ */
+XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
+
+/******* Canonical representation *******/
+
+/*
+ * The default return values from XXH functions are unsigned 32 and 64 bit
+ * integers.
+ * This the simplest and fastest format for further post-processing.
+ *
+ * However, this leaves open the question of what is the order on the byte level,
+ * since little and big endian conventions will store the same number differently.
+ *
+ * The canonical representation settles this issue by mandating big-endian
+ * convention, the same convention as human-readable numbers (large digits first).
+ *
+ * When writing hash values to storage, sending them over a network, or printing
+ * them, it's highly recommended to use the canonical representation to ensure
+ * portability across a wider range of systems, present and future.
+ *
+ * The following functions allow transformation of hash values to and from
+ * canonical format.
+ */
+
+/*!
+ * @brief Canonical (big endian) representation of @ref XXH32_hash_t.
+ */
+typedef struct {
+ unsigned char digest[4]; /*!< Hash bytes, big endian */
+} XXH32_canonical_t;
+
+/*!
+ * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t.
+ *
+ * @param dst The @ref XXH32_canonical_t pointer to be stored to.
+ * @param hash The @ref XXH32_hash_t to be converted.
+ *
+ * @pre
+ * @p dst must not be `NULL`.
+ */
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
+
+/*!
+ * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t.
+ *
+ * @param src The @ref XXH32_canonical_t to convert.
+ *
+ * @pre
+ * @p src must not be `NULL`.
+ *
+ * @return The converted hash.
+ */
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
+
+
+#ifdef __has_attribute
+# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x)
+#else
+# define XXH_HAS_ATTRIBUTE(x) 0
+#endif
+
+/* C-language Attributes are added in C23. */
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute)
+# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
+#else
+# define XXH_HAS_C_ATTRIBUTE(x) 0
+#endif
+
+#if defined(__cplusplus) && defined(__has_cpp_attribute)
+# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+# define XXH_HAS_CPP_ATTRIBUTE(x) 0
+#endif
+
+/*
+Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute
+introduced in CPP17 and C23.
+CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough
+C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough
+*/
+#if XXH_HAS_C_ATTRIBUTE(x)
+# define XXH_FALLTHROUGH [[fallthrough]]
+#elif XXH_HAS_CPP_ATTRIBUTE(x)
+# define XXH_FALLTHROUGH [[fallthrough]]
+#elif XXH_HAS_ATTRIBUTE(__fallthrough__)
+# define XXH_FALLTHROUGH __attribute__ ((fallthrough))
+#else
+# define XXH_FALLTHROUGH
+#endif
+
+/*!
+ * @}
+ * @ingroup public
+ * @{
+ */
+
+#ifndef XXH_NO_LONG_LONG
+/*-**********************************************************************
+* 64-bit hash
+************************************************************************/
+#if defined(XXH_DOXYGEN) /* don't include <stdint.h> */
+/*!
+ * @brief An unsigned 64-bit integer.
+ *
+ * Not necessarily defined to `uint64_t` but functionally equivalent.
+ */
+typedef uint64_t XXH64_hash_t;
+#elif !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+ typedef uint64_t XXH64_hash_t;
+#else
+# include <limits.h>
+# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL
+ /* LP64 ABI says uint64_t is unsigned long */
+ typedef unsigned long XXH64_hash_t;
+# else
+ /* the following type must have a width of 64-bit */
+ typedef unsigned long long XXH64_hash_t;
+# endif
+#endif
+
+/*!
+ * @}
+ *
+ * @defgroup xxh64_family XXH64 family
+ * @ingroup public
+ * @{
+ * Contains functions used in the classic 64-bit xxHash algorithm.
+ *
+ * @note
+ * XXH3 provides competitive speed for both 32-bit and 64-bit systems,
+ * and offers true 64/128 bit hash results.
+ * It provides better speed for systems with vector processing capabilities.
+ */
+
+
+/*!
+ * @brief Calculates the 64-bit hash of @p input using xxHash64.
+ *
+ * This function usually runs faster on 64-bit systems, but slower on 32-bit
+ * systems (see benchmark).
+ *
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ * @param seed The 64-bit seed to alter the hash's output predictably.
+ *
+ * @pre
+ * The memory between @p input and @p input + @p length must be valid,
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 64-bit hash.
+ *
+ * @see
+ * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
+ * Direct equivalents for the other variants of xxHash.
+ * @see
+ * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version.
+ */
+XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed);
+
+/******* Streaming *******/
+/*!
+ * @brief The opaque state struct for the XXH64 streaming API.
+ *
+ * @see XXH64_state_s for details.
+ */
+typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed);
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
+
+/******* Canonical representation *******/
+typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t;
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
+
+/*!
+ * @}
+ * ************************************************************************
+ * @defgroup xxh3_family XXH3 family
+ * @ingroup public
+ * @{
+ *
+ * XXH3 is a more recent hash algorithm featuring:
+ * - Improved speed for both small and large inputs
+ * - True 64-bit and 128-bit outputs
+ * - SIMD acceleration
+ * - Improved 32-bit viability
+ *
+ * Speed analysis methodology is explained here:
+ *
+ * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
+ *
+ * Compared to XXH64, expect XXH3 to run approximately
+ * ~2x faster on large inputs and >3x faster on small ones,
+ * exact differences vary depending on platform.
+ *
+ * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic,
+ * but does not require it.
+ * Any 32-bit and 64-bit targets that can run XXH32 smoothly
+ * can run XXH3 at competitive speeds, even without vector support.
+ * Further details are explained in the implementation.
+ *
+ * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8,
+ * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro.
+ *
+ * XXH3 implementation is portable:
+ * it has a generic C90 formulation that can be compiled on any platform,
+ * all implementations generage exactly the same hash value on all platforms.
+ * Starting from v0.8.0, it's also labelled "stable", meaning that
+ * any future version will also generate the same hash value.
+ *
+ * XXH3 offers 2 variants, _64bits and _128bits.
+ *
+ * When only 64 bits are needed, prefer invoking the _64bits variant, as it
+ * reduces the amount of mixing, resulting in faster speed on small inputs.
+ * It's also generally simpler to manipulate a scalar return type than a struct.
+ *
+ * The API supports one-shot hashing, streaming mode, and custom secrets.
+ */
+
+/*-**********************************************************************
+* XXH3 64-bit variant
+************************************************************************/
+
+/* XXH3_64bits():
+ * default 64-bit variant, using default secret and default seed of 0.
+ * It's the fastest variant. */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len);
+
+/*
+ * XXH3_64bits_withSeed():
+ * This variant generates a custom secret on the fly
+ * based on default secret altered using the `seed` value.
+ * While this operation is decently fast, note that it's not completely free.
+ * Note: seed==0 produces the same results as XXH3_64bits().
+ */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);
+
+/*!
+ * The bare minimum size for a custom secret.
+ *
+ * @see
+ * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(),
+ * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret().
+ */
+#define XXH3_SECRET_SIZE_MIN 136
+
+/*
+ * XXH3_64bits_withSecret():
+ * It's possible to provide any blob of bytes as a "secret" to generate the hash.
+ * This makes it more difficult for an external actor to prepare an intentional collision.
+ * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN).
+ * However, the quality of the secret impacts the dispersion of the hash algorithm.
+ * Therefore, the secret _must_ look like a bunch of random bytes.
+ * Avoid "trivial" or structured data such as repeated sequences or a text document.
+ * Whenever in doubt about the "randomness" of the blob of bytes,
+ * consider employing "XXH3_generateSecret()" instead (see below).
+ * It will generate a proper high entropy secret derived from the blob of bytes.
+ * Another advantage of using XXH3_generateSecret() is that
+ * it guarantees that all bits within the initial blob of bytes
+ * will impact every bit of the output.
+ * This is not necessarily the case when using the blob of bytes directly
+ * because, when hashing _small_ inputs, only a portion of the secret is employed.
+ */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize);
+
+
+/******* Streaming *******/
+/*
+ * Streaming requires state maintenance.
+ * This operation costs memory and CPU.
+ * As a consequence, streaming is slower than one-shot hashing.
+ * For better performance, prefer one-shot functions whenever applicable.
+ */
+
+/*!
+ * @brief The state struct for the XXH3 streaming API.
+ *
+ * @see XXH3_state_s for details.
+ */
+typedef struct XXH3_state_s XXH3_state_t;
+XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void);
+XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr);
+XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state);
+
+/*
+ * XXH3_64bits_reset():
+ * Initialize with default parameters.
+ * digest will be equivalent to `XXH3_64bits()`.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr);
+/*
+ * XXH3_64bits_reset_withSeed():
+ * Generate a custom secret from `seed`, and store it into `statePtr`.
+ * digest will be equivalent to `XXH3_64bits_withSeed()`.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed);
+/*
+ * XXH3_64bits_reset_withSecret():
+ * `secret` is referenced, it _must outlive_ the hash streaming session.
+ * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`,
+ * and the quality of produced hash values depends on secret's entropy
+ * (secret's content should look like a bunch of random bytes).
+ * When in doubt about the randomness of a candidate `secret`,
+ * consider employing `XXH3_generateSecret()` instead (see below).
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize);
+
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr);
+
+/* note : canonical representation of XXH3 is the same as XXH64
+ * since they both produce XXH64_hash_t values */
+
+
+/*-**********************************************************************
+* XXH3 128-bit variant
+************************************************************************/
+
+/*!
+ * @brief The return value from 128-bit hashes.
+ *
+ * Stored in little endian order, although the fields themselves are in native
+ * endianness.
+ */
+typedef struct {
+ XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */
+ XXH64_hash_t high64; /*!< `value >> 64` */
+} XXH128_hash_t;
+
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len);
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize);
+
+/******* Streaming *******/
+/*
+ * Streaming requires state maintenance.
+ * This operation costs memory and CPU.
+ * As a consequence, streaming is slower than one-shot hashing.
+ * For better performance, prefer one-shot functions whenever applicable.
+ *
+ * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits().
+ * Use already declared XXH3_createState() and XXH3_freeState().
+ *
+ * All reset and streaming functions have same meaning as their 64-bit counterpart.
+ */
+
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr);
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed);
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize);
+
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr);
+
+/* Following helper functions make it possible to compare XXH128_hast_t values.
+ * Since XXH128_hash_t is a structure, this capability is not offered by the language.
+ * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */
+
+/*!
+ * XXH128_isEqual():
+ * Return: 1 if `h1` and `h2` are equal, 0 if they are not.
+ */
+XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);
+
+/*!
+ * XXH128_cmp():
+ *
+ * This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
+ *
+ * return: >0 if *h128_1 > *h128_2
+ * =0 if *h128_1 == *h128_2
+ * <0 if *h128_1 < *h128_2
+ */
+XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2);
+
+
+/******* Canonical representation *******/
+typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t;
+XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash);
+XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src);
+
+
+#endif /* XXH_NO_LONG_LONG */
+
+/*!
+ * @}
+ */
+#endif /* XXHASH_H_5627135585666179 */
+
+
+
+#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742)
+#define XXHASH_H_STATIC_13879238742
+/* ****************************************************************************
+ * This section contains declarations which are not guaranteed to remain stable.
+ * They may change in future versions, becoming incompatible with a different
+ * version of the library.
+ * These declarations should only be used with static linking.
+ * Never use them in association with dynamic linking!
+ ***************************************************************************** */
+
+/*
+ * These definitions are only present to allow static allocation
+ * of XXH states, on stack or in a struct, for example.
+ * Never **ever** access their members directly.
+ */
+
+/*!
+ * @internal
+ * @brief Structure for XXH32 streaming API.
+ *
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
+ * an opaque type. This allows fields to safely be changed.
+ *
+ * Typedef'd to @ref XXH32_state_t.
+ * Do not access the members of this struct directly.
+ * @see XXH64_state_s, XXH3_state_s
+ */
+struct XXH32_state_s {
+ XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */
+ XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */
+ XXH32_hash_t v[4]; /*!< Accumulator lanes */
+ XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */
+ XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */
+ XXH32_hash_t reserved; /*!< Reserved field. Do not read or write to it, it may be removed. */
+}; /* typedef'd to XXH32_state_t */
+
+
+#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */
+
+/*!
+ * @internal
+ * @brief Structure for XXH64 streaming API.
+ *
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
+ * an opaque type. This allows fields to safely be changed.
+ *
+ * Typedef'd to @ref XXH64_state_t.
+ * Do not access the members of this struct directly.
+ * @see XXH32_state_s, XXH3_state_s
+ */
+struct XXH64_state_s {
+ XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */
+ XXH64_hash_t v[4]; /*!< Accumulator lanes */
+ XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */
+ XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */
+ XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/
+ XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it, it may be removed. */
+}; /* typedef'd to XXH64_state_t */
+
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */
+# include <stdalign.h>
+# define XXH_ALIGN(n) alignas(n)
+#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
+/* In C++ alignas() is a keyword */
+# define XXH_ALIGN(n) alignas(n)
+#elif defined(__GNUC__)
+# define XXH_ALIGN(n) __attribute__ ((aligned(n)))
+#elif defined(_MSC_VER)
+# define XXH_ALIGN(n) __declspec(align(n))
+#else
+# define XXH_ALIGN(n) /* disabled */
+#endif
+
+/* Old GCC versions only accept the attribute after the type in structures. */
+#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \
+ && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \
+ && defined(__GNUC__)
+# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align)
+#else
+# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type
+#endif
+
+/*!
+ * @brief The size of the internal XXH3 buffer.
+ *
+ * This is the optimal update size for incremental hashing.
+ *
+ * @see XXH3_64b_update(), XXH3_128b_update().
+ */
+#define XXH3_INTERNALBUFFER_SIZE 256
+
+/*!
+ * @brief Default size of the secret buffer (and @ref XXH3_kSecret).
+ *
+ * This is the size used in @ref XXH3_kSecret and the seeded functions.
+ *
+ * Not to be confused with @ref XXH3_SECRET_SIZE_MIN.
+ */
+#define XXH3_SECRET_DEFAULT_SIZE 192
+
+/*!
+ * @internal
+ * @brief Structure for XXH3 streaming API.
+ *
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined.
+ * Otherwise it is an opaque type.
+ * Never use this definition in combination with dynamic library.
+ * This allows fields to safely be changed in the future.
+ *
+ * @note ** This structure has a strict alignment requirement of 64 bytes!! **
+ * Do not allocate this with `malloc()` or `new`,
+ * it will not be sufficiently aligned.
+ * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation.
+ *
+ * Typedef'd to @ref XXH3_state_t.
+ * Do never access the members of this struct directly.
+ *
+ * @see XXH3_INITSTATE() for stack initialization.
+ * @see XXH3_createState(), XXH3_freeState().
+ * @see XXH32_state_s, XXH64_state_s
+ */
+struct XXH3_state_s {
+ XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]);
+ /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */
+ XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]);
+ /*!< Used to store a custom secret generated from a seed. */
+ XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]);
+ /*!< The internal buffer. @see XXH32_state_s::mem32 */
+ XXH32_hash_t bufferedSize;
+ /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */
+ XXH32_hash_t useSeed;
+ /*!< Reserved field. Needed for padding on 64-bit. */
+ size_t nbStripesSoFar;
+ /*!< Number or stripes processed. */
+ XXH64_hash_t totalLen;
+ /*!< Total length hashed. 64-bit even on 32-bit targets. */
+ size_t nbStripesPerBlock;
+ /*!< Number of stripes per block. */
+ size_t secretLimit;
+ /*!< Size of @ref customSecret or @ref extSecret */
+ XXH64_hash_t seed;
+ /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */
+ XXH64_hash_t reserved64;
+ /*!< Reserved field. */
+ const unsigned char* extSecret;
+ /*!< Reference to an external secret for the _withSecret variants, NULL
+ * for other variants. */
+ /* note: there may be some padding at the end due to alignment on 64 bytes */
+}; /* typedef'd to XXH3_state_t */
+
+#undef XXH_ALIGN_MEMBER
+
+/*!
+ * @brief Initializes a stack-allocated `XXH3_state_s`.
+ *
+ * When the @ref XXH3_state_t structure is merely emplaced on stack,
+ * it should be initialized with XXH3_INITSTATE() or a memset()
+ * in case its first reset uses XXH3_NNbits_reset_withSeed().
+ * This init can be omitted if the first reset uses default or _withSecret mode.
+ * This operation isn't necessary when the state is created with XXH3_createState().
+ * Note that this doesn't prepare the state for a streaming operation,
+ * it's still necessary to use XXH3_NNbits_reset*() afterwards.
+ */
+#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; }
+
+
+/* XXH128() :
+ * simple alias to pre-selected XXH3_128bits variant
+ */
+XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed);
+
+
+/* === Experimental API === */
+/* Symbols defined below must be considered tied to a specific library version. */
+
+/*
+ * XXH3_generateSecret():
+ *
+ * Derive a high-entropy secret from any user-defined content, named customSeed.
+ * The generated secret can be used in combination with `*_withSecret()` functions.
+ * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed,
+ * as it becomes much more difficult for an external actor to guess how to impact the calculation logic.
+ *
+ * The function accepts as input a custom seed of any length and any content,
+ * and derives from it a high-entropy secret of length @secretSize
+ * into an already allocated buffer @secretBuffer.
+ * @secretSize must be >= XXH3_SECRET_SIZE_MIN
+ *
+ * The generated secret can then be used with any `*_withSecret()` variant.
+ * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`,
+ * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()`
+ * are part of this list. They all accept a `secret` parameter
+ * which must be large enough for implementation reasons (>= XXH3_SECRET_SIZE_MIN)
+ * _and_ feature very high entropy (consist of random-looking bytes).
+ * These conditions can be a high bar to meet, so
+ * XXH3_generateSecret() can be employed to ensure proper quality.
+ *
+ * customSeed can be anything. It can have any size, even small ones,
+ * and its content can be anything, even "poor entropy" sources such as a bunch of zeroes.
+ * The resulting `secret` will nonetheless provide all required qualities.
+ *
+ * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior.
+ */
+XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize);
+
+
+/*
+ * XXH3_generateSecret_fromSeed():
+ *
+ * Generate the same secret as the _withSeed() variants.
+ *
+ * The resulting secret has a length of XXH3_SECRET_DEFAULT_SIZE (necessarily).
+ * @secretBuffer must be already allocated, of size at least XXH3_SECRET_DEFAULT_SIZE bytes.
+ *
+ * The generated secret can be used in combination with
+ *`*_withSecret()` and `_withSecretandSeed()` variants.
+ * This generator is notably useful in combination with `_withSecretandSeed()`,
+ * as a way to emulate a faster `_withSeed()` variant.
+ */
+XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed);
+
+/*
+ * *_withSecretandSeed() :
+ * These variants generate hash values using either
+ * @seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes)
+ * or @secret for "large" keys (>= XXH3_MIDSIZE_MAX).
+ *
+ * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`.
+ * `_withSeed()` has to generate the secret on the fly for "large" keys.
+ * It's fast, but can be perceptible for "not so large" keys (< 1 KB).
+ * `_withSecret()` has to generate the masks on the fly for "small" keys,
+ * which requires more instructions than _withSeed() variants.
+ * Therefore, _withSecretandSeed variant combines the best of both worlds.
+ *
+ * When @secret has been generated by XXH3_generateSecret_fromSeed(),
+ * this variant produces *exactly* the same results as `_withSeed()` variant,
+ * hence offering only a pure speed benefit on "large" input,
+ * by skipping the need to regenerate the secret for every large input.
+ *
+ * Another usage scenario is to hash the secret to a 64-bit hash value,
+ * for example with XXH3_64bits(), which then becomes the seed,
+ * and then employ both the seed and the secret in _withSecretandSeed().
+ * On top of speed, an added benefit is that each bit in the secret
+ * has a 50% chance to swap each bit in the output,
+ * via its impact to the seed.
+ * This is not guaranteed when using the secret directly in "small data" scenarios,
+ * because only portions of the secret are employed for small data.
+ */
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSecretandSeed(const void* data, size_t len,
+ const void* secret, size_t secretSize,
+ XXH64_hash_t seed);
+
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSecretandSeed(const void* data, size_t len,
+ const void* secret, size_t secretSize,
+ XXH64_hash_t seed64);
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr,
+ const void* secret, size_t secretSize,
+ XXH64_hash_t seed64);
+
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr,
+ const void* secret, size_t secretSize,
+ XXH64_hash_t seed64);
+
+
+#endif /* XXH_NO_LONG_LONG */
+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
+# define XXH_IMPLEMENTATION
+#endif
+
+#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */
+
+
+/* ======================================================================== */
+/* ======================================================================== */
+/* ======================================================================== */
+
+
+/*-**********************************************************************
+ * xxHash implementation
+ *-**********************************************************************
+ * xxHash's implementation used to be hosted inside xxhash.c.
+ *
+ * However, inlining requires implementation to be visible to the compiler,
+ * hence be included alongside the header.
+ * Previously, implementation was hosted inside xxhash.c,
+ * which was then #included when inlining was activated.
+ * This construction created issues with a few build and install systems,
+ * as it required xxhash.c to be stored in /include directory.
+ *
+ * xxHash implementation is now directly integrated within xxhash.h.
+ * As a consequence, xxhash.c is no longer needed in /include.
+ *
+ * xxhash.c is still available and is still useful.
+ * In a "normal" setup, when xxhash is not inlined,
+ * xxhash.h only exposes the prototypes and public symbols,
+ * while xxhash.c can be built into an object file xxhash.o
+ * which can then be linked into the final binary.
+ ************************************************************************/
+
+#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \
+ || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387)
+# define XXH_IMPLEM_13a8737387
+
+/* *************************************
+* Tuning parameters
+***************************************/
+
+/*!
+ * @defgroup tuning Tuning parameters
+ * @{
+ *
+ * Various macros to control xxHash's behavior.
+ */
+#ifdef XXH_DOXYGEN
+/*!
+ * @brief Define this to disable 64-bit code.
+ *
+ * Useful if only using the @ref xxh32_family and you have a strict C90 compiler.
+ */
+# define XXH_NO_LONG_LONG
+# undef XXH_NO_LONG_LONG /* don't actually */
+/*!
+ * @brief Controls how unaligned memory is accessed.
+ *
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is
+ * safe and portable.
+ *
+ * Unfortunately, on some target/compiler combinations, the generated assembly
+ * is sub-optimal.
+ *
+ * The below switch allow selection of a different access method
+ * in the search for improved performance.
+ *
+ * @par Possible options:
+ *
+ * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy`
+ * @par
+ * Use `memcpy()`. Safe and portable. Note that most modern compilers will
+ * eliminate the function call and treat it as an unaligned access.
+ *
+ * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))`
+ * @par
+ * Depends on compiler extensions and is therefore not portable.
+ * This method is safe _if_ your compiler supports it,
+ * and *generally* as fast or faster than `memcpy`.
+ *
+ * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast
+ * @par
+ * Casts directly and dereferences. This method doesn't depend on the
+ * compiler, but it violates the C standard as it directly dereferences an
+ * unaligned pointer. It can generate buggy code on targets which do not
+ * support unaligned memory accesses, but in some circumstances, it's the
+ * only known way to get the most performance.
+ *
+ * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift
+ * @par
+ * Also portable. This can generate the best code on old compilers which don't
+ * inline small `memcpy()` calls, and it might also be faster on big-endian
+ * systems which lack a native byteswap instruction. However, some compilers
+ * will emit literal byteshifts even if the target supports unaligned access.
+ * .
+ *
+ * @warning
+ * Methods 1 and 2 rely on implementation-defined behavior. Use these with
+ * care, as what works on one compiler/platform/optimization level may cause
+ * another to read garbage data or even crash.
+ *
+ * See http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
+ *
+ * Prefer these methods in priority order (0 > 3 > 1 > 2)
+ */
+# define XXH_FORCE_MEMORY_ACCESS 0
+
+/*!
+ * @def XXH_FORCE_ALIGN_CHECK
+ * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32()
+ * and XXH64() only).
+ *
+ * This is an important performance trick for architectures without decent
+ * unaligned memory access performance.
+ *
+ * It checks for input alignment, and when conditions are met, uses a "fast
+ * path" employing direct 32-bit/64-bit reads, resulting in _dramatically
+ * faster_ read speed.
+ *
+ * The check costs one initial branch per hash, which is generally negligible,
+ * but not zero.
+ *
+ * Moreover, it's not useful to generate an additional code path if memory
+ * access uses the same instruction for both aligned and unaligned
+ * addresses (e.g. x86 and aarch64).
+ *
+ * In these cases, the alignment check can be removed by setting this macro to 0.
+ * Then the code will always use unaligned memory access.
+ * Align check is automatically disabled on x86, x64 & arm64,
+ * which are platforms known to offer good unaligned memory accesses performance.
+ *
+ * This option does not affect XXH3 (only XXH32 and XXH64).
+ */
+# define XXH_FORCE_ALIGN_CHECK 0
+
+/*!
+ * @def XXH_NO_INLINE_HINTS
+ * @brief When non-zero, sets all functions to `static`.
+ *
+ * By default, xxHash tries to force the compiler to inline almost all internal
+ * functions.
+ *
+ * This can usually improve performance due to reduced jumping and improved
+ * constant folding, but significantly increases the size of the binary which
+ * might not be favorable.
+ *
+ * Additionally, sometimes the forced inlining can be detrimental to performance,
+ * depending on the architecture.
+ *
+ * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the
+ * compiler full control on whether to inline or not.
+ *
+ * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using
+ * -fno-inline with GCC or Clang, this will automatically be defined.
+ */
+# define XXH_NO_INLINE_HINTS 0
+
+/*!
+ * @def XXH32_ENDJMP
+ * @brief Whether to use a jump for `XXH32_finalize`.
+ *
+ * For performance, `XXH32_finalize` uses multiple branches in the finalizer.
+ * This is generally preferable for performance,
+ * but depending on exact architecture, a jmp may be preferable.
+ *
+ * This setting is only possibly making a difference for very small inputs.
+ */
+# define XXH32_ENDJMP 0
+
+/*!
+ * @internal
+ * @brief Redefines old internal names.
+ *
+ * For compatibility with code that uses xxHash's internals before the names
+ * were changed to improve namespacing. There is no other reason to use this.
+ */
+# define XXH_OLD_NAMES
+# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */
+#endif /* XXH_DOXYGEN */
+/*!
+ * @}
+ */
+
+#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
+ /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */
+# if !defined(__clang__) && \
+( \
+ (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \
+ ( \
+ defined(__GNUC__) && ( \
+ (defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \
+ ( \
+ defined(__mips__) && \
+ (__mips <= 5 || __mips_isa_rev < 6) && \
+ (!defined(__mips16) || defined(__mips_mips16e2)) \
+ ) \
+ ) \
+ ) \
+)
+# define XXH_FORCE_MEMORY_ACCESS 1
+# endif
+#endif
+
+#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
+# if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) \
+ || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */
+# define XXH_FORCE_ALIGN_CHECK 0
+# else
+# define XXH_FORCE_ALIGN_CHECK 1
+# endif
+#endif
+
+#ifndef XXH_NO_INLINE_HINTS
+# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \
+ || defined(__NO_INLINE__) /* -O0, -fno-inline */
+# define XXH_NO_INLINE_HINTS 1
+# else
+# define XXH_NO_INLINE_HINTS 0
+# endif
+#endif
+
+#ifndef XXH32_ENDJMP
+/* generally preferable for performance */
+# define XXH32_ENDJMP 0
+#endif
+
+/*!
+ * @defgroup impl Implementation
+ * @{
+ */
+
+
+/* *************************************
+* Includes & Memory related functions
+***************************************/
+/*
+ * Modify the local functions below should you wish to use
+ * different memory routines for malloc() and free()
+ */
+#include <stdlib.h>
+
+/*!
+ * @internal
+ * @brief Modify this function to use a different routine than malloc().
+ */
+static void* XXH_malloc(size_t s) { return malloc(s); }
+
+/*!
+ * @internal
+ * @brief Modify this function to use a different routine than free().
+ */
+static void XXH_free(void* p) { free(p); }
+
+#include <string.h>
+
+/*!
+ * @internal
+ * @brief Modify this function to use a different routine than memcpy().
+ */
+static void* XXH_memcpy(void* dest, const void* src, size_t size)
+{
+ return memcpy(dest,src,size);
+}
+
+#include <limits.h> /* ULLONG_MAX */
+
+
+/* *************************************
+* Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER /* Visual Studio warning fix */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+#endif
+
+#if XXH_NO_INLINE_HINTS /* disable inlining hints */
+# if defined(__GNUC__) || defined(__clang__)
+# define XXH_FORCE_INLINE static __attribute__((unused))
+# else
+# define XXH_FORCE_INLINE static
+# endif
+# define XXH_NO_INLINE static
+/* enable inlining hints */
+#elif defined(__GNUC__) || defined(__clang__)
+# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused))
+# define XXH_NO_INLINE static __attribute__((noinline))
+#elif defined(_MSC_VER) /* Visual Studio */
+# define XXH_FORCE_INLINE static __forceinline
+# define XXH_NO_INLINE static __declspec(noinline)
+#elif defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */
+# define XXH_FORCE_INLINE static inline
+# define XXH_NO_INLINE static
+#else
+# define XXH_FORCE_INLINE static
+# define XXH_NO_INLINE static
+#endif
+
+
+
+/* *************************************
+* Debug
+***************************************/
+/*!
+ * @ingroup tuning
+ * @def XXH_DEBUGLEVEL
+ * @brief Sets the debugging level.
+ *
+ * XXH_DEBUGLEVEL is expected to be defined externally, typically via the
+ * compiler's command line options. The value must be a number.
+ */
+#ifndef XXH_DEBUGLEVEL
+# ifdef DEBUGLEVEL /* backwards compat */
+# define XXH_DEBUGLEVEL DEBUGLEVEL
+# else
+# define XXH_DEBUGLEVEL 0
+# endif
+#endif
+
+#if (XXH_DEBUGLEVEL>=1)
+# include <assert.h> /* note: can still be disabled with NDEBUG */
+# define XXH_ASSERT(c) assert(c)
+#else
+# define XXH_ASSERT(c) ((void)0)
+#endif
+
+/* note: use after variable declarations */
+#ifndef XXH_STATIC_ASSERT
+# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
+# include <assert.h>
+# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
+# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */
+# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
+# else
+# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0)
+# endif
+# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c)
+#endif
+
+/*!
+ * @internal
+ * @def XXH_COMPILER_GUARD(var)
+ * @brief Used to prevent unwanted optimizations for @p var.
+ *
+ * It uses an empty GCC inline assembly statement with a register constraint
+ * which forces @p var into a general purpose register (eg eax, ebx, ecx
+ * on x86) and marks it as modified.
+ *
+ * This is used in a few places to avoid unwanted autovectorization (e.g.
+ * XXH32_round()). All vectorization we want is explicit via intrinsics,
+ * and _usually_ isn't wanted elsewhere.
+ *
+ * We also use it to prevent unwanted constant folding for AArch64 in
+ * XXH3_initCustomSecret_scalar().
+ */
+#if defined(__GNUC__) || defined(__clang__)
+# define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r" (var))
+#else
+# define XXH_COMPILER_GUARD(var) ((void)0)
+#endif
+
+/* *************************************
+* Basic Types
+***************************************/
+#if !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+ typedef uint8_t xxh_u8;
+#else
+ typedef unsigned char xxh_u8;
+#endif
+typedef XXH32_hash_t xxh_u32;
+
+#ifdef XXH_OLD_NAMES
+# define BYTE xxh_u8
+# define U8 xxh_u8
+# define U32 xxh_u32
+#endif
+
+/* *** Memory access *** */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_read32(const void* ptr)
+ * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ *
+ * @param ptr The pointer to read from.
+ * @return The 32-bit native endian integer from the bytes at @p ptr.
+ */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_readLE32(const void* ptr)
+ * @brief Reads an unaligned 32-bit little endian integer from @p ptr.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ *
+ * @param ptr The pointer to read from.
+ * @return The 32-bit little endian integer from the bytes at @p ptr.
+ */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_readBE32(const void* ptr)
+ * @brief Reads an unaligned 32-bit big endian integer from @p ptr.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ *
+ * @param ptr The pointer to read from.
+ * @return The 32-bit big endian integer from the bytes at @p ptr.
+ */
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align)
+ * @brief Like @ref XXH_readLE32(), but has an option for aligned reads.
+ *
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
+ * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is
+ * always @ref XXH_alignment::XXH_unaligned.
+ *
+ * @param ptr The pointer to read from.
+ * @param align Whether @p ptr is aligned.
+ * @pre
+ * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte
+ * aligned.
+ * @return The 32-bit little endian integer from the bytes at @p ptr.
+ */
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+/*
+ * Manual byteshift. Best for old compilers which don't inline memcpy.
+ * We actually directly use XXH_readLE32 and XXH_readBE32.
+ */
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/*
+ * Force direct memory access. Only works on CPU which support unaligned memory
+ * access in hardware.
+ */
+static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/*
+ * __pack instructions are safer but compiler specific, hence potentially
+ * problematic for some compilers.
+ *
+ * Currently only defined for GCC and ICC.
+ */
+#ifdef XXH_OLD_NAMES
+typedef union { xxh_u32 u32; } __attribute__((packed)) unalign;
+#endif
+static xxh_u32 XXH_read32(const void* ptr)
+{
+ typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign;
+ return ((const xxh_unalign*)ptr)->u32;
+}
+
+#else
+
+/*
+ * Portable and safe solution. Generally efficient.
+ * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
+ */
+static xxh_u32 XXH_read32(const void* memPtr)
+{
+ xxh_u32 val;
+ XXH_memcpy(&val, memPtr, sizeof(val));
+ return val;
+}
+
+#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+
+/* *** Endianness *** */
+
+/*!
+ * @ingroup tuning
+ * @def XXH_CPU_LITTLE_ENDIAN
+ * @brief Whether the target is little endian.
+ *
+ * Defined to 1 if the target is little endian, or 0 if it is big endian.
+ * It can be defined externally, for example on the compiler command line.
+ *
+ * If it is not defined,
+ * a runtime check (which is usually constant folded) is used instead.
+ *
+ * @note
+ * This is not necessarily defined to an integer constant.
+ *
+ * @see XXH_isLittleEndian() for the runtime check.
+ */
+#ifndef XXH_CPU_LITTLE_ENDIAN
+/*
+ * Try to detect endianness automatically, to avoid the nonstandard behavior
+ * in `XXH_isLittleEndian()`
+ */
+# if defined(_WIN32) /* Windows is always little endian */ \
+ || defined(__LITTLE_ENDIAN__) \
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+# define XXH_CPU_LITTLE_ENDIAN 1
+# elif defined(__BIG_ENDIAN__) \
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+# define XXH_CPU_LITTLE_ENDIAN 0
+# else
+/*!
+ * @internal
+ * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN.
+ *
+ * Most compilers will constant fold this.
+ */
+static int XXH_isLittleEndian(void)
+{
+ /*
+ * Portable and well-defined behavior.
+ * Don't use static: it is detrimental to performance.
+ */
+ const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 };
+ return one.c[0];
+}
+# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian()
+# endif
+#endif
+
+
+
+
+/* ****************************************
+* Compiler-specific Functions and Macros
+******************************************/
+#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+#ifdef __has_builtin
+# define XXH_HAS_BUILTIN(x) __has_builtin(x)
+#else
+# define XXH_HAS_BUILTIN(x) 0
+#endif
+
+/*!
+ * @internal
+ * @def XXH_rotl32(x,r)
+ * @brief 32-bit rotate left.
+ *
+ * @param x The 32-bit integer to be rotated.
+ * @param r The number of bits to rotate.
+ * @pre
+ * @p r > 0 && @p r < 32
+ * @note
+ * @p x and @p r may be evaluated multiple times.
+ * @return The rotated result.
+ */
+#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \
+ && XXH_HAS_BUILTIN(__builtin_rotateleft64)
+# define XXH_rotl32 __builtin_rotateleft32
+# define XXH_rotl64 __builtin_rotateleft64
+/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
+#elif defined(_MSC_VER)
+# define XXH_rotl32(x,r) _rotl(x,r)
+# define XXH_rotl64(x,r) _rotl64(x,r)
+#else
+# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
+# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r))))
+#endif
+
+/*!
+ * @internal
+ * @fn xxh_u32 XXH_swap32(xxh_u32 x)
+ * @brief A 32-bit byteswap.
+ *
+ * @param x The 32-bit integer to byteswap.
+ * @return @p x, byteswapped.
+ */
+#if defined(_MSC_VER) /* Visual Studio */
+# define XXH_swap32 _byteswap_ulong
+#elif XXH_GCC_VERSION >= 403
+# define XXH_swap32 __builtin_bswap32
+#else
+static xxh_u32 XXH_swap32 (xxh_u32 x)
+{
+ return ((x << 24) & 0xff000000 ) |
+ ((x << 8) & 0x00ff0000 ) |
+ ((x >> 8) & 0x0000ff00 ) |
+ ((x >> 24) & 0x000000ff );
+}
+#endif
+
+
+/* ***************************
+* Memory reads
+*****************************/
+
+/*!
+ * @internal
+ * @brief Enum to indicate whether a pointer is aligned.
+ */
+typedef enum {
+ XXH_aligned, /*!< Aligned */
+ XXH_unaligned /*!< Possibly unaligned */
+} XXH_alignment;
+
+/*
+ * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load.
+ *
+ * This is ideal for older compilers which don't inline memcpy.
+ */
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+
+XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr)
+{
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+ return bytePtr[0]
+ | ((xxh_u32)bytePtr[1] << 8)
+ | ((xxh_u32)bytePtr[2] << 16)
+ | ((xxh_u32)bytePtr[3] << 24);
+}
+
+XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr)
+{
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+ return bytePtr[3]
+ | ((xxh_u32)bytePtr[2] << 8)
+ | ((xxh_u32)bytePtr[1] << 16)
+ | ((xxh_u32)bytePtr[0] << 24);
+}
+
+#else
+XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
+}
+
+static xxh_u32 XXH_readBE32(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
+}
+#endif
+
+XXH_FORCE_INLINE xxh_u32
+XXH_readLE32_align(const void* ptr, XXH_alignment align)
+{
+ if (align==XXH_unaligned) {
+ return XXH_readLE32(ptr);
+ } else {
+ return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr);
+ }
+}
+
+
+/* *************************************
+* Misc
+***************************************/
+/*! @ingroup public */
+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
+
+
+/* *******************************************************************
+* 32-bit hash functions
+*********************************************************************/
+/*!
+ * @}
+ * @defgroup xxh32_impl XXH32 implementation
+ * @ingroup impl
+ * @{
+ */
+ /* #define instead of static const, to be used as initializers */
+#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */
+#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */
+#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */
+#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */
+#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */
+
+#ifdef XXH_OLD_NAMES
+# define PRIME32_1 XXH_PRIME32_1
+# define PRIME32_2 XXH_PRIME32_2
+# define PRIME32_3 XXH_PRIME32_3
+# define PRIME32_4 XXH_PRIME32_4
+# define PRIME32_5 XXH_PRIME32_5
+#endif
+
+/*!
+ * @internal
+ * @brief Normal stripe processing routine.
+ *
+ * This shuffles the bits so that any bit from @p input impacts several bits in
+ * @p acc.
+ *
+ * @param acc The accumulator lane.
+ * @param input The stripe of input to mix.
+ * @return The mixed accumulator lane.
+ */
+static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input)
+{
+ acc += input * XXH_PRIME32_2;
+ acc = XXH_rotl32(acc, 13);
+ acc *= XXH_PRIME32_1;
+#if (defined(__SSE4_1__) || defined(__aarch64__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
+ /*
+ * UGLY HACK:
+ * A compiler fence is the only thing that prevents GCC and Clang from
+ * autovectorizing the XXH32 loop (pragmas and attributes don't work for some
+ * reason) without globally disabling SSE4.1.
+ *
+ * The reason we want to avoid vectorization is because despite working on
+ * 4 integers at a time, there are multiple factors slowing XXH32 down on
+ * SSE4:
+ * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on
+ * newer chips!) making it slightly slower to multiply four integers at
+ * once compared to four integers independently. Even when pmulld was
+ * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE
+ * just to multiply unless doing a long operation.
+ *
+ * - Four instructions are required to rotate,
+ * movqda tmp, v // not required with VEX encoding
+ * pslld tmp, 13 // tmp <<= 13
+ * psrld v, 19 // x >>= 19
+ * por v, tmp // x |= tmp
+ * compared to one for scalar:
+ * roll v, 13 // reliably fast across the board
+ * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason
+ *
+ * - Instruction level parallelism is actually more beneficial here because
+ * the SIMD actually serializes this operation: While v1 is rotating, v2
+ * can load data, while v3 can multiply. SSE forces them to operate
+ * together.
+ *
+ * This is also enabled on AArch64, as Clang autovectorizes it incorrectly
+ * and it is pointless writing a NEON implementation that is basically the
+ * same speed as scalar for XXH32.
+ */
+ XXH_COMPILER_GUARD(acc);
+#endif
+ return acc;
+}
+
+/*!
+ * @internal
+ * @brief Mixes all bits to finalize the hash.
+ *
+ * The final mix ensures that all input bits have a chance to impact any bit in
+ * the output digest, resulting in an unbiased distribution.
+ *
+ * @param h32 The hash to avalanche.
+ * @return The avalanched hash.
+ */
+static xxh_u32 XXH32_avalanche(xxh_u32 h32)
+{
+ h32 ^= h32 >> 15;
+ h32 *= XXH_PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= XXH_PRIME32_3;
+ h32 ^= h32 >> 16;
+ return(h32);
+}
+
+#define XXH_get32bits(p) XXH_readLE32_align(p, align)
+
+/*!
+ * @internal
+ * @brief Processes the last 0-15 bytes of @p ptr.
+ *
+ * There may be up to 15 bytes remaining to consume from the input.
+ * This final stage will digest them to ensure that all input bytes are present
+ * in the final mix.
+ *
+ * @param h32 The hash to finalize.
+ * @param ptr The pointer to the remaining input.
+ * @param len The remaining length, modulo 16.
+ * @param align Whether @p ptr is aligned.
+ * @return The finalized hash.
+ */
+static xxh_u32
+XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align)
+{
+#define XXH_PROCESS1 do { \
+ h32 += (*ptr++) * XXH_PRIME32_5; \
+ h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \
+} while (0)
+
+#define XXH_PROCESS4 do { \
+ h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \
+ ptr += 4; \
+ h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \
+} while (0)
+
+ if (ptr==NULL) XXH_ASSERT(len == 0);
+
+ /* Compact rerolled version; generally faster */
+ if (!XXH32_ENDJMP) {
+ len &= 15;
+ while (len >= 4) {
+ XXH_PROCESS4;
+ len -= 4;
+ }
+ while (len > 0) {
+ XXH_PROCESS1;
+ --len;
+ }
+ return XXH32_avalanche(h32);
+ } else {
+ switch(len&15) /* or switch(bEnd - p) */ {
+ case 12: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 8: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 4: XXH_PROCESS4;
+ return XXH32_avalanche(h32);
+
+ case 13: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 9: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 5: XXH_PROCESS4;
+ XXH_PROCESS1;
+ return XXH32_avalanche(h32);
+
+ case 14: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 10: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 6: XXH_PROCESS4;
+ XXH_PROCESS1;
+ XXH_PROCESS1;
+ return XXH32_avalanche(h32);
+
+ case 15: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 11: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 7: XXH_PROCESS4;
+ XXH_FALLTHROUGH;
+ case 3: XXH_PROCESS1;
+ XXH_FALLTHROUGH;
+ case 2: XXH_PROCESS1;
+ XXH_FALLTHROUGH;
+ case 1: XXH_PROCESS1;
+ XXH_FALLTHROUGH;
+ case 0: return XXH32_avalanche(h32);
+ }
+ XXH_ASSERT(0);
+ return h32; /* reaching this point is deemed impossible */
+ }
+}
+
+#ifdef XXH_OLD_NAMES
+# define PROCESS1 XXH_PROCESS1
+# define PROCESS4 XXH_PROCESS4
+#else
+# undef XXH_PROCESS1
+# undef XXH_PROCESS4
+#endif
+
+/*!
+ * @internal
+ * @brief The implementation for @ref XXH32().
+ *
+ * @param input , len , seed Directly passed from @ref XXH32().
+ * @param align Whether @p input is aligned.
+ * @return The calculated hash.
+ */
+XXH_FORCE_INLINE xxh_u32
+XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
+{
+ xxh_u32 h32;
+
+ if (input==NULL) XXH_ASSERT(len == 0);
+
+ if (len>=16) {
+ const xxh_u8* const bEnd = input + len;
+ const xxh_u8* const limit = bEnd - 15;
+ xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
+ xxh_u32 v2 = seed + XXH_PRIME32_2;
+ xxh_u32 v3 = seed + 0;
+ xxh_u32 v4 = seed - XXH_PRIME32_1;
+
+ do {
+ v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4;
+ v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4;
+ v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4;
+ v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4;
+ } while (input < limit);
+
+ h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7)
+ + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+ } else {
+ h32 = seed + XXH_PRIME32_5;
+ }
+
+ h32 += (xxh_u32)len;
+
+ return XXH32_finalize(h32, input, len&15, align);
+}
+
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
+{
+#if 0
+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+ XXH32_state_t state;
+ XXH32_reset(&state, seed);
+ XXH32_update(&state, (const xxh_u8*)input, len);
+ return XXH32_digest(&state);
+#else
+ if (XXH_FORCE_ALIGN_CHECK) {
+ if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
+ return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
+ } }
+
+ return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
+#endif
+}
+
+
+
+/******* Hash streaming *******/
+/*!
+ * @ingroup xxh32_family
+ */
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
+{
+ return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
+}
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+{
+ XXH_free(statePtr);
+ return XXH_OK;
+}
+
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
+{
+ XXH_memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
+{
+ XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+ memset(&state, 0, sizeof(state));
+ state.v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
+ state.v[1] = seed + XXH_PRIME32_2;
+ state.v[2] = seed + 0;
+ state.v[3] = seed - XXH_PRIME32_1;
+ /* do not write into reserved, planned to be removed in a future version */
+ XXH_memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));
+ return XXH_OK;
+}
+
+
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH32_update(XXH32_state_t* state, const void* input, size_t len)
+{
+ if (input==NULL) {
+ XXH_ASSERT(len == 0);
+ return XXH_OK;
+ }
+
+ { const xxh_u8* p = (const xxh_u8*)input;
+ const xxh_u8* const bEnd = p + len;
+
+ state->total_len_32 += (XXH32_hash_t)len;
+ state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
+
+ if (state->memsize + len < 16) { /* fill in tmp buffer */
+ XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len);
+ state->memsize += (XXH32_hash_t)len;
+ return XXH_OK;
+ }
+
+ if (state->memsize) { /* some data left from previous update */
+ XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize);
+ { const xxh_u32* p32 = state->mem32;
+ state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++;
+ state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++;
+ state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++;
+ state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32));
+ }
+ p += 16-state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p <= bEnd-16) {
+ const xxh_u8* const limit = bEnd - 16;
+
+ do {
+ state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4;
+ state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4;
+ state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4;
+ state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4;
+ } while (p<=limit);
+
+ }
+
+ if (p < bEnd) {
+ XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+ state->memsize = (unsigned)(bEnd-p);
+ }
+ }
+
+ return XXH_OK;
+}
+
+
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state)
+{
+ xxh_u32 h32;
+
+ if (state->large_len) {
+ h32 = XXH_rotl32(state->v[0], 1)
+ + XXH_rotl32(state->v[1], 7)
+ + XXH_rotl32(state->v[2], 12)
+ + XXH_rotl32(state->v[3], 18);
+ } else {
+ h32 = state->v[2] /* == seed */ + XXH_PRIME32_5;
+ }
+
+ h32 += state->total_len_32;
+
+ return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned);
+}
+
+
+/******* Canonical representation *******/
+
+/*!
+ * @ingroup xxh32_family
+ * The default return values from XXH functions are unsigned 32 and 64 bit
+ * integers.
+ *
+ * The canonical representation uses big endian convention, the same convention
+ * as human-readable numbers (large digits first).
+ *
+ * This way, hash values can be written into a file or buffer, remaining
+ * comparable across different systems.
+ *
+ * The following functions allow transformation of hash values to and from their
+ * canonical format.
+ */
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
+ XXH_memcpy(dst, &hash, sizeof(*dst));
+}
+/*! @ingroup xxh32_family */
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
+{
+ return XXH_readBE32(src);
+}
+
+
+#ifndef XXH_NO_LONG_LONG
+
+/* *******************************************************************
+* 64-bit hash functions
+*********************************************************************/
+/*!
+ * @}
+ * @ingroup impl
+ * @{
+ */
+/******* Memory access *******/
+
+typedef XXH64_hash_t xxh_u64;
+
+#ifdef XXH_OLD_NAMES
+# define U64 xxh_u64
+#endif
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+/*
+ * Manual byteshift. Best for old compilers which don't inline memcpy.
+ * We actually directly use XXH_readLE64 and XXH_readBE64.
+ */
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static xxh_u64 XXH_read64(const void* memPtr)
+{
+ return *(const xxh_u64*) memPtr;
+}
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/*
+ * __pack instructions are safer, but compiler specific, hence potentially
+ * problematic for some compilers.
+ *
+ * Currently only defined for GCC and ICC.
+ */
+#ifdef XXH_OLD_NAMES
+typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64;
+#endif
+static xxh_u64 XXH_read64(const void* ptr)
+{
+ typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64;
+ return ((const xxh_unalign64*)ptr)->u64;
+}
+
+#else
+
+/*
+ * Portable and safe solution. Generally efficient.
+ * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
+ */
+static xxh_u64 XXH_read64(const void* memPtr)
+{
+ xxh_u64 val;
+ XXH_memcpy(&val, memPtr, sizeof(val));
+ return val;
+}
+
+#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+#if defined(_MSC_VER) /* Visual Studio */
+# define XXH_swap64 _byteswap_uint64
+#elif XXH_GCC_VERSION >= 403
+# define XXH_swap64 __builtin_bswap64
+#else
+static xxh_u64 XXH_swap64(xxh_u64 x)
+{
+ return ((x << 56) & 0xff00000000000000ULL) |
+ ((x << 40) & 0x00ff000000000000ULL) |
+ ((x << 24) & 0x0000ff0000000000ULL) |
+ ((x << 8) & 0x000000ff00000000ULL) |
+ ((x >> 8) & 0x00000000ff000000ULL) |
+ ((x >> 24) & 0x0000000000ff0000ULL) |
+ ((x >> 40) & 0x000000000000ff00ULL) |
+ ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+
+/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
+
+XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr)
+{
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+ return bytePtr[0]
+ | ((xxh_u64)bytePtr[1] << 8)
+ | ((xxh_u64)bytePtr[2] << 16)
+ | ((xxh_u64)bytePtr[3] << 24)
+ | ((xxh_u64)bytePtr[4] << 32)
+ | ((xxh_u64)bytePtr[5] << 40)
+ | ((xxh_u64)bytePtr[6] << 48)
+ | ((xxh_u64)bytePtr[7] << 56);
+}
+
+XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr)
+{
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
+ return bytePtr[7]
+ | ((xxh_u64)bytePtr[6] << 8)
+ | ((xxh_u64)bytePtr[5] << 16)
+ | ((xxh_u64)bytePtr[4] << 24)
+ | ((xxh_u64)bytePtr[3] << 32)
+ | ((xxh_u64)bytePtr[2] << 40)
+ | ((xxh_u64)bytePtr[1] << 48)
+ | ((xxh_u64)bytePtr[0] << 56);
+}
+
+#else
+XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+}
+
+static xxh_u64 XXH_readBE64(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
+}
+#endif
+
+XXH_FORCE_INLINE xxh_u64
+XXH_readLE64_align(const void* ptr, XXH_alignment align)
+{
+ if (align==XXH_unaligned)
+ return XXH_readLE64(ptr);
+ else
+ return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr);
+}
+
+
+/******* xxh64 *******/
+/*!
+ * @}
+ * @defgroup xxh64_impl XXH64 implementation
+ * @ingroup impl
+ * @{
+ */
+/* #define rather that static const, to be used as initializers */
+#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */
+#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */
+#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */
+#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */
+#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */
+
+#ifdef XXH_OLD_NAMES
+# define PRIME64_1 XXH_PRIME64_1
+# define PRIME64_2 XXH_PRIME64_2
+# define PRIME64_3 XXH_PRIME64_3
+# define PRIME64_4 XXH_PRIME64_4
+# define PRIME64_5 XXH_PRIME64_5
+#endif
+
+static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input)
+{
+ acc += input * XXH_PRIME64_2;
+ acc = XXH_rotl64(acc, 31);
+ acc *= XXH_PRIME64_1;
+ return acc;
+}
+
+static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val)
+{
+ val = XXH64_round(0, val);
+ acc ^= val;
+ acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
+ return acc;
+}
+
+static xxh_u64 XXH64_avalanche(xxh_u64 h64)
+{
+ h64 ^= h64 >> 33;
+ h64 *= XXH_PRIME64_2;
+ h64 ^= h64 >> 29;
+ h64 *= XXH_PRIME64_3;
+ h64 ^= h64 >> 32;
+ return h64;
+}
+
+
+#define XXH_get64bits(p) XXH_readLE64_align(p, align)
+
+static xxh_u64
+XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align)
+{
+ if (ptr==NULL) XXH_ASSERT(len == 0);
+ len &= 31;
+ while (len >= 8) {
+ xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr));
+ ptr += 8;
+ h64 ^= k1;
+ h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4;
+ len -= 8;
+ }
+ if (len >= 4) {
+ h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1;
+ ptr += 4;
+ h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
+ len -= 4;
+ }
+ while (len > 0) {
+ h64 ^= (*ptr++) * XXH_PRIME64_5;
+ h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1;
+ --len;
+ }
+ return XXH64_avalanche(h64);
+}
+
+#ifdef XXH_OLD_NAMES
+# define PROCESS1_64 XXH_PROCESS1_64
+# define PROCESS4_64 XXH_PROCESS4_64
+# define PROCESS8_64 XXH_PROCESS8_64
+#else
+# undef XXH_PROCESS1_64
+# undef XXH_PROCESS4_64
+# undef XXH_PROCESS8_64
+#endif
+
+XXH_FORCE_INLINE xxh_u64
+XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align)
+{
+ xxh_u64 h64;
+ if (input==NULL) XXH_ASSERT(len == 0);
+
+ if (len>=32) {
+ const xxh_u8* const bEnd = input + len;
+ const xxh_u8* const limit = bEnd - 31;
+ xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
+ xxh_u64 v2 = seed + XXH_PRIME64_2;
+ xxh_u64 v3 = seed + 0;
+ xxh_u64 v4 = seed - XXH_PRIME64_1;
+
+ do {
+ v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8;
+ v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8;
+ v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8;
+ v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8;
+ } while (input<limit);
+
+ h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+ h64 = XXH64_mergeRound(h64, v1);
+ h64 = XXH64_mergeRound(h64, v2);
+ h64 = XXH64_mergeRound(h64, v3);
+ h64 = XXH64_mergeRound(h64, v4);
+
+ } else {
+ h64 = seed + XXH_PRIME64_5;
+ }
+
+ h64 += (xxh_u64) len;
+
+ return XXH64_finalize(h64, input, len, align);
+}
+
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t len, XXH64_hash_t seed)
+{
+#if 0
+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+ XXH64_state_t state;
+ XXH64_reset(&state, seed);
+ XXH64_update(&state, (const xxh_u8*)input, len);
+ return XXH64_digest(&state);
+#else
+ if (XXH_FORCE_ALIGN_CHECK) {
+ if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
+ return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
+ } }
+
+ return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
+
+#endif
+}
+
+/******* Hash Streaming *******/
+
+/*! @ingroup xxh64_family*/
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
+{
+ return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+ XXH_free(statePtr);
+ return XXH_OK;
+}
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
+{
+ XXH_memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed)
+{
+ XXH64_state_t state; /* use a local state to memcpy() in order to avoid strict-aliasing warnings */
+ memset(&state, 0, sizeof(state));
+ state.v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
+ state.v[1] = seed + XXH_PRIME64_2;
+ state.v[2] = seed + 0;
+ state.v[3] = seed - XXH_PRIME64_1;
+ /* do not write into reserved64, might be removed in a future version */
+ XXH_memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved64));
+ return XXH_OK;
+}
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH64_update (XXH64_state_t* state, const void* input, size_t len)
+{
+ if (input==NULL) {
+ XXH_ASSERT(len == 0);
+ return XXH_OK;
+ }
+
+ { const xxh_u8* p = (const xxh_u8*)input;
+ const xxh_u8* const bEnd = p + len;
+
+ state->total_len += len;
+
+ if (state->memsize + len < 32) { /* fill in tmp buffer */
+ XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len);
+ state->memsize += (xxh_u32)len;
+ return XXH_OK;
+ }
+
+ if (state->memsize) { /* tmp buffer is full */
+ XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize);
+ state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0));
+ state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1));
+ state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2));
+ state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3));
+ p += 32 - state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p+32 <= bEnd) {
+ const xxh_u8* const limit = bEnd - 32;
+
+ do {
+ state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8;
+ state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8;
+ state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8;
+ state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8;
+ } while (p<=limit);
+
+ }
+
+ if (p < bEnd) {
+ XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+ state->memsize = (unsigned)(bEnd-p);
+ }
+ }
+
+ return XXH_OK;
+}
+
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state)
+{
+ xxh_u64 h64;
+
+ if (state->total_len >= 32) {
+ h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18);
+ h64 = XXH64_mergeRound(h64, state->v[0]);
+ h64 = XXH64_mergeRound(h64, state->v[1]);
+ h64 = XXH64_mergeRound(h64, state->v[2]);
+ h64 = XXH64_mergeRound(h64, state->v[3]);
+ } else {
+ h64 = state->v[2] /*seed*/ + XXH_PRIME64_5;
+ }
+
+ h64 += (xxh_u64) state->total_len;
+
+ return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned);
+}
+
+
+/******* Canonical representation *******/
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
+ XXH_memcpy(dst, &hash, sizeof(*dst));
+}
+
+/*! @ingroup xxh64_family */
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
+{
+ return XXH_readBE64(src);
+}
+
+#ifndef XXH_NO_XXH3
+
+/* *********************************************************************
+* XXH3
+* New generation hash designed for speed on small keys and vectorization
+************************************************************************ */
+/*!
+ * @}
+ * @defgroup xxh3_impl XXH3 implementation
+ * @ingroup impl
+ * @{
+ */
+
+/* === Compiler specifics === */
+
+#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
+# define XXH_RESTRICT /* disable */
+#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */
+# define XXH_RESTRICT restrict
+#else
+/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */
+# define XXH_RESTRICT /* disable */
+#endif
+
+#if (defined(__GNUC__) && (__GNUC__ >= 3)) \
+ || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
+ || defined(__clang__)
+# define XXH_likely(x) __builtin_expect(x, 1)
+# define XXH_unlikely(x) __builtin_expect(x, 0)
+#else
+# define XXH_likely(x) (x)
+# define XXH_unlikely(x) (x)
+#endif
+
+#if defined(__GNUC__)
+# if defined(__AVX2__)
+# include <immintrin.h>
+# elif defined(__SSE2__)
+# include <emmintrin.h>
+# elif defined(__ARM_NEON__) || defined(__ARM_NEON)
+# define inline __inline__ /* circumvent a clang bug */
+# include <arm_neon.h>
+# undef inline
+# endif
+#elif defined(_MSC_VER)
+# include <intrin.h>
+#endif
+
+/*
+ * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while
+ * remaining a true 64-bit/128-bit hash function.
+ *
+ * This is done by prioritizing a subset of 64-bit operations that can be
+ * emulated without too many steps on the average 32-bit machine.
+ *
+ * For example, these two lines seem similar, and run equally fast on 64-bit:
+ *
+ * xxh_u64 x;
+ * x ^= (x >> 47); // good
+ * x ^= (x >> 13); // bad
+ *
+ * However, to a 32-bit machine, there is a major difference.
+ *
+ * x ^= (x >> 47) looks like this:
+ *
+ * x.lo ^= (x.hi >> (47 - 32));
+ *
+ * while x ^= (x >> 13) looks like this:
+ *
+ * // note: funnel shifts are not usually cheap.
+ * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13));
+ * x.hi ^= (x.hi >> 13);
+ *
+ * The first one is significantly faster than the second, simply because the
+ * shift is larger than 32. This means:
+ * - All the bits we need are in the upper 32 bits, so we can ignore the lower
+ * 32 bits in the shift.
+ * - The shift result will always fit in the lower 32 bits, and therefore,
+ * we can ignore the upper 32 bits in the xor.
+ *
+ * Thanks to this optimization, XXH3 only requires these features to be efficient:
+ *
+ * - Usable unaligned access
+ * - A 32-bit or 64-bit ALU
+ * - If 32-bit, a decent ADC instruction
+ * - A 32 or 64-bit multiply with a 64-bit result
+ * - For the 128-bit variant, a decent byteswap helps short inputs.
+ *
+ * The first two are already required by XXH32, and almost all 32-bit and 64-bit
+ * platforms which can run XXH32 can run XXH3 efficiently.
+ *
+ * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one
+ * notable exception.
+ *
+ * First of all, Thumb-1 lacks support for the UMULL instruction which
+ * performs the important long multiply. This means numerous __aeabi_lmul
+ * calls.
+ *
+ * Second of all, the 8 functional registers are just not enough.
+ * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
+ * Lo registers, and this shuffling results in thousands more MOVs than A32.
+ *
+ * A32 and T32 don't have this limitation. They can access all 14 registers,
+ * do a 32->64 multiply with UMULL, and the flexible operand allowing free
+ * shifts is helpful, too.
+ *
+ * Therefore, we do a quick sanity check.
+ *
+ * If compiling Thumb-1 for a target which supports ARM instructions, we will
+ * emit a warning, as it is not a "sane" platform to compile for.
+ *
+ * Usually, if this happens, it is because of an accident and you probably need
+ * to specify -march, as you likely meant to compile for a newer architecture.
+ *
+ * Credit: large sections of the vectorial and asm source code paths
+ * have been contributed by @easyaspi314
+ */
+#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
+# warning "XXH3 is highly inefficient without ARM or Thumb-2."
+#endif
+
+/* ==========================================
+ * Vectorization detection
+ * ========================================== */
+
+#ifdef XXH_DOXYGEN
+/*!
+ * @ingroup tuning
+ * @brief Overrides the vectorization implementation chosen for XXH3.
+ *
+ * Can be defined to 0 to disable SIMD or any of the values mentioned in
+ * @ref XXH_VECTOR_TYPE.
+ *
+ * If this is not defined, it uses predefined macros to determine the best
+ * implementation.
+ */
+# define XXH_VECTOR XXH_SCALAR
+/*!
+ * @ingroup tuning
+ * @brief Possible values for @ref XXH_VECTOR.
+ *
+ * Note that these are actually implemented as macros.
+ *
+ * If this is not defined, it is detected automatically.
+ * @ref XXH_X86DISPATCH overrides this.
+ */
+enum XXH_VECTOR_TYPE /* fake enum */ {
+ XXH_SCALAR = 0, /*!< Portable scalar version */
+ XXH_SSE2 = 1, /*!<
+ * SSE2 for Pentium 4, Opteron, all x86_64.
+ *
+ * @note SSE2 is also guaranteed on Windows 10, macOS, and
+ * Android x86.
+ */
+ XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */
+ XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */
+ XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */
+ XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */
+};
+/*!
+ * @ingroup tuning
+ * @brief Selects the minimum alignment for XXH3's accumulators.
+ *
+ * When using SIMD, this should match the alignment reqired for said vector
+ * type, so, for example, 32 for AVX2.
+ *
+ * Default: Auto detected.
+ */
+# define XXH_ACC_ALIGN 8
+#endif
+
+/* Actual definition */
+#ifndef XXH_DOXYGEN
+# define XXH_SCALAR 0
+# define XXH_SSE2 1
+# define XXH_AVX2 2
+# define XXH_AVX512 3
+# define XXH_NEON 4
+# define XXH_VSX 5
+#endif
+
+#ifndef XXH_VECTOR /* can be defined on command line */
+# if defined(__AVX512F__)
+# define XXH_VECTOR XXH_AVX512
+# elif defined(__AVX2__)
+# define XXH_VECTOR XXH_AVX2
+# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
+# define XXH_VECTOR XXH_SSE2
+# elif ( \
+ defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \
+ || defined(_M_ARM64) || defined(_M_ARM_ARMV7VE) /* msvc */ \
+ ) && ( \
+ defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
+ )
+# define XXH_VECTOR XXH_NEON
+# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
+ || (defined(__s390x__) && defined(__VEC__)) \
+ && defined(__GNUC__) /* TODO: IBM XL */
+# define XXH_VECTOR XXH_VSX
+# else
+# define XXH_VECTOR XXH_SCALAR
+# endif
+#endif
+
+/*
+ * Controls the alignment of the accumulator,
+ * for compatibility with aligned vector loads, which are usually faster.
+ */
+#ifndef XXH_ACC_ALIGN
+# if defined(XXH_X86DISPATCH)
+# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */
+# elif XXH_VECTOR == XXH_SCALAR /* scalar */
+# define XXH_ACC_ALIGN 8
+# elif XXH_VECTOR == XXH_SSE2 /* sse2 */
+# define XXH_ACC_ALIGN 16
+# elif XXH_VECTOR == XXH_AVX2 /* avx2 */
+# define XXH_ACC_ALIGN 32
+# elif XXH_VECTOR == XXH_NEON /* neon */
+# define XXH_ACC_ALIGN 16
+# elif XXH_VECTOR == XXH_VSX /* vsx */
+# define XXH_ACC_ALIGN 16
+# elif XXH_VECTOR == XXH_AVX512 /* avx512 */
+# define XXH_ACC_ALIGN 64
+# endif
+#endif
+
+#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \
+ || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512
+# define XXH_SEC_ALIGN XXH_ACC_ALIGN
+#else
+# define XXH_SEC_ALIGN 8
+#endif
+
+/*
+ * UGLY HACK:
+ * GCC usually generates the best code with -O3 for xxHash.
+ *
+ * However, when targeting AVX2, it is overzealous in its unrolling resulting
+ * in code roughly 3/4 the speed of Clang.
+ *
+ * There are other issues, such as GCC splitting _mm256_loadu_si256 into
+ * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which
+ * only applies to Sandy and Ivy Bridge... which don't even support AVX2.
+ *
+ * That is why when compiling the AVX2 version, it is recommended to use either
+ * -O2 -mavx2 -march=haswell
+ * or
+ * -O2 -mavx2 -mno-avx256-split-unaligned-load
+ * for decent performance, or to use Clang instead.
+ *
+ * Fortunately, we can control the first one with a pragma that forces GCC into
+ * -O2, but the other one we can't control without "failed to inline always
+ * inline function due to target mismatch" warnings.
+ */
+#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
+ && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+ && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */
+# pragma GCC push_options
+# pragma GCC optimize("-O2")
+#endif
+
+
+#if XXH_VECTOR == XXH_NEON
+/*
+ * NEON's setup for vmlal_u32 is a little more complicated than it is on
+ * SSE2, AVX2, and VSX.
+ *
+ * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast.
+ *
+ * To do the same operation, the 128-bit 'Q' register needs to be split into
+ * two 64-bit 'D' registers, performing this operation::
+ *
+ * [ a | b ]
+ * | '---------. .--------' |
+ * | x |
+ * | .---------' '--------. |
+ * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ]
+ *
+ * Due to significant changes in aarch64, the fastest method for aarch64 is
+ * completely different than the fastest method for ARMv7-A.
+ *
+ * ARMv7-A treats D registers as unions overlaying Q registers, so modifying
+ * D11 will modify the high half of Q5. This is similar to how modifying AH
+ * will only affect bits 8-15 of AX on x86.
+ *
+ * VZIP takes two registers, and puts even lanes in one register and odd lanes
+ * in the other.
+ *
+ * On ARMv7-A, this strangely modifies both parameters in place instead of
+ * taking the usual 3-operand form.
+ *
+ * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the
+ * lower and upper halves of the Q register to end up with the high and low
+ * halves where we want - all in one instruction.
+ *
+ * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] }
+ *
+ * Unfortunately we need inline assembly for this: Instructions modifying two
+ * registers at once is not possible in GCC or Clang's IR, and they have to
+ * create a copy.
+ *
+ * aarch64 requires a different approach.
+ *
+ * In order to make it easier to write a decent compiler for aarch64, many
+ * quirks were removed, such as conditional execution.
+ *
+ * NEON was also affected by this.
+ *
+ * aarch64 cannot access the high bits of a Q-form register, and writes to a
+ * D-form register zero the high bits, similar to how writes to W-form scalar
+ * registers (or DWORD registers on x86_64) work.
+ *
+ * The formerly free vget_high intrinsics now require a vext (with a few
+ * exceptions)
+ *
+ * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent
+ * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one
+ * operand.
+ *
+ * The equivalent of the VZIP.32 on the lower and upper halves would be this
+ * mess:
+ *
+ * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] }
+ * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] }
+ * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] }
+ *
+ * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN):
+ *
+ * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32);
+ * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF);
+ *
+ * This is available on ARMv7-A, but is less efficient than a single VZIP.32.
+ */
+
+/*!
+ * Function-like macro:
+ * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi)
+ * {
+ * outLo = (uint32x2_t)(in & 0xFFFFFFFF);
+ * outHi = (uint32x2_t)(in >> 32);
+ * in = UNDEFINED;
+ * }
+ */
+# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \
+ && defined(__GNUC__) \
+ && !defined(__aarch64__) && !defined(__arm64__) && !defined(_M_ARM64)
+# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \
+ do { \
+ /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \
+ /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \
+ /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \
+ __asm__("vzip.32 %e0, %f0" : "+w" (in)); \
+ (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \
+ (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \
+ } while (0)
+# else
+# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \
+ do { \
+ (outLo) = vmovn_u64 (in); \
+ (outHi) = vshrn_n_u64 ((in), 32); \
+ } while (0)
+# endif
+#endif /* XXH_VECTOR == XXH_NEON */
+
+/*
+ * VSX and Z Vector helpers.
+ *
+ * This is very messy, and any pull requests to clean this up are welcome.
+ *
+ * There are a lot of problems with supporting VSX and s390x, due to
+ * inconsistent intrinsics, spotty coverage, and multiple endiannesses.
+ */
+#if XXH_VECTOR == XXH_VSX
+# if defined(__s390x__)
+# include <s390intrin.h>
+# else
+/* gcc's altivec.h can have the unwanted consequence to unconditionally
+ * #define bool, vector, and pixel keywords,
+ * with bad consequences for programs already using these keywords for other purposes.
+ * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined.
+ * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler,
+ * but it seems that, in some cases, it isn't.
+ * Force the build macro to be defined, so that keywords are not altered.
+ */
+# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__)
+# define __APPLE_ALTIVEC__
+# endif
+# include <altivec.h>
+# endif
+
+typedef __vector unsigned long long xxh_u64x2;
+typedef __vector unsigned char xxh_u8x16;
+typedef __vector unsigned xxh_u32x4;
+
+# ifndef XXH_VSX_BE
+# if defined(__BIG_ENDIAN__) \
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+# define XXH_VSX_BE 1
+# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
+# warning "-maltivec=be is not recommended. Please use native endianness."
+# define XXH_VSX_BE 1
+# else
+# define XXH_VSX_BE 0
+# endif
+# endif /* !defined(XXH_VSX_BE) */
+
+# if XXH_VSX_BE
+# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
+# define XXH_vec_revb vec_revb
+# else
+/*!
+ * A polyfill for POWER9's vec_revb().
+ */
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
+{
+ xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+ 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
+ return vec_perm(val, val, vByteSwap);
+}
+# endif
+# endif /* XXH_VSX_BE */
+
+/*!
+ * Performs an unaligned vector load and byte swaps it on big endian.
+ */
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr)
+{
+ xxh_u64x2 ret;
+ XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2));
+# if XXH_VSX_BE
+ ret = XXH_vec_revb(ret);
+# endif
+ return ret;
+}
+
+/*
+ * vec_mulo and vec_mule are very problematic intrinsics on PowerPC
+ *
+ * These intrinsics weren't added until GCC 8, despite existing for a while,
+ * and they are endian dependent. Also, their meaning swap depending on version.
+ * */
+# if defined(__s390x__)
+ /* s390x is always big endian, no issue on this platform */
+# define XXH_vec_mulo vec_mulo
+# define XXH_vec_mule vec_mule
+# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw)
+/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */
+# define XXH_vec_mulo __builtin_altivec_vmulouw
+# define XXH_vec_mule __builtin_altivec_vmuleuw
+# else
+/* gcc needs inline assembly */
+/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b)
+{
+ xxh_u64x2 result;
+ __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
+ return result;
+}
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
+{
+ xxh_u64x2 result;
+ __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
+ return result;
+}
+# endif /* XXH_vec_mulo, XXH_vec_mule */
+#endif /* XXH_VECTOR == XXH_VSX */
+
+
+/* prefetch
+ * can be disabled, by declaring XXH_NO_PREFETCH build macro */
+#if defined(XXH_NO_PREFETCH)
+# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
+#else
+# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */
+# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
+# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+# else
+# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
+# endif
+#endif /* XXH_NO_PREFETCH */
+
+
+/* ==========================================
+ * XXH3 default settings
+ * ========================================== */
+
+#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */
+
+#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
+# error "default keyset is not large enough"
+#endif
+
+/*! Pseudorandom secret taken directly from FARSH. */
+XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = {
+ 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
+ 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
+ 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
+ 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
+ 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
+ 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
+ 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
+ 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
+ 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
+ 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
+ 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
+ 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
+};
+
+
+#ifdef XXH_OLD_NAMES
+# define kSecret XXH3_kSecret
+#endif
+
+#ifdef XXH_DOXYGEN
+/*!
+ * @brief Calculates a 32-bit to 64-bit long multiply.
+ *
+ * Implemented as a macro.
+ *
+ * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't
+ * need to (but it shouldn't need to anyways, it is about 7 instructions to do
+ * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we
+ * use that instead of the normal method.
+ *
+ * If you are compiling for platforms like Thumb-1 and don't have a better option,
+ * you may also want to write your own long multiply routine here.
+ *
+ * @param x, y Numbers to be multiplied
+ * @return 64-bit product of the low 32 bits of @p x and @p y.
+ */
+XXH_FORCE_INLINE xxh_u64
+XXH_mult32to64(xxh_u64 x, xxh_u64 y)
+{
+ return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF);
+}
+#elif defined(_MSC_VER) && defined(_M_IX86)
+# include <intrin.h>
+# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y))
+#else
+/*
+ * Downcast + upcast is usually better than masking on older compilers like
+ * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers.
+ *
+ * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands
+ * and perform a full 64x64 multiply -- entirely redundant on 32-bit.
+ */
+# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y))
+#endif
+
+/*!
+ * @brief Calculates a 64->128-bit long multiply.
+ *
+ * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar
+ * version.
+ *
+ * @param lhs , rhs The 64-bit integers to be multiplied
+ * @return The 128-bit result represented in an @ref XXH128_hash_t.
+ */
+static XXH128_hash_t
+XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
+{
+ /*
+ * GCC/Clang __uint128_t method.
+ *
+ * On most 64-bit targets, GCC and Clang define a __uint128_t type.
+ * This is usually the best way as it usually uses a native long 64-bit
+ * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
+ *
+ * Usually.
+ *
+ * Despite being a 32-bit platform, Clang (and emscripten) define this type
+ * despite not having the arithmetic for it. This results in a laggy
+ * compiler builtin call which calculates a full 128-bit multiply.
+ * In that case it is best to use the portable one.
+ * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
+ */
+#if defined(__GNUC__) && !defined(__wasm__) \
+ && defined(__SIZEOF_INT128__) \
+ || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
+
+ __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs;
+ XXH128_hash_t r128;
+ r128.low64 = (xxh_u64)(product);
+ r128.high64 = (xxh_u64)(product >> 64);
+ return r128;
+
+ /*
+ * MSVC for x64's _umul128 method.
+ *
+ * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
+ *
+ * This compiles to single operand MUL on x64.
+ */
+#elif defined(_M_X64) || defined(_M_IA64)
+
+#ifndef _MSC_VER
+# pragma intrinsic(_umul128)
+#endif
+ xxh_u64 product_high;
+ xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
+ XXH128_hash_t r128;
+ r128.low64 = product_low;
+ r128.high64 = product_high;
+ return r128;
+
+ /*
+ * MSVC for ARM64's __umulh method.
+ *
+ * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method.
+ */
+#elif defined(_M_ARM64)
+
+#ifndef _MSC_VER
+# pragma intrinsic(__umulh)
+#endif
+ XXH128_hash_t r128;
+ r128.low64 = lhs * rhs;
+ r128.high64 = __umulh(lhs, rhs);
+ return r128;
+
+#else
+ /*
+ * Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
+ *
+ * This is a fast and simple grade school multiply, which is shown below
+ * with base 10 arithmetic instead of base 0x100000000.
+ *
+ * 9 3 // D2 lhs = 93
+ * x 7 5 // D2 rhs = 75
+ * ----------
+ * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15
+ * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45
+ * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21
+ * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63
+ * ---------
+ * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27
+ * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67
+ * ---------
+ * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975
+ *
+ * The reasons for adding the products like this are:
+ * 1. It avoids manual carry tracking. Just like how
+ * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX.
+ * This avoids a lot of complexity.
+ *
+ * 2. It hints for, and on Clang, compiles to, the powerful UMAAL
+ * instruction available in ARM's Digital Signal Processing extension
+ * in 32-bit ARMv6 and later, which is shown below:
+ *
+ * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm)
+ * {
+ * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm;
+ * *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
+ * *RdHi = (xxh_u32)(product >> 32);
+ * }
+ *
+ * This instruction was designed for efficient long multiplication, and
+ * allows this to be calculated in only 4 instructions at speeds
+ * comparable to some 64-bit ALUs.
+ *
+ * 3. It isn't terrible on other platforms. Usually this will be a couple
+ * of 32-bit ADD/ADCs.
+ */
+
+ /* First calculate all of the cross products. */
+ xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
+ xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF);
+ xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
+ xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32);
+
+ /* Now add the products together. These will never overflow. */
+ xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
+ xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi;
+ xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);
+
+ XXH128_hash_t r128;
+ r128.low64 = lower;
+ r128.high64 = upper;
+ return r128;
+#endif
+}
+
+/*!
+ * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it.
+ *
+ * The reason for the separate function is to prevent passing too many structs
+ * around by value. This will hopefully inline the multiply, but we don't force it.
+ *
+ * @param lhs , rhs The 64-bit integers to multiply
+ * @return The low 64 bits of the product XOR'd by the high 64 bits.
+ * @see XXH_mult64to128()
+ */
+static xxh_u64
+XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
+{
+ XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
+ return product.low64 ^ product.high64;
+}
+
+/*! Seems to produce slightly better code on GCC for some reason. */
+XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift)
+{
+ XXH_ASSERT(0 <= shift && shift < 64);
+ return v64 ^ (v64 >> shift);
+}
+
+/*
+ * This is a fast avalanche stage,
+ * suitable when input bits are already partially mixed
+ */
+static XXH64_hash_t XXH3_avalanche(xxh_u64 h64)
+{
+ h64 = XXH_xorshift64(h64, 37);
+ h64 *= 0x165667919E3779F9ULL;
+ h64 = XXH_xorshift64(h64, 32);
+ return h64;
+}
+
+/*
+ * This is a stronger avalanche,
+ * inspired by Pelle Evensen's rrmxmx
+ * preferable when input has not been previously mixed
+ */
+static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len)
+{
+ /* this mix is inspired by Pelle Evensen's rrmxmx */
+ h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24);
+ h64 *= 0x9FB21C651E98DF25ULL;
+ h64 ^= (h64 >> 35) + len ;
+ h64 *= 0x9FB21C651E98DF25ULL;
+ return XXH_xorshift64(h64, 28);
+}
+
+
+/* ==========================================
+ * Short keys
+ * ==========================================
+ * One of the shortcomings of XXH32 and XXH64 was that their performance was
+ * sub-optimal on short lengths. It used an iterative algorithm which strongly
+ * favored lengths that were a multiple of 4 or 8.
+ *
+ * Instead of iterating over individual inputs, we use a set of single shot
+ * functions which piece together a range of lengths and operate in constant time.
+ *
+ * Additionally, the number of multiplies has been significantly reduced. This
+ * reduces latency, especially when emulating 64-bit multiplies on 32-bit.
+ *
+ * Depending on the platform, this may or may not be faster than XXH32, but it
+ * is almost guaranteed to be faster than XXH64.
+ */
+
+/*
+ * At very short lengths, there isn't enough input to fully hide secrets, or use
+ * the entire secret.
+ *
+ * There is also only a limited amount of mixing we can do before significantly
+ * impacting performance.
+ *
+ * Therefore, we use different sections of the secret and always mix two secret
+ * samples with an XOR. This should have no effect on performance on the
+ * seedless or withSeed variants because everything _should_ be constant folded
+ * by modern compilers.
+ *
+ * The XOR mixing hides individual parts of the secret and increases entropy.
+ *
+ * This adds an extra layer of strength for custom secrets.
+ */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(1 <= len && len <= 3);
+ XXH_ASSERT(secret != NULL);
+ /*
+ * len = 1: combined = { input[0], 0x01, input[0], input[0] }
+ * len = 2: combined = { input[1], 0x02, input[0], input[1] }
+ * len = 3: combined = { input[2], 0x03, input[0], input[1] }
+ */
+ { xxh_u8 const c1 = input[0];
+ xxh_u8 const c2 = input[len >> 1];
+ xxh_u8 const c3 = input[len - 1];
+ xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24)
+ | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
+ xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
+ xxh_u64 const keyed = (xxh_u64)combined ^ bitflip;
+ return XXH64_avalanche(keyed);
+ }
+}
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(secret != NULL);
+ XXH_ASSERT(4 <= len && len <= 8);
+ seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
+ { xxh_u32 const input1 = XXH_readLE32(input);
+ xxh_u32 const input2 = XXH_readLE32(input + len - 4);
+ xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed;
+ xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32);
+ xxh_u64 const keyed = input64 ^ bitflip;
+ return XXH3_rrmxmx(keyed, len);
+ }
+}
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(secret != NULL);
+ XXH_ASSERT(9 <= len && len <= 16);
+ { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed;
+ xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed;
+ xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1;
+ xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2;
+ xxh_u64 const acc = len
+ + XXH_swap64(input_lo) + input_hi
+ + XXH3_mul128_fold64(input_lo, input_hi);
+ return XXH3_avalanche(acc);
+ }
+}
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(len <= 16);
+ { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed);
+ if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed);
+ if (len) return XXH3_len_1to3_64b(input, len, secret, seed);
+ return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64)));
+ }
+}
+
+/*
+ * DISCLAIMER: There are known *seed-dependent* multicollisions here due to
+ * multiplication by zero, affecting hashes of lengths 17 to 240.
+ *
+ * However, they are very unlikely.
+ *
+ * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
+ * unseeded non-cryptographic hashes, it does not attempt to defend itself
+ * against specially crafted inputs, only random inputs.
+ *
+ * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
+ * cancelling out the secret is taken an arbitrary number of times (addressed
+ * in XXH3_accumulate_512), this collision is very unlikely with random inputs
+ * and/or proper seeding:
+ *
+ * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
+ * function that is only called up to 16 times per hash with up to 240 bytes of
+ * input.
+ *
+ * This is not too bad for a non-cryptographic hash function, especially with
+ * only 64 bit outputs.
+ *
+ * The 128-bit variant (which trades some speed for strength) is NOT affected
+ * by this, although it is always a good idea to use a proper seed if you care
+ * about strength.
+ */
+XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
+ const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
+{
+#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+ && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \
+ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */
+ /*
+ * UGLY HACK:
+ * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in
+ * slower code.
+ *
+ * By forcing seed64 into a register, we disrupt the cost model and
+ * cause it to scalarize. See `XXH32_round()`
+ *
+ * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600,
+ * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on
+ * GCC 9.2, despite both emitting scalar code.
+ *
+ * GCC generates much better scalar code than Clang for the rest of XXH3,
+ * which is why finding a more optimal codepath is an interest.
+ */
+ XXH_COMPILER_GUARD(seed64);
+#endif
+ { xxh_u64 const input_lo = XXH_readLE64(input);
+ xxh_u64 const input_hi = XXH_readLE64(input+8);
+ return XXH3_mul128_fold64(
+ input_lo ^ (XXH_readLE64(secret) + seed64),
+ input_hi ^ (XXH_readLE64(secret+8) - seed64)
+ );
+ }
+}
+
+/* For mid range keys, XXH3 uses a Mum-hash variant. */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH64_hash_t seed)
+{
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+ XXH_ASSERT(16 < len && len <= 128);
+
+ { xxh_u64 acc = len * XXH_PRIME64_1;
+ if (len > 32) {
+ if (len > 64) {
+ if (len > 96) {
+ acc += XXH3_mix16B(input+48, secret+96, seed);
+ acc += XXH3_mix16B(input+len-64, secret+112, seed);
+ }
+ acc += XXH3_mix16B(input+32, secret+64, seed);
+ acc += XXH3_mix16B(input+len-48, secret+80, seed);
+ }
+ acc += XXH3_mix16B(input+16, secret+32, seed);
+ acc += XXH3_mix16B(input+len-32, secret+48, seed);
+ }
+ acc += XXH3_mix16B(input+0, secret+0, seed);
+ acc += XXH3_mix16B(input+len-16, secret+16, seed);
+
+ return XXH3_avalanche(acc);
+ }
+}
+
+#define XXH3_MIDSIZE_MAX 240
+
+XXH_NO_INLINE XXH64_hash_t
+XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH64_hash_t seed)
+{
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+ XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
+
+ #define XXH3_MIDSIZE_STARTOFFSET 3
+ #define XXH3_MIDSIZE_LASTOFFSET 17
+
+ { xxh_u64 acc = len * XXH_PRIME64_1;
+ int const nbRounds = (int)len / 16;
+ int i;
+ for (i=0; i<8; i++) {
+ acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed);
+ }
+ acc = XXH3_avalanche(acc);
+ XXH_ASSERT(nbRounds >= 8);
+#if defined(__clang__) /* Clang */ \
+ && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
+ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
+ /*
+ * UGLY HACK:
+ * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86.
+ * In everywhere else, it uses scalar code.
+ *
+ * For 64->128-bit multiplies, even if the NEON was 100% optimal, it
+ * would still be slower than UMAAL (see XXH_mult64to128).
+ *
+ * Unfortunately, Clang doesn't handle the long multiplies properly and
+ * converts them to the nonexistent "vmulq_u64" intrinsic, which is then
+ * scalarized into an ugly mess of VMOV.32 instructions.
+ *
+ * This mess is difficult to avoid without turning autovectorization
+ * off completely, but they are usually relatively minor and/or not
+ * worth it to fix.
+ *
+ * This loop is the easiest to fix, as unlike XXH32, this pragma
+ * _actually works_ because it is a loop vectorization instead of an
+ * SLP vectorization.
+ */
+ #pragma clang loop vectorize(disable)
+#endif
+ for (i=8 ; i < nbRounds; i++) {
+ acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
+ }
+ /* last bytes */
+ acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
+ return XXH3_avalanche(acc);
+ }
+}
+
+
+/* ======= Long Keys ======= */
+
+#define XXH_STRIPE_LEN 64
+#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */
+#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64))
+
+#ifdef XXH_OLD_NAMES
+# define STRIPE_LEN XXH_STRIPE_LEN
+# define ACC_NB XXH_ACC_NB
+#endif
+
+XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
+{
+ if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
+ XXH_memcpy(dst, &v64, sizeof(v64));
+}
+
+/* Several intrinsic functions below are supposed to accept __int64 as argument,
+ * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ .
+ * However, several environments do not define __int64 type,
+ * requiring a workaround.
+ */
+#if !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+ typedef int64_t xxh_i64;
+#else
+ /* the following type must have a width of 64-bit */
+ typedef long long xxh_i64;
+#endif
+
+/*
+ * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized.
+ *
+ * It is a hardened version of UMAC, based off of FARSH's implementation.
+ *
+ * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD
+ * implementations, and it is ridiculously fast.
+ *
+ * We harden it by mixing the original input to the accumulators as well as the product.
+ *
+ * This means that in the (relatively likely) case of a multiply by zero, the
+ * original input is preserved.
+ *
+ * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve
+ * cross-pollination, as otherwise the upper and lower halves would be
+ * essentially independent.
+ *
+ * This doesn't matter on 64-bit hashes since they all get merged together in
+ * the end, so we skip the extra step.
+ *
+ * Both XXH3_64bits and XXH3_128bits use this subroutine.
+ */
+
+#if (XXH_VECTOR == XXH_AVX512) \
+ || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0)
+
+#ifndef XXH_TARGET_AVX512
+# define XXH_TARGET_AVX512 /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ __m512i* const xacc = (__m512i *) acc;
+ XXH_ASSERT((((size_t)acc) & 63) == 0);
+ XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
+
+ {
+ /* data_vec = input[0]; */
+ __m512i const data_vec = _mm512_loadu_si512 (input);
+ /* key_vec = secret[0]; */
+ __m512i const key_vec = _mm512_loadu_si512 (secret);
+ /* data_key = data_vec ^ key_vec; */
+ __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec);
+ /* data_key_lo = data_key >> 32; */
+ __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1));
+ /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+ __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo);
+ /* xacc[0] += swap(data_vec); */
+ __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2));
+ __m512i const sum = _mm512_add_epi64(*xacc, data_swap);
+ /* xacc[0] += product; */
+ *xacc = _mm512_add_epi64(product, sum);
+ }
+}
+
+/*
+ * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing.
+ *
+ * Multiplication isn't perfect, as explained by Google in HighwayHash:
+ *
+ * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
+ * // varying degrees. In descending order of goodness, bytes
+ * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
+ * // As expected, the upper and lower bytes are much worse.
+ *
+ * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291
+ *
+ * Since our algorithm uses a pseudorandom secret to add some variance into the
+ * mix, we don't need to (or want to) mix as often or as much as HighwayHash does.
+ *
+ * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid
+ * extraction.
+ *
+ * Both XXH3_64bits and XXH3_128bits use this subroutine.
+ */
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 63) == 0);
+ XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
+ { __m512i* const xacc = (__m512i*) acc;
+ const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1);
+
+ /* xacc[0] ^= (xacc[0] >> 47) */
+ __m512i const acc_vec = *xacc;
+ __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47);
+ __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted);
+ /* xacc[0] ^= secret; */
+ __m512i const key_vec = _mm512_loadu_si512 (secret);
+ __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec);
+
+ /* xacc[0] *= XXH_PRIME32_1; */
+ __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1));
+ __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32);
+ __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32);
+ *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32));
+ }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
+XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0);
+ XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64);
+ XXH_ASSERT(((size_t)customSecret & 63) == 0);
+ (void)(&XXH_writeLE64);
+ { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
+ __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64));
+
+ const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret);
+ __m512i* const dest = ( __m512i*) customSecret;
+ int i;
+ XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */
+ XXH_ASSERT(((size_t)dest & 63) == 0);
+ for (i=0; i < nbRounds; ++i) {
+ /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*',
+ * this will warn "discards 'const' qualifier". */
+ union {
+ const __m512i* cp;
+ void* p;
+ } remote_const_void;
+ remote_const_void.cp = src + i;
+ dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed);
+ } }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_AVX2) \
+ || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0)
+
+#ifndef XXH_TARGET_AVX2
+# define XXH_TARGET_AVX2 /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void
+XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 31) == 0);
+ { __m256i* const xacc = (__m256i *) acc;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
+ const __m256i* const xinput = (const __m256i *) input;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
+ const __m256i* const xsecret = (const __m256i *) secret;
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
+ /* data_vec = xinput[i]; */
+ __m256i const data_vec = _mm256_loadu_si256 (xinput+i);
+ /* key_vec = xsecret[i]; */
+ __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
+ /* data_key = data_vec ^ key_vec; */
+ __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
+ /* data_key_lo = data_key >> 32; */
+ __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+ /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+ __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo);
+ /* xacc[i] += swap(data_vec); */
+ __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2));
+ __m256i const sum = _mm256_add_epi64(xacc[i], data_swap);
+ /* xacc[i] += product; */
+ xacc[i] = _mm256_add_epi64(product, sum);
+ } }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void
+XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 31) == 0);
+ { __m256i* const xacc = (__m256i*) acc;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
+ const __m256i* const xsecret = (const __m256i *) secret;
+ const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1);
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
+ /* xacc[i] ^= (xacc[i] >> 47) */
+ __m256i const acc_vec = xacc[i];
+ __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47);
+ __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted);
+ /* xacc[i] ^= xsecret; */
+ __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
+ __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
+
+ /* xacc[i] *= XXH_PRIME32_1; */
+ __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+ __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32);
+ __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32);
+ xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
+ }
+ }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0);
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6);
+ XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64);
+ (void)(&XXH_writeLE64);
+ XXH_PREFETCH(customSecret);
+ { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64);
+
+ const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret);
+ __m256i* dest = ( __m256i*) customSecret;
+
+# if defined(__GNUC__) || defined(__clang__)
+ /*
+ * On GCC & Clang, marking 'dest' as modified will cause the compiler:
+ * - do not extract the secret from sse registers in the internal loop
+ * - use less common registers, and avoid pushing these reg into stack
+ */
+ XXH_COMPILER_GUARD(dest);
+# endif
+ XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */
+ XXH_ASSERT(((size_t)dest & 31) == 0);
+
+ /* GCC -O2 need unroll loop manually */
+ dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed);
+ dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed);
+ dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed);
+ dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed);
+ dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed);
+ dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed);
+ }
+}
+
+#endif
+
+/* x86dispatch always generates SSE2 */
+#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH)
+
+#ifndef XXH_TARGET_SSE2
+# define XXH_TARGET_SSE2 /* disable attribute target */
+#endif
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void
+XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ /* SSE2 is just a half-scale version of the AVX2 version. */
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+ { __m128i* const xacc = (__m128i *) acc;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+ const __m128i* const xinput = (const __m128i *) input;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+ const __m128i* const xsecret = (const __m128i *) secret;
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
+ /* data_vec = xinput[i]; */
+ __m128i const data_vec = _mm_loadu_si128 (xinput+i);
+ /* key_vec = xsecret[i]; */
+ __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
+ /* data_key = data_vec ^ key_vec; */
+ __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
+ /* data_key_lo = data_key >> 32; */
+ __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+ /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+ __m128i const product = _mm_mul_epu32 (data_key, data_key_lo);
+ /* xacc[i] += swap(data_vec); */
+ __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
+ __m128i const sum = _mm_add_epi64(xacc[i], data_swap);
+ /* xacc[i] += product; */
+ xacc[i] = _mm_add_epi64(product, sum);
+ } }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void
+XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+ { __m128i* const xacc = (__m128i*) acc;
+ /* Unaligned. This is mainly for pointer arithmetic, and because
+ * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
+ const __m128i* const xsecret = (const __m128i *) secret;
+ const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1);
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
+ /* xacc[i] ^= (xacc[i] >> 47) */
+ __m128i const acc_vec = xacc[i];
+ __m128i const shifted = _mm_srli_epi64 (acc_vec, 47);
+ __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted);
+ /* xacc[i] ^= xsecret[i]; */
+ __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
+ __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
+
+ /* xacc[i] *= XXH_PRIME32_1; */
+ __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
+ __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32);
+ __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32);
+ xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
+ }
+ }
+}
+
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
+ (void)(&XXH_writeLE64);
+ { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i);
+
+# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900
+ /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */
+ XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) };
+ __m128i const seed = _mm_load_si128((__m128i const*)seed64x2);
+# else
+ __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64);
+# endif
+ int i;
+
+ const void* const src16 = XXH3_kSecret;
+ __m128i* dst16 = (__m128i*) customSecret;
+# if defined(__GNUC__) || defined(__clang__)
+ /*
+ * On GCC & Clang, marking 'dest' as modified will cause the compiler:
+ * - do not extract the secret from sse registers in the internal loop
+ * - use less common registers, and avoid pushing these reg into stack
+ */
+ XXH_COMPILER_GUARD(dst16);
+# endif
+ XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */
+ XXH_ASSERT(((size_t)dst16 & 15) == 0);
+
+ for (i=0; i < nbRounds; ++i) {
+ dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed);
+ } }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_NEON)
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_neon( void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+ {
+ uint64x2_t* const xacc = (uint64x2_t *) acc;
+ /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
+ uint8_t const* const xinput = (const uint8_t *) input;
+ uint8_t const* const xsecret = (const uint8_t *) secret;
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) {
+ /* data_vec = xinput[i]; */
+ uint8x16_t data_vec = vld1q_u8(xinput + (i * 16));
+ /* key_vec = xsecret[i]; */
+ uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16));
+ uint64x2_t data_key;
+ uint32x2_t data_key_lo, data_key_hi;
+ /* xacc[i] += swap(data_vec); */
+ uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec);
+ uint64x2_t const swapped = vextq_u64(data64, data64, 1);
+ xacc[i] = vaddq_u64 (xacc[i], swapped);
+ /* data_key = data_vec ^ key_vec; */
+ data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec));
+ /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF);
+ * data_key_hi = (uint32x2_t) (data_key >> 32);
+ * data_key = UNDEFINED; */
+ XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi);
+ /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */
+ xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi);
+
+ }
+ }
+}
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+
+ { uint64x2_t* xacc = (uint64x2_t*) acc;
+ uint8_t const* xsecret = (uint8_t const*) secret;
+ uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1);
+
+ size_t i;
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(uint64x2_t); i++) {
+ /* xacc[i] ^= (xacc[i] >> 47); */
+ uint64x2_t acc_vec = xacc[i];
+ uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47);
+ uint64x2_t data_vec = veorq_u64 (acc_vec, shifted);
+
+ /* xacc[i] ^= xsecret[i]; */
+ uint8x16_t key_vec = vld1q_u8 (xsecret + (i * 16));
+ uint64x2_t data_key = veorq_u64 (data_vec, vreinterpretq_u64_u8(key_vec));
+
+ /* xacc[i] *= XXH_PRIME32_1 */
+ uint32x2_t data_key_lo, data_key_hi;
+ /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF);
+ * data_key_hi = (uint32x2_t) (xacc[i] >> 32);
+ * xacc[i] = UNDEFINED; */
+ XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi);
+ { /*
+ * prod_hi = (data_key >> 32) * XXH_PRIME32_1;
+ *
+ * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will
+ * incorrectly "optimize" this:
+ * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b));
+ * shifted = vshll_n_u32(tmp, 32);
+ * to this:
+ * tmp = "vmulq_u64"(a, b); // no such thing!
+ * shifted = vshlq_n_u64(tmp, 32);
+ *
+ * However, unlike SSE, Clang lacks a 64-bit multiply routine
+ * for NEON, and it scalarizes two 64-bit multiplies instead.
+ *
+ * vmull_u32 has the same timing as vmul_u32, and it avoids
+ * this bug completely.
+ * See https://bugs.llvm.org/show_bug.cgi?id=39967
+ */
+ uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime);
+ /* xacc[i] = prod_hi << 32; */
+ xacc[i] = vshlq_n_u64(prod_hi, 32);
+ /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */
+ xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime);
+ }
+ } }
+}
+
+#endif
+
+#if (XXH_VECTOR == XXH_VSX)
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ /* presumed aligned */
+ unsigned long long* const xacc = (unsigned long long*) acc;
+ xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */
+ xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */
+ xxh_u64x2 const v32 = { 32, 32 };
+ size_t i;
+ for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
+ /* data_vec = xinput[i]; */
+ xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i);
+ /* key_vec = xsecret[i]; */
+ xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i);
+ xxh_u64x2 const data_key = data_vec ^ key_vec;
+ /* shuffled = (data_key << 32) | (data_key >> 32); */
+ xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32);
+ /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */
+ xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled);
+ /* acc_vec = xacc[i]; */
+ xxh_u64x2 acc_vec = vec_xl(0, xacc + 2 * i);
+ acc_vec += product;
+
+ /* swap high and low halves */
+#ifdef __s390x__
+ acc_vec += vec_permi(data_vec, data_vec, 2);
+#else
+ acc_vec += vec_xxpermdi(data_vec, data_vec, 2);
+#endif
+ /* xacc[i] = acc_vec; */
+ vec_xst(acc_vec, 0, xacc + 2 * i);
+ }
+}
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+
+ { xxh_u64x2* const xacc = (xxh_u64x2*) acc;
+ const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret;
+ /* constants */
+ xxh_u64x2 const v32 = { 32, 32 };
+ xxh_u64x2 const v47 = { 47, 47 };
+ xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 };
+ size_t i;
+ for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
+ /* xacc[i] ^= (xacc[i] >> 47); */
+ xxh_u64x2 const acc_vec = xacc[i];
+ xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47);
+
+ /* xacc[i] ^= xsecret[i]; */
+ xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i);
+ xxh_u64x2 const data_key = data_vec ^ key_vec;
+
+ /* xacc[i] *= XXH_PRIME32_1 */
+ /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */
+ xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime);
+ /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */
+ xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime);
+ xacc[i] = prod_odd + (prod_even << v32);
+ } }
+}
+
+#endif
+
+/* scalar variants - universal */
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */
+ const xxh_u8* const xinput = (const xxh_u8*) input; /* no alignment restriction */
+ const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */
+ size_t i;
+ XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
+ for (i=0; i < XXH_ACC_NB; i++) {
+ xxh_u64 const data_val = XXH_readLE64(xinput + 8*i);
+ xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i*8);
+ xacc[i ^ 1] += data_val; /* swap adjacent lanes */
+ xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32);
+ }
+}
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */
+ const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */
+ size_t i;
+ XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
+ for (i=0; i < XXH_ACC_NB; i++) {
+ xxh_u64 const key64 = XXH_readLE64(xsecret + 8*i);
+ xxh_u64 acc64 = xacc[i];
+ acc64 = XXH_xorshift64(acc64, 47);
+ acc64 ^= key64;
+ acc64 *= XXH_PRIME32_1;
+ xacc[i] = acc64;
+ }
+}
+
+XXH_FORCE_INLINE void
+XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
+{
+ /*
+ * We need a separate pointer for the hack below,
+ * which requires a non-const pointer.
+ * Any decent compiler will optimize this out otherwise.
+ */
+ const xxh_u8* kSecretPtr = XXH3_kSecret;
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
+
+#if defined(__clang__) && defined(__aarch64__)
+ /*
+ * UGLY HACK:
+ * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are
+ * placed sequentially, in order, at the top of the unrolled loop.
+ *
+ * While MOVK is great for generating constants (2 cycles for a 64-bit
+ * constant compared to 4 cycles for LDR), long MOVK chains stall the
+ * integer pipelines:
+ * I L S
+ * MOVK
+ * MOVK
+ * MOVK
+ * MOVK
+ * ADD
+ * SUB STR
+ * STR
+ * By forcing loads from memory (as the asm line causes Clang to assume
+ * that XXH3_kSecretPtr has been changed), the pipelines are used more
+ * efficiently:
+ * I L S
+ * LDR
+ * ADD LDR
+ * SUB STR
+ * STR
+ * XXH3_64bits_withSeed, len == 256, Snapdragon 835
+ * without hack: 2654.4 MB/s
+ * with hack: 3202.9 MB/s
+ */
+ XXH_COMPILER_GUARD(kSecretPtr);
+#endif
+ /*
+ * Note: in debug mode, this overrides the asm optimization
+ * and Clang will emit MOVK chains again.
+ */
+ XXH_ASSERT(kSecretPtr == XXH3_kSecret);
+
+ { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
+ int i;
+ for (i=0; i < nbRounds; i++) {
+ /*
+ * The asm hack causes Clang to assume that kSecretPtr aliases with
+ * customSecret, and on aarch64, this prevented LDP from merging two
+ * loads together for free. Putting the loads together before the stores
+ * properly generates LDP.
+ */
+ xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64;
+ xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64;
+ XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo);
+ XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi);
+ } }
+}
+
+
+typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*);
+typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*);
+typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64);
+
+
+#if (XXH_VECTOR == XXH_AVX512)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_avx512
+#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512
+#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512
+
+#elif (XXH_VECTOR == XXH_AVX2)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_avx2
+#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2
+#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2
+
+#elif (XXH_VECTOR == XXH_SSE2)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_sse2
+#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2
+#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2
+
+#elif (XXH_VECTOR == XXH_NEON)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_neon
+#define XXH3_scrambleAcc XXH3_scrambleAcc_neon
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#elif (XXH_VECTOR == XXH_VSX)
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_vsx
+#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#else /* scalar */
+
+#define XXH3_accumulate_512 XXH3_accumulate_512_scalar
+#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
+#endif
+
+
+
+#ifndef XXH_PREFETCH_DIST
+# ifdef __clang__
+# define XXH_PREFETCH_DIST 320
+# else
+# if (XXH_VECTOR == XXH_AVX512)
+# define XXH_PREFETCH_DIST 512
+# else
+# define XXH_PREFETCH_DIST 384
+# endif
+# endif /* __clang__ */
+#endif /* XXH_PREFETCH_DIST */
+
+/*
+ * XXH3_accumulate()
+ * Loops over XXH3_accumulate_512().
+ * Assumption: nbStripes will not overflow the secret size
+ */
+XXH_FORCE_INLINE void
+XXH3_accumulate( xxh_u64* XXH_RESTRICT acc,
+ const xxh_u8* XXH_RESTRICT input,
+ const xxh_u8* XXH_RESTRICT secret,
+ size_t nbStripes,
+ XXH3_f_accumulate_512 f_acc512)
+{
+ size_t n;
+ for (n = 0; n < nbStripes; n++ ) {
+ const xxh_u8* const in = input + n*XXH_STRIPE_LEN;
+ XXH_PREFETCH(in + XXH_PREFETCH_DIST);
+ f_acc512(acc,
+ in,
+ secret + n*XXH_SECRET_CONSUME_RATE);
+ }
+}
+
+XXH_FORCE_INLINE void
+XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc,
+ const xxh_u8* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble)
+{
+ size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
+ size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock;
+ size_t const nb_blocks = (len - 1) / block_len;
+
+ size_t n;
+
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
+
+ for (n = 0; n < nb_blocks; n++) {
+ XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512);
+ f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN);
+ }
+
+ /* last partial block */
+ XXH_ASSERT(len > XXH_STRIPE_LEN);
+ { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
+ XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
+ XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512);
+
+ /* last stripe */
+ { const xxh_u8* const p = input + len - XXH_STRIPE_LEN;
+#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */
+ f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START);
+ } }
+}
+
+XXH_FORCE_INLINE xxh_u64
+XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret)
+{
+ return XXH3_mul128_fold64(
+ acc[0] ^ XXH_readLE64(secret),
+ acc[1] ^ XXH_readLE64(secret+8) );
+}
+
+static XXH64_hash_t
+XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
+{
+ xxh_u64 result64 = start;
+ size_t i = 0;
+
+ for (i = 0; i < 4; i++) {
+ result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i);
+#if defined(__clang__) /* Clang */ \
+ && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \
+ && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
+ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
+ /*
+ * UGLY HACK:
+ * Prevent autovectorization on Clang ARMv7-a. Exact same problem as
+ * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b.
+ * XXH3_64bits, len == 256, Snapdragon 835:
+ * without hack: 2063.7 MB/s
+ * with hack: 2560.7 MB/s
+ */
+ XXH_COMPILER_GUARD(result64);
+#endif
+ }
+
+ return XXH3_avalanche(result64);
+}
+
+#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
+ XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len,
+ const void* XXH_RESTRICT secret, size_t secretSize,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble)
+{
+ XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
+
+ XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble);
+
+ /* converge into final hash */
+ XXH_STATIC_ASSERT(sizeof(acc) == 64);
+ /* do not align on 8, so that the secret is different from the accumulator */
+#define XXH_SECRET_MERGEACCS_START 11
+ XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+ return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1);
+}
+
+/*
+ * It's important for performance to transmit secret's size (when it's static)
+ * so that the compiler can properly optimize the vectorized loop.
+ * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set.
+ */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+ (void)seed64;
+ return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+/*
+ * It's preferable for performance that XXH3_hashLong is not inlined,
+ * as it results in a smaller function for small data, easier to the instruction cache.
+ * Note that inside this no_inline function, we do inline the internal loop,
+ * and provide a statically defined secret size to allow optimization of vector loop.
+ */
+XXH_NO_INLINE XXH64_hash_t
+XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+ (void)seed64; (void)secret; (void)secretLen;
+ return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+/*
+ * XXH3_hashLong_64b_withSeed():
+ * Generate a custom key based on alteration of default XXH3_kSecret with the seed,
+ * and then use this key for long mode hashing.
+ *
+ * This operation is decently fast but nonetheless costs a little bit of time.
+ * Try to avoid it whenever possible (typically when seed==0).
+ *
+ * It's important for performance that XXH3_hashLong is not inlined. Not sure
+ * why (uop cache maybe?), but the difference is large and easily measurable.
+ */
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len,
+ XXH64_hash_t seed,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble,
+ XXH3_f_initCustomSecret f_initSec)
+{
+ if (seed == 0)
+ return XXH3_hashLong_64b_internal(input, len,
+ XXH3_kSecret, sizeof(XXH3_kSecret),
+ f_acc512, f_scramble);
+ { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
+ f_initSec(secret, seed);
+ return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret),
+ f_acc512, f_scramble);
+ }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH64_hash_t
+XXH3_hashLong_64b_withSeed(const void* input, size_t len,
+ XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen)
+{
+ (void)secret; (void)secretLen;
+ return XXH3_hashLong_64b_withSeed_internal(input, len, seed,
+ XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret);
+}
+
+
+typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t,
+ XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t);
+
+XXH_FORCE_INLINE XXH64_hash_t
+XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
+ XXH3_hashLong64_f f_hashLong)
+{
+ XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
+ /*
+ * If an action is to be taken if `secretLen` condition is not respected,
+ * it should be done here.
+ * For now, it's a contract pre-condition.
+ * Adding a check and a branch here would cost performance at every hash.
+ * Also, note that function signature doesn't offer room to return an error.
+ */
+ if (len <= 16)
+ return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
+ if (len <= 128)
+ return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+ if (len <= XXH3_MIDSIZE_MAX)
+ return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+ return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen);
+}
+
+
+/* === Public entry point === */
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len)
+{
+ return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
+{
+ return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
+{
+ return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed);
+}
+
+XXH_PUBLIC_API XXH64_hash_t
+XXH3_64bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed)
+{
+ if (len <= XXH3_MIDSIZE_MAX)
+ return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
+ return XXH3_hashLong_64b_withSecret(input, len, seed, (const xxh_u8*)secret, secretSize);
+}
+
+
+/* === XXH3 streaming === */
+
+/*
+ * Malloc's a pointer that is always aligned to align.
+ *
+ * This must be freed with `XXH_alignedFree()`.
+ *
+ * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte
+ * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2
+ * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON.
+ *
+ * This underalignment previously caused a rather obvious crash which went
+ * completely unnoticed due to XXH3_createState() not actually being tested.
+ * Credit to RedSpah for noticing this bug.
+ *
+ * The alignment is done manually: Functions like posix_memalign or _mm_malloc
+ * are avoided: To maintain portability, we would have to write a fallback
+ * like this anyways, and besides, testing for the existence of library
+ * functions without relying on external build tools is impossible.
+ *
+ * The method is simple: Overallocate, manually align, and store the offset
+ * to the original behind the returned pointer.
+ *
+ * Align must be a power of 2 and 8 <= align <= 128.
+ */
+static void* XXH_alignedMalloc(size_t s, size_t align)
+{
+ XXH_ASSERT(align <= 128 && align >= 8); /* range check */
+ XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */
+ XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */
+ { /* Overallocate to make room for manual realignment and an offset byte */
+ xxh_u8* base = (xxh_u8*)XXH_malloc(s + align);
+ if (base != NULL) {
+ /*
+ * Get the offset needed to align this pointer.
+ *
+ * Even if the returned pointer is aligned, there will always be
+ * at least one byte to store the offset to the original pointer.
+ */
+ size_t offset = align - ((size_t)base & (align - 1)); /* base % align */
+ /* Add the offset for the now-aligned pointer */
+ xxh_u8* ptr = base + offset;
+
+ XXH_ASSERT((size_t)ptr % align == 0);
+
+ /* Store the offset immediately before the returned pointer. */
+ ptr[-1] = (xxh_u8)offset;
+ return ptr;
+ }
+ return NULL;
+ }
+}
+/*
+ * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass
+ * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout.
+ */
+static void XXH_alignedFree(void* p)
+{
+ if (p != NULL) {
+ xxh_u8* ptr = (xxh_u8*)p;
+ /* Get the offset byte we added in XXH_malloc. */
+ xxh_u8 offset = ptr[-1];
+ /* Free the original malloc'd pointer */
+ xxh_u8* base = ptr - offset;
+ XXH_free(base);
+ }
+}
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
+{
+ XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64);
+ if (state==NULL) return NULL;
+ XXH3_INITSTATE(state);
+ return state;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
+{
+ XXH_alignedFree(statePtr);
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API void
+XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state)
+{
+ XXH_memcpy(dst_state, src_state, sizeof(*dst_state));
+}
+
+static void
+XXH3_reset_internal(XXH3_state_t* statePtr,
+ XXH64_hash_t seed,
+ const void* secret, size_t secretSize)
+{
+ size_t const initStart = offsetof(XXH3_state_t, bufferedSize);
+ size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart;
+ XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart);
+ XXH_ASSERT(statePtr != NULL);
+ /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */
+ memset((char*)statePtr + initStart, 0, initLength);
+ statePtr->acc[0] = XXH_PRIME32_3;
+ statePtr->acc[1] = XXH_PRIME64_1;
+ statePtr->acc[2] = XXH_PRIME64_2;
+ statePtr->acc[3] = XXH_PRIME64_3;
+ statePtr->acc[4] = XXH_PRIME64_4;
+ statePtr->acc[5] = XXH_PRIME32_2;
+ statePtr->acc[6] = XXH_PRIME64_5;
+ statePtr->acc[7] = XXH_PRIME32_1;
+ statePtr->seed = seed;
+ statePtr->useSeed = (seed != 0);
+ statePtr->extSecret = (const unsigned char*)secret;
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
+ statePtr->secretLimit = secretSize - XXH_STRIPE_LEN;
+ statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset(XXH3_state_t* statePtr)
+{
+ if (statePtr == NULL) return XXH_ERROR;
+ XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
+{
+ if (statePtr == NULL) return XXH_ERROR;
+ XXH3_reset_internal(statePtr, 0, secret, secretSize);
+ if (secret == NULL) return XXH_ERROR;
+ if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
+{
+ if (statePtr == NULL) return XXH_ERROR;
+ if (seed==0) return XXH3_64bits_reset(statePtr);
+ if ((seed != statePtr->seed) || (statePtr->extSecret != NULL))
+ XXH3_initCustomSecret(statePtr->customSecret, seed);
+ XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE);
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed64)
+{
+ if (statePtr == NULL) return XXH_ERROR;
+ if (secret == NULL) return XXH_ERROR;
+ if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
+ XXH3_reset_internal(statePtr, seed64, secret, secretSize);
+ statePtr->useSeed = 1; /* always, even if seed64==0 */
+ return XXH_OK;
+}
+
+/* Note : when XXH3_consumeStripes() is invoked,
+ * there must be a guarantee that at least one more byte must be consumed from input
+ * so that the function can blindly consume all stripes using the "normal" secret segment */
+XXH_FORCE_INLINE void
+XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc,
+ size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock,
+ const xxh_u8* XXH_RESTRICT input, size_t nbStripes,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretLimit,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble)
+{
+ XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */
+ XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock);
+ if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) {
+ /* need a scrambling operation */
+ size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr;
+ size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock;
+ XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512);
+ f_scramble(acc, secret + secretLimit);
+ XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512);
+ *nbStripesSoFarPtr = nbStripesAfterBlock;
+ } else {
+ XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512);
+ *nbStripesSoFarPtr += nbStripes;
+ }
+}
+
+#ifndef XXH3_STREAM_USE_STACK
+# ifndef __clang__ /* clang doesn't need additional stack space */
+# define XXH3_STREAM_USE_STACK 1
+# endif
+#endif
+/*
+ * Both XXH3_64bits_update and XXH3_128bits_update use this routine.
+ */
+XXH_FORCE_INLINE XXH_errorcode
+XXH3_update(XXH3_state_t* XXH_RESTRICT const state,
+ const xxh_u8* XXH_RESTRICT input, size_t len,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble)
+{
+ if (input==NULL) {
+ XXH_ASSERT(len == 0);
+ return XXH_OK;
+ }
+
+ XXH_ASSERT(state != NULL);
+ { const xxh_u8* const bEnd = input + len;
+ const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
+ /* For some reason, gcc and MSVC seem to suffer greatly
+ * when operating accumulators directly into state.
+ * Operating into stack space seems to enable proper optimization.
+ * clang, on the other hand, doesn't seem to need this trick */
+ XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; memcpy(acc, state->acc, sizeof(acc));
+#else
+ xxh_u64* XXH_RESTRICT const acc = state->acc;
+#endif
+ state->totalLen += len;
+ XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE);
+
+ /* small input : just fill in tmp buffer */
+ if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) {
+ XXH_memcpy(state->buffer + state->bufferedSize, input, len);
+ state->bufferedSize += (XXH32_hash_t)len;
+ return XXH_OK;
+ }
+
+ /* total input is now > XXH3_INTERNALBUFFER_SIZE */
+ #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN)
+ XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */
+
+ /*
+ * Internal buffer is partially filled (always, except at beginning)
+ * Complete it, then consume it.
+ */
+ if (state->bufferedSize) {
+ size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize;
+ XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize);
+ input += loadSize;
+ XXH3_consumeStripes(acc,
+ &state->nbStripesSoFar, state->nbStripesPerBlock,
+ state->buffer, XXH3_INTERNALBUFFER_STRIPES,
+ secret, state->secretLimit,
+ f_acc512, f_scramble);
+ state->bufferedSize = 0;
+ }
+ XXH_ASSERT(input < bEnd);
+
+ /* large input to consume : ingest per full block */
+ if ((size_t)(bEnd - input) > state->nbStripesPerBlock * XXH_STRIPE_LEN) {
+ size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN;
+ XXH_ASSERT(state->nbStripesPerBlock >= state->nbStripesSoFar);
+ /* join to current block's end */
+ { size_t const nbStripesToEnd = state->nbStripesPerBlock - state->nbStripesSoFar;
+ XXH_ASSERT(nbStripes <= nbStripes);
+ XXH3_accumulate(acc, input, secret + state->nbStripesSoFar * XXH_SECRET_CONSUME_RATE, nbStripesToEnd, f_acc512);
+ f_scramble(acc, secret + state->secretLimit);
+ state->nbStripesSoFar = 0;
+ input += nbStripesToEnd * XXH_STRIPE_LEN;
+ nbStripes -= nbStripesToEnd;
+ }
+ /* consume per entire blocks */
+ while(nbStripes >= state->nbStripesPerBlock) {
+ XXH3_accumulate(acc, input, secret, state->nbStripesPerBlock, f_acc512);
+ f_scramble(acc, secret + state->secretLimit);
+ input += state->nbStripesPerBlock * XXH_STRIPE_LEN;
+ nbStripes -= state->nbStripesPerBlock;
+ }
+ /* consume last partial block */
+ XXH3_accumulate(acc, input, secret, nbStripes, f_acc512);
+ input += nbStripes * XXH_STRIPE_LEN;
+ XXH_ASSERT(input < bEnd); /* at least some bytes left */
+ state->nbStripesSoFar = nbStripes;
+ /* buffer predecessor of last partial stripe */
+ XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
+ XXH_ASSERT(bEnd - input <= XXH_STRIPE_LEN);
+ } else {
+ /* content to consume <= block size */
+ /* Consume input by a multiple of internal buffer size */
+ if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) {
+ const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE;
+ do {
+ XXH3_consumeStripes(acc,
+ &state->nbStripesSoFar, state->nbStripesPerBlock,
+ input, XXH3_INTERNALBUFFER_STRIPES,
+ secret, state->secretLimit,
+ f_acc512, f_scramble);
+ input += XXH3_INTERNALBUFFER_SIZE;
+ } while (input<limit);
+ /* buffer predecessor of last partial stripe */
+ XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
+ }
+ }
+
+ /* Some remaining input (always) : buffer it */
+ XXH_ASSERT(input < bEnd);
+ XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE);
+ XXH_ASSERT(state->bufferedSize == 0);
+ XXH_memcpy(state->buffer, input, (size_t)(bEnd-input));
+ state->bufferedSize = (XXH32_hash_t)(bEnd-input);
+#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
+ /* save stack accumulators into state */
+ memcpy(state->acc, acc, sizeof(acc));
+#endif
+ }
+
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len)
+{
+ return XXH3_update(state, (const xxh_u8*)input, len,
+ XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+
+XXH_FORCE_INLINE void
+XXH3_digest_long (XXH64_hash_t* acc,
+ const XXH3_state_t* state,
+ const unsigned char* secret)
+{
+ /*
+ * Digest on a local copy. This way, the state remains unaltered, and it can
+ * continue ingesting more input afterwards.
+ */
+ XXH_memcpy(acc, state->acc, sizeof(state->acc));
+ if (state->bufferedSize >= XXH_STRIPE_LEN) {
+ size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN;
+ size_t nbStripesSoFar = state->nbStripesSoFar;
+ XXH3_consumeStripes(acc,
+ &nbStripesSoFar, state->nbStripesPerBlock,
+ state->buffer, nbStripes,
+ secret, state->secretLimit,
+ XXH3_accumulate_512, XXH3_scrambleAcc);
+ /* last stripe */
+ XXH3_accumulate_512(acc,
+ state->buffer + state->bufferedSize - XXH_STRIPE_LEN,
+ secret + state->secretLimit - XXH_SECRET_LASTACC_START);
+ } else { /* bufferedSize < XXH_STRIPE_LEN */
+ xxh_u8 lastStripe[XXH_STRIPE_LEN];
+ size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize;
+ XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */
+ XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize);
+ XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize);
+ XXH3_accumulate_512(acc,
+ lastStripe,
+ secret + state->secretLimit - XXH_SECRET_LASTACC_START);
+ }
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state)
+{
+ const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+ if (state->totalLen > XXH3_MIDSIZE_MAX) {
+ XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
+ XXH3_digest_long(acc, state, secret);
+ return XXH3_mergeAccs(acc,
+ secret + XXH_SECRET_MERGEACCS_START,
+ (xxh_u64)state->totalLen * XXH_PRIME64_1);
+ }
+ /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
+ if (state->useSeed)
+ return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
+ return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen),
+ secret, state->secretLimit + XXH_STRIPE_LEN);
+}
+
+
+
+/* ==========================================
+ * XXH3 128 bits (a.k.a XXH128)
+ * ==========================================
+ * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
+ * even without counting the significantly larger output size.
+ *
+ * For example, extra steps are taken to avoid the seed-dependent collisions
+ * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B).
+ *
+ * This strength naturally comes at the cost of some speed, especially on short
+ * lengths. Note that longer hashes are about as fast as the 64-bit version
+ * due to it using only a slight modification of the 64-bit loop.
+ *
+ * XXH128 is also more oriented towards 64-bit machines. It is still extremely
+ * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64).
+ */
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ /* A doubled version of 1to3_64b with different constants. */
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(1 <= len && len <= 3);
+ XXH_ASSERT(secret != NULL);
+ /*
+ * len = 1: combinedl = { input[0], 0x01, input[0], input[0] }
+ * len = 2: combinedl = { input[1], 0x02, input[0], input[1] }
+ * len = 3: combinedl = { input[2], 0x03, input[0], input[1] }
+ */
+ { xxh_u8 const c1 = input[0];
+ xxh_u8 const c2 = input[len >> 1];
+ xxh_u8 const c3 = input[len - 1];
+ xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24)
+ | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
+ xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13);
+ xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
+ xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed;
+ xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl;
+ xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph;
+ XXH128_hash_t h128;
+ h128.low64 = XXH64_avalanche(keyed_lo);
+ h128.high64 = XXH64_avalanche(keyed_hi);
+ return h128;
+ }
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(secret != NULL);
+ XXH_ASSERT(4 <= len && len <= 8);
+ seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
+ { xxh_u32 const input_lo = XXH_readLE32(input);
+ xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
+ xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32);
+ xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed;
+ xxh_u64 const keyed = input_64 ^ bitflip;
+
+ /* Shift len to the left to ensure it is even, this avoids even multiplies. */
+ XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2));
+
+ m128.high64 += (m128.low64 << 1);
+ m128.low64 ^= (m128.high64 >> 3);
+
+ m128.low64 = XXH_xorshift64(m128.low64, 35);
+ m128.low64 *= 0x9FB21C651E98DF25ULL;
+ m128.low64 = XXH_xorshift64(m128.low64, 28);
+ m128.high64 = XXH3_avalanche(m128.high64);
+ return m128;
+ }
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(secret != NULL);
+ XXH_ASSERT(9 <= len && len <= 16);
+ { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed;
+ xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed;
+ xxh_u64 const input_lo = XXH_readLE64(input);
+ xxh_u64 input_hi = XXH_readLE64(input + len - 8);
+ XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1);
+ /*
+ * Put len in the middle of m128 to ensure that the length gets mixed to
+ * both the low and high bits in the 128x64 multiply below.
+ */
+ m128.low64 += (xxh_u64)(len - 1) << 54;
+ input_hi ^= bitfliph;
+ /*
+ * Add the high 32 bits of input_hi to the high 32 bits of m128, then
+ * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to
+ * the high 64 bits of m128.
+ *
+ * The best approach to this operation is different on 32-bit and 64-bit.
+ */
+ if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */
+ /*
+ * 32-bit optimized version, which is more readable.
+ *
+ * On 32-bit, it removes an ADC and delays a dependency between the two
+ * halves of m128.high64, but it generates an extra mask on 64-bit.
+ */
+ m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2);
+ } else {
+ /*
+ * 64-bit optimized (albeit more confusing) version.
+ *
+ * Uses some properties of addition and multiplication to remove the mask:
+ *
+ * Let:
+ * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF)
+ * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000)
+ * c = XXH_PRIME32_2
+ *
+ * a + (b * c)
+ * Inverse Property: x + y - x == y
+ * a + (b * (1 + c - 1))
+ * Distributive Property: x * (y + z) == (x * y) + (x * z)
+ * a + (b * 1) + (b * (c - 1))
+ * Identity Property: x * 1 == x
+ * a + b + (b * (c - 1))
+ *
+ * Substitute a, b, and c:
+ * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
+ *
+ * Since input_hi.hi + input_hi.lo == input_hi, we get this:
+ * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
+ */
+ m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1);
+ }
+ /* m128 ^= XXH_swap64(m128 >> 64); */
+ m128.low64 ^= XXH_swap64(m128.high64);
+
+ { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */
+ XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2);
+ h128.high64 += m128.high64 * XXH_PRIME64_2;
+
+ h128.low64 = XXH3_avalanche(h128.low64);
+ h128.high64 = XXH3_avalanche(h128.high64);
+ return h128;
+ } }
+}
+
+/*
+ * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN
+ */
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
+{
+ XXH_ASSERT(len <= 16);
+ { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed);
+ if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed);
+ if (len) return XXH3_len_1to3_128b(input, len, secret, seed);
+ { XXH128_hash_t h128;
+ xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72);
+ xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88);
+ h128.low64 = XXH64_avalanche(seed ^ bitflipl);
+ h128.high64 = XXH64_avalanche( seed ^ bitfliph);
+ return h128;
+ } }
+}
+
+/*
+ * A bit slower than XXH3_mix16B, but handles multiply by zero better.
+ */
+XXH_FORCE_INLINE XXH128_hash_t
+XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2,
+ const xxh_u8* secret, XXH64_hash_t seed)
+{
+ acc.low64 += XXH3_mix16B (input_1, secret+0, seed);
+ acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8);
+ acc.high64 += XXH3_mix16B (input_2, secret+16, seed);
+ acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8);
+ return acc;
+}
+
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH64_hash_t seed)
+{
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+ XXH_ASSERT(16 < len && len <= 128);
+
+ { XXH128_hash_t acc;
+ acc.low64 = len * XXH_PRIME64_1;
+ acc.high64 = 0;
+ if (len > 32) {
+ if (len > 64) {
+ if (len > 96) {
+ acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed);
+ }
+ acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed);
+ }
+ acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed);
+ }
+ acc = XXH128_mix32B(acc, input, input+len-16, secret, seed);
+ { XXH128_hash_t h128;
+ h128.low64 = acc.low64 + acc.high64;
+ h128.high64 = (acc.low64 * XXH_PRIME64_1)
+ + (acc.high64 * XXH_PRIME64_4)
+ + ((len - seed) * XXH_PRIME64_2);
+ h128.low64 = XXH3_avalanche(h128.low64);
+ h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
+ return h128;
+ }
+ }
+}
+
+XXH_NO_INLINE XXH128_hash_t
+XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH64_hash_t seed)
+{
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
+ XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
+
+ { XXH128_hash_t acc;
+ int const nbRounds = (int)len / 32;
+ int i;
+ acc.low64 = len * XXH_PRIME64_1;
+ acc.high64 = 0;
+ for (i=0; i<4; i++) {
+ acc = XXH128_mix32B(acc,
+ input + (32 * i),
+ input + (32 * i) + 16,
+ secret + (32 * i),
+ seed);
+ }
+ acc.low64 = XXH3_avalanche(acc.low64);
+ acc.high64 = XXH3_avalanche(acc.high64);
+ XXH_ASSERT(nbRounds >= 4);
+ for (i=4 ; i < nbRounds; i++) {
+ acc = XXH128_mix32B(acc,
+ input + (32 * i),
+ input + (32 * i) + 16,
+ secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)),
+ seed);
+ }
+ /* last bytes */
+ acc = XXH128_mix32B(acc,
+ input + len - 16,
+ input + len - 32,
+ secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16,
+ 0ULL - seed);
+
+ { XXH128_hash_t h128;
+ h128.low64 = acc.low64 + acc.high64;
+ h128.high64 = (acc.low64 * XXH_PRIME64_1)
+ + (acc.high64 * XXH_PRIME64_4)
+ + ((len - seed) * XXH_PRIME64_2);
+ h128.low64 = XXH3_avalanche(h128.low64);
+ h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
+ return h128;
+ }
+ }
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len,
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble)
+{
+ XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
+
+ XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble);
+
+ /* converge into final hash */
+ XXH_STATIC_ASSERT(sizeof(acc) == 64);
+ XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+ { XXH128_hash_t h128;
+ h128.low64 = XXH3_mergeAccs(acc,
+ secret + XXH_SECRET_MERGEACCS_START,
+ (xxh_u64)len * XXH_PRIME64_1);
+ h128.high64 = XXH3_mergeAccs(acc,
+ secret + secretSize
+ - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
+ ~((xxh_u64)len * XXH_PRIME64_2));
+ return h128;
+ }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH128_hash_t
+XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64,
+ const void* XXH_RESTRICT secret, size_t secretLen)
+{
+ (void)seed64; (void)secret; (void)secretLen;
+ return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret),
+ XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+/*
+ * It's important for performance to pass @secretLen (when it's static)
+ * to the compiler, so that it can properly optimize the vectorized loop.
+ */
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64,
+ const void* XXH_RESTRICT secret, size_t secretLen)
+{
+ (void)seed64;
+ return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen,
+ XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64,
+ XXH3_f_accumulate_512 f_acc512,
+ XXH3_f_scrambleAcc f_scramble,
+ XXH3_f_initCustomSecret f_initSec)
+{
+ if (seed64 == 0)
+ return XXH3_hashLong_128b_internal(input, len,
+ XXH3_kSecret, sizeof(XXH3_kSecret),
+ f_acc512, f_scramble);
+ { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
+ f_initSec(secret, seed64);
+ return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret),
+ f_acc512, f_scramble);
+ }
+}
+
+/*
+ * It's important for performance that XXH3_hashLong is not inlined.
+ */
+XXH_NO_INLINE XXH128_hash_t
+XXH3_hashLong_128b_withSeed(const void* input, size_t len,
+ XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen)
+{
+ (void)secret; (void)secretLen;
+ return XXH3_hashLong_128b_withSeed_internal(input, len, seed64,
+ XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret);
+}
+
+typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t,
+ XXH64_hash_t, const void* XXH_RESTRICT, size_t);
+
+XXH_FORCE_INLINE XXH128_hash_t
+XXH3_128bits_internal(const void* input, size_t len,
+ XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
+ XXH3_hashLong128_f f_hl128)
+{
+ XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
+ /*
+ * If an action is to be taken if `secret` conditions are not respected,
+ * it should be done here.
+ * For now, it's a contract pre-condition.
+ * Adding a check and a branch here would cost performance at every hash.
+ */
+ if (len <= 16)
+ return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
+ if (len <= 128)
+ return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+ if (len <= XXH3_MIDSIZE_MAX)
+ return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
+ return f_hl128(input, len, seed64, secret, secretLen);
+}
+
+
+/* === Public XXH128 API === */
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len)
+{
+ return XXH3_128bits_internal(input, len, 0,
+ XXH3_kSecret, sizeof(XXH3_kSecret),
+ XXH3_hashLong_128b_default);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
+{
+ return XXH3_128bits_internal(input, len, 0,
+ (const xxh_u8*)secret, secretSize,
+ XXH3_hashLong_128b_withSecret);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
+{
+ return XXH3_128bits_internal(input, len, seed,
+ XXH3_kSecret, sizeof(XXH3_kSecret),
+ XXH3_hashLong_128b_withSeed);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH3_128bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed)
+{
+ if (len <= XXH3_MIDSIZE_MAX)
+ return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
+ return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH128(const void* input, size_t len, XXH64_hash_t seed)
+{
+ return XXH3_128bits_withSeed(input, len, seed);
+}
+
+
+/* === XXH3 128-bit streaming === */
+
+/*
+ * All initialization and update functions are identical to 64-bit streaming variant.
+ * The only difference is the finalization routine.
+ */
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset(XXH3_state_t* statePtr)
+{
+ return XXH3_64bits_reset(statePtr);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
+{
+ return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
+{
+ return XXH3_64bits_reset_withSeed(statePtr, seed);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed)
+{
+ return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len)
+{
+ return XXH3_update(state, (const xxh_u8*)input, len,
+ XXH3_accumulate_512, XXH3_scrambleAcc);
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state)
+{
+ const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
+ if (state->totalLen > XXH3_MIDSIZE_MAX) {
+ XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
+ XXH3_digest_long(acc, state, secret);
+ XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
+ { XXH128_hash_t h128;
+ h128.low64 = XXH3_mergeAccs(acc,
+ secret + XXH_SECRET_MERGEACCS_START,
+ (xxh_u64)state->totalLen * XXH_PRIME64_1);
+ h128.high64 = XXH3_mergeAccs(acc,
+ secret + state->secretLimit + XXH_STRIPE_LEN
+ - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
+ ~((xxh_u64)state->totalLen * XXH_PRIME64_2));
+ return h128;
+ }
+ }
+ /* len <= XXH3_MIDSIZE_MAX : short code */
+ if (state->seed)
+ return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
+ return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen),
+ secret, state->secretLimit + XXH_STRIPE_LEN);
+}
+
+/* 128-bit utility functions */
+
+#include <string.h> /* memcmp, memcpy */
+
+/* return : 1 is equal, 0 if different */
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
+{
+ /* note : XXH128_hash_t is compact, it has no padding byte */
+ return !(memcmp(&h1, &h2, sizeof(h1)));
+}
+
+/* This prototype is compatible with stdlib's qsort().
+ * return : >0 if *h128_1 > *h128_2
+ * <0 if *h128_1 < *h128_2
+ * =0 if *h128_1 == *h128_2 */
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2)
+{
+ XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1;
+ XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2;
+ int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
+ /* note : bets that, in most cases, hash values are different */
+ if (hcmp) return hcmp;
+ return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
+}
+
+
+/*====== Canonical representation ======*/
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API void
+XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) {
+ hash.high64 = XXH_swap64(hash.high64);
+ hash.low64 = XXH_swap64(hash.low64);
+ }
+ XXH_memcpy(dst, &hash.high64, sizeof(hash.high64));
+ XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH128_hash_t
+XXH128_hashFromCanonical(const XXH128_canonical_t* src)
+{
+ XXH128_hash_t h;
+ h.high64 = XXH_readBE64(src);
+ h.low64 = XXH_readBE64(src->digest + 8);
+ return h;
+}
+
+
+
+/* ==========================================
+ * Secret generators
+ * ==========================================
+ */
+#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x))
+
+static void XXH3_combine16(void* dst, XXH128_hash_t h128)
+{
+ XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 );
+ XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 );
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API XXH_errorcode
+XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize)
+{
+ XXH_ASSERT(secretBuffer != NULL);
+ if (secretBuffer == NULL) return XXH_ERROR;
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
+ if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
+ if (customSeedSize == 0) {
+ customSeed = XXH3_kSecret;
+ customSeedSize = XXH_SECRET_DEFAULT_SIZE;
+ }
+ XXH_ASSERT(customSeed != NULL);
+ if (customSeed == NULL) return XXH_ERROR;
+
+ /* Fill secretBuffer with a copy of customSeed - repeat as needed */
+ { size_t pos = 0;
+ while (pos < secretSize) {
+ size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize);
+ memcpy((char*)secretBuffer + pos, customSeed, toCopy);
+ pos += toCopy;
+ } }
+
+ { size_t const nbSeg16 = secretSize / 16;
+ size_t n;
+ XXH128_canonical_t scrambler;
+ XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0));
+ for (n=0; n<nbSeg16; n++) {
+ XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n);
+ XXH3_combine16((char*)secretBuffer + n*16, h128);
+ }
+ /* last segment */
+ XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler));
+ }
+ return XXH_OK;
+}
+
+/*! @ingroup xxh3_family */
+XXH_PUBLIC_API void
+XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed)
+{
+ XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
+ XXH3_initCustomSecret(secret, seed);
+ XXH_ASSERT(secretBuffer != NULL);
+ memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE);
+}
+
+
+
+/* Pop our optimization override from above */
+#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
+ && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
+ && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */
+# pragma GCC pop_options
+#endif
+
+#endif /* XXH_NO_LONG_LONG */
+
+#endif /* XXH_NO_XXH3 */
+
+/*!
+ * @}
+ */
+#endif /* XXH_IMPLEMENTATION */
+
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/contrib/zstd/CHANGELOG b/contrib/zstd/CHANGELOG
new file mode 100644
index 0000000..0ed939a
--- /dev/null
+++ b/contrib/zstd/CHANGELOG
@@ -0,0 +1,555 @@
+v1.4.5
+fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln
+perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln
+perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta)
+perf: Improved fast compression speed on aarch64 (#2040, ~+3%, by @caoyzh)
+perf: Small level 1 compression speed gains (depending on compiler)
+cli : New --patch-from command, create and apply patches from files, by @bimbashreshta
+cli : New --filelist= : Provide a list of files to operate upon from a file
+cli : -b -d command can now benchmark decompression on multiple files
+cli : New --no-content-size command
+cli : New --show-default-cparams information command
+api : ZDICT_finalizeDictionary() is promoted to stable (#2111)
+api : new experimental parameter ZSTD_d_stableOutBuffer (#2094)
+build: Generate a single-file libzstd library (#2065, by @cwoffenden)
+build: Relative includes no longer require -I compiler flags for zstd lib subdirs (#2103, by @felixhandte)
+build: zstd now compiles cleanly under -pedantic (#2099)
+build: zstd now compiles with make-4.3
+build: Support mingw cross-compilation from Linux, by @Ericson2314
+build: Meson multi-thread build fix on windows
+build: Some misc icc fixes backed by new ci test on travis
+misc: bitflip analyzer tool, by @felixhandte
+misc: Extend largeNbDicts benchmark to compression
+misc: Edit-distance match finder in contrib/
+doc : Improved beginner CONTRIBUTING.md docs
+doc : New issue templates for zstd
+
+v1.4.4
+perf: Improved decompression speed, by > 10%, by @terrelln
+perf: Better compression speed when re-using a context, by @felixhandte
+perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42
+perf: zstd reference encoder can generate RLE blocks, by @bimbashrestha
+perf: minor generic speed optimization, by @davidbolvansky
+api: new ability to extract sequences from the parser for analysis, by @bimbashrestha
+api: fixed decoding of magic-less frames, by @terrelln
+api: fixed ZSTD_initCStream_advanced() performance with fast modes, reported by @QrczakMK
+cli: Named pipes support, by @bimbashrestha
+cli: short tar's extension support, by @stokito
+cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42
+cli: commands --stream-size=# and --size-hint=#, by @nmagerko
+cli: command --exclude-compressed, by @shashank0791
+cli: faster `-t` test mode
+cli: improved some error messages, by @vangyzen
+cli: fix command `-D dictionary` on Windows, reported by @artyompetrov
+cli: fix rare deadlock condition within dictionary builder, by @terrelln
+build: single-file decoder with emscripten compilation script, by @cwoffenden
+build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive
+build: fixed deprecation warning for certain gcc version, reported by @jasonma163
+build: fix compilation on old gcc versions, by @cemeyer
+build: improved installation directories for cmake script, by Dmitri Shubin
+pack: modified pkgconfig, for better integration into openwrt, requested by @neheb
+misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format
+misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro
+
+v1.4.3
+bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709)
+bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722)
+build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705)
+
+v1.4.2
+bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696)
+bug: Fix seekable decompression in-memory API by @iburinoc (#1695)
+misc: Validate blocks are smaller than size limit by @vivekmg (#1685)
+misc: Restructure source files by @ephiepark (#1679)
+
+v1.4.1
+bug: Fix data corruption in niche use cases by @terrelln (#1659)
+bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595)
+bug: Fix out of bounds read by @terrelln (#1590)
+perf: Improve decode speed by ~7% @mgrice (#1668)
+perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681)
+perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658)
+perf: Improve compression ratio for small windowLog by @cyan4973 (#1624)
+perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635)
+api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656)
+cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640)
+cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631)
+cli: Restrict read permissions on destination files by @chungy (#1644)
+cli: zstdgrep: handle -f flag by @felixhandte (#1618)
+cli: zstdcat: follow symlinks by @vejnar (#1604)
+doc: Remove extra size limit on compressed blocks by @felixhandte (#1689)
+doc: Fix typo by @yk-tanigawa (#1633)
+doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629)
+build: CMake: support building with LZ4 @leeyoung624 (#1626)
+build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647)
+build: CMake: respect existing uninstall target by @j301scott (#1619)
+build: Make: skip multithread tests when built without support by @michaelforney (#1620)
+build: Make: Fix examples/ test target by @sjnam (#1603)
+build: Meson: rename options out of deprecated namespace by @lzutao (#1665)
+build: Meson: fix build by @lzutao (#1602)
+build: Visual Studio: don't export symbols in static lib by @scharan (#1650)
+build: Visual Studio: fix linking by @absotively (#1639)
+build: Fix MinGW-W64 build by @myzhang1029 (#1600)
+misc: Expand decodecorpus coverage by @ephiepark (#1664)
+
+v1.4.0
+perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln
+api: Move the advanced API, including all functions in the staging section, to the stable section
+api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress
+api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
+api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
+api: Don't export ZSTDMT functions from the shared library by default
+api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT
+api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao
+api: Fix ZSTD_decompressDCtx() corner cases with a dictionary
+api: Move ZSTD_getDictID_*() functions to the stable section
+api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln
+api: Allow compression parameters to be set when a dictionary is used
+api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called
+api: Fix ZSTD_estimateCStreamSize_usingCCtxParams()
+api: Setting ZSTD_d_maxWindowLog to 0 means use the default
+cli: Ensure that a dictionary is not used to compress itself by @shakeelrao
+cli: Add --[no-]compress-literals flag to enable or disable literal compression
+doc: Update the examples to use the advanced API
+doc: Explain how to transition from old streaming functions to the advanced API in the header
+build: Improve the Windows release packages
+build: Improve CMake build by @hjmjohnson
+build: Build fixes for FreeBSD by @lwhsu
+build: Remove redundant warnings by @thatsafunnyname
+build: Fix tests on OpenBSD by @bket
+build: Extend fuzzer build system to work with the new clang engine
+build: CMake now creates the libzstd.so.1 symlink
+build: Improve Menson build by @lzutao
+misc: Fix symbolic link detection on FreeBSD
+misc: Use physical core count for -T0 on FreeBSD by @cemeyer
+misc: Fix zstd --list on truncated files by @kostmo
+misc: Improve logging in debug mode by @felixhandte
+misc: Add CirrusCI tests by @lwhsu
+misc: Optimize dictionary memory usage in corner cases
+misc: Improve the dictionary builder on small or homogeneous data
+misc: Fix spelling across the repo by @jsoref
+
+v1.3.8
+perf: better decompression speed on large files (+7%) and cold dictionaries (+15%)
+perf: slightly better compression ratio at high compression modes
+api : finalized advanced API, last stage before "stable" status
+api : new --rsyncable mode, by @terrelln
+api : support decompression of empty frames into NULL (used to be an error) (#1385)
+build: new set of macros to build a minimal size decoder, by @felixhandte
+build: fix compilation on MIPS32, reported by @clbr (#1441)
+build: fix compilation with multiple -arch flags, by @ryandesign
+build: highly upgraded meson build, by @lzutao
+build: improved buck support, by @obelisk
+build: fix cmake script : can create debug build, by @pitrou
+build: Makefile : grep works on both colored consoles and systems without color support
+build: fixed zstd-pgo, by @bmwiedemann
+cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423)
+cli : --no-progress flag, preserving final summary (#1371), by @terrelln
+cli : ensure destination file is not source file (#1422)
+cli : clearer error messages, especially when input file not present
+doc : clarified zstd_compression_format.md, by @ulikunitz
+misc: fixed zstdgrep, returns 1 on failure, by @lzutao
+misc: NEWS renamed as CHANGELOG, in accordance with fboss
+
+v1.3.7
+perf: slightly better decompression speed on clang (depending on hardware target)
+fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
+build: no longer build backtrace by default in release mode; restrict further automatic mode
+build: control backtrace support through build macro BACKTRACE
+misc: added man pages for zstdless and zstdgrep, by @samrussell
+
+v1.3.6
+perf: much faster dictionary builder, by @jenniferliu
+perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
+perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
+cli : fix : does no longer overwrite destination when source does not exist (#1082)
+cli : new command --adapt, for automatic compression level adaptation
+api : fix : block api can be streamed with > 4 GB, reported by @catid
+api : reduced ZSTD_DDict size by 2 KB
+api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
+build: support Haiku target, by @korli
+build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT.
+doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
+misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
+
+v1.3.5
+perf: much faster dictionary compression, by @felixhandte
+perf: small quality improvement for dictionary generation, by @terrelln
+perf: slightly improved high compression levels (notably level 19)
+mem : automatic memory release for long duration contexts
+cli : fix : overlapLog can be manually set
+cli : fix : decoding invalid lz4 frames
+api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln
+api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln
+build: select custom libzstd scope through control macros, by @GeorgeLu97
+build: OpenBSD patch, by @bket
+build: make and make all are compatible with -j
+doc : clarify zstd_compression_format.md, updated for IETF RFC process
+misc: pzstd compatible with reproducible compilation, by @lamby
+
+v1.3.4
+perf: faster speed (especially decoding speed) on recent cpus (haswell+)
+perf: much better performance associating --long with multi-threading, by @terrelln
+perf: better compression at levels 13-15
+cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior)
+cli : smoother status report in multi-threading mode
+cli : added command --fast=#, for faster compression modes
+cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb)
+api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode
+api : compression levels can be negative, for even more speed
+api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime
+api : ZSTDMT can accept new compression parameters during compression
+api : implemented all advanced dictionary decompression prototypes
+build: improved meson recipe, by Shawn Landden (@shawnl)
+build: VS2017 scripts, by @HaydnTrigg
+misc: all /contrib projects fixed
+misc: added /contrib/docker script by @gyscos
+
+v1.3.3
+perf: faster zstd_opt strategy (levels 16-19)
+fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
+cli : fix : content size written in header by default
+cli : fix : improved LZ4 format support, by @felixhandte
+cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
+api : fix : support large skippable frames, by @terrelln
+api : fix : streaming interface was adding a useless 3-bytes null block to small frames
+api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown"
+build: fix : compilation under rhel6 and centos6, reported by @pixelb
+build: added `check` target
+
+v1.3.2
+new : long range mode, using --long command, by Stella Lau (@stellamplau)
+new : ability to generate and decode magicless frames (#591)
+changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode
+fix : multi-threading compression works with custom allocators
+fix : ZSTD_sizeof_CStream() was over-evaluating memory usage
+fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22)
+fix : 32-bits build can now decode large offsets (levels 21+)
+cli : added LZ4 frame support by default, by Felix Handte (@felixhandte)
+cli : improved --list output
+cli : new : can split input file for dictionary training, using command -B#
+cli : new : clean operation artefact on Ctrl-C interruption
+cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851)
+cli : fix : write file size in header in multiple-files mode
+api : added macro ZSTD_COMPRESSBOUND() for static allocation
+api : experimental : new advanced decompression API
+api : fix : sizeof_CCtx() used to over-estimate
+build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819)
+build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818)
+example : added streaming_memory_usage
+license : changed /examples license to BSD + GPLv2
+license : fix a few header files to reflect new license (#825)
+
+v1.3.1
+New license : BSD + GPLv2
+perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk)
+perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
+cli : improved and fixed --list command, by @ib (#772)
+cli : command -vV to list supported formats, by @ib (#771)
+build : fixed binary variants, reported by @svenha (#788)
+build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718)
+API exp : breaking change : ZSTD_getframeHeader() provides more information
+API exp : breaking change : pinned down values of error codes
+doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz)
+new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74)
+new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau)
+updated : contrib/linux-kernel, by Nick Terrell (@terrelln)
+
+v1.3.0
+cli : new : `--list` command, by Paul Cruz
+cli : changed : xz/lzma support enabled by default
+cli : changed : `-t *` continue processing list after a decompression error
+API : added : ZSTD_versionString()
+API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell
+API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
+API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
+API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
+API exp : clarified memory estimation / measurement functions.
+API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1
+tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz
+new : contrib/seekable_format, demo and API, by Sean Purcell
+changed : contrib/linux-kernel, updated version and license, by Nick Terrell
+
+v1.2.0
+cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
+cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
+cli : new : zstdmt symlink hardwired to `zstd -T0`
+cli : new : command --threads=# (#671)
+cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
+cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
+cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
+cli : fix : does not output compressed data on console
+cli : fix : ignore symbolic links unless --force specified,
+API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
+API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
+API : improved: ZSTDMT_compressCCtx() reduced memory usage
+API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
+API : fix : src size stored in frame header is controlled at end of frame
+API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
+API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
+build: improved cmake script, by @Majlen
+build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
+tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
+new : contrib/linux-kernel version, by Nick Terrell
+
+v1.1.4
+cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
+cli : new : advanced benchmark command --priority=rt
+cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
+cli : fix : --rm remains silent when input is stdin
+cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
+speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
+memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
+arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
+API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
+API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
+build : new: meson build system in contrib/meson, by Dima Krasner
+build : improved cmake script, by @Majlen
+build : added -Wformat-security flag, as recommended by Padraig Brady
+doc : new : educational decoder, by Sean Purcell
+
+v1.1.3
+cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
+cli : new : experimental target `make zstdmt`, with multi-threading support
+cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano.
+cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
+cli : fix zstdless on Mac OS-X, by Andrew Janke
+cli : fix #232 "compress non-files"
+dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
+API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
+API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
+API : new : ZDICT_finalizeDictionary()
+API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
+API : fix : all symbols properly exposed in libzstd, by Nick Terrell
+build : support for Solaris target, by Przemyslaw Skibinski
+doc : clarified specification, by Sean Purcell
+
+v1.1.2
+API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
+API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
+API : zbuff : changed : prototypes now generate deprecation warnings
+lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
+lib : changed : only public ZSTD_ symbols are now exposed
+lib : changed : reduced usage of stack memory
+lib : fixed : several corner case bugs, by Nick Terrell
+cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
+cli : new : preserve file attributes
+cli : new : added zstdless and zstdgrep tools
+cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
+cli : fixed : zstdcat
+zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
+install : better compatibility with FreeBSD, by Dimitry Andric
+source tree : changed : zbuff source files moved to lib/deprecated
+
+v1.1.1
+New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
+New : doc/zstd_manual.html, by Przemyslaw Skibinski
+Improved : slightly better compression ratio at --ultra levels (>= 20)
+Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report
+Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
+Added : example/multiple_streaming_compression.c
+Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h)
+Updated man page
+Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
+
+v1.1.0
+New : contrib/pzstd, parallel version of zstd, by Nick Terrell
+added : NetBSD install target (#338)
+Improved : speed for batches of small files
+Improved : speed of zlib wrapper, by Przemyslaw Skibinski
+Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier
+Fixed : CLI -d output to stdout by default when input is stdin (#322)
+Fixed : CLI correctly detects console on Mac OS-X
+Fixed : CLI supports recursive mode `-r` on Mac OS-X
+Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
+Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
+Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
+Fixed : zstd-pgo, reported by octoploid (#329)
+
+v1.0.0
+Change Licensing, all project is now BSD, Copyright Facebook
+Small decompression speed improvement
+API : Streaming API supports legacy format
+API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter()
+CLI supports legacy formats v0.4+
+Fixed : compression fails on certain huge files, reported by Jesse McGrew
+Enhanced documentation, by Przemyslaw Skibinski
+
+v0.8.1
+New streaming API
+Changed : --ultra now enables levels beyond 19
+Changed : -i# now selects benchmark time in second
+Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
+Fixed : speed regression on specific patterns (#272)
+Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
+Fixed : ICC compilation, by Przemyslaw Skibinski
+
+v0.8.0
+Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
+New : Build on FreeBSD and DragonFly, thanks to JrMarino
+Changed : modified API : ZSTD_compressEnd()
+Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist
+Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers
+Fixed : large dictionaries (> 384 KB), reported by Ilona Papava
+Fixed : checksum correctly checked in single-pass mode
+Fixed : combined --test amd --rm, reported by Andreas M. Nilsson
+Modified : minor compression level adaptations
+Updated : compression format specification to v0.2.0
+changed : zstd.h moved to /lib directory
+
+v0.7.5
+Transition version, supporting decoding of v0.8.x
+
+v0.7.4
+Added : homebrew for Mac, by Daniel Cade
+Added : more examples
+Fixed : segfault when using small dictionaries, reported by Felix Handte
+Modified : default compression level for CLI is now 3
+Updated : specification, to v0.1.1
+
+v0.7.3
+New : compression format specification
+New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
+New : `ZSTD_getDecompressedSize()`
+New : OpenBSD target, by Juan Francisco Cantero Hurtado
+New : `examples` directory
+fixed : dictBuilder using HC levels, reported by Bartosz Taudul
+fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte
+fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski
+modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
+modified : legacy functions no longer need magic number
+
+v0.7.2
+fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
+fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
+fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
+
+v0.7.1
+fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
+fixed : dictBuilder fails if first sample is too small, reported by РуÑлан Ковалёв
+fixed : corruption issue, reported by cj
+modified : checksum enabled by default in command line mode
+
+v0.7.0
+New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
+New : Command `--rm`, to remove source file after successful de/compression
+New : Visual build scripts, by Christophe Chevalier
+New : Support for Sparse File-systems (do not use space for zero-filled sectors)
+New : Frame checksum support
+New : Support pass-through mode (when using `-df`)
+API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()`
+API : create dictionary files from custom content, by Giuseppe Ottaviano
+API : support for custom malloc/free functions
+New : controllable Dictionary ID
+New : Support for skippable frames
+
+v0.6.1
+New : zlib wrapper API, thanks to Przemyslaw Skibinski
+New : Ability to compile compressor / decompressor separately
+Changed : new lib directory structure
+Fixed : Legacy codec v0.5 compatible with dictionary decompression
+Fixed : Decoder corruption error (#173)
+Fixed : null-string roundtrip (#176)
+New : benchmark mode can select directory as input
+Experimental : midipix support, VMS support
+
+v0.6.0
+Stronger high compression modes, thanks to Przemyslaw Skibinski
+API : ZSTD_getFrameParams() provides size of decompressed content
+New : highest compression modes require `--ultra` command to fully unleash their capacity
+Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner
+
+v0.5.1
+New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski
+Changed : Dictionary builder integrated into libzstd and zstd cli
+Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`.
+Fix : high compression modes for big-endian platforms
+New : zstd cli : `-t` | `--test` command
+
+v0.5.0
+New : dictionary builder utility
+Changed : streaming & dictionary API
+Improved : better compression of small data
+
+v0.4.7
+Improved : small compression speed improvement in HC mode
+Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
+fix : bt search bug
+
+v0.4.6
+fix : fast compression mode on Windows
+New : cmake configuration file, thanks to Artyom Dymchenko
+Improved : high compression mode on repetitive data
+New : block-level API
+New : ZSTD_duplicateCCtx()
+
+v0.4.5
+new : -m/--multiple : compress/decompress multiple files
+
+v0.4.4
+Fixed : high compression modes for Windows 32 bits
+new : external dictionary API extended to buffered mode and accessible through command line
+new : windows DLL project, thanks to Christophe Chevalier
+
+v0.4.3 :
+new : external dictionary API
+new : zstd-frugal
+
+v0.4.2 :
+Generic minor improvements for small blocks
+Fixed : big-endian compatibility, by Peter Harris (#85)
+
+v0.4.1
+Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
+removed `zstd.c`
+
+v0.4.0
+Command line utility compatible with high compression levels
+Removed zstdhc => merged into zstd
+Added : ZBUFF API (see zstd_buffered.h)
+Rolling buffer support
+
+v0.3.6
+small blocks params
+
+v0.3.5
+minor generic compression improvements
+
+v0.3.4
+Faster fast cLevels
+
+v0.3.3
+Small compression ratio improvement
+
+v0.3.2
+Fixed Visual Studio
+
+v0.3.1 :
+Small compression ratio improvement
+
+v0.3
+HC mode : compression levels 2-26
+
+v0.2.2
+Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
+
+v0.2.1
+Fix : Read errors, advanced fuzzer tests, by Hanno Böck
+
+v0.2.0
+**Breaking format change**
+Faster decompression speed
+Can still decode v0.1 format
+
+v0.1.3
+fix uninitialization warning, reported by Evan Nemerson
+
+v0.1.2
+frame concatenation support
+
+v0.1.1
+fix compression bug
+detects write-flush errors
+
+v0.1.0
+first release
diff --git a/contrib/zstd/CMakeLists.txt b/contrib/zstd/CMakeLists.txt
new file mode 100644
index 0000000..4601e53
--- /dev/null
+++ b/contrib/zstd/CMakeLists.txt
@@ -0,0 +1,27 @@
+SET(ZSTDSRC
+ debug.c
+ divsufsort.c
+ entropy_common.c
+ error_private.c
+ fse_compress.c
+ fse_decompress.c
+ hist.c
+ huf_compress.c
+ huf_decompress.c
+ pool.c
+ zstd_common.c
+ zstd_compress.c
+ zstd_compress_literals.c
+ zstd_compress_sequences.c
+ zstd_compress_superblock.c
+ zstd_ddict.c
+ zstd_decompress.c
+ zstd_decompress_block.c
+ zstd_double_fast.c
+ zstd_fast.c
+ zstd_lazy.c
+ zstd_ldm.c
+ zstd_opt.c)
+
+ADD_LIBRARY(rspamd-zstd STATIC ${ZSTDSRC})
+ADD_DEFINITIONS(-DZSTD_DISABLE_ASM) \ No newline at end of file
diff --git a/contrib/zstd/LICENSE b/contrib/zstd/LICENSE
new file mode 100644
index 0000000..a793a80
--- /dev/null
+++ b/contrib/zstd/LICENSE
@@ -0,0 +1,30 @@
+BSD License
+
+For Zstandard software
+
+Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ * Neither the name Facebook nor the names of its contributors may be used to
+ endorse or promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/contrib/zstd/PATENTS b/contrib/zstd/PATENTS
new file mode 100644
index 0000000..15b4a2e
--- /dev/null
+++ b/contrib/zstd/PATENTS
@@ -0,0 +1,33 @@
+Additional Grant of Patent Rights Version 2
+
+"Software" means the Zstandard software distributed by Facebook, Inc.
+
+Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software
+("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable
+(subject to the termination provision below) license under any Necessary
+Claims, to make, have made, use, sell, offer to sell, import, and otherwise
+transfer the Software. For avoidance of doubt, no license is granted under
+Facebook’s rights in any patent claims that are infringed by (i) modifications
+to the Software made by you or any third party or (ii) the Software in
+combination with any software or other technology.
+
+The license granted hereunder will terminate, automatically and without notice,
+if you (or any of your subsidiaries, corporate affiliates or agents) initiate
+directly or indirectly, or take a direct financial interest in, any Patent
+Assertion: (i) against Facebook or any of its subsidiaries or corporate
+affiliates, (ii) against any party if such Patent Assertion arises in whole or
+in part from any software, technology, product or service of Facebook or any of
+its subsidiaries or corporate affiliates, or (iii) against any party relating
+to the Software. Notwithstanding the foregoing, if Facebook or any of its
+subsidiaries or corporate affiliates files a lawsuit alleging patent
+infringement against you in the first instance, and you respond by filing a
+patent infringement counterclaim in that lawsuit against that party that is
+unrelated to the Software, the license granted hereunder will not terminate
+under section (i) of this paragraph due to such counterclaim.
+
+A "Necessary Claim" is a claim of a patent owned by Facebook that is
+necessarily infringed by the Software standing alone.
+
+A "Patent Assertion" is any lawsuit or other action alleging direct, indirect,
+or contributory infringement or inducement to infringe any patent, including a
+cross-claim or counterclaim.
diff --git a/contrib/zstd/README.md b/contrib/zstd/README.md
new file mode 100644
index 0000000..e4af990
--- /dev/null
+++ b/contrib/zstd/README.md
@@ -0,0 +1,94 @@
+ **Zstd**, short for Zstandard, is a fast lossless compression algorithm,
+ targeting real-time compression scenarios at zlib-level and better compression ratios.
+
+It is provided as an open-source BSD-licensed **C** library.
+For other programming languages,
+you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages).
+
+|Branch |Status |
+|------------|---------|
+|master | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=master)](https://travis-ci.org/facebook/zstd) |
+|dev | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=dev)](https://travis-ci.org/facebook/zstd) |
+
+As a reference, several fast compression algorithms were tested and compared on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with gcc 5.4.0, with the [Silesia compression corpus].
+
+[lzbench]: https://github.com/inikep/lzbench
+[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
+
+
+|Name | Ratio | C.speed | D.speed |
+|-----------------|-------|--------:|--------:|
+| | | MB/s | MB/s |
+|**zstd 0.8.2 -1**|**2.877**|**330**| **940** |
+| [zlib] 1.2.8 -1 | 2.730 | 95 | 360 |
+| brotli 0.4 -0 | 2.708 | 320 | 375 |
+| QuickLZ 1.5 | 2.237 | 510 | 605 |
+| LZO 2.09 | 2.106 | 610 | 870 |
+| [LZ4] r131 | 2.101 | 620 | 3100 |
+| Snappy 1.1.3 | 2.091 | 480 | 1600 |
+| LZF 3.6 | 2.077 | 375 | 790 |
+
+[zlib]:http://www.zlib.net/
+[LZ4]: http://www.lz4.org/
+
+Zstd can also offer stronger compression ratios at the cost of compression speed.
+Speed vs Compression trade-off is configurable by small increment. Decompression speed is preserved and remain roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.
+
+The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with gcc 5.2.1, on the [Silesia compression corpus].
+
+Compression Speed vs Ratio | Decompression Speed
+---------------------------|--------------------
+![Compression Speed vs Ratio](images/Cspeed4.png "Compression Speed vs Ratio") | ![Decompression Speed](images/Dspeed4.png "Decompression Speed")
+
+Several algorithms can produce higher compression ratio but at slower speed, falling outside of the graph.
+For a larger picture including very slow modes, [click on this link](images/DCspeed5.png) .
+
+
+### The case for Small Data compression
+
+Previous charts provide results applicable to typical files and streams scenarios (several MB). Small data come with different perspectives. The smaller the amount of data to compress, the more difficult it is to achieve any significant compression.
+
+This problem is common to any compression algorithm. The reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new file, there is no "past" to build upon.
+
+To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data, by providing it with a few samples. The result of the training is stored in a file called "dictionary", which can be loaded before compression and decompression. Using this dictionary, the compression ratio achievable on small data improves dramatically :
+
+![Compressing Small Data](images/smallData.png "Compressing Small Data")
+
+These compression gains are achieved while simultaneously providing faster compression and decompression speeds.
+
+Dictionary work if there is some correlation in a family of small data (there is no _universal dictionary_).
+Hence, deploying one dictionary per type of data will provide the greater benefits. Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will rely more and more on previously decoded content to compress the rest of the file.
+
+#### Dictionary compression How To :
+
+1) Create the dictionary
+
+`zstd --train FullPathToTrainingSet/* -o dictionaryName`
+
+2) Compress with dictionary
+
+`zstd FILE -D dictionaryName`
+
+3) Decompress with dictionary
+
+`zstd --decompress FILE.zst -D dictionaryName`
+
+### Status
+
+Zstandard is currently deployed within Facebook. It is used daily to compress and decompress very large amount of data in multiple formats and use cases.
+Zstandard is considered safe for production environments.
+
+### License
+
+Zstandard is [BSD-licensed](LICENSE). We also provide an [additional patent grant](PATENTS).
+
+### Contributing
+
+The "dev" branch is the one where all contributions will be merged before reaching "master".
+If you plan to propose a patch, please commit into the "dev" branch or its own feature branch.
+Direct commit to "master" are not permitted.
+For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
+
+### Miscellaneous
+
+Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entropy library](https://github.com/Cyan4973/FiniteStateEntropy).
diff --git a/contrib/zstd/bits.h b/contrib/zstd/bits.h
new file mode 100644
index 0000000..7939f3d
--- /dev/null
+++ b/contrib/zstd/bits.h
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_BITS_H
+#define ZSTD_BITS_H
+
+#include "mem.h"
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
+{
+ assert(val != 0);
+ {
+ static const int DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3,
+ 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7,
+ 26, 12, 18, 6, 11, 5, 10, 9};
+ return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27];
+ }
+}
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
+{
+ assert(val != 0);
+# if defined(_MSC_VER)
+# if STATIC_BMI2 == 1
+ return _tzcnt_u32(val);
+# else
+ if (val != 0) {
+ unsigned long r;
+ _BitScanForward(&r, val);
+ return (unsigned)r;
+ } else {
+ /* Should not reach this code path */
+ __assume(0);
+ }
+# endif
+# elif defined(__GNUC__) && (__GNUC__ >= 4)
+ return (unsigned)__builtin_ctz(val);
+# else
+ return ZSTD_countTrailingZeros32_fallback(val);
+# endif
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
+ assert(val != 0);
+ {
+ static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29,
+ 11, 14, 16, 18, 22, 25, 3, 30,
+ 8, 12, 20, 28, 15, 17, 24, 7,
+ 19, 27, 23, 6, 26, 5, 4, 31};
+ val |= val >> 1;
+ val |= val >> 2;
+ val |= val >> 4;
+ val |= val >> 8;
+ val |= val >> 16;
+ return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27];
+ }
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
+{
+ assert(val != 0);
+# if defined(_MSC_VER)
+# if STATIC_BMI2 == 1
+ return _lzcnt_u32(val);
+# else
+ if (val != 0) {
+ unsigned long r;
+ _BitScanReverse(&r, val);
+ return (unsigned)(31 - r);
+ } else {
+ /* Should not reach this code path */
+ __assume(0);
+ }
+# endif
+# elif defined(__GNUC__) && (__GNUC__ >= 4)
+ return (unsigned)__builtin_clz(val);
+# else
+ return ZSTD_countLeadingZeros32_fallback(val);
+# endif
+}
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
+{
+ assert(val != 0);
+# if defined(_MSC_VER) && defined(_WIN64)
+# if STATIC_BMI2 == 1
+ return _tzcnt_u64(val);
+# else
+ if (val != 0) {
+ unsigned long r;
+ _BitScanForward64(&r, val);
+ return (unsigned)r;
+ } else {
+ /* Should not reach this code path */
+ __assume(0);
+ }
+# endif
+# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
+ return (unsigned)__builtin_ctzll(val);
+# else
+ {
+ U32 mostSignificantWord = (U32)(val >> 32);
+ U32 leastSignificantWord = (U32)val;
+ if (leastSignificantWord == 0) {
+ return 32 + ZSTD_countTrailingZeros32(mostSignificantWord);
+ } else {
+ return ZSTD_countTrailingZeros32(leastSignificantWord);
+ }
+ }
+# endif
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
+{
+ assert(val != 0);
+# if defined(_MSC_VER) && defined(_WIN64)
+# if STATIC_BMI2 == 1
+ return _lzcnt_u64(val);
+# else
+ if (val != 0) {
+ unsigned long r;
+ _BitScanReverse64(&r, val);
+ return (unsigned)(63 - r);
+ } else {
+ /* Should not reach this code path */
+ __assume(0);
+ }
+# endif
+# elif defined(__GNUC__) && (__GNUC__ >= 4)
+ return (unsigned)(__builtin_clzll(val));
+# else
+ {
+ U32 mostSignificantWord = (U32)(val >> 32);
+ U32 leastSignificantWord = (U32)val;
+ if (mostSignificantWord == 0) {
+ return 32 + ZSTD_countLeadingZeros32(leastSignificantWord);
+ } else {
+ return ZSTD_countLeadingZeros32(mostSignificantWord);
+ }
+ }
+# endif
+}
+
+MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val)
+{
+ if (MEM_isLittleEndian()) {
+ if (MEM_64bits()) {
+ return ZSTD_countTrailingZeros64((U64)val) >> 3;
+ } else {
+ return ZSTD_countTrailingZeros32((U32)val) >> 3;
+ }
+ } else { /* Big Endian CPU */
+ if (MEM_64bits()) {
+ return ZSTD_countLeadingZeros64((U64)val) >> 3;
+ } else {
+ return ZSTD_countLeadingZeros32((U32)val) >> 3;
+ }
+ }
+}
+
+MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */
+{
+ assert(val != 0);
+ return 31 - ZSTD_countLeadingZeros32(val);
+}
+
+#endif /* ZSTD_BITS_H */
diff --git a/contrib/zstd/bitstream.h b/contrib/zstd/bitstream.h
new file mode 100644
index 0000000..db1b4cf
--- /dev/null
+++ b/contrib/zstd/bitstream.h
@@ -0,0 +1,437 @@
+/* ******************************************************************
+ * bitstream
+ * Part of FSE library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+#ifndef BITSTREAM_H_MODULE
+#define BITSTREAM_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+/*
+* This API consists of small unitary functions, which must be inlined for best performance.
+* Since link-time-optimization is not available for all compilers,
+* these functions are defined into a .h to be included.
+*/
+
+/*-****************************************
+* Dependencies
+******************************************/
+#include "mem.h" /* unaligned access routines */
+#include "compiler.h" /* UNLIKELY() */
+#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */
+#include "error_private.h" /* error codes and messages */
+#include "bits.h" /* ZSTD_highbit32 */
+
+
+/*=========================================
+* Target specific
+=========================================*/
+#ifndef ZSTD_NO_INTRINSICS
+# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__)
+# include <immintrin.h> /* support for bextr (experimental)/bzhi */
+# elif defined(__ICCARM__)
+# include <intrinsics.h>
+# endif
+#endif
+
+#define STREAM_ACCUMULATOR_MIN_32 25
+#define STREAM_ACCUMULATOR_MIN_64 57
+#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
+
+
+/*-******************************************
+* bitStream encoding API (write forward)
+********************************************/
+/* bitStream can mix input from multiple sources.
+ * A critical property of these streams is that they encode and decode in **reverse** direction.
+ * So the first bit sequence you add will be the last to be read, like a LIFO stack.
+ */
+typedef struct {
+ size_t bitContainer;
+ unsigned bitPos;
+ char* startPtr;
+ char* ptr;
+ char* endPtr;
+} BIT_CStream_t;
+
+MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
+MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
+MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
+MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
+
+/* Start with initCStream, providing the size of buffer to write into.
+* bitStream will never write outside of this buffer.
+* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
+*
+* bits are first added to a local register.
+* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
+* Writing data into memory is an explicit operation, performed by the flushBits function.
+* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
+* After a flushBits, a maximum of 7 bits might still be stored into local register.
+*
+* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
+*
+* Last operation is to close the bitStream.
+* The function returns the final size of CStream in bytes.
+* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
+*/
+
+
+/*-********************************************
+* bitStream decoding API (read backward)
+**********************************************/
+typedef struct {
+ size_t bitContainer;
+ unsigned bitsConsumed;
+ const char* ptr;
+ const char* start;
+ const char* limitPtr;
+} BIT_DStream_t;
+
+typedef enum { BIT_DStream_unfinished = 0,
+ BIT_DStream_endOfBuffer = 1,
+ BIT_DStream_completed = 2,
+ BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */
+ /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
+
+MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
+MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
+MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
+MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
+
+
+/* Start by invoking BIT_initDStream().
+* A chunk of the bitStream is then stored into a local register.
+* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
+* You can then retrieve bitFields stored into the local register, **in reverse order**.
+* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
+* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
+* Otherwise, it can be less than that, so proceed accordingly.
+* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
+*/
+
+
+/*-****************************************
+* unsafe API
+******************************************/
+MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
+/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
+
+MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
+/* unsafe version; does not check buffer overflow */
+
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
+/* faster, but works only if nbBits >= 1 */
+
+/*===== Local Constants =====*/
+static const unsigned BIT_mask[] = {
+ 0, 1, 3, 7, 0xF, 0x1F,
+ 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF,
+ 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,
+ 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
+ 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
+ 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */
+#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))
+
+/*-**************************************************************
+* bitStream encoding
+****************************************************************/
+/*! BIT_initCStream() :
+ * `dstCapacity` must be > sizeof(size_t)
+ * @return : 0 if success,
+ * otherwise an error code (can be tested using ERR_isError()) */
+MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
+ void* startPtr, size_t dstCapacity)
+{
+ bitC->bitContainer = 0;
+ bitC->bitPos = 0;
+ bitC->startPtr = (char*)startPtr;
+ bitC->ptr = bitC->startPtr;
+ bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer);
+ if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall);
+ return 0;
+}
+
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
+{
+#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS)
+ return _bzhi_u64(bitContainer, nbBits);
+#else
+ assert(nbBits < BIT_MASK_SIZE);
+ return bitContainer & BIT_mask[nbBits];
+#endif
+}
+
+/*! BIT_addBits() :
+ * can add up to 31 bits into `bitC`.
+ * Note : does not check for register overflow ! */
+MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
+ size_t value, unsigned nbBits)
+{
+ DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
+ assert(nbBits < BIT_MASK_SIZE);
+ assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+ bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos;
+ bitC->bitPos += nbBits;
+}
+
+/*! BIT_addBitsFast() :
+ * works only if `value` is _clean_,
+ * meaning all high bits above nbBits are 0 */
+MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
+ size_t value, unsigned nbBits)
+{
+ assert((value>>nbBits) == 0);
+ assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+ bitC->bitContainer |= value << bitC->bitPos;
+ bitC->bitPos += nbBits;
+}
+
+/*! BIT_flushBitsFast() :
+ * assumption : bitContainer has not overflowed
+ * unsafe version; does not check buffer overflow */
+MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
+{
+ size_t const nbBytes = bitC->bitPos >> 3;
+ assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+ assert(bitC->ptr <= bitC->endPtr);
+ MEM_writeLEST(bitC->ptr, bitC->bitContainer);
+ bitC->ptr += nbBytes;
+ bitC->bitPos &= 7;
+ bitC->bitContainer >>= nbBytes*8;
+}
+
+/*! BIT_flushBits() :
+ * assumption : bitContainer has not overflowed
+ * safe version; check for buffer overflow, and prevents it.
+ * note : does not signal buffer overflow.
+ * overflow will be revealed later on using BIT_closeCStream() */
+MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
+{
+ size_t const nbBytes = bitC->bitPos >> 3;
+ assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
+ assert(bitC->ptr <= bitC->endPtr);
+ MEM_writeLEST(bitC->ptr, bitC->bitContainer);
+ bitC->ptr += nbBytes;
+ if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
+ bitC->bitPos &= 7;
+ bitC->bitContainer >>= nbBytes*8;
+}
+
+/*! BIT_closeCStream() :
+ * @return : size of CStream, in bytes,
+ * or 0 if it could not fit into dstBuffer */
+MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
+{
+ BIT_addBitsFast(bitC, 1, 1); /* endMark */
+ BIT_flushBits(bitC);
+ if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
+ return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
+}
+
+
+/*-********************************************************
+* bitStream decoding
+**********************************************************/
+/*! BIT_initDStream() :
+ * Initialize a BIT_DStream_t.
+ * `bitD` : a pointer to an already allocated BIT_DStream_t structure.
+ * `srcSize` must be the *exact* size of the bitStream, in bytes.
+ * @return : size of stream (== srcSize), or an errorCode if a problem is detected
+ */
+MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
+{
+ if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
+
+ bitD->start = (const char*)srcBuffer;
+ bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer);
+
+ if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
+ bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
+ bitD->bitContainer = MEM_readLEST(bitD->ptr);
+ { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
+ bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
+ if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
+ } else {
+ bitD->ptr = bitD->start;
+ bitD->bitContainer = *(const BYTE*)(bitD->start);
+ switch(srcSize)
+ {
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
+ ZSTD_FALLTHROUGH;
+
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
+ ZSTD_FALLTHROUGH;
+
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
+ ZSTD_FALLTHROUGH;
+
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
+ ZSTD_FALLTHROUGH;
+
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
+ ZSTD_FALLTHROUGH;
+
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
+ ZSTD_FALLTHROUGH;
+
+ default: break;
+ }
+ { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
+ bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
+ if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */
+ }
+ bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
+ }
+
+ return srcSize;
+}
+
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
+{
+ return bitContainer >> start;
+}
+
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
+{
+ U32 const regMask = sizeof(bitContainer)*8 - 1;
+ /* if start > regMask, bitstream is corrupted, and result is undefined */
+ assert(nbBits < BIT_MASK_SIZE);
+ /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better
+ * than accessing memory. When bmi2 instruction is not present, we consider
+ * such cpus old (pre-Haswell, 2013) and their performance is not of that
+ * importance.
+ */
+#if defined(__x86_64__) || defined(_M_X86)
+ return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
+#else
+ return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
+#endif
+}
+
+/*! BIT_lookBits() :
+ * Provides next n bits from local register.
+ * local register is not modified.
+ * On 32-bits, maxNbBits==24.
+ * On 64-bits, maxNbBits==56.
+ * @return : value extracted */
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
+{
+ /* arbitrate between double-shift and shift+mask */
+#if 1
+ /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8,
+ * bitstream is likely corrupted, and result is undefined */
+ return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
+#else
+ /* this code path is slower on my os-x laptop */
+ U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
+ return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
+#endif
+}
+
+/*! BIT_lookBitsFast() :
+ * unsafe version; only works if nbBits >= 1 */
+MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
+{
+ U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
+ assert(nbBits >= 1);
+ return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
+}
+
+MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
+{
+ bitD->bitsConsumed += nbBits;
+}
+
+/*! BIT_readBits() :
+ * Read (consume) next n bits from local register and update.
+ * Pay attention to not read more than nbBits contained into local register.
+ * @return : extracted value. */
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
+{
+ size_t const value = BIT_lookBits(bitD, nbBits);
+ BIT_skipBits(bitD, nbBits);
+ return value;
+}
+
+/*! BIT_readBitsFast() :
+ * unsafe version; only works if nbBits >= 1 */
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
+{
+ size_t const value = BIT_lookBitsFast(bitD, nbBits);
+ assert(nbBits >= 1);
+ BIT_skipBits(bitD, nbBits);
+ return value;
+}
+
+/*! BIT_reloadDStreamFast() :
+ * Similar to BIT_reloadDStream(), but with two differences:
+ * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
+ * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this
+ * point you must use BIT_reloadDStream() to reload.
+ */
+MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
+{
+ if (UNLIKELY(bitD->ptr < bitD->limitPtr))
+ return BIT_DStream_overflow;
+ assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
+ bitD->ptr -= bitD->bitsConsumed >> 3;
+ bitD->bitsConsumed &= 7;
+ bitD->bitContainer = MEM_readLEST(bitD->ptr);
+ return BIT_DStream_unfinished;
+}
+
+/*! BIT_reloadDStream() :
+ * Refill `bitD` from buffer previously set in BIT_initDStream() .
+ * This function is safe, it guarantees it will not read beyond src buffer.
+ * @return : status of `BIT_DStream_t` internal register.
+ * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
+MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
+{
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */
+ return BIT_DStream_overflow;
+
+ if (bitD->ptr >= bitD->limitPtr) {
+ return BIT_reloadDStreamFast(bitD);
+ }
+ if (bitD->ptr == bitD->start) {
+ if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
+ return BIT_DStream_completed;
+ }
+ /* start < ptr < limitPtr */
+ { U32 nbBytes = bitD->bitsConsumed >> 3;
+ BIT_DStream_status result = BIT_DStream_unfinished;
+ if (bitD->ptr - nbBytes < bitD->start) {
+ nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
+ result = BIT_DStream_endOfBuffer;
+ }
+ bitD->ptr -= nbBytes;
+ bitD->bitsConsumed -= nbBytes*8;
+ bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */
+ return result;
+ }
+}
+
+/*! BIT_endOfDStream() :
+ * @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
+ */
+MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
+{
+ return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* BITSTREAM_H_MODULE */
diff --git a/contrib/zstd/clevels.h b/contrib/zstd/clevels.h
new file mode 100644
index 0000000..650a34f
--- /dev/null
+++ b/contrib/zstd/clevels.h
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_CLEVELS_H
+#define ZSTD_CLEVELS_H
+
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
+#include "zstd.h"
+
+/*-===== Pre-defined compression levels =====-*/
+
+#define ZSTD_MAX_CLEVEL 22
+
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+
+static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
+{ /* "default" - for any srcSize > 256 KB */
+ /* W, C, H, S, L, TL, strat */
+ { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */
+ { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */
+ { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */
+ { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */
+ { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */
+ { 21, 18, 19, 3, 5, 2, ZSTD_greedy }, /* level 5 */
+ { 21, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6 */
+ { 21, 19, 20, 4, 5, 8, ZSTD_lazy }, /* level 7 */
+ { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 8 */
+ { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */
+ { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 10 */
+ { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 11 */
+ { 22, 22, 23, 6, 5, 32, ZSTD_lazy2 }, /* level 12 */
+ { 22, 22, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */
+ { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */
+ { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */
+ { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */
+ { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */
+ { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */
+ { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */
+ { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */
+ { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */
+ { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */
+},
+{ /* for srcSize <= 256 KB */
+ /* W, C, H, S, L, T, strat */
+ { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
+ { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */
+ { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */
+ { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */
+ { 18, 16, 17, 3, 5, 2, ZSTD_greedy }, /* level 4.*/
+ { 18, 17, 18, 5, 5, 2, ZSTD_greedy }, /* level 5.*/
+ { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/
+ { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */
+ { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
+ { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
+ { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
+ { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/
+ { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/
+ { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */
+ { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
+ { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/
+ { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/
+ { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/
+ { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/
+ { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
+ { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/
+ { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/
+ { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/
+},
+{ /* for srcSize <= 128 KB */
+ /* W, C, H, S, L, T, strat */
+ { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
+ { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */
+ { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */
+ { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */
+ { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */
+ { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */
+ { 17, 16, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */
+ { 17, 16, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */
+ { 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
+ { 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
+ { 17, 16, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
+ { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */
+ { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */
+ { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/
+ { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
+ { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/
+ { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/
+ { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/
+ { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/
+ { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/
+ { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/
+ { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
+ { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/
+},
+{ /* for srcSize <= 16 KB */
+ /* W, C, H, S, L, T, strat */
+ { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
+ { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */
+ { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */
+ { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */
+ { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */
+ { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/
+ { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */
+ { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */
+ { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/
+ { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/
+ { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/
+ { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/
+ { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/
+ { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/
+ { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/
+ { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/
+ { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/
+ { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/
+ { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/
+ { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
+ { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/
+ { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
+ { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/
+},
+};
+
+
+
+#endif /* ZSTD_CLEVELS_H */
diff --git a/contrib/zstd/compiler.h b/contrib/zstd/compiler.h
new file mode 100644
index 0000000..d4f2f28
--- /dev/null
+++ b/contrib/zstd/compiler.h
@@ -0,0 +1,354 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMPILER_H
+#define ZSTD_COMPILER_H
+
+#include "portability_macros.h"
+
+/*-*******************************************************
+* Compiler specifics
+*********************************************************/
+/* force inlining */
+
+#if !defined(ZSTD_NO_INLINE)
+#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# define INLINE_KEYWORD inline
+#else
+# define INLINE_KEYWORD
+#endif
+
+#if defined(__GNUC__) || defined(__ICCARM__)
+# define FORCE_INLINE_ATTR __attribute__((always_inline))
+#elif defined(_MSC_VER)
+# define FORCE_INLINE_ATTR __forceinline
+#else
+# define FORCE_INLINE_ATTR
+#endif
+
+#else
+
+#define INLINE_KEYWORD
+#define FORCE_INLINE_ATTR
+
+#endif
+
+/**
+ On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC).
+ This explicitly marks such functions as __cdecl so that the code will still compile
+ if a CC other than __cdecl has been made the default.
+*/
+#if defined(_MSC_VER)
+# define WIN_CDECL __cdecl
+#else
+# define WIN_CDECL
+#endif
+
+/**
+ * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
+ * parameters. They must be inlined for the compiler to eliminate the constant
+ * branches.
+ */
+#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
+/**
+ * HINT_INLINE is used to help the compiler generate better code. It is *not*
+ * used for "templates", so it can be tweaked based on the compilers
+ * performance.
+ *
+ * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the
+ * always_inline attribute.
+ *
+ * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline
+ * attribute.
+ */
+#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5
+# define HINT_INLINE static INLINE_KEYWORD
+#else
+# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR
+#endif
+
+/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
+#if defined(__GNUC__)
+# define UNUSED_ATTR __attribute__((unused))
+#else
+# define UNUSED_ATTR
+#endif
+
+/* force no inlining */
+#ifdef _MSC_VER
+# define FORCE_NOINLINE static __declspec(noinline)
+#else
+# if defined(__GNUC__) || defined(__ICCARM__)
+# define FORCE_NOINLINE static __attribute__((__noinline__))
+# else
+# define FORCE_NOINLINE static
+# endif
+#endif
+
+
+/* target attribute */
+#if defined(__GNUC__) || defined(__ICCARM__)
+# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
+#else
+# define TARGET_ATTRIBUTE(target)
+#endif
+
+/* Target attribute for BMI2 dynamic dispatch.
+ * Enable lzcnt, bmi, and bmi2.
+ * We test for bmi1 & bmi2. lzcnt is included in bmi1.
+ */
+#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2")
+
+/* prefetch
+ * can be disabled, by declaring NO_PREFETCH build macro */
+#if defined(NO_PREFETCH)
+# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
+# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
+#else
+# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
+# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
+# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
+# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
+# elif defined(__aarch64__)
+# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr)))
+# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr)))
+# else
+# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
+# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
+# endif
+#endif /* NO_PREFETCH */
+
+#define CACHELINE_SIZE 64
+
+#define PREFETCH_AREA(p, s) { \
+ const char* const _ptr = (const char*)(p); \
+ size_t const _size = (size_t)(s); \
+ size_t _pos; \
+ for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \
+ PREFETCH_L2(_ptr + _pos); \
+ } \
+}
+
+/* vectorization
+ * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax,
+ * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */
+#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__)
+# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5)
+# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
+# else
+# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")")
+# endif
+#else
+# define DONT_VECTORIZE
+#endif
+
+/* Tell the compiler that a branch is likely or unlikely.
+ * Only use these macros if it causes the compiler to generate better code.
+ * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc
+ * and clang, please do.
+ */
+#if defined(__GNUC__)
+#define LIKELY(x) (__builtin_expect((x), 1))
+#define UNLIKELY(x) (__builtin_expect((x), 0))
+#else
+#define LIKELY(x) (x)
+#define UNLIKELY(x) (x)
+#endif
+
+#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5)))
+# define ZSTD_UNREACHABLE { assert(0), __builtin_unreachable(); }
+#else
+# define ZSTD_UNREACHABLE { assert(0); }
+#endif
+
+/* disable warnings */
+#ifdef _MSC_VER /* Visual Studio */
+# include <intrin.h> /* For Visual 2005 */
+# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
+# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
+# pragma warning(disable : 4324) /* disable: C4324: padded structure */
+#endif
+
+/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/
+#ifndef STATIC_BMI2
+# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))
+# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2
+# define STATIC_BMI2 1
+# endif
+# elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__)
+# define STATIC_BMI2 1
+# endif
+#endif
+
+#ifndef STATIC_BMI2
+ #define STATIC_BMI2 0
+#endif
+
+/* compile time determination of SIMD support */
+#if !defined(ZSTD_NO_INTRINSICS)
+# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
+# define ZSTD_ARCH_X86_SSE2
+# endif
+# if defined(__ARM_NEON) || defined(_M_ARM64)
+# define ZSTD_ARCH_ARM_NEON
+# endif
+#
+# if defined(ZSTD_ARCH_X86_SSE2)
+# include <emmintrin.h>
+# elif defined(ZSTD_ARCH_ARM_NEON)
+# include <arm_neon.h>
+# endif
+#endif
+
+/* C-language Attributes are added in C23. */
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute)
+# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
+#else
+# define ZSTD_HAS_C_ATTRIBUTE(x) 0
+#endif
+
+/* Only use C++ attributes in C++. Some compilers report support for C++
+ * attributes when compiling with C.
+ */
+#if defined(__cplusplus) && defined(__has_cpp_attribute)
+# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0
+#endif
+
+/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute.
+ * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough
+ * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough
+ * - Else: __attribute__((__fallthrough__))
+ */
+#ifndef ZSTD_FALLTHROUGH
+# if ZSTD_HAS_C_ATTRIBUTE(fallthrough)
+# define ZSTD_FALLTHROUGH [[fallthrough]]
+# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough)
+# define ZSTD_FALLTHROUGH [[fallthrough]]
+# elif __has_attribute(__fallthrough__)
+/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon
+ * gcc complains about: a label can only be part of a statement and a declaration is not a statement.
+ */
+# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__))
+# else
+# define ZSTD_FALLTHROUGH
+# endif
+#endif
+
+/*-**************************************************************
+* Alignment check
+*****************************************************************/
+
+/* this test was initially positioned in mem.h,
+ * but this file is removed (or replaced) for linux kernel
+ * so it's now hosted in compiler.h,
+ * which remains valid for both user & kernel spaces.
+ */
+
+#ifndef ZSTD_ALIGNOF
+# if defined(__GNUC__) || defined(_MSC_VER)
+/* covers gcc, clang & MSVC */
+/* note : this section must come first, before C11,
+ * due to a limitation in the kernel source generator */
+# define ZSTD_ALIGNOF(T) __alignof(T)
+
+# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
+/* C11 support */
+# include <stdalign.h>
+# define ZSTD_ALIGNOF(T) alignof(T)
+
+# else
+/* No known support for alignof() - imperfect backup */
+# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T))
+
+# endif
+#endif /* ZSTD_ALIGNOF */
+
+/*-**************************************************************
+* Sanitizer
+*****************************************************************/
+
+/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an
+ * abundance of caution, disable our custom poisoning on mingw. */
+#ifdef __MINGW32__
+#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE
+#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1
+#endif
+#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE
+#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1
+#endif
+#endif
+
+#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE)
+/* Not all platforms that support msan provide sanitizers/msan_interface.h.
+ * We therefore declare the functions we need ourselves, rather than trying to
+ * include the header file... */
+#include <stddef.h> /* size_t */
+#define ZSTD_DEPS_NEED_STDINT
+#include "zstd_deps.h" /* intptr_t */
+
+/* Make memory region fully initialized (without changing its contents). */
+void __msan_unpoison(const volatile void *a, size_t size);
+
+/* Make memory region fully uninitialized (without changing its contents).
+ This is a legacy interface that does not update origin information. Use
+ __msan_allocated_memory() instead. */
+void __msan_poison(const volatile void *a, size_t size);
+
+/* Returns the offset of the first (at least partially) poisoned byte in the
+ memory range, or -1 if the whole range is good. */
+intptr_t __msan_test_shadow(const volatile void *x, size_t size);
+#endif
+
+#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE)
+/* Not all platforms that support asan provide sanitizers/asan_interface.h.
+ * We therefore declare the functions we need ourselves, rather than trying to
+ * include the header file... */
+#include <stddef.h> /* size_t */
+
+/**
+ * Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable.
+ *
+ * This memory must be previously allocated by your program. Instrumented
+ * code is forbidden from accessing addresses in this region until it is
+ * unpoisoned. This function is not guaranteed to poison the entire region -
+ * it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan
+ * alignment restrictions.
+ *
+ * \note This function is not thread-safe because no two threads can poison or
+ * unpoison memory in the same memory region simultaneously.
+ *
+ * \param addr Start of memory region.
+ * \param size Size of memory region. */
+void __asan_poison_memory_region(void const volatile *addr, size_t size);
+
+/**
+ * Marks a memory region (<c>[addr, addr+size)</c>) as addressable.
+ *
+ * This memory must be previously allocated by your program. Accessing
+ * addresses in this region is allowed until this region is poisoned again.
+ * This function could unpoison a super-region of <c>[addr, addr+size)</c> due
+ * to ASan alignment restrictions.
+ *
+ * \note This function is not thread-safe because no two threads can
+ * poison or unpoison memory in the same memory region simultaneously.
+ *
+ * \param addr Start of memory region.
+ * \param size Size of memory region. */
+void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
+#endif
+
+#endif /* ZSTD_COMPILER_H */
diff --git a/contrib/zstd/cpu.h b/contrib/zstd/cpu.h
new file mode 100644
index 0000000..8bc34a3
--- /dev/null
+++ b/contrib/zstd/cpu.h
@@ -0,0 +1,213 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMMON_CPU_H
+#define ZSTD_COMMON_CPU_H
+
+/**
+ * Implementation taken from folly/CpuId.h
+ * https://github.com/facebook/folly/blob/master/folly/CpuId.h
+ */
+
+#include "mem.h"
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+
+typedef struct {
+ U32 f1c;
+ U32 f1d;
+ U32 f7b;
+ U32 f7c;
+} ZSTD_cpuid_t;
+
+MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
+ U32 f1c = 0;
+ U32 f1d = 0;
+ U32 f7b = 0;
+ U32 f7c = 0;
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
+ int reg[4];
+ __cpuid((int*)reg, 0);
+ {
+ int const n = reg[0];
+ if (n >= 1) {
+ __cpuid((int*)reg, 1);
+ f1c = (U32)reg[2];
+ f1d = (U32)reg[3];
+ }
+ if (n >= 7) {
+ __cpuidex((int*)reg, 7, 0);
+ f7b = (U32)reg[1];
+ f7c = (U32)reg[2];
+ }
+ }
+#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__)
+ /* The following block like the normal cpuid branch below, but gcc
+ * reserves ebx for use of its pic register so we must specially
+ * handle the save and restore to avoid clobbering the register
+ */
+ U32 n;
+ __asm__(
+ "pushl %%ebx\n\t"
+ "cpuid\n\t"
+ "popl %%ebx\n\t"
+ : "=a"(n)
+ : "a"(0)
+ : "ecx", "edx");
+ if (n >= 1) {
+ U32 f1a;
+ __asm__(
+ "pushl %%ebx\n\t"
+ "cpuid\n\t"
+ "popl %%ebx\n\t"
+ : "=a"(f1a), "=c"(f1c), "=d"(f1d)
+ : "a"(1));
+ }
+ if (n >= 7) {
+ __asm__(
+ "pushl %%ebx\n\t"
+ "cpuid\n\t"
+ "movl %%ebx, %%eax\n\t"
+ "popl %%ebx"
+ : "=a"(f7b), "=c"(f7c)
+ : "a"(7), "c"(0)
+ : "edx");
+ }
+#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
+ U32 n;
+ __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx");
+ if (n >= 1) {
+ U32 f1a;
+ __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx");
+ }
+ if (n >= 7) {
+ U32 f7a;
+ __asm__("cpuid"
+ : "=a"(f7a), "=b"(f7b), "=c"(f7c)
+ : "a"(7), "c"(0)
+ : "edx");
+ }
+#endif
+ {
+ ZSTD_cpuid_t cpuid;
+ cpuid.f1c = f1c;
+ cpuid.f1d = f1d;
+ cpuid.f7b = f7b;
+ cpuid.f7c = f7c;
+ return cpuid;
+ }
+}
+
+#define X(name, r, bit) \
+ MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \
+ return ((cpuid.r) & (1U << bit)) != 0; \
+ }
+
+/* cpuid(1): Processor Info and Feature Bits. */
+#define C(name, bit) X(name, f1c, bit)
+ C(sse3, 0)
+ C(pclmuldq, 1)
+ C(dtes64, 2)
+ C(monitor, 3)
+ C(dscpl, 4)
+ C(vmx, 5)
+ C(smx, 6)
+ C(eist, 7)
+ C(tm2, 8)
+ C(ssse3, 9)
+ C(cnxtid, 10)
+ C(fma, 12)
+ C(cx16, 13)
+ C(xtpr, 14)
+ C(pdcm, 15)
+ C(pcid, 17)
+ C(dca, 18)
+ C(sse41, 19)
+ C(sse42, 20)
+ C(x2apic, 21)
+ C(movbe, 22)
+ C(popcnt, 23)
+ C(tscdeadline, 24)
+ C(aes, 25)
+ C(xsave, 26)
+ C(osxsave, 27)
+ C(avx, 28)
+ C(f16c, 29)
+ C(rdrand, 30)
+#undef C
+#define D(name, bit) X(name, f1d, bit)
+ D(fpu, 0)
+ D(vme, 1)
+ D(de, 2)
+ D(pse, 3)
+ D(tsc, 4)
+ D(msr, 5)
+ D(pae, 6)
+ D(mce, 7)
+ D(cx8, 8)
+ D(apic, 9)
+ D(sep, 11)
+ D(mtrr, 12)
+ D(pge, 13)
+ D(mca, 14)
+ D(cmov, 15)
+ D(pat, 16)
+ D(pse36, 17)
+ D(psn, 18)
+ D(clfsh, 19)
+ D(ds, 21)
+ D(acpi, 22)
+ D(mmx, 23)
+ D(fxsr, 24)
+ D(sse, 25)
+ D(sse2, 26)
+ D(ss, 27)
+ D(htt, 28)
+ D(tm, 29)
+ D(pbe, 31)
+#undef D
+
+/* cpuid(7): Extended Features. */
+#define B(name, bit) X(name, f7b, bit)
+ B(bmi1, 3)
+ B(hle, 4)
+ B(avx2, 5)
+ B(smep, 7)
+ B(bmi2, 8)
+ B(erms, 9)
+ B(invpcid, 10)
+ B(rtm, 11)
+ B(mpx, 14)
+ B(avx512f, 16)
+ B(avx512dq, 17)
+ B(rdseed, 18)
+ B(adx, 19)
+ B(smap, 20)
+ B(avx512ifma, 21)
+ B(pcommit, 22)
+ B(clflushopt, 23)
+ B(clwb, 24)
+ B(avx512pf, 26)
+ B(avx512er, 27)
+ B(avx512cd, 28)
+ B(sha, 29)
+ B(avx512bw, 30)
+ B(avx512vl, 31)
+#undef B
+#define C(name, bit) X(name, f7c, bit)
+ C(prefetchwt1, 0)
+ C(avx512vbmi, 1)
+#undef C
+
+#undef X
+
+#endif /* ZSTD_COMMON_CPU_H */
diff --git a/contrib/zstd/debug.c b/contrib/zstd/debug.c
new file mode 100644
index 0000000..ebf7bfc
--- /dev/null
+++ b/contrib/zstd/debug.c
@@ -0,0 +1,24 @@
+/* ******************************************************************
+ * debug
+ * Part of FSE library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+
+/*
+ * This module only hosts one global variable
+ * which can be used to dynamically influence the verbosity of traces,
+ * such as DEBUGLOG and RAWLOG
+ */
+
+#include "debug.h"
+
+int g_debuglevel = DEBUGLEVEL;
diff --git a/contrib/zstd/debug.h b/contrib/zstd/debug.h
new file mode 100644
index 0000000..0e9817e
--- /dev/null
+++ b/contrib/zstd/debug.h
@@ -0,0 +1,107 @@
+/* ******************************************************************
+ * debug
+ * Part of FSE library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+
+/*
+ * The purpose of this header is to enable debug functions.
+ * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,
+ * and DEBUG_STATIC_ASSERT() for compile-time.
+ *
+ * By default, DEBUGLEVEL==0, which means run-time debug is disabled.
+ *
+ * Level 1 enables assert() only.
+ * Starting level 2, traces can be generated and pushed to stderr.
+ * The higher the level, the more verbose the traces.
+ *
+ * It's possible to dynamically adjust level using variable g_debug_level,
+ * which is only declared if DEBUGLEVEL>=2,
+ * and is a global variable, not multi-thread protected (use with care)
+ */
+
+#ifndef DEBUG_H_12987983217
+#define DEBUG_H_12987983217
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* static assert is triggered at compile time, leaving no runtime artefact.
+ * static assert only works with compile-time constants.
+ * Also, this variant can only be used inside a function. */
+#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
+
+
+/* DEBUGLEVEL is expected to be defined externally,
+ * typically through compiler command line.
+ * Value must be a number. */
+#ifndef DEBUGLEVEL
+# define DEBUGLEVEL 0
+#endif
+
+
+/* recommended values for DEBUGLEVEL :
+ * 0 : release mode, no debug, all run-time checks disabled
+ * 1 : enables assert() only, no display
+ * 2 : reserved, for currently active debug path
+ * 3 : events once per object lifetime (CCtx, CDict, etc.)
+ * 4 : events once per frame
+ * 5 : events once per block
+ * 6 : events once per sequence (verbose)
+ * 7+: events at every position (*very* verbose)
+ *
+ * It's generally inconvenient to output traces > 5.
+ * In which case, it's possible to selectively trigger high verbosity levels
+ * by modifying g_debug_level.
+ */
+
+#if (DEBUGLEVEL>=1)
+# define ZSTD_DEPS_NEED_ASSERT
+# include "zstd_deps.h"
+#else
+# ifndef assert /* assert may be already defined, due to prior #include <assert.h> */
+# define assert(condition) ((void)0) /* disable assert (default) */
+# endif
+#endif
+
+#if (DEBUGLEVEL>=2)
+# define ZSTD_DEPS_NEED_IO
+# include "zstd_deps.h"
+extern int g_debuglevel; /* the variable is only declared,
+ it actually lives in debug.c,
+ and is shared by the whole process.
+ It's not thread-safe.
+ It's useful when enabling very verbose levels
+ on selective conditions (such as position in src) */
+
+# define RAWLOG(l, ...) { \
+ if (l<=g_debuglevel) { \
+ ZSTD_DEBUG_PRINT(__VA_ARGS__); \
+ } }
+# define DEBUGLOG(l, ...) { \
+ if (l<=g_debuglevel) { \
+ ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \
+ ZSTD_DEBUG_PRINT(" \n"); \
+ } }
+#else
+# define RAWLOG(l, ...) {} /* disabled */
+# define DEBUGLOG(l, ...) {} /* disabled */
+#endif
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* DEBUG_H_12987983217 */
diff --git a/contrib/zstd/divsufsort.c b/contrib/zstd/divsufsort.c
new file mode 100644
index 0000000..60cceb0
--- /dev/null
+++ b/contrib/zstd/divsufsort.c
@@ -0,0 +1,1913 @@
+/*
+ * divsufsort.c for libdivsufsort-lite
+ * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*- Compiler specifics -*/
+#ifdef __clang__
+#pragma clang diagnostic ignored "-Wshorten-64-to-32"
+#endif
+
+#if defined(_MSC_VER)
+# pragma warning(disable : 4244)
+# pragma warning(disable : 4127) /* C4127 : Condition expression is constant */
+#endif
+
+
+/*- Dependencies -*/
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "divsufsort.h"
+
+/*- Constants -*/
+#if defined(INLINE)
+# undef INLINE
+#endif
+#if !defined(INLINE)
+# define INLINE __inline
+#endif
+#if defined(ALPHABET_SIZE) && (ALPHABET_SIZE < 1)
+# undef ALPHABET_SIZE
+#endif
+#if !defined(ALPHABET_SIZE)
+# define ALPHABET_SIZE (256)
+#endif
+#define BUCKET_A_SIZE (ALPHABET_SIZE)
+#define BUCKET_B_SIZE (ALPHABET_SIZE * ALPHABET_SIZE)
+#if defined(SS_INSERTIONSORT_THRESHOLD)
+# if SS_INSERTIONSORT_THRESHOLD < 1
+# undef SS_INSERTIONSORT_THRESHOLD
+# define SS_INSERTIONSORT_THRESHOLD (1)
+# endif
+#else
+# define SS_INSERTIONSORT_THRESHOLD (8)
+#endif
+#if defined(SS_BLOCKSIZE)
+# if SS_BLOCKSIZE < 0
+# undef SS_BLOCKSIZE
+# define SS_BLOCKSIZE (0)
+# elif 32768 <= SS_BLOCKSIZE
+# undef SS_BLOCKSIZE
+# define SS_BLOCKSIZE (32767)
+# endif
+#else
+# define SS_BLOCKSIZE (1024)
+#endif
+/* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */
+#if SS_BLOCKSIZE == 0
+# define SS_MISORT_STACKSIZE (96)
+#elif SS_BLOCKSIZE <= 4096
+# define SS_MISORT_STACKSIZE (16)
+#else
+# define SS_MISORT_STACKSIZE (24)
+#endif
+#define SS_SMERGE_STACKSIZE (32)
+#define TR_INSERTIONSORT_THRESHOLD (8)
+#define TR_STACKSIZE (64)
+
+
+/*- Macros -*/
+#ifndef SWAP
+# define SWAP(_a, _b) do { t = (_a); (_a) = (_b); (_b) = t; } while(0)
+#endif /* SWAP */
+#ifndef MIN
+# define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b))
+#endif /* MIN */
+#ifndef MAX
+# define MAX(_a, _b) (((_a) > (_b)) ? (_a) : (_b))
+#endif /* MAX */
+#define STACK_PUSH(_a, _b, _c, _d)\
+ do {\
+ assert(ssize < STACK_SIZE);\
+ stack[ssize].a = (_a), stack[ssize].b = (_b),\
+ stack[ssize].c = (_c), stack[ssize++].d = (_d);\
+ } while(0)
+#define STACK_PUSH5(_a, _b, _c, _d, _e)\
+ do {\
+ assert(ssize < STACK_SIZE);\
+ stack[ssize].a = (_a), stack[ssize].b = (_b),\
+ stack[ssize].c = (_c), stack[ssize].d = (_d), stack[ssize++].e = (_e);\
+ } while(0)
+#define STACK_POP(_a, _b, _c, _d)\
+ do {\
+ assert(0 <= ssize);\
+ if(ssize == 0) { return; }\
+ (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\
+ (_c) = stack[ssize].c, (_d) = stack[ssize].d;\
+ } while(0)
+#define STACK_POP5(_a, _b, _c, _d, _e)\
+ do {\
+ assert(0 <= ssize);\
+ if(ssize == 0) { return; }\
+ (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\
+ (_c) = stack[ssize].c, (_d) = stack[ssize].d, (_e) = stack[ssize].e;\
+ } while(0)
+#define BUCKET_A(_c0) bucket_A[(_c0)]
+#if ALPHABET_SIZE == 256
+#define BUCKET_B(_c0, _c1) (bucket_B[((_c1) << 8) | (_c0)])
+#define BUCKET_BSTAR(_c0, _c1) (bucket_B[((_c0) << 8) | (_c1)])
+#else
+#define BUCKET_B(_c0, _c1) (bucket_B[(_c1) * ALPHABET_SIZE + (_c0)])
+#define BUCKET_BSTAR(_c0, _c1) (bucket_B[(_c0) * ALPHABET_SIZE + (_c1)])
+#endif
+
+
+/*- Private Functions -*/
+
+static const int lg_table[256]= {
+ -1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
+ 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,
+ 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
+ 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7
+};
+
+#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
+
+static INLINE
+int
+ss_ilg(int n) {
+#if SS_BLOCKSIZE == 0
+ return (n & 0xffff0000) ?
+ ((n & 0xff000000) ?
+ 24 + lg_table[(n >> 24) & 0xff] :
+ 16 + lg_table[(n >> 16) & 0xff]) :
+ ((n & 0x0000ff00) ?
+ 8 + lg_table[(n >> 8) & 0xff] :
+ 0 + lg_table[(n >> 0) & 0xff]);
+#elif SS_BLOCKSIZE < 256
+ return lg_table[n];
+#else
+ return (n & 0xff00) ?
+ 8 + lg_table[(n >> 8) & 0xff] :
+ 0 + lg_table[(n >> 0) & 0xff];
+#endif
+}
+
+#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */
+
+#if SS_BLOCKSIZE != 0
+
+static const int sqq_table[256] = {
+ 0, 16, 22, 27, 32, 35, 39, 42, 45, 48, 50, 53, 55, 57, 59, 61,
+ 64, 65, 67, 69, 71, 73, 75, 76, 78, 80, 81, 83, 84, 86, 87, 89,
+ 90, 91, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 106, 107, 108, 109,
+110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126,
+128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142,
+143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155,
+156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168,
+169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180,
+181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191,
+192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201,
+202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211,
+212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221,
+221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230,
+230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238,
+239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247,
+247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255
+};
+
+static INLINE
+int
+ss_isqrt(int x) {
+ int y, e;
+
+ if(x >= (SS_BLOCKSIZE * SS_BLOCKSIZE)) { return SS_BLOCKSIZE; }
+ e = (x & 0xffff0000) ?
+ ((x & 0xff000000) ?
+ 24 + lg_table[(x >> 24) & 0xff] :
+ 16 + lg_table[(x >> 16) & 0xff]) :
+ ((x & 0x0000ff00) ?
+ 8 + lg_table[(x >> 8) & 0xff] :
+ 0 + lg_table[(x >> 0) & 0xff]);
+
+ if(e >= 16) {
+ y = sqq_table[x >> ((e - 6) - (e & 1))] << ((e >> 1) - 7);
+ if(e >= 24) { y = (y + 1 + x / y) >> 1; }
+ y = (y + 1 + x / y) >> 1;
+ } else if(e >= 8) {
+ y = (sqq_table[x >> ((e - 6) - (e & 1))] >> (7 - (e >> 1))) + 1;
+ } else {
+ return sqq_table[x] >> 4;
+ }
+
+ return (x < (y * y)) ? y - 1 : y;
+}
+
+#endif /* SS_BLOCKSIZE != 0 */
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Compares two suffixes. */
+static INLINE
+int
+ss_compare(const unsigned char *T,
+ const int *p1, const int *p2,
+ int depth) {
+ const unsigned char *U1, *U2, *U1n, *U2n;
+
+ for(U1 = T + depth + *p1,
+ U2 = T + depth + *p2,
+ U1n = T + *(p1 + 1) + 2,
+ U2n = T + *(p2 + 1) + 2;
+ (U1 < U1n) && (U2 < U2n) && (*U1 == *U2);
+ ++U1, ++U2) {
+ }
+
+ return U1 < U1n ?
+ (U2 < U2n ? *U1 - *U2 : 1) :
+ (U2 < U2n ? -1 : 0);
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+#if (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1)
+
+/* Insertionsort for small size groups */
+static
+void
+ss_insertionsort(const unsigned char *T, const int *PA,
+ int *first, int *last, int depth) {
+ int *i, *j;
+ int t;
+ int r;
+
+ for(i = last - 2; first <= i; --i) {
+ for(t = *i, j = i + 1; 0 < (r = ss_compare(T, PA + t, PA + *j, depth));) {
+ do { *(j - 1) = *j; } while((++j < last) && (*j < 0));
+ if(last <= j) { break; }
+ }
+ if(r == 0) { *j = ~*j; }
+ *(j - 1) = t;
+ }
+}
+
+#endif /* (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) */
+
+
+/*---------------------------------------------------------------------------*/
+
+#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
+
+static INLINE
+void
+ss_fixdown(const unsigned char *Td, const int *PA,
+ int *SA, int i, int size) {
+ int j, k;
+ int v;
+ int c, d, e;
+
+ for(v = SA[i], c = Td[PA[v]]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) {
+ d = Td[PA[SA[k = j++]]];
+ if(d < (e = Td[PA[SA[j]]])) { k = j; d = e; }
+ if(d <= c) { break; }
+ }
+ SA[i] = v;
+}
+
+/* Simple top-down heapsort. */
+static
+void
+ss_heapsort(const unsigned char *Td, const int *PA, int *SA, int size) {
+ int i, m;
+ int t;
+
+ m = size;
+ if((size % 2) == 0) {
+ m--;
+ if(Td[PA[SA[m / 2]]] < Td[PA[SA[m]]]) { SWAP(SA[m], SA[m / 2]); }
+ }
+
+ for(i = m / 2 - 1; 0 <= i; --i) { ss_fixdown(Td, PA, SA, i, m); }
+ if((size % 2) == 0) { SWAP(SA[0], SA[m]); ss_fixdown(Td, PA, SA, 0, m); }
+ for(i = m - 1; 0 < i; --i) {
+ t = SA[0], SA[0] = SA[i];
+ ss_fixdown(Td, PA, SA, 0, i);
+ SA[i] = t;
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Returns the median of three elements. */
+static INLINE
+int *
+ss_median3(const unsigned char *Td, const int *PA,
+ int *v1, int *v2, int *v3) {
+ int *t;
+ if(Td[PA[*v1]] > Td[PA[*v2]]) { SWAP(v1, v2); }
+ if(Td[PA[*v2]] > Td[PA[*v3]]) {
+ if(Td[PA[*v1]] > Td[PA[*v3]]) { return v1; }
+ else { return v3; }
+ }
+ return v2;
+}
+
+/* Returns the median of five elements. */
+static INLINE
+int *
+ss_median5(const unsigned char *Td, const int *PA,
+ int *v1, int *v2, int *v3, int *v4, int *v5) {
+ int *t;
+ if(Td[PA[*v2]] > Td[PA[*v3]]) { SWAP(v2, v3); }
+ if(Td[PA[*v4]] > Td[PA[*v5]]) { SWAP(v4, v5); }
+ if(Td[PA[*v2]] > Td[PA[*v4]]) { SWAP(v2, v4); SWAP(v3, v5); }
+ if(Td[PA[*v1]] > Td[PA[*v3]]) { SWAP(v1, v3); }
+ if(Td[PA[*v1]] > Td[PA[*v4]]) { SWAP(v1, v4); SWAP(v3, v5); }
+ if(Td[PA[*v3]] > Td[PA[*v4]]) { return v4; }
+ return v3;
+}
+
+/* Returns the pivot element. */
+static INLINE
+int *
+ss_pivot(const unsigned char *Td, const int *PA, int *first, int *last) {
+ int *middle;
+ int t;
+
+ t = last - first;
+ middle = first + t / 2;
+
+ if(t <= 512) {
+ if(t <= 32) {
+ return ss_median3(Td, PA, first, middle, last - 1);
+ } else {
+ t >>= 2;
+ return ss_median5(Td, PA, first, first + t, middle, last - 1 - t, last - 1);
+ }
+ }
+ t >>= 3;
+ first = ss_median3(Td, PA, first, first + t, first + (t << 1));
+ middle = ss_median3(Td, PA, middle - t, middle, middle + t);
+ last = ss_median3(Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1);
+ return ss_median3(Td, PA, first, middle, last);
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Binary partition for substrings. */
+static INLINE
+int *
+ss_partition(const int *PA,
+ int *first, int *last, int depth) {
+ int *a, *b;
+ int t;
+ for(a = first - 1, b = last;;) {
+ for(; (++a < b) && ((PA[*a] + depth) >= (PA[*a + 1] + 1));) { *a = ~*a; }
+ for(; (a < --b) && ((PA[*b] + depth) < (PA[*b + 1] + 1));) { }
+ if(b <= a) { break; }
+ t = ~*b;
+ *b = *a;
+ *a = t;
+ }
+ if(first < a) { *first = ~*first; }
+ return a;
+}
+
+/* Multikey introsort for medium size groups. */
+static
+void
+ss_mintrosort(const unsigned char *T, const int *PA,
+ int *first, int *last,
+ int depth) {
+#define STACK_SIZE SS_MISORT_STACKSIZE
+ struct { int *a, *b, c; int d; } stack[STACK_SIZE];
+ const unsigned char *Td;
+ int *a, *b, *c, *d, *e, *f;
+ int s, t;
+ int ssize;
+ int limit;
+ int v, x = 0;
+
+ for(ssize = 0, limit = ss_ilg(last - first);;) {
+
+ if((last - first) <= SS_INSERTIONSORT_THRESHOLD) {
+#if 1 < SS_INSERTIONSORT_THRESHOLD
+ if(1 < (last - first)) { ss_insertionsort(T, PA, first, last, depth); }
+#endif
+ STACK_POP(first, last, depth, limit);
+ continue;
+ }
+
+ Td = T + depth;
+ if(limit-- == 0) { ss_heapsort(Td, PA, first, last - first); }
+ if(limit < 0) {
+ for(a = first + 1, v = Td[PA[*first]]; a < last; ++a) {
+ if((x = Td[PA[*a]]) != v) {
+ if(1 < (a - first)) { break; }
+ v = x;
+ first = a;
+ }
+ }
+ if(Td[PA[*first] - 1] < v) {
+ first = ss_partition(PA, first, a, depth);
+ }
+ if((a - first) <= (last - a)) {
+ if(1 < (a - first)) {
+ STACK_PUSH(a, last, depth, -1);
+ last = a, depth += 1, limit = ss_ilg(a - first);
+ } else {
+ first = a, limit = -1;
+ }
+ } else {
+ if(1 < (last - a)) {
+ STACK_PUSH(first, a, depth + 1, ss_ilg(a - first));
+ first = a, limit = -1;
+ } else {
+ last = a, depth += 1, limit = ss_ilg(a - first);
+ }
+ }
+ continue;
+ }
+
+ /* choose pivot */
+ a = ss_pivot(Td, PA, first, last);
+ v = Td[PA[*a]];
+ SWAP(*first, *a);
+
+ /* partition */
+ for(b = first; (++b < last) && ((x = Td[PA[*b]]) == v);) { }
+ if(((a = b) < last) && (x < v)) {
+ for(; (++b < last) && ((x = Td[PA[*b]]) <= v);) {
+ if(x == v) { SWAP(*b, *a); ++a; }
+ }
+ }
+ for(c = last; (b < --c) && ((x = Td[PA[*c]]) == v);) { }
+ if((b < (d = c)) && (x > v)) {
+ for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) {
+ if(x == v) { SWAP(*c, *d); --d; }
+ }
+ }
+ for(; b < c;) {
+ SWAP(*b, *c);
+ for(; (++b < c) && ((x = Td[PA[*b]]) <= v);) {
+ if(x == v) { SWAP(*b, *a); ++a; }
+ }
+ for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) {
+ if(x == v) { SWAP(*c, *d); --d; }
+ }
+ }
+
+ if(a <= d) {
+ c = b - 1;
+
+ if((s = a - first) > (t = b - a)) { s = t; }
+ for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
+ if((s = d - c) > (t = last - d - 1)) { s = t; }
+ for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
+
+ a = first + (b - a), c = last - (d - c);
+ b = (v <= Td[PA[*a] - 1]) ? a : ss_partition(PA, a, c, depth);
+
+ if((a - first) <= (last - c)) {
+ if((last - c) <= (c - b)) {
+ STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
+ STACK_PUSH(c, last, depth, limit);
+ last = a;
+ } else if((a - first) <= (c - b)) {
+ STACK_PUSH(c, last, depth, limit);
+ STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
+ last = a;
+ } else {
+ STACK_PUSH(c, last, depth, limit);
+ STACK_PUSH(first, a, depth, limit);
+ first = b, last = c, depth += 1, limit = ss_ilg(c - b);
+ }
+ } else {
+ if((a - first) <= (c - b)) {
+ STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
+ STACK_PUSH(first, a, depth, limit);
+ first = c;
+ } else if((last - c) <= (c - b)) {
+ STACK_PUSH(first, a, depth, limit);
+ STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
+ first = c;
+ } else {
+ STACK_PUSH(first, a, depth, limit);
+ STACK_PUSH(c, last, depth, limit);
+ first = b, last = c, depth += 1, limit = ss_ilg(c - b);
+ }
+ }
+ } else {
+ limit += 1;
+ if(Td[PA[*first] - 1] < v) {
+ first = ss_partition(PA, first, last, depth);
+ limit = ss_ilg(last - first);
+ }
+ depth += 1;
+ }
+ }
+#undef STACK_SIZE
+}
+
+#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */
+
+
+/*---------------------------------------------------------------------------*/
+
+#if SS_BLOCKSIZE != 0
+
+static INLINE
+void
+ss_blockswap(int *a, int *b, int n) {
+ int t;
+ for(; 0 < n; --n, ++a, ++b) {
+ t = *a, *a = *b, *b = t;
+ }
+}
+
+static INLINE
+void
+ss_rotate(int *first, int *middle, int *last) {
+ int *a, *b, t;
+ int l, r;
+ l = middle - first, r = last - middle;
+ for(; (0 < l) && (0 < r);) {
+ if(l == r) { ss_blockswap(first, middle, l); break; }
+ if(l < r) {
+ a = last - 1, b = middle - 1;
+ t = *a;
+ do {
+ *a-- = *b, *b-- = *a;
+ if(b < first) {
+ *a = t;
+ last = a;
+ if((r -= l + 1) <= l) { break; }
+ a -= 1, b = middle - 1;
+ t = *a;
+ }
+ } while(1);
+ } else {
+ a = first, b = middle;
+ t = *a;
+ do {
+ *a++ = *b, *b++ = *a;
+ if(last <= b) {
+ *a = t;
+ first = a + 1;
+ if((l -= r + 1) <= r) { break; }
+ a += 1, b = middle;
+ t = *a;
+ }
+ } while(1);
+ }
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+static
+void
+ss_inplacemerge(const unsigned char *T, const int *PA,
+ int *first, int *middle, int *last,
+ int depth) {
+ const int *p;
+ int *a, *b;
+ int len, half;
+ int q, r;
+ int x;
+
+ for(;;) {
+ if(*(last - 1) < 0) { x = 1; p = PA + ~*(last - 1); }
+ else { x = 0; p = PA + *(last - 1); }
+ for(a = first, len = middle - first, half = len >> 1, r = -1;
+ 0 < len;
+ len = half, half >>= 1) {
+ b = a + half;
+ q = ss_compare(T, PA + ((0 <= *b) ? *b : ~*b), p, depth);
+ if(q < 0) {
+ a = b + 1;
+ half -= (len & 1) ^ 1;
+ } else {
+ r = q;
+ }
+ }
+ if(a < middle) {
+ if(r == 0) { *a = ~*a; }
+ ss_rotate(a, middle, last);
+ last -= middle - a;
+ middle = a;
+ if(first == middle) { break; }
+ }
+ --last;
+ if(x != 0) { while(*--last < 0) { } }
+ if(middle == last) { break; }
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Merge-forward with internal buffer. */
+static
+void
+ss_mergeforward(const unsigned char *T, const int *PA,
+ int *first, int *middle, int *last,
+ int *buf, int depth) {
+ int *a, *b, *c, *bufend;
+ int t;
+ int r;
+
+ bufend = buf + (middle - first) - 1;
+ ss_blockswap(buf, first, middle - first);
+
+ for(t = *(a = first), b = buf, c = middle;;) {
+ r = ss_compare(T, PA + *b, PA + *c, depth);
+ if(r < 0) {
+ do {
+ *a++ = *b;
+ if(bufend <= b) { *bufend = t; return; }
+ *b++ = *a;
+ } while(*b < 0);
+ } else if(r > 0) {
+ do {
+ *a++ = *c, *c++ = *a;
+ if(last <= c) {
+ while(b < bufend) { *a++ = *b, *b++ = *a; }
+ *a = *b, *b = t;
+ return;
+ }
+ } while(*c < 0);
+ } else {
+ *c = ~*c;
+ do {
+ *a++ = *b;
+ if(bufend <= b) { *bufend = t; return; }
+ *b++ = *a;
+ } while(*b < 0);
+
+ do {
+ *a++ = *c, *c++ = *a;
+ if(last <= c) {
+ while(b < bufend) { *a++ = *b, *b++ = *a; }
+ *a = *b, *b = t;
+ return;
+ }
+ } while(*c < 0);
+ }
+ }
+}
+
+/* Merge-backward with internal buffer. */
+static
+void
+ss_mergebackward(const unsigned char *T, const int *PA,
+ int *first, int *middle, int *last,
+ int *buf, int depth) {
+ const int *p1, *p2;
+ int *a, *b, *c, *bufend;
+ int t;
+ int r;
+ int x;
+
+ bufend = buf + (last - middle) - 1;
+ ss_blockswap(buf, middle, last - middle);
+
+ x = 0;
+ if(*bufend < 0) { p1 = PA + ~*bufend; x |= 1; }
+ else { p1 = PA + *bufend; }
+ if(*(middle - 1) < 0) { p2 = PA + ~*(middle - 1); x |= 2; }
+ else { p2 = PA + *(middle - 1); }
+ for(t = *(a = last - 1), b = bufend, c = middle - 1;;) {
+ r = ss_compare(T, p1, p2, depth);
+ if(0 < r) {
+ if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; }
+ *a-- = *b;
+ if(b <= buf) { *buf = t; break; }
+ *b-- = *a;
+ if(*b < 0) { p1 = PA + ~*b; x |= 1; }
+ else { p1 = PA + *b; }
+ } else if(r < 0) {
+ if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; }
+ *a-- = *c, *c-- = *a;
+ if(c < first) {
+ while(buf < b) { *a-- = *b, *b-- = *a; }
+ *a = *b, *b = t;
+ break;
+ }
+ if(*c < 0) { p2 = PA + ~*c; x |= 2; }
+ else { p2 = PA + *c; }
+ } else {
+ if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; }
+ *a-- = ~*b;
+ if(b <= buf) { *buf = t; break; }
+ *b-- = *a;
+ if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; }
+ *a-- = *c, *c-- = *a;
+ if(c < first) {
+ while(buf < b) { *a-- = *b, *b-- = *a; }
+ *a = *b, *b = t;
+ break;
+ }
+ if(*b < 0) { p1 = PA + ~*b; x |= 1; }
+ else { p1 = PA + *b; }
+ if(*c < 0) { p2 = PA + ~*c; x |= 2; }
+ else { p2 = PA + *c; }
+ }
+ }
+}
+
+/* D&C based merge. */
+static
+void
+ss_swapmerge(const unsigned char *T, const int *PA,
+ int *first, int *middle, int *last,
+ int *buf, int bufsize, int depth) {
+#define STACK_SIZE SS_SMERGE_STACKSIZE
+#define GETIDX(a) ((0 <= (a)) ? (a) : (~(a)))
+#define MERGE_CHECK(a, b, c)\
+ do {\
+ if(((c) & 1) ||\
+ (((c) & 2) && (ss_compare(T, PA + GETIDX(*((a) - 1)), PA + *(a), depth) == 0))) {\
+ *(a) = ~*(a);\
+ }\
+ if(((c) & 4) && ((ss_compare(T, PA + GETIDX(*((b) - 1)), PA + *(b), depth) == 0))) {\
+ *(b) = ~*(b);\
+ }\
+ } while(0)
+ struct { int *a, *b, *c; int d; } stack[STACK_SIZE];
+ int *l, *r, *lm, *rm;
+ int m, len, half;
+ int ssize;
+ int check, next;
+
+ for(check = 0, ssize = 0;;) {
+ if((last - middle) <= bufsize) {
+ if((first < middle) && (middle < last)) {
+ ss_mergebackward(T, PA, first, middle, last, buf, depth);
+ }
+ MERGE_CHECK(first, last, check);
+ STACK_POP(first, middle, last, check);
+ continue;
+ }
+
+ if((middle - first) <= bufsize) {
+ if(first < middle) {
+ ss_mergeforward(T, PA, first, middle, last, buf, depth);
+ }
+ MERGE_CHECK(first, last, check);
+ STACK_POP(first, middle, last, check);
+ continue;
+ }
+
+ for(m = 0, len = MIN(middle - first, last - middle), half = len >> 1;
+ 0 < len;
+ len = half, half >>= 1) {
+ if(ss_compare(T, PA + GETIDX(*(middle + m + half)),
+ PA + GETIDX(*(middle - m - half - 1)), depth) < 0) {
+ m += half + 1;
+ half -= (len & 1) ^ 1;
+ }
+ }
+
+ if(0 < m) {
+ lm = middle - m, rm = middle + m;
+ ss_blockswap(lm, middle, m);
+ l = r = middle, next = 0;
+ if(rm < last) {
+ if(*rm < 0) {
+ *rm = ~*rm;
+ if(first < lm) { for(; *--l < 0;) { } next |= 4; }
+ next |= 1;
+ } else if(first < lm) {
+ for(; *r < 0; ++r) { }
+ next |= 2;
+ }
+ }
+
+ if((l - first) <= (last - r)) {
+ STACK_PUSH(r, rm, last, (next & 3) | (check & 4));
+ middle = lm, last = l, check = (check & 3) | (next & 4);
+ } else {
+ if((next & 2) && (r == middle)) { next ^= 6; }
+ STACK_PUSH(first, lm, l, (check & 3) | (next & 4));
+ first = r, middle = rm, check = (next & 3) | (check & 4);
+ }
+ } else {
+ if(ss_compare(T, PA + GETIDX(*(middle - 1)), PA + *middle, depth) == 0) {
+ *middle = ~*middle;
+ }
+ MERGE_CHECK(first, last, check);
+ STACK_POP(first, middle, last, check);
+ }
+ }
+#undef STACK_SIZE
+}
+
+#endif /* SS_BLOCKSIZE != 0 */
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Substring sort */
+static
+void
+sssort(const unsigned char *T, const int *PA,
+ int *first, int *last,
+ int *buf, int bufsize,
+ int depth, int n, int lastsuffix) {
+ int *a;
+#if SS_BLOCKSIZE != 0
+ int *b, *middle, *curbuf;
+ int j, k, curbufsize, limit;
+#endif
+ int i;
+
+ if(lastsuffix != 0) { ++first; }
+
+#if SS_BLOCKSIZE == 0
+ ss_mintrosort(T, PA, first, last, depth);
+#else
+ if((bufsize < SS_BLOCKSIZE) &&
+ (bufsize < (last - first)) &&
+ (bufsize < (limit = ss_isqrt(last - first)))) {
+ if(SS_BLOCKSIZE < limit) { limit = SS_BLOCKSIZE; }
+ buf = middle = last - limit, bufsize = limit;
+ } else {
+ middle = last, limit = 0;
+ }
+ for(a = first, i = 0; SS_BLOCKSIZE < (middle - a); a += SS_BLOCKSIZE, ++i) {
+#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
+ ss_mintrosort(T, PA, a, a + SS_BLOCKSIZE, depth);
+#elif 1 < SS_BLOCKSIZE
+ ss_insertionsort(T, PA, a, a + SS_BLOCKSIZE, depth);
+#endif
+ curbufsize = last - (a + SS_BLOCKSIZE);
+ curbuf = a + SS_BLOCKSIZE;
+ if(curbufsize <= bufsize) { curbufsize = bufsize, curbuf = buf; }
+ for(b = a, k = SS_BLOCKSIZE, j = i; j & 1; b -= k, k <<= 1, j >>= 1) {
+ ss_swapmerge(T, PA, b - k, b, b + k, curbuf, curbufsize, depth);
+ }
+ }
+#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
+ ss_mintrosort(T, PA, a, middle, depth);
+#elif 1 < SS_BLOCKSIZE
+ ss_insertionsort(T, PA, a, middle, depth);
+#endif
+ for(k = SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) {
+ if(i & 1) {
+ ss_swapmerge(T, PA, a - k, a, middle, buf, bufsize, depth);
+ a -= k;
+ }
+ }
+ if(limit != 0) {
+#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
+ ss_mintrosort(T, PA, middle, last, depth);
+#elif 1 < SS_BLOCKSIZE
+ ss_insertionsort(T, PA, middle, last, depth);
+#endif
+ ss_inplacemerge(T, PA, first, middle, last, depth);
+ }
+#endif
+
+ if(lastsuffix != 0) {
+ /* Insert last type B* suffix. */
+ int PAi[2]; PAi[0] = PA[*(first - 1)], PAi[1] = n - 2;
+ for(a = first, i = *(first - 1);
+ (a < last) && ((*a < 0) || (0 < ss_compare(T, &(PAi[0]), PA + *a, depth)));
+ ++a) {
+ *(a - 1) = *a;
+ }
+ *(a - 1) = i;
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+static INLINE
+int
+tr_ilg(int n) {
+ return (n & 0xffff0000) ?
+ ((n & 0xff000000) ?
+ 24 + lg_table[(n >> 24) & 0xff] :
+ 16 + lg_table[(n >> 16) & 0xff]) :
+ ((n & 0x0000ff00) ?
+ 8 + lg_table[(n >> 8) & 0xff] :
+ 0 + lg_table[(n >> 0) & 0xff]);
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Simple insertionsort for small size groups. */
+static
+void
+tr_insertionsort(const int *ISAd, int *first, int *last) {
+ int *a, *b;
+ int t, r;
+
+ for(a = first + 1; a < last; ++a) {
+ for(t = *a, b = a - 1; 0 > (r = ISAd[t] - ISAd[*b]);) {
+ do { *(b + 1) = *b; } while((first <= --b) && (*b < 0));
+ if(b < first) { break; }
+ }
+ if(r == 0) { *b = ~*b; }
+ *(b + 1) = t;
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+static INLINE
+void
+tr_fixdown(const int *ISAd, int *SA, int i, int size) {
+ int j, k;
+ int v;
+ int c, d, e;
+
+ for(v = SA[i], c = ISAd[v]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) {
+ d = ISAd[SA[k = j++]];
+ if(d < (e = ISAd[SA[j]])) { k = j; d = e; }
+ if(d <= c) { break; }
+ }
+ SA[i] = v;
+}
+
+/* Simple top-down heapsort. */
+static
+void
+tr_heapsort(const int *ISAd, int *SA, int size) {
+ int i, m;
+ int t;
+
+ m = size;
+ if((size % 2) == 0) {
+ m--;
+ if(ISAd[SA[m / 2]] < ISAd[SA[m]]) { SWAP(SA[m], SA[m / 2]); }
+ }
+
+ for(i = m / 2 - 1; 0 <= i; --i) { tr_fixdown(ISAd, SA, i, m); }
+ if((size % 2) == 0) { SWAP(SA[0], SA[m]); tr_fixdown(ISAd, SA, 0, m); }
+ for(i = m - 1; 0 < i; --i) {
+ t = SA[0], SA[0] = SA[i];
+ tr_fixdown(ISAd, SA, 0, i);
+ SA[i] = t;
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Returns the median of three elements. */
+static INLINE
+int *
+tr_median3(const int *ISAd, int *v1, int *v2, int *v3) {
+ int *t;
+ if(ISAd[*v1] > ISAd[*v2]) { SWAP(v1, v2); }
+ if(ISAd[*v2] > ISAd[*v3]) {
+ if(ISAd[*v1] > ISAd[*v3]) { return v1; }
+ else { return v3; }
+ }
+ return v2;
+}
+
+/* Returns the median of five elements. */
+static INLINE
+int *
+tr_median5(const int *ISAd,
+ int *v1, int *v2, int *v3, int *v4, int *v5) {
+ int *t;
+ if(ISAd[*v2] > ISAd[*v3]) { SWAP(v2, v3); }
+ if(ISAd[*v4] > ISAd[*v5]) { SWAP(v4, v5); }
+ if(ISAd[*v2] > ISAd[*v4]) { SWAP(v2, v4); SWAP(v3, v5); }
+ if(ISAd[*v1] > ISAd[*v3]) { SWAP(v1, v3); }
+ if(ISAd[*v1] > ISAd[*v4]) { SWAP(v1, v4); SWAP(v3, v5); }
+ if(ISAd[*v3] > ISAd[*v4]) { return v4; }
+ return v3;
+}
+
+/* Returns the pivot element. */
+static INLINE
+int *
+tr_pivot(const int *ISAd, int *first, int *last) {
+ int *middle;
+ int t;
+
+ t = last - first;
+ middle = first + t / 2;
+
+ if(t <= 512) {
+ if(t <= 32) {
+ return tr_median3(ISAd, first, middle, last - 1);
+ } else {
+ t >>= 2;
+ return tr_median5(ISAd, first, first + t, middle, last - 1 - t, last - 1);
+ }
+ }
+ t >>= 3;
+ first = tr_median3(ISAd, first, first + t, first + (t << 1));
+ middle = tr_median3(ISAd, middle - t, middle, middle + t);
+ last = tr_median3(ISAd, last - 1 - (t << 1), last - 1 - t, last - 1);
+ return tr_median3(ISAd, first, middle, last);
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+typedef struct _trbudget_t trbudget_t;
+struct _trbudget_t {
+ int chance;
+ int remain;
+ int incval;
+ int count;
+};
+
+static INLINE
+void
+trbudget_init(trbudget_t *budget, int chance, int incval) {
+ budget->chance = chance;
+ budget->remain = budget->incval = incval;
+}
+
+static INLINE
+int
+trbudget_check(trbudget_t *budget, int size) {
+ if(size <= budget->remain) { budget->remain -= size; return 1; }
+ if(budget->chance == 0) { budget->count += size; return 0; }
+ budget->remain += budget->incval - size;
+ budget->chance -= 1;
+ return 1;
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+static INLINE
+void
+tr_partition(const int *ISAd,
+ int *first, int *middle, int *last,
+ int **pa, int **pb, int v) {
+ int *a, *b, *c, *d, *e, *f;
+ int t, s;
+ int x = 0;
+
+ for(b = middle - 1; (++b < last) && ((x = ISAd[*b]) == v);) { }
+ if(((a = b) < last) && (x < v)) {
+ for(; (++b < last) && ((x = ISAd[*b]) <= v);) {
+ if(x == v) { SWAP(*b, *a); ++a; }
+ }
+ }
+ for(c = last; (b < --c) && ((x = ISAd[*c]) == v);) { }
+ if((b < (d = c)) && (x > v)) {
+ for(; (b < --c) && ((x = ISAd[*c]) >= v);) {
+ if(x == v) { SWAP(*c, *d); --d; }
+ }
+ }
+ for(; b < c;) {
+ SWAP(*b, *c);
+ for(; (++b < c) && ((x = ISAd[*b]) <= v);) {
+ if(x == v) { SWAP(*b, *a); ++a; }
+ }
+ for(; (b < --c) && ((x = ISAd[*c]) >= v);) {
+ if(x == v) { SWAP(*c, *d); --d; }
+ }
+ }
+
+ if(a <= d) {
+ c = b - 1;
+ if((s = a - first) > (t = b - a)) { s = t; }
+ for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
+ if((s = d - c) > (t = last - d - 1)) { s = t; }
+ for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
+ first += (b - a), last -= (d - c);
+ }
+ *pa = first, *pb = last;
+}
+
+static
+void
+tr_copy(int *ISA, const int *SA,
+ int *first, int *a, int *b, int *last,
+ int depth) {
+ /* sort suffixes of middle partition
+ by using sorted order of suffixes of left and right partition. */
+ int *c, *d, *e;
+ int s, v;
+
+ v = b - SA - 1;
+ for(c = first, d = a - 1; c <= d; ++c) {
+ if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
+ *++d = s;
+ ISA[s] = d - SA;
+ }
+ }
+ for(c = last - 1, e = d + 1, d = b; e < d; --c) {
+ if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
+ *--d = s;
+ ISA[s] = d - SA;
+ }
+ }
+}
+
+static
+void
+tr_partialcopy(int *ISA, const int *SA,
+ int *first, int *a, int *b, int *last,
+ int depth) {
+ int *c, *d, *e;
+ int s, v;
+ int rank, lastrank, newrank = -1;
+
+ v = b - SA - 1;
+ lastrank = -1;
+ for(c = first, d = a - 1; c <= d; ++c) {
+ if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
+ *++d = s;
+ rank = ISA[s + depth];
+ if(lastrank != rank) { lastrank = rank; newrank = d - SA; }
+ ISA[s] = newrank;
+ }
+ }
+
+ lastrank = -1;
+ for(e = d; first <= e; --e) {
+ rank = ISA[*e];
+ if(lastrank != rank) { lastrank = rank; newrank = e - SA; }
+ if(newrank != rank) { ISA[*e] = newrank; }
+ }
+
+ lastrank = -1;
+ for(c = last - 1, e = d + 1, d = b; e < d; --c) {
+ if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
+ *--d = s;
+ rank = ISA[s + depth];
+ if(lastrank != rank) { lastrank = rank; newrank = d - SA; }
+ ISA[s] = newrank;
+ }
+ }
+}
+
+static
+void
+tr_introsort(int *ISA, const int *ISAd,
+ int *SA, int *first, int *last,
+ trbudget_t *budget) {
+#define STACK_SIZE TR_STACKSIZE
+ struct { const int *a; int *b, *c; int d, e; }stack[STACK_SIZE];
+ int *a, *b, *c;
+ int t;
+ int v, x = 0;
+ int incr = ISAd - ISA;
+ int limit, next;
+ int ssize, trlink = -1;
+
+ for(ssize = 0, limit = tr_ilg(last - first);;) {
+
+ if(limit < 0) {
+ if(limit == -1) {
+ /* tandem repeat partition */
+ tr_partition(ISAd - incr, first, first, last, &a, &b, last - SA - 1);
+
+ /* update ranks */
+ if(a < last) {
+ for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; }
+ }
+ if(b < last) {
+ for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; }
+ }
+
+ /* push */
+ if(1 < (b - a)) {
+ STACK_PUSH5(NULL, a, b, 0, 0);
+ STACK_PUSH5(ISAd - incr, first, last, -2, trlink);
+ trlink = ssize - 2;
+ }
+ if((a - first) <= (last - b)) {
+ if(1 < (a - first)) {
+ STACK_PUSH5(ISAd, b, last, tr_ilg(last - b), trlink);
+ last = a, limit = tr_ilg(a - first);
+ } else if(1 < (last - b)) {
+ first = b, limit = tr_ilg(last - b);
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ } else {
+ if(1 < (last - b)) {
+ STACK_PUSH5(ISAd, first, a, tr_ilg(a - first), trlink);
+ first = b, limit = tr_ilg(last - b);
+ } else if(1 < (a - first)) {
+ last = a, limit = tr_ilg(a - first);
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ }
+ } else if(limit == -2) {
+ /* tandem repeat copy */
+ a = stack[--ssize].b, b = stack[ssize].c;
+ if(stack[ssize].d == 0) {
+ tr_copy(ISA, SA, first, a, b, last, ISAd - ISA);
+ } else {
+ if(0 <= trlink) { stack[trlink].d = -1; }
+ tr_partialcopy(ISA, SA, first, a, b, last, ISAd - ISA);
+ }
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ } else {
+ /* sorted partition */
+ if(0 <= *first) {
+ a = first;
+ do { ISA[*a] = a - SA; } while((++a < last) && (0 <= *a));
+ first = a;
+ }
+ if(first < last) {
+ a = first; do { *a = ~*a; } while(*++a < 0);
+ next = (ISA[*a] != ISAd[*a]) ? tr_ilg(a - first + 1) : -1;
+ if(++a < last) { for(b = first, v = a - SA - 1; b < a; ++b) { ISA[*b] = v; } }
+
+ /* push */
+ if(trbudget_check(budget, a - first)) {
+ if((a - first) <= (last - a)) {
+ STACK_PUSH5(ISAd, a, last, -3, trlink);
+ ISAd += incr, last = a, limit = next;
+ } else {
+ if(1 < (last - a)) {
+ STACK_PUSH5(ISAd + incr, first, a, next, trlink);
+ first = a, limit = -3;
+ } else {
+ ISAd += incr, last = a, limit = next;
+ }
+ }
+ } else {
+ if(0 <= trlink) { stack[trlink].d = -1; }
+ if(1 < (last - a)) {
+ first = a, limit = -3;
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ }
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ }
+ continue;
+ }
+
+ if((last - first) <= TR_INSERTIONSORT_THRESHOLD) {
+ tr_insertionsort(ISAd, first, last);
+ limit = -3;
+ continue;
+ }
+
+ if(limit-- == 0) {
+ tr_heapsort(ISAd, first, last - first);
+ for(a = last - 1; first < a; a = b) {
+ for(x = ISAd[*a], b = a - 1; (first <= b) && (ISAd[*b] == x); --b) { *b = ~*b; }
+ }
+ limit = -3;
+ continue;
+ }
+
+ /* choose pivot */
+ a = tr_pivot(ISAd, first, last);
+ SWAP(*first, *a);
+ v = ISAd[*first];
+
+ /* partition */
+ tr_partition(ISAd, first, first + 1, last, &a, &b, v);
+ if((last - first) != (b - a)) {
+ next = (ISA[*a] != v) ? tr_ilg(b - a) : -1;
+
+ /* update ranks */
+ for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; }
+ if(b < last) { for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } }
+
+ /* push */
+ if((1 < (b - a)) && (trbudget_check(budget, b - a))) {
+ if((a - first) <= (last - b)) {
+ if((last - b) <= (b - a)) {
+ if(1 < (a - first)) {
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ last = a;
+ } else if(1 < (last - b)) {
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ first = b;
+ } else {
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ } else if((a - first) <= (b - a)) {
+ if(1 < (a - first)) {
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ last = a;
+ } else {
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ } else {
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ } else {
+ if((a - first) <= (b - a)) {
+ if(1 < (last - b)) {
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ first = b;
+ } else if(1 < (a - first)) {
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ last = a;
+ } else {
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ } else if((last - b) <= (b - a)) {
+ if(1 < (last - b)) {
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ STACK_PUSH5(ISAd + incr, a, b, next, trlink);
+ first = b;
+ } else {
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ } else {
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ ISAd += incr, first = a, last = b, limit = next;
+ }
+ }
+ } else {
+ if((1 < (b - a)) && (0 <= trlink)) { stack[trlink].d = -1; }
+ if((a - first) <= (last - b)) {
+ if(1 < (a - first)) {
+ STACK_PUSH5(ISAd, b, last, limit, trlink);
+ last = a;
+ } else if(1 < (last - b)) {
+ first = b;
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ } else {
+ if(1 < (last - b)) {
+ STACK_PUSH5(ISAd, first, a, limit, trlink);
+ first = b;
+ } else if(1 < (a - first)) {
+ last = a;
+ } else {
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ }
+ }
+ } else {
+ if(trbudget_check(budget, last - first)) {
+ limit = tr_ilg(last - first), ISAd += incr;
+ } else {
+ if(0 <= trlink) { stack[trlink].d = -1; }
+ STACK_POP5(ISAd, first, last, limit, trlink);
+ }
+ }
+ }
+#undef STACK_SIZE
+}
+
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Tandem repeat sort */
+static
+void
+trsort(int *ISA, int *SA, int n, int depth) {
+ int *ISAd;
+ int *first, *last;
+ trbudget_t budget;
+ int t, skip, unsorted;
+
+ trbudget_init(&budget, tr_ilg(n) * 2 / 3, n);
+/* trbudget_init(&budget, tr_ilg(n) * 3 / 4, n); */
+ for(ISAd = ISA + depth; -n < *SA; ISAd += ISAd - ISA) {
+ first = SA;
+ skip = 0;
+ unsorted = 0;
+ do {
+ if((t = *first) < 0) { first -= t; skip += t; }
+ else {
+ if(skip != 0) { *(first + skip) = skip; skip = 0; }
+ last = SA + ISA[t] + 1;
+ if(1 < (last - first)) {
+ budget.count = 0;
+ tr_introsort(ISA, ISAd, SA, first, last, &budget);
+ if(budget.count != 0) { unsorted += budget.count; }
+ else { skip = first - last; }
+ } else if((last - first) == 1) {
+ skip = -1;
+ }
+ first = last;
+ }
+ } while(first < (SA + n));
+ if(skip != 0) { *(first + skip) = skip; }
+ if(unsorted == 0) { break; }
+ }
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/* Sorts suffixes of type B*. */
+static
+int
+sort_typeBstar(const unsigned char *T, int *SA,
+ int *bucket_A, int *bucket_B,
+ int n, int openMP) {
+ int *PAb, *ISAb, *buf;
+#ifdef LIBBSC_OPENMP
+ int *curbuf;
+ int l;
+#endif
+ int i, j, k, t, m, bufsize;
+ int c0, c1;
+#ifdef LIBBSC_OPENMP
+ int d0, d1;
+#endif
+ (void)openMP;
+
+ /* Initialize bucket arrays. */
+ for(i = 0; i < BUCKET_A_SIZE; ++i) { bucket_A[i] = 0; }
+ for(i = 0; i < BUCKET_B_SIZE; ++i) { bucket_B[i] = 0; }
+
+ /* Count the number of occurrences of the first one or two characters of each
+ type A, B and B* suffix. Moreover, store the beginning position of all
+ type B* suffixes into the array SA. */
+ for(i = n - 1, m = n, c0 = T[n - 1]; 0 <= i;) {
+ /* type A suffix. */
+ do { ++BUCKET_A(c1 = c0); } while((0 <= --i) && ((c0 = T[i]) >= c1));
+ if(0 <= i) {
+ /* type B* suffix. */
+ ++BUCKET_BSTAR(c0, c1);
+ SA[--m] = i;
+ /* type B suffix. */
+ for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) {
+ ++BUCKET_B(c0, c1);
+ }
+ }
+ }
+ m = n - m;
+/*
+note:
+ A type B* suffix is lexicographically smaller than a type B suffix that
+ begins with the same first two characters.
+*/
+
+ /* Calculate the index of start/end point of each bucket. */
+ for(c0 = 0, i = 0, j = 0; c0 < ALPHABET_SIZE; ++c0) {
+ t = i + BUCKET_A(c0);
+ BUCKET_A(c0) = i + j; /* start point */
+ i = t + BUCKET_B(c0, c0);
+ for(c1 = c0 + 1; c1 < ALPHABET_SIZE; ++c1) {
+ j += BUCKET_BSTAR(c0, c1);
+ BUCKET_BSTAR(c0, c1) = j; /* end point */
+ i += BUCKET_B(c0, c1);
+ }
+ }
+
+ if(0 < m) {
+ /* Sort the type B* suffixes by their first two characters. */
+ PAb = SA + n - m; ISAb = SA + m;
+ for(i = m - 2; 0 <= i; --i) {
+ t = PAb[i], c0 = T[t], c1 = T[t + 1];
+ SA[--BUCKET_BSTAR(c0, c1)] = i;
+ }
+ t = PAb[m - 1], c0 = T[t], c1 = T[t + 1];
+ SA[--BUCKET_BSTAR(c0, c1)] = m - 1;
+
+ /* Sort the type B* substrings using sssort. */
+#ifdef LIBBSC_OPENMP
+ if (openMP)
+ {
+ buf = SA + m;
+ c0 = ALPHABET_SIZE - 2, c1 = ALPHABET_SIZE - 1, j = m;
+#pragma omp parallel default(shared) private(bufsize, curbuf, k, l, d0, d1)
+ {
+ bufsize = (n - (2 * m)) / omp_get_num_threads();
+ curbuf = buf + omp_get_thread_num() * bufsize;
+ k = 0;
+ for(;;) {
+ #pragma omp critical(sssort_lock)
+ {
+ if(0 < (l = j)) {
+ d0 = c0, d1 = c1;
+ do {
+ k = BUCKET_BSTAR(d0, d1);
+ if(--d1 <= d0) {
+ d1 = ALPHABET_SIZE - 1;
+ if(--d0 < 0) { break; }
+ }
+ } while(((l - k) <= 1) && (0 < (l = k)));
+ c0 = d0, c1 = d1, j = k;
+ }
+ }
+ if(l == 0) { break; }
+ sssort(T, PAb, SA + k, SA + l,
+ curbuf, bufsize, 2, n, *(SA + k) == (m - 1));
+ }
+ }
+ }
+ else
+ {
+ buf = SA + m, bufsize = n - (2 * m);
+ for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) {
+ for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) {
+ i = BUCKET_BSTAR(c0, c1);
+ if(1 < (j - i)) {
+ sssort(T, PAb, SA + i, SA + j,
+ buf, bufsize, 2, n, *(SA + i) == (m - 1));
+ }
+ }
+ }
+ }
+#else
+ buf = SA + m, bufsize = n - (2 * m);
+ for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) {
+ for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) {
+ i = BUCKET_BSTAR(c0, c1);
+ if(1 < (j - i)) {
+ sssort(T, PAb, SA + i, SA + j,
+ buf, bufsize, 2, n, *(SA + i) == (m - 1));
+ }
+ }
+ }
+#endif
+
+ /* Compute ranks of type B* substrings. */
+ for(i = m - 1; 0 <= i; --i) {
+ if(0 <= SA[i]) {
+ j = i;
+ do { ISAb[SA[i]] = i; } while((0 <= --i) && (0 <= SA[i]));
+ SA[i + 1] = i - j;
+ if(i <= 0) { break; }
+ }
+ j = i;
+ do { ISAb[SA[i] = ~SA[i]] = j; } while(SA[--i] < 0);
+ ISAb[SA[i]] = j;
+ }
+
+ /* Construct the inverse suffix array of type B* suffixes using trsort. */
+ trsort(ISAb, SA, m, 1);
+
+ /* Set the sorted order of tyoe B* suffixes. */
+ for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) {
+ for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { }
+ if(0 <= i) {
+ t = i;
+ for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { }
+ SA[ISAb[--j]] = ((t == 0) || (1 < (t - i))) ? t : ~t;
+ }
+ }
+
+ /* Calculate the index of start/end point of each bucket. */
+ BUCKET_B(ALPHABET_SIZE - 1, ALPHABET_SIZE - 1) = n; /* end point */
+ for(c0 = ALPHABET_SIZE - 2, k = m - 1; 0 <= c0; --c0) {
+ i = BUCKET_A(c0 + 1) - 1;
+ for(c1 = ALPHABET_SIZE - 1; c0 < c1; --c1) {
+ t = i - BUCKET_B(c0, c1);
+ BUCKET_B(c0, c1) = i; /* end point */
+
+ /* Move all type B* suffixes to the correct position. */
+ for(i = t, j = BUCKET_BSTAR(c0, c1);
+ j <= k;
+ --i, --k) { SA[i] = SA[k]; }
+ }
+ BUCKET_BSTAR(c0, c0 + 1) = i - BUCKET_B(c0, c0) + 1; /* start point */
+ BUCKET_B(c0, c0) = i; /* end point */
+ }
+ }
+
+ return m;
+}
+
+/* Constructs the suffix array by using the sorted order of type B* suffixes. */
+static
+void
+construct_SA(const unsigned char *T, int *SA,
+ int *bucket_A, int *bucket_B,
+ int n, int m) {
+ int *i, *j, *k;
+ int s;
+ int c0, c1, c2;
+
+ if(0 < m) {
+ /* Construct the sorted order of type B suffixes by using
+ the sorted order of type B* suffixes. */
+ for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
+ /* Scan the suffix array from right to left. */
+ for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
+ j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
+ i <= j;
+ --j) {
+ if(0 < (s = *j)) {
+ assert(T[s] == c1);
+ assert(((s + 1) < n) && (T[s] <= T[s + 1]));
+ assert(T[s - 1] <= T[s]);
+ *j = ~s;
+ c0 = T[--s];
+ if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
+ if(c0 != c2) {
+ if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
+ k = SA + BUCKET_B(c2 = c0, c1);
+ }
+ assert(k < j);
+ *k-- = s;
+ } else {
+ assert(((s == 0) && (T[s] == c1)) || (s < 0));
+ *j = ~s;
+ }
+ }
+ }
+ }
+
+ /* Construct the suffix array by using
+ the sorted order of type B suffixes. */
+ k = SA + BUCKET_A(c2 = T[n - 1]);
+ *k++ = (T[n - 2] < c2) ? ~(n - 1) : (n - 1);
+ /* Scan the suffix array from left to right. */
+ for(i = SA, j = SA + n; i < j; ++i) {
+ if(0 < (s = *i)) {
+ assert(T[s - 1] >= T[s]);
+ c0 = T[--s];
+ if((s == 0) || (T[s - 1] < c0)) { s = ~s; }
+ if(c0 != c2) {
+ BUCKET_A(c2) = k - SA;
+ k = SA + BUCKET_A(c2 = c0);
+ }
+ assert(i < k);
+ *k++ = s;
+ } else {
+ assert(s < 0);
+ *i = ~s;
+ }
+ }
+}
+
+/* Constructs the burrows-wheeler transformed string directly
+ by using the sorted order of type B* suffixes. */
+static
+int
+construct_BWT(const unsigned char *T, int *SA,
+ int *bucket_A, int *bucket_B,
+ int n, int m) {
+ int *i, *j, *k, *orig;
+ int s;
+ int c0, c1, c2;
+
+ if(0 < m) {
+ /* Construct the sorted order of type B suffixes by using
+ the sorted order of type B* suffixes. */
+ for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
+ /* Scan the suffix array from right to left. */
+ for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
+ j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
+ i <= j;
+ --j) {
+ if(0 < (s = *j)) {
+ assert(T[s] == c1);
+ assert(((s + 1) < n) && (T[s] <= T[s + 1]));
+ assert(T[s - 1] <= T[s]);
+ c0 = T[--s];
+ *j = ~((int)c0);
+ if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
+ if(c0 != c2) {
+ if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
+ k = SA + BUCKET_B(c2 = c0, c1);
+ }
+ assert(k < j);
+ *k-- = s;
+ } else if(s != 0) {
+ *j = ~s;
+#ifndef NDEBUG
+ } else {
+ assert(T[s] == c1);
+#endif
+ }
+ }
+ }
+ }
+
+ /* Construct the BWTed string by using
+ the sorted order of type B suffixes. */
+ k = SA + BUCKET_A(c2 = T[n - 1]);
+ *k++ = (T[n - 2] < c2) ? ~((int)T[n - 2]) : (n - 1);
+ /* Scan the suffix array from left to right. */
+ for(i = SA, j = SA + n, orig = SA; i < j; ++i) {
+ if(0 < (s = *i)) {
+ assert(T[s - 1] >= T[s]);
+ c0 = T[--s];
+ *i = c0;
+ if((0 < s) && (T[s - 1] < c0)) { s = ~((int)T[s - 1]); }
+ if(c0 != c2) {
+ BUCKET_A(c2) = k - SA;
+ k = SA + BUCKET_A(c2 = c0);
+ }
+ assert(i < k);
+ *k++ = s;
+ } else if(s != 0) {
+ *i = ~s;
+ } else {
+ orig = i;
+ }
+ }
+
+ return orig - SA;
+}
+
+/* Constructs the burrows-wheeler transformed string directly
+ by using the sorted order of type B* suffixes. */
+static
+int
+construct_BWT_indexes(const unsigned char *T, int *SA,
+ int *bucket_A, int *bucket_B,
+ int n, int m,
+ unsigned char * num_indexes, int * indexes) {
+ int *i, *j, *k, *orig;
+ int s;
+ int c0, c1, c2;
+
+ int mod = n / 8;
+ {
+ mod |= mod >> 1; mod |= mod >> 2;
+ mod |= mod >> 4; mod |= mod >> 8;
+ mod |= mod >> 16; mod >>= 1;
+
+ *num_indexes = (unsigned char)((n - 1) / (mod + 1));
+ }
+
+ if(0 < m) {
+ /* Construct the sorted order of type B suffixes by using
+ the sorted order of type B* suffixes. */
+ for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
+ /* Scan the suffix array from right to left. */
+ for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
+ j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
+ i <= j;
+ --j) {
+ if(0 < (s = *j)) {
+ assert(T[s] == c1);
+ assert(((s + 1) < n) && (T[s] <= T[s + 1]));
+ assert(T[s - 1] <= T[s]);
+
+ if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = j - SA;
+
+ c0 = T[--s];
+ *j = ~((int)c0);
+ if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
+ if(c0 != c2) {
+ if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
+ k = SA + BUCKET_B(c2 = c0, c1);
+ }
+ assert(k < j);
+ *k-- = s;
+ } else if(s != 0) {
+ *j = ~s;
+#ifndef NDEBUG
+ } else {
+ assert(T[s] == c1);
+#endif
+ }
+ }
+ }
+ }
+
+ /* Construct the BWTed string by using
+ the sorted order of type B suffixes. */
+ k = SA + BUCKET_A(c2 = T[n - 1]);
+ if (T[n - 2] < c2) {
+ if (((n - 1) & mod) == 0) indexes[(n - 1) / (mod + 1) - 1] = k - SA;
+ *k++ = ~((int)T[n - 2]);
+ }
+ else {
+ *k++ = n - 1;
+ }
+
+ /* Scan the suffix array from left to right. */
+ for(i = SA, j = SA + n, orig = SA; i < j; ++i) {
+ if(0 < (s = *i)) {
+ assert(T[s - 1] >= T[s]);
+
+ if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = i - SA;
+
+ c0 = T[--s];
+ *i = c0;
+ if(c0 != c2) {
+ BUCKET_A(c2) = k - SA;
+ k = SA + BUCKET_A(c2 = c0);
+ }
+ assert(i < k);
+ if((0 < s) && (T[s - 1] < c0)) {
+ if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = k - SA;
+ *k++ = ~((int)T[s - 1]);
+ } else
+ *k++ = s;
+ } else if(s != 0) {
+ *i = ~s;
+ } else {
+ orig = i;
+ }
+ }
+
+ return orig - SA;
+}
+
+
+/*---------------------------------------------------------------------------*/
+
+/*- Function -*/
+
+int
+divsufsort(const unsigned char *T, int *SA, int n, int openMP) {
+ int *bucket_A, *bucket_B;
+ int m;
+ int err = 0;
+
+ /* Check arguments. */
+ if((T == NULL) || (SA == NULL) || (n < 0)) { return -1; }
+ else if(n == 0) { return 0; }
+ else if(n == 1) { SA[0] = 0; return 0; }
+ else if(n == 2) { m = (T[0] < T[1]); SA[m ^ 1] = 0, SA[m] = 1; return 0; }
+
+ bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int));
+ bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int));
+
+ /* Suffixsort. */
+ if((bucket_A != NULL) && (bucket_B != NULL)) {
+ m = sort_typeBstar(T, SA, bucket_A, bucket_B, n, openMP);
+ construct_SA(T, SA, bucket_A, bucket_B, n, m);
+ } else {
+ err = -2;
+ }
+
+ free(bucket_B);
+ free(bucket_A);
+
+ return err;
+}
+
+int
+divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP) {
+ int *B;
+ int *bucket_A, *bucket_B;
+ int m, pidx, i;
+
+ /* Check arguments. */
+ if((T == NULL) || (U == NULL) || (n < 0)) { return -1; }
+ else if(n <= 1) { if(n == 1) { U[0] = T[0]; } return n; }
+
+ if((B = A) == NULL) { B = (int *)malloc((size_t)(n + 1) * sizeof(int)); }
+ bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int));
+ bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int));
+
+ /* Burrows-Wheeler Transform. */
+ if((B != NULL) && (bucket_A != NULL) && (bucket_B != NULL)) {
+ m = sort_typeBstar(T, B, bucket_A, bucket_B, n, openMP);
+
+ if (num_indexes == NULL || indexes == NULL) {
+ pidx = construct_BWT(T, B, bucket_A, bucket_B, n, m);
+ } else {
+ pidx = construct_BWT_indexes(T, B, bucket_A, bucket_B, n, m, num_indexes, indexes);
+ }
+
+ /* Copy to output string. */
+ U[0] = T[n - 1];
+ for(i = 0; i < pidx; ++i) { U[i + 1] = (unsigned char)B[i]; }
+ for(i += 1; i < n; ++i) { U[i] = (unsigned char)B[i]; }
+ pidx += 1;
+ } else {
+ pidx = -2;
+ }
+
+ free(bucket_B);
+ free(bucket_A);
+ if(A == NULL) { free(B); }
+
+ return pidx;
+}
diff --git a/contrib/zstd/divsufsort.h b/contrib/zstd/divsufsort.h
new file mode 100644
index 0000000..5440994
--- /dev/null
+++ b/contrib/zstd/divsufsort.h
@@ -0,0 +1,67 @@
+/*
+ * divsufsort.h for libdivsufsort-lite
+ * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _DIVSUFSORT_H
+#define _DIVSUFSORT_H 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+
+/*- Prototypes -*/
+
+/**
+ * Constructs the suffix array of a given string.
+ * @param T [0..n-1] The input string.
+ * @param SA [0..n-1] The output array of suffixes.
+ * @param n The length of the given string.
+ * @param openMP enables OpenMP optimization.
+ * @return 0 if no error occurred, -1 or -2 otherwise.
+ */
+int
+divsufsort(const unsigned char *T, int *SA, int n, int openMP);
+
+/**
+ * Constructs the burrows-wheeler transformed string of a given string.
+ * @param T [0..n-1] The input string.
+ * @param U [0..n-1] The output string. (can be T)
+ * @param A [0..n-1] The temporary array. (can be NULL)
+ * @param n The length of the given string.
+ * @param num_indexes The length of secondary indexes array. (can be NULL)
+ * @param indexes The secondary indexes array. (can be NULL)
+ * @param openMP enables OpenMP optimization.
+ * @return The primary index if no error occurred, -1 or -2 otherwise.
+ */
+int
+divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP);
+
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif /* __cplusplus */
+
+#endif /* _DIVSUFSORT_H */
diff --git a/contrib/zstd/entropy_common.c b/contrib/zstd/entropy_common.c
new file mode 100644
index 0000000..e2173af
--- /dev/null
+++ b/contrib/zstd/entropy_common.c
@@ -0,0 +1,340 @@
+/* ******************************************************************
+ * Common functions of New Generation Entropy library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* *************************************
+* Dependencies
+***************************************/
+#include "mem.h"
+#include "error_private.h" /* ERR_*, ERROR */
+#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
+#include "fse.h"
+#include "huf.h"
+#include "bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */
+
+
+/*=== Version ===*/
+unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
+
+
+/*=== Error Management ===*/
+unsigned FSE_isError(size_t code) { return ERR_isError(code); }
+const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
+
+unsigned HUF_isError(size_t code) { return ERR_isError(code); }
+const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
+
+
+/*-**************************************************************
+* FSE NCount encoding-decoding
+****************************************************************/
+FORCE_INLINE_TEMPLATE
+size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+ const void* headerBuffer, size_t hbSize)
+{
+ const BYTE* const istart = (const BYTE*) headerBuffer;
+ const BYTE* const iend = istart + hbSize;
+ const BYTE* ip = istart;
+ int nbBits;
+ int remaining;
+ int threshold;
+ U32 bitStream;
+ int bitCount;
+ unsigned charnum = 0;
+ unsigned const maxSV1 = *maxSVPtr + 1;
+ int previous0 = 0;
+
+ if (hbSize < 8) {
+ /* This function only works when hbSize >= 8 */
+ char buffer[8] = {0};
+ ZSTD_memcpy(buffer, headerBuffer, hbSize);
+ { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
+ buffer, sizeof(buffer));
+ if (FSE_isError(countSize)) return countSize;
+ if (countSize > hbSize) return ERROR(corruption_detected);
+ return countSize;
+ } }
+ assert(hbSize >= 8);
+
+ /* init */
+ ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
+ bitStream = MEM_readLE32(ip);
+ nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
+ if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
+ bitStream >>= 4;
+ bitCount = 4;
+ *tableLogPtr = nbBits;
+ remaining = (1<<nbBits)+1;
+ threshold = 1<<nbBits;
+ nbBits++;
+
+ for (;;) {
+ if (previous0) {
+ /* Count the number of repeats. Each time the
+ * 2-bit repeat code is 0b11 there is another
+ * repeat.
+ * Avoid UB by setting the high bit to 1.
+ */
+ int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
+ while (repeats >= 12) {
+ charnum += 3 * 12;
+ if (LIKELY(ip <= iend-7)) {
+ ip += 3;
+ } else {
+ bitCount -= (int)(8 * (iend - 7 - ip));
+ bitCount &= 31;
+ ip = iend - 4;
+ }
+ bitStream = MEM_readLE32(ip) >> bitCount;
+ repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
+ }
+ charnum += 3 * repeats;
+ bitStream >>= 2 * repeats;
+ bitCount += 2 * repeats;
+
+ /* Add the final repeat which isn't 0b11. */
+ assert((bitStream & 3) < 3);
+ charnum += bitStream & 3;
+ bitCount += 2;
+
+ /* This is an error, but break and return an error
+ * at the end, because returning out of a loop makes
+ * it harder for the compiler to optimize.
+ */
+ if (charnum >= maxSV1) break;
+
+ /* We don't need to set the normalized count to 0
+ * because we already memset the whole buffer to 0.
+ */
+
+ if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
+ assert((bitCount >> 3) <= 3); /* For first condition to work */
+ ip += bitCount>>3;
+ bitCount &= 7;
+ } else {
+ bitCount -= (int)(8 * (iend - 4 - ip));
+ bitCount &= 31;
+ ip = iend - 4;
+ }
+ bitStream = MEM_readLE32(ip) >> bitCount;
+ }
+ {
+ int const max = (2*threshold-1) - remaining;
+ int count;
+
+ if ((bitStream & (threshold-1)) < (U32)max) {
+ count = bitStream & (threshold-1);
+ bitCount += nbBits-1;
+ } else {
+ count = bitStream & (2*threshold-1);
+ if (count >= threshold) count -= max;
+ bitCount += nbBits;
+ }
+
+ count--; /* extra accuracy */
+ /* When it matters (small blocks), this is a
+ * predictable branch, because we don't use -1.
+ */
+ if (count >= 0) {
+ remaining -= count;
+ } else {
+ assert(count == -1);
+ remaining += count;
+ }
+ normalizedCounter[charnum++] = (short)count;
+ previous0 = !count;
+
+ assert(threshold > 1);
+ if (remaining < threshold) {
+ /* This branch can be folded into the
+ * threshold update condition because we
+ * know that threshold > 1.
+ */
+ if (remaining <= 1) break;
+ nbBits = ZSTD_highbit32(remaining) + 1;
+ threshold = 1 << (nbBits - 1);
+ }
+ if (charnum >= maxSV1) break;
+
+ if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
+ ip += bitCount>>3;
+ bitCount &= 7;
+ } else {
+ bitCount -= (int)(8 * (iend - 4 - ip));
+ bitCount &= 31;
+ ip = iend - 4;
+ }
+ bitStream = MEM_readLE32(ip) >> bitCount;
+ } }
+ if (remaining != 1) return ERROR(corruption_detected);
+ /* Only possible when there are too many zeros. */
+ if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall);
+ if (bitCount > 32) return ERROR(corruption_detected);
+ *maxSVPtr = charnum-1;
+
+ ip += (bitCount+7)>>3;
+ return ip-istart;
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static size_t FSE_readNCount_body_default(
+ short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+ const void* headerBuffer, size_t hbSize)
+{
+ return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+}
+
+#if DYNAMIC_BMI2
+BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2(
+ short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+ const void* headerBuffer, size_t hbSize)
+{
+ return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+}
+#endif
+
+size_t FSE_readNCount_bmi2(
+ short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+ const void* headerBuffer, size_t hbSize, int bmi2)
+{
+#if DYNAMIC_BMI2
+ if (bmi2) {
+ return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+ }
+#endif
+ (void)bmi2;
+ return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
+}
+
+size_t FSE_readNCount(
+ short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+ const void* headerBuffer, size_t hbSize)
+{
+ return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0);
+}
+
+
+/*! HUF_readStats() :
+ Read compact Huffman tree, saved by HUF_writeCTable().
+ `huffWeight` is destination buffer.
+ `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
+ @return : size read from `src` , or an error Code .
+ Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
+*/
+size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+ U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize)
+{
+ U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
+ return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0);
+}
+
+FORCE_INLINE_TEMPLATE size_t
+HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+ U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t wkspSize,
+ int bmi2)
+{
+ U32 weightTotal;
+ const BYTE* ip = (const BYTE*) src;
+ size_t iSize;
+ size_t oSize;
+
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
+ /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
+
+ if (iSize >= 128) { /* special header */
+ oSize = iSize - 127;
+ iSize = ((oSize+1)/2);
+ if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
+ if (oSize >= hwSize) return ERROR(corruption_detected);
+ ip += 1;
+ { U32 n;
+ for (n=0; n<oSize; n+=2) {
+ huffWeight[n] = ip[n/2] >> 4;
+ huffWeight[n+1] = ip[n/2] & 15;
+ } } }
+ else { /* header compressed with FSE (normal case) */
+ if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
+ /* max (hwSize-1) values decoded, as last one is implied */
+ oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2);
+ if (FSE_isError(oSize)) return oSize;
+ }
+
+ /* collect weight stats */
+ ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
+ weightTotal = 0;
+ { U32 n; for (n=0; n<oSize; n++) {
+ if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
+ rankStats[huffWeight[n]]++;
+ weightTotal += (1 << huffWeight[n]) >> 1;
+ } }
+ if (weightTotal == 0) return ERROR(corruption_detected);
+
+ /* get last non-null symbol weight (implied, total must be 2^n) */
+ { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1;
+ if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
+ *tableLogPtr = tableLog;
+ /* determine last weight */
+ { U32 const total = 1 << tableLog;
+ U32 const rest = total - weightTotal;
+ U32 const verif = 1 << ZSTD_highbit32(rest);
+ U32 const lastWeight = ZSTD_highbit32(rest) + 1;
+ if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
+ huffWeight[oSize] = (BYTE)lastWeight;
+ rankStats[lastWeight]++;
+ } }
+
+ /* check tree construction validity */
+ if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
+
+ /* results */
+ *nbSymbolsPtr = (U32)(oSize+1);
+ return iSize+1;
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+ U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t wkspSize)
+{
+ return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0);
+}
+
+#if DYNAMIC_BMI2
+static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+ U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t wkspSize)
+{
+ return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1);
+}
+#endif
+
+size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+ U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t wkspSize,
+ int flags)
+{
+#if DYNAMIC_BMI2
+ if (flags & HUF_flags_bmi2) {
+ return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
+ }
+#endif
+ (void)flags;
+ return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
+}
diff --git a/contrib/zstd/error_private.c b/contrib/zstd/error_private.c
new file mode 100644
index 0000000..075fc5e
--- /dev/null
+++ b/contrib/zstd/error_private.c
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* The purpose of this file is to have a single list of error strings embedded in binary */
+
+#include "error_private.h"
+
+const char* ERR_getErrorString(ERR_enum code)
+{
+#ifdef ZSTD_STRIP_ERROR_STRINGS
+ (void)code;
+ return "Error strings stripped";
+#else
+ static const char* const notErrorCode = "Unspecified error code";
+ switch( code )
+ {
+ case PREFIX(no_error): return "No error detected";
+ case PREFIX(GENERIC): return "Error (generic)";
+ case PREFIX(prefix_unknown): return "Unknown frame descriptor";
+ case PREFIX(version_unsupported): return "Version not supported";
+ case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
+ case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
+ case PREFIX(corruption_detected): return "Data corruption detected";
+ case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+ case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification";
+ case PREFIX(parameter_unsupported): return "Unsupported parameter";
+ case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters";
+ case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
+ case PREFIX(init_missing): return "Context should be init first";
+ case PREFIX(memory_allocation): return "Allocation error : not enough memory";
+ case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";
+ case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
+ case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
+ case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
+ case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
+ case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected";
+ case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
+ case PREFIX(dictionary_wrong): return "Dictionary mismatch";
+ case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
+ case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
+ case PREFIX(srcSize_wrong): return "Src size is incorrect";
+ case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
+ case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full";
+ case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty";
+ /* following error codes are not stable and may be removed or changed in a future version */
+ case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
+ case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
+ case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
+ case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
+ case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code";
+ case PREFIX(externalSequences_invalid): return "External sequences are not valid";
+ case PREFIX(maxCode):
+ default: return notErrorCode;
+ }
+#endif
+}
diff --git a/contrib/zstd/error_private.h b/contrib/zstd/error_private.h
new file mode 100644
index 0000000..d99c961
--- /dev/null
+++ b/contrib/zstd/error_private.h
@@ -0,0 +1,159 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* Note : this module is expected to remain private, do not expose it */
+
+#ifndef ERROR_H_MODULE
+#define ERROR_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* ****************************************
+* Dependencies
+******************************************/
+#include "zstd_errors.h" /* enum list */
+#include "compiler.h"
+#include "debug.h"
+#include "zstd_deps.h" /* size_t */
+
+
+/* ****************************************
+* Compiler-specific
+******************************************/
+#if defined(__GNUC__)
+# define ERR_STATIC static __attribute__((unused))
+#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+# define ERR_STATIC static inline
+#elif defined(_MSC_VER)
+# define ERR_STATIC static __inline
+#else
+# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
+#endif
+
+
+/*-****************************************
+* Customization (error_public.h)
+******************************************/
+typedef ZSTD_ErrorCode ERR_enum;
+#define PREFIX(name) ZSTD_error_##name
+
+
+/*-****************************************
+* Error codes handling
+******************************************/
+#undef ERROR /* already defined on Visual Studio */
+#define ERROR(name) ZSTD_ERROR(name)
+#define ZSTD_ERROR(name) ((size_t)-PREFIX(name))
+
+ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
+
+ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
+
+/* check and forward error code */
+#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
+#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
+
+
+/*-****************************************
+* Error Strings
+******************************************/
+
+const char* ERR_getErrorString(ERR_enum code); /* error_private.c */
+
+ERR_STATIC const char* ERR_getErrorName(size_t code)
+{
+ return ERR_getErrorString(ERR_getErrorCode(code));
+}
+
+/**
+ * Ignore: this is an internal helper.
+ *
+ * This is a helper function to help force C99-correctness during compilation.
+ * Under strict compilation modes, variadic macro arguments can't be empty.
+ * However, variadic function arguments can be. Using a function therefore lets
+ * us statically check that at least one (string) argument was passed,
+ * independent of the compilation flags.
+ */
+static INLINE_KEYWORD UNUSED_ATTR
+void _force_has_format_string(const char *format, ...) {
+ (void)format;
+}
+
+/**
+ * Ignore: this is an internal helper.
+ *
+ * We want to force this function invocation to be syntactically correct, but
+ * we don't want to force runtime evaluation of its arguments.
+ */
+#define _FORCE_HAS_FORMAT_STRING(...) \
+ if (0) { \
+ _force_has_format_string(__VA_ARGS__); \
+ }
+
+#define ERR_QUOTE(str) #str
+
+/**
+ * Return the specified error if the condition evaluates to true.
+ *
+ * In debug modes, prints additional information.
+ * In order to do that (particularly, printing the conditional that failed),
+ * this can't just wrap RETURN_ERROR().
+ */
+#define RETURN_ERROR_IF(cond, err, ...) \
+ if (cond) { \
+ RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \
+ __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \
+ _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
+ RAWLOG(3, ": " __VA_ARGS__); \
+ RAWLOG(3, "\n"); \
+ return ERROR(err); \
+ }
+
+/**
+ * Unconditionally return the specified error.
+ *
+ * In debug modes, prints additional information.
+ */
+#define RETURN_ERROR(err, ...) \
+ do { \
+ RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \
+ __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \
+ _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
+ RAWLOG(3, ": " __VA_ARGS__); \
+ RAWLOG(3, "\n"); \
+ return ERROR(err); \
+ } while(0);
+
+/**
+ * If the provided expression evaluates to an error code, returns that error code.
+ *
+ * In debug modes, prints additional information.
+ */
+#define FORWARD_IF_ERROR(err, ...) \
+ do { \
+ size_t const err_code = (err); \
+ if (ERR_isError(err_code)) { \
+ RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \
+ __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \
+ _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
+ RAWLOG(3, ": " __VA_ARGS__); \
+ RAWLOG(3, "\n"); \
+ return err_code; \
+ } \
+ } while(0);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ERROR_H_MODULE */
diff --git a/contrib/zstd/error_public.h b/contrib/zstd/error_public.h
new file mode 100644
index 0000000..d46abd2
--- /dev/null
+++ b/contrib/zstd/error_public.h
@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#ifndef ERROR_PUBLIC_H_MODULE
+#define ERROR_PUBLIC_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*===== dependency =====*/
+#include <stddef.h> /* size_t */
+
+
+/*-****************************************
+* error codes list
+******************************************/
+typedef enum {
+ ZSTD_error_no_error,
+ ZSTD_error_GENERIC,
+ ZSTD_error_prefix_unknown,
+ ZSTD_error_version_unsupported,
+ ZSTD_error_parameter_unknown,
+ ZSTD_error_frameParameter_unsupported,
+ ZSTD_error_frameParameter_unsupportedBy32bits,
+ ZSTD_error_compressionParameter_unsupported,
+ ZSTD_error_init_missing,
+ ZSTD_error_memory_allocation,
+ ZSTD_error_stage_wrong,
+ ZSTD_error_dstSize_tooSmall,
+ ZSTD_error_srcSize_wrong,
+ ZSTD_error_corruption_detected,
+ ZSTD_error_checksum_wrong,
+ ZSTD_error_tableLog_tooLarge,
+ ZSTD_error_maxSymbolValue_tooLarge,
+ ZSTD_error_maxSymbolValue_tooSmall,
+ ZSTD_error_dictionary_corrupted,
+ ZSTD_error_dictionary_wrong,
+ ZSTD_error_maxCode
+} ZSTD_ErrorCode;
+
+/*! ZSTD_getErrorCode() :
+ convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
+ which can be used to compare directly with enum list published into "error_public.h" */
+ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ERROR_PUBLIC_H_MODULE */
diff --git a/contrib/zstd/fse.h b/contrib/zstd/fse.h
new file mode 100644
index 0000000..02a1f0b
--- /dev/null
+++ b/contrib/zstd/fse.h
@@ -0,0 +1,639 @@
+/* ******************************************************************
+ * FSE : Finite State Entropy codec
+ * Public Prototypes declaration
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef FSE_H
+#define FSE_H
+
+
+/*-*****************************************
+* Dependencies
+******************************************/
+#include "zstd_deps.h" /* size_t, ptrdiff_t */
+
+
+/*-*****************************************
+* FSE_PUBLIC_API : control library symbols visibility
+******************************************/
+#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
+# define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
+#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
+# define FSE_PUBLIC_API __declspec(dllexport)
+#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
+# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define FSE_PUBLIC_API
+#endif
+
+/*------ Version ------*/
+#define FSE_VERSION_MAJOR 0
+#define FSE_VERSION_MINOR 9
+#define FSE_VERSION_RELEASE 0
+
+#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
+#define FSE_QUOTE(str) #str
+#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
+#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
+
+#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
+FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
+
+
+/*-*****************************************
+* Tool functions
+******************************************/
+FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
+
+/* Error Management */
+FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
+FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
+
+
+/*-*****************************************
+* FSE detailed API
+******************************************/
+/*!
+FSE_compress() does the following:
+1. count symbol occurrence from source[] into table count[] (see hist.h)
+2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
+3. save normalized counters to memory buffer using writeNCount()
+4. build encoding table 'CTable' from normalized counters
+5. encode the data stream using encoding table 'CTable'
+
+FSE_decompress() does the following:
+1. read normalized counters with readNCount()
+2. build decoding table 'DTable' from normalized counters
+3. decode the data stream using decoding table 'DTable'
+
+The following API allows targeting specific sub-functions for advanced tasks.
+For example, it's possible to compress several blocks using the same 'CTable',
+or to save and provide normalized distribution using external method.
+*/
+
+/* *** COMPRESSION *** */
+
+/*! FSE_optimalTableLog():
+ dynamically downsize 'tableLog' when conditions are met.
+ It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
+ @return : recommended tableLog (necessarily <= 'maxTableLog') */
+FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
+
+/*! FSE_normalizeCount():
+ normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
+ 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
+ useLowProbCount is a boolean parameter which trades off compressed size for
+ faster header decoding. When it is set to 1, the compressed data will be slightly
+ smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be
+ faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0
+ is a good default, since header deserialization makes a big speed difference.
+ Otherwise, useLowProbCount=1 is a good default, since the speed difference is small.
+ @return : tableLog,
+ or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
+ const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount);
+
+/*! FSE_NCountWriteBound():
+ Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
+ Typically useful for allocation purpose. */
+FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
+
+/*! FSE_writeNCount():
+ Compactly save 'normalizedCounter' into 'buffer'.
+ @return : size of the compressed table,
+ or an errorCode, which can be tested using FSE_isError(). */
+FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
+ const short* normalizedCounter,
+ unsigned maxSymbolValue, unsigned tableLog);
+
+/*! Constructor and Destructor of FSE_CTable.
+ Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
+typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
+
+/*! FSE_buildCTable():
+ Builds `ct`, which must be already allocated, using FSE_createCTable().
+ @return : 0, or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
+
+/*! FSE_compress_usingCTable():
+ Compress `src` using `ct` into `dst` which must be already allocated.
+ @return : size of compressed data (<= `dstCapacity`),
+ or 0 if compressed data could not fit into `dst`,
+ or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
+
+/*!
+Tutorial :
+----------
+The first step is to count all symbols. FSE_count() does this job very fast.
+Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
+'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
+maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
+FSE_count() will return the number of occurrence of the most frequent symbol.
+This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
+
+The next step is to normalize the frequencies.
+FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
+It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
+You can use 'tableLog'==0 to mean "use default tableLog value".
+If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
+which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
+
+The result of FSE_normalizeCount() will be saved into a table,
+called 'normalizedCounter', which is a table of signed short.
+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
+The return value is tableLog if everything proceeded as expected.
+It is 0 if there is a single symbol within distribution.
+If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
+
+'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
+'buffer' must be already allocated.
+For guaranteed success, buffer size must be at least FSE_headerBound().
+The result of the function is the number of bytes written into 'buffer'.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
+
+'normalizedCounter' can then be used to create the compression table 'CTable'.
+The space required by 'CTable' must be already allocated, using FSE_createCTable().
+You can then use FSE_buildCTable() to fill 'CTable'.
+If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
+
+'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
+Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
+The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
+If it returns '0', compressed data could not fit into 'dst'.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
+*/
+
+
+/* *** DECOMPRESSION *** */
+
+/*! FSE_readNCount():
+ Read compactly saved 'normalizedCounter' from 'rBuffer'.
+ @return : size read from 'rBuffer',
+ or an errorCode, which can be tested using FSE_isError().
+ maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
+FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
+ unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
+ const void* rBuffer, size_t rBuffSize);
+
+/*! FSE_readNCount_bmi2():
+ * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.
+ */
+FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
+ unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
+ const void* rBuffer, size_t rBuffSize, int bmi2);
+
+typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
+
+/*!
+Tutorial :
+----------
+(Note : these functions only decompress FSE-compressed blocks.
+ If block is uncompressed, use memcpy() instead
+ If block is a single repeated byte, use memset() instead )
+
+The first step is to obtain the normalized frequencies of symbols.
+This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
+In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
+or size the table to handle worst case situations (typically 256).
+FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
+The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
+Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
+If there is an error, the function will return an error code, which can be tested using FSE_isError().
+
+The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
+This is performed by the function FSE_buildDTable().
+The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
+If there is an error, the function will return an error code, which can be tested using FSE_isError().
+
+`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
+`cSrcSize` must be strictly correct, otherwise decompression will fail.
+FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
+If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
+*/
+
+#endif /* FSE_H */
+
+#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
+#define FSE_H_FSE_STATIC_LINKING_ONLY
+
+/* *** Dependency *** */
+#include "bitstream.h"
+
+
+/* *****************************************
+* Static allocation
+*******************************************/
+/* FSE buffer bounds */
+#define FSE_NCOUNTBOUND 512
+#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */)
+#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
+
+/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
+#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2))
+#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog)))
+
+/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */
+#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))
+#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable))
+
+
+/* *****************************************
+ * FSE advanced API
+ ***************************************** */
+
+unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
+/**< same as FSE_optimalTableLog(), which used `minus==2` */
+
+size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
+/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
+
+/* FSE_buildCTable_wksp() :
+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
+ * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`.
+ * See FSE_buildCTable_wksp() for breakdown of workspace usage.
+ */
+#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */)
+#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog))
+size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
+
+#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8)
+#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned))
+FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
+/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
+
+#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
+#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
+size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
+/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`.
+ * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */
+
+typedef enum {
+ FSE_repeat_none, /**< Cannot use the previous table */
+ FSE_repeat_check, /**< Can use the previous table but it must be checked */
+ FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */
+ } FSE_repeat;
+
+/* *****************************************
+* FSE symbol compression API
+*******************************************/
+/*!
+ This API consists of small unitary functions, which highly benefit from being inlined.
+ Hence their body are included in next section.
+*/
+typedef struct {
+ ptrdiff_t value;
+ const void* stateTable;
+ const void* symbolTT;
+ unsigned stateLog;
+} FSE_CState_t;
+
+static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
+
+static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
+
+static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
+
+/**<
+These functions are inner components of FSE_compress_usingCTable().
+They allow the creation of custom streams, mixing multiple tables and bit sources.
+
+A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
+So the first symbol you will encode is the last you will decode, like a LIFO stack.
+
+You will need a few variables to track your CStream. They are :
+
+FSE_CTable ct; // Provided by FSE_buildCTable()
+BIT_CStream_t bitStream; // bitStream tracking structure
+FSE_CState_t state; // State tracking structure (can have several)
+
+
+The first thing to do is to init bitStream and state.
+ size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
+ FSE_initCState(&state, ct);
+
+Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
+You can then encode your input data, byte after byte.
+FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
+Remember decoding will be done in reverse direction.
+ FSE_encodeByte(&bitStream, &state, symbol);
+
+At any time, you can also add any bit sequence.
+Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
+ BIT_addBits(&bitStream, bitField, nbBits);
+
+The above methods don't commit data to memory, they just store it into local register, for speed.
+Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
+Writing data to memory is a manual operation, performed by the flushBits function.
+ BIT_flushBits(&bitStream);
+
+Your last FSE encoding operation shall be to flush your last state value(s).
+ FSE_flushState(&bitStream, &state);
+
+Finally, you must close the bitStream.
+The function returns the size of CStream in bytes.
+If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
+If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
+ size_t size = BIT_closeCStream(&bitStream);
+*/
+
+
+/* *****************************************
+* FSE symbol decompression API
+*******************************************/
+typedef struct {
+ size_t state;
+ const void* table; /* precise table may vary, depending on U16 */
+} FSE_DState_t;
+
+
+static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
+
+static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
+
+static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
+
+/**<
+Let's now decompose FSE_decompress_usingDTable() into its unitary components.
+You will decode FSE-encoded symbols from the bitStream,
+and also any other bitFields you put in, **in reverse order**.
+
+You will need a few variables to track your bitStream. They are :
+
+BIT_DStream_t DStream; // Stream context
+FSE_DState_t DState; // State context. Multiple ones are possible
+FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
+
+The first thing to do is to init the bitStream.
+ errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
+
+You should then retrieve your initial state(s)
+(in reverse flushing order if you have several ones) :
+ errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
+
+You can then decode your data, symbol after symbol.
+For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
+Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
+ unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
+
+You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
+Note : maximum allowed nbBits is 25, for 32-bits compatibility
+ size_t bitField = BIT_readBits(&DStream, nbBits);
+
+All above operations only read from local register (which size depends on size_t).
+Refueling the register from memory is manually performed by the reload method.
+ endSignal = FSE_reloadDStream(&DStream);
+
+BIT_reloadDStream() result tells if there is still some more data to read from DStream.
+BIT_DStream_unfinished : there is still some data left into the DStream.
+BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
+BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
+BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
+
+When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
+to properly detect the exact end of stream.
+After each decoded symbol, check if DStream is fully consumed using this simple test :
+ BIT_reloadDStream(&DStream) >= BIT_DStream_completed
+
+When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
+Checking if DStream has reached its end is performed by :
+ BIT_endOfDStream(&DStream);
+Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
+ FSE_endOfDState(&DState);
+*/
+
+
+/* *****************************************
+* FSE unsafe API
+*******************************************/
+static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
+/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
+
+
+/* *****************************************
+* Implementation of inlined functions
+*******************************************/
+typedef struct {
+ int deltaFindState;
+ U32 deltaNbBits;
+} FSE_symbolCompressionTransform; /* total 8 bytes */
+
+MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
+{
+ const void* ptr = ct;
+ const U16* u16ptr = (const U16*) ptr;
+ const U32 tableLog = MEM_read16(ptr);
+ statePtr->value = (ptrdiff_t)1<<tableLog;
+ statePtr->stateTable = u16ptr+2;
+ statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
+ statePtr->stateLog = tableLog;
+}
+
+
+/*! FSE_initCState2() :
+* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
+* uses the smallest state value possible, saving the cost of this symbol */
+MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
+{
+ FSE_initCState(statePtr, ct);
+ { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
+ const U16* stateTable = (const U16*)(statePtr->stateTable);
+ U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
+ statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
+ statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
+ }
+}
+
+MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
+{
+ FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
+ const U16* const stateTable = (const U16*)(statePtr->stateTable);
+ U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
+ BIT_addBits(bitC, statePtr->value, nbBitsOut);
+ statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
+}
+
+MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
+{
+ BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
+ BIT_flushBits(bitC);
+}
+
+
+/* FSE_getMaxNbBits() :
+ * Approximate maximum cost of a symbol, in bits.
+ * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
+MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
+{
+ const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
+ return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
+}
+
+/* FSE_bitCost() :
+ * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
+MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
+{
+ const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
+ U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
+ U32 const threshold = (minNbBits+1) << 16;
+ assert(tableLog < 16);
+ assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */
+ { U32 const tableSize = 1 << tableLog;
+ U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
+ U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */
+ U32 const bitMultiplier = 1 << accuracyLog;
+ assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
+ assert(normalizedDeltaFromThreshold <= bitMultiplier);
+ return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
+ }
+}
+
+
+/* ====== Decompression ====== */
+
+typedef struct {
+ U16 tableLog;
+ U16 fastMode;
+} FSE_DTableHeader; /* sizeof U32 */
+
+typedef struct
+{
+ unsigned short newState;
+ unsigned char symbol;
+ unsigned char nbBits;
+} FSE_decode_t; /* size == U32 */
+
+MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
+{
+ const void* ptr = dt;
+ const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
+ DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
+ BIT_reloadDStream(bitD);
+ DStatePtr->table = dt + 1;
+}
+
+MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+ return DInfo.symbol;
+}
+
+MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+ U32 const nbBits = DInfo.nbBits;
+ size_t const lowBits = BIT_readBits(bitD, nbBits);
+ DStatePtr->state = DInfo.newState + lowBits;
+}
+
+MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+ U32 const nbBits = DInfo.nbBits;
+ BYTE const symbol = DInfo.symbol;
+ size_t const lowBits = BIT_readBits(bitD, nbBits);
+
+ DStatePtr->state = DInfo.newState + lowBits;
+ return symbol;
+}
+
+/*! FSE_decodeSymbolFast() :
+ unsafe, only works if no symbol has a probability > 50% */
+MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
+ U32 const nbBits = DInfo.nbBits;
+ BYTE const symbol = DInfo.symbol;
+ size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
+
+ DStatePtr->state = DInfo.newState + lowBits;
+ return symbol;
+}
+
+MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
+{
+ return DStatePtr->state == 0;
+}
+
+
+
+#ifndef FSE_COMMONDEFS_ONLY
+
+/* **************************************************************
+* Tuning parameters
+****************************************************************/
+/*!MEMORY_USAGE :
+* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+* Increasing memory usage improves compression ratio
+* Reduced memory usage can improve speed, due to cache effect
+* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
+#ifndef FSE_MAX_MEMORY_USAGE
+# define FSE_MAX_MEMORY_USAGE 14
+#endif
+#ifndef FSE_DEFAULT_MEMORY_USAGE
+# define FSE_DEFAULT_MEMORY_USAGE 13
+#endif
+#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE)
+# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE"
+#endif
+
+/*!FSE_MAX_SYMBOL_VALUE :
+* Maximum symbol value authorized.
+* Required for proper stack allocation */
+#ifndef FSE_MAX_SYMBOL_VALUE
+# define FSE_MAX_SYMBOL_VALUE 255
+#endif
+
+/* **************************************************************
+* template functions type & suffix
+****************************************************************/
+#define FSE_FUNCTION_TYPE BYTE
+#define FSE_FUNCTION_EXTENSION
+#define FSE_DECODE_TYPE FSE_decode_t
+
+
+#endif /* !FSE_COMMONDEFS_ONLY */
+
+
+/* ***************************************************************
+* Constants
+*****************************************************************/
+#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
+#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
+#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
+#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
+#define FSE_MIN_TABLELOG 5
+
+#define FSE_TABLELOG_ABSOLUTE_MAX 15
+#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
+# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
+#endif
+
+#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3)
+
+
+#endif /* FSE_STATIC_LINKING_ONLY */
+
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/contrib/zstd/fse_compress.c b/contrib/zstd/fse_compress.c
new file mode 100644
index 0000000..3fec4a3
--- /dev/null
+++ b/contrib/zstd/fse_compress.c
@@ -0,0 +1,624 @@
+/* ******************************************************************
+ * FSE : Finite State Entropy encoder
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* **************************************************************
+* Includes
+****************************************************************/
+#include "compiler.h"
+#include "mem.h" /* U32, U16, etc. */
+#include "debug.h" /* assert, DEBUGLOG */
+#include "hist.h" /* HIST_count_wksp */
+#include "bitstream.h"
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "error_private.h"
+#define ZSTD_DEPS_NEED_MALLOC
+#define ZSTD_DEPS_NEED_MATH64
+#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */
+#include "bits.h" /* ZSTD_highbit32 */
+
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define FSE_isError ERR_isError
+
+
+/* **************************************************************
+* Templates
+****************************************************************/
+/*
+ designed to be included
+ for type-specific functions (template emulation in C)
+ Objective is to write these functions only once, for improved maintenance
+*/
+
+/* safety checks */
+#ifndef FSE_FUNCTION_EXTENSION
+# error "FSE_FUNCTION_EXTENSION must be defined"
+#endif
+#ifndef FSE_FUNCTION_TYPE
+# error "FSE_FUNCTION_TYPE must be defined"
+#endif
+
+/* Function names */
+#define FSE_CAT(X,Y) X##Y
+#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
+#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
+
+
+/* Function templates */
+
+/* FSE_buildCTable_wksp() :
+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
+ * wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
+ * workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
+ */
+size_t FSE_buildCTable_wksp(FSE_CTable* ct,
+ const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
+ void* workSpace, size_t wkspSize)
+{
+ U32 const tableSize = 1 << tableLog;
+ U32 const tableMask = tableSize - 1;
+ void* const ptr = ct;
+ U16* const tableU16 = ( (U16*) ptr) + 2;
+ void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
+ FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
+ U32 const step = FSE_TABLESTEP(tableSize);
+ U32 const maxSV1 = maxSymbolValue+1;
+
+ U16* cumul = (U16*)workSpace; /* size = maxSV1 */
+ FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */
+
+ U32 highThreshold = tableSize-1;
+
+ assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */
+ if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge);
+ /* CTable header */
+ tableU16[-2] = (U16) tableLog;
+ tableU16[-1] = (U16) maxSymbolValue;
+ assert(tableLog < 16); /* required for threshold strategy to work */
+
+ /* For explanations on how to distribute symbol values over the table :
+ * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
+
+ #ifdef __clang_analyzer__
+ ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
+ #endif
+
+ /* symbol start positions */
+ { U32 u;
+ cumul[0] = 0;
+ for (u=1; u <= maxSV1; u++) {
+ if (normalizedCounter[u-1]==-1) { /* Low proba symbol */
+ cumul[u] = cumul[u-1] + 1;
+ tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
+ } else {
+ assert(normalizedCounter[u-1] >= 0);
+ cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1];
+ assert(cumul[u] >= cumul[u-1]); /* no overflow */
+ } }
+ cumul[maxSV1] = (U16)(tableSize+1);
+ }
+
+ /* Spread symbols */
+ if (highThreshold == tableSize - 1) {
+ /* Case for no low prob count symbols. Lay down 8 bytes at a time
+ * to reduce branch misses since we are operating on a small block
+ */
+ BYTE* const spread = tableSymbol + tableSize; /* size = tableSize + 8 (may write beyond tableSize) */
+ { U64 const add = 0x0101010101010101ull;
+ size_t pos = 0;
+ U64 sv = 0;
+ U32 s;
+ for (s=0; s<maxSV1; ++s, sv += add) {
+ int i;
+ int const n = normalizedCounter[s];
+ MEM_write64(spread + pos, sv);
+ for (i = 8; i < n; i += 8) {
+ MEM_write64(spread + pos + i, sv);
+ }
+ assert(n>=0);
+ pos += (size_t)n;
+ }
+ }
+ /* Spread symbols across the table. Lack of lowprob symbols means that
+ * we don't need variable sized inner loop, so we can unroll the loop and
+ * reduce branch misses.
+ */
+ { size_t position = 0;
+ size_t s;
+ size_t const unroll = 2; /* Experimentally determined optimal unroll */
+ assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
+ for (s = 0; s < (size_t)tableSize; s += unroll) {
+ size_t u;
+ for (u = 0; u < unroll; ++u) {
+ size_t const uPosition = (position + (u * step)) & tableMask;
+ tableSymbol[uPosition] = spread[s + u];
+ }
+ position = (position + (unroll * step)) & tableMask;
+ }
+ assert(position == 0); /* Must have initialized all positions */
+ }
+ } else {
+ U32 position = 0;
+ U32 symbol;
+ for (symbol=0; symbol<maxSV1; symbol++) {
+ int nbOccurrences;
+ int const freq = normalizedCounter[symbol];
+ for (nbOccurrences=0; nbOccurrences<freq; nbOccurrences++) {
+ tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
+ position = (position + step) & tableMask;
+ while (position > highThreshold)
+ position = (position + step) & tableMask; /* Low proba area */
+ } }
+ assert(position==0); /* Must have initialized all positions */
+ }
+
+ /* Build table */
+ { U32 u; for (u=0; u<tableSize; u++) {
+ FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
+ tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */
+ } }
+
+ /* Build Symbol Transformation Table */
+ { unsigned total = 0;
+ unsigned s;
+ for (s=0; s<=maxSymbolValue; s++) {
+ switch (normalizedCounter[s])
+ {
+ case 0:
+ /* filling nonetheless, for compatibility with FSE_getMaxNbBits() */
+ symbolTT[s].deltaNbBits = ((tableLog+1) << 16) - (1<<tableLog);
+ break;
+
+ case -1:
+ case 1:
+ symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
+ assert(total <= INT_MAX);
+ symbolTT[s].deltaFindState = (int)(total - 1);
+ total ++;
+ break;
+ default :
+ assert(normalizedCounter[s] > 1);
+ { U32 const maxBitsOut = tableLog - ZSTD_highbit32 ((U32)normalizedCounter[s]-1);
+ U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut;
+ symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
+ symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]);
+ total += (unsigned)normalizedCounter[s];
+ } } } }
+
+#if 0 /* debug : symbol costs */
+ DEBUGLOG(5, "\n --- table statistics : ");
+ { U32 symbol;
+ for (symbol=0; symbol<=maxSymbolValue; symbol++) {
+ DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f",
+ symbol, normalizedCounter[symbol],
+ FSE_getMaxNbBits(symbolTT, symbol),
+ (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256);
+ } }
+#endif
+
+ return 0;
+}
+
+
+
+#ifndef FSE_COMMONDEFS_ONLY
+
+/*-**************************************************************
+* FSE NCount encoding
+****************************************************************/
+size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
+{
+ size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog
+ + 4 /* bitCount initialized at 4 */
+ + 2 /* first two symbols may use one additional bit each */) / 8)
+ + 1 /* round up to whole nb bytes */
+ + 2 /* additional two bytes for bitstream flush */;
+ return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
+}
+
+static size_t
+FSE_writeNCount_generic (void* header, size_t headerBufferSize,
+ const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
+ unsigned writeIsSafe)
+{
+ BYTE* const ostart = (BYTE*) header;
+ BYTE* out = ostart;
+ BYTE* const oend = ostart + headerBufferSize;
+ int nbBits;
+ const int tableSize = 1 << tableLog;
+ int remaining;
+ int threshold;
+ U32 bitStream = 0;
+ int bitCount = 0;
+ unsigned symbol = 0;
+ unsigned const alphabetSize = maxSymbolValue + 1;
+ int previousIs0 = 0;
+
+ /* Table Size */
+ bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
+ bitCount += 4;
+
+ /* Init */
+ remaining = tableSize+1; /* +1 for extra accuracy */
+ threshold = tableSize;
+ nbBits = tableLog+1;
+
+ while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */
+ if (previousIs0) {
+ unsigned start = symbol;
+ while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++;
+ if (symbol == alphabetSize) break; /* incorrect distribution */
+ while (symbol >= start+24) {
+ start+=24;
+ bitStream += 0xFFFFU << bitCount;
+ if ((!writeIsSafe) && (out > oend-2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE) bitStream;
+ out[1] = (BYTE)(bitStream>>8);
+ out+=2;
+ bitStream>>=16;
+ }
+ while (symbol >= start+3) {
+ start+=3;
+ bitStream += 3 << bitCount;
+ bitCount += 2;
+ }
+ bitStream += (symbol-start) << bitCount;
+ bitCount += 2;
+ if (bitCount>16) {
+ if ((!writeIsSafe) && (out > oend - 2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE)bitStream;
+ out[1] = (BYTE)(bitStream>>8);
+ out += 2;
+ bitStream >>= 16;
+ bitCount -= 16;
+ } }
+ { int count = normalizedCounter[symbol++];
+ int const max = (2*threshold-1) - remaining;
+ remaining -= count < 0 ? -count : count;
+ count++; /* +1 for extra accuracy */
+ if (count>=threshold)
+ count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
+ bitStream += count << bitCount;
+ bitCount += nbBits;
+ bitCount -= (count<max);
+ previousIs0 = (count==1);
+ if (remaining<1) return ERROR(GENERIC);
+ while (remaining<threshold) { nbBits--; threshold>>=1; }
+ }
+ if (bitCount>16) {
+ if ((!writeIsSafe) && (out > oend - 2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE)bitStream;
+ out[1] = (BYTE)(bitStream>>8);
+ out += 2;
+ bitStream >>= 16;
+ bitCount -= 16;
+ } }
+
+ if (remaining != 1)
+ return ERROR(GENERIC); /* incorrect normalized distribution */
+ assert(symbol <= alphabetSize);
+
+ /* flush remaining bitStream */
+ if ((!writeIsSafe) && (out > oend - 2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE)bitStream;
+ out[1] = (BYTE)(bitStream>>8);
+ out+= (bitCount+7) /8;
+
+ return (out-ostart);
+}
+
+
+size_t FSE_writeNCount (void* buffer, size_t bufferSize,
+ const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
+{
+ if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */
+ if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */
+
+ if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
+
+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */);
+}
+
+
+/*-**************************************************************
+* FSE Compression Code
+****************************************************************/
+
+/* provides the minimum logSize to safely represent a distribution */
+static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
+{
+ U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1;
+ U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2;
+ U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
+ assert(srcSize > 1); /* Not supported, RLE should be used instead */
+ return minBits;
+}
+
+unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
+{
+ U32 maxBitsSrc = ZSTD_highbit32((U32)(srcSize - 1)) - minus;
+ U32 tableLog = maxTableLog;
+ U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
+ assert(srcSize > 1); /* Not supported, RLE should be used instead */
+ if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
+ if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
+ if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
+ if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
+ if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
+ return tableLog;
+}
+
+unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
+{
+ return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
+}
+
+/* Secondary normalization method.
+ To be used when primary method fails. */
+
+static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount)
+{
+ short const NOT_YET_ASSIGNED = -2;
+ U32 s;
+ U32 distributed = 0;
+ U32 ToDistribute;
+
+ /* Init */
+ U32 const lowThreshold = (U32)(total >> tableLog);
+ U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
+
+ for (s=0; s<=maxSymbolValue; s++) {
+ if (count[s] == 0) {
+ norm[s]=0;
+ continue;
+ }
+ if (count[s] <= lowThreshold) {
+ norm[s] = lowProbCount;
+ distributed++;
+ total -= count[s];
+ continue;
+ }
+ if (count[s] <= lowOne) {
+ norm[s] = 1;
+ distributed++;
+ total -= count[s];
+ continue;
+ }
+
+ norm[s]=NOT_YET_ASSIGNED;
+ }
+ ToDistribute = (1 << tableLog) - distributed;
+
+ if (ToDistribute == 0)
+ return 0;
+
+ if ((total / ToDistribute) > lowOne) {
+ /* risk of rounding to zero */
+ lowOne = (U32)((total * 3) / (ToDistribute * 2));
+ for (s=0; s<=maxSymbolValue; s++) {
+ if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
+ norm[s] = 1;
+ distributed++;
+ total -= count[s];
+ continue;
+ } }
+ ToDistribute = (1 << tableLog) - distributed;
+ }
+
+ if (distributed == maxSymbolValue+1) {
+ /* all values are pretty poor;
+ probably incompressible data (should have already been detected);
+ find max, then give all remaining points to max */
+ U32 maxV = 0, maxC = 0;
+ for (s=0; s<=maxSymbolValue; s++)
+ if (count[s] > maxC) { maxV=s; maxC=count[s]; }
+ norm[maxV] += (short)ToDistribute;
+ return 0;
+ }
+
+ if (total == 0) {
+ /* all of the symbols were low enough for the lowOne or lowThreshold */
+ for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1))
+ if (norm[s] > 0) { ToDistribute--; norm[s]++; }
+ return 0;
+ }
+
+ { U64 const vStepLog = 62 - tableLog;
+ U64 const mid = (1ULL << (vStepLog-1)) - 1;
+ U64 const rStep = ZSTD_div64((((U64)1<<vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */
+ U64 tmpTotal = mid;
+ for (s=0; s<=maxSymbolValue; s++) {
+ if (norm[s]==NOT_YET_ASSIGNED) {
+ U64 const end = tmpTotal + (count[s] * rStep);
+ U32 const sStart = (U32)(tmpTotal >> vStepLog);
+ U32 const sEnd = (U32)(end >> vStepLog);
+ U32 const weight = sEnd - sStart;
+ if (weight < 1)
+ return ERROR(GENERIC);
+ norm[s] = (short)weight;
+ tmpTotal = end;
+ } } }
+
+ return 0;
+}
+
+size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
+ const unsigned* count, size_t total,
+ unsigned maxSymbolValue, unsigned useLowProbCount)
+{
+ /* Sanity checks */
+ if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
+ if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */
+ if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */
+ if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
+
+ { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
+ short const lowProbCount = useLowProbCount ? -1 : 1;
+ U64 const scale = 62 - tableLog;
+ U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */
+ U64 const vStep = 1ULL<<(scale-20);
+ int stillToDistribute = 1<<tableLog;
+ unsigned s;
+ unsigned largest=0;
+ short largestP=0;
+ U32 lowThreshold = (U32)(total >> tableLog);
+
+ for (s=0; s<=maxSymbolValue; s++) {
+ if (count[s] == total) return 0; /* rle special case */
+ if (count[s] == 0) { normalizedCounter[s]=0; continue; }
+ if (count[s] <= lowThreshold) {
+ normalizedCounter[s] = lowProbCount;
+ stillToDistribute--;
+ } else {
+ short proba = (short)((count[s]*step) >> scale);
+ if (proba<8) {
+ U64 restToBeat = vStep * rtbTable[proba];
+ proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat;
+ }
+ if (proba > largestP) { largestP=proba; largest=s; }
+ normalizedCounter[s] = proba;
+ stillToDistribute -= proba;
+ } }
+ if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
+ /* corner case, need another normalization method */
+ size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount);
+ if (FSE_isError(errorCode)) return errorCode;
+ }
+ else normalizedCounter[largest] += (short)stillToDistribute;
+ }
+
+#if 0
+ { /* Print Table (debug) */
+ U32 s;
+ U32 nTotal = 0;
+ for (s=0; s<=maxSymbolValue; s++)
+ RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]);
+ for (s=0; s<=maxSymbolValue; s++)
+ nTotal += abs(normalizedCounter[s]);
+ if (nTotal != (1U<<tableLog))
+ RAWLOG(2, "Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog);
+ getchar();
+ }
+#endif
+
+ return tableLog;
+}
+
+/* fake FSE_CTable, for rle input (always same symbol) */
+size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
+{
+ void* ptr = ct;
+ U16* tableU16 = ( (U16*) ptr) + 2;
+ void* FSCTptr = (U32*)ptr + 2;
+ FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr;
+
+ /* header */
+ tableU16[-2] = (U16) 0;
+ tableU16[-1] = (U16) symbolValue;
+
+ /* Build table */
+ tableU16[0] = 0;
+ tableU16[1] = 0; /* just in case */
+
+ /* Build Symbol Transformation Table */
+ symbolTT[symbolValue].deltaNbBits = 0;
+ symbolTT[symbolValue].deltaFindState = 0;
+
+ return 0;
+}
+
+
+static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const FSE_CTable* ct, const unsigned fast)
+{
+ const BYTE* const istart = (const BYTE*) src;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* ip=iend;
+
+ BIT_CStream_t bitC;
+ FSE_CState_t CState1, CState2;
+
+ /* init */
+ if (srcSize <= 2) return 0;
+ { size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
+ if (FSE_isError(initError)) return 0; /* not enough space available to write a bitstream */ }
+
+#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
+
+ if (srcSize & 1) {
+ FSE_initCState2(&CState1, ct, *--ip);
+ FSE_initCState2(&CState2, ct, *--ip);
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+ FSE_FLUSHBITS(&bitC);
+ } else {
+ FSE_initCState2(&CState2, ct, *--ip);
+ FSE_initCState2(&CState1, ct, *--ip);
+ }
+
+ /* join to mod 4 */
+ srcSize -= 2;
+ if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+ FSE_FLUSHBITS(&bitC);
+ }
+
+ /* 2 or 4 encoding per loop */
+ while ( ip>istart ) {
+
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
+
+ if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
+ FSE_FLUSHBITS(&bitC);
+
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+
+ if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+ }
+
+ FSE_FLUSHBITS(&bitC);
+ }
+
+ FSE_flushCState(&bitC, &CState2);
+ FSE_flushCState(&bitC, &CState1);
+ return BIT_closeCStream(&bitC);
+}
+
+size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const FSE_CTable* ct)
+{
+ unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
+
+ if (fast)
+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
+ else
+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
+}
+
+
+size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
+
+#endif /* FSE_COMMONDEFS_ONLY */
diff --git a/contrib/zstd/fse_decompress.c b/contrib/zstd/fse_decompress.c
new file mode 100644
index 0000000..1e1c9f9
--- /dev/null
+++ b/contrib/zstd/fse_decompress.c
@@ -0,0 +1,311 @@
+/* ******************************************************************
+ * FSE : Finite State Entropy decoder
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+
+/* **************************************************************
+* Includes
+****************************************************************/
+#include "debug.h" /* assert */
+#include "bitstream.h"
+#include "compiler.h"
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "error_private.h"
+#define ZSTD_DEPS_NEED_MALLOC
+#include "zstd_deps.h"
+#include "bits.h" /* ZSTD_highbit32 */
+
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define FSE_isError ERR_isError
+#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
+
+
+/* **************************************************************
+* Templates
+****************************************************************/
+/*
+ designed to be included
+ for type-specific functions (template emulation in C)
+ Objective is to write these functions only once, for improved maintenance
+*/
+
+/* safety checks */
+#ifndef FSE_FUNCTION_EXTENSION
+# error "FSE_FUNCTION_EXTENSION must be defined"
+#endif
+#ifndef FSE_FUNCTION_TYPE
+# error "FSE_FUNCTION_TYPE must be defined"
+#endif
+
+/* Function names */
+#define FSE_CAT(X,Y) X##Y
+#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
+#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
+
+static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
+{
+ void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
+ FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
+ U16* symbolNext = (U16*)workSpace;
+ BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1);
+
+ U32 const maxSV1 = maxSymbolValue + 1;
+ U32 const tableSize = 1 << tableLog;
+ U32 highThreshold = tableSize-1;
+
+ /* Sanity Checks */
+ if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge);
+ if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
+ if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
+
+ /* Init, lay down lowprob symbols */
+ { FSE_DTableHeader DTableH;
+ DTableH.tableLog = (U16)tableLog;
+ DTableH.fastMode = 1;
+ { S16 const largeLimit= (S16)(1 << (tableLog-1));
+ U32 s;
+ for (s=0; s<maxSV1; s++) {
+ if (normalizedCounter[s]==-1) {
+ tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
+ symbolNext[s] = 1;
+ } else {
+ if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
+ symbolNext[s] = normalizedCounter[s];
+ } } }
+ ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
+ }
+
+ /* Spread symbols */
+ if (highThreshold == tableSize - 1) {
+ size_t const tableMask = tableSize-1;
+ size_t const step = FSE_TABLESTEP(tableSize);
+ /* First lay down the symbols in order.
+ * We use a uint64_t to lay down 8 bytes at a time. This reduces branch
+ * misses since small blocks generally have small table logs, so nearly
+ * all symbols have counts <= 8. We ensure we have 8 bytes at the end of
+ * our buffer to handle the over-write.
+ */
+ {
+ U64 const add = 0x0101010101010101ull;
+ size_t pos = 0;
+ U64 sv = 0;
+ U32 s;
+ for (s=0; s<maxSV1; ++s, sv += add) {
+ int i;
+ int const n = normalizedCounter[s];
+ MEM_write64(spread + pos, sv);
+ for (i = 8; i < n; i += 8) {
+ MEM_write64(spread + pos + i, sv);
+ }
+ pos += n;
+ }
+ }
+ /* Now we spread those positions across the table.
+ * The benefit of doing it in two stages is that we avoid the
+ * variable size inner loop, which caused lots of branch misses.
+ * Now we can run through all the positions without any branch misses.
+ * We unroll the loop twice, since that is what empirically worked best.
+ */
+ {
+ size_t position = 0;
+ size_t s;
+ size_t const unroll = 2;
+ assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
+ for (s = 0; s < (size_t)tableSize; s += unroll) {
+ size_t u;
+ for (u = 0; u < unroll; ++u) {
+ size_t const uPosition = (position + (u * step)) & tableMask;
+ tableDecode[uPosition].symbol = spread[s + u];
+ }
+ position = (position + (unroll * step)) & tableMask;
+ }
+ assert(position == 0);
+ }
+ } else {
+ U32 const tableMask = tableSize-1;
+ U32 const step = FSE_TABLESTEP(tableSize);
+ U32 s, position = 0;
+ for (s=0; s<maxSV1; s++) {
+ int i;
+ for (i=0; i<normalizedCounter[s]; i++) {
+ tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
+ position = (position + step) & tableMask;
+ while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
+ } }
+ if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
+ }
+
+ /* Build Decoding table */
+ { U32 u;
+ for (u=0; u<tableSize; u++) {
+ FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
+ U32 const nextState = symbolNext[symbol]++;
+ tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
+ tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
+ } }
+
+ return 0;
+}
+
+size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
+{
+ return FSE_buildDTable_internal(dt, normalizedCounter, maxSymbolValue, tableLog, workSpace, wkspSize);
+}
+
+
+#ifndef FSE_COMMONDEFS_ONLY
+
+/*-*******************************************************
+* Decompression (Byte symbols)
+*********************************************************/
+
+FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
+ void* dst, size_t maxDstSize,
+ const void* cSrc, size_t cSrcSize,
+ const FSE_DTable* dt, const unsigned fast)
+{
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* op = ostart;
+ BYTE* const omax = op + maxDstSize;
+ BYTE* const olimit = omax-3;
+
+ BIT_DStream_t bitD;
+ FSE_DState_t state1;
+ FSE_DState_t state2;
+
+ /* Init */
+ CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
+
+ FSE_initDState(&state1, &bitD, dt);
+ FSE_initDState(&state2, &bitD, dt);
+
+#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
+
+ /* 4 symbols per loop */
+ for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) {
+ op[0] = FSE_GETSYMBOL(&state1);
+
+ if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
+ BIT_reloadDStream(&bitD);
+
+ op[1] = FSE_GETSYMBOL(&state2);
+
+ if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
+ { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
+
+ op[2] = FSE_GETSYMBOL(&state1);
+
+ if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
+ BIT_reloadDStream(&bitD);
+
+ op[3] = FSE_GETSYMBOL(&state2);
+ }
+
+ /* tail */
+ /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
+ while (1) {
+ if (op>(omax-2)) return ERROR(dstSize_tooSmall);
+ *op++ = FSE_GETSYMBOL(&state1);
+ if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
+ *op++ = FSE_GETSYMBOL(&state2);
+ break;
+ }
+
+ if (op>(omax-2)) return ERROR(dstSize_tooSmall);
+ *op++ = FSE_GETSYMBOL(&state2);
+ if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
+ *op++ = FSE_GETSYMBOL(&state1);
+ break;
+ } }
+
+ return op-ostart;
+}
+
+typedef struct {
+ short ncount[FSE_MAX_SYMBOL_VALUE + 1];
+ FSE_DTable dtable[1]; /* Dynamically sized */
+} FSE_DecompressWksp;
+
+
+FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
+ void* dst, size_t dstCapacity,
+ const void* cSrc, size_t cSrcSize,
+ unsigned maxLog, void* workSpace, size_t wkspSize,
+ int bmi2)
+{
+ const BYTE* const istart = (const BYTE*)cSrc;
+ const BYTE* ip = istart;
+ unsigned tableLog;
+ unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
+ FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace;
+
+ DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0);
+ if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC);
+
+ /* normal FSE decoding mode */
+ {
+ size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2);
+ if (FSE_isError(NCountLength)) return NCountLength;
+ if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
+ assert(NCountLength <= cSrcSize);
+ ip += NCountLength;
+ cSrcSize -= NCountLength;
+ }
+
+ if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
+ assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize);
+ workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
+ wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
+
+ CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
+
+ {
+ const void* ptr = wksp->dtable;
+ const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
+ const U32 fastMode = DTableH->fastMode;
+
+ /* select fast mode (static) */
+ if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1);
+ return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0);
+ }
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
+{
+ return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0);
+}
+
+#if DYNAMIC_BMI2
+BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
+{
+ return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1);
+}
+#endif
+
+size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2)
+{
+#if DYNAMIC_BMI2
+ if (bmi2) {
+ return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
+ }
+#endif
+ (void)bmi2;
+ return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
+}
+
+#endif /* FSE_COMMONDEFS_ONLY */
diff --git a/contrib/zstd/hist.c b/contrib/zstd/hist.c
new file mode 100644
index 0000000..ebb1a1a
--- /dev/null
+++ b/contrib/zstd/hist.c
@@ -0,0 +1,181 @@
+/* ******************************************************************
+ * hist : Histogram functions
+ * part of Finite State Entropy project
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* --- dependencies --- */
+#include "mem.h" /* U32, BYTE, etc. */
+#include "debug.h" /* assert, DEBUGLOG */
+#include "error_private.h" /* ERROR */
+#include "hist.h"
+
+
+/* --- Error management --- */
+unsigned HIST_isError(size_t code) { return ERR_isError(code); }
+
+/*-**************************************************************
+ * Histogram functions
+ ****************************************************************/
+unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize)
+{
+ const BYTE* ip = (const BYTE*)src;
+ const BYTE* const end = ip + srcSize;
+ unsigned maxSymbolValue = *maxSymbolValuePtr;
+ unsigned largestCount=0;
+
+ ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count));
+ if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
+
+ while (ip<end) {
+ assert(*ip <= maxSymbolValue);
+ count[*ip++]++;
+ }
+
+ while (!count[maxSymbolValue]) maxSymbolValue--;
+ *maxSymbolValuePtr = maxSymbolValue;
+
+ { U32 s;
+ for (s=0; s<=maxSymbolValue; s++)
+ if (count[s] > largestCount) largestCount = count[s];
+ }
+
+ return largestCount;
+}
+
+typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e;
+
+/* HIST_count_parallel_wksp() :
+ * store histogram into 4 intermediate tables, recombined at the end.
+ * this design makes better use of OoO cpus,
+ * and is noticeably faster when some values are heavily repeated.
+ * But it needs some additional workspace for intermediate tables.
+ * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32.
+ * @return : largest histogram frequency,
+ * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */
+static size_t HIST_count_parallel_wksp(
+ unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* source, size_t sourceSize,
+ HIST_checkInput_e check,
+ U32* const workSpace)
+{
+ const BYTE* ip = (const BYTE*)source;
+ const BYTE* const iend = ip+sourceSize;
+ size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count);
+ unsigned max=0;
+ U32* const Counting1 = workSpace;
+ U32* const Counting2 = Counting1 + 256;
+ U32* const Counting3 = Counting2 + 256;
+ U32* const Counting4 = Counting3 + 256;
+
+ /* safety checks */
+ assert(*maxSymbolValuePtr <= 255);
+ if (!sourceSize) {
+ ZSTD_memset(count, 0, countSize);
+ *maxSymbolValuePtr = 0;
+ return 0;
+ }
+ ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned));
+
+ /* by stripes of 16 bytes */
+ { U32 cached = MEM_read32(ip); ip += 4;
+ while (ip < iend-15) {
+ U32 c = cached; cached = MEM_read32(ip); ip += 4;
+ Counting1[(BYTE) c ]++;
+ Counting2[(BYTE)(c>>8) ]++;
+ Counting3[(BYTE)(c>>16)]++;
+ Counting4[ c>>24 ]++;
+ c = cached; cached = MEM_read32(ip); ip += 4;
+ Counting1[(BYTE) c ]++;
+ Counting2[(BYTE)(c>>8) ]++;
+ Counting3[(BYTE)(c>>16)]++;
+ Counting4[ c>>24 ]++;
+ c = cached; cached = MEM_read32(ip); ip += 4;
+ Counting1[(BYTE) c ]++;
+ Counting2[(BYTE)(c>>8) ]++;
+ Counting3[(BYTE)(c>>16)]++;
+ Counting4[ c>>24 ]++;
+ c = cached; cached = MEM_read32(ip); ip += 4;
+ Counting1[(BYTE) c ]++;
+ Counting2[(BYTE)(c>>8) ]++;
+ Counting3[(BYTE)(c>>16)]++;
+ Counting4[ c>>24 ]++;
+ }
+ ip-=4;
+ }
+
+ /* finish last symbols */
+ while (ip<iend) Counting1[*ip++]++;
+
+ { U32 s;
+ for (s=0; s<256; s++) {
+ Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
+ if (Counting1[s] > max) max = Counting1[s];
+ } }
+
+ { unsigned maxSymbolValue = 255;
+ while (!Counting1[maxSymbolValue]) maxSymbolValue--;
+ if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall);
+ *maxSymbolValuePtr = maxSymbolValue;
+ ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */
+ }
+ return (size_t)max;
+}
+
+/* HIST_countFast_wksp() :
+ * Same as HIST_countFast(), but using an externally provided scratch buffer.
+ * `workSpace` is a writable buffer which must be 4-bytes aligned,
+ * `workSpaceSize` must be >= HIST_WKSP_SIZE
+ */
+size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* source, size_t sourceSize,
+ void* workSpace, size_t workSpaceSize)
+{
+ if (sourceSize < 1500) /* heuristic threshold */
+ return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize);
+ if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
+ if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
+ return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace);
+}
+
+/* HIST_count_wksp() :
+ * Same as HIST_count(), but using an externally provided scratch buffer.
+ * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
+size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* source, size_t sourceSize,
+ void* workSpace, size_t workSpaceSize)
+{
+ if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
+ if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
+ if (*maxSymbolValuePtr < 255)
+ return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace);
+ *maxSymbolValuePtr = 255;
+ return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize);
+}
+
+#ifndef ZSTD_NO_UNUSED_FUNCTIONS
+/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
+size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* source, size_t sourceSize)
+{
+ unsigned tmpCounters[HIST_WKSP_SIZE_U32];
+ return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
+}
+
+size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize)
+{
+ unsigned tmpCounters[HIST_WKSP_SIZE_U32];
+ return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
+}
+#endif
diff --git a/contrib/zstd/hist.h b/contrib/zstd/hist.h
new file mode 100644
index 0000000..b8d5ba7
--- /dev/null
+++ b/contrib/zstd/hist.h
@@ -0,0 +1,75 @@
+/* ******************************************************************
+ * hist : Histogram functions
+ * part of Finite State Entropy project
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* --- dependencies --- */
+#include "zstd_deps.h" /* size_t */
+
+
+/* --- simple histogram functions --- */
+
+/*! HIST_count():
+ * Provides the precise count of each byte within a table 'count'.
+ * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
+ * Updates *maxSymbolValuePtr with actual largest symbol value detected.
+ * @return : count of the most frequent symbol (which isn't identified).
+ * or an error code, which can be tested using HIST_isError().
+ * note : if return == srcSize, there is only one symbol.
+ */
+size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize);
+
+unsigned HIST_isError(size_t code); /**< tells if a return value is an error code */
+
+
+/* --- advanced histogram functions --- */
+
+#define HIST_WKSP_SIZE_U32 1024
+#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned))
+/** HIST_count_wksp() :
+ * Same as HIST_count(), but using an externally provided scratch buffer.
+ * Benefit is this function will use very little stack space.
+ * `workSpace` is a writable buffer which must be 4-bytes aligned,
+ * `workSpaceSize` must be >= HIST_WKSP_SIZE
+ */
+size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t workSpaceSize);
+
+/** HIST_countFast() :
+ * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr.
+ * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr`
+ */
+size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize);
+
+/** HIST_countFast_wksp() :
+ * Same as HIST_countFast(), but using an externally provided scratch buffer.
+ * `workSpace` is a writable buffer which must be 4-bytes aligned,
+ * `workSpaceSize` must be >= HIST_WKSP_SIZE
+ */
+size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t workSpaceSize);
+
+/*! HIST_count_simple() :
+ * Same as HIST_countFast(), this function is unsafe,
+ * and will segfault if any value within `src` is `> *maxSymbolValuePtr`.
+ * It is also a bit slower for large inputs.
+ * However, it does not need any additional memory (not even on stack).
+ * @return : count of the most frequent symbol.
+ * Note this function doesn't produce any error (i.e. it must succeed).
+ */
+unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize);
diff --git a/contrib/zstd/huf.h b/contrib/zstd/huf.h
new file mode 100644
index 0000000..73d1ee5
--- /dev/null
+++ b/contrib/zstd/huf.h
@@ -0,0 +1,273 @@
+/* ******************************************************************
+ * huff0 huffman codec,
+ * part of Finite State Entropy library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef HUF_H_298734234
+#define HUF_H_298734234
+
+/* *** Dependencies *** */
+#include "zstd_deps.h" /* size_t */
+#include "mem.h" /* U32 */
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+
+
+/* *** Tool functions *** */
+#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
+size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
+
+/* Error Management */
+unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
+const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
+
+
+#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
+#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
+
+/* *** Constants *** */
+#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
+#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
+#define HUF_SYMBOLVALUE_MAX 255
+
+#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
+#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
+# error "HUF_TABLELOG_MAX is too large !"
+#endif
+
+
+/* ****************************************
+* Static allocation
+******************************************/
+/* HUF buffer bounds */
+#define HUF_CTABLEBOUND 129
+#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */
+#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
+
+/* static allocation of HUF's Compression Table */
+/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */
+typedef size_t HUF_CElt; /* consider it an incomplete type */
+#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */
+#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t))
+#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
+ HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */
+
+/* static allocation of HUF's DTable */
+typedef U32 HUF_DTable;
+#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
+#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
+ HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
+#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
+ HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
+
+
+/* ****************************************
+* Advanced decompression functions
+******************************************/
+
+/**
+ * Huffman flags bitset.
+ * For all flags, 0 is the default value.
+ */
+typedef enum {
+ /**
+ * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime.
+ * Otherwise: Ignored.
+ */
+ HUF_flags_bmi2 = (1 << 0),
+ /**
+ * If set: Test possible table depths to find the one that produces the smallest header + encoded size.
+ * If unset: Use heuristic to find the table depth.
+ */
+ HUF_flags_optimalDepth = (1 << 1),
+ /**
+ * If set: If the previous table can encode the input, always reuse the previous table.
+ * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output.
+ */
+ HUF_flags_preferRepeat = (1 << 2),
+ /**
+ * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress.
+ * If unset: Always histogram the entire input.
+ */
+ HUF_flags_suspectUncompressible = (1 << 3),
+ /**
+ * If set: Don't use assembly implementations
+ * If unset: Allow using assembly implementations
+ */
+ HUF_flags_disableAsm = (1 << 4),
+ /**
+ * If set: Don't use the fast decoding loop, always use the fallback decoding loop.
+ * If unset: Use the fast decoding loop when possible.
+ */
+ HUF_flags_disableFast = (1 << 5)
+} HUF_flags_e;
+
+
+/* ****************************************
+ * HUF detailed API
+ * ****************************************/
+#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
+
+/*! HUF_compress() does the following:
+ * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
+ * 2. (optional) refine tableLog using HUF_optimalTableLog()
+ * 3. build Huffman table from count using HUF_buildCTable()
+ * 4. save Huffman table to memory buffer using HUF_writeCTable()
+ * 5. encode the data stream using HUF_compress4X_usingCTable()
+ *
+ * The following API allows targeting specific sub-functions for advanced tasks.
+ * For example, it's possible to compress several blocks using the same 'CTable',
+ * or to save and regenerate 'CTable' using external methods.
+ */
+unsigned HUF_minTableLog(unsigned symbolCardinality);
+unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue);
+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace,
+ size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */
+size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
+size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
+int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
+
+typedef enum {
+ HUF_repeat_none, /**< Cannot use the previous table */
+ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
+ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
+ } HUF_repeat;
+
+/** HUF_compress4X_repeat() :
+ * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+ * If it uses hufTable it does not modify hufTable or repeat.
+ * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+ * If preferRepeat then the old table will always be used if valid.
+ * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
+size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned tableLog,
+ void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
+ HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
+
+/** HUF_buildCTable_wksp() :
+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
+ * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
+ */
+#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192)
+#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
+size_t HUF_buildCTable_wksp (HUF_CElt* tree,
+ const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
+ void* workSpace, size_t wkspSize);
+
+/*! HUF_readStats() :
+ * Read compact Huffman tree, saved by HUF_writeCTable().
+ * `huffWeight` is destination buffer.
+ * @return : size read from `src` , or an error Code .
+ * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
+size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
+ U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize);
+
+/*! HUF_readStats_wksp() :
+ * Same as HUF_readStats() but takes an external workspace which must be
+ * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE.
+ * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
+ */
+#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1)
+#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned))
+size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
+ U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
+ const void* src, size_t srcSize,
+ void* workspace, size_t wkspSize,
+ int flags);
+
+/** HUF_readCTable() :
+ * Loading a CTable saved with HUF_writeCTable() */
+size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights);
+
+/** HUF_getNbBitsFromCTable() :
+ * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
+ * Note 1 : is not inlined, as HUF_CElt definition is private */
+U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue);
+
+/*
+ * HUF_decompress() does the following:
+ * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
+ * 2. build Huffman table from save, using HUF_readDTableX?()
+ * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
+ */
+
+/** HUF_selectDecoder() :
+ * Tells which decoder is likely to decode faster,
+ * based on a set of pre-computed metrics.
+ * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
+ * Assumption : 0 < dstSize <= 128 KB */
+U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
+
+/**
+ * The minimum workspace size for the `workSpace` used in
+ * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
+ *
+ * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
+ * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
+ * Buffer overflow errors may potentially occur if code modifications result in
+ * a required workspace size greater than that specified in the following
+ * macro.
+ */
+#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
+#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
+
+
+/* ====================== */
+/* single stream variants */
+/* ====================== */
+
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
+/** HUF_compress1X_repeat() :
+ * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+ * If it uses hufTable it does not modify hufTable or repeat.
+ * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+ * If preferRepeat then the old table will always be used if valid.
+ * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
+size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned tableLog,
+ void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
+ HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
+
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
+#ifndef HUF_FORCE_DECOMPRESS_X1
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */
+#endif
+
+/* BMI2 variants.
+ * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
+ */
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
+#ifndef HUF_FORCE_DECOMPRESS_X2
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
+#endif
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
+#ifndef HUF_FORCE_DECOMPRESS_X2
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
+#endif
+#ifndef HUF_FORCE_DECOMPRESS_X1
+size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
+#endif
+
+#endif /* HUF_H_298734234 */
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/contrib/zstd/huf_compress.c b/contrib/zstd/huf_compress.c
new file mode 100644
index 0000000..b5f0917
--- /dev/null
+++ b/contrib/zstd/huf_compress.c
@@ -0,0 +1,1435 @@
+/* ******************************************************************
+ * Huffman encoder, part of New Generation Entropy library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* **************************************************************
+* Compiler specifics
+****************************************************************/
+#ifdef _MSC_VER /* Visual Studio */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+#endif
+
+
+/* **************************************************************
+* Includes
+****************************************************************/
+#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */
+#include "compiler.h"
+#include "bitstream.h"
+#include "hist.h"
+#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
+#include "fse.h" /* header compression */
+#include "huf.h"
+#include "error_private.h"
+#include "bits.h" /* ZSTD_highbit32 */
+
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define HUF_isError ERR_isError
+#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
+
+
+/* **************************************************************
+* Required declarations
+****************************************************************/
+typedef struct nodeElt_s {
+ U32 count;
+ U16 parent;
+ BYTE byte;
+ BYTE nbBits;
+} nodeElt;
+
+
+/* **************************************************************
+* Debug Traces
+****************************************************************/
+
+#if DEBUGLEVEL >= 2
+
+static size_t showU32(const U32* arr, size_t size)
+{
+ size_t u;
+ for (u=0; u<size; u++) {
+ RAWLOG(6, " %u", arr[u]); (void)arr;
+ }
+ RAWLOG(6, " \n");
+ return size;
+}
+
+static size_t HUF_getNbBits(HUF_CElt elt);
+
+static size_t showCTableBits(const HUF_CElt* ctable, size_t size)
+{
+ size_t u;
+ for (u=0; u<size; u++) {
+ RAWLOG(6, " %zu", HUF_getNbBits(ctable[u])); (void)ctable;
+ }
+ RAWLOG(6, " \n");
+ return size;
+
+}
+
+static size_t showHNodeSymbols(const nodeElt* hnode, size_t size)
+{
+ size_t u;
+ for (u=0; u<size; u++) {
+ RAWLOG(6, " %u", hnode[u].byte); (void)hnode;
+ }
+ RAWLOG(6, " \n");
+ return size;
+}
+
+static size_t showHNodeBits(const nodeElt* hnode, size_t size)
+{
+ size_t u;
+ for (u=0; u<size; u++) {
+ RAWLOG(6, " %u", hnode[u].nbBits); (void)hnode;
+ }
+ RAWLOG(6, " \n");
+ return size;
+}
+
+#endif
+
+
+/* *******************************************************
+* HUF : Huffman block compression
+*********************************************************/
+#define HUF_WORKSPACE_MAX_ALIGNMENT 8
+
+static void* HUF_alignUpWorkspace(void* workspace, size_t* workspaceSizePtr, size_t align)
+{
+ size_t const mask = align - 1;
+ size_t const rem = (size_t)workspace & mask;
+ size_t const add = (align - rem) & mask;
+ BYTE* const aligned = (BYTE*)workspace + add;
+ assert((align & (align - 1)) == 0); /* pow 2 */
+ assert(align <= HUF_WORKSPACE_MAX_ALIGNMENT);
+ if (*workspaceSizePtr >= add) {
+ assert(add < align);
+ assert(((size_t)aligned & mask) == 0);
+ *workspaceSizePtr -= add;
+ return aligned;
+ } else {
+ *workspaceSizePtr = 0;
+ return NULL;
+ }
+}
+
+
+/* HUF_compressWeights() :
+ * Same as FSE_compress(), but dedicated to huff0's weights compression.
+ * The use case needs much less stack memory.
+ * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
+ */
+#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
+
+typedef struct {
+ FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
+ U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)];
+ unsigned count[HUF_TABLELOG_MAX+1];
+ S16 norm[HUF_TABLELOG_MAX+1];
+} HUF_CompressWeightsWksp;
+
+static size_t
+HUF_compressWeights(void* dst, size_t dstSize,
+ const void* weightTable, size_t wtSize,
+ void* workspace, size_t workspaceSize)
+{
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* op = ostart;
+ BYTE* const oend = ostart + dstSize;
+
+ unsigned maxSymbolValue = HUF_TABLELOG_MAX;
+ U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
+ HUF_CompressWeightsWksp* wksp = (HUF_CompressWeightsWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32));
+
+ if (workspaceSize < sizeof(HUF_CompressWeightsWksp)) return ERROR(GENERIC);
+
+ /* init conditions */
+ if (wtSize <= 1) return 0; /* Not compressible */
+
+ /* Scan input and build symbol stats */
+ { unsigned const maxCount = HIST_count_simple(wksp->count, &maxSymbolValue, weightTable, wtSize); /* never fails */
+ if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */
+ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
+ }
+
+ tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
+ CHECK_F( FSE_normalizeCount(wksp->norm, tableLog, wksp->count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) );
+
+ /* Write table description header */
+ { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), wksp->norm, maxSymbolValue, tableLog) );
+ op += hSize;
+ }
+
+ /* Compress */
+ CHECK_F( FSE_buildCTable_wksp(wksp->CTable, wksp->norm, maxSymbolValue, tableLog, wksp->scratchBuffer, sizeof(wksp->scratchBuffer)) );
+ { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, wksp->CTable) );
+ if (cSize == 0) return 0; /* not enough space for compressed data */
+ op += cSize;
+ }
+
+ return (size_t)(op-ostart);
+}
+
+static size_t HUF_getNbBits(HUF_CElt elt)
+{
+ return elt & 0xFF;
+}
+
+static size_t HUF_getNbBitsFast(HUF_CElt elt)
+{
+ return elt;
+}
+
+static size_t HUF_getValue(HUF_CElt elt)
+{
+ return elt & ~(size_t)0xFF;
+}
+
+static size_t HUF_getValueFast(HUF_CElt elt)
+{
+ return elt;
+}
+
+static void HUF_setNbBits(HUF_CElt* elt, size_t nbBits)
+{
+ assert(nbBits <= HUF_TABLELOG_ABSOLUTEMAX);
+ *elt = nbBits;
+}
+
+static void HUF_setValue(HUF_CElt* elt, size_t value)
+{
+ size_t const nbBits = HUF_getNbBits(*elt);
+ if (nbBits > 0) {
+ assert((value >> nbBits) == 0);
+ *elt |= value << (sizeof(HUF_CElt) * 8 - nbBits);
+ }
+}
+
+typedef struct {
+ HUF_CompressWeightsWksp wksp;
+ BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
+ BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
+} HUF_WriteCTableWksp;
+
+size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
+ const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog,
+ void* workspace, size_t workspaceSize)
+{
+ HUF_CElt const* const ct = CTable + 1;
+ BYTE* op = (BYTE*)dst;
+ U32 n;
+ HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32));
+
+ HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp));
+
+ /* check conditions */
+ if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC);
+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
+
+ /* convert to weight */
+ wksp->bitsToWeight[0] = 0;
+ for (n=1; n<huffLog+1; n++)
+ wksp->bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
+ for (n=0; n<maxSymbolValue; n++)
+ wksp->huffWeight[n] = wksp->bitsToWeight[HUF_getNbBits(ct[n])];
+
+ /* attempt weights compression by FSE */
+ if (maxDstSize < 1) return ERROR(dstSize_tooSmall);
+ { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) );
+ if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */
+ op[0] = (BYTE)hSize;
+ return hSize+1;
+ } }
+
+ /* write raw values as 4-bits (max : 15) */
+ if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
+ if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
+ op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
+ wksp->huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
+ for (n=0; n<maxSymbolValue; n+=2)
+ op[(n/2)+1] = (BYTE)((wksp->huffWeight[n] << 4) + wksp->huffWeight[n+1]);
+ return ((maxSymbolValue+1)/2) + 1;
+}
+
+
+size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights)
+{
+ BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
+ U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
+ U32 tableLog = 0;
+ U32 nbSymbols = 0;
+ HUF_CElt* const ct = CTable + 1;
+
+ /* get symbol weights */
+ CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize));
+ *hasZeroWeights = (rankVal[0] > 0);
+
+ /* check result */
+ if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
+ if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall);
+
+ CTable[0] = tableLog;
+
+ /* Prepare base value per rank */
+ { U32 n, nextRankStart = 0;
+ for (n=1; n<=tableLog; n++) {
+ U32 curr = nextRankStart;
+ nextRankStart += (rankVal[n] << (n-1));
+ rankVal[n] = curr;
+ } }
+
+ /* fill nbBits */
+ { U32 n; for (n=0; n<nbSymbols; n++) {
+ const U32 w = huffWeight[n];
+ HUF_setNbBits(ct + n, (BYTE)(tableLog + 1 - w) & -(w != 0));
+ } }
+
+ /* fill val */
+ { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
+ U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
+ { U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[HUF_getNbBits(ct[n])]++; }
+ /* determine stating value per rank */
+ valPerRank[tableLog+1] = 0; /* for w==0 */
+ { U16 min = 0;
+ U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
+ valPerRank[n] = min; /* get starting value within each rank */
+ min += nbPerRank[n];
+ min >>= 1;
+ } }
+ /* assign value within rank, symbol order */
+ { U32 n; for (n=0; n<nbSymbols; n++) HUF_setValue(ct + n, valPerRank[HUF_getNbBits(ct[n])]++); }
+ }
+
+ *maxSymbolValuePtr = nbSymbols - 1;
+ return readSize;
+}
+
+U32 HUF_getNbBitsFromCTable(HUF_CElt const* CTable, U32 symbolValue)
+{
+ const HUF_CElt* const ct = CTable + 1;
+ assert(symbolValue <= HUF_SYMBOLVALUE_MAX);
+ return (U32)HUF_getNbBits(ct[symbolValue]);
+}
+
+
+/**
+ * HUF_setMaxHeight():
+ * Try to enforce @targetNbBits on the Huffman tree described in @huffNode.
+ *
+ * It attempts to convert all nodes with nbBits > @targetNbBits
+ * to employ @targetNbBits instead. Then it adjusts the tree
+ * so that it remains a valid canonical Huffman tree.
+ *
+ * @pre The sum of the ranks of each symbol == 2^largestBits,
+ * where largestBits == huffNode[lastNonNull].nbBits.
+ * @post The sum of the ranks of each symbol == 2^largestBits,
+ * where largestBits is the return value (expected <= targetNbBits).
+ *
+ * @param huffNode The Huffman tree modified in place to enforce targetNbBits.
+ * It's presumed sorted, from most frequent to rarest symbol.
+ * @param lastNonNull The symbol with the lowest count in the Huffman tree.
+ * @param targetNbBits The allowed number of bits, which the Huffman tree
+ * may not respect. After this function the Huffman tree will
+ * respect targetNbBits.
+ * @return The maximum number of bits of the Huffman tree after adjustment.
+ */
+static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits)
+{
+ const U32 largestBits = huffNode[lastNonNull].nbBits;
+ /* early exit : no elt > targetNbBits, so the tree is already valid. */
+ if (largestBits <= targetNbBits) return largestBits;
+
+ DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits);
+
+ /* there are several too large elements (at least >= 2) */
+ { int totalCost = 0;
+ const U32 baseCost = 1 << (largestBits - targetNbBits);
+ int n = (int)lastNonNull;
+
+ /* Adjust any ranks > targetNbBits to targetNbBits.
+ * Compute totalCost, which is how far the sum of the ranks is
+ * we are over 2^largestBits after adjust the offending ranks.
+ */
+ while (huffNode[n].nbBits > targetNbBits) {
+ totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
+ huffNode[n].nbBits = (BYTE)targetNbBits;
+ n--;
+ }
+ /* n stops at huffNode[n].nbBits <= targetNbBits */
+ assert(huffNode[n].nbBits <= targetNbBits);
+ /* n end at index of smallest symbol using < targetNbBits */
+ while (huffNode[n].nbBits == targetNbBits) --n;
+
+ /* renorm totalCost from 2^largestBits to 2^targetNbBits
+ * note : totalCost is necessarily a multiple of baseCost */
+ assert(((U32)totalCost & (baseCost - 1)) == 0);
+ totalCost >>= (largestBits - targetNbBits);
+ assert(totalCost > 0);
+
+ /* repay normalized cost */
+ { U32 const noSymbol = 0xF0F0F0F0;
+ U32 rankLast[HUF_TABLELOG_MAX+2];
+
+ /* Get pos of last (smallest = lowest cum. count) symbol per rank */
+ ZSTD_memset(rankLast, 0xF0, sizeof(rankLast));
+ { U32 currentNbBits = targetNbBits;
+ int pos;
+ for (pos=n ; pos >= 0; pos--) {
+ if (huffNode[pos].nbBits >= currentNbBits) continue;
+ currentNbBits = huffNode[pos].nbBits; /* < targetNbBits */
+ rankLast[targetNbBits-currentNbBits] = (U32)pos;
+ } }
+
+ while (totalCost > 0) {
+ /* Try to reduce the next power of 2 above totalCost because we
+ * gain back half the rank.
+ */
+ U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1;
+ for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
+ U32 const highPos = rankLast[nBitsToDecrease];
+ U32 const lowPos = rankLast[nBitsToDecrease-1];
+ if (highPos == noSymbol) continue;
+ /* Decrease highPos if no symbols of lowPos or if it is
+ * not cheaper to remove 2 lowPos than highPos.
+ */
+ if (lowPos == noSymbol) break;
+ { U32 const highTotal = huffNode[highPos].count;
+ U32 const lowTotal = 2 * huffNode[lowPos].count;
+ if (highTotal <= lowTotal) break;
+ } }
+ /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
+ assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1);
+ /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
+ while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol))
+ nBitsToDecrease++;
+ assert(rankLast[nBitsToDecrease] != noSymbol);
+ /* Increase the number of bits to gain back half the rank cost. */
+ totalCost -= 1 << (nBitsToDecrease-1);
+ huffNode[rankLast[nBitsToDecrease]].nbBits++;
+
+ /* Fix up the new rank.
+ * If the new rank was empty, this symbol is now its smallest.
+ * Otherwise, this symbol will be the largest in the new rank so no adjustment.
+ */
+ if (rankLast[nBitsToDecrease-1] == noSymbol)
+ rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease];
+ /* Fix up the old rank.
+ * If the symbol was at position 0, meaning it was the highest weight symbol in the tree,
+ * it must be the only symbol in its rank, so the old rank now has no symbols.
+ * Otherwise, since the Huffman nodes are sorted by count, the previous position is now
+ * the smallest node in the rank. If the previous position belongs to a different rank,
+ * then the rank is now empty.
+ */
+ if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
+ rankLast[nBitsToDecrease] = noSymbol;
+ else {
+ rankLast[nBitsToDecrease]--;
+ if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease)
+ rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
+ }
+ } /* while (totalCost > 0) */
+
+ /* If we've removed too much weight, then we have to add it back.
+ * To avoid overshooting again, we only adjust the smallest rank.
+ * We take the largest nodes from the lowest rank 0 and move them
+ * to rank 1. There's guaranteed to be enough rank 0 symbols because
+ * TODO.
+ */
+ while (totalCost < 0) { /* Sometimes, cost correction overshoot */
+ /* special case : no rank 1 symbol (using targetNbBits-1);
+ * let's create one from largest rank 0 (using targetNbBits).
+ */
+ if (rankLast[1] == noSymbol) {
+ while (huffNode[n].nbBits == targetNbBits) n--;
+ huffNode[n+1].nbBits--;
+ assert(n >= 0);
+ rankLast[1] = (U32)(n+1);
+ totalCost++;
+ continue;
+ }
+ huffNode[ rankLast[1] + 1 ].nbBits--;
+ rankLast[1]++;
+ totalCost ++;
+ }
+ } /* repay normalized cost */
+ } /* there are several too large elements (at least >= 2) */
+
+ return targetNbBits;
+}
+
+typedef struct {
+ U16 base;
+ U16 curr;
+} rankPos;
+
+typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)];
+
+/* Number of buckets available for HUF_sort() */
+#define RANK_POSITION_TABLE_SIZE 192
+
+typedef struct {
+ huffNodeTable huffNodeTbl;
+ rankPos rankPosition[RANK_POSITION_TABLE_SIZE];
+} HUF_buildCTable_wksp_tables;
+
+/* RANK_POSITION_DISTINCT_COUNT_CUTOFF == Cutoff point in HUF_sort() buckets for which we use log2 bucketing.
+ * Strategy is to use as many buckets as possible for representing distinct
+ * counts while using the remainder to represent all "large" counts.
+ *
+ * To satisfy this requirement for 192 buckets, we can do the following:
+ * Let buckets 0-166 represent distinct counts of [0, 166]
+ * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing.
+ */
+#define RANK_POSITION_MAX_COUNT_LOG 32
+#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */)
+#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */)
+
+/* Return the appropriate bucket index for a given count. See definition of
+ * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy.
+ */
+static U32 HUF_getIndex(U32 const count) {
+ return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF)
+ ? count
+ : ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN;
+}
+
+/* Helper swap function for HUF_quickSortPartition() */
+static void HUF_swapNodes(nodeElt* a, nodeElt* b) {
+ nodeElt tmp = *a;
+ *a = *b;
+ *b = tmp;
+}
+
+/* Returns 0 if the huffNode array is not sorted by descending count */
+MEM_STATIC int HUF_isSorted(nodeElt huffNode[], U32 const maxSymbolValue1) {
+ U32 i;
+ for (i = 1; i < maxSymbolValue1; ++i) {
+ if (huffNode[i].count > huffNode[i-1].count) {
+ return 0;
+ }
+ }
+ return 1;
+}
+
+/* Insertion sort by descending order */
+HINT_INLINE void HUF_insertionSort(nodeElt huffNode[], int const low, int const high) {
+ int i;
+ int const size = high-low+1;
+ huffNode += low;
+ for (i = 1; i < size; ++i) {
+ nodeElt const key = huffNode[i];
+ int j = i - 1;
+ while (j >= 0 && huffNode[j].count < key.count) {
+ huffNode[j + 1] = huffNode[j];
+ j--;
+ }
+ huffNode[j + 1] = key;
+ }
+}
+
+/* Pivot helper function for quicksort. */
+static int HUF_quickSortPartition(nodeElt arr[], int const low, int const high) {
+ /* Simply select rightmost element as pivot. "Better" selectors like
+ * median-of-three don't experimentally appear to have any benefit.
+ */
+ U32 const pivot = arr[high].count;
+ int i = low - 1;
+ int j = low;
+ for ( ; j < high; j++) {
+ if (arr[j].count > pivot) {
+ i++;
+ HUF_swapNodes(&arr[i], &arr[j]);
+ }
+ }
+ HUF_swapNodes(&arr[i + 1], &arr[high]);
+ return i + 1;
+}
+
+/* Classic quicksort by descending with partially iterative calls
+ * to reduce worst case callstack size.
+ */
+static void HUF_simpleQuickSort(nodeElt arr[], int low, int high) {
+ int const kInsertionSortThreshold = 8;
+ if (high - low < kInsertionSortThreshold) {
+ HUF_insertionSort(arr, low, high);
+ return;
+ }
+ while (low < high) {
+ int const idx = HUF_quickSortPartition(arr, low, high);
+ if (idx - low < high - idx) {
+ HUF_simpleQuickSort(arr, low, idx - 1);
+ low = idx + 1;
+ } else {
+ HUF_simpleQuickSort(arr, idx + 1, high);
+ high = idx - 1;
+ }
+ }
+}
+
+/**
+ * HUF_sort():
+ * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order.
+ * This is a typical bucket sorting strategy that uses either quicksort or insertion sort to sort each bucket.
+ *
+ * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled.
+ * Must have (maxSymbolValue + 1) entries.
+ * @param[in] count Histogram of the symbols.
+ * @param[in] maxSymbolValue Maximum symbol value.
+ * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries.
+ */
+static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSymbolValue, rankPos rankPosition[]) {
+ U32 n;
+ U32 const maxSymbolValue1 = maxSymbolValue+1;
+
+ /* Compute base and set curr to base.
+ * For symbol s let lowerRank = HUF_getIndex(count[n]) and rank = lowerRank + 1.
+ * See HUF_getIndex to see bucketing strategy.
+ * We attribute each symbol to lowerRank's base value, because we want to know where
+ * each rank begins in the output, so for rank R we want to count ranks R+1 and above.
+ */
+ ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE);
+ for (n = 0; n < maxSymbolValue1; ++n) {
+ U32 lowerRank = HUF_getIndex(count[n]);
+ assert(lowerRank < RANK_POSITION_TABLE_SIZE - 1);
+ rankPosition[lowerRank].base++;
+ }
+
+ assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0);
+ /* Set up the rankPosition table */
+ for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) {
+ rankPosition[n-1].base += rankPosition[n].base;
+ rankPosition[n-1].curr = rankPosition[n-1].base;
+ }
+
+ /* Insert each symbol into their appropriate bucket, setting up rankPosition table. */
+ for (n = 0; n < maxSymbolValue1; ++n) {
+ U32 const c = count[n];
+ U32 const r = HUF_getIndex(c) + 1;
+ U32 const pos = rankPosition[r].curr++;
+ assert(pos < maxSymbolValue1);
+ huffNode[pos].count = c;
+ huffNode[pos].byte = (BYTE)n;
+ }
+
+ /* Sort each bucket. */
+ for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) {
+ int const bucketSize = rankPosition[n].curr - rankPosition[n].base;
+ U32 const bucketStartIdx = rankPosition[n].base;
+ if (bucketSize > 1) {
+ assert(bucketStartIdx < maxSymbolValue1);
+ HUF_simpleQuickSort(huffNode + bucketStartIdx, 0, bucketSize-1);
+ }
+ }
+
+ assert(HUF_isSorted(huffNode, maxSymbolValue1));
+}
+
+
+/** HUF_buildCTable_wksp() :
+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables).
+ */
+#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
+
+/* HUF_buildTree():
+ * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree.
+ *
+ * @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array.
+ * @param maxSymbolValue The maximum symbol value.
+ * @return The smallest node in the Huffman tree (by count).
+ */
+static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue)
+{
+ nodeElt* const huffNode0 = huffNode - 1;
+ int nonNullRank;
+ int lowS, lowN;
+ int nodeNb = STARTNODE;
+ int n, nodeRoot;
+ DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1);
+ /* init for parents */
+ nonNullRank = (int)maxSymbolValue;
+ while(huffNode[nonNullRank].count == 0) nonNullRank--;
+ lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
+ huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
+ huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb;
+ nodeNb++; lowS-=2;
+ for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
+ huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */
+
+ /* create parents */
+ while (nodeNb <= nodeRoot) {
+ int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
+ int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
+ huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
+ huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb;
+ nodeNb++;
+ }
+
+ /* distribute weights (unlimited tree height) */
+ huffNode[nodeRoot].nbBits = 0;
+ for (n=nodeRoot-1; n>=STARTNODE; n--)
+ huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
+ for (n=0; n<=nonNullRank; n++)
+ huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
+
+ DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1));
+
+ return nonNullRank;
+}
+
+/**
+ * HUF_buildCTableFromTree():
+ * Build the CTable given the Huffman tree in huffNode.
+ *
+ * @param[out] CTable The output Huffman CTable.
+ * @param huffNode The Huffman tree.
+ * @param nonNullRank The last and smallest node in the Huffman tree.
+ * @param maxSymbolValue The maximum symbol value.
+ * @param maxNbBits The exact maximum number of bits used in the Huffman tree.
+ */
+static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits)
+{
+ HUF_CElt* const ct = CTable + 1;
+ /* fill result into ctable (val, nbBits) */
+ int n;
+ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
+ U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
+ int const alphabetSize = (int)(maxSymbolValue + 1);
+ for (n=0; n<=nonNullRank; n++)
+ nbPerRank[huffNode[n].nbBits]++;
+ /* determine starting value per rank */
+ { U16 min = 0;
+ for (n=(int)maxNbBits; n>0; n--) {
+ valPerRank[n] = min; /* get starting value within each rank */
+ min += nbPerRank[n];
+ min >>= 1;
+ } }
+ for (n=0; n<alphabetSize; n++)
+ HUF_setNbBits(ct + huffNode[n].byte, huffNode[n].nbBits); /* push nbBits per symbol, symbol order */
+ for (n=0; n<alphabetSize; n++)
+ HUF_setValue(ct + n, valPerRank[HUF_getNbBits(ct[n])]++); /* assign value within rank, symbol order */
+ CTable[0] = maxNbBits;
+}
+
+size_t
+HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
+ void* workSpace, size_t wkspSize)
+{
+ HUF_buildCTable_wksp_tables* const wksp_tables =
+ (HUF_buildCTable_wksp_tables*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(U32));
+ nodeElt* const huffNode0 = wksp_tables->huffNodeTbl;
+ nodeElt* const huffNode = huffNode0+1;
+ int nonNullRank;
+
+ HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables));
+
+ DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1);
+
+ /* safety checks */
+ if (wkspSize < sizeof(HUF_buildCTable_wksp_tables))
+ return ERROR(workSpace_tooSmall);
+ if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
+ return ERROR(maxSymbolValue_tooLarge);
+ ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable));
+
+ /* sort, decreasing order */
+ HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition);
+ DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1));
+
+ /* build tree */
+ nonNullRank = HUF_buildTree(huffNode, maxSymbolValue);
+
+ /* determine and enforce maxTableLog */
+ maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits);
+ if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
+
+ HUF_buildCTableFromTree(CTable, huffNode, nonNullRank, maxSymbolValue, maxNbBits);
+
+ return maxNbBits;
+}
+
+size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue)
+{
+ HUF_CElt const* ct = CTable + 1;
+ size_t nbBits = 0;
+ int s;
+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
+ nbBits += HUF_getNbBits(ct[s]) * count[s];
+ }
+ return nbBits >> 3;
+}
+
+int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
+ HUF_CElt const* ct = CTable + 1;
+ int bad = 0;
+ int s;
+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
+ bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0);
+ }
+ return !bad;
+}
+
+size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
+
+/** HUF_CStream_t:
+ * Huffman uses its own BIT_CStream_t implementation.
+ * There are three major differences from BIT_CStream_t:
+ * 1. HUF_addBits() takes a HUF_CElt (size_t) which is
+ * the pair (nbBits, value) in the format:
+ * format:
+ * - Bits [0, 4) = nbBits
+ * - Bits [4, 64 - nbBits) = 0
+ * - Bits [64 - nbBits, 64) = value
+ * 2. The bitContainer is built from the upper bits and
+ * right shifted. E.g. to add a new value of N bits
+ * you right shift the bitContainer by N, then or in
+ * the new value into the N upper bits.
+ * 3. The bitstream has two bit containers. You can add
+ * bits to the second container and merge them into
+ * the first container.
+ */
+
+#define HUF_BITS_IN_CONTAINER (sizeof(size_t) * 8)
+
+typedef struct {
+ size_t bitContainer[2];
+ size_t bitPos[2];
+
+ BYTE* startPtr;
+ BYTE* ptr;
+ BYTE* endPtr;
+} HUF_CStream_t;
+
+/**! HUF_initCStream():
+ * Initializes the bitstream.
+ * @returns 0 or an error code.
+ */
+static size_t HUF_initCStream(HUF_CStream_t* bitC,
+ void* startPtr, size_t dstCapacity)
+{
+ ZSTD_memset(bitC, 0, sizeof(*bitC));
+ bitC->startPtr = (BYTE*)startPtr;
+ bitC->ptr = bitC->startPtr;
+ bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer[0]);
+ if (dstCapacity <= sizeof(bitC->bitContainer[0])) return ERROR(dstSize_tooSmall);
+ return 0;
+}
+
+/*! HUF_addBits():
+ * Adds the symbol stored in HUF_CElt elt to the bitstream.
+ *
+ * @param elt The element we're adding. This is a (nbBits, value) pair.
+ * See the HUF_CStream_t docs for the format.
+ * @param idx Insert into the bitstream at this idx.
+ * @param kFast This is a template parameter. If the bitstream is guaranteed
+ * to have at least 4 unused bits after this call it may be 1,
+ * otherwise it must be 0. HUF_addBits() is faster when fast is set.
+ */
+FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int idx, int kFast)
+{
+ assert(idx <= 1);
+ assert(HUF_getNbBits(elt) <= HUF_TABLELOG_ABSOLUTEMAX);
+ /* This is efficient on x86-64 with BMI2 because shrx
+ * only reads the low 6 bits of the register. The compiler
+ * knows this and elides the mask. When fast is set,
+ * every operation can use the same value loaded from elt.
+ */
+ bitC->bitContainer[idx] >>= HUF_getNbBits(elt);
+ bitC->bitContainer[idx] |= kFast ? HUF_getValueFast(elt) : HUF_getValue(elt);
+ /* We only read the low 8 bits of bitC->bitPos[idx] so it
+ * doesn't matter that the high bits have noise from the value.
+ */
+ bitC->bitPos[idx] += HUF_getNbBitsFast(elt);
+ assert((bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER);
+ /* The last 4-bits of elt are dirty if fast is set,
+ * so we must not be overwriting bits that have already been
+ * inserted into the bit container.
+ */
+#if DEBUGLEVEL >= 1
+ {
+ size_t const nbBits = HUF_getNbBits(elt);
+ size_t const dirtyBits = nbBits == 0 ? 0 : ZSTD_highbit32((U32)nbBits) + 1;
+ (void)dirtyBits;
+ /* Middle bits are 0. */
+ assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0);
+ /* We didn't overwrite any bits in the bit container. */
+ assert(!kFast || (bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER);
+ (void)dirtyBits;
+ }
+#endif
+}
+
+FORCE_INLINE_TEMPLATE void HUF_zeroIndex1(HUF_CStream_t* bitC)
+{
+ bitC->bitContainer[1] = 0;
+ bitC->bitPos[1] = 0;
+}
+
+/*! HUF_mergeIndex1() :
+ * Merges the bit container @ index 1 into the bit container @ index 0
+ * and zeros the bit container @ index 1.
+ */
+FORCE_INLINE_TEMPLATE void HUF_mergeIndex1(HUF_CStream_t* bitC)
+{
+ assert((bitC->bitPos[1] & 0xFF) < HUF_BITS_IN_CONTAINER);
+ bitC->bitContainer[0] >>= (bitC->bitPos[1] & 0xFF);
+ bitC->bitContainer[0] |= bitC->bitContainer[1];
+ bitC->bitPos[0] += bitC->bitPos[1];
+ assert((bitC->bitPos[0] & 0xFF) <= HUF_BITS_IN_CONTAINER);
+}
+
+/*! HUF_flushBits() :
+* Flushes the bits in the bit container @ index 0.
+*
+* @post bitPos will be < 8.
+* @param kFast If kFast is set then we must know a-priori that
+* the bit container will not overflow.
+*/
+FORCE_INLINE_TEMPLATE void HUF_flushBits(HUF_CStream_t* bitC, int kFast)
+{
+ /* The upper bits of bitPos are noisy, so we must mask by 0xFF. */
+ size_t const nbBits = bitC->bitPos[0] & 0xFF;
+ size_t const nbBytes = nbBits >> 3;
+ /* The top nbBits bits of bitContainer are the ones we need. */
+ size_t const bitContainer = bitC->bitContainer[0] >> (HUF_BITS_IN_CONTAINER - nbBits);
+ /* Mask bitPos to account for the bytes we consumed. */
+ bitC->bitPos[0] &= 7;
+ assert(nbBits > 0);
+ assert(nbBits <= sizeof(bitC->bitContainer[0]) * 8);
+ assert(bitC->ptr <= bitC->endPtr);
+ MEM_writeLEST(bitC->ptr, bitContainer);
+ bitC->ptr += nbBytes;
+ assert(!kFast || bitC->ptr <= bitC->endPtr);
+ if (!kFast && bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
+ /* bitContainer doesn't need to be modified because the leftover
+ * bits are already the top bitPos bits. And we don't care about
+ * noise in the lower values.
+ */
+}
+
+/*! HUF_endMark()
+ * @returns The Huffman stream end mark: A 1-bit value = 1.
+ */
+static HUF_CElt HUF_endMark(void)
+{
+ HUF_CElt endMark;
+ HUF_setNbBits(&endMark, 1);
+ HUF_setValue(&endMark, 1);
+ return endMark;
+}
+
+/*! HUF_closeCStream() :
+ * @return Size of CStream, in bytes,
+ * or 0 if it could not fit into dstBuffer */
+static size_t HUF_closeCStream(HUF_CStream_t* bitC)
+{
+ HUF_addBits(bitC, HUF_endMark(), /* idx */ 0, /* kFast */ 0);
+ HUF_flushBits(bitC, /* kFast */ 0);
+ {
+ size_t const nbBits = bitC->bitPos[0] & 0xFF;
+ if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
+ return (size_t)(bitC->ptr - bitC->startPtr) + (nbBits > 0);
+ }
+}
+
+FORCE_INLINE_TEMPLATE void
+HUF_encodeSymbol(HUF_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable, int idx, int fast)
+{
+ HUF_addBits(bitCPtr, CTable[symbol], idx, fast);
+}
+
+FORCE_INLINE_TEMPLATE void
+HUF_compress1X_usingCTable_internal_body_loop(HUF_CStream_t* bitC,
+ const BYTE* ip, size_t srcSize,
+ const HUF_CElt* ct,
+ int kUnroll, int kFastFlush, int kLastFast)
+{
+ /* Join to kUnroll */
+ int n = (int)srcSize;
+ int rem = n % kUnroll;
+ if (rem > 0) {
+ for (; rem > 0; --rem) {
+ HUF_encodeSymbol(bitC, ip[--n], ct, 0, /* fast */ 0);
+ }
+ HUF_flushBits(bitC, kFastFlush);
+ }
+ assert(n % kUnroll == 0);
+
+ /* Join to 2 * kUnroll */
+ if (n % (2 * kUnroll)) {
+ int u;
+ for (u = 1; u < kUnroll; ++u) {
+ HUF_encodeSymbol(bitC, ip[n - u], ct, 0, 1);
+ }
+ HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, 0, kLastFast);
+ HUF_flushBits(bitC, kFastFlush);
+ n -= kUnroll;
+ }
+ assert(n % (2 * kUnroll) == 0);
+
+ for (; n>0; n-= 2 * kUnroll) {
+ /* Encode kUnroll symbols into the bitstream @ index 0. */
+ int u;
+ for (u = 1; u < kUnroll; ++u) {
+ HUF_encodeSymbol(bitC, ip[n - u], ct, /* idx */ 0, /* fast */ 1);
+ }
+ HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, /* idx */ 0, /* fast */ kLastFast);
+ HUF_flushBits(bitC, kFastFlush);
+ /* Encode kUnroll symbols into the bitstream @ index 1.
+ * This allows us to start filling the bit container
+ * without any data dependencies.
+ */
+ HUF_zeroIndex1(bitC);
+ for (u = 1; u < kUnroll; ++u) {
+ HUF_encodeSymbol(bitC, ip[n - kUnroll - u], ct, /* idx */ 1, /* fast */ 1);
+ }
+ HUF_encodeSymbol(bitC, ip[n - kUnroll - kUnroll], ct, /* idx */ 1, /* fast */ kLastFast);
+ /* Merge bitstream @ index 1 into the bitstream @ index 0 */
+ HUF_mergeIndex1(bitC);
+ HUF_flushBits(bitC, kFastFlush);
+ }
+ assert(n == 0);
+
+}
+
+/**
+ * Returns a tight upper bound on the output space needed by Huffman
+ * with 8 bytes buffer to handle over-writes. If the output is at least
+ * this large we don't need to do bounds checks during Huffman encoding.
+ */
+static size_t HUF_tightCompressBound(size_t srcSize, size_t tableLog)
+{
+ return ((srcSize * tableLog) >> 3) + 8;
+}
+
+
+FORCE_INLINE_TEMPLATE size_t
+HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable)
+{
+ U32 const tableLog = (U32)CTable[0];
+ HUF_CElt const* ct = CTable + 1;
+ const BYTE* ip = (const BYTE*) src;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* op = ostart;
+ HUF_CStream_t bitC;
+
+ /* init */
+ if (dstSize < 8) return 0; /* not enough space to compress */
+ { size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op));
+ if (HUF_isError(initErr)) return 0; }
+
+ if (dstSize < HUF_tightCompressBound(srcSize, (size_t)tableLog) || tableLog > 11)
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ MEM_32bits() ? 2 : 4, /* kFast */ 0, /* kLastFast */ 0);
+ else {
+ if (MEM_32bits()) {
+ switch (tableLog) {
+ case 11:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 0);
+ break;
+ case 10: ZSTD_FALLTHROUGH;
+ case 9: ZSTD_FALLTHROUGH;
+ case 8:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 1);
+ break;
+ case 7: ZSTD_FALLTHROUGH;
+ default:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 3, /* kFastFlush */ 1, /* kLastFast */ 1);
+ break;
+ }
+ } else {
+ switch (tableLog) {
+ case 11:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 0);
+ break;
+ case 10:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 1);
+ break;
+ case 9:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 6, /* kFastFlush */ 1, /* kLastFast */ 0);
+ break;
+ case 8:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 7, /* kFastFlush */ 1, /* kLastFast */ 0);
+ break;
+ case 7:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 8, /* kFastFlush */ 1, /* kLastFast */ 0);
+ break;
+ case 6: ZSTD_FALLTHROUGH;
+ default:
+ HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 9, /* kFastFlush */ 1, /* kLastFast */ 1);
+ break;
+ }
+ }
+ }
+ assert(bitC.ptr <= bitC.endPtr);
+
+ return HUF_closeCStream(&bitC);
+}
+
+#if DYNAMIC_BMI2
+
+static BMI2_TARGET_ATTRIBUTE size_t
+HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable)
+{
+ return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
+}
+
+static size_t
+HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable)
+{
+ return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
+}
+
+static size_t
+HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable, const int flags)
+{
+ if (flags & HUF_flags_bmi2) {
+ return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable);
+ }
+ return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable);
+}
+
+#else
+
+static size_t
+HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable, const int flags)
+{
+ (void)flags;
+ return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
+}
+
+#endif
+
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
+{
+ return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
+}
+
+static size_t
+HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ const HUF_CElt* CTable, int flags)
+{
+ size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */
+ const BYTE* ip = (const BYTE*) src;
+ const BYTE* const iend = ip + srcSize;
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* op = ostart;
+
+ if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */
+ if (srcSize < 12) return 0; /* no saving possible : too small input */
+ op += 6; /* jumpTable */
+
+ assert(op <= oend);
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
+ if (cSize == 0 || cSize > 65535) return 0;
+ MEM_writeLE16(ostart, (U16)cSize);
+ op += cSize;
+ }
+
+ ip += segmentSize;
+ assert(op <= oend);
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
+ if (cSize == 0 || cSize > 65535) return 0;
+ MEM_writeLE16(ostart+2, (U16)cSize);
+ op += cSize;
+ }
+
+ ip += segmentSize;
+ assert(op <= oend);
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
+ if (cSize == 0 || cSize > 65535) return 0;
+ MEM_writeLE16(ostart+4, (U16)cSize);
+ op += cSize;
+ }
+
+ ip += segmentSize;
+ assert(op <= oend);
+ assert(ip <= iend);
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) );
+ if (cSize == 0 || cSize > 65535) return 0;
+ op += cSize;
+ }
+
+ return (size_t)(op-ostart);
+}
+
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
+{
+ return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
+}
+
+typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
+
+static size_t HUF_compressCTable_internal(
+ BYTE* const ostart, BYTE* op, BYTE* const oend,
+ const void* src, size_t srcSize,
+ HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags)
+{
+ size_t const cSize = (nbStreams==HUF_singleStream) ?
+ HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) :
+ HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags);
+ if (HUF_isError(cSize)) { return cSize; }
+ if (cSize==0) { return 0; } /* uncompressible */
+ op += cSize;
+ /* check compressibility */
+ assert(op >= ostart);
+ if ((size_t)(op-ostart) >= srcSize-1) { return 0; }
+ return (size_t)(op-ostart);
+}
+
+typedef struct {
+ unsigned count[HUF_SYMBOLVALUE_MAX + 1];
+ HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)];
+ union {
+ HUF_buildCTable_wksp_tables buildCTable_wksp;
+ HUF_WriteCTableWksp writeCTable_wksp;
+ U32 hist_wksp[HIST_WKSP_SIZE_U32];
+ } wksps;
+} HUF_compress_tables_t;
+
+#define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096
+#define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */
+
+unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue)
+{
+ unsigned cardinality = 0;
+ unsigned i;
+
+ for (i = 0; i < maxSymbolValue + 1; i++) {
+ if (count[i] != 0) cardinality += 1;
+ }
+
+ return cardinality;
+}
+
+unsigned HUF_minTableLog(unsigned symbolCardinality)
+{
+ U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1;
+ return minBitsSymbols;
+}
+
+unsigned HUF_optimalTableLog(
+ unsigned maxTableLog,
+ size_t srcSize,
+ unsigned maxSymbolValue,
+ void* workSpace, size_t wkspSize,
+ HUF_CElt* table,
+ const unsigned* count,
+ int flags)
+{
+ assert(srcSize > 1); /* Not supported, RLE should be used instead */
+ assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables));
+
+ if (!(flags & HUF_flags_optimalDepth)) {
+ /* cheap evaluation, based on FSE */
+ return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+ }
+
+ { BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
+ size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp);
+ size_t maxBits, hSize, newSize;
+ const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
+ const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
+ size_t optSize = ((size_t) ~0) - 1;
+ unsigned optLog = maxTableLog, optLogGuess;
+
+ DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize);
+
+ /* Search until size increases */
+ for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
+ DEBUGLOG(7, "checking for huffLog=%u", optLogGuess);
+ maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
+ if (ERR_isError(maxBits)) continue;
+
+ if (maxBits < optLogGuess && optLogGuess > minTableLog) break;
+
+ hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
+
+ if (ERR_isError(hSize)) continue;
+
+ newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
+
+ if (newSize > optSize + 1) {
+ break;
+ }
+
+ if (newSize < optSize) {
+ optSize = newSize;
+ optLog = optLogGuess;
+ }
+ }
+ assert(optLog <= HUF_TABLELOG_MAX);
+ return optLog;
+ }
+}
+
+/* HUF_compress_internal() :
+ * `workSpace_align4` must be aligned on 4-bytes boundaries,
+ * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */
+static size_t
+HUF_compress_internal (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned huffLog,
+ HUF_nbStreams_e nbStreams,
+ void* workSpace, size_t wkspSize,
+ HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags)
+{
+ HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t));
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* op = ostart;
+
+ DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize);
+ HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE);
+
+ /* checks & inits */
+ if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall);
+ if (!srcSize) return 0; /* Uncompressed */
+ if (!dstSize) return 0; /* cannot fit anything within dst budget */
+ if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
+ if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
+ if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
+ if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
+
+ /* Heuristic : If old table is valid, use it for small inputs */
+ if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) {
+ return HUF_compressCTable_internal(ostart, op, oend,
+ src, srcSize,
+ nbStreams, oldHufTable, flags);
+ }
+
+ /* If uncompressible data is suspected, do a smaller sampling first */
+ DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2);
+ if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) {
+ size_t largestTotal = 0;
+ DEBUGLOG(5, "input suspected incompressible : sampling to check");
+ { unsigned maxSymbolValueBegin = maxSymbolValue;
+ CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) );
+ largestTotal += largestBegin;
+ }
+ { unsigned maxSymbolValueEnd = maxSymbolValue;
+ CHECK_V_F(largestEnd, HIST_count_simple (table->count, &maxSymbolValueEnd, (const BYTE*)src + srcSize - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) );
+ largestTotal += largestEnd;
+ }
+ if (largestTotal <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7)+4) return 0; /* heuristic : probably not compressible enough */
+ }
+
+ /* Scan input and build symbol stats */
+ { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->wksps.hist_wksp, sizeof(table->wksps.hist_wksp)) );
+ if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
+ if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */
+ }
+ DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1));
+
+ /* Check validity of previous table */
+ if ( repeat
+ && *repeat == HUF_repeat_check
+ && !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) {
+ *repeat = HUF_repeat_none;
+ }
+ /* Heuristic : use existing table for small inputs */
+ if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) {
+ return HUF_compressCTable_internal(ostart, op, oend,
+ src, srcSize,
+ nbStreams, oldHufTable, flags);
+ }
+
+ /* Build Huffman Tree */
+ huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags);
+ { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
+ maxSymbolValue, huffLog,
+ &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp));
+ CHECK_F(maxBits);
+ huffLog = (U32)maxBits;
+ DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1));
+ }
+ /* Zero unused symbols in CTable, so we can check it for validity */
+ {
+ size_t const ctableSize = HUF_CTABLE_SIZE_ST(maxSymbolValue);
+ size_t const unusedSize = sizeof(table->CTable) - ctableSize * sizeof(HUF_CElt);
+ ZSTD_memset(table->CTable + ctableSize, 0, unusedSize);
+ }
+
+ /* Write table description header */
+ { CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog,
+ &table->wksps.writeCTable_wksp, sizeof(table->wksps.writeCTable_wksp)) );
+ /* Check if using previous huffman table is beneficial */
+ if (repeat && *repeat != HUF_repeat_none) {
+ size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue);
+ size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue);
+ if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
+ return HUF_compressCTable_internal(ostart, op, oend,
+ src, srcSize,
+ nbStreams, oldHufTable, flags);
+ } }
+
+ /* Use the new huffman table */
+ if (hSize + 12ul >= srcSize) { return 0; }
+ op += hSize;
+ if (repeat) { *repeat = HUF_repeat_none; }
+ if (oldHufTable)
+ ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */
+ }
+ return HUF_compressCTable_internal(ostart, op, oend,
+ src, srcSize,
+ nbStreams, table->CTable, flags);
+}
+
+size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned huffLog,
+ void* workSpace, size_t wkspSize,
+ HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
+{
+ DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize);
+ return HUF_compress_internal(dst, dstSize, src, srcSize,
+ maxSymbolValue, huffLog, HUF_singleStream,
+ workSpace, wkspSize, hufTable,
+ repeat, flags);
+}
+
+/* HUF_compress4X_repeat():
+ * compress input using 4 streams.
+ * consider skipping quickly
+ * re-use an existing huffman compression table */
+size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned huffLog,
+ void* workSpace, size_t wkspSize,
+ HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
+{
+ DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize);
+ return HUF_compress_internal(dst, dstSize, src, srcSize,
+ maxSymbolValue, huffLog, HUF_fourStreams,
+ workSpace, wkspSize,
+ hufTable, repeat, flags);
+}
diff --git a/contrib/zstd/huf_decompress.c b/contrib/zstd/huf_decompress.c
new file mode 100644
index 0000000..ea72fe4
--- /dev/null
+++ b/contrib/zstd/huf_decompress.c
@@ -0,0 +1,1882 @@
+/* ******************************************************************
+ * huff0 huffman decoder,
+ * part of Finite State Entropy library
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * You can contact the author at :
+ * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+****************************************************************** */
+
+/* **************************************************************
+* Dependencies
+****************************************************************/
+#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */
+#include "compiler.h"
+#include "bitstream.h" /* BIT_* */
+#include "fse.h" /* to compress headers */
+#include "huf.h"
+#include "error_private.h"
+#include "zstd_internal.h"
+#include "bits.h" /* ZSTD_highbit32, ZSTD_countTrailingZeros64 */
+
+/* **************************************************************
+* Constants
+****************************************************************/
+
+#define HUF_DECODER_FAST_TABLELOG 11
+
+/* **************************************************************
+* Macros
+****************************************************************/
+
+/* These two optional macros force the use one way or another of the two
+ * Huffman decompression implementations. You can't force in both directions
+ * at the same time.
+ */
+#if defined(HUF_FORCE_DECOMPRESS_X1) && \
+ defined(HUF_FORCE_DECOMPRESS_X2)
+#error "Cannot force the use of the X1 and X2 decoders at the same time!"
+#endif
+
+/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is
+ * supported at runtime, so we can add the BMI2 target attribute.
+ * When it is disabled, we will still get BMI2 if it is enabled statically.
+ */
+#if DYNAMIC_BMI2
+# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
+#else
+# define HUF_FAST_BMI2_ATTRS
+#endif
+
+#ifdef __cplusplus
+# define HUF_EXTERN_C extern "C"
+#else
+# define HUF_EXTERN_C
+#endif
+#define HUF_ASM_DECL HUF_EXTERN_C
+
+#if DYNAMIC_BMI2
+# define HUF_NEED_BMI2_FUNCTION 1
+#else
+# define HUF_NEED_BMI2_FUNCTION 0
+#endif
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define HUF_isError ERR_isError
+
+
+/* **************************************************************
+* Byte alignment for workSpace management
+****************************************************************/
+#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1)
+#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
+
+
+/* **************************************************************
+* BMI2 Variant Wrappers
+****************************************************************/
+typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize,
+ const void *cSrc,
+ size_t cSrcSize,
+ const HUF_DTable *DTable);
+
+#if DYNAMIC_BMI2
+
+#define HUF_DGEN(fn) \
+ \
+ static size_t fn##_default( \
+ void* dst, size_t dstSize, \
+ const void* cSrc, size_t cSrcSize, \
+ const HUF_DTable* DTable) \
+ { \
+ return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
+ } \
+ \
+ static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \
+ void* dst, size_t dstSize, \
+ const void* cSrc, size_t cSrcSize, \
+ const HUF_DTable* DTable) \
+ { \
+ return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
+ } \
+ \
+ static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
+ size_t cSrcSize, HUF_DTable const* DTable, int flags) \
+ { \
+ if (flags & HUF_flags_bmi2) { \
+ return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \
+ } \
+ return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \
+ }
+
+#else
+
+#define HUF_DGEN(fn) \
+ static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
+ size_t cSrcSize, HUF_DTable const* DTable, int flags) \
+ { \
+ (void)flags; \
+ return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
+ }
+
+#endif
+
+
+/*-***************************/
+/* generic DTableDesc */
+/*-***************************/
+typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;
+
+static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
+{
+ DTableDesc dtd;
+ ZSTD_memcpy(&dtd, table, sizeof(dtd));
+ return dtd;
+}
+
+static size_t HUF_initFastDStream(BYTE const* ip) {
+ BYTE const lastByte = ip[7];
+ size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
+ size_t const value = MEM_readLEST(ip) | 1;
+ assert(bitsConsumed <= 8);
+ assert(sizeof(size_t) == 8);
+ return value << bitsConsumed;
+}
+
+
+/**
+ * The input/output arguments to the Huffman fast decoding loop:
+ *
+ * ip [in/out] - The input pointers, must be updated to reflect what is consumed.
+ * op [in/out] - The output pointers, must be updated to reflect what is written.
+ * bits [in/out] - The bitstream containers, must be updated to reflect the current state.
+ * dt [in] - The decoding table.
+ * ilimit [in] - The input limit, stop when any input pointer is below ilimit.
+ * oend [in] - The end of the output stream. op[3] must not cross oend.
+ * iend [in] - The end of each input stream. ip[i] may cross iend[i],
+ * as long as it is above ilimit, but that indicates corruption.
+ */
+typedef struct {
+ BYTE const* ip[4];
+ BYTE* op[4];
+ U64 bits[4];
+ void const* dt;
+ BYTE const* ilimit;
+ BYTE* oend;
+ BYTE const* iend[4];
+} HUF_DecompressFastArgs;
+
+typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*);
+
+/**
+ * Initializes args for the fast decoding loop.
+ * @returns 1 on success
+ * 0 if the fallback implementation should be used.
+ * Or an error code on failure.
+ */
+static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
+{
+ void const* dt = DTable + 1;
+ U32 const dtLog = HUF_getDTableDesc(DTable).tableLog;
+
+ const BYTE* const ilimit = (const BYTE*)src + 6 + 8;
+
+ BYTE* const oend = (BYTE*)dst + dstSize;
+
+ /* The fast decoding loop assumes 64-bit little-endian.
+ * This condition is false on x32.
+ */
+ if (!MEM_isLittleEndian() || MEM_32bits())
+ return 0;
+
+ /* strict minimum : jump table + 1 byte per stream */
+ if (srcSize < 10)
+ return ERROR(corruption_detected);
+
+ /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers.
+ * If table log is not correct at this point, fallback to the old decoder.
+ * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder.
+ */
+ if (dtLog != HUF_DECODER_FAST_TABLELOG)
+ return 0;
+
+ /* Read the jump table. */
+ {
+ const BYTE* const istart = (const BYTE*)src;
+ size_t const length1 = MEM_readLE16(istart);
+ size_t const length2 = MEM_readLE16(istart+2);
+ size_t const length3 = MEM_readLE16(istart+4);
+ size_t const length4 = srcSize - (length1 + length2 + length3 + 6);
+ args->iend[0] = istart + 6; /* jumpTable */
+ args->iend[1] = args->iend[0] + length1;
+ args->iend[2] = args->iend[1] + length2;
+ args->iend[3] = args->iend[2] + length3;
+
+ /* HUF_initFastDStream() requires this, and this small of an input
+ * won't benefit from the ASM loop anyways.
+ * length1 must be >= 16 so that ip[0] >= ilimit before the loop
+ * starts.
+ */
+ if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8)
+ return 0;
+ if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */
+ }
+ /* ip[] contains the position that is currently loaded into bits[]. */
+ args->ip[0] = args->iend[1] - sizeof(U64);
+ args->ip[1] = args->iend[2] - sizeof(U64);
+ args->ip[2] = args->iend[3] - sizeof(U64);
+ args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64);
+
+ /* op[] contains the output pointers. */
+ args->op[0] = (BYTE*)dst;
+ args->op[1] = args->op[0] + (dstSize+3)/4;
+ args->op[2] = args->op[1] + (dstSize+3)/4;
+ args->op[3] = args->op[2] + (dstSize+3)/4;
+
+ /* No point to call the ASM loop for tiny outputs. */
+ if (args->op[3] >= oend)
+ return 0;
+
+ /* bits[] is the bit container.
+ * It is read from the MSB down to the LSB.
+ * It is shifted left as it is read, and zeros are
+ * shifted in. After the lowest valid bit a 1 is
+ * set, so that CountTrailingZeros(bits[]) can be used
+ * to count how many bits we've consumed.
+ */
+ args->bits[0] = HUF_initFastDStream(args->ip[0]);
+ args->bits[1] = HUF_initFastDStream(args->ip[1]);
+ args->bits[2] = HUF_initFastDStream(args->ip[2]);
+ args->bits[3] = HUF_initFastDStream(args->ip[3]);
+
+ /* If ip[] >= ilimit, it is guaranteed to be safe to
+ * reload bits[]. It may be beyond its section, but is
+ * guaranteed to be valid (>= istart).
+ */
+ args->ilimit = ilimit;
+
+ args->oend = oend;
+ args->dt = dt;
+
+ return 1;
+}
+
+static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd)
+{
+ /* Validate that we haven't overwritten. */
+ if (args->op[stream] > segmentEnd)
+ return ERROR(corruption_detected);
+ /* Validate that we haven't read beyond iend[].
+ * Note that ip[] may be < iend[] because the MSB is
+ * the next bit to read, and we may have consumed 100%
+ * of the stream, so down to iend[i] - 8 is valid.
+ */
+ if (args->ip[stream] < args->iend[stream] - 8)
+ return ERROR(corruption_detected);
+
+ /* Construct the BIT_DStream_t. */
+ assert(sizeof(size_t) == 8);
+ bit->bitContainer = MEM_readLEST(args->ip[stream]);
+ bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]);
+ bit->start = (const char*)args->iend[0];
+ bit->limitPtr = bit->start + sizeof(size_t);
+ bit->ptr = (const char*)args->ip[stream];
+
+ return 0;
+}
+
+
+#ifndef HUF_FORCE_DECOMPRESS_X2
+
+/*-***************************/
+/* single-symbol decoding */
+/*-***************************/
+typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */
+
+/**
+ * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at
+ * a time.
+ */
+static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) {
+ U64 D4;
+ if (MEM_isLittleEndian()) {
+ D4 = (U64)((symbol << 8) + nbBits);
+ } else {
+ D4 = (U64)(symbol + (nbBits << 8));
+ }
+ assert(D4 < (1U << 16));
+ D4 *= 0x0001000100010001ULL;
+ return D4;
+}
+
+/**
+ * Increase the tableLog to targetTableLog and rescales the stats.
+ * If tableLog > targetTableLog this is a no-op.
+ * @returns New tableLog
+ */
+static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog)
+{
+ if (tableLog > targetTableLog)
+ return tableLog;
+ if (tableLog < targetTableLog) {
+ U32 const scale = targetTableLog - tableLog;
+ U32 s;
+ /* Increase the weight for all non-zero probability symbols by scale. */
+ for (s = 0; s < nbSymbols; ++s) {
+ huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale);
+ }
+ /* Update rankVal to reflect the new weights.
+ * All weights except 0 get moved to weight + scale.
+ * Weights [1, scale] are empty.
+ */
+ for (s = targetTableLog; s > scale; --s) {
+ rankVal[s] = rankVal[s - scale];
+ }
+ for (s = scale; s > 0; --s) {
+ rankVal[s] = 0;
+ }
+ }
+ return targetTableLog;
+}
+
+typedef struct {
+ U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
+ U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1];
+ U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
+ BYTE symbols[HUF_SYMBOLVALUE_MAX + 1];
+ BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
+} HUF_ReadDTableX1_Workspace;
+
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags)
+{
+ U32 tableLog = 0;
+ U32 nbSymbols = 0;
+ size_t iSize;
+ void* const dtPtr = DTable + 1;
+ HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr;
+ HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace;
+
+ DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp));
+ if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge);
+
+ DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
+ /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
+
+ iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags);
+ if (HUF_isError(iSize)) return iSize;
+
+
+ /* Table header */
+ { DTableDesc dtd = HUF_getDTableDesc(DTable);
+ U32 const maxTableLog = dtd.maxTableLog + 1;
+ U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG);
+ tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog);
+ if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */
+ dtd.tableType = 0;
+ dtd.tableLog = (BYTE)tableLog;
+ ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
+ }
+
+ /* Compute symbols and rankStart given rankVal:
+ *
+ * rankVal already contains the number of values of each weight.
+ *
+ * symbols contains the symbols ordered by weight. First are the rankVal[0]
+ * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on.
+ * symbols[0] is filled (but unused) to avoid a branch.
+ *
+ * rankStart contains the offset where each rank belongs in the DTable.
+ * rankStart[0] is not filled because there are no entries in the table for
+ * weight 0.
+ */
+ { int n;
+ U32 nextRankStart = 0;
+ int const unroll = 4;
+ int const nLimit = (int)nbSymbols - unroll + 1;
+ for (n=0; n<(int)tableLog+1; n++) {
+ U32 const curr = nextRankStart;
+ nextRankStart += wksp->rankVal[n];
+ wksp->rankStart[n] = curr;
+ }
+ for (n=0; n < nLimit; n += unroll) {
+ int u;
+ for (u=0; u < unroll; ++u) {
+ size_t const w = wksp->huffWeight[n+u];
+ wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u);
+ }
+ }
+ for (; n < (int)nbSymbols; ++n) {
+ size_t const w = wksp->huffWeight[n];
+ wksp->symbols[wksp->rankStart[w]++] = (BYTE)n;
+ }
+ }
+
+ /* fill DTable
+ * We fill all entries of each weight in order.
+ * That way length is a constant for each iteration of the outer loop.
+ * We can switch based on the length to a different inner loop which is
+ * optimized for that particular case.
+ */
+ { U32 w;
+ int symbol = wksp->rankVal[0];
+ int rankStart = 0;
+ for (w=1; w<tableLog+1; ++w) {
+ int const symbolCount = wksp->rankVal[w];
+ int const length = (1 << w) >> 1;
+ int uStart = rankStart;
+ BYTE const nbBits = (BYTE)(tableLog + 1 - w);
+ int s;
+ int u;
+ switch (length) {
+ case 1:
+ for (s=0; s<symbolCount; ++s) {
+ HUF_DEltX1 D;
+ D.byte = wksp->symbols[symbol + s];
+ D.nbBits = nbBits;
+ dt[uStart] = D;
+ uStart += 1;
+ }
+ break;
+ case 2:
+ for (s=0; s<symbolCount; ++s) {
+ HUF_DEltX1 D;
+ D.byte = wksp->symbols[symbol + s];
+ D.nbBits = nbBits;
+ dt[uStart+0] = D;
+ dt[uStart+1] = D;
+ uStart += 2;
+ }
+ break;
+ case 4:
+ for (s=0; s<symbolCount; ++s) {
+ U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
+ MEM_write64(dt + uStart, D4);
+ uStart += 4;
+ }
+ break;
+ case 8:
+ for (s=0; s<symbolCount; ++s) {
+ U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
+ MEM_write64(dt + uStart, D4);
+ MEM_write64(dt + uStart + 4, D4);
+ uStart += 8;
+ }
+ break;
+ default:
+ for (s=0; s<symbolCount; ++s) {
+ U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
+ for (u=0; u < length; u += 16) {
+ MEM_write64(dt + uStart + u + 0, D4);
+ MEM_write64(dt + uStart + u + 4, D4);
+ MEM_write64(dt + uStart + u + 8, D4);
+ MEM_write64(dt + uStart + u + 12, D4);
+ }
+ assert(u == length);
+ uStart += length;
+ }
+ break;
+ }
+ symbol += symbolCount;
+ rankStart += symbolCount * length;
+ }
+ }
+ return iSize;
+}
+
+FORCE_INLINE_TEMPLATE BYTE
+HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog)
+{
+ size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
+ BYTE const c = dt[val].byte;
+ BIT_skipBits(Dstream, dt[val].nbBits);
+ return c;
+}
+
+#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \
+ *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog)
+
+#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \
+ if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
+ HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr)
+
+#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \
+ if (MEM_64bits()) \
+ HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr)
+
+HINT_INLINE size_t
+HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog)
+{
+ BYTE* const pStart = p;
+
+ /* up to 4 symbols at a time */
+ if ((pEnd - p) > 3) {
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) {
+ HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX1_1(p, bitDPtr);
+ HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
+ }
+ } else {
+ BIT_reloadDStream(bitDPtr);
+ }
+
+ /* [0-3] symbols remaining */
+ if (MEM_32bits())
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd))
+ HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
+
+ /* no more data to retrieve from bitstream, no need to reload */
+ while (p < pEnd)
+ HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
+
+ return (size_t)(pEnd-pStart);
+}
+
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress1X1_usingDTable_internal_body(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable)
+{
+ BYTE* op = (BYTE*)dst;
+ BYTE* const oend = op + dstSize;
+ const void* dtPtr = DTable + 1;
+ const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
+ BIT_DStream_t bitD;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ U32 const dtLog = dtd.tableLog;
+
+ CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
+
+ HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog);
+
+ if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
+
+ return dstSize;
+}
+
+/* HUF_decompress4X1_usingDTable_internal_body():
+ * Conditions :
+ * @dstSize >= 6
+ */
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress4X1_usingDTable_internal_body(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable)
+{
+ /* Check */
+ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
+
+ { const BYTE* const istart = (const BYTE*) cSrc;
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* const olimit = oend - 3;
+ const void* const dtPtr = DTable + 1;
+ const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
+
+ /* Init */
+ BIT_DStream_t bitD1;
+ BIT_DStream_t bitD2;
+ BIT_DStream_t bitD3;
+ BIT_DStream_t bitD4;
+ size_t const length1 = MEM_readLE16(istart);
+ size_t const length2 = MEM_readLE16(istart+2);
+ size_t const length3 = MEM_readLE16(istart+4);
+ size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
+ const BYTE* const istart1 = istart + 6; /* jumpTable */
+ const BYTE* const istart2 = istart1 + length1;
+ const BYTE* const istart3 = istart2 + length2;
+ const BYTE* const istart4 = istart3 + length3;
+ const size_t segmentSize = (dstSize+3) / 4;
+ BYTE* const opStart2 = ostart + segmentSize;
+ BYTE* const opStart3 = opStart2 + segmentSize;
+ BYTE* const opStart4 = opStart3 + segmentSize;
+ BYTE* op1 = ostart;
+ BYTE* op2 = opStart2;
+ BYTE* op3 = opStart3;
+ BYTE* op4 = opStart4;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ U32 const dtLog = dtd.tableLog;
+ U32 endSignal = 1;
+
+ if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
+ if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */
+ if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */
+ CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
+ CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
+ CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
+ CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
+
+ /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */
+ if ((size_t)(oend - op4) >= sizeof(size_t)) {
+ for ( ; (endSignal) & (op4 < olimit) ; ) {
+ HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX1_1(op1, &bitD1);
+ HUF_DECODE_SYMBOLX1_1(op2, &bitD2);
+ HUF_DECODE_SYMBOLX1_1(op3, &bitD3);
+ HUF_DECODE_SYMBOLX1_1(op4, &bitD4);
+ HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX1_0(op1, &bitD1);
+ HUF_DECODE_SYMBOLX1_0(op2, &bitD2);
+ HUF_DECODE_SYMBOLX1_0(op3, &bitD3);
+ HUF_DECODE_SYMBOLX1_0(op4, &bitD4);
+ endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
+ endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
+ endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
+ endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
+ }
+ }
+
+ /* check corruption */
+ /* note : should not be necessary : op# advance in lock step, and we control op4.
+ * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */
+ if (op1 > opStart2) return ERROR(corruption_detected);
+ if (op2 > opStart3) return ERROR(corruption_detected);
+ if (op3 > opStart4) return ERROR(corruption_detected);
+ /* note : op4 supposed already verified within main loop */
+
+ /* finish bitStreams one by one */
+ HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog);
+ HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog);
+ HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog);
+ HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog);
+
+ /* check */
+ { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
+ if (!endCheck) return ERROR(corruption_detected); }
+
+ /* decoded size */
+ return dstSize;
+ }
+}
+
+#if HUF_NEED_BMI2_FUNCTION
+static BMI2_TARGET_ATTRIBUTE
+size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable) {
+ return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+#endif
+
+static
+size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable) {
+ return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2
+
+HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
+
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+ U64 bits[4];
+ BYTE const* ip[4];
+ BYTE* op[4];
+ U16 const* const dtable = (U16 const*)args->dt;
+ BYTE* const oend = args->oend;
+ BYTE const* const ilimit = args->ilimit;
+
+ /* Copy the arguments to local variables */
+ ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+ ZSTD_memcpy(&ip, &args->ip, sizeof(ip));
+ ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+ assert(MEM_isLittleEndian());
+ assert(!MEM_32bits());
+
+ for (;;) {
+ BYTE* olimit;
+ int stream;
+ int symbol;
+
+ /* Assert loop preconditions */
+#ifndef NDEBUG
+ for (stream = 0; stream < 4; ++stream) {
+ assert(op[stream] <= (stream == 3 ? oend : op[stream + 1]));
+ assert(ip[stream] >= ilimit);
+ }
+#endif
+ /* Compute olimit */
+ {
+ /* Each iteration produces 5 output symbols per stream */
+ size_t const oiters = (size_t)(oend - op[3]) / 5;
+ /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes
+ * per stream.
+ */
+ size_t const iiters = (size_t)(ip[0] - ilimit) / 7;
+ /* We can safely run iters iterations before running bounds checks */
+ size_t const iters = MIN(oiters, iiters);
+ size_t const symbols = iters * 5;
+
+ /* We can simply check that op[3] < olimit, instead of checking all
+ * of our bounds, since we can't hit the other bounds until we've run
+ * iters iterations, which only happens when op[3] == olimit.
+ */
+ olimit = op[3] + symbols;
+
+ /* Exit fast decoding loop once we get close to the end. */
+ if (op[3] + 20 > olimit)
+ break;
+
+ /* Exit the decoding loop if any input pointer has crossed the
+ * previous one. This indicates corruption, and a precondition
+ * to our loop is that ip[i] >= ip[0].
+ */
+ for (stream = 1; stream < 4; ++stream) {
+ if (ip[stream] < ip[stream - 1])
+ goto _out;
+ }
+ }
+
+#ifndef NDEBUG
+ for (stream = 1; stream < 4; ++stream) {
+ assert(ip[stream] >= ip[stream - 1]);
+ }
+#endif
+
+ do {
+ /* Decode 5 symbols in each of the 4 streams */
+ for (symbol = 0; symbol < 5; ++symbol) {
+ for (stream = 0; stream < 4; ++stream) {
+ int const index = (int)(bits[stream] >> 53);
+ int const entry = (int)dtable[index];
+ bits[stream] <<= (entry & 63);
+ op[stream][symbol] = (BYTE)((entry >> 8) & 0xFF);
+ }
+ }
+ /* Reload the bitstreams */
+ for (stream = 0; stream < 4; ++stream) {
+ int const ctz = ZSTD_countTrailingZeros64(bits[stream]);
+ int const nbBits = ctz & 7;
+ int const nbBytes = ctz >> 3;
+ op[stream] += 5;
+ ip[stream] -= nbBytes;
+ bits[stream] = MEM_read64(ip[stream]) | 1;
+ bits[stream] <<= nbBits;
+ }
+ } while (op[3] < olimit);
+ }
+
+_out:
+
+ /* Save the final values of each of the state variables back to args. */
+ ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+ ZSTD_memcpy(&args->ip, &ip, sizeof(ip));
+ ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+/**
+ * @returns @p dstSize on success (>= 6)
+ * 0 if the fallback implementation should be used
+ * An error if an error occurred
+ */
+static HUF_FAST_BMI2_ATTRS
+size_t
+HUF_decompress4X1_usingDTable_internal_fast(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable,
+ HUF_DecompressFastLoopFn loopFn)
+{
+ void const* dt = DTable + 1;
+ const BYTE* const iend = (const BYTE*)cSrc + 6;
+ BYTE* const oend = (BYTE*)dst + dstSize;
+ HUF_DecompressFastArgs args;
+ { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+ FORWARD_IF_ERROR(ret, "Failed to init fast loop args");
+ if (ret == 0)
+ return 0;
+ }
+
+ assert(args.ip[0] >= args.ilimit);
+ loopFn(&args);
+
+ /* Our loop guarantees that ip[] >= ilimit and that we haven't
+ * overwritten any op[].
+ */
+ assert(args.ip[0] >= iend);
+ assert(args.ip[1] >= iend);
+ assert(args.ip[2] >= iend);
+ assert(args.ip[3] >= iend);
+ assert(args.op[3] <= oend);
+ (void)iend;
+
+ /* finish bit streams one by one. */
+ { size_t const segmentSize = (dstSize+3) / 4;
+ BYTE* segmentEnd = (BYTE*)dst;
+ int i;
+ for (i = 0; i < 4; ++i) {
+ BIT_DStream_t bit;
+ if (segmentSize <= (size_t)(oend - segmentEnd))
+ segmentEnd += segmentSize;
+ else
+ segmentEnd = oend;
+ FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
+ /* Decompress and validate that we've produced exactly the expected length. */
+ args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG);
+ if (args.op[i] != segmentEnd) return ERROR(corruption_detected);
+ }
+ }
+
+ /* decoded size */
+ assert(dstSize != 0);
+ return dstSize;
+}
+
+HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
+
+static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable, int flags)
+{
+ HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default;
+ HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop;
+
+#if DYNAMIC_BMI2
+ if (flags & HUF_flags_bmi2) {
+ fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2;
+# if ZSTD_ENABLE_ASM_X86_64_BMI2
+ if (!(flags & HUF_flags_disableAsm)) {
+ loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+ }
+# endif
+ } else {
+ return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+ }
+#endif
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
+ if (!(flags & HUF_flags_disableAsm)) {
+ loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+ }
+#endif
+
+ if (!(flags & HUF_flags_disableFast)) {
+ size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+ if (ret != 0)
+ return ret;
+ }
+ return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ void* workSpace, size_t wkspSize, int flags)
+{
+ const BYTE* ip = (const BYTE*) cSrc;
+
+ size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
+ if (HUF_isError(hSize)) return hSize;
+ if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+ ip += hSize; cSrcSize -= hSize;
+
+ return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
+}
+
+#endif /* HUF_FORCE_DECOMPRESS_X2 */
+
+
+#ifndef HUF_FORCE_DECOMPRESS_X1
+
+/* *************************/
+/* double-symbols decoding */
+/* *************************/
+
+typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */
+typedef struct { BYTE symbol; } sortedSymbol_t;
+typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
+typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX];
+
+/**
+ * Constructs a HUF_DEltX2 in a U32.
+ */
+static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level)
+{
+ U32 seq;
+ DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0);
+ DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2);
+ DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3);
+ DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32));
+ if (MEM_isLittleEndian()) {
+ seq = level == 1 ? symbol : (baseSeq + (symbol << 8));
+ return seq + (nbBits << 16) + ((U32)level << 24);
+ } else {
+ seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol);
+ return (seq << 16) + (nbBits << 8) + (U32)level;
+ }
+}
+
+/**
+ * Constructs a HUF_DEltX2.
+ */
+static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level)
+{
+ HUF_DEltX2 DElt;
+ U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
+ DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val));
+ ZSTD_memcpy(&DElt, &val, sizeof(val));
+ return DElt;
+}
+
+/**
+ * Constructs 2 HUF_DEltX2s and packs them into a U64.
+ */
+static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level)
+{
+ U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
+ return (U64)DElt + ((U64)DElt << 32);
+}
+
+/**
+ * Fills the DTable rank with all the symbols from [begin, end) that are each
+ * nbBits long.
+ *
+ * @param DTableRank The start of the rank in the DTable.
+ * @param begin The first symbol to fill (inclusive).
+ * @param end The last symbol to fill (exclusive).
+ * @param nbBits Each symbol is nbBits long.
+ * @param tableLog The table log.
+ * @param baseSeq If level == 1 { 0 } else { the first level symbol }
+ * @param level The level in the table. Must be 1 or 2.
+ */
+static void HUF_fillDTableX2ForWeight(
+ HUF_DEltX2* DTableRank,
+ sortedSymbol_t const* begin, sortedSymbol_t const* end,
+ U32 nbBits, U32 tableLog,
+ U16 baseSeq, int const level)
+{
+ U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */);
+ const sortedSymbol_t* ptr;
+ assert(level >= 1 && level <= 2);
+ switch (length) {
+ case 1:
+ for (ptr = begin; ptr != end; ++ptr) {
+ HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
+ *DTableRank++ = DElt;
+ }
+ break;
+ case 2:
+ for (ptr = begin; ptr != end; ++ptr) {
+ HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
+ DTableRank[0] = DElt;
+ DTableRank[1] = DElt;
+ DTableRank += 2;
+ }
+ break;
+ case 4:
+ for (ptr = begin; ptr != end; ++ptr) {
+ U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
+ ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
+ DTableRank += 4;
+ }
+ break;
+ case 8:
+ for (ptr = begin; ptr != end; ++ptr) {
+ U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
+ ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
+ DTableRank += 8;
+ }
+ break;
+ default:
+ for (ptr = begin; ptr != end; ++ptr) {
+ U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
+ HUF_DEltX2* const DTableRankEnd = DTableRank + length;
+ for (; DTableRank != DTableRankEnd; DTableRank += 8) {
+ ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
+ }
+ }
+ break;
+ }
+}
+
+/* HUF_fillDTableX2Level2() :
+ * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
+static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits,
+ const U32* rankVal, const int minWeight, const int maxWeight1,
+ const sortedSymbol_t* sortedSymbols, U32 const* rankStart,
+ U32 nbBitsBaseline, U16 baseSeq)
+{
+ /* Fill skipped values (all positions up to rankVal[minWeight]).
+ * These are positions only get a single symbol because the combined weight
+ * is too large.
+ */
+ if (minWeight>1) {
+ U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */);
+ U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1);
+ int const skipSize = rankVal[minWeight];
+ assert(length > 1);
+ assert((U32)skipSize < length);
+ switch (length) {
+ case 2:
+ assert(skipSize == 1);
+ ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2));
+ break;
+ case 4:
+ assert(skipSize <= 4);
+ ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2));
+ break;
+ default:
+ {
+ int i;
+ for (i = 0; i < skipSize; i += 8) {
+ ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2));
+ ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2));
+ }
+ }
+ }
+ }
+
+ /* Fill each of the second level symbols by weight. */
+ {
+ int w;
+ for (w = minWeight; w < maxWeight1; ++w) {
+ int const begin = rankStart[w];
+ int const end = rankStart[w+1];
+ U32 const nbBits = nbBitsBaseline - w;
+ U32 const totalBits = nbBits + consumedBits;
+ HUF_fillDTableX2ForWeight(
+ DTable + rankVal[w],
+ sortedSymbols + begin, sortedSymbols + end,
+ totalBits, targetLog,
+ baseSeq, /* level */ 2);
+ }
+ }
+}
+
+static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
+ const sortedSymbol_t* sortedList,
+ const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight,
+ const U32 nbBitsBaseline)
+{
+ U32* const rankVal = rankValOrigin[0];
+ const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
+ const U32 minBits = nbBitsBaseline - maxWeight;
+ int w;
+ int const wEnd = (int)maxWeight + 1;
+
+ /* Fill DTable in order of weight. */
+ for (w = 1; w < wEnd; ++w) {
+ int const begin = (int)rankStart[w];
+ int const end = (int)rankStart[w+1];
+ U32 const nbBits = nbBitsBaseline - w;
+
+ if (targetLog-nbBits >= minBits) {
+ /* Enough room for a second symbol. */
+ int start = rankVal[w];
+ U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */);
+ int minWeight = nbBits + scaleLog;
+ int s;
+ if (minWeight < 1) minWeight = 1;
+ /* Fill the DTable for every symbol of weight w.
+ * These symbols get at least 1 second symbol.
+ */
+ for (s = begin; s != end; ++s) {
+ HUF_fillDTableX2Level2(
+ DTable + start, targetLog, nbBits,
+ rankValOrigin[nbBits], minWeight, wEnd,
+ sortedList, rankStart,
+ nbBitsBaseline, sortedList[s].symbol);
+ start += length;
+ }
+ } else {
+ /* Only a single symbol. */
+ HUF_fillDTableX2ForWeight(
+ DTable + rankVal[w],
+ sortedList + begin, sortedList + end,
+ nbBits, targetLog,
+ /* baseSeq */ 0, /* level */ 1);
+ }
+ }
+}
+
+typedef struct {
+ rankValCol_t rankVal[HUF_TABLELOG_MAX];
+ U32 rankStats[HUF_TABLELOG_MAX + 1];
+ U32 rankStart0[HUF_TABLELOG_MAX + 3];
+ sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
+ BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
+ U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
+} HUF_ReadDTableX2_Workspace;
+
+size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
+ const void* src, size_t srcSize,
+ void* workSpace, size_t wkspSize, int flags)
+{
+ U32 tableLog, maxW, nbSymbols;
+ DTableDesc dtd = HUF_getDTableDesc(DTable);
+ U32 maxTableLog = dtd.maxTableLog;
+ size_t iSize;
+ void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
+ HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
+ U32 *rankStart;
+
+ HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace;
+
+ if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC);
+
+ rankStart = wksp->rankStart0 + 1;
+ ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats));
+ ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0));
+
+ DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
+ if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
+ /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
+
+ iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags);
+ if (HUF_isError(iSize)) return iSize;
+
+ /* check result */
+ if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
+ if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG;
+
+ /* find maxWeight */
+ for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
+
+ /* Get start index of each weight */
+ { U32 w, nextRankStart = 0;
+ for (w=1; w<maxW+1; w++) {
+ U32 curr = nextRankStart;
+ nextRankStart += wksp->rankStats[w];
+ rankStart[w] = curr;
+ }
+ rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
+ rankStart[maxW+1] = nextRankStart;
+ }
+
+ /* sort symbols by weight */
+ { U32 s;
+ for (s=0; s<nbSymbols; s++) {
+ U32 const w = wksp->weightList[s];
+ U32 const r = rankStart[w]++;
+ wksp->sortedSymbol[r].symbol = (BYTE)s;
+ }
+ rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
+ }
+
+ /* Build rankVal */
+ { U32* const rankVal0 = wksp->rankVal[0];
+ { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */
+ U32 nextRankVal = 0;
+ U32 w;
+ for (w=1; w<maxW+1; w++) {
+ U32 curr = nextRankVal;
+ nextRankVal += wksp->rankStats[w] << (w+rescale);
+ rankVal0[w] = curr;
+ } }
+ { U32 const minBits = tableLog+1 - maxW;
+ U32 consumed;
+ for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
+ U32* const rankValPtr = wksp->rankVal[consumed];
+ U32 w;
+ for (w = 1; w < maxW+1; w++) {
+ rankValPtr[w] = rankVal0[w] >> consumed;
+ } } } }
+
+ HUF_fillDTableX2(dt, maxTableLog,
+ wksp->sortedSymbol,
+ wksp->rankStart0, wksp->rankVal, maxW,
+ tableLog+1);
+
+ dtd.tableLog = (BYTE)maxTableLog;
+ dtd.tableType = 1;
+ ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
+ return iSize;
+}
+
+
+FORCE_INLINE_TEMPLATE U32
+HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
+{
+ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
+ ZSTD_memcpy(op, &dt[val].sequence, 2);
+ BIT_skipBits(DStream, dt[val].nbBits);
+ return dt[val].length;
+}
+
+FORCE_INLINE_TEMPLATE U32
+HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
+{
+ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
+ ZSTD_memcpy(op, &dt[val].sequence, 1);
+ if (dt[val].length==1) {
+ BIT_skipBits(DStream, dt[val].nbBits);
+ } else {
+ if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
+ BIT_skipBits(DStream, dt[val].nbBits);
+ if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
+ /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
+ DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8);
+ }
+ }
+ return 1;
+}
+
+#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \
+ ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
+
+#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
+ if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
+ ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
+
+#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
+ if (MEM_64bits()) \
+ ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
+
+HINT_INLINE size_t
+HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd,
+ const HUF_DEltX2* const dt, const U32 dtLog)
+{
+ BYTE* const pStart = p;
+
+ /* up to 8 symbols at a time */
+ if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) {
+ if (dtLog <= 11 && MEM_64bits()) {
+ /* up to 10 symbols at a time */
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) {
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ }
+ } else {
+ /* up to 8 symbols at a time */
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) {
+ HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ }
+ }
+ } else {
+ BIT_reloadDStream(bitDPtr);
+ }
+
+ /* closer to end : up to 2 symbols at a time */
+ if ((size_t)(pEnd - p) >= 2) {
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2))
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+
+ while (p <= pEnd-2)
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
+ }
+
+ if (p < pEnd)
+ p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog);
+
+ return p-pStart;
+}
+
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress1X2_usingDTable_internal_body(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable)
+{
+ BIT_DStream_t bitD;
+
+ /* Init */
+ CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
+
+ /* decode */
+ { BYTE* const ostart = (BYTE*) dst;
+ BYTE* const oend = ostart + dstSize;
+ const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */
+ const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog);
+ }
+
+ /* check */
+ if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
+
+ /* decoded size */
+ return dstSize;
+}
+
+/* HUF_decompress4X2_usingDTable_internal_body():
+ * Conditions:
+ * @dstSize >= 6
+ */
+FORCE_INLINE_TEMPLATE size_t
+HUF_decompress4X2_usingDTable_internal_body(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable)
+{
+ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
+
+ { const BYTE* const istart = (const BYTE*) cSrc;
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* const olimit = oend - (sizeof(size_t)-1);
+ const void* const dtPtr = DTable+1;
+ const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
+
+ /* Init */
+ BIT_DStream_t bitD1;
+ BIT_DStream_t bitD2;
+ BIT_DStream_t bitD3;
+ BIT_DStream_t bitD4;
+ size_t const length1 = MEM_readLE16(istart);
+ size_t const length2 = MEM_readLE16(istart+2);
+ size_t const length3 = MEM_readLE16(istart+4);
+ size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
+ const BYTE* const istart1 = istart + 6; /* jumpTable */
+ const BYTE* const istart2 = istart1 + length1;
+ const BYTE* const istart3 = istart2 + length2;
+ const BYTE* const istart4 = istart3 + length3;
+ size_t const segmentSize = (dstSize+3) / 4;
+ BYTE* const opStart2 = ostart + segmentSize;
+ BYTE* const opStart3 = opStart2 + segmentSize;
+ BYTE* const opStart4 = opStart3 + segmentSize;
+ BYTE* op1 = ostart;
+ BYTE* op2 = opStart2;
+ BYTE* op3 = opStart3;
+ BYTE* op4 = opStart4;
+ U32 endSignal = 1;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ U32 const dtLog = dtd.tableLog;
+
+ if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
+ if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */
+ if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */
+ CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
+ CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
+ CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
+ CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
+
+ /* 16-32 symbols per loop (4-8 symbols per stream) */
+ if ((size_t)(oend - op4) >= sizeof(size_t)) {
+ for ( ; (endSignal) & (op4 < olimit); ) {
+#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
+ endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
+ endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
+ endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
+ endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
+#else
+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
+ endSignal = (U32)LIKELY((U32)
+ (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished)
+ & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished)
+ & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished)
+ & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished));
+#endif
+ }
+ }
+
+ /* check corruption */
+ if (op1 > opStart2) return ERROR(corruption_detected);
+ if (op2 > opStart3) return ERROR(corruption_detected);
+ if (op3 > opStart4) return ERROR(corruption_detected);
+ /* note : op4 already verified within main loop */
+
+ /* finish bitStreams one by one */
+ HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
+ HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
+ HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
+ HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
+
+ /* check */
+ { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
+ if (!endCheck) return ERROR(corruption_detected); }
+
+ /* decoded size */
+ return dstSize;
+ }
+}
+
+#if HUF_NEED_BMI2_FUNCTION
+static BMI2_TARGET_ATTRIBUTE
+size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable) {
+ return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+#endif
+
+static
+size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable) {
+ return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2
+
+HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
+
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+ U64 bits[4];
+ BYTE const* ip[4];
+ BYTE* op[4];
+ BYTE* oend[4];
+ HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt;
+ BYTE const* const ilimit = args->ilimit;
+
+ /* Copy the arguments to local registers. */
+ ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+ ZSTD_memcpy(&ip, &args->ip, sizeof(ip));
+ ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+ oend[0] = op[1];
+ oend[1] = op[2];
+ oend[2] = op[3];
+ oend[3] = args->oend;
+
+ assert(MEM_isLittleEndian());
+ assert(!MEM_32bits());
+
+ for (;;) {
+ BYTE* olimit;
+ int stream;
+ int symbol;
+
+ /* Assert loop preconditions */
+#ifndef NDEBUG
+ for (stream = 0; stream < 4; ++stream) {
+ assert(op[stream] <= oend[stream]);
+ assert(ip[stream] >= ilimit);
+ }
+#endif
+ /* Compute olimit */
+ {
+ /* Each loop does 5 table lookups for each of the 4 streams.
+ * Each table lookup consumes up to 11 bits of input, and produces
+ * up to 2 bytes of output.
+ */
+ /* We can consume up to 7 bytes of input per iteration per stream.
+ * We also know that each input pointer is >= ip[0]. So we can run
+ * iters loops before running out of input.
+ */
+ size_t iters = (size_t)(ip[0] - ilimit) / 7;
+ /* Each iteration can produce up to 10 bytes of output per stream.
+ * Each output stream my advance at different rates. So take the
+ * minimum number of safe iterations among all the output streams.
+ */
+ for (stream = 0; stream < 4; ++stream) {
+ size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10;
+ iters = MIN(iters, oiters);
+ }
+
+ /* Each iteration produces at least 5 output symbols. So until
+ * op[3] crosses olimit, we know we haven't executed iters
+ * iterations yet. This saves us maintaining an iters counter,
+ * at the expense of computing the remaining # of iterations
+ * more frequently.
+ */
+ olimit = op[3] + (iters * 5);
+
+ /* Exit the fast decoding loop if we are too close to the end. */
+ if (op[3] + 10 > olimit)
+ break;
+
+ /* Exit the decoding loop if any input pointer has crossed the
+ * previous one. This indicates corruption, and a precondition
+ * to our loop is that ip[i] >= ip[0].
+ */
+ for (stream = 1; stream < 4; ++stream) {
+ if (ip[stream] < ip[stream - 1])
+ goto _out;
+ }
+ }
+
+#ifndef NDEBUG
+ for (stream = 1; stream < 4; ++stream) {
+ assert(ip[stream] >= ip[stream - 1]);
+ }
+#endif
+
+ do {
+ /* Do 5 table lookups for each of the first 3 streams */
+ for (symbol = 0; symbol < 5; ++symbol) {
+ for (stream = 0; stream < 3; ++stream) {
+ int const index = (int)(bits[stream] >> 53);
+ HUF_DEltX2 const entry = dtable[index];
+ MEM_write16(op[stream], entry.sequence);
+ bits[stream] <<= (entry.nbBits);
+ op[stream] += (entry.length);
+ }
+ }
+ /* Do 1 table lookup from the final stream */
+ {
+ int const index = (int)(bits[3] >> 53);
+ HUF_DEltX2 const entry = dtable[index];
+ MEM_write16(op[3], entry.sequence);
+ bits[3] <<= (entry.nbBits);
+ op[3] += (entry.length);
+ }
+ /* Do 4 table lookups from the final stream & reload bitstreams */
+ for (stream = 0; stream < 4; ++stream) {
+ /* Do a table lookup from the final stream.
+ * This is interleaved with the reloading to reduce register
+ * pressure. This shouldn't be necessary, but compilers can
+ * struggle with codegen with high register pressure.
+ */
+ {
+ int const index = (int)(bits[3] >> 53);
+ HUF_DEltX2 const entry = dtable[index];
+ MEM_write16(op[3], entry.sequence);
+ bits[3] <<= (entry.nbBits);
+ op[3] += (entry.length);
+ }
+ /* Reload the bistreams. The final bitstream must be reloaded
+ * after the 5th symbol was decoded.
+ */
+ {
+ int const ctz = ZSTD_countTrailingZeros64(bits[stream]);
+ int const nbBits = ctz & 7;
+ int const nbBytes = ctz >> 3;
+ ip[stream] -= nbBytes;
+ bits[stream] = MEM_read64(ip[stream]) | 1;
+ bits[stream] <<= nbBits;
+ }
+ }
+ } while (op[3] < olimit);
+ }
+
+_out:
+
+ /* Save the final values of each of the state variables back to args. */
+ ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+ ZSTD_memcpy(&args->ip, &ip, sizeof(ip));
+ ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+
+static HUF_FAST_BMI2_ATTRS size_t
+HUF_decompress4X2_usingDTable_internal_fast(
+ void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ const HUF_DTable* DTable,
+ HUF_DecompressFastLoopFn loopFn) {
+ void const* dt = DTable + 1;
+ const BYTE* const iend = (const BYTE*)cSrc + 6;
+ BYTE* const oend = (BYTE*)dst + dstSize;
+ HUF_DecompressFastArgs args;
+ {
+ size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+ FORWARD_IF_ERROR(ret, "Failed to init asm args");
+ if (ret == 0)
+ return 0;
+ }
+
+ assert(args.ip[0] >= args.ilimit);
+ loopFn(&args);
+
+ /* note : op4 already verified within main loop */
+ assert(args.ip[0] >= iend);
+ assert(args.ip[1] >= iend);
+ assert(args.ip[2] >= iend);
+ assert(args.ip[3] >= iend);
+ assert(args.op[3] <= oend);
+ (void)iend;
+
+ /* finish bitStreams one by one */
+ {
+ size_t const segmentSize = (dstSize+3) / 4;
+ BYTE* segmentEnd = (BYTE*)dst;
+ int i;
+ for (i = 0; i < 4; ++i) {
+ BIT_DStream_t bit;
+ if (segmentSize <= (size_t)(oend - segmentEnd))
+ segmentEnd += segmentSize;
+ else
+ segmentEnd = oend;
+ FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
+ args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG);
+ if (args.op[i] != segmentEnd)
+ return ERROR(corruption_detected);
+ }
+ }
+
+ /* decoded size */
+ return dstSize;
+}
+
+static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
+ size_t cSrcSize, HUF_DTable const* DTable, int flags)
+{
+ HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default;
+ HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop;
+
+#if DYNAMIC_BMI2
+ if (flags & HUF_flags_bmi2) {
+ fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2;
+# if ZSTD_ENABLE_ASM_X86_64_BMI2
+ if (!(flags & HUF_flags_disableAsm)) {
+ loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+ }
+# endif
+ } else {
+ return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+ }
+#endif
+
+#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
+ if (!(flags & HUF_flags_disableAsm)) {
+ loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+ }
+#endif
+
+ if (!(flags & HUF_flags_disableFast)) {
+ size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+ if (ret != 0)
+ return ret;
+ }
+ return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+HUF_DGEN(HUF_decompress1X2_usingDTable_internal)
+
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ void* workSpace, size_t wkspSize, int flags)
+{
+ const BYTE* ip = (const BYTE*) cSrc;
+
+ size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize,
+ workSpace, wkspSize, flags);
+ if (HUF_isError(hSize)) return hSize;
+ if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+ ip += hSize; cSrcSize -= hSize;
+
+ return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags);
+}
+
+static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ void* workSpace, size_t wkspSize, int flags)
+{
+ const BYTE* ip = (const BYTE*) cSrc;
+
+ size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize,
+ workSpace, wkspSize, flags);
+ if (HUF_isError(hSize)) return hSize;
+ if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+ ip += hSize; cSrcSize -= hSize;
+
+ return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
+}
+
+#endif /* HUF_FORCE_DECOMPRESS_X1 */
+
+
+/* ***********************************/
+/* Universal decompression selectors */
+/* ***********************************/
+
+
+#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
+typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
+static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] =
+{
+ /* single, double, quad */
+ {{0,0}, {1,1}}, /* Q==0 : impossible */
+ {{0,0}, {1,1}}, /* Q==1 : impossible */
+ {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */
+ {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */
+ {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */
+ {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */
+ {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */
+ {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */
+ {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */
+ {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */
+ {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */
+ {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */
+ {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */
+ {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */
+ {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */
+ {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */
+};
+#endif
+
+/** HUF_selectDecoder() :
+ * Tells which decoder is likely to decode faster,
+ * based on a set of pre-computed metrics.
+ * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
+ * Assumption : 0 < dstSize <= 128 KB */
+U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
+{
+ assert(dstSize > 0);
+ assert(dstSize <= 128*1024);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+ (void)dstSize;
+ (void)cSrcSize;
+ return 0;
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+ (void)dstSize;
+ (void)cSrcSize;
+ return 1;
+#else
+ /* decoder timing evaluation */
+ { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */
+ U32 const D256 = (U32)(dstSize >> 8);
+ U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
+ U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
+ DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */
+ return DTime1 < DTime0;
+ }
+#endif
+}
+
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
+ const void* cSrc, size_t cSrcSize,
+ void* workSpace, size_t wkspSize, int flags)
+{
+ /* validation checks */
+ if (dstSize == 0) return ERROR(dstSize_tooSmall);
+ if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
+ if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
+ if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
+
+ { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+ (void)algoNb;
+ assert(algoNb == 0);
+ return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
+ cSrcSize, workSpace, wkspSize, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+ (void)algoNb;
+ assert(algoNb == 1);
+ return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
+ cSrcSize, workSpace, wkspSize, flags);
+#else
+ return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
+ cSrcSize, workSpace, wkspSize, flags):
+ HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
+ cSrcSize, workSpace, wkspSize, flags);
+#endif
+ }
+}
+
+
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
+{
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+ (void)dtd;
+ assert(dtd.tableType == 0);
+ return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+ (void)dtd;
+ assert(dtd.tableType == 1);
+ return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#else
+ return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+ HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#endif
+}
+
+#ifndef HUF_FORCE_DECOMPRESS_X2
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
+{
+ const BYTE* ip = (const BYTE*) cSrc;
+
+ size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
+ if (HUF_isError(hSize)) return hSize;
+ if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
+ ip += hSize; cSrcSize -= hSize;
+
+ return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
+}
+#endif
+
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
+{
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+ (void)dtd;
+ assert(dtd.tableType == 0);
+ return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+ (void)dtd;
+ assert(dtd.tableType == 1);
+ return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#else
+ return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+ HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
+#endif
+}
+
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
+{
+ /* validation checks */
+ if (dstSize == 0) return ERROR(dstSize_tooSmall);
+ if (cSrcSize == 0) return ERROR(corruption_detected);
+
+ { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
+#if defined(HUF_FORCE_DECOMPRESS_X1)
+ (void)algoNb;
+ assert(algoNb == 0);
+ return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
+#elif defined(HUF_FORCE_DECOMPRESS_X2)
+ (void)algoNb;
+ assert(algoNb == 1);
+ return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
+#else
+ return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) :
+ HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
+#endif
+ }
+}
diff --git a/contrib/zstd/mem.h b/contrib/zstd/mem.h
new file mode 100644
index 0000000..98dd47a
--- /dev/null
+++ b/contrib/zstd/mem.h
@@ -0,0 +1,435 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef MEM_H_MODULE
+#define MEM_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-****************************************
+* Dependencies
+******************************************/
+#include <stddef.h> /* size_t, ptrdiff_t */
+#include "compiler.h" /* __has_builtin */
+#include "debug.h" /* DEBUG_STATIC_ASSERT */
+#include "zstd_deps.h" /* ZSTD_memcpy */
+
+
+/*-****************************************
+* Compiler specifics
+******************************************/
+#if defined(_MSC_VER) /* Visual Studio */
+# include <stdlib.h> /* _byteswap_ulong */
+# include <intrin.h> /* _byteswap_* */
+#endif
+#if defined(__GNUC__)
+# define MEM_STATIC static __inline __attribute__((unused))
+#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+# define MEM_STATIC static inline
+#elif defined(_MSC_VER)
+# define MEM_STATIC static __inline
+#else
+# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
+#endif
+
+/*-**************************************************************
+* Basic Types
+*****************************************************************/
+#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# if defined(_AIX)
+# include <inttypes.h>
+# else
+# include <stdint.h> /* intptr_t */
+# endif
+ typedef uint8_t BYTE;
+ typedef uint8_t U8;
+ typedef int8_t S8;
+ typedef uint16_t U16;
+ typedef int16_t S16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+ typedef uint64_t U64;
+ typedef int64_t S64;
+#else
+# include <limits.h>
+#if CHAR_BIT != 8
+# error "this implementation requires char to be exactly 8-bit type"
+#endif
+ typedef unsigned char BYTE;
+ typedef unsigned char U8;
+ typedef signed char S8;
+#if USHRT_MAX != 65535
+# error "this implementation requires short to be exactly 16-bit type"
+#endif
+ typedef unsigned short U16;
+ typedef signed short S16;
+#if UINT_MAX != 4294967295
+# error "this implementation requires int to be exactly 32-bit type"
+#endif
+ typedef unsigned int U32;
+ typedef signed int S32;
+/* note : there are no limits defined for long long type in C90.
+ * limits exist in C99, however, in such case, <stdint.h> is preferred */
+ typedef unsigned long long U64;
+ typedef signed long long S64;
+#endif
+
+
+/*-**************************************************************
+* Memory I/O API
+*****************************************************************/
+/*=== Static platform detection ===*/
+MEM_STATIC unsigned MEM_32bits(void);
+MEM_STATIC unsigned MEM_64bits(void);
+MEM_STATIC unsigned MEM_isLittleEndian(void);
+
+/*=== Native unaligned read/write ===*/
+MEM_STATIC U16 MEM_read16(const void* memPtr);
+MEM_STATIC U32 MEM_read32(const void* memPtr);
+MEM_STATIC U64 MEM_read64(const void* memPtr);
+MEM_STATIC size_t MEM_readST(const void* memPtr);
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value);
+MEM_STATIC void MEM_write32(void* memPtr, U32 value);
+MEM_STATIC void MEM_write64(void* memPtr, U64 value);
+
+/*=== Little endian unaligned read/write ===*/
+MEM_STATIC U16 MEM_readLE16(const void* memPtr);
+MEM_STATIC U32 MEM_readLE24(const void* memPtr);
+MEM_STATIC U32 MEM_readLE32(const void* memPtr);
+MEM_STATIC U64 MEM_readLE64(const void* memPtr);
+MEM_STATIC size_t MEM_readLEST(const void* memPtr);
+
+MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val);
+MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val);
+MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32);
+MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64);
+MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val);
+
+/*=== Big endian unaligned read/write ===*/
+MEM_STATIC U32 MEM_readBE32(const void* memPtr);
+MEM_STATIC U64 MEM_readBE64(const void* memPtr);
+MEM_STATIC size_t MEM_readBEST(const void* memPtr);
+
+MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32);
+MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64);
+MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val);
+
+/*=== Byteswap ===*/
+MEM_STATIC U32 MEM_swap32(U32 in);
+MEM_STATIC U64 MEM_swap64(U64 in);
+MEM_STATIC size_t MEM_swapST(size_t in);
+
+
+/*-**************************************************************
+* Memory I/O Implementation
+*****************************************************************/
+/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
+ * Method 0 : always use `memcpy()`. Safe and portable.
+ * Method 1 : Use compiler extension to set unaligned access.
+ * Method 2 : direct access. This method is portable but violate C standard.
+ * It can generate buggy code on targets depending on alignment.
+ * Default : method 1 if supported, else method 0
+ */
+#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
+# ifdef __GNUC__
+# define MEM_FORCE_MEMORY_ACCESS 1
+# endif
+#endif
+
+MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
+MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
+
+MEM_STATIC unsigned MEM_isLittleEndian(void)
+{
+#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+ return 1;
+#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+ return 0;
+#elif defined(__clang__) && __LITTLE_ENDIAN__
+ return 1;
+#elif defined(__clang__) && __BIG_ENDIAN__
+ return 0;
+#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86)
+ return 1;
+#elif defined(__DMC__) && defined(_M_IX86)
+ return 1;
+#else
+ const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
+ return one.c[0];
+#endif
+}
+
+#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
+
+/* violates C standard, by lying on structure alignment.
+Only use if no other choice to achieve best performance on target platform */
+MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
+MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
+MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
+MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
+
+#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
+
+typedef __attribute__((aligned(1))) U16 unalign16;
+typedef __attribute__((aligned(1))) U32 unalign32;
+typedef __attribute__((aligned(1))) U64 unalign64;
+typedef __attribute__((aligned(1))) size_t unalignArch;
+
+MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
+MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; }
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; }
+
+#else
+
+/* default method, safe and standard.
+ can sometimes prove slower */
+
+MEM_STATIC U16 MEM_read16(const void* memPtr)
+{
+ U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC U32 MEM_read32(const void* memPtr)
+{
+ U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC U64 MEM_read64(const void* memPtr)
+{
+ U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC size_t MEM_readST(const void* memPtr)
+{
+ size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value)
+{
+ ZSTD_memcpy(memPtr, &value, sizeof(value));
+}
+
+MEM_STATIC void MEM_write32(void* memPtr, U32 value)
+{
+ ZSTD_memcpy(memPtr, &value, sizeof(value));
+}
+
+MEM_STATIC void MEM_write64(void* memPtr, U64 value)
+{
+ ZSTD_memcpy(memPtr, &value, sizeof(value));
+}
+
+#endif /* MEM_FORCE_MEMORY_ACCESS */
+
+MEM_STATIC U32 MEM_swap32_fallback(U32 in)
+{
+ return ((in << 24) & 0xff000000 ) |
+ ((in << 8) & 0x00ff0000 ) |
+ ((in >> 8) & 0x0000ff00 ) |
+ ((in >> 24) & 0x000000ff );
+}
+
+MEM_STATIC U32 MEM_swap32(U32 in)
+{
+#if defined(_MSC_VER) /* Visual Studio */
+ return _byteswap_ulong(in);
+#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
+ || (defined(__clang__) && __has_builtin(__builtin_bswap32))
+ return __builtin_bswap32(in);
+#else
+ return MEM_swap32_fallback(in);
+#endif
+}
+
+MEM_STATIC U64 MEM_swap64_fallback(U64 in)
+{
+ return ((in << 56) & 0xff00000000000000ULL) |
+ ((in << 40) & 0x00ff000000000000ULL) |
+ ((in << 24) & 0x0000ff0000000000ULL) |
+ ((in << 8) & 0x000000ff00000000ULL) |
+ ((in >> 8) & 0x00000000ff000000ULL) |
+ ((in >> 24) & 0x0000000000ff0000ULL) |
+ ((in >> 40) & 0x000000000000ff00ULL) |
+ ((in >> 56) & 0x00000000000000ffULL);
+}
+
+MEM_STATIC U64 MEM_swap64(U64 in)
+{
+#if defined(_MSC_VER) /* Visual Studio */
+ return _byteswap_uint64(in);
+#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
+ || (defined(__clang__) && __has_builtin(__builtin_bswap64))
+ return __builtin_bswap64(in);
+#else
+ return MEM_swap64_fallback(in);
+#endif
+}
+
+MEM_STATIC size_t MEM_swapST(size_t in)
+{
+ if (MEM_32bits())
+ return (size_t)MEM_swap32((U32)in);
+ else
+ return (size_t)MEM_swap64((U64)in);
+}
+
+/*=== Little endian r/w ===*/
+
+MEM_STATIC U16 MEM_readLE16(const void* memPtr)
+{
+ if (MEM_isLittleEndian())
+ return MEM_read16(memPtr);
+ else {
+ const BYTE* p = (const BYTE*)memPtr;
+ return (U16)(p[0] + (p[1]<<8));
+ }
+}
+
+MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
+{
+ if (MEM_isLittleEndian()) {
+ MEM_write16(memPtr, val);
+ } else {
+ BYTE* p = (BYTE*)memPtr;
+ p[0] = (BYTE)val;
+ p[1] = (BYTE)(val>>8);
+ }
+}
+
+MEM_STATIC U32 MEM_readLE24(const void* memPtr)
+{
+ return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16);
+}
+
+MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
+{
+ MEM_writeLE16(memPtr, (U16)val);
+ ((BYTE*)memPtr)[2] = (BYTE)(val>>16);
+}
+
+MEM_STATIC U32 MEM_readLE32(const void* memPtr)
+{
+ if (MEM_isLittleEndian())
+ return MEM_read32(memPtr);
+ else
+ return MEM_swap32(MEM_read32(memPtr));
+}
+
+MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
+{
+ if (MEM_isLittleEndian())
+ MEM_write32(memPtr, val32);
+ else
+ MEM_write32(memPtr, MEM_swap32(val32));
+}
+
+MEM_STATIC U64 MEM_readLE64(const void* memPtr)
+{
+ if (MEM_isLittleEndian())
+ return MEM_read64(memPtr);
+ else
+ return MEM_swap64(MEM_read64(memPtr));
+}
+
+MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
+{
+ if (MEM_isLittleEndian())
+ MEM_write64(memPtr, val64);
+ else
+ MEM_write64(memPtr, MEM_swap64(val64));
+}
+
+MEM_STATIC size_t MEM_readLEST(const void* memPtr)
+{
+ if (MEM_32bits())
+ return (size_t)MEM_readLE32(memPtr);
+ else
+ return (size_t)MEM_readLE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
+{
+ if (MEM_32bits())
+ MEM_writeLE32(memPtr, (U32)val);
+ else
+ MEM_writeLE64(memPtr, (U64)val);
+}
+
+/*=== Big endian r/w ===*/
+
+MEM_STATIC U32 MEM_readBE32(const void* memPtr)
+{
+ if (MEM_isLittleEndian())
+ return MEM_swap32(MEM_read32(memPtr));
+ else
+ return MEM_read32(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
+{
+ if (MEM_isLittleEndian())
+ MEM_write32(memPtr, MEM_swap32(val32));
+ else
+ MEM_write32(memPtr, val32);
+}
+
+MEM_STATIC U64 MEM_readBE64(const void* memPtr)
+{
+ if (MEM_isLittleEndian())
+ return MEM_swap64(MEM_read64(memPtr));
+ else
+ return MEM_read64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
+{
+ if (MEM_isLittleEndian())
+ MEM_write64(memPtr, MEM_swap64(val64));
+ else
+ MEM_write64(memPtr, val64);
+}
+
+MEM_STATIC size_t MEM_readBEST(const void* memPtr)
+{
+ if (MEM_32bits())
+ return (size_t)MEM_readBE32(memPtr);
+ else
+ return (size_t)MEM_readBE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
+{
+ if (MEM_32bits())
+ MEM_writeBE32(memPtr, (U32)val);
+ else
+ MEM_writeBE64(memPtr, (U64)val);
+}
+
+/* code only tested on 32 and 64 bits systems */
+MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* MEM_H_MODULE */
diff --git a/contrib/zstd/pool.c b/contrib/zstd/pool.c
new file mode 100644
index 0000000..f3d9d08
--- /dev/null
+++ b/contrib/zstd/pool.c
@@ -0,0 +1,371 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* ====== Dependencies ======= */
+#include "zstd_deps.h" /* size_t */
+#include "debug.h" /* assert */
+#include "zstd_internal.h" /* ZSTD_customCalloc, ZSTD_customFree */
+#include "pool.h"
+
+/* ====== Compiler specifics ====== */
+#if defined(_MSC_VER)
+# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
+#endif
+
+
+#ifdef ZSTD_MULTITHREAD
+
+#include "threading.h" /* pthread adaptation */
+
+/* A job is a function and an opaque argument */
+typedef struct POOL_job_s {
+ POOL_function function;
+ void *opaque;
+} POOL_job;
+
+struct POOL_ctx_s {
+ ZSTD_customMem customMem;
+ /* Keep track of the threads */
+ ZSTD_pthread_t* threads;
+ size_t threadCapacity;
+ size_t threadLimit;
+
+ /* The queue is a circular buffer */
+ POOL_job *queue;
+ size_t queueHead;
+ size_t queueTail;
+ size_t queueSize;
+
+ /* The number of threads working on jobs */
+ size_t numThreadsBusy;
+ /* Indicates if the queue is empty */
+ int queueEmpty;
+
+ /* The mutex protects the queue */
+ ZSTD_pthread_mutex_t queueMutex;
+ /* Condition variable for pushers to wait on when the queue is full */
+ ZSTD_pthread_cond_t queuePushCond;
+ /* Condition variables for poppers to wait on when the queue is empty */
+ ZSTD_pthread_cond_t queuePopCond;
+ /* Indicates if the queue is shutting down */
+ int shutdown;
+};
+
+/* POOL_thread() :
+ * Work thread for the thread pool.
+ * Waits for jobs and executes them.
+ * @returns : NULL on failure else non-null.
+ */
+static void* POOL_thread(void* opaque) {
+ POOL_ctx* const ctx = (POOL_ctx*)opaque;
+ if (!ctx) { return NULL; }
+ for (;;) {
+ /* Lock the mutex and wait for a non-empty queue or until shutdown */
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+
+ while ( ctx->queueEmpty
+ || (ctx->numThreadsBusy >= ctx->threadLimit) ) {
+ if (ctx->shutdown) {
+ /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit),
+ * a few threads will be shutdown while !queueEmpty,
+ * but enough threads will remain active to finish the queue */
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ return opaque;
+ }
+ ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
+ }
+ /* Pop a job off the queue */
+ { POOL_job const job = ctx->queue[ctx->queueHead];
+ ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
+ ctx->numThreadsBusy++;
+ ctx->queueEmpty = (ctx->queueHead == ctx->queueTail);
+ /* Unlock the mutex, signal a pusher, and run the job */
+ ZSTD_pthread_cond_signal(&ctx->queuePushCond);
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+
+ job.function(job.opaque);
+
+ /* If the intended queue size was 0, signal after finishing job */
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ ctx->numThreadsBusy--;
+ ZSTD_pthread_cond_signal(&ctx->queuePushCond);
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ }
+ } /* for (;;) */
+ assert(0); /* Unreachable */
+}
+
+/* ZSTD_createThreadPool() : public access point */
+POOL_ctx* ZSTD_createThreadPool(size_t numThreads) {
+ return POOL_create (numThreads, 0);
+}
+
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
+ return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
+}
+
+POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
+ ZSTD_customMem customMem)
+{
+ POOL_ctx* ctx;
+ /* Check parameters */
+ if (!numThreads) { return NULL; }
+ /* Allocate the context and zero initialize */
+ ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem);
+ if (!ctx) { return NULL; }
+ /* Initialize the job queue.
+ * It needs one extra space since one space is wasted to differentiate
+ * empty and full queues.
+ */
+ ctx->queueSize = queueSize + 1;
+ ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem);
+ ctx->queueHead = 0;
+ ctx->queueTail = 0;
+ ctx->numThreadsBusy = 0;
+ ctx->queueEmpty = 1;
+ {
+ int error = 0;
+ error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
+ error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
+ error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
+ if (error) { POOL_free(ctx); return NULL; }
+ }
+ ctx->shutdown = 0;
+ /* Allocate space for the thread handles */
+ ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
+ ctx->threadCapacity = 0;
+ ctx->customMem = customMem;
+ /* Check for errors */
+ if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
+ /* Initialize the threads */
+ { size_t i;
+ for (i = 0; i < numThreads; ++i) {
+ if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
+ ctx->threadCapacity = i;
+ POOL_free(ctx);
+ return NULL;
+ } }
+ ctx->threadCapacity = numThreads;
+ ctx->threadLimit = numThreads;
+ }
+ return ctx;
+}
+
+/*! POOL_join() :
+ Shutdown the queue, wake any sleeping threads, and join all of the threads.
+*/
+static void POOL_join(POOL_ctx* ctx) {
+ /* Shut down the queue */
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ ctx->shutdown = 1;
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ /* Wake up sleeping threads */
+ ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
+ ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
+ /* Join all of the threads */
+ { size_t i;
+ for (i = 0; i < ctx->threadCapacity; ++i) {
+ ZSTD_pthread_join(ctx->threads[i]); /* note : could fail */
+ } }
+}
+
+void POOL_free(POOL_ctx *ctx) {
+ if (!ctx) { return; }
+ POOL_join(ctx);
+ ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
+ ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
+ ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
+ ZSTD_customFree(ctx->queue, ctx->customMem);
+ ZSTD_customFree(ctx->threads, ctx->customMem);
+ ZSTD_customFree(ctx, ctx->customMem);
+}
+
+/*! POOL_joinJobs() :
+ * Waits for all queued jobs to finish executing.
+ */
+void POOL_joinJobs(POOL_ctx* ctx) {
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) {
+ ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
+ }
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+}
+
+void ZSTD_freeThreadPool (ZSTD_threadPool* pool) {
+ POOL_free (pool);
+}
+
+size_t POOL_sizeof(const POOL_ctx* ctx) {
+ if (ctx==NULL) return 0; /* supports sizeof NULL */
+ return sizeof(*ctx)
+ + ctx->queueSize * sizeof(POOL_job)
+ + ctx->threadCapacity * sizeof(ZSTD_pthread_t);
+}
+
+
+/* @return : 0 on success, 1 on error */
+static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
+{
+ if (numThreads <= ctx->threadCapacity) {
+ if (!numThreads) return 1;
+ ctx->threadLimit = numThreads;
+ return 0;
+ }
+ /* numThreads > threadCapacity */
+ { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
+ if (!threadPool) return 1;
+ /* replace existing thread pool */
+ ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
+ ZSTD_customFree(ctx->threads, ctx->customMem);
+ ctx->threads = threadPool;
+ /* Initialize additional threads */
+ { size_t threadId;
+ for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
+ if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
+ ctx->threadCapacity = threadId;
+ return 1;
+ } }
+ } }
+ /* successfully expanded */
+ ctx->threadCapacity = numThreads;
+ ctx->threadLimit = numThreads;
+ return 0;
+}
+
+/* @return : 0 on success, 1 on error */
+int POOL_resize(POOL_ctx* ctx, size_t numThreads)
+{
+ int result;
+ if (ctx==NULL) return 1;
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ result = POOL_resize_internal(ctx, numThreads);
+ ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ return result;
+}
+
+/**
+ * Returns 1 if the queue is full and 0 otherwise.
+ *
+ * When queueSize is 1 (pool was created with an intended queueSize of 0),
+ * then a queue is empty if there is a thread free _and_ no job is waiting.
+ */
+static int isQueueFull(POOL_ctx const* ctx) {
+ if (ctx->queueSize > 1) {
+ return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
+ } else {
+ return (ctx->numThreadsBusy == ctx->threadLimit) ||
+ !ctx->queueEmpty;
+ }
+}
+
+
+static void
+POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
+{
+ POOL_job job;
+ job.function = function;
+ job.opaque = opaque;
+ assert(ctx != NULL);
+ if (ctx->shutdown) return;
+
+ ctx->queueEmpty = 0;
+ ctx->queue[ctx->queueTail] = job;
+ ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
+ ZSTD_pthread_cond_signal(&ctx->queuePopCond);
+}
+
+void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque)
+{
+ assert(ctx != NULL);
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ /* Wait until there is space in the queue for the new job */
+ while (isQueueFull(ctx) && (!ctx->shutdown)) {
+ ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
+ }
+ POOL_add_internal(ctx, function, opaque);
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+}
+
+
+int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque)
+{
+ assert(ctx != NULL);
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+ if (isQueueFull(ctx)) {
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ return 0;
+ }
+ POOL_add_internal(ctx, function, opaque);
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+ return 1;
+}
+
+
+#else /* ZSTD_MULTITHREAD not defined */
+
+/* ========================== */
+/* No multi-threading support */
+/* ========================== */
+
+
+/* We don't need any data, but if it is empty, malloc() might return NULL. */
+struct POOL_ctx_s {
+ int dummy;
+};
+static POOL_ctx g_poolCtx;
+
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
+ return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
+}
+
+POOL_ctx*
+POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem)
+{
+ (void)numThreads;
+ (void)queueSize;
+ (void)customMem;
+ return &g_poolCtx;
+}
+
+void POOL_free(POOL_ctx* ctx) {
+ assert(!ctx || ctx == &g_poolCtx);
+ (void)ctx;
+}
+
+void POOL_joinJobs(POOL_ctx* ctx){
+ assert(!ctx || ctx == &g_poolCtx);
+ (void)ctx;
+}
+
+int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
+ (void)ctx; (void)numThreads;
+ return 0;
+}
+
+void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
+ (void)ctx;
+ function(opaque);
+}
+
+int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
+ (void)ctx;
+ function(opaque);
+ return 1;
+}
+
+size_t POOL_sizeof(const POOL_ctx* ctx) {
+ if (ctx==NULL) return 0; /* supports sizeof NULL */
+ assert(ctx == &g_poolCtx);
+ return sizeof(*ctx);
+}
+
+#endif /* ZSTD_MULTITHREAD */
diff --git a/contrib/zstd/pool.h b/contrib/zstd/pool.h
new file mode 100644
index 0000000..14e883f
--- /dev/null
+++ b/contrib/zstd/pool.h
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef POOL_H
+#define POOL_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+#include "zstd_deps.h"
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */
+#include "zstd.h"
+
+typedef struct POOL_ctx_s POOL_ctx;
+
+/*! POOL_create() :
+ * Create a thread pool with at most `numThreads` threads.
+ * `numThreads` must be at least 1.
+ * The maximum number of queued jobs before blocking is `queueSize`.
+ * @return : POOL_ctx pointer on success, else NULL.
+*/
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
+
+POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
+ ZSTD_customMem customMem);
+
+/*! POOL_free() :
+ * Free a thread pool returned by POOL_create().
+ */
+void POOL_free(POOL_ctx* ctx);
+
+
+/*! POOL_joinJobs() :
+ * Waits for all queued jobs to finish executing.
+ */
+void POOL_joinJobs(POOL_ctx* ctx);
+
+/*! POOL_resize() :
+ * Expands or shrinks pool's number of threads.
+ * This is more efficient than releasing + creating a new context,
+ * since it tries to preserve and re-use existing threads.
+ * `numThreads` must be at least 1.
+ * @return : 0 when resize was successful,
+ * !0 (typically 1) if there is an error.
+ * note : only numThreads can be resized, queueSize remains unchanged.
+ */
+int POOL_resize(POOL_ctx* ctx, size_t numThreads);
+
+/*! POOL_sizeof() :
+ * @return threadpool memory usage
+ * note : compatible with NULL (returns 0 in this case)
+ */
+size_t POOL_sizeof(const POOL_ctx* ctx);
+
+/*! POOL_function :
+ * The function type that can be added to a thread pool.
+ */
+typedef void (*POOL_function)(void*);
+
+/*! POOL_add() :
+ * Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
+ * Possibly blocks until there is room in the queue.
+ * Note : The function may be executed asynchronously,
+ * therefore, `opaque` must live until function has been completed.
+ */
+void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
+
+
+/*! POOL_tryAdd() :
+ * Add the job `function(opaque)` to thread pool _if_ a queue slot is available.
+ * Returns immediately even if not (does not block).
+ * @return : 1 if successful, 0 if not.
+ */
+int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif
diff --git a/contrib/zstd/portability_macros.h b/contrib/zstd/portability_macros.h
new file mode 100644
index 0000000..8fd6ea8
--- /dev/null
+++ b/contrib/zstd/portability_macros.h
@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_PORTABILITY_MACROS_H
+#define ZSTD_PORTABILITY_MACROS_H
+
+/**
+ * This header file contains macro definitions to support portability.
+ * This header is shared between C and ASM code, so it MUST only
+ * contain macro definitions. It MUST not contain any C code.
+ *
+ * This header ONLY defines macros to detect platforms/feature support.
+ *
+ */
+
+
+/* compat. with non-clang compilers */
+#ifndef __has_attribute
+ #define __has_attribute(x) 0
+#endif
+
+/* compat. with non-clang compilers */
+#ifndef __has_builtin
+# define __has_builtin(x) 0
+#endif
+
+/* compat. with non-clang compilers */
+#ifndef __has_feature
+# define __has_feature(x) 0
+#endif
+
+/* detects whether we are being compiled under msan */
+#ifndef ZSTD_MEMORY_SANITIZER
+# if __has_feature(memory_sanitizer)
+# define ZSTD_MEMORY_SANITIZER 1
+# else
+# define ZSTD_MEMORY_SANITIZER 0
+# endif
+#endif
+
+/* detects whether we are being compiled under asan */
+#ifndef ZSTD_ADDRESS_SANITIZER
+# if __has_feature(address_sanitizer)
+# define ZSTD_ADDRESS_SANITIZER 1
+# elif defined(__SANITIZE_ADDRESS__)
+# define ZSTD_ADDRESS_SANITIZER 1
+# else
+# define ZSTD_ADDRESS_SANITIZER 0
+# endif
+#endif
+
+/* detects whether we are being compiled under dfsan */
+#ifndef ZSTD_DATAFLOW_SANITIZER
+# if __has_feature(dataflow_sanitizer)
+# define ZSTD_DATAFLOW_SANITIZER 1
+# else
+# define ZSTD_DATAFLOW_SANITIZER 0
+# endif
+#endif
+
+/* Mark the internal assembly functions as hidden */
+#ifdef __ELF__
+# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
+#else
+# define ZSTD_HIDE_ASM_FUNCTION(func)
+#endif
+
+/* Enable runtime BMI2 dispatch based on the CPU.
+ * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
+ */
+#ifndef DYNAMIC_BMI2
+ #if ((defined(__clang__) && __has_attribute(__target__)) \
+ || (defined(__GNUC__) \
+ && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
+ && (defined(__x86_64__) || defined(_M_X64)) \
+ && !defined(__BMI2__)
+ # define DYNAMIC_BMI2 1
+ #else
+ # define DYNAMIC_BMI2 0
+ #endif
+#endif
+
+/**
+ * Only enable assembly for GNUC compatible compilers,
+ * because other platforms may not support GAS assembly syntax.
+ *
+ * Only enable assembly for Linux / MacOS, other platforms may
+ * work, but they haven't been tested. This could likely be
+ * extended to BSD systems.
+ *
+ * Disable assembly when MSAN is enabled, because MSAN requires
+ * 100% of code to be instrumented to work.
+ */
+#if defined(__GNUC__)
+# if defined(__linux__) || defined(__linux) || defined(__APPLE__)
+# if ZSTD_MEMORY_SANITIZER
+# define ZSTD_ASM_SUPPORTED 0
+# elif ZSTD_DATAFLOW_SANITIZER
+# define ZSTD_ASM_SUPPORTED 0
+# else
+# define ZSTD_ASM_SUPPORTED 1
+# endif
+# else
+# define ZSTD_ASM_SUPPORTED 0
+# endif
+#else
+# define ZSTD_ASM_SUPPORTED 0
+#endif
+
+/**
+ * Determines whether we should enable assembly for x86-64
+ * with BMI2.
+ *
+ * Enable if all of the following conditions hold:
+ * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM
+ * - Assembly is supported
+ * - We are compiling for x86-64 and either:
+ * - DYNAMIC_BMI2 is enabled
+ * - BMI2 is supported at compile time
+ */
+#if !defined(ZSTD_DISABLE_ASM) && \
+ ZSTD_ASM_SUPPORTED && \
+ defined(__x86_64__) && \
+ (DYNAMIC_BMI2 || defined(__BMI2__))
+# define ZSTD_ENABLE_ASM_X86_64_BMI2 1
+#else
+# define ZSTD_ENABLE_ASM_X86_64_BMI2 0
+#endif
+
+/*
+ * For x86 ELF targets, add .note.gnu.property section for Intel CET in
+ * assembly sources when CET is enabled.
+ *
+ * Additionally, any function that may be called indirectly must begin
+ * with ZSTD_CET_ENDBRANCH.
+ */
+#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \
+ && defined(__has_include)
+# if __has_include(<cet.h>)
+# include <cet.h>
+# define ZSTD_CET_ENDBRANCH _CET_ENDBR
+# endif
+#endif
+
+#ifndef ZSTD_CET_ENDBRANCH
+# define ZSTD_CET_ENDBRANCH
+#endif
+
+#endif /* ZSTD_PORTABILITY_MACROS_H */
diff --git a/contrib/zstd/zdict.h b/contrib/zstd/zdict.h
new file mode 100644
index 0000000..2268f94
--- /dev/null
+++ b/contrib/zstd/zdict.h
@@ -0,0 +1,474 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef ZSTD_ZDICT_H
+#define ZSTD_ZDICT_H
+
+/*====== Dependencies ======*/
+#include <stddef.h> /* size_t */
+
+
+/* ===== ZDICTLIB_API : control library symbols visibility ===== */
+#ifndef ZDICTLIB_VISIBLE
+ /* Backwards compatibility with old macro name */
+# ifdef ZDICTLIB_VISIBILITY
+# define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY
+# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default")))
+# else
+# define ZDICTLIB_VISIBLE
+# endif
+#endif
+
+#ifndef ZDICTLIB_HIDDEN
+# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+# else
+# define ZDICTLIB_HIDDEN
+# endif
+#endif
+
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define ZDICTLIB_API ZDICTLIB_VISIBLE
+#endif
+
+/*******************************************************************************
+ * Zstd dictionary builder
+ *
+ * FAQ
+ * ===
+ * Why should I use a dictionary?
+ * ------------------------------
+ *
+ * Zstd can use dictionaries to improve compression ratio of small data.
+ * Traditionally small files don't compress well because there is very little
+ * repetition in a single sample, since it is small. But, if you are compressing
+ * many similar files, like a bunch of JSON records that share the same
+ * structure, you can train a dictionary on ahead of time on some samples of
+ * these files. Then, zstd can use the dictionary to find repetitions that are
+ * present across samples. This can vastly improve compression ratio.
+ *
+ * When is a dictionary useful?
+ * ----------------------------
+ *
+ * Dictionaries are useful when compressing many small files that are similar.
+ * The larger a file is, the less benefit a dictionary will have. Generally,
+ * we don't expect dictionary compression to be effective past 100KB. And the
+ * smaller a file is, the more we would expect the dictionary to help.
+ *
+ * How do I use a dictionary?
+ * --------------------------
+ *
+ * Simply pass the dictionary to the zstd compressor with
+ * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to
+ * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other
+ * more advanced functions that allow selecting some options, see zstd.h for
+ * complete documentation.
+ *
+ * What is a zstd dictionary?
+ * --------------------------
+ *
+ * A zstd dictionary has two pieces: Its header, and its content. The header
+ * contains a magic number, the dictionary ID, and entropy tables. These
+ * entropy tables allow zstd to save on header costs in the compressed file,
+ * which really matters for small data. The content is just bytes, which are
+ * repeated content that is common across many samples.
+ *
+ * What is a raw content dictionary?
+ * ---------------------------------
+ *
+ * A raw content dictionary is just bytes. It doesn't have a zstd dictionary
+ * header, a dictionary ID, or entropy tables. Any buffer is a valid raw
+ * content dictionary.
+ *
+ * How do I train a dictionary?
+ * ----------------------------
+ *
+ * Gather samples from your use case. These samples should be similar to each
+ * other. If you have several use cases, you could try to train one dictionary
+ * per use case.
+ *
+ * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your
+ * dictionary. There are a few advanced versions of this function, but this
+ * is a great starting point. If you want to further tune your dictionary
+ * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow
+ * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`.
+ *
+ * If the dictionary training function fails, that is likely because you
+ * either passed too few samples, or a dictionary would not be effective
+ * for your data. Look at the messages that the dictionary trainer printed,
+ * if it doesn't say too few samples, then a dictionary would not be effective.
+ *
+ * How large should my dictionary be?
+ * ----------------------------------
+ *
+ * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB.
+ * The zstd CLI defaults to a 110KB dictionary. You likely don't need a
+ * dictionary larger than that. But, most use cases can get away with a
+ * smaller dictionary. The advanced dictionary builders can automatically
+ * shrink the dictionary for you, and select the smallest size that doesn't
+ * hurt compression ratio too much. See the `shrinkDict` parameter.
+ * A smaller dictionary can save memory, and potentially speed up
+ * compression.
+ *
+ * How many samples should I provide to the dictionary builder?
+ * ------------------------------------------------------------
+ *
+ * We generally recommend passing ~100x the size of the dictionary
+ * in samples. A few thousand should suffice. Having too few samples
+ * can hurt the dictionaries effectiveness. Having more samples will
+ * only improve the dictionaries effectiveness. But having too many
+ * samples can slow down the dictionary builder.
+ *
+ * How do I determine if a dictionary will be effective?
+ * -----------------------------------------------------
+ *
+ * Simply train a dictionary and try it out. You can use zstd's built in
+ * benchmarking tool to test the dictionary effectiveness.
+ *
+ * # Benchmark levels 1-3 without a dictionary
+ * zstd -b1e3 -r /path/to/my/files
+ * # Benchmark levels 1-3 with a dictionary
+ * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary
+ *
+ * When should I retrain a dictionary?
+ * -----------------------------------
+ *
+ * You should retrain a dictionary when its effectiveness drops. Dictionary
+ * effectiveness drops as the data you are compressing changes. Generally, we do
+ * expect dictionaries to "decay" over time, as your data changes, but the rate
+ * at which they decay depends on your use case. Internally, we regularly
+ * retrain dictionaries, and if the new dictionary performs significantly
+ * better than the old dictionary, we will ship the new dictionary.
+ *
+ * I have a raw content dictionary, how do I turn it into a zstd dictionary?
+ * -------------------------------------------------------------------------
+ *
+ * If you have a raw content dictionary, e.g. by manually constructing it, or
+ * using a third-party dictionary builder, you can turn it into a zstd
+ * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to
+ * provide some samples of the data. It will add the zstd header to the
+ * raw content, which contains a dictionary ID and entropy tables, which
+ * will improve compression ratio, and allow zstd to write the dictionary ID
+ * into the frame, if you so choose.
+ *
+ * Do I have to use zstd's dictionary builder?
+ * -------------------------------------------
+ *
+ * No! You can construct dictionary content however you please, it is just
+ * bytes. It will always be valid as a raw content dictionary. If you want
+ * a zstd dictionary, which can improve compression ratio, use
+ * `ZDICT_finalizeDictionary()`.
+ *
+ * What is the attack surface of a zstd dictionary?
+ * ------------------------------------------------
+ *
+ * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so
+ * zstd should never crash, or access out-of-bounds memory no matter what
+ * the dictionary is. However, if an attacker can control the dictionary
+ * during decompression, they can cause zstd to generate arbitrary bytes,
+ * just like if they controlled the compressed data.
+ *
+ ******************************************************************************/
+
+
+/*! ZDICT_trainFromBuffer():
+ * Train a dictionary from an array of samples.
+ * Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4,
+ * f=20, and accel=1.
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ * The resulting dictionary will be saved into `dictBuffer`.
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * Note: Dictionary training will fail if there are not enough samples to construct a
+ * dictionary, or if most of the samples are too small (< 8 bytes being the lower limit).
+ * If dictionary training fails, you should use zstd without a dictionary, as the dictionary
+ * would've been ineffective anyways. If you believe your samples would benefit from a dictionary
+ * please open an issue with details, and we can look into it.
+ * Note: ZDICT_trainFromBuffer()'s memory usage is about 6 MB.
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+ */
+ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer,
+ const size_t* samplesSizes, unsigned nbSamples);
+
+typedef struct {
+ int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */
+ unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
+ unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value)
+ * NOTE: The zstd format reserves some dictionary IDs for future use.
+ * You may use them in private settings, but be warned that they
+ * may be used by zstd in a public dictionary registry in the future.
+ * These dictionary IDs are:
+ * - low range : <= 32767
+ * - high range : >= (2^31)
+ */
+} ZDICT_params_t;
+
+/*! ZDICT_finalizeDictionary():
+ * Given a custom content as a basis for dictionary, and a set of samples,
+ * finalize dictionary by adding headers and statistics according to the zstd
+ * dictionary format.
+ *
+ * Samples must be stored concatenated in a flat buffer `samplesBuffer`,
+ * supplied with an array of sizes `samplesSizes`, providing the size of each
+ * sample in order. The samples are used to construct the statistics, so they
+ * should be representative of what you will compress with this dictionary.
+ *
+ * The compression level can be set in `parameters`. You should pass the
+ * compression level you expect to use in production. The statistics for each
+ * compression level differ, so tuning the dictionary for the compression level
+ * can help quite a bit.
+ *
+ * You can set an explicit dictionary ID in `parameters`, or allow us to pick
+ * a random dictionary ID for you, but we can't guarantee no collisions.
+ *
+ * The dstDictBuffer and the dictContent may overlap, and the content will be
+ * appended to the end of the header. If the header + the content doesn't fit in
+ * maxDictSize the beginning of the content is truncated to make room, since it
+ * is presumed that the most profitable content is at the end of the dictionary,
+ * since that is the cheapest to reference.
+ *
+ * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
+ *
+ * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
+ * or an error code, which can be tested by ZDICT_isError().
+ * Note: ZDICT_finalizeDictionary() will push notifications into stderr if
+ * instructed to, using notificationLevel>0.
+ * NOTE: This function currently may fail in several edge cases including:
+ * * Not enough samples
+ * * Samples are uncompressible
+ * * Samples are all exactly the same
+ */
+ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
+ const void* dictContent, size_t dictContentSize,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_params_t parameters);
+
+
+/*====== Helper functions ======*/
+ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
+ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
+ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
+ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
+
+#endif /* ZSTD_ZDICT_H */
+
+#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC)
+#define ZSTD_ZDICT_H_STATIC
+
+/* This can be overridden externally to hide static symbols. */
+#ifndef ZDICTLIB_STATIC_API
+# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE
+# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE
+# else
+# define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE
+# endif
+#endif
+
+/* ====================================================================================
+ * The definitions in this section are considered experimental.
+ * They should never be used with a dynamic library, as they may change in the future.
+ * They are provided for advanced usages.
+ * Use them only in association with static linking.
+ * ==================================================================================== */
+
+#define ZDICT_DICTSIZE_MIN 256
+/* Deprecated: Remove in v1.6.0 */
+#define ZDICT_CONTENTSIZE_MIN 128
+
+/*! ZDICT_cover_params_t:
+ * k and d are the only required parameters.
+ * For others, value 0 means default.
+ */
+typedef struct {
+ unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
+ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
+ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
+ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
+ double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
+ ZDICT_params_t zParams;
+} ZDICT_cover_params_t;
+
+typedef struct {
+ unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
+ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
+ unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/
+ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
+ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
+ double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
+ unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
+
+ ZDICT_params_t zParams;
+} ZDICT_fastCover_params_t;
+
+/*! ZDICT_trainFromBuffer_cover():
+ * Train a dictionary from an array of samples using the COVER algorithm.
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ * The resulting dictionary will be saved into `dictBuffer`.
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * See ZDICT_trainFromBuffer() for details on failure modes.
+ * Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte.
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+ */
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
+ void *dictBuffer, size_t dictBufferCapacity,
+ const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
+ ZDICT_cover_params_t parameters);
+
+/*! ZDICT_optimizeTrainFromBuffer_cover():
+ * The same requirements as above hold for all the parameters except `parameters`.
+ * This function tries many parameter combinations and picks the best parameters.
+ * `*parameters` is filled with the best parameters found,
+ * dictionary constructed with those parameters is stored in `dictBuffer`.
+ *
+ * All of the parameters d, k, steps are optional.
+ * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
+ * if steps is zero it defaults to its default value.
+ * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
+ *
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * On success `*parameters` contains the parameters selected.
+ * See ZDICT_trainFromBuffer() for details on failure modes.
+ * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
+ */
+ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
+ void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_cover_params_t* parameters);
+
+/*! ZDICT_trainFromBuffer_fastCover():
+ * Train a dictionary from an array of samples using a modified version of COVER algorithm.
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ * d and k are required.
+ * All other parameters are optional, will use default values if not provided
+ * The resulting dictionary will be saved into `dictBuffer`.
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * See ZDICT_trainFromBuffer() for details on failure modes.
+ * Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory.
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+ */
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
+ size_t dictBufferCapacity, const void *samplesBuffer,
+ const size_t *samplesSizes, unsigned nbSamples,
+ ZDICT_fastCover_params_t parameters);
+
+/*! ZDICT_optimizeTrainFromBuffer_fastCover():
+ * The same requirements as above hold for all the parameters except `parameters`.
+ * This function tries many parameter combinations (specifically, k and d combinations)
+ * and picks the best parameters. `*parameters` is filled with the best parameters found,
+ * dictionary constructed with those parameters is stored in `dictBuffer`.
+ * All of the parameters d, k, steps, f, and accel are optional.
+ * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
+ * if steps is zero it defaults to its default value.
+ * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
+ * If f is zero, default value of 20 is used.
+ * If accel is zero, default value of 1 is used.
+ *
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * On success `*parameters` contains the parameters selected.
+ * See ZDICT_trainFromBuffer() for details on failure modes.
+ * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
+ */
+ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
+ size_t dictBufferCapacity, const void* samplesBuffer,
+ const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_fastCover_params_t* parameters);
+
+typedef struct {
+ unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
+ ZDICT_params_t zParams;
+} ZDICT_legacy_params_t;
+
+/*! ZDICT_trainFromBuffer_legacy():
+ * Train a dictionary from an array of samples.
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ * The resulting dictionary will be saved into `dictBuffer`.
+ * `parameters` is optional and can be provided with values set to 0 to mean "default".
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ * or an error code, which can be tested with ZDICT_isError().
+ * See ZDICT_trainFromBuffer() for details on failure modes.
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+ * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
+ */
+ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy(
+ void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_legacy_params_t parameters);
+
+
+/* Deprecation warnings */
+/* It is generally possible to disable deprecation warnings from compiler,
+ for example with -Wno-deprecated-declarations for gcc
+ or _CRT_SECURE_NO_WARNINGS in Visual.
+ Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
+#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
+# define ZDICT_DEPRECATED(message) /* disable deprecation warnings */
+#else
+# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+# define ZDICT_DEPRECATED(message) [[deprecated(message)]]
+# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
+# define ZDICT_DEPRECATED(message) __attribute__((deprecated(message)))
+# elif (ZDICT_GCC_VERSION >= 301)
+# define ZDICT_DEPRECATED(message) __attribute__((deprecated))
+# elif defined(_MSC_VER)
+# define ZDICT_DEPRECATED(message) __declspec(deprecated(message))
+# else
+# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
+# define ZDICT_DEPRECATED(message)
+# endif
+#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
+
+ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
+ZDICTLIB_STATIC_API
+size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
+
+
+#endif /* ZSTD_ZDICT_H_STATIC */
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/contrib/zstd/zstd.h b/contrib/zstd/zstd.h
new file mode 100644
index 0000000..95aac07
--- /dev/null
+++ b/contrib/zstd/zstd.h
@@ -0,0 +1,2974 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef ZSTD_H_235446
+#define ZSTD_H_235446
+
+/* ====== Dependencies ======*/
+#include <limits.h> /* INT_MAX */
+#include <stddef.h> /* size_t */
+
+
+/* ===== ZSTDLIB_API : control library symbols visibility ===== */
+#ifndef ZSTDLIB_VISIBLE
+ /* Backwards compatibility with old macro name */
+# ifdef ZSTDLIB_VISIBILITY
+# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY
+# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
+# else
+# define ZSTDLIB_VISIBLE
+# endif
+#endif
+
+#ifndef ZSTDLIB_HIDDEN
+# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+# else
+# define ZSTDLIB_HIDDEN
+# endif
+#endif
+
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define ZSTDLIB_API ZSTDLIB_VISIBLE
+#endif
+
+/* Deprecation warnings :
+ * Should these warnings be a problem, it is generally possible to disable them,
+ * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
+ * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
+ */
+#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
+# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */
+#else
+# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+# define ZSTD_DEPRECATED(message) [[deprecated(message)]]
+# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
+# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
+# elif defined(__GNUC__) && (__GNUC__ >= 3)
+# define ZSTD_DEPRECATED(message) __attribute__((deprecated))
+# elif defined(_MSC_VER)
+# define ZSTD_DEPRECATED(message) __declspec(deprecated(message))
+# else
+# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
+# define ZSTD_DEPRECATED(message)
+# endif
+#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
+
+
+/*******************************************************************************
+ Introduction
+
+ zstd, short for Zstandard, is a fast lossless compression algorithm, targeting
+ real-time compression scenarios at zlib-level and better compression ratios.
+ The zstd compression library provides in-memory compression and decompression
+ functions.
+
+ The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
+ which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
+ caution, as they require more memory. The library also offers negative
+ compression levels, which extend the range of speed vs. ratio preferences.
+ The lower the level, the faster the speed (at the cost of compression).
+
+ Compression can be done in:
+ - a single step (described as Simple API)
+ - a single step, reusing a context (described as Explicit context)
+ - unbounded multiple steps (described as Streaming compression)
+
+ The compression ratio achievable on small data can be highly improved using
+ a dictionary. Dictionary compression can be performed in:
+ - a single step (described as Simple dictionary API)
+ - a single step, reusing a dictionary (described as Bulk-processing
+ dictionary API)
+
+ Advanced experimental functions can be accessed using
+ `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.
+
+ Advanced experimental APIs should never be used with a dynamically-linked
+ library. They are not "stable"; their definitions or signatures may change in
+ the future. Only static linking is allowed.
+*******************************************************************************/
+
+/*------ Version ------*/
+#define ZSTD_VERSION_MAJOR 1
+#define ZSTD_VERSION_MINOR 5
+#define ZSTD_VERSION_RELEASE 4
+#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
+
+/*! ZSTD_versionNumber() :
+ * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */
+ZSTDLIB_API unsigned ZSTD_versionNumber(void);
+
+#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
+#define ZSTD_QUOTE(str) #str
+#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
+#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
+
+/*! ZSTD_versionString() :
+ * Return runtime library version, like "1.4.5". Requires v1.3.0+. */
+ZSTDLIB_API const char* ZSTD_versionString(void);
+
+/* *************************************
+ * Default constant
+ ***************************************/
+#ifndef ZSTD_CLEVEL_DEFAULT
+# define ZSTD_CLEVEL_DEFAULT 3
+#endif
+
+/* *************************************
+ * Constants
+ ***************************************/
+
+/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
+#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
+#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */
+#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
+#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0
+
+#define ZSTD_BLOCKSIZELOG_MAX 17
+#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX)
+
+
+/***************************************
+* Simple API
+***************************************/
+/*! ZSTD_compress() :
+ * Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
+ * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
+ * @return : compressed size written into `dst` (<= `dstCapacity),
+ * or an error code if it fails (which can be tested using ZSTD_isError()). */
+ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel);
+
+/*! ZSTD_decompress() :
+ * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
+ * `dstCapacity` is an upper bound of originalSize to regenerate.
+ * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
+ * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+ * or an errorCode if it fails (which can be tested using ZSTD_isError()). */
+ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
+ const void* src, size_t compressedSize);
+
+/*! ZSTD_getFrameContentSize() : requires v1.3.0+
+ * `src` should point to the start of a ZSTD encoded frame.
+ * `srcSize` must be at least as large as the frame header.
+ * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
+ * @return : - decompressed size of `src` frame content, if known
+ * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+ * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
+ * note 1 : a 0 return value means the frame is valid but "empty".
+ * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
+ * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+ * In which case, it's necessary to use streaming mode to decompress data.
+ * Optionally, application can rely on some implicit limit,
+ * as ZSTD_decompress() only needs an upper bound of decompressed size.
+ * (For example, data could be necessarily cut into blocks <= 16 KB).
+ * note 3 : decompressed size is always present when compression is completed using single-pass functions,
+ * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
+ * note 4 : decompressed size can be very large (64-bits value),
+ * potentially larger than what local system can handle as a single memory segment.
+ * In which case, it's necessary to use streaming mode to decompress data.
+ * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ * Always ensure return value fits within application's authorized limits.
+ * Each application can set its own limits.
+ * note 6 : This function replaces ZSTD_getDecompressedSize() */
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
+ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
+
+/*! ZSTD_getDecompressedSize() :
+ * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
+ * Both functions work the same way, but ZSTD_getDecompressedSize() blends
+ * "empty", "unknown" and "error" results to the same return value (0),
+ * while ZSTD_getFrameContentSize() gives them separate return values.
+ * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
+ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+ZSTDLIB_API
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+
+/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
+ * `src` should point to the start of a ZSTD frame or skippable frame.
+ * `srcSize` must be >= first frame size
+ * @return : the compressed size of the first frame starting at `src`,
+ * suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
+ * or an error code if input is invalid */
+ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+
+
+/*====== Helper functions ======*/
+/* ZSTD_compressBound() :
+ * maximum compressed size in worst case single-pass scenario.
+ * When invoking `ZSTD_compress()` or any other one-pass compression function,
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ * as it eliminates one potential failure scenario,
+ * aka not enough room in dst buffer to write the compressed frame.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
+ * In which case, ZSTD_compressBound() will return an error code
+ * which can be tested using ZSTD_isError().
+ *
+ * ZSTD_COMPRESSBOUND() :
+ * same as ZSTD_compressBound(), but as a macro.
+ * It can be used to produce constants, which can be useful for static allocation,
+ * for example to size a static array on stack.
+ * Will produce constant value 0 if srcSize too large.
+ */
+#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
+#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
+ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+/* ZSTD_isError() :
+ * Most ZSTD_* functions returning a size_t value can be tested for error,
+ * using ZSTD_isError().
+ * @return 1 if error, 0 otherwise
+ */
+ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
+ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
+ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */
+ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
+ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */
+
+
+/***************************************
+* Explicit context
+***************************************/
+/*= Compression context
+ * When compressing many times,
+ * it is recommended to allocate a context just once,
+ * and re-use it for each successive compression operation.
+ * This will make workload friendlier for system's memory.
+ * Note : re-using context is just a speed / resource optimization.
+ * It doesn't change the compression ratio, which remains identical.
+ * Note 2 : In multi-threaded environments,
+ * use one different context per thread for parallel execution.
+ */
+typedef struct ZSTD_CCtx_s ZSTD_CCtx;
+ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
+ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */
+
+/*! ZSTD_compressCCtx() :
+ * Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
+ * Important : in order to behave similarly to `ZSTD_compress()`,
+ * this function compresses at requested compression level,
+ * __ignoring any other parameter__ .
+ * If any advanced parameter was set using the advanced API,
+ * they will all be reset. Only `compressionLevel` remains.
+ */
+ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel);
+
+/*= Decompression context
+ * When decompressing many times,
+ * it is recommended to allocate a context only once,
+ * and re-use it for each successive compression operation.
+ * This will make workload friendlier for system's memory.
+ * Use one context per thread for parallel execution. */
+typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
+ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */
+
+/*! ZSTD_decompressDCtx() :
+ * Same as ZSTD_decompress(),
+ * requires an allocated ZSTD_DCtx.
+ * Compatible with sticky parameters.
+ */
+ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize);
+
+
+/*********************************************
+* Advanced compression API (Requires v1.4.0+)
+**********************************************/
+
+/* API design :
+ * Parameters are pushed one by one into an existing context,
+ * using ZSTD_CCtx_set*() functions.
+ * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
+ * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
+ * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .
+ *
+ * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
+ *
+ * This API supersedes all other "advanced" API entry points in the experimental section.
+ * In the future, we expect to remove from experimental API entry points which are redundant with this API.
+ */
+
+
+/* Compression strategies, listed from fastest to strongest */
+typedef enum { ZSTD_fast=1,
+ ZSTD_dfast=2,
+ ZSTD_greedy=3,
+ ZSTD_lazy=4,
+ ZSTD_lazy2=5,
+ ZSTD_btlazy2=6,
+ ZSTD_btopt=7,
+ ZSTD_btultra=8,
+ ZSTD_btultra2=9
+ /* note : new strategies _might_ be added in the future.
+ Only the order (from fast to strong) is guaranteed */
+} ZSTD_strategy;
+
+typedef enum {
+
+ /* compression parameters
+ * Note: When compressing with a ZSTD_CDict these parameters are superseded
+ * by the parameters used to construct the ZSTD_CDict.
+ * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
+ ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
+ * Note that exact compression parameters are dynamically determined,
+ * depending on both compression level and srcSize (when known).
+ * Default level is ZSTD_CLEVEL_DEFAULT==3.
+ * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
+ * Note 1 : it's possible to pass a negative compression level.
+ * Note 2 : setting a level does not automatically set all other compression parameters
+ * to default. Setting this will however eventually dynamically impact the compression
+ * parameters which have not been manually set. The manually set
+ * ones will 'stick'. */
+ /* Advanced compression parameters :
+ * It's possible to pin down compression parameters to some specific values.
+ * In which case, these values are no longer dynamically selected by the compressor */
+ ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
+ * This will set a memory budget for streaming decompression,
+ * with larger values requiring more memory
+ * and typically compressing more.
+ * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
+ * Special: value 0 means "use default windowLog".
+ * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
+ * requires explicitly allowing such size at streaming decompression stage. */
+ ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
+ * Resulting memory usage is (1 << (hashLog+2)).
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
+ * Larger tables improve compression ratio of strategies <= dFast,
+ * and improve speed of strategies > dFast.
+ * Special: value 0 means "use default hashLog". */
+ ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2.
+ * Resulting memory usage is (1 << (chainLog+2)).
+ * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
+ * Larger tables result in better and slower compression.
+ * This parameter is useless for "fast" strategy.
+ * It's still useful when using "dfast" strategy,
+ * in which case it defines a secondary probe table.
+ * Special: value 0 means "use default chainLog". */
+ ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
+ * More attempts result in better and slower compression.
+ * This parameter is useless for "fast" and "dFast" strategies.
+ * Special: value 0 means "use default searchLog". */
+ ZSTD_c_minMatch=105, /* Minimum size of searched matches.
+ * Note that Zstandard can still find matches of smaller size,
+ * it just tweaks its search algorithm to look for this size and larger.
+ * Larger values increase compression and decompression speed, but decrease ratio.
+ * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
+ * Note that currently, for all strategies < btopt, effective minimum is 4.
+ * , for all strategies > fast, effective maximum is 6.
+ * Special: value 0 means "use default minMatchLength". */
+ ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
+ * For strategies btopt, btultra & btultra2:
+ * Length of Match considered "good enough" to stop search.
+ * Larger values make compression stronger, and slower.
+ * For strategy fast:
+ * Distance between match sampling.
+ * Larger values make compression faster, and weaker.
+ * Special: value 0 means "use default targetLength". */
+ ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition.
+ * The higher the value of selected strategy, the more complex it is,
+ * resulting in stronger and slower compression.
+ * Special: value 0 means "use default strategy". */
+ /* LDM mode parameters */
+ ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
+ * This parameter is designed to improve compression ratio
+ * for large inputs, by finding large matches at long distance.
+ * It increases memory usage and window size.
+ * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
+ * except when expressly set to a different value.
+ * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
+ * compression strategy >= ZSTD_btopt (== compression level 16+) */
+ ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
+ * Larger values increase memory usage and compression ratio,
+ * but decrease compression speed.
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
+ * default: windowlog - 7.
+ * Special: value 0 means "automatically determine hashlog". */
+ ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher.
+ * Larger/too small values usually decrease compression ratio.
+ * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
+ * Special: value 0 means "use default value" (default: 64). */
+ ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
+ * Larger values improve collision resolution but decrease compression speed.
+ * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
+ * Special: value 0 means "use default value" (default: 3). */
+ ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
+ * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
+ * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
+ * Larger values improve compression speed.
+ * Deviating far from default value will likely result in a compression ratio decrease.
+ * Special: value 0 means "automatically determine hashRateLog". */
+
+ /* frame parameters */
+ ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
+ * Content size must be known at the beginning of compression.
+ * This is automatically the case when using ZSTD_compress2(),
+ * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
+ ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
+ ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
+
+ /* multi-threading parameters */
+ /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
+ * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
+ * In a situation where it's unknown if the linked library supports multi-threading or not,
+ * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
+ */
+ ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
+ * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
+ * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
+ * while compression is performed in parallel, within worker thread(s).
+ * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
+ * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
+ * More workers improve speed, but also increase memory usage.
+ * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
+ * compression is performed inside Caller's thread, and all invocations are blocking */
+ ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
+ * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
+ * 0 means default, which is dynamically determined based on compression parameters.
+ * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
+ * The minimum size is automatically and transparently enforced. */
+ ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
+ * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
+ * It helps preserve compression ratio, while each job is compressed in parallel.
+ * This value is enforced only when nbWorkers >= 1.
+ * Larger values increase compression ratio, but decrease speed.
+ * Possible values range from 0 to 9 :
+ * - 0 means "default" : value will be determined by the library, depending on strategy
+ * - 1 means "no overlap"
+ * - 9 means "full overlap", using a full window size.
+ * Each intermediate rank increases/decreases load size by a factor 2 :
+ * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default
+ * default value varies between 6 and 9, depending on strategy */
+
+ /* note : additional experimental parameters are also available
+ * within the experimental section of the API.
+ * At the time of this writing, they include :
+ * ZSTD_c_rsyncable
+ * ZSTD_c_format
+ * ZSTD_c_forceMaxWindow
+ * ZSTD_c_forceAttachDict
+ * ZSTD_c_literalCompressionMode
+ * ZSTD_c_targetCBlockSize
+ * ZSTD_c_srcSizeHint
+ * ZSTD_c_enableDedicatedDictSearch
+ * ZSTD_c_stableInBuffer
+ * ZSTD_c_stableOutBuffer
+ * ZSTD_c_blockDelimiters
+ * ZSTD_c_validateSequences
+ * ZSTD_c_useBlockSplitter
+ * ZSTD_c_useRowMatchFinder
+ * ZSTD_c_prefetchCDictTables
+ * ZSTD_c_enableSeqProducerFallback
+ * ZSTD_c_maxBlockSize
+ * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+ * note : never ever use experimentalParam? names directly;
+ * also, the enums values themselves are unstable and can still change.
+ */
+ ZSTD_c_experimentalParam1=500,
+ ZSTD_c_experimentalParam2=10,
+ ZSTD_c_experimentalParam3=1000,
+ ZSTD_c_experimentalParam4=1001,
+ ZSTD_c_experimentalParam5=1002,
+ ZSTD_c_experimentalParam6=1003,
+ ZSTD_c_experimentalParam7=1004,
+ ZSTD_c_experimentalParam8=1005,
+ ZSTD_c_experimentalParam9=1006,
+ ZSTD_c_experimentalParam10=1007,
+ ZSTD_c_experimentalParam11=1008,
+ ZSTD_c_experimentalParam12=1009,
+ ZSTD_c_experimentalParam13=1010,
+ ZSTD_c_experimentalParam14=1011,
+ ZSTD_c_experimentalParam15=1012,
+ ZSTD_c_experimentalParam16=1013,
+ ZSTD_c_experimentalParam17=1014,
+ ZSTD_c_experimentalParam18=1015,
+ ZSTD_c_experimentalParam19=1016
+} ZSTD_cParameter;
+
+typedef struct {
+ size_t error;
+ int lowerBound;
+ int upperBound;
+} ZSTD_bounds;
+
+/*! ZSTD_cParam_getBounds() :
+ * All parameters must belong to an interval with lower and upper bounds,
+ * otherwise they will either trigger an error or be automatically clamped.
+ * @return : a structure, ZSTD_bounds, which contains
+ * - an error status field, which must be tested using ZSTD_isError()
+ * - lower and upper bounds, both inclusive
+ */
+ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
+
+/*! ZSTD_CCtx_setParameter() :
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
+ * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
+ * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ * Setting a parameter is generally only possible during frame initialization (before starting compression).
+ * Exception : when using multi-threading mode (nbWorkers >= 1),
+ * the following parameters can be updated _during_ compression (within same frame):
+ * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
+ * new parameters will be active for next job only (after a flush()).
+ * @return : an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
+
+/*! ZSTD_CCtx_setPledgedSrcSize() :
+ * Total input data size to be compressed as a single frame.
+ * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
+ * This value will also be controlled at end of frame, and trigger an error if not respected.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
+ * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
+ * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
+ * Note 2 : pledgedSrcSize is only valid once, for the next frame.
+ * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
+ * Note 3 : Whenever all input data is provided and consumed in a single round,
+ * for example with ZSTD_compress2(),
+ * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
+ * this value is automatically overridden by srcSize instead.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
+
+typedef enum {
+ ZSTD_reset_session_only = 1,
+ ZSTD_reset_parameters = 2,
+ ZSTD_reset_session_and_parameters = 3
+} ZSTD_ResetDirective;
+
+/*! ZSTD_CCtx_reset() :
+ * There are 2 different things that can be reset, independently or jointly :
+ * - The session : will stop compressing current frame, and make CCtx ready to start a new one.
+ * Useful after an error, or to interrupt any ongoing compression.
+ * Any internal data not yet flushed is cancelled.
+ * Compression parameters and dictionary remain unchanged.
+ * They will be used to compress next frame.
+ * Resetting session never fails.
+ * - The parameters : changes all parameters back to "default".
+ * This also removes any reference to any dictionary or external sequence producer.
+ * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
+ * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
+ * - Both : similar to resetting the session, followed by resetting parameters.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
+
+/*! ZSTD_compress2() :
+ * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
+ * ZSTD_compress2() always starts a new frame.
+ * Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
+ * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ * - The function is always blocking, returns when compression is completed.
+ * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
+ * @return : compressed size written into `dst` (<= `dstCapacity),
+ * or an error code if it fails (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize);
+
+
+/***********************************************
+* Advanced decompression API (Requires v1.4.0+)
+************************************************/
+
+/* The advanced API pushes parameters one by one into an existing DCtx context.
+ * Parameters are sticky, and remain valid for all following frames
+ * using the same DCtx context.
+ * It's possible to reset parameters to default values using ZSTD_DCtx_reset().
+ * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream().
+ * Therefore, no new decompression function is necessary.
+ */
+
+typedef enum {
+
+ ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
+ * the streaming API will refuse to allocate memory buffer
+ * in order to protect the host from unreasonable memory requirements.
+ * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+ * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
+ * Special: value 0 means "use default maximum windowLog". */
+
+ /* note : additional experimental parameters are also available
+ * within the experimental section of the API.
+ * At the time of this writing, they include :
+ * ZSTD_d_format
+ * ZSTD_d_stableOutBuffer
+ * ZSTD_d_forceIgnoreChecksum
+ * ZSTD_d_refMultipleDDicts
+ * ZSTD_d_disableHuffmanAssembly
+ * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+ * note : never ever use experimentalParam? names directly
+ */
+ ZSTD_d_experimentalParam1=1000,
+ ZSTD_d_experimentalParam2=1001,
+ ZSTD_d_experimentalParam3=1002,
+ ZSTD_d_experimentalParam4=1003,
+ ZSTD_d_experimentalParam5=1004
+
+} ZSTD_dParameter;
+
+/*! ZSTD_dParam_getBounds() :
+ * All parameters must belong to an interval with lower and upper bounds,
+ * otherwise they will either trigger an error or be automatically clamped.
+ * @return : a structure, ZSTD_bounds, which contains
+ * - an error status field, which must be tested using ZSTD_isError()
+ * - both lower and upper bounds, inclusive
+ */
+ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
+
+/*! ZSTD_DCtx_setParameter() :
+ * Set one compression parameter, selected by enum ZSTD_dParameter.
+ * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
+ * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ * Setting a parameter is only possible during frame initialization (before starting decompression).
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
+
+/*! ZSTD_DCtx_reset() :
+ * Return a DCtx to clean state.
+ * Session and parameters can be reset jointly or separately.
+ * Parameters can only be reset when no active frame is being decompressed.
+ * @return : 0, or an error code, which can be tested with ZSTD_isError()
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
+
+
+/****************************
+* Streaming
+****************************/
+
+typedef struct ZSTD_inBuffer_s {
+ const void* src; /**< start of input buffer */
+ size_t size; /**< size of input buffer */
+ size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_inBuffer;
+
+typedef struct ZSTD_outBuffer_s {
+ void* dst; /**< start of output buffer */
+ size_t size; /**< size of output buffer */
+ size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_outBuffer;
+
+
+
+/*-***********************************************************************
+* Streaming compression - HowTo
+*
+* A ZSTD_CStream object is required to track streaming operation.
+* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
+* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
+*
+* For parallel execution, use one separate ZSTD_CStream per thread.
+*
+* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
+*
+* Parameters are sticky : when starting a new compression on the same context,
+* it will re-use the same sticky parameters as previous compression session.
+* When in doubt, it's recommended to fully initialize the context before usage.
+* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
+* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
+* set more specific parameters, the pledged source size, or load a dictionary.
+*
+* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to
+* consume input stream. The function will automatically update both `pos`
+* fields within `input` and `output`.
+* Note that the function may not consume the entire input, for example, because
+* the output buffer is already full, in which case `input.pos < input.size`.
+* The caller must check if input has been entirely consumed.
+* If not, the caller must make some room to receive more compressed data,
+* and then present again remaining input data.
+* note: ZSTD_e_continue is guaranteed to make some forward progress when called,
+* but doesn't guarantee maximal forward progress. This is especially relevant
+* when compressing with multiple threads. The call won't block if it can
+* consume some input, but if it can't it will wait for some, but not all,
+* output to be flushed.
+* @return : provides a minimum amount of data remaining to be flushed from internal buffers
+* or an error code, which can be tested using ZSTD_isError().
+*
+* At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
+* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
+* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).
+* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
+* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
+* operation.
+* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will
+* block until the flush is complete or the output buffer is full.
+* @return : 0 if internal buffers are entirely flushed,
+* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+* or an error code, which can be tested using ZSTD_isError().
+*
+* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.
+* It will perform a flush and write frame epilogue.
+* The epilogue is required for decoders to consider a frame completed.
+* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
+* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
+* start a new frame.
+* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will
+* block until the flush is complete or the output buffer is full.
+* @return : 0 if frame fully completed and fully flushed,
+* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
+* or an error code, which can be tested using ZSTD_isError().
+*
+* *******************************************************************/
+
+typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
+ /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
+/*===== ZSTD_CStream management functions =====*/
+ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
+ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */
+
+/*===== Streaming compression functions =====*/
+typedef enum {
+ ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
+ ZSTD_e_flush=1, /* flush any data provided so far,
+ * it creates (at least) one new block, that can be decoded immediately on reception;
+ * frame will continue: any future data can still reference previously compressed data, improving compression.
+ * note : multithreaded compression will block to flush as much output as possible. */
+ ZSTD_e_end=2 /* flush any remaining data _and_ close current frame.
+ * note that frame is only closed after compressed data is fully flushed (return value == 0).
+ * After that point, any additional data starts a new frame.
+ * note : each frame is independent (does not reference any content from previous frame).
+ : note : multithreaded compression will block to flush as much output as possible. */
+} ZSTD_EndDirective;
+
+/*! ZSTD_compressStream2() : Requires v1.4.0+
+ * Behaves about the same as ZSTD_compressStream, with additional control on end directive.
+ * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
+ * - output->pos must be <= dstCapacity, input->pos must be <= srcSize
+ * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+ * - endOp must be a valid directive
+ * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
+ * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
+ * and then immediately returns, just indicating that there is some data remaining to be flushed.
+ * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
+ * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
+ * - @return provides a minimum amount of data remaining to be flushed from internal buffers
+ * or an error code, which can be tested using ZSTD_isError().
+ * if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
+ * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
+ * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
+ * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
+ * only ZSTD_e_end or ZSTD_e_flush operations are allowed.
+ * Before starting a new compression job, or changing compression parameters,
+ * it is required to fully flush internal buffers.
+ */
+ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
+ ZSTD_outBuffer* output,
+ ZSTD_inBuffer* input,
+ ZSTD_EndDirective endOp);
+
+
+/* These buffer sizes are softly recommended.
+ * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
+ * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
+ * reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
+ *
+ * However, note that these recommendations are from the perspective of a C caller program.
+ * If the streaming interface is invoked from some other language,
+ * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
+ * a major performance rule is to reduce crossing such interface to an absolute minimum.
+ * It's not rare that performance ends being spent more into the interface, rather than compression itself.
+ * In which cases, prefer using large buffers, as large as practical,
+ * for both input and output, to reduce the nb of roundtrips.
+ */
+ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
+
+
+/* *****************************************************************************
+ * This following is a legacy streaming API, available since v1.0+ .
+ * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
+ * It is redundant, but remains fully supported.
+ ******************************************************************************/
+
+/*!
+ * Equivalent to:
+ *
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *
+ * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
+ * to compress with a dictionary.
+ */
+ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+/*!
+ * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
+ * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
+ * the next read size (if non-zero and not an error). ZSTD_compressStream2()
+ * returns the minimum nb of bytes left to flush (if non-zero and not an error).
+ */
+ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
+ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
+ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+
+
+/*-***************************************************************************
+* Streaming decompression - HowTo
+*
+* A ZSTD_DStream object is required to track streaming operations.
+* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
+* ZSTD_DStream objects can be re-used multiple times.
+*
+* Use ZSTD_initDStream() to start a new decompression operation.
+* @return : recommended first input size
+* Alternatively, use advanced API to set specific properties.
+*
+* Use ZSTD_decompressStream() repetitively to consume your input.
+* The function will update both `pos` fields.
+* If `input.pos < input.size`, some input has not been consumed.
+* It's up to the caller to present again remaining data.
+* The function tries to flush all data decoded immediately, respecting output buffer size.
+* If `output.pos < output.size`, decoder has flushed everything it could.
+* But if `output.pos == output.size`, there might be some data left within internal buffers.,
+* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
+* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
+* @return : 0 when a frame is completely decoded and fully flushed,
+* or an error code, which can be tested using ZSTD_isError(),
+* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
+* the return value is a suggested next input size (just a hint for better latency)
+* that will never request more than the remaining frame size.
+* *******************************************************************************/
+
+typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
+ /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
+/*===== ZSTD_DStream management functions =====*/
+ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
+ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */
+
+/*===== Streaming decompression functions =====*/
+
+/*! ZSTD_initDStream() :
+ * Initialize/reset DStream state for new decompression operation.
+ * Call before new decompression operation using same DStream.
+ *
+ * Note : This function is redundant with the advanced API and equivalent to:
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ * ZSTD_DCtx_refDDict(zds, NULL);
+ */
+ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
+
+/*! ZSTD_decompressStream() :
+ * Streaming decompression function.
+ * Call repetitively to consume full input updating it as necessary.
+ * Function will update both input and output `pos` fields exposing current state via these fields:
+ * - `input.pos < input.size`, some input remaining and caller should provide remaining input
+ * on the next call.
+ * - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
+ * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
+ * call ZSTD_decompressStream() again to flush remaining data to output.
+ * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
+ *
+ * @return : 0 when a frame is completely decoded and fully flushed,
+ * or an error code, which can be tested using ZSTD_isError(),
+ * or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
+ */
+ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
+
+
+/**************************
+* Simple dictionary API
+***************************/
+/*! ZSTD_compress_usingDict() :
+ * Compression at an explicit compression level using a Dictionary.
+ * A dictionary can be any arbitrary data segment (also called a prefix),
+ * or a buffer with specified information (see zdict.h).
+ * Note : This function loads the dictionary, resulting in significant startup delay.
+ * It's intended for a dictionary used only once.
+ * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ int compressionLevel);
+
+/*! ZSTD_decompress_usingDict() :
+ * Decompression using a known Dictionary.
+ * Dictionary must be identical to the one used during compression.
+ * Note : This function loads the dictionary, resulting in significant startup delay.
+ * It's intended for a dictionary used only once.
+ * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize);
+
+
+/***********************************
+ * Bulk processing dictionary API
+ **********************************/
+typedef struct ZSTD_CDict_s ZSTD_CDict;
+
+/*! ZSTD_createCDict() :
+ * When compressing multiple messages or blocks using the same dictionary,
+ * it's recommended to digest the dictionary only once, since it's a costly operation.
+ * ZSTD_createCDict() will create a state from digesting a dictionary.
+ * The resulting state can be used for future compression operations with very limited startup cost.
+ * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
+ * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
+ * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
+ * in which case the only thing that it transports is the @compressionLevel.
+ * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
+ * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
+ int compressionLevel);
+
+/*! ZSTD_freeCDict() :
+ * Function frees memory allocated by ZSTD_createCDict().
+ * If a NULL pointer is passed, no operation is performed. */
+ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
+
+/*! ZSTD_compress_usingCDict() :
+ * Compression using a digested Dictionary.
+ * Recommended when same dictionary is used multiple times.
+ * Note : compression level is _decided at dictionary creation time_,
+ * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
+ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict);
+
+
+typedef struct ZSTD_DDict_s ZSTD_DDict;
+
+/*! ZSTD_createDDict() :
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
+ * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
+
+/*! ZSTD_freeDDict() :
+ * Function frees memory allocated with ZSTD_createDDict()
+ * If a NULL pointer is passed, no operation is performed. */
+ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
+
+/*! ZSTD_decompress_usingDDict() :
+ * Decompression using a digested Dictionary.
+ * Recommended when same dictionary is used multiple times. */
+ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_DDict* ddict);
+
+
+/********************************
+ * Dictionary helper functions
+ *******************************/
+
+/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+
+ * Provides the dictID stored within dictionary.
+ * if @return == 0, the dictionary is not conformant with Zstandard specification.
+ * It can still be loaded, but as a content-only dictionary. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+
+/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+
+ * Provides the dictID of the dictionary loaded into `cdict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
+
+/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+
+ * Provides the dictID of the dictionary loaded into `ddict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+
+/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+
+ * Provides the dictID required to decompressed the frame stored within `src`.
+ * If @return == 0, the dictID could not be decoded.
+ * This could for one of the following reasons :
+ * - The frame does not require a dictionary to be decoded (most common case).
+ * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information.
+ * Note : this use case also happens when using a non-conformant dictionary.
+ * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ * - This is not a Zstandard frame.
+ * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+
+
+/*******************************************************************************
+ * Advanced dictionary and prefix API (Requires v1.4.0+)
+ *
+ * This API allows dictionaries to be used with ZSTD_compress2(),
+ * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
+ * only reset with the context is reset with ZSTD_reset_parameters or
+ * ZSTD_reset_session_and_parameters. Prefixes are single-use.
+ ******************************************************************************/
+
+
+/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+
+ * Create an internal CDict from `dict` buffer.
+ * Decompression will have to use same dictionary.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
+ * meaning "return to no-dictionary mode".
+ * Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
+ * until parameters are reset, a new dictionary is loaded, or the dictionary
+ * is explicitly invalidated by loading a NULL dictionary.
+ * Note 2 : Loading a dictionary involves building tables.
+ * It's also a CPU consuming operation, with non-negligible impact on latency.
+ * Tables are dependent on compression parameters, and for this reason,
+ * compression parameters can no longer be changed after loading a dictionary.
+ * Note 3 :`dict` content will be copied internally.
+ * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
+ * In such a case, dictionary buffer must outlive its users.
+ * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
+ * to precisely select how dictionary content must be interpreted. */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
+ * Reference a prepared dictionary, to be used for all future compressed frames.
+ * Note that compression parameters are enforced from within CDict,
+ * and supersede any compression parameter previously set within CCtx.
+ * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
+ * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
+ * The dictionary will remain valid for future compressed frames using same CCtx.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special : Referencing a NULL CDict means "return to no-dictionary mode".
+ * Note 1 : Currently, only one dictionary can be managed.
+ * Referencing a new dictionary effectively "discards" any previous one.
+ * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
+ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
+
+/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+
+ * Reference a prefix (single-usage dictionary) for next compressed frame.
+ * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
+ * Decompression will need same prefix to properly regenerate data.
+ * Compressing with a prefix is similar in outcome as performing a diff and compressing it,
+ * but performs much faster, especially during decompression (compression speed is tunable with compression level).
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
+ * Note 1 : Prefix buffer is referenced. It **must** outlive compression.
+ * Its content must remain unmodified during compression.
+ * Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
+ * ensure that the window size is large enough to contain the entire source.
+ * See ZSTD_c_windowLog.
+ * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+ * It's a CPU consuming operation, with non-negligible impact on latency.
+ * If there is a need to use the same prefix multiple times, consider loadDictionary instead.
+ * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
+ * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
+ const void* prefix, size_t prefixSize);
+
+/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
+ * Create an internal DDict from dict buffer, to be used to decompress all future frames.
+ * The dictionary remains valid for all future frames, until explicitly invalidated, or
+ * a new dictionary is loaded.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
+ * meaning "return to no-dictionary mode".
+ * Note 1 : Loading a dictionary involves building tables,
+ * which has a non-negligible impact on CPU usage and latency.
+ * It's recommended to "load once, use many times", to amortize the cost
+ * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
+ * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
+ * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
+ * how dictionary content is loaded and interpreted.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+
+ * Reference a prepared dictionary, to be used to decompress next frames.
+ * The dictionary remains active for decompression of future frames using same DCtx.
+ *
+ * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
+ * will store the DDict references in a table, and the DDict used for decompression
+ * will be determined at decompression time, as per the dict ID in the frame.
+ * The memory for the table is allocated on the first call to refDDict, and can be
+ * freed with ZSTD_freeDCtx().
+ *
+ * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
+ * will be managed, and referencing a dictionary effectively "discards" any previous one.
+ *
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special: referencing a NULL DDict means "return to no-dictionary mode".
+ * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+
+ * Reference a prefix (single-usage dictionary) to decompress next frame.
+ * This is the reverse operation of ZSTD_CCtx_refPrefix(),
+ * and must use the same prefix as the one used during compression.
+ * Prefix is **only used once**. Reference is discarded at end of frame.
+ * End of frame is reached when ZSTD_decompressStream() returns 0.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
+ * Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
+ * Prefix buffer must remain unmodified up to the end of frame,
+ * reached when ZSTD_decompressStream() returns 0.
+ * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
+ * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
+ * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
+ * A full dictionary is more costly, as it requires building tables.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
+ const void* prefix, size_t prefixSize);
+
+/* === Memory management === */
+
+/*! ZSTD_sizeof_*() : Requires v1.4.0+
+ * These functions give the _current_ memory usage of selected object.
+ * Note that object memory usage can evolve (increase or decrease) over time. */
+ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
+#endif /* ZSTD_H_235446 */
+
+
+/* **************************************************************************************
+ * ADVANCED AND EXPERIMENTAL FUNCTIONS
+ ****************************************************************************************
+ * The definitions in the following section are considered experimental.
+ * They are provided for advanced scenarios.
+ * They should never be used with a dynamic library, as prototypes may change in the future.
+ * Use them only in association with static linking.
+ * ***************************************************************************************/
+
+#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
+#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
+
+/* This can be overridden externally to hide static symbols. */
+#ifndef ZSTDLIB_STATIC_API
+# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE
+# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE
+# else
+# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE
+# endif
+#endif
+
+/****************************************************************************************
+ * experimental API (static linking only)
+ ****************************************************************************************
+ * The following symbols and constants
+ * are not planned to join "stable API" status in the near future.
+ * They can still change in future versions.
+ * Some of them are planned to remain in the static_only section indefinitely.
+ * Some of them might be removed in the future (especially when redundant with existing stable functions)
+ * ***************************************************************************************/
+
+#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */
+#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2)
+#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
+#define ZSTD_SKIPPABLEHEADERSIZE 8
+
+/* compression parameter bounds */
+#define ZSTD_WINDOWLOG_MAX_32 30
+#define ZSTD_WINDOWLOG_MAX_64 31
+#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
+#define ZSTD_WINDOWLOG_MIN 10
+#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)
+#define ZSTD_HASHLOG_MIN 6
+#define ZSTD_CHAINLOG_MAX_32 29
+#define ZSTD_CHAINLOG_MAX_64 30
+#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))
+#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
+#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
+#define ZSTD_SEARCHLOG_MIN 1
+#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
+#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */
+#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
+#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
+#define ZSTD_STRATEGY_MIN ZSTD_fast
+#define ZSTD_STRATEGY_MAX ZSTD_btultra2
+#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
+
+
+#define ZSTD_OVERLAPLOG_MIN 0
+#define ZSTD_OVERLAPLOG_MAX 9
+
+#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame
+ * requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size,
+ * to preserve host's memory from unreasonable requirements.
+ * This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).
+ * The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */
+
+
+/* LDM parameter bounds */
+#define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN
+#define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX
+#define ZSTD_LDM_MINMATCH_MIN 4
+#define ZSTD_LDM_MINMATCH_MAX 4096
+#define ZSTD_LDM_BUCKETSIZELOG_MIN 1
+#define ZSTD_LDM_BUCKETSIZELOG_MAX 8
+#define ZSTD_LDM_HASHRATELOG_MIN 0
+#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
+
+/* Advanced parameter bounds */
+#define ZSTD_TARGETCBLOCKSIZE_MIN 64
+#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
+#define ZSTD_SRCSIZEHINT_MIN 0
+#define ZSTD_SRCSIZEHINT_MAX INT_MAX
+
+
+/* --- Advanced types --- */
+
+typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
+
+typedef struct {
+ unsigned int offset; /* The offset of the match. (NOT the same as the offset code)
+ * If offset == 0 and matchLength == 0, this sequence represents the last
+ * literals in the block of litLength size.
+ */
+
+ unsigned int litLength; /* Literal length of the sequence. */
+ unsigned int matchLength; /* Match length of the sequence. */
+
+ /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
+ * In this case, we will treat the sequence as a marker for a block boundary.
+ */
+
+ unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'.
+ * Ranges from [0, 3].
+ *
+ * Repeat offsets are essentially previous offsets from previous sequences sorted in
+ * recency order. For more detail, see doc/zstd_compression_format.md
+ *
+ * If rep == 0, then 'offset' does not contain a repeat offset.
+ * If rep > 0:
+ * If litLength != 0:
+ * rep == 1 --> offset == repeat_offset_1
+ * rep == 2 --> offset == repeat_offset_2
+ * rep == 3 --> offset == repeat_offset_3
+ * If litLength == 0:
+ * rep == 1 --> offset == repeat_offset_2
+ * rep == 2 --> offset == repeat_offset_3
+ * rep == 3 --> offset == repeat_offset_1 - 1
+ *
+ * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
+ * 'rep', but repeat offsets do not necessarily need to be calculated from an external
+ * sequence provider's perspective. For example, ZSTD_compressSequences() does not
+ * use this 'rep' field at all (as of now).
+ */
+} ZSTD_Sequence;
+
+typedef struct {
+ unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
+ unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
+ unsigned hashLog; /**< dispatch table : larger == faster, more memory */
+ unsigned searchLog; /**< nb of searches : larger == more compression, slower */
+ unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */
+ unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
+ ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */
+} ZSTD_compressionParameters;
+
+typedef struct {
+ int contentSizeFlag; /**< 1: content size will be in frame header (when known) */
+ int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */
+ int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */
+} ZSTD_frameParameters;
+
+typedef struct {
+ ZSTD_compressionParameters cParams;
+ ZSTD_frameParameters fParams;
+} ZSTD_parameters;
+
+typedef enum {
+ ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
+ ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
+ ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */
+} ZSTD_dictContentType_e;
+
+typedef enum {
+ ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
+ ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
+} ZSTD_dictLoadMethod_e;
+
+typedef enum {
+ ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
+ ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
+ * Useful to save 4 bytes per generated frame.
+ * Decoder cannot recognise automatically this format, requiring this instruction. */
+} ZSTD_format_e;
+
+typedef enum {
+ /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
+ ZSTD_d_validateChecksum = 0,
+ ZSTD_d_ignoreChecksum = 1
+} ZSTD_forceIgnoreChecksum_e;
+
+typedef enum {
+ /* Note: this enum controls ZSTD_d_refMultipleDDicts */
+ ZSTD_rmd_refSingleDDict = 0,
+ ZSTD_rmd_refMultipleDDicts = 1
+} ZSTD_refMultipleDDicts_e;
+
+typedef enum {
+ /* Note: this enum and the behavior it controls are effectively internal
+ * implementation details of the compressor. They are expected to continue
+ * to evolve and should be considered only in the context of extremely
+ * advanced performance tuning.
+ *
+ * Zstd currently supports the use of a CDict in three ways:
+ *
+ * - The contents of the CDict can be copied into the working context. This
+ * means that the compression can search both the dictionary and input
+ * while operating on a single set of internal tables. This makes
+ * the compression faster per-byte of input. However, the initial copy of
+ * the CDict's tables incurs a fixed cost at the beginning of the
+ * compression. For small compressions (< 8 KB), that copy can dominate
+ * the cost of the compression.
+ *
+ * - The CDict's tables can be used in-place. In this model, compression is
+ * slower per input byte, because the compressor has to search two sets of
+ * tables. However, this model incurs no start-up cost (as long as the
+ * working context's tables can be reused). For small inputs, this can be
+ * faster than copying the CDict's tables.
+ *
+ * - The CDict's tables are not used at all, and instead we use the working
+ * context alone to reload the dictionary and use params based on the source
+ * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
+ * This method is effective when the dictionary sizes are very small relative
+ * to the input size, and the input size is fairly large to begin with.
+ *
+ * Zstd has a simple internal heuristic that selects which strategy to use
+ * at the beginning of a compression. However, if experimentation shows that
+ * Zstd is making poor choices, it is possible to override that choice with
+ * this enum.
+ */
+ ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
+ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
+ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
+ ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
+} ZSTD_dictAttachPref_e;
+
+typedef enum {
+ ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level.
+ * Negative compression levels will be uncompressed, and positive compression
+ * levels will be compressed. */
+ ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
+ * emitted if Huffman compression is not profitable. */
+ ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
+} ZSTD_literalCompressionMode_e;
+
+typedef enum {
+ /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
+ * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
+ * or ZSTD_ps_disable allow for a force enable/disable the feature.
+ */
+ ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */
+ ZSTD_ps_enable = 1, /* Force-enable the feature */
+ ZSTD_ps_disable = 2 /* Do not use the feature */
+} ZSTD_paramSwitch_e;
+
+/***************************************
+* Frame size functions
+***************************************/
+
+/*! ZSTD_findDecompressedSize() :
+ * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
+ * `srcSize` must be the _exact_ size of this series
+ * (i.e. there should be a frame boundary at `src + srcSize`)
+ * @return : - decompressed size of all data in all successive frames
+ * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
+ * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+ *
+ * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+ * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+ * In which case, it's necessary to use streaming mode to decompress data.
+ * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+ * note 3 : decompressed size can be very large (64-bits value),
+ * potentially larger than what local system can handle as a single memory segment.
+ * In which case, it's necessary to use streaming mode to decompress data.
+ * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ * Always ensure result fits within application's authorized limits.
+ * Each application can set its own limits.
+ * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
+ * read each contained frame header. This is fast as most of the data is skipped,
+ * however it does mean that all frame data must be present and valid. */
+ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
+
+/*! ZSTD_decompressBound() :
+ * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
+ * `srcSize` must be the _exact_ size of this series
+ * (i.e. there should be a frame boundary at `src + srcSize`)
+ * @return : - upper-bound for the decompressed size of all data in all successive frames
+ * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+ *
+ * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
+ * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
+ * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
+ * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
+ * upper-bound = # blocks * min(128 KB, Window_Size)
+ */
+ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
+
+/*! ZSTD_frameHeaderSize() :
+ * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
+ * @return : size of the Frame Header,
+ * or an error code (if srcSize is too small) */
+ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
+
+/*! ZSTD_decompressionMargin() :
+ * Zstd supports in-place decompression, where the input and output buffers overlap.
+ * In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
+ * and the input buffer must be at the end of the output buffer.
+ *
+ * _______________________ Output Buffer ________________________
+ * | |
+ * | ____ Input Buffer ____|
+ * | | |
+ * v v v
+ * |---------------------------------------|-----------|----------|
+ * ^ ^ ^
+ * |___________________ Output_Size ___________________|_ Margin _|
+ *
+ * NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
+ * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
+ * ZSTD_decompressDCtx().
+ * NOTE: This function supports multi-frame input.
+ *
+ * @param src The compressed frame(s)
+ * @param srcSize The size of the compressed frame(s)
+ * @returns The decompression margin or an error that can be checked with ZSTD_isError().
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
+
+/*! ZSTD_DECOMPRESS_MARGIN() :
+ * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
+ * the compressed frame, compute it from the original size and the blockSizeLog.
+ * See ZSTD_decompressionMargin() for details.
+ *
+ * WARNING: This macro does not support multi-frame input, the input must be a single
+ * zstd frame. If you need that support use the function, or implement it yourself.
+ *
+ * @param originalSize The original uncompressed size of the data.
+ * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
+ * Unless you explicitly set the windowLog smaller than
+ * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
+ */
+#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \
+ ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \
+ 4 /* checksum */ + \
+ ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
+ (blockSize) /* One block of margin */ \
+ ))
+
+typedef enum {
+ ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
+ ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
+} ZSTD_sequenceFormat_e;
+
+/*! ZSTD_sequenceBound() :
+ * `srcSize` : size of the input buffer
+ * @return : upper-bound for the number of sequences that can be generated
+ * from a buffer of srcSize bytes
+ *
+ * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
+
+/*! ZSTD_generateSequences() :
+ * Generate sequences using ZSTD_compress2(), given a source buffer.
+ *
+ * Each block will end with a dummy sequence
+ * with offset == 0, matchLength == 0, and litLength == length of last literals.
+ * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
+ * simply acts as a block delimiter.
+ *
+ * @zc can be used to insert custom compression params.
+ * This function invokes ZSTD_compress2().
+ *
+ * The output of this function can be fed into ZSTD_compressSequences() with CCtx
+ * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
+ * @return : number of sequences generated
+ */
+
+ZSTDLIB_STATIC_API size_t
+ZSTD_generateSequences( ZSTD_CCtx* zc,
+ ZSTD_Sequence* outSeqs, size_t outSeqsSize,
+ const void* src, size_t srcSize);
+
+/*! ZSTD_mergeBlockDelimiters() :
+ * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
+ * by merging them into the literals of the next sequence.
+ *
+ * As such, the final generated result has no explicit representation of block boundaries,
+ * and the final last literals segment is not represented in the sequences.
+ *
+ * The output of this function can be fed into ZSTD_compressSequences() with CCtx
+ * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
+ * @return : number of sequences left after merging
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
+
+/*! ZSTD_compressSequences() :
+ * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
+ * @src contains the entire input (not just the literals).
+ * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
+ * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
+ * The entire source is compressed into a single frame.
+ *
+ * The compression behavior changes based on cctx params. In particular:
+ * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
+ * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
+ * the block size derived from the cctx, and sequences may be split. This is the default setting.
+ *
+ * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
+ * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
+ *
+ * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
+ * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
+ * specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
+ *
+ * In addition to the two adjustable experimental params, there are other important cctx params.
+ * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
+ * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
+ * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
+ * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
+ *
+ * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
+ * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
+ * and cannot emit an RLE block that disagrees with the repcode history
+ * @return : final compressed size, or a ZSTD error code.
+ */
+ZSTDLIB_STATIC_API size_t
+ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
+ const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+ const void* src, size_t srcSize);
+
+
+/*! ZSTD_writeSkippableFrame() :
+ * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ *
+ * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
+ * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
+ * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
+ * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
+ *
+ * Returns an error if destination buffer is not large enough, if the source size is not representable
+ * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
+ *
+ * @return : number of bytes written or a ZSTD error.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, unsigned magicVariant);
+
+/*! ZSTD_readSkippableFrame() :
+ * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ *
+ * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
+ * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
+ * in the magicVariant.
+ *
+ * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
+ *
+ * @return : number of bytes written or a ZSTD error.
+ */
+ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
+ const void* src, size_t srcSize);
+
+/*! ZSTD_isSkippableFrame() :
+ * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
+ */
+ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
+
+
+
+/***************************************
+* Memory management
+***************************************/
+
+/*! ZSTD_estimate*() :
+ * These functions make it possible to estimate memory usage
+ * of a future {D,C}Ctx, before its creation.
+ *
+ * ZSTD_estimateCCtxSize() will provide a memory budget large enough
+ * for any compression level up to selected one.
+ * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate
+ * does not include space for a window buffer.
+ * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
+ * The estimate will assume the input may be arbitrarily large,
+ * which is the worst case.
+ *
+ * When srcSize can be bound by a known and rather "small" value,
+ * this fact can be used to provide a tighter estimation
+ * because the CCtx compression context will need less memory.
+ * This tighter estimation can be provided by more advanced functions
+ * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
+ * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
+ * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
+ *
+ * Note : only single-threaded compression is supported.
+ * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *
+ * Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+ * Size estimates assume that no external sequence producer is registered.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
+
+/*! ZSTD_estimateCStreamSize() :
+ * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
+ * It will also consider src size to be arbitrarily "large", which is worst case.
+ * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
+ * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+ * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
+ * Note : CStream size estimation is only correct for single-threaded compression.
+ * ZSTD_DStream memory budget depends on window Size.
+ * This information can be passed manually, using ZSTD_estimateDStreamSize,
+ * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
+ * Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
+ * an internal ?Dict will be created, which additional size is not estimated here.
+ * In this case, get total size by adding ZSTD_estimate?DictSize
+ * Note 2 : only single-threaded compression is supported.
+ * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ * Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+ * Size estimates assume that no external sequence producer is registered.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
+
+/*! ZSTD_estimate?DictSize() :
+ * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
+ * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
+ * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
+
+/*! ZSTD_initStatic*() :
+ * Initialize an object using a pre-allocated fixed-size buffer.
+ * workspace: The memory area to emplace the object into.
+ * Provided pointer *must be 8-bytes aligned*.
+ * Buffer must outlive object.
+ * workspaceSize: Use ZSTD_estimate*Size() to determine
+ * how large workspace must be to support target scenario.
+ * @return : pointer to object (same address as workspace, just different type),
+ * or NULL if error (size too small, incorrect alignment, etc.)
+ * Note : zstd will never resize nor malloc() when using a static buffer.
+ * If the object requires more memory than available,
+ * zstd will just error out (typically ZSTD_error_memory_allocation).
+ * Note 2 : there is no corresponding "free" function.
+ * Since workspace is allocated externally, it must be freed externally too.
+ * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level
+ * into its associated cParams.
+ * Limitation 1 : currently not compatible with internal dictionary creation, triggered by
+ * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict().
+ * Limitation 2 : static cctx currently not compatible with multi-threading.
+ * Limitation 3 : static dctx is incompatible with legacy support.
+ */
+ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
+ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
+
+ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
+ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
+
+ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict(
+ void* workspace, size_t workspaceSize,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_compressionParameters cParams);
+
+ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict(
+ void* workspace, size_t workspaceSize,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType);
+
+
+/*! Custom memory allocation :
+ * These prototypes make it possible to pass your own allocation/free functions.
+ * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
+ * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
+ */
+typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
+typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
+static
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
+
+ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
+ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_compressionParameters cParams,
+ ZSTD_customMem customMem);
+
+/*! Thread pool :
+ * These prototypes make it possible to share a thread pool among multiple compression contexts.
+ * This can limit resources for applications with multiple threads where each one uses
+ * a threaded compression mode (via ZSTD_c_nbWorkers parameter).
+ * ZSTD_createThreadPool creates a new thread pool with a given number of threads.
+ * Note that the lifetime of such pool must exist while being used.
+ * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
+ * to use an internal thread pool).
+ * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer.
+ */
+typedef struct POOL_ctx_s ZSTD_threadPool;
+ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
+ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
+
+
+/*
+ * This API is temporary and is expected to change or disappear in the future!
+ */
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2(
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ const ZSTD_CCtx_params* cctxParams,
+ ZSTD_customMem customMem);
+
+ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced(
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_customMem customMem);
+
+
+/***************************************
+* Advanced compression functions
+***************************************/
+
+/*! ZSTD_createCDict_byReference() :
+ * Create a digested dictionary for compression
+ * Dictionary content is just referenced, not duplicated.
+ * As a consequence, `dictBuffer` **must** outlive CDict,
+ * and its content must remain unmodified throughout the lifetime of CDict.
+ * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
+ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
+
+/*! ZSTD_getCParams() :
+ * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
+ * `estimatedSrcSize` value is optional, select 0 if not known */
+ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+
+/*! ZSTD_getParams() :
+ * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
+ * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
+ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+
+/*! ZSTD_checkCParams() :
+ * Ensure param values remain within authorized range.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
+ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
+
+/*! ZSTD_adjustCParams() :
+ * optimize params for a given `srcSize` and `dictSize`.
+ * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
+ * `dictSize` must be `0` when there is no dictionary.
+ * cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
+ * This function never fails (wide contract) */
+ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
+
+/*! ZSTD_CCtx_setCParams() :
+ * Set all parameters provided within @cparams into the working @cctx.
+ * Note : if modifying parameters during compression (MT mode only),
+ * note that changes to the .windowLog parameter will be ignored.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
+
+/*! ZSTD_compress_advanced() :
+ * Note : this function is now DEPRECATED.
+ * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
+ * This prototype will generate compilation warnings. */
+ZSTD_DEPRECATED("use ZSTD_compress2")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ ZSTD_parameters params);
+
+/*! ZSTD_compress_usingCDict_advanced() :
+ * Note : this function is now DEPRECATED.
+ * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
+ * This prototype will generate compilation warnings. */
+ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict,
+ ZSTD_frameParameters fParams);
+
+
+/*! ZSTD_CCtx_loadDictionary_byReference() :
+ * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
+ * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_loadDictionary_advanced() :
+ * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
+ * how to load the dictionary (by copy ? by reference ?)
+ * and how to interpret it (automatic ? force raw mode ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_CCtx_refPrefix_advanced() :
+ * Same as ZSTD_CCtx_refPrefix(), but gives finer control over
+ * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+
+/* === experimental parameters === */
+/* these parameters can be used with ZSTD_setParameter()
+ * they are not guaranteed to remain supported in the future */
+
+ /* Enables rsyncable mode,
+ * which makes compressed files more rsync friendly
+ * by adding periodic synchronization points to the compressed data.
+ * The target average block size is ZSTD_c_jobSize / 2.
+ * It's possible to modify the job size to increase or decrease
+ * the granularity of the synchronization point.
+ * Once the jobSize is smaller than the window size,
+ * it will result in compression ratio degradation.
+ * NOTE 1: rsyncable mode only works when multithreading is enabled.
+ * NOTE 2: rsyncable performs poorly in combination with long range mode,
+ * since it will decrease the effectiveness of synchronization points,
+ * though mileage may vary.
+ * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s.
+ * If the selected compression level is already running significantly slower,
+ * the overall speed won't be significantly impacted.
+ */
+ #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1
+
+/* Select a compression format.
+ * The value must be of type ZSTD_format_e.
+ * See ZSTD_format_e enum definition for details */
+#define ZSTD_c_format ZSTD_c_experimentalParam2
+
+/* Force back-reference distances to remain < windowSize,
+ * even when referencing into Dictionary content (default:0) */
+#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3
+
+/* Controls whether the contents of a CDict
+ * are used in place, or copied into the working context.
+ * Accepts values from the ZSTD_dictAttachPref_e enum.
+ * See the comments on that enum for an explanation of the feature. */
+#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
+
+/* Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never compress literals.
+ * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
+ * may still be emitted if huffman is not beneficial to use.)
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * literals compression based on the compression parameters - specifically,
+ * negative compression levels do not use literal compression.
+ */
+#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
+
+/* Tries to fit compressed block size to be around targetCBlockSize.
+ * No target when targetCBlockSize == 0.
+ * There is no guarantee on compressed block size (default:0) */
+#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6
+
+/* User's best guess of source size.
+ * Hint is not valid when srcSizeHint == 0.
+ * There is no guarantee that hint is close to actual source size,
+ * but compression ratio may regress significantly if guess considerably underestimates */
+#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
+
+/* Controls whether the new and experimental "dedicated dictionary search
+ * structure" can be used. This feature is still rough around the edges, be
+ * prepared for surprising behavior!
+ *
+ * How to use it:
+ *
+ * When using a CDict, whether to use this feature or not is controlled at
+ * CDict creation, and it must be set in a CCtxParams set passed into that
+ * construction (via ZSTD_createCDict_advanced2()). A compression will then
+ * use the feature or not based on how the CDict was constructed; the value of
+ * this param, set in the CCtx, will have no effect.
+ *
+ * However, when a dictionary buffer is passed into a CCtx, such as via
+ * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control
+ * whether the CDict that is created internally can use the feature or not.
+ *
+ * What it does:
+ *
+ * Normally, the internal data structures of the CDict are analogous to what
+ * would be stored in a CCtx after compressing the contents of a dictionary.
+ * To an approximation, a compression using a dictionary can then use those
+ * data structures to simply continue what is effectively a streaming
+ * compression where the simulated compression of the dictionary left off.
+ * Which is to say, the search structures in the CDict are normally the same
+ * format as in the CCtx.
+ *
+ * It is possible to do better, since the CDict is not like a CCtx: the search
+ * structures are written once during CDict creation, and then are only read
+ * after that, while the search structures in the CCtx are both read and
+ * written as the compression goes along. This means we can choose a search
+ * structure for the dictionary that is read-optimized.
+ *
+ * This feature enables the use of that different structure.
+ *
+ * Note that some of the members of the ZSTD_compressionParameters struct have
+ * different semantics and constraints in the dedicated search structure. It is
+ * highly recommended that you simply set a compression level in the CCtxParams
+ * you pass into the CDict creation call, and avoid messing with the cParams
+ * directly.
+ *
+ * Effects:
+ *
+ * This will only have any effect when the selected ZSTD_strategy
+ * implementation supports this feature. Currently, that's limited to
+ * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2.
+ *
+ * Note that this means that the CDict tables can no longer be copied into the
+ * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
+ * usable. The dictionary can only be attached or reloaded.
+ *
+ * In general, you should expect compression to be faster--sometimes very much
+ * so--and CDict creation to be slightly slower. Eventually, we will probably
+ * make this mode the default.
+ */
+#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
+
+/* ZSTD_c_stableInBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells the compressor that input data presented with ZSTD_inBuffer
+ * will ALWAYS be the same between calls.
+ * Technically, the @src pointer must never be changed,
+ * and the @pos field can only be updated by zstd.
+ * However, it's possible to increase the @size field,
+ * allowing scenarios where more data can be appended after compressions starts.
+ * These conditions are checked by the compressor,
+ * and compression will fail if they are not respected.
+ * Also, data in the ZSTD_inBuffer within the range [src, src + pos)
+ * MUST not be modified during compression or it will result in data corruption.
+ *
+ * When this flag is enabled zstd won't allocate an input window buffer,
+ * because the user guarantees it can reference the ZSTD_inBuffer until
+ * the frame is complete. But, it will still allocate an output buffer
+ * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
+ * avoid the memcpy() from the input buffer to the input window buffer.
+ *
+ * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
+ * memory. However, compression WILL fail if conditions are not respected.
+ *
+ * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
+ * not be modified during compression or it will result in data corruption.
+ * This is because zstd needs to reference data in the ZSTD_inBuffer to find
+ * matches. Normally zstd maintains its own window buffer for this purpose,
+ * but passing this flag tells zstd to rely on user provided buffer instead.
+ */
+#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
+
+/* ZSTD_c_stableOutBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells he compressor that the ZSTD_outBuffer will not be resized between
+ * calls. Specifically: (out.size - out.pos) will never grow. This gives the
+ * compressor the freedom to say: If the compressed data doesn't fit in the
+ * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
+ * always decompress directly into the output buffer, instead of decompressing
+ * into an internal buffer and copying to the output buffer.
+ *
+ * When this flag is enabled zstd won't allocate an output buffer, because
+ * it can write directly to the ZSTD_outBuffer. It will still allocate the
+ * input window buffer (see ZSTD_c_stableInBuffer).
+ *
+ * Zstd will check that (out.size - out.pos) never grows and return an error
+ * if it does. While not strictly necessary, this should prevent surprises.
+ */
+#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
+
+/* ZSTD_c_blockDelimiters
+ * Default is 0 == ZSTD_sf_noBlockDelimiters.
+ *
+ * For use with sequence compression API: ZSTD_compressSequences().
+ *
+ * Designates whether or not the given array of ZSTD_Sequence contains block delimiters
+ * and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
+ * See the definition of ZSTD_Sequence for more specifics.
+ */
+#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11
+
+/* ZSTD_c_validateSequences
+ * Default is 0 == disabled. Set to 1 to enable sequence validation.
+ *
+ * For use with sequence compression API: ZSTD_compressSequences().
+ * Designates whether or not we validate sequences provided to ZSTD_compressSequences()
+ * during function execution.
+ *
+ * Without validation, providing a sequence that does not conform to the zstd spec will cause
+ * undefined behavior, and may produce a corrupted block.
+ *
+ * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for
+ * specifics regarding offset/matchlength requirements) then the function will bail out and
+ * return an error.
+ *
+ */
+#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
+
+/* ZSTD_c_useBlockSplitter
+ * Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never use block splitter.
+ * Set to ZSTD_ps_enable to always use block splitter.
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * block splitting based on the compression parameters.
+ */
+#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13
+
+/* ZSTD_c_useRowMatchFinder
+ * Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never use row-based matchfinder.
+ * Set to ZSTD_ps_enable to force usage of row-based matchfinder.
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * the row-based matchfinder based on support for SIMD instructions and the window log.
+ * Note that this only pertains to compression strategies: greedy, lazy, and lazy2
+ */
+#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
+
+/* ZSTD_c_deterministicRefPrefix
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Zstd produces different results for prefix compression when the prefix is
+ * directly adjacent to the data about to be compressed vs. when it isn't.
+ * This is because zstd detects that the two buffers are contiguous and it can
+ * use a more efficient match finding algorithm. However, this produces different
+ * results than when the two buffers are non-contiguous. This flag forces zstd
+ * to always load the prefix in non-contiguous mode, even if it happens to be
+ * adjacent to the data, to guarantee determinism.
+ *
+ * If you really care about determinism when using a dictionary or prefix,
+ * like when doing delta compression, you should select this option. It comes
+ * at a speed penalty of about ~2.5% if the dictionary and data happened to be
+ * contiguous, and is free if they weren't contiguous. We don't expect that
+ * intentionally making the dictionary and data contiguous will be worth the
+ * cost to memcpy() the data.
+ */
+#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
+
+/* ZSTD_c_prefetchCDictTables
+ * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto.
+ *
+ * In some situations, zstd uses CDict tables in-place rather than copying them
+ * into the working context. (See docs on ZSTD_dictAttachPref_e above for details).
+ * In such situations, compression speed is seriously impacted when CDict tables are
+ * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables
+ * when they are used in-place.
+ *
+ * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit.
+ * For sufficiently large inputs, zstd will by default memcpy() CDict tables
+ * into the working context, so there is no need to prefetch. This parameter is
+ * targeted at a middle range of input sizes, where a prefetch is cheap enough to be
+ * useful but memcpy() is too expensive. The exact range of input sizes where this
+ * makes sense is best determined by careful experimentation.
+ *
+ * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable,
+ * but in the future zstd may conditionally enable this feature via an auto-detection
+ * heuristic for cold CDicts.
+ * Use ZSTD_ps_disable to opt out of prefetching under any circumstances.
+ */
+#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
+
+/* ZSTD_c_enableSeqProducerFallback
+ * Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
+ *
+ * Controls whether zstd will fall back to an internal sequence producer if an
+ * external sequence producer is registered and returns an error code. This fallback
+ * is block-by-block: the internal sequence producer will only be called for blocks
+ * where the external sequence producer returns an error code. Fallback parsing will
+ * follow any other cParam settings, such as compression level, the same as in a
+ * normal (fully-internal) compression operation.
+ *
+ * The user is strongly encouraged to read the full Block-Level Sequence Producer API
+ * documentation (below) before setting this parameter. */
+#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17
+
+/* ZSTD_c_maxBlockSize
+ * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+ *
+ * This parameter can be used to set an upper bound on the blocksize
+ * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
+ * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
+ * compressBound() innacurate). Only currently meant to be used for testing.
+ *
+ */
+#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
+
+/* ZSTD_c_searchForExternalRepcodes
+ * This parameter affects how zstd parses external sequences, such as sequences
+ * provided through the compressSequences() API or from an external block-level
+ * sequence producer.
+ *
+ * If set to ZSTD_ps_enable, the library will check for repeated offsets in
+ * external sequences, even if those repcodes are not explicitly indicated in
+ * the "rep" field. Note that this is the only way to exploit repcode matches
+ * while using compressSequences() or an external sequence producer, since zstd
+ * currently ignores the "rep" field of external sequences.
+ *
+ * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
+ * external sequences, regardless of whether the "rep" field has been set. This
+ * reduces sequence compression overhead by about 25% while sacrificing some
+ * compression ratio.
+ *
+ * The default value is ZSTD_ps_auto, for which the library will enable/disable
+ * based on compression level.
+ *
+ * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
+ * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
+ */
+#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
+
+/*! ZSTD_CCtx_getParameter() :
+ * Get the requested compression parameter value, selected by enum ZSTD_cParameter,
+ * and store it into int* value.
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
+
+
+/*! ZSTD_CCtx_params :
+ * Quick howto :
+ * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
+ * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
+ * an existing ZSTD_CCtx_params structure.
+ * This is similar to
+ * ZSTD_CCtx_setParameter().
+ * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
+ * an existing CCtx.
+ * These parameters will be applied to
+ * all subsequent frames.
+ * - ZSTD_compressStream2() : Do compression using the CCtx.
+ * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer.
+ *
+ * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
+ * for static allocation of CCtx for single-threaded compression.
+ */
+ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
+ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */
+
+/*! ZSTD_CCtxParams_reset() :
+ * Reset params to default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
+
+/*! ZSTD_CCtxParams_init() :
+ * Initializes the compression parameters of cctxParams according to
+ * compression level. All other parameters are reset to their default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
+
+/*! ZSTD_CCtxParams_init_advanced() :
+ * Initializes the compression and frame parameters of cctxParams according to
+ * params. All other parameters are reset to their default values.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
+
+/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+
+ * Similar to ZSTD_CCtx_setParameter.
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
+ * Parameters must be applied to a ZSTD_CCtx using
+ * ZSTD_CCtx_setParametersUsingCCtxParams().
+ * @result : a code representing success or failure (which can be tested with
+ * ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
+
+/*! ZSTD_CCtxParams_getParameter() :
+ * Similar to ZSTD_CCtx_getParameter.
+ * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+
+/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
+ * Apply a set of ZSTD_CCtx_params to the compression context.
+ * This can be done even after compression is started,
+ * if nbWorkers==0, this will have no impact until a new compression is started.
+ * if nbWorkers>=1, new parameters will be picked up at next job,
+ * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
+ ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
+
+/*! ZSTD_compressStream2_simpleArgs() :
+ * Same as ZSTD_compressStream2(),
+ * but using only integral types as arguments.
+ * This variant might be helpful for binders from dynamic languages
+ * which have troubles handling structures containing memory pointers.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs (
+ ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos,
+ ZSTD_EndDirective endOp);
+
+
+/***************************************
+* Advanced decompression functions
+***************************************/
+
+/*! ZSTD_isFrame() :
+ * Tells if the content of `buffer` starts with a valid Frame Identifier.
+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ * Note 3 : Skippable Frame Identifiers are considered valid. */
+ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
+
+/*! ZSTD_createDDict_byReference() :
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
+ * Dictionary content is referenced, and therefore stays in dictBuffer.
+ * It is important that dictBuffer outlives DDict,
+ * it must remain read accessible throughout the lifetime of DDict */
+ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
+
+/*! ZSTD_DCtx_loadDictionary_byReference() :
+ * Same as ZSTD_DCtx_loadDictionary(),
+ * but references `dict` content instead of copying it into `dctx`.
+ * This saves memory if `dict` remains around.,
+ * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_DCtx_loadDictionary_advanced() :
+ * Same as ZSTD_DCtx_loadDictionary(),
+ * but gives direct control over
+ * how to load the dictionary (by copy ? by reference ?)
+ * and how to interpret it (automatic ? force raw mode ? full mode only ?). */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_DCtx_refPrefix_advanced() :
+ * Same as ZSTD_DCtx_refPrefix(), but gives finer control over
+ * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_DCtx_setMaxWindowSize() :
+ * Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
+ * This protects a decoder context from reserving too much memory for itself (potential attack scenario).
+ * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+ * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
+
+/*! ZSTD_DCtx_getParameter() :
+ * Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
+ * and store it into int* value.
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
+
+/* ZSTD_d_format
+ * experimental parameter,
+ * allowing selection between ZSTD_format_e input compression formats
+ */
+#define ZSTD_d_format ZSTD_d_experimentalParam1
+/* ZSTD_d_stableOutBuffer
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable.
+ *
+ * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same
+ * between calls, except for the modifications that zstd makes to pos (the
+ * caller must not modify pos). This is checked by the decompressor, and
+ * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer
+ * MUST be large enough to fit the entire decompressed frame. This will be
+ * checked when the frame content size is known. The data in the ZSTD_outBuffer
+ * in the range [dst, dst + pos) MUST not be modified during decompression
+ * or you will get data corruption.
+ *
+ * When this flag is enabled zstd won't allocate an output buffer, because
+ * it can write directly to the ZSTD_outBuffer, but it will still allocate
+ * an input buffer large enough to fit any compressed block. This will also
+ * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
+ * If you need to avoid the input buffer allocation use the buffer-less
+ * streaming API.
+ *
+ * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
+ * memory. However, decompression WILL fail if you violate the preconditions.
+ *
+ * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST
+ * not be modified during decompression or you will get data corruption. This
+ * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate
+ * matches. Normally zstd maintains its own buffer for this purpose, but passing
+ * this flag tells zstd to use the user provided buffer.
+ */
+#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2
+
+/* ZSTD_d_forceIgnoreChecksum
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable
+ *
+ * Tells the decompressor to skip checksum validation during decompression, regardless
+ * of whether checksumming was specified during compression. This offers some
+ * slight performance benefits, and may be useful for debugging.
+ * Param has values of type ZSTD_forceIgnoreChecksum_e
+ */
+#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
+
+/* ZSTD_d_refMultipleDDicts
+ * Experimental parameter.
+ * Default is 0 == disabled. Set to 1 to enable
+ *
+ * If enabled and dctx is allocated on the heap, then additional memory will be allocated
+ * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
+ * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
+ * store all references. At decompression time, the appropriate dictID is selected
+ * from the set of DDicts based on the dictID in the frame.
+ *
+ * Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
+ *
+ * Param has values of byte ZSTD_refMultipleDDicts_e
+ *
+ * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
+ * allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
+ * Memory is allocated as per ZSTD_DCtx::customMem.
+ *
+ * Although this function allocates memory for the table, the user is still responsible for
+ * memory management of the underlying ZSTD_DDict* themselves.
+ */
+#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
+
+/* ZSTD_d_disableHuffmanAssembly
+ * Set to 1 to disable the Huffman assembly implementation.
+ * The default value is 0, which allows zstd to use the Huffman assembly
+ * implementation if available.
+ *
+ * This parameter can be used to disable Huffman assembly at runtime.
+ * If you want to disable it at compile time you can define the macro
+ * ZSTD_DISABLE_ASM.
+ */
+#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5
+
+
+/*! ZSTD_DCtx_setFormat() :
+ * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
+ * Instruct the decoder context about what kind of data to decode next.
+ * This instruction is mandatory to decode data without a fully-formed header,
+ * such ZSTD_f_zstd1_magicless for example.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
+ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+ZSTDLIB_STATIC_API
+size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
+
+/*! ZSTD_decompressStream_simpleArgs() :
+ * Same as ZSTD_decompressStream(),
+ * but using only integral types as arguments.
+ * This can be helpful for binders from dynamic languages
+ * which have troubles handling structures containing memory pointers.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
+ ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos);
+
+
+/********************************************************************
+* Advanced streaming functions
+* Warning : most of these functions are now redundant with the Advanced API.
+* Once Advanced API reaches "stable" status,
+* redundant functions will be deprecated, and then at some point removed.
+********************************************************************/
+
+/*===== Advanced Streaming compression functions =====*/
+
+/*! ZSTD_initCStream_srcSize() :
+ * This function is DEPRECATED, and equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+ * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ *
+ * pledgedSrcSize must be correct. If it is not known at init time, use
+ * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
+ * "0" also disables frame content size field. It may be enabled in the future.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
+ int compressionLevel,
+ unsigned long long pledgedSrcSize);
+
+/*! ZSTD_initCStream_usingDict() :
+ * This function is DEPRECATED, and is equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * Creates of an internal CDict (incompatible with static CCtx), except if
+ * dict == NULL or dictSize < 8, in which case no dict is used.
+ * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
+ * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
+ const void* dict, size_t dictSize,
+ int compressionLevel);
+
+/*! ZSTD_initCStream_advanced() :
+ * This function is DEPRECATED, and is approximately equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * // Pseudocode: Set each zstd parameter and leave the rest as-is.
+ * for ((param, value) : params) {
+ * ZSTD_CCtx_setParameter(zcs, param, value);
+ * }
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+ *
+ * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
+ * pledgedSrcSize must be correct.
+ * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+ const void* dict, size_t dictSize,
+ ZSTD_parameters params,
+ unsigned long long pledgedSrcSize);
+
+/*! ZSTD_initCStream_usingCDict() :
+ * This function is DEPRECATED, and equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * note : cdict will just be referenced, and must outlive compression session
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
+
+/*! ZSTD_initCStream_usingCDict_advanced() :
+ * This function is DEPRECATED, and is approximately equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
+ * for ((fParam, value) : fParams) {
+ * ZSTD_CCtx_setParameter(zcs, fParam, value);
+ * }
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ * ZSTD_CCtx_refCDict(zcs, cdict);
+ *
+ * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
+ * pledgedSrcSize must be correct. If srcSize is not known at init time, use
+ * value ZSTD_CONTENTSIZE_UNKNOWN.
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+ const ZSTD_CDict* cdict,
+ ZSTD_frameParameters fParams,
+ unsigned long long pledgedSrcSize);
+
+/*! ZSTD_resetCStream() :
+ * This function is DEPRECATED, and is equivalent to:
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
+ * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
+ * explicitly specified.
+ *
+ * start a new frame, using same parameters from previous frame.
+ * This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
+ * Note that zcs must be init at least once before using ZSTD_resetCStream().
+ * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
+ * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
+ * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
+ * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError())
+ * This prototype will generate compilation warnings.
+ */
+ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
+
+
+typedef struct {
+ unsigned long long ingested; /* nb input bytes read and buffered */
+ unsigned long long consumed; /* nb input bytes actually compressed */
+ unsigned long long produced; /* nb of compressed bytes generated and buffered */
+ unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
+ unsigned currentJobID; /* MT only : latest started job nb */
+ unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */
+} ZSTD_frameProgression;
+
+/* ZSTD_getFrameProgression() :
+ * tells how much data has been ingested (read from input)
+ * consumed (input actually compressed) and produced (output) for current frame.
+ * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
+ * Aggregates progression inside active worker threads.
+ */
+ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
+
+/*! ZSTD_toFlushNow() :
+ * Tell how many bytes are ready to be flushed immediately.
+ * Useful for multithreading scenarios (nbWorkers >= 1).
+ * Probe the oldest active job, defined as oldest job not yet entirely flushed,
+ * and check its output buffer.
+ * @return : amount of data stored in oldest job and ready to be flushed immediately.
+ * if @return == 0, it means either :
+ * + there is no active job (could be checked with ZSTD_frameProgression()), or
+ * + oldest job is still actively compressing data,
+ * but everything it has produced has also been flushed so far,
+ * therefore flush speed is limited by production speed of oldest job
+ * irrespective of the speed of concurrent (and newer) jobs.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
+
+
+/*===== Advanced Streaming decompression functions =====*/
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ * ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
+ *
+ * note: no dictionary will be used if dict == NULL or dictSize < 8
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ * ZSTD_DCtx_refDDict(zds, ddict);
+ *
+ * note : ddict is referenced, it must outlive decompression session
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+
+/*!
+ * This function is deprecated, and is equivalent to:
+ *
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *
+ * re-use decompression parameters from previous init; saves dictionary loading
+ */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
+
+
+/*********************************************************************
+* Buffer-less and synchronous inner streaming functions
+*
+* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
+* But it's also a complex one, with several restrictions, documented below.
+* Prefer normal streaming API for an easier experience.
+********************************************************************* */
+
+/**
+ Buffer-less streaming compression (synchronous mode)
+
+ A ZSTD_CCtx object is required to track streaming operations.
+ Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
+ ZSTD_CCtx object can be re-used multiple times within successive compression operations.
+
+ Start by initializing a context.
+ Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
+
+ Then, consume your input using ZSTD_compressContinue().
+ There are some important considerations to keep in mind when using this advanced function :
+ - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
+ - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
+ - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
+ Worst case evaluation is provided by ZSTD_compressBound().
+ ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
+ - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
+ It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
+ - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
+ In which case, it will "discard" the relevant memory section from its history.
+
+ Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
+ It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
+ Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
+
+ `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
+*/
+
+/*===== Buffer-less streaming compression functions =====*/
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
+
+ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
+
+ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
+ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
+ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
+size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
+/**
+ Buffer-less streaming decompression (synchronous mode)
+
+ A ZSTD_DCtx object is required to track streaming operations.
+ Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
+ A ZSTD_DCtx object can be re-used multiple times.
+
+ First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
+ Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
+ Data fragment must be large enough to ensure successful decoding.
+ `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
+ result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
+ >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
+ errorCode, which can be tested using ZSTD_isError().
+
+ It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
+ such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
+ Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
+ As a consequence, check that values remain within valid application range.
+ For example, do not allocate memory blindly, check that `windowSize` is within expectation.
+ Each application can set its own limits, depending on local restrictions.
+ For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
+
+ ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
+ ZSTD_decompressContinue() is very sensitive to contiguity,
+ if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
+ or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
+ There are multiple ways to guarantee this condition.
+
+ The most memory efficient way is to use a round buffer of sufficient size.
+ Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
+ which can return an error code if required value is too large for current system (in 32-bits mode).
+ In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
+ up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
+ which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
+ At which point, decoding can resume from the beginning of the buffer.
+ Note that already decoded data stored in the buffer should be flushed before being overwritten.
+
+ There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
+
+ Finally, if you control the compression process, you can also ignore all buffer size rules,
+ as long as the encoder and decoder progress in "lock-step",
+ aka use exactly the same buffer sizes, break contiguity at the same place, etc.
+
+ Once buffers are setup, start decompression, with ZSTD_decompressBegin().
+ If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
+
+ Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
+ ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
+ ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
+
+ result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+ It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
+ It can also be an error code, which can be tested with ZSTD_isError().
+
+ A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
+ Context can then be reset to start a new decompression.
+
+ Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
+ This information is not required to properly decode a frame.
+
+ == Special case : skippable frames ==
+
+ Skippable frames allow integration of user-defined data into a flow of concatenated frames.
+ Skippable frames will be ignored (skipped) by decompressor.
+ The format of skippable frames is as follows :
+ a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
+ b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
+ c) Frame Content - any content (User Data) of length equal to Frame Size
+ For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
+ For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
+*/
+
+/*===== Buffer-less streaming decompression functions =====*/
+typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
+typedef struct {
+ unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
+ unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
+ unsigned blockSizeMax;
+ ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
+ unsigned headerSize;
+ unsigned dictID;
+ unsigned checksumFlag;
+ unsigned _reserved1;
+ unsigned _reserved2;
+} ZSTD_frameHeader;
+
+/*! ZSTD_getFrameHeader() :
+ * decode Frame Header, or requires larger `srcSize`.
+ * @return : 0, `zfhPtr` is correctly filled,
+ * >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ * or an error code, which can be tested using ZSTD_isError() */
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
+/*! ZSTD_getFrameHeader_advanced() :
+ * same as ZSTD_getFrameHeader(),
+ * with added capability to select a format (like ZSTD_f_zstd1_magicless) */
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
+ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
+
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* misc */
+ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
+typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
+ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
+
+
+
+
+/* ============================ */
+/** Block level API */
+/* ============================ */
+
+/*!
+ Block functions produce and decode raw zstd blocks, without frame metadata.
+ Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
+ But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
+
+ A few rules to respect :
+ - Compressing and decompressing require a context structure
+ + Use ZSTD_createCCtx() and ZSTD_createDCtx()
+ - It is necessary to init context before starting
+ + compression : any ZSTD_compressBegin*() variant, including with dictionary
+ + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
+ - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
+ + If input is larger than a block size, it's necessary to split input data into multiple blocks
+ + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
+ Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
+ - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
+ ===> In which case, nothing is produced into `dst` !
+ + User __must__ test for such outcome and deal directly with uncompressed data
+ + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
+ Doing so would mess up with statistics history, leading to potential data corruption.
+ + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
+ + In case of multiple successive blocks, should some of them be uncompressed,
+ decoder must be informed of their existence in order to follow proper history.
+ Use ZSTD_insertBlock() for such a case.
+*/
+
+/*===== Raw zstd block functions =====*/
+ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
+ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
+
+
+/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
+ *
+ * *** OVERVIEW ***
+ * The Block-Level Sequence Producer API allows users to provide their own custom
+ * sequence producer which libzstd invokes to process each block. The produced list
+ * of sequences (literals and matches) is then post-processed by libzstd to produce
+ * valid compressed blocks.
+ *
+ * This block-level offload API is a more granular complement of the existing
+ * frame-level offload API compressSequences() (introduced in v1.5.1). It offers
+ * an easier migration story for applications already integrated with libzstd: the
+ * user application continues to invoke the same compression functions
+ * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits
+ * from the specific advantages of the external sequence producer. For example,
+ * the sequence producer could be tuned to take advantage of known characteristics
+ * of the input, to offer better speed / ratio, or could leverage hardware
+ * acceleration not available within libzstd itself.
+ *
+ * See contrib/externalSequenceProducer for an example program employing the
+ * Block-Level Sequence Producer API.
+ *
+ * *** USAGE ***
+ * The user is responsible for implementing a function of type
+ * ZSTD_sequenceProducer_F. For each block, zstd will pass the following
+ * arguments to the user-provided function:
+ *
+ * - sequenceProducerState: a pointer to a user-managed state for the sequence
+ * producer.
+ *
+ * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer.
+ * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory
+ * backing outSeqs is managed by the CCtx.
+ *
+ * - src, srcSize: an input buffer for the sequence producer to parse.
+ * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
+ *
+ * - dict, dictSize: a history buffer, which may be empty, which the sequence
+ * producer may reference as it parses the src buffer. Currently, zstd will
+ * always pass dictSize == 0 into external sequence producers, but this will
+ * change in the future.
+ *
+ * - compressionLevel: a signed integer representing the zstd compression level
+ * set by the user for the current operation. The sequence producer may choose
+ * to use this information to change its compression strategy and speed/ratio
+ * tradeoff. Note: the compression level does not reflect zstd parameters set
+ * through the advanced API.
+ *
+ * - windowSize: a size_t representing the maximum allowed offset for external
+ * sequences. Note that sequence offsets are sometimes allowed to exceed the
+ * windowSize if a dictionary is present, see doc/zstd_compression_format.md
+ * for details.
+ *
+ * The user-provided function shall return a size_t representing the number of
+ * sequences written to outSeqs. This return value will be treated as an error
+ * code if it is greater than outSeqsCapacity. The return value must be non-zero
+ * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided
+ * for convenience, but any value greater than outSeqsCapacity will be treated as
+ * an error code.
+ *
+ * If the user-provided function does not return an error code, the sequences
+ * written to outSeqs must be a valid parse of the src buffer. Data corruption may
+ * occur if the parse is not valid. A parse is defined to be valid if the
+ * following conditions hold:
+ * - The sum of matchLengths and literalLengths must equal srcSize.
+ * - All sequences in the parse, except for the final sequence, must have
+ * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have
+ * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0.
+ * - All offsets must respect the windowSize parameter as specified in
+ * doc/zstd_compression_format.md.
+ * - If the final sequence has matchLength == 0, it must also have offset == 0.
+ *
+ * zstd will only validate these conditions (and fail compression if they do not
+ * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
+ * validation has a performance cost.
+ *
+ * If the user-provided function returns an error, zstd will either fall back
+ * to an internal sequence producer or fail the compression operation. The user can
+ * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback
+ * cParam. Fallback compression will follow any other cParam settings, such as
+ * compression level, the same as in a normal compression operation.
+ *
+ * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F
+ * function by calling
+ * ZSTD_registerSequenceProducer(cctx,
+ * sequenceProducerState,
+ * sequenceProducer)
+ * This setting will persist until the next parameter reset of the CCtx.
+ *
+ * The sequenceProducerState must be initialized by the user before calling
+ * ZSTD_registerSequenceProducer(). The user is responsible for destroying the
+ * sequenceProducerState.
+ *
+ * *** LIMITATIONS ***
+ * This API is compatible with all zstd compression APIs which respect advanced parameters.
+ * However, there are three limitations:
+ *
+ * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported.
+ * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level
+ * external sequence producer.
+ * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some
+ * cases (see its documentation for details). Users must explicitly set
+ * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external
+ * sequence producer is registered.
+ * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
+ * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
+ * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence
+ * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog).
+ *
+ * Second, history buffers are not currently supported. Concretely, zstd will always pass
+ * dictSize == 0 to the external sequence producer (for now). This has two implications:
+ * - Dictionaries are not currently supported. Compression will *not* fail if the user
+ * references a dictionary, but the dictionary won't have any effect.
+ * - Stream history is not currently supported. All advanced compression APIs, including
+ * streaming APIs, work with external sequence producers, but each block is treated as
+ * an independent chunk without history from previous blocks.
+ *
+ * Third, multi-threading within a single compression is not currently supported. In other words,
+ * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered.
+ * Multi-threading across compressions is fine: simply create one CCtx per thread.
+ *
+ * Long-term, we plan to overcome all three limitations. There is no technical blocker to
+ * overcoming them. It is purely a question of engineering effort.
+ */
+
+#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1))
+
+typedef size_t ZSTD_sequenceProducer_F (
+ void* sequenceProducerState,
+ ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize,
+ int compressionLevel,
+ size_t windowSize
+);
+
+/*! ZSTD_registerSequenceProducer() :
+ * Instruct zstd to use a block-level external sequence producer function.
+ *
+ * The sequenceProducerState must be initialized by the caller, and the caller is
+ * responsible for managing its lifetime. This parameter is sticky across
+ * compressions. It will remain set until the user explicitly resets compression
+ * parameters.
+ *
+ * Sequence producer registration is considered to be an "advanced parameter",
+ * part of the "advanced API". This means it will only have an effect on compression
+ * APIs which respect advanced parameters, such as compress2() and compressStream2().
+ * Older compression APIs such as compressCCtx(), which predate the introduction of
+ * "advanced parameters", will ignore any external sequence producer setting.
+ *
+ * The sequence producer can be "cleared" by registering a NULL function pointer. This
+ * removes all limitations described above in the "LIMITATIONS" section of the API docs.
+ *
+ * The user is strongly encouraged to read the full API documentation (above) before
+ * calling this function. */
+ZSTDLIB_STATIC_API void
+ZSTD_registerSequenceProducer(
+ ZSTD_CCtx* cctx,
+ void* sequenceProducerState,
+ ZSTD_sequenceProducer_F* sequenceProducer
+);
+
+#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/contrib/zstd/zstd_common.c b/contrib/zstd/zstd_common.c
new file mode 100644
index 0000000..3208552
--- /dev/null
+++ b/contrib/zstd/zstd_common.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+
+/*-*************************************
+* Dependencies
+***************************************/
+#define ZSTD_DEPS_NEED_MALLOC
+#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
+#include "error_private.h"
+#include "zstd_internal.h"
+
+
+/*-****************************************
+* Version
+******************************************/
+unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; }
+
+const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
+
+
+/*-****************************************
+* ZSTD Error Management
+******************************************/
+#undef ZSTD_isError /* defined within zstd_internal.h */
+/*! ZSTD_isError() :
+ * tells if a return value is an error code
+ * symbol is required for external callers */
+unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
+
+/*! ZSTD_getErrorName() :
+ * provides error code string from function result (useful for debugging) */
+const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
+
+/*! ZSTD_getError() :
+ * convert a `size_t` function result into a proper ZSTD_errorCode enum */
+ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
+
+/*! ZSTD_getErrorString() :
+ * provides error code string from enum */
+const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
+
+
+
+/*=**************************************************************
+* Custom allocator
+****************************************************************/
+void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
+{
+ if (customMem.customAlloc)
+ return customMem.customAlloc(customMem.opaque, size);
+ return ZSTD_malloc(size);
+}
+
+void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
+{
+ if (customMem.customAlloc) {
+ /* calloc implemented as malloc+memset;
+ * not as efficient as calloc, but next best guess for custom malloc */
+ void* const ptr = customMem.customAlloc(customMem.opaque, size);
+ ZSTD_memset(ptr, 0, size);
+ return ptr;
+ }
+ return ZSTD_calloc(1, size);
+}
+
+void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
+{
+ if (ptr!=NULL) {
+ if (customMem.customFree)
+ customMem.customFree(customMem.opaque, ptr);
+ else
+ ZSTD_free(ptr);
+ }
+}
diff --git a/contrib/zstd/zstd_compress.c b/contrib/zstd/zstd_compress.c
new file mode 100644
index 0000000..e11252c
--- /dev/null
+++ b/contrib/zstd/zstd_compress.c
@@ -0,0 +1,6927 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */
+#include "mem.h"
+#include "hist.h" /* HIST_countFast_wksp */
+#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */
+#include "fse.h"
+#include "huf.h"
+#include "zstd_compress_internal.h"
+#include "zstd_compress_sequences.h"
+#include "zstd_compress_literals.h"
+#include "zstd_fast.h"
+#include "zstd_double_fast.h"
+#include "zstd_lazy.h"
+#include "zstd_opt.h"
+#include "zstd_ldm.h"
+#include "zstd_compress_superblock.h"
+#include "bits.h" /* ZSTD_highbit32 */
+
+/* ***************************************************************
+* Tuning parameters
+*****************************************************************/
+/*!
+ * COMPRESS_HEAPMODE :
+ * Select how default decompression function ZSTD_compress() allocates its context,
+ * on stack (0, default), or into heap (1).
+ * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected.
+ */
+#ifndef ZSTD_COMPRESS_HEAPMODE
+# define ZSTD_COMPRESS_HEAPMODE 0
+#endif
+
+/*!
+ * ZSTD_HASHLOG3_MAX :
+ * Maximum size of the hash table dedicated to find 3-bytes matches,
+ * in log format, aka 17 => 1 << 17 == 128Ki positions.
+ * This structure is only used in zstd_opt.
+ * Since allocation is centralized for all strategies, it has to be known here.
+ * The actual (selected) size of the hash table is then stored in ZSTD_matchState_t.hashLog3,
+ * so that zstd_opt.c doesn't need to know about this constant.
+ */
+#ifndef ZSTD_HASHLOG3_MAX
+# define ZSTD_HASHLOG3_MAX 17
+#endif
+
+/*-*************************************
+* Helper functions
+***************************************/
+/* ZSTD_compressBound()
+ * Note that the result from this function is only valid for
+ * the one-pass compression functions.
+ * When employing the streaming mode,
+ * if flushes are frequently altering the size of blocks,
+ * the overhead from block headers can make the compressed data larger
+ * than the return value of ZSTD_compressBound().
+ */
+size_t ZSTD_compressBound(size_t srcSize) {
+ size_t const r = ZSTD_COMPRESSBOUND(srcSize);
+ if (r==0) return ERROR(srcSize_wrong);
+ return r;
+}
+
+
+/*-*************************************
+* Context memory management
+***************************************/
+struct ZSTD_CDict_s {
+ const void* dictContent;
+ size_t dictContentSize;
+ ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */
+ U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */
+ ZSTD_cwksp workspace;
+ ZSTD_matchState_t matchState;
+ ZSTD_compressedBlockState_t cBlockState;
+ ZSTD_customMem customMem;
+ U32 dictID;
+ int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */
+ ZSTD_paramSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use
+ * row-based matchfinder. Unless the cdict is reloaded, we will use
+ * the same greedy/lazy matchfinder at compression time.
+ */
+}; /* typedef'd to ZSTD_CDict within "zstd.h" */
+
+ZSTD_CCtx* ZSTD_createCCtx(void)
+{
+ return ZSTD_createCCtx_advanced(ZSTD_defaultCMem);
+}
+
+static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager)
+{
+ assert(cctx != NULL);
+ ZSTD_memset(cctx, 0, sizeof(*cctx));
+ cctx->customMem = memManager;
+ cctx->bmi2 = ZSTD_cpuSupportsBmi2();
+ { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
+ assert(!ZSTD_isError(err));
+ (void)err;
+ }
+}
+
+ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
+{
+ ZSTD_STATIC_ASSERT(zcss_init==0);
+ ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1));
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+ { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem);
+ if (!cctx) return NULL;
+ ZSTD_initCCtx(cctx, customMem);
+ return cctx;
+ }
+}
+
+ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize)
+{
+ ZSTD_cwksp ws;
+ ZSTD_CCtx* cctx;
+ if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */
+ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */
+ ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc);
+
+ cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx));
+ if (cctx == NULL) return NULL;
+
+ ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx));
+ ZSTD_cwksp_move(&cctx->workspace, &ws);
+ cctx->staticSize = workspaceSize;
+
+ /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */
+ if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL;
+ cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t));
+ cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t));
+ cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE);
+ cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
+ return cctx;
+}
+
+/**
+ * Clears and frees all of the dictionaries in the CCtx.
+ */
+static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx)
+{
+ ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem);
+ ZSTD_freeCDict(cctx->localDict.cdict);
+ ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict));
+ ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict));
+ cctx->cdict = NULL;
+}
+
+static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict)
+{
+ size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0;
+ size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict);
+ return bufferSize + cdictSize;
+}
+
+static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx)
+{
+ assert(cctx != NULL);
+ assert(cctx->staticSize == 0);
+ ZSTD_clearAllDicts(cctx);
+#ifdef ZSTD_MULTITHREAD
+ ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL;
+#endif
+ ZSTD_cwksp_free(&cctx->workspace, cctx->customMem);
+}
+
+size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
+{
+ if (cctx==NULL) return 0; /* support free on NULL */
+ RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
+ "not compatible with static CCtx");
+ { int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
+ ZSTD_freeCCtxContent(cctx);
+ if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem);
+ }
+ return 0;
+}
+
+
+static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx)
+{
+#ifdef ZSTD_MULTITHREAD
+ return ZSTDMT_sizeof_CCtx(cctx->mtctx);
+#else
+ (void)cctx;
+ return 0;
+#endif
+}
+
+
+size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
+{
+ if (cctx==NULL) return 0; /* support sizeof on NULL */
+ /* cctx may be in the workspace */
+ return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx))
+ + ZSTD_cwksp_sizeof(&cctx->workspace)
+ + ZSTD_sizeof_localDict(cctx->localDict)
+ + ZSTD_sizeof_mtctx(cctx);
+}
+
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
+{
+ return ZSTD_sizeof_CCtx(zcs); /* same object */
+}
+
+/* private API call, for dictBuilder only */
+const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
+
+/* Returns true if the strategy supports using a row based matchfinder */
+static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) {
+ return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2);
+}
+
+/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder
+ * for this compression.
+ */
+static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_paramSwitch_e mode) {
+ assert(mode != ZSTD_ps_auto);
+ return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable);
+}
+
+/* Returns row matchfinder usage given an initial mode and cParams */
+static ZSTD_paramSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_paramSwitch_e mode,
+ const ZSTD_compressionParameters* const cParams) {
+#if defined(ZSTD_ARCH_X86_SSE2) || defined(ZSTD_ARCH_ARM_NEON)
+ int const kHasSIMD128 = 1;
+#else
+ int const kHasSIMD128 = 0;
+#endif
+ if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */
+ mode = ZSTD_ps_disable;
+ if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode;
+ if (kHasSIMD128) {
+ if (cParams->windowLog > 14) mode = ZSTD_ps_enable;
+ } else {
+ if (cParams->windowLog > 17) mode = ZSTD_ps_enable;
+ }
+ return mode;
+}
+
+/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */
+static ZSTD_paramSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_paramSwitch_e mode,
+ const ZSTD_compressionParameters* const cParams) {
+ if (mode != ZSTD_ps_auto) return mode;
+ return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable;
+}
+
+/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */
+static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
+ const ZSTD_paramSwitch_e useRowMatchFinder,
+ const U32 forDDSDict) {
+ assert(useRowMatchFinder != ZSTD_ps_auto);
+ /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate.
+ * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder.
+ */
+ return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder));
+}
+
+/* Returns ZSTD_ps_enable if compression parameters are such that we should
+ * enable long distance matching (wlog >= 27, strategy >= btopt).
+ * Returns ZSTD_ps_disable otherwise.
+ */
+static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
+ const ZSTD_compressionParameters* const cParams) {
+ if (mode != ZSTD_ps_auto) return mode;
+ return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable;
+}
+
+static int ZSTD_resolveExternalSequenceValidation(int mode) {
+ return mode;
+}
+
+/* Resolves maxBlockSize to the default if no value is present. */
+static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
+ if (maxBlockSize == 0) {
+ return ZSTD_BLOCKSIZE_MAX;
+ } else {
+ return maxBlockSize;
+ }
+}
+
+static ZSTD_paramSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_paramSwitch_e value, int cLevel) {
+ if (value != ZSTD_ps_auto) return value;
+ if (cLevel < 10) {
+ return ZSTD_ps_disable;
+ } else {
+ return ZSTD_ps_enable;
+ }
+}
+
+/* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
+ * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
+static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
+ return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast;
+}
+
+static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
+ ZSTD_compressionParameters cParams)
+{
+ ZSTD_CCtx_params cctxParams;
+ /* should not matter, as all cParams are presumed properly defined */
+ ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT);
+ cctxParams.cParams = cParams;
+
+ /* Adjust advanced params according to cParams */
+ cctxParams.ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams.ldmParams.enableLdm, &cParams);
+ if (cctxParams.ldmParams.enableLdm == ZSTD_ps_enable) {
+ ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams);
+ assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog);
+ assert(cctxParams.ldmParams.hashRateLog < 32);
+ }
+ cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
+ cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
+ cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
+ cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize);
+ cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes,
+ cctxParams.compressionLevel);
+ assert(!ZSTD_checkCParams(cParams));
+ return cctxParams;
+}
+
+static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced(
+ ZSTD_customMem customMem)
+{
+ ZSTD_CCtx_params* params;
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+ params = (ZSTD_CCtx_params*)ZSTD_customCalloc(
+ sizeof(ZSTD_CCtx_params), customMem);
+ if (!params) { return NULL; }
+ ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT);
+ params->customMem = customMem;
+ return params;
+}
+
+ZSTD_CCtx_params* ZSTD_createCCtxParams(void)
+{
+ return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem);
+}
+
+size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params)
+{
+ if (params == NULL) { return 0; }
+ ZSTD_customFree(params, params->customMem);
+ return 0;
+}
+
+size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params)
+{
+ return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT);
+}
+
+size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) {
+ RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!");
+ ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
+ cctxParams->compressionLevel = compressionLevel;
+ cctxParams->fParams.contentSizeFlag = 1;
+ return 0;
+}
+
+#define ZSTD_NO_CLEVEL 0
+
+/**
+ * Initializes `cctxParams` from `params` and `compressionLevel`.
+ * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL.
+ */
+static void
+ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams,
+ const ZSTD_parameters* params,
+ int compressionLevel)
+{
+ assert(!ZSTD_checkCParams(params->cParams));
+ ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
+ cctxParams->cParams = params->cParams;
+ cctxParams->fParams = params->fParams;
+ /* Should not matter, as all cParams are presumed properly defined.
+ * But, set it for tracing anyway.
+ */
+ cctxParams->compressionLevel = compressionLevel;
+ cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, &params->cParams);
+ cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, &params->cParams);
+ cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, &params->cParams);
+ cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
+ cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize);
+ cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel);
+ DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
+ cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
+}
+
+size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
+{
+ RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!");
+ FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , "");
+ ZSTD_CCtxParams_init_internal(cctxParams, &params, ZSTD_NO_CLEVEL);
+ return 0;
+}
+
+/**
+ * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone.
+ * @param params Validated zstd parameters.
+ */
+static void ZSTD_CCtxParams_setZstdParams(
+ ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params)
+{
+ assert(!ZSTD_checkCParams(params->cParams));
+ cctxParams->cParams = params->cParams;
+ cctxParams->fParams = params->fParams;
+ /* Should not matter, as all cParams are presumed properly defined.
+ * But, set it for tracing anyway.
+ */
+ cctxParams->compressionLevel = ZSTD_NO_CLEVEL;
+}
+
+ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
+{
+ ZSTD_bounds bounds = { 0, 0, 0 };
+
+ switch(param)
+ {
+ case ZSTD_c_compressionLevel:
+ bounds.lowerBound = ZSTD_minCLevel();
+ bounds.upperBound = ZSTD_maxCLevel();
+ return bounds;
+
+ case ZSTD_c_windowLog:
+ bounds.lowerBound = ZSTD_WINDOWLOG_MIN;
+ bounds.upperBound = ZSTD_WINDOWLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_hashLog:
+ bounds.lowerBound = ZSTD_HASHLOG_MIN;
+ bounds.upperBound = ZSTD_HASHLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_chainLog:
+ bounds.lowerBound = ZSTD_CHAINLOG_MIN;
+ bounds.upperBound = ZSTD_CHAINLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_searchLog:
+ bounds.lowerBound = ZSTD_SEARCHLOG_MIN;
+ bounds.upperBound = ZSTD_SEARCHLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_minMatch:
+ bounds.lowerBound = ZSTD_MINMATCH_MIN;
+ bounds.upperBound = ZSTD_MINMATCH_MAX;
+ return bounds;
+
+ case ZSTD_c_targetLength:
+ bounds.lowerBound = ZSTD_TARGETLENGTH_MIN;
+ bounds.upperBound = ZSTD_TARGETLENGTH_MAX;
+ return bounds;
+
+ case ZSTD_c_strategy:
+ bounds.lowerBound = ZSTD_STRATEGY_MIN;
+ bounds.upperBound = ZSTD_STRATEGY_MAX;
+ return bounds;
+
+ case ZSTD_c_contentSizeFlag:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_checksumFlag:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_dictIDFlag:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_nbWorkers:
+ bounds.lowerBound = 0;
+#ifdef ZSTD_MULTITHREAD
+ bounds.upperBound = ZSTDMT_NBWORKERS_MAX;
+#else
+ bounds.upperBound = 0;
+#endif
+ return bounds;
+
+ case ZSTD_c_jobSize:
+ bounds.lowerBound = 0;
+#ifdef ZSTD_MULTITHREAD
+ bounds.upperBound = ZSTDMT_JOBSIZE_MAX;
+#else
+ bounds.upperBound = 0;
+#endif
+ return bounds;
+
+ case ZSTD_c_overlapLog:
+#ifdef ZSTD_MULTITHREAD
+ bounds.lowerBound = ZSTD_OVERLAPLOG_MIN;
+ bounds.upperBound = ZSTD_OVERLAPLOG_MAX;
+#else
+ bounds.lowerBound = 0;
+ bounds.upperBound = 0;
+#endif
+ return bounds;
+
+ case ZSTD_c_enableDedicatedDictSearch:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_enableLongDistanceMatching:
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ case ZSTD_c_ldmHashLog:
+ bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN;
+ bounds.upperBound = ZSTD_LDM_HASHLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_ldmMinMatch:
+ bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN;
+ bounds.upperBound = ZSTD_LDM_MINMATCH_MAX;
+ return bounds;
+
+ case ZSTD_c_ldmBucketSizeLog:
+ bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN;
+ bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX;
+ return bounds;
+
+ case ZSTD_c_ldmHashRateLog:
+ bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN;
+ bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX;
+ return bounds;
+
+ /* experimental parameters */
+ case ZSTD_c_rsyncable:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_forceMaxWindow :
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_format:
+ ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
+ bounds.lowerBound = ZSTD_f_zstd1;
+ bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */
+ return bounds;
+
+ case ZSTD_c_forceAttachDict:
+ ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad);
+ bounds.lowerBound = ZSTD_dictDefaultAttach;
+ bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */
+ return bounds;
+
+ case ZSTD_c_literalCompressionMode:
+ ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_enable && ZSTD_ps_enable < ZSTD_ps_disable);
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ case ZSTD_c_targetCBlockSize:
+ bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN;
+ bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX;
+ return bounds;
+
+ case ZSTD_c_srcSizeHint:
+ bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN;
+ bounds.upperBound = ZSTD_SRCSIZEHINT_MAX;
+ return bounds;
+
+ case ZSTD_c_stableInBuffer:
+ case ZSTD_c_stableOutBuffer:
+ bounds.lowerBound = (int)ZSTD_bm_buffered;
+ bounds.upperBound = (int)ZSTD_bm_stable;
+ return bounds;
+
+ case ZSTD_c_blockDelimiters:
+ bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters;
+ bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters;
+ return bounds;
+
+ case ZSTD_c_validateSequences:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_useBlockSplitter:
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ case ZSTD_c_useRowMatchFinder:
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ case ZSTD_c_deterministicRefPrefix:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_prefetchCDictTables:
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ case ZSTD_c_enableSeqProducerFallback:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_maxBlockSize:
+ bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN;
+ bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
+ return bounds;
+
+ case ZSTD_c_searchForExternalRepcodes:
+ bounds.lowerBound = (int)ZSTD_ps_auto;
+ bounds.upperBound = (int)ZSTD_ps_disable;
+ return bounds;
+
+ default:
+ bounds.error = ERROR(parameter_unsupported);
+ return bounds;
+ }
+}
+
+/* ZSTD_cParam_clampBounds:
+ * Clamps the value into the bounded range.
+ */
+static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value)
+{
+ ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);
+ if (ZSTD_isError(bounds.error)) return bounds.error;
+ if (*value < bounds.lowerBound) *value = bounds.lowerBound;
+ if (*value > bounds.upperBound) *value = bounds.upperBound;
+ return 0;
+}
+
+#define BOUNDCHECK(cParam, val) { \
+ RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \
+ parameter_outOfBound, "Param out of bounds"); \
+}
+
+
+static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
+{
+ switch(param)
+ {
+ case ZSTD_c_compressionLevel:
+ case ZSTD_c_hashLog:
+ case ZSTD_c_chainLog:
+ case ZSTD_c_searchLog:
+ case ZSTD_c_minMatch:
+ case ZSTD_c_targetLength:
+ case ZSTD_c_strategy:
+ return 1;
+
+ case ZSTD_c_format:
+ case ZSTD_c_windowLog:
+ case ZSTD_c_contentSizeFlag:
+ case ZSTD_c_checksumFlag:
+ case ZSTD_c_dictIDFlag:
+ case ZSTD_c_forceMaxWindow :
+ case ZSTD_c_nbWorkers:
+ case ZSTD_c_jobSize:
+ case ZSTD_c_overlapLog:
+ case ZSTD_c_rsyncable:
+ case ZSTD_c_enableDedicatedDictSearch:
+ case ZSTD_c_enableLongDistanceMatching:
+ case ZSTD_c_ldmHashLog:
+ case ZSTD_c_ldmMinMatch:
+ case ZSTD_c_ldmBucketSizeLog:
+ case ZSTD_c_ldmHashRateLog:
+ case ZSTD_c_forceAttachDict:
+ case ZSTD_c_literalCompressionMode:
+ case ZSTD_c_targetCBlockSize:
+ case ZSTD_c_srcSizeHint:
+ case ZSTD_c_stableInBuffer:
+ case ZSTD_c_stableOutBuffer:
+ case ZSTD_c_blockDelimiters:
+ case ZSTD_c_validateSequences:
+ case ZSTD_c_useBlockSplitter:
+ case ZSTD_c_useRowMatchFinder:
+ case ZSTD_c_deterministicRefPrefix:
+ case ZSTD_c_prefetchCDictTables:
+ case ZSTD_c_enableSeqProducerFallback:
+ case ZSTD_c_maxBlockSize:
+ case ZSTD_c_searchForExternalRepcodes:
+ default:
+ return 0;
+ }
+}
+
+size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
+{
+ DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value);
+ if (cctx->streamStage != zcss_init) {
+ if (ZSTD_isUpdateAuthorized(param)) {
+ cctx->cParamsChanged = 1;
+ } else {
+ RETURN_ERROR(stage_wrong, "can only set params in cctx init stage");
+ } }
+
+ switch(param)
+ {
+ case ZSTD_c_nbWorkers:
+ RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported,
+ "MT not compatible with static alloc");
+ break;
+
+ case ZSTD_c_compressionLevel:
+ case ZSTD_c_windowLog:
+ case ZSTD_c_hashLog:
+ case ZSTD_c_chainLog:
+ case ZSTD_c_searchLog:
+ case ZSTD_c_minMatch:
+ case ZSTD_c_targetLength:
+ case ZSTD_c_strategy:
+ case ZSTD_c_ldmHashRateLog:
+ case ZSTD_c_format:
+ case ZSTD_c_contentSizeFlag:
+ case ZSTD_c_checksumFlag:
+ case ZSTD_c_dictIDFlag:
+ case ZSTD_c_forceMaxWindow:
+ case ZSTD_c_forceAttachDict:
+ case ZSTD_c_literalCompressionMode:
+ case ZSTD_c_jobSize:
+ case ZSTD_c_overlapLog:
+ case ZSTD_c_rsyncable:
+ case ZSTD_c_enableDedicatedDictSearch:
+ case ZSTD_c_enableLongDistanceMatching:
+ case ZSTD_c_ldmHashLog:
+ case ZSTD_c_ldmMinMatch:
+ case ZSTD_c_ldmBucketSizeLog:
+ case ZSTD_c_targetCBlockSize:
+ case ZSTD_c_srcSizeHint:
+ case ZSTD_c_stableInBuffer:
+ case ZSTD_c_stableOutBuffer:
+ case ZSTD_c_blockDelimiters:
+ case ZSTD_c_validateSequences:
+ case ZSTD_c_useBlockSplitter:
+ case ZSTD_c_useRowMatchFinder:
+ case ZSTD_c_deterministicRefPrefix:
+ case ZSTD_c_prefetchCDictTables:
+ case ZSTD_c_enableSeqProducerFallback:
+ case ZSTD_c_maxBlockSize:
+ case ZSTD_c_searchForExternalRepcodes:
+ break;
+
+ default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
+ }
+ return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value);
+}
+
+size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
+ ZSTD_cParameter param, int value)
+{
+ DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value);
+ switch(param)
+ {
+ case ZSTD_c_format :
+ BOUNDCHECK(ZSTD_c_format, value);
+ CCtxParams->format = (ZSTD_format_e)value;
+ return (size_t)CCtxParams->format;
+
+ case ZSTD_c_compressionLevel : {
+ FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), "");
+ if (value == 0)
+ CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */
+ else
+ CCtxParams->compressionLevel = value;
+ if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel;
+ return 0; /* return type (size_t) cannot represent negative values */
+ }
+
+ case ZSTD_c_windowLog :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_windowLog, value);
+ CCtxParams->cParams.windowLog = (U32)value;
+ return CCtxParams->cParams.windowLog;
+
+ case ZSTD_c_hashLog :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_hashLog, value);
+ CCtxParams->cParams.hashLog = (U32)value;
+ return CCtxParams->cParams.hashLog;
+
+ case ZSTD_c_chainLog :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_chainLog, value);
+ CCtxParams->cParams.chainLog = (U32)value;
+ return CCtxParams->cParams.chainLog;
+
+ case ZSTD_c_searchLog :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_searchLog, value);
+ CCtxParams->cParams.searchLog = (U32)value;
+ return (size_t)value;
+
+ case ZSTD_c_minMatch :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_minMatch, value);
+ CCtxParams->cParams.minMatch = (U32)value;
+ return CCtxParams->cParams.minMatch;
+
+ case ZSTD_c_targetLength :
+ BOUNDCHECK(ZSTD_c_targetLength, value);
+ CCtxParams->cParams.targetLength = (U32)value;
+ return CCtxParams->cParams.targetLength;
+
+ case ZSTD_c_strategy :
+ if (value!=0) /* 0 => use default */
+ BOUNDCHECK(ZSTD_c_strategy, value);
+ CCtxParams->cParams.strategy = (ZSTD_strategy)value;
+ return (size_t)CCtxParams->cParams.strategy;
+
+ case ZSTD_c_contentSizeFlag :
+ /* Content size written in frame header _when known_ (default:1) */
+ DEBUGLOG(4, "set content size flag = %u", (value!=0));
+ CCtxParams->fParams.contentSizeFlag = value != 0;
+ return (size_t)CCtxParams->fParams.contentSizeFlag;
+
+ case ZSTD_c_checksumFlag :
+ /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
+ CCtxParams->fParams.checksumFlag = value != 0;
+ return (size_t)CCtxParams->fParams.checksumFlag;
+
+ case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
+ DEBUGLOG(4, "set dictIDFlag = %u", (value!=0));
+ CCtxParams->fParams.noDictIDFlag = !value;
+ return !CCtxParams->fParams.noDictIDFlag;
+
+ case ZSTD_c_forceMaxWindow :
+ CCtxParams->forceWindow = (value != 0);
+ return (size_t)CCtxParams->forceWindow;
+
+ case ZSTD_c_forceAttachDict : {
+ const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value;
+ BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref);
+ CCtxParams->attachDictPref = pref;
+ return CCtxParams->attachDictPref;
+ }
+
+ case ZSTD_c_literalCompressionMode : {
+ const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value;
+ BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm);
+ CCtxParams->literalCompressionMode = lcm;
+ return CCtxParams->literalCompressionMode;
+ }
+
+ case ZSTD_c_nbWorkers :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
+ return 0;
+#else
+ FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), "");
+ CCtxParams->nbWorkers = value;
+ return CCtxParams->nbWorkers;
+#endif
+
+ case ZSTD_c_jobSize :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
+ return 0;
+#else
+ /* Adjust to the minimum non-default value. */
+ if (value != 0 && value < ZSTDMT_JOBSIZE_MIN)
+ value = ZSTDMT_JOBSIZE_MIN;
+ FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), "");
+ assert(value >= 0);
+ CCtxParams->jobSize = value;
+ return CCtxParams->jobSize;
+#endif
+
+ case ZSTD_c_overlapLog :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
+ return 0;
+#else
+ FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), "");
+ CCtxParams->overlapLog = value;
+ return CCtxParams->overlapLog;
+#endif
+
+ case ZSTD_c_rsyncable :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
+ return 0;
+#else
+ FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), "");
+ CCtxParams->rsyncable = value;
+ return CCtxParams->rsyncable;
+#endif
+
+ case ZSTD_c_enableDedicatedDictSearch :
+ CCtxParams->enableDedicatedDictSearch = (value!=0);
+ return (size_t)CCtxParams->enableDedicatedDictSearch;
+
+ case ZSTD_c_enableLongDistanceMatching :
+ BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value);
+ CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value;
+ return CCtxParams->ldmParams.enableLdm;
+
+ case ZSTD_c_ldmHashLog :
+ if (value!=0) /* 0 ==> auto */
+ BOUNDCHECK(ZSTD_c_ldmHashLog, value);
+ CCtxParams->ldmParams.hashLog = (U32)value;
+ return CCtxParams->ldmParams.hashLog;
+
+ case ZSTD_c_ldmMinMatch :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_ldmMinMatch, value);
+ CCtxParams->ldmParams.minMatchLength = (U32)value;
+ return CCtxParams->ldmParams.minMatchLength;
+
+ case ZSTD_c_ldmBucketSizeLog :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value);
+ CCtxParams->ldmParams.bucketSizeLog = (U32)value;
+ return CCtxParams->ldmParams.bucketSizeLog;
+
+ case ZSTD_c_ldmHashRateLog :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_ldmHashRateLog, value);
+ CCtxParams->ldmParams.hashRateLog = (U32)value;
+ return CCtxParams->ldmParams.hashRateLog;
+
+ case ZSTD_c_targetCBlockSize :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_targetCBlockSize, value);
+ CCtxParams->targetCBlockSize = (U32)value;
+ return CCtxParams->targetCBlockSize;
+
+ case ZSTD_c_srcSizeHint :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_srcSizeHint, value);
+ CCtxParams->srcSizeHint = value;
+ return (size_t)CCtxParams->srcSizeHint;
+
+ case ZSTD_c_stableInBuffer:
+ BOUNDCHECK(ZSTD_c_stableInBuffer, value);
+ CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value;
+ return CCtxParams->inBufferMode;
+
+ case ZSTD_c_stableOutBuffer:
+ BOUNDCHECK(ZSTD_c_stableOutBuffer, value);
+ CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value;
+ return CCtxParams->outBufferMode;
+
+ case ZSTD_c_blockDelimiters:
+ BOUNDCHECK(ZSTD_c_blockDelimiters, value);
+ CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value;
+ return CCtxParams->blockDelimiters;
+
+ case ZSTD_c_validateSequences:
+ BOUNDCHECK(ZSTD_c_validateSequences, value);
+ CCtxParams->validateSequences = value;
+ return CCtxParams->validateSequences;
+
+ case ZSTD_c_useBlockSplitter:
+ BOUNDCHECK(ZSTD_c_useBlockSplitter, value);
+ CCtxParams->useBlockSplitter = (ZSTD_paramSwitch_e)value;
+ return CCtxParams->useBlockSplitter;
+
+ case ZSTD_c_useRowMatchFinder:
+ BOUNDCHECK(ZSTD_c_useRowMatchFinder, value);
+ CCtxParams->useRowMatchFinder = (ZSTD_paramSwitch_e)value;
+ return CCtxParams->useRowMatchFinder;
+
+ case ZSTD_c_deterministicRefPrefix:
+ BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value);
+ CCtxParams->deterministicRefPrefix = !!value;
+ return CCtxParams->deterministicRefPrefix;
+
+ case ZSTD_c_prefetchCDictTables:
+ BOUNDCHECK(ZSTD_c_prefetchCDictTables, value);
+ CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value;
+ return CCtxParams->prefetchCDictTables;
+
+ case ZSTD_c_enableSeqProducerFallback:
+ BOUNDCHECK(ZSTD_c_enableSeqProducerFallback, value);
+ CCtxParams->enableMatchFinderFallback = value;
+ return CCtxParams->enableMatchFinderFallback;
+
+ case ZSTD_c_maxBlockSize:
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_maxBlockSize, value);
+ CCtxParams->maxBlockSize = value;
+ return CCtxParams->maxBlockSize;
+
+ case ZSTD_c_searchForExternalRepcodes:
+ BOUNDCHECK(ZSTD_c_searchForExternalRepcodes, value);
+ CCtxParams->searchForExternalRepcodes = (ZSTD_paramSwitch_e)value;
+ return CCtxParams->searchForExternalRepcodes;
+
+ default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
+ }
+}
+
+size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value)
+{
+ return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value);
+}
+
+size_t ZSTD_CCtxParams_getParameter(
+ ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value)
+{
+ switch(param)
+ {
+ case ZSTD_c_format :
+ *value = CCtxParams->format;
+ break;
+ case ZSTD_c_compressionLevel :
+ *value = CCtxParams->compressionLevel;
+ break;
+ case ZSTD_c_windowLog :
+ *value = (int)CCtxParams->cParams.windowLog;
+ break;
+ case ZSTD_c_hashLog :
+ *value = (int)CCtxParams->cParams.hashLog;
+ break;
+ case ZSTD_c_chainLog :
+ *value = (int)CCtxParams->cParams.chainLog;
+ break;
+ case ZSTD_c_searchLog :
+ *value = CCtxParams->cParams.searchLog;
+ break;
+ case ZSTD_c_minMatch :
+ *value = CCtxParams->cParams.minMatch;
+ break;
+ case ZSTD_c_targetLength :
+ *value = CCtxParams->cParams.targetLength;
+ break;
+ case ZSTD_c_strategy :
+ *value = (unsigned)CCtxParams->cParams.strategy;
+ break;
+ case ZSTD_c_contentSizeFlag :
+ *value = CCtxParams->fParams.contentSizeFlag;
+ break;
+ case ZSTD_c_checksumFlag :
+ *value = CCtxParams->fParams.checksumFlag;
+ break;
+ case ZSTD_c_dictIDFlag :
+ *value = !CCtxParams->fParams.noDictIDFlag;
+ break;
+ case ZSTD_c_forceMaxWindow :
+ *value = CCtxParams->forceWindow;
+ break;
+ case ZSTD_c_forceAttachDict :
+ *value = CCtxParams->attachDictPref;
+ break;
+ case ZSTD_c_literalCompressionMode :
+ *value = CCtxParams->literalCompressionMode;
+ break;
+ case ZSTD_c_nbWorkers :
+#ifndef ZSTD_MULTITHREAD
+ assert(CCtxParams->nbWorkers == 0);
+#endif
+ *value = CCtxParams->nbWorkers;
+ break;
+ case ZSTD_c_jobSize :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
+#else
+ assert(CCtxParams->jobSize <= INT_MAX);
+ *value = (int)CCtxParams->jobSize;
+ break;
+#endif
+ case ZSTD_c_overlapLog :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
+#else
+ *value = CCtxParams->overlapLog;
+ break;
+#endif
+ case ZSTD_c_rsyncable :
+#ifndef ZSTD_MULTITHREAD
+ RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
+#else
+ *value = CCtxParams->rsyncable;
+ break;
+#endif
+ case ZSTD_c_enableDedicatedDictSearch :
+ *value = CCtxParams->enableDedicatedDictSearch;
+ break;
+ case ZSTD_c_enableLongDistanceMatching :
+ *value = CCtxParams->ldmParams.enableLdm;
+ break;
+ case ZSTD_c_ldmHashLog :
+ *value = CCtxParams->ldmParams.hashLog;
+ break;
+ case ZSTD_c_ldmMinMatch :
+ *value = CCtxParams->ldmParams.minMatchLength;
+ break;
+ case ZSTD_c_ldmBucketSizeLog :
+ *value = CCtxParams->ldmParams.bucketSizeLog;
+ break;
+ case ZSTD_c_ldmHashRateLog :
+ *value = CCtxParams->ldmParams.hashRateLog;
+ break;
+ case ZSTD_c_targetCBlockSize :
+ *value = (int)CCtxParams->targetCBlockSize;
+ break;
+ case ZSTD_c_srcSizeHint :
+ *value = (int)CCtxParams->srcSizeHint;
+ break;
+ case ZSTD_c_stableInBuffer :
+ *value = (int)CCtxParams->inBufferMode;
+ break;
+ case ZSTD_c_stableOutBuffer :
+ *value = (int)CCtxParams->outBufferMode;
+ break;
+ case ZSTD_c_blockDelimiters :
+ *value = (int)CCtxParams->blockDelimiters;
+ break;
+ case ZSTD_c_validateSequences :
+ *value = (int)CCtxParams->validateSequences;
+ break;
+ case ZSTD_c_useBlockSplitter :
+ *value = (int)CCtxParams->useBlockSplitter;
+ break;
+ case ZSTD_c_useRowMatchFinder :
+ *value = (int)CCtxParams->useRowMatchFinder;
+ break;
+ case ZSTD_c_deterministicRefPrefix:
+ *value = (int)CCtxParams->deterministicRefPrefix;
+ break;
+ case ZSTD_c_prefetchCDictTables:
+ *value = (int)CCtxParams->prefetchCDictTables;
+ break;
+ case ZSTD_c_enableSeqProducerFallback:
+ *value = CCtxParams->enableMatchFinderFallback;
+ break;
+ case ZSTD_c_maxBlockSize:
+ *value = (int)CCtxParams->maxBlockSize;
+ break;
+ case ZSTD_c_searchForExternalRepcodes:
+ *value = (int)CCtxParams->searchForExternalRepcodes;
+ break;
+ default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
+ }
+ return 0;
+}
+
+/** ZSTD_CCtx_setParametersUsingCCtxParams() :
+ * just applies `params` into `cctx`
+ * no action is performed, parameters are merely stored.
+ * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx.
+ * This is possible even if a compression is ongoing.
+ * In which case, new parameters will be applied on the fly, starting with next compression job.
+ */
+size_t ZSTD_CCtx_setParametersUsingCCtxParams(
+ ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params)
+{
+ DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams");
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "The context is in the wrong stage!");
+ RETURN_ERROR_IF(cctx->cdict, stage_wrong,
+ "Can't override parameters with cdict attached (some must "
+ "be inherited from the cdict).");
+
+ cctx->requestedParams = *params;
+ return 0;
+}
+
+size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams)
+{
+ DEBUGLOG(4, "ZSTD_CCtx_setCParams");
+ assert(cctx != NULL);
+ if (cctx->streamStage != zcss_init) {
+ /* All parameters in @cparams are allowed to be updated during MT compression.
+ * This must be signaled, so that MT compression picks up the changes */
+ cctx->cParamsChanged = 1;
+ }
+ /* only update if parameters are valid */
+ FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), "");
+ cctx->requestedParams.cParams = cparams;
+ return 0;
+}
+
+size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
+{
+ DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize);
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't set pledgedSrcSize when not in init stage.");
+ cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
+ return 0;
+}
+
+static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(
+ int const compressionLevel,
+ size_t const dictSize);
+static int ZSTD_dedicatedDictSearch_isSupported(
+ const ZSTD_compressionParameters* cParams);
+static void ZSTD_dedicatedDictSearch_revertCParams(
+ ZSTD_compressionParameters* cParams);
+
+/**
+ * Initializes the local dict using the requested parameters.
+ * NOTE: This does not use the pledged src size, because it may be used for more
+ * than one compression.
+ */
+static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx)
+{
+ ZSTD_localDict* const dl = &cctx->localDict;
+ if (dl->dict == NULL) {
+ /* No local dictionary. */
+ assert(dl->dictBuffer == NULL);
+ assert(dl->cdict == NULL);
+ assert(dl->dictSize == 0);
+ return 0;
+ }
+ if (dl->cdict != NULL) {
+ assert(cctx->cdict == dl->cdict);
+ /* Local dictionary already initialized. */
+ return 0;
+ }
+ assert(dl->dictSize > 0);
+ assert(cctx->cdict == NULL);
+ assert(cctx->prefixDict.dict == NULL);
+
+ dl->cdict = ZSTD_createCDict_advanced2(
+ dl->dict,
+ dl->dictSize,
+ ZSTD_dlm_byRef,
+ dl->dictContentType,
+ &cctx->requestedParams,
+ cctx->customMem);
+ RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed");
+ cctx->cdict = dl->cdict;
+ return 0;
+}
+
+size_t ZSTD_CCtx_loadDictionary_advanced(
+ ZSTD_CCtx* cctx, const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
+{
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't load a dictionary when ctx is not in init stage.");
+ DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize);
+ ZSTD_clearAllDicts(cctx); /* in case one already exists */
+ if (dict == NULL || dictSize == 0) /* no dictionary mode */
+ return 0;
+ if (dictLoadMethod == ZSTD_dlm_byRef) {
+ cctx->localDict.dict = dict;
+ } else {
+ void* dictBuffer;
+ RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
+ "no malloc for static CCtx");
+ dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem);
+ RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!");
+ ZSTD_memcpy(dictBuffer, dict, dictSize);
+ cctx->localDict.dictBuffer = dictBuffer;
+ cctx->localDict.dict = dictBuffer;
+ }
+ cctx->localDict.dictSize = dictSize;
+ cctx->localDict.dictContentType = dictContentType;
+ return 0;
+}
+
+size_t ZSTD_CCtx_loadDictionary_byReference(
+ ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
+{
+ return ZSTD_CCtx_loadDictionary_advanced(
+ cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
+}
+
+size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
+{
+ return ZSTD_CCtx_loadDictionary_advanced(
+ cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
+}
+
+
+size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
+{
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't ref a dict when ctx not in init stage.");
+ /* Free the existing local cdict (if any) to save memory. */
+ ZSTD_clearAllDicts(cctx);
+ cctx->cdict = cdict;
+ return 0;
+}
+
+size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool)
+{
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't ref a pool when ctx not in init stage.");
+ cctx->pool = pool;
+ return 0;
+}
+
+size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
+{
+ return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent);
+}
+
+size_t ZSTD_CCtx_refPrefix_advanced(
+ ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
+{
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't ref a prefix when ctx not in init stage.");
+ ZSTD_clearAllDicts(cctx);
+ if (prefix != NULL && prefixSize > 0) {
+ cctx->prefixDict.dict = prefix;
+ cctx->prefixDict.dictSize = prefixSize;
+ cctx->prefixDict.dictContentType = dictContentType;
+ }
+ return 0;
+}
+
+/*! ZSTD_CCtx_reset() :
+ * Also dumps dictionary */
+size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
+{
+ if ( (reset == ZSTD_reset_session_only)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ cctx->streamStage = zcss_init;
+ cctx->pledgedSrcSizePlusOne = 0;
+ }
+ if ( (reset == ZSTD_reset_parameters)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+ "Can't reset parameters only when not in init stage.");
+ ZSTD_clearAllDicts(cctx);
+ ZSTD_memset(&cctx->externalMatchCtx, 0, sizeof(cctx->externalMatchCtx));
+ return ZSTD_CCtxParams_reset(&cctx->requestedParams);
+ }
+ return 0;
+}
+
+
+/** ZSTD_checkCParams() :
+ control CParam values remain within authorized range.
+ @return : 0, or an error code if one value is beyond authorized range */
+size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
+{
+ BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog);
+ BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog);
+ BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog);
+ BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog);
+ BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch);
+ BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength);
+ BOUNDCHECK(ZSTD_c_strategy, cParams.strategy);
+ return 0;
+}
+
+/** ZSTD_clampCParams() :
+ * make CParam values within valid range.
+ * @return : valid CParams */
+static ZSTD_compressionParameters
+ZSTD_clampCParams(ZSTD_compressionParameters cParams)
+{
+# define CLAMP_TYPE(cParam, val, type) { \
+ ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \
+ if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound; \
+ else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \
+ }
+# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned)
+ CLAMP(ZSTD_c_windowLog, cParams.windowLog);
+ CLAMP(ZSTD_c_chainLog, cParams.chainLog);
+ CLAMP(ZSTD_c_hashLog, cParams.hashLog);
+ CLAMP(ZSTD_c_searchLog, cParams.searchLog);
+ CLAMP(ZSTD_c_minMatch, cParams.minMatch);
+ CLAMP(ZSTD_c_targetLength,cParams.targetLength);
+ CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy);
+ return cParams;
+}
+
+/** ZSTD_cycleLog() :
+ * condition for correct operation : hashLog > 1 */
+U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
+{
+ U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2);
+ return hashLog - btScale;
+}
+
+/** ZSTD_dictAndWindowLog() :
+ * Returns an adjusted window log that is large enough to fit the source and the dictionary.
+ * The zstd format says that the entire dictionary is valid if one byte of the dictionary
+ * is within the window. So the hashLog and chainLog should be large enough to reference both
+ * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing
+ * the hashLog and windowLog.
+ * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN.
+ */
+static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize)
+{
+ const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX;
+ /* No dictionary ==> No change */
+ if (dictSize == 0) {
+ return windowLog;
+ }
+ assert(windowLog <= ZSTD_WINDOWLOG_MAX);
+ assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */
+ {
+ U64 const windowSize = 1ULL << windowLog;
+ U64 const dictAndWindowSize = dictSize + windowSize;
+ /* If the window size is already large enough to fit both the source and the dictionary
+ * then just use the window size. Otherwise adjust so that it fits the dictionary and
+ * the window.
+ */
+ if (windowSize >= dictSize + srcSize) {
+ return windowLog; /* Window size large enough already */
+ } else if (dictAndWindowSize >= maxWindowSize) {
+ return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */
+ } else {
+ return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1;
+ }
+ }
+}
+
+/** ZSTD_adjustCParams_internal() :
+ * optimize `cPar` for a specified input (`srcSize` and `dictSize`).
+ * mostly downsize to reduce memory consumption and initialization latency.
+ * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known.
+ * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`.
+ * note : `srcSize==0` means 0!
+ * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */
+static ZSTD_compressionParameters
+ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
+ unsigned long long srcSize,
+ size_t dictSize,
+ ZSTD_cParamMode_e mode,
+ ZSTD_paramSwitch_e useRowMatchFinder)
+{
+ const U64 minSrcSize = 513; /* (1<<9) + 1 */
+ const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
+ assert(ZSTD_checkCParams(cPar)==0);
+
+ switch (mode) {
+ case ZSTD_cpm_unknown:
+ case ZSTD_cpm_noAttachDict:
+ /* If we don't know the source size, don't make any
+ * assumptions about it. We will already have selected
+ * smaller parameters if a dictionary is in use.
+ */
+ break;
+ case ZSTD_cpm_createCDict:
+ /* Assume a small source size when creating a dictionary
+ * with an unknown source size.
+ */
+ if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN)
+ srcSize = minSrcSize;
+ break;
+ case ZSTD_cpm_attachDict:
+ /* Dictionary has its own dedicated parameters which have
+ * already been selected. We are selecting parameters
+ * for only the source.
+ */
+ dictSize = 0;
+ break;
+ default:
+ assert(0);
+ break;
+ }
+
+ /* resize windowLog if input is small enough, to use less memory */
+ if ( (srcSize <= maxWindowResize)
+ && (dictSize <= maxWindowResize) ) {
+ U32 const tSize = (U32)(srcSize + dictSize);
+ static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
+ U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :
+ ZSTD_highbit32(tSize-1) + 1;
+ if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
+ }
+ if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+ U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize);
+ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
+ if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1;
+ if (cycleLog > dictAndWindowLog)
+ cPar.chainLog -= (cycleLog - dictAndWindowLog);
+ }
+
+ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
+ cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */
+
+ /* We can't use more than 32 bits of hash in total, so that means that we require:
+ * (hashLog + 8) <= 32 && (chainLog + 8) <= 32
+ */
+ if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) {
+ U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS;
+ if (cPar.hashLog > maxShortCacheHashLog) {
+ cPar.hashLog = maxShortCacheHashLog;
+ }
+ if (cPar.chainLog > maxShortCacheHashLog) {
+ cPar.chainLog = maxShortCacheHashLog;
+ }
+ }
+
+
+ /* At this point, we aren't 100% sure if we are using the row match finder.
+ * Unless it is explicitly disabled, conservatively assume that it is enabled.
+ * In this case it will only be disabled for small sources, so shrinking the
+ * hash log a little bit shouldn't result in any ratio loss.
+ */
+ if (useRowMatchFinder == ZSTD_ps_auto)
+ useRowMatchFinder = ZSTD_ps_enable;
+
+ /* We can't hash more than 32-bits in total. So that means that we require:
+ * (hashLog - rowLog + 8) <= 32
+ */
+ if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) {
+ /* Switch to 32-entry rows if searchLog is 5 (or more) */
+ U32 const rowLog = BOUNDED(4, cPar.searchLog, 6);
+ U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS;
+ U32 const maxHashLog = maxRowHashLog + rowLog;
+ assert(cPar.hashLog >= rowLog);
+ if (cPar.hashLog > maxHashLog) {
+ cPar.hashLog = maxHashLog;
+ }
+ }
+
+ return cPar;
+}
+
+ZSTD_compressionParameters
+ZSTD_adjustCParams(ZSTD_compressionParameters cPar,
+ unsigned long long srcSize,
+ size_t dictSize)
+{
+ cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */
+ if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN;
+ return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto);
+}
+
+static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
+static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
+
+static void ZSTD_overrideCParams(
+ ZSTD_compressionParameters* cParams,
+ const ZSTD_compressionParameters* overrides)
+{
+ if (overrides->windowLog) cParams->windowLog = overrides->windowLog;
+ if (overrides->hashLog) cParams->hashLog = overrides->hashLog;
+ if (overrides->chainLog) cParams->chainLog = overrides->chainLog;
+ if (overrides->searchLog) cParams->searchLog = overrides->searchLog;
+ if (overrides->minMatch) cParams->minMatch = overrides->minMatch;
+ if (overrides->targetLength) cParams->targetLength = overrides->targetLength;
+ if (overrides->strategy) cParams->strategy = overrides->strategy;
+}
+
+ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
+ const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
+{
+ ZSTD_compressionParameters cParams;
+ if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) {
+ srcSizeHint = CCtxParams->srcSizeHint;
+ }
+ cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode);
+ if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
+ ZSTD_overrideCParams(&cParams, &CCtxParams->cParams);
+ assert(!ZSTD_checkCParams(cParams));
+ /* srcSizeHint == 0 means 0 */
+ return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder);
+}
+
+static size_t
+ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
+ const ZSTD_paramSwitch_e useRowMatchFinder,
+ const U32 enableDedicatedDictSearch,
+ const U32 forCCtx)
+{
+ /* chain table size should be 0 for fast or row-hash strategies */
+ size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx)
+ ? ((size_t)1 << cParams->chainLog)
+ : 0;
+ size_t const hSize = ((size_t)1) << cParams->hashLog;
+ U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
+ size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
+ /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't
+ * surrounded by redzones in ASAN. */
+ size_t const tableSpace = chainSize * sizeof(U32)
+ + hSize * sizeof(U32)
+ + h3Size * sizeof(U32);
+ size_t const optPotentialSpace =
+ ZSTD_cwksp_aligned_alloc_size((MaxML+1) * sizeof(U32))
+ + ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32))
+ + ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32))
+ + ZSTD_cwksp_aligned_alloc_size((1<<Litbits) * sizeof(U32))
+ + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t))
+ + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
+ size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)
+ ? ZSTD_cwksp_aligned_alloc_size(hSize*sizeof(U16))
+ : 0;
+ size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
+ ? optPotentialSpace
+ : 0;
+ size_t const slackSpace = ZSTD_cwksp_slack_space_required();
+
+ /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */
+ ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4);
+ assert(useRowMatchFinder != ZSTD_ps_auto);
+
+ DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u",
+ (U32)chainSize, (U32)hSize, (U32)h3Size);
+ return tableSpace + optSpace + slackSpace + lazyAdditionalSpace;
+}
+
+/* Helper function for calculating memory requirements.
+ * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */
+static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) {
+ U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4;
+ return blockSize / divider;
+}
+
+static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
+ const ZSTD_compressionParameters* cParams,
+ const ldmParams_t* ldmParams,
+ const int isStatic,
+ const ZSTD_paramSwitch_e useRowMatchFinder,
+ const size_t buffInSize,
+ const size_t buffOutSize,
+ const U64 pledgedSrcSize,
+ int useSequenceProducer,
+ size_t maxBlockSize)
+{
+ size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
+ size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
+ size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer);
+ size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
+ + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
+ + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
+ size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE);
+ size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
+ size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1);
+
+ size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams);
+ size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize);
+ size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ?
+ ZSTD_cwksp_aligned_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0;
+
+
+ size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize)
+ + ZSTD_cwksp_alloc_size(buffOutSize);
+
+ size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
+
+ size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+ size_t const externalSeqSpace = useSequenceProducer
+ ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
+ : 0;
+
+ size_t const neededSpace =
+ cctxSpace +
+ entropySpace +
+ blockStateSpace +
+ ldmSpace +
+ ldmSeqSpace +
+ matchStateSize +
+ tokenSpace +
+ bufferSpace +
+ externalSeqSpace;
+
+ DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
+ return neededSpace;
+}
+
+size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
+{
+ ZSTD_compressionParameters const cParams =
+ ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
+ ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder,
+ &cParams);
+
+ RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
+ /* estimateCCtxSize is for one-shot compression. So no buffers should
+ * be needed. However, we still allocate two 0-sized buffers, which can
+ * take space under ASAN. */
+ return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
+ &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize);
+}
+
+size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
+{
+ ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams);
+ if (ZSTD_rowMatchFinderSupported(cParams.strategy)) {
+ /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */
+ size_t noRowCCtxSize;
+ size_t rowCCtxSize;
+ initialParams.useRowMatchFinder = ZSTD_ps_disable;
+ noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
+ initialParams.useRowMatchFinder = ZSTD_ps_enable;
+ rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
+ return MAX(noRowCCtxSize, rowCCtxSize);
+ } else {
+ return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
+ }
+}
+
+static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel)
+{
+ int tier = 0;
+ size_t largestSize = 0;
+ static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN};
+ for (; tier < 4; ++tier) {
+ /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict);
+ largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize);
+ }
+ return largestSize;
+}
+
+size_t ZSTD_estimateCCtxSize(int compressionLevel)
+{
+ int level;
+ size_t memBudget = 0;
+ for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
+ /* Ensure monotonically increasing memory usage as compression level increases */
+ size_t const newMB = ZSTD_estimateCCtxSize_internal(level);
+ if (newMB > memBudget) memBudget = newMB;
+ }
+ return memBudget;
+}
+
+size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
+{
+ RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
+ { ZSTD_compressionParameters const cParams =
+ ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
+ size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog);
+ size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered)
+ ? ((size_t)1 << cParams.windowLog) + blockSize
+ : 0;
+ size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered)
+ ? ZSTD_compressBound(blockSize) + 1
+ : 0;
+ ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, &params->cParams);
+
+ return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
+ &cParams, &params->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
+ ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize);
+ }
+}
+
+size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams)
+{
+ ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams);
+ if (ZSTD_rowMatchFinderSupported(cParams.strategy)) {
+ /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */
+ size_t noRowCCtxSize;
+ size_t rowCCtxSize;
+ initialParams.useRowMatchFinder = ZSTD_ps_disable;
+ noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
+ initialParams.useRowMatchFinder = ZSTD_ps_enable;
+ rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
+ return MAX(noRowCCtxSize, rowCCtxSize);
+ } else {
+ return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
+ }
+}
+
+static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel)
+{
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
+ return ZSTD_estimateCStreamSize_usingCParams(cParams);
+}
+
+size_t ZSTD_estimateCStreamSize(int compressionLevel)
+{
+ int level;
+ size_t memBudget = 0;
+ for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
+ size_t const newMB = ZSTD_estimateCStreamSize_internal(level);
+ if (newMB > memBudget) memBudget = newMB;
+ }
+ return memBudget;
+}
+
+/* ZSTD_getFrameProgression():
+ * tells how much data has been consumed (input) and produced (output) for current frame.
+ * able to count progression inside worker threads (non-blocking mode).
+ */
+ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx)
+{
+#ifdef ZSTD_MULTITHREAD
+ if (cctx->appliedParams.nbWorkers > 0) {
+ return ZSTDMT_getFrameProgression(cctx->mtctx);
+ }
+#endif
+ { ZSTD_frameProgression fp;
+ size_t const buffered = (cctx->inBuff == NULL) ? 0 :
+ cctx->inBuffPos - cctx->inToCompress;
+ if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress);
+ assert(buffered <= ZSTD_BLOCKSIZE_MAX);
+ fp.ingested = cctx->consumedSrcSize + buffered;
+ fp.consumed = cctx->consumedSrcSize;
+ fp.produced = cctx->producedCSize;
+ fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */
+ fp.currentJobID = 0;
+ fp.nbActiveWorkers = 0;
+ return fp;
+} }
+
+/*! ZSTD_toFlushNow()
+ * Only useful for multithreading scenarios currently (nbWorkers >= 1).
+ */
+size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx)
+{
+#ifdef ZSTD_MULTITHREAD
+ if (cctx->appliedParams.nbWorkers > 0) {
+ return ZSTDMT_toFlushNow(cctx->mtctx);
+ }
+#endif
+ (void)cctx;
+ return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */
+}
+
+static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1,
+ ZSTD_compressionParameters cParams2)
+{
+ (void)cParams1;
+ (void)cParams2;
+ assert(cParams1.windowLog == cParams2.windowLog);
+ assert(cParams1.chainLog == cParams2.chainLog);
+ assert(cParams1.hashLog == cParams2.hashLog);
+ assert(cParams1.searchLog == cParams2.searchLog);
+ assert(cParams1.minMatch == cParams2.minMatch);
+ assert(cParams1.targetLength == cParams2.targetLength);
+ assert(cParams1.strategy == cParams2.strategy);
+}
+
+void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs)
+{
+ int i;
+ for (i = 0; i < ZSTD_REP_NUM; ++i)
+ bs->rep[i] = repStartValue[i];
+ bs->entropy.huf.repeatMode = HUF_repeat_none;
+ bs->entropy.fse.offcode_repeatMode = FSE_repeat_none;
+ bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none;
+ bs->entropy.fse.litlength_repeatMode = FSE_repeat_none;
+}
+
+/*! ZSTD_invalidateMatchState()
+ * Invalidate all the matches in the match finder tables.
+ * Requires nextSrc and base to be set (can be NULL).
+ */
+static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms)
+{
+ ZSTD_window_clear(&ms->window);
+
+ ms->nextToUpdate = ms->window.dictLimit;
+ ms->loadedDictEnd = 0;
+ ms->opt.litLengthSum = 0; /* force reset of btopt stats */
+ ms->dictMatchState = NULL;
+}
+
+/**
+ * Controls, for this matchState reset, whether the tables need to be cleared /
+ * prepared for the coming compression (ZSTDcrp_makeClean), or whether the
+ * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a
+ * subsequent operation will overwrite the table space anyways (e.g., copying
+ * the matchState contents in from a CDict).
+ */
+typedef enum {
+ ZSTDcrp_makeClean,
+ ZSTDcrp_leaveDirty
+} ZSTD_compResetPolicy_e;
+
+/**
+ * Controls, for this matchState reset, whether indexing can continue where it
+ * left off (ZSTDirp_continue), or whether it needs to be restarted from zero
+ * (ZSTDirp_reset).
+ */
+typedef enum {
+ ZSTDirp_continue,
+ ZSTDirp_reset
+} ZSTD_indexResetPolicy_e;
+
+typedef enum {
+ ZSTD_resetTarget_CDict,
+ ZSTD_resetTarget_CCtx
+} ZSTD_resetTarget_e;
+
+
+static size_t
+ZSTD_reset_matchState(ZSTD_matchState_t* ms,
+ ZSTD_cwksp* ws,
+ const ZSTD_compressionParameters* cParams,
+ const ZSTD_paramSwitch_e useRowMatchFinder,
+ const ZSTD_compResetPolicy_e crp,
+ const ZSTD_indexResetPolicy_e forceResetIndex,
+ const ZSTD_resetTarget_e forWho)
+{
+ /* disable chain table allocation for fast or row-based strategies */
+ size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder,
+ ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict))
+ ? ((size_t)1 << cParams->chainLog)
+ : 0;
+ size_t const hSize = ((size_t)1) << cParams->hashLog;
+ U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
+ size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
+
+ DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset);
+ assert(useRowMatchFinder != ZSTD_ps_auto);
+ if (forceResetIndex == ZSTDirp_reset) {
+ ZSTD_window_init(&ms->window);
+ ZSTD_cwksp_mark_tables_dirty(ws);
+ }
+
+ ms->hashLog3 = hashLog3;
+
+ ZSTD_invalidateMatchState(ms);
+
+ assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */
+
+ ZSTD_cwksp_clear_tables(ws);
+
+ DEBUGLOG(5, "reserving table space");
+ /* table Space */
+ ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32));
+ ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32));
+ ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32));
+ RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
+ "failed a workspace allocation in ZSTD_reset_matchState");
+
+ DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty);
+ if (crp!=ZSTDcrp_leaveDirty) {
+ /* reset tables only */
+ ZSTD_cwksp_clean_tables(ws);
+ }
+
+ /* opt parser space */
+ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) {
+ DEBUGLOG(4, "reserving optimal parser space");
+ ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<<Litbits) * sizeof(unsigned));
+ ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned));
+ ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned));
+ ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned));
+ ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t));
+ ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
+ }
+
+ if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
+ { /* Row match finder needs an additional table of hashes ("tags") */
+ size_t const tagTableSize = hSize*sizeof(U16);
+ ms->tagTable = (U16*)ZSTD_cwksp_reserve_aligned(ws, tagTableSize);
+ if (ms->tagTable) ZSTD_memset(ms->tagTable, 0, tagTableSize);
+ }
+ { /* Switch to 32-entry rows if searchLog is 5 (or more) */
+ U32 const rowLog = BOUNDED(4, cParams->searchLog, 6);
+ assert(cParams->hashLog >= rowLog);
+ ms->rowHashLog = cParams->hashLog - rowLog;
+ }
+ }
+
+ ms->cParams = *cParams;
+
+ RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
+ "failed a workspace allocation in ZSTD_reset_matchState");
+ return 0;
+}
+
+/* ZSTD_indexTooCloseToMax() :
+ * minor optimization : prefer memset() rather than reduceIndex()
+ * which is measurably slow in some circumstances (reported for Visual Studio).
+ * Works when re-using a context for a lot of smallish inputs :
+ * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN,
+ * memset() will be triggered before reduceIndex().
+ */
+#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB)
+static int ZSTD_indexTooCloseToMax(ZSTD_window_t w)
+{
+ return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN);
+}
+
+/** ZSTD_dictTooBig():
+ * When dictionaries are larger than ZSTD_CHUNKSIZE_MAX they can't be loaded in
+ * one go generically. So we ensure that in that case we reset the tables to zero,
+ * so that we can load as much of the dictionary as possible.
+ */
+static int ZSTD_dictTooBig(size_t const loadedDictSize)
+{
+ return loadedDictSize > ZSTD_CHUNKSIZE_MAX;
+}
+
+/*! ZSTD_resetCCtx_internal() :
+ * @param loadedDictSize The size of the dictionary to be loaded
+ * into the context, if any. If no dictionary is used, or the
+ * dictionary is being attached / copied, then pass 0.
+ * note : `params` are assumed fully validated at this stage.
+ */
+static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
+ ZSTD_CCtx_params const* params,
+ U64 const pledgedSrcSize,
+ size_t const loadedDictSize,
+ ZSTD_compResetPolicy_e const crp,
+ ZSTD_buffered_policy_e const zbuff)
+{
+ ZSTD_cwksp* const ws = &zc->workspace;
+ DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d",
+ (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->useBlockSplitter);
+ assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
+
+ zc->isFirstBlock = 1;
+
+ /* Set applied params early so we can modify them for LDM,
+ * and point params at the applied params.
+ */
+ zc->appliedParams = *params;
+ params = &zc->appliedParams;
+
+ assert(params->useRowMatchFinder != ZSTD_ps_auto);
+ assert(params->useBlockSplitter != ZSTD_ps_auto);
+ assert(params->ldmParams.enableLdm != ZSTD_ps_auto);
+ assert(params->maxBlockSize != 0);
+ if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
+ /* Adjust long distance matching parameters */
+ ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, &params->cParams);
+ assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog);
+ assert(params->ldmParams.hashRateLog < 32);
+ }
+
+ { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
+ size_t const blockSize = MIN(params->maxBlockSize, windowSize);
+ size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useSequenceProducer);
+ size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
+ ? ZSTD_compressBound(blockSize) + 1
+ : 0;
+ size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered)
+ ? windowSize + blockSize
+ : 0;
+ size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize);
+
+ int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window);
+ int const dictTooBig = ZSTD_dictTooBig(loadedDictSize);
+ ZSTD_indexResetPolicy_e needsIndexReset =
+ (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue;
+
+ size_t const neededSpace =
+ ZSTD_estimateCCtxSize_usingCCtxParams_internal(
+ &params->cParams, &params->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
+ buffInSize, buffOutSize, pledgedSrcSize, params->useSequenceProducer, params->maxBlockSize);
+ int resizeWorkspace;
+
+ FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
+
+ if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0);
+
+ { /* Check if workspace is large enough, alloc a new one if needed */
+ int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace;
+ int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace);
+ resizeWorkspace = workspaceTooSmall || workspaceWasteful;
+ DEBUGLOG(4, "Need %zu B workspace", neededSpace);
+ DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize);
+
+ if (resizeWorkspace) {
+ DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB",
+ ZSTD_cwksp_sizeof(ws) >> 10,
+ neededSpace >> 10);
+
+ RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize");
+
+ needsIndexReset = ZSTDirp_reset;
+
+ ZSTD_cwksp_free(ws, zc->customMem);
+ FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), "");
+
+ DEBUGLOG(5, "reserving object space");
+ /* Statically sized space.
+ * entropyWorkspace never moves,
+ * though prev/next block swap places */
+ assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t)));
+ zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t));
+ RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock");
+ zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t));
+ RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock");
+ zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE);
+ RETURN_ERROR_IF(zc->entropyWorkspace == NULL, memory_allocation, "couldn't allocate entropyWorkspace");
+ } }
+
+ ZSTD_cwksp_clear(ws);
+
+ /* init params */
+ zc->blockState.matchState.cParams = params->cParams;
+ zc->blockState.matchState.prefetchCDictTables = params->prefetchCDictTables == ZSTD_ps_enable;
+ zc->pledgedSrcSizePlusOne = pledgedSrcSize+1;
+ zc->consumedSrcSize = 0;
+ zc->producedCSize = 0;
+ if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)
+ zc->appliedParams.fParams.contentSizeFlag = 0;
+ DEBUGLOG(4, "pledged content size : %u ; flag : %u",
+ (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag);
+ zc->blockSize = blockSize;
+
+ XXH64_reset(&zc->xxhState, 0);
+ zc->stage = ZSTDcs_init;
+ zc->dictID = 0;
+ zc->dictContentSize = 0;
+
+ ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock);
+
+ /* ZSTD_wildcopy() is used to copy into the literals buffer,
+ * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes.
+ */
+ zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH);
+ zc->seqStore.maxNbLit = blockSize;
+
+ /* buffers */
+ zc->bufferedPolicy = zbuff;
+ zc->inBuffSize = buffInSize;
+ zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize);
+ zc->outBuffSize = buffOutSize;
+ zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize);
+
+ /* ldm bucketOffsets table */
+ if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
+ /* TODO: avoid memset? */
+ size_t const numBuckets =
+ ((size_t)1) << (params->ldmParams.hashLog -
+ params->ldmParams.bucketSizeLog);
+ zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets);
+ ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets);
+ }
+
+ /* sequences storage */
+ ZSTD_referenceExternalSequences(zc, NULL, 0);
+ zc->seqStore.maxNbSeq = maxNbSeq;
+ zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
+ zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
+ zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
+ zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef));
+
+ FORWARD_IF_ERROR(ZSTD_reset_matchState(
+ &zc->blockState.matchState,
+ ws,
+ &params->cParams,
+ params->useRowMatchFinder,
+ crp,
+ needsIndexReset,
+ ZSTD_resetTarget_CCtx), "");
+
+ /* ldm hash table */
+ if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
+ /* TODO: avoid memset? */
+ size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog;
+ zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t));
+ ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t));
+ zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
+ zc->maxNbLdmSequences = maxNbLdmSeq;
+
+ ZSTD_window_init(&zc->ldmState.window);
+ zc->ldmState.loadedDictEnd = 0;
+ }
+
+ /* reserve space for block-level external sequences */
+ if (params->useSequenceProducer) {
+ size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+ zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq;
+ zc->externalMatchCtx.seqBuffer =
+ (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
+ }
+
+ DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
+ assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace));
+
+ zc->initialized = 1;
+
+ return 0;
+ }
+}
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ * do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
+ int i;
+ for (i=0; i<ZSTD_REP_NUM; i++) cctx->blockState.prevCBlock->rep[i] = 0;
+ assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window));
+}
+
+/* These are the approximate sizes for each strategy past which copying the
+ * dictionary tables into the working context is faster than using them
+ * in-place.
+ */
+static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = {
+ 8 KB, /* unused */
+ 8 KB, /* ZSTD_fast */
+ 16 KB, /* ZSTD_dfast */
+ 32 KB, /* ZSTD_greedy */
+ 32 KB, /* ZSTD_lazy */
+ 32 KB, /* ZSTD_lazy2 */
+ 32 KB, /* ZSTD_btlazy2 */
+ 32 KB, /* ZSTD_btopt */
+ 8 KB, /* ZSTD_btultra */
+ 8 KB /* ZSTD_btultra2 */
+};
+
+static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params,
+ U64 pledgedSrcSize)
+{
+ size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy];
+ int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch;
+ return dedicatedDictSearch
+ || ( ( pledgedSrcSize <= cutoff
+ || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
+ || params->attachDictPref == ZSTD_dictForceAttach )
+ && params->attachDictPref != ZSTD_dictForceCopy
+ && !params->forceWindow ); /* dictMatchState isn't correctly
+ * handled in _enforceMaxDist */
+}
+
+static size_t
+ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
+ const ZSTD_CDict* cdict,
+ ZSTD_CCtx_params params,
+ U64 pledgedSrcSize,
+ ZSTD_buffered_policy_e zbuff)
+{
+ DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu",
+ (unsigned long long)pledgedSrcSize);
+ {
+ ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams;
+ unsigned const windowLog = params.cParams.windowLog;
+ assert(windowLog != 0);
+ /* Resize working context table params for input only, since the dict
+ * has its own tables. */
+ /* pledgedSrcSize == 0 means 0! */
+
+ if (cdict->matchState.dedicatedDictSearch) {
+ ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams);
+ }
+
+ params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize,
+ cdict->dictContentSize, ZSTD_cpm_attachDict,
+ params.useRowMatchFinder);
+ params.cParams.windowLog = windowLog;
+ params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */
+ FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, &params, pledgedSrcSize,
+ /* loadedDictSize */ 0,
+ ZSTDcrp_makeClean, zbuff), "");
+ assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy);
+ }
+
+ { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc
+ - cdict->matchState.window.base);
+ const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit;
+ if (cdictLen == 0) {
+ /* don't even attach dictionaries with no contents */
+ DEBUGLOG(4, "skipping attaching empty dictionary");
+ } else {
+ DEBUGLOG(4, "attaching dictionary into context");
+ cctx->blockState.matchState.dictMatchState = &cdict->matchState;
+
+ /* prep working match state so dict matches never have negative indices
+ * when they are translated to the working context's index space. */
+ if (cctx->blockState.matchState.window.dictLimit < cdictEnd) {
+ cctx->blockState.matchState.window.nextSrc =
+ cctx->blockState.matchState.window.base + cdictEnd;
+ ZSTD_window_clear(&cctx->blockState.matchState.window);
+ }
+ /* loadedDictEnd is expressed within the referential of the active context */
+ cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit;
+ } }
+
+ cctx->dictID = cdict->dictID;
+ cctx->dictContentSize = cdict->dictContentSize;
+
+ /* copy block state */
+ ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState));
+
+ return 0;
+}
+
+static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize,
+ ZSTD_compressionParameters const* cParams) {
+ if (ZSTD_CDictIndicesAreTagged(cParams)){
+ /* Remove tags from the CDict table if they are present.
+ * See docs on "short cache" in zstd_compress_internal.h for context. */
+ size_t i;
+ for (i = 0; i < tableSize; i++) {
+ U32 const taggedIndex = src[i];
+ U32 const index = taggedIndex >> ZSTD_SHORT_CACHE_TAG_BITS;
+ dst[i] = index;
+ }
+ } else {
+ ZSTD_memcpy(dst, src, tableSize * sizeof(U32));
+ }
+}
+
+static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
+ const ZSTD_CDict* cdict,
+ ZSTD_CCtx_params params,
+ U64 pledgedSrcSize,
+ ZSTD_buffered_policy_e zbuff)
+{
+ const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
+
+ assert(!cdict->matchState.dedicatedDictSearch);
+ DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu",
+ (unsigned long long)pledgedSrcSize);
+
+ { unsigned const windowLog = params.cParams.windowLog;
+ assert(windowLog != 0);
+ /* Copy only compression parameters related to tables. */
+ params.cParams = *cdict_cParams;
+ params.cParams.windowLog = windowLog;
+ params.useRowMatchFinder = cdict->useRowMatchFinder;
+ FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, &params, pledgedSrcSize,
+ /* loadedDictSize */ 0,
+ ZSTDcrp_leaveDirty, zbuff), "");
+ assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
+ assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog);
+ assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog);
+ }
+
+ ZSTD_cwksp_mark_tables_dirty(&cctx->workspace);
+ assert(params.useRowMatchFinder != ZSTD_ps_auto);
+
+ /* copy tables */
+ { size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */)
+ ? ((size_t)1 << cdict_cParams->chainLog)
+ : 0;
+ size_t const hSize = (size_t)1 << cdict_cParams->hashLog;
+
+ ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable,
+ cdict->matchState.hashTable,
+ hSize, cdict_cParams);
+
+ /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */
+ if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) {
+ ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable,
+ cdict->matchState.chainTable,
+ chainSize, cdict_cParams);
+ }
+ /* copy tag table */
+ if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) {
+ size_t const tagTableSize = hSize*sizeof(U16);
+ ZSTD_memcpy(cctx->blockState.matchState.tagTable,
+ cdict->matchState.tagTable,
+ tagTableSize);
+ }
+ }
+
+ /* Zero the hashTable3, since the cdict never fills it */
+ { int const h3log = cctx->blockState.matchState.hashLog3;
+ size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
+ assert(cdict->matchState.hashLog3 == 0);
+ ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32));
+ }
+
+ ZSTD_cwksp_mark_tables_clean(&cctx->workspace);
+
+ /* copy dictionary offsets */
+ { ZSTD_matchState_t const* srcMatchState = &cdict->matchState;
+ ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState;
+ dstMatchState->window = srcMatchState->window;
+ dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
+ dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
+ }
+
+ cctx->dictID = cdict->dictID;
+ cctx->dictContentSize = cdict->dictContentSize;
+
+ /* copy block state */
+ ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState));
+
+ return 0;
+}
+
+/* We have a choice between copying the dictionary context into the working
+ * context, or referencing the dictionary context from the working context
+ * in-place. We decide here which strategy to use. */
+static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
+ const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params,
+ U64 pledgedSrcSize,
+ ZSTD_buffered_policy_e zbuff)
+{
+
+ DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)",
+ (unsigned)pledgedSrcSize);
+
+ if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) {
+ return ZSTD_resetCCtx_byAttachingCDict(
+ cctx, cdict, *params, pledgedSrcSize, zbuff);
+ } else {
+ return ZSTD_resetCCtx_byCopyingCDict(
+ cctx, cdict, *params, pledgedSrcSize, zbuff);
+ }
+}
+
+/*! ZSTD_copyCCtx_internal() :
+ * Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
+ * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
+ * The "context", in this case, refers to the hash and chain tables,
+ * entropy tables, and dictionary references.
+ * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx.
+ * @return : 0, or an error code */
+static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
+ const ZSTD_CCtx* srcCCtx,
+ ZSTD_frameParameters fParams,
+ U64 pledgedSrcSize,
+ ZSTD_buffered_policy_e zbuff)
+{
+ RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong,
+ "Can't copy a ctx that's not in init stage.");
+ DEBUGLOG(5, "ZSTD_copyCCtx_internal");
+ ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
+ { ZSTD_CCtx_params params = dstCCtx->requestedParams;
+ /* Copy only compression parameters related to tables. */
+ params.cParams = srcCCtx->appliedParams.cParams;
+ assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto);
+ assert(srcCCtx->appliedParams.useBlockSplitter != ZSTD_ps_auto);
+ assert(srcCCtx->appliedParams.ldmParams.enableLdm != ZSTD_ps_auto);
+ params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder;
+ params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter;
+ params.ldmParams = srcCCtx->appliedParams.ldmParams;
+ params.fParams = fParams;
+ params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize;
+ ZSTD_resetCCtx_internal(dstCCtx, &params, pledgedSrcSize,
+ /* loadedDictSize */ 0,
+ ZSTDcrp_leaveDirty, zbuff);
+ assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog);
+ assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy);
+ assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog);
+ assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog);
+ assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3);
+ }
+
+ ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace);
+
+ /* copy tables */
+ { size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy,
+ srcCCtx->appliedParams.useRowMatchFinder,
+ 0 /* forDDSDict */)
+ ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog)
+ : 0;
+ size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
+ int const h3log = srcCCtx->blockState.matchState.hashLog3;
+ size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
+
+ ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable,
+ srcCCtx->blockState.matchState.hashTable,
+ hSize * sizeof(U32));
+ ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable,
+ srcCCtx->blockState.matchState.chainTable,
+ chainSize * sizeof(U32));
+ ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3,
+ srcCCtx->blockState.matchState.hashTable3,
+ h3Size * sizeof(U32));
+ }
+
+ ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace);
+
+ /* copy dictionary offsets */
+ {
+ const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState;
+ ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState;
+ dstMatchState->window = srcMatchState->window;
+ dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
+ dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
+ }
+ dstCCtx->dictID = srcCCtx->dictID;
+ dstCCtx->dictContentSize = srcCCtx->dictContentSize;
+
+ /* copy block state */
+ ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock));
+
+ return 0;
+}
+
+/*! ZSTD_copyCCtx() :
+ * Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
+ * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
+ * pledgedSrcSize==0 means "unknown".
+* @return : 0, or an error code */
+size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize)
+{
+ ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
+ ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy;
+ ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1);
+ if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
+ fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN);
+
+ return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx,
+ fParams, pledgedSrcSize,
+ zbuff);
+}
+
+
+#define ZSTD_ROWSIZE 16
+/*! ZSTD_reduceTable() :
+ * reduce table indexes by `reducerValue`, or squash to zero.
+ * PreserveMark preserves "unsorted mark" for btlazy2 strategy.
+ * It must be set to a clear 0/1 value, to remove branch during inlining.
+ * Presume table size is a multiple of ZSTD_ROWSIZE
+ * to help auto-vectorization */
+FORCE_INLINE_TEMPLATE void
+ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark)
+{
+ int const nbRows = (int)size / ZSTD_ROWSIZE;
+ int cellNb = 0;
+ int rowNb;
+ /* Protect special index values < ZSTD_WINDOW_START_INDEX. */
+ U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX;
+ assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */
+ assert(size < (1U<<31)); /* can be casted to int */
+
+#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
+ /* To validate that the table re-use logic is sound, and that we don't
+ * access table space that we haven't cleaned, we re-"poison" the table
+ * space every time we mark it dirty.
+ *
+ * This function however is intended to operate on those dirty tables and
+ * re-clean them. So when this function is used correctly, we can unpoison
+ * the memory it operated on. This introduces a blind spot though, since
+ * if we now try to operate on __actually__ poisoned memory, we will not
+ * detect that. */
+ __msan_unpoison(table, size * sizeof(U32));
+#endif
+
+ for (rowNb=0 ; rowNb < nbRows ; rowNb++) {
+ int column;
+ for (column=0; column<ZSTD_ROWSIZE; column++) {
+ U32 newVal;
+ if (preserveMark && table[cellNb] == ZSTD_DUBT_UNSORTED_MARK) {
+ /* This write is pointless, but is required(?) for the compiler
+ * to auto-vectorize the loop. */
+ newVal = ZSTD_DUBT_UNSORTED_MARK;
+ } else if (table[cellNb] < reducerThreshold) {
+ newVal = 0;
+ } else {
+ newVal = table[cellNb] - reducerValue;
+ }
+ table[cellNb] = newVal;
+ cellNb++;
+ } }
+}
+
+static void ZSTD_reduceTable(U32* const table, U32 const size, U32 const reducerValue)
+{
+ ZSTD_reduceTable_internal(table, size, reducerValue, 0);
+}
+
+static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const reducerValue)
+{
+ ZSTD_reduceTable_internal(table, size, reducerValue, 1);
+}
+
+/*! ZSTD_reduceIndex() :
+* rescale all indexes to avoid future overflow (indexes are U32) */
+static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue)
+{
+ { U32 const hSize = (U32)1 << params->cParams.hashLog;
+ ZSTD_reduceTable(ms->hashTable, hSize, reducerValue);
+ }
+
+ if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) {
+ U32 const chainSize = (U32)1 << params->cParams.chainLog;
+ if (params->cParams.strategy == ZSTD_btlazy2)
+ ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue);
+ else
+ ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue);
+ }
+
+ if (ms->hashLog3) {
+ U32 const h3Size = (U32)1 << ms->hashLog3;
+ ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue);
+ }
+}
+
+
+/*-*******************************************************
+* Block entropic compression
+*********************************************************/
+
+/* See doc/zstd_compression_format.md for detailed format description */
+
+int ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
+{
+ const seqDef* const sequences = seqStorePtr->sequencesStart;
+ BYTE* const llCodeTable = seqStorePtr->llCode;
+ BYTE* const ofCodeTable = seqStorePtr->ofCode;
+ BYTE* const mlCodeTable = seqStorePtr->mlCode;
+ U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ U32 u;
+ int longOffsets = 0;
+ assert(nbSeq <= seqStorePtr->maxNbSeq);
+ for (u=0; u<nbSeq; u++) {
+ U32 const llv = sequences[u].litLength;
+ U32 const ofCode = ZSTD_highbit32(sequences[u].offBase);
+ U32 const mlv = sequences[u].mlBase;
+ llCodeTable[u] = (BYTE)ZSTD_LLcode(llv);
+ ofCodeTable[u] = (BYTE)ofCode;
+ mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
+ assert(!(MEM_64bits() && ofCode >= STREAM_ACCUMULATOR_MIN));
+ if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN)
+ longOffsets = 1;
+ }
+ if (seqStorePtr->longLengthType==ZSTD_llt_literalLength)
+ llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
+ if (seqStorePtr->longLengthType==ZSTD_llt_matchLength)
+ mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
+ return longOffsets;
+}
+
+/* ZSTD_useTargetCBlockSize():
+ * Returns if target compressed block size param is being used.
+ * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize.
+ * Returns 1 if true, 0 otherwise. */
+static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams)
+{
+ DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize);
+ return (cctxParams->targetCBlockSize != 0);
+}
+
+/* ZSTD_blockSplitterEnabled():
+ * Returns if block splitting param is being used
+ * If used, compression will do best effort to split a block in order to improve compression ratio.
+ * At the time this function is called, the parameter must be finalized.
+ * Returns 1 if true, 0 otherwise. */
+static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams)
+{
+ DEBUGLOG(5, "ZSTD_blockSplitterEnabled (useBlockSplitter=%d)", cctxParams->useBlockSplitter);
+ assert(cctxParams->useBlockSplitter != ZSTD_ps_auto);
+ return (cctxParams->useBlockSplitter == ZSTD_ps_enable);
+}
+
+/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types
+ * and size of the sequences statistics
+ */
+typedef struct {
+ U32 LLtype;
+ U32 Offtype;
+ U32 MLtype;
+ size_t size;
+ size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
+ int longOffsets;
+} ZSTD_symbolEncodingTypeStats_t;
+
+/* ZSTD_buildSequencesStatistics():
+ * Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field.
+ * Modifies `nextEntropy` to have the appropriate values as a side effect.
+ * nbSeq must be greater than 0.
+ *
+ * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32)
+ */
+static ZSTD_symbolEncodingTypeStats_t
+ZSTD_buildSequencesStatistics(
+ const seqStore_t* seqStorePtr, size_t nbSeq,
+ const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
+ BYTE* dst, const BYTE* const dstEnd,
+ ZSTD_strategy strategy, unsigned* countWorkspace,
+ void* entropyWorkspace, size_t entropyWkspSize)
+{
+ BYTE* const ostart = dst;
+ const BYTE* const oend = dstEnd;
+ BYTE* op = ostart;
+ FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable;
+ FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable;
+ FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable;
+ const BYTE* const ofCodeTable = seqStorePtr->ofCode;
+ const BYTE* const llCodeTable = seqStorePtr->llCode;
+ const BYTE* const mlCodeTable = seqStorePtr->mlCode;
+ ZSTD_symbolEncodingTypeStats_t stats;
+
+ stats.lastCountSize = 0;
+ /* convert length/distances into codes */
+ stats.longOffsets = ZSTD_seqToCodes(seqStorePtr);
+ assert(op <= oend);
+ assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */
+ /* build CTable for Literal Lengths */
+ { unsigned max = MaxLL;
+ size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
+ DEBUGLOG(5, "Building LL table");
+ nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
+ stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode,
+ countWorkspace, max, mostFrequent, nbSeq,
+ LLFSELog, prevEntropy->litlengthCTable,
+ LL_defaultNorm, LL_defaultNormLog,
+ ZSTD_defaultAllowed, strategy);
+ assert(set_basic < set_compressed && set_rle < set_compressed);
+ assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
+ { size_t const countSize = ZSTD_buildCTable(
+ op, (size_t)(oend - op),
+ CTable_LitLength, LLFSELog, (symbolEncodingType_e)stats.LLtype,
+ countWorkspace, max, llCodeTable, nbSeq,
+ LL_defaultNorm, LL_defaultNormLog, MaxLL,
+ prevEntropy->litlengthCTable,
+ sizeof(prevEntropy->litlengthCTable),
+ entropyWorkspace, entropyWkspSize);
+ if (ZSTD_isError(countSize)) {
+ DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed");
+ stats.size = countSize;
+ return stats;
+ }
+ if (stats.LLtype == set_compressed)
+ stats.lastCountSize = countSize;
+ op += countSize;
+ assert(op <= oend);
+ } }
+ /* build CTable for Offsets */
+ { unsigned max = MaxOff;
+ size_t const mostFrequent = HIST_countFast_wksp(
+ countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
+ /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
+ ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
+ DEBUGLOG(5, "Building OF table");
+ nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
+ stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode,
+ countWorkspace, max, mostFrequent, nbSeq,
+ OffFSELog, prevEntropy->offcodeCTable,
+ OF_defaultNorm, OF_defaultNormLog,
+ defaultPolicy, strategy);
+ assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
+ { size_t const countSize = ZSTD_buildCTable(
+ op, (size_t)(oend - op),
+ CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)stats.Offtype,
+ countWorkspace, max, ofCodeTable, nbSeq,
+ OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
+ prevEntropy->offcodeCTable,
+ sizeof(prevEntropy->offcodeCTable),
+ entropyWorkspace, entropyWkspSize);
+ if (ZSTD_isError(countSize)) {
+ DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed");
+ stats.size = countSize;
+ return stats;
+ }
+ if (stats.Offtype == set_compressed)
+ stats.lastCountSize = countSize;
+ op += countSize;
+ assert(op <= oend);
+ } }
+ /* build CTable for MatchLengths */
+ { unsigned max = MaxML;
+ size_t const mostFrequent = HIST_countFast_wksp(
+ countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
+ DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
+ nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
+ stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode,
+ countWorkspace, max, mostFrequent, nbSeq,
+ MLFSELog, prevEntropy->matchlengthCTable,
+ ML_defaultNorm, ML_defaultNormLog,
+ ZSTD_defaultAllowed, strategy);
+ assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
+ { size_t const countSize = ZSTD_buildCTable(
+ op, (size_t)(oend - op),
+ CTable_MatchLength, MLFSELog, (symbolEncodingType_e)stats.MLtype,
+ countWorkspace, max, mlCodeTable, nbSeq,
+ ML_defaultNorm, ML_defaultNormLog, MaxML,
+ prevEntropy->matchlengthCTable,
+ sizeof(prevEntropy->matchlengthCTable),
+ entropyWorkspace, entropyWkspSize);
+ if (ZSTD_isError(countSize)) {
+ DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed");
+ stats.size = countSize;
+ return stats;
+ }
+ if (stats.MLtype == set_compressed)
+ stats.lastCountSize = countSize;
+ op += countSize;
+ assert(op <= oend);
+ } }
+ stats.size = (size_t)(op-ostart);
+ return stats;
+}
+
+/* ZSTD_entropyCompressSeqStore_internal():
+ * compresses both literals and sequences
+ * Returns compressed size of block, or a zstd error.
+ */
+#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20
+MEM_STATIC size_t
+ZSTD_entropyCompressSeqStore_internal(
+ const seqStore_t* seqStorePtr,
+ const ZSTD_entropyCTables_t* prevEntropy,
+ ZSTD_entropyCTables_t* nextEntropy,
+ const ZSTD_CCtx_params* cctxParams,
+ void* dst, size_t dstCapacity,
+ void* entropyWorkspace, size_t entropyWkspSize,
+ const int bmi2)
+{
+ ZSTD_strategy const strategy = cctxParams->cParams.strategy;
+ unsigned* count = (unsigned*)entropyWorkspace;
+ FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
+ FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
+ FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
+ const seqDef* const sequences = seqStorePtr->sequencesStart;
+ const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ const BYTE* const ofCodeTable = seqStorePtr->ofCode;
+ const BYTE* const llCodeTable = seqStorePtr->llCode;
+ const BYTE* const mlCodeTable = seqStorePtr->mlCode;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstCapacity;
+ BYTE* op = ostart;
+ size_t lastCountSize;
+ int longOffsets = 0;
+
+ entropyWorkspace = count + (MaxSeq + 1);
+ entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
+
+ DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity);
+ ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
+ assert(entropyWkspSize >= HUF_WORKSPACE_SIZE);
+
+ /* Compress literals */
+ { const BYTE* const literals = seqStorePtr->litStart;
+ size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
+ /* Base suspicion of uncompressibility on ratio of literals to sequences */
+ unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
+ size_t const litSize = (size_t)(seqStorePtr->lit - literals);
+
+ size_t const cSize = ZSTD_compressLiterals(
+ op, dstCapacity,
+ literals, litSize,
+ entropyWorkspace, entropyWkspSize,
+ &prevEntropy->huf, &nextEntropy->huf,
+ cctxParams->cParams.strategy,
+ ZSTD_literalsCompressionIsDisabled(cctxParams),
+ suspectUncompressible, bmi2);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed");
+ assert(cSize <= dstCapacity);
+ op += cSize;
+ }
+
+ /* Sequences Header */
+ RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
+ dstSize_tooSmall, "Can't fit seq hdr in output buf!");
+ if (nbSeq < 128) {
+ *op++ = (BYTE)nbSeq;
+ } else if (nbSeq < LONGNBSEQ) {
+ op[0] = (BYTE)((nbSeq>>8) + 0x80);
+ op[1] = (BYTE)nbSeq;
+ op+=2;
+ } else {
+ op[0]=0xFF;
+ MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ));
+ op+=3;
+ }
+ assert(op <= oend);
+ if (nbSeq==0) {
+ /* Copy the old tables over as if we repeated them */
+ ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse));
+ return (size_t)(op - ostart);
+ }
+ { BYTE* const seqHead = op++;
+ /* build stats for sequences */
+ const ZSTD_symbolEncodingTypeStats_t stats =
+ ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
+ &prevEntropy->fse, &nextEntropy->fse,
+ op, oend,
+ strategy, count,
+ entropyWorkspace, entropyWkspSize);
+ FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!");
+ *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2));
+ lastCountSize = stats.lastCountSize;
+ op += stats.size;
+ longOffsets = stats.longOffsets;
+ }
+
+ { size_t const bitstreamSize = ZSTD_encodeSequences(
+ op, (size_t)(oend - op),
+ CTable_MatchLength, mlCodeTable,
+ CTable_OffsetBits, ofCodeTable,
+ CTable_LitLength, llCodeTable,
+ sequences, nbSeq,
+ longOffsets, bmi2);
+ FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed");
+ op += bitstreamSize;
+ assert(op <= oend);
+ /* zstd versions <= 1.3.4 mistakenly report corruption when
+ * FSE_readNCount() receives a buffer < 4 bytes.
+ * Fixed by https://github.com/facebook/zstd/pull/1146.
+ * This can happen when the last set_compressed table present is 2
+ * bytes and the bitstream is only one byte.
+ * In this exceedingly rare case, we will simply emit an uncompressed
+ * block, since it isn't worth optimizing.
+ */
+ if (lastCountSize && (lastCountSize + bitstreamSize) < 4) {
+ /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */
+ assert(lastCountSize + bitstreamSize == 3);
+ DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by "
+ "emitting an uncompressed block.");
+ return 0;
+ }
+ }
+
+ DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart));
+ return (size_t)(op - ostart);
+}
+
+MEM_STATIC size_t
+ZSTD_entropyCompressSeqStore(
+ const seqStore_t* seqStorePtr,
+ const ZSTD_entropyCTables_t* prevEntropy,
+ ZSTD_entropyCTables_t* nextEntropy,
+ const ZSTD_CCtx_params* cctxParams,
+ void* dst, size_t dstCapacity,
+ size_t srcSize,
+ void* entropyWorkspace, size_t entropyWkspSize,
+ int bmi2)
+{
+ size_t const cSize = ZSTD_entropyCompressSeqStore_internal(
+ seqStorePtr, prevEntropy, nextEntropy, cctxParams,
+ dst, dstCapacity,
+ entropyWorkspace, entropyWkspSize, bmi2);
+ if (cSize == 0) return 0;
+ /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
+ * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
+ */
+ if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) {
+ DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity);
+ return 0; /* block not compressed */
+ }
+ FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed");
+
+ /* Check compressibility */
+ { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
+ if (cSize >= maxCSize) return 0; /* block not compressed */
+ }
+ DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
+ /* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly.
+ * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above.
+ */
+ assert(cSize < ZSTD_BLOCKSIZE_MAX);
+ return cSize;
+}
+
+/* ZSTD_selectBlockCompressor() :
+ * Not static, but internal use only (used by long distance matcher)
+ * assumption : strat is a valid strategy */
+ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode)
+{
+ static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = {
+ { ZSTD_compressBlock_fast /* default for 0 */,
+ ZSTD_compressBlock_fast,
+ ZSTD_compressBlock_doubleFast,
+ ZSTD_compressBlock_greedy,
+ ZSTD_compressBlock_lazy,
+ ZSTD_compressBlock_lazy2,
+ ZSTD_compressBlock_btlazy2,
+ ZSTD_compressBlock_btopt,
+ ZSTD_compressBlock_btultra,
+ ZSTD_compressBlock_btultra2 },
+ { ZSTD_compressBlock_fast_extDict /* default for 0 */,
+ ZSTD_compressBlock_fast_extDict,
+ ZSTD_compressBlock_doubleFast_extDict,
+ ZSTD_compressBlock_greedy_extDict,
+ ZSTD_compressBlock_lazy_extDict,
+ ZSTD_compressBlock_lazy2_extDict,
+ ZSTD_compressBlock_btlazy2_extDict,
+ ZSTD_compressBlock_btopt_extDict,
+ ZSTD_compressBlock_btultra_extDict,
+ ZSTD_compressBlock_btultra_extDict },
+ { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */,
+ ZSTD_compressBlock_fast_dictMatchState,
+ ZSTD_compressBlock_doubleFast_dictMatchState,
+ ZSTD_compressBlock_greedy_dictMatchState,
+ ZSTD_compressBlock_lazy_dictMatchState,
+ ZSTD_compressBlock_lazy2_dictMatchState,
+ ZSTD_compressBlock_btlazy2_dictMatchState,
+ ZSTD_compressBlock_btopt_dictMatchState,
+ ZSTD_compressBlock_btultra_dictMatchState,
+ ZSTD_compressBlock_btultra_dictMatchState },
+ { NULL /* default for 0 */,
+ NULL,
+ NULL,
+ ZSTD_compressBlock_greedy_dedicatedDictSearch,
+ ZSTD_compressBlock_lazy_dedicatedDictSearch,
+ ZSTD_compressBlock_lazy2_dedicatedDictSearch,
+ NULL,
+ NULL,
+ NULL,
+ NULL }
+ };
+ ZSTD_blockCompressor selectedCompressor;
+ ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1);
+
+ assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat));
+ DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder);
+ if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) {
+ static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = {
+ { ZSTD_compressBlock_greedy_row,
+ ZSTD_compressBlock_lazy_row,
+ ZSTD_compressBlock_lazy2_row },
+ { ZSTD_compressBlock_greedy_extDict_row,
+ ZSTD_compressBlock_lazy_extDict_row,
+ ZSTD_compressBlock_lazy2_extDict_row },
+ { ZSTD_compressBlock_greedy_dictMatchState_row,
+ ZSTD_compressBlock_lazy_dictMatchState_row,
+ ZSTD_compressBlock_lazy2_dictMatchState_row },
+ { ZSTD_compressBlock_greedy_dedicatedDictSearch_row,
+ ZSTD_compressBlock_lazy_dedicatedDictSearch_row,
+ ZSTD_compressBlock_lazy2_dedicatedDictSearch_row }
+ };
+ DEBUGLOG(4, "Selecting a row-based matchfinder");
+ assert(useRowMatchFinder != ZSTD_ps_auto);
+ selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy];
+ } else {
+ selectedCompressor = blockCompressor[(int)dictMode][(int)strat];
+ }
+ assert(selectedCompressor != NULL);
+ return selectedCompressor;
+}
+
+static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr,
+ const BYTE* anchor, size_t lastLLSize)
+{
+ ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize);
+ seqStorePtr->lit += lastLLSize;
+}
+
+void ZSTD_resetSeqStore(seqStore_t* ssPtr)
+{
+ ssPtr->lit = ssPtr->litStart;
+ ssPtr->sequences = ssPtr->sequencesStart;
+ ssPtr->longLengthType = ZSTD_llt_none;
+}
+
+/* ZSTD_postProcessSequenceProducerResult() :
+ * Validates and post-processes sequences obtained through the external matchfinder API:
+ * - Checks whether nbExternalSeqs represents an error condition.
+ * - Appends a block delimiter to outSeqs if one is not already present.
+ * See zstd.h for context regarding block delimiters.
+ * Returns the number of sequences after post-processing, or an error code. */
+static size_t ZSTD_postProcessSequenceProducerResult(
+ ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize
+) {
+ RETURN_ERROR_IF(
+ nbExternalSeqs > outSeqsCapacity,
+ sequenceProducer_failed,
+ "External sequence producer returned error code %lu",
+ (unsigned long)nbExternalSeqs
+ );
+
+ RETURN_ERROR_IF(
+ nbExternalSeqs == 0 && srcSize > 0,
+ sequenceProducer_failed,
+ "Got zero sequences from external sequence producer for a non-empty src buffer!"
+ );
+
+ if (srcSize == 0) {
+ ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence));
+ return 1;
+ }
+
+ {
+ ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1];
+
+ /* We can return early if lastSeq is already a block delimiter. */
+ if (lastSeq.offset == 0 && lastSeq.matchLength == 0) {
+ return nbExternalSeqs;
+ }
+
+ /* This error condition is only possible if the external matchfinder
+ * produced an invalid parse, by definition of ZSTD_sequenceBound(). */
+ RETURN_ERROR_IF(
+ nbExternalSeqs == outSeqsCapacity,
+ sequenceProducer_failed,
+ "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!"
+ );
+
+ /* lastSeq is not a block delimiter, so we need to append one. */
+ ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence));
+ return nbExternalSeqs + 1;
+ }
+}
+
+/* ZSTD_fastSequenceLengthSum() :
+ * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*.
+ * Similar to another function in zstd_compress.c (determine_blockSize),
+ * except it doesn't check for a block delimiter to end summation.
+ * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P).
+ * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */
+static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) {
+ size_t matchLenSum, litLenSum, i;
+ matchLenSum = 0;
+ litLenSum = 0;
+ for (i = 0; i < seqBufSize; i++) {
+ litLenSum += seqBuf[i].litLength;
+ matchLenSum += seqBuf[i].matchLength;
+ }
+ return litLenSum + matchLenSum;
+}
+
+typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;
+
+static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
+{
+ ZSTD_matchState_t* const ms = &zc->blockState.matchState;
+ DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize);
+ assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
+ /* Assert that we have correctly flushed the ctx params into the ms's copy */
+ ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams);
+ /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
+ * additional 1. We need to revisit and change this logic to be more consistent */
+ if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) {
+ if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) {
+ ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize);
+ } else {
+ ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch);
+ }
+ return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */
+ }
+ ZSTD_resetSeqStore(&(zc->seqStore));
+ /* required for optimal parser to read stats from dictionary */
+ ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy;
+ /* tell the optimal parser how we expect to compress literals */
+ ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode;
+ /* a gap between an attached dict and the current window is not safe,
+ * they must remain adjacent,
+ * and when that stops being the case, the dict must be unset */
+ assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit);
+
+ /* limited update after a very long match */
+ { const BYTE* const base = ms->window.base;
+ const BYTE* const istart = (const BYTE*)src;
+ const U32 curr = (U32)(istart-base);
+ if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */
+ if (curr > ms->nextToUpdate + 384)
+ ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384));
+ }
+
+ /* select and store sequences */
+ { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms);
+ size_t lastLLSize;
+ { int i;
+ for (i = 0; i < ZSTD_REP_NUM; ++i)
+ zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i];
+ }
+ if (zc->externSeqStore.pos < zc->externSeqStore.size) {
+ assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable);
+
+ /* External matchfinder + LDM is technically possible, just not implemented yet.
+ * We need to revisit soon and implement it. */
+ RETURN_ERROR_IF(
+ zc->appliedParams.useSequenceProducer,
+ parameter_combination_unsupported,
+ "Long-distance matching with external sequence producer enabled is not currently supported."
+ );
+
+ /* Updates ldmSeqStore.pos */
+ lastLLSize =
+ ZSTD_ldm_blockCompress(&zc->externSeqStore,
+ ms, &zc->seqStore,
+ zc->blockState.nextCBlock->rep,
+ zc->appliedParams.useRowMatchFinder,
+ src, srcSize);
+ assert(zc->externSeqStore.pos <= zc->externSeqStore.size);
+ } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) {
+ rawSeqStore_t ldmSeqStore = kNullRawSeqStore;
+
+ /* External matchfinder + LDM is technically possible, just not implemented yet.
+ * We need to revisit soon and implement it. */
+ RETURN_ERROR_IF(
+ zc->appliedParams.useSequenceProducer,
+ parameter_combination_unsupported,
+ "Long-distance matching with external sequence producer enabled is not currently supported."
+ );
+
+ ldmSeqStore.seq = zc->ldmSequences;
+ ldmSeqStore.capacity = zc->maxNbLdmSequences;
+ /* Updates ldmSeqStore.size */
+ FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore,
+ &zc->appliedParams.ldmParams,
+ src, srcSize), "");
+ /* Updates ldmSeqStore.pos */
+ lastLLSize =
+ ZSTD_ldm_blockCompress(&ldmSeqStore,
+ ms, &zc->seqStore,
+ zc->blockState.nextCBlock->rep,
+ zc->appliedParams.useRowMatchFinder,
+ src, srcSize);
+ assert(ldmSeqStore.pos == ldmSeqStore.size);
+ } else if (zc->appliedParams.useSequenceProducer) {
+ assert(
+ zc->externalMatchCtx.seqBufferCapacity >= ZSTD_sequenceBound(srcSize)
+ );
+ assert(zc->externalMatchCtx.mFinder != NULL);
+
+ { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog;
+
+ size_t const nbExternalSeqs = (zc->externalMatchCtx.mFinder)(
+ zc->externalMatchCtx.mState,
+ zc->externalMatchCtx.seqBuffer,
+ zc->externalMatchCtx.seqBufferCapacity,
+ src, srcSize,
+ NULL, 0, /* dict and dictSize, currently not supported */
+ zc->appliedParams.compressionLevel,
+ windowSize
+ );
+
+ size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult(
+ zc->externalMatchCtx.seqBuffer,
+ nbExternalSeqs,
+ zc->externalMatchCtx.seqBufferCapacity,
+ srcSize
+ );
+
+ /* Return early if there is no error, since we don't need to worry about last literals */
+ if (!ZSTD_isError(nbPostProcessedSeqs)) {
+ ZSTD_sequencePosition seqPos = {0,0,0};
+ size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs);
+ RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!");
+ FORWARD_IF_ERROR(
+ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(
+ zc, &seqPos,
+ zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs,
+ src, srcSize,
+ zc->appliedParams.searchForExternalRepcodes
+ ),
+ "Failed to copy external sequences to seqStore!"
+ );
+ ms->ldmSeqStore = NULL;
+ DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs);
+ return ZSTDbss_compress;
+ }
+
+ /* Propagate the error if fallback is disabled */
+ if (!zc->appliedParams.enableMatchFinderFallback) {
+ return nbPostProcessedSeqs;
+ }
+
+ /* Fallback to software matchfinder */
+ { ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
+ zc->appliedParams.useRowMatchFinder,
+ dictMode);
+ ms->ldmSeqStore = NULL;
+ DEBUGLOG(
+ 5,
+ "External sequence producer returned error code %lu. Falling back to internal parser.",
+ (unsigned long)nbExternalSeqs
+ );
+ lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
+ } }
+ } else { /* not long range mode and no external matchfinder */
+ ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
+ zc->appliedParams.useRowMatchFinder,
+ dictMode);
+ ms->ldmSeqStore = NULL;
+ lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
+ }
+ { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize;
+ ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize);
+ } }
+ return ZSTDbss_compress;
+}
+
+static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
+{
+ const seqStore_t* seqStore = ZSTD_getSeqStore(zc);
+ const seqDef* seqStoreSeqs = seqStore->sequencesStart;
+ size_t seqStoreSeqSize = seqStore->sequences - seqStoreSeqs;
+ size_t seqStoreLiteralsSize = (size_t)(seqStore->lit - seqStore->litStart);
+ size_t literalsRead = 0;
+ size_t lastLLSize;
+
+ ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex];
+ size_t i;
+ repcodes_t updatedRepcodes;
+
+ assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences);
+ /* Ensure we have enough space for last literals "sequence" */
+ assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1);
+ ZSTD_memcpy(updatedRepcodes.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
+ for (i = 0; i < seqStoreSeqSize; ++i) {
+ U32 rawOffset = seqStoreSeqs[i].offBase - ZSTD_REP_NUM;
+ outSeqs[i].litLength = seqStoreSeqs[i].litLength;
+ outSeqs[i].matchLength = seqStoreSeqs[i].mlBase + MINMATCH;
+ outSeqs[i].rep = 0;
+
+ if (i == seqStore->longLengthPos) {
+ if (seqStore->longLengthType == ZSTD_llt_literalLength) {
+ outSeqs[i].litLength += 0x10000;
+ } else if (seqStore->longLengthType == ZSTD_llt_matchLength) {
+ outSeqs[i].matchLength += 0x10000;
+ }
+ }
+
+ if (seqStoreSeqs[i].offBase <= ZSTD_REP_NUM) {
+ /* Derive the correct offset corresponding to a repcode */
+ outSeqs[i].rep = seqStoreSeqs[i].offBase;
+ if (outSeqs[i].litLength != 0) {
+ rawOffset = updatedRepcodes.rep[outSeqs[i].rep - 1];
+ } else {
+ if (outSeqs[i].rep == 3) {
+ rawOffset = updatedRepcodes.rep[0] - 1;
+ } else {
+ rawOffset = updatedRepcodes.rep[outSeqs[i].rep];
+ }
+ }
+ }
+ outSeqs[i].offset = rawOffset;
+ /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode
+ so we provide seqStoreSeqs[i].offset - 1 */
+ ZSTD_updateRep(updatedRepcodes.rep,
+ seqStoreSeqs[i].offBase,
+ seqStoreSeqs[i].litLength == 0);
+ literalsRead += outSeqs[i].litLength;
+ }
+ /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0.
+ * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker
+ * for the block boundary, according to the API.
+ */
+ assert(seqStoreLiteralsSize >= literalsRead);
+ lastLLSize = seqStoreLiteralsSize - literalsRead;
+ outSeqs[i].litLength = (U32)lastLLSize;
+ outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0;
+ seqStoreSeqSize++;
+ zc->seqCollector.seqIndex += seqStoreSeqSize;
+}
+
+size_t ZSTD_sequenceBound(size_t srcSize) {
+ return (srcSize / ZSTD_MINMATCH_MIN) + 1;
+}
+
+size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
+ size_t outSeqsSize, const void* src, size_t srcSize)
+{
+ const size_t dstCapacity = ZSTD_compressBound(srcSize);
+ void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem);
+ SeqCollector seqCollector;
+
+ RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!");
+
+ seqCollector.collectSequences = 1;
+ seqCollector.seqStart = outSeqs;
+ seqCollector.seqIndex = 0;
+ seqCollector.maxSequences = outSeqsSize;
+ zc->seqCollector = seqCollector;
+
+ ZSTD_compress2(zc, dst, dstCapacity, src, srcSize);
+ ZSTD_customFree(dst, ZSTD_defaultCMem);
+ return zc->seqCollector.seqIndex;
+}
+
+size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) {
+ size_t in = 0;
+ size_t out = 0;
+ for (; in < seqsSize; ++in) {
+ if (sequences[in].offset == 0 && sequences[in].matchLength == 0) {
+ if (in != seqsSize - 1) {
+ sequences[in+1].litLength += sequences[in].litLength;
+ }
+ } else {
+ sequences[out] = sequences[in];
+ ++out;
+ }
+ }
+ return out;
+}
+
+/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */
+static int ZSTD_isRLE(const BYTE* src, size_t length) {
+ const BYTE* ip = src;
+ const BYTE value = ip[0];
+ const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL);
+ const size_t unrollSize = sizeof(size_t) * 4;
+ const size_t unrollMask = unrollSize - 1;
+ const size_t prefixLength = length & unrollMask;
+ size_t i;
+ if (length == 1) return 1;
+ /* Check if prefix is RLE first before using unrolled loop */
+ if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) {
+ return 0;
+ }
+ for (i = prefixLength; i != length; i += unrollSize) {
+ size_t u;
+ for (u = 0; u < unrollSize; u += sizeof(size_t)) {
+ if (MEM_readST(ip + i + u) != valueST) {
+ return 0;
+ } } }
+ return 1;
+}
+
+/* Returns true if the given block may be RLE.
+ * This is just a heuristic based on the compressibility.
+ * It may return both false positives and false negatives.
+ */
+static int ZSTD_maybeRLE(seqStore_t const* seqStore)
+{
+ size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
+ size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart);
+
+ return nbSeqs < 4 && nbLits < 10;
+}
+
+static void
+ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
+{
+ ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock;
+ bs->prevCBlock = bs->nextCBlock;
+ bs->nextCBlock = tmp;
+}
+
+/* Writes the block header */
+static void
+writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock)
+{
+ U32 const cBlockHeader = cSize == 1 ?
+ lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
+ lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
+ MEM_writeLE24(op, cBlockHeader);
+ DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock);
+}
+
+/** ZSTD_buildBlockEntropyStats_literals() :
+ * Builds entropy for the literals.
+ * Stores literals block type (raw, rle, compressed, repeat) and
+ * huffman description table to hufMetadata.
+ * Requires ENTROPY_WORKSPACE_SIZE workspace
+ * @return : size of huffman description table, or an error code
+ */
+static size_t
+ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
+ const ZSTD_hufCTables_t* prevHuf,
+ ZSTD_hufCTables_t* nextHuf,
+ ZSTD_hufCTablesMetadata_t* hufMetadata,
+ const int literalsCompressionIsDisabled,
+ void* workspace, size_t wkspSize,
+ int hufFlags)
+{
+ BYTE* const wkspStart = (BYTE*)workspace;
+ BYTE* const wkspEnd = wkspStart + wkspSize;
+ BYTE* const countWkspStart = wkspStart;
+ unsigned* const countWksp = (unsigned*)workspace;
+ const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned);
+ BYTE* const nodeWksp = countWkspStart + countWkspSize;
+ const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp);
+ unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
+ unsigned huffLog = LitHufLog;
+ HUF_repeat repeat = prevHuf->repeatMode;
+ DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize);
+
+ /* Prepare nextEntropy assuming reusing the existing table */
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+
+ if (literalsCompressionIsDisabled) {
+ DEBUGLOG(5, "set_basic - disabled");
+ hufMetadata->hType = set_basic;
+ return 0;
+ }
+
+ /* small ? don't even attempt compression (speed opt) */
+#ifndef COMPRESS_LITERALS_SIZE_MIN
+# define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */
+#endif
+ { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
+ if (srcSize <= minLitSize) {
+ DEBUGLOG(5, "set_basic - too small");
+ hufMetadata->hType = set_basic;
+ return 0;
+ } }
+
+ /* Scan input and build symbol stats */
+ { size_t const largest =
+ HIST_count_wksp (countWksp, &maxSymbolValue,
+ (const BYTE*)src, srcSize,
+ workspace, wkspSize);
+ FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
+ if (largest == srcSize) {
+ /* only one literal symbol */
+ DEBUGLOG(5, "set_rle");
+ hufMetadata->hType = set_rle;
+ return 0;
+ }
+ if (largest <= (srcSize >> 7)+4) {
+ /* heuristic: likely not compressible */
+ DEBUGLOG(5, "set_basic - no gain");
+ hufMetadata->hType = set_basic;
+ return 0;
+ } }
+
+ /* Validate the previous Huffman table */
+ if (repeat == HUF_repeat_check
+ && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
+ repeat = HUF_repeat_none;
+ }
+
+ /* Build Huffman Tree */
+ ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
+ huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags);
+ assert(huffLog <= LitHufLog);
+ { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
+ maxSymbolValue, huffLog,
+ nodeWksp, nodeWkspSize);
+ FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
+ huffLog = (U32)maxBits;
+ }
+ { /* Build and write the CTable */
+ size_t const newCSize = HUF_estimateCompressedSize(
+ (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
+ size_t const hSize = HUF_writeCTable_wksp(
+ hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
+ (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
+ nodeWksp, nodeWkspSize);
+ /* Check against repeating the previous CTable */
+ if (repeat != HUF_repeat_none) {
+ size_t const oldCSize = HUF_estimateCompressedSize(
+ (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
+ if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
+ DEBUGLOG(5, "set_repeat - smaller");
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+ hufMetadata->hType = set_repeat;
+ return 0;
+ } }
+ if (newCSize + hSize >= srcSize) {
+ DEBUGLOG(5, "set_basic - no gains");
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+ hufMetadata->hType = set_basic;
+ return 0;
+ }
+ DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
+ hufMetadata->hType = set_compressed;
+ nextHuf->repeatMode = HUF_repeat_check;
+ return hSize;
+ }
+}
+
+
+/* ZSTD_buildDummySequencesStatistics():
+ * Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic,
+ * and updates nextEntropy to the appropriate repeatMode.
+ */
+static ZSTD_symbolEncodingTypeStats_t
+ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy)
+{
+ ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0};
+ nextEntropy->litlength_repeatMode = FSE_repeat_none;
+ nextEntropy->offcode_repeatMode = FSE_repeat_none;
+ nextEntropy->matchlength_repeatMode = FSE_repeat_none;
+ return stats;
+}
+
+/** ZSTD_buildBlockEntropyStats_sequences() :
+ * Builds entropy for the sequences.
+ * Stores symbol compression modes and fse table to fseMetadata.
+ * Requires ENTROPY_WORKSPACE_SIZE wksp.
+ * @return : size of fse tables or error code */
+static size_t
+ZSTD_buildBlockEntropyStats_sequences(
+ const seqStore_t* seqStorePtr,
+ const ZSTD_fseCTables_t* prevEntropy,
+ ZSTD_fseCTables_t* nextEntropy,
+ const ZSTD_CCtx_params* cctxParams,
+ ZSTD_fseCTablesMetadata_t* fseMetadata,
+ void* workspace, size_t wkspSize)
+{
+ ZSTD_strategy const strategy = cctxParams->cParams.strategy;
+ size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ BYTE* const ostart = fseMetadata->fseTablesBuffer;
+ BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
+ BYTE* op = ostart;
+ unsigned* countWorkspace = (unsigned*)workspace;
+ unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1);
+ size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace);
+ ZSTD_symbolEncodingTypeStats_t stats;
+
+ DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq);
+ stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
+ prevEntropy, nextEntropy, op, oend,
+ strategy, countWorkspace,
+ entropyWorkspace, entropyWorkspaceSize)
+ : ZSTD_buildDummySequencesStatistics(nextEntropy);
+ FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!");
+ fseMetadata->llType = (symbolEncodingType_e) stats.LLtype;
+ fseMetadata->ofType = (symbolEncodingType_e) stats.Offtype;
+ fseMetadata->mlType = (symbolEncodingType_e) stats.MLtype;
+ fseMetadata->lastCountSize = stats.lastCountSize;
+ return stats.size;
+}
+
+
+/** ZSTD_buildBlockEntropyStats() :
+ * Builds entropy for the block.
+ * Requires workspace size ENTROPY_WORKSPACE_SIZE
+ * @return : 0 on success, or an error code
+ * Note : also employed in superblock
+ */
+size_t ZSTD_buildBlockEntropyStats(
+ const seqStore_t* seqStorePtr,
+ const ZSTD_entropyCTables_t* prevEntropy,
+ ZSTD_entropyCTables_t* nextEntropy,
+ const ZSTD_CCtx_params* cctxParams,
+ ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ void* workspace, size_t wkspSize)
+{
+ size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
+ int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD);
+ int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0;
+
+ entropyMetadata->hufMetadata.hufDesSize =
+ ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
+ &prevEntropy->huf, &nextEntropy->huf,
+ &entropyMetadata->hufMetadata,
+ ZSTD_literalsCompressionIsDisabled(cctxParams),
+ workspace, wkspSize, hufFlags);
+
+ FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
+ entropyMetadata->fseMetadata.fseTablesSize =
+ ZSTD_buildBlockEntropyStats_sequences(seqStorePtr,
+ &prevEntropy->fse, &nextEntropy->fse,
+ cctxParams,
+ &entropyMetadata->fseMetadata,
+ workspace, wkspSize);
+ FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed");
+ return 0;
+}
+
+/* Returns the size estimate for the literals section (header + content) of a block */
+static size_t
+ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
+ const ZSTD_hufCTables_t* huf,
+ const ZSTD_hufCTablesMetadata_t* hufMetadata,
+ void* workspace, size_t wkspSize,
+ int writeEntropy)
+{
+ unsigned* const countWksp = (unsigned*)workspace;
+ unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
+ size_t literalSectionHeaderSize = 3 + (litSize >= 1 KB) + (litSize >= 16 KB);
+ U32 singleStream = litSize < 256;
+
+ if (hufMetadata->hType == set_basic) return litSize;
+ else if (hufMetadata->hType == set_rle) return 1;
+ else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) {
+ size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize);
+ if (ZSTD_isError(largest)) return litSize;
+ { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue);
+ if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize;
+ if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */
+ return cLitSizeEstimate + literalSectionHeaderSize;
+ } }
+ assert(0); /* impossible */
+ return 0;
+}
+
+/* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */
+static size_t
+ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
+ const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
+ const FSE_CTable* fseCTable,
+ const U8* additionalBits,
+ short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+ void* workspace, size_t wkspSize)
+{
+ unsigned* const countWksp = (unsigned*)workspace;
+ const BYTE* ctp = codeTable;
+ const BYTE* const ctStart = ctp;
+ const BYTE* const ctEnd = ctStart + nbSeq;
+ size_t cSymbolTypeSizeEstimateInBits = 0;
+ unsigned max = maxCode;
+
+ HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */
+ if (type == set_basic) {
+ /* We selected this encoding type, so it must be valid. */
+ assert(max <= defaultMax);
+ (void)defaultMax;
+ cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max);
+ } else if (type == set_rle) {
+ cSymbolTypeSizeEstimateInBits = 0;
+ } else if (type == set_compressed || type == set_repeat) {
+ cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max);
+ }
+ if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) {
+ return nbSeq * 10;
+ }
+ while (ctp < ctEnd) {
+ if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp];
+ else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */
+ ctp++;
+ }
+ return cSymbolTypeSizeEstimateInBits >> 3;
+}
+
+/* Returns the size estimate for the sequences section (header + content) of a block */
+static size_t
+ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable,
+ const BYTE* llCodeTable,
+ const BYTE* mlCodeTable,
+ size_t nbSeq,
+ const ZSTD_fseCTables_t* fseTables,
+ const ZSTD_fseCTablesMetadata_t* fseMetadata,
+ void* workspace, size_t wkspSize,
+ int writeEntropy)
+{
+ size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ);
+ size_t cSeqSizeEstimate = 0;
+ cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff,
+ fseTables->offcodeCTable, NULL,
+ OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
+ workspace, wkspSize);
+ cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL,
+ fseTables->litlengthCTable, LL_bits,
+ LL_defaultNorm, LL_defaultNormLog, MaxLL,
+ workspace, wkspSize);
+ cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML,
+ fseTables->matchlengthCTable, ML_bits,
+ ML_defaultNorm, ML_defaultNormLog, MaxML,
+ workspace, wkspSize);
+ if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
+ return cSeqSizeEstimate + sequencesSectionHeaderSize;
+}
+
+/* Returns the size estimate for a given stream of literals, of, ll, ml */
+static size_t
+ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
+ const BYTE* ofCodeTable,
+ const BYTE* llCodeTable,
+ const BYTE* mlCodeTable,
+ size_t nbSeq,
+ const ZSTD_entropyCTables_t* entropy,
+ const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ void* workspace, size_t wkspSize,
+ int writeLitEntropy, int writeSeqEntropy)
+{
+ size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize,
+ &entropy->huf, &entropyMetadata->hufMetadata,
+ workspace, wkspSize, writeLitEntropy);
+ size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
+ nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
+ workspace, wkspSize, writeSeqEntropy);
+ return seqSize + literalsSize + ZSTD_blockHeaderSize;
+}
+
+/* Builds entropy statistics and uses them for blocksize estimation.
+ *
+ * @return: estimated compressed size of the seqStore, or a zstd error.
+ */
+static size_t
+ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc)
+{
+ ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
+ DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()");
+ FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore,
+ &zc->blockState.prevCBlock->entropy,
+ &zc->blockState.nextCBlock->entropy,
+ &zc->appliedParams,
+ entropyMetadata,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), "");
+ return ZSTD_estimateBlockSize(
+ seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
+ seqStore->ofCode, seqStore->llCode, seqStore->mlCode,
+ (size_t)(seqStore->sequences - seqStore->sequencesStart),
+ &zc->blockState.nextCBlock->entropy,
+ entropyMetadata,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
+ (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1);
+}
+
+/* Returns literals bytes represented in a seqStore */
+static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore)
+{
+ size_t literalsBytes = 0;
+ size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
+ size_t i;
+ for (i = 0; i < nbSeqs; ++i) {
+ seqDef const seq = seqStore->sequencesStart[i];
+ literalsBytes += seq.litLength;
+ if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) {
+ literalsBytes += 0x10000;
+ } }
+ return literalsBytes;
+}
+
+/* Returns match bytes represented in a seqStore */
+static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore)
+{
+ size_t matchBytes = 0;
+ size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
+ size_t i;
+ for (i = 0; i < nbSeqs; ++i) {
+ seqDef seq = seqStore->sequencesStart[i];
+ matchBytes += seq.mlBase + MINMATCH;
+ if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) {
+ matchBytes += 0x10000;
+ } }
+ return matchBytes;
+}
+
+/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx).
+ * Stores the result in resultSeqStore.
+ */
+static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore,
+ const seqStore_t* originalSeqStore,
+ size_t startIdx, size_t endIdx)
+{
+ *resultSeqStore = *originalSeqStore;
+ if (startIdx > 0) {
+ resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx;
+ resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
+ }
+
+ /* Move longLengthPos into the correct position if necessary */
+ if (originalSeqStore->longLengthType != ZSTD_llt_none) {
+ if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) {
+ resultSeqStore->longLengthType = ZSTD_llt_none;
+ } else {
+ resultSeqStore->longLengthPos -= (U32)startIdx;
+ }
+ }
+ resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx;
+ resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx;
+ if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) {
+ /* This accounts for possible last literals if the derived chunk reaches the end of the block */
+ assert(resultSeqStore->lit == originalSeqStore->lit);
+ } else {
+ size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
+ resultSeqStore->lit = resultSeqStore->litStart + literalsBytes;
+ }
+ resultSeqStore->llCode += startIdx;
+ resultSeqStore->mlCode += startIdx;
+ resultSeqStore->ofCode += startIdx;
+}
+
+/**
+ * Returns the raw offset represented by the combination of offBase, ll0, and repcode history.
+ * offBase must represent a repcode in the numeric representation of ZSTD_storeSeq().
+ */
+static U32
+ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0)
+{
+ U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */
+ assert(OFFBASE_IS_REPCODE(offBase));
+ if (adjustedRepCode == ZSTD_REP_NUM) {
+ assert(ll0);
+ /* litlength == 0 and offCode == 2 implies selection of first repcode - 1
+ * This is only valid if it results in a valid offset value, aka > 0.
+ * Note : it may happen that `rep[0]==1` in exceptional circumstances.
+ * In which case this function will return 0, which is an invalid offset.
+ * It's not an issue though, since this value will be
+ * compared and discarded within ZSTD_seqStore_resolveOffCodes().
+ */
+ return rep[0] - 1;
+ }
+ return rep[adjustedRepCode];
+}
+
+/**
+ * ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise
+ * due to emission of RLE/raw blocks that disturb the offset history,
+ * and replaces any repcodes within the seqStore that may be invalid.
+ *
+ * dRepcodes are updated as would be on the decompression side.
+ * cRepcodes are updated exactly in accordance with the seqStore.
+ *
+ * Note : this function assumes seq->offBase respects the following numbering scheme :
+ * 0 : invalid
+ * 1-3 : repcode 1-3
+ * 4+ : real_offset+3
+ */
+static void
+ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
+ const seqStore_t* const seqStore, U32 const nbSeq)
+{
+ U32 idx = 0;
+ for (; idx < nbSeq; ++idx) {
+ seqDef* const seq = seqStore->sequencesStart + idx;
+ U32 const ll0 = (seq->litLength == 0);
+ U32 const offBase = seq->offBase;
+ assert(offBase > 0);
+ if (OFFBASE_IS_REPCODE(offBase)) {
+ U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0);
+ U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0);
+ /* Adjust simulated decompression repcode history if we come across a mismatch. Replace
+ * the repcode with the offset it actually references, determined by the compression
+ * repcode history.
+ */
+ if (dRawOffset != cRawOffset) {
+ seq->offBase = OFFSET_TO_OFFBASE(cRawOffset);
+ }
+ }
+ /* Compression repcode history is always updated with values directly from the unmodified seqStore.
+ * Decompression repcode history may use modified seq->offset value taken from compression repcode history.
+ */
+ ZSTD_updateRep(dRepcodes->rep, seq->offBase, ll0);
+ ZSTD_updateRep(cRepcodes->rep, offBase, ll0);
+ }
+}
+
+/* ZSTD_compressSeqStore_singleBlock():
+ * Compresses a seqStore into a block with a block header, into the buffer dst.
+ *
+ * Returns the total size of that block (including header) or a ZSTD error code.
+ */
+static size_t
+ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc,
+ const seqStore_t* const seqStore,
+ repcodes_t* const dRep, repcodes_t* const cRep,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ U32 lastBlock, U32 isPartition)
+{
+ const U32 rleMaxLength = 25;
+ BYTE* op = (BYTE*)dst;
+ const BYTE* ip = (const BYTE*)src;
+ size_t cSize;
+ size_t cSeqsSize;
+
+ /* In case of an RLE or raw block, the simulated decompression repcode history must be reset */
+ repcodes_t const dRepOriginal = *dRep;
+ DEBUGLOG(5, "ZSTD_compressSeqStore_singleBlock");
+ if (isPartition)
+ ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart));
+
+ RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "Block header doesn't fit");
+ cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore,
+ &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
+ &zc->appliedParams,
+ op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize,
+ srcSize,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
+ zc->bmi2);
+ FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!");
+
+ if (!zc->isFirstBlock &&
+ cSeqsSize < rleMaxLength &&
+ ZSTD_isRLE((BYTE const*)src, srcSize)) {
+ /* We don't want to emit our first block as a RLE even if it qualifies because
+ * doing so will cause the decoder (cli only) to throw a "should consume all input error."
+ * This is only an issue for zstd <= v1.4.3
+ */
+ cSeqsSize = 1;
+ }
+
+ if (zc->seqCollector.collectSequences) {
+ ZSTD_copyBlockSequences(zc);
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
+ return 0;
+ }
+
+ if (cSeqsSize == 0) {
+ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "Nocompress block failed");
+ DEBUGLOG(4, "Writing out nocompress block, size: %zu", cSize);
+ *dRep = dRepOriginal; /* reset simulated decompression repcode history */
+ } else if (cSeqsSize == 1) {
+ cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "RLE compress block failed");
+ DEBUGLOG(4, "Writing out RLE block, size: %zu", cSize);
+ *dRep = dRepOriginal; /* reset simulated decompression repcode history */
+ } else {
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
+ writeBlockHeader(op, cSeqsSize, srcSize, lastBlock);
+ cSize = ZSTD_blockHeaderSize + cSeqsSize;
+ DEBUGLOG(4, "Writing out compressed block, size: %zu", cSize);
+ }
+
+ if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+ zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+
+ return cSize;
+}
+
+/* Struct to keep track of where we are in our recursive calls. */
+typedef struct {
+ U32* splitLocations; /* Array of split indices */
+ size_t idx; /* The current index within splitLocations being worked on */
+} seqStoreSplits;
+
+#define MIN_SEQUENCES_BLOCK_SPLITTING 300
+
+/* Helper function to perform the recursive search for block splits.
+ * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half.
+ * If advantageous to split, then we recurse down the two sub-blocks.
+ * If not, or if an error occurred in estimation, then we do not recurse.
+ *
+ * Note: The recursion depth is capped by a heuristic minimum number of sequences,
+ * defined by MIN_SEQUENCES_BLOCK_SPLITTING.
+ * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING).
+ * In practice, recursion depth usually doesn't go beyond 4.
+ *
+ * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS.
+ * At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize
+ * maximum of 128 KB, this value is actually impossible to reach.
+ */
+static void
+ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx,
+ ZSTD_CCtx* zc, const seqStore_t* origSeqStore)
+{
+ seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
+ seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
+ seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
+ size_t estimatedOriginalSize;
+ size_t estimatedFirstHalfSize;
+ size_t estimatedSecondHalfSize;
+ size_t midIdx = (startIdx + endIdx)/2;
+
+ DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx);
+ assert(endIdx >= startIdx);
+ if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) {
+ DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx);
+ return;
+ }
+ ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx);
+ ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx);
+ ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx);
+ estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(fullSeqStoreChunk, zc);
+ estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(firstHalfSeqStore, zc);
+ estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(secondHalfSeqStore, zc);
+ DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu",
+ estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize);
+ if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) {
+ return;
+ }
+ if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) {
+ DEBUGLOG(5, "split decided at seqNb:%zu", midIdx);
+ ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore);
+ splits->splitLocations[splits->idx] = (U32)midIdx;
+ splits->idx++;
+ ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore);
+ }
+}
+
+/* Base recursive function.
+ * Populates a table with intra-block partition indices that can improve compression ratio.
+ *
+ * @return: number of splits made (which equals the size of the partition table - 1).
+ */
+static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq)
+{
+ seqStoreSplits splits;
+ splits.splitLocations = partitions;
+ splits.idx = 0;
+ if (nbSeq <= 4) {
+ DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq);
+ /* Refuse to try and split anything with less than 4 sequences */
+ return 0;
+ }
+ ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore);
+ splits.splitLocations[splits.idx] = nbSeq;
+ DEBUGLOG(5, "ZSTD_deriveBlockSplits: final nb partitions: %zu", splits.idx+1);
+ return splits.idx;
+}
+
+/* ZSTD_compressBlock_splitBlock():
+ * Attempts to split a given block into multiple blocks to improve compression ratio.
+ *
+ * Returns combined size of all blocks (which includes headers), or a ZSTD error code.
+ */
+static size_t
+ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t blockSize,
+ U32 lastBlock, U32 nbSeq)
+{
+ size_t cSize = 0;
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* op = (BYTE*)dst;
+ size_t i = 0;
+ size_t srcBytesTotal = 0;
+ U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */
+ seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
+ seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore;
+ size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
+
+ /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history
+ * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two
+ * separate repcode histories that simulate repcode history on compression and decompression side,
+ * and use the histories to determine whether we must replace a particular repcode with its raw offset.
+ *
+ * 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed
+ * or RLE. This allows us to retrieve the offset value that an invalid repcode references within
+ * a nocompress/RLE block.
+ * 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use
+ * the replacement offset value rather than the original repcode to update the repcode history.
+ * dRep also will be the final repcode history sent to the next block.
+ *
+ * See ZSTD_seqStore_resolveOffCodes() for more details.
+ */
+ repcodes_t dRep;
+ repcodes_t cRep;
+ ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
+ ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
+ ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t));
+
+ DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
+ (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
+ (unsigned)zc->blockState.matchState.nextToUpdate);
+
+ if (numSplits == 0) {
+ size_t cSizeSingleBlock =
+ ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
+ &dRep, &cRep,
+ op, dstCapacity,
+ ip, blockSize,
+ lastBlock, 0 /* isPartition */);
+ FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!");
+ DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits");
+ assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX);
+ assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize);
+ return cSizeSingleBlock;
+ }
+
+ ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]);
+ for (i = 0; i <= numSplits; ++i) {
+ size_t cSizeChunk;
+ U32 const lastPartition = (i == numSplits);
+ U32 lastBlockEntireSrc = 0;
+
+ size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore);
+ srcBytesTotal += srcBytes;
+ if (lastPartition) {
+ /* This is the final partition, need to account for possible last literals */
+ srcBytes += blockSize - srcBytesTotal;
+ lastBlockEntireSrc = lastBlock;
+ } else {
+ ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]);
+ }
+
+ cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, currSeqStore,
+ &dRep, &cRep,
+ op, dstCapacity,
+ ip, srcBytes,
+ lastBlockEntireSrc, 1 /* isPartition */);
+ DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size",
+ ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk);
+ FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!");
+
+ ip += srcBytes;
+ op += cSizeChunk;
+ dstCapacity -= cSizeChunk;
+ cSize += cSizeChunk;
+ *currSeqStore = *nextSeqStore;
+ assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize);
+ }
+ /* cRep and dRep may have diverged during the compression.
+ * If so, we use the dRep repcodes for the next block.
+ */
+ ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t));
+ return cSize;
+}
+
+static size_t
+ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, U32 lastBlock)
+{
+ U32 nbSeq;
+ size_t cSize;
+ DEBUGLOG(4, "ZSTD_compressBlock_splitBlock");
+ assert(zc->appliedParams.useBlockSplitter == ZSTD_ps_enable);
+
+ { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
+ FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
+ if (bss == ZSTDbss_noCompress) {
+ if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+ zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+ cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
+ DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block");
+ return cSize;
+ }
+ nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart);
+ }
+
+ cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq);
+ FORWARD_IF_ERROR(cSize, "Splitting blocks failed!");
+ return cSize;
+}
+
+static size_t
+ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, U32 frame)
+{
+ /* This is an estimated upper bound for the length of an rle block.
+ * This isn't the actual upper bound.
+ * Finding the real threshold needs further investigation.
+ */
+ const U32 rleMaxLength = 25;
+ size_t cSize;
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* op = (BYTE*)dst;
+ DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
+ (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
+ (unsigned)zc->blockState.matchState.nextToUpdate);
+
+ { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
+ FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
+ if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; }
+ }
+
+ if (zc->seqCollector.collectSequences) {
+ ZSTD_copyBlockSequences(zc);
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
+ return 0;
+ }
+
+ /* encode sequences and literals */
+ cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore,
+ &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
+ &zc->appliedParams,
+ dst, dstCapacity,
+ srcSize,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
+ zc->bmi2);
+
+ if (frame &&
+ /* We don't want to emit our first block as a RLE even if it qualifies because
+ * doing so will cause the decoder (cli only) to throw a "should consume all input error."
+ * This is only an issue for zstd <= v1.4.3
+ */
+ !zc->isFirstBlock &&
+ cSize < rleMaxLength &&
+ ZSTD_isRLE(ip, srcSize))
+ {
+ cSize = 1;
+ op[0] = ip[0];
+ }
+
+out:
+ if (!ZSTD_isError(cSize) && cSize > 1) {
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
+ }
+ /* We check that dictionaries have offset codes available for the first
+ * block. After the first block, the offcode table might not have large
+ * enough codes to represent the offsets in the data.
+ */
+ if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+ zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+
+ return cSize;
+}
+
+static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const size_t bss, U32 lastBlock)
+{
+ DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()");
+ if (bss == ZSTDbss_compress) {
+ if (/* We don't want to emit our first block as a RLE even if it qualifies because
+ * doing so will cause the decoder (cli only) to throw a "should consume all input error."
+ * This is only an issue for zstd <= v1.4.3
+ */
+ !zc->isFirstBlock &&
+ ZSTD_maybeRLE(&zc->seqStore) &&
+ ZSTD_isRLE((BYTE const*)src, srcSize))
+ {
+ return ZSTD_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock);
+ }
+ /* Attempt superblock compression.
+ *
+ * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the
+ * standard ZSTD_compressBound(). This is a problem, because even if we have
+ * space now, taking an extra byte now could cause us to run out of space later
+ * and violate ZSTD_compressBound().
+ *
+ * Define blockBound(blockSize) = blockSize + ZSTD_blockHeaderSize.
+ *
+ * In order to respect ZSTD_compressBound() we must attempt to emit a raw
+ * uncompressed block in these cases:
+ * * cSize == 0: Return code for an uncompressed block.
+ * * cSize == dstSize_tooSmall: We may have expanded beyond blockBound(srcSize).
+ * ZSTD_noCompressBlock() will return dstSize_tooSmall if we are really out of
+ * output space.
+ * * cSize >= blockBound(srcSize): We have expanded the block too much so
+ * emit an uncompressed block.
+ */
+ { size_t const cSize =
+ ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
+ if (cSize != ERROR(dstSize_tooSmall)) {
+ size_t const maxCSize =
+ srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed");
+ if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) {
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
+ return cSize;
+ }
+ }
+ }
+ } /* if (bss == ZSTDbss_compress)*/
+
+ DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()");
+ /* Superblock compression failed, attempt to emit a single no compress block.
+ * The decoder will be able to stream this block since it is uncompressed.
+ */
+ return ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock);
+}
+
+static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ U32 lastBlock)
+{
+ size_t cSize = 0;
+ const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
+ DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)",
+ (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize);
+ FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
+
+ cSize = ZSTD_compressBlock_targetCBlockSize_body(zc, dst, dstCapacity, src, srcSize, bss, lastBlock);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed");
+
+ if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+ zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+
+ return cSize;
+}
+
+static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
+ ZSTD_cwksp* ws,
+ ZSTD_CCtx_params const* params,
+ void const* ip,
+ void const* iend)
+{
+ U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy);
+ U32 const maxDist = (U32)1 << params->cParams.windowLog;
+ if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)) {
+ U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip);
+ ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+ ZSTD_cwksp_mark_tables_dirty(ws);
+ ZSTD_reduceIndex(ms, params, correction);
+ ZSTD_cwksp_mark_tables_clean(ws);
+ if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
+ else ms->nextToUpdate -= correction;
+ /* invalidate dictionaries on overflow correction */
+ ms->loadedDictEnd = 0;
+ ms->dictMatchState = NULL;
+ }
+}
+
+/*! ZSTD_compress_frameChunk() :
+* Compress a chunk of data into one or multiple blocks.
+* All blocks will be terminated, all input will be consumed.
+* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content.
+* Frame is supposed already started (header already produced)
+* @return : compressed size, or an error code
+*/
+static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ U32 lastFrameChunk)
+{
+ size_t blockSize = cctx->blockSize;
+ size_t remaining = srcSize;
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* op = ostart;
+ U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
+
+ assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX);
+
+ DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize);
+ if (cctx->appliedParams.fParams.checksumFlag && srcSize)
+ XXH64_update(&cctx->xxhState, src, srcSize);
+
+ while (remaining) {
+ ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
+ U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
+
+ /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
+ * additional 1. We need to revisit and change this logic to be more consistent */
+ RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1,
+ dstSize_tooSmall,
+ "not enough space to store compressed block");
+ if (remaining < blockSize) blockSize = remaining;
+
+ ZSTD_overflowCorrectIfNeeded(
+ ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize);
+ ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
+ ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
+
+ /* Ensure hash/chain table insertion resumes no sooner than lowlimit */
+ if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit;
+
+ { size_t cSize;
+ if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) {
+ cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed");
+ assert(cSize > 0);
+ assert(cSize <= blockSize + ZSTD_blockHeaderSize);
+ } else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) {
+ cSize = ZSTD_compressBlock_splitBlock(cctx, op, dstCapacity, ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_splitBlock failed");
+ assert(cSize > 0 || cctx->seqCollector.collectSequences == 1);
+ } else {
+ cSize = ZSTD_compressBlock_internal(cctx,
+ op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
+ ip, blockSize, 1 /* frame */);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_internal failed");
+
+ if (cSize == 0) { /* block is not compressible */
+ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
+ } else {
+ U32 const cBlockHeader = cSize == 1 ?
+ lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
+ lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
+ MEM_writeLE24(op, cBlockHeader);
+ cSize += ZSTD_blockHeaderSize;
+ }
+ } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/
+
+
+ ip += blockSize;
+ assert(remaining >= blockSize);
+ remaining -= blockSize;
+ op += cSize;
+ assert(dstCapacity >= cSize);
+ dstCapacity -= cSize;
+ cctx->isFirstBlock = 0;
+ DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u",
+ (unsigned)cSize);
+ } }
+
+ if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending;
+ return (size_t)(op-ostart);
+}
+
+
+static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
+ const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID)
+{ BYTE* const op = (BYTE*)dst;
+ U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
+ U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */
+ U32 const checksumFlag = params->fParams.checksumFlag>0;
+ U32 const windowSize = (U32)1 << params->cParams.windowLog;
+ U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
+ BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
+ U32 const fcsCode = params->fParams.contentSizeFlag ?
+ (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */
+ BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
+ size_t pos=0;
+
+ assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN));
+ RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall,
+ "dst buf is too small to fit worst-case frame header size.");
+ DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
+ !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode);
+ if (params->format == ZSTD_f_zstd1) {
+ MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
+ pos = 4;
+ }
+ op[pos++] = frameHeaderDescriptionByte;
+ if (!singleSegment) op[pos++] = windowLogByte;
+ switch(dictIDSizeCode)
+ {
+ default:
+ assert(0); /* impossible */
+ ZSTD_FALLTHROUGH;
+ case 0 : break;
+ case 1 : op[pos] = (BYTE)(dictID); pos++; break;
+ case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break;
+ case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break;
+ }
+ switch(fcsCode)
+ {
+ default:
+ assert(0); /* impossible */
+ ZSTD_FALLTHROUGH;
+ case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break;
+ case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break;
+ case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break;
+ case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break;
+ }
+ return pos;
+}
+
+/* ZSTD_writeSkippableFrame_advanced() :
+ * Writes out a skippable frame with the specified magic number variant (16 are supported),
+ * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data.
+ *
+ * Returns the total number of bytes written, or a ZSTD error code.
+ */
+size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, unsigned magicVariant) {
+ BYTE* op = (BYTE*)dst;
+ RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */,
+ dstSize_tooSmall, "Not enough room for skippable frame");
+ RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame");
+ RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported");
+
+ MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant));
+ MEM_writeLE32(op+4, (U32)srcSize);
+ ZSTD_memcpy(op+8, src, srcSize);
+ return srcSize + ZSTD_SKIPPABLEHEADERSIZE;
+}
+
+/* ZSTD_writeLastEmptyBlock() :
+ * output an empty Block with end-of-frame mark to complete a frame
+ * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h))
+ * or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize)
+ */
+size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity)
+{
+ RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall,
+ "dst buf is too small to write frame trailer empty block.");
+ { U32 const cBlockHeader24 = 1 /*lastBlock*/ + (((U32)bt_raw)<<1); /* 0 size */
+ MEM_writeLE24(dst, cBlockHeader24);
+ return ZSTD_blockHeaderSize;
+ }
+}
+
+size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq)
+{
+ RETURN_ERROR_IF(cctx->stage != ZSTDcs_init, stage_wrong,
+ "wrong cctx stage");
+ RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable,
+ parameter_unsupported,
+ "incompatible with ldm");
+ cctx->externSeqStore.seq = seq;
+ cctx->externSeqStore.size = nbSeq;
+ cctx->externSeqStore.capacity = nbSeq;
+ cctx->externSeqStore.pos = 0;
+ cctx->externSeqStore.posInSequence = 0;
+ return 0;
+}
+
+
+static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ U32 frame, U32 lastFrameChunk)
+{
+ ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
+ size_t fhSize = 0;
+
+ DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u",
+ cctx->stage, (unsigned)srcSize);
+ RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong,
+ "missing init (ZSTD_compressBegin)");
+
+ if (frame && (cctx->stage==ZSTDcs_init)) {
+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams,
+ cctx->pledgedSrcSizePlusOne-1, cctx->dictID);
+ FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed");
+ assert(fhSize <= dstCapacity);
+ dstCapacity -= fhSize;
+ dst = (char*)dst + fhSize;
+ cctx->stage = ZSTDcs_ongoing;
+ }
+
+ if (!srcSize) return fhSize; /* do not generate an empty block if no input */
+
+ if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) {
+ ms->forceNonContiguous = 0;
+ ms->nextToUpdate = ms->window.dictLimit;
+ }
+ if (cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) {
+ ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0);
+ }
+
+ if (!frame) {
+ /* overflow check and correction for block mode */
+ ZSTD_overflowCorrectIfNeeded(
+ ms, &cctx->workspace, &cctx->appliedParams,
+ src, (BYTE const*)src + srcSize);
+ }
+
+ DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize);
+ { size_t const cSize = frame ?
+ ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
+ ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */);
+ FORWARD_IF_ERROR(cSize, "%s", frame ? "ZSTD_compress_frameChunk failed" : "ZSTD_compressBlock_internal failed");
+ cctx->consumedSrcSize += srcSize;
+ cctx->producedCSize += (cSize + fhSize);
+ assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0));
+ if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */
+ ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1);
+ RETURN_ERROR_IF(
+ cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne,
+ srcSize_wrong,
+ "error : pledgedSrcSize = %u, while realSrcSize >= %u",
+ (unsigned)cctx->pledgedSrcSizePlusOne-1,
+ (unsigned)cctx->consumedSrcSize);
+ }
+ return cSize + fhSize;
+ }
+}
+
+size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize);
+ return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */);
+}
+
+
+size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
+{
+ ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams;
+ assert(!ZSTD_checkCParams(cParams));
+ return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog);
+}
+
+size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize);
+ { size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
+ RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); }
+
+ return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
+}
+
+/*! ZSTD_loadDictionaryContent() :
+ * @return : 0, or an error code
+ */
+static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
+ ldmState_t* ls,
+ ZSTD_cwksp* ws,
+ ZSTD_CCtx_params const* params,
+ const void* src, size_t srcSize,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp)
+{
+ const BYTE* ip = (const BYTE*) src;
+ const BYTE* const iend = ip + srcSize;
+ int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL;
+
+ /* Assert that the ms params match the params we're being given */
+ ZSTD_assertEqualCParams(params->cParams, ms->cParams);
+
+ { /* Ensure large dictionaries can't cause index overflow */
+
+ /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX.
+ * Dictionaries right at the edge will immediately trigger overflow
+ * correction, but I don't want to insert extra constraints here.
+ */
+ U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX;
+
+ int const CDictTaggedIndices = ZSTD_CDictIndicesAreTagged(&params->cParams);
+ if (CDictTaggedIndices && tfp == ZSTD_tfp_forCDict) {
+ /* Some dictionary matchfinders in zstd use "short cache",
+ * which treats the lower ZSTD_SHORT_CACHE_TAG_BITS of each
+ * CDict hashtable entry as a tag rather than as part of an index.
+ * When short cache is used, we need to truncate the dictionary
+ * so that its indices don't overlap with the tag. */
+ U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX;
+ maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize);
+ assert(!loadLdmDict);
+ }
+
+ /* If the dictionary is too large, only load the suffix of the dictionary. */
+ if (srcSize > maxDictSize) {
+ ip = iend - maxDictSize;
+ src = ip;
+ srcSize = maxDictSize;
+ } }
+
+ if (srcSize > ZSTD_CHUNKSIZE_MAX) {
+ /* We must have cleared our windows when our source is this large. */
+ assert(ZSTD_window_isEmpty(ms->window));
+ if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window));
+ }
+
+ DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder);
+ ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0);
+ ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
+ ms->forceNonContiguous = params->deterministicRefPrefix;
+
+ if (loadLdmDict) {
+ ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0);
+ ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base);
+ }
+
+ if (srcSize <= HASH_READ_SIZE) return 0;
+
+ ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend);
+
+ if (loadLdmDict)
+ ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
+
+ switch(params->cParams.strategy)
+ {
+ case ZSTD_fast:
+ ZSTD_fillHashTable(ms, iend, dtlm, tfp);
+ break;
+ case ZSTD_dfast:
+ ZSTD_fillDoubleHashTable(ms, iend, dtlm, tfp);
+ break;
+
+ case ZSTD_greedy:
+ case ZSTD_lazy:
+ case ZSTD_lazy2:
+ assert(srcSize >= HASH_READ_SIZE);
+ if (ms->dedicatedDictSearch) {
+ assert(ms->chainTable != NULL);
+ ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE);
+ } else {
+ assert(params->useRowMatchFinder != ZSTD_ps_auto);
+ if (params->useRowMatchFinder == ZSTD_ps_enable) {
+ size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16);
+ ZSTD_memset(ms->tagTable, 0, tagTableSize);
+ ZSTD_row_update(ms, iend-HASH_READ_SIZE);
+ DEBUGLOG(4, "Using row-based hash table for lazy dict");
+ } else {
+ ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE);
+ DEBUGLOG(4, "Using chain-based hash table for lazy dict");
+ }
+ }
+ break;
+
+ case ZSTD_btlazy2: /* we want the dictionary table fully sorted */
+ case ZSTD_btopt:
+ case ZSTD_btultra:
+ case ZSTD_btultra2:
+ assert(srcSize >= HASH_READ_SIZE);
+ ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
+ break;
+
+ default:
+ assert(0); /* not possible : not a valid strategy id */
+ }
+
+ ms->nextToUpdate = (U32)(iend - ms->window.base);
+ return 0;
+}
+
+
+/* Dictionaries that assign zero probability to symbols that show up causes problems
+ * when FSE encoding. Mark dictionaries with zero probability symbols as FSE_repeat_check
+ * and only dictionaries with 100% valid symbols can be assumed valid.
+ */
+static FSE_repeat ZSTD_dictNCountRepeat(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue)
+{
+ U32 s;
+ if (dictMaxSymbolValue < maxSymbolValue) {
+ return FSE_repeat_check;
+ }
+ for (s = 0; s <= maxSymbolValue; ++s) {
+ if (normalizedCounter[s] == 0) {
+ return FSE_repeat_check;
+ }
+ }
+ return FSE_repeat_valid;
+}
+
+size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
+ const void* const dict, size_t dictSize)
+{
+ short offcodeNCount[MaxOff+1];
+ unsigned offcodeMaxValue = MaxOff;
+ const BYTE* dictPtr = (const BYTE*)dict; /* skip magic num and dict ID */
+ const BYTE* const dictEnd = dictPtr + dictSize;
+ dictPtr += 8;
+ bs->entropy.huf.repeatMode = HUF_repeat_check;
+
+ { unsigned maxSymbolValue = 255;
+ unsigned hasZeroWeights = 1;
+ size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr,
+ dictEnd-dictPtr, &hasZeroWeights);
+
+ /* We only set the loaded table as valid if it contains all non-zero
+ * weights. Otherwise, we set it to check */
+ if (!hasZeroWeights)
+ bs->entropy.huf.repeatMode = HUF_repeat_valid;
+
+ RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted, "");
+ dictPtr += hufHeaderSize;
+ }
+
+ { unsigned offcodeLog;
+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
+ RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
+ /* fill all offset symbols to avoid garbage at end of table */
+ RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
+ bs->entropy.fse.offcodeCTable,
+ offcodeNCount, MaxOff, offcodeLog,
+ workspace, HUF_WORKSPACE_SIZE)),
+ dictionary_corrupted, "");
+ /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
+ dictPtr += offcodeHeaderSize;
+ }
+
+ { short matchlengthNCount[MaxML+1];
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
+ RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
+ RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
+ bs->entropy.fse.matchlengthCTable,
+ matchlengthNCount, matchlengthMaxValue, matchlengthLog,
+ workspace, HUF_WORKSPACE_SIZE)),
+ dictionary_corrupted, "");
+ bs->entropy.fse.matchlength_repeatMode = ZSTD_dictNCountRepeat(matchlengthNCount, matchlengthMaxValue, MaxML);
+ dictPtr += matchlengthHeaderSize;
+ }
+
+ { short litlengthNCount[MaxLL+1];
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
+ RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
+ RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
+ bs->entropy.fse.litlengthCTable,
+ litlengthNCount, litlengthMaxValue, litlengthLog,
+ workspace, HUF_WORKSPACE_SIZE)),
+ dictionary_corrupted, "");
+ bs->entropy.fse.litlength_repeatMode = ZSTD_dictNCountRepeat(litlengthNCount, litlengthMaxValue, MaxLL);
+ dictPtr += litlengthHeaderSize;
+ }
+
+ RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, "");
+ bs->rep[0] = MEM_readLE32(dictPtr+0);
+ bs->rep[1] = MEM_readLE32(dictPtr+4);
+ bs->rep[2] = MEM_readLE32(dictPtr+8);
+ dictPtr += 12;
+
+ { size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
+ U32 offcodeMax = MaxOff;
+ if (dictContentSize <= ((U32)-1) - 128 KB) {
+ U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */
+ offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */
+ }
+ /* All offset values <= dictContentSize + 128 KB must be representable for a valid table */
+ bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff));
+
+ /* All repCodes must be <= dictContentSize and != 0 */
+ { U32 u;
+ for (u=0; u<3; u++) {
+ RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, "");
+ RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, "");
+ } } }
+
+ return dictPtr - (const BYTE*)dict;
+}
+
+/* Dictionary format :
+ * See :
+ * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format
+ */
+/*! ZSTD_loadZstdDictionary() :
+ * @return : dictID, or an error code
+ * assumptions : magic number supposed already checked
+ * dictSize supposed >= 8
+ */
+static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
+ ZSTD_matchState_t* ms,
+ ZSTD_cwksp* ws,
+ ZSTD_CCtx_params const* params,
+ const void* dict, size_t dictSize,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp,
+ void* workspace)
+{
+ const BYTE* dictPtr = (const BYTE*)dict;
+ const BYTE* const dictEnd = dictPtr + dictSize;
+ size_t dictID;
+ size_t eSize;
+ ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
+ assert(dictSize >= 8);
+ assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY);
+
+ dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ );
+ eSize = ZSTD_loadCEntropy(bs, workspace, dict, dictSize);
+ FORWARD_IF_ERROR(eSize, "ZSTD_loadCEntropy failed");
+ dictPtr += eSize;
+
+ {
+ size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
+ FORWARD_IF_ERROR(ZSTD_loadDictionaryContent(
+ ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), "");
+ }
+ return dictID;
+}
+
+/** ZSTD_compress_insertDictionary() :
+* @return : dictID, or an error code */
+static size_t
+ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
+ ZSTD_matchState_t* ms,
+ ldmState_t* ls,
+ ZSTD_cwksp* ws,
+ const ZSTD_CCtx_params* params,
+ const void* dict, size_t dictSize,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp,
+ void* workspace)
+{
+ DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
+ if ((dict==NULL) || (dictSize<8)) {
+ RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, "");
+ return 0;
+ }
+
+ ZSTD_reset_compressedBlockState(bs);
+
+ /* dict restricted modes */
+ if (dictContentType == ZSTD_dct_rawContent)
+ return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm, tfp);
+
+ if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) {
+ if (dictContentType == ZSTD_dct_auto) {
+ DEBUGLOG(4, "raw content dictionary detected");
+ return ZSTD_loadDictionaryContent(
+ ms, ls, ws, params, dict, dictSize, dtlm, tfp);
+ }
+ RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, "");
+ assert(0); /* impossible */
+ }
+
+ /* dict as full zstd dictionary */
+ return ZSTD_loadZstdDictionary(
+ bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace);
+}
+
+#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB)
+#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL)
+
+/*! ZSTD_compressBegin_internal() :
+ * Assumption : either @dict OR @cdict (or none) is non-NULL, never both
+ * @return : 0, or an error code */
+static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
+ const void* dict, size_t dictSize,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params, U64 pledgedSrcSize,
+ ZSTD_buffered_policy_e zbuff)
+{
+ size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize;
+#if ZSTD_TRACE
+ cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0;
+#endif
+ DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog);
+ /* params are supposed to be fully validated at this point */
+ assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
+ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
+ if ( (cdict)
+ && (cdict->dictContentSize > 0)
+ && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
+ || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
+ || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
+ || cdict->compressionLevel == 0)
+ && (params->attachDictPref != ZSTD_dictForceLoad) ) {
+ return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff);
+ }
+
+ FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
+ dictContentSize,
+ ZSTDcrp_makeClean, zbuff) , "");
+ { size_t const dictID = cdict ?
+ ZSTD_compress_insertDictionary(
+ cctx->blockState.prevCBlock, &cctx->blockState.matchState,
+ &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent,
+ cdict->dictContentSize, cdict->dictContentType, dtlm,
+ ZSTD_tfp_forCCtx, cctx->entropyWorkspace)
+ : ZSTD_compress_insertDictionary(
+ cctx->blockState.prevCBlock, &cctx->blockState.matchState,
+ &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize,
+ dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->entropyWorkspace);
+ FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
+ assert(dictID <= UINT_MAX);
+ cctx->dictID = (U32)dictID;
+ cctx->dictContentSize = dictContentSize;
+ }
+ return 0;
+}
+
+size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
+ const void* dict, size_t dictSize,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params,
+ unsigned long long pledgedSrcSize)
+{
+ DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog);
+ /* compression parameters verification and optimization */
+ FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , "");
+ return ZSTD_compressBegin_internal(cctx,
+ dict, dictSize, dictContentType, dtlm,
+ cdict,
+ params, pledgedSrcSize,
+ ZSTDb_not_buffered);
+}
+
+/*! ZSTD_compressBegin_advanced() :
+* @return : 0, or an error code */
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
+ const void* dict, size_t dictSize,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+ ZSTD_CCtx_params cctxParams;
+ ZSTD_CCtxParams_init_internal(&cctxParams, &params, ZSTD_NO_CLEVEL);
+ return ZSTD_compressBegin_advanced_internal(cctx,
+ dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast,
+ NULL /*cdict*/,
+ &cctxParams, pledgedSrcSize);
+}
+
+size_t
+ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
+{
+ ZSTD_CCtx_params cctxParams;
+ { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
+ ZSTD_CCtxParams_init_internal(&cctxParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel);
+ }
+ DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize);
+ return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
+ &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
+}
+
+size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
+{
+ return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
+}
+
+
+/*! ZSTD_writeEpilogue() :
+* Ends a frame.
+* @return : nb of bytes written into dst (or an error code) */
+static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
+{
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* op = ostart;
+ size_t fhSize = 0;
+
+ DEBUGLOG(4, "ZSTD_writeEpilogue");
+ RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing");
+
+ /* special case : empty frame */
+ if (cctx->stage == ZSTDcs_init) {
+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0);
+ FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed");
+ dstCapacity -= fhSize;
+ op += fhSize;
+ cctx->stage = ZSTDcs_ongoing;
+ }
+
+ if (cctx->stage != ZSTDcs_ending) {
+ /* write one last empty block, make it the "last" block */
+ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0;
+ RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue");
+ MEM_writeLE32(op, cBlockHeader24);
+ op += ZSTD_blockHeaderSize;
+ dstCapacity -= ZSTD_blockHeaderSize;
+ }
+
+ if (cctx->appliedParams.fParams.checksumFlag) {
+ U32 const checksum = (U32) XXH64_digest(&cctx->xxhState);
+ RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum");
+ DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum);
+ MEM_writeLE32(op, checksum);
+ op += 4;
+ }
+
+ cctx->stage = ZSTDcs_created; /* return to "created but no init" status */
+ return op-ostart;
+}
+
+void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize)
+{
+#if ZSTD_TRACE
+ if (cctx->traceCtx && ZSTD_trace_compress_end != NULL) {
+ int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0;
+ ZSTD_Trace trace;
+ ZSTD_memset(&trace, 0, sizeof(trace));
+ trace.version = ZSTD_VERSION_NUMBER;
+ trace.streaming = streaming;
+ trace.dictionaryID = cctx->dictID;
+ trace.dictionarySize = cctx->dictContentSize;
+ trace.uncompressedSize = cctx->consumedSrcSize;
+ trace.compressedSize = cctx->producedCSize + extraCSize;
+ trace.params = &cctx->appliedParams;
+ trace.cctx = cctx;
+ ZSTD_trace_compress_end(cctx->traceCtx, &trace);
+ }
+ cctx->traceCtx = 0;
+#else
+ (void)cctx;
+ (void)extraCSize;
+#endif
+}
+
+size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ size_t endResult;
+ size_t const cSize = ZSTD_compressContinue_internal(cctx,
+ dst, dstCapacity, src, srcSize,
+ 1 /* frame mode */, 1 /* last chunk */);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressContinue_internal failed");
+ endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
+ FORWARD_IF_ERROR(endResult, "ZSTD_writeEpilogue failed");
+ assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0));
+ if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */
+ ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1);
+ DEBUGLOG(4, "end of frame : controlling src size");
+ RETURN_ERROR_IF(
+ cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1,
+ srcSize_wrong,
+ "error : pledgedSrcSize = %u, while realSrcSize = %u",
+ (unsigned)cctx->pledgedSrcSizePlusOne-1,
+ (unsigned)cctx->consumedSrcSize);
+ }
+ ZSTD_CCtx_trace(cctx, endResult);
+ return cSize + endResult;
+}
+
+size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ ZSTD_parameters params)
+{
+ DEBUGLOG(4, "ZSTD_compress_advanced");
+ FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), "");
+ ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, &params, ZSTD_NO_CLEVEL);
+ return ZSTD_compress_advanced_internal(cctx,
+ dst, dstCapacity,
+ src, srcSize,
+ dict, dictSize,
+ &cctx->simpleApiParams);
+}
+
+/* Internal */
+size_t ZSTD_compress_advanced_internal(
+ ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ const ZSTD_CCtx_params* params)
+{
+ DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize);
+ FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
+ dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
+ params, srcSize, ZSTDb_not_buffered) , "");
+ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
+}
+
+size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize,
+ int compressionLevel)
+{
+ {
+ ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict);
+ assert(params.fParams.contentSizeFlag == 1);
+ ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel);
+ }
+ DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize);
+ return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams);
+}
+
+size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel)
+{
+ DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize);
+ assert(cctx != NULL);
+ return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel);
+}
+
+size_t ZSTD_compress(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel)
+{
+ size_t result;
+#if ZSTD_COMPRESS_HEAPMODE
+ ZSTD_CCtx* cctx = ZSTD_createCCtx();
+ RETURN_ERROR_IF(!cctx, memory_allocation, "ZSTD_createCCtx failed");
+ result = ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel);
+ ZSTD_freeCCtx(cctx);
+#else
+ ZSTD_CCtx ctxBody;
+ ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem);
+ result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
+ ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */
+#endif
+ return result;
+}
+
+
+/* ===== Dictionary API ===== */
+
+/*! ZSTD_estimateCDictSize_advanced() :
+ * Estimate amount of memory that will be needed to create a dictionary with following arguments */
+size_t ZSTD_estimateCDictSize_advanced(
+ size_t dictSize, ZSTD_compressionParameters cParams,
+ ZSTD_dictLoadMethod_e dictLoadMethod)
+{
+ DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict));
+ return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
+ /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small
+ * in case we are using DDS with row-hash. */
+ + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams),
+ /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0)
+ + (dictLoadMethod == ZSTD_dlm_byRef ? 0
+ : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *))));
+}
+
+size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel)
+{
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
+ return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
+}
+
+size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
+{
+ if (cdict==NULL) return 0; /* support sizeof on NULL */
+ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict));
+ /* cdict may be in the workspace */
+ return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict))
+ + ZSTD_cwksp_sizeof(&cdict->workspace);
+}
+
+static size_t ZSTD_initCDict_internal(
+ ZSTD_CDict* cdict,
+ const void* dictBuffer, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_CCtx_params params)
+{
+ DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType);
+ assert(!ZSTD_checkCParams(params.cParams));
+ cdict->matchState.cParams = params.cParams;
+ cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch;
+ if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) {
+ cdict->dictContent = dictBuffer;
+ } else {
+ void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*)));
+ RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!");
+ cdict->dictContent = internalBuffer;
+ ZSTD_memcpy(internalBuffer, dictBuffer, dictSize);
+ }
+ cdict->dictContentSize = dictSize;
+ cdict->dictContentType = dictContentType;
+
+ cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE);
+
+
+ /* Reset the state to no dictionary */
+ ZSTD_reset_compressedBlockState(&cdict->cBlockState);
+ FORWARD_IF_ERROR(ZSTD_reset_matchState(
+ &cdict->matchState,
+ &cdict->workspace,
+ &params.cParams,
+ params.useRowMatchFinder,
+ ZSTDcrp_makeClean,
+ ZSTDirp_reset,
+ ZSTD_resetTarget_CDict), "");
+ /* (Maybe) load the dictionary
+ * Skips loading the dictionary if it is < 8 bytes.
+ */
+ { params.compressionLevel = ZSTD_CLEVEL_DEFAULT;
+ params.fParams.contentSizeFlag = 1;
+ { size_t const dictID = ZSTD_compress_insertDictionary(
+ &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace,
+ &params, cdict->dictContent, cdict->dictContentSize,
+ dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace);
+ FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
+ assert(dictID <= (size_t)(U32)-1);
+ cdict->dictID = (U32)dictID;
+ }
+ }
+
+ return 0;
+}
+
+static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_compressionParameters cParams,
+ ZSTD_paramSwitch_e useRowMatchFinder,
+ U32 enableDedicatedDictSearch,
+ ZSTD_customMem customMem)
+{
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+
+ { size_t const workspaceSize =
+ ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) +
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) +
+ ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) +
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
+ : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
+ void* const workspace = ZSTD_customMalloc(workspaceSize, customMem);
+ ZSTD_cwksp ws;
+ ZSTD_CDict* cdict;
+
+ if (!workspace) {
+ ZSTD_customFree(workspace, customMem);
+ return NULL;
+ }
+
+ ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc);
+
+ cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict));
+ assert(cdict != NULL);
+ ZSTD_cwksp_move(&cdict->workspace, &ws);
+ cdict->customMem = customMem;
+ cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */
+ cdict->useRowMatchFinder = useRowMatchFinder;
+ return cdict;
+ }
+}
+
+ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_compressionParameters cParams,
+ ZSTD_customMem customMem)
+{
+ ZSTD_CCtx_params cctxParams;
+ ZSTD_memset(&cctxParams, 0, sizeof(cctxParams));
+ ZSTD_CCtxParams_init(&cctxParams, 0);
+ cctxParams.cParams = cParams;
+ cctxParams.customMem = customMem;
+ return ZSTD_createCDict_advanced2(
+ dictBuffer, dictSize,
+ dictLoadMethod, dictContentType,
+ &cctxParams, customMem);
+}
+
+ZSTD_CDict* ZSTD_createCDict_advanced2(
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ const ZSTD_CCtx_params* originalCctxParams,
+ ZSTD_customMem customMem)
+{
+ ZSTD_CCtx_params cctxParams = *originalCctxParams;
+ ZSTD_compressionParameters cParams;
+ ZSTD_CDict* cdict;
+
+ DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType);
+ if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
+
+ if (cctxParams.enableDedicatedDictSearch) {
+ cParams = ZSTD_dedicatedDictSearch_getCParams(
+ cctxParams.compressionLevel, dictSize);
+ ZSTD_overrideCParams(&cParams, &cctxParams.cParams);
+ } else {
+ cParams = ZSTD_getCParamsFromCCtxParams(
+ &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
+ }
+
+ if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) {
+ /* Fall back to non-DDSS params */
+ cctxParams.enableDedicatedDictSearch = 0;
+ cParams = ZSTD_getCParamsFromCCtxParams(
+ &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
+ }
+
+ DEBUGLOG(3, "ZSTD_createCDict_advanced2: DDS: %u", cctxParams.enableDedicatedDictSearch);
+ cctxParams.cParams = cParams;
+ cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
+
+ cdict = ZSTD_createCDict_advanced_internal(dictSize,
+ dictLoadMethod, cctxParams.cParams,
+ cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch,
+ customMem);
+
+ if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
+ dict, dictSize,
+ dictLoadMethod, dictContentType,
+ cctxParams) )) {
+ ZSTD_freeCDict(cdict);
+ return NULL;
+ }
+
+ return cdict;
+}
+
+ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
+{
+ ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
+ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize,
+ ZSTD_dlm_byCopy, ZSTD_dct_auto,
+ cParams, ZSTD_defaultCMem);
+ if (cdict)
+ cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel;
+ return cdict;
+}
+
+ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
+{
+ ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
+ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize,
+ ZSTD_dlm_byRef, ZSTD_dct_auto,
+ cParams, ZSTD_defaultCMem);
+ if (cdict)
+ cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel;
+ return cdict;
+}
+
+size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
+{
+ if (cdict==NULL) return 0; /* support free on NULL */
+ { ZSTD_customMem const cMem = cdict->customMem;
+ int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict);
+ ZSTD_cwksp_free(&cdict->workspace, cMem);
+ if (!cdictInWorkspace) {
+ ZSTD_customFree(cdict, cMem);
+ }
+ return 0;
+ }
+}
+
+/*! ZSTD_initStaticCDict_advanced() :
+ * Generate a digested dictionary in provided memory area.
+ * workspace: The memory area to emplace the dictionary into.
+ * Provided pointer must 8-bytes aligned.
+ * It must outlive dictionary usage.
+ * workspaceSize: Use ZSTD_estimateCDictSize()
+ * to determine how large workspace must be.
+ * cParams : use ZSTD_getCParams() to transform a compression level
+ * into its relevants cParams.
+ * @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
+ * Note : there is no corresponding "free" function.
+ * Since workspace was allocated externally, it must be freed externally.
+ */
+const ZSTD_CDict* ZSTD_initStaticCDict(
+ void* workspace, size_t workspaceSize,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_compressionParameters cParams)
+{
+ ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams);
+ /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */
+ size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0);
+ size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ + (dictLoadMethod == ZSTD_dlm_byRef ? 0
+ : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))))
+ + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
+ + matchStateSize;
+ ZSTD_CDict* cdict;
+ ZSTD_CCtx_params params;
+
+ if ((size_t)workspace & 7) return NULL; /* 8-aligned */
+
+ {
+ ZSTD_cwksp ws;
+ ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc);
+ cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict));
+ if (cdict == NULL) return NULL;
+ ZSTD_cwksp_move(&cdict->workspace, &ws);
+ }
+
+ DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u",
+ (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize));
+ if (workspaceSize < neededSize) return NULL;
+
+ ZSTD_CCtxParams_init(&params, 0);
+ params.cParams = cParams;
+ params.useRowMatchFinder = useRowMatchFinder;
+ cdict->useRowMatchFinder = useRowMatchFinder;
+
+ if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
+ dict, dictSize,
+ dictLoadMethod, dictContentType,
+ params) ))
+ return NULL;
+
+ return cdict;
+}
+
+ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict)
+{
+ assert(cdict != NULL);
+ return cdict->matchState.cParams;
+}
+
+/*! ZSTD_getDictID_fromCDict() :
+ * Provides the dictID of the dictionary loaded into `cdict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict)
+{
+ if (cdict==NULL) return 0;
+ return cdict->dictID;
+}
+
+/* ZSTD_compressBegin_usingCDict_internal() :
+ * Implementation of various ZSTD_compressBegin_usingCDict* functions.
+ */
+static size_t ZSTD_compressBegin_usingCDict_internal(
+ ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
+ ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
+{
+ ZSTD_CCtx_params cctxParams;
+ DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal");
+ RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!");
+ /* Initialize the cctxParams from the cdict */
+ {
+ ZSTD_parameters params;
+ params.fParams = fParams;
+ params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
+ || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
+ || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
+ || cdict->compressionLevel == 0 ) ?
+ ZSTD_getCParamsFromCDict(cdict)
+ : ZSTD_getCParams(cdict->compressionLevel,
+ pledgedSrcSize,
+ cdict->dictContentSize);
+ ZSTD_CCtxParams_init_internal(&cctxParams, &params, cdict->compressionLevel);
+ }
+ /* Increase window log to fit the entire dictionary and source if the
+ * source size is known. Limit the increase to 19, which is the
+ * window log for compression level 1 with the largest source size.
+ */
+ if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+ U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19);
+ U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1;
+ cctxParams.cParams.windowLog = MAX(cctxParams.cParams.windowLog, limitedSrcLog);
+ }
+ return ZSTD_compressBegin_internal(cctx,
+ NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast,
+ cdict,
+ &cctxParams, pledgedSrcSize,
+ ZSTDb_not_buffered);
+}
+
+
+/* ZSTD_compressBegin_usingCDict_advanced() :
+ * This function is DEPRECATED.
+ * cdict must be != NULL */
+size_t ZSTD_compressBegin_usingCDict_advanced(
+ ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
+ ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
+{
+ return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize);
+}
+
+/* ZSTD_compressBegin_usingCDict() :
+ * cdict must be != NULL */
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
+{
+ ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
+ return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN);
+}
+
+/*! ZSTD_compress_usingCDict_internal():
+ * Implementation of various ZSTD_compress_usingCDict* functions.
+ */
+static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
+{
+ FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */
+ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
+}
+
+/*! ZSTD_compress_usingCDict_advanced():
+ * This function is DEPRECATED.
+ */
+size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
+{
+ return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams);
+}
+
+/*! ZSTD_compress_usingCDict() :
+ * Compression using a digested Dictionary.
+ * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
+ * Note that compression parameters are decided at CDict creation time
+ * while frame parameters are hardcoded */
+size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict)
+{
+ ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
+ return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams);
+}
+
+
+
+/* ******************************************************************
+* Streaming
+********************************************************************/
+
+ZSTD_CStream* ZSTD_createCStream(void)
+{
+ DEBUGLOG(3, "ZSTD_createCStream");
+ return ZSTD_createCStream_advanced(ZSTD_defaultCMem);
+}
+
+ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize)
+{
+ return ZSTD_initStaticCCtx(workspace, workspaceSize);
+}
+
+ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
+{ /* CStream and CCtx are now same object */
+ return ZSTD_createCCtx_advanced(customMem);
+}
+
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
+{
+ return ZSTD_freeCCtx(zcs); /* same object */
+}
+
+
+
+/*====== Initialization ======*/
+
+size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; }
+
+size_t ZSTD_CStreamOutSize(void)
+{
+ return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
+}
+
+static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize)
+{
+ if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize))
+ return ZSTD_cpm_attachDict;
+ else
+ return ZSTD_cpm_noAttachDict;
+}
+
+/* ZSTD_resetCStream():
+ * pledgedSrcSize == 0 means "unknown" */
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss)
+{
+ /* temporary : 0 interpreted as "unknown" during transition period.
+ * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN.
+ * 0 will be interpreted as "empty" in the future.
+ */
+ U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
+ DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize);
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
+ return 0;
+}
+
+/*! ZSTD_initCStream_internal() :
+ * Note : for lib/compress only. Used by zstdmt_compress.c.
+ * Assumption 1 : params are valid
+ * Assumption 2 : either dict, or cdict, is defined, not both */
+size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
+ const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params,
+ unsigned long long pledgedSrcSize)
+{
+ DEBUGLOG(4, "ZSTD_initCStream_internal");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
+ assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
+ zcs->requestedParams = *params;
+ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
+ if (dict) {
+ FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , "");
+ } else {
+ /* Dictionary is cleared if !cdict */
+ FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , "");
+ }
+ return 0;
+}
+
+/* ZSTD_initCStream_usingCDict_advanced() :
+ * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+ const ZSTD_CDict* cdict,
+ ZSTD_frameParameters fParams,
+ unsigned long long pledgedSrcSize)
+{
+ DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
+ zcs->requestedParams.fParams = fParams;
+ FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , "");
+ return 0;
+}
+
+/* note : cdict must outlive compression session */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
+{
+ DEBUGLOG(4, "ZSTD_initCStream_usingCDict");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , "");
+ return 0;
+}
+
+
+/* ZSTD_initCStream_advanced() :
+ * pledgedSrcSize must be exact.
+ * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
+ * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+ const void* dict, size_t dictSize,
+ ZSTD_parameters params, unsigned long long pss)
+{
+ /* for compatibility with older programs relying on this behavior.
+ * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN.
+ * This line will be removed in the future.
+ */
+ U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
+ DEBUGLOG(4, "ZSTD_initCStream_advanced");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
+ FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , "");
+ ZSTD_CCtxParams_setZstdParams(&zcs->requestedParams, &params);
+ FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , "");
+ return 0;
+}
+
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
+{
+ DEBUGLOG(4, "ZSTD_initCStream_usingDict");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , "");
+ return 0;
+}
+
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss)
+{
+ /* temporary : 0 interpreted as "unknown" during transition period.
+ * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN.
+ * 0 will be interpreted as "empty" in the future.
+ */
+ U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
+ DEBUGLOG(4, "ZSTD_initCStream_srcSize");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , "");
+ return 0;
+}
+
+size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
+{
+ DEBUGLOG(4, "ZSTD_initCStream");
+ FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , "");
+ FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , "");
+ return 0;
+}
+
+/*====== Compression ======*/
+
+static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx)
+{
+ if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
+ return cctx->blockSize - cctx->stableIn_notConsumed;
+ }
+ assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered);
+ { size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos;
+ if (hintInSize==0) hintInSize = cctx->blockSize;
+ return hintInSize;
+ }
+}
+
+/** ZSTD_compressStream_generic():
+ * internal function for all *compressStream*() variants
+ * @return : hint size for next input to complete ongoing block */
+static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
+ ZSTD_outBuffer* output,
+ ZSTD_inBuffer* input,
+ ZSTD_EndDirective const flushMode)
+{
+ const char* const istart = (assert(input != NULL), (const char*)input->src);
+ const char* const iend = (istart != NULL) ? istart + input->size : istart;
+ const char* ip = (istart != NULL) ? istart + input->pos : istart;
+ char* const ostart = (assert(output != NULL), (char*)output->dst);
+ char* const oend = (ostart != NULL) ? ostart + output->size : ostart;
+ char* op = (ostart != NULL) ? ostart + output->pos : ostart;
+ U32 someMoreWork = 1;
+
+ /* check expectations */
+ DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos);
+ assert(zcs != NULL);
+ if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) {
+ assert(input->pos >= zcs->stableIn_notConsumed);
+ input->pos -= zcs->stableIn_notConsumed;
+ ip -= zcs->stableIn_notConsumed;
+ zcs->stableIn_notConsumed = 0;
+ }
+ if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) {
+ assert(zcs->inBuff != NULL);
+ assert(zcs->inBuffSize > 0);
+ }
+ if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) {
+ assert(zcs->outBuff != NULL);
+ assert(zcs->outBuffSize > 0);
+ }
+ if (input->src == NULL) assert(input->size == 0);
+ assert(input->pos <= input->size);
+ if (output->dst == NULL) assert(output->size == 0);
+ assert(output->pos <= output->size);
+ assert((U32)flushMode <= (U32)ZSTD_e_end);
+
+ while (someMoreWork) {
+ switch(zcs->streamStage)
+ {
+ case zcss_init:
+ RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!");
+
+ case zcss_load:
+ if ( (flushMode == ZSTD_e_end)
+ && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */
+ || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */
+ && (zcs->inBuffPos == 0) ) {
+ /* shortcut to compression pass directly into output buffer */
+ size_t const cSize = ZSTD_compressEnd(zcs,
+ op, oend-op, ip, iend-ip);
+ DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed");
+ ip = iend;
+ op += cSize;
+ zcs->frameEnded = 1;
+ ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ someMoreWork = 0; break;
+ }
+ /* complete loading into inBuffer in buffered mode */
+ if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) {
+ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
+ size_t const loaded = ZSTD_limitCopy(
+ zcs->inBuff + zcs->inBuffPos, toLoad,
+ ip, iend-ip);
+ zcs->inBuffPos += loaded;
+ if (ip) ip += loaded;
+ if ( (flushMode == ZSTD_e_continue)
+ && (zcs->inBuffPos < zcs->inBuffTarget) ) {
+ /* not enough input to fill full block : stop here */
+ someMoreWork = 0; break;
+ }
+ if ( (flushMode == ZSTD_e_flush)
+ && (zcs->inBuffPos == zcs->inToCompress) ) {
+ /* empty */
+ someMoreWork = 0; break;
+ }
+ } else {
+ assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable);
+ if ( (flushMode == ZSTD_e_continue)
+ && ( (size_t)(iend - ip) < zcs->blockSize) ) {
+ /* can't compress a full block : stop here */
+ zcs->stableIn_notConsumed = (size_t)(iend - ip);
+ ip = iend; /* pretend to have consumed input */
+ someMoreWork = 0; break;
+ }
+ if ( (flushMode == ZSTD_e_flush)
+ && (ip == iend) ) {
+ /* empty */
+ someMoreWork = 0; break;
+ }
+ }
+ /* compress current block (note : this stage cannot be stopped in the middle) */
+ DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode);
+ { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered);
+ void* cDst;
+ size_t cSize;
+ size_t oSize = oend-op;
+ size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress
+ : MIN((size_t)(iend - ip), zcs->blockSize);
+ if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable)
+ cDst = op; /* compress into output buffer, to skip flush stage */
+ else
+ cDst = zcs->outBuff, oSize = zcs->outBuffSize;
+ if (inputBuffered) {
+ unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
+ cSize = lastBlock ?
+ ZSTD_compressEnd(zcs, cDst, oSize,
+ zcs->inBuff + zcs->inToCompress, iSize) :
+ ZSTD_compressContinue(zcs, cDst, oSize,
+ zcs->inBuff + zcs->inToCompress, iSize);
+ FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
+ zcs->frameEnded = lastBlock;
+ /* prepare next block */
+ zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
+ if (zcs->inBuffTarget > zcs->inBuffSize)
+ zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize;
+ DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u",
+ (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize);
+ if (!lastBlock)
+ assert(zcs->inBuffTarget <= zcs->inBuffSize);
+ zcs->inToCompress = zcs->inBuffPos;
+ } else { /* !inputBuffered, hence ZSTD_bm_stable */
+ unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend);
+ cSize = lastBlock ?
+ ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) :
+ ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize);
+ /* Consume the input prior to error checking to mirror buffered mode. */
+ if (ip) ip += iSize;
+ FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
+ zcs->frameEnded = lastBlock;
+ if (lastBlock) assert(ip == iend);
+ }
+ if (cDst == op) { /* no need to flush */
+ op += cSize;
+ if (zcs->frameEnded) {
+ DEBUGLOG(5, "Frame completed directly in outBuffer");
+ someMoreWork = 0;
+ ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ }
+ break;
+ }
+ zcs->outBuffContentSize = cSize;
+ zcs->outBuffFlushedSize = 0;
+ zcs->streamStage = zcss_flush; /* pass-through to flush stage */
+ }
+ ZSTD_FALLTHROUGH;
+ case zcss_flush:
+ DEBUGLOG(5, "flush stage");
+ assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered);
+ { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
+ size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op),
+ zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
+ DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u",
+ (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed);
+ if (flushed)
+ op += flushed;
+ zcs->outBuffFlushedSize += flushed;
+ if (toFlush!=flushed) {
+ /* flush not fully completed, presumably because dst is too small */
+ assert(op==oend);
+ someMoreWork = 0;
+ break;
+ }
+ zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
+ if (zcs->frameEnded) {
+ DEBUGLOG(5, "Frame completed on flush");
+ someMoreWork = 0;
+ ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+ break;
+ }
+ zcs->streamStage = zcss_load;
+ break;
+ }
+
+ default: /* impossible */
+ assert(0);
+ }
+ }
+
+ input->pos = ip - istart;
+ output->pos = op - ostart;
+ if (zcs->frameEnded) return 0;
+ return ZSTD_nextInputSizeHint(zcs);
+}
+
+static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx)
+{
+#ifdef ZSTD_MULTITHREAD
+ if (cctx->appliedParams.nbWorkers >= 1) {
+ assert(cctx->mtctx != NULL);
+ return ZSTDMT_nextInputSizeHint(cctx->mtctx);
+ }
+#endif
+ return ZSTD_nextInputSizeHint(cctx);
+
+}
+
+size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+ FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) , "");
+ return ZSTD_nextInputSizeHint_MTorST(zcs);
+}
+
+/* After a compression call set the expected input/output buffer.
+ * This is validated at the start of the next compression call.
+ */
+static void
+ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, const ZSTD_outBuffer* output, const ZSTD_inBuffer* input)
+{
+ DEBUGLOG(5, "ZSTD_setBufferExpectations (for advanced stable in/out modes)");
+ if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
+ cctx->expectedInBuffer = *input;
+ }
+ if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) {
+ cctx->expectedOutBufferSize = output->size - output->pos;
+ }
+}
+
+/* Validate that the input/output buffers match the expectations set by
+ * ZSTD_setBufferExpectations.
+ */
+static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx,
+ ZSTD_outBuffer const* output,
+ ZSTD_inBuffer const* input,
+ ZSTD_EndDirective endOp)
+{
+ if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
+ ZSTD_inBuffer const expect = cctx->expectedInBuffer;
+ if (expect.src != input->src || expect.pos != input->pos)
+ RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableInBuffer enabled but input differs!");
+ }
+ (void)endOp;
+ if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) {
+ size_t const outBufferSize = output->size - output->pos;
+ if (cctx->expectedOutBufferSize != outBufferSize)
+ RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableOutBuffer enabled but output size differs!");
+ }
+ return 0;
+}
+
+static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
+ ZSTD_EndDirective endOp,
+ size_t inSize)
+{
+ ZSTD_CCtx_params params = cctx->requestedParams;
+ ZSTD_prefixDict const prefixDict = cctx->prefixDict;
+ FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */
+ ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */
+ assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */
+ if (cctx->cdict && !cctx->localDict.cdict) {
+ /* Let the cdict's compression level take priority over the requested params.
+ * But do not take the cdict's compression level if the "cdict" is actually a localDict
+ * generated from ZSTD_initLocalDict().
+ */
+ params.compressionLevel = cctx->cdict->compressionLevel;
+ }
+ DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
+ if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSize */
+
+ { size_t const dictSize = prefixDict.dict
+ ? prefixDict.dictSize
+ : (cctx->cdict ? cctx->cdict->dictContentSize : 0);
+ ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, &params, cctx->pledgedSrcSizePlusOne - 1);
+ params.cParams = ZSTD_getCParamsFromCCtxParams(
+ &params, cctx->pledgedSrcSizePlusOne-1,
+ dictSize, mode);
+ }
+
+ params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, &params.cParams);
+ params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, &params.cParams);
+ params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
+ params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
+ params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize);
+ params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel);
+
+#ifdef ZSTD_MULTITHREAD
+ /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */
+ RETURN_ERROR_IF(
+ params.useSequenceProducer == 1 && params.nbWorkers >= 1,
+ parameter_combination_unsupported,
+ "External sequence producer isn't supported with nbWorkers >= 1"
+ );
+
+ if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
+ params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
+ }
+ if (params.nbWorkers > 0) {
+#if ZSTD_TRACE
+ cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0;
+#endif
+ /* mt context creation */
+ if (cctx->mtctx == NULL) {
+ DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u",
+ params.nbWorkers);
+ cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem, cctx->pool);
+ RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!");
+ }
+ /* mt compression */
+ DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers);
+ FORWARD_IF_ERROR( ZSTDMT_initCStream_internal(
+ cctx->mtctx,
+ prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
+ cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , "");
+ cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0;
+ cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize;
+ cctx->consumedSrcSize = 0;
+ cctx->producedCSize = 0;
+ cctx->streamStage = zcss_load;
+ cctx->appliedParams = params;
+ } else
+#endif /* ZSTD_MULTITHREAD */
+ { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1;
+ assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
+ FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
+ prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, ZSTD_dtlm_fast,
+ cctx->cdict,
+ &params, pledgedSrcSize,
+ ZSTDb_buffered) , "");
+ assert(cctx->appliedParams.nbWorkers == 0);
+ cctx->inToCompress = 0;
+ cctx->inBuffPos = 0;
+ if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) {
+ /* for small input: avoid automatic flush on reaching end of block, since
+ * it would require to add a 3-bytes null block to end frame
+ */
+ cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize);
+ } else {
+ cctx->inBuffTarget = 0;
+ }
+ cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0;
+ cctx->streamStage = zcss_load;
+ cctx->frameEnded = 0;
+ }
+ return 0;
+}
+
+/* @return provides a minimum amount of data remaining to be flushed from internal buffers
+ */
+size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
+ ZSTD_outBuffer* output,
+ ZSTD_inBuffer* input,
+ ZSTD_EndDirective endOp)
+{
+ DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp);
+ /* check conditions */
+ RETURN_ERROR_IF(output->pos > output->size, dstSize_tooSmall, "invalid output buffer");
+ RETURN_ERROR_IF(input->pos > input->size, srcSize_wrong, "invalid input buffer");
+ RETURN_ERROR_IF((U32)endOp > (U32)ZSTD_e_end, parameter_outOfBound, "invalid endDirective");
+ assert(cctx != NULL);
+
+ /* transparent initialization stage */
+ if (cctx->streamStage == zcss_init) {
+ size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */
+ size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed;
+ if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */
+ && (endOp == ZSTD_e_continue) /* no flush requested, more input to come */
+ && (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */
+ if (cctx->stableIn_notConsumed) { /* not the first time */
+ /* check stable source guarantees */
+ RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer");
+ RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos");
+ }
+ /* pretend input was consumed, to give a sense forward progress */
+ input->pos = input->size;
+ /* save stable inBuffer, for later control, and flush/end */
+ cctx->expectedInBuffer = *input;
+ /* but actually input wasn't consumed, so keep track of position from where compression shall resume */
+ cctx->stableIn_notConsumed += inputSize;
+ /* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */
+ return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */
+ }
+ FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed");
+ ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */
+ }
+ /* end of transparent initialization stage */
+
+ FORWARD_IF_ERROR(ZSTD_checkBufferStability(cctx, output, input, endOp), "invalid buffers");
+ /* compression stage */
+#ifdef ZSTD_MULTITHREAD
+ if (cctx->appliedParams.nbWorkers > 0) {
+ size_t flushMin;
+ if (cctx->cParamsChanged) {
+ ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams);
+ cctx->cParamsChanged = 0;
+ }
+ if (cctx->stableIn_notConsumed) {
+ assert(cctx->appliedParams.inBufferMode == ZSTD_bm_stable);
+ /* some early data was skipped - make it available for consumption */
+ assert(input->pos >= cctx->stableIn_notConsumed);
+ input->pos -= cctx->stableIn_notConsumed;
+ cctx->stableIn_notConsumed = 0;
+ }
+ for (;;) {
+ size_t const ipos = input->pos;
+ size_t const opos = output->pos;
+ flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
+ cctx->consumedSrcSize += (U64)(input->pos - ipos);
+ cctx->producedCSize += (U64)(output->pos - opos);
+ if ( ZSTD_isError(flushMin)
+ || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
+ if (flushMin == 0)
+ ZSTD_CCtx_trace(cctx, 0);
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
+ }
+ FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed");
+
+ if (endOp == ZSTD_e_continue) {
+ /* We only require some progress with ZSTD_e_continue, not maximal progress.
+ * We're done if we've consumed or produced any bytes, or either buffer is
+ * full.
+ */
+ if (input->pos != ipos || output->pos != opos || input->pos == input->size || output->pos == output->size)
+ break;
+ } else {
+ assert(endOp == ZSTD_e_flush || endOp == ZSTD_e_end);
+ /* We require maximal progress. We're done when the flush is complete or the
+ * output buffer is full.
+ */
+ if (flushMin == 0 || output->pos == output->size)
+ break;
+ }
+ }
+ DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic");
+ /* Either we don't require maximum forward progress, we've finished the
+ * flush, or we are out of output space.
+ */
+ assert(endOp == ZSTD_e_continue || flushMin == 0 || output->pos == output->size);
+ ZSTD_setBufferExpectations(cctx, output, input);
+ return flushMin;
+ }
+#endif /* ZSTD_MULTITHREAD */
+ FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , "");
+ DEBUGLOG(5, "completed ZSTD_compressStream2");
+ ZSTD_setBufferExpectations(cctx, output, input);
+ return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
+}
+
+size_t ZSTD_compressStream2_simpleArgs (
+ ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos,
+ ZSTD_EndDirective endOp)
+{
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ output.dst = dst;
+ output.size = dstCapacity;
+ output.pos = *dstPos;
+ input.src = src;
+ input.size = srcSize;
+ input.pos = *srcPos;
+ /* ZSTD_compressStream2() will check validity of dstPos and srcPos */
+ { size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp);
+ *dstPos = output.pos;
+ *srcPos = input.pos;
+ return cErr;
+ }
+}
+
+size_t ZSTD_compress2(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode;
+ ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode;
+ DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize);
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
+ /* Enable stable input/output buffers. */
+ cctx->requestedParams.inBufferMode = ZSTD_bm_stable;
+ cctx->requestedParams.outBufferMode = ZSTD_bm_stable;
+ { size_t oPos = 0;
+ size_t iPos = 0;
+ size_t const result = ZSTD_compressStream2_simpleArgs(cctx,
+ dst, dstCapacity, &oPos,
+ src, srcSize, &iPos,
+ ZSTD_e_end);
+ /* Reset to the original values. */
+ cctx->requestedParams.inBufferMode = originalInBufferMode;
+ cctx->requestedParams.outBufferMode = originalOutBufferMode;
+
+ FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed");
+ if (result != 0) { /* compression not completed, due to lack of output space */
+ assert(oPos == dstCapacity);
+ RETURN_ERROR(dstSize_tooSmall, "");
+ }
+ assert(iPos == srcSize); /* all input is expected consumed */
+ return oPos;
+ }
+}
+
+/* ZSTD_validateSequence() :
+ * @offCode : is presumed to follow format required by ZSTD_storeSeq()
+ * @returns a ZSTD error code if sequence is not valid
+ */
+static size_t
+ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch,
+ size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer)
+{
+ U32 const windowSize = 1u << windowLog;
+ /* posInSrc represents the amount of data the decoder would decode up to this point.
+ * As long as the amount of data decoded is less than or equal to window size, offsets may be
+ * larger than the total length of output decoded in order to reference the dict, even larger than
+ * window size. After output surpasses windowSize, we're limited to windowSize offsets again.
+ */
+ size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize;
+ size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4;
+ RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!");
+ /* Validate maxNbSeq is large enough for the given matchLength and minMatch */
+ RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch");
+ return 0;
+}
+
+/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */
+static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0)
+{
+ U32 offBase = OFFSET_TO_OFFBASE(rawOffset);
+
+ if (!ll0 && rawOffset == rep[0]) {
+ offBase = REPCODE1_TO_OFFBASE;
+ } else if (rawOffset == rep[1]) {
+ offBase = REPCODE_TO_OFFBASE(2 - ll0);
+ } else if (rawOffset == rep[2]) {
+ offBase = REPCODE_TO_OFFBASE(3 - ll0);
+ } else if (ll0 && rawOffset == rep[0] - 1) {
+ offBase = REPCODE3_TO_OFFBASE;
+ }
+ return offBase;
+}
+
+size_t
+ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
+ ZSTD_sequencePosition* seqPos,
+ const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+ const void* src, size_t blockSize,
+ ZSTD_paramSwitch_e externalRepSearch)
+{
+ U32 idx = seqPos->idx;
+ U32 const startIdx = idx;
+ BYTE const* ip = (BYTE const*)(src);
+ const BYTE* const iend = ip + blockSize;
+ repcodes_t updatedRepcodes;
+ U32 dictSize;
+
+ DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreExplicitBlockDelim (blockSize = %zu)", blockSize);
+
+ if (cctx->cdict) {
+ dictSize = (U32)cctx->cdict->dictContentSize;
+ } else if (cctx->prefixDict.dict) {
+ dictSize = (U32)cctx->prefixDict.dictSize;
+ } else {
+ dictSize = 0;
+ }
+ ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
+ for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) {
+ U32 const litLength = inSeqs[idx].litLength;
+ U32 const matchLength = inSeqs[idx].matchLength;
+ U32 offBase;
+
+ if (externalRepSearch == ZSTD_ps_disable) {
+ offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset);
+ } else {
+ U32 const ll0 = (litLength == 0);
+ offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0);
+ ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
+ }
+
+ DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
+ if (cctx->appliedParams.validateSequences) {
+ seqPos->posInSrc += litLength + matchLength;
+ FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
+ cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer),
+ "Sequence validation failed");
+ }
+ RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
+ "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
+ ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
+ ip += matchLength + litLength;
+ }
+
+ /* If we skipped repcode search while parsing, we need to update repcodes now */
+ assert(externalRepSearch != ZSTD_ps_auto);
+ assert(idx >= startIdx);
+ if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) {
+ U32* const rep = updatedRepcodes.rep;
+ U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */
+
+ if (lastSeqIdx >= startIdx + 2) {
+ rep[2] = inSeqs[lastSeqIdx - 2].offset;
+ rep[1] = inSeqs[lastSeqIdx - 1].offset;
+ rep[0] = inSeqs[lastSeqIdx].offset;
+ } else if (lastSeqIdx == startIdx + 1) {
+ rep[2] = rep[0];
+ rep[1] = inSeqs[lastSeqIdx - 1].offset;
+ rep[0] = inSeqs[lastSeqIdx].offset;
+ } else {
+ assert(lastSeqIdx == startIdx);
+ rep[2] = rep[1];
+ rep[1] = rep[0];
+ rep[0] = inSeqs[lastSeqIdx].offset;
+ }
+ }
+
+ ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t));
+
+ if (inSeqs[idx].litLength) {
+ DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength);
+ ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength);
+ ip += inSeqs[idx].litLength;
+ seqPos->posInSrc += inSeqs[idx].litLength;
+ }
+ RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!");
+ seqPos->idx = idx+1;
+ return 0;
+}
+
+size_t
+ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
+ const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+ const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch)
+{
+ U32 idx = seqPos->idx;
+ U32 startPosInSequence = seqPos->posInSequence;
+ U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize;
+ size_t dictSize;
+ BYTE const* ip = (BYTE const*)(src);
+ BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */
+ repcodes_t updatedRepcodes;
+ U32 bytesAdjustment = 0;
+ U32 finalMatchSplit = 0;
+
+ /* TODO(embg) support fast parsing mode in noBlockDelim mode */
+ (void)externalRepSearch;
+
+ if (cctx->cdict) {
+ dictSize = cctx->cdict->dictContentSize;
+ } else if (cctx->prefixDict.dict) {
+ dictSize = cctx->prefixDict.dictSize;
+ } else {
+ dictSize = 0;
+ }
+ DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreNoBlockDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize);
+ DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
+ ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
+ while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) {
+ const ZSTD_Sequence currSeq = inSeqs[idx];
+ U32 litLength = currSeq.litLength;
+ U32 matchLength = currSeq.matchLength;
+ U32 const rawOffset = currSeq.offset;
+ U32 offBase;
+
+ /* Modify the sequence depending on where endPosInSequence lies */
+ if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) {
+ if (startPosInSequence >= litLength) {
+ startPosInSequence -= litLength;
+ litLength = 0;
+ matchLength -= startPosInSequence;
+ } else {
+ litLength -= startPosInSequence;
+ }
+ /* Move to the next sequence */
+ endPosInSequence -= currSeq.litLength + currSeq.matchLength;
+ startPosInSequence = 0;
+ } else {
+ /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence
+ does not reach the end of the match. So, we have to split the sequence */
+ DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u",
+ currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence);
+ if (endPosInSequence > litLength) {
+ U32 firstHalfMatchLength;
+ litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence;
+ firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength;
+ if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) {
+ /* Only ever split the match if it is larger than the block size */
+ U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence;
+ if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) {
+ /* Move the endPosInSequence backward so that it creates match of minMatch length */
+ endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength;
+ bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength;
+ firstHalfMatchLength -= bytesAdjustment;
+ }
+ matchLength = firstHalfMatchLength;
+ /* Flag that we split the last match - after storing the sequence, exit the loop,
+ but keep the value of endPosInSequence */
+ finalMatchSplit = 1;
+ } else {
+ /* Move the position in sequence backwards so that we don't split match, and break to store
+ * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence
+ * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so
+ * would cause the first half of the match to be too small
+ */
+ bytesAdjustment = endPosInSequence - currSeq.litLength;
+ endPosInSequence = currSeq.litLength;
+ break;
+ }
+ } else {
+ /* This sequence ends inside the literals, break to store the last literals */
+ break;
+ }
+ }
+ /* Check if this offset can be represented with a repcode */
+ { U32 const ll0 = (litLength == 0);
+ offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0);
+ ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
+ }
+
+ if (cctx->appliedParams.validateSequences) {
+ seqPos->posInSrc += litLength + matchLength;
+ FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
+ cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer),
+ "Sequence validation failed");
+ }
+ DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
+ RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
+ "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
+ ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
+ ip += matchLength + litLength;
+ if (!finalMatchSplit)
+ idx++; /* Next Sequence */
+ }
+ DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
+ assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength);
+ seqPos->idx = idx;
+ seqPos->posInSequence = endPosInSequence;
+ ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t));
+
+ iend -= bytesAdjustment;
+ if (ip != iend) {
+ /* Store any last literals */
+ U32 lastLLSize = (U32)(iend - ip);
+ assert(ip <= iend);
+ DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize);
+ ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize);
+ seqPos->posInSrc += lastLLSize;
+ }
+
+ return bytesAdjustment;
+}
+
+typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
+ const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+ const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
+static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode)
+{
+ ZSTD_sequenceCopier sequenceCopier = NULL;
+ assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode));
+ if (mode == ZSTD_sf_explicitBlockDelimiters) {
+ return ZSTD_copySequencesToSeqStoreExplicitBlockDelim;
+ } else if (mode == ZSTD_sf_noBlockDelimiters) {
+ return ZSTD_copySequencesToSeqStoreNoBlockDelim;
+ }
+ assert(sequenceCopier != NULL);
+ return sequenceCopier;
+}
+
+/* Discover the size of next block by searching for the delimiter.
+ * Note that a block delimiter **must** exist in this mode,
+ * otherwise it's an input error.
+ * The block size retrieved will be later compared to ensure it remains within bounds */
+static size_t
+blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos)
+{
+ int end = 0;
+ size_t blockSize = 0;
+ size_t spos = seqPos.idx;
+ DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize);
+ assert(spos <= inSeqsSize);
+ while (spos < inSeqsSize) {
+ end = (inSeqs[spos].offset == 0);
+ blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength;
+ if (end) {
+ if (inSeqs[spos].matchLength != 0)
+ RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0");
+ break;
+ }
+ spos++;
+ }
+ if (!end)
+ RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter");
+ return blockSize;
+}
+
+/* More a "target" block size */
+static size_t blockSize_noDelimiter(size_t blockSize, size_t remaining)
+{
+ int const lastBlock = (remaining <= blockSize);
+ return lastBlock ? remaining : blockSize;
+}
+
+static size_t determine_blockSize(ZSTD_sequenceFormat_e mode,
+ size_t blockSize, size_t remaining,
+ const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos)
+{
+ DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining);
+ if (mode == ZSTD_sf_noBlockDelimiters)
+ return blockSize_noDelimiter(blockSize, remaining);
+ { size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos);
+ FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters");
+ if (explicitBlockSize > blockSize)
+ RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block");
+ if (explicitBlockSize > remaining)
+ RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source");
+ return explicitBlockSize;
+ }
+}
+
+/* Compress, block-by-block, all of the sequences given.
+ *
+ * Returns the cumulative size of all compressed blocks (including their headers),
+ * otherwise a ZSTD error.
+ */
+static size_t
+ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+ const void* src, size_t srcSize)
+{
+ size_t cSize = 0;
+ size_t remaining = srcSize;
+ ZSTD_sequencePosition seqPos = {0, 0, 0};
+
+ BYTE const* ip = (BYTE const*)src;
+ BYTE* op = (BYTE*)dst;
+ ZSTD_sequenceCopier const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters);
+
+ DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize);
+ /* Special case: empty frame */
+ if (remaining == 0) {
+ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1);
+ RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header");
+ MEM_writeLE32(op, cBlockHeader24);
+ op += ZSTD_blockHeaderSize;
+ dstCapacity -= ZSTD_blockHeaderSize;
+ cSize += ZSTD_blockHeaderSize;
+ }
+
+ while (remaining) {
+ size_t compressedSeqsSize;
+ size_t cBlockSize;
+ size_t additionalByteAdjustment;
+ size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters,
+ cctx->blockSize, remaining,
+ inSeqs, inSeqsSize, seqPos);
+ U32 const lastBlock = (blockSize == remaining);
+ FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size");
+ assert(blockSize <= remaining);
+ ZSTD_resetSeqStore(&cctx->seqStore);
+ DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize);
+
+ additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes);
+ FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy");
+ blockSize -= additionalByteAdjustment;
+
+ /* If blocks are too small, emit as a nocompress block */
+ /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
+ * additional 1. We need to revisit and change this logic to be more consistent */
+ if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) {
+ cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed");
+ DEBUGLOG(5, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize);
+ cSize += cBlockSize;
+ ip += blockSize;
+ op += cBlockSize;
+ remaining -= blockSize;
+ dstCapacity -= cBlockSize;
+ continue;
+ }
+
+ RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block");
+ compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore,
+ &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy,
+ &cctx->appliedParams,
+ op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize,
+ blockSize,
+ cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
+ cctx->bmi2);
+ FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed");
+ DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize);
+
+ if (!cctx->isFirstBlock &&
+ ZSTD_maybeRLE(&cctx->seqStore) &&
+ ZSTD_isRLE(ip, blockSize)) {
+ /* We don't want to emit our first block as a RLE even if it qualifies because
+ * doing so will cause the decoder (cli only) to throw a "should consume all input error."
+ * This is only an issue for zstd <= v1.4.3
+ */
+ compressedSeqsSize = 1;
+ }
+
+ if (compressedSeqsSize == 0) {
+ /* ZSTD_noCompressBlock writes the block header as well */
+ cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed");
+ DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize);
+ } else if (compressedSeqsSize == 1) {
+ cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock);
+ FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed");
+ DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize);
+ } else {
+ U32 cBlockHeader;
+ /* Error checking and repcodes update */
+ ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState);
+ if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+ cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+
+ /* Write block header into beginning of block*/
+ cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3);
+ MEM_writeLE24(op, cBlockHeader);
+ cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize;
+ DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize);
+ }
+
+ cSize += cBlockSize;
+
+ if (lastBlock) {
+ break;
+ } else {
+ ip += blockSize;
+ op += cBlockSize;
+ remaining -= blockSize;
+ dstCapacity -= cBlockSize;
+ cctx->isFirstBlock = 0;
+ }
+ DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity);
+ }
+
+ DEBUGLOG(4, "cSize final total: %zu", cSize);
+ return cSize;
+}
+
+size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+ const void* src, size_t srcSize)
+{
+ BYTE* op = (BYTE*)dst;
+ size_t cSize = 0;
+ size_t compressedBlocksSize = 0;
+ size_t frameHeaderSize = 0;
+
+ /* Transparent initialization stage, same as compressStream2() */
+ DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity);
+ assert(cctx != NULL);
+ FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed");
+ /* Begin writing output, starting with frame header */
+ frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID);
+ op += frameHeaderSize;
+ dstCapacity -= frameHeaderSize;
+ cSize += frameHeaderSize;
+ if (cctx->appliedParams.fParams.checksumFlag && srcSize) {
+ XXH64_update(&cctx->xxhState, src, srcSize);
+ }
+ /* cSize includes block header size and compressed sequences size */
+ compressedBlocksSize = ZSTD_compressSequences_internal(cctx,
+ op, dstCapacity,
+ inSeqs, inSeqsSize,
+ src, srcSize);
+ FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!");
+ cSize += compressedBlocksSize;
+ dstCapacity -= compressedBlocksSize;
+
+ if (cctx->appliedParams.fParams.checksumFlag) {
+ U32 const checksum = (U32) XXH64_digest(&cctx->xxhState);
+ RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum");
+ DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum);
+ MEM_writeLE32((char*)dst + cSize, checksum);
+ cSize += 4;
+ }
+
+ DEBUGLOG(4, "Final compressed size: %zu", cSize);
+ return cSize;
+}
+
+/*====== Finalize ======*/
+
+static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs)
+{
+ const ZSTD_inBuffer nullInput = { NULL, 0, 0 };
+ const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable);
+ return stableInput ? zcs->expectedInBuffer : nullInput;
+}
+
+/*! ZSTD_flushStream() :
+ * @return : amount of data remaining to flush */
+size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
+{
+ ZSTD_inBuffer input = inBuffer_forEndFlush(zcs);
+ input.size = input.pos; /* do not ingest more input during flush */
+ return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush);
+}
+
+
+size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
+{
+ ZSTD_inBuffer input = inBuffer_forEndFlush(zcs);
+ size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end);
+ FORWARD_IF_ERROR(remainingToFlush , "ZSTD_compressStream2(,,ZSTD_e_end) failed");
+ if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */
+ /* single thread mode : attempt to calculate remaining to flush more precisely */
+ { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;
+ size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4);
+ size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize;
+ DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush);
+ return toFlush;
+ }
+}
+
+
+/*-===== Pre-defined compression levels =====-*/
+#include "clevels.h"
+
+int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
+int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; }
+int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; }
+
+static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize)
+{
+ ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict);
+ switch (cParams.strategy) {
+ case ZSTD_fast:
+ case ZSTD_dfast:
+ break;
+ case ZSTD_greedy:
+ case ZSTD_lazy:
+ case ZSTD_lazy2:
+ cParams.hashLog += ZSTD_LAZY_DDSS_BUCKET_LOG;
+ break;
+ case ZSTD_btlazy2:
+ case ZSTD_btopt:
+ case ZSTD_btultra:
+ case ZSTD_btultra2:
+ break;
+ }
+ return cParams;
+}
+
+static int ZSTD_dedicatedDictSearch_isSupported(
+ ZSTD_compressionParameters const* cParams)
+{
+ return (cParams->strategy >= ZSTD_greedy)
+ && (cParams->strategy <= ZSTD_lazy2)
+ && (cParams->hashLog > cParams->chainLog)
+ && (cParams->chainLog <= 24);
+}
+
+/**
+ * Reverses the adjustment applied to cparams when enabling dedicated dict
+ * search. This is used to recover the params set to be used in the working
+ * context. (Otherwise, those tables would also grow.)
+ */
+static void ZSTD_dedicatedDictSearch_revertCParams(
+ ZSTD_compressionParameters* cParams) {
+ switch (cParams->strategy) {
+ case ZSTD_fast:
+ case ZSTD_dfast:
+ break;
+ case ZSTD_greedy:
+ case ZSTD_lazy:
+ case ZSTD_lazy2:
+ cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG;
+ if (cParams->hashLog < ZSTD_HASHLOG_MIN) {
+ cParams->hashLog = ZSTD_HASHLOG_MIN;
+ }
+ break;
+ case ZSTD_btlazy2:
+ case ZSTD_btopt:
+ case ZSTD_btultra:
+ case ZSTD_btultra2:
+ break;
+ }
+}
+
+static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
+{
+ switch (mode) {
+ case ZSTD_cpm_unknown:
+ case ZSTD_cpm_noAttachDict:
+ case ZSTD_cpm_createCDict:
+ break;
+ case ZSTD_cpm_attachDict:
+ dictSize = 0;
+ break;
+ default:
+ assert(0);
+ break;
+ }
+ { int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN;
+ size_t const addedSize = unknown && dictSize > 0 ? 500 : 0;
+ return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize;
+ }
+}
+
+/*! ZSTD_getCParams_internal() :
+ * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize.
+ * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown.
+ * Use dictSize == 0 for unknown or unused.
+ * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */
+static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
+{
+ U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode);
+ U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB);
+ int row;
+ DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel);
+
+ /* row */
+ if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */
+ else if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */
+ else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL;
+ else row = compressionLevel;
+
+ { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row];
+ DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy);
+ /* acceleration factor */
+ if (compressionLevel < 0) {
+ int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel);
+ cp.targetLength = (unsigned)(-clampedCompressionLevel);
+ }
+ /* refine parameters based on srcSize & dictSize */
+ return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto);
+ }
+}
+
+/*! ZSTD_getCParams() :
+ * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize.
+ * Size values are optional, provide 0 if not known or unused */
+ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)
+{
+ if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN;
+ return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
+}
+
+/*! ZSTD_getParams() :
+ * same idea as ZSTD_getCParams()
+ * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`).
+ * Fields of `ZSTD_frameParameters` are set to default values */
+static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) {
+ ZSTD_parameters params;
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode);
+ DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel);
+ ZSTD_memset(&params, 0, sizeof(params));
+ params.cParams = cParams;
+ params.fParams.contentSizeFlag = 1;
+ return params;
+}
+
+/*! ZSTD_getParams() :
+ * same idea as ZSTD_getCParams()
+ * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`).
+ * Fields of `ZSTD_frameParameters` are set to default values */
+ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) {
+ if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN;
+ return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
+}
+
+void ZSTD_registerSequenceProducer(
+ ZSTD_CCtx* zc, void* mState,
+ ZSTD_sequenceProducer_F* mFinder
+) {
+ if (mFinder != NULL) {
+ ZSTD_externalMatchCtx emctx;
+ emctx.mState = mState;
+ emctx.mFinder = mFinder;
+ emctx.seqBuffer = NULL;
+ emctx.seqBufferCapacity = 0;
+ zc->externalMatchCtx = emctx;
+ zc->requestedParams.useSequenceProducer = 1;
+ } else {
+ ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx));
+ zc->requestedParams.useSequenceProducer = 0;
+ }
+}
diff --git a/contrib/zstd/zstd_compress.h b/contrib/zstd/zstd_compress.h
new file mode 100644
index 0000000..94606ed
--- /dev/null
+++ b/contrib/zstd/zstd_compress.h
@@ -0,0 +1,307 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#ifndef ZSTD_COMPRESS_H
+#define ZSTD_COMPRESS_H
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "zstd_internal.h"
+#ifdef ZSTD_MULTITHREAD
+# include "zstdmt_compress.h"
+#endif
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-*************************************
+* Constants
+***************************************/
+static const U32 g_searchStrength = 8;
+#define HASH_READ_SIZE 8
+
+
+/*-*************************************
+* Context memory management
+***************************************/
+typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
+typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
+
+typedef struct ZSTD_prefixDict_s {
+ const void* dict;
+ size_t dictSize;
+ ZSTD_dictMode_e dictMode;
+} ZSTD_prefixDict;
+
+struct ZSTD_CCtx_s {
+ const BYTE* nextSrc; /* next block here to continue on current prefix */
+ const BYTE* base; /* All regular indexes relative to this position */
+ const BYTE* dictBase; /* extDict indexes relative to this position */
+ U32 dictLimit; /* below that point, need extDict */
+ U32 lowLimit; /* below that point, no more data */
+ U32 nextToUpdate; /* index from which to continue dictionary update */
+ U32 nextToUpdate3; /* index from which to continue dictionary update */
+ U32 hashLog3; /* dispatch table : larger == faster, more memory */
+ U32 loadedDictEnd; /* index of end of dictionary */
+ ZSTD_compressionStage_e stage;
+ U32 dictID;
+ ZSTD_CCtx_params requestedParams;
+ ZSTD_CCtx_params appliedParams;
+ void* workSpace;
+ size_t workSpaceSize;
+ size_t blockSize;
+ U64 pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */
+ U64 consumedSrcSize;
+ XXH64_state_t xxhState;
+ ZSTD_customMem customMem;
+ size_t staticSize;
+
+ seqStore_t seqStore; /* sequences storage ptrs */
+ optState_t optState;
+ ldmState_t ldmState; /* long distance matching state */
+ U32* hashTable;
+ U32* hashTable3;
+ U32* chainTable;
+ ZSTD_entropyCTables_t* entropy;
+
+ /* streaming */
+ char* inBuff;
+ size_t inBuffSize;
+ size_t inToCompress;
+ size_t inBuffPos;
+ size_t inBuffTarget;
+ char* outBuff;
+ size_t outBuffSize;
+ size_t outBuffContentSize;
+ size_t outBuffFlushedSize;
+ ZSTD_cStreamStage streamStage;
+ U32 frameEnded;
+
+ /* Dictionary */
+ ZSTD_CDict* cdictLocal;
+ const ZSTD_CDict* cdict;
+ ZSTD_prefixDict prefixDict; /* single-usage dictionary */
+
+ /* Multi-threading */
+#ifdef ZSTD_MULTITHREAD
+ ZSTDMT_CCtx* mtctx;
+#endif
+};
+
+
+static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 16, 17, 17, 18, 18, 19, 19,
+ 20, 20, 20, 20, 21, 21, 21, 21,
+ 22, 22, 22, 22, 22, 22, 22, 22,
+ 23, 23, 23, 23, 23, 23, 23, 23,
+ 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24 };
+
+static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
+ 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
+ 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
+ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
+ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
+
+/*! ZSTD_storeSeq() :
+ Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
+ `offsetCode` : distance to match, or 0 == repCode.
+ `matchCode` : matchLength - MINMATCH
+*/
+MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
+{
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6)
+ static const BYTE* g_start = NULL;
+ U32 const pos = (U32)((const BYTE*)literals - g_start);
+ if (g_start==NULL) g_start = (const BYTE*)literals;
+ if ((pos > 0) && (pos < 1000000000))
+ DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u",
+ pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
+#endif
+ /* copy Literals */
+ assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB);
+ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
+ seqStorePtr->lit += litLength;
+
+ /* literal Length */
+ if (litLength>0xFFFF) {
+ seqStorePtr->longLengthID = 1;
+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ }
+ seqStorePtr->sequences[0].litLength = (U16)litLength;
+
+ /* match offset */
+ seqStorePtr->sequences[0].offset = offsetCode + 1;
+
+ /* match Length */
+ if (matchCode>0xFFFF) {
+ seqStorePtr->longLengthID = 2;
+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ }
+ seqStorePtr->sequences[0].matchLength = (U16)matchCode;
+
+ seqStorePtr->sequences++;
+}
+
+
+/*-*************************************
+* Match length counter
+***************************************/
+static unsigned ZSTD_NbCommonBytes (register size_t val)
+{
+ if (MEM_isLittleEndian()) {
+ if (MEM_64bits()) {
+# if defined(_MSC_VER) && defined(_WIN64)
+ unsigned long r = 0;
+ _BitScanForward64( &r, (U64)val );
+ return (unsigned)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 4)
+ return (__builtin_ctzll((U64)val) >> 3);
+# else
+ static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
+ 0, 3, 1, 3, 1, 4, 2, 7,
+ 0, 2, 3, 6, 1, 5, 3, 5,
+ 1, 3, 4, 4, 2, 5, 6, 7,
+ 7, 0, 1, 2, 3, 3, 4, 6,
+ 2, 6, 5, 5, 3, 4, 5, 6,
+ 7, 1, 2, 4, 6, 4, 4, 5,
+ 7, 2, 6, 5, 7, 6, 7, 7 };
+ return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+# endif
+ } else { /* 32 bits */
+# if defined(_MSC_VER)
+ unsigned long r=0;
+ _BitScanForward( &r, (U32)val );
+ return (unsigned)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 3)
+ return (__builtin_ctz((U32)val) >> 3);
+# else
+ static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
+ 3, 2, 2, 1, 3, 2, 0, 1,
+ 3, 3, 1, 2, 2, 2, 2, 0,
+ 3, 1, 2, 0, 1, 0, 1, 1 };
+ return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+# endif
+ }
+ } else { /* Big Endian CPU */
+ if (MEM_64bits()) {
+# if defined(_MSC_VER) && defined(_WIN64)
+ unsigned long r = 0;
+ _BitScanReverse64( &r, val );
+ return (unsigned)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 4)
+ return (__builtin_clzll(val) >> 3);
+# else
+ unsigned r;
+ const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
+ if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
+ if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
+ r += (!val);
+ return r;
+# endif
+ } else { /* 32 bits */
+# if defined(_MSC_VER)
+ unsigned long r = 0;
+ _BitScanReverse( &r, (unsigned long)val );
+ return (unsigned)(r>>3);
+# elif defined(__GNUC__) && (__GNUC__ >= 3)
+ return (__builtin_clz((U32)val) >> 3);
+# else
+ unsigned r;
+ if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
+ r += (!val);
+ return r;
+# endif
+ } }
+}
+
+
+MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
+{
+ const BYTE* const pStart = pIn;
+ const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
+
+ while (pIn < pInLoopLimit) {
+ size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+ if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
+ pIn += ZSTD_NbCommonBytes(diff);
+ return (size_t)(pIn - pStart);
+ }
+ if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
+ if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
+ if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
+ return (size_t)(pIn - pStart);
+}
+
+/** ZSTD_count_2segments() :
+* can count match length with `ip` & `match` in 2 different segments.
+* convention : on reaching mEnd, match count continue starting from iStart
+*/
+MEM_STATIC size_t ZSTD_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart)
+{
+ const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd);
+ size_t const matchLength = ZSTD_count(ip, match, vEnd);
+ if (match + matchLength != mEnd) return matchLength;
+ return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd);
+}
+
+
+/*-*************************************
+* Hashes
+***************************************/
+static const U32 prime3bytes = 506832829U;
+static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; }
+MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */
+
+static const U32 prime4bytes = 2654435761U;
+static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
+static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); }
+
+static const U64 prime5bytes = 889523592379ULL;
+static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; }
+static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); }
+
+static const U64 prime6bytes = 227718039650203ULL;
+static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
+static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
+
+static const U64 prime7bytes = 58295818150454627ULL;
+static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; }
+static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); }
+
+static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
+static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
+static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
+
+MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
+{
+ switch(mls)
+ {
+ default:
+ case 4: return ZSTD_hash4Ptr(p, hBits);
+ case 5: return ZSTD_hash5Ptr(p, hBits);
+ case 6: return ZSTD_hash6Ptr(p, hBits);
+ case 7: return ZSTD_hash7Ptr(p, hBits);
+ case 8: return ZSTD_hash8Ptr(p, hBits);
+ }
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_COMPRESS_H */
diff --git a/contrib/zstd/zstd_compress_internal.h b/contrib/zstd/zstd_compress_internal.h
new file mode 100644
index 0000000..e53b3a5
--- /dev/null
+++ b/contrib/zstd/zstd_compress_internal.h
@@ -0,0 +1,1478 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* This header contains definitions
+ * that shall **only** be used by modules within lib/compress.
+ */
+
+#ifndef ZSTD_COMPRESS_H
+#define ZSTD_COMPRESS_H
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "zstd_internal.h"
+#include "zstd_cwksp.h"
+#ifdef ZSTD_MULTITHREAD
+# include "zstdmt_compress.h"
+#endif
+#include "bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-*************************************
+* Constants
+***************************************/
+#define kSearchStrength 8
+#define HASH_READ_SIZE 8
+#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted".
+ It could be confused for a real successor at index "1", if sorted as larger than its predecessor.
+ It's not a big deal though : candidate will just be sorted again.
+ Additionally, candidate position 1 will be lost.
+ But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss.
+ The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy.
+ This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */
+
+
+/*-*************************************
+* Context memory management
+***************************************/
+typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
+typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
+
+typedef struct ZSTD_prefixDict_s {
+ const void* dict;
+ size_t dictSize;
+ ZSTD_dictContentType_e dictContentType;
+} ZSTD_prefixDict;
+
+typedef struct {
+ void* dictBuffer;
+ void const* dict;
+ size_t dictSize;
+ ZSTD_dictContentType_e dictContentType;
+ ZSTD_CDict* cdict;
+} ZSTD_localDict;
+
+typedef struct {
+ HUF_CElt CTable[HUF_CTABLE_SIZE_ST(255)];
+ HUF_repeat repeatMode;
+} ZSTD_hufCTables_t;
+
+typedef struct {
+ FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
+ FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
+ FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
+ FSE_repeat offcode_repeatMode;
+ FSE_repeat matchlength_repeatMode;
+ FSE_repeat litlength_repeatMode;
+} ZSTD_fseCTables_t;
+
+typedef struct {
+ ZSTD_hufCTables_t huf;
+ ZSTD_fseCTables_t fse;
+} ZSTD_entropyCTables_t;
+
+/***********************************************
+* Entropy buffer statistics structs and funcs *
+***********************************************/
+/** ZSTD_hufCTablesMetadata_t :
+ * Stores Literals Block Type for a super-block in hType, and
+ * huffman tree description in hufDesBuffer.
+ * hufDesSize refers to the size of huffman tree description in bytes.
+ * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */
+typedef struct {
+ symbolEncodingType_e hType;
+ BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE];
+ size_t hufDesSize;
+} ZSTD_hufCTablesMetadata_t;
+
+/** ZSTD_fseCTablesMetadata_t :
+ * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
+ * fse tables in fseTablesBuffer.
+ * fseTablesSize refers to the size of fse tables in bytes.
+ * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */
+typedef struct {
+ symbolEncodingType_e llType;
+ symbolEncodingType_e ofType;
+ symbolEncodingType_e mlType;
+ BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE];
+ size_t fseTablesSize;
+ size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
+} ZSTD_fseCTablesMetadata_t;
+
+typedef struct {
+ ZSTD_hufCTablesMetadata_t hufMetadata;
+ ZSTD_fseCTablesMetadata_t fseMetadata;
+} ZSTD_entropyCTablesMetadata_t;
+
+/** ZSTD_buildBlockEntropyStats() :
+ * Builds entropy for the block.
+ * @return : 0 on success or error code */
+size_t ZSTD_buildBlockEntropyStats(
+ const seqStore_t* seqStorePtr,
+ const ZSTD_entropyCTables_t* prevEntropy,
+ ZSTD_entropyCTables_t* nextEntropy,
+ const ZSTD_CCtx_params* cctxParams,
+ ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ void* workspace, size_t wkspSize);
+
+/*********************************
+* Compression internals structs *
+*********************************/
+
+typedef struct {
+ U32 off; /* Offset sumtype code for the match, using ZSTD_storeSeq() format */
+ U32 len; /* Raw length of match */
+} ZSTD_match_t;
+
+typedef struct {
+ U32 offset; /* Offset of sequence */
+ U32 litLength; /* Length of literals prior to match */
+ U32 matchLength; /* Raw length of match */
+} rawSeq;
+
+typedef struct {
+ rawSeq* seq; /* The start of the sequences */
+ size_t pos; /* The index in seq where reading stopped. pos <= size. */
+ size_t posInSequence; /* The position within the sequence at seq[pos] where reading
+ stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */
+ size_t size; /* The number of sequences. <= capacity. */
+ size_t capacity; /* The capacity starting from `seq` pointer */
+} rawSeqStore_t;
+
+typedef struct {
+ U32 idx; /* Index in array of ZSTD_Sequence */
+ U32 posInSequence; /* Position within sequence at idx */
+ size_t posInSrc; /* Number of bytes given by sequences provided so far */
+} ZSTD_sequencePosition;
+
+UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
+
+typedef struct {
+ int price;
+ U32 off;
+ U32 mlen;
+ U32 litlen;
+ U32 rep[ZSTD_REP_NUM];
+} ZSTD_optimal_t;
+
+typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e;
+
+typedef struct {
+ /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */
+ unsigned* litFreq; /* table of literals statistics, of size 256 */
+ unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */
+ unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */
+ unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */
+ ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */
+ ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */
+
+ U32 litSum; /* nb of literals */
+ U32 litLengthSum; /* nb of litLength codes */
+ U32 matchLengthSum; /* nb of matchLength codes */
+ U32 offCodeSum; /* nb of offset codes */
+ U32 litSumBasePrice; /* to compare to log2(litfreq) */
+ U32 litLengthSumBasePrice; /* to compare to log2(llfreq) */
+ U32 matchLengthSumBasePrice;/* to compare to log2(mlfreq) */
+ U32 offCodeSumBasePrice; /* to compare to log2(offreq) */
+ ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */
+ const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */
+ ZSTD_paramSwitch_e literalCompressionMode;
+} optState_t;
+
+typedef struct {
+ ZSTD_entropyCTables_t entropy;
+ U32 rep[ZSTD_REP_NUM];
+} ZSTD_compressedBlockState_t;
+
+typedef struct {
+ BYTE const* nextSrc; /* next block here to continue on current prefix */
+ BYTE const* base; /* All regular indexes relative to this position */
+ BYTE const* dictBase; /* extDict indexes relative to this position */
+ U32 dictLimit; /* below that point, need extDict */
+ U32 lowLimit; /* below that point, no more valid data */
+ U32 nbOverflowCorrections; /* Number of times overflow correction has run since
+ * ZSTD_window_init(). Useful for debugging coredumps
+ * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY.
+ */
+} ZSTD_window_t;
+
+#define ZSTD_WINDOW_START_INDEX 2
+
+typedef struct ZSTD_matchState_t ZSTD_matchState_t;
+
+#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */
+
+struct ZSTD_matchState_t {
+ ZSTD_window_t window; /* State for window round buffer management */
+ U32 loadedDictEnd; /* index of end of dictionary, within context's referential.
+ * When loadedDictEnd != 0, a dictionary is in use, and still valid.
+ * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance.
+ * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity().
+ * When dict referential is copied into active context (i.e. not attached),
+ * loadedDictEnd == dictSize, since referential starts from zero.
+ */
+ U32 nextToUpdate; /* index from which to continue table update */
+ U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */
+
+ U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/
+ U16* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */
+ U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */
+
+ U32* hashTable;
+ U32* hashTable3;
+ U32* chainTable;
+
+ U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */
+
+ int dedicatedDictSearch; /* Indicates whether this matchState is using the
+ * dedicated dictionary search structure.
+ */
+ optState_t opt; /* optimal parser state */
+ const ZSTD_matchState_t* dictMatchState;
+ ZSTD_compressionParameters cParams;
+ const rawSeqStore_t* ldmSeqStore;
+
+ /* Controls prefetching in some dictMatchState matchfinders.
+ * This behavior is controlled from the cctx ms.
+ * This parameter has no effect in the cdict ms. */
+ int prefetchCDictTables;
+};
+
+typedef struct {
+ ZSTD_compressedBlockState_t* prevCBlock;
+ ZSTD_compressedBlockState_t* nextCBlock;
+ ZSTD_matchState_t matchState;
+} ZSTD_blockState_t;
+
+typedef struct {
+ U32 offset;
+ U32 checksum;
+} ldmEntry_t;
+
+typedef struct {
+ BYTE const* split;
+ U32 hash;
+ U32 checksum;
+ ldmEntry_t* bucket;
+} ldmMatchCandidate_t;
+
+#define LDM_BATCH_SIZE 64
+
+typedef struct {
+ ZSTD_window_t window; /* State for the window round buffer management */
+ ldmEntry_t* hashTable;
+ U32 loadedDictEnd;
+ BYTE* bucketOffsets; /* Next position in bucket to insert entry */
+ size_t splitIndices[LDM_BATCH_SIZE];
+ ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE];
+} ldmState_t;
+
+typedef struct {
+ ZSTD_paramSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */
+ U32 hashLog; /* Log size of hashTable */
+ U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
+ U32 minMatchLength; /* Minimum match length */
+ U32 hashRateLog; /* Log number of entries to skip */
+ U32 windowLog; /* Window log for the LDM */
+} ldmParams_t;
+
+typedef struct {
+ int collectSequences;
+ ZSTD_Sequence* seqStart;
+ size_t seqIndex;
+ size_t maxSequences;
+} SeqCollector;
+
+struct ZSTD_CCtx_params_s {
+ ZSTD_format_e format;
+ ZSTD_compressionParameters cParams;
+ ZSTD_frameParameters fParams;
+
+ int compressionLevel;
+ int forceWindow; /* force back-references to respect limit of
+ * 1<<wLog, even for dictionary */
+ size_t targetCBlockSize; /* Tries to fit compressed block size to be around targetCBlockSize.
+ * No target when targetCBlockSize == 0.
+ * There is no guarantee on compressed block size */
+ int srcSizeHint; /* User's best guess of source size.
+ * Hint is not valid when srcSizeHint == 0.
+ * There is no guarantee that hint is close to actual source size */
+
+ ZSTD_dictAttachPref_e attachDictPref;
+ ZSTD_paramSwitch_e literalCompressionMode;
+
+ /* Multithreading: used to pass parameters to mtctx */
+ int nbWorkers;
+ size_t jobSize;
+ int overlapLog;
+ int rsyncable;
+
+ /* Long distance matching parameters */
+ ldmParams_t ldmParams;
+
+ /* Dedicated dict search algorithm trigger */
+ int enableDedicatedDictSearch;
+
+ /* Input/output buffer modes */
+ ZSTD_bufferMode_e inBufferMode;
+ ZSTD_bufferMode_e outBufferMode;
+
+ /* Sequence compression API */
+ ZSTD_sequenceFormat_e blockDelimiters;
+ int validateSequences;
+
+ /* Block splitting */
+ ZSTD_paramSwitch_e useBlockSplitter;
+
+ /* Param for deciding whether to use row-based matchfinder */
+ ZSTD_paramSwitch_e useRowMatchFinder;
+
+ /* Always load a dictionary in ext-dict mode (not prefix mode)? */
+ int deterministicRefPrefix;
+
+ /* Internal use, for createCCtxParams() and freeCCtxParams() only */
+ ZSTD_customMem customMem;
+
+ /* Controls prefetching in some dictMatchState matchfinders */
+ ZSTD_paramSwitch_e prefetchCDictTables;
+
+ /* Controls whether zstd will fall back to an internal matchfinder
+ * if the external matchfinder returns an error code. */
+ int enableMatchFinderFallback;
+
+ /* Indicates whether an external matchfinder has been referenced.
+ * Users can't set this externally.
+ * It is set internally in ZSTD_registerSequenceProducer(). */
+ int useSequenceProducer;
+
+ /* Adjust the max block size*/
+ size_t maxBlockSize;
+
+ /* Controls repcode search in external sequence parsing */
+ ZSTD_paramSwitch_e searchForExternalRepcodes;
+}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
+
+#define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
+#define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE)
+
+/**
+ * Indicates whether this compression proceeds directly from user-provided
+ * source buffer to user-provided destination buffer (ZSTDb_not_buffered), or
+ * whether the context needs to buffer the input/output (ZSTDb_buffered).
+ */
+typedef enum {
+ ZSTDb_not_buffered,
+ ZSTDb_buffered
+} ZSTD_buffered_policy_e;
+
+/**
+ * Struct that contains all elements of block splitter that should be allocated
+ * in a wksp.
+ */
+#define ZSTD_MAX_NB_BLOCK_SPLITS 196
+typedef struct {
+ seqStore_t fullSeqStoreChunk;
+ seqStore_t firstHalfSeqStore;
+ seqStore_t secondHalfSeqStore;
+ seqStore_t currSeqStore;
+ seqStore_t nextSeqStore;
+
+ U32 partitions[ZSTD_MAX_NB_BLOCK_SPLITS];
+ ZSTD_entropyCTablesMetadata_t entropyMetadata;
+} ZSTD_blockSplitCtx;
+
+/* Context for block-level external matchfinder API */
+typedef struct {
+ void* mState;
+ ZSTD_sequenceProducer_F* mFinder;
+ ZSTD_Sequence* seqBuffer;
+ size_t seqBufferCapacity;
+} ZSTD_externalMatchCtx;
+
+struct ZSTD_CCtx_s {
+ ZSTD_compressionStage_e stage;
+ int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
+ int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
+ ZSTD_CCtx_params requestedParams;
+ ZSTD_CCtx_params appliedParams;
+ ZSTD_CCtx_params simpleApiParams; /* Param storage used by the simple API - not sticky. Must only be used in top-level simple API functions for storage. */
+ U32 dictID;
+ size_t dictContentSize;
+
+ ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */
+ size_t blockSize;
+ unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */
+ unsigned long long consumedSrcSize;
+ unsigned long long producedCSize;
+ XXH64_state_t xxhState;
+ ZSTD_customMem customMem;
+ ZSTD_threadPool* pool;
+ size_t staticSize;
+ SeqCollector seqCollector;
+ int isFirstBlock;
+ int initialized;
+
+ seqStore_t seqStore; /* sequences storage ptrs */
+ ldmState_t ldmState; /* long distance matching state */
+ rawSeq* ldmSequences; /* Storage for the ldm output sequences */
+ size_t maxNbLdmSequences;
+ rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */
+ ZSTD_blockState_t blockState;
+ U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */
+
+ /* Whether we are streaming or not */
+ ZSTD_buffered_policy_e bufferedPolicy;
+
+ /* streaming */
+ char* inBuff;
+ size_t inBuffSize;
+ size_t inToCompress;
+ size_t inBuffPos;
+ size_t inBuffTarget;
+ char* outBuff;
+ size_t outBuffSize;
+ size_t outBuffContentSize;
+ size_t outBuffFlushedSize;
+ ZSTD_cStreamStage streamStage;
+ U32 frameEnded;
+
+ /* Stable in/out buffer verification */
+ ZSTD_inBuffer expectedInBuffer;
+ size_t stableIn_notConsumed; /* nb bytes within stable input buffer that are said to be consumed but are not */
+ size_t expectedOutBufferSize;
+
+ /* Dictionary */
+ ZSTD_localDict localDict;
+ const ZSTD_CDict* cdict;
+ ZSTD_prefixDict prefixDict; /* single-usage dictionary */
+
+ /* Multi-threading */
+#ifdef ZSTD_MULTITHREAD
+ ZSTDMT_CCtx* mtctx;
+#endif
+
+ /* Tracing */
+#if ZSTD_TRACE
+ ZSTD_TraceCtx traceCtx;
+#endif
+
+ /* Workspace for block splitter */
+ ZSTD_blockSplitCtx blockSplitCtx;
+
+ /* Workspace for external matchfinder */
+ ZSTD_externalMatchCtx externalMatchCtx;
+};
+
+typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
+typedef enum { ZSTD_tfp_forCCtx, ZSTD_tfp_forCDict } ZSTD_tableFillPurpose_e;
+
+typedef enum {
+ ZSTD_noDict = 0,
+ ZSTD_extDict = 1,
+ ZSTD_dictMatchState = 2,
+ ZSTD_dedicatedDictSearch = 3
+} ZSTD_dictMode_e;
+
+typedef enum {
+ ZSTD_cpm_noAttachDict = 0, /* Compression with ZSTD_noDict or ZSTD_extDict.
+ * In this mode we use both the srcSize and the dictSize
+ * when selecting and adjusting parameters.
+ */
+ ZSTD_cpm_attachDict = 1, /* Compression with ZSTD_dictMatchState or ZSTD_dedicatedDictSearch.
+ * In this mode we only take the srcSize into account when selecting
+ * and adjusting parameters.
+ */
+ ZSTD_cpm_createCDict = 2, /* Creating a CDict.
+ * In this mode we take both the source size and the dictionary size
+ * into account when selecting and adjusting the parameters.
+ */
+ ZSTD_cpm_unknown = 3 /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
+ * We don't know what these parameters are for. We default to the legacy
+ * behavior of taking both the source size and the dict size into account
+ * when selecting and adjusting parameters.
+ */
+} ZSTD_cParamMode_e;
+
+typedef size_t (*ZSTD_blockCompressor) (
+ ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
+
+
+MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
+{
+ static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 16, 17, 17, 18, 18, 19, 19,
+ 20, 20, 20, 20, 21, 21, 21, 21,
+ 22, 22, 22, 22, 22, 22, 22, 22,
+ 23, 23, 23, 23, 23, 23, 23, 23,
+ 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24 };
+ static const U32 LL_deltaCode = 19;
+ return (litLength > 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
+}
+
+/* ZSTD_MLcode() :
+ * note : mlBase = matchLength - MINMATCH;
+ * because it's the format it's stored in seqStore->sequences */
+MEM_STATIC U32 ZSTD_MLcode(U32 mlBase)
+{
+ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
+ 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
+ 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
+ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
+ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
+ static const U32 ML_deltaCode = 36;
+ return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
+}
+
+/* ZSTD_cParam_withinBounds:
+ * @return 1 if value is within cParam bounds,
+ * 0 otherwise */
+MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
+{
+ ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);
+ if (ZSTD_isError(bounds.error)) return 0;
+ if (value < bounds.lowerBound) return 0;
+ if (value > bounds.upperBound) return 0;
+ return 1;
+}
+
+/* ZSTD_noCompressBlock() :
+ * Writes uncompressed block to dst buffer from given src.
+ * Returns the size of the block */
+MEM_STATIC size_t
+ZSTD_noCompressBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock)
+{
+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3);
+ DEBUGLOG(5, "ZSTD_noCompressBlock (srcSize=%zu, dstCapacity=%zu)", srcSize, dstCapacity);
+ RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity,
+ dstSize_tooSmall, "dst buf too small for uncompressed block");
+ MEM_writeLE24(dst, cBlockHeader24);
+ ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
+ return ZSTD_blockHeaderSize + srcSize;
+}
+
+MEM_STATIC size_t
+ZSTD_rleCompressBlock(void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock)
+{
+ BYTE* const op = (BYTE*)dst;
+ U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3);
+ RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, "");
+ MEM_writeLE24(op, cBlockHeader);
+ op[3] = src;
+ return 4;
+}
+
+
+/* ZSTD_minGain() :
+ * minimum compression required
+ * to generate a compress block or a compressed literals section.
+ * note : use same formula for both situations */
+MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
+{
+ U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6;
+ ZSTD_STATIC_ASSERT(ZSTD_btultra == 8);
+ assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat));
+ return (srcSize >> minlog) + 2;
+}
+
+MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams)
+{
+ switch (cctxParams->literalCompressionMode) {
+ case ZSTD_ps_enable:
+ return 0;
+ case ZSTD_ps_disable:
+ return 1;
+ default:
+ assert(0 /* impossible: pre-validated */);
+ ZSTD_FALLTHROUGH;
+ case ZSTD_ps_auto:
+ return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
+ }
+}
+
+/*! ZSTD_safecopyLiterals() :
+ * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w.
+ * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single
+ * large copies.
+ */
+static void
+ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w)
+{
+ assert(iend > ilimit_w);
+ if (ip <= ilimit_w) {
+ ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap);
+ op += ilimit_w - ip;
+ ip = ilimit_w;
+ }
+ while (ip < iend) *op++ = *ip++;
+}
+
+
+#define REPCODE1_TO_OFFBASE REPCODE_TO_OFFBASE(1)
+#define REPCODE2_TO_OFFBASE REPCODE_TO_OFFBASE(2)
+#define REPCODE3_TO_OFFBASE REPCODE_TO_OFFBASE(3)
+#define REPCODE_TO_OFFBASE(r) (assert((r)>=1), assert((r)<=ZSTD_REP_NUM), (r)) /* accepts IDs 1,2,3 */
+#define OFFSET_TO_OFFBASE(o) (assert((o)>0), o + ZSTD_REP_NUM)
+#define OFFBASE_IS_OFFSET(o) ((o) > ZSTD_REP_NUM)
+#define OFFBASE_IS_REPCODE(o) ( 1 <= (o) && (o) <= ZSTD_REP_NUM)
+#define OFFBASE_TO_OFFSET(o) (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM)
+#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o)) /* returns ID 1,2,3 */
+
+/*! ZSTD_storeSeq() :
+ * Store a sequence (litlen, litPtr, offBase and matchLength) into seqStore_t.
+ * @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE().
+ * @matchLength : must be >= MINMATCH
+ * Allowed to over-read literals up to litLimit.
+*/
+HINT_INLINE UNUSED_ATTR void
+ZSTD_storeSeq(seqStore_t* seqStorePtr,
+ size_t litLength, const BYTE* literals, const BYTE* litLimit,
+ U32 offBase,
+ size_t matchLength)
+{
+ BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH;
+ BYTE const* const litEnd = literals + litLength;
+#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
+ static const BYTE* g_start = NULL;
+ if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
+ { U32 const pos = (U32)((const BYTE*)literals - g_start);
+ DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u",
+ pos, (U32)litLength, (U32)matchLength, (U32)offBase);
+ }
+#endif
+ assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq);
+ /* copy Literals */
+ assert(seqStorePtr->maxNbLit <= 128 KB);
+ assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit);
+ assert(literals + litLength <= litLimit);
+ if (litEnd <= litLimit_w) {
+ /* Common case we can use wildcopy.
+ * First copy 16 bytes, because literals are likely short.
+ */
+ ZSTD_STATIC_ASSERT(WILDCOPY_OVERLENGTH >= 16);
+ ZSTD_copy16(seqStorePtr->lit, literals);
+ if (litLength > 16) {
+ ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap);
+ }
+ } else {
+ ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w);
+ }
+ seqStorePtr->lit += litLength;
+
+ /* literal Length */
+ if (litLength>0xFFFF) {
+ assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
+ seqStorePtr->longLengthType = ZSTD_llt_literalLength;
+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ }
+ seqStorePtr->sequences[0].litLength = (U16)litLength;
+
+ /* match offset */
+ seqStorePtr->sequences[0].offBase = offBase;
+
+ /* match Length */
+ assert(matchLength >= MINMATCH);
+ { size_t const mlBase = matchLength - MINMATCH;
+ if (mlBase>0xFFFF) {
+ assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
+ seqStorePtr->longLengthType = ZSTD_llt_matchLength;
+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ }
+ seqStorePtr->sequences[0].mlBase = (U16)mlBase;
+ }
+
+ seqStorePtr->sequences++;
+}
+
+/* ZSTD_updateRep() :
+ * updates in-place @rep (array of repeat offsets)
+ * @offBase : sum-type, using numeric representation of ZSTD_storeSeq()
+ */
+MEM_STATIC void
+ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0)
+{
+ if (OFFBASE_IS_OFFSET(offBase)) { /* full offset */
+ rep[2] = rep[1];
+ rep[1] = rep[0];
+ rep[0] = OFFBASE_TO_OFFSET(offBase);
+ } else { /* repcode */
+ U32 const repCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0;
+ if (repCode > 0) { /* note : if repCode==0, no change */
+ U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
+ rep[2] = (repCode >= 2) ? rep[1] : rep[2];
+ rep[1] = rep[0];
+ rep[0] = currentOffset;
+ } else { /* repCode == 0 */
+ /* nothing to do */
+ }
+ }
+}
+
+typedef struct repcodes_s {
+ U32 rep[3];
+} repcodes_t;
+
+MEM_STATIC repcodes_t
+ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0)
+{
+ repcodes_t newReps;
+ ZSTD_memcpy(&newReps, rep, sizeof(newReps));
+ ZSTD_updateRep(newReps.rep, offBase, ll0);
+ return newReps;
+}
+
+
+/*-*************************************
+* Match length counter
+***************************************/
+MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
+{
+ const BYTE* const pStart = pIn;
+ const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
+
+ if (pIn < pInLoopLimit) {
+ { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+ if (diff) return ZSTD_NbCommonBytes(diff); }
+ pIn+=sizeof(size_t); pMatch+=sizeof(size_t);
+ while (pIn < pInLoopLimit) {
+ size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+ if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
+ pIn += ZSTD_NbCommonBytes(diff);
+ return (size_t)(pIn - pStart);
+ } }
+ if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
+ if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
+ if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
+ return (size_t)(pIn - pStart);
+}
+
+/** ZSTD_count_2segments() :
+ * can count match length with `ip` & `match` in 2 different segments.
+ * convention : on reaching mEnd, match count continue starting from iStart
+ */
+MEM_STATIC size_t
+ZSTD_count_2segments(const BYTE* ip, const BYTE* match,
+ const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart)
+{
+ const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd);
+ size_t const matchLength = ZSTD_count(ip, match, vEnd);
+ if (match + matchLength != mEnd) return matchLength;
+ DEBUGLOG(7, "ZSTD_count_2segments: found a 2-parts match (current length==%zu)", matchLength);
+ DEBUGLOG(7, "distance from match beginning to end dictionary = %zi", mEnd - match);
+ DEBUGLOG(7, "distance from current pos to end buffer = %zi", iEnd - ip);
+ DEBUGLOG(7, "next byte : ip==%02X, istart==%02X", ip[matchLength], *iStart);
+ DEBUGLOG(7, "final match length = %zu", matchLength + ZSTD_count(ip+matchLength, iStart, iEnd));
+ return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd);
+}
+
+
+/*-*************************************
+ * Hashes
+ ***************************************/
+static const U32 prime3bytes = 506832829U;
+static U32 ZSTD_hash3(U32 u, U32 h) { assert(h <= 32); return ((u << (32-24)) * prime3bytes) >> (32-h) ; }
+MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */
+
+static const U32 prime4bytes = 2654435761U;
+static U32 ZSTD_hash4(U32 u, U32 h) { assert(h <= 32); return (u * prime4bytes) >> (32-h) ; }
+static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_readLE32(ptr), h); }
+
+static const U64 prime5bytes = 889523592379ULL;
+static size_t ZSTD_hash5(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; }
+static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); }
+
+static const U64 prime6bytes = 227718039650203ULL;
+static size_t ZSTD_hash6(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
+static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
+
+static const U64 prime7bytes = 58295818150454627ULL;
+static size_t ZSTD_hash7(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; }
+static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); }
+
+static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
+static size_t ZSTD_hash8(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
+static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
+
+MEM_STATIC FORCE_INLINE_ATTR
+size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
+{
+ /* Although some of these hashes do support hBits up to 64, some do not.
+ * To be on the safe side, always avoid hBits > 32. */
+ assert(hBits <= 32);
+
+ switch(mls)
+ {
+ default:
+ case 4: return ZSTD_hash4Ptr(p, hBits);
+ case 5: return ZSTD_hash5Ptr(p, hBits);
+ case 6: return ZSTD_hash6Ptr(p, hBits);
+ case 7: return ZSTD_hash7Ptr(p, hBits);
+ case 8: return ZSTD_hash8Ptr(p, hBits);
+ }
+}
+
+/** ZSTD_ipow() :
+ * Return base^exponent.
+ */
+static U64 ZSTD_ipow(U64 base, U64 exponent)
+{
+ U64 power = 1;
+ while (exponent) {
+ if (exponent & 1) power *= base;
+ exponent >>= 1;
+ base *= base;
+ }
+ return power;
+}
+
+#define ZSTD_ROLL_HASH_CHAR_OFFSET 10
+
+/** ZSTD_rollingHash_append() :
+ * Add the buffer to the hash value.
+ */
+static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size)
+{
+ BYTE const* istart = (BYTE const*)buf;
+ size_t pos;
+ for (pos = 0; pos < size; ++pos) {
+ hash *= prime8bytes;
+ hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET;
+ }
+ return hash;
+}
+
+/** ZSTD_rollingHash_compute() :
+ * Compute the rolling hash value of the buffer.
+ */
+MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size)
+{
+ return ZSTD_rollingHash_append(0, buf, size);
+}
+
+/** ZSTD_rollingHash_primePower() :
+ * Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash
+ * over a window of length bytes.
+ */
+MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length)
+{
+ return ZSTD_ipow(prime8bytes, length - 1);
+}
+
+/** ZSTD_rollingHash_rotate() :
+ * Rotate the rolling hash by one byte.
+ */
+MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower)
+{
+ hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower;
+ hash *= prime8bytes;
+ hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET;
+ return hash;
+}
+
+/*-*************************************
+* Round buffer management
+***************************************/
+#if (ZSTD_WINDOWLOG_MAX_64 > 31)
+# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX"
+#endif
+/* Max current allowed */
+#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX))
+/* Maximum chunk size before overflow correction needs to be called again */
+#define ZSTD_CHUNKSIZE_MAX \
+ ( ((U32)-1) /* Maximum ending current index */ \
+ - ZSTD_CURRENT_MAX) /* Maximum beginning lowLimit */
+
+/**
+ * ZSTD_window_clear():
+ * Clears the window containing the history by simply setting it to empty.
+ */
+MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window)
+{
+ size_t const endT = (size_t)(window->nextSrc - window->base);
+ U32 const end = (U32)endT;
+
+ window->lowLimit = end;
+ window->dictLimit = end;
+}
+
+MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window)
+{
+ return window.dictLimit == ZSTD_WINDOW_START_INDEX &&
+ window.lowLimit == ZSTD_WINDOW_START_INDEX &&
+ (window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX;
+}
+
+/**
+ * ZSTD_window_hasExtDict():
+ * Returns non-zero if the window has a non-empty extDict.
+ */
+MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window)
+{
+ return window.lowLimit < window.dictLimit;
+}
+
+/**
+ * ZSTD_matchState_dictMode():
+ * Inspects the provided matchState and figures out what dictMode should be
+ * passed to the compressor.
+ */
+MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms)
+{
+ return ZSTD_window_hasExtDict(ms->window) ?
+ ZSTD_extDict :
+ ms->dictMatchState != NULL ?
+ (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) :
+ ZSTD_noDict;
+}
+
+/* Defining this macro to non-zero tells zstd to run the overflow correction
+ * code much more frequently. This is very inefficient, and should only be
+ * used for tests and fuzzers.
+ */
+#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY
+# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1
+# else
+# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0
+# endif
+#endif
+
+/**
+ * ZSTD_window_canOverflowCorrect():
+ * Returns non-zero if the indices are large enough for overflow correction
+ * to work correctly without impacting compression ratio.
+ */
+MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window,
+ U32 cycleLog,
+ U32 maxDist,
+ U32 loadedDictEnd,
+ void const* src)
+{
+ U32 const cycleSize = 1u << cycleLog;
+ U32 const curr = (U32)((BYTE const*)src - window.base);
+ U32 const minIndexToOverflowCorrect = cycleSize
+ + MAX(maxDist, cycleSize)
+ + ZSTD_WINDOW_START_INDEX;
+
+ /* Adjust the min index to backoff the overflow correction frequency,
+ * so we don't waste too much CPU in overflow correction. If this
+ * computation overflows we don't really care, we just need to make
+ * sure it is at least minIndexToOverflowCorrect.
+ */
+ U32 const adjustment = window.nbOverflowCorrections + 1;
+ U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment,
+ minIndexToOverflowCorrect);
+ U32 const indexLargeEnough = curr > adjustedIndex;
+
+ /* Only overflow correct early if the dictionary is invalidated already,
+ * so we don't hurt compression ratio.
+ */
+ U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd;
+
+ return indexLargeEnough && dictionaryInvalidated;
+}
+
+/**
+ * ZSTD_window_needOverflowCorrection():
+ * Returns non-zero if the indices are getting too large and need overflow
+ * protection.
+ */
+MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
+ U32 cycleLog,
+ U32 maxDist,
+ U32 loadedDictEnd,
+ void const* src,
+ void const* srcEnd)
+{
+ U32 const curr = (U32)((BYTE const*)srcEnd - window.base);
+ if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
+ if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) {
+ return 1;
+ }
+ }
+ return curr > ZSTD_CURRENT_MAX;
+}
+
+/**
+ * ZSTD_window_correctOverflow():
+ * Reduces the indices to protect from index overflow.
+ * Returns the correction made to the indices, which must be applied to every
+ * stored index.
+ *
+ * The least significant cycleLog bits of the indices must remain the same,
+ * which may be 0. Every index up to maxDist in the past must be valid.
+ */
+MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
+ U32 maxDist, void const* src)
+{
+ /* preemptive overflow correction:
+ * 1. correction is large enough:
+ * lowLimit > (3<<29) ==> current > 3<<29 + 1<<windowLog
+ * 1<<windowLog <= newCurrent < 1<<chainLog + 1<<windowLog
+ *
+ * current - newCurrent
+ * > (3<<29 + 1<<windowLog) - (1<<windowLog + 1<<chainLog)
+ * > (3<<29) - (1<<chainLog)
+ * > (3<<29) - (1<<30) (NOTE: chainLog <= 30)
+ * > 1<<29
+ *
+ * 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow:
+ * After correction, current is less than (1<<chainLog + 1<<windowLog).
+ * In 64-bit mode we are safe, because we have 64-bit ptrdiff_t.
+ * In 32-bit mode we are safe, because (chainLog <= 29), so
+ * ip+ZSTD_CHUNKSIZE_MAX - cctx->base < 1<<32.
+ * 3. (cctx->lowLimit + 1<<windowLog) < 1<<32:
+ * windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
+ */
+ U32 const cycleSize = 1u << cycleLog;
+ U32 const cycleMask = cycleSize - 1;
+ U32 const curr = (U32)((BYTE const*)src - window->base);
+ U32 const currentCycle = curr & cycleMask;
+ /* Ensure newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX. */
+ U32 const currentCycleCorrection = currentCycle < ZSTD_WINDOW_START_INDEX
+ ? MAX(cycleSize, ZSTD_WINDOW_START_INDEX)
+ : 0;
+ U32 const newCurrent = currentCycle
+ + currentCycleCorrection
+ + MAX(maxDist, cycleSize);
+ U32 const correction = curr - newCurrent;
+ /* maxDist must be a power of two so that:
+ * (newCurrent & cycleMask) == (curr & cycleMask)
+ * This is required to not corrupt the chains / binary tree.
+ */
+ assert((maxDist & (maxDist - 1)) == 0);
+ assert((curr & cycleMask) == (newCurrent & cycleMask));
+ assert(curr > newCurrent);
+ if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
+ /* Loose bound, should be around 1<<29 (see above) */
+ assert(correction > 1<<28);
+ }
+
+ window->base += correction;
+ window->dictBase += correction;
+ if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) {
+ window->lowLimit = ZSTD_WINDOW_START_INDEX;
+ } else {
+ window->lowLimit -= correction;
+ }
+ if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) {
+ window->dictLimit = ZSTD_WINDOW_START_INDEX;
+ } else {
+ window->dictLimit -= correction;
+ }
+
+ /* Ensure we can still reference the full window. */
+ assert(newCurrent >= maxDist);
+ assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX);
+ /* Ensure that lowLimit and dictLimit didn't underflow. */
+ assert(window->lowLimit <= newCurrent);
+ assert(window->dictLimit <= newCurrent);
+
+ ++window->nbOverflowCorrections;
+
+ DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction,
+ window->lowLimit);
+ return correction;
+}
+
+/**
+ * ZSTD_window_enforceMaxDist():
+ * Updates lowLimit so that:
+ * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd
+ *
+ * It ensures index is valid as long as index >= lowLimit.
+ * This must be called before a block compression call.
+ *
+ * loadedDictEnd is only defined if a dictionary is in use for current compression.
+ * As the name implies, loadedDictEnd represents the index at end of dictionary.
+ * The value lies within context's referential, it can be directly compared to blockEndIdx.
+ *
+ * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0.
+ * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit.
+ * This is because dictionaries are allowed to be referenced fully
+ * as long as the last byte of the dictionary is in the window.
+ * Once input has progressed beyond window size, dictionary cannot be referenced anymore.
+ *
+ * In normal dict mode, the dictionary lies between lowLimit and dictLimit.
+ * In dictMatchState mode, lowLimit and dictLimit are the same,
+ * and the dictionary is below them.
+ * forceWindow and dictMatchState are therefore incompatible.
+ */
+MEM_STATIC void
+ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
+ const void* blockEnd,
+ U32 maxDist,
+ U32* loadedDictEndPtr,
+ const ZSTD_matchState_t** dictMatchStatePtr)
+{
+ U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
+ U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
+ DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
+ (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
+
+ /* - When there is no dictionary : loadedDictEnd == 0.
+ In which case, the test (blockEndIdx > maxDist) is merely to avoid
+ overflowing next operation `newLowLimit = blockEndIdx - maxDist`.
+ - When there is a standard dictionary :
+ Index referential is copied from the dictionary,
+ which means it starts from 0.
+ In which case, loadedDictEnd == dictSize,
+ and it makes sense to compare `blockEndIdx > maxDist + dictSize`
+ since `blockEndIdx` also starts from zero.
+ - When there is an attached dictionary :
+ loadedDictEnd is expressed within the referential of the context,
+ so it can be directly compared against blockEndIdx.
+ */
+ if (blockEndIdx > maxDist + loadedDictEnd) {
+ U32 const newLowLimit = blockEndIdx - maxDist;
+ if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit;
+ if (window->dictLimit < window->lowLimit) {
+ DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
+ (unsigned)window->dictLimit, (unsigned)window->lowLimit);
+ window->dictLimit = window->lowLimit;
+ }
+ /* On reaching window size, dictionaries are invalidated */
+ if (loadedDictEndPtr) *loadedDictEndPtr = 0;
+ if (dictMatchStatePtr) *dictMatchStatePtr = NULL;
+ }
+}
+
+/* Similar to ZSTD_window_enforceMaxDist(),
+ * but only invalidates dictionary
+ * when input progresses beyond window size.
+ * assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL)
+ * loadedDictEnd uses same referential as window->base
+ * maxDist is the window size */
+MEM_STATIC void
+ZSTD_checkDictValidity(const ZSTD_window_t* window,
+ const void* blockEnd,
+ U32 maxDist,
+ U32* loadedDictEndPtr,
+ const ZSTD_matchState_t** dictMatchStatePtr)
+{
+ assert(loadedDictEndPtr != NULL);
+ assert(dictMatchStatePtr != NULL);
+ { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
+ U32 const loadedDictEnd = *loadedDictEndPtr;
+ DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
+ (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
+ assert(blockEndIdx >= loadedDictEnd);
+
+ if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) {
+ /* On reaching window size, dictionaries are invalidated.
+ * For simplification, if window size is reached anywhere within next block,
+ * the dictionary is invalidated for the full block.
+ *
+ * We also have to invalidate the dictionary if ZSTD_window_update() has detected
+ * non-contiguous segments, which means that loadedDictEnd != window->dictLimit.
+ * loadedDictEnd may be 0, if forceWindow is true, but in that case we never use
+ * dictMatchState, so setting it to NULL is not a problem.
+ */
+ DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)");
+ *loadedDictEndPtr = 0;
+ *dictMatchStatePtr = NULL;
+ } else {
+ if (*loadedDictEndPtr != 0) {
+ DEBUGLOG(6, "dictionary considered valid for current block");
+ } } }
+}
+
+MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
+ ZSTD_memset(window, 0, sizeof(*window));
+ window->base = (BYTE const*)" ";
+ window->dictBase = (BYTE const*)" ";
+ ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX); /* Start above ZSTD_DUBT_UNSORTED_MARK */
+ window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */
+ window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */
+ window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */
+ window->nbOverflowCorrections = 0;
+}
+
+/**
+ * ZSTD_window_update():
+ * Updates the window by appending [src, src + srcSize) to the window.
+ * If it is not contiguous, the current prefix becomes the extDict, and we
+ * forget about the extDict. Handles overlap of the prefix and extDict.
+ * Returns non-zero if the segment is contiguous.
+ */
+MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
+ void const* src, size_t srcSize,
+ int forceNonContiguous)
+{
+ BYTE const* const ip = (BYTE const*)src;
+ U32 contiguous = 1;
+ DEBUGLOG(5, "ZSTD_window_update");
+ if (srcSize == 0)
+ return contiguous;
+ assert(window->base != NULL);
+ assert(window->dictBase != NULL);
+ /* Check if blocks follow each other */
+ if (src != window->nextSrc || forceNonContiguous) {
+ /* not contiguous */
+ size_t const distanceFromBase = (size_t)(window->nextSrc - window->base);
+ DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit);
+ window->lowLimit = window->dictLimit;
+ assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */
+ window->dictLimit = (U32)distanceFromBase;
+ window->dictBase = window->base;
+ window->base = ip - distanceFromBase;
+ /* ms->nextToUpdate = window->dictLimit; */
+ if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */
+ contiguous = 0;
+ }
+ window->nextSrc = ip + srcSize;
+ /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
+ if ( (ip+srcSize > window->dictBase + window->lowLimit)
+ & (ip < window->dictBase + window->dictLimit)) {
+ ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase;
+ U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx;
+ window->lowLimit = lowLimitMax;
+ DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit);
+ }
+ return contiguous;
+}
+
+/**
+ * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix.
+ */
+MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
+{
+ U32 const maxDistance = 1U << windowLog;
+ U32 const lowestValid = ms->window.lowLimit;
+ U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
+ U32 const isDictionary = (ms->loadedDictEnd != 0);
+ /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary
+ * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't
+ * valid for the entire block. So this check is sufficient to find the lowest valid match index.
+ */
+ U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
+ return matchLowest;
+}
+
+/**
+ * Returns the lowest allowed match index in the prefix.
+ */
+MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
+{
+ U32 const maxDistance = 1U << windowLog;
+ U32 const lowestValid = ms->window.dictLimit;
+ U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
+ U32 const isDictionary = (ms->loadedDictEnd != 0);
+ /* When computing the lowest prefix index we need to take the dictionary into account to handle
+ * the edge case where the dictionary and the source are contiguous in memory.
+ */
+ U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
+ return matchLowest;
+}
+
+
+
+/* debug functions */
+#if (DEBUGLEVEL>=2)
+
+MEM_STATIC double ZSTD_fWeight(U32 rawStat)
+{
+ U32 const fp_accuracy = 8;
+ U32 const fp_multiplier = (1 << fp_accuracy);
+ U32 const newStat = rawStat + 1;
+ U32 const hb = ZSTD_highbit32(newStat);
+ U32 const BWeight = hb * fp_multiplier;
+ U32 const FWeight = (newStat << fp_accuracy) >> hb;
+ U32 const weight = BWeight + FWeight;
+ assert(hb + fp_accuracy < 31);
+ return (double)weight / fp_multiplier;
+}
+
+/* display a table content,
+ * listing each element, its frequency, and its predicted bit cost */
+MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
+{
+ unsigned u, sum;
+ for (u=0, sum=0; u<=max; u++) sum += table[u];
+ DEBUGLOG(2, "total nb elts: %u", sum);
+ for (u=0; u<=max; u++) {
+ DEBUGLOG(2, "%2u: %5u (%.2f)",
+ u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) );
+ }
+}
+
+#endif
+
+/* Short Cache */
+
+/* Normally, zstd matchfinders follow this flow:
+ * 1. Compute hash at ip
+ * 2. Load index from hashTable[hash]
+ * 3. Check if *ip == *(base + index)
+ * In dictionary compression, loading *(base + index) is often an L2 or even L3 miss.
+ *
+ * Short cache is an optimization which allows us to avoid step 3 most of the time
+ * when the data doesn't actually match. With short cache, the flow becomes:
+ * 1. Compute (hash, currentTag) at ip. currentTag is an 8-bit independent hash at ip.
+ * 2. Load (index, matchTag) from hashTable[hash]. See ZSTD_writeTaggedIndex to understand how this works.
+ * 3. Only if currentTag == matchTag, check *ip == *(base + index). Otherwise, continue.
+ *
+ * Currently, short cache is only implemented in CDict hashtables. Thus, its use is limited to
+ * dictMatchState matchfinders.
+ */
+#define ZSTD_SHORT_CACHE_TAG_BITS 8
+#define ZSTD_SHORT_CACHE_TAG_MASK ((1u << ZSTD_SHORT_CACHE_TAG_BITS) - 1)
+
+/* Helper function for ZSTD_fillHashTable and ZSTD_fillDoubleHashTable.
+ * Unpacks hashAndTag into (hash, tag), then packs (index, tag) into hashTable[hash]. */
+MEM_STATIC void ZSTD_writeTaggedIndex(U32* const hashTable, size_t hashAndTag, U32 index) {
+ size_t const hash = hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS;
+ U32 const tag = (U32)(hashAndTag & ZSTD_SHORT_CACHE_TAG_MASK);
+ assert(index >> (32 - ZSTD_SHORT_CACHE_TAG_BITS) == 0);
+ hashTable[hash] = (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag;
+}
+
+/* Helper function for short cache matchfinders.
+ * Unpacks tag1 and tag2 from lower bits of packedTag1 and packedTag2, then checks if the tags match. */
+MEM_STATIC int ZSTD_comparePackedTags(size_t packedTag1, size_t packedTag2) {
+ U32 const tag1 = packedTag1 & ZSTD_SHORT_CACHE_TAG_MASK;
+ U32 const tag2 = packedTag2 & ZSTD_SHORT_CACHE_TAG_MASK;
+ return tag1 == tag2;
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+/* ===============================================================
+ * Shared internal declarations
+ * These prototypes may be called from sources not in lib/compress
+ * =============================================================== */
+
+/* ZSTD_loadCEntropy() :
+ * dict : must point at beginning of a valid zstd dictionary.
+ * return : size of dictionary header (size of magic number + dict ID + entropy tables)
+ * assumptions : magic number supposed already checked
+ * and dictSize >= 8 */
+size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
+ const void* const dict, size_t dictSize);
+
+void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
+
+/* ==============================================================
+ * Private declarations
+ * These prototypes shall only be called from within lib/compress
+ * ============================================================== */
+
+/* ZSTD_getCParamsFromCCtxParams() :
+ * cParams are built depending on compressionLevel, src size hints,
+ * LDM and manually set compression parameters.
+ * Note: srcSizeHint == 0 means 0!
+ */
+ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
+ const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
+
+/*! ZSTD_initCStream_internal() :
+ * Private use only. Init streaming operation.
+ * expects params to be valid.
+ * must receive dict, or cdict, or none, but not both.
+ * @return : 0, or an error code */
+size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
+ const void* dict, size_t dictSize,
+ const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize);
+
+void ZSTD_resetSeqStore(seqStore_t* ssPtr);
+
+/*! ZSTD_getCParamsFromCDict() :
+ * as the name implies */
+ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
+
+/* ZSTD_compressBegin_advanced_internal() :
+ * Private use only. To be called from zstdmt_compress.c. */
+size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
+ const void* dict, size_t dictSize,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ const ZSTD_CDict* cdict,
+ const ZSTD_CCtx_params* params,
+ unsigned long long pledgedSrcSize);
+
+/* ZSTD_compress_advanced_internal() :
+ * Private use only. To be called from zstdmt_compress.c. */
+size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ const ZSTD_CCtx_params* params);
+
+
+/* ZSTD_writeLastEmptyBlock() :
+ * output an empty Block with end-of-frame mark to complete a frame
+ * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h))
+ * or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize)
+ */
+size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity);
+
+
+/* ZSTD_referenceExternalSequences() :
+ * Must be called before starting a compression operation.
+ * seqs must parse a prefix of the source.
+ * This cannot be used when long range matching is enabled.
+ * Zstd will use these sequences, and pass the literals to a secondary block
+ * compressor.
+ * @return : An error code on failure.
+ * NOTE: seqs are not verified! Invalid sequences can cause out-of-bounds memory
+ * access and data corruption.
+ */
+size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq);
+
+/** ZSTD_cycleLog() :
+ * condition for correct operation : hashLog > 1 */
+U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
+
+/** ZSTD_CCtx_trace() :
+ * Trace the end of a compression call.
+ */
+void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
+
+/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
+ * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
+ * Note that the block delimiter must include the last literals of the block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
+ ZSTD_sequencePosition* seqPos,
+ const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+ const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
+
+/* Returns the number of bytes to move the current read position back by.
+ * Only non-zero if we ended up splitting a sequence.
+ * Otherwise, it may return a ZSTD error if something went wrong.
+ *
+ * This function will attempt to scan through blockSize bytes
+ * represented by the sequences in @inSeqs,
+ * storing any (partial) sequences.
+ *
+ * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
+ * avoid splitting a match, or to avoid splitting a match such that it would produce a match
+ * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
+ const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+ const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
+
+#endif /* ZSTD_COMPRESS_H */
diff --git a/contrib/zstd/zstd_compress_literals.c b/contrib/zstd/zstd_compress_literals.c
new file mode 100644
index 0000000..bfd4f11
--- /dev/null
+++ b/contrib/zstd/zstd_compress_literals.c
@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+ /*-*************************************
+ * Dependencies
+ ***************************************/
+#include "zstd_compress_literals.h"
+
+
+/* **************************************************************
+* Debug Traces
+****************************************************************/
+#if DEBUGLEVEL >= 2
+
+static size_t showHexa(const void* src, size_t srcSize)
+{
+ const BYTE* const ip = (const BYTE*)src;
+ size_t u;
+ for (u=0; u<srcSize; u++) {
+ RAWLOG(5, " %02X", ip[u]); (void)ip;
+ }
+ RAWLOG(5, " \n");
+ return srcSize;
+}
+
+#endif
+
+
+/* **************************************************************
+* Literals compression - special cases
+****************************************************************/
+size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ BYTE* const ostart = (BYTE*)dst;
+ U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
+
+ DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity);
+
+ RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
+
+ switch(flSize)
+ {
+ case 1: /* 2 - 1 - 5 */
+ ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3));
+ break;
+ case 2: /* 2 - 2 - 12 */
+ MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4)));
+ break;
+ case 3: /* 2 - 2 - 20 */
+ MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4)));
+ break;
+ default: /* not necessary : flSize is {1,2,3} */
+ assert(0);
+ }
+
+ ZSTD_memcpy(ostart + flSize, src, srcSize);
+ DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
+ return srcSize + flSize;
+}
+
+static int allBytesIdentical(const void* src, size_t srcSize)
+{
+ assert(srcSize >= 1);
+ assert(src != NULL);
+ { const BYTE b = ((const BYTE*)src)[0];
+ size_t p;
+ for (p=1; p<srcSize; p++) {
+ if (((const BYTE*)src)[p] != b) return 0;
+ }
+ return 1;
+ }
+}
+
+size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ BYTE* const ostart = (BYTE*)dst;
+ U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
+
+ assert(dstCapacity >= 4); (void)dstCapacity;
+ assert(allBytesIdentical(src, srcSize));
+
+ switch(flSize)
+ {
+ case 1: /* 2 - 1 - 5 */
+ ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3));
+ break;
+ case 2: /* 2 - 2 - 12 */
+ MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4)));
+ break;
+ case 3: /* 2 - 2 - 20 */
+ MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4)));
+ break;
+ default: /* not necessary : flSize is {1,2,3} */
+ assert(0);
+ }
+
+ ostart[flSize] = *(const BYTE*)src;
+ DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1);
+ return flSize+1;
+}
+
+/* ZSTD_minLiteralsToCompress() :
+ * returns minimal amount of literals
+ * for literal compression to even be attempted.
+ * Minimum is made tighter as compression strategy increases.
+ */
+static size_t
+ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat)
+{
+ assert((int)strategy >= 0);
+ assert((int)strategy <= 9);
+ /* btultra2 : min 8 bytes;
+ * then 2x larger for each successive compression strategy
+ * max threshold 64 bytes */
+ { int const shift = MIN(9-(int)strategy, 3);
+ size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift;
+ DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc);
+ return mintc;
+ }
+}
+
+size_t ZSTD_compressLiterals (
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ void* entropyWorkspace, size_t entropyWorkspaceSize,
+ const ZSTD_hufCTables_t* prevHuf,
+ ZSTD_hufCTables_t* nextHuf,
+ ZSTD_strategy strategy,
+ int disableLiteralCompression,
+ int suspectUncompressible,
+ int bmi2)
+{
+ size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
+ BYTE* const ostart = (BYTE*)dst;
+ U32 singleStream = srcSize < 256;
+ symbolEncodingType_e hType = set_compressed;
+ size_t cLitSize;
+
+ DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)",
+ disableLiteralCompression, (U32)srcSize, dstCapacity);
+
+ DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize));
+
+ /* Prepare nextEntropy assuming reusing the existing table */
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+
+ if (disableLiteralCompression)
+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+
+ /* if too small, don't even attempt compression (speed opt) */
+ if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode))
+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+
+ RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
+ { HUF_repeat repeat = prevHuf->repeatMode;
+ int const flags = 0
+ | (bmi2 ? HUF_flags_bmi2 : 0)
+ | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0)
+ | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0)
+ | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0);
+
+ typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int);
+ huf_compress_f huf_compress;
+ if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
+ huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
+ cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize,
+ src, srcSize,
+ HUF_SYMBOLVALUE_MAX, LitHufLog,
+ entropyWorkspace, entropyWorkspaceSize,
+ (HUF_CElt*)nextHuf->CTable,
+ &repeat, flags);
+ DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize);
+ if (repeat != HUF_repeat_none) {
+ /* reused the existing table */
+ DEBUGLOG(5, "reusing statistics from previous huffman block");
+ hType = set_repeat;
+ }
+ }
+
+ { size_t const minGain = ZSTD_minGain(srcSize, strategy);
+ if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+ } }
+ if (cLitSize==1) {
+ /* A return value of 1 signals that the alphabet consists of a single symbol.
+ * However, in some rare circumstances, it could be the compressed size (a single byte).
+ * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`.
+ * (it's also necessary to not generate statistics).
+ * Therefore, in such a case, actively check that all bytes are identical. */
+ if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) {
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+ return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
+ } }
+
+ if (hType == set_compressed) {
+ /* using a newly constructed table */
+ nextHuf->repeatMode = HUF_repeat_check;
+ }
+
+ /* Build header */
+ switch(lhSize)
+ {
+ case 3: /* 2 - 2 - 10 - 10 */
+ if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
+ { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
+ MEM_writeLE24(ostart, lhc);
+ break;
+ }
+ case 4: /* 2 - 2 - 14 - 14 */
+ assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
+ { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18);
+ MEM_writeLE32(ostart, lhc);
+ break;
+ }
+ case 5: /* 2 - 2 - 18 - 18 */
+ assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
+ { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22);
+ MEM_writeLE32(ostart, lhc);
+ ostart[4] = (BYTE)(cLitSize >> 10);
+ break;
+ }
+ default: /* not possible : lhSize is {3,4,5} */
+ assert(0);
+ }
+ DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize));
+ return lhSize+cLitSize;
+}
diff --git a/contrib/zstd/zstd_compress_literals.h b/contrib/zstd/zstd_compress_literals.h
new file mode 100644
index 0000000..b060c8a
--- /dev/null
+++ b/contrib/zstd/zstd_compress_literals.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMPRESS_LITERALS_H
+#define ZSTD_COMPRESS_LITERALS_H
+
+#include "zstd_compress_internal.h" /* ZSTD_hufCTables_t, ZSTD_minGain() */
+
+
+size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* ZSTD_compressRleLiteralsBlock() :
+ * Conditions :
+ * - All bytes in @src are identical
+ * - dstCapacity >= 4 */
+size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+/* ZSTD_compressLiterals():
+ * @entropyWorkspace: must be aligned on 4-bytes boundaries
+ * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE
+ * @suspectUncompressible: sampling checks, to potentially skip huffman coding
+ */
+size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ void* entropyWorkspace, size_t entropyWorkspaceSize,
+ const ZSTD_hufCTables_t* prevHuf,
+ ZSTD_hufCTables_t* nextHuf,
+ ZSTD_strategy strategy, int disableLiteralCompression,
+ int suspectUncompressible,
+ int bmi2);
+
+#endif /* ZSTD_COMPRESS_LITERALS_H */
diff --git a/contrib/zstd/zstd_compress_sequences.c b/contrib/zstd/zstd_compress_sequences.c
new file mode 100644
index 0000000..8872d4d
--- /dev/null
+++ b/contrib/zstd/zstd_compress_sequences.c
@@ -0,0 +1,442 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+ /*-*************************************
+ * Dependencies
+ ***************************************/
+#include "zstd_compress_sequences.h"
+
+/**
+ * -log2(x / 256) lookup table for x in [0, 256).
+ * If x == 0: Return 0
+ * Else: Return floor(-log2(x / 256) * 256)
+ */
+static unsigned const kInverseProbabilityLog256[256] = {
+ 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162,
+ 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889,
+ 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734,
+ 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626,
+ 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542,
+ 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473,
+ 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415,
+ 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366,
+ 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322,
+ 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282,
+ 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247,
+ 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215,
+ 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185,
+ 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157,
+ 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132,
+ 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108,
+ 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85,
+ 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64,
+ 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44,
+ 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25,
+ 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7,
+ 5, 4, 2, 1,
+};
+
+static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) {
+ void const* ptr = ctable;
+ U16 const* u16ptr = (U16 const*)ptr;
+ U32 const maxSymbolValue = MEM_read16(u16ptr + 1);
+ return maxSymbolValue;
+}
+
+/**
+ * Returns true if we should use ncount=-1 else we should
+ * use ncount=1 for low probability symbols instead.
+ */
+static unsigned ZSTD_useLowProbCount(size_t const nbSeq)
+{
+ /* Heuristic: This should cover most blocks <= 16K and
+ * start to fade out after 16K to about 32K depending on
+ * compressibility.
+ */
+ return nbSeq >= 2048;
+}
+
+/**
+ * Returns the cost in bytes of encoding the normalized count header.
+ * Returns an error if any of the helper functions return an error.
+ */
+static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
+ size_t const nbSeq, unsigned const FSELog)
+{
+ BYTE wksp[FSE_NCOUNTBOUND];
+ S16 norm[MaxSeq + 1];
+ const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
+ FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), "");
+ return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog);
+}
+
+/**
+ * Returns the cost in bits of encoding the distribution described by count
+ * using the entropy bound.
+ */
+static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total)
+{
+ unsigned cost = 0;
+ unsigned s;
+
+ assert(total > 0);
+ for (s = 0; s <= max; ++s) {
+ unsigned norm = (unsigned)((256 * count[s]) / total);
+ if (count[s] != 0 && norm == 0)
+ norm = 1;
+ assert(count[s] < total);
+ cost += count[s] * kInverseProbabilityLog256[norm];
+ }
+ return cost >> 8;
+}
+
+/**
+ * Returns the cost in bits of encoding the distribution in count using ctable.
+ * Returns an error if ctable cannot represent all the symbols in count.
+ */
+size_t ZSTD_fseBitCost(
+ FSE_CTable const* ctable,
+ unsigned const* count,
+ unsigned const max)
+{
+ unsigned const kAccuracyLog = 8;
+ size_t cost = 0;
+ unsigned s;
+ FSE_CState_t cstate;
+ FSE_initCState(&cstate, ctable);
+ if (ZSTD_getFSEMaxSymbolValue(ctable) < max) {
+ DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u",
+ ZSTD_getFSEMaxSymbolValue(ctable), max);
+ return ERROR(GENERIC);
+ }
+ for (s = 0; s <= max; ++s) {
+ unsigned const tableLog = cstate.stateLog;
+ unsigned const badCost = (tableLog + 1) << kAccuracyLog;
+ unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog);
+ if (count[s] == 0)
+ continue;
+ if (bitCost >= badCost) {
+ DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s);
+ return ERROR(GENERIC);
+ }
+ cost += (size_t)count[s] * bitCost;
+ }
+ return cost >> kAccuracyLog;
+}
+
+/**
+ * Returns the cost in bits of encoding the distribution in count using the
+ * table described by norm. The max symbol support by norm is assumed >= max.
+ * norm must be valid for every symbol with non-zero probability in count.
+ */
+size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
+ unsigned const* count, unsigned const max)
+{
+ unsigned const shift = 8 - accuracyLog;
+ size_t cost = 0;
+ unsigned s;
+ assert(accuracyLog <= 8);
+ for (s = 0; s <= max; ++s) {
+ unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1;
+ unsigned const norm256 = normAcc << shift;
+ assert(norm256 > 0);
+ assert(norm256 < 256);
+ cost += count[s] * kInverseProbabilityLog256[norm256];
+ }
+ return cost >> 8;
+}
+
+symbolEncodingType_e
+ZSTD_selectEncodingType(
+ FSE_repeat* repeatMode, unsigned const* count, unsigned const max,
+ size_t const mostFrequent, size_t nbSeq, unsigned const FSELog,
+ FSE_CTable const* prevCTable,
+ short const* defaultNorm, U32 defaultNormLog,
+ ZSTD_defaultPolicy_e const isDefaultAllowed,
+ ZSTD_strategy const strategy)
+{
+ ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0);
+ if (mostFrequent == nbSeq) {
+ *repeatMode = FSE_repeat_none;
+ if (isDefaultAllowed && nbSeq <= 2) {
+ /* Prefer set_basic over set_rle when there are 2 or fewer symbols,
+ * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol.
+ * If basic encoding isn't possible, always choose RLE.
+ */
+ DEBUGLOG(5, "Selected set_basic");
+ return set_basic;
+ }
+ DEBUGLOG(5, "Selected set_rle");
+ return set_rle;
+ }
+ if (strategy < ZSTD_lazy) {
+ if (isDefaultAllowed) {
+ size_t const staticFse_nbSeq_max = 1000;
+ size_t const mult = 10 - strategy;
+ size_t const baseLog = 3;
+ size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */
+ assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */
+ assert(mult <= 9 && mult >= 7);
+ if ( (*repeatMode == FSE_repeat_valid)
+ && (nbSeq < staticFse_nbSeq_max) ) {
+ DEBUGLOG(5, "Selected set_repeat");
+ return set_repeat;
+ }
+ if ( (nbSeq < dynamicFse_nbSeq_min)
+ || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) {
+ DEBUGLOG(5, "Selected set_basic");
+ /* The format allows default tables to be repeated, but it isn't useful.
+ * When using simple heuristics to select encoding type, we don't want
+ * to confuse these tables with dictionaries. When running more careful
+ * analysis, we don't need to waste time checking both repeating tables
+ * and default tables.
+ */
+ *repeatMode = FSE_repeat_none;
+ return set_basic;
+ }
+ }
+ } else {
+ size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC);
+ size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC);
+ size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog);
+ size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq);
+
+ if (isDefaultAllowed) {
+ assert(!ZSTD_isError(basicCost));
+ assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost)));
+ }
+ assert(!ZSTD_isError(NCountCost));
+ assert(compressedCost < ERROR(maxCode));
+ DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u",
+ (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost);
+ if (basicCost <= repeatCost && basicCost <= compressedCost) {
+ DEBUGLOG(5, "Selected set_basic");
+ assert(isDefaultAllowed);
+ *repeatMode = FSE_repeat_none;
+ return set_basic;
+ }
+ if (repeatCost <= compressedCost) {
+ DEBUGLOG(5, "Selected set_repeat");
+ assert(!ZSTD_isError(repeatCost));
+ return set_repeat;
+ }
+ assert(compressedCost < basicCost && compressedCost < repeatCost);
+ }
+ DEBUGLOG(5, "Selected set_compressed");
+ *repeatMode = FSE_repeat_check;
+ return set_compressed;
+}
+
+typedef struct {
+ S16 norm[MaxSeq + 1];
+ U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)];
+} ZSTD_BuildCTableWksp;
+
+size_t
+ZSTD_buildCTable(void* dst, size_t dstCapacity,
+ FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
+ unsigned* count, U32 max,
+ const BYTE* codeTable, size_t nbSeq,
+ const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+ const FSE_CTable* prevCTable, size_t prevCTableSize,
+ void* entropyWorkspace, size_t entropyWorkspaceSize)
+{
+ BYTE* op = (BYTE*)dst;
+ const BYTE* const oend = op + dstCapacity;
+ DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity);
+
+ switch (type) {
+ case set_rle:
+ FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), "");
+ RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space");
+ *op = codeTable[0];
+ return 1;
+ case set_repeat:
+ ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize);
+ return 0;
+ case set_basic:
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */
+ return 0;
+ case set_compressed: {
+ ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace;
+ size_t nbSeq_1 = nbSeq;
+ const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
+ if (count[codeTable[nbSeq-1]] > 1) {
+ count[codeTable[nbSeq-1]]--;
+ nbSeq_1--;
+ }
+ assert(nbSeq_1 > 1);
+ assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp));
+ (void)entropyWorkspaceSize;
+ FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed");
+ assert(oend >= op);
+ { size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */
+ FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed");
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed");
+ return NCountSize;
+ }
+ }
+ default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach");
+ }
+}
+
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_encodeSequences_body(
+ void* dst, size_t dstCapacity,
+ FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+ FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+ FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+ seqDef const* sequences, size_t nbSeq, int longOffsets)
+{
+ BIT_CStream_t blockStream;
+ FSE_CState_t stateMatchLength;
+ FSE_CState_t stateOffsetBits;
+ FSE_CState_t stateLitLength;
+
+ RETURN_ERROR_IF(
+ ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)),
+ dstSize_tooSmall, "not enough space remaining");
+ DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)",
+ (int)(blockStream.endPtr - blockStream.startPtr),
+ (unsigned)dstCapacity);
+
+ /* first symbols */
+ FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]);
+ FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]);
+ FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]);
+ BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]);
+ if (MEM_32bits()) BIT_flushBits(&blockStream);
+ BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]);
+ if (MEM_32bits()) BIT_flushBits(&blockStream);
+ if (longOffsets) {
+ U32 const ofBits = ofCodeTable[nbSeq-1];
+ unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+ if (extraBits) {
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits);
+ BIT_flushBits(&blockStream);
+ }
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits,
+ ofBits - extraBits);
+ } else {
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]);
+ }
+ BIT_flushBits(&blockStream);
+
+ { size_t n;
+ for (n=nbSeq-2 ; n<nbSeq ; n--) { /* intentional underflow */
+ BYTE const llCode = llCodeTable[n];
+ BYTE const ofCode = ofCodeTable[n];
+ BYTE const mlCode = mlCodeTable[n];
+ U32 const llBits = LL_bits[llCode];
+ U32 const ofBits = ofCode;
+ U32 const mlBits = ML_bits[mlCode];
+ DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u",
+ (unsigned)sequences[n].litLength,
+ (unsigned)sequences[n].mlBase + MINMATCH,
+ (unsigned)sequences[n].offBase);
+ /* 32b*/ /* 64b*/
+ /* (7)*/ /* (7)*/
+ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
+ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */
+ if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/
+ FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */
+ if (MEM_32bits() || (ofBits+mlBits+llBits >= 64-7-(LLFSELog+MLFSELog+OffFSELog)))
+ BIT_flushBits(&blockStream); /* (7)*/
+ BIT_addBits(&blockStream, sequences[n].litLength, llBits);
+ if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
+ BIT_addBits(&blockStream, sequences[n].mlBase, mlBits);
+ if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream);
+ if (longOffsets) {
+ unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+ if (extraBits) {
+ BIT_addBits(&blockStream, sequences[n].offBase, extraBits);
+ BIT_flushBits(&blockStream); /* (7)*/
+ }
+ BIT_addBits(&blockStream, sequences[n].offBase >> extraBits,
+ ofBits - extraBits); /* 31 */
+ } else {
+ BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */
+ }
+ BIT_flushBits(&blockStream); /* (7)*/
+ DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr));
+ } }
+
+ DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog);
+ FSE_flushCState(&blockStream, &stateMatchLength);
+ DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog);
+ FSE_flushCState(&blockStream, &stateOffsetBits);
+ DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog);
+ FSE_flushCState(&blockStream, &stateLitLength);
+
+ { size_t const streamSize = BIT_closeCStream(&blockStream);
+ RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space");
+ return streamSize;
+ }
+}
+
+static size_t
+ZSTD_encodeSequences_default(
+ void* dst, size_t dstCapacity,
+ FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+ FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+ FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+ seqDef const* sequences, size_t nbSeq, int longOffsets)
+{
+ return ZSTD_encodeSequences_body(dst, dstCapacity,
+ CTable_MatchLength, mlCodeTable,
+ CTable_OffsetBits, ofCodeTable,
+ CTable_LitLength, llCodeTable,
+ sequences, nbSeq, longOffsets);
+}
+
+
+#if DYNAMIC_BMI2
+
+static BMI2_TARGET_ATTRIBUTE size_t
+ZSTD_encodeSequences_bmi2(
+ void* dst, size_t dstCapacity,
+ FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+ FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+ FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+ seqDef const* sequences, size_t nbSeq, int longOffsets)
+{
+ return ZSTD_encodeSequences_body(dst, dstCapacity,
+ CTable_MatchLength, mlCodeTable,
+ CTable_OffsetBits, ofCodeTable,
+ CTable_LitLength, llCodeTable,
+ sequences, nbSeq, longOffsets);
+}
+
+#endif
+
+size_t ZSTD_encodeSequences(
+ void* dst, size_t dstCapacity,
+ FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+ FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+ FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+ seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2)
+{
+ DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity);
+#if DYNAMIC_BMI2
+ if (bmi2) {
+ return ZSTD_encodeSequences_bmi2(dst, dstCapacity,
+ CTable_MatchLength, mlCodeTable,
+ CTable_OffsetBits, ofCodeTable,
+ CTable_LitLength, llCodeTable,
+ sequences, nbSeq, longOffsets);
+ }
+#endif
+ (void)bmi2;
+ return ZSTD_encodeSequences_default(dst, dstCapacity,
+ CTable_MatchLength, mlCodeTable,
+ CTable_OffsetBits, ofCodeTable,
+ CTable_LitLength, llCodeTable,
+ sequences, nbSeq, longOffsets);
+}
diff --git a/contrib/zstd/zstd_compress_sequences.h b/contrib/zstd/zstd_compress_sequences.h
new file mode 100644
index 0000000..37848d8
--- /dev/null
+++ b/contrib/zstd/zstd_compress_sequences.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMPRESS_SEQUENCES_H
+#define ZSTD_COMPRESS_SEQUENCES_H
+
+#include "fse.h" /* FSE_repeat, FSE_CTable */
+#include "zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */
+
+typedef enum {
+ ZSTD_defaultDisallowed = 0,
+ ZSTD_defaultAllowed = 1
+} ZSTD_defaultPolicy_e;
+
+symbolEncodingType_e
+ZSTD_selectEncodingType(
+ FSE_repeat* repeatMode, unsigned const* count, unsigned const max,
+ size_t const mostFrequent, size_t nbSeq, unsigned const FSELog,
+ FSE_CTable const* prevCTable,
+ short const* defaultNorm, U32 defaultNormLog,
+ ZSTD_defaultPolicy_e const isDefaultAllowed,
+ ZSTD_strategy const strategy);
+
+size_t
+ZSTD_buildCTable(void* dst, size_t dstCapacity,
+ FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
+ unsigned* count, U32 max,
+ const BYTE* codeTable, size_t nbSeq,
+ const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+ const FSE_CTable* prevCTable, size_t prevCTableSize,
+ void* entropyWorkspace, size_t entropyWorkspaceSize);
+
+size_t ZSTD_encodeSequences(
+ void* dst, size_t dstCapacity,
+ FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+ FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+ FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+ seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2);
+
+size_t ZSTD_fseBitCost(
+ FSE_CTable const* ctable,
+ unsigned const* count,
+ unsigned const max);
+
+size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
+ unsigned const* count, unsigned const max);
+#endif /* ZSTD_COMPRESS_SEQUENCES_H */
diff --git a/contrib/zstd/zstd_compress_superblock.c b/contrib/zstd/zstd_compress_superblock.c
new file mode 100644
index 0000000..8349085
--- /dev/null
+++ b/contrib/zstd/zstd_compress_superblock.c
@@ -0,0 +1,577 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+ /*-*************************************
+ * Dependencies
+ ***************************************/
+#include "zstd_compress_superblock.h"
+
+#include "zstd_internal.h" /* ZSTD_getSequenceLength */
+#include "hist.h" /* HIST_countFast_wksp */
+#include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */
+#include "zstd_compress_sequences.h"
+#include "zstd_compress_literals.h"
+
+/** ZSTD_compressSubBlock_literal() :
+ * Compresses literals section for a sub-block.
+ * When we have to write the Huffman table we will sometimes choose a header
+ * size larger than necessary. This is because we have to pick the header size
+ * before we know the table size + compressed size, so we have a bound on the
+ * table size. If we guessed incorrectly, we fall back to uncompressed literals.
+ *
+ * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded
+ * in writing the header, otherwise it is set to 0.
+ *
+ * hufMetadata->hType has literals block type info.
+ * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block.
+ * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block.
+ * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block
+ * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block
+ * and the following sub-blocks' literals sections will be Treeless_Literals_Block.
+ * @return : compressed size of literals section of a sub-block
+ * Or 0 if unable to compress.
+ * Or error code */
+static size_t
+ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
+ const ZSTD_hufCTablesMetadata_t* hufMetadata,
+ const BYTE* literals, size_t litSize,
+ void* dst, size_t dstSize,
+ const int bmi2, int writeEntropy, int* entropyWritten)
+{
+ size_t const header = writeEntropy ? 200 : 0;
+ size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header));
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstSize;
+ BYTE* op = ostart + lhSize;
+ U32 const singleStream = lhSize == 3;
+ symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
+ size_t cLitSize = 0;
+
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy);
+
+ *entropyWritten = 0;
+ if (litSize == 0 || hufMetadata->hType == set_basic) {
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal");
+ return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize);
+ } else if (hufMetadata->hType == set_rle) {
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal");
+ return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize);
+ }
+
+ assert(litSize > 0);
+ assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat);
+
+ if (writeEntropy && hufMetadata->hType == set_compressed) {
+ ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize);
+ op += hufMetadata->hufDesSize;
+ cLitSize += hufMetadata->hufDesSize;
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
+ }
+
+ { int const flags = bmi2 ? HUF_flags_bmi2 : 0;
+ const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable, flags)
+ : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable, flags);
+ op += cSize;
+ cLitSize += cSize;
+ if (cSize == 0 || ERR_isError(cSize)) {
+ DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize));
+ return 0;
+ }
+ /* If we expand and we aren't writing a header then emit uncompressed */
+ if (!writeEntropy && cLitSize >= litSize) {
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible");
+ return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize);
+ }
+ /* If we are writing headers then allow expansion that doesn't change our header size. */
+ if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) {
+ assert(cLitSize > litSize);
+ DEBUGLOG(5, "Literals expanded beyond allowed header size");
+ return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize);
+ }
+ DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize);
+ }
+
+ /* Build header */
+ switch(lhSize)
+ {
+ case 3: /* 2 - 2 - 10 - 10 */
+ { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14);
+ MEM_writeLE24(ostart, lhc);
+ break;
+ }
+ case 4: /* 2 - 2 - 14 - 14 */
+ { U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18);
+ MEM_writeLE32(ostart, lhc);
+ break;
+ }
+ case 5: /* 2 - 2 - 18 - 18 */
+ { U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22);
+ MEM_writeLE32(ostart, lhc);
+ ostart[4] = (BYTE)(cLitSize >> 10);
+ break;
+ }
+ default: /* not possible : lhSize is {3,4,5} */
+ assert(0);
+ }
+ *entropyWritten = 1;
+ DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart));
+ return op-ostart;
+}
+
+static size_t
+ZSTD_seqDecompressedSize(seqStore_t const* seqStore,
+ const seqDef* sequences, size_t nbSeq,
+ size_t litSize, int lastSequence)
+{
+ const seqDef* const sstart = sequences;
+ const seqDef* const send = sequences + nbSeq;
+ const seqDef* sp = sstart;
+ size_t matchLengthSum = 0;
+ size_t litLengthSum = 0;
+ (void)(litLengthSum); /* suppress unused variable warning on some environments */
+ while (send-sp > 0) {
+ ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp);
+ litLengthSum += seqLen.litLength;
+ matchLengthSum += seqLen.matchLength;
+ sp++;
+ }
+ assert(litLengthSum <= litSize);
+ if (!lastSequence) {
+ assert(litLengthSum == litSize);
+ }
+ return matchLengthSum + litSize;
+}
+
+/** ZSTD_compressSubBlock_sequences() :
+ * Compresses sequences section for a sub-block.
+ * fseMetadata->llType, fseMetadata->ofType, and fseMetadata->mlType have
+ * symbol compression modes for the super-block.
+ * The first successfully compressed block will have these in its header.
+ * We set entropyWritten=1 when we succeed in compressing the sequences.
+ * The following sub-blocks will always have repeat mode.
+ * @return : compressed size of sequences section of a sub-block
+ * Or 0 if it is unable to compress
+ * Or error code. */
+static size_t
+ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
+ const ZSTD_fseCTablesMetadata_t* fseMetadata,
+ const seqDef* sequences, size_t nbSeq,
+ const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
+ const ZSTD_CCtx_params* cctxParams,
+ void* dst, size_t dstCapacity,
+ const int bmi2, int writeEntropy, int* entropyWritten)
+{
+ const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstCapacity;
+ BYTE* op = ostart;
+ BYTE* seqHead;
+
+ DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets);
+
+ *entropyWritten = 0;
+ /* Sequences Header */
+ RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
+ dstSize_tooSmall, "");
+ if (nbSeq < 0x7F)
+ *op++ = (BYTE)nbSeq;
+ else if (nbSeq < LONGNBSEQ)
+ op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
+ else
+ op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
+ if (nbSeq==0) {
+ return op - ostart;
+ }
+
+ /* seqHead : flags for FSE encoding type */
+ seqHead = op++;
+
+ DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart));
+
+ if (writeEntropy) {
+ const U32 LLtype = fseMetadata->llType;
+ const U32 Offtype = fseMetadata->ofType;
+ const U32 MLtype = fseMetadata->mlType;
+ DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize);
+ *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
+ ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize);
+ op += fseMetadata->fseTablesSize;
+ } else {
+ const U32 repeat = set_repeat;
+ *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2));
+ }
+
+ { size_t const bitstreamSize = ZSTD_encodeSequences(
+ op, oend - op,
+ fseTables->matchlengthCTable, mlCode,
+ fseTables->offcodeCTable, ofCode,
+ fseTables->litlengthCTable, llCode,
+ sequences, nbSeq,
+ longOffsets, bmi2);
+ FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed");
+ op += bitstreamSize;
+ /* zstd versions <= 1.3.4 mistakenly report corruption when
+ * FSE_readNCount() receives a buffer < 4 bytes.
+ * Fixed by https://github.com/facebook/zstd/pull/1146.
+ * This can happen when the last set_compressed table present is 2
+ * bytes and the bitstream is only one byte.
+ * In this exceedingly rare case, we will simply emit an uncompressed
+ * block, since it isn't worth optimizing.
+ */
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) {
+ /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */
+ assert(fseMetadata->lastCountSize + bitstreamSize == 3);
+ DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by "
+ "emitting an uncompressed block.");
+ return 0;
+ }
+#endif
+ DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize);
+ }
+
+ /* zstd versions <= 1.4.0 mistakenly report error when
+ * sequences section body size is less than 3 bytes.
+ * Fixed by https://github.com/facebook/zstd/pull/1664.
+ * This can happen when the previous sequences section block is compressed
+ * with rle mode and the current block's sequences section is compressed
+ * with repeat mode where sequences section body size can be 1 byte.
+ */
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ if (op-seqHead < 4) {
+ DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting "
+ "an uncompressed block when sequences are < 4 bytes");
+ return 0;
+ }
+#endif
+
+ *entropyWritten = 1;
+ return op - ostart;
+}
+
+/** ZSTD_compressSubBlock() :
+ * Compresses a single sub-block.
+ * @return : compressed size of the sub-block
+ * Or 0 if it failed to compress. */
+static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
+ const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ const seqDef* sequences, size_t nbSeq,
+ const BYTE* literals, size_t litSize,
+ const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
+ const ZSTD_CCtx_params* cctxParams,
+ void* dst, size_t dstCapacity,
+ const int bmi2,
+ int writeLitEntropy, int writeSeqEntropy,
+ int* litEntropyWritten, int* seqEntropyWritten,
+ U32 lastBlock)
+{
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstCapacity;
+ BYTE* op = ostart + ZSTD_blockHeaderSize;
+ DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)",
+ litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock);
+ { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable,
+ &entropyMetadata->hufMetadata, literals, litSize,
+ op, oend-op, bmi2, writeLitEntropy, litEntropyWritten);
+ FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed");
+ if (cLitSize == 0) return 0;
+ op += cLitSize;
+ }
+ { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse,
+ &entropyMetadata->fseMetadata,
+ sequences, nbSeq,
+ llCode, mlCode, ofCode,
+ cctxParams,
+ op, oend-op,
+ bmi2, writeSeqEntropy, seqEntropyWritten);
+ FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed");
+ if (cSeqSize == 0) return 0;
+ op += cSeqSize;
+ }
+ /* Write block header */
+ { size_t cSize = (op-ostart)-ZSTD_blockHeaderSize;
+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
+ MEM_writeLE24(ostart, cBlockHeader24);
+ }
+ return op-ostart;
+}
+
+static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize,
+ const ZSTD_hufCTables_t* huf,
+ const ZSTD_hufCTablesMetadata_t* hufMetadata,
+ void* workspace, size_t wkspSize,
+ int writeEntropy)
+{
+ unsigned* const countWksp = (unsigned*)workspace;
+ unsigned maxSymbolValue = 255;
+ size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
+
+ if (hufMetadata->hType == set_basic) return litSize;
+ else if (hufMetadata->hType == set_rle) return 1;
+ else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) {
+ size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize);
+ if (ZSTD_isError(largest)) return litSize;
+ { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue);
+ if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize;
+ return cLitSizeEstimate + literalSectionHeaderSize;
+ } }
+ assert(0); /* impossible */
+ return 0;
+}
+
+static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
+ const BYTE* codeTable, unsigned maxCode,
+ size_t nbSeq, const FSE_CTable* fseCTable,
+ const U8* additionalBits,
+ short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+ void* workspace, size_t wkspSize)
+{
+ unsigned* const countWksp = (unsigned*)workspace;
+ const BYTE* ctp = codeTable;
+ const BYTE* const ctStart = ctp;
+ const BYTE* const ctEnd = ctStart + nbSeq;
+ size_t cSymbolTypeSizeEstimateInBits = 0;
+ unsigned max = maxCode;
+
+ HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */
+ if (type == set_basic) {
+ /* We selected this encoding type, so it must be valid. */
+ assert(max <= defaultMax);
+ cSymbolTypeSizeEstimateInBits = max <= defaultMax
+ ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max)
+ : ERROR(GENERIC);
+ } else if (type == set_rle) {
+ cSymbolTypeSizeEstimateInBits = 0;
+ } else if (type == set_compressed || type == set_repeat) {
+ cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max);
+ }
+ if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10;
+ while (ctp < ctEnd) {
+ if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp];
+ else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */
+ ctp++;
+ }
+ return cSymbolTypeSizeEstimateInBits / 8;
+}
+
+static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable,
+ const BYTE* llCodeTable,
+ const BYTE* mlCodeTable,
+ size_t nbSeq,
+ const ZSTD_fseCTables_t* fseTables,
+ const ZSTD_fseCTablesMetadata_t* fseMetadata,
+ void* workspace, size_t wkspSize,
+ int writeEntropy)
+{
+ size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
+ size_t cSeqSizeEstimate = 0;
+ if (nbSeq == 0) return sequencesSectionHeaderSize;
+ cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff,
+ nbSeq, fseTables->offcodeCTable, NULL,
+ OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
+ workspace, wkspSize);
+ cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL,
+ nbSeq, fseTables->litlengthCTable, LL_bits,
+ LL_defaultNorm, LL_defaultNormLog, MaxLL,
+ workspace, wkspSize);
+ cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML,
+ nbSeq, fseTables->matchlengthCTable, ML_bits,
+ ML_defaultNorm, ML_defaultNormLog, MaxML,
+ workspace, wkspSize);
+ if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
+ return cSeqSizeEstimate + sequencesSectionHeaderSize;
+}
+
+static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize,
+ const BYTE* ofCodeTable,
+ const BYTE* llCodeTable,
+ const BYTE* mlCodeTable,
+ size_t nbSeq,
+ const ZSTD_entropyCTables_t* entropy,
+ const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ void* workspace, size_t wkspSize,
+ int writeLitEntropy, int writeSeqEntropy) {
+ size_t cSizeEstimate = 0;
+ cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize,
+ &entropy->huf, &entropyMetadata->hufMetadata,
+ workspace, wkspSize, writeLitEntropy);
+ cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
+ nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
+ workspace, wkspSize, writeSeqEntropy);
+ return cSizeEstimate + ZSTD_blockHeaderSize;
+}
+
+static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata)
+{
+ if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle)
+ return 1;
+ if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle)
+ return 1;
+ if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle)
+ return 1;
+ return 0;
+}
+
+/** ZSTD_compressSubBlock_multi() :
+ * Breaks super-block into multiple sub-blocks and compresses them.
+ * Entropy will be written to the first block.
+ * The following blocks will use repeat mode to compress.
+ * All sub-blocks are compressed blocks (no raw or rle blocks).
+ * @return : compressed size of the super block (which is multiple ZSTD blocks)
+ * Or 0 if it failed to compress. */
+static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
+ const ZSTD_compressedBlockState_t* prevCBlock,
+ ZSTD_compressedBlockState_t* nextCBlock,
+ const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+ const ZSTD_CCtx_params* cctxParams,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const int bmi2, U32 lastBlock,
+ void* workspace, size_t wkspSize)
+{
+ const seqDef* const sstart = seqStorePtr->sequencesStart;
+ const seqDef* const send = seqStorePtr->sequences;
+ const seqDef* sp = sstart;
+ const BYTE* const lstart = seqStorePtr->litStart;
+ const BYTE* const lend = seqStorePtr->lit;
+ const BYTE* lp = lstart;
+ BYTE const* ip = (BYTE const*)src;
+ BYTE const* const iend = ip + srcSize;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + dstCapacity;
+ BYTE* op = ostart;
+ const BYTE* llCodePtr = seqStorePtr->llCode;
+ const BYTE* mlCodePtr = seqStorePtr->mlCode;
+ const BYTE* ofCodePtr = seqStorePtr->ofCode;
+ size_t targetCBlockSize = cctxParams->targetCBlockSize;
+ size_t litSize, seqCount;
+ int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed;
+ int writeSeqEntropy = 1;
+ int lastSequence = 0;
+
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)",
+ (unsigned)(lend-lp), (unsigned)(send-sstart));
+
+ litSize = 0;
+ seqCount = 0;
+ do {
+ size_t cBlockSizeEstimate = 0;
+ if (sstart == send) {
+ lastSequence = 1;
+ } else {
+ const seqDef* const sequence = sp + seqCount;
+ lastSequence = sequence == send - 1;
+ litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength;
+ seqCount++;
+ }
+ if (lastSequence) {
+ assert(lp <= lend);
+ assert(litSize <= (size_t)(lend - lp));
+ litSize = (size_t)(lend - lp);
+ }
+ /* I think there is an optimization opportunity here.
+ * Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful
+ * since it recalculates estimate from scratch.
+ * For example, it would recount literal distribution and symbol codes every time.
+ */
+ cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount,
+ &nextCBlock->entropy, entropyMetadata,
+ workspace, wkspSize, writeLitEntropy, writeSeqEntropy);
+ if (cBlockSizeEstimate > targetCBlockSize || lastSequence) {
+ int litEntropyWritten = 0;
+ int seqEntropyWritten = 0;
+ const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence);
+ const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata,
+ sp, seqCount,
+ lp, litSize,
+ llCodePtr, mlCodePtr, ofCodePtr,
+ cctxParams,
+ op, oend-op,
+ bmi2, writeLitEntropy, writeSeqEntropy,
+ &litEntropyWritten, &seqEntropyWritten,
+ lastBlock && lastSequence);
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed");
+ if (cSize > 0 && cSize < decompressedSize) {
+ DEBUGLOG(5, "Committed the sub-block");
+ assert(ip + decompressedSize <= iend);
+ ip += decompressedSize;
+ sp += seqCount;
+ lp += litSize;
+ op += cSize;
+ llCodePtr += seqCount;
+ mlCodePtr += seqCount;
+ ofCodePtr += seqCount;
+ litSize = 0;
+ seqCount = 0;
+ /* Entropy only needs to be written once */
+ if (litEntropyWritten) {
+ writeLitEntropy = 0;
+ }
+ if (seqEntropyWritten) {
+ writeSeqEntropy = 0;
+ }
+ }
+ }
+ } while (!lastSequence);
+ if (writeLitEntropy) {
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten");
+ ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf));
+ }
+ if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) {
+ /* If we haven't written our entropy tables, then we've violated our contract and
+ * must emit an uncompressed block.
+ */
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten");
+ return 0;
+ }
+ if (ip < iend) {
+ size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock);
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip));
+ FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
+ assert(cSize != 0);
+ op += cSize;
+ /* We have to regenerate the repcodes because we've skipped some sequences */
+ if (sp < send) {
+ seqDef const* seq;
+ repcodes_t rep;
+ ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
+ for (seq = sstart; seq < sp; ++seq) {
+ ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
+ }
+ ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep));
+ }
+ }
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed");
+ return op-ostart;
+}
+
+size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ void const* src, size_t srcSize,
+ unsigned lastBlock) {
+ ZSTD_entropyCTablesMetadata_t entropyMetadata;
+
+ FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore,
+ &zc->blockState.prevCBlock->entropy,
+ &zc->blockState.nextCBlock->entropy,
+ &zc->appliedParams,
+ &entropyMetadata,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
+
+ return ZSTD_compressSubBlock_multi(&zc->seqStore,
+ zc->blockState.prevCBlock,
+ zc->blockState.nextCBlock,
+ &entropyMetadata,
+ &zc->appliedParams,
+ dst, dstCapacity,
+ src, srcSize,
+ zc->bmi2, lastBlock,
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
+}
diff --git a/contrib/zstd/zstd_compress_superblock.h b/contrib/zstd/zstd_compress_superblock.h
new file mode 100644
index 0000000..1ac733b
--- /dev/null
+++ b/contrib/zstd/zstd_compress_superblock.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMPRESS_ADVANCED_H
+#define ZSTD_COMPRESS_ADVANCED_H
+
+/*-*************************************
+* Dependencies
+***************************************/
+
+#include "zstd.h" /* ZSTD_CCtx */
+
+/*-*************************************
+* Target Compressed Block Size
+***************************************/
+
+/* ZSTD_compressSuperBlock() :
+ * Used to compress a super block when targetCBlockSize is being used.
+ * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */
+size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ void const* src, size_t srcSize,
+ unsigned lastBlock);
+
+#endif /* ZSTD_COMPRESS_ADVANCED_H */
diff --git a/contrib/zstd/zstd_cwksp.h b/contrib/zstd/zstd_cwksp.h
new file mode 100644
index 0000000..4d007c0
--- /dev/null
+++ b/contrib/zstd/zstd_cwksp.h
@@ -0,0 +1,678 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_CWKSP_H
+#define ZSTD_CWKSP_H
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "zstd_internal.h"
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-*************************************
+* Constants
+***************************************/
+
+/* Since the workspace is effectively its own little malloc implementation /
+ * arena, when we run under ASAN, we should similarly insert redzones between
+ * each internal element of the workspace, so ASAN will catch overruns that
+ * reach outside an object but that stay inside the workspace.
+ *
+ * This defines the size of that redzone.
+ */
+#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE
+#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128
+#endif
+
+
+/* Set our tables and aligneds to align by 64 bytes */
+#define ZSTD_CWKSP_ALIGNMENT_BYTES 64
+
+/*-*************************************
+* Structures
+***************************************/
+typedef enum {
+ ZSTD_cwksp_alloc_objects,
+ ZSTD_cwksp_alloc_buffers,
+ ZSTD_cwksp_alloc_aligned
+} ZSTD_cwksp_alloc_phase_e;
+
+/**
+ * Used to describe whether the workspace is statically allocated (and will not
+ * necessarily ever be freed), or if it's dynamically allocated and we can
+ * expect a well-formed caller to free this.
+ */
+typedef enum {
+ ZSTD_cwksp_dynamic_alloc,
+ ZSTD_cwksp_static_alloc
+} ZSTD_cwksp_static_alloc_e;
+
+/**
+ * Zstd fits all its internal datastructures into a single continuous buffer,
+ * so that it only needs to perform a single OS allocation (or so that a buffer
+ * can be provided to it and it can perform no allocations at all). This buffer
+ * is called the workspace.
+ *
+ * Several optimizations complicate that process of allocating memory ranges
+ * from this workspace for each internal datastructure:
+ *
+ * - These different internal datastructures have different setup requirements:
+ *
+ * - The static objects need to be cleared once and can then be trivially
+ * reused for each compression.
+ *
+ * - Various buffers don't need to be initialized at all--they are always
+ * written into before they're read.
+ *
+ * - The matchstate tables have a unique requirement that they don't need
+ * their memory to be totally cleared, but they do need the memory to have
+ * some bound, i.e., a guarantee that all values in the memory they've been
+ * allocated is less than some maximum value (which is the starting value
+ * for the indices that they will then use for compression). When this
+ * guarantee is provided to them, they can use the memory without any setup
+ * work. When it can't, they have to clear the area.
+ *
+ * - These buffers also have different alignment requirements.
+ *
+ * - We would like to reuse the objects in the workspace for multiple
+ * compressions without having to perform any expensive reallocation or
+ * reinitialization work.
+ *
+ * - We would like to be able to efficiently reuse the workspace across
+ * multiple compressions **even when the compression parameters change** and
+ * we need to resize some of the objects (where possible).
+ *
+ * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp
+ * abstraction was created. It works as follows:
+ *
+ * Workspace Layout:
+ *
+ * [ ... workspace ... ]
+ * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers]
+ *
+ * The various objects that live in the workspace are divided into the
+ * following categories, and are allocated separately:
+ *
+ * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict,
+ * so that literally everything fits in a single buffer. Note: if present,
+ * this must be the first object in the workspace, since ZSTD_customFree{CCtx,
+ * CDict}() rely on a pointer comparison to see whether one or two frees are
+ * required.
+ *
+ * - Fixed size objects: these are fixed-size, fixed-count objects that are
+ * nonetheless "dynamically" allocated in the workspace so that we can
+ * control how they're initialized separately from the broader ZSTD_CCtx.
+ * Examples:
+ * - Entropy Workspace
+ * - 2 x ZSTD_compressedBlockState_t
+ * - CDict dictionary contents
+ *
+ * - Tables: these are any of several different datastructures (hash tables,
+ * chain tables, binary trees) that all respect a common format: they are
+ * uint32_t arrays, all of whose values are between 0 and (nextSrc - base).
+ * Their sizes depend on the cparams. These tables are 64-byte aligned.
+ *
+ * - Aligned: these buffers are used for various purposes that require 4 byte
+ * alignment, but don't require any initialization before they're used. These
+ * buffers are each aligned to 64 bytes.
+ *
+ * - Buffers: these buffers are used for various purposes that don't require
+ * any alignment or initialization before they're used. This means they can
+ * be moved around at no cost for a new compression.
+ *
+ * Allocating Memory:
+ *
+ * The various types of objects must be allocated in order, so they can be
+ * correctly packed into the workspace buffer. That order is:
+ *
+ * 1. Objects
+ * 2. Buffers
+ * 3. Aligned/Tables
+ *
+ * Attempts to reserve objects of different types out of order will fail.
+ */
+typedef struct {
+ void* workspace;
+ void* workspaceEnd;
+
+ void* objectEnd;
+ void* tableEnd;
+ void* tableValidEnd;
+ void* allocStart;
+
+ BYTE allocFailed;
+ int workspaceOversizedDuration;
+ ZSTD_cwksp_alloc_phase_e phase;
+ ZSTD_cwksp_static_alloc_e isStatic;
+} ZSTD_cwksp;
+
+/*-*************************************
+* Functions
+***************************************/
+
+MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws);
+
+MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
+ (void)ws;
+ assert(ws->workspace <= ws->objectEnd);
+ assert(ws->objectEnd <= ws->tableEnd);
+ assert(ws->objectEnd <= ws->tableValidEnd);
+ assert(ws->tableEnd <= ws->allocStart);
+ assert(ws->tableValidEnd <= ws->allocStart);
+ assert(ws->allocStart <= ws->workspaceEnd);
+}
+
+/**
+ * Align must be a power of 2.
+ */
+MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
+ size_t const mask = align - 1;
+ assert((align & mask) == 0);
+ return (size + mask) & ~mask;
+}
+
+/**
+ * Use this to determine how much space in the workspace we will consume to
+ * allocate this object. (Normally it should be exactly the size of the object,
+ * but under special conditions, like ASAN, where we pad each object, it might
+ * be larger.)
+ *
+ * Since tables aren't currently redzoned, you don't need to call through this
+ * to figure out how much space you need for the matchState tables. Everything
+ * else is though.
+ *
+ * Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size().
+ */
+MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
+ if (size == 0)
+ return 0;
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
+#else
+ return size;
+#endif
+}
+
+/**
+ * Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes.
+ * Used to determine the number of bytes required for a given "aligned".
+ */
+MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) {
+ return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES));
+}
+
+/**
+ * Returns the amount of additional space the cwksp must allocate
+ * for internal purposes (currently only alignment).
+ */
+MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) {
+ /* For alignment, the wksp will always allocate an additional n_1=[1, 64] bytes
+ * to align the beginning of tables section, as well as another n_2=[0, 63] bytes
+ * to align the beginning of the aligned section.
+ *
+ * n_1 + n_2 == 64 bytes if the cwksp is freshly allocated, due to tables and
+ * aligneds being sized in multiples of 64 bytes.
+ */
+ size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES;
+ return slackSpace;
+}
+
+
+/**
+ * Return the number of additional bytes required to align a pointer to the given number of bytes.
+ * alignBytes must be a power of two.
+ */
+MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) {
+ size_t const alignBytesMask = alignBytes - 1;
+ size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask;
+ assert((alignBytes & alignBytesMask) == 0);
+ assert(bytes != ZSTD_CWKSP_ALIGNMENT_BYTES);
+ return bytes;
+}
+
+/**
+ * Internal function. Do not use directly.
+ * Reserves the given number of bytes within the aligned/buffer segment of the wksp,
+ * which counts from the end of the wksp (as opposed to the object/table segment).
+ *
+ * Returns a pointer to the beginning of that space.
+ */
+MEM_STATIC void*
+ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes)
+{
+ void* const alloc = (BYTE*)ws->allocStart - bytes;
+ void* const bottom = ws->tableEnd;
+ DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
+ alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
+ ZSTD_cwksp_assert_internal_consistency(ws);
+ assert(alloc >= bottom);
+ if (alloc < bottom) {
+ DEBUGLOG(4, "cwksp: alloc failed!");
+ ws->allocFailed = 1;
+ return NULL;
+ }
+ /* the area is reserved from the end of wksp.
+ * If it overlaps with tableValidEnd, it voids guarantees on values' range */
+ if (alloc < ws->tableValidEnd) {
+ ws->tableValidEnd = alloc;
+ }
+ ws->allocStart = alloc;
+ return alloc;
+}
+
+/**
+ * Moves the cwksp to the next phase, and does any necessary allocations.
+ * cwksp initialization must necessarily go through each phase in order.
+ * Returns a 0 on success, or zstd error
+ */
+MEM_STATIC size_t
+ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase)
+{
+ assert(phase >= ws->phase);
+ if (phase > ws->phase) {
+ /* Going from allocating objects to allocating buffers */
+ if (ws->phase < ZSTD_cwksp_alloc_buffers &&
+ phase >= ZSTD_cwksp_alloc_buffers) {
+ ws->tableValidEnd = ws->objectEnd;
+ }
+
+ /* Going from allocating buffers to allocating aligneds/tables */
+ if (ws->phase < ZSTD_cwksp_alloc_aligned &&
+ phase >= ZSTD_cwksp_alloc_aligned) {
+ { /* Align the start of the "aligned" to 64 bytes. Use [1, 64] bytes. */
+ size_t const bytesToAlign =
+ ZSTD_CWKSP_ALIGNMENT_BYTES - ZSTD_cwksp_bytes_to_align_ptr(ws->allocStart, ZSTD_CWKSP_ALIGNMENT_BYTES);
+ DEBUGLOG(5, "reserving aligned alignment addtl space: %zu", bytesToAlign);
+ ZSTD_STATIC_ASSERT((ZSTD_CWKSP_ALIGNMENT_BYTES & (ZSTD_CWKSP_ALIGNMENT_BYTES - 1)) == 0); /* power of 2 */
+ RETURN_ERROR_IF(!ZSTD_cwksp_reserve_internal_buffer_space(ws, bytesToAlign),
+ memory_allocation, "aligned phase - alignment initial allocation failed!");
+ }
+ { /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */
+ void* const alloc = ws->objectEnd;
+ size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES);
+ void* const objectEnd = (BYTE*)alloc + bytesToAlign;
+ DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign);
+ RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation,
+ "table phase - alignment initial allocation failed!");
+ ws->objectEnd = objectEnd;
+ ws->tableEnd = objectEnd; /* table area starts being empty */
+ if (ws->tableValidEnd < ws->tableEnd) {
+ ws->tableValidEnd = ws->tableEnd;
+ } } }
+ ws->phase = phase;
+ ZSTD_cwksp_assert_internal_consistency(ws);
+ }
+ return 0;
+}
+
+/**
+ * Returns whether this object/buffer/etc was allocated in this workspace.
+ */
+MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr)
+{
+ return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
+}
+
+/**
+ * Internal function. Do not use directly.
+ */
+MEM_STATIC void*
+ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase)
+{
+ void* alloc;
+ if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase)) || bytes == 0) {
+ return NULL;
+ }
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* over-reserve space */
+ bytes += 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
+#endif
+
+ alloc = ZSTD_cwksp_reserve_internal_buffer_space(ws, bytes);
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
+ * either size. */
+ if (alloc) {
+ alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
+ if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
+ /* We need to keep the redzone poisoned while unpoisoning the bytes that
+ * are actually allocated. */
+ __asan_unpoison_memory_region(alloc, bytes - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE);
+ }
+ }
+#endif
+
+ return alloc;
+}
+
+/**
+ * Reserves and returns unaligned memory.
+ */
+MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes)
+{
+ return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers);
+}
+
+/**
+ * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
+ */
+MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes)
+{
+ void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES),
+ ZSTD_cwksp_alloc_aligned);
+ assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
+ return ptr;
+}
+
+/**
+ * Aligned on 64 bytes. These buffers have the special property that
+ * their values remain constrained, allowing us to re-use them without
+ * memset()-ing them.
+ */
+MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes)
+{
+ const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned;
+ void* alloc;
+ void* end;
+ void* top;
+
+ if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) {
+ return NULL;
+ }
+ alloc = ws->tableEnd;
+ end = (BYTE *)alloc + bytes;
+ top = ws->allocStart;
+
+ DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining",
+ alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
+ assert((bytes & (sizeof(U32)-1)) == 0);
+ ZSTD_cwksp_assert_internal_consistency(ws);
+ assert(end <= top);
+ if (end > top) {
+ DEBUGLOG(4, "cwksp: table alloc failed!");
+ ws->allocFailed = 1;
+ return NULL;
+ }
+ ws->tableEnd = end;
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
+ __asan_unpoison_memory_region(alloc, bytes);
+ }
+#endif
+
+ assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
+ assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
+ return alloc;
+}
+
+/**
+ * Aligned on sizeof(void*).
+ * Note : should happen only once, at workspace first initialization
+ */
+MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes)
+{
+ size_t const roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*));
+ void* alloc = ws->objectEnd;
+ void* end = (BYTE*)alloc + roundedBytes;
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* over-reserve space */
+ end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
+#endif
+
+ DEBUGLOG(4,
+ "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining",
+ alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes);
+ assert((size_t)alloc % ZSTD_ALIGNOF(void*) == 0);
+ assert(bytes % ZSTD_ALIGNOF(void*) == 0);
+ ZSTD_cwksp_assert_internal_consistency(ws);
+ /* we must be in the first phase, no advance is possible */
+ if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) {
+ DEBUGLOG(3, "cwksp: object alloc failed!");
+ ws->allocFailed = 1;
+ return NULL;
+ }
+ ws->objectEnd = end;
+ ws->tableEnd = end;
+ ws->tableValidEnd = end;
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
+ * either size. */
+ alloc = (BYTE*)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
+ if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
+ __asan_unpoison_memory_region(alloc, bytes);
+ }
+#endif
+
+ return alloc;
+}
+
+MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws)
+{
+ DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty");
+
+#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
+ /* To validate that the table re-use logic is sound, and that we don't
+ * access table space that we haven't cleaned, we re-"poison" the table
+ * space every time we mark it dirty. */
+ {
+ size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
+ assert(__msan_test_shadow(ws->objectEnd, size) == -1);
+ __msan_poison(ws->objectEnd, size);
+ }
+#endif
+
+ assert(ws->tableValidEnd >= ws->objectEnd);
+ assert(ws->tableValidEnd <= ws->allocStart);
+ ws->tableValidEnd = ws->objectEnd;
+ ZSTD_cwksp_assert_internal_consistency(ws);
+}
+
+MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) {
+ DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean");
+ assert(ws->tableValidEnd >= ws->objectEnd);
+ assert(ws->tableValidEnd <= ws->allocStart);
+ if (ws->tableValidEnd < ws->tableEnd) {
+ ws->tableValidEnd = ws->tableEnd;
+ }
+ ZSTD_cwksp_assert_internal_consistency(ws);
+}
+
+/**
+ * Zero the part of the allocated tables not already marked clean.
+ */
+MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
+ DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables");
+ assert(ws->tableValidEnd >= ws->objectEnd);
+ assert(ws->tableValidEnd <= ws->allocStart);
+ if (ws->tableValidEnd < ws->tableEnd) {
+ ZSTD_memset(ws->tableValidEnd, 0, (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd));
+ }
+ ZSTD_cwksp_mark_tables_clean(ws);
+}
+
+/**
+ * Invalidates table allocations.
+ * All other allocations remain valid.
+ */
+MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
+ DEBUGLOG(4, "cwksp: clearing tables!");
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* We don't do this when the workspace is statically allocated, because
+ * when that is the case, we have no capability to hook into the end of the
+ * workspace's lifecycle to unpoison the memory.
+ */
+ if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
+ size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
+ __asan_poison_memory_region(ws->objectEnd, size);
+ }
+#endif
+
+ ws->tableEnd = ws->objectEnd;
+ ZSTD_cwksp_assert_internal_consistency(ws);
+}
+
+/**
+ * Invalidates all buffer, aligned, and table allocations.
+ * Object allocations remain valid.
+ */
+MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
+ DEBUGLOG(4, "cwksp: clearing!");
+
+#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
+ /* To validate that the context re-use logic is sound, and that we don't
+ * access stuff that this compression hasn't initialized, we re-"poison"
+ * the workspace (or at least the non-static, non-table parts of it)
+ * every time we start a new compression. */
+ {
+ size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd;
+ __msan_poison(ws->tableValidEnd, size);
+ }
+#endif
+
+#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
+ /* We don't do this when the workspace is statically allocated, because
+ * when that is the case, we have no capability to hook into the end of the
+ * workspace's lifecycle to unpoison the memory.
+ */
+ if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
+ size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd;
+ __asan_poison_memory_region(ws->objectEnd, size);
+ }
+#endif
+
+ ws->tableEnd = ws->objectEnd;
+ ws->allocStart = ws->workspaceEnd;
+ ws->allocFailed = 0;
+ if (ws->phase > ZSTD_cwksp_alloc_buffers) {
+ ws->phase = ZSTD_cwksp_alloc_buffers;
+ }
+ ZSTD_cwksp_assert_internal_consistency(ws);
+}
+
+/**
+ * The provided workspace takes ownership of the buffer [start, start+size).
+ * Any existing values in the workspace are ignored (the previously managed
+ * buffer, if present, must be separately freed).
+ */
+MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_cwksp_static_alloc_e isStatic) {
+ DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size);
+ assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */
+ ws->workspace = start;
+ ws->workspaceEnd = (BYTE*)start + size;
+ ws->objectEnd = ws->workspace;
+ ws->tableValidEnd = ws->objectEnd;
+ ws->phase = ZSTD_cwksp_alloc_objects;
+ ws->isStatic = isStatic;
+ ZSTD_cwksp_clear(ws);
+ ws->workspaceOversizedDuration = 0;
+ ZSTD_cwksp_assert_internal_consistency(ws);
+}
+
+MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) {
+ void* workspace = ZSTD_customMalloc(size, customMem);
+ DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size);
+ RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!");
+ ZSTD_cwksp_init(ws, workspace, size, ZSTD_cwksp_dynamic_alloc);
+ return 0;
+}
+
+MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) {
+ void *ptr = ws->workspace;
+ DEBUGLOG(4, "cwksp: freeing workspace");
+ ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp));
+ ZSTD_customFree(ptr, customMem);
+}
+
+/**
+ * Moves the management of a workspace from one cwksp to another. The src cwksp
+ * is left in an invalid state (src must be re-init()'ed before it's used again).
+ */
+MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
+ *dst = *src;
+ ZSTD_memset(src, 0, sizeof(ZSTD_cwksp));
+}
+
+MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
+ return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
+}
+
+MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) {
+ return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace)
+ + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart);
+}
+
+MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
+ return ws->allocFailed;
+}
+
+/*-*************************************
+* Functions Checking Free Space
+***************************************/
+
+/* ZSTD_alignmentSpaceWithinBounds() :
+ * Returns if the estimated space needed for a wksp is within an acceptable limit of the
+ * actual amount of space used.
+ */
+MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp* const ws,
+ size_t const estimatedSpace, int resizedWorkspace) {
+ if (resizedWorkspace) {
+ /* Resized/newly allocated wksp should have exact bounds */
+ return ZSTD_cwksp_used(ws) == estimatedSpace;
+ } else {
+ /* Due to alignment, when reusing a workspace, we can actually consume 63 fewer or more bytes
+ * than estimatedSpace. See the comments in zstd_cwksp.h for details.
+ */
+ return (ZSTD_cwksp_used(ws) >= estimatedSpace - 63) && (ZSTD_cwksp_used(ws) <= estimatedSpace + 63);
+ }
+}
+
+
+MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) {
+ return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd);
+}
+
+MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
+ return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace;
+}
+
+MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
+ return ZSTD_cwksp_check_available(
+ ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR);
+}
+
+MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
+ return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)
+ && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION;
+}
+
+MEM_STATIC void ZSTD_cwksp_bump_oversized_duration(
+ ZSTD_cwksp* ws, size_t additionalNeededSpace) {
+ if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) {
+ ws->workspaceOversizedDuration++;
+ } else {
+ ws->workspaceOversizedDuration = 0;
+ }
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_CWKSP_H */
diff --git a/contrib/zstd/zstd_ddict.c b/contrib/zstd/zstd_ddict.c
new file mode 100644
index 0000000..06bd996
--- /dev/null
+++ b/contrib/zstd/zstd_ddict.c
@@ -0,0 +1,243 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* zstd_ddict.c :
+ * concentrates all logic that needs to know the internals of ZSTD_DDict object */
+
+/*-*******************************************************
+* Dependencies
+*********************************************************/
+#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
+#include "cpu.h" /* bmi2 */
+#include "mem.h" /* low level memory routines */
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "huf.h"
+#include "zstd_decompress_internal.h"
+#include "zstd_ddict.h"
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+# include "../legacy/zstd_legacy.h"
+#endif
+
+
+
+/*-*******************************************************
+* Types
+*********************************************************/
+struct ZSTD_DDict_s {
+ void* dictBuffer;
+ const void* dictContent;
+ size_t dictSize;
+ ZSTD_entropyDTables_t entropy;
+ U32 dictID;
+ U32 entropyPresent;
+ ZSTD_customMem cMem;
+}; /* typedef'd to ZSTD_DDict within "zstd.h" */
+
+const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
+{
+ assert(ddict != NULL);
+ return ddict->dictContent;
+}
+
+size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
+{
+ assert(ddict != NULL);
+ return ddict->dictSize;
+}
+
+void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
+{
+ DEBUGLOG(4, "ZSTD_copyDDictParameters");
+ assert(dctx != NULL);
+ assert(ddict != NULL);
+ dctx->dictID = ddict->dictID;
+ dctx->prefixStart = ddict->dictContent;
+ dctx->virtualStart = ddict->dictContent;
+ dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
+ dctx->previousDstEnd = dctx->dictEnd;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ dctx->dictContentBeginForFuzzing = dctx->prefixStart;
+ dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
+#endif
+ if (ddict->entropyPresent) {
+ dctx->litEntropy = 1;
+ dctx->fseEntropy = 1;
+ dctx->LLTptr = ddict->entropy.LLTable;
+ dctx->MLTptr = ddict->entropy.MLTable;
+ dctx->OFTptr = ddict->entropy.OFTable;
+ dctx->HUFptr = ddict->entropy.hufTable;
+ dctx->entropy.rep[0] = ddict->entropy.rep[0];
+ dctx->entropy.rep[1] = ddict->entropy.rep[1];
+ dctx->entropy.rep[2] = ddict->entropy.rep[2];
+ } else {
+ dctx->litEntropy = 0;
+ dctx->fseEntropy = 0;
+ }
+}
+
+
+static size_t
+ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
+ ZSTD_dictContentType_e dictContentType)
+{
+ ddict->dictID = 0;
+ ddict->entropyPresent = 0;
+ if (dictContentType == ZSTD_dct_rawContent) return 0;
+
+ if (ddict->dictSize < 8) {
+ if (dictContentType == ZSTD_dct_fullDict)
+ return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
+ return 0; /* pure content mode */
+ }
+ { U32 const magic = MEM_readLE32(ddict->dictContent);
+ if (magic != ZSTD_MAGIC_DICTIONARY) {
+ if (dictContentType == ZSTD_dct_fullDict)
+ return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
+ return 0; /* pure content mode */
+ }
+ }
+ ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
+
+ /* load entropy tables */
+ RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
+ &ddict->entropy, ddict->dictContent, ddict->dictSize)),
+ dictionary_corrupted, "");
+ ddict->entropyPresent = 1;
+ return 0;
+}
+
+
+static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType)
+{
+ if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
+ ddict->dictBuffer = NULL;
+ ddict->dictContent = dict;
+ if (!dict) dictSize = 0;
+ } else {
+ void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem);
+ ddict->dictBuffer = internalBuffer;
+ ddict->dictContent = internalBuffer;
+ if (!internalBuffer) return ERROR(memory_allocation);
+ ZSTD_memcpy(internalBuffer, dict, dictSize);
+ }
+ ddict->dictSize = dictSize;
+ ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */
+
+ /* parse dictionary content */
+ FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
+
+ return 0;
+}
+
+ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType,
+ ZSTD_customMem customMem)
+{
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+
+ { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem);
+ if (ddict == NULL) return NULL;
+ ddict->cMem = customMem;
+ { size_t const initResult = ZSTD_initDDict_internal(ddict,
+ dict, dictSize,
+ dictLoadMethod, dictContentType);
+ if (ZSTD_isError(initResult)) {
+ ZSTD_freeDDict(ddict);
+ return NULL;
+ } }
+ return ddict;
+ }
+}
+
+/*! ZSTD_createDDict() :
+* Create a digested dictionary, to start decompression without startup delay.
+* `dict` content is copied inside DDict.
+* Consequently, `dict` can be released after `ZSTD_DDict` creation */
+ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
+{
+ ZSTD_customMem const allocator = { NULL, NULL, NULL };
+ return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
+}
+
+/*! ZSTD_createDDict_byReference() :
+ * Create a digested dictionary, to start decompression without startup delay.
+ * Dictionary content is simply referenced, it will be accessed during decompression.
+ * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
+ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
+{
+ ZSTD_customMem const allocator = { NULL, NULL, NULL };
+ return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
+}
+
+
+const ZSTD_DDict* ZSTD_initStaticDDict(
+ void* sBuffer, size_t sBufferSize,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType)
+{
+ size_t const neededSpace = sizeof(ZSTD_DDict)
+ + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
+ ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
+ assert(sBuffer != NULL);
+ assert(dict != NULL);
+ if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
+ if (sBufferSize < neededSpace) return NULL;
+ if (dictLoadMethod == ZSTD_dlm_byCopy) {
+ ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */
+ dict = ddict+1;
+ }
+ if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
+ dict, dictSize,
+ ZSTD_dlm_byRef, dictContentType) ))
+ return NULL;
+ return ddict;
+}
+
+
+size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
+{
+ if (ddict==NULL) return 0; /* support free on NULL */
+ { ZSTD_customMem const cMem = ddict->cMem;
+ ZSTD_customFree(ddict->dictBuffer, cMem);
+ ZSTD_customFree(ddict, cMem);
+ return 0;
+ }
+}
+
+/*! ZSTD_estimateDDictSize() :
+ * Estimate amount of memory that will be needed to create a dictionary for decompression.
+ * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
+size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
+{
+ return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
+}
+
+size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
+{
+ if (ddict==NULL) return 0; /* support sizeof on NULL */
+ return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
+}
+
+/*! ZSTD_getDictID_fromDDict() :
+ * Provides the dictID of the dictionary loaded into `ddict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
+{
+ if (ddict==NULL) return 0;
+ return ddict->dictID;
+}
diff --git a/contrib/zstd/zstd_ddict.h b/contrib/zstd/zstd_ddict.h
new file mode 100644
index 0000000..f49c570
--- /dev/null
+++ b/contrib/zstd/zstd_ddict.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#ifndef ZSTD_DDICT_H
+#define ZSTD_DDICT_H
+
+/*-*******************************************************
+ * Dependencies
+ *********************************************************/
+#include "zstd_deps.h" /* size_t */
+#include "zstd.h" /* ZSTD_DDict, and several public functions */
+
+
+/*-*******************************************************
+ * Interface
+ *********************************************************/
+
+/* note: several prototypes are already published in `zstd.h` :
+ * ZSTD_createDDict()
+ * ZSTD_createDDict_byReference()
+ * ZSTD_createDDict_advanced()
+ * ZSTD_freeDDict()
+ * ZSTD_initStaticDDict()
+ * ZSTD_sizeof_DDict()
+ * ZSTD_estimateDDictSize()
+ * ZSTD_getDictID_fromDict()
+ */
+
+const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict);
+size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict);
+
+void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+
+
+#endif /* ZSTD_DDICT_H */
diff --git a/contrib/zstd/zstd_decompress.c b/contrib/zstd/zstd_decompress.c
new file mode 100644
index 0000000..05704ce
--- /dev/null
+++ b/contrib/zstd/zstd_decompress.c
@@ -0,0 +1,2352 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* ***************************************************************
+* Tuning parameters
+*****************************************************************/
+/*!
+ * HEAPMODE :
+ * Select how default decompression function ZSTD_decompress() allocates its context,
+ * on stack (0), or into heap (1, default; requires malloc()).
+ * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected.
+ */
+#ifndef ZSTD_HEAPMODE
+# define ZSTD_HEAPMODE 1
+#endif
+
+/*!
+* LEGACY_SUPPORT :
+* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+)
+*/
+#ifndef ZSTD_LEGACY_SUPPORT
+# define ZSTD_LEGACY_SUPPORT 0
+#endif
+
+/*!
+ * MAXWINDOWSIZE_DEFAULT :
+ * maximum window size accepted by DStream __by default__.
+ * Frames requiring more memory will be rejected.
+ * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize().
+ */
+#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
+# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1)
+#endif
+
+/*!
+ * NO_FORWARD_PROGRESS_MAX :
+ * maximum allowed nb of calls to ZSTD_decompressStream()
+ * without any forward progress
+ * (defined as: no byte read from input, and no byte flushed to output)
+ * before triggering an error.
+ */
+#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX
+# define ZSTD_NO_FORWARD_PROGRESS_MAX 16
+#endif
+
+
+/*-*******************************************************
+* Dependencies
+*********************************************************/
+#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
+#include "mem.h" /* low level memory routines */
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "huf.h"
+#include "xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
+#include "zstd_internal.h" /* blockProperties_t */
+#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
+#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
+#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */
+#include "bits.h" /* ZSTD_highbit32 */
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+# include "../legacy/zstd_legacy.h"
+#endif
+
+
+
+/*************************************
+ * Multiple DDicts Hashset internals *
+ *************************************/
+
+#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4
+#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
+ * Currently, that means a 0.75 load factor.
+ * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
+ * the load factor of the ddict hash set.
+ */
+
+#define DDICT_HASHSET_TABLE_BASE_SIZE 64
+#define DDICT_HASHSET_RESIZE_FACTOR 2
+
+/* Hash function to determine starting position of dict insertion within the table
+ * Returns an index between [0, hashSet->ddictPtrTableSize]
+ */
+static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) {
+ const U64 hash = XXH64(&dictID, sizeof(U32), 0);
+ /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */
+ return hash & (hashSet->ddictPtrTableSize - 1);
+}
+
+/* Adds DDict to a hashset without resizing it.
+ * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set.
+ * Returns 0 if successful, or a zstd error code if something went wrong.
+ */
+static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) {
+ const U32 dictID = ZSTD_getDictID_fromDDict(ddict);
+ size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+ const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
+ RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
+ DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
+ while (hashSet->ddictPtrTable[idx] != NULL) {
+ /* Replace existing ddict if inserting ddict with same dictID */
+ if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) {
+ DEBUGLOG(4, "DictID already exists, replacing rather than adding");
+ hashSet->ddictPtrTable[idx] = ddict;
+ return 0;
+ }
+ idx &= idxRangeMask;
+ idx++;
+ }
+ DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
+ hashSet->ddictPtrTable[idx] = ddict;
+ hashSet->ddictPtrCount++;
+ return 0;
+}
+
+/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and
+ * rehashes all values, allocates new table, frees old table.
+ * Returns 0 on success, otherwise a zstd error code.
+ */
+static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
+ size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR;
+ const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem);
+ const ZSTD_DDict** oldTable = hashSet->ddictPtrTable;
+ size_t oldTableSize = hashSet->ddictPtrTableSize;
+ size_t i;
+
+ DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize);
+ RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
+ hashSet->ddictPtrTable = newTable;
+ hashSet->ddictPtrTableSize = newTableSize;
+ hashSet->ddictPtrCount = 0;
+ for (i = 0; i < oldTableSize; ++i) {
+ if (oldTable[i] != NULL) {
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), "");
+ }
+ }
+ ZSTD_customFree((void*)oldTable, customMem);
+ DEBUGLOG(4, "Finished re-hash");
+ return 0;
+}
+
+/* Fetches a DDict with the given dictID
+ * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL.
+ */
+static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) {
+ size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
+ const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
+ DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
+ for (;;) {
+ size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
+ if (currDictID == dictID || currDictID == 0) {
+ /* currDictID == 0 implies a NULL ddict entry */
+ break;
+ } else {
+ idx &= idxRangeMask; /* Goes to start of table when we reach the end */
+ idx++;
+ }
+ }
+ DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
+ return hashSet->ddictPtrTable[idx];
+}
+
+/* Allocates space for and returns a ddict hash set
+ * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with.
+ * Returns NULL if allocation failed.
+ */
+static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
+ ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
+ DEBUGLOG(4, "Allocating new hash set");
+ if (!ret)
+ return NULL;
+ ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem);
+ if (!ret->ddictPtrTable) {
+ ZSTD_customFree(ret, customMem);
+ return NULL;
+ }
+ ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
+ ret->ddictPtrCount = 0;
+ return ret;
+}
+
+/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself.
+ * Note: The ZSTD_DDict* within the table are NOT freed.
+ */
+static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
+ DEBUGLOG(4, "Freeing ddict hash set");
+ if (hashSet && hashSet->ddictPtrTable) {
+ ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem);
+ }
+ if (hashSet) {
+ ZSTD_customFree(hashSet, customMem);
+ }
+}
+
+/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
+ * Returns 0 on success, or a ZSTD error.
+ */
+static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) {
+ DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
+ if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) {
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
+ }
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), "");
+ return 0;
+}
+
+/*-*************************************************************
+* Context management
+***************************************************************/
+size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
+{
+ if (dctx==NULL) return 0; /* support sizeof NULL */
+ return sizeof(*dctx)
+ + ZSTD_sizeof_DDict(dctx->ddictLocal)
+ + dctx->inBuffSize + dctx->outBuffSize;
+}
+
+size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
+
+
+static size_t ZSTD_startingInputLength(ZSTD_format_e format)
+{
+ size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format);
+ /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
+ assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) );
+ return startingInputLength;
+}
+
+static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
+{
+ assert(dctx->streamStage == zdss_init);
+ dctx->format = ZSTD_f_zstd1;
+ dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
+ dctx->outBufferMode = ZSTD_bm_buffered;
+ dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
+ dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
+ dctx->disableHufAsm = 0;
+}
+
+static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
+{
+ dctx->staticSize = 0;
+ dctx->ddict = NULL;
+ dctx->ddictLocal = NULL;
+ dctx->dictEnd = NULL;
+ dctx->ddictIsCold = 0;
+ dctx->dictUses = ZSTD_dont_use;
+ dctx->inBuff = NULL;
+ dctx->inBuffSize = 0;
+ dctx->outBuffSize = 0;
+ dctx->streamStage = zdss_init;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+ dctx->legacyContext = NULL;
+ dctx->previousLegacyVersion = 0;
+#endif
+ dctx->noForwardProgress = 0;
+ dctx->oversizedDuration = 0;
+#if DYNAMIC_BMI2
+ dctx->bmi2 = ZSTD_cpuSupportsBmi2();
+#endif
+ dctx->ddictSet = NULL;
+ ZSTD_DCtx_resetParameters(dctx);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ dctx->dictContentEndForFuzzing = NULL;
+#endif
+}
+
+ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
+{
+ ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace;
+
+ if ((size_t)workspace & 7) return NULL; /* 8-aligned */
+ if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */
+
+ ZSTD_initDCtx_internal(dctx);
+ dctx->staticSize = workspaceSize;
+ dctx->inBuff = (char*)(dctx+1);
+ return dctx;
+}
+
+static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) {
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
+
+ { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem);
+ if (!dctx) return NULL;
+ dctx->customMem = customMem;
+ ZSTD_initDCtx_internal(dctx);
+ return dctx;
+ }
+}
+
+ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
+{
+ return ZSTD_createDCtx_internal(customMem);
+}
+
+ZSTD_DCtx* ZSTD_createDCtx(void)
+{
+ DEBUGLOG(3, "ZSTD_createDCtx");
+ return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
+}
+
+static void ZSTD_clearDict(ZSTD_DCtx* dctx)
+{
+ ZSTD_freeDDict(dctx->ddictLocal);
+ dctx->ddictLocal = NULL;
+ dctx->ddict = NULL;
+ dctx->dictUses = ZSTD_dont_use;
+}
+
+size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
+{
+ if (dctx==NULL) return 0; /* support free on NULL */
+ RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");
+ { ZSTD_customMem const cMem = dctx->customMem;
+ ZSTD_clearDict(dctx);
+ ZSTD_customFree(dctx->inBuff, cMem);
+ dctx->inBuff = NULL;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+ if (dctx->legacyContext)
+ ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
+#endif
+ if (dctx->ddictSet) {
+ ZSTD_freeDDictHashSet(dctx->ddictSet, cMem);
+ dctx->ddictSet = NULL;
+ }
+ ZSTD_customFree(dctx, cMem);
+ return 0;
+ }
+}
+
+/* no longer useful */
+void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
+{
+ size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);
+ ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */
+}
+
+/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on
+ * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then
+ * accordingly sets the ddict to be used to decompress the frame.
+ *
+ * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is.
+ *
+ * ZSTD_d_refMultipleDDicts must be enabled for this function to be called.
+ */
+static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) {
+ assert(dctx->refMultipleDDicts && dctx->ddictSet);
+ DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame");
+ if (dctx->ddict) {
+ const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
+ if (frameDDict) {
+ DEBUGLOG(4, "DDict found!");
+ ZSTD_clearDict(dctx);
+ dctx->dictID = dctx->fParams.dictID;
+ dctx->ddict = frameDDict;
+ dctx->dictUses = ZSTD_use_indefinitely;
+ }
+ }
+}
+
+
+/*-*************************************************************
+ * Frame header decoding
+ ***************************************************************/
+
+/*! ZSTD_isFrame() :
+ * Tells if the content of `buffer` starts with a valid Frame Identifier.
+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ * Note 3 : Skippable Frame Identifiers are considered valid. */
+unsigned ZSTD_isFrame(const void* buffer, size_t size)
+{
+ if (size < ZSTD_FRAMEIDSIZE) return 0;
+ { U32 const magic = MEM_readLE32(buffer);
+ if (magic == ZSTD_MAGICNUMBER) return 1;
+ if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
+ }
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+ if (ZSTD_isLegacy(buffer, size)) return 1;
+#endif
+ return 0;
+}
+
+/*! ZSTD_isSkippableFrame() :
+ * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ */
+unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size)
+{
+ if (size < ZSTD_FRAMEIDSIZE) return 0;
+ { U32 const magic = MEM_readLE32(buffer);
+ if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
+ }
+ return 0;
+}
+
+/** ZSTD_frameHeaderSize_internal() :
+ * srcSize must be large enough to reach header size fields.
+ * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless.
+ * @return : size of the Frame Header
+ * or an error code, which can be tested with ZSTD_isError() */
+static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format)
+{
+ size_t const minInputSize = ZSTD_startingInputLength(format);
+ RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, "");
+
+ { BYTE const fhd = ((const BYTE*)src)[minInputSize-1];
+ U32 const dictID= fhd & 3;
+ U32 const singleSegment = (fhd >> 5) & 1;
+ U32 const fcsId = fhd >> 6;
+ return minInputSize + !singleSegment
+ + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
+ + (singleSegment && !fcsId);
+ }
+}
+
+/** ZSTD_frameHeaderSize() :
+ * srcSize must be >= ZSTD_frameHeaderSize_prefix.
+ * @return : size of the Frame Header,
+ * or an error code (if srcSize is too small) */
+size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
+{
+ return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1);
+}
+
+
+/** ZSTD_getFrameHeader_advanced() :
+ * decode Frame Header, or require larger `srcSize`.
+ * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless
+ * @return : 0, `zfhPtr` is correctly filled,
+ * >0, `srcSize` is too small, value is wanted `srcSize` amount,
+** or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
+{
+ const BYTE* ip = (const BYTE*)src;
+ size_t const minInputSize = ZSTD_startingInputLength(format);
+
+ DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize);
+
+ if (srcSize > 0) {
+ /* note : technically could be considered an assert(), since it's an invalid entry */
+ RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0");
+ }
+ if (srcSize < minInputSize) {
+ if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) {
+ /* when receiving less than @minInputSize bytes,
+ * control these bytes at least correspond to a supported magic number
+ * in order to error out early if they don't.
+ **/
+ size_t const toCopy = MIN(4, srcSize);
+ unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER);
+ assert(src != NULL);
+ ZSTD_memcpy(hbuf, src, toCopy);
+ if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) {
+ /* not a zstd frame : let's check if it's a skippable frame */
+ MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START);
+ ZSTD_memcpy(hbuf, src, toCopy);
+ if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) {
+ RETURN_ERROR(prefix_unknown,
+ "first bytes don't correspond to any supported magic number");
+ } } }
+ return minInputSize;
+ }
+
+ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */
+ if ( (format != ZSTD_f_zstd1_magicless)
+ && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
+ if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+ /* skippable frame */
+ if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
+ return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
+ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));
+ zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
+ zfhPtr->frameType = ZSTD_skippableFrame;
+ return 0;
+ }
+ RETURN_ERROR(prefix_unknown, "");
+ }
+
+ /* ensure there is enough `srcSize` to fully read/decode frame header */
+ { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format);
+ if (srcSize < fhsize) return fhsize;
+ zfhPtr->headerSize = (U32)fhsize;
+ }
+
+ { BYTE const fhdByte = ip[minInputSize-1];
+ size_t pos = minInputSize;
+ U32 const dictIDSizeCode = fhdByte&3;
+ U32 const checksumFlag = (fhdByte>>2)&1;
+ U32 const singleSegment = (fhdByte>>5)&1;
+ U32 const fcsID = fhdByte>>6;
+ U64 windowSize = 0;
+ U32 dictID = 0;
+ U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
+ RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported,
+ "reserved bits, must be zero");
+
+ if (!singleSegment) {
+ BYTE const wlByte = ip[pos++];
+ U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
+ RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, "");
+ windowSize = (1ULL << windowLog);
+ windowSize += (windowSize >> 3) * (wlByte&7);
+ }
+ switch(dictIDSizeCode)
+ {
+ default:
+ assert(0); /* impossible */
+ ZSTD_FALLTHROUGH;
+ case 0 : break;
+ case 1 : dictID = ip[pos]; pos++; break;
+ case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
+ case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
+ }
+ switch(fcsID)
+ {
+ default:
+ assert(0); /* impossible */
+ ZSTD_FALLTHROUGH;
+ case 0 : if (singleSegment) frameContentSize = ip[pos]; break;
+ case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
+ case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
+ case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
+ }
+ if (singleSegment) windowSize = frameContentSize;
+
+ zfhPtr->frameType = ZSTD_frame;
+ zfhPtr->frameContentSize = frameContentSize;
+ zfhPtr->windowSize = windowSize;
+ zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
+ zfhPtr->dictID = dictID;
+ zfhPtr->checksumFlag = checksumFlag;
+ }
+ return 0;
+}
+
+/** ZSTD_getFrameHeader() :
+ * decode Frame Header, or require larger `srcSize`.
+ * note : this function does not consume input, it only reads it.
+ * @return : 0, `zfhPtr` is correctly filled,
+ * >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ * or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
+{
+ return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);
+}
+
+/** ZSTD_getFrameContentSize() :
+ * compatible with legacy mode
+ * @return : decompressed size of the single frame pointed to be `src` if known, otherwise
+ * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+ * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
+unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
+{
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+ if (ZSTD_isLegacy(src, srcSize)) {
+ unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize);
+ return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
+ }
+#endif
+ { ZSTD_frameHeader zfh;
+ if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0)
+ return ZSTD_CONTENTSIZE_ERROR;
+ if (zfh.frameType == ZSTD_skippableFrame) {
+ return 0;
+ } else {
+ return zfh.frameContentSize;
+ } }
+}
+
+static size_t readSkippableFrameSize(void const* src, size_t srcSize)
+{
+ size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE;
+ U32 sizeU32;
+
+ RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, "");
+
+ sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE);
+ RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32,
+ frameParameter_unsupported, "");
+ {
+ size_t const skippableSize = skippableHeaderSize + sizeU32;
+ RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, "");
+ return skippableSize;
+ }
+}
+
+/*! ZSTD_readSkippableFrame() :
+ * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ *
+ * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
+ * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
+ * in the magicVariant.
+ *
+ * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
+ *
+ * @return : number of bytes written or a ZSTD error.
+ */
+ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
+ const void* src, size_t srcSize)
+{
+ U32 const magicNumber = MEM_readLE32(src);
+ size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
+ size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
+
+ /* check input validity */
+ RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
+ RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
+ RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
+
+ /* deliver payload */
+ if (skippableContentSize > 0 && dst != NULL)
+ ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
+ if (magicVariant != NULL)
+ *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
+ return skippableContentSize;
+}
+
+/** ZSTD_findDecompressedSize() :
+ * compatible with legacy mode
+ * `srcSize` must be the exact length of some number of ZSTD compressed and/or
+ * skippable frames
+ * @return : decompressed size of the frames contained */
+unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
+{
+ unsigned long long totalDstSize = 0;
+
+ while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) {
+ U32 const magicNumber = MEM_readLE32(src);
+
+ if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+ size_t const skippableSize = readSkippableFrameSize(src, srcSize);
+ if (ZSTD_isError(skippableSize)) {
+ return ZSTD_CONTENTSIZE_ERROR;
+ }
+ assert(skippableSize <= srcSize);
+
+ src = (const BYTE *)src + skippableSize;
+ srcSize -= skippableSize;
+ continue;
+ }
+
+ { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
+ if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
+
+ /* check for overflow */
+ if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
+ totalDstSize += ret;
+ }
+ { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
+ if (ZSTD_isError(frameSrcSize)) {
+ return ZSTD_CONTENTSIZE_ERROR;
+ }
+
+ src = (const BYTE *)src + frameSrcSize;
+ srcSize -= frameSrcSize;
+ }
+ } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
+
+ if (srcSize) return ZSTD_CONTENTSIZE_ERROR;
+
+ return totalDstSize;
+}
+
+/** ZSTD_getDecompressedSize() :
+ * compatible with legacy mode
+ * @return : decompressed size if known, 0 otherwise
+ note : 0 can mean any of the following :
+ - frame content is empty
+ - decompressed size field is not present in frame header
+ - frame header unknown / not supported
+ - frame header not complete (`srcSize` too small) */
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
+{
+ unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
+ ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN);
+ return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret;
+}
+
+
+/** ZSTD_decodeFrameHeader() :
+ * `headerSize` must be the size provided by ZSTD_frameHeaderSize().
+ * If multiple DDict references are enabled, also will choose the correct DDict to use.
+ * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
+static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
+{
+ size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);
+ if (ZSTD_isError(result)) return result; /* invalid header */
+ RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
+
+ /* Reference DDict requested by frame if dctx references multiple ddicts */
+ if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) {
+ ZSTD_DCtx_selectFrameDDict(dctx);
+ }
+
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ /* Skip the dictID check in fuzzing mode, because it makes the search
+ * harder.
+ */
+ RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),
+ dictionary_wrong, "");
+#endif
+ dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0;
+ if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0);
+ dctx->processedCSize += headerSize;
+ return 0;
+}
+
+static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret)
+{
+ ZSTD_frameSizeInfo frameSizeInfo;
+ frameSizeInfo.compressedSize = ret;
+ frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
+ return frameSizeInfo;
+}
+
+static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize)
+{
+ ZSTD_frameSizeInfo frameSizeInfo;
+ ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+ if (ZSTD_isLegacy(src, srcSize))
+ return ZSTD_findFrameSizeInfoLegacy(src, srcSize);
+#endif
+
+ if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
+ && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+ frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize);
+ assert(ZSTD_isError(frameSizeInfo.compressedSize) ||
+ frameSizeInfo.compressedSize <= srcSize);
+ return frameSizeInfo;
+ } else {
+ const BYTE* ip = (const BYTE*)src;
+ const BYTE* const ipstart = ip;
+ size_t remainingSize = srcSize;
+ size_t nbBlocks = 0;
+ ZSTD_frameHeader zfh;
+
+ /* Extract Frame Header */
+ { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
+ if (ZSTD_isError(ret))
+ return ZSTD_errorFrameSizeInfo(ret);
+ if (ret > 0)
+ return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
+ }
+
+ ip += zfh.headerSize;
+ remainingSize -= zfh.headerSize;
+
+ /* Iterate over each block */
+ while (1) {
+ blockProperties_t blockProperties;
+ size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
+ if (ZSTD_isError(cBlockSize))
+ return ZSTD_errorFrameSizeInfo(cBlockSize);
+
+ if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
+ return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
+
+ ip += ZSTD_blockHeaderSize + cBlockSize;
+ remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
+ nbBlocks++;
+
+ if (blockProperties.lastBlock) break;
+ }
+
+ /* Final frame content checksum */
+ if (zfh.checksumFlag) {
+ if (remainingSize < 4)
+ return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
+ ip += 4;
+ }
+
+ frameSizeInfo.nbBlocks = nbBlocks;
+ frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
+ frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
+ ? zfh.frameContentSize
+ : (unsigned long long)nbBlocks * zfh.blockSizeMax;
+ return frameSizeInfo;
+ }
+}
+
+/** ZSTD_findFrameCompressedSize() :
+ * compatible with legacy mode
+ * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
+ * `srcSize` must be at least as large as the frame contained
+ * @return : the compressed size of the frame starting at `src` */
+size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
+{
+ ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+ return frameSizeInfo.compressedSize;
+}
+
+/** ZSTD_decompressBound() :
+ * compatible with legacy mode
+ * `src` must point to the start of a ZSTD frame or a skippeable frame
+ * `srcSize` must be at least as large as the frame contained
+ * @return : the maximum decompressed size of the compressed source
+ */
+unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
+{
+ unsigned long long bound = 0;
+ /* Iterate over each frame */
+ while (srcSize > 0) {
+ ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+ size_t const compressedSize = frameSizeInfo.compressedSize;
+ unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
+ if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
+ return ZSTD_CONTENTSIZE_ERROR;
+ assert(srcSize >= compressedSize);
+ src = (const BYTE*)src + compressedSize;
+ srcSize -= compressedSize;
+ bound += decompressedBound;
+ }
+ return bound;
+}
+
+size_t ZSTD_decompressionMargin(void const* src, size_t srcSize)
+{
+ size_t margin = 0;
+ unsigned maxBlockSize = 0;
+
+ /* Iterate over each frame */
+ while (srcSize > 0) {
+ ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+ size_t const compressedSize = frameSizeInfo.compressedSize;
+ unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
+ ZSTD_frameHeader zfh;
+
+ FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), "");
+ if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
+ return ERROR(corruption_detected);
+
+ if (zfh.frameType == ZSTD_frame) {
+ /* Add the frame header to our margin */
+ margin += zfh.headerSize;
+ /* Add the checksum to our margin */
+ margin += zfh.checksumFlag ? 4 : 0;
+ /* Add 3 bytes per block */
+ margin += 3 * frameSizeInfo.nbBlocks;
+
+ /* Compute the max block size */
+ maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax);
+ } else {
+ assert(zfh.frameType == ZSTD_skippableFrame);
+ /* Add the entire skippable frame size to our margin. */
+ margin += compressedSize;
+ }
+
+ assert(srcSize >= compressedSize);
+ src = (const BYTE*)src + compressedSize;
+ srcSize -= compressedSize;
+ }
+
+ /* Add the max block size back to the margin. */
+ margin += maxBlockSize;
+
+ return margin;
+}
+
+/*-*************************************************************
+ * Frame decoding
+ ***************************************************************/
+
+/** ZSTD_insertBlock() :
+ * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
+size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
+{
+ DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize);
+ ZSTD_checkContinuity(dctx, blockStart, blockSize);
+ dctx->previousDstEnd = (const char*)blockStart + blockSize;
+ return blockSize;
+}
+
+
+static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_copyRawBlock");
+ RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
+ if (dst == NULL) {
+ if (srcSize == 0) return 0;
+ RETURN_ERROR(dstBuffer_null, "");
+ }
+ ZSTD_memmove(dst, src, srcSize);
+ return srcSize;
+}
+
+static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
+ BYTE b,
+ size_t regenSize)
+{
+ RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
+ if (dst == NULL) {
+ if (regenSize == 0) return 0;
+ RETURN_ERROR(dstBuffer_null, "");
+ }
+ ZSTD_memset(dst, b, regenSize);
+ return regenSize;
+}
+
+static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
+{
+#if ZSTD_TRACE
+ if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) {
+ ZSTD_Trace trace;
+ ZSTD_memset(&trace, 0, sizeof(trace));
+ trace.version = ZSTD_VERSION_NUMBER;
+ trace.streaming = streaming;
+ if (dctx->ddict) {
+ trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict);
+ trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict);
+ trace.dictionaryIsCold = dctx->ddictIsCold;
+ }
+ trace.uncompressedSize = (size_t)uncompressedSize;
+ trace.compressedSize = (size_t)compressedSize;
+ trace.dctx = dctx;
+ ZSTD_trace_decompress_end(dctx->traceCtx, &trace);
+ }
+#else
+ (void)dctx;
+ (void)uncompressedSize;
+ (void)compressedSize;
+ (void)streaming;
+#endif
+}
+
+
+/*! ZSTD_decompressFrame() :
+ * @dctx must be properly initialized
+ * will update *srcPtr and *srcSizePtr,
+ * to make *srcPtr progress by one frame. */
+static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void** srcPtr, size_t *srcSizePtr)
+{
+ const BYTE* const istart = (const BYTE*)(*srcPtr);
+ const BYTE* ip = istart;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
+ BYTE* op = ostart;
+ size_t remainingSrcSize = *srcSizePtr;
+
+ DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr);
+
+ /* check */
+ RETURN_ERROR_IF(
+ remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize,
+ srcSize_wrong, "");
+
+ /* Frame Header */
+ { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal(
+ ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format);
+ if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
+ RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
+ srcSize_wrong, "");
+ FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , "");
+ ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
+ }
+
+ /* Loop on each block */
+ while (1) {
+ BYTE* oBlockEnd = oend;
+ size_t decodedSize;
+ blockProperties_t blockProperties;
+ size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties);
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
+
+ ip += ZSTD_blockHeaderSize;
+ remainingSrcSize -= ZSTD_blockHeaderSize;
+ RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, "");
+
+ if (ip >= op && ip < oBlockEnd) {
+ /* We are decompressing in-place. Limit the output pointer so that we
+ * don't overwrite the block that we are currently reading. This will
+ * fail decompression if the input & output pointers aren't spaced
+ * far enough apart.
+ *
+ * This is important to set, even when the pointers are far enough
+ * apart, because ZSTD_decompressBlock_internal() can decide to store
+ * literals in the output buffer, after the block it is decompressing.
+ * Since we don't want anything to overwrite our input, we have to tell
+ * ZSTD_decompressBlock_internal to never write past ip.
+ *
+ * See ZSTD_allocateLiteralsBuffer() for reference.
+ */
+ oBlockEnd = op + (ip - op);
+ }
+
+ switch(blockProperties.blockType)
+ {
+ case bt_compressed:
+ decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, /* frame */ 1, not_streaming);
+ break;
+ case bt_raw :
+ /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */
+ decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize);
+ break;
+ case bt_rle :
+ decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize);
+ break;
+ case bt_reserved :
+ default:
+ RETURN_ERROR(corruption_detected, "invalid block type");
+ }
+
+ if (ZSTD_isError(decodedSize)) return decodedSize;
+ if (dctx->validateChecksum)
+ XXH64_update(&dctx->xxhState, op, decodedSize);
+ if (decodedSize != 0)
+ op += decodedSize;
+ assert(ip != NULL);
+ ip += cBlockSize;
+ remainingSrcSize -= cBlockSize;
+ if (blockProperties.lastBlock) break;
+ }
+
+ if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) {
+ RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize,
+ corruption_detected, "");
+ }
+ if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
+ RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, "");
+ if (!dctx->forceIgnoreChecksum) {
+ U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
+ U32 checkRead;
+ checkRead = MEM_readLE32(ip);
+ RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, "");
+ }
+ ip += 4;
+ remainingSrcSize -= 4;
+ }
+ ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
+ /* Allow caller to get size read */
+ DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr);
+ *srcPtr = ip;
+ *srcSizePtr = remainingSrcSize;
+ return (size_t)(op-ostart);
+}
+
+static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize,
+ const ZSTD_DDict* ddict)
+{
+ void* const dststart = dst;
+ int moreThan1Frame = 0;
+
+ DEBUGLOG(5, "ZSTD_decompressMultiFrame");
+ assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */
+
+ if (ddict) {
+ dict = ZSTD_DDict_dictContent(ddict);
+ dictSize = ZSTD_DDict_dictSize(ddict);
+ }
+
+ while (srcSize >= ZSTD_startingInputLength(dctx->format)) {
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+ if (ZSTD_isLegacy(src, srcSize)) {
+ size_t decodedSize;
+ size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
+ if (ZSTD_isError(frameSize)) return frameSize;
+ RETURN_ERROR_IF(dctx->staticSize, memory_allocation,
+ "legacy support is not compatible with static dctx");
+
+ decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
+ if (ZSTD_isError(decodedSize)) return decodedSize;
+
+ assert(decodedSize <= dstCapacity);
+ dst = (BYTE*)dst + decodedSize;
+ dstCapacity -= decodedSize;
+
+ src = (const BYTE*)src + frameSize;
+ srcSize -= frameSize;
+
+ continue;
+ }
+#endif
+
+ { U32 const magicNumber = MEM_readLE32(src);
+ DEBUGLOG(4, "reading magic number %08X (expecting %08X)",
+ (unsigned)magicNumber, ZSTD_MAGICNUMBER);
+ if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+ size_t const skippableSize = readSkippableFrameSize(src, srcSize);
+ FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed");
+ assert(skippableSize <= srcSize);
+
+ src = (const BYTE *)src + skippableSize;
+ srcSize -= skippableSize;
+ continue;
+ } }
+
+ if (ddict) {
+ /* we were called from ZSTD_decompress_usingDDict */
+ FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), "");
+ } else {
+ /* this will initialize correctly with no dict if dict == NULL, so
+ * use this in all cases but ddict */
+ FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), "");
+ }
+ ZSTD_checkContinuity(dctx, dst, dstCapacity);
+
+ { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
+ &src, &srcSize);
+ RETURN_ERROR_IF(
+ (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown)
+ && (moreThan1Frame==1),
+ srcSize_wrong,
+ "At least one frame successfully completed, "
+ "but following bytes are garbage: "
+ "it's more likely to be a srcSize error, "
+ "specifying more input bytes than size of frame(s). "
+ "Note: one could be unlucky, it might be a corruption error instead, "
+ "happening right at the place where we expect zstd magic bytes. "
+ "But this is _much_ less likely than a srcSize field error.");
+ if (ZSTD_isError(res)) return res;
+ assert(res <= dstCapacity);
+ if (res != 0)
+ dst = (BYTE*)dst + res;
+ dstCapacity -= res;
+ }
+ moreThan1Frame = 1;
+ } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
+
+ RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed");
+
+ return (size_t)((BYTE*)dst - (BYTE*)dststart);
+}
+
+size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize)
+{
+ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
+}
+
+
+static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx)
+{
+ switch (dctx->dictUses) {
+ default:
+ assert(0 /* Impossible */);
+ ZSTD_FALLTHROUGH;
+ case ZSTD_dont_use:
+ ZSTD_clearDict(dctx);
+ return NULL;
+ case ZSTD_use_indefinitely:
+ return dctx->ddict;
+ case ZSTD_use_once:
+ dctx->dictUses = ZSTD_dont_use;
+ return dctx->ddict;
+ }
+}
+
+size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx));
+}
+
+
+size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
+ size_t regenSize;
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem);
+ RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!");
+ regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
+ ZSTD_freeDCtx(dctx);
+ return regenSize;
+#else /* stack mode */
+ ZSTD_DCtx dctx;
+ ZSTD_initDCtx_internal(&dctx);
+ return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
+#endif
+}
+
+
+/*-**************************************
+* Advanced Streaming Decompression API
+* Bufferless and synchronous
+****************************************/
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; }
+
+/**
+ * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we
+ * allow taking a partial block as the input. Currently only raw uncompressed blocks can
+ * be streamed.
+ *
+ * For blocks that can be streamed, this allows us to reduce the latency until we produce
+ * output, and avoid copying the input.
+ *
+ * @param inputSize - The total amount of input that the caller currently has.
+ */
+static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) {
+ if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock))
+ return dctx->expected;
+ if (dctx->bType != bt_raw)
+ return dctx->expected;
+ return BOUNDED(1, inputSize, dctx->expected);
+}
+
+ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
+ switch(dctx->stage)
+ {
+ default: /* should not happen */
+ assert(0);
+ ZSTD_FALLTHROUGH;
+ case ZSTDds_getFrameHeaderSize:
+ ZSTD_FALLTHROUGH;
+ case ZSTDds_decodeFrameHeader:
+ return ZSTDnit_frameHeader;
+ case ZSTDds_decodeBlockHeader:
+ return ZSTDnit_blockHeader;
+ case ZSTDds_decompressBlock:
+ return ZSTDnit_block;
+ case ZSTDds_decompressLastBlock:
+ return ZSTDnit_lastBlock;
+ case ZSTDds_checkChecksum:
+ return ZSTDnit_checksum;
+ case ZSTDds_decodeSkippableHeader:
+ ZSTD_FALLTHROUGH;
+ case ZSTDds_skipFrame:
+ return ZSTDnit_skippableFrame;
+ }
+}
+
+static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; }
+
+/** ZSTD_decompressContinue() :
+ * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress())
+ * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
+ * or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);
+ /* Sanity check */
+ RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");
+ ZSTD_checkContinuity(dctx, dst, dstCapacity);
+
+ dctx->processedCSize += srcSize;
+
+ switch (dctx->stage)
+ {
+ case ZSTDds_getFrameHeaderSize :
+ assert(src != NULL);
+ if (dctx->format == ZSTD_f_zstd1) { /* allows header */
+ assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */
+ if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
+ ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
+ dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */
+ dctx->stage = ZSTDds_decodeSkippableHeader;
+ return 0;
+ } }
+ dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format);
+ if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
+ ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
+ dctx->expected = dctx->headerSize - srcSize;
+ dctx->stage = ZSTDds_decodeFrameHeader;
+ return 0;
+
+ case ZSTDds_decodeFrameHeader:
+ assert(src != NULL);
+ ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
+ FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), "");
+ dctx->expected = ZSTD_blockHeaderSize;
+ dctx->stage = ZSTDds_decodeBlockHeader;
+ return 0;
+
+ case ZSTDds_decodeBlockHeader:
+ { blockProperties_t bp;
+ size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
+ RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum");
+ dctx->expected = cBlockSize;
+ dctx->bType = bp.blockType;
+ dctx->rleSize = bp.origSize;
+ if (cBlockSize) {
+ dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock;
+ return 0;
+ }
+ /* empty block */
+ if (bp.lastBlock) {
+ if (dctx->fParams.checksumFlag) {
+ dctx->expected = 4;
+ dctx->stage = ZSTDds_checkChecksum;
+ } else {
+ dctx->expected = 0; /* end of frame */
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ }
+ } else {
+ dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */
+ dctx->stage = ZSTDds_decodeBlockHeader;
+ }
+ return 0;
+ }
+
+ case ZSTDds_decompressLastBlock:
+ case ZSTDds_decompressBlock:
+ DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock");
+ { size_t rSize;
+ switch(dctx->bType)
+ {
+ case bt_compressed:
+ DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed");
+ rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming);
+ dctx->expected = 0; /* Streaming not supported */
+ break;
+ case bt_raw :
+ assert(srcSize <= dctx->expected);
+ rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
+ FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed");
+ assert(rSize == srcSize);
+ dctx->expected -= rSize;
+ break;
+ case bt_rle :
+ rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize);
+ dctx->expected = 0; /* Streaming not supported */
+ break;
+ case bt_reserved : /* should never happen */
+ default:
+ RETURN_ERROR(corruption_detected, "invalid block type");
+ }
+ FORWARD_IF_ERROR(rSize, "");
+ RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum");
+ DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
+ dctx->decodedSize += rSize;
+ if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize);
+ dctx->previousDstEnd = (char*)dst + rSize;
+
+ /* Stay on the same stage until we are finished streaming the block. */
+ if (dctx->expected > 0) {
+ return rSize;
+ }
+
+ if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
+ DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize);
+ RETURN_ERROR_IF(
+ dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
+ && dctx->decodedSize != dctx->fParams.frameContentSize,
+ corruption_detected, "");
+ if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
+ dctx->expected = 4;
+ dctx->stage = ZSTDds_checkChecksum;
+ } else {
+ ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
+ dctx->expected = 0; /* ends here */
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ }
+ } else {
+ dctx->stage = ZSTDds_decodeBlockHeader;
+ dctx->expected = ZSTD_blockHeaderSize;
+ }
+ return rSize;
+ }
+
+ case ZSTDds_checkChecksum:
+ assert(srcSize == 4); /* guaranteed by dctx->expected */
+ {
+ if (dctx->validateChecksum) {
+ U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
+ U32 const check32 = MEM_readLE32(src);
+ DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
+ RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
+ }
+ ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
+ dctx->expected = 0;
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ return 0;
+ }
+
+ case ZSTDds_decodeSkippableHeader:
+ assert(src != NULL);
+ assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE);
+ ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */
+ dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */
+ dctx->stage = ZSTDds_skipFrame;
+ return 0;
+
+ case ZSTDds_skipFrame:
+ dctx->expected = 0;
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ return 0;
+
+ default:
+ assert(0); /* impossible */
+ RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */
+ }
+}
+
+
+static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ dctx->dictEnd = dctx->previousDstEnd;
+ dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
+ dctx->prefixStart = dict;
+ dctx->previousDstEnd = (const char*)dict + dictSize;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ dctx->dictContentBeginForFuzzing = dctx->prefixStart;
+ dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
+#endif
+ return 0;
+}
+
+/*! ZSTD_loadDEntropy() :
+ * dict : must point at beginning of a valid zstd dictionary.
+ * @return : size of entropy tables read */
+size_t
+ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
+ const void* const dict, size_t const dictSize)
+{
+ const BYTE* dictPtr = (const BYTE*)dict;
+ const BYTE* const dictEnd = dictPtr + dictSize;
+
+ RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small");
+ assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */
+ dictPtr += 8; /* skip header = magic + dictID */
+
+ ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable));
+ ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable));
+ ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE);
+ { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */
+ size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable);
+#ifdef HUF_FORCE_DECOMPRESS_X1
+ /* in minimal huffman, we always use X1 variants */
+ size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable,
+ dictPtr, dictEnd - dictPtr,
+ workspace, workspaceSize, /* flags */ 0);
+#else
+ size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
+ dictPtr, (size_t)(dictEnd - dictPtr),
+ workspace, workspaceSize, /* flags */ 0);
+#endif
+ RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
+ dictPtr += hSize;
+ }
+
+ { short offcodeNCount[MaxOff+1];
+ unsigned offcodeMaxValue = MaxOff, offcodeLog;
+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr));
+ RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, "");
+ RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
+ ZSTD_buildFSETable( entropy->OFTable,
+ offcodeNCount, offcodeMaxValue,
+ OF_base, OF_bits,
+ offcodeLog,
+ entropy->workspace, sizeof(entropy->workspace),
+ /* bmi2 */0);
+ dictPtr += offcodeHeaderSize;
+ }
+
+ { short matchlengthNCount[MaxML+1];
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
+ RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, "");
+ RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
+ ZSTD_buildFSETable( entropy->MLTable,
+ matchlengthNCount, matchlengthMaxValue,
+ ML_base, ML_bits,
+ matchlengthLog,
+ entropy->workspace, sizeof(entropy->workspace),
+ /* bmi2 */ 0);
+ dictPtr += matchlengthHeaderSize;
+ }
+
+ { short litlengthNCount[MaxLL+1];
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
+ RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
+ RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, "");
+ RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
+ ZSTD_buildFSETable( entropy->LLTable,
+ litlengthNCount, litlengthMaxValue,
+ LL_base, LL_bits,
+ litlengthLog,
+ entropy->workspace, sizeof(entropy->workspace),
+ /* bmi2 */ 0);
+ dictPtr += litlengthHeaderSize;
+ }
+
+ RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, "");
+ { int i;
+ size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
+ for (i=0; i<3; i++) {
+ U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
+ RETURN_ERROR_IF(rep==0 || rep > dictContentSize,
+ dictionary_corrupted, "");
+ entropy->rep[i] = rep;
+ } }
+
+ return (size_t)(dictPtr - (const BYTE*)dict);
+}
+
+static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
+ { U32 const magic = MEM_readLE32(dict);
+ if (magic != ZSTD_MAGIC_DICTIONARY) {
+ return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
+ } }
+ dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
+
+ /* load entropy tables */
+ { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize);
+ RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, "");
+ dict = (const char*)dict + eSize;
+ dictSize -= eSize;
+ }
+ dctx->litEntropy = dctx->fseEntropy = 1;
+
+ /* reference dictionary content */
+ return ZSTD_refDictContent(dctx, dict, dictSize);
+}
+
+size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
+{
+ assert(dctx != NULL);
+#if ZSTD_TRACE
+ dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0;
+#endif
+ dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ dctx->processedCSize = 0;
+ dctx->decodedSize = 0;
+ dctx->previousDstEnd = NULL;
+ dctx->prefixStart = NULL;
+ dctx->virtualStart = NULL;
+ dctx->dictEnd = NULL;
+ dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */
+ dctx->litEntropy = dctx->fseEntropy = 0;
+ dctx->dictID = 0;
+ dctx->bType = bt_reserved;
+ ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
+ ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
+ dctx->LLTptr = dctx->entropy.LLTable;
+ dctx->MLTptr = dctx->entropy.MLTable;
+ dctx->OFTptr = dctx->entropy.OFTable;
+ dctx->HUFptr = dctx->entropy.hufTable;
+ return 0;
+}
+
+size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
+ if (dict && dictSize)
+ RETURN_ERROR_IF(
+ ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)),
+ dictionary_corrupted, "");
+ return 0;
+}
+
+
+/* ====== ZSTD_DDict ====== */
+
+size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
+{
+ DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict");
+ assert(dctx != NULL);
+ if (ddict) {
+ const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict);
+ size_t const dictSize = ZSTD_DDict_dictSize(ddict);
+ const void* const dictEnd = dictStart + dictSize;
+ dctx->ddictIsCold = (dctx->dictEnd != dictEnd);
+ DEBUGLOG(4, "DDict is %s",
+ dctx->ddictIsCold ? "~cold~" : "hot!");
+ }
+ FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
+ if (ddict) { /* NULL ddict is equivalent to no dictionary */
+ ZSTD_copyDDictParameters(dctx, ddict);
+ }
+ return 0;
+}
+
+/*! ZSTD_getDictID_fromDict() :
+ * Provides the dictID stored within dictionary.
+ * if @return == 0, the dictionary is not conformant with Zstandard specification.
+ * It can still be loaded, but as a content-only dictionary. */
+unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
+{
+ if (dictSize < 8) return 0;
+ if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
+ return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
+}
+
+/*! ZSTD_getDictID_fromFrame() :
+ * Provides the dictID required to decompress frame stored within `src`.
+ * If @return == 0, the dictID could not be decoded.
+ * This could for one of the following reasons :
+ * - The frame does not require a dictionary (most common case).
+ * - The frame was built with dictID intentionally removed.
+ * Needed dictionary is a hidden piece of information.
+ * Note : this use case also happens when using a non-conformant dictionary.
+ * - `srcSize` is too small, and as a result, frame header could not be decoded.
+ * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`.
+ * - This is not a Zstandard frame.
+ * When identifying the exact failure cause, it's possible to use
+ * ZSTD_getFrameHeader(), which will provide a more precise error code. */
+unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
+{
+ ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 };
+ size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
+ if (ZSTD_isError(hError)) return 0;
+ return zfp.dictID;
+}
+
+
+/*! ZSTD_decompress_usingDDict() :
+* Decompression using a pre-digested Dictionary
+* Use dictionary without significant overhead. */
+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_DDict* ddict)
+{
+ /* pass content and size in case legacy frames are encountered */
+ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
+ NULL, 0,
+ ddict);
+}
+
+
+/*=====================================
+* Streaming decompression
+*====================================*/
+
+ZSTD_DStream* ZSTD_createDStream(void)
+{
+ DEBUGLOG(3, "ZSTD_createDStream");
+ return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
+}
+
+ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
+{
+ return ZSTD_initStaticDCtx(workspace, workspaceSize);
+}
+
+ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
+{
+ return ZSTD_createDCtx_internal(customMem);
+}
+
+size_t ZSTD_freeDStream(ZSTD_DStream* zds)
+{
+ return ZSTD_freeDCtx(zds);
+}
+
+
+/* *** Initialization *** */
+
+size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
+size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
+
+size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
+ const void* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType)
+{
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+ ZSTD_clearDict(dctx);
+ if (dict && dictSize != 0) {
+ dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
+ RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!");
+ dctx->ddict = dctx->ddictLocal;
+ dctx->dictUses = ZSTD_use_indefinitely;
+ }
+ return 0;
+}
+
+size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
+}
+
+size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+ return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
+}
+
+size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
+{
+ FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), "");
+ dctx->dictUses = ZSTD_use_once;
+ return 0;
+}
+
+size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize)
+{
+ return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent);
+}
+
+
+/* ZSTD_initDStream_usingDict() :
+ * return : expected size, aka ZSTD_startingInputLength().
+ * this function cannot fail */
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
+{
+ DEBUGLOG(4, "ZSTD_initDStream_usingDict");
+ FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , "");
+ return ZSTD_startingInputLength(zds->format);
+}
+
+/* note : this variant can't fail */
+size_t ZSTD_initDStream(ZSTD_DStream* zds)
+{
+ DEBUGLOG(4, "ZSTD_initDStream");
+ FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), "");
+ FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), "");
+ return ZSTD_startingInputLength(zds->format);
+}
+
+/* ZSTD_initDStream_usingDDict() :
+ * ddict will just be referenced, and must outlive decompression session
+ * this function cannot fail */
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
+{
+ DEBUGLOG(4, "ZSTD_initDStream_usingDDict");
+ FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");
+ FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");
+ return ZSTD_startingInputLength(dctx->format);
+}
+
+/* ZSTD_resetDStream() :
+ * return : expected size, aka ZSTD_startingInputLength().
+ * this function cannot fail */
+size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
+{
+ DEBUGLOG(4, "ZSTD_resetDStream");
+ FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");
+ return ZSTD_startingInputLength(dctx->format);
+}
+
+
+size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
+{
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+ ZSTD_clearDict(dctx);
+ if (ddict) {
+ dctx->ddict = ddict;
+ dctx->dictUses = ZSTD_use_indefinitely;
+ if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) {
+ if (dctx->ddictSet == NULL) {
+ dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
+ if (!dctx->ddictSet) {
+ RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!");
+ }
+ }
+ assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), "");
+ }
+ }
+ return 0;
+}
+
+/* ZSTD_DCtx_setMaxWindowSize() :
+ * note : no direct equivalence in ZSTD_DCtx_setParameter,
+ * since this version sets windowSize, and the other sets windowLog */
+size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
+{
+ ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
+ size_t const min = (size_t)1 << bounds.lowerBound;
+ size_t const max = (size_t)1 << bounds.upperBound;
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+ RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, "");
+ RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, "");
+ dctx->maxWindowSize = maxWindowSize;
+ return 0;
+}
+
+size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
+{
+ return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format);
+}
+
+ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
+{
+ ZSTD_bounds bounds = { 0, 0, 0 };
+ switch(dParam) {
+ case ZSTD_d_windowLogMax:
+ bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN;
+ bounds.upperBound = ZSTD_WINDOWLOG_MAX;
+ return bounds;
+ case ZSTD_d_format:
+ bounds.lowerBound = (int)ZSTD_f_zstd1;
+ bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
+ ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
+ return bounds;
+ case ZSTD_d_stableOutBuffer:
+ bounds.lowerBound = (int)ZSTD_bm_buffered;
+ bounds.upperBound = (int)ZSTD_bm_stable;
+ return bounds;
+ case ZSTD_d_forceIgnoreChecksum:
+ bounds.lowerBound = (int)ZSTD_d_validateChecksum;
+ bounds.upperBound = (int)ZSTD_d_ignoreChecksum;
+ return bounds;
+ case ZSTD_d_refMultipleDDicts:
+ bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
+ bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
+ return bounds;
+ case ZSTD_d_disableHuffmanAssembly:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ default:;
+ }
+ bounds.error = ERROR(parameter_unsupported);
+ return bounds;
+}
+
+/* ZSTD_dParam_withinBounds:
+ * @return 1 if value is within dParam bounds,
+ * 0 otherwise */
+static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
+{
+ ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam);
+ if (ZSTD_isError(bounds.error)) return 0;
+ if (value < bounds.lowerBound) return 0;
+ if (value > bounds.upperBound) return 0;
+ return 1;
+}
+
+#define CHECK_DBOUNDS(p,v) { \
+ RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \
+}
+
+size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value)
+{
+ switch (param) {
+ case ZSTD_d_windowLogMax:
+ *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize);
+ return 0;
+ case ZSTD_d_format:
+ *value = (int)dctx->format;
+ return 0;
+ case ZSTD_d_stableOutBuffer:
+ *value = (int)dctx->outBufferMode;
+ return 0;
+ case ZSTD_d_forceIgnoreChecksum:
+ *value = (int)dctx->forceIgnoreChecksum;
+ return 0;
+ case ZSTD_d_refMultipleDDicts:
+ *value = (int)dctx->refMultipleDDicts;
+ return 0;
+ case ZSTD_d_disableHuffmanAssembly:
+ *value = (int)dctx->disableHufAsm;
+ return 0;
+ default:;
+ }
+ RETURN_ERROR(parameter_unsupported, "");
+}
+
+size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
+{
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+ switch(dParam) {
+ case ZSTD_d_windowLogMax:
+ if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
+ CHECK_DBOUNDS(ZSTD_d_windowLogMax, value);
+ dctx->maxWindowSize = ((size_t)1) << value;
+ return 0;
+ case ZSTD_d_format:
+ CHECK_DBOUNDS(ZSTD_d_format, value);
+ dctx->format = (ZSTD_format_e)value;
+ return 0;
+ case ZSTD_d_stableOutBuffer:
+ CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value);
+ dctx->outBufferMode = (ZSTD_bufferMode_e)value;
+ return 0;
+ case ZSTD_d_forceIgnoreChecksum:
+ CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value);
+ dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value;
+ return 0;
+ case ZSTD_d_refMultipleDDicts:
+ CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
+ if (dctx->staticSize != 0) {
+ RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!");
+ }
+ dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
+ return 0;
+ case ZSTD_d_disableHuffmanAssembly:
+ CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value);
+ dctx->disableHufAsm = value != 0;
+ return 0;
+ default:;
+ }
+ RETURN_ERROR(parameter_unsupported, "");
+}
+
+size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
+{
+ if ( (reset == ZSTD_reset_session_only)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ dctx->streamStage = zdss_init;
+ dctx->noForwardProgress = 0;
+ }
+ if ( (reset == ZSTD_reset_parameters)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
+ ZSTD_clearDict(dctx);
+ ZSTD_DCtx_resetParameters(dctx);
+ }
+ return 0;
+}
+
+
+size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx)
+{
+ return ZSTD_sizeof_DCtx(dctx);
+}
+
+size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize)
+{
+ size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
+ /* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/
+ unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2);
+ unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
+ size_t const minRBSize = (size_t) neededSize;
+ RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
+ frameParameter_windowTooLarge, "");
+ return minRBSize;
+}
+
+size_t ZSTD_estimateDStreamSize(size_t windowSize)
+{
+ size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
+ size_t const inBuffSize = blockSize; /* no block can be larger */
+ size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN);
+ return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
+}
+
+size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
+{
+ U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */
+ ZSTD_frameHeader zfh;
+ size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
+ if (ZSTD_isError(err)) return err;
+ RETURN_ERROR_IF(err>0, srcSize_wrong, "");
+ RETURN_ERROR_IF(zfh.windowSize > windowSizeMax,
+ frameParameter_windowTooLarge, "");
+ return ZSTD_estimateDStreamSize((size_t)zfh.windowSize);
+}
+
+
+/* ***** Decompression ***** */
+
+static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
+{
+ return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR;
+}
+
+static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
+{
+ if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize))
+ zds->oversizedDuration++;
+ else
+ zds->oversizedDuration = 0;
+}
+
+static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds)
+{
+ return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION;
+}
+
+/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */
+static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output)
+{
+ ZSTD_outBuffer const expect = zds->expectedOutBuffer;
+ /* No requirement when ZSTD_obm_stable is not enabled. */
+ if (zds->outBufferMode != ZSTD_bm_stable)
+ return 0;
+ /* Any buffer is allowed in zdss_init, this must be the same for every other call until
+ * the context is reset.
+ */
+ if (zds->streamStage == zdss_init)
+ return 0;
+ /* The buffer must match our expectation exactly. */
+ if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size)
+ return 0;
+ RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!");
+}
+
+/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream()
+ * and updates the stage and the output buffer state. This call is extracted so it can be
+ * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode.
+ * NOTE: You must break after calling this function since the streamStage is modified.
+ */
+static size_t ZSTD_decompressContinueStream(
+ ZSTD_DStream* zds, char** op, char* oend,
+ void const* src, size_t srcSize) {
+ int const isSkipFrame = ZSTD_isSkipFrame(zds);
+ if (zds->outBufferMode == ZSTD_bm_buffered) {
+ size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart;
+ size_t const decodedSize = ZSTD_decompressContinue(zds,
+ zds->outBuff + zds->outStart, dstSize, src, srcSize);
+ FORWARD_IF_ERROR(decodedSize, "");
+ if (!decodedSize && !isSkipFrame) {
+ zds->streamStage = zdss_read;
+ } else {
+ zds->outEnd = zds->outStart + decodedSize;
+ zds->streamStage = zdss_flush;
+ }
+ } else {
+ /* Write directly into the output buffer */
+ size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op);
+ size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize);
+ FORWARD_IF_ERROR(decodedSize, "");
+ *op += decodedSize;
+ /* Flushing is not needed. */
+ zds->streamStage = zdss_read;
+ assert(*op <= oend);
+ assert(zds->outBufferMode == ZSTD_bm_stable);
+ }
+ return 0;
+}
+
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+ const char* const src = (const char*)input->src;
+ const char* const istart = input->pos != 0 ? src + input->pos : src;
+ const char* const iend = input->size != 0 ? src + input->size : src;
+ const char* ip = istart;
+ char* const dst = (char*)output->dst;
+ char* const ostart = output->pos != 0 ? dst + output->pos : dst;
+ char* const oend = output->size != 0 ? dst + output->size : dst;
+ char* op = ostart;
+ U32 someMoreWork = 1;
+
+ DEBUGLOG(5, "ZSTD_decompressStream");
+ RETURN_ERROR_IF(
+ input->pos > input->size,
+ srcSize_wrong,
+ "forbidden. in: pos: %u vs size: %u",
+ (U32)input->pos, (U32)input->size);
+ RETURN_ERROR_IF(
+ output->pos > output->size,
+ dstSize_tooSmall,
+ "forbidden. out: pos: %u vs size: %u",
+ (U32)output->pos, (U32)output->size);
+ DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
+ FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), "");
+
+ while (someMoreWork) {
+ switch(zds->streamStage)
+ {
+ case zdss_init :
+ DEBUGLOG(5, "stage zdss_init => transparent reset ");
+ zds->streamStage = zdss_loadHeader;
+ zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+ zds->legacyVersion = 0;
+#endif
+ zds->hostageByte = 0;
+ zds->expectedOutBuffer = *output;
+ ZSTD_FALLTHROUGH;
+
+ case zdss_loadHeader :
+ DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip));
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+ if (zds->legacyVersion) {
+ RETURN_ERROR_IF(zds->staticSize, memory_allocation,
+ "legacy support is incompatible with static dctx");
+ { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
+ if (hint==0) zds->streamStage = zdss_init;
+ return hint;
+ } }
+#endif
+ { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
+ if (zds->refMultipleDDicts && zds->ddictSet) {
+ ZSTD_DCtx_selectFrameDDict(zds);
+ }
+ if (ZSTD_isError(hSize)) {
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
+ if (legacyVersion) {
+ ZSTD_DDict const* const ddict = ZSTD_getDDict(zds);
+ const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL;
+ size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0;
+ DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
+ RETURN_ERROR_IF(zds->staticSize, memory_allocation,
+ "legacy support is incompatible with static dctx");
+ FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext,
+ zds->previousLegacyVersion, legacyVersion,
+ dict, dictSize), "");
+ zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
+ { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
+ if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */
+ return hint;
+ } }
+#endif
+ return hSize; /* error */
+ }
+ if (hSize != 0) { /* need more input */
+ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
+ size_t const remainingInput = (size_t)(iend-ip);
+ assert(iend >= ip);
+ if (toLoad > remainingInput) { /* not enough input to load full header */
+ if (remainingInput > 0) {
+ ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
+ zds->lhSize += remainingInput;
+ }
+ input->pos = input->size;
+ /* check first few bytes */
+ FORWARD_IF_ERROR(
+ ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format),
+ "First few bytes detected incorrect" );
+ /* return hint input size */
+ return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
+ }
+ assert(ip != NULL);
+ ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
+ break;
+ } }
+
+ /* check for single-pass mode opportunity */
+ if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
+ && zds->fParams.frameType != ZSTD_skippableFrame
+ && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
+ size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart));
+ if (cSize <= (size_t)(iend-istart)) {
+ /* shortcut : using single-pass mode */
+ size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds));
+ if (ZSTD_isError(decompressedSize)) return decompressedSize;
+ DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
+ assert(istart != NULL);
+ ip = istart + cSize;
+ op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */
+ zds->expected = 0;
+ zds->streamStage = zdss_init;
+ someMoreWork = 0;
+ break;
+ } }
+
+ /* Check output buffer is large enough for ZSTD_odm_stable. */
+ if (zds->outBufferMode == ZSTD_bm_stable
+ && zds->fParams.frameType != ZSTD_skippableFrame
+ && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
+ && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) {
+ RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small");
+ }
+
+ /* Consume header (see ZSTDds_decodeFrameHeader) */
+ DEBUGLOG(4, "Consume header");
+ FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), "");
+
+ if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
+ zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
+ zds->stage = ZSTDds_skipFrame;
+ } else {
+ FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), "");
+ zds->expected = ZSTD_blockHeaderSize;
+ zds->stage = ZSTDds_decodeBlockHeader;
+ }
+
+ /* control buffer memory usage */
+ DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)",
+ (U32)(zds->fParams.windowSize >>10),
+ (U32)(zds->maxWindowSize >> 10) );
+ zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
+ RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize,
+ frameParameter_windowTooLarge, "");
+
+ /* Adapt buffer sizes to frame header instructions */
+ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
+ size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered
+ ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize)
+ : 0;
+
+ ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize);
+
+ { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize);
+ int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds);
+
+ if (tooSmall || tooLarge) {
+ size_t const bufferSize = neededInBuffSize + neededOutBuffSize;
+ DEBUGLOG(4, "inBuff : from %u to %u",
+ (U32)zds->inBuffSize, (U32)neededInBuffSize);
+ DEBUGLOG(4, "outBuff : from %u to %u",
+ (U32)zds->outBuffSize, (U32)neededOutBuffSize);
+ if (zds->staticSize) { /* static DCtx */
+ DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
+ assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */
+ RETURN_ERROR_IF(
+ bufferSize > zds->staticSize - sizeof(ZSTD_DCtx),
+ memory_allocation, "");
+ } else {
+ ZSTD_customFree(zds->inBuff, zds->customMem);
+ zds->inBuffSize = 0;
+ zds->outBuffSize = 0;
+ zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem);
+ RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, "");
+ }
+ zds->inBuffSize = neededInBuffSize;
+ zds->outBuff = zds->inBuff + zds->inBuffSize;
+ zds->outBuffSize = neededOutBuffSize;
+ } } }
+ zds->streamStage = zdss_read;
+ ZSTD_FALLTHROUGH;
+
+ case zdss_read:
+ DEBUGLOG(5, "stage zdss_read");
+ { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip));
+ DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
+ if (neededInSize==0) { /* end of frame */
+ zds->streamStage = zdss_init;
+ someMoreWork = 0;
+ break;
+ }
+ if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
+ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), "");
+ assert(ip != NULL);
+ ip += neededInSize;
+ /* Function modifies the stage so we must break */
+ break;
+ } }
+ if (ip==iend) { someMoreWork = 0; break; } /* no more input */
+ zds->streamStage = zdss_load;
+ ZSTD_FALLTHROUGH;
+
+ case zdss_load:
+ { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
+ size_t const toLoad = neededInSize - zds->inPos;
+ int const isSkipFrame = ZSTD_isSkipFrame(zds);
+ size_t loadedSize;
+ /* At this point we shouldn't be decompressing a block that we can stream. */
+ assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)));
+ if (isSkipFrame) {
+ loadedSize = MIN(toLoad, (size_t)(iend-ip));
+ } else {
+ RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos,
+ corruption_detected,
+ "should never happen");
+ loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip));
+ }
+ if (loadedSize != 0) {
+ /* ip may be NULL */
+ ip += loadedSize;
+ zds->inPos += loadedSize;
+ }
+ if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
+
+ /* decode loaded input */
+ zds->inPos = 0; /* input is consumed */
+ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), "");
+ /* Function modifies the stage so we must break */
+ break;
+ }
+ case zdss_flush:
+ {
+ size_t const toFlushSize = zds->outEnd - zds->outStart;
+ size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
+
+ op = op ? op + flushedSize : op;
+
+ zds->outStart += flushedSize;
+ if (flushedSize == toFlushSize) { /* flush completed */
+ zds->streamStage = zdss_read;
+ if ( (zds->outBuffSize < zds->fParams.frameContentSize)
+ && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
+ DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)",
+ (int)(zds->outBuffSize - zds->outStart),
+ (U32)zds->fParams.blockSizeMax);
+ zds->outStart = zds->outEnd = 0;
+ }
+ break;
+ } }
+ /* cannot complete flush */
+ someMoreWork = 0;
+ break;
+
+ default:
+ assert(0); /* impossible */
+ RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */
+ } }
+
+ /* result */
+ input->pos = (size_t)(ip - (const char*)(input->src));
+ output->pos = (size_t)(op - (char*)(output->dst));
+
+ /* Update the expected output buffer for ZSTD_obm_stable. */
+ zds->expectedOutBuffer = *output;
+
+ if ((ip==istart) && (op==ostart)) { /* no forward progress */
+ zds->noForwardProgress ++;
+ if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
+ RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, "");
+ RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, "");
+ assert(0);
+ }
+ } else {
+ zds->noForwardProgress = 0;
+ }
+ { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds);
+ if (!nextSrcSizeHint) { /* frame fully decoded */
+ if (zds->outEnd == zds->outStart) { /* output fully flushed */
+ if (zds->hostageByte) {
+ if (input->pos >= input->size) {
+ /* can't release hostage (not present) */
+ zds->streamStage = zdss_read;
+ return 1;
+ }
+ input->pos++; /* release hostage */
+ } /* zds->hostageByte */
+ return 0;
+ } /* zds->outEnd == zds->outStart */
+ if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
+ input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
+ zds->hostageByte=1;
+ }
+ return 1;
+ } /* nextSrcSizeHint==0 */
+ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */
+ assert(zds->inPos <= nextSrcSizeHint);
+ nextSrcSizeHint -= zds->inPos; /* part already loaded*/
+ return nextSrcSizeHint;
+ }
+}
+
+size_t ZSTD_decompressStream_simpleArgs (
+ ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos)
+{
+ ZSTD_outBuffer output;
+ ZSTD_inBuffer input;
+ output.dst = dst;
+ output.size = dstCapacity;
+ output.pos = *dstPos;
+ input.src = src;
+ input.size = srcSize;
+ input.pos = *srcPos;
+ { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
+ *dstPos = output.pos;
+ *srcPos = input.pos;
+ return cErr;
+ }
+}
diff --git a/contrib/zstd/zstd_decompress_block.c b/contrib/zstd/zstd_decompress_block.c
new file mode 100644
index 0000000..add277e
--- /dev/null
+++ b/contrib/zstd/zstd_decompress_block.c
@@ -0,0 +1,2193 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* zstd_decompress_block :
+ * this module takes care of decompressing _compressed_ block */
+
+/*-*******************************************************
+* Dependencies
+*********************************************************/
+#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
+#include "compiler.h" /* prefetch */
+#include "cpu.h" /* bmi2 */
+#include "mem.h" /* low level memory routines */
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "huf.h"
+#include "zstd_internal.h"
+#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
+#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
+#include "zstd_decompress_block.h"
+#include "bits.h" /* ZSTD_highbit32 */
+
+/*_*******************************************************
+* Macros
+**********************************************************/
+
+/* These two optional macros force the use one way or another of the two
+ * ZSTD_decompressSequences implementations. You can't force in both directions
+ * at the same time.
+ */
+#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
+ defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
+#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!"
+#endif
+
+
+/*_*******************************************************
+* Memory operations
+**********************************************************/
+static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); }
+
+
+/*-*************************************************************
+ * Block decoding
+ ***************************************************************/
+
+/*! ZSTD_getcBlockSize() :
+ * Provides the size of compressed block from block header `src` */
+size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
+ blockProperties_t* bpPtr)
+{
+ RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, "");
+
+ { U32 const cBlockHeader = MEM_readLE24(src);
+ U32 const cSize = cBlockHeader >> 3;
+ bpPtr->lastBlock = cBlockHeader & 1;
+ bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
+ bpPtr->origSize = cSize; /* only useful for RLE */
+ if (bpPtr->blockType == bt_rle) return 1;
+ RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, "");
+ return cSize;
+ }
+}
+
+/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */
+static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize,
+ const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately)
+{
+ if (streaming == not_streaming && dstCapacity > ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH)
+ {
+ /* room for litbuffer to fit without read faulting */
+ dctx->litBuffer = (BYTE*)dst + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH;
+ dctx->litBufferEnd = dctx->litBuffer + litSize;
+ dctx->litBufferLocation = ZSTD_in_dst;
+ }
+ else if (litSize > ZSTD_LITBUFFEREXTRASIZE)
+ {
+ /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
+ if (splitImmediately) {
+ /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
+ dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
+ dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE;
+ }
+ else {
+ /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */
+ dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize;
+ dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize;
+ }
+ dctx->litBufferLocation = ZSTD_split;
+ }
+ else
+ {
+ /* fits entirely within litExtraBuffer, so no split is necessary */
+ dctx->litBuffer = dctx->litExtraBuffer;
+ dctx->litBufferEnd = dctx->litBuffer + litSize;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ }
+}
+
+/* Hidden declaration for fullbench */
+size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
+ const void* src, size_t srcSize,
+ void* dst, size_t dstCapacity, const streaming_operation streaming);
+/*! ZSTD_decodeLiteralsBlock() :
+ * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored
+ * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current
+ * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being
+ * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write.
+ *
+ * @return : nb of bytes read from src (< srcSize )
+ * note : symbol not declared but exposed for fullbench */
+size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
+ const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */
+ void* dst, size_t dstCapacity, const streaming_operation streaming)
+{
+ DEBUGLOG(5, "ZSTD_decodeLiteralsBlock");
+ RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");
+
+ { const BYTE* const istart = (const BYTE*) src;
+ symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
+
+ switch(litEncType)
+ {
+ case set_repeat:
+ DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block");
+ RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, "");
+ ZSTD_FALLTHROUGH;
+
+ case set_compressed:
+ RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3");
+ { size_t lhSize, litSize, litCSize;
+ U32 singleStream=0;
+ U32 const lhlCode = (istart[0] >> 2) & 3;
+ U32 const lhc = MEM_readLE32(istart);
+ size_t hufSuccess;
+ size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
+ int const flags = 0
+ | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0)
+ | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0);
+ switch(lhlCode)
+ {
+ case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
+ /* 2 - 2 - 10 - 10 */
+ singleStream = !lhlCode;
+ lhSize = 3;
+ litSize = (lhc >> 4) & 0x3FF;
+ litCSize = (lhc >> 14) & 0x3FF;
+ break;
+ case 2:
+ /* 2 - 2 - 14 - 14 */
+ lhSize = 4;
+ litSize = (lhc >> 4) & 0x3FFF;
+ litCSize = lhc >> 18;
+ break;
+ case 3:
+ /* 2 - 2 - 18 - 18 */
+ lhSize = 5;
+ litSize = (lhc >> 4) & 0x3FFFF;
+ litCSize = (lhc >> 22) + ((size_t)istart[4] << 10);
+ break;
+ }
+ RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
+ RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
+ if (!singleStream)
+ RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong,
+ "Not enough literals (%zu) for the 4-streams mode (min %u)",
+ litSize, MIN_LITERALS_FOR_4_STREAMS);
+ RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
+ RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
+ ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
+
+ /* prefetch huffman table if cold */
+ if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) {
+ PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable));
+ }
+
+ if (litEncType==set_repeat) {
+ if (singleStream) {
+ hufSuccess = HUF_decompress1X_usingDTable(
+ dctx->litBuffer, litSize, istart+lhSize, litCSize,
+ dctx->HUFptr, flags);
+ } else {
+ assert(litSize >= MIN_LITERALS_FOR_4_STREAMS);
+ hufSuccess = HUF_decompress4X_usingDTable(
+ dctx->litBuffer, litSize, istart+lhSize, litCSize,
+ dctx->HUFptr, flags);
+ }
+ } else {
+ if (singleStream) {
+#if defined(HUF_FORCE_DECOMPRESS_X2)
+ hufSuccess = HUF_decompress1X_DCtx_wksp(
+ dctx->entropy.hufTable, dctx->litBuffer, litSize,
+ istart+lhSize, litCSize, dctx->workspace,
+ sizeof(dctx->workspace), flags);
+#else
+ hufSuccess = HUF_decompress1X1_DCtx_wksp(
+ dctx->entropy.hufTable, dctx->litBuffer, litSize,
+ istart+lhSize, litCSize, dctx->workspace,
+ sizeof(dctx->workspace), flags);
+#endif
+ } else {
+ hufSuccess = HUF_decompress4X_hufOnly_wksp(
+ dctx->entropy.hufTable, dctx->litBuffer, litSize,
+ istart+lhSize, litCSize, dctx->workspace,
+ sizeof(dctx->workspace), flags);
+ }
+ }
+ if (dctx->litBufferLocation == ZSTD_split)
+ {
+ ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
+ ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE);
+ dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
+ dctx->litBufferEnd -= WILDCOPY_OVERLENGTH;
+ }
+
+ RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, "");
+
+ dctx->litPtr = dctx->litBuffer;
+ dctx->litSize = litSize;
+ dctx->litEntropy = 1;
+ if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
+ return litCSize + lhSize;
+ }
+
+ case set_basic:
+ { size_t litSize, lhSize;
+ U32 const lhlCode = ((istart[0]) >> 2) & 3;
+ size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
+ switch(lhlCode)
+ {
+ case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
+ lhSize = 1;
+ litSize = istart[0] >> 3;
+ break;
+ case 1:
+ lhSize = 2;
+ litSize = MEM_readLE16(istart) >> 4;
+ break;
+ case 3:
+ lhSize = 3;
+ RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3");
+ litSize = MEM_readLE24(istart) >> 4;
+ break;
+ }
+
+ RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
+ RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
+ ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
+ if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
+ RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, "");
+ if (dctx->litBufferLocation == ZSTD_split)
+ {
+ ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE);
+ ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
+ }
+ else
+ {
+ ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize);
+ }
+ dctx->litPtr = dctx->litBuffer;
+ dctx->litSize = litSize;
+ return lhSize+litSize;
+ }
+ /* direct reference into compressed stream */
+ dctx->litPtr = istart+lhSize;
+ dctx->litSize = litSize;
+ dctx->litBufferEnd = dctx->litPtr + litSize;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ return lhSize+litSize;
+ }
+
+ case set_rle:
+ { U32 const lhlCode = ((istart[0]) >> 2) & 3;
+ size_t litSize, lhSize;
+ size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
+ switch(lhlCode)
+ {
+ case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
+ lhSize = 1;
+ litSize = istart[0] >> 3;
+ break;
+ case 1:
+ lhSize = 2;
+ RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3");
+ litSize = MEM_readLE16(istart) >> 4;
+ break;
+ case 3:
+ lhSize = 3;
+ RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4");
+ litSize = MEM_readLE24(istart) >> 4;
+ break;
+ }
+ RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
+ RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
+ RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
+ ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
+ if (dctx->litBufferLocation == ZSTD_split)
+ {
+ ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE);
+ ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE);
+ }
+ else
+ {
+ ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize);
+ }
+ dctx->litPtr = dctx->litBuffer;
+ dctx->litSize = litSize;
+ return lhSize+1;
+ }
+ default:
+ RETURN_ERROR(corruption_detected, "impossible");
+ }
+ }
+}
+
+/* Default FSE distribution tables.
+ * These are pre-calculated FSE decoding tables using default distributions as defined in specification :
+ * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions
+ * They were generated programmatically with following method :
+ * - start from default distributions, present in /lib/common/zstd_internal.h
+ * - generate tables normally, using ZSTD_buildFSETable()
+ * - printout the content of tables
+ * - pretify output, report below, test with fuzzer to ensure it's correct */
+
+/* Default FSE distribution table for Literal Lengths */
+static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {
+ { 1, 1, 1, LL_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
+ /* nextState, nbAddBits, nbBits, baseVal */
+ { 0, 0, 4, 0}, { 16, 0, 4, 0},
+ { 32, 0, 5, 1}, { 0, 0, 5, 3},
+ { 0, 0, 5, 4}, { 0, 0, 5, 6},
+ { 0, 0, 5, 7}, { 0, 0, 5, 9},
+ { 0, 0, 5, 10}, { 0, 0, 5, 12},
+ { 0, 0, 6, 14}, { 0, 1, 5, 16},
+ { 0, 1, 5, 20}, { 0, 1, 5, 22},
+ { 0, 2, 5, 28}, { 0, 3, 5, 32},
+ { 0, 4, 5, 48}, { 32, 6, 5, 64},
+ { 0, 7, 5, 128}, { 0, 8, 6, 256},
+ { 0, 10, 6, 1024}, { 0, 12, 6, 4096},
+ { 32, 0, 4, 0}, { 0, 0, 4, 1},
+ { 0, 0, 5, 2}, { 32, 0, 5, 4},
+ { 0, 0, 5, 5}, { 32, 0, 5, 7},
+ { 0, 0, 5, 8}, { 32, 0, 5, 10},
+ { 0, 0, 5, 11}, { 0, 0, 6, 13},
+ { 32, 1, 5, 16}, { 0, 1, 5, 18},
+ { 32, 1, 5, 22}, { 0, 2, 5, 24},
+ { 32, 3, 5, 32}, { 0, 3, 5, 40},
+ { 0, 6, 4, 64}, { 16, 6, 4, 64},
+ { 32, 7, 5, 128}, { 0, 9, 6, 512},
+ { 0, 11, 6, 2048}, { 48, 0, 4, 0},
+ { 16, 0, 4, 1}, { 32, 0, 5, 2},
+ { 32, 0, 5, 3}, { 32, 0, 5, 5},
+ { 32, 0, 5, 6}, { 32, 0, 5, 8},
+ { 32, 0, 5, 9}, { 32, 0, 5, 11},
+ { 32, 0, 5, 12}, { 0, 0, 6, 15},
+ { 32, 1, 5, 18}, { 32, 1, 5, 20},
+ { 32, 2, 5, 24}, { 32, 2, 5, 28},
+ { 32, 3, 5, 40}, { 32, 4, 5, 48},
+ { 0, 16, 6,65536}, { 0, 15, 6,32768},
+ { 0, 14, 6,16384}, { 0, 13, 6, 8192},
+}; /* LL_defaultDTable */
+
+/* Default FSE distribution table for Offset Codes */
+static const ZSTD_seqSymbol OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {
+ { 1, 1, 1, OF_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
+ /* nextState, nbAddBits, nbBits, baseVal */
+ { 0, 0, 5, 0}, { 0, 6, 4, 61},
+ { 0, 9, 5, 509}, { 0, 15, 5,32765},
+ { 0, 21, 5,2097149}, { 0, 3, 5, 5},
+ { 0, 7, 4, 125}, { 0, 12, 5, 4093},
+ { 0, 18, 5,262141}, { 0, 23, 5,8388605},
+ { 0, 5, 5, 29}, { 0, 8, 4, 253},
+ { 0, 14, 5,16381}, { 0, 20, 5,1048573},
+ { 0, 2, 5, 1}, { 16, 7, 4, 125},
+ { 0, 11, 5, 2045}, { 0, 17, 5,131069},
+ { 0, 22, 5,4194301}, { 0, 4, 5, 13},
+ { 16, 8, 4, 253}, { 0, 13, 5, 8189},
+ { 0, 19, 5,524285}, { 0, 1, 5, 1},
+ { 16, 6, 4, 61}, { 0, 10, 5, 1021},
+ { 0, 16, 5,65533}, { 0, 28, 5,268435453},
+ { 0, 27, 5,134217725}, { 0, 26, 5,67108861},
+ { 0, 25, 5,33554429}, { 0, 24, 5,16777213},
+}; /* OF_defaultDTable */
+
+
+/* Default FSE distribution table for Match Lengths */
+static const ZSTD_seqSymbol ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = {
+ { 1, 1, 1, ML_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
+ /* nextState, nbAddBits, nbBits, baseVal */
+ { 0, 0, 6, 3}, { 0, 0, 4, 4},
+ { 32, 0, 5, 5}, { 0, 0, 5, 6},
+ { 0, 0, 5, 8}, { 0, 0, 5, 9},
+ { 0, 0, 5, 11}, { 0, 0, 6, 13},
+ { 0, 0, 6, 16}, { 0, 0, 6, 19},
+ { 0, 0, 6, 22}, { 0, 0, 6, 25},
+ { 0, 0, 6, 28}, { 0, 0, 6, 31},
+ { 0, 0, 6, 34}, { 0, 1, 6, 37},
+ { 0, 1, 6, 41}, { 0, 2, 6, 47},
+ { 0, 3, 6, 59}, { 0, 4, 6, 83},
+ { 0, 7, 6, 131}, { 0, 9, 6, 515},
+ { 16, 0, 4, 4}, { 0, 0, 4, 5},
+ { 32, 0, 5, 6}, { 0, 0, 5, 7},
+ { 32, 0, 5, 9}, { 0, 0, 5, 10},
+ { 0, 0, 6, 12}, { 0, 0, 6, 15},
+ { 0, 0, 6, 18}, { 0, 0, 6, 21},
+ { 0, 0, 6, 24}, { 0, 0, 6, 27},
+ { 0, 0, 6, 30}, { 0, 0, 6, 33},
+ { 0, 1, 6, 35}, { 0, 1, 6, 39},
+ { 0, 2, 6, 43}, { 0, 3, 6, 51},
+ { 0, 4, 6, 67}, { 0, 5, 6, 99},
+ { 0, 8, 6, 259}, { 32, 0, 4, 4},
+ { 48, 0, 4, 4}, { 16, 0, 4, 5},
+ { 32, 0, 5, 7}, { 32, 0, 5, 8},
+ { 32, 0, 5, 10}, { 32, 0, 5, 11},
+ { 0, 0, 6, 14}, { 0, 0, 6, 17},
+ { 0, 0, 6, 20}, { 0, 0, 6, 23},
+ { 0, 0, 6, 26}, { 0, 0, 6, 29},
+ { 0, 0, 6, 32}, { 0, 16, 6,65539},
+ { 0, 15, 6,32771}, { 0, 14, 6,16387},
+ { 0, 13, 6, 8195}, { 0, 12, 6, 4099},
+ { 0, 11, 6, 2051}, { 0, 10, 6, 1027},
+}; /* ML_defaultDTable */
+
+
+static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U8 nbAddBits)
+{
+ void* ptr = dt;
+ ZSTD_seqSymbol_header* const DTableH = (ZSTD_seqSymbol_header*)ptr;
+ ZSTD_seqSymbol* const cell = dt + 1;
+
+ DTableH->tableLog = 0;
+ DTableH->fastMode = 0;
+
+ cell->nbBits = 0;
+ cell->nextState = 0;
+ assert(nbAddBits < 255);
+ cell->nbAdditionalBits = nbAddBits;
+ cell->baseValue = baseValue;
+}
+
+
+/* ZSTD_buildFSETable() :
+ * generate FSE decoding table for one symbol (ll, ml or off)
+ * cannot fail if input is valid =>
+ * all inputs are presumed validated at this stage */
+FORCE_INLINE_TEMPLATE
+void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
+ const short* normalizedCounter, unsigned maxSymbolValue,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ unsigned tableLog, void* wksp, size_t wkspSize)
+{
+ ZSTD_seqSymbol* const tableDecode = dt+1;
+ U32 const maxSV1 = maxSymbolValue + 1;
+ U32 const tableSize = 1 << tableLog;
+
+ U16* symbolNext = (U16*)wksp;
+ BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1);
+ U32 highThreshold = tableSize - 1;
+
+
+ /* Sanity Checks */
+ assert(maxSymbolValue <= MaxSeq);
+ assert(tableLog <= MaxFSELog);
+ assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE);
+ (void)wkspSize;
+ /* Init, lay down lowprob symbols */
+ { ZSTD_seqSymbol_header DTableH;
+ DTableH.tableLog = tableLog;
+ DTableH.fastMode = 1;
+ { S16 const largeLimit= (S16)(1 << (tableLog-1));
+ U32 s;
+ for (s=0; s<maxSV1; s++) {
+ if (normalizedCounter[s]==-1) {
+ tableDecode[highThreshold--].baseValue = s;
+ symbolNext[s] = 1;
+ } else {
+ if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
+ assert(normalizedCounter[s]>=0);
+ symbolNext[s] = (U16)normalizedCounter[s];
+ } } }
+ ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
+ }
+
+ /* Spread symbols */
+ assert(tableSize <= 512);
+ /* Specialized symbol spreading for the case when there are
+ * no low probability (-1 count) symbols. When compressing
+ * small blocks we avoid low probability symbols to hit this
+ * case, since header decoding speed matters more.
+ */
+ if (highThreshold == tableSize - 1) {
+ size_t const tableMask = tableSize-1;
+ size_t const step = FSE_TABLESTEP(tableSize);
+ /* First lay down the symbols in order.
+ * We use a uint64_t to lay down 8 bytes at a time. This reduces branch
+ * misses since small blocks generally have small table logs, so nearly
+ * all symbols have counts <= 8. We ensure we have 8 bytes at the end of
+ * our buffer to handle the over-write.
+ */
+ {
+ U64 const add = 0x0101010101010101ull;
+ size_t pos = 0;
+ U64 sv = 0;
+ U32 s;
+ for (s=0; s<maxSV1; ++s, sv += add) {
+ int i;
+ int const n = normalizedCounter[s];
+ MEM_write64(spread + pos, sv);
+ for (i = 8; i < n; i += 8) {
+ MEM_write64(spread + pos + i, sv);
+ }
+ assert(n>=0);
+ pos += (size_t)n;
+ }
+ }
+ /* Now we spread those positions across the table.
+ * The benefit of doing it in two stages is that we avoid the
+ * variable size inner loop, which caused lots of branch misses.
+ * Now we can run through all the positions without any branch misses.
+ * We unroll the loop twice, since that is what empirically worked best.
+ */
+ {
+ size_t position = 0;
+ size_t s;
+ size_t const unroll = 2;
+ assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
+ for (s = 0; s < (size_t)tableSize; s += unroll) {
+ size_t u;
+ for (u = 0; u < unroll; ++u) {
+ size_t const uPosition = (position + (u * step)) & tableMask;
+ tableDecode[uPosition].baseValue = spread[s + u];
+ }
+ position = (position + (unroll * step)) & tableMask;
+ }
+ assert(position == 0);
+ }
+ } else {
+ U32 const tableMask = tableSize-1;
+ U32 const step = FSE_TABLESTEP(tableSize);
+ U32 s, position = 0;
+ for (s=0; s<maxSV1; s++) {
+ int i;
+ int const n = normalizedCounter[s];
+ for (i=0; i<n; i++) {
+ tableDecode[position].baseValue = s;
+ position = (position + step) & tableMask;
+ while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask; /* lowprob area */
+ } }
+ assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
+ }
+
+ /* Build Decoding table */
+ {
+ U32 u;
+ for (u=0; u<tableSize; u++) {
+ U32 const symbol = tableDecode[u].baseValue;
+ U32 const nextState = symbolNext[symbol]++;
+ tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
+ tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
+ assert(nbAdditionalBits[symbol] < 255);
+ tableDecode[u].nbAdditionalBits = nbAdditionalBits[symbol];
+ tableDecode[u].baseValue = baseValue[symbol];
+ }
+ }
+}
+
+/* Avoids the FORCE_INLINE of the _body() function. */
+static void ZSTD_buildFSETable_body_default(ZSTD_seqSymbol* dt,
+ const short* normalizedCounter, unsigned maxSymbolValue,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ unsigned tableLog, void* wksp, size_t wkspSize)
+{
+ ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
+ baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+}
+
+#if DYNAMIC_BMI2
+BMI2_TARGET_ATTRIBUTE static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* dt,
+ const short* normalizedCounter, unsigned maxSymbolValue,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ unsigned tableLog, void* wksp, size_t wkspSize)
+{
+ ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
+ baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+}
+#endif
+
+void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
+ const short* normalizedCounter, unsigned maxSymbolValue,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ unsigned tableLog, void* wksp, size_t wkspSize, int bmi2)
+{
+#if DYNAMIC_BMI2
+ if (bmi2) {
+ ZSTD_buildFSETable_body_bmi2(dt, normalizedCounter, maxSymbolValue,
+ baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+ return;
+ }
+#endif
+ (void)bmi2;
+ ZSTD_buildFSETable_body_default(dt, normalizedCounter, maxSymbolValue,
+ baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
+}
+
+
+/*! ZSTD_buildSeqTable() :
+ * @return : nb bytes read from src,
+ * or an error code if it fails */
+static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr,
+ symbolEncodingType_e type, unsigned max, U32 maxLog,
+ const void* src, size_t srcSize,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable,
+ int ddictIsCold, int nbSeq, U32* wksp, size_t wkspSize,
+ int bmi2)
+{
+ switch(type)
+ {
+ case set_rle :
+ RETURN_ERROR_IF(!srcSize, srcSize_wrong, "");
+ RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected, "");
+ { U32 const symbol = *(const BYTE*)src;
+ U32 const baseline = baseValue[symbol];
+ U8 const nbBits = nbAdditionalBits[symbol];
+ ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits);
+ }
+ *DTablePtr = DTableSpace;
+ return 1;
+ case set_basic :
+ *DTablePtr = defaultTable;
+ return 0;
+ case set_repeat:
+ RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, "");
+ /* prefetch FSE table if used */
+ if (ddictIsCold && (nbSeq > 24 /* heuristic */)) {
+ const void* const pStart = *DTablePtr;
+ size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog));
+ PREFETCH_AREA(pStart, pSize);
+ }
+ return 0;
+ case set_compressed :
+ { unsigned tableLog;
+ S16 norm[MaxSeq+1];
+ size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
+ RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, "");
+ RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, "");
+ ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2);
+ *DTablePtr = DTableSpace;
+ return headerSize;
+ }
+ default :
+ assert(0);
+ RETURN_ERROR(GENERIC, "impossible");
+ }
+}
+
+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
+ const void* src, size_t srcSize)
+{
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* ip = istart;
+ int nbSeq;
+ DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
+
+ /* check */
+ RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, "");
+
+ /* SeqHead */
+ nbSeq = *ip++;
+ if (!nbSeq) {
+ *nbSeqPtr=0;
+ RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, "");
+ return 1;
+ }
+ if (nbSeq > 0x7F) {
+ if (nbSeq == 0xFF) {
+ RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, "");
+ nbSeq = MEM_readLE16(ip) + LONGNBSEQ;
+ ip+=2;
+ } else {
+ RETURN_ERROR_IF(ip >= iend, srcSize_wrong, "");
+ nbSeq = ((nbSeq-0x80)<<8) + *ip++;
+ }
+ }
+ *nbSeqPtr = nbSeq;
+
+ /* FSE table descriptors */
+ RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */
+ { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
+ symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
+ symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
+ ip++;
+
+ /* Build DTables */
+ { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
+ LLtype, MaxLL, LLFSELog,
+ ip, iend-ip,
+ LL_base, LL_bits,
+ LL_defaultDTable, dctx->fseEntropy,
+ dctx->ddictIsCold, nbSeq,
+ dctx->workspace, sizeof(dctx->workspace),
+ ZSTD_DCtx_get_bmi2(dctx));
+ RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed");
+ ip += llhSize;
+ }
+
+ { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
+ OFtype, MaxOff, OffFSELog,
+ ip, iend-ip,
+ OF_base, OF_bits,
+ OF_defaultDTable, dctx->fseEntropy,
+ dctx->ddictIsCold, nbSeq,
+ dctx->workspace, sizeof(dctx->workspace),
+ ZSTD_DCtx_get_bmi2(dctx));
+ RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed");
+ ip += ofhSize;
+ }
+
+ { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
+ MLtype, MaxML, MLFSELog,
+ ip, iend-ip,
+ ML_base, ML_bits,
+ ML_defaultDTable, dctx->fseEntropy,
+ dctx->ddictIsCold, nbSeq,
+ dctx->workspace, sizeof(dctx->workspace),
+ ZSTD_DCtx_get_bmi2(dctx));
+ RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed");
+ ip += mlhSize;
+ }
+ }
+
+ return ip-istart;
+}
+
+
+typedef struct {
+ size_t litLength;
+ size_t matchLength;
+ size_t offset;
+} seq_t;
+
+typedef struct {
+ size_t state;
+ const ZSTD_seqSymbol* table;
+} ZSTD_fseState;
+
+typedef struct {
+ BIT_DStream_t DStream;
+ ZSTD_fseState stateLL;
+ ZSTD_fseState stateOffb;
+ ZSTD_fseState stateML;
+ size_t prevOffset[ZSTD_REP_NUM];
+} seqState_t;
+
+/*! ZSTD_overlapCopy8() :
+ * Copies 8 bytes from ip to op and updates op and ip where ip <= op.
+ * If the offset is < 8 then the offset is spread to at least 8 bytes.
+ *
+ * Precondition: *ip <= *op
+ * Postcondition: *op - *op >= 8
+ */
+HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) {
+ assert(*ip <= *op);
+ if (offset < 8) {
+ /* close range match, overlap */
+ static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
+ int const sub2 = dec64table[offset];
+ (*op)[0] = (*ip)[0];
+ (*op)[1] = (*ip)[1];
+ (*op)[2] = (*ip)[2];
+ (*op)[3] = (*ip)[3];
+ *ip += dec32table[offset];
+ ZSTD_copy4(*op+4, *ip);
+ *ip -= sub2;
+ } else {
+ ZSTD_copy8(*op, *ip);
+ }
+ *ip += 8;
+ *op += 8;
+ assert(*op - *ip >= 8);
+}
+
+/*! ZSTD_safecopy() :
+ * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer
+ * and write up to 16 bytes past oend_w (op >= oend_w is allowed).
+ * This function is only called in the uncommon case where the sequence is near the end of the block. It
+ * should be fast for a single long sequence, but can be slow for several short sequences.
+ *
+ * @param ovtype controls the overlap detection
+ * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
+ * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
+ * The src buffer must be before the dst buffer.
+ */
+static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) {
+ ptrdiff_t const diff = op - ip;
+ BYTE* const oend = op + length;
+
+ assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) ||
+ (ovtype == ZSTD_overlap_src_before_dst && diff >= 0));
+
+ if (length < 8) {
+ /* Handle short lengths. */
+ while (op < oend) *op++ = *ip++;
+ return;
+ }
+ if (ovtype == ZSTD_overlap_src_before_dst) {
+ /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */
+ assert(length >= 8);
+ ZSTD_overlapCopy8(&op, &ip, diff);
+ length -= 8;
+ assert(op - ip >= 8);
+ assert(op <= oend);
+ }
+
+ if (oend <= oend_w) {
+ /* No risk of overwrite. */
+ ZSTD_wildcopy(op, ip, length, ovtype);
+ return;
+ }
+ if (op <= oend_w) {
+ /* Wildcopy until we get close to the end. */
+ assert(oend > oend_w);
+ ZSTD_wildcopy(op, ip, oend_w - op, ovtype);
+ ip += oend_w - op;
+ op += oend_w - op;
+ }
+ /* Handle the leftovers. */
+ while (op < oend) *op++ = *ip++;
+}
+
+/* ZSTD_safecopyDstBeforeSrc():
+ * This version allows overlap with dst before src, or handles the non-overlap case with dst after src
+ * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */
+static void ZSTD_safecopyDstBeforeSrc(BYTE* op, BYTE const* ip, ptrdiff_t length) {
+ ptrdiff_t const diff = op - ip;
+ BYTE* const oend = op + length;
+
+ if (length < 8 || diff > -8) {
+ /* Handle short lengths, close overlaps, and dst not before src. */
+ while (op < oend) *op++ = *ip++;
+ return;
+ }
+
+ if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) {
+ ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap);
+ ip += oend - WILDCOPY_OVERLENGTH - op;
+ op += oend - WILDCOPY_OVERLENGTH - op;
+ }
+
+ /* Handle the leftovers. */
+ while (op < oend) *op++ = *ip++;
+}
+
+/* ZSTD_execSequenceEnd():
+ * This version handles cases that are near the end of the output buffer. It requires
+ * more careful checks to make sure there is no overflow. By separating out these hard
+ * and unlikely cases, we can speed up the common cases.
+ *
+ * NOTE: This function needs to be fast for a single long sequence, but doesn't need
+ * to be optimized for many small sequences, since those fall into ZSTD_execSequence().
+ */
+FORCE_NOINLINE
+size_t ZSTD_execSequenceEnd(BYTE* op,
+ BYTE* const oend, seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit,
+ const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+ BYTE* const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE* match = oLitEnd - sequence.offset;
+ BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
+
+ /* bounds checks : careful of address space overflow in 32-bit mode */
+ RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
+ RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
+ assert(op < op + sequenceLength);
+ assert(oLitEnd < op + sequenceLength);
+
+ /* copy literals */
+ ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap);
+ op = oLitEnd;
+ *litPtr = iLitEnd;
+
+ /* copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+ /* offset beyond prefix */
+ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
+ match = dictEnd - (prefixStart - match);
+ if (match + sequence.matchLength <= dictEnd) {
+ ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ { size_t const length1 = dictEnd - match;
+ ZSTD_memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = prefixStart;
+ }
+ }
+ ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
+ return sequenceLength;
+}
+
+/* ZSTD_execSequenceEndSplitLitBuffer():
+ * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case.
+ */
+FORCE_NOINLINE
+size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op,
+ BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit,
+ const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+ BYTE* const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE* match = oLitEnd - sequence.offset;
+
+
+ /* bounds checks : careful of address space overflow in 32-bit mode */
+ RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
+ RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
+ assert(op < op + sequenceLength);
+ assert(oLitEnd < op + sequenceLength);
+
+ /* copy literals */
+ RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer");
+ ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength);
+ op = oLitEnd;
+ *litPtr = iLitEnd;
+
+ /* copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+ /* offset beyond prefix */
+ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
+ match = dictEnd - (prefixStart - match);
+ if (match + sequence.matchLength <= dictEnd) {
+ ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ { size_t const length1 = dictEnd - match;
+ ZSTD_memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = prefixStart;
+ }
+ }
+ ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
+ return sequenceLength;
+}
+
+HINT_INLINE
+size_t ZSTD_execSequence(BYTE* op,
+ BYTE* const oend, seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit,
+ const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+ BYTE* const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
+ BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE* match = oLitEnd - sequence.offset;
+
+ assert(op != NULL /* Precondition */);
+ assert(oend_w < oend /* No underflow */);
+
+#if defined(__aarch64__)
+ /* prefetch sequence starting from match that will be used for copy later */
+ PREFETCH_L1(match);
+#endif
+ /* Handle edge cases in a slow path:
+ * - Read beyond end of literals
+ * - Match end is within WILDCOPY_OVERLIMIT of oend
+ * - 32-bit mode and the match length overflows
+ */
+ if (UNLIKELY(
+ iLitEnd > litLimit ||
+ oMatchEnd > oend_w ||
+ (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
+ return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
+
+ /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
+ assert(op <= oLitEnd /* No overflow */);
+ assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
+ assert(oMatchEnd <= oend /* No underflow */);
+ assert(iLitEnd <= litLimit /* Literal length is in bounds */);
+ assert(oLitEnd <= oend_w /* Can wildcopy literals */);
+ assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
+
+ /* Copy Literals:
+ * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
+ * We likely don't need the full 32-byte wildcopy.
+ */
+ assert(WILDCOPY_OVERLENGTH >= 16);
+ ZSTD_copy16(op, (*litPtr));
+ if (UNLIKELY(sequence.litLength > 16)) {
+ ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap);
+ }
+ op = oLitEnd;
+ *litPtr = iLitEnd; /* update for next sequence */
+
+ /* Copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+ /* offset beyond prefix -> go into extDict */
+ RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
+ match = dictEnd + (match - prefixStart);
+ if (match + sequence.matchLength <= dictEnd) {
+ ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ { size_t const length1 = dictEnd - match;
+ ZSTD_memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = prefixStart;
+ }
+ }
+ /* Match within prefix of 1 or more bytes */
+ assert(op <= oMatchEnd);
+ assert(oMatchEnd <= oend_w);
+ assert(match >= prefixStart);
+ assert(sequence.matchLength >= 1);
+
+ /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
+ * without overlap checking.
+ */
+ if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
+ /* We bet on a full wildcopy for matches, since we expect matches to be
+ * longer than literals (in general). In silesia, ~10% of matches are longer
+ * than 16 bytes.
+ */
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
+ return sequenceLength;
+ }
+ assert(sequence.offset < WILDCOPY_VECLEN);
+
+ /* Copy 8 bytes and spread the offset to be >= 8. */
+ ZSTD_overlapCopy8(&op, &match, sequence.offset);
+
+ /* If the match length is > 8 bytes, then continue with the wildcopy. */
+ if (sequence.matchLength > 8) {
+ assert(op < oMatchEnd);
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst);
+ }
+ return sequenceLength;
+}
+
+HINT_INLINE
+size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op,
+ BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit,
+ const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
+{
+ BYTE* const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE* match = oLitEnd - sequence.offset;
+
+ assert(op != NULL /* Precondition */);
+ assert(oend_w < oend /* No underflow */);
+ /* Handle edge cases in a slow path:
+ * - Read beyond end of literals
+ * - Match end is within WILDCOPY_OVERLIMIT of oend
+ * - 32-bit mode and the match length overflows
+ */
+ if (UNLIKELY(
+ iLitEnd > litLimit ||
+ oMatchEnd > oend_w ||
+ (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
+ return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
+
+ /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
+ assert(op <= oLitEnd /* No overflow */);
+ assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
+ assert(oMatchEnd <= oend /* No underflow */);
+ assert(iLitEnd <= litLimit /* Literal length is in bounds */);
+ assert(oLitEnd <= oend_w /* Can wildcopy literals */);
+ assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
+
+ /* Copy Literals:
+ * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
+ * We likely don't need the full 32-byte wildcopy.
+ */
+ assert(WILDCOPY_OVERLENGTH >= 16);
+ ZSTD_copy16(op, (*litPtr));
+ if (UNLIKELY(sequence.litLength > 16)) {
+ ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap);
+ }
+ op = oLitEnd;
+ *litPtr = iLitEnd; /* update for next sequence */
+
+ /* Copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
+ /* offset beyond prefix -> go into extDict */
+ RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
+ match = dictEnd + (match - prefixStart);
+ if (match + sequence.matchLength <= dictEnd) {
+ ZSTD_memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ { size_t const length1 = dictEnd - match;
+ ZSTD_memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = prefixStart;
+ } }
+ /* Match within prefix of 1 or more bytes */
+ assert(op <= oMatchEnd);
+ assert(oMatchEnd <= oend_w);
+ assert(match >= prefixStart);
+ assert(sequence.matchLength >= 1);
+
+ /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
+ * without overlap checking.
+ */
+ if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
+ /* We bet on a full wildcopy for matches, since we expect matches to be
+ * longer than literals (in general). In silesia, ~10% of matches are longer
+ * than 16 bytes.
+ */
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
+ return sequenceLength;
+ }
+ assert(sequence.offset < WILDCOPY_VECLEN);
+
+ /* Copy 8 bytes and spread the offset to be >= 8. */
+ ZSTD_overlapCopy8(&op, &match, sequence.offset);
+
+ /* If the match length is > 8 bytes, then continue with the wildcopy. */
+ if (sequence.matchLength > 8) {
+ assert(op < oMatchEnd);
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst);
+ }
+ return sequenceLength;
+}
+
+
+static void
+ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt)
+{
+ const void* ptr = dt;
+ const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr;
+ DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
+ DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits",
+ (U32)DStatePtr->state, DTableH->tableLog);
+ BIT_reloadDStream(bitD);
+ DStatePtr->table = dt + 1;
+}
+
+FORCE_INLINE_TEMPLATE void
+ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits)
+{
+ size_t const lowBits = BIT_readBits(bitD, nbBits);
+ DStatePtr->state = nextState + lowBits;
+}
+
+/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
+ * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32
+ * bits before reloading. This value is the maximum number of bytes we read
+ * after reloading when we are decoding long offsets.
+ */
+#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \
+ (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \
+ ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \
+ : 0)
+
+typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e;
+
+FORCE_INLINE_TEMPLATE seq_t
+ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
+{
+ seq_t seq;
+ /*
+ * ZSTD_seqSymbol is a structure with a total of 64 bits wide. So it can be
+ * loaded in one operation and extracted its fields by simply shifting or
+ * bit-extracting on aarch64.
+ * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh
+ * operations that cause performance drop. This can be avoided by using this
+ * ZSTD_memcpy hack.
+ */
+#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__))
+ ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS;
+ ZSTD_seqSymbol* const llDInfo = &llDInfoS;
+ ZSTD_seqSymbol* const mlDInfo = &mlDInfoS;
+ ZSTD_seqSymbol* const ofDInfo = &ofDInfoS;
+ ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol));
+ ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol));
+ ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol));
+#else
+ const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
+ const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
+ const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
+#endif
+ seq.matchLength = mlDInfo->baseValue;
+ seq.litLength = llDInfo->baseValue;
+ { U32 const ofBase = ofDInfo->baseValue;
+ BYTE const llBits = llDInfo->nbAdditionalBits;
+ BYTE const mlBits = mlDInfo->nbAdditionalBits;
+ BYTE const ofBits = ofDInfo->nbAdditionalBits;
+ BYTE const totalBits = llBits+mlBits+ofBits;
+
+ U16 const llNext = llDInfo->nextState;
+ U16 const mlNext = mlDInfo->nextState;
+ U16 const ofNext = ofDInfo->nextState;
+ U32 const llnbBits = llDInfo->nbBits;
+ U32 const mlnbBits = mlDInfo->nbBits;
+ U32 const ofnbBits = ofDInfo->nbBits;
+
+ assert(llBits <= MaxLLBits);
+ assert(mlBits <= MaxMLBits);
+ assert(ofBits <= MaxOff);
+ /*
+ * As gcc has better branch and block analyzers, sometimes it is only
+ * valuable to mark likeliness for clang, it gives around 3-4% of
+ * performance.
+ */
+
+ /* sequence */
+ { size_t offset;
+ #if defined(__clang__)
+ if (LIKELY(ofBits > 1)) {
+ #else
+ if (ofBits > 1) {
+ #endif
+ ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
+ ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
+ ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32);
+ ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits);
+ if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) {
+ /* Always read extra bits, this keeps the logic simple,
+ * avoids branches, and avoids accidentally reading 0 bits.
+ */
+ U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32;
+ offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
+ BIT_reloadDStream(&seqState->DStream);
+ offset += BIT_readBitsFast(&seqState->DStream, extraBits);
+ } else {
+ offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
+ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
+ }
+ seqState->prevOffset[2] = seqState->prevOffset[1];
+ seqState->prevOffset[1] = seqState->prevOffset[0];
+ seqState->prevOffset[0] = offset;
+ } else {
+ U32 const ll0 = (llDInfo->baseValue == 0);
+ if (LIKELY((ofBits == 0))) {
+ offset = seqState->prevOffset[ll0];
+ seqState->prevOffset[1] = seqState->prevOffset[!ll0];
+ seqState->prevOffset[0] = offset;
+ } else {
+ offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1);
+ { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
+ if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
+ seqState->prevOffset[1] = seqState->prevOffset[0];
+ seqState->prevOffset[0] = offset = temp;
+ } } }
+ seq.offset = offset;
+ }
+
+ #if defined(__clang__)
+ if (UNLIKELY(mlBits > 0))
+ #else
+ if (mlBits > 0)
+ #endif
+ seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/);
+
+ if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
+ BIT_reloadDStream(&seqState->DStream);
+ if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
+ BIT_reloadDStream(&seqState->DStream);
+ /* Ensure there are enough bits to read the rest of data in 64-bit mode. */
+ ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
+
+ #if defined(__clang__)
+ if (UNLIKELY(llBits > 0))
+ #else
+ if (llBits > 0)
+ #endif
+ seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
+
+ if (MEM_32bits())
+ BIT_reloadDStream(&seqState->DStream);
+
+ DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
+ (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
+
+ ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */
+ ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */
+ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
+ ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */
+ }
+
+ return seq;
+}
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd)
+{
+ size_t const windowSize = dctx->fParams.windowSize;
+ /* No dictionary used. */
+ if (dctx->dictContentEndForFuzzing == NULL) return 0;
+ /* Dictionary is our prefix. */
+ if (prefixStart == dctx->dictContentBeginForFuzzing) return 1;
+ /* Dictionary is not our ext-dict. */
+ if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0;
+ /* Dictionary is not within our window size. */
+ if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0;
+ /* Dictionary is active. */
+ return 1;
+}
+
+MEM_STATIC void ZSTD_assertValidSequence(
+ ZSTD_DCtx const* dctx,
+ BYTE const* op, BYTE const* oend,
+ seq_t const seq,
+ BYTE const* prefixStart, BYTE const* virtualStart)
+{
+#if DEBUGLEVEL >= 1
+ size_t const windowSize = dctx->fParams.windowSize;
+ size_t const sequenceSize = seq.litLength + seq.matchLength;
+ BYTE const* const oLitEnd = op + seq.litLength;
+ DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u",
+ (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
+ assert(op <= oend);
+ assert((size_t)(oend - op) >= sequenceSize);
+ assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX);
+ if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) {
+ size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing);
+ /* Offset must be within the dictionary. */
+ assert(seq.offset <= (size_t)(oLitEnd - virtualStart));
+ assert(seq.offset <= windowSize + dictSize);
+ } else {
+ /* Offset must be within our window. */
+ assert(seq.offset <= windowSize);
+ }
+#else
+ (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart;
+#endif
+}
+#endif
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+
+
+FORCE_INLINE_TEMPLATE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ const BYTE* ip = (const BYTE*)seqStart;
+ const BYTE* const iend = ip + seqSize;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = ostart + maxDstSize;
+ BYTE* op = ostart;
+ const BYTE* litPtr = dctx->litPtr;
+ const BYTE* litBufferEnd = dctx->litBufferEnd;
+ const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
+ const BYTE* const vBase = (const BYTE*) (dctx->virtualStart);
+ const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
+ DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer");
+ (void)frame;
+
+ /* Regen sequences */
+ if (nbSeq) {
+ seqState_t seqState;
+ dctx->fseEntropy = 1;
+ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
+ RETURN_ERROR_IF(
+ ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
+ corruption_detected, "");
+ ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+ ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+ ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+ assert(dst != NULL);
+
+ ZSTD_STATIC_ASSERT(
+ BIT_DStream_unfinished < BIT_DStream_completed &&
+ BIT_DStream_endOfBuffer < BIT_DStream_completed &&
+ BIT_DStream_completed < BIT_DStream_overflow);
+
+ /* decompress without overrunning litPtr begins */
+ {
+ seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ /* Align the decompression loop to 32 + 16 bytes.
+ *
+ * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression
+ * speed swings based on the alignment of the decompression loop. This
+ * performance swing is caused by parts of the decompression loop falling
+ * out of the DSB. The entire decompression loop should fit in the DSB,
+ * when it can't we get much worse performance. You can measure if you've
+ * hit the good case or the bad case with this perf command for some
+ * compressed file test.zst:
+ *
+ * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \
+ * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst
+ *
+ * If you see most cycles served out of the MITE you've hit the bad case.
+ * If you see most cycles served out of the DSB you've hit the good case.
+ * If it is pretty even then you may be in an okay case.
+ *
+ * This issue has been reproduced on the following CPUs:
+ * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9
+ * Use Instruments->Counters to get DSB/MITE cycles.
+ * I never got performance swings, but I was able to
+ * go from the good case of mostly DSB to half of the
+ * cycles served from MITE.
+ * - Coffeelake: Intel i9-9900k
+ * - Coffeelake: Intel i7-9700k
+ *
+ * I haven't been able to reproduce the instability or DSB misses on any
+ * of the following CPUS:
+ * - Haswell
+ * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH
+ * - Skylake
+ *
+ * Alignment is done for each of the three major decompression loops:
+ * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer
+ * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer
+ * - ZSTD_decompressSequences_body
+ * Alignment choices are made to minimize large swings on bad cases and influence on performance
+ * from changes external to this code, rather than to overoptimize on the current commit.
+ *
+ * If you are seeing performance stability this script can help test.
+ * It tests on 4 commits in zstd where I saw performance change.
+ *
+ * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4
+ */
+#if defined(__GNUC__) && defined(__x86_64__)
+ __asm__(".p2align 6");
+# if __GNUC__ >= 7
+ /* good for gcc-7, gcc-9, and gcc-11 */
+ __asm__("nop");
+ __asm__(".p2align 5");
+ __asm__("nop");
+ __asm__(".p2align 4");
+# if __GNUC__ == 8 || __GNUC__ == 10
+ /* good for gcc-8 and gcc-10 */
+ __asm__("nop");
+ __asm__(".p2align 3");
+# endif
+# endif
+#endif
+
+ /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */
+ for (; litPtr + sequence.litLength <= dctx->litBufferEnd; ) {
+ size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+ if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+ return oneSeqSize;
+ DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+ op += oneSeqSize;
+ if (UNLIKELY(!--nbSeq))
+ break;
+ BIT_reloadDStream(&(seqState.DStream));
+ sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ }
+
+ /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */
+ if (nbSeq > 0) {
+ const size_t leftoverLit = dctx->litBufferEnd - litPtr;
+ if (leftoverLit)
+ {
+ RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
+ ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
+ sequence.litLength -= leftoverLit;
+ op += leftoverLit;
+ }
+ litPtr = dctx->litExtraBuffer;
+ litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ {
+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+ if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+ return oneSeqSize;
+ DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+ op += oneSeqSize;
+ if (--nbSeq)
+ BIT_reloadDStream(&(seqState.DStream));
+ }
+ }
+ }
+
+ if (nbSeq > 0) /* there is remaining lit from extra buffer */
+ {
+
+#if defined(__GNUC__) && defined(__x86_64__)
+ __asm__(".p2align 6");
+ __asm__("nop");
+# if __GNUC__ != 7
+ /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */
+ __asm__(".p2align 4");
+ __asm__("nop");
+ __asm__(".p2align 3");
+# elif __GNUC__ >= 11
+ __asm__(".p2align 3");
+# else
+ __asm__(".p2align 5");
+ __asm__("nop");
+ __asm__(".p2align 3");
+# endif
+#endif
+
+ for (; ; ) {
+ seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+ if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+ return oneSeqSize;
+ DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+ op += oneSeqSize;
+ if (UNLIKELY(!--nbSeq))
+ break;
+ BIT_reloadDStream(&(seqState.DStream));
+ }
+ }
+
+ /* check if reached exact end */
+ DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq);
+ RETURN_ERROR_IF(nbSeq, corruption_detected, "");
+ RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, "");
+ /* save reps for next block */
+ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
+ }
+
+ /* last literal segment */
+ if (dctx->litBufferLocation == ZSTD_split) /* split hasn't been reached yet, first get dst then copy litExtraBuffer */
+ {
+ size_t const lastLLSize = litBufferEnd - litPtr;
+ RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
+ if (op != NULL) {
+ ZSTD_memmove(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+ litPtr = dctx->litExtraBuffer;
+ litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ }
+ { size_t const lastLLSize = litBufferEnd - litPtr;
+ RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
+ if (op != NULL) {
+ ZSTD_memcpy(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+ }
+
+ return op-ostart;
+}
+
+FORCE_INLINE_TEMPLATE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ const BYTE* ip = (const BYTE*)seqStart;
+ const BYTE* const iend = ip + seqSize;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ostart + maxDstSize : dctx->litBuffer;
+ BYTE* op = ostart;
+ const BYTE* litPtr = dctx->litPtr;
+ const BYTE* const litEnd = litPtr + dctx->litSize;
+ const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
+ const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
+ const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
+ DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq);
+ (void)frame;
+
+ /* Regen sequences */
+ if (nbSeq) {
+ seqState_t seqState;
+ dctx->fseEntropy = 1;
+ { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
+ RETURN_ERROR_IF(
+ ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)),
+ corruption_detected, "");
+ ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+ ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+ ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+ assert(dst != NULL);
+
+ ZSTD_STATIC_ASSERT(
+ BIT_DStream_unfinished < BIT_DStream_completed &&
+ BIT_DStream_endOfBuffer < BIT_DStream_completed &&
+ BIT_DStream_completed < BIT_DStream_overflow);
+
+#if defined(__GNUC__) && defined(__x86_64__)
+ __asm__(".p2align 6");
+ __asm__("nop");
+# if __GNUC__ >= 7
+ __asm__(".p2align 5");
+ __asm__("nop");
+ __asm__(".p2align 3");
+# else
+ __asm__(".p2align 4");
+ __asm__("nop");
+ __asm__(".p2align 3");
+# endif
+#endif
+
+ for ( ; ; ) {
+ seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
+#endif
+ if (UNLIKELY(ZSTD_isError(oneSeqSize)))
+ return oneSeqSize;
+ DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
+ op += oneSeqSize;
+ if (UNLIKELY(!--nbSeq))
+ break;
+ BIT_reloadDStream(&(seqState.DStream));
+ }
+
+ /* check if reached exact end */
+ DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq);
+ RETURN_ERROR_IF(nbSeq, corruption_detected, "");
+ RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, "");
+ /* save reps for next block */
+ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
+ }
+
+ /* last literal segment */
+ { size_t const lastLLSize = litEnd - litPtr;
+ RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
+ if (op != NULL) {
+ ZSTD_memcpy(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+ }
+
+ return op-ostart;
+}
+
+static size_t
+ZSTD_decompressSequences_default(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+
+static size_t
+ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence,
+ const BYTE* const prefixStart, const BYTE* const dictEnd)
+{
+ prefetchPos += sequence.litLength;
+ { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart;
+ const BYTE* const match = matchBase + prefetchPos - sequence.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted.
+ * No consequence though : memory address is only used for prefetching, not for dereferencing */
+ PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
+ }
+ return prefetchPos + sequence.matchLength;
+}
+
+/* This decoding function employs prefetching
+ * to reduce latency impact of cache misses.
+ * It's generally employed when block contains a significant portion of long-distance matches
+ * or when coupled with a "cold" dictionary */
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_decompressSequencesLong_body(
+ ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ const BYTE* ip = (const BYTE*)seqStart;
+ const BYTE* const iend = ip + seqSize;
+ BYTE* const ostart = (BYTE*)dst;
+ BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ostart + maxDstSize;
+ BYTE* op = ostart;
+ const BYTE* litPtr = dctx->litPtr;
+ const BYTE* litBufferEnd = dctx->litBufferEnd;
+ const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
+ const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart);
+ const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
+ (void)frame;
+
+ /* Regen sequences */
+ if (nbSeq) {
+#define STORED_SEQS 8
+#define STORED_SEQS_MASK (STORED_SEQS-1)
+#define ADVANCED_SEQS STORED_SEQS
+ seq_t sequences[STORED_SEQS];
+ int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
+ seqState_t seqState;
+ int seqNb;
+ size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */
+
+ dctx->fseEntropy = 1;
+ { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
+ assert(dst != NULL);
+ assert(iend >= ip);
+ RETURN_ERROR_IF(
+ ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
+ corruption_detected, "");
+ ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+ ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+ ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+
+ /* prepare in advance */
+ for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNb<seqAdvance); seqNb++) {
+ seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
+ sequences[seqNb] = sequence;
+ }
+ RETURN_ERROR_IF(seqNb<seqAdvance, corruption_detected, "");
+
+ /* decompress without stomping litBuffer */
+ for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb < nbSeq); seqNb++) {
+ seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
+ size_t oneSeqSize;
+
+ if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd)
+ {
+ /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */
+ const size_t leftoverLit = dctx->litBufferEnd - litPtr;
+ if (leftoverLit)
+ {
+ RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
+ ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
+ sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit;
+ op += leftoverLit;
+ }
+ litPtr = dctx->litExtraBuffer;
+ litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+ if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+
+ prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
+ sequences[seqNb & STORED_SEQS_MASK] = sequence;
+ op += oneSeqSize;
+ }
+ else
+ {
+ /* lit buffer is either wholly contained in first or second split, or not split at all*/
+ oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
+ ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
+ ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+ if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+
+ prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
+ sequences[seqNb & STORED_SEQS_MASK] = sequence;
+ op += oneSeqSize;
+ }
+ }
+ RETURN_ERROR_IF(seqNb<nbSeq, corruption_detected, "");
+
+ /* finish queue */
+ seqNb -= seqAdvance;
+ for ( ; seqNb<nbSeq ; seqNb++) {
+ seq_t *sequence = &(sequences[seqNb&STORED_SEQS_MASK]);
+ if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd)
+ {
+ const size_t leftoverLit = dctx->litBufferEnd - litPtr;
+ if (leftoverLit)
+ {
+ RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
+ ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
+ sequence->litLength -= leftoverLit;
+ op += leftoverLit;
+ }
+ litPtr = dctx->litExtraBuffer;
+ litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+ dctx->litBufferLocation = ZSTD_not_in_dst;
+ {
+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+ if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+ op += oneSeqSize;
+ }
+ }
+ else
+ {
+ size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
+ ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
+ ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
+#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
+ assert(!ZSTD_isError(oneSeqSize));
+ if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
+#endif
+ if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+ op += oneSeqSize;
+ }
+ }
+
+ /* save reps for next block */
+ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
+ }
+
+ /* last literal segment */
+ if (dctx->litBufferLocation == ZSTD_split) /* first deplete literal buffer in dst, then copy litExtraBuffer */
+ {
+ size_t const lastLLSize = litBufferEnd - litPtr;
+ RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
+ if (op != NULL) {
+ ZSTD_memmove(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+ litPtr = dctx->litExtraBuffer;
+ litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
+ }
+ { size_t const lastLLSize = litBufferEnd - litPtr;
+ RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
+ if (op != NULL) {
+ ZSTD_memmove(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+ }
+
+ return op-ostart;
+}
+
+static size_t
+ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
+
+
+
+#if DYNAMIC_BMI2
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+static BMI2_TARGET_ATTRIBUTE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+static BMI2_TARGET_ATTRIBUTE size_t
+DONT_VECTORIZE
+ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+static BMI2_TARGET_ATTRIBUTE size_t
+ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
+
+#endif /* DYNAMIC_BMI2 */
+
+typedef size_t (*ZSTD_decompressSequences_t)(
+ ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame);
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+static size_t
+ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ DEBUGLOG(5, "ZSTD_decompressSequences");
+#if DYNAMIC_BMI2
+ if (ZSTD_DCtx_get_bmi2(dctx)) {
+ return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+ }
+#endif
+ return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+static size_t
+ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer");
+#if DYNAMIC_BMI2
+ if (ZSTD_DCtx_get_bmi2(dctx)) {
+ return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+ }
+#endif
+ return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
+
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+/* ZSTD_decompressSequencesLong() :
+ * decompression function triggered when a minimum share of offsets is considered "long",
+ * aka out of cache.
+ * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance".
+ * This function will try to mitigate main memory latency through the use of prefetching */
+static size_t
+ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize, int nbSeq,
+ const ZSTD_longOffset_e isLongOffset,
+ const int frame)
+{
+ DEBUGLOG(5, "ZSTD_decompressSequencesLong");
+#if DYNAMIC_BMI2
+ if (ZSTD_DCtx_get_bmi2(dctx)) {
+ return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+ }
+#endif
+ return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
+}
+#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
+
+
+/**
+ * @returns The total size of the history referencable by zstd, including
+ * both the prefix and the extDict. At @p op any offset larger than this
+ * is invalid.
+ */
+static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart)
+{
+ return (size_t)(op - virtualStart);
+}
+
+typedef struct {
+ unsigned longOffsetShare;
+ unsigned maxNbAdditionalBits;
+} ZSTD_OffsetInfo;
+
+/* ZSTD_getOffsetInfo() :
+ * condition : offTable must be valid
+ * @return : "share" of long offsets (arbitrarily defined as > (1<<23))
+ * compared to maximum possible of (1<<OffFSELog),
+ * as well as the maximum number additional bits required.
+ */
+static ZSTD_OffsetInfo
+ZSTD_getOffsetInfo(const ZSTD_seqSymbol* offTable, int nbSeq)
+{
+ ZSTD_OffsetInfo info = {0, 0};
+ /* If nbSeq == 0, then the offTable is uninitialized, but we have
+ * no sequences, so both values should be 0.
+ */
+ if (nbSeq != 0) {
+ const void* ptr = offTable;
+ U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
+ const ZSTD_seqSymbol* table = offTable + 1;
+ U32 const max = 1 << tableLog;
+ U32 u;
+ DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
+
+ assert(max <= (1 << OffFSELog)); /* max not too large */
+ for (u=0; u<max; u++) {
+ info.maxNbAdditionalBits = MAX(info.maxNbAdditionalBits, table[u].nbAdditionalBits);
+ if (table[u].nbAdditionalBits > 22) info.longOffsetShare += 1;
+ }
+
+ assert(tableLog <= OffFSELog);
+ info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */
+ }
+
+ return info;
+}
+
+/**
+ * @returns The maximum offset we can decode in one read of our bitstream, without
+ * reloading more bits in the middle of the offset bits read. Any offsets larger
+ * than this must use the long offset decoder.
+ */
+static size_t ZSTD_maxShortOffset(void)
+{
+ if (MEM_64bits()) {
+ /* We can decode any offset without reloading bits.
+ * This might change if the max window size grows.
+ */
+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+ return (size_t)-1;
+ } else {
+ /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1.
+ * This offBase would require STREAM_ACCUMULATOR_MIN extra bits.
+ * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset.
+ */
+ size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1;
+ size_t const maxOffset = maxOffbase - ZSTD_REP_NUM;
+ assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN);
+ return maxOffset;
+ }
+}
+
+size_t
+ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, const int frame, const streaming_operation streaming)
+{ /* blockType == blockCompressed */
+ const BYTE* ip = (const BYTE*)src;
+ DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
+
+ /* Note : the wording of the specification
+ * allows compressed block to be sized exactly ZSTD_BLOCKSIZE_MAX.
+ * This generally does not happen, as it makes little sense,
+ * since an uncompressed block would feature same size and have no decompression cost.
+ * Also, note that decoder from reference libzstd before < v1.5.4
+ * would consider this edge case as an error.
+ * As a consequence, avoid generating compressed blocks of size ZSTD_BLOCKSIZE_MAX
+ * for broader compatibility with the deployed ecosystem of zstd decoders */
+ RETURN_ERROR_IF(srcSize > ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
+
+ /* Decode literals section */
+ { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
+ DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize);
+ if (ZSTD_isError(litCSize)) return litCSize;
+ ip += litCSize;
+ srcSize -= litCSize;
+ }
+
+ /* Build Decoding Tables */
+ {
+ /* Compute the maximum block size, which must also work when !frame and fParams are unset.
+ * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t.
+ */
+ size_t const blockSizeMax = MIN(dstCapacity, (frame ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX));
+ size_t const totalHistorySize = ZSTD_totalHistorySize((BYTE*)dst + blockSizeMax, (BYTE const*)dctx->virtualStart);
+ /* isLongOffset must be true if there are long offsets.
+ * Offsets are long if they are larger than ZSTD_maxShortOffset().
+ * We don't expect that to be the case in 64-bit mode.
+ *
+ * We check here to see if our history is large enough to allow long offsets.
+ * If it isn't, then we can't possible have (valid) long offsets. If the offset
+ * is invalid, then it is okay to read it incorrectly.
+ *
+ * If isLongOffsets is true, then we will later check our decoding table to see
+ * if it is even possible to generate long offsets.
+ */
+ ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset()));
+ /* These macros control at build-time which decompressor implementation
+ * we use. If neither is defined, we do some inspection and dispatch at
+ * runtime.
+ */
+#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
+ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
+ int usePrefetchDecoder = dctx->ddictIsCold;
+#else
+ /* Set to 1 to avoid computing offset info if we don't need to.
+ * Otherwise this value is ignored.
+ */
+ int usePrefetchDecoder = 1;
+#endif
+ int nbSeq;
+ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize);
+ if (ZSTD_isError(seqHSize)) return seqHSize;
+ ip += seqHSize;
+ srcSize -= seqHSize;
+
+ RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
+
+ /* If we could potentially have long offsets, or we might want to use the prefetch decoder,
+ * compute information about the share of long offsets, and the maximum nbAdditionalBits.
+ * NOTE: could probably use a larger nbSeq limit
+ */
+ if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) {
+ ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq);
+ if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) {
+ /* If isLongOffset, but the maximum number of additional bits that we see in our table is small
+ * enough, then we know it is impossible to have too long an offset in this block, so we can
+ * use the regular offset decoder.
+ */
+ isLongOffset = ZSTD_lo_isRegularOffset;
+ }
+ if (!usePrefetchDecoder) {
+ U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
+ usePrefetchDecoder = (info.longOffsetShare >= minShare);
+ }
+ }
+
+ dctx->ddictIsCold = 0;
+
+#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
+ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
+ if (usePrefetchDecoder) {
+#else
+ (void)usePrefetchDecoder;
+ {
+#endif
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+ return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
+#endif
+ }
+
+#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+ /* else */
+ if (dctx->litBufferLocation == ZSTD_split)
+ return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
+ else
+ return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
+#endif
+ }
+}
+
+
+void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize)
+{
+ if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */
+ dctx->dictEnd = dctx->previousDstEnd;
+ dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
+ dctx->prefixStart = dst;
+ dctx->previousDstEnd = dst;
+ }
+}
+
+
+size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ size_t dSize;
+ ZSTD_checkContinuity(dctx, dst, dstCapacity);
+ dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0, not_streaming);
+ dctx->previousDstEnd = (char*)dst + dSize;
+ return dSize;
+}
diff --git a/contrib/zstd/zstd_decompress_block.h b/contrib/zstd/zstd_decompress_block.h
new file mode 100644
index 0000000..418985c
--- /dev/null
+++ b/contrib/zstd/zstd_decompress_block.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#ifndef ZSTD_DEC_BLOCK_H
+#define ZSTD_DEC_BLOCK_H
+
+/*-*******************************************************
+ * Dependencies
+ *********************************************************/
+#include "zstd_deps.h" /* size_t */
+#include "zstd.h" /* DCtx, and some public functions */
+#include "zstd_internal.h" /* blockProperties_t, and some public functions */
+#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
+
+
+/* === Prototypes === */
+
+/* note: prototypes already published within `zstd.h` :
+ * ZSTD_decompressBlock()
+ */
+
+/* note: prototypes already published within `zstd_internal.h` :
+ * ZSTD_getcBlockSize()
+ * ZSTD_decodeSeqHeaders()
+ */
+
+
+ /* Streaming state is used to inform allocation of the literal buffer */
+typedef enum {
+ not_streaming = 0,
+ is_streaming = 1
+} streaming_operation;
+
+/* ZSTD_decompressBlock_internal() :
+ * decompress block, starting at `src`,
+ * into destination buffer `dst`.
+ * @return : decompressed block size,
+ * or an error code (which can be tested using ZSTD_isError())
+ */
+size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize, const int frame, const streaming_operation streaming);
+
+/* ZSTD_buildFSETable() :
+ * generate FSE decoding table for one symbol (ll, ml or off)
+ * this function must be called with valid parameters only
+ * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.)
+ * in which case it cannot fail.
+ * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is
+ * defined in zstd_decompress_internal.h.
+ * Internal use only.
+ */
+void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
+ const short* normalizedCounter, unsigned maxSymbolValue,
+ const U32* baseValue, const U8* nbAdditionalBits,
+ unsigned tableLog, void* wksp, size_t wkspSize,
+ int bmi2);
+
+
+#endif /* ZSTD_DEC_BLOCK_H */
diff --git a/contrib/zstd/zstd_decompress_internal.h b/contrib/zstd/zstd_decompress_internal.h
new file mode 100644
index 0000000..a164de3
--- /dev/null
+++ b/contrib/zstd/zstd_decompress_internal.h
@@ -0,0 +1,238 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* zstd_decompress_internal:
+ * objects and definitions shared within lib/decompress modules */
+
+ #ifndef ZSTD_DECOMPRESS_INTERNAL_H
+ #define ZSTD_DECOMPRESS_INTERNAL_H
+
+
+/*-*******************************************************
+ * Dependencies
+ *********************************************************/
+#include "mem.h" /* BYTE, U16, U32 */
+#include "zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */
+
+
+
+/*-*******************************************************
+ * Constants
+ *********************************************************/
+static UNUSED_ATTR const U32 LL_base[MaxLL+1] = {
+ 0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 18, 20, 22, 24, 28, 32, 40,
+ 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
+ 0x2000, 0x4000, 0x8000, 0x10000 };
+
+static UNUSED_ATTR const U32 OF_base[MaxOff+1] = {
+ 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
+ 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
+ 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
+ 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };
+
+static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = {
+ 0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31 };
+
+static UNUSED_ATTR const U32 ML_base[MaxML+1] = {
+ 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18,
+ 19, 20, 21, 22, 23, 24, 25, 26,
+ 27, 28, 29, 30, 31, 32, 33, 34,
+ 35, 37, 39, 41, 43, 47, 51, 59,
+ 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
+ 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
+
+
+/*-*******************************************************
+ * Decompression types
+ *********************************************************/
+ typedef struct {
+ U32 fastMode;
+ U32 tableLog;
+ } ZSTD_seqSymbol_header;
+
+ typedef struct {
+ U16 nextState;
+ BYTE nbAdditionalBits;
+ BYTE nbBits;
+ U32 baseValue;
+ } ZSTD_seqSymbol;
+
+ #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
+
+#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64))
+#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32))
+#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12
+
+typedef struct {
+ ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
+ ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
+ ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
+ HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */
+ U32 rep[ZSTD_REP_NUM];
+ U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
+} ZSTD_entropyDTables_t;
+
+typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
+ ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
+ ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
+ ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
+
+typedef enum { zdss_init=0, zdss_loadHeader,
+ zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
+
+typedef enum {
+ ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */
+ ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */
+ ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */
+} ZSTD_dictUses_e;
+
+/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */
+typedef struct {
+ const ZSTD_DDict** ddictPtrTable;
+ size_t ddictPtrTableSize;
+ size_t ddictPtrCount;
+} ZSTD_DDictHashSet;
+
+#ifndef ZSTD_DECODER_INTERNAL_BUFFER
+# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16)
+#endif
+
+#define ZSTD_LBMIN 64
+#define ZSTD_LBMAX (128 << 10)
+
+/* extra buffer, compensates when dst is not large enough to store litBuffer */
+#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX)
+
+typedef enum {
+ ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */
+ ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */
+ ZSTD_split = 2 /* Split between litExtraBuffer and dst */
+} ZSTD_litLocation_e;
+
+struct ZSTD_DCtx_s
+{
+ const ZSTD_seqSymbol* LLTptr;
+ const ZSTD_seqSymbol* MLTptr;
+ const ZSTD_seqSymbol* OFTptr;
+ const HUF_DTable* HUFptr;
+ ZSTD_entropyDTables_t entropy;
+ U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */
+ const void* previousDstEnd; /* detect continuity */
+ const void* prefixStart; /* start of current segment */
+ const void* virtualStart; /* virtual start of previous segment if it was just before current one */
+ const void* dictEnd; /* end of previous segment */
+ size_t expected;
+ ZSTD_frameHeader fParams;
+ U64 processedCSize;
+ U64 decodedSize;
+ blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
+ ZSTD_dStage stage;
+ U32 litEntropy;
+ U32 fseEntropy;
+ XXH64_state_t xxhState;
+ size_t headerSize;
+ ZSTD_format_e format;
+ ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */
+ U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */
+ const BYTE* litPtr;
+ ZSTD_customMem customMem;
+ size_t litSize;
+ size_t rleSize;
+ size_t staticSize;
+#if DYNAMIC_BMI2 != 0
+ int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
+#endif
+
+ /* dictionary */
+ ZSTD_DDict* ddictLocal;
+ const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
+ U32 dictID;
+ int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
+ ZSTD_dictUses_e dictUses;
+ ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */
+ ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
+ int disableHufAsm;
+
+ /* streaming */
+ ZSTD_dStreamStage streamStage;
+ char* inBuff;
+ size_t inBuffSize;
+ size_t inPos;
+ size_t maxWindowSize;
+ char* outBuff;
+ size_t outBuffSize;
+ size_t outStart;
+ size_t outEnd;
+ size_t lhSize;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+ void* legacyContext;
+ U32 previousLegacyVersion;
+ U32 legacyVersion;
+#endif
+ U32 hostageByte;
+ int noForwardProgress;
+ ZSTD_bufferMode_e outBufferMode;
+ ZSTD_outBuffer expectedOutBuffer;
+
+ /* workspace */
+ BYTE* litBuffer;
+ const BYTE* litBufferEnd;
+ ZSTD_litLocation_e litBufferLocation;
+ BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */
+ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+
+ size_t oversizedDuration;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ void const* dictContentBeginForFuzzing;
+ void const* dictContentEndForFuzzing;
+#endif
+
+ /* Tracing */
+#if ZSTD_TRACE
+ ZSTD_TraceCtx traceCtx;
+#endif
+}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
+
+MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) {
+#if DYNAMIC_BMI2 != 0
+ return dctx->bmi2;
+#else
+ (void)dctx;
+ return 0;
+#endif
+}
+
+/*-*******************************************************
+ * Shared internal functions
+ *********************************************************/
+
+/*! ZSTD_loadDEntropy() :
+ * dict : must point at beginning of a valid zstd dictionary.
+ * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */
+size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
+ const void* const dict, size_t const dictSize);
+
+/*! ZSTD_checkContinuity() :
+ * check if next `dst` follows previous position, where decompression ended.
+ * If yes, do nothing (continue on current segment).
+ * If not, classify previous segment as "external dictionary", and start a new segment.
+ * This function cannot fail. */
+void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize);
+
+
+#endif /* ZSTD_DECOMPRESS_INTERNAL_H */
diff --git a/contrib/zstd/zstd_deps.h b/contrib/zstd/zstd_deps.h
new file mode 100644
index 0000000..4d767ae
--- /dev/null
+++ b/contrib/zstd/zstd_deps.h
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* This file provides common libc dependencies that zstd requires.
+ * The purpose is to allow replacing this file with a custom implementation
+ * to compile zstd without libc support.
+ */
+
+/* Need:
+ * NULL
+ * INT_MAX
+ * UINT_MAX
+ * ZSTD_memcpy()
+ * ZSTD_memset()
+ * ZSTD_memmove()
+ */
+#ifndef ZSTD_DEPS_COMMON
+#define ZSTD_DEPS_COMMON
+
+#include <limits.h>
+#include <stddef.h>
+#include <string.h>
+
+#if defined(__GNUC__) && __GNUC__ >= 4
+# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l))
+# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l))
+# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l))
+#else
+# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l))
+# define ZSTD_memmove(d,s,l) memmove((d),(s),(l))
+# define ZSTD_memset(p,v,l) memset((p),(v),(l))
+#endif
+
+#endif /* ZSTD_DEPS_COMMON */
+
+/* Need:
+ * ZSTD_malloc()
+ * ZSTD_free()
+ * ZSTD_calloc()
+ */
+#ifdef ZSTD_DEPS_NEED_MALLOC
+#ifndef ZSTD_DEPS_MALLOC
+#define ZSTD_DEPS_MALLOC
+
+#include <stdlib.h>
+
+#define ZSTD_malloc(s) malloc(s)
+#define ZSTD_calloc(n,s) calloc((n), (s))
+#define ZSTD_free(p) free((p))
+
+#endif /* ZSTD_DEPS_MALLOC */
+#endif /* ZSTD_DEPS_NEED_MALLOC */
+
+/*
+ * Provides 64-bit math support.
+ * Need:
+ * U64 ZSTD_div64(U64 dividend, U32 divisor)
+ */
+#ifdef ZSTD_DEPS_NEED_MATH64
+#ifndef ZSTD_DEPS_MATH64
+#define ZSTD_DEPS_MATH64
+
+#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor))
+
+#endif /* ZSTD_DEPS_MATH64 */
+#endif /* ZSTD_DEPS_NEED_MATH64 */
+
+/* Need:
+ * assert()
+ */
+#ifdef ZSTD_DEPS_NEED_ASSERT
+#ifndef ZSTD_DEPS_ASSERT
+#define ZSTD_DEPS_ASSERT
+
+#include <assert.h>
+
+#endif /* ZSTD_DEPS_ASSERT */
+#endif /* ZSTD_DEPS_NEED_ASSERT */
+
+/* Need:
+ * ZSTD_DEBUG_PRINT()
+ */
+#ifdef ZSTD_DEPS_NEED_IO
+#ifndef ZSTD_DEPS_IO
+#define ZSTD_DEPS_IO
+
+#include <stdio.h>
+#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)
+
+#endif /* ZSTD_DEPS_IO */
+#endif /* ZSTD_DEPS_NEED_IO */
+
+/* Only requested when <stdint.h> is known to be present.
+ * Need:
+ * intptr_t
+ */
+#ifdef ZSTD_DEPS_NEED_STDINT
+#ifndef ZSTD_DEPS_STDINT
+#define ZSTD_DEPS_STDINT
+
+#include <stdint.h>
+
+#endif /* ZSTD_DEPS_STDINT */
+#endif /* ZSTD_DEPS_NEED_STDINT */
diff --git a/contrib/zstd/zstd_double_fast.c b/contrib/zstd/zstd_double_fast.c
new file mode 100644
index 0000000..0ad88ff
--- /dev/null
+++ b/contrib/zstd/zstd_double_fast.c
@@ -0,0 +1,758 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h"
+#include "zstd_double_fast.h"
+
+static void ZSTD_fillDoubleHashTableForCDict(ZSTD_matchState_t* ms,
+ void const* end, ZSTD_dictTableLoadMethod_e dtlm)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashLarge = ms->hashTable;
+ U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
+ U32 const mls = cParams->minMatch;
+ U32* const hashSmall = ms->chainTable;
+ U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
+ const BYTE* const base = ms->window.base;
+ const BYTE* ip = base + ms->nextToUpdate;
+ const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
+ const U32 fastHashFillStep = 3;
+
+ /* Always insert every fastHashFillStep position into the hash tables.
+ * Insert the other positions into the large hash table if their entry
+ * is empty.
+ */
+ for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
+ U32 const curr = (U32)(ip - base);
+ U32 i;
+ for (i = 0; i < fastHashFillStep; ++i) {
+ size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls);
+ size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8);
+ if (i == 0) {
+ ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i);
+ }
+ if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) {
+ ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i);
+ }
+ /* Only load extra positions for ZSTD_dtlm_full */
+ if (dtlm == ZSTD_dtlm_fast)
+ break;
+ } }
+}
+
+static void ZSTD_fillDoubleHashTableForCCtx(ZSTD_matchState_t* ms,
+ void const* end, ZSTD_dictTableLoadMethod_e dtlm)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashLarge = ms->hashTable;
+ U32 const hBitsL = cParams->hashLog;
+ U32 const mls = cParams->minMatch;
+ U32* const hashSmall = ms->chainTable;
+ U32 const hBitsS = cParams->chainLog;
+ const BYTE* const base = ms->window.base;
+ const BYTE* ip = base + ms->nextToUpdate;
+ const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
+ const U32 fastHashFillStep = 3;
+
+ /* Always insert every fastHashFillStep position into the hash tables.
+ * Insert the other positions into the large hash table if their entry
+ * is empty.
+ */
+ for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
+ U32 const curr = (U32)(ip - base);
+ U32 i;
+ for (i = 0; i < fastHashFillStep; ++i) {
+ size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls);
+ size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8);
+ if (i == 0)
+ hashSmall[smHash] = curr + i;
+ if (i == 0 || hashLarge[lgHash] == 0)
+ hashLarge[lgHash] = curr + i;
+ /* Only load extra positions for ZSTD_dtlm_full */
+ if (dtlm == ZSTD_dtlm_fast)
+ break;
+ } }
+}
+
+void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
+ const void* const end,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp)
+{
+ if (tfp == ZSTD_tfp_forCDict) {
+ ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm);
+ } else {
+ ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm);
+ }
+}
+
+
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_compressBlock_doubleFast_noDict_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize, U32 const mls /* template */)
+{
+ ZSTD_compressionParameters const* cParams = &ms->cParams;
+ U32* const hashLong = ms->hashTable;
+ const U32 hBitsL = cParams->hashLog;
+ U32* const hashSmall = ms->chainTable;
+ const U32 hBitsS = cParams->chainLog;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* anchor = istart;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ /* presumes that, if there is a dictionary, it must be using Attach mode */
+ const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
+ const BYTE* const prefixLowest = base + prefixLowestIndex;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - HASH_READ_SIZE;
+ U32 offset_1=rep[0], offset_2=rep[1];
+ U32 offsetSaved1 = 0, offsetSaved2 = 0;
+
+ size_t mLength;
+ U32 offset;
+ U32 curr;
+
+ /* how many positions to search before increasing step size */
+ const size_t kStepIncr = 1 << kSearchStrength;
+ /* the position at which to increment the step size if no match is found */
+ const BYTE* nextStep;
+ size_t step; /* the current step size */
+
+ size_t hl0; /* the long hash at ip */
+ size_t hl1; /* the long hash at ip1 */
+
+ U32 idxl0; /* the long match index for ip */
+ U32 idxl1; /* the long match index for ip1 */
+
+ const BYTE* matchl0; /* the long match for ip */
+ const BYTE* matchs0; /* the short match for ip */
+ const BYTE* matchl1; /* the long match for ip1 */
+
+ const BYTE* ip = istart; /* the current position */
+ const BYTE* ip1; /* the next position */
+
+ DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic");
+
+ /* init */
+ ip += ((ip - prefixLowest) == 0);
+ {
+ U32 const current = (U32)(ip - base);
+ U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog);
+ U32 const maxRep = current - windowLow;
+ if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0;
+ if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0;
+ }
+
+ /* Outer Loop: one iteration per match found and stored */
+ while (1) {
+ step = 1;
+ nextStep = ip + kStepIncr;
+ ip1 = ip + step;
+
+ if (ip1 > ilimit) {
+ goto _cleanup;
+ }
+
+ hl0 = ZSTD_hashPtr(ip, hBitsL, 8);
+ idxl0 = hashLong[hl0];
+ matchl0 = base + idxl0;
+
+ /* Inner Loop: one iteration per search / position */
+ do {
+ const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls);
+ const U32 idxs0 = hashSmall[hs0];
+ curr = (U32)(ip-base);
+ matchs0 = base + idxs0;
+
+ hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */
+
+ /* check noDict repcode */
+ if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
+ mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
+ ip++;
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
+ goto _match_stored;
+ }
+
+ hl1 = ZSTD_hashPtr(ip1, hBitsL, 8);
+
+ if (idxl0 > prefixLowestIndex) {
+ /* check prefix long match */
+ if (MEM_read64(matchl0) == MEM_read64(ip)) {
+ mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8;
+ offset = (U32)(ip-matchl0);
+ while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */
+ goto _match_found;
+ }
+ }
+
+ idxl1 = hashLong[hl1];
+ matchl1 = base + idxl1;
+
+ if (idxs0 > prefixLowestIndex) {
+ /* check prefix short match */
+ if (MEM_read32(matchs0) == MEM_read32(ip)) {
+ goto _search_next_long;
+ }
+ }
+
+ if (ip1 >= nextStep) {
+ PREFETCH_L1(ip1 + 64);
+ PREFETCH_L1(ip1 + 128);
+ step++;
+ nextStep += kStepIncr;
+ }
+ ip = ip1;
+ ip1 += step;
+
+ hl0 = hl1;
+ idxl0 = idxl1;
+ matchl0 = matchl1;
+ #if defined(__aarch64__)
+ PREFETCH_L1(ip+256);
+ #endif
+ } while (ip1 <= ilimit);
+
+_cleanup:
+ /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
+ * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
+ offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
+
+ /* save reps for next block */
+ rep[0] = offset_1 ? offset_1 : offsetSaved1;
+ rep[1] = offset_2 ? offset_2 : offsetSaved2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+
+_search_next_long:
+
+ /* check prefix long +1 match */
+ if (idxl1 > prefixLowestIndex) {
+ if (MEM_read64(matchl1) == MEM_read64(ip1)) {
+ ip = ip1;
+ mLength = ZSTD_count(ip+8, matchl1+8, iend) + 8;
+ offset = (U32)(ip-matchl1);
+ while (((ip>anchor) & (matchl1>prefixLowest)) && (ip[-1] == matchl1[-1])) { ip--; matchl1--; mLength++; } /* catch up */
+ goto _match_found;
+ }
+ }
+
+ /* if no long +1 match, explore the short match we found */
+ mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
+ offset = (U32)(ip - matchs0);
+ while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* catch up */
+
+ /* fall-through */
+
+_match_found: /* requires ip, offset, mLength */
+ offset_2 = offset_1;
+ offset_1 = offset;
+
+ if (step < 4) {
+ /* It is unsafe to write this value back to the hashtable when ip1 is
+ * greater than or equal to the new ip we will have after we're done
+ * processing this match. Rather than perform that test directly
+ * (ip1 >= ip + mLength), which costs speed in practice, we do a simpler
+ * more predictable test. The minmatch even if we take a short match is
+ * 4 bytes, so as long as step, the distance between ip and ip1
+ * (initially) is less than 4, we know ip1 < new ip. */
+ hashLong[hl1] = (U32)(ip1 - base);
+ }
+
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+
+_match_stored:
+ /* match found */
+ ip += mLength;
+ anchor = ip;
+
+ if (ip <= ilimit) {
+ /* Complementary insertion */
+ /* done after iLimit test, as candidates could be > iend-8 */
+ { U32 const indexToInsert = curr+2;
+ hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
+ hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
+ hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
+ hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
+ }
+
+ /* check immediate repcode */
+ while ( (ip <= ilimit)
+ && ( (offset_2>0)
+ & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
+ /* store sequence */
+ size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
+ U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength);
+ ip += rLength;
+ anchor = ip;
+ continue; /* faster when present ... (?) */
+ }
+ }
+ }
+}
+
+
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize,
+ U32 const mls /* template */)
+{
+ ZSTD_compressionParameters const* cParams = &ms->cParams;
+ U32* const hashLong = ms->hashTable;
+ const U32 hBitsL = cParams->hashLog;
+ U32* const hashSmall = ms->chainTable;
+ const U32 hBitsS = cParams->chainLog;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip = istart;
+ const BYTE* anchor = istart;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ /* presumes that, if there is a dictionary, it must be using Attach mode */
+ const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
+ const BYTE* const prefixLowest = base + prefixLowestIndex;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - HASH_READ_SIZE;
+ U32 offset_1=rep[0], offset_2=rep[1];
+
+ const ZSTD_matchState_t* const dms = ms->dictMatchState;
+ const ZSTD_compressionParameters* const dictCParams = &dms->cParams;
+ const U32* const dictHashLong = dms->hashTable;
+ const U32* const dictHashSmall = dms->chainTable;
+ const U32 dictStartIndex = dms->window.dictLimit;
+ const BYTE* const dictBase = dms->window.base;
+ const BYTE* const dictStart = dictBase + dictStartIndex;
+ const BYTE* const dictEnd = dms->window.nextSrc;
+ const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase);
+ const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
+ const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
+ const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart));
+
+ DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic");
+
+ /* if a dictionary is attached, it must be within window range */
+ assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex);
+
+ if (ms->prefetchCDictTables) {
+ size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32);
+ size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32);
+ PREFETCH_AREA(dictHashLong, hashTableBytes)
+ PREFETCH_AREA(dictHashSmall, chainTableBytes)
+ }
+
+ /* init */
+ ip += (dictAndPrefixLength == 0);
+
+ /* dictMatchState repCode checks don't currently handle repCode == 0
+ * disabling. */
+ assert(offset_1 <= dictAndPrefixLength);
+ assert(offset_2 <= dictAndPrefixLength);
+
+ /* Main Search Loop */
+ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
+ size_t mLength;
+ U32 offset;
+ size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
+ size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
+ size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8);
+ size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls);
+ U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS];
+ U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS];
+ int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL);
+ int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS);
+ U32 const curr = (U32)(ip-base);
+ U32 const matchIndexL = hashLong[h2];
+ U32 matchIndexS = hashSmall[h];
+ const BYTE* matchLong = base + matchIndexL;
+ const BYTE* match = base + matchIndexS;
+ const U32 repIndex = curr + 1 - offset_1;
+ const BYTE* repMatch = (repIndex < prefixLowestIndex) ?
+ dictBase + (repIndex - dictIndexDelta) :
+ base + repIndex;
+ hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
+
+ /* check repcode */
+ if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
+ const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
+ mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
+ ip++;
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
+ goto _match_stored;
+ }
+
+ if (matchIndexL > prefixLowestIndex) {
+ /* check prefix long match */
+ if (MEM_read64(matchLong) == MEM_read64(ip)) {
+ mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
+ offset = (U32)(ip-matchLong);
+ while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
+ goto _match_found;
+ }
+ } else if (dictTagsMatchL) {
+ /* check dictMatchState long match */
+ U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS;
+ const BYTE* dictMatchL = dictBase + dictMatchIndexL;
+ assert(dictMatchL < dictEnd);
+
+ if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) {
+ mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8;
+ offset = (U32)(curr - dictMatchIndexL - dictIndexDelta);
+ while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */
+ goto _match_found;
+ } }
+
+ if (matchIndexS > prefixLowestIndex) {
+ /* check prefix short match */
+ if (MEM_read32(match) == MEM_read32(ip)) {
+ goto _search_next_long;
+ }
+ } else if (dictTagsMatchS) {
+ /* check dictMatchState short match */
+ U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS;
+ match = dictBase + dictMatchIndexS;
+ matchIndexS = dictMatchIndexS + dictIndexDelta;
+
+ if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) {
+ goto _search_next_long;
+ } }
+
+ ip += ((ip-anchor) >> kSearchStrength) + 1;
+#if defined(__aarch64__)
+ PREFETCH_L1(ip+256);
+#endif
+ continue;
+
+_search_next_long:
+ { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
+ size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8);
+ U32 const matchIndexL3 = hashLong[hl3];
+ U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS];
+ int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3);
+ const BYTE* matchL3 = base + matchIndexL3;
+ hashLong[hl3] = curr + 1;
+
+ /* check prefix long +1 match */
+ if (matchIndexL3 > prefixLowestIndex) {
+ if (MEM_read64(matchL3) == MEM_read64(ip+1)) {
+ mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
+ ip++;
+ offset = (U32)(ip-matchL3);
+ while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
+ goto _match_found;
+ }
+ } else if (dictTagsMatchL3) {
+ /* check dict long +1 match */
+ U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS;
+ const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3;
+ assert(dictMatchL3 < dictEnd);
+ if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) {
+ mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8;
+ ip++;
+ offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta);
+ while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */
+ goto _match_found;
+ } } }
+
+ /* if no long +1 match, explore the short match we found */
+ if (matchIndexS < prefixLowestIndex) {
+ mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4;
+ offset = (U32)(curr - matchIndexS);
+ while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
+ } else {
+ mLength = ZSTD_count(ip+4, match+4, iend) + 4;
+ offset = (U32)(ip - match);
+ while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
+ }
+
+_match_found:
+ offset_2 = offset_1;
+ offset_1 = offset;
+
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+
+_match_stored:
+ /* match found */
+ ip += mLength;
+ anchor = ip;
+
+ if (ip <= ilimit) {
+ /* Complementary insertion */
+ /* done after iLimit test, as candidates could be > iend-8 */
+ { U32 const indexToInsert = curr+2;
+ hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
+ hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
+ hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
+ hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
+ }
+
+ /* check immediate repcode */
+ while (ip <= ilimit) {
+ U32 const current2 = (U32)(ip-base);
+ U32 const repIndex2 = current2 - offset_2;
+ const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ?
+ dictBase + repIndex2 - dictIndexDelta :
+ base + repIndex2;
+ if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
+ && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
+ const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
+ size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
+ U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
+ ip += repLength2;
+ anchor = ip;
+ continue;
+ }
+ break;
+ }
+ }
+ } /* while (ip < ilimit) */
+
+ /* save reps for next block */
+ rep[0] = offset_1;
+ rep[1] = offset_2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+#define ZSTD_GEN_DFAST_FN(dictMode, mls) \
+ static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
+ void const* src, size_t srcSize) \
+ { \
+ return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \
+ }
+
+ZSTD_GEN_DFAST_FN(noDict, 4)
+ZSTD_GEN_DFAST_FN(noDict, 5)
+ZSTD_GEN_DFAST_FN(noDict, 6)
+ZSTD_GEN_DFAST_FN(noDict, 7)
+
+ZSTD_GEN_DFAST_FN(dictMatchState, 4)
+ZSTD_GEN_DFAST_FN(dictMatchState, 5)
+ZSTD_GEN_DFAST_FN(dictMatchState, 6)
+ZSTD_GEN_DFAST_FN(dictMatchState, 7)
+
+
+size_t ZSTD_compressBlock_doubleFast(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ const U32 mls = ms->cParams.minMatch;
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize);
+ }
+}
+
+
+size_t ZSTD_compressBlock_doubleFast_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ const U32 mls = ms->cParams.minMatch;
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize);
+ }
+}
+
+
+static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize,
+ U32 const mls /* template */)
+{
+ ZSTD_compressionParameters const* cParams = &ms->cParams;
+ U32* const hashLong = ms->hashTable;
+ U32 const hBitsL = cParams->hashLog;
+ U32* const hashSmall = ms->chainTable;
+ U32 const hBitsS = cParams->chainLog;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip = istart;
+ const BYTE* anchor = istart;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - 8;
+ const BYTE* const base = ms->window.base;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog);
+ const U32 dictStartIndex = lowLimit;
+ const U32 dictLimit = ms->window.dictLimit;
+ const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit;
+ const BYTE* const prefixStart = base + prefixStartIndex;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const BYTE* const dictStart = dictBase + dictStartIndex;
+ const BYTE* const dictEnd = dictBase + prefixStartIndex;
+ U32 offset_1=rep[0], offset_2=rep[1];
+
+ DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize);
+
+ /* if extDict is invalidated due to maxDistance, switch to "regular" variant */
+ if (prefixStartIndex == dictStartIndex)
+ return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize);
+
+ /* Search Loop */
+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
+ const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls);
+ const U32 matchIndex = hashSmall[hSmall];
+ const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base;
+ const BYTE* match = matchBase + matchIndex;
+
+ const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8);
+ const U32 matchLongIndex = hashLong[hLong];
+ const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base;
+ const BYTE* matchLong = matchLongBase + matchLongIndex;
+
+ const U32 curr = (U32)(ip-base);
+ const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
+ const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
+ const BYTE* const repMatch = repBase + repIndex;
+ size_t mLength;
+ hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
+
+ if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */
+ & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */
+ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
+ const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
+ mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
+ ip++;
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
+ } else {
+ if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
+ const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend;
+ const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart;
+ U32 offset;
+ mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8;
+ offset = curr - matchLongIndex;
+ while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
+ offset_2 = offset_1;
+ offset_1 = offset;
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+
+ } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) {
+ size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
+ U32 const matchIndex3 = hashLong[h3];
+ const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base;
+ const BYTE* match3 = match3Base + matchIndex3;
+ U32 offset;
+ hashLong[h3] = curr + 1;
+ if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) {
+ const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend;
+ const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart;
+ mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8;
+ ip++;
+ offset = curr+1 - matchIndex3;
+ while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */
+ } else {
+ const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend;
+ const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart;
+ mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
+ offset = curr - matchIndex;
+ while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
+ }
+ offset_2 = offset_1;
+ offset_1 = offset;
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+
+ } else {
+ ip += ((ip-anchor) >> kSearchStrength) + 1;
+ continue;
+ } }
+
+ /* move to next sequence start */
+ ip += mLength;
+ anchor = ip;
+
+ if (ip <= ilimit) {
+ /* Complementary insertion */
+ /* done after iLimit test, as candidates could be > iend-8 */
+ { U32 const indexToInsert = curr+2;
+ hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
+ hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
+ hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
+ hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
+ }
+
+ /* check immediate repcode */
+ while (ip <= ilimit) {
+ U32 const current2 = (U32)(ip-base);
+ U32 const repIndex2 = current2 - offset_2;
+ const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
+ if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */
+ & (offset_2 <= current2 - dictStartIndex))
+ && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
+ const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
+ size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
+ U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
+ ip += repLength2;
+ anchor = ip;
+ continue;
+ }
+ break;
+ } } }
+
+ /* save reps for next block */
+ rep[0] = offset_1;
+ rep[1] = offset_2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+ZSTD_GEN_DFAST_FN(extDict, 4)
+ZSTD_GEN_DFAST_FN(extDict, 5)
+ZSTD_GEN_DFAST_FN(extDict, 6)
+ZSTD_GEN_DFAST_FN(extDict, 7)
+
+size_t ZSTD_compressBlock_doubleFast_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ U32 const mls = ms->cParams.minMatch;
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize);
+ }
+}
diff --git a/contrib/zstd/zstd_double_fast.h b/contrib/zstd/zstd_double_fast.h
new file mode 100644
index 0000000..e8ce20c
--- /dev/null
+++ b/contrib/zstd/zstd_double_fast.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_DOUBLE_FAST_H
+#define ZSTD_DOUBLE_FAST_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include "mem.h" /* U32 */
+#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */
+
+void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
+ void const* end, ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp);
+size_t ZSTD_compressBlock_doubleFast(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_doubleFast_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_doubleFast_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_DOUBLE_FAST_H */
diff --git a/contrib/zstd/zstd_errors.h b/contrib/zstd/zstd_errors.h
new file mode 100644
index 0000000..dc75eee
--- /dev/null
+++ b/contrib/zstd/zstd_errors.h
@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_ERRORS_H_398273423
+#define ZSTD_ERRORS_H_398273423
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*===== dependency =====*/
+#include <stddef.h> /* size_t */
+
+
+/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
+#ifndef ZSTDERRORLIB_VISIBLE
+ /* Backwards compatibility with old macro name */
+# ifdef ZSTDERRORLIB_VISIBILITY
+# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY
+# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default")))
+# else
+# define ZSTDERRORLIB_VISIBLE
+# endif
+#endif
+
+#ifndef ZSTDERRORLIB_HIDDEN
+# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+# else
+# define ZSTDERRORLIB_HIDDEN
+# endif
+#endif
+
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
+#endif
+
+/*-*********************************************
+ * Error codes list
+ *-*********************************************
+ * Error codes _values_ are pinned down since v1.3.1 only.
+ * Therefore, don't rely on values if you may link to any version < v1.3.1.
+ *
+ * Only values < 100 are considered stable.
+ *
+ * note 1 : this API shall be used with static linking only.
+ * dynamic linking is not yet officially supported.
+ * note 2 : Prefer relying on the enum than on its value whenever possible
+ * This is the only supported way to use the error list < v1.3.1
+ * note 3 : ZSTD_isError() is always correct, whatever the library version.
+ **********************************************/
+typedef enum {
+ ZSTD_error_no_error = 0,
+ ZSTD_error_GENERIC = 1,
+ ZSTD_error_prefix_unknown = 10,
+ ZSTD_error_version_unsupported = 12,
+ ZSTD_error_frameParameter_unsupported = 14,
+ ZSTD_error_frameParameter_windowTooLarge = 16,
+ ZSTD_error_corruption_detected = 20,
+ ZSTD_error_checksum_wrong = 22,
+ ZSTD_error_literals_headerWrong = 24,
+ ZSTD_error_dictionary_corrupted = 30,
+ ZSTD_error_dictionary_wrong = 32,
+ ZSTD_error_dictionaryCreation_failed = 34,
+ ZSTD_error_parameter_unsupported = 40,
+ ZSTD_error_parameter_combination_unsupported = 41,
+ ZSTD_error_parameter_outOfBound = 42,
+ ZSTD_error_tableLog_tooLarge = 44,
+ ZSTD_error_maxSymbolValue_tooLarge = 46,
+ ZSTD_error_maxSymbolValue_tooSmall = 48,
+ ZSTD_error_stabilityCondition_notRespected = 50,
+ ZSTD_error_stage_wrong = 60,
+ ZSTD_error_init_missing = 62,
+ ZSTD_error_memory_allocation = 64,
+ ZSTD_error_workSpace_tooSmall= 66,
+ ZSTD_error_dstSize_tooSmall = 70,
+ ZSTD_error_srcSize_wrong = 72,
+ ZSTD_error_dstBuffer_null = 74,
+ ZSTD_error_noForwardProgress_destFull = 80,
+ ZSTD_error_noForwardProgress_inputEmpty = 82,
+ /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
+ ZSTD_error_frameIndex_tooLarge = 100,
+ ZSTD_error_seekableIO = 102,
+ ZSTD_error_dstBuffer_wrong = 104,
+ ZSTD_error_srcBuffer_wrong = 105,
+ ZSTD_error_sequenceProducer_failed = 106,
+ ZSTD_error_externalSequences_invalid = 107,
+ ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
+} ZSTD_ErrorCode;
+
+/*! ZSTD_getErrorCode() :
+ convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
+ which can be used to compare with enum list published above */
+ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_ERRORS_H_398273423 */
diff --git a/contrib/zstd/zstd_fast.c b/contrib/zstd/zstd_fast.c
new file mode 100644
index 0000000..5f2c6a2
--- /dev/null
+++ b/contrib/zstd/zstd_fast.c
@@ -0,0 +1,960 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */
+#include "zstd_fast.h"
+
+static void ZSTD_fillHashTableForCDict(ZSTD_matchState_t* ms,
+ const void* const end,
+ ZSTD_dictTableLoadMethod_e dtlm)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hBits = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
+ U32 const mls = cParams->minMatch;
+ const BYTE* const base = ms->window.base;
+ const BYTE* ip = base + ms->nextToUpdate;
+ const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
+ const U32 fastHashFillStep = 3;
+
+ /* Currently, we always use ZSTD_dtlm_full for filling CDict tables.
+ * Feel free to remove this assert if there's a good reason! */
+ assert(dtlm == ZSTD_dtlm_full);
+
+ /* Always insert every fastHashFillStep position into the hash table.
+ * Insert the other positions if their hash entry is empty.
+ */
+ for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) {
+ U32 const curr = (U32)(ip - base);
+ { size_t const hashAndTag = ZSTD_hashPtr(ip, hBits, mls);
+ ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr); }
+
+ if (dtlm == ZSTD_dtlm_fast) continue;
+ /* Only load extra positions for ZSTD_dtlm_full */
+ { U32 p;
+ for (p = 1; p < fastHashFillStep; ++p) {
+ size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls);
+ if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */
+ ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p);
+ } } } }
+}
+
+static void ZSTD_fillHashTableForCCtx(ZSTD_matchState_t* ms,
+ const void* const end,
+ ZSTD_dictTableLoadMethod_e dtlm)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hBits = cParams->hashLog;
+ U32 const mls = cParams->minMatch;
+ const BYTE* const base = ms->window.base;
+ const BYTE* ip = base + ms->nextToUpdate;
+ const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
+ const U32 fastHashFillStep = 3;
+
+ /* Currently, we always use ZSTD_dtlm_fast for filling CCtx tables.
+ * Feel free to remove this assert if there's a good reason! */
+ assert(dtlm == ZSTD_dtlm_fast);
+
+ /* Always insert every fastHashFillStep position into the hash table.
+ * Insert the other positions if their hash entry is empty.
+ */
+ for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) {
+ U32 const curr = (U32)(ip - base);
+ size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls);
+ hashTable[hash0] = curr;
+ if (dtlm == ZSTD_dtlm_fast) continue;
+ /* Only load extra positions for ZSTD_dtlm_full */
+ { U32 p;
+ for (p = 1; p < fastHashFillStep; ++p) {
+ size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls);
+ if (hashTable[hash] == 0) { /* not yet filled */
+ hashTable[hash] = curr + p;
+ } } } }
+}
+
+void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
+ const void* const end,
+ ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp)
+{
+ if (tfp == ZSTD_tfp_forCDict) {
+ ZSTD_fillHashTableForCDict(ms, end, dtlm);
+ } else {
+ ZSTD_fillHashTableForCCtx(ms, end, dtlm);
+ }
+}
+
+
+/**
+ * If you squint hard enough (and ignore repcodes), the search operation at any
+ * given position is broken into 4 stages:
+ *
+ * 1. Hash (map position to hash value via input read)
+ * 2. Lookup (map hash val to index via hashtable read)
+ * 3. Load (map index to value at that position via input read)
+ * 4. Compare
+ *
+ * Each of these steps involves a memory read at an address which is computed
+ * from the previous step. This means these steps must be sequenced and their
+ * latencies are cumulative.
+ *
+ * Rather than do 1->2->3->4 sequentially for a single position before moving
+ * onto the next, this implementation interleaves these operations across the
+ * next few positions:
+ *
+ * R = Repcode Read & Compare
+ * H = Hash
+ * T = Table Lookup
+ * M = Match Read & Compare
+ *
+ * Pos | Time -->
+ * ----+-------------------
+ * N | ... M
+ * N+1 | ... TM
+ * N+2 | R H T M
+ * N+3 | H TM
+ * N+4 | R H T M
+ * N+5 | H ...
+ * N+6 | R ...
+ *
+ * This is very much analogous to the pipelining of execution in a CPU. And just
+ * like a CPU, we have to dump the pipeline when we find a match (i.e., take a
+ * branch).
+ *
+ * When this happens, we throw away our current state, and do the following prep
+ * to re-enter the loop:
+ *
+ * Pos | Time -->
+ * ----+-------------------
+ * N | H T
+ * N+1 | H
+ *
+ * This is also the work we do at the beginning to enter the loop initially.
+ */
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_compressBlock_fast_noDict_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize,
+ U32 const mls, U32 const hasStep)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hlog = cParams->hashLog;
+ /* support stepSize of 0 */
+ size_t const stepSize = hasStep ? (cParams->targetLength + !(cParams->targetLength) + 1) : 2;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const istart = (const BYTE*)src;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
+ const BYTE* const prefixStart = base + prefixStartIndex;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - HASH_READ_SIZE;
+
+ const BYTE* anchor = istart;
+ const BYTE* ip0 = istart;
+ const BYTE* ip1;
+ const BYTE* ip2;
+ const BYTE* ip3;
+ U32 current0;
+
+ U32 rep_offset1 = rep[0];
+ U32 rep_offset2 = rep[1];
+ U32 offsetSaved1 = 0, offsetSaved2 = 0;
+
+ size_t hash0; /* hash for ip0 */
+ size_t hash1; /* hash for ip1 */
+ U32 idx; /* match idx for ip0 */
+ U32 mval; /* src value at match idx */
+
+ U32 offcode;
+ const BYTE* match0;
+ size_t mLength;
+
+ /* ip0 and ip1 are always adjacent. The targetLength skipping and
+ * uncompressibility acceleration is applied to every other position,
+ * matching the behavior of #1562. step therefore represents the gap
+ * between pairs of positions, from ip0 to ip2 or ip1 to ip3. */
+ size_t step;
+ const BYTE* nextStep;
+ const size_t kStepIncr = (1 << (kSearchStrength - 1));
+
+ DEBUGLOG(5, "ZSTD_compressBlock_fast_generic");
+ ip0 += (ip0 == prefixStart);
+ { U32 const curr = (U32)(ip0 - base);
+ U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog);
+ U32 const maxRep = curr - windowLow;
+ if (rep_offset2 > maxRep) offsetSaved2 = rep_offset2, rep_offset2 = 0;
+ if (rep_offset1 > maxRep) offsetSaved1 = rep_offset1, rep_offset1 = 0;
+ }
+
+ /* start each op */
+_start: /* Requires: ip0 */
+
+ step = stepSize;
+ nextStep = ip0 + kStepIncr;
+
+ /* calculate positions, ip0 - anchor == 0, so we skip step calc */
+ ip1 = ip0 + 1;
+ ip2 = ip0 + step;
+ ip3 = ip2 + 1;
+
+ if (ip3 >= ilimit) {
+ goto _cleanup;
+ }
+
+ hash0 = ZSTD_hashPtr(ip0, hlog, mls);
+ hash1 = ZSTD_hashPtr(ip1, hlog, mls);
+
+ idx = hashTable[hash0];
+
+ do {
+ /* load repcode match for ip[2]*/
+ const U32 rval = MEM_read32(ip2 - rep_offset1);
+
+ /* write back hash table entry */
+ current0 = (U32)(ip0 - base);
+ hashTable[hash0] = current0;
+
+ /* check repcode at ip[2] */
+ if ((MEM_read32(ip2) == rval) & (rep_offset1 > 0)) {
+ ip0 = ip2;
+ match0 = ip0 - rep_offset1;
+ mLength = ip0[-1] == match0[-1];
+ ip0 -= mLength;
+ match0 -= mLength;
+ offcode = REPCODE1_TO_OFFBASE;
+ mLength += 4;
+
+ /* First write next hash table entry; we've already calculated it.
+ * This write is known to be safe because the ip1 is before the
+ * repcode (ip2). */
+ hashTable[hash1] = (U32)(ip1 - base);
+
+ goto _match;
+ }
+
+ /* load match for ip[0] */
+ if (idx >= prefixStartIndex) {
+ mval = MEM_read32(base + idx);
+ } else {
+ mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */
+ }
+
+ /* check match at ip[0] */
+ if (MEM_read32(ip0) == mval) {
+ /* found a match! */
+
+ /* First write next hash table entry; we've already calculated it.
+ * This write is known to be safe because the ip1 == ip0 + 1, so
+ * we know we will resume searching after ip1 */
+ hashTable[hash1] = (U32)(ip1 - base);
+
+ goto _offset;
+ }
+
+ /* lookup ip[1] */
+ idx = hashTable[hash1];
+
+ /* hash ip[2] */
+ hash0 = hash1;
+ hash1 = ZSTD_hashPtr(ip2, hlog, mls);
+
+ /* advance to next positions */
+ ip0 = ip1;
+ ip1 = ip2;
+ ip2 = ip3;
+
+ /* write back hash table entry */
+ current0 = (U32)(ip0 - base);
+ hashTable[hash0] = current0;
+
+ /* load match for ip[0] */
+ if (idx >= prefixStartIndex) {
+ mval = MEM_read32(base + idx);
+ } else {
+ mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */
+ }
+
+ /* check match at ip[0] */
+ if (MEM_read32(ip0) == mval) {
+ /* found a match! */
+
+ /* first write next hash table entry; we've already calculated it */
+ if (step <= 4) {
+ /* We need to avoid writing an index into the hash table >= the
+ * position at which we will pick up our searching after we've
+ * taken this match.
+ *
+ * The minimum possible match has length 4, so the earliest ip0
+ * can be after we take this match will be the current ip0 + 4.
+ * ip1 is ip0 + step - 1. If ip1 is >= ip0 + 4, we can't safely
+ * write this position.
+ */
+ hashTable[hash1] = (U32)(ip1 - base);
+ }
+
+ goto _offset;
+ }
+
+ /* lookup ip[1] */
+ idx = hashTable[hash1];
+
+ /* hash ip[2] */
+ hash0 = hash1;
+ hash1 = ZSTD_hashPtr(ip2, hlog, mls);
+
+ /* advance to next positions */
+ ip0 = ip1;
+ ip1 = ip2;
+ ip2 = ip0 + step;
+ ip3 = ip1 + step;
+
+ /* calculate step */
+ if (ip2 >= nextStep) {
+ step++;
+ PREFETCH_L1(ip1 + 64);
+ PREFETCH_L1(ip1 + 128);
+ nextStep += kStepIncr;
+ }
+ } while (ip3 < ilimit);
+
+_cleanup:
+ /* Note that there are probably still a couple positions we could search.
+ * However, it seems to be a meaningful performance hit to try to search
+ * them. So let's not. */
+
+ /* When the repcodes are outside of the prefix, we set them to zero before the loop.
+ * When the offsets are still zero, we need to restore them after the block to have a correct
+ * repcode history. If only one offset was invalid, it is easy. The tricky case is when both
+ * offsets were invalid. We need to figure out which offset to refill with.
+ * - If both offsets are zero they are in the same order.
+ * - If both offsets are non-zero, we won't restore the offsets from `offsetSaved[12]`.
+ * - If only one is zero, we need to decide which offset to restore.
+ * - If rep_offset1 is non-zero, then rep_offset2 must be offsetSaved1.
+ * - It is impossible for rep_offset2 to be non-zero.
+ *
+ * So if rep_offset1 started invalid (offsetSaved1 != 0) and became valid (rep_offset1 != 0), then
+ * set rep[0] = rep_offset1 and rep[1] = offsetSaved1.
+ */
+ offsetSaved2 = ((offsetSaved1 != 0) && (rep_offset1 != 0)) ? offsetSaved1 : offsetSaved2;
+
+ /* save reps for next block */
+ rep[0] = rep_offset1 ? rep_offset1 : offsetSaved1;
+ rep[1] = rep_offset2 ? rep_offset2 : offsetSaved2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+
+_offset: /* Requires: ip0, idx */
+
+ /* Compute the offset code. */
+ match0 = base + idx;
+ rep_offset2 = rep_offset1;
+ rep_offset1 = (U32)(ip0-match0);
+ offcode = OFFSET_TO_OFFBASE(rep_offset1);
+ mLength = 4;
+
+ /* Count the backwards match length. */
+ while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) {
+ ip0--;
+ match0--;
+ mLength++;
+ }
+
+_match: /* Requires: ip0, match0, offcode */
+
+ /* Count the forward length. */
+ mLength += ZSTD_count(ip0 + mLength, match0 + mLength, iend);
+
+ ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength);
+
+ ip0 += mLength;
+ anchor = ip0;
+
+ /* Fill table and check for immediate repcode. */
+ if (ip0 <= ilimit) {
+ /* Fill Table */
+ assert(base+current0+2 > istart); /* check base overflow */
+ hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */
+ hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
+
+ if (rep_offset2 > 0) { /* rep_offset2==0 means rep_offset2 is invalidated */
+ while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - rep_offset2)) ) {
+ /* store sequence */
+ size_t const rLength = ZSTD_count(ip0+4, ip0+4-rep_offset2, iend) + 4;
+ { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */
+ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
+ ip0 += rLength;
+ ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, REPCODE1_TO_OFFBASE, rLength);
+ anchor = ip0;
+ continue; /* faster when present (confirmed on gcc-8) ... (?) */
+ } } }
+
+ goto _start;
+}
+
+#define ZSTD_GEN_FAST_FN(dictMode, mls, step) \
+ static size_t ZSTD_compressBlock_fast_##dictMode##_##mls##_##step( \
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
+ void const* src, size_t srcSize) \
+ { \
+ return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, step); \
+ }
+
+ZSTD_GEN_FAST_FN(noDict, 4, 1)
+ZSTD_GEN_FAST_FN(noDict, 5, 1)
+ZSTD_GEN_FAST_FN(noDict, 6, 1)
+ZSTD_GEN_FAST_FN(noDict, 7, 1)
+
+ZSTD_GEN_FAST_FN(noDict, 4, 0)
+ZSTD_GEN_FAST_FN(noDict, 5, 0)
+ZSTD_GEN_FAST_FN(noDict, 6, 0)
+ZSTD_GEN_FAST_FN(noDict, 7, 0)
+
+size_t ZSTD_compressBlock_fast(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ U32 const mls = ms->cParams.minMatch;
+ assert(ms->dictMatchState == NULL);
+ if (ms->cParams.targetLength > 1) {
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize);
+ }
+ } else {
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize);
+ }
+
+ }
+}
+
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_compressBlock_fast_dictMatchState_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hlog = cParams->hashLog;
+ /* support stepSize of 0 */
+ U32 const stepSize = cParams->targetLength + !(cParams->targetLength);
+ const BYTE* const base = ms->window.base;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip0 = istart;
+ const BYTE* ip1 = ip0 + stepSize; /* we assert below that stepSize >= 1 */
+ const BYTE* anchor = istart;
+ const U32 prefixStartIndex = ms->window.dictLimit;
+ const BYTE* const prefixStart = base + prefixStartIndex;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - HASH_READ_SIZE;
+ U32 offset_1=rep[0], offset_2=rep[1];
+
+ const ZSTD_matchState_t* const dms = ms->dictMatchState;
+ const ZSTD_compressionParameters* const dictCParams = &dms->cParams ;
+ const U32* const dictHashTable = dms->hashTable;
+ const U32 dictStartIndex = dms->window.dictLimit;
+ const BYTE* const dictBase = dms->window.base;
+ const BYTE* const dictStart = dictBase + dictStartIndex;
+ const BYTE* const dictEnd = dms->window.nextSrc;
+ const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase);
+ const U32 dictAndPrefixLength = (U32)(istart - prefixStart + dictEnd - dictStart);
+ const U32 dictHBits = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
+
+ /* if a dictionary is still attached, it necessarily means that
+ * it is within window size. So we just check it. */
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ assert(endIndex - prefixStartIndex <= maxDistance);
+ (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */
+
+ (void)hasStep; /* not currently specialized on whether it's accelerated */
+
+ /* ensure there will be no underflow
+ * when translating a dict index into a local index */
+ assert(prefixStartIndex >= (U32)(dictEnd - dictBase));
+
+ if (ms->prefetchCDictTables) {
+ size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32);
+ PREFETCH_AREA(dictHashTable, hashTableBytes)
+ }
+
+ /* init */
+ DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic");
+ ip0 += (dictAndPrefixLength == 0);
+ /* dictMatchState repCode checks don't currently handle repCode == 0
+ * disabling. */
+ assert(offset_1 <= dictAndPrefixLength);
+ assert(offset_2 <= dictAndPrefixLength);
+
+ /* Outer search loop */
+ assert(stepSize >= 1);
+ while (ip1 <= ilimit) { /* repcode check at (ip0 + 1) is safe because ip0 < ip1 */
+ size_t mLength;
+ size_t hash0 = ZSTD_hashPtr(ip0, hlog, mls);
+
+ size_t const dictHashAndTag0 = ZSTD_hashPtr(ip0, dictHBits, mls);
+ U32 dictMatchIndexAndTag = dictHashTable[dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS];
+ int dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag0);
+
+ U32 matchIndex = hashTable[hash0];
+ U32 curr = (U32)(ip0 - base);
+ size_t step = stepSize;
+ const size_t kStepIncr = 1 << kSearchStrength;
+ const BYTE* nextStep = ip0 + kStepIncr;
+
+ /* Inner search loop */
+ while (1) {
+ const BYTE* match = base + matchIndex;
+ const U32 repIndex = curr + 1 - offset_1;
+ const BYTE* repMatch = (repIndex < prefixStartIndex) ?
+ dictBase + (repIndex - dictIndexDelta) :
+ base + repIndex;
+ const size_t hash1 = ZSTD_hashPtr(ip1, hlog, mls);
+ size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls);
+ hashTable[hash0] = curr; /* update hash table */
+
+ if (((U32) ((prefixStartIndex - 1) - repIndex) >=
+ 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
+ && (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) {
+ const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
+ mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4;
+ ip0++;
+ ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
+ break;
+ }
+
+ if (dictTagsMatch) {
+ /* Found a possible dict match */
+ const U32 dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS;
+ const BYTE* dictMatch = dictBase + dictMatchIndex;
+ if (dictMatchIndex > dictStartIndex &&
+ MEM_read32(dictMatch) == MEM_read32(ip0)) {
+ /* To replicate extDict parse behavior, we only use dict matches when the normal matchIndex is invalid */
+ if (matchIndex <= prefixStartIndex) {
+ U32 const offset = (U32) (curr - dictMatchIndex - dictIndexDelta);
+ mLength = ZSTD_count_2segments(ip0 + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4;
+ while (((ip0 > anchor) & (dictMatch > dictStart))
+ && (ip0[-1] == dictMatch[-1])) {
+ ip0--;
+ dictMatch--;
+ mLength++;
+ } /* catch up */
+ offset_2 = offset_1;
+ offset_1 = offset;
+ ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+ break;
+ }
+ }
+ }
+
+ if (matchIndex > prefixStartIndex && MEM_read32(match) == MEM_read32(ip0)) {
+ /* found a regular match */
+ U32 const offset = (U32) (ip0 - match);
+ mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4;
+ while (((ip0 > anchor) & (match > prefixStart))
+ && (ip0[-1] == match[-1])) {
+ ip0--;
+ match--;
+ mLength++;
+ } /* catch up */
+ offset_2 = offset_1;
+ offset_1 = offset;
+ ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+ break;
+ }
+
+ /* Prepare for next iteration */
+ dictMatchIndexAndTag = dictHashTable[dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS];
+ dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag1);
+ matchIndex = hashTable[hash1];
+
+ if (ip1 >= nextStep) {
+ step++;
+ nextStep += kStepIncr;
+ }
+ ip0 = ip1;
+ ip1 = ip1 + step;
+ if (ip1 > ilimit) goto _cleanup;
+
+ curr = (U32)(ip0 - base);
+ hash0 = hash1;
+ } /* end inner search loop */
+
+ /* match found */
+ assert(mLength);
+ ip0 += mLength;
+ anchor = ip0;
+
+ if (ip0 <= ilimit) {
+ /* Fill Table */
+ assert(base+curr+2 > istart); /* check base overflow */
+ hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */
+ hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
+
+ /* check immediate repcode */
+ while (ip0 <= ilimit) {
+ U32 const current2 = (U32)(ip0-base);
+ U32 const repIndex2 = current2 - offset_2;
+ const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
+ dictBase - dictIndexDelta + repIndex2 :
+ base + repIndex2;
+ if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
+ && (MEM_read32(repMatch2) == MEM_read32(ip0))) {
+ const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
+ size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
+ U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
+ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = current2;
+ ip0 += repLength2;
+ anchor = ip0;
+ continue;
+ }
+ break;
+ }
+ }
+
+ /* Prepare for next iteration */
+ assert(ip0 == anchor);
+ ip1 = ip0 + stepSize;
+ }
+
+_cleanup:
+ /* save reps for next block */
+ rep[0] = offset_1;
+ rep[1] = offset_2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+
+ZSTD_GEN_FAST_FN(dictMatchState, 4, 0)
+ZSTD_GEN_FAST_FN(dictMatchState, 5, 0)
+ZSTD_GEN_FAST_FN(dictMatchState, 6, 0)
+ZSTD_GEN_FAST_FN(dictMatchState, 7, 0)
+
+size_t ZSTD_compressBlock_fast_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ U32 const mls = ms->cParams.minMatch;
+ assert(ms->dictMatchState != NULL);
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize);
+ }
+}
+
+
+static size_t ZSTD_compressBlock_fast_extDict_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hlog = cParams->hashLog;
+ /* support stepSize of 0 */
+ size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* anchor = istart;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog);
+ const U32 dictStartIndex = lowLimit;
+ const BYTE* const dictStart = dictBase + dictStartIndex;
+ const U32 dictLimit = ms->window.dictLimit;
+ const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit;
+ const BYTE* const prefixStart = base + prefixStartIndex;
+ const BYTE* const dictEnd = dictBase + prefixStartIndex;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - 8;
+ U32 offset_1=rep[0], offset_2=rep[1];
+ U32 offsetSaved1 = 0, offsetSaved2 = 0;
+
+ const BYTE* ip0 = istart;
+ const BYTE* ip1;
+ const BYTE* ip2;
+ const BYTE* ip3;
+ U32 current0;
+
+
+ size_t hash0; /* hash for ip0 */
+ size_t hash1; /* hash for ip1 */
+ U32 idx; /* match idx for ip0 */
+ const BYTE* idxBase; /* base pointer for idx */
+
+ U32 offcode;
+ const BYTE* match0;
+ size_t mLength;
+ const BYTE* matchEnd = 0; /* initialize to avoid warning, assert != 0 later */
+
+ size_t step;
+ const BYTE* nextStep;
+ const size_t kStepIncr = (1 << (kSearchStrength - 1));
+
+ (void)hasStep; /* not currently specialized on whether it's accelerated */
+
+ DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1);
+
+ /* switch to "regular" variant if extDict is invalidated due to maxDistance */
+ if (prefixStartIndex == dictStartIndex)
+ return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize);
+
+ { U32 const curr = (U32)(ip0 - base);
+ U32 const maxRep = curr - dictStartIndex;
+ if (offset_2 >= maxRep) offsetSaved2 = offset_2, offset_2 = 0;
+ if (offset_1 >= maxRep) offsetSaved1 = offset_1, offset_1 = 0;
+ }
+
+ /* start each op */
+_start: /* Requires: ip0 */
+
+ step = stepSize;
+ nextStep = ip0 + kStepIncr;
+
+ /* calculate positions, ip0 - anchor == 0, so we skip step calc */
+ ip1 = ip0 + 1;
+ ip2 = ip0 + step;
+ ip3 = ip2 + 1;
+
+ if (ip3 >= ilimit) {
+ goto _cleanup;
+ }
+
+ hash0 = ZSTD_hashPtr(ip0, hlog, mls);
+ hash1 = ZSTD_hashPtr(ip1, hlog, mls);
+
+ idx = hashTable[hash0];
+ idxBase = idx < prefixStartIndex ? dictBase : base;
+
+ do {
+ { /* load repcode match for ip[2] */
+ U32 const current2 = (U32)(ip2 - base);
+ U32 const repIndex = current2 - offset_1;
+ const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
+ U32 rval;
+ if ( ((U32)(prefixStartIndex - repIndex) >= 4) /* intentional underflow */
+ & (offset_1 > 0) ) {
+ rval = MEM_read32(repBase + repIndex);
+ } else {
+ rval = MEM_read32(ip2) ^ 1; /* guaranteed to not match. */
+ }
+
+ /* write back hash table entry */
+ current0 = (U32)(ip0 - base);
+ hashTable[hash0] = current0;
+
+ /* check repcode at ip[2] */
+ if (MEM_read32(ip2) == rval) {
+ ip0 = ip2;
+ match0 = repBase + repIndex;
+ matchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
+ assert((match0 != prefixStart) & (match0 != dictStart));
+ mLength = ip0[-1] == match0[-1];
+ ip0 -= mLength;
+ match0 -= mLength;
+ offcode = REPCODE1_TO_OFFBASE;
+ mLength += 4;
+ goto _match;
+ } }
+
+ { /* load match for ip[0] */
+ U32 const mval = idx >= dictStartIndex ?
+ MEM_read32(idxBase + idx) :
+ MEM_read32(ip0) ^ 1; /* guaranteed not to match */
+
+ /* check match at ip[0] */
+ if (MEM_read32(ip0) == mval) {
+ /* found a match! */
+ goto _offset;
+ } }
+
+ /* lookup ip[1] */
+ idx = hashTable[hash1];
+ idxBase = idx < prefixStartIndex ? dictBase : base;
+
+ /* hash ip[2] */
+ hash0 = hash1;
+ hash1 = ZSTD_hashPtr(ip2, hlog, mls);
+
+ /* advance to next positions */
+ ip0 = ip1;
+ ip1 = ip2;
+ ip2 = ip3;
+
+ /* write back hash table entry */
+ current0 = (U32)(ip0 - base);
+ hashTable[hash0] = current0;
+
+ { /* load match for ip[0] */
+ U32 const mval = idx >= dictStartIndex ?
+ MEM_read32(idxBase + idx) :
+ MEM_read32(ip0) ^ 1; /* guaranteed not to match */
+
+ /* check match at ip[0] */
+ if (MEM_read32(ip0) == mval) {
+ /* found a match! */
+ goto _offset;
+ } }
+
+ /* lookup ip[1] */
+ idx = hashTable[hash1];
+ idxBase = idx < prefixStartIndex ? dictBase : base;
+
+ /* hash ip[2] */
+ hash0 = hash1;
+ hash1 = ZSTD_hashPtr(ip2, hlog, mls);
+
+ /* advance to next positions */
+ ip0 = ip1;
+ ip1 = ip2;
+ ip2 = ip0 + step;
+ ip3 = ip1 + step;
+
+ /* calculate step */
+ if (ip2 >= nextStep) {
+ step++;
+ PREFETCH_L1(ip1 + 64);
+ PREFETCH_L1(ip1 + 128);
+ nextStep += kStepIncr;
+ }
+ } while (ip3 < ilimit);
+
+_cleanup:
+ /* Note that there are probably still a couple positions we could search.
+ * However, it seems to be a meaningful performance hit to try to search
+ * them. So let's not. */
+
+ /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
+ * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
+ offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
+
+ /* save reps for next block */
+ rep[0] = offset_1 ? offset_1 : offsetSaved1;
+ rep[1] = offset_2 ? offset_2 : offsetSaved2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+
+_offset: /* Requires: ip0, idx, idxBase */
+
+ /* Compute the offset code. */
+ { U32 const offset = current0 - idx;
+ const BYTE* const lowMatchPtr = idx < prefixStartIndex ? dictStart : prefixStart;
+ matchEnd = idx < prefixStartIndex ? dictEnd : iend;
+ match0 = idxBase + idx;
+ offset_2 = offset_1;
+ offset_1 = offset;
+ offcode = OFFSET_TO_OFFBASE(offset);
+ mLength = 4;
+
+ /* Count the backwards match length. */
+ while (((ip0>anchor) & (match0>lowMatchPtr)) && (ip0[-1] == match0[-1])) {
+ ip0--;
+ match0--;
+ mLength++;
+ } }
+
+_match: /* Requires: ip0, match0, offcode, matchEnd */
+
+ /* Count the forward length. */
+ assert(matchEnd != 0);
+ mLength += ZSTD_count_2segments(ip0 + mLength, match0 + mLength, iend, matchEnd, prefixStart);
+
+ ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength);
+
+ ip0 += mLength;
+ anchor = ip0;
+
+ /* write next hash table entry */
+ if (ip1 < ip0) {
+ hashTable[hash1] = (U32)(ip1 - base);
+ }
+
+ /* Fill table and check for immediate repcode. */
+ if (ip0 <= ilimit) {
+ /* Fill Table */
+ assert(base+current0+2 > istart); /* check base overflow */
+ hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */
+ hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
+
+ while (ip0 <= ilimit) {
+ U32 const repIndex2 = (U32)(ip0-base) - offset_2;
+ const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
+ if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 > 0)) /* intentional underflow */
+ && (MEM_read32(repMatch2) == MEM_read32(ip0)) ) {
+ const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
+ size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
+ { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
+ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
+ ip0 += repLength2;
+ anchor = ip0;
+ continue;
+ }
+ break;
+ } }
+
+ goto _start;
+}
+
+ZSTD_GEN_FAST_FN(extDict, 4, 0)
+ZSTD_GEN_FAST_FN(extDict, 5, 0)
+ZSTD_GEN_FAST_FN(extDict, 6, 0)
+ZSTD_GEN_FAST_FN(extDict, 7, 0)
+
+size_t ZSTD_compressBlock_fast_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ U32 const mls = ms->cParams.minMatch;
+ assert(ms->dictMatchState == NULL);
+ switch(mls)
+ {
+ default: /* includes case 3 */
+ case 4 :
+ return ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize);
+ case 5 :
+ return ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize);
+ case 6 :
+ return ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize);
+ case 7 :
+ return ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize);
+ }
+}
diff --git a/contrib/zstd/zstd_fast.h b/contrib/zstd/zstd_fast.h
new file mode 100644
index 0000000..3e81de4
--- /dev/null
+++ b/contrib/zstd/zstd_fast.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_FAST_H
+#define ZSTD_FAST_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include "mem.h" /* U32 */
+#include "zstd_compress_internal.h"
+
+void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
+ void const* end, ZSTD_dictTableLoadMethod_e dtlm,
+ ZSTD_tableFillPurpose_e tfp);
+size_t ZSTD_compressBlock_fast(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_fast_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_fast_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_FAST_H */
diff --git a/contrib/zstd/zstd_internal.h b/contrib/zstd/zstd_internal.h
new file mode 100644
index 0000000..8f4f64b
--- /dev/null
+++ b/contrib/zstd/zstd_internal.h
@@ -0,0 +1,397 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_CCOMMON_H_MODULE
+#define ZSTD_CCOMMON_H_MODULE
+
+/* this module contains definitions which must be identical
+ * across compression, decompression and dictBuilder.
+ * It also contains a few functions useful to at least 2 of them
+ * and which benefit from being inlined */
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "compiler.h"
+#include "cpu.h"
+#include "mem.h"
+#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */
+#include "error_private.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#include "huf.h"
+#ifndef XXH_STATIC_LINKING_ONLY
+# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
+#endif
+#include "xxhash.h" /* XXH_reset, update, digest */
+#ifndef ZSTD_NO_TRACE
+# include "zstd_trace.h"
+#else
+# define ZSTD_TRACE 0
+#endif
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* ---- static assert (debug) --- */
+#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
+#define ZSTD_isError ERR_isError /* for inlining */
+#define FSE_isError ERR_isError
+#define HUF_isError ERR_isError
+
+
+/*-*************************************
+* shared macros
+***************************************/
+#undef MIN
+#undef MAX
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+#define MAX(a,b) ((a)>(b) ? (a) : (b))
+#define BOUNDED(min,val,max) (MAX(min,MIN(val,max)))
+
+
+/*-*************************************
+* Common constants
+***************************************/
+#define ZSTD_OPT_NUM (1<<12)
+
+#define ZSTD_REP_NUM 3 /* number of repcodes */
+static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define BIT7 128
+#define BIT6 64
+#define BIT5 32
+#define BIT4 16
+#define BIT1 2
+#define BIT0 1
+
+#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
+static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
+static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
+
+#define ZSTD_FRAMEIDSIZE 4 /* magic number size */
+
+#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
+static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
+typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
+
+#define ZSTD_FRAMECHECKSUMSIZE 4
+
+#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
+#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */
+#define MIN_LITERALS_FOR_4_STREAMS 6
+
+typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
+
+#define LONGNBSEQ 0x7F00
+
+#define MINMATCH 3
+
+#define Litbits 8
+#define LitHufLog 11
+#define MaxLit ((1<<Litbits) - 1)
+#define MaxML 52
+#define MaxLL 35
+#define DefaultMaxOff 28
+#define MaxOff 31
+#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
+#define MLFSELog 9
+#define LLFSELog 9
+#define OffFSELog 8
+#define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
+#define MaxMLBits 16
+#define MaxLLBits 16
+
+#define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */
+/* Each table cannot take more than #symbols * FSELog bits */
+#define ZSTD_MAX_FSE_HEADERS_SIZE (((MaxML + 1) * MLFSELog + (MaxLL + 1) * LLFSELog + (MaxOff + 1) * OffFSELog + 7) / 8)
+
+static UNUSED_ATTR const U8 LL_bits[MaxLL+1] = {
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 1, 1, 1, 2, 2, 3, 3,
+ 4, 6, 7, 8, 9,10,11,12,
+ 13,14,15,16
+};
+static UNUSED_ATTR const S16 LL_defaultNorm[MaxLL+1] = {
+ 4, 3, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 1, 1, 1,
+ 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 3, 2, 1, 1, 1, 1, 1,
+ -1,-1,-1,-1
+};
+#define LL_DEFAULTNORMLOG 6 /* for static allocation */
+static UNUSED_ATTR const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
+
+static UNUSED_ATTR const U8 ML_bits[MaxML+1] = {
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 1, 1, 1, 2, 2, 3, 3,
+ 4, 4, 5, 7, 8, 9,10,11,
+ 12,13,14,15,16
+};
+static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = {
+ 1, 4, 3, 2, 2, 2, 2, 2,
+ 2, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1,-1,-1,
+ -1,-1,-1,-1,-1
+};
+#define ML_DEFAULTNORMLOG 6 /* for static allocation */
+static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
+
+static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = {
+ 1, 1, 1, 1, 1, 1, 2, 2,
+ 2, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ -1,-1,-1,-1,-1
+};
+#define OF_DEFAULTNORMLOG 5 /* for static allocation */
+static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
+
+
+/*-*******************************************
+* Shared functions to include for inlining
+*********************************************/
+static void ZSTD_copy8(void* dst, const void* src) {
+#if defined(ZSTD_ARCH_ARM_NEON)
+ vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src));
+#else
+ ZSTD_memcpy(dst, src, 8);
+#endif
+}
+#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
+
+/* Need to use memmove here since the literal buffer can now be located within
+ the dst buffer. In circumstances where the op "catches up" to where the
+ literal buffer is, there can be partial overlaps in this call on the final
+ copy if the literal is being shifted by less than 16 bytes. */
+static void ZSTD_copy16(void* dst, const void* src) {
+#if defined(ZSTD_ARCH_ARM_NEON)
+ vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src));
+#elif defined(ZSTD_ARCH_X86_SSE2)
+ _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src));
+#elif defined(__clang__)
+ ZSTD_memmove(dst, src, 16);
+#else
+ /* ZSTD_memmove is not inlined properly by gcc */
+ BYTE copy16_buf[16];
+ ZSTD_memcpy(copy16_buf, src, 16);
+ ZSTD_memcpy(dst, copy16_buf, 16);
+#endif
+}
+#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
+
+#define WILDCOPY_OVERLENGTH 32
+#define WILDCOPY_VECLEN 16
+
+typedef enum {
+ ZSTD_no_overlap,
+ ZSTD_overlap_src_before_dst
+ /* ZSTD_overlap_dst_before_src, */
+} ZSTD_overlap_e;
+
+/*! ZSTD_wildcopy() :
+ * Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0)
+ * @param ovtype controls the overlap detection
+ * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
+ * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart.
+ * The src buffer must be before the dst buffer.
+ */
+MEM_STATIC FORCE_INLINE_ATTR
+void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype)
+{
+ ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* op = (BYTE*)dst;
+ BYTE* const oend = op + length;
+
+ if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) {
+ /* Handle short offset copies. */
+ do {
+ COPY8(op, ip)
+ } while (op < oend);
+ } else {
+ assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN);
+ /* Separate out the first COPY16() call because the copy length is
+ * almost certain to be short, so the branches have different
+ * probabilities. Since it is almost certain to be short, only do
+ * one COPY16() in the first call. Then, do two calls per loop since
+ * at that point it is more likely to have a high trip count.
+ */
+ ZSTD_copy16(op, ip);
+ if (16 >= length) return;
+ op += 16;
+ ip += 16;
+ do {
+ COPY16(op, ip);
+ COPY16(op, ip);
+ }
+ while (op < oend);
+ }
+}
+
+MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+ size_t const length = MIN(dstCapacity, srcSize);
+ if (length > 0) {
+ ZSTD_memcpy(dst, src, length);
+ }
+ return length;
+}
+
+/* define "workspace is too large" as this number of times larger than needed */
+#define ZSTD_WORKSPACETOOLARGE_FACTOR 3
+
+/* when workspace is continuously too large
+ * during at least this number of times,
+ * context's memory usage is considered wasteful,
+ * because it's sized to handle a worst case scenario which rarely happens.
+ * In which case, resize it down to free some memory */
+#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
+
+/* Controls whether the input/output buffer is buffered or stable. */
+typedef enum {
+ ZSTD_bm_buffered = 0, /* Buffer the input/output */
+ ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */
+} ZSTD_bufferMode_e;
+
+
+/*-*******************************************
+* Private declarations
+*********************************************/
+typedef struct seqDef_s {
+ U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
+ U16 litLength;
+ U16 mlBase; /* mlBase == matchLength - MINMATCH */
+} seqDef;
+
+/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
+typedef enum {
+ ZSTD_llt_none = 0, /* no longLengthType */
+ ZSTD_llt_literalLength = 1, /* represents a long literal */
+ ZSTD_llt_matchLength = 2 /* represents a long match */
+} ZSTD_longLengthType_e;
+
+typedef struct {
+ seqDef* sequencesStart;
+ seqDef* sequences; /* ptr to end of sequences */
+ BYTE* litStart;
+ BYTE* lit; /* ptr to end of literals */
+ BYTE* llCode;
+ BYTE* mlCode;
+ BYTE* ofCode;
+ size_t maxNbSeq;
+ size_t maxNbLit;
+
+ /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
+ * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
+ * the existing value of the litLength or matchLength by 0x10000.
+ */
+ ZSTD_longLengthType_e longLengthType;
+ U32 longLengthPos; /* Index of the sequence to apply long length modification to */
+} seqStore_t;
+
+typedef struct {
+ U32 litLength;
+ U32 matchLength;
+} ZSTD_sequenceLength;
+
+/**
+ * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences
+ * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
+ */
+MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq)
+{
+ ZSTD_sequenceLength seqLen;
+ seqLen.litLength = seq->litLength;
+ seqLen.matchLength = seq->mlBase + MINMATCH;
+ if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
+ if (seqStore->longLengthType == ZSTD_llt_literalLength) {
+ seqLen.litLength += 0x10000;
+ }
+ if (seqStore->longLengthType == ZSTD_llt_matchLength) {
+ seqLen.matchLength += 0x10000;
+ }
+ }
+ return seqLen;
+}
+
+/**
+ * Contains the compressed frame size and an upper-bound for the decompressed frame size.
+ * Note: before using `compressedSize`, check for errors using ZSTD_isError().
+ * similarly, before using `decompressedBound`, check for errors using:
+ * `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
+ */
+typedef struct {
+ size_t nbBlocks;
+ size_t compressedSize;
+ unsigned long long decompressedBound;
+} ZSTD_frameSizeInfo; /* decompress & legacy */
+
+const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
+int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
+
+/* custom memory allocation functions */
+void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);
+void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem);
+void ZSTD_customFree(void* ptr, ZSTD_customMem customMem);
+
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ * do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */
+
+
+typedef struct {
+ blockType_e blockType;
+ U32 lastBlock;
+ U32 origSize;
+} blockProperties_t; /* declared here for decompress and fullbench */
+
+/*! ZSTD_getcBlockSize() :
+ * Provides the size of compressed block from block header `src` */
+/* Used by: decompress, fullbench (does not get its definition from here) */
+size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
+ blockProperties_t* bpPtr);
+
+/*! ZSTD_decodeSeqHeaders() :
+ * decode sequence header from src */
+/* Used by: decompress, fullbench (does not get its definition from here) */
+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
+ const void* src, size_t srcSize);
+
+/**
+ * @returns true iff the CPU supports dynamic BMI2 dispatch.
+ */
+MEM_STATIC int ZSTD_cpuSupportsBmi2(void)
+{
+ ZSTD_cpuid_t cpuid = ZSTD_cpuid();
+ return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid);
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_CCOMMON_H_MODULE */
diff --git a/contrib/zstd/zstd_lazy.c b/contrib/zstd/zstd_lazy.c
new file mode 100644
index 0000000..297f520
--- /dev/null
+++ b/contrib/zstd/zstd_lazy.c
@@ -0,0 +1,2127 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h"
+#include "zstd_lazy.h"
+#include "bits.h" /* ZSTD_countTrailingZeros64 */
+
+
+/*-*************************************
+* Binary Tree search
+***************************************/
+
+static void
+ZSTD_updateDUBT(ZSTD_matchState_t* ms,
+ const BYTE* ip, const BYTE* iend,
+ U32 mls)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hashLog = cParams->hashLog;
+
+ U32* const bt = ms->chainTable;
+ U32 const btLog = cParams->chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+
+ const BYTE* const base = ms->window.base;
+ U32 const target = (U32)(ip - base);
+ U32 idx = ms->nextToUpdate;
+
+ if (idx != target)
+ DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)",
+ idx, target, ms->window.dictLimit);
+ assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */
+ (void)iend;
+
+ assert(idx >= ms->window.dictLimit); /* condition for valid base+idx */
+ for ( ; idx < target ; idx++) {
+ size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */
+ U32 const matchIndex = hashTable[h];
+
+ U32* const nextCandidatePtr = bt + 2*(idx&btMask);
+ U32* const sortMarkPtr = nextCandidatePtr + 1;
+
+ DEBUGLOG(8, "ZSTD_updateDUBT: insert %u", idx);
+ hashTable[h] = idx; /* Update Hash Table */
+ *nextCandidatePtr = matchIndex; /* update BT like a chain */
+ *sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK;
+ }
+ ms->nextToUpdate = target;
+}
+
+
+/** ZSTD_insertDUBT1() :
+ * sort one already inserted but unsorted position
+ * assumption : curr >= btlow == (curr - btmask)
+ * doesn't fail */
+static void
+ZSTD_insertDUBT1(const ZSTD_matchState_t* ms,
+ U32 curr, const BYTE* inputEnd,
+ U32 nbCompares, U32 btLow,
+ const ZSTD_dictMode_e dictMode)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const bt = ms->chainTable;
+ U32 const btLog = cParams->chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+ size_t commonLengthSmaller=0, commonLengthLarger=0;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const ip = (curr>=dictLimit) ? base + curr : dictBase + curr;
+ const BYTE* const iend = (curr>=dictLimit) ? inputEnd : dictBase + dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ const BYTE* match;
+ U32* smallerPtr = bt + 2*(curr&btMask);
+ U32* largerPtr = smallerPtr + 1;
+ U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */
+ U32 dummy32; /* to be nullified at the end */
+ U32 const windowValid = ms->window.lowLimit;
+ U32 const maxDistance = 1U << cParams->windowLog;
+ U32 const windowLow = (curr - windowValid > maxDistance) ? curr - maxDistance : windowValid;
+
+
+ DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)",
+ curr, dictLimit, windowLow);
+ assert(curr >= btLow);
+ assert(ip < iend); /* condition for ZSTD_count */
+
+ for (; nbCompares && (matchIndex > windowLow); --nbCompares) {
+ U32* const nextPtr = bt + 2*(matchIndex & btMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ assert(matchIndex < curr);
+ /* note : all candidates are now supposed sorted,
+ * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK
+ * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */
+
+ if ( (dictMode != ZSTD_extDict)
+ || (matchIndex+matchLength >= dictLimit) /* both in current segment*/
+ || (curr < dictLimit) /* both in extDict */) {
+ const BYTE* const mBase = ( (dictMode != ZSTD_extDict)
+ || (matchIndex+matchLength >= dictLimit)) ?
+ base : dictBase;
+ assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */
+ || (curr < dictLimit) );
+ match = mBase + matchIndex;
+ matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
+ } else {
+ match = dictBase + matchIndex;
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
+ if (matchIndex+matchLength >= dictLimit)
+ match = base + matchIndex; /* preparation for next read of match[matchLength] */
+ }
+
+ DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ",
+ curr, matchIndex, (U32)matchLength);
+
+ if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
+ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */
+ }
+
+ if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */
+ /* match is smaller than current */
+ *smallerPtr = matchIndex; /* update smaller idx */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */
+ DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is smaller : next => %u",
+ matchIndex, btLow, nextPtr[1]);
+ smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */
+ matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */
+ } else {
+ /* match is larger than current */
+ *largerPtr = matchIndex;
+ commonLengthLarger = matchLength;
+ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */
+ DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is larger => %u",
+ matchIndex, btLow, nextPtr[0]);
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ } }
+
+ *smallerPtr = *largerPtr = 0;
+}
+
+
+static size_t
+ZSTD_DUBT_findBetterDictMatch (
+ const ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iend,
+ size_t* offsetPtr,
+ size_t bestLength,
+ U32 nbCompares,
+ U32 const mls,
+ const ZSTD_dictMode_e dictMode)
+{
+ const ZSTD_matchState_t * const dms = ms->dictMatchState;
+ const ZSTD_compressionParameters* const dmsCParams = &dms->cParams;
+ const U32 * const dictHashTable = dms->hashTable;
+ U32 const hashLog = dmsCParams->hashLog;
+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
+ U32 dictMatchIndex = dictHashTable[h];
+
+ const BYTE* const base = ms->window.base;
+ const BYTE* const prefixStart = base + ms->window.dictLimit;
+ U32 const curr = (U32)(ip-base);
+ const BYTE* const dictBase = dms->window.base;
+ const BYTE* const dictEnd = dms->window.nextSrc;
+ U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base);
+ U32 const dictLowLimit = dms->window.lowLimit;
+ U32 const dictIndexDelta = ms->window.lowLimit - dictHighLimit;
+
+ U32* const dictBt = dms->chainTable;
+ U32 const btLog = dmsCParams->chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+ U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask;
+
+ size_t commonLengthSmaller=0, commonLengthLarger=0;
+
+ (void)dictMode;
+ assert(dictMode == ZSTD_dictMatchState);
+
+ for (; nbCompares && (dictMatchIndex > dictLowLimit); --nbCompares) {
+ U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ const BYTE* match = dictBase + dictMatchIndex;
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
+ if (dictMatchIndex+matchLength >= dictHighLimit)
+ match = base + dictMatchIndex + dictIndexDelta; /* to prepare for next usage of match[matchLength] */
+
+ if (matchLength > bestLength) {
+ U32 matchIndex = dictMatchIndex + dictIndexDelta;
+ if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) {
+ DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
+ curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, OFFSET_TO_OFFBASE(curr - matchIndex), dictMatchIndex, matchIndex);
+ bestLength = matchLength, *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
+ }
+ if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */
+ break; /* drop, to guarantee consistency (miss a little bit of compression) */
+ }
+ }
+
+ if (match[matchLength] < ip[matchLength]) {
+ if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
+ } else {
+ /* match is larger than current */
+ if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */
+ commonLengthLarger = matchLength;
+ dictMatchIndex = nextPtr[0];
+ }
+ }
+
+ if (bestLength >= MINMATCH) {
+ U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offsetPtr); (void)mIndex;
+ DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
+ curr, (U32)bestLength, (U32)*offsetPtr, mIndex);
+ }
+ return bestLength;
+
+}
+
+
+static size_t
+ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iend,
+ size_t* offBasePtr,
+ U32 const mls,
+ const ZSTD_dictMode_e dictMode)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hashLog = cParams->hashLog;
+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
+ U32 matchIndex = hashTable[h];
+
+ const BYTE* const base = ms->window.base;
+ U32 const curr = (U32)(ip-base);
+ U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog);
+
+ U32* const bt = ms->chainTable;
+ U32 const btLog = cParams->chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+ U32 const btLow = (btMask >= curr) ? 0 : curr - btMask;
+ U32 const unsortLimit = MAX(btLow, windowLow);
+
+ U32* nextCandidate = bt + 2*(matchIndex&btMask);
+ U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1;
+ U32 nbCompares = 1U << cParams->searchLog;
+ U32 nbCandidates = nbCompares;
+ U32 previousCandidate = 0;
+
+ DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", curr);
+ assert(ip <= iend-8); /* required for h calculation */
+ assert(dictMode != ZSTD_dedicatedDictSearch);
+
+ /* reach end of unsorted candidates list */
+ while ( (matchIndex > unsortLimit)
+ && (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK)
+ && (nbCandidates > 1) ) {
+ DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted",
+ matchIndex);
+ *unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */
+ previousCandidate = matchIndex;
+ matchIndex = *nextCandidate;
+ nextCandidate = bt + 2*(matchIndex&btMask);
+ unsortedMark = bt + 2*(matchIndex&btMask) + 1;
+ nbCandidates --;
+ }
+
+ /* nullify last candidate if it's still unsorted
+ * simplification, detrimental to compression ratio, beneficial for speed */
+ if ( (matchIndex > unsortLimit)
+ && (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) {
+ DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u",
+ matchIndex);
+ *nextCandidate = *unsortedMark = 0;
+ }
+
+ /* batch sort stacked candidates */
+ matchIndex = previousCandidate;
+ while (matchIndex) { /* will end on matchIndex == 0 */
+ U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1;
+ U32 const nextCandidateIdx = *nextCandidateIdxPtr;
+ ZSTD_insertDUBT1(ms, matchIndex, iend,
+ nbCandidates, unsortLimit, dictMode);
+ matchIndex = nextCandidateIdx;
+ nbCandidates++;
+ }
+
+ /* find longest match */
+ { size_t commonLengthSmaller = 0, commonLengthLarger = 0;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ U32* smallerPtr = bt + 2*(curr&btMask);
+ U32* largerPtr = bt + 2*(curr&btMask) + 1;
+ U32 matchEndIdx = curr + 8 + 1;
+ U32 dummy32; /* to be nullified at the end */
+ size_t bestLength = 0;
+
+ matchIndex = hashTable[h];
+ hashTable[h] = curr; /* Update Hash Table */
+
+ for (; nbCompares && (matchIndex > windowLow); --nbCompares) {
+ U32* const nextPtr = bt + 2*(matchIndex & btMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ const BYTE* match;
+
+ if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) {
+ match = base + matchIndex;
+ matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
+ } else {
+ match = dictBase + matchIndex;
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
+ if (matchIndex+matchLength >= dictLimit)
+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
+ }
+
+ if (matchLength > bestLength) {
+ if (matchLength > matchEndIdx - matchIndex)
+ matchEndIdx = matchIndex + (U32)matchLength;
+ if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)*offBasePtr)) )
+ bestLength = matchLength, *offBasePtr = OFFSET_TO_OFFBASE(curr - matchIndex);
+ if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
+ if (dictMode == ZSTD_dictMatchState) {
+ nbCompares = 0; /* in addition to avoiding checking any
+ * further in this loop, make sure we
+ * skip checking in the dictionary. */
+ }
+ break; /* drop, to guarantee consistency (miss a little bit of compression) */
+ }
+ }
+
+ if (match[matchLength] < ip[matchLength]) {
+ /* match is smaller than current */
+ *smallerPtr = matchIndex; /* update smaller idx */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
+ } else {
+ /* match is larger than current */
+ *largerPtr = matchIndex;
+ commonLengthLarger = matchLength;
+ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ } }
+
+ *smallerPtr = *largerPtr = 0;
+
+ assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
+ if (dictMode == ZSTD_dictMatchState && nbCompares) {
+ bestLength = ZSTD_DUBT_findBetterDictMatch(
+ ms, ip, iend,
+ offBasePtr, bestLength, nbCompares,
+ mls, dictMode);
+ }
+
+ assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */
+ ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
+ if (bestLength >= MINMATCH) {
+ U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offBasePtr); (void)mIndex;
+ DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
+ curr, (U32)bestLength, (U32)*offBasePtr, mIndex);
+ }
+ return bestLength;
+ }
+}
+
+
+/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iLimit,
+ size_t* offBasePtr,
+ const U32 mls /* template */,
+ const ZSTD_dictMode_e dictMode)
+{
+ DEBUGLOG(7, "ZSTD_BtFindBestMatch");
+ if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */
+ ZSTD_updateDUBT(ms, ip, iLimit, mls);
+ return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offBasePtr, mls, dictMode);
+}
+
+/***********************************
+* Dedicated dict search
+***********************************/
+
+void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip)
+{
+ const BYTE* const base = ms->window.base;
+ U32 const target = (U32)(ip - base);
+ U32* const hashTable = ms->hashTable;
+ U32* const chainTable = ms->chainTable;
+ U32 const chainSize = 1 << ms->cParams.chainLog;
+ U32 idx = ms->nextToUpdate;
+ U32 const minChain = chainSize < target - idx ? target - chainSize : idx;
+ U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG;
+ U32 const cacheSize = bucketSize - 1;
+ U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize;
+ U32 const chainLimit = chainAttempts > 255 ? 255 : chainAttempts;
+
+ /* We know the hashtable is oversized by a factor of `bucketSize`.
+ * We are going to temporarily pretend `bucketSize == 1`, keeping only a
+ * single entry. We will use the rest of the space to construct a temporary
+ * chaintable.
+ */
+ U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG;
+ U32* const tmpHashTable = hashTable;
+ U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog);
+ U32 const tmpChainSize = (U32)((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog;
+ U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx;
+ U32 hashIdx;
+
+ assert(ms->cParams.chainLog <= 24);
+ assert(ms->cParams.hashLog > ms->cParams.chainLog);
+ assert(idx != 0);
+ assert(tmpMinChain <= minChain);
+
+ /* fill conventional hash table and conventional chain table */
+ for ( ; idx < target; idx++) {
+ U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch);
+ if (idx >= tmpMinChain) {
+ tmpChainTable[idx - tmpMinChain] = hashTable[h];
+ }
+ tmpHashTable[h] = idx;
+ }
+
+ /* sort chains into ddss chain table */
+ {
+ U32 chainPos = 0;
+ for (hashIdx = 0; hashIdx < (1U << hashLog); hashIdx++) {
+ U32 count;
+ U32 countBeyondMinChain = 0;
+ U32 i = tmpHashTable[hashIdx];
+ for (count = 0; i >= tmpMinChain && count < cacheSize; count++) {
+ /* skip through the chain to the first position that won't be
+ * in the hash cache bucket */
+ if (i < minChain) {
+ countBeyondMinChain++;
+ }
+ i = tmpChainTable[i - tmpMinChain];
+ }
+ if (count == cacheSize) {
+ for (count = 0; count < chainLimit;) {
+ if (i < minChain) {
+ if (!i || ++countBeyondMinChain > cacheSize) {
+ /* only allow pulling `cacheSize` number of entries
+ * into the cache or chainTable beyond `minChain`,
+ * to replace the entries pulled out of the
+ * chainTable into the cache. This lets us reach
+ * back further without increasing the total number
+ * of entries in the chainTable, guaranteeing the
+ * DDSS chain table will fit into the space
+ * allocated for the regular one. */
+ break;
+ }
+ }
+ chainTable[chainPos++] = i;
+ count++;
+ if (i < tmpMinChain) {
+ break;
+ }
+ i = tmpChainTable[i - tmpMinChain];
+ }
+ } else {
+ count = 0;
+ }
+ if (count) {
+ tmpHashTable[hashIdx] = ((chainPos - count) << 8) + count;
+ } else {
+ tmpHashTable[hashIdx] = 0;
+ }
+ }
+ assert(chainPos <= chainSize); /* I believe this is guaranteed... */
+ }
+
+ /* move chain pointers into the last entry of each hash bucket */
+ for (hashIdx = (1 << hashLog); hashIdx; ) {
+ U32 const bucketIdx = --hashIdx << ZSTD_LAZY_DDSS_BUCKET_LOG;
+ U32 const chainPackedPointer = tmpHashTable[hashIdx];
+ U32 i;
+ for (i = 0; i < cacheSize; i++) {
+ hashTable[bucketIdx + i] = 0;
+ }
+ hashTable[bucketIdx + bucketSize - 1] = chainPackedPointer;
+ }
+
+ /* fill the buckets of the hash table */
+ for (idx = ms->nextToUpdate; idx < target; idx++) {
+ U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch)
+ << ZSTD_LAZY_DDSS_BUCKET_LOG;
+ U32 i;
+ /* Shift hash cache down 1. */
+ for (i = cacheSize - 1; i; i--)
+ hashTable[h + i] = hashTable[h + i - 1];
+ hashTable[h] = idx;
+ }
+
+ ms->nextToUpdate = target;
+}
+
+/* Returns the longest match length found in the dedicated dict search structure.
+ * If none are longer than the argument ml, then ml will be returned.
+ */
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts,
+ const ZSTD_matchState_t* const dms,
+ const BYTE* const ip, const BYTE* const iLimit,
+ const BYTE* const prefixStart, const U32 curr,
+ const U32 dictLimit, const size_t ddsIdx) {
+ const U32 ddsLowestIndex = dms->window.dictLimit;
+ const BYTE* const ddsBase = dms->window.base;
+ const BYTE* const ddsEnd = dms->window.nextSrc;
+ const U32 ddsSize = (U32)(ddsEnd - ddsBase);
+ const U32 ddsIndexDelta = dictLimit - ddsSize;
+ const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG);
+ const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1;
+ U32 ddsAttempt;
+ U32 matchIndex;
+
+ for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) {
+ PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]);
+ }
+
+ {
+ U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1];
+ U32 const chainIndex = chainPackedPointer >> 8;
+
+ PREFETCH_L1(&dms->chainTable[chainIndex]);
+ }
+
+ for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) {
+ size_t currentMl=0;
+ const BYTE* match;
+ matchIndex = dms->hashTable[ddsIdx + ddsAttempt];
+ match = ddsBase + matchIndex;
+
+ if (!matchIndex) {
+ return ml;
+ }
+
+ /* guaranteed by table construction */
+ (void)ddsLowestIndex;
+ assert(matchIndex >= ddsLowestIndex);
+ assert(match+4 <= ddsEnd);
+ if (MEM_read32(match) == MEM_read32(ip)) {
+ /* assumption : matchIndex <= dictLimit-4 (by table construction) */
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4;
+ }
+
+ /* save best solution */
+ if (currentMl > ml) {
+ ml = currentMl;
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta));
+ if (ip+currentMl == iLimit) {
+ /* best possible, avoids read overflow on next attempt */
+ return ml;
+ }
+ }
+ }
+
+ {
+ U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1];
+ U32 chainIndex = chainPackedPointer >> 8;
+ U32 const chainLength = chainPackedPointer & 0xFF;
+ U32 const chainAttempts = nbAttempts - ddsAttempt;
+ U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts;
+ U32 chainAttempt;
+
+ for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) {
+ PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]);
+ }
+
+ for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) {
+ size_t currentMl=0;
+ const BYTE* match;
+ matchIndex = dms->chainTable[chainIndex];
+ match = ddsBase + matchIndex;
+
+ /* guaranteed by table construction */
+ assert(matchIndex >= ddsLowestIndex);
+ assert(match+4 <= ddsEnd);
+ if (MEM_read32(match) == MEM_read32(ip)) {
+ /* assumption : matchIndex <= dictLimit-4 (by table construction) */
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4;
+ }
+
+ /* save best solution */
+ if (currentMl > ml) {
+ ml = currentMl;
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta));
+ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
+ }
+ }
+ }
+ return ml;
+}
+
+
+/* *********************************
+* Hash Chain
+***********************************/
+#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)]
+
+/* Update chains up to ip (excluded)
+ Assumption : always within prefix (i.e. not within extDict) */
+FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal(
+ ZSTD_matchState_t* ms,
+ const ZSTD_compressionParameters* const cParams,
+ const BYTE* ip, U32 const mls)
+{
+ U32* const hashTable = ms->hashTable;
+ const U32 hashLog = cParams->hashLog;
+ U32* const chainTable = ms->chainTable;
+ const U32 chainMask = (1 << cParams->chainLog) - 1;
+ const BYTE* const base = ms->window.base;
+ const U32 target = (U32)(ip - base);
+ U32 idx = ms->nextToUpdate;
+
+ while(idx < target) { /* catch up */
+ size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls);
+ NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
+ hashTable[h] = idx;
+ idx++;
+ }
+
+ ms->nextToUpdate = target;
+ return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
+}
+
+U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch);
+}
+
+/* inlining is important to hardwire a hot branch (template emulation) */
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_HcFindBestMatch(
+ ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iLimit,
+ size_t* offsetPtr,
+ const U32 mls, const ZSTD_dictMode_e dictMode)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const chainTable = ms->chainTable;
+ const U32 chainSize = (1 << cParams->chainLog);
+ const U32 chainMask = chainSize-1;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const U32 curr = (U32)(ip-base);
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 lowestValid = ms->window.lowLimit;
+ const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
+ const U32 isDictionary = (ms->loadedDictEnd != 0);
+ const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance;
+ const U32 minChain = curr > chainSize ? curr - chainSize : 0;
+ U32 nbAttempts = 1U << cParams->searchLog;
+ size_t ml=4-1;
+
+ const ZSTD_matchState_t* const dms = ms->dictMatchState;
+ const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch
+ ? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0;
+ const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch
+ ? ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG : 0;
+
+ U32 matchIndex;
+
+ if (dictMode == ZSTD_dedicatedDictSearch) {
+ const U32* entry = &dms->hashTable[ddsIdx];
+ PREFETCH_L1(entry);
+ }
+
+ /* HC4 match finder */
+ matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls);
+
+ for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) {
+ size_t currentMl=0;
+ if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
+ const BYTE* const match = base + matchIndex;
+ assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
+ /* read 4B starting from (match + ml + 1 - sizeof(U32)) */
+ if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */
+ currentMl = ZSTD_count(ip, match, iLimit);
+ } else {
+ const BYTE* const match = dictBase + matchIndex;
+ assert(match+4 <= dictEnd);
+ if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4;
+ }
+
+ /* save best solution */
+ if (currentMl > ml) {
+ ml = currentMl;
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
+ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
+ }
+
+ if (matchIndex <= minChain) break;
+ matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
+ }
+
+ assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
+ if (dictMode == ZSTD_dedicatedDictSearch) {
+ ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms,
+ ip, iLimit, prefixStart, curr, dictLimit, ddsIdx);
+ } else if (dictMode == ZSTD_dictMatchState) {
+ const U32* const dmsChainTable = dms->chainTable;
+ const U32 dmsChainSize = (1 << dms->cParams.chainLog);
+ const U32 dmsChainMask = dmsChainSize - 1;
+ const U32 dmsLowestIndex = dms->window.dictLimit;
+ const BYTE* const dmsBase = dms->window.base;
+ const BYTE* const dmsEnd = dms->window.nextSrc;
+ const U32 dmsSize = (U32)(dmsEnd - dmsBase);
+ const U32 dmsIndexDelta = dictLimit - dmsSize;
+ const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0;
+
+ matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)];
+
+ for ( ; (matchIndex>=dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) {
+ size_t currentMl=0;
+ const BYTE* const match = dmsBase + matchIndex;
+ assert(match+4 <= dmsEnd);
+ if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4;
+
+ /* save best solution */
+ if (currentMl > ml) {
+ ml = currentMl;
+ assert(curr > matchIndex + dmsIndexDelta);
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta));
+ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
+ }
+
+ if (matchIndex <= dmsMinChain) break;
+
+ matchIndex = dmsChainTable[matchIndex & dmsChainMask];
+ }
+ }
+
+ return ml;
+}
+
+/* *********************************
+* (SIMD) Row-based matchfinder
+***********************************/
+/* Constants for row-based hash */
+#define ZSTD_ROW_HASH_TAG_OFFSET 16 /* byte offset of hashes in the match state's tagTable from the beginning of a row */
+#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1)
+#define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */
+
+#define ZSTD_ROW_HASH_CACHE_MASK (ZSTD_ROW_HASH_CACHE_SIZE - 1)
+
+typedef U64 ZSTD_VecMask; /* Clarifies when we are interacting with a U64 representing a mask of matches */
+
+/* ZSTD_VecMask_next():
+ * Starting from the LSB, returns the idx of the next non-zero bit.
+ * Basically counting the nb of trailing zeroes.
+ */
+MEM_STATIC U32 ZSTD_VecMask_next(ZSTD_VecMask val) {
+ return ZSTD_countTrailingZeros64(val);
+}
+
+/* ZSTD_rotateRight_*():
+ * Rotates a bitfield to the right by "count" bits.
+ * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts
+ */
+FORCE_INLINE_TEMPLATE
+U64 ZSTD_rotateRight_U64(U64 const value, U32 count) {
+ assert(count < 64);
+ count &= 0x3F; /* for fickle pattern recognition */
+ return (value >> count) | (U64)(value << ((0U - count) & 0x3F));
+}
+
+FORCE_INLINE_TEMPLATE
+U32 ZSTD_rotateRight_U32(U32 const value, U32 count) {
+ assert(count < 32);
+ count &= 0x1F; /* for fickle pattern recognition */
+ return (value >> count) | (U32)(value << ((0U - count) & 0x1F));
+}
+
+FORCE_INLINE_TEMPLATE
+U16 ZSTD_rotateRight_U16(U16 const value, U32 count) {
+ assert(count < 16);
+ count &= 0x0F; /* for fickle pattern recognition */
+ return (value >> count) | (U16)(value << ((0U - count) & 0x0F));
+}
+
+/* ZSTD_row_nextIndex():
+ * Returns the next index to insert at within a tagTable row, and updates the "head"
+ * value to reflect the update. Essentially cycles backwards from [0, {entries per row})
+ */
+FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) {
+ U32 const next = (*tagRow - 1) & rowMask;
+ *tagRow = (BYTE)next;
+ return next;
+}
+
+/* ZSTD_isAligned():
+ * Checks that a pointer is aligned to "align" bytes which must be a power of 2.
+ */
+MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) {
+ assert((align & (align - 1)) == 0);
+ return (((size_t)ptr) & (align - 1)) == 0;
+}
+
+/* ZSTD_row_prefetch():
+ * Performs prefetching for the hashTable and tagTable at a given row.
+ */
+FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, U16 const* tagTable, U32 const relRow, U32 const rowLog) {
+ PREFETCH_L1(hashTable + relRow);
+ if (rowLog >= 5) {
+ PREFETCH_L1(hashTable + relRow + 16);
+ /* Note: prefetching more of the hash table does not appear to be beneficial for 128-entry rows */
+ }
+ PREFETCH_L1(tagTable + relRow);
+ if (rowLog == 6) {
+ PREFETCH_L1(tagTable + relRow + 32);
+ }
+ assert(rowLog == 4 || rowLog == 5 || rowLog == 6);
+ assert(ZSTD_isAligned(hashTable + relRow, 64)); /* prefetched hash row always 64-byte aligned */
+ assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on correct multiple of bytes (32,64,128) */
+}
+
+/* ZSTD_row_fillHashCache():
+ * Fill up the hash cache starting at idx, prefetching up to ZSTD_ROW_HASH_CACHE_SIZE entries,
+ * but not beyond iLimit.
+ */
+FORCE_INLINE_TEMPLATE void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const BYTE* base,
+ U32 const rowLog, U32 const mls,
+ U32 idx, const BYTE* const iLimit)
+{
+ U32 const* const hashTable = ms->hashTable;
+ U16 const* const tagTable = ms->tagTable;
+ U32 const hashLog = ms->rowHashLog;
+ U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1);
+ U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch);
+
+ for (; idx < lim; ++idx) {
+ U32 const hash = (U32)ZSTD_hashPtr(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
+ U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
+ ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
+ ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash;
+ }
+
+ DEBUGLOG(6, "ZSTD_row_fillHashCache(): [%u %u %u %u %u %u %u %u]", ms->hashCache[0], ms->hashCache[1],
+ ms->hashCache[2], ms->hashCache[3], ms->hashCache[4],
+ ms->hashCache[5], ms->hashCache[6], ms->hashCache[7]);
+}
+
+/* ZSTD_row_nextCachedHash():
+ * Returns the hash of base + idx, and replaces the hash in the hash cache with the byte at
+ * base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable.
+ */
+FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable,
+ U16 const* tagTable, BYTE const* base,
+ U32 idx, U32 const hashLog,
+ U32 const rowLog, U32 const mls)
+{
+ U32 const newHash = (U32)ZSTD_hashPtr(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
+ U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
+ ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
+ { U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK];
+ cache[idx & ZSTD_ROW_HASH_CACHE_MASK] = newHash;
+ return hash;
+ }
+}
+
+/* ZSTD_row_update_internalImpl():
+ * Updates the hash table with positions starting from updateStartIdx until updateEndIdx.
+ */
+FORCE_INLINE_TEMPLATE void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms,
+ U32 updateStartIdx, U32 const updateEndIdx,
+ U32 const mls, U32 const rowLog,
+ U32 const rowMask, U32 const useCache)
+{
+ U32* const hashTable = ms->hashTable;
+ U16* const tagTable = ms->tagTable;
+ U32 const hashLog = ms->rowHashLog;
+ const BYTE* const base = ms->window.base;
+
+ DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx);
+ for (; updateStartIdx < updateEndIdx; ++updateStartIdx) {
+ U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls)
+ : (U32)ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
+ U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
+ U32* const row = hashTable + relRow;
+ BYTE* tagRow = (BYTE*)(tagTable + relRow); /* Though tagTable is laid out as a table of U16, each tag is only 1 byte.
+ Explicit cast allows us to get exact desired position within each row */
+ U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
+
+ assert(hash == ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls));
+ ((BYTE*)tagRow)[pos + ZSTD_ROW_HASH_TAG_OFFSET] = hash & ZSTD_ROW_HASH_TAG_MASK;
+ row[pos] = updateStartIdx;
+ }
+}
+
+/* ZSTD_row_update_internal():
+ * Inserts the byte at ip into the appropriate position in the hash table, and updates ms->nextToUpdate.
+ * Skips sections of long matches as is necessary.
+ */
+FORCE_INLINE_TEMPLATE void ZSTD_row_update_internal(ZSTD_matchState_t* ms, const BYTE* ip,
+ U32 const mls, U32 const rowLog,
+ U32 const rowMask, U32 const useCache)
+{
+ U32 idx = ms->nextToUpdate;
+ const BYTE* const base = ms->window.base;
+ const U32 target = (U32)(ip - base);
+ const U32 kSkipThreshold = 384;
+ const U32 kMaxMatchStartPositionsToUpdate = 96;
+ const U32 kMaxMatchEndPositionsToUpdate = 32;
+
+ if (useCache) {
+ /* Only skip positions when using hash cache, i.e.
+ * if we are loading a dict, don't skip anything.
+ * If we decide to skip, then we only update a set number
+ * of positions at the beginning and end of the match.
+ */
+ if (UNLIKELY(target - idx > kSkipThreshold)) {
+ U32 const bound = idx + kMaxMatchStartPositionsToUpdate;
+ ZSTD_row_update_internalImpl(ms, idx, bound, mls, rowLog, rowMask, useCache);
+ idx = target - kMaxMatchEndPositionsToUpdate;
+ ZSTD_row_fillHashCache(ms, base, rowLog, mls, idx, ip+1);
+ }
+ }
+ assert(target >= idx);
+ ZSTD_row_update_internalImpl(ms, idx, target, mls, rowLog, rowMask, useCache);
+ ms->nextToUpdate = target;
+}
+
+/* ZSTD_row_update():
+ * External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary
+ * processing.
+ */
+void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) {
+ const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
+ const U32 rowMask = (1u << rowLog) - 1;
+ const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */);
+
+ DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog);
+ ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* don't use cache */);
+}
+
+/* Returns the mask width of bits group of which will be set to 1. Given not all
+ * architectures have easy movemask instruction, this helps to iterate over
+ * groups of bits easier and faster.
+ */
+FORCE_INLINE_TEMPLATE U32
+ZSTD_row_matchMaskGroupWidth(const U32 rowEntries)
+{
+ assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
+ assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES);
+ (void)rowEntries;
+#if defined(ZSTD_ARCH_ARM_NEON)
+ /* NEON path only works for little endian */
+ if (!MEM_isLittleEndian()) {
+ return 1;
+ }
+ if (rowEntries == 16) {
+ return 4;
+ }
+ if (rowEntries == 32) {
+ return 2;
+ }
+ if (rowEntries == 64) {
+ return 1;
+ }
+#endif
+ return 1;
+}
+
+#if defined(ZSTD_ARCH_X86_SSE2)
+FORCE_INLINE_TEMPLATE ZSTD_VecMask
+ZSTD_row_getSSEMask(int nbChunks, const BYTE* const src, const BYTE tag, const U32 head)
+{
+ const __m128i comparisonMask = _mm_set1_epi8((char)tag);
+ int matches[4] = {0};
+ int i;
+ assert(nbChunks == 1 || nbChunks == 2 || nbChunks == 4);
+ for (i=0; i<nbChunks; i++) {
+ const __m128i chunk = _mm_loadu_si128((const __m128i*)(const void*)(src + 16*i));
+ const __m128i equalMask = _mm_cmpeq_epi8(chunk, comparisonMask);
+ matches[i] = _mm_movemask_epi8(equalMask);
+ }
+ if (nbChunks == 1) return ZSTD_rotateRight_U16((U16)matches[0], head);
+ if (nbChunks == 2) return ZSTD_rotateRight_U32((U32)matches[1] << 16 | (U32)matches[0], head);
+ assert(nbChunks == 4);
+ return ZSTD_rotateRight_U64((U64)matches[3] << 48 | (U64)matches[2] << 32 | (U64)matches[1] << 16 | (U64)matches[0], head);
+}
+#endif
+
+#if defined(ZSTD_ARCH_ARM_NEON)
+FORCE_INLINE_TEMPLATE ZSTD_VecMask
+ZSTD_row_getNEONMask(const U32 rowEntries, const BYTE* const src, const BYTE tag, const U32 headGrouped)
+{
+ assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
+ if (rowEntries == 16) {
+ /* vshrn_n_u16 shifts by 4 every u16 and narrows to 8 lower bits.
+ * After that groups of 4 bits represent the equalMask. We lower
+ * all bits except the highest in these groups by doing AND with
+ * 0x88 = 0b10001000.
+ */
+ const uint8x16_t chunk = vld1q_u8(src);
+ const uint16x8_t equalMask = vreinterpretq_u16_u8(vceqq_u8(chunk, vdupq_n_u8(tag)));
+ const uint8x8_t res = vshrn_n_u16(equalMask, 4);
+ const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0);
+ return ZSTD_rotateRight_U64(matches, headGrouped) & 0x8888888888888888ull;
+ } else if (rowEntries == 32) {
+ /* Same idea as with rowEntries == 16 but doing AND with
+ * 0x55 = 0b01010101.
+ */
+ const uint16x8x2_t chunk = vld2q_u16((const uint16_t*)(const void*)src);
+ const uint8x16_t chunk0 = vreinterpretq_u8_u16(chunk.val[0]);
+ const uint8x16_t chunk1 = vreinterpretq_u8_u16(chunk.val[1]);
+ const uint8x16_t dup = vdupq_n_u8(tag);
+ const uint8x8_t t0 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk0, dup)), 6);
+ const uint8x8_t t1 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk1, dup)), 6);
+ const uint8x8_t res = vsli_n_u8(t0, t1, 4);
+ const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0) ;
+ return ZSTD_rotateRight_U64(matches, headGrouped) & 0x5555555555555555ull;
+ } else { /* rowEntries == 64 */
+ const uint8x16x4_t chunk = vld4q_u8(src);
+ const uint8x16_t dup = vdupq_n_u8(tag);
+ const uint8x16_t cmp0 = vceqq_u8(chunk.val[0], dup);
+ const uint8x16_t cmp1 = vceqq_u8(chunk.val[1], dup);
+ const uint8x16_t cmp2 = vceqq_u8(chunk.val[2], dup);
+ const uint8x16_t cmp3 = vceqq_u8(chunk.val[3], dup);
+
+ const uint8x16_t t0 = vsriq_n_u8(cmp1, cmp0, 1);
+ const uint8x16_t t1 = vsriq_n_u8(cmp3, cmp2, 1);
+ const uint8x16_t t2 = vsriq_n_u8(t1, t0, 2);
+ const uint8x16_t t3 = vsriq_n_u8(t2, t2, 4);
+ const uint8x8_t t4 = vshrn_n_u16(vreinterpretq_u16_u8(t3), 4);
+ const U64 matches = vget_lane_u64(vreinterpret_u64_u8(t4), 0);
+ return ZSTD_rotateRight_U64(matches, headGrouped);
+ }
+}
+#endif
+
+/* Returns a ZSTD_VecMask (U64) that has the nth group (determined by
+ * ZSTD_row_matchMaskGroupWidth) of bits set to 1 if the newly-computed "tag"
+ * matches the hash at the nth position in a row of the tagTable.
+ * Each row is a circular buffer beginning at the value of "headGrouped". So we
+ * must rotate the "matches" bitfield to match up with the actual layout of the
+ * entries within the hashTable */
+FORCE_INLINE_TEMPLATE ZSTD_VecMask
+ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGrouped, const U32 rowEntries)
+{
+ const BYTE* const src = tagRow + ZSTD_ROW_HASH_TAG_OFFSET;
+ assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
+ assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES);
+ assert(ZSTD_row_matchMaskGroupWidth(rowEntries) * rowEntries <= sizeof(ZSTD_VecMask) * 8);
+
+#if defined(ZSTD_ARCH_X86_SSE2)
+
+ return ZSTD_row_getSSEMask(rowEntries / 16, src, tag, headGrouped);
+
+#else /* SW or NEON-LE */
+
+# if defined(ZSTD_ARCH_ARM_NEON)
+ /* This NEON path only works for little endian - otherwise use SWAR below */
+ if (MEM_isLittleEndian()) {
+ return ZSTD_row_getNEONMask(rowEntries, src, tag, headGrouped);
+ }
+# endif /* ZSTD_ARCH_ARM_NEON */
+ /* SWAR */
+ { const int chunkSize = sizeof(size_t);
+ const size_t shiftAmount = ((chunkSize * 8) - chunkSize);
+ const size_t xFF = ~((size_t)0);
+ const size_t x01 = xFF / 0xFF;
+ const size_t x80 = x01 << 7;
+ const size_t splatChar = tag * x01;
+ ZSTD_VecMask matches = 0;
+ int i = rowEntries - chunkSize;
+ assert((sizeof(size_t) == 4) || (sizeof(size_t) == 8));
+ if (MEM_isLittleEndian()) { /* runtime check so have two loops */
+ const size_t extractMagic = (xFF / 0x7F) >> chunkSize;
+ do {
+ size_t chunk = MEM_readST(&src[i]);
+ chunk ^= splatChar;
+ chunk = (((chunk | x80) - x01) | chunk) & x80;
+ matches <<= chunkSize;
+ matches |= (chunk * extractMagic) >> shiftAmount;
+ i -= chunkSize;
+ } while (i >= 0);
+ } else { /* big endian: reverse bits during extraction */
+ const size_t msb = xFF ^ (xFF >> 1);
+ const size_t extractMagic = (msb / 0x1FF) | msb;
+ do {
+ size_t chunk = MEM_readST(&src[i]);
+ chunk ^= splatChar;
+ chunk = (((chunk | x80) - x01) | chunk) & x80;
+ matches <<= chunkSize;
+ matches |= ((chunk >> 7) * extractMagic) >> shiftAmount;
+ i -= chunkSize;
+ } while (i >= 0);
+ }
+ matches = ~matches;
+ if (rowEntries == 16) {
+ return ZSTD_rotateRight_U16((U16)matches, headGrouped);
+ } else if (rowEntries == 32) {
+ return ZSTD_rotateRight_U32((U32)matches, headGrouped);
+ } else {
+ return ZSTD_rotateRight_U64((U64)matches, headGrouped);
+ }
+ }
+#endif
+}
+
+/* The high-level approach of the SIMD row based match finder is as follows:
+ * - Figure out where to insert the new entry:
+ * - Generate a hash from a byte along with an additional 1-byte "short hash". The additional byte is our "tag"
+ * - The hashTable is effectively split into groups or "rows" of 16 or 32 entries of U32, and the hash determines
+ * which row to insert into.
+ * - Determine the correct position within the row to insert the entry into. Each row of 16 or 32 can
+ * be considered as a circular buffer with a "head" index that resides in the tagTable.
+ * - Also insert the "tag" into the equivalent row and position in the tagTable.
+ * - Note: The tagTable has 17 or 33 1-byte entries per row, due to 16 or 32 tags, and 1 "head" entry.
+ * The 17 or 33 entry rows are spaced out to occur every 32 or 64 bytes, respectively,
+ * for alignment/performance reasons, leaving some bytes unused.
+ * - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte "short hash" and
+ * generate a bitfield that we can cycle through to check the collisions in the hash table.
+ * - Pick the longest match.
+ */
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_RowFindBestMatch(
+ ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iLimit,
+ size_t* offsetPtr,
+ const U32 mls, const ZSTD_dictMode_e dictMode,
+ const U32 rowLog)
+{
+ U32* const hashTable = ms->hashTable;
+ U16* const tagTable = ms->tagTable;
+ U32* const hashCache = ms->hashCache;
+ const U32 hashLog = ms->rowHashLog;
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const U32 curr = (U32)(ip-base);
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 lowestValid = ms->window.lowLimit;
+ const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
+ const U32 isDictionary = (ms->loadedDictEnd != 0);
+ const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance;
+ const U32 rowEntries = (1U << rowLog);
+ const U32 rowMask = rowEntries - 1;
+ const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */
+ const U32 groupWidth = ZSTD_row_matchMaskGroupWidth(rowEntries);
+ U32 nbAttempts = 1U << cappedSearchLog;
+ size_t ml=4-1;
+
+ /* DMS/DDS variables that may be referenced laster */
+ const ZSTD_matchState_t* const dms = ms->dictMatchState;
+
+ /* Initialize the following variables to satisfy static analyzer */
+ size_t ddsIdx = 0;
+ U32 ddsExtraAttempts = 0; /* cctx hash tables are limited in searches, but allow extra searches into DDS */
+ U32 dmsTag = 0;
+ U32* dmsRow = NULL;
+ BYTE* dmsTagRow = NULL;
+
+ if (dictMode == ZSTD_dedicatedDictSearch) {
+ const U32 ddsHashLog = dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG;
+ { /* Prefetch DDS hashtable entry */
+ ddsIdx = ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG;
+ PREFETCH_L1(&dms->hashTable[ddsIdx]);
+ }
+ ddsExtraAttempts = cParams->searchLog > rowLog ? 1U << (cParams->searchLog - rowLog) : 0;
+ }
+
+ if (dictMode == ZSTD_dictMatchState) {
+ /* Prefetch DMS rows */
+ U32* const dmsHashTable = dms->hashTable;
+ U16* const dmsTagTable = dms->tagTable;
+ U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
+ U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
+ dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK;
+ dmsTagRow = (BYTE*)(dmsTagTable + dmsRelRow);
+ dmsRow = dmsHashTable + dmsRelRow;
+ ZSTD_row_prefetch(dmsHashTable, dmsTagTable, dmsRelRow, rowLog);
+ }
+
+ /* Update the hashTable and tagTable up to (but not including) ip */
+ ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */);
+ { /* Get the hash for ip, compute the appropriate row */
+ U32 const hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls);
+ U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
+ U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK;
+ U32* const row = hashTable + relRow;
+ BYTE* tagRow = (BYTE*)(tagTable + relRow);
+ U32 const headGrouped = (*tagRow & rowMask) * groupWidth;
+ U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES];
+ size_t numMatches = 0;
+ size_t currMatch = 0;
+ ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, headGrouped, rowEntries);
+
+ /* Cycle through the matches and prefetch */
+ for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) {
+ U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask;
+ U32 const matchIndex = row[matchPos];
+ assert(numMatches < rowEntries);
+ if (matchIndex < lowLimit)
+ break;
+ if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
+ PREFETCH_L1(base + matchIndex);
+ } else {
+ PREFETCH_L1(dictBase + matchIndex);
+ }
+ matchBuffer[numMatches++] = matchIndex;
+ }
+
+ /* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop
+ in ZSTD_row_update_internal() at the next search. */
+ {
+ U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
+ tagRow[pos + ZSTD_ROW_HASH_TAG_OFFSET] = (BYTE)tag;
+ row[pos] = ms->nextToUpdate++;
+ }
+
+ /* Return the longest match */
+ for (; currMatch < numMatches; ++currMatch) {
+ U32 const matchIndex = matchBuffer[currMatch];
+ size_t currentMl=0;
+ assert(matchIndex < curr);
+ assert(matchIndex >= lowLimit);
+
+ if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
+ const BYTE* const match = base + matchIndex;
+ assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
+ /* read 4B starting from (match + ml + 1 - sizeof(U32)) */
+ if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */
+ currentMl = ZSTD_count(ip, match, iLimit);
+ } else {
+ const BYTE* const match = dictBase + matchIndex;
+ assert(match+4 <= dictEnd);
+ if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4;
+ }
+
+ /* Save best solution */
+ if (currentMl > ml) {
+ ml = currentMl;
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
+ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
+ }
+ }
+ }
+
+ assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
+ if (dictMode == ZSTD_dedicatedDictSearch) {
+ ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms,
+ ip, iLimit, prefixStart, curr, dictLimit, ddsIdx);
+ } else if (dictMode == ZSTD_dictMatchState) {
+ /* TODO: Measure and potentially add prefetching to DMS */
+ const U32 dmsLowestIndex = dms->window.dictLimit;
+ const BYTE* const dmsBase = dms->window.base;
+ const BYTE* const dmsEnd = dms->window.nextSrc;
+ const U32 dmsSize = (U32)(dmsEnd - dmsBase);
+ const U32 dmsIndexDelta = dictLimit - dmsSize;
+
+ { U32 const headGrouped = (*dmsTagRow & rowMask) * groupWidth;
+ U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES];
+ size_t numMatches = 0;
+ size_t currMatch = 0;
+ ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, headGrouped, rowEntries);
+
+ for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) {
+ U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask;
+ U32 const matchIndex = dmsRow[matchPos];
+ if (matchIndex < dmsLowestIndex)
+ break;
+ PREFETCH_L1(dmsBase + matchIndex);
+ matchBuffer[numMatches++] = matchIndex;
+ }
+
+ /* Return the longest match */
+ for (; currMatch < numMatches; ++currMatch) {
+ U32 const matchIndex = matchBuffer[currMatch];
+ size_t currentMl=0;
+ assert(matchIndex >= dmsLowestIndex);
+ assert(matchIndex < curr);
+
+ { const BYTE* const match = dmsBase + matchIndex;
+ assert(match+4 <= dmsEnd);
+ if (MEM_read32(match) == MEM_read32(ip))
+ currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4;
+ }
+
+ if (currentMl > ml) {
+ ml = currentMl;
+ assert(curr > matchIndex + dmsIndexDelta);
+ *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta));
+ if (ip+currentMl == iLimit) break;
+ }
+ }
+ }
+ }
+ return ml;
+}
+
+
+/**
+ * Generate search functions templated on (dictMode, mls, rowLog).
+ * These functions are outlined for code size & compilation time.
+ * ZSTD_searchMax() dispatches to the correct implementation function.
+ *
+ * TODO: The start of the search function involves loading and calculating a
+ * bunch of constants from the ZSTD_matchState_t. These computations could be
+ * done in an initialization function, and saved somewhere in the match state.
+ * Then we could pass a pointer to the saved state instead of the match state,
+ * and avoid duplicate computations.
+ *
+ * TODO: Move the match re-winding into searchMax. This improves compression
+ * ratio, and unlocks further simplifications with the next TODO.
+ *
+ * TODO: Try moving the repcode search into searchMax. After the re-winding
+ * and repcode search are in searchMax, there is no more logic in the match
+ * finder loop that requires knowledge about the dictMode. So we should be
+ * able to avoid force inlining it, and we can join the extDict loop with
+ * the single segment loop. It should go in searchMax instead of its own
+ * function to avoid having multiple virtual function calls per search.
+ */
+
+#define ZSTD_BT_SEARCH_FN(dictMode, mls) ZSTD_BtFindBestMatch_##dictMode##_##mls
+#define ZSTD_HC_SEARCH_FN(dictMode, mls) ZSTD_HcFindBestMatch_##dictMode##_##mls
+#define ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog
+
+#define ZSTD_SEARCH_FN_ATTRS FORCE_NOINLINE
+
+#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls) \
+ ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)( \
+ ZSTD_matchState_t* ms, \
+ const BYTE* ip, const BYTE* const iLimit, \
+ size_t* offBasePtr) \
+ { \
+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \
+ return ZSTD_BtFindBestMatch(ms, ip, iLimit, offBasePtr, mls, ZSTD_##dictMode); \
+ } \
+
+#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls) \
+ ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)( \
+ ZSTD_matchState_t* ms, \
+ const BYTE* ip, const BYTE* const iLimit, \
+ size_t* offsetPtr) \
+ { \
+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \
+ return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \
+ } \
+
+#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) \
+ ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)( \
+ ZSTD_matchState_t* ms, \
+ const BYTE* ip, const BYTE* const iLimit, \
+ size_t* offsetPtr) \
+ { \
+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \
+ assert(MAX(4, MIN(6, ms->cParams.searchLog)) == rowLog); \
+ return ZSTD_RowFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode, rowLog); \
+ } \
+
+#define ZSTD_FOR_EACH_ROWLOG(X, dictMode, mls) \
+ X(dictMode, mls, 4) \
+ X(dictMode, mls, 5) \
+ X(dictMode, mls, 6)
+
+#define ZSTD_FOR_EACH_MLS_ROWLOG(X, dictMode) \
+ ZSTD_FOR_EACH_ROWLOG(X, dictMode, 4) \
+ ZSTD_FOR_EACH_ROWLOG(X, dictMode, 5) \
+ ZSTD_FOR_EACH_ROWLOG(X, dictMode, 6)
+
+#define ZSTD_FOR_EACH_MLS(X, dictMode) \
+ X(dictMode, 4) \
+ X(dictMode, 5) \
+ X(dictMode, 6)
+
+#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \
+ X(__VA_ARGS__, noDict) \
+ X(__VA_ARGS__, extDict) \
+ X(__VA_ARGS__, dictMatchState) \
+ X(__VA_ARGS__, dedicatedDictSearch)
+
+/* Generate row search fns for each combination of (dictMode, mls, rowLog) */
+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_SEARCH_FN)
+/* Generate binary Tree search fns for each combination of (dictMode, mls) */
+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_SEARCH_FN)
+/* Generate hash chain search fns for each combination of (dictMode, mls) */
+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_SEARCH_FN)
+
+typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e;
+
+#define GEN_ZSTD_CALL_BT_SEARCH_FN(dictMode, mls) \
+ case mls: \
+ return ZSTD_BT_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr);
+#define GEN_ZSTD_CALL_HC_SEARCH_FN(dictMode, mls) \
+ case mls: \
+ return ZSTD_HC_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr);
+#define GEN_ZSTD_CALL_ROW_SEARCH_FN(dictMode, mls, rowLog) \
+ case rowLog: \
+ return ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)(ms, ip, iend, offsetPtr);
+
+#define ZSTD_SWITCH_MLS(X, dictMode) \
+ switch (mls) { \
+ ZSTD_FOR_EACH_MLS(X, dictMode) \
+ }
+
+#define ZSTD_SWITCH_ROWLOG(dictMode, mls) \
+ case mls: \
+ switch (rowLog) { \
+ ZSTD_FOR_EACH_ROWLOG(GEN_ZSTD_CALL_ROW_SEARCH_FN, dictMode, mls) \
+ } \
+ ZSTD_UNREACHABLE; \
+ break;
+
+#define ZSTD_SWITCH_SEARCH_METHOD(dictMode) \
+ switch (searchMethod) { \
+ case search_hashChain: \
+ ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_HC_SEARCH_FN, dictMode) \
+ break; \
+ case search_binaryTree: \
+ ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_BT_SEARCH_FN, dictMode) \
+ break; \
+ case search_rowHash: \
+ ZSTD_SWITCH_MLS(ZSTD_SWITCH_ROWLOG, dictMode) \
+ break; \
+ } \
+ ZSTD_UNREACHABLE;
+
+/**
+ * Searches for the longest match at @p ip.
+ * Dispatches to the correct implementation function based on the
+ * (searchMethod, dictMode, mls, rowLog). We use switch statements
+ * here instead of using an indirect function call through a function
+ * pointer because after Spectre and Meltdown mitigations, indirect
+ * function calls can be very costly, especially in the kernel.
+ *
+ * NOTE: dictMode and searchMethod should be templated, so those switch
+ * statements should be optimized out. Only the mls & rowLog switches
+ * should be left.
+ *
+ * @param ms The match state.
+ * @param ip The position to search at.
+ * @param iend The end of the input data.
+ * @param[out] offsetPtr Stores the match offset into this pointer.
+ * @param mls The minimum search length, in the range [4, 6].
+ * @param rowLog The row log (if applicable), in the range [4, 6].
+ * @param searchMethod The search method to use (templated).
+ * @param dictMode The dictMode (templated).
+ *
+ * @returns The length of the longest match found, or < mls if no match is found.
+ * If a match is found its offset is stored in @p offsetPtr.
+ */
+FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax(
+ ZSTD_matchState_t* ms,
+ const BYTE* ip,
+ const BYTE* iend,
+ size_t* offsetPtr,
+ U32 const mls,
+ U32 const rowLog,
+ searchMethod_e const searchMethod,
+ ZSTD_dictMode_e const dictMode)
+{
+ if (dictMode == ZSTD_noDict) {
+ ZSTD_SWITCH_SEARCH_METHOD(noDict)
+ } else if (dictMode == ZSTD_extDict) {
+ ZSTD_SWITCH_SEARCH_METHOD(extDict)
+ } else if (dictMode == ZSTD_dictMatchState) {
+ ZSTD_SWITCH_SEARCH_METHOD(dictMatchState)
+ } else if (dictMode == ZSTD_dedicatedDictSearch) {
+ ZSTD_SWITCH_SEARCH_METHOD(dedicatedDictSearch)
+ }
+ ZSTD_UNREACHABLE;
+ return 0;
+}
+
+/* *******************************
+* Common parser - lazy strategy
+*********************************/
+
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_compressBlock_lazy_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore,
+ U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize,
+ const searchMethod_e searchMethod, const U32 depth,
+ ZSTD_dictMode_e const dictMode)
+{
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip = istart;
+ const BYTE* anchor = istart;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = (searchMethod == search_rowHash) ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8;
+ const BYTE* const base = ms->window.base;
+ const U32 prefixLowestIndex = ms->window.dictLimit;
+ const BYTE* const prefixLowest = base + prefixLowestIndex;
+ const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6);
+ const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
+
+ U32 offset_1 = rep[0], offset_2 = rep[1];
+ U32 offsetSaved1 = 0, offsetSaved2 = 0;
+
+ const int isDMS = dictMode == ZSTD_dictMatchState;
+ const int isDDS = dictMode == ZSTD_dedicatedDictSearch;
+ const int isDxS = isDMS || isDDS;
+ const ZSTD_matchState_t* const dms = ms->dictMatchState;
+ const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0;
+ const BYTE* const dictBase = isDxS ? dms->window.base : NULL;
+ const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL;
+ const BYTE* const dictEnd = isDxS ? dms->window.nextSrc : NULL;
+ const U32 dictIndexDelta = isDxS ?
+ prefixLowestIndex - (U32)(dictEnd - dictBase) :
+ 0;
+ const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest));
+
+ DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod);
+ ip += (dictAndPrefixLength == 0);
+ if (dictMode == ZSTD_noDict) {
+ U32 const curr = (U32)(ip - base);
+ U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog);
+ U32 const maxRep = curr - windowLow;
+ if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0;
+ if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0;
+ }
+ if (isDxS) {
+ /* dictMatchState repCode checks don't currently handle repCode == 0
+ * disabling. */
+ assert(offset_1 <= dictAndPrefixLength);
+ assert(offset_2 <= dictAndPrefixLength);
+ }
+
+ if (searchMethod == search_rowHash) {
+ ZSTD_row_fillHashCache(ms, base, rowLog,
+ MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */),
+ ms->nextToUpdate, ilimit);
+ }
+
+ /* Match Loop */
+#if defined(__GNUC__) && defined(__x86_64__)
+ /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the
+ * code alignment is perturbed. To fix the instability align the loop on 32-bytes.
+ */
+ __asm__(".p2align 5");
+#endif
+ while (ip < ilimit) {
+ size_t matchLength=0;
+ size_t offBase = REPCODE1_TO_OFFBASE;
+ const BYTE* start=ip+1;
+ DEBUGLOG(7, "search baseline (depth 0)");
+
+ /* check repCode */
+ if (isDxS) {
+ const U32 repIndex = (U32)(ip - base) + 1 - offset_1;
+ const BYTE* repMatch = ((dictMode == ZSTD_dictMatchState || dictMode == ZSTD_dedicatedDictSearch)
+ && repIndex < prefixLowestIndex) ?
+ dictBase + (repIndex - dictIndexDelta) :
+ base + repIndex;
+ if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
+ const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
+ matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
+ if (depth==0) goto _storeSequence;
+ }
+ }
+ if ( dictMode == ZSTD_noDict
+ && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) {
+ matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
+ if (depth==0) goto _storeSequence;
+ }
+
+ /* first search (depth 0) */
+ { size_t offbaseFound = 999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offbaseFound, mls, rowLog, searchMethod, dictMode);
+ if (ml2 > matchLength)
+ matchLength = ml2, start = ip, offBase = offbaseFound;
+ }
+
+ if (matchLength < 4) {
+ ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */
+ continue;
+ }
+
+ /* let's try to find a better solution */
+ if (depth>=1)
+ while (ip<ilimit) {
+ DEBUGLOG(7, "search depth 1");
+ ip ++;
+ if ( (dictMode == ZSTD_noDict)
+ && (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
+ size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
+ int const gain2 = (int)(mlRep * 3);
+ int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((mlRep >= 4) && (gain2 > gain1))
+ matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ }
+ if (isDxS) {
+ const U32 repIndex = (U32)(ip - base) - offset_1;
+ const BYTE* repMatch = repIndex < prefixLowestIndex ?
+ dictBase + (repIndex - dictIndexDelta) :
+ base + repIndex;
+ if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+ && (MEM_read32(repMatch) == MEM_read32(ip)) ) {
+ const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
+ size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
+ int const gain2 = (int)(mlRep * 3);
+ int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((mlRep >= 4) && (gain2 > gain1))
+ matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ }
+ }
+ { size_t ofbCandidate=999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode);
+ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4);
+ if ((ml2 >= 4) && (gain2 > gain1)) {
+ matchLength = ml2, offBase = ofbCandidate, start = ip;
+ continue; /* search a better one */
+ } }
+
+ /* let's find an even better one */
+ if ((depth==2) && (ip<ilimit)) {
+ DEBUGLOG(7, "search depth 2");
+ ip ++;
+ if ( (dictMode == ZSTD_noDict)
+ && (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
+ size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
+ int const gain2 = (int)(mlRep * 4);
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((mlRep >= 4) && (gain2 > gain1))
+ matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ }
+ if (isDxS) {
+ const U32 repIndex = (U32)(ip - base) - offset_1;
+ const BYTE* repMatch = repIndex < prefixLowestIndex ?
+ dictBase + (repIndex - dictIndexDelta) :
+ base + repIndex;
+ if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
+ && (MEM_read32(repMatch) == MEM_read32(ip)) ) {
+ const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
+ size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
+ int const gain2 = (int)(mlRep * 4);
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((mlRep >= 4) && (gain2 > gain1))
+ matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ }
+ }
+ { size_t ofbCandidate=999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode);
+ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7);
+ if ((ml2 >= 4) && (gain2 > gain1)) {
+ matchLength = ml2, offBase = ofbCandidate, start = ip;
+ continue;
+ } } }
+ break; /* nothing found : store previous solution */
+ }
+
+ /* NOTE:
+ * Pay attention that `start[-value]` can lead to strange undefined behavior
+ * notably if `value` is unsigned, resulting in a large positive `-value`.
+ */
+ /* catch up */
+ if (OFFBASE_IS_OFFSET(offBase)) {
+ if (dictMode == ZSTD_noDict) {
+ while ( ((start > anchor) & (start - OFFBASE_TO_OFFSET(offBase) > prefixLowest))
+ && (start[-1] == (start-OFFBASE_TO_OFFSET(offBase))[-1]) ) /* only search for offset within prefix */
+ { start--; matchLength++; }
+ }
+ if (isDxS) {
+ U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase));
+ const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex;
+ const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest;
+ while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
+ }
+ offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase);
+ }
+ /* store sequence */
+_storeSequence:
+ { size_t const litLength = (size_t)(start - anchor);
+ ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength);
+ anchor = ip = start + matchLength;
+ }
+
+ /* check immediate repcode */
+ if (isDxS) {
+ while (ip <= ilimit) {
+ U32 const current2 = (U32)(ip-base);
+ U32 const repIndex = current2 - offset_2;
+ const BYTE* repMatch = repIndex < prefixLowestIndex ?
+ dictBase - dictIndexDelta + repIndex :
+ base + repIndex;
+ if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */)
+ && (MEM_read32(repMatch) == MEM_read32(ip)) ) {
+ const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend;
+ matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4;
+ offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
+ ip += matchLength;
+ anchor = ip;
+ continue;
+ }
+ break;
+ }
+ }
+
+ if (dictMode == ZSTD_noDict) {
+ while ( ((ip <= ilimit) & (offset_2>0))
+ && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) {
+ /* store sequence */
+ matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
+ offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap repcodes */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
+ ip += matchLength;
+ anchor = ip;
+ continue; /* faster when present ... (?) */
+ } } }
+
+ /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
+ * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
+ offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
+
+ /* save reps for next block */
+ rep[0] = offset_1 ? offset_1 : offsetSaved1;
+ rep[1] = offset_2 ? offset_2 : offsetSaved2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+
+size_t ZSTD_compressBlock_btlazy2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_lazy2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_lazy(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_greedy(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_btlazy2_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_lazy2_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_lazy_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_greedy_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState);
+}
+
+
+size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch);
+}
+
+size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch);
+}
+
+size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch);
+}
+
+/* Row-based matchfinder */
+size_t ZSTD_compressBlock_lazy2_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_lazy_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_greedy_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_lazy_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_greedy_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState);
+}
+
+
+size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch);
+}
+
+size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch);
+}
+
+size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch);
+}
+
+FORCE_INLINE_TEMPLATE
+size_t ZSTD_compressBlock_lazy_extDict_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore,
+ U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize,
+ const searchMethod_e searchMethod, const U32 depth)
+{
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip = istart;
+ const BYTE* anchor = istart;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = searchMethod == search_rowHash ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8;
+ const BYTE* const base = ms->window.base;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const BYTE* const dictStart = dictBase + ms->window.lowLimit;
+ const U32 windowLog = ms->cParams.windowLog;
+ const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6);
+ const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
+
+ U32 offset_1 = rep[0], offset_2 = rep[1];
+
+ DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod);
+
+ /* init */
+ ip += (ip == prefixStart);
+ if (searchMethod == search_rowHash) {
+ ZSTD_row_fillHashCache(ms, base, rowLog,
+ MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */),
+ ms->nextToUpdate, ilimit);
+ }
+
+ /* Match Loop */
+#if defined(__GNUC__) && defined(__x86_64__)
+ /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the
+ * code alignment is perturbed. To fix the instability align the loop on 32-bytes.
+ */
+ __asm__(".p2align 5");
+#endif
+ while (ip < ilimit) {
+ size_t matchLength=0;
+ size_t offBase = REPCODE1_TO_OFFBASE;
+ const BYTE* start=ip+1;
+ U32 curr = (U32)(ip-base);
+
+ /* check repCode */
+ { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr+1, windowLog);
+ const U32 repIndex = (U32)(curr+1 - offset_1);
+ const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE* const repMatch = repBase + repIndex;
+ if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
+ & (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */
+ if (MEM_read32(ip+1) == MEM_read32(repMatch)) {
+ /* repcode detected we should take it */
+ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4;
+ if (depth==0) goto _storeSequence;
+ } }
+
+ /* first search (depth 0) */
+ { size_t ofbCandidate = 999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
+ if (ml2 > matchLength)
+ matchLength = ml2, start = ip, offBase = ofbCandidate;
+ }
+
+ if (matchLength < 4) {
+ ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */
+ continue;
+ }
+
+ /* let's try to find a better solution */
+ if (depth>=1)
+ while (ip<ilimit) {
+ ip ++;
+ curr++;
+ /* check repCode */
+ if (offBase) {
+ const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
+ const U32 repIndex = (U32)(curr - offset_1);
+ const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE* const repMatch = repBase + repIndex;
+ if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
+ & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
+ if (MEM_read32(ip) == MEM_read32(repMatch)) {
+ /* repcode detected */
+ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
+ int const gain2 = (int)(repLength * 3);
+ int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((repLength >= 4) && (gain2 > gain1))
+ matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ } }
+
+ /* search match, depth 1 */
+ { size_t ofbCandidate = 999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
+ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4);
+ if ((ml2 >= 4) && (gain2 > gain1)) {
+ matchLength = ml2, offBase = ofbCandidate, start = ip;
+ continue; /* search a better one */
+ } }
+
+ /* let's find an even better one */
+ if ((depth==2) && (ip<ilimit)) {
+ ip ++;
+ curr++;
+ /* check repCode */
+ if (offBase) {
+ const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
+ const U32 repIndex = (U32)(curr - offset_1);
+ const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE* const repMatch = repBase + repIndex;
+ if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
+ & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
+ if (MEM_read32(ip) == MEM_read32(repMatch)) {
+ /* repcode detected */
+ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
+ int const gain2 = (int)(repLength * 4);
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
+ if ((repLength >= 4) && (gain2 > gain1))
+ matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip;
+ } }
+
+ /* search match, depth 2 */
+ { size_t ofbCandidate = 999999999;
+ size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
+ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
+ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7);
+ if ((ml2 >= 4) && (gain2 > gain1)) {
+ matchLength = ml2, offBase = ofbCandidate, start = ip;
+ continue;
+ } } }
+ break; /* nothing found : store previous solution */
+ }
+
+ /* catch up */
+ if (OFFBASE_IS_OFFSET(offBase)) {
+ U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase));
+ const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
+ const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
+ while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
+ offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase);
+ }
+
+ /* store sequence */
+_storeSequence:
+ { size_t const litLength = (size_t)(start - anchor);
+ ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength);
+ anchor = ip = start + matchLength;
+ }
+
+ /* check immediate repcode */
+ while (ip <= ilimit) {
+ const U32 repCurrent = (U32)(ip-base);
+ const U32 windowLow = ZSTD_getLowestMatchIndex(ms, repCurrent, windowLog);
+ const U32 repIndex = repCurrent - offset_2;
+ const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE* const repMatch = repBase + repIndex;
+ if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
+ & (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
+ if (MEM_read32(ip) == MEM_read32(repMatch)) {
+ /* repcode detected we should take it */
+ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
+ offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset history */
+ ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
+ ip += matchLength;
+ anchor = ip;
+ continue; /* faster when present ... (?) */
+ }
+ break;
+ } }
+
+ /* Save reps for next block */
+ rep[0] = offset_1;
+ rep[1] = offset_2;
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+
+size_t ZSTD_compressBlock_greedy_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0);
+}
+
+size_t ZSTD_compressBlock_lazy_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1);
+}
+
+size_t ZSTD_compressBlock_lazy2_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2);
+}
+
+size_t ZSTD_compressBlock_btlazy2_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2);
+}
+
+size_t ZSTD_compressBlock_greedy_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0);
+}
+
+size_t ZSTD_compressBlock_lazy_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1);
+}
+
+size_t ZSTD_compressBlock_lazy2_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2);
+}
diff --git a/contrib/zstd/zstd_lazy.h b/contrib/zstd/zstd_lazy.h
new file mode 100644
index 0000000..3bde673
--- /dev/null
+++ b/contrib/zstd/zstd_lazy.h
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_LAZY_H
+#define ZSTD_LAZY_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include "zstd_compress_internal.h"
+
+/**
+ * Dedicated Dictionary Search Structure bucket log. In the
+ * ZSTD_dedicatedDictSearch mode, the hashTable has
+ * 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just
+ * one.
+ */
+#define ZSTD_LAZY_DDSS_BUCKET_LOG 2
+
+#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */
+
+U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
+void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
+
+void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip);
+
+void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */
+
+size_t ZSTD_compressBlock_btlazy2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+size_t ZSTD_compressBlock_btlazy2_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_dictMatchState_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+size_t ZSTD_compressBlock_greedy_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_greedy_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_lazy2_extDict_row(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_btlazy2_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_LAZY_H */
diff --git a/contrib/zstd/zstd_ldm.c b/contrib/zstd/zstd_ldm.c
new file mode 100644
index 0000000..b8dcbca
--- /dev/null
+++ b/contrib/zstd/zstd_ldm.c
@@ -0,0 +1,724 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_ldm.h"
+
+#include "debug.h"
+#include "xxhash.h"
+#include "zstd_fast.h" /* ZSTD_fillHashTable() */
+#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */
+#include "zstd_ldm_geartab.h"
+
+#define LDM_BUCKET_SIZE_LOG 3
+#define LDM_MIN_MATCH_LENGTH 64
+#define LDM_HASH_RLOG 7
+
+typedef struct {
+ U64 rolling;
+ U64 stopMask;
+} ldmRollingHashState_t;
+
+/** ZSTD_ldm_gear_init():
+ *
+ * Initializes the rolling hash state such that it will honor the
+ * settings in params. */
+static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params)
+{
+ unsigned maxBitsInMask = MIN(params->minMatchLength, 64);
+ unsigned hashRateLog = params->hashRateLog;
+
+ state->rolling = ~(U32)0;
+
+ /* The choice of the splitting criterion is subject to two conditions:
+ * 1. it has to trigger on average every 2^(hashRateLog) bytes;
+ * 2. ideally, it has to depend on a window of minMatchLength bytes.
+ *
+ * In the gear hash algorithm, bit n depends on the last n bytes;
+ * so in order to obtain a good quality splitting criterion it is
+ * preferable to use bits with high weight.
+ *
+ * To match condition 1 we use a mask with hashRateLog bits set
+ * and, because of the previous remark, we make sure these bits
+ * have the highest possible weight while still respecting
+ * condition 2.
+ */
+ if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) {
+ state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog);
+ } else {
+ /* In this degenerate case we simply honor the hash rate. */
+ state->stopMask = ((U64)1 << hashRateLog) - 1;
+ }
+}
+
+/** ZSTD_ldm_gear_reset()
+ * Feeds [data, data + minMatchLength) into the hash without registering any
+ * splits. This effectively resets the hash state. This is used when skipping
+ * over data, either at the beginning of a block, or skipping sections.
+ */
+static void ZSTD_ldm_gear_reset(ldmRollingHashState_t* state,
+ BYTE const* data, size_t minMatchLength)
+{
+ U64 hash = state->rolling;
+ size_t n = 0;
+
+#define GEAR_ITER_ONCE() do { \
+ hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \
+ n += 1; \
+ } while (0)
+ while (n + 3 < minMatchLength) {
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ }
+ while (n < minMatchLength) {
+ GEAR_ITER_ONCE();
+ }
+#undef GEAR_ITER_ONCE
+}
+
+/** ZSTD_ldm_gear_feed():
+ *
+ * Registers in the splits array all the split points found in the first
+ * size bytes following the data pointer. This function terminates when
+ * either all the data has been processed or LDM_BATCH_SIZE splits are
+ * present in the splits array.
+ *
+ * Precondition: The splits array must not be full.
+ * Returns: The number of bytes processed. */
+static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state,
+ BYTE const* data, size_t size,
+ size_t* splits, unsigned* numSplits)
+{
+ size_t n;
+ U64 hash, mask;
+
+ hash = state->rolling;
+ mask = state->stopMask;
+ n = 0;
+
+#define GEAR_ITER_ONCE() do { \
+ hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \
+ n += 1; \
+ if (UNLIKELY((hash & mask) == 0)) { \
+ splits[*numSplits] = n; \
+ *numSplits += 1; \
+ if (*numSplits == LDM_BATCH_SIZE) \
+ goto done; \
+ } \
+ } while (0)
+
+ while (n + 3 < size) {
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ GEAR_ITER_ONCE();
+ }
+ while (n < size) {
+ GEAR_ITER_ONCE();
+ }
+
+#undef GEAR_ITER_ONCE
+
+done:
+ state->rolling = hash;
+ return n;
+}
+
+void ZSTD_ldm_adjustParameters(ldmParams_t* params,
+ ZSTD_compressionParameters const* cParams)
+{
+ params->windowLog = cParams->windowLog;
+ ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
+ DEBUGLOG(4, "ZSTD_ldm_adjustParameters");
+ if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
+ if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH;
+ if (params->hashLog == 0) {
+ params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
+ assert(params->hashLog <= ZSTD_HASHLOG_MAX);
+ }
+ if (params->hashRateLog == 0) {
+ params->hashRateLog = params->windowLog < params->hashLog
+ ? 0
+ : params->windowLog - params->hashLog;
+ }
+ params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog);
+}
+
+size_t ZSTD_ldm_getTableSize(ldmParams_t params)
+{
+ size_t const ldmHSize = ((size_t)1) << params.hashLog;
+ size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog);
+ size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog);
+ size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize)
+ + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t));
+ return params.enableLdm == ZSTD_ps_enable ? totalSize : 0;
+}
+
+size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize)
+{
+ return params.enableLdm == ZSTD_ps_enable ? (maxChunkSize / params.minMatchLength) : 0;
+}
+
+/** ZSTD_ldm_getBucket() :
+ * Returns a pointer to the start of the bucket associated with hash. */
+static ldmEntry_t* ZSTD_ldm_getBucket(
+ ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams)
+{
+ return ldmState->hashTable + (hash << ldmParams.bucketSizeLog);
+}
+
+/** ZSTD_ldm_insertEntry() :
+ * Insert the entry with corresponding hash into the hash table */
+static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
+ size_t const hash, const ldmEntry_t entry,
+ ldmParams_t const ldmParams)
+{
+ BYTE* const pOffset = ldmState->bucketOffsets + hash;
+ unsigned const offset = *pOffset;
+
+ *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry;
+ *pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1));
+
+}
+
+/** ZSTD_ldm_countBackwardsMatch() :
+ * Returns the number of bytes that match backwards before pIn and pMatch.
+ *
+ * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */
+static size_t ZSTD_ldm_countBackwardsMatch(
+ const BYTE* pIn, const BYTE* pAnchor,
+ const BYTE* pMatch, const BYTE* pMatchBase)
+{
+ size_t matchLength = 0;
+ while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) {
+ pIn--;
+ pMatch--;
+ matchLength++;
+ }
+ return matchLength;
+}
+
+/** ZSTD_ldm_countBackwardsMatch_2segments() :
+ * Returns the number of bytes that match backwards from pMatch,
+ * even with the backwards match spanning 2 different segments.
+ *
+ * On reaching `pMatchBase`, start counting from mEnd */
+static size_t ZSTD_ldm_countBackwardsMatch_2segments(
+ const BYTE* pIn, const BYTE* pAnchor,
+ const BYTE* pMatch, const BYTE* pMatchBase,
+ const BYTE* pExtDictStart, const BYTE* pExtDictEnd)
+{
+ size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase);
+ if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) {
+ /* If backwards match is entirely in the extDict or prefix, immediately return */
+ return matchLength;
+ }
+ DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength);
+ matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart);
+ DEBUGLOG(7, "final backwards match length = %zu", matchLength);
+ return matchLength;
+}
+
+/** ZSTD_ldm_fillFastTables() :
+ *
+ * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies.
+ * This is similar to ZSTD_loadDictionaryContent.
+ *
+ * The tables for the other strategies are filled within their
+ * block compressors. */
+static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
+ void const* end)
+{
+ const BYTE* const iend = (const BYTE*)end;
+
+ switch(ms->cParams.strategy)
+ {
+ case ZSTD_fast:
+ ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx);
+ break;
+
+ case ZSTD_dfast:
+ ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx);
+ break;
+
+ case ZSTD_greedy:
+ case ZSTD_lazy:
+ case ZSTD_lazy2:
+ case ZSTD_btlazy2:
+ case ZSTD_btopt:
+ case ZSTD_btultra:
+ case ZSTD_btultra2:
+ break;
+ default:
+ assert(0); /* not possible : not a valid strategy id */
+ }
+
+ return 0;
+}
+
+void ZSTD_ldm_fillHashTable(
+ ldmState_t* ldmState, const BYTE* ip,
+ const BYTE* iend, ldmParams_t const* params)
+{
+ U32 const minMatchLength = params->minMatchLength;
+ U32 const hBits = params->hashLog - params->bucketSizeLog;
+ BYTE const* const base = ldmState->window.base;
+ BYTE const* const istart = ip;
+ ldmRollingHashState_t hashState;
+ size_t* const splits = ldmState->splitIndices;
+ unsigned numSplits;
+
+ DEBUGLOG(5, "ZSTD_ldm_fillHashTable");
+
+ ZSTD_ldm_gear_init(&hashState, params);
+ while (ip < iend) {
+ size_t hashed;
+ unsigned n;
+
+ numSplits = 0;
+ hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits);
+
+ for (n = 0; n < numSplits; n++) {
+ if (ip + splits[n] >= istart + minMatchLength) {
+ BYTE const* const split = ip + splits[n] - minMatchLength;
+ U64 const xxhash = XXH64(split, minMatchLength, 0);
+ U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1));
+ ldmEntry_t entry;
+
+ entry.offset = (U32)(split - base);
+ entry.checksum = (U32)(xxhash >> 32);
+ ZSTD_ldm_insertEntry(ldmState, hash, entry, *params);
+ }
+ }
+
+ ip += hashed;
+ }
+}
+
+
+/** ZSTD_ldm_limitTableUpdate() :
+ *
+ * Sets cctx->nextToUpdate to a position corresponding closer to anchor
+ * if it is far way
+ * (after a long match, only update tables a limited amount). */
+static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
+{
+ U32 const curr = (U32)(anchor - ms->window.base);
+ if (curr > ms->nextToUpdate + 1024) {
+ ms->nextToUpdate =
+ curr - MIN(512, curr - ms->nextToUpdate - 1024);
+ }
+}
+
+static size_t ZSTD_ldm_generateSequences_internal(
+ ldmState_t* ldmState, rawSeqStore_t* rawSeqStore,
+ ldmParams_t const* params, void const* src, size_t srcSize)
+{
+ /* LDM parameters */
+ int const extDict = ZSTD_window_hasExtDict(ldmState->window);
+ U32 const minMatchLength = params->minMatchLength;
+ U32 const entsPerBucket = 1U << params->bucketSizeLog;
+ U32 const hBits = params->hashLog - params->bucketSizeLog;
+ /* Prefix and extDict parameters */
+ U32 const dictLimit = ldmState->window.dictLimit;
+ U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
+ BYTE const* const base = ldmState->window.base;
+ BYTE const* const dictBase = extDict ? ldmState->window.dictBase : NULL;
+ BYTE const* const dictStart = extDict ? dictBase + lowestIndex : NULL;
+ BYTE const* const dictEnd = extDict ? dictBase + dictLimit : NULL;
+ BYTE const* const lowPrefixPtr = base + dictLimit;
+ /* Input bounds */
+ BYTE const* const istart = (BYTE const*)src;
+ BYTE const* const iend = istart + srcSize;
+ BYTE const* const ilimit = iend - HASH_READ_SIZE;
+ /* Input positions */
+ BYTE const* anchor = istart;
+ BYTE const* ip = istart;
+ /* Rolling hash state */
+ ldmRollingHashState_t hashState;
+ /* Arrays for staged-processing */
+ size_t* const splits = ldmState->splitIndices;
+ ldmMatchCandidate_t* const candidates = ldmState->matchCandidates;
+ unsigned numSplits;
+
+ if (srcSize < minMatchLength)
+ return iend - anchor;
+
+ /* Initialize the rolling hash state with the first minMatchLength bytes */
+ ZSTD_ldm_gear_init(&hashState, params);
+ ZSTD_ldm_gear_reset(&hashState, ip, minMatchLength);
+ ip += minMatchLength;
+
+ while (ip < ilimit) {
+ size_t hashed;
+ unsigned n;
+
+ numSplits = 0;
+ hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip,
+ splits, &numSplits);
+
+ for (n = 0; n < numSplits; n++) {
+ BYTE const* const split = ip + splits[n] - minMatchLength;
+ U64 const xxhash = XXH64(split, minMatchLength, 0);
+ U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1));
+
+ candidates[n].split = split;
+ candidates[n].hash = hash;
+ candidates[n].checksum = (U32)(xxhash >> 32);
+ candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params);
+ PREFETCH_L1(candidates[n].bucket);
+ }
+
+ for (n = 0; n < numSplits; n++) {
+ size_t forwardMatchLength = 0, backwardMatchLength = 0,
+ bestMatchLength = 0, mLength;
+ U32 offset;
+ BYTE const* const split = candidates[n].split;
+ U32 const checksum = candidates[n].checksum;
+ U32 const hash = candidates[n].hash;
+ ldmEntry_t* const bucket = candidates[n].bucket;
+ ldmEntry_t const* cur;
+ ldmEntry_t const* bestEntry = NULL;
+ ldmEntry_t newEntry;
+
+ newEntry.offset = (U32)(split - base);
+ newEntry.checksum = checksum;
+
+ /* If a split point would generate a sequence overlapping with
+ * the previous one, we merely register it in the hash table and
+ * move on */
+ if (split < anchor) {
+ ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
+ continue;
+ }
+
+ for (cur = bucket; cur < bucket + entsPerBucket; cur++) {
+ size_t curForwardMatchLength, curBackwardMatchLength,
+ curTotalMatchLength;
+ if (cur->checksum != checksum || cur->offset <= lowestIndex) {
+ continue;
+ }
+ if (extDict) {
+ BYTE const* const curMatchBase =
+ cur->offset < dictLimit ? dictBase : base;
+ BYTE const* const pMatch = curMatchBase + cur->offset;
+ BYTE const* const matchEnd =
+ cur->offset < dictLimit ? dictEnd : iend;
+ BYTE const* const lowMatchPtr =
+ cur->offset < dictLimit ? dictStart : lowPrefixPtr;
+ curForwardMatchLength =
+ ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr);
+ if (curForwardMatchLength < minMatchLength) {
+ continue;
+ }
+ curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments(
+ split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd);
+ } else { /* !extDict */
+ BYTE const* const pMatch = base + cur->offset;
+ curForwardMatchLength = ZSTD_count(split, pMatch, iend);
+ if (curForwardMatchLength < minMatchLength) {
+ continue;
+ }
+ curBackwardMatchLength =
+ ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr);
+ }
+ curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength;
+
+ if (curTotalMatchLength > bestMatchLength) {
+ bestMatchLength = curTotalMatchLength;
+ forwardMatchLength = curForwardMatchLength;
+ backwardMatchLength = curBackwardMatchLength;
+ bestEntry = cur;
+ }
+ }
+
+ /* No match found -- insert an entry into the hash table
+ * and process the next candidate match */
+ if (bestEntry == NULL) {
+ ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
+ continue;
+ }
+
+ /* Match found */
+ offset = (U32)(split - base) - bestEntry->offset;
+ mLength = forwardMatchLength + backwardMatchLength;
+ {
+ rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
+
+ /* Out of sequence storage */
+ if (rawSeqStore->size == rawSeqStore->capacity)
+ return ERROR(dstSize_tooSmall);
+ seq->litLength = (U32)(split - backwardMatchLength - anchor);
+ seq->matchLength = (U32)mLength;
+ seq->offset = offset;
+ rawSeqStore->size++;
+ }
+
+ /* Insert the current entry into the hash table --- it must be
+ * done after the previous block to avoid clobbering bestEntry */
+ ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
+
+ anchor = split + forwardMatchLength;
+
+ /* If we find a match that ends after the data that we've hashed
+ * then we have a repeating, overlapping, pattern. E.g. all zeros.
+ * If one repetition of the pattern matches our `stopMask` then all
+ * repetitions will. We don't need to insert them all into out table,
+ * only the first one. So skip over overlapping matches.
+ * This is a major speed boost (20x) for compressing a single byte
+ * repeated, when that byte ends up in the table.
+ */
+ if (anchor > ip + hashed) {
+ ZSTD_ldm_gear_reset(&hashState, anchor - minMatchLength, minMatchLength);
+ /* Continue the outer loop at anchor (ip + hashed == anchor). */
+ ip = anchor - hashed;
+ break;
+ }
+ }
+
+ ip += hashed;
+ }
+
+ return iend - anchor;
+}
+
+/*! ZSTD_ldm_reduceTable() :
+ * reduce table indexes by `reducerValue` */
+static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size,
+ U32 const reducerValue)
+{
+ U32 u;
+ for (u = 0; u < size; u++) {
+ if (table[u].offset < reducerValue) table[u].offset = 0;
+ else table[u].offset -= reducerValue;
+ }
+}
+
+size_t ZSTD_ldm_generateSequences(
+ ldmState_t* ldmState, rawSeqStore_t* sequences,
+ ldmParams_t const* params, void const* src, size_t srcSize)
+{
+ U32 const maxDist = 1U << params->windowLog;
+ BYTE const* const istart = (BYTE const*)src;
+ BYTE const* const iend = istart + srcSize;
+ size_t const kMaxChunkSize = 1 << 20;
+ size_t const nbChunks = (srcSize / kMaxChunkSize) + ((srcSize % kMaxChunkSize) != 0);
+ size_t chunk;
+ size_t leftoverSize = 0;
+
+ assert(ZSTD_CHUNKSIZE_MAX >= kMaxChunkSize);
+ /* Check that ZSTD_window_update() has been called for this chunk prior
+ * to passing it to this function.
+ */
+ assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize);
+ /* The input could be very large (in zstdmt), so it must be broken up into
+ * chunks to enforce the maximum distance and handle overflow correction.
+ */
+ assert(sequences->pos <= sequences->size);
+ assert(sequences->size <= sequences->capacity);
+ for (chunk = 0; chunk < nbChunks && sequences->size < sequences->capacity; ++chunk) {
+ BYTE const* const chunkStart = istart + chunk * kMaxChunkSize;
+ size_t const remaining = (size_t)(iend - chunkStart);
+ BYTE const *const chunkEnd =
+ (remaining < kMaxChunkSize) ? iend : chunkStart + kMaxChunkSize;
+ size_t const chunkSize = chunkEnd - chunkStart;
+ size_t newLeftoverSize;
+ size_t const prevSize = sequences->size;
+
+ assert(chunkStart < iend);
+ /* 1. Perform overflow correction if necessary. */
+ if (ZSTD_window_needOverflowCorrection(ldmState->window, 0, maxDist, ldmState->loadedDictEnd, chunkStart, chunkEnd)) {
+ U32 const ldmHSize = 1U << params->hashLog;
+ U32 const correction = ZSTD_window_correctOverflow(
+ &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart);
+ ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction);
+ /* invalidate dictionaries on overflow correction */
+ ldmState->loadedDictEnd = 0;
+ }
+ /* 2. We enforce the maximum offset allowed.
+ *
+ * kMaxChunkSize should be small enough that we don't lose too much of
+ * the window through early invalidation.
+ * TODO: * Test the chunk size.
+ * * Try invalidation after the sequence generation and test the
+ * offset against maxDist directly.
+ *
+ * NOTE: Because of dictionaries + sequence splitting we MUST make sure
+ * that any offset used is valid at the END of the sequence, since it may
+ * be split into two sequences. This condition holds when using
+ * ZSTD_window_enforceMaxDist(), but if we move to checking offsets
+ * against maxDist directly, we'll have to carefully handle that case.
+ */
+ ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL);
+ /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */
+ newLeftoverSize = ZSTD_ldm_generateSequences_internal(
+ ldmState, sequences, params, chunkStart, chunkSize);
+ if (ZSTD_isError(newLeftoverSize))
+ return newLeftoverSize;
+ /* 4. We add the leftover literals from previous iterations to the first
+ * newly generated sequence, or add the `newLeftoverSize` if none are
+ * generated.
+ */
+ /* Prepend the leftover literals from the last call */
+ if (prevSize < sequences->size) {
+ sequences->seq[prevSize].litLength += (U32)leftoverSize;
+ leftoverSize = newLeftoverSize;
+ } else {
+ assert(newLeftoverSize == chunkSize);
+ leftoverSize += chunkSize;
+ }
+ }
+ return 0;
+}
+
+void
+ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch)
+{
+ while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) {
+ rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos;
+ if (srcSize <= seq->litLength) {
+ /* Skip past srcSize literals */
+ seq->litLength -= (U32)srcSize;
+ return;
+ }
+ srcSize -= seq->litLength;
+ seq->litLength = 0;
+ if (srcSize < seq->matchLength) {
+ /* Skip past the first srcSize of the match */
+ seq->matchLength -= (U32)srcSize;
+ if (seq->matchLength < minMatch) {
+ /* The match is too short, omit it */
+ if (rawSeqStore->pos + 1 < rawSeqStore->size) {
+ seq[1].litLength += seq[0].matchLength;
+ }
+ rawSeqStore->pos++;
+ }
+ return;
+ }
+ srcSize -= seq->matchLength;
+ seq->matchLength = 0;
+ rawSeqStore->pos++;
+ }
+}
+
+/**
+ * If the sequence length is longer than remaining then the sequence is split
+ * between this block and the next.
+ *
+ * Returns the current sequence to handle, or if the rest of the block should
+ * be literals, it returns a sequence with offset == 0.
+ */
+static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
+ U32 const remaining, U32 const minMatch)
+{
+ rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos];
+ assert(sequence.offset > 0);
+ /* Likely: No partial sequence */
+ if (remaining >= sequence.litLength + sequence.matchLength) {
+ rawSeqStore->pos++;
+ return sequence;
+ }
+ /* Cut the sequence short (offset == 0 ==> rest is literals). */
+ if (remaining <= sequence.litLength) {
+ sequence.offset = 0;
+ } else if (remaining < sequence.litLength + sequence.matchLength) {
+ sequence.matchLength = remaining - sequence.litLength;
+ if (sequence.matchLength < minMatch) {
+ sequence.offset = 0;
+ }
+ }
+ /* Skip past `remaining` bytes for the future sequences. */
+ ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch);
+ return sequence;
+}
+
+void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
+ U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
+ while (currPos && rawSeqStore->pos < rawSeqStore->size) {
+ rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
+ if (currPos >= currSeq.litLength + currSeq.matchLength) {
+ currPos -= currSeq.litLength + currSeq.matchLength;
+ rawSeqStore->pos++;
+ } else {
+ rawSeqStore->posInSequence = currPos;
+ break;
+ }
+ }
+ if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
+ rawSeqStore->posInSequence = 0;
+ }
+}
+
+size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ ZSTD_paramSwitch_e useRowMatchFinder,
+ void const* src, size_t srcSize)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ unsigned const minMatch = cParams->minMatch;
+ ZSTD_blockCompressor const blockCompressor =
+ ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms));
+ /* Input bounds */
+ BYTE const* const istart = (BYTE const*)src;
+ BYTE const* const iend = istart + srcSize;
+ /* Input positions */
+ BYTE const* ip = istart;
+
+ DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize);
+ /* If using opt parser, use LDMs only as candidates rather than always accepting them */
+ if (cParams->strategy >= ZSTD_btopt) {
+ size_t lastLLSize;
+ ms->ldmSeqStore = rawSeqStore;
+ lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize);
+ ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize);
+ return lastLLSize;
+ }
+
+ assert(rawSeqStore->pos <= rawSeqStore->size);
+ assert(rawSeqStore->size <= rawSeqStore->capacity);
+ /* Loop through each sequence and apply the block compressor to the literals */
+ while (rawSeqStore->pos < rawSeqStore->size && ip < iend) {
+ /* maybeSplitSequence updates rawSeqStore->pos */
+ rawSeq const sequence = maybeSplitSequence(rawSeqStore,
+ (U32)(iend - ip), minMatch);
+ int i;
+ /* End signal */
+ if (sequence.offset == 0)
+ break;
+
+ assert(ip + sequence.litLength + sequence.matchLength <= iend);
+
+ /* Fill tables for block compressor */
+ ZSTD_ldm_limitTableUpdate(ms, ip);
+ ZSTD_ldm_fillFastTables(ms, ip);
+ /* Run the block compressor */
+ DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength);
+ {
+ size_t const newLitLength =
+ blockCompressor(ms, seqStore, rep, ip, sequence.litLength);
+ ip += sequence.litLength;
+ /* Update the repcodes */
+ for (i = ZSTD_REP_NUM - 1; i > 0; i--)
+ rep[i] = rep[i-1];
+ rep[0] = sequence.offset;
+ /* Store the sequence */
+ ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend,
+ OFFSET_TO_OFFBASE(sequence.offset),
+ sequence.matchLength);
+ ip += sequence.matchLength;
+ }
+ }
+ /* Fill the tables for the block compressor */
+ ZSTD_ldm_limitTableUpdate(ms, ip);
+ ZSTD_ldm_fillFastTables(ms, ip);
+ /* Compress the last literals */
+ return blockCompressor(ms, seqStore, rep, ip, iend - ip);
+}
diff --git a/contrib/zstd/zstd_ldm.h b/contrib/zstd/zstd_ldm.h
new file mode 100644
index 0000000..38ef9fb
--- /dev/null
+++ b/contrib/zstd/zstd_ldm.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_LDM_H
+#define ZSTD_LDM_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include "zstd_compress_internal.h" /* ldmParams_t, U32 */
+#include "zstd.h" /* ZSTD_CCtx, size_t */
+
+/*-*************************************
+* Long distance matching
+***************************************/
+
+#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT
+
+void ZSTD_ldm_fillHashTable(
+ ldmState_t* state, const BYTE* ip,
+ const BYTE* iend, ldmParams_t const* params);
+
+/**
+ * ZSTD_ldm_generateSequences():
+ *
+ * Generates the sequences using the long distance match finder.
+ * Generates long range matching sequences in `sequences`, which parse a prefix
+ * of the source. `sequences` must be large enough to store every sequence,
+ * which can be checked with `ZSTD_ldm_getMaxNbSeq()`.
+ * @returns 0 or an error code.
+ *
+ * NOTE: The user must have called ZSTD_window_update() for all of the input
+ * they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks.
+ * NOTE: This function returns an error if it runs out of space to store
+ * sequences.
+ */
+size_t ZSTD_ldm_generateSequences(
+ ldmState_t* ldms, rawSeqStore_t* sequences,
+ ldmParams_t const* params, void const* src, size_t srcSize);
+
+/**
+ * ZSTD_ldm_blockCompress():
+ *
+ * Compresses a block using the predefined sequences, along with a secondary
+ * block compressor. The literals section of every sequence is passed to the
+ * secondary block compressor, and those sequences are interspersed with the
+ * predefined sequences. Returns the length of the last literals.
+ * Updates `rawSeqStore.pos` to indicate how many sequences have been consumed.
+ * `rawSeqStore.seq` may also be updated to split the last sequence between two
+ * blocks.
+ * @return The length of the last literals.
+ *
+ * NOTE: The source must be at most the maximum block size, but the predefined
+ * sequences can be any size, and may be longer than the block. In the case that
+ * they are longer than the block, the last sequences may need to be split into
+ * two. We handle that case correctly, and update `rawSeqStore` appropriately.
+ * NOTE: This function does not return any errors.
+ */
+size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ ZSTD_paramSwitch_e useRowMatchFinder,
+ void const* src, size_t srcSize);
+
+/**
+ * ZSTD_ldm_skipSequences():
+ *
+ * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`.
+ * Avoids emitting matches less than `minMatch` bytes.
+ * Must be called for data that is not passed to ZSTD_ldm_blockCompress().
+ */
+void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize,
+ U32 const minMatch);
+
+/* ZSTD_ldm_skipRawSeqStoreBytes():
+ * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'.
+ * Not to be used in conjunction with ZSTD_ldm_skipSequences().
+ * Must be called for data with is not passed to ZSTD_ldm_blockCompress().
+ */
+void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes);
+
+/** ZSTD_ldm_getTableSize() :
+ * Estimate the space needed for long distance matching tables or 0 if LDM is
+ * disabled.
+ */
+size_t ZSTD_ldm_getTableSize(ldmParams_t params);
+
+/** ZSTD_ldm_getSeqSpace() :
+ * Return an upper bound on the number of sequences that can be produced by
+ * the long distance matcher, or 0 if LDM is disabled.
+ */
+size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize);
+
+/** ZSTD_ldm_adjustParameters() :
+ * If the params->hashRateLog is not set, set it to its default value based on
+ * windowLog and params->hashLog.
+ *
+ * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to
+ * params->hashLog if it is not).
+ *
+ * Ensures that the minMatchLength >= targetLength during optimal parsing.
+ */
+void ZSTD_ldm_adjustParameters(ldmParams_t* params,
+ ZSTD_compressionParameters const* cParams);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_FAST_H */
diff --git a/contrib/zstd/zstd_ldm_geartab.h b/contrib/zstd/zstd_ldm_geartab.h
new file mode 100644
index 0000000..92455a0
--- /dev/null
+++ b/contrib/zstd/zstd_ldm_geartab.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_LDM_GEARTAB_H
+#define ZSTD_LDM_GEARTAB_H
+
+#include "compiler.h" /* UNUSED_ATTR */
+#include "mem.h" /* U64 */
+
+static UNUSED_ATTR const U64 ZSTD_ldm_gearTab[256] = {
+ 0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc,
+ 0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05,
+ 0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e,
+ 0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889,
+ 0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e,
+ 0x37b628620b628, 0x49a8d455d88caf5, 0x8556d711e6958140,
+ 0x4f7ae74fc605c1f, 0x829f0c3468bd3a20, 0x4ffdc885c625179e,
+ 0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f,
+ 0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391,
+ 0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210,
+ 0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be,
+ 0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a,
+ 0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b,
+ 0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4,
+ 0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb,
+ 0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312,
+ 0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01,
+ 0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc,
+ 0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967,
+ 0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553,
+ 0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f,
+ 0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2,
+ 0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d,
+ 0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a,
+ 0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74,
+ 0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3,
+ 0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1,
+ 0xff452823dbb010a, 0x9d42ed614f3dd267, 0x5b9313c06257c57b,
+ 0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568,
+ 0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a,
+ 0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1,
+ 0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9,
+ 0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463,
+ 0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba,
+ 0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9,
+ 0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61,
+ 0x24a5483879c453e3, 0x88026889192b4b9, 0x28da96671782dbec,
+ 0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6,
+ 0xbc135a0a704b70ba, 0x69cd868f7622ada, 0xbc37ba89e0b9c0ab,
+ 0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5,
+ 0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59,
+ 0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7,
+ 0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc,
+ 0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb,
+ 0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be,
+ 0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312,
+ 0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1,
+ 0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc,
+ 0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d,
+ 0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445,
+ 0x820d471e20b348e, 0x1874383cb83d46dc, 0x97edeec7a1efe11c,
+ 0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5,
+ 0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5,
+ 0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28,
+ 0xaf846af6ab7d0bf4, 0xe5af208eb666e49, 0x5e6622f73534cd6a,
+ 0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9,
+ 0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15,
+ 0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef,
+ 0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2,
+ 0x9f90e4c5fd508d8, 0xa34e5956fbaf3385, 0x2e2f8e151d3ef375,
+ 0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3,
+ 0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595,
+ 0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389,
+ 0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4,
+ 0x4228e364c5b5ed7, 0x9d7a3edf0da43911, 0x8edcfeda24686756,
+ 0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc,
+ 0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45,
+ 0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea,
+ 0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f,
+ 0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc,
+ 0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c,
+ 0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a,
+ 0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17,
+ 0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3,
+ 0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4,
+ 0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91,
+ 0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40,
+ 0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741,
+ 0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f,
+ 0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4,
+ 0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad,
+ 0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047,
+ 0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2,
+ 0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e,
+ 0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b,
+ 0x2b4da14f2613d8f4
+};
+
+#endif /* ZSTD_LDM_GEARTAB_H */
diff --git a/contrib/zstd/zstd_opt.c b/contrib/zstd/zstd_opt.c
new file mode 100644
index 0000000..fdd7f9d
--- /dev/null
+++ b/contrib/zstd/zstd_opt.c
@@ -0,0 +1,1470 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h"
+#include "hist.h"
+#include "zstd_opt.h"
+
+
+#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */
+#define ZSTD_MAX_PRICE (1<<30)
+
+#define ZSTD_PREDEF_THRESHOLD 8 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
+
+
+/*-*************************************
+* Price functions for optimal parser
+***************************************/
+
+#if 0 /* approximation at bit level (for tests) */
+# define BITCOST_ACCURACY 0
+# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
+# define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat))
+#elif 0 /* fractional bit accuracy (for tests) */
+# define BITCOST_ACCURACY 8
+# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
+# define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat))
+#else /* opt==approx, ultra==accurate */
+# define BITCOST_ACCURACY 8
+# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
+# define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
+#endif
+
+/* ZSTD_bitWeight() :
+ * provide estimated "cost" of a stat in full bits only */
+MEM_STATIC U32 ZSTD_bitWeight(U32 stat)
+{
+ return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER);
+}
+
+/* ZSTD_fracWeight() :
+ * provide fractional-bit "cost" of a stat,
+ * using linear interpolation approximation */
+MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
+{
+ U32 const stat = rawStat + 1;
+ U32 const hb = ZSTD_highbit32(stat);
+ U32 const BWeight = hb * BITCOST_MULTIPLIER;
+ /* Fweight was meant for "Fractional weight"
+ * but it's effectively a value between 1 and 2
+ * using fixed point arithmetic */
+ U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb;
+ U32 const weight = BWeight + FWeight;
+ assert(hb + BITCOST_ACCURACY < 31);
+ return weight;
+}
+
+#if (DEBUGLEVEL>=2)
+/* debugging function,
+ * @return price in bytes as fractional value
+ * for debug messages only */
+MEM_STATIC double ZSTD_fCost(int price)
+{
+ return (double)price / (BITCOST_MULTIPLIER*8);
+}
+#endif
+
+static int ZSTD_compressedLiterals(optState_t const* const optPtr)
+{
+ return optPtr->literalCompressionMode != ZSTD_ps_disable;
+}
+
+static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
+{
+ if (ZSTD_compressedLiterals(optPtr))
+ optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel);
+ optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel);
+ optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel);
+ optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel);
+}
+
+
+static U32 sum_u32(const unsigned table[], size_t nbElts)
+{
+ size_t n;
+ U32 total = 0;
+ for (n=0; n<nbElts; n++) {
+ total += table[n];
+ }
+ return total;
+}
+
+typedef enum { base_0possible=0, base_1guaranteed=1 } base_directive_e;
+
+static U32
+ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift, base_directive_e base1)
+{
+ U32 s, sum=0;
+ DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)",
+ (unsigned)lastEltIndex+1, (unsigned)shift );
+ assert(shift < 30);
+ for (s=0; s<lastEltIndex+1; s++) {
+ unsigned const base = base1 ? 1 : (table[s]>0);
+ unsigned const newStat = base + (table[s] >> shift);
+ sum += newStat;
+ table[s] = newStat;
+ }
+ return sum;
+}
+
+/* ZSTD_scaleStats() :
+ * reduce all elt frequencies in table if sum too large
+ * return the resulting sum of elements */
+static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
+{
+ U32 const prevsum = sum_u32(table, lastEltIndex+1);
+ U32 const factor = prevsum >> logTarget;
+ DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget);
+ assert(logTarget < 30);
+ if (factor <= 1) return prevsum;
+ return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed);
+}
+
+/* ZSTD_rescaleFreqs() :
+ * if first block (detected by optPtr->litLengthSum == 0) : init statistics
+ * take hints from dictionary if there is one
+ * and init from zero if there is none,
+ * using src for literals stats, and baseline stats for sequence symbols
+ * otherwise downscale existing stats, to be used as seed for next block.
+ */
+static void
+ZSTD_rescaleFreqs(optState_t* const optPtr,
+ const BYTE* const src, size_t const srcSize,
+ int const optLevel)
+{
+ int const compressedLiterals = ZSTD_compressedLiterals(optPtr);
+ DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
+ optPtr->priceType = zop_dynamic;
+
+ if (optPtr->litLengthSum == 0) { /* no literals stats collected -> first block assumed -> init */
+
+ /* heuristic: use pre-defined stats for too small inputs */
+ if (srcSize <= ZSTD_PREDEF_THRESHOLD) {
+ DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD);
+ optPtr->priceType = zop_predef;
+ }
+
+ assert(optPtr->symbolCosts != NULL);
+ if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
+
+ /* huffman stats covering the full value set : table presumed generated by dictionary */
+ optPtr->priceType = zop_dynamic;
+
+ if (compressedLiterals) {
+ /* generate literals statistics from huffman table */
+ unsigned lit;
+ assert(optPtr->litFreq != NULL);
+ optPtr->litSum = 0;
+ for (lit=0; lit<=MaxLit; lit++) {
+ U32 const scaleLog = 11; /* scale to 2K */
+ U32 const bitCost = HUF_getNbBitsFromCTable(optPtr->symbolCosts->huf.CTable, lit);
+ assert(bitCost <= scaleLog);
+ optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
+ optPtr->litSum += optPtr->litFreq[lit];
+ } }
+
+ { unsigned ll;
+ FSE_CState_t llstate;
+ FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable);
+ optPtr->litLengthSum = 0;
+ for (ll=0; ll<=MaxLL; ll++) {
+ U32 const scaleLog = 10; /* scale to 1K */
+ U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll);
+ assert(bitCost < scaleLog);
+ optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
+ optPtr->litLengthSum += optPtr->litLengthFreq[ll];
+ } }
+
+ { unsigned ml;
+ FSE_CState_t mlstate;
+ FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable);
+ optPtr->matchLengthSum = 0;
+ for (ml=0; ml<=MaxML; ml++) {
+ U32 const scaleLog = 10;
+ U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml);
+ assert(bitCost < scaleLog);
+ optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
+ optPtr->matchLengthSum += optPtr->matchLengthFreq[ml];
+ } }
+
+ { unsigned of;
+ FSE_CState_t ofstate;
+ FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable);
+ optPtr->offCodeSum = 0;
+ for (of=0; of<=MaxOff; of++) {
+ U32 const scaleLog = 10;
+ U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of);
+ assert(bitCost < scaleLog);
+ optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
+ optPtr->offCodeSum += optPtr->offCodeFreq[of];
+ } }
+
+ } else { /* first block, no dictionary */
+
+ assert(optPtr->litFreq != NULL);
+ if (compressedLiterals) {
+ /* base initial cost of literals on direct frequency within src */
+ unsigned lit = MaxLit;
+ HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */
+ optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible);
+ }
+
+ { unsigned const baseLLfreqs[MaxLL+1] = {
+ 4, 2, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1
+ };
+ ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs));
+ optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1);
+ }
+
+ { unsigned ml;
+ for (ml=0; ml<=MaxML; ml++)
+ optPtr->matchLengthFreq[ml] = 1;
+ }
+ optPtr->matchLengthSum = MaxML+1;
+
+ { unsigned const baseOFCfreqs[MaxOff+1] = {
+ 6, 2, 1, 1, 2, 3, 4, 4,
+ 4, 3, 2, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1
+ };
+ ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs));
+ optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1);
+ }
+
+ }
+
+ } else { /* new block : scale down accumulated statistics */
+
+ if (compressedLiterals)
+ optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12);
+ optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11);
+ optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11);
+ optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11);
+ }
+
+ ZSTD_setBasePrices(optPtr, optLevel);
+}
+
+/* ZSTD_rawLiteralsCost() :
+ * price of literals (only) in specified segment (which length can be 0).
+ * does not include price of literalLength symbol */
+static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
+ const optState_t* const optPtr,
+ int optLevel)
+{
+ if (litLength == 0) return 0;
+
+ if (!ZSTD_compressedLiterals(optPtr))
+ return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */
+
+ if (optPtr->priceType == zop_predef)
+ return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */
+
+ /* dynamic statistics */
+ { U32 price = optPtr->litSumBasePrice * litLength;
+ U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER;
+ U32 u;
+ assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER);
+ for (u=0; u < litLength; u++) {
+ U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel);
+ if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax;
+ price -= litPrice;
+ }
+ return price;
+ }
+}
+
+/* ZSTD_litLengthPrice() :
+ * cost of literalLength symbol */
+static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel)
+{
+ assert(litLength <= ZSTD_BLOCKSIZE_MAX);
+ if (optPtr->priceType == zop_predef)
+ return WEIGHT(litLength, optLevel);
+
+ /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX
+ * because it isn't representable in the zstd format.
+ * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1.
+ * In such a case, the block would be all literals.
+ */
+ if (litLength == ZSTD_BLOCKSIZE_MAX)
+ return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel);
+
+ /* dynamic statistics */
+ { U32 const llCode = ZSTD_LLcode(litLength);
+ return (LL_bits[llCode] * BITCOST_MULTIPLIER)
+ + optPtr->litLengthSumBasePrice
+ - WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
+ }
+}
+
+/* ZSTD_getMatchPrice() :
+ * Provides the cost of the match part (offset + matchLength) of a sequence.
+ * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
+ * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq()
+ * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency)
+ */
+FORCE_INLINE_TEMPLATE U32
+ZSTD_getMatchPrice(U32 const offBase,
+ U32 const matchLength,
+ const optState_t* const optPtr,
+ int const optLevel)
+{
+ U32 price;
+ U32 const offCode = ZSTD_highbit32(offBase);
+ U32 const mlBase = matchLength - MINMATCH;
+ assert(matchLength >= MINMATCH);
+
+ if (optPtr->priceType == zop_predef) /* fixed scheme, does not use statistics */
+ return WEIGHT(mlBase, optLevel)
+ + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */
+
+ /* dynamic statistics */
+ price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel));
+ if ((optLevel<2) /*static*/ && offCode >= 20)
+ price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */
+
+ /* match Length */
+ { U32 const mlCode = ZSTD_MLcode(mlBase);
+ price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel));
+ }
+
+ price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */
+
+ DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price);
+ return price;
+}
+
+/* ZSTD_updateStats() :
+ * assumption : literals + litLength <= iend */
+static void ZSTD_updateStats(optState_t* const optPtr,
+ U32 litLength, const BYTE* literals,
+ U32 offBase, U32 matchLength)
+{
+ /* literals */
+ if (ZSTD_compressedLiterals(optPtr)) {
+ U32 u;
+ for (u=0; u < litLength; u++)
+ optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
+ optPtr->litSum += litLength*ZSTD_LITFREQ_ADD;
+ }
+
+ /* literal Length */
+ { U32 const llCode = ZSTD_LLcode(litLength);
+ optPtr->litLengthFreq[llCode]++;
+ optPtr->litLengthSum++;
+ }
+
+ /* offset code : follows storeSeq() numeric representation */
+ { U32 const offCode = ZSTD_highbit32(offBase);
+ assert(offCode <= MaxOff);
+ optPtr->offCodeFreq[offCode]++;
+ optPtr->offCodeSum++;
+ }
+
+ /* match Length */
+ { U32 const mlBase = matchLength - MINMATCH;
+ U32 const mlCode = ZSTD_MLcode(mlBase);
+ optPtr->matchLengthFreq[mlCode]++;
+ optPtr->matchLengthSum++;
+ }
+}
+
+
+/* ZSTD_readMINMATCH() :
+ * function safe only for comparisons
+ * assumption : memPtr must be at least 4 bytes before end of buffer */
+MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length)
+{
+ switch (length)
+ {
+ default :
+ case 4 : return MEM_read32(memPtr);
+ case 3 : if (MEM_isLittleEndian())
+ return MEM_read32(memPtr)<<8;
+ else
+ return MEM_read32(memPtr)>>8;
+ }
+}
+
+
+/* Update hashTable3 up to ip (excluded)
+ Assumption : always within prefix (i.e. not within extDict) */
+static U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_matchState_t* ms,
+ U32* nextToUpdate3,
+ const BYTE* const ip)
+{
+ U32* const hashTable3 = ms->hashTable3;
+ U32 const hashLog3 = ms->hashLog3;
+ const BYTE* const base = ms->window.base;
+ U32 idx = *nextToUpdate3;
+ U32 const target = (U32)(ip - base);
+ size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3);
+ assert(hashLog3 > 0);
+
+ while(idx < target) {
+ hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx;
+ idx++;
+ }
+
+ *nextToUpdate3 = target;
+ return hashTable3[hash3];
+}
+
+
+/*-*************************************
+* Binary Tree search
+***************************************/
+/** ZSTD_insertBt1() : add one or multiple positions to tree.
+ * @param ip assumed <= iend-8 .
+ * @param target The target of ZSTD_updateTree_internal() - we are filling to this position
+ * @return : nb of positions added */
+static U32 ZSTD_insertBt1(
+ const ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iend,
+ U32 const target,
+ U32 const mls, const int extDict)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32* const hashTable = ms->hashTable;
+ U32 const hashLog = cParams->hashLog;
+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
+ U32* const bt = ms->chainTable;
+ U32 const btLog = cParams->chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+ U32 matchIndex = hashTable[h];
+ size_t commonLengthSmaller=0, commonLengthLarger=0;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const dictBase = ms->window.dictBase;
+ const U32 dictLimit = ms->window.dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ const BYTE* match;
+ const U32 curr = (U32)(ip-base);
+ const U32 btLow = btMask >= curr ? 0 : curr - btMask;
+ U32* smallerPtr = bt + 2*(curr&btMask);
+ U32* largerPtr = smallerPtr + 1;
+ U32 dummy32; /* to be nullified at the end */
+ /* windowLow is based on target because
+ * we only need positions that will be in the window at the end of the tree update.
+ */
+ U32 const windowLow = ZSTD_getLowestMatchIndex(ms, target, cParams->windowLog);
+ U32 matchEndIdx = curr+8+1;
+ size_t bestLength = 8;
+ U32 nbCompares = 1U << cParams->searchLog;
+#ifdef ZSTD_C_PREDICT
+ U32 predictedSmall = *(bt + 2*((curr-1)&btMask) + 0);
+ U32 predictedLarge = *(bt + 2*((curr-1)&btMask) + 1);
+ predictedSmall += (predictedSmall>0);
+ predictedLarge += (predictedLarge>0);
+#endif /* ZSTD_C_PREDICT */
+
+ DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr);
+
+ assert(curr <= target);
+ assert(ip <= iend-8); /* required for h calculation */
+ hashTable[h] = curr; /* Update Hash Table */
+
+ assert(windowLow > 0);
+ for (; nbCompares && (matchIndex >= windowLow); --nbCompares) {
+ U32* const nextPtr = bt + 2*(matchIndex & btMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ assert(matchIndex < curr);
+
+#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */
+ const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */
+ if (matchIndex == predictedSmall) {
+ /* no need to check length, result known */
+ *smallerPtr = matchIndex;
+ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
+ predictedSmall = predictPtr[1] + (predictPtr[1]>0);
+ continue;
+ }
+ if (matchIndex == predictedLarge) {
+ *largerPtr = matchIndex;
+ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ predictedLarge = predictPtr[0] + (predictPtr[0]>0);
+ continue;
+ }
+#endif
+
+ if (!extDict || (matchIndex+matchLength >= dictLimit)) {
+ assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */
+ match = base + matchIndex;
+ matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
+ } else {
+ match = dictBase + matchIndex;
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
+ if (matchIndex+matchLength >= dictLimit)
+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
+ }
+
+ if (matchLength > bestLength) {
+ bestLength = matchLength;
+ if (matchLength > matchEndIdx - matchIndex)
+ matchEndIdx = matchIndex + (U32)matchLength;
+ }
+
+ if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
+ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */
+ }
+
+ if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */
+ /* match is smaller than current */
+ *smallerPtr = matchIndex; /* update smaller idx */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */
+ smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */
+ matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */
+ } else {
+ /* match is larger than current */
+ *largerPtr = matchIndex;
+ commonLengthLarger = matchLength;
+ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ } }
+
+ *smallerPtr = *largerPtr = 0;
+ { U32 positions = 0;
+ if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */
+ assert(matchEndIdx > curr + 8);
+ return MAX(positions, matchEndIdx - (curr + 8));
+ }
+}
+
+FORCE_INLINE_TEMPLATE
+void ZSTD_updateTree_internal(
+ ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iend,
+ const U32 mls, const ZSTD_dictMode_e dictMode)
+{
+ const BYTE* const base = ms->window.base;
+ U32 const target = (U32)(ip - base);
+ U32 idx = ms->nextToUpdate;
+ DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)",
+ idx, target, dictMode);
+
+ while(idx < target) {
+ U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, target, mls, dictMode == ZSTD_extDict);
+ assert(idx < (U32)(idx + forward));
+ idx += forward;
+ }
+ assert((size_t)(ip - base) <= (size_t)(U32)(-1));
+ assert((size_t)(iend - base) <= (size_t)(U32)(-1));
+ ms->nextToUpdate = target;
+}
+
+void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
+ ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
+}
+
+FORCE_INLINE_TEMPLATE U32
+ZSTD_insertBtAndGetAllMatches (
+ ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */
+ ZSTD_matchState_t* ms,
+ U32* nextToUpdate3,
+ const BYTE* const ip, const BYTE* const iLimit,
+ const ZSTD_dictMode_e dictMode,
+ const U32 rep[ZSTD_REP_NUM],
+ const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
+ const U32 lengthToBeat,
+ const U32 mls /* template */)
+{
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+ U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
+ const BYTE* const base = ms->window.base;
+ U32 const curr = (U32)(ip-base);
+ U32 const hashLog = cParams->hashLog;
+ U32 const minMatch = (mls==3) ? 3 : 4;
+ U32* const hashTable = ms->hashTable;
+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
+ U32 matchIndex = hashTable[h];
+ U32* const bt = ms->chainTable;
+ U32 const btLog = cParams->chainLog - 1;
+ U32 const btMask= (1U << btLog) - 1;
+ size_t commonLengthSmaller=0, commonLengthLarger=0;
+ const BYTE* const dictBase = ms->window.dictBase;
+ U32 const dictLimit = ms->window.dictLimit;
+ const BYTE* const dictEnd = dictBase + dictLimit;
+ const BYTE* const prefixStart = base + dictLimit;
+ U32 const btLow = (btMask >= curr) ? 0 : curr - btMask;
+ U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog);
+ U32 const matchLow = windowLow ? windowLow : 1;
+ U32* smallerPtr = bt + 2*(curr&btMask);
+ U32* largerPtr = bt + 2*(curr&btMask) + 1;
+ U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */
+ U32 dummy32; /* to be nullified at the end */
+ U32 mnum = 0;
+ U32 nbCompares = 1U << cParams->searchLog;
+
+ const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL;
+ const ZSTD_compressionParameters* const dmsCParams =
+ dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL;
+ const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL;
+ const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL;
+ U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0;
+ U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0;
+ U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0;
+ U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog;
+ U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog;
+ U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0;
+ U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit;
+
+ size_t bestLength = lengthToBeat-1;
+ DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr);
+
+ /* check repCode */
+ assert(ll0 <= 1); /* necessarily 1 or 0 */
+ { U32 const lastR = ZSTD_REP_NUM + ll0;
+ U32 repCode;
+ for (repCode = ll0; repCode < lastR; repCode++) {
+ U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
+ U32 const repIndex = curr - repOffset;
+ U32 repLen = 0;
+ assert(curr >= dictLimit);
+ if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */
+ /* We must validate the repcode offset because when we're using a dictionary the
+ * valid offset range shrinks when the dictionary goes out of bounds.
+ */
+ if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) {
+ repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch;
+ }
+ } else { /* repIndex < dictLimit || repIndex >= curr */
+ const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ?
+ dmsBase + repIndex - dmsIndexDelta :
+ dictBase + repIndex;
+ assert(curr >= windowLow);
+ if ( dictMode == ZSTD_extDict
+ && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */
+ & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */)
+ && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
+ repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch;
+ }
+ if (dictMode == ZSTD_dictMatchState
+ && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */
+ & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */
+ && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
+ repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch;
+ } }
+ /* save longer solution */
+ if (repLen > bestLength) {
+ DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u",
+ repCode, ll0, repOffset, repLen);
+ bestLength = repLen;
+ matches[mnum].off = REPCODE_TO_OFFBASE(repCode - ll0 + 1); /* expect value between 1 and 3 */
+ matches[mnum].len = (U32)repLen;
+ mnum++;
+ if ( (repLen > sufficient_len)
+ | (ip+repLen == iLimit) ) { /* best possible */
+ return mnum;
+ } } } }
+
+ /* HC3 match finder */
+ if ((mls == 3) /*static*/ && (bestLength < mls)) {
+ U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip);
+ if ((matchIndex3 >= matchLow)
+ & (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) {
+ size_t mlen;
+ if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) {
+ const BYTE* const match = base + matchIndex3;
+ mlen = ZSTD_count(ip, match, iLimit);
+ } else {
+ const BYTE* const match = dictBase + matchIndex3;
+ mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart);
+ }
+
+ /* save best solution */
+ if (mlen >= mls /* == 3 > bestLength */) {
+ DEBUGLOG(8, "found small match with hlog3, of length %u",
+ (U32)mlen);
+ bestLength = mlen;
+ assert(curr > matchIndex3);
+ assert(mnum==0); /* no prior solution */
+ matches[0].off = OFFSET_TO_OFFBASE(curr - matchIndex3);
+ matches[0].len = (U32)mlen;
+ mnum = 1;
+ if ( (mlen > sufficient_len) |
+ (ip+mlen == iLimit) ) { /* best possible length */
+ ms->nextToUpdate = curr+1; /* skip insertion */
+ return 1;
+ } } }
+ /* no dictMatchState lookup: dicts don't have a populated HC3 table */
+ } /* if (mls == 3) */
+
+ hashTable[h] = curr; /* Update Hash Table */
+
+ for (; nbCompares && (matchIndex >= matchLow); --nbCompares) {
+ U32* const nextPtr = bt + 2*(matchIndex & btMask);
+ const BYTE* match;
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ assert(curr > matchIndex);
+
+ if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) {
+ assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */
+ match = base + matchIndex;
+ if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */
+ matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit);
+ } else {
+ match = dictBase + matchIndex;
+ assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart);
+ if (matchIndex+matchLength >= dictLimit)
+ match = base + matchIndex; /* prepare for match[matchLength] read */
+ }
+
+ if (matchLength > bestLength) {
+ DEBUGLOG(8, "found match of length %u at distance %u (offBase=%u)",
+ (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex));
+ assert(matchEndIdx > matchIndex);
+ if (matchLength > matchEndIdx - matchIndex)
+ matchEndIdx = matchIndex + (U32)matchLength;
+ bestLength = matchLength;
+ matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex);
+ matches[mnum].len = (U32)matchLength;
+ mnum++;
+ if ( (matchLength > ZSTD_OPT_NUM)
+ | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) {
+ if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */
+ break; /* drop, to preserve bt consistency (miss a little bit of compression) */
+ } }
+
+ if (match[matchLength] < ip[matchLength]) {
+ /* match smaller than current */
+ *smallerPtr = matchIndex; /* update smaller idx */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */
+ matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */
+ } else {
+ *largerPtr = matchIndex;
+ commonLengthLarger = matchLength;
+ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ } }
+
+ *smallerPtr = *largerPtr = 0;
+
+ assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
+ if (dictMode == ZSTD_dictMatchState && nbCompares) {
+ size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls);
+ U32 dictMatchIndex = dms->hashTable[dmsH];
+ const U32* const dmsBt = dms->chainTable;
+ commonLengthSmaller = commonLengthLarger = 0;
+ for (; nbCompares && (dictMatchIndex > dmsLowLimit); --nbCompares) {
+ const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ const BYTE* match = dmsBase + dictMatchIndex;
+ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart);
+ if (dictMatchIndex+matchLength >= dmsHighLimit)
+ match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */
+
+ if (matchLength > bestLength) {
+ matchIndex = dictMatchIndex + dmsIndexDelta;
+ DEBUGLOG(8, "found dms match of length %u at distance %u (offBase=%u)",
+ (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex));
+ if (matchLength > matchEndIdx - matchIndex)
+ matchEndIdx = matchIndex + (U32)matchLength;
+ bestLength = matchLength;
+ matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex);
+ matches[mnum].len = (U32)matchLength;
+ mnum++;
+ if ( (matchLength > ZSTD_OPT_NUM)
+ | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) {
+ break; /* drop, to guarantee consistency (miss a little bit of compression) */
+ } }
+
+ if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */
+ if (match[matchLength] < ip[matchLength]) {
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
+ } else {
+ /* match is larger than current */
+ commonLengthLarger = matchLength;
+ dictMatchIndex = nextPtr[0];
+ } } } /* if (dictMode == ZSTD_dictMatchState) */
+
+ assert(matchEndIdx > curr+8);
+ ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
+ return mnum;
+}
+
+typedef U32 (*ZSTD_getAllMatchesFn)(
+ ZSTD_match_t*,
+ ZSTD_matchState_t*,
+ U32*,
+ const BYTE*,
+ const BYTE*,
+ const U32 rep[ZSTD_REP_NUM],
+ U32 const ll0,
+ U32 const lengthToBeat);
+
+FORCE_INLINE_TEMPLATE U32 ZSTD_btGetAllMatches_internal(
+ ZSTD_match_t* matches,
+ ZSTD_matchState_t* ms,
+ U32* nextToUpdate3,
+ const BYTE* ip,
+ const BYTE* const iHighLimit,
+ const U32 rep[ZSTD_REP_NUM],
+ U32 const ll0,
+ U32 const lengthToBeat,
+ const ZSTD_dictMode_e dictMode,
+ const U32 mls)
+{
+ assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls);
+ DEBUGLOG(8, "ZSTD_BtGetAllMatches(dictMode=%d, mls=%u)", (int)dictMode, mls);
+ if (ip < ms->window.base + ms->nextToUpdate)
+ return 0; /* skipped area */
+ ZSTD_updateTree_internal(ms, ip, iHighLimit, mls, dictMode);
+ return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, mls);
+}
+
+#define ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls) ZSTD_btGetAllMatches_##dictMode##_##mls
+
+#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \
+ static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \
+ ZSTD_match_t* matches, \
+ ZSTD_matchState_t* ms, \
+ U32* nextToUpdate3, \
+ const BYTE* ip, \
+ const BYTE* const iHighLimit, \
+ const U32 rep[ZSTD_REP_NUM], \
+ U32 const ll0, \
+ U32 const lengthToBeat) \
+ { \
+ return ZSTD_btGetAllMatches_internal( \
+ matches, ms, nextToUpdate3, ip, iHighLimit, \
+ rep, ll0, lengthToBeat, ZSTD_##dictMode, mls); \
+ }
+
+#define GEN_ZSTD_BT_GET_ALL_MATCHES(dictMode) \
+ GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 3) \
+ GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 4) \
+ GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 5) \
+ GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 6)
+
+GEN_ZSTD_BT_GET_ALL_MATCHES(noDict)
+GEN_ZSTD_BT_GET_ALL_MATCHES(extDict)
+GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState)
+
+#define ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMode) \
+ { \
+ ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 3), \
+ ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 4), \
+ ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 5), \
+ ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 6) \
+ }
+
+static ZSTD_getAllMatchesFn
+ZSTD_selectBtGetAllMatches(ZSTD_matchState_t const* ms, ZSTD_dictMode_e const dictMode)
+{
+ ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = {
+ ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict),
+ ZSTD_BT_GET_ALL_MATCHES_ARRAY(extDict),
+ ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMatchState)
+ };
+ U32 const mls = BOUNDED(3, ms->cParams.minMatch, 6);
+ assert((U32)dictMode < 3);
+ assert(mls - 3 < 4);
+ return getAllMatchesFns[(int)dictMode][mls - 3];
+}
+
+/*************************
+* LDM helper functions *
+*************************/
+
+/* Struct containing info needed to make decision about ldm inclusion */
+typedef struct {
+ rawSeqStore_t seqStore; /* External match candidates store for this block */
+ U32 startPosInBlock; /* Start position of the current match candidate */
+ U32 endPosInBlock; /* End position of the current match candidate */
+ U32 offset; /* Offset of the match candidate */
+} ZSTD_optLdm_t;
+
+/* ZSTD_optLdm_skipRawSeqStoreBytes():
+ * Moves forward in @rawSeqStore by @nbBytes,
+ * which will update the fields 'pos' and 'posInSequence'.
+ */
+static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes)
+{
+ U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
+ while (currPos && rawSeqStore->pos < rawSeqStore->size) {
+ rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
+ if (currPos >= currSeq.litLength + currSeq.matchLength) {
+ currPos -= currSeq.litLength + currSeq.matchLength;
+ rawSeqStore->pos++;
+ } else {
+ rawSeqStore->posInSequence = currPos;
+ break;
+ }
+ }
+ if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
+ rawSeqStore->posInSequence = 0;
+ }
+}
+
+/* ZSTD_opt_getNextMatchAndUpdateSeqStore():
+ * Calculates the beginning and end of the next match in the current block.
+ * Updates 'pos' and 'posInSequence' of the ldmSeqStore.
+ */
+static void
+ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock,
+ U32 blockBytesRemaining)
+{
+ rawSeq currSeq;
+ U32 currBlockEndPos;
+ U32 literalsBytesRemaining;
+ U32 matchBytesRemaining;
+
+ /* Setting match end position to MAX to ensure we never use an LDM during this block */
+ if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
+ optLdm->startPosInBlock = UINT_MAX;
+ optLdm->endPosInBlock = UINT_MAX;
+ return;
+ }
+ /* Calculate appropriate bytes left in matchLength and litLength
+ * after adjusting based on ldmSeqStore->posInSequence */
+ currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos];
+ assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength);
+ currBlockEndPos = currPosInBlock + blockBytesRemaining;
+ literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ?
+ currSeq.litLength - (U32)optLdm->seqStore.posInSequence :
+ 0;
+ matchBytesRemaining = (literalsBytesRemaining == 0) ?
+ currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) :
+ currSeq.matchLength;
+
+ /* If there are more literal bytes than bytes remaining in block, no ldm is possible */
+ if (literalsBytesRemaining >= blockBytesRemaining) {
+ optLdm->startPosInBlock = UINT_MAX;
+ optLdm->endPosInBlock = UINT_MAX;
+ ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining);
+ return;
+ }
+
+ /* Matches may be < MINMATCH by this process. In that case, we will reject them
+ when we are deciding whether or not to add the ldm */
+ optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining;
+ optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining;
+ optLdm->offset = currSeq.offset;
+
+ if (optLdm->endPosInBlock > currBlockEndPos) {
+ /* Match ends after the block ends, we can't use the whole match */
+ optLdm->endPosInBlock = currBlockEndPos;
+ ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock);
+ } else {
+ /* Consume nb of bytes equal to size of sequence left */
+ ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining);
+ }
+}
+
+/* ZSTD_optLdm_maybeAddMatch():
+ * Adds a match if it's long enough,
+ * based on it's 'matchStartPosInBlock' and 'matchEndPosInBlock',
+ * into 'matches'. Maintains the correct ordering of 'matches'.
+ */
+static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
+ const ZSTD_optLdm_t* optLdm, U32 currPosInBlock)
+{
+ U32 const posDiff = currPosInBlock - optLdm->startPosInBlock;
+ /* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */
+ U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff;
+
+ /* Ensure that current block position is not outside of the match */
+ if (currPosInBlock < optLdm->startPosInBlock
+ || currPosInBlock >= optLdm->endPosInBlock
+ || candidateMatchLength < MINMATCH) {
+ return;
+ }
+
+ if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) {
+ U32 const candidateOffBase = OFFSET_TO_OFFBASE(optLdm->offset);
+ DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offBase: %u matchLength %u) at block position=%u",
+ candidateOffBase, candidateMatchLength, currPosInBlock);
+ matches[*nbMatches].len = candidateMatchLength;
+ matches[*nbMatches].off = candidateOffBase;
+ (*nbMatches)++;
+ }
+}
+
+/* ZSTD_optLdm_processMatchCandidate():
+ * Wrapper function to update ldm seq store and call ldm functions as necessary.
+ */
+static void
+ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm,
+ ZSTD_match_t* matches, U32* nbMatches,
+ U32 currPosInBlock, U32 remainingBytes)
+{
+ if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
+ return;
+ }
+
+ if (currPosInBlock >= optLdm->endPosInBlock) {
+ if (currPosInBlock > optLdm->endPosInBlock) {
+ /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily
+ * at the end of a match from the ldm seq store, and will often be some bytes
+ * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots"
+ */
+ U32 const posOvershoot = currPosInBlock - optLdm->endPosInBlock;
+ ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot);
+ }
+ ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes);
+ }
+ ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock);
+}
+
+
+/*-*******************************
+* Optimal parser
+*********************************/
+
+static U32 ZSTD_totalLen(ZSTD_optimal_t sol)
+{
+ return sol.litlen + sol.mlen;
+}
+
+#if 0 /* debug */
+
+static void
+listStats(const U32* table, int lastEltID)
+{
+ int const nbElts = lastEltID + 1;
+ int enb;
+ for (enb=0; enb < nbElts; enb++) {
+ (void)table;
+ /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */
+ RAWLOG(2, "%4i,", table[enb]);
+ }
+ RAWLOG(2, " \n");
+}
+
+#endif
+
+FORCE_INLINE_TEMPLATE size_t
+ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
+ seqStore_t* seqStore,
+ U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize,
+ const int optLevel,
+ const ZSTD_dictMode_e dictMode)
+{
+ optState_t* const optStatePtr = &ms->opt;
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* ip = istart;
+ const BYTE* anchor = istart;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* const ilimit = iend - 8;
+ const BYTE* const base = ms->window.base;
+ const BYTE* const prefixStart = base + ms->window.dictLimit;
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
+
+ ZSTD_getAllMatchesFn getAllMatches = ZSTD_selectBtGetAllMatches(ms, dictMode);
+
+ U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
+ U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
+ U32 nextToUpdate3 = ms->nextToUpdate;
+
+ ZSTD_optimal_t* const opt = optStatePtr->priceTable;
+ ZSTD_match_t* const matches = optStatePtr->matchTable;
+ ZSTD_optimal_t lastSequence;
+ ZSTD_optLdm_t optLdm;
+
+ optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore;
+ optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0;
+ ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip));
+
+ /* init */
+ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
+ (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
+ assert(optLevel <= 2);
+ ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
+ ip += (ip==prefixStart);
+
+ /* Match Loop */
+ while (ip < ilimit) {
+ U32 cur, last_pos = 0;
+
+ /* find first match */
+ { U32 const litlen = (U32)(ip - anchor);
+ U32 const ll0 = !litlen;
+ U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch);
+ ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
+ (U32)(ip-istart), (U32)(iend - ip));
+ if (!nbMatches) { ip++; continue; }
+
+ /* initialize opt[0] */
+ { U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; }
+ opt[0].mlen = 0; /* means is_a_literal */
+ opt[0].litlen = litlen;
+ /* We don't need to include the actual price of the literals because
+ * it is static for the duration of the forward pass, and is included
+ * in every price. We include the literal length to avoid negative
+ * prices when we subtract the previous literal length.
+ */
+ opt[0].price = (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel);
+
+ /* large match -> immediate encoding */
+ { U32 const maxML = matches[nbMatches-1].len;
+ U32 const maxOffBase = matches[nbMatches-1].off;
+ DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffBase=%u at cPos=%u => start new series",
+ nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart));
+
+ if (maxML > sufficient_len) {
+ lastSequence.litlen = litlen;
+ lastSequence.mlen = maxML;
+ lastSequence.off = maxOffBase;
+ DEBUGLOG(6, "large match (%u>%u), immediate encoding",
+ maxML, sufficient_len);
+ cur = 0;
+ last_pos = ZSTD_totalLen(lastSequence);
+ goto _shortestPath;
+ } }
+
+ /* set prices for first matches starting position == 0 */
+ assert(opt[0].price >= 0);
+ { U32 const literalsPrice = (U32)opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
+ U32 pos;
+ U32 matchNb;
+ for (pos = 1; pos < minMatch; pos++) {
+ opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */
+ }
+ for (matchNb = 0; matchNb < nbMatches; matchNb++) {
+ U32 const offBase = matches[matchNb].off;
+ U32 const end = matches[matchNb].len;
+ for ( ; pos <= end ; pos++ ) {
+ U32 const matchPrice = ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel);
+ U32 const sequencePrice = literalsPrice + matchPrice;
+ DEBUGLOG(7, "rPos:%u => set initial price : %.2f",
+ pos, ZSTD_fCost((int)sequencePrice));
+ opt[pos].mlen = pos;
+ opt[pos].off = offBase;
+ opt[pos].litlen = litlen;
+ opt[pos].price = (int)sequencePrice;
+ } }
+ last_pos = pos-1;
+ }
+ }
+
+ /* check further positions */
+ for (cur = 1; cur <= last_pos; cur++) {
+ const BYTE* const inr = ip + cur;
+ assert(cur < ZSTD_OPT_NUM);
+ DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur)
+
+ /* Fix current position with one literal if cheaper */
+ { U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1;
+ int const price = opt[cur-1].price
+ + (int)ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel)
+ + (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel)
+ - (int)ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel);
+ assert(price < 1000000000); /* overflow check */
+ if (price <= opt[cur].price) {
+ DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)",
+ inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen,
+ opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]);
+ opt[cur].mlen = 0;
+ opt[cur].off = 0;
+ opt[cur].litlen = litlen;
+ opt[cur].price = price;
+ } else {
+ DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)",
+ inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price),
+ opt[cur].rep[0], opt[cur].rep[1], opt[cur].rep[2]);
+ }
+ }
+
+ /* Set the repcodes of the current position. We must do it here
+ * because we rely on the repcodes of the 2nd to last sequence being
+ * correct to set the next chunks repcodes during the backward
+ * traversal.
+ */
+ ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t));
+ assert(cur >= opt[cur].mlen);
+ if (opt[cur].mlen != 0) {
+ U32 const prev = cur - opt[cur].mlen;
+ repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0);
+ ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t));
+ } else {
+ ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t));
+ }
+
+ /* last match must start at a minimum distance of 8 from oend */
+ if (inr > ilimit) continue;
+
+ if (cur == last_pos) break;
+
+ if ( (optLevel==0) /*static_test*/
+ && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) {
+ DEBUGLOG(7, "move to next rPos:%u : price is <=", cur+1);
+ continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */
+ }
+
+ assert(opt[cur].price >= 0);
+ { U32 const ll0 = (opt[cur].mlen != 0);
+ U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0;
+ U32 const previousPrice = (U32)opt[cur].price;
+ U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
+ U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, inr, iend, opt[cur].rep, ll0, minMatch);
+ U32 matchNb;
+
+ ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
+ (U32)(inr-istart), (U32)(iend-inr));
+
+ if (!nbMatches) {
+ DEBUGLOG(7, "rPos:%u : no match found", cur);
+ continue;
+ }
+
+ { U32 const maxML = matches[nbMatches-1].len;
+ DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of maxLength=%u",
+ inr-istart, cur, nbMatches, maxML);
+
+ if ( (maxML > sufficient_len)
+ || (cur + maxML >= ZSTD_OPT_NUM) ) {
+ lastSequence.mlen = maxML;
+ lastSequence.off = matches[nbMatches-1].off;
+ lastSequence.litlen = litlen;
+ cur -= (opt[cur].mlen==0) ? opt[cur].litlen : 0; /* last sequence is actually only literals, fix cur to last match - note : may underflow, in which case, it's first sequence, and it's okay */
+ last_pos = cur + ZSTD_totalLen(lastSequence);
+ if (cur > ZSTD_OPT_NUM) cur = 0; /* underflow => first match */
+ goto _shortestPath;
+ } }
+
+ /* set prices using matches found at position == cur */
+ for (matchNb = 0; matchNb < nbMatches; matchNb++) {
+ U32 const offset = matches[matchNb].off;
+ U32 const lastML = matches[matchNb].len;
+ U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch;
+ U32 mlen;
+
+ DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u",
+ matchNb, matches[matchNb].off, lastML, litlen);
+
+ for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */
+ U32 const pos = cur + mlen;
+ int const price = (int)basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel);
+
+ if ((pos > last_pos) || (price < opt[pos].price)) {
+ DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)",
+ pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price));
+ while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */
+ opt[pos].mlen = mlen;
+ opt[pos].off = offset;
+ opt[pos].litlen = litlen;
+ opt[pos].price = price;
+ } else {
+ DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)",
+ pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price));
+ if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */
+ }
+ } } }
+ } /* for (cur = 1; cur <= last_pos; cur++) */
+
+ lastSequence = opt[last_pos];
+ cur = last_pos > ZSTD_totalLen(lastSequence) ? last_pos - ZSTD_totalLen(lastSequence) : 0; /* single sequence, and it starts before `ip` */
+ assert(cur < ZSTD_OPT_NUM); /* control overflow*/
+
+_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
+ assert(opt[0].mlen == 0);
+
+ /* Set the next chunk's repcodes based on the repcodes of the beginning
+ * of the last match, and the last sequence. This avoids us having to
+ * update them while traversing the sequences.
+ */
+ if (lastSequence.mlen != 0) {
+ repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0);
+ ZSTD_memcpy(rep, &reps, sizeof(reps));
+ } else {
+ ZSTD_memcpy(rep, opt[cur].rep, sizeof(repcodes_t));
+ }
+
+ { U32 const storeEnd = cur + 1;
+ U32 storeStart = storeEnd;
+ U32 seqPos = cur;
+
+ DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)",
+ last_pos, cur); (void)last_pos;
+ assert(storeEnd < ZSTD_OPT_NUM);
+ DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
+ storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off);
+ opt[storeEnd] = lastSequence;
+ while (seqPos > 0) {
+ U32 const backDist = ZSTD_totalLen(opt[seqPos]);
+ storeStart--;
+ DEBUGLOG(6, "sequence from rPos=%u copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
+ seqPos, storeStart, opt[seqPos].litlen, opt[seqPos].mlen, opt[seqPos].off);
+ opt[storeStart] = opt[seqPos];
+ seqPos = (seqPos > backDist) ? seqPos - backDist : 0;
+ }
+
+ /* save sequences */
+ DEBUGLOG(6, "sending selected sequences into seqStore")
+ { U32 storePos;
+ for (storePos=storeStart; storePos <= storeEnd; storePos++) {
+ U32 const llen = opt[storePos].litlen;
+ U32 const mlen = opt[storePos].mlen;
+ U32 const offBase = opt[storePos].off;
+ U32 const advance = llen + mlen;
+ DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u",
+ anchor - istart, (unsigned)llen, (unsigned)mlen);
+
+ if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */
+ assert(storePos == storeEnd); /* must be last sequence */
+ ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */
+ continue; /* will finish */
+ }
+
+ assert(anchor + llen <= iend);
+ ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen);
+ ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen);
+ anchor += advance;
+ ip = anchor;
+ } }
+ ZSTD_setBasePrices(optStatePtr, optLevel);
+ }
+ } /* while (ip < ilimit) */
+
+ /* Return the last literals size */
+ return (size_t)(iend - anchor);
+}
+
+static size_t ZSTD_compressBlock_opt0(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
+{
+ return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode);
+}
+
+static size_t ZSTD_compressBlock_opt2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
+{
+ return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode);
+}
+
+size_t ZSTD_compressBlock_btopt(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_compressBlock_btopt");
+ return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_noDict);
+}
+
+
+
+
+/* ZSTD_initStats_ultra():
+ * make a first compression pass, just to seed stats with more accurate starting values.
+ * only works on first block, with no dictionary and no ldm.
+ * this function cannot error out, its narrow contract must be respected.
+ */
+static void
+ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
+ seqStore_t* seqStore,
+ U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
+ ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep));
+
+ DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
+ assert(ms->opt.litLengthSum == 0); /* first block */
+ assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
+ assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
+ assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
+
+ ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict); /* generate stats into ms->opt*/
+
+ /* invalidate first scan from history, only keep entropy stats */
+ ZSTD_resetSeqStore(seqStore);
+ ms->window.base -= srcSize;
+ ms->window.dictLimit += (U32)srcSize;
+ ms->window.lowLimit = ms->window.dictLimit;
+ ms->nextToUpdate = ms->window.dictLimit;
+
+}
+
+size_t ZSTD_compressBlock_btultra(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
+ return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_btultra2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ U32 const curr = (U32)((const BYTE*)src - ms->window.base);
+ DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
+
+ /* 2-passes strategy:
+ * this strategy makes a first pass over first block to collect statistics
+ * in order to seed next round's statistics with it.
+ * After 1st pass, function forgets history, and starts a new block.
+ * Consequently, this can only work if no data has been previously loaded in tables,
+ * aka, no dictionary, no prefix, no ldm preprocessing.
+ * The compression ratio gain is generally small (~0.5% on first block),
+ ** the cost is 2x cpu time on first block. */
+ assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
+ if ( (ms->opt.litLengthSum==0) /* first block */
+ && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
+ && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
+ && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */
+ && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */
+ ) {
+ ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
+ }
+
+ return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict);
+}
+
+size_t ZSTD_compressBlock_btopt_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_btultra_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState);
+}
+
+size_t ZSTD_compressBlock_btopt_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict);
+}
+
+size_t ZSTD_compressBlock_btultra_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ const void* src, size_t srcSize)
+{
+ return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict);
+}
+
+/* note : no btultra2 variant for extDict nor dictMatchState,
+ * because btultra2 is not meant to work with dictionaries
+ * and is only specific for the first block (no prefix) */
diff --git a/contrib/zstd/zstd_opt.h b/contrib/zstd/zstd_opt.h
new file mode 100644
index 0000000..342e5a3
--- /dev/null
+++ b/contrib/zstd/zstd_opt.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_OPT_H
+#define ZSTD_OPT_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include "zstd_compress_internal.h"
+
+/* used in ZSTD_loadDictionaryContent() */
+void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend);
+
+size_t ZSTD_compressBlock_btopt(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_btultra(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_btultra2(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+
+size_t ZSTD_compressBlock_btopt_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_btultra_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+size_t ZSTD_compressBlock_btopt_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+size_t ZSTD_compressBlock_btultra_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize);
+
+ /* note : no btultra2 variant for extDict nor dictMatchState,
+ * because btultra2 is not meant to work with dictionaries
+ * and is only specific for the first block (no prefix) */
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_OPT_H */
diff --git a/contrib/zstd/zstd_trace.h b/contrib/zstd/zstd_trace.h
new file mode 100644
index 0000000..da20534
--- /dev/null
+++ b/contrib/zstd/zstd_trace.h
@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_TRACE_H
+#define ZSTD_TRACE_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include <stddef.h>
+
+/* weak symbol support
+ * For now, enable conservatively:
+ * - Only GNUC
+ * - Only ELF
+ * - Only x86-64, i386 and aarch64
+ * Also, explicitly disable on platforms known not to work so they aren't
+ * forgotten in the future.
+ */
+#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \
+ defined(__GNUC__) && defined(__ELF__) && \
+ (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(__aarch64__)) && \
+ !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
+ !defined(__CYGWIN__) && !defined(_AIX)
+# define ZSTD_HAVE_WEAK_SYMBOLS 1
+#else
+# define ZSTD_HAVE_WEAK_SYMBOLS 0
+#endif
+#if ZSTD_HAVE_WEAK_SYMBOLS
+# define ZSTD_WEAK_ATTR __attribute__((__weak__))
+#else
+# define ZSTD_WEAK_ATTR
+#endif
+
+/* Only enable tracing when weak symbols are available. */
+#ifndef ZSTD_TRACE
+# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
+#endif
+
+#if ZSTD_TRACE
+
+struct ZSTD_CCtx_s;
+struct ZSTD_DCtx_s;
+struct ZSTD_CCtx_params_s;
+
+typedef struct {
+ /**
+ * ZSTD_VERSION_NUMBER
+ *
+ * This is guaranteed to be the first member of ZSTD_trace.
+ * Otherwise, this struct is not stable between versions. If
+ * the version number does not match your expectation, you
+ * should not interpret the rest of the struct.
+ */
+ unsigned version;
+ /**
+ * Non-zero if streaming (de)compression is used.
+ */
+ unsigned streaming;
+ /**
+ * The dictionary ID.
+ */
+ unsigned dictionaryID;
+ /**
+ * Is the dictionary cold?
+ * Only set on decompression.
+ */
+ unsigned dictionaryIsCold;
+ /**
+ * The dictionary size or zero if no dictionary.
+ */
+ size_t dictionarySize;
+ /**
+ * The uncompressed size of the data.
+ */
+ size_t uncompressedSize;
+ /**
+ * The compressed size of the data.
+ */
+ size_t compressedSize;
+ /**
+ * The fully resolved CCtx parameters (NULL on decompression).
+ */
+ struct ZSTD_CCtx_params_s const* params;
+ /**
+ * The ZSTD_CCtx pointer (NULL on decompression).
+ */
+ struct ZSTD_CCtx_s const* cctx;
+ /**
+ * The ZSTD_DCtx pointer (NULL on compression).
+ */
+ struct ZSTD_DCtx_s const* dctx;
+} ZSTD_Trace;
+
+/**
+ * A tracing context. It must be 0 when tracing is disabled.
+ * Otherwise, any non-zero value returned by a tracing begin()
+ * function is presented to any subsequent calls to end().
+ *
+ * Any non-zero value is treated as tracing is enabled and not
+ * interpreted by the library.
+ *
+ * Two possible uses are:
+ * * A timestamp for when the begin() function was called.
+ * * A unique key identifying the (de)compression, like the
+ * address of the [dc]ctx pointer if you need to track
+ * more information than just a timestamp.
+ */
+typedef unsigned long long ZSTD_TraceCtx;
+
+/**
+ * Trace the beginning of a compression call.
+ * @param cctx The dctx pointer for the compression.
+ * It can be used as a key to map begin() to end().
+ * @returns Non-zero if tracing is enabled. The return value is
+ * passed to ZSTD_trace_compress_end().
+ */
+ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(
+ struct ZSTD_CCtx_s const* cctx);
+
+/**
+ * Trace the end of a compression call.
+ * @param ctx The return value of ZSTD_trace_compress_begin().
+ * @param trace The zstd tracing info.
+ */
+ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(
+ ZSTD_TraceCtx ctx,
+ ZSTD_Trace const* trace);
+
+/**
+ * Trace the beginning of a decompression call.
+ * @param dctx The dctx pointer for the decompression.
+ * It can be used as a key to map begin() to end().
+ * @returns Non-zero if tracing is enabled. The return value is
+ * passed to ZSTD_trace_compress_end().
+ */
+ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(
+ struct ZSTD_DCtx_s const* dctx);
+
+/**
+ * Trace the end of a decompression call.
+ * @param ctx The return value of ZSTD_trace_decompress_begin().
+ * @param trace The zstd tracing info.
+ */
+ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
+ ZSTD_TraceCtx ctx,
+ ZSTD_Trace const* trace);
+
+#endif /* ZSTD_TRACE */
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_TRACE_H */
diff --git a/dist.sh b/dist.sh
new file mode 100755
index 0000000..12e05a2
--- /dev/null
+++ b/dist.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+# Creates a tarball with the concatenation of a git tree and the submodules.
+# Hidden files such as .gitignore are skipped.
+
+# GNU tar
+TAR=${2:-"tar"}
+
+if [ $# -lt 1 ] ; then
+ echo "Usage: dist.sh <filename> [tar_command]"
+ exit 1
+fi
+
+FNAME=$1
+PREFIX=`basename $FNAME | sed -e 's/\.tar.*$//'`
+
+ALL_TAR=$(mktemp) || { echo "mktemp is missing!"; exit 1; }
+TMP_TAR=$(mktemp) || { echo "mktemp is missing!"; exit 1; }
+trap 'rm -f "$TMP_TAR" "$ALL_TAR"' EXIT
+
+# Create tarball for main repo contents.
+git archive --prefix="$PREFIX/" HEAD ":!.*" ":!**/.*" > "$ALL_TAR"
+
+# Append submodule contents, if any.
+export PREFIX TMP_TAR ALL_TAR
+git submodule --quiet foreach --recursive \
+ 'git archive --prefix="$PREFIX/$displaypath/" HEAD ":!.*" ":!**/.*" > "$TMP_TAR";
+ tar Af "$ALL_TAR" "$TMP_TAR"'
+
+xz < "$ALL_TAR" > "$FNAME"
diff --git a/doc/Makefile b/doc/Makefile
new file mode 100644
index 0000000..53ed9b6
--- /dev/null
+++ b/doc/Makefile
@@ -0,0 +1,94 @@
+# A simple makefile to generate documentation from .md using pandoc
+
+PANDOC ?= pandoc
+LUADOC ?= doxydown/doxydown.pl
+LLUADOC ?= ${LUADOC} -l lua -e lua
+
+all: man
+
+man: rspamd.8 rspamc.1 rspamadm.1
+
+rspamd.8: rspamd.8.md
+ $(PANDOC) -s -f markdown -t man -o rspamd.8 rspamd.8.md
+rspamc.1: rspamc.1.md
+ $(PANDOC) -s -f markdown -t man -o rspamc.1 rspamc.1.md
+rspamadm.1: rspamadm.1.md
+ $(PANDOC) -s -f markdown -t man -o rspamadm.1 rspamadm.1.md
+
+lua-dirs:
+ mkdir -p markdown/lua
+
+lua-doc: lua-dirs rspamd_regexp rspamd_ip rspamd_config rspamd_task ucl rspamd_http rspamd_trie \
+ rspamd_resolver rspamd_redis rspamd_upstream_list rspamd_expression rspamd_mimepart rspamd_logger rspamd_url \
+ rspamd_tcp rspamd_mempool rspamd_html rspamd_util rspamd_sqlite3 rspamd_cryptobox rspamd_map \
+ lua_redis lua_util lua_maps lua_clickhouse lua_selectors rspamd_udp rspamd_text lua_mime rspamd_parsers \
+ rspamd_cdb
+
+lua_redis:
+ $(LLUADOC) < ../lualib/lua_redis.lua > markdown/lua/lua_redis.md
+
+lua_util:
+ $(LLUADOC) < ../lualib/lua_util.lua > markdown/lua/lua_util.md
+
+lua_maps:
+ $(LLUADOC) < ../lualib/lua_maps.lua > markdown/lua/lua_maps.md
+
+lua_clickhouse:
+ $(LLUADOC) < ../lualib/lua_clickhouse.lua > markdown/lua/lua_clickhouse.md
+
+lua_selectors:
+ $(LLUADOC) < ../lualib/lua_selectors/init.lua > markdown/lua/lua_selectors.md
+
+lua_mime:
+ $(LLUADOC) < ../lualib/lua_mime.lua > markdown/lua/lua_mime.md
+
+rspamd_regexp: ../src/lua/lua_regexp.c
+ $(LUADOC) < ../src/lua/lua_regexp.c > markdown/lua/rspamd_regexp.md
+rspamd_ip: ../src/lua/lua_ip.c
+ $(LUADOC) < ../src/lua/lua_ip.c > markdown/lua/rspamd_ip.md
+rspamd_config: ../src/lua/lua_config.c
+ $(LUADOC) < ../src/lua/lua_config.c > markdown/lua/rspamd_config.md
+rspamd_task: ../src/lua/lua_task.c
+ $(LUADOC) < ../src/lua/lua_task.c > markdown/lua/rspamd_task.md
+ucl: ../contrib/libucl/lua_ucl.c
+ $(LUADOC) < ../contrib/libucl/lua_ucl.c > markdown/lua/ucl.md
+rspamd_http: ../src/lua/lua_http.c
+ $(LUADOC) < ../src/lua/lua_http.c > markdown/lua/rspamd_http.md
+rspamd_trie: ../src/lua/lua_trie.c
+ $(LUADOC) < ../src/lua/lua_trie.c > markdown/lua/rspamd_trie.md
+rspamd_resolver: ../src/lua/lua_dns_resolver.c
+ $(LUADOC) < ../src/lua/lua_dns_resolver.c > markdown/lua/rspamd_resolver.md
+rspamd_redis: ../src/lua/lua_redis.c
+ $(LUADOC) < ../src/lua/lua_redis.c > markdown/lua/rspamd_redis.md
+rspamd_upstream_list: ../src/lua/lua_upstream.c
+ $(LUADOC) < ../src/lua/lua_upstream.c > markdown/lua/rspamd_upstream.md
+rspamd_expression: ../src/lua/lua_expression.c
+ $(LUADOC) < ../src/lua/lua_expression.c > markdown/lua/rspamd_expression.md
+rspamd_mimepart: ../src/lua/lua_mimepart.c
+ $(LUADOC) < ../src/lua/lua_mimepart.c > markdown/lua/rspamd_mimepart.md
+rspamd_logger: ../src/lua/lua_logger.c
+ $(LUADOC) < ../src/lua/lua_logger.c > markdown/lua/rspamd_logger.md
+rspamd_url: ../src/lua/lua_url.c
+ $(LUADOC) < ../src/lua/lua_url.c > markdown/lua/rspamd_url.md
+rspamd_tcp: ../src/lua/lua_tcp.c
+ $(LUADOC) < ../src/lua/lua_tcp.c > markdown/lua/rspamd_tcp.md
+rspamd_mempool: ../src/lua/lua_mempool.c
+ $(LUADOC) < ../src/lua/lua_mempool.c > markdown/lua/rspamd_mempool.md
+rspamd_html: ../src/lua/lua_html.cxx
+ $(LUADOC) < ../src/lua/lua_html.cxx > markdown/lua/rspamd_html.md
+rspamd_util: ../src/lua/lua_util.c
+ $(LUADOC) < ../src/lua/lua_util.c > markdown/lua/rspamd_util.md
+rspamd_sqlite3: ../src/lua/lua_sqlite3.c
+ $(LUADOC) < ../src/lua/lua_sqlite3.c > markdown/lua/rspamd_sqlite3.md
+rspamd_cryptobox: ../src/lua/lua_cryptobox.c
+ $(LUADOC) < ../src/lua/lua_cryptobox.c > markdown/lua/rspamd_cryptobox.md
+rspamd_map: ../src/lua/lua_map.c
+ $(LUADOC) < ../src/lua/lua_map.c > markdown/lua/rspamd_map.md
+rspamd_udp: ../src/lua/lua_udp.c
+ $(LUADOC) < ../src/lua/lua_udp.c > markdown/lua/rspamd_udp.md
+rspamd_text: ../src/lua/lua_text.c
+ $(LUADOC) < ../src/lua/lua_text.c > markdown/lua/rspamd_text.md
+rspamd_parsers: ../src/lua/lua_parsers.c
+ $(LUADOC) < ../src/lua/lua_parsers.c > markdown/lua/rspamd_parsers.md
+rspamd_cdb: ../src/lua/lua_cdb.c
+ $(LUADOC) < ../src/lua/lua_cdb.c > markdown/lua/rspamd_cdb.md \ No newline at end of file
diff --git a/doc/doxydown/.gitignore b/doc/doxydown/.gitignore
new file mode 100644
index 0000000..eaca02e
--- /dev/null
+++ b/doc/doxydown/.gitignore
@@ -0,0 +1,19 @@
+/blib/
+/.build/
+_build/
+cover_db/
+inc/
+Build
+!Build/
+Build.bat
+.last_cover_stats
+/Makefile
+/Makefile.old
+/MANIFEST.bak
+/META.yml
+/META.json
+/MYMETA.*
+nytprof.out
+/pm_to_blib
+*.o
+*.bs
diff --git a/doc/doxydown/LICENSE b/doc/doxydown/LICENSE
new file mode 100644
index 0000000..d39277a
--- /dev/null
+++ b/doc/doxydown/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Vsevolod Stakhov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/doc/doxydown/README.md b/doc/doxydown/README.md
new file mode 100644
index 0000000..86d4c30
--- /dev/null
+++ b/doc/doxydown/README.md
@@ -0,0 +1,139 @@
+# Doxydown - documentation utility
+
+## Introduction
+
+Doxydown is an utility to convert `doxygen`-like comments from the source code to markdown.
+Unlike other documentation systems, `doxydown` is specifically designed to generate markdown output only.
+At the moment, doxydown can work with C and lua comments and produce kramdown/pandoc or github
+flavoured markdown. Doxydown produces output with anchors, links and table of content.
+It also can highlight syntax for examples in the documentation.
+
+### Why markdown
+
+Markdown is used by many contemporary engines and can be rendered to HTML using
+advanced templates, styles and scripts. Markdown provides excellent formatting
+capabilities while it doesn't require authors to be web designers to create
+documentation. Markdown is rendered by [`github`](https://github.com) and
+doxydown can generate documentation easily viewed directly inside github. Moreover,
+doxydown supports pandoc style of markdown and that means that markdown output
+can be converted to all formats supported by pandoc (html, pdf, latex,
+man pages and many others).
+
+### Why not `other documentation generator`
+
+Doxydown is extremely simple as it can output markdown only but it is very
+convenient tool to generate nice markdown with all features required from the
+documentation system. Doxydown uses input format that is very close to `doxygen`
+that allows to re-use the existing documentation comments. Currently, doxydown
+does not support many features but they could be easily added on demand.
+
+## Input format
+
+Doxydown extracts documentation from the comments blocks. The start of block is indicated by
+
+ /***
+
+in `C` or by
+
+ --[[[
+
+in `lua`. The end of documentation block is the normal multiline comment ending
+specific for the input language. Doxydown also strips an initial comment character,
+therefore the following inputs are equal:
+
+~~~c
+/***
+ * some text
+ * other text
+ *
+ */
+~~~
+and
+
+~~~c
+/***
+some text
+other text
+
+*/
+~~~
+
+Note that doxydown preserves empty lines and all markdown elements.
+
+### Documentation blocks
+
+Each documentation block describes either module or function/method. Modules are
+logical compounds of functions and methods. The difference between method and
+function is significant for languages with methods support (e.g. by `lua` via
+metatables). To define method or function you can use the following:
+
+ /***
+ @function my_awesome_function(param1[, param2])
+ This function is awesome.
+ */
+
+All text met in the current documentation block is used as function or method description.
+You can also define parameters and return values for functions and methods:
+
+ @param {type} param1 mandatory param
+
+Here, `{type}` is optional type description for a parameter, `param1` is parameter's name
+and the rest of the string is parameter description. Currently, you cannot split
+parameter description by newline character. In future versions of doxydown this might
+be fixed.
+
+You can specify return type of your function by using of `@return` tag:
+
+ @return {type} some cool result
+
+This tag is similar to `@param` and has the same limitation regarding newlines.
+You can also add some example code by using of `@example` tag:
+
+ @example
+ my_awesome_function('hello'); // returns 42
+
+All text after `@example` tag and until documentation block end is used as an example
+and is highlighted in markdown. Also you can switch the language of example by using
+the extended `@example` tag:
+
+ @example lua
+
+In this example, the code will be highlighted as `lua` code.
+
+Modules descriptions uses the same conventions, but `@param` and `@return` are
+meaningless for the modules. Function and methods blocks that follows some `@module`
+block are automatically attached to that module.
+
+Both modules and function can use links to other functions and methods by using of
+`@see` tag:
+
+ @see my_awesome_function
+
+This inserts a hyperlink to the specified function definition to the markdown.
+
+## Output format
+
+Doxydown can generate github flavoured markdown and pandoc/kramdown compatible
+markdown. The main difference is in how anchors are organized. In kramdown and
+pandoc it is possible to specify an explicit id for each header, whilst in
+GH flavoured markdown we can use only implicit anchors.
+
+### Examples
+
+You can see an example of github flavoured markdown render at
+[libucl github page](https://github.com/vstakhov/libucl/blob/master/doc/lua_api.md).
+The same page bu rendered by kramdown engine in `jekyll` platform can be
+accessed by [this address](https://rspamd.com/doc/lua/ucl.html).
+
+## Program invocation
+
+ doxydown [-hg] [-l language] < input_source > markdown.md
+
+* `-h`: help message
+* `-e`: sets default example language (default: lua)
+* `-l`: sets input language (default: c)
+* `-g`: use github flavoured markdown (default: kramdown/pandoc)
+
+## License
+
+Doxydown is published by terms of `MIT` license. \ No newline at end of file
diff --git a/doc/doxydown/doxydown.pl b/doc/doxydown/doxydown.pl
new file mode 100755
index 0000000..5002f58
--- /dev/null
+++ b/doc/doxydown/doxydown.pl
@@ -0,0 +1,624 @@
+#!/usr/bin/env perl
+
+$VERSION = "0.1.4";
+
+use strict;
+use warnings;
+use Data::Dumper;
+use Digest::MD5 qw(md5_hex);
+
+my @modules;
+my %options = ();
+my $cur_module;
+my $example_language = "lua";
+
+my %languages = (
+ c => {
+ start => qr/^\s*\/\*\*\*(?:\s*|(\s+\S.+\s*))$/,
+ end => qr/^\s*\*+\/\s*$/,
+ filter => qr/^(?:\s*\*+\s?)?(\s*\S.+)\s*$/,
+ },
+ lua => {
+ start => qr/^\s*\--(?:\[\[\[+|-+)\s*$/,
+ end => qr/^\s*--(:?\]\]+|-+)\s*$/,
+ filter => qr/^(?:\s*--!?\s?)?(\s*\S?.+)\s*$/,
+ },
+ sql => {
+ start => qr/^\s*\--(?:\[\[+|-+)\s*$/,
+ end => qr/^\s*--(:?\]\]+|-+)\s*$/,
+ filter => qr/^(?:\s*--\s?)?(\s*\S.+)\s*$/,
+ },
+ pl => {
+ start => qr/^\s*\##+\s*$/,
+ end => qr/^\s*##+\s*/,
+ filter => qr/^(?:\s*#+\s?)(\s*\S.+)\s*$/,
+ },
+);
+
+my $function_re = qr/^\s*\@(function|fn|method)\s*(\S.+)$/oi;
+my $struct_re = qr/^\s*\@(table|struct)\s*(\S.+)$/oi;
+my $module_re = qr/^\s*\@(?:module|file)\s*(\S.+)$/oi;
+my $language;
+
+# /function print_module_markdown
+sub print_module_markdown {
+ my ( $mname, $m ) = @_;
+ my $idline = $options{g} ? "" : " {#$m->{'id'}}";
+
+ print <<EOD;
+## Module `$mname`$idline
+
+$m->{'data'}
+EOD
+ if ( $m->{'example'} ) {
+ print <<EOD;
+
+### Example:
+
+~~~$m->{'example_language'}
+$m->{'example'}
+~~~
+EOD
+ }
+
+ sub print_func {
+ my ($f) = @_;
+
+ my $name = $f->{'name'};
+ my $id = $f->{'id'};
+
+ if ($f->{'brief'}) {
+ print "[`$name`](#$id) | ". $f->{'brief'} . "\n";
+ } else {
+ print "[`$name`](#$id) | No description\n";
+ }
+ }
+
+ sub print_table {
+ my ($f) = @_;
+
+ my $name = $f->{'name'};
+ my $id = $f->{'id'};
+
+ if ($f->{'brief'}) {
+ print "> [`$name`](#$id): ". $f->{'brief'} . "\n\n";
+ } else {
+ print "> [`$name`](#$id)\n\n";
+ }
+ }
+
+ print "\n### Brief content:\n\n";
+
+ if ($m->{'functions'}) {
+ if (scalar(@{ $m->{'functions'} }) > 0) {
+ print "**Functions**:\n\n";
+
+ print " Function | Description\n";
+ print "----------|------------\n";
+ foreach ( @{ $m->{'functions'} } ) {
+ print_func($_);
+ }
+ }
+ }
+
+ if ($m->{'methods'}) {
+ if (scalar(@{ $m->{'methods'} }) > 0) {
+ print "\n\n**Methods**:\n\n";
+ print " Method | Description\n";
+ print "----------|------------\n";
+ foreach ( @{ $m->{'methods'} } ) {
+ print_func($_);
+ }
+ }
+ }
+
+ if ($m->{'tables'}) {
+ if (scalar(@{ $m->{'tables'} }) > 0) {
+ print "\n\n**Tables**:\n\n";
+
+ foreach ( @{ $m->{'tables'} } ) {
+ print_table($_);
+ }
+ }
+ }
+
+ if ($m->{'structs'}) {
+ if (scalar(@{ $m->{'structs'} }) > 0) {
+ print "\n\n**Structs**:\n\n";
+
+ foreach ( @{ $m->{'structs'} } ) {
+ print_table($_);
+ }
+ }
+ }
+}
+
+# /function print_function_markdown
+sub print_function_markdown {
+ my ( $type, $fname, $f ) = @_;
+
+ my $idline = $options{g} ? "" : " {#$f->{'id'}}";
+ print <<EOD;
+### $type `$fname`$idline
+
+$f->{'data'}
+EOD
+ print "\n**Parameters:**\n\n";
+
+ if ( $f->{'params'} && scalar @{ $f->{'params'} } > 0 ) {
+ foreach ( @{ $f->{'params'} } ) {
+ if ( $_->{'type'} ) {
+ print
+ "- `$_->{'name'} \{$_->{'type'}\}`: $_->{'description'}\n";
+ } else {
+ print "- `$_->{'name'}`: $_->{'description'}\n";
+ }
+ }
+ } else {
+ print "No parameters\n";
+ }
+
+ print "\n**Returns:**\n\n";
+
+ if ( $f->{'returns'} && scalar @{ $f->{'returns'} } > 0 ) {
+ foreach ( @{ $f->{'returns'} } ) {
+ if ( $_->{'type'} ) {
+ print "- `\{$_->{'type'}\}`: $_->{'description'}\n";
+ } else {
+ print "- $_->{'description'}\n";
+ }
+ }
+ } else {
+ print "No return\n";
+ }
+
+ if ( $f->{'available'} ) {
+ printf "\n**Available in:** %s\n", $f->{'available'};
+ }
+
+ if ( $f->{'example'} ) {
+ print <<EOD;
+
+### Example:
+
+~~~$f->{'example_language'}
+$f->{'example'}
+~~~
+EOD
+ }
+}
+
+# /function print_struct_markdown
+sub print_struct_markdown {
+ my ( $type, $fname, $f ) = @_;
+
+ my $idline = $options{g} ? "" : " {#$f->{'id'}}";
+ print <<EOD;
+### $type `$fname`$idline
+
+$f->{'data'}
+EOD
+ print "\n**Elements:**\n\n";
+
+ if ( $f->{'params'} && scalar @{ $f->{'params'} } > 0 ) {
+ foreach ( @{ $f->{'params'} } ) {
+ if ( $_->{'type'} ) {
+ print
+ "- `$_->{'name'} \{$_->{'type'}\}`: $_->{'description'}\n";
+ } else {
+ print "- `$_->{'name'}`: $_->{'description'}\n";
+ }
+ }
+ } else {
+ print "No elements\n";
+ }
+
+ if ( $f->{'example'} ) {
+ print <<EOD;
+
+### Example:
+
+~~~$f->{'example_language'}
+$f->{'example'}
+~~~
+EOD
+ }
+}
+
+# /function print_markdown
+sub print_markdown {
+ for my $m (@modules) {
+ my $mname = $m->{name};
+
+ print_module_markdown( $mname, $m );
+
+ if ($m->{'functions'}) {
+ if ( scalar(@{ $m->{'functions'} }) > 0 ) {
+ print "\n## Functions\n\nThe module `$mname` defines the following functions.\n\n";
+
+ foreach ( @{ $m->{'functions'} } ) {
+ print_function_markdown( "Function", $_->{'name'}, $_ );
+
+ print "\nBack to [module description](#$m->{'id'}).\n\n";
+
+ }
+ }
+ }
+
+ if ($m->{'methods'}) {
+ if ( scalar(@{ $m->{'methods'} }) > 0 ) {
+ print "\n## Methods\n\nThe module `$mname` defines the following methods.\n\n";
+
+ foreach ( @{ $m->{'methods'} } ) {
+ print_function_markdown( "Method", $_->{'name'}, $_ );
+
+ print "\nBack to [module description](#$m->{'id'}).\n\n";
+
+ }
+ }
+ }
+
+ if ($m->{'tables'}) {
+ if ( scalar(@{ $m->{'tables'} }) > 0 ) {
+ print "\n## Tables\n\nThe module `$mname` defines the following tables.\n\n";
+
+ foreach ( @{ $m->{'tables'} } ) {
+ print_struct_markdown( "Table", $_->{'name'}, $_ );
+
+ print "\nBack to [module description](#$m->{'id'}).\n\n";
+
+ }
+ }
+ }
+
+ if ($m->{'structs'}) {
+ if ( scalar(@{ $m->{'structs'} }) > 0 ) {
+ print "\n## Structs\n\nThe module `$mname` defines the following structs.\n\n";
+
+ foreach ( @{ $m->{'structs'} } ) {
+ print_struct_markdown( "Struct", $_->{'name'}, $_ );
+
+ print "\nBack to [module description](#$m->{'id'}).\n\n";
+
+ }
+ }
+ }
+
+ print "\nBack to [top](#).\n\n";
+ }
+}
+
+# /function make_id
+sub make_id {
+ my ( $name, $prefix ) = @_;
+
+ if ( !$prefix ) {
+ $prefix = "f";
+ }
+
+ if ( !$options{g} ) {
+
+ # Kramdown/pandoc version of ID's
+ $name =~ /^(\S+).*$/;
+
+ return substr( substr( $prefix, 0, 1 ) . md5_hex($1), 0, 6 );
+ } else {
+ my $input = lc $prefix . "-" . $name;
+ my $id = join '-', split /\s+/, $input;
+
+ $id =~ s/[^\w_-]+//g;
+
+ return $id;
+ }
+}
+
+# /function substitute_data_keywords
+sub substitute_data_keywords {
+ my ($line) = @_;
+
+ if ( $line =~ /^.*\@see\s+(\S+)\s*.*$/ ) {
+ my $name = $1;
+ my $id = make_id($name);
+
+ return $line =~ s/\@see\s+\S+/[`$name`](#$id)/r;
+ }
+
+ return $line;
+}
+
+# /function parse_function
+sub parse_function {
+ my ( $func, @data ) = @_;
+
+ my ( $type, $name ) = ( $func =~ $function_re );
+ chomp $name;
+
+ my $f = {
+ name => $name,
+ data => '',
+ example => undef,
+ example_language => $example_language,
+ id => make_id( $name, $type ),
+ };
+ my $example = 0;
+
+ foreach ( @data ) {
+ if ( /^\s*\@param\s*(?:\{([^}]+)\})?\s*(\S+)\s*(.+)?\s*$/ ) {
+ my $p = {
+ name => $2,
+ type => $1 || "no type",
+ description => $3 || "no description"
+ };
+
+ push @{ $f->{'params'} }, $p;
+ } elsif ( /^\s*\@return\s*(?:\{([^}]+)\})?\s*(.+)?\s*$/ ) {
+ my $r = {
+ type => $1,
+ description => $2 || "no description"
+ };
+
+ push @{ $f->{'returns'} }, $r;
+ } elsif ( /^\s*\@brief\s*(\S.+)$/ ) {
+ $f->{'brief'} = $1;
+ } elsif ( /^\s*\@available\s*(\S.+)$/ ) {
+ $f->{'available'} = $1;
+ }
+ elsif ( /^\s*\@example\s*(\S)?\s*$/ ) {
+ $example = 1;
+ if ( $1 ) {
+ $f->{'example_language'} = $1;
+ }
+ } elsif ( $_ ne $func ) {
+ if ( $example ) {
+ $f->{'example'} .= $_;
+ } else {
+ $f->{'data'} .= substitute_data_keywords($_);
+ }
+ }
+ }
+
+ if ( $f->{'data'} ) {
+ chomp $f->{'data'};
+ } elsif ($f->{'brief'}) {
+ chomp $f->{'brief'};
+ $f->{'data'} = $f->{'brief'};
+ }
+
+ if ( $f->{'example'} ) {
+ chomp $f->{'example'};
+ }
+
+ if ( $f->{'available'} ) {
+ chomp $f->{'available'}
+ }
+
+ if ( !$f->{'brief'} && $f->{'data'} ) {
+
+
+ if ( $f->{'data'} =~ /^(.*?)(?:(?:[.:]\s|$)|\n).*/ ) {
+ $f->{'brief'} = "$1";
+ chomp $f->{'brief'};
+
+ if ( $f->{'brief'} !~ /\.$/) {
+ $f->{'brief'} .= ".";
+ }
+ }
+ }
+
+ if ( $type eq "method" ) {
+ push @{ $cur_module->{'methods'} }, $f;
+ } elsif ( $type eq "function" || $type eq "fn") {
+ push @{ $cur_module->{'functions'} }, $f;
+ }
+}
+
+# /function parse_struct
+sub parse_struct {
+ my ( $func, @data ) = @_;
+
+ my ( $type, $name ) = ( $func =~ $struct_re );
+ chomp $name;
+
+ my $f = {
+ name => $name,
+ data => '',
+ example => undef,
+ example_language => $example_language,
+ id => make_id( $name, $type ),
+ };
+ my $example = 0;
+
+ foreach ( @data ) {
+ if ( /^\s*\@param\s*(?:\{([^}]+)\})?\s*(\S+)\s*(.+)?\s*$/ ) {
+ my $p = {
+ name => $2,
+ type => $1,
+ description => $3
+ };
+
+ push @{ $f->{'params'} }, $p;
+ } elsif ( /^\s*\@brief\s*(\S.+)$/ ) {
+ $f->{'brief'} = $1;
+ } elsif ( /^\s*\@example\s*(\S)?\s*$/ ) {
+ $example = 1;
+ if ( $1 ) {
+ $f->{'example_language'} = $1;
+ }
+ } elsif ( $_ ne $func ) {
+ if ( $example ) {
+ $f->{'example'} .= $_;
+ } else {
+ $f->{'data'} .= substitute_data_keywords($_);
+ }
+ }
+ }
+
+ if ( $f->{'data'} ) {
+ chomp $f->{'data'};
+ } elsif ($f->{'brief'}) {
+ chomp $f->{'brief'};
+ $f->{'data'} = $f->{'brief'};
+ }
+
+ if ( $f->{'example'} ) {
+ chomp $f->{'example'};
+ }
+
+ if ( $type eq "table" ) {
+ push @{ $cur_module->{'tables'} }, $f;
+ } elsif ( $type eq "struct" ) {
+ push @{ $cur_module->{'structs'} }, $f;
+ }
+}
+
+# /function parse_module
+sub parse_module {
+ my ( $module, @data ) = @_;
+ my ( $name ) = ( $module =~ $module_re );
+
+ chomp $name;
+
+ my $f = {
+ name => $name,
+ functions => [],
+ methods => [],
+ data => '',
+ example => undef,
+ example_language => $example_language,
+ id => make_id( $name, "module" ),
+ };
+
+ my $example = 0;
+
+ foreach ( @data ) {
+ if ( /^\s*\@example\s*(\S)?\s*$/ ) {
+ $example = 1;
+ if ($1) {
+ $f->{'example_language'} = $1;
+ }
+ } elsif ( /^\s*\@brief\s*(\S.+)$/ ) {
+ $f->{'brief'} = $1;
+ } elsif ( $_ ne $module ) {
+ if ( $example ) {
+ $f->{'example'} .= $_;
+ } else {
+ $f->{'data'} .= substitute_data_keywords($_);
+ }
+ }
+ }
+
+ if ( $f->{'data'} ) {
+ chomp $f->{'data'};
+ } elsif ( $f->{'brief'} ) {
+ chomp $f->{'brief'};
+
+ $f->{'data'} = $f->{'brief'};
+ }
+
+ if ( $f->{'example'} ) {
+ chomp $f->{'example'};
+ }
+
+ $cur_module = $f;
+ push @modules, $f;
+}
+
+# /function parse_content
+sub parse_content {
+ #
+ my @func = grep /$function_re/, @_;
+ if ( scalar @func > 0 ) {
+ parse_function( $func[0], @_ );
+ }
+
+ #
+ my @struct = grep /$struct_re/, @_;
+ if ( scalar @struct > 0 ) {
+ parse_struct( $struct[0], @_ );
+ }
+
+ #
+ my @module = grep /$module_re/, @_;
+ if ( scalar @module > 0 ) {
+ parse_module( $module[0], @_ );
+ }
+}
+
+sub HELP_MESSAGE {
+ print STDERR <<EOF;
+Utility to convert doxygen comments to markdown.
+
+usage: $0 [-hg] [-l language] < input_source > markdown.md
+
+ -h : this (help) message
+ -e : sets default example language (default: lua)
+ -l : sets input language (default: c)
+ -g : use github flavoured markdown (default: kramdown/pandoc)
+EOF
+
+ exit;
+}
+
+$Getopt::Std::STANDARD_HELP_VERSION = 1;
+use Getopt::Std;
+
+getopts( 'he:gl:', \%options );
+
+HELP_MESSAGE() if $options{h};
+
+$example_language = $options{e} if $options{e};
+$language = $languages{ lc $options{l} } if $options{l};
+
+if ( !$language ) {
+ $language = $languages{c};
+}
+
+## TODO: select language based on file extension
+## TODO: change calling structure to allow looping through directory
+
+use constant {
+ STATE_READ_SKIP => 0,
+ STATE_READ_CONTENT => 1,
+ STATE_READ_ENUM => 2,
+ STATE_READ_STRUCT => 3,
+};
+
+my $state = STATE_READ_SKIP;
+my $content;
+
+while ( <> ) {
+ if ( $state == STATE_READ_SKIP ) {
+ if ( $_ =~ $language->{start} ) {
+ $state = STATE_READ_CONTENT;
+
+ if (defined($1)) {
+ chomp($content = $1);
+ $content =~ tr/\r//d;
+ $content .= "\n";
+ } else {
+ $content = "";
+ }
+ }
+ } elsif ( $state == STATE_READ_CONTENT ) {
+ if ( $_ =~ $language->{end} ) {
+ $state = STATE_READ_SKIP;
+
+ parse_content( split /^/, $content );
+
+ $content = "";
+ } else {
+ my ($line) = ( $_ =~ $language->{filter} );
+
+ if ( $line ) {
+ $line =~ tr/\r//d;
+ $content .= $line . "\n";
+ } else {
+ # Preserve empty lines
+ $content .= "\n";
+ }
+ }
+ }
+}
+
+#print Dumper( \@modules );
+print_markdown;
diff --git a/doc/rspamadm.1 b/doc/rspamadm.1
new file mode 100644
index 0000000..bd3b02a
--- /dev/null
+++ b/doc/rspamadm.1
@@ -0,0 +1,134 @@
+.\" Automatically generated by Pandoc 1.17.2
+.\"
+.TH "RSPAMADM" "1" "" "Rspamd User Manual" ""
+.hy
+.SH NAME
+.PP
+rspamadm \- rspamd administration utility
+.SH SYNOPSIS
+.PP
+rspamadm [\f[I]global_options\f[]] [\f[I]command\f[]]
+[\f[I]command_options\f[]]...
+.SH DESCRIPTION
+.PP
+\f[C]rspamadm\f[] is a routine to manage rspamd spam filtering system.
+It is intended to perform such actions as merging databases, performing
+configuration tests, encrypting passwords, signing configurations and so
+on.
+You can get a list of available \f[B]commands\f[] by running
+.IP
+.nf
+\f[C]
+rspamadm\ \-l
+\f[]
+.fi
+.PP
+Also for each command you can check list of available
+\f[B]command_options\f[] by running
+.IP
+.nf
+\f[C]
+rspamadm\ help\ command
+rspamadm\ command\ \-\-help
+\f[]
+.fi
+.SH OPTIONS
+.TP
+.B \-h, \-\-help
+Show help message
+.RS
+.RE
+.TP
+.B \-v, \-\-verbose
+Enable verbose output
+.RS
+.RE
+.TP
+.B \-l, \-\-list\-commands
+List available commands
+.RS
+.RE
+.TP
+.B \-\-version
+Show version
+.RS
+.RE
+.TP
+.B \-\-var=\f[I]value\f[]
+Redefine ucl variable in format \f[C]VARIABLE=VALUE\f[]
+.RS
+.RE
+.SH RETURN VALUE
+.PP
+On exit \f[C]rspamadm\f[] returns \f[C]0\f[] if operation was successful
+and an error code otherwise.
+.SH EXAMPLES
+.PP
+Get help for pw command:
+.IP
+.nf
+\f[C]
+rspamadm\ help\ pw
+rspamadm\ pw\ \-\-help
+\f[]
+.fi
+.PP
+Encrypt controller\[aq]s password:
+.IP
+.nf
+\f[C]
+rspamadm\ pw\ encrypt
+\f[]
+.fi
+.PP
+Merge fuzzy databases:
+.IP
+.nf
+\f[C]
+rspamadm\ fuzzy_merge\ \-s\ data1.sqlite\ \-s\ data2.sqlite\ \-t\ dest.sqlite
+\f[]
+.fi
+.PP
+Perform configuration test:
+.IP
+.nf
+\f[C]
+rspamadm\ configtest\ \-c\ rspamd.conf
+\f[]
+.fi
+.PP
+Test configuration strictly and redefine some ucl vars:
+.IP
+.nf
+\f[C]
+rspamadm\ \-\-var=DBDIR=/tmp\ configtest\ \-c\ ./rspamd.conf\ \-s
+\f[]
+.fi
+.PP
+Dump the processed configuration:
+.IP
+.nf
+\f[C]
+rspamadm\ configdump
+\f[]
+.fi
+.PP
+Dump the processed configuration as JSON string:
+.IP
+.nf
+\f[C]
+rspamadm\ configdump\ \-j
+\f[]
+.fi
+.PP
+Generate a keypair to use for HTTPCrypt encryption:
+.IP
+.nf
+\f[C]
+rspamadm\ keypair
+\f[]
+.fi
+.SH SEE ALSO
+.PP
+Rspamd documentation and source codes may be downloaded from
+<https://rspamd.com/>.
diff --git a/doc/rspamadm.1.md b/doc/rspamadm.1.md
new file mode 100644
index 0000000..83c6c3e
--- /dev/null
+++ b/doc/rspamadm.1.md
@@ -0,0 +1,85 @@
+% RSPAMADM(1) Rspamd User Manual
+
+# NAME
+
+rspamadm - rspamd administration utility
+
+# SYNOPSIS
+
+rspamadm [*global_options*] [*command*] [*command_options*]...
+
+# DESCRIPTION
+
+`rspamadm` is a routine to manage rspamd spam filtering system. It is intended to perform
+such actions as merging databases, performing configuration tests, encrypting passwords,
+signing configurations and so on. You can get a list of available **commands** by running
+
+ rspamadm -l
+
+Also for each command you can check list of available **command_options** by running
+
+ rspamadm help command
+ rspamadm command --help
+
+
+# OPTIONS
+
+-h, \--help
+: Show help message
+
+-v, \--verbose
+: Enable verbose output
+
+-l, \--list-commands
+: List available commands
+
+\--version
+: Show version
+
+\--var=*value*
+: Redefine ucl variable in format `VARIABLE=VALUE`
+
+# RETURN VALUE
+
+On exit `rspamadm` returns `0` if operation was successful and an error code otherwise.
+
+# EXAMPLES
+
+Get help for pw command:
+
+ rspamadm help pw
+ rspamadm pw --help
+
+Encrypt controller's password:
+
+ rspamadm pw encrypt
+
+Merge fuzzy databases:
+
+ rspamadm fuzzy_merge -s data1.sqlite -s data2.sqlite -t dest.sqlite
+
+Perform configuration test:
+
+ rspamadm configtest -c rspamd.conf
+
+Test configuration strictly and redefine some ucl vars:
+
+ rspamadm --var=DBDIR=/tmp configtest -c ./rspamd.conf -s
+
+
+Dump the processed configuration:
+
+ rspamadm configdump
+
+Dump the processed configuration as JSON string:
+
+ rspamadm configdump -j
+
+Generate a keypair to use for HTTPCrypt encryption:
+
+ rspamadm keypair
+
+# SEE ALSO
+
+Rspamd documentation and source codes may be downloaded from
+<https://rspamd.com/>.
diff --git a/doc/rspamc.1 b/doc/rspamc.1
new file mode 100644
index 0000000..0f823e6
--- /dev/null
+++ b/doc/rspamc.1
@@ -0,0 +1,307 @@
+.\" Automatically generated by Pandoc 2.2.2.1
+.\"
+.TH "RSPAMC" "1" "" "Rspamd User Manual" ""
+.hy
+.SH NAME
+.PP
+\f[C]rspamc\f[] \- rspamd command line client
+.SH SYNOPSIS
+.PP
+rspamc [\f[I]options\f[]] [\f[I]command\f[]] [\f[I]input\-file\f[]]\&...
+.PP
+rspamc \[en]help
+.SH DESCRIPTION
+.PP
+\f[C]rspamc\f[] is a simple rspamd client, primarily for classifying or
+learning messages.
+\f[C]rspamc\f[] supports the following commands:
+.IP \[bu] 2
+Scan commands:
+.RS 2
+.IP \[bu] 2
+\f[C]symbols\f[]: scan message and show symbols (default command)
+.RE
+.IP \[bu] 2
+Control commands
+.RS 2
+.IP \[bu] 2
+\f[C]learn_spam\f[]: learn message as spam
+.IP \[bu] 2
+\f[C]learn_ham\f[]: learn message as ham
+.IP \[bu] 2
+\f[C]fuzzy_add\f[]: add message to fuzzy storage (check \f[C]\-f\f[] and
+\f[C]\-w\f[] options for this command)
+.IP \[bu] 2
+\f[C]fuzzy_del\f[]: delete message from fuzzy storage (check
+\f[C]\-f\f[] option for this command)
+.IP \[bu] 2
+\f[C]stat\f[]: show rspamd statistics
+.IP \[bu] 2
+\f[C]stat_reset\f[]: show and reset rspamd statistics (useful for
+graphs)
+.IP \[bu] 2
+\f[C]counters\f[]: display rspamd symbols statistics
+.IP \[bu] 2
+\f[C]uptime\f[]: show rspamd uptime
+.IP \[bu] 2
+\f[C]add_symbol\f[]: add or modify symbol settings in rspamd
+.IP \[bu] 2
+\f[C]add_action\f[]: add or modify action settings
+.RE
+.PP
+Control commands that modify rspamd state are considered privileged and
+require a password to be specified with the \f[C]\-P\f[] option (see
+\f[B]OPTIONS\f[], below, for details).
+.PD 0
+.P
+.PD
+This depends on a controller's settings and is discussed in the
+\f[C]rspamd\-workers\f[] page (see \f[B]SEE ALSO\f[], below, for
+details).
+.PP
+\f[C]Input\ files\f[] may be either regular file(s) or a directory to
+scan.
+If no files are specified \f[C]rspamc\f[] reads from the standard input.
+Controller commands usually do not accept any input, however learn* and
+fuzzy* commands requires input.
+.SH OPTIONS
+.TP
+.B \-h \f[I]host[:port]\f[], \-\-connect=\f[I]host[:port]\f[]
+Specify host and port
+.RS
+.RE
+.TP
+.B \-P \f[I]password\f[], \-\-password=\f[I]password\f[]
+Specify control password
+.RS
+.RE
+.TP
+.B \-c \f[I]name\f[], \-\-classifier=\f[I]name\f[]
+Classifier to learn spam or ham (bayes is used by default)
+.RS
+.RE
+.TP
+.B \-w \f[I]weight\f[], \-\-weight=\f[I]weight\f[]
+Weight for fuzzy operations
+.RS
+.RE
+.TP
+.B \-f \f[I]number\f[], \-\-flag=\f[I]number\f[]
+Flag for fuzzy operations
+.RS
+.RE
+.TP
+.B \-p, \-\-pass
+Pass all filters
+.RS
+.RE
+.TP
+.B \-v, \-\-verbose
+More verbose output
+.RS
+.RE
+.TP
+.B \-i \f[I]ip address\f[], \-\-ip=\f[I]ip address\f[]
+Emulate that message was received from specified ip address
+.RS
+.RE
+.TP
+.B \-u \f[I]username\f[], \-\-user=\f[I]username\f[]
+Emulate that message was received from specified authenticated user
+.RS
+.RE
+.TP
+.B \-d \f[I]user\@domain\f[], \-\-deliver=\f[I]user\@domain\f[]
+Emulate that message was delivered to specified user (for
+LDA/statistics)
+.RS
+.RE
+.TP
+.B \-F \f[I]user\@domain\f[], \-\-from=\f[I]user\@domain\f[]
+Emulate that message has specified SMTP FROM address
+.RS
+.RE
+.TP
+.B \-r \f[I]user\@domain\f[], \-\-rcpt=\f[I]user\@domain\f[]
+Emulate that message has specified SMTP RCPT address
+.RS
+.RE
+.TP
+.B \-\-helo=\f[I]helo_string\f[]
+Imitate SMTP HELO passing from MTA
+.RS
+.RE
+.TP
+.B \-\-hostname=\f[I]hostname\f[]
+Imitate hostname passing from MTA (rspamd assumes that it is verified by
+MTA)
+.RS
+.RE
+.TP
+.B \-t \f[I]seconds\f[], \-\-timeout=\f[I]seconds\f[]
+Timeout for waiting for a reply (can be floating point number,
+e.g.\ 0.1)
+.RS
+.RE
+.TP
+.B \-b \f[I]host:port\f[], \-\-bind=\f[I]host:port\f[]
+Bind to specified ip address
+.RS
+.RE
+.TP
+.B \-R, \-\-human
+Output human readable report.
+The first line of the output contains the message score and three
+threshold scores, in this format:
+.IP
+.nf
+\f[C]
+ score/greylist/addheader/reject,action=N:ACTION,spam=0|1,skipped=0|1
+\f[]
+.fi
+.RS
+.RE
+.TP
+.B \-j, \-\-json
+Output formatted JSON
+.RS
+.RE
+.TP
+.B \-\-ucl
+Output UCL
+.RS
+.RE
+.TP
+.B \-\-raw
+Output raw data received from rspamd (compacted JSON)
+.RS
+.RE
+.TP
+.B \-\-headers
+Output HTTP headers from a reply
+.RS
+.RE
+.TP
+.B \-\-extended\-urls
+Output URLs in an extended format, showing full URL, host and the part
+of host that was used by surbl module (if enabled).
+.RS
+.RE
+.TP
+.B \-n \f[I]parallel_count\f[], \-\-max\-requests=\f[I]parallel_count\f[]
+Maximum number of requests to rspamd executed in parallel (8 by default)
+.RS
+.RE
+.TP
+.B \-e \f[I]command\f[], \-\-execute=\f[I]command\f[]
+Execute the specified command with either mime output (if \f[C]mime\f[]
+option is also specified) or formatted rspamd output
+.RS
+.RE
+.TP
+.B \-\-mime
+Output the full mime message instead of scanning results only
+.RS
+.RE
+.TP
+.B \-\-header=\f[I]header\f[]
+Add custom HTTP header for a request.
+You may specify header in format \f[C]name=value\f[] or just
+\f[C]name\f[] for an empty header.
+This option can be repeated multiple times.
+.RS
+.RE
+.TP
+.B \-\-sort=\f[I]type\f[]
+Sort output according to a specific field.
+For \f[C]counters\f[] command the allowed values for this key are
+\f[C]name\f[], \f[C]weight\f[], \f[C]frequency\f[] and \f[C]hits\f[].
+Appending \f[C]:desc\f[] to any of these types inverts sorting order.
+.RS
+.RE
+.TP
+.B \-\-commands
+List available commands
+.RS
+.RE
+.SH RETURN VALUE
+.PP
+On exit \f[C]rspamc\f[] returns \f[C]0\f[] if operation was successful
+and an error code otherwise.
+.SH EXAMPLES
+.PP
+Check stdin:
+.IP
+.nf
+\f[C]
+rspamc\ <\ some_file
+\f[]
+.fi
+.PP
+Check files:
+.IP
+.nf
+\f[C]
+rspamc\ symbols\ file1\ file2\ file3
+\f[]
+.fi
+.PP
+Learn files:
+.IP
+.nf
+\f[C]
+rspamc\ \-P\ pass\ learn_spam\ file1\ file2\ file3
+\f[]
+.fi
+.PP
+Add fuzzy hash to set 2:
+.IP
+.nf
+\f[C]
+rspamc\ \-P\ pass\ \-f\ 2\ \-w\ 10\ fuzzy_add\ file1\ file2
+\f[]
+.fi
+.PP
+Delete fuzzy hash from other server:
+.IP
+.nf
+\f[C]
+rspamc\ \-P\ pass\ \-h\ hostname:11334\ \-f\ 2\ fuzzy_del\ file1\ file2
+\f[]
+.fi
+.PP
+Get statistics:
+.IP
+.nf
+\f[C]
+rspamc\ stat
+\f[]
+.fi
+.PP
+Get uptime:
+.IP
+.nf
+\f[C]
+rspamc\ uptime
+\f[]
+.fi
+.PP
+Add custom rule's weight:
+.IP
+.nf
+\f[C]
+rspamc\ add_symbol\ test\ 1.5
+\f[]
+.fi
+.PP
+Add custom action's weight:
+.IP
+.nf
+\f[C]
+rspamc\ add_action\ reject\ 7.1
+\f[]
+.fi
+.SH SEE ALSO
+.PP
+Rspamd documentation and source code may be downloaded from
+<https://rspamd.com/>.
diff --git a/doc/rspamc.1.md b/doc/rspamc.1.md
new file mode 100644
index 0000000..335c225
--- /dev/null
+++ b/doc/rspamc.1.md
@@ -0,0 +1,173 @@
+% RSPAMC(1) Rspamd User Manual
+
+# NAME
+
+`rspamc` - rspamd command line client
+
+# SYNOPSIS
+
+rspamc [*options*] [*command*] [*input-file*]...
+
+rspamc --help
+
+# DESCRIPTION
+
+`rspamc` is a simple rspamd client, primarily for classifying or learning messages.
+`rspamc` supports the following commands:
+
+* Scan commands:
+ * `symbols`: scan message and show symbols (default command)
+* Control commands
+ * `learn_spam`: learn message as spam
+ * `learn_ham`: learn message as ham
+ * `fuzzy_add`: add message to fuzzy storage (check `-f` and `-w` options for this command)
+ * `fuzzy_del`: delete message from fuzzy storage (check `-f` option for this command)
+ * `stat`: show rspamd statistics
+ * `stat_reset`: show and reset rspamd statistics (useful for graphs)
+ * `counters`: display rspamd symbols statistics
+ * `uptime`: show rspamd uptime
+ * `add_symbol`: add or modify symbol settings in rspamd
+ * `add_action`: add or modify action settings
+
+Control commands that modify rspamd state are considered privileged and require a password to be specified with the `-P` option (see **OPTIONS**, below, for details).
+This depends on a controller's settings and is discussed in the `rspamd-workers` page (see **SEE ALSO**, below, for details).
+
+`Input files` may be either regular file(s) or a directory to scan. If no files are specified `rspamc` reads
+from the standard input. Controller commands usually do not accept any input, however learn* and fuzzy* commands
+requires input.
+
+# OPTIONS
+
+-h *host[:port]*, \--connect=*host[:port]*
+: Specify host and port
+
+-P *password*, \--password=*password*
+: Specify control password
+
+-c *name*, \--classifier=*name*
+: Classifier to learn spam or ham (bayes is used by default)
+
+-w *weight*, \--weight=*weight*
+: Weight for fuzzy operations
+
+-f *number*, \--flag=*number*
+: Flag for fuzzy operations
+
+-p, \--pass
+: Pass all filters
+
+-v, \--verbose
+: More verbose output
+
+-i *ip address*, \--ip=*ip address*
+: Emulate that message was received from specified ip address
+
+-u *username*, \--user=*username*
+: Emulate that message was received from specified authenticated user
+
+-d *user@domain*, \--deliver=*user@domain*
+: Emulate that message was delivered to specified user (for LDA/statistics)
+
+-F *user@domain*, \--from=*user@domain*
+: Emulate that message has specified SMTP FROM address
+
+-r *user@domain*, \--rcpt=*user@domain*
+: Emulate that message has specified SMTP RCPT address
+
+\--helo=*helo_string*
+: Imitate SMTP HELO passing from MTA
+
+\--hostname=*hostname*
+: Imitate hostname passing from MTA (rspamd assumes that it is verified by MTA)
+
+-t *seconds*, \--timeout=*seconds*
+: Timeout for waiting for a reply (can be floating point number, e.g. 0.1)
+
+-b *host:port*, \--bind=*host:port*
+: Bind to specified ip address
+
+-R, \--human
+: Output human readable report. The first line of the output contains the message score and three threshold scores, in this format:
+: score/greylist/addheader/reject,action=N:ACTION,spam=0|1,skipped=0|1
+
+-j, \--json
+: Output formatted JSON
+
+\--ucl
+: Output UCL
+
+\--raw
+: Output raw data received from rspamd (compacted JSON)
+
+\--headers
+: Output HTTP headers from a reply
+
+\--extended-urls
+: Output URLs in an extended format, showing full URL, host and the part of host that was used by surbl module (if enabled).
+
+-n *parallel_count*, \--max-requests=*parallel_count*
+: Maximum number of requests to rspamd executed in parallel (8 by default)
+
+-e *command*, \--execute=*command*
+: Execute the specified command with either mime output (if `mime` option is also specified) or formatted rspamd output
+
+\--mime
+: Output the full mime message instead of scanning results only
+
+\--header=*header*
+: Add custom HTTP header for a request. You may specify header in format `name=value` or just `name` for an empty header. This option can be repeated multiple times.
+
+\--sort=*type*
+: Sort output according to a specific field. For `counters` command the allowed values for this key are `name`, `weight`, `frequency` and `hits`. Appending `:asc` to any of these types inverts sorting order.
+
+\--commands
+: List available commands
+
+# RETURN VALUE
+
+On exit `rspamc` returns `0` if operation was successful and an error code otherwise.
+
+# EXAMPLES
+
+Check stdin:
+
+ rspamc < some_file
+
+Check files:
+
+ rspamc symbols file1 file2 file3
+
+Learn files:
+
+ rspamc -P pass learn_spam file1 file2 file3
+
+Add fuzzy hash to set 2:
+
+ rspamc -P pass -f 2 -w 10 fuzzy_add file1 file2
+
+Delete fuzzy hash from other server:
+
+ rspamc -P pass -h hostname:11334 -f 2 fuzzy_del file1 file2
+
+Get statistics:
+
+ rspamc stat
+
+Get uptime:
+
+ rspamc uptime
+
+Add custom rule's weight:
+
+ rspamc add_symbol test 1.5
+
+Add custom action's weight:
+
+ rspamc add_action reject 7.1
+
+# SEE ALSO
+
+Rspamd documentation and source code may be downloaded from
+<https://rspamd.com/>.
+
+[rspamd-workers]: https://rspamd.com/doc/workers/
diff --git a/doc/rspamd.8 b/doc/rspamd.8
new file mode 100644
index 0000000..325a507
--- /dev/null
+++ b/doc/rspamd.8
@@ -0,0 +1,78 @@
+.TH "RSPAMD" "8" "" "Rspamd User Manual" ""
+.SH NAME
+.PP
+rspamd \- main daemon for rapid spam filtering system
+.SH SYNOPSIS
+.PP
+rspamd [\f[I]options\f[]]...
+.PP
+rspamd \-\-help
+.SH DESCRIPTION
+.PP
+Rspamd filtering system is designed to be fast, modular and easily
+scalable system.
+Rspamd core is written in \f[C]C\f[] language using event driven
+processing model.
+Plugins for rspamd can be written in \f[C]Lua\f[] programming language.
+Rspamd is designed to process connections completely asynchronous and do
+not block anywhere in code.
+.SH OPTIONS
+.TP
+.B \-f, \-\-no\-fork
+Do not daemonize main process
+.RS
+.RE
+.TP
+.B \-c \f[I]path\f[], \-\-config=\f[I]path\f[]
+Specify config file(s)
+.RS
+.RE
+.TP
+.B \-u \f[I]username\f[], \-\-user=\f[I]username\f[]
+User to run rspamd as
+.RS
+.RE
+.TP
+.B \-g \f[I]groupname\f[], \-\-group=\f[I]groupname\f[]
+Group to run rspamd as
+.RS
+.RE
+.TP
+.B \-p \f[I]path\f[], \-\-pid=\f[I]path\f[]
+Path to pidfile
+.RS
+.RE
+.TP
+.B \-i, \-\-insecure
+Ignore running workers as privileged users (insecure)
+.RS
+.RE
+.SH EXAMPLES
+.PP
+Run rspamd daemon with default configuration:
+.IP
+.nf
+\f[C]
+rspamd
+\f[]
+.fi
+.PP
+Run rspamd in foreground with custom configuration:
+.IP
+.nf
+\f[C]
+rspamd\ \-f\ \-c\ ~/rspamd.conf
+\f[]
+.fi
+.PP
+Run rspamd specifying user and group:
+.IP
+.nf
+\f[C]
+rspamd\ \-u\ rspamd\ \-g\ rspamd\ \-c\ /etc/rspamd/rspamd.conf
+\f[]
+.fi
+.SH SEE ALSO
+.PP
+Rspamd documentation and source codes may be downloaded from
+<https://rspamd.com/>.
diff --git a/doc/rspamd.8.md b/doc/rspamd.8.md
new file mode 100644
index 0000000..c7c811c
--- /dev/null
+++ b/doc/rspamd.8.md
@@ -0,0 +1,57 @@
+% RSPAMD(8) Rspamd User Manual
+
+# NAME
+
+rspamd - main daemon for rapid spam filtering system
+
+# SYNOPSIS
+
+rspamd [*options*]...
+
+rspamd --help
+
+# DESCRIPTION
+
+Rspamd filtering system is designed to be fast, modular and easily scalable system.
+Rspamd core is written in `C` language using event driven processing model.
+Plugins for rspamd can be written in `Lua` programming language.
+Rspamd is designed to process connections completely asynchronous and do not block anywhere in code.
+
+# OPTIONS
+
+-f, \--no-fork
+: Do not daemonize main process
+
+-c *path*, \--config=*path*
+: Specify config file(s)
+
+-u *username*, \--user=*username*
+: User to run rspamd as
+
+-g *groupname*, \--group=*groupname*
+: Group to run rspamd as
+
+-p *path*, \--pid=*path*
+: Path to pidfile
+
+-i, \--insecure
+: Ignore running workers as privileged users (insecure)
+
+# EXAMPLES
+
+Run rspamd daemon with default configuration:
+
+ rspamd
+
+Run rspamd in foreground with custom configuration:
+
+ rspamd -f -c ~/rspamd.conf
+
+Run rspamd specifying user and group:
+
+ rspamd -u rspamd -g rspamd -c /etc/rspamd/rspamd.conf
+
+# SEE ALSO
+
+Rspamd documentation and source codes may be downloaded from
+<https://rspamd.com/>. \ No newline at end of file
diff --git a/docker/README.md b/docker/README.md
new file mode 100644
index 0000000..2b1b181
--- /dev/null
+++ b/docker/README.md
@@ -0,0 +1,3 @@
+# Rspamd Docker image 📨 ðŸ‹
+
+[See here](https://github.com/rspamd/rspamd-docker).
diff --git a/freebsd/cmake/PkgCreate.cmake b/freebsd/cmake/PkgCreate.cmake
new file mode 100644
index 0000000..547e8c9
--- /dev/null
+++ b/freebsd/cmake/PkgCreate.cmake
@@ -0,0 +1,56 @@
+# PkgCreate creates FreeBSD package for cmake
+# USAGE : ADD_FREEBSD_PACKAGE ( PKG_TARGET_NAME [DESCRIPTION] )
+
+FIND_PROGRAM(PKGCREATE
+ NAMES pkg_create
+ PATHS "/usr/sbin")
+
+IF ( PKGCREATE )
+ GET_FILENAME_COMPONENT(PKGCREATE_PATH ${PKGCREATE} ABSOLUTE)
+ MESSAGE(STATUS "Found pkg_create : ${PKGCREATE_PATH}")
+ SET(PKGCREATE_FOUND "YES")
+ELSE ( PKGCREATE )
+ MESSAGE(STATUS "pkg_create NOT found. package generation will not be available")
+ SET(PKGCREATE_FOUND "NO")
+ENDIF ( PKGCREATE )
+
+MACRO(ADD_FREEBSD_PACKAGE PKGNAME PLIST_FILE)
+ # let's create a directory to call 'make install DESTDIR=...' into:
+ SET ( FREEBSD_DIR ${CMAKE_BINARY_DIR}/${PACKAGE_NAME}_${PACKAGE_VERSION}-${PACKAGE_RELEASE} )
+ FILE ( REMOVE ${FREEBSD_DIR} )
+ FILE ( MAKE_DIRECTORY ${FREEBSD_DIR} )
+ FILE ( MAKE_DIRECTORY ${FREEBSD_DIR}/pkg )
+ # Calling "make install DESTDIR=${FREEBSD_DIR}"
+ ADD_CUSTOM_TARGET(pkg_destdir_install
+ COMMAND ${CMAKE_MAKE_PROGRAM} install CMAKE_INSTALL_PREFIX=/ DESTDIR=${FREEBSD_DIR}
+ DEPENDS ${CMAKE_BINARY_DIR}/cmake_install.cmake
+ COMMENT "Installing with DESTDIR = ${FREEBSD_DIR}"
+ )
+ ADD_DEPENDENCIES(pkg_destdir_install pkg_destdir_preinstall)
+ ADD_CUSTOM_TARGET(pkg_destdir_preinstall
+ COMMAND ${CMAKE_COMMAND} -DCMAKE_INSTALL_PREFIX=/ -DDESTDIR=${FREEBSD_DIR} .
+ DEPENDS ${CMAKE_BINARY_DIR}/cmake_install.cmake
+ COMMENT "Configuring with DESTDIR = ${FREEBSD_DIR}"
+ )
+ ADD_DEPENDENCIES(pkg_destdir_preinstall all preinstall)
+
+
+ ADD_CUSTOM_COMMAND(
+ OUTPUT ${CMAKE_BINARY_DIR}/${PACKAGE_NAME}_${PACKAGE_VERSION}-${PACKAGE_RELEASE}.tbz
+ COMMAND ${PKGCREATE_PATH} -c -"${PACKAGE_DESCRIPTION_SUMMARY}"
+ -d -"${PACKAGE_DESCRIPTION}"
+ -f ${PLIST_FILE}
+ -p ${FREEBSD_DIR}
+ ${CMAKE_BINARY_DIR}/${PACKAGE_NAME}_${PACKAGE_VERSION}-${PACKAGE_RELEASE}.tbz
+ # removing control, so its (re)generated each time we need to build the package
+ DEPENDS ${PLIST_FILE}
+ COMMENT "Generating freebsd package"
+ )
+
+ # the final target:
+ ADD_CUSTOM_TARGET(${PKGNAME}_pkg
+ DEPENDS ${CMAKE_BINARY_DIR}/${PACKAGE_NAME}_${PACKAGE_VERSION}-${PACKAGE_RELEASE}.tbz
+ )
+
+ ADD_DEPENDENCIES(${PKGNAME}_pkg pkg_destdir_install)
+ENDMACRO(ADD_FREEBSD_PACKAGE PKGNAME) \ No newline at end of file
diff --git a/freebsd/rspamd.sh.in b/freebsd/rspamd.sh.in
new file mode 100755
index 0000000..c527a6d
--- /dev/null
+++ b/freebsd/rspamd.sh.in
@@ -0,0 +1,52 @@
+#!/bin/sh
+#
+# $Id$
+#
+# PROVIDE: rspamd
+# REQUIRE: LOGIN
+# KEYWORD: shutdown
+
+#
+# Add the following line to /etc/rc.conf to enable countd:
+# rspamd (bool): Set to "NO" by default.
+# Set it to "YES" to enable rspamd.
+
+. /etc/rc.subr
+
+name="rspamd"
+rcvar=`set_rcvar`
+procname="@CMAKE_INSTALL_PREFIX@/bin/rspamd"
+restart_precmd="rspamd_checkconfig"
+reload_precmd="rspamd_checkconfig"
+configtest_cmd="rspamd_checkconfig"
+reopenlog_cmd="reopenlog_cmd"
+
+load_rc_config $name
+
+: ${rspamd_enable="NO"}
+: ${rspamd_pidfile="/var/run/rspamd/rspamd.pid"}
+: ${rspamd_run_user="@RSPAMD_USER@"}
+: ${rspamd_run_group="@RSPAMD_GROUP@"}
+
+pidfile="$rspamd_pidfile"
+
+stop_postcmd="rm -f $rspamd_pidfile"
+
+rspamd_checkconfig()
+{
+ echo "Performing sanity check on rspamd configuration:"
+ eval ${command} -t
+}
+
+reopenlog_cmd()
+{
+ pkill -USR1 -F $rspamd_pidfile
+}
+
+extra_commands="reload configtest reopenlog"
+sig_reload="HUP"
+
+command="$procname"
+command_args="-u ${rspamd_run_user} -g ${rspamd_run_group} -c @CMAKE_INSTALL_PREFIX@/etc/rspamd.xml"
+
+run_rc_command "$1"
diff --git a/interface/README.md b/interface/README.md
new file mode 100644
index 0000000..a56aa66
--- /dev/null
+++ b/interface/README.md
@@ -0,0 +1,51 @@
+# Rspamd web interface
+
+## Overview
+
+This is a simple control interface for rspamd spam filtering system.
+It provides basic functions for setting metric actions, scores,
+viewing statistic and learning.
+
+<img src="https://rspamd.com/img/webui.png" class="img-responsive" alt="Webui screenshot">
+<img src="https://rspamd.com/img/webui_throughput.png" class="img-responsive" alt="Webui screenshot">
+
+## Rspamd setup
+
+It is required to configure dynamic settings to store configured values.
+Basically this can be done by providing the following line in options settings:
+
+~~~ucl
+options {
+ dynamic_conf = "/var/lib/rspamd/rspamd_dynamic";
+}
+~~~
+
+Please note that this path must have write access for rspamd user.
+
+Then controller worker should be configured:
+
+~~~ucl
+worker {
+ type = "controller";
+ bind_socket = "localhost:11334";
+ count = 1;
+ # Password for normal commands (use rspamadm pw)
+ password = "$2$anydoddx67ggcs74owybhcwqsq3z67q4$udympbo8pfcfqkeiiuj7gegabk5jpt8edmhseujhar9ooyuzig5b";
+ # Password for privileged commands (use rspamadm pw)
+ enable_password = "$2$nx6sqkxtewx9c5s3hxjmabaxdcr46pk9$45qajkbyqx77abapiqugpjpsojj38zcqn7xnp3ekqyu674koux4b";
+ # Path to webiu static files
+ static_dir = "${WWWDIR}";
+}
+~~~
+
+Password option should be changed for sure for your specific configuration. Encrypted password using is encouraged (`rspamadm pw --encrypt`).
+
+## Interface setup
+
+Interface itself is written in pure HTML5/js and, hence, it requires zero setup.
+Just enter a password for webui access and you are ready.
+
+## Contact information
+
+Rspamd interface is distributed under the terms of [MIT license](http://opensource.org/licenses/MIT). For all questions related to this
+product please see the [support page](https://rspamd.com/support.html)
diff --git a/interface/apple-touch-icon.png b/interface/apple-touch-icon.png
new file mode 100644
index 0000000..a8054ac
--- /dev/null
+++ b/interface/apple-touch-icon.png
Binary files differ
diff --git a/interface/browserconfig.xml b/interface/browserconfig.xml
new file mode 100644
index 0000000..31789c9
--- /dev/null
+++ b/interface/browserconfig.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig>
+ <msapplication>
+ <tile>
+ <square150x150logo src="/mstile-150x150.png?v=3"/>
+ <TileColor>#2b5797</TileColor>
+ </tile>
+ </msapplication>
+</browserconfig>
diff --git a/interface/css/FooTable.Glyphicons.css b/interface/css/FooTable.Glyphicons.css
new file mode 100644
index 0000000..c68f66e
--- /dev/null
+++ b/interface/css/FooTable.Glyphicons.css
@@ -0,0 +1,62 @@
+/* Glyphicons Icons - We're not actually using Glyphicons classes but instead provide a simple mapping
+ from Glyphicons to FooTable class names. */
+.fooicon {
+ position: relative;
+ top: 1px;
+ display: inline-block;
+ /* stylelint-disable-next-line font-family-no-missing-generic-family-keyword */
+ font-family: "Glyphicons Halflings" !important;
+ font-style: normal;
+ font-weight: 400;
+ line-height: 1;
+ -webkit-font-smoothing: antialiased;
+ -moz-osx-font-smoothing: grayscale;
+}
+.fooicon::before,
+.fooicon::after {
+ -webkit-box-sizing: border-box;
+ -moz-box-sizing: border-box;
+ box-sizing: border-box;
+}
+.fooicon-loader::before {
+ content: "\e030";
+}
+.fooicon-plus::before {
+ content: "\2b";
+}
+.fooicon-minus::before {
+ content: "\2212";
+}
+.fooicon-search::before {
+ content: "\e003";
+}
+.fooicon-remove::before {
+ content: "\e014";
+}
+.fooicon-sort::before {
+ content: "\e150";
+}
+.fooicon-sort-asc::before {
+ content: "\e155";
+}
+.fooicon-sort-desc::before {
+ content: "\e156";
+}
+.fooicon-pencil::before {
+ content: "\270f";
+}
+.fooicon-trash::before {
+ content: "\e020";
+}
+.fooicon-eye-close::before {
+ content: "\e106";
+}
+.fooicon-flash::before {
+ content: "\e162";
+}
+.fooicon-cog::before {
+ content: "\e019";
+}
+.fooicon-stats::before {
+ content: "\e185";
+}
diff --git a/interface/css/bootstrap.min.css b/interface/css/bootstrap.min.css
new file mode 100644
index 0000000..f5910ac
--- /dev/null
+++ b/interface/css/bootstrap.min.css
@@ -0,0 +1,6 @@
+@charset "UTF-8";/*!
+ * Bootstrap v5.3.2 (https://getbootstrap.com/)
+ * Copyright 2011-2023 The Bootstrap Authors
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)
+ */:root,[data-bs-theme=light]{--bs-blue:#0d6efd;--bs-indigo:#6610f2;--bs-purple:#6f42c1;--bs-pink:#d63384;--bs-red:#dc3545;--bs-orange:#fd7e14;--bs-yellow:#ffc107;--bs-green:#198754;--bs-teal:#20c997;--bs-cyan:#0dcaf0;--bs-black:#000;--bs-white:#fff;--bs-gray:#6c757d;--bs-gray-dark:#343a40;--bs-gray-100:#f8f9fa;--bs-gray-200:#e9ecef;--bs-gray-300:#dee2e6;--bs-gray-400:#ced4da;--bs-gray-500:#adb5bd;--bs-gray-600:#6c757d;--bs-gray-700:#495057;--bs-gray-800:#343a40;--bs-gray-900:#212529;--bs-primary:#0d6efd;--bs-secondary:#6c757d;--bs-success:#198754;--bs-info:#0dcaf0;--bs-warning:#ffc107;--bs-danger:#dc3545;--bs-light:#f8f9fa;--bs-dark:#212529;--bs-primary-rgb:13,110,253;--bs-secondary-rgb:108,117,125;--bs-success-rgb:25,135,84;--bs-info-rgb:13,202,240;--bs-warning-rgb:255,193,7;--bs-danger-rgb:220,53,69;--bs-light-rgb:248,249,250;--bs-dark-rgb:33,37,41;--bs-primary-text-emphasis:#052c65;--bs-secondary-text-emphasis:#2b2f32;--bs-success-text-emphasis:#0a3622;--bs-info-text-emphasis:#055160;--bs-warning-text-emphasis:#664d03;--bs-danger-text-emphasis:#58151c;--bs-light-text-emphasis:#495057;--bs-dark-text-emphasis:#495057;--bs-primary-bg-subtle:#cfe2ff;--bs-secondary-bg-subtle:#e2e3e5;--bs-success-bg-subtle:#d1e7dd;--bs-info-bg-subtle:#cff4fc;--bs-warning-bg-subtle:#fff3cd;--bs-danger-bg-subtle:#f8d7da;--bs-light-bg-subtle:#fcfcfd;--bs-dark-bg-subtle:#ced4da;--bs-primary-border-subtle:#9ec5fe;--bs-secondary-border-subtle:#c4c8cb;--bs-success-border-subtle:#a3cfbb;--bs-info-border-subtle:#9eeaf9;--bs-warning-border-subtle:#ffe69c;--bs-danger-border-subtle:#f1aeb5;--bs-light-border-subtle:#e9ecef;--bs-dark-border-subtle:#adb5bd;--bs-white-rgb:255,255,255;--bs-black-rgb:0,0,0;--bs-font-sans-serif:system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue","Noto Sans","Liberation Sans",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--bs-font-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;--bs-gradient:linear-gradient(180deg, rgba(255, 255, 255, 0.15), rgba(255, 255, 255, 0));--bs-body-font-family:var(--bs-font-sans-serif);--bs-body-font-size:1rem;--bs-body-font-weight:400;--bs-body-line-height:1.5;--bs-body-color:#212529;--bs-body-color-rgb:33,37,41;--bs-body-bg:#fff;--bs-body-bg-rgb:255,255,255;--bs-emphasis-color:#000;--bs-emphasis-color-rgb:0,0,0;--bs-secondary-color:rgba(33, 37, 41, 0.75);--bs-secondary-color-rgb:33,37,41;--bs-secondary-bg:#e9ecef;--bs-secondary-bg-rgb:233,236,239;--bs-tertiary-color:rgba(33, 37, 41, 0.5);--bs-tertiary-color-rgb:33,37,41;--bs-tertiary-bg:#f8f9fa;--bs-tertiary-bg-rgb:248,249,250;--bs-heading-color:inherit;--bs-link-color:#0d6efd;--bs-link-color-rgb:13,110,253;--bs-link-decoration:underline;--bs-link-hover-color:#0a58ca;--bs-link-hover-color-rgb:10,88,202;--bs-code-color:#d63384;--bs-highlight-color:#212529;--bs-highlight-bg:#fff3cd;--bs-border-width:1px;--bs-border-style:solid;--bs-border-color:#dee2e6;--bs-border-color-translucent:rgba(0, 0, 0, 0.175);--bs-border-radius:0.375rem;--bs-border-radius-sm:0.25rem;--bs-border-radius-lg:0.5rem;--bs-border-radius-xl:1rem;--bs-border-radius-xxl:2rem;--bs-border-radius-2xl:var(--bs-border-radius-xxl);--bs-border-radius-pill:50rem;--bs-box-shadow:0 0.5rem 1rem rgba(0, 0, 0, 0.15);--bs-box-shadow-sm:0 0.125rem 0.25rem rgba(0, 0, 0, 0.075);--bs-box-shadow-lg:0 1rem 3rem rgba(0, 0, 0, 0.175);--bs-box-shadow-inset:inset 0 1px 2px rgba(0, 0, 0, 0.075);--bs-focus-ring-width:0.25rem;--bs-focus-ring-opacity:0.25;--bs-focus-ring-color:rgba(13, 110, 253, 0.25);--bs-form-valid-color:#198754;--bs-form-valid-border-color:#198754;--bs-form-invalid-color:#dc3545;--bs-form-invalid-border-color:#dc3545}[data-bs-theme=dark]{color-scheme:dark;--bs-body-color:#dee2e6;--bs-body-color-rgb:222,226,230;--bs-body-bg:#212529;--bs-body-bg-rgb:33,37,41;--bs-emphasis-color:#fff;--bs-emphasis-color-rgb:255,255,255;--bs-secondary-color:rgba(222, 226, 230, 0.75);--bs-secondary-color-rgb:222,226,230;--bs-secondary-bg:#343a40;--bs-secondary-bg-rgb:52,58,64;--bs-tertiary-color:rgba(222, 226, 230, 0.5);--bs-tertiary-color-rgb:222,226,230;--bs-tertiary-bg:#2b3035;--bs-tertiary-bg-rgb:43,48,53;--bs-primary-text-emphasis:#6ea8fe;--bs-secondary-text-emphasis:#a7acb1;--bs-success-text-emphasis:#75b798;--bs-info-text-emphasis:#6edff6;--bs-warning-text-emphasis:#ffda6a;--bs-danger-text-emphasis:#ea868f;--bs-light-text-emphasis:#f8f9fa;--bs-dark-text-emphasis:#dee2e6;--bs-primary-bg-subtle:#031633;--bs-secondary-bg-subtle:#161719;--bs-success-bg-subtle:#051b11;--bs-info-bg-subtle:#032830;--bs-warning-bg-subtle:#332701;--bs-danger-bg-subtle:#2c0b0e;--bs-light-bg-subtle:#343a40;--bs-dark-bg-subtle:#1a1d20;--bs-primary-border-subtle:#084298;--bs-secondary-border-subtle:#41464b;--bs-success-border-subtle:#0f5132;--bs-info-border-subtle:#087990;--bs-warning-border-subtle:#997404;--bs-danger-border-subtle:#842029;--bs-light-border-subtle:#495057;--bs-dark-border-subtle:#343a40;--bs-heading-color:inherit;--bs-link-color:#6ea8fe;--bs-link-hover-color:#8bb9fe;--bs-link-color-rgb:110,168,254;--bs-link-hover-color-rgb:139,185,254;--bs-code-color:#e685b5;--bs-highlight-color:#dee2e6;--bs-highlight-bg:#664d03;--bs-border-color:#495057;--bs-border-color-translucent:rgba(255, 255, 255, 0.15);--bs-form-valid-color:#75b798;--bs-form-valid-border-color:#75b798;--bs-form-invalid-color:#ea868f;--bs-form-invalid-border-color:#ea868f}*,::after,::before{box-sizing:border-box}@media (prefers-reduced-motion:no-preference){:root{scroll-behavior:smooth}}body{margin:0;font-family:var(--bs-body-font-family);font-size:var(--bs-body-font-size);font-weight:var(--bs-body-font-weight);line-height:var(--bs-body-line-height);color:var(--bs-body-color);text-align:var(--bs-body-text-align);background-color:var(--bs-body-bg);-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}hr{margin:1rem 0;color:inherit;border:0;border-top:var(--bs-border-width) solid;opacity:.25}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem;font-weight:500;line-height:1.2;color:var(--bs-heading-color)}.h1,h1{font-size:calc(1.375rem + 1.5vw)}@media (min-width:1200px){.h1,h1{font-size:2.5rem}}.h2,h2{font-size:calc(1.325rem + .9vw)}@media (min-width:1200px){.h2,h2{font-size:2rem}}.h3,h3{font-size:calc(1.3rem + .6vw)}@media (min-width:1200px){.h3,h3{font-size:1.75rem}}.h4,h4{font-size:calc(1.275rem + .3vw)}@media (min-width:1200px){.h4,h4{font-size:1.5rem}}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}p{margin-top:0;margin-bottom:1rem}abbr[title]{-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}ol,ul{padding-left:2rem}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}.small,small{font-size:.875em}.mark,mark{padding:.1875em;color:var(--bs-highlight-color);background-color:var(--bs-highlight-bg)}sub,sup{position:relative;font-size:.75em;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:rgba(var(--bs-link-color-rgb),var(--bs-link-opacity,1));text-decoration:underline}a:hover{--bs-link-color-rgb:var(--bs-link-hover-color-rgb)}a:not([href]):not([class]),a:not([href]):not([class]):hover{color:inherit;text-decoration:none}code,kbd,pre,samp{font-family:var(--bs-font-monospace);font-size:1em}pre{display:block;margin-top:0;margin-bottom:1rem;overflow:auto;font-size:.875em}pre code{font-size:inherit;color:inherit;word-break:normal}code{font-size:.875em;color:var(--bs-code-color);word-wrap:break-word}a>code{color:inherit}kbd{padding:.1875rem .375rem;font-size:.875em;color:var(--bs-body-bg);background-color:var(--bs-body-color);border-radius:.25rem}kbd kbd{padding:0;font-size:1em}figure{margin:0 0 1rem}img,svg{vertical-align:middle}table{caption-side:bottom;border-collapse:collapse}caption{padding-top:.5rem;padding-bottom:.5rem;color:var(--bs-secondary-color);text-align:left}th{text-align:inherit;text-align:-webkit-match-parent}tbody,td,tfoot,th,thead,tr{border-color:inherit;border-style:solid;border-width:0}label{display:inline-block}button{border-radius:0}button:focus:not(:focus-visible){outline:0}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,select{text-transform:none}[role=button]{cursor:pointer}select{word-wrap:normal}select:disabled{opacity:1}[list]:not([type=date]):not([type=datetime-local]):not([type=month]):not([type=week]):not([type=time])::-webkit-calendar-picker-indicator{display:none!important}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}::-moz-focus-inner{padding:0;border-style:none}textarea{resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{float:left;width:100%;padding:0;margin-bottom:.5rem;font-size:calc(1.275rem + .3vw);line-height:inherit}@media (min-width:1200px){legend{font-size:1.5rem}}legend+*{clear:left}::-webkit-datetime-edit-day-field,::-webkit-datetime-edit-fields-wrapper,::-webkit-datetime-edit-hour-field,::-webkit-datetime-edit-minute,::-webkit-datetime-edit-month-field,::-webkit-datetime-edit-text,::-webkit-datetime-edit-year-field{padding:0}::-webkit-inner-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-color-swatch-wrapper{padding:0}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}::file-selector-button{font:inherit;-webkit-appearance:button}output{display:inline-block}iframe{border:0}summary{display:list-item;cursor:pointer}progress{vertical-align:baseline}[hidden]{display:none!important}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:calc(1.625rem + 4.5vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-1{font-size:5rem}}.display-2{font-size:calc(1.575rem + 3.9vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-2{font-size:4.5rem}}.display-3{font-size:calc(1.525rem + 3.3vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-3{font-size:4rem}}.display-4{font-size:calc(1.475rem + 2.7vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-4{font-size:3.5rem}}.display-5{font-size:calc(1.425rem + 2.1vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-5{font-size:3rem}}.display-6{font-size:calc(1.375rem + 1.5vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-6{font-size:2.5rem}}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:.875em;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote>:last-child{margin-bottom:0}.blockquote-footer{margin-top:-1rem;margin-bottom:1rem;font-size:.875em;color:#6c757d}.blockquote-footer::before{content:"— "}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:var(--bs-body-bg);border:var(--bs-border-width) solid var(--bs-border-color);border-radius:var(--bs-border-radius);max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:.875em;color:var(--bs-secondary-color)}.container,.container-fluid,.container-lg,.container-md,.container-sm,.container-xl,.container-xxl{--bs-gutter-x:1.5rem;--bs-gutter-y:0;width:100%;padding-right:calc(var(--bs-gutter-x) * .5);padding-left:calc(var(--bs-gutter-x) * .5);margin-right:auto;margin-left:auto}@media (min-width:576px){.container,.container-sm{max-width:540px}}@media (min-width:768px){.container,.container-md,.container-sm{max-width:720px}}@media (min-width:992px){.container,.container-lg,.container-md,.container-sm{max-width:960px}}@media (min-width:1200px){.container,.container-lg,.container-md,.container-sm,.container-xl{max-width:1140px}}@media (min-width:1400px){.container,.container-lg,.container-md,.container-sm,.container-xl,.container-xxl{max-width:1320px}}:root{--bs-breakpoint-xs:0;--bs-breakpoint-sm:576px;--bs-breakpoint-md:768px;--bs-breakpoint-lg:992px;--bs-breakpoint-xl:1200px;--bs-breakpoint-xxl:1400px}.row{--bs-gutter-x:1.5rem;--bs-gutter-y:0;display:flex;flex-wrap:wrap;margin-top:calc(-1 * var(--bs-gutter-y));margin-right:calc(-.5 * var(--bs-gutter-x));margin-left:calc(-.5 * var(--bs-gutter-x))}.row>*{flex-shrink:0;width:100%;max-width:100%;padding-right:calc(var(--bs-gutter-x) * .5);padding-left:calc(var(--bs-gutter-x) * .5);margin-top:var(--bs-gutter-y)}.col{flex:1 0 0%}.row-cols-auto>*{flex:0 0 auto;width:auto}.row-cols-1>*{flex:0 0 auto;width:100%}.row-cols-2>*{flex:0 0 auto;width:50%}.row-cols-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-4>*{flex:0 0 auto;width:25%}.row-cols-5>*{flex:0 0 auto;width:20%}.row-cols-6>*{flex:0 0 auto;width:16.66666667%}.col-auto{flex:0 0 auto;width:auto}.col-1{flex:0 0 auto;width:8.33333333%}.col-2{flex:0 0 auto;width:16.66666667%}.col-3{flex:0 0 auto;width:25%}.col-4{flex:0 0 auto;width:33.33333333%}.col-5{flex:0 0 auto;width:41.66666667%}.col-6{flex:0 0 auto;width:50%}.col-7{flex:0 0 auto;width:58.33333333%}.col-8{flex:0 0 auto;width:66.66666667%}.col-9{flex:0 0 auto;width:75%}.col-10{flex:0 0 auto;width:83.33333333%}.col-11{flex:0 0 auto;width:91.66666667%}.col-12{flex:0 0 auto;width:100%}.offset-1{margin-left:8.33333333%}.offset-2{margin-left:16.66666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.33333333%}.offset-5{margin-left:41.66666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.33333333%}.offset-8{margin-left:66.66666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.33333333%}.offset-11{margin-left:91.66666667%}.g-0,.gx-0{--bs-gutter-x:0}.g-0,.gy-0{--bs-gutter-y:0}.g-1,.gx-1{--bs-gutter-x:0.25rem}.g-1,.gy-1{--bs-gutter-y:0.25rem}.g-2,.gx-2{--bs-gutter-x:0.5rem}.g-2,.gy-2{--bs-gutter-y:0.5rem}.g-3,.gx-3{--bs-gutter-x:1rem}.g-3,.gy-3{--bs-gutter-y:1rem}.g-4,.gx-4{--bs-gutter-x:1.5rem}.g-4,.gy-4{--bs-gutter-y:1.5rem}.g-5,.gx-5{--bs-gutter-x:3rem}.g-5,.gy-5{--bs-gutter-y:3rem}@media (min-width:576px){.col-sm{flex:1 0 0%}.row-cols-sm-auto>*{flex:0 0 auto;width:auto}.row-cols-sm-1>*{flex:0 0 auto;width:100%}.row-cols-sm-2>*{flex:0 0 auto;width:50%}.row-cols-sm-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-sm-4>*{flex:0 0 auto;width:25%}.row-cols-sm-5>*{flex:0 0 auto;width:20%}.row-cols-sm-6>*{flex:0 0 auto;width:16.66666667%}.col-sm-auto{flex:0 0 auto;width:auto}.col-sm-1{flex:0 0 auto;width:8.33333333%}.col-sm-2{flex:0 0 auto;width:16.66666667%}.col-sm-3{flex:0 0 auto;width:25%}.col-sm-4{flex:0 0 auto;width:33.33333333%}.col-sm-5{flex:0 0 auto;width:41.66666667%}.col-sm-6{flex:0 0 auto;width:50%}.col-sm-7{flex:0 0 auto;width:58.33333333%}.col-sm-8{flex:0 0 auto;width:66.66666667%}.col-sm-9{flex:0 0 auto;width:75%}.col-sm-10{flex:0 0 auto;width:83.33333333%}.col-sm-11{flex:0 0 auto;width:91.66666667%}.col-sm-12{flex:0 0 auto;width:100%}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.33333333%}.offset-sm-2{margin-left:16.66666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.33333333%}.offset-sm-5{margin-left:41.66666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.33333333%}.offset-sm-8{margin-left:66.66666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.33333333%}.offset-sm-11{margin-left:91.66666667%}.g-sm-0,.gx-sm-0{--bs-gutter-x:0}.g-sm-0,.gy-sm-0{--bs-gutter-y:0}.g-sm-1,.gx-sm-1{--bs-gutter-x:0.25rem}.g-sm-1,.gy-sm-1{--bs-gutter-y:0.25rem}.g-sm-2,.gx-sm-2{--bs-gutter-x:0.5rem}.g-sm-2,.gy-sm-2{--bs-gutter-y:0.5rem}.g-sm-3,.gx-sm-3{--bs-gutter-x:1rem}.g-sm-3,.gy-sm-3{--bs-gutter-y:1rem}.g-sm-4,.gx-sm-4{--bs-gutter-x:1.5rem}.g-sm-4,.gy-sm-4{--bs-gutter-y:1.5rem}.g-sm-5,.gx-sm-5{--bs-gutter-x:3rem}.g-sm-5,.gy-sm-5{--bs-gutter-y:3rem}}@media (min-width:768px){.col-md{flex:1 0 0%}.row-cols-md-auto>*{flex:0 0 auto;width:auto}.row-cols-md-1>*{flex:0 0 auto;width:100%}.row-cols-md-2>*{flex:0 0 auto;width:50%}.row-cols-md-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-md-4>*{flex:0 0 auto;width:25%}.row-cols-md-5>*{flex:0 0 auto;width:20%}.row-cols-md-6>*{flex:0 0 auto;width:16.66666667%}.col-md-auto{flex:0 0 auto;width:auto}.col-md-1{flex:0 0 auto;width:8.33333333%}.col-md-2{flex:0 0 auto;width:16.66666667%}.col-md-3{flex:0 0 auto;width:25%}.col-md-4{flex:0 0 auto;width:33.33333333%}.col-md-5{flex:0 0 auto;width:41.66666667%}.col-md-6{flex:0 0 auto;width:50%}.col-md-7{flex:0 0 auto;width:58.33333333%}.col-md-8{flex:0 0 auto;width:66.66666667%}.col-md-9{flex:0 0 auto;width:75%}.col-md-10{flex:0 0 auto;width:83.33333333%}.col-md-11{flex:0 0 auto;width:91.66666667%}.col-md-12{flex:0 0 auto;width:100%}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.33333333%}.offset-md-2{margin-left:16.66666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.33333333%}.offset-md-5{margin-left:41.66666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.33333333%}.offset-md-8{margin-left:66.66666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.33333333%}.offset-md-11{margin-left:91.66666667%}.g-md-0,.gx-md-0{--bs-gutter-x:0}.g-md-0,.gy-md-0{--bs-gutter-y:0}.g-md-1,.gx-md-1{--bs-gutter-x:0.25rem}.g-md-1,.gy-md-1{--bs-gutter-y:0.25rem}.g-md-2,.gx-md-2{--bs-gutter-x:0.5rem}.g-md-2,.gy-md-2{--bs-gutter-y:0.5rem}.g-md-3,.gx-md-3{--bs-gutter-x:1rem}.g-md-3,.gy-md-3{--bs-gutter-y:1rem}.g-md-4,.gx-md-4{--bs-gutter-x:1.5rem}.g-md-4,.gy-md-4{--bs-gutter-y:1.5rem}.g-md-5,.gx-md-5{--bs-gutter-x:3rem}.g-md-5,.gy-md-5{--bs-gutter-y:3rem}}@media (min-width:992px){.col-lg{flex:1 0 0%}.row-cols-lg-auto>*{flex:0 0 auto;width:auto}.row-cols-lg-1>*{flex:0 0 auto;width:100%}.row-cols-lg-2>*{flex:0 0 auto;width:50%}.row-cols-lg-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-lg-4>*{flex:0 0 auto;width:25%}.row-cols-lg-5>*{flex:0 0 auto;width:20%}.row-cols-lg-6>*{flex:0 0 auto;width:16.66666667%}.col-lg-auto{flex:0 0 auto;width:auto}.col-lg-1{flex:0 0 auto;width:8.33333333%}.col-lg-2{flex:0 0 auto;width:16.66666667%}.col-lg-3{flex:0 0 auto;width:25%}.col-lg-4{flex:0 0 auto;width:33.33333333%}.col-lg-5{flex:0 0 auto;width:41.66666667%}.col-lg-6{flex:0 0 auto;width:50%}.col-lg-7{flex:0 0 auto;width:58.33333333%}.col-lg-8{flex:0 0 auto;width:66.66666667%}.col-lg-9{flex:0 0 auto;width:75%}.col-lg-10{flex:0 0 auto;width:83.33333333%}.col-lg-11{flex:0 0 auto;width:91.66666667%}.col-lg-12{flex:0 0 auto;width:100%}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.33333333%}.offset-lg-2{margin-left:16.66666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.33333333%}.offset-lg-5{margin-left:41.66666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.33333333%}.offset-lg-8{margin-left:66.66666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.33333333%}.offset-lg-11{margin-left:91.66666667%}.g-lg-0,.gx-lg-0{--bs-gutter-x:0}.g-lg-0,.gy-lg-0{--bs-gutter-y:0}.g-lg-1,.gx-lg-1{--bs-gutter-x:0.25rem}.g-lg-1,.gy-lg-1{--bs-gutter-y:0.25rem}.g-lg-2,.gx-lg-2{--bs-gutter-x:0.5rem}.g-lg-2,.gy-lg-2{--bs-gutter-y:0.5rem}.g-lg-3,.gx-lg-3{--bs-gutter-x:1rem}.g-lg-3,.gy-lg-3{--bs-gutter-y:1rem}.g-lg-4,.gx-lg-4{--bs-gutter-x:1.5rem}.g-lg-4,.gy-lg-4{--bs-gutter-y:1.5rem}.g-lg-5,.gx-lg-5{--bs-gutter-x:3rem}.g-lg-5,.gy-lg-5{--bs-gutter-y:3rem}}@media (min-width:1200px){.col-xl{flex:1 0 0%}.row-cols-xl-auto>*{flex:0 0 auto;width:auto}.row-cols-xl-1>*{flex:0 0 auto;width:100%}.row-cols-xl-2>*{flex:0 0 auto;width:50%}.row-cols-xl-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-xl-4>*{flex:0 0 auto;width:25%}.row-cols-xl-5>*{flex:0 0 auto;width:20%}.row-cols-xl-6>*{flex:0 0 auto;width:16.66666667%}.col-xl-auto{flex:0 0 auto;width:auto}.col-xl-1{flex:0 0 auto;width:8.33333333%}.col-xl-2{flex:0 0 auto;width:16.66666667%}.col-xl-3{flex:0 0 auto;width:25%}.col-xl-4{flex:0 0 auto;width:33.33333333%}.col-xl-5{flex:0 0 auto;width:41.66666667%}.col-xl-6{flex:0 0 auto;width:50%}.col-xl-7{flex:0 0 auto;width:58.33333333%}.col-xl-8{flex:0 0 auto;width:66.66666667%}.col-xl-9{flex:0 0 auto;width:75%}.col-xl-10{flex:0 0 auto;width:83.33333333%}.col-xl-11{flex:0 0 auto;width:91.66666667%}.col-xl-12{flex:0 0 auto;width:100%}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.33333333%}.offset-xl-2{margin-left:16.66666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.33333333%}.offset-xl-5{margin-left:41.66666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.33333333%}.offset-xl-8{margin-left:66.66666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.33333333%}.offset-xl-11{margin-left:91.66666667%}.g-xl-0,.gx-xl-0{--bs-gutter-x:0}.g-xl-0,.gy-xl-0{--bs-gutter-y:0}.g-xl-1,.gx-xl-1{--bs-gutter-x:0.25rem}.g-xl-1,.gy-xl-1{--bs-gutter-y:0.25rem}.g-xl-2,.gx-xl-2{--bs-gutter-x:0.5rem}.g-xl-2,.gy-xl-2{--bs-gutter-y:0.5rem}.g-xl-3,.gx-xl-3{--bs-gutter-x:1rem}.g-xl-3,.gy-xl-3{--bs-gutter-y:1rem}.g-xl-4,.gx-xl-4{--bs-gutter-x:1.5rem}.g-xl-4,.gy-xl-4{--bs-gutter-y:1.5rem}.g-xl-5,.gx-xl-5{--bs-gutter-x:3rem}.g-xl-5,.gy-xl-5{--bs-gutter-y:3rem}}@media (min-width:1400px){.col-xxl{flex:1 0 0%}.row-cols-xxl-auto>*{flex:0 0 auto;width:auto}.row-cols-xxl-1>*{flex:0 0 auto;width:100%}.row-cols-xxl-2>*{flex:0 0 auto;width:50%}.row-cols-xxl-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-xxl-4>*{flex:0 0 auto;width:25%}.row-cols-xxl-5>*{flex:0 0 auto;width:20%}.row-cols-xxl-6>*{flex:0 0 auto;width:16.66666667%}.col-xxl-auto{flex:0 0 auto;width:auto}.col-xxl-1{flex:0 0 auto;width:8.33333333%}.col-xxl-2{flex:0 0 auto;width:16.66666667%}.col-xxl-3{flex:0 0 auto;width:25%}.col-xxl-4{flex:0 0 auto;width:33.33333333%}.col-xxl-5{flex:0 0 auto;width:41.66666667%}.col-xxl-6{flex:0 0 auto;width:50%}.col-xxl-7{flex:0 0 auto;width:58.33333333%}.col-xxl-8{flex:0 0 auto;width:66.66666667%}.col-xxl-9{flex:0 0 auto;width:75%}.col-xxl-10{flex:0 0 auto;width:83.33333333%}.col-xxl-11{flex:0 0 auto;width:91.66666667%}.col-xxl-12{flex:0 0 auto;width:100%}.offset-xxl-0{margin-left:0}.offset-xxl-1{margin-left:8.33333333%}.offset-xxl-2{margin-left:16.66666667%}.offset-xxl-3{margin-left:25%}.offset-xxl-4{margin-left:33.33333333%}.offset-xxl-5{margin-left:41.66666667%}.offset-xxl-6{margin-left:50%}.offset-xxl-7{margin-left:58.33333333%}.offset-xxl-8{margin-left:66.66666667%}.offset-xxl-9{margin-left:75%}.offset-xxl-10{margin-left:83.33333333%}.offset-xxl-11{margin-left:91.66666667%}.g-xxl-0,.gx-xxl-0{--bs-gutter-x:0}.g-xxl-0,.gy-xxl-0{--bs-gutter-y:0}.g-xxl-1,.gx-xxl-1{--bs-gutter-x:0.25rem}.g-xxl-1,.gy-xxl-1{--bs-gutter-y:0.25rem}.g-xxl-2,.gx-xxl-2{--bs-gutter-x:0.5rem}.g-xxl-2,.gy-xxl-2{--bs-gutter-y:0.5rem}.g-xxl-3,.gx-xxl-3{--bs-gutter-x:1rem}.g-xxl-3,.gy-xxl-3{--bs-gutter-y:1rem}.g-xxl-4,.gx-xxl-4{--bs-gutter-x:1.5rem}.g-xxl-4,.gy-xxl-4{--bs-gutter-y:1.5rem}.g-xxl-5,.gx-xxl-5{--bs-gutter-x:3rem}.g-xxl-5,.gy-xxl-5{--bs-gutter-y:3rem}}.table{--bs-table-color-type:initial;--bs-table-bg-type:initial;--bs-table-color-state:initial;--bs-table-bg-state:initial;--bs-table-color:var(--bs-emphasis-color);--bs-table-bg:var(--bs-body-bg);--bs-table-border-color:var(--bs-border-color);--bs-table-accent-bg:transparent;--bs-table-striped-color:var(--bs-emphasis-color);--bs-table-striped-bg:rgba(var(--bs-emphasis-color-rgb), 0.05);--bs-table-active-color:var(--bs-emphasis-color);--bs-table-active-bg:rgba(var(--bs-emphasis-color-rgb), 0.1);--bs-table-hover-color:var(--bs-emphasis-color);--bs-table-hover-bg:rgba(var(--bs-emphasis-color-rgb), 0.075);width:100%;margin-bottom:1rem;vertical-align:top;border-color:var(--bs-table-border-color)}.table>:not(caption)>*>*{padding:.5rem .5rem;color:var(--bs-table-color-state,var(--bs-table-color-type,var(--bs-table-color)));background-color:var(--bs-table-bg);border-bottom-width:var(--bs-border-width);box-shadow:inset 0 0 0 9999px var(--bs-table-bg-state,var(--bs-table-bg-type,var(--bs-table-accent-bg)))}.table>tbody{vertical-align:inherit}.table>thead{vertical-align:bottom}.table-group-divider{border-top:calc(var(--bs-border-width) * 2) solid currentcolor}.caption-top{caption-side:top}.table-sm>:not(caption)>*>*{padding:.25rem .25rem}.table-bordered>:not(caption)>*{border-width:var(--bs-border-width) 0}.table-bordered>:not(caption)>*>*{border-width:0 var(--bs-border-width)}.table-borderless>:not(caption)>*>*{border-bottom-width:0}.table-borderless>:not(:first-child){border-top-width:0}.table-striped>tbody>tr:nth-of-type(odd)>*{--bs-table-color-type:var(--bs-table-striped-color);--bs-table-bg-type:var(--bs-table-striped-bg)}.table-striped-columns>:not(caption)>tr>:nth-child(2n){--bs-table-color-type:var(--bs-table-striped-color);--bs-table-bg-type:var(--bs-table-striped-bg)}.table-active{--bs-table-color-state:var(--bs-table-active-color);--bs-table-bg-state:var(--bs-table-active-bg)}.table-hover>tbody>tr:hover>*{--bs-table-color-state:var(--bs-table-hover-color);--bs-table-bg-state:var(--bs-table-hover-bg)}.table-primary{--bs-table-color:#000;--bs-table-bg:#cfe2ff;--bs-table-border-color:#a6b5cc;--bs-table-striped-bg:#c5d7f2;--bs-table-striped-color:#000;--bs-table-active-bg:#bacbe6;--bs-table-active-color:#000;--bs-table-hover-bg:#bfd1ec;--bs-table-hover-color:#000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-secondary{--bs-table-color:#000;--bs-table-bg:#e2e3e5;--bs-table-border-color:#b5b6b7;--bs-table-striped-bg:#d7d8da;--bs-table-striped-color:#000;--bs-table-active-bg:#cbccce;--bs-table-active-color:#000;--bs-table-hover-bg:#d1d2d4;--bs-table-hover-color:#000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-success{--bs-table-color:#000;--bs-table-bg:#d1e7dd;--bs-table-border-color:#a7b9b1;--bs-table-striped-bg:#c7dbd2;--bs-table-striped-color:#000;--bs-table-active-bg:#bcd0c7;--bs-table-active-color:#000;--bs-table-hover-bg:#c1d6cc;--bs-table-hover-color:#000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-info{--bs-table-color:#000;--bs-table-bg:#cff4fc;--bs-table-border-color:#a6c3ca;--bs-table-striped-bg:#c5e8ef;--bs-table-striped-color:#000;--bs-table-active-bg:#badce3;--bs-table-active-color:#000;--bs-table-hover-bg:#bfe2e9;--bs-table-hover-color:#000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-warning{--bs-table-color:#000;--bs-table-bg:#fff3cd;--bs-table-border-color:#ccc2a4;--bs-table-striped-bg:#f2e7c3;--bs-table-striped-color:#000;--bs-table-active-bg:#e6dbb9;--bs-table-active-color:#000;--bs-table-hover-bg:#ece1be;--bs-table-hover-color:#000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-danger{--bs-table-color:#000;--bs-table-bg:#f8d7da;--bs-table-border-color:#c6acae;--bs-table-striped-bg:#eccccf;--bs-table-striped-color:#000;--bs-table-active-bg:#dfc2c4;--bs-table-active-color:#000;--bs-table-hover-bg:#e5c7ca;--bs-table-hover-color:#000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-light{--bs-table-color:#000;--bs-table-bg:#f8f9fa;--bs-table-border-color:#c6c7c8;--bs-table-striped-bg:#ecedee;--bs-table-striped-color:#000;--bs-table-active-bg:#dfe0e1;--bs-table-active-color:#000;--bs-table-hover-bg:#e5e6e7;--bs-table-hover-color:#000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-dark{--bs-table-color:#fff;--bs-table-bg:#212529;--bs-table-border-color:#4d5154;--bs-table-striped-bg:#2c3034;--bs-table-striped-color:#fff;--bs-table-active-bg:#373b3e;--bs-table-active-color:#fff;--bs-table-hover-bg:#323539;--bs-table-hover-color:#fff;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-responsive{overflow-x:auto;-webkit-overflow-scrolling:touch}@media (max-width:575.98px){.table-responsive-sm{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:767.98px){.table-responsive-md{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:991.98px){.table-responsive-lg{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:1199.98px){.table-responsive-xl{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:1399.98px){.table-responsive-xxl{overflow-x:auto;-webkit-overflow-scrolling:touch}}.form-label{margin-bottom:.5rem}.col-form-label{padding-top:calc(.375rem + var(--bs-border-width));padding-bottom:calc(.375rem + var(--bs-border-width));margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(.5rem + var(--bs-border-width));padding-bottom:calc(.5rem + var(--bs-border-width));font-size:1.25rem}.col-form-label-sm{padding-top:calc(.25rem + var(--bs-border-width));padding-bottom:calc(.25rem + var(--bs-border-width));font-size:.875rem}.form-text{margin-top:.25rem;font-size:.875em;color:var(--bs-secondary-color)}.form-control{display:block;width:100%;padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:var(--bs-body-color);-webkit-appearance:none;-moz-appearance:none;appearance:none;background-color:var(--bs-body-bg);background-clip:padding-box;border:var(--bs-border-width) solid var(--bs-border-color);border-radius:var(--bs-border-radius);transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control{transition:none}}.form-control[type=file]{overflow:hidden}.form-control[type=file]:not(:disabled):not([readonly]){cursor:pointer}.form-control:focus{color:var(--bs-body-color);background-color:var(--bs-body-bg);border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.form-control::-webkit-date-and-time-value{min-width:85px;height:1.5em;margin:0}.form-control::-webkit-datetime-edit{display:block;padding:0}.form-control::-moz-placeholder{color:var(--bs-secondary-color);opacity:1}.form-control::placeholder{color:var(--bs-secondary-color);opacity:1}.form-control:disabled{background-color:var(--bs-secondary-bg);opacity:1}.form-control::-webkit-file-upload-button{padding:.375rem .75rem;margin:-.375rem -.75rem;-webkit-margin-end:.75rem;margin-inline-end:.75rem;color:var(--bs-body-color);background-color:var(--bs-tertiary-bg);pointer-events:none;border-color:inherit;border-style:solid;border-width:0;border-inline-end-width:var(--bs-border-width);border-radius:0;-webkit-transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}.form-control::file-selector-button{padding:.375rem .75rem;margin:-.375rem -.75rem;-webkit-margin-end:.75rem;margin-inline-end:.75rem;color:var(--bs-body-color);background-color:var(--bs-tertiary-bg);pointer-events:none;border-color:inherit;border-style:solid;border-width:0;border-inline-end-width:var(--bs-border-width);border-radius:0;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control::-webkit-file-upload-button{-webkit-transition:none;transition:none}.form-control::file-selector-button{transition:none}}.form-control:hover:not(:disabled):not([readonly])::-webkit-file-upload-button{background-color:var(--bs-secondary-bg)}.form-control:hover:not(:disabled):not([readonly])::file-selector-button{background-color:var(--bs-secondary-bg)}.form-control-plaintext{display:block;width:100%;padding:.375rem 0;margin-bottom:0;line-height:1.5;color:var(--bs-body-color);background-color:transparent;border:solid transparent;border-width:var(--bs-border-width) 0}.form-control-plaintext:focus{outline:0}.form-control-plaintext.form-control-lg,.form-control-plaintext.form-control-sm{padding-right:0;padding-left:0}.form-control-sm{min-height:calc(1.5em + .5rem + calc(var(--bs-border-width) * 2));padding:.25rem .5rem;font-size:.875rem;border-radius:var(--bs-border-radius-sm)}.form-control-sm::-webkit-file-upload-button{padding:.25rem .5rem;margin:-.25rem -.5rem;-webkit-margin-end:.5rem;margin-inline-end:.5rem}.form-control-sm::file-selector-button{padding:.25rem .5rem;margin:-.25rem -.5rem;-webkit-margin-end:.5rem;margin-inline-end:.5rem}.form-control-lg{min-height:calc(1.5em + 1rem + calc(var(--bs-border-width) * 2));padding:.5rem 1rem;font-size:1.25rem;border-radius:var(--bs-border-radius-lg)}.form-control-lg::-webkit-file-upload-button{padding:.5rem 1rem;margin:-.5rem -1rem;-webkit-margin-end:1rem;margin-inline-end:1rem}.form-control-lg::file-selector-button{padding:.5rem 1rem;margin:-.5rem -1rem;-webkit-margin-end:1rem;margin-inline-end:1rem}textarea.form-control{min-height:calc(1.5em + .75rem + calc(var(--bs-border-width) * 2))}textarea.form-control-sm{min-height:calc(1.5em + .5rem + calc(var(--bs-border-width) * 2))}textarea.form-control-lg{min-height:calc(1.5em + 1rem + calc(var(--bs-border-width) * 2))}.form-control-color{width:3rem;height:calc(1.5em + .75rem + calc(var(--bs-border-width) * 2));padding:.375rem}.form-control-color:not(:disabled):not([readonly]){cursor:pointer}.form-control-color::-moz-color-swatch{border:0!important;border-radius:var(--bs-border-radius)}.form-control-color::-webkit-color-swatch{border:0!important;border-radius:var(--bs-border-radius)}.form-control-color.form-control-sm{height:calc(1.5em + .5rem + calc(var(--bs-border-width) * 2))}.form-control-color.form-control-lg{height:calc(1.5em + 1rem + calc(var(--bs-border-width) * 2))}.form-select{--bs-form-select-bg-img:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23343a40' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='m2 5 6 6 6-6'/%3e%3c/svg%3e");display:block;width:100%;padding:.375rem 2.25rem .375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:var(--bs-body-color);-webkit-appearance:none;-moz-appearance:none;appearance:none;background-color:var(--bs-body-bg);background-image:var(--bs-form-select-bg-img),var(--bs-form-select-bg-icon,none);background-repeat:no-repeat;background-position:right .75rem center;background-size:16px 12px;border:var(--bs-border-width) solid var(--bs-border-color);border-radius:var(--bs-border-radius);transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-select{transition:none}}.form-select:focus{border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.form-select[multiple],.form-select[size]:not([size="1"]){padding-right:.75rem;background-image:none}.form-select:disabled{background-color:var(--bs-secondary-bg)}.form-select:-moz-focusring{color:transparent;text-shadow:0 0 0 var(--bs-body-color)}.form-select-sm{padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:.875rem;border-radius:var(--bs-border-radius-sm)}.form-select-lg{padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem;border-radius:var(--bs-border-radius-lg)}[data-bs-theme=dark] .form-select{--bs-form-select-bg-img:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23dee2e6' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='m2 5 6 6 6-6'/%3e%3c/svg%3e")}.form-check{display:block;min-height:1.5rem;padding-left:1.5em;margin-bottom:.125rem}.form-check .form-check-input{float:left;margin-left:-1.5em}.form-check-reverse{padding-right:1.5em;padding-left:0;text-align:right}.form-check-reverse .form-check-input{float:right;margin-right:-1.5em;margin-left:0}.form-check-input{--bs-form-check-bg:var(--bs-body-bg);flex-shrink:0;width:1em;height:1em;margin-top:.25em;vertical-align:top;-webkit-appearance:none;-moz-appearance:none;appearance:none;background-color:var(--bs-form-check-bg);background-image:var(--bs-form-check-bg-image);background-repeat:no-repeat;background-position:center;background-size:contain;border:var(--bs-border-width) solid var(--bs-border-color);-webkit-print-color-adjust:exact;color-adjust:exact;print-color-adjust:exact}.form-check-input[type=checkbox]{border-radius:.25em}.form-check-input[type=radio]{border-radius:50%}.form-check-input:active{filter:brightness(90%)}.form-check-input:focus{border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.form-check-input:checked{background-color:#0d6efd;border-color:#0d6efd}.form-check-input:checked[type=checkbox]{--bs-form-check-bg-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20'%3e%3cpath fill='none' stroke='%23fff' stroke-linecap='round' stroke-linejoin='round' stroke-width='3' d='m6 10 3 3 6-6'/%3e%3c/svg%3e")}.form-check-input:checked[type=radio]{--bs-form-check-bg-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='2' fill='%23fff'/%3e%3c/svg%3e")}.form-check-input[type=checkbox]:indeterminate{background-color:#0d6efd;border-color:#0d6efd;--bs-form-check-bg-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20'%3e%3cpath fill='none' stroke='%23fff' stroke-linecap='round' stroke-linejoin='round' stroke-width='3' d='M6 10h8'/%3e%3c/svg%3e")}.form-check-input:disabled{pointer-events:none;filter:none;opacity:.5}.form-check-input:disabled~.form-check-label,.form-check-input[disabled]~.form-check-label{cursor:default;opacity:.5}.form-switch{padding-left:2.5em}.form-switch .form-check-input{--bs-form-switch-bg:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%280, 0, 0, 0.25%29'/%3e%3c/svg%3e");width:2em;margin-left:-2.5em;background-image:var(--bs-form-switch-bg);background-position:left center;border-radius:2em;transition:background-position .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-switch .form-check-input{transition:none}}.form-switch .form-check-input:focus{--bs-form-switch-bg:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%2386b7fe'/%3e%3c/svg%3e")}.form-switch .form-check-input:checked{background-position:right center;--bs-form-switch-bg:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.form-switch.form-check-reverse{padding-right:2.5em;padding-left:0}.form-switch.form-check-reverse .form-check-input{margin-right:-2.5em;margin-left:0}.form-check-inline{display:inline-block;margin-right:1rem}.btn-check{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.btn-check:disabled+.btn,.btn-check[disabled]+.btn{pointer-events:none;filter:none;opacity:.65}[data-bs-theme=dark] .form-switch .form-check-input:not(:checked):not(:focus){--bs-form-switch-bg:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%28255, 255, 255, 0.25%29'/%3e%3c/svg%3e")}.form-range{width:100%;height:1.5rem;padding:0;-webkit-appearance:none;-moz-appearance:none;appearance:none;background-color:transparent}.form-range:focus{outline:0}.form-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .25rem rgba(13,110,253,.25)}.form-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .25rem rgba(13,110,253,.25)}.form-range::-moz-focus-outer{border:0}.form-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-.25rem;-webkit-appearance:none;appearance:none;background-color:#0d6efd;border:0;border-radius:1rem;-webkit-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-range::-webkit-slider-thumb{-webkit-transition:none;transition:none}}.form-range::-webkit-slider-thumb:active{background-color:#b6d4fe}.form-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:var(--bs-secondary-bg);border-color:transparent;border-radius:1rem}.form-range::-moz-range-thumb{width:1rem;height:1rem;-moz-appearance:none;appearance:none;background-color:#0d6efd;border:0;border-radius:1rem;-moz-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-range::-moz-range-thumb{-moz-transition:none;transition:none}}.form-range::-moz-range-thumb:active{background-color:#b6d4fe}.form-range::-moz-range-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:var(--bs-secondary-bg);border-color:transparent;border-radius:1rem}.form-range:disabled{pointer-events:none}.form-range:disabled::-webkit-slider-thumb{background-color:var(--bs-secondary-color)}.form-range:disabled::-moz-range-thumb{background-color:var(--bs-secondary-color)}.form-floating{position:relative}.form-floating>.form-control,.form-floating>.form-control-plaintext,.form-floating>.form-select{height:calc(3.5rem + calc(var(--bs-border-width) * 2));min-height:calc(3.5rem + calc(var(--bs-border-width) * 2));line-height:1.25}.form-floating>label{position:absolute;top:0;left:0;z-index:2;height:100%;padding:1rem .75rem;overflow:hidden;text-align:start;text-overflow:ellipsis;white-space:nowrap;pointer-events:none;border:var(--bs-border-width) solid transparent;transform-origin:0 0;transition:opacity .1s ease-in-out,transform .1s ease-in-out}@media (prefers-reduced-motion:reduce){.form-floating>label{transition:none}}.form-floating>.form-control,.form-floating>.form-control-plaintext{padding:1rem .75rem}.form-floating>.form-control-plaintext::-moz-placeholder,.form-floating>.form-control::-moz-placeholder{color:transparent}.form-floating>.form-control-plaintext::placeholder,.form-floating>.form-control::placeholder{color:transparent}.form-floating>.form-control-plaintext:not(:-moz-placeholder-shown),.form-floating>.form-control:not(:-moz-placeholder-shown){padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control-plaintext:focus,.form-floating>.form-control-plaintext:not(:placeholder-shown),.form-floating>.form-control:focus,.form-floating>.form-control:not(:placeholder-shown){padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control-plaintext:-webkit-autofill,.form-floating>.form-control:-webkit-autofill{padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-select{padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control:not(:-moz-placeholder-shown)~label{color:rgba(var(--bs-body-color-rgb),.65);transform:scale(.85) translateY(-.5rem) translateX(.15rem)}.form-floating>.form-control-plaintext~label,.form-floating>.form-control:focus~label,.form-floating>.form-control:not(:placeholder-shown)~label,.form-floating>.form-select~label{color:rgba(var(--bs-body-color-rgb),.65);transform:scale(.85) translateY(-.5rem) translateX(.15rem)}.form-floating>.form-control:not(:-moz-placeholder-shown)~label::after{position:absolute;inset:1rem 0.375rem;z-index:-1;height:1.5em;content:"";background-color:var(--bs-body-bg);border-radius:var(--bs-border-radius)}.form-floating>.form-control-plaintext~label::after,.form-floating>.form-control:focus~label::after,.form-floating>.form-control:not(:placeholder-shown)~label::after,.form-floating>.form-select~label::after{position:absolute;inset:1rem 0.375rem;z-index:-1;height:1.5em;content:"";background-color:var(--bs-body-bg);border-radius:var(--bs-border-radius)}.form-floating>.form-control:-webkit-autofill~label{color:rgba(var(--bs-body-color-rgb),.65);transform:scale(.85) translateY(-.5rem) translateX(.15rem)}.form-floating>.form-control-plaintext~label{border-width:var(--bs-border-width) 0}.form-floating>.form-control:disabled~label,.form-floating>:disabled~label{color:#6c757d}.form-floating>.form-control:disabled~label::after,.form-floating>:disabled~label::after{background-color:var(--bs-secondary-bg)}.input-group{position:relative;display:flex;flex-wrap:wrap;align-items:stretch;width:100%}.input-group>.form-control,.input-group>.form-floating,.input-group>.form-select{position:relative;flex:1 1 auto;width:1%;min-width:0}.input-group>.form-control:focus,.input-group>.form-floating:focus-within,.input-group>.form-select:focus{z-index:5}.input-group .btn{position:relative;z-index:2}.input-group .btn:focus{z-index:5}.input-group-text{display:flex;align-items:center;padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:var(--bs-body-color);text-align:center;white-space:nowrap;background-color:var(--bs-tertiary-bg);border:var(--bs-border-width) solid var(--bs-border-color);border-radius:var(--bs-border-radius)}.input-group-lg>.btn,.input-group-lg>.form-control,.input-group-lg>.form-select,.input-group-lg>.input-group-text{padding:.5rem 1rem;font-size:1.25rem;border-radius:var(--bs-border-radius-lg)}.input-group-sm>.btn,.input-group-sm>.form-control,.input-group-sm>.form-select,.input-group-sm>.input-group-text{padding:.25rem .5rem;font-size:.875rem;border-radius:var(--bs-border-radius-sm)}.input-group-lg>.form-select,.input-group-sm>.form-select{padding-right:3rem}.input-group:not(.has-validation)>.dropdown-toggle:nth-last-child(n+3),.input-group:not(.has-validation)>.form-floating:not(:last-child)>.form-control,.input-group:not(.has-validation)>.form-floating:not(:last-child)>.form-select,.input-group:not(.has-validation)>:not(:last-child):not(.dropdown-toggle):not(.dropdown-menu):not(.form-floating){border-top-right-radius:0;border-bottom-right-radius:0}.input-group.has-validation>.dropdown-toggle:nth-last-child(n+4),.input-group.has-validation>.form-floating:nth-last-child(n+3)>.form-control,.input-group.has-validation>.form-floating:nth-last-child(n+3)>.form-select,.input-group.has-validation>:nth-last-child(n+3):not(.dropdown-toggle):not(.dropdown-menu):not(.form-floating){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>:not(:first-child):not(.dropdown-menu):not(.valid-tooltip):not(.valid-feedback):not(.invalid-tooltip):not(.invalid-feedback){margin-left:calc(var(--bs-border-width) * -1);border-top-left-radius:0;border-bottom-left-radius:0}.input-group>.form-floating:not(:first-child)>.form-control,.input-group>.form-floating:not(:first-child)>.form-select{border-top-left-radius:0;border-bottom-left-radius:0}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:.875em;color:var(--bs-form-valid-color)}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;color:#fff;background-color:var(--bs-success);border-radius:var(--bs-border-radius)}.is-valid~.valid-feedback,.is-valid~.valid-tooltip,.was-validated :valid~.valid-feedback,.was-validated :valid~.valid-tooltip{display:block}.form-control.is-valid,.was-validated .form-control:valid{border-color:var(--bs-form-valid-border-color);padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%23198754' d='M2.3 6.73.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-valid:focus,.was-validated .form-control:valid:focus{border-color:var(--bs-form-valid-border-color);box-shadow:0 0 0 .25rem rgba(var(--bs-success-rgb),.25)}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.form-select.is-valid,.was-validated .form-select:valid{border-color:var(--bs-form-valid-border-color)}.form-select.is-valid:not([multiple]):not([size]),.form-select.is-valid:not([multiple])[size="1"],.was-validated .form-select:valid:not([multiple]):not([size]),.was-validated .form-select:valid:not([multiple])[size="1"]{--bs-form-select-bg-icon:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%23198754' d='M2.3 6.73.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");padding-right:4.125rem;background-position:right .75rem center,center right 2.25rem;background-size:16px 12px,calc(.75em + .375rem) calc(.75em + .375rem)}.form-select.is-valid:focus,.was-validated .form-select:valid:focus{border-color:var(--bs-form-valid-border-color);box-shadow:0 0 0 .25rem rgba(var(--bs-success-rgb),.25)}.form-control-color.is-valid,.was-validated .form-control-color:valid{width:calc(3rem + calc(1.5em + .75rem))}.form-check-input.is-valid,.was-validated .form-check-input:valid{border-color:var(--bs-form-valid-border-color)}.form-check-input.is-valid:checked,.was-validated .form-check-input:valid:checked{background-color:var(--bs-form-valid-color)}.form-check-input.is-valid:focus,.was-validated .form-check-input:valid:focus{box-shadow:0 0 0 .25rem rgba(var(--bs-success-rgb),.25)}.form-check-input.is-valid~.form-check-label,.was-validated .form-check-input:valid~.form-check-label{color:var(--bs-form-valid-color)}.form-check-inline .form-check-input~.valid-feedback{margin-left:.5em}.input-group>.form-control:not(:focus).is-valid,.input-group>.form-floating:not(:focus-within).is-valid,.input-group>.form-select:not(:focus).is-valid,.was-validated .input-group>.form-control:not(:focus):valid,.was-validated .input-group>.form-floating:not(:focus-within):valid,.was-validated .input-group>.form-select:not(:focus):valid{z-index:3}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:.875em;color:var(--bs-form-invalid-color)}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;color:#fff;background-color:var(--bs-danger);border-radius:var(--bs-border-radius)}.is-invalid~.invalid-feedback,.is-invalid~.invalid-tooltip,.was-validated :invalid~.invalid-feedback,.was-validated :invalid~.invalid-tooltip{display:block}.form-control.is-invalid,.was-validated .form-control:invalid{border-color:var(--bs-form-invalid-border-color);padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 12 12' width='12' height='12' fill='none' stroke='%23dc3545'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-invalid:focus,.was-validated .form-control:invalid:focus{border-color:var(--bs-form-invalid-border-color);box-shadow:0 0 0 .25rem rgba(var(--bs-danger-rgb),.25)}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.form-select.is-invalid,.was-validated .form-select:invalid{border-color:var(--bs-form-invalid-border-color)}.form-select.is-invalid:not([multiple]):not([size]),.form-select.is-invalid:not([multiple])[size="1"],.was-validated .form-select:invalid:not([multiple]):not([size]),.was-validated .form-select:invalid:not([multiple])[size="1"]{--bs-form-select-bg-icon:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 12 12' width='12' height='12' fill='none' stroke='%23dc3545'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");padding-right:4.125rem;background-position:right .75rem center,center right 2.25rem;background-size:16px 12px,calc(.75em + .375rem) calc(.75em + .375rem)}.form-select.is-invalid:focus,.was-validated .form-select:invalid:focus{border-color:var(--bs-form-invalid-border-color);box-shadow:0 0 0 .25rem rgba(var(--bs-danger-rgb),.25)}.form-control-color.is-invalid,.was-validated .form-control-color:invalid{width:calc(3rem + calc(1.5em + .75rem))}.form-check-input.is-invalid,.was-validated .form-check-input:invalid{border-color:var(--bs-form-invalid-border-color)}.form-check-input.is-invalid:checked,.was-validated .form-check-input:invalid:checked{background-color:var(--bs-form-invalid-color)}.form-check-input.is-invalid:focus,.was-validated .form-check-input:invalid:focus{box-shadow:0 0 0 .25rem rgba(var(--bs-danger-rgb),.25)}.form-check-input.is-invalid~.form-check-label,.was-validated .form-check-input:invalid~.form-check-label{color:var(--bs-form-invalid-color)}.form-check-inline .form-check-input~.invalid-feedback{margin-left:.5em}.input-group>.form-control:not(:focus).is-invalid,.input-group>.form-floating:not(:focus-within).is-invalid,.input-group>.form-select:not(:focus).is-invalid,.was-validated .input-group>.form-control:not(:focus):invalid,.was-validated .input-group>.form-floating:not(:focus-within):invalid,.was-validated .input-group>.form-select:not(:focus):invalid{z-index:4}.btn{--bs-btn-padding-x:0.75rem;--bs-btn-padding-y:0.375rem;--bs-btn-font-family: ;--bs-btn-font-size:1rem;--bs-btn-font-weight:400;--bs-btn-line-height:1.5;--bs-btn-color:var(--bs-body-color);--bs-btn-bg:transparent;--bs-btn-border-width:var(--bs-border-width);--bs-btn-border-color:transparent;--bs-btn-border-radius:var(--bs-border-radius);--bs-btn-hover-border-color:transparent;--bs-btn-box-shadow:inset 0 1px 0 rgba(255, 255, 255, 0.15),0 1px 1px rgba(0, 0, 0, 0.075);--bs-btn-disabled-opacity:0.65;--bs-btn-focus-box-shadow:0 0 0 0.25rem rgba(var(--bs-btn-focus-shadow-rgb), .5);display:inline-block;padding:var(--bs-btn-padding-y) var(--bs-btn-padding-x);font-family:var(--bs-btn-font-family);font-size:var(--bs-btn-font-size);font-weight:var(--bs-btn-font-weight);line-height:var(--bs-btn-line-height);color:var(--bs-btn-color);text-align:center;text-decoration:none;vertical-align:middle;cursor:pointer;-webkit-user-select:none;-moz-user-select:none;user-select:none;border:var(--bs-btn-border-width) solid var(--bs-btn-border-color);border-radius:var(--bs-btn-border-radius);background-color:var(--bs-btn-bg);transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.btn{transition:none}}.btn:hover{color:var(--bs-btn-hover-color);background-color:var(--bs-btn-hover-bg);border-color:var(--bs-btn-hover-border-color)}.btn-check+.btn:hover{color:var(--bs-btn-color);background-color:var(--bs-btn-bg);border-color:var(--bs-btn-border-color)}.btn:focus-visible{color:var(--bs-btn-hover-color);background-color:var(--bs-btn-hover-bg);border-color:var(--bs-btn-hover-border-color);outline:0;box-shadow:var(--bs-btn-focus-box-shadow)}.btn-check:focus-visible+.btn{border-color:var(--bs-btn-hover-border-color);outline:0;box-shadow:var(--bs-btn-focus-box-shadow)}.btn-check:checked+.btn,.btn.active,.btn.show,.btn:first-child:active,:not(.btn-check)+.btn:active{color:var(--bs-btn-active-color);background-color:var(--bs-btn-active-bg);border-color:var(--bs-btn-active-border-color)}.btn-check:checked+.btn:focus-visible,.btn.active:focus-visible,.btn.show:focus-visible,.btn:first-child:active:focus-visible,:not(.btn-check)+.btn:active:focus-visible{box-shadow:var(--bs-btn-focus-box-shadow)}.btn.disabled,.btn:disabled,fieldset:disabled .btn{color:var(--bs-btn-disabled-color);pointer-events:none;background-color:var(--bs-btn-disabled-bg);border-color:var(--bs-btn-disabled-border-color);opacity:var(--bs-btn-disabled-opacity)}.btn-primary{--bs-btn-color:#fff;--bs-btn-bg:#0d6efd;--bs-btn-border-color:#0d6efd;--bs-btn-hover-color:#fff;--bs-btn-hover-bg:#0b5ed7;--bs-btn-hover-border-color:#0a58ca;--bs-btn-focus-shadow-rgb:49,132,253;--bs-btn-active-color:#fff;--bs-btn-active-bg:#0a58ca;--bs-btn-active-border-color:#0a53be;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#fff;--bs-btn-disabled-bg:#0d6efd;--bs-btn-disabled-border-color:#0d6efd}.btn-secondary{--bs-btn-color:#fff;--bs-btn-bg:#6c757d;--bs-btn-border-color:#6c757d;--bs-btn-hover-color:#fff;--bs-btn-hover-bg:#5c636a;--bs-btn-hover-border-color:#565e64;--bs-btn-focus-shadow-rgb:130,138,145;--bs-btn-active-color:#fff;--bs-btn-active-bg:#565e64;--bs-btn-active-border-color:#51585e;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#fff;--bs-btn-disabled-bg:#6c757d;--bs-btn-disabled-border-color:#6c757d}.btn-success{--bs-btn-color:#fff;--bs-btn-bg:#198754;--bs-btn-border-color:#198754;--bs-btn-hover-color:#fff;--bs-btn-hover-bg:#157347;--bs-btn-hover-border-color:#146c43;--bs-btn-focus-shadow-rgb:60,153,110;--bs-btn-active-color:#fff;--bs-btn-active-bg:#146c43;--bs-btn-active-border-color:#13653f;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#fff;--bs-btn-disabled-bg:#198754;--bs-btn-disabled-border-color:#198754}.btn-info{--bs-btn-color:#000;--bs-btn-bg:#0dcaf0;--bs-btn-border-color:#0dcaf0;--bs-btn-hover-color:#000;--bs-btn-hover-bg:#31d2f2;--bs-btn-hover-border-color:#25cff2;--bs-btn-focus-shadow-rgb:11,172,204;--bs-btn-active-color:#000;--bs-btn-active-bg:#3dd5f3;--bs-btn-active-border-color:#25cff2;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#000;--bs-btn-disabled-bg:#0dcaf0;--bs-btn-disabled-border-color:#0dcaf0}.btn-warning{--bs-btn-color:#000;--bs-btn-bg:#ffc107;--bs-btn-border-color:#ffc107;--bs-btn-hover-color:#000;--bs-btn-hover-bg:#ffca2c;--bs-btn-hover-border-color:#ffc720;--bs-btn-focus-shadow-rgb:217,164,6;--bs-btn-active-color:#000;--bs-btn-active-bg:#ffcd39;--bs-btn-active-border-color:#ffc720;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#000;--bs-btn-disabled-bg:#ffc107;--bs-btn-disabled-border-color:#ffc107}.btn-danger{--bs-btn-color:#fff;--bs-btn-bg:#dc3545;--bs-btn-border-color:#dc3545;--bs-btn-hover-color:#fff;--bs-btn-hover-bg:#bb2d3b;--bs-btn-hover-border-color:#b02a37;--bs-btn-focus-shadow-rgb:225,83,97;--bs-btn-active-color:#fff;--bs-btn-active-bg:#b02a37;--bs-btn-active-border-color:#a52834;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#fff;--bs-btn-disabled-bg:#dc3545;--bs-btn-disabled-border-color:#dc3545}.btn-light{--bs-btn-color:#000;--bs-btn-bg:#f8f9fa;--bs-btn-border-color:#f8f9fa;--bs-btn-hover-color:#000;--bs-btn-hover-bg:#d3d4d5;--bs-btn-hover-border-color:#c6c7c8;--bs-btn-focus-shadow-rgb:211,212,213;--bs-btn-active-color:#000;--bs-btn-active-bg:#c6c7c8;--bs-btn-active-border-color:#babbbc;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#000;--bs-btn-disabled-bg:#f8f9fa;--bs-btn-disabled-border-color:#f8f9fa}.btn-dark{--bs-btn-color:#fff;--bs-btn-bg:#212529;--bs-btn-border-color:#212529;--bs-btn-hover-color:#fff;--bs-btn-hover-bg:#424649;--bs-btn-hover-border-color:#373b3e;--bs-btn-focus-shadow-rgb:66,70,73;--bs-btn-active-color:#fff;--bs-btn-active-bg:#4d5154;--bs-btn-active-border-color:#373b3e;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#fff;--bs-btn-disabled-bg:#212529;--bs-btn-disabled-border-color:#212529}.btn-outline-primary{--bs-btn-color:#0d6efd;--bs-btn-border-color:#0d6efd;--bs-btn-hover-color:#fff;--bs-btn-hover-bg:#0d6efd;--bs-btn-hover-border-color:#0d6efd;--bs-btn-focus-shadow-rgb:13,110,253;--bs-btn-active-color:#fff;--bs-btn-active-bg:#0d6efd;--bs-btn-active-border-color:#0d6efd;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#0d6efd;--bs-btn-disabled-bg:transparent;--bs-btn-disabled-border-color:#0d6efd;--bs-gradient:none}.btn-outline-secondary{--bs-btn-color:#6c757d;--bs-btn-border-color:#6c757d;--bs-btn-hover-color:#fff;--bs-btn-hover-bg:#6c757d;--bs-btn-hover-border-color:#6c757d;--bs-btn-focus-shadow-rgb:108,117,125;--bs-btn-active-color:#fff;--bs-btn-active-bg:#6c757d;--bs-btn-active-border-color:#6c757d;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#6c757d;--bs-btn-disabled-bg:transparent;--bs-btn-disabled-border-color:#6c757d;--bs-gradient:none}.btn-outline-success{--bs-btn-color:#198754;--bs-btn-border-color:#198754;--bs-btn-hover-color:#fff;--bs-btn-hover-bg:#198754;--bs-btn-hover-border-color:#198754;--bs-btn-focus-shadow-rgb:25,135,84;--bs-btn-active-color:#fff;--bs-btn-active-bg:#198754;--bs-btn-active-border-color:#198754;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#198754;--bs-btn-disabled-bg:transparent;--bs-btn-disabled-border-color:#198754;--bs-gradient:none}.btn-outline-info{--bs-btn-color:#0dcaf0;--bs-btn-border-color:#0dcaf0;--bs-btn-hover-color:#000;--bs-btn-hover-bg:#0dcaf0;--bs-btn-hover-border-color:#0dcaf0;--bs-btn-focus-shadow-rgb:13,202,240;--bs-btn-active-color:#000;--bs-btn-active-bg:#0dcaf0;--bs-btn-active-border-color:#0dcaf0;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#0dcaf0;--bs-btn-disabled-bg:transparent;--bs-btn-disabled-border-color:#0dcaf0;--bs-gradient:none}.btn-outline-warning{--bs-btn-color:#ffc107;--bs-btn-border-color:#ffc107;--bs-btn-hover-color:#000;--bs-btn-hover-bg:#ffc107;--bs-btn-hover-border-color:#ffc107;--bs-btn-focus-shadow-rgb:255,193,7;--bs-btn-active-color:#000;--bs-btn-active-bg:#ffc107;--bs-btn-active-border-color:#ffc107;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#ffc107;--bs-btn-disabled-bg:transparent;--bs-btn-disabled-border-color:#ffc107;--bs-gradient:none}.btn-outline-danger{--bs-btn-color:#dc3545;--bs-btn-border-color:#dc3545;--bs-btn-hover-color:#fff;--bs-btn-hover-bg:#dc3545;--bs-btn-hover-border-color:#dc3545;--bs-btn-focus-shadow-rgb:220,53,69;--bs-btn-active-color:#fff;--bs-btn-active-bg:#dc3545;--bs-btn-active-border-color:#dc3545;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#dc3545;--bs-btn-disabled-bg:transparent;--bs-btn-disabled-border-color:#dc3545;--bs-gradient:none}.btn-outline-light{--bs-btn-color:#f8f9fa;--bs-btn-border-color:#f8f9fa;--bs-btn-hover-color:#000;--bs-btn-hover-bg:#f8f9fa;--bs-btn-hover-border-color:#f8f9fa;--bs-btn-focus-shadow-rgb:248,249,250;--bs-btn-active-color:#000;--bs-btn-active-bg:#f8f9fa;--bs-btn-active-border-color:#f8f9fa;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#f8f9fa;--bs-btn-disabled-bg:transparent;--bs-btn-disabled-border-color:#f8f9fa;--bs-gradient:none}.btn-outline-dark{--bs-btn-color:#212529;--bs-btn-border-color:#212529;--bs-btn-hover-color:#fff;--bs-btn-hover-bg:#212529;--bs-btn-hover-border-color:#212529;--bs-btn-focus-shadow-rgb:33,37,41;--bs-btn-active-color:#fff;--bs-btn-active-bg:#212529;--bs-btn-active-border-color:#212529;--bs-btn-active-shadow:inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color:#212529;--bs-btn-disabled-bg:transparent;--bs-btn-disabled-border-color:#212529;--bs-gradient:none}.btn-link{--bs-btn-font-weight:400;--bs-btn-color:var(--bs-link-color);--bs-btn-bg:transparent;--bs-btn-border-color:transparent;--bs-btn-hover-color:var(--bs-link-hover-color);--bs-btn-hover-border-color:transparent;--bs-btn-active-color:var(--bs-link-hover-color);--bs-btn-active-border-color:transparent;--bs-btn-disabled-color:#6c757d;--bs-btn-disabled-border-color:transparent;--bs-btn-box-shadow:0 0 0 #000;--bs-btn-focus-shadow-rgb:49,132,253;text-decoration:underline}.btn-link:focus-visible{color:var(--bs-btn-color)}.btn-link:hover{color:var(--bs-btn-hover-color)}.btn-group-lg>.btn,.btn-lg{--bs-btn-padding-y:0.5rem;--bs-btn-padding-x:1rem;--bs-btn-font-size:1.25rem;--bs-btn-border-radius:var(--bs-border-radius-lg)}.btn-group-sm>.btn,.btn-sm{--bs-btn-padding-y:0.25rem;--bs-btn-padding-x:0.5rem;--bs-btn-font-size:0.875rem;--bs-btn-border-radius:var(--bs-border-radius-sm)}.fade{transition:opacity .15s linear}@media (prefers-reduced-motion:reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{height:0;overflow:hidden;transition:height .35s ease}@media (prefers-reduced-motion:reduce){.collapsing{transition:none}}.collapsing.collapse-horizontal{width:0;height:auto;transition:width .35s ease}@media (prefers-reduced-motion:reduce){.collapsing.collapse-horizontal{transition:none}}.dropdown,.dropdown-center,.dropend,.dropstart,.dropup,.dropup-center{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-bottom:0;border-left:.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{--bs-dropdown-zindex:1000;--bs-dropdown-min-width:10rem;--bs-dropdown-padding-x:0;--bs-dropdown-padding-y:0.5rem;--bs-dropdown-spacer:0.125rem;--bs-dropdown-font-size:1rem;--bs-dropdown-color:var(--bs-body-color);--bs-dropdown-bg:var(--bs-body-bg);--bs-dropdown-border-color:var(--bs-border-color-translucent);--bs-dropdown-border-radius:var(--bs-border-radius);--bs-dropdown-border-width:var(--bs-border-width);--bs-dropdown-inner-border-radius:calc(var(--bs-border-radius) - var(--bs-border-width));--bs-dropdown-divider-bg:var(--bs-border-color-translucent);--bs-dropdown-divider-margin-y:0.5rem;--bs-dropdown-box-shadow:var(--bs-box-shadow);--bs-dropdown-link-color:var(--bs-body-color);--bs-dropdown-link-hover-color:var(--bs-body-color);--bs-dropdown-link-hover-bg:var(--bs-tertiary-bg);--bs-dropdown-link-active-color:#fff;--bs-dropdown-link-active-bg:#0d6efd;--bs-dropdown-link-disabled-color:var(--bs-tertiary-color);--bs-dropdown-item-padding-x:1rem;--bs-dropdown-item-padding-y:0.25rem;--bs-dropdown-header-color:#6c757d;--bs-dropdown-header-padding-x:1rem;--bs-dropdown-header-padding-y:0.5rem;position:absolute;z-index:var(--bs-dropdown-zindex);display:none;min-width:var(--bs-dropdown-min-width);padding:var(--bs-dropdown-padding-y) var(--bs-dropdown-padding-x);margin:0;font-size:var(--bs-dropdown-font-size);color:var(--bs-dropdown-color);text-align:left;list-style:none;background-color:var(--bs-dropdown-bg);background-clip:padding-box;border:var(--bs-dropdown-border-width) solid var(--bs-dropdown-border-color);border-radius:var(--bs-dropdown-border-radius)}.dropdown-menu[data-bs-popper]{top:100%;left:0;margin-top:var(--bs-dropdown-spacer)}.dropdown-menu-start{--bs-position:start}.dropdown-menu-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-end{--bs-position:end}.dropdown-menu-end[data-bs-popper]{right:0;left:auto}@media (min-width:576px){.dropdown-menu-sm-start{--bs-position:start}.dropdown-menu-sm-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-sm-end{--bs-position:end}.dropdown-menu-sm-end[data-bs-popper]{right:0;left:auto}}@media (min-width:768px){.dropdown-menu-md-start{--bs-position:start}.dropdown-menu-md-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-md-end{--bs-position:end}.dropdown-menu-md-end[data-bs-popper]{right:0;left:auto}}@media (min-width:992px){.dropdown-menu-lg-start{--bs-position:start}.dropdown-menu-lg-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-lg-end{--bs-position:end}.dropdown-menu-lg-end[data-bs-popper]{right:0;left:auto}}@media (min-width:1200px){.dropdown-menu-xl-start{--bs-position:start}.dropdown-menu-xl-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-xl-end{--bs-position:end}.dropdown-menu-xl-end[data-bs-popper]{right:0;left:auto}}@media (min-width:1400px){.dropdown-menu-xxl-start{--bs-position:start}.dropdown-menu-xxl-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-xxl-end{--bs-position:end}.dropdown-menu-xxl-end[data-bs-popper]{right:0;left:auto}}.dropup .dropdown-menu[data-bs-popper]{top:auto;bottom:100%;margin-top:0;margin-bottom:var(--bs-dropdown-spacer)}.dropup .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid transparent;border-bottom:.3em solid;border-left:.3em solid transparent}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropend .dropdown-menu[data-bs-popper]{top:0;right:auto;left:100%;margin-top:0;margin-left:var(--bs-dropdown-spacer)}.dropend .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:0;border-bottom:.3em solid transparent;border-left:.3em solid}.dropend .dropdown-toggle:empty::after{margin-left:0}.dropend .dropdown-toggle::after{vertical-align:0}.dropstart .dropdown-menu[data-bs-popper]{top:0;right:100%;left:auto;margin-top:0;margin-right:var(--bs-dropdown-spacer)}.dropstart .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropstart .dropdown-toggle::after{display:none}.dropstart .dropdown-toggle::before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:.3em solid;border-bottom:.3em solid transparent}.dropstart .dropdown-toggle:empty::after{margin-left:0}.dropstart .dropdown-toggle::before{vertical-align:0}.dropdown-divider{height:0;margin:var(--bs-dropdown-divider-margin-y) 0;overflow:hidden;border-top:1px solid var(--bs-dropdown-divider-bg);opacity:1}.dropdown-item{display:block;width:100%;padding:var(--bs-dropdown-item-padding-y) var(--bs-dropdown-item-padding-x);clear:both;font-weight:400;color:var(--bs-dropdown-link-color);text-align:inherit;text-decoration:none;white-space:nowrap;background-color:transparent;border:0;border-radius:var(--bs-dropdown-item-border-radius,0)}.dropdown-item:focus,.dropdown-item:hover{color:var(--bs-dropdown-link-hover-color);background-color:var(--bs-dropdown-link-hover-bg)}.dropdown-item.active,.dropdown-item:active{color:var(--bs-dropdown-link-active-color);text-decoration:none;background-color:var(--bs-dropdown-link-active-bg)}.dropdown-item.disabled,.dropdown-item:disabled{color:var(--bs-dropdown-link-disabled-color);pointer-events:none;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:var(--bs-dropdown-header-padding-y) var(--bs-dropdown-header-padding-x);margin-bottom:0;font-size:.875rem;color:var(--bs-dropdown-header-color);white-space:nowrap}.dropdown-item-text{display:block;padding:var(--bs-dropdown-item-padding-y) var(--bs-dropdown-item-padding-x);color:var(--bs-dropdown-link-color)}.dropdown-menu-dark{--bs-dropdown-color:#dee2e6;--bs-dropdown-bg:#343a40;--bs-dropdown-border-color:var(--bs-border-color-translucent);--bs-dropdown-box-shadow: ;--bs-dropdown-link-color:#dee2e6;--bs-dropdown-link-hover-color:#fff;--bs-dropdown-divider-bg:var(--bs-border-color-translucent);--bs-dropdown-link-hover-bg:rgba(255, 255, 255, 0.15);--bs-dropdown-link-active-color:#fff;--bs-dropdown-link-active-bg:#0d6efd;--bs-dropdown-link-disabled-color:#adb5bd;--bs-dropdown-header-color:#adb5bd}.btn-group,.btn-group-vertical{position:relative;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;flex:1 1 auto}.btn-group-vertical>.btn-check:checked+.btn,.btn-group-vertical>.btn-check:focus+.btn,.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group-vertical>.btn:hover,.btn-group>.btn-check:checked+.btn,.btn-group>.btn-check:focus+.btn,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus,.btn-group>.btn:hover{z-index:1}.btn-toolbar{display:flex;flex-wrap:wrap;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group{border-radius:var(--bs-border-radius)}.btn-group>.btn-group:not(:first-child),.btn-group>:not(.btn-check:first-child)+.btn{margin-left:calc(var(--bs-border-width) * -1)}.btn-group>.btn-group:not(:last-child)>.btn,.btn-group>.btn.dropdown-toggle-split:first-child,.btn-group>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:not(:first-child)>.btn,.btn-group>.btn:nth-child(n+3),.btn-group>:not(.btn-check)+.btn{border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split::after,.dropend .dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after{margin-left:0}.dropstart .dropdown-toggle-split::before{margin-right:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{flex-direction:column;align-items:flex-start;justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn-group:not(:first-child),.btn-group-vertical>.btn:not(:first-child){margin-top:calc(var(--bs-border-width) * -1)}.btn-group-vertical>.btn-group:not(:last-child)>.btn,.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child)>.btn,.btn-group-vertical>.btn~.btn{border-top-left-radius:0;border-top-right-radius:0}.nav{--bs-nav-link-padding-x:1rem;--bs-nav-link-padding-y:0.5rem;--bs-nav-link-font-weight: ;--bs-nav-link-color:var(--bs-link-color);--bs-nav-link-hover-color:var(--bs-link-hover-color);--bs-nav-link-disabled-color:var(--bs-secondary-color);display:flex;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:var(--bs-nav-link-padding-y) var(--bs-nav-link-padding-x);font-size:var(--bs-nav-link-font-size);font-weight:var(--bs-nav-link-font-weight);color:var(--bs-nav-link-color);text-decoration:none;background:0 0;border:0;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out}@media (prefers-reduced-motion:reduce){.nav-link{transition:none}}.nav-link:focus,.nav-link:hover{color:var(--bs-nav-link-hover-color)}.nav-link:focus-visible{outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.nav-link.disabled,.nav-link:disabled{color:var(--bs-nav-link-disabled-color);pointer-events:none;cursor:default}.nav-tabs{--bs-nav-tabs-border-width:var(--bs-border-width);--bs-nav-tabs-border-color:var(--bs-border-color);--bs-nav-tabs-border-radius:var(--bs-border-radius);--bs-nav-tabs-link-hover-border-color:var(--bs-secondary-bg) var(--bs-secondary-bg) var(--bs-border-color);--bs-nav-tabs-link-active-color:var(--bs-emphasis-color);--bs-nav-tabs-link-active-bg:var(--bs-body-bg);--bs-nav-tabs-link-active-border-color:var(--bs-border-color) var(--bs-border-color) var(--bs-body-bg);border-bottom:var(--bs-nav-tabs-border-width) solid var(--bs-nav-tabs-border-color)}.nav-tabs .nav-link{margin-bottom:calc(-1 * var(--bs-nav-tabs-border-width));border:var(--bs-nav-tabs-border-width) solid transparent;border-top-left-radius:var(--bs-nav-tabs-border-radius);border-top-right-radius:var(--bs-nav-tabs-border-radius)}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{isolation:isolate;border-color:var(--bs-nav-tabs-link-hover-border-color)}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:var(--bs-nav-tabs-link-active-color);background-color:var(--bs-nav-tabs-link-active-bg);border-color:var(--bs-nav-tabs-link-active-border-color)}.nav-tabs .dropdown-menu{margin-top:calc(-1 * var(--bs-nav-tabs-border-width));border-top-left-radius:0;border-top-right-radius:0}.nav-pills{--bs-nav-pills-border-radius:var(--bs-border-radius);--bs-nav-pills-link-active-color:#fff;--bs-nav-pills-link-active-bg:#0d6efd}.nav-pills .nav-link{border-radius:var(--bs-nav-pills-border-radius)}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:var(--bs-nav-pills-link-active-color);background-color:var(--bs-nav-pills-link-active-bg)}.nav-underline{--bs-nav-underline-gap:1rem;--bs-nav-underline-border-width:0.125rem;--bs-nav-underline-link-active-color:var(--bs-emphasis-color);gap:var(--bs-nav-underline-gap)}.nav-underline .nav-link{padding-right:0;padding-left:0;border-bottom:var(--bs-nav-underline-border-width) solid transparent}.nav-underline .nav-link:focus,.nav-underline .nav-link:hover{border-bottom-color:currentcolor}.nav-underline .nav-link.active,.nav-underline .show>.nav-link{font-weight:700;color:var(--bs-nav-underline-link-active-color);border-bottom-color:currentcolor}.nav-fill .nav-item,.nav-fill>.nav-link{flex:1 1 auto;text-align:center}.nav-justified .nav-item,.nav-justified>.nav-link{flex-basis:0;flex-grow:1;text-align:center}.nav-fill .nav-item .nav-link,.nav-justified .nav-item .nav-link{width:100%}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{--bs-navbar-padding-x:0;--bs-navbar-padding-y:0.5rem;--bs-navbar-color:rgba(var(--bs-emphasis-color-rgb), 0.65);--bs-navbar-hover-color:rgba(var(--bs-emphasis-color-rgb), 0.8);--bs-navbar-disabled-color:rgba(var(--bs-emphasis-color-rgb), 0.3);--bs-navbar-active-color:rgba(var(--bs-emphasis-color-rgb), 1);--bs-navbar-brand-padding-y:0.3125rem;--bs-navbar-brand-margin-end:1rem;--bs-navbar-brand-font-size:1.25rem;--bs-navbar-brand-color:rgba(var(--bs-emphasis-color-rgb), 1);--bs-navbar-brand-hover-color:rgba(var(--bs-emphasis-color-rgb), 1);--bs-navbar-nav-link-padding-x:0.5rem;--bs-navbar-toggler-padding-y:0.25rem;--bs-navbar-toggler-padding-x:0.75rem;--bs-navbar-toggler-font-size:1.25rem;--bs-navbar-toggler-icon-bg:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%2833, 37, 41, 0.75%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e");--bs-navbar-toggler-border-color:rgba(var(--bs-emphasis-color-rgb), 0.15);--bs-navbar-toggler-border-radius:var(--bs-border-radius);--bs-navbar-toggler-focus-width:0.25rem;--bs-navbar-toggler-transition:box-shadow 0.15s ease-in-out;position:relative;display:flex;flex-wrap:wrap;align-items:center;justify-content:space-between;padding:var(--bs-navbar-padding-y) var(--bs-navbar-padding-x)}.navbar>.container,.navbar>.container-fluid,.navbar>.container-lg,.navbar>.container-md,.navbar>.container-sm,.navbar>.container-xl,.navbar>.container-xxl{display:flex;flex-wrap:inherit;align-items:center;justify-content:space-between}.navbar-brand{padding-top:var(--bs-navbar-brand-padding-y);padding-bottom:var(--bs-navbar-brand-padding-y);margin-right:var(--bs-navbar-brand-margin-end);font-size:var(--bs-navbar-brand-font-size);color:var(--bs-navbar-brand-color);text-decoration:none;white-space:nowrap}.navbar-brand:focus,.navbar-brand:hover{color:var(--bs-navbar-brand-hover-color)}.navbar-nav{--bs-nav-link-padding-x:0;--bs-nav-link-padding-y:0.5rem;--bs-nav-link-font-weight: ;--bs-nav-link-color:var(--bs-navbar-color);--bs-nav-link-hover-color:var(--bs-navbar-hover-color);--bs-nav-link-disabled-color:var(--bs-navbar-disabled-color);display:flex;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link.active,.navbar-nav .nav-link.show{color:var(--bs-navbar-active-color)}.navbar-nav .dropdown-menu{position:static}.navbar-text{padding-top:.5rem;padding-bottom:.5rem;color:var(--bs-navbar-color)}.navbar-text a,.navbar-text a:focus,.navbar-text a:hover{color:var(--bs-navbar-active-color)}.navbar-collapse{flex-basis:100%;flex-grow:1;align-items:center}.navbar-toggler{padding:var(--bs-navbar-toggler-padding-y) var(--bs-navbar-toggler-padding-x);font-size:var(--bs-navbar-toggler-font-size);line-height:1;color:var(--bs-navbar-color);background-color:transparent;border:var(--bs-border-width) solid var(--bs-navbar-toggler-border-color);border-radius:var(--bs-navbar-toggler-border-radius);transition:var(--bs-navbar-toggler-transition)}@media (prefers-reduced-motion:reduce){.navbar-toggler{transition:none}}.navbar-toggler:hover{text-decoration:none}.navbar-toggler:focus{text-decoration:none;outline:0;box-shadow:0 0 0 var(--bs-navbar-toggler-focus-width)}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;background-image:var(--bs-navbar-toggler-icon-bg);background-repeat:no-repeat;background-position:center;background-size:100%}.navbar-nav-scroll{max-height:var(--bs-scroll-height,75vh);overflow-y:auto}@media (min-width:576px){.navbar-expand-sm{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-sm .navbar-nav{flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-sm .navbar-nav-scroll{overflow:visible}.navbar-expand-sm .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}.navbar-expand-sm .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand-sm .offcanvas .offcanvas-header{display:none}.navbar-expand-sm .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:768px){.navbar-expand-md{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-md .navbar-nav{flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-md .navbar-nav-scroll{overflow:visible}.navbar-expand-md .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}.navbar-expand-md .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand-md .offcanvas .offcanvas-header{display:none}.navbar-expand-md .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:992px){.navbar-expand-lg{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-lg .navbar-nav{flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-lg .navbar-nav-scroll{overflow:visible}.navbar-expand-lg .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}.navbar-expand-lg .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand-lg .offcanvas .offcanvas-header{display:none}.navbar-expand-lg .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:1200px){.navbar-expand-xl{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-xl .navbar-nav{flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-xl .navbar-nav-scroll{overflow:visible}.navbar-expand-xl .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}.navbar-expand-xl .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand-xl .offcanvas .offcanvas-header{display:none}.navbar-expand-xl .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:1400px){.navbar-expand-xxl{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-xxl .navbar-nav{flex-direction:row}.navbar-expand-xxl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xxl .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-xxl .navbar-nav-scroll{overflow:visible}.navbar-expand-xxl .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-xxl .navbar-toggler{display:none}.navbar-expand-xxl .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand-xxl .offcanvas .offcanvas-header{display:none}.navbar-expand-xxl .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}.navbar-expand{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand .navbar-nav{flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand .navbar-nav-scroll{overflow:visible}.navbar-expand .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-expand .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand .offcanvas .offcanvas-header{display:none}.navbar-expand .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}.navbar-dark,.navbar[data-bs-theme=dark]{--bs-navbar-color:rgba(255, 255, 255, 0.55);--bs-navbar-hover-color:rgba(255, 255, 255, 0.75);--bs-navbar-disabled-color:rgba(255, 255, 255, 0.25);--bs-navbar-active-color:#fff;--bs-navbar-brand-color:#fff;--bs-navbar-brand-hover-color:#fff;--bs-navbar-toggler-border-color:rgba(255, 255, 255, 0.1);--bs-navbar-toggler-icon-bg:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%28255, 255, 255, 0.55%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}[data-bs-theme=dark] .navbar-toggler-icon{--bs-navbar-toggler-icon-bg:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%28255, 255, 255, 0.55%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.card{--bs-card-spacer-y:1rem;--bs-card-spacer-x:1rem;--bs-card-title-spacer-y:0.5rem;--bs-card-title-color: ;--bs-card-subtitle-color: ;--bs-card-border-width:var(--bs-border-width);--bs-card-border-color:var(--bs-border-color-translucent);--bs-card-border-radius:var(--bs-border-radius);--bs-card-box-shadow: ;--bs-card-inner-border-radius:calc(var(--bs-border-radius) - (var(--bs-border-width)));--bs-card-cap-padding-y:0.5rem;--bs-card-cap-padding-x:1rem;--bs-card-cap-bg:rgba(var(--bs-body-color-rgb), 0.03);--bs-card-cap-color: ;--bs-card-height: ;--bs-card-color: ;--bs-card-bg:var(--bs-body-bg);--bs-card-img-overlay-padding:1rem;--bs-card-group-margin:0.75rem;position:relative;display:flex;flex-direction:column;min-width:0;height:var(--bs-card-height);color:var(--bs-body-color);word-wrap:break-word;background-color:var(--bs-card-bg);background-clip:border-box;border:var(--bs-card-border-width) solid var(--bs-card-border-color);border-radius:var(--bs-card-border-radius)}.card>hr{margin-right:0;margin-left:0}.card>.list-group{border-top:inherit;border-bottom:inherit}.card>.list-group:first-child{border-top-width:0;border-top-left-radius:var(--bs-card-inner-border-radius);border-top-right-radius:var(--bs-card-inner-border-radius)}.card>.list-group:last-child{border-bottom-width:0;border-bottom-right-radius:var(--bs-card-inner-border-radius);border-bottom-left-radius:var(--bs-card-inner-border-radius)}.card>.card-header+.list-group,.card>.list-group+.card-footer{border-top:0}.card-body{flex:1 1 auto;padding:var(--bs-card-spacer-y) var(--bs-card-spacer-x);color:var(--bs-card-color)}.card-title{margin-bottom:var(--bs-card-title-spacer-y);color:var(--bs-card-title-color)}.card-subtitle{margin-top:calc(-.5 * var(--bs-card-title-spacer-y));margin-bottom:0;color:var(--bs-card-subtitle-color)}.card-text:last-child{margin-bottom:0}.card-link+.card-link{margin-left:var(--bs-card-spacer-x)}.card-header{padding:var(--bs-card-cap-padding-y) var(--bs-card-cap-padding-x);margin-bottom:0;color:var(--bs-card-cap-color);background-color:var(--bs-card-cap-bg);border-bottom:var(--bs-card-border-width) solid var(--bs-card-border-color)}.card-header:first-child{border-radius:var(--bs-card-inner-border-radius) var(--bs-card-inner-border-radius) 0 0}.card-footer{padding:var(--bs-card-cap-padding-y) var(--bs-card-cap-padding-x);color:var(--bs-card-cap-color);background-color:var(--bs-card-cap-bg);border-top:var(--bs-card-border-width) solid var(--bs-card-border-color)}.card-footer:last-child{border-radius:0 0 var(--bs-card-inner-border-radius) var(--bs-card-inner-border-radius)}.card-header-tabs{margin-right:calc(-.5 * var(--bs-card-cap-padding-x));margin-bottom:calc(-1 * var(--bs-card-cap-padding-y));margin-left:calc(-.5 * var(--bs-card-cap-padding-x));border-bottom:0}.card-header-tabs .nav-link.active{background-color:var(--bs-card-bg);border-bottom-color:var(--bs-card-bg)}.card-header-pills{margin-right:calc(-.5 * var(--bs-card-cap-padding-x));margin-left:calc(-.5 * var(--bs-card-cap-padding-x))}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:var(--bs-card-img-overlay-padding);border-radius:var(--bs-card-inner-border-radius)}.card-img,.card-img-bottom,.card-img-top{width:100%}.card-img,.card-img-top{border-top-left-radius:var(--bs-card-inner-border-radius);border-top-right-radius:var(--bs-card-inner-border-radius)}.card-img,.card-img-bottom{border-bottom-right-radius:var(--bs-card-inner-border-radius);border-bottom-left-radius:var(--bs-card-inner-border-radius)}.card-group>.card{margin-bottom:var(--bs-card-group-margin)}@media (min-width:576px){.card-group{display:flex;flex-flow:row wrap}.card-group>.card{flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:not(:last-child) .card-header,.card-group>.card:not(:last-child) .card-img-top{border-top-right-radius:0}.card-group>.card:not(:last-child) .card-footer,.card-group>.card:not(:last-child) .card-img-bottom{border-bottom-right-radius:0}.card-group>.card:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:not(:first-child) .card-header,.card-group>.card:not(:first-child) .card-img-top{border-top-left-radius:0}.card-group>.card:not(:first-child) .card-footer,.card-group>.card:not(:first-child) .card-img-bottom{border-bottom-left-radius:0}}.accordion{--bs-accordion-color:var(--bs-body-color);--bs-accordion-bg:var(--bs-body-bg);--bs-accordion-transition:color 0.15s ease-in-out,background-color 0.15s ease-in-out,border-color 0.15s ease-in-out,box-shadow 0.15s ease-in-out,border-radius 0.15s ease;--bs-accordion-border-color:var(--bs-border-color);--bs-accordion-border-width:var(--bs-border-width);--bs-accordion-border-radius:var(--bs-border-radius);--bs-accordion-inner-border-radius:calc(var(--bs-border-radius) - (var(--bs-border-width)));--bs-accordion-btn-padding-x:1.25rem;--bs-accordion-btn-padding-y:1rem;--bs-accordion-btn-color:var(--bs-body-color);--bs-accordion-btn-bg:var(--bs-accordion-bg);--bs-accordion-btn-icon:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23212529'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e");--bs-accordion-btn-icon-width:1.25rem;--bs-accordion-btn-icon-transform:rotate(-180deg);--bs-accordion-btn-icon-transition:transform 0.2s ease-in-out;--bs-accordion-btn-active-icon:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23052c65'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e");--bs-accordion-btn-focus-border-color:#86b7fe;--bs-accordion-btn-focus-box-shadow:0 0 0 0.25rem rgba(13, 110, 253, 0.25);--bs-accordion-body-padding-x:1.25rem;--bs-accordion-body-padding-y:1rem;--bs-accordion-active-color:var(--bs-primary-text-emphasis);--bs-accordion-active-bg:var(--bs-primary-bg-subtle)}.accordion-button{position:relative;display:flex;align-items:center;width:100%;padding:var(--bs-accordion-btn-padding-y) var(--bs-accordion-btn-padding-x);font-size:1rem;color:var(--bs-accordion-btn-color);text-align:left;background-color:var(--bs-accordion-btn-bg);border:0;border-radius:0;overflow-anchor:none;transition:var(--bs-accordion-transition)}@media (prefers-reduced-motion:reduce){.accordion-button{transition:none}}.accordion-button:not(.collapsed){color:var(--bs-accordion-active-color);background-color:var(--bs-accordion-active-bg);box-shadow:inset 0 calc(-1 * var(--bs-accordion-border-width)) 0 var(--bs-accordion-border-color)}.accordion-button:not(.collapsed)::after{background-image:var(--bs-accordion-btn-active-icon);transform:var(--bs-accordion-btn-icon-transform)}.accordion-button::after{flex-shrink:0;width:var(--bs-accordion-btn-icon-width);height:var(--bs-accordion-btn-icon-width);margin-left:auto;content:"";background-image:var(--bs-accordion-btn-icon);background-repeat:no-repeat;background-size:var(--bs-accordion-btn-icon-width);transition:var(--bs-accordion-btn-icon-transition)}@media (prefers-reduced-motion:reduce){.accordion-button::after{transition:none}}.accordion-button:hover{z-index:2}.accordion-button:focus{z-index:3;border-color:var(--bs-accordion-btn-focus-border-color);outline:0;box-shadow:var(--bs-accordion-btn-focus-box-shadow)}.accordion-header{margin-bottom:0}.accordion-item{color:var(--bs-accordion-color);background-color:var(--bs-accordion-bg);border:var(--bs-accordion-border-width) solid var(--bs-accordion-border-color)}.accordion-item:first-of-type{border-top-left-radius:var(--bs-accordion-border-radius);border-top-right-radius:var(--bs-accordion-border-radius)}.accordion-item:first-of-type .accordion-button{border-top-left-radius:var(--bs-accordion-inner-border-radius);border-top-right-radius:var(--bs-accordion-inner-border-radius)}.accordion-item:not(:first-of-type){border-top:0}.accordion-item:last-of-type{border-bottom-right-radius:var(--bs-accordion-border-radius);border-bottom-left-radius:var(--bs-accordion-border-radius)}.accordion-item:last-of-type .accordion-button.collapsed{border-bottom-right-radius:var(--bs-accordion-inner-border-radius);border-bottom-left-radius:var(--bs-accordion-inner-border-radius)}.accordion-item:last-of-type .accordion-collapse{border-bottom-right-radius:var(--bs-accordion-border-radius);border-bottom-left-radius:var(--bs-accordion-border-radius)}.accordion-body{padding:var(--bs-accordion-body-padding-y) var(--bs-accordion-body-padding-x)}.accordion-flush .accordion-collapse{border-width:0}.accordion-flush .accordion-item{border-right:0;border-left:0;border-radius:0}.accordion-flush .accordion-item:first-child{border-top:0}.accordion-flush .accordion-item:last-child{border-bottom:0}.accordion-flush .accordion-item .accordion-button,.accordion-flush .accordion-item .accordion-button.collapsed{border-radius:0}[data-bs-theme=dark] .accordion-button::after{--bs-accordion-btn-icon:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%236ea8fe'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e");--bs-accordion-btn-active-icon:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%236ea8fe'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e")}.breadcrumb{--bs-breadcrumb-padding-x:0;--bs-breadcrumb-padding-y:0;--bs-breadcrumb-margin-bottom:1rem;--bs-breadcrumb-bg: ;--bs-breadcrumb-border-radius: ;--bs-breadcrumb-divider-color:var(--bs-secondary-color);--bs-breadcrumb-item-padding-x:0.5rem;--bs-breadcrumb-item-active-color:var(--bs-secondary-color);display:flex;flex-wrap:wrap;padding:var(--bs-breadcrumb-padding-y) var(--bs-breadcrumb-padding-x);margin-bottom:var(--bs-breadcrumb-margin-bottom);font-size:var(--bs-breadcrumb-font-size);list-style:none;background-color:var(--bs-breadcrumb-bg);border-radius:var(--bs-breadcrumb-border-radius)}.breadcrumb-item+.breadcrumb-item{padding-left:var(--bs-breadcrumb-item-padding-x)}.breadcrumb-item+.breadcrumb-item::before{float:left;padding-right:var(--bs-breadcrumb-item-padding-x);color:var(--bs-breadcrumb-divider-color);content:var(--bs-breadcrumb-divider, "/")}.breadcrumb-item.active{color:var(--bs-breadcrumb-item-active-color)}.pagination{--bs-pagination-padding-x:0.75rem;--bs-pagination-padding-y:0.375rem;--bs-pagination-font-size:1rem;--bs-pagination-color:var(--bs-link-color);--bs-pagination-bg:var(--bs-body-bg);--bs-pagination-border-width:var(--bs-border-width);--bs-pagination-border-color:var(--bs-border-color);--bs-pagination-border-radius:var(--bs-border-radius);--bs-pagination-hover-color:var(--bs-link-hover-color);--bs-pagination-hover-bg:var(--bs-tertiary-bg);--bs-pagination-hover-border-color:var(--bs-border-color);--bs-pagination-focus-color:var(--bs-link-hover-color);--bs-pagination-focus-bg:var(--bs-secondary-bg);--bs-pagination-focus-box-shadow:0 0 0 0.25rem rgba(13, 110, 253, 0.25);--bs-pagination-active-color:#fff;--bs-pagination-active-bg:#0d6efd;--bs-pagination-active-border-color:#0d6efd;--bs-pagination-disabled-color:var(--bs-secondary-color);--bs-pagination-disabled-bg:var(--bs-secondary-bg);--bs-pagination-disabled-border-color:var(--bs-border-color);display:flex;padding-left:0;list-style:none}.page-link{position:relative;display:block;padding:var(--bs-pagination-padding-y) var(--bs-pagination-padding-x);font-size:var(--bs-pagination-font-size);color:var(--bs-pagination-color);text-decoration:none;background-color:var(--bs-pagination-bg);border:var(--bs-pagination-border-width) solid var(--bs-pagination-border-color);transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.page-link{transition:none}}.page-link:hover{z-index:2;color:var(--bs-pagination-hover-color);background-color:var(--bs-pagination-hover-bg);border-color:var(--bs-pagination-hover-border-color)}.page-link:focus{z-index:3;color:var(--bs-pagination-focus-color);background-color:var(--bs-pagination-focus-bg);outline:0;box-shadow:var(--bs-pagination-focus-box-shadow)}.active>.page-link,.page-link.active{z-index:3;color:var(--bs-pagination-active-color);background-color:var(--bs-pagination-active-bg);border-color:var(--bs-pagination-active-border-color)}.disabled>.page-link,.page-link.disabled{color:var(--bs-pagination-disabled-color);pointer-events:none;background-color:var(--bs-pagination-disabled-bg);border-color:var(--bs-pagination-disabled-border-color)}.page-item:not(:first-child) .page-link{margin-left:calc(var(--bs-border-width) * -1)}.page-item:first-child .page-link{border-top-left-radius:var(--bs-pagination-border-radius);border-bottom-left-radius:var(--bs-pagination-border-radius)}.page-item:last-child .page-link{border-top-right-radius:var(--bs-pagination-border-radius);border-bottom-right-radius:var(--bs-pagination-border-radius)}.pagination-lg{--bs-pagination-padding-x:1.5rem;--bs-pagination-padding-y:0.75rem;--bs-pagination-font-size:1.25rem;--bs-pagination-border-radius:var(--bs-border-radius-lg)}.pagination-sm{--bs-pagination-padding-x:0.5rem;--bs-pagination-padding-y:0.25rem;--bs-pagination-font-size:0.875rem;--bs-pagination-border-radius:var(--bs-border-radius-sm)}.badge{--bs-badge-padding-x:0.65em;--bs-badge-padding-y:0.35em;--bs-badge-font-size:0.75em;--bs-badge-font-weight:700;--bs-badge-color:#fff;--bs-badge-border-radius:var(--bs-border-radius);display:inline-block;padding:var(--bs-badge-padding-y) var(--bs-badge-padding-x);font-size:var(--bs-badge-font-size);font-weight:var(--bs-badge-font-weight);line-height:1;color:var(--bs-badge-color);text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:var(--bs-badge-border-radius)}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.alert{--bs-alert-bg:transparent;--bs-alert-padding-x:1rem;--bs-alert-padding-y:1rem;--bs-alert-margin-bottom:1rem;--bs-alert-color:inherit;--bs-alert-border-color:transparent;--bs-alert-border:var(--bs-border-width) solid var(--bs-alert-border-color);--bs-alert-border-radius:var(--bs-border-radius);--bs-alert-link-color:inherit;position:relative;padding:var(--bs-alert-padding-y) var(--bs-alert-padding-x);margin-bottom:var(--bs-alert-margin-bottom);color:var(--bs-alert-color);background-color:var(--bs-alert-bg);border:var(--bs-alert-border);border-radius:var(--bs-alert-border-radius)}.alert-heading{color:inherit}.alert-link{font-weight:700;color:var(--bs-alert-link-color)}.alert-dismissible{padding-right:3rem}.alert-dismissible .btn-close{position:absolute;top:0;right:0;z-index:2;padding:1.25rem 1rem}.alert-primary{--bs-alert-color:var(--bs-primary-text-emphasis);--bs-alert-bg:var(--bs-primary-bg-subtle);--bs-alert-border-color:var(--bs-primary-border-subtle);--bs-alert-link-color:var(--bs-primary-text-emphasis)}.alert-secondary{--bs-alert-color:var(--bs-secondary-text-emphasis);--bs-alert-bg:var(--bs-secondary-bg-subtle);--bs-alert-border-color:var(--bs-secondary-border-subtle);--bs-alert-link-color:var(--bs-secondary-text-emphasis)}.alert-success{--bs-alert-color:var(--bs-success-text-emphasis);--bs-alert-bg:var(--bs-success-bg-subtle);--bs-alert-border-color:var(--bs-success-border-subtle);--bs-alert-link-color:var(--bs-success-text-emphasis)}.alert-info{--bs-alert-color:var(--bs-info-text-emphasis);--bs-alert-bg:var(--bs-info-bg-subtle);--bs-alert-border-color:var(--bs-info-border-subtle);--bs-alert-link-color:var(--bs-info-text-emphasis)}.alert-warning{--bs-alert-color:var(--bs-warning-text-emphasis);--bs-alert-bg:var(--bs-warning-bg-subtle);--bs-alert-border-color:var(--bs-warning-border-subtle);--bs-alert-link-color:var(--bs-warning-text-emphasis)}.alert-danger{--bs-alert-color:var(--bs-danger-text-emphasis);--bs-alert-bg:var(--bs-danger-bg-subtle);--bs-alert-border-color:var(--bs-danger-border-subtle);--bs-alert-link-color:var(--bs-danger-text-emphasis)}.alert-light{--bs-alert-color:var(--bs-light-text-emphasis);--bs-alert-bg:var(--bs-light-bg-subtle);--bs-alert-border-color:var(--bs-light-border-subtle);--bs-alert-link-color:var(--bs-light-text-emphasis)}.alert-dark{--bs-alert-color:var(--bs-dark-text-emphasis);--bs-alert-bg:var(--bs-dark-bg-subtle);--bs-alert-border-color:var(--bs-dark-border-subtle);--bs-alert-link-color:var(--bs-dark-text-emphasis)}@keyframes progress-bar-stripes{0%{background-position-x:1rem}}.progress,.progress-stacked{--bs-progress-height:1rem;--bs-progress-font-size:0.75rem;--bs-progress-bg:var(--bs-secondary-bg);--bs-progress-border-radius:var(--bs-border-radius);--bs-progress-box-shadow:var(--bs-box-shadow-inset);--bs-progress-bar-color:#fff;--bs-progress-bar-bg:#0d6efd;--bs-progress-bar-transition:width 0.6s ease;display:flex;height:var(--bs-progress-height);overflow:hidden;font-size:var(--bs-progress-font-size);background-color:var(--bs-progress-bg);border-radius:var(--bs-progress-border-radius)}.progress-bar{display:flex;flex-direction:column;justify-content:center;overflow:hidden;color:var(--bs-progress-bar-color);text-align:center;white-space:nowrap;background-color:var(--bs-progress-bar-bg);transition:var(--bs-progress-bar-transition)}@media (prefers-reduced-motion:reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-size:var(--bs-progress-height) var(--bs-progress-height)}.progress-stacked>.progress{overflow:visible}.progress-stacked>.progress>.progress-bar{width:100%}.progress-bar-animated{animation:1s linear infinite progress-bar-stripes}@media (prefers-reduced-motion:reduce){.progress-bar-animated{animation:none}}.list-group{--bs-list-group-color:var(--bs-body-color);--bs-list-group-bg:var(--bs-body-bg);--bs-list-group-border-color:var(--bs-border-color);--bs-list-group-border-width:var(--bs-border-width);--bs-list-group-border-radius:var(--bs-border-radius);--bs-list-group-item-padding-x:1rem;--bs-list-group-item-padding-y:0.5rem;--bs-list-group-action-color:var(--bs-secondary-color);--bs-list-group-action-hover-color:var(--bs-emphasis-color);--bs-list-group-action-hover-bg:var(--bs-tertiary-bg);--bs-list-group-action-active-color:var(--bs-body-color);--bs-list-group-action-active-bg:var(--bs-secondary-bg);--bs-list-group-disabled-color:var(--bs-secondary-color);--bs-list-group-disabled-bg:var(--bs-body-bg);--bs-list-group-active-color:#fff;--bs-list-group-active-bg:#0d6efd;--bs-list-group-active-border-color:#0d6efd;display:flex;flex-direction:column;padding-left:0;margin-bottom:0;border-radius:var(--bs-list-group-border-radius)}.list-group-numbered{list-style-type:none;counter-reset:section}.list-group-numbered>.list-group-item::before{content:counters(section, ".") ". ";counter-increment:section}.list-group-item-action{width:100%;color:var(--bs-list-group-action-color);text-align:inherit}.list-group-item-action:focus,.list-group-item-action:hover{z-index:1;color:var(--bs-list-group-action-hover-color);text-decoration:none;background-color:var(--bs-list-group-action-hover-bg)}.list-group-item-action:active{color:var(--bs-list-group-action-active-color);background-color:var(--bs-list-group-action-active-bg)}.list-group-item{position:relative;display:block;padding:var(--bs-list-group-item-padding-y) var(--bs-list-group-item-padding-x);color:var(--bs-list-group-color);text-decoration:none;background-color:var(--bs-list-group-bg);border:var(--bs-list-group-border-width) solid var(--bs-list-group-border-color)}.list-group-item:first-child{border-top-left-radius:inherit;border-top-right-radius:inherit}.list-group-item:last-child{border-bottom-right-radius:inherit;border-bottom-left-radius:inherit}.list-group-item.disabled,.list-group-item:disabled{color:var(--bs-list-group-disabled-color);pointer-events:none;background-color:var(--bs-list-group-disabled-bg)}.list-group-item.active{z-index:2;color:var(--bs-list-group-active-color);background-color:var(--bs-list-group-active-bg);border-color:var(--bs-list-group-active-border-color)}.list-group-item+.list-group-item{border-top-width:0}.list-group-item+.list-group-item.active{margin-top:calc(-1 * var(--bs-list-group-border-width));border-top-width:var(--bs-list-group-border-width)}.list-group-horizontal{flex-direction:row}.list-group-horizontal>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal>.list-group-item.active{margin-top:0}.list-group-horizontal>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}@media (min-width:576px){.list-group-horizontal-sm{flex-direction:row}.list-group-horizontal-sm>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal-sm>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal-sm>.list-group-item.active{margin-top:0}.list-group-horizontal-sm>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-sm>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media (min-width:768px){.list-group-horizontal-md{flex-direction:row}.list-group-horizontal-md>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal-md>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal-md>.list-group-item.active{margin-top:0}.list-group-horizontal-md>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-md>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media (min-width:992px){.list-group-horizontal-lg{flex-direction:row}.list-group-horizontal-lg>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal-lg>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal-lg>.list-group-item.active{margin-top:0}.list-group-horizontal-lg>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-lg>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media (min-width:1200px){.list-group-horizontal-xl{flex-direction:row}.list-group-horizontal-xl>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal-xl>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal-xl>.list-group-item.active{margin-top:0}.list-group-horizontal-xl>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-xl>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media (min-width:1400px){.list-group-horizontal-xxl{flex-direction:row}.list-group-horizontal-xxl>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal-xxl>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal-xxl>.list-group-item.active{margin-top:0}.list-group-horizontal-xxl>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-xxl>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}.list-group-flush{border-radius:0}.list-group-flush>.list-group-item{border-width:0 0 var(--bs-list-group-border-width)}.list-group-flush>.list-group-item:last-child{border-bottom-width:0}.list-group-item-primary{--bs-list-group-color:var(--bs-primary-text-emphasis);--bs-list-group-bg:var(--bs-primary-bg-subtle);--bs-list-group-border-color:var(--bs-primary-border-subtle);--bs-list-group-action-hover-color:var(--bs-emphasis-color);--bs-list-group-action-hover-bg:var(--bs-primary-border-subtle);--bs-list-group-action-active-color:var(--bs-emphasis-color);--bs-list-group-action-active-bg:var(--bs-primary-border-subtle);--bs-list-group-active-color:var(--bs-primary-bg-subtle);--bs-list-group-active-bg:var(--bs-primary-text-emphasis);--bs-list-group-active-border-color:var(--bs-primary-text-emphasis)}.list-group-item-secondary{--bs-list-group-color:var(--bs-secondary-text-emphasis);--bs-list-group-bg:var(--bs-secondary-bg-subtle);--bs-list-group-border-color:var(--bs-secondary-border-subtle);--bs-list-group-action-hover-color:var(--bs-emphasis-color);--bs-list-group-action-hover-bg:var(--bs-secondary-border-subtle);--bs-list-group-action-active-color:var(--bs-emphasis-color);--bs-list-group-action-active-bg:var(--bs-secondary-border-subtle);--bs-list-group-active-color:var(--bs-secondary-bg-subtle);--bs-list-group-active-bg:var(--bs-secondary-text-emphasis);--bs-list-group-active-border-color:var(--bs-secondary-text-emphasis)}.list-group-item-success{--bs-list-group-color:var(--bs-success-text-emphasis);--bs-list-group-bg:var(--bs-success-bg-subtle);--bs-list-group-border-color:var(--bs-success-border-subtle);--bs-list-group-action-hover-color:var(--bs-emphasis-color);--bs-list-group-action-hover-bg:var(--bs-success-border-subtle);--bs-list-group-action-active-color:var(--bs-emphasis-color);--bs-list-group-action-active-bg:var(--bs-success-border-subtle);--bs-list-group-active-color:var(--bs-success-bg-subtle);--bs-list-group-active-bg:var(--bs-success-text-emphasis);--bs-list-group-active-border-color:var(--bs-success-text-emphasis)}.list-group-item-info{--bs-list-group-color:var(--bs-info-text-emphasis);--bs-list-group-bg:var(--bs-info-bg-subtle);--bs-list-group-border-color:var(--bs-info-border-subtle);--bs-list-group-action-hover-color:var(--bs-emphasis-color);--bs-list-group-action-hover-bg:var(--bs-info-border-subtle);--bs-list-group-action-active-color:var(--bs-emphasis-color);--bs-list-group-action-active-bg:var(--bs-info-border-subtle);--bs-list-group-active-color:var(--bs-info-bg-subtle);--bs-list-group-active-bg:var(--bs-info-text-emphasis);--bs-list-group-active-border-color:var(--bs-info-text-emphasis)}.list-group-item-warning{--bs-list-group-color:var(--bs-warning-text-emphasis);--bs-list-group-bg:var(--bs-warning-bg-subtle);--bs-list-group-border-color:var(--bs-warning-border-subtle);--bs-list-group-action-hover-color:var(--bs-emphasis-color);--bs-list-group-action-hover-bg:var(--bs-warning-border-subtle);--bs-list-group-action-active-color:var(--bs-emphasis-color);--bs-list-group-action-active-bg:var(--bs-warning-border-subtle);--bs-list-group-active-color:var(--bs-warning-bg-subtle);--bs-list-group-active-bg:var(--bs-warning-text-emphasis);--bs-list-group-active-border-color:var(--bs-warning-text-emphasis)}.list-group-item-danger{--bs-list-group-color:var(--bs-danger-text-emphasis);--bs-list-group-bg:var(--bs-danger-bg-subtle);--bs-list-group-border-color:var(--bs-danger-border-subtle);--bs-list-group-action-hover-color:var(--bs-emphasis-color);--bs-list-group-action-hover-bg:var(--bs-danger-border-subtle);--bs-list-group-action-active-color:var(--bs-emphasis-color);--bs-list-group-action-active-bg:var(--bs-danger-border-subtle);--bs-list-group-active-color:var(--bs-danger-bg-subtle);--bs-list-group-active-bg:var(--bs-danger-text-emphasis);--bs-list-group-active-border-color:var(--bs-danger-text-emphasis)}.list-group-item-light{--bs-list-group-color:var(--bs-light-text-emphasis);--bs-list-group-bg:var(--bs-light-bg-subtle);--bs-list-group-border-color:var(--bs-light-border-subtle);--bs-list-group-action-hover-color:var(--bs-emphasis-color);--bs-list-group-action-hover-bg:var(--bs-light-border-subtle);--bs-list-group-action-active-color:var(--bs-emphasis-color);--bs-list-group-action-active-bg:var(--bs-light-border-subtle);--bs-list-group-active-color:var(--bs-light-bg-subtle);--bs-list-group-active-bg:var(--bs-light-text-emphasis);--bs-list-group-active-border-color:var(--bs-light-text-emphasis)}.list-group-item-dark{--bs-list-group-color:var(--bs-dark-text-emphasis);--bs-list-group-bg:var(--bs-dark-bg-subtle);--bs-list-group-border-color:var(--bs-dark-border-subtle);--bs-list-group-action-hover-color:var(--bs-emphasis-color);--bs-list-group-action-hover-bg:var(--bs-dark-border-subtle);--bs-list-group-action-active-color:var(--bs-emphasis-color);--bs-list-group-action-active-bg:var(--bs-dark-border-subtle);--bs-list-group-active-color:var(--bs-dark-bg-subtle);--bs-list-group-active-bg:var(--bs-dark-text-emphasis);--bs-list-group-active-border-color:var(--bs-dark-text-emphasis)}.btn-close{--bs-btn-close-color:#000;--bs-btn-close-bg:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23000'%3e%3cpath d='M.293.293a1 1 0 0 1 1.414 0L8 6.586 14.293.293a1 1 0 1 1 1.414 1.414L9.414 8l6.293 6.293a1 1 0 0 1-1.414 1.414L8 9.414l-6.293 6.293a1 1 0 0 1-1.414-1.414L6.586 8 .293 1.707a1 1 0 0 1 0-1.414z'/%3e%3c/svg%3e");--bs-btn-close-opacity:0.5;--bs-btn-close-hover-opacity:0.75;--bs-btn-close-focus-shadow:0 0 0 0.25rem rgba(13, 110, 253, 0.25);--bs-btn-close-focus-opacity:1;--bs-btn-close-disabled-opacity:0.25;--bs-btn-close-white-filter:invert(1) grayscale(100%) brightness(200%);box-sizing:content-box;width:1em;height:1em;padding:.25em .25em;color:var(--bs-btn-close-color);background:transparent var(--bs-btn-close-bg) center/1em auto no-repeat;border:0;border-radius:.375rem;opacity:var(--bs-btn-close-opacity)}.btn-close:hover{color:var(--bs-btn-close-color);text-decoration:none;opacity:var(--bs-btn-close-hover-opacity)}.btn-close:focus{outline:0;box-shadow:var(--bs-btn-close-focus-shadow);opacity:var(--bs-btn-close-focus-opacity)}.btn-close.disabled,.btn-close:disabled{pointer-events:none;-webkit-user-select:none;-moz-user-select:none;user-select:none;opacity:var(--bs-btn-close-disabled-opacity)}.btn-close-white{filter:var(--bs-btn-close-white-filter)}[data-bs-theme=dark] .btn-close{filter:var(--bs-btn-close-white-filter)}.toast{--bs-toast-zindex:1090;--bs-toast-padding-x:0.75rem;--bs-toast-padding-y:0.5rem;--bs-toast-spacing:1.5rem;--bs-toast-max-width:350px;--bs-toast-font-size:0.875rem;--bs-toast-color: ;--bs-toast-bg:rgba(var(--bs-body-bg-rgb), 0.85);--bs-toast-border-width:var(--bs-border-width);--bs-toast-border-color:var(--bs-border-color-translucent);--bs-toast-border-radius:var(--bs-border-radius);--bs-toast-box-shadow:var(--bs-box-shadow);--bs-toast-header-color:var(--bs-secondary-color);--bs-toast-header-bg:rgba(var(--bs-body-bg-rgb), 0.85);--bs-toast-header-border-color:var(--bs-border-color-translucent);width:var(--bs-toast-max-width);max-width:100%;font-size:var(--bs-toast-font-size);color:var(--bs-toast-color);pointer-events:auto;background-color:var(--bs-toast-bg);background-clip:padding-box;border:var(--bs-toast-border-width) solid var(--bs-toast-border-color);box-shadow:var(--bs-toast-box-shadow);border-radius:var(--bs-toast-border-radius)}.toast.showing{opacity:0}.toast:not(.show){display:none}.toast-container{--bs-toast-zindex:1090;position:absolute;z-index:var(--bs-toast-zindex);width:-webkit-max-content;width:-moz-max-content;width:max-content;max-width:100%;pointer-events:none}.toast-container>:not(:last-child){margin-bottom:var(--bs-toast-spacing)}.toast-header{display:flex;align-items:center;padding:var(--bs-toast-padding-y) var(--bs-toast-padding-x);color:var(--bs-toast-header-color);background-color:var(--bs-toast-header-bg);background-clip:padding-box;border-bottom:var(--bs-toast-border-width) solid var(--bs-toast-header-border-color);border-top-left-radius:calc(var(--bs-toast-border-radius) - var(--bs-toast-border-width));border-top-right-radius:calc(var(--bs-toast-border-radius) - var(--bs-toast-border-width))}.toast-header .btn-close{margin-right:calc(-.5 * var(--bs-toast-padding-x));margin-left:var(--bs-toast-padding-x)}.toast-body{padding:var(--bs-toast-padding-x);word-wrap:break-word}.modal{--bs-modal-zindex:1055;--bs-modal-width:500px;--bs-modal-padding:1rem;--bs-modal-margin:0.5rem;--bs-modal-color: ;--bs-modal-bg:var(--bs-body-bg);--bs-modal-border-color:var(--bs-border-color-translucent);--bs-modal-border-width:var(--bs-border-width);--bs-modal-border-radius:var(--bs-border-radius-lg);--bs-modal-box-shadow:var(--bs-box-shadow-sm);--bs-modal-inner-border-radius:calc(var(--bs-border-radius-lg) - (var(--bs-border-width)));--bs-modal-header-padding-x:1rem;--bs-modal-header-padding-y:1rem;--bs-modal-header-padding:1rem 1rem;--bs-modal-header-border-color:var(--bs-border-color);--bs-modal-header-border-width:var(--bs-border-width);--bs-modal-title-line-height:1.5;--bs-modal-footer-gap:0.5rem;--bs-modal-footer-bg: ;--bs-modal-footer-border-color:var(--bs-border-color);--bs-modal-footer-border-width:var(--bs-border-width);position:fixed;top:0;left:0;z-index:var(--bs-modal-zindex);display:none;width:100%;height:100%;overflow-x:hidden;overflow-y:auto;outline:0}.modal-dialog{position:relative;width:auto;margin:var(--bs-modal-margin);pointer-events:none}.modal.fade .modal-dialog{transition:transform .3s ease-out;transform:translate(0,-50px)}@media (prefers-reduced-motion:reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{transform:none}.modal.modal-static .modal-dialog{transform:scale(1.02)}.modal-dialog-scrollable{height:calc(100% - var(--bs-modal-margin) * 2)}.modal-dialog-scrollable .modal-content{max-height:100%;overflow:hidden}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:flex;align-items:center;min-height:calc(100% - var(--bs-modal-margin) * 2)}.modal-content{position:relative;display:flex;flex-direction:column;width:100%;color:var(--bs-modal-color);pointer-events:auto;background-color:var(--bs-modal-bg);background-clip:padding-box;border:var(--bs-modal-border-width) solid var(--bs-modal-border-color);border-radius:var(--bs-modal-border-radius);outline:0}.modal-backdrop{--bs-backdrop-zindex:1050;--bs-backdrop-bg:#000;--bs-backdrop-opacity:0.5;position:fixed;top:0;left:0;z-index:var(--bs-backdrop-zindex);width:100vw;height:100vh;background-color:var(--bs-backdrop-bg)}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:var(--bs-backdrop-opacity)}.modal-header{display:flex;flex-shrink:0;align-items:center;justify-content:space-between;padding:var(--bs-modal-header-padding);border-bottom:var(--bs-modal-header-border-width) solid var(--bs-modal-header-border-color);border-top-left-radius:var(--bs-modal-inner-border-radius);border-top-right-radius:var(--bs-modal-inner-border-radius)}.modal-header .btn-close{padding:calc(var(--bs-modal-header-padding-y) * .5) calc(var(--bs-modal-header-padding-x) * .5);margin:calc(-.5 * var(--bs-modal-header-padding-y)) calc(-.5 * var(--bs-modal-header-padding-x)) calc(-.5 * var(--bs-modal-header-padding-y)) auto}.modal-title{margin-bottom:0;line-height:var(--bs-modal-title-line-height)}.modal-body{position:relative;flex:1 1 auto;padding:var(--bs-modal-padding)}.modal-footer{display:flex;flex-shrink:0;flex-wrap:wrap;align-items:center;justify-content:flex-end;padding:calc(var(--bs-modal-padding) - var(--bs-modal-footer-gap) * .5);background-color:var(--bs-modal-footer-bg);border-top:var(--bs-modal-footer-border-width) solid var(--bs-modal-footer-border-color);border-bottom-right-radius:var(--bs-modal-inner-border-radius);border-bottom-left-radius:var(--bs-modal-inner-border-radius)}.modal-footer>*{margin:calc(var(--bs-modal-footer-gap) * .5)}@media (min-width:576px){.modal{--bs-modal-margin:1.75rem;--bs-modal-box-shadow:var(--bs-box-shadow)}.modal-dialog{max-width:var(--bs-modal-width);margin-right:auto;margin-left:auto}.modal-sm{--bs-modal-width:300px}}@media (min-width:992px){.modal-lg,.modal-xl{--bs-modal-width:800px}}@media (min-width:1200px){.modal-xl{--bs-modal-width:1140px}}.modal-fullscreen{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen .modal-footer,.modal-fullscreen .modal-header{border-radius:0}.modal-fullscreen .modal-body{overflow-y:auto}@media (max-width:575.98px){.modal-fullscreen-sm-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-sm-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-sm-down .modal-footer,.modal-fullscreen-sm-down .modal-header{border-radius:0}.modal-fullscreen-sm-down .modal-body{overflow-y:auto}}@media (max-width:767.98px){.modal-fullscreen-md-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-md-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-md-down .modal-footer,.modal-fullscreen-md-down .modal-header{border-radius:0}.modal-fullscreen-md-down .modal-body{overflow-y:auto}}@media (max-width:991.98px){.modal-fullscreen-lg-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-lg-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-lg-down .modal-footer,.modal-fullscreen-lg-down .modal-header{border-radius:0}.modal-fullscreen-lg-down .modal-body{overflow-y:auto}}@media (max-width:1199.98px){.modal-fullscreen-xl-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-xl-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-xl-down .modal-footer,.modal-fullscreen-xl-down .modal-header{border-radius:0}.modal-fullscreen-xl-down .modal-body{overflow-y:auto}}@media (max-width:1399.98px){.modal-fullscreen-xxl-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-xxl-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-xxl-down .modal-footer,.modal-fullscreen-xxl-down .modal-header{border-radius:0}.modal-fullscreen-xxl-down .modal-body{overflow-y:auto}}.tooltip{--bs-tooltip-zindex:1080;--bs-tooltip-max-width:200px;--bs-tooltip-padding-x:0.5rem;--bs-tooltip-padding-y:0.25rem;--bs-tooltip-margin: ;--bs-tooltip-font-size:0.875rem;--bs-tooltip-color:var(--bs-body-bg);--bs-tooltip-bg:var(--bs-emphasis-color);--bs-tooltip-border-radius:var(--bs-border-radius);--bs-tooltip-opacity:0.9;--bs-tooltip-arrow-width:0.8rem;--bs-tooltip-arrow-height:0.4rem;z-index:var(--bs-tooltip-zindex);display:block;margin:var(--bs-tooltip-margin);font-family:var(--bs-font-sans-serif);font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;white-space:normal;word-spacing:normal;line-break:auto;font-size:var(--bs-tooltip-font-size);word-wrap:break-word;opacity:0}.tooltip.show{opacity:var(--bs-tooltip-opacity)}.tooltip .tooltip-arrow{display:block;width:var(--bs-tooltip-arrow-width);height:var(--bs-tooltip-arrow-height)}.tooltip .tooltip-arrow::before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-auto[data-popper-placement^=top] .tooltip-arrow,.bs-tooltip-top .tooltip-arrow{bottom:calc(-1 * var(--bs-tooltip-arrow-height))}.bs-tooltip-auto[data-popper-placement^=top] .tooltip-arrow::before,.bs-tooltip-top .tooltip-arrow::before{top:-1px;border-width:var(--bs-tooltip-arrow-height) calc(var(--bs-tooltip-arrow-width) * .5) 0;border-top-color:var(--bs-tooltip-bg)}.bs-tooltip-auto[data-popper-placement^=right] .tooltip-arrow,.bs-tooltip-end .tooltip-arrow{left:calc(-1 * var(--bs-tooltip-arrow-height));width:var(--bs-tooltip-arrow-height);height:var(--bs-tooltip-arrow-width)}.bs-tooltip-auto[data-popper-placement^=right] .tooltip-arrow::before,.bs-tooltip-end .tooltip-arrow::before{right:-1px;border-width:calc(var(--bs-tooltip-arrow-width) * .5) var(--bs-tooltip-arrow-height) calc(var(--bs-tooltip-arrow-width) * .5) 0;border-right-color:var(--bs-tooltip-bg)}.bs-tooltip-auto[data-popper-placement^=bottom] .tooltip-arrow,.bs-tooltip-bottom .tooltip-arrow{top:calc(-1 * var(--bs-tooltip-arrow-height))}.bs-tooltip-auto[data-popper-placement^=bottom] .tooltip-arrow::before,.bs-tooltip-bottom .tooltip-arrow::before{bottom:-1px;border-width:0 calc(var(--bs-tooltip-arrow-width) * .5) var(--bs-tooltip-arrow-height);border-bottom-color:var(--bs-tooltip-bg)}.bs-tooltip-auto[data-popper-placement^=left] .tooltip-arrow,.bs-tooltip-start .tooltip-arrow{right:calc(-1 * var(--bs-tooltip-arrow-height));width:var(--bs-tooltip-arrow-height);height:var(--bs-tooltip-arrow-width)}.bs-tooltip-auto[data-popper-placement^=left] .tooltip-arrow::before,.bs-tooltip-start .tooltip-arrow::before{left:-1px;border-width:calc(var(--bs-tooltip-arrow-width) * .5) 0 calc(var(--bs-tooltip-arrow-width) * .5) var(--bs-tooltip-arrow-height);border-left-color:var(--bs-tooltip-bg)}.tooltip-inner{max-width:var(--bs-tooltip-max-width);padding:var(--bs-tooltip-padding-y) var(--bs-tooltip-padding-x);color:var(--bs-tooltip-color);text-align:center;background-color:var(--bs-tooltip-bg);border-radius:var(--bs-tooltip-border-radius)}.popover{--bs-popover-zindex:1070;--bs-popover-max-width:276px;--bs-popover-font-size:0.875rem;--bs-popover-bg:var(--bs-body-bg);--bs-popover-border-width:var(--bs-border-width);--bs-popover-border-color:var(--bs-border-color-translucent);--bs-popover-border-radius:var(--bs-border-radius-lg);--bs-popover-inner-border-radius:calc(var(--bs-border-radius-lg) - var(--bs-border-width));--bs-popover-box-shadow:var(--bs-box-shadow);--bs-popover-header-padding-x:1rem;--bs-popover-header-padding-y:0.5rem;--bs-popover-header-font-size:1rem;--bs-popover-header-color:inherit;--bs-popover-header-bg:var(--bs-secondary-bg);--bs-popover-body-padding-x:1rem;--bs-popover-body-padding-y:1rem;--bs-popover-body-color:var(--bs-body-color);--bs-popover-arrow-width:1rem;--bs-popover-arrow-height:0.5rem;--bs-popover-arrow-border:var(--bs-popover-border-color);z-index:var(--bs-popover-zindex);display:block;max-width:var(--bs-popover-max-width);font-family:var(--bs-font-sans-serif);font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;white-space:normal;word-spacing:normal;line-break:auto;font-size:var(--bs-popover-font-size);word-wrap:break-word;background-color:var(--bs-popover-bg);background-clip:padding-box;border:var(--bs-popover-border-width) solid var(--bs-popover-border-color);border-radius:var(--bs-popover-border-radius)}.popover .popover-arrow{display:block;width:var(--bs-popover-arrow-width);height:var(--bs-popover-arrow-height)}.popover .popover-arrow::after,.popover .popover-arrow::before{position:absolute;display:block;content:"";border-color:transparent;border-style:solid;border-width:0}.bs-popover-auto[data-popper-placement^=top]>.popover-arrow,.bs-popover-top>.popover-arrow{bottom:calc(-1 * (var(--bs-popover-arrow-height)) - var(--bs-popover-border-width))}.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::before,.bs-popover-top>.popover-arrow::after,.bs-popover-top>.popover-arrow::before{border-width:var(--bs-popover-arrow-height) calc(var(--bs-popover-arrow-width) * .5) 0}.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::before,.bs-popover-top>.popover-arrow::before{bottom:0;border-top-color:var(--bs-popover-arrow-border)}.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::after,.bs-popover-top>.popover-arrow::after{bottom:var(--bs-popover-border-width);border-top-color:var(--bs-popover-bg)}.bs-popover-auto[data-popper-placement^=right]>.popover-arrow,.bs-popover-end>.popover-arrow{left:calc(-1 * (var(--bs-popover-arrow-height)) - var(--bs-popover-border-width));width:var(--bs-popover-arrow-height);height:var(--bs-popover-arrow-width)}.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::before,.bs-popover-end>.popover-arrow::after,.bs-popover-end>.popover-arrow::before{border-width:calc(var(--bs-popover-arrow-width) * .5) var(--bs-popover-arrow-height) calc(var(--bs-popover-arrow-width) * .5) 0}.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::before,.bs-popover-end>.popover-arrow::before{left:0;border-right-color:var(--bs-popover-arrow-border)}.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::after,.bs-popover-end>.popover-arrow::after{left:var(--bs-popover-border-width);border-right-color:var(--bs-popover-bg)}.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow,.bs-popover-bottom>.popover-arrow{top:calc(-1 * (var(--bs-popover-arrow-height)) - var(--bs-popover-border-width))}.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::before,.bs-popover-bottom>.popover-arrow::after,.bs-popover-bottom>.popover-arrow::before{border-width:0 calc(var(--bs-popover-arrow-width) * .5) var(--bs-popover-arrow-height)}.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::before,.bs-popover-bottom>.popover-arrow::before{top:0;border-bottom-color:var(--bs-popover-arrow-border)}.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::after,.bs-popover-bottom>.popover-arrow::after{top:var(--bs-popover-border-width);border-bottom-color:var(--bs-popover-bg)}.bs-popover-auto[data-popper-placement^=bottom] .popover-header::before,.bs-popover-bottom .popover-header::before{position:absolute;top:0;left:50%;display:block;width:var(--bs-popover-arrow-width);margin-left:calc(-.5 * var(--bs-popover-arrow-width));content:"";border-bottom:var(--bs-popover-border-width) solid var(--bs-popover-header-bg)}.bs-popover-auto[data-popper-placement^=left]>.popover-arrow,.bs-popover-start>.popover-arrow{right:calc(-1 * (var(--bs-popover-arrow-height)) - var(--bs-popover-border-width));width:var(--bs-popover-arrow-height);height:var(--bs-popover-arrow-width)}.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::before,.bs-popover-start>.popover-arrow::after,.bs-popover-start>.popover-arrow::before{border-width:calc(var(--bs-popover-arrow-width) * .5) 0 calc(var(--bs-popover-arrow-width) * .5) var(--bs-popover-arrow-height)}.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::before,.bs-popover-start>.popover-arrow::before{right:0;border-left-color:var(--bs-popover-arrow-border)}.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::after,.bs-popover-start>.popover-arrow::after{right:var(--bs-popover-border-width);border-left-color:var(--bs-popover-bg)}.popover-header{padding:var(--bs-popover-header-padding-y) var(--bs-popover-header-padding-x);margin-bottom:0;font-size:var(--bs-popover-header-font-size);color:var(--bs-popover-header-color);background-color:var(--bs-popover-header-bg);border-bottom:var(--bs-popover-border-width) solid var(--bs-popover-border-color);border-top-left-radius:var(--bs-popover-inner-border-radius);border-top-right-radius:var(--bs-popover-inner-border-radius)}.popover-header:empty{display:none}.popover-body{padding:var(--bs-popover-body-padding-y) var(--bs-popover-body-padding-x);color:var(--bs-popover-body-color)}.carousel{position:relative}.carousel.pointer-event{touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner::after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;-webkit-backface-visibility:hidden;backface-visibility:hidden;transition:transform .6s ease-in-out}@media (prefers-reduced-motion:reduce){.carousel-item{transition:none}}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:block}.active.carousel-item-end,.carousel-item-next:not(.carousel-item-start){transform:translateX(100%)}.active.carousel-item-start,.carousel-item-prev:not(.carousel-item-end){transform:translateX(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;transform:none}.carousel-fade .carousel-item-next.carousel-item-start,.carousel-fade .carousel-item-prev.carousel-item-end,.carousel-fade .carousel-item.active{z-index:1;opacity:1}.carousel-fade .active.carousel-item-end,.carousel-fade .active.carousel-item-start{z-index:0;opacity:0;transition:opacity 0s .6s}@media (prefers-reduced-motion:reduce){.carousel-fade .active.carousel-item-end,.carousel-fade .active.carousel-item-start{transition:none}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;z-index:1;display:flex;align-items:center;justify-content:center;width:15%;padding:0;color:#fff;text-align:center;background:0 0;border:0;opacity:.5;transition:opacity .15s ease}@media (prefers-reduced-motion:reduce){.carousel-control-next,.carousel-control-prev{transition:none}}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:2rem;height:2rem;background-repeat:no-repeat;background-position:50%;background-size:100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23fff'%3e%3cpath d='M11.354 1.646a.5.5 0 0 1 0 .708L5.707 8l5.647 5.646a.5.5 0 0 1-.708.708l-6-6a.5.5 0 0 1 0-.708l6-6a.5.5 0 0 1 .708 0z'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23fff'%3e%3cpath d='M4.646 1.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1 0 .708l-6 6a.5.5 0 0 1-.708-.708L10.293 8 4.646 2.354a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:2;display:flex;justify-content:center;padding:0;margin-right:15%;margin-bottom:1rem;margin-left:15%}.carousel-indicators [data-bs-target]{box-sizing:content-box;flex:0 1 auto;width:30px;height:3px;padding:0;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:#fff;background-clip:padding-box;border:0;border-top:10px solid transparent;border-bottom:10px solid transparent;opacity:.5;transition:opacity .6s ease}@media (prefers-reduced-motion:reduce){.carousel-indicators [data-bs-target]{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:1.25rem;left:15%;padding-top:1.25rem;padding-bottom:1.25rem;color:#fff;text-align:center}.carousel-dark .carousel-control-next-icon,.carousel-dark .carousel-control-prev-icon{filter:invert(1) grayscale(100)}.carousel-dark .carousel-indicators [data-bs-target]{background-color:#000}.carousel-dark .carousel-caption{color:#000}[data-bs-theme=dark] .carousel .carousel-control-next-icon,[data-bs-theme=dark] .carousel .carousel-control-prev-icon,[data-bs-theme=dark].carousel .carousel-control-next-icon,[data-bs-theme=dark].carousel .carousel-control-prev-icon{filter:invert(1) grayscale(100)}[data-bs-theme=dark] .carousel .carousel-indicators [data-bs-target],[data-bs-theme=dark].carousel .carousel-indicators [data-bs-target]{background-color:#000}[data-bs-theme=dark] .carousel .carousel-caption,[data-bs-theme=dark].carousel .carousel-caption{color:#000}.spinner-border,.spinner-grow{display:inline-block;width:var(--bs-spinner-width);height:var(--bs-spinner-height);vertical-align:var(--bs-spinner-vertical-align);border-radius:50%;animation:var(--bs-spinner-animation-speed) linear infinite var(--bs-spinner-animation-name)}@keyframes spinner-border{to{transform:rotate(360deg)}}.spinner-border{--bs-spinner-width:2rem;--bs-spinner-height:2rem;--bs-spinner-vertical-align:-0.125em;--bs-spinner-border-width:0.25em;--bs-spinner-animation-speed:0.75s;--bs-spinner-animation-name:spinner-border;border:var(--bs-spinner-border-width) solid currentcolor;border-right-color:transparent}.spinner-border-sm{--bs-spinner-width:1rem;--bs-spinner-height:1rem;--bs-spinner-border-width:0.2em}@keyframes spinner-grow{0%{transform:scale(0)}50%{opacity:1;transform:none}}.spinner-grow{--bs-spinner-width:2rem;--bs-spinner-height:2rem;--bs-spinner-vertical-align:-0.125em;--bs-spinner-animation-speed:0.75s;--bs-spinner-animation-name:spinner-grow;background-color:currentcolor;opacity:0}.spinner-grow-sm{--bs-spinner-width:1rem;--bs-spinner-height:1rem}@media (prefers-reduced-motion:reduce){.spinner-border,.spinner-grow{--bs-spinner-animation-speed:1.5s}}.offcanvas,.offcanvas-lg,.offcanvas-md,.offcanvas-sm,.offcanvas-xl,.offcanvas-xxl{--bs-offcanvas-zindex:1045;--bs-offcanvas-width:400px;--bs-offcanvas-height:30vh;--bs-offcanvas-padding-x:1rem;--bs-offcanvas-padding-y:1rem;--bs-offcanvas-color:var(--bs-body-color);--bs-offcanvas-bg:var(--bs-body-bg);--bs-offcanvas-border-width:var(--bs-border-width);--bs-offcanvas-border-color:var(--bs-border-color-translucent);--bs-offcanvas-box-shadow:var(--bs-box-shadow-sm);--bs-offcanvas-transition:transform 0.3s ease-in-out;--bs-offcanvas-title-line-height:1.5}@media (max-width:575.98px){.offcanvas-sm{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media (max-width:575.98px) and (prefers-reduced-motion:reduce){.offcanvas-sm{transition:none}}@media (max-width:575.98px){.offcanvas-sm.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas-sm.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas-sm.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-sm.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-sm.show:not(.hiding),.offcanvas-sm.showing{transform:none}.offcanvas-sm.hiding,.offcanvas-sm.show,.offcanvas-sm.showing{visibility:visible}}@media (min-width:576px){.offcanvas-sm{--bs-offcanvas-height:auto;--bs-offcanvas-border-width:0;background-color:transparent!important}.offcanvas-sm .offcanvas-header{display:none}.offcanvas-sm .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible;background-color:transparent!important}}@media (max-width:767.98px){.offcanvas-md{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media (max-width:767.98px) and (prefers-reduced-motion:reduce){.offcanvas-md{transition:none}}@media (max-width:767.98px){.offcanvas-md.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas-md.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas-md.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-md.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-md.show:not(.hiding),.offcanvas-md.showing{transform:none}.offcanvas-md.hiding,.offcanvas-md.show,.offcanvas-md.showing{visibility:visible}}@media (min-width:768px){.offcanvas-md{--bs-offcanvas-height:auto;--bs-offcanvas-border-width:0;background-color:transparent!important}.offcanvas-md .offcanvas-header{display:none}.offcanvas-md .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible;background-color:transparent!important}}@media (max-width:991.98px){.offcanvas-lg{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media (max-width:991.98px) and (prefers-reduced-motion:reduce){.offcanvas-lg{transition:none}}@media (max-width:991.98px){.offcanvas-lg.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas-lg.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas-lg.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-lg.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-lg.show:not(.hiding),.offcanvas-lg.showing{transform:none}.offcanvas-lg.hiding,.offcanvas-lg.show,.offcanvas-lg.showing{visibility:visible}}@media (min-width:992px){.offcanvas-lg{--bs-offcanvas-height:auto;--bs-offcanvas-border-width:0;background-color:transparent!important}.offcanvas-lg .offcanvas-header{display:none}.offcanvas-lg .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible;background-color:transparent!important}}@media (max-width:1199.98px){.offcanvas-xl{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media (max-width:1199.98px) and (prefers-reduced-motion:reduce){.offcanvas-xl{transition:none}}@media (max-width:1199.98px){.offcanvas-xl.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas-xl.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas-xl.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-xl.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-xl.show:not(.hiding),.offcanvas-xl.showing{transform:none}.offcanvas-xl.hiding,.offcanvas-xl.show,.offcanvas-xl.showing{visibility:visible}}@media (min-width:1200px){.offcanvas-xl{--bs-offcanvas-height:auto;--bs-offcanvas-border-width:0;background-color:transparent!important}.offcanvas-xl .offcanvas-header{display:none}.offcanvas-xl .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible;background-color:transparent!important}}@media (max-width:1399.98px){.offcanvas-xxl{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media (max-width:1399.98px) and (prefers-reduced-motion:reduce){.offcanvas-xxl{transition:none}}@media (max-width:1399.98px){.offcanvas-xxl.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas-xxl.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas-xxl.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-xxl.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-xxl.show:not(.hiding),.offcanvas-xxl.showing{transform:none}.offcanvas-xxl.hiding,.offcanvas-xxl.show,.offcanvas-xxl.showing{visibility:visible}}@media (min-width:1400px){.offcanvas-xxl{--bs-offcanvas-height:auto;--bs-offcanvas-border-width:0;background-color:transparent!important}.offcanvas-xxl .offcanvas-header{display:none}.offcanvas-xxl .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible;background-color:transparent!important}}.offcanvas{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}@media (prefers-reduced-motion:reduce){.offcanvas{transition:none}}.offcanvas.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas.show:not(.hiding),.offcanvas.showing{transform:none}.offcanvas.hiding,.offcanvas.show,.offcanvas.showing{visibility:visible}.offcanvas-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.offcanvas-backdrop.fade{opacity:0}.offcanvas-backdrop.show{opacity:.5}.offcanvas-header{display:flex;align-items:center;justify-content:space-between;padding:var(--bs-offcanvas-padding-y) var(--bs-offcanvas-padding-x)}.offcanvas-header .btn-close{padding:calc(var(--bs-offcanvas-padding-y) * .5) calc(var(--bs-offcanvas-padding-x) * .5);margin-top:calc(-.5 * var(--bs-offcanvas-padding-y));margin-right:calc(-.5 * var(--bs-offcanvas-padding-x));margin-bottom:calc(-.5 * var(--bs-offcanvas-padding-y))}.offcanvas-title{margin-bottom:0;line-height:var(--bs-offcanvas-title-line-height)}.offcanvas-body{flex-grow:1;padding:var(--bs-offcanvas-padding-y) var(--bs-offcanvas-padding-x);overflow-y:auto}.placeholder{display:inline-block;min-height:1em;vertical-align:middle;cursor:wait;background-color:currentcolor;opacity:.5}.placeholder.btn::before{display:inline-block;content:""}.placeholder-xs{min-height:.6em}.placeholder-sm{min-height:.8em}.placeholder-lg{min-height:1.2em}.placeholder-glow .placeholder{animation:placeholder-glow 2s ease-in-out infinite}@keyframes placeholder-glow{50%{opacity:.2}}.placeholder-wave{-webkit-mask-image:linear-gradient(130deg,#000 55%,rgba(0,0,0,0.8) 75%,#000 95%);mask-image:linear-gradient(130deg,#000 55%,rgba(0,0,0,0.8) 75%,#000 95%);-webkit-mask-size:200% 100%;mask-size:200% 100%;animation:placeholder-wave 2s linear infinite}@keyframes placeholder-wave{100%{-webkit-mask-position:-200% 0%;mask-position:-200% 0%}}.clearfix::after{display:block;clear:both;content:""}.text-bg-primary{color:#fff!important;background-color:RGBA(var(--bs-primary-rgb),var(--bs-bg-opacity,1))!important}.text-bg-secondary{color:#fff!important;background-color:RGBA(var(--bs-secondary-rgb),var(--bs-bg-opacity,1))!important}.text-bg-success{color:#fff!important;background-color:RGBA(var(--bs-success-rgb),var(--bs-bg-opacity,1))!important}.text-bg-info{color:#000!important;background-color:RGBA(var(--bs-info-rgb),var(--bs-bg-opacity,1))!important}.text-bg-warning{color:#000!important;background-color:RGBA(var(--bs-warning-rgb),var(--bs-bg-opacity,1))!important}.text-bg-danger{color:#fff!important;background-color:RGBA(var(--bs-danger-rgb),var(--bs-bg-opacity,1))!important}.text-bg-light{color:#000!important;background-color:RGBA(var(--bs-light-rgb),var(--bs-bg-opacity,1))!important}.text-bg-dark{color:#fff!important;background-color:RGBA(var(--bs-dark-rgb),var(--bs-bg-opacity,1))!important}.link-primary{color:RGBA(var(--bs-primary-rgb),var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(var(--bs-primary-rgb),var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(var(--bs-primary-rgb),var(--bs-link-underline-opacity,1))!important}.link-primary:focus,.link-primary:hover{color:RGBA(10,88,202,var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(10,88,202,var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(10,88,202,var(--bs-link-underline-opacity,1))!important}.link-secondary{color:RGBA(var(--bs-secondary-rgb),var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(var(--bs-secondary-rgb),var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(var(--bs-secondary-rgb),var(--bs-link-underline-opacity,1))!important}.link-secondary:focus,.link-secondary:hover{color:RGBA(86,94,100,var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(86,94,100,var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(86,94,100,var(--bs-link-underline-opacity,1))!important}.link-success{color:RGBA(var(--bs-success-rgb),var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(var(--bs-success-rgb),var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(var(--bs-success-rgb),var(--bs-link-underline-opacity,1))!important}.link-success:focus,.link-success:hover{color:RGBA(20,108,67,var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(20,108,67,var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(20,108,67,var(--bs-link-underline-opacity,1))!important}.link-info{color:RGBA(var(--bs-info-rgb),var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(var(--bs-info-rgb),var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(var(--bs-info-rgb),var(--bs-link-underline-opacity,1))!important}.link-info:focus,.link-info:hover{color:RGBA(61,213,243,var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(61,213,243,var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(61,213,243,var(--bs-link-underline-opacity,1))!important}.link-warning{color:RGBA(var(--bs-warning-rgb),var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(var(--bs-warning-rgb),var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(var(--bs-warning-rgb),var(--bs-link-underline-opacity,1))!important}.link-warning:focus,.link-warning:hover{color:RGBA(255,205,57,var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(255,205,57,var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(255,205,57,var(--bs-link-underline-opacity,1))!important}.link-danger{color:RGBA(var(--bs-danger-rgb),var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(var(--bs-danger-rgb),var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(var(--bs-danger-rgb),var(--bs-link-underline-opacity,1))!important}.link-danger:focus,.link-danger:hover{color:RGBA(176,42,55,var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(176,42,55,var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(176,42,55,var(--bs-link-underline-opacity,1))!important}.link-light{color:RGBA(var(--bs-light-rgb),var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(var(--bs-light-rgb),var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(var(--bs-light-rgb),var(--bs-link-underline-opacity,1))!important}.link-light:focus,.link-light:hover{color:RGBA(249,250,251,var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(249,250,251,var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(249,250,251,var(--bs-link-underline-opacity,1))!important}.link-dark{color:RGBA(var(--bs-dark-rgb),var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(var(--bs-dark-rgb),var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(var(--bs-dark-rgb),var(--bs-link-underline-opacity,1))!important}.link-dark:focus,.link-dark:hover{color:RGBA(26,30,33,var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(26,30,33,var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(26,30,33,var(--bs-link-underline-opacity,1))!important}.link-body-emphasis{color:RGBA(var(--bs-emphasis-color-rgb),var(--bs-link-opacity,1))!important;-webkit-text-decoration-color:RGBA(var(--bs-emphasis-color-rgb),var(--bs-link-underline-opacity,1))!important;text-decoration-color:RGBA(var(--bs-emphasis-color-rgb),var(--bs-link-underline-opacity,1))!important}.link-body-emphasis:focus,.link-body-emphasis:hover{color:RGBA(var(--bs-emphasis-color-rgb),var(--bs-link-opacity,.75))!important;-webkit-text-decoration-color:RGBA(var(--bs-emphasis-color-rgb),var(--bs-link-underline-opacity,0.75))!important;text-decoration-color:RGBA(var(--bs-emphasis-color-rgb),var(--bs-link-underline-opacity,0.75))!important}.focus-ring:focus{outline:0;box-shadow:var(--bs-focus-ring-x,0) var(--bs-focus-ring-y,0) var(--bs-focus-ring-blur,0) var(--bs-focus-ring-width) var(--bs-focus-ring-color)}.icon-link{display:inline-flex;gap:.375rem;align-items:center;-webkit-text-decoration-color:rgba(var(--bs-link-color-rgb),var(--bs-link-opacity,0.5));text-decoration-color:rgba(var(--bs-link-color-rgb),var(--bs-link-opacity,0.5));text-underline-offset:0.25em;-webkit-backface-visibility:hidden;backface-visibility:hidden}.icon-link>.bi{flex-shrink:0;width:1em;height:1em;fill:currentcolor;transition:.2s ease-in-out transform}@media (prefers-reduced-motion:reduce){.icon-link>.bi{transition:none}}.icon-link-hover:focus-visible>.bi,.icon-link-hover:hover>.bi{transform:var(--bs-icon-link-transform,translate3d(.25em,0,0))}.ratio{position:relative;width:100%}.ratio::before{display:block;padding-top:var(--bs-aspect-ratio);content:""}.ratio>*{position:absolute;top:0;left:0;width:100%;height:100%}.ratio-1x1{--bs-aspect-ratio:100%}.ratio-4x3{--bs-aspect-ratio:75%}.ratio-16x9{--bs-aspect-ratio:56.25%}.ratio-21x9{--bs-aspect-ratio:42.8571428571%}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}.sticky-bottom{position:-webkit-sticky;position:sticky;bottom:0;z-index:1020}@media (min-width:576px){.sticky-sm-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}.sticky-sm-bottom{position:-webkit-sticky;position:sticky;bottom:0;z-index:1020}}@media (min-width:768px){.sticky-md-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}.sticky-md-bottom{position:-webkit-sticky;position:sticky;bottom:0;z-index:1020}}@media (min-width:992px){.sticky-lg-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}.sticky-lg-bottom{position:-webkit-sticky;position:sticky;bottom:0;z-index:1020}}@media (min-width:1200px){.sticky-xl-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}.sticky-xl-bottom{position:-webkit-sticky;position:sticky;bottom:0;z-index:1020}}@media (min-width:1400px){.sticky-xxl-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}.sticky-xxl-bottom{position:-webkit-sticky;position:sticky;bottom:0;z-index:1020}}.hstack{display:flex;flex-direction:row;align-items:center;align-self:stretch}.vstack{display:flex;flex:1 1 auto;flex-direction:column;align-self:stretch}.visually-hidden,.visually-hidden-focusable:not(:focus):not(:focus-within){width:1px!important;height:1px!important;padding:0!important;margin:-1px!important;overflow:hidden!important;clip:rect(0,0,0,0)!important;white-space:nowrap!important;border:0!important}.visually-hidden-focusable:not(:focus):not(:focus-within):not(caption),.visually-hidden:not(caption){position:absolute!important}.stretched-link::after{position:absolute;top:0;right:0;bottom:0;left:0;z-index:1;content:""}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.vr{display:inline-block;align-self:stretch;width:var(--bs-border-width);min-height:1em;background-color:currentcolor;opacity:.25}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.float-start{float:left!important}.float-end{float:right!important}.float-none{float:none!important}.object-fit-contain{-o-object-fit:contain!important;object-fit:contain!important}.object-fit-cover{-o-object-fit:cover!important;object-fit:cover!important}.object-fit-fill{-o-object-fit:fill!important;object-fit:fill!important}.object-fit-scale{-o-object-fit:scale-down!important;object-fit:scale-down!important}.object-fit-none{-o-object-fit:none!important;object-fit:none!important}.opacity-0{opacity:0!important}.opacity-25{opacity:.25!important}.opacity-50{opacity:.5!important}.opacity-75{opacity:.75!important}.opacity-100{opacity:1!important}.overflow-auto{overflow:auto!important}.overflow-hidden{overflow:hidden!important}.overflow-visible{overflow:visible!important}.overflow-scroll{overflow:scroll!important}.overflow-x-auto{overflow-x:auto!important}.overflow-x-hidden{overflow-x:hidden!important}.overflow-x-visible{overflow-x:visible!important}.overflow-x-scroll{overflow-x:scroll!important}.overflow-y-auto{overflow-y:auto!important}.overflow-y-hidden{overflow-y:hidden!important}.overflow-y-visible{overflow-y:visible!important}.overflow-y-scroll{overflow-y:scroll!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-grid{display:grid!important}.d-inline-grid{display:inline-grid!important}.d-table{display:table!important}.d-table-row{display:table-row!important}.d-table-cell{display:table-cell!important}.d-flex{display:flex!important}.d-inline-flex{display:inline-flex!important}.d-none{display:none!important}.shadow{box-shadow:var(--bs-box-shadow)!important}.shadow-sm{box-shadow:var(--bs-box-shadow-sm)!important}.shadow-lg{box-shadow:var(--bs-box-shadow-lg)!important}.shadow-none{box-shadow:none!important}.focus-ring-primary{--bs-focus-ring-color:rgba(var(--bs-primary-rgb), var(--bs-focus-ring-opacity))}.focus-ring-secondary{--bs-focus-ring-color:rgba(var(--bs-secondary-rgb), var(--bs-focus-ring-opacity))}.focus-ring-success{--bs-focus-ring-color:rgba(var(--bs-success-rgb), var(--bs-focus-ring-opacity))}.focus-ring-info{--bs-focus-ring-color:rgba(var(--bs-info-rgb), var(--bs-focus-ring-opacity))}.focus-ring-warning{--bs-focus-ring-color:rgba(var(--bs-warning-rgb), var(--bs-focus-ring-opacity))}.focus-ring-danger{--bs-focus-ring-color:rgba(var(--bs-danger-rgb), var(--bs-focus-ring-opacity))}.focus-ring-light{--bs-focus-ring-color:rgba(var(--bs-light-rgb), var(--bs-focus-ring-opacity))}.focus-ring-dark{--bs-focus-ring-color:rgba(var(--bs-dark-rgb), var(--bs-focus-ring-opacity))}.position-static{position:static!important}.position-relative{position:relative!important}.position-absolute{position:absolute!important}.position-fixed{position:fixed!important}.position-sticky{position:-webkit-sticky!important;position:sticky!important}.top-0{top:0!important}.top-50{top:50%!important}.top-100{top:100%!important}.bottom-0{bottom:0!important}.bottom-50{bottom:50%!important}.bottom-100{bottom:100%!important}.start-0{left:0!important}.start-50{left:50%!important}.start-100{left:100%!important}.end-0{right:0!important}.end-50{right:50%!important}.end-100{right:100%!important}.translate-middle{transform:translate(-50%,-50%)!important}.translate-middle-x{transform:translateX(-50%)!important}.translate-middle-y{transform:translateY(-50%)!important}.border{border:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color)!important}.border-0{border:0!important}.border-top{border-top:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color)!important}.border-top-0{border-top:0!important}.border-end{border-right:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color)!important}.border-end-0{border-right:0!important}.border-bottom{border-bottom:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color)!important}.border-bottom-0{border-bottom:0!important}.border-start{border-left:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color)!important}.border-start-0{border-left:0!important}.border-primary{--bs-border-opacity:1;border-color:rgba(var(--bs-primary-rgb),var(--bs-border-opacity))!important}.border-secondary{--bs-border-opacity:1;border-color:rgba(var(--bs-secondary-rgb),var(--bs-border-opacity))!important}.border-success{--bs-border-opacity:1;border-color:rgba(var(--bs-success-rgb),var(--bs-border-opacity))!important}.border-info{--bs-border-opacity:1;border-color:rgba(var(--bs-info-rgb),var(--bs-border-opacity))!important}.border-warning{--bs-border-opacity:1;border-color:rgba(var(--bs-warning-rgb),var(--bs-border-opacity))!important}.border-danger{--bs-border-opacity:1;border-color:rgba(var(--bs-danger-rgb),var(--bs-border-opacity))!important}.border-light{--bs-border-opacity:1;border-color:rgba(var(--bs-light-rgb),var(--bs-border-opacity))!important}.border-dark{--bs-border-opacity:1;border-color:rgba(var(--bs-dark-rgb),var(--bs-border-opacity))!important}.border-black{--bs-border-opacity:1;border-color:rgba(var(--bs-black-rgb),var(--bs-border-opacity))!important}.border-white{--bs-border-opacity:1;border-color:rgba(var(--bs-white-rgb),var(--bs-border-opacity))!important}.border-primary-subtle{border-color:var(--bs-primary-border-subtle)!important}.border-secondary-subtle{border-color:var(--bs-secondary-border-subtle)!important}.border-success-subtle{border-color:var(--bs-success-border-subtle)!important}.border-info-subtle{border-color:var(--bs-info-border-subtle)!important}.border-warning-subtle{border-color:var(--bs-warning-border-subtle)!important}.border-danger-subtle{border-color:var(--bs-danger-border-subtle)!important}.border-light-subtle{border-color:var(--bs-light-border-subtle)!important}.border-dark-subtle{border-color:var(--bs-dark-border-subtle)!important}.border-1{border-width:1px!important}.border-2{border-width:2px!important}.border-3{border-width:3px!important}.border-4{border-width:4px!important}.border-5{border-width:5px!important}.border-opacity-10{--bs-border-opacity:0.1}.border-opacity-25{--bs-border-opacity:0.25}.border-opacity-50{--bs-border-opacity:0.5}.border-opacity-75{--bs-border-opacity:0.75}.border-opacity-100{--bs-border-opacity:1}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.w-auto{width:auto!important}.mw-100{max-width:100%!important}.vw-100{width:100vw!important}.min-vw-100{min-width:100vw!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.h-auto{height:auto!important}.mh-100{max-height:100%!important}.vh-100{height:100vh!important}.min-vh-100{min-height:100vh!important}.flex-fill{flex:1 1 auto!important}.flex-row{flex-direction:row!important}.flex-column{flex-direction:column!important}.flex-row-reverse{flex-direction:row-reverse!important}.flex-column-reverse{flex-direction:column-reverse!important}.flex-grow-0{flex-grow:0!important}.flex-grow-1{flex-grow:1!important}.flex-shrink-0{flex-shrink:0!important}.flex-shrink-1{flex-shrink:1!important}.flex-wrap{flex-wrap:wrap!important}.flex-nowrap{flex-wrap:nowrap!important}.flex-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-start{justify-content:flex-start!important}.justify-content-end{justify-content:flex-end!important}.justify-content-center{justify-content:center!important}.justify-content-between{justify-content:space-between!important}.justify-content-around{justify-content:space-around!important}.justify-content-evenly{justify-content:space-evenly!important}.align-items-start{align-items:flex-start!important}.align-items-end{align-items:flex-end!important}.align-items-center{align-items:center!important}.align-items-baseline{align-items:baseline!important}.align-items-stretch{align-items:stretch!important}.align-content-start{align-content:flex-start!important}.align-content-end{align-content:flex-end!important}.align-content-center{align-content:center!important}.align-content-between{align-content:space-between!important}.align-content-around{align-content:space-around!important}.align-content-stretch{align-content:stretch!important}.align-self-auto{align-self:auto!important}.align-self-start{align-self:flex-start!important}.align-self-end{align-self:flex-end!important}.align-self-center{align-self:center!important}.align-self-baseline{align-self:baseline!important}.align-self-stretch{align-self:stretch!important}.order-first{order:-1!important}.order-0{order:0!important}.order-1{order:1!important}.order-2{order:2!important}.order-3{order:3!important}.order-4{order:4!important}.order-5{order:5!important}.order-last{order:6!important}.m-0{margin:0!important}.m-1{margin:.25rem!important}.m-2{margin:.5rem!important}.m-3{margin:1rem!important}.m-4{margin:1.5rem!important}.m-5{margin:3rem!important}.m-auto{margin:auto!important}.mx-0{margin-right:0!important;margin-left:0!important}.mx-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-3{margin-right:1rem!important;margin-left:1rem!important}.mx-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-5{margin-right:3rem!important;margin-left:3rem!important}.mx-auto{margin-right:auto!important;margin-left:auto!important}.my-0{margin-top:0!important;margin-bottom:0!important}.my-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-0{margin-top:0!important}.mt-1{margin-top:.25rem!important}.mt-2{margin-top:.5rem!important}.mt-3{margin-top:1rem!important}.mt-4{margin-top:1.5rem!important}.mt-5{margin-top:3rem!important}.mt-auto{margin-top:auto!important}.me-0{margin-right:0!important}.me-1{margin-right:.25rem!important}.me-2{margin-right:.5rem!important}.me-3{margin-right:1rem!important}.me-4{margin-right:1.5rem!important}.me-5{margin-right:3rem!important}.me-auto{margin-right:auto!important}.mb-0{margin-bottom:0!important}.mb-1{margin-bottom:.25rem!important}.mb-2{margin-bottom:.5rem!important}.mb-3{margin-bottom:1rem!important}.mb-4{margin-bottom:1.5rem!important}.mb-5{margin-bottom:3rem!important}.mb-auto{margin-bottom:auto!important}.ms-0{margin-left:0!important}.ms-1{margin-left:.25rem!important}.ms-2{margin-left:.5rem!important}.ms-3{margin-left:1rem!important}.ms-4{margin-left:1.5rem!important}.ms-5{margin-left:3rem!important}.ms-auto{margin-left:auto!important}.p-0{padding:0!important}.p-1{padding:.25rem!important}.p-2{padding:.5rem!important}.p-3{padding:1rem!important}.p-4{padding:1.5rem!important}.p-5{padding:3rem!important}.px-0{padding-right:0!important;padding-left:0!important}.px-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-3{padding-right:1rem!important;padding-left:1rem!important}.px-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-5{padding-right:3rem!important;padding-left:3rem!important}.py-0{padding-top:0!important;padding-bottom:0!important}.py-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-0{padding-top:0!important}.pt-1{padding-top:.25rem!important}.pt-2{padding-top:.5rem!important}.pt-3{padding-top:1rem!important}.pt-4{padding-top:1.5rem!important}.pt-5{padding-top:3rem!important}.pe-0{padding-right:0!important}.pe-1{padding-right:.25rem!important}.pe-2{padding-right:.5rem!important}.pe-3{padding-right:1rem!important}.pe-4{padding-right:1.5rem!important}.pe-5{padding-right:3rem!important}.pb-0{padding-bottom:0!important}.pb-1{padding-bottom:.25rem!important}.pb-2{padding-bottom:.5rem!important}.pb-3{padding-bottom:1rem!important}.pb-4{padding-bottom:1.5rem!important}.pb-5{padding-bottom:3rem!important}.ps-0{padding-left:0!important}.ps-1{padding-left:.25rem!important}.ps-2{padding-left:.5rem!important}.ps-3{padding-left:1rem!important}.ps-4{padding-left:1.5rem!important}.ps-5{padding-left:3rem!important}.gap-0{gap:0!important}.gap-1{gap:.25rem!important}.gap-2{gap:.5rem!important}.gap-3{gap:1rem!important}.gap-4{gap:1.5rem!important}.gap-5{gap:3rem!important}.row-gap-0{row-gap:0!important}.row-gap-1{row-gap:.25rem!important}.row-gap-2{row-gap:.5rem!important}.row-gap-3{row-gap:1rem!important}.row-gap-4{row-gap:1.5rem!important}.row-gap-5{row-gap:3rem!important}.column-gap-0{-moz-column-gap:0!important;column-gap:0!important}.column-gap-1{-moz-column-gap:0.25rem!important;column-gap:.25rem!important}.column-gap-2{-moz-column-gap:0.5rem!important;column-gap:.5rem!important}.column-gap-3{-moz-column-gap:1rem!important;column-gap:1rem!important}.column-gap-4{-moz-column-gap:1.5rem!important;column-gap:1.5rem!important}.column-gap-5{-moz-column-gap:3rem!important;column-gap:3rem!important}.font-monospace{font-family:var(--bs-font-monospace)!important}.fs-1{font-size:calc(1.375rem + 1.5vw)!important}.fs-2{font-size:calc(1.325rem + .9vw)!important}.fs-3{font-size:calc(1.3rem + .6vw)!important}.fs-4{font-size:calc(1.275rem + .3vw)!important}.fs-5{font-size:1.25rem!important}.fs-6{font-size:1rem!important}.fst-italic{font-style:italic!important}.fst-normal{font-style:normal!important}.fw-lighter{font-weight:lighter!important}.fw-light{font-weight:300!important}.fw-normal{font-weight:400!important}.fw-medium{font-weight:500!important}.fw-semibold{font-weight:600!important}.fw-bold{font-weight:700!important}.fw-bolder{font-weight:bolder!important}.lh-1{line-height:1!important}.lh-sm{line-height:1.25!important}.lh-base{line-height:1.5!important}.lh-lg{line-height:2!important}.text-start{text-align:left!important}.text-end{text-align:right!important}.text-center{text-align:center!important}.text-decoration-none{text-decoration:none!important}.text-decoration-underline{text-decoration:underline!important}.text-decoration-line-through{text-decoration:line-through!important}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.text-wrap{white-space:normal!important}.text-nowrap{white-space:nowrap!important}.text-break{word-wrap:break-word!important;word-break:break-word!important}.text-primary{--bs-text-opacity:1;color:rgba(var(--bs-primary-rgb),var(--bs-text-opacity))!important}.text-secondary{--bs-text-opacity:1;color:rgba(var(--bs-secondary-rgb),var(--bs-text-opacity))!important}.text-success{--bs-text-opacity:1;color:rgba(var(--bs-success-rgb),var(--bs-text-opacity))!important}.text-info{--bs-text-opacity:1;color:rgba(var(--bs-info-rgb),var(--bs-text-opacity))!important}.text-warning{--bs-text-opacity:1;color:rgba(var(--bs-warning-rgb),var(--bs-text-opacity))!important}.text-danger{--bs-text-opacity:1;color:rgba(var(--bs-danger-rgb),var(--bs-text-opacity))!important}.text-light{--bs-text-opacity:1;color:rgba(var(--bs-light-rgb),var(--bs-text-opacity))!important}.text-dark{--bs-text-opacity:1;color:rgba(var(--bs-dark-rgb),var(--bs-text-opacity))!important}.text-black{--bs-text-opacity:1;color:rgba(var(--bs-black-rgb),var(--bs-text-opacity))!important}.text-white{--bs-text-opacity:1;color:rgba(var(--bs-white-rgb),var(--bs-text-opacity))!important}.text-body{--bs-text-opacity:1;color:rgba(var(--bs-body-color-rgb),var(--bs-text-opacity))!important}.text-muted{--bs-text-opacity:1;color:var(--bs-secondary-color)!important}.text-black-50{--bs-text-opacity:1;color:rgba(0,0,0,.5)!important}.text-white-50{--bs-text-opacity:1;color:rgba(255,255,255,.5)!important}.text-body-secondary{--bs-text-opacity:1;color:var(--bs-secondary-color)!important}.text-body-tertiary{--bs-text-opacity:1;color:var(--bs-tertiary-color)!important}.text-body-emphasis{--bs-text-opacity:1;color:var(--bs-emphasis-color)!important}.text-reset{--bs-text-opacity:1;color:inherit!important}.text-opacity-25{--bs-text-opacity:0.25}.text-opacity-50{--bs-text-opacity:0.5}.text-opacity-75{--bs-text-opacity:0.75}.text-opacity-100{--bs-text-opacity:1}.text-primary-emphasis{color:var(--bs-primary-text-emphasis)!important}.text-secondary-emphasis{color:var(--bs-secondary-text-emphasis)!important}.text-success-emphasis{color:var(--bs-success-text-emphasis)!important}.text-info-emphasis{color:var(--bs-info-text-emphasis)!important}.text-warning-emphasis{color:var(--bs-warning-text-emphasis)!important}.text-danger-emphasis{color:var(--bs-danger-text-emphasis)!important}.text-light-emphasis{color:var(--bs-light-text-emphasis)!important}.text-dark-emphasis{color:var(--bs-dark-text-emphasis)!important}.link-opacity-10{--bs-link-opacity:0.1}.link-opacity-10-hover:hover{--bs-link-opacity:0.1}.link-opacity-25{--bs-link-opacity:0.25}.link-opacity-25-hover:hover{--bs-link-opacity:0.25}.link-opacity-50{--bs-link-opacity:0.5}.link-opacity-50-hover:hover{--bs-link-opacity:0.5}.link-opacity-75{--bs-link-opacity:0.75}.link-opacity-75-hover:hover{--bs-link-opacity:0.75}.link-opacity-100{--bs-link-opacity:1}.link-opacity-100-hover:hover{--bs-link-opacity:1}.link-offset-1{text-underline-offset:0.125em!important}.link-offset-1-hover:hover{text-underline-offset:0.125em!important}.link-offset-2{text-underline-offset:0.25em!important}.link-offset-2-hover:hover{text-underline-offset:0.25em!important}.link-offset-3{text-underline-offset:0.375em!important}.link-offset-3-hover:hover{text-underline-offset:0.375em!important}.link-underline-primary{--bs-link-underline-opacity:1;-webkit-text-decoration-color:rgba(var(--bs-primary-rgb),var(--bs-link-underline-opacity))!important;text-decoration-color:rgba(var(--bs-primary-rgb),var(--bs-link-underline-opacity))!important}.link-underline-secondary{--bs-link-underline-opacity:1;-webkit-text-decoration-color:rgba(var(--bs-secondary-rgb),var(--bs-link-underline-opacity))!important;text-decoration-color:rgba(var(--bs-secondary-rgb),var(--bs-link-underline-opacity))!important}.link-underline-success{--bs-link-underline-opacity:1;-webkit-text-decoration-color:rgba(var(--bs-success-rgb),var(--bs-link-underline-opacity))!important;text-decoration-color:rgba(var(--bs-success-rgb),var(--bs-link-underline-opacity))!important}.link-underline-info{--bs-link-underline-opacity:1;-webkit-text-decoration-color:rgba(var(--bs-info-rgb),var(--bs-link-underline-opacity))!important;text-decoration-color:rgba(var(--bs-info-rgb),var(--bs-link-underline-opacity))!important}.link-underline-warning{--bs-link-underline-opacity:1;-webkit-text-decoration-color:rgba(var(--bs-warning-rgb),var(--bs-link-underline-opacity))!important;text-decoration-color:rgba(var(--bs-warning-rgb),var(--bs-link-underline-opacity))!important}.link-underline-danger{--bs-link-underline-opacity:1;-webkit-text-decoration-color:rgba(var(--bs-danger-rgb),var(--bs-link-underline-opacity))!important;text-decoration-color:rgba(var(--bs-danger-rgb),var(--bs-link-underline-opacity))!important}.link-underline-light{--bs-link-underline-opacity:1;-webkit-text-decoration-color:rgba(var(--bs-light-rgb),var(--bs-link-underline-opacity))!important;text-decoration-color:rgba(var(--bs-light-rgb),var(--bs-link-underline-opacity))!important}.link-underline-dark{--bs-link-underline-opacity:1;-webkit-text-decoration-color:rgba(var(--bs-dark-rgb),var(--bs-link-underline-opacity))!important;text-decoration-color:rgba(var(--bs-dark-rgb),var(--bs-link-underline-opacity))!important}.link-underline{--bs-link-underline-opacity:1;-webkit-text-decoration-color:rgba(var(--bs-link-color-rgb),var(--bs-link-underline-opacity,1))!important;text-decoration-color:rgba(var(--bs-link-color-rgb),var(--bs-link-underline-opacity,1))!important}.link-underline-opacity-0{--bs-link-underline-opacity:0}.link-underline-opacity-0-hover:hover{--bs-link-underline-opacity:0}.link-underline-opacity-10{--bs-link-underline-opacity:0.1}.link-underline-opacity-10-hover:hover{--bs-link-underline-opacity:0.1}.link-underline-opacity-25{--bs-link-underline-opacity:0.25}.link-underline-opacity-25-hover:hover{--bs-link-underline-opacity:0.25}.link-underline-opacity-50{--bs-link-underline-opacity:0.5}.link-underline-opacity-50-hover:hover{--bs-link-underline-opacity:0.5}.link-underline-opacity-75{--bs-link-underline-opacity:0.75}.link-underline-opacity-75-hover:hover{--bs-link-underline-opacity:0.75}.link-underline-opacity-100{--bs-link-underline-opacity:1}.link-underline-opacity-100-hover:hover{--bs-link-underline-opacity:1}.bg-primary{--bs-bg-opacity:1;background-color:rgba(var(--bs-primary-rgb),var(--bs-bg-opacity))!important}.bg-secondary{--bs-bg-opacity:1;background-color:rgba(var(--bs-secondary-rgb),var(--bs-bg-opacity))!important}.bg-success{--bs-bg-opacity:1;background-color:rgba(var(--bs-success-rgb),var(--bs-bg-opacity))!important}.bg-info{--bs-bg-opacity:1;background-color:rgba(var(--bs-info-rgb),var(--bs-bg-opacity))!important}.bg-warning{--bs-bg-opacity:1;background-color:rgba(var(--bs-warning-rgb),var(--bs-bg-opacity))!important}.bg-danger{--bs-bg-opacity:1;background-color:rgba(var(--bs-danger-rgb),var(--bs-bg-opacity))!important}.bg-light{--bs-bg-opacity:1;background-color:rgba(var(--bs-light-rgb),var(--bs-bg-opacity))!important}.bg-dark{--bs-bg-opacity:1;background-color:rgba(var(--bs-dark-rgb),var(--bs-bg-opacity))!important}.bg-black{--bs-bg-opacity:1;background-color:rgba(var(--bs-black-rgb),var(--bs-bg-opacity))!important}.bg-white{--bs-bg-opacity:1;background-color:rgba(var(--bs-white-rgb),var(--bs-bg-opacity))!important}.bg-body{--bs-bg-opacity:1;background-color:rgba(var(--bs-body-bg-rgb),var(--bs-bg-opacity))!important}.bg-transparent{--bs-bg-opacity:1;background-color:transparent!important}.bg-body-secondary{--bs-bg-opacity:1;background-color:rgba(var(--bs-secondary-bg-rgb),var(--bs-bg-opacity))!important}.bg-body-tertiary{--bs-bg-opacity:1;background-color:rgba(var(--bs-tertiary-bg-rgb),var(--bs-bg-opacity))!important}.bg-opacity-10{--bs-bg-opacity:0.1}.bg-opacity-25{--bs-bg-opacity:0.25}.bg-opacity-50{--bs-bg-opacity:0.5}.bg-opacity-75{--bs-bg-opacity:0.75}.bg-opacity-100{--bs-bg-opacity:1}.bg-primary-subtle{background-color:var(--bs-primary-bg-subtle)!important}.bg-secondary-subtle{background-color:var(--bs-secondary-bg-subtle)!important}.bg-success-subtle{background-color:var(--bs-success-bg-subtle)!important}.bg-info-subtle{background-color:var(--bs-info-bg-subtle)!important}.bg-warning-subtle{background-color:var(--bs-warning-bg-subtle)!important}.bg-danger-subtle{background-color:var(--bs-danger-bg-subtle)!important}.bg-light-subtle{background-color:var(--bs-light-bg-subtle)!important}.bg-dark-subtle{background-color:var(--bs-dark-bg-subtle)!important}.bg-gradient{background-image:var(--bs-gradient)!important}.user-select-all{-webkit-user-select:all!important;-moz-user-select:all!important;user-select:all!important}.user-select-auto{-webkit-user-select:auto!important;-moz-user-select:auto!important;user-select:auto!important}.user-select-none{-webkit-user-select:none!important;-moz-user-select:none!important;user-select:none!important}.pe-none{pointer-events:none!important}.pe-auto{pointer-events:auto!important}.rounded{border-radius:var(--bs-border-radius)!important}.rounded-0{border-radius:0!important}.rounded-1{border-radius:var(--bs-border-radius-sm)!important}.rounded-2{border-radius:var(--bs-border-radius)!important}.rounded-3{border-radius:var(--bs-border-radius-lg)!important}.rounded-4{border-radius:var(--bs-border-radius-xl)!important}.rounded-5{border-radius:var(--bs-border-radius-xxl)!important}.rounded-circle{border-radius:50%!important}.rounded-pill{border-radius:var(--bs-border-radius-pill)!important}.rounded-top{border-top-left-radius:var(--bs-border-radius)!important;border-top-right-radius:var(--bs-border-radius)!important}.rounded-top-0{border-top-left-radius:0!important;border-top-right-radius:0!important}.rounded-top-1{border-top-left-radius:var(--bs-border-radius-sm)!important;border-top-right-radius:var(--bs-border-radius-sm)!important}.rounded-top-2{border-top-left-radius:var(--bs-border-radius)!important;border-top-right-radius:var(--bs-border-radius)!important}.rounded-top-3{border-top-left-radius:var(--bs-border-radius-lg)!important;border-top-right-radius:var(--bs-border-radius-lg)!important}.rounded-top-4{border-top-left-radius:var(--bs-border-radius-xl)!important;border-top-right-radius:var(--bs-border-radius-xl)!important}.rounded-top-5{border-top-left-radius:var(--bs-border-radius-xxl)!important;border-top-right-radius:var(--bs-border-radius-xxl)!important}.rounded-top-circle{border-top-left-radius:50%!important;border-top-right-radius:50%!important}.rounded-top-pill{border-top-left-radius:var(--bs-border-radius-pill)!important;border-top-right-radius:var(--bs-border-radius-pill)!important}.rounded-end{border-top-right-radius:var(--bs-border-radius)!important;border-bottom-right-radius:var(--bs-border-radius)!important}.rounded-end-0{border-top-right-radius:0!important;border-bottom-right-radius:0!important}.rounded-end-1{border-top-right-radius:var(--bs-border-radius-sm)!important;border-bottom-right-radius:var(--bs-border-radius-sm)!important}.rounded-end-2{border-top-right-radius:var(--bs-border-radius)!important;border-bottom-right-radius:var(--bs-border-radius)!important}.rounded-end-3{border-top-right-radius:var(--bs-border-radius-lg)!important;border-bottom-right-radius:var(--bs-border-radius-lg)!important}.rounded-end-4{border-top-right-radius:var(--bs-border-radius-xl)!important;border-bottom-right-radius:var(--bs-border-radius-xl)!important}.rounded-end-5{border-top-right-radius:var(--bs-border-radius-xxl)!important;border-bottom-right-radius:var(--bs-border-radius-xxl)!important}.rounded-end-circle{border-top-right-radius:50%!important;border-bottom-right-radius:50%!important}.rounded-end-pill{border-top-right-radius:var(--bs-border-radius-pill)!important;border-bottom-right-radius:var(--bs-border-radius-pill)!important}.rounded-bottom{border-bottom-right-radius:var(--bs-border-radius)!important;border-bottom-left-radius:var(--bs-border-radius)!important}.rounded-bottom-0{border-bottom-right-radius:0!important;border-bottom-left-radius:0!important}.rounded-bottom-1{border-bottom-right-radius:var(--bs-border-radius-sm)!important;border-bottom-left-radius:var(--bs-border-radius-sm)!important}.rounded-bottom-2{border-bottom-right-radius:var(--bs-border-radius)!important;border-bottom-left-radius:var(--bs-border-radius)!important}.rounded-bottom-3{border-bottom-right-radius:var(--bs-border-radius-lg)!important;border-bottom-left-radius:var(--bs-border-radius-lg)!important}.rounded-bottom-4{border-bottom-right-radius:var(--bs-border-radius-xl)!important;border-bottom-left-radius:var(--bs-border-radius-xl)!important}.rounded-bottom-5{border-bottom-right-radius:var(--bs-border-radius-xxl)!important;border-bottom-left-radius:var(--bs-border-radius-xxl)!important}.rounded-bottom-circle{border-bottom-right-radius:50%!important;border-bottom-left-radius:50%!important}.rounded-bottom-pill{border-bottom-right-radius:var(--bs-border-radius-pill)!important;border-bottom-left-radius:var(--bs-border-radius-pill)!important}.rounded-start{border-bottom-left-radius:var(--bs-border-radius)!important;border-top-left-radius:var(--bs-border-radius)!important}.rounded-start-0{border-bottom-left-radius:0!important;border-top-left-radius:0!important}.rounded-start-1{border-bottom-left-radius:var(--bs-border-radius-sm)!important;border-top-left-radius:var(--bs-border-radius-sm)!important}.rounded-start-2{border-bottom-left-radius:var(--bs-border-radius)!important;border-top-left-radius:var(--bs-border-radius)!important}.rounded-start-3{border-bottom-left-radius:var(--bs-border-radius-lg)!important;border-top-left-radius:var(--bs-border-radius-lg)!important}.rounded-start-4{border-bottom-left-radius:var(--bs-border-radius-xl)!important;border-top-left-radius:var(--bs-border-radius-xl)!important}.rounded-start-5{border-bottom-left-radius:var(--bs-border-radius-xxl)!important;border-top-left-radius:var(--bs-border-radius-xxl)!important}.rounded-start-circle{border-bottom-left-radius:50%!important;border-top-left-radius:50%!important}.rounded-start-pill{border-bottom-left-radius:var(--bs-border-radius-pill)!important;border-top-left-radius:var(--bs-border-radius-pill)!important}.visible{visibility:visible!important}.invisible{visibility:hidden!important}.z-n1{z-index:-1!important}.z-0{z-index:0!important}.z-1{z-index:1!important}.z-2{z-index:2!important}.z-3{z-index:3!important}@media (min-width:576px){.float-sm-start{float:left!important}.float-sm-end{float:right!important}.float-sm-none{float:none!important}.object-fit-sm-contain{-o-object-fit:contain!important;object-fit:contain!important}.object-fit-sm-cover{-o-object-fit:cover!important;object-fit:cover!important}.object-fit-sm-fill{-o-object-fit:fill!important;object-fit:fill!important}.object-fit-sm-scale{-o-object-fit:scale-down!important;object-fit:scale-down!important}.object-fit-sm-none{-o-object-fit:none!important;object-fit:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-grid{display:grid!important}.d-sm-inline-grid{display:inline-grid!important}.d-sm-table{display:table!important}.d-sm-table-row{display:table-row!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:flex!important}.d-sm-inline-flex{display:inline-flex!important}.d-sm-none{display:none!important}.flex-sm-fill{flex:1 1 auto!important}.flex-sm-row{flex-direction:row!important}.flex-sm-column{flex-direction:column!important}.flex-sm-row-reverse{flex-direction:row-reverse!important}.flex-sm-column-reverse{flex-direction:column-reverse!important}.flex-sm-grow-0{flex-grow:0!important}.flex-sm-grow-1{flex-grow:1!important}.flex-sm-shrink-0{flex-shrink:0!important}.flex-sm-shrink-1{flex-shrink:1!important}.flex-sm-wrap{flex-wrap:wrap!important}.flex-sm-nowrap{flex-wrap:nowrap!important}.flex-sm-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-sm-start{justify-content:flex-start!important}.justify-content-sm-end{justify-content:flex-end!important}.justify-content-sm-center{justify-content:center!important}.justify-content-sm-between{justify-content:space-between!important}.justify-content-sm-around{justify-content:space-around!important}.justify-content-sm-evenly{justify-content:space-evenly!important}.align-items-sm-start{align-items:flex-start!important}.align-items-sm-end{align-items:flex-end!important}.align-items-sm-center{align-items:center!important}.align-items-sm-baseline{align-items:baseline!important}.align-items-sm-stretch{align-items:stretch!important}.align-content-sm-start{align-content:flex-start!important}.align-content-sm-end{align-content:flex-end!important}.align-content-sm-center{align-content:center!important}.align-content-sm-between{align-content:space-between!important}.align-content-sm-around{align-content:space-around!important}.align-content-sm-stretch{align-content:stretch!important}.align-self-sm-auto{align-self:auto!important}.align-self-sm-start{align-self:flex-start!important}.align-self-sm-end{align-self:flex-end!important}.align-self-sm-center{align-self:center!important}.align-self-sm-baseline{align-self:baseline!important}.align-self-sm-stretch{align-self:stretch!important}.order-sm-first{order:-1!important}.order-sm-0{order:0!important}.order-sm-1{order:1!important}.order-sm-2{order:2!important}.order-sm-3{order:3!important}.order-sm-4{order:4!important}.order-sm-5{order:5!important}.order-sm-last{order:6!important}.m-sm-0{margin:0!important}.m-sm-1{margin:.25rem!important}.m-sm-2{margin:.5rem!important}.m-sm-3{margin:1rem!important}.m-sm-4{margin:1.5rem!important}.m-sm-5{margin:3rem!important}.m-sm-auto{margin:auto!important}.mx-sm-0{margin-right:0!important;margin-left:0!important}.mx-sm-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-sm-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-sm-3{margin-right:1rem!important;margin-left:1rem!important}.mx-sm-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-sm-5{margin-right:3rem!important;margin-left:3rem!important}.mx-sm-auto{margin-right:auto!important;margin-left:auto!important}.my-sm-0{margin-top:0!important;margin-bottom:0!important}.my-sm-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-sm-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-sm-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-sm-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-sm-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-sm-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-sm-0{margin-top:0!important}.mt-sm-1{margin-top:.25rem!important}.mt-sm-2{margin-top:.5rem!important}.mt-sm-3{margin-top:1rem!important}.mt-sm-4{margin-top:1.5rem!important}.mt-sm-5{margin-top:3rem!important}.mt-sm-auto{margin-top:auto!important}.me-sm-0{margin-right:0!important}.me-sm-1{margin-right:.25rem!important}.me-sm-2{margin-right:.5rem!important}.me-sm-3{margin-right:1rem!important}.me-sm-4{margin-right:1.5rem!important}.me-sm-5{margin-right:3rem!important}.me-sm-auto{margin-right:auto!important}.mb-sm-0{margin-bottom:0!important}.mb-sm-1{margin-bottom:.25rem!important}.mb-sm-2{margin-bottom:.5rem!important}.mb-sm-3{margin-bottom:1rem!important}.mb-sm-4{margin-bottom:1.5rem!important}.mb-sm-5{margin-bottom:3rem!important}.mb-sm-auto{margin-bottom:auto!important}.ms-sm-0{margin-left:0!important}.ms-sm-1{margin-left:.25rem!important}.ms-sm-2{margin-left:.5rem!important}.ms-sm-3{margin-left:1rem!important}.ms-sm-4{margin-left:1.5rem!important}.ms-sm-5{margin-left:3rem!important}.ms-sm-auto{margin-left:auto!important}.p-sm-0{padding:0!important}.p-sm-1{padding:.25rem!important}.p-sm-2{padding:.5rem!important}.p-sm-3{padding:1rem!important}.p-sm-4{padding:1.5rem!important}.p-sm-5{padding:3rem!important}.px-sm-0{padding-right:0!important;padding-left:0!important}.px-sm-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-sm-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-sm-3{padding-right:1rem!important;padding-left:1rem!important}.px-sm-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-sm-5{padding-right:3rem!important;padding-left:3rem!important}.py-sm-0{padding-top:0!important;padding-bottom:0!important}.py-sm-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-sm-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-sm-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-sm-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-sm-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-sm-0{padding-top:0!important}.pt-sm-1{padding-top:.25rem!important}.pt-sm-2{padding-top:.5rem!important}.pt-sm-3{padding-top:1rem!important}.pt-sm-4{padding-top:1.5rem!important}.pt-sm-5{padding-top:3rem!important}.pe-sm-0{padding-right:0!important}.pe-sm-1{padding-right:.25rem!important}.pe-sm-2{padding-right:.5rem!important}.pe-sm-3{padding-right:1rem!important}.pe-sm-4{padding-right:1.5rem!important}.pe-sm-5{padding-right:3rem!important}.pb-sm-0{padding-bottom:0!important}.pb-sm-1{padding-bottom:.25rem!important}.pb-sm-2{padding-bottom:.5rem!important}.pb-sm-3{padding-bottom:1rem!important}.pb-sm-4{padding-bottom:1.5rem!important}.pb-sm-5{padding-bottom:3rem!important}.ps-sm-0{padding-left:0!important}.ps-sm-1{padding-left:.25rem!important}.ps-sm-2{padding-left:.5rem!important}.ps-sm-3{padding-left:1rem!important}.ps-sm-4{padding-left:1.5rem!important}.ps-sm-5{padding-left:3rem!important}.gap-sm-0{gap:0!important}.gap-sm-1{gap:.25rem!important}.gap-sm-2{gap:.5rem!important}.gap-sm-3{gap:1rem!important}.gap-sm-4{gap:1.5rem!important}.gap-sm-5{gap:3rem!important}.row-gap-sm-0{row-gap:0!important}.row-gap-sm-1{row-gap:.25rem!important}.row-gap-sm-2{row-gap:.5rem!important}.row-gap-sm-3{row-gap:1rem!important}.row-gap-sm-4{row-gap:1.5rem!important}.row-gap-sm-5{row-gap:3rem!important}.column-gap-sm-0{-moz-column-gap:0!important;column-gap:0!important}.column-gap-sm-1{-moz-column-gap:0.25rem!important;column-gap:.25rem!important}.column-gap-sm-2{-moz-column-gap:0.5rem!important;column-gap:.5rem!important}.column-gap-sm-3{-moz-column-gap:1rem!important;column-gap:1rem!important}.column-gap-sm-4{-moz-column-gap:1.5rem!important;column-gap:1.5rem!important}.column-gap-sm-5{-moz-column-gap:3rem!important;column-gap:3rem!important}.text-sm-start{text-align:left!important}.text-sm-end{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.float-md-start{float:left!important}.float-md-end{float:right!important}.float-md-none{float:none!important}.object-fit-md-contain{-o-object-fit:contain!important;object-fit:contain!important}.object-fit-md-cover{-o-object-fit:cover!important;object-fit:cover!important}.object-fit-md-fill{-o-object-fit:fill!important;object-fit:fill!important}.object-fit-md-scale{-o-object-fit:scale-down!important;object-fit:scale-down!important}.object-fit-md-none{-o-object-fit:none!important;object-fit:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-grid{display:grid!important}.d-md-inline-grid{display:inline-grid!important}.d-md-table{display:table!important}.d-md-table-row{display:table-row!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:flex!important}.d-md-inline-flex{display:inline-flex!important}.d-md-none{display:none!important}.flex-md-fill{flex:1 1 auto!important}.flex-md-row{flex-direction:row!important}.flex-md-column{flex-direction:column!important}.flex-md-row-reverse{flex-direction:row-reverse!important}.flex-md-column-reverse{flex-direction:column-reverse!important}.flex-md-grow-0{flex-grow:0!important}.flex-md-grow-1{flex-grow:1!important}.flex-md-shrink-0{flex-shrink:0!important}.flex-md-shrink-1{flex-shrink:1!important}.flex-md-wrap{flex-wrap:wrap!important}.flex-md-nowrap{flex-wrap:nowrap!important}.flex-md-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-md-start{justify-content:flex-start!important}.justify-content-md-end{justify-content:flex-end!important}.justify-content-md-center{justify-content:center!important}.justify-content-md-between{justify-content:space-between!important}.justify-content-md-around{justify-content:space-around!important}.justify-content-md-evenly{justify-content:space-evenly!important}.align-items-md-start{align-items:flex-start!important}.align-items-md-end{align-items:flex-end!important}.align-items-md-center{align-items:center!important}.align-items-md-baseline{align-items:baseline!important}.align-items-md-stretch{align-items:stretch!important}.align-content-md-start{align-content:flex-start!important}.align-content-md-end{align-content:flex-end!important}.align-content-md-center{align-content:center!important}.align-content-md-between{align-content:space-between!important}.align-content-md-around{align-content:space-around!important}.align-content-md-stretch{align-content:stretch!important}.align-self-md-auto{align-self:auto!important}.align-self-md-start{align-self:flex-start!important}.align-self-md-end{align-self:flex-end!important}.align-self-md-center{align-self:center!important}.align-self-md-baseline{align-self:baseline!important}.align-self-md-stretch{align-self:stretch!important}.order-md-first{order:-1!important}.order-md-0{order:0!important}.order-md-1{order:1!important}.order-md-2{order:2!important}.order-md-3{order:3!important}.order-md-4{order:4!important}.order-md-5{order:5!important}.order-md-last{order:6!important}.m-md-0{margin:0!important}.m-md-1{margin:.25rem!important}.m-md-2{margin:.5rem!important}.m-md-3{margin:1rem!important}.m-md-4{margin:1.5rem!important}.m-md-5{margin:3rem!important}.m-md-auto{margin:auto!important}.mx-md-0{margin-right:0!important;margin-left:0!important}.mx-md-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-md-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-md-3{margin-right:1rem!important;margin-left:1rem!important}.mx-md-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-md-5{margin-right:3rem!important;margin-left:3rem!important}.mx-md-auto{margin-right:auto!important;margin-left:auto!important}.my-md-0{margin-top:0!important;margin-bottom:0!important}.my-md-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-md-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-md-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-md-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-md-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-md-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-md-0{margin-top:0!important}.mt-md-1{margin-top:.25rem!important}.mt-md-2{margin-top:.5rem!important}.mt-md-3{margin-top:1rem!important}.mt-md-4{margin-top:1.5rem!important}.mt-md-5{margin-top:3rem!important}.mt-md-auto{margin-top:auto!important}.me-md-0{margin-right:0!important}.me-md-1{margin-right:.25rem!important}.me-md-2{margin-right:.5rem!important}.me-md-3{margin-right:1rem!important}.me-md-4{margin-right:1.5rem!important}.me-md-5{margin-right:3rem!important}.me-md-auto{margin-right:auto!important}.mb-md-0{margin-bottom:0!important}.mb-md-1{margin-bottom:.25rem!important}.mb-md-2{margin-bottom:.5rem!important}.mb-md-3{margin-bottom:1rem!important}.mb-md-4{margin-bottom:1.5rem!important}.mb-md-5{margin-bottom:3rem!important}.mb-md-auto{margin-bottom:auto!important}.ms-md-0{margin-left:0!important}.ms-md-1{margin-left:.25rem!important}.ms-md-2{margin-left:.5rem!important}.ms-md-3{margin-left:1rem!important}.ms-md-4{margin-left:1.5rem!important}.ms-md-5{margin-left:3rem!important}.ms-md-auto{margin-left:auto!important}.p-md-0{padding:0!important}.p-md-1{padding:.25rem!important}.p-md-2{padding:.5rem!important}.p-md-3{padding:1rem!important}.p-md-4{padding:1.5rem!important}.p-md-5{padding:3rem!important}.px-md-0{padding-right:0!important;padding-left:0!important}.px-md-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-md-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-md-3{padding-right:1rem!important;padding-left:1rem!important}.px-md-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-md-5{padding-right:3rem!important;padding-left:3rem!important}.py-md-0{padding-top:0!important;padding-bottom:0!important}.py-md-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-md-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-md-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-md-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-md-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-md-0{padding-top:0!important}.pt-md-1{padding-top:.25rem!important}.pt-md-2{padding-top:.5rem!important}.pt-md-3{padding-top:1rem!important}.pt-md-4{padding-top:1.5rem!important}.pt-md-5{padding-top:3rem!important}.pe-md-0{padding-right:0!important}.pe-md-1{padding-right:.25rem!important}.pe-md-2{padding-right:.5rem!important}.pe-md-3{padding-right:1rem!important}.pe-md-4{padding-right:1.5rem!important}.pe-md-5{padding-right:3rem!important}.pb-md-0{padding-bottom:0!important}.pb-md-1{padding-bottom:.25rem!important}.pb-md-2{padding-bottom:.5rem!important}.pb-md-3{padding-bottom:1rem!important}.pb-md-4{padding-bottom:1.5rem!important}.pb-md-5{padding-bottom:3rem!important}.ps-md-0{padding-left:0!important}.ps-md-1{padding-left:.25rem!important}.ps-md-2{padding-left:.5rem!important}.ps-md-3{padding-left:1rem!important}.ps-md-4{padding-left:1.5rem!important}.ps-md-5{padding-left:3rem!important}.gap-md-0{gap:0!important}.gap-md-1{gap:.25rem!important}.gap-md-2{gap:.5rem!important}.gap-md-3{gap:1rem!important}.gap-md-4{gap:1.5rem!important}.gap-md-5{gap:3rem!important}.row-gap-md-0{row-gap:0!important}.row-gap-md-1{row-gap:.25rem!important}.row-gap-md-2{row-gap:.5rem!important}.row-gap-md-3{row-gap:1rem!important}.row-gap-md-4{row-gap:1.5rem!important}.row-gap-md-5{row-gap:3rem!important}.column-gap-md-0{-moz-column-gap:0!important;column-gap:0!important}.column-gap-md-1{-moz-column-gap:0.25rem!important;column-gap:.25rem!important}.column-gap-md-2{-moz-column-gap:0.5rem!important;column-gap:.5rem!important}.column-gap-md-3{-moz-column-gap:1rem!important;column-gap:1rem!important}.column-gap-md-4{-moz-column-gap:1.5rem!important;column-gap:1.5rem!important}.column-gap-md-5{-moz-column-gap:3rem!important;column-gap:3rem!important}.text-md-start{text-align:left!important}.text-md-end{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.float-lg-start{float:left!important}.float-lg-end{float:right!important}.float-lg-none{float:none!important}.object-fit-lg-contain{-o-object-fit:contain!important;object-fit:contain!important}.object-fit-lg-cover{-o-object-fit:cover!important;object-fit:cover!important}.object-fit-lg-fill{-o-object-fit:fill!important;object-fit:fill!important}.object-fit-lg-scale{-o-object-fit:scale-down!important;object-fit:scale-down!important}.object-fit-lg-none{-o-object-fit:none!important;object-fit:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-grid{display:grid!important}.d-lg-inline-grid{display:inline-grid!important}.d-lg-table{display:table!important}.d-lg-table-row{display:table-row!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:flex!important}.d-lg-inline-flex{display:inline-flex!important}.d-lg-none{display:none!important}.flex-lg-fill{flex:1 1 auto!important}.flex-lg-row{flex-direction:row!important}.flex-lg-column{flex-direction:column!important}.flex-lg-row-reverse{flex-direction:row-reverse!important}.flex-lg-column-reverse{flex-direction:column-reverse!important}.flex-lg-grow-0{flex-grow:0!important}.flex-lg-grow-1{flex-grow:1!important}.flex-lg-shrink-0{flex-shrink:0!important}.flex-lg-shrink-1{flex-shrink:1!important}.flex-lg-wrap{flex-wrap:wrap!important}.flex-lg-nowrap{flex-wrap:nowrap!important}.flex-lg-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-lg-start{justify-content:flex-start!important}.justify-content-lg-end{justify-content:flex-end!important}.justify-content-lg-center{justify-content:center!important}.justify-content-lg-between{justify-content:space-between!important}.justify-content-lg-around{justify-content:space-around!important}.justify-content-lg-evenly{justify-content:space-evenly!important}.align-items-lg-start{align-items:flex-start!important}.align-items-lg-end{align-items:flex-end!important}.align-items-lg-center{align-items:center!important}.align-items-lg-baseline{align-items:baseline!important}.align-items-lg-stretch{align-items:stretch!important}.align-content-lg-start{align-content:flex-start!important}.align-content-lg-end{align-content:flex-end!important}.align-content-lg-center{align-content:center!important}.align-content-lg-between{align-content:space-between!important}.align-content-lg-around{align-content:space-around!important}.align-content-lg-stretch{align-content:stretch!important}.align-self-lg-auto{align-self:auto!important}.align-self-lg-start{align-self:flex-start!important}.align-self-lg-end{align-self:flex-end!important}.align-self-lg-center{align-self:center!important}.align-self-lg-baseline{align-self:baseline!important}.align-self-lg-stretch{align-self:stretch!important}.order-lg-first{order:-1!important}.order-lg-0{order:0!important}.order-lg-1{order:1!important}.order-lg-2{order:2!important}.order-lg-3{order:3!important}.order-lg-4{order:4!important}.order-lg-5{order:5!important}.order-lg-last{order:6!important}.m-lg-0{margin:0!important}.m-lg-1{margin:.25rem!important}.m-lg-2{margin:.5rem!important}.m-lg-3{margin:1rem!important}.m-lg-4{margin:1.5rem!important}.m-lg-5{margin:3rem!important}.m-lg-auto{margin:auto!important}.mx-lg-0{margin-right:0!important;margin-left:0!important}.mx-lg-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-lg-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-lg-3{margin-right:1rem!important;margin-left:1rem!important}.mx-lg-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-lg-5{margin-right:3rem!important;margin-left:3rem!important}.mx-lg-auto{margin-right:auto!important;margin-left:auto!important}.my-lg-0{margin-top:0!important;margin-bottom:0!important}.my-lg-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-lg-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-lg-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-lg-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-lg-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-lg-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-lg-0{margin-top:0!important}.mt-lg-1{margin-top:.25rem!important}.mt-lg-2{margin-top:.5rem!important}.mt-lg-3{margin-top:1rem!important}.mt-lg-4{margin-top:1.5rem!important}.mt-lg-5{margin-top:3rem!important}.mt-lg-auto{margin-top:auto!important}.me-lg-0{margin-right:0!important}.me-lg-1{margin-right:.25rem!important}.me-lg-2{margin-right:.5rem!important}.me-lg-3{margin-right:1rem!important}.me-lg-4{margin-right:1.5rem!important}.me-lg-5{margin-right:3rem!important}.me-lg-auto{margin-right:auto!important}.mb-lg-0{margin-bottom:0!important}.mb-lg-1{margin-bottom:.25rem!important}.mb-lg-2{margin-bottom:.5rem!important}.mb-lg-3{margin-bottom:1rem!important}.mb-lg-4{margin-bottom:1.5rem!important}.mb-lg-5{margin-bottom:3rem!important}.mb-lg-auto{margin-bottom:auto!important}.ms-lg-0{margin-left:0!important}.ms-lg-1{margin-left:.25rem!important}.ms-lg-2{margin-left:.5rem!important}.ms-lg-3{margin-left:1rem!important}.ms-lg-4{margin-left:1.5rem!important}.ms-lg-5{margin-left:3rem!important}.ms-lg-auto{margin-left:auto!important}.p-lg-0{padding:0!important}.p-lg-1{padding:.25rem!important}.p-lg-2{padding:.5rem!important}.p-lg-3{padding:1rem!important}.p-lg-4{padding:1.5rem!important}.p-lg-5{padding:3rem!important}.px-lg-0{padding-right:0!important;padding-left:0!important}.px-lg-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-lg-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-lg-3{padding-right:1rem!important;padding-left:1rem!important}.px-lg-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-lg-5{padding-right:3rem!important;padding-left:3rem!important}.py-lg-0{padding-top:0!important;padding-bottom:0!important}.py-lg-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-lg-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-lg-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-lg-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-lg-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-lg-0{padding-top:0!important}.pt-lg-1{padding-top:.25rem!important}.pt-lg-2{padding-top:.5rem!important}.pt-lg-3{padding-top:1rem!important}.pt-lg-4{padding-top:1.5rem!important}.pt-lg-5{padding-top:3rem!important}.pe-lg-0{padding-right:0!important}.pe-lg-1{padding-right:.25rem!important}.pe-lg-2{padding-right:.5rem!important}.pe-lg-3{padding-right:1rem!important}.pe-lg-4{padding-right:1.5rem!important}.pe-lg-5{padding-right:3rem!important}.pb-lg-0{padding-bottom:0!important}.pb-lg-1{padding-bottom:.25rem!important}.pb-lg-2{padding-bottom:.5rem!important}.pb-lg-3{padding-bottom:1rem!important}.pb-lg-4{padding-bottom:1.5rem!important}.pb-lg-5{padding-bottom:3rem!important}.ps-lg-0{padding-left:0!important}.ps-lg-1{padding-left:.25rem!important}.ps-lg-2{padding-left:.5rem!important}.ps-lg-3{padding-left:1rem!important}.ps-lg-4{padding-left:1.5rem!important}.ps-lg-5{padding-left:3rem!important}.gap-lg-0{gap:0!important}.gap-lg-1{gap:.25rem!important}.gap-lg-2{gap:.5rem!important}.gap-lg-3{gap:1rem!important}.gap-lg-4{gap:1.5rem!important}.gap-lg-5{gap:3rem!important}.row-gap-lg-0{row-gap:0!important}.row-gap-lg-1{row-gap:.25rem!important}.row-gap-lg-2{row-gap:.5rem!important}.row-gap-lg-3{row-gap:1rem!important}.row-gap-lg-4{row-gap:1.5rem!important}.row-gap-lg-5{row-gap:3rem!important}.column-gap-lg-0{-moz-column-gap:0!important;column-gap:0!important}.column-gap-lg-1{-moz-column-gap:0.25rem!important;column-gap:.25rem!important}.column-gap-lg-2{-moz-column-gap:0.5rem!important;column-gap:.5rem!important}.column-gap-lg-3{-moz-column-gap:1rem!important;column-gap:1rem!important}.column-gap-lg-4{-moz-column-gap:1.5rem!important;column-gap:1.5rem!important}.column-gap-lg-5{-moz-column-gap:3rem!important;column-gap:3rem!important}.text-lg-start{text-align:left!important}.text-lg-end{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.float-xl-start{float:left!important}.float-xl-end{float:right!important}.float-xl-none{float:none!important}.object-fit-xl-contain{-o-object-fit:contain!important;object-fit:contain!important}.object-fit-xl-cover{-o-object-fit:cover!important;object-fit:cover!important}.object-fit-xl-fill{-o-object-fit:fill!important;object-fit:fill!important}.object-fit-xl-scale{-o-object-fit:scale-down!important;object-fit:scale-down!important}.object-fit-xl-none{-o-object-fit:none!important;object-fit:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-grid{display:grid!important}.d-xl-inline-grid{display:inline-grid!important}.d-xl-table{display:table!important}.d-xl-table-row{display:table-row!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:flex!important}.d-xl-inline-flex{display:inline-flex!important}.d-xl-none{display:none!important}.flex-xl-fill{flex:1 1 auto!important}.flex-xl-row{flex-direction:row!important}.flex-xl-column{flex-direction:column!important}.flex-xl-row-reverse{flex-direction:row-reverse!important}.flex-xl-column-reverse{flex-direction:column-reverse!important}.flex-xl-grow-0{flex-grow:0!important}.flex-xl-grow-1{flex-grow:1!important}.flex-xl-shrink-0{flex-shrink:0!important}.flex-xl-shrink-1{flex-shrink:1!important}.flex-xl-wrap{flex-wrap:wrap!important}.flex-xl-nowrap{flex-wrap:nowrap!important}.flex-xl-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-xl-start{justify-content:flex-start!important}.justify-content-xl-end{justify-content:flex-end!important}.justify-content-xl-center{justify-content:center!important}.justify-content-xl-between{justify-content:space-between!important}.justify-content-xl-around{justify-content:space-around!important}.justify-content-xl-evenly{justify-content:space-evenly!important}.align-items-xl-start{align-items:flex-start!important}.align-items-xl-end{align-items:flex-end!important}.align-items-xl-center{align-items:center!important}.align-items-xl-baseline{align-items:baseline!important}.align-items-xl-stretch{align-items:stretch!important}.align-content-xl-start{align-content:flex-start!important}.align-content-xl-end{align-content:flex-end!important}.align-content-xl-center{align-content:center!important}.align-content-xl-between{align-content:space-between!important}.align-content-xl-around{align-content:space-around!important}.align-content-xl-stretch{align-content:stretch!important}.align-self-xl-auto{align-self:auto!important}.align-self-xl-start{align-self:flex-start!important}.align-self-xl-end{align-self:flex-end!important}.align-self-xl-center{align-self:center!important}.align-self-xl-baseline{align-self:baseline!important}.align-self-xl-stretch{align-self:stretch!important}.order-xl-first{order:-1!important}.order-xl-0{order:0!important}.order-xl-1{order:1!important}.order-xl-2{order:2!important}.order-xl-3{order:3!important}.order-xl-4{order:4!important}.order-xl-5{order:5!important}.order-xl-last{order:6!important}.m-xl-0{margin:0!important}.m-xl-1{margin:.25rem!important}.m-xl-2{margin:.5rem!important}.m-xl-3{margin:1rem!important}.m-xl-4{margin:1.5rem!important}.m-xl-5{margin:3rem!important}.m-xl-auto{margin:auto!important}.mx-xl-0{margin-right:0!important;margin-left:0!important}.mx-xl-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-xl-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-xl-3{margin-right:1rem!important;margin-left:1rem!important}.mx-xl-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-xl-5{margin-right:3rem!important;margin-left:3rem!important}.mx-xl-auto{margin-right:auto!important;margin-left:auto!important}.my-xl-0{margin-top:0!important;margin-bottom:0!important}.my-xl-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-xl-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-xl-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-xl-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-xl-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-xl-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-xl-0{margin-top:0!important}.mt-xl-1{margin-top:.25rem!important}.mt-xl-2{margin-top:.5rem!important}.mt-xl-3{margin-top:1rem!important}.mt-xl-4{margin-top:1.5rem!important}.mt-xl-5{margin-top:3rem!important}.mt-xl-auto{margin-top:auto!important}.me-xl-0{margin-right:0!important}.me-xl-1{margin-right:.25rem!important}.me-xl-2{margin-right:.5rem!important}.me-xl-3{margin-right:1rem!important}.me-xl-4{margin-right:1.5rem!important}.me-xl-5{margin-right:3rem!important}.me-xl-auto{margin-right:auto!important}.mb-xl-0{margin-bottom:0!important}.mb-xl-1{margin-bottom:.25rem!important}.mb-xl-2{margin-bottom:.5rem!important}.mb-xl-3{margin-bottom:1rem!important}.mb-xl-4{margin-bottom:1.5rem!important}.mb-xl-5{margin-bottom:3rem!important}.mb-xl-auto{margin-bottom:auto!important}.ms-xl-0{margin-left:0!important}.ms-xl-1{margin-left:.25rem!important}.ms-xl-2{margin-left:.5rem!important}.ms-xl-3{margin-left:1rem!important}.ms-xl-4{margin-left:1.5rem!important}.ms-xl-5{margin-left:3rem!important}.ms-xl-auto{margin-left:auto!important}.p-xl-0{padding:0!important}.p-xl-1{padding:.25rem!important}.p-xl-2{padding:.5rem!important}.p-xl-3{padding:1rem!important}.p-xl-4{padding:1.5rem!important}.p-xl-5{padding:3rem!important}.px-xl-0{padding-right:0!important;padding-left:0!important}.px-xl-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-xl-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-xl-3{padding-right:1rem!important;padding-left:1rem!important}.px-xl-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-xl-5{padding-right:3rem!important;padding-left:3rem!important}.py-xl-0{padding-top:0!important;padding-bottom:0!important}.py-xl-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-xl-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-xl-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-xl-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-xl-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-xl-0{padding-top:0!important}.pt-xl-1{padding-top:.25rem!important}.pt-xl-2{padding-top:.5rem!important}.pt-xl-3{padding-top:1rem!important}.pt-xl-4{padding-top:1.5rem!important}.pt-xl-5{padding-top:3rem!important}.pe-xl-0{padding-right:0!important}.pe-xl-1{padding-right:.25rem!important}.pe-xl-2{padding-right:.5rem!important}.pe-xl-3{padding-right:1rem!important}.pe-xl-4{padding-right:1.5rem!important}.pe-xl-5{padding-right:3rem!important}.pb-xl-0{padding-bottom:0!important}.pb-xl-1{padding-bottom:.25rem!important}.pb-xl-2{padding-bottom:.5rem!important}.pb-xl-3{padding-bottom:1rem!important}.pb-xl-4{padding-bottom:1.5rem!important}.pb-xl-5{padding-bottom:3rem!important}.ps-xl-0{padding-left:0!important}.ps-xl-1{padding-left:.25rem!important}.ps-xl-2{padding-left:.5rem!important}.ps-xl-3{padding-left:1rem!important}.ps-xl-4{padding-left:1.5rem!important}.ps-xl-5{padding-left:3rem!important}.gap-xl-0{gap:0!important}.gap-xl-1{gap:.25rem!important}.gap-xl-2{gap:.5rem!important}.gap-xl-3{gap:1rem!important}.gap-xl-4{gap:1.5rem!important}.gap-xl-5{gap:3rem!important}.row-gap-xl-0{row-gap:0!important}.row-gap-xl-1{row-gap:.25rem!important}.row-gap-xl-2{row-gap:.5rem!important}.row-gap-xl-3{row-gap:1rem!important}.row-gap-xl-4{row-gap:1.5rem!important}.row-gap-xl-5{row-gap:3rem!important}.column-gap-xl-0{-moz-column-gap:0!important;column-gap:0!important}.column-gap-xl-1{-moz-column-gap:0.25rem!important;column-gap:.25rem!important}.column-gap-xl-2{-moz-column-gap:0.5rem!important;column-gap:.5rem!important}.column-gap-xl-3{-moz-column-gap:1rem!important;column-gap:1rem!important}.column-gap-xl-4{-moz-column-gap:1.5rem!important;column-gap:1.5rem!important}.column-gap-xl-5{-moz-column-gap:3rem!important;column-gap:3rem!important}.text-xl-start{text-align:left!important}.text-xl-end{text-align:right!important}.text-xl-center{text-align:center!important}}@media (min-width:1400px){.float-xxl-start{float:left!important}.float-xxl-end{float:right!important}.float-xxl-none{float:none!important}.object-fit-xxl-contain{-o-object-fit:contain!important;object-fit:contain!important}.object-fit-xxl-cover{-o-object-fit:cover!important;object-fit:cover!important}.object-fit-xxl-fill{-o-object-fit:fill!important;object-fit:fill!important}.object-fit-xxl-scale{-o-object-fit:scale-down!important;object-fit:scale-down!important}.object-fit-xxl-none{-o-object-fit:none!important;object-fit:none!important}.d-xxl-inline{display:inline!important}.d-xxl-inline-block{display:inline-block!important}.d-xxl-block{display:block!important}.d-xxl-grid{display:grid!important}.d-xxl-inline-grid{display:inline-grid!important}.d-xxl-table{display:table!important}.d-xxl-table-row{display:table-row!important}.d-xxl-table-cell{display:table-cell!important}.d-xxl-flex{display:flex!important}.d-xxl-inline-flex{display:inline-flex!important}.d-xxl-none{display:none!important}.flex-xxl-fill{flex:1 1 auto!important}.flex-xxl-row{flex-direction:row!important}.flex-xxl-column{flex-direction:column!important}.flex-xxl-row-reverse{flex-direction:row-reverse!important}.flex-xxl-column-reverse{flex-direction:column-reverse!important}.flex-xxl-grow-0{flex-grow:0!important}.flex-xxl-grow-1{flex-grow:1!important}.flex-xxl-shrink-0{flex-shrink:0!important}.flex-xxl-shrink-1{flex-shrink:1!important}.flex-xxl-wrap{flex-wrap:wrap!important}.flex-xxl-nowrap{flex-wrap:nowrap!important}.flex-xxl-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-xxl-start{justify-content:flex-start!important}.justify-content-xxl-end{justify-content:flex-end!important}.justify-content-xxl-center{justify-content:center!important}.justify-content-xxl-between{justify-content:space-between!important}.justify-content-xxl-around{justify-content:space-around!important}.justify-content-xxl-evenly{justify-content:space-evenly!important}.align-items-xxl-start{align-items:flex-start!important}.align-items-xxl-end{align-items:flex-end!important}.align-items-xxl-center{align-items:center!important}.align-items-xxl-baseline{align-items:baseline!important}.align-items-xxl-stretch{align-items:stretch!important}.align-content-xxl-start{align-content:flex-start!important}.align-content-xxl-end{align-content:flex-end!important}.align-content-xxl-center{align-content:center!important}.align-content-xxl-between{align-content:space-between!important}.align-content-xxl-around{align-content:space-around!important}.align-content-xxl-stretch{align-content:stretch!important}.align-self-xxl-auto{align-self:auto!important}.align-self-xxl-start{align-self:flex-start!important}.align-self-xxl-end{align-self:flex-end!important}.align-self-xxl-center{align-self:center!important}.align-self-xxl-baseline{align-self:baseline!important}.align-self-xxl-stretch{align-self:stretch!important}.order-xxl-first{order:-1!important}.order-xxl-0{order:0!important}.order-xxl-1{order:1!important}.order-xxl-2{order:2!important}.order-xxl-3{order:3!important}.order-xxl-4{order:4!important}.order-xxl-5{order:5!important}.order-xxl-last{order:6!important}.m-xxl-0{margin:0!important}.m-xxl-1{margin:.25rem!important}.m-xxl-2{margin:.5rem!important}.m-xxl-3{margin:1rem!important}.m-xxl-4{margin:1.5rem!important}.m-xxl-5{margin:3rem!important}.m-xxl-auto{margin:auto!important}.mx-xxl-0{margin-right:0!important;margin-left:0!important}.mx-xxl-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-xxl-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-xxl-3{margin-right:1rem!important;margin-left:1rem!important}.mx-xxl-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-xxl-5{margin-right:3rem!important;margin-left:3rem!important}.mx-xxl-auto{margin-right:auto!important;margin-left:auto!important}.my-xxl-0{margin-top:0!important;margin-bottom:0!important}.my-xxl-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-xxl-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-xxl-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-xxl-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-xxl-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-xxl-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-xxl-0{margin-top:0!important}.mt-xxl-1{margin-top:.25rem!important}.mt-xxl-2{margin-top:.5rem!important}.mt-xxl-3{margin-top:1rem!important}.mt-xxl-4{margin-top:1.5rem!important}.mt-xxl-5{margin-top:3rem!important}.mt-xxl-auto{margin-top:auto!important}.me-xxl-0{margin-right:0!important}.me-xxl-1{margin-right:.25rem!important}.me-xxl-2{margin-right:.5rem!important}.me-xxl-3{margin-right:1rem!important}.me-xxl-4{margin-right:1.5rem!important}.me-xxl-5{margin-right:3rem!important}.me-xxl-auto{margin-right:auto!important}.mb-xxl-0{margin-bottom:0!important}.mb-xxl-1{margin-bottom:.25rem!important}.mb-xxl-2{margin-bottom:.5rem!important}.mb-xxl-3{margin-bottom:1rem!important}.mb-xxl-4{margin-bottom:1.5rem!important}.mb-xxl-5{margin-bottom:3rem!important}.mb-xxl-auto{margin-bottom:auto!important}.ms-xxl-0{margin-left:0!important}.ms-xxl-1{margin-left:.25rem!important}.ms-xxl-2{margin-left:.5rem!important}.ms-xxl-3{margin-left:1rem!important}.ms-xxl-4{margin-left:1.5rem!important}.ms-xxl-5{margin-left:3rem!important}.ms-xxl-auto{margin-left:auto!important}.p-xxl-0{padding:0!important}.p-xxl-1{padding:.25rem!important}.p-xxl-2{padding:.5rem!important}.p-xxl-3{padding:1rem!important}.p-xxl-4{padding:1.5rem!important}.p-xxl-5{padding:3rem!important}.px-xxl-0{padding-right:0!important;padding-left:0!important}.px-xxl-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-xxl-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-xxl-3{padding-right:1rem!important;padding-left:1rem!important}.px-xxl-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-xxl-5{padding-right:3rem!important;padding-left:3rem!important}.py-xxl-0{padding-top:0!important;padding-bottom:0!important}.py-xxl-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-xxl-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-xxl-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-xxl-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-xxl-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-xxl-0{padding-top:0!important}.pt-xxl-1{padding-top:.25rem!important}.pt-xxl-2{padding-top:.5rem!important}.pt-xxl-3{padding-top:1rem!important}.pt-xxl-4{padding-top:1.5rem!important}.pt-xxl-5{padding-top:3rem!important}.pe-xxl-0{padding-right:0!important}.pe-xxl-1{padding-right:.25rem!important}.pe-xxl-2{padding-right:.5rem!important}.pe-xxl-3{padding-right:1rem!important}.pe-xxl-4{padding-right:1.5rem!important}.pe-xxl-5{padding-right:3rem!important}.pb-xxl-0{padding-bottom:0!important}.pb-xxl-1{padding-bottom:.25rem!important}.pb-xxl-2{padding-bottom:.5rem!important}.pb-xxl-3{padding-bottom:1rem!important}.pb-xxl-4{padding-bottom:1.5rem!important}.pb-xxl-5{padding-bottom:3rem!important}.ps-xxl-0{padding-left:0!important}.ps-xxl-1{padding-left:.25rem!important}.ps-xxl-2{padding-left:.5rem!important}.ps-xxl-3{padding-left:1rem!important}.ps-xxl-4{padding-left:1.5rem!important}.ps-xxl-5{padding-left:3rem!important}.gap-xxl-0{gap:0!important}.gap-xxl-1{gap:.25rem!important}.gap-xxl-2{gap:.5rem!important}.gap-xxl-3{gap:1rem!important}.gap-xxl-4{gap:1.5rem!important}.gap-xxl-5{gap:3rem!important}.row-gap-xxl-0{row-gap:0!important}.row-gap-xxl-1{row-gap:.25rem!important}.row-gap-xxl-2{row-gap:.5rem!important}.row-gap-xxl-3{row-gap:1rem!important}.row-gap-xxl-4{row-gap:1.5rem!important}.row-gap-xxl-5{row-gap:3rem!important}.column-gap-xxl-0{-moz-column-gap:0!important;column-gap:0!important}.column-gap-xxl-1{-moz-column-gap:0.25rem!important;column-gap:.25rem!important}.column-gap-xxl-2{-moz-column-gap:0.5rem!important;column-gap:.5rem!important}.column-gap-xxl-3{-moz-column-gap:1rem!important;column-gap:1rem!important}.column-gap-xxl-4{-moz-column-gap:1.5rem!important;column-gap:1.5rem!important}.column-gap-xxl-5{-moz-column-gap:3rem!important;column-gap:3rem!important}.text-xxl-start{text-align:left!important}.text-xxl-end{text-align:right!important}.text-xxl-center{text-align:center!important}}@media (min-width:1200px){.fs-1{font-size:2.5rem!important}.fs-2{font-size:2rem!important}.fs-3{font-size:1.75rem!important}.fs-4{font-size:1.5rem!important}}@media print{.d-print-inline{display:inline!important}.d-print-inline-block{display:inline-block!important}.d-print-block{display:block!important}.d-print-grid{display:grid!important}.d-print-inline-grid{display:inline-grid!important}.d-print-table{display:table!important}.d-print-table-row{display:table-row!important}.d-print-table-cell{display:table-cell!important}.d-print-flex{display:flex!important}.d-print-inline-flex{display:inline-flex!important}.d-print-none{display:none!important}}
+/*# sourceMappingURL=bootstrap.min.css.map */ \ No newline at end of file
diff --git a/interface/css/codejar-linenumbers.css b/interface/css/codejar-linenumbers.css
new file mode 100644
index 0000000..cc19f41
--- /dev/null
+++ b/interface/css/codejar-linenumbers.css
@@ -0,0 +1,17 @@
+.codejar-linenumbers-inner-wrap {
+ position: absolute;
+ top: 0px;
+ left: 0px;
+ bottom: 0px;
+ overflow: hidden;
+}
+
+.codejar-linenumbers {
+ mix-blend-mode: initial;
+ height: 100%;
+}
+
+.codejar-linenumber {
+ position: relative;
+ top: 0px;
+}
diff --git a/interface/css/d3evolution.css b/interface/css/d3evolution.css
new file mode 100644
index 0000000..beb3deb
--- /dev/null
+++ b/interface/css/d3evolution.css
@@ -0,0 +1,66 @@
+.d3evolution svg {
+ background-color: white;
+}
+.d3evolution .chart-title {
+ font-size: 17px;
+}
+.d3evolution .y.label {
+ font-size: 11px;
+ font-weight: normal;
+}
+.d3evolution .grid line {
+ stroke: lightgrey;
+ stroke-opacity: .7;
+ shape-rendering: crispEdges;
+}
+.d3evolution .grid path {
+ stroke-width: 0;
+}
+.d3evolution .axis,
+.d3evolution .legend {
+ font-size: 12px;
+}
+.d3evolution .legend .value {
+ font-size: 10px;
+}
+.d3evolution .cursor-time {
+ font-size: 10px;
+}
+.d3evolution .cursor {
+ shape-rendering: crispEdges;
+}
+.d3evolution .cursor .background {
+ stroke: white;
+}
+.d3evolution .cursor .foreground {
+ stroke-dasharray: 4, 2;
+}
+.d3evolution .cursor .x.foreground {
+ stroke: blue;
+}
+.d3evolution .cursor circle {
+ shape-rendering: geometricPrecision;
+}
+.d3evolution .cursor circle.foreground {
+ stroke-dasharray: none;
+ stroke: black;
+ opacity: .5;
+}
+/*
+path.path {
+ shape-rendering: crispEdges;
+}
+*/
+.d3evolution .axis path,
+.d3evolution .axis line {
+ fill: none;
+ stroke: grey;
+ shape-rendering: crispEdges;
+}
+.d3evolution .legend circle {
+ stroke-width: 2px;
+}
+.d3evolution .path-null {
+ fill: steelblue;
+ fill-opacity: .1;
+}
diff --git a/interface/css/d3pie.css b/interface/css/d3pie.css
new file mode 100644
index 0000000..813ce9f
--- /dev/null
+++ b/interface/css/d3pie.css
@@ -0,0 +1,49 @@
+.d3pie .chart-title {
+ font-family: Arial, sans-serif;
+ font-size: 24px;
+ text-anchor: middle;
+}
+.d3pie-tooltip {
+ pointer-events: none;
+ position: absolute;
+ padding: 5px;
+ color: white;
+ background-color: rgb(0 0 0 /50%);
+ border-radius: 4px;
+ opacity: 0;
+ line-height: normal;
+}
+
+.d3pie .total-text {
+ text-anchor: middle;
+ dominant-baseline: central;
+}
+.d3pie .total-value {
+ font-family: Arial, sans-serif;
+}
+
+.d3pie .inner-label {
+ fill: #eeeeee;
+ pointer-events: none;
+ text-anchor: middle;
+}
+
+/* pie placeholder */
+.d3pie .slice-g:first-of-type .inner-label {
+ fill: #cccccc;
+}
+.d3pie defs radialGradient:first-of-type stop {
+ stop-opacity: 0.1;
+}
+
+/* gradient color */
+.d3pie .grad-stop-1 {
+ stop-color: black;
+}
+
+.outer-label-g {
+ opacity: 0;
+}
+.link {
+ fill: none;
+}
diff --git a/interface/css/font-glyphicons.css b/interface/css/font-glyphicons.css
new file mode 100644
index 0000000..ecb237b
--- /dev/null
+++ b/interface/css/font-glyphicons.css
@@ -0,0 +1,805 @@
+/* https://gist.github.com/itchief/e3aa0888c959ee367cd69f5d83ff06f9 */
+
+@font-face {
+ font-family: "Glyphicons Halflings";
+ src: url("../fonts/glyphicons-halflings-regular.eot");
+ src: url("../fonts/glyphicons-halflings-regular.eot?#iefix") format("embedded-opentype"), url("../fonts/glyphicons-halflings-regular.woff2") format("woff2"), url("../fonts/glyphicons-halflings-regular.woff") format("woff"), url("../fonts/glyphicons-halflings-regular.ttf") format("truetype"), url("../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular") format("svg");
+}
+.glyphicon {
+ position: relative;
+ top: 1px;
+ display: inline-block;
+ font-family: "Glyphicons Halflings"; /* stylelint-disable-line font-family-no-missing-generic-family-keyword */
+ font-style: normal;
+ font-weight: 400;
+ line-height: 1;
+ -webkit-font-smoothing: antialiased;
+ -moz-osx-font-smoothing: grayscale;
+}
+.glyphicon-asterisk::before {
+ content: "\002a";
+}
+.glyphicon-plus::before {
+ content: "\002b";
+}
+.glyphicon-euro::before,
+.glyphicon-eur::before {
+ content: "\20ac";
+}
+.glyphicon-minus::before {
+ content: "\2212";
+}
+.glyphicon-cloud::before {
+ content: "\2601";
+}
+.glyphicon-envelope::before {
+ content: "\2709";
+}
+.glyphicon-pencil::before {
+ content: "\270f";
+}
+.glyphicon-glass::before {
+ content: "\e001";
+}
+.glyphicon-music::before {
+ content: "\e002";
+}
+.glyphicon-search::before {
+ content: "\e003";
+}
+.glyphicon-heart::before {
+ content: "\e005";
+}
+.glyphicon-star::before {
+ content: "\e006";
+}
+.glyphicon-star-empty::before {
+ content: "\e007";
+}
+.glyphicon-user::before {
+ content: "\e008";
+}
+.glyphicon-film::before {
+ content: "\e009";
+}
+.glyphicon-th-large::before {
+ content: "\e010";
+}
+.glyphicon-th::before {
+ content: "\e011";
+}
+.glyphicon-th-list::before {
+ content: "\e012";
+}
+.glyphicon-ok::before {
+ content: "\e013";
+}
+.glyphicon-remove::before {
+ content: "\e014";
+}
+.glyphicon-zoom-in::before {
+ content: "\e015";
+}
+.glyphicon-zoom-out::before {
+ content: "\e016";
+}
+.glyphicon-off::before {
+ content: "\e017";
+}
+.glyphicon-signal::before {
+ content: "\e018";
+}
+.glyphicon-cog::before {
+ content: "\e019";
+}
+.glyphicon-trash::before {
+ content: "\e020";
+}
+.glyphicon-home::before {
+ content: "\e021";
+}
+.glyphicon-file::before {
+ content: "\e022";
+}
+.glyphicon-time::before {
+ content: "\e023";
+}
+.glyphicon-road::before {
+ content: "\e024";
+}
+.glyphicon-download-alt::before {
+ content: "\e025";
+}
+.glyphicon-download::before {
+ content: "\e026";
+}
+.glyphicon-upload::before {
+ content: "\e027";
+}
+.glyphicon-inbox::before {
+ content: "\e028";
+}
+.glyphicon-play-circle::before {
+ content: "\e029";
+}
+.glyphicon-repeat::before {
+ content: "\e030";
+}
+.glyphicon-refresh::before {
+ content: "\e031";
+}
+.glyphicon-list-alt::before {
+ content: "\e032";
+}
+.glyphicon-lock::before {
+ content: "\e033";
+}
+.glyphicon-flag::before {
+ content: "\e034";
+}
+.glyphicon-headphones::before {
+ content: "\e035";
+}
+.glyphicon-volume-off::before {
+ content: "\e036";
+}
+.glyphicon-volume-down::before {
+ content: "\e037";
+}
+.glyphicon-volume-up::before {
+ content: "\e038";
+}
+.glyphicon-qrcode::before {
+ content: "\e039";
+}
+.glyphicon-barcode::before {
+ content: "\e040";
+}
+.glyphicon-tag::before {
+ content: "\e041";
+}
+.glyphicon-tags::before {
+ content: "\e042";
+}
+.glyphicon-book::before {
+ content: "\e043";
+}
+.glyphicon-bookmark::before {
+ content: "\e044";
+}
+.glyphicon-print::before {
+ content: "\e045";
+}
+.glyphicon-camera::before {
+ content: "\e046";
+}
+.glyphicon-font::before {
+ content: "\e047";
+}
+.glyphicon-bold::before {
+ content: "\e048";
+}
+.glyphicon-italic::before {
+ content: "\e049";
+}
+.glyphicon-text-height::before {
+ content: "\e050";
+}
+.glyphicon-text-width::before {
+ content: "\e051";
+}
+.glyphicon-align-left::before {
+ content: "\e052";
+}
+.glyphicon-align-center::before {
+ content: "\e053";
+}
+.glyphicon-align-right::before {
+ content: "\e054";
+}
+.glyphicon-align-justify::before {
+ content: "\e055";
+}
+.glyphicon-list::before {
+ content: "\e056";
+}
+.glyphicon-indent-left::before {
+ content: "\e057";
+}
+.glyphicon-indent-right::before {
+ content: "\e058";
+}
+.glyphicon-facetime-video::before {
+ content: "\e059";
+}
+.glyphicon-picture::before {
+ content: "\e060";
+}
+.glyphicon-map-marker::before {
+ content: "\e062";
+}
+.glyphicon-adjust::before {
+ content: "\e063";
+}
+.glyphicon-tint::before {
+ content: "\e064";
+}
+.glyphicon-edit::before {
+ content: "\e065";
+}
+.glyphicon-share::before {
+ content: "\e066";
+}
+.glyphicon-check::before {
+ content: "\e067";
+}
+.glyphicon-move::before {
+ content: "\e068";
+}
+.glyphicon-step-backward::before {
+ content: "\e069";
+}
+.glyphicon-fast-backward::before {
+ content: "\e070";
+}
+.glyphicon-backward::before {
+ content: "\e071";
+}
+.glyphicon-play::before {
+ content: "\e072";
+}
+.glyphicon-pause::before {
+ content: "\e073";
+}
+.glyphicon-stop::before {
+ content: "\e074";
+}
+.glyphicon-forward::before {
+ content: "\e075";
+}
+.glyphicon-fast-forward::before {
+ content: "\e076";
+}
+.glyphicon-step-forward::before {
+ content: "\e077";
+}
+.glyphicon-eject::before {
+ content: "\e078";
+}
+.glyphicon-chevron-left::before {
+ content: "\e079";
+}
+.glyphicon-chevron-right::before {
+ content: "\e080";
+}
+.glyphicon-plus-sign::before {
+ content: "\e081";
+}
+.glyphicon-minus-sign::before {
+ content: "\e082";
+}
+.glyphicon-remove-sign::before {
+ content: "\e083";
+}
+.glyphicon-ok-sign::before {
+ content: "\e084";
+}
+.glyphicon-question-sign::before {
+ content: "\e085";
+}
+.glyphicon-info-sign::before {
+ content: "\e086";
+}
+.glyphicon-screenshot::before {
+ content: "\e087";
+}
+.glyphicon-remove-circle::before {
+ content: "\e088";
+}
+.glyphicon-ok-circle::before {
+ content: "\e089";
+}
+.glyphicon-ban-circle::before {
+ content: "\e090";
+}
+.glyphicon-arrow-left::before {
+ content: "\e091";
+}
+.glyphicon-arrow-right::before {
+ content: "\e092";
+}
+.glyphicon-arrow-up::before {
+ content: "\e093";
+}
+.glyphicon-arrow-down::before {
+ content: "\e094";
+}
+.glyphicon-share-alt::before {
+ content: "\e095";
+}
+.glyphicon-resize-full::before {
+ content: "\e096";
+}
+.glyphicon-resize-small::before {
+ content: "\e097";
+}
+.glyphicon-exclamation-sign::before {
+ content: "\e101";
+}
+.glyphicon-gift::before {
+ content: "\e102";
+}
+.glyphicon-leaf::before {
+ content: "\e103";
+}
+.glyphicon-fire::before {
+ content: "\e104";
+}
+.glyphicon-eye-open::before {
+ content: "\e105";
+}
+.glyphicon-eye-close::before {
+ content: "\e106";
+}
+.glyphicon-warning-sign::before {
+ content: "\e107";
+}
+.glyphicon-plane::before {
+ content: "\e108";
+}
+.glyphicon-calendar::before {
+ content: "\e109";
+}
+.glyphicon-random::before {
+ content: "\e110";
+}
+.glyphicon-comment::before {
+ content: "\e111";
+}
+.glyphicon-magnet::before {
+ content: "\e112";
+}
+.glyphicon-chevron-up::before {
+ content: "\e113";
+}
+.glyphicon-chevron-down::before {
+ content: "\e114";
+}
+.glyphicon-retweet::before {
+ content: "\e115";
+}
+.glyphicon-shopping-cart::before {
+ content: "\e116";
+}
+.glyphicon-folder-close::before {
+ content: "\e117";
+}
+.glyphicon-folder-open::before {
+ content: "\e118";
+}
+.glyphicon-resize-vertical::before {
+ content: "\e119";
+}
+.glyphicon-resize-horizontal::before {
+ content: "\e120";
+}
+.glyphicon-hdd::before {
+ content: "\e121";
+}
+.glyphicon-bullhorn::before {
+ content: "\e122";
+}
+.glyphicon-bell::before {
+ content: "\e123";
+}
+.glyphicon-certificate::before {
+ content: "\e124";
+}
+.glyphicon-thumbs-up::before {
+ content: "\e125";
+}
+.glyphicon-thumbs-down::before {
+ content: "\e126";
+}
+.glyphicon-hand-right::before {
+ content: "\e127";
+}
+.glyphicon-hand-left::before {
+ content: "\e128";
+}
+.glyphicon-hand-up::before {
+ content: "\e129";
+}
+.glyphicon-hand-down::before {
+ content: "\e130";
+}
+.glyphicon-circle-arrow-right::before {
+ content: "\e131";
+}
+.glyphicon-circle-arrow-left::before {
+ content: "\e132";
+}
+.glyphicon-circle-arrow-up::before {
+ content: "\e133";
+}
+.glyphicon-circle-arrow-down::before {
+ content: "\e134";
+}
+.glyphicon-globe::before {
+ content: "\e135";
+}
+.glyphicon-wrench::before {
+ content: "\e136";
+}
+.glyphicon-tasks::before {
+ content: "\e137";
+}
+.glyphicon-filter::before {
+ content: "\e138";
+}
+.glyphicon-briefcase::before {
+ content: "\e139";
+}
+.glyphicon-fullscreen::before {
+ content: "\e140";
+}
+.glyphicon-dashboard::before {
+ content: "\e141";
+}
+.glyphicon-paperclip::before {
+ content: "\e142";
+}
+.glyphicon-heart-empty::before {
+ content: "\e143";
+}
+.glyphicon-link::before {
+ content: "\e144";
+}
+.glyphicon-phone::before {
+ content: "\e145";
+}
+.glyphicon-pushpin::before {
+ content: "\e146";
+}
+.glyphicon-usd::before {
+ content: "\e148";
+}
+.glyphicon-gbp::before {
+ content: "\e149";
+}
+.glyphicon-sort::before {
+ content: "\e150";
+}
+.glyphicon-sort-by-alphabet::before {
+ content: "\e151";
+}
+.glyphicon-sort-by-alphabet-alt::before {
+ content: "\e152";
+}
+.glyphicon-sort-by-order::before {
+ content: "\e153";
+}
+.glyphicon-sort-by-order-alt::before {
+ content: "\e154";
+}
+.glyphicon-sort-by-attributes::before {
+ content: "\e155";
+}
+.glyphicon-sort-by-attributes-alt::before {
+ content: "\e156";
+}
+.glyphicon-unchecked::before {
+ content: "\e157";
+}
+.glyphicon-expand::before {
+ content: "\e158";
+}
+.glyphicon-collapse-down::before {
+ content: "\e159";
+}
+.glyphicon-collapse-up::before {
+ content: "\e160";
+}
+.glyphicon-log-in::before {
+ content: "\e161";
+}
+.glyphicon-flash::before {
+ content: "\e162";
+}
+.glyphicon-log-out::before {
+ content: "\e163";
+}
+.glyphicon-new-window::before {
+ content: "\e164";
+}
+.glyphicon-record::before {
+ content: "\e165";
+}
+.glyphicon-save::before {
+ content: "\e166";
+}
+.glyphicon-open::before {
+ content: "\e167";
+}
+.glyphicon-saved::before {
+ content: "\e168";
+}
+.glyphicon-import::before {
+ content: "\e169";
+}
+.glyphicon-export::before {
+ content: "\e170";
+}
+.glyphicon-send::before {
+ content: "\e171";
+}
+.glyphicon-floppy-disk::before {
+ content: "\e172";
+}
+.glyphicon-floppy-saved::before {
+ content: "\e173";
+}
+.glyphicon-floppy-remove::before {
+ content: "\e174";
+}
+.glyphicon-floppy-save::before {
+ content: "\e175";
+}
+.glyphicon-floppy-open::before {
+ content: "\e176";
+}
+.glyphicon-credit-card::before {
+ content: "\e177";
+}
+.glyphicon-transfer::before {
+ content: "\e178";
+}
+.glyphicon-cutlery::before {
+ content: "\e179";
+}
+.glyphicon-header::before {
+ content: "\e180";
+}
+.glyphicon-compressed::before {
+ content: "\e181";
+}
+.glyphicon-earphone::before {
+ content: "\e182";
+}
+.glyphicon-phone-alt::before {
+ content: "\e183";
+}
+.glyphicon-tower::before {
+ content: "\e184";
+}
+.glyphicon-stats::before {
+ content: "\e185";
+}
+.glyphicon-sd-video::before {
+ content: "\e186";
+}
+.glyphicon-hd-video::before {
+ content: "\e187";
+}
+.glyphicon-subtitles::before {
+ content: "\e188";
+}
+.glyphicon-sound-stereo::before {
+ content: "\e189";
+}
+.glyphicon-sound-dolby::before {
+ content: "\e190";
+}
+.glyphicon-sound-5-1::before {
+ content: "\e191";
+}
+.glyphicon-sound-6-1::before {
+ content: "\e192";
+}
+.glyphicon-sound-7-1::before {
+ content: "\e193";
+}
+.glyphicon-copyright-mark::before {
+ content: "\e194";
+}
+.glyphicon-registration-mark::before {
+ content: "\e195";
+}
+.glyphicon-cloud-download::before {
+ content: "\e197";
+}
+.glyphicon-cloud-upload::before {
+ content: "\e198";
+}
+.glyphicon-tree-conifer::before {
+ content: "\e199";
+}
+.glyphicon-tree-deciduous::before {
+ content: "\e200";
+}
+.glyphicon-cd::before {
+ content: "\e201";
+}
+.glyphicon-save-file::before {
+ content: "\e202";
+}
+.glyphicon-open-file::before {
+ content: "\e203";
+}
+.glyphicon-level-up::before {
+ content: "\e204";
+}
+.glyphicon-copy::before {
+ content: "\e205";
+}
+.glyphicon-paste::before {
+ content: "\e206";
+}
+.glyphicon-alert::before {
+ content: "\e209";
+}
+.glyphicon-equalizer::before {
+ content: "\e210";
+}
+.glyphicon-king::before {
+ content: "\e211";
+}
+.glyphicon-queen::before {
+ content: "\e212";
+}
+.glyphicon-pawn::before {
+ content: "\e213";
+}
+.glyphicon-bishop::before {
+ content: "\e214";
+}
+.glyphicon-knight::before {
+ content: "\e215";
+}
+.glyphicon-baby-formula::before {
+ content: "\e216";
+}
+.glyphicon-tent::before {
+ content: "\26fa";
+}
+.glyphicon-blackboard::before {
+ content: "\e218";
+}
+.glyphicon-bed::before {
+ content: "\e219";
+}
+.glyphicon-apple::before {
+ content: "\f8ff";
+}
+.glyphicon-erase::before {
+ content: "\e221";
+}
+.glyphicon-hourglass::before {
+ content: "\231b";
+}
+.glyphicon-lamp::before {
+ content: "\e223";
+}
+.glyphicon-duplicate::before {
+ content: "\e224";
+}
+.glyphicon-piggy-bank::before {
+ content: "\e225";
+}
+.glyphicon-scissors::before {
+ content: "\e226";
+}
+.glyphicon-bitcoin::before {
+ content: "\e227";
+}
+.glyphicon-btc::before {
+ content: "\e227";
+}
+.glyphicon-xbt::before {
+ content: "\e227";
+}
+.glyphicon-yen::before {
+ content: "\00a5";
+}
+.glyphicon-jpy::before {
+ content: "\00a5";
+}
+.glyphicon-ruble::before {
+ content: "\20bd";
+}
+.glyphicon-rub::before {
+ content: "\20bd";
+}
+.glyphicon-scale::before {
+ content: "\e230";
+}
+.glyphicon-ice-lolly::before {
+ content: "\e231";
+}
+.glyphicon-ice-lolly-tasted::before {
+ content: "\e232";
+}
+.glyphicon-education::before {
+ content: "\e233";
+}
+.glyphicon-option-horizontal::before {
+ content: "\e234";
+}
+.glyphicon-option-vertical::before {
+ content: "\e235";
+}
+.glyphicon-menu-hamburger::before {
+ content: "\e236";
+}
+.glyphicon-modal-window::before {
+ content: "\e237";
+}
+.glyphicon-oil::before {
+ content: "\e238";
+}
+.glyphicon-grain::before {
+ content: "\e239";
+}
+.glyphicon-sunglasses::before {
+ content: "\e240";
+}
+.glyphicon-text-size::before {
+ content: "\e241";
+}
+.glyphicon-text-color::before {
+ content: "\e242";
+}
+.glyphicon-text-background::before {
+ content: "\e243";
+}
+.glyphicon-object-align-top::before {
+ content: "\e244";
+}
+.glyphicon-object-align-bottom::before {
+ content: "\e245";
+}
+.glyphicon-object-align-horizontal::before {
+ content: "\e246";
+}
+.glyphicon-object-align-left::before {
+ content: "\e247";
+}
+.glyphicon-object-align-vertical::before {
+ content: "\e248";
+}
+.glyphicon-object-align-right::before {
+ content: "\e249";
+}
+.glyphicon-triangle-right::before {
+ content: "\e250";
+}
+.glyphicon-triangle-left::before {
+ content: "\e251";
+}
+.glyphicon-triangle-bottom::before {
+ content: "\e252";
+}
+.glyphicon-triangle-top::before {
+ content: "\e253";
+}
+.glyphicon-console::before {
+ content: "\e254";
+}
+.glyphicon-superscript::before {
+ content: "\e255";
+}
+.glyphicon-subscript::before {
+ content: "\e256";
+}
+.glyphicon-menu-left::before {
+ content: "\e257";
+}
+.glyphicon-menu-right::before {
+ content: "\e258";
+}
+.glyphicon-menu-down::before {
+ content: "\e259";
+}
+.glyphicon-menu-up::before {
+ content: "\e260";
+}
diff --git a/interface/css/footable.standalone.min.css b/interface/css/footable.standalone.min.css
new file mode 100644
index 0000000..0ac5adf
--- /dev/null
+++ b/interface/css/footable.standalone.min.css
@@ -0,0 +1 @@
+.footable .btn,.footable .caret{display:inline-block;vertical-align:middle}.footable-details.table,.footable-details.table *,.footable.table,.footable.table *{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.footable-details.table th,.footable.table th{text-align:left}.footable-details.table,.footable.table{width:100%;max-width:100%;margin-bottom:20px}.footable-details.table>caption+thead>tr:first-child>td,.footable-details.table>caption+thead>tr:first-child>th,.footable-details.table>colgroup+thead>tr:first-child>td,.footable-details.table>colgroup+thead>tr:first-child>th,.footable-details.table>thead:first-child>tr:first-child>td,.footable-details.table>thead:first-child>tr:first-child>th,.footable.table>caption+thead>tr:first-child>td,.footable.table>caption+thead>tr:first-child>th,.footable.table>colgroup+thead>tr:first-child>td,.footable.table>colgroup+thead>tr:first-child>th,.footable.table>thead:first-child>tr:first-child>td,.footable.table>thead:first-child>tr:first-child>th{border-top:0}.footable-details.table>tbody>tr>td,.footable-details.table>tbody>tr>th,.footable-details.table>tfoot>tr>td,.footable-details.table>tfoot>tr>th,.footable-details.table>thead>tr>td,.footable-details.table>thead>tr>th,.footable.table>tbody>tr>td,.footable.table>tbody>tr>th,.footable.table>tfoot>tr>td,.footable.table>tfoot>tr>th,.footable.table>thead>tr>td,.footable.table>thead>tr>th{padding:8px;line-height:1.42857143;vertical-align:top;border-top:1px solid #ddd}.footable-details.table>thead>tr>td,.footable-details.table>thead>tr>th,.footable.table>thead>tr>td,.footable.table>thead>tr>th{vertical-align:bottom;border-bottom:2px solid #ddd}.footable-details.table-condensed>tbody>tr>td,.footable-details.table-condensed>tbody>tr>th,.footable-details.table-condensed>tfoot>tr>td,.footable-details.table-condensed>tfoot>tr>th,.footable-details.table-condensed>thead>tr>td,.footable-details.table-condensed>thead>tr>th,.footable.table-condensed>tbody>tr>td,.footable.table-condensed>tbody>tr>th,.footable.table-condensed>tfoot>tr>td,.footable.table-condensed>tfoot>tr>th,.footable.table-condensed>thead>tr>td,.footable.table-condensed>thead>tr>th{padding:5px}.footable-details.table-bordered,.footable-details.table-bordered>tbody>tr>td,.footable-details.table-bordered>tbody>tr>th,.footable-details.table-bordered>tfoot>tr>td,.footable-details.table-bordered>tfoot>tr>th,.footable-details.table-bordered>thead>tr>td,.footable-details.table-bordered>thead>tr>th,.footable.table-bordered,.footable.table-bordered>tbody>tr>td,.footable.table-bordered>tbody>tr>th,.footable.table-bordered>tfoot>tr>td,.footable.table-bordered>tfoot>tr>th,.footable.table-bordered>thead>tr>td,.footable.table-bordered>thead>tr>th{border:1px solid #ddd}.footable-details.table-bordered>thead>tr>td,.footable-details.table-bordered>thead>tr>th,.footable.table-bordered>thead>tr>td,.footable.table-bordered>thead>tr>th{border-bottom-width:2px}.footable-details.table-striped>tbody>tr:nth-child(odd),.footable.table-striped>tbody>tr:nth-child(odd){background-color:#f9f9f9}.footable-details.table-hover>tbody>tr:hover,.footable.table-hover>tbody>tr:hover{background-color:#f5f5f5}.footable .btn{padding:6px 12px;margin-bottom:0;font-size:14px;font-weight:400;line-height:1.42857143;text-align:center;white-space:nowrap;-ms-touch-action:manipulation;touch-action:manipulation;cursor:pointer;-webkit-appearance:button;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-image:none;border:1px solid transparent;border-radius:4px;overflow:visible;text-transform:none}.footable .btn.focus,.footable .btn:focus,.footable .btn:hover{color:#333;text-decoration:none}.footable .btn-default{color:#333;background-color:#fff;border-color:#ccc}.footable .btn-default.active,.footable .btn-default.focus,.footable .btn-default:active,.footable .btn-default:focus,.footable .btn-default:hover,.footable .open>.dropdown-toggle.btn-default{color:#333;background-color:#e6e6e6;border-color:#adadad}.footable .btn-primary{color:#fff;background-color:#337ab7;border-color:#2e6da4}.footable .btn-primary.active,.footable .btn-primary.focus,.footable .btn-primary:active,.footable .btn-primary:focus,.footable .btn-primary:hover,.footable .open>.dropdown-toggle.btn-primary{color:#fff;background-color:#286090;border-color:#204d74}.footable .btn-group,.footable .btn-group-vertical{position:relative;display:inline-block;vertical-align:middle}.footable .btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.footable .btn-group>.btn:first-child{margin-left:0}.footable .btn-group-vertical>.btn,.footable .btn-group>.btn{position:relative;float:left}.footable .btn-group-xs>.btn,.footable .btn-xs{padding:1px 5px;font-size:12px;line-height:1.5;border-radius:3px}.footable .btn-group-sm>.btn,.footable .btn-sm{padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}.footable .btn-group-lg>.btn,.footable .btn-lg{padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}.footable .caret{width:0;height:0;margin-left:2px;border-top:4px solid;border-right:4px solid transparent;border-left:4px solid transparent}.footable .btn .caret{margin-left:0}.form-group{margin-bottom:15px}.footable .form-control{display:block;width:100%;height:34px;padding:6px 12px;margin:0;font-family:inherit;font-size:14px;line-height:1.42857143;color:#555;background-color:#fff;background-image:none;border:1px solid #ccc;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075);-webkit-transition:border-color ease-in-out .15s,-webkit-box-shadow ease-in-out .15s;-o-transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s}.footable .input-group{position:relative;display:table;border-collapse:separate}.footable .input-group .form-control{position:relative;z-index:2;float:left;width:100%;margin-bottom:0}.footable .input-group-btn{position:relative;font-size:0}.footable .input-group-addon,.footable .input-group-btn{width:1%;white-space:nowrap;vertical-align:middle}.footable .input-group .form-control,.footable .input-group-addon,.footable .input-group-btn{display:table-cell}.footable .input-group-btn:last-child>.btn,.footable .input-group-btn:last-child>.btn-group,.footable .input-group-btn>.btn+.btn{margin-left:-1px}.footable .input-group-btn>.btn{position:relative}.footable .input-group-btn>.btn:active,.footable .input-group-btn>.btn:focus,.footable .input-group-btn>.btn:hover{z-index:2}.footable .input-group .form-control:first-child,.footable .input-group-addon:first-child,.footable .input-group-btn:first-child>.btn,.footable .input-group-btn:first-child>.btn-group>.btn,.footable .input-group-btn:first-child>.dropdown-toggle,.footable .input-group-btn:last-child>.btn-group:not(:last-child)>.btn,.footable .input-group-btn:last-child>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.footable .input-group .form-control:last-child,.footable .input-group-addon:last-child,.footable .input-group-btn:first-child>.btn-group:not(:first-child)>.btn,.footable .input-group-btn:first-child>.btn:not(:first-child),.footable .input-group-btn:last-child>.btn,.footable .input-group-btn:last-child>.btn-group>.btn,.footable .input-group-btn:last-child>.dropdown-toggle{border-top-left-radius:0;border-bottom-left-radius:0}.footable .checkbox,.footable .radio{position:relative;display:block;margin-top:10px;margin-bottom:10px}.footable .checkbox label,.footable .radio label{max-width:100%;min-height:20px;padding-left:20px;margin-bottom:0;font-weight:400;cursor:pointer}.footable .checkbox input[type=checkbox],.footable .checkbox-inline input[type=checkbox],.footable .radio input[type=radio],.footable .radio-inline input[type=radio]{position:absolute;margin:4px 0 0 -20px;line-height:normal}.footable .dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;font-size:14px;text-align:left;list-style:none;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid #ccc;border:1px solid rgba(0,0,0,.15);border-radius:4px;-webkit-box-shadow:0 6px 12px rgba(0,0,0,.175);box-shadow:0 6px 12px rgba(0,0,0,.175)}.footable .open>.dropdown-menu{display:block}.footable .dropdown-menu-right{right:0;left:auto}.footable .dropdown-menu>li>a{display:block;padding:3px 20px;clear:both;font-weight:400;line-height:1.42857143;color:#333;white-space:nowrap}.footable .dropdown-menu>li>a:focus,.footable .dropdown-menu>li>a:hover{color:#262626;text-decoration:none;background-color:#f5f5f5}.footable .pagination{display:inline-block;padding-left:0;margin:20px 0;border-radius:4px}.footable .pagination>li{display:inline}.footable .pagination>li:first-child>a,.footable .pagination>li:first-child>span{margin-left:0;border-top-left-radius:4px;border-bottom-left-radius:4px}.footable .pagination>li>a,.footable .pagination>li>span{position:relative;float:left;padding:6px 12px;margin-left:-1px;line-height:1.42857143;color:#337ab7;text-decoration:none;background-color:#fff;border:1px solid #ddd}.footable .pagination>li>a:focus,.footable .pagination>li>a:hover,.footable .pagination>li>span:focus,.footable .pagination>li>span:hover{color:#23527c;background-color:#eee;border-color:#ddd}.footable .pagination>.active>a,.footable .pagination>.active>a:focus,.footable .pagination>.active>a:hover,.footable .pagination>.active>span,.footable .pagination>.active>span:focus,.footable .pagination>.active>span:hover{z-index:2;color:#fff;cursor:default;background-color:#337ab7;border-color:#337ab7}.footable .pagination>.disabled>a,.footable .pagination>.disabled>a:focus,.footable .pagination>.disabled>a:hover,.footable .pagination>.disabled>span,.footable .pagination>.disabled>span:focus,.footable .pagination>.disabled>span:hover{color:#777;cursor:not-allowed;background-color:#fff;border-color:#ddd}.footable .label{display:inline;padding:.2em .6em .3em;font-size:75%;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25em}.footable .label-default{background-color:#777}.footable-loader.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #e3e3e3;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.05);box-shadow:inset 0 1px 1px rgba(0,0,0,.05)}.footable .sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);border:0}table.footable-details,table.footable>thead>tr.footable-filtering>th div.form-group{margin-bottom:0}@media (min-width:768px){.footable .form-inline .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.footable .form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.footable .form-inline .input-group{display:inline-table;vertical-align:middle}.footable .form-inline .input-group .form-control,.footable .form-inline .input-group .input-group-addon,.footable .form-inline .input-group .input-group-btn{width:auto}.footable .form-inline .input-group>.form-control{width:100%}}table.footable,table.footable-details{position:relative;width:100%;border-spacing:0;border-collapse:collapse}table.footable-hide-fouc{display:none}table>tbody>tr>td>span.footable-toggle{margin-right:8px;opacity:.3}table>tbody>tr>td>span.footable-toggle.last-column{margin-left:8px;float:right}table.table-condensed>tbody>tr>td>span.footable-toggle{margin-right:5px}table.footable-details>tbody>tr>th:nth-child(1){min-width:40px;width:120px}table.footable-details>tbody>tr>td:nth-child(2){word-break:break-all}table.footable-details>tbody>tr:first-child>td,table.footable-details>tbody>tr:first-child>th,table.footable-details>tfoot>tr:first-child>td,table.footable-details>tfoot>tr:first-child>th,table.footable-details>thead>tr:first-child>td,table.footable-details>thead>tr:first-child>th{border-top-width:0}table.footable-details.table-bordered>tbody>tr:first-child>td,table.footable-details.table-bordered>tbody>tr:first-child>th,table.footable-details.table-bordered>tfoot>tr:first-child>td,table.footable-details.table-bordered>tfoot>tr:first-child>th,table.footable-details.table-bordered>thead>tr:first-child>td,table.footable-details.table-bordered>thead>tr:first-child>th{border-top-width:1px}div.footable-loader{vertical-align:middle;text-align:center;height:300px;position:relative}div.footable-loader>span.fooicon{display:inline-block;opacity:.3;font-size:30px;line-height:32px;width:32px;height:32px;margin-top:-16px;margin-left:-16px;position:absolute;top:50%;left:50%;-webkit-animation:fooicon-spin-r 2s infinite linear;animation:fooicon-spin-r 2s infinite linear}table.footable>tbody>tr.footable-empty>td{vertical-align:middle;text-align:center;font-size:30px}table.footable>tbody>tr>td,table.footable>tbody>tr>th{display:none}table.footable>tbody>tr.footable-detail-row>td,table.footable>tbody>tr.footable-detail-row>th,table.footable>tbody>tr.footable-empty>td,table.footable>tbody>tr.footable-empty>th{display:table-cell}@-webkit-keyframes fooicon-spin-r{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fooicon-spin-r{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fooicon{display:inline-block;font-size:inherit;font-family:FontAwesome!important;font-style:normal;font-weight:400;line-height:1;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;transform:translate(0,0)}.fooicon:after,.fooicon:before{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.fooicon-loader:before{content:"\f01e"}.fooicon-plus:before{content:"\f067"}.fooicon-minus:before{content:"\f068"}.fooicon-search:before{content:"\f002"}.fooicon-remove:before{content:"\f00d"}.fooicon-sort:before{content:"\f0dc"}.fooicon-sort-asc:before{content:"\f160"}.fooicon-sort-desc:before{content:"\f161"}.fooicon-pencil:before{content:"\f040"}.fooicon-trash:before{content:"\f1f8"}.fooicon-eye-close:before{content:"\f070"}.fooicon-flash:before{content:"\f0e7"}.fooicon-cog:before{content:"\f013"}.fooicon-stats:before{content:"\f080"}table.footable>thead>tr.footable-filtering>th{border-bottom-width:1px;font-weight:400}.footable-filtering-external.footable-filtering-right,table.footable.footable-filtering-right>thead>tr.footable-filtering>th,table.footable>thead>tr.footable-filtering>th{text-align:right}.footable-filtering-external.footable-filtering-left,table.footable.footable-filtering-left>thead>tr.footable-filtering>th{text-align:left}.footable-filtering-external.footable-filtering-center,.footable-paging-external.footable-paging-center,table.footable-paging-center>tfoot>tr.footable-paging>td,table.footable.footable-filtering-center>thead>tr.footable-filtering>th,table.footable>tfoot>tr.footable-paging>td{text-align:center}table.footable>thead>tr.footable-filtering>th div.form-group+div.form-group{margin-top:5px}table.footable>thead>tr.footable-filtering>th div.input-group{width:100%}.footable-filtering-external ul.dropdown-menu>li>a.checkbox,table.footable>thead>tr.footable-filtering>th ul.dropdown-menu>li>a.checkbox{margin:0;display:block;position:relative}.footable-filtering-external ul.dropdown-menu>li>a.checkbox>label,table.footable>thead>tr.footable-filtering>th ul.dropdown-menu>li>a.checkbox>label{display:block;padding-left:20px}.footable-filtering-external ul.dropdown-menu>li>a.checkbox input[type=checkbox],table.footable>thead>tr.footable-filtering>th ul.dropdown-menu>li>a.checkbox input[type=checkbox]{position:absolute;margin-left:-20px}@media (min-width:768px){table.footable>thead>tr.footable-filtering>th div.input-group{width:auto}table.footable>thead>tr.footable-filtering>th div.form-group{margin-left:2px;margin-right:2px}table.footable>thead>tr.footable-filtering>th div.form-group+div.form-group{margin-top:0}}table.footable>tbody>tr>td.footable-sortable,table.footable>tbody>tr>th.footable-sortable,table.footable>tfoot>tr>td.footable-sortable,table.footable>tfoot>tr>th.footable-sortable,table.footable>thead>tr>td.footable-sortable,table.footable>thead>tr>th.footable-sortable{position:relative;padding-right:30px;cursor:pointer}td.footable-sortable>span.fooicon,th.footable-sortable>span.fooicon{position:absolute;right:6px;top:50%;margin-top:-7px;opacity:0;transition:opacity .3s ease-in}td.footable-sortable.footable-asc>span.fooicon,td.footable-sortable.footable-desc>span.fooicon,td.footable-sortable:hover>span.fooicon,th.footable-sortable.footable-asc>span.fooicon,th.footable-sortable.footable-desc>span.fooicon,th.footable-sortable:hover>span.fooicon{opacity:1}table.footable-sorting-disabled td.footable-sortable.footable-asc>span.fooicon,table.footable-sorting-disabled td.footable-sortable.footable-desc>span.fooicon,table.footable-sorting-disabled td.footable-sortable:hover>span.fooicon,table.footable-sorting-disabled th.footable-sortable.footable-asc>span.fooicon,table.footable-sorting-disabled th.footable-sortable.footable-desc>span.fooicon,table.footable-sorting-disabled th.footable-sortable:hover>span.fooicon{opacity:0;visibility:hidden}.footable-paging-external ul.pagination,table.footable>tfoot>tr.footable-paging>td>ul.pagination{margin:10px 0 0}.footable-paging-external span.label,table.footable>tfoot>tr.footable-paging>td>span.label{display:inline-block;margin:0 0 10px;padding:4px 10px}.footable-paging-external.footable-paging-left,table.footable-paging-left>tfoot>tr.footable-paging>td{text-align:left}.footable-paging-external.footable-paging-right,table.footable-editing-right td.footable-editing,table.footable-editing-right tr.footable-editing,table.footable-paging-right>tfoot>tr.footable-paging>td{text-align:right}ul.pagination>li.footable-page{display:none}ul.pagination>li.footable-page.visible{display:inline}td.footable-editing{width:90px;max-width:90px}table.footable-editing-no-delete td.footable-editing,table.footable-editing-no-edit td.footable-editing,table.footable-editing-no-view td.footable-editing{width:70px;max-width:70px}table.footable-editing-no-delete.footable-editing-no-view td.footable-editing,table.footable-editing-no-edit.footable-editing-no-delete td.footable-editing,table.footable-editing-no-edit.footable-editing-no-view td.footable-editing{width:50px;max-width:50px}table.footable-editing-no-edit.footable-editing-no-delete.footable-editing-no-view td.footable-editing,table.footable-editing-no-edit.footable-editing-no-delete.footable-editing-no-view th.footable-editing{width:0;max-width:0;display:none!important}table.footable-editing-left td.footable-editing,table.footable-editing-left tr.footable-editing{text-align:left}table.footable-editing button.footable-add,table.footable-editing button.footable-hide,table.footable-editing-show button.footable-show,table.footable-editing.footable-editing-always-show button.footable-hide,table.footable-editing.footable-editing-always-show button.footable-show,table.footable-editing.footable-editing-always-show.footable-editing-no-add tr.footable-editing{display:none}table.footable-editing.footable-editing-always-show button.footable-add,table.footable-editing.footable-editing-show button.footable-add,table.footable-editing.footable-editing-show button.footable-hide{display:inline-block} \ No newline at end of file
diff --git a/interface/css/nprogress.css b/interface/css/nprogress.css
new file mode 100644
index 0000000..6752d7f
--- /dev/null
+++ b/interface/css/nprogress.css
@@ -0,0 +1,74 @@
+/* Make clicks pass-through */
+#nprogress {
+ pointer-events: none;
+}
+
+#nprogress .bar {
+ background: #29d;
+
+ position: fixed;
+ z-index: 1031;
+ top: 0;
+ left: 0;
+
+ width: 100%;
+ height: 2px;
+}
+
+/* Fancy blur effect */
+#nprogress .peg {
+ display: block;
+ position: absolute;
+ right: 0px;
+ width: 100px;
+ height: 100%;
+ box-shadow: 0 0 10px #29d, 0 0 5px #29d;
+ opacity: 1.0;
+
+ -webkit-transform: rotate(3deg) translate(0px, -4px);
+ -ms-transform: rotate(3deg) translate(0px, -4px);
+ transform: rotate(3deg) translate(0px, -4px);
+}
+
+/* Remove these to get rid of the spinner */
+#nprogress .spinner {
+ display: block;
+ position: fixed;
+ z-index: 1031;
+ top: 15px;
+ right: 15px;
+}
+
+#nprogress .spinner-icon {
+ width: 18px;
+ height: 18px;
+ box-sizing: border-box;
+
+ border: solid 2px transparent;
+ border-top-color: #29d;
+ border-left-color: #29d;
+ border-radius: 50%;
+
+ -webkit-animation: nprogress-spinner 400ms linear infinite;
+ animation: nprogress-spinner 400ms linear infinite;
+}
+
+.nprogress-custom-parent {
+ overflow: hidden;
+ position: relative;
+}
+
+.nprogress-custom-parent #nprogress .spinner,
+.nprogress-custom-parent #nprogress .bar {
+ position: absolute;
+}
+
+@-webkit-keyframes nprogress-spinner {
+ 0% { -webkit-transform: rotate(0deg); }
+ 100% { -webkit-transform: rotate(360deg); }
+}
+@keyframes nprogress-spinner {
+ 0% { transform: rotate(0deg); }
+ 100% { transform: rotate(360deg); }
+}
+
diff --git a/interface/css/prism.css b/interface/css/prism.css
new file mode 100644
index 0000000..b806939
--- /dev/null
+++ b/interface/css/prism.css
@@ -0,0 +1,4 @@
+/* PrismJS 1.29.0
+https://prismjs.com/download.html#themes=prism-okaidia&languages=clike&plugins=show-invisibles */
+code[class*=language-],pre[class*=language-]{color:#f8f8f2;background:0 0;text-shadow:0 1px rgba(0,0,0,.3);font-family:Consolas,Monaco,'Andale Mono','Ubuntu Mono',monospace;font-size:1em;text-align:left;white-space:pre;word-spacing:normal;word-break:normal;word-wrap:normal;line-height:1.5;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-hyphens:none;-moz-hyphens:none;-ms-hyphens:none;hyphens:none}pre[class*=language-]{padding:1em;margin:.5em 0;overflow:auto;border-radius:.3em}:not(pre)>code[class*=language-],pre[class*=language-]{background:#272822}:not(pre)>code[class*=language-]{padding:.1em;border-radius:.3em;white-space:normal}.token.cdata,.token.comment,.token.doctype,.token.prolog{color:#8292a2}.token.punctuation{color:#f8f8f2}.token.namespace{opacity:.7}.token.constant,.token.deleted,.token.property,.token.symbol,.token.tag{color:#f92672}.token.boolean,.token.number{color:#ae81ff}.token.attr-name,.token.builtin,.token.char,.token.inserted,.token.selector,.token.string{color:#a6e22e}.language-css .token.string,.style .token.string,.token.entity,.token.operator,.token.url,.token.variable{color:#f8f8f2}.token.atrule,.token.attr-value,.token.class-name,.token.function{color:#e6db74}.token.keyword{color:#66d9ef}.token.important,.token.regex{color:#fd971f}.token.bold,.token.important{font-weight:700}.token.italic{font-style:italic}.token.entity{cursor:help}
+.token.cr,.token.lf,.token.space,.token.tab:not(:empty){position:relative}.token.cr:before,.token.lf:before,.token.space:before,.token.tab:not(:empty):before{color:grey;opacity:.6;position:absolute}.token.tab:not(:empty):before{content:'\21E5'}.token.cr:before{content:'\240D'}.token.crlf:before{content:'\240D\240A'}.token.lf:before{content:'\240A'}.token.space:before{content:'\00B7'}
diff --git a/interface/css/rspamd.css b/interface/css/rspamd.css
new file mode 100644
index 0000000..566e1db
--- /dev/null
+++ b/interface/css/rspamd.css
@@ -0,0 +1,598 @@
+/*
+The MIT License (MIT)
+
+Copyright (C) 2012-2013 Anton Simonov <untone@gmail.com>
+Copyright (C) 2014-2015 Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/* stylelint-disable selector-id-pattern */
+
+:root {
+ font-size: 14px;
+
+ /* Tweak bootstrap 5 colors for better accessibility */
+ --bs-danger-rgb: 221, 0, 0;
+ --bs-success-rgb: 40, 139, 69;
+}
+
+/* bootstrap 4 overrides */
+body {
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+code {
+ font-size: 90%;
+}
+small,
+.small {
+ font-size: 85%;
+}
+.text-secondary {
+ color: #666 !important;
+}
+.navbar {
+ padding-top: 0;
+ padding-bottom: 0;
+ margin-bottom: 20px;
+ border-bottom: 1px solid rgb(231 231 231);
+}
+.nav-pills .nav-link.active {
+ background-color: #e7e7e7;
+}
+.danger > td {
+ background-color: #fbe9e5;
+}
+.success > td {
+ background-color: #eef9e7;
+}
+td.warning {
+ background-color: #fff8e6;
+}
+@media (max-width: 1199px) {
+ .navbar-collapse.order-3 {
+ border-top: 1px solid #dee2e6 !important;
+ }
+ /* Avoid navbar toggler hiding on navbar collapse */
+ .navbar-toggler {
+ display: block !important;
+ }
+}
+
+/* Tweak FooTable for Bootstrap 5 */
+.footable .btn,
+.footable .form-control {
+ border-radius: 0.375rem;
+}
+
+/* bootstrap 4 additionals */
+.btn-group-xs > .btn,
+.btn-xs {
+ padding: .06rem .3rem;
+ font-size: .875rem;
+ line-height: 1.5;
+ border-radius: .2rem;
+}
+.btn.disabled,
+.btn[disabled],
+fieldset[disabled] .btn {
+ pointer-events: auto;
+ cursor: not-allowed;
+}
+.w-1 {
+ width: 1%;
+}
+
+a {
+ outline: none;
+}
+textarea {
+ font-family: "Courier New", Courier, monospace;
+ resize: vertical;
+}
+
+/* Tweak FooTable for Bootstrap 4 */
+.footable .dropdown-toggle::after {
+ content: none;
+}
+.footable .btn-outline-secondary {
+ border-color: rgb(108 117 125);
+}
+.footable .btn-group > .btn:not(:first-child),
+.footable .btn-group > .btn-group:not(:first-child) > .btn {
+ border-top-left-radius: 0;
+ border-bottom-left-radius: 0;
+}
+.footable .input-sm {
+ height: 30px;
+}
+
+.footable-header .fooicon {
+ font-size: 12px;
+}
+.footable .pagination > li:last-child > a {
+ border-top-right-radius:4px;
+ border-bottom-right-radius:4px
+}
+
+/* local overrides */
+.navbar-brand > img {
+ height: 50px;
+}
+.btn-group > .btn.radius-right {
+ border-top-right-radius: .25rem !important;
+ border-bottom-right-radius: .25rem !important;
+}
+
+input.form-control[type="number"] {
+ width: 4em;
+ padding-left: 0;
+ padding-right: 0;
+ text-align: center;
+}
+input.action-scores {
+ margin: 5px -7em 5px 0;
+}
+table#symbolsTable input[type="number"] {
+ width: 6em;
+ font-size: 11px;
+}
+
+.notification-area {
+ position: fixed;
+ z-index: 1050;
+ top: 44px;
+ left: 0;
+ right: 0;
+ padding: 8px;
+}
+.alert {
+ margin-bottom: 4px;
+}
+.alert.alert-modal {
+ top: 0;
+}
+.alert strong {
+ display: inline-block;
+ padding-left: 35px;
+}
+.alert,
+.alert h4 {
+ color: #c09853;
+}
+.alert h4 {
+ margin: 0;
+}
+.alert-success {
+ color: #468847;
+ background: #dff0d8;
+ border-color: #d6e9c6;
+}
+.alert-success h4 {
+ color: #468847;
+}
+.alert-danger,
+.alert-error {
+ color: #b94a48;
+ background: #f2dede;
+ border-color: #eed3d7;
+}
+.alert-danger h4,
+.alert-error h4 {
+ color: #b94a48;
+}
+.alert-info {
+ color: #3a87ad;
+ background: #d9edf7;
+ border-color: #bce8f1;
+}
+.alert-info h4 {
+ color: #3a87ad;
+}
+
+#authInvalidCharFeedback,
+#authUnauthorizedFeedback {
+ position: unset;
+ padding-top: 0.1rem;
+ padding-bottom: 0.1rem;
+}
+
+/* widget */
+.card-header,
+.modal-header {
+ background-color: #f3f3f3;
+ background-image: linear-gradient(to bottom, #fdfdfd, #eaeaea);
+}
+.card-header > .icon > svg {
+ vertical-align: middle;
+}
+.card-header .h6 {
+ font-size: 0.857rem;
+}
+
+.stat-box {
+ background-color: #f3f3f3;
+ background-image: linear-gradient(to bottom, #f9f9f9, #ededed);
+ line-height: 1;
+}
+.stat-box:not(.float-end) {
+ min-width: 90px;
+}
+.stat-box .widget {
+ font-size: 10px;
+}
+.stat-box .widget strong {
+ font-size: 26px;
+}
+
+/* Symbols coloring */
+.symbol-default {
+ border-radius: 2px;
+ padding-left: 2px;
+ padding-right: 2px;
+}
+.symbol-default:hover {
+ background-color: #e6e6e6;
+}
+.symbol-negative.symbol-negative {
+ background-color: #eef9e7;
+}
+.symbol-positive.symbol-positive {
+ background-color: #fbe9e5;
+}
+.symbol-special {
+ background-color: #e2e9fe;
+}
+.symbol-negative:hover {
+ background-color: #dcf9d3;
+}
+.symbol-positive:hover {
+ background-color: #fbd6d1;
+}
+.symbol-special:hover {
+ background-color: #cddbff;
+}
+
+.map-link {
+ display: block;
+ color: #0088cc;
+ cursor: pointer;
+}
+.map-link:hover,
+.map-link:focus {
+ color: #005580;
+ text-decoration: underline;
+}
+
+/* Font Awesome icons size */
+.svg-inline--fa { /* stylelint-disable-line selector-class-pattern */
+ font-size: 16px;
+}
+/* Increase refresh button spinner speed */
+#refresh .fa-spin {
+ -webkit-animation: fa-spin 1s linear infinite;
+ animation: fa-spin 1s linear infinite;
+}
+
+/* Some spacing tweaks */
+.notification-area div > button:not(.close) {
+ margin-right: 9px;
+}
+
+.status-table td:last-child {
+ border-right: none;
+}
+.status-table tr:last-child td {
+ border-bottom: none;
+}
+.status-table thead th:last-child {
+ border-right: none;
+}
+.footable-header,
+.footable tr:not(.footable-detail-row) > td {
+ font-size: 11px;
+}
+.status-table tr:last-child td:first-child {
+ border-radius: 0 0 0 calc(.25rem - 1px);
+}
+.status-table tr:last-child td:last-child {
+ border-radius: 0 0 calc(.25rem - 1px) 0;
+}
+
+/* RRD summary */
+#summary-row {
+ padding-left: 80px;
+ padding-right: 80px;
+}
+.col-fixed,
+.col-fluid {
+ position: relative;
+ float: left;
+}
+.col-fixed {
+ width: 200px;
+ min-height: 1px; /* make an empty div take space */
+}
+.col-fluid {
+ width: calc(100% - 200px);
+}
+#rrd-table_toggle {
+ position: absolute;
+ top: 0;
+ height: 100%;
+ width: 100%;
+}
+#rrd-table {
+ margin-bottom: 2px;
+ width: 100% !important;
+ text-align: left;
+ font-size: 12px;
+ z-index: 100;
+}
+#rrd-table td {
+ color: inherit;
+ padding-top: 2px;
+ padding-bottom: 2px;
+}
+#rrd-total {
+ padding-left: 8px;
+ margin-bottom: 10px;
+ text-align: left;
+ font-size: 12px;
+}
+
+/* Throughput graph controls */
+#graph_controls select {
+ margin: 10px 20px 0;
+ display: inline-block;
+ width: auto;
+ border: 1px solid grey;
+}
+
+/* history table */
+.footable-details.table {
+ margin-bottom: 0;
+}
+#historyTable_scan > tbody > tr > td,
+#historyTable_scan > thead > tr > th,
+#historyTable_history > tbody > tr > td,
+#historyTable_history > thead > tr > th {
+ padding: 4px;
+}
+#historyTable_scan > thead > tr > th,
+#historyTable_history > thead > tr > th {
+ padding-right: 20px;
+}
+@media (min-width: 576px) and (max-width: 1199px) {
+ .history-col-time {
+ /* Avoid taking multiple lines in every row when one of rows has long ID */
+ white-space: nowrap;
+ }
+}
+.footable-filtering-search .dropdown-menu .sym-order-toggle {
+ display: none;
+}
+
+#history_page_size {
+ width: 6em !important;
+ text-align: center;
+}
+
+.scorebar-spam {
+ background-color: rgba(240 0 0 / 0.1) !important;
+}
+.scorebar-ham {
+ background: rgba(100 230 80 / 0.1) !important;
+}
+
+.danger .icon {
+ color: #b94a48;
+}
+.success .icon {
+ color: #468847;
+}
+
+#learnServers {
+ display: flex;
+}
+
+#nprogress .bar {
+ height: 1px;
+}
+
+@media (min-width: 992px) {
+ #selectors > .card {
+ height: calc(100vh - 96px);
+ }
+ #row-main {
+ /* necessary to hide collapsed sidebar */
+ overflow-x: hidden;
+ }
+ #content > div {
+ display: flex;
+ }
+}
+#content {
+ transition: all 0.3s ease;
+ transition-property: flex-basis, max-width, width;
+}
+
+.sidebar {
+ padding: 8px;
+ background-color: #ffe;
+ transition: margin 0.3s ease;
+}
+.collapsed {
+ /* hide it for small displays */
+ display: none;
+}
+@media (min-width: 992px) {
+ .collapsed {
+ display: block;
+ }
+ #sidebar-left.collapsed {
+ /* same width as sidebar */
+ margin-left: -25%;
+ }
+ #sidebar-right.collapsed {
+ /* same width as sidebar */
+ margin-right: -25%;
+ }
+}
+
+#selectors > .card > .card-body {
+ min-height: 0;
+}
+
+.sidebar-nav {
+ width: 20px;
+}
+.sidebar-nav .nav-link,
+.sidebar-nav .nav-link:hover {
+ border: 1px solid #ddd;
+}
+#sidebar-tab-left > a,
+#sidebar-tab-right > a {
+ background-color: #ffe;
+ margin-left: 12px;
+ margin-right: 12px;
+}
+#sidebar-tab-left {
+ transform: rotate(180deg);
+}
+#sidebar-tab-text-left {
+ transform: rotate(180deg);
+}
+@media (min-width: 992px) {
+ #sidebar-left {
+ border-bottom-left-radius: 3.5px;
+ }
+ #sidebar-right {
+ border-bottom-right-radius: 3.5px;
+ }
+ .sidebar-nav {
+ padding-right: 0;
+ display: block;
+ }
+ #content {
+ border-left: 1px solid #ddd;
+ border-right: 1px solid #ddd;
+ }
+ #sidebar-tab-left {
+ display: flex;
+ transform: translateX(-50%) rotate(90deg) translate(50%, -50%);
+ }
+ #sidebar-tab-right {
+ float: right;
+ transform: translateX(50%) rotate(-90deg) translate(-50%, -50%);
+ }
+}
+@media (max-width: 991.98px) {
+ #sidebar-right {
+ border-bottom-left-radius: 3.5px;
+ border-bottom-right-radius: 3.5px;
+ }
+ #content {
+ border-top: 1px solid #ddd;
+ border-bottom: 1px solid #ddd;
+ }
+ #sidebar-tab-right {
+ bottom: 0;
+ right: 0;
+ }
+}
+
+#navBar .navbar-nav .nav-link {
+ padding: 15px;
+}
+
+#modalDialog > .modal-dialog {
+ /* Center the modal vertically */
+ top: 50%;
+ transform: translate(0, -50%);
+ -webkit-transform: translate(0, -50%);
+}
+
+.codejar-wrap,
+#editor.map-textarea {
+ border-radius: 6px;
+ max-height: calc(100vh - 178px);
+ max-width: 100%;
+ min-width: 100%;
+ overflow: auto;
+ resize: both;
+}
+#editor.map-textarea {
+ height: calc(100vh - 178px);
+ width: calc(100vw - 36px - 3rem);
+}
+.codejar-wrap,
+#editor.map-textarea,
+#editor.map-textarea:focus {
+ background: rgb(0 47 79);
+ color: silver;
+}
+.codejar-wrap {
+ /* Fix line wrapping */
+ scrollbar-width: thin;
+}
+.codejar-linenumbers-inner-wrap {
+ bottom: unset;
+}
+.codejar-linenumbers {
+ background: rgba(255 255 255 / 0.07) !important;
+}
+.codejar-linenumber {
+ color: rgba(120 120 120 / 1) !important;
+ text-align: right;
+}
+.editor {
+ font-family: monospace;
+ font-size: 14px;
+ font-weight: 400;
+ letter-spacing: normal;
+ margin-left: 10px;
+ margin-right: 10px;
+ resize: unset !important;
+ tab-size: 4;
+ -moz-tab-size: 4;
+ overflow: unset !important;
+}
+
+/* Prism show-invisibles plugin overrides */
+.token.tab:not(:empty)::before {
+ content: "\23af\27F6";
+}
+/* Temporarily remove CR and LF tokens as they overflow line width */
+.token.cr::before,
+.token.crlf::before,
+.token.lf::before {
+ content: "";
+}
+
+/* Preloader */
+.blinking {
+ animation: blinker 1.2s ease-in-out infinite;
+}
+@keyframes blinker {
+ 50% {
+ -webkit-filter: invert(1);
+ filter: invert(1);
+ }
+}
diff --git a/interface/css/svg-with-js.min.css b/interface/css/svg-with-js.min.css
new file mode 100644
index 0000000..2640f93
--- /dev/null
+++ b/interface/css/svg-with-js.min.css
@@ -0,0 +1,5 @@
+/*!
+ * Font Awesome Free 5.13.1 by @fontawesome - https://fontawesome.com
+ * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+ */
+.svg-inline--fa,svg:not(:root).svg-inline--fa{overflow:visible}.svg-inline--fa{display:inline-block;font-size:inherit;height:1em;vertical-align:-.125em}.svg-inline--fa.fa-lg{vertical-align:-.225em}.svg-inline--fa.fa-w-1{width:.0625em}.svg-inline--fa.fa-w-2{width:.125em}.svg-inline--fa.fa-w-3{width:.1875em}.svg-inline--fa.fa-w-4{width:.25em}.svg-inline--fa.fa-w-5{width:.3125em}.svg-inline--fa.fa-w-6{width:.375em}.svg-inline--fa.fa-w-7{width:.4375em}.svg-inline--fa.fa-w-8{width:.5em}.svg-inline--fa.fa-w-9{width:.5625em}.svg-inline--fa.fa-w-10{width:.625em}.svg-inline--fa.fa-w-11{width:.6875em}.svg-inline--fa.fa-w-12{width:.75em}.svg-inline--fa.fa-w-13{width:.8125em}.svg-inline--fa.fa-w-14{width:.875em}.svg-inline--fa.fa-w-15{width:.9375em}.svg-inline--fa.fa-w-16{width:1em}.svg-inline--fa.fa-w-17{width:1.0625em}.svg-inline--fa.fa-w-18{width:1.125em}.svg-inline--fa.fa-w-19{width:1.1875em}.svg-inline--fa.fa-w-20{width:1.25em}.svg-inline--fa.fa-pull-left{margin-right:.3em;width:auto}.svg-inline--fa.fa-pull-right{margin-left:.3em;width:auto}.svg-inline--fa.fa-border{height:1.5em}.svg-inline--fa.fa-li{width:2em}.svg-inline--fa.fa-fw{width:1.25em}.fa-layers svg.svg-inline--fa{bottom:0;left:0;margin:auto;position:absolute;right:0;top:0}.fa-layers{display:inline-block;height:1em;position:relative;text-align:center;vertical-align:-.125em;width:1em}.fa-layers svg.svg-inline--fa{-webkit-transform-origin:center center;transform-origin:center center}.fa-layers-counter,.fa-layers-text{display:inline-block;position:absolute;text-align:center}.fa-layers-text{left:50%;top:50%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%);-webkit-transform-origin:center center;transform-origin:center center}.fa-layers-counter{background-color:#ff253a;border-radius:1em;-webkit-box-sizing:border-box;box-sizing:border-box;color:#fff;height:1.5em;line-height:1;max-width:5em;min-width:1.5em;overflow:hidden;padding:.25em;right:0;text-overflow:ellipsis;top:0;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:top right;transform-origin:top right}.fa-layers-bottom-right{bottom:0;right:0;top:auto;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:bottom right;transform-origin:bottom right}.fa-layers-bottom-left{bottom:0;left:0;right:auto;top:auto;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:bottom left;transform-origin:bottom left}.fa-layers-top-right{right:0;top:0;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:top right;transform-origin:top right}.fa-layers-top-left{left:0;right:auto;top:0;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:top left;transform-origin:top left}.fa-lg{font-size:1.33333em;line-height:.75em;vertical-align:-.0667em}.fa-xs{font-size:.75em}.fa-sm{font-size:.875em}.fa-1x{font-size:1em}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-6x{font-size:6em}.fa-7x{font-size:7em}.fa-8x{font-size:8em}.fa-9x{font-size:9em}.fa-10x{font-size:10em}.fa-fw{text-align:center;width:1.25em}.fa-ul{list-style-type:none;margin-left:2.5em;padding-left:0}.fa-ul>li{position:relative}.fa-li{left:-2em;position:absolute;text-align:center;width:2em;line-height:inherit}.fa-border{border:.08em solid #eee;border-radius:.1em;padding:.2em .25em .15em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left,.fab.fa-pull-left,.fal.fa-pull-left,.far.fa-pull-left,.fas.fa-pull-left{margin-right:.3em}.fa.fa-pull-right,.fab.fa-pull-right,.fal.fa-pull-right,.far.fa-pull-right,.fas.fa-pull-right{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s linear infinite;animation:fa-spin 2s linear infinite}.fa-pulse{-webkit-animation:fa-spin 1s steps(8) infinite;animation:fa-spin 1s steps(8) infinite}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{-webkit-transform:rotate(1turn);transform:rotate(1turn)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{-webkit-transform:rotate(1turn);transform:rotate(1turn)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scaleX(-1);transform:scaleX(-1)}.fa-flip-vertical{-webkit-transform:scaleY(-1);transform:scaleY(-1)}.fa-flip-both,.fa-flip-horizontal.fa-flip-vertical,.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)"}.fa-flip-both,.fa-flip-horizontal.fa-flip-vertical{-webkit-transform:scale(-1);transform:scale(-1)}:root .fa-flip-both,:root .fa-flip-horizontal,:root .fa-flip-vertical,:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270{-webkit-filter:none;filter:none}.fa-stack{display:inline-block;height:2em;position:relative;width:2.5em}.fa-stack-1x,.fa-stack-2x{bottom:0;left:0;margin:auto;position:absolute;right:0;top:0}.svg-inline--fa.fa-stack-1x{height:1em;width:1.25em}.svg-inline--fa.fa-stack-2x{height:2em;width:2.5em}.fa-inverse{color:#fff}.sr-only{border:0;clip:rect(0,0,0,0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.sr-only-focusable:active,.sr-only-focusable:focus{clip:auto;height:auto;margin:0;overflow:visible;position:static;width:auto}.svg-inline--fa .fa-primary{fill:var(--fa-primary-color,currentColor);opacity:1;opacity:var(--fa-primary-opacity,1)}.svg-inline--fa .fa-secondary{fill:var(--fa-secondary-color,currentColor)}.svg-inline--fa .fa-secondary,.svg-inline--fa.fa-swap-opacity .fa-primary{opacity:.4;opacity:var(--fa-secondary-opacity,.4)}.svg-inline--fa.fa-swap-opacity .fa-secondary{opacity:1;opacity:var(--fa-primary-opacity,1)}.svg-inline--fa mask .fa-primary,.svg-inline--fa mask .fa-secondary{fill:#000}.fad.fa-inverse{color:#fff} \ No newline at end of file
diff --git a/interface/favicon-16x16.png b/interface/favicon-16x16.png
new file mode 100644
index 0000000..cbc338d
--- /dev/null
+++ b/interface/favicon-16x16.png
Binary files differ
diff --git a/interface/favicon-32x32.png b/interface/favicon-32x32.png
new file mode 100644
index 0000000..71d0d07
--- /dev/null
+++ b/interface/favicon-32x32.png
Binary files differ
diff --git a/interface/favicon.ico b/interface/favicon.ico
new file mode 100644
index 0000000..002bebc
--- /dev/null
+++ b/interface/favicon.ico
Binary files differ
diff --git a/interface/fonts/glyphicons-halflings-regular.ttf b/interface/fonts/glyphicons-halflings-regular.ttf
new file mode 100644
index 0000000..1413fc6
--- /dev/null
+++ b/interface/fonts/glyphicons-halflings-regular.ttf
Binary files differ
diff --git a/interface/fonts/glyphicons-halflings-regular.woff b/interface/fonts/glyphicons-halflings-regular.woff
new file mode 100644
index 0000000..9e61285
--- /dev/null
+++ b/interface/fonts/glyphicons-halflings-regular.woff
Binary files differ
diff --git a/interface/fonts/glyphicons-halflings-regular.woff2 b/interface/fonts/glyphicons-halflings-regular.woff2
new file mode 100644
index 0000000..64539b5
--- /dev/null
+++ b/interface/fonts/glyphicons-halflings-regular.woff2
Binary files differ
diff --git a/interface/img/asc.png b/interface/img/asc.png
new file mode 100644
index 0000000..3021615
--- /dev/null
+++ b/interface/img/asc.png
Binary files differ
diff --git a/interface/img/desc.png b/interface/img/desc.png
new file mode 100644
index 0000000..068c549
--- /dev/null
+++ b/interface/img/desc.png
Binary files differ
diff --git a/interface/img/rspamd_logo_navbar.png b/interface/img/rspamd_logo_navbar.png
new file mode 100644
index 0000000..89d1222
--- /dev/null
+++ b/interface/img/rspamd_logo_navbar.png
Binary files differ
diff --git a/interface/index.html b/interface/index.html
new file mode 100644
index 0000000..2607348
--- /dev/null
+++ b/interface/index.html
@@ -0,0 +1,737 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <title>Rspamd Web Interface</title>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <meta name="description" content="">
+ <meta name="author" content="">
+
+ <link rel="icon" type="image/png" sizes="32x32" href="favicon-32x32.png?v=3">
+ <link rel="icon" type="image/png" sizes="16x16" href="favicon-16x16.png?v=3">
+ <link rel="apple-touch-icon" sizes="180x180" href="apple-touch-icon.png?v=3">
+ <link rel="mask-icon" href="safari-pinned-tab.svg?v=3" color="#5bbad5">
+ <link rel="shortcut icon" href="favicon.ico?v=3">
+ <meta name="msapplication-TileColor" content="#2b5797">
+
+ <link href="./css/bootstrap.min.css" rel="stylesheet">
+ <link rel="stylesheet" type="text/css" href="./css/font-glyphicons.css">
+ <link rel="stylesheet" type="text/css" href="./css/footable.standalone.min.css"/>
+ <link rel="stylesheet" type="text/css" href="./css/FooTable.Glyphicons.css"/>
+ <link rel="stylesheet" type="text/css" href="./css/svg-with-js.min.css">
+ <link rel="stylesheet" type="text/css" href="./css/d3evolution.css">
+ <link rel="stylesheet" type="text/css" href="./css/d3pie.css">
+ <link rel="stylesheet" type="text/css" href="./css/nprogress.css"/>
+ <link rel="stylesheet" type="text/css" href="./css/codejar-linenumbers.css"/>
+ <link rel="stylesheet" type="text/css" href="./css/prism.css"/>
+ <link href="./css/rspamd.css" rel="stylesheet">
+</head>
+
+<body>
+
+<nav class="navbar navbar-light bg-light navbar-expand-xl d-none" id="navBar">
+ <div class="container-fluid">
+ <div class="navbar-header navbar-brand p-0">
+ <img src="./img/rspamd_logo_navbar.png" alt="Rspamd">
+ </div>
+ <div class="collapse navbar-collapse order-3 order-xl-2">
+ <form class="my-2 me-auto">
+ <select id="selSrv" class="form-select"></select>
+ </form>
+ </div>
+
+ <div class="collapse navbar-collapse order-4 order-xl-3 justify-content-center">
+ <ul class="nav navbar-nav nav-pills nav-tabs-sticky" id="tablist" role="tablist">
+ <li role="presentation" class="nav-item"><a id="status_nav" aria-controls="status" role="tab" href="#status" data-bs-toggle="tab" class="nav-link">Status</a></li>
+
+ <li role="presentation" class="nav-item"><a id="throughput_nav" aria-controls="throughput" role="tab" href="#throughput" data-bs-toggle="tab" class="nav-link">Throughput</a></li>
+
+ <li role="presentation" class="nav-item"><a id="configuration_nav" aria-controls="configuration" role="tab" href="#configuration" data-bs-toggle="tab" class="nav-link">Configuration</a></li>
+ <li role="presentation" class="nav-item"><a id="symbols_nav" aria-controls="symbols" role="tab" href="#symbols" data-bs-toggle="tab" class="nav-link">Symbols</a></li>
+ <li role="presentation" class="nav-item"><a id="scan_nav" aria-controls="scan" role="tab" href="#scan" data-bs-toggle="tab" class="nav-link">Scan<span class="ro-hide" style="display: none;">/Learn</span></a></li>
+ <li role="presentation" class="nav-item"><a id="selectors_nav" aria-controls="selectors" role="tab" href="#selectors" data-bs-toggle="tab" class="nav-link ro-hide" style="display: none;">Test selectors</a></li>
+ <li role="presentation" class="nav-item"><a id="history_nav" aria-controls="history" role="tab" href="#history" data-bs-toggle="tab" class="nav-link">History</a></li>
+ </ul>
+ </div>
+
+ <div class="d-flex flex-row order-2 order-xl-4">
+ <form class="ms-auto">
+ <div class="btn-group">
+ <button class="btn btn-outline-secondary" id="refresh" style="display: none;"><i class="fas fa-sync-alt"></i> Refresh</button>
+ <button class="btn btn-outline-secondary dropdown-toggle dropdown-toggle-split" data-bs-toggle="dropdown" id="autoRefresh" title="Auto-refresh time interval">
+ <span id="countdown">--:--</span>
+ </button>
+ <div class="dropdown-menu">
+ <a class="dropdown-item preset" href="#" data-value=null>Disable</a>
+ <div class="dropdown-divider preset" role="separator"></div>
+ <a class="dropdown-item preset active" href="#" data-value="10000">10 seconds</a>
+ <a class="dropdown-item preset" href="#" data-value="30000">30 seconds</a>
+ <div class="dropdown-divider preset" role="separator"></div>
+ <a class="dropdown-item preset" href="#" data-value="60000">1 minute</a>
+ <a class="dropdown-item preset" href="#" data-value="600000">10 minutes</a>
+ <a class="dropdown-item preset" href="#" data-value="1800000">30 minutes</a>
+ <div class="dropdown-divider preset" role="separator"></div>
+ <a class="dropdown-item preset" href="#" data-value="3600000">1 hour</a>
+
+ <a class="dropdown-item history active" href="#" data-value=null>Disable</a>
+ <div class="dropdown-divider history" role="separator"></div>
+ <a class="dropdown-item history" href="#" data-value="10000">10 seconds</a>
+ <a class="dropdown-item history" href="#" data-value="30000">30 seconds</a>
+ <div class="dropdown-divider history" role="separator"></div>
+ <a class="dropdown-item history" href="#" data-value="60000">1 minute</a>
+ <a class="dropdown-item history" href="#" data-value="600000">10 minutes</a>
+ <a class="dropdown-item history" href="#" data-value="1800000">30 minutes</a>
+ <div class="dropdown-divider history" role="separator"></div>
+ <a class="dropdown-item history" href="#" data-value="3600000">1 hour</a>
+
+ <a class="dropdown-item dynamic" href="#" data-value=null>Disable</a>
+ <div class="dropdown-divider dynamic" role="separator"></div>
+ <a class="dropdown-item dynamic active" href="#" id="dynamic-item" data-value="3600000">1 hour</a>
+ </div>
+ </div>
+ <button class="btn btn-outline-secondary ms-2" id="disconnect" title="Disconnect"><i class="fas fa-power-off"></i></button>
+ <button class="btn btn-outline-secondary ms-2" id="settings" title="WebUI settings"><i class="fas fa-cog"></i></button>
+ <div class="d-none">
+ <div id="settings-popover">
+ <div class="card">
+ <div class="card-body">
+ <h6 class="card-title fw-bolder">Date and time locale</h6>
+ <label class="ms-2">
+ <input type="radio" class="me-2" name="locale" value="browser" checked>
+ Use browser settings
+ </label>
+ <div class="input-group input-group-sm">
+ <div class="input-group-text">
+ <label class="my-auto">
+ <input type="radio" class="me-2" name="locale" value="custom">
+ Custom
+ </label>
+ </div>
+ <input type="text" id="locale" placeholder="Enter locale" class="form-control" />
+ </div>
+ <p class="mt-2 mb-0">Date: <span id="date-example"></span></p>
+ </div>
+ </div>
+
+ <div class="card mt-1">
+ <div class="card-body">
+ <h6 class="card-title fw-bolder">HTTP requests timeout, ms</h6>
+ <div class="input-group input-group-sm was-validated">
+ <input type="number" id="ajax-timeout" class="form-control" min="0" step="any" />
+ <div class="input-group-sm">
+ <button id="ajax-timeout-restore" class="btn btn-secondary">Restore default</button>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </form>
+ <button class="navbar-toggler ms-2" type="button" data-bs-toggle="collapse" data-bs-target=".navbar-collapse">
+ <span class="navbar-toggler-icon"></span>
+ </button>
+ </div>
+ </div>
+</nav>
+
+<div class="notification-area">
+ <noscript>
+ <div class="alert alert-error"><strong>Please enable JavaScript</strong></div>
+ </noscript>
+</div>
+
+<div id="preloader" class="container-fluid">
+ <div id="loading" class="position-absolute align-items-center my-3 text-primary blinking d-none">
+ <div class="spinner-border align-middle" role="status" aria-hidden="true"></div>
+ <strong class="ms-3">Loading...</strong>
+ </div>
+ <script>document.getElementById("loading").classList.remove("d-none");</script>
+ <div class="row position-absolute w-100 h-100 align-items-center text-center">
+ <img class="img-fluid w-auto mh-100 mx-auto" src="./img/rspamd_logo_navbar.png" alt="Rspamd" />
+ </div>
+</div>
+
+<div id="mainUI" class="d-none">
+
+ <div class="container-fluid">
+
+ <div class="tab-content">
+ <div class="tab-pane" id="status">
+ <div class="row">
+ <div id="statWidgets" class="col-lg-12 stat-boxes fw-bold text-secondary" style="display: none;">
+ </div>
+ </div>
+ <div class="row">
+ <div class="col-lg-6">
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-2">
+ <span class="icon me-3"><i class="fas fa-server"></i></span>
+ <span class="h6 fw-bolder my-2">Servers</span>
+ </div>
+ <div class="card-body p-0 table-responsive">
+ <table class="table status-table table-sm table-hover table-bordered text-nowrap mb-0" id="clusterTable">
+ <thead class="text-secondary">
+ <tr>
+ <th></th>
+ <th>Server name</th>
+ <th>Host</th>
+ <th class="w-1">Status</th>
+ <th class="w-1">Scan time</th>
+ <th class="w-1">Uptime</th>
+ <th class="w-1">Version</th>
+ <th>Configuration ID</th>
+ </tr>
+ </thead>
+ <tbody>
+ </tbody>
+ </table>
+ </div>
+ </div>
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-2">
+ <span class="icon me-3"><i class="fas fa-dice"></i></span>
+ <span class="h6 fw-bolder my-2">Bayesian statistics</span>
+ </div>
+ <div class="card-body p-0 table-responsive">
+ <table class="table status-table table-sm table-bordered text-nowrap mb-0" id="bayesTable">
+ <thead class="text-secondary">
+ <tr>
+ <th>Server name</th>
+ <th>Symbol</th>
+ <th>Type</th>
+ <th>Learns</th>
+ <th>Users</th>
+ </tr>
+ </thead>
+ <tbody>
+ </tbody>
+ </table>
+ </div>
+ </div>
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-2">
+ <span class="icon me-3"><i class="fas fa-hashtag"></i></span>
+ <span class="h6 fw-bolder my-2">Fuzzy hashes</span>
+ </div>
+ <div class="card-body p-0 table-responsive">
+ <table class="table status-table table-sm table-bordered text-nowrap mb-0" id="fuzzyTable">
+ <thead class="text-secondary">
+ <tr>
+ <th>Server name</th>
+ <th>Storage</th>
+ <th>Hashes</th>
+ </tr>
+ </thead>
+ <tbody>
+ </tbody>
+ </table>
+ </div>
+ </div>
+ </div>
+ <div class="col-lg-6">
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-2">
+ <span class="icon me-3"><i class="fas fa-chart-pie"></i></span>
+ <span class="h6 fw-bolder my-2">Statistics</span>
+ </div>
+ <div class="card-body">
+ <div class="row">
+ <div class="bg-white w-auto mx-auto" id="chart"></div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div class="tab-pane" id="throughput">
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-2">
+ <span class="icon me-3"><i class="fas fa-chart-area"></i></span>
+ <span class="h6 fw-bolder my-2">Throughput</span>
+ </div>
+ <div class="card-body text-center">
+ <div class="d-inline-block bg-white">
+ <div class="row">
+ <div id="graph" class="mx-auto"></div>
+ </div>
+ <div id="summary-row" class="row">
+ <div class="col-fixed" id="rrd-pie"></div>
+ <div class="col-fluid">
+ <table id="rrd-table" class="table table-light table-striped table-hover"></table>
+ <div id="rrd-table_toggle"></div>
+ <div id="rrd-total">Total messages: <span id="rrd-total-value"></span></div>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div class="card-footer pt-0 text-center">
+ <form id="graph_controls" action="#">
+ Dataset:
+ <select id="selData" class="form-select">
+ <option value="day" selected>By day</option>
+ <option value="week">By week</option>
+ <option value="month">By month</option>
+ <option value="year">By year</option>
+ </select>
+ Y-scale:
+ <select id="selYScale" class="form-select">
+ <option value="lin" selected>Lin</option>
+ <option value="log">Log</option>
+ </select>
+ Convert to:
+ <select id="selConvert" class="form-select">
+ <option value="" selected>--</option>
+ <option value="percentage">percentage</option>
+ </select>
+ Chart type:
+ <select id="selType" class="form-select">
+ <option value="line" selected>Line</option>
+ <option value="area">Stacked area</option>
+ </select>
+ <a title="&ldquo;Curves&rdquo; section of &ldquo;d3-shape&rdquo; library documentation" href="https://github.com/d3/d3-shape#curves" target="_blank">Interpolation mode</a>:
+ <select id="selInterpolate" class="form-select">
+ <option value="curveLinear" selected>linear</option>
+ <option value="curveStep">step</option>
+ <option value="curveStepBefore">stepBefore</option>
+ <option value="curveStepAfter">stepAfter</option>
+ <option value="curveBasis">basis</option>
+ <option value="curveBasisOpen">basisOpen</option>
+ <option value="curveBundle">bundle</option>
+ <option value="curveCardinal">cardinal</option>
+ <option value="curveCardinalOpen">cardinalOpen</option>
+ <option value="curveMonotoneX">monotoneX</option>
+ </select>
+ </form>
+ </div>
+ </div>
+ </div>
+
+ <div class="tab-pane" id="configuration">
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-2">
+ <span class="icon me-3"><i class="fas fa-tasks"></i></span>
+ <span class="h6 fw-bolder my-2">Actions</span>
+ </div>
+ <div class="card-body pb-2">
+ <form id="actionsForm">
+ <fieldset id="actionsFormField" class="ro-disable"></fieldset>
+ </form>
+ </div>
+ <div class="card-footer ro-hide">
+ <div class="btn-group">
+ <button class="btn btn-primary" type="button" id="saveActionsBtn">Save actions</button>
+ <button class="btn btn-warning" type="button" id="saveActionsClusterBtn">Save cluster</button>
+ </div>
+ </div>
+ </div>
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-2 d-flex">
+ <span class="icon me-3"><i class="fas fa-list"></i></span>
+ <span class="h6 fw-bolder my-2">Lists</span>
+ <div class="input-group-sm align-self-center ms-auto me-1">
+ Editor:
+ <div id="btnGroupEditor" class="btn-group btn-group-xs ms-1">
+ <input type="radio" class="btn-check" name="editorMode" id="editorModeBascic" autocomplete="off" value="basic">
+ <label class="btn btn-outline-secondary" for="editorModeBascic">Basic</label>
+
+ <input type="radio" class="btn-check" name="editorMode" id="editorModeAdvanced"autocomplete="off" value="advanced" checked>
+ <label class="btn btn-outline-secondary" for="editorModeAdvanced">Advanced</label>
+ </div>
+ </div>
+ </div>
+ <div class="card-body p-0">
+ <table class="table table-sm table-hover mb-0" id="listMaps">
+ </table>
+ </div>
+ </div>
+ </div>
+
+ <div class="tab-pane" id="symbols">
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-1 d-flex">
+ <span class="icon me-3"><i class="fas fa-tasks"></i></span>
+ <span class="h6 fw-bolder my-2 ms-0">Symbols and rules</span>
+ <div class="align-self-center ms-auto me-1">
+ <button class="btn btn-info btn-sm" id="updateSymbols">
+ <i class="fas fa-redo-alt"></i> Update
+ </button>
+ </div>
+ </div>
+ <div class="alert alert-info sticky-top py-2 mb-0 rounded-0 d-none" id="save-alert">
+ <span class="icon me-3 align-middle"><i class="fas fa-save"></i></span>
+ <span class="align-middle">The values have been updated. The changes must be saved to take effect.</span>
+ <div class="float-end">
+ <button title="Save changes to the selected server" type="button" class="btn btn-primary btn-sm">
+ Save
+ </button>
+ <button data-save="All SERVERS" title="Save changes to all servers" type="button" class="btn btn-warning btn-sm">
+ Save in cluster
+ </button>
+ </div>
+ </div>
+ <div class="card-body p-0">
+ <table class="table table-hover" id="symbolsTable"></table>
+ </div>
+ </div>
+ </div>
+
+ <div class="tab-pane" id="scan">
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-2">
+ <span class="icon me-3"><i class="fas fa-envelope"></i></span>
+ <span class="h6 fw-bolder my-2">Scan suspected message</span>
+ </div>
+ <div class="card-body">
+ <div class="row">
+ <form class="col-lg-12" id="scanForm">
+ <div class="mb-0">
+ <label class="form-label" for="scanMsgSource">Message source:</label>
+ <textarea class="form-control" id="scanMsgSource" rows="10" placeholder="Paste raw message source"></textarea>
+ </div>
+ <div class="collapse row mt-3" id="scanOptions">
+ <div class="col-lg-6">
+ <div class="row mb-3 text-nowrap" title="Emulate IP address from which this message was received">
+ <label for="scan-opt-ip" class="col-form-label col-sm-2">IP:</label>
+ <div class="col-sm-10">
+ <input name="scan-opt-ip" id="scan-opt-ip" class="form-control" type="text"/>
+ </div>
+ </div>
+ <div class="row mb-3 text-nowrap" title="Emulate username of the authenticated SMTP client">
+ <label for="scan-opt-user" class="col-form-label col-sm-2">User:</label>
+ <div class="col-sm-10">
+ <input name="scan-opt-user" id="scan-opt-user" class="form-control" type="text"/>
+ </div>
+ </div>
+ <div class="row mb-3 text-nowrap" title="Emulate SMTP 'MAIL FROM' command data">
+ <label for="scan-opt-from" class="col-form-label col-sm-2">From:</label>
+ <div class="col-sm-10">
+ <input name="scan-opt-from" id="scan-opt-from" class="form-control" type="text"/>
+ </div>
+ </div>
+ </div>
+ <div class="col-lg-6">
+ <div class="row mb-3 text-nowrap" title="Emulate SMTP 'RCPT TO' command data">
+ <label for="scan-opt-rcpt" class="col-form-label col-sm-2">Rcpt:</label>
+ <div class="col-sm-10">
+ <input name="scan-opt-rcpt" id="scan-opt-rcpt" class="form-control" type="text"/>
+ </div>
+ </div>
+ <div class="row mb-3 text-nowrap" title="Imitate SMTP 'HELO' passing from the MTA">
+ <label for="scan-opt-helo" class="col-form-label col-sm-2">Helo:</label>
+ <div class="col-sm-10">
+ <input name="scan-opt-helo" id="scan-opt-helo" class="form-control" type="text"/>
+ </div>
+ </div>
+ <div class="row mb-3 text-nowrap" title="Imitate resolved hostname passing from the MTA">
+ <label for="scan-opt-hostname" class="col-form-label col-sm-2">Hostname:</label>
+ <div class="col-sm-10">
+ <input name="scan-opt-hostname" id="scan-opt-hostname" class="form-control" type="text"/>
+ </div>
+ </div>
+ <div class="row text-nowrap ps-0" title="Pass all filters">
+ <label for="scan-opt-pass-all" class="form-check-label col-sm-2">Pass: all</label>
+ <div class="col-sm-10">
+ <input name="scan-opt-pass-all" id="scan-opt-pass-all" class="form-check-input-reverse" type="checkbox"/>
+ </div>
+ </div>
+ </div>
+ </div>
+ </form>
+ </div>
+ </div>
+ <div class="card-footer d-md-flex justify-content-between py-1">
+ <div class="input-group d-inline-flex w-auto my-1">
+ <button type="submit" class="btn btn-primary" data-upload="scan"><i class="fas fa-search"></i> Scan message</button>
+ <button class="btn btn-secondary d-inline-block" id="scanOptionsToggle" data-bs-toggle="collapse" data-bs-target="#scanOptions"><i class="fas fa-bars"></i> Options</button>
+ </div>
+ <div class="input-group d-inline-flex w-auto my-1">
+ <label for="fuzzy-flag" class="input-group-text">Flag</label>
+ <input id="fuzzy-flag" class="form-control" value="1" min="1" type="number">
+ <button class="btn btn-warning" data-upload="compute-fuzzy"><i class="fas fa-hashtag"></i> Compute fuzzy hashes</button>
+ </div>
+ <div class="float-end my-1">
+ <button class="btn btn-secondary" id="scanClean"><i class="fas fa-trash-alt"></i> Clean form</button>
+ </div>
+ </div>
+ </div>
+ <div class="card ro-hide" style="display: none;">
+ <div class="card-header text-secondary py-1 d-flex">
+ <span class="icon me-3"><i class="fas fa-graduation-cap"></i></span>
+ <span class="h6 fw-bolder my-2">Learn Rspamd</span>
+ <div id="learnServers" class="input-group-sm align-items-center text-nowrap ms-auto me-1">
+ <label for="selLearnServers">Learn servers:</label>
+ <select id="selLearnServers" class="form-select ms-1">
+ <option value="random" selected>random</option>
+ <option value="all">all</option>
+ </select>
+ </div>
+ </div>
+ <div class="card-body">
+ <div class="row">
+ <div class="col-lg-6">
+ <div class="card bg-light shadow card-body card p-2">
+ <p>Learn Bayesian classifier:</p>
+ <form>
+ <div class="btn-group">
+ <button class="btn btn-success" type="button" data-upload="ham" disabled><i class="fas fa-thumbs-up"></i> Upload HAM</button>
+ <button class="btn btn-danger" type="button" data-upload="spam" disabled><i class="fas fa-thumbs-down"></i> Upload SPAM</button>
+ </div>
+ </form>
+ </div>
+ </div>
+ <div class="col-lg-6">
+ <div class="card bg-light shadow card-body card p-2">
+ <p>Learn Fuzzy storage:</p>
+ <form class="d-flex">
+ <div class="d-flex align-items-center">
+ <label for="fuzzyFlagText">Flag:</label>
+ <input name="fuzzyFlagText" id="fuzzyFlagText" class="form-control ms-1" type="number" value="1"/>
+ </div>
+ <div class="d-flex align-items-center ms-2">
+ <label for="fuzzyWeightText">Weight:</label>
+ <input name="fuzzyWeightText" id="fuzzyWeightText" class="form-control ms-1" type="number" value="1"/>
+ </div>
+ <button class="btn btn-warning ms-2" data-upload="fuzzy" disabled><i class="fas fa-upload"></i> Upload FUZZY</button>
+ </form>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div id="hash-card" class="card bg-light shadow my-3" style="display: none;">
+ <div class="card-header text-secondary py-2">
+ <span class="icon me-3"><i class="fas fa-hashtag"></i></span>
+ <span class="h6 fw-bolder my-2">Fuzzy hashes</span>
+ <button type="button" class="card-close-btn btn-close float-end" aria-label="Close"></button>
+ </div>
+ <div class="card-body p-0 table-responsive">
+ <table class="table status-table table-sm table-bordered text-nowrap mb-0" id="hashTable">
+ <thead class="text-secondary">
+ <tr>
+ <th>Rule name</th>
+ <th>Hashes</th>
+ </tr>
+ </thead>
+ <tbody>
+ </tbody>
+ </table>
+ </div>
+ </div>
+
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-1 d-flex">
+ <span class="icon me-3"><i class="fas fa-eye"></i></span>
+ <span class="h6 fw-bolder my-2 ms-0">Scan results history</span>
+ <div id="scanResult" class="d-flex input-group-sm align-items-center text-nowrap ms-auto me-1">
+ <label for="selSymOrder_scan">Symbols order:</label>
+ <select id="selSymOrder_scan" class="form-select ms-1">
+ <option value="magnitude" selected>Score magnitude</option>
+ <option value="score">Score value</option>
+ <option value="name">Name</option>
+ </select>
+ <label for="scan_page_size" class="ms-2">Rows per page:</label>
+ <input id="scan_page_size" class="form-control ms-1" value="25" min="1" type="number">
+ <button class="btn btn-secondary btn-sm ms-2" id="cleanScanHistory" disabled>
+ <i class="fas fa-trash-alt"></i> Clean history
+ </button>
+ </div>
+ </div>
+ <div class="card-body p-0">
+ <div id="scanLog">
+ <table class="table mb-0" id="historyTable_scan"></table>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div class="tab-pane" id="selectors">
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-2">
+ <span class="icon me-3"><i class="fas fa-envelope"></i></span>
+ <span class="h6 fw-bolder my-2">Test Rspamd selectors</span>
+ </div>
+ <div class="card-body p-0">
+ <div class="row h-100 m-0" id="row-main">
+ <div class="col-lg-3 sidebar h-100" id="sidebar-left">
+ <div class="p-0 table-responsive mh-100 bg-white">
+ <table class="table table-sm small table-striped table-hover table-bordered mb-0" id="selectorsTable-extractors">
+ <thead><tr><th>Name</th><th>Description</th></tr></thead>
+ <tbody/>
+ </table>
+ </div>
+ </div>
+
+ <div class="col-lg-6 col-12 h-100" id="content">
+
+ <div class="row navbar-light h-100">
+ <ul class="nav navbar-nav nav-tabs sidebar-nav border-bottom-0">
+ <li class="active text-nowrap nav-item" id="sidebar-tab-left">
+ <a class="nav-link border-bottom-0 px-2 py-0" href=""><div id="sidebar-tab-text-left">&#x2195;&ensp;List of extractors</div></a>
+ </li>
+ </ul>
+
+ <div class="col-lg col-12 mh-100 overflow-auto">
+ <div class="row h-100">
+ <form class="col-12 d-flex flex-column">
+ <div class="row pt-3">
+ <div class="col">
+ <div class="form-group">
+ <label class="form-label" for="selectorsMsgArea">Message source:</label>
+ <textarea class="form-control" id="selectorsMsgArea" rows="9" placeholder="Paste raw message source"></textarea>
+ </div>
+ <button class="btn btn-secondary float-end" id="selectorsMsgClean"><i class="fas fa-trash-alt"></i> Clean form</button>
+ </div>
+ </div>
+ <div class="row pt-3">
+ <div class="col">
+ <div class="form-group">
+ <label class="form-label" for="selectorsSelArea">Selector(s):</label>
+ <textarea class="form-control" id="selectorsSelArea" rows="1" placeholder="extractor.transform(arg);extractor.transform(arg);..."></textarea>
+ </div>
+ <button type="submit" class="btn btn-primary" id="selectorsChkMsgBtn"><i class="fas fa-search"></i> Check message</button>
+ <button class="btn btn-secondary float-end" id="selectorsClean"><i class="fas fa-trash-alt"></i> Clean form</button>
+ </div>
+ </div>
+ <div class="row pt-3 flex-grow-1">
+ <div class="col d-flex flex-column">
+ <div class="form-group h-100 d-flex flex-column">
+ <label class="form-label" for="selectorsResArea">Result:</label>
+ <textarea class="form-control flex-grow-1" id="selectorsResArea" disabled readonly></textarea>
+ </div>
+ </div>
+ </div>
+ </form>
+ </div>
+ </div>
+
+ <ul class="nav navbar-nav nav-tabs sidebar-nav border-bottom-0">
+ <li class="active text-nowrap nav-item" id="sidebar-tab-right">
+ <a class="nav-link border-bottom-0 px-2 py-0" href="">&#x2195;&ensp;List of transforms</a>
+ </li>
+ </ul>
+ </div>
+ </div>
+
+ <div class="col-lg-3 sidebar h-100" id="sidebar-right">
+ <div class="p-0 table-responsive mh-100 bg-white">
+ <table class="table table-sm small table-striped table-hover table-bordered mb-0" id="selectorsTable-transforms">
+ <thead><tr><th>Name</th><th>Description</th></tr></thead>
+ <tbody/>
+ </table>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div class="tab-pane" id="history">
+
+ <div class="card bg-light shadow my-3">
+ <div class="card-header text-secondary py-1 d-flex">
+ <span class="icon me-3"><i class="fas fa-eye"></i></span>
+ <span class="h6 fw-bolder my-2 ms-0">History</span>
+ <a href="https://rspamd.com/doc/modules/history_redis.html" target="_blank" rel="noopener noreferrer"
+ title="If you'd like to use the modern version of History, please enable History redis module."
+ id="legacy-history-badge" class="my-2 ms-2 badge text-bg-info" style="display: none;">Legacy version</a>
+ <div class="d-flex input-group-sm align-items-center text-nowrap ms-auto me-1">
+ <label for="selSymOrder_history">Symbols order:</label>
+ <select id="selSymOrder_history" class="form-select ms-1">
+ <option value="magnitude" selected>Score magnitude</option>
+ <option value="score">Score value</option>
+ <option value="name">Name</option>
+ </select>
+ <label for="history_page_size" class="ms-2">Rows per page:</label>
+ <input id="history_page_size" class="form-control ms-1" value="25" min="1" type="number">
+ <button class="btn btn-danger btn-sm ms-2 ro-hide" id="resetHistory">
+ <i class="fas fa-times-circle"></i> Reset
+ </button>
+ <button class="btn btn-info btn-sm ms-2" id="updateHistory">
+ <i class="fas fa-redo-alt"></i> Update
+ </button>
+ </div>
+ </div>
+ <div class="card-body p-0">
+ <div id="historyLog">
+ <table class="table" id="historyTable_history"></table>
+ </div>
+ </div>
+ </div>
+ <div class="card bg-light shadow my-3 ro-hide" id="errors-history">
+ <div class="card-header text-secondary py-1 d-flex">
+ <span class="icon me-3"><i class="fas fa-exclamation-triangle"></i></span>
+ <span class="h6 fw-bolder my-2 ms-0">Errors</span>
+ <div class="align-self-center ms-auto me-1">
+ <button class="btn btn-info btn-sm" id="updateErrors">
+ <i class="fas fa-redo-alt"></i> Update
+ </button>
+ </div>
+ </div>
+ <div class="card-body p-0">
+ <table class="table table-hover mb-0" id="errorsLog"></table>
+ </div>
+ </div>
+
+ </div>
+
+ </div>
+ </div>
+
+</div>
+
+<!-- Common modal -->
+<div id="modalDialog" class="modal fade text-center" data-backdrop="static" tabindex="-1" role="dialog">
+ <div class="modal-dialog modal-xl d-inline-block mw-100 my-auto text-start">
+ <div class="modal-content shadow">
+ <div class="modal-header text-secondary py-2">
+ <i class="fas my-auto"></i><h6 class="modal-title fw-bolder mx-3" id="modalTitle"></h6>
+ <button type="button" class="btn-close" data-bs-dismiss="modal" aria-hidden="true"></button>
+ </div>
+ <div class="modal-body" id="modalBody"></div>
+ <div class="modal-footer">
+ <div class="btn-group" id="modalSaveGroup">
+ <button class="btn btn-primary" id="modalSave">Save changes</button>
+ <button class="btn btn-warning" id="modalSaveAll">Save on cluster</button>
+ </div>
+ <button class="btn btn-secondary" data-bs-dismiss="modal" aria-hidden="true" id="modalClose">Close</button>
+ </div>
+ </div>
+ </div>
+</div>
+
+<!-- login modal -->
+<div id="connectDialog" class="modal fade" data-bs-backdrop="static" data-bs-keyboard="false" role="dialog">
+ <div class="modal-dialog modal-sm modal-dialog-centered">
+ <div class="modal-content shadow">
+ <div class="modal-header text-secondary py-2">
+ <i class="fas fa-key my-auto"></i>
+ <div id="authInvalidCharFeedback" class="invalid-tooltip">Invalid characters</div>
+ <div id="authUnauthorizedFeedback" class="invalid-tooltip">Wrong password</div>
+ <h6 class="modal-title fw-bolder">Login to Rspamd</h6>
+ </div>
+ <div class="modal-body" id="connectBody">
+ <form id="connectForm">
+ <!-- In recent browser versions username is required to save credentials in a password manager.
+ Browser detects passwords by form.elements[n].type == "password" and then detects
+ the username field by searching backwards through form elements for the text field
+ immediately before the password fields. -->
+ <input value="Rspamd controller password" style="display: none;"/>
+ <div class="input-group">
+ <input class="form-control"
+ type="password"
+ id="connectPassword"
+ placeholder="Password"
+ data-cip-id="connectPassword">
+ <button type="submit" id="connectButton" class="btn btn-primary">Connect</button>
+ </div>
+ </form>
+ </div>
+ </div>
+ </div>
+</div>
+
+<script data-main="./js/main.js" src="./js/lib/require.min.js"></script>
+</body>
+</html>
diff --git a/interface/js/app/common.js b/interface/js/app/common.js
new file mode 100644
index 0000000..ea6102f
--- /dev/null
+++ b/interface/js/app/common.js
@@ -0,0 +1,233 @@
+/* global jQuery */
+
+define(["jquery", "nprogress"],
+ ($, NProgress) => {
+ "use strict";
+ const ui = {
+ chartLegend: [
+ {label: "reject", color: "#FF0000"},
+ {label: "soft reject", color: "#BF8040"},
+ {label: "rewrite subject", color: "#FF6600"},
+ {label: "add header", color: "#FFAD00"},
+ {label: "greylist", color: "#436EEE"},
+ {label: "no action", color: "#66CC00"}
+ ],
+ locale: (localStorage.getItem("selected_locale") === "custom") ? localStorage.getItem("custom_locale") : null,
+ neighbours: [],
+ page_size: {
+ scan: 25,
+ errors: 25,
+ history: 25
+ },
+ symbols: {
+ scan: [],
+ history: []
+ },
+ tables: {}
+ };
+
+
+ NProgress.configure({
+ minimum: 0.01,
+ showSpinner: false,
+ });
+
+ function getPassword() {
+ return sessionStorage.getItem("Password");
+ }
+
+ function alertMessage(alertClass, alertText) {
+ const a = $("<div class=\"alert " + alertClass + " alert-dismissible fade in show\">" +
+ "<button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"alert\" title=\"Dismiss\"></button>" +
+ "<strong>" + alertText + "</strong>");
+ $(".notification-area").append(a);
+
+ setTimeout(() => {
+ $(a).fadeTo(500, 0).slideUp(500, function () {
+ $(this).alert("close");
+ });
+ }, 5000);
+ }
+
+ function queryServer(neighbours_status, ind, req_url, o) {
+ neighbours_status[ind].checked = false;
+ neighbours_status[ind].data = {};
+ neighbours_status[ind].status = false;
+ const req_params = {
+ jsonp: false,
+ data: o.data,
+ headers: $.extend({Password: getPassword()}, o.headers),
+ url: neighbours_status[ind].url + req_url,
+ xhr: function () {
+ const xhr = $.ajaxSettings.xhr();
+ // Download progress
+ if (req_url !== "neighbours") {
+ xhr.addEventListener("progress", (e) => {
+ if (e.lengthComputable) {
+ neighbours_status[ind].percentComplete = e.loaded / e.total;
+ const percentComplete = neighbours_status
+ .reduce((prev, curr) => (curr.percentComplete ? curr.percentComplete + prev : prev), 0);
+ NProgress.set(percentComplete / neighbours_status.length);
+ }
+ }, false);
+ }
+ return xhr;
+ },
+ success: function (json) {
+ neighbours_status[ind].checked = true;
+ neighbours_status[ind].status = true;
+ neighbours_status[ind].data = json;
+ },
+ error: function (jqXHR, textStatus, errorThrown) {
+ neighbours_status[ind].checked = true;
+ function errorMessage() {
+ alertMessage("alert-error", neighbours_status[ind].name + " > " +
+ (o.errorMessage ? o.errorMessage : "Request failed") +
+ (errorThrown ? ": " + errorThrown : ""));
+ }
+ if (o.error) {
+ o.error(neighbours_status[ind],
+ jqXHR, textStatus, errorThrown);
+ } else if (o.errorOnceId) {
+ const alert_status = o.errorOnceId + neighbours_status[ind].name;
+ if (!(alert_status in sessionStorage)) {
+ sessionStorage.setItem(alert_status, true);
+ errorMessage();
+ }
+ } else {
+ errorMessage();
+ }
+ },
+ complete: function (jqXHR) {
+ if (neighbours_status.every((elt) => elt.checked)) {
+ if (neighbours_status.some((elt) => elt.status)) {
+ if (o.success) {
+ o.success(neighbours_status, jqXHR);
+ } else {
+ alertMessage("alert-success", "Request completed");
+ }
+ } else {
+ alertMessage("alert-error", "Request failed");
+ }
+ if (o.complete) o.complete();
+ NProgress.done();
+ }
+ },
+ statusCode: o.statusCode
+ };
+ if (o.method) {
+ req_params.method = o.method;
+ }
+ if (o.params) {
+ $.each(o.params, (k, v) => {
+ req_params[k] = v;
+ });
+ }
+ $.ajax(req_params);
+ }
+
+
+ // Public functions
+
+ ui.alertMessage = alertMessage;
+ ui.getPassword = getPassword;
+
+ // Get selectors' current state
+ ui.getSelector = function (id) {
+ const e = document.getElementById(id);
+ return e.options[e.selectedIndex].value;
+ };
+
+ /**
+ * @param {string} url - A string containing the URL to which the request is sent
+ * @param {Object} [options] - A set of key/value pairs that configure the Ajax request. All settings are optional.
+ *
+ * @param {Function} [options.complete] - A function to be called when the requests to all neighbours complete.
+ * @param {Object|string|Array} [options.data] - Data to be sent to the server.
+ * @param {Function} [options.error] - A function to be called if the request fails.
+ * @param {string} [options.errorMessage] - Text to display in the alert message if the request fails.
+ * @param {string} [options.errorOnceId] - A prefix of the alert ID to be added to the session storage. If the
+ * parameter is set, the error for each server will be displayed only once per session.
+ * @param {Object} [options.headers] - An object of additional header key/value pairs to send along with requests
+ * using the XMLHttpRequest transport.
+ * @param {string} [options.method] - The HTTP method to use for the request.
+ * @param {Object} [options.params] - An object of additional jQuery.ajax() settings key/value pairs.
+ * @param {string} [options.server] - A server to which send the request.
+ * @param {Function} [options.success] - A function to be called if the request succeeds.
+ *
+ * @returns {undefined}
+ */
+ ui.query = function (url, options) {
+ // Force options to be an object
+ const o = options || {};
+ Object.keys(o).forEach((option) => {
+ if (["complete", "data", "error", "errorMessage", "errorOnceId", "headers", "method", "params", "server",
+ "statusCode", "success"]
+ .indexOf(option) < 0) {
+ throw new Error("Unknown option: " + option);
+ }
+ });
+
+ let neighbours_status = [{
+ name: "local",
+ host: "local",
+ url: "",
+ }];
+ o.server = o.server || ui.getSelector("selSrv");
+ if (o.server === "All SERVERS") {
+ queryServer(neighbours_status, 0, "neighbours", {
+ success: function (json) {
+ const [{data}] = json;
+ if (jQuery.isEmptyObject(data)) {
+ ui.neighbours = {
+ local: {
+ host: window.location.host,
+ url: window.location.origin + window.location.pathname
+ }
+ };
+ } else {
+ ui.neighbours = data;
+ }
+ neighbours_status = [];
+ $.each(ui.neighbours, (ind) => {
+ neighbours_status.push({
+ name: ind,
+ host: ui.neighbours[ind].host,
+ url: ui.neighbours[ind].url,
+ });
+ });
+ $.each(neighbours_status, (ind) => {
+ queryServer(neighbours_status, ind, url, o);
+ });
+ },
+ errorMessage: "Cannot receive neighbours data"
+ });
+ } else {
+ if (o.server !== "local") {
+ neighbours_status = [{
+ name: o.server,
+ host: ui.neighbours[o.server].host,
+ url: ui.neighbours[o.server].url,
+ }];
+ }
+ queryServer(neighbours_status, 0, url, o);
+ }
+ };
+
+ ui.escapeHTML = function (string) {
+ const htmlEscaper = /[&<>"'/`=]/g;
+ const htmlEscapes = {
+ "&": "&amp;",
+ "<": "&lt;",
+ ">": "&gt;",
+ "\"": "&quot;",
+ "'": "&#39;",
+ "/": "&#x2F;",
+ "`": "&#x60;",
+ "=": "&#x3D;"
+ };
+ return String(string).replace(htmlEscaper, (match) => htmlEscapes[match]);
+ };
+
+ return ui;
+ });
diff --git a/interface/js/app/config.js b/interface/js/app/config.js
new file mode 100644
index 0000000..6be1075
--- /dev/null
+++ b/interface/js/app/config.js
@@ -0,0 +1,246 @@
+/*
+ The MIT License (MIT)
+
+ Copyright (C) 2017 Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+
+/* global require */
+
+define(["jquery", "app/common"],
+ ($, common) => {
+ "use strict";
+ const ui = {};
+
+ ui.getActions = function getActions(checked_server) {
+ common.query("actions", {
+ success: function (data) {
+ $("#actionsFormField").empty();
+ const items = [];
+ $.each(data[0].data, (i, item) => {
+ const actionsOrder = ["greylist", "add header", "rewrite subject", "reject"];
+ const idx = actionsOrder.indexOf(item.action);
+ if (idx >= 0) {
+ items.push({
+ idx: idx,
+ html:
+ '<div class="form-group">' +
+ '<label class="col-form-label col-md-2 float-start">' + item.action + "</label>" +
+ '<div class="controls slider-controls col-md-10">' +
+ '<input class="action-scores form-control" data-id="action" type="number" value="' +
+ item.value + '">' +
+ "</div>" +
+ "</div>"
+ });
+ }
+ });
+
+ items.sort((a, b) => a.idx - b.idx);
+
+ $("#actionsFormField").html(
+ items.map((e) => e.html).join(""));
+ },
+ server: (checked_server === "All SERVERS") ? "local" : checked_server
+ });
+ };
+
+ ui.saveActions = function (server) {
+ function descending(arr) {
+ let desc = true;
+ const filtered = arr.filter((el) => el !== null);
+ for (let i = 0; i < filtered.length - 1; i++) {
+ if (filtered[i + 1] >= filtered[i]) {
+ desc = false;
+ break;
+ }
+ }
+ return desc;
+ }
+
+ const elts = (function () {
+ const values = [];
+ const inputs = $("#actionsForm :input[data-id=\"action\"]");
+ // Rspamd order: [spam, rewrite_subject, probable_spam, greylist]
+ values[0] = parseFloat(inputs[3].value);
+ values[1] = parseFloat(inputs[2].value);
+ values[2] = parseFloat(inputs[1].value);
+ values[3] = parseFloat(inputs[0].value);
+
+ return JSON.stringify(values);
+ }());
+ // String to array for comparison
+ const eltsArray = JSON.parse(elts);
+ if (eltsArray[0] < 0) {
+ common.alertMessage("alert-modal alert-error", "Spam can not be negative");
+ } else if (eltsArray[1] < 0) {
+ common.alertMessage("alert-modal alert-error", "Rewrite subject can not be negative");
+ } else if (eltsArray[2] < 0) {
+ common.alertMessage("alert-modal alert-error", "Probable spam can not be negative");
+ } else if (eltsArray[3] < 0) {
+ common.alertMessage("alert-modal alert-error", "Greylist can not be negative");
+ } else if (descending(eltsArray)) {
+ common.query("saveactions", {
+ method: "POST",
+ params: {
+ data: elts,
+ dataType: "json"
+ },
+ server: server
+ });
+ } else {
+ common.alertMessage("alert-modal alert-error", "Incorrect order of actions thresholds");
+ }
+ };
+
+ ui.getMaps = function (checked_server) {
+ const $listmaps = $("#listMaps");
+ $listmaps.closest(".card").hide();
+ common.query("maps", {
+ success: function (json) {
+ const [{data}] = json;
+ $listmaps.empty();
+ $("#modalBody").empty();
+ const $tbody = $("<tbody>");
+
+ $.each(data, (i, item) => {
+ let $td = '<td><span class="badge text-bg-secondary">Read</span></td>';
+ if (!(item.editable === false || common.read_only)) {
+ $td = $($td).append('&nbsp;<span class="badge text-bg-success">Write</span>');
+ }
+ const $tr = $("<tr>").append($td);
+
+ const $span = $('<span class="map-link" data-bs-toggle="modal" data-bs-target="#modalDialog">' +
+ item.uri + "</span>").data("item", item);
+ $span.wrap("<td>").parent().appendTo($tr);
+ $("<td>" + item.description + "</td>").appendTo($tr);
+ $tr.appendTo($tbody);
+ });
+ $tbody.appendTo($listmaps);
+ $listmaps.closest(".card").show();
+ },
+ server: (checked_server === "All SERVERS") ? "local" : checked_server
+ });
+ };
+
+
+ let jar = {};
+ const editor = {
+ advanced: {
+ codejar: true,
+ elt: "div",
+ class: "editor language-clike",
+ readonly_attr: {contenteditable: false},
+ },
+ basic: {
+ elt: "textarea",
+ class: "form-control map-textarea",
+ readonly_attr: {readonly: true},
+ }
+ };
+ let mode = "advanced";
+
+ // Modal form for maps
+ $(document).on("click", "[data-bs-toggle=\"modal\"]", function () {
+ const checked_server = common.getSelector("selSrv");
+ const item = $(this).data("item");
+ common.query("getmap", {
+ headers: {
+ Map: item.map
+ },
+ success: function (data) {
+ // Highlighting a large amount of text is unresponsive
+ mode = (new Blob([data[0].data]).size > 5120) ? "basic" : $("input[name=editorMode]:checked").val();
+
+ $("<" + editor[mode].elt + ' id="editor" class="' + editor[mode].class + '" data-id="' + item.map +
+ '"></' + editor[mode].elt + ">").appendTo("#modalBody");
+
+ if (editor[mode].codejar) {
+ require(["codejar", "linenumbers", "prism"], (CodeJar, withLineNumbers, Prism) => {
+ jar = new CodeJar(
+ document.querySelector("#editor"),
+ withLineNumbers((el) => Prism.highlightElement(el))
+ );
+ jar.updateCode(data[0].data);
+ });
+ } else {
+ document.querySelector("#editor").innerHTML = common.escapeHTML(data[0].data);
+ }
+
+ let icon = "fa-edit";
+ if (item.editable === false || common.read_only) {
+ $("#editor").attr(editor[mode].readonly_attr);
+ icon = "fa-eye";
+ $("#modalSaveGroup").hide();
+ } else {
+ $("#modalSaveGroup").show();
+ }
+ $("#modalDialog .modal-header").find("[data-fa-i2svg]").addClass(icon);
+ $("#modalTitle").html(item.uri);
+
+ $("#modalDialog").modal("show");
+ },
+ errorMessage: "Cannot receive maps data",
+ server: (checked_server === "All SERVERS") ? "local" : checked_server
+ });
+ return false;
+ });
+ $("#modalDialog").on("hidden.bs.modal", () => {
+ if (editor[mode].codejar) {
+ jar.destroy();
+ $(".codejar-wrap").remove();
+ } else {
+ $("#editor").remove();
+ }
+ });
+
+ $("#saveActionsBtn").on("click", () => {
+ ui.saveActions();
+ });
+ $("#saveActionsClusterBtn").on("click", () => {
+ ui.saveActions("All SERVERS");
+ });
+
+ function saveMap(server) {
+ common.query("savemap", {
+ success: function () {
+ common.alertMessage("alert-success", "Map data successfully saved");
+ $("#modalDialog").modal("hide");
+ },
+ errorMessage: "Save map error",
+ method: "POST",
+ headers: {
+ Map: $("#editor").data("id"),
+ },
+ params: {
+ data: editor[mode].codejar ? jar.toString() : $("#editor").val(),
+ dataType: "text",
+ },
+ server: server
+ });
+ }
+ $("#modalSave").on("click", () => {
+ saveMap();
+ });
+ $("#modalSaveAll").on("click", () => {
+ saveMap("All SERVERS");
+ });
+
+ return ui;
+ });
diff --git a/interface/js/app/graph.js b/interface/js/app/graph.js
new file mode 100644
index 0000000..71306f4
--- /dev/null
+++ b/interface/js/app/graph.js
@@ -0,0 +1,252 @@
+/*
+ The MIT License (MIT)
+
+ Copyright (C) 2017 Vsevolod Stakhov <vsevolod@highsecure.ru>
+ Copyright (C) 2017 Alexander Moisseev
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+
+/* global FooTable */
+
+define(["jquery", "app/common", "d3evolution", "d3pie", "d3", "footable"],
+ ($, common, D3Evolution, D3Pie, d3) => {
+ "use strict";
+
+ const rrd_pie_config = {
+ cornerRadius: 2,
+ size: {
+ canvasWidth: 400,
+ canvasHeight: 180,
+ pieInnerRadius: "50%",
+ pieOuterRadius: "80%"
+ },
+ labels: {
+ outer: {
+ format: "none"
+ },
+ inner: {
+ hideWhenLessThanPercentage: 8,
+ offset: 0
+ },
+ },
+ padAngle: 0.02,
+ pieCenterOffset: {
+ x: -120,
+ y: 10,
+ },
+ total: {
+ enabled: true
+ },
+ };
+
+ const ui = {};
+ let prevUnit = "msg/s";
+
+ ui.draw = function (graphs, neighbours, checked_server, type) {
+ const graph_options = {
+ title: "Rspamd throughput",
+ width: 1060,
+ height: 370,
+ yAxisLabel: "Message rate, msg/s",
+
+ legend: {
+ space: 140,
+ entries: common.chartLegend
+ }
+ };
+
+ function initGraph() {
+ const graph = new D3Evolution("graph", $.extend({}, graph_options, {
+ yScale: common.getSelector("selYScale"),
+ type: common.getSelector("selType"),
+ interpolate: common.getSelector("selInterpolate"),
+ convert: common.getSelector("selConvert"),
+ }));
+ $("#selYScale").change(function () {
+ graph.yScale(this.value);
+ });
+ $("#selConvert").change(function () {
+ graph.convert(this.value);
+ });
+ $("#selType").change(function () {
+ graph.type(this.value);
+ });
+ $("#selInterpolate").change(function () {
+ graph.interpolate(this.value);
+ });
+
+ return graph;
+ }
+
+ function getRrdSummary(json, scaleFactor) {
+ const xExtents = d3.extent(d3.merge(json), (d) => d.x);
+ const timeInterval = xExtents[1] - xExtents[0];
+
+ let total = 0;
+ const rows = json.map((curr, i) => {
+ // Time intervals that don't have data are excluded from average calculation as d3.mean()ignores nulls
+ const avg = d3.mean(curr, (d) => d.y);
+ // To find an integral on the whole time interval we need to convert nulls to zeroes
+ // eslint-disable-next-line no-bitwise
+ const value = d3.mean(curr, (d) => Number(d.y)) * timeInterval / scaleFactor ^ 0;
+ const yExtents = d3.extent(curr, (d) => d.y);
+
+ total += value;
+ return {
+ label: graph_options.legend.entries[i].label,
+ value: value,
+ min: Number(yExtents[0].toFixed(6)),
+ avg: Number(avg.toFixed(6)),
+ max: Number(yExtents[1].toFixed(6)),
+ last: Number(curr[curr.length - 1].y.toFixed(6)),
+ color: graph_options.legend.entries[i].color,
+ };
+ }, []);
+
+ return {
+ rows: rows,
+ total: total
+ };
+ }
+
+ function initSummaryTable(rows, unit) {
+ common.tables.rrd_summary = FooTable.init("#rrd-table", {
+ sorting: {
+ enabled: true
+ },
+ columns: [
+ {name: "label", title: "Action"},
+ {name: "value", title: "Messages", defaultContent: ""},
+ {name: "min", title: "Minimum, <span class=\"unit\">" + unit + "</span>", defaultContent: ""},
+ {name: "avg", title: "Average, <span class=\"unit\">" + unit + "</span>", defaultContent: ""},
+ {name: "max", title: "Maximum, <span class=\"unit\">" + unit + "</span>", defaultContent: ""},
+ {name: "last", title: "Last, " + unit},
+ ],
+ rows: rows.map((curr, i) => ({
+ options: {
+ style: {
+ color: graph_options.legend.entries[i].color
+ }
+ },
+ value: curr
+ }), [])
+ });
+ }
+
+ function drawRrdTable(rows, unit) {
+ if (Object.prototype.hasOwnProperty.call(common.tables, "rrd_summary")) {
+ $.each(common.tables.rrd_summary.rows.all, (i, row) => {
+ row.val(rows[i], false, true);
+ });
+ } else {
+ initSummaryTable(rows, unit);
+ }
+ }
+
+ function updateWidgets(data) {
+ let rrd_summary = {rows: []};
+ let unit = "msg/s";
+
+ if (data) {
+ // Autoranging
+ let scaleFactor = 1;
+ const yMax = d3.max(d3.merge(data), (d) => d.y);
+ if (yMax < 1) {
+ scaleFactor = 60;
+ unit = "msg/min";
+ data.forEach((s) => {
+ s.forEach((d) => {
+ if (d.y !== null) { d.y *= scaleFactor; }
+ });
+ });
+ }
+
+ rrd_summary = getRrdSummary(data, scaleFactor);
+ }
+
+ if (!graphs.rrd_pie) graphs.rrd_pie = new D3Pie("rrd-pie", rrd_pie_config);
+ graphs.rrd_pie.data(rrd_summary.rows);
+
+ graphs.graph.data(data);
+ if (unit !== prevUnit) {
+ graphs.graph.yAxisLabel("Message rate, " + unit);
+ $(".unit").text(unit);
+ prevUnit = unit;
+ }
+ drawRrdTable(rrd_summary.rows, unit);
+ document.getElementById("rrd-total-value").innerHTML = rrd_summary.total;
+ }
+
+ if (!graphs.graph) {
+ graphs.graph = initGraph();
+ }
+
+
+ common.query("graph", {
+ success: function (req_data) {
+ let data = null;
+ const neighbours_data = req_data
+ .filter((d) => d.status) // filter out unavailable neighbours
+ .map((d) => d.data);
+
+ if (neighbours_data.length === 1) {
+ [data] = neighbours_data;
+ } else {
+ let time_match = true;
+ neighbours_data.reduce((res, curr, _, arr) => {
+ if ((curr[0][0].x !== res[0][0].x) ||
+ (curr[0][curr[0].length - 1].x !== res[0][res[0].length - 1].x)) {
+ time_match = false;
+ common.alertMessage("alert-error",
+ "Neighbours time extents do not match. Check if time is synchronized on all servers.");
+ arr.splice(1); // Break out of .reduce() by mutating the source array
+ }
+ return curr;
+ });
+
+ if (time_match) {
+ data = neighbours_data.reduce((res, curr) => curr.map((action, j) => action.map((d, i) => ({
+ x: d.x,
+ y: (res[j][i].y === null) ? d.y : res[j][i].y + d.y
+ }))));
+ }
+ }
+ updateWidgets(data);
+ },
+ complete: function () { $("#refresh").removeAttr("disabled").removeClass("disabled"); },
+ errorMessage: "Cannot receive throughput data",
+ errorOnceId: "alerted_graph_",
+ data: {type: type}
+ });
+ };
+
+
+ // Handling mouse events on overlapping elements
+ $("#rrd-pie").mouseover(() => {
+ $("#rrd-pie,#rrd-pie-tooltip").css("z-index", "200");
+ $("#rrd-table_toggle").css("z-index", "300");
+ });
+ $("#rrd-table_toggle").mouseover(() => {
+ $("#rrd-pie,#rrd-pie-tooltip").css("z-index", "0");
+ $("#rrd-table_toggle").css("z-index", "0");
+ });
+
+ return ui;
+ });
diff --git a/interface/js/app/history.js b/interface/js/app/history.js
new file mode 100644
index 0000000..0d953ec
--- /dev/null
+++ b/interface/js/app/history.js
@@ -0,0 +1,306 @@
+/*
+ The MIT License (MIT)
+
+ Copyright (C) 2017 Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+
+/* global FooTable */
+
+define(["jquery", "app/common", "app/libft", "footable"],
+ ($, common, libft) => {
+ "use strict";
+ const ui = {};
+ let prevVersion = null;
+
+ function process_history_legacy(data) {
+ const items = [];
+
+ function compare(e1, e2) { return e1.name.localeCompare(e2.name); }
+
+ $("#selSymOrder_history, label[for='selSymOrder_history']").hide();
+
+ $.each(data, (i, item) => {
+ item.time = libft.unix_time_format(item.unix_time);
+ libft.preprocess_item(item);
+ item.symbols = Object.keys(item.symbols)
+ .map((key) => item.symbols[key])
+ .sort(compare)
+ .map((e) => e.name)
+ .join(", ");
+ item.time = {
+ value: libft.unix_time_format(item.unix_time),
+ options: {
+ sortValue: item.unix_time
+ }
+ };
+
+ items.push(item);
+ });
+
+ return {items: items};
+ }
+
+ function columns_legacy() {
+ return [{
+ name: "id",
+ title: "ID",
+ style: {
+ width: 300,
+ maxWidth: 300,
+ overflow: "hidden",
+ textOverflow: "ellipsis",
+ wordBreak: "keep-all",
+ whiteSpace: "nowrap"
+ }
+ }, {
+ name: "ip",
+ title: "IP address",
+ breakpoints: "xs sm",
+ style: {width: 150, maxWidth: 150}
+ }, {
+ name: "action",
+ title: "Action",
+ style: {width: 110, maxWidth: 110}
+ }, {
+ name: "score",
+ title: "Score",
+ style: {maxWidth: 110},
+ sortValue: function (val) { return Number(val.options.sortValue); }
+ }, {
+ name: "symbols",
+ title: "Symbols",
+ breakpoints: "all",
+ style: {width: 550, maxWidth: 550}
+ }, {
+ name: "size",
+ title: "Message size",
+ breakpoints: "xs sm",
+ style: {width: 120, maxWidth: 120},
+ formatter: libft.formatBytesIEC
+ }, {
+ name: "scan_time",
+ title: "Scan time",
+ breakpoints: "xs sm",
+ style: {maxWidth: 80},
+ sortValue: function (val) { return Number(val); }
+ }, {
+ sorted: true,
+ direction: "DESC",
+ name: "time",
+ title: "Time",
+ sortValue: function (val) { return Number(val.options.sortValue); }
+ }, {
+ name: "user",
+ title: "Authenticated user",
+ breakpoints: "xs sm",
+ style: {width: 200, maxWidth: 200}
+ }];
+ }
+
+ const columns = {
+ 2: libft.columns_v2("history"),
+ legacy: columns_legacy()
+ };
+
+ function process_history_data(data) {
+ const process_functions = {
+ 2: libft.process_history_v2,
+ legacy: process_history_legacy
+ };
+ let pf = process_functions.legacy;
+
+ if (data.version) {
+ const strkey = data.version.toString();
+ if (process_functions[strkey]) {
+ pf = process_functions[strkey];
+ }
+ }
+
+ return pf(data, "history");
+ }
+
+ function get_history_columns(data) {
+ let func = columns.legacy;
+
+ if (data.version) {
+ const strkey = data.version.toString();
+ if (columns[strkey]) {
+ func = columns[strkey];
+ }
+ }
+
+ return func;
+ }
+
+ ui.getHistory = function () {
+ common.query("history", {
+ success: function (req_data) {
+ function differentVersions(neighbours_data) {
+ const dv = neighbours_data.some((e) => e.version !== neighbours_data[0].version);
+ if (dv) {
+ common.alertMessage("alert-error",
+ "Neighbours history backend versions do not match. Cannot display history.");
+ return true;
+ }
+ return false;
+ }
+
+ const neighbours_data = req_data
+ .filter((d) => d.status) // filter out unavailable neighbours
+ .map((d) => d.data);
+ if (neighbours_data.length && !differentVersions(neighbours_data)) {
+ let data = {};
+ const [{version}] = neighbours_data;
+ if (version) {
+ data.rows = [].concat.apply([], neighbours_data
+ .map((e) => e.rows));
+ data.version = version;
+ $("#legacy-history-badge").hide();
+ } else {
+ // Legacy version
+ data = [].concat.apply([], neighbours_data);
+ $("#legacy-history-badge").show();
+ }
+ const o = process_history_data(data);
+ const {items} = o;
+ common.symbols.history = o.symbols;
+
+ if (Object.prototype.hasOwnProperty.call(common.tables, "history") &&
+ version === prevVersion) {
+ common.tables.history.rows.load(items);
+ } else {
+ libft.destroyTable("history");
+ // Is there a way to get an event when the table is destroyed?
+ setTimeout(() => {
+ libft.initHistoryTable(data, items, "history", get_history_columns(data), false);
+ }, 200);
+ }
+ prevVersion = version;
+ } else {
+ libft.destroyTable("history");
+ }
+ },
+ complete: function () { $("#refresh").removeAttr("disabled").removeClass("disabled"); },
+ errorMessage: "Cannot receive history",
+ });
+ };
+
+ function initErrorsTable(rows) {
+ common.tables.errors = FooTable.init("#errorsLog", {
+ columns: [
+ {sorted: true,
+ direction: "DESC",
+ name: "ts",
+ title: "Time",
+ style: {width: 300, maxWidth: 300},
+ sortValue: function (val) { return Number(val.options.sortValue); }},
+ {name: "type",
+ title: "Worker type",
+ breakpoints: "xs sm",
+ style: {width: 150, maxWidth: 150}},
+ {name: "pid",
+ title: "PID",
+ breakpoints: "xs sm",
+ style: {width: 110, maxWidth: 110}},
+ {name: "module", title: "Module"},
+ {name: "id", title: "Internal ID"},
+ {name: "message", title: "Message", breakpoints: "xs sm"},
+ ],
+ rows: rows,
+ paging: {
+ enabled: true,
+ limit: 5,
+ size: common.page_size.errors
+ },
+ filtering: {
+ enabled: true,
+ position: "left",
+ connectors: false
+ },
+ sorting: {
+ enabled: true
+ }
+ });
+ }
+
+ ui.getErrors = function () {
+ if (common.read_only) return;
+
+ common.query("errors", {
+ success: function (data) {
+ const neighbours_data = data
+ .filter((d) => d.status) // filter out unavailable neighbours
+ .map((d) => d.data);
+ const rows = [].concat.apply([], neighbours_data);
+ $.each(rows, (i, item) => {
+ item.ts = {
+ value: libft.unix_time_format(item.ts),
+ options: {
+ sortValue: item.ts
+ }
+ };
+ });
+ if (Object.prototype.hasOwnProperty.call(common.tables, "errors")) {
+ common.tables.errors.rows.load(rows);
+ } else {
+ initErrorsTable(rows);
+ }
+ }
+ });
+
+ $("#updateErrors").off("click");
+ $("#updateErrors").on("click", (e) => {
+ e.preventDefault();
+ ui.getErrors();
+ });
+ };
+
+
+ libft.set_page_size("history", $("#history_page_size").val());
+ libft.bindHistoryTableEventHandlers("history", 8);
+
+ $("#updateHistory").off("click");
+ $("#updateHistory").on("click", (e) => {
+ e.preventDefault();
+ ui.getHistory();
+ });
+
+ // @reset history log
+ $("#resetHistory").off("click");
+ $("#resetHistory").on("click", (e) => {
+ e.preventDefault();
+ if (!confirm("Are you sure you want to reset history log?")) { // eslint-disable-line no-alert
+ return;
+ }
+ libft.destroyTable("history");
+ libft.destroyTable("errors");
+
+ common.query("historyreset", {
+ success: function () {
+ ui.getHistory();
+ ui.getErrors();
+ },
+ errorMessage: "Cannot reset history log"
+ });
+ });
+
+ return ui;
+ });
diff --git a/interface/js/app/libft.js b/interface/js/app/libft.js
new file mode 100644
index 0000000..c69f74d
--- /dev/null
+++ b/interface/js/app/libft.js
@@ -0,0 +1,524 @@
+/* global FooTable */
+
+define(["jquery", "app/common", "footable"],
+ ($, common) => {
+ "use strict";
+ const ui = {};
+
+ let pageSizeTimerId = null;
+ let pageSizeInvocationCounter = 0;
+
+ function get_compare_function(table) {
+ const compare_functions = {
+ magnitude: function (e1, e2) {
+ return Math.abs(e2.score) - Math.abs(e1.score);
+ },
+ name: function (e1, e2) {
+ return e1.name.localeCompare(e2.name);
+ },
+ score: function (e1, e2) {
+ return e2.score - e1.score;
+ }
+ };
+
+ return compare_functions[common.getSelector("selSymOrder_" + table)];
+ }
+
+ function sort_symbols(o, compare_function) {
+ return Object.keys(o)
+ .map((key) => o[key])
+ .sort(compare_function)
+ .map((e) => e.str)
+ .join("<br>\n");
+ }
+
+
+ // Public functions
+
+ ui.formatBytesIEC = function (bytes) {
+ // FooTable represents data as text even column type is "number".
+ if (!Number.isInteger(Number(bytes)) || bytes < 0) return "NaN";
+
+ const base = 1024;
+ const exponent = Math.floor(Math.log(bytes) / Math.log(base));
+
+ if (exponent > 8) return "∞";
+
+ const value = parseFloat((bytes / (base ** exponent)).toPrecision(3));
+ let unit = "BKMGTPEZY"[exponent];
+ if (exponent) unit += "iB";
+
+ return value + " " + unit;
+ };
+
+ ui.columns_v2 = function (table) {
+ return [{
+ name: "id",
+ title: "ID",
+ style: {
+ minWidth: 130,
+ overflow: "hidden",
+ textOverflow: "ellipsis",
+ wordBreak: "break-all",
+ whiteSpace: "normal"
+ }
+ }, {
+ name: "ip",
+ title: "IP address",
+ breakpoints: "xs sm md",
+ style: {
+ "minWidth": "calc(7.6em + 8px)",
+ "word-break": "break-all"
+ },
+ // Normalize IPv4
+ sortValue: (ip) => ((typeof ip === "string") ? ip.split(".").map((x) => x.padStart(3, "0")).join("") : "0")
+ }, {
+ name: "sender_mime",
+ title: "[Envelope From] From",
+ breakpoints: "xs sm md",
+ style: {
+ "minWidth": 100,
+ "maxWidth": 200,
+ "word-wrap": "break-word"
+ }
+ }, {
+ name: "rcpt_mime_short",
+ title: "[Envelope To] To/Cc/Bcc",
+ breakpoints: "xs sm md",
+ filterable: false,
+ classes: "d-none d-xl-table-cell",
+ style: {
+ "minWidth": 100,
+ "maxWidth": 200,
+ "word-wrap": "break-word"
+ }
+ }, {
+ name: "rcpt_mime",
+ title: "[Envelope To] To/Cc/Bcc",
+ breakpoints: "all",
+ style: {"word-wrap": "break-word"}
+ }, {
+ name: "subject",
+ title: "Subject",
+ breakpoints: "xs sm md",
+ style: {
+ "word-break": "break-all",
+ "minWidth": 150
+ }
+ }, {
+ name: "action",
+ title: "Action",
+ style: {minwidth: 82}
+ }, {
+ name: "passthrough_module",
+ title: '<div title="The module that has set the pre-result">Pass-through module</div>',
+ breakpoints: "xs sm md"
+ }, {
+ name: "score",
+ title: "Score",
+ style: {
+ "maxWidth": 110,
+ "text-align": "right",
+ "white-space": "nowrap"
+ },
+ sortValue: function (val) { return Number(val.options.sortValue); }
+ }, {
+ name: "symbols",
+ title: "Symbols" +
+ '<div class="sym-order-toggle">' +
+ '<br><span style="font-weight:normal;">Sort by:</span><br>' +
+ '<div class="btn-group btn-group-xs btn-sym-order-' + table + '">' +
+ '<label type="button" class="btn btn-outline-secondary btn-sym-' + table + '-magnitude">' +
+ '<input type="radio" class="btn-check" value="magnitude">Magnitude</label>' +
+ '<label type="button" class="btn btn-outline-secondary btn-sym-' + table + '-score">' +
+ '<input type="radio" class="btn-check" value="score">Value</label>' +
+ '<label type="button" class="btn btn-outline-secondary btn-sym-' + table + '-name">' +
+ '<input type="radio" class="btn-check" value="name">Name</label>' +
+ "</div>" +
+ "</div>",
+ breakpoints: "all",
+ style: {width: 550, maxWidth: 550}
+ }, {
+ name: "size",
+ title: "Msg size",
+ breakpoints: "xs sm md",
+ style: {minwidth: 50},
+ formatter: ui.formatBytesIEC
+ }, {
+ name: "time_real",
+ title: "Scan time",
+ breakpoints: "xs sm md",
+ style: {maxWidth: 72},
+ sortValue: function (val) { return Number(val); }
+ }, {
+ classes: "history-col-time",
+ sorted: true,
+ direction: "DESC",
+ name: "time",
+ title: "Time",
+ sortValue: function (val) { return Number(val.options.sortValue); }
+ }, {
+ name: "user",
+ title: "Authenticated user",
+ breakpoints: "xs sm md",
+ style: {
+ "minWidth": 100,
+ "maxWidth": 130,
+ "word-wrap": "break-word"
+ }
+ }].filter((col) => {
+ switch (table) {
+ case "history":
+ return (col.name !== "passthrough_module");
+ case "scan":
+ return ["ip", "sender_mime", "rcpt_mime_short", "rcpt_mime", "subject", "size", "user"]
+ .every((name) => col.name !== name);
+ default:
+ return null;
+ }
+ });
+ };
+
+ ui.set_page_size = function (table, page_size, changeTablePageSize) {
+ const n = parseInt(page_size, 10); // HTML Input elements return string representing a number
+ if (n > 0) {
+ common.page_size[table] = n;
+
+ if (changeTablePageSize &&
+ $("#historyTable_" + table + " tbody").is(":parent")) { // Table is not empty
+ clearTimeout(pageSizeTimerId);
+ const t = FooTable.get("#historyTable_" + table);
+ if (t) {
+ pageSizeInvocationCounter = 0;
+ // Wait for input finish
+ pageSizeTimerId = setTimeout(() => t.pageSize(n), 1000);
+ } else if (++pageSizeInvocationCounter < 10) {
+ // Wait for FooTable instance ready
+ pageSizeTimerId = setTimeout(() => ui.set_page_size(table, n, true), 1000);
+ }
+ }
+ }
+ };
+
+ ui.bindHistoryTableEventHandlers = function (table, symbolsCol) {
+ function change_symbols_order(order) {
+ $(".btn-sym-" + table + "-" + order).addClass("active").siblings().removeClass("active");
+ const compare_function = get_compare_function(table);
+ $.each(common.tables[table].rows.all, (i, row) => {
+ const cell_val = sort_symbols(common.symbols[table][i], compare_function);
+ row.cells[symbolsCol].val(cell_val, false, true);
+ });
+ }
+
+ $("#selSymOrder_" + table).unbind().change(function () {
+ const order = this.value;
+ change_symbols_order(order);
+ });
+ $("#" + table + "_page_size").change((e) => ui.set_page_size(table, e.target.value, true));
+ $(document).on("click", ".btn-sym-order-" + table + " input", function () {
+ const order = this.value;
+ $("#selSymOrder_" + table).val(order);
+ change_symbols_order(order);
+ });
+ };
+
+ ui.destroyTable = function (table) {
+ if (common.tables[table]) {
+ common.tables[table].destroy();
+ delete common.tables[table];
+ }
+ };
+
+ ui.initHistoryTable = function (data, items, table, columns, expandFirst) {
+ /* eslint-disable no-underscore-dangle */
+ FooTable.Cell.extend("collapse", function () {
+ // call the original method
+ this._super();
+ // Copy cell classes to detail row tr element
+ this._setClasses(this.$detail);
+ });
+ /* eslint-enable no-underscore-dangle */
+
+ /* eslint-disable consistent-this, no-underscore-dangle, one-var-declaration-per-line */
+ FooTable.actionFilter = FooTable.Filtering.extend({
+ construct: function (instance) {
+ this._super(instance);
+ this.actions = ["reject", "add header", "greylist",
+ "no action", "soft reject", "rewrite subject"];
+ this.def = "Any action";
+ this.$action = null;
+ },
+ $create: function () {
+ this._super();
+ const self = this;
+ const $form_grp = $("<div/>", {
+ class: "form-group d-inline-flex align-items-center"
+ }).append($("<label/>", {
+ class: "sr-only",
+ text: "Action"
+ })).prependTo(self.$form);
+
+ $("<div/>", {
+ class: "form-check form-check-inline",
+ title: "Invert action match."
+ }).append(
+ self.$not = $("<input/>", {
+ type: "checkbox",
+ class: "form-check-input",
+ id: "not_" + table
+ }).on("change", {self: self}, self._onStatusDropdownChanged),
+ $("<label/>", {
+ class: "form-check-label",
+ for: "not_" + table,
+ text: "not"
+ })
+ ).appendTo($form_grp);
+
+ self.$action = $("<select/>", {
+ class: "form-select"
+ }).on("change", {
+ self: self
+ }, self._onStatusDropdownChanged).append(
+ $("<option/>", {
+ text: self.def
+ })).appendTo($form_grp);
+
+ $.each(self.actions, (i, action) => {
+ self.$action.append($("<option/>").text(action));
+ });
+ },
+ _onStatusDropdownChanged: function (e) {
+ const {self} = e.data;
+ const selected = self.$action.val();
+ if (selected !== self.def) {
+ const not = self.$not.is(":checked");
+ let query = null;
+
+ if (selected === "reject") {
+ query = not ? "-reject OR soft" : "reject -soft";
+ } else {
+ query = not ? selected.replace(/(\b\w+\b)/g, "-$1") : selected;
+ }
+
+ self.addFilter("action", query, ["action"]);
+ } else {
+ self.removeFilter("action");
+ }
+ self.filter();
+ }
+ });
+ /* eslint-enable consistent-this, no-underscore-dangle, one-var-declaration-per-line */
+
+ common.tables[table] = FooTable.init("#historyTable_" + table, {
+ columns: columns,
+ rows: items,
+ expandFirst: expandFirst,
+ paging: {
+ enabled: true,
+ limit: 5,
+ size: common.page_size[table]
+ },
+ filtering: {
+ enabled: true,
+ position: "left",
+ connectors: false
+ },
+ sorting: {
+ enabled: true
+ },
+ components: {
+ filtering: FooTable.actionFilter
+ },
+ on: {
+ "expand.ft.row": function (e, ft, row) {
+ setTimeout(() => {
+ const detail_row = row.$el.next();
+ const order = common.getSelector("selSymOrder_" + table);
+ detail_row.find(".btn-sym-" + table + "-" + order)
+ .addClass("active").siblings().removeClass("active");
+ }, 5);
+ }
+ }
+ });
+ };
+
+ ui.preprocess_item = function (item) {
+ function escape_HTML_array(arr) {
+ arr.forEach((d, i) => { arr[i] = common.escapeHTML(d); });
+ }
+
+ for (const prop in item) {
+ if (!{}.hasOwnProperty.call(item, prop)) continue;
+ switch (prop) {
+ case "rcpt_mime":
+ case "rcpt_smtp":
+ escape_HTML_array(item[prop]);
+ break;
+ case "symbols":
+ Object.keys(item.symbols).forEach((key) => {
+ const sym = item.symbols[key];
+ if (!sym.name) {
+ sym.name = key;
+ }
+ sym.name = common.escapeHTML(sym.name);
+ if (sym.description) {
+ sym.description = common.escapeHTML(sym.description);
+ }
+
+ if (sym.options) {
+ escape_HTML_array(sym.options);
+ }
+ });
+ break;
+ default:
+ if (typeof item[prop] === "string") {
+ item[prop] = common.escapeHTML(item[prop]);
+ }
+ }
+ }
+
+ if (item.action === "clean" || item.action === "no action") {
+ item.action = "<div style='font-size:11px' class='badge text-bg-success'>" + item.action + "</div>";
+ } else if (item.action === "rewrite subject" || item.action === "add header" || item.action === "probable spam") {
+ item.action = "<div style='font-size:11px' class='badge text-bg-warning'>" + item.action + "</div>";
+ } else if (item.action === "spam" || item.action === "reject") {
+ item.action = "<div style='font-size:11px' class='badge text-bg-danger'>" + item.action + "</div>";
+ } else {
+ item.action = "<div style='font-size:11px' class='badge text-bg-info'>" + item.action + "</div>";
+ }
+
+ const score_content = (item.score < item.required_score)
+ ? "<span class='text-success'>" + item.score.toFixed(2) + " / " + item.required_score + "</span>"
+ : "<span class='text-danger'>" + item.score.toFixed(2) + " / " + item.required_score + "</span>";
+
+ item.score = {
+ options: {
+ sortValue: item.score
+ },
+ value: score_content
+ };
+ };
+
+ ui.unix_time_format = function (tm) {
+ const date = new Date(tm ? tm * 1000 : 0);
+ return (common.locale)
+ ? date.toLocaleString(common.locale)
+ : date.toLocaleString();
+ };
+
+ ui.process_history_v2 = function (data, table) {
+ // Display no more than rcpt_lim recipients
+ const rcpt_lim = 3;
+ const items = [];
+ const unsorted_symbols = [];
+ const compare_function = get_compare_function(table);
+
+ $("#selSymOrder_" + table + ", label[for='selSymOrder_" + table + "']").show();
+
+ $.each(data.rows,
+ (i, item) => {
+ function more(p) {
+ const l = item[p].length;
+ return (l > rcpt_lim) ? " … (" + l + ")" : "";
+ }
+ function format_rcpt(smtp, mime) {
+ let full = "";
+ let shrt = "";
+ if (smtp) {
+ full = "[" + item.rcpt_smtp.join(", ") + "] ";
+ shrt = "[" + item.rcpt_smtp.slice(0, rcpt_lim).join(",&#8203;") + more("rcpt_smtp") + "]";
+ if (mime) {
+ full += " ";
+ shrt += " ";
+ }
+ }
+ if (mime) {
+ full += item.rcpt_mime.join(", ");
+ shrt += item.rcpt_mime.slice(0, rcpt_lim).join(",&#8203;") + more("rcpt_mime");
+ }
+ return {full: full, shrt: shrt};
+ }
+
+ function get_symbol_class(name, score) {
+ if (name.match(/^GREYLIST$/)) {
+ return "symbol-special";
+ }
+
+ if (score < 0) {
+ return "symbol-negative";
+ } else if (score > 0) {
+ return "symbol-positive";
+ }
+ return null;
+ }
+
+ ui.preprocess_item(item);
+ Object.values(item.symbols).forEach((sym) => {
+ sym.str = '<span class="symbol-default ' + get_symbol_class(sym.name, sym.score) + '"><strong>';
+
+ if (sym.description) {
+ sym.str += '<abbr title="' + sym.description + '">' + sym.name + "</abbr>";
+ } else {
+ sym.str += sym.name;
+ }
+ sym.str += "</strong> (" + sym.score + ")</span>";
+
+ if (sym.options) {
+ sym.str += " [" + sym.options.join(",") + "]";
+ }
+ });
+ unsorted_symbols.push(item.symbols);
+ item.symbols = sort_symbols(item.symbols, compare_function);
+ if (table === "scan") {
+ item.unix_time = (new Date()).getTime() / 1000;
+ }
+ item.time = {
+ value: ui.unix_time_format(item.unix_time),
+ options: {
+ sortValue: item.unix_time
+ }
+ };
+ item.time_real = item.time_real.toFixed(3);
+ item.id = item["message-id"];
+
+ if (table === "history") {
+ let rcpt = {};
+ if (!item.rcpt_mime.length) {
+ rcpt = format_rcpt(true, false);
+ } else if (
+ $(item.rcpt_mime).not(item.rcpt_smtp).length !== 0 ||
+ $(item.rcpt_smtp).not(item.rcpt_mime).length !== 0
+ ) {
+ rcpt = format_rcpt(true, true);
+ } else {
+ rcpt = format_rcpt(false, true);
+ }
+ item.rcpt_mime_short = rcpt.shrt;
+ item.rcpt_mime = rcpt.full;
+
+ if (item.sender_mime !== item.sender_smtp) {
+ item.sender_mime = "[" + item.sender_smtp + "] " + item.sender_mime;
+ }
+ }
+ items.push(item);
+ });
+
+ return {items: items, symbols: unsorted_symbols};
+ };
+
+ ui.waitForRowsDisplayed = function (table, rows_total, callback, iteration) {
+ let i = (typeof iteration === "undefined") ? 10 : iteration;
+ const num_rows = $("#historyTable_" + table + " > tbody > tr:not(.footable-detail-row)").length;
+ if (num_rows === common.page_size[table] ||
+ num_rows === rows_total) {
+ return callback();
+ } else if (--i) {
+ setTimeout(() => {
+ ui.waitForRowsDisplayed(table, rows_total, callback, i);
+ }, 500);
+ }
+ return null;
+ };
+
+ return ui;
+ });
diff --git a/interface/js/app/rspamd.js b/interface/js/app/rspamd.js
new file mode 100644
index 0000000..938f048
--- /dev/null
+++ b/interface/js/app/rspamd.js
@@ -0,0 +1,486 @@
+/*
+ The MIT License (MIT)
+
+ Copyright (C) 2012-2013 Anton Simonov <untone@gmail.com>
+ Copyright (C) 2014-2017 Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+
+/* global require, Visibility */
+
+define(["jquery", "app/common", "stickytabs", "visibility",
+ "bootstrap", "fontawesome"],
+($, common) => {
+ "use strict";
+ const ui = {};
+
+ const defaultAjaxTimeout = 20000;
+
+ const ajaxTimeoutBox = ".popover #settings-popover #ajax-timeout";
+ const graphs = {};
+ let checked_server = "All SERVERS";
+ const timer_id = [];
+
+ function ajaxSetup(ajax_timeout, setFieldValue, saveToLocalStorage) {
+ const timeout = (ajax_timeout && ajax_timeout >= 0) ? ajax_timeout : defaultAjaxTimeout;
+ if (saveToLocalStorage) localStorage.setItem("ajax_timeout", timeout);
+ if (setFieldValue) $(ajaxTimeoutBox).val(timeout);
+
+ $.ajaxSetup({
+ timeout: timeout,
+ jsonp: false
+ });
+ }
+
+ function cleanCredentials() {
+ sessionStorage.clear();
+ $("#statWidgets").empty();
+ $("#listMaps").empty();
+ $("#modalBody").empty();
+ }
+
+ function stopTimers() {
+ for (const key in timer_id) {
+ if (!{}.hasOwnProperty.call(timer_id, key)) continue;
+ Visibility.stop(timer_id[key]);
+ }
+ }
+
+ function disconnect() {
+ [graphs, common.tables].forEach((o) => {
+ Object.keys(o).forEach((key) => {
+ o[key].destroy();
+ delete o[key];
+ });
+ });
+
+ // Remove jquery-stickytabs listeners
+ $(window).off("hashchange");
+ $(".nav-tabs-sticky > .nav-item > .nav-link").off("click").removeClass("active");
+
+ stopTimers();
+ cleanCredentials();
+ ui.connect();
+ }
+
+ function tabClick(id) {
+ let tab_id = id;
+ if ($(id).attr("disabled")) return;
+ let navBarControls = $("#selSrv, #navBar li, #navBar a, #navBar button");
+ if (id !== "#autoRefresh") navBarControls.attr("disabled", true).addClass("disabled", true);
+
+ stopTimers();
+
+ if (id === "#refresh" || id === "#autoRefresh") {
+ tab_id = "#" + $(".nav-link.active").attr("id");
+ }
+
+ $("#autoRefresh").hide();
+ $("#refresh").addClass("radius-right");
+
+ function setAutoRefresh(refreshInterval, timer, callback) {
+ function countdown(interval) {
+ Visibility.stop(timer_id.countdown);
+ if (!interval) {
+ $("#countdown").text("--:--");
+ return;
+ }
+
+ let timeLeft = interval;
+ $("#countdown").text("00:00");
+ timer_id.countdown = Visibility.every(1000, 1000, () => {
+ timeLeft -= 1000;
+ $("#countdown").text(new Date(timeLeft).toISOString().substr(14, 5));
+ if (timeLeft <= 0) Visibility.stop(timer_id.countdown);
+ });
+ }
+
+ $("#refresh").removeClass("radius-right");
+ $("#autoRefresh").show();
+
+ countdown(refreshInterval);
+ if (!refreshInterval) return;
+ timer_id[timer] = Visibility.every(refreshInterval, () => {
+ countdown(refreshInterval);
+ if ($("#refresh").attr("disabled")) return;
+ $("#refresh").attr("disabled", true).addClass("disabled", true);
+ callback();
+ });
+ }
+
+ if (["#scan_nav", "#selectors_nav", "#disconnect"].indexOf(tab_id) !== -1) {
+ $("#refresh").hide();
+ } else {
+ $("#refresh").show();
+ }
+
+ switch (tab_id) {
+ case "#status_nav":
+ require(["app/stats"], (module) => {
+ const refreshInterval = $(".dropdown-menu a.active.preset").data("value");
+ setAutoRefresh(refreshInterval, "status",
+ () => module.statWidgets(graphs, checked_server));
+ if (id !== "#autoRefresh") module.statWidgets(graphs, checked_server);
+
+ $(".preset").show();
+ $(".history").hide();
+ $(".dynamic").hide();
+ });
+ break;
+ case "#throughput_nav":
+ require(["app/graph"], (module) => {
+ const selData = common.getSelector("selData"); // Graph's dataset selector state
+ const step = {
+ day: 60000,
+ week: 300000
+ };
+ let refreshInterval = step[selData] || 3600000;
+ $("#dynamic-item").text((refreshInterval / 60000) + " min");
+
+ if (!$(".dropdown-menu a.active.dynamic").data("value")) {
+ refreshInterval = null;
+ }
+ setAutoRefresh(refreshInterval, "throughput",
+ () => module.draw(graphs, common.neighbours, checked_server, selData));
+ if (id !== "#autoRefresh") module.draw(graphs, common.neighbours, checked_server, selData);
+
+ $(".preset").hide();
+ $(".history").hide();
+ $(".dynamic").show();
+ });
+ break;
+ case "#configuration_nav":
+ require(["app/config"], (module) => {
+ module.getActions(checked_server);
+ module.getMaps(checked_server);
+ });
+ break;
+ case "#symbols_nav":
+ require(["app/symbols"], (module) => module.getSymbols(checked_server));
+ break;
+ case "#scan_nav":
+ require(["app/upload"]);
+ break;
+ case "#selectors_nav":
+ require(["app/selectors"], (module) => module.displayUI());
+ break;
+ case "#history_nav":
+ require(["app/history"], (module) => {
+ function getHistoryAndErrors() {
+ module.getHistory();
+ module.getErrors();
+ }
+ const refreshInterval = $(".dropdown-menu a.active.history").data("value");
+ setAutoRefresh(refreshInterval, "history",
+ () => getHistoryAndErrors());
+ if (id !== "#autoRefresh") getHistoryAndErrors();
+
+ $(".preset").hide();
+ $(".history").show();
+ $(".dynamic").hide();
+ });
+ break;
+ case "#disconnect":
+ disconnect();
+ break;
+ default:
+ }
+
+ setTimeout(() => {
+ // Do not enable Refresh button until AJAX requests to all neighbours are finished
+ if (tab_id === "#history_nav") navBarControls = $(navBarControls).not("#refresh");
+
+ navBarControls.removeAttr("disabled").removeClass("disabled");
+ }, (id === "#autoRefresh") ? 0 : 1000);
+ }
+
+ function saveCredentials(password) {
+ sessionStorage.setItem("Password", password);
+ }
+
+ function displayUI() {
+ // In many browsers local storage can only store string.
+ // So when we store the boolean true or false, it actually stores the strings "true" or "false".
+ common.read_only = sessionStorage.getItem("read_only") === "true";
+
+ common.query("auth", {
+ success: function (neighbours_status) {
+ $("#selSrv").empty();
+ $("#selSrv").append($('<option value="All SERVERS">All SERVERS</option>'));
+ neighbours_status.forEach((e) => {
+ $("#selSrv").append($('<option value="' + e.name + '">' + e.name + "</option>"));
+ if (checked_server === e.name) {
+ $('#selSrv [value="' + e.name + '"]').prop("selected", true);
+ } else if (!e.status) {
+ $('#selSrv [value="' + e.name + '"]').prop("disabled", true);
+ }
+ });
+ },
+ complete: function () {
+ ajaxSetup(localStorage.getItem("ajax_timeout"));
+
+ if (common.read_only) {
+ $(".ro-disable").attr("disabled", true);
+ $(".ro-hide").hide();
+ } else {
+ $(".ro-disable").removeAttr("disabled", true);
+ $(".ro-hide").show();
+ }
+
+ $("#preloader").addClass("d-none");
+ $("#navBar, #mainUI").removeClass("d-none");
+ $(".nav-tabs-sticky").stickyTabs({initialTab: "#status_nav"});
+ },
+ errorMessage: "Cannot get server status",
+ server: "All SERVERS"
+ });
+ }
+
+
+ // Public functions
+
+ ui.connect = function () {
+ // Prevent locking out of the WebUI if timeout is too low.
+ let timeout = localStorage.getItem("ajax_timeout");
+ if (timeout < defaultAjaxTimeout) timeout = defaultAjaxTimeout;
+ ajaxSetup(timeout);
+
+ // Query "/stat" to check if user is already logged in or client ip matches "secure_ip"
+ $.ajax({
+ type: "GET",
+ url: "stat",
+ success: function (data) {
+ sessionStorage.setItem("read_only", data.read_only);
+ displayUI();
+ },
+ error: function () {
+ function clearFeedback() {
+ $("#connectPassword").off("input").removeClass("is-invalid");
+ $("#authInvalidCharFeedback,#authUnauthorizedFeedback").hide();
+ }
+
+ $("#connectDialog")
+ .on("show.bs.modal", () => {
+ $("#connectDialog").off("show.bs.modal");
+ clearFeedback();
+ })
+ .on("shown.bs.modal", () => {
+ $("#connectDialog").off("shown.bs.modal");
+ $("#connectPassword").focus();
+ })
+ .modal("show");
+
+ $("#connectForm").off("submit").on("submit", (e) => {
+ e.preventDefault();
+ const password = $("#connectPassword").val();
+
+ function invalidFeedback(tooltip) {
+ $("#connectPassword")
+ .addClass("is-invalid")
+ .off("input").on("input", () => clearFeedback());
+ $(tooltip).show();
+ }
+
+ if (!(/^[\u0020-\u007e]*$/).test(password)) {
+ invalidFeedback("#authInvalidCharFeedback");
+ $("#connectPassword").focus();
+ return;
+ }
+
+ common.query("auth", {
+ headers: {
+ Password: password
+ },
+ success: function (json) {
+ const [{data}] = json;
+ $("#connectPassword").val("");
+ if (data.auth === "ok") {
+ sessionStorage.setItem("read_only", data.read_only);
+ saveCredentials(password);
+ $("#connectForm").off("submit");
+ $("#connectDialog").modal("hide");
+ displayUI();
+ }
+ },
+ error: function (jqXHR, textStatus) {
+ if (textStatus.statusText === "Unauthorized") {
+ invalidFeedback("#authUnauthorizedFeedback");
+ } else {
+ common.alertMessage("alert-modal alert-error", textStatus.statusText);
+ }
+ $("#connectPassword").val("");
+ $("#connectPassword").focus();
+ },
+ params: {
+ global: false,
+ },
+ server: "local"
+ });
+ });
+ }
+ });
+ };
+
+
+ (function initSettings() {
+ let selected_locale = null;
+ let custom_locale = null;
+ const localeTextbox = ".popover #settings-popover #locale";
+
+ function validateLocale(saveToLocalStorage) {
+ function toggle_form_group_class(remove, add) {
+ $(localeTextbox).removeClass("is-" + remove).addClass("is-" + add);
+ }
+
+ const now = new Date();
+
+ if (custom_locale.length) {
+ try {
+ now.toLocaleString(custom_locale);
+
+ if (saveToLocalStorage) localStorage.setItem("custom_locale", custom_locale);
+ common.locale = (selected_locale === "custom") ? custom_locale : null;
+ toggle_form_group_class("invalid", "valid");
+ } catch (err) {
+ common.locale = null;
+ toggle_form_group_class("valid", "invalid");
+ }
+ } else {
+ if (saveToLocalStorage) localStorage.setItem("custom_locale", null);
+ common.locale = null;
+ $(localeTextbox).removeClass("is-valid is-invalid");
+ }
+
+ // Display date example
+ $(".popover #settings-popover #date-example").text(
+ (common.locale)
+ ? now.toLocaleString(common.locale)
+ : now.toLocaleString()
+ );
+ }
+
+ $("#settings").popover({
+ container: "body",
+ placement: "bottom",
+ html: true,
+ sanitize: false,
+ content: function () {
+ // Using .clone() has the side-effect of producing elements with duplicate id attributes.
+ return $("#settings-popover").clone();
+ }
+ // Restore the tooltip of the element that the popover is attached to.
+ }).attr("title", function () {
+ return $(this).attr("data-original-title");
+ });
+ $("#settings").on("click", (e) => {
+ e.preventDefault();
+ });
+ $("#settings").on("inserted.bs.popover", () => {
+ selected_locale = localStorage.getItem("selected_locale") || "browser";
+ custom_locale = localStorage.getItem("custom_locale") || "";
+ validateLocale();
+
+ $('.popover #settings-popover input:radio[name="locale"]').val([selected_locale]);
+ $(localeTextbox).val(custom_locale);
+
+ ajaxSetup(localStorage.getItem("ajax_timeout"), true);
+ });
+ $(document).on("change", '.popover #settings-popover input:radio[name="locale"]', function () {
+ selected_locale = this.value;
+ localStorage.setItem("selected_locale", selected_locale);
+ validateLocale();
+ });
+ $(document).on("input", localeTextbox, () => {
+ custom_locale = $(localeTextbox).val();
+ validateLocale(true);
+ });
+ $(document).on("input", ajaxTimeoutBox, () => {
+ ajaxSetup($(ajaxTimeoutBox).val(), false, true);
+ });
+ $(document).on("click", ".popover #settings-popover #ajax-timeout-restore", () => {
+ ajaxSetup(null, true, true);
+ });
+
+ // Dismiss Bootstrap popover by clicking outside
+ $("body").on("click", (e) => {
+ $(".popover").each(function () {
+ if (
+ // Popover's descendant
+ $(this).has(e.target).length ||
+ // Button (or icon within a button) that triggers the popover.
+ $(e.target).closest("button").attr("aria-describedby") === this.id
+ ) return;
+ $("#settings").popover("hide");
+ });
+ });
+ }());
+
+ $("#selData").change(() => {
+ tabClick("#throughput_nav");
+ });
+
+ $(document).ajaxStart(() => {
+ $("#refresh > svg").addClass("fa-spin");
+ });
+ $(document).ajaxComplete(() => {
+ setTimeout(() => {
+ $("#refresh > svg").removeClass("fa-spin");
+ }, 1000);
+ });
+
+ $('a[data-bs-toggle="tab"]').on("shown.bs.tab", function () {
+ tabClick("#" + $(this).attr("id"));
+ });
+ $("#refresh, #disconnect").on("click", function (e) {
+ e.preventDefault();
+ tabClick("#" + $(this).attr("id"));
+ });
+ $(".dropdown-menu a").click(function (e) {
+ e.preventDefault();
+ const classList = $(this).attr("class");
+ const [menuClass] = (/\b(?:dynamic|history|preset)\b/).exec(classList);
+ $(".dropdown-menu a.active." + menuClass).removeClass("active");
+ $(this).addClass("active");
+ tabClick("#autoRefresh");
+ });
+
+ $("#selSrv").change(function () {
+ checked_server = this.value;
+ $("#selSrv [value=\"" + checked_server + "\"]").prop("checked", true);
+ if (checked_server === "All SERVERS") {
+ $("#learnServers").show();
+ } else {
+ $("#learnServers").hide();
+ }
+ tabClick("#" + $("#tablist > .nav-item > .nav-link.active").attr("id"));
+ });
+
+ // Radio buttons
+ $(document).on("click", "input:radio[name=\"clusterName\"]", function () {
+ if (!this.disabled) {
+ checked_server = this.value;
+ tabClick("#status_nav");
+ }
+ });
+
+ $("#loading").addClass("d-none");
+
+ return ui;
+});
diff --git a/interface/js/app/selectors.js b/interface/js/app/selectors.js
new file mode 100644
index 0000000..53240d8
--- /dev/null
+++ b/interface/js/app/selectors.js
@@ -0,0 +1,145 @@
+define(["jquery", "app/common"],
+ ($, common) => {
+ "use strict";
+ const ui = {};
+
+ function enable_disable_check_btn() {
+ $("#selectorsChkMsgBtn").prop("disabled", (
+ $.trim($("#selectorsMsgArea").val()).length === 0 ||
+ !$("#selectorsSelArea").hasClass("is-valid")
+ ));
+ }
+
+ function get_server() {
+ const checked_server = common.getSelector("selSrv");
+ return (checked_server === "All SERVERS") ? "local" : checked_server;
+ }
+
+ function checkMsg(data) {
+ const selector = $("#selectorsSelArea").val();
+ common.query("plugins/selectors/check_message?selector=" + encodeURIComponent(selector), {
+ data: data,
+ method: "POST",
+ success: function (neighbours_status) {
+ const json = neighbours_status[0].data;
+ if (json.success) {
+ common.alertMessage("alert-success", "Message successfully processed");
+ $("#selectorsResArea")
+ .val(Object.prototype.hasOwnProperty.call(json, "data") ? json.data.toString() : "");
+ } else {
+ common.alertMessage("alert-error", "Unexpected error processing message");
+ }
+ },
+ server: get_server()
+ });
+ }
+
+ function checkSelectors() {
+ function toggle_form_group_class(remove, add) {
+ $("#selectorsSelArea").removeClass("is-" + remove).addClass("is-" + add);
+ enable_disable_check_btn();
+ }
+ const selector = $("#selectorsSelArea").val();
+ if (selector.length && !common.read_only) {
+ common.query("plugins/selectors/check_selector?selector=" + encodeURIComponent(selector), {
+ method: "GET",
+ success: function (json) {
+ if (json[0].data.success) {
+ toggle_form_group_class("invalid", "valid");
+ } else {
+ toggle_form_group_class("valid", "invalid");
+ }
+ },
+ server: get_server()
+ });
+ } else {
+ $("#selectorsSelArea").removeClass("is-valid is-invalid");
+ enable_disable_check_btn();
+ }
+ }
+
+ function buildLists() {
+ function build_table_from_json(json, table_id) {
+ Object.keys(json).forEach((key) => {
+ const td = $("<td/>");
+ const tr = $("<tr/>")
+ .append(td.clone().html("<code>" + key + "</code>"))
+ .append(td.clone().html(json[key].description));
+ $(table_id + " tbody").append(tr);
+ });
+ }
+
+ function getList(list) {
+ common.query("plugins/selectors/list_" + list, {
+ method: "GET",
+ success: function (neighbours_status) {
+ const json = neighbours_status[0].data;
+ build_table_from_json(json, "#selectorsTable-" + list);
+ },
+ server: get_server()
+ });
+ }
+
+ getList("extractors");
+ getList("transforms");
+ }
+
+ ui.displayUI = function () {
+ if (!common.read_only &&
+ !$("#selectorsTable-extractors>tbody>tr").length &&
+ !$("#selectorsTable-transforms>tbody>tr").length) buildLists();
+ if (!$("#selectorsSelArea").is(".is-valid, .is-invalid")) checkSelectors();
+ };
+
+
+ function toggleSidebar(side) {
+ $("#sidebar-" + side).toggleClass("collapsed");
+ let contentClass = "col-lg-6";
+ const openSidebarsCount = $("#sidebar-left").hasClass("collapsed") +
+ $("#sidebar-right").hasClass("collapsed");
+ switch (openSidebarsCount) {
+ case 1:
+ contentClass = "col-lg-9";
+ break;
+ case 2:
+ contentClass = "col-lg-12";
+ break;
+ default:
+ }
+ $("#content").removeClass("col-lg-12 col-lg-9 col-lg-6")
+ .addClass(contentClass);
+ }
+ $("#sidebar-tab-left>a").click(() => {
+ toggleSidebar("left");
+ return false;
+ });
+ $("#sidebar-tab-right>a").click(() => {
+ toggleSidebar("right");
+ return false;
+ });
+
+ $("#selectorsMsgClean").on("click", () => {
+ $("#selectorsChkMsgBtn").attr("disabled", true);
+ $("#selectorsMsgArea").val("");
+ return false;
+ });
+ $("#selectorsClean").on("click", () => {
+ $("#selectorsSelArea").val("");
+ checkSelectors();
+ return false;
+ });
+ $("#selectorsChkMsgBtn").on("click", () => {
+ $("#selectorsResArea").val("");
+ checkMsg($("#selectorsMsgArea").val());
+ return false;
+ });
+
+ $("#selectorsMsgArea").on("input", () => {
+ enable_disable_check_btn();
+ });
+ $("#selectorsSelArea").on("input", () => {
+ checkSelectors();
+ });
+
+ return ui;
+ });
diff --git a/interface/js/app/stats.js b/interface/js/app/stats.js
new file mode 100644
index 0000000..04b4a75
--- /dev/null
+++ b/interface/js/app/stats.js
@@ -0,0 +1,372 @@
+/*
+ The MIT License (MIT)
+
+ Copyright (C) 2017 Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+
+define(["jquery", "app/common", "d3pie", "d3"],
+ ($, common, D3Pie, d3) => {
+ "use strict";
+ // @ ms to date
+ function msToTime(seconds) {
+ if (!Number.isFinite(seconds)) return "???";
+ /* eslint-disable no-bitwise */
+ const years = seconds / 31536000 >> 0; // 3600*24*365
+ const months = seconds % 31536000 / 2628000 >> 0; // 3600*24*365/12
+ const days = seconds % 31536000 % 2628000 / 86400 >> 0; // 24*3600
+ const hours = seconds % 31536000 % 2628000 % 86400 / 3600 >> 0;
+ const minutes = seconds % 31536000 % 2628000 % 86400 % 3600 / 60 >> 0;
+ /* eslint-enable no-bitwise */
+ let out = null;
+ if (years > 0) {
+ if (months > 0) {
+ out = years + "yr " + months + "mth";
+ } else {
+ out = years + "yr " + days + "d";
+ }
+ } else if (months > 0) {
+ out = months + "mth " + days + "d";
+ } else if (days > 0) {
+ out = days + "d " + hours + "hr";
+ } else if (hours > 0) {
+ out = hours + "hr " + minutes + "min";
+ } else {
+ out = minutes + "min";
+ }
+ return out;
+ }
+
+ function displayStatWidgets(checked_server) {
+ const servers = JSON.parse(sessionStorage.getItem("Credentials"));
+ let data = {};
+ if (servers && servers[checked_server]) {
+ ({data} = servers[checked_server]);
+ }
+
+ const stat_w = [];
+ $("#statWidgets").empty().hide();
+ $.each(data, (i, item) => {
+ const widgetsOrder = ["scanned", "no action", "greylist", "add header", "rewrite subject", "reject", "learned"];
+
+ function widget(k, v, cls) {
+ const c = (typeof cls === "undefined") ? "" : cls;
+ const titleAtt = d3.format(",")(v) + " " + k;
+ return '<div class="card stat-box d-inline-block text-center shadow-sm me-3 px-3">' +
+ '<div class="widget overflow-hidden p-2' + c + '" title="' + titleAtt +
+ '"><strong class="d-block mt-2 mb-1 fw-bold">' +
+ d3.format(".3~s")(v) + "</strong>" + k + "</div></div>";
+ }
+
+ if (i === "auth" || i === "error") return; // Skip to the next iteration
+ if (i === "uptime" || i === "version") {
+ let cls = "border-end ";
+ let val = item;
+ if (i === "uptime") {
+ cls = "";
+ val = msToTime(item);
+ }
+ $('<div class="' + cls + 'float-start px-3"><strong class="d-block mt-2 mb-1 fw-bold">' +
+ val + "</strong>" + i + "</div>")
+ .appendTo("#statWidgets");
+ } else if (i === "actions") {
+ $.each(item, (action, count) => {
+ stat_w[widgetsOrder.indexOf(action)] = widget(action, count);
+ });
+ } else {
+ stat_w[widgetsOrder.indexOf(i)] = widget(i, item, " text-capitalize");
+ }
+ });
+ $.each(stat_w, (i, item) => {
+ $(item).appendTo("#statWidgets");
+ });
+ $("#statWidgets > div:not(.stat-box)")
+ .wrapAll('<div class="card stat-box text-center shadow-sm float-end">' +
+ '<div class="widget overflow-hidden p-2 text-capitalize"></div></div>');
+ $("#statWidgets").find("div.float-end").appendTo("#statWidgets");
+ $("#statWidgets").show();
+
+ $("#clusterTable tbody").empty();
+ $("#selSrv").empty();
+ $.each(servers, (key, val) => {
+ let row_class = "danger";
+ let glyph_status = "fas fa-times";
+ let version = "???";
+ let uptime = "???";
+ let short_id = "???";
+ let scan_times = {
+ data: "???",
+ title: ""
+ };
+ if (val.status) {
+ row_class = "success";
+ glyph_status = "fas fa-check";
+ if (Number.isFinite(val.data.uptime)) {
+ uptime = msToTime(val.data.uptime);
+ }
+ if ("version" in val.data) {
+ ({version} = val.data);
+ }
+ if (key === "All SERVERS") {
+ short_id = "";
+ scan_times.data = "";
+ } else {
+ if ("config_id" in val.data) {
+ short_id = val.data.config_id.substring(0, 8);
+ }
+ if ("scan_times" in val.data) {
+ const [min, max] = d3.extent(val.data.scan_times);
+ if (max) {
+ const f = d3.format(".3f");
+ scan_times = {
+ data: "<small>" + f(min) + "/</small>" +
+ f(d3.mean(val.data.scan_times)) +
+ "<small>/" + f(max) + "</small>",
+ title: ' title="min/avg/max"'
+ };
+ } else {
+ scan_times = {
+ data: "-",
+ title: ' title="Have not scanned anything yet"'
+ };
+ }
+ }
+ }
+ }
+
+ $("#clusterTable tbody").append('<tr class="' + row_class + '">' +
+ '<td class="align-middle"><input type="radio" class="form-check m-auto" name="clusterName" value="' +
+ key + '"></td>' +
+ "<td>" + key + "</td>" +
+ "<td>" + val.host + "</td>" +
+ '<td class="text-center"><span class="icon"><i class="' + glyph_status + '"></i></span></td>' +
+ '<td class="text-center"' + scan_times.title + ">" + scan_times.data + "</td>" +
+ '<td class="text-end' +
+ ((Number.isFinite(val.data.uptime) && val.data.uptime < 3600)
+ ? ' warning" title="Has been restarted within the last hour"'
+ : "") +
+ '">' + uptime + "</td>" +
+ "<td>" + version + "</td>" +
+ "<td>" + short_id + "</td></tr>");
+
+ $("#selSrv").append($('<option value="' + key + '">' + key + "</option>"));
+
+ if (checked_server === key) {
+ $('#clusterTable tbody [value="' + key + '"]').prop("checked", true);
+ $('#selSrv [value="' + key + '"]').prop("selected", true);
+ } else if (!val.status) {
+ $('#clusterTable tbody [value="' + key + '"]').prop("disabled", true);
+ $('#selSrv [value="' + key + '"]').prop("disabled", true);
+ }
+ });
+
+ function addStatfiles(server, statfiles) {
+ $.each(statfiles, (i, statfile) => {
+ let cls = "";
+ switch (statfile.symbol) {
+ case "BAYES_SPAM":
+ cls = "symbol-positive";
+ break;
+ case "BAYES_HAM":
+ cls = "symbol-negative";
+ break;
+ default:
+ }
+ $("#bayesTable tbody").append("<tr>" +
+ (i === 0 ? '<td rowspan="' + statfiles.length + '">' + server + "</td>" : "") +
+ '<td class="' + cls + '">' + statfile.symbol + "</td>" +
+ '<td class="' + cls + '">' + statfile.type + "</td>" +
+ '<td class="text-end ' + cls + '">' + statfile.revision + "</td>" +
+ '<td class="text-end ' + cls + '">' + statfile.users + "</td></tr>");
+ });
+ }
+
+ function addFuzzyStorage(server, storages) {
+ let i = 0;
+ $.each(storages, (storage, hashes) => {
+ $("#fuzzyTable tbody").append("<tr>" +
+ (i === 0 ? '<td rowspan="' + Object.keys(storages).length + '">' + server + "</td>" : "") +
+ "<td>" + storage + "</td>" +
+ '<td class="text-end">' + hashes + "</td></tr>");
+ i++;
+ });
+ }
+
+ $("#bayesTable tbody, #fuzzyTable tbody").empty();
+ if (checked_server === "All SERVERS") {
+ $.each(servers, (server, val) => {
+ if (server !== "All SERVERS") {
+ addStatfiles(server, val.data.statfiles);
+ addFuzzyStorage(server, val.data.fuzzy_hashes);
+ }
+ });
+ } else {
+ addStatfiles(checked_server, data.statfiles);
+ addFuzzyStorage(checked_server, data.fuzzy_hashes);
+ }
+ }
+
+ function getChart(graphs, checked_server) {
+ if (!graphs.chart) {
+ graphs.chart = new D3Pie("chart", {
+ labels: {
+ inner: {
+ offset: 0
+ },
+ outer: {
+ collideHeight: 18,
+ }
+ },
+ size: {
+ pieInnerRadius: "50%"
+ },
+ title: "Rspamd filter stats",
+ total: {
+ enabled: true,
+ label: "Scanned"
+ }
+ });
+ }
+
+ const data = [];
+ const creds = JSON.parse(sessionStorage.getItem("Credentials"));
+ // Controller doesn't return the 'actions' object until at least one message is scanned
+ if (creds && creds[checked_server] && creds[checked_server].data.scanned) {
+ const {actions} = creds[checked_server].data;
+
+ ["no action", "soft reject", "add header", "rewrite subject", "greylist", "reject"]
+ .forEach((action) => {
+ data.push({
+ color: common.chartLegend.find((item) => item.label === action).color,
+ label: action,
+ value: actions[action]
+ });
+ });
+ }
+ graphs.chart.data(data);
+ }
+
+ // Public API
+ const ui = {
+ statWidgets: function (graphs, checked_server) {
+ common.query("stat", {
+ success: function (neighbours_status) {
+ const neighbours_sum = {
+ version: neighbours_status[0].data.version,
+ uptime: 0,
+ scanned: 0,
+ learned: 0,
+ actions: {
+ "no action": 0,
+ "add header": 0,
+ "rewrite subject": 0,
+ "greylist": 0,
+ "reject": 0,
+ "soft reject": 0,
+ }
+ };
+ let status_count = 0;
+ const promises = [];
+ const to_Credentials = {
+ "All SERVERS": {
+ name: "All SERVERS",
+ url: "",
+ host: "",
+ checked: true,
+ status: true
+ }
+ };
+
+ function process_node_stat(e) {
+ const {data} = neighbours_status[e];
+ // Controller doesn't return the 'actions' object until at least one message is scanned
+ if (data.scanned) {
+ for (const action in neighbours_sum.actions) {
+ if ({}.hasOwnProperty.call(neighbours_sum.actions, action)) {
+ neighbours_sum.actions[action] += data.actions[action];
+ }
+ }
+ }
+ ["learned", "scanned", "uptime"].forEach((p) => {
+ neighbours_sum[p] += data[p];
+ });
+ status_count++;
+ }
+
+ // Get config_id, version and uptime using /auth query for Rspamd 2.5 and earlier
+ function get_legacy_stat(e) {
+ const alerted = "alerted_stats_legacy_" + neighbours_status[e].name;
+ promises.push($.ajax({
+ url: neighbours_status[e].url + "auth",
+ headers: {Password: common.getPassword()},
+ success: function (data) {
+ sessionStorage.removeItem(alerted);
+ ["config_id", "version", "uptime"].forEach((p) => {
+ neighbours_status[e].data[p] = data[p];
+ });
+ process_node_stat(e);
+ },
+ error: function (jqXHR, textStatus, errorThrown) {
+ if (!(alerted in sessionStorage)) {
+ sessionStorage.setItem(alerted, true);
+ common.alertMessage("alert-error", neighbours_status[e].name + " > " +
+ "Cannot receive legacy stats data" + (errorThrown ? ": " + errorThrown : ""));
+ }
+ process_node_stat(e);
+ }
+ }));
+ }
+
+ for (const e in neighbours_status) {
+ if ({}.hasOwnProperty.call(neighbours_status, e)) {
+ to_Credentials[neighbours_status[e].name] = neighbours_status[e];
+ if (neighbours_status[e].status === true) {
+ // Remove alert status
+ sessionStorage.removeItem("alerted_stats_" + neighbours_status[e].name);
+
+ if ({}.hasOwnProperty.call(neighbours_status[e].data, "version")) {
+ process_node_stat(e);
+ } else {
+ get_legacy_stat(e);
+ }
+ }
+ }
+ }
+ setTimeout(() => {
+ $.when.apply($, promises).always(() => {
+ neighbours_sum.uptime = Math.floor(neighbours_sum.uptime / status_count);
+ to_Credentials["All SERVERS"].data = neighbours_sum;
+ sessionStorage.setItem("Credentials", JSON.stringify(to_Credentials));
+ displayStatWidgets(checked_server);
+ getChart(graphs, checked_server);
+ });
+ }, promises.length ? 100 : 0);
+ },
+ complete: function () { $("#refresh").removeAttr("disabled").removeClass("disabled"); },
+ errorMessage: "Cannot receive stats data",
+ errorOnceId: "alerted_stats_",
+ server: "All SERVERS"
+ });
+ },
+ };
+
+ return ui;
+ }
+);
diff --git a/interface/js/app/symbols.js b/interface/js/app/symbols.js
new file mode 100644
index 0000000..1e3fb5d
--- /dev/null
+++ b/interface/js/app/symbols.js
@@ -0,0 +1,260 @@
+/*
+ The MIT License (MIT)
+
+ Copyright (C) 2017 Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+
+/* global FooTable */
+
+define(["jquery", "app/common", "footable"],
+ ($, common) => {
+ "use strict";
+ const ui = {};
+ let altered = {};
+
+ function clear_altered() {
+ $("#save-alert").addClass("d-none");
+ altered = {};
+ }
+
+ function saveSymbols(server) {
+ $("#save-alert button").attr("disabled", true);
+
+ const values = [];
+ Object.entries(altered).forEach(([key, value]) => values.push({name: key, value: value}));
+
+ common.query("./savesymbols", {
+ success: function () {
+ clear_altered();
+ common.alertMessage("alert-modal alert-success", "Symbols successfully saved");
+ },
+ complete: () => $("#save-alert button").removeAttr("disabled", true),
+ errorMessage: "Save symbols error",
+ method: "POST",
+ params: {
+ data: JSON.stringify(values),
+ dataType: "json",
+ },
+ server: server
+ });
+ }
+
+ function process_symbols_data(data) {
+ const items = [];
+ const lookup = {};
+ const freqs = [];
+ const distinct_groups = [];
+
+ data.forEach((group) => {
+ group.rules.forEach((item) => {
+ const formatter = new Intl.NumberFormat("en", {
+ minimumFractionDigits: 2,
+ maximumFractionDigits: 6,
+ useGrouping: false
+ });
+ item.group = group.group;
+ let label_class = "";
+ if (item.weight < 0) {
+ label_class = "scorebar-ham";
+ } else if (item.weight > 0) {
+ label_class = "scorebar-spam";
+ }
+ item.weight = '<input class="form-control input-sm mb-disabled scorebar ' + label_class +
+ '" autocomplete="off" type="number" step="0.01" tabindex="1" ' +
+ 'value="' + formatter.format(item.weight) + '" id="_sym_' + item.symbol + '"></input>';
+ if (!item.time) {
+ item.time = 0;
+ }
+ item.time = Number(item.time).toFixed(2) + "s";
+ if (!item.frequency) {
+ item.frequency = 0;
+ }
+ freqs.push(item.frequency);
+ item.frequency = Number(item.frequency).toFixed(2);
+ if (!(item.group in lookup)) {
+ lookup[item.group] = 1;
+ distinct_groups.push(item.group);
+ }
+ items.push(item);
+ });
+ });
+
+ // For better mean calculations
+ const avg_freq = freqs
+ .sort((a, b) => Number(a) < Number(b))
+ .reduce((f1, acc) => f1 + acc) / (freqs.length !== 0 ? freqs.length : 1.0);
+ let mult = 1.0;
+ let exp = 0.0;
+
+ if (avg_freq > 0.0) {
+ while (mult * avg_freq < 1.0) {
+ mult *= 10;
+ exp++;
+ }
+ }
+ $.each(items, (i, item) => {
+ item.frequency = Number(item.frequency) * mult;
+
+ if (exp > 0) {
+ item.frequency = item.frequency.toFixed(2) + "e-" + exp;
+ } else {
+ item.frequency = item.frequency.toFixed(2);
+ }
+ });
+ return [items, distinct_groups];
+ }
+ // @get symbols into modal form
+ ui.getSymbols = function (checked_server) {
+ clear_altered();
+ common.query("symbols", {
+ success: function (json) {
+ const [{data}] = json;
+ const items = process_symbols_data(data);
+
+ /* eslint-disable consistent-this, no-underscore-dangle, one-var-declaration-per-line */
+ FooTable.groupFilter = FooTable.Filtering.extend({
+ construct: function (instance) {
+ this._super(instance);
+ [,this.groups] = items;
+ this.def = "Any group";
+ this.$group = null;
+ },
+ $create: function () {
+ this._super();
+ const self = this;
+ const $form_grp = $("<div/>", {
+ class: "form-group"
+ }).append($("<label/>", {
+ class: "sr-only",
+ text: "Group"
+ })).prependTo(self.$form);
+
+ self.$group = $("<select/>", {
+ class: "form-select"
+ }).on("change", {
+ self: self
+ }, self._onStatusDropdownChanged).append(
+ $("<option/>", {
+ text: self.def
+ })).appendTo($form_grp);
+
+ $.each(self.groups, (i, group) => {
+ self.$group.append($("<option/>").text(group));
+ });
+ },
+ _onStatusDropdownChanged: function (e) {
+ const {self} = e.data;
+ const selected = $(this).val();
+ if (selected !== self.def) {
+ self.addFilter("group", selected, ["group"]);
+ } else {
+ self.removeFilter("group");
+ }
+ self.filter();
+ },
+ draw: function () {
+ this._super();
+ const group = this.find("group");
+ if (group instanceof FooTable.Filter) {
+ this.$group.val(group.query.val());
+ } else {
+ this.$group.val(this.def);
+ }
+ }
+ });
+ /* eslint-enable consistent-this, no-underscore-dangle, one-var-declaration-per-line */
+
+ common.tables.symbols = FooTable.init("#symbolsTable", {
+ columns: [
+ {sorted: true, direction: "ASC", name: "group", title: "Group"},
+ {name: "symbol", title: "Symbol"},
+ {name: "description", title: "Description", breakpoints: "xs sm"},
+ {name: "weight", title: "Score"},
+ {name: "frequency",
+ title: "Frequency",
+ breakpoints: "xs sm",
+ sortValue: function (value) { return Number(value).toFixed(2); }},
+ {name: "time", title: "Avg. time", breakpoints: "xs sm"},
+ ],
+ rows: items[0],
+ paging: {
+ enabled: true,
+ limit: 5,
+ size: 25
+ },
+ filtering: {
+ enabled: true,
+ position: "left",
+ connectors: false
+ },
+ sorting: {
+ enabled: true
+ },
+ components: {
+ filtering: FooTable.groupFilter
+ },
+ on: {
+ "ready.ft.table": function () {
+ if (common.read_only) {
+ $(".mb-disabled").attr("disabled", true);
+ }
+ }
+ }
+ });
+ },
+ server: (checked_server === "All SERVERS") ? "local" : checked_server
+ });
+ };
+
+
+ $("#updateSymbols").on("click", (e) => {
+ e.preventDefault();
+ clear_altered();
+ const checked_server = common.getSelector("selSrv");
+ common.query("symbols", {
+ success: function (data) {
+ const [items] = process_symbols_data(data[0].data);
+ common.tables.symbols.rows.load(items);
+ },
+ server: (checked_server === "All SERVERS") ? "local" : checked_server
+ });
+ });
+
+ $("#symbolsTable")
+ .on("input", ".scorebar", ({target}) => {
+ const t = $(target);
+ t.removeClass("scorebar-ham scorebar-spam");
+ if (target.value < 0) {
+ t.addClass("scorebar-ham");
+ } else if (target.value > 0) {
+ t.addClass("scorebar-spam");
+ }
+ })
+ .on("change", ".scorebar", ({target}) => {
+ altered[$(target).attr("id").substring(5)] = parseFloat(target.value);
+ $("#save-alert").removeClass("d-none");
+ });
+
+ $("#save-alert button")
+ .on("click", ({target}) => saveSymbols($(target).data("save")));
+
+ return ui;
+ });
diff --git a/interface/js/app/upload.js b/interface/js/app/upload.js
new file mode 100644
index 0000000..546f5cc
--- /dev/null
+++ b/interface/js/app/upload.js
@@ -0,0 +1,243 @@
+/*
+ The MIT License (MIT)
+
+ Copyright (C) 2017 Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+
+/* global require */
+
+define(["jquery", "app/common", "app/libft"],
+ ($, common, libft) => {
+ "use strict";
+ const ui = {};
+
+ function cleanTextUpload(source) {
+ $("#" + source + "TextSource").val("");
+ }
+
+ // @upload text
+ function uploadText(data, source, headers) {
+ let url = null;
+ if (source === "spam") {
+ url = "learnspam";
+ } else if (source === "ham") {
+ url = "learnham";
+ } else if (source === "fuzzy") {
+ url = "fuzzyadd";
+ } else if (source === "scan") {
+ url = "checkv2";
+ }
+
+ function server() {
+ if (common.getSelector("selSrv") === "All SERVERS" &&
+ common.getSelector("selLearnServers") === "random") {
+ const servers = $("#selSrv option").slice(1).map((_, o) => o.value);
+ return servers[Math.floor(Math.random() * servers.length)];
+ }
+ return null;
+ }
+
+ common.query(url, {
+ data: data,
+ params: {
+ processData: false,
+ },
+ method: "POST",
+ headers: headers,
+ success: function (json, jqXHR) {
+ cleanTextUpload(source);
+ common.alertMessage("alert-success", "Data successfully uploaded");
+ if (jqXHR.status !== 200) {
+ common.alertMessage("alert-info", jqXHR.statusText);
+ }
+ },
+ server: server()
+ });
+ }
+
+ function get_server() {
+ const checked_server = common.getSelector("selSrv");
+ return (checked_server === "All SERVERS") ? "local" : checked_server;
+ }
+
+ // @upload text
+ function scanText(data, headers) {
+ common.query("checkv2", {
+ data: data,
+ params: {
+ processData: false,
+ },
+ method: "POST",
+ headers: headers,
+ success: function (neighbours_status) {
+ function scrollTop(rows_total) {
+ // Is there a way to get an event when all rows are loaded?
+ libft.waitForRowsDisplayed("scan", rows_total, () => {
+ $("#cleanScanHistory").removeAttr("disabled", true);
+ $("html, body").animate({
+ scrollTop: $("#scanResult").offset().top
+ }, 1000);
+ });
+ }
+
+ const json = neighbours_status[0].data;
+ if (json.action) {
+ common.alertMessage("alert-success", "Data successfully scanned");
+
+ const rows_total = $("#historyTable_scan > tbody > tr:not(.footable-detail-row)").length + 1;
+ const o = libft.process_history_v2({rows: [json]}, "scan");
+ const {items} = o;
+ common.symbols.scan.push(o.symbols[0]);
+
+ if (Object.prototype.hasOwnProperty.call(common.tables, "scan")) {
+ common.tables.scan.rows.load(items, true);
+ scrollTop(rows_total);
+ } else {
+ libft.destroyTable("scan");
+ require(["footable"], () => {
+ // Is there a way to get an event when the table is destroyed?
+ setTimeout(() => {
+ libft.initHistoryTable(data, items, "scan", libft.columns_v2("scan"), true);
+ scrollTop(rows_total);
+ }, 200);
+ });
+ }
+ } else {
+ common.alertMessage("alert-error", "Cannot scan data");
+ }
+ },
+ errorMessage: "Cannot upload data",
+ statusCode: {
+ 404: function () {
+ common.alertMessage("alert-error", "Cannot upload data, no server found");
+ },
+ 500: function () {
+ common.alertMessage("alert-error", "Cannot tokenize message: no text data");
+ },
+ 503: function () {
+ common.alertMessage("alert-error", "Cannot tokenize message: no text data");
+ }
+ },
+ server: get_server()
+ });
+ }
+
+ function getFuzzyHashes(data) {
+ function fillHashTable(rules) {
+ $("#hashTable tbody").empty();
+ for (const [rule, hashes] of Object.entries(rules)) {
+ hashes.forEach((hash, i) => {
+ $("#hashTable tbody").append("<tr>" +
+ (i === 0 ? '<td rowspan="' + Object.keys(hashes).length + '">' + rule + "</td>" : "") +
+ "<td>" + hash + "</td></tr>");
+ });
+ }
+ $("#hash-card").slideDown();
+ }
+
+ common.query("plugins/fuzzy/hashes?flag=" + $("#fuzzy-flag").val(), {
+ data: data,
+ params: {
+ processData: false,
+ },
+ method: "POST",
+ success: function (neighbours_status) {
+ const json = neighbours_status[0].data;
+ if (json.success) {
+ common.alertMessage("alert-success", "Message successfully processed");
+ fillHashTable(json.hashes);
+ } else {
+ common.alertMessage("alert-error", "Unexpected error processing message");
+ }
+ },
+ server: get_server()
+ });
+ }
+
+
+ libft.set_page_size("scan", $("#scan_page_size").val());
+ libft.bindHistoryTableEventHandlers("scan", 3);
+
+ $("#cleanScanHistory").off("click");
+ $("#cleanScanHistory").on("click", (e) => {
+ e.preventDefault();
+ if (!confirm("Are you sure you want to clean scan history?")) { // eslint-disable-line no-alert
+ return;
+ }
+ libft.destroyTable("scan");
+ common.symbols.scan.length = 0;
+ $("#cleanScanHistory").attr("disabled", true);
+ });
+
+ function enable_disable_scan_btn() {
+ $("#scan button:not(#cleanScanHistory, #scanOptionsToggle)")
+ .prop("disabled", ($.trim($("textarea").val()).length === 0));
+ }
+ enable_disable_scan_btn();
+ $("textarea").on("input", () => {
+ enable_disable_scan_btn();
+ });
+
+ $("#scanClean").on("click", () => {
+ $("#scan button:not(#cleanScanHistory, #scanOptionsToggle)").attr("disabled", true);
+ $("#scanForm")[0].reset();
+ $("#scanResult").hide();
+ $("#scanOutput tbody").remove();
+ $("html, body").animate({scrollTop: 0}, 1000);
+ return false;
+ });
+
+ $(".card-close-btn").on("click", function () {
+ $(this).closest(".card").slideUp();
+ });
+
+ $("[data-upload]").on("click", function () {
+ const source = $(this).data("upload");
+ const data = $("#scanMsgSource").val();
+ let headers = {};
+ if ($.trim(data).length > 0) {
+ if (source === "scan") {
+ headers = ["IP", "User", "From", "Rcpt", "Helo", "Hostname"].reduce((o, header) => {
+ const value = $("#scan-opt-" + header.toLowerCase()).val();
+ if (value !== "") o[header] = value;
+ return o;
+ }, {});
+ if ($("#scan-opt-pass-all").prop("checked")) headers.Pass = "all";
+ scanText(data, headers);
+ } else if (source === "compute-fuzzy") {
+ getFuzzyHashes(data);
+ } else {
+ if (source === "fuzzy") {
+ headers = {
+ flag: $("#fuzzyFlagText").val(),
+ weight: $("#fuzzyWeightText").val()
+ };
+ }
+ uploadText(data, source, headers);
+ }
+ } else {
+ common.alertMessage("alert-error", "Message source field cannot be blank");
+ }
+ return false;
+ });
+
+ return ui;
+ });
diff --git a/interface/js/lib/bootstrap.bundle.min.js b/interface/js/lib/bootstrap.bundle.min.js
new file mode 100644
index 0000000..b1999d9
--- /dev/null
+++ b/interface/js/lib/bootstrap.bundle.min.js
@@ -0,0 +1,7 @@
+/*!
+ * Bootstrap v5.3.2 (https://getbootstrap.com/)
+ * Copyright 2011-2023 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors)
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)
+ */
+!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e():"function"==typeof define&&define.amd?define(e):(t="undefined"!=typeof globalThis?globalThis:t||self).bootstrap=e()}(this,(function(){"use strict";const t=new Map,e={set(e,i,n){t.has(e)||t.set(e,new Map);const s=t.get(e);s.has(i)||0===s.size?s.set(i,n):console.error(`Bootstrap doesn't allow more than one instance per element. Bound instance: ${Array.from(s.keys())[0]}.`)},get:(e,i)=>t.has(e)&&t.get(e).get(i)||null,remove(e,i){if(!t.has(e))return;const n=t.get(e);n.delete(i),0===n.size&&t.delete(e)}},i="transitionend",n=t=>(t&&window.CSS&&window.CSS.escape&&(t=t.replace(/#([^\s"#']+)/g,((t,e)=>`#${CSS.escape(e)}`))),t),s=t=>{t.dispatchEvent(new Event(i))},o=t=>!(!t||"object"!=typeof t)&&(void 0!==t.jquery&&(t=t[0]),void 0!==t.nodeType),r=t=>o(t)?t.jquery?t[0]:t:"string"==typeof t&&t.length>0?document.querySelector(n(t)):null,a=t=>{if(!o(t)||0===t.getClientRects().length)return!1;const e="visible"===getComputedStyle(t).getPropertyValue("visibility"),i=t.closest("details:not([open])");if(!i)return e;if(i!==t){const e=t.closest("summary");if(e&&e.parentNode!==i)return!1;if(null===e)return!1}return e},l=t=>!t||t.nodeType!==Node.ELEMENT_NODE||!!t.classList.contains("disabled")||(void 0!==t.disabled?t.disabled:t.hasAttribute("disabled")&&"false"!==t.getAttribute("disabled")),c=t=>{if(!document.documentElement.attachShadow)return null;if("function"==typeof t.getRootNode){const e=t.getRootNode();return e instanceof ShadowRoot?e:null}return t instanceof ShadowRoot?t:t.parentNode?c(t.parentNode):null},h=()=>{},d=t=>{t.offsetHeight},u=()=>window.jQuery&&!document.body.hasAttribute("data-bs-no-jquery")?window.jQuery:null,f=[],p=()=>"rtl"===document.documentElement.dir,m=t=>{var e;e=()=>{const e=u();if(e){const i=t.NAME,n=e.fn[i];e.fn[i]=t.jQueryInterface,e.fn[i].Constructor=t,e.fn[i].noConflict=()=>(e.fn[i]=n,t.jQueryInterface)}},"loading"===document.readyState?(f.length||document.addEventListener("DOMContentLoaded",(()=>{for(const t of f)t()})),f.push(e)):e()},g=(t,e=[],i=t)=>"function"==typeof t?t(...e):i,_=(t,e,n=!0)=>{if(!n)return void g(t);const o=(t=>{if(!t)return 0;let{transitionDuration:e,transitionDelay:i}=window.getComputedStyle(t);const n=Number.parseFloat(e),s=Number.parseFloat(i);return n||s?(e=e.split(",")[0],i=i.split(",")[0],1e3*(Number.parseFloat(e)+Number.parseFloat(i))):0})(e)+5;let r=!1;const a=({target:n})=>{n===e&&(r=!0,e.removeEventListener(i,a),g(t))};e.addEventListener(i,a),setTimeout((()=>{r||s(e)}),o)},b=(t,e,i,n)=>{const s=t.length;let o=t.indexOf(e);return-1===o?!i&&n?t[s-1]:t[0]:(o+=i?1:-1,n&&(o=(o+s)%s),t[Math.max(0,Math.min(o,s-1))])},v=/[^.]*(?=\..*)\.|.*/,y=/\..*/,w=/::\d+$/,A={};let E=1;const T={mouseenter:"mouseover",mouseleave:"mouseout"},C=new Set(["click","dblclick","mouseup","mousedown","contextmenu","mousewheel","DOMMouseScroll","mouseover","mouseout","mousemove","selectstart","selectend","keydown","keypress","keyup","orientationchange","touchstart","touchmove","touchend","touchcancel","pointerdown","pointermove","pointerup","pointerleave","pointercancel","gesturestart","gesturechange","gestureend","focus","blur","change","reset","select","submit","focusin","focusout","load","unload","beforeunload","resize","move","DOMContentLoaded","readystatechange","error","abort","scroll"]);function O(t,e){return e&&`${e}::${E++}`||t.uidEvent||E++}function x(t){const e=O(t);return t.uidEvent=e,A[e]=A[e]||{},A[e]}function k(t,e,i=null){return Object.values(t).find((t=>t.callable===e&&t.delegationSelector===i))}function L(t,e,i){const n="string"==typeof e,s=n?i:e||i;let o=I(t);return C.has(o)||(o=t),[n,s,o]}function S(t,e,i,n,s){if("string"!=typeof e||!t)return;let[o,r,a]=L(e,i,n);if(e in T){const t=t=>function(e){if(!e.relatedTarget||e.relatedTarget!==e.delegateTarget&&!e.delegateTarget.contains(e.relatedTarget))return t.call(this,e)};r=t(r)}const l=x(t),c=l[a]||(l[a]={}),h=k(c,r,o?i:null);if(h)return void(h.oneOff=h.oneOff&&s);const d=O(r,e.replace(v,"")),u=o?function(t,e,i){return function n(s){const o=t.querySelectorAll(e);for(let{target:r}=s;r&&r!==this;r=r.parentNode)for(const a of o)if(a===r)return P(s,{delegateTarget:r}),n.oneOff&&N.off(t,s.type,e,i),i.apply(r,[s])}}(t,i,r):function(t,e){return function i(n){return P(n,{delegateTarget:t}),i.oneOff&&N.off(t,n.type,e),e.apply(t,[n])}}(t,r);u.delegationSelector=o?i:null,u.callable=r,u.oneOff=s,u.uidEvent=d,c[d]=u,t.addEventListener(a,u,o)}function D(t,e,i,n,s){const o=k(e[i],n,s);o&&(t.removeEventListener(i,o,Boolean(s)),delete e[i][o.uidEvent])}function $(t,e,i,n){const s=e[i]||{};for(const[o,r]of Object.entries(s))o.includes(n)&&D(t,e,i,r.callable,r.delegationSelector)}function I(t){return t=t.replace(y,""),T[t]||t}const N={on(t,e,i,n){S(t,e,i,n,!1)},one(t,e,i,n){S(t,e,i,n,!0)},off(t,e,i,n){if("string"!=typeof e||!t)return;const[s,o,r]=L(e,i,n),a=r!==e,l=x(t),c=l[r]||{},h=e.startsWith(".");if(void 0===o){if(h)for(const i of Object.keys(l))$(t,l,i,e.slice(1));for(const[i,n]of Object.entries(c)){const s=i.replace(w,"");a&&!e.includes(s)||D(t,l,r,n.callable,n.delegationSelector)}}else{if(!Object.keys(c).length)return;D(t,l,r,o,s?i:null)}},trigger(t,e,i){if("string"!=typeof e||!t)return null;const n=u();let s=null,o=!0,r=!0,a=!1;e!==I(e)&&n&&(s=n.Event(e,i),n(t).trigger(s),o=!s.isPropagationStopped(),r=!s.isImmediatePropagationStopped(),a=s.isDefaultPrevented());const l=P(new Event(e,{bubbles:o,cancelable:!0}),i);return a&&l.preventDefault(),r&&t.dispatchEvent(l),l.defaultPrevented&&s&&s.preventDefault(),l}};function P(t,e={}){for(const[i,n]of Object.entries(e))try{t[i]=n}catch(e){Object.defineProperty(t,i,{configurable:!0,get:()=>n})}return t}function M(t){if("true"===t)return!0;if("false"===t)return!1;if(t===Number(t).toString())return Number(t);if(""===t||"null"===t)return null;if("string"!=typeof t)return t;try{return JSON.parse(decodeURIComponent(t))}catch(e){return t}}function j(t){return t.replace(/[A-Z]/g,(t=>`-${t.toLowerCase()}`))}const F={setDataAttribute(t,e,i){t.setAttribute(`data-bs-${j(e)}`,i)},removeDataAttribute(t,e){t.removeAttribute(`data-bs-${j(e)}`)},getDataAttributes(t){if(!t)return{};const e={},i=Object.keys(t.dataset).filter((t=>t.startsWith("bs")&&!t.startsWith("bsConfig")));for(const n of i){let i=n.replace(/^bs/,"");i=i.charAt(0).toLowerCase()+i.slice(1,i.length),e[i]=M(t.dataset[n])}return e},getDataAttribute:(t,e)=>M(t.getAttribute(`data-bs-${j(e)}`))};class H{static get Default(){return{}}static get DefaultType(){return{}}static get NAME(){throw new Error('You have to implement the static method "NAME", for each component!')}_getConfig(t){return t=this._mergeConfigObj(t),t=this._configAfterMerge(t),this._typeCheckConfig(t),t}_configAfterMerge(t){return t}_mergeConfigObj(t,e){const i=o(e)?F.getDataAttribute(e,"config"):{};return{...this.constructor.Default,..."object"==typeof i?i:{},...o(e)?F.getDataAttributes(e):{},..."object"==typeof t?t:{}}}_typeCheckConfig(t,e=this.constructor.DefaultType){for(const[n,s]of Object.entries(e)){const e=t[n],r=o(e)?"element":null==(i=e)?`${i}`:Object.prototype.toString.call(i).match(/\s([a-z]+)/i)[1].toLowerCase();if(!new RegExp(s).test(r))throw new TypeError(`${this.constructor.NAME.toUpperCase()}: Option "${n}" provided type "${r}" but expected type "${s}".`)}var i}}class W extends H{constructor(t,i){super(),(t=r(t))&&(this._element=t,this._config=this._getConfig(i),e.set(this._element,this.constructor.DATA_KEY,this))}dispose(){e.remove(this._element,this.constructor.DATA_KEY),N.off(this._element,this.constructor.EVENT_KEY);for(const t of Object.getOwnPropertyNames(this))this[t]=null}_queueCallback(t,e,i=!0){_(t,e,i)}_getConfig(t){return t=this._mergeConfigObj(t,this._element),t=this._configAfterMerge(t),this._typeCheckConfig(t),t}static getInstance(t){return e.get(r(t),this.DATA_KEY)}static getOrCreateInstance(t,e={}){return this.getInstance(t)||new this(t,"object"==typeof e?e:null)}static get VERSION(){return"5.3.2"}static get DATA_KEY(){return`bs.${this.NAME}`}static get EVENT_KEY(){return`.${this.DATA_KEY}`}static eventName(t){return`${t}${this.EVENT_KEY}`}}const B=t=>{let e=t.getAttribute("data-bs-target");if(!e||"#"===e){let i=t.getAttribute("href");if(!i||!i.includes("#")&&!i.startsWith("."))return null;i.includes("#")&&!i.startsWith("#")&&(i=`#${i.split("#")[1]}`),e=i&&"#"!==i?n(i.trim()):null}return e},z={find:(t,e=document.documentElement)=>[].concat(...Element.prototype.querySelectorAll.call(e,t)),findOne:(t,e=document.documentElement)=>Element.prototype.querySelector.call(e,t),children:(t,e)=>[].concat(...t.children).filter((t=>t.matches(e))),parents(t,e){const i=[];let n=t.parentNode.closest(e);for(;n;)i.push(n),n=n.parentNode.closest(e);return i},prev(t,e){let i=t.previousElementSibling;for(;i;){if(i.matches(e))return[i];i=i.previousElementSibling}return[]},next(t,e){let i=t.nextElementSibling;for(;i;){if(i.matches(e))return[i];i=i.nextElementSibling}return[]},focusableChildren(t){const e=["a","button","input","textarea","select","details","[tabindex]",'[contenteditable="true"]'].map((t=>`${t}:not([tabindex^="-"])`)).join(",");return this.find(e,t).filter((t=>!l(t)&&a(t)))},getSelectorFromElement(t){const e=B(t);return e&&z.findOne(e)?e:null},getElementFromSelector(t){const e=B(t);return e?z.findOne(e):null},getMultipleElementsFromSelector(t){const e=B(t);return e?z.find(e):[]}},R=(t,e="hide")=>{const i=`click.dismiss${t.EVENT_KEY}`,n=t.NAME;N.on(document,i,`[data-bs-dismiss="${n}"]`,(function(i){if(["A","AREA"].includes(this.tagName)&&i.preventDefault(),l(this))return;const s=z.getElementFromSelector(this)||this.closest(`.${n}`);t.getOrCreateInstance(s)[e]()}))},q=".bs.alert",V=`close${q}`,K=`closed${q}`;class Q extends W{static get NAME(){return"alert"}close(){if(N.trigger(this._element,V).defaultPrevented)return;this._element.classList.remove("show");const t=this._element.classList.contains("fade");this._queueCallback((()=>this._destroyElement()),this._element,t)}_destroyElement(){this._element.remove(),N.trigger(this._element,K),this.dispose()}static jQueryInterface(t){return this.each((function(){const e=Q.getOrCreateInstance(this);if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t](this)}}))}}R(Q,"close"),m(Q);const X='[data-bs-toggle="button"]';class Y extends W{static get NAME(){return"button"}toggle(){this._element.setAttribute("aria-pressed",this._element.classList.toggle("active"))}static jQueryInterface(t){return this.each((function(){const e=Y.getOrCreateInstance(this);"toggle"===t&&e[t]()}))}}N.on(document,"click.bs.button.data-api",X,(t=>{t.preventDefault();const e=t.target.closest(X);Y.getOrCreateInstance(e).toggle()})),m(Y);const U=".bs.swipe",G=`touchstart${U}`,J=`touchmove${U}`,Z=`touchend${U}`,tt=`pointerdown${U}`,et=`pointerup${U}`,it={endCallback:null,leftCallback:null,rightCallback:null},nt={endCallback:"(function|null)",leftCallback:"(function|null)",rightCallback:"(function|null)"};class st extends H{constructor(t,e){super(),this._element=t,t&&st.isSupported()&&(this._config=this._getConfig(e),this._deltaX=0,this._supportPointerEvents=Boolean(window.PointerEvent),this._initEvents())}static get Default(){return it}static get DefaultType(){return nt}static get NAME(){return"swipe"}dispose(){N.off(this._element,U)}_start(t){this._supportPointerEvents?this._eventIsPointerPenTouch(t)&&(this._deltaX=t.clientX):this._deltaX=t.touches[0].clientX}_end(t){this._eventIsPointerPenTouch(t)&&(this._deltaX=t.clientX-this._deltaX),this._handleSwipe(),g(this._config.endCallback)}_move(t){this._deltaX=t.touches&&t.touches.length>1?0:t.touches[0].clientX-this._deltaX}_handleSwipe(){const t=Math.abs(this._deltaX);if(t<=40)return;const e=t/this._deltaX;this._deltaX=0,e&&g(e>0?this._config.rightCallback:this._config.leftCallback)}_initEvents(){this._supportPointerEvents?(N.on(this._element,tt,(t=>this._start(t))),N.on(this._element,et,(t=>this._end(t))),this._element.classList.add("pointer-event")):(N.on(this._element,G,(t=>this._start(t))),N.on(this._element,J,(t=>this._move(t))),N.on(this._element,Z,(t=>this._end(t))))}_eventIsPointerPenTouch(t){return this._supportPointerEvents&&("pen"===t.pointerType||"touch"===t.pointerType)}static isSupported(){return"ontouchstart"in document.documentElement||navigator.maxTouchPoints>0}}const ot=".bs.carousel",rt=".data-api",at="next",lt="prev",ct="left",ht="right",dt=`slide${ot}`,ut=`slid${ot}`,ft=`keydown${ot}`,pt=`mouseenter${ot}`,mt=`mouseleave${ot}`,gt=`dragstart${ot}`,_t=`load${ot}${rt}`,bt=`click${ot}${rt}`,vt="carousel",yt="active",wt=".active",At=".carousel-item",Et=wt+At,Tt={ArrowLeft:ht,ArrowRight:ct},Ct={interval:5e3,keyboard:!0,pause:"hover",ride:!1,touch:!0,wrap:!0},Ot={interval:"(number|boolean)",keyboard:"boolean",pause:"(string|boolean)",ride:"(boolean|string)",touch:"boolean",wrap:"boolean"};class xt extends W{constructor(t,e){super(t,e),this._interval=null,this._activeElement=null,this._isSliding=!1,this.touchTimeout=null,this._swipeHelper=null,this._indicatorsElement=z.findOne(".carousel-indicators",this._element),this._addEventListeners(),this._config.ride===vt&&this.cycle()}static get Default(){return Ct}static get DefaultType(){return Ot}static get NAME(){return"carousel"}next(){this._slide(at)}nextWhenVisible(){!document.hidden&&a(this._element)&&this.next()}prev(){this._slide(lt)}pause(){this._isSliding&&s(this._element),this._clearInterval()}cycle(){this._clearInterval(),this._updateInterval(),this._interval=setInterval((()=>this.nextWhenVisible()),this._config.interval)}_maybeEnableCycle(){this._config.ride&&(this._isSliding?N.one(this._element,ut,(()=>this.cycle())):this.cycle())}to(t){const e=this._getItems();if(t>e.length-1||t<0)return;if(this._isSliding)return void N.one(this._element,ut,(()=>this.to(t)));const i=this._getItemIndex(this._getActive());if(i===t)return;const n=t>i?at:lt;this._slide(n,e[t])}dispose(){this._swipeHelper&&this._swipeHelper.dispose(),super.dispose()}_configAfterMerge(t){return t.defaultInterval=t.interval,t}_addEventListeners(){this._config.keyboard&&N.on(this._element,ft,(t=>this._keydown(t))),"hover"===this._config.pause&&(N.on(this._element,pt,(()=>this.pause())),N.on(this._element,mt,(()=>this._maybeEnableCycle()))),this._config.touch&&st.isSupported()&&this._addTouchEventListeners()}_addTouchEventListeners(){for(const t of z.find(".carousel-item img",this._element))N.on(t,gt,(t=>t.preventDefault()));const t={leftCallback:()=>this._slide(this._directionToOrder(ct)),rightCallback:()=>this._slide(this._directionToOrder(ht)),endCallback:()=>{"hover"===this._config.pause&&(this.pause(),this.touchTimeout&&clearTimeout(this.touchTimeout),this.touchTimeout=setTimeout((()=>this._maybeEnableCycle()),500+this._config.interval))}};this._swipeHelper=new st(this._element,t)}_keydown(t){if(/input|textarea/i.test(t.target.tagName))return;const e=Tt[t.key];e&&(t.preventDefault(),this._slide(this._directionToOrder(e)))}_getItemIndex(t){return this._getItems().indexOf(t)}_setActiveIndicatorElement(t){if(!this._indicatorsElement)return;const e=z.findOne(wt,this._indicatorsElement);e.classList.remove(yt),e.removeAttribute("aria-current");const i=z.findOne(`[data-bs-slide-to="${t}"]`,this._indicatorsElement);i&&(i.classList.add(yt),i.setAttribute("aria-current","true"))}_updateInterval(){const t=this._activeElement||this._getActive();if(!t)return;const e=Number.parseInt(t.getAttribute("data-bs-interval"),10);this._config.interval=e||this._config.defaultInterval}_slide(t,e=null){if(this._isSliding)return;const i=this._getActive(),n=t===at,s=e||b(this._getItems(),i,n,this._config.wrap);if(s===i)return;const o=this._getItemIndex(s),r=e=>N.trigger(this._element,e,{relatedTarget:s,direction:this._orderToDirection(t),from:this._getItemIndex(i),to:o});if(r(dt).defaultPrevented)return;if(!i||!s)return;const a=Boolean(this._interval);this.pause(),this._isSliding=!0,this._setActiveIndicatorElement(o),this._activeElement=s;const l=n?"carousel-item-start":"carousel-item-end",c=n?"carousel-item-next":"carousel-item-prev";s.classList.add(c),d(s),i.classList.add(l),s.classList.add(l),this._queueCallback((()=>{s.classList.remove(l,c),s.classList.add(yt),i.classList.remove(yt,c,l),this._isSliding=!1,r(ut)}),i,this._isAnimated()),a&&this.cycle()}_isAnimated(){return this._element.classList.contains("slide")}_getActive(){return z.findOne(Et,this._element)}_getItems(){return z.find(At,this._element)}_clearInterval(){this._interval&&(clearInterval(this._interval),this._interval=null)}_directionToOrder(t){return p()?t===ct?lt:at:t===ct?at:lt}_orderToDirection(t){return p()?t===lt?ct:ht:t===lt?ht:ct}static jQueryInterface(t){return this.each((function(){const e=xt.getOrCreateInstance(this,t);if("number"!=typeof t){if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t]()}}else e.to(t)}))}}N.on(document,bt,"[data-bs-slide], [data-bs-slide-to]",(function(t){const e=z.getElementFromSelector(this);if(!e||!e.classList.contains(vt))return;t.preventDefault();const i=xt.getOrCreateInstance(e),n=this.getAttribute("data-bs-slide-to");return n?(i.to(n),void i._maybeEnableCycle()):"next"===F.getDataAttribute(this,"slide")?(i.next(),void i._maybeEnableCycle()):(i.prev(),void i._maybeEnableCycle())})),N.on(window,_t,(()=>{const t=z.find('[data-bs-ride="carousel"]');for(const e of t)xt.getOrCreateInstance(e)})),m(xt);const kt=".bs.collapse",Lt=`show${kt}`,St=`shown${kt}`,Dt=`hide${kt}`,$t=`hidden${kt}`,It=`click${kt}.data-api`,Nt="show",Pt="collapse",Mt="collapsing",jt=`:scope .${Pt} .${Pt}`,Ft='[data-bs-toggle="collapse"]',Ht={parent:null,toggle:!0},Wt={parent:"(null|element)",toggle:"boolean"};class Bt extends W{constructor(t,e){super(t,e),this._isTransitioning=!1,this._triggerArray=[];const i=z.find(Ft);for(const t of i){const e=z.getSelectorFromElement(t),i=z.find(e).filter((t=>t===this._element));null!==e&&i.length&&this._triggerArray.push(t)}this._initializeChildren(),this._config.parent||this._addAriaAndCollapsedClass(this._triggerArray,this._isShown()),this._config.toggle&&this.toggle()}static get Default(){return Ht}static get DefaultType(){return Wt}static get NAME(){return"collapse"}toggle(){this._isShown()?this.hide():this.show()}show(){if(this._isTransitioning||this._isShown())return;let t=[];if(this._config.parent&&(t=this._getFirstLevelChildren(".collapse.show, .collapse.collapsing").filter((t=>t!==this._element)).map((t=>Bt.getOrCreateInstance(t,{toggle:!1})))),t.length&&t[0]._isTransitioning)return;if(N.trigger(this._element,Lt).defaultPrevented)return;for(const e of t)e.hide();const e=this._getDimension();this._element.classList.remove(Pt),this._element.classList.add(Mt),this._element.style[e]=0,this._addAriaAndCollapsedClass(this._triggerArray,!0),this._isTransitioning=!0;const i=`scroll${e[0].toUpperCase()+e.slice(1)}`;this._queueCallback((()=>{this._isTransitioning=!1,this._element.classList.remove(Mt),this._element.classList.add(Pt,Nt),this._element.style[e]="",N.trigger(this._element,St)}),this._element,!0),this._element.style[e]=`${this._element[i]}px`}hide(){if(this._isTransitioning||!this._isShown())return;if(N.trigger(this._element,Dt).defaultPrevented)return;const t=this._getDimension();this._element.style[t]=`${this._element.getBoundingClientRect()[t]}px`,d(this._element),this._element.classList.add(Mt),this._element.classList.remove(Pt,Nt);for(const t of this._triggerArray){const e=z.getElementFromSelector(t);e&&!this._isShown(e)&&this._addAriaAndCollapsedClass([t],!1)}this._isTransitioning=!0,this._element.style[t]="",this._queueCallback((()=>{this._isTransitioning=!1,this._element.classList.remove(Mt),this._element.classList.add(Pt),N.trigger(this._element,$t)}),this._element,!0)}_isShown(t=this._element){return t.classList.contains(Nt)}_configAfterMerge(t){return t.toggle=Boolean(t.toggle),t.parent=r(t.parent),t}_getDimension(){return this._element.classList.contains("collapse-horizontal")?"width":"height"}_initializeChildren(){if(!this._config.parent)return;const t=this._getFirstLevelChildren(Ft);for(const e of t){const t=z.getElementFromSelector(e);t&&this._addAriaAndCollapsedClass([e],this._isShown(t))}}_getFirstLevelChildren(t){const e=z.find(jt,this._config.parent);return z.find(t,this._config.parent).filter((t=>!e.includes(t)))}_addAriaAndCollapsedClass(t,e){if(t.length)for(const i of t)i.classList.toggle("collapsed",!e),i.setAttribute("aria-expanded",e)}static jQueryInterface(t){const e={};return"string"==typeof t&&/show|hide/.test(t)&&(e.toggle=!1),this.each((function(){const i=Bt.getOrCreateInstance(this,e);if("string"==typeof t){if(void 0===i[t])throw new TypeError(`No method named "${t}"`);i[t]()}}))}}N.on(document,It,Ft,(function(t){("A"===t.target.tagName||t.delegateTarget&&"A"===t.delegateTarget.tagName)&&t.preventDefault();for(const t of z.getMultipleElementsFromSelector(this))Bt.getOrCreateInstance(t,{toggle:!1}).toggle()})),m(Bt);var zt="top",Rt="bottom",qt="right",Vt="left",Kt="auto",Qt=[zt,Rt,qt,Vt],Xt="start",Yt="end",Ut="clippingParents",Gt="viewport",Jt="popper",Zt="reference",te=Qt.reduce((function(t,e){return t.concat([e+"-"+Xt,e+"-"+Yt])}),[]),ee=[].concat(Qt,[Kt]).reduce((function(t,e){return t.concat([e,e+"-"+Xt,e+"-"+Yt])}),[]),ie="beforeRead",ne="read",se="afterRead",oe="beforeMain",re="main",ae="afterMain",le="beforeWrite",ce="write",he="afterWrite",de=[ie,ne,se,oe,re,ae,le,ce,he];function ue(t){return t?(t.nodeName||"").toLowerCase():null}function fe(t){if(null==t)return window;if("[object Window]"!==t.toString()){var e=t.ownerDocument;return e&&e.defaultView||window}return t}function pe(t){return t instanceof fe(t).Element||t instanceof Element}function me(t){return t instanceof fe(t).HTMLElement||t instanceof HTMLElement}function ge(t){return"undefined"!=typeof ShadowRoot&&(t instanceof fe(t).ShadowRoot||t instanceof ShadowRoot)}const _e={name:"applyStyles",enabled:!0,phase:"write",fn:function(t){var e=t.state;Object.keys(e.elements).forEach((function(t){var i=e.styles[t]||{},n=e.attributes[t]||{},s=e.elements[t];me(s)&&ue(s)&&(Object.assign(s.style,i),Object.keys(n).forEach((function(t){var e=n[t];!1===e?s.removeAttribute(t):s.setAttribute(t,!0===e?"":e)})))}))},effect:function(t){var e=t.state,i={popper:{position:e.options.strategy,left:"0",top:"0",margin:"0"},arrow:{position:"absolute"},reference:{}};return Object.assign(e.elements.popper.style,i.popper),e.styles=i,e.elements.arrow&&Object.assign(e.elements.arrow.style,i.arrow),function(){Object.keys(e.elements).forEach((function(t){var n=e.elements[t],s=e.attributes[t]||{},o=Object.keys(e.styles.hasOwnProperty(t)?e.styles[t]:i[t]).reduce((function(t,e){return t[e]="",t}),{});me(n)&&ue(n)&&(Object.assign(n.style,o),Object.keys(s).forEach((function(t){n.removeAttribute(t)})))}))}},requires:["computeStyles"]};function be(t){return t.split("-")[0]}var ve=Math.max,ye=Math.min,we=Math.round;function Ae(){var t=navigator.userAgentData;return null!=t&&t.brands&&Array.isArray(t.brands)?t.brands.map((function(t){return t.brand+"/"+t.version})).join(" "):navigator.userAgent}function Ee(){return!/^((?!chrome|android).)*safari/i.test(Ae())}function Te(t,e,i){void 0===e&&(e=!1),void 0===i&&(i=!1);var n=t.getBoundingClientRect(),s=1,o=1;e&&me(t)&&(s=t.offsetWidth>0&&we(n.width)/t.offsetWidth||1,o=t.offsetHeight>0&&we(n.height)/t.offsetHeight||1);var r=(pe(t)?fe(t):window).visualViewport,a=!Ee()&&i,l=(n.left+(a&&r?r.offsetLeft:0))/s,c=(n.top+(a&&r?r.offsetTop:0))/o,h=n.width/s,d=n.height/o;return{width:h,height:d,top:c,right:l+h,bottom:c+d,left:l,x:l,y:c}}function Ce(t){var e=Te(t),i=t.offsetWidth,n=t.offsetHeight;return Math.abs(e.width-i)<=1&&(i=e.width),Math.abs(e.height-n)<=1&&(n=e.height),{x:t.offsetLeft,y:t.offsetTop,width:i,height:n}}function Oe(t,e){var i=e.getRootNode&&e.getRootNode();if(t.contains(e))return!0;if(i&&ge(i)){var n=e;do{if(n&&t.isSameNode(n))return!0;n=n.parentNode||n.host}while(n)}return!1}function xe(t){return fe(t).getComputedStyle(t)}function ke(t){return["table","td","th"].indexOf(ue(t))>=0}function Le(t){return((pe(t)?t.ownerDocument:t.document)||window.document).documentElement}function Se(t){return"html"===ue(t)?t:t.assignedSlot||t.parentNode||(ge(t)?t.host:null)||Le(t)}function De(t){return me(t)&&"fixed"!==xe(t).position?t.offsetParent:null}function $e(t){for(var e=fe(t),i=De(t);i&&ke(i)&&"static"===xe(i).position;)i=De(i);return i&&("html"===ue(i)||"body"===ue(i)&&"static"===xe(i).position)?e:i||function(t){var e=/firefox/i.test(Ae());if(/Trident/i.test(Ae())&&me(t)&&"fixed"===xe(t).position)return null;var i=Se(t);for(ge(i)&&(i=i.host);me(i)&&["html","body"].indexOf(ue(i))<0;){var n=xe(i);if("none"!==n.transform||"none"!==n.perspective||"paint"===n.contain||-1!==["transform","perspective"].indexOf(n.willChange)||e&&"filter"===n.willChange||e&&n.filter&&"none"!==n.filter)return i;i=i.parentNode}return null}(t)||e}function Ie(t){return["top","bottom"].indexOf(t)>=0?"x":"y"}function Ne(t,e,i){return ve(t,ye(e,i))}function Pe(t){return Object.assign({},{top:0,right:0,bottom:0,left:0},t)}function Me(t,e){return e.reduce((function(e,i){return e[i]=t,e}),{})}const je={name:"arrow",enabled:!0,phase:"main",fn:function(t){var e,i=t.state,n=t.name,s=t.options,o=i.elements.arrow,r=i.modifiersData.popperOffsets,a=be(i.placement),l=Ie(a),c=[Vt,qt].indexOf(a)>=0?"height":"width";if(o&&r){var h=function(t,e){return Pe("number"!=typeof(t="function"==typeof t?t(Object.assign({},e.rects,{placement:e.placement})):t)?t:Me(t,Qt))}(s.padding,i),d=Ce(o),u="y"===l?zt:Vt,f="y"===l?Rt:qt,p=i.rects.reference[c]+i.rects.reference[l]-r[l]-i.rects.popper[c],m=r[l]-i.rects.reference[l],g=$e(o),_=g?"y"===l?g.clientHeight||0:g.clientWidth||0:0,b=p/2-m/2,v=h[u],y=_-d[c]-h[f],w=_/2-d[c]/2+b,A=Ne(v,w,y),E=l;i.modifiersData[n]=((e={})[E]=A,e.centerOffset=A-w,e)}},effect:function(t){var e=t.state,i=t.options.element,n=void 0===i?"[data-popper-arrow]":i;null!=n&&("string"!=typeof n||(n=e.elements.popper.querySelector(n)))&&Oe(e.elements.popper,n)&&(e.elements.arrow=n)},requires:["popperOffsets"],requiresIfExists:["preventOverflow"]};function Fe(t){return t.split("-")[1]}var He={top:"auto",right:"auto",bottom:"auto",left:"auto"};function We(t){var e,i=t.popper,n=t.popperRect,s=t.placement,o=t.variation,r=t.offsets,a=t.position,l=t.gpuAcceleration,c=t.adaptive,h=t.roundOffsets,d=t.isFixed,u=r.x,f=void 0===u?0:u,p=r.y,m=void 0===p?0:p,g="function"==typeof h?h({x:f,y:m}):{x:f,y:m};f=g.x,m=g.y;var _=r.hasOwnProperty("x"),b=r.hasOwnProperty("y"),v=Vt,y=zt,w=window;if(c){var A=$e(i),E="clientHeight",T="clientWidth";A===fe(i)&&"static"!==xe(A=Le(i)).position&&"absolute"===a&&(E="scrollHeight",T="scrollWidth"),(s===zt||(s===Vt||s===qt)&&o===Yt)&&(y=Rt,m-=(d&&A===w&&w.visualViewport?w.visualViewport.height:A[E])-n.height,m*=l?1:-1),s!==Vt&&(s!==zt&&s!==Rt||o!==Yt)||(v=qt,f-=(d&&A===w&&w.visualViewport?w.visualViewport.width:A[T])-n.width,f*=l?1:-1)}var C,O=Object.assign({position:a},c&&He),x=!0===h?function(t,e){var i=t.x,n=t.y,s=e.devicePixelRatio||1;return{x:we(i*s)/s||0,y:we(n*s)/s||0}}({x:f,y:m},fe(i)):{x:f,y:m};return f=x.x,m=x.y,l?Object.assign({},O,((C={})[y]=b?"0":"",C[v]=_?"0":"",C.transform=(w.devicePixelRatio||1)<=1?"translate("+f+"px, "+m+"px)":"translate3d("+f+"px, "+m+"px, 0)",C)):Object.assign({},O,((e={})[y]=b?m+"px":"",e[v]=_?f+"px":"",e.transform="",e))}const Be={name:"computeStyles",enabled:!0,phase:"beforeWrite",fn:function(t){var e=t.state,i=t.options,n=i.gpuAcceleration,s=void 0===n||n,o=i.adaptive,r=void 0===o||o,a=i.roundOffsets,l=void 0===a||a,c={placement:be(e.placement),variation:Fe(e.placement),popper:e.elements.popper,popperRect:e.rects.popper,gpuAcceleration:s,isFixed:"fixed"===e.options.strategy};null!=e.modifiersData.popperOffsets&&(e.styles.popper=Object.assign({},e.styles.popper,We(Object.assign({},c,{offsets:e.modifiersData.popperOffsets,position:e.options.strategy,adaptive:r,roundOffsets:l})))),null!=e.modifiersData.arrow&&(e.styles.arrow=Object.assign({},e.styles.arrow,We(Object.assign({},c,{offsets:e.modifiersData.arrow,position:"absolute",adaptive:!1,roundOffsets:l})))),e.attributes.popper=Object.assign({},e.attributes.popper,{"data-popper-placement":e.placement})},data:{}};var ze={passive:!0};const Re={name:"eventListeners",enabled:!0,phase:"write",fn:function(){},effect:function(t){var e=t.state,i=t.instance,n=t.options,s=n.scroll,o=void 0===s||s,r=n.resize,a=void 0===r||r,l=fe(e.elements.popper),c=[].concat(e.scrollParents.reference,e.scrollParents.popper);return o&&c.forEach((function(t){t.addEventListener("scroll",i.update,ze)})),a&&l.addEventListener("resize",i.update,ze),function(){o&&c.forEach((function(t){t.removeEventListener("scroll",i.update,ze)})),a&&l.removeEventListener("resize",i.update,ze)}},data:{}};var qe={left:"right",right:"left",bottom:"top",top:"bottom"};function Ve(t){return t.replace(/left|right|bottom|top/g,(function(t){return qe[t]}))}var Ke={start:"end",end:"start"};function Qe(t){return t.replace(/start|end/g,(function(t){return Ke[t]}))}function Xe(t){var e=fe(t);return{scrollLeft:e.pageXOffset,scrollTop:e.pageYOffset}}function Ye(t){return Te(Le(t)).left+Xe(t).scrollLeft}function Ue(t){var e=xe(t),i=e.overflow,n=e.overflowX,s=e.overflowY;return/auto|scroll|overlay|hidden/.test(i+s+n)}function Ge(t){return["html","body","#document"].indexOf(ue(t))>=0?t.ownerDocument.body:me(t)&&Ue(t)?t:Ge(Se(t))}function Je(t,e){var i;void 0===e&&(e=[]);var n=Ge(t),s=n===(null==(i=t.ownerDocument)?void 0:i.body),o=fe(n),r=s?[o].concat(o.visualViewport||[],Ue(n)?n:[]):n,a=e.concat(r);return s?a:a.concat(Je(Se(r)))}function Ze(t){return Object.assign({},t,{left:t.x,top:t.y,right:t.x+t.width,bottom:t.y+t.height})}function ti(t,e,i){return e===Gt?Ze(function(t,e){var i=fe(t),n=Le(t),s=i.visualViewport,o=n.clientWidth,r=n.clientHeight,a=0,l=0;if(s){o=s.width,r=s.height;var c=Ee();(c||!c&&"fixed"===e)&&(a=s.offsetLeft,l=s.offsetTop)}return{width:o,height:r,x:a+Ye(t),y:l}}(t,i)):pe(e)?function(t,e){var i=Te(t,!1,"fixed"===e);return i.top=i.top+t.clientTop,i.left=i.left+t.clientLeft,i.bottom=i.top+t.clientHeight,i.right=i.left+t.clientWidth,i.width=t.clientWidth,i.height=t.clientHeight,i.x=i.left,i.y=i.top,i}(e,i):Ze(function(t){var e,i=Le(t),n=Xe(t),s=null==(e=t.ownerDocument)?void 0:e.body,o=ve(i.scrollWidth,i.clientWidth,s?s.scrollWidth:0,s?s.clientWidth:0),r=ve(i.scrollHeight,i.clientHeight,s?s.scrollHeight:0,s?s.clientHeight:0),a=-n.scrollLeft+Ye(t),l=-n.scrollTop;return"rtl"===xe(s||i).direction&&(a+=ve(i.clientWidth,s?s.clientWidth:0)-o),{width:o,height:r,x:a,y:l}}(Le(t)))}function ei(t){var e,i=t.reference,n=t.element,s=t.placement,o=s?be(s):null,r=s?Fe(s):null,a=i.x+i.width/2-n.width/2,l=i.y+i.height/2-n.height/2;switch(o){case zt:e={x:a,y:i.y-n.height};break;case Rt:e={x:a,y:i.y+i.height};break;case qt:e={x:i.x+i.width,y:l};break;case Vt:e={x:i.x-n.width,y:l};break;default:e={x:i.x,y:i.y}}var c=o?Ie(o):null;if(null!=c){var h="y"===c?"height":"width";switch(r){case Xt:e[c]=e[c]-(i[h]/2-n[h]/2);break;case Yt:e[c]=e[c]+(i[h]/2-n[h]/2)}}return e}function ii(t,e){void 0===e&&(e={});var i=e,n=i.placement,s=void 0===n?t.placement:n,o=i.strategy,r=void 0===o?t.strategy:o,a=i.boundary,l=void 0===a?Ut:a,c=i.rootBoundary,h=void 0===c?Gt:c,d=i.elementContext,u=void 0===d?Jt:d,f=i.altBoundary,p=void 0!==f&&f,m=i.padding,g=void 0===m?0:m,_=Pe("number"!=typeof g?g:Me(g,Qt)),b=u===Jt?Zt:Jt,v=t.rects.popper,y=t.elements[p?b:u],w=function(t,e,i,n){var s="clippingParents"===e?function(t){var e=Je(Se(t)),i=["absolute","fixed"].indexOf(xe(t).position)>=0&&me(t)?$e(t):t;return pe(i)?e.filter((function(t){return pe(t)&&Oe(t,i)&&"body"!==ue(t)})):[]}(t):[].concat(e),o=[].concat(s,[i]),r=o[0],a=o.reduce((function(e,i){var s=ti(t,i,n);return e.top=ve(s.top,e.top),e.right=ye(s.right,e.right),e.bottom=ye(s.bottom,e.bottom),e.left=ve(s.left,e.left),e}),ti(t,r,n));return a.width=a.right-a.left,a.height=a.bottom-a.top,a.x=a.left,a.y=a.top,a}(pe(y)?y:y.contextElement||Le(t.elements.popper),l,h,r),A=Te(t.elements.reference),E=ei({reference:A,element:v,strategy:"absolute",placement:s}),T=Ze(Object.assign({},v,E)),C=u===Jt?T:A,O={top:w.top-C.top+_.top,bottom:C.bottom-w.bottom+_.bottom,left:w.left-C.left+_.left,right:C.right-w.right+_.right},x=t.modifiersData.offset;if(u===Jt&&x){var k=x[s];Object.keys(O).forEach((function(t){var e=[qt,Rt].indexOf(t)>=0?1:-1,i=[zt,Rt].indexOf(t)>=0?"y":"x";O[t]+=k[i]*e}))}return O}function ni(t,e){void 0===e&&(e={});var i=e,n=i.placement,s=i.boundary,o=i.rootBoundary,r=i.padding,a=i.flipVariations,l=i.allowedAutoPlacements,c=void 0===l?ee:l,h=Fe(n),d=h?a?te:te.filter((function(t){return Fe(t)===h})):Qt,u=d.filter((function(t){return c.indexOf(t)>=0}));0===u.length&&(u=d);var f=u.reduce((function(e,i){return e[i]=ii(t,{placement:i,boundary:s,rootBoundary:o,padding:r})[be(i)],e}),{});return Object.keys(f).sort((function(t,e){return f[t]-f[e]}))}const si={name:"flip",enabled:!0,phase:"main",fn:function(t){var e=t.state,i=t.options,n=t.name;if(!e.modifiersData[n]._skip){for(var s=i.mainAxis,o=void 0===s||s,r=i.altAxis,a=void 0===r||r,l=i.fallbackPlacements,c=i.padding,h=i.boundary,d=i.rootBoundary,u=i.altBoundary,f=i.flipVariations,p=void 0===f||f,m=i.allowedAutoPlacements,g=e.options.placement,_=be(g),b=l||(_!==g&&p?function(t){if(be(t)===Kt)return[];var e=Ve(t);return[Qe(t),e,Qe(e)]}(g):[Ve(g)]),v=[g].concat(b).reduce((function(t,i){return t.concat(be(i)===Kt?ni(e,{placement:i,boundary:h,rootBoundary:d,padding:c,flipVariations:p,allowedAutoPlacements:m}):i)}),[]),y=e.rects.reference,w=e.rects.popper,A=new Map,E=!0,T=v[0],C=0;C<v.length;C++){var O=v[C],x=be(O),k=Fe(O)===Xt,L=[zt,Rt].indexOf(x)>=0,S=L?"width":"height",D=ii(e,{placement:O,boundary:h,rootBoundary:d,altBoundary:u,padding:c}),$=L?k?qt:Vt:k?Rt:zt;y[S]>w[S]&&($=Ve($));var I=Ve($),N=[];if(o&&N.push(D[x]<=0),a&&N.push(D[$]<=0,D[I]<=0),N.every((function(t){return t}))){T=O,E=!1;break}A.set(O,N)}if(E)for(var P=function(t){var e=v.find((function(e){var i=A.get(e);if(i)return i.slice(0,t).every((function(t){return t}))}));if(e)return T=e,"break"},M=p?3:1;M>0&&"break"!==P(M);M--);e.placement!==T&&(e.modifiersData[n]._skip=!0,e.placement=T,e.reset=!0)}},requiresIfExists:["offset"],data:{_skip:!1}};function oi(t,e,i){return void 0===i&&(i={x:0,y:0}),{top:t.top-e.height-i.y,right:t.right-e.width+i.x,bottom:t.bottom-e.height+i.y,left:t.left-e.width-i.x}}function ri(t){return[zt,qt,Rt,Vt].some((function(e){return t[e]>=0}))}const ai={name:"hide",enabled:!0,phase:"main",requiresIfExists:["preventOverflow"],fn:function(t){var e=t.state,i=t.name,n=e.rects.reference,s=e.rects.popper,o=e.modifiersData.preventOverflow,r=ii(e,{elementContext:"reference"}),a=ii(e,{altBoundary:!0}),l=oi(r,n),c=oi(a,s,o),h=ri(l),d=ri(c);e.modifiersData[i]={referenceClippingOffsets:l,popperEscapeOffsets:c,isReferenceHidden:h,hasPopperEscaped:d},e.attributes.popper=Object.assign({},e.attributes.popper,{"data-popper-reference-hidden":h,"data-popper-escaped":d})}},li={name:"offset",enabled:!0,phase:"main",requires:["popperOffsets"],fn:function(t){var e=t.state,i=t.options,n=t.name,s=i.offset,o=void 0===s?[0,0]:s,r=ee.reduce((function(t,i){return t[i]=function(t,e,i){var n=be(t),s=[Vt,zt].indexOf(n)>=0?-1:1,o="function"==typeof i?i(Object.assign({},e,{placement:t})):i,r=o[0],a=o[1];return r=r||0,a=(a||0)*s,[Vt,qt].indexOf(n)>=0?{x:a,y:r}:{x:r,y:a}}(i,e.rects,o),t}),{}),a=r[e.placement],l=a.x,c=a.y;null!=e.modifiersData.popperOffsets&&(e.modifiersData.popperOffsets.x+=l,e.modifiersData.popperOffsets.y+=c),e.modifiersData[n]=r}},ci={name:"popperOffsets",enabled:!0,phase:"read",fn:function(t){var e=t.state,i=t.name;e.modifiersData[i]=ei({reference:e.rects.reference,element:e.rects.popper,strategy:"absolute",placement:e.placement})},data:{}},hi={name:"preventOverflow",enabled:!0,phase:"main",fn:function(t){var e=t.state,i=t.options,n=t.name,s=i.mainAxis,o=void 0===s||s,r=i.altAxis,a=void 0!==r&&r,l=i.boundary,c=i.rootBoundary,h=i.altBoundary,d=i.padding,u=i.tether,f=void 0===u||u,p=i.tetherOffset,m=void 0===p?0:p,g=ii(e,{boundary:l,rootBoundary:c,padding:d,altBoundary:h}),_=be(e.placement),b=Fe(e.placement),v=!b,y=Ie(_),w="x"===y?"y":"x",A=e.modifiersData.popperOffsets,E=e.rects.reference,T=e.rects.popper,C="function"==typeof m?m(Object.assign({},e.rects,{placement:e.placement})):m,O="number"==typeof C?{mainAxis:C,altAxis:C}:Object.assign({mainAxis:0,altAxis:0},C),x=e.modifiersData.offset?e.modifiersData.offset[e.placement]:null,k={x:0,y:0};if(A){if(o){var L,S="y"===y?zt:Vt,D="y"===y?Rt:qt,$="y"===y?"height":"width",I=A[y],N=I+g[S],P=I-g[D],M=f?-T[$]/2:0,j=b===Xt?E[$]:T[$],F=b===Xt?-T[$]:-E[$],H=e.elements.arrow,W=f&&H?Ce(H):{width:0,height:0},B=e.modifiersData["arrow#persistent"]?e.modifiersData["arrow#persistent"].padding:{top:0,right:0,bottom:0,left:0},z=B[S],R=B[D],q=Ne(0,E[$],W[$]),V=v?E[$]/2-M-q-z-O.mainAxis:j-q-z-O.mainAxis,K=v?-E[$]/2+M+q+R+O.mainAxis:F+q+R+O.mainAxis,Q=e.elements.arrow&&$e(e.elements.arrow),X=Q?"y"===y?Q.clientTop||0:Q.clientLeft||0:0,Y=null!=(L=null==x?void 0:x[y])?L:0,U=I+K-Y,G=Ne(f?ye(N,I+V-Y-X):N,I,f?ve(P,U):P);A[y]=G,k[y]=G-I}if(a){var J,Z="x"===y?zt:Vt,tt="x"===y?Rt:qt,et=A[w],it="y"===w?"height":"width",nt=et+g[Z],st=et-g[tt],ot=-1!==[zt,Vt].indexOf(_),rt=null!=(J=null==x?void 0:x[w])?J:0,at=ot?nt:et-E[it]-T[it]-rt+O.altAxis,lt=ot?et+E[it]+T[it]-rt-O.altAxis:st,ct=f&&ot?function(t,e,i){var n=Ne(t,e,i);return n>i?i:n}(at,et,lt):Ne(f?at:nt,et,f?lt:st);A[w]=ct,k[w]=ct-et}e.modifiersData[n]=k}},requiresIfExists:["offset"]};function di(t,e,i){void 0===i&&(i=!1);var n,s,o=me(e),r=me(e)&&function(t){var e=t.getBoundingClientRect(),i=we(e.width)/t.offsetWidth||1,n=we(e.height)/t.offsetHeight||1;return 1!==i||1!==n}(e),a=Le(e),l=Te(t,r,i),c={scrollLeft:0,scrollTop:0},h={x:0,y:0};return(o||!o&&!i)&&(("body"!==ue(e)||Ue(a))&&(c=(n=e)!==fe(n)&&me(n)?{scrollLeft:(s=n).scrollLeft,scrollTop:s.scrollTop}:Xe(n)),me(e)?((h=Te(e,!0)).x+=e.clientLeft,h.y+=e.clientTop):a&&(h.x=Ye(a))),{x:l.left+c.scrollLeft-h.x,y:l.top+c.scrollTop-h.y,width:l.width,height:l.height}}function ui(t){var e=new Map,i=new Set,n=[];function s(t){i.add(t.name),[].concat(t.requires||[],t.requiresIfExists||[]).forEach((function(t){if(!i.has(t)){var n=e.get(t);n&&s(n)}})),n.push(t)}return t.forEach((function(t){e.set(t.name,t)})),t.forEach((function(t){i.has(t.name)||s(t)})),n}var fi={placement:"bottom",modifiers:[],strategy:"absolute"};function pi(){for(var t=arguments.length,e=new Array(t),i=0;i<t;i++)e[i]=arguments[i];return!e.some((function(t){return!(t&&"function"==typeof t.getBoundingClientRect)}))}function mi(t){void 0===t&&(t={});var e=t,i=e.defaultModifiers,n=void 0===i?[]:i,s=e.defaultOptions,o=void 0===s?fi:s;return function(t,e,i){void 0===i&&(i=o);var s,r,a={placement:"bottom",orderedModifiers:[],options:Object.assign({},fi,o),modifiersData:{},elements:{reference:t,popper:e},attributes:{},styles:{}},l=[],c=!1,h={state:a,setOptions:function(i){var s="function"==typeof i?i(a.options):i;d(),a.options=Object.assign({},o,a.options,s),a.scrollParents={reference:pe(t)?Je(t):t.contextElement?Je(t.contextElement):[],popper:Je(e)};var r,c,u=function(t){var e=ui(t);return de.reduce((function(t,i){return t.concat(e.filter((function(t){return t.phase===i})))}),[])}((r=[].concat(n,a.options.modifiers),c=r.reduce((function(t,e){var i=t[e.name];return t[e.name]=i?Object.assign({},i,e,{options:Object.assign({},i.options,e.options),data:Object.assign({},i.data,e.data)}):e,t}),{}),Object.keys(c).map((function(t){return c[t]}))));return a.orderedModifiers=u.filter((function(t){return t.enabled})),a.orderedModifiers.forEach((function(t){var e=t.name,i=t.options,n=void 0===i?{}:i,s=t.effect;if("function"==typeof s){var o=s({state:a,name:e,instance:h,options:n});l.push(o||function(){})}})),h.update()},forceUpdate:function(){if(!c){var t=a.elements,e=t.reference,i=t.popper;if(pi(e,i)){a.rects={reference:di(e,$e(i),"fixed"===a.options.strategy),popper:Ce(i)},a.reset=!1,a.placement=a.options.placement,a.orderedModifiers.forEach((function(t){return a.modifiersData[t.name]=Object.assign({},t.data)}));for(var n=0;n<a.orderedModifiers.length;n++)if(!0!==a.reset){var s=a.orderedModifiers[n],o=s.fn,r=s.options,l=void 0===r?{}:r,d=s.name;"function"==typeof o&&(a=o({state:a,options:l,name:d,instance:h})||a)}else a.reset=!1,n=-1}}},update:(s=function(){return new Promise((function(t){h.forceUpdate(),t(a)}))},function(){return r||(r=new Promise((function(t){Promise.resolve().then((function(){r=void 0,t(s())}))}))),r}),destroy:function(){d(),c=!0}};if(!pi(t,e))return h;function d(){l.forEach((function(t){return t()})),l=[]}return h.setOptions(i).then((function(t){!c&&i.onFirstUpdate&&i.onFirstUpdate(t)})),h}}var gi=mi(),_i=mi({defaultModifiers:[Re,ci,Be,_e]}),bi=mi({defaultModifiers:[Re,ci,Be,_e,li,si,hi,je,ai]});const vi=Object.freeze(Object.defineProperty({__proto__:null,afterMain:ae,afterRead:se,afterWrite:he,applyStyles:_e,arrow:je,auto:Kt,basePlacements:Qt,beforeMain:oe,beforeRead:ie,beforeWrite:le,bottom:Rt,clippingParents:Ut,computeStyles:Be,createPopper:bi,createPopperBase:gi,createPopperLite:_i,detectOverflow:ii,end:Yt,eventListeners:Re,flip:si,hide:ai,left:Vt,main:re,modifierPhases:de,offset:li,placements:ee,popper:Jt,popperGenerator:mi,popperOffsets:ci,preventOverflow:hi,read:ne,reference:Zt,right:qt,start:Xt,top:zt,variationPlacements:te,viewport:Gt,write:ce},Symbol.toStringTag,{value:"Module"})),yi="dropdown",wi=".bs.dropdown",Ai=".data-api",Ei="ArrowUp",Ti="ArrowDown",Ci=`hide${wi}`,Oi=`hidden${wi}`,xi=`show${wi}`,ki=`shown${wi}`,Li=`click${wi}${Ai}`,Si=`keydown${wi}${Ai}`,Di=`keyup${wi}${Ai}`,$i="show",Ii='[data-bs-toggle="dropdown"]:not(.disabled):not(:disabled)',Ni=`${Ii}.${$i}`,Pi=".dropdown-menu",Mi=p()?"top-end":"top-start",ji=p()?"top-start":"top-end",Fi=p()?"bottom-end":"bottom-start",Hi=p()?"bottom-start":"bottom-end",Wi=p()?"left-start":"right-start",Bi=p()?"right-start":"left-start",zi={autoClose:!0,boundary:"clippingParents",display:"dynamic",offset:[0,2],popperConfig:null,reference:"toggle"},Ri={autoClose:"(boolean|string)",boundary:"(string|element)",display:"string",offset:"(array|string|function)",popperConfig:"(null|object|function)",reference:"(string|element|object)"};class qi extends W{constructor(t,e){super(t,e),this._popper=null,this._parent=this._element.parentNode,this._menu=z.next(this._element,Pi)[0]||z.prev(this._element,Pi)[0]||z.findOne(Pi,this._parent),this._inNavbar=this._detectNavbar()}static get Default(){return zi}static get DefaultType(){return Ri}static get NAME(){return yi}toggle(){return this._isShown()?this.hide():this.show()}show(){if(l(this._element)||this._isShown())return;const t={relatedTarget:this._element};if(!N.trigger(this._element,xi,t).defaultPrevented){if(this._createPopper(),"ontouchstart"in document.documentElement&&!this._parent.closest(".navbar-nav"))for(const t of[].concat(...document.body.children))N.on(t,"mouseover",h);this._element.focus(),this._element.setAttribute("aria-expanded",!0),this._menu.classList.add($i),this._element.classList.add($i),N.trigger(this._element,ki,t)}}hide(){if(l(this._element)||!this._isShown())return;const t={relatedTarget:this._element};this._completeHide(t)}dispose(){this._popper&&this._popper.destroy(),super.dispose()}update(){this._inNavbar=this._detectNavbar(),this._popper&&this._popper.update()}_completeHide(t){if(!N.trigger(this._element,Ci,t).defaultPrevented){if("ontouchstart"in document.documentElement)for(const t of[].concat(...document.body.children))N.off(t,"mouseover",h);this._popper&&this._popper.destroy(),this._menu.classList.remove($i),this._element.classList.remove($i),this._element.setAttribute("aria-expanded","false"),F.removeDataAttribute(this._menu,"popper"),N.trigger(this._element,Oi,t)}}_getConfig(t){if("object"==typeof(t=super._getConfig(t)).reference&&!o(t.reference)&&"function"!=typeof t.reference.getBoundingClientRect)throw new TypeError(`${yi.toUpperCase()}: Option "reference" provided type "object" without a required "getBoundingClientRect" method.`);return t}_createPopper(){if(void 0===vi)throw new TypeError("Bootstrap's dropdowns require Popper (https://popper.js.org)");let t=this._element;"parent"===this._config.reference?t=this._parent:o(this._config.reference)?t=r(this._config.reference):"object"==typeof this._config.reference&&(t=this._config.reference);const e=this._getPopperConfig();this._popper=bi(t,this._menu,e)}_isShown(){return this._menu.classList.contains($i)}_getPlacement(){const t=this._parent;if(t.classList.contains("dropend"))return Wi;if(t.classList.contains("dropstart"))return Bi;if(t.classList.contains("dropup-center"))return"top";if(t.classList.contains("dropdown-center"))return"bottom";const e="end"===getComputedStyle(this._menu).getPropertyValue("--bs-position").trim();return t.classList.contains("dropup")?e?ji:Mi:e?Hi:Fi}_detectNavbar(){return null!==this._element.closest(".navbar")}_getOffset(){const{offset:t}=this._config;return"string"==typeof t?t.split(",").map((t=>Number.parseInt(t,10))):"function"==typeof t?e=>t(e,this._element):t}_getPopperConfig(){const t={placement:this._getPlacement(),modifiers:[{name:"preventOverflow",options:{boundary:this._config.boundary}},{name:"offset",options:{offset:this._getOffset()}}]};return(this._inNavbar||"static"===this._config.display)&&(F.setDataAttribute(this._menu,"popper","static"),t.modifiers=[{name:"applyStyles",enabled:!1}]),{...t,...g(this._config.popperConfig,[t])}}_selectMenuItem({key:t,target:e}){const i=z.find(".dropdown-menu .dropdown-item:not(.disabled):not(:disabled)",this._menu).filter((t=>a(t)));i.length&&b(i,e,t===Ti,!i.includes(e)).focus()}static jQueryInterface(t){return this.each((function(){const e=qi.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t]()}}))}static clearMenus(t){if(2===t.button||"keyup"===t.type&&"Tab"!==t.key)return;const e=z.find(Ni);for(const i of e){const e=qi.getInstance(i);if(!e||!1===e._config.autoClose)continue;const n=t.composedPath(),s=n.includes(e._menu);if(n.includes(e._element)||"inside"===e._config.autoClose&&!s||"outside"===e._config.autoClose&&s)continue;if(e._menu.contains(t.target)&&("keyup"===t.type&&"Tab"===t.key||/input|select|option|textarea|form/i.test(t.target.tagName)))continue;const o={relatedTarget:e._element};"click"===t.type&&(o.clickEvent=t),e._completeHide(o)}}static dataApiKeydownHandler(t){const e=/input|textarea/i.test(t.target.tagName),i="Escape"===t.key,n=[Ei,Ti].includes(t.key);if(!n&&!i)return;if(e&&!i)return;t.preventDefault();const s=this.matches(Ii)?this:z.prev(this,Ii)[0]||z.next(this,Ii)[0]||z.findOne(Ii,t.delegateTarget.parentNode),o=qi.getOrCreateInstance(s);if(n)return t.stopPropagation(),o.show(),void o._selectMenuItem(t);o._isShown()&&(t.stopPropagation(),o.hide(),s.focus())}}N.on(document,Si,Ii,qi.dataApiKeydownHandler),N.on(document,Si,Pi,qi.dataApiKeydownHandler),N.on(document,Li,qi.clearMenus),N.on(document,Di,qi.clearMenus),N.on(document,Li,Ii,(function(t){t.preventDefault(),qi.getOrCreateInstance(this).toggle()})),m(qi);const Vi="backdrop",Ki="show",Qi=`mousedown.bs.${Vi}`,Xi={className:"modal-backdrop",clickCallback:null,isAnimated:!1,isVisible:!0,rootElement:"body"},Yi={className:"string",clickCallback:"(function|null)",isAnimated:"boolean",isVisible:"boolean",rootElement:"(element|string)"};class Ui extends H{constructor(t){super(),this._config=this._getConfig(t),this._isAppended=!1,this._element=null}static get Default(){return Xi}static get DefaultType(){return Yi}static get NAME(){return Vi}show(t){if(!this._config.isVisible)return void g(t);this._append();const e=this._getElement();this._config.isAnimated&&d(e),e.classList.add(Ki),this._emulateAnimation((()=>{g(t)}))}hide(t){this._config.isVisible?(this._getElement().classList.remove(Ki),this._emulateAnimation((()=>{this.dispose(),g(t)}))):g(t)}dispose(){this._isAppended&&(N.off(this._element,Qi),this._element.remove(),this._isAppended=!1)}_getElement(){if(!this._element){const t=document.createElement("div");t.className=this._config.className,this._config.isAnimated&&t.classList.add("fade"),this._element=t}return this._element}_configAfterMerge(t){return t.rootElement=r(t.rootElement),t}_append(){if(this._isAppended)return;const t=this._getElement();this._config.rootElement.append(t),N.on(t,Qi,(()=>{g(this._config.clickCallback)})),this._isAppended=!0}_emulateAnimation(t){_(t,this._getElement(),this._config.isAnimated)}}const Gi=".bs.focustrap",Ji=`focusin${Gi}`,Zi=`keydown.tab${Gi}`,tn="backward",en={autofocus:!0,trapElement:null},nn={autofocus:"boolean",trapElement:"element"};class sn extends H{constructor(t){super(),this._config=this._getConfig(t),this._isActive=!1,this._lastTabNavDirection=null}static get Default(){return en}static get DefaultType(){return nn}static get NAME(){return"focustrap"}activate(){this._isActive||(this._config.autofocus&&this._config.trapElement.focus(),N.off(document,Gi),N.on(document,Ji,(t=>this._handleFocusin(t))),N.on(document,Zi,(t=>this._handleKeydown(t))),this._isActive=!0)}deactivate(){this._isActive&&(this._isActive=!1,N.off(document,Gi))}_handleFocusin(t){const{trapElement:e}=this._config;if(t.target===document||t.target===e||e.contains(t.target))return;const i=z.focusableChildren(e);0===i.length?e.focus():this._lastTabNavDirection===tn?i[i.length-1].focus():i[0].focus()}_handleKeydown(t){"Tab"===t.key&&(this._lastTabNavDirection=t.shiftKey?tn:"forward")}}const on=".fixed-top, .fixed-bottom, .is-fixed, .sticky-top",rn=".sticky-top",an="padding-right",ln="margin-right";class cn{constructor(){this._element=document.body}getWidth(){const t=document.documentElement.clientWidth;return Math.abs(window.innerWidth-t)}hide(){const t=this.getWidth();this._disableOverFlow(),this._setElementAttributes(this._element,an,(e=>e+t)),this._setElementAttributes(on,an,(e=>e+t)),this._setElementAttributes(rn,ln,(e=>e-t))}reset(){this._resetElementAttributes(this._element,"overflow"),this._resetElementAttributes(this._element,an),this._resetElementAttributes(on,an),this._resetElementAttributes(rn,ln)}isOverflowing(){return this.getWidth()>0}_disableOverFlow(){this._saveInitialAttribute(this._element,"overflow"),this._element.style.overflow="hidden"}_setElementAttributes(t,e,i){const n=this.getWidth();this._applyManipulationCallback(t,(t=>{if(t!==this._element&&window.innerWidth>t.clientWidth+n)return;this._saveInitialAttribute(t,e);const s=window.getComputedStyle(t).getPropertyValue(e);t.style.setProperty(e,`${i(Number.parseFloat(s))}px`)}))}_saveInitialAttribute(t,e){const i=t.style.getPropertyValue(e);i&&F.setDataAttribute(t,e,i)}_resetElementAttributes(t,e){this._applyManipulationCallback(t,(t=>{const i=F.getDataAttribute(t,e);null!==i?(F.removeDataAttribute(t,e),t.style.setProperty(e,i)):t.style.removeProperty(e)}))}_applyManipulationCallback(t,e){if(o(t))e(t);else for(const i of z.find(t,this._element))e(i)}}const hn=".bs.modal",dn=`hide${hn}`,un=`hidePrevented${hn}`,fn=`hidden${hn}`,pn=`show${hn}`,mn=`shown${hn}`,gn=`resize${hn}`,_n=`click.dismiss${hn}`,bn=`mousedown.dismiss${hn}`,vn=`keydown.dismiss${hn}`,yn=`click${hn}.data-api`,wn="modal-open",An="show",En="modal-static",Tn={backdrop:!0,focus:!0,keyboard:!0},Cn={backdrop:"(boolean|string)",focus:"boolean",keyboard:"boolean"};class On extends W{constructor(t,e){super(t,e),this._dialog=z.findOne(".modal-dialog",this._element),this._backdrop=this._initializeBackDrop(),this._focustrap=this._initializeFocusTrap(),this._isShown=!1,this._isTransitioning=!1,this._scrollBar=new cn,this._addEventListeners()}static get Default(){return Tn}static get DefaultType(){return Cn}static get NAME(){return"modal"}toggle(t){return this._isShown?this.hide():this.show(t)}show(t){this._isShown||this._isTransitioning||N.trigger(this._element,pn,{relatedTarget:t}).defaultPrevented||(this._isShown=!0,this._isTransitioning=!0,this._scrollBar.hide(),document.body.classList.add(wn),this._adjustDialog(),this._backdrop.show((()=>this._showElement(t))))}hide(){this._isShown&&!this._isTransitioning&&(N.trigger(this._element,dn).defaultPrevented||(this._isShown=!1,this._isTransitioning=!0,this._focustrap.deactivate(),this._element.classList.remove(An),this._queueCallback((()=>this._hideModal()),this._element,this._isAnimated())))}dispose(){N.off(window,hn),N.off(this._dialog,hn),this._backdrop.dispose(),this._focustrap.deactivate(),super.dispose()}handleUpdate(){this._adjustDialog()}_initializeBackDrop(){return new Ui({isVisible:Boolean(this._config.backdrop),isAnimated:this._isAnimated()})}_initializeFocusTrap(){return new sn({trapElement:this._element})}_showElement(t){document.body.contains(this._element)||document.body.append(this._element),this._element.style.display="block",this._element.removeAttribute("aria-hidden"),this._element.setAttribute("aria-modal",!0),this._element.setAttribute("role","dialog"),this._element.scrollTop=0;const e=z.findOne(".modal-body",this._dialog);e&&(e.scrollTop=0),d(this._element),this._element.classList.add(An),this._queueCallback((()=>{this._config.focus&&this._focustrap.activate(),this._isTransitioning=!1,N.trigger(this._element,mn,{relatedTarget:t})}),this._dialog,this._isAnimated())}_addEventListeners(){N.on(this._element,vn,(t=>{"Escape"===t.key&&(this._config.keyboard?this.hide():this._triggerBackdropTransition())})),N.on(window,gn,(()=>{this._isShown&&!this._isTransitioning&&this._adjustDialog()})),N.on(this._element,bn,(t=>{N.one(this._element,_n,(e=>{this._element===t.target&&this._element===e.target&&("static"!==this._config.backdrop?this._config.backdrop&&this.hide():this._triggerBackdropTransition())}))}))}_hideModal(){this._element.style.display="none",this._element.setAttribute("aria-hidden",!0),this._element.removeAttribute("aria-modal"),this._element.removeAttribute("role"),this._isTransitioning=!1,this._backdrop.hide((()=>{document.body.classList.remove(wn),this._resetAdjustments(),this._scrollBar.reset(),N.trigger(this._element,fn)}))}_isAnimated(){return this._element.classList.contains("fade")}_triggerBackdropTransition(){if(N.trigger(this._element,un).defaultPrevented)return;const t=this._element.scrollHeight>document.documentElement.clientHeight,e=this._element.style.overflowY;"hidden"===e||this._element.classList.contains(En)||(t||(this._element.style.overflowY="hidden"),this._element.classList.add(En),this._queueCallback((()=>{this._element.classList.remove(En),this._queueCallback((()=>{this._element.style.overflowY=e}),this._dialog)}),this._dialog),this._element.focus())}_adjustDialog(){const t=this._element.scrollHeight>document.documentElement.clientHeight,e=this._scrollBar.getWidth(),i=e>0;if(i&&!t){const t=p()?"paddingLeft":"paddingRight";this._element.style[t]=`${e}px`}if(!i&&t){const t=p()?"paddingRight":"paddingLeft";this._element.style[t]=`${e}px`}}_resetAdjustments(){this._element.style.paddingLeft="",this._element.style.paddingRight=""}static jQueryInterface(t,e){return this.each((function(){const i=On.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===i[t])throw new TypeError(`No method named "${t}"`);i[t](e)}}))}}N.on(document,yn,'[data-bs-toggle="modal"]',(function(t){const e=z.getElementFromSelector(this);["A","AREA"].includes(this.tagName)&&t.preventDefault(),N.one(e,pn,(t=>{t.defaultPrevented||N.one(e,fn,(()=>{a(this)&&this.focus()}))}));const i=z.findOne(".modal.show");i&&On.getInstance(i).hide(),On.getOrCreateInstance(e).toggle(this)})),R(On),m(On);const xn=".bs.offcanvas",kn=".data-api",Ln=`load${xn}${kn}`,Sn="show",Dn="showing",$n="hiding",In=".offcanvas.show",Nn=`show${xn}`,Pn=`shown${xn}`,Mn=`hide${xn}`,jn=`hidePrevented${xn}`,Fn=`hidden${xn}`,Hn=`resize${xn}`,Wn=`click${xn}${kn}`,Bn=`keydown.dismiss${xn}`,zn={backdrop:!0,keyboard:!0,scroll:!1},Rn={backdrop:"(boolean|string)",keyboard:"boolean",scroll:"boolean"};class qn extends W{constructor(t,e){super(t,e),this._isShown=!1,this._backdrop=this._initializeBackDrop(),this._focustrap=this._initializeFocusTrap(),this._addEventListeners()}static get Default(){return zn}static get DefaultType(){return Rn}static get NAME(){return"offcanvas"}toggle(t){return this._isShown?this.hide():this.show(t)}show(t){this._isShown||N.trigger(this._element,Nn,{relatedTarget:t}).defaultPrevented||(this._isShown=!0,this._backdrop.show(),this._config.scroll||(new cn).hide(),this._element.setAttribute("aria-modal",!0),this._element.setAttribute("role","dialog"),this._element.classList.add(Dn),this._queueCallback((()=>{this._config.scroll&&!this._config.backdrop||this._focustrap.activate(),this._element.classList.add(Sn),this._element.classList.remove(Dn),N.trigger(this._element,Pn,{relatedTarget:t})}),this._element,!0))}hide(){this._isShown&&(N.trigger(this._element,Mn).defaultPrevented||(this._focustrap.deactivate(),this._element.blur(),this._isShown=!1,this._element.classList.add($n),this._backdrop.hide(),this._queueCallback((()=>{this._element.classList.remove(Sn,$n),this._element.removeAttribute("aria-modal"),this._element.removeAttribute("role"),this._config.scroll||(new cn).reset(),N.trigger(this._element,Fn)}),this._element,!0)))}dispose(){this._backdrop.dispose(),this._focustrap.deactivate(),super.dispose()}_initializeBackDrop(){const t=Boolean(this._config.backdrop);return new Ui({className:"offcanvas-backdrop",isVisible:t,isAnimated:!0,rootElement:this._element.parentNode,clickCallback:t?()=>{"static"!==this._config.backdrop?this.hide():N.trigger(this._element,jn)}:null})}_initializeFocusTrap(){return new sn({trapElement:this._element})}_addEventListeners(){N.on(this._element,Bn,(t=>{"Escape"===t.key&&(this._config.keyboard?this.hide():N.trigger(this._element,jn))}))}static jQueryInterface(t){return this.each((function(){const e=qn.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t](this)}}))}}N.on(document,Wn,'[data-bs-toggle="offcanvas"]',(function(t){const e=z.getElementFromSelector(this);if(["A","AREA"].includes(this.tagName)&&t.preventDefault(),l(this))return;N.one(e,Fn,(()=>{a(this)&&this.focus()}));const i=z.findOne(In);i&&i!==e&&qn.getInstance(i).hide(),qn.getOrCreateInstance(e).toggle(this)})),N.on(window,Ln,(()=>{for(const t of z.find(In))qn.getOrCreateInstance(t).show()})),N.on(window,Hn,(()=>{for(const t of z.find("[aria-modal][class*=show][class*=offcanvas-]"))"fixed"!==getComputedStyle(t).position&&qn.getOrCreateInstance(t).hide()})),R(qn),m(qn);const Vn={"*":["class","dir","id","lang","role",/^aria-[\w-]*$/i],a:["target","href","title","rel"],area:[],b:[],br:[],col:[],code:[],div:[],em:[],hr:[],h1:[],h2:[],h3:[],h4:[],h5:[],h6:[],i:[],img:["src","srcset","alt","title","width","height"],li:[],ol:[],p:[],pre:[],s:[],small:[],span:[],sub:[],sup:[],strong:[],u:[],ul:[]},Kn=new Set(["background","cite","href","itemtype","longdesc","poster","src","xlink:href"]),Qn=/^(?!javascript:)(?:[a-z0-9+.-]+:|[^&:/?#]*(?:[/?#]|$))/i,Xn=(t,e)=>{const i=t.nodeName.toLowerCase();return e.includes(i)?!Kn.has(i)||Boolean(Qn.test(t.nodeValue)):e.filter((t=>t instanceof RegExp)).some((t=>t.test(i)))},Yn={allowList:Vn,content:{},extraClass:"",html:!1,sanitize:!0,sanitizeFn:null,template:"<div></div>"},Un={allowList:"object",content:"object",extraClass:"(string|function)",html:"boolean",sanitize:"boolean",sanitizeFn:"(null|function)",template:"string"},Gn={entry:"(string|element|function|null)",selector:"(string|element)"};class Jn extends H{constructor(t){super(),this._config=this._getConfig(t)}static get Default(){return Yn}static get DefaultType(){return Un}static get NAME(){return"TemplateFactory"}getContent(){return Object.values(this._config.content).map((t=>this._resolvePossibleFunction(t))).filter(Boolean)}hasContent(){return this.getContent().length>0}changeContent(t){return this._checkContent(t),this._config.content={...this._config.content,...t},this}toHtml(){const t=document.createElement("div");t.innerHTML=this._maybeSanitize(this._config.template);for(const[e,i]of Object.entries(this._config.content))this._setContent(t,i,e);const e=t.children[0],i=this._resolvePossibleFunction(this._config.extraClass);return i&&e.classList.add(...i.split(" ")),e}_typeCheckConfig(t){super._typeCheckConfig(t),this._checkContent(t.content)}_checkContent(t){for(const[e,i]of Object.entries(t))super._typeCheckConfig({selector:e,entry:i},Gn)}_setContent(t,e,i){const n=z.findOne(i,t);n&&((e=this._resolvePossibleFunction(e))?o(e)?this._putElementInTemplate(r(e),n):this._config.html?n.innerHTML=this._maybeSanitize(e):n.textContent=e:n.remove())}_maybeSanitize(t){return this._config.sanitize?function(t,e,i){if(!t.length)return t;if(i&&"function"==typeof i)return i(t);const n=(new window.DOMParser).parseFromString(t,"text/html"),s=[].concat(...n.body.querySelectorAll("*"));for(const t of s){const i=t.nodeName.toLowerCase();if(!Object.keys(e).includes(i)){t.remove();continue}const n=[].concat(...t.attributes),s=[].concat(e["*"]||[],e[i]||[]);for(const e of n)Xn(e,s)||t.removeAttribute(e.nodeName)}return n.body.innerHTML}(t,this._config.allowList,this._config.sanitizeFn):t}_resolvePossibleFunction(t){return g(t,[this])}_putElementInTemplate(t,e){if(this._config.html)return e.innerHTML="",void e.append(t);e.textContent=t.textContent}}const Zn=new Set(["sanitize","allowList","sanitizeFn"]),ts="fade",es="show",is=".modal",ns="hide.bs.modal",ss="hover",os="focus",rs={AUTO:"auto",TOP:"top",RIGHT:p()?"left":"right",BOTTOM:"bottom",LEFT:p()?"right":"left"},as={allowList:Vn,animation:!0,boundary:"clippingParents",container:!1,customClass:"",delay:0,fallbackPlacements:["top","right","bottom","left"],html:!1,offset:[0,6],placement:"top",popperConfig:null,sanitize:!0,sanitizeFn:null,selector:!1,template:'<div class="tooltip" role="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>',title:"",trigger:"hover focus"},ls={allowList:"object",animation:"boolean",boundary:"(string|element)",container:"(string|element|boolean)",customClass:"(string|function)",delay:"(number|object)",fallbackPlacements:"array",html:"boolean",offset:"(array|string|function)",placement:"(string|function)",popperConfig:"(null|object|function)",sanitize:"boolean",sanitizeFn:"(null|function)",selector:"(string|boolean)",template:"string",title:"(string|element|function)",trigger:"string"};class cs extends W{constructor(t,e){if(void 0===vi)throw new TypeError("Bootstrap's tooltips require Popper (https://popper.js.org)");super(t,e),this._isEnabled=!0,this._timeout=0,this._isHovered=null,this._activeTrigger={},this._popper=null,this._templateFactory=null,this._newContent=null,this.tip=null,this._setListeners(),this._config.selector||this._fixTitle()}static get Default(){return as}static get DefaultType(){return ls}static get NAME(){return"tooltip"}enable(){this._isEnabled=!0}disable(){this._isEnabled=!1}toggleEnabled(){this._isEnabled=!this._isEnabled}toggle(){this._isEnabled&&(this._activeTrigger.click=!this._activeTrigger.click,this._isShown()?this._leave():this._enter())}dispose(){clearTimeout(this._timeout),N.off(this._element.closest(is),ns,this._hideModalHandler),this._element.getAttribute("data-bs-original-title")&&this._element.setAttribute("title",this._element.getAttribute("data-bs-original-title")),this._disposePopper(),super.dispose()}show(){if("none"===this._element.style.display)throw new Error("Please use show on visible elements");if(!this._isWithContent()||!this._isEnabled)return;const t=N.trigger(this._element,this.constructor.eventName("show")),e=(c(this._element)||this._element.ownerDocument.documentElement).contains(this._element);if(t.defaultPrevented||!e)return;this._disposePopper();const i=this._getTipElement();this._element.setAttribute("aria-describedby",i.getAttribute("id"));const{container:n}=this._config;if(this._element.ownerDocument.documentElement.contains(this.tip)||(n.append(i),N.trigger(this._element,this.constructor.eventName("inserted"))),this._popper=this._createPopper(i),i.classList.add(es),"ontouchstart"in document.documentElement)for(const t of[].concat(...document.body.children))N.on(t,"mouseover",h);this._queueCallback((()=>{N.trigger(this._element,this.constructor.eventName("shown")),!1===this._isHovered&&this._leave(),this._isHovered=!1}),this.tip,this._isAnimated())}hide(){if(this._isShown()&&!N.trigger(this._element,this.constructor.eventName("hide")).defaultPrevented){if(this._getTipElement().classList.remove(es),"ontouchstart"in document.documentElement)for(const t of[].concat(...document.body.children))N.off(t,"mouseover",h);this._activeTrigger.click=!1,this._activeTrigger[os]=!1,this._activeTrigger[ss]=!1,this._isHovered=null,this._queueCallback((()=>{this._isWithActiveTrigger()||(this._isHovered||this._disposePopper(),this._element.removeAttribute("aria-describedby"),N.trigger(this._element,this.constructor.eventName("hidden")))}),this.tip,this._isAnimated())}}update(){this._popper&&this._popper.update()}_isWithContent(){return Boolean(this._getTitle())}_getTipElement(){return this.tip||(this.tip=this._createTipElement(this._newContent||this._getContentForTemplate())),this.tip}_createTipElement(t){const e=this._getTemplateFactory(t).toHtml();if(!e)return null;e.classList.remove(ts,es),e.classList.add(`bs-${this.constructor.NAME}-auto`);const i=(t=>{do{t+=Math.floor(1e6*Math.random())}while(document.getElementById(t));return t})(this.constructor.NAME).toString();return e.setAttribute("id",i),this._isAnimated()&&e.classList.add(ts),e}setContent(t){this._newContent=t,this._isShown()&&(this._disposePopper(),this.show())}_getTemplateFactory(t){return this._templateFactory?this._templateFactory.changeContent(t):this._templateFactory=new Jn({...this._config,content:t,extraClass:this._resolvePossibleFunction(this._config.customClass)}),this._templateFactory}_getContentForTemplate(){return{".tooltip-inner":this._getTitle()}}_getTitle(){return this._resolvePossibleFunction(this._config.title)||this._element.getAttribute("data-bs-original-title")}_initializeOnDelegatedTarget(t){return this.constructor.getOrCreateInstance(t.delegateTarget,this._getDelegateConfig())}_isAnimated(){return this._config.animation||this.tip&&this.tip.classList.contains(ts)}_isShown(){return this.tip&&this.tip.classList.contains(es)}_createPopper(t){const e=g(this._config.placement,[this,t,this._element]),i=rs[e.toUpperCase()];return bi(this._element,t,this._getPopperConfig(i))}_getOffset(){const{offset:t}=this._config;return"string"==typeof t?t.split(",").map((t=>Number.parseInt(t,10))):"function"==typeof t?e=>t(e,this._element):t}_resolvePossibleFunction(t){return g(t,[this._element])}_getPopperConfig(t){const e={placement:t,modifiers:[{name:"flip",options:{fallbackPlacements:this._config.fallbackPlacements}},{name:"offset",options:{offset:this._getOffset()}},{name:"preventOverflow",options:{boundary:this._config.boundary}},{name:"arrow",options:{element:`.${this.constructor.NAME}-arrow`}},{name:"preSetPlacement",enabled:!0,phase:"beforeMain",fn:t=>{this._getTipElement().setAttribute("data-popper-placement",t.state.placement)}}]};return{...e,...g(this._config.popperConfig,[e])}}_setListeners(){const t=this._config.trigger.split(" ");for(const e of t)if("click"===e)N.on(this._element,this.constructor.eventName("click"),this._config.selector,(t=>{this._initializeOnDelegatedTarget(t).toggle()}));else if("manual"!==e){const t=e===ss?this.constructor.eventName("mouseenter"):this.constructor.eventName("focusin"),i=e===ss?this.constructor.eventName("mouseleave"):this.constructor.eventName("focusout");N.on(this._element,t,this._config.selector,(t=>{const e=this._initializeOnDelegatedTarget(t);e._activeTrigger["focusin"===t.type?os:ss]=!0,e._enter()})),N.on(this._element,i,this._config.selector,(t=>{const e=this._initializeOnDelegatedTarget(t);e._activeTrigger["focusout"===t.type?os:ss]=e._element.contains(t.relatedTarget),e._leave()}))}this._hideModalHandler=()=>{this._element&&this.hide()},N.on(this._element.closest(is),ns,this._hideModalHandler)}_fixTitle(){const t=this._element.getAttribute("title");t&&(this._element.getAttribute("aria-label")||this._element.textContent.trim()||this._element.setAttribute("aria-label",t),this._element.setAttribute("data-bs-original-title",t),this._element.removeAttribute("title"))}_enter(){this._isShown()||this._isHovered?this._isHovered=!0:(this._isHovered=!0,this._setTimeout((()=>{this._isHovered&&this.show()}),this._config.delay.show))}_leave(){this._isWithActiveTrigger()||(this._isHovered=!1,this._setTimeout((()=>{this._isHovered||this.hide()}),this._config.delay.hide))}_setTimeout(t,e){clearTimeout(this._timeout),this._timeout=setTimeout(t,e)}_isWithActiveTrigger(){return Object.values(this._activeTrigger).includes(!0)}_getConfig(t){const e=F.getDataAttributes(this._element);for(const t of Object.keys(e))Zn.has(t)&&delete e[t];return t={...e,..."object"==typeof t&&t?t:{}},t=this._mergeConfigObj(t),t=this._configAfterMerge(t),this._typeCheckConfig(t),t}_configAfterMerge(t){return t.container=!1===t.container?document.body:r(t.container),"number"==typeof t.delay&&(t.delay={show:t.delay,hide:t.delay}),"number"==typeof t.title&&(t.title=t.title.toString()),"number"==typeof t.content&&(t.content=t.content.toString()),t}_getDelegateConfig(){const t={};for(const[e,i]of Object.entries(this._config))this.constructor.Default[e]!==i&&(t[e]=i);return t.selector=!1,t.trigger="manual",t}_disposePopper(){this._popper&&(this._popper.destroy(),this._popper=null),this.tip&&(this.tip.remove(),this.tip=null)}static jQueryInterface(t){return this.each((function(){const e=cs.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t]()}}))}}m(cs);const hs={...cs.Default,content:"",offset:[0,8],placement:"right",template:'<div class="popover" role="tooltip"><div class="popover-arrow"></div><h3 class="popover-header"></h3><div class="popover-body"></div></div>',trigger:"click"},ds={...cs.DefaultType,content:"(null|string|element|function)"};class us extends cs{static get Default(){return hs}static get DefaultType(){return ds}static get NAME(){return"popover"}_isWithContent(){return this._getTitle()||this._getContent()}_getContentForTemplate(){return{".popover-header":this._getTitle(),".popover-body":this._getContent()}}_getContent(){return this._resolvePossibleFunction(this._config.content)}static jQueryInterface(t){return this.each((function(){const e=us.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t]()}}))}}m(us);const fs=".bs.scrollspy",ps=`activate${fs}`,ms=`click${fs}`,gs=`load${fs}.data-api`,_s="active",bs="[href]",vs=".nav-link",ys=`${vs}, .nav-item > ${vs}, .list-group-item`,ws={offset:null,rootMargin:"0px 0px -25%",smoothScroll:!1,target:null,threshold:[.1,.5,1]},As={offset:"(number|null)",rootMargin:"string",smoothScroll:"boolean",target:"element",threshold:"array"};class Es extends W{constructor(t,e){super(t,e),this._targetLinks=new Map,this._observableSections=new Map,this._rootElement="visible"===getComputedStyle(this._element).overflowY?null:this._element,this._activeTarget=null,this._observer=null,this._previousScrollData={visibleEntryTop:0,parentScrollTop:0},this.refresh()}static get Default(){return ws}static get DefaultType(){return As}static get NAME(){return"scrollspy"}refresh(){this._initializeTargetsAndObservables(),this._maybeEnableSmoothScroll(),this._observer?this._observer.disconnect():this._observer=this._getNewObserver();for(const t of this._observableSections.values())this._observer.observe(t)}dispose(){this._observer.disconnect(),super.dispose()}_configAfterMerge(t){return t.target=r(t.target)||document.body,t.rootMargin=t.offset?`${t.offset}px 0px -30%`:t.rootMargin,"string"==typeof t.threshold&&(t.threshold=t.threshold.split(",").map((t=>Number.parseFloat(t)))),t}_maybeEnableSmoothScroll(){this._config.smoothScroll&&(N.off(this._config.target,ms),N.on(this._config.target,ms,bs,(t=>{const e=this._observableSections.get(t.target.hash);if(e){t.preventDefault();const i=this._rootElement||window,n=e.offsetTop-this._element.offsetTop;if(i.scrollTo)return void i.scrollTo({top:n,behavior:"smooth"});i.scrollTop=n}})))}_getNewObserver(){const t={root:this._rootElement,threshold:this._config.threshold,rootMargin:this._config.rootMargin};return new IntersectionObserver((t=>this._observerCallback(t)),t)}_observerCallback(t){const e=t=>this._targetLinks.get(`#${t.target.id}`),i=t=>{this._previousScrollData.visibleEntryTop=t.target.offsetTop,this._process(e(t))},n=(this._rootElement||document.documentElement).scrollTop,s=n>=this._previousScrollData.parentScrollTop;this._previousScrollData.parentScrollTop=n;for(const o of t){if(!o.isIntersecting){this._activeTarget=null,this._clearActiveClass(e(o));continue}const t=o.target.offsetTop>=this._previousScrollData.visibleEntryTop;if(s&&t){if(i(o),!n)return}else s||t||i(o)}}_initializeTargetsAndObservables(){this._targetLinks=new Map,this._observableSections=new Map;const t=z.find(bs,this._config.target);for(const e of t){if(!e.hash||l(e))continue;const t=z.findOne(decodeURI(e.hash),this._element);a(t)&&(this._targetLinks.set(decodeURI(e.hash),e),this._observableSections.set(e.hash,t))}}_process(t){this._activeTarget!==t&&(this._clearActiveClass(this._config.target),this._activeTarget=t,t.classList.add(_s),this._activateParents(t),N.trigger(this._element,ps,{relatedTarget:t}))}_activateParents(t){if(t.classList.contains("dropdown-item"))z.findOne(".dropdown-toggle",t.closest(".dropdown")).classList.add(_s);else for(const e of z.parents(t,".nav, .list-group"))for(const t of z.prev(e,ys))t.classList.add(_s)}_clearActiveClass(t){t.classList.remove(_s);const e=z.find(`${bs}.${_s}`,t);for(const t of e)t.classList.remove(_s)}static jQueryInterface(t){return this.each((function(){const e=Es.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t]()}}))}}N.on(window,gs,(()=>{for(const t of z.find('[data-bs-spy="scroll"]'))Es.getOrCreateInstance(t)})),m(Es);const Ts=".bs.tab",Cs=`hide${Ts}`,Os=`hidden${Ts}`,xs=`show${Ts}`,ks=`shown${Ts}`,Ls=`click${Ts}`,Ss=`keydown${Ts}`,Ds=`load${Ts}`,$s="ArrowLeft",Is="ArrowRight",Ns="ArrowUp",Ps="ArrowDown",Ms="Home",js="End",Fs="active",Hs="fade",Ws="show",Bs=".dropdown-toggle",zs=`:not(${Bs})`,Rs='[data-bs-toggle="tab"], [data-bs-toggle="pill"], [data-bs-toggle="list"]',qs=`.nav-link${zs}, .list-group-item${zs}, [role="tab"]${zs}, ${Rs}`,Vs=`.${Fs}[data-bs-toggle="tab"], .${Fs}[data-bs-toggle="pill"], .${Fs}[data-bs-toggle="list"]`;class Ks extends W{constructor(t){super(t),this._parent=this._element.closest('.list-group, .nav, [role="tablist"]'),this._parent&&(this._setInitialAttributes(this._parent,this._getChildren()),N.on(this._element,Ss,(t=>this._keydown(t))))}static get NAME(){return"tab"}show(){const t=this._element;if(this._elemIsActive(t))return;const e=this._getActiveElem(),i=e?N.trigger(e,Cs,{relatedTarget:t}):null;N.trigger(t,xs,{relatedTarget:e}).defaultPrevented||i&&i.defaultPrevented||(this._deactivate(e,t),this._activate(t,e))}_activate(t,e){t&&(t.classList.add(Fs),this._activate(z.getElementFromSelector(t)),this._queueCallback((()=>{"tab"===t.getAttribute("role")?(t.removeAttribute("tabindex"),t.setAttribute("aria-selected",!0),this._toggleDropDown(t,!0),N.trigger(t,ks,{relatedTarget:e})):t.classList.add(Ws)}),t,t.classList.contains(Hs)))}_deactivate(t,e){t&&(t.classList.remove(Fs),t.blur(),this._deactivate(z.getElementFromSelector(t)),this._queueCallback((()=>{"tab"===t.getAttribute("role")?(t.setAttribute("aria-selected",!1),t.setAttribute("tabindex","-1"),this._toggleDropDown(t,!1),N.trigger(t,Os,{relatedTarget:e})):t.classList.remove(Ws)}),t,t.classList.contains(Hs)))}_keydown(t){if(![$s,Is,Ns,Ps,Ms,js].includes(t.key))return;t.stopPropagation(),t.preventDefault();const e=this._getChildren().filter((t=>!l(t)));let i;if([Ms,js].includes(t.key))i=e[t.key===Ms?0:e.length-1];else{const n=[Is,Ps].includes(t.key);i=b(e,t.target,n,!0)}i&&(i.focus({preventScroll:!0}),Ks.getOrCreateInstance(i).show())}_getChildren(){return z.find(qs,this._parent)}_getActiveElem(){return this._getChildren().find((t=>this._elemIsActive(t)))||null}_setInitialAttributes(t,e){this._setAttributeIfNotExists(t,"role","tablist");for(const t of e)this._setInitialAttributesOnChild(t)}_setInitialAttributesOnChild(t){t=this._getInnerElement(t);const e=this._elemIsActive(t),i=this._getOuterElement(t);t.setAttribute("aria-selected",e),i!==t&&this._setAttributeIfNotExists(i,"role","presentation"),e||t.setAttribute("tabindex","-1"),this._setAttributeIfNotExists(t,"role","tab"),this._setInitialAttributesOnTargetPanel(t)}_setInitialAttributesOnTargetPanel(t){const e=z.getElementFromSelector(t);e&&(this._setAttributeIfNotExists(e,"role","tabpanel"),t.id&&this._setAttributeIfNotExists(e,"aria-labelledby",`${t.id}`))}_toggleDropDown(t,e){const i=this._getOuterElement(t);if(!i.classList.contains("dropdown"))return;const n=(t,n)=>{const s=z.findOne(t,i);s&&s.classList.toggle(n,e)};n(Bs,Fs),n(".dropdown-menu",Ws),i.setAttribute("aria-expanded",e)}_setAttributeIfNotExists(t,e,i){t.hasAttribute(e)||t.setAttribute(e,i)}_elemIsActive(t){return t.classList.contains(Fs)}_getInnerElement(t){return t.matches(qs)?t:z.findOne(qs,t)}_getOuterElement(t){return t.closest(".nav-item, .list-group-item")||t}static jQueryInterface(t){return this.each((function(){const e=Ks.getOrCreateInstance(this);if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t]()}}))}}N.on(document,Ls,Rs,(function(t){["A","AREA"].includes(this.tagName)&&t.preventDefault(),l(this)||Ks.getOrCreateInstance(this).show()})),N.on(window,Ds,(()=>{for(const t of z.find(Vs))Ks.getOrCreateInstance(t)})),m(Ks);const Qs=".bs.toast",Xs=`mouseover${Qs}`,Ys=`mouseout${Qs}`,Us=`focusin${Qs}`,Gs=`focusout${Qs}`,Js=`hide${Qs}`,Zs=`hidden${Qs}`,to=`show${Qs}`,eo=`shown${Qs}`,io="hide",no="show",so="showing",oo={animation:"boolean",autohide:"boolean",delay:"number"},ro={animation:!0,autohide:!0,delay:5e3};class ao extends W{constructor(t,e){super(t,e),this._timeout=null,this._hasMouseInteraction=!1,this._hasKeyboardInteraction=!1,this._setListeners()}static get Default(){return ro}static get DefaultType(){return oo}static get NAME(){return"toast"}show(){N.trigger(this._element,to).defaultPrevented||(this._clearTimeout(),this._config.animation&&this._element.classList.add("fade"),this._element.classList.remove(io),d(this._element),this._element.classList.add(no,so),this._queueCallback((()=>{this._element.classList.remove(so),N.trigger(this._element,eo),this._maybeScheduleHide()}),this._element,this._config.animation))}hide(){this.isShown()&&(N.trigger(this._element,Js).defaultPrevented||(this._element.classList.add(so),this._queueCallback((()=>{this._element.classList.add(io),this._element.classList.remove(so,no),N.trigger(this._element,Zs)}),this._element,this._config.animation)))}dispose(){this._clearTimeout(),this.isShown()&&this._element.classList.remove(no),super.dispose()}isShown(){return this._element.classList.contains(no)}_maybeScheduleHide(){this._config.autohide&&(this._hasMouseInteraction||this._hasKeyboardInteraction||(this._timeout=setTimeout((()=>{this.hide()}),this._config.delay)))}_onInteraction(t,e){switch(t.type){case"mouseover":case"mouseout":this._hasMouseInteraction=e;break;case"focusin":case"focusout":this._hasKeyboardInteraction=e}if(e)return void this._clearTimeout();const i=t.relatedTarget;this._element===i||this._element.contains(i)||this._maybeScheduleHide()}_setListeners(){N.on(this._element,Xs,(t=>this._onInteraction(t,!0))),N.on(this._element,Ys,(t=>this._onInteraction(t,!1))),N.on(this._element,Us,(t=>this._onInteraction(t,!0))),N.on(this._element,Gs,(t=>this._onInteraction(t,!1)))}_clearTimeout(){clearTimeout(this._timeout),this._timeout=null}static jQueryInterface(t){return this.each((function(){const e=ao.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t](this)}}))}}return R(ao),m(ao),{Alert:Q,Button:Y,Carousel:xt,Collapse:Bt,Dropdown:qi,Modal:On,Offcanvas:qn,Popover:us,ScrollSpy:Es,Tab:Ks,Toast:ao,Tooltip:cs}}));
+//# sourceMappingURL=bootstrap.bundle.min.js.map \ No newline at end of file
diff --git a/interface/js/lib/codejar-linenumbers.min.js b/interface/js/lib/codejar-linenumbers.min.js
new file mode 100644
index 0000000..d4629b9
--- /dev/null
+++ b/interface/js/lib/codejar-linenumbers.min.js
@@ -0,0 +1,6 @@
+/*!
+ * codejar-linenumbers v1.0.1 (https://github.com/julianpoemp/codejar-linenumbers)
+ * Copyright (c) 2023, Julian Poemp, MIT
+ */
+function withLineNumbers(r,e={}){const d={class:"codejar-linenumbers",wrapClass:"codejar-wrap",width:"35px",backgroundColor:"rgba(128, 128, 128, 0.15)",color:"",...e};let l;return function(e){r(e),l||(l=init(e,d),e.addEventListener("scroll",()=>l.style.top=`-${e.scrollTop}px`));var t=(e.textContent||"").replace(/\n$/g,"").split("\n").length;let o="";for(let e=0;e<t;e++)o+=e+1+`
+`;l.innerText=o}}function init(e,t){var o=getComputedStyle(e),r=document.createElement("div"),d=(r.className=t.wrapClass,r.style.position="relative",document.createElement("div")),l=(d.className="codejar-linenumbers-inner-wrap",d.style.background=o.background,d.style.marginTop=o.borderTopWidth,d.style.marginBottom=o.borderBottomWidth,d.style.marginLeft=o.borderLeftWidth,d.style.borderTopLeftRadius=o.borderTopLeftRadius,d.style.borderBottomLeftRadius=o.borderBottomLeftRadius,document.createElement("div")),d=(l.className=t.class,d.appendChild(l),r.appendChild(d),l.style.width=t.width,l.style.overflow="hidden",l.style.backgroundColor=t.backgroundColor,l.style.fontFamily=o.fontFamily,l.style.fontSize=o.fontSize,l.style.lineHeight=o.lineHeight,l.style.paddingTop=`calc(${o.paddingTop})`,l.style.paddingLeft=o.paddingLeft,l.style.borderTopLeftRadius=o.borderTopLeftRadius,l.style.borderBottomLeftRadius=o.borderBottomLeftRadius,document.createElement("div"));return d.setAttribute("class","codejar-linenumber"),d.style.color=t.color||o.color,d.style.setProperty("mix-blend-mode","unset"),l.appendChild(d),e.style.paddingLeft=`calc(${t.width} + ${l.style.paddingLeft} + 5px)`,e.style.whiteSpace="pre",e.parentNode.insertBefore(r,e),r.appendChild(e),d} \ No newline at end of file
diff --git a/interface/js/lib/codejar.min.js b/interface/js/lib/codejar.min.js
new file mode 100644
index 0000000..6326ea3
--- /dev/null
+++ b/interface/js/lib/codejar.min.js
@@ -0,0 +1,5 @@
+/*!
+ * CodeJar 4.2.0 (https://github.com/antonmedv/codejar)
+ * Copyright (c) 2020, Anton Medvedev, MIT
+ */
+const globalWindow=window;function CodeJar(l,n,e={}){const c={tab:"\t",indentOn:/[({\[]$/,moveToNewLine:/^[)}\]]/,spellcheck:!1,catchTab:!0,preserveIdent:!0,addClosing:!0,history:!0,window:globalWindow,...e},o=c.window,f=o.document,r=[],u=[];let p=-1,a=!1,i=()=>{},h;l.setAttribute("contenteditable","plaintext-only"),l.setAttribute("spellcheck",c.spellcheck?"true":"false"),l.style.outline="none",l.style.overflowWrap="break-word",l.style.overflowY="auto",l.style.whiteSpace="pre-wrap";const d=(e,t)=>{n(e,t)};let g=!1;(g="plaintext-only"!==l.contentEditable?!0:g)&&l.setAttribute("contenteditable","true");const t=S(()=>{var e=v();d(l,e),T(e)},30);let y=!1;const N=e=>!O(e)&&!M(e)&&"Meta"!==e.key&&"Control"!==e.key&&"Alt"!==e.key&&!e.key.startsWith("Arrow"),s=S(e=>{N(e)&&(x(),y=!1)},300);e=(e,t)=>{r.push([e,t]),l.addEventListener(e,t)};function v(){var e=H();const t={start:0,end:0,dir:void 0};let{anchorNode:n,anchorOffset:r,focusNode:o,focusOffset:a}=e;if(n&&o)return n===l&&o===l?(t.start=0<r&&l.textContent?l.textContent.length:0,t.end=0<a&&l.textContent?l.textContent.length:0,t.dir=a>=r?"->":"<-"):(n.nodeType===Node.ELEMENT_NODE&&(e=f.createTextNode(""),n.insertBefore(e,n.childNodes[r]),n=e,r=0),o.nodeType===Node.ELEMENT_NODE&&(e=f.createTextNode(""),o.insertBefore(e,o.childNodes[a]),o=e,a=0),w(l,e=>{if(e===n&&e===o)return t.start+=r,t.end+=a,t.dir=r<=a?"->":"<-","stop";if(e===n){if(t.start+=r,t.dir)return"stop";t.dir="->"}else if(e===o){if(t.end+=a,t.dir)return"stop";t.dir="<-"}e.nodeType===Node.TEXT_NODE&&("->"!=t.dir&&(t.start+=e.nodeValue.length),"<-"!=t.dir)&&(t.end+=e.nodeValue.length)}),l.normalize()),t;throw"error1"}function T(n){var e=H();let r,o=0,a,i=0,d=(n.dir||(n.dir="->"),n.start<0&&(n.start=0),n.end<0&&(n.end=0),"<-"==n.dir&&({start:t,end:s}=n,n.start=s,n.end=t),0);w(l,e=>{var t;if(e.nodeType===Node.TEXT_NODE)return t=(e.nodeValue||"").length,d+t>n.start&&(r||(r=e,o=n.start-d),d+t>n.end)?(a=e,i=n.end-d,"stop"):void(d+=t)}),r||(r=l,o=l.childNodes.length),a||(a=l,i=l.childNodes.length),"<-"==n.dir&&([r,o,a,i]=[a,i,r,o]);var t,s=C(r),s=(s&&(t=f.createTextNode(""),s.parentNode?.insertBefore(t,s),r=t,o=0),C(a));s&&(t=f.createTextNode(""),s.parentNode?.insertBefore(t,s),a=t,i=0),e.setBaseAndExtent(r,o,a,i),l.normalize()}function C(e){for(;e&&e!==l;){if(e.nodeType===Node.ELEMENT_NODE){var t=e;if("false"==t.getAttribute("contenteditable"))return t}e=e.parentNode}}function E(){var e=H().getRangeAt(0),t=f.createRange();return t.selectNodeContents(l),t.setEnd(e.startContainer,e.startOffset),t.toString()}function m(){var e=H().getRangeAt(0),t=f.createRange();return t.selectNodeContents(l),t.setStart(e.endContainer,e.endOffset),t.toString()}function b(e){g&&"Enter"===e.key&&(B(e),e.stopPropagation(),""==m()?(L("\n "),(e=v()).start=--e.end,T(e)):L("\n"))}function x(){var e,t,n;a&&(e=l.innerHTML,t=v(),(n=u[p])&&n.html===e&&n.pos.start===t.start&&n.pos.end===t.end||(p++,u[p]={html:e,pos:t},u.splice(p+1),300<p&&(p=300,u.splice(0,1))))}function w(e,t){var n=[];e.firstChild&&n.push(e.firstChild);let r=n.pop();for(;r&&"stop"!==t(r);)r.nextSibling&&n.push(r.nextSibling),r.firstChild&&n.push(r.firstChild),r=n.pop()}function k(e){return e.metaKey||e.ctrlKey}function O(e){return k(e)&&!e.shiftKey&&"Z"===D(e)}function M(e){return k(e)&&e.shiftKey&&"Z"===D(e)}function D(e){e=e.key||e.keyCode||e.which;if(e)return("string"==typeof e?e:String.fromCharCode(e)).toUpperCase()}function L(e){e=e.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;").replace(/'/g,"&#039;"),f.execCommand("insertHTML",!1,e)}function S(t,n){let r=0;return(...e)=>{clearTimeout(r),r=o.setTimeout(()=>t(...e),n)}}function A(e){let t=e.length-1;for(;0<=t&&"\n"!==e[t];)t--;let n=++t;for(;n<e.length&&/[ \t]/.test(e[n]);)n++;return[e.substring(t,n)||"",t,n]}function _(){return l.textContent||""}function B(e){e.preventDefault()}function H(){return(l.parentNode?.nodeType==Node.DOCUMENT_FRAGMENT_NODE?l.parentNode:o).getSelection()}return e("keydown",e=>{if(!e.defaultPrevented){if(h=_(),c.preserveIdent){var t=e;if("Enter"===t.key){var n=E(),r=m(),[o]=A(n);let e=o;c.indentOn.test(n)&&(e+=c.tab),0<e.length?(B(t),t.stopPropagation(),L("\n"+e)):b(t),e!==o&&c.moveToNewLine.test(r)&&(n=v(),L("\n"+o),T(n))}}else b(e);if(c.catchTab){t=e;"Tab"===t.key&&(B(t),t.shiftKey?([t,r]=A(E()),0<t.length&&(o=v(),t=Math.min(c.tab.length,t.length),T({start:r,end:r+t}),f.execCommand("delete"),o.start-=t,o.end-=t,T(o))):L(c.tab))}if(c.addClosing){var a,n=e,i=`([{'"`;i.includes(n.key)&&(B(n),a=v(),d=a.start==a.end?"":H().toString(),L(n.key+d+`)]}'"`[i.indexOf(n.key)]),a.start++,a.end++,T(a))}var d,s;c.history&&(O(d=e)&&(B(d),p--,(s=u[p])&&(l.innerHTML=s.html,T(s.pos)),p<0)&&(p=0),M(d)&&(B(d),p++,(s=u[p])&&(l.innerHTML=s.html,T(s.pos)),p>=u.length)&&p--,N(e))&&!y&&(x(),y=!0),g&&(!k(i=e)||"C"!==D(i))&&T(v())}}),e("keyup",e=>{e.defaultPrevented||e.isComposing||(h!==_()&&t(),s(e),i(_()))}),e("focus",e=>{a=!0}),e("blur",e=>{a=!1}),e("paste",e=>{var t;x(),(e=e).defaultPrevented||(B(e),e=(e.originalEvent??e).clipboardData.getData("text/plain").replace(/\r\n?/g,"\n"),t=v(),L(e),d(l),T({start:Math.min(t.start,t.end)+e.length,end:Math.min(t.start,t.end)+e.length,dir:"<-"})),x(),i(_())}),e("cut",e=>{var t,n;x(),e=e,t=v(),n=H(),(e.originalEvent??e).clipboardData.setData("text/plain",n.toString()),f.execCommand("delete"),d(l),T({start:Math.min(t.start,t.end),end:Math.min(t.start,t.end),dir:"<-"}),B(e),x(),i(_())}),{updateOptions(e){Object.assign(c,e)},updateCode(e){l.textContent=e,d(l),i(e)},onUpdate(e){i=e},toString:_,save:v,restore:T,recordHistory:x,destroy(){for(var[e,t]of r)l.removeEventListener(e,t)}}} \ No newline at end of file
diff --git a/interface/js/lib/d3.min.js b/interface/js/lib/d3.min.js
new file mode 100644
index 0000000..8d56002
--- /dev/null
+++ b/interface/js/lib/d3.min.js
@@ -0,0 +1,2 @@
+// https://d3js.org v7.8.5 Copyright 2010-2023 Mike Bostock
+!function(t,n){"object"==typeof exports&&"undefined"!=typeof module?n(exports):"function"==typeof define&&define.amd?define(["exports"],n):n((t="undefined"!=typeof globalThis?globalThis:t||self).d3=t.d3||{})}(this,(function(t){"use strict";function n(t,n){return null==t||null==n?NaN:t<n?-1:t>n?1:t>=n?0:NaN}function e(t,n){return null==t||null==n?NaN:n<t?-1:n>t?1:n>=t?0:NaN}function r(t){let r,o,a;function u(t,n,e=0,i=t.length){if(e<i){if(0!==r(n,n))return i;do{const r=e+i>>>1;o(t[r],n)<0?e=r+1:i=r}while(e<i)}return e}return 2!==t.length?(r=n,o=(e,r)=>n(t(e),r),a=(n,e)=>t(n)-e):(r=t===n||t===e?t:i,o=t,a=t),{left:u,center:function(t,n,e=0,r=t.length){const i=u(t,n,e,r-1);return i>e&&a(t[i-1],n)>-a(t[i],n)?i-1:i},right:function(t,n,e=0,i=t.length){if(e<i){if(0!==r(n,n))return i;do{const r=e+i>>>1;o(t[r],n)<=0?e=r+1:i=r}while(e<i)}return e}}}function i(){return 0}function o(t){return null===t?NaN:+t}const a=r(n),u=a.right,c=a.left,f=r(o).center;var s=u;const l=d(y),h=d((function(t){const n=y(t);return(t,e,r,i,o)=>{n(t,e,(r<<=2)+0,(i<<=2)+0,o<<=2),n(t,e,r+1,i+1,o),n(t,e,r+2,i+2,o),n(t,e,r+3,i+3,o)}}));function d(t){return function(n,e,r=e){if(!((e=+e)>=0))throw new RangeError("invalid rx");if(!((r=+r)>=0))throw new RangeError("invalid ry");let{data:i,width:o,height:a}=n;if(!((o=Math.floor(o))>=0))throw new RangeError("invalid width");if(!((a=Math.floor(void 0!==a?a:i.length/o))>=0))throw new RangeError("invalid height");if(!o||!a||!e&&!r)return n;const u=e&&t(e),c=r&&t(r),f=i.slice();return u&&c?(p(u,f,i,o,a),p(u,i,f,o,a),p(u,f,i,o,a),g(c,i,f,o,a),g(c,f,i,o,a),g(c,i,f,o,a)):u?(p(u,i,f,o,a),p(u,f,i,o,a),p(u,i,f,o,a)):c&&(g(c,i,f,o,a),g(c,f,i,o,a),g(c,i,f,o,a)),n}}function p(t,n,e,r,i){for(let o=0,a=r*i;o<a;)t(n,e,o,o+=r,1)}function g(t,n,e,r,i){for(let o=0,a=r*i;o<r;++o)t(n,e,o,o+a,r)}function y(t){const n=Math.floor(t);if(n===t)return function(t){const n=2*t+1;return(e,r,i,o,a)=>{if(!((o-=a)>=i))return;let u=t*r[i];const c=a*t;for(let t=i,n=i+c;t<n;t+=a)u+=r[Math.min(o,t)];for(let t=i,f=o;t<=f;t+=a)u+=r[Math.min(o,t+c)],e[t]=u/n,u-=r[Math.max(i,t-c)]}}(t);const e=t-n,r=2*t+1;return(t,i,o,a,u)=>{if(!((a-=u)>=o))return;let c=n*i[o];const f=u*n,s=f+u;for(let t=o,n=o+f;t<n;t+=u)c+=i[Math.min(a,t)];for(let n=o,l=a;n<=l;n+=u)c+=i[Math.min(a,n+f)],t[n]=(c+e*(i[Math.max(o,n-s)]+i[Math.min(a,n+s)]))/r,c-=i[Math.max(o,n-f)]}}function v(t,n){let e=0;if(void 0===n)for(let n of t)null!=n&&(n=+n)>=n&&++e;else{let r=-1;for(let i of t)null!=(i=n(i,++r,t))&&(i=+i)>=i&&++e}return e}function _(t){return 0|t.length}function b(t){return!(t>0)}function m(t){return"object"!=typeof t||"length"in t?t:Array.from(t)}function x(t,n){let e,r=0,i=0,o=0;if(void 0===n)for(let n of t)null!=n&&(n=+n)>=n&&(e=n-i,i+=e/++r,o+=e*(n-i));else{let a=-1;for(let u of t)null!=(u=n(u,++a,t))&&(u=+u)>=u&&(e=u-i,i+=e/++r,o+=e*(u-i))}if(r>1)return o/(r-1)}function w(t,n){const e=x(t,n);return e?Math.sqrt(e):e}function M(t,n){let e,r;if(void 0===n)for(const n of t)null!=n&&(void 0===e?n>=n&&(e=r=n):(e>n&&(e=n),r<n&&(r=n)));else{let i=-1;for(let o of t)null!=(o=n(o,++i,t))&&(void 0===e?o>=o&&(e=r=o):(e>o&&(e=o),r<o&&(r=o)))}return[e,r]}class T{constructor(){this._partials=new Float64Array(32),this._n=0}add(t){const n=this._partials;let e=0;for(let r=0;r<this._n&&r<32;r++){const i=n[r],o=t+i,a=Math.abs(t)<Math.abs(i)?t-(o-i):i-(o-t);a&&(n[e++]=a),t=o}return n[e]=t,this._n=e+1,this}valueOf(){const t=this._partials;let n,e,r,i=this._n,o=0;if(i>0){for(o=t[--i];i>0&&(n=o,e=t[--i],o=n+e,r=e-(o-n),!r););i>0&&(r<0&&t[i-1]<0||r>0&&t[i-1]>0)&&(e=2*r,n=o+e,e==n-o&&(o=n))}return o}}class InternMap extends Map{constructor(t,n=N){if(super(),Object.defineProperties(this,{_intern:{value:new Map},_key:{value:n}}),null!=t)for(const[n,e]of t)this.set(n,e)}get(t){return super.get(A(this,t))}has(t){return super.has(A(this,t))}set(t,n){return super.set(S(this,t),n)}delete(t){return super.delete(E(this,t))}}class InternSet extends Set{constructor(t,n=N){if(super(),Object.defineProperties(this,{_intern:{value:new Map},_key:{value:n}}),null!=t)for(const n of t)this.add(n)}has(t){return super.has(A(this,t))}add(t){return super.add(S(this,t))}delete(t){return super.delete(E(this,t))}}function A({_intern:t,_key:n},e){const r=n(e);return t.has(r)?t.get(r):e}function S({_intern:t,_key:n},e){const r=n(e);return t.has(r)?t.get(r):(t.set(r,e),e)}function E({_intern:t,_key:n},e){const r=n(e);return t.has(r)&&(e=t.get(r),t.delete(r)),e}function N(t){return null!==t&&"object"==typeof t?t.valueOf():t}function k(t){return t}function C(t,...n){return F(t,k,k,n)}function P(t,...n){return F(t,Array.from,k,n)}function z(t,n){for(let e=1,r=n.length;e<r;++e)t=t.flatMap((t=>t.pop().map((([n,e])=>[...t,n,e]))));return t}function $(t,n,...e){return F(t,k,n,e)}function D(t,n,...e){return F(t,Array.from,n,e)}function R(t){if(1!==t.length)throw new Error("duplicate key");return t[0]}function F(t,n,e,r){return function t(i,o){if(o>=r.length)return e(i);const a=new InternMap,u=r[o++];let c=-1;for(const t of i){const n=u(t,++c,i),e=a.get(n);e?e.push(t):a.set(n,[t])}for(const[n,e]of a)a.set(n,t(e,o));return n(a)}(t,0)}function q(t,n){return Array.from(n,(n=>t[n]))}function U(t,...n){if("function"!=typeof t[Symbol.iterator])throw new TypeError("values is not iterable");t=Array.from(t);let[e]=n;if(e&&2!==e.length||n.length>1){const r=Uint32Array.from(t,((t,n)=>n));return n.length>1?(n=n.map((n=>t.map(n))),r.sort(((t,e)=>{for(const r of n){const n=O(r[t],r[e]);if(n)return n}}))):(e=t.map(e),r.sort(((t,n)=>O(e[t],e[n])))),q(t,r)}return t.sort(I(e))}function I(t=n){if(t===n)return O;if("function"!=typeof t)throw new TypeError("compare is not a function");return(n,e)=>{const r=t(n,e);return r||0===r?r:(0===t(e,e))-(0===t(n,n))}}function O(t,n){return(null==t||!(t>=t))-(null==n||!(n>=n))||(t<n?-1:t>n?1:0)}var B=Array.prototype.slice;function Y(t){return()=>t}const L=Math.sqrt(50),j=Math.sqrt(10),H=Math.sqrt(2);function X(t,n,e){const r=(n-t)/Math.max(0,e),i=Math.floor(Math.log10(r)),o=r/Math.pow(10,i),a=o>=L?10:o>=j?5:o>=H?2:1;let u,c,f;return i<0?(f=Math.pow(10,-i)/a,u=Math.round(t*f),c=Math.round(n*f),u/f<t&&++u,c/f>n&&--c,f=-f):(f=Math.pow(10,i)*a,u=Math.round(t/f),c=Math.round(n/f),u*f<t&&++u,c*f>n&&--c),c<u&&.5<=e&&e<2?X(t,n,2*e):[u,c,f]}function G(t,n,e){if(!((e=+e)>0))return[];if((t=+t)===(n=+n))return[t];const r=n<t,[i,o,a]=r?X(n,t,e):X(t,n,e);if(!(o>=i))return[];const u=o-i+1,c=new Array(u);if(r)if(a<0)for(let t=0;t<u;++t)c[t]=(o-t)/-a;else for(let t=0;t<u;++t)c[t]=(o-t)*a;else if(a<0)for(let t=0;t<u;++t)c[t]=(i+t)/-a;else for(let t=0;t<u;++t)c[t]=(i+t)*a;return c}function V(t,n,e){return X(t=+t,n=+n,e=+e)[2]}function W(t,n,e){e=+e;const r=(n=+n)<(t=+t),i=r?V(n,t,e):V(t,n,e);return(r?-1:1)*(i<0?1/-i:i)}function Z(t,n,e){let r;for(;;){const i=V(t,n,e);if(i===r||0===i||!isFinite(i))return[t,n];i>0?(t=Math.floor(t/i)*i,n=Math.ceil(n/i)*i):i<0&&(t=Math.ceil(t*i)/i,n=Math.floor(n*i)/i),r=i}}function K(t){return Math.max(1,Math.ceil(Math.log(v(t))/Math.LN2)+1)}function Q(){var t=k,n=M,e=K;function r(r){Array.isArray(r)||(r=Array.from(r));var i,o,a,u=r.length,c=new Array(u);for(i=0;i<u;++i)c[i]=t(r[i],i,r);var f=n(c),l=f[0],h=f[1],d=e(c,l,h);if(!Array.isArray(d)){const t=h,e=+d;if(n===M&&([l,h]=Z(l,h,e)),(d=G(l,h,e))[0]<=l&&(a=V(l,h,e)),d[d.length-1]>=h)if(t>=h&&n===M){const t=V(l,h,e);isFinite(t)&&(t>0?h=(Math.floor(h/t)+1)*t:t<0&&(h=(Math.ceil(h*-t)+1)/-t))}else d.pop()}for(var p=d.length,g=0,y=p;d[g]<=l;)++g;for(;d[y-1]>h;)--y;(g||y<p)&&(d=d.slice(g,y),p=y-g);var v,_=new Array(p+1);for(i=0;i<=p;++i)(v=_[i]=[]).x0=i>0?d[i-1]:l,v.x1=i<p?d[i]:h;if(isFinite(a)){if(a>0)for(i=0;i<u;++i)null!=(o=c[i])&&l<=o&&o<=h&&_[Math.min(p,Math.floor((o-l)/a))].push(r[i]);else if(a<0)for(i=0;i<u;++i)if(null!=(o=c[i])&&l<=o&&o<=h){const t=Math.floor((l-o)*a);_[Math.min(p,t+(d[t]<=o))].push(r[i])}}else for(i=0;i<u;++i)null!=(o=c[i])&&l<=o&&o<=h&&_[s(d,o,0,p)].push(r[i]);return _}return r.value=function(n){return arguments.length?(t="function"==typeof n?n:Y(n),r):t},r.domain=function(t){return arguments.length?(n="function"==typeof t?t:Y([t[0],t[1]]),r):n},r.thresholds=function(t){return arguments.length?(e="function"==typeof t?t:Y(Array.isArray(t)?B.call(t):t),r):e},r}function J(t,n){let e;if(void 0===n)for(const n of t)null!=n&&(e<n||void 0===e&&n>=n)&&(e=n);else{let r=-1;for(let i of t)null!=(i=n(i,++r,t))&&(e<i||void 0===e&&i>=i)&&(e=i)}return e}function tt(t,n){let e,r=-1,i=-1;if(void 0===n)for(const n of t)++i,null!=n&&(e<n||void 0===e&&n>=n)&&(e=n,r=i);else for(let o of t)null!=(o=n(o,++i,t))&&(e<o||void 0===e&&o>=o)&&(e=o,r=i);return r}function nt(t,n){let e;if(void 0===n)for(const n of t)null!=n&&(e>n||void 0===e&&n>=n)&&(e=n);else{let r=-1;for(let i of t)null!=(i=n(i,++r,t))&&(e>i||void 0===e&&i>=i)&&(e=i)}return e}function et(t,n){let e,r=-1,i=-1;if(void 0===n)for(const n of t)++i,null!=n&&(e>n||void 0===e&&n>=n)&&(e=n,r=i);else for(let o of t)null!=(o=n(o,++i,t))&&(e>o||void 0===e&&o>=o)&&(e=o,r=i);return r}function rt(t,n,e=0,r=1/0,i){if(n=Math.floor(n),e=Math.floor(Math.max(0,e)),r=Math.floor(Math.min(t.length-1,r)),!(e<=n&&n<=r))return t;for(i=void 0===i?O:I(i);r>e;){if(r-e>600){const o=r-e+1,a=n-e+1,u=Math.log(o),c=.5*Math.exp(2*u/3),f=.5*Math.sqrt(u*c*(o-c)/o)*(a-o/2<0?-1:1);rt(t,n,Math.max(e,Math.floor(n-a*c/o+f)),Math.min(r,Math.floor(n+(o-a)*c/o+f)),i)}const o=t[n];let a=e,u=r;for(it(t,e,n),i(t[r],o)>0&&it(t,e,r);a<u;){for(it(t,a,u),++a,--u;i(t[a],o)<0;)++a;for(;i(t[u],o)>0;)--u}0===i(t[e],o)?it(t,e,u):(++u,it(t,u,r)),u<=n&&(e=u+1),n<=u&&(r=u-1)}return t}function it(t,n,e){const r=t[n];t[n]=t[e],t[e]=r}function ot(t,e=n){let r,i=!1;if(1===e.length){let o;for(const a of t){const t=e(a);(i?n(t,o)>0:0===n(t,t))&&(r=a,o=t,i=!0)}}else for(const n of t)(i?e(n,r)>0:0===e(n,n))&&(r=n,i=!0);return r}function at(t,n,e){if(t=Float64Array.from(function*(t,n){if(void 0===n)for(let n of t)null!=n&&(n=+n)>=n&&(yield n);else{let e=-1;for(let r of t)null!=(r=n(r,++e,t))&&(r=+r)>=r&&(yield r)}}(t,e)),(r=t.length)&&!isNaN(n=+n)){if(n<=0||r<2)return nt(t);if(n>=1)return J(t);var r,i=(r-1)*n,o=Math.floor(i),a=J(rt(t,o).subarray(0,o+1));return a+(nt(t.subarray(o+1))-a)*(i-o)}}function ut(t,n,e=o){if((r=t.length)&&!isNaN(n=+n)){if(n<=0||r<2)return+e(t[0],0,t);if(n>=1)return+e(t[r-1],r-1,t);var r,i=(r-1)*n,a=Math.floor(i),u=+e(t[a],a,t);return u+(+e(t[a+1],a+1,t)-u)*(i-a)}}function ct(t,n,e=o){if(!isNaN(n=+n)){if(r=Float64Array.from(t,((n,r)=>o(e(t[r],r,t)))),n<=0)return et(r);if(n>=1)return tt(r);var r,i=Uint32Array.from(t,((t,n)=>n)),a=r.length-1,u=Math.floor(a*n);return rt(i,u,0,a,((t,n)=>O(r[t],r[n]))),(u=ot(i.subarray(0,u+1),(t=>r[t])))>=0?u:-1}}function ft(t){return Array.from(function*(t){for(const n of t)yield*n}(t))}function st(t,n){return[t,n]}function lt(t,n,e){t=+t,n=+n,e=(i=arguments.length)<2?(n=t,t=0,1):i<3?1:+e;for(var r=-1,i=0|Math.max(0,Math.ceil((n-t)/e)),o=new Array(i);++r<i;)o[r]=t+r*e;return o}function ht(t,e=n){if(1===e.length)return et(t,e);let r,i=-1,o=-1;for(const n of t)++o,(i<0?0===e(n,n):e(n,r)<0)&&(r=n,i=o);return i}var dt=pt(Math.random);function pt(t){return function(n,e=0,r=n.length){let i=r-(e=+e);for(;i;){const r=t()*i--|0,o=n[i+e];n[i+e]=n[r+e],n[r+e]=o}return n}}function gt(t){if(!(i=t.length))return[];for(var n=-1,e=nt(t,yt),r=new Array(e);++n<e;)for(var i,o=-1,a=r[n]=new Array(i);++o<i;)a[o]=t[o][n];return r}function yt(t){return t.length}function vt(t){return t instanceof InternSet?t:new InternSet(t)}function _t(t,n){const e=t[Symbol.iterator](),r=new Set;for(const t of n){const n=bt(t);if(r.has(n))continue;let i,o;for(;({value:i,done:o}=e.next());){if(o)return!1;const t=bt(i);if(r.add(t),Object.is(n,t))break}}return!0}function bt(t){return null!==t&&"object"==typeof t?t.valueOf():t}function mt(t){return t}var xt=1,wt=2,Mt=3,Tt=4,At=1e-6;function St(t){return"translate("+t+",0)"}function Et(t){return"translate(0,"+t+")"}function Nt(t){return n=>+t(n)}function kt(t,n){return n=Math.max(0,t.bandwidth()-2*n)/2,t.round()&&(n=Math.round(n)),e=>+t(e)+n}function Ct(){return!this.__axis}function Pt(t,n){var e=[],r=null,i=null,o=6,a=6,u=3,c="undefined"!=typeof window&&window.devicePixelRatio>1?0:.5,f=t===xt||t===Tt?-1:1,s=t===Tt||t===wt?"x":"y",l=t===xt||t===Mt?St:Et;function h(h){var d=null==r?n.ticks?n.ticks.apply(n,e):n.domain():r,p=null==i?n.tickFormat?n.tickFormat.apply(n,e):mt:i,g=Math.max(o,0)+u,y=n.range(),v=+y[0]+c,_=+y[y.length-1]+c,b=(n.bandwidth?kt:Nt)(n.copy(),c),m=h.selection?h.selection():h,x=m.selectAll(".domain").data([null]),w=m.selectAll(".tick").data(d,n).order(),M=w.exit(),T=w.enter().append("g").attr("class","tick"),A=w.select("line"),S=w.select("text");x=x.merge(x.enter().insert("path",".tick").attr("class","domain").attr("stroke","currentColor")),w=w.merge(T),A=A.merge(T.append("line").attr("stroke","currentColor").attr(s+"2",f*o)),S=S.merge(T.append("text").attr("fill","currentColor").attr(s,f*g).attr("dy",t===xt?"0em":t===Mt?"0.71em":"0.32em")),h!==m&&(x=x.transition(h),w=w.transition(h),A=A.transition(h),S=S.transition(h),M=M.transition(h).attr("opacity",At).attr("transform",(function(t){return isFinite(t=b(t))?l(t+c):this.getAttribute("transform")})),T.attr("opacity",At).attr("transform",(function(t){var n=this.parentNode.__axis;return l((n&&isFinite(n=n(t))?n:b(t))+c)}))),M.remove(),x.attr("d",t===Tt||t===wt?a?"M"+f*a+","+v+"H"+c+"V"+_+"H"+f*a:"M"+c+","+v+"V"+_:a?"M"+v+","+f*a+"V"+c+"H"+_+"V"+f*a:"M"+v+","+c+"H"+_),w.attr("opacity",1).attr("transform",(function(t){return l(b(t)+c)})),A.attr(s+"2",f*o),S.attr(s,f*g).text(p),m.filter(Ct).attr("fill","none").attr("font-size",10).attr("font-family","sans-serif").attr("text-anchor",t===wt?"start":t===Tt?"end":"middle"),m.each((function(){this.__axis=b}))}return h.scale=function(t){return arguments.length?(n=t,h):n},h.ticks=function(){return e=Array.from(arguments),h},h.tickArguments=function(t){return arguments.length?(e=null==t?[]:Array.from(t),h):e.slice()},h.tickValues=function(t){return arguments.length?(r=null==t?null:Array.from(t),h):r&&r.slice()},h.tickFormat=function(t){return arguments.length?(i=t,h):i},h.tickSize=function(t){return arguments.length?(o=a=+t,h):o},h.tickSizeInner=function(t){return arguments.length?(o=+t,h):o},h.tickSizeOuter=function(t){return arguments.length?(a=+t,h):a},h.tickPadding=function(t){return arguments.length?(u=+t,h):u},h.offset=function(t){return arguments.length?(c=+t,h):c},h}var zt={value:()=>{}};function $t(){for(var t,n=0,e=arguments.length,r={};n<e;++n){if(!(t=arguments[n]+"")||t in r||/[\s.]/.test(t))throw new Error("illegal type: "+t);r[t]=[]}return new Dt(r)}function Dt(t){this._=t}function Rt(t,n){for(var e,r=0,i=t.length;r<i;++r)if((e=t[r]).name===n)return e.value}function Ft(t,n,e){for(var r=0,i=t.length;r<i;++r)if(t[r].name===n){t[r]=zt,t=t.slice(0,r).concat(t.slice(r+1));break}return null!=e&&t.push({name:n,value:e}),t}Dt.prototype=$t.prototype={constructor:Dt,on:function(t,n){var e,r,i=this._,o=(r=i,(t+"").trim().split(/^|\s+/).map((function(t){var n="",e=t.indexOf(".");if(e>=0&&(n=t.slice(e+1),t=t.slice(0,e)),t&&!r.hasOwnProperty(t))throw new Error("unknown type: "+t);return{type:t,name:n}}))),a=-1,u=o.length;if(!(arguments.length<2)){if(null!=n&&"function"!=typeof n)throw new Error("invalid callback: "+n);for(;++a<u;)if(e=(t=o[a]).type)i[e]=Ft(i[e],t.name,n);else if(null==n)for(e in i)i[e]=Ft(i[e],t.name,null);return this}for(;++a<u;)if((e=(t=o[a]).type)&&(e=Rt(i[e],t.name)))return e},copy:function(){var t={},n=this._;for(var e in n)t[e]=n[e].slice();return new Dt(t)},call:function(t,n){if((e=arguments.length-2)>0)for(var e,r,i=new Array(e),o=0;o<e;++o)i[o]=arguments[o+2];if(!this._.hasOwnProperty(t))throw new Error("unknown type: "+t);for(o=0,e=(r=this._[t]).length;o<e;++o)r[o].value.apply(n,i)},apply:function(t,n,e){if(!this._.hasOwnProperty(t))throw new Error("unknown type: "+t);for(var r=this._[t],i=0,o=r.length;i<o;++i)r[i].value.apply(n,e)}};var qt="http://www.w3.org/1999/xhtml",Ut={svg:"http://www.w3.org/2000/svg",xhtml:qt,xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"};function It(t){var n=t+="",e=n.indexOf(":");return e>=0&&"xmlns"!==(n=t.slice(0,e))&&(t=t.slice(e+1)),Ut.hasOwnProperty(n)?{space:Ut[n],local:t}:t}function Ot(t){return function(){var n=this.ownerDocument,e=this.namespaceURI;return e===qt&&n.documentElement.namespaceURI===qt?n.createElement(t):n.createElementNS(e,t)}}function Bt(t){return function(){return this.ownerDocument.createElementNS(t.space,t.local)}}function Yt(t){var n=It(t);return(n.local?Bt:Ot)(n)}function Lt(){}function jt(t){return null==t?Lt:function(){return this.querySelector(t)}}function Ht(t){return null==t?[]:Array.isArray(t)?t:Array.from(t)}function Xt(){return[]}function Gt(t){return null==t?Xt:function(){return this.querySelectorAll(t)}}function Vt(t){return function(){return this.matches(t)}}function Wt(t){return function(n){return n.matches(t)}}var Zt=Array.prototype.find;function Kt(){return this.firstElementChild}var Qt=Array.prototype.filter;function Jt(){return Array.from(this.children)}function tn(t){return new Array(t.length)}function nn(t,n){this.ownerDocument=t.ownerDocument,this.namespaceURI=t.namespaceURI,this._next=null,this._parent=t,this.__data__=n}function en(t,n,e,r,i,o){for(var a,u=0,c=n.length,f=o.length;u<f;++u)(a=n[u])?(a.__data__=o[u],r[u]=a):e[u]=new nn(t,o[u]);for(;u<c;++u)(a=n[u])&&(i[u]=a)}function rn(t,n,e,r,i,o,a){var u,c,f,s=new Map,l=n.length,h=o.length,d=new Array(l);for(u=0;u<l;++u)(c=n[u])&&(d[u]=f=a.call(c,c.__data__,u,n)+"",s.has(f)?i[u]=c:s.set(f,c));for(u=0;u<h;++u)f=a.call(t,o[u],u,o)+"",(c=s.get(f))?(r[u]=c,c.__data__=o[u],s.delete(f)):e[u]=new nn(t,o[u]);for(u=0;u<l;++u)(c=n[u])&&s.get(d[u])===c&&(i[u]=c)}function on(t){return t.__data__}function an(t){return"object"==typeof t&&"length"in t?t:Array.from(t)}function un(t,n){return t<n?-1:t>n?1:t>=n?0:NaN}function cn(t){return function(){this.removeAttribute(t)}}function fn(t){return function(){this.removeAttributeNS(t.space,t.local)}}function sn(t,n){return function(){this.setAttribute(t,n)}}function ln(t,n){return function(){this.setAttributeNS(t.space,t.local,n)}}function hn(t,n){return function(){var e=n.apply(this,arguments);null==e?this.removeAttribute(t):this.setAttribute(t,e)}}function dn(t,n){return function(){var e=n.apply(this,arguments);null==e?this.removeAttributeNS(t.space,t.local):this.setAttributeNS(t.space,t.local,e)}}function pn(t){return t.ownerDocument&&t.ownerDocument.defaultView||t.document&&t||t.defaultView}function gn(t){return function(){this.style.removeProperty(t)}}function yn(t,n,e){return function(){this.style.setProperty(t,n,e)}}function vn(t,n,e){return function(){var r=n.apply(this,arguments);null==r?this.style.removeProperty(t):this.style.setProperty(t,r,e)}}function _n(t,n){return t.style.getPropertyValue(n)||pn(t).getComputedStyle(t,null).getPropertyValue(n)}function bn(t){return function(){delete this[t]}}function mn(t,n){return function(){this[t]=n}}function xn(t,n){return function(){var e=n.apply(this,arguments);null==e?delete this[t]:this[t]=e}}function wn(t){return t.trim().split(/^|\s+/)}function Mn(t){return t.classList||new Tn(t)}function Tn(t){this._node=t,this._names=wn(t.getAttribute("class")||"")}function An(t,n){for(var e=Mn(t),r=-1,i=n.length;++r<i;)e.add(n[r])}function Sn(t,n){for(var e=Mn(t),r=-1,i=n.length;++r<i;)e.remove(n[r])}function En(t){return function(){An(this,t)}}function Nn(t){return function(){Sn(this,t)}}function kn(t,n){return function(){(n.apply(this,arguments)?An:Sn)(this,t)}}function Cn(){this.textContent=""}function Pn(t){return function(){this.textContent=t}}function zn(t){return function(){var n=t.apply(this,arguments);this.textContent=null==n?"":n}}function $n(){this.innerHTML=""}function Dn(t){return function(){this.innerHTML=t}}function Rn(t){return function(){var n=t.apply(this,arguments);this.innerHTML=null==n?"":n}}function Fn(){this.nextSibling&&this.parentNode.appendChild(this)}function qn(){this.previousSibling&&this.parentNode.insertBefore(this,this.parentNode.firstChild)}function Un(){return null}function In(){var t=this.parentNode;t&&t.removeChild(this)}function On(){var t=this.cloneNode(!1),n=this.parentNode;return n?n.insertBefore(t,this.nextSibling):t}function Bn(){var t=this.cloneNode(!0),n=this.parentNode;return n?n.insertBefore(t,this.nextSibling):t}function Yn(t){return function(){var n=this.__on;if(n){for(var e,r=0,i=-1,o=n.length;r<o;++r)e=n[r],t.type&&e.type!==t.type||e.name!==t.name?n[++i]=e:this.removeEventListener(e.type,e.listener,e.options);++i?n.length=i:delete this.__on}}}function Ln(t,n,e){return function(){var r,i=this.__on,o=function(t){return function(n){t.call(this,n,this.__data__)}}(n);if(i)for(var a=0,u=i.length;a<u;++a)if((r=i[a]).type===t.type&&r.name===t.name)return this.removeEventListener(r.type,r.listener,r.options),this.addEventListener(r.type,r.listener=o,r.options=e),void(r.value=n);this.addEventListener(t.type,o,e),r={type:t.type,name:t.name,value:n,listener:o,options:e},i?i.push(r):this.__on=[r]}}function jn(t,n,e){var r=pn(t),i=r.CustomEvent;"function"==typeof i?i=new i(n,e):(i=r.document.createEvent("Event"),e?(i.initEvent(n,e.bubbles,e.cancelable),i.detail=e.detail):i.initEvent(n,!1,!1)),t.dispatchEvent(i)}function Hn(t,n){return function(){return jn(this,t,n)}}function Xn(t,n){return function(){return jn(this,t,n.apply(this,arguments))}}nn.prototype={constructor:nn,appendChild:function(t){return this._parent.insertBefore(t,this._next)},insertBefore:function(t,n){return this._parent.insertBefore(t,n)},querySelector:function(t){return this._parent.querySelector(t)},querySelectorAll:function(t){return this._parent.querySelectorAll(t)}},Tn.prototype={add:function(t){this._names.indexOf(t)<0&&(this._names.push(t),this._node.setAttribute("class",this._names.join(" ")))},remove:function(t){var n=this._names.indexOf(t);n>=0&&(this._names.splice(n,1),this._node.setAttribute("class",this._names.join(" ")))},contains:function(t){return this._names.indexOf(t)>=0}};var Gn=[null];function Vn(t,n){this._groups=t,this._parents=n}function Wn(){return new Vn([[document.documentElement]],Gn)}function Zn(t){return"string"==typeof t?new Vn([[document.querySelector(t)]],[document.documentElement]):new Vn([[t]],Gn)}Vn.prototype=Wn.prototype={constructor:Vn,select:function(t){"function"!=typeof t&&(t=jt(t));for(var n=this._groups,e=n.length,r=new Array(e),i=0;i<e;++i)for(var o,a,u=n[i],c=u.length,f=r[i]=new Array(c),s=0;s<c;++s)(o=u[s])&&(a=t.call(o,o.__data__,s,u))&&("__data__"in o&&(a.__data__=o.__data__),f[s]=a);return new Vn(r,this._parents)},selectAll:function(t){t="function"==typeof t?function(t){return function(){return Ht(t.apply(this,arguments))}}(t):Gt(t);for(var n=this._groups,e=n.length,r=[],i=[],o=0;o<e;++o)for(var a,u=n[o],c=u.length,f=0;f<c;++f)(a=u[f])&&(r.push(t.call(a,a.__data__,f,u)),i.push(a));return new Vn(r,i)},selectChild:function(t){return this.select(null==t?Kt:function(t){return function(){return Zt.call(this.children,t)}}("function"==typeof t?t:Wt(t)))},selectChildren:function(t){return this.selectAll(null==t?Jt:function(t){return function(){return Qt.call(this.children,t)}}("function"==typeof t?t:Wt(t)))},filter:function(t){"function"!=typeof t&&(t=Vt(t));for(var n=this._groups,e=n.length,r=new Array(e),i=0;i<e;++i)for(var o,a=n[i],u=a.length,c=r[i]=[],f=0;f<u;++f)(o=a[f])&&t.call(o,o.__data__,f,a)&&c.push(o);return new Vn(r,this._parents)},data:function(t,n){if(!arguments.length)return Array.from(this,on);var e=n?rn:en,r=this._parents,i=this._groups;"function"!=typeof t&&(t=function(t){return function(){return t}}(t));for(var o=i.length,a=new Array(o),u=new Array(o),c=new Array(o),f=0;f<o;++f){var s=r[f],l=i[f],h=l.length,d=an(t.call(s,s&&s.__data__,f,r)),p=d.length,g=u[f]=new Array(p),y=a[f]=new Array(p);e(s,l,g,y,c[f]=new Array(h),d,n);for(var v,_,b=0,m=0;b<p;++b)if(v=g[b]){for(b>=m&&(m=b+1);!(_=y[m])&&++m<p;);v._next=_||null}}return(a=new Vn(a,r))._enter=u,a._exit=c,a},enter:function(){return new Vn(this._enter||this._groups.map(tn),this._parents)},exit:function(){return new Vn(this._exit||this._groups.map(tn),this._parents)},join:function(t,n,e){var r=this.enter(),i=this,o=this.exit();return"function"==typeof t?(r=t(r))&&(r=r.selection()):r=r.append(t+""),null!=n&&(i=n(i))&&(i=i.selection()),null==e?o.remove():e(o),r&&i?r.merge(i).order():i},merge:function(t){for(var n=t.selection?t.selection():t,e=this._groups,r=n._groups,i=e.length,o=r.length,a=Math.min(i,o),u=new Array(i),c=0;c<a;++c)for(var f,s=e[c],l=r[c],h=s.length,d=u[c]=new Array(h),p=0;p<h;++p)(f=s[p]||l[p])&&(d[p]=f);for(;c<i;++c)u[c]=e[c];return new Vn(u,this._parents)},selection:function(){return this},order:function(){for(var t=this._groups,n=-1,e=t.length;++n<e;)for(var r,i=t[n],o=i.length-1,a=i[o];--o>=0;)(r=i[o])&&(a&&4^r.compareDocumentPosition(a)&&a.parentNode.insertBefore(r,a),a=r);return this},sort:function(t){function n(n,e){return n&&e?t(n.__data__,e.__data__):!n-!e}t||(t=un);for(var e=this._groups,r=e.length,i=new Array(r),o=0;o<r;++o){for(var a,u=e[o],c=u.length,f=i[o]=new Array(c),s=0;s<c;++s)(a=u[s])&&(f[s]=a);f.sort(n)}return new Vn(i,this._parents).order()},call:function(){var t=arguments[0];return arguments[0]=this,t.apply(null,arguments),this},nodes:function(){return Array.from(this)},node:function(){for(var t=this._groups,n=0,e=t.length;n<e;++n)for(var r=t[n],i=0,o=r.length;i<o;++i){var a=r[i];if(a)return a}return null},size:function(){let t=0;for(const n of this)++t;return t},empty:function(){return!this.node()},each:function(t){for(var n=this._groups,e=0,r=n.length;e<r;++e)for(var i,o=n[e],a=0,u=o.length;a<u;++a)(i=o[a])&&t.call(i,i.__data__,a,o);return this},attr:function(t,n){var e=It(t);if(arguments.length<2){var r=this.node();return e.local?r.getAttributeNS(e.space,e.local):r.getAttribute(e)}return this.each((null==n?e.local?fn:cn:"function"==typeof n?e.local?dn:hn:e.local?ln:sn)(e,n))},style:function(t,n,e){return arguments.length>1?this.each((null==n?gn:"function"==typeof n?vn:yn)(t,n,null==e?"":e)):_n(this.node(),t)},property:function(t,n){return arguments.length>1?this.each((null==n?bn:"function"==typeof n?xn:mn)(t,n)):this.node()[t]},classed:function(t,n){var e=wn(t+"");if(arguments.length<2){for(var r=Mn(this.node()),i=-1,o=e.length;++i<o;)if(!r.contains(e[i]))return!1;return!0}return this.each(("function"==typeof n?kn:n?En:Nn)(e,n))},text:function(t){return arguments.length?this.each(null==t?Cn:("function"==typeof t?zn:Pn)(t)):this.node().textContent},html:function(t){return arguments.length?this.each(null==t?$n:("function"==typeof t?Rn:Dn)(t)):this.node().innerHTML},raise:function(){return this.each(Fn)},lower:function(){return this.each(qn)},append:function(t){var n="function"==typeof t?t:Yt(t);return this.select((function(){return this.appendChild(n.apply(this,arguments))}))},insert:function(t,n){var e="function"==typeof t?t:Yt(t),r=null==n?Un:"function"==typeof n?n:jt(n);return this.select((function(){return this.insertBefore(e.apply(this,arguments),r.apply(this,arguments)||null)}))},remove:function(){return this.each(In)},clone:function(t){return this.select(t?Bn:On)},datum:function(t){return arguments.length?this.property("__data__",t):this.node().__data__},on:function(t,n,e){var r,i,o=function(t){return t.trim().split(/^|\s+/).map((function(t){var n="",e=t.indexOf(".");return e>=0&&(n=t.slice(e+1),t=t.slice(0,e)),{type:t,name:n}}))}(t+""),a=o.length;if(!(arguments.length<2)){for(u=n?Ln:Yn,r=0;r<a;++r)this.each(u(o[r],n,e));return this}var u=this.node().__on;if(u)for(var c,f=0,s=u.length;f<s;++f)for(r=0,c=u[f];r<a;++r)if((i=o[r]).type===c.type&&i.name===c.name)return c.value},dispatch:function(t,n){return this.each(("function"==typeof n?Xn:Hn)(t,n))},[Symbol.iterator]:function*(){for(var t=this._groups,n=0,e=t.length;n<e;++n)for(var r,i=t[n],o=0,a=i.length;o<a;++o)(r=i[o])&&(yield r)}};var Kn=0;function Qn(){return new Jn}function Jn(){this._="@"+(++Kn).toString(36)}function te(t){let n;for(;n=t.sourceEvent;)t=n;return t}function ne(t,n){if(t=te(t),void 0===n&&(n=t.currentTarget),n){var e=n.ownerSVGElement||n;if(e.createSVGPoint){var r=e.createSVGPoint();return r.x=t.clientX,r.y=t.clientY,[(r=r.matrixTransform(n.getScreenCTM().inverse())).x,r.y]}if(n.getBoundingClientRect){var i=n.getBoundingClientRect();return[t.clientX-i.left-n.clientLeft,t.clientY-i.top-n.clientTop]}}return[t.pageX,t.pageY]}Jn.prototype=Qn.prototype={constructor:Jn,get:function(t){for(var n=this._;!(n in t);)if(!(t=t.parentNode))return;return t[n]},set:function(t,n){return t[this._]=n},remove:function(t){return this._ in t&&delete t[this._]},toString:function(){return this._}};const ee={passive:!1},re={capture:!0,passive:!1};function ie(t){t.stopImmediatePropagation()}function oe(t){t.preventDefault(),t.stopImmediatePropagation()}function ae(t){var n=t.document.documentElement,e=Zn(t).on("dragstart.drag",oe,re);"onselectstart"in n?e.on("selectstart.drag",oe,re):(n.__noselect=n.style.MozUserSelect,n.style.MozUserSelect="none")}function ue(t,n){var e=t.document.documentElement,r=Zn(t).on("dragstart.drag",null);n&&(r.on("click.drag",oe,re),setTimeout((function(){r.on("click.drag",null)}),0)),"onselectstart"in e?r.on("selectstart.drag",null):(e.style.MozUserSelect=e.__noselect,delete e.__noselect)}var ce=t=>()=>t;function fe(t,{sourceEvent:n,subject:e,target:r,identifier:i,active:o,x:a,y:u,dx:c,dy:f,dispatch:s}){Object.defineProperties(this,{type:{value:t,enumerable:!0,configurable:!0},sourceEvent:{value:n,enumerable:!0,configurable:!0},subject:{value:e,enumerable:!0,configurable:!0},target:{value:r,enumerable:!0,configurable:!0},identifier:{value:i,enumerable:!0,configurable:!0},active:{value:o,enumerable:!0,configurable:!0},x:{value:a,enumerable:!0,configurable:!0},y:{value:u,enumerable:!0,configurable:!0},dx:{value:c,enumerable:!0,configurable:!0},dy:{value:f,enumerable:!0,configurable:!0},_:{value:s}})}function se(t){return!t.ctrlKey&&!t.button}function le(){return this.parentNode}function he(t,n){return null==n?{x:t.x,y:t.y}:n}function de(){return navigator.maxTouchPoints||"ontouchstart"in this}function pe(t,n,e){t.prototype=n.prototype=e,e.constructor=t}function ge(t,n){var e=Object.create(t.prototype);for(var r in n)e[r]=n[r];return e}function ye(){}fe.prototype.on=function(){var t=this._.on.apply(this._,arguments);return t===this._?this:t};var ve=.7,_e=1/ve,be="\\s*([+-]?\\d+)\\s*",me="\\s*([+-]?(?:\\d*\\.)?\\d+(?:[eE][+-]?\\d+)?)\\s*",xe="\\s*([+-]?(?:\\d*\\.)?\\d+(?:[eE][+-]?\\d+)?)%\\s*",we=/^#([0-9a-f]{3,8})$/,Me=new RegExp(`^rgb\\(${be},${be},${be}\\)$`),Te=new RegExp(`^rgb\\(${xe},${xe},${xe}\\)$`),Ae=new RegExp(`^rgba\\(${be},${be},${be},${me}\\)$`),Se=new RegExp(`^rgba\\(${xe},${xe},${xe},${me}\\)$`),Ee=new RegExp(`^hsl\\(${me},${xe},${xe}\\)$`),Ne=new RegExp(`^hsla\\(${me},${xe},${xe},${me}\\)$`),ke={aliceblue:15792383,antiquewhite:16444375,aqua:65535,aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794160,hotpink:16738740,indianred:13458524,indigo:4915330,ivory:16777200,khaki:15787660,lavender:15132410,lavenderblush:16773365,lawngreen:8190976,lemonchiffon:16775885,lightblue:11393254,lightcoral:15761536,lightcyan:14745599,lightgoldenrodyellow:16448210,lightgray:13882323,lightgreen:9498256,lightgrey:13882323,lightpink:16758465,lightsalmon:16752762,lightseagreen:2142890,lightskyblue:8900346,lightslategray:7833753,lightslategrey:7833753,lightsteelblue:11584734,lightyellow:16777184,lime:65280,limegreen:3329330,linen:16445670,magenta:16711935,maroon:8388608,mediumaquamarine:6737322,mediumblue:205,mediumorchid:12211667,mediumpurple:9662683,mediumseagreen:3978097,mediumslateblue:8087790,mediumspringgreen:64154,mediumturquoise:4772300,mediumvioletred:13047173,midnightblue:1644912,mintcream:16121850,mistyrose:16770273,moccasin:16770229,navajowhite:16768685,navy:128,oldlace:16643558,olive:8421376,olivedrab:7048739,orange:16753920,orangered:16729344,orchid:14315734,palegoldenrod:15657130,palegreen:10025880,paleturquoise:11529966,palevioletred:14381203,papayawhip:16773077,peachpuff:16767673,peru:13468991,pink:16761035,plum:14524637,powderblue:11591910,purple:8388736,rebeccapurple:6697881,red:16711680,rosybrown:12357519,royalblue:4286945,saddlebrown:9127187,salmon:16416882,sandybrown:16032864,seagreen:3050327,seashell:16774638,sienna:10506797,silver:12632256,skyblue:8900331,slateblue:6970061,slategray:7372944,slategrey:7372944,snow:16775930,springgreen:65407,steelblue:4620980,tan:13808780,teal:32896,thistle:14204888,tomato:16737095,turquoise:4251856,violet:15631086,wheat:16113331,white:16777215,whitesmoke:16119285,yellow:16776960,yellowgreen:10145074};function Ce(){return this.rgb().formatHex()}function Pe(){return this.rgb().formatRgb()}function ze(t){var n,e;return t=(t+"").trim().toLowerCase(),(n=we.exec(t))?(e=n[1].length,n=parseInt(n[1],16),6===e?$e(n):3===e?new qe(n>>8&15|n>>4&240,n>>4&15|240&n,(15&n)<<4|15&n,1):8===e?De(n>>24&255,n>>16&255,n>>8&255,(255&n)/255):4===e?De(n>>12&15|n>>8&240,n>>8&15|n>>4&240,n>>4&15|240&n,((15&n)<<4|15&n)/255):null):(n=Me.exec(t))?new qe(n[1],n[2],n[3],1):(n=Te.exec(t))?new qe(255*n[1]/100,255*n[2]/100,255*n[3]/100,1):(n=Ae.exec(t))?De(n[1],n[2],n[3],n[4]):(n=Se.exec(t))?De(255*n[1]/100,255*n[2]/100,255*n[3]/100,n[4]):(n=Ee.exec(t))?Le(n[1],n[2]/100,n[3]/100,1):(n=Ne.exec(t))?Le(n[1],n[2]/100,n[3]/100,n[4]):ke.hasOwnProperty(t)?$e(ke[t]):"transparent"===t?new qe(NaN,NaN,NaN,0):null}function $e(t){return new qe(t>>16&255,t>>8&255,255&t,1)}function De(t,n,e,r){return r<=0&&(t=n=e=NaN),new qe(t,n,e,r)}function Re(t){return t instanceof ye||(t=ze(t)),t?new qe((t=t.rgb()).r,t.g,t.b,t.opacity):new qe}function Fe(t,n,e,r){return 1===arguments.length?Re(t):new qe(t,n,e,null==r?1:r)}function qe(t,n,e,r){this.r=+t,this.g=+n,this.b=+e,this.opacity=+r}function Ue(){return`#${Ye(this.r)}${Ye(this.g)}${Ye(this.b)}`}function Ie(){const t=Oe(this.opacity);return`${1===t?"rgb(":"rgba("}${Be(this.r)}, ${Be(this.g)}, ${Be(this.b)}${1===t?")":`, ${t})`}`}function Oe(t){return isNaN(t)?1:Math.max(0,Math.min(1,t))}function Be(t){return Math.max(0,Math.min(255,Math.round(t)||0))}function Ye(t){return((t=Be(t))<16?"0":"")+t.toString(16)}function Le(t,n,e,r){return r<=0?t=n=e=NaN:e<=0||e>=1?t=n=NaN:n<=0&&(t=NaN),new Xe(t,n,e,r)}function je(t){if(t instanceof Xe)return new Xe(t.h,t.s,t.l,t.opacity);if(t instanceof ye||(t=ze(t)),!t)return new Xe;if(t instanceof Xe)return t;var n=(t=t.rgb()).r/255,e=t.g/255,r=t.b/255,i=Math.min(n,e,r),o=Math.max(n,e,r),a=NaN,u=o-i,c=(o+i)/2;return u?(a=n===o?(e-r)/u+6*(e<r):e===o?(r-n)/u+2:(n-e)/u+4,u/=c<.5?o+i:2-o-i,a*=60):u=c>0&&c<1?0:a,new Xe(a,u,c,t.opacity)}function He(t,n,e,r){return 1===arguments.length?je(t):new Xe(t,n,e,null==r?1:r)}function Xe(t,n,e,r){this.h=+t,this.s=+n,this.l=+e,this.opacity=+r}function Ge(t){return(t=(t||0)%360)<0?t+360:t}function Ve(t){return Math.max(0,Math.min(1,t||0))}function We(t,n,e){return 255*(t<60?n+(e-n)*t/60:t<180?e:t<240?n+(e-n)*(240-t)/60:n)}pe(ye,ze,{copy(t){return Object.assign(new this.constructor,this,t)},displayable(){return this.rgb().displayable()},hex:Ce,formatHex:Ce,formatHex8:function(){return this.rgb().formatHex8()},formatHsl:function(){return je(this).formatHsl()},formatRgb:Pe,toString:Pe}),pe(qe,Fe,ge(ye,{brighter(t){return t=null==t?_e:Math.pow(_e,t),new qe(this.r*t,this.g*t,this.b*t,this.opacity)},darker(t){return t=null==t?ve:Math.pow(ve,t),new qe(this.r*t,this.g*t,this.b*t,this.opacity)},rgb(){return this},clamp(){return new qe(Be(this.r),Be(this.g),Be(this.b),Oe(this.opacity))},displayable(){return-.5<=this.r&&this.r<255.5&&-.5<=this.g&&this.g<255.5&&-.5<=this.b&&this.b<255.5&&0<=this.opacity&&this.opacity<=1},hex:Ue,formatHex:Ue,formatHex8:function(){return`#${Ye(this.r)}${Ye(this.g)}${Ye(this.b)}${Ye(255*(isNaN(this.opacity)?1:this.opacity))}`},formatRgb:Ie,toString:Ie})),pe(Xe,He,ge(ye,{brighter(t){return t=null==t?_e:Math.pow(_e,t),new Xe(this.h,this.s,this.l*t,this.opacity)},darker(t){return t=null==t?ve:Math.pow(ve,t),new Xe(this.h,this.s,this.l*t,this.opacity)},rgb(){var t=this.h%360+360*(this.h<0),n=isNaN(t)||isNaN(this.s)?0:this.s,e=this.l,r=e+(e<.5?e:1-e)*n,i=2*e-r;return new qe(We(t>=240?t-240:t+120,i,r),We(t,i,r),We(t<120?t+240:t-120,i,r),this.opacity)},clamp(){return new Xe(Ge(this.h),Ve(this.s),Ve(this.l),Oe(this.opacity))},displayable(){return(0<=this.s&&this.s<=1||isNaN(this.s))&&0<=this.l&&this.l<=1&&0<=this.opacity&&this.opacity<=1},formatHsl(){const t=Oe(this.opacity);return`${1===t?"hsl(":"hsla("}${Ge(this.h)}, ${100*Ve(this.s)}%, ${100*Ve(this.l)}%${1===t?")":`, ${t})`}`}}));const Ze=Math.PI/180,Ke=180/Math.PI,Qe=.96422,Je=1,tr=.82521,nr=4/29,er=6/29,rr=3*er*er,ir=er*er*er;function or(t){if(t instanceof ur)return new ur(t.l,t.a,t.b,t.opacity);if(t instanceof pr)return gr(t);t instanceof qe||(t=Re(t));var n,e,r=lr(t.r),i=lr(t.g),o=lr(t.b),a=cr((.2225045*r+.7168786*i+.0606169*o)/Je);return r===i&&i===o?n=e=a:(n=cr((.4360747*r+.3850649*i+.1430804*o)/Qe),e=cr((.0139322*r+.0971045*i+.7141733*o)/tr)),new ur(116*a-16,500*(n-a),200*(a-e),t.opacity)}function ar(t,n,e,r){return 1===arguments.length?or(t):new ur(t,n,e,null==r?1:r)}function ur(t,n,e,r){this.l=+t,this.a=+n,this.b=+e,this.opacity=+r}function cr(t){return t>ir?Math.pow(t,1/3):t/rr+nr}function fr(t){return t>er?t*t*t:rr*(t-nr)}function sr(t){return 255*(t<=.0031308?12.92*t:1.055*Math.pow(t,1/2.4)-.055)}function lr(t){return(t/=255)<=.04045?t/12.92:Math.pow((t+.055)/1.055,2.4)}function hr(t){if(t instanceof pr)return new pr(t.h,t.c,t.l,t.opacity);if(t instanceof ur||(t=or(t)),0===t.a&&0===t.b)return new pr(NaN,0<t.l&&t.l<100?0:NaN,t.l,t.opacity);var n=Math.atan2(t.b,t.a)*Ke;return new pr(n<0?n+360:n,Math.sqrt(t.a*t.a+t.b*t.b),t.l,t.opacity)}function dr(t,n,e,r){return 1===arguments.length?hr(t):new pr(t,n,e,null==r?1:r)}function pr(t,n,e,r){this.h=+t,this.c=+n,this.l=+e,this.opacity=+r}function gr(t){if(isNaN(t.h))return new ur(t.l,0,0,t.opacity);var n=t.h*Ze;return new ur(t.l,Math.cos(n)*t.c,Math.sin(n)*t.c,t.opacity)}pe(ur,ar,ge(ye,{brighter(t){return new ur(this.l+18*(null==t?1:t),this.a,this.b,this.opacity)},darker(t){return new ur(this.l-18*(null==t?1:t),this.a,this.b,this.opacity)},rgb(){var t=(this.l+16)/116,n=isNaN(this.a)?t:t+this.a/500,e=isNaN(this.b)?t:t-this.b/200;return new qe(sr(3.1338561*(n=Qe*fr(n))-1.6168667*(t=Je*fr(t))-.4906146*(e=tr*fr(e))),sr(-.9787684*n+1.9161415*t+.033454*e),sr(.0719453*n-.2289914*t+1.4052427*e),this.opacity)}})),pe(pr,dr,ge(ye,{brighter(t){return new pr(this.h,this.c,this.l+18*(null==t?1:t),this.opacity)},darker(t){return new pr(this.h,this.c,this.l-18*(null==t?1:t),this.opacity)},rgb(){return gr(this).rgb()}}));var yr=-.14861,vr=1.78277,_r=-.29227,br=-.90649,mr=1.97294,xr=mr*br,wr=mr*vr,Mr=vr*_r-br*yr;function Tr(t,n,e,r){return 1===arguments.length?function(t){if(t instanceof Ar)return new Ar(t.h,t.s,t.l,t.opacity);t instanceof qe||(t=Re(t));var n=t.r/255,e=t.g/255,r=t.b/255,i=(Mr*r+xr*n-wr*e)/(Mr+xr-wr),o=r-i,a=(mr*(e-i)-_r*o)/br,u=Math.sqrt(a*a+o*o)/(mr*i*(1-i)),c=u?Math.atan2(a,o)*Ke-120:NaN;return new Ar(c<0?c+360:c,u,i,t.opacity)}(t):new Ar(t,n,e,null==r?1:r)}function Ar(t,n,e,r){this.h=+t,this.s=+n,this.l=+e,this.opacity=+r}function Sr(t,n,e,r,i){var o=t*t,a=o*t;return((1-3*t+3*o-a)*n+(4-6*o+3*a)*e+(1+3*t+3*o-3*a)*r+a*i)/6}function Er(t){var n=t.length-1;return function(e){var r=e<=0?e=0:e>=1?(e=1,n-1):Math.floor(e*n),i=t[r],o=t[r+1],a=r>0?t[r-1]:2*i-o,u=r<n-1?t[r+2]:2*o-i;return Sr((e-r/n)*n,a,i,o,u)}}function Nr(t){var n=t.length;return function(e){var r=Math.floor(((e%=1)<0?++e:e)*n),i=t[(r+n-1)%n],o=t[r%n],a=t[(r+1)%n],u=t[(r+2)%n];return Sr((e-r/n)*n,i,o,a,u)}}pe(Ar,Tr,ge(ye,{brighter(t){return t=null==t?_e:Math.pow(_e,t),new Ar(this.h,this.s,this.l*t,this.opacity)},darker(t){return t=null==t?ve:Math.pow(ve,t),new Ar(this.h,this.s,this.l*t,this.opacity)},rgb(){var t=isNaN(this.h)?0:(this.h+120)*Ze,n=+this.l,e=isNaN(this.s)?0:this.s*n*(1-n),r=Math.cos(t),i=Math.sin(t);return new qe(255*(n+e*(yr*r+vr*i)),255*(n+e*(_r*r+br*i)),255*(n+e*(mr*r)),this.opacity)}}));var kr=t=>()=>t;function Cr(t,n){return function(e){return t+e*n}}function Pr(t,n){var e=n-t;return e?Cr(t,e>180||e<-180?e-360*Math.round(e/360):e):kr(isNaN(t)?n:t)}function zr(t){return 1==(t=+t)?$r:function(n,e){return e-n?function(t,n,e){return t=Math.pow(t,e),n=Math.pow(n,e)-t,e=1/e,function(r){return Math.pow(t+r*n,e)}}(n,e,t):kr(isNaN(n)?e:n)}}function $r(t,n){var e=n-t;return e?Cr(t,e):kr(isNaN(t)?n:t)}var Dr=function t(n){var e=zr(n);function r(t,n){var r=e((t=Fe(t)).r,(n=Fe(n)).r),i=e(t.g,n.g),o=e(t.b,n.b),a=$r(t.opacity,n.opacity);return function(n){return t.r=r(n),t.g=i(n),t.b=o(n),t.opacity=a(n),t+""}}return r.gamma=t,r}(1);function Rr(t){return function(n){var e,r,i=n.length,o=new Array(i),a=new Array(i),u=new Array(i);for(e=0;e<i;++e)r=Fe(n[e]),o[e]=r.r||0,a[e]=r.g||0,u[e]=r.b||0;return o=t(o),a=t(a),u=t(u),r.opacity=1,function(t){return r.r=o(t),r.g=a(t),r.b=u(t),r+""}}}var Fr=Rr(Er),qr=Rr(Nr);function Ur(t,n){n||(n=[]);var e,r=t?Math.min(n.length,t.length):0,i=n.slice();return function(o){for(e=0;e<r;++e)i[e]=t[e]*(1-o)+n[e]*o;return i}}function Ir(t){return ArrayBuffer.isView(t)&&!(t instanceof DataView)}function Or(t,n){var e,r=n?n.length:0,i=t?Math.min(r,t.length):0,o=new Array(i),a=new Array(r);for(e=0;e<i;++e)o[e]=Gr(t[e],n[e]);for(;e<r;++e)a[e]=n[e];return function(t){for(e=0;e<i;++e)a[e]=o[e](t);return a}}function Br(t,n){var e=new Date;return t=+t,n=+n,function(r){return e.setTime(t*(1-r)+n*r),e}}function Yr(t,n){return t=+t,n=+n,function(e){return t*(1-e)+n*e}}function Lr(t,n){var e,r={},i={};for(e in null!==t&&"object"==typeof t||(t={}),null!==n&&"object"==typeof n||(n={}),n)e in t?r[e]=Gr(t[e],n[e]):i[e]=n[e];return function(t){for(e in r)i[e]=r[e](t);return i}}var jr=/[-+]?(?:\d+\.?\d*|\.?\d+)(?:[eE][-+]?\d+)?/g,Hr=new RegExp(jr.source,"g");function Xr(t,n){var e,r,i,o=jr.lastIndex=Hr.lastIndex=0,a=-1,u=[],c=[];for(t+="",n+="";(e=jr.exec(t))&&(r=Hr.exec(n));)(i=r.index)>o&&(i=n.slice(o,i),u[a]?u[a]+=i:u[++a]=i),(e=e[0])===(r=r[0])?u[a]?u[a]+=r:u[++a]=r:(u[++a]=null,c.push({i:a,x:Yr(e,r)})),o=Hr.lastIndex;return o<n.length&&(i=n.slice(o),u[a]?u[a]+=i:u[++a]=i),u.length<2?c[0]?function(t){return function(n){return t(n)+""}}(c[0].x):function(t){return function(){return t}}(n):(n=c.length,function(t){for(var e,r=0;r<n;++r)u[(e=c[r]).i]=e.x(t);return u.join("")})}function Gr(t,n){var e,r=typeof n;return null==n||"boolean"===r?kr(n):("number"===r?Yr:"string"===r?(e=ze(n))?(n=e,Dr):Xr:n instanceof ze?Dr:n instanceof Date?Br:Ir(n)?Ur:Array.isArray(n)?Or:"function"!=typeof n.valueOf&&"function"!=typeof n.toString||isNaN(n)?Lr:Yr)(t,n)}function Vr(t,n){return t=+t,n=+n,function(e){return Math.round(t*(1-e)+n*e)}}var Wr,Zr=180/Math.PI,Kr={translateX:0,translateY:0,rotate:0,skewX:0,scaleX:1,scaleY:1};function Qr(t,n,e,r,i,o){var a,u,c;return(a=Math.sqrt(t*t+n*n))&&(t/=a,n/=a),(c=t*e+n*r)&&(e-=t*c,r-=n*c),(u=Math.sqrt(e*e+r*r))&&(e/=u,r/=u,c/=u),t*r<n*e&&(t=-t,n=-n,c=-c,a=-a),{translateX:i,translateY:o,rotate:Math.atan2(n,t)*Zr,skewX:Math.atan(c)*Zr,scaleX:a,scaleY:u}}function Jr(t,n,e,r){function i(t){return t.length?t.pop()+" ":""}return function(o,a){var u=[],c=[];return o=t(o),a=t(a),function(t,r,i,o,a,u){if(t!==i||r!==o){var c=a.push("translate(",null,n,null,e);u.push({i:c-4,x:Yr(t,i)},{i:c-2,x:Yr(r,o)})}else(i||o)&&a.push("translate("+i+n+o+e)}(o.translateX,o.translateY,a.translateX,a.translateY,u,c),function(t,n,e,o){t!==n?(t-n>180?n+=360:n-t>180&&(t+=360),o.push({i:e.push(i(e)+"rotate(",null,r)-2,x:Yr(t,n)})):n&&e.push(i(e)+"rotate("+n+r)}(o.rotate,a.rotate,u,c),function(t,n,e,o){t!==n?o.push({i:e.push(i(e)+"skewX(",null,r)-2,x:Yr(t,n)}):n&&e.push(i(e)+"skewX("+n+r)}(o.skewX,a.skewX,u,c),function(t,n,e,r,o,a){if(t!==e||n!==r){var u=o.push(i(o)+"scale(",null,",",null,")");a.push({i:u-4,x:Yr(t,e)},{i:u-2,x:Yr(n,r)})}else 1===e&&1===r||o.push(i(o)+"scale("+e+","+r+")")}(o.scaleX,o.scaleY,a.scaleX,a.scaleY,u,c),o=a=null,function(t){for(var n,e=-1,r=c.length;++e<r;)u[(n=c[e]).i]=n.x(t);return u.join("")}}}var ti=Jr((function(t){const n=new("function"==typeof DOMMatrix?DOMMatrix:WebKitCSSMatrix)(t+"");return n.isIdentity?Kr:Qr(n.a,n.b,n.c,n.d,n.e,n.f)}),"px, ","px)","deg)"),ni=Jr((function(t){return null==t?Kr:(Wr||(Wr=document.createElementNS("http://www.w3.org/2000/svg","g")),Wr.setAttribute("transform",t),(t=Wr.transform.baseVal.consolidate())?Qr((t=t.matrix).a,t.b,t.c,t.d,t.e,t.f):Kr)}),", ",")",")");function ei(t){return((t=Math.exp(t))+1/t)/2}var ri=function t(n,e,r){function i(t,i){var o,a,u=t[0],c=t[1],f=t[2],s=i[0],l=i[1],h=i[2],d=s-u,p=l-c,g=d*d+p*p;if(g<1e-12)a=Math.log(h/f)/n,o=function(t){return[u+t*d,c+t*p,f*Math.exp(n*t*a)]};else{var y=Math.sqrt(g),v=(h*h-f*f+r*g)/(2*f*e*y),_=(h*h-f*f-r*g)/(2*h*e*y),b=Math.log(Math.sqrt(v*v+1)-v),m=Math.log(Math.sqrt(_*_+1)-_);a=(m-b)/n,o=function(t){var r=t*a,i=ei(b),o=f/(e*y)*(i*function(t){return((t=Math.exp(2*t))-1)/(t+1)}(n*r+b)-function(t){return((t=Math.exp(t))-1/t)/2}(b));return[u+o*d,c+o*p,f*i/ei(n*r+b)]}}return o.duration=1e3*a*n/Math.SQRT2,o}return i.rho=function(n){var e=Math.max(.001,+n),r=e*e;return t(e,r,r*r)},i}(Math.SQRT2,2,4);function ii(t){return function(n,e){var r=t((n=He(n)).h,(e=He(e)).h),i=$r(n.s,e.s),o=$r(n.l,e.l),a=$r(n.opacity,e.opacity);return function(t){return n.h=r(t),n.s=i(t),n.l=o(t),n.opacity=a(t),n+""}}}var oi=ii(Pr),ai=ii($r);function ui(t){return function(n,e){var r=t((n=dr(n)).h,(e=dr(e)).h),i=$r(n.c,e.c),o=$r(n.l,e.l),a=$r(n.opacity,e.opacity);return function(t){return n.h=r(t),n.c=i(t),n.l=o(t),n.opacity=a(t),n+""}}}var ci=ui(Pr),fi=ui($r);function si(t){return function n(e){function r(n,r){var i=t((n=Tr(n)).h,(r=Tr(r)).h),o=$r(n.s,r.s),a=$r(n.l,r.l),u=$r(n.opacity,r.opacity);return function(t){return n.h=i(t),n.s=o(t),n.l=a(Math.pow(t,e)),n.opacity=u(t),n+""}}return e=+e,r.gamma=n,r}(1)}var li=si(Pr),hi=si($r);function di(t,n){void 0===n&&(n=t,t=Gr);for(var e=0,r=n.length-1,i=n[0],o=new Array(r<0?0:r);e<r;)o[e]=t(i,i=n[++e]);return function(t){var n=Math.max(0,Math.min(r-1,Math.floor(t*=r)));return o[n](t-n)}}var pi,gi,yi=0,vi=0,_i=0,bi=1e3,mi=0,xi=0,wi=0,Mi="object"==typeof performance&&performance.now?performance:Date,Ti="object"==typeof window&&window.requestAnimationFrame?window.requestAnimationFrame.bind(window):function(t){setTimeout(t,17)};function Ai(){return xi||(Ti(Si),xi=Mi.now()+wi)}function Si(){xi=0}function Ei(){this._call=this._time=this._next=null}function Ni(t,n,e){var r=new Ei;return r.restart(t,n,e),r}function ki(){Ai(),++yi;for(var t,n=pi;n;)(t=xi-n._time)>=0&&n._call.call(void 0,t),n=n._next;--yi}function Ci(){xi=(mi=Mi.now())+wi,yi=vi=0;try{ki()}finally{yi=0,function(){var t,n,e=pi,r=1/0;for(;e;)e._call?(r>e._time&&(r=e._time),t=e,e=e._next):(n=e._next,e._next=null,e=t?t._next=n:pi=n);gi=t,zi(r)}(),xi=0}}function Pi(){var t=Mi.now(),n=t-mi;n>bi&&(wi-=n,mi=t)}function zi(t){yi||(vi&&(vi=clearTimeout(vi)),t-xi>24?(t<1/0&&(vi=setTimeout(Ci,t-Mi.now()-wi)),_i&&(_i=clearInterval(_i))):(_i||(mi=Mi.now(),_i=setInterval(Pi,bi)),yi=1,Ti(Ci)))}function $i(t,n,e){var r=new Ei;return n=null==n?0:+n,r.restart((e=>{r.stop(),t(e+n)}),n,e),r}Ei.prototype=Ni.prototype={constructor:Ei,restart:function(t,n,e){if("function"!=typeof t)throw new TypeError("callback is not a function");e=(null==e?Ai():+e)+(null==n?0:+n),this._next||gi===this||(gi?gi._next=this:pi=this,gi=this),this._call=t,this._time=e,zi()},stop:function(){this._call&&(this._call=null,this._time=1/0,zi())}};var Di=$t("start","end","cancel","interrupt"),Ri=[],Fi=0,qi=1,Ui=2,Ii=3,Oi=4,Bi=5,Yi=6;function Li(t,n,e,r,i,o){var a=t.__transition;if(a){if(e in a)return}else t.__transition={};!function(t,n,e){var r,i=t.__transition;function o(t){e.state=qi,e.timer.restart(a,e.delay,e.time),e.delay<=t&&a(t-e.delay)}function a(o){var f,s,l,h;if(e.state!==qi)return c();for(f in i)if((h=i[f]).name===e.name){if(h.state===Ii)return $i(a);h.state===Oi?(h.state=Yi,h.timer.stop(),h.on.call("interrupt",t,t.__data__,h.index,h.group),delete i[f]):+f<n&&(h.state=Yi,h.timer.stop(),h.on.call("cancel",t,t.__data__,h.index,h.group),delete i[f])}if($i((function(){e.state===Ii&&(e.state=Oi,e.timer.restart(u,e.delay,e.time),u(o))})),e.state=Ui,e.on.call("start",t,t.__data__,e.index,e.group),e.state===Ui){for(e.state=Ii,r=new Array(l=e.tween.length),f=0,s=-1;f<l;++f)(h=e.tween[f].value.call(t,t.__data__,e.index,e.group))&&(r[++s]=h);r.length=s+1}}function u(n){for(var i=n<e.duration?e.ease.call(null,n/e.duration):(e.timer.restart(c),e.state=Bi,1),o=-1,a=r.length;++o<a;)r[o].call(t,i);e.state===Bi&&(e.on.call("end",t,t.__data__,e.index,e.group),c())}function c(){for(var r in e.state=Yi,e.timer.stop(),delete i[n],i)return;delete t.__transition}i[n]=e,e.timer=Ni(o,0,e.time)}(t,e,{name:n,index:r,group:i,on:Di,tween:Ri,time:o.time,delay:o.delay,duration:o.duration,ease:o.ease,timer:null,state:Fi})}function ji(t,n){var e=Xi(t,n);if(e.state>Fi)throw new Error("too late; already scheduled");return e}function Hi(t,n){var e=Xi(t,n);if(e.state>Ii)throw new Error("too late; already running");return e}function Xi(t,n){var e=t.__transition;if(!e||!(e=e[n]))throw new Error("transition not found");return e}function Gi(t,n){var e,r,i,o=t.__transition,a=!0;if(o){for(i in n=null==n?null:n+"",o)(e=o[i]).name===n?(r=e.state>Ui&&e.state<Bi,e.state=Yi,e.timer.stop(),e.on.call(r?"interrupt":"cancel",t,t.__data__,e.index,e.group),delete o[i]):a=!1;a&&delete t.__transition}}function Vi(t,n){var e,r;return function(){var i=Hi(this,t),o=i.tween;if(o!==e)for(var a=0,u=(r=e=o).length;a<u;++a)if(r[a].name===n){(r=r.slice()).splice(a,1);break}i.tween=r}}function Wi(t,n,e){var r,i;if("function"!=typeof e)throw new Error;return function(){var o=Hi(this,t),a=o.tween;if(a!==r){i=(r=a).slice();for(var u={name:n,value:e},c=0,f=i.length;c<f;++c)if(i[c].name===n){i[c]=u;break}c===f&&i.push(u)}o.tween=i}}function Zi(t,n,e){var r=t._id;return t.each((function(){var t=Hi(this,r);(t.value||(t.value={}))[n]=e.apply(this,arguments)})),function(t){return Xi(t,r).value[n]}}function Ki(t,n){var e;return("number"==typeof n?Yr:n instanceof ze?Dr:(e=ze(n))?(n=e,Dr):Xr)(t,n)}function Qi(t){return function(){this.removeAttribute(t)}}function Ji(t){return function(){this.removeAttributeNS(t.space,t.local)}}function to(t,n,e){var r,i,o=e+"";return function(){var a=this.getAttribute(t);return a===o?null:a===r?i:i=n(r=a,e)}}function no(t,n,e){var r,i,o=e+"";return function(){var a=this.getAttributeNS(t.space,t.local);return a===o?null:a===r?i:i=n(r=a,e)}}function eo(t,n,e){var r,i,o;return function(){var a,u,c=e(this);if(null!=c)return(a=this.getAttribute(t))===(u=c+"")?null:a===r&&u===i?o:(i=u,o=n(r=a,c));this.removeAttribute(t)}}function ro(t,n,e){var r,i,o;return function(){var a,u,c=e(this);if(null!=c)return(a=this.getAttributeNS(t.space,t.local))===(u=c+"")?null:a===r&&u===i?o:(i=u,o=n(r=a,c));this.removeAttributeNS(t.space,t.local)}}function io(t,n){var e,r;function i(){var i=n.apply(this,arguments);return i!==r&&(e=(r=i)&&function(t,n){return function(e){this.setAttributeNS(t.space,t.local,n.call(this,e))}}(t,i)),e}return i._value=n,i}function oo(t,n){var e,r;function i(){var i=n.apply(this,arguments);return i!==r&&(e=(r=i)&&function(t,n){return function(e){this.setAttribute(t,n.call(this,e))}}(t,i)),e}return i._value=n,i}function ao(t,n){return function(){ji(this,t).delay=+n.apply(this,arguments)}}function uo(t,n){return n=+n,function(){ji(this,t).delay=n}}function co(t,n){return function(){Hi(this,t).duration=+n.apply(this,arguments)}}function fo(t,n){return n=+n,function(){Hi(this,t).duration=n}}var so=Wn.prototype.constructor;function lo(t){return function(){this.style.removeProperty(t)}}var ho=0;function po(t,n,e,r){this._groups=t,this._parents=n,this._name=e,this._id=r}function go(t){return Wn().transition(t)}function yo(){return++ho}var vo=Wn.prototype;po.prototype=go.prototype={constructor:po,select:function(t){var n=this._name,e=this._id;"function"!=typeof t&&(t=jt(t));for(var r=this._groups,i=r.length,o=new Array(i),a=0;a<i;++a)for(var u,c,f=r[a],s=f.length,l=o[a]=new Array(s),h=0;h<s;++h)(u=f[h])&&(c=t.call(u,u.__data__,h,f))&&("__data__"in u&&(c.__data__=u.__data__),l[h]=c,Li(l[h],n,e,h,l,Xi(u,e)));return new po(o,this._parents,n,e)},selectAll:function(t){var n=this._name,e=this._id;"function"!=typeof t&&(t=Gt(t));for(var r=this._groups,i=r.length,o=[],a=[],u=0;u<i;++u)for(var c,f=r[u],s=f.length,l=0;l<s;++l)if(c=f[l]){for(var h,d=t.call(c,c.__data__,l,f),p=Xi(c,e),g=0,y=d.length;g<y;++g)(h=d[g])&&Li(h,n,e,g,d,p);o.push(d),a.push(c)}return new po(o,a,n,e)},selectChild:vo.selectChild,selectChildren:vo.selectChildren,filter:function(t){"function"!=typeof t&&(t=Vt(t));for(var n=this._groups,e=n.length,r=new Array(e),i=0;i<e;++i)for(var o,a=n[i],u=a.length,c=r[i]=[],f=0;f<u;++f)(o=a[f])&&t.call(o,o.__data__,f,a)&&c.push(o);return new po(r,this._parents,this._name,this._id)},merge:function(t){if(t._id!==this._id)throw new Error;for(var n=this._groups,e=t._groups,r=n.length,i=e.length,o=Math.min(r,i),a=new Array(r),u=0;u<o;++u)for(var c,f=n[u],s=e[u],l=f.length,h=a[u]=new Array(l),d=0;d<l;++d)(c=f[d]||s[d])&&(h[d]=c);for(;u<r;++u)a[u]=n[u];return new po(a,this._parents,this._name,this._id)},selection:function(){return new so(this._groups,this._parents)},transition:function(){for(var t=this._name,n=this._id,e=yo(),r=this._groups,i=r.length,o=0;o<i;++o)for(var a,u=r[o],c=u.length,f=0;f<c;++f)if(a=u[f]){var s=Xi(a,n);Li(a,t,e,f,u,{time:s.time+s.delay+s.duration,delay:0,duration:s.duration,ease:s.ease})}return new po(r,this._parents,t,e)},call:vo.call,nodes:vo.nodes,node:vo.node,size:vo.size,empty:vo.empty,each:vo.each,on:function(t,n){var e=this._id;return arguments.length<2?Xi(this.node(),e).on.on(t):this.each(function(t,n,e){var r,i,o=function(t){return(t+"").trim().split(/^|\s+/).every((function(t){var n=t.indexOf(".");return n>=0&&(t=t.slice(0,n)),!t||"start"===t}))}(n)?ji:Hi;return function(){var a=o(this,t),u=a.on;u!==r&&(i=(r=u).copy()).on(n,e),a.on=i}}(e,t,n))},attr:function(t,n){var e=It(t),r="transform"===e?ni:Ki;return this.attrTween(t,"function"==typeof n?(e.local?ro:eo)(e,r,Zi(this,"attr."+t,n)):null==n?(e.local?Ji:Qi)(e):(e.local?no:to)(e,r,n))},attrTween:function(t,n){var e="attr."+t;if(arguments.length<2)return(e=this.tween(e))&&e._value;if(null==n)return this.tween(e,null);if("function"!=typeof n)throw new Error;var r=It(t);return this.tween(e,(r.local?io:oo)(r,n))},style:function(t,n,e){var r="transform"==(t+="")?ti:Ki;return null==n?this.styleTween(t,function(t,n){var e,r,i;return function(){var o=_n(this,t),a=(this.style.removeProperty(t),_n(this,t));return o===a?null:o===e&&a===r?i:i=n(e=o,r=a)}}(t,r)).on("end.style."+t,lo(t)):"function"==typeof n?this.styleTween(t,function(t,n,e){var r,i,o;return function(){var a=_n(this,t),u=e(this),c=u+"";return null==u&&(this.style.removeProperty(t),c=u=_n(this,t)),a===c?null:a===r&&c===i?o:(i=c,o=n(r=a,u))}}(t,r,Zi(this,"style."+t,n))).each(function(t,n){var e,r,i,o,a="style."+n,u="end."+a;return function(){var c=Hi(this,t),f=c.on,s=null==c.value[a]?o||(o=lo(n)):void 0;f===e&&i===s||(r=(e=f).copy()).on(u,i=s),c.on=r}}(this._id,t)):this.styleTween(t,function(t,n,e){var r,i,o=e+"";return function(){var a=_n(this,t);return a===o?null:a===r?i:i=n(r=a,e)}}(t,r,n),e).on("end.style."+t,null)},styleTween:function(t,n,e){var r="style."+(t+="");if(arguments.length<2)return(r=this.tween(r))&&r._value;if(null==n)return this.tween(r,null);if("function"!=typeof n)throw new Error;return this.tween(r,function(t,n,e){var r,i;function o(){var o=n.apply(this,arguments);return o!==i&&(r=(i=o)&&function(t,n,e){return function(r){this.style.setProperty(t,n.call(this,r),e)}}(t,o,e)),r}return o._value=n,o}(t,n,null==e?"":e))},text:function(t){return this.tween("text","function"==typeof t?function(t){return function(){var n=t(this);this.textContent=null==n?"":n}}(Zi(this,"text",t)):function(t){return function(){this.textContent=t}}(null==t?"":t+""))},textTween:function(t){var n="text";if(arguments.length<1)return(n=this.tween(n))&&n._value;if(null==t)return this.tween(n,null);if("function"!=typeof t)throw new Error;return this.tween(n,function(t){var n,e;function r(){var r=t.apply(this,arguments);return r!==e&&(n=(e=r)&&function(t){return function(n){this.textContent=t.call(this,n)}}(r)),n}return r._value=t,r}(t))},remove:function(){return this.on("end.remove",function(t){return function(){var n=this.parentNode;for(var e in this.__transition)if(+e!==t)return;n&&n.removeChild(this)}}(this._id))},tween:function(t,n){var e=this._id;if(t+="",arguments.length<2){for(var r,i=Xi(this.node(),e).tween,o=0,a=i.length;o<a;++o)if((r=i[o]).name===t)return r.value;return null}return this.each((null==n?Vi:Wi)(e,t,n))},delay:function(t){var n=this._id;return arguments.length?this.each(("function"==typeof t?ao:uo)(n,t)):Xi(this.node(),n).delay},duration:function(t){var n=this._id;return arguments.length?this.each(("function"==typeof t?co:fo)(n,t)):Xi(this.node(),n).duration},ease:function(t){var n=this._id;return arguments.length?this.each(function(t,n){if("function"!=typeof n)throw new Error;return function(){Hi(this,t).ease=n}}(n,t)):Xi(this.node(),n).ease},easeVarying:function(t){if("function"!=typeof t)throw new Error;return this.each(function(t,n){return function(){var e=n.apply(this,arguments);if("function"!=typeof e)throw new Error;Hi(this,t).ease=e}}(this._id,t))},end:function(){var t,n,e=this,r=e._id,i=e.size();return new Promise((function(o,a){var u={value:a},c={value:function(){0==--i&&o()}};e.each((function(){var e=Hi(this,r),i=e.on;i!==t&&((n=(t=i).copy())._.cancel.push(u),n._.interrupt.push(u),n._.end.push(c)),e.on=n})),0===i&&o()}))},[Symbol.iterator]:vo[Symbol.iterator]};function _o(t){return((t*=2)<=1?t*t:--t*(2-t)+1)/2}function bo(t){return((t*=2)<=1?t*t*t:(t-=2)*t*t+2)/2}var mo=function t(n){function e(t){return Math.pow(t,n)}return n=+n,e.exponent=t,e}(3),xo=function t(n){function e(t){return 1-Math.pow(1-t,n)}return n=+n,e.exponent=t,e}(3),wo=function t(n){function e(t){return((t*=2)<=1?Math.pow(t,n):2-Math.pow(2-t,n))/2}return n=+n,e.exponent=t,e}(3),Mo=Math.PI,To=Mo/2;function Ao(t){return(1-Math.cos(Mo*t))/2}function So(t){return 1.0009775171065494*(Math.pow(2,-10*t)-.0009765625)}function Eo(t){return((t*=2)<=1?So(1-t):2-So(t-1))/2}function No(t){return((t*=2)<=1?1-Math.sqrt(1-t*t):Math.sqrt(1-(t-=2)*t)+1)/2}var ko=4/11,Co=6/11,Po=8/11,zo=3/4,$o=9/11,Do=10/11,Ro=15/16,Fo=21/22,qo=63/64,Uo=1/ko/ko;function Io(t){return(t=+t)<ko?Uo*t*t:t<Po?Uo*(t-=Co)*t+zo:t<Do?Uo*(t-=$o)*t+Ro:Uo*(t-=Fo)*t+qo}var Oo=1.70158,Bo=function t(n){function e(t){return(t=+t)*t*(n*(t-1)+t)}return n=+n,e.overshoot=t,e}(Oo),Yo=function t(n){function e(t){return--t*t*((t+1)*n+t)+1}return n=+n,e.overshoot=t,e}(Oo),Lo=function t(n){function e(t){return((t*=2)<1?t*t*((n+1)*t-n):(t-=2)*t*((n+1)*t+n)+2)/2}return n=+n,e.overshoot=t,e}(Oo),jo=2*Math.PI,Ho=function t(n,e){var r=Math.asin(1/(n=Math.max(1,n)))*(e/=jo);function i(t){return n*So(- --t)*Math.sin((r-t)/e)}return i.amplitude=function(n){return t(n,e*jo)},i.period=function(e){return t(n,e)},i}(1,.3),Xo=function t(n,e){var r=Math.asin(1/(n=Math.max(1,n)))*(e/=jo);function i(t){return 1-n*So(t=+t)*Math.sin((t+r)/e)}return i.amplitude=function(n){return t(n,e*jo)},i.period=function(e){return t(n,e)},i}(1,.3),Go=function t(n,e){var r=Math.asin(1/(n=Math.max(1,n)))*(e/=jo);function i(t){return((t=2*t-1)<0?n*So(-t)*Math.sin((r-t)/e):2-n*So(t)*Math.sin((r+t)/e))/2}return i.amplitude=function(n){return t(n,e*jo)},i.period=function(e){return t(n,e)},i}(1,.3),Vo={time:null,delay:0,duration:250,ease:bo};function Wo(t,n){for(var e;!(e=t.__transition)||!(e=e[n]);)if(!(t=t.parentNode))throw new Error(`transition ${n} not found`);return e}Wn.prototype.interrupt=function(t){return this.each((function(){Gi(this,t)}))},Wn.prototype.transition=function(t){var n,e;t instanceof po?(n=t._id,t=t._name):(n=yo(),(e=Vo).time=Ai(),t=null==t?null:t+"");for(var r=this._groups,i=r.length,o=0;o<i;++o)for(var a,u=r[o],c=u.length,f=0;f<c;++f)(a=u[f])&&Li(a,t,n,f,u,e||Wo(a,n));return new po(r,this._parents,t,n)};var Zo=[null];var Ko=t=>()=>t;function Qo(t,{sourceEvent:n,target:e,selection:r,mode:i,dispatch:o}){Object.defineProperties(this,{type:{value:t,enumerable:!0,configurable:!0},sourceEvent:{value:n,enumerable:!0,configurable:!0},target:{value:e,enumerable:!0,configurable:!0},selection:{value:r,enumerable:!0,configurable:!0},mode:{value:i,enumerable:!0,configurable:!0},_:{value:o}})}function Jo(t){t.preventDefault(),t.stopImmediatePropagation()}var ta={name:"drag"},na={name:"space"},ea={name:"handle"},ra={name:"center"};const{abs:ia,max:oa,min:aa}=Math;function ua(t){return[+t[0],+t[1]]}function ca(t){return[ua(t[0]),ua(t[1])]}var fa={name:"x",handles:["w","e"].map(va),input:function(t,n){return null==t?null:[[+t[0],n[0][1]],[+t[1],n[1][1]]]},output:function(t){return t&&[t[0][0],t[1][0]]}},sa={name:"y",handles:["n","s"].map(va),input:function(t,n){return null==t?null:[[n[0][0],+t[0]],[n[1][0],+t[1]]]},output:function(t){return t&&[t[0][1],t[1][1]]}},la={name:"xy",handles:["n","w","e","s","nw","ne","sw","se"].map(va),input:function(t){return null==t?null:ca(t)},output:function(t){return t}},ha={overlay:"crosshair",selection:"move",n:"ns-resize",e:"ew-resize",s:"ns-resize",w:"ew-resize",nw:"nwse-resize",ne:"nesw-resize",se:"nwse-resize",sw:"nesw-resize"},da={e:"w",w:"e",nw:"ne",ne:"nw",se:"sw",sw:"se"},pa={n:"s",s:"n",nw:"sw",ne:"se",se:"ne",sw:"nw"},ga={overlay:1,selection:1,n:null,e:1,s:null,w:-1,nw:-1,ne:1,se:1,sw:-1},ya={overlay:1,selection:1,n:-1,e:null,s:1,w:null,nw:-1,ne:-1,se:1,sw:1};function va(t){return{type:t}}function _a(t){return!t.ctrlKey&&!t.button}function ba(){var t=this.ownerSVGElement||this;return t.hasAttribute("viewBox")?[[(t=t.viewBox.baseVal).x,t.y],[t.x+t.width,t.y+t.height]]:[[0,0],[t.width.baseVal.value,t.height.baseVal.value]]}function ma(){return navigator.maxTouchPoints||"ontouchstart"in this}function xa(t){for(;!t.__brush;)if(!(t=t.parentNode))return;return t.__brush}function wa(t){var n,e=ba,r=_a,i=ma,o=!0,a=$t("start","brush","end"),u=6;function c(n){var e=n.property("__brush",g).selectAll(".overlay").data([va("overlay")]);e.enter().append("rect").attr("class","overlay").attr("pointer-events","all").attr("cursor",ha.overlay).merge(e).each((function(){var t=xa(this).extent;Zn(this).attr("x",t[0][0]).attr("y",t[0][1]).attr("width",t[1][0]-t[0][0]).attr("height",t[1][1]-t[0][1])})),n.selectAll(".selection").data([va("selection")]).enter().append("rect").attr("class","selection").attr("cursor",ha.selection).attr("fill","#777").attr("fill-opacity",.3).attr("stroke","#fff").attr("shape-rendering","crispEdges");var r=n.selectAll(".handle").data(t.handles,(function(t){return t.type}));r.exit().remove(),r.enter().append("rect").attr("class",(function(t){return"handle handle--"+t.type})).attr("cursor",(function(t){return ha[t.type]})),n.each(f).attr("fill","none").attr("pointer-events","all").on("mousedown.brush",h).filter(i).on("touchstart.brush",h).on("touchmove.brush",d).on("touchend.brush touchcancel.brush",p).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function f(){var t=Zn(this),n=xa(this).selection;n?(t.selectAll(".selection").style("display",null).attr("x",n[0][0]).attr("y",n[0][1]).attr("width",n[1][0]-n[0][0]).attr("height",n[1][1]-n[0][1]),t.selectAll(".handle").style("display",null).attr("x",(function(t){return"e"===t.type[t.type.length-1]?n[1][0]-u/2:n[0][0]-u/2})).attr("y",(function(t){return"s"===t.type[0]?n[1][1]-u/2:n[0][1]-u/2})).attr("width",(function(t){return"n"===t.type||"s"===t.type?n[1][0]-n[0][0]+u:u})).attr("height",(function(t){return"e"===t.type||"w"===t.type?n[1][1]-n[0][1]+u:u}))):t.selectAll(".selection,.handle").style("display","none").attr("x",null).attr("y",null).attr("width",null).attr("height",null)}function s(t,n,e){var r=t.__brush.emitter;return!r||e&&r.clean?new l(t,n,e):r}function l(t,n,e){this.that=t,this.args=n,this.state=t.__brush,this.active=0,this.clean=e}function h(e){if((!n||e.touches)&&r.apply(this,arguments)){var i,a,u,c,l,h,d,p,g,y,v,_=this,b=e.target.__data__.type,m="selection"===(o&&e.metaKey?b="overlay":b)?ta:o&&e.altKey?ra:ea,x=t===sa?null:ga[b],w=t===fa?null:ya[b],M=xa(_),T=M.extent,A=M.selection,S=T[0][0],E=T[0][1],N=T[1][0],k=T[1][1],C=0,P=0,z=x&&w&&o&&e.shiftKey,$=Array.from(e.touches||[e],(t=>{const n=t.identifier;return(t=ne(t,_)).point0=t.slice(),t.identifier=n,t}));Gi(_);var D=s(_,arguments,!0).beforestart();if("overlay"===b){A&&(g=!0);const n=[$[0],$[1]||$[0]];M.selection=A=[[i=t===sa?S:aa(n[0][0],n[1][0]),u=t===fa?E:aa(n[0][1],n[1][1])],[l=t===sa?N:oa(n[0][0],n[1][0]),d=t===fa?k:oa(n[0][1],n[1][1])]],$.length>1&&I(e)}else i=A[0][0],u=A[0][1],l=A[1][0],d=A[1][1];a=i,c=u,h=l,p=d;var R=Zn(_).attr("pointer-events","none"),F=R.selectAll(".overlay").attr("cursor",ha[b]);if(e.touches)D.moved=U,D.ended=O;else{var q=Zn(e.view).on("mousemove.brush",U,!0).on("mouseup.brush",O,!0);o&&q.on("keydown.brush",(function(t){switch(t.keyCode){case 16:z=x&&w;break;case 18:m===ea&&(x&&(l=h-C*x,i=a+C*x),w&&(d=p-P*w,u=c+P*w),m=ra,I(t));break;case 32:m!==ea&&m!==ra||(x<0?l=h-C:x>0&&(i=a-C),w<0?d=p-P:w>0&&(u=c-P),m=na,F.attr("cursor",ha.selection),I(t));break;default:return}Jo(t)}),!0).on("keyup.brush",(function(t){switch(t.keyCode){case 16:z&&(y=v=z=!1,I(t));break;case 18:m===ra&&(x<0?l=h:x>0&&(i=a),w<0?d=p:w>0&&(u=c),m=ea,I(t));break;case 32:m===na&&(t.altKey?(x&&(l=h-C*x,i=a+C*x),w&&(d=p-P*w,u=c+P*w),m=ra):(x<0?l=h:x>0&&(i=a),w<0?d=p:w>0&&(u=c),m=ea),F.attr("cursor",ha[b]),I(t));break;default:return}Jo(t)}),!0),ae(e.view)}f.call(_),D.start(e,m.name)}function U(t){for(const n of t.changedTouches||[t])for(const t of $)t.identifier===n.identifier&&(t.cur=ne(n,_));if(z&&!y&&!v&&1===$.length){const t=$[0];ia(t.cur[0]-t[0])>ia(t.cur[1]-t[1])?v=!0:y=!0}for(const t of $)t.cur&&(t[0]=t.cur[0],t[1]=t.cur[1]);g=!0,Jo(t),I(t)}function I(t){const n=$[0],e=n.point0;var r;switch(C=n[0]-e[0],P=n[1]-e[1],m){case na:case ta:x&&(C=oa(S-i,aa(N-l,C)),a=i+C,h=l+C),w&&(P=oa(E-u,aa(k-d,P)),c=u+P,p=d+P);break;case ea:$[1]?(x&&(a=oa(S,aa(N,$[0][0])),h=oa(S,aa(N,$[1][0])),x=1),w&&(c=oa(E,aa(k,$[0][1])),p=oa(E,aa(k,$[1][1])),w=1)):(x<0?(C=oa(S-i,aa(N-i,C)),a=i+C,h=l):x>0&&(C=oa(S-l,aa(N-l,C)),a=i,h=l+C),w<0?(P=oa(E-u,aa(k-u,P)),c=u+P,p=d):w>0&&(P=oa(E-d,aa(k-d,P)),c=u,p=d+P));break;case ra:x&&(a=oa(S,aa(N,i-C*x)),h=oa(S,aa(N,l+C*x))),w&&(c=oa(E,aa(k,u-P*w)),p=oa(E,aa(k,d+P*w)))}h<a&&(x*=-1,r=i,i=l,l=r,r=a,a=h,h=r,b in da&&F.attr("cursor",ha[b=da[b]])),p<c&&(w*=-1,r=u,u=d,d=r,r=c,c=p,p=r,b in pa&&F.attr("cursor",ha[b=pa[b]])),M.selection&&(A=M.selection),y&&(a=A[0][0],h=A[1][0]),v&&(c=A[0][1],p=A[1][1]),A[0][0]===a&&A[0][1]===c&&A[1][0]===h&&A[1][1]===p||(M.selection=[[a,c],[h,p]],f.call(_),D.brush(t,m.name))}function O(t){if(function(t){t.stopImmediatePropagation()}(t),t.touches){if(t.touches.length)return;n&&clearTimeout(n),n=setTimeout((function(){n=null}),500)}else ue(t.view,g),q.on("keydown.brush keyup.brush mousemove.brush mouseup.brush",null);R.attr("pointer-events","all"),F.attr("cursor",ha.overlay),M.selection&&(A=M.selection),function(t){return t[0][0]===t[1][0]||t[0][1]===t[1][1]}(A)&&(M.selection=null,f.call(_)),D.end(t,m.name)}}function d(t){s(this,arguments).moved(t)}function p(t){s(this,arguments).ended(t)}function g(){var n=this.__brush||{selection:null};return n.extent=ca(e.apply(this,arguments)),n.dim=t,n}return c.move=function(n,e,r){n.tween?n.on("start.brush",(function(t){s(this,arguments).beforestart().start(t)})).on("interrupt.brush end.brush",(function(t){s(this,arguments).end(t)})).tween("brush",(function(){var n=this,r=n.__brush,i=s(n,arguments),o=r.selection,a=t.input("function"==typeof e?e.apply(this,arguments):e,r.extent),u=Gr(o,a);function c(t){r.selection=1===t&&null===a?null:u(t),f.call(n),i.brush()}return null!==o&&null!==a?c:c(1)})):n.each((function(){var n=this,i=arguments,o=n.__brush,a=t.input("function"==typeof e?e.apply(n,i):e,o.extent),u=s(n,i).beforestart();Gi(n),o.selection=null===a?null:a,f.call(n),u.start(r).brush(r).end(r)}))},c.clear=function(t,n){c.move(t,null,n)},l.prototype={beforestart:function(){return 1==++this.active&&(this.state.emitter=this,this.starting=!0),this},start:function(t,n){return this.starting?(this.starting=!1,this.emit("start",t,n)):this.emit("brush",t),this},brush:function(t,n){return this.emit("brush",t,n),this},end:function(t,n){return 0==--this.active&&(delete this.state.emitter,this.emit("end",t,n)),this},emit:function(n,e,r){var i=Zn(this.that).datum();a.call(n,this.that,new Qo(n,{sourceEvent:e,target:c,selection:t.output(this.state.selection),mode:r,dispatch:a}),i)}},c.extent=function(t){return arguments.length?(e="function"==typeof t?t:Ko(ca(t)),c):e},c.filter=function(t){return arguments.length?(r="function"==typeof t?t:Ko(!!t),c):r},c.touchable=function(t){return arguments.length?(i="function"==typeof t?t:Ko(!!t),c):i},c.handleSize=function(t){return arguments.length?(u=+t,c):u},c.keyModifiers=function(t){return arguments.length?(o=!!t,c):o},c.on=function(){var t=a.on.apply(a,arguments);return t===a?c:t},c}var Ma=Math.abs,Ta=Math.cos,Aa=Math.sin,Sa=Math.PI,Ea=Sa/2,Na=2*Sa,ka=Math.max,Ca=1e-12;function Pa(t,n){return Array.from({length:n-t},((n,e)=>t+e))}function za(t,n){var e=0,r=null,i=null,o=null;function a(a){var u,c=a.length,f=new Array(c),s=Pa(0,c),l=new Array(c*c),h=new Array(c),d=0;a=Float64Array.from({length:c*c},n?(t,n)=>a[n%c][n/c|0]:(t,n)=>a[n/c|0][n%c]);for(let n=0;n<c;++n){let e=0;for(let r=0;r<c;++r)e+=a[n*c+r]+t*a[r*c+n];d+=f[n]=e}u=(d=ka(0,Na-e*c)/d)?e:Na/c;{let n=0;r&&s.sort(((t,n)=>r(f[t],f[n])));for(const e of s){const r=n;if(t){const t=Pa(1+~c,c).filter((t=>t<0?a[~t*c+e]:a[e*c+t]));i&&t.sort(((t,n)=>i(t<0?-a[~t*c+e]:a[e*c+t],n<0?-a[~n*c+e]:a[e*c+n])));for(const r of t)if(r<0){(l[~r*c+e]||(l[~r*c+e]={source:null,target:null})).target={index:e,startAngle:n,endAngle:n+=a[~r*c+e]*d,value:a[~r*c+e]}}else{(l[e*c+r]||(l[e*c+r]={source:null,target:null})).source={index:e,startAngle:n,endAngle:n+=a[e*c+r]*d,value:a[e*c+r]}}h[e]={index:e,startAngle:r,endAngle:n,value:f[e]}}else{const t=Pa(0,c).filter((t=>a[e*c+t]||a[t*c+e]));i&&t.sort(((t,n)=>i(a[e*c+t],a[e*c+n])));for(const r of t){let t;if(e<r?(t=l[e*c+r]||(l[e*c+r]={source:null,target:null}),t.source={index:e,startAngle:n,endAngle:n+=a[e*c+r]*d,value:a[e*c+r]}):(t=l[r*c+e]||(l[r*c+e]={source:null,target:null}),t.target={index:e,startAngle:n,endAngle:n+=a[e*c+r]*d,value:a[e*c+r]},e===r&&(t.source=t.target)),t.source&&t.target&&t.source.value<t.target.value){const n=t.source;t.source=t.target,t.target=n}}h[e]={index:e,startAngle:r,endAngle:n,value:f[e]}}n+=u}}return(l=Object.values(l)).groups=h,o?l.sort(o):l}return a.padAngle=function(t){return arguments.length?(e=ka(0,t),a):e},a.sortGroups=function(t){return arguments.length?(r=t,a):r},a.sortSubgroups=function(t){return arguments.length?(i=t,a):i},a.sortChords=function(t){return arguments.length?(null==t?o=null:(n=t,o=function(t,e){return n(t.source.value+t.target.value,e.source.value+e.target.value)})._=t,a):o&&o._;var n},a}const $a=Math.PI,Da=2*$a,Ra=1e-6,Fa=Da-Ra;function qa(t){this._+=t[0];for(let n=1,e=t.length;n<e;++n)this._+=arguments[n]+t[n]}let Ua=class{constructor(t){this._x0=this._y0=this._x1=this._y1=null,this._="",this._append=null==t?qa:function(t){let n=Math.floor(t);if(!(n>=0))throw new Error(`invalid digits: ${t}`);if(n>15)return qa;const e=10**n;return function(t){this._+=t[0];for(let n=1,r=t.length;n<r;++n)this._+=Math.round(arguments[n]*e)/e+t[n]}}(t)}moveTo(t,n){this._append`M${this._x0=this._x1=+t},${this._y0=this._y1=+n}`}closePath(){null!==this._x1&&(this._x1=this._x0,this._y1=this._y0,this._append`Z`)}lineTo(t,n){this._append`L${this._x1=+t},${this._y1=+n}`}quadraticCurveTo(t,n,e,r){this._append`Q${+t},${+n},${this._x1=+e},${this._y1=+r}`}bezierCurveTo(t,n,e,r,i,o){this._append`C${+t},${+n},${+e},${+r},${this._x1=+i},${this._y1=+o}`}arcTo(t,n,e,r,i){if(t=+t,n=+n,e=+e,r=+r,(i=+i)<0)throw new Error(`negative radius: ${i}`);let o=this._x1,a=this._y1,u=e-t,c=r-n,f=o-t,s=a-n,l=f*f+s*s;if(null===this._x1)this._append`M${this._x1=t},${this._y1=n}`;else if(l>Ra)if(Math.abs(s*u-c*f)>Ra&&i){let h=e-o,d=r-a,p=u*u+c*c,g=h*h+d*d,y=Math.sqrt(p),v=Math.sqrt(l),_=i*Math.tan(($a-Math.acos((p+l-g)/(2*y*v)))/2),b=_/v,m=_/y;Math.abs(b-1)>Ra&&this._append`L${t+b*f},${n+b*s}`,this._append`A${i},${i},0,0,${+(s*h>f*d)},${this._x1=t+m*u},${this._y1=n+m*c}`}else this._append`L${this._x1=t},${this._y1=n}`;else;}arc(t,n,e,r,i,o){if(t=+t,n=+n,o=!!o,(e=+e)<0)throw new Error(`negative radius: ${e}`);let a=e*Math.cos(r),u=e*Math.sin(r),c=t+a,f=n+u,s=1^o,l=o?r-i:i-r;null===this._x1?this._append`M${c},${f}`:(Math.abs(this._x1-c)>Ra||Math.abs(this._y1-f)>Ra)&&this._append`L${c},${f}`,e&&(l<0&&(l=l%Da+Da),l>Fa?this._append`A${e},${e},0,1,${s},${t-a},${n-u}A${e},${e},0,1,${s},${this._x1=c},${this._y1=f}`:l>Ra&&this._append`A${e},${e},0,${+(l>=$a)},${s},${this._x1=t+e*Math.cos(i)},${this._y1=n+e*Math.sin(i)}`)}rect(t,n,e,r){this._append`M${this._x0=this._x1=+t},${this._y0=this._y1=+n}h${e=+e}v${+r}h${-e}Z`}toString(){return this._}};function Ia(){return new Ua}Ia.prototype=Ua.prototype;var Oa=Array.prototype.slice;function Ba(t){return function(){return t}}function Ya(t){return t.source}function La(t){return t.target}function ja(t){return t.radius}function Ha(t){return t.startAngle}function Xa(t){return t.endAngle}function Ga(){return 0}function Va(){return 10}function Wa(t){var n=Ya,e=La,r=ja,i=ja,o=Ha,a=Xa,u=Ga,c=null;function f(){var f,s=n.apply(this,arguments),l=e.apply(this,arguments),h=u.apply(this,arguments)/2,d=Oa.call(arguments),p=+r.apply(this,(d[0]=s,d)),g=o.apply(this,d)-Ea,y=a.apply(this,d)-Ea,v=+i.apply(this,(d[0]=l,d)),_=o.apply(this,d)-Ea,b=a.apply(this,d)-Ea;if(c||(c=f=Ia()),h>Ca&&(Ma(y-g)>2*h+Ca?y>g?(g+=h,y-=h):(g-=h,y+=h):g=y=(g+y)/2,Ma(b-_)>2*h+Ca?b>_?(_+=h,b-=h):(_-=h,b+=h):_=b=(_+b)/2),c.moveTo(p*Ta(g),p*Aa(g)),c.arc(0,0,p,g,y),g!==_||y!==b)if(t){var m=v-+t.apply(this,arguments),x=(_+b)/2;c.quadraticCurveTo(0,0,m*Ta(_),m*Aa(_)),c.lineTo(v*Ta(x),v*Aa(x)),c.lineTo(m*Ta(b),m*Aa(b))}else c.quadraticCurveTo(0,0,v*Ta(_),v*Aa(_)),c.arc(0,0,v,_,b);if(c.quadraticCurveTo(0,0,p*Ta(g),p*Aa(g)),c.closePath(),f)return c=null,f+""||null}return t&&(f.headRadius=function(n){return arguments.length?(t="function"==typeof n?n:Ba(+n),f):t}),f.radius=function(t){return arguments.length?(r=i="function"==typeof t?t:Ba(+t),f):r},f.sourceRadius=function(t){return arguments.length?(r="function"==typeof t?t:Ba(+t),f):r},f.targetRadius=function(t){return arguments.length?(i="function"==typeof t?t:Ba(+t),f):i},f.startAngle=function(t){return arguments.length?(o="function"==typeof t?t:Ba(+t),f):o},f.endAngle=function(t){return arguments.length?(a="function"==typeof t?t:Ba(+t),f):a},f.padAngle=function(t){return arguments.length?(u="function"==typeof t?t:Ba(+t),f):u},f.source=function(t){return arguments.length?(n=t,f):n},f.target=function(t){return arguments.length?(e=t,f):e},f.context=function(t){return arguments.length?(c=null==t?null:t,f):c},f}var Za=Array.prototype.slice;function Ka(t,n){return t-n}var Qa=t=>()=>t;function Ja(t,n){for(var e,r=-1,i=n.length;++r<i;)if(e=tu(t,n[r]))return e;return 0}function tu(t,n){for(var e=n[0],r=n[1],i=-1,o=0,a=t.length,u=a-1;o<a;u=o++){var c=t[o],f=c[0],s=c[1],l=t[u],h=l[0],d=l[1];if(nu(c,l,n))return 0;s>r!=d>r&&e<(h-f)*(r-s)/(d-s)+f&&(i=-i)}return i}function nu(t,n,e){var r,i,o,a;return function(t,n,e){return(n[0]-t[0])*(e[1]-t[1])==(e[0]-t[0])*(n[1]-t[1])}(t,n,e)&&(i=t[r=+(t[0]===n[0])],o=e[r],a=n[r],i<=o&&o<=a||a<=o&&o<=i)}function eu(){}var ru=[[],[[[1,1.5],[.5,1]]],[[[1.5,1],[1,1.5]]],[[[1.5,1],[.5,1]]],[[[1,.5],[1.5,1]]],[[[1,1.5],[.5,1]],[[1,.5],[1.5,1]]],[[[1,.5],[1,1.5]]],[[[1,.5],[.5,1]]],[[[.5,1],[1,.5]]],[[[1,1.5],[1,.5]]],[[[.5,1],[1,.5]],[[1.5,1],[1,1.5]]],[[[1.5,1],[1,.5]]],[[[.5,1],[1.5,1]]],[[[1,1.5],[1.5,1]]],[[[.5,1],[1,1.5]]],[]];function iu(){var t=1,n=1,e=K,r=u;function i(t){var n=e(t);if(Array.isArray(n))n=n.slice().sort(Ka);else{const e=M(t,ou);for(n=G(...Z(e[0],e[1],n),n);n[n.length-1]>=e[1];)n.pop();for(;n[1]<e[0];)n.shift()}return n.map((n=>o(t,n)))}function o(e,i){const o=null==i?NaN:+i;if(isNaN(o))throw new Error(`invalid value: ${i}`);var u=[],c=[];return function(e,r,i){var o,u,c,f,s,l,h=new Array,d=new Array;o=u=-1,f=au(e[0],r),ru[f<<1].forEach(p);for(;++o<t-1;)c=f,f=au(e[o+1],r),ru[c|f<<1].forEach(p);ru[f<<0].forEach(p);for(;++u<n-1;){for(o=-1,f=au(e[u*t+t],r),s=au(e[u*t],r),ru[f<<1|s<<2].forEach(p);++o<t-1;)c=f,f=au(e[u*t+t+o+1],r),l=s,s=au(e[u*t+o+1],r),ru[c|f<<1|s<<2|l<<3].forEach(p);ru[f|s<<3].forEach(p)}o=-1,s=e[u*t]>=r,ru[s<<2].forEach(p);for(;++o<t-1;)l=s,s=au(e[u*t+o+1],r),ru[s<<2|l<<3].forEach(p);function p(t){var n,e,r=[t[0][0]+o,t[0][1]+u],c=[t[1][0]+o,t[1][1]+u],f=a(r),s=a(c);(n=d[f])?(e=h[s])?(delete d[n.end],delete h[e.start],n===e?(n.ring.push(c),i(n.ring)):h[n.start]=d[e.end]={start:n.start,end:e.end,ring:n.ring.concat(e.ring)}):(delete d[n.end],n.ring.push(c),d[n.end=s]=n):(n=h[s])?(e=d[f])?(delete h[n.start],delete d[e.end],n===e?(n.ring.push(c),i(n.ring)):h[e.start]=d[n.end]={start:e.start,end:n.end,ring:e.ring.concat(n.ring)}):(delete h[n.start],n.ring.unshift(r),h[n.start=f]=n):h[f]=d[s]={start:f,end:s,ring:[r,c]}}ru[s<<3].forEach(p)}(e,o,(function(t){r(t,e,o),function(t){for(var n=0,e=t.length,r=t[e-1][1]*t[0][0]-t[e-1][0]*t[0][1];++n<e;)r+=t[n-1][1]*t[n][0]-t[n-1][0]*t[n][1];return r}(t)>0?u.push([t]):c.push(t)})),c.forEach((function(t){for(var n,e=0,r=u.length;e<r;++e)if(-1!==Ja((n=u[e])[0],t))return void n.push(t)})),{type:"MultiPolygon",value:i,coordinates:u}}function a(n){return 2*n[0]+n[1]*(t+1)*4}function u(e,r,i){e.forEach((function(e){var o=e[0],a=e[1],u=0|o,c=0|a,f=uu(r[c*t+u]);o>0&&o<t&&u===o&&(e[0]=cu(o,uu(r[c*t+u-1]),f,i)),a>0&&a<n&&c===a&&(e[1]=cu(a,uu(r[(c-1)*t+u]),f,i))}))}return i.contour=o,i.size=function(e){if(!arguments.length)return[t,n];var r=Math.floor(e[0]),o=Math.floor(e[1]);if(!(r>=0&&o>=0))throw new Error("invalid size");return t=r,n=o,i},i.thresholds=function(t){return arguments.length?(e="function"==typeof t?t:Array.isArray(t)?Qa(Za.call(t)):Qa(t),i):e},i.smooth=function(t){return arguments.length?(r=t?u:eu,i):r===u},i}function ou(t){return isFinite(t)?t:NaN}function au(t,n){return null!=t&&+t>=n}function uu(t){return null==t||isNaN(t=+t)?-1/0:t}function cu(t,n,e,r){const i=r-n,o=e-n,a=isFinite(i)||isFinite(o)?i/o:Math.sign(i)/Math.sign(o);return isNaN(a)?t:t+a-.5}function fu(t){return t[0]}function su(t){return t[1]}function lu(){return 1}const hu=134217729,du=33306690738754706e-32;function pu(t,n,e,r,i){let o,a,u,c,f=n[0],s=r[0],l=0,h=0;s>f==s>-f?(o=f,f=n[++l]):(o=s,s=r[++h]);let d=0;if(l<t&&h<e)for(s>f==s>-f?(a=f+o,u=o-(a-f),f=n[++l]):(a=s+o,u=o-(a-s),s=r[++h]),o=a,0!==u&&(i[d++]=u);l<t&&h<e;)s>f==s>-f?(a=o+f,c=a-o,u=o-(a-c)+(f-c),f=n[++l]):(a=o+s,c=a-o,u=o-(a-c)+(s-c),s=r[++h]),o=a,0!==u&&(i[d++]=u);for(;l<t;)a=o+f,c=a-o,u=o-(a-c)+(f-c),f=n[++l],o=a,0!==u&&(i[d++]=u);for(;h<e;)a=o+s,c=a-o,u=o-(a-c)+(s-c),s=r[++h],o=a,0!==u&&(i[d++]=u);return 0===o&&0!==d||(i[d++]=o),d}function gu(t){return new Float64Array(t)}const yu=22204460492503146e-32,vu=11093356479670487e-47,_u=gu(4),bu=gu(8),mu=gu(12),xu=gu(16),wu=gu(4);function Mu(t,n,e,r,i,o){const a=(n-o)*(e-i),u=(t-i)*(r-o),c=a-u,f=Math.abs(a+u);return Math.abs(c)>=33306690738754716e-32*f?c:-function(t,n,e,r,i,o,a){let u,c,f,s,l,h,d,p,g,y,v,_,b,m,x,w,M,T;const A=t-i,S=e-i,E=n-o,N=r-o;m=A*N,h=hu*A,d=h-(h-A),p=A-d,h=hu*N,g=h-(h-N),y=N-g,x=p*y-(m-d*g-p*g-d*y),w=E*S,h=hu*E,d=h-(h-E),p=E-d,h=hu*S,g=h-(h-S),y=S-g,M=p*y-(w-d*g-p*g-d*y),v=x-M,l=x-v,_u[0]=x-(v+l)+(l-M),_=m+v,l=_-m,b=m-(_-l)+(v-l),v=b-w,l=b-v,_u[1]=b-(v+l)+(l-w),T=_+v,l=T-_,_u[2]=_-(T-l)+(v-l),_u[3]=T;let k=function(t,n){let e=n[0];for(let r=1;r<t;r++)e+=n[r];return e}(4,_u),C=yu*a;if(k>=C||-k>=C)return k;if(l=t-A,u=t-(A+l)+(l-i),l=e-S,f=e-(S+l)+(l-i),l=n-E,c=n-(E+l)+(l-o),l=r-N,s=r-(N+l)+(l-o),0===u&&0===c&&0===f&&0===s)return k;if(C=vu*a+du*Math.abs(k),k+=A*s+N*u-(E*f+S*c),k>=C||-k>=C)return k;m=u*N,h=hu*u,d=h-(h-u),p=u-d,h=hu*N,g=h-(h-N),y=N-g,x=p*y-(m-d*g-p*g-d*y),w=c*S,h=hu*c,d=h-(h-c),p=c-d,h=hu*S,g=h-(h-S),y=S-g,M=p*y-(w-d*g-p*g-d*y),v=x-M,l=x-v,wu[0]=x-(v+l)+(l-M),_=m+v,l=_-m,b=m-(_-l)+(v-l),v=b-w,l=b-v,wu[1]=b-(v+l)+(l-w),T=_+v,l=T-_,wu[2]=_-(T-l)+(v-l),wu[3]=T;const P=pu(4,_u,4,wu,bu);m=A*s,h=hu*A,d=h-(h-A),p=A-d,h=hu*s,g=h-(h-s),y=s-g,x=p*y-(m-d*g-p*g-d*y),w=E*f,h=hu*E,d=h-(h-E),p=E-d,h=hu*f,g=h-(h-f),y=f-g,M=p*y-(w-d*g-p*g-d*y),v=x-M,l=x-v,wu[0]=x-(v+l)+(l-M),_=m+v,l=_-m,b=m-(_-l)+(v-l),v=b-w,l=b-v,wu[1]=b-(v+l)+(l-w),T=_+v,l=T-_,wu[2]=_-(T-l)+(v-l),wu[3]=T;const z=pu(P,bu,4,wu,mu);m=u*s,h=hu*u,d=h-(h-u),p=u-d,h=hu*s,g=h-(h-s),y=s-g,x=p*y-(m-d*g-p*g-d*y),w=c*f,h=hu*c,d=h-(h-c),p=c-d,h=hu*f,g=h-(h-f),y=f-g,M=p*y-(w-d*g-p*g-d*y),v=x-M,l=x-v,wu[0]=x-(v+l)+(l-M),_=m+v,l=_-m,b=m-(_-l)+(v-l),v=b-w,l=b-v,wu[1]=b-(v+l)+(l-w),T=_+v,l=T-_,wu[2]=_-(T-l)+(v-l),wu[3]=T;const $=pu(z,mu,4,wu,xu);return xu[$-1]}(t,n,e,r,i,o,f)}const Tu=Math.pow(2,-52),Au=new Uint32Array(512);class Su{static from(t,n=zu,e=$u){const r=t.length,i=new Float64Array(2*r);for(let o=0;o<r;o++){const r=t[o];i[2*o]=n(r),i[2*o+1]=e(r)}return new Su(i)}constructor(t){const n=t.length>>1;if(n>0&&"number"!=typeof t[0])throw new Error("Expected coords to contain numbers.");this.coords=t;const e=Math.max(2*n-5,0);this._triangles=new Uint32Array(3*e),this._halfedges=new Int32Array(3*e),this._hashSize=Math.ceil(Math.sqrt(n)),this._hullPrev=new Uint32Array(n),this._hullNext=new Uint32Array(n),this._hullTri=new Uint32Array(n),this._hullHash=new Int32Array(this._hashSize).fill(-1),this._ids=new Uint32Array(n),this._dists=new Float64Array(n),this.update()}update(){const{coords:t,_hullPrev:n,_hullNext:e,_hullTri:r,_hullHash:i}=this,o=t.length>>1;let a=1/0,u=1/0,c=-1/0,f=-1/0;for(let n=0;n<o;n++){const e=t[2*n],r=t[2*n+1];e<a&&(a=e),r<u&&(u=r),e>c&&(c=e),r>f&&(f=r),this._ids[n]=n}const s=(a+c)/2,l=(u+f)/2;let h,d,p,g=1/0;for(let n=0;n<o;n++){const e=Eu(s,l,t[2*n],t[2*n+1]);e<g&&(h=n,g=e)}const y=t[2*h],v=t[2*h+1];g=1/0;for(let n=0;n<o;n++){if(n===h)continue;const e=Eu(y,v,t[2*n],t[2*n+1]);e<g&&e>0&&(d=n,g=e)}let _=t[2*d],b=t[2*d+1],m=1/0;for(let n=0;n<o;n++){if(n===h||n===d)continue;const e=ku(y,v,_,b,t[2*n],t[2*n+1]);e<m&&(p=n,m=e)}let x=t[2*p],w=t[2*p+1];if(m===1/0){for(let n=0;n<o;n++)this._dists[n]=t[2*n]-t[0]||t[2*n+1]-t[1];Cu(this._ids,this._dists,0,o-1);const n=new Uint32Array(o);let e=0;for(let t=0,r=-1/0;t<o;t++){const i=this._ids[t];this._dists[i]>r&&(n[e++]=i,r=this._dists[i])}return this.hull=n.subarray(0,e),this.triangles=new Uint32Array(0),void(this.halfedges=new Uint32Array(0))}if(Mu(y,v,_,b,x,w)<0){const t=d,n=_,e=b;d=p,_=x,b=w,p=t,x=n,w=e}const M=function(t,n,e,r,i,o){const a=e-t,u=r-n,c=i-t,f=o-n,s=a*a+u*u,l=c*c+f*f,h=.5/(a*f-u*c),d=t+(f*s-u*l)*h,p=n+(a*l-c*s)*h;return{x:d,y:p}}(y,v,_,b,x,w);this._cx=M.x,this._cy=M.y;for(let n=0;n<o;n++)this._dists[n]=Eu(t[2*n],t[2*n+1],M.x,M.y);Cu(this._ids,this._dists,0,o-1),this._hullStart=h;let T=3;e[h]=n[p]=d,e[d]=n[h]=p,e[p]=n[d]=h,r[h]=0,r[d]=1,r[p]=2,i.fill(-1),i[this._hashKey(y,v)]=h,i[this._hashKey(_,b)]=d,i[this._hashKey(x,w)]=p,this.trianglesLen=0,this._addTriangle(h,d,p,-1,-1,-1);for(let o,a,u=0;u<this._ids.length;u++){const c=this._ids[u],f=t[2*c],s=t[2*c+1];if(u>0&&Math.abs(f-o)<=Tu&&Math.abs(s-a)<=Tu)continue;if(o=f,a=s,c===h||c===d||c===p)continue;let l=0;for(let t=0,n=this._hashKey(f,s);t<this._hashSize&&(l=i[(n+t)%this._hashSize],-1===l||l===e[l]);t++);l=n[l];let g,y=l;for(;g=e[y],Mu(f,s,t[2*y],t[2*y+1],t[2*g],t[2*g+1])>=0;)if(y=g,y===l){y=-1;break}if(-1===y)continue;let v=this._addTriangle(y,c,e[y],-1,-1,r[y]);r[c]=this._legalize(v+2),r[y]=v,T++;let _=e[y];for(;g=e[_],Mu(f,s,t[2*_],t[2*_+1],t[2*g],t[2*g+1])<0;)v=this._addTriangle(_,c,g,r[c],-1,r[_]),r[c]=this._legalize(v+2),e[_]=_,T--,_=g;if(y===l)for(;g=n[y],Mu(f,s,t[2*g],t[2*g+1],t[2*y],t[2*y+1])<0;)v=this._addTriangle(g,c,y,-1,r[y],r[g]),this._legalize(v+2),r[g]=v,e[y]=y,T--,y=g;this._hullStart=n[c]=y,e[y]=n[_]=c,e[c]=_,i[this._hashKey(f,s)]=c,i[this._hashKey(t[2*y],t[2*y+1])]=y}this.hull=new Uint32Array(T);for(let t=0,n=this._hullStart;t<T;t++)this.hull[t]=n,n=e[n];this.triangles=this._triangles.subarray(0,this.trianglesLen),this.halfedges=this._halfedges.subarray(0,this.trianglesLen)}_hashKey(t,n){return Math.floor(function(t,n){const e=t/(Math.abs(t)+Math.abs(n));return(n>0?3-e:1+e)/4}(t-this._cx,n-this._cy)*this._hashSize)%this._hashSize}_legalize(t){const{_triangles:n,_halfedges:e,coords:r}=this;let i=0,o=0;for(;;){const a=e[t],u=t-t%3;if(o=u+(t+2)%3,-1===a){if(0===i)break;t=Au[--i];continue}const c=a-a%3,f=u+(t+1)%3,s=c+(a+2)%3,l=n[o],h=n[t],d=n[f],p=n[s];if(Nu(r[2*l],r[2*l+1],r[2*h],r[2*h+1],r[2*d],r[2*d+1],r[2*p],r[2*p+1])){n[t]=p,n[a]=l;const r=e[s];if(-1===r){let n=this._hullStart;do{if(this._hullTri[n]===s){this._hullTri[n]=t;break}n=this._hullPrev[n]}while(n!==this._hullStart)}this._link(t,r),this._link(a,e[o]),this._link(o,s);const u=c+(a+1)%3;i<Au.length&&(Au[i++]=u)}else{if(0===i)break;t=Au[--i]}}return o}_link(t,n){this._halfedges[t]=n,-1!==n&&(this._halfedges[n]=t)}_addTriangle(t,n,e,r,i,o){const a=this.trianglesLen;return this._triangles[a]=t,this._triangles[a+1]=n,this._triangles[a+2]=e,this._link(a,r),this._link(a+1,i),this._link(a+2,o),this.trianglesLen+=3,a}}function Eu(t,n,e,r){const i=t-e,o=n-r;return i*i+o*o}function Nu(t,n,e,r,i,o,a,u){const c=t-a,f=n-u,s=e-a,l=r-u,h=i-a,d=o-u,p=s*s+l*l,g=h*h+d*d;return c*(l*g-p*d)-f*(s*g-p*h)+(c*c+f*f)*(s*d-l*h)<0}function ku(t,n,e,r,i,o){const a=e-t,u=r-n,c=i-t,f=o-n,s=a*a+u*u,l=c*c+f*f,h=.5/(a*f-u*c),d=(f*s-u*l)*h,p=(a*l-c*s)*h;return d*d+p*p}function Cu(t,n,e,r){if(r-e<=20)for(let i=e+1;i<=r;i++){const r=t[i],o=n[r];let a=i-1;for(;a>=e&&n[t[a]]>o;)t[a+1]=t[a--];t[a+1]=r}else{let i=e+1,o=r;Pu(t,e+r>>1,i),n[t[e]]>n[t[r]]&&Pu(t,e,r),n[t[i]]>n[t[r]]&&Pu(t,i,r),n[t[e]]>n[t[i]]&&Pu(t,e,i);const a=t[i],u=n[a];for(;;){do{i++}while(n[t[i]]<u);do{o--}while(n[t[o]]>u);if(o<i)break;Pu(t,i,o)}t[e+1]=t[o],t[o]=a,r-i+1>=o-e?(Cu(t,n,i,r),Cu(t,n,e,o-1)):(Cu(t,n,e,o-1),Cu(t,n,i,r))}}function Pu(t,n,e){const r=t[n];t[n]=t[e],t[e]=r}function zu(t){return t[0]}function $u(t){return t[1]}const Du=1e-6;class Ru{constructor(){this._x0=this._y0=this._x1=this._y1=null,this._=""}moveTo(t,n){this._+=`M${this._x0=this._x1=+t},${this._y0=this._y1=+n}`}closePath(){null!==this._x1&&(this._x1=this._x0,this._y1=this._y0,this._+="Z")}lineTo(t,n){this._+=`L${this._x1=+t},${this._y1=+n}`}arc(t,n,e){const r=(t=+t)+(e=+e),i=n=+n;if(e<0)throw new Error("negative radius");null===this._x1?this._+=`M${r},${i}`:(Math.abs(this._x1-r)>Du||Math.abs(this._y1-i)>Du)&&(this._+="L"+r+","+i),e&&(this._+=`A${e},${e},0,1,1,${t-e},${n}A${e},${e},0,1,1,${this._x1=r},${this._y1=i}`)}rect(t,n,e,r){this._+=`M${this._x0=this._x1=+t},${this._y0=this._y1=+n}h${+e}v${+r}h${-e}Z`}value(){return this._||null}}class Fu{constructor(){this._=[]}moveTo(t,n){this._.push([t,n])}closePath(){this._.push(this._[0].slice())}lineTo(t,n){this._.push([t,n])}value(){return this._.length?this._:null}}class qu{constructor(t,[n,e,r,i]=[0,0,960,500]){if(!((r=+r)>=(n=+n)&&(i=+i)>=(e=+e)))throw new Error("invalid bounds");this.delaunay=t,this._circumcenters=new Float64Array(2*t.points.length),this.vectors=new Float64Array(2*t.points.length),this.xmax=r,this.xmin=n,this.ymax=i,this.ymin=e,this._init()}update(){return this.delaunay.update(),this._init(),this}_init(){const{delaunay:{points:t,hull:n,triangles:e},vectors:r}=this;let i,o;const a=this.circumcenters=this._circumcenters.subarray(0,e.length/3*2);for(let r,u,c=0,f=0,s=e.length;c<s;c+=3,f+=2){const s=2*e[c],l=2*e[c+1],h=2*e[c+2],d=t[s],p=t[s+1],g=t[l],y=t[l+1],v=t[h],_=t[h+1],b=g-d,m=y-p,x=v-d,w=_-p,M=2*(b*w-m*x);if(Math.abs(M)<1e-9){if(void 0===i){i=o=0;for(const e of n)i+=t[2*e],o+=t[2*e+1];i/=n.length,o/=n.length}const e=1e9*Math.sign((i-d)*w-(o-p)*x);r=(d+v)/2-e*w,u=(p+_)/2+e*x}else{const t=1/M,n=b*b+m*m,e=x*x+w*w;r=d+(w*n-m*e)*t,u=p+(b*e-x*n)*t}a[f]=r,a[f+1]=u}let u,c,f,s=n[n.length-1],l=4*s,h=t[2*s],d=t[2*s+1];r.fill(0);for(let e=0;e<n.length;++e)s=n[e],u=l,c=h,f=d,l=4*s,h=t[2*s],d=t[2*s+1],r[u+2]=r[l]=f-d,r[u+3]=r[l+1]=h-c}render(t){const n=null==t?t=new Ru:void 0,{delaunay:{halfedges:e,inedges:r,hull:i},circumcenters:o,vectors:a}=this;if(i.length<=1)return null;for(let n=0,r=e.length;n<r;++n){const r=e[n];if(r<n)continue;const i=2*Math.floor(n/3),a=2*Math.floor(r/3),u=o[i],c=o[i+1],f=o[a],s=o[a+1];this._renderSegment(u,c,f,s,t)}let u,c=i[i.length-1];for(let n=0;n<i.length;++n){u=c,c=i[n];const e=2*Math.floor(r[c]/3),f=o[e],s=o[e+1],l=4*u,h=this._project(f,s,a[l+2],a[l+3]);h&&this._renderSegment(f,s,h[0],h[1],t)}return n&&n.value()}renderBounds(t){const n=null==t?t=new Ru:void 0;return t.rect(this.xmin,this.ymin,this.xmax-this.xmin,this.ymax-this.ymin),n&&n.value()}renderCell(t,n){const e=null==n?n=new Ru:void 0,r=this._clip(t);if(null===r||!r.length)return;n.moveTo(r[0],r[1]);let i=r.length;for(;r[0]===r[i-2]&&r[1]===r[i-1]&&i>1;)i-=2;for(let t=2;t<i;t+=2)r[t]===r[t-2]&&r[t+1]===r[t-1]||n.lineTo(r[t],r[t+1]);return n.closePath(),e&&e.value()}*cellPolygons(){const{delaunay:{points:t}}=this;for(let n=0,e=t.length/2;n<e;++n){const t=this.cellPolygon(n);t&&(t.index=n,yield t)}}cellPolygon(t){const n=new Fu;return this.renderCell(t,n),n.value()}_renderSegment(t,n,e,r,i){let o;const a=this._regioncode(t,n),u=this._regioncode(e,r);0===a&&0===u?(i.moveTo(t,n),i.lineTo(e,r)):(o=this._clipSegment(t,n,e,r,a,u))&&(i.moveTo(o[0],o[1]),i.lineTo(o[2],o[3]))}contains(t,n,e){return(n=+n)==n&&(e=+e)==e&&this.delaunay._step(t,n,e)===t}*neighbors(t){const n=this._clip(t);if(n)for(const e of this.delaunay.neighbors(t)){const t=this._clip(e);if(t)t:for(let r=0,i=n.length;r<i;r+=2)for(let o=0,a=t.length;o<a;o+=2)if(n[r]===t[o]&&n[r+1]===t[o+1]&&n[(r+2)%i]===t[(o+a-2)%a]&&n[(r+3)%i]===t[(o+a-1)%a]){yield e;break t}}}_cell(t){const{circumcenters:n,delaunay:{inedges:e,halfedges:r,triangles:i}}=this,o=e[t];if(-1===o)return null;const a=[];let u=o;do{const e=Math.floor(u/3);if(a.push(n[2*e],n[2*e+1]),u=u%3==2?u-2:u+1,i[u]!==t)break;u=r[u]}while(u!==o&&-1!==u);return a}_clip(t){if(0===t&&1===this.delaunay.hull.length)return[this.xmax,this.ymin,this.xmax,this.ymax,this.xmin,this.ymax,this.xmin,this.ymin];const n=this._cell(t);if(null===n)return null;const{vectors:e}=this,r=4*t;return this._simplify(e[r]||e[r+1]?this._clipInfinite(t,n,e[r],e[r+1],e[r+2],e[r+3]):this._clipFinite(t,n))}_clipFinite(t,n){const e=n.length;let r,i,o,a,u=null,c=n[e-2],f=n[e-1],s=this._regioncode(c,f),l=0;for(let h=0;h<e;h+=2)if(r=c,i=f,c=n[h],f=n[h+1],o=s,s=this._regioncode(c,f),0===o&&0===s)a=l,l=0,u?u.push(c,f):u=[c,f];else{let n,e,h,d,p;if(0===o){if(null===(n=this._clipSegment(r,i,c,f,o,s)))continue;[e,h,d,p]=n}else{if(null===(n=this._clipSegment(c,f,r,i,s,o)))continue;[d,p,e,h]=n,a=l,l=this._edgecode(e,h),a&&l&&this._edge(t,a,l,u,u.length),u?u.push(e,h):u=[e,h]}a=l,l=this._edgecode(d,p),a&&l&&this._edge(t,a,l,u,u.length),u?u.push(d,p):u=[d,p]}if(u)a=l,l=this._edgecode(u[0],u[1]),a&&l&&this._edge(t,a,l,u,u.length);else if(this.contains(t,(this.xmin+this.xmax)/2,(this.ymin+this.ymax)/2))return[this.xmax,this.ymin,this.xmax,this.ymax,this.xmin,this.ymax,this.xmin,this.ymin];return u}_clipSegment(t,n,e,r,i,o){const a=i<o;for(a&&([t,n,e,r,i,o]=[e,r,t,n,o,i]);;){if(0===i&&0===o)return a?[e,r,t,n]:[t,n,e,r];if(i&o)return null;let u,c,f=i||o;8&f?(u=t+(e-t)*(this.ymax-n)/(r-n),c=this.ymax):4&f?(u=t+(e-t)*(this.ymin-n)/(r-n),c=this.ymin):2&f?(c=n+(r-n)*(this.xmax-t)/(e-t),u=this.xmax):(c=n+(r-n)*(this.xmin-t)/(e-t),u=this.xmin),i?(t=u,n=c,i=this._regioncode(t,n)):(e=u,r=c,o=this._regioncode(e,r))}}_clipInfinite(t,n,e,r,i,o){let a,u=Array.from(n);if((a=this._project(u[0],u[1],e,r))&&u.unshift(a[0],a[1]),(a=this._project(u[u.length-2],u[u.length-1],i,o))&&u.push(a[0],a[1]),u=this._clipFinite(t,u))for(let n,e=0,r=u.length,i=this._edgecode(u[r-2],u[r-1]);e<r;e+=2)n=i,i=this._edgecode(u[e],u[e+1]),n&&i&&(e=this._edge(t,n,i,u,e),r=u.length);else this.contains(t,(this.xmin+this.xmax)/2,(this.ymin+this.ymax)/2)&&(u=[this.xmin,this.ymin,this.xmax,this.ymin,this.xmax,this.ymax,this.xmin,this.ymax]);return u}_edge(t,n,e,r,i){for(;n!==e;){let e,o;switch(n){case 5:n=4;continue;case 4:n=6,e=this.xmax,o=this.ymin;break;case 6:n=2;continue;case 2:n=10,e=this.xmax,o=this.ymax;break;case 10:n=8;continue;case 8:n=9,e=this.xmin,o=this.ymax;break;case 9:n=1;continue;case 1:n=5,e=this.xmin,o=this.ymin}r[i]===e&&r[i+1]===o||!this.contains(t,e,o)||(r.splice(i,0,e,o),i+=2)}return i}_project(t,n,e,r){let i,o,a,u=1/0;if(r<0){if(n<=this.ymin)return null;(i=(this.ymin-n)/r)<u&&(a=this.ymin,o=t+(u=i)*e)}else if(r>0){if(n>=this.ymax)return null;(i=(this.ymax-n)/r)<u&&(a=this.ymax,o=t+(u=i)*e)}if(e>0){if(t>=this.xmax)return null;(i=(this.xmax-t)/e)<u&&(o=this.xmax,a=n+(u=i)*r)}else if(e<0){if(t<=this.xmin)return null;(i=(this.xmin-t)/e)<u&&(o=this.xmin,a=n+(u=i)*r)}return[o,a]}_edgecode(t,n){return(t===this.xmin?1:t===this.xmax?2:0)|(n===this.ymin?4:n===this.ymax?8:0)}_regioncode(t,n){return(t<this.xmin?1:t>this.xmax?2:0)|(n<this.ymin?4:n>this.ymax?8:0)}_simplify(t){if(t&&t.length>4){for(let n=0;n<t.length;n+=2){const e=(n+2)%t.length,r=(n+4)%t.length;(t[n]===t[e]&&t[e]===t[r]||t[n+1]===t[e+1]&&t[e+1]===t[r+1])&&(t.splice(e,2),n-=2)}t.length||(t=null)}return t}}const Uu=2*Math.PI,Iu=Math.pow;function Ou(t){return t[0]}function Bu(t){return t[1]}function Yu(t,n,e){return[t+Math.sin(t+n)*e,n+Math.cos(t-n)*e]}class Lu{static from(t,n=Ou,e=Bu,r){return new Lu("length"in t?function(t,n,e,r){const i=t.length,o=new Float64Array(2*i);for(let a=0;a<i;++a){const i=t[a];o[2*a]=n.call(r,i,a,t),o[2*a+1]=e.call(r,i,a,t)}return o}(t,n,e,r):Float64Array.from(function*(t,n,e,r){let i=0;for(const o of t)yield n.call(r,o,i,t),yield e.call(r,o,i,t),++i}(t,n,e,r)))}constructor(t){this._delaunator=new Su(t),this.inedges=new Int32Array(t.length/2),this._hullIndex=new Int32Array(t.length/2),this.points=this._delaunator.coords,this._init()}update(){return this._delaunator.update(),this._init(),this}_init(){const t=this._delaunator,n=this.points;if(t.hull&&t.hull.length>2&&function(t){const{triangles:n,coords:e}=t;for(let t=0;t<n.length;t+=3){const r=2*n[t],i=2*n[t+1],o=2*n[t+2];if((e[o]-e[r])*(e[i+1]-e[r+1])-(e[i]-e[r])*(e[o+1]-e[r+1])>1e-10)return!1}return!0}(t)){this.collinear=Int32Array.from({length:n.length/2},((t,n)=>n)).sort(((t,e)=>n[2*t]-n[2*e]||n[2*t+1]-n[2*e+1]));const t=this.collinear[0],e=this.collinear[this.collinear.length-1],r=[n[2*t],n[2*t+1],n[2*e],n[2*e+1]],i=1e-8*Math.hypot(r[3]-r[1],r[2]-r[0]);for(let t=0,e=n.length/2;t<e;++t){const e=Yu(n[2*t],n[2*t+1],i);n[2*t]=e[0],n[2*t+1]=e[1]}this._delaunator=new Su(n)}else delete this.collinear;const e=this.halfedges=this._delaunator.halfedges,r=this.hull=this._delaunator.hull,i=this.triangles=this._delaunator.triangles,o=this.inedges.fill(-1),a=this._hullIndex.fill(-1);for(let t=0,n=e.length;t<n;++t){const n=i[t%3==2?t-2:t+1];-1!==e[t]&&-1!==o[n]||(o[n]=t)}for(let t=0,n=r.length;t<n;++t)a[r[t]]=t;r.length<=2&&r.length>0&&(this.triangles=new Int32Array(3).fill(-1),this.halfedges=new Int32Array(3).fill(-1),this.triangles[0]=r[0],o[r[0]]=1,2===r.length&&(o[r[1]]=0,this.triangles[1]=r[1],this.triangles[2]=r[1]))}voronoi(t){return new qu(this,t)}*neighbors(t){const{inedges:n,hull:e,_hullIndex:r,halfedges:i,triangles:o,collinear:a}=this;if(a){const n=a.indexOf(t);return n>0&&(yield a[n-1]),void(n<a.length-1&&(yield a[n+1]))}const u=n[t];if(-1===u)return;let c=u,f=-1;do{if(yield f=o[c],c=c%3==2?c-2:c+1,o[c]!==t)return;if(c=i[c],-1===c){const n=e[(r[t]+1)%e.length];return void(n!==f&&(yield n))}}while(c!==u)}find(t,n,e=0){if((t=+t)!=t||(n=+n)!=n)return-1;const r=e;let i;for(;(i=this._step(e,t,n))>=0&&i!==e&&i!==r;)e=i;return i}_step(t,n,e){const{inedges:r,hull:i,_hullIndex:o,halfedges:a,triangles:u,points:c}=this;if(-1===r[t]||!c.length)return(t+1)%(c.length>>1);let f=t,s=Iu(n-c[2*t],2)+Iu(e-c[2*t+1],2);const l=r[t];let h=l;do{let r=u[h];const l=Iu(n-c[2*r],2)+Iu(e-c[2*r+1],2);if(l<s&&(s=l,f=r),h=h%3==2?h-2:h+1,u[h]!==t)break;if(h=a[h],-1===h){if(h=i[(o[t]+1)%i.length],h!==r&&Iu(n-c[2*h],2)+Iu(e-c[2*h+1],2)<s)return h;break}}while(h!==l);return f}render(t){const n=null==t?t=new Ru:void 0,{points:e,halfedges:r,triangles:i}=this;for(let n=0,o=r.length;n<o;++n){const o=r[n];if(o<n)continue;const a=2*i[n],u=2*i[o];t.moveTo(e[a],e[a+1]),t.lineTo(e[u],e[u+1])}return this.renderHull(t),n&&n.value()}renderPoints(t,n){void 0!==n||t&&"function"==typeof t.moveTo||(n=t,t=null),n=null==n?2:+n;const e=null==t?t=new Ru:void 0,{points:r}=this;for(let e=0,i=r.length;e<i;e+=2){const i=r[e],o=r[e+1];t.moveTo(i+n,o),t.arc(i,o,n,0,Uu)}return e&&e.value()}renderHull(t){const n=null==t?t=new Ru:void 0,{hull:e,points:r}=this,i=2*e[0],o=e.length;t.moveTo(r[i],r[i+1]);for(let n=1;n<o;++n){const i=2*e[n];t.lineTo(r[i],r[i+1])}return t.closePath(),n&&n.value()}hullPolygon(){const t=new Fu;return this.renderHull(t),t.value()}renderTriangle(t,n){const e=null==n?n=new Ru:void 0,{points:r,triangles:i}=this,o=2*i[t*=3],a=2*i[t+1],u=2*i[t+2];return n.moveTo(r[o],r[o+1]),n.lineTo(r[a],r[a+1]),n.lineTo(r[u],r[u+1]),n.closePath(),e&&e.value()}*trianglePolygons(){const{triangles:t}=this;for(let n=0,e=t.length/3;n<e;++n)yield this.trianglePolygon(n)}trianglePolygon(t){const n=new Fu;return this.renderTriangle(t,n),n.value()}}var ju={},Hu={},Xu=34,Gu=10,Vu=13;function Wu(t){return new Function("d","return {"+t.map((function(t,n){return JSON.stringify(t)+": d["+n+'] || ""'})).join(",")+"}")}function Zu(t){var n=Object.create(null),e=[];return t.forEach((function(t){for(var r in t)r in n||e.push(n[r]=r)})),e}function Ku(t,n){var e=t+"",r=e.length;return r<n?new Array(n-r+1).join(0)+e:e}function Qu(t){var n,e=t.getUTCHours(),r=t.getUTCMinutes(),i=t.getUTCSeconds(),o=t.getUTCMilliseconds();return isNaN(t)?"Invalid Date":((n=t.getUTCFullYear())<0?"-"+Ku(-n,6):n>9999?"+"+Ku(n,6):Ku(n,4))+"-"+Ku(t.getUTCMonth()+1,2)+"-"+Ku(t.getUTCDate(),2)+(o?"T"+Ku(e,2)+":"+Ku(r,2)+":"+Ku(i,2)+"."+Ku(o,3)+"Z":i?"T"+Ku(e,2)+":"+Ku(r,2)+":"+Ku(i,2)+"Z":r||e?"T"+Ku(e,2)+":"+Ku(r,2)+"Z":"")}function Ju(t){var n=new RegExp('["'+t+"\n\r]"),e=t.charCodeAt(0);function r(t,n){var r,i=[],o=t.length,a=0,u=0,c=o<=0,f=!1;function s(){if(c)return Hu;if(f)return f=!1,ju;var n,r,i=a;if(t.charCodeAt(i)===Xu){for(;a++<o&&t.charCodeAt(a)!==Xu||t.charCodeAt(++a)===Xu;);return(n=a)>=o?c=!0:(r=t.charCodeAt(a++))===Gu?f=!0:r===Vu&&(f=!0,t.charCodeAt(a)===Gu&&++a),t.slice(i+1,n-1).replace(/""/g,'"')}for(;a<o;){if((r=t.charCodeAt(n=a++))===Gu)f=!0;else if(r===Vu)f=!0,t.charCodeAt(a)===Gu&&++a;else if(r!==e)continue;return t.slice(i,n)}return c=!0,t.slice(i,o)}for(t.charCodeAt(o-1)===Gu&&--o,t.charCodeAt(o-1)===Vu&&--o;(r=s())!==Hu;){for(var l=[];r!==ju&&r!==Hu;)l.push(r),r=s();n&&null==(l=n(l,u++))||i.push(l)}return i}function i(n,e){return n.map((function(n){return e.map((function(t){return a(n[t])})).join(t)}))}function o(n){return n.map(a).join(t)}function a(t){return null==t?"":t instanceof Date?Qu(t):n.test(t+="")?'"'+t.replace(/"/g,'""')+'"':t}return{parse:function(t,n){var e,i,o=r(t,(function(t,r){if(e)return e(t,r-1);i=t,e=n?function(t,n){var e=Wu(t);return function(r,i){return n(e(r),i,t)}}(t,n):Wu(t)}));return o.columns=i||[],o},parseRows:r,format:function(n,e){return null==e&&(e=Zu(n)),[e.map(a).join(t)].concat(i(n,e)).join("\n")},formatBody:function(t,n){return null==n&&(n=Zu(t)),i(t,n).join("\n")},formatRows:function(t){return t.map(o).join("\n")},formatRow:o,formatValue:a}}var tc=Ju(","),nc=tc.parse,ec=tc.parseRows,rc=tc.format,ic=tc.formatBody,oc=tc.formatRows,ac=tc.formatRow,uc=tc.formatValue,cc=Ju("\t"),fc=cc.parse,sc=cc.parseRows,lc=cc.format,hc=cc.formatBody,dc=cc.formatRows,pc=cc.formatRow,gc=cc.formatValue;const yc=new Date("2019-01-01T00:00").getHours()||new Date("2019-07-01T00:00").getHours();function vc(t){if(!t.ok)throw new Error(t.status+" "+t.statusText);return t.blob()}function _c(t){if(!t.ok)throw new Error(t.status+" "+t.statusText);return t.arrayBuffer()}function bc(t){if(!t.ok)throw new Error(t.status+" "+t.statusText);return t.text()}function mc(t,n){return fetch(t,n).then(bc)}function xc(t){return function(n,e,r){return 2===arguments.length&&"function"==typeof e&&(r=e,e=void 0),mc(n,e).then((function(n){return t(n,r)}))}}var wc=xc(nc),Mc=xc(fc);function Tc(t){if(!t.ok)throw new Error(t.status+" "+t.statusText);if(204!==t.status&&205!==t.status)return t.json()}function Ac(t){return(n,e)=>mc(n,e).then((n=>(new DOMParser).parseFromString(n,t)))}var Sc=Ac("application/xml"),Ec=Ac("text/html"),Nc=Ac("image/svg+xml");function kc(t,n,e,r){if(isNaN(n)||isNaN(e))return t;var i,o,a,u,c,f,s,l,h,d=t._root,p={data:r},g=t._x0,y=t._y0,v=t._x1,_=t._y1;if(!d)return t._root=p,t;for(;d.length;)if((f=n>=(o=(g+v)/2))?g=o:v=o,(s=e>=(a=(y+_)/2))?y=a:_=a,i=d,!(d=d[l=s<<1|f]))return i[l]=p,t;if(u=+t._x.call(null,d.data),c=+t._y.call(null,d.data),n===u&&e===c)return p.next=d,i?i[l]=p:t._root=p,t;do{i=i?i[l]=new Array(4):t._root=new Array(4),(f=n>=(o=(g+v)/2))?g=o:v=o,(s=e>=(a=(y+_)/2))?y=a:_=a}while((l=s<<1|f)==(h=(c>=a)<<1|u>=o));return i[h]=d,i[l]=p,t}function Cc(t,n,e,r,i){this.node=t,this.x0=n,this.y0=e,this.x1=r,this.y1=i}function Pc(t){return t[0]}function zc(t){return t[1]}function $c(t,n,e){var r=new Dc(null==n?Pc:n,null==e?zc:e,NaN,NaN,NaN,NaN);return null==t?r:r.addAll(t)}function Dc(t,n,e,r,i,o){this._x=t,this._y=n,this._x0=e,this._y0=r,this._x1=i,this._y1=o,this._root=void 0}function Rc(t){for(var n={data:t.data},e=n;t=t.next;)e=e.next={data:t.data};return n}var Fc=$c.prototype=Dc.prototype;function qc(t){return function(){return t}}function Uc(t){return 1e-6*(t()-.5)}function Ic(t){return t.x+t.vx}function Oc(t){return t.y+t.vy}function Bc(t){return t.index}function Yc(t,n){var e=t.get(n);if(!e)throw new Error("node not found: "+n);return e}Fc.copy=function(){var t,n,e=new Dc(this._x,this._y,this._x0,this._y0,this._x1,this._y1),r=this._root;if(!r)return e;if(!r.length)return e._root=Rc(r),e;for(t=[{source:r,target:e._root=new Array(4)}];r=t.pop();)for(var i=0;i<4;++i)(n=r.source[i])&&(n.length?t.push({source:n,target:r.target[i]=new Array(4)}):r.target[i]=Rc(n));return e},Fc.add=function(t){const n=+this._x.call(null,t),e=+this._y.call(null,t);return kc(this.cover(n,e),n,e,t)},Fc.addAll=function(t){var n,e,r,i,o=t.length,a=new Array(o),u=new Array(o),c=1/0,f=1/0,s=-1/0,l=-1/0;for(e=0;e<o;++e)isNaN(r=+this._x.call(null,n=t[e]))||isNaN(i=+this._y.call(null,n))||(a[e]=r,u[e]=i,r<c&&(c=r),r>s&&(s=r),i<f&&(f=i),i>l&&(l=i));if(c>s||f>l)return this;for(this.cover(c,f).cover(s,l),e=0;e<o;++e)kc(this,a[e],u[e],t[e]);return this},Fc.cover=function(t,n){if(isNaN(t=+t)||isNaN(n=+n))return this;var e=this._x0,r=this._y0,i=this._x1,o=this._y1;if(isNaN(e))i=(e=Math.floor(t))+1,o=(r=Math.floor(n))+1;else{for(var a,u,c=i-e||1,f=this._root;e>t||t>=i||r>n||n>=o;)switch(u=(n<r)<<1|t<e,(a=new Array(4))[u]=f,f=a,c*=2,u){case 0:i=e+c,o=r+c;break;case 1:e=i-c,o=r+c;break;case 2:i=e+c,r=o-c;break;case 3:e=i-c,r=o-c}this._root&&this._root.length&&(this._root=f)}return this._x0=e,this._y0=r,this._x1=i,this._y1=o,this},Fc.data=function(){var t=[];return this.visit((function(n){if(!n.length)do{t.push(n.data)}while(n=n.next)})),t},Fc.extent=function(t){return arguments.length?this.cover(+t[0][0],+t[0][1]).cover(+t[1][0],+t[1][1]):isNaN(this._x0)?void 0:[[this._x0,this._y0],[this._x1,this._y1]]},Fc.find=function(t,n,e){var r,i,o,a,u,c,f,s=this._x0,l=this._y0,h=this._x1,d=this._y1,p=[],g=this._root;for(g&&p.push(new Cc(g,s,l,h,d)),null==e?e=1/0:(s=t-e,l=n-e,h=t+e,d=n+e,e*=e);c=p.pop();)if(!(!(g=c.node)||(i=c.x0)>h||(o=c.y0)>d||(a=c.x1)<s||(u=c.y1)<l))if(g.length){var y=(i+a)/2,v=(o+u)/2;p.push(new Cc(g[3],y,v,a,u),new Cc(g[2],i,v,y,u),new Cc(g[1],y,o,a,v),new Cc(g[0],i,o,y,v)),(f=(n>=v)<<1|t>=y)&&(c=p[p.length-1],p[p.length-1]=p[p.length-1-f],p[p.length-1-f]=c)}else{var _=t-+this._x.call(null,g.data),b=n-+this._y.call(null,g.data),m=_*_+b*b;if(m<e){var x=Math.sqrt(e=m);s=t-x,l=n-x,h=t+x,d=n+x,r=g.data}}return r},Fc.remove=function(t){if(isNaN(o=+this._x.call(null,t))||isNaN(a=+this._y.call(null,t)))return this;var n,e,r,i,o,a,u,c,f,s,l,h,d=this._root,p=this._x0,g=this._y0,y=this._x1,v=this._y1;if(!d)return this;if(d.length)for(;;){if((f=o>=(u=(p+y)/2))?p=u:y=u,(s=a>=(c=(g+v)/2))?g=c:v=c,n=d,!(d=d[l=s<<1|f]))return this;if(!d.length)break;(n[l+1&3]||n[l+2&3]||n[l+3&3])&&(e=n,h=l)}for(;d.data!==t;)if(r=d,!(d=d.next))return this;return(i=d.next)&&delete d.next,r?(i?r.next=i:delete r.next,this):n?(i?n[l]=i:delete n[l],(d=n[0]||n[1]||n[2]||n[3])&&d===(n[3]||n[2]||n[1]||n[0])&&!d.length&&(e?e[h]=d:this._root=d),this):(this._root=i,this)},Fc.removeAll=function(t){for(var n=0,e=t.length;n<e;++n)this.remove(t[n]);return this},Fc.root=function(){return this._root},Fc.size=function(){var t=0;return this.visit((function(n){if(!n.length)do{++t}while(n=n.next)})),t},Fc.visit=function(t){var n,e,r,i,o,a,u=[],c=this._root;for(c&&u.push(new Cc(c,this._x0,this._y0,this._x1,this._y1));n=u.pop();)if(!t(c=n.node,r=n.x0,i=n.y0,o=n.x1,a=n.y1)&&c.length){var f=(r+o)/2,s=(i+a)/2;(e=c[3])&&u.push(new Cc(e,f,s,o,a)),(e=c[2])&&u.push(new Cc(e,r,s,f,a)),(e=c[1])&&u.push(new Cc(e,f,i,o,s)),(e=c[0])&&u.push(new Cc(e,r,i,f,s))}return this},Fc.visitAfter=function(t){var n,e=[],r=[];for(this._root&&e.push(new Cc(this._root,this._x0,this._y0,this._x1,this._y1));n=e.pop();){var i=n.node;if(i.length){var o,a=n.x0,u=n.y0,c=n.x1,f=n.y1,s=(a+c)/2,l=(u+f)/2;(o=i[0])&&e.push(new Cc(o,a,u,s,l)),(o=i[1])&&e.push(new Cc(o,s,u,c,l)),(o=i[2])&&e.push(new Cc(o,a,l,s,f)),(o=i[3])&&e.push(new Cc(o,s,l,c,f))}r.push(n)}for(;n=r.pop();)t(n.node,n.x0,n.y0,n.x1,n.y1);return this},Fc.x=function(t){return arguments.length?(this._x=t,this):this._x},Fc.y=function(t){return arguments.length?(this._y=t,this):this._y};const Lc=1664525,jc=1013904223,Hc=4294967296;function Xc(t){return t.x}function Gc(t){return t.y}var Vc=Math.PI*(3-Math.sqrt(5));function Wc(t,n){if((e=(t=n?t.toExponential(n-1):t.toExponential()).indexOf("e"))<0)return null;var e,r=t.slice(0,e);return[r.length>1?r[0]+r.slice(2):r,+t.slice(e+1)]}function Zc(t){return(t=Wc(Math.abs(t)))?t[1]:NaN}var Kc,Qc=/^(?:(.)?([<>=^]))?([+\-( ])?([$#])?(0)?(\d+)?(,)?(\.\d+)?(~)?([a-z%])?$/i;function Jc(t){if(!(n=Qc.exec(t)))throw new Error("invalid format: "+t);var n;return new tf({fill:n[1],align:n[2],sign:n[3],symbol:n[4],zero:n[5],width:n[6],comma:n[7],precision:n[8]&&n[8].slice(1),trim:n[9],type:n[10]})}function tf(t){this.fill=void 0===t.fill?" ":t.fill+"",this.align=void 0===t.align?">":t.align+"",this.sign=void 0===t.sign?"-":t.sign+"",this.symbol=void 0===t.symbol?"":t.symbol+"",this.zero=!!t.zero,this.width=void 0===t.width?void 0:+t.width,this.comma=!!t.comma,this.precision=void 0===t.precision?void 0:+t.precision,this.trim=!!t.trim,this.type=void 0===t.type?"":t.type+""}function nf(t,n){var e=Wc(t,n);if(!e)return t+"";var r=e[0],i=e[1];return i<0?"0."+new Array(-i).join("0")+r:r.length>i+1?r.slice(0,i+1)+"."+r.slice(i+1):r+new Array(i-r.length+2).join("0")}Jc.prototype=tf.prototype,tf.prototype.toString=function(){return this.fill+this.align+this.sign+this.symbol+(this.zero?"0":"")+(void 0===this.width?"":Math.max(1,0|this.width))+(this.comma?",":"")+(void 0===this.precision?"":"."+Math.max(0,0|this.precision))+(this.trim?"~":"")+this.type};var ef={"%":(t,n)=>(100*t).toFixed(n),b:t=>Math.round(t).toString(2),c:t=>t+"",d:function(t){return Math.abs(t=Math.round(t))>=1e21?t.toLocaleString("en").replace(/,/g,""):t.toString(10)},e:(t,n)=>t.toExponential(n),f:(t,n)=>t.toFixed(n),g:(t,n)=>t.toPrecision(n),o:t=>Math.round(t).toString(8),p:(t,n)=>nf(100*t,n),r:nf,s:function(t,n){var e=Wc(t,n);if(!e)return t+"";var r=e[0],i=e[1],o=i-(Kc=3*Math.max(-8,Math.min(8,Math.floor(i/3))))+1,a=r.length;return o===a?r:o>a?r+new Array(o-a+1).join("0"):o>0?r.slice(0,o)+"."+r.slice(o):"0."+new Array(1-o).join("0")+Wc(t,Math.max(0,n+o-1))[0]},X:t=>Math.round(t).toString(16).toUpperCase(),x:t=>Math.round(t).toString(16)};function rf(t){return t}var of,af=Array.prototype.map,uf=["y","z","a","f","p","n","µ","m","","k","M","G","T","P","E","Z","Y"];function cf(t){var n,e,r=void 0===t.grouping||void 0===t.thousands?rf:(n=af.call(t.grouping,Number),e=t.thousands+"",function(t,r){for(var i=t.length,o=[],a=0,u=n[0],c=0;i>0&&u>0&&(c+u+1>r&&(u=Math.max(1,r-c)),o.push(t.substring(i-=u,i+u)),!((c+=u+1)>r));)u=n[a=(a+1)%n.length];return o.reverse().join(e)}),i=void 0===t.currency?"":t.currency[0]+"",o=void 0===t.currency?"":t.currency[1]+"",a=void 0===t.decimal?".":t.decimal+"",u=void 0===t.numerals?rf:function(t){return function(n){return n.replace(/[0-9]/g,(function(n){return t[+n]}))}}(af.call(t.numerals,String)),c=void 0===t.percent?"%":t.percent+"",f=void 0===t.minus?"−":t.minus+"",s=void 0===t.nan?"NaN":t.nan+"";function l(t){var n=(t=Jc(t)).fill,e=t.align,l=t.sign,h=t.symbol,d=t.zero,p=t.width,g=t.comma,y=t.precision,v=t.trim,_=t.type;"n"===_?(g=!0,_="g"):ef[_]||(void 0===y&&(y=12),v=!0,_="g"),(d||"0"===n&&"="===e)&&(d=!0,n="0",e="=");var b="$"===h?i:"#"===h&&/[boxX]/.test(_)?"0"+_.toLowerCase():"",m="$"===h?o:/[%p]/.test(_)?c:"",x=ef[_],w=/[defgprs%]/.test(_);function M(t){var i,o,c,h=b,M=m;if("c"===_)M=x(t)+M,t="";else{var T=(t=+t)<0||1/t<0;if(t=isNaN(t)?s:x(Math.abs(t),y),v&&(t=function(t){t:for(var n,e=t.length,r=1,i=-1;r<e;++r)switch(t[r]){case".":i=n=r;break;case"0":0===i&&(i=r),n=r;break;default:if(!+t[r])break t;i>0&&(i=0)}return i>0?t.slice(0,i)+t.slice(n+1):t}(t)),T&&0==+t&&"+"!==l&&(T=!1),h=(T?"("===l?l:f:"-"===l||"("===l?"":l)+h,M=("s"===_?uf[8+Kc/3]:"")+M+(T&&"("===l?")":""),w)for(i=-1,o=t.length;++i<o;)if(48>(c=t.charCodeAt(i))||c>57){M=(46===c?a+t.slice(i+1):t.slice(i))+M,t=t.slice(0,i);break}}g&&!d&&(t=r(t,1/0));var A=h.length+t.length+M.length,S=A<p?new Array(p-A+1).join(n):"";switch(g&&d&&(t=r(S+t,S.length?p-M.length:1/0),S=""),e){case"<":t=h+t+M+S;break;case"=":t=h+S+t+M;break;case"^":t=S.slice(0,A=S.length>>1)+h+t+M+S.slice(A);break;default:t=S+h+t+M}return u(t)}return y=void 0===y?6:/[gprs]/.test(_)?Math.max(1,Math.min(21,y)):Math.max(0,Math.min(20,y)),M.toString=function(){return t+""},M}return{format:l,formatPrefix:function(t,n){var e=l(((t=Jc(t)).type="f",t)),r=3*Math.max(-8,Math.min(8,Math.floor(Zc(n)/3))),i=Math.pow(10,-r),o=uf[8+r/3];return function(t){return e(i*t)+o}}}}function ff(n){return of=cf(n),t.format=of.format,t.formatPrefix=of.formatPrefix,of}function sf(t){return Math.max(0,-Zc(Math.abs(t)))}function lf(t,n){return Math.max(0,3*Math.max(-8,Math.min(8,Math.floor(Zc(n)/3)))-Zc(Math.abs(t)))}function hf(t,n){return t=Math.abs(t),n=Math.abs(n)-t,Math.max(0,Zc(n)-Zc(t))+1}t.format=void 0,t.formatPrefix=void 0,ff({thousands:",",grouping:[3],currency:["$",""]});var df=1e-6,pf=1e-12,gf=Math.PI,yf=gf/2,vf=gf/4,_f=2*gf,bf=180/gf,mf=gf/180,xf=Math.abs,wf=Math.atan,Mf=Math.atan2,Tf=Math.cos,Af=Math.ceil,Sf=Math.exp,Ef=Math.hypot,Nf=Math.log,kf=Math.pow,Cf=Math.sin,Pf=Math.sign||function(t){return t>0?1:t<0?-1:0},zf=Math.sqrt,$f=Math.tan;function Df(t){return t>1?0:t<-1?gf:Math.acos(t)}function Rf(t){return t>1?yf:t<-1?-yf:Math.asin(t)}function Ff(t){return(t=Cf(t/2))*t}function qf(){}function Uf(t,n){t&&Of.hasOwnProperty(t.type)&&Of[t.type](t,n)}var If={Feature:function(t,n){Uf(t.geometry,n)},FeatureCollection:function(t,n){for(var e=t.features,r=-1,i=e.length;++r<i;)Uf(e[r].geometry,n)}},Of={Sphere:function(t,n){n.sphere()},Point:function(t,n){t=t.coordinates,n.point(t[0],t[1],t[2])},MultiPoint:function(t,n){for(var e=t.coordinates,r=-1,i=e.length;++r<i;)t=e[r],n.point(t[0],t[1],t[2])},LineString:function(t,n){Bf(t.coordinates,n,0)},MultiLineString:function(t,n){for(var e=t.coordinates,r=-1,i=e.length;++r<i;)Bf(e[r],n,0)},Polygon:function(t,n){Yf(t.coordinates,n)},MultiPolygon:function(t,n){for(var e=t.coordinates,r=-1,i=e.length;++r<i;)Yf(e[r],n)},GeometryCollection:function(t,n){for(var e=t.geometries,r=-1,i=e.length;++r<i;)Uf(e[r],n)}};function Bf(t,n,e){var r,i=-1,o=t.length-e;for(n.lineStart();++i<o;)r=t[i],n.point(r[0],r[1],r[2]);n.lineEnd()}function Yf(t,n){var e=-1,r=t.length;for(n.polygonStart();++e<r;)Bf(t[e],n,1);n.polygonEnd()}function Lf(t,n){t&&If.hasOwnProperty(t.type)?If[t.type](t,n):Uf(t,n)}var jf,Hf,Xf,Gf,Vf,Wf,Zf,Kf,Qf,Jf,ts,ns,es,rs,is,os,as=new T,us=new T,cs={point:qf,lineStart:qf,lineEnd:qf,polygonStart:function(){as=new T,cs.lineStart=fs,cs.lineEnd=ss},polygonEnd:function(){var t=+as;us.add(t<0?_f+t:t),this.lineStart=this.lineEnd=this.point=qf},sphere:function(){us.add(_f)}};function fs(){cs.point=ls}function ss(){hs(jf,Hf)}function ls(t,n){cs.point=hs,jf=t,Hf=n,Xf=t*=mf,Gf=Tf(n=(n*=mf)/2+vf),Vf=Cf(n)}function hs(t,n){var e=(t*=mf)-Xf,r=e>=0?1:-1,i=r*e,o=Tf(n=(n*=mf)/2+vf),a=Cf(n),u=Vf*a,c=Gf*o+u*Tf(i),f=u*r*Cf(i);as.add(Mf(f,c)),Xf=t,Gf=o,Vf=a}function ds(t){return[Mf(t[1],t[0]),Rf(t[2])]}function ps(t){var n=t[0],e=t[1],r=Tf(e);return[r*Tf(n),r*Cf(n),Cf(e)]}function gs(t,n){return t[0]*n[0]+t[1]*n[1]+t[2]*n[2]}function ys(t,n){return[t[1]*n[2]-t[2]*n[1],t[2]*n[0]-t[0]*n[2],t[0]*n[1]-t[1]*n[0]]}function vs(t,n){t[0]+=n[0],t[1]+=n[1],t[2]+=n[2]}function _s(t,n){return[t[0]*n,t[1]*n,t[2]*n]}function bs(t){var n=zf(t[0]*t[0]+t[1]*t[1]+t[2]*t[2]);t[0]/=n,t[1]/=n,t[2]/=n}var ms,xs,ws,Ms,Ts,As,Ss,Es,Ns,ks,Cs,Ps,zs,$s,Ds,Rs,Fs={point:qs,lineStart:Is,lineEnd:Os,polygonStart:function(){Fs.point=Bs,Fs.lineStart=Ys,Fs.lineEnd=Ls,rs=new T,cs.polygonStart()},polygonEnd:function(){cs.polygonEnd(),Fs.point=qs,Fs.lineStart=Is,Fs.lineEnd=Os,as<0?(Wf=-(Kf=180),Zf=-(Qf=90)):rs>df?Qf=90:rs<-df&&(Zf=-90),os[0]=Wf,os[1]=Kf},sphere:function(){Wf=-(Kf=180),Zf=-(Qf=90)}};function qs(t,n){is.push(os=[Wf=t,Kf=t]),n<Zf&&(Zf=n),n>Qf&&(Qf=n)}function Us(t,n){var e=ps([t*mf,n*mf]);if(es){var r=ys(es,e),i=ys([r[1],-r[0],0],r);bs(i),i=ds(i);var o,a=t-Jf,u=a>0?1:-1,c=i[0]*bf*u,f=xf(a)>180;f^(u*Jf<c&&c<u*t)?(o=i[1]*bf)>Qf&&(Qf=o):f^(u*Jf<(c=(c+360)%360-180)&&c<u*t)?(o=-i[1]*bf)<Zf&&(Zf=o):(n<Zf&&(Zf=n),n>Qf&&(Qf=n)),f?t<Jf?js(Wf,t)>js(Wf,Kf)&&(Kf=t):js(t,Kf)>js(Wf,Kf)&&(Wf=t):Kf>=Wf?(t<Wf&&(Wf=t),t>Kf&&(Kf=t)):t>Jf?js(Wf,t)>js(Wf,Kf)&&(Kf=t):js(t,Kf)>js(Wf,Kf)&&(Wf=t)}else is.push(os=[Wf=t,Kf=t]);n<Zf&&(Zf=n),n>Qf&&(Qf=n),es=e,Jf=t}function Is(){Fs.point=Us}function Os(){os[0]=Wf,os[1]=Kf,Fs.point=qs,es=null}function Bs(t,n){if(es){var e=t-Jf;rs.add(xf(e)>180?e+(e>0?360:-360):e)}else ts=t,ns=n;cs.point(t,n),Us(t,n)}function Ys(){cs.lineStart()}function Ls(){Bs(ts,ns),cs.lineEnd(),xf(rs)>df&&(Wf=-(Kf=180)),os[0]=Wf,os[1]=Kf,es=null}function js(t,n){return(n-=t)<0?n+360:n}function Hs(t,n){return t[0]-n[0]}function Xs(t,n){return t[0]<=t[1]?t[0]<=n&&n<=t[1]:n<t[0]||t[1]<n}var Gs={sphere:qf,point:Vs,lineStart:Zs,lineEnd:Js,polygonStart:function(){Gs.lineStart=tl,Gs.lineEnd=nl},polygonEnd:function(){Gs.lineStart=Zs,Gs.lineEnd=Js}};function Vs(t,n){t*=mf;var e=Tf(n*=mf);Ws(e*Tf(t),e*Cf(t),Cf(n))}function Ws(t,n,e){++ms,ws+=(t-ws)/ms,Ms+=(n-Ms)/ms,Ts+=(e-Ts)/ms}function Zs(){Gs.point=Ks}function Ks(t,n){t*=mf;var e=Tf(n*=mf);$s=e*Tf(t),Ds=e*Cf(t),Rs=Cf(n),Gs.point=Qs,Ws($s,Ds,Rs)}function Qs(t,n){t*=mf;var e=Tf(n*=mf),r=e*Tf(t),i=e*Cf(t),o=Cf(n),a=Mf(zf((a=Ds*o-Rs*i)*a+(a=Rs*r-$s*o)*a+(a=$s*i-Ds*r)*a),$s*r+Ds*i+Rs*o);xs+=a,As+=a*($s+($s=r)),Ss+=a*(Ds+(Ds=i)),Es+=a*(Rs+(Rs=o)),Ws($s,Ds,Rs)}function Js(){Gs.point=Vs}function tl(){Gs.point=el}function nl(){rl(Ps,zs),Gs.point=Vs}function el(t,n){Ps=t,zs=n,t*=mf,n*=mf,Gs.point=rl;var e=Tf(n);$s=e*Tf(t),Ds=e*Cf(t),Rs=Cf(n),Ws($s,Ds,Rs)}function rl(t,n){t*=mf;var e=Tf(n*=mf),r=e*Tf(t),i=e*Cf(t),o=Cf(n),a=Ds*o-Rs*i,u=Rs*r-$s*o,c=$s*i-Ds*r,f=Ef(a,u,c),s=Rf(f),l=f&&-s/f;Ns.add(l*a),ks.add(l*u),Cs.add(l*c),xs+=s,As+=s*($s+($s=r)),Ss+=s*(Ds+(Ds=i)),Es+=s*(Rs+(Rs=o)),Ws($s,Ds,Rs)}function il(t){return function(){return t}}function ol(t,n){function e(e,r){return e=t(e,r),n(e[0],e[1])}return t.invert&&n.invert&&(e.invert=function(e,r){return(e=n.invert(e,r))&&t.invert(e[0],e[1])}),e}function al(t,n){return xf(t)>gf&&(t-=Math.round(t/_f)*_f),[t,n]}function ul(t,n,e){return(t%=_f)?n||e?ol(fl(t),sl(n,e)):fl(t):n||e?sl(n,e):al}function cl(t){return function(n,e){return xf(n+=t)>gf&&(n-=Math.round(n/_f)*_f),[n,e]}}function fl(t){var n=cl(t);return n.invert=cl(-t),n}function sl(t,n){var e=Tf(t),r=Cf(t),i=Tf(n),o=Cf(n);function a(t,n){var a=Tf(n),u=Tf(t)*a,c=Cf(t)*a,f=Cf(n),s=f*e+u*r;return[Mf(c*i-s*o,u*e-f*r),Rf(s*i+c*o)]}return a.invert=function(t,n){var a=Tf(n),u=Tf(t)*a,c=Cf(t)*a,f=Cf(n),s=f*i-c*o;return[Mf(c*i+f*o,u*e+s*r),Rf(s*e-u*r)]},a}function ll(t){function n(n){return(n=t(n[0]*mf,n[1]*mf))[0]*=bf,n[1]*=bf,n}return t=ul(t[0]*mf,t[1]*mf,t.length>2?t[2]*mf:0),n.invert=function(n){return(n=t.invert(n[0]*mf,n[1]*mf))[0]*=bf,n[1]*=bf,n},n}function hl(t,n,e,r,i,o){if(e){var a=Tf(n),u=Cf(n),c=r*e;null==i?(i=n+r*_f,o=n-c/2):(i=dl(a,i),o=dl(a,o),(r>0?i<o:i>o)&&(i+=r*_f));for(var f,s=i;r>0?s>o:s<o;s-=c)f=ds([a,-u*Tf(s),-u*Cf(s)]),t.point(f[0],f[1])}}function dl(t,n){(n=ps(n))[0]-=t,bs(n);var e=Df(-n[1]);return((-n[2]<0?-e:e)+_f-df)%_f}function pl(){var t,n=[];return{point:function(n,e,r){t.push([n,e,r])},lineStart:function(){n.push(t=[])},lineEnd:qf,rejoin:function(){n.length>1&&n.push(n.pop().concat(n.shift()))},result:function(){var e=n;return n=[],t=null,e}}}function gl(t,n){return xf(t[0]-n[0])<df&&xf(t[1]-n[1])<df}function yl(t,n,e,r){this.x=t,this.z=n,this.o=e,this.e=r,this.v=!1,this.n=this.p=null}function vl(t,n,e,r,i){var o,a,u=[],c=[];if(t.forEach((function(t){if(!((n=t.length-1)<=0)){var n,e,r=t[0],a=t[n];if(gl(r,a)){if(!r[2]&&!a[2]){for(i.lineStart(),o=0;o<n;++o)i.point((r=t[o])[0],r[1]);return void i.lineEnd()}a[0]+=2*df}u.push(e=new yl(r,t,null,!0)),c.push(e.o=new yl(r,null,e,!1)),u.push(e=new yl(a,t,null,!1)),c.push(e.o=new yl(a,null,e,!0))}})),u.length){for(c.sort(n),_l(u),_l(c),o=0,a=c.length;o<a;++o)c[o].e=e=!e;for(var f,s,l=u[0];;){for(var h=l,d=!0;h.v;)if((h=h.n)===l)return;f=h.z,i.lineStart();do{if(h.v=h.o.v=!0,h.e){if(d)for(o=0,a=f.length;o<a;++o)i.point((s=f[o])[0],s[1]);else r(h.x,h.n.x,1,i);h=h.n}else{if(d)for(f=h.p.z,o=f.length-1;o>=0;--o)i.point((s=f[o])[0],s[1]);else r(h.x,h.p.x,-1,i);h=h.p}f=(h=h.o).z,d=!d}while(!h.v);i.lineEnd()}}}function _l(t){if(n=t.length){for(var n,e,r=0,i=t[0];++r<n;)i.n=e=t[r],e.p=i,i=e;i.n=e=t[0],e.p=i}}function bl(t){return xf(t[0])<=gf?t[0]:Pf(t[0])*((xf(t[0])+gf)%_f-gf)}function ml(t,n){var e=bl(n),r=n[1],i=Cf(r),o=[Cf(e),-Tf(e),0],a=0,u=0,c=new T;1===i?r=yf+df:-1===i&&(r=-yf-df);for(var f=0,s=t.length;f<s;++f)if(h=(l=t[f]).length)for(var l,h,d=l[h-1],p=bl(d),g=d[1]/2+vf,y=Cf(g),v=Tf(g),_=0;_<h;++_,p=m,y=w,v=M,d=b){var b=l[_],m=bl(b),x=b[1]/2+vf,w=Cf(x),M=Tf(x),A=m-p,S=A>=0?1:-1,E=S*A,N=E>gf,k=y*w;if(c.add(Mf(k*S*Cf(E),v*M+k*Tf(E))),a+=N?A+S*_f:A,N^p>=e^m>=e){var C=ys(ps(d),ps(b));bs(C);var P=ys(o,C);bs(P);var z=(N^A>=0?-1:1)*Rf(P[2]);(r>z||r===z&&(C[0]||C[1]))&&(u+=N^A>=0?1:-1)}}return(a<-df||a<df&&c<-pf)^1&u}function xl(t,n,e,r){return function(i){var o,a,u,c=n(i),f=pl(),s=n(f),l=!1,h={point:d,lineStart:g,lineEnd:y,polygonStart:function(){h.point=v,h.lineStart=_,h.lineEnd=b,a=[],o=[]},polygonEnd:function(){h.point=d,h.lineStart=g,h.lineEnd=y,a=ft(a);var t=ml(o,r);a.length?(l||(i.polygonStart(),l=!0),vl(a,Ml,t,e,i)):t&&(l||(i.polygonStart(),l=!0),i.lineStart(),e(null,null,1,i),i.lineEnd()),l&&(i.polygonEnd(),l=!1),a=o=null},sphere:function(){i.polygonStart(),i.lineStart(),e(null,null,1,i),i.lineEnd(),i.polygonEnd()}};function d(n,e){t(n,e)&&i.point(n,e)}function p(t,n){c.point(t,n)}function g(){h.point=p,c.lineStart()}function y(){h.point=d,c.lineEnd()}function v(t,n){u.push([t,n]),s.point(t,n)}function _(){s.lineStart(),u=[]}function b(){v(u[0][0],u[0][1]),s.lineEnd();var t,n,e,r,c=s.clean(),h=f.result(),d=h.length;if(u.pop(),o.push(u),u=null,d)if(1&c){if((n=(e=h[0]).length-1)>0){for(l||(i.polygonStart(),l=!0),i.lineStart(),t=0;t<n;++t)i.point((r=e[t])[0],r[1]);i.lineEnd()}}else d>1&&2&c&&h.push(h.pop().concat(h.shift())),a.push(h.filter(wl))}return h}}function wl(t){return t.length>1}function Ml(t,n){return((t=t.x)[0]<0?t[1]-yf-df:yf-t[1])-((n=n.x)[0]<0?n[1]-yf-df:yf-n[1])}al.invert=al;var Tl=xl((function(){return!0}),(function(t){var n,e=NaN,r=NaN,i=NaN;return{lineStart:function(){t.lineStart(),n=1},point:function(o,a){var u=o>0?gf:-gf,c=xf(o-e);xf(c-gf)<df?(t.point(e,r=(r+a)/2>0?yf:-yf),t.point(i,r),t.lineEnd(),t.lineStart(),t.point(u,r),t.point(o,r),n=0):i!==u&&c>=gf&&(xf(e-i)<df&&(e-=i*df),xf(o-u)<df&&(o-=u*df),r=function(t,n,e,r){var i,o,a=Cf(t-e);return xf(a)>df?wf((Cf(n)*(o=Tf(r))*Cf(e)-Cf(r)*(i=Tf(n))*Cf(t))/(i*o*a)):(n+r)/2}(e,r,o,a),t.point(i,r),t.lineEnd(),t.lineStart(),t.point(u,r),n=0),t.point(e=o,r=a),i=u},lineEnd:function(){t.lineEnd(),e=r=NaN},clean:function(){return 2-n}}}),(function(t,n,e,r){var i;if(null==t)i=e*yf,r.point(-gf,i),r.point(0,i),r.point(gf,i),r.point(gf,0),r.point(gf,-i),r.point(0,-i),r.point(-gf,-i),r.point(-gf,0),r.point(-gf,i);else if(xf(t[0]-n[0])>df){var o=t[0]<n[0]?gf:-gf;i=e*o/2,r.point(-o,i),r.point(0,i),r.point(o,i)}else r.point(n[0],n[1])}),[-gf,-yf]);function Al(t){var n=Tf(t),e=6*mf,r=n>0,i=xf(n)>df;function o(t,e){return Tf(t)*Tf(e)>n}function a(t,e,r){var i=[1,0,0],o=ys(ps(t),ps(e)),a=gs(o,o),u=o[0],c=a-u*u;if(!c)return!r&&t;var f=n*a/c,s=-n*u/c,l=ys(i,o),h=_s(i,f);vs(h,_s(o,s));var d=l,p=gs(h,d),g=gs(d,d),y=p*p-g*(gs(h,h)-1);if(!(y<0)){var v=zf(y),_=_s(d,(-p-v)/g);if(vs(_,h),_=ds(_),!r)return _;var b,m=t[0],x=e[0],w=t[1],M=e[1];x<m&&(b=m,m=x,x=b);var T=x-m,A=xf(T-gf)<df;if(!A&&M<w&&(b=w,w=M,M=b),A||T<df?A?w+M>0^_[1]<(xf(_[0]-m)<df?w:M):w<=_[1]&&_[1]<=M:T>gf^(m<=_[0]&&_[0]<=x)){var S=_s(d,(-p+v)/g);return vs(S,h),[_,ds(S)]}}}function u(n,e){var i=r?t:gf-t,o=0;return n<-i?o|=1:n>i&&(o|=2),e<-i?o|=4:e>i&&(o|=8),o}return xl(o,(function(t){var n,e,c,f,s;return{lineStart:function(){f=c=!1,s=1},point:function(l,h){var d,p=[l,h],g=o(l,h),y=r?g?0:u(l,h):g?u(l+(l<0?gf:-gf),h):0;if(!n&&(f=c=g)&&t.lineStart(),g!==c&&(!(d=a(n,p))||gl(n,d)||gl(p,d))&&(p[2]=1),g!==c)s=0,g?(t.lineStart(),d=a(p,n),t.point(d[0],d[1])):(d=a(n,p),t.point(d[0],d[1],2),t.lineEnd()),n=d;else if(i&&n&&r^g){var v;y&e||!(v=a(p,n,!0))||(s=0,r?(t.lineStart(),t.point(v[0][0],v[0][1]),t.point(v[1][0],v[1][1]),t.lineEnd()):(t.point(v[1][0],v[1][1]),t.lineEnd(),t.lineStart(),t.point(v[0][0],v[0][1],3)))}!g||n&&gl(n,p)||t.point(p[0],p[1]),n=p,c=g,e=y},lineEnd:function(){c&&t.lineEnd(),n=null},clean:function(){return s|(f&&c)<<1}}}),(function(n,r,i,o){hl(o,t,e,i,n,r)}),r?[0,-t]:[-gf,t-gf])}var Sl,El,Nl,kl,Cl=1e9,Pl=-Cl;function zl(t,n,e,r){function i(i,o){return t<=i&&i<=e&&n<=o&&o<=r}function o(i,o,u,f){var s=0,l=0;if(null==i||(s=a(i,u))!==(l=a(o,u))||c(i,o)<0^u>0)do{f.point(0===s||3===s?t:e,s>1?r:n)}while((s=(s+u+4)%4)!==l);else f.point(o[0],o[1])}function a(r,i){return xf(r[0]-t)<df?i>0?0:3:xf(r[0]-e)<df?i>0?2:1:xf(r[1]-n)<df?i>0?1:0:i>0?3:2}function u(t,n){return c(t.x,n.x)}function c(t,n){var e=a(t,1),r=a(n,1);return e!==r?e-r:0===e?n[1]-t[1]:1===e?t[0]-n[0]:2===e?t[1]-n[1]:n[0]-t[0]}return function(a){var c,f,s,l,h,d,p,g,y,v,_,b=a,m=pl(),x={point:w,lineStart:function(){x.point=M,f&&f.push(s=[]);v=!0,y=!1,p=g=NaN},lineEnd:function(){c&&(M(l,h),d&&y&&m.rejoin(),c.push(m.result()));x.point=w,y&&b.lineEnd()},polygonStart:function(){b=m,c=[],f=[],_=!0},polygonEnd:function(){var n=function(){for(var n=0,e=0,i=f.length;e<i;++e)for(var o,a,u=f[e],c=1,s=u.length,l=u[0],h=l[0],d=l[1];c<s;++c)o=h,a=d,h=(l=u[c])[0],d=l[1],a<=r?d>r&&(h-o)*(r-a)>(d-a)*(t-o)&&++n:d<=r&&(h-o)*(r-a)<(d-a)*(t-o)&&--n;return n}(),e=_&&n,i=(c=ft(c)).length;(e||i)&&(a.polygonStart(),e&&(a.lineStart(),o(null,null,1,a),a.lineEnd()),i&&vl(c,u,n,o,a),a.polygonEnd());b=a,c=f=s=null}};function w(t,n){i(t,n)&&b.point(t,n)}function M(o,a){var u=i(o,a);if(f&&s.push([o,a]),v)l=o,h=a,d=u,v=!1,u&&(b.lineStart(),b.point(o,a));else if(u&&y)b.point(o,a);else{var c=[p=Math.max(Pl,Math.min(Cl,p)),g=Math.max(Pl,Math.min(Cl,g))],m=[o=Math.max(Pl,Math.min(Cl,o)),a=Math.max(Pl,Math.min(Cl,a))];!function(t,n,e,r,i,o){var a,u=t[0],c=t[1],f=0,s=1,l=n[0]-u,h=n[1]-c;if(a=e-u,l||!(a>0)){if(a/=l,l<0){if(a<f)return;a<s&&(s=a)}else if(l>0){if(a>s)return;a>f&&(f=a)}if(a=i-u,l||!(a<0)){if(a/=l,l<0){if(a>s)return;a>f&&(f=a)}else if(l>0){if(a<f)return;a<s&&(s=a)}if(a=r-c,h||!(a>0)){if(a/=h,h<0){if(a<f)return;a<s&&(s=a)}else if(h>0){if(a>s)return;a>f&&(f=a)}if(a=o-c,h||!(a<0)){if(a/=h,h<0){if(a>s)return;a>f&&(f=a)}else if(h>0){if(a<f)return;a<s&&(s=a)}return f>0&&(t[0]=u+f*l,t[1]=c+f*h),s<1&&(n[0]=u+s*l,n[1]=c+s*h),!0}}}}}(c,m,t,n,e,r)?u&&(b.lineStart(),b.point(o,a),_=!1):(y||(b.lineStart(),b.point(c[0],c[1])),b.point(m[0],m[1]),u||b.lineEnd(),_=!1)}p=o,g=a,y=u}return x}}var $l={sphere:qf,point:qf,lineStart:function(){$l.point=Rl,$l.lineEnd=Dl},lineEnd:qf,polygonStart:qf,polygonEnd:qf};function Dl(){$l.point=$l.lineEnd=qf}function Rl(t,n){El=t*=mf,Nl=Cf(n*=mf),kl=Tf(n),$l.point=Fl}function Fl(t,n){t*=mf;var e=Cf(n*=mf),r=Tf(n),i=xf(t-El),o=Tf(i),a=r*Cf(i),u=kl*e-Nl*r*o,c=Nl*e+kl*r*o;Sl.add(Mf(zf(a*a+u*u),c)),El=t,Nl=e,kl=r}function ql(t){return Sl=new T,Lf(t,$l),+Sl}var Ul=[null,null],Il={type:"LineString",coordinates:Ul};function Ol(t,n){return Ul[0]=t,Ul[1]=n,ql(Il)}var Bl={Feature:function(t,n){return Ll(t.geometry,n)},FeatureCollection:function(t,n){for(var e=t.features,r=-1,i=e.length;++r<i;)if(Ll(e[r].geometry,n))return!0;return!1}},Yl={Sphere:function(){return!0},Point:function(t,n){return jl(t.coordinates,n)},MultiPoint:function(t,n){for(var e=t.coordinates,r=-1,i=e.length;++r<i;)if(jl(e[r],n))return!0;return!1},LineString:function(t,n){return Hl(t.coordinates,n)},MultiLineString:function(t,n){for(var e=t.coordinates,r=-1,i=e.length;++r<i;)if(Hl(e[r],n))return!0;return!1},Polygon:function(t,n){return Xl(t.coordinates,n)},MultiPolygon:function(t,n){for(var e=t.coordinates,r=-1,i=e.length;++r<i;)if(Xl(e[r],n))return!0;return!1},GeometryCollection:function(t,n){for(var e=t.geometries,r=-1,i=e.length;++r<i;)if(Ll(e[r],n))return!0;return!1}};function Ll(t,n){return!(!t||!Yl.hasOwnProperty(t.type))&&Yl[t.type](t,n)}function jl(t,n){return 0===Ol(t,n)}function Hl(t,n){for(var e,r,i,o=0,a=t.length;o<a;o++){if(0===(r=Ol(t[o],n)))return!0;if(o>0&&(i=Ol(t[o],t[o-1]))>0&&e<=i&&r<=i&&(e+r-i)*(1-Math.pow((e-r)/i,2))<pf*i)return!0;e=r}return!1}function Xl(t,n){return!!ml(t.map(Gl),Vl(n))}function Gl(t){return(t=t.map(Vl)).pop(),t}function Vl(t){return[t[0]*mf,t[1]*mf]}function Wl(t,n,e){var r=lt(t,n-df,e).concat(n);return function(t){return r.map((function(n){return[t,n]}))}}function Zl(t,n,e){var r=lt(t,n-df,e).concat(n);return function(t){return r.map((function(n){return[n,t]}))}}function Kl(){var t,n,e,r,i,o,a,u,c,f,s,l,h=10,d=h,p=90,g=360,y=2.5;function v(){return{type:"MultiLineString",coordinates:_()}}function _(){return lt(Af(r/p)*p,e,p).map(s).concat(lt(Af(u/g)*g,a,g).map(l)).concat(lt(Af(n/h)*h,t,h).filter((function(t){return xf(t%p)>df})).map(c)).concat(lt(Af(o/d)*d,i,d).filter((function(t){return xf(t%g)>df})).map(f))}return v.lines=function(){return _().map((function(t){return{type:"LineString",coordinates:t}}))},v.outline=function(){return{type:"Polygon",coordinates:[s(r).concat(l(a).slice(1),s(e).reverse().slice(1),l(u).reverse().slice(1))]}},v.extent=function(t){return arguments.length?v.extentMajor(t).extentMinor(t):v.extentMinor()},v.extentMajor=function(t){return arguments.length?(r=+t[0][0],e=+t[1][0],u=+t[0][1],a=+t[1][1],r>e&&(t=r,r=e,e=t),u>a&&(t=u,u=a,a=t),v.precision(y)):[[r,u],[e,a]]},v.extentMinor=function(e){return arguments.length?(n=+e[0][0],t=+e[1][0],o=+e[0][1],i=+e[1][1],n>t&&(e=n,n=t,t=e),o>i&&(e=o,o=i,i=e),v.precision(y)):[[n,o],[t,i]]},v.step=function(t){return arguments.length?v.stepMajor(t).stepMinor(t):v.stepMinor()},v.stepMajor=function(t){return arguments.length?(p=+t[0],g=+t[1],v):[p,g]},v.stepMinor=function(t){return arguments.length?(h=+t[0],d=+t[1],v):[h,d]},v.precision=function(h){return arguments.length?(y=+h,c=Wl(o,i,90),f=Zl(n,t,y),s=Wl(u,a,90),l=Zl(r,e,y),v):y},v.extentMajor([[-180,-90+df],[180,90-df]]).extentMinor([[-180,-80-df],[180,80+df]])}var Ql,Jl,th,nh,eh=t=>t,rh=new T,ih=new T,oh={point:qf,lineStart:qf,lineEnd:qf,polygonStart:function(){oh.lineStart=ah,oh.lineEnd=fh},polygonEnd:function(){oh.lineStart=oh.lineEnd=oh.point=qf,rh.add(xf(ih)),ih=new T},result:function(){var t=rh/2;return rh=new T,t}};function ah(){oh.point=uh}function uh(t,n){oh.point=ch,Ql=th=t,Jl=nh=n}function ch(t,n){ih.add(nh*t-th*n),th=t,nh=n}function fh(){ch(Ql,Jl)}var sh=oh,lh=1/0,hh=lh,dh=-lh,ph=dh,gh={point:function(t,n){t<lh&&(lh=t);t>dh&&(dh=t);n<hh&&(hh=n);n>ph&&(ph=n)},lineStart:qf,lineEnd:qf,polygonStart:qf,polygonEnd:qf,result:function(){var t=[[lh,hh],[dh,ph]];return dh=ph=-(hh=lh=1/0),t}};var yh,vh,_h,bh,mh=gh,xh=0,wh=0,Mh=0,Th=0,Ah=0,Sh=0,Eh=0,Nh=0,kh=0,Ch={point:Ph,lineStart:zh,lineEnd:Rh,polygonStart:function(){Ch.lineStart=Fh,Ch.lineEnd=qh},polygonEnd:function(){Ch.point=Ph,Ch.lineStart=zh,Ch.lineEnd=Rh},result:function(){var t=kh?[Eh/kh,Nh/kh]:Sh?[Th/Sh,Ah/Sh]:Mh?[xh/Mh,wh/Mh]:[NaN,NaN];return xh=wh=Mh=Th=Ah=Sh=Eh=Nh=kh=0,t}};function Ph(t,n){xh+=t,wh+=n,++Mh}function zh(){Ch.point=$h}function $h(t,n){Ch.point=Dh,Ph(_h=t,bh=n)}function Dh(t,n){var e=t-_h,r=n-bh,i=zf(e*e+r*r);Th+=i*(_h+t)/2,Ah+=i*(bh+n)/2,Sh+=i,Ph(_h=t,bh=n)}function Rh(){Ch.point=Ph}function Fh(){Ch.point=Uh}function qh(){Ih(yh,vh)}function Uh(t,n){Ch.point=Ih,Ph(yh=_h=t,vh=bh=n)}function Ih(t,n){var e=t-_h,r=n-bh,i=zf(e*e+r*r);Th+=i*(_h+t)/2,Ah+=i*(bh+n)/2,Sh+=i,Eh+=(i=bh*t-_h*n)*(_h+t),Nh+=i*(bh+n),kh+=3*i,Ph(_h=t,bh=n)}var Oh=Ch;function Bh(t){this._context=t}Bh.prototype={_radius:4.5,pointRadius:function(t){return this._radius=t,this},polygonStart:function(){this._line=0},polygonEnd:function(){this._line=NaN},lineStart:function(){this._point=0},lineEnd:function(){0===this._line&&this._context.closePath(),this._point=NaN},point:function(t,n){switch(this._point){case 0:this._context.moveTo(t,n),this._point=1;break;case 1:this._context.lineTo(t,n);break;default:this._context.moveTo(t+this._radius,n),this._context.arc(t,n,this._radius,0,_f)}},result:qf};var Yh,Lh,jh,Hh,Xh,Gh=new T,Vh={point:qf,lineStart:function(){Vh.point=Wh},lineEnd:function(){Yh&&Zh(Lh,jh),Vh.point=qf},polygonStart:function(){Yh=!0},polygonEnd:function(){Yh=null},result:function(){var t=+Gh;return Gh=new T,t}};function Wh(t,n){Vh.point=Zh,Lh=Hh=t,jh=Xh=n}function Zh(t,n){Hh-=t,Xh-=n,Gh.add(zf(Hh*Hh+Xh*Xh)),Hh=t,Xh=n}var Kh=Vh;let Qh,Jh,td,nd;class ed{constructor(t){this._append=null==t?rd:function(t){const n=Math.floor(t);if(!(n>=0))throw new RangeError(`invalid digits: ${t}`);if(n>15)return rd;if(n!==Qh){const t=10**n;Qh=n,Jh=function(n){let e=1;this._+=n[0];for(const r=n.length;e<r;++e)this._+=Math.round(arguments[e]*t)/t+n[e]}}return Jh}(t),this._radius=4.5,this._=""}pointRadius(t){return this._radius=+t,this}polygonStart(){this._line=0}polygonEnd(){this._line=NaN}lineStart(){this._point=0}lineEnd(){0===this._line&&(this._+="Z"),this._point=NaN}point(t,n){switch(this._point){case 0:this._append`M${t},${n}`,this._point=1;break;case 1:this._append`L${t},${n}`;break;default:if(this._append`M${t},${n}`,this._radius!==td||this._append!==Jh){const t=this._radius,n=this._;this._="",this._append`m0,${t}a${t},${t} 0 1,1 0,${-2*t}a${t},${t} 0 1,1 0,${2*t}z`,td=t,Jh=this._append,nd=this._,this._=n}this._+=nd}}result(){const t=this._;return this._="",t.length?t:null}}function rd(t){let n=1;this._+=t[0];for(const e=t.length;n<e;++n)this._+=arguments[n]+t[n]}function id(t){return function(n){var e=new od;for(var r in t)e[r]=t[r];return e.stream=n,e}}function od(){}function ad(t,n,e){var r=t.clipExtent&&t.clipExtent();return t.scale(150).translate([0,0]),null!=r&&t.clipExtent(null),Lf(e,t.stream(mh)),n(mh.result()),null!=r&&t.clipExtent(r),t}function ud(t,n,e){return ad(t,(function(e){var r=n[1][0]-n[0][0],i=n[1][1]-n[0][1],o=Math.min(r/(e[1][0]-e[0][0]),i/(e[1][1]-e[0][1])),a=+n[0][0]+(r-o*(e[1][0]+e[0][0]))/2,u=+n[0][1]+(i-o*(e[1][1]+e[0][1]))/2;t.scale(150*o).translate([a,u])}),e)}function cd(t,n,e){return ud(t,[[0,0],n],e)}function fd(t,n,e){return ad(t,(function(e){var r=+n,i=r/(e[1][0]-e[0][0]),o=(r-i*(e[1][0]+e[0][0]))/2,a=-i*e[0][1];t.scale(150*i).translate([o,a])}),e)}function sd(t,n,e){return ad(t,(function(e){var r=+n,i=r/(e[1][1]-e[0][1]),o=-i*e[0][0],a=(r-i*(e[1][1]+e[0][1]))/2;t.scale(150*i).translate([o,a])}),e)}od.prototype={constructor:od,point:function(t,n){this.stream.point(t,n)},sphere:function(){this.stream.sphere()},lineStart:function(){this.stream.lineStart()},lineEnd:function(){this.stream.lineEnd()},polygonStart:function(){this.stream.polygonStart()},polygonEnd:function(){this.stream.polygonEnd()}};var ld=16,hd=Tf(30*mf);function dd(t,n){return+n?function(t,n){function e(r,i,o,a,u,c,f,s,l,h,d,p,g,y){var v=f-r,_=s-i,b=v*v+_*_;if(b>4*n&&g--){var m=a+h,x=u+d,w=c+p,M=zf(m*m+x*x+w*w),T=Rf(w/=M),A=xf(xf(w)-1)<df||xf(o-l)<df?(o+l)/2:Mf(x,m),S=t(A,T),E=S[0],N=S[1],k=E-r,C=N-i,P=_*k-v*C;(P*P/b>n||xf((v*k+_*C)/b-.5)>.3||a*h+u*d+c*p<hd)&&(e(r,i,o,a,u,c,E,N,A,m/=M,x/=M,w,g,y),y.point(E,N),e(E,N,A,m,x,w,f,s,l,h,d,p,g,y))}}return function(n){var r,i,o,a,u,c,f,s,l,h,d,p,g={point:y,lineStart:v,lineEnd:b,polygonStart:function(){n.polygonStart(),g.lineStart=m},polygonEnd:function(){n.polygonEnd(),g.lineStart=v}};function y(e,r){e=t(e,r),n.point(e[0],e[1])}function v(){s=NaN,g.point=_,n.lineStart()}function _(r,i){var o=ps([r,i]),a=t(r,i);e(s,l,f,h,d,p,s=a[0],l=a[1],f=r,h=o[0],d=o[1],p=o[2],ld,n),n.point(s,l)}function b(){g.point=y,n.lineEnd()}function m(){v(),g.point=x,g.lineEnd=w}function x(t,n){_(r=t,n),i=s,o=l,a=h,u=d,c=p,g.point=_}function w(){e(s,l,f,h,d,p,i,o,r,a,u,c,ld,n),g.lineEnd=b,b()}return g}}(t,n):function(t){return id({point:function(n,e){n=t(n,e),this.stream.point(n[0],n[1])}})}(t)}var pd=id({point:function(t,n){this.stream.point(t*mf,n*mf)}});function gd(t,n,e,r,i,o){if(!o)return function(t,n,e,r,i){function o(o,a){return[n+t*(o*=r),e-t*(a*=i)]}return o.invert=function(o,a){return[(o-n)/t*r,(e-a)/t*i]},o}(t,n,e,r,i);var a=Tf(o),u=Cf(o),c=a*t,f=u*t,s=a/t,l=u/t,h=(u*e-a*n)/t,d=(u*n+a*e)/t;function p(t,o){return[c*(t*=r)-f*(o*=i)+n,e-f*t-c*o]}return p.invert=function(t,n){return[r*(s*t-l*n+h),i*(d-l*t-s*n)]},p}function yd(t){return vd((function(){return t}))()}function vd(t){var n,e,r,i,o,a,u,c,f,s,l=150,h=480,d=250,p=0,g=0,y=0,v=0,_=0,b=0,m=1,x=1,w=null,M=Tl,T=null,A=eh,S=.5;function E(t){return c(t[0]*mf,t[1]*mf)}function N(t){return(t=c.invert(t[0],t[1]))&&[t[0]*bf,t[1]*bf]}function k(){var t=gd(l,0,0,m,x,b).apply(null,n(p,g)),r=gd(l,h-t[0],d-t[1],m,x,b);return e=ul(y,v,_),u=ol(n,r),c=ol(e,u),a=dd(u,S),C()}function C(){return f=s=null,E}return E.stream=function(t){return f&&s===t?f:f=pd(function(t){return id({point:function(n,e){var r=t(n,e);return this.stream.point(r[0],r[1])}})}(e)(M(a(A(s=t)))))},E.preclip=function(t){return arguments.length?(M=t,w=void 0,C()):M},E.postclip=function(t){return arguments.length?(A=t,T=r=i=o=null,C()):A},E.clipAngle=function(t){return arguments.length?(M=+t?Al(w=t*mf):(w=null,Tl),C()):w*bf},E.clipExtent=function(t){return arguments.length?(A=null==t?(T=r=i=o=null,eh):zl(T=+t[0][0],r=+t[0][1],i=+t[1][0],o=+t[1][1]),C()):null==T?null:[[T,r],[i,o]]},E.scale=function(t){return arguments.length?(l=+t,k()):l},E.translate=function(t){return arguments.length?(h=+t[0],d=+t[1],k()):[h,d]},E.center=function(t){return arguments.length?(p=t[0]%360*mf,g=t[1]%360*mf,k()):[p*bf,g*bf]},E.rotate=function(t){return arguments.length?(y=t[0]%360*mf,v=t[1]%360*mf,_=t.length>2?t[2]%360*mf:0,k()):[y*bf,v*bf,_*bf]},E.angle=function(t){return arguments.length?(b=t%360*mf,k()):b*bf},E.reflectX=function(t){return arguments.length?(m=t?-1:1,k()):m<0},E.reflectY=function(t){return arguments.length?(x=t?-1:1,k()):x<0},E.precision=function(t){return arguments.length?(a=dd(u,S=t*t),C()):zf(S)},E.fitExtent=function(t,n){return ud(E,t,n)},E.fitSize=function(t,n){return cd(E,t,n)},E.fitWidth=function(t,n){return fd(E,t,n)},E.fitHeight=function(t,n){return sd(E,t,n)},function(){return n=t.apply(this,arguments),E.invert=n.invert&&N,k()}}function _d(t){var n=0,e=gf/3,r=vd(t),i=r(n,e);return i.parallels=function(t){return arguments.length?r(n=t[0]*mf,e=t[1]*mf):[n*bf,e*bf]},i}function bd(t,n){var e=Cf(t),r=(e+Cf(n))/2;if(xf(r)<df)return function(t){var n=Tf(t);function e(t,e){return[t*n,Cf(e)/n]}return e.invert=function(t,e){return[t/n,Rf(e*n)]},e}(t);var i=1+e*(2*r-e),o=zf(i)/r;function a(t,n){var e=zf(i-2*r*Cf(n))/r;return[e*Cf(t*=r),o-e*Tf(t)]}return a.invert=function(t,n){var e=o-n,a=Mf(t,xf(e))*Pf(e);return e*r<0&&(a-=gf*Pf(t)*Pf(e)),[a/r,Rf((i-(t*t+e*e)*r*r)/(2*r))]},a}function md(){return _d(bd).scale(155.424).center([0,33.6442])}function xd(){return md().parallels([29.5,45.5]).scale(1070).translate([480,250]).rotate([96,0]).center([-.6,38.7])}function wd(t){return function(n,e){var r=Tf(n),i=Tf(e),o=t(r*i);return o===1/0?[2,0]:[o*i*Cf(n),o*Cf(e)]}}function Md(t){return function(n,e){var r=zf(n*n+e*e),i=t(r),o=Cf(i),a=Tf(i);return[Mf(n*o,r*a),Rf(r&&e*o/r)]}}var Td=wd((function(t){return zf(2/(1+t))}));Td.invert=Md((function(t){return 2*Rf(t/2)}));var Ad=wd((function(t){return(t=Df(t))&&t/Cf(t)}));function Sd(t,n){return[t,Nf($f((yf+n)/2))]}function Ed(t){var n,e,r,i=yd(t),o=i.center,a=i.scale,u=i.translate,c=i.clipExtent,f=null;function s(){var o=gf*a(),u=i(ll(i.rotate()).invert([0,0]));return c(null==f?[[u[0]-o,u[1]-o],[u[0]+o,u[1]+o]]:t===Sd?[[Math.max(u[0]-o,f),n],[Math.min(u[0]+o,e),r]]:[[f,Math.max(u[1]-o,n)],[e,Math.min(u[1]+o,r)]])}return i.scale=function(t){return arguments.length?(a(t),s()):a()},i.translate=function(t){return arguments.length?(u(t),s()):u()},i.center=function(t){return arguments.length?(o(t),s()):o()},i.clipExtent=function(t){return arguments.length?(null==t?f=n=e=r=null:(f=+t[0][0],n=+t[0][1],e=+t[1][0],r=+t[1][1]),s()):null==f?null:[[f,n],[e,r]]},s()}function Nd(t){return $f((yf+t)/2)}function kd(t,n){var e=Tf(t),r=t===n?Cf(t):Nf(e/Tf(n))/Nf(Nd(n)/Nd(t)),i=e*kf(Nd(t),r)/r;if(!r)return Sd;function o(t,n){i>0?n<-yf+df&&(n=-yf+df):n>yf-df&&(n=yf-df);var e=i/kf(Nd(n),r);return[e*Cf(r*t),i-e*Tf(r*t)]}return o.invert=function(t,n){var e=i-n,o=Pf(r)*zf(t*t+e*e),a=Mf(t,xf(e))*Pf(e);return e*r<0&&(a-=gf*Pf(t)*Pf(e)),[a/r,2*wf(kf(i/o,1/r))-yf]},o}function Cd(t,n){return[t,n]}function Pd(t,n){var e=Tf(t),r=t===n?Cf(t):(e-Tf(n))/(n-t),i=e/r+t;if(xf(r)<df)return Cd;function o(t,n){var e=i-n,o=r*t;return[e*Cf(o),i-e*Tf(o)]}return o.invert=function(t,n){var e=i-n,o=Mf(t,xf(e))*Pf(e);return e*r<0&&(o-=gf*Pf(t)*Pf(e)),[o/r,i-Pf(r)*zf(t*t+e*e)]},o}Ad.invert=Md((function(t){return t})),Sd.invert=function(t,n){return[t,2*wf(Sf(n))-yf]},Cd.invert=Cd;var zd=1.340264,$d=-.081106,Dd=893e-6,Rd=.003796,Fd=zf(3)/2;function qd(t,n){var e=Rf(Fd*Cf(n)),r=e*e,i=r*r*r;return[t*Tf(e)/(Fd*(zd+3*$d*r+i*(7*Dd+9*Rd*r))),e*(zd+$d*r+i*(Dd+Rd*r))]}function Ud(t,n){var e=Tf(n),r=Tf(t)*e;return[e*Cf(t)/r,Cf(n)/r]}function Id(t,n){var e=n*n,r=e*e;return[t*(.8707-.131979*e+r*(r*(.003971*e-.001529*r)-.013791)),n*(1.007226+e*(.015085+r*(.028874*e-.044475-.005916*r)))]}function Od(t,n){return[Tf(n)*Cf(t),Cf(n)]}function Bd(t,n){var e=Tf(n),r=1+Tf(t)*e;return[e*Cf(t)/r,Cf(n)/r]}function Yd(t,n){return[Nf($f((yf+n)/2)),-t]}function Ld(t,n){return t.parent===n.parent?1:2}function jd(t,n){return t+n.x}function Hd(t,n){return Math.max(t,n.y)}function Xd(t){var n=0,e=t.children,r=e&&e.length;if(r)for(;--r>=0;)n+=e[r].value;else n=1;t.value=n}function Gd(t,n){t instanceof Map?(t=[void 0,t],void 0===n&&(n=Wd)):void 0===n&&(n=Vd);for(var e,r,i,o,a,u=new Qd(t),c=[u];e=c.pop();)if((i=n(e.data))&&(a=(i=Array.from(i)).length))for(e.children=i,o=a-1;o>=0;--o)c.push(r=i[o]=new Qd(i[o])),r.parent=e,r.depth=e.depth+1;return u.eachBefore(Kd)}function Vd(t){return t.children}function Wd(t){return Array.isArray(t)?t[1]:null}function Zd(t){void 0!==t.data.value&&(t.value=t.data.value),t.data=t.data.data}function Kd(t){var n=0;do{t.height=n}while((t=t.parent)&&t.height<++n)}function Qd(t){this.data=t,this.depth=this.height=0,this.parent=null}function Jd(t){return null==t?null:tp(t)}function tp(t){if("function"!=typeof t)throw new Error;return t}function np(){return 0}function ep(t){return function(){return t}}qd.invert=function(t,n){for(var e,r=n,i=r*r,o=i*i*i,a=0;a<12&&(o=(i=(r-=e=(r*(zd+$d*i+o*(Dd+Rd*i))-n)/(zd+3*$d*i+o*(7*Dd+9*Rd*i)))*r)*i*i,!(xf(e)<pf));++a);return[Fd*t*(zd+3*$d*i+o*(7*Dd+9*Rd*i))/Tf(r),Rf(Cf(r)/Fd)]},Ud.invert=Md(wf),Id.invert=function(t,n){var e,r=n,i=25;do{var o=r*r,a=o*o;r-=e=(r*(1.007226+o*(.015085+a*(.028874*o-.044475-.005916*a)))-n)/(1.007226+o*(.045255+a*(.259866*o-.311325-.005916*11*a)))}while(xf(e)>df&&--i>0);return[t/(.8707+(o=r*r)*(o*(o*o*o*(.003971-.001529*o)-.013791)-.131979)),r]},Od.invert=Md(Rf),Bd.invert=Md((function(t){return 2*wf(t)})),Yd.invert=function(t,n){return[-n,2*wf(Sf(t))-yf]},Qd.prototype=Gd.prototype={constructor:Qd,count:function(){return this.eachAfter(Xd)},each:function(t,n){let e=-1;for(const r of this)t.call(n,r,++e,this);return this},eachAfter:function(t,n){for(var e,r,i,o=this,a=[o],u=[],c=-1;o=a.pop();)if(u.push(o),e=o.children)for(r=0,i=e.length;r<i;++r)a.push(e[r]);for(;o=u.pop();)t.call(n,o,++c,this);return this},eachBefore:function(t,n){for(var e,r,i=this,o=[i],a=-1;i=o.pop();)if(t.call(n,i,++a,this),e=i.children)for(r=e.length-1;r>=0;--r)o.push(e[r]);return this},find:function(t,n){let e=-1;for(const r of this)if(t.call(n,r,++e,this))return r},sum:function(t){return this.eachAfter((function(n){for(var e=+t(n.data)||0,r=n.children,i=r&&r.length;--i>=0;)e+=r[i].value;n.value=e}))},sort:function(t){return this.eachBefore((function(n){n.children&&n.children.sort(t)}))},path:function(t){for(var n=this,e=function(t,n){if(t===n)return t;var e=t.ancestors(),r=n.ancestors(),i=null;t=e.pop(),n=r.pop();for(;t===n;)i=t,t=e.pop(),n=r.pop();return i}(n,t),r=[n];n!==e;)n=n.parent,r.push(n);for(var i=r.length;t!==e;)r.splice(i,0,t),t=t.parent;return r},ancestors:function(){for(var t=this,n=[t];t=t.parent;)n.push(t);return n},descendants:function(){return Array.from(this)},leaves:function(){var t=[];return this.eachBefore((function(n){n.children||t.push(n)})),t},links:function(){var t=this,n=[];return t.each((function(e){e!==t&&n.push({source:e.parent,target:e})})),n},copy:function(){return Gd(this).eachBefore(Zd)},[Symbol.iterator]:function*(){var t,n,e,r,i=this,o=[i];do{for(t=o.reverse(),o=[];i=t.pop();)if(yield i,n=i.children)for(e=0,r=n.length;e<r;++e)o.push(n[e])}while(o.length)}};const rp=1664525,ip=1013904223,op=4294967296;function ap(){let t=1;return()=>(t=(rp*t+ip)%op)/op}function up(t,n){for(var e,r,i=0,o=(t=function(t,n){let e,r,i=t.length;for(;i;)r=n()*i--|0,e=t[i],t[i]=t[r],t[r]=e;return t}(Array.from(t),n)).length,a=[];i<o;)e=t[i],r&&sp(r,e)?++i:(r=hp(a=cp(a,e)),i=0);return r}function cp(t,n){var e,r;if(lp(n,t))return[n];for(e=0;e<t.length;++e)if(fp(n,t[e])&&lp(dp(t[e],n),t))return[t[e],n];for(e=0;e<t.length-1;++e)for(r=e+1;r<t.length;++r)if(fp(dp(t[e],t[r]),n)&&fp(dp(t[e],n),t[r])&&fp(dp(t[r],n),t[e])&&lp(pp(t[e],t[r],n),t))return[t[e],t[r],n];throw new Error}function fp(t,n){var e=t.r-n.r,r=n.x-t.x,i=n.y-t.y;return e<0||e*e<r*r+i*i}function sp(t,n){var e=t.r-n.r+1e-9*Math.max(t.r,n.r,1),r=n.x-t.x,i=n.y-t.y;return e>0&&e*e>r*r+i*i}function lp(t,n){for(var e=0;e<n.length;++e)if(!sp(t,n[e]))return!1;return!0}function hp(t){switch(t.length){case 1:return function(t){return{x:t.x,y:t.y,r:t.r}}(t[0]);case 2:return dp(t[0],t[1]);case 3:return pp(t[0],t[1],t[2])}}function dp(t,n){var e=t.x,r=t.y,i=t.r,o=n.x,a=n.y,u=n.r,c=o-e,f=a-r,s=u-i,l=Math.sqrt(c*c+f*f);return{x:(e+o+c/l*s)/2,y:(r+a+f/l*s)/2,r:(l+i+u)/2}}function pp(t,n,e){var r=t.x,i=t.y,o=t.r,a=n.x,u=n.y,c=n.r,f=e.x,s=e.y,l=e.r,h=r-a,d=r-f,p=i-u,g=i-s,y=c-o,v=l-o,_=r*r+i*i-o*o,b=_-a*a-u*u+c*c,m=_-f*f-s*s+l*l,x=d*p-h*g,w=(p*m-g*b)/(2*x)-r,M=(g*y-p*v)/x,T=(d*b-h*m)/(2*x)-i,A=(h*v-d*y)/x,S=M*M+A*A-1,E=2*(o+w*M+T*A),N=w*w+T*T-o*o,k=-(Math.abs(S)>1e-6?(E+Math.sqrt(E*E-4*S*N))/(2*S):N/E);return{x:r+w+M*k,y:i+T+A*k,r:k}}function gp(t,n,e){var r,i,o,a,u=t.x-n.x,c=t.y-n.y,f=u*u+c*c;f?(i=n.r+e.r,i*=i,a=t.r+e.r,i>(a*=a)?(r=(f+a-i)/(2*f),o=Math.sqrt(Math.max(0,a/f-r*r)),e.x=t.x-r*u-o*c,e.y=t.y-r*c+o*u):(r=(f+i-a)/(2*f),o=Math.sqrt(Math.max(0,i/f-r*r)),e.x=n.x+r*u-o*c,e.y=n.y+r*c+o*u)):(e.x=n.x+e.r,e.y=n.y)}function yp(t,n){var e=t.r+n.r-1e-6,r=n.x-t.x,i=n.y-t.y;return e>0&&e*e>r*r+i*i}function vp(t){var n=t._,e=t.next._,r=n.r+e.r,i=(n.x*e.r+e.x*n.r)/r,o=(n.y*e.r+e.y*n.r)/r;return i*i+o*o}function _p(t){this._=t,this.next=null,this.previous=null}function bp(t,n){if(!(o=(t=function(t){return"object"==typeof t&&"length"in t?t:Array.from(t)}(t)).length))return 0;var e,r,i,o,a,u,c,f,s,l,h;if((e=t[0]).x=0,e.y=0,!(o>1))return e.r;if(r=t[1],e.x=-r.r,r.x=e.r,r.y=0,!(o>2))return e.r+r.r;gp(r,e,i=t[2]),e=new _p(e),r=new _p(r),i=new _p(i),e.next=i.previous=r,r.next=e.previous=i,i.next=r.previous=e;t:for(c=3;c<o;++c){gp(e._,r._,i=t[c]),i=new _p(i),f=r.next,s=e.previous,l=r._.r,h=e._.r;do{if(l<=h){if(yp(f._,i._)){r=f,e.next=r,r.previous=e,--c;continue t}l+=f._.r,f=f.next}else{if(yp(s._,i._)){(e=s).next=r,r.previous=e,--c;continue t}h+=s._.r,s=s.previous}}while(f!==s.next);for(i.previous=e,i.next=r,e.next=r.previous=r=i,a=vp(e);(i=i.next)!==r;)(u=vp(i))<a&&(e=i,a=u);r=e.next}for(e=[r._],i=r;(i=i.next)!==r;)e.push(i._);for(i=up(e,n),c=0;c<o;++c)(e=t[c]).x-=i.x,e.y-=i.y;return i.r}function mp(t){return Math.sqrt(t.value)}function xp(t){return function(n){n.children||(n.r=Math.max(0,+t(n)||0))}}function wp(t,n,e){return function(r){if(i=r.children){var i,o,a,u=i.length,c=t(r)*n||0;if(c)for(o=0;o<u;++o)i[o].r+=c;if(a=bp(i,e),c)for(o=0;o<u;++o)i[o].r-=c;r.r=a+c}}}function Mp(t){return function(n){var e=n.parent;n.r*=t,e&&(n.x=e.x+t*n.x,n.y=e.y+t*n.y)}}function Tp(t){t.x0=Math.round(t.x0),t.y0=Math.round(t.y0),t.x1=Math.round(t.x1),t.y1=Math.round(t.y1)}function Ap(t,n,e,r,i){for(var o,a=t.children,u=-1,c=a.length,f=t.value&&(r-n)/t.value;++u<c;)(o=a[u]).y0=e,o.y1=i,o.x0=n,o.x1=n+=o.value*f}var Sp={depth:-1},Ep={},Np={};function kp(t){return t.id}function Cp(t){return t.parentId}function Pp(t){let n=t.length;if(n<2)return"";for(;--n>1&&!zp(t,n););return t.slice(0,n)}function zp(t,n){if("/"===t[n]){let e=0;for(;n>0&&"\\"===t[--n];)++e;if(0==(1&e))return!0}return!1}function $p(t,n){return t.parent===n.parent?1:2}function Dp(t){var n=t.children;return n?n[0]:t.t}function Rp(t){var n=t.children;return n?n[n.length-1]:t.t}function Fp(t,n,e){var r=e/(n.i-t.i);n.c-=r,n.s+=e,t.c+=r,n.z+=e,n.m+=e}function qp(t,n,e){return t.a.parent===n.parent?t.a:e}function Up(t,n){this._=t,this.parent=null,this.children=null,this.A=null,this.a=this,this.z=0,this.m=0,this.c=0,this.s=0,this.t=null,this.i=n}function Ip(t,n,e,r,i){for(var o,a=t.children,u=-1,c=a.length,f=t.value&&(i-e)/t.value;++u<c;)(o=a[u]).x0=n,o.x1=r,o.y0=e,o.y1=e+=o.value*f}Up.prototype=Object.create(Qd.prototype);var Op=(1+Math.sqrt(5))/2;function Bp(t,n,e,r,i,o){for(var a,u,c,f,s,l,h,d,p,g,y,v=[],_=n.children,b=0,m=0,x=_.length,w=n.value;b<x;){c=i-e,f=o-r;do{s=_[m++].value}while(!s&&m<x);for(l=h=s,y=s*s*(g=Math.max(f/c,c/f)/(w*t)),p=Math.max(h/y,y/l);m<x;++m){if(s+=u=_[m].value,u<l&&(l=u),u>h&&(h=u),y=s*s*g,(d=Math.max(h/y,y/l))>p){s-=u;break}p=d}v.push(a={value:s,dice:c<f,children:_.slice(b,m)}),a.dice?Ap(a,e,r,i,w?r+=f*s/w:o):Ip(a,e,r,w?e+=c*s/w:i,o),w-=s,b=m}return v}var Yp=function t(n){function e(t,e,r,i,o){Bp(n,t,e,r,i,o)}return e.ratio=function(n){return t((n=+n)>1?n:1)},e}(Op);var Lp=function t(n){function e(t,e,r,i,o){if((a=t._squarify)&&a.ratio===n)for(var a,u,c,f,s,l=-1,h=a.length,d=t.value;++l<h;){for(c=(u=a[l]).children,f=u.value=0,s=c.length;f<s;++f)u.value+=c[f].value;u.dice?Ap(u,e,r,i,d?r+=(o-r)*u.value/d:o):Ip(u,e,r,d?e+=(i-e)*u.value/d:i,o),d-=u.value}else t._squarify=a=Bp(n,t,e,r,i,o),a.ratio=n}return e.ratio=function(n){return t((n=+n)>1?n:1)},e}(Op);function jp(t,n,e){return(n[0]-t[0])*(e[1]-t[1])-(n[1]-t[1])*(e[0]-t[0])}function Hp(t,n){return t[0]-n[0]||t[1]-n[1]}function Xp(t){const n=t.length,e=[0,1];let r,i=2;for(r=2;r<n;++r){for(;i>1&&jp(t[e[i-2]],t[e[i-1]],t[r])<=0;)--i;e[i++]=r}return e.slice(0,i)}var Gp=Math.random,Vp=function t(n){function e(t,e){return t=null==t?0:+t,e=null==e?1:+e,1===arguments.length?(e=t,t=0):e-=t,function(){return n()*e+t}}return e.source=t,e}(Gp),Wp=function t(n){function e(t,e){return arguments.length<2&&(e=t,t=0),t=Math.floor(t),e=Math.floor(e)-t,function(){return Math.floor(n()*e+t)}}return e.source=t,e}(Gp),Zp=function t(n){function e(t,e){var r,i;return t=null==t?0:+t,e=null==e?1:+e,function(){var o;if(null!=r)o=r,r=null;else do{r=2*n()-1,o=2*n()-1,i=r*r+o*o}while(!i||i>1);return t+e*o*Math.sqrt(-2*Math.log(i)/i)}}return e.source=t,e}(Gp),Kp=function t(n){var e=Zp.source(n);function r(){var t=e.apply(this,arguments);return function(){return Math.exp(t())}}return r.source=t,r}(Gp),Qp=function t(n){function e(t){return(t=+t)<=0?()=>0:function(){for(var e=0,r=t;r>1;--r)e+=n();return e+r*n()}}return e.source=t,e}(Gp),Jp=function t(n){var e=Qp.source(n);function r(t){if(0==(t=+t))return n;var r=e(t);return function(){return r()/t}}return r.source=t,r}(Gp),tg=function t(n){function e(t){return function(){return-Math.log1p(-n())/t}}return e.source=t,e}(Gp),ng=function t(n){function e(t){if((t=+t)<0)throw new RangeError("invalid alpha");return t=1/-t,function(){return Math.pow(1-n(),t)}}return e.source=t,e}(Gp),eg=function t(n){function e(t){if((t=+t)<0||t>1)throw new RangeError("invalid p");return function(){return Math.floor(n()+t)}}return e.source=t,e}(Gp),rg=function t(n){function e(t){if((t=+t)<0||t>1)throw new RangeError("invalid p");return 0===t?()=>1/0:1===t?()=>1:(t=Math.log1p(-t),function(){return 1+Math.floor(Math.log1p(-n())/t)})}return e.source=t,e}(Gp),ig=function t(n){var e=Zp.source(n)();function r(t,r){if((t=+t)<0)throw new RangeError("invalid k");if(0===t)return()=>0;if(r=null==r?1:+r,1===t)return()=>-Math.log1p(-n())*r;var i=(t<1?t+1:t)-1/3,o=1/(3*Math.sqrt(i)),a=t<1?()=>Math.pow(n(),1/t):()=>1;return function(){do{do{var t=e(),u=1+o*t}while(u<=0);u*=u*u;var c=1-n()}while(c>=1-.0331*t*t*t*t&&Math.log(c)>=.5*t*t+i*(1-u+Math.log(u)));return i*u*a()*r}}return r.source=t,r}(Gp),og=function t(n){var e=ig.source(n);function r(t,n){var r=e(t),i=e(n);return function(){var t=r();return 0===t?0:t/(t+i())}}return r.source=t,r}(Gp),ag=function t(n){var e=rg.source(n),r=og.source(n);function i(t,n){return t=+t,(n=+n)>=1?()=>t:n<=0?()=>0:function(){for(var i=0,o=t,a=n;o*a>16&&o*(1-a)>16;){var u=Math.floor((o+1)*a),c=r(u,o-u+1)();c<=a?(i+=u,o-=u,a=(a-c)/(1-c)):(o=u-1,a/=c)}for(var f=a<.5,s=e(f?a:1-a),l=s(),h=0;l<=o;++h)l+=s();return i+(f?h:o-h)}}return i.source=t,i}(Gp),ug=function t(n){function e(t,e,r){var i;return 0==(t=+t)?i=t=>-Math.log(t):(t=1/t,i=n=>Math.pow(n,t)),e=null==e?0:+e,r=null==r?1:+r,function(){return e+r*i(-Math.log1p(-n()))}}return e.source=t,e}(Gp),cg=function t(n){function e(t,e){return t=null==t?0:+t,e=null==e?1:+e,function(){return t+e*Math.tan(Math.PI*n())}}return e.source=t,e}(Gp),fg=function t(n){function e(t,e){return t=null==t?0:+t,e=null==e?1:+e,function(){var r=n();return t+e*Math.log(r/(1-r))}}return e.source=t,e}(Gp),sg=function t(n){var e=ig.source(n),r=ag.source(n);function i(t){return function(){for(var i=0,o=t;o>16;){var a=Math.floor(.875*o),u=e(a)();if(u>o)return i+r(a-1,o/u)();i+=a,o-=u}for(var c=-Math.log1p(-n()),f=0;c<=o;++f)c-=Math.log1p(-n());return i+f}}return i.source=t,i}(Gp);const lg=1/4294967296;function hg(t,n){switch(arguments.length){case 0:break;case 1:this.range(t);break;default:this.range(n).domain(t)}return this}function dg(t,n){switch(arguments.length){case 0:break;case 1:"function"==typeof t?this.interpolator(t):this.range(t);break;default:this.domain(t),"function"==typeof n?this.interpolator(n):this.range(n)}return this}const pg=Symbol("implicit");function gg(){var t=new InternMap,n=[],e=[],r=pg;function i(i){let o=t.get(i);if(void 0===o){if(r!==pg)return r;t.set(i,o=n.push(i)-1)}return e[o%e.length]}return i.domain=function(e){if(!arguments.length)return n.slice();n=[],t=new InternMap;for(const r of e)t.has(r)||t.set(r,n.push(r)-1);return i},i.range=function(t){return arguments.length?(e=Array.from(t),i):e.slice()},i.unknown=function(t){return arguments.length?(r=t,i):r},i.copy=function(){return gg(n,e).unknown(r)},hg.apply(i,arguments),i}function yg(){var t,n,e=gg().unknown(void 0),r=e.domain,i=e.range,o=0,a=1,u=!1,c=0,f=0,s=.5;function l(){var e=r().length,l=a<o,h=l?a:o,d=l?o:a;t=(d-h)/Math.max(1,e-c+2*f),u&&(t=Math.floor(t)),h+=(d-h-t*(e-c))*s,n=t*(1-c),u&&(h=Math.round(h),n=Math.round(n));var p=lt(e).map((function(n){return h+t*n}));return i(l?p.reverse():p)}return delete e.unknown,e.domain=function(t){return arguments.length?(r(t),l()):r()},e.range=function(t){return arguments.length?([o,a]=t,o=+o,a=+a,l()):[o,a]},e.rangeRound=function(t){return[o,a]=t,o=+o,a=+a,u=!0,l()},e.bandwidth=function(){return n},e.step=function(){return t},e.round=function(t){return arguments.length?(u=!!t,l()):u},e.padding=function(t){return arguments.length?(c=Math.min(1,f=+t),l()):c},e.paddingInner=function(t){return arguments.length?(c=Math.min(1,t),l()):c},e.paddingOuter=function(t){return arguments.length?(f=+t,l()):f},e.align=function(t){return arguments.length?(s=Math.max(0,Math.min(1,t)),l()):s},e.copy=function(){return yg(r(),[o,a]).round(u).paddingInner(c).paddingOuter(f).align(s)},hg.apply(l(),arguments)}function vg(t){var n=t.copy;return t.padding=t.paddingOuter,delete t.paddingInner,delete t.paddingOuter,t.copy=function(){return vg(n())},t}function _g(t){return+t}var bg=[0,1];function mg(t){return t}function xg(t,n){return(n-=t=+t)?function(e){return(e-t)/n}:function(t){return function(){return t}}(isNaN(n)?NaN:.5)}function wg(t,n,e){var r=t[0],i=t[1],o=n[0],a=n[1];return i<r?(r=xg(i,r),o=e(a,o)):(r=xg(r,i),o=e(o,a)),function(t){return o(r(t))}}function Mg(t,n,e){var r=Math.min(t.length,n.length)-1,i=new Array(r),o=new Array(r),a=-1;for(t[r]<t[0]&&(t=t.slice().reverse(),n=n.slice().reverse());++a<r;)i[a]=xg(t[a],t[a+1]),o[a]=e(n[a],n[a+1]);return function(n){var e=s(t,n,1,r)-1;return o[e](i[e](n))}}function Tg(t,n){return n.domain(t.domain()).range(t.range()).interpolate(t.interpolate()).clamp(t.clamp()).unknown(t.unknown())}function Ag(){var t,n,e,r,i,o,a=bg,u=bg,c=Gr,f=mg;function s(){var t=Math.min(a.length,u.length);return f!==mg&&(f=function(t,n){var e;return t>n&&(e=t,t=n,n=e),function(e){return Math.max(t,Math.min(n,e))}}(a[0],a[t-1])),r=t>2?Mg:wg,i=o=null,l}function l(n){return null==n||isNaN(n=+n)?e:(i||(i=r(a.map(t),u,c)))(t(f(n)))}return l.invert=function(e){return f(n((o||(o=r(u,a.map(t),Yr)))(e)))},l.domain=function(t){return arguments.length?(a=Array.from(t,_g),s()):a.slice()},l.range=function(t){return arguments.length?(u=Array.from(t),s()):u.slice()},l.rangeRound=function(t){return u=Array.from(t),c=Vr,s()},l.clamp=function(t){return arguments.length?(f=!!t||mg,s()):f!==mg},l.interpolate=function(t){return arguments.length?(c=t,s()):c},l.unknown=function(t){return arguments.length?(e=t,l):e},function(e,r){return t=e,n=r,s()}}function Sg(){return Ag()(mg,mg)}function Eg(n,e,r,i){var o,a=W(n,e,r);switch((i=Jc(null==i?",f":i)).type){case"s":var u=Math.max(Math.abs(n),Math.abs(e));return null!=i.precision||isNaN(o=lf(a,u))||(i.precision=o),t.formatPrefix(i,u);case"":case"e":case"g":case"p":case"r":null!=i.precision||isNaN(o=hf(a,Math.max(Math.abs(n),Math.abs(e))))||(i.precision=o-("e"===i.type));break;case"f":case"%":null!=i.precision||isNaN(o=sf(a))||(i.precision=o-2*("%"===i.type))}return t.format(i)}function Ng(t){var n=t.domain;return t.ticks=function(t){var e=n();return G(e[0],e[e.length-1],null==t?10:t)},t.tickFormat=function(t,e){var r=n();return Eg(r[0],r[r.length-1],null==t?10:t,e)},t.nice=function(e){null==e&&(e=10);var r,i,o=n(),a=0,u=o.length-1,c=o[a],f=o[u],s=10;for(f<c&&(i=c,c=f,f=i,i=a,a=u,u=i);s-- >0;){if((i=V(c,f,e))===r)return o[a]=c,o[u]=f,n(o);if(i>0)c=Math.floor(c/i)*i,f=Math.ceil(f/i)*i;else{if(!(i<0))break;c=Math.ceil(c*i)/i,f=Math.floor(f*i)/i}r=i}return t},t}function kg(t,n){var e,r=0,i=(t=t.slice()).length-1,o=t[r],a=t[i];return a<o&&(e=r,r=i,i=e,e=o,o=a,a=e),t[r]=n.floor(o),t[i]=n.ceil(a),t}function Cg(t){return Math.log(t)}function Pg(t){return Math.exp(t)}function zg(t){return-Math.log(-t)}function $g(t){return-Math.exp(-t)}function Dg(t){return isFinite(t)?+("1e"+t):t<0?0:t}function Rg(t){return(n,e)=>-t(-n,e)}function Fg(n){const e=n(Cg,Pg),r=e.domain;let i,o,a=10;function u(){return i=function(t){return t===Math.E?Math.log:10===t&&Math.log10||2===t&&Math.log2||(t=Math.log(t),n=>Math.log(n)/t)}(a),o=function(t){return 10===t?Dg:t===Math.E?Math.exp:n=>Math.pow(t,n)}(a),r()[0]<0?(i=Rg(i),o=Rg(o),n(zg,$g)):n(Cg,Pg),e}return e.base=function(t){return arguments.length?(a=+t,u()):a},e.domain=function(t){return arguments.length?(r(t),u()):r()},e.ticks=t=>{const n=r();let e=n[0],u=n[n.length-1];const c=u<e;c&&([e,u]=[u,e]);let f,s,l=i(e),h=i(u);const d=null==t?10:+t;let p=[];if(!(a%1)&&h-l<d){if(l=Math.floor(l),h=Math.ceil(h),e>0){for(;l<=h;++l)for(f=1;f<a;++f)if(s=l<0?f/o(-l):f*o(l),!(s<e)){if(s>u)break;p.push(s)}}else for(;l<=h;++l)for(f=a-1;f>=1;--f)if(s=l>0?f/o(-l):f*o(l),!(s<e)){if(s>u)break;p.push(s)}2*p.length<d&&(p=G(e,u,d))}else p=G(l,h,Math.min(h-l,d)).map(o);return c?p.reverse():p},e.tickFormat=(n,r)=>{if(null==n&&(n=10),null==r&&(r=10===a?"s":","),"function"!=typeof r&&(a%1||null!=(r=Jc(r)).precision||(r.trim=!0),r=t.format(r)),n===1/0)return r;const u=Math.max(1,a*n/e.ticks().length);return t=>{let n=t/o(Math.round(i(t)));return n*a<a-.5&&(n*=a),n<=u?r(t):""}},e.nice=()=>r(kg(r(),{floor:t=>o(Math.floor(i(t))),ceil:t=>o(Math.ceil(i(t)))})),e}function qg(t){return function(n){return Math.sign(n)*Math.log1p(Math.abs(n/t))}}function Ug(t){return function(n){return Math.sign(n)*Math.expm1(Math.abs(n))*t}}function Ig(t){var n=1,e=t(qg(n),Ug(n));return e.constant=function(e){return arguments.length?t(qg(n=+e),Ug(n)):n},Ng(e)}function Og(t){return function(n){return n<0?-Math.pow(-n,t):Math.pow(n,t)}}function Bg(t){return t<0?-Math.sqrt(-t):Math.sqrt(t)}function Yg(t){return t<0?-t*t:t*t}function Lg(t){var n=t(mg,mg),e=1;return n.exponent=function(n){return arguments.length?1===(e=+n)?t(mg,mg):.5===e?t(Bg,Yg):t(Og(e),Og(1/e)):e},Ng(n)}function jg(){var t=Lg(Ag());return t.copy=function(){return Tg(t,jg()).exponent(t.exponent())},hg.apply(t,arguments),t}function Hg(t){return Math.sign(t)*t*t}const Xg=new Date,Gg=new Date;function Vg(t,n,e,r){function i(n){return t(n=0===arguments.length?new Date:new Date(+n)),n}return i.floor=n=>(t(n=new Date(+n)),n),i.ceil=e=>(t(e=new Date(e-1)),n(e,1),t(e),e),i.round=t=>{const n=i(t),e=i.ceil(t);return t-n<e-t?n:e},i.offset=(t,e)=>(n(t=new Date(+t),null==e?1:Math.floor(e)),t),i.range=(e,r,o)=>{const a=[];if(e=i.ceil(e),o=null==o?1:Math.floor(o),!(e<r&&o>0))return a;let u;do{a.push(u=new Date(+e)),n(e,o),t(e)}while(u<e&&e<r);return a},i.filter=e=>Vg((n=>{if(n>=n)for(;t(n),!e(n);)n.setTime(n-1)}),((t,r)=>{if(t>=t)if(r<0)for(;++r<=0;)for(;n(t,-1),!e(t););else for(;--r>=0;)for(;n(t,1),!e(t););})),e&&(i.count=(n,r)=>(Xg.setTime(+n),Gg.setTime(+r),t(Xg),t(Gg),Math.floor(e(Xg,Gg))),i.every=t=>(t=Math.floor(t),isFinite(t)&&t>0?t>1?i.filter(r?n=>r(n)%t==0:n=>i.count(0,n)%t==0):i:null)),i}const Wg=Vg((()=>{}),((t,n)=>{t.setTime(+t+n)}),((t,n)=>n-t));Wg.every=t=>(t=Math.floor(t),isFinite(t)&&t>0?t>1?Vg((n=>{n.setTime(Math.floor(n/t)*t)}),((n,e)=>{n.setTime(+n+e*t)}),((n,e)=>(e-n)/t)):Wg:null);const Zg=Wg.range,Kg=1e3,Qg=6e4,Jg=36e5,ty=864e5,ny=6048e5,ey=2592e6,ry=31536e6,iy=Vg((t=>{t.setTime(t-t.getMilliseconds())}),((t,n)=>{t.setTime(+t+n*Kg)}),((t,n)=>(n-t)/Kg),(t=>t.getUTCSeconds())),oy=iy.range,ay=Vg((t=>{t.setTime(t-t.getMilliseconds()-t.getSeconds()*Kg)}),((t,n)=>{t.setTime(+t+n*Qg)}),((t,n)=>(n-t)/Qg),(t=>t.getMinutes())),uy=ay.range,cy=Vg((t=>{t.setUTCSeconds(0,0)}),((t,n)=>{t.setTime(+t+n*Qg)}),((t,n)=>(n-t)/Qg),(t=>t.getUTCMinutes())),fy=cy.range,sy=Vg((t=>{t.setTime(t-t.getMilliseconds()-t.getSeconds()*Kg-t.getMinutes()*Qg)}),((t,n)=>{t.setTime(+t+n*Jg)}),((t,n)=>(n-t)/Jg),(t=>t.getHours())),ly=sy.range,hy=Vg((t=>{t.setUTCMinutes(0,0,0)}),((t,n)=>{t.setTime(+t+n*Jg)}),((t,n)=>(n-t)/Jg),(t=>t.getUTCHours())),dy=hy.range,py=Vg((t=>t.setHours(0,0,0,0)),((t,n)=>t.setDate(t.getDate()+n)),((t,n)=>(n-t-(n.getTimezoneOffset()-t.getTimezoneOffset())*Qg)/ty),(t=>t.getDate()-1)),gy=py.range,yy=Vg((t=>{t.setUTCHours(0,0,0,0)}),((t,n)=>{t.setUTCDate(t.getUTCDate()+n)}),((t,n)=>(n-t)/ty),(t=>t.getUTCDate()-1)),vy=yy.range,_y=Vg((t=>{t.setUTCHours(0,0,0,0)}),((t,n)=>{t.setUTCDate(t.getUTCDate()+n)}),((t,n)=>(n-t)/ty),(t=>Math.floor(t/ty))),by=_y.range;function my(t){return Vg((n=>{n.setDate(n.getDate()-(n.getDay()+7-t)%7),n.setHours(0,0,0,0)}),((t,n)=>{t.setDate(t.getDate()+7*n)}),((t,n)=>(n-t-(n.getTimezoneOffset()-t.getTimezoneOffset())*Qg)/ny))}const xy=my(0),wy=my(1),My=my(2),Ty=my(3),Ay=my(4),Sy=my(5),Ey=my(6),Ny=xy.range,ky=wy.range,Cy=My.range,Py=Ty.range,zy=Ay.range,$y=Sy.range,Dy=Ey.range;function Ry(t){return Vg((n=>{n.setUTCDate(n.getUTCDate()-(n.getUTCDay()+7-t)%7),n.setUTCHours(0,0,0,0)}),((t,n)=>{t.setUTCDate(t.getUTCDate()+7*n)}),((t,n)=>(n-t)/ny))}const Fy=Ry(0),qy=Ry(1),Uy=Ry(2),Iy=Ry(3),Oy=Ry(4),By=Ry(5),Yy=Ry(6),Ly=Fy.range,jy=qy.range,Hy=Uy.range,Xy=Iy.range,Gy=Oy.range,Vy=By.range,Wy=Yy.range,Zy=Vg((t=>{t.setDate(1),t.setHours(0,0,0,0)}),((t,n)=>{t.setMonth(t.getMonth()+n)}),((t,n)=>n.getMonth()-t.getMonth()+12*(n.getFullYear()-t.getFullYear())),(t=>t.getMonth())),Ky=Zy.range,Qy=Vg((t=>{t.setUTCDate(1),t.setUTCHours(0,0,0,0)}),((t,n)=>{t.setUTCMonth(t.getUTCMonth()+n)}),((t,n)=>n.getUTCMonth()-t.getUTCMonth()+12*(n.getUTCFullYear()-t.getUTCFullYear())),(t=>t.getUTCMonth())),Jy=Qy.range,tv=Vg((t=>{t.setMonth(0,1),t.setHours(0,0,0,0)}),((t,n)=>{t.setFullYear(t.getFullYear()+n)}),((t,n)=>n.getFullYear()-t.getFullYear()),(t=>t.getFullYear()));tv.every=t=>isFinite(t=Math.floor(t))&&t>0?Vg((n=>{n.setFullYear(Math.floor(n.getFullYear()/t)*t),n.setMonth(0,1),n.setHours(0,0,0,0)}),((n,e)=>{n.setFullYear(n.getFullYear()+e*t)})):null;const nv=tv.range,ev=Vg((t=>{t.setUTCMonth(0,1),t.setUTCHours(0,0,0,0)}),((t,n)=>{t.setUTCFullYear(t.getUTCFullYear()+n)}),((t,n)=>n.getUTCFullYear()-t.getUTCFullYear()),(t=>t.getUTCFullYear()));ev.every=t=>isFinite(t=Math.floor(t))&&t>0?Vg((n=>{n.setUTCFullYear(Math.floor(n.getUTCFullYear()/t)*t),n.setUTCMonth(0,1),n.setUTCHours(0,0,0,0)}),((n,e)=>{n.setUTCFullYear(n.getUTCFullYear()+e*t)})):null;const rv=ev.range;function iv(t,n,e,i,o,a){const u=[[iy,1,Kg],[iy,5,5e3],[iy,15,15e3],[iy,30,3e4],[a,1,Qg],[a,5,3e5],[a,15,9e5],[a,30,18e5],[o,1,Jg],[o,3,108e5],[o,6,216e5],[o,12,432e5],[i,1,ty],[i,2,1728e5],[e,1,ny],[n,1,ey],[n,3,7776e6],[t,1,ry]];function c(n,e,i){const o=Math.abs(e-n)/i,a=r((([,,t])=>t)).right(u,o);if(a===u.length)return t.every(W(n/ry,e/ry,i));if(0===a)return Wg.every(Math.max(W(n,e,i),1));const[c,f]=u[o/u[a-1][2]<u[a][2]/o?a-1:a];return c.every(f)}return[function(t,n,e){const r=n<t;r&&([t,n]=[n,t]);const i=e&&"function"==typeof e.range?e:c(t,n,e),o=i?i.range(t,+n+1):[];return r?o.reverse():o},c]}const[ov,av]=iv(ev,Qy,Fy,_y,hy,cy),[uv,cv]=iv(tv,Zy,xy,py,sy,ay);function fv(t){if(0<=t.y&&t.y<100){var n=new Date(-1,t.m,t.d,t.H,t.M,t.S,t.L);return n.setFullYear(t.y),n}return new Date(t.y,t.m,t.d,t.H,t.M,t.S,t.L)}function sv(t){if(0<=t.y&&t.y<100){var n=new Date(Date.UTC(-1,t.m,t.d,t.H,t.M,t.S,t.L));return n.setUTCFullYear(t.y),n}return new Date(Date.UTC(t.y,t.m,t.d,t.H,t.M,t.S,t.L))}function lv(t,n,e){return{y:t,m:n,d:e,H:0,M:0,S:0,L:0}}function hv(t){var n=t.dateTime,e=t.date,r=t.time,i=t.periods,o=t.days,a=t.shortDays,u=t.months,c=t.shortMonths,f=mv(i),s=xv(i),l=mv(o),h=xv(o),d=mv(a),p=xv(a),g=mv(u),y=xv(u),v=mv(c),_=xv(c),b={a:function(t){return a[t.getDay()]},A:function(t){return o[t.getDay()]},b:function(t){return c[t.getMonth()]},B:function(t){return u[t.getMonth()]},c:null,d:Yv,e:Yv,f:Gv,g:i_,G:a_,H:Lv,I:jv,j:Hv,L:Xv,m:Vv,M:Wv,p:function(t){return i[+(t.getHours()>=12)]},q:function(t){return 1+~~(t.getMonth()/3)},Q:k_,s:C_,S:Zv,u:Kv,U:Qv,V:t_,w:n_,W:e_,x:null,X:null,y:r_,Y:o_,Z:u_,"%":N_},m={a:function(t){return a[t.getUTCDay()]},A:function(t){return o[t.getUTCDay()]},b:function(t){return c[t.getUTCMonth()]},B:function(t){return u[t.getUTCMonth()]},c:null,d:c_,e:c_,f:d_,g:T_,G:S_,H:f_,I:s_,j:l_,L:h_,m:p_,M:g_,p:function(t){return i[+(t.getUTCHours()>=12)]},q:function(t){return 1+~~(t.getUTCMonth()/3)},Q:k_,s:C_,S:y_,u:v_,U:__,V:m_,w:x_,W:w_,x:null,X:null,y:M_,Y:A_,Z:E_,"%":N_},x={a:function(t,n,e){var r=d.exec(n.slice(e));return r?(t.w=p.get(r[0].toLowerCase()),e+r[0].length):-1},A:function(t,n,e){var r=l.exec(n.slice(e));return r?(t.w=h.get(r[0].toLowerCase()),e+r[0].length):-1},b:function(t,n,e){var r=v.exec(n.slice(e));return r?(t.m=_.get(r[0].toLowerCase()),e+r[0].length):-1},B:function(t,n,e){var r=g.exec(n.slice(e));return r?(t.m=y.get(r[0].toLowerCase()),e+r[0].length):-1},c:function(t,e,r){return T(t,n,e,r)},d:zv,e:zv,f:Uv,g:Nv,G:Ev,H:Dv,I:Dv,j:$v,L:qv,m:Pv,M:Rv,p:function(t,n,e){var r=f.exec(n.slice(e));return r?(t.p=s.get(r[0].toLowerCase()),e+r[0].length):-1},q:Cv,Q:Ov,s:Bv,S:Fv,u:Mv,U:Tv,V:Av,w:wv,W:Sv,x:function(t,n,r){return T(t,e,n,r)},X:function(t,n,e){return T(t,r,n,e)},y:Nv,Y:Ev,Z:kv,"%":Iv};function w(t,n){return function(e){var r,i,o,a=[],u=-1,c=0,f=t.length;for(e instanceof Date||(e=new Date(+e));++u<f;)37===t.charCodeAt(u)&&(a.push(t.slice(c,u)),null!=(i=pv[r=t.charAt(++u)])?r=t.charAt(++u):i="e"===r?" ":"0",(o=n[r])&&(r=o(e,i)),a.push(r),c=u+1);return a.push(t.slice(c,u)),a.join("")}}function M(t,n){return function(e){var r,i,o=lv(1900,void 0,1);if(T(o,t,e+="",0)!=e.length)return null;if("Q"in o)return new Date(o.Q);if("s"in o)return new Date(1e3*o.s+("L"in o?o.L:0));if(n&&!("Z"in o)&&(o.Z=0),"p"in o&&(o.H=o.H%12+12*o.p),void 0===o.m&&(o.m="q"in o?o.q:0),"V"in o){if(o.V<1||o.V>53)return null;"w"in o||(o.w=1),"Z"in o?(i=(r=sv(lv(o.y,0,1))).getUTCDay(),r=i>4||0===i?qy.ceil(r):qy(r),r=yy.offset(r,7*(o.V-1)),o.y=r.getUTCFullYear(),o.m=r.getUTCMonth(),o.d=r.getUTCDate()+(o.w+6)%7):(i=(r=fv(lv(o.y,0,1))).getDay(),r=i>4||0===i?wy.ceil(r):wy(r),r=py.offset(r,7*(o.V-1)),o.y=r.getFullYear(),o.m=r.getMonth(),o.d=r.getDate()+(o.w+6)%7)}else("W"in o||"U"in o)&&("w"in o||(o.w="u"in o?o.u%7:"W"in o?1:0),i="Z"in o?sv(lv(o.y,0,1)).getUTCDay():fv(lv(o.y,0,1)).getDay(),o.m=0,o.d="W"in o?(o.w+6)%7+7*o.W-(i+5)%7:o.w+7*o.U-(i+6)%7);return"Z"in o?(o.H+=o.Z/100|0,o.M+=o.Z%100,sv(o)):fv(o)}}function T(t,n,e,r){for(var i,o,a=0,u=n.length,c=e.length;a<u;){if(r>=c)return-1;if(37===(i=n.charCodeAt(a++))){if(i=n.charAt(a++),!(o=x[i in pv?n.charAt(a++):i])||(r=o(t,e,r))<0)return-1}else if(i!=e.charCodeAt(r++))return-1}return r}return b.x=w(e,b),b.X=w(r,b),b.c=w(n,b),m.x=w(e,m),m.X=w(r,m),m.c=w(n,m),{format:function(t){var n=w(t+="",b);return n.toString=function(){return t},n},parse:function(t){var n=M(t+="",!1);return n.toString=function(){return t},n},utcFormat:function(t){var n=w(t+="",m);return n.toString=function(){return t},n},utcParse:function(t){var n=M(t+="",!0);return n.toString=function(){return t},n}}}var dv,pv={"-":"",_:" ",0:"0"},gv=/^\s*\d+/,yv=/^%/,vv=/[\\^$*+?|[\]().{}]/g;function _v(t,n,e){var r=t<0?"-":"",i=(r?-t:t)+"",o=i.length;return r+(o<e?new Array(e-o+1).join(n)+i:i)}function bv(t){return t.replace(vv,"\\$&")}function mv(t){return new RegExp("^(?:"+t.map(bv).join("|")+")","i")}function xv(t){return new Map(t.map(((t,n)=>[t.toLowerCase(),n])))}function wv(t,n,e){var r=gv.exec(n.slice(e,e+1));return r?(t.w=+r[0],e+r[0].length):-1}function Mv(t,n,e){var r=gv.exec(n.slice(e,e+1));return r?(t.u=+r[0],e+r[0].length):-1}function Tv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.U=+r[0],e+r[0].length):-1}function Av(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.V=+r[0],e+r[0].length):-1}function Sv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.W=+r[0],e+r[0].length):-1}function Ev(t,n,e){var r=gv.exec(n.slice(e,e+4));return r?(t.y=+r[0],e+r[0].length):-1}function Nv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.y=+r[0]+(+r[0]>68?1900:2e3),e+r[0].length):-1}function kv(t,n,e){var r=/^(Z)|([+-]\d\d)(?::?(\d\d))?/.exec(n.slice(e,e+6));return r?(t.Z=r[1]?0:-(r[2]+(r[3]||"00")),e+r[0].length):-1}function Cv(t,n,e){var r=gv.exec(n.slice(e,e+1));return r?(t.q=3*r[0]-3,e+r[0].length):-1}function Pv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.m=r[0]-1,e+r[0].length):-1}function zv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.d=+r[0],e+r[0].length):-1}function $v(t,n,e){var r=gv.exec(n.slice(e,e+3));return r?(t.m=0,t.d=+r[0],e+r[0].length):-1}function Dv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.H=+r[0],e+r[0].length):-1}function Rv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.M=+r[0],e+r[0].length):-1}function Fv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.S=+r[0],e+r[0].length):-1}function qv(t,n,e){var r=gv.exec(n.slice(e,e+3));return r?(t.L=+r[0],e+r[0].length):-1}function Uv(t,n,e){var r=gv.exec(n.slice(e,e+6));return r?(t.L=Math.floor(r[0]/1e3),e+r[0].length):-1}function Iv(t,n,e){var r=yv.exec(n.slice(e,e+1));return r?e+r[0].length:-1}function Ov(t,n,e){var r=gv.exec(n.slice(e));return r?(t.Q=+r[0],e+r[0].length):-1}function Bv(t,n,e){var r=gv.exec(n.slice(e));return r?(t.s=+r[0],e+r[0].length):-1}function Yv(t,n){return _v(t.getDate(),n,2)}function Lv(t,n){return _v(t.getHours(),n,2)}function jv(t,n){return _v(t.getHours()%12||12,n,2)}function Hv(t,n){return _v(1+py.count(tv(t),t),n,3)}function Xv(t,n){return _v(t.getMilliseconds(),n,3)}function Gv(t,n){return Xv(t,n)+"000"}function Vv(t,n){return _v(t.getMonth()+1,n,2)}function Wv(t,n){return _v(t.getMinutes(),n,2)}function Zv(t,n){return _v(t.getSeconds(),n,2)}function Kv(t){var n=t.getDay();return 0===n?7:n}function Qv(t,n){return _v(xy.count(tv(t)-1,t),n,2)}function Jv(t){var n=t.getDay();return n>=4||0===n?Ay(t):Ay.ceil(t)}function t_(t,n){return t=Jv(t),_v(Ay.count(tv(t),t)+(4===tv(t).getDay()),n,2)}function n_(t){return t.getDay()}function e_(t,n){return _v(wy.count(tv(t)-1,t),n,2)}function r_(t,n){return _v(t.getFullYear()%100,n,2)}function i_(t,n){return _v((t=Jv(t)).getFullYear()%100,n,2)}function o_(t,n){return _v(t.getFullYear()%1e4,n,4)}function a_(t,n){var e=t.getDay();return _v((t=e>=4||0===e?Ay(t):Ay.ceil(t)).getFullYear()%1e4,n,4)}function u_(t){var n=t.getTimezoneOffset();return(n>0?"-":(n*=-1,"+"))+_v(n/60|0,"0",2)+_v(n%60,"0",2)}function c_(t,n){return _v(t.getUTCDate(),n,2)}function f_(t,n){return _v(t.getUTCHours(),n,2)}function s_(t,n){return _v(t.getUTCHours()%12||12,n,2)}function l_(t,n){return _v(1+yy.count(ev(t),t),n,3)}function h_(t,n){return _v(t.getUTCMilliseconds(),n,3)}function d_(t,n){return h_(t,n)+"000"}function p_(t,n){return _v(t.getUTCMonth()+1,n,2)}function g_(t,n){return _v(t.getUTCMinutes(),n,2)}function y_(t,n){return _v(t.getUTCSeconds(),n,2)}function v_(t){var n=t.getUTCDay();return 0===n?7:n}function __(t,n){return _v(Fy.count(ev(t)-1,t),n,2)}function b_(t){var n=t.getUTCDay();return n>=4||0===n?Oy(t):Oy.ceil(t)}function m_(t,n){return t=b_(t),_v(Oy.count(ev(t),t)+(4===ev(t).getUTCDay()),n,2)}function x_(t){return t.getUTCDay()}function w_(t,n){return _v(qy.count(ev(t)-1,t),n,2)}function M_(t,n){return _v(t.getUTCFullYear()%100,n,2)}function T_(t,n){return _v((t=b_(t)).getUTCFullYear()%100,n,2)}function A_(t,n){return _v(t.getUTCFullYear()%1e4,n,4)}function S_(t,n){var e=t.getUTCDay();return _v((t=e>=4||0===e?Oy(t):Oy.ceil(t)).getUTCFullYear()%1e4,n,4)}function E_(){return"+0000"}function N_(){return"%"}function k_(t){return+t}function C_(t){return Math.floor(+t/1e3)}function P_(n){return dv=hv(n),t.timeFormat=dv.format,t.timeParse=dv.parse,t.utcFormat=dv.utcFormat,t.utcParse=dv.utcParse,dv}t.timeFormat=void 0,t.timeParse=void 0,t.utcFormat=void 0,t.utcParse=void 0,P_({dateTime:"%x, %X",date:"%-m/%-d/%Y",time:"%-I:%M:%S %p",periods:["AM","PM"],days:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],shortDays:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],months:["January","February","March","April","May","June","July","August","September","October","November","December"],shortMonths:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]});var z_="%Y-%m-%dT%H:%M:%S.%LZ";var $_=Date.prototype.toISOString?function(t){return t.toISOString()}:t.utcFormat(z_),D_=$_;var R_=+new Date("2000-01-01T00:00:00.000Z")?function(t){var n=new Date(t);return isNaN(n)?null:n}:t.utcParse(z_),F_=R_;function q_(t){return new Date(t)}function U_(t){return t instanceof Date?+t:+new Date(+t)}function I_(t,n,e,r,i,o,a,u,c,f){var s=Sg(),l=s.invert,h=s.domain,d=f(".%L"),p=f(":%S"),g=f("%I:%M"),y=f("%I %p"),v=f("%a %d"),_=f("%b %d"),b=f("%B"),m=f("%Y");function x(t){return(c(t)<t?d:u(t)<t?p:a(t)<t?g:o(t)<t?y:r(t)<t?i(t)<t?v:_:e(t)<t?b:m)(t)}return s.invert=function(t){return new Date(l(t))},s.domain=function(t){return arguments.length?h(Array.from(t,U_)):h().map(q_)},s.ticks=function(n){var e=h();return t(e[0],e[e.length-1],null==n?10:n)},s.tickFormat=function(t,n){return null==n?x:f(n)},s.nice=function(t){var e=h();return t&&"function"==typeof t.range||(t=n(e[0],e[e.length-1],null==t?10:t)),t?h(kg(e,t)):s},s.copy=function(){return Tg(s,I_(t,n,e,r,i,o,a,u,c,f))},s}function O_(){var t,n,e,r,i,o=0,a=1,u=mg,c=!1;function f(n){return null==n||isNaN(n=+n)?i:u(0===e?.5:(n=(r(n)-t)*e,c?Math.max(0,Math.min(1,n)):n))}function s(t){return function(n){var e,r;return arguments.length?([e,r]=n,u=t(e,r),f):[u(0),u(1)]}}return f.domain=function(i){return arguments.length?([o,a]=i,t=r(o=+o),n=r(a=+a),e=t===n?0:1/(n-t),f):[o,a]},f.clamp=function(t){return arguments.length?(c=!!t,f):c},f.interpolator=function(t){return arguments.length?(u=t,f):u},f.range=s(Gr),f.rangeRound=s(Vr),f.unknown=function(t){return arguments.length?(i=t,f):i},function(i){return r=i,t=i(o),n=i(a),e=t===n?0:1/(n-t),f}}function B_(t,n){return n.domain(t.domain()).interpolator(t.interpolator()).clamp(t.clamp()).unknown(t.unknown())}function Y_(){var t=Lg(O_());return t.copy=function(){return B_(t,Y_()).exponent(t.exponent())},dg.apply(t,arguments)}function L_(){var t,n,e,r,i,o,a,u=0,c=.5,f=1,s=1,l=mg,h=!1;function d(t){return isNaN(t=+t)?a:(t=.5+((t=+o(t))-n)*(s*t<s*n?r:i),l(h?Math.max(0,Math.min(1,t)):t))}function p(t){return function(n){var e,r,i;return arguments.length?([e,r,i]=n,l=di(t,[e,r,i]),d):[l(0),l(.5),l(1)]}}return d.domain=function(a){return arguments.length?([u,c,f]=a,t=o(u=+u),n=o(c=+c),e=o(f=+f),r=t===n?0:.5/(n-t),i=n===e?0:.5/(e-n),s=n<t?-1:1,d):[u,c,f]},d.clamp=function(t){return arguments.length?(h=!!t,d):h},d.interpolator=function(t){return arguments.length?(l=t,d):l},d.range=p(Gr),d.rangeRound=p(Vr),d.unknown=function(t){return arguments.length?(a=t,d):a},function(a){return o=a,t=a(u),n=a(c),e=a(f),r=t===n?0:.5/(n-t),i=n===e?0:.5/(e-n),s=n<t?-1:1,d}}function j_(){var t=Lg(L_());return t.copy=function(){return B_(t,j_()).exponent(t.exponent())},dg.apply(t,arguments)}function H_(t){for(var n=t.length/6|0,e=new Array(n),r=0;r<n;)e[r]="#"+t.slice(6*r,6*++r);return e}var X_=H_("1f77b4ff7f0e2ca02cd627289467bd8c564be377c27f7f7fbcbd2217becf"),G_=H_("7fc97fbeaed4fdc086ffff99386cb0f0027fbf5b17666666"),V_=H_("1b9e77d95f027570b3e7298a66a61ee6ab02a6761d666666"),W_=H_("a6cee31f78b4b2df8a33a02cfb9a99e31a1cfdbf6fff7f00cab2d66a3d9affff99b15928"),Z_=H_("fbb4aeb3cde3ccebc5decbe4fed9a6ffffcce5d8bdfddaecf2f2f2"),K_=H_("b3e2cdfdcdaccbd5e8f4cae4e6f5c9fff2aef1e2cccccccc"),Q_=H_("e41a1c377eb84daf4a984ea3ff7f00ffff33a65628f781bf999999"),J_=H_("66c2a5fc8d628da0cbe78ac3a6d854ffd92fe5c494b3b3b3"),tb=H_("8dd3c7ffffb3bebadafb807280b1d3fdb462b3de69fccde5d9d9d9bc80bdccebc5ffed6f"),nb=H_("4e79a7f28e2ce1575976b7b259a14fedc949af7aa1ff9da79c755fbab0ab"),eb=t=>Fr(t[t.length-1]),rb=new Array(3).concat("d8b365f5f5f55ab4ac","a6611adfc27d80cdc1018571","a6611adfc27df5f5f580cdc1018571","8c510ad8b365f6e8c3c7eae55ab4ac01665e","8c510ad8b365f6e8c3f5f5f5c7eae55ab4ac01665e","8c510abf812ddfc27df6e8c3c7eae580cdc135978f01665e","8c510abf812ddfc27df6e8c3f5f5f5c7eae580cdc135978f01665e","5430058c510abf812ddfc27df6e8c3c7eae580cdc135978f01665e003c30","5430058c510abf812ddfc27df6e8c3f5f5f5c7eae580cdc135978f01665e003c30").map(H_),ib=eb(rb),ob=new Array(3).concat("af8dc3f7f7f77fbf7b","7b3294c2a5cfa6dba0008837","7b3294c2a5cff7f7f7a6dba0008837","762a83af8dc3e7d4e8d9f0d37fbf7b1b7837","762a83af8dc3e7d4e8f7f7f7d9f0d37fbf7b1b7837","762a839970abc2a5cfe7d4e8d9f0d3a6dba05aae611b7837","762a839970abc2a5cfe7d4e8f7f7f7d9f0d3a6dba05aae611b7837","40004b762a839970abc2a5cfe7d4e8d9f0d3a6dba05aae611b783700441b","40004b762a839970abc2a5cfe7d4e8f7f7f7d9f0d3a6dba05aae611b783700441b").map(H_),ab=eb(ob),ub=new Array(3).concat("e9a3c9f7f7f7a1d76a","d01c8bf1b6dab8e1864dac26","d01c8bf1b6daf7f7f7b8e1864dac26","c51b7de9a3c9fde0efe6f5d0a1d76a4d9221","c51b7de9a3c9fde0eff7f7f7e6f5d0a1d76a4d9221","c51b7dde77aef1b6dafde0efe6f5d0b8e1867fbc414d9221","c51b7dde77aef1b6dafde0eff7f7f7e6f5d0b8e1867fbc414d9221","8e0152c51b7dde77aef1b6dafde0efe6f5d0b8e1867fbc414d9221276419","8e0152c51b7dde77aef1b6dafde0eff7f7f7e6f5d0b8e1867fbc414d9221276419").map(H_),cb=eb(ub),fb=new Array(3).concat("998ec3f7f7f7f1a340","5e3c99b2abd2fdb863e66101","5e3c99b2abd2f7f7f7fdb863e66101","542788998ec3d8daebfee0b6f1a340b35806","542788998ec3d8daebf7f7f7fee0b6f1a340b35806","5427888073acb2abd2d8daebfee0b6fdb863e08214b35806","5427888073acb2abd2d8daebf7f7f7fee0b6fdb863e08214b35806","2d004b5427888073acb2abd2d8daebfee0b6fdb863e08214b358067f3b08","2d004b5427888073acb2abd2d8daebf7f7f7fee0b6fdb863e08214b358067f3b08").map(H_),sb=eb(fb),lb=new Array(3).concat("ef8a62f7f7f767a9cf","ca0020f4a58292c5de0571b0","ca0020f4a582f7f7f792c5de0571b0","b2182bef8a62fddbc7d1e5f067a9cf2166ac","b2182bef8a62fddbc7f7f7f7d1e5f067a9cf2166ac","b2182bd6604df4a582fddbc7d1e5f092c5de4393c32166ac","b2182bd6604df4a582fddbc7f7f7f7d1e5f092c5de4393c32166ac","67001fb2182bd6604df4a582fddbc7d1e5f092c5de4393c32166ac053061","67001fb2182bd6604df4a582fddbc7f7f7f7d1e5f092c5de4393c32166ac053061").map(H_),hb=eb(lb),db=new Array(3).concat("ef8a62ffffff999999","ca0020f4a582bababa404040","ca0020f4a582ffffffbababa404040","b2182bef8a62fddbc7e0e0e09999994d4d4d","b2182bef8a62fddbc7ffffffe0e0e09999994d4d4d","b2182bd6604df4a582fddbc7e0e0e0bababa8787874d4d4d","b2182bd6604df4a582fddbc7ffffffe0e0e0bababa8787874d4d4d","67001fb2182bd6604df4a582fddbc7e0e0e0bababa8787874d4d4d1a1a1a","67001fb2182bd6604df4a582fddbc7ffffffe0e0e0bababa8787874d4d4d1a1a1a").map(H_),pb=eb(db),gb=new Array(3).concat("fc8d59ffffbf91bfdb","d7191cfdae61abd9e92c7bb6","d7191cfdae61ffffbfabd9e92c7bb6","d73027fc8d59fee090e0f3f891bfdb4575b4","d73027fc8d59fee090ffffbfe0f3f891bfdb4575b4","d73027f46d43fdae61fee090e0f3f8abd9e974add14575b4","d73027f46d43fdae61fee090ffffbfe0f3f8abd9e974add14575b4","a50026d73027f46d43fdae61fee090e0f3f8abd9e974add14575b4313695","a50026d73027f46d43fdae61fee090ffffbfe0f3f8abd9e974add14575b4313695").map(H_),yb=eb(gb),vb=new Array(3).concat("fc8d59ffffbf91cf60","d7191cfdae61a6d96a1a9641","d7191cfdae61ffffbfa6d96a1a9641","d73027fc8d59fee08bd9ef8b91cf601a9850","d73027fc8d59fee08bffffbfd9ef8b91cf601a9850","d73027f46d43fdae61fee08bd9ef8ba6d96a66bd631a9850","d73027f46d43fdae61fee08bffffbfd9ef8ba6d96a66bd631a9850","a50026d73027f46d43fdae61fee08bd9ef8ba6d96a66bd631a9850006837","a50026d73027f46d43fdae61fee08bffffbfd9ef8ba6d96a66bd631a9850006837").map(H_),_b=eb(vb),bb=new Array(3).concat("fc8d59ffffbf99d594","d7191cfdae61abdda42b83ba","d7191cfdae61ffffbfabdda42b83ba","d53e4ffc8d59fee08be6f59899d5943288bd","d53e4ffc8d59fee08bffffbfe6f59899d5943288bd","d53e4ff46d43fdae61fee08be6f598abdda466c2a53288bd","d53e4ff46d43fdae61fee08bffffbfe6f598abdda466c2a53288bd","9e0142d53e4ff46d43fdae61fee08be6f598abdda466c2a53288bd5e4fa2","9e0142d53e4ff46d43fdae61fee08bffffbfe6f598abdda466c2a53288bd5e4fa2").map(H_),mb=eb(bb),xb=new Array(3).concat("e5f5f999d8c92ca25f","edf8fbb2e2e266c2a4238b45","edf8fbb2e2e266c2a42ca25f006d2c","edf8fbccece699d8c966c2a42ca25f006d2c","edf8fbccece699d8c966c2a441ae76238b45005824","f7fcfde5f5f9ccece699d8c966c2a441ae76238b45005824","f7fcfde5f5f9ccece699d8c966c2a441ae76238b45006d2c00441b").map(H_),wb=eb(xb),Mb=new Array(3).concat("e0ecf49ebcda8856a7","edf8fbb3cde38c96c688419d","edf8fbb3cde38c96c68856a7810f7c","edf8fbbfd3e69ebcda8c96c68856a7810f7c","edf8fbbfd3e69ebcda8c96c68c6bb188419d6e016b","f7fcfde0ecf4bfd3e69ebcda8c96c68c6bb188419d6e016b","f7fcfde0ecf4bfd3e69ebcda8c96c68c6bb188419d810f7c4d004b").map(H_),Tb=eb(Mb),Ab=new Array(3).concat("e0f3dba8ddb543a2ca","f0f9e8bae4bc7bccc42b8cbe","f0f9e8bae4bc7bccc443a2ca0868ac","f0f9e8ccebc5a8ddb57bccc443a2ca0868ac","f0f9e8ccebc5a8ddb57bccc44eb3d32b8cbe08589e","f7fcf0e0f3dbccebc5a8ddb57bccc44eb3d32b8cbe08589e","f7fcf0e0f3dbccebc5a8ddb57bccc44eb3d32b8cbe0868ac084081").map(H_),Sb=eb(Ab),Eb=new Array(3).concat("fee8c8fdbb84e34a33","fef0d9fdcc8afc8d59d7301f","fef0d9fdcc8afc8d59e34a33b30000","fef0d9fdd49efdbb84fc8d59e34a33b30000","fef0d9fdd49efdbb84fc8d59ef6548d7301f990000","fff7ecfee8c8fdd49efdbb84fc8d59ef6548d7301f990000","fff7ecfee8c8fdd49efdbb84fc8d59ef6548d7301fb300007f0000").map(H_),Nb=eb(Eb),kb=new Array(3).concat("ece2f0a6bddb1c9099","f6eff7bdc9e167a9cf02818a","f6eff7bdc9e167a9cf1c9099016c59","f6eff7d0d1e6a6bddb67a9cf1c9099016c59","f6eff7d0d1e6a6bddb67a9cf3690c002818a016450","fff7fbece2f0d0d1e6a6bddb67a9cf3690c002818a016450","fff7fbece2f0d0d1e6a6bddb67a9cf3690c002818a016c59014636").map(H_),Cb=eb(kb),Pb=new Array(3).concat("ece7f2a6bddb2b8cbe","f1eef6bdc9e174a9cf0570b0","f1eef6bdc9e174a9cf2b8cbe045a8d","f1eef6d0d1e6a6bddb74a9cf2b8cbe045a8d","f1eef6d0d1e6a6bddb74a9cf3690c00570b0034e7b","fff7fbece7f2d0d1e6a6bddb74a9cf3690c00570b0034e7b","fff7fbece7f2d0d1e6a6bddb74a9cf3690c00570b0045a8d023858").map(H_),zb=eb(Pb),$b=new Array(3).concat("e7e1efc994c7dd1c77","f1eef6d7b5d8df65b0ce1256","f1eef6d7b5d8df65b0dd1c77980043","f1eef6d4b9dac994c7df65b0dd1c77980043","f1eef6d4b9dac994c7df65b0e7298ace125691003f","f7f4f9e7e1efd4b9dac994c7df65b0e7298ace125691003f","f7f4f9e7e1efd4b9dac994c7df65b0e7298ace125698004367001f").map(H_),Db=eb($b),Rb=new Array(3).concat("fde0ddfa9fb5c51b8a","feebe2fbb4b9f768a1ae017e","feebe2fbb4b9f768a1c51b8a7a0177","feebe2fcc5c0fa9fb5f768a1c51b8a7a0177","feebe2fcc5c0fa9fb5f768a1dd3497ae017e7a0177","fff7f3fde0ddfcc5c0fa9fb5f768a1dd3497ae017e7a0177","fff7f3fde0ddfcc5c0fa9fb5f768a1dd3497ae017e7a017749006a").map(H_),Fb=eb(Rb),qb=new Array(3).concat("edf8b17fcdbb2c7fb8","ffffcca1dab441b6c4225ea8","ffffcca1dab441b6c42c7fb8253494","ffffccc7e9b47fcdbb41b6c42c7fb8253494","ffffccc7e9b47fcdbb41b6c41d91c0225ea80c2c84","ffffd9edf8b1c7e9b47fcdbb41b6c41d91c0225ea80c2c84","ffffd9edf8b1c7e9b47fcdbb41b6c41d91c0225ea8253494081d58").map(H_),Ub=eb(qb),Ib=new Array(3).concat("f7fcb9addd8e31a354","ffffccc2e69978c679238443","ffffccc2e69978c67931a354006837","ffffccd9f0a3addd8e78c67931a354006837","ffffccd9f0a3addd8e78c67941ab5d238443005a32","ffffe5f7fcb9d9f0a3addd8e78c67941ab5d238443005a32","ffffe5f7fcb9d9f0a3addd8e78c67941ab5d238443006837004529").map(H_),Ob=eb(Ib),Bb=new Array(3).concat("fff7bcfec44fd95f0e","ffffd4fed98efe9929cc4c02","ffffd4fed98efe9929d95f0e993404","ffffd4fee391fec44ffe9929d95f0e993404","ffffd4fee391fec44ffe9929ec7014cc4c028c2d04","ffffe5fff7bcfee391fec44ffe9929ec7014cc4c028c2d04","ffffe5fff7bcfee391fec44ffe9929ec7014cc4c02993404662506").map(H_),Yb=eb(Bb),Lb=new Array(3).concat("ffeda0feb24cf03b20","ffffb2fecc5cfd8d3ce31a1c","ffffb2fecc5cfd8d3cf03b20bd0026","ffffb2fed976feb24cfd8d3cf03b20bd0026","ffffb2fed976feb24cfd8d3cfc4e2ae31a1cb10026","ffffccffeda0fed976feb24cfd8d3cfc4e2ae31a1cb10026","ffffccffeda0fed976feb24cfd8d3cfc4e2ae31a1cbd0026800026").map(H_),jb=eb(Lb),Hb=new Array(3).concat("deebf79ecae13182bd","eff3ffbdd7e76baed62171b5","eff3ffbdd7e76baed63182bd08519c","eff3ffc6dbef9ecae16baed63182bd08519c","eff3ffc6dbef9ecae16baed64292c62171b5084594","f7fbffdeebf7c6dbef9ecae16baed64292c62171b5084594","f7fbffdeebf7c6dbef9ecae16baed64292c62171b508519c08306b").map(H_),Xb=eb(Hb),Gb=new Array(3).concat("e5f5e0a1d99b31a354","edf8e9bae4b374c476238b45","edf8e9bae4b374c47631a354006d2c","edf8e9c7e9c0a1d99b74c47631a354006d2c","edf8e9c7e9c0a1d99b74c47641ab5d238b45005a32","f7fcf5e5f5e0c7e9c0a1d99b74c47641ab5d238b45005a32","f7fcf5e5f5e0c7e9c0a1d99b74c47641ab5d238b45006d2c00441b").map(H_),Vb=eb(Gb),Wb=new Array(3).concat("f0f0f0bdbdbd636363","f7f7f7cccccc969696525252","f7f7f7cccccc969696636363252525","f7f7f7d9d9d9bdbdbd969696636363252525","f7f7f7d9d9d9bdbdbd969696737373525252252525","fffffff0f0f0d9d9d9bdbdbd969696737373525252252525","fffffff0f0f0d9d9d9bdbdbd969696737373525252252525000000").map(H_),Zb=eb(Wb),Kb=new Array(3).concat("efedf5bcbddc756bb1","f2f0f7cbc9e29e9ac86a51a3","f2f0f7cbc9e29e9ac8756bb154278f","f2f0f7dadaebbcbddc9e9ac8756bb154278f","f2f0f7dadaebbcbddc9e9ac8807dba6a51a34a1486","fcfbfdefedf5dadaebbcbddc9e9ac8807dba6a51a34a1486","fcfbfdefedf5dadaebbcbddc9e9ac8807dba6a51a354278f3f007d").map(H_),Qb=eb(Kb),Jb=new Array(3).concat("fee0d2fc9272de2d26","fee5d9fcae91fb6a4acb181d","fee5d9fcae91fb6a4ade2d26a50f15","fee5d9fcbba1fc9272fb6a4ade2d26a50f15","fee5d9fcbba1fc9272fb6a4aef3b2ccb181d99000d","fff5f0fee0d2fcbba1fc9272fb6a4aef3b2ccb181d99000d","fff5f0fee0d2fcbba1fc9272fb6a4aef3b2ccb181da50f1567000d").map(H_),tm=eb(Jb),nm=new Array(3).concat("fee6cefdae6be6550d","feeddefdbe85fd8d3cd94701","feeddefdbe85fd8d3ce6550da63603","feeddefdd0a2fdae6bfd8d3ce6550da63603","feeddefdd0a2fdae6bfd8d3cf16913d948018c2d04","fff5ebfee6cefdd0a2fdae6bfd8d3cf16913d948018c2d04","fff5ebfee6cefdd0a2fdae6bfd8d3cf16913d94801a636037f2704").map(H_),em=eb(nm);var rm=hi(Tr(300,.5,0),Tr(-240,.5,1)),im=hi(Tr(-100,.75,.35),Tr(80,1.5,.8)),om=hi(Tr(260,.75,.35),Tr(80,1.5,.8)),am=Tr();var um=Fe(),cm=Math.PI/3,fm=2*Math.PI/3;function sm(t){var n=t.length;return function(e){return t[Math.max(0,Math.min(n-1,Math.floor(e*n)))]}}var lm=sm(H_("44015444025645045745055946075a46085c460a5d460b5e470d60470e6147106347116447136548146748166848176948186a481a6c481b6d481c6e481d6f481f70482071482173482374482475482576482677482878482979472a7a472c7a472d7b472e7c472f7d46307e46327e46337f463480453581453781453882443983443a83443b84433d84433e85423f854240864241864142874144874045884046883f47883f48893e49893e4a893e4c8a3d4d8a3d4e8a3c4f8a3c508b3b518b3b528b3a538b3a548c39558c39568c38588c38598c375a8c375b8d365c8d365d8d355e8d355f8d34608d34618d33628d33638d32648e32658e31668e31678e31688e30698e306a8e2f6b8e2f6c8e2e6d8e2e6e8e2e6f8e2d708e2d718e2c718e2c728e2c738e2b748e2b758e2a768e2a778e2a788e29798e297a8e297b8e287c8e287d8e277e8e277f8e27808e26818e26828e26828e25838e25848e25858e24868e24878e23888e23898e238a8d228b8d228c8d228d8d218e8d218f8d21908d21918c20928c20928c20938c1f948c1f958b1f968b1f978b1f988b1f998a1f9a8a1e9b8a1e9c891e9d891f9e891f9f881fa0881fa1881fa1871fa28720a38620a48621a58521a68522a78522a88423a98324aa8325ab8225ac8226ad8127ad8128ae8029af7f2ab07f2cb17e2db27d2eb37c2fb47c31b57b32b67a34b67935b77937b87838b9773aba763bbb753dbc743fbc7340bd7242be7144bf7046c06f48c16e4ac16d4cc26c4ec36b50c46a52c56954c56856c66758c7655ac8645cc8635ec96260ca6063cb5f65cb5e67cc5c69cd5b6ccd5a6ece5870cf5773d05675d05477d1537ad1517cd2507fd34e81d34d84d44b86d54989d5488bd6468ed64590d74393d74195d84098d83e9bd93c9dd93ba0da39a2da37a5db36a8db34aadc32addc30b0dd2fb2dd2db5de2bb8de29bade28bddf26c0df25c2df23c5e021c8e020cae11fcde11dd0e11cd2e21bd5e21ad8e219dae319dde318dfe318e2e418e5e419e7e419eae51aece51befe51cf1e51df4e61ef6e620f8e621fbe723fde725")),hm=sm(H_("00000401000501010601010802010902020b02020d03030f03031204041405041606051806051a07061c08071e0907200a08220b09240c09260d0a290e0b2b100b2d110c2f120d31130d34140e36150e38160f3b180f3d19103f1a10421c10441d11471e114920114b21114e22115024125325125527125829115a2a115c2c115f2d11612f116331116533106734106936106b38106c390f6e3b0f703d0f713f0f72400f74420f75440f764510774710784910784a10794c117a4e117b4f127b51127c52137c54137d56147d57157e59157e5a167e5c167f5d177f5f187f601880621980641a80651a80671b80681c816a1c816b1d816d1d816e1e81701f81721f817320817521817621817822817922827b23827c23827e24828025828125818326818426818627818827818928818b29818c29818e2a81902a81912b81932b80942c80962c80982d80992d809b2e7f9c2e7f9e2f7fa02f7fa1307ea3307ea5317ea6317da8327daa337dab337cad347cae347bb0357bb2357bb3367ab5367ab73779b83779ba3878bc3978bd3977bf3a77c03a76c23b75c43c75c53c74c73d73c83e73ca3e72cc3f71cd4071cf4070d0416fd2426fd3436ed5446dd6456cd8456cd9466bdb476adc4869de4968df4a68e04c67e24d66e34e65e44f64e55064e75263e85362e95462ea5661eb5760ec5860ed5a5fee5b5eef5d5ef05f5ef1605df2625df2645cf3655cf4675cf4695cf56b5cf66c5cf66e5cf7705cf7725cf8745cf8765cf9785df9795df97b5dfa7d5efa7f5efa815ffb835ffb8560fb8761fc8961fc8a62fc8c63fc8e64fc9065fd9266fd9467fd9668fd9869fd9a6afd9b6bfe9d6cfe9f6dfea16efea36ffea571fea772fea973feaa74feac76feae77feb078feb27afeb47bfeb67cfeb77efeb97ffebb81febd82febf84fec185fec287fec488fec68afec88cfeca8dfecc8ffecd90fecf92fed194fed395fed597fed799fed89afdda9cfddc9efddea0fde0a1fde2a3fde3a5fde5a7fde7a9fde9aafdebacfcecaefceeb0fcf0b2fcf2b4fcf4b6fcf6b8fcf7b9fcf9bbfcfbbdfcfdbf")),dm=sm(H_("00000401000501010601010802010a02020c02020e03021004031204031405041706041907051b08051d09061f0a07220b07240c08260d08290e092b10092d110a30120a32140b34150b37160b39180c3c190c3e1b0c411c0c431e0c451f0c48210c4a230c4c240c4f260c51280b53290b552b0b572d0b592f0a5b310a5c320a5e340a5f3609613809623909633b09643d09653e0966400a67420a68440a68450a69470b6a490b6a4a0c6b4c0c6b4d0d6c4f0d6c510e6c520e6d540f6d550f6d57106e59106e5a116e5c126e5d126e5f136e61136e62146e64156e65156e67166e69166e6a176e6c186e6d186e6f196e71196e721a6e741a6e751b6e771c6d781c6d7a1d6d7c1d6d7d1e6d7f1e6c801f6c82206c84206b85216b87216b88226a8a226a8c23698d23698f24699025689225689326679526679727669827669a28659b29649d29649f2a63a02a63a22b62a32c61a52c60a62d60a82e5fa92e5eab2f5ead305dae305cb0315bb1325ab3325ab43359b63458b73557b93556ba3655bc3754bd3853bf3952c03a51c13a50c33b4fc43c4ec63d4dc73e4cc83f4bca404acb4149cc4248ce4347cf4446d04545d24644d34743d44842d54a41d74b3fd84c3ed94d3dda4e3cdb503bdd513ade5238df5337e05536e15635e25734e35933e45a31e55c30e65d2fe75e2ee8602de9612bea632aeb6429eb6628ec6726ed6925ee6a24ef6c23ef6e21f06f20f1711ff1731df2741cf3761bf37819f47918f57b17f57d15f67e14f68013f78212f78410f8850ff8870ef8890cf98b0bf98c0af98e09fa9008fa9207fa9407fb9606fb9706fb9906fb9b06fb9d07fc9f07fca108fca309fca50afca60cfca80dfcaa0ffcac11fcae12fcb014fcb216fcb418fbb61afbb81dfbba1ffbbc21fbbe23fac026fac228fac42afac62df9c72ff9c932f9cb35f8cd37f8cf3af7d13df7d340f6d543f6d746f5d949f5db4cf4dd4ff4df53f4e156f3e35af3e55df2e661f2e865f2ea69f1ec6df1ed71f1ef75f1f179f2f27df2f482f3f586f3f68af4f88ef5f992f6fa96f8fb9af9fc9dfafda1fcffa4")),pm=sm(H_("0d088710078813078916078a19068c1b068d1d068e20068f2206902406912605912805922a05932c05942e05952f059631059733059735049837049938049a3a049a3c049b3e049c3f049c41049d43039e44039e46039f48039f4903a04b03a14c02a14e02a25002a25102a35302a35502a45601a45801a45901a55b01a55c01a65e01a66001a66100a76300a76400a76600a76700a86900a86a00a86c00a86e00a86f00a87100a87201a87401a87501a87701a87801a87a02a87b02a87d03a87e03a88004a88104a78305a78405a78606a68707a68808a68a09a58b0aa58d0ba58e0ca48f0da4910ea3920fa39410a29511a19613a19814a099159f9a169f9c179e9d189d9e199da01a9ca11b9ba21d9aa31e9aa51f99a62098a72197a82296aa2395ab2494ac2694ad2793ae2892b02991b12a90b22b8fb32c8eb42e8db52f8cb6308bb7318ab83289ba3388bb3488bc3587bd3786be3885bf3984c03a83c13b82c23c81c33d80c43e7fc5407ec6417dc7427cc8437bc9447aca457acb4679cc4778cc4977cd4a76ce4b75cf4c74d04d73d14e72d24f71d35171d45270d5536fd5546ed6556dd7566cd8576bd9586ada5a6ada5b69db5c68dc5d67dd5e66de5f65de6164df6263e06363e16462e26561e26660e3685fe4695ee56a5de56b5de66c5ce76e5be76f5ae87059e97158e97257ea7457eb7556eb7655ec7754ed7953ed7a52ee7b51ef7c51ef7e50f07f4ff0804ef1814df1834cf2844bf3854bf3874af48849f48948f58b47f58c46f68d45f68f44f79044f79143f79342f89441f89540f9973ff9983ef99a3efa9b3dfa9c3cfa9e3bfb9f3afba139fba238fca338fca537fca636fca835fca934fdab33fdac33fdae32fdaf31fdb130fdb22ffdb42ffdb52efeb72dfeb82cfeba2cfebb2bfebd2afebe2afec029fdc229fdc328fdc527fdc627fdc827fdca26fdcb26fccd25fcce25fcd025fcd225fbd324fbd524fbd724fad824fada24f9dc24f9dd25f8df25f8e125f7e225f7e425f6e626f6e826f5e926f5eb27f4ed27f3ee27f3f027f2f227f1f426f1f525f0f724f0f921"));function gm(t){return function(){return t}}const ym=Math.abs,vm=Math.atan2,_m=Math.cos,bm=Math.max,mm=Math.min,xm=Math.sin,wm=Math.sqrt,Mm=1e-12,Tm=Math.PI,Am=Tm/2,Sm=2*Tm;function Em(t){return t>=1?Am:t<=-1?-Am:Math.asin(t)}function Nm(t){let n=3;return t.digits=function(e){if(!arguments.length)return n;if(null==e)n=null;else{const t=Math.floor(e);if(!(t>=0))throw new RangeError(`invalid digits: ${e}`);n=t}return t},()=>new Ua(n)}function km(t){return t.innerRadius}function Cm(t){return t.outerRadius}function Pm(t){return t.startAngle}function zm(t){return t.endAngle}function $m(t){return t&&t.padAngle}function Dm(t,n,e,r,i,o,a){var u=t-e,c=n-r,f=(a?o:-o)/wm(u*u+c*c),s=f*c,l=-f*u,h=t+s,d=n+l,p=e+s,g=r+l,y=(h+p)/2,v=(d+g)/2,_=p-h,b=g-d,m=_*_+b*b,x=i-o,w=h*g-p*d,M=(b<0?-1:1)*wm(bm(0,x*x*m-w*w)),T=(w*b-_*M)/m,A=(-w*_-b*M)/m,S=(w*b+_*M)/m,E=(-w*_+b*M)/m,N=T-y,k=A-v,C=S-y,P=E-v;return N*N+k*k>C*C+P*P&&(T=S,A=E),{cx:T,cy:A,x01:-s,y01:-l,x11:T*(i/x-1),y11:A*(i/x-1)}}var Rm=Array.prototype.slice;function Fm(t){return"object"==typeof t&&"length"in t?t:Array.from(t)}function qm(t){this._context=t}function Um(t){return new qm(t)}function Im(t){return t[0]}function Om(t){return t[1]}function Bm(t,n){var e=gm(!0),r=null,i=Um,o=null,a=Nm(u);function u(u){var c,f,s,l=(u=Fm(u)).length,h=!1;for(null==r&&(o=i(s=a())),c=0;c<=l;++c)!(c<l&&e(f=u[c],c,u))===h&&((h=!h)?o.lineStart():o.lineEnd()),h&&o.point(+t(f,c,u),+n(f,c,u));if(s)return o=null,s+""||null}return t="function"==typeof t?t:void 0===t?Im:gm(t),n="function"==typeof n?n:void 0===n?Om:gm(n),u.x=function(n){return arguments.length?(t="function"==typeof n?n:gm(+n),u):t},u.y=function(t){return arguments.length?(n="function"==typeof t?t:gm(+t),u):n},u.defined=function(t){return arguments.length?(e="function"==typeof t?t:gm(!!t),u):e},u.curve=function(t){return arguments.length?(i=t,null!=r&&(o=i(r)),u):i},u.context=function(t){return arguments.length?(null==t?r=o=null:o=i(r=t),u):r},u}function Ym(t,n,e){var r=null,i=gm(!0),o=null,a=Um,u=null,c=Nm(f);function f(f){var s,l,h,d,p,g=(f=Fm(f)).length,y=!1,v=new Array(g),_=new Array(g);for(null==o&&(u=a(p=c())),s=0;s<=g;++s){if(!(s<g&&i(d=f[s],s,f))===y)if(y=!y)l=s,u.areaStart(),u.lineStart();else{for(u.lineEnd(),u.lineStart(),h=s-1;h>=l;--h)u.point(v[h],_[h]);u.lineEnd(),u.areaEnd()}y&&(v[s]=+t(d,s,f),_[s]=+n(d,s,f),u.point(r?+r(d,s,f):v[s],e?+e(d,s,f):_[s]))}if(p)return u=null,p+""||null}function s(){return Bm().defined(i).curve(a).context(o)}return t="function"==typeof t?t:void 0===t?Im:gm(+t),n="function"==typeof n?n:gm(void 0===n?0:+n),e="function"==typeof e?e:void 0===e?Om:gm(+e),f.x=function(n){return arguments.length?(t="function"==typeof n?n:gm(+n),r=null,f):t},f.x0=function(n){return arguments.length?(t="function"==typeof n?n:gm(+n),f):t},f.x1=function(t){return arguments.length?(r=null==t?null:"function"==typeof t?t:gm(+t),f):r},f.y=function(t){return arguments.length?(n="function"==typeof t?t:gm(+t),e=null,f):n},f.y0=function(t){return arguments.length?(n="function"==typeof t?t:gm(+t),f):n},f.y1=function(t){return arguments.length?(e=null==t?null:"function"==typeof t?t:gm(+t),f):e},f.lineX0=f.lineY0=function(){return s().x(t).y(n)},f.lineY1=function(){return s().x(t).y(e)},f.lineX1=function(){return s().x(r).y(n)},f.defined=function(t){return arguments.length?(i="function"==typeof t?t:gm(!!t),f):i},f.curve=function(t){return arguments.length?(a=t,null!=o&&(u=a(o)),f):a},f.context=function(t){return arguments.length?(null==t?o=u=null:u=a(o=t),f):o},f}function Lm(t,n){return n<t?-1:n>t?1:n>=t?0:NaN}function jm(t){return t}qm.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._point=0},lineEnd:function(){(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2;default:this._context.lineTo(t,n)}}};var Hm=Gm(Um);function Xm(t){this._curve=t}function Gm(t){function n(n){return new Xm(t(n))}return n._curve=t,n}function Vm(t){var n=t.curve;return t.angle=t.x,delete t.x,t.radius=t.y,delete t.y,t.curve=function(t){return arguments.length?n(Gm(t)):n()._curve},t}function Wm(){return Vm(Bm().curve(Hm))}function Zm(){var t=Ym().curve(Hm),n=t.curve,e=t.lineX0,r=t.lineX1,i=t.lineY0,o=t.lineY1;return t.angle=t.x,delete t.x,t.startAngle=t.x0,delete t.x0,t.endAngle=t.x1,delete t.x1,t.radius=t.y,delete t.y,t.innerRadius=t.y0,delete t.y0,t.outerRadius=t.y1,delete t.y1,t.lineStartAngle=function(){return Vm(e())},delete t.lineX0,t.lineEndAngle=function(){return Vm(r())},delete t.lineX1,t.lineInnerRadius=function(){return Vm(i())},delete t.lineY0,t.lineOuterRadius=function(){return Vm(o())},delete t.lineY1,t.curve=function(t){return arguments.length?n(Gm(t)):n()._curve},t}function Km(t,n){return[(n=+n)*Math.cos(t-=Math.PI/2),n*Math.sin(t)]}Xm.prototype={areaStart:function(){this._curve.areaStart()},areaEnd:function(){this._curve.areaEnd()},lineStart:function(){this._curve.lineStart()},lineEnd:function(){this._curve.lineEnd()},point:function(t,n){this._curve.point(n*Math.sin(t),n*-Math.cos(t))}};class Qm{constructor(t,n){this._context=t,this._x=n}areaStart(){this._line=0}areaEnd(){this._line=NaN}lineStart(){this._point=0}lineEnd(){(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line}point(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2;default:this._x?this._context.bezierCurveTo(this._x0=(this._x0+t)/2,this._y0,this._x0,n,t,n):this._context.bezierCurveTo(this._x0,this._y0=(this._y0+n)/2,t,this._y0,t,n)}this._x0=t,this._y0=n}}class Jm{constructor(t){this._context=t}lineStart(){this._point=0}lineEnd(){}point(t,n){if(t=+t,n=+n,0===this._point)this._point=1;else{const e=Km(this._x0,this._y0),r=Km(this._x0,this._y0=(this._y0+n)/2),i=Km(t,this._y0),o=Km(t,n);this._context.moveTo(...e),this._context.bezierCurveTo(...r,...i,...o)}this._x0=t,this._y0=n}}function tx(t){return new Qm(t,!0)}function nx(t){return new Qm(t,!1)}function ex(t){return new Jm(t)}function rx(t){return t.source}function ix(t){return t.target}function ox(t){let n=rx,e=ix,r=Im,i=Om,o=null,a=null,u=Nm(c);function c(){let c;const f=Rm.call(arguments),s=n.apply(this,f),l=e.apply(this,f);if(null==o&&(a=t(c=u())),a.lineStart(),f[0]=s,a.point(+r.apply(this,f),+i.apply(this,f)),f[0]=l,a.point(+r.apply(this,f),+i.apply(this,f)),a.lineEnd(),c)return a=null,c+""||null}return c.source=function(t){return arguments.length?(n=t,c):n},c.target=function(t){return arguments.length?(e=t,c):e},c.x=function(t){return arguments.length?(r="function"==typeof t?t:gm(+t),c):r},c.y=function(t){return arguments.length?(i="function"==typeof t?t:gm(+t),c):i},c.context=function(n){return arguments.length?(null==n?o=a=null:a=t(o=n),c):o},c}const ax=wm(3);var ux={draw(t,n){const e=.59436*wm(n+mm(n/28,.75)),r=e/2,i=r*ax;t.moveTo(0,e),t.lineTo(0,-e),t.moveTo(-i,-r),t.lineTo(i,r),t.moveTo(-i,r),t.lineTo(i,-r)}},cx={draw(t,n){const e=wm(n/Tm);t.moveTo(e,0),t.arc(0,0,e,0,Sm)}},fx={draw(t,n){const e=wm(n/5)/2;t.moveTo(-3*e,-e),t.lineTo(-e,-e),t.lineTo(-e,-3*e),t.lineTo(e,-3*e),t.lineTo(e,-e),t.lineTo(3*e,-e),t.lineTo(3*e,e),t.lineTo(e,e),t.lineTo(e,3*e),t.lineTo(-e,3*e),t.lineTo(-e,e),t.lineTo(-3*e,e),t.closePath()}};const sx=wm(1/3),lx=2*sx;var hx={draw(t,n){const e=wm(n/lx),r=e*sx;t.moveTo(0,-e),t.lineTo(r,0),t.lineTo(0,e),t.lineTo(-r,0),t.closePath()}},dx={draw(t,n){const e=.62625*wm(n);t.moveTo(0,-e),t.lineTo(e,0),t.lineTo(0,e),t.lineTo(-e,0),t.closePath()}},px={draw(t,n){const e=.87559*wm(n-mm(n/7,2));t.moveTo(-e,0),t.lineTo(e,0),t.moveTo(0,e),t.lineTo(0,-e)}},gx={draw(t,n){const e=wm(n),r=-e/2;t.rect(r,r,e,e)}},yx={draw(t,n){const e=.4431*wm(n);t.moveTo(e,e),t.lineTo(e,-e),t.lineTo(-e,-e),t.lineTo(-e,e),t.closePath()}};const vx=xm(Tm/10)/xm(7*Tm/10),_x=xm(Sm/10)*vx,bx=-_m(Sm/10)*vx;var mx={draw(t,n){const e=wm(.8908130915292852*n),r=_x*e,i=bx*e;t.moveTo(0,-e),t.lineTo(r,i);for(let n=1;n<5;++n){const o=Sm*n/5,a=_m(o),u=xm(o);t.lineTo(u*e,-a*e),t.lineTo(a*r-u*i,u*r+a*i)}t.closePath()}};const xx=wm(3);var wx={draw(t,n){const e=-wm(n/(3*xx));t.moveTo(0,2*e),t.lineTo(-xx*e,-e),t.lineTo(xx*e,-e),t.closePath()}};const Mx=wm(3);var Tx={draw(t,n){const e=.6824*wm(n),r=e/2,i=e*Mx/2;t.moveTo(0,-e),t.lineTo(i,r),t.lineTo(-i,r),t.closePath()}};const Ax=-.5,Sx=wm(3)/2,Ex=1/wm(12),Nx=3*(Ex/2+1);var kx={draw(t,n){const e=wm(n/Nx),r=e/2,i=e*Ex,o=r,a=e*Ex+e,u=-o,c=a;t.moveTo(r,i),t.lineTo(o,a),t.lineTo(u,c),t.lineTo(Ax*r-Sx*i,Sx*r+Ax*i),t.lineTo(Ax*o-Sx*a,Sx*o+Ax*a),t.lineTo(Ax*u-Sx*c,Sx*u+Ax*c),t.lineTo(Ax*r+Sx*i,Ax*i-Sx*r),t.lineTo(Ax*o+Sx*a,Ax*a-Sx*o),t.lineTo(Ax*u+Sx*c,Ax*c-Sx*u),t.closePath()}},Cx={draw(t,n){const e=.6189*wm(n-mm(n/6,1.7));t.moveTo(-e,-e),t.lineTo(e,e),t.moveTo(-e,e),t.lineTo(e,-e)}};const Px=[cx,fx,hx,gx,mx,wx,kx],zx=[cx,px,Cx,Tx,ux,yx,dx];function $x(){}function Dx(t,n,e){t._context.bezierCurveTo((2*t._x0+t._x1)/3,(2*t._y0+t._y1)/3,(t._x0+2*t._x1)/3,(t._y0+2*t._y1)/3,(t._x0+4*t._x1+n)/6,(t._y0+4*t._y1+e)/6)}function Rx(t){this._context=t}function Fx(t){this._context=t}function qx(t){this._context=t}function Ux(t,n){this._basis=new Rx(t),this._beta=n}Rx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._y0=this._y1=NaN,this._point=0},lineEnd:function(){switch(this._point){case 3:Dx(this,this._x1,this._y1);case 2:this._context.lineTo(this._x1,this._y1)}(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2;break;case 2:this._point=3,this._context.lineTo((5*this._x0+this._x1)/6,(5*this._y0+this._y1)/6);default:Dx(this,t,n)}this._x0=this._x1,this._x1=t,this._y0=this._y1,this._y1=n}},Fx.prototype={areaStart:$x,areaEnd:$x,lineStart:function(){this._x0=this._x1=this._x2=this._x3=this._x4=this._y0=this._y1=this._y2=this._y3=this._y4=NaN,this._point=0},lineEnd:function(){switch(this._point){case 1:this._context.moveTo(this._x2,this._y2),this._context.closePath();break;case 2:this._context.moveTo((this._x2+2*this._x3)/3,(this._y2+2*this._y3)/3),this._context.lineTo((this._x3+2*this._x2)/3,(this._y3+2*this._y2)/3),this._context.closePath();break;case 3:this.point(this._x2,this._y2),this.point(this._x3,this._y3),this.point(this._x4,this._y4)}},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._x2=t,this._y2=n;break;case 1:this._point=2,this._x3=t,this._y3=n;break;case 2:this._point=3,this._x4=t,this._y4=n,this._context.moveTo((this._x0+4*this._x1+t)/6,(this._y0+4*this._y1+n)/6);break;default:Dx(this,t,n)}this._x0=this._x1,this._x1=t,this._y0=this._y1,this._y1=n}},qx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._y0=this._y1=NaN,this._point=0},lineEnd:function(){(this._line||0!==this._line&&3===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1;break;case 1:this._point=2;break;case 2:this._point=3;var e=(this._x0+4*this._x1+t)/6,r=(this._y0+4*this._y1+n)/6;this._line?this._context.lineTo(e,r):this._context.moveTo(e,r);break;case 3:this._point=4;default:Dx(this,t,n)}this._x0=this._x1,this._x1=t,this._y0=this._y1,this._y1=n}},Ux.prototype={lineStart:function(){this._x=[],this._y=[],this._basis.lineStart()},lineEnd:function(){var t=this._x,n=this._y,e=t.length-1;if(e>0)for(var r,i=t[0],o=n[0],a=t[e]-i,u=n[e]-o,c=-1;++c<=e;)r=c/e,this._basis.point(this._beta*t[c]+(1-this._beta)*(i+r*a),this._beta*n[c]+(1-this._beta)*(o+r*u));this._x=this._y=null,this._basis.lineEnd()},point:function(t,n){this._x.push(+t),this._y.push(+n)}};var Ix=function t(n){function e(t){return 1===n?new Rx(t):new Ux(t,n)}return e.beta=function(n){return t(+n)},e}(.85);function Ox(t,n,e){t._context.bezierCurveTo(t._x1+t._k*(t._x2-t._x0),t._y1+t._k*(t._y2-t._y0),t._x2+t._k*(t._x1-n),t._y2+t._k*(t._y1-e),t._x2,t._y2)}function Bx(t,n){this._context=t,this._k=(1-n)/6}Bx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._point=0},lineEnd:function(){switch(this._point){case 2:this._context.lineTo(this._x2,this._y2);break;case 3:Ox(this,this._x1,this._y1)}(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2,this._x1=t,this._y1=n;break;case 2:this._point=3;default:Ox(this,t,n)}this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var Yx=function t(n){function e(t){return new Bx(t,n)}return e.tension=function(n){return t(+n)},e}(0);function Lx(t,n){this._context=t,this._k=(1-n)/6}Lx.prototype={areaStart:$x,areaEnd:$x,lineStart:function(){this._x0=this._x1=this._x2=this._x3=this._x4=this._x5=this._y0=this._y1=this._y2=this._y3=this._y4=this._y5=NaN,this._point=0},lineEnd:function(){switch(this._point){case 1:this._context.moveTo(this._x3,this._y3),this._context.closePath();break;case 2:this._context.lineTo(this._x3,this._y3),this._context.closePath();break;case 3:this.point(this._x3,this._y3),this.point(this._x4,this._y4),this.point(this._x5,this._y5)}},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._x3=t,this._y3=n;break;case 1:this._point=2,this._context.moveTo(this._x4=t,this._y4=n);break;case 2:this._point=3,this._x5=t,this._y5=n;break;default:Ox(this,t,n)}this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var jx=function t(n){function e(t){return new Lx(t,n)}return e.tension=function(n){return t(+n)},e}(0);function Hx(t,n){this._context=t,this._k=(1-n)/6}Hx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._point=0},lineEnd:function(){(this._line||0!==this._line&&3===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1;break;case 1:this._point=2;break;case 2:this._point=3,this._line?this._context.lineTo(this._x2,this._y2):this._context.moveTo(this._x2,this._y2);break;case 3:this._point=4;default:Ox(this,t,n)}this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var Xx=function t(n){function e(t){return new Hx(t,n)}return e.tension=function(n){return t(+n)},e}(0);function Gx(t,n,e){var r=t._x1,i=t._y1,o=t._x2,a=t._y2;if(t._l01_a>Mm){var u=2*t._l01_2a+3*t._l01_a*t._l12_a+t._l12_2a,c=3*t._l01_a*(t._l01_a+t._l12_a);r=(r*u-t._x0*t._l12_2a+t._x2*t._l01_2a)/c,i=(i*u-t._y0*t._l12_2a+t._y2*t._l01_2a)/c}if(t._l23_a>Mm){var f=2*t._l23_2a+3*t._l23_a*t._l12_a+t._l12_2a,s=3*t._l23_a*(t._l23_a+t._l12_a);o=(o*f+t._x1*t._l23_2a-n*t._l12_2a)/s,a=(a*f+t._y1*t._l23_2a-e*t._l12_2a)/s}t._context.bezierCurveTo(r,i,o,a,t._x2,t._y2)}function Vx(t,n){this._context=t,this._alpha=n}Vx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._l01_a=this._l12_a=this._l23_a=this._l01_2a=this._l12_2a=this._l23_2a=this._point=0},lineEnd:function(){switch(this._point){case 2:this._context.lineTo(this._x2,this._y2);break;case 3:this.point(this._x2,this._y2)}(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){if(t=+t,n=+n,this._point){var e=this._x2-t,r=this._y2-n;this._l23_a=Math.sqrt(this._l23_2a=Math.pow(e*e+r*r,this._alpha))}switch(this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2;break;case 2:this._point=3;default:Gx(this,t,n)}this._l01_a=this._l12_a,this._l12_a=this._l23_a,this._l01_2a=this._l12_2a,this._l12_2a=this._l23_2a,this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var Wx=function t(n){function e(t){return n?new Vx(t,n):new Bx(t,0)}return e.alpha=function(n){return t(+n)},e}(.5);function Zx(t,n){this._context=t,this._alpha=n}Zx.prototype={areaStart:$x,areaEnd:$x,lineStart:function(){this._x0=this._x1=this._x2=this._x3=this._x4=this._x5=this._y0=this._y1=this._y2=this._y3=this._y4=this._y5=NaN,this._l01_a=this._l12_a=this._l23_a=this._l01_2a=this._l12_2a=this._l23_2a=this._point=0},lineEnd:function(){switch(this._point){case 1:this._context.moveTo(this._x3,this._y3),this._context.closePath();break;case 2:this._context.lineTo(this._x3,this._y3),this._context.closePath();break;case 3:this.point(this._x3,this._y3),this.point(this._x4,this._y4),this.point(this._x5,this._y5)}},point:function(t,n){if(t=+t,n=+n,this._point){var e=this._x2-t,r=this._y2-n;this._l23_a=Math.sqrt(this._l23_2a=Math.pow(e*e+r*r,this._alpha))}switch(this._point){case 0:this._point=1,this._x3=t,this._y3=n;break;case 1:this._point=2,this._context.moveTo(this._x4=t,this._y4=n);break;case 2:this._point=3,this._x5=t,this._y5=n;break;default:Gx(this,t,n)}this._l01_a=this._l12_a,this._l12_a=this._l23_a,this._l01_2a=this._l12_2a,this._l12_2a=this._l23_2a,this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var Kx=function t(n){function e(t){return n?new Zx(t,n):new Lx(t,0)}return e.alpha=function(n){return t(+n)},e}(.5);function Qx(t,n){this._context=t,this._alpha=n}Qx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._l01_a=this._l12_a=this._l23_a=this._l01_2a=this._l12_2a=this._l23_2a=this._point=0},lineEnd:function(){(this._line||0!==this._line&&3===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){if(t=+t,n=+n,this._point){var e=this._x2-t,r=this._y2-n;this._l23_a=Math.sqrt(this._l23_2a=Math.pow(e*e+r*r,this._alpha))}switch(this._point){case 0:this._point=1;break;case 1:this._point=2;break;case 2:this._point=3,this._line?this._context.lineTo(this._x2,this._y2):this._context.moveTo(this._x2,this._y2);break;case 3:this._point=4;default:Gx(this,t,n)}this._l01_a=this._l12_a,this._l12_a=this._l23_a,this._l01_2a=this._l12_2a,this._l12_2a=this._l23_2a,this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var Jx=function t(n){function e(t){return n?new Qx(t,n):new Hx(t,0)}return e.alpha=function(n){return t(+n)},e}(.5);function tw(t){this._context=t}function nw(t){return t<0?-1:1}function ew(t,n,e){var r=t._x1-t._x0,i=n-t._x1,o=(t._y1-t._y0)/(r||i<0&&-0),a=(e-t._y1)/(i||r<0&&-0),u=(o*i+a*r)/(r+i);return(nw(o)+nw(a))*Math.min(Math.abs(o),Math.abs(a),.5*Math.abs(u))||0}function rw(t,n){var e=t._x1-t._x0;return e?(3*(t._y1-t._y0)/e-n)/2:n}function iw(t,n,e){var r=t._x0,i=t._y0,o=t._x1,a=t._y1,u=(o-r)/3;t._context.bezierCurveTo(r+u,i+u*n,o-u,a-u*e,o,a)}function ow(t){this._context=t}function aw(t){this._context=new uw(t)}function uw(t){this._context=t}function cw(t){this._context=t}function fw(t){var n,e,r=t.length-1,i=new Array(r),o=new Array(r),a=new Array(r);for(i[0]=0,o[0]=2,a[0]=t[0]+2*t[1],n=1;n<r-1;++n)i[n]=1,o[n]=4,a[n]=4*t[n]+2*t[n+1];for(i[r-1]=2,o[r-1]=7,a[r-1]=8*t[r-1]+t[r],n=1;n<r;++n)e=i[n]/o[n-1],o[n]-=e,a[n]-=e*a[n-1];for(i[r-1]=a[r-1]/o[r-1],n=r-2;n>=0;--n)i[n]=(a[n]-i[n+1])/o[n];for(o[r-1]=(t[r]+i[r-1])/2,n=0;n<r-1;++n)o[n]=2*t[n+1]-i[n+1];return[i,o]}function sw(t,n){this._context=t,this._t=n}function lw(t,n){if((i=t.length)>1)for(var e,r,i,o=1,a=t[n[0]],u=a.length;o<i;++o)for(r=a,a=t[n[o]],e=0;e<u;++e)a[e][1]+=a[e][0]=isNaN(r[e][1])?r[e][0]:r[e][1]}function hw(t){for(var n=t.length,e=new Array(n);--n>=0;)e[n]=n;return e}function dw(t,n){return t[n]}function pw(t){const n=[];return n.key=t,n}function gw(t){var n=t.map(yw);return hw(t).sort((function(t,e){return n[t]-n[e]}))}function yw(t){for(var n,e=-1,r=0,i=t.length,o=-1/0;++e<i;)(n=+t[e][1])>o&&(o=n,r=e);return r}function vw(t){var n=t.map(_w);return hw(t).sort((function(t,e){return n[t]-n[e]}))}function _w(t){for(var n,e=0,r=-1,i=t.length;++r<i;)(n=+t[r][1])&&(e+=n);return e}tw.prototype={areaStart:$x,areaEnd:$x,lineStart:function(){this._point=0},lineEnd:function(){this._point&&this._context.closePath()},point:function(t,n){t=+t,n=+n,this._point?this._context.lineTo(t,n):(this._point=1,this._context.moveTo(t,n))}},ow.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._y0=this._y1=this._t0=NaN,this._point=0},lineEnd:function(){switch(this._point){case 2:this._context.lineTo(this._x1,this._y1);break;case 3:iw(this,this._t0,rw(this,this._t0))}(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){var e=NaN;if(n=+n,(t=+t)!==this._x1||n!==this._y1){switch(this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2;break;case 2:this._point=3,iw(this,rw(this,e=ew(this,t,n)),e);break;default:iw(this,this._t0,e=ew(this,t,n))}this._x0=this._x1,this._x1=t,this._y0=this._y1,this._y1=n,this._t0=e}}},(aw.prototype=Object.create(ow.prototype)).point=function(t,n){ow.prototype.point.call(this,n,t)},uw.prototype={moveTo:function(t,n){this._context.moveTo(n,t)},closePath:function(){this._context.closePath()},lineTo:function(t,n){this._context.lineTo(n,t)},bezierCurveTo:function(t,n,e,r,i,o){this._context.bezierCurveTo(n,t,r,e,o,i)}},cw.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x=[],this._y=[]},lineEnd:function(){var t=this._x,n=this._y,e=t.length;if(e)if(this._line?this._context.lineTo(t[0],n[0]):this._context.moveTo(t[0],n[0]),2===e)this._context.lineTo(t[1],n[1]);else for(var r=fw(t),i=fw(n),o=0,a=1;a<e;++o,++a)this._context.bezierCurveTo(r[0][o],i[0][o],r[1][o],i[1][o],t[a],n[a]);(this._line||0!==this._line&&1===e)&&this._context.closePath(),this._line=1-this._line,this._x=this._y=null},point:function(t,n){this._x.push(+t),this._y.push(+n)}},sw.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x=this._y=NaN,this._point=0},lineEnd:function(){0<this._t&&this._t<1&&2===this._point&&this._context.lineTo(this._x,this._y),(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line>=0&&(this._t=1-this._t,this._line=1-this._line)},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2;default:if(this._t<=0)this._context.lineTo(this._x,n),this._context.lineTo(t,n);else{var e=this._x*(1-this._t)+t*this._t;this._context.lineTo(e,this._y),this._context.lineTo(e,n)}}this._x=t,this._y=n}};var bw=t=>()=>t;function mw(t,{sourceEvent:n,target:e,transform:r,dispatch:i}){Object.defineProperties(this,{type:{value:t,enumerable:!0,configurable:!0},sourceEvent:{value:n,enumerable:!0,configurable:!0},target:{value:e,enumerable:!0,configurable:!0},transform:{value:r,enumerable:!0,configurable:!0},_:{value:i}})}function xw(t,n,e){this.k=t,this.x=n,this.y=e}xw.prototype={constructor:xw,scale:function(t){return 1===t?this:new xw(this.k*t,this.x,this.y)},translate:function(t,n){return 0===t&0===n?this:new xw(this.k,this.x+this.k*t,this.y+this.k*n)},apply:function(t){return[t[0]*this.k+this.x,t[1]*this.k+this.y]},applyX:function(t){return t*this.k+this.x},applyY:function(t){return t*this.k+this.y},invert:function(t){return[(t[0]-this.x)/this.k,(t[1]-this.y)/this.k]},invertX:function(t){return(t-this.x)/this.k},invertY:function(t){return(t-this.y)/this.k},rescaleX:function(t){return t.copy().domain(t.range().map(this.invertX,this).map(t.invert,t))},rescaleY:function(t){return t.copy().domain(t.range().map(this.invertY,this).map(t.invert,t))},toString:function(){return"translate("+this.x+","+this.y+") scale("+this.k+")"}};var ww=new xw(1,0,0);function Mw(t){for(;!t.__zoom;)if(!(t=t.parentNode))return ww;return t.__zoom}function Tw(t){t.stopImmediatePropagation()}function Aw(t){t.preventDefault(),t.stopImmediatePropagation()}function Sw(t){return!(t.ctrlKey&&"wheel"!==t.type||t.button)}function Ew(){var t=this;return t instanceof SVGElement?(t=t.ownerSVGElement||t).hasAttribute("viewBox")?[[(t=t.viewBox.baseVal).x,t.y],[t.x+t.width,t.y+t.height]]:[[0,0],[t.width.baseVal.value,t.height.baseVal.value]]:[[0,0],[t.clientWidth,t.clientHeight]]}function Nw(){return this.__zoom||ww}function kw(t){return-t.deltaY*(1===t.deltaMode?.05:t.deltaMode?1:.002)*(t.ctrlKey?10:1)}function Cw(){return navigator.maxTouchPoints||"ontouchstart"in this}function Pw(t,n,e){var r=t.invertX(n[0][0])-e[0][0],i=t.invertX(n[1][0])-e[1][0],o=t.invertY(n[0][1])-e[0][1],a=t.invertY(n[1][1])-e[1][1];return t.translate(i>r?(r+i)/2:Math.min(0,r)||Math.max(0,i),a>o?(o+a)/2:Math.min(0,o)||Math.max(0,a))}Mw.prototype=xw.prototype,t.Adder=T,t.Delaunay=Lu,t.FormatSpecifier=tf,t.InternMap=InternMap,t.InternSet=InternSet,t.Node=Qd,t.Path=Ua,t.Voronoi=qu,t.ZoomTransform=xw,t.active=function(t,n){var e,r,i=t.__transition;if(i)for(r in n=null==n?null:n+"",i)if((e=i[r]).state>qi&&e.name===n)return new po([[t]],Zo,n,+r);return null},t.arc=function(){var t=km,n=Cm,e=gm(0),r=null,i=Pm,o=zm,a=$m,u=null,c=Nm(f);function f(){var f,s,l=+t.apply(this,arguments),h=+n.apply(this,arguments),d=i.apply(this,arguments)-Am,p=o.apply(this,arguments)-Am,g=ym(p-d),y=p>d;if(u||(u=f=c()),h<l&&(s=h,h=l,l=s),h>Mm)if(g>Sm-Mm)u.moveTo(h*_m(d),h*xm(d)),u.arc(0,0,h,d,p,!y),l>Mm&&(u.moveTo(l*_m(p),l*xm(p)),u.arc(0,0,l,p,d,y));else{var v,_,b=d,m=p,x=d,w=p,M=g,T=g,A=a.apply(this,arguments)/2,S=A>Mm&&(r?+r.apply(this,arguments):wm(l*l+h*h)),E=mm(ym(h-l)/2,+e.apply(this,arguments)),N=E,k=E;if(S>Mm){var C=Em(S/l*xm(A)),P=Em(S/h*xm(A));(M-=2*C)>Mm?(x+=C*=y?1:-1,w-=C):(M=0,x=w=(d+p)/2),(T-=2*P)>Mm?(b+=P*=y?1:-1,m-=P):(T=0,b=m=(d+p)/2)}var z=h*_m(b),$=h*xm(b),D=l*_m(w),R=l*xm(w);if(E>Mm){var F,q=h*_m(m),U=h*xm(m),I=l*_m(x),O=l*xm(x);if(g<Tm)if(F=function(t,n,e,r,i,o,a,u){var c=e-t,f=r-n,s=a-i,l=u-o,h=l*c-s*f;if(!(h*h<Mm))return[t+(h=(s*(n-o)-l*(t-i))/h)*c,n+h*f]}(z,$,I,O,q,U,D,R)){var B=z-F[0],Y=$-F[1],L=q-F[0],j=U-F[1],H=1/xm(function(t){return t>1?0:t<-1?Tm:Math.acos(t)}((B*L+Y*j)/(wm(B*B+Y*Y)*wm(L*L+j*j)))/2),X=wm(F[0]*F[0]+F[1]*F[1]);N=mm(E,(l-X)/(H-1)),k=mm(E,(h-X)/(H+1))}else N=k=0}T>Mm?k>Mm?(v=Dm(I,O,z,$,h,k,y),_=Dm(q,U,D,R,h,k,y),u.moveTo(v.cx+v.x01,v.cy+v.y01),k<E?u.arc(v.cx,v.cy,k,vm(v.y01,v.x01),vm(_.y01,_.x01),!y):(u.arc(v.cx,v.cy,k,vm(v.y01,v.x01),vm(v.y11,v.x11),!y),u.arc(0,0,h,vm(v.cy+v.y11,v.cx+v.x11),vm(_.cy+_.y11,_.cx+_.x11),!y),u.arc(_.cx,_.cy,k,vm(_.y11,_.x11),vm(_.y01,_.x01),!y))):(u.moveTo(z,$),u.arc(0,0,h,b,m,!y)):u.moveTo(z,$),l>Mm&&M>Mm?N>Mm?(v=Dm(D,R,q,U,l,-N,y),_=Dm(z,$,I,O,l,-N,y),u.lineTo(v.cx+v.x01,v.cy+v.y01),N<E?u.arc(v.cx,v.cy,N,vm(v.y01,v.x01),vm(_.y01,_.x01),!y):(u.arc(v.cx,v.cy,N,vm(v.y01,v.x01),vm(v.y11,v.x11),!y),u.arc(0,0,l,vm(v.cy+v.y11,v.cx+v.x11),vm(_.cy+_.y11,_.cx+_.x11),y),u.arc(_.cx,_.cy,N,vm(_.y11,_.x11),vm(_.y01,_.x01),!y))):u.arc(0,0,l,w,x,y):u.lineTo(D,R)}else u.moveTo(0,0);if(u.closePath(),f)return u=null,f+""||null}return f.centroid=function(){var e=(+t.apply(this,arguments)+ +n.apply(this,arguments))/2,r=(+i.apply(this,arguments)+ +o.apply(this,arguments))/2-Tm/2;return[_m(r)*e,xm(r)*e]},f.innerRadius=function(n){return arguments.length?(t="function"==typeof n?n:gm(+n),f):t},f.outerRadius=function(t){return arguments.length?(n="function"==typeof t?t:gm(+t),f):n},f.cornerRadius=function(t){return arguments.length?(e="function"==typeof t?t:gm(+t),f):e},f.padRadius=function(t){return arguments.length?(r=null==t?null:"function"==typeof t?t:gm(+t),f):r},f.startAngle=function(t){return arguments.length?(i="function"==typeof t?t:gm(+t),f):i},f.endAngle=function(t){return arguments.length?(o="function"==typeof t?t:gm(+t),f):o},f.padAngle=function(t){return arguments.length?(a="function"==typeof t?t:gm(+t),f):a},f.context=function(t){return arguments.length?(u=null==t?null:t,f):u},f},t.area=Ym,t.areaRadial=Zm,t.ascending=n,t.autoType=function(t){for(var n in t){var e,r,i=t[n].trim();if(i)if("true"===i)i=!0;else if("false"===i)i=!1;else if("NaN"===i)i=NaN;else if(isNaN(e=+i)){if(!(r=i.match(/^([-+]\d{2})?\d{4}(-\d{2}(-\d{2})?)?(T\d{2}:\d{2}(:\d{2}(\.\d{3})?)?(Z|[-+]\d{2}:\d{2})?)?$/)))continue;yc&&r[4]&&!r[7]&&(i=i.replace(/-/g,"/").replace(/T/," ")),i=new Date(i)}else i=e;else i=null;t[n]=i}return t},t.axisBottom=function(t){return Pt(Mt,t)},t.axisLeft=function(t){return Pt(Tt,t)},t.axisRight=function(t){return Pt(wt,t)},t.axisTop=function(t){return Pt(xt,t)},t.bin=Q,t.bisect=s,t.bisectCenter=f,t.bisectLeft=c,t.bisectRight=u,t.bisector=r,t.blob=function(t,n){return fetch(t,n).then(vc)},t.blur=function(t,n){if(!((n=+n)>=0))throw new RangeError("invalid r");let e=t.length;if(!((e=Math.floor(e))>=0))throw new RangeError("invalid length");if(!e||!n)return t;const r=y(n),i=t.slice();return r(t,i,0,e,1),r(i,t,0,e,1),r(t,i,0,e,1),t},t.blur2=l,t.blurImage=h,t.brush=function(){return wa(la)},t.brushSelection=function(t){var n=t.__brush;return n?n.dim.output(n.selection):null},t.brushX=function(){return wa(fa)},t.brushY=function(){return wa(sa)},t.buffer=function(t,n){return fetch(t,n).then(_c)},t.chord=function(){return za(!1,!1)},t.chordDirected=function(){return za(!0,!1)},t.chordTranspose=function(){return za(!1,!0)},t.cluster=function(){var t=Ld,n=1,e=1,r=!1;function i(i){var o,a=0;i.eachAfter((function(n){var e=n.children;e?(n.x=function(t){return t.reduce(jd,0)/t.length}(e),n.y=function(t){return 1+t.reduce(Hd,0)}(e)):(n.x=o?a+=t(n,o):0,n.y=0,o=n)}));var u=function(t){for(var n;n=t.children;)t=n[0];return t}(i),c=function(t){for(var n;n=t.children;)t=n[n.length-1];return t}(i),f=u.x-t(u,c)/2,s=c.x+t(c,u)/2;return i.eachAfter(r?function(t){t.x=(t.x-i.x)*n,t.y=(i.y-t.y)*e}:function(t){t.x=(t.x-f)/(s-f)*n,t.y=(1-(i.y?t.y/i.y:1))*e})}return i.separation=function(n){return arguments.length?(t=n,i):t},i.size=function(t){return arguments.length?(r=!1,n=+t[0],e=+t[1],i):r?null:[n,e]},i.nodeSize=function(t){return arguments.length?(r=!0,n=+t[0],e=+t[1],i):r?[n,e]:null},i},t.color=ze,t.contourDensity=function(){var t=fu,n=su,e=lu,r=960,i=500,o=20,a=2,u=3*o,c=r+2*u>>a,f=i+2*u>>a,s=Qa(20);function h(r){var i=new Float32Array(c*f),s=Math.pow(2,-a),h=-1;for(const o of r){var d=(t(o,++h,r)+u)*s,p=(n(o,h,r)+u)*s,g=+e(o,h,r);if(g&&d>=0&&d<c&&p>=0&&p<f){var y=Math.floor(d),v=Math.floor(p),_=d-y-.5,b=p-v-.5;i[y+v*c]+=(1-_)*(1-b)*g,i[y+1+v*c]+=_*(1-b)*g,i[y+1+(v+1)*c]+=_*b*g,i[y+(v+1)*c]+=(1-_)*b*g}}return l({data:i,width:c,height:f},o*s),i}function d(t){var n=h(t),e=s(n),r=Math.pow(2,2*a);return Array.isArray(e)||(e=G(Number.MIN_VALUE,J(n)/r,e)),iu().size([c,f]).thresholds(e.map((t=>t*r)))(n).map(((t,n)=>(t.value=+e[n],p(t))))}function p(t){return t.coordinates.forEach(g),t}function g(t){t.forEach(y)}function y(t){t.forEach(v)}function v(t){t[0]=t[0]*Math.pow(2,a)-u,t[1]=t[1]*Math.pow(2,a)-u}function _(){return c=r+2*(u=3*o)>>a,f=i+2*u>>a,d}return d.contours=function(t){var n=h(t),e=iu().size([c,f]),r=Math.pow(2,2*a),i=t=>{t=+t;var i=p(e.contour(n,t*r));return i.value=t,i};return Object.defineProperty(i,"max",{get:()=>J(n)/r}),i},d.x=function(n){return arguments.length?(t="function"==typeof n?n:Qa(+n),d):t},d.y=function(t){return arguments.length?(n="function"==typeof t?t:Qa(+t),d):n},d.weight=function(t){return arguments.length?(e="function"==typeof t?t:Qa(+t),d):e},d.size=function(t){if(!arguments.length)return[r,i];var n=+t[0],e=+t[1];if(!(n>=0&&e>=0))throw new Error("invalid size");return r=n,i=e,_()},d.cellSize=function(t){if(!arguments.length)return 1<<a;if(!((t=+t)>=1))throw new Error("invalid cell size");return a=Math.floor(Math.log(t)/Math.LN2),_()},d.thresholds=function(t){return arguments.length?(s="function"==typeof t?t:Array.isArray(t)?Qa(Za.call(t)):Qa(t),d):s},d.bandwidth=function(t){if(!arguments.length)return Math.sqrt(o*(o+1));if(!((t=+t)>=0))throw new Error("invalid bandwidth");return o=(Math.sqrt(4*t*t+1)-1)/2,_()},d},t.contours=iu,t.count=v,t.create=function(t){return Zn(Yt(t).call(document.documentElement))},t.creator=Yt,t.cross=function(...t){const n="function"==typeof t[t.length-1]&&function(t){return n=>t(...n)}(t.pop()),e=(t=t.map(m)).map(_),r=t.length-1,i=new Array(r+1).fill(0),o=[];if(r<0||e.some(b))return o;for(;;){o.push(i.map(((n,e)=>t[e][n])));let a=r;for(;++i[a]===e[a];){if(0===a)return n?o.map(n):o;i[a--]=0}}},t.csv=wc,t.csvFormat=rc,t.csvFormatBody=ic,t.csvFormatRow=ac,t.csvFormatRows=oc,t.csvFormatValue=uc,t.csvParse=nc,t.csvParseRows=ec,t.cubehelix=Tr,t.cumsum=function(t,n){var e=0,r=0;return Float64Array.from(t,void 0===n?t=>e+=+t||0:i=>e+=+n(i,r++,t)||0)},t.curveBasis=function(t){return new Rx(t)},t.curveBasisClosed=function(t){return new Fx(t)},t.curveBasisOpen=function(t){return new qx(t)},t.curveBumpX=tx,t.curveBumpY=nx,t.curveBundle=Ix,t.curveCardinal=Yx,t.curveCardinalClosed=jx,t.curveCardinalOpen=Xx,t.curveCatmullRom=Wx,t.curveCatmullRomClosed=Kx,t.curveCatmullRomOpen=Jx,t.curveLinear=Um,t.curveLinearClosed=function(t){return new tw(t)},t.curveMonotoneX=function(t){return new ow(t)},t.curveMonotoneY=function(t){return new aw(t)},t.curveNatural=function(t){return new cw(t)},t.curveStep=function(t){return new sw(t,.5)},t.curveStepAfter=function(t){return new sw(t,1)},t.curveStepBefore=function(t){return new sw(t,0)},t.descending=e,t.deviation=w,t.difference=function(t,...n){t=new InternSet(t);for(const e of n)for(const n of e)t.delete(n);return t},t.disjoint=function(t,n){const e=n[Symbol.iterator](),r=new InternSet;for(const n of t){if(r.has(n))return!1;let t,i;for(;({value:t,done:i}=e.next())&&!i;){if(Object.is(n,t))return!1;r.add(t)}}return!0},t.dispatch=$t,t.drag=function(){var t,n,e,r,i=se,o=le,a=he,u=de,c={},f=$t("start","drag","end"),s=0,l=0;function h(t){t.on("mousedown.drag",d).filter(u).on("touchstart.drag",y).on("touchmove.drag",v,ee).on("touchend.drag touchcancel.drag",_).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function d(a,u){if(!r&&i.call(this,a,u)){var c=b(this,o.call(this,a,u),a,u,"mouse");c&&(Zn(a.view).on("mousemove.drag",p,re).on("mouseup.drag",g,re),ae(a.view),ie(a),e=!1,t=a.clientX,n=a.clientY,c("start",a))}}function p(r){if(oe(r),!e){var i=r.clientX-t,o=r.clientY-n;e=i*i+o*o>l}c.mouse("drag",r)}function g(t){Zn(t.view).on("mousemove.drag mouseup.drag",null),ue(t.view,e),oe(t),c.mouse("end",t)}function y(t,n){if(i.call(this,t,n)){var e,r,a=t.changedTouches,u=o.call(this,t,n),c=a.length;for(e=0;e<c;++e)(r=b(this,u,t,n,a[e].identifier,a[e]))&&(ie(t),r("start",t,a[e]))}}function v(t){var n,e,r=t.changedTouches,i=r.length;for(n=0;n<i;++n)(e=c[r[n].identifier])&&(oe(t),e("drag",t,r[n]))}function _(t){var n,e,i=t.changedTouches,o=i.length;for(r&&clearTimeout(r),r=setTimeout((function(){r=null}),500),n=0;n<o;++n)(e=c[i[n].identifier])&&(ie(t),e("end",t,i[n]))}function b(t,n,e,r,i,o){var u,l,d,p=f.copy(),g=ne(o||e,n);if(null!=(d=a.call(t,new fe("beforestart",{sourceEvent:e,target:h,identifier:i,active:s,x:g[0],y:g[1],dx:0,dy:0,dispatch:p}),r)))return u=d.x-g[0]||0,l=d.y-g[1]||0,function e(o,a,f){var y,v=g;switch(o){case"start":c[i]=e,y=s++;break;case"end":delete c[i],--s;case"drag":g=ne(f||a,n),y=s}p.call(o,t,new fe(o,{sourceEvent:a,subject:d,target:h,identifier:i,active:y,x:g[0]+u,y:g[1]+l,dx:g[0]-v[0],dy:g[1]-v[1],dispatch:p}),r)}}return h.filter=function(t){return arguments.length?(i="function"==typeof t?t:ce(!!t),h):i},h.container=function(t){return arguments.length?(o="function"==typeof t?t:ce(t),h):o},h.subject=function(t){return arguments.length?(a="function"==typeof t?t:ce(t),h):a},h.touchable=function(t){return arguments.length?(u="function"==typeof t?t:ce(!!t),h):u},h.on=function(){var t=f.on.apply(f,arguments);return t===f?h:t},h.clickDistance=function(t){return arguments.length?(l=(t=+t)*t,h):Math.sqrt(l)},h},t.dragDisable=ae,t.dragEnable=ue,t.dsv=function(t,n,e,r){3===arguments.length&&"function"==typeof e&&(r=e,e=void 0);var i=Ju(t);return mc(n,e).then((function(t){return i.parse(t,r)}))},t.dsvFormat=Ju,t.easeBack=Lo,t.easeBackIn=Bo,t.easeBackInOut=Lo,t.easeBackOut=Yo,t.easeBounce=Io,t.easeBounceIn=function(t){return 1-Io(1-t)},t.easeBounceInOut=function(t){return((t*=2)<=1?1-Io(1-t):Io(t-1)+1)/2},t.easeBounceOut=Io,t.easeCircle=No,t.easeCircleIn=function(t){return 1-Math.sqrt(1-t*t)},t.easeCircleInOut=No,t.easeCircleOut=function(t){return Math.sqrt(1- --t*t)},t.easeCubic=bo,t.easeCubicIn=function(t){return t*t*t},t.easeCubicInOut=bo,t.easeCubicOut=function(t){return--t*t*t+1},t.easeElastic=Xo,t.easeElasticIn=Ho,t.easeElasticInOut=Go,t.easeElasticOut=Xo,t.easeExp=Eo,t.easeExpIn=function(t){return So(1-+t)},t.easeExpInOut=Eo,t.easeExpOut=function(t){return 1-So(t)},t.easeLinear=t=>+t,t.easePoly=wo,t.easePolyIn=mo,t.easePolyInOut=wo,t.easePolyOut=xo,t.easeQuad=_o,t.easeQuadIn=function(t){return t*t},t.easeQuadInOut=_o,t.easeQuadOut=function(t){return t*(2-t)},t.easeSin=Ao,t.easeSinIn=function(t){return 1==+t?1:1-Math.cos(t*To)},t.easeSinInOut=Ao,t.easeSinOut=function(t){return Math.sin(t*To)},t.every=function(t,n){if("function"!=typeof n)throw new TypeError("test is not a function");let e=-1;for(const r of t)if(!n(r,++e,t))return!1;return!0},t.extent=M,t.fcumsum=function(t,n){const e=new T;let r=-1;return Float64Array.from(t,void 0===n?t=>e.add(+t||0):i=>e.add(+n(i,++r,t)||0))},t.filter=function(t,n){if("function"!=typeof n)throw new TypeError("test is not a function");const e=[];let r=-1;for(const i of t)n(i,++r,t)&&e.push(i);return e},t.flatGroup=function(t,...n){return z(P(t,...n),n)},t.flatRollup=function(t,n,...e){return z(D(t,n,...e),e)},t.forceCenter=function(t,n){var e,r=1;function i(){var i,o,a=e.length,u=0,c=0;for(i=0;i<a;++i)u+=(o=e[i]).x,c+=o.y;for(u=(u/a-t)*r,c=(c/a-n)*r,i=0;i<a;++i)(o=e[i]).x-=u,o.y-=c}return null==t&&(t=0),null==n&&(n=0),i.initialize=function(t){e=t},i.x=function(n){return arguments.length?(t=+n,i):t},i.y=function(t){return arguments.length?(n=+t,i):n},i.strength=function(t){return arguments.length?(r=+t,i):r},i},t.forceCollide=function(t){var n,e,r,i=1,o=1;function a(){for(var t,a,c,f,s,l,h,d=n.length,p=0;p<o;++p)for(a=$c(n,Ic,Oc).visitAfter(u),t=0;t<d;++t)c=n[t],l=e[c.index],h=l*l,f=c.x+c.vx,s=c.y+c.vy,a.visit(g);function g(t,n,e,o,a){var u=t.data,d=t.r,p=l+d;if(!u)return n>f+p||o<f-p||e>s+p||a<s-p;if(u.index>c.index){var g=f-u.x-u.vx,y=s-u.y-u.vy,v=g*g+y*y;v<p*p&&(0===g&&(v+=(g=Uc(r))*g),0===y&&(v+=(y=Uc(r))*y),v=(p-(v=Math.sqrt(v)))/v*i,c.vx+=(g*=v)*(p=(d*=d)/(h+d)),c.vy+=(y*=v)*p,u.vx-=g*(p=1-p),u.vy-=y*p)}}}function u(t){if(t.data)return t.r=e[t.data.index];for(var n=t.r=0;n<4;++n)t[n]&&t[n].r>t.r&&(t.r=t[n].r)}function c(){if(n){var r,i,o=n.length;for(e=new Array(o),r=0;r<o;++r)i=n[r],e[i.index]=+t(i,r,n)}}return"function"!=typeof t&&(t=qc(null==t?1:+t)),a.initialize=function(t,e){n=t,r=e,c()},a.iterations=function(t){return arguments.length?(o=+t,a):o},a.strength=function(t){return arguments.length?(i=+t,a):i},a.radius=function(n){return arguments.length?(t="function"==typeof n?n:qc(+n),c(),a):t},a},t.forceLink=function(t){var n,e,r,i,o,a,u=Bc,c=function(t){return 1/Math.min(i[t.source.index],i[t.target.index])},f=qc(30),s=1;function l(r){for(var i=0,u=t.length;i<s;++i)for(var c,f,l,h,d,p,g,y=0;y<u;++y)f=(c=t[y]).source,h=(l=c.target).x+l.vx-f.x-f.vx||Uc(a),d=l.y+l.vy-f.y-f.vy||Uc(a),h*=p=((p=Math.sqrt(h*h+d*d))-e[y])/p*r*n[y],d*=p,l.vx-=h*(g=o[y]),l.vy-=d*g,f.vx+=h*(g=1-g),f.vy+=d*g}function h(){if(r){var a,c,f=r.length,s=t.length,l=new Map(r.map(((t,n)=>[u(t,n,r),t])));for(a=0,i=new Array(f);a<s;++a)(c=t[a]).index=a,"object"!=typeof c.source&&(c.source=Yc(l,c.source)),"object"!=typeof c.target&&(c.target=Yc(l,c.target)),i[c.source.index]=(i[c.source.index]||0)+1,i[c.target.index]=(i[c.target.index]||0)+1;for(a=0,o=new Array(s);a<s;++a)c=t[a],o[a]=i[c.source.index]/(i[c.source.index]+i[c.target.index]);n=new Array(s),d(),e=new Array(s),p()}}function d(){if(r)for(var e=0,i=t.length;e<i;++e)n[e]=+c(t[e],e,t)}function p(){if(r)for(var n=0,i=t.length;n<i;++n)e[n]=+f(t[n],n,t)}return null==t&&(t=[]),l.initialize=function(t,n){r=t,a=n,h()},l.links=function(n){return arguments.length?(t=n,h(),l):t},l.id=function(t){return arguments.length?(u=t,l):u},l.iterations=function(t){return arguments.length?(s=+t,l):s},l.strength=function(t){return arguments.length?(c="function"==typeof t?t:qc(+t),d(),l):c},l.distance=function(t){return arguments.length?(f="function"==typeof t?t:qc(+t),p(),l):f},l},t.forceManyBody=function(){var t,n,e,r,i,o=qc(-30),a=1,u=1/0,c=.81;function f(e){var i,o=t.length,a=$c(t,Xc,Gc).visitAfter(l);for(r=e,i=0;i<o;++i)n=t[i],a.visit(h)}function s(){if(t){var n,e,r=t.length;for(i=new Array(r),n=0;n<r;++n)e=t[n],i[e.index]=+o(e,n,t)}}function l(t){var n,e,r,o,a,u=0,c=0;if(t.length){for(r=o=a=0;a<4;++a)(n=t[a])&&(e=Math.abs(n.value))&&(u+=n.value,c+=e,r+=e*n.x,o+=e*n.y);t.x=r/c,t.y=o/c}else{(n=t).x=n.data.x,n.y=n.data.y;do{u+=i[n.data.index]}while(n=n.next)}t.value=u}function h(t,o,f,s){if(!t.value)return!0;var l=t.x-n.x,h=t.y-n.y,d=s-o,p=l*l+h*h;if(d*d/c<p)return p<u&&(0===l&&(p+=(l=Uc(e))*l),0===h&&(p+=(h=Uc(e))*h),p<a&&(p=Math.sqrt(a*p)),n.vx+=l*t.value*r/p,n.vy+=h*t.value*r/p),!0;if(!(t.length||p>=u)){(t.data!==n||t.next)&&(0===l&&(p+=(l=Uc(e))*l),0===h&&(p+=(h=Uc(e))*h),p<a&&(p=Math.sqrt(a*p)));do{t.data!==n&&(d=i[t.data.index]*r/p,n.vx+=l*d,n.vy+=h*d)}while(t=t.next)}}return f.initialize=function(n,r){t=n,e=r,s()},f.strength=function(t){return arguments.length?(o="function"==typeof t?t:qc(+t),s(),f):o},f.distanceMin=function(t){return arguments.length?(a=t*t,f):Math.sqrt(a)},f.distanceMax=function(t){return arguments.length?(u=t*t,f):Math.sqrt(u)},f.theta=function(t){return arguments.length?(c=t*t,f):Math.sqrt(c)},f},t.forceRadial=function(t,n,e){var r,i,o,a=qc(.1);function u(t){for(var a=0,u=r.length;a<u;++a){var c=r[a],f=c.x-n||1e-6,s=c.y-e||1e-6,l=Math.sqrt(f*f+s*s),h=(o[a]-l)*i[a]*t/l;c.vx+=f*h,c.vy+=s*h}}function c(){if(r){var n,e=r.length;for(i=new Array(e),o=new Array(e),n=0;n<e;++n)o[n]=+t(r[n],n,r),i[n]=isNaN(o[n])?0:+a(r[n],n,r)}}return"function"!=typeof t&&(t=qc(+t)),null==n&&(n=0),null==e&&(e=0),u.initialize=function(t){r=t,c()},u.strength=function(t){return arguments.length?(a="function"==typeof t?t:qc(+t),c(),u):a},u.radius=function(n){return arguments.length?(t="function"==typeof n?n:qc(+n),c(),u):t},u.x=function(t){return arguments.length?(n=+t,u):n},u.y=function(t){return arguments.length?(e=+t,u):e},u},t.forceSimulation=function(t){var n,e=1,r=.001,i=1-Math.pow(r,1/300),o=0,a=.6,u=new Map,c=Ni(l),f=$t("tick","end"),s=function(){let t=1;return()=>(t=(Lc*t+jc)%Hc)/Hc}();function l(){h(),f.call("tick",n),e<r&&(c.stop(),f.call("end",n))}function h(r){var c,f,s=t.length;void 0===r&&(r=1);for(var l=0;l<r;++l)for(e+=(o-e)*i,u.forEach((function(t){t(e)})),c=0;c<s;++c)null==(f=t[c]).fx?f.x+=f.vx*=a:(f.x=f.fx,f.vx=0),null==f.fy?f.y+=f.vy*=a:(f.y=f.fy,f.vy=0);return n}function d(){for(var n,e=0,r=t.length;e<r;++e){if((n=t[e]).index=e,null!=n.fx&&(n.x=n.fx),null!=n.fy&&(n.y=n.fy),isNaN(n.x)||isNaN(n.y)){var i=10*Math.sqrt(.5+e),o=e*Vc;n.x=i*Math.cos(o),n.y=i*Math.sin(o)}(isNaN(n.vx)||isNaN(n.vy))&&(n.vx=n.vy=0)}}function p(n){return n.initialize&&n.initialize(t,s),n}return null==t&&(t=[]),d(),n={tick:h,restart:function(){return c.restart(l),n},stop:function(){return c.stop(),n},nodes:function(e){return arguments.length?(t=e,d(),u.forEach(p),n):t},alpha:function(t){return arguments.length?(e=+t,n):e},alphaMin:function(t){return arguments.length?(r=+t,n):r},alphaDecay:function(t){return arguments.length?(i=+t,n):+i},alphaTarget:function(t){return arguments.length?(o=+t,n):o},velocityDecay:function(t){return arguments.length?(a=1-t,n):1-a},randomSource:function(t){return arguments.length?(s=t,u.forEach(p),n):s},force:function(t,e){return arguments.length>1?(null==e?u.delete(t):u.set(t,p(e)),n):u.get(t)},find:function(n,e,r){var i,o,a,u,c,f=0,s=t.length;for(null==r?r=1/0:r*=r,f=0;f<s;++f)(a=(i=n-(u=t[f]).x)*i+(o=e-u.y)*o)<r&&(c=u,r=a);return c},on:function(t,e){return arguments.length>1?(f.on(t,e),n):f.on(t)}}},t.forceX=function(t){var n,e,r,i=qc(.1);function o(t){for(var i,o=0,a=n.length;o<a;++o)(i=n[o]).vx+=(r[o]-i.x)*e[o]*t}function a(){if(n){var o,a=n.length;for(e=new Array(a),r=new Array(a),o=0;o<a;++o)e[o]=isNaN(r[o]=+t(n[o],o,n))?0:+i(n[o],o,n)}}return"function"!=typeof t&&(t=qc(null==t?0:+t)),o.initialize=function(t){n=t,a()},o.strength=function(t){return arguments.length?(i="function"==typeof t?t:qc(+t),a(),o):i},o.x=function(n){return arguments.length?(t="function"==typeof n?n:qc(+n),a(),o):t},o},t.forceY=function(t){var n,e,r,i=qc(.1);function o(t){for(var i,o=0,a=n.length;o<a;++o)(i=n[o]).vy+=(r[o]-i.y)*e[o]*t}function a(){if(n){var o,a=n.length;for(e=new Array(a),r=new Array(a),o=0;o<a;++o)e[o]=isNaN(r[o]=+t(n[o],o,n))?0:+i(n[o],o,n)}}return"function"!=typeof t&&(t=qc(null==t?0:+t)),o.initialize=function(t){n=t,a()},o.strength=function(t){return arguments.length?(i="function"==typeof t?t:qc(+t),a(),o):i},o.y=function(n){return arguments.length?(t="function"==typeof n?n:qc(+n),a(),o):t},o},t.formatDefaultLocale=ff,t.formatLocale=cf,t.formatSpecifier=Jc,t.fsum=function(t,n){const e=new T;if(void 0===n)for(let n of t)(n=+n)&&e.add(n);else{let r=-1;for(let i of t)(i=+n(i,++r,t))&&e.add(i)}return+e},t.geoAlbers=xd,t.geoAlbersUsa=function(){var t,n,e,r,i,o,a=xd(),u=md().rotate([154,0]).center([-2,58.5]).parallels([55,65]),c=md().rotate([157,0]).center([-3,19.9]).parallels([8,18]),f={point:function(t,n){o=[t,n]}};function s(t){var n=t[0],a=t[1];return o=null,e.point(n,a),o||(r.point(n,a),o)||(i.point(n,a),o)}function l(){return t=n=null,s}return s.invert=function(t){var n=a.scale(),e=a.translate(),r=(t[0]-e[0])/n,i=(t[1]-e[1])/n;return(i>=.12&&i<.234&&r>=-.425&&r<-.214?u:i>=.166&&i<.234&&r>=-.214&&r<-.115?c:a).invert(t)},s.stream=function(e){return t&&n===e?t:(r=[a.stream(n=e),u.stream(e),c.stream(e)],i=r.length,t={point:function(t,n){for(var e=-1;++e<i;)r[e].point(t,n)},sphere:function(){for(var t=-1;++t<i;)r[t].sphere()},lineStart:function(){for(var t=-1;++t<i;)r[t].lineStart()},lineEnd:function(){for(var t=-1;++t<i;)r[t].lineEnd()},polygonStart:function(){for(var t=-1;++t<i;)r[t].polygonStart()},polygonEnd:function(){for(var t=-1;++t<i;)r[t].polygonEnd()}});var r,i},s.precision=function(t){return arguments.length?(a.precision(t),u.precision(t),c.precision(t),l()):a.precision()},s.scale=function(t){return arguments.length?(a.scale(t),u.scale(.35*t),c.scale(t),s.translate(a.translate())):a.scale()},s.translate=function(t){if(!arguments.length)return a.translate();var n=a.scale(),o=+t[0],s=+t[1];return e=a.translate(t).clipExtent([[o-.455*n,s-.238*n],[o+.455*n,s+.238*n]]).stream(f),r=u.translate([o-.307*n,s+.201*n]).clipExtent([[o-.425*n+df,s+.12*n+df],[o-.214*n-df,s+.234*n-df]]).stream(f),i=c.translate([o-.205*n,s+.212*n]).clipExtent([[o-.214*n+df,s+.166*n+df],[o-.115*n-df,s+.234*n-df]]).stream(f),l()},s.fitExtent=function(t,n){return ud(s,t,n)},s.fitSize=function(t,n){return cd(s,t,n)},s.fitWidth=function(t,n){return fd(s,t,n)},s.fitHeight=function(t,n){return sd(s,t,n)},s.scale(1070)},t.geoArea=function(t){return us=new T,Lf(t,cs),2*us},t.geoAzimuthalEqualArea=function(){return yd(Td).scale(124.75).clipAngle(179.999)},t.geoAzimuthalEqualAreaRaw=Td,t.geoAzimuthalEquidistant=function(){return yd(Ad).scale(79.4188).clipAngle(179.999)},t.geoAzimuthalEquidistantRaw=Ad,t.geoBounds=function(t){var n,e,r,i,o,a,u;if(Qf=Kf=-(Wf=Zf=1/0),is=[],Lf(t,Fs),e=is.length){for(is.sort(Hs),n=1,o=[r=is[0]];n<e;++n)Xs(r,(i=is[n])[0])||Xs(r,i[1])?(js(r[0],i[1])>js(r[0],r[1])&&(r[1]=i[1]),js(i[0],r[1])>js(r[0],r[1])&&(r[0]=i[0])):o.push(r=i);for(a=-1/0,n=0,r=o[e=o.length-1];n<=e;r=i,++n)i=o[n],(u=js(r[1],i[0]))>a&&(a=u,Wf=i[0],Kf=r[1])}return is=os=null,Wf===1/0||Zf===1/0?[[NaN,NaN],[NaN,NaN]]:[[Wf,Zf],[Kf,Qf]]},t.geoCentroid=function(t){ms=xs=ws=Ms=Ts=As=Ss=Es=0,Ns=new T,ks=new T,Cs=new T,Lf(t,Gs);var n=+Ns,e=+ks,r=+Cs,i=Ef(n,e,r);return i<pf&&(n=As,e=Ss,r=Es,xs<df&&(n=ws,e=Ms,r=Ts),(i=Ef(n,e,r))<pf)?[NaN,NaN]:[Mf(e,n)*bf,Rf(r/i)*bf]},t.geoCircle=function(){var t,n,e=il([0,0]),r=il(90),i=il(6),o={point:function(e,r){t.push(e=n(e,r)),e[0]*=bf,e[1]*=bf}};function a(){var a=e.apply(this,arguments),u=r.apply(this,arguments)*mf,c=i.apply(this,arguments)*mf;return t=[],n=ul(-a[0]*mf,-a[1]*mf,0).invert,hl(o,u,c,1),a={type:"Polygon",coordinates:[t]},t=n=null,a}return a.center=function(t){return arguments.length?(e="function"==typeof t?t:il([+t[0],+t[1]]),a):e},a.radius=function(t){return arguments.length?(r="function"==typeof t?t:il(+t),a):r},a.precision=function(t){return arguments.length?(i="function"==typeof t?t:il(+t),a):i},a},t.geoClipAntimeridian=Tl,t.geoClipCircle=Al,t.geoClipExtent=function(){var t,n,e,r=0,i=0,o=960,a=500;return e={stream:function(e){return t&&n===e?t:t=zl(r,i,o,a)(n=e)},extent:function(u){return arguments.length?(r=+u[0][0],i=+u[0][1],o=+u[1][0],a=+u[1][1],t=n=null,e):[[r,i],[o,a]]}}},t.geoClipRectangle=zl,t.geoConicConformal=function(){return _d(kd).scale(109.5).parallels([30,30])},t.geoConicConformalRaw=kd,t.geoConicEqualArea=md,t.geoConicEqualAreaRaw=bd,t.geoConicEquidistant=function(){return _d(Pd).scale(131.154).center([0,13.9389])},t.geoConicEquidistantRaw=Pd,t.geoContains=function(t,n){return(t&&Bl.hasOwnProperty(t.type)?Bl[t.type]:Ll)(t,n)},t.geoDistance=Ol,t.geoEqualEarth=function(){return yd(qd).scale(177.158)},t.geoEqualEarthRaw=qd,t.geoEquirectangular=function(){return yd(Cd).scale(152.63)},t.geoEquirectangularRaw=Cd,t.geoGnomonic=function(){return yd(Ud).scale(144.049).clipAngle(60)},t.geoGnomonicRaw=Ud,t.geoGraticule=Kl,t.geoGraticule10=function(){return Kl()()},t.geoIdentity=function(){var t,n,e,r,i,o,a,u=1,c=0,f=0,s=1,l=1,h=0,d=null,p=1,g=1,y=id({point:function(t,n){var e=b([t,n]);this.stream.point(e[0],e[1])}}),v=eh;function _(){return p=u*s,g=u*l,o=a=null,b}function b(e){var r=e[0]*p,i=e[1]*g;if(h){var o=i*t-r*n;r=r*t+i*n,i=o}return[r+c,i+f]}return b.invert=function(e){var r=e[0]-c,i=e[1]-f;if(h){var o=i*t+r*n;r=r*t-i*n,i=o}return[r/p,i/g]},b.stream=function(t){return o&&a===t?o:o=y(v(a=t))},b.postclip=function(t){return arguments.length?(v=t,d=e=r=i=null,_()):v},b.clipExtent=function(t){return arguments.length?(v=null==t?(d=e=r=i=null,eh):zl(d=+t[0][0],e=+t[0][1],r=+t[1][0],i=+t[1][1]),_()):null==d?null:[[d,e],[r,i]]},b.scale=function(t){return arguments.length?(u=+t,_()):u},b.translate=function(t){return arguments.length?(c=+t[0],f=+t[1],_()):[c,f]},b.angle=function(e){return arguments.length?(n=Cf(h=e%360*mf),t=Tf(h),_()):h*bf},b.reflectX=function(t){return arguments.length?(s=t?-1:1,_()):s<0},b.reflectY=function(t){return arguments.length?(l=t?-1:1,_()):l<0},b.fitExtent=function(t,n){return ud(b,t,n)},b.fitSize=function(t,n){return cd(b,t,n)},b.fitWidth=function(t,n){return fd(b,t,n)},b.fitHeight=function(t,n){return sd(b,t,n)},b},t.geoInterpolate=function(t,n){var e=t[0]*mf,r=t[1]*mf,i=n[0]*mf,o=n[1]*mf,a=Tf(r),u=Cf(r),c=Tf(o),f=Cf(o),s=a*Tf(e),l=a*Cf(e),h=c*Tf(i),d=c*Cf(i),p=2*Rf(zf(Ff(o-r)+a*c*Ff(i-e))),g=Cf(p),y=p?function(t){var n=Cf(t*=p)/g,e=Cf(p-t)/g,r=e*s+n*h,i=e*l+n*d,o=e*u+n*f;return[Mf(i,r)*bf,Mf(o,zf(r*r+i*i))*bf]}:function(){return[e*bf,r*bf]};return y.distance=p,y},t.geoLength=ql,t.geoMercator=function(){return Ed(Sd).scale(961/_f)},t.geoMercatorRaw=Sd,t.geoNaturalEarth1=function(){return yd(Id).scale(175.295)},t.geoNaturalEarth1Raw=Id,t.geoOrthographic=function(){return yd(Od).scale(249.5).clipAngle(90+df)},t.geoOrthographicRaw=Od,t.geoPath=function(t,n){let e,r,i=3,o=4.5;function a(t){return t&&("function"==typeof o&&r.pointRadius(+o.apply(this,arguments)),Lf(t,e(r))),r.result()}return a.area=function(t){return Lf(t,e(sh)),sh.result()},a.measure=function(t){return Lf(t,e(Kh)),Kh.result()},a.bounds=function(t){return Lf(t,e(mh)),mh.result()},a.centroid=function(t){return Lf(t,e(Oh)),Oh.result()},a.projection=function(n){return arguments.length?(e=null==n?(t=null,eh):(t=n).stream,a):t},a.context=function(t){return arguments.length?(r=null==t?(n=null,new ed(i)):new Bh(n=t),"function"!=typeof o&&r.pointRadius(o),a):n},a.pointRadius=function(t){return arguments.length?(o="function"==typeof t?t:(r.pointRadius(+t),+t),a):o},a.digits=function(t){if(!arguments.length)return i;if(null==t)i=null;else{const n=Math.floor(t);if(!(n>=0))throw new RangeError(`invalid digits: ${t}`);i=n}return null===n&&(r=new ed(i)),a},a.projection(t).digits(i).context(n)},t.geoProjection=yd,t.geoProjectionMutator=vd,t.geoRotation=ll,t.geoStereographic=function(){return yd(Bd).scale(250).clipAngle(142)},t.geoStereographicRaw=Bd,t.geoStream=Lf,t.geoTransform=function(t){return{stream:id(t)}},t.geoTransverseMercator=function(){var t=Ed(Yd),n=t.center,e=t.rotate;return t.center=function(t){return arguments.length?n([-t[1],t[0]]):[(t=n())[1],-t[0]]},t.rotate=function(t){return arguments.length?e([t[0],t[1],t.length>2?t[2]+90:90]):[(t=e())[0],t[1],t[2]-90]},e([0,0,90]).scale(159.155)},t.geoTransverseMercatorRaw=Yd,t.gray=function(t,n){return new ur(t,0,0,null==n?1:n)},t.greatest=ot,t.greatestIndex=function(t,e=n){if(1===e.length)return tt(t,e);let r,i=-1,o=-1;for(const n of t)++o,(i<0?0===e(n,n):e(n,r)>0)&&(r=n,i=o);return i},t.group=C,t.groupSort=function(t,e,r){return(2!==e.length?U($(t,e,r),(([t,e],[r,i])=>n(e,i)||n(t,r))):U(C(t,r),(([t,r],[i,o])=>e(r,o)||n(t,i)))).map((([t])=>t))},t.groups=P,t.hcl=dr,t.hierarchy=Gd,t.histogram=Q,t.hsl=He,t.html=Ec,t.image=function(t,n){return new Promise((function(e,r){var i=new Image;for(var o in n)i[o]=n[o];i.onerror=r,i.onload=function(){e(i)},i.src=t}))},t.index=function(t,...n){return F(t,k,R,n)},t.indexes=function(t,...n){return F(t,Array.from,R,n)},t.interpolate=Gr,t.interpolateArray=function(t,n){return(Ir(n)?Ur:Or)(t,n)},t.interpolateBasis=Er,t.interpolateBasisClosed=Nr,t.interpolateBlues=Xb,t.interpolateBrBG=ib,t.interpolateBuGn=wb,t.interpolateBuPu=Tb,t.interpolateCividis=function(t){return t=Math.max(0,Math.min(1,t)),"rgb("+Math.max(0,Math.min(255,Math.round(-4.54-t*(35.34-t*(2381.73-t*(6402.7-t*(7024.72-2710.57*t)))))))+", "+Math.max(0,Math.min(255,Math.round(32.49+t*(170.73+t*(52.82-t*(131.46-t*(176.58-67.37*t)))))))+", "+Math.max(0,Math.min(255,Math.round(81.24+t*(442.36-t*(2482.43-t*(6167.24-t*(6614.94-2475.67*t)))))))+")"},t.interpolateCool=om,t.interpolateCubehelix=li,t.interpolateCubehelixDefault=rm,t.interpolateCubehelixLong=hi,t.interpolateDate=Br,t.interpolateDiscrete=function(t){var n=t.length;return function(e){return t[Math.max(0,Math.min(n-1,Math.floor(e*n)))]}},t.interpolateGnBu=Sb,t.interpolateGreens=Vb,t.interpolateGreys=Zb,t.interpolateHcl=ci,t.interpolateHclLong=fi,t.interpolateHsl=oi,t.interpolateHslLong=ai,t.interpolateHue=function(t,n){var e=Pr(+t,+n);return function(t){var n=e(t);return n-360*Math.floor(n/360)}},t.interpolateInferno=dm,t.interpolateLab=function(t,n){var e=$r((t=ar(t)).l,(n=ar(n)).l),r=$r(t.a,n.a),i=$r(t.b,n.b),o=$r(t.opacity,n.opacity);return function(n){return t.l=e(n),t.a=r(n),t.b=i(n),t.opacity=o(n),t+""}},t.interpolateMagma=hm,t.interpolateNumber=Yr,t.interpolateNumberArray=Ur,t.interpolateObject=Lr,t.interpolateOrRd=Nb,t.interpolateOranges=em,t.interpolatePRGn=ab,t.interpolatePiYG=cb,t.interpolatePlasma=pm,t.interpolatePuBu=zb,t.interpolatePuBuGn=Cb,t.interpolatePuOr=sb,t.interpolatePuRd=Db,t.interpolatePurples=Qb,t.interpolateRainbow=function(t){(t<0||t>1)&&(t-=Math.floor(t));var n=Math.abs(t-.5);return am.h=360*t-100,am.s=1.5-1.5*n,am.l=.8-.9*n,am+""},t.interpolateRdBu=hb,t.interpolateRdGy=pb,t.interpolateRdPu=Fb,t.interpolateRdYlBu=yb,t.interpolateRdYlGn=_b,t.interpolateReds=tm,t.interpolateRgb=Dr,t.interpolateRgbBasis=Fr,t.interpolateRgbBasisClosed=qr,t.interpolateRound=Vr,t.interpolateSinebow=function(t){var n;return t=(.5-t)*Math.PI,um.r=255*(n=Math.sin(t))*n,um.g=255*(n=Math.sin(t+cm))*n,um.b=255*(n=Math.sin(t+fm))*n,um+""},t.interpolateSpectral=mb,t.interpolateString=Xr,t.interpolateTransformCss=ti,t.interpolateTransformSvg=ni,t.interpolateTurbo=function(t){return t=Math.max(0,Math.min(1,t)),"rgb("+Math.max(0,Math.min(255,Math.round(34.61+t*(1172.33-t*(10793.56-t*(33300.12-t*(38394.49-14825.05*t)))))))+", "+Math.max(0,Math.min(255,Math.round(23.31+t*(557.33+t*(1225.33-t*(3574.96-t*(1073.77+707.56*t)))))))+", "+Math.max(0,Math.min(255,Math.round(27.2+t*(3211.1-t*(15327.97-t*(27814-t*(22569.18-6838.66*t)))))))+")"},t.interpolateViridis=lm,t.interpolateWarm=im,t.interpolateYlGn=Ob,t.interpolateYlGnBu=Ub,t.interpolateYlOrBr=Yb,t.interpolateYlOrRd=jb,t.interpolateZoom=ri,t.interrupt=Gi,t.intersection=function(t,...n){t=new InternSet(t),n=n.map(vt);t:for(const e of t)for(const r of n)if(!r.has(e)){t.delete(e);continue t}return t},t.interval=function(t,n,e){var r=new Ei,i=n;return null==n?(r.restart(t,n,e),r):(r._restart=r.restart,r.restart=function(t,n,e){n=+n,e=null==e?Ai():+e,r._restart((function o(a){a+=i,r._restart(o,i+=n,e),t(a)}),n,e)},r.restart(t,n,e),r)},t.isoFormat=D_,t.isoParse=F_,t.json=function(t,n){return fetch(t,n).then(Tc)},t.lab=ar,t.lch=function(t,n,e,r){return 1===arguments.length?hr(t):new pr(e,n,t,null==r?1:r)},t.least=function(t,e=n){let r,i=!1;if(1===e.length){let o;for(const a of t){const t=e(a);(i?n(t,o)<0:0===n(t,t))&&(r=a,o=t,i=!0)}}else for(const n of t)(i?e(n,r)<0:0===e(n,n))&&(r=n,i=!0);return r},t.leastIndex=ht,t.line=Bm,t.lineRadial=Wm,t.link=ox,t.linkHorizontal=function(){return ox(tx)},t.linkRadial=function(){const t=ox(ex);return t.angle=t.x,delete t.x,t.radius=t.y,delete t.y,t},t.linkVertical=function(){return ox(nx)},t.local=Qn,t.map=function(t,n){if("function"!=typeof t[Symbol.iterator])throw new TypeError("values is not iterable");if("function"!=typeof n)throw new TypeError("mapper is not a function");return Array.from(t,((e,r)=>n(e,r,t)))},t.matcher=Vt,t.max=J,t.maxIndex=tt,t.mean=function(t,n){let e=0,r=0;if(void 0===n)for(let n of t)null!=n&&(n=+n)>=n&&(++e,r+=n);else{let i=-1;for(let o of t)null!=(o=n(o,++i,t))&&(o=+o)>=o&&(++e,r+=o)}if(e)return r/e},t.median=function(t,n){return at(t,.5,n)},t.medianIndex=function(t,n){return ct(t,.5,n)},t.merge=ft,t.min=nt,t.minIndex=et,t.mode=function(t,n){const e=new InternMap;if(void 0===n)for(let n of t)null!=n&&n>=n&&e.set(n,(e.get(n)||0)+1);else{let r=-1;for(let i of t)null!=(i=n(i,++r,t))&&i>=i&&e.set(i,(e.get(i)||0)+1)}let r,i=0;for(const[t,n]of e)n>i&&(i=n,r=t);return r},t.namespace=It,t.namespaces=Ut,t.nice=Z,t.now=Ai,t.pack=function(){var t=null,n=1,e=1,r=np;function i(i){const o=ap();return i.x=n/2,i.y=e/2,t?i.eachBefore(xp(t)).eachAfter(wp(r,.5,o)).eachBefore(Mp(1)):i.eachBefore(xp(mp)).eachAfter(wp(np,1,o)).eachAfter(wp(r,i.r/Math.min(n,e),o)).eachBefore(Mp(Math.min(n,e)/(2*i.r))),i}return i.radius=function(n){return arguments.length?(t=Jd(n),i):t},i.size=function(t){return arguments.length?(n=+t[0],e=+t[1],i):[n,e]},i.padding=function(t){return arguments.length?(r="function"==typeof t?t:ep(+t),i):r},i},t.packEnclose=function(t){return up(t,ap())},t.packSiblings=function(t){return bp(t,ap()),t},t.pairs=function(t,n=st){const e=[];let r,i=!1;for(const o of t)i&&e.push(n(r,o)),r=o,i=!0;return e},t.partition=function(){var t=1,n=1,e=0,r=!1;function i(i){var o=i.height+1;return i.x0=i.y0=e,i.x1=t,i.y1=n/o,i.eachBefore(function(t,n){return function(r){r.children&&Ap(r,r.x0,t*(r.depth+1)/n,r.x1,t*(r.depth+2)/n);var i=r.x0,o=r.y0,a=r.x1-e,u=r.y1-e;a<i&&(i=a=(i+a)/2),u<o&&(o=u=(o+u)/2),r.x0=i,r.y0=o,r.x1=a,r.y1=u}}(n,o)),r&&i.eachBefore(Tp),i}return i.round=function(t){return arguments.length?(r=!!t,i):r},i.size=function(e){return arguments.length?(t=+e[0],n=+e[1],i):[t,n]},i.padding=function(t){return arguments.length?(e=+t,i):e},i},t.path=Ia,t.pathRound=function(t=3){return new Ua(+t)},t.permute=q,t.pie=function(){var t=jm,n=Lm,e=null,r=gm(0),i=gm(Sm),o=gm(0);function a(a){var u,c,f,s,l,h=(a=Fm(a)).length,d=0,p=new Array(h),g=new Array(h),y=+r.apply(this,arguments),v=Math.min(Sm,Math.max(-Sm,i.apply(this,arguments)-y)),_=Math.min(Math.abs(v)/h,o.apply(this,arguments)),b=_*(v<0?-1:1);for(u=0;u<h;++u)(l=g[p[u]=u]=+t(a[u],u,a))>0&&(d+=l);for(null!=n?p.sort((function(t,e){return n(g[t],g[e])})):null!=e&&p.sort((function(t,n){return e(a[t],a[n])})),u=0,f=d?(v-h*b)/d:0;u<h;++u,y=s)c=p[u],s=y+((l=g[c])>0?l*f:0)+b,g[c]={data:a[c],index:u,value:l,startAngle:y,endAngle:s,padAngle:_};return g}return a.value=function(n){return arguments.length?(t="function"==typeof n?n:gm(+n),a):t},a.sortValues=function(t){return arguments.length?(n=t,e=null,a):n},a.sort=function(t){return arguments.length?(e=t,n=null,a):e},a.startAngle=function(t){return arguments.length?(r="function"==typeof t?t:gm(+t),a):r},a.endAngle=function(t){return arguments.length?(i="function"==typeof t?t:gm(+t),a):i},a.padAngle=function(t){return arguments.length?(o="function"==typeof t?t:gm(+t),a):o},a},t.piecewise=di,t.pointRadial=Km,t.pointer=ne,t.pointers=function(t,n){return t.target&&(t=te(t),void 0===n&&(n=t.currentTarget),t=t.touches||[t]),Array.from(t,(t=>ne(t,n)))},t.polygonArea=function(t){for(var n,e=-1,r=t.length,i=t[r-1],o=0;++e<r;)n=i,i=t[e],o+=n[1]*i[0]-n[0]*i[1];return o/2},t.polygonCentroid=function(t){for(var n,e,r=-1,i=t.length,o=0,a=0,u=t[i-1],c=0;++r<i;)n=u,u=t[r],c+=e=n[0]*u[1]-u[0]*n[1],o+=(n[0]+u[0])*e,a+=(n[1]+u[1])*e;return[o/(c*=3),a/c]},t.polygonContains=function(t,n){for(var e,r,i=t.length,o=t[i-1],a=n[0],u=n[1],c=o[0],f=o[1],s=!1,l=0;l<i;++l)e=(o=t[l])[0],(r=o[1])>u!=f>u&&a<(c-e)*(u-r)/(f-r)+e&&(s=!s),c=e,f=r;return s},t.polygonHull=function(t){if((e=t.length)<3)return null;var n,e,r=new Array(e),i=new Array(e);for(n=0;n<e;++n)r[n]=[+t[n][0],+t[n][1],n];for(r.sort(Hp),n=0;n<e;++n)i[n]=[r[n][0],-r[n][1]];var o=Xp(r),a=Xp(i),u=a[0]===o[0],c=a[a.length-1]===o[o.length-1],f=[];for(n=o.length-1;n>=0;--n)f.push(t[r[o[n]][2]]);for(n=+u;n<a.length-c;++n)f.push(t[r[a[n]][2]]);return f},t.polygonLength=function(t){for(var n,e,r=-1,i=t.length,o=t[i-1],a=o[0],u=o[1],c=0;++r<i;)n=a,e=u,n-=a=(o=t[r])[0],e-=u=o[1],c+=Math.hypot(n,e);return c},t.precisionFixed=sf,t.precisionPrefix=lf,t.precisionRound=hf,t.quadtree=$c,t.quantile=at,t.quantileIndex=ct,t.quantileSorted=ut,t.quantize=function(t,n){for(var e=new Array(n),r=0;r<n;++r)e[r]=t(r/(n-1));return e},t.quickselect=rt,t.radialArea=Zm,t.radialLine=Wm,t.randomBates=Jp,t.randomBernoulli=eg,t.randomBeta=og,t.randomBinomial=ag,t.randomCauchy=cg,t.randomExponential=tg,t.randomGamma=ig,t.randomGeometric=rg,t.randomInt=Wp,t.randomIrwinHall=Qp,t.randomLcg=function(t=Math.random()){let n=0|(0<=t&&t<1?t/lg:Math.abs(t));return()=>(n=1664525*n+1013904223|0,lg*(n>>>0))},t.randomLogNormal=Kp,t.randomLogistic=fg,t.randomNormal=Zp,t.randomPareto=ng,t.randomPoisson=sg,t.randomUniform=Vp,t.randomWeibull=ug,t.range=lt,t.rank=function(t,e=n){if("function"!=typeof t[Symbol.iterator])throw new TypeError("values is not iterable");let r=Array.from(t);const i=new Float64Array(r.length);2!==e.length&&(r=r.map(e),e=n);const o=(t,n)=>e(r[t],r[n]);let a,u;return(t=Uint32Array.from(r,((t,n)=>n))).sort(e===n?(t,n)=>O(r[t],r[n]):I(o)),t.forEach(((t,n)=>{const e=o(t,void 0===a?t:a);e>=0?((void 0===a||e>0)&&(a=t,u=n),i[t]=u):i[t]=NaN})),i},t.reduce=function(t,n,e){if("function"!=typeof n)throw new TypeError("reducer is not a function");const r=t[Symbol.iterator]();let i,o,a=-1;if(arguments.length<3){if(({done:i,value:e}=r.next()),i)return;++a}for(;({done:i,value:o}=r.next()),!i;)e=n(e,o,++a,t);return e},t.reverse=function(t){if("function"!=typeof t[Symbol.iterator])throw new TypeError("values is not iterable");return Array.from(t).reverse()},t.rgb=Fe,t.ribbon=function(){return Wa()},t.ribbonArrow=function(){return Wa(Va)},t.rollup=$,t.rollups=D,t.scaleBand=yg,t.scaleDiverging=function t(){var n=Ng(L_()(mg));return n.copy=function(){return B_(n,t())},dg.apply(n,arguments)},t.scaleDivergingLog=function t(){var n=Fg(L_()).domain([.1,1,10]);return n.copy=function(){return B_(n,t()).base(n.base())},dg.apply(n,arguments)},t.scaleDivergingPow=j_,t.scaleDivergingSqrt=function(){return j_.apply(null,arguments).exponent(.5)},t.scaleDivergingSymlog=function t(){var n=Ig(L_());return n.copy=function(){return B_(n,t()).constant(n.constant())},dg.apply(n,arguments)},t.scaleIdentity=function t(n){var e;function r(t){return null==t||isNaN(t=+t)?e:t}return r.invert=r,r.domain=r.range=function(t){return arguments.length?(n=Array.from(t,_g),r):n.slice()},r.unknown=function(t){return arguments.length?(e=t,r):e},r.copy=function(){return t(n).unknown(e)},n=arguments.length?Array.from(n,_g):[0,1],Ng(r)},t.scaleImplicit=pg,t.scaleLinear=function t(){var n=Sg();return n.copy=function(){return Tg(n,t())},hg.apply(n,arguments),Ng(n)},t.scaleLog=function t(){const n=Fg(Ag()).domain([1,10]);return n.copy=()=>Tg(n,t()).base(n.base()),hg.apply(n,arguments),n},t.scaleOrdinal=gg,t.scalePoint=function(){return vg(yg.apply(null,arguments).paddingInner(1))},t.scalePow=jg,t.scaleQuantile=function t(){var e,r=[],i=[],o=[];function a(){var t=0,n=Math.max(1,i.length);for(o=new Array(n-1);++t<n;)o[t-1]=ut(r,t/n);return u}function u(t){return null==t||isNaN(t=+t)?e:i[s(o,t)]}return u.invertExtent=function(t){var n=i.indexOf(t);return n<0?[NaN,NaN]:[n>0?o[n-1]:r[0],n<o.length?o[n]:r[r.length-1]]},u.domain=function(t){if(!arguments.length)return r.slice();r=[];for(let n of t)null==n||isNaN(n=+n)||r.push(n);return r.sort(n),a()},u.range=function(t){return arguments.length?(i=Array.from(t),a()):i.slice()},u.unknown=function(t){return arguments.length?(e=t,u):e},u.quantiles=function(){return o.slice()},u.copy=function(){return t().domain(r).range(i).unknown(e)},hg.apply(u,arguments)},t.scaleQuantize=function t(){var n,e=0,r=1,i=1,o=[.5],a=[0,1];function u(t){return null!=t&&t<=t?a[s(o,t,0,i)]:n}function c(){var t=-1;for(o=new Array(i);++t<i;)o[t]=((t+1)*r-(t-i)*e)/(i+1);return u}return u.domain=function(t){return arguments.length?([e,r]=t,e=+e,r=+r,c()):[e,r]},u.range=function(t){return arguments.length?(i=(a=Array.from(t)).length-1,c()):a.slice()},u.invertExtent=function(t){var n=a.indexOf(t);return n<0?[NaN,NaN]:n<1?[e,o[0]]:n>=i?[o[i-1],r]:[o[n-1],o[n]]},u.unknown=function(t){return arguments.length?(n=t,u):u},u.thresholds=function(){return o.slice()},u.copy=function(){return t().domain([e,r]).range(a).unknown(n)},hg.apply(Ng(u),arguments)},t.scaleRadial=function t(){var n,e=Sg(),r=[0,1],i=!1;function o(t){var r=function(t){return Math.sign(t)*Math.sqrt(Math.abs(t))}(e(t));return isNaN(r)?n:i?Math.round(r):r}return o.invert=function(t){return e.invert(Hg(t))},o.domain=function(t){return arguments.length?(e.domain(t),o):e.domain()},o.range=function(t){return arguments.length?(e.range((r=Array.from(t,_g)).map(Hg)),o):r.slice()},o.rangeRound=function(t){return o.range(t).round(!0)},o.round=function(t){return arguments.length?(i=!!t,o):i},o.clamp=function(t){return arguments.length?(e.clamp(t),o):e.clamp()},o.unknown=function(t){return arguments.length?(n=t,o):n},o.copy=function(){return t(e.domain(),r).round(i).clamp(e.clamp()).unknown(n)},hg.apply(o,arguments),Ng(o)},t.scaleSequential=function t(){var n=Ng(O_()(mg));return n.copy=function(){return B_(n,t())},dg.apply(n,arguments)},t.scaleSequentialLog=function t(){var n=Fg(O_()).domain([1,10]);return n.copy=function(){return B_(n,t()).base(n.base())},dg.apply(n,arguments)},t.scaleSequentialPow=Y_,t.scaleSequentialQuantile=function t(){var e=[],r=mg;function i(t){if(null!=t&&!isNaN(t=+t))return r((s(e,t,1)-1)/(e.length-1))}return i.domain=function(t){if(!arguments.length)return e.slice();e=[];for(let n of t)null==n||isNaN(n=+n)||e.push(n);return e.sort(n),i},i.interpolator=function(t){return arguments.length?(r=t,i):r},i.range=function(){return e.map(((t,n)=>r(n/(e.length-1))))},i.quantiles=function(t){return Array.from({length:t+1},((n,r)=>at(e,r/t)))},i.copy=function(){return t(r).domain(e)},dg.apply(i,arguments)},t.scaleSequentialSqrt=function(){return Y_.apply(null,arguments).exponent(.5)},t.scaleSequentialSymlog=function t(){var n=Ig(O_());return n.copy=function(){return B_(n,t()).constant(n.constant())},dg.apply(n,arguments)},t.scaleSqrt=function(){return jg.apply(null,arguments).exponent(.5)},t.scaleSymlog=function t(){var n=Ig(Ag());return n.copy=function(){return Tg(n,t()).constant(n.constant())},hg.apply(n,arguments)},t.scaleThreshold=function t(){var n,e=[.5],r=[0,1],i=1;function o(t){return null!=t&&t<=t?r[s(e,t,0,i)]:n}return o.domain=function(t){return arguments.length?(e=Array.from(t),i=Math.min(e.length,r.length-1),o):e.slice()},o.range=function(t){return arguments.length?(r=Array.from(t),i=Math.min(e.length,r.length-1),o):r.slice()},o.invertExtent=function(t){var n=r.indexOf(t);return[e[n-1],e[n]]},o.unknown=function(t){return arguments.length?(n=t,o):n},o.copy=function(){return t().domain(e).range(r).unknown(n)},hg.apply(o,arguments)},t.scaleTime=function(){return hg.apply(I_(uv,cv,tv,Zy,xy,py,sy,ay,iy,t.timeFormat).domain([new Date(2e3,0,1),new Date(2e3,0,2)]),arguments)},t.scaleUtc=function(){return hg.apply(I_(ov,av,ev,Qy,Fy,yy,hy,cy,iy,t.utcFormat).domain([Date.UTC(2e3,0,1),Date.UTC(2e3,0,2)]),arguments)},t.scan=function(t,n){const e=ht(t,n);return e<0?void 0:e},t.schemeAccent=G_,t.schemeBlues=Hb,t.schemeBrBG=rb,t.schemeBuGn=xb,t.schemeBuPu=Mb,t.schemeCategory10=X_,t.schemeDark2=V_,t.schemeGnBu=Ab,t.schemeGreens=Gb,t.schemeGreys=Wb,t.schemeOrRd=Eb,t.schemeOranges=nm,t.schemePRGn=ob,t.schemePaired=W_,t.schemePastel1=Z_,t.schemePastel2=K_,t.schemePiYG=ub,t.schemePuBu=Pb,t.schemePuBuGn=kb,t.schemePuOr=fb,t.schemePuRd=$b,t.schemePurples=Kb,t.schemeRdBu=lb,t.schemeRdGy=db,t.schemeRdPu=Rb,t.schemeRdYlBu=gb,t.schemeRdYlGn=vb,t.schemeReds=Jb,t.schemeSet1=Q_,t.schemeSet2=J_,t.schemeSet3=tb,t.schemeSpectral=bb,t.schemeTableau10=nb,t.schemeYlGn=Ib,t.schemeYlGnBu=qb,t.schemeYlOrBr=Bb,t.schemeYlOrRd=Lb,t.select=Zn,t.selectAll=function(t){return"string"==typeof t?new Vn([document.querySelectorAll(t)],[document.documentElement]):new Vn([Ht(t)],Gn)},t.selection=Wn,t.selector=jt,t.selectorAll=Gt,t.shuffle=dt,t.shuffler=pt,t.some=function(t,n){if("function"!=typeof n)throw new TypeError("test is not a function");let e=-1;for(const r of t)if(n(r,++e,t))return!0;return!1},t.sort=U,t.stack=function(){var t=gm([]),n=hw,e=lw,r=dw;function i(i){var o,a,u=Array.from(t.apply(this,arguments),pw),c=u.length,f=-1;for(const t of i)for(o=0,++f;o<c;++o)(u[o][f]=[0,+r(t,u[o].key,f,i)]).data=t;for(o=0,a=Fm(n(u));o<c;++o)u[a[o]].index=o;return e(u,a),u}return i.keys=function(n){return arguments.length?(t="function"==typeof n?n:gm(Array.from(n)),i):t},i.value=function(t){return arguments.length?(r="function"==typeof t?t:gm(+t),i):r},i.order=function(t){return arguments.length?(n=null==t?hw:"function"==typeof t?t:gm(Array.from(t)),i):n},i.offset=function(t){return arguments.length?(e=null==t?lw:t,i):e},i},t.stackOffsetDiverging=function(t,n){if((u=t.length)>0)for(var e,r,i,o,a,u,c=0,f=t[n[0]].length;c<f;++c)for(o=a=0,e=0;e<u;++e)(i=(r=t[n[e]][c])[1]-r[0])>0?(r[0]=o,r[1]=o+=i):i<0?(r[1]=a,r[0]=a+=i):(r[0]=0,r[1]=i)},t.stackOffsetExpand=function(t,n){if((r=t.length)>0){for(var e,r,i,o=0,a=t[0].length;o<a;++o){for(i=e=0;e<r;++e)i+=t[e][o][1]||0;if(i)for(e=0;e<r;++e)t[e][o][1]/=i}lw(t,n)}},t.stackOffsetNone=lw,t.stackOffsetSilhouette=function(t,n){if((e=t.length)>0){for(var e,r=0,i=t[n[0]],o=i.length;r<o;++r){for(var a=0,u=0;a<e;++a)u+=t[a][r][1]||0;i[r][1]+=i[r][0]=-u/2}lw(t,n)}},t.stackOffsetWiggle=function(t,n){if((i=t.length)>0&&(r=(e=t[n[0]]).length)>0){for(var e,r,i,o=0,a=1;a<r;++a){for(var u=0,c=0,f=0;u<i;++u){for(var s=t[n[u]],l=s[a][1]||0,h=(l-(s[a-1][1]||0))/2,d=0;d<u;++d){var p=t[n[d]];h+=(p[a][1]||0)-(p[a-1][1]||0)}c+=l,f+=h*l}e[a-1][1]+=e[a-1][0]=o,c&&(o-=f/c)}e[a-1][1]+=e[a-1][0]=o,lw(t,n)}},t.stackOrderAppearance=gw,t.stackOrderAscending=vw,t.stackOrderDescending=function(t){return vw(t).reverse()},t.stackOrderInsideOut=function(t){var n,e,r=t.length,i=t.map(_w),o=gw(t),a=0,u=0,c=[],f=[];for(n=0;n<r;++n)e=o[n],a<u?(a+=i[e],c.push(e)):(u+=i[e],f.push(e));return f.reverse().concat(c)},t.stackOrderNone=hw,t.stackOrderReverse=function(t){return hw(t).reverse()},t.stratify=function(){var t,n=kp,e=Cp;function r(r){var i,o,a,u,c,f,s,l,h=Array.from(r),d=n,p=e,g=new Map;if(null!=t){const n=h.map(((n,e)=>function(t){t=`${t}`;let n=t.length;zp(t,n-1)&&!zp(t,n-2)&&(t=t.slice(0,-1));return"/"===t[0]?t:`/${t}`}(t(n,e,r)))),e=n.map(Pp),i=new Set(n).add("");for(const t of e)i.has(t)||(i.add(t),n.push(t),e.push(Pp(t)),h.push(Np));d=(t,e)=>n[e],p=(t,n)=>e[n]}for(a=0,i=h.length;a<i;++a)o=h[a],f=h[a]=new Qd(o),null!=(s=d(o,a,r))&&(s+="")&&(l=f.id=s,g.set(l,g.has(l)?Ep:f)),null!=(s=p(o,a,r))&&(s+="")&&(f.parent=s);for(a=0;a<i;++a)if(s=(f=h[a]).parent){if(!(c=g.get(s)))throw new Error("missing: "+s);if(c===Ep)throw new Error("ambiguous: "+s);c.children?c.children.push(f):c.children=[f],f.parent=c}else{if(u)throw new Error("multiple roots");u=f}if(!u)throw new Error("no root");if(null!=t){for(;u.data===Np&&1===u.children.length;)u=u.children[0],--i;for(let t=h.length-1;t>=0&&(f=h[t]).data===Np;--t)f.data=null}if(u.parent=Sp,u.eachBefore((function(t){t.depth=t.parent.depth+1,--i})).eachBefore(Kd),u.parent=null,i>0)throw new Error("cycle");return u}return r.id=function(t){return arguments.length?(n=Jd(t),r):n},r.parentId=function(t){return arguments.length?(e=Jd(t),r):e},r.path=function(n){return arguments.length?(t=Jd(n),r):t},r},t.style=_n,t.subset=function(t,n){return _t(n,t)},t.sum=function(t,n){let e=0;if(void 0===n)for(let n of t)(n=+n)&&(e+=n);else{let r=-1;for(let i of t)(i=+n(i,++r,t))&&(e+=i)}return e},t.superset=_t,t.svg=Nc,t.symbol=function(t,n){let e=null,r=Nm(i);function i(){let i;if(e||(e=i=r()),t.apply(this,arguments).draw(e,+n.apply(this,arguments)),i)return e=null,i+""||null}return t="function"==typeof t?t:gm(t||cx),n="function"==typeof n?n:gm(void 0===n?64:+n),i.type=function(n){return arguments.length?(t="function"==typeof n?n:gm(n),i):t},i.size=function(t){return arguments.length?(n="function"==typeof t?t:gm(+t),i):n},i.context=function(t){return arguments.length?(e=null==t?null:t,i):e},i},t.symbolAsterisk=ux,t.symbolCircle=cx,t.symbolCross=fx,t.symbolDiamond=hx,t.symbolDiamond2=dx,t.symbolPlus=px,t.symbolSquare=gx,t.symbolSquare2=yx,t.symbolStar=mx,t.symbolTimes=Cx,t.symbolTriangle=wx,t.symbolTriangle2=Tx,t.symbolWye=kx,t.symbolX=Cx,t.symbols=Px,t.symbolsFill=Px,t.symbolsStroke=zx,t.text=mc,t.thresholdFreedmanDiaconis=function(t,n,e){const r=v(t),i=at(t,.75)-at(t,.25);return r&&i?Math.ceil((e-n)/(2*i*Math.pow(r,-1/3))):1},t.thresholdScott=function(t,n,e){const r=v(t),i=w(t);return r&&i?Math.ceil((e-n)*Math.cbrt(r)/(3.49*i)):1},t.thresholdSturges=K,t.tickFormat=Eg,t.tickIncrement=V,t.tickStep=W,t.ticks=G,t.timeDay=py,t.timeDays=gy,t.timeFormatDefaultLocale=P_,t.timeFormatLocale=hv,t.timeFriday=Sy,t.timeFridays=$y,t.timeHour=sy,t.timeHours=ly,t.timeInterval=Vg,t.timeMillisecond=Wg,t.timeMilliseconds=Zg,t.timeMinute=ay,t.timeMinutes=uy,t.timeMonday=wy,t.timeMondays=ky,t.timeMonth=Zy,t.timeMonths=Ky,t.timeSaturday=Ey,t.timeSaturdays=Dy,t.timeSecond=iy,t.timeSeconds=oy,t.timeSunday=xy,t.timeSundays=Ny,t.timeThursday=Ay,t.timeThursdays=zy,t.timeTickInterval=cv,t.timeTicks=uv,t.timeTuesday=My,t.timeTuesdays=Cy,t.timeWednesday=Ty,t.timeWednesdays=Py,t.timeWeek=xy,t.timeWeeks=Ny,t.timeYear=tv,t.timeYears=nv,t.timeout=$i,t.timer=Ni,t.timerFlush=ki,t.transition=go,t.transpose=gt,t.tree=function(){var t=$p,n=1,e=1,r=null;function i(i){var c=function(t){for(var n,e,r,i,o,a=new Up(t,0),u=[a];n=u.pop();)if(r=n._.children)for(n.children=new Array(o=r.length),i=o-1;i>=0;--i)u.push(e=n.children[i]=new Up(r[i],i)),e.parent=n;return(a.parent=new Up(null,0)).children=[a],a}(i);if(c.eachAfter(o),c.parent.m=-c.z,c.eachBefore(a),r)i.eachBefore(u);else{var f=i,s=i,l=i;i.eachBefore((function(t){t.x<f.x&&(f=t),t.x>s.x&&(s=t),t.depth>l.depth&&(l=t)}));var h=f===s?1:t(f,s)/2,d=h-f.x,p=n/(s.x+h+d),g=e/(l.depth||1);i.eachBefore((function(t){t.x=(t.x+d)*p,t.y=t.depth*g}))}return i}function o(n){var e=n.children,r=n.parent.children,i=n.i?r[n.i-1]:null;if(e){!function(t){for(var n,e=0,r=0,i=t.children,o=i.length;--o>=0;)(n=i[o]).z+=e,n.m+=e,e+=n.s+(r+=n.c)}(n);var o=(e[0].z+e[e.length-1].z)/2;i?(n.z=i.z+t(n._,i._),n.m=n.z-o):n.z=o}else i&&(n.z=i.z+t(n._,i._));n.parent.A=function(n,e,r){if(e){for(var i,o=n,a=n,u=e,c=o.parent.children[0],f=o.m,s=a.m,l=u.m,h=c.m;u=Rp(u),o=Dp(o),u&&o;)c=Dp(c),(a=Rp(a)).a=n,(i=u.z+l-o.z-f+t(u._,o._))>0&&(Fp(qp(u,n,r),n,i),f+=i,s+=i),l+=u.m,f+=o.m,h+=c.m,s+=a.m;u&&!Rp(a)&&(a.t=u,a.m+=l-s),o&&!Dp(c)&&(c.t=o,c.m+=f-h,r=n)}return r}(n,i,n.parent.A||r[0])}function a(t){t._.x=t.z+t.parent.m,t.m+=t.parent.m}function u(t){t.x*=n,t.y=t.depth*e}return i.separation=function(n){return arguments.length?(t=n,i):t},i.size=function(t){return arguments.length?(r=!1,n=+t[0],e=+t[1],i):r?null:[n,e]},i.nodeSize=function(t){return arguments.length?(r=!0,n=+t[0],e=+t[1],i):r?[n,e]:null},i},t.treemap=function(){var t=Yp,n=!1,e=1,r=1,i=[0],o=np,a=np,u=np,c=np,f=np;function s(t){return t.x0=t.y0=0,t.x1=e,t.y1=r,t.eachBefore(l),i=[0],n&&t.eachBefore(Tp),t}function l(n){var e=i[n.depth],r=n.x0+e,s=n.y0+e,l=n.x1-e,h=n.y1-e;l<r&&(r=l=(r+l)/2),h<s&&(s=h=(s+h)/2),n.x0=r,n.y0=s,n.x1=l,n.y1=h,n.children&&(e=i[n.depth+1]=o(n)/2,r+=f(n)-e,s+=a(n)-e,(l-=u(n)-e)<r&&(r=l=(r+l)/2),(h-=c(n)-e)<s&&(s=h=(s+h)/2),t(n,r,s,l,h))}return s.round=function(t){return arguments.length?(n=!!t,s):n},s.size=function(t){return arguments.length?(e=+t[0],r=+t[1],s):[e,r]},s.tile=function(n){return arguments.length?(t=tp(n),s):t},s.padding=function(t){return arguments.length?s.paddingInner(t).paddingOuter(t):s.paddingInner()},s.paddingInner=function(t){return arguments.length?(o="function"==typeof t?t:ep(+t),s):o},s.paddingOuter=function(t){return arguments.length?s.paddingTop(t).paddingRight(t).paddingBottom(t).paddingLeft(t):s.paddingTop()},s.paddingTop=function(t){return arguments.length?(a="function"==typeof t?t:ep(+t),s):a},s.paddingRight=function(t){return arguments.length?(u="function"==typeof t?t:ep(+t),s):u},s.paddingBottom=function(t){return arguments.length?(c="function"==typeof t?t:ep(+t),s):c},s.paddingLeft=function(t){return arguments.length?(f="function"==typeof t?t:ep(+t),s):f},s},t.treemapBinary=function(t,n,e,r,i){var o,a,u=t.children,c=u.length,f=new Array(c+1);for(f[0]=a=o=0;o<c;++o)f[o+1]=a+=u[o].value;!function t(n,e,r,i,o,a,c){if(n>=e-1){var s=u[n];return s.x0=i,s.y0=o,s.x1=a,void(s.y1=c)}var l=f[n],h=r/2+l,d=n+1,p=e-1;for(;d<p;){var g=d+p>>>1;f[g]<h?d=g+1:p=g}h-f[d-1]<f[d]-h&&n+1<d&&--d;var y=f[d]-l,v=r-y;if(a-i>c-o){var _=r?(i*v+a*y)/r:a;t(n,d,y,i,o,_,c),t(d,e,v,_,o,a,c)}else{var b=r?(o*v+c*y)/r:c;t(n,d,y,i,o,a,b),t(d,e,v,i,b,a,c)}}(0,c,t.value,n,e,r,i)},t.treemapDice=Ap,t.treemapResquarify=Lp,t.treemapSlice=Ip,t.treemapSliceDice=function(t,n,e,r,i){(1&t.depth?Ip:Ap)(t,n,e,r,i)},t.treemapSquarify=Yp,t.tsv=Mc,t.tsvFormat=lc,t.tsvFormatBody=hc,t.tsvFormatRow=pc,t.tsvFormatRows=dc,t.tsvFormatValue=gc,t.tsvParse=fc,t.tsvParseRows=sc,t.union=function(...t){const n=new InternSet;for(const e of t)for(const t of e)n.add(t);return n},t.unixDay=_y,t.unixDays=by,t.utcDay=yy,t.utcDays=vy,t.utcFriday=By,t.utcFridays=Vy,t.utcHour=hy,t.utcHours=dy,t.utcMillisecond=Wg,t.utcMilliseconds=Zg,t.utcMinute=cy,t.utcMinutes=fy,t.utcMonday=qy,t.utcMondays=jy,t.utcMonth=Qy,t.utcMonths=Jy,t.utcSaturday=Yy,t.utcSaturdays=Wy,t.utcSecond=iy,t.utcSeconds=oy,t.utcSunday=Fy,t.utcSundays=Ly,t.utcThursday=Oy,t.utcThursdays=Gy,t.utcTickInterval=av,t.utcTicks=ov,t.utcTuesday=Uy,t.utcTuesdays=Hy,t.utcWednesday=Iy,t.utcWednesdays=Xy,t.utcWeek=Fy,t.utcWeeks=Ly,t.utcYear=ev,t.utcYears=rv,t.variance=x,t.version="7.8.5",t.window=pn,t.xml=Sc,t.zip=function(){return gt(arguments)},t.zoom=function(){var t,n,e,r=Sw,i=Ew,o=Pw,a=kw,u=Cw,c=[0,1/0],f=[[-1/0,-1/0],[1/0,1/0]],s=250,l=ri,h=$t("start","zoom","end"),d=500,p=150,g=0,y=10;function v(t){t.property("__zoom",Nw).on("wheel.zoom",T,{passive:!1}).on("mousedown.zoom",A).on("dblclick.zoom",S).filter(u).on("touchstart.zoom",E).on("touchmove.zoom",N).on("touchend.zoom touchcancel.zoom",k).style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function _(t,n){return(n=Math.max(c[0],Math.min(c[1],n)))===t.k?t:new xw(n,t.x,t.y)}function b(t,n,e){var r=n[0]-e[0]*t.k,i=n[1]-e[1]*t.k;return r===t.x&&i===t.y?t:new xw(t.k,r,i)}function m(t){return[(+t[0][0]+ +t[1][0])/2,(+t[0][1]+ +t[1][1])/2]}function x(t,n,e,r){t.on("start.zoom",(function(){w(this,arguments).event(r).start()})).on("interrupt.zoom end.zoom",(function(){w(this,arguments).event(r).end()})).tween("zoom",(function(){var t=this,o=arguments,a=w(t,o).event(r),u=i.apply(t,o),c=null==e?m(u):"function"==typeof e?e.apply(t,o):e,f=Math.max(u[1][0]-u[0][0],u[1][1]-u[0][1]),s=t.__zoom,h="function"==typeof n?n.apply(t,o):n,d=l(s.invert(c).concat(f/s.k),h.invert(c).concat(f/h.k));return function(t){if(1===t)t=h;else{var n=d(t),e=f/n[2];t=new xw(e,c[0]-n[0]*e,c[1]-n[1]*e)}a.zoom(null,t)}}))}function w(t,n,e){return!e&&t.__zooming||new M(t,n)}function M(t,n){this.that=t,this.args=n,this.active=0,this.sourceEvent=null,this.extent=i.apply(t,n),this.taps=0}function T(t,...n){if(r.apply(this,arguments)){var e=w(this,n).event(t),i=this.__zoom,u=Math.max(c[0],Math.min(c[1],i.k*Math.pow(2,a.apply(this,arguments)))),s=ne(t);if(e.wheel)e.mouse[0][0]===s[0]&&e.mouse[0][1]===s[1]||(e.mouse[1]=i.invert(e.mouse[0]=s)),clearTimeout(e.wheel);else{if(i.k===u)return;e.mouse=[s,i.invert(s)],Gi(this),e.start()}Aw(t),e.wheel=setTimeout((function(){e.wheel=null,e.end()}),p),e.zoom("mouse",o(b(_(i,u),e.mouse[0],e.mouse[1]),e.extent,f))}}function A(t,...n){if(!e&&r.apply(this,arguments)){var i=t.currentTarget,a=w(this,n,!0).event(t),u=Zn(t.view).on("mousemove.zoom",(function(t){if(Aw(t),!a.moved){var n=t.clientX-s,e=t.clientY-l;a.moved=n*n+e*e>g}a.event(t).zoom("mouse",o(b(a.that.__zoom,a.mouse[0]=ne(t,i),a.mouse[1]),a.extent,f))}),!0).on("mouseup.zoom",(function(t){u.on("mousemove.zoom mouseup.zoom",null),ue(t.view,a.moved),Aw(t),a.event(t).end()}),!0),c=ne(t,i),s=t.clientX,l=t.clientY;ae(t.view),Tw(t),a.mouse=[c,this.__zoom.invert(c)],Gi(this),a.start()}}function S(t,...n){if(r.apply(this,arguments)){var e=this.__zoom,a=ne(t.changedTouches?t.changedTouches[0]:t,this),u=e.invert(a),c=e.k*(t.shiftKey?.5:2),l=o(b(_(e,c),a,u),i.apply(this,n),f);Aw(t),s>0?Zn(this).transition().duration(s).call(x,l,a,t):Zn(this).call(v.transform,l,a,t)}}function E(e,...i){if(r.apply(this,arguments)){var o,a,u,c,f=e.touches,s=f.length,l=w(this,i,e.changedTouches.length===s).event(e);for(Tw(e),a=0;a<s;++a)c=[c=ne(u=f[a],this),this.__zoom.invert(c),u.identifier],l.touch0?l.touch1||l.touch0[2]===c[2]||(l.touch1=c,l.taps=0):(l.touch0=c,o=!0,l.taps=1+!!t);t&&(t=clearTimeout(t)),o&&(l.taps<2&&(n=c[0],t=setTimeout((function(){t=null}),d)),Gi(this),l.start())}}function N(t,...n){if(this.__zooming){var e,r,i,a,u=w(this,n).event(t),c=t.changedTouches,s=c.length;for(Aw(t),e=0;e<s;++e)i=ne(r=c[e],this),u.touch0&&u.touch0[2]===r.identifier?u.touch0[0]=i:u.touch1&&u.touch1[2]===r.identifier&&(u.touch1[0]=i);if(r=u.that.__zoom,u.touch1){var l=u.touch0[0],h=u.touch0[1],d=u.touch1[0],p=u.touch1[1],g=(g=d[0]-l[0])*g+(g=d[1]-l[1])*g,y=(y=p[0]-h[0])*y+(y=p[1]-h[1])*y;r=_(r,Math.sqrt(g/y)),i=[(l[0]+d[0])/2,(l[1]+d[1])/2],a=[(h[0]+p[0])/2,(h[1]+p[1])/2]}else{if(!u.touch0)return;i=u.touch0[0],a=u.touch0[1]}u.zoom("touch",o(b(r,i,a),u.extent,f))}}function k(t,...r){if(this.__zooming){var i,o,a=w(this,r).event(t),u=t.changedTouches,c=u.length;for(Tw(t),e&&clearTimeout(e),e=setTimeout((function(){e=null}),d),i=0;i<c;++i)o=u[i],a.touch0&&a.touch0[2]===o.identifier?delete a.touch0:a.touch1&&a.touch1[2]===o.identifier&&delete a.touch1;if(a.touch1&&!a.touch0&&(a.touch0=a.touch1,delete a.touch1),a.touch0)a.touch0[1]=this.__zoom.invert(a.touch0[0]);else if(a.end(),2===a.taps&&(o=ne(o,this),Math.hypot(n[0]-o[0],n[1]-o[1])<y)){var f=Zn(this).on("dblclick.zoom");f&&f.apply(this,arguments)}}}return v.transform=function(t,n,e,r){var i=t.selection?t.selection():t;i.property("__zoom",Nw),t!==i?x(t,n,e,r):i.interrupt().each((function(){w(this,arguments).event(r).start().zoom(null,"function"==typeof n?n.apply(this,arguments):n).end()}))},v.scaleBy=function(t,n,e,r){v.scaleTo(t,(function(){return this.__zoom.k*("function"==typeof n?n.apply(this,arguments):n)}),e,r)},v.scaleTo=function(t,n,e,r){v.transform(t,(function(){var t=i.apply(this,arguments),r=this.__zoom,a=null==e?m(t):"function"==typeof e?e.apply(this,arguments):e,u=r.invert(a),c="function"==typeof n?n.apply(this,arguments):n;return o(b(_(r,c),a,u),t,f)}),e,r)},v.translateBy=function(t,n,e,r){v.transform(t,(function(){return o(this.__zoom.translate("function"==typeof n?n.apply(this,arguments):n,"function"==typeof e?e.apply(this,arguments):e),i.apply(this,arguments),f)}),null,r)},v.translateTo=function(t,n,e,r,a){v.transform(t,(function(){var t=i.apply(this,arguments),a=this.__zoom,u=null==r?m(t):"function"==typeof r?r.apply(this,arguments):r;return o(ww.translate(u[0],u[1]).scale(a.k).translate("function"==typeof n?-n.apply(this,arguments):-n,"function"==typeof e?-e.apply(this,arguments):-e),t,f)}),r,a)},M.prototype={event:function(t){return t&&(this.sourceEvent=t),this},start:function(){return 1==++this.active&&(this.that.__zooming=this,this.emit("start")),this},zoom:function(t,n){return this.mouse&&"mouse"!==t&&(this.mouse[1]=n.invert(this.mouse[0])),this.touch0&&"touch"!==t&&(this.touch0[1]=n.invert(this.touch0[0])),this.touch1&&"touch"!==t&&(this.touch1[1]=n.invert(this.touch1[0])),this.that.__zoom=n,this.emit("zoom"),this},end:function(){return 0==--this.active&&(delete this.that.__zooming,this.emit("end")),this},emit:function(t){var n=Zn(this.that).datum();h.call(t,this.that,new mw(t,{sourceEvent:this.sourceEvent,target:v,type:t,transform:this.that.__zoom,dispatch:h}),n)}},v.wheelDelta=function(t){return arguments.length?(a="function"==typeof t?t:bw(+t),v):a},v.filter=function(t){return arguments.length?(r="function"==typeof t?t:bw(!!t),v):r},v.touchable=function(t){return arguments.length?(u="function"==typeof t?t:bw(!!t),v):u},v.extent=function(t){return arguments.length?(i="function"==typeof t?t:bw([[+t[0][0],+t[0][1]],[+t[1][0],+t[1][1]]]),v):i},v.scaleExtent=function(t){return arguments.length?(c[0]=+t[0],c[1]=+t[1],v):[c[0],c[1]]},v.translateExtent=function(t){return arguments.length?(f[0][0]=+t[0][0],f[1][0]=+t[1][0],f[0][1]=+t[0][1],f[1][1]=+t[1][1],v):[[f[0][0],f[0][1]],[f[1][0],f[1][1]]]},v.constrain=function(t){return arguments.length?(o=t,v):o},v.duration=function(t){return arguments.length?(s=+t,v):s},v.interpolate=function(t){return arguments.length?(l=t,v):l},v.on=function(){var t=h.on.apply(h,arguments);return t===h?v:t},v.clickDistance=function(t){return arguments.length?(g=(t=+t)*t,v):Math.sqrt(g)},v.tapDistance=function(t){return arguments.length?(y=+t,v):y},v},t.zoomIdentity=ww,t.zoomTransform=Mw}));
diff --git a/interface/js/lib/d3evolution.min.js b/interface/js/lib/d3evolution.min.js
new file mode 100644
index 0000000..5c50ee8
--- /dev/null
+++ b/interface/js/lib/d3evolution.min.js
@@ -0,0 +1,5 @@
+/*!
+ * D3Evolution 2.0.1 (https://github.com/moisseev/D3Evolution)
+ * Copyright (c) 2016-2017, Alexander Moisseev, BSD 2-Clause
+ */
+function D3Evolution(t,e){"use strict";var l=$.extend(!0,{title:"",width:800,height:400,margin:{top:80,right:60,bottom:40,left:60},yAxisLabel:"",type:"line",yScale:"lin",duration:1250,interpolate:"curveLinear",legend:{buttonRadius:7,space:130,entries:[]}},e);this.destroy=function(){d3.selectAll("#"+t+" svg").remove()},this.destroy();const n={curveLinear:d3.curveLinear,curveStep:d3.curveStep,curveStepBefore:d3.curveStepBefore,curveStepAfter:d3.curveStepAfter,curveMonotoneX:d3.curveMonotoneX,curveBasis:d3.curveBasis,curveBasisOpen:d3.curveBasisOpen,curveBundle:d3.curveBundle,curveCardinal:d3.curveCardinal,curveCardinalOpen:d3.curveCardinalOpen,curveCatmullRom:d3.curveCatmullRom,curveCatmullRomOpen:d3.curveCatmullRomOpen,curveNatural:d3.curveNatural};var o=null,i=null,c=null,u=l.width-l.margin.left-l.margin.right,r=l.height-l.margin.top-l.margin.bottom,s=d3.scaleTime().range([0,u]),d=null,p=null;function a(){p="log"===l.yScale?(d=d3.scaleLog().clamp(!0).range([r,0]),d3.scaleLog().range([r-30,0])):(d=d3.scaleLinear().range([r,0])).copy()}a();function y(t){return void 0!==l.legend.entries[t]&&void 0!==l.legend.entries[t].color?l.legend.entries[t].color:H(t)}function f(t){return void 0!==l.legend.entries[t]&&void 0!==l.legend.entries[t].label?l.legend.entries[t].label:"path_"+t}var g=d3.axisBottom().scale(s),h=d3.axisLeft().scale(p).ticks(5),m=d3.axisBottom().tickFormat("").scale(s).tickSize(-r,0),v=d3.axisLeft().tickFormat("").scale(p).tickSize(-u,0),D=d3.scaleQuantize().range([r,0]),T=d3.area().x(function(t){return s(t.x)}).y0(function(){return r}).y1(function(t){return D(null===t.y)}).curve(d3.curveStep),x=d3.line().defined(function(t){return null!==t.y}).x(function(t){return s(t.x)}).y(function(t){return d(t.y)}).curve(n[l.interpolate]),A=d3.area().defined(function(t){return null!==t.y}).x(function(t){return s(t.x)}).y0(function(t){return d(t.y0)}).y1(function(t){return d(t.y0+t.y)}).curve(n[l.interpolate]),X=function(t){t.reduce(function(n,t){return t.forEach(function(t,e){t.y0=n.length?n[e].y+n[e].y0:0}),t},[])},H=d3.scaleOrdinal(d3.schemeCategory10),Q=function(t){var n=t.reduce(function(n,t){return t.map(function(t,e){return t.y+(n[e]||0)})},[]),t=$.extend(!0,[],t);return t.forEach(function(t){t.forEach(function(t,e){n[e]&&(t.y/=n[e])})}),t};function k(){if("log"===l.yScale){const e=d.invert(r);o.forEach(function(t){t.forEach(function(t){return 0===t.y?t.y:e})})}}var e=d3.select("#"+t).append("svg").classed("d3evolution",!0).attr("width",l.width).attr("height",l.height),b=e.append("g").attr("class","legend"),S=e.append("g").attr("width",u).attr("height",r).attr("transform","translate("+l.margin.left+", "+l.margin.top+")"),L=(S.append("g").attr("class","x grid").attr("transform","translate(0,"+r+")").call(m),S.append("g").attr("class","y grid").attr("transform","translate(0,0)").call(v),S.append("g").attr("class","x axis").attr("transform","translate(0,"+r+")").call(g),S.append("g").attr("class","y axis").attr("transform","translate(0,0)").call(h),d3.scaleOrdinal().domain([0]).range([r])),w=d3.axisLeft().scale(L),R=(S.append("g").attr("class","y-zero axis").call(w),S.append("text").attr("class","y label").attr("x",20-l.margin.left).attr("y",-20).style("opacity","percentage"===l.convert?0:1).text(l.yAxisLabel)),B=e.append("svg:text").attr("x",l.width/2).attr("y",l.margin.top/3).attr("text-anchor","middle"),L=(B.append("tspan").attr("class","chart-title").text(l.title+" "),B.timeRange=B.append("tspan"),e.append("svg:text").attr("x",l.width-20).attr("y",l.margin.top/3).attr("text-anchor","end")),V=L.append("tspan").attr("class","cursor-time");L.append("svg:title").text("Current cursor position");const C=function(t){return d3.timeFormat("%Y-%m-%d %H:%M:%S")(new Date(t))};function M(e){var n=o.map(function(t){return t[e]}),t=n[0].x;return V.text(C(t)),b.selectAll("text.value").text(function(t,e){return null===n[e].y?null:d3.format("percentage"===l.convert?".2~%":".6~")(n[e].y)}),n}var O=null,E=null;function Y(t){var e=d3.bisector(function(t){return t.x}).left,t=s.invert(d3.pointer(t)[0]),n=M(e(o[0],t)-1);O.selectAll(".x,.cursor circle").attr("transform","translate("+s(n[0].x)+",0)"),O.selectAll(".y").attr("transform",function(t,e){e=n[e];return"translate(0,"+(("area"===l.type?d(e.y0+e.y):d(e.y))||0)+")"}).style("display",function(t,e){return n[e].y?null:"none"})}function j(){O.style("display","none"),M(E)}function q(){O.style("display",null)}S.append("rect").style("fill","none").style("pointer-events","all").attr("width",u).attr("height",r).on("mousemove",Y).on("mouseout",j).on("mouseover",q);function G(){o="percentage"===l.convert?(R.transition().duration(l.duration).style("opacity",0),Q(i)):(R.transition().duration(l.duration).style("opacity",1),i),E=i[0].length-1,z()}var F=S.append("g"),_=S.append("g"),z=((O=S.append("g").attr("class","cursor").style("pointer-events","none").style("display","none")).append("line").attr("class","x background").attr("y1",0).attr("y2",r),O.append("line").attr("class","x foreground").attr("y1",0).attr("y2",r),function(){var t=[];if(t="area"===l.type?(X(o),"log"===l.yScale?d3.extent(d3.merge(o),function(t){return t.y0+t.y===0?null:t.y0+t.y}):d3.extent(d3.merge(o),function(t){return t.y0+t.y})):"log"===l.yScale?d3.extent(d3.merge(o),function(t){return 0===t.y?null:t.y}):d3.extent(d3.merge(o),function(t){return t.y}),"log"===l.yScale?(void 0===t[0]?t=[.0095,.0105]:t[0]===t[1]&&(t[0]*=.9),p.domain([t[0],t[1]]),e=p.invert(r),d.domain([e,t[1]])):(d.domain([0<t[0]?0:t[0],t[1]]),p.domain(d.domain())),"percentage"===l.convert){var a={y:r};const i=d3.format(".0%");w.tickFormat(i),h.tickFormat("log"===l.yScale?function(t){return t=t,e=a,n=i,r=Math.pow(10,Math.round(Math.log(t)/Math.LN10)),Math.abs(r-t)<1e-6||!(Math.abs(d(r)-d(t))<15||e.y-d(t)<15)?(e.y=d(t),n(t)):"";var e,n,r}:i)}else w.tickFormat(null),h.tickFormat(null);h.tickValues(p.ticks().length?null:[t[0],t[1]]);var e=d3.transition().duration(l.duration);S.select(".y.grid").transition(e).call(v.scale(p)),S.select(".y.axis").transition(e).call(h.scale(p)),S.select(".y-zero.axis").call(w)});function I(){b.selectAll("g").transition().duration(l.duration).attr("transform",function(t,e){return"translate("+(c+l.legend.space*e+2*l.legend.buttonRadius)+","+2*l.margin.top/3+")"})}var N=[];this.data=function(t){var e=e=>{e.on("click",t=>{t=e.nodes().indexOf(t.currentTarget);N[t]=0===N[t]?1:0,d3.select("#circle_"+t).transition().duration(l.duration).style("fill-opacity",N[t]+.2),d3.select("#path_"+t).transition().duration(l.duration).style("opacity",N[t])})};function r(e,n){function r(t){return!1===n?N[t]:t===e?1:0===N[t]?0:.4}d3.select("#circle_"+e).attr("r",l.legend.buttonRadius*(!1===n?1:1.3)),_.selectAll("path.path").style("opacity",function(t,e){return r(e)}).style("fill-opacity",function(t,e){return r(e)})}i=$.extend(!0,[],t),c=l.width-l.margin.right-l.legend.space*i.length,i.forEach(function(t){t.forEach(function(t){t.x*=1e3})});var t=d3.extent(d3.merge(i),function(t){return t.x}),t=(s.domain([t[0],t[1]]),B.timeRange.text("[ "+C(t[0])+" / "+C(t[1])+" ]"),F.selectAll("path.path-null").data(i)),t=(t.enter().append("path").attr("class","path-null"),F.selectAll("path.path-null").transition().duration(l.duration/2).style("opacity",0).on("end",function(){F.selectAll("path.path-null").attr("d",T).transition().duration(l.duration/2).style("opacity",1)}),t.exit().remove(),G(),k(),_.selectAll("path.path").data(o)),t=(e(t.enter().append("path").merge(t).attr("class","path").attr("id",function(t,e){return"path_"+e}).on("mousemove",Y).on("mouseover",function(t,e,n){r(n),q()}).on("mouseout",function(t,e,n){r(n,!1),j()})),t.exit().remove(),t=_.selectAll("path.path"),"area"===l.type?t.style("fill",function(t,e){return y(e)}).style("stroke","none").style("fill-opacity",function(t,e){return N[e]}):t.style("fill","none").style("stroke",function(t,e){return y(e)}).style("opacity",function(t,e){return N[e]}),t.transition().duration(l.duration).attr("d","area"===l.type?A:x),d3.transition().duration(l.duration)),t=(S.select(".x.grid").transition(t).call(m.scale(s)),S.select(".x.axis").transition(t).call(g.scale(s)),O.selectAll(".y").data(o)),n=t.enter().append("g").attr("class","y").style("stroke",function(t,e){return y(e)}),n=(n.append("circle").attr("class","background"),n.append("circle").attr("class","foreground"),n.selectAll("circle").attr("r",7).style("fill","none"),n.append("line").attr("class","background"),n.append("line").attr("class","foreground"),n.selectAll("line").attr("x1",0).attr("x2",u),t.exit().remove(),b.selectAll("circle").data(o)),t=(e(n.enter().append("circle").attr("id",function(t,e){return"circle_"+e}).attr("cy",2*l.margin.top/3).attr("r",l.legend.buttonRadius).style("fill",function(t,e){return y(e)}).style("stroke",function(t,e){return y(e)}).style("fill-opacity",function(t,e){return N[e]+.2}).on("mouseover",function(t,e,n){r(n)}).on("mouseout",function(t,e,n){r(n,!1)})),n.exit().remove(),b.selectAll("circle").transition().duration(l.duration).attr("cx",function(t,e){return c+l.legend.space*e}),b.selectAll("g").data(o)),n=t.enter().append("g"),a=(e(n.append("text").attr("class","name").attr("dy","0.3em").text(function(t,e){return f(e)}).on("mouseover",function(t,e,n){r(n)}).on("mouseout",function(t,e,n){r(n,!1)})),n.append("text").attr("class","value").attr("dy","20"),t.exit().remove(),I(),b.selectAll("text.value"));return a.transition("opacity").duration(l.duration/2).style("opacity",0).on("end",function(){M(E),a.transition("opacity").duration(l.duration/2).style("opacity",1)}),this},this.legend=function(t){return $.extend(!0,l.legend,t),b.selectAll("circle").transition().duration(l.duration).attr("cx",function(t,e){return c+l.legend.space*e}).attr("r",l.legend.buttonRadius).style("fill",function(t,e){return y(e)}).style("stroke",function(t,e){return y(e)}),b.selectAll("text.name").text(function(t,e){return f(e)}),I(),_.selectAll("path.path").transition().duration(l.duration).style("fill","area"===l.type?function(t,e){return y(e)}:"none").style("stroke","area"!==l.type?function(t,e){return y(e)}:"none"),O.selectAll(".y").style("stroke",function(t,e){return y(e)}),this},this.convert=function(t){return l.convert=t,G(),M(E),_.selectAll("path.path").data(o).transition().duration(l.duration).attr("d","area"===l.type?A:x),this},this.interpolate=function(t){return l.interpolate=t,A.curve(n[l.interpolate]),x.curve(n[l.interpolate]),_.selectAll("path.path").attr("d","area"===l.type?A:x),this},this.type=function(t){return l.type=t,z(),_.selectAll("path.path").style("stroke","area"!==l.type?function(t,e){return y(e)}:"none").style("fill","area"===l.type?function(t,e){return y(e)}:"none").transition().duration(l.duration).attr("d","area"===l.type?A:x),this},this.yAxisLabel=function(t){return l.yAxisLabel=t,R.transition().duration(l.duration/2).style("opacity",0).on("end",function(){R.text(l.yAxisLabel).transition().duration(l.duration/2).style("opacity",1)}),this},this.yScale=function(t){return l.yScale=t,a(),k(),z(),_.selectAll("path.path").transition().duration(l.duration).attr("d","area"===l.type?A:x),this}} \ No newline at end of file
diff --git a/interface/js/lib/d3pie.min.js b/interface/js/lib/d3pie.min.js
new file mode 100644
index 0000000..823f697
--- /dev/null
+++ b/interface/js/lib/d3pie.min.js
@@ -0,0 +1,5 @@
+/*!
+ * rspamd-D3Pie 1.1.0 (https://github.com/moisseev/rspamd-D3Pie)
+ * Copyright (c) 2022, Alexander Moisseev, BSD 2-Clause
+ */
+function D3Pie(v,t){"use strict";const A=$.extend(!0,{canvasPadding:5,cornerRadius:3,duration:1250,gradient:{enabled:!0,percentage:100},labels:{inner:{hideWhenLessThanPercentage:4,offset:.15},outer:{collideHeight:13,format:"label",pieDistance:30}},padAngle:.01,pieCenterOffset:{x:0,y:0},size:{canvasHeight:400,canvasWidth:600,pieInnerRadius:"20%",pieOuterRadius:"85%"},title:"",total:{enabled:!1}},t),e=(this.destroy=function(){d3.selectAll("#"+v+" svg, #"+v+"-tooltip").remove()},this.destroy(),d3.select("#"+v).append("svg").attr("class","d3pie").attr("width",A.size.canvasWidth).attr("height",A.size.canvasHeight));let l=0;if(""!==A.title){const a=e.append("svg:text").attr("class","chart-title").attr("x",A.size.canvasWidth/2);a.append("tspan").text(A.title+" "),l=a.node().getBBox().height,a.attr("y",l+A.canvasPadding)}const x=e.append("g").attr("transform","translate("+(A.size.canvasWidth/2+A.pieCenterOffset.x)+","+(A.size.canvasHeight/2+l/2+A.pieCenterOffset.y)+")"),y={},m={},{outerRadius:M,innerRadius:R}=function(){function t(t,e){var a;return/%/u.test(t)?(a=Math.max(0,Math.min(99,parseInt(t.replace(/[\D]/u,""),10)))/100,Math.floor(e*a)):parseInt(t,10)}var e=A.size.canvasWidth-2*A.canvasPadding,a=A.size.canvasHeight-2*A.canvasPadding-l;let n=Math.min(e,a)/2;"none"!==A.labels.outer.format&&(e=parseInt(A.labels.outer.pieDistance,10),n>e&&(n-=e));a=t(A.size.pieOuterRadius,n);return{outerRadius:a,innerRadius:t(A.size.pieInnerRadius,a)}}(),w=M+A.labels.outer.pieDistance,z=d3.line().curve(d3.curveCatmullRomOpen),P=d3.select("body").append("div").attr("id",v+"-tooltip").attr("class","d3pie-tooltip"),r=P.append("span").attr("id",v+"-tooltip-text");function I(t){t.on("mouseover",function(t,e){var a=P.datum().total,n=a?Math.round(100*e.data.value/a):NaN;e.data.value?(P.transition().duration(300).style("opacity",1),r.text(e.data.label+(a?": "+e.data.value+" ("+n+"%)":""))):P.transition().duration(300).style("opacity",0),P.each(function(t){t.height=this.getBoundingClientRect().height})}).on("mouseout",function(){P.transition().duration(300).style("opacity",0)}).on("mousemove",t=>{const{pageX:e,pageY:a}=t;P.style("left",e+"px").style("top",function(t){return a-t.height-2+"px"})})}const H=x.append("g");if(I(H),H.append("circle").attr("r",R).style("opacity",0),A.total.enabled){const n=H.append("text").attr("class","total-text");n.append("tspan").attr("class","total-value").style("font-size",.6*R+"px"),n.append("tspan").attr("x","0").attr("dy",.5*R).text(void 0!==A.total.label?A.total.label:"Total")}const O=e.append("defs");this.data=function(t){let l=$.extend(!0,[],t);const d=[],e=l.reduce(function(t,e){return t+(e.value||0)},0),a=(P.datum({total:e}),H.datum({data:{label:void 0!==A.total.label?A.total.label:"Total",value:e}}),A.total.enabled&&H.select(".total-value").text(d3.format(".3~s")(e)),l.unshift({label:"undefined",color:A.gradient.enabled?"steelblue":"#ecf1f5",value:0===e?1:0}),d3.scaleOrdinal(d3.schemeSet1));function n(t,e){return void 0!==t&&void 0!==t.color?t.color:a(e)}function s(t,e,a=e){return d3.arc().innerRadius(e).outerRadius(a).centroid(t)}function c(t){return d3.interpolate(y[t.data.label],t)}function u(t){var e=w-.1;return Math.max(-e,Math.min(e,t))}function r(r,i){m[r.data.label].newAngle=function(){var t=u(d[i].y);let e=Math.sqrt(Math.pow(w,2)-Math.pow(t,2)),a=((r.endAngle+r.startAngle)/2>Math.PI&&(e*=-1),Math.PI/2-Math.atan2(-t,e));return a<0&&(a+=2*Math.PI),{startAngle:a,endAngle:a}}();const o=d3.interpolate(m[r.data.label].currentAngle,m[r.data.label].newAngle);return function(t){var e=s(o(t),w),[a,n]=e,l=0<a?{dx:5,textAnchor:"start"}:{dx:-5,textAnchor:"end"};return d3.select(x.selectAll(".link").nodes()[i]).datum([s(c(r)(t),M),s(c(r)(t),M+5),[a,n],[a+l.dx,n]]).attr("d",z),d3.select(x.selectAll(".outer-label").nodes()[i]).attr("dx",l.dx).style("text-anchor",l.textAnchor),"translate("+e+")"}}t=d3.transition().duration(A.duration);if(A.gradient.enabled){const g=O.selectAll("radialGradient").data(l,function(t){return t.label}),h=g.enter().append("radialGradient").attr("gradientUnits","userSpaceOnUse").attr("cx",0).attr("cy",0).attr("r","120%").attr("id",function(t,e){return v+"-grad"+e});h.append("stop").attr("class","grad-stop-0").style("stop-color",n),h.append("stop").attr("class","grad-stop-1").attr("offset",A.gradient.percentage+"%"),O.selectAll("radialGradient").select(".grad-stop-0").transition(t).style("stop-color",n)}function i(t){return t.data.label}const o=d3.pie().sort(null).value(function(t){return t.value}),p=x.selectAll(".slice-g").data(o(l),i),f=p.enter().append("g").attr("class","slice-g");if(I(f),f.append("path").attr("id",function(t,e){return v+"-slice"+e}).attr("class","slice").attr("fill",function(t,e){return A.gradient.enabled?"url(#"+v+"-grad"+e+")":n(t.data,e)}),p.exit().each(function(t,e){l[e]={value:0,label:t.data.label}}),o(l).forEach(function(t,e){void 0===y[t.data.label]&&(e=e?y[o(l)[e-1].data.label].endAngle:0,y[t.data.label]={startAngle:e,endAngle:e})}),x.selectAll(".slice").data(o(l),i).transition(t).attrTween("d",function(e){return function(t){return d3.arc().padAngle(A.padAngle).cornerRadius(A.cornerRadius).innerRadius(R).outerRadius(M)(c(e)(t))}}).end().then(function(){l=l.filter(function(t,e){return 0===e||t.value}),O.selectAll("radialGradient").data(l,function(t){return t.label}).exit().remove(),x.selectAll(".slice-g").data(o(l),i).exit().each(function(t){delete y[t.data.label],delete m[t.data.label]}).remove();for(const t of o(l))y[t.data.label]=t,"none"!==A.labels.outer.format&&(m[t.data.label].currentAngle=m[t.data.label].newAngle)}).catch(function(t){console.warn(t)}),f.append("text").attr("class","inner-label").attr("dy",".35em"),x.selectAll(".inner-label").data(o(l),i).text(function(t){return"undefined"===t.data.label?"undefined":Math.round(100*t.data.value/e)+"%"}).transition(t).attrTween("opacity",function(e){return e.data.value?function(t){t=c(e)(t);return 100*(t.endAngle-t.startAngle)/(2*Math.PI)<A.labels.inner.hideWhenLessThanPercentage?0:1}:function(){return 0}}).attrTween("transform",function(e){return function(t){return"translate("+s(c(e)(t),R*(1-A.labels.inner.offset),M*(1+A.labels.inner.offset))+")"}}),"none"!==A.labels.outer.format){o(l).forEach(function(t,e){void 0===m[t.data.label]&&(e=e?y[o(l)[e-1].data.label].endAngle:0,m[t.data.label]={currentAngle:{startAngle:e,endAngle:e},newAngle:{startAngle:e,endAngle:e}});let a=0;var[e,n]=s(t,w);t.data.value&&(a=0<=e?A.labels.outer.collideHeight:-A.labels.outer.collideHeight),d.push({fx:a,y:n})}),d3.forceSimulation(d).alphaMin(.5).force("collide",d3.forceCollide(A.labels.outer.collideHeight/2)).force("boundY",function(){for(const t of d)t.y=u(t.y)}).tick(30);const b=f.append("g").attr("class","outer-label-g");b.append("text").attr("class","outer-label").attr("dy",".35em").text(i),b.append("path").attr("class","link"),x.selectAll(".outer-label-g").data(o(l),i).transition(t).style("opacity",function(t,e){return e&&t.value?1:0}).each(function(t,e){$(this).children(".link").attr("stroke",n(t.data,e))}),x.selectAll(".outer-label").data(o(l),i).transition(t).attrTween("transform",r)}}} \ No newline at end of file
diff --git a/interface/js/lib/fontawesome.min.js b/interface/js/lib/fontawesome.min.js
new file mode 100644
index 0000000..ab008ed
--- /dev/null
+++ b/interface/js/lib/fontawesome.min.js
@@ -0,0 +1,5 @@
+/*!
+ * Font Awesome Free 5.13.1 by @fontawesome - https://fontawesome.com
+ * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+ */
+!function(){"use strict";function i(t){return(i="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}function r(t,e){for(var n=0;n<e.length;n++){var a=e[n];a.enumerable=a.enumerable||!1,a.configurable=!0,"value"in a&&(a.writable=!0),Object.defineProperty(t,a.key,a)}}function Q(r){for(var t=1;t<arguments.length;t++){var i=null!=arguments[t]?arguments[t]:{},e=Object.keys(i);"function"==typeof Object.getOwnPropertySymbols&&(e=e.concat(Object.getOwnPropertySymbols(i).filter(function(t){return Object.getOwnPropertyDescriptor(i,t).enumerable}))),e.forEach(function(t){var e,n,a;e=r,a=i[n=t],n in e?Object.defineProperty(e,n,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[n]=a})}return r}function p(t,e){return function(t){if(Array.isArray(t))return t}(t)||function(t,e){var n=[],a=!0,r=!1,i=void 0;try{for(var o,c=t[Symbol.iterator]();!(a=(o=c.next()).done)&&(n.push(o.value),!e||n.length!==e);a=!0);}catch(t){r=!0,i=t}finally{try{a||null==c.return||c.return()}finally{if(r)throw i}}return n}(t,e)||function(){throw new TypeError("Invalid attempt to destructure non-iterable instance")}()}function d(t){return function(t){if(Array.isArray(t)){for(var e=0,n=new Array(t.length);e<t.length;e++)n[e]=t[e];return n}}(t)||function(t){if(Symbol.iterator in Object(t)||"[object Arguments]"===Object.prototype.toString.call(t))return Array.from(t)}(t)||function(){throw new TypeError("Invalid attempt to spread non-iterable instance")}()}var t=function(){},e={},n={},a=null,o={mark:t,measure:t};try{"undefined"!=typeof window&&(e=window),"undefined"!=typeof document&&(n=document),"undefined"!=typeof MutationObserver&&(a=MutationObserver),"undefined"!=typeof performance&&(o=performance)}catch(t){}var c=(e.navigator||{}).userAgent,s=void 0===c?"":c,g=e,v=n,l=a,f=o,u=!!g.document,m=!!v.documentElement&&!!v.head&&"function"==typeof v.addEventListener&&"function"==typeof v.createElement,k=~s.indexOf("MSIE")||~s.indexOf("Trident/"),h="___FONT_AWESOME___",A=16,b="fa",y="svg-inline--fa",Z="data-fa-i2svg",w="data-fa-pseudo-element",x="data-fa-pseudo-element-pending",C="data-prefix",O="data-icon",S="fontawesome-i2svg",P="async",N=["HTML","HEAD","STYLE","SCRIPT"],M=function(){try{return!0}catch(t){return!1}}(),z={fas:"solid",far:"regular",fal:"light",fad:"duotone",fab:"brands",fa:"solid"},E={solid:"fas",regular:"far",light:"fal",duotone:"fad",brands:"fab"},j="fa-layers-text",L=/Font Awesome 5 (Solid|Regular|Light|Duotone|Brands|Free|Pro)/,R={900:"fas",400:"far",normal:"far",300:"fal"},I=[1,2,3,4,5,6,7,8,9,10],_=I.concat([11,12,13,14,15,16,17,18,19,20]),T=["class","data-prefix","data-icon","data-fa-transform","data-fa-mask"],Y={GROUP:"group",SWAP_OPACITY:"swap-opacity",PRIMARY:"primary",SECONDARY:"secondary"},F=["xs","sm","lg","fw","ul","li","border","pull-left","pull-right","spin","pulse","rotate-90","rotate-180","rotate-270","flip-horizontal","flip-vertical","flip-both","stack","stack-1x","stack-2x","inverse","layers","layers-text","layers-counter",Y.GROUP,Y.SWAP_OPACITY,Y.PRIMARY,Y.SECONDARY].concat(I.map(function(t){return"".concat(t,"x")})).concat(_.map(function(t){return"w-".concat(t)})),H=g.FontAwesomeConfig||{};if(v&&"function"==typeof v.querySelector){[["data-family-prefix","familyPrefix"],["data-replacement-class","replacementClass"],["data-auto-replace-svg","autoReplaceSvg"],["data-auto-add-css","autoAddCss"],["data-auto-a11y","autoA11y"],["data-search-pseudo-elements","searchPseudoElements"],["data-observe-mutations","observeMutations"],["data-mutate-approach","mutateApproach"],["data-keep-original-source","keepOriginalSource"],["data-measure-performance","measurePerformance"],["data-show-missing-icons","showMissingIcons"]].forEach(function(t){var e,n=p(t,2),a=n[0],r=n[1],i=""===(e=function(t){var e=v.querySelector("script["+t+"]");if(e)return e.getAttribute(t)}(a))||"false"!==e&&("true"===e||e);null!=i&&(H[r]=i)})}var D=Q({},{familyPrefix:b,replacementClass:y,autoReplaceSvg:!0,autoAddCss:!0,autoA11y:!0,searchPseudoElements:!1,observeMutations:!0,mutateApproach:"async",keepOriginalSource:!0,measurePerformance:!1,showMissingIcons:!0},H);D.autoReplaceSvg||(D.observeMutations=!1);var $=Q({},D);g.FontAwesomeConfig=$;var U=g||{};U[h]||(U[h]={}),U[h].styles||(U[h].styles={}),U[h].hooks||(U[h].hooks={}),U[h].shims||(U[h].shims=[]);var W=U[h],q=[],X=!1;function B(t){m&&(X?setTimeout(t,0):q.push(t))}m&&((X=(v.documentElement.doScroll?/^loaded|^c/:/^loaded|^i|^c/).test(v.readyState))||v.addEventListener("DOMContentLoaded",function t(){v.removeEventListener("DOMContentLoaded",t),X=1,q.map(function(t){return t()})}));var V,K="pending",G="settled",J="fulfilled",tt="rejected",et=function(){},nt="undefined"!=typeof global&&void 0!==global.process&&"function"==typeof global.process.emit,at="undefined"==typeof setImmediate?setTimeout:setImmediate,rt=[];function it(){for(var t=0;t<rt.length;t++)rt[t][0](rt[t][1]);V=!(rt=[])}function ot(t,e){rt.push([t,e]),V||(V=!0,at(it,0))}function ct(t){var e=t.owner,n=e._state,a=e._data,r=t[n],i=t.then;if("function"==typeof r){n=J;try{a=r(a)}catch(t){ut(i,t)}}st(i,a)||(n===J&&lt(i,a),n===tt&&ut(i,a))}function st(e,n){var a;try{if(e===n)throw new TypeError("A promises callback cannot return that same promise.");if(n&&("function"==typeof n||"object"===i(n))){var t=n.then;if("function"==typeof t)return t.call(n,function(t){a||(a=!0,n===t?ft(e,t):lt(e,t))},function(t){a||(a=!0,ut(e,t))}),!0}}catch(t){return a||ut(e,t),!0}return!1}function lt(t,e){t!==e&&st(t,e)||ft(t,e)}function ft(t,e){t._state===K&&(t._state=G,t._data=e,ot(mt,t))}function ut(t,e){t._state===K&&(t._state=G,t._data=e,ot(pt,t))}function dt(t){t._then=t._then.forEach(ct)}function mt(t){t._state=J,dt(t)}function pt(t){t._state=tt,dt(t),!t._handled&&nt&&global.process.emit("unhandledRejection",t._data,t)}function ht(t){global.process.emit("rejectionHandled",t)}function gt(t){if("function"!=typeof t)throw new TypeError("Promise resolver "+t+" is not a function");if(this instanceof gt==!1)throw new TypeError("Failed to construct 'Promise': Please use the 'new' operator, this object constructor cannot be called as a function.");this._then=[],function(t,e){function n(t){ut(e,t)}try{t(function(t){lt(e,t)},n)}catch(t){n(t)}}(t,this)}gt.prototype={constructor:gt,_state:K,_then:null,_data:void 0,_handled:!1,then:function(t,e){var n={owner:this,then:new this.constructor(et),fulfilled:t,rejected:e};return!e&&!t||this._handled||(this._handled=!0,this._state===tt&&nt&&ot(ht,this)),this._state===J||this._state===tt?ot(ct,n):this._then.push(n),n.then},catch:function(t){return this.then(null,t)}},gt.all=function(c){if(!Array.isArray(c))throw new TypeError("You must pass an array to Promise.all().");return new gt(function(n,t){var a=[],r=0;function e(e){return r++,function(t){a[e]=t,--r||n(a)}}for(var i,o=0;o<c.length;o++)(i=c[o])&&"function"==typeof i.then?i.then(e(o),t):a[o]=i;r||n(a)})},gt.race=function(r){if(!Array.isArray(r))throw new TypeError("You must pass an array to Promise.race().");return new gt(function(t,e){for(var n,a=0;a<r.length;a++)(n=r[a])&&"function"==typeof n.then?n.then(t,e):t(n)})},gt.resolve=function(e){return e&&"object"===i(e)&&e.constructor===gt?e:new gt(function(t){t(e)})},gt.reject=function(n){return new gt(function(t,e){e(n)})};var vt="function"==typeof Promise?Promise:gt,bt=A,yt={size:16,x:0,y:0,rotate:0,flipX:!1,flipY:!1};function wt(t){if(t&&m){var e=v.createElement("style");e.setAttribute("type","text/css"),e.innerHTML=t;for(var n=v.head.childNodes,a=null,r=n.length-1;-1<r;r--){var i=n[r],o=(i.tagName||"").toUpperCase();-1<["STYLE","LINK"].indexOf(o)&&(a=i)}return v.head.insertBefore(e,a),t}}var xt="0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";function kt(){for(var t=12,e="";0<t--;)e+=xt[62*Math.random()|0];return e}function At(t){for(var e=[],n=(t||[]).length>>>0;n--;)e[n]=t[n];return e}function Ct(t){return t.classList?At(t.classList):(t.getAttribute("class")||"").split(" ").filter(function(t){return t})}function Ot(t,e){var n,a=e.split("-"),r=a[0],i=a.slice(1).join("-");return r!==t||""===i||(n=i,~F.indexOf(n))?null:i}function St(t){return"".concat(t).replace(/&/g,"&amp;").replace(/"/g,"&quot;").replace(/'/g,"&#39;").replace(/</g,"&lt;").replace(/>/g,"&gt;")}function Pt(n){return Object.keys(n||{}).reduce(function(t,e){return t+"".concat(e,": ").concat(n[e],";")},"")}function Nt(t){return t.size!==yt.size||t.x!==yt.x||t.y!==yt.y||t.rotate!==yt.rotate||t.flipX||t.flipY}function Mt(t){var e=t.transform,n=t.containerWidth,a=t.iconWidth,r={transform:"translate(".concat(n/2," 256)")},i="translate(".concat(32*e.x,", ").concat(32*e.y,") "),o="scale(".concat(e.size/16*(e.flipX?-1:1),", ").concat(e.size/16*(e.flipY?-1:1),") "),c="rotate(".concat(e.rotate," 0 0)");return{outer:r,inner:{transform:"".concat(i," ").concat(o," ").concat(c)},path:{transform:"translate(".concat(a/2*-1," -256)")}}}var zt={x:0,y:0,width:"100%",height:"100%"};function Et(t){var e=!(1<arguments.length&&void 0!==arguments[1])||arguments[1];return t.attributes&&(t.attributes.fill||e)&&(t.attributes.fill="black"),t}function jt(t){var e=t.icons,n=e.main,a=e.mask,r=t.prefix,i=t.iconName,o=t.transform,c=t.symbol,s=t.title,l=t.maskId,f=t.titleId,u=t.extra,d=t.watchable,m=void 0!==d&&d,p=a.found?a:n,h=p.width,g=p.height,v="fa-w-".concat(Math.ceil(h/g*16)),b=[$.replacementClass,i?"".concat($.familyPrefix,"-").concat(i):"",v].filter(function(t){return-1===u.classes.indexOf(t)}).concat(u.classes).join(" "),y={children:[],attributes:Q({},u.attributes,{"data-prefix":r,"data-icon":i,class:b,role:u.attributes.role||"img",xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 ".concat(h," ").concat(g)})};m&&(y.attributes[Z]=""),s&&y.children.push({tag:"title",attributes:{id:y.attributes["aria-labelledby"]||"title-".concat(f||kt())},children:[s]});var w,x,k,A,C,O,S,P,N,M,z,E,j,L,R,I,_,T,Y,F,H,D,U,W,q,X,B,V=Q({},y,{prefix:r,iconName:i,main:n,mask:a,maskId:l,transform:o,symbol:c,styles:u.styles}),K=a.found&&n.found?(k=(w=V).children,A=w.attributes,C=w.main,O=w.mask,S=w.maskId,P=w.transform,N=C.width,M=C.icon,z=O.width,E=O.icon,j=Mt({transform:P,containerWidth:z,iconWidth:N}),L={tag:"rect",attributes:Q({},zt,{fill:"white"})},R=M.children?{children:M.children.map(Et)}:{},I={tag:"g",attributes:Q({},j.inner),children:[Et(Q({tag:M.tag,attributes:Q({},M.attributes,j.path)},R))]},_={tag:"g",attributes:Q({},j.outer),children:[I]},T="mask-".concat(S||kt()),Y="clip-".concat(S||kt()),F={tag:"mask",attributes:Q({},zt,{id:T,maskUnits:"userSpaceOnUse",maskContentUnits:"userSpaceOnUse"}),children:[L,_]},H={tag:"defs",children:[{tag:"clipPath",attributes:{id:Y},children:(x=E,"g"===x.tag?x.children:[x])},F]},k.push(H,{tag:"rect",attributes:Q({fill:"currentColor","clip-path":"url(#".concat(Y,")"),mask:"url(#".concat(T,")")},zt)}),{children:k,attributes:A}):function(t){var e=t.children,n=t.attributes,a=t.main,r=t.transform,i=Pt(t.styles);if(0<i.length&&(n.style=i),Nt(r)){var o=Mt({transform:r,containerWidth:a.width,iconWidth:a.width});e.push({tag:"g",attributes:Q({},o.outer),children:[{tag:"g",attributes:Q({},o.inner),children:[{tag:a.icon.tag,children:a.icon.children,attributes:Q({},a.icon.attributes,o.path)}]}]})}else e.push(a.icon);return{children:e,attributes:n}}(V),G=K.children,J=K.attributes;return V.children=G,V.attributes=J,c?(U=(D=V).prefix,W=D.iconName,q=D.children,X=D.attributes,B=D.symbol,[{tag:"svg",attributes:{style:"display: none;"},children:[{tag:"symbol",attributes:Q({},X,{id:!0===B?"".concat(U,"-").concat($.familyPrefix,"-").concat(W):B}),children:q}]}]):function(t){var e=t.children,n=t.main,a=t.mask,r=t.attributes,i=t.styles,o=t.transform;if(Nt(o)&&n.found&&!a.found){var c=n.width/n.height/2,s=.5;r.style=Pt(Q({},i,{"transform-origin":"".concat(c+o.x/16,"em ").concat(s+o.y/16,"em")}))}return[{tag:"svg",attributes:r,children:e}]}(V)}function Lt(t){var e=t.content,n=t.width,a=t.height,r=t.transform,i=t.title,o=t.extra,c=t.watchable,s=void 0!==c&&c,l=Q({},o.attributes,i?{title:i}:{},{class:o.classes.join(" ")});s&&(l[Z]="");var f,u,d,m,p,h,g,v,b,y=Q({},o.styles);Nt(r)&&(y.transform=(u=(f={transform:r,startCentered:!0,width:n,height:a}).transform,d=f.width,m=void 0===d?A:d,p=f.height,h=void 0===p?A:p,g=f.startCentered,b="",b+=(v=void 0!==g&&g)&&k?"translate(".concat(u.x/bt-m/2,"em, ").concat(u.y/bt-h/2,"em) "):v?"translate(calc(-50% + ".concat(u.x/bt,"em), calc(-50% + ").concat(u.y/bt,"em)) "):"translate(".concat(u.x/bt,"em, ").concat(u.y/bt,"em) "),b+="scale(".concat(u.size/bt*(u.flipX?-1:1),", ").concat(u.size/bt*(u.flipY?-1:1),") "),b+="rotate(".concat(u.rotate,"deg) ")),y["-webkit-transform"]=y.transform);var w=Pt(y);0<w.length&&(l.style=w);var x=[];return x.push({tag:"span",attributes:l,children:[e]}),i&&x.push({tag:"span",attributes:{class:"sr-only"},children:[i]}),x}var Rt=function(){},It=$.measurePerformance&&f&&f.mark&&f.measure?f:{mark:Rt,measure:Rt},_t='FA "5.13.1"',Tt=function(t){It.mark("".concat(_t," ").concat(t," ends")),It.measure("".concat(_t," ").concat(t),"".concat(_t," ").concat(t," begins"),"".concat(_t," ").concat(t," ends"))},Yt={begin:function(t){return It.mark("".concat(_t," ").concat(t," begins")),function(){return Tt(t)}},end:Tt},Ft=function(t,e,n,a){var r,i,o,c,s,l=Object.keys(t),f=l.length,u=void 0!==a?(c=e,s=a,function(t,e,n,a){return c.call(s,t,e,n,a)}):e;for(o=void 0===n?(r=1,t[l[0]]):(r=0,n);r<f;r++)o=u(o,t[i=l[r]],i,t);return o};function Ht(t){for(var e="",n=0;n<t.length;n++){e+=("000"+t.charCodeAt(n).toString(16)).slice(-4)}return e}var Dt=W.styles,Ut=W.shims,Wt={},qt={},Xt={},Bt=function(){var t=function(a){return Ft(Dt,function(t,e,n){return t[n]=Ft(e,a,{}),t},{})};Wt=t(function(t,e,n){return e[3]&&(t[e[3]]=n),t}),qt=t(function(e,t,n){var a=t[2];return e[n]=n,a.forEach(function(t){e[t]=n}),e});var i="far"in Dt;Xt=Ft(Ut,function(t,e){var n=e[0],a=e[1],r=e[2];return"far"!==a||i||(a="fas"),t[n]={prefix:a,iconName:r},t},{})};function Vt(t,e){return(Wt[t]||{})[e]}Bt();var Kt=W.styles,Gt=function(){return{prefix:null,iconName:null,rest:[]}};function Jt(t){return t.reduce(function(t,e){var n=Ot($.familyPrefix,e);if(Kt[e])t.prefix=e;else if($.autoFetchSvg&&-1<["fas","far","fal","fad","fab","fa"].indexOf(e))t.prefix=e;else if(n){var a="fa"===t.prefix?Xt[n]||{prefix:null,iconName:null}:{};t.iconName=a.iconName||n,t.prefix=a.prefix||t.prefix}else e!==$.replacementClass&&0!==e.indexOf("fa-w-")&&t.rest.push(e);return t},Gt())}function Qt(t,e,n){if(t&&t[e]&&t[e][n])return{prefix:e,iconName:n,icon:t[e][n]}}function Zt(t){var n,e=t.tag,a=t.attributes,r=void 0===a?{}:a,i=t.children,o=void 0===i?[]:i;return"string"==typeof t?St(t):"<".concat(e," ").concat((n=r,Object.keys(n||{}).reduce(function(t,e){return t+"".concat(e,'="').concat(St(n[e]),'" ')},"").trim()),">").concat(o.map(Zt).join(""),"</").concat(e,">")}var $t=function(){};function te(t){return"string"==typeof(t.getAttribute?t.getAttribute(Z):null)}var ee={replace:function(t){var e=t[0],n=t[1].map(function(t){return Zt(t)}).join("\n");if(e.parentNode&&e.outerHTML)e.outerHTML=n+($.keepOriginalSource&&"svg"!==e.tagName.toLowerCase()?"\x3c!-- ".concat(e.outerHTML," --\x3e"):"");else if(e.parentNode){var a=document.createElement("span");e.parentNode.replaceChild(a,e),a.outerHTML=n}},nest:function(t){var e=t[0],n=t[1];if(~Ct(e).indexOf($.replacementClass))return ee.replace(t);var a=new RegExp("".concat($.familyPrefix,"-.*"));delete n[0].attributes.style,delete n[0].attributes.id;var r=n[0].attributes.class.split(" ").reduce(function(t,e){return e===$.replacementClass||e.match(a)?t.toSvg.push(e):t.toNode.push(e),t},{toNode:[],toSvg:[]});n[0].attributes.class=r.toSvg.join(" ");var i=n.map(function(t){return Zt(t)}).join("\n");e.setAttribute("class",r.toNode.join(" ")),e.setAttribute(Z,""),e.innerHTML=i}};function ne(t){t()}function ae(n,t){var a="function"==typeof t?t:$t;if(0===n.length)a();else{var e=ne;$.mutateApproach===P&&(e=g.requestAnimationFrame||ne),e(function(){var t=!0===$.autoReplaceSvg?ee.replace:ee[$.autoReplaceSvg]||ee.replace,e=Yt.begin("mutate");n.map(t),e(),a()})}}var re=!1;function ie(){re=!1}var oe=null;function ce(t){if(l&&$.observeMutations){var r=t.treeCallback,i=t.nodeCallback,o=t.pseudoElementsCallback,e=t.observeMutationsRoot,n=void 0===e?v:e;oe=new l(function(t){re||At(t).forEach(function(t){if("childList"===t.type&&0<t.addedNodes.length&&!te(t.addedNodes[0])&&($.searchPseudoElements&&o(t.target),r(t.target)),"attributes"===t.type&&t.target.parentNode&&$.searchPseudoElements&&o(t.target.parentNode),"attributes"===t.type&&te(t.target)&&~T.indexOf(t.attributeName))if("class"===t.attributeName){var e=Jt(Ct(t.target)),n=e.prefix,a=e.iconName;n&&t.target.setAttribute("data-prefix",n),a&&t.target.setAttribute("data-icon",a)}else i(t.target)})}),m&&oe.observe(n,{childList:!0,attributes:!0,characterData:!0,subtree:!0})}}function se(t){var e,n,a=t.getAttribute("data-prefix"),r=t.getAttribute("data-icon"),i=void 0!==t.innerText?t.innerText.trim():"",o=Jt(Ct(t));return a&&r&&(o.prefix=a,o.iconName=r),o.prefix&&1<i.length?o.iconName=(e=o.prefix,n=t.innerText,(qt[e]||{})[n]):o.prefix&&1===i.length&&(o.iconName=Vt(o.prefix,Ht(t.innerText))),o}var le=function(t){var e={size:16,x:0,y:0,flipX:!1,flipY:!1,rotate:0};return t?t.toLowerCase().split(" ").reduce(function(t,e){var n=e.toLowerCase().split("-"),a=n[0],r=n.slice(1).join("-");if(a&&"h"===r)return t.flipX=!0,t;if(a&&"v"===r)return t.flipY=!0,t;if(r=parseFloat(r),isNaN(r))return t;switch(a){case"grow":t.size=t.size+r;break;case"shrink":t.size=t.size-r;break;case"left":t.x=t.x-r;break;case"right":t.x=t.x+r;break;case"up":t.y=t.y-r;break;case"down":t.y=t.y+r;break;case"rotate":t.rotate=t.rotate+r}return t},e):e};function fe(t){var e,n,a,r,i,o,c,s,l=se(t),f=l.iconName,u=l.prefix,d=l.rest,m=(e=t.getAttribute("style"),n=[],e&&(n=e.split(";").reduce(function(t,e){var n=e.split(":"),a=n[0],r=n.slice(1);return a&&0<r.length&&(t[a]=r.join(":").trim()),t},{})),n),p=le(t.getAttribute("data-fa-transform")),h=null!==(a=t.getAttribute("data-fa-symbol"))&&(""===a||a),g=(i=At((r=t).attributes).reduce(function(t,e){return"class"!==t.name&&"style"!==t.name&&(t[e.name]=e.value),t},{}),o=r.getAttribute("title"),c=r.getAttribute("data-fa-title-id"),$.autoA11y&&(o?i["aria-labelledby"]="".concat($.replacementClass,"-title-").concat(c||kt()):(i["aria-hidden"]="true",i.focusable="false")),i),v=(s=t.getAttribute("data-fa-mask"))?Jt(s.split(" ").map(function(t){return t.trim()})):Gt();return{iconName:f,title:t.getAttribute("title"),titleId:t.getAttribute("data-fa-title-id"),prefix:u,transform:p,symbol:h,mask:v,maskId:t.getAttribute("data-fa-mask-id"),extra:{classes:d,styles:m,attributes:g}}}function ue(t){this.name="MissingIcon",this.message=t||"Icon unavailable",this.stack=(new Error).stack}(ue.prototype=Object.create(Error.prototype)).constructor=ue;var de={fill:"currentColor"},me={attributeType:"XML",repeatCount:"indefinite",dur:"2s"},pe={tag:"path",attributes:Q({},de,{d:"M156.5,447.7l-12.6,29.5c-18.7-9.5-35.9-21.2-51.5-34.9l22.7-22.7C127.6,430.5,141.5,440,156.5,447.7z M40.6,272H8.5 c1.4,21.2,5.4,41.7,11.7,61.1L50,321.2C45.1,305.5,41.8,289,40.6,272z M40.6,240c1.4-18.8,5.2-37,11.1-54.1l-29.5-12.6 C14.7,194.3,10,216.7,8.5,240H40.6z M64.3,156.5c7.8-14.9,17.2-28.8,28.1-41.5L69.7,92.3c-13.7,15.6-25.5,32.8-34.9,51.5 L64.3,156.5z M397,419.6c-13.9,12-29.4,22.3-46.1,30.4l11.9,29.8c20.7-9.9,39.8-22.6,56.9-37.6L397,419.6z M115,92.4 c13.9-12,29.4-22.3,46.1-30.4l-11.9-29.8c-20.7,9.9-39.8,22.6-56.8,37.6L115,92.4z M447.7,355.5c-7.8,14.9-17.2,28.8-28.1,41.5 l22.7,22.7c13.7-15.6,25.5-32.9,34.9-51.5L447.7,355.5z M471.4,272c-1.4,18.8-5.2,37-11.1,54.1l29.5,12.6 c7.5-21.1,12.2-43.5,13.6-66.8H471.4z M321.2,462c-15.7,5-32.2,8.2-49.2,9.4v32.1c21.2-1.4,41.7-5.4,61.1-11.7L321.2,462z M240,471.4c-18.8-1.4-37-5.2-54.1-11.1l-12.6,29.5c21.1,7.5,43.5,12.2,66.8,13.6V471.4z M462,190.8c5,15.7,8.2,32.2,9.4,49.2h32.1 c-1.4-21.2-5.4-41.7-11.7-61.1L462,190.8z M92.4,397c-12-13.9-22.3-29.4-30.4-46.1l-29.8,11.9c9.9,20.7,22.6,39.8,37.6,56.9 L92.4,397z M272,40.6c18.8,1.4,36.9,5.2,54.1,11.1l12.6-29.5C317.7,14.7,295.3,10,272,8.5V40.6z M190.8,50 c15.7-5,32.2-8.2,49.2-9.4V8.5c-21.2,1.4-41.7,5.4-61.1,11.7L190.8,50z M442.3,92.3L419.6,115c12,13.9,22.3,29.4,30.5,46.1 l29.8-11.9C470,128.5,457.3,109.4,442.3,92.3z M397,92.4l22.7-22.7c-15.6-13.7-32.8-25.5-51.5-34.9l-12.6,29.5 C370.4,72.1,384.4,81.5,397,92.4z"})},he=Q({},me,{attributeName:"opacity"}),ge={tag:"g",children:[pe,{tag:"circle",attributes:Q({},de,{cx:"256",cy:"364",r:"28"}),children:[{tag:"animate",attributes:Q({},me,{attributeName:"r",values:"28;14;28;28;14;28;"})},{tag:"animate",attributes:Q({},he,{values:"1;0;1;1;0;1;"})}]},{tag:"path",attributes:Q({},de,{opacity:"1",d:"M263.7,312h-16c-6.6,0-12-5.4-12-12c0-71,77.4-63.9,77.4-107.8c0-20-17.8-40.2-57.4-40.2c-29.1,0-44.3,9.6-59.2,28.7 c-3.9,5-11.1,6-16.2,2.4l-13.1-9.2c-5.6-3.9-6.9-11.8-2.6-17.2c21.2-27.2,46.4-44.7,91.2-44.7c52.3,0,97.4,29.8,97.4,80.2 c0,67.6-77.4,63.5-77.4,107.8C275.7,306.6,270.3,312,263.7,312z"}),children:[{tag:"animate",attributes:Q({},he,{values:"1;0;0;0;0;1;"})}]},{tag:"path",attributes:Q({},de,{opacity:"0",d:"M232.5,134.5l7,168c0.3,6.4,5.6,11.5,12,11.5h9c6.4,0,11.7-5.1,12-11.5l7-168c0.3-6.8-5.2-12.5-12-12.5h-23 C237.7,122,232.2,127.7,232.5,134.5z"}),children:[{tag:"animate",attributes:Q({},he,{values:"0;0;1;1;0;0;"})}]}]},ve=W.styles;function be(t){var e=t[0],n=t[1],a=p(t.slice(4),1)[0];return{found:!0,width:e,height:n,icon:Array.isArray(a)?{tag:"g",attributes:{class:"".concat($.familyPrefix,"-").concat(Y.GROUP)},children:[{tag:"path",attributes:{class:"".concat($.familyPrefix,"-").concat(Y.SECONDARY),fill:"currentColor",d:a[0]}},{tag:"path",attributes:{class:"".concat($.familyPrefix,"-").concat(Y.PRIMARY),fill:"currentColor",d:a[1]}}]}:{tag:"path",attributes:{fill:"currentColor",d:a}}}}function ye(a,r){return new vt(function(t,e){var n={found:!1,width:512,height:512,icon:ge};if(a&&r&&ve[r]&&ve[r][a])return t(be(ve[r][a]));"object"===i(g.FontAwesomeKitConfig)&&"string"==typeof window.FontAwesomeKitConfig.token&&g.FontAwesomeKitConfig.token,a&&r&&!$.showMissingIcons?e(new ue("Icon is missing for prefix ".concat(r," with icon name ").concat(a))):t(n)})}var we=W.styles;function xe(t){var i,e,o,c,s,l,f,u,n,d,m,a=fe(t);return~a.extra.classes.indexOf(j)?function(t,e){var n=e.title,a=e.transform,r=e.extra,i=null,o=null;if(k){var c=parseInt(getComputedStyle(t).fontSize,10),s=t.getBoundingClientRect();i=s.width/c,o=s.height/c}return $.autoA11y&&!n&&(r.attributes["aria-hidden"]="true"),vt.resolve([t,Lt({content:t.innerHTML,width:i,height:o,transform:a,title:n,extra:r,watchable:!0})])}(t,a):(i=t,o=(e=a).iconName,c=e.title,s=e.titleId,l=e.prefix,f=e.transform,u=e.symbol,n=e.mask,d=e.maskId,m=e.extra,new vt(function(r,t){vt.all([ye(o,l),ye(n.iconName,n.prefix)]).then(function(t){var e=p(t,2),n=e[0],a=e[1];r([i,jt({icons:{main:n,mask:a},prefix:l,iconName:o,transform:f,symbol:u,mask:a,maskId:d,title:c,titleId:s,extra:m,watchable:!0})])})}))}function ke(t){var n=1<arguments.length&&void 0!==arguments[1]?arguments[1]:null;if(m){var e=v.documentElement.classList,a=function(t){return e.add("".concat(S,"-").concat(t))},r=function(t){return e.remove("".concat(S,"-").concat(t))},i=$.autoFetchSvg?Object.keys(z):Object.keys(we),o=[".".concat(j,":not([").concat(Z,"])")].concat(i.map(function(t){return".".concat(t,":not([").concat(Z,"])")})).join(", ");if(0!==o.length){var c=[];try{c=At(t.querySelectorAll(o))}catch(t){}if(0<c.length){a("pending"),r("complete");var s=Yt.begin("onTree"),l=c.reduce(function(t,e){try{var n=xe(e);n&&t.push(n)}catch(t){M||t instanceof ue&&console.error(t)}return t},[]);return new vt(function(e,t){vt.all(l).then(function(t){ae(t,function(){a("active"),a("complete"),r("pending"),"function"==typeof n&&n(),s(),e()})}).catch(function(){s(),t()})})}}}}function Ae(t){var e=1<arguments.length&&void 0!==arguments[1]?arguments[1]:null;xe(t).then(function(t){t&&ae([t],e)})}function Ce(m,p){var h="".concat(x).concat(p.replace(":","-"));return new vt(function(a,t){if(null!==m.getAttribute(h))return a();var e=At(m.children).filter(function(t){return t.getAttribute(w)===p})[0],n=g.getComputedStyle(m,p),r=n.getPropertyValue("font-family").match(L),i=n.getPropertyValue("font-weight"),o=n.getPropertyValue("content");if(e&&!r)return m.removeChild(e),a();if(r&&"none"!==o&&""!==o){var c=~["Solid","Regular","Light","Duotone","Brands"].indexOf(r[1])?E[r[1].toLowerCase()]:R[i],s=Ht(3===o.length?o.substr(1,1):o),l=Vt(c,s),f=l;if(!l||e&&e.getAttribute(C)===c&&e.getAttribute(O)===f)a();else{m.setAttribute(h,f),e&&m.removeChild(e);var u={iconName:null,title:null,titleId:null,prefix:null,transform:yt,symbol:!1,mask:null,maskId:null,extra:{classes:[],styles:{},attributes:{}}},d=u.extra;d.attributes[w]=p,ye(l,c).then(function(t){var e=jt(Q({},u,{icons:{main:t,mask:Gt()},prefix:c,iconName:f,extra:d,watchable:!0})),n=v.createElement("svg");":before"===p?m.insertBefore(n,m.firstChild):m.appendChild(n),n.outerHTML=e.map(function(t){return Zt(t)}).join("\n"),m.removeAttribute(h),a()}).catch(t)}}else a()})}function Oe(t){return vt.all([Ce(t,":before"),Ce(t,":after")])}function Se(t){return!(t.parentNode===document.head||~N.indexOf(t.tagName.toUpperCase())||t.getAttribute(w)||t.parentNode&&"svg"===t.parentNode.tagName)}function Pe(r){if(m)return new vt(function(t,e){var n=At(r.querySelectorAll("*")).filter(Se).map(Oe),a=Yt.begin("searchPseudoElements");re=!0,vt.all(n).then(function(){a(),ie(),t()}).catch(function(){a(),ie(),e()})})}var Ne="svg:not(:root).svg-inline--fa{overflow:visible}.svg-inline--fa{display:inline-block;font-size:inherit;height:1em;overflow:visible;vertical-align:-.125em}.svg-inline--fa.fa-lg{vertical-align:-.225em}.svg-inline--fa.fa-w-1{width:.0625em}.svg-inline--fa.fa-w-2{width:.125em}.svg-inline--fa.fa-w-3{width:.1875em}.svg-inline--fa.fa-w-4{width:.25em}.svg-inline--fa.fa-w-5{width:.3125em}.svg-inline--fa.fa-w-6{width:.375em}.svg-inline--fa.fa-w-7{width:.4375em}.svg-inline--fa.fa-w-8{width:.5em}.svg-inline--fa.fa-w-9{width:.5625em}.svg-inline--fa.fa-w-10{width:.625em}.svg-inline--fa.fa-w-11{width:.6875em}.svg-inline--fa.fa-w-12{width:.75em}.svg-inline--fa.fa-w-13{width:.8125em}.svg-inline--fa.fa-w-14{width:.875em}.svg-inline--fa.fa-w-15{width:.9375em}.svg-inline--fa.fa-w-16{width:1em}.svg-inline--fa.fa-w-17{width:1.0625em}.svg-inline--fa.fa-w-18{width:1.125em}.svg-inline--fa.fa-w-19{width:1.1875em}.svg-inline--fa.fa-w-20{width:1.25em}.svg-inline--fa.fa-pull-left{margin-right:.3em;width:auto}.svg-inline--fa.fa-pull-right{margin-left:.3em;width:auto}.svg-inline--fa.fa-border{height:1.5em}.svg-inline--fa.fa-li{width:2em}.svg-inline--fa.fa-fw{width:1.25em}.fa-layers svg.svg-inline--fa{bottom:0;left:0;margin:auto;position:absolute;right:0;top:0}.fa-layers{display:inline-block;height:1em;position:relative;text-align:center;vertical-align:-.125em;width:1em}.fa-layers svg.svg-inline--fa{-webkit-transform-origin:center center;transform-origin:center center}.fa-layers-counter,.fa-layers-text{display:inline-block;position:absolute;text-align:center}.fa-layers-text{left:50%;top:50%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%);-webkit-transform-origin:center center;transform-origin:center center}.fa-layers-counter{background-color:#ff253a;border-radius:1em;-webkit-box-sizing:border-box;box-sizing:border-box;color:#fff;height:1.5em;line-height:1;max-width:5em;min-width:1.5em;overflow:hidden;padding:.25em;right:0;text-overflow:ellipsis;top:0;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:top right;transform-origin:top right}.fa-layers-bottom-right{bottom:0;right:0;top:auto;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:bottom right;transform-origin:bottom right}.fa-layers-bottom-left{bottom:0;left:0;right:auto;top:auto;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:bottom left;transform-origin:bottom left}.fa-layers-top-right{right:0;top:0;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:top right;transform-origin:top right}.fa-layers-top-left{left:0;right:auto;top:0;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:top left;transform-origin:top left}.fa-lg{font-size:1.3333333333em;line-height:.75em;vertical-align:-.0667em}.fa-xs{font-size:.75em}.fa-sm{font-size:.875em}.fa-1x{font-size:1em}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-6x{font-size:6em}.fa-7x{font-size:7em}.fa-8x{font-size:8em}.fa-9x{font-size:9em}.fa-10x{font-size:10em}.fa-fw{text-align:center;width:1.25em}.fa-ul{list-style-type:none;margin-left:2.5em;padding-left:0}.fa-ul>li{position:relative}.fa-li{left:-2em;position:absolute;text-align:center;width:2em;line-height:inherit}.fa-border{border:solid .08em #eee;border-radius:.1em;padding:.2em .25em .15em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left,.fab.fa-pull-left,.fal.fa-pull-left,.far.fa-pull-left,.fas.fa-pull-left{margin-right:.3em}.fa.fa-pull-right,.fab.fa-pull-right,.fal.fa-pull-right,.far.fa-pull-right,.fas.fa-pull-right{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s infinite linear;animation:fa-spin 2s infinite linear}.fa-pulse{-webkit-animation:fa-spin 1s infinite steps(8);animation:fa-spin 1s infinite steps(8)}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.fa-rotate-90{-webkit-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-webkit-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-webkit-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-webkit-transform:scale(-1,1);transform:scale(-1,1)}.fa-flip-vertical{-webkit-transform:scale(1,-1);transform:scale(1,-1)}.fa-flip-both,.fa-flip-horizontal.fa-flip-vertical{-webkit-transform:scale(-1,-1);transform:scale(-1,-1)}:root .fa-flip-both,:root .fa-flip-horizontal,:root .fa-flip-vertical,:root .fa-rotate-180,:root .fa-rotate-270,:root .fa-rotate-90{-webkit-filter:none;filter:none}.fa-stack{display:inline-block;height:2em;position:relative;width:2.5em}.fa-stack-1x,.fa-stack-2x{bottom:0;left:0;margin:auto;position:absolute;right:0;top:0}.svg-inline--fa.fa-stack-1x{height:1em;width:1.25em}.svg-inline--fa.fa-stack-2x{height:2em;width:2.5em}.fa-inverse{color:#fff}.sr-only{border:0;clip:rect(0,0,0,0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.sr-only-focusable:active,.sr-only-focusable:focus{clip:auto;height:auto;margin:0;overflow:visible;position:static;width:auto}.svg-inline--fa .fa-primary{fill:var(--fa-primary-color,currentColor);opacity:1;opacity:var(--fa-primary-opacity,1)}.svg-inline--fa .fa-secondary{fill:var(--fa-secondary-color,currentColor);opacity:.4;opacity:var(--fa-secondary-opacity,.4)}.svg-inline--fa.fa-swap-opacity .fa-primary{opacity:.4;opacity:var(--fa-secondary-opacity,.4)}.svg-inline--fa.fa-swap-opacity .fa-secondary{opacity:1;opacity:var(--fa-primary-opacity,1)}.svg-inline--fa mask .fa-primary,.svg-inline--fa mask .fa-secondary{fill:#000}.fad.fa-inverse{color:#fff}";function Me(){var t=b,e=y,n=$.familyPrefix,a=$.replacementClass,r=Ne;if(n!==t||a!==e){var i=new RegExp("\\.".concat(t,"\\-"),"g"),o=new RegExp("\\--".concat(t,"\\-"),"g"),c=new RegExp("\\.".concat(e),"g");r=r.replace(i,".".concat(n,"-")).replace(o,"--".concat(n,"-")).replace(c,".".concat(a))}return r}function ze(){$.autoAddCss&&!Ie&&(wt(Me()),Ie=!0)}function Ee(e,t){return Object.defineProperty(e,"abstract",{get:t}),Object.defineProperty(e,"html",{get:function(){return e.abstract.map(function(t){return Zt(t)})}}),Object.defineProperty(e,"node",{get:function(){if(m){var t=v.createElement("div");return t.innerHTML=e.html,t.children}}}),e}function je(t){var e=t.prefix,n=void 0===e?"fa":e,a=t.iconName;if(a)return Qt(Re.definitions,n,a)||Qt(W.styles,n,a)}var Le,Re=new(function(){function t(){!function(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}(this,t),this.definitions={}}var e,n,a;return e=t,(n=[{key:"add",value:function(){for(var e=this,t=arguments.length,n=new Array(t),a=0;a<t;a++)n[a]=arguments[a];var r=n.reduce(this._pullDefinitions,{});Object.keys(r).forEach(function(t){e.definitions[t]=Q({},e.definitions[t]||{},r[t]),function t(e,a){var n=(2<arguments.length&&void 0!==arguments[2]?arguments[2]:{}).skipHooks,r=void 0!==n&&n,i=Object.keys(a).reduce(function(t,e){var n=a[e];return n.icon?t[n.iconName]=n.icon:t[e]=n,t},{});"function"!=typeof W.hooks.addPack||r?W.styles[e]=Q({},W.styles[e]||{},i):W.hooks.addPack(e,i),"fas"===e&&t("fa",a)}(t,r[t]),Bt()})}},{key:"reset",value:function(){this.definitions={}}},{key:"_pullDefinitions",value:function(i,t){var o=t.prefix&&t.iconName&&t.icon?{0:t}:t;return Object.keys(o).map(function(t){var e=o[t],n=e.prefix,a=e.iconName,r=e.icon;i[n]||(i[n]={}),i[n][a]=r}),i}}])&&r(e.prototype,n),a&&r(e,a),t}()),Ie=!1,_e={i2svg:function(){var t=0<arguments.length&&void 0!==arguments[0]?arguments[0]:{};if(m){ze();var e=t.node,n=void 0===e?v:e,a=t.callback,r=void 0===a?function(){}:a;return $.searchPseudoElements&&Pe(n),ke(n,r)}return vt.reject("Operation requires a DOM of some kind.")},css:Me,insertCss:function(){Ie||(wt(Me()),Ie=!0)},watch:function(){var t=0<arguments.length&&void 0!==arguments[0]?arguments[0]:{},e=t.autoReplaceSvgRoot,n=t.observeMutationsRoot;!1===$.autoReplaceSvg&&($.autoReplaceSvg=!0),$.observeMutations=!0,B(function(){Fe({autoReplaceSvgRoot:e}),ce({treeCallback:ke,nodeCallback:Ae,pseudoElementsCallback:Pe,observeMutationsRoot:n})})}},Te=(Le=function(t){var e=1<arguments.length&&void 0!==arguments[1]?arguments[1]:{},n=e.transform,a=void 0===n?yt:n,r=e.symbol,i=void 0!==r&&r,o=e.mask,c=void 0===o?null:o,s=e.maskId,l=void 0===s?null:s,f=e.title,u=void 0===f?null:f,d=e.titleId,m=void 0===d?null:d,p=e.classes,h=void 0===p?[]:p,g=e.attributes,v=void 0===g?{}:g,b=e.styles,y=void 0===b?{}:b;if(t){var w=t.prefix,x=t.iconName,k=t.icon;return Ee(Q({type:"icon"},t),function(){return ze(),$.autoA11y&&(u?v["aria-labelledby"]="".concat($.replacementClass,"-title-").concat(m||kt()):(v["aria-hidden"]="true",v.focusable="false")),jt({icons:{main:be(k),mask:c?be(c.icon):{found:!1,width:null,height:null,icon:{}}},prefix:w,iconName:x,transform:Q({},yt,a),symbol:i,title:u,maskId:l,titleId:m,extra:{attributes:v,styles:y,classes:h}})})}},function(t){var e=1<arguments.length&&void 0!==arguments[1]?arguments[1]:{},n=(t||{}).icon?t:je(t||{}),a=e.mask;return a&&(a=(a||{}).icon?a:je(a||{})),Le(n,Q({},e,{mask:a}))}),Ye={noAuto:function(){$.autoReplaceSvg=!1,$.observeMutations=!1,oe&&oe.disconnect()},config:$,dom:_e,library:Re,parse:{transform:function(t){return le(t)}},findIconDefinition:je,icon:Te,text:function(t){var e=1<arguments.length&&void 0!==arguments[1]?arguments[1]:{},n=e.transform,a=void 0===n?yt:n,r=e.title,i=void 0===r?null:r,o=e.classes,c=void 0===o?[]:o,s=e.attributes,l=void 0===s?{}:s,f=e.styles,u=void 0===f?{}:f;return Ee({type:"text",content:t},function(){return ze(),Lt({content:t,transform:Q({},yt,a),title:i,extra:{attributes:l,styles:u,classes:["".concat($.familyPrefix,"-layers-text")].concat(d(c))}})})},counter:function(t){var e=1<arguments.length&&void 0!==arguments[1]?arguments[1]:{},n=e.title,a=void 0===n?null:n,r=e.classes,i=void 0===r?[]:r,o=e.attributes,c=void 0===o?{}:o,s=e.styles,l=void 0===s?{}:s;return Ee({type:"counter",content:t},function(){return ze(),function(t){var e=t.content,n=t.title,a=t.extra,r=Q({},a.attributes,n?{title:n}:{},{class:a.classes.join(" ")}),i=Pt(a.styles);0<i.length&&(r.style=i);var o=[];return o.push({tag:"span",attributes:r,children:[e]}),n&&o.push({tag:"span",attributes:{class:"sr-only"},children:[n]}),o}({content:t.toString(),title:a,extra:{attributes:c,styles:l,classes:["".concat($.familyPrefix,"-layers-counter")].concat(d(i))}})})},layer:function(t){var e=(1<arguments.length&&void 0!==arguments[1]?arguments[1]:{}).classes,n=void 0===e?[]:e;return Ee({type:"layer"},function(){ze();var e=[];return t(function(t){Array.isArray(t)?t.map(function(t){e=e.concat(t.abstract)}):e=e.concat(t.abstract)}),[{tag:"span",attributes:{class:["".concat($.familyPrefix,"-layers")].concat(d(n)).join(" ")},children:e}]})},toHtml:Zt},Fe=function(){var t=(0<arguments.length&&void 0!==arguments[0]?arguments[0]:{}).autoReplaceSvgRoot,e=void 0===t?v:t;(0<Object.keys(W.styles).length||$.autoFetchSvg)&&m&&$.autoReplaceSvg&&Ye.dom.i2svg({node:e})};!function(t){try{t()}catch(t){if(!M)throw t}}(function(){u&&(g.FontAwesome||(g.FontAwesome=Ye),B(function(){Fe(),ce({treeCallback:ke,nodeCallback:Ae,pseudoElementsCallback:Pe})})),W.hooks=Q({},W.hooks,{addPack:function(t,e){W.styles[t]=Q({},W.styles[t]||{},e),Bt(),Fe()},addShims:function(t){var e;(e=W.shims).push.apply(e,d(t)),Bt(),Fe()}})})}(); \ No newline at end of file
diff --git a/interface/js/lib/footable.min.js b/interface/js/lib/footable.min.js
new file mode 100644
index 0000000..53398d1
--- /dev/null
+++ b/interface/js/lib/footable.min.js
@@ -0,0 +1,10 @@
+/*
+* FooTable v3 - FooTable is a jQuery plugin that aims to make HTML tables on smaller devices look awesome.
+* @version 3.1.6
+* @link http://fooplugins.com
+* @copyright Steven Usher & Brad Vincent 2015
+* @license Released under the GPLv3 license.
+*/
+!function(a,b){window.console=window.console||{log:function(){},error:function(){}},a.fn.footable=function(a,c){return a=a||{},this.filter("table").each(function(d,e){b.init(e,a,c)})};var c={events:[]};b.__debug__=JSON.parse(localStorage.getItem("footable_debug"))||!1,b.__debug_options__=JSON.parse(localStorage.getItem("footable_debug_options"))||c,b.debug=function(d,e){return b.is["boolean"](d)?(b.__debug__=d,void(b.__debug__?(localStorage.setItem("footable_debug",JSON.stringify(b.__debug__)),b.__debug_options__=a.extend(!0,{},c,e||{}),b.is.hash(e)&&localStorage.setItem("footable_debug_options",JSON.stringify(b.__debug_options__))):(localStorage.removeItem("footable_debug"),localStorage.removeItem("footable_debug_options")))):b.__debug__},b.get=function(b){return a(b).first().data("__FooTable__")},b.init=function(a,c,d){var e=b.get(a);return e instanceof b.Table&&e.destroy(),new b.Table(a,c,d)},b.getRow=function(b){var c=a(b).closest("tr");return c.hasClass("footable-detail-row")&&(c=c.prev()),c.data("__FooTableRow__")}}(jQuery,FooTable=window.FooTable||{}),function(a){var b=function(){return!0};a.arr={},a.arr.each=function(b,c){if(a.is.array(b)&&a.is.fn(c))for(var d=0,e=b.length;e>d&&c(b[d],d)!==!1;d++);},a.arr.get=function(b,c){var d=[];if(!a.is.array(b))return d;if(!a.is.fn(c))return b;for(var e=0,f=b.length;f>e;e++)c(b[e],e)&&d.push(b[e]);return d},a.arr.any=function(c,d){if(!a.is.array(c))return!1;d=a.is.fn(d)?d:b;for(var e=0,f=c.length;f>e;e++)if(d(c[e],e))return!0;return!1},a.arr.contains=function(b,c){if(!a.is.array(b)||a.is.undef(c))return!1;for(var d=0,e=b.length;e>d;d++)if(b[d]==c)return!0;return!1},a.arr.first=function(c,d){if(!a.is.array(c))return null;d=a.is.fn(d)?d:b;for(var e=0,f=c.length;f>e;e++)if(d(c[e],e))return c[e];return null},a.arr.map=function(b,c){var d=[],e=null;if(!a.is.array(b)||!a.is.fn(c))return d;for(var f=0,g=b.length;g>f;f++)null!=(e=c(b[f],f))&&d.push(e);return d},a.arr.remove=function(b,c){var d=[],e=[];if(!a.is.array(b)||!a.is.fn(c))return e;for(var f=0,g=b.length;g>f;f++)c(b[f],f,e)&&(d.push(f),e.push(b[f]));for(d.sort(function(a,b){return b-a}),f=0,g=d.length;g>f;f++){var h=d[f]-f;b.splice(h,1)}return e},a.arr["delete"]=function(b,c){var d=-1,e=null;if(!a.is.array(b)||a.is.undef(c))return e;for(var f=0,g=b.length;g>f;f++)if(b[f]==c){d=f,e=b[f];break}return-1!=d&&b.splice(d,1),e},a.arr.replace=function(a,b,c){var d=a.indexOf(b);-1!==d&&(a[d]=c)}}(FooTable),function(a){a.is={},a.is.type=function(a,b){return typeof a===b},a.is.defined=function(a){return"undefined"!=typeof a},a.is.undef=function(a){return"undefined"==typeof a},a.is.array=function(a){return"[object Array]"===Object.prototype.toString.call(a)},a.is.date=function(a){return"[object Date]"===Object.prototype.toString.call(a)&&!isNaN(a.getTime())},a.is["boolean"]=function(a){return"[object Boolean]"===Object.prototype.toString.call(a)},a.is.string=function(a){return"[object String]"===Object.prototype.toString.call(a)},a.is.number=function(a){return"[object Number]"===Object.prototype.toString.call(a)&&!isNaN(a)},a.is.fn=function(b){return a.is.defined(window)&&b===window.alert||"[object Function]"===Object.prototype.toString.call(b)},a.is.error=function(a){return"[object Error]"===Object.prototype.toString.call(a)},a.is.object=function(a){return"[object Object]"===Object.prototype.toString.call(a)},a.is.hash=function(b){return a.is.object(b)&&b.constructor===Object&&!b.nodeType&&!b.setInterval},a.is.element=function(a){return"object"==typeof HTMLElement?a instanceof HTMLElement:a&&"object"==typeof a&&null!==a&&1===a.nodeType&&"string"==typeof a.nodeName},a.is.promise=function(b){return a.is.object(b)&&a.is.fn(b.then)&&a.is.fn(b.promise)},a.is.jq=function(b){return a.is.defined(window.jQuery)&&b instanceof jQuery&&b.length>0},a.is.moment=function(b){return a.is.defined(window.moment)&&a.is.object(b)&&a.is["boolean"](b._isAMomentObject)},a.is.emptyObject=function(b){if(!a.is.hash(b))return!1;for(var c in b)if(b.hasOwnProperty(c))return!1;return!0},a.is.emptyArray=function(b){return a.is.array(b)?0===b.length:!0},a.is.emptyString=function(b){return a.is.string(b)?0===b.length:!0}}(FooTable),function(a){a.str={},a.str.contains=function(b,c,d){return a.is.emptyString(b)||a.is.emptyString(c)?!1:c.length<=b.length&&-1!==(d?b.toUpperCase().indexOf(c.toUpperCase()):b.indexOf(c))},a.str.containsExact=function(b,c,d){return a.is.emptyString(b)||a.is.emptyString(c)||c.length>b.length?!1:new RegExp("\\b"+a.str.escapeRegExp(c)+"\\b",d?"i":"").test(b)},a.str.containsWord=function(b,c,d){if(a.is.emptyString(b)||a.is.emptyString(c)||b.length<c.length)return!1;for(var e=b.split(/\W/),f=0,g=e.length;g>f;f++)if(d?e[f].toUpperCase()==c.toUpperCase():e[f]==c)return!0;return!1},a.str.from=function(b,c){return a.is.emptyString(b)?b:a.str.contains(b,c)?b.substring(b.indexOf(c)+1):b},a.str.startsWith=function(b,c){return a.is.emptyString(b)?b==c:b.slice(0,c.length)==c},a.str.toCamelCase=function(b){return a.is.emptyString(b)?b:b.toUpperCase()===b?b.toLowerCase():b.replace(/^([A-Z])|[-\s_](\w)/g,function(b,c,d){return a.is.string(d)?d.toUpperCase():c.toLowerCase()})},a.str.random=function(b){return b=a.is.emptyString(b)?"":b,b+Math.random().toString(36).substr(2,9)},a.str.escapeRegExp=function(b){return a.is.emptyString(b)?b:b.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}}(FooTable),function(a){"use strict";function b(){}Object.create||(Object.create=function(){var b=function(){};return function(c){if(arguments.length>1)throw Error("Second argument not supported");if(!a.is.object(c))throw TypeError("Argument must be an object");b.prototype=c;var d=new b;return b.prototype=null,d}}());var c=/xyz/.test(function(){xyz})?/\b_super\b/:/.*/;b.__extend__=function(b,d,e,f){b[d]=a.is.fn(f)&&c.test(e)?function(a,b){return function(){var a,c;return a=this._super,this._super=f,c=b.apply(this,arguments),this._super=a,c}}(d,e):e},b.extend=function(d,e){function f(b,d,e,f){b[d]=a.is.fn(f)&&c.test(e)?function(a,b,c){return function(){var a,d;return a=this._super,this._super=c,d=b.apply(this,arguments),this._super=a,d}}(d,e,f):e}var g=Array.prototype.slice.call(arguments);if(d=g.shift(),e=g.shift(),a.is.hash(d)){var h=Object.create(this.prototype),i=this.prototype;for(var j in d)"__ctor__"!==j&&f(h,j,d[j],i[j]);var k=a.is.fn(h.__ctor__)?h.__ctor__:function(){if(!a.is.fn(this.construct))throw new SyntaxError('FooTable class objects must be constructed with the "new" keyword.');this.construct.apply(this,arguments)};return h.construct=a.is.fn(h.construct)?h.construct:function(){},k.prototype=h,h.constructor=k,k.extend=b.extend,k}a.is.string(d)&&a.is.fn(e)&&f(this.prototype,d,e,this.prototype[d])},a.Class=b,a.ClassFactory=a.Class.extend({construct:function(){this.registered={}},contains:function(b){return a.is.defined(this.registered[b])},names:function(){var a,b=[];for(a in this.registered)this.registered.hasOwnProperty(a)&&b.push(a);return b},register:function(b,c,d){if(a.is.string(b)&&a.is.fn(c)){var e=this.registered[b];this.registered[b]={name:b,klass:c,priority:a.is.number(d)?d:a.is.defined(e)?e.priority:0}}},load:function(b,c,d){var e,f,g=this,h=Array.prototype.slice.call(arguments),i=[],j=[];b=h.shift()||{};for(e in g.registered)if(g.registered.hasOwnProperty(e)){var k=g.registered[e];b.hasOwnProperty(e)&&(f=b[e],a.is.string(f)&&(f=a.getFnPointer(b[e])),a.is.fn(f)&&(k={name:e,klass:f,priority:g.registered[e].priority})),i.push(k)}for(e in b)b.hasOwnProperty(e)&&!g.registered.hasOwnProperty(e)&&(f=b[e],a.is.string(f)&&(f=a.getFnPointer(b[e])),a.is.fn(f)&&i.push({name:e,klass:f,priority:0}));return i.sort(function(a,b){return b.priority-a.priority}),a.arr.each(i,function(b){a.is.fn(b.klass)&&j.push(g._make(b.klass,h))}),j},make:function(b,c,d){var e,f=this,g=Array.prototype.slice.call(arguments);return b=g.shift(),e=f.registered[b],a.is.fn(e.klass)?f._make(e.klass,g):null},_make:function(a,b){function c(){return a.apply(this,b)}return c.prototype=a.prototype,new c}})}(FooTable),function(a,b){b.css2json=function(c){if(b.is.emptyString(c))return{};for(var d,e,f,g={},h=c.split(";"),i=0,j=h.length;j>i;i++)b.is.emptyString(h[i])||(d=h[i].split(":"),b.is.emptyString(d[0])||b.is.emptyString(d[1])||(e=b.str.toCamelCase(a.trim(d[0])),f=a.trim(d[1]),g[e]=f));return g},b.getFnPointer=function(a){if(b.is.emptyString(a))return null;var c=window,d=a.split(".");return b.arr.each(d,function(a){c[a]&&(c=c[a])}),b.is.fn(c)?c:null},b.checkFnValue=function(a,c,d){function e(a,c,d){return b.is.fn(c)?function(){return c.apply(a,arguments)}:d}return d=b.is.fn(d)?d:null,b.is.fn(c)?e(a,c,d):b.is.type(c,"string")?e(a,b.getFnPointer(c),d):d}}(jQuery,FooTable),function(a,b){b.Cell=b.Class.extend({construct:function(a,b,c,d){this.ft=a,this.row=b,this.column=c,this.created=!1,this.define(d)},define:function(c){this.$el=b.is.element(c)||b.is.jq(c)?a(c):null,this.$detail=null;var d=b.is.hash(c)&&b.is.hash(c.options)&&b.is.defined(c.value);this.value=this.column.parser.call(this.column,b.is.jq(this.$el)?this.$el:d?c.value:c,this.ft.o),this.o=a.extend(!0,{classes:null,style:null},d?c.options:{}),this.classes=b.is.jq(this.$el)&&this.$el.attr("class")?this.$el.attr("class").match(/\S+/g):b.is.array(this.o.classes)?this.o.classes:b.is.string(this.o.classes)?this.o.classes.match(/\S+/g):[],this.style=b.is.jq(this.$el)&&this.$el.attr("style")?b.css2json(this.$el.attr("style")):b.is.hash(this.o.style)?this.o.style:b.is.string(this.o.style)?b.css2json(this.o.style):{}},$create:function(){this.created||((this.$el=b.is.jq(this.$el)?this.$el:a("<td/>")).data("value",this.value).contents().detach().end().append(this.format(this.value)),this._setClasses(this.$el),this._setStyle(this.$el),this.$detail=a("<tr/>").addClass(this.row.classes.join(" ")).data("__FooTableCell__",this).append(a("<th/>")).append(a("<td/>")),this.created=!0)},collapse:function(){this.created&&(this.$detail.children("th").html(this.column.title),this.$el.clone().attr("id",this.$el.attr("id")?this.$el.attr("id")+"-detail":void 0).css("display","table-cell").html("").append(this.$el.contents().detach()).replaceAll(this.$detail.children("td").first()),b.is.jq(this.$detail.parent())||this.$detail.appendTo(this.row.$details.find(".footable-details > tbody")))},restore:function(){if(this.created){if(b.is.jq(this.$detail.parent())){var a=this.$detail.children("td").first();this.$el.attr("class",a.attr("class")).attr("style",a.attr("style")).css("display",this.column.hidden||!this.column.visible?"none":"table-cell").append(a.contents().detach())}this.$detail.detach()}},parse:function(){return this.column.parser.call(this.column,this.$el,this.ft.o)},format:function(a){return this.column.formatter.call(this.column,a,this.ft.o,this.row.value)},val:function(c,d,e){if(b.is.undef(c))return this.value;var f=this,g=b.is.hash(c)&&b.is.hash(c.options)&&b.is.defined(c.value);if(this.o=a.extend(!0,{classes:f.classes,style:f.style},g?c.options:{}),this.value=g?c.value:c,this.classes=b.is.array(this.o.classes)?this.o.classes:b.is.string(this.o.classes)?this.o.classes.match(/\S+/g):[],this.style=b.is.hash(this.o.style)?this.o.style:b.is.string(this.o.style)?b.css2json(this.o.style):{},e=b.is["boolean"](e)?e:!0,this.created&&e){this.$el.data("value",this.value).empty();var h=this.$detail.children("td").first().empty(),i=b.is.jq(this.$detail.parent())?h:this.$el;i.append(this.format(this.value)),this._setClasses(i),this._setStyle(i),(b.is["boolean"](d)?d:!0)&&this.row.draw()}},_setClasses:function(a){var c=!b.is.emptyArray(this.column.classes),d=!b.is.emptyArray(this.classes),e=null;a.removeAttr("class"),(c||d)&&(c&&d?e=this.classes.concat(this.column.classes).join(" "):c?e=this.column.classes.join(" "):d&&(e=this.classes.join(" ")),b.is.emptyString(e)||a.addClass(e))},_setStyle:function(c){var d=!b.is.emptyObject(this.column.style),e=!b.is.emptyObject(this.style),f=null;c.removeAttr("style"),(d||e)&&(d&&e?f=a.extend({},this.column.style,this.style):d?f=this.column.style:e&&(f=this.style),b.is.hash(f)&&c.css(f))}})}(jQuery,FooTable),function(a,b){b.Column=b.Class.extend({construct:function(a,c,d){this.ft=a,this.type=b.is.emptyString(d)?"text":d,this.virtual=b.is["boolean"](c.virtual)?c.virtual:!1,this.$el=b.is.jq(c.$el)?c.$el:null,this.index=b.is.number(c.index)?c.index:-1,this.internal=!1,this.define(c),this.$create()},define:function(a){this.hidden=b.is["boolean"](a.hidden)?a.hidden:!1,this.visible=b.is["boolean"](a.visible)?a.visible:!0,this.name=b.is.string(a.name)?a.name:null,null==this.name&&(this.name="col"+(a.index+1)),this.title=b.is.string(a.title)?a.title:null,!this.virtual&&null==this.title&&b.is.jq(this.$el)&&(this.title=this.$el.html()),null==this.title&&(this.title="Column "+(a.index+1)),this.style=b.is.hash(a.style)?a.style:b.is.string(a.style)?b.css2json(a.style):{},this.classes=b.is.array(a.classes)?a.classes:b.is.string(a.classes)?a.classes.match(/\S+/g):[],this.parser=b.checkFnValue(this,a.parser,this.parser),this.formatter=b.checkFnValue(this,a.formatter,this.formatter)},$create:function(){(this.$el=!this.virtual&&b.is.jq(this.$el)?this.$el:a("<th/>")).html(this.title).addClass(this.classes.join(" ")).css(this.style)},parser:function(c){if(b.is.element(c)||b.is.jq(c)){var d=a(c).data("value");return b.is.defined(d)?d:a(c).html()}return b.is.defined(c)&&null!=c?c+"":null},formatter:function(a,b,c){return null==a?"":a},createCell:function(a){var c=b.is.jq(a.$el)?a.$el.children("td,th").get(this.index):null,d=b.is.hash(a.value)?a.value[this.name]:null;return new b.Cell(this.ft,a,this,c||d)}}),b.columns=new b.ClassFactory,b.columns.register("text",b.Column)}(jQuery,FooTable),function(a,b){b.Component=b.Class.extend({construct:function(a,c){if(!(a instanceof b.Table))throw new TypeError("The instance parameter must be an instance of FooTable.Table.");this.ft=a,this.enabled=b.is["boolean"](c)?c:!1},preinit:function(a){},init:function(){},destroy:function(){},predraw:function(){},draw:function(){},postdraw:function(){}}),b.components=new b.ClassFactory}(jQuery,FooTable),function(a,b){b.Defaults=function(){this.stopPropagation=!1,this.on=null},b.defaults=new b.Defaults}(jQuery,FooTable),function(a,b){b.Row=b.Class.extend({construct:function(a,b,c){this.ft=a,this.columns=b,this.created=!1,this.define(c)},define:function(c){this.$el=b.is.element(c)||b.is.jq(c)?a(c):null,this.$toggle=a("<span/>",{"class":"footable-toggle fooicon fooicon-plus"});var d=b.is.hash(c),e=d&&b.is.hash(c.options)&&b.is.hash(c.value);this.value=d?e?c.value:c:null,this.o=a.extend(!0,{expanded:!1,classes:null,style:null},e?c.options:{}),this.expanded=b.is.jq(this.$el)?this.$el.data("expanded")||this.o.expanded:this.o.expanded,this.classes=b.is.jq(this.$el)&&this.$el.attr("class")?this.$el.attr("class").match(/\S+/g):b.is.array(this.o.classes)?this.o.classes:b.is.string(this.o.classes)?this.o.classes.match(/\S+/g):[],this.style=b.is.jq(this.$el)&&this.$el.attr("style")?b.css2json(this.$el.attr("style")):b.is.hash(this.o.style)?this.o.style:b.is.string(this.o.style)?b.css2json(this.o.style):{},this.cells=this.createCells();var f=this;f.value={},b.arr.each(f.cells,function(a){f.value[a.column.name]=a.val()})},$create:function(){if(!this.created){(this.$el=b.is.jq(this.$el)?this.$el:a("<tr/>")).data("__FooTableRow__",this),this._setClasses(this.$el),this._setStyle(this.$el),"last"==this.ft.rows.toggleColumn&&this.$toggle.addClass("last-column"),this.$details=a("<tr/>",{"class":"footable-detail-row"}).append(a("<td/>",{colspan:this.ft.columns.visibleColspan}).append(a("<table/>",{"class":"footable-details "+this.ft.classes.join(" ")}).append("<tbody/>")));var c=this;b.arr.each(c.cells,function(a){a.created||a.$create(),c.$el.append(a.$el)}),c.$el.off("click.ft.row").on("click.ft.row",{self:c},c._onToggle),this.created=!0}},createCells:function(){var a=this;return b.arr.map(a.columns,function(b){return b.createCell(a)})},val:function(c,d,e){var f=this;if(!b.is.hash(c))return b.is.hash(this.value)&&!b.is.emptyObject(this.value)||(this.value={},b.arr.each(this.cells,function(a){a.column.internal||(f.value[a.column.name]=a.val())})),this.value;this.collapse(!1);var g=b.is.hash(c),h=g&&b.is.hash(c.options)&&b.is.hash(c.value);if(this.o=a.extend(!0,{expanded:f.expanded,classes:f.classes,style:f.style},h?c.options:{}),this.expanded=this.o.expanded,this.classes=b.is.array(this.o.classes)?this.o.classes:b.is.string(this.o.classes)?this.o.classes.match(/\S+/g):[],this.style=b.is.hash(this.o.style)?this.o.style:b.is.string(this.o.style)?b.css2json(this.o.style):{},g)if(h&&(c=c.value),b.is.hash(this.value))for(var i in c)c.hasOwnProperty(i)&&(this.value[i]=c[i]);else this.value=c;else this.value=null;e=b.is["boolean"](e)?e:!0,b.arr.each(this.cells,function(a){!a.column.internal&&b.is.defined(f.value[a.column.name])&&a.val(f.value[a.column.name],!1,e)}),this.created&&e&&(this._setClasses(this.$el),this._setStyle(this.$el),(b.is["boolean"](d)?d:!0)&&this.draw())},_setClasses:function(a){var c=!b.is.emptyArray(this.classes),d=null;a.removeAttr("class"),c&&(d=this.classes.join(" "),b.is.emptyString(d)||a.addClass(d))},_setStyle:function(a){var c=!b.is.emptyObject(this.style),d=null;a.removeAttr("style"),c&&(d=this.style,b.is.hash(d)&&a.css(d))},expand:function(){if(this.created){var a=this;a.ft.raise("expand.ft.row",[a]).then(function(){a.__hidden__=b.arr.map(a.cells,function(a){return a.column.hidden&&a.column.visible?a:null}),a.__hidden__.length>0&&(a.$details.insertAfter(a.$el).children("td").first().attr("colspan",a.ft.columns.visibleColspan),b.arr.each(a.__hidden__,function(a){a.collapse()})),a.$el.attr("data-expanded",!0),a.$toggle.removeClass("fooicon-plus").addClass("fooicon-minus"),a.expanded=!0,a.ft.raise("expanded.ft.row",[a])})}},collapse:function(a){if(this.created){var c=this;c.ft.raise("collapse.ft.row",[c]).then(function(){b.arr.each(c.__hidden__,function(a){a.restore()}),c.$details.detach(),c.$el.removeAttr("data-expanded"),c.$toggle.removeClass("fooicon-minus").addClass("fooicon-plus"),(b.is["boolean"](a)?a:!0)&&(c.expanded=!1),c.ft.raise("collapsed.ft.row",[c])})}},predraw:function(a){this.created&&(this.expanded&&this.collapse(!1),this.$toggle.detach(),a=b.is["boolean"](a)?a:!0,a&&this.$el.detach())},draw:function(a){this.created||this.$create(),b.is.jq(a)&&a.append(this.$el);var c=this;b.arr.each(c.cells,function(a){a.$el.css("display",a.column.hidden||!a.column.visible?"none":"table-cell"),c.ft.rows.showToggle&&c.ft.columns.hasHidden&&("first"==c.ft.rows.toggleColumn&&a.column.index==c.ft.columns.firstVisibleIndex||"last"==c.ft.rows.toggleColumn&&a.column.index==c.ft.columns.lastVisibleIndex)&&a.$el.prepend(c.$toggle),a.$el.add(a.column.$el).removeClass("footable-first-visible footable-last-visible"),a.column.index==c.ft.columns.firstVisibleIndex&&a.$el.add(a.column.$el).addClass("footable-first-visible"),a.column.index==c.ft.columns.lastVisibleIndex&&a.$el.add(a.column.$el).addClass("footable-last-visible")}),this.expanded&&this.expand()},toggle:function(){this.created&&this.ft.columns.hasHidden&&(this.expanded?this.collapse():this.expand())},_onToggle:function(b){var c=b.data.self;a(b.target).is(c.ft.rows.toggleSelector)&&c.toggle()}})}(jQuery,FooTable),function(a,b){b.instances=[],b.Table=b.Class.extend({construct:function(c,d,e){this._resizeTimeout=null,this.id=b.instances.push(this),this.initialized=!1,this.$el=(b.is.jq(c)?c:a(c)).first(),this.$loader=a("<div/>",{"class":"footable-loader"}).append(a("<span/>",{"class":"fooicon fooicon-loader"})),this.o=a.extend(!0,{},b.defaults,d),this.data=this.$el.data()||{},this.classes=[],this.components=b.components.load(b.is.hash(this.data.components)?this.data.components:this.o.components,this),this.breakpoints=this.use(FooTable.Breakpoints),this.columns=this.use(FooTable.Columns),this.rows=this.use(FooTable.Rows),this._construct(e)},_construct:function(a){var c=this;return this._preinit().then(function(){return c._init().then(function(){return c.raise("ready.ft.table").then(function(){b.is.fn(a)&&a.call(c,c)})})}).always(function(a){c.$el.show(),b.is.error(a)&&console.error("FooTable: unhandled error thrown during initialization.",a)})},_preinit:function(){var a=this;return this.raise("preinit.ft.table",[a.data]).then(function(){var c=(a.$el.attr("class")||"").match(/\S+/g)||[];a.o.ajax=b.checkFnValue(a,a.data.ajax,a.o.ajax),a.o.stopPropagation=b.is["boolean"](a.data.stopPropagation)?a.data.stopPropagation:a.o.stopPropagation;for(var d=0,e=c.length;e>d;d++)b.str.startsWith(c[d],"footable")||a.classes.push(c[d]);return a.$el.hide().after(a.$loader),a.execute(!1,!1,"preinit",a.data)})},_init:function(){var c=this;return c.raise("init.ft.table").then(function(){var d=c.$el.children("thead"),e=c.$el.children("tbody"),f=c.$el.children("tfoot");return c.$el.addClass("footable footable-"+c.id),b.is.hash(c.o.on)&&c.$el.on(c.o.on),0==f.length&&c.$el.append(f=a("<tfoot/>")),0==e.length&&c.$el.append("<tbody/>"),0==d.length&&c.$el.prepend(d=a("<thead/>")),c.execute(!1,!0,"init").then(function(){return c.$el.data("__FooTable__",c),0==f.children("tr").length&&f.remove(),0==d.children("tr").length&&d.remove(),c.raise("postinit.ft.table").then(function(){return c.draw()}).always(function(){a(window).off("resize.ft"+c.id,c._onWindowResize).on("resize.ft"+c.id,{self:c},c._onWindowResize),c.initialized=!0})})})},destroy:function(){var c=this;return c.raise("destroy.ft.table").then(function(){return c.execute(!0,!0,"destroy").then(function(){c.$el.removeData("__FooTable__").removeClass("footable-"+c.id),b.is.hash(c.o.on)&&c.$el.off(c.o.on),a(window).off("resize.ft"+c.id,c._onWindowResize),c.initialized=!1,b.instances[c.id]=null})}).fail(function(a){b.is.error(a)&&console.error("FooTable: unhandled error thrown while destroying the plugin.",a)})},raise:function(c,d){var e=this,f=b.__debug__&&(b.is.emptyArray(b.__debug_options__.events)||b.arr.any(b.__debug_options__.events,function(a){return b.str.contains(c,a)}));return d=d||[],d.unshift(this),a.Deferred(function(b){var g=a.Event(c);1==e.o.stopPropagation&&e.$el.one(c,function(a){a.stopPropagation()}),f&&console.log("FooTable:"+c+": ",d),e.$el.trigger(g,d),g.isDefaultPrevented()?(f&&console.log('FooTable: default prevented for the "'+c+'" event.'),b.reject(g)):b.resolve(g)})},use:function(a){for(var b=0,c=this.components.length;c>b;b++)if(this.components[b]instanceof a)return this.components[b];return null},draw:function(){var a=this,c=a.$el.clone().insertBefore(a.$el);return a.$el.detach(),a.execute(!1,!0,"predraw").then(function(){return a.raise("predraw.ft.table").then(function(){return a.execute(!1,!0,"draw").then(function(){return a.raise("draw.ft.table").then(function(){return a.execute(!1,!0,"postdraw").then(function(){return a.raise("postdraw.ft.table")})})})})}).fail(function(a){b.is.error(a)&&console.error("FooTable: unhandled error thrown during a draw operation.",a)}).always(function(){c.replaceWith(a.$el),a.$loader.remove()})},execute:function(a,c,d,e,f){var g=this,h=Array.prototype.slice.call(arguments);a=h.shift(),c=h.shift();var i=c?b.arr.get(g.components,function(a){return a.enabled}):g.components.slice(0);return h.unshift(a?i.reverse():i),g._execute.apply(g,h)},_execute:function(c,d,e,f){if(!c||!c.length)return a.when();var g,h=this,i=Array.prototype.slice.call(arguments);return c=i.shift(),d=i.shift(),g=c.shift(),b.is.fn(g[d])?a.Deferred(function(a){try{var c=g[d].apply(g,i);if(b.is.promise(c))return c.then(a.resolve,a.reject);a.resolve(c)}catch(e){a.reject(e)}}).then(function(){return h._execute.apply(h,[c,d].concat(i))}):h._execute.apply(h,[c,d].concat(i))},_onWindowResize:function(a){var b=a.data.self;null!=b._resizeTimeout&&clearTimeout(b._resizeTimeout),b._resizeTimeout=setTimeout(function(){b._resizeTimeout=null,b.raise("resize.ft.table").then(function(){b.breakpoints.check()})},300)}})}(jQuery,FooTable),function(a,b){b.ArrayColumn=b.Column.extend({construct:function(a,b){this._super(a,b,"array")},parser:function(c){if(b.is.element(c)||b.is.jq(c)){var d=a(c),e=d.data("value");if(b.is.array(e))return e;e=d.html();try{e=JSON.parse(e)}catch(f){e=null}return b.is.array(e)?e:null}return b.is.array(c)?c:null},formatter:function(a,c,d){return b.is.array(a)?JSON.stringify(a):""}}),b.columns.register("array",b.ArrayColumn)}(jQuery,FooTable),function(a,b){b.is.undef(window.moment)||(b.DateColumn=b.Column.extend({construct:function(a,c){this._super(a,c,"date"),this.formatString=b.is.string(c.formatString)?c.formatString:"MM-DD-YYYY"},parser:function(c){if(b.is.element(c)||b.is.jq(c)){var d=a(c).data("value");c=b.is.defined(d)?d:a(c).text(),b.is.string(c)&&(c=isNaN(c)?c:+c)}if(b.is.date(c))return moment(c);if(b.is.object(c)&&b.is["boolean"](c._isAMomentObject))return c;if(b.is.string(c)){if(isNaN(c))return moment(c,this.formatString);c=+c}return b.is.number(c)?moment(c):null},formatter:function(a,c,d){return b.is.object(a)&&b.is["boolean"](a._isAMomentObject)&&a.isValid()?a.format(this.formatString):""},filterValue:function(c){if((b.is.element(c)||b.is.jq(c))&&(c=a(c).data("filterValue")||a(c).text()),b.is.hash(c)&&b.is.hash(c.options)&&(b.is.string(c.options.filterValue)&&(c=c.options.filterValue),b.is.defined(c.value)&&(c=c.value)),b.is.object(c)&&b.is["boolean"](c._isAMomentObject))return c.format(this.formatString);if(b.is.string(c)){if(isNaN(c))return c;c=+c}return b.is.number(c)||b.is.date(c)?moment(c).format(this.formatString):b.is.defined(c)&&null!=c?c+"":""}}),b.columns.register("date",b.DateColumn))}(jQuery,FooTable),function(a,b){b.HTMLColumn=b.Column.extend({construct:function(a,b){this._super(a,b,"html")},parser:function(c){if(b.is.string(c)&&(c=a(a.trim(c))),b.is.element(c)&&(c=a(c)),b.is.jq(c)){var d=c.prop("tagName").toLowerCase();if("td"==d||"th"==d){var e=c.data("value");return b.is.defined(e)?e:c.contents()}return c}return null}}),b.columns.register("html",b.HTMLColumn)}(jQuery,FooTable),function(a,b){b.NumberColumn=b.Column.extend({construct:function(a,c){this._super(a,c,"number"),this.decimalSeparator=b.is.string(c.decimalSeparator)?c.decimalSeparator:".",this.thousandSeparator=b.is.string(c.thousandSeparator)?c.thousandSeparator:",",this.decimalSeparatorRegex=new RegExp(b.str.escapeRegExp(this.decimalSeparator),"g"),this.thousandSeparatorRegex=new RegExp(b.str.escapeRegExp(this.thousandSeparator),"g"),this.cleanRegex=new RegExp("[^-0-9"+b.str.escapeRegExp(this.decimalSeparator)+"]","g")},parser:function(c){if(b.is.element(c)||b.is.jq(c)){var d=a(c).data("value");c=b.is.defined(d)?d:a(c).text().replace(this.cleanRegex,"")}return b.is.string(c)&&(c=c.replace(this.thousandSeparatorRegex,"").replace(this.decimalSeparatorRegex,"."),c=parseFloat(c)),b.is.number(c)?c:null},formatter:function(a,b,c){if(null==a)return"";var d=(a+"").split(".");return 2==d.length&&d[0].length>3&&(d[0]=d[0].replace(/\B(?=(?:\d{3})+(?!\d))/g,this.thousandSeparator)),d.join(this.decimalSeparator)}}),b.columns.register("number",b.NumberColumn)}(jQuery,FooTable),function(a,b){b.ObjectColumn=b.Column.extend({construct:function(a,b){this._super(a,b,"object")},parser:function(c){if(b.is.element(c)||b.is.jq(c)){var d=a(c),e=d.data("value");if(b.is.object(e))return e;e=d.html();try{e=JSON.parse(e)}catch(f){e=null}return b.is.object(e)?e:null}return b.is.object(c)?c:null},formatter:function(a,c,d){return b.is.object(a)?JSON.stringify(a):""}}),b.columns.register("object",b.ObjectColumn)}(jQuery,FooTable),function(a,b){b.Breakpoint=b.Class.extend({construct:function(a,b){this.name=a,this.width=b}})}(jQuery,FooTable),function(a,b){b.Breakpoints=b.Component.extend({construct:function(a){this._super(a,!0),this.o=a.o,this.current=null,this.array=[],this.cascade=this.o.cascade,this.useParentWidth=this.o.useParentWidth,this.hidden=null,this._classNames="",this.getWidth=b.checkFnValue(this,this.o.getWidth,this.getWidth)},preinit:function(a){var c=this;return this.ft.raise("preinit.ft.breakpoints",[a]).then(function(){c.cascade=b.is["boolean"](a.cascade)?a.cascade:c.cascade,c.o.breakpoints=b.is.hash(a.breakpoints)?a.breakpoints:c.o.breakpoints,c.getWidth=b.checkFnValue(c,a.getWidth,c.getWidth),null==c.o.breakpoints&&(c.o.breakpoints={xs:480,sm:768,md:992,lg:1200});for(var d in c.o.breakpoints)c.o.breakpoints.hasOwnProperty(d)&&(c.array.push(new b.Breakpoint(d,c.o.breakpoints[d])),c._classNames+="breakpoint-"+d+" ");c.array.sort(function(a,b){return b.width-a.width})})},init:function(){var a=this;return this.ft.raise("init.ft.breakpoints").then(function(){a.current=a.get()})},draw:function(){this.ft.$el.removeClass(this._classNames).addClass("breakpoint-"+this.current.name)},calculate:function(){for(var a,c=this,d=null,e=[],f=null,g=c.getWidth(),h=0,i=c.array.length;i>h;h++)a=c.array[h],(!d&&h==i-1||g>=a.width&&(f instanceof b.Breakpoint?g<f.width:!0))&&(d=a),d||e.push(a.name),f=a;return e.push(d.name),c.hidden=e.join(" "),d},visible:function(a){if(b.is.emptyString(a))return!0;if("all"===a)return!1;for(var c=a.split(" "),d=0,e=c.length;e>d;d++)if(this.cascade?b.str.containsWord(this.hidden,c[d]):c[d]==this.current.name)return!1;return!0},check:function(){var a=this,c=a.get();c instanceof b.Breakpoint&&c!=a.current&&a.ft.raise("before.ft.breakpoints",[a.current,c]).then(function(){var b=a.current;return a.current=c,a.ft.draw().then(function(){a.ft.raise("after.ft.breakpoints",[a.current,b])})})},get:function(a){return b.is.undef(a)?this.calculate():a instanceof b.Breakpoint?a:b.is.string(a)?b.arr.first(this.array,function(b){return b.name==a}):b.is.number(a)&&a>=0&&a<this.array.length?this.array[a]:null},getWidth:function(){return b.is.fn(this.o.getWidth)?this.o.getWidth(this.ft):1==this.useParentWidth?this.getParentWidth():this.getViewportWidth()},getParentWidth:function(){return this.ft.$el.parent().width()},getViewportWidth:function(){return Math.max(document.documentElement.clientWidth,window.innerWidth,0)}}),b.components.register("breakpoints",b.Breakpoints,1e3)}(jQuery,FooTable),function(a){a.Column.prototype.breakpoints=null,a.Column.prototype.__breakpoints_define__=function(b){this.breakpoints=a.is.emptyString(b.breakpoints)?null:b.breakpoints},a.Column.extend("define",function(a){this._super(a),this.__breakpoints_define__(a)})}(FooTable),function(a){a.Defaults.prototype.breakpoints=null,a.Defaults.prototype.cascade=!1,a.Defaults.prototype.useParentWidth=!1,a.Defaults.prototype.getWidth=null}(FooTable),function(a,b){b.Columns=b.Component.extend({construct:function(a){this._super(a,!0),this.o=a.o,this.array=[],this.$header=null,this.showHeader=a.o.showHeader,this._fromHTML=b.is.emptyArray(a.o.columns)&&!b.is.promise(a.o.columns)},parse:function(c){var d=this;return a.Deferred(function(c){function e(c,d){var e=[];if(0==c.length||0==d.length)e=c.concat(d);else{var f=0;b.arr.each(c.concat(d),function(a){a.index>f&&(f=a.index)}),f++;for(var g,h,i=0;f>i;i++)g={},b.arr.each(c,function(a){return a.index==i?(g=a,!1):void 0}),h={},b.arr.each(d,function(a){return a.index==i?(h=a,!1):void 0}),e.push(a.extend(!0,{},g,h))}return e}var f,g,h=[],i=[],j=d.ft.$el.find("tr.footable-header, thead > tr:last:has([data-breakpoints]), tbody > tr:first:has([data-breakpoints]), thead > tr:last, tbody > tr:first").first();if(j.length>0){var k=j.parent().is("tbody")&&j.children().length==j.children("td").length;k||(d.$header=j.addClass("footable-header")),j.children("td,th").each(function(b,c){f=a(c),g=f.data(),g.index=b,g.$el=f,g.virtual=k,i.push(g)}),k&&(d.showHeader=!1)}b.is.array(d.o.columns)&&!b.is.emptyArray(d.o.columns)?(b.arr.each(d.o.columns,function(a,b){a.index=b,h.push(a)}),d.parseFinalize(c,e(h,i))):b.is.promise(d.o.columns)?d.o.columns.then(function(a){b.arr.each(a,function(a,b){a.index=b,h.push(a)}),d.parseFinalize(c,e(h,i))},function(a){c.reject(Error("Columns ajax request error: "+a.status+" ("+a.statusText+")"))}):d.parseFinalize(c,e(h,i))})},parseFinalize:function(a,c){var d,e=this,f=[];b.arr.each(c,function(a){(d=b.columns.contains(a.type)?b.columns.make(a.type,e.ft,a):new b.Column(e.ft,a))&&f.push(d)}),b.is.emptyArray(f)?a.reject(Error("No columns supplied.")):(f.sort(function(a,b){
+return a.index-b.index}),a.resolve(f))},preinit:function(a){var c=this;return c.ft.raise("preinit.ft.columns",[a]).then(function(){return c.parse(a).then(function(d){c.array=d,c.showHeader=b.is["boolean"](a.showHeader)?a.showHeader:c.showHeader})})},init:function(){var a=this;return this.ft.raise("init.ft.columns",[a.array]).then(function(){a.$create()})},destroy:function(){var a=this;this.ft.raise("destroy.ft.columns").then(function(){a._fromHTML||a.$header.remove()})},predraw:function(){var a=this,c=!0;a.visibleColspan=0,a.firstVisibleIndex=0,a.lastVisibleIndex=0,a.hasHidden=!1,b.arr.each(a.array,function(b){b.hidden=!a.ft.breakpoints.visible(b.breakpoints),!b.hidden&&b.visible&&(c&&(a.firstVisibleIndex=b.index,c=!1),a.lastVisibleIndex=b.index,a.visibleColspan++),b.hidden&&(a.hasHidden=!0)}),a.ft.$el.toggleClass("breakpoint",a.hasHidden)},draw:function(){b.arr.each(this.array,function(a){a.$el.css("display",a.hidden||!a.visible?"none":"table-cell")}),!this.showHeader&&b.is.jq(this.$header.parent())&&this.$header.detach()},$create:function(){var c=this;c.$header=b.is.jq(c.$header)?c.$header:a("<tr/>",{"class":"footable-header"}),c.$header.children("th,td").detach(),b.arr.each(c.array,function(a){c.$header.append(a.$el)}),c.showHeader&&!b.is.jq(c.$header.parent())&&c.ft.$el.children("thead").append(c.$header)},get:function(a){return a instanceof b.Column?a:b.is.string(a)?b.arr.first(this.array,function(b){return b.name==a}):b.is.number(a)?b.arr.first(this.array,function(b){return b.index==a}):b.is.fn(a)?b.arr.get(this.array,a):null},ensure:function(a){var c=this,d=[];return b.is.array(a)?(b.arr.each(a,function(a){d.push(c.get(a))}),d):d}}),b.components.register("columns",b.Columns,900)}(jQuery,FooTable),function(a){a.Defaults.prototype.columns=[],a.Defaults.prototype.showHeader=!0}(FooTable),function(a,b){b.Rows=b.Component.extend({construct:function(a){this._super(a,!0),this.o=a.o,this.array=[],this.all=[],this.showToggle=a.o.showToggle,this.toggleSelector=a.o.toggleSelector,this.toggleColumn=a.o.toggleColumn,this.emptyString=a.o.empty,this.expandFirst=a.o.expandFirst,this.expandAll=a.o.expandAll,this.$empty=null,this._fromHTML=b.is.emptyArray(a.o.rows)&&!b.is.promise(a.o.rows)},parse:function(){var c=this;return a.Deferred(function(a){var d=c.ft.$el.children("tbody").children("tr");b.is.array(c.o.rows)&&c.o.rows.length>0?c.parseFinalize(a,c.o.rows):b.is.promise(c.o.rows)?c.o.rows.then(function(b){c.parseFinalize(a,b)},function(b){a.reject(Error("Rows ajax request error: "+b.status+" ("+b.statusText+")"))}):b.is.jq(d)?(c.parseFinalize(a,d),d.detach()):c.parseFinalize(a,[])})},parseFinalize:function(c,d){var e=this,f=a.map(d,function(a){return new b.Row(e.ft,e.ft.columns.array,a)});c.resolve(f)},preinit:function(a){var c=this;return c.ft.raise("preinit.ft.rows",[a]).then(function(){return c.parse().then(function(d){c.all=d,c.array=c.all.slice(0),c.showToggle=b.is["boolean"](a.showToggle)?a.showToggle:c.showToggle,c.toggleSelector=b.is.string(a.toggleSelector)?a.toggleSelector:c.toggleSelector,c.toggleColumn=b.is.string(a.toggleColumn)?a.toggleColumn:c.toggleColumn,"first"!=c.toggleColumn&&"last"!=c.toggleColumn&&(c.toggleColumn="first"),c.emptyString=b.is.string(a.empty)?a.empty:c.emptyString,c.expandFirst=b.is["boolean"](a.expandFirst)?a.expandFirst:c.expandFirst,c.expandAll=b.is["boolean"](a.expandAll)?a.expandAll:c.expandAll})})},init:function(){var a=this;return a.ft.raise("init.ft.rows",[a.all]).then(function(){a.$create()})},destroy:function(){var a=this;this.ft.raise("destroy.ft.rows").then(function(){b.arr.each(a.array,function(b){b.predraw(!a._fromHTML)}),a.all=a.array=[]})},predraw:function(){b.arr.each(this.array,function(a){a.predraw()}),this.array=this.all.slice(0)},$create:function(){this.$empty=a("<tr/>",{"class":"footable-empty"}).append(a("<td/>").text(this.emptyString))},draw:function(){var a=this,c=a.ft.$el.children("tbody"),d=!0;a.array.length>0?(a.$empty.detach(),b.arr.each(a.array,function(b){(a.expandFirst&&d||a.expandAll)&&(b.expanded=!0,d=!1),b.draw(c)})):(a.$empty.children("td").attr("colspan",a.ft.columns.visibleColspan),c.append(a.$empty))},load:function(c,d){var e=this,f=a.map(c,function(a){return new b.Row(e.ft,e.ft.columns.array,a)});b.arr.each(this.array,function(a){a.predraw()}),this.all=(b.is["boolean"](d)?d:!1)?this.all.concat(f):f,this.array=this.all.slice(0),this.ft.draw()},expand:function(){b.arr.each(this.array,function(a){a.expand()})},collapse:function(){b.arr.each(this.array,function(a){a.collapse()})}}),b.components.register("rows",b.Rows,800)}(jQuery,FooTable),function(a){a.Defaults.prototype.rows=[],a.Defaults.prototype.empty="No results",a.Defaults.prototype.showToggle=!0,a.Defaults.prototype.toggleSelector="tr,td,.footable-toggle",a.Defaults.prototype.toggleColumn="first",a.Defaults.prototype.expandFirst=!1,a.Defaults.prototype.expandAll=!1}(FooTable),function(a){a.Table.prototype.loadRows=function(a,b){this.rows.load(a,b)}}(FooTable),function(a){a.Filter=a.Class.extend({construct:function(b,c,d,e,f,g,h){this.name=b,this.space=!a.is.string(e)||"OR"!=e&&"AND"!=e?"AND":e,this.connectors=a.is["boolean"](f)?f:!0,this.ignoreCase=a.is["boolean"](g)?g:!0,this.hidden=a.is["boolean"](h)?h:!1,this.query=c instanceof a.Query?c:new a.Query(c,this.space,this.connectors,this.ignoreCase),this.columns=d},match:function(b){return a.is.string(b)?(a.is.string(this.query)&&(this.query=new a.Query(this.query,this.space,this.connectors,this.ignoreCase)),this.query instanceof a.Query?this.query.match(b):!1):!1},matchRow:function(b){var c=this,d=a.arr.map(b.cells,function(b){return a.arr.contains(c.columns,b.column)?b.filterValue:null}).join(" ");return c.match(d)}})}(FooTable),function(a,b){b.Filtering=b.Component.extend({construct:function(a){this._super(a,a.o.filtering.enabled),this.filters=a.o.filtering.filters,this.delay=a.o.filtering.delay,this.min=a.o.filtering.min,this.space=a.o.filtering.space,this.connectors=a.o.filtering.connectors,this.ignoreCase=a.o.filtering.ignoreCase,this.exactMatch=a.o.filtering.exactMatch,this.placeholder=a.o.filtering.placeholder,this.dropdownTitle=a.o.filtering.dropdownTitle,this.position=a.o.filtering.position,this.focus=a.o.filtering.focus,this.container=a.o.filtering.container,this.$container=null,this.$row=null,this.$cell=null,this.$form=null,this.$dropdown=null,this.$input=null,this.$button=null,this._filterTimeout=null,this._exactRegExp=/^"(.*?)"$/},preinit:function(a){var c=this;return c.ft.raise("preinit.ft.filtering").then(function(){c.ft.$el.hasClass("footable-filtering")&&(c.enabled=!0),c.enabled=b.is["boolean"](a.filtering)?a.filtering:c.enabled,c.enabled&&(c.space=b.is.string(a.filterSpace)?a.filterSpace:c.space,c.min=b.is.number(a.filterMin)?a.filterMin:c.min,c.connectors=b.is["boolean"](a.filterConnectors)?a.filterConnectors:c.connectors,c.ignoreCase=b.is["boolean"](a.filterIgnoreCase)?a.filterIgnoreCase:c.ignoreCase,c.exactMatch=b.is["boolean"](a.filterExactMatch)?a.filterExactMatch:c.exactMatch,c.focus=b.is["boolean"](a.filterFocus)?a.filterFocus:c.focus,c.delay=b.is.number(a.filterDelay)?a.filterDelay:c.delay,c.placeholder=b.is.string(a.filterPlaceholder)?a.filterPlaceholder:c.placeholder,c.dropdownTitle=b.is.string(a.filterDropdownTitle)?a.filterDropdownTitle:c.dropdownTitle,c.container=b.is.string(a.filterContainer)?a.filterContainer:c.container,c.filters=b.is.array(a.filterFilters)?c.ensure(a.filterFilters):c.ensure(c.filters),c.ft.$el.hasClass("footable-filtering-left")&&(c.position="left"),c.ft.$el.hasClass("footable-filtering-center")&&(c.position="center"),c.ft.$el.hasClass("footable-filtering-right")&&(c.position="right"),c.position=b.is.string(a.filterPosition)?a.filterPosition:c.position)},function(){c.enabled=!1})},init:function(){var a=this;return a.ft.raise("init.ft.filtering").then(function(){a.$create()},function(){a.enabled=!1})},destroy:function(){var a=this;return a.ft.raise("destroy.ft.filtering").then(function(){a.ft.$el.removeClass("footable-filtering").find("thead > tr.footable-filtering").remove()})},$create:function(){var c,d=this,e=a("<div/>",{"class":"form-group footable-filtering-search"}).append(a("<label/>",{"class":"sr-only",text:"Search"})),f=a("<div/>",{"class":"input-group"}).appendTo(e),g=a("<div/>",{"class":"input-group-btn"}),h=a("<button/>",{type:"button","class":"btn btn-default dropdown-toggle"}).on("click",{self:d},d._onDropdownToggleClicked).append(a("<span/>",{"class":"caret"}));switch(d.position){case"left":c="footable-filtering-left";break;case"center":c="footable-filtering-center";break;default:c="footable-filtering-right"}d.ft.$el.addClass("footable-filtering").addClass(c),d.$container=null===d.container?a():a(d.container).first(),d.$container.length?d.$container.addClass("footable-filtering-external").addClass(c):(d.$row=a("<tr/>",{"class":"footable-filtering"}).prependTo(d.ft.$el.children("thead")),d.$cell=a("<th/>").attr("colspan",d.ft.columns.visibleColspan).appendTo(d.$row),d.$container=d.$cell),d.$form=a("<form/>",{"class":"form-inline"}).append(e).appendTo(d.$container),d.$input=a("<input/>",{type:"text","class":"form-control",placeholder:d.placeholder}),d.$button=a("<button/>",{type:"button","class":"btn btn-primary"}).on("click",{self:d},d._onSearchButtonClicked).append(a("<span/>",{"class":"fooicon fooicon-search"})),d.$dropdown=a("<ul/>",{"class":"dropdown-menu dropdown-menu-right"}),b.is.emptyString(d.dropdownTitle)||d.$dropdown.append(a("<li/>",{"class":"dropdown-header",text:d.dropdownTitle})),d.$dropdown.append(b.arr.map(d.ft.columns.array,function(b){return b.filterable?a("<li/>").append(a("<a/>",{"class":"checkbox"}).append(a("<label/>",{html:b.title}).prepend(a("<input/>",{type:"checkbox",checked:!0}).data("__FooTableColumn__",b)))):null})),d.delay>0&&(d.$input.on("keypress keyup paste",{self:d},d._onSearchInputChanged),d.$dropdown.on("click",'input[type="checkbox"]',{self:d},d._onSearchColumnClicked)),g.append(d.$button,h,d.$dropdown),f.append(d.$input,g)},predraw:function(){if(!b.is.emptyArray(this.filters)){var c=this;c.ft.rows.array=a.grep(c.ft.rows.array,function(a){return a.filtered(c.filters)})}},draw:function(){b.is.jq(this.$cell)&&this.$cell.attr("colspan",this.ft.columns.visibleColspan);var a=this.find("search");if(a instanceof b.Filter){var c=a.query.val();this.exactMatch&&this._exactRegExp.test(c)&&(c=c.replace(this._exactRegExp,"$1")),this.$input.val(c)}else this.$input.val(null);this.setButton(!b.arr.any(this.filters,function(a){return!a.hidden}))},addFilter:function(a,c,d,e,f,g,h){var i=this.createFilter(a,c,d,e,f,g,h);i instanceof b.Filter&&(this.removeFilter(i.name),this.filters.push(i))},removeFilter:function(a){b.arr.remove(this.filters,function(b){return b.name==a})},filter:function(a){var b=this;return b.filters=b.ensure(b.filters),b.ft.raise("before.ft.filtering",[b.filters]).then(function(){if(b.filters=b.ensure(b.filters),a)var c=b.$input.prop("selectionStart"),d=b.$input.prop("selectionEnd");return b.ft.draw().then(function(){a&&b.$input.focus().prop({selectionStart:c,selectionEnd:d}),b.ft.raise("after.ft.filtering",[b.filters])})})},clear:function(){return this.filters=b.arr.get(this.filters,function(a){return a.hidden}),this.filter(this.focus)},setButton:function(a){a?this.$button.children(".fooicon").removeClass("fooicon-remove").addClass("fooicon-search"):this.$button.children(".fooicon").removeClass("fooicon-search").addClass("fooicon-remove")},find:function(a){return b.arr.first(this.filters,function(b){return b.name==a})},columns:function(){return b.is.jq(this.$dropdown)?this.$dropdown.find("input:checked").map(function(){return a(this).data("__FooTableColumn__")}).get():this.ft.columns.get(function(a){return a.filterable})},ensure:function(a){var c=this,d=[],e=c.columns();return b.is.emptyArray(a)||b.arr.each(a,function(a){a=c._ensure(a,e),a instanceof b.Filter&&d.push(a)}),d},createFilter:function(a,c,d,e,f,g,h){return b.is.string(a)&&(a={name:a,query:c,columns:d,ignoreCase:e,connectors:f,space:g,hidden:h}),this._ensure(a,this.columns())},_ensure:function(a,c){return(b.is.hash(a)||a instanceof b.Filter)&&!b.is.emptyString(a.name)&&(!b.is.emptyString(a.query)||a.query instanceof b.Query)?(a.columns=b.is.emptyArray(a.columns)?c:this.ft.columns.ensure(a.columns),a.ignoreCase=b.is["boolean"](a.ignoreCase)?a.ignoreCase:this.ignoreCase,a.connectors=b.is["boolean"](a.connectors)?a.connectors:this.connectors,a.hidden=b.is["boolean"](a.hidden)?a.hidden:!1,a.space=!b.is.string(a.space)||"AND"!==a.space&&"OR"!==a.space?this.space:a.space,a.query=b.is.string(a.query)?new b.Query(a.query,a.space,a.connectors,a.ignoreCase):a.query,a instanceof b.Filter?a:new b.Filter(a.name,a.query,a.columns,a.space,a.connectors,a.ignoreCase,a.hidden)):null},_onSearchInputChanged:function(a){var c=a.data.self,d="keypress"==a.type&&!b.is.emptyString(String.fromCharCode(a.charCode)),e="keyup"==a.type&&(8==a.which||46==a.which),f="paste"==a.type;(d||e||f)&&(13==a.which&&a.preventDefault(),null!=c._filterTimeout&&clearTimeout(c._filterTimeout),c._filterTimeout=setTimeout(function(){c._filterTimeout=null;var a=c.$input.val();a.length>=c.min?(c.exactMatch&&!c._exactRegExp.test(a)&&(a='"'+a+'"'),c.addFilter("search",a),c.filter(c.focus)):b.is.emptyString(a)&&c.clear()},c.delay))},_onSearchButtonClicked:function(a){a.preventDefault();var b=a.data.self;null!=b._filterTimeout&&clearTimeout(b._filterTimeout);var c=b.$button.children(".fooicon");if(c.hasClass("fooicon-remove"))b.clear();else{var d=b.$input.val();d.length>=b.min&&(b.exactMatch&&!b._exactRegExp.test(d)&&(d='"'+d+'"'),b.addFilter("search",d),b.filter(b.focus))}},_onSearchColumnClicked:function(a){var b=a.data.self;null!=b._filterTimeout&&clearTimeout(b._filterTimeout),b._filterTimeout=setTimeout(function(){b._filterTimeout=null;var a=b.$button.children(".fooicon");a.hasClass("fooicon-remove")&&(a.removeClass("fooicon-remove").addClass("fooicon-search"),b.addFilter("search",b.$input.val()),b.filter())},b.delay)},_onDropdownToggleClicked:function(b){b.preventDefault(),b.stopPropagation();var c=b.data.self;c.$dropdown.parent().toggleClass("open"),c.$dropdown.parent().hasClass("open")?a(document).on("click.footable",{self:c},c._onDocumentClicked):a(document).off("click.footable",c._onDocumentClicked)},_onDocumentClicked:function(b){if(0==a(b.target).closest(".dropdown-menu").length){b.preventDefault();var c=b.data.self;c.$dropdown.parent().removeClass("open"),a(document).off("click.footable",c._onDocumentClicked)}}}),b.components.register("filtering",b.Filtering,500)}(jQuery,FooTable),function(a){a.Query=a.Class.extend({construct:function(b,c,d,e){this._original=null,this._value=null,this.space=!a.is.string(c)||"OR"!=c&&"AND"!=c?"AND":c,this.connectors=a.is["boolean"](d)?d:!0,this.ignoreCase=a.is["boolean"](e)?e:!0,this.left=null,this.right=null,this.parts=[],this.operator=null,this.val(b)},val:function(b){if(a.is.emptyString(b))return this._value;if(a.is.emptyString(this._original))this._original=b;else if(this._original==b)return;this._value=b,this._parse()},match:function(b){return a.is.emptyString(this.operator)||"OR"===this.operator?this._left(b,!1)||this._match(b,!1)||this._right(b,!1):"AND"===this.operator?this._left(b,!0)&&this._match(b,!0)&&this._right(b,!0):void 0},_match:function(b,c){var d=this,e=!1,f=a.is.emptyString(b);return a.is.emptyArray(d.parts)&&d.left instanceof a.Query?c:a.is.emptyArray(d.parts)?e:("OR"===d.space?a.arr.each(d.parts,function(c){if(c.empty&&f){if(e=!0,c.negate)return e=!1}else{var g=(c.exact?a.str.containsExact:a.str.contains)(b,c.query,d.ignoreCase);if(g&&!c.negate&&(e=!0),g&&c.negate)return e=!1}}):(e=!0,a.arr.each(d.parts,function(c){if(c.empty)return(!f&&!c.negate||f&&c.negate)&&(e=!1),e;var g=(c.exact?a.str.containsExact:a.str.contains)(b,c.query,d.ignoreCase);return(!g&&!c.negate||g&&c.negate)&&(e=!1),e})),e)},_left:function(b,c){return this.left instanceof a.Query?this.left.match(b):c},_right:function(b,c){return this.right instanceof a.Query?this.right.match(b):c},_parse:function(){if(!a.is.emptyString(this._value))if(/\sOR\s/.test(this._value)){this.operator="OR";var b=this._value.split(/(?:\sOR\s)(.*)?/);this.left=new a.Query(b[0],this.space,this.connectors,this.ignoreCase),this.right=new a.Query(b[1],this.space,this.connectors,this.ignoreCase)}else if(/\sAND\s/.test(this._value)){this.operator="AND";var c=this._value.split(/(?:\sAND\s)(.*)?/);this.left=new a.Query(c[0],this.space,this.connectors,this.ignoreCase),this.right=new a.Query(c[1],this.space,this.connectors,this.ignoreCase)}else{var d=this;this.parts=a.arr.map(this._value.match(/(?:[^\s"]+|"[^"]*")+/g),function(a){return d._part(a)})}},_part:function(b){var c={query:b,negate:!1,phrase:!1,exact:!1,empty:!1};return a.str.startsWith(c.query,"-")&&(c.query=a.str.from(c.query,"-"),c.negate=!0),/^"(.*?)"$/.test(c.query)?(c.query=c.query.replace(/^"(.*?)"$/,"$1"),c.phrase=!0,c.exact=!0):this.connectors&&/(?:\w)+?([-_\+\.])(?:\w)+?/.test(c.query)&&(c.query=c.query.replace(/(?:\w)+?([-_\+\.])(?:\w)+?/g,function(a,b){return a.replace(b," ")}),c.phrase=!0),c.empty=c.phrase&&a.is.emptyString(c.query),c}})}(FooTable),function(a){a.Cell.prototype.filterValue=null,a.Cell.prototype.__filtering_define__=function(a){this.filterValue=this.column.filterValue.call(this.column,a)},a.Cell.prototype.__filtering_val__=function(b){a.is.defined(b)&&(this.filterValue=this.column.filterValue.call(this.column,b))},a.Cell.extend("define",function(a){this._super(a),this.__filtering_define__(a)}),a.Cell.extend("val",function(a,b,c){var d=this._super(a,b,c);return this.__filtering_val__(a),d})}(FooTable),function(a,b){b.Column.prototype.filterable=!0,b.Column.prototype.filterValue=function(c){if(b.is.element(c)||b.is.jq(c)){var d=a(c).data("filterValue");return b.is.defined(d)?""+d:a(c).text()}if(b.is.hash(c)&&b.is.hash(c.options)){if(b.is.string(c.options.filterValue))return c.options.filterValue;b.is.defined(c.value)&&(c=c.value)}return b.is.defined(c)&&null!=c?c+"":""},b.Column.prototype.__filtering_define__=function(a){this.filterable=b.is["boolean"](a.filterable)?a.filterable:this.filterable,this.filterValue=b.checkFnValue(this,a.filterValue,this.filterValue)},b.Column.extend("define",function(a){this._super(a),this.__filtering_define__(a)})}(jQuery,FooTable),function(a){a.Defaults.prototype.filtering={enabled:!1,filters:[],delay:1200,min:1,space:"AND",placeholder:"Search",dropdownTitle:null,position:"right",connectors:!0,ignoreCase:!0,exactMatch:!1,focus:!0,container:null}}(FooTable),function(a){a.Row.prototype.filtered=function(b){var c=!0,d=this;return a.arr.each(b,function(a){return 0==(c=a.matchRow(d))?!1:void 0}),c}}(FooTable),function(a,b){b.Sorter=b.Class.extend({construct:function(a,b){this.column=a,this.direction=b}})}(jQuery,FooTable),function(a,b){b.Sorting=b.Component.extend({construct:function(a){this._super(a,a.o.sorting.enabled),this.o=a.o.sorting,this.column=null,this.allowed=!0,this.initial=null},preinit:function(a){var c=this;this.ft.raise("preinit.ft.sorting",[a]).then(function(){c.ft.$el.hasClass("footable-sorting")&&(c.enabled=!0),c.enabled=b.is["boolean"](a.sorting)?a.sorting:c.enabled,c.enabled&&(c.column=b.arr.first(c.ft.columns.array,function(a){return a.sorted}))},function(){c.enabled=!1})},init:function(){var c=this;this.ft.raise("init.ft.sorting").then(function(){if(!c.initial){var d=!!c.column;c.initial={isset:d,rows:c.ft.rows.all.slice(0),column:d?c.column.name:null,direction:d?c.column.direction:null}}b.arr.each(c.ft.columns.array,function(b){b.sortable&&b.$el.addClass("footable-sortable").append(a("<span/>",{"class":"fooicon fooicon-sort"}))}),c.ft.$el.on("click.footable",".footable-sortable",{self:c},c._onSortClicked)},function(){c.enabled=!1})},destroy:function(){var a=this;this.ft.raise("destroy.ft.paging").then(function(){a.ft.$el.off("click.footable",".footable-sortable",a._onSortClicked),a.ft.$el.children("thead").children("tr.footable-header").children(".footable-sortable").removeClass("footable-sortable footable-asc footable-desc").find("span.fooicon").remove()})},predraw:function(){if(this.column){var a=this,b=a.column;a.ft.rows.array.sort(function(a,c){return"DESC"==b.direction?b.sorter(c.cells[b.index].sortValue,a.cells[b.index].sortValue):b.sorter(a.cells[b.index].sortValue,c.cells[b.index].sortValue)})}},draw:function(){if(this.column){var a=this,b=a.ft.$el.find("thead > tr > .footable-sortable"),c=a.column.$el;b.removeClass("footable-asc footable-desc").children(".fooicon").removeClass("fooicon-sort fooicon-sort-asc fooicon-sort-desc"),b.not(c).children(".fooicon").addClass("fooicon-sort"),c.addClass("DESC"==a.column.direction?"footable-desc":"footable-asc").children(".fooicon").addClass("DESC"==a.column.direction?"fooicon-sort-desc":"fooicon-sort-asc")}},sort:function(a,b){return this._sort(a,b)},toggleAllowed:function(a){a=b.is["boolean"](a)?a:!this.allowed,this.allowed=a,this.ft.$el.toggleClass("footable-sorting-disabled",!this.allowed)},hasChanged:function(){return!(!this.initial||!this.column||this.column.name===this.initial.column&&(this.column.direction===this.initial.direction||null===this.initial.direction&&"ASC"===this.column.direction))},reset:function(){this.initial&&(this.initial.isset?this.sort(this.initial.column,this.initial.direction):(this.column&&(this.column.$el.removeClass("footable-asc footable-desc"),this.column=null),this.ft.rows.all=this.initial.rows,this.ft.draw()))},_sort:function(c,d){if(!this.allowed)return a.Deferred().reject("sorting disabled");var e=this,f=new b.Sorter(e.ft.columns.get(c),b.Sorting.dir(d));return e.ft.raise("before.ft.sorting",[f]).then(function(){return b.arr.each(e.ft.columns.array,function(a){a!=e.column&&(a.direction=null)}),e.column=e.ft.columns.get(f.column),e.column&&(e.column.direction=b.Sorting.dir(f.direction)),e.ft.draw().then(function(){e.ft.raise("after.ft.sorting",[f])})})},_onSortClicked:function(b){var c=b.data.self,d=a(this).closest("th,td"),e=d.is(".footable-asc, .footable-desc")?d.hasClass("footable-desc")?"ASC":"DESC":"ASC";c._sort(d.index(),e)}}),b.Sorting.dir=function(a){return!b.is.string(a)||"ASC"!=a&&"DESC"!=a?"ASC":a},b.components.register("sorting",b.Sorting,600)}(jQuery,FooTable),function(a){a.Cell.prototype.sortValue=null,a.Cell.prototype.__sorting_define__=function(a){this.sortValue=this.column.sortValue.call(this.column,a)},a.Cell.prototype.__sorting_val__=function(b){a.is.defined(b)&&(this.sortValue=this.column.sortValue.call(this.column,b))},a.Cell.extend("define",function(a){this._super(a),this.__sorting_define__(a)}),a.Cell.extend("val",function(a,b,c){var d=this._super(a,b,c);return this.__sorting_val__(a),d})}(FooTable),function(a,b){b.Column.prototype.direction=null,b.Column.prototype.sortable=!0,b.Column.prototype.sorted=!1,b.Column.prototype.sorter=function(a,b){return"string"==typeof a&&(a=a.toLowerCase()),"string"==typeof b&&(b=b.toLowerCase()),a===b?0:b>a?-1:1},b.Column.prototype.sortValue=function(c){if(b.is.element(c)||b.is.jq(c)){var d=a(c).data("sortValue");return b.is.defined(d)?d:this.parser(c)}if(b.is.hash(c)&&b.is.hash(c.options)){if(b.is.string(c.options.sortValue))return c.options.sortValue;b.is.defined(c.value)&&(c=c.value)}return b.is.defined(c)&&null!=c?c:null},b.Column.prototype.__sorting_define__=function(a){this.sorter=b.checkFnValue(this,a.sorter,this.sorter),this.direction=b.is.type(a.direction,"string")?b.Sorting.dir(a.direction):null,this.sortable=b.is["boolean"](a.sortable)?a.sortable:!0,this.sorted=b.is["boolean"](a.sorted)?a.sorted:!1,this.sortValue=b.checkFnValue(this,a.sortValue,this.sortValue)},b.Column.extend("define",function(a){this._super(a),this.__sorting_define__(a)})}(jQuery,FooTable),function(a){a.Defaults.prototype.sorting={enabled:!1}}(FooTable),function(a,b){b.HTMLColumn.extend("__sorting_define__",function(c){this._super(c),this.sortUse=b.is.string(c.sortUse)&&-1!==a.inArray(c.sortUse,["html","text"])?c.sortUse:"html"}),b.HTMLColumn.prototype.sortValue=function(c){if(b.is.element(c)||b.is.jq(c)){var d=a(c).data("sortValue");return b.is.defined(d)?d:this.parser(c)}if(b.is.hash(c)&&b.is.hash(c.options)){if(b.is.string(c.options.sortValue))return c.options.sortValue;b.is.defined(c.value)&&(c=c.value)}return b.is.defined(c)&&null!=c?c:null}}(jQuery,FooTable),function(a,b){b.NumberColumn.prototype.sortValue=function(c){if(b.is.element(c)||b.is.jq(c)){var d=a(c).data("sortValue");return b.is.number(d)?d:this.parser(c)}if(b.is.hash(c)&&b.is.hash(c.options)){if(b.is.string(c.options.sortValue))return this.parser(c);if(b.is.number(c.options.sortValue))return c.options.sortValue;if(b.is.number(c.value))return c.value}return b.is.string(c)?this.parser(c):b.is.number(c)?c:null}}(jQuery,FooTable),function(a){a.Table.prototype.sort=function(b,c){return this.use(a.Sorting).sort(b,c)}}(FooTable),function(a,b){b.Pager=b.Class.extend({construct:function(a,b,c,d,e){this.total=a,this.current=b,this.size=c,this.page=d,this.forward=e}})}(jQuery,FooTable),function(a,b){b.Paging=b.Component.extend({construct:function(a){this._super(a,a.o.paging.enabled),this.strings=a.o.paging.strings,this.current=a.o.paging.current,this.size=a.o.paging.size,this.limit=a.o.paging.limit,this.position=a.o.paging.position,this.countFormat=a.o.paging.countFormat,this.container=a.o.paging.container,this.total=-1,this.totalRows=0,this.previous=-1,this.formattedCount=null,this.$container=null,this.$wrapper=null,this.$row=null,this.$cell=null,this.$pagination=null,this.$count=null,this.detached=!0,this._createdLinks=0},preinit:function(a){var c=this;this.ft.raise("preinit.ft.paging",[a]).then(function(){c.ft.$el.hasClass("footable-paging")&&(c.enabled=!0),c.enabled=b.is["boolean"](a.paging)?a.paging:c.enabled,c.enabled&&(c.size=b.is.number(a.pagingSize)?a.pagingSize:c.size,c.current=b.is.number(a.pagingCurrent)?a.pagingCurrent:c.current,c.limit=b.is.number(a.pagingLimit)?a.pagingLimit:c.limit,c.ft.$el.hasClass("footable-paging-left")&&(c.position="left"),c.ft.$el.hasClass("footable-paging-center")&&(c.position="center"),c.ft.$el.hasClass("footable-paging-right")&&(c.position="right"),c.position=b.is.string(a.pagingPosition)?a.pagingPosition:c.position,c.countFormat=b.is.string(a.pagingCountFormat)?a.pagingCountFormat:c.countFormat,c.container=b.is.string(a.pagingContainer)?a.pagingContainer:c.container,c.total=Math.ceil(c.ft.rows.all.length/c.size))},function(){c.enabled=!1})},init:function(){var a=this;this.ft.raise("init.ft.paging").then(function(){a.$create()},function(){a.enabled=!1})},destroy:function(){var a=this;this.ft.raise("destroy.ft.paging").then(function(){a.ft.$el.removeClass("footable-paging").find("tfoot > tr.footable-paging").remove(),a.detached=!0,a._createdLinks=0})},predraw:function(){this.total=Math.ceil(this.ft.rows.array.length/this.size),this.current=this.current>this.total?this.total:this.current<1?1:this.current,this.totalRows=this.ft.rows.array.length,this.totalRows>this.size&&(this.ft.rows.array=this.ft.rows.array.splice((this.current-1)*this.size,this.size)),this.formattedCount=this.format(this.countFormat)},draw:function(){if(this.total<=1)this.detached||(this.$row?this.$row.detach():this.$wrapper.detach(),this.detached=!0);else{if(this.detached){if(this.$row){var c=this.ft.$el.children("tfoot");0==c.length&&(c=a("<tfoot/>"),this.ft.$el.append(c)),this.$row.appendTo(c)}else this.$wrapper.appendTo(this.$container);this.detached=!1}b.is.jq(this.$cell)&&this.$cell.attr("colspan",this.ft.columns.visibleColspan),this._createLinks(),this._setVisible(this.current,this.current>this.previous),this._setNavigation(!0),this.$count.text(this.formattedCount)}},$create:function(){this._createdLinks=0;var c="footable-paging-center";switch(this.position){case"left":c="footable-paging-left";break;case"right":c="footable-paging-right"}if(this.ft.$el.addClass("footable-paging").addClass(c),this.$container=null===this.container?null:a(this.container).first(),b.is.jq(this.$container))this.$container.addClass("footable-paging-external").addClass(c);else{var d=this.ft.$el.children("tfoot");0==d.length&&(d=a("<tfoot/>"),this.ft.$el.append(d)),this.$row=a("<tr/>",{"class":"footable-paging"}).prependTo(d),this.$container=this.$cell=a("<td/>").attr("colspan",this.ft.columns.visibleColspan).appendTo(this.$row)}this.$wrapper=a("<div/>",{"class":"footable-pagination-wrapper"}).appendTo(this.$container),this.$pagination=a("<ul/>",{"class":"pagination"}).on("click.footable","a.footable-page-link",{self:this},this._onPageClicked),this.$count=a("<span/>",{"class":"label label-default"}),this.$wrapper.append(this.$pagination,a("<div/>",{"class":"divider"}),this.$count),this.detached=!1},format:function(a){var b=this.size*(this.current-1)+1,c=this.size*this.current;return 0==this.ft.rows.array.length?(b=0,c=0):c=c>this.totalRows?this.totalRows:c,a.replace(/\{CP}/g,this.current).replace(/\{TP}/g,this.total).replace(/\{PF}/g,b).replace(/\{PL}/g,c).replace(/\{TR}/g,this.totalRows)},first:function(){return this._set(1)},prev:function(){return this._set(this.current-1>0?this.current-1:1)},next:function(){return this._set(this.current+1<this.total?this.current+1:this.total)},last:function(){return this._set(this.total)},"goto":function(a){return this._set(a>this.total?this.total:1>a?1:a)},prevPages:function(){var a=this.$pagination.children("li.footable-page.visible:first").data("page")-1;this._setVisible(a,!0),this._setNavigation(!1)},nextPages:function(){var a=this.$pagination.children("li.footable-page.visible:last").data("page")+1;this._setVisible(a,!1),this._setNavigation(!1)},pageSize:function(a){return a=parseInt(a),isNaN(a)?this.size:(this.size=a,this.total=Math.ceil(this.ft.rows.all.length/this.size),b.is.jq(this.$wrapper)&&(this.$container.is("td")?this.$row.remove():this.$wrapper.remove()),this.$create(),void this.ft.draw())},_set:function(c){var d=this,e=new b.Pager(d.total,d.current,d.size,c,c>d.current);return d.ft.raise("before.ft.paging",[e]).then(function(){return e.page=e.page>e.total?e.total:e.page,e.page=e.page<1?1:e.page,d.current==c?a.when():(d.previous=d.current,d.current=e.page,d.ft.draw().then(function(){d.ft.raise("after.ft.paging",[e])}))})},_createLinks:function(){if(this._createdLinks!==this.total){var b=this,c=b.total>1,d=function(b,c,d){return a("<li/>",{"class":d}).attr("data-page",b).append(a("<a/>",{"class":"footable-page-link",href:"#"}).data("page",b).html(c))};b.$pagination.empty(),c&&(b.$pagination.append(d("first",b.strings.first,"footable-page-nav")),b.$pagination.append(d("prev",b.strings.prev,"footable-page-nav")),b.limit>0&&b.limit<b.total&&b.$pagination.append(d("prev-limit",b.strings.prevPages,"footable-page-nav")));for(var e,f=0;f<b.total;f++)e=d(f+1,f+1,"footable-page"),b.$pagination.append(e);c&&(b.limit>0&&b.limit<b.total&&b.$pagination.append(d("next-limit",b.strings.nextPages,"footable-page-nav")),b.$pagination.append(d("next",b.strings.next,"footable-page-nav")),b.$pagination.append(d("last",b.strings.last,"footable-page-nav"))),b._createdLinks=b.total}},_setNavigation:function(a){1==this.current?this.$pagination.children('li[data-page="first"],li[data-page="prev"]').addClass("disabled"):this.$pagination.children('li[data-page="first"],li[data-page="prev"]').removeClass("disabled"),this.current==this.total?this.$pagination.children('li[data-page="next"],li[data-page="last"]').addClass("disabled"):this.$pagination.children('li[data-page="next"],li[data-page="last"]').removeClass("disabled"),1==(this.$pagination.children("li.footable-page.visible:first").data("page")||1)?this.$pagination.children('li[data-page="prev-limit"]').addClass("disabled"):this.$pagination.children('li[data-page="prev-limit"]').removeClass("disabled"),(this.$pagination.children("li.footable-page.visible:last").data("page")||this.limit)==this.total?this.$pagination.children('li[data-page="next-limit"]').addClass("disabled"):this.$pagination.children('li[data-page="next-limit"]').removeClass("disabled"),this.limit>0&&this.total<this.limit?this.$pagination.children('li[data-page="prev-limit"],li[data-page="next-limit"]').css("display","none"):this.$pagination.children('li[data-page="prev-limit"],li[data-page="next-limit"]').css("display",""),
+a&&this.$pagination.children("li.footable-page").removeClass("active").filter('li[data-page="'+this.current+'"]').addClass("active")},_setVisible:function(a,b){if(this.limit>0&&this.total>this.limit){if(!this.$pagination.children('li.footable-page[data-page="'+a+'"]').hasClass("visible")){var c=0,d=0;1==b?(d=a>this.total?this.total:a,c=d-this.limit):(c=1>a?0:a-1,d=c+this.limit),0>c&&(c=0,d=this.limit>this.total?this.total:this.limit),d>this.total&&(d=this.total,c=this.total-this.limit<0?0:this.total-this.limit),this.$pagination.children("li.footable-page").removeClass("visible").slice(c,d).addClass("visible")}}else this.$pagination.children("li.footable-page").removeClass("visible").slice(0,this.total).addClass("visible")},_onPageClicked:function(b){if(b.preventDefault(),!a(b.target).closest("li").is(".active,.disabled")){var c=b.data.self,d=a(this).data("page");switch(d){case"first":return void c.first();case"prev":return void c.prev();case"next":return void c.next();case"last":return void c.last();case"prev-limit":return void c.prevPages();case"next-limit":return void c.nextPages();default:return void c._set(d)}}}}),b.components.register("paging",b.Paging,400)}(jQuery,FooTable),function(a){a.Defaults.prototype.paging={enabled:!1,countFormat:"{CP} of {TP}",current:1,limit:5,position:"center",size:10,container:null,strings:{first:"&laquo;",prev:"&lsaquo;",next:"&rsaquo;",last:"&raquo;",prevPages:"...",nextPages:"..."}}}(FooTable),function(a){a.Table.prototype.gotoPage=function(b){return this.use(a.Paging)["goto"](b)},a.Table.prototype.nextPage=function(){return this.use(a.Paging).next()},a.Table.prototype.prevPage=function(){return this.use(a.Paging).prev()},a.Table.prototype.firstPage=function(){return this.use(a.Paging).first()},a.Table.prototype.lastPage=function(){return this.use(a.Paging).last()},a.Table.prototype.nextPages=function(){return this.use(a.Paging).nextPages()},a.Table.prototype.prevPages=function(){return this.use(a.Paging).prevPages()},a.Table.prototype.pageSize=function(b){return this.use(a.Paging).pageSize(b)}}(FooTable),function(a,b){b.Editing=b.Component.extend({construct:function(c){this._super(c,c.o.editing.enabled),this.pageToNew=c.o.editing.pageToNew,this.alwaysShow=c.o.editing.alwaysShow,this.column=a.extend(!0,{},c.o.editing.column,{visible:this.alwaysShow}),this.position=c.o.editing.position,this.showText=c.o.editing.showText,this.hideText=c.o.editing.hideText,this.addText=c.o.editing.addText,this.editText=c.o.editing.editText,this.deleteText=c.o.editing.deleteText,this.viewText=c.o.editing.viewText,this.allowAdd=c.o.editing.allowAdd,this.allowEdit=c.o.editing.allowEdit,this.allowDelete=c.o.editing.allowDelete,this.allowView=c.o.editing.allowView,this._$buttons=null,this.callbacks={addRow:b.checkFnValue(this,c.o.editing.addRow),editRow:b.checkFnValue(this,c.o.editing.editRow),deleteRow:b.checkFnValue(this,c.o.editing.deleteRow),viewRow:b.checkFnValue(this,c.o.editing.viewRow)}},preinit:function(c){var d=this;this.ft.raise("preinit.ft.editing",[c]).then(function(){if(d.ft.$el.hasClass("footable-editing")&&(d.enabled=!0),d.enabled=b.is["boolean"](c.editing)?c.editing:d.enabled,d.enabled){if(d.pageToNew=b.is["boolean"](c.editingPageToNew)?c.editingPageToNew:d.pageToNew,d.alwaysShow=b.is["boolean"](c.editingAlwaysShow)?c.editingAlwaysShow:d.alwaysShow,d.position=b.is.string(c.editingPosition)?c.editingPosition:d.position,d.showText=b.is.string(c.editingShowText)?c.editingShowText:d.showText,d.hideText=b.is.string(c.editingHideText)?c.editingHideText:d.hideText,d.addText=b.is.string(c.editingAddText)?c.editingAddText:d.addText,d.editText=b.is.string(c.editingEditText)?c.editingEditText:d.editText,d.deleteText=b.is.string(c.editingDeleteText)?c.editingDeleteText:d.deleteText,d.viewText=b.is.string(c.editingViewText)?c.editingViewText:d.viewText,d.allowAdd=b.is["boolean"](c.editingAllowAdd)?c.editingAllowAdd:d.allowAdd,d.allowEdit=b.is["boolean"](c.editingAllowEdit)?c.editingAllowEdit:d.allowEdit,d.allowDelete=b.is["boolean"](c.editingAllowDelete)?c.editingAllowDelete:d.allowDelete,d.allowView=b.is["boolean"](c.editingAllowView)?c.editingAllowView:d.allowView,d.column=new b.EditingColumn(d.ft,d,a.extend(!0,{},d.column,c.editingColumn,{visible:d.alwaysShow})),d.ft.$el.hasClass("footable-editing-left")&&(d.position="left"),d.ft.$el.hasClass("footable-editing-right")&&(d.position="right"),"right"===d.position)d.column.index=d.ft.columns.array.length;else{d.column.index=0;for(var e=0,f=d.ft.columns.array.length;f>e;e++)d.ft.columns.array[e].index+=1}d.ft.columns.array.push(d.column),d.ft.columns.array.sort(function(a,b){return a.index-b.index}),d.callbacks.addRow=b.checkFnValue(d,c.editingAddRow,d.callbacks.addRow),d.callbacks.editRow=b.checkFnValue(d,c.editingEditRow,d.callbacks.editRow),d.callbacks.deleteRow=b.checkFnValue(d,c.editingDeleteRow,d.callbacks.deleteRow),d.callbacks.viewRow=b.checkFnValue(d,c.editingViewRow,d.callbacks.viewRow)}},function(){d.enabled=!1})},init:function(){var a=this;this.ft.raise("init.ft.editing").then(function(){a.$create()},function(){a.enabled=!1})},destroy:function(){var a=this;this.ft.raise("destroy.ft.editing").then(function(){a.ft.$el.removeClass("footable-editing footable-editing-always-show footable-editing-no-add footable-editing-no-edit footable-editing-no-delete footable-editing-no-view").off("click.ft.editing").find("tfoot > tr.footable-editing").remove()})},$create:function(){var b=this,c="right"===b.position?"footable-editing-right":"footable-editing-left";b.ft.$el.addClass("footable-editing").addClass(c).on("click.ft.editing",".footable-show",{self:b},b._onShowClick).on("click.ft.editing",".footable-hide",{self:b},b._onHideClick).on("click.ft.editing",".footable-edit",{self:b},b._onEditClick).on("click.ft.editing",".footable-delete",{self:b},b._onDeleteClick).on("click.ft.editing",".footable-view",{self:b},b._onViewClick).on("click.ft.editing",".footable-add",{self:b},b._onAddClick),b.$cell=a("<td/>").attr("colspan",b.ft.columns.visibleColspan).append(b.$buttonShow()),b.allowAdd&&b.$cell.append(b.$buttonAdd()),b.$cell.append(b.$buttonHide()),b.alwaysShow&&b.ft.$el.addClass("footable-editing-always-show"),b.allowAdd||b.ft.$el.addClass("footable-editing-no-add"),b.allowEdit||b.ft.$el.addClass("footable-editing-no-edit"),b.allowDelete||b.ft.$el.addClass("footable-editing-no-delete"),b.allowView||b.ft.$el.addClass("footable-editing-no-view");var d=b.ft.$el.children("tfoot");0==d.length&&(d=a("<tfoot/>"),b.ft.$el.append(d)),b.$row=a("<tr/>",{"class":"footable-editing"}).append(b.$cell).appendTo(d)},$buttonShow:function(){return'<button type="button" class="btn btn-primary footable-show">'+this.showText+"</button>"},$buttonHide:function(){return'<button type="button" class="btn btn-default footable-hide">'+this.hideText+"</button>"},$buttonAdd:function(){return'<button type="button" class="btn btn-primary footable-add">'+this.addText+"</button> "},$buttonEdit:function(){return'<button type="button" class="btn btn-default footable-edit">'+this.editText+"</button> "},$buttonDelete:function(){return'<button type="button" class="btn btn-default footable-delete">'+this.deleteText+"</button>"},$buttonView:function(){return'<button type="button" class="btn btn-default footable-view">'+this.viewText+"</button> "},$rowButtons:function(){return b.is.jq(this._$buttons)?this._$buttons.clone():(this._$buttons=a('<div class="btn-group btn-group-xs" role="group"></div>'),this.allowView&&this._$buttons.append(this.$buttonView()),this.allowEdit&&this._$buttons.append(this.$buttonEdit()),this.allowDelete&&this._$buttons.append(this.$buttonDelete()),this._$buttons)},draw:function(){this.$cell.attr("colspan",this.ft.columns.visibleColspan)},_onEditClick:function(c){c.preventDefault();var d=c.data.self,e=a(this).closest("tr").data("__FooTableRow__");e instanceof b.Row&&d.ft.raise("edit.ft.editing",[e]).then(function(){d.callbacks.editRow.call(d.ft,e)})},_onDeleteClick:function(c){c.preventDefault();var d=c.data.self,e=a(this).closest("tr").data("__FooTableRow__");e instanceof b.Row&&d.ft.raise("delete.ft.editing",[e]).then(function(){d.callbacks.deleteRow.call(d.ft,e)})},_onViewClick:function(c){c.preventDefault();var d=c.data.self,e=a(this).closest("tr").data("__FooTableRow__");e instanceof b.Row&&d.ft.raise("view.ft.editing",[e]).then(function(){d.callbacks.viewRow.call(d.ft,e)})},_onAddClick:function(a){a.preventDefault();var b=a.data.self;b.ft.raise("add.ft.editing").then(function(){b.callbacks.addRow.call(b.ft)})},_onShowClick:function(a){a.preventDefault();var b=a.data.self;b.ft.raise("show.ft.editing").then(function(){b.ft.$el.addClass("footable-editing-show"),b.column.visible=!0,b.ft.draw()})},_onHideClick:function(a){a.preventDefault();var b=a.data.self;b.ft.raise("hide.ft.editing").then(function(){b.ft.$el.removeClass("footable-editing-show"),b.column.visible=!1,b.ft.draw()})}}),b.components.register("editing",b.Editing,850)}(jQuery,FooTable),function(a,b){b.EditingColumn=b.Column.extend({construct:function(a,b,c){this._super(a,c,"editing"),this.editing=b,this.internal=!0},$create:function(){(this.$el=!this.virtual&&b.is.jq(this.$el)?this.$el:a("<th/>",{"class":"footable-editing"})).html(this.title)},parser:function(c){if(b.is.string(c)&&(c=a(a.trim(c))),b.is.element(c)&&(c=a(c)),b.is.jq(c)){var d=c.prop("tagName").toLowerCase();return"td"==d||"th"==d?c.data("value")||c.contents():c}return null},createCell:function(c){var d=this.editing.$rowButtons(),e=a("<td/>").append(d);return b.is.jq(c.$el)&&(0===this.index?e.prependTo(c.$el):e.insertAfter(c.$el.children().eq(this.index-1))),new b.Cell(this.ft,c,this,e||e.html())}}),b.columns.register("editing",b.EditingColumn)}(jQuery,FooTable),function(a,b){b.Defaults.prototype.editing={enabled:!1,pageToNew:!0,position:"right",alwaysShow:!1,addRow:function(){},editRow:function(a){},deleteRow:function(a){},viewRow:function(a){},showText:'<span class="fooicon fooicon-pencil" aria-hidden="true"></span> Edit rows',hideText:"Cancel",addText:"New row",editText:'<span class="fooicon fooicon-pencil" aria-hidden="true"></span>',deleteText:'<span class="fooicon fooicon-trash" aria-hidden="true"></span>',viewText:'<span class="fooicon fooicon-stats" aria-hidden="true"></span>',allowAdd:!0,allowEdit:!0,allowDelete:!0,allowView:!1,column:{classes:"footable-editing",name:"editing",title:"",filterable:!1,sortable:!1}}}(jQuery,FooTable),function(a,b){b.is.defined(b.Paging)&&(b.Paging.prototype.unpaged=[],b.Paging.extend("predraw",function(){this.unpaged=this.ft.rows.array.slice(0),this._super()}))}(jQuery,FooTable),function(a,b){b.Row.prototype.add=function(c){c=b.is["boolean"](c)?c:!0;var d=this;return a.Deferred(function(a){var b=d.ft.rows.all.push(d)-1;return c?d.ft.draw().then(function(){a.resolve(b)}):void a.resolve(b)})},b.Row.prototype["delete"]=function(c){c=b.is["boolean"](c)?c:!0;var d=this;return a.Deferred(function(a){var e=d.ft.rows.all.indexOf(d);return b.is.number(e)&&e>=0&&e<d.ft.rows.all.length&&(d.ft.rows.all.splice(e,1),c)?d.ft.draw().then(function(){a.resolve(d)}):void a.resolve(d)})},b.is.defined(b.Paging)&&b.Row.extend("add",function(a){a=b.is["boolean"](a)?a:!0;var c,d=this,e=this._super(a),f=d.ft.use(b.Editing);return f&&f.pageToNew&&(c=d.ft.use(b.Paging))&&a?e.then(function(){var a=c.unpaged.indexOf(d),b=Math.ceil((a+1)/c.size);return c.current!==b?c["goto"](b):void 0}):e}),b.is.defined(b.Sorting)&&b.Row.extend("val",function(a,c){c=b.is["boolean"](c)?c:!0;var d=this._super(a);if(!b.is.hash(a))return d;var e=this;return c&&e.ft.draw().then(function(){var a,c=e.ft.use(b.Editing);if(b.is.defined(b.Paging)&&c&&c.pageToNew&&(a=e.ft.use(b.Paging))){var d=a.unpaged.indexOf(e),f=Math.ceil((d+1)/a.size);if(a.current!==f)return a["goto"](f)}}),d})}(jQuery,FooTable),function(a){a.Rows.prototype.add=function(b,c){var d=b;a.is.hash(b)&&(d=new FooTable.Row(this.ft,this.ft.columns.array,b)),d instanceof FooTable.Row&&d.add(c)},a.Rows.prototype.update=function(b,c,d){var e=this.ft.rows.all.length,f=b;a.is.number(b)&&b>=0&&e>b&&(f=this.ft.rows.all[b]),f instanceof FooTable.Row&&a.is.hash(c)&&f.val(c,d)},a.Rows.prototype["delete"]=function(b,c){var d=this.ft.rows.all.length,e=b;a.is.number(b)&&b>=0&&d>b&&(e=this.ft.rows.all[b]),e instanceof FooTable.Row&&e["delete"](c)}}(FooTable),function(a,b){var c=0,d=function(a){var b,c,d=2166136261;for(b=0,c=a.length;c>b;b++)d^=a.charCodeAt(b),d+=(d<<1)+(d<<4)+(d<<7)+(d<<8)+(d<<24);return d>>>0}(location.origin+location.pathname);b.State=b.Component.extend({construct:function(a){this._super(a,a.o.state.enabled),this._key="1",this.key=this._key+(b.is.string(a.o.state.key)?a.o.state.key:this._uid()),this.filtering=b.is["boolean"](a.o.state.filtering)?a.o.state.filtering:!0,this.paging=b.is["boolean"](a.o.state.paging)?a.o.state.paging:!0,this.sorting=b.is["boolean"](a.o.state.sorting)?a.o.state.sorting:!0},preinit:function(a){var c=this;this.ft.raise("preinit.ft.state",[a]).then(function(){c.enabled=b.is["boolean"](a.state)?a.state:c.enabled,c.enabled&&(c.key=c._key+(b.is.string(a.stateKey)?a.stateKey:c.key),c.filtering=b.is["boolean"](a.stateFiltering)?a.stateFiltering:c.filtering,c.paging=b.is["boolean"](a.statePaging)?a.statePaging:c.paging,c.sorting=b.is["boolean"](a.stateSorting)?a.stateSorting:c.sorting)},function(){c.enabled=!1})},get:function(a){return JSON.parse(localStorage.getItem(this.key+":"+a))},set:function(a,b){localStorage.setItem(this.key+":"+a,JSON.stringify(b))},remove:function(a){localStorage.removeItem(this.key+":"+a)},read:function(){this.ft.execute(!1,!0,"readState")},write:function(){this.ft.execute(!1,!0,"writeState")},clear:function(){this.ft.execute(!1,!0,"clearState")},_uid:function(){var a=this.ft.$el.attr("id");return d+"_"+(b.is.string(a)?a:++c)}}),b.components.register("state",b.State,700)}(jQuery,FooTable),function(a){a.Component.prototype.readState=function(){},a.Component.prototype.writeState=function(){},a.Component.prototype.clearState=function(){}}(FooTable),function(a){a.Defaults.prototype.state={enabled:!1,filtering:!0,paging:!0,sorting:!0,key:null}}(FooTable),function(a){a.Filtering&&(a.Filtering.prototype.readState=function(){if(this.ft.state.filtering){var b=this.ft.state.get("filtering");a.is.hash(b)&&!a.is.emptyArray(b.filters)&&(this.filters=this.ensure(b.filters))}},a.Filtering.prototype.writeState=function(){if(this.ft.state.filtering){var b=a.arr.map(this.filters,function(b){return{name:b.name,query:b.query instanceof a.Query?b.query.val():b.query,columns:a.arr.map(b.columns,function(a){return a.name}),hidden:b.hidden,space:b.space,connectors:b.connectors,ignoreCase:b.ignoreCase}});this.ft.state.set("filtering",{filters:b})}},a.Filtering.prototype.clearState=function(){this.ft.state.filtering&&this.ft.state.remove("filtering")})}(FooTable),function(a){a.Paging&&(a.Paging.prototype.readState=function(){if(this.ft.state.paging){var b=this.ft.state.get("paging");a.is.hash(b)&&(this.current=b.current,this.size=b.size)}},a.Paging.prototype.writeState=function(){this.ft.state.paging&&this.ft.state.set("paging",{current:this.current,size:this.size})},a.Paging.prototype.clearState=function(){this.ft.state.paging&&this.ft.state.remove("paging")})}(FooTable),function(a){a.Sorting&&(a.Sorting.prototype.readState=function(){if(this.ft.state.sorting){var b=this.ft.state.get("sorting");if(a.is.hash(b)){var c=this.ft.columns.get(b.column);c instanceof a.Column&&(this.column=c,this.column.direction=b.direction)}}},a.Sorting.prototype.writeState=function(){this.ft.state.sorting&&this.column instanceof a.Column&&this.ft.state.set("sorting",{column:this.column.name,direction:this.column.direction})},a.Sorting.prototype.clearState=function(){this.ft.state.sorting&&this.ft.state.remove("sorting")})}(FooTable),function(a){a.Table.extend("_construct",function(a){return this.state=this.use(FooTable.State),this._super(a)}),a.Table.extend("_preinit",function(){var a=this;return a._super().then(function(){a.state.enabled&&a.state.read()})}),a.Table.extend("draw",function(){var a=this;return a._super().then(function(){a.state.enabled&&a.state.write()})})}(FooTable),function(a,b){b.Export=b.Component.extend({construct:function(a){this._super(a,!0),this.snapshot=[]},predraw:function(){this.snapshot=this.ft.rows.array.slice(0)},columns:function(){var a=[];return b.arr.each(this.ft.columns.array,function(b){b.internal||a.push({type:b.type,name:b.name,title:b.title,visible:b.visible,hidden:b.hidden,classes:b.classes,style:b.style})}),a},rows:function(a){a=b.is["boolean"](a)?a:!1;var c=a?this.ft.rows.all:this.snapshot,d=[];return b.arr.each(c,function(a){d.push(a.val())}),d},json:function(a){return JSON.parse(JSON.stringify({columns:this.columns(),rows:this.rows(a)}))},csv:function(a){var c,d,e="",f=this.columns();b.arr.each(f,function(a,b){d='"'+a.title.replace(/"/g,'""')+'"',e+=0===b?d:","+d}),e+="\n";var g=a?this.ft.rows.all:this.snapshot;return b.arr.each(g,function(a){b.arr.each(a.cells,function(a,b){a.column.internal||(c=a.column.stringify.call(a.column,a.value,a.ft.o,a.row.value),d='"'+c.replace(/"/g,'""')+'"',e+=0===b?d:","+d)}),e+="\n"}),e}}),b.components.register("export",b.Export,490)}(jQuery,FooTable),function(a){a.Column.prototype.__export_define__=function(b){this.stringify=a.checkFnValue(this,b.stringify,this.stringify)},a.Column.extend("define",function(a){this._super(a),this.__export_define__(a)}),a.Column.prototype.stringify=function(a,b,c){return a+""},a.is.defined(a.DateColumn)&&(a.DateColumn.prototype.stringify=function(b,c,d){return a.is.object(b)&&a.is["boolean"](b._isAMomentObject)&&b.isValid()?b.format(this.formatString):""}),a.ObjectColumn.prototype.stringify=function(b,c,d){return a.is.object(b)?JSON.stringify(b):""},a.ArrayColumn.prototype.stringify=function(b,c,d){return a.is.array(b)?JSON.stringify(b):""}}(FooTable),function(a){a.Table.prototype.toJSON=function(b){return this.use(a.Export).json(b)},a.Table.prototype.toCSV=function(b){return this.use(a.Export).csv(b)}}(FooTable); \ No newline at end of file
diff --git a/interface/js/lib/jquery-3.7.1.min.js b/interface/js/lib/jquery-3.7.1.min.js
new file mode 100644
index 0000000..7f37b5d
--- /dev/null
+++ b/interface/js/lib/jquery-3.7.1.min.js
@@ -0,0 +1,2 @@
+/*! jQuery v3.7.1 | (c) OpenJS Foundation and other contributors | jquery.org/license */
+!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(ie,e){"use strict";var oe=[],r=Object.getPrototypeOf,ae=oe.slice,g=oe.flat?function(e){return oe.flat.call(e)}:function(e){return oe.concat.apply([],e)},s=oe.push,se=oe.indexOf,n={},i=n.toString,ue=n.hasOwnProperty,o=ue.toString,a=o.call(Object),le={},v=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},y=function(e){return null!=e&&e===e.window},C=ie.document,u={type:!0,src:!0,nonce:!0,noModule:!0};function m(e,t,n){var r,i,o=(n=n||C).createElement("script");if(o.text=e,t)for(r in u)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function x(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[i.call(e)]||"object":typeof e}var t="3.7.1",l=/HTML$/i,ce=function(e,t){return new ce.fn.init(e,t)};function c(e){var t=!!e&&"length"in e&&e.length,n=x(e);return!v(e)&&!y(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}function fe(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}ce.fn=ce.prototype={jquery:t,constructor:ce,length:0,toArray:function(){return ae.call(this)},get:function(e){return null==e?ae.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=ce.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return ce.each(this,e)},map:function(n){return this.pushStack(ce.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(ae.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},even:function(){return this.pushStack(ce.grep(this,function(e,t){return(t+1)%2}))},odd:function(){return this.pushStack(ce.grep(this,function(e,t){return t%2}))},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:s,sort:oe.sort,splice:oe.splice},ce.extend=ce.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||v(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(ce.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||ce.isPlainObject(n)?n:{},i=!1,a[t]=ce.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},ce.extend({expando:"jQuery"+(t+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==i.call(e))&&(!(t=r(e))||"function"==typeof(n=ue.call(t,"constructor")&&t.constructor)&&o.call(n)===a)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t,n){m(e,{nonce:t&&t.nonce},n)},each:function(e,t){var n,r=0;if(c(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},text:function(e){var t,n="",r=0,i=e.nodeType;if(!i)while(t=e[r++])n+=ce.text(t);return 1===i||11===i?e.textContent:9===i?e.documentElement.textContent:3===i||4===i?e.nodeValue:n},makeArray:function(e,t){var n=t||[];return null!=e&&(c(Object(e))?ce.merge(n,"string"==typeof e?[e]:e):s.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:se.call(t,e,n)},isXMLDoc:function(e){var t=e&&e.namespaceURI,n=e&&(e.ownerDocument||e).documentElement;return!l.test(t||n&&n.nodeName||"HTML")},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(c(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g(a)},guid:1,support:le}),"function"==typeof Symbol&&(ce.fn[Symbol.iterator]=oe[Symbol.iterator]),ce.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var pe=oe.pop,de=oe.sort,he=oe.splice,ge="[\\x20\\t\\r\\n\\f]",ve=new RegExp("^"+ge+"+|((?:^|[^\\\\])(?:\\\\.)*)"+ge+"+$","g");ce.contains=function(e,t){var n=t&&t.parentNode;return e===n||!(!n||1!==n.nodeType||!(e.contains?e.contains(n):e.compareDocumentPosition&&16&e.compareDocumentPosition(n)))};var f=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\x80-\uFFFF\w-]/g;function p(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e}ce.escapeSelector=function(e){return(e+"").replace(f,p)};var ye=C,me=s;!function(){var e,b,w,o,a,T,r,C,d,i,k=me,S=ce.expando,E=0,n=0,s=W(),c=W(),u=W(),h=W(),l=function(e,t){return e===t&&(a=!0),0},f="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",t="(?:\\\\[\\da-fA-F]{1,6}"+ge+"?|\\\\[^\\r\\n\\f]|[\\w-]|[^\0-\\x7f])+",p="\\["+ge+"*("+t+")(?:"+ge+"*([*^$|!~]?=)"+ge+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+t+"))|)"+ge+"*\\]",g=":("+t+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+p+")*)|.*)\\)|)",v=new RegExp(ge+"+","g"),y=new RegExp("^"+ge+"*,"+ge+"*"),m=new RegExp("^"+ge+"*([>+~]|"+ge+")"+ge+"*"),x=new RegExp(ge+"|>"),j=new RegExp(g),A=new RegExp("^"+t+"$"),D={ID:new RegExp("^#("+t+")"),CLASS:new RegExp("^\\.("+t+")"),TAG:new RegExp("^("+t+"|[*])"),ATTR:new RegExp("^"+p),PSEUDO:new RegExp("^"+g),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+ge+"*(even|odd|(([+-]|)(\\d*)n|)"+ge+"*(?:([+-]|)"+ge+"*(\\d+)|))"+ge+"*\\)|)","i"),bool:new RegExp("^(?:"+f+")$","i"),needsContext:new RegExp("^"+ge+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+ge+"*((?:-\\d)?\\d*)"+ge+"*\\)|)(?=[^-]|$)","i")},N=/^(?:input|select|textarea|button)$/i,q=/^h\d$/i,L=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,H=/[+~]/,O=new RegExp("\\\\[\\da-fA-F]{1,6}"+ge+"?|\\\\([^\\r\\n\\f])","g"),P=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},M=function(){V()},R=J(function(e){return!0===e.disabled&&fe(e,"fieldset")},{dir:"parentNode",next:"legend"});try{k.apply(oe=ae.call(ye.childNodes),ye.childNodes),oe[ye.childNodes.length].nodeType}catch(e){k={apply:function(e,t){me.apply(e,ae.call(t))},call:function(e){me.apply(e,ae.call(arguments,1))}}}function I(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(V(e),e=e||T,C)){if(11!==p&&(u=L.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return k.call(n,a),n}else if(f&&(a=f.getElementById(i))&&I.contains(e,a)&&a.id===i)return k.call(n,a),n}else{if(u[2])return k.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&e.getElementsByClassName)return k.apply(n,e.getElementsByClassName(i)),n}if(!(h[t+" "]||d&&d.test(t))){if(c=t,f=e,1===p&&(x.test(t)||m.test(t))){(f=H.test(t)&&U(e.parentNode)||e)==e&&le.scope||((s=e.getAttribute("id"))?s=ce.escapeSelector(s):e.setAttribute("id",s=S)),o=(l=Y(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+Q(l[o]);c=l.join(",")}try{return k.apply(n,f.querySelectorAll(c)),n}catch(e){h(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return re(t.replace(ve,"$1"),e,n,r)}function W(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function F(e){return e[S]=!0,e}function $(e){var t=T.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function B(t){return function(e){return fe(e,"input")&&e.type===t}}function _(t){return function(e){return(fe(e,"input")||fe(e,"button"))&&e.type===t}}function z(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&R(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function X(a){return F(function(o){return o=+o,F(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function U(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}function V(e){var t,n=e?e.ownerDocument||e:ye;return n!=T&&9===n.nodeType&&n.documentElement&&(r=(T=n).documentElement,C=!ce.isXMLDoc(T),i=r.matches||r.webkitMatchesSelector||r.msMatchesSelector,r.msMatchesSelector&&ye!=T&&(t=T.defaultView)&&t.top!==t&&t.addEventListener("unload",M),le.getById=$(function(e){return r.appendChild(e).id=ce.expando,!T.getElementsByName||!T.getElementsByName(ce.expando).length}),le.disconnectedMatch=$(function(e){return i.call(e,"*")}),le.scope=$(function(){return T.querySelectorAll(":scope")}),le.cssHas=$(function(){try{return T.querySelector(":has(*,:jqfake)"),!1}catch(e){return!0}}),le.getById?(b.filter.ID=function(e){var t=e.replace(O,P);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&C){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(O,P);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&C){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):t.querySelectorAll(e)},b.find.CLASS=function(e,t){if("undefined"!=typeof t.getElementsByClassName&&C)return t.getElementsByClassName(e)},d=[],$(function(e){var t;r.appendChild(e).innerHTML="<a id='"+S+"' href='' disabled='disabled'></a><select id='"+S+"-\r\\' disabled='disabled'><option selected=''></option></select>",e.querySelectorAll("[selected]").length||d.push("\\["+ge+"*(?:value|"+f+")"),e.querySelectorAll("[id~="+S+"-]").length||d.push("~="),e.querySelectorAll("a#"+S+"+*").length||d.push(".#.+[+~]"),e.querySelectorAll(":checked").length||d.push(":checked"),(t=T.createElement("input")).setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),r.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&d.push(":enabled",":disabled"),(t=T.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||d.push("\\["+ge+"*name"+ge+"*="+ge+"*(?:''|\"\")")}),le.cssHas||d.push(":has"),d=d.length&&new RegExp(d.join("|")),l=function(e,t){if(e===t)return a=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!le.sortDetached&&t.compareDocumentPosition(e)===n?e===T||e.ownerDocument==ye&&I.contains(ye,e)?-1:t===T||t.ownerDocument==ye&&I.contains(ye,t)?1:o?se.call(o,e)-se.call(o,t):0:4&n?-1:1)}),T}for(e in I.matches=function(e,t){return I(e,null,null,t)},I.matchesSelector=function(e,t){if(V(e),C&&!h[t+" "]&&(!d||!d.test(t)))try{var n=i.call(e,t);if(n||le.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){h(t,!0)}return 0<I(t,T,null,[e]).length},I.contains=function(e,t){return(e.ownerDocument||e)!=T&&V(e),ce.contains(e,t)},I.attr=function(e,t){(e.ownerDocument||e)!=T&&V(e);var n=b.attrHandle[t.toLowerCase()],r=n&&ue.call(b.attrHandle,t.toLowerCase())?n(e,t,!C):void 0;return void 0!==r?r:e.getAttribute(t)},I.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},ce.uniqueSort=function(e){var t,n=[],r=0,i=0;if(a=!le.sortStable,o=!le.sortStable&&ae.call(e,0),de.call(e,l),a){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)he.call(e,n[r],1)}return o=null,e},ce.fn.uniqueSort=function(){return this.pushStack(ce.uniqueSort(ae.apply(this)))},(b=ce.expr={cacheLength:50,createPseudo:F,match:D,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(O,P),e[3]=(e[3]||e[4]||e[5]||"").replace(O,P),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||I.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&I.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return D.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&j.test(n)&&(t=Y(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(O,P).toLowerCase();return"*"===e?function(){return!0}:function(e){return fe(e,t)}},CLASS:function(e){var t=s[e+" "];return t||(t=new RegExp("(^|"+ge+")"+e+"("+ge+"|$)"))&&s(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=I.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(v," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(d,e,t,h,g){var v="nth"!==d.slice(0,3),y="last"!==d.slice(-4),m="of-type"===e;return 1===h&&0===g?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u=v!==y?"nextSibling":"previousSibling",l=e.parentNode,c=m&&e.nodeName.toLowerCase(),f=!n&&!m,p=!1;if(l){if(v){while(u){o=e;while(o=o[u])if(m?fe(o,c):1===o.nodeType)return!1;s=u="only"===d&&!s&&"nextSibling"}return!0}if(s=[y?l.firstChild:l.lastChild],y&&f){p=(a=(r=(i=l[S]||(l[S]={}))[d]||[])[0]===E&&r[1])&&r[2],o=a&&l.childNodes[a];while(o=++a&&o&&o[u]||(p=a=0)||s.pop())if(1===o.nodeType&&++p&&o===e){i[d]=[E,a,p];break}}else if(f&&(p=a=(r=(i=e[S]||(e[S]={}))[d]||[])[0]===E&&r[1]),!1===p)while(o=++a&&o&&o[u]||(p=a=0)||s.pop())if((m?fe(o,c):1===o.nodeType)&&++p&&(f&&((i=o[S]||(o[S]={}))[d]=[E,p]),o===e))break;return(p-=g)===h||p%h==0&&0<=p/h}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||I.error("unsupported pseudo: "+e);return a[S]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?F(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=se.call(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:F(function(e){var r=[],i=[],s=ne(e.replace(ve,"$1"));return s[S]?F(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:F(function(t){return function(e){return 0<I(t,e).length}}),contains:F(function(t){return t=t.replace(O,P),function(e){return-1<(e.textContent||ce.text(e)).indexOf(t)}}),lang:F(function(n){return A.test(n||"")||I.error("unsupported lang: "+n),n=n.replace(O,P).toLowerCase(),function(e){var t;do{if(t=C?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=ie.location&&ie.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===r},focus:function(e){return e===function(){try{return T.activeElement}catch(e){}}()&&T.hasFocus()&&!!(e.type||e.href||~e.tabIndex)},enabled:z(!1),disabled:z(!0),checked:function(e){return fe(e,"input")&&!!e.checked||fe(e,"option")&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return q.test(e.nodeName)},input:function(e){return N.test(e.nodeName)},button:function(e){return fe(e,"input")&&"button"===e.type||fe(e,"button")},text:function(e){var t;return fe(e,"input")&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:X(function(){return[0]}),last:X(function(e,t){return[t-1]}),eq:X(function(e,t,n){return[n<0?n+t:n]}),even:X(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:X(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:X(function(e,t,n){var r;for(r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:X(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=B(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=_(e);function G(){}function Y(e,t){var n,r,i,o,a,s,u,l=c[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=y.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=m.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace(ve," ")}),a=a.slice(n.length)),b.filter)!(r=D[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?I.error(e):c(e,s).slice(0)}function Q(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function J(a,e,t){var s=e.dir,u=e.next,l=u||s,c=t&&"parentNode"===l,f=n++;return e.first?function(e,t,n){while(e=e[s])if(1===e.nodeType||c)return a(e,t,n);return!1}:function(e,t,n){var r,i,o=[E,f];if(n){while(e=e[s])if((1===e.nodeType||c)&&a(e,t,n))return!0}else while(e=e[s])if(1===e.nodeType||c)if(i=e[S]||(e[S]={}),u&&fe(e,u))e=e[s]||e;else{if((r=i[l])&&r[0]===E&&r[1]===f)return o[2]=r[2];if((i[l]=o)[2]=a(e,t,n))return!0}return!1}}function K(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Z(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function ee(d,h,g,v,y,e){return v&&!v[S]&&(v=ee(v)),y&&!y[S]&&(y=ee(y,e)),F(function(e,t,n,r){var i,o,a,s,u=[],l=[],c=t.length,f=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)I(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),p=!d||!e&&h?f:Z(f,u,d,n,r);if(g?g(p,s=y||(e?d:c||v)?[]:t,n,r):s=p,v){i=Z(s,l),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(s[l[o]]=!(p[l[o]]=a))}if(e){if(y||d){if(y){i=[],o=s.length;while(o--)(a=s[o])&&i.push(p[o]=a);y(null,s=[],i,r)}o=s.length;while(o--)(a=s[o])&&-1<(i=y?se.call(e,a):u[o])&&(e[i]=!(t[i]=a))}}else s=Z(s===t?s.splice(c,s.length):s),y?y(null,t,s,r):k.apply(t,s)})}function te(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=J(function(e){return e===i},a,!0),l=J(function(e){return-1<se.call(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!=w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[J(K(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[S]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return ee(1<s&&K(c),1<s&&Q(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace(ve,"$1"),t,s<n&&te(e.slice(s,n)),n<r&&te(e=e.slice(n)),n<r&&Q(e))}c.push(t)}return K(c)}function ne(e,t){var n,v,y,m,x,r,i=[],o=[],a=u[e+" "];if(!a){t||(t=Y(e)),n=t.length;while(n--)(a=te(t[n]))[S]?i.push(a):o.push(a);(a=u(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=E+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t==T||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument==T||(V(o),n=!C);while(s=v[a++])if(s(o,t||T,n)){k.call(r,o);break}i&&(E=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=pe.call(r));f=Z(f)}k.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&ce.uniqueSort(r)}return i&&(E=h,w=p),c},m?F(r):r))).selector=e}return a}function re(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&Y(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&C&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(O,P),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=D.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(O,P),H.test(o[0].type)&&U(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&Q(o)))return k.apply(n,r),n;break}}}return(l||ne(e,c))(r,t,!C,n,!t||H.test(e)&&U(t.parentNode)||t),n}G.prototype=b.filters=b.pseudos,b.setFilters=new G,le.sortStable=S.split("").sort(l).join("")===S,V(),le.sortDetached=$(function(e){return 1&e.compareDocumentPosition(T.createElement("fieldset"))}),ce.find=I,ce.expr[":"]=ce.expr.pseudos,ce.unique=ce.uniqueSort,I.compile=ne,I.select=re,I.setDocument=V,I.tokenize=Y,I.escape=ce.escapeSelector,I.getText=ce.text,I.isXML=ce.isXMLDoc,I.selectors=ce.expr,I.support=ce.support,I.uniqueSort=ce.uniqueSort}();var d=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&ce(e).is(n))break;r.push(e)}return r},h=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},b=ce.expr.match.needsContext,w=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function T(e,n,r){return v(n)?ce.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?ce.grep(e,function(e){return e===n!==r}):"string"!=typeof n?ce.grep(e,function(e){return-1<se.call(n,e)!==r}):ce.filter(n,e,r)}ce.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?ce.find.matchesSelector(r,e)?[r]:[]:ce.find.matches(e,ce.grep(t,function(e){return 1===e.nodeType}))},ce.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(ce(e).filter(function(){for(t=0;t<r;t++)if(ce.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)ce.find(e,i[t],n);return 1<r?ce.uniqueSort(n):n},filter:function(e){return this.pushStack(T(this,e||[],!1))},not:function(e){return this.pushStack(T(this,e||[],!0))},is:function(e){return!!T(this,"string"==typeof e&&b.test(e)?ce(e):e||[],!1).length}});var k,S=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(ce.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||k,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:S.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof ce?t[0]:t,ce.merge(this,ce.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:C,!0)),w.test(r[1])&&ce.isPlainObject(t))for(r in t)v(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=C.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):v(e)?void 0!==n.ready?n.ready(e):e(ce):ce.makeArray(e,this)}).prototype=ce.fn,k=ce(C);var E=/^(?:parents|prev(?:Until|All))/,j={children:!0,contents:!0,next:!0,prev:!0};function A(e,t){while((e=e[t])&&1!==e.nodeType);return e}ce.fn.extend({has:function(e){var t=ce(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(ce.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&ce(e);if(!b.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&ce.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?ce.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?se.call(ce(e),this[0]):se.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(ce.uniqueSort(ce.merge(this.get(),ce(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),ce.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return d(e,"parentNode")},parentsUntil:function(e,t,n){return d(e,"parentNode",n)},next:function(e){return A(e,"nextSibling")},prev:function(e){return A(e,"previousSibling")},nextAll:function(e){return d(e,"nextSibling")},prevAll:function(e){return d(e,"previousSibling")},nextUntil:function(e,t,n){return d(e,"nextSibling",n)},prevUntil:function(e,t,n){return d(e,"previousSibling",n)},siblings:function(e){return h((e.parentNode||{}).firstChild,e)},children:function(e){return h(e.firstChild)},contents:function(e){return null!=e.contentDocument&&r(e.contentDocument)?e.contentDocument:(fe(e,"template")&&(e=e.content||e),ce.merge([],e.childNodes))}},function(r,i){ce.fn[r]=function(e,t){var n=ce.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=ce.filter(t,n)),1<this.length&&(j[r]||ce.uniqueSort(n),E.test(r)&&n.reverse()),this.pushStack(n)}});var D=/[^\x20\t\r\n\f]+/g;function N(e){return e}function q(e){throw e}function L(e,t,n,r){var i;try{e&&v(i=e.promise)?i.call(e).done(t).fail(n):e&&v(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}ce.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},ce.each(e.match(D)||[],function(e,t){n[t]=!0}),n):ce.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){ce.each(e,function(e,t){v(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==x(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return ce.each(arguments,function(e,t){var n;while(-1<(n=ce.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<ce.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},ce.extend({Deferred:function(e){var o=[["notify","progress",ce.Callbacks("memory"),ce.Callbacks("memory"),2],["resolve","done",ce.Callbacks("once memory"),ce.Callbacks("once memory"),0,"resolved"],["reject","fail",ce.Callbacks("once memory"),ce.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return ce.Deferred(function(r){ce.each(o,function(e,t){var n=v(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&v(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,v(t)?s?t.call(e,l(u,o,N,s),l(u,o,q,s)):(u++,t.call(e,l(u,o,N,s),l(u,o,q,s),l(u,o,N,o.notifyWith))):(a!==N&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){ce.Deferred.exceptionHook&&ce.Deferred.exceptionHook(e,t.error),u<=i+1&&(a!==q&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(ce.Deferred.getErrorHook?t.error=ce.Deferred.getErrorHook():ce.Deferred.getStackHook&&(t.error=ce.Deferred.getStackHook()),ie.setTimeout(t))}}return ce.Deferred(function(e){o[0][3].add(l(0,e,v(r)?r:N,e.notifyWith)),o[1][3].add(l(0,e,v(t)?t:N)),o[2][3].add(l(0,e,v(n)?n:q))}).promise()},promise:function(e){return null!=e?ce.extend(e,a):a}},s={};return ce.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=ae.call(arguments),o=ce.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?ae.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(L(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||v(i[t]&&i[t].then)))return o.then();while(t--)L(i[t],a(t),o.reject);return o.promise()}});var H=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;ce.Deferred.exceptionHook=function(e,t){ie.console&&ie.console.warn&&e&&H.test(e.name)&&ie.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},ce.readyException=function(e){ie.setTimeout(function(){throw e})};var O=ce.Deferred();function P(){C.removeEventListener("DOMContentLoaded",P),ie.removeEventListener("load",P),ce.ready()}ce.fn.ready=function(e){return O.then(e)["catch"](function(e){ce.readyException(e)}),this},ce.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--ce.readyWait:ce.isReady)||(ce.isReady=!0)!==e&&0<--ce.readyWait||O.resolveWith(C,[ce])}}),ce.ready.then=O.then,"complete"===C.readyState||"loading"!==C.readyState&&!C.documentElement.doScroll?ie.setTimeout(ce.ready):(C.addEventListener("DOMContentLoaded",P),ie.addEventListener("load",P));var M=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===x(n))for(s in i=!0,n)M(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,v(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(ce(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},R=/^-ms-/,I=/-([a-z])/g;function W(e,t){return t.toUpperCase()}function F(e){return e.replace(R,"ms-").replace(I,W)}var $=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function B(){this.expando=ce.expando+B.uid++}B.uid=1,B.prototype={cache:function(e){var t=e[this.expando];return t||(t={},$(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[F(t)]=n;else for(r in t)i[F(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][F(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(F):(t=F(t))in r?[t]:t.match(D)||[]).length;while(n--)delete r[t[n]]}(void 0===t||ce.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!ce.isEmptyObject(t)}};var _=new B,z=new B,X=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,U=/[A-Z]/g;function V(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(U,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:X.test(i)?JSON.parse(i):i)}catch(e){}z.set(e,t,n)}else n=void 0;return n}ce.extend({hasData:function(e){return z.hasData(e)||_.hasData(e)},data:function(e,t,n){return z.access(e,t,n)},removeData:function(e,t){z.remove(e,t)},_data:function(e,t,n){return _.access(e,t,n)},_removeData:function(e,t){_.remove(e,t)}}),ce.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=z.get(o),1===o.nodeType&&!_.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=F(r.slice(5)),V(o,r,i[r]));_.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){z.set(this,n)}):M(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=z.get(o,n))?t:void 0!==(t=V(o,n))?t:void 0;this.each(function(){z.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){z.remove(this,e)})}}),ce.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=_.get(e,t),n&&(!r||Array.isArray(n)?r=_.access(e,t,ce.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=ce.queue(e,t),r=n.length,i=n.shift(),o=ce._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){ce.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return _.get(e,n)||_.access(e,n,{empty:ce.Callbacks("once memory").add(function(){_.remove(e,[t+"queue",n])})})}}),ce.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?ce.queue(this[0],t):void 0===n?this:this.each(function(){var e=ce.queue(this,t,n);ce._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&ce.dequeue(this,t)})},dequeue:function(e){return this.each(function(){ce.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=ce.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=_.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var G=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,Y=new RegExp("^(?:([+-])=|)("+G+")([a-z%]*)$","i"),Q=["Top","Right","Bottom","Left"],J=C.documentElement,K=function(e){return ce.contains(e.ownerDocument,e)},Z={composed:!0};J.getRootNode&&(K=function(e){return ce.contains(e.ownerDocument,e)||e.getRootNode(Z)===e.ownerDocument});var ee=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&K(e)&&"none"===ce.css(e,"display")};function te(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return ce.css(e,t,"")},u=s(),l=n&&n[3]||(ce.cssNumber[t]?"":"px"),c=e.nodeType&&(ce.cssNumber[t]||"px"!==l&&+u)&&Y.exec(ce.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)ce.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,ce.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ne={};function re(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=_.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&ee(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ne[s])||(o=a.body.appendChild(a.createElement(s)),u=ce.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ne[s]=u)))):"none"!==n&&(l[c]="none",_.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}ce.fn.extend({show:function(){return re(this,!0)},hide:function(){return re(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){ee(this)?ce(this).show():ce(this).hide()})}});var xe,be,we=/^(?:checkbox|radio)$/i,Te=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,Ce=/^$|^module$|\/(?:java|ecma)script/i;xe=C.createDocumentFragment().appendChild(C.createElement("div")),(be=C.createElement("input")).setAttribute("type","radio"),be.setAttribute("checked","checked"),be.setAttribute("name","t"),xe.appendChild(be),le.checkClone=xe.cloneNode(!0).cloneNode(!0).lastChild.checked,xe.innerHTML="<textarea>x</textarea>",le.noCloneChecked=!!xe.cloneNode(!0).lastChild.defaultValue,xe.innerHTML="<option></option>",le.option=!!xe.lastChild;var ke={thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function Se(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&fe(e,t)?ce.merge([e],n):n}function Ee(e,t){for(var n=0,r=e.length;n<r;n++)_.set(e[n],"globalEval",!t||_.get(t[n],"globalEval"))}ke.tbody=ke.tfoot=ke.colgroup=ke.caption=ke.thead,ke.th=ke.td,le.option||(ke.optgroup=ke.option=[1,"<select multiple='multiple'>","</select>"]);var je=/<|&#?\w+;/;function Ae(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===x(o))ce.merge(p,o.nodeType?[o]:o);else if(je.test(o)){a=a||f.appendChild(t.createElement("div")),s=(Te.exec(o)||["",""])[1].toLowerCase(),u=ke[s]||ke._default,a.innerHTML=u[1]+ce.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;ce.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<ce.inArray(o,r))i&&i.push(o);else if(l=K(o),a=Se(f.appendChild(o),"script"),l&&Ee(a),n){c=0;while(o=a[c++])Ce.test(o.type||"")&&n.push(o)}return f}var De=/^([^.]*)(?:\.(.+)|)/;function Ne(){return!0}function qe(){return!1}function Le(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)Le(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=qe;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return ce().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=ce.guid++)),e.each(function(){ce.event.add(this,t,i,r,n)})}function He(e,r,t){t?(_.set(e,r,!1),ce.event.add(e,r,{namespace:!1,handler:function(e){var t,n=_.get(this,r);if(1&e.isTrigger&&this[r]){if(n)(ce.event.special[r]||{}).delegateType&&e.stopPropagation();else if(n=ae.call(arguments),_.set(this,r,n),this[r](),t=_.get(this,r),_.set(this,r,!1),n!==t)return e.stopImmediatePropagation(),e.preventDefault(),t}else n&&(_.set(this,r,ce.event.trigger(n[0],n.slice(1),this)),e.stopPropagation(),e.isImmediatePropagationStopped=Ne)}})):void 0===_.get(e,r)&&ce.event.add(e,r,Ne)}ce.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=_.get(t);if($(t)){n.handler&&(n=(o=n).handler,i=o.selector),i&&ce.find.matchesSelector(J,i),n.guid||(n.guid=ce.guid++),(u=v.events)||(u=v.events=Object.create(null)),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof ce&&ce.event.triggered!==e.type?ce.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(D)||[""]).length;while(l--)d=g=(s=De.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=ce.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=ce.event.special[d]||{},c=ce.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&ce.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),ce.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=_.hasData(e)&&_.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(D)||[""]).length;while(l--)if(d=g=(s=De.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=ce.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||ce.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)ce.event.remove(e,d+t[l],n,r,!0);ce.isEmptyObject(u)&&_.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=new Array(arguments.length),u=ce.event.fix(e),l=(_.get(this,"events")||Object.create(null))[u.type]||[],c=ce.event.special[u.type]||{};for(s[0]=u,t=1;t<arguments.length;t++)s[t]=arguments[t];if(u.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,u)){a=ce.event.handlers.call(this,u,l),t=0;while((i=a[t++])&&!u.isPropagationStopped()){u.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!u.isImmediatePropagationStopped())u.rnamespace&&!1!==o.namespace&&!u.rnamespace.test(o.namespace)||(u.handleObj=o,u.data=o.data,void 0!==(r=((ce.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,s))&&!1===(u.result=r)&&(u.preventDefault(),u.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,u),u.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<ce(i,this).index(l):ce.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(ce.Event.prototype,t,{enumerable:!0,configurable:!0,get:v(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[ce.expando]?e:new ce.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return we.test(t.type)&&t.click&&fe(t,"input")&&He(t,"click",!0),!1},trigger:function(e){var t=this||e;return we.test(t.type)&&t.click&&fe(t,"input")&&He(t,"click"),!0},_default:function(e){var t=e.target;return we.test(t.type)&&t.click&&fe(t,"input")&&_.get(t,"click")||fe(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},ce.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},ce.Event=function(e,t){if(!(this instanceof ce.Event))return new ce.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?Ne:qe,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&ce.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[ce.expando]=!0},ce.Event.prototype={constructor:ce.Event,isDefaultPrevented:qe,isPropagationStopped:qe,isImmediatePropagationStopped:qe,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=Ne,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=Ne,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=Ne,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},ce.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:!0},ce.event.addProp),ce.each({focus:"focusin",blur:"focusout"},function(r,i){function o(e){if(C.documentMode){var t=_.get(this,"handle"),n=ce.event.fix(e);n.type="focusin"===e.type?"focus":"blur",n.isSimulated=!0,t(e),n.target===n.currentTarget&&t(n)}else ce.event.simulate(i,e.target,ce.event.fix(e))}ce.event.special[r]={setup:function(){var e;if(He(this,r,!0),!C.documentMode)return!1;(e=_.get(this,i))||this.addEventListener(i,o),_.set(this,i,(e||0)+1)},trigger:function(){return He(this,r),!0},teardown:function(){var e;if(!C.documentMode)return!1;(e=_.get(this,i)-1)?_.set(this,i,e):(this.removeEventListener(i,o),_.remove(this,i))},_default:function(e){return _.get(e.target,r)},delegateType:i},ce.event.special[i]={setup:function(){var e=this.ownerDocument||this.document||this,t=C.documentMode?this:e,n=_.get(t,i);n||(C.documentMode?this.addEventListener(i,o):e.addEventListener(r,o,!0)),_.set(t,i,(n||0)+1)},teardown:function(){var e=this.ownerDocument||this.document||this,t=C.documentMode?this:e,n=_.get(t,i)-1;n?_.set(t,i,n):(C.documentMode?this.removeEventListener(i,o):e.removeEventListener(r,o,!0),_.remove(t,i))}}}),ce.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){ce.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||ce.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),ce.fn.extend({on:function(e,t,n,r){return Le(this,e,t,n,r)},one:function(e,t,n,r){return Le(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,ce(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=qe),this.each(function(){ce.event.remove(this,e,n,t)})}});var Oe=/<script|<style|<link/i,Pe=/checked\s*(?:[^=]|=\s*.checked.)/i,Me=/^\s*<!\[CDATA\[|\]\]>\s*$/g;function Re(e,t){return fe(e,"table")&&fe(11!==t.nodeType?t:t.firstChild,"tr")&&ce(e).children("tbody")[0]||e}function Ie(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function We(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Fe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(_.hasData(e)&&(s=_.get(e).events))for(i in _.remove(t,"handle events"),s)for(n=0,r=s[i].length;n<r;n++)ce.event.add(t,i,s[i][n]);z.hasData(e)&&(o=z.access(e),a=ce.extend({},o),z.set(t,a))}}function $e(n,r,i,o){r=g(r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=v(d);if(h||1<f&&"string"==typeof d&&!le.checkClone&&Pe.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),$e(t,r,i,o)});if(f&&(t=(e=Ae(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=ce.map(Se(e,"script"),Ie)).length;c<f;c++)u=e,c!==p&&(u=ce.clone(u,!0,!0),s&&ce.merge(a,Se(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,ce.map(a,We),c=0;c<s;c++)u=a[c],Ce.test(u.type||"")&&!_.access(u,"globalEval")&&ce.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?ce._evalUrl&&!u.noModule&&ce._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")},l):m(u.textContent.replace(Me,""),u,l))}return n}function Be(e,t,n){for(var r,i=t?ce.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||ce.cleanData(Se(r)),r.parentNode&&(n&&K(r)&&Ee(Se(r,"script")),r.parentNode.removeChild(r));return e}ce.extend({htmlPrefilter:function(e){return e},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=K(e);if(!(le.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||ce.isXMLDoc(e)))for(a=Se(c),r=0,i=(o=Se(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&we.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||Se(e),a=a||Se(c),r=0,i=o.length;r<i;r++)Fe(o[r],a[r]);else Fe(e,c);return 0<(a=Se(c,"script")).length&&Ee(a,!f&&Se(e,"script")),c},cleanData:function(e){for(var t,n,r,i=ce.event.special,o=0;void 0!==(n=e[o]);o++)if($(n)){if(t=n[_.expando]){if(t.events)for(r in t.events)i[r]?ce.event.remove(n,r):ce.removeEvent(n,r,t.handle);n[_.expando]=void 0}n[z.expando]&&(n[z.expando]=void 0)}}}),ce.fn.extend({detach:function(e){return Be(this,e,!0)},remove:function(e){return Be(this,e)},text:function(e){return M(this,function(e){return void 0===e?ce.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return $e(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||Re(this,e).appendChild(e)})},prepend:function(){return $e(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Re(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return $e(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return $e(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(ce.cleanData(Se(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return ce.clone(this,e,t)})},html:function(e){return M(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!Oe.test(e)&&!ke[(Te.exec(e)||["",""])[1].toLowerCase()]){e=ce.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(ce.cleanData(Se(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return $e(this,arguments,function(e){var t=this.parentNode;ce.inArray(this,n)<0&&(ce.cleanData(Se(this)),t&&t.replaceChild(e,this))},n)}}),ce.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){ce.fn[e]=function(e){for(var t,n=[],r=ce(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),ce(r[o])[a](t),s.apply(n,t.get());return this.pushStack(n)}});var _e=new RegExp("^("+G+")(?!px)[a-z%]+$","i"),ze=/^--/,Xe=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=ie),t.getComputedStyle(e)},Ue=function(e,t,n){var r,i,o={};for(i in t)o[i]=e.style[i],e.style[i]=t[i];for(i in r=n.call(e),t)e.style[i]=o[i];return r},Ve=new RegExp(Q.join("|"),"i");function Ge(e,t,n){var r,i,o,a,s=ze.test(t),u=e.style;return(n=n||Xe(e))&&(a=n.getPropertyValue(t)||n[t],s&&a&&(a=a.replace(ve,"$1")||void 0),""!==a||K(e)||(a=ce.style(e,t)),!le.pixelBoxStyles()&&_e.test(a)&&Ve.test(t)&&(r=u.width,i=u.minWidth,o=u.maxWidth,u.minWidth=u.maxWidth=u.width=a,a=n.width,u.width=r,u.minWidth=i,u.maxWidth=o)),void 0!==a?a+"":a}function Ye(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(l){u.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",l.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",J.appendChild(u).appendChild(l);var e=ie.getComputedStyle(l);n="1%"!==e.top,s=12===t(e.marginLeft),l.style.right="60%",o=36===t(e.right),r=36===t(e.width),l.style.position="absolute",i=12===t(l.offsetWidth/3),J.removeChild(u),l=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s,u=C.createElement("div"),l=C.createElement("div");l.style&&(l.style.backgroundClip="content-box",l.cloneNode(!0).style.backgroundClip="",le.clearCloneStyle="content-box"===l.style.backgroundClip,ce.extend(le,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),s},scrollboxSize:function(){return e(),i},reliableTrDimensions:function(){var e,t,n,r;return null==a&&(e=C.createElement("table"),t=C.createElement("tr"),n=C.createElement("div"),e.style.cssText="position:absolute;left:-11111px;border-collapse:separate",t.style.cssText="box-sizing:content-box;border:1px solid",t.style.height="1px",n.style.height="9px",n.style.display="block",J.appendChild(e).appendChild(t).appendChild(n),r=ie.getComputedStyle(t),a=parseInt(r.height,10)+parseInt(r.borderTopWidth,10)+parseInt(r.borderBottomWidth,10)===t.offsetHeight,J.removeChild(e)),a}}))}();var Qe=["Webkit","Moz","ms"],Je=C.createElement("div").style,Ke={};function Ze(e){var t=ce.cssProps[e]||Ke[e];return t||(e in Je?e:Ke[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=Qe.length;while(n--)if((e=Qe[n]+t)in Je)return e}(e)||e)}var et=/^(none|table(?!-c[ea]).+)/,tt={position:"absolute",visibility:"hidden",display:"block"},nt={letterSpacing:"0",fontWeight:"400"};function rt(e,t,n){var r=Y.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function it(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0,l=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(l+=ce.css(e,n+Q[a],!0,i)),r?("content"===n&&(u-=ce.css(e,"padding"+Q[a],!0,i)),"margin"!==n&&(u-=ce.css(e,"border"+Q[a]+"Width",!0,i))):(u+=ce.css(e,"padding"+Q[a],!0,i),"padding"!==n?u+=ce.css(e,"border"+Q[a]+"Width",!0,i):s+=ce.css(e,"border"+Q[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u+l}function ot(e,t,n){var r=Xe(e),i=(!le.boxSizingReliable()||n)&&"border-box"===ce.css(e,"boxSizing",!1,r),o=i,a=Ge(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if(_e.test(a)){if(!n)return a;a="auto"}return(!le.boxSizingReliable()&&i||!le.reliableTrDimensions()&&fe(e,"tr")||"auto"===a||!parseFloat(a)&&"inline"===ce.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===ce.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+it(e,t,n||(i?"border":"content"),o,r,a)+"px"}function at(e,t,n,r,i){return new at.prototype.init(e,t,n,r,i)}ce.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Ge(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,aspectRatio:!0,borderImageSlice:!0,columnCount:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,scale:!0,widows:!0,zIndex:!0,zoom:!0,fillOpacity:!0,floodOpacity:!0,stopOpacity:!0,strokeMiterlimit:!0,strokeOpacity:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=F(t),u=ze.test(t),l=e.style;if(u||(t=Ze(s)),a=ce.cssHooks[t]||ce.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=Y.exec(n))&&i[1]&&(n=te(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(ce.cssNumber[s]?"":"px")),le.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=F(t);return ze.test(t)||(t=Ze(s)),(a=ce.cssHooks[t]||ce.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=Ge(e,t,r)),"normal"===i&&t in nt&&(i=nt[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),ce.each(["height","width"],function(e,u){ce.cssHooks[u]={get:function(e,t,n){if(t)return!et.test(ce.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?ot(e,u,n):Ue(e,tt,function(){return ot(e,u,n)})},set:function(e,t,n){var r,i=Xe(e),o=!le.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===ce.css(e,"boxSizing",!1,i),s=n?it(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-it(e,u,"border",!1,i)-.5)),s&&(r=Y.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=ce.css(e,u)),rt(0,t,s)}}}),ce.cssHooks.marginLeft=Ye(le.reliableMarginLeft,function(e,t){if(t)return(parseFloat(Ge(e,"marginLeft"))||e.getBoundingClientRect().left-Ue(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),ce.each({margin:"",padding:"",border:"Width"},function(i,o){ce.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+Q[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(ce.cssHooks[i+o].set=rt)}),ce.fn.extend({css:function(e,t){return M(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Xe(e),i=t.length;a<i;a++)o[t[a]]=ce.css(e,t[a],!1,r);return o}return void 0!==n?ce.style(e,t,n):ce.css(e,t)},e,t,1<arguments.length)}}),((ce.Tween=at).prototype={constructor:at,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||ce.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(ce.cssNumber[n]?"":"px")},cur:function(){var e=at.propHooks[this.prop];return e&&e.get?e.get(this):at.propHooks._default.get(this)},run:function(e){var t,n=at.propHooks[this.prop];return this.options.duration?this.pos=t=ce.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):at.propHooks._default.set(this),this}}).init.prototype=at.prototype,(at.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=ce.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){ce.fx.step[e.prop]?ce.fx.step[e.prop](e):1!==e.elem.nodeType||!ce.cssHooks[e.prop]&&null==e.elem.style[Ze(e.prop)]?e.elem[e.prop]=e.now:ce.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=at.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},ce.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},ce.fx=at.prototype.init,ce.fx.step={};var st,ut,lt,ct,ft=/^(?:toggle|show|hide)$/,pt=/queueHooks$/;function dt(){ut&&(!1===C.hidden&&ie.requestAnimationFrame?ie.requestAnimationFrame(dt):ie.setTimeout(dt,ce.fx.interval),ce.fx.tick())}function ht(){return ie.setTimeout(function(){st=void 0}),st=Date.now()}function gt(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=Q[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function vt(e,t,n){for(var r,i=(yt.tweeners[t]||[]).concat(yt.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function yt(o,e,t){var n,a,r=0,i=yt.prefilters.length,s=ce.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=st||ht(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:ce.extend({},e),opts:ce.extend(!0,{specialEasing:{},easing:ce.easing._default},t),originalProperties:e,originalOptions:t,startTime:st||ht(),duration:t.duration,tweens:[],createTween:function(e,t){var n=ce.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=F(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=ce.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=yt.prefilters[r].call(l,o,c,l.opts))return v(n.stop)&&(ce._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return ce.map(c,vt,l),v(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),ce.fx.timer(ce.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}ce.Animation=ce.extend(yt,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return te(n.elem,e,Y.exec(t),n),n}]},tweener:function(e,t){v(e)?(t=e,e=["*"]):e=e.match(D);for(var n,r=0,i=e.length;r<i;r++)n=e[r],yt.tweeners[n]=yt.tweeners[n]||[],yt.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&ee(e),v=_.get(e,"fxshow");for(r in n.queue||(null==(a=ce._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,ce.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],ft.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||ce.style(e,r)}if((u=!ce.isEmptyObject(t))||!ce.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=_.get(e,"display")),"none"===(c=ce.css(e,"display"))&&(l?c=l:(re([e],!0),l=e.style.display||l,c=ce.css(e,"display"),re([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===ce.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=_.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&re([e],!0),p.done(function(){for(r in g||re([e]),_.remove(e,"fxshow"),d)ce.style(e,r,d[r])})),u=vt(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?yt.prefilters.unshift(e):yt.prefilters.push(e)}}),ce.speed=function(e,t,n){var r=e&&"object"==typeof e?ce.extend({},e):{complete:n||!n&&t||v(e)&&e,duration:e,easing:n&&t||t&&!v(t)&&t};return ce.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in ce.fx.speeds?r.duration=ce.fx.speeds[r.duration]:r.duration=ce.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){v(r.old)&&r.old.call(this),r.queue&&ce.dequeue(this,r.queue)},r},ce.fn.extend({fadeTo:function(e,t,n,r){return this.filter(ee).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=ce.isEmptyObject(t),o=ce.speed(e,n,r),a=function(){var e=yt(this,ce.extend({},t),o);(i||_.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=ce.timers,r=_.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&pt.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||ce.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=_.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=ce.timers,o=n?n.length:0;for(t.finish=!0,ce.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),ce.each(["toggle","show","hide"],function(e,r){var i=ce.fn[r];ce.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(gt(r,!0),e,t,n)}}),ce.each({slideDown:gt("show"),slideUp:gt("hide"),slideToggle:gt("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){ce.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),ce.timers=[],ce.fx.tick=function(){var e,t=0,n=ce.timers;for(st=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||ce.fx.stop(),st=void 0},ce.fx.timer=function(e){ce.timers.push(e),ce.fx.start()},ce.fx.interval=13,ce.fx.start=function(){ut||(ut=!0,dt())},ce.fx.stop=function(){ut=null},ce.fx.speeds={slow:600,fast:200,_default:400},ce.fn.delay=function(r,e){return r=ce.fx&&ce.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=ie.setTimeout(e,r);t.stop=function(){ie.clearTimeout(n)}})},lt=C.createElement("input"),ct=C.createElement("select").appendChild(C.createElement("option")),lt.type="checkbox",le.checkOn=""!==lt.value,le.optSelected=ct.selected,(lt=C.createElement("input")).value="t",lt.type="radio",le.radioValue="t"===lt.value;var mt,xt=ce.expr.attrHandle;ce.fn.extend({attr:function(e,t){return M(this,ce.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){ce.removeAttr(this,e)})}}),ce.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?ce.prop(e,t,n):(1===o&&ce.isXMLDoc(e)||(i=ce.attrHooks[t.toLowerCase()]||(ce.expr.match.bool.test(t)?mt:void 0)),void 0!==n?null===n?void ce.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=ce.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!le.radioValue&&"radio"===t&&fe(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(D);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),mt={set:function(e,t,n){return!1===t?ce.removeAttr(e,n):e.setAttribute(n,n),n}},ce.each(ce.expr.match.bool.source.match(/\w+/g),function(e,t){var a=xt[t]||ce.find.attr;xt[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=xt[o],xt[o]=r,r=null!=a(e,t,n)?o:null,xt[o]=i),r}});var bt=/^(?:input|select|textarea|button)$/i,wt=/^(?:a|area)$/i;function Tt(e){return(e.match(D)||[]).join(" ")}function Ct(e){return e.getAttribute&&e.getAttribute("class")||""}function kt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(D)||[]}ce.fn.extend({prop:function(e,t){return M(this,ce.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[ce.propFix[e]||e]})}}),ce.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&ce.isXMLDoc(e)||(t=ce.propFix[t]||t,i=ce.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=ce.find.attr(e,"tabindex");return t?parseInt(t,10):bt.test(e.nodeName)||wt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),le.optSelected||(ce.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),ce.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){ce.propFix[this.toLowerCase()]=this}),ce.fn.extend({addClass:function(t){var e,n,r,i,o,a;return v(t)?this.each(function(e){ce(this).addClass(t.call(this,e,Ct(this)))}):(e=kt(t)).length?this.each(function(){if(r=Ct(this),n=1===this.nodeType&&" "+Tt(r)+" "){for(o=0;o<e.length;o++)i=e[o],n.indexOf(" "+i+" ")<0&&(n+=i+" ");a=Tt(n),r!==a&&this.setAttribute("class",a)}}):this},removeClass:function(t){var e,n,r,i,o,a;return v(t)?this.each(function(e){ce(this).removeClass(t.call(this,e,Ct(this)))}):arguments.length?(e=kt(t)).length?this.each(function(){if(r=Ct(this),n=1===this.nodeType&&" "+Tt(r)+" "){for(o=0;o<e.length;o++){i=e[o];while(-1<n.indexOf(" "+i+" "))n=n.replace(" "+i+" "," ")}a=Tt(n),r!==a&&this.setAttribute("class",a)}}):this:this.attr("class","")},toggleClass:function(t,n){var e,r,i,o,a=typeof t,s="string"===a||Array.isArray(t);return v(t)?this.each(function(e){ce(this).toggleClass(t.call(this,e,Ct(this),n),n)}):"boolean"==typeof n&&s?n?this.addClass(t):this.removeClass(t):(e=kt(t),this.each(function(){if(s)for(o=ce(this),i=0;i<e.length;i++)r=e[i],o.hasClass(r)?o.removeClass(r):o.addClass(r);else void 0!==t&&"boolean"!==a||((r=Ct(this))&&_.set(this,"__className__",r),this.setAttribute&&this.setAttribute("class",r||!1===t?"":_.get(this,"__className__")||""))}))},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+Tt(Ct(n))+" ").indexOf(t))return!0;return!1}});var St=/\r/g;ce.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=v(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,ce(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=ce.map(t,function(e){return null==e?"":e+""})),(r=ce.valHooks[this.type]||ce.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=ce.valHooks[t.type]||ce.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(St,""):null==e?"":e:void 0}}),ce.extend({valHooks:{option:{get:function(e){var t=ce.find.attr(e,"value");return null!=t?t:Tt(ce.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!fe(n.parentNode,"optgroup"))){if(t=ce(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=ce.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<ce.inArray(ce.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),ce.each(["radio","checkbox"],function(){ce.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<ce.inArray(ce(e).val(),t)}},le.checkOn||(ce.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})});var Et=ie.location,jt={guid:Date.now()},At=/\?/;ce.parseXML=function(e){var t,n;if(!e||"string"!=typeof e)return null;try{t=(new ie.DOMParser).parseFromString(e,"text/xml")}catch(e){}return n=t&&t.getElementsByTagName("parsererror")[0],t&&!n||ce.error("Invalid XML: "+(n?ce.map(n.childNodes,function(e){return e.textContent}).join("\n"):e)),t};var Dt=/^(?:focusinfocus|focusoutblur)$/,Nt=function(e){e.stopPropagation()};ce.extend(ce.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||C],d=ue.call(e,"type")?e.type:e,h=ue.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||C,3!==n.nodeType&&8!==n.nodeType&&!Dt.test(d+ce.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[ce.expando]?e:new ce.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:ce.makeArray(t,[e]),c=ce.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!y(n)){for(s=c.delegateType||d,Dt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||C)&&p.push(a.defaultView||a.parentWindow||ie)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(_.get(o,"events")||Object.create(null))[e.type]&&_.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&$(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!$(n)||u&&v(n[d])&&!y(n)&&((a=n[u])&&(n[u]=null),ce.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,Nt),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,Nt),ce.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=ce.extend(new ce.Event,n,{type:e,isSimulated:!0});ce.event.trigger(r,null,t)}}),ce.fn.extend({trigger:function(e,t){return this.each(function(){ce.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return ce.event.trigger(e,t,n,!0)}});var qt=/\[\]$/,Lt=/\r?\n/g,Ht=/^(?:submit|button|image|reset|file)$/i,Ot=/^(?:input|select|textarea|keygen)/i;function Pt(n,e,r,i){var t;if(Array.isArray(e))ce.each(e,function(e,t){r||qt.test(n)?i(n,t):Pt(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==x(e))i(n,e);else for(t in e)Pt(n+"["+t+"]",e[t],r,i)}ce.param=function(e,t){var n,r=[],i=function(e,t){var n=v(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!ce.isPlainObject(e))ce.each(e,function(){i(this.name,this.value)});else for(n in e)Pt(n,e[n],t,i);return r.join("&")},ce.fn.extend({serialize:function(){return ce.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=ce.prop(this,"elements");return e?ce.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!ce(this).is(":disabled")&&Ot.test(this.nodeName)&&!Ht.test(e)&&(this.checked||!we.test(e))}).map(function(e,t){var n=ce(this).val();return null==n?null:Array.isArray(n)?ce.map(n,function(e){return{name:t.name,value:e.replace(Lt,"\r\n")}}):{name:t.name,value:n.replace(Lt,"\r\n")}}).get()}});var Mt=/%20/g,Rt=/#.*$/,It=/([?&])_=[^&]*/,Wt=/^(.*?):[ \t]*([^\r\n]*)$/gm,Ft=/^(?:GET|HEAD)$/,$t=/^\/\//,Bt={},_t={},zt="*/".concat("*"),Xt=C.createElement("a");function Ut(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(D)||[];if(v(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function Vt(t,i,o,a){var s={},u=t===_t;function l(e){var r;return s[e]=!0,ce.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function Gt(e,t){var n,r,i=ce.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&ce.extend(!0,e,r),e}Xt.href=Et.href,ce.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Et.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(Et.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":zt,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":ce.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?Gt(Gt(e,ce.ajaxSettings),t):Gt(ce.ajaxSettings,e)},ajaxPrefilter:Ut(Bt),ajaxTransport:Ut(_t),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=ce.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?ce(y):ce.event,x=ce.Deferred(),b=ce.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=Wt.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||Et.href)+"").replace($t,Et.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(D)||[""],null==v.crossDomain){r=C.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Xt.protocol+"//"+Xt.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=ce.param(v.data,v.traditional)),Vt(Bt,v,t,T),h)return T;for(i in(g=ce.event&&v.global)&&0==ce.active++&&ce.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Ft.test(v.type),f=v.url.replace(Rt,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(Mt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(At.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(It,"$1"),o=(At.test(f)?"&":"?")+"_="+jt.guid+++o),v.url=f+o),v.ifModified&&(ce.lastModified[f]&&T.setRequestHeader("If-Modified-Since",ce.lastModified[f]),ce.etag[f]&&T.setRequestHeader("If-None-Match",ce.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+zt+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=Vt(_t,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=ie.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&ie.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),!i&&-1<ce.inArray("script",v.dataTypes)&&ce.inArray("json",v.dataTypes)<0&&(v.converters["text script"]=function(){}),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(ce.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(ce.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--ce.active||ce.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return ce.get(e,t,n,"json")},getScript:function(e,t){return ce.get(e,void 0,t,"script")}}),ce.each(["get","post"],function(e,i){ce[i]=function(e,t,n,r){return v(t)&&(r=r||n,n=t,t=void 0),ce.ajax(ce.extend({url:e,type:i,dataType:r,data:t,success:n},ce.isPlainObject(e)&&e))}}),ce.ajaxPrefilter(function(e){var t;for(t in e.headers)"content-type"===t.toLowerCase()&&(e.contentType=e.headers[t]||"")}),ce._evalUrl=function(e,t,n){return ce.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){ce.globalEval(e,t,n)}})},ce.fn.extend({wrapAll:function(e){var t;return this[0]&&(v(e)&&(e=e.call(this[0])),t=ce(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return v(n)?this.each(function(e){ce(this).wrapInner(n.call(this,e))}):this.each(function(){var e=ce(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=v(t);return this.each(function(e){ce(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){ce(this).replaceWith(this.childNodes)}),this}}),ce.expr.pseudos.hidden=function(e){return!ce.expr.pseudos.visible(e)},ce.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},ce.ajaxSettings.xhr=function(){try{return new ie.XMLHttpRequest}catch(e){}};var Yt={0:200,1223:204},Qt=ce.ajaxSettings.xhr();le.cors=!!Qt&&"withCredentials"in Qt,le.ajax=Qt=!!Qt,ce.ajaxTransport(function(i){var o,a;if(le.cors||Qt&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(Yt[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&ie.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),ce.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),ce.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return ce.globalEval(e),e}}}),ce.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),ce.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=ce("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),C.head.appendChild(r[0])},abort:function(){i&&i()}}});var Jt,Kt=[],Zt=/(=)\?(?=&|$)|\?\?/;ce.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Kt.pop()||ce.expando+"_"+jt.guid++;return this[e]=!0,e}}),ce.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Zt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Zt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=v(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Zt,"$1"+r):!1!==e.jsonp&&(e.url+=(At.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||ce.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=ie[r],ie[r]=function(){o=arguments},n.always(function(){void 0===i?ce(ie).removeProp(r):ie[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Kt.push(r)),o&&v(i)&&i(o[0]),o=i=void 0}),"script"}),le.createHTMLDocument=((Jt=C.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===Jt.childNodes.length),ce.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(le.createHTMLDocument?((r=(t=C.implementation.createHTMLDocument("")).createElement("base")).href=C.location.href,t.head.appendChild(r)):t=C),o=!n&&[],(i=w.exec(e))?[t.createElement(i[1])]:(i=Ae([e],t,o),o&&o.length&&ce(o).remove(),ce.merge([],i.childNodes)));var r,i,o},ce.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=Tt(e.slice(s)),e=e.slice(0,s)),v(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&ce.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?ce("<div>").append(ce.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},ce.expr.pseudos.animated=function(t){return ce.grep(ce.timers,function(e){return t===e.elem}).length},ce.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=ce.css(e,"position"),c=ce(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=ce.css(e,"top"),u=ce.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),v(t)&&(t=t.call(e,n,ce.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},ce.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){ce.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===ce.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===ce.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=ce(e).offset()).top+=ce.css(e,"borderTopWidth",!0),i.left+=ce.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-ce.css(r,"marginTop",!0),left:t.left-i.left-ce.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===ce.css(e,"position"))e=e.offsetParent;return e||J})}}),ce.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;ce.fn[t]=function(e){return M(this,function(e,t,n){var r;if(y(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),ce.each(["top","left"],function(e,n){ce.cssHooks[n]=Ye(le.pixelPosition,function(e,t){if(t)return t=Ge(e,n),_e.test(t)?ce(e).position()[n]+"px":t})}),ce.each({Height:"height",Width:"width"},function(a,s){ce.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){ce.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return M(this,function(e,t,n){var r;return y(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?ce.css(e,t,i):ce.style(e,t,n,i)},s,n?e:void 0,n)}})}),ce.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){ce.fn[t]=function(e){return this.on(t,e)}}),ce.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.on("mouseenter",e).on("mouseleave",t||e)}}),ce.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){ce.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}});var en=/^[\s\uFEFF\xA0]+|([^\s\uFEFF\xA0])[\s\uFEFF\xA0]+$/g;ce.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),v(e))return r=ae.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(ae.call(arguments)))}).guid=e.guid=e.guid||ce.guid++,i},ce.holdReady=function(e){e?ce.readyWait++:ce.ready(!0)},ce.isArray=Array.isArray,ce.parseJSON=JSON.parse,ce.nodeName=fe,ce.isFunction=v,ce.isWindow=y,ce.camelCase=F,ce.type=x,ce.now=Date.now,ce.isNumeric=function(e){var t=ce.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},ce.trim=function(e){return null==e?"":(e+"").replace(en,"$1")},"function"==typeof define&&define.amd&&define("jquery",[],function(){return ce});var tn=ie.jQuery,nn=ie.$;return ce.noConflict=function(e){return ie.$===ce&&(ie.$=nn),e&&ie.jQuery===ce&&(ie.jQuery=tn),ce},"undefined"==typeof e&&(ie.jQuery=ie.$=ce),ce});
diff --git a/interface/js/lib/jquery.stickytabs.min.js b/interface/js/lib/jquery.stickytabs.min.js
new file mode 100644
index 0000000..96967ea
--- /dev/null
+++ b/interface/js/lib/jquery.stickytabs.min.js
@@ -0,0 +1,9 @@
+/*!
+ * jQuery Plugin: Sticky Tabs
+ *
+ * @author Aidan Lister <aidan@php.net>
+ * @version 1.2.1
+ * @see https://github.com/aidanlister/jquery-stickytabs
+ * @license MIT
+ */
+!function(c){c.fn.stickyTabs=function(o){function t(){var o="href"==n.selectorAttribute?window.location.hash:window.location.hash.substring(1),t=o?"a["+n.selectorAttribute+'="'+o+'"]':n.initialTab;c(t,i).tab("show"),setTimeout(l,1)}var i=this,n=c.extend({getHashCallback:function(o,t){return o},selectorAttribute:"href",backToTop:!1,initialTab:c("li.active > a",i)},o),l=function(){!0===n.backToTop&&window.scrollTo(0,0)};return t(),c(window).on("hashchange",t),c("a",i).on("click",function(o){var t=this.href.split("#")[1];!function(o){history&&history.pushState?history.pushState(null,null,window.location.pathname+window.location.search+"#"+o):(scrollV=document.body.scrollTop,scrollH=document.body.scrollLeft,window.location.hash=o,document.body.scrollTop=scrollV,document.body.scrollLeft=scrollH)}(n.getHashCallback(t,this)),setTimeout(l,1)}),this}}(jQuery); \ No newline at end of file
diff --git a/interface/js/lib/nprogress.min.js b/interface/js/lib/nprogress.min.js
new file mode 100644
index 0000000..4173215
--- /dev/null
+++ b/interface/js/lib/nprogress.min.js
@@ -0,0 +1,4 @@
+/* NProgress, (c) 2013, 2014 Rico Sta. Cruz - http://ricostacruz.com/nprogress
+ * @version 0.2.0
+ * @license MIT */
+!function(e,n){"function"==typeof define&&define.amd?define(n):"object"==typeof exports?module.exports=n():e.NProgress=n()}(this,function(){var n,t,o={version:"0.2.0"},a=o.settings={minimum:.08,easing:"ease",positionUsing:"",speed:200,trickle:!0,trickleRate:.02,trickleSpeed:800,showSpinner:!0,barSelector:'[role="bar"]',spinnerSelector:'[role="spinner"]',parent:"body",template:'<div class="bar" role="bar"><div class="peg"></div></div><div class="spinner" role="spinner"><div class="spinner-icon"></div></div>'};function u(e,n,t){return e<n?n:t<e?t:e}function c(e){return 100*(-1+e)}o.configure=function(e){var n,t;for(n in e)void 0!==(t=e[n])&&e.hasOwnProperty(n)&&(a[n]=t);return this},o.status=null,o.set=function(n){var e=o.isStarted();n=u(n,a.minimum,1),o.status=1===n?null:n;var t=o.render(!e),r=t.querySelector(a.barSelector),i=a.speed,s=a.easing;return t.offsetWidth,l(function(e){""===a.positionUsing&&(a.positionUsing=o.getPositioningCSS()),f(r,function(e,n,t){var r;r="translate3d"===a.positionUsing?{transform:"translate3d("+c(e)+"%,0,0)"}:"translate"===a.positionUsing?{transform:"translate("+c(e)+"%,0)"}:{"margin-left":c(e)+"%"};return r.transition="all "+n+"ms "+t,r}(n,i,s)),1===n?(f(t,{transition:"none",opacity:1}),t.offsetWidth,setTimeout(function(){f(t,{transition:"all "+i+"ms linear",opacity:0}),setTimeout(function(){o.remove(),e()},i)},i)):setTimeout(e,i)}),this},o.isStarted=function(){return"number"==typeof o.status},o.start=function(){o.status||o.set(0);var e=function(){setTimeout(function(){o.status&&(o.trickle(),e())},a.trickleSpeed)};return a.trickle&&e(),this},o.done=function(e){return e||o.status?o.inc(.3+.5*Math.random()).set(1):this},o.inc=function(e){var n=o.status;return n?("number"!=typeof e&&(e=(1-n)*u(Math.random()*n,.1,.95)),n=u(n+e,0,.994),o.set(n)):o.start()},o.trickle=function(){return o.inc(Math.random()*a.trickleRate)},t=n=0,o.promise=function(e){return e&&"resolved"!==e.state()&&(0===t&&o.start(),n++,t++,e.always(function(){0==--t?(n=0,o.done()):o.set((n-t)/n)})),this},o.render=function(e){if(o.isRendered())return document.getElementById("nprogress");d(document.documentElement,"nprogress-busy");var n=document.createElement("div");n.id="nprogress",n.innerHTML=a.template;var t,r=n.querySelector(a.barSelector),i=e?"-100":c(o.status||0),s=document.querySelector(a.parent);return f(r,{transition:"all 0 linear",transform:"translate3d("+i+"%,0,0)"}),a.showSpinner||(t=n.querySelector(a.spinnerSelector))&&m(t),s!=document.body&&d(s,"nprogress-custom-parent"),s.appendChild(n),n},o.remove=function(){r(document.documentElement,"nprogress-busy"),r(document.querySelector(a.parent),"nprogress-custom-parent");var e=document.getElementById("nprogress");e&&m(e)},o.isRendered=function(){return!!document.getElementById("nprogress")},o.getPositioningCSS=function(){var e=document.body.style,n="WebkitTransform"in e?"Webkit":"MozTransform"in e?"Moz":"msTransform"in e?"ms":"OTransform"in e?"O":"";return n+"Perspective"in e?"translate3d":n+"Transform"in e?"translate":"margin"};var l=function(){var n=[];function t(){var e=n.shift();e&&e(t)}return function(e){n.push(e),1==n.length&&t()}}(),f=function(){var s=["Webkit","O","Moz","ms"],n={};function r(e){return e=e.replace(/^-ms-/,"ms-").replace(/-([\da-z])/gi,function(e,n){return n.toUpperCase()}),n[e]||(n[e]=function(e){var n=document.body.style;if(e in n)return e;for(var t,r=s.length,i=e.charAt(0).toUpperCase()+e.slice(1);r--;)if((t=s[r]+i)in n)return t;return e}(e))}function o(e,n,t){n=r(n),e.style[n]=t}return function(e,n){var t,r,i=arguments;if(2==i.length)for(t in n)void 0!==(r=n[t])&&n.hasOwnProperty(t)&&o(e,t,r);else o(e,i[1],i[2])}}();function i(e,n){return 0<=("string"==typeof e?e:s(e)).indexOf(" "+n+" ")}function d(e,n){var t=s(e),r=t+n;i(t,n)||(e.className=r.substring(1))}function r(e,n){var t,r=s(e);i(e,n)&&(t=r.replace(" "+n+" "," "),e.className=t.substring(1,t.length-1))}function s(e){return(" "+(e.className||"")+" ").replace(/\s+/gi," ")}function m(e){e&&e.parentNode&&e.parentNode.removeChild(e)}return o});
diff --git a/interface/js/lib/prism.js b/interface/js/lib/prism.js
new file mode 100644
index 0000000..ca0eb98
--- /dev/null
+++ b/interface/js/lib/prism.js
@@ -0,0 +1,5 @@
+/* PrismJS 1.29.0
+https://prismjs.com/download.html#themes=prism-okaidia&languages=clike&plugins=show-invisibles */
+var _self="undefined"!=typeof window?window:"undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?self:{},Prism=function(e){var n=/(?:^|\s)lang(?:uage)?-([\w-]+)(?=\s|$)/i,t=0,r={},a={manual:e.Prism&&e.Prism.manual,disableWorkerMessageHandler:e.Prism&&e.Prism.disableWorkerMessageHandler,util:{encode:function e(n){return n instanceof i?new i(n.type,e(n.content),n.alias):Array.isArray(n)?n.map(e):n.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/\u00a0/g," ")},type:function(e){return Object.prototype.toString.call(e).slice(8,-1)},objId:function(e){return e.__id||Object.defineProperty(e,"__id",{value:++t}),e.__id},clone:function e(n,t){var r,i;switch(t=t||{},a.util.type(n)){case"Object":if(i=a.util.objId(n),t[i])return t[i];for(var l in r={},t[i]=r,n)n.hasOwnProperty(l)&&(r[l]=e(n[l],t));return r;case"Array":return i=a.util.objId(n),t[i]?t[i]:(r=[],t[i]=r,n.forEach((function(n,a){r[a]=e(n,t)})),r);default:return n}},getLanguage:function(e){for(;e;){var t=n.exec(e.className);if(t)return t[1].toLowerCase();e=e.parentElement}return"none"},setLanguage:function(e,t){e.className=e.className.replace(RegExp(n,"gi"),""),e.classList.add("language-"+t)},currentScript:function(){if("undefined"==typeof document)return null;if("currentScript"in document)return document.currentScript;try{throw new Error}catch(r){var e=(/at [^(\r\n]*\((.*):[^:]+:[^:]+\)$/i.exec(r.stack)||[])[1];if(e){var n=document.getElementsByTagName("script");for(var t in n)if(n[t].src==e)return n[t]}return null}},isActive:function(e,n,t){for(var r="no-"+n;e;){var a=e.classList;if(a.contains(n))return!0;if(a.contains(r))return!1;e=e.parentElement}return!!t}},languages:{plain:r,plaintext:r,text:r,txt:r,extend:function(e,n){var t=a.util.clone(a.languages[e]);for(var r in n)t[r]=n[r];return t},insertBefore:function(e,n,t,r){var i=(r=r||a.languages)[e],l={};for(var o in i)if(i.hasOwnProperty(o)){if(o==n)for(var s in t)t.hasOwnProperty(s)&&(l[s]=t[s]);t.hasOwnProperty(o)||(l[o]=i[o])}var u=r[e];return r[e]=l,a.languages.DFS(a.languages,(function(n,t){t===u&&n!=e&&(this[n]=l)})),l},DFS:function e(n,t,r,i){i=i||{};var l=a.util.objId;for(var o in n)if(n.hasOwnProperty(o)){t.call(n,o,n[o],r||o);var s=n[o],u=a.util.type(s);"Object"!==u||i[l(s)]?"Array"!==u||i[l(s)]||(i[l(s)]=!0,e(s,t,o,i)):(i[l(s)]=!0,e(s,t,null,i))}}},plugins:{},highlightAll:function(e,n){a.highlightAllUnder(document,e,n)},highlightAllUnder:function(e,n,t){var r={callback:t,container:e,selector:'code[class*="language-"], [class*="language-"] code, code[class*="lang-"], [class*="lang-"] code'};a.hooks.run("before-highlightall",r),r.elements=Array.prototype.slice.apply(r.container.querySelectorAll(r.selector)),a.hooks.run("before-all-elements-highlight",r);for(var i,l=0;i=r.elements[l++];)a.highlightElement(i,!0===n,r.callback)},highlightElement:function(n,t,r){var i=a.util.getLanguage(n),l=a.languages[i];a.util.setLanguage(n,i);var o=n.parentElement;o&&"pre"===o.nodeName.toLowerCase()&&a.util.setLanguage(o,i);var s={element:n,language:i,grammar:l,code:n.textContent};function u(e){s.highlightedCode=e,a.hooks.run("before-insert",s),s.element.innerHTML=s.highlightedCode,a.hooks.run("after-highlight",s),a.hooks.run("complete",s),r&&r.call(s.element)}if(a.hooks.run("before-sanity-check",s),(o=s.element.parentElement)&&"pre"===o.nodeName.toLowerCase()&&!o.hasAttribute("tabindex")&&o.setAttribute("tabindex","0"),!s.code)return a.hooks.run("complete",s),void(r&&r.call(s.element));if(a.hooks.run("before-highlight",s),s.grammar)if(t&&e.Worker){var c=new Worker(a.filename);c.onmessage=function(e){u(e.data)},c.postMessage(JSON.stringify({language:s.language,code:s.code,immediateClose:!0}))}else u(a.highlight(s.code,s.grammar,s.language));else u(a.util.encode(s.code))},highlight:function(e,n,t){var r={code:e,grammar:n,language:t};if(a.hooks.run("before-tokenize",r),!r.grammar)throw new Error('The language "'+r.language+'" has no grammar.');return r.tokens=a.tokenize(r.code,r.grammar),a.hooks.run("after-tokenize",r),i.stringify(a.util.encode(r.tokens),r.language)},tokenize:function(e,n){var t=n.rest;if(t){for(var r in t)n[r]=t[r];delete n.rest}var a=new s;return u(a,a.head,e),o(e,a,n,a.head,0),function(e){for(var n=[],t=e.head.next;t!==e.tail;)n.push(t.value),t=t.next;return n}(a)},hooks:{all:{},add:function(e,n){var t=a.hooks.all;t[e]=t[e]||[],t[e].push(n)},run:function(e,n){var t=a.hooks.all[e];if(t&&t.length)for(var r,i=0;r=t[i++];)r(n)}},Token:i};function i(e,n,t,r){this.type=e,this.content=n,this.alias=t,this.length=0|(r||"").length}function l(e,n,t,r){e.lastIndex=n;var a=e.exec(t);if(a&&r&&a[1]){var i=a[1].length;a.index+=i,a[0]=a[0].slice(i)}return a}function o(e,n,t,r,s,g){for(var f in t)if(t.hasOwnProperty(f)&&t[f]){var h=t[f];h=Array.isArray(h)?h:[h];for(var d=0;d<h.length;++d){if(g&&g.cause==f+","+d)return;var v=h[d],p=v.inside,m=!!v.lookbehind,y=!!v.greedy,k=v.alias;if(y&&!v.pattern.global){var x=v.pattern.toString().match(/[imsuy]*$/)[0];v.pattern=RegExp(v.pattern.source,x+"g")}for(var b=v.pattern||v,w=r.next,A=s;w!==n.tail&&!(g&&A>=g.reach);A+=w.value.length,w=w.next){var E=w.value;if(n.length>e.length)return;if(!(E instanceof i)){var P,L=1;if(y){if(!(P=l(b,A,e,m))||P.index>=e.length)break;var S=P.index,O=P.index+P[0].length,j=A;for(j+=w.value.length;S>=j;)j+=(w=w.next).value.length;if(A=j-=w.value.length,w.value instanceof i)continue;for(var C=w;C!==n.tail&&(j<O||"string"==typeof C.value);C=C.next)L++,j+=C.value.length;L--,E=e.slice(A,j),P.index-=A}else if(!(P=l(b,0,E,m)))continue;S=P.index;var N=P[0],_=E.slice(0,S),M=E.slice(S+N.length),W=A+E.length;g&&W>g.reach&&(g.reach=W);var z=w.prev;if(_&&(z=u(n,z,_),A+=_.length),c(n,z,L),w=u(n,z,new i(f,p?a.tokenize(N,p):N,k,N)),M&&u(n,w,M),L>1){var I={cause:f+","+d,reach:W};o(e,n,t,w.prev,A,I),g&&I.reach>g.reach&&(g.reach=I.reach)}}}}}}function s(){var e={value:null,prev:null,next:null},n={value:null,prev:e,next:null};e.next=n,this.head=e,this.tail=n,this.length=0}function u(e,n,t){var r=n.next,a={value:t,prev:n,next:r};return n.next=a,r.prev=a,e.length++,a}function c(e,n,t){for(var r=n.next,a=0;a<t&&r!==e.tail;a++)r=r.next;n.next=r,r.prev=n,e.length-=a}if(e.Prism=a,i.stringify=function e(n,t){if("string"==typeof n)return n;if(Array.isArray(n)){var r="";return n.forEach((function(n){r+=e(n,t)})),r}var i={type:n.type,content:e(n.content,t),tag:"span",classes:["token",n.type],attributes:{},language:t},l=n.alias;l&&(Array.isArray(l)?Array.prototype.push.apply(i.classes,l):i.classes.push(l)),a.hooks.run("wrap",i);var o="";for(var s in i.attributes)o+=" "+s+'="'+(i.attributes[s]||"").replace(/"/g,"&quot;")+'"';return"<"+i.tag+' class="'+i.classes.join(" ")+'"'+o+">"+i.content+"</"+i.tag+">"},!e.document)return e.addEventListener?(a.disableWorkerMessageHandler||e.addEventListener("message",(function(n){var t=JSON.parse(n.data),r=t.language,i=t.code,l=t.immediateClose;e.postMessage(a.highlight(i,a.languages[r],r)),l&&e.close()}),!1),a):a;var g=a.util.currentScript();function f(){a.manual||a.highlightAll()}if(g&&(a.filename=g.src,g.hasAttribute("data-manual")&&(a.manual=!0)),!a.manual){var h=document.readyState;"loading"===h||"interactive"===h&&g&&g.defer?document.addEventListener("DOMContentLoaded",f):window.requestAnimationFrame?window.requestAnimationFrame(f):window.setTimeout(f,16)}return a}(_self);"undefined"!=typeof module&&module.exports&&(module.exports=Prism),"undefined"!=typeof global&&(global.Prism=Prism);
+Prism.languages.clike={comment:[{pattern:/(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,lookbehind:!0,greedy:!0},{pattern:/(^|[^\\:])\/\/.*/,lookbehind:!0,greedy:!0}],string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"class-name":{pattern:/(\b(?:class|extends|implements|instanceof|interface|new|trait)\s+|\bcatch\s+\()[\w.\\]+/i,lookbehind:!0,inside:{punctuation:/[.\\]/}},keyword:/\b(?:break|catch|continue|do|else|finally|for|function|if|in|instanceof|new|null|return|throw|try|while)\b/,boolean:/\b(?:false|true)\b/,function:/\b\w+(?=\()/,number:/\b0x[\da-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?/i,operator:/[<>]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/,punctuation:/[{}[\];(),.:]/};
+!function(){if("undefined"!=typeof Prism){var r={tab:/\t/,crlf:/\r\n/,lf:/\n/,cr:/\r/,space:/ /};Prism.hooks.add("before-highlight",(function(r){i(r.grammar)}))}function e(r,a){var n=r[a];switch(Prism.util.type(n)){case"RegExp":var t={};r[a]={pattern:n,inside:t},i(t);break;case"Array":for(var f=0,s=n.length;f<s;f++)e(n,f);break;default:i(t=n.inside||(n.inside={}))}}function i(a){if(a&&!a.tab){for(var n in r)r.hasOwnProperty(n)&&(a[n]=r[n]);for(var n in a)a.hasOwnProperty(n)&&!r[n]&&("rest"===n?i(a.rest):e(a,n))}}}();
diff --git a/interface/js/lib/require.min.js b/interface/js/lib/require.min.js
new file mode 100644
index 0000000..a4203f0
--- /dev/null
+++ b/interface/js/lib/require.min.js
@@ -0,0 +1,5 @@
+/** vim: et:ts=4:sw=4:sts=4
+ * @license RequireJS 2.3.6 Copyright jQuery Foundation and other contributors.
+ * Released under MIT license, https://github.com/requirejs/requirejs/blob/master/LICENSE
+ */
+var requirejs,require,define;!function(global,setTimeout){var req,s,head,baseElement,dataMain,src,interactiveScript,currentlyAddingScript,mainScript,subPath,version="2.3.6",commentRegExp=/\/\*[\s\S]*?\*\/|([^:"'=]|^)\/\/.*$/gm,cjsRequireRegExp=/[^.]\s*require\s*\(\s*["']([^'"\s]+)["']\s*\)/g,jsSuffixRegExp=/\.js$/,currDirRegExp=/^\.\//,op=Object.prototype,ostring=op.toString,hasOwn=op.hasOwnProperty,isBrowser=!("undefined"==typeof window||"undefined"==typeof navigator||!window.document),isWebWorker=!isBrowser&&"undefined"!=typeof importScripts,readyRegExp=isBrowser&&"PLAYSTATION 3"===navigator.platform?/^complete$/:/^(complete|loaded)$/,defContextName="_",isOpera="undefined"!=typeof opera&&"[object Opera]"===opera.toString(),contexts={},cfg={},globalDefQueue=[],useInteractive=!1;function commentReplace(e,t){return t||""}function isFunction(e){return"[object Function]"===ostring.call(e)}function isArray(e){return"[object Array]"===ostring.call(e)}function each(e,t){var i;if(e)for(i=0;i<e.length&&(!e[i]||!t(e[i],i,e));i+=1);}function eachReverse(e,t){var i;if(e)for(i=e.length-1;-1<i&&(!e[i]||!t(e[i],i,e));i-=1);}function hasProp(e,t){return hasOwn.call(e,t)}function getOwn(e,t){return hasProp(e,t)&&e[t]}function eachProp(e,t){var i;for(i in e)if(hasProp(e,i)&&t(e[i],i))break}function mixin(i,e,r,n){return e&&eachProp(e,function(e,t){!r&&hasProp(i,t)||(!n||"object"!=typeof e||!e||isArray(e)||isFunction(e)||e instanceof RegExp?i[t]=e:(i[t]||(i[t]={}),mixin(i[t],e,r,n)))}),i}function bind(e,t){return function(){return t.apply(e,arguments)}}function scripts(){return document.getElementsByTagName("script")}function defaultOnError(e){throw e}function getGlobal(e){if(!e)return e;var t=global;return each(e.split("."),function(e){t=t[e]}),t}function makeError(e,t,i,r){var n=new Error(t+"\nhttps://requirejs.org/docs/errors.html#"+e);return n.requireType=e,n.requireModules=r,i&&(n.originalError=i),n}if(void 0===define){if(void 0!==requirejs){if(isFunction(requirejs))return;cfg=requirejs,requirejs=void 0}void 0===require||isFunction(require)||(cfg=require,require=void 0),req=requirejs=function(e,t,i,r){var n,o,a=defContextName;return isArray(e)||"string"==typeof e||(o=e,isArray(t)?(e=t,t=i,i=r):e=[]),o&&o.context&&(a=o.context),(n=getOwn(contexts,a))||(n=contexts[a]=req.s.newContext(a)),o&&n.configure(o),n.require(e,t,i)},req.config=function(e){return req(e)},req.nextTick=void 0!==setTimeout?function(e){setTimeout(e,4)}:function(e){e()},require||(require=req),req.version=version,req.jsExtRegExp=/^\/|:|\?|\.js$/,req.isBrowser=isBrowser,s=req.s={contexts:contexts,newContext:newContext},req({}),each(["toUrl","undef","defined","specified"],function(t){req[t]=function(){var e=contexts[defContextName];return e.require[t].apply(e,arguments)}}),isBrowser&&(head=s.head=document.getElementsByTagName("head")[0],baseElement=document.getElementsByTagName("base")[0],baseElement&&(head=s.head=baseElement.parentNode)),req.onError=defaultOnError,req.createNode=function(e,t,i){var r=e.xhtml?document.createElementNS("http://www.w3.org/1999/xhtml","html:script"):document.createElement("script");return r.type=e.scriptType||"text/javascript",r.charset="utf-8",r.async=!0,r},req.load=function(t,i,r){var e,n=t&&t.config||{};if(isBrowser)return(e=req.createNode(n,i,r)).setAttribute("data-requirecontext",t.contextName),e.setAttribute("data-requiremodule",i),!e.attachEvent||e.attachEvent.toString&&e.attachEvent.toString().indexOf("[native code")<0||isOpera?(e.addEventListener("load",t.onScriptLoad,!1),e.addEventListener("error",t.onScriptError,!1)):(useInteractive=!0,e.attachEvent("onreadystatechange",t.onScriptLoad)),e.src=r,n.onNodeCreated&&n.onNodeCreated(e,n,i,r),currentlyAddingScript=e,baseElement?head.insertBefore(e,baseElement):head.appendChild(e),currentlyAddingScript=null,e;if(isWebWorker)try{setTimeout(function(){},0),importScripts(r),t.completeLoad(i)}catch(e){t.onError(makeError("importscripts","importScripts failed for "+i+" at "+r,e,[i]))}},isBrowser&&!cfg.skipDataMain&&eachReverse(scripts(),function(e){if(head||(head=e.parentNode),dataMain=e.getAttribute("data-main"))return mainScript=dataMain,cfg.baseUrl||-1!==mainScript.indexOf("!")||(mainScript=(src=mainScript.split("/")).pop(),subPath=src.length?src.join("/")+"/":"./",cfg.baseUrl=subPath),mainScript=mainScript.replace(jsSuffixRegExp,""),req.jsExtRegExp.test(mainScript)&&(mainScript=dataMain),cfg.deps=cfg.deps?cfg.deps.concat(mainScript):[mainScript],!0}),define=function(e,i,t){var r,n;"string"!=typeof e&&(t=i,i=e,e=null),isArray(i)||(t=i,i=null),!i&&isFunction(t)&&(i=[],t.length&&(t.toString().replace(commentRegExp,commentReplace).replace(cjsRequireRegExp,function(e,t){i.push(t)}),i=(1===t.length?["require"]:["require","exports","module"]).concat(i))),useInteractive&&(r=currentlyAddingScript||getInteractiveScript())&&(e||(e=r.getAttribute("data-requiremodule")),n=contexts[r.getAttribute("data-requirecontext")]),n?(n.defQueue.push([e,i,t]),n.defQueueMap[e]=!0):globalDefQueue.push([e,i,t])},define.amd={jQuery:!0},req.exec=function(text){return eval(text)},req(cfg)}function newContext(u){var i,e,l,c,d,g={waitSeconds:7,baseUrl:"./",paths:{},bundles:{},pkgs:{},shim:{},config:{}},p={},f={},r={},h=[],m={},n={},v={},x=1,b=1;function q(e,t,i){var r,n,o,a,s,u,c,d,p,f,l=t&&t.split("/"),h=g.map,m=h&&h["*"];if(e&&(u=(e=e.split("/")).length-1,g.nodeIdCompat&&jsSuffixRegExp.test(e[u])&&(e[u]=e[u].replace(jsSuffixRegExp,"")),"."===e[0].charAt(0)&&l&&(e=l.slice(0,l.length-1).concat(e)),function(e){var t,i;for(t=0;t<e.length;t++)if("."===(i=e[t]))e.splice(t,1),t-=1;else if(".."===i){if(0===t||1===t&&".."===e[2]||".."===e[t-1])continue;0<t&&(e.splice(t-1,2),t-=2)}}(e),e=e.join("/")),i&&h&&(l||m)){e:for(o=(n=e.split("/")).length;0<o;o-=1){if(s=n.slice(0,o).join("/"),l)for(a=l.length;0<a;a-=1)if((r=getOwn(h,l.slice(0,a).join("/")))&&(r=getOwn(r,s))){c=r,d=o;break e}!p&&m&&getOwn(m,s)&&(p=getOwn(m,s),f=o)}!c&&p&&(c=p,d=f),c&&(n.splice(0,d,c),e=n.join("/"))}return getOwn(g.pkgs,e)||e}function E(t){isBrowser&&each(scripts(),function(e){if(e.getAttribute("data-requiremodule")===t&&e.getAttribute("data-requirecontext")===l.contextName)return e.parentNode.removeChild(e),!0})}function w(e){var t=getOwn(g.paths,e);if(t&&isArray(t)&&1<t.length)return t.shift(),l.require.undef(e),l.makeRequire(null,{skipMap:!0})([e]),!0}function y(e){var t,i=e?e.indexOf("!"):-1;return-1<i&&(t=e.substring(0,i),e=e.substring(i+1,e.length)),[t,e]}function S(e,t,i,r){var n,o,a,s,u=null,c=t?t.name:null,d=e,p=!0,f="";return e||(p=!1,e="_@r"+(x+=1)),u=(s=y(e))[0],e=s[1],u&&(u=q(u,c,r),o=getOwn(m,u)),e&&(u?f=i?e:o&&o.normalize?o.normalize(e,function(e){return q(e,c,r)}):-1===e.indexOf("!")?q(e,c,r):e:(u=(s=y(f=q(e,c,r)))[0],f=s[1],i=!0,n=l.nameToUrl(f))),{prefix:u,name:f,parentMap:t,unnormalized:!!(a=!u||o||i?"":"_unnormalized"+(b+=1)),url:n,originalName:d,isDefine:p,id:(u?u+"!"+f:f)+a}}function k(e){var t=e.id,i=getOwn(p,t);return i||(i=p[t]=new l.Module(e)),i}function M(e,t,i){var r=e.id,n=getOwn(p,r);!hasProp(m,r)||n&&!n.defineEmitComplete?(n=k(e)).error&&"error"===t?i(n.error):n.on(t,i):"defined"===t&&i(m[r])}function O(i,e){var t=i.requireModules,r=!1;e?e(i):(each(t,function(e){var t=getOwn(p,e);t&&(t.error=i,t.events.error&&(r=!0,t.emit("error",i)))}),r||req.onError(i))}function j(){globalDefQueue.length&&(each(globalDefQueue,function(e){var t=e[0];"string"==typeof t&&(l.defQueueMap[t]=!0),h.push(e)}),globalDefQueue=[])}function P(e){delete p[e],delete f[e]}function R(){var e,r,t=1e3*g.waitSeconds,n=t&&l.startTime+t<(new Date).getTime(),o=[],a=[],s=!1,u=!0;if(!i){if(i=!0,eachProp(f,function(e){var t=e.map,i=t.id;if(e.enabled&&(t.isDefine||a.push(e),!e.error))if(!e.inited&&n)w(i)?s=r=!0:(o.push(i),E(i));else if(!e.inited&&e.fetched&&t.isDefine&&(s=!0,!t.prefix))return u=!1}),n&&o.length)return(e=makeError("timeout","Load timeout for modules: "+o,null,o)).contextName=l.contextName,O(e);u&&each(a,function(e){!function n(o,a,s){var e=o.map.id;o.error?o.emit("error",o.error):(a[e]=!0,each(o.depMaps,function(e,t){var i=e.id,r=getOwn(p,i);!r||o.depMatched[t]||s[i]||(getOwn(a,i)?(o.defineDep(t,m[i]),o.check()):n(r,a,s))}),s[e]=!0)}(e,{},{})}),n&&!r||!s||!isBrowser&&!isWebWorker||d||(d=setTimeout(function(){d=0,R()},50)),i=!1}}function a(e){hasProp(m,e[0])||k(S(e[0],null,!0)).init(e[1],e[2])}function o(e,t,i,r){e.detachEvent&&!isOpera?r&&e.detachEvent(r,t):e.removeEventListener(i,t,!1)}function s(e){var t=e.currentTarget||e.srcElement;return o(t,l.onScriptLoad,"load","onreadystatechange"),o(t,l.onScriptError,"error"),{node:t,id:t&&t.getAttribute("data-requiremodule")}}function T(){var e;for(j();h.length;){if(null===(e=h.shift())[0])return O(makeError("mismatch","Mismatched anonymous define() module: "+e[e.length-1]));a(e)}l.defQueueMap={}}return c={require:function(e){return e.require?e.require:e.require=l.makeRequire(e.map)},exports:function(e){if(e.usingExports=!0,e.map.isDefine)return e.exports?m[e.map.id]=e.exports:e.exports=m[e.map.id]={}},module:function(e){return e.module?e.module:e.module={id:e.map.id,uri:e.map.url,config:function(){return getOwn(g.config,e.map.id)||{}},exports:e.exports||(e.exports={})}}},(e=function(e){this.events=getOwn(r,e.id)||{},this.map=e,this.shim=getOwn(g.shim,e.id),this.depExports=[],this.depMaps=[],this.depMatched=[],this.pluginMaps={},this.depCount=0}).prototype={init:function(e,t,i,r){r=r||{},this.inited||(this.factory=t,i?this.on("error",i):this.events.error&&(i=bind(this,function(e){this.emit("error",e)})),this.depMaps=e&&e.slice(0),this.errback=i,this.inited=!0,this.ignore=r.ignore,r.enabled||this.enabled?this.enable():this.check())},defineDep:function(e,t){this.depMatched[e]||(this.depMatched[e]=!0,this.depCount-=1,this.depExports[e]=t)},fetch:function(){if(!this.fetched){this.fetched=!0,l.startTime=(new Date).getTime();var e=this.map;if(!this.shim)return e.prefix?this.callPlugin():this.load();l.makeRequire(this.map,{enableBuildCallback:!0})(this.shim.deps||[],bind(this,function(){return e.prefix?this.callPlugin():this.load()}))}},load:function(){var e=this.map.url;n[e]||(n[e]=!0,l.load(this.map.id,e))},check:function(){if(this.enabled&&!this.enabling){var t,e,i=this.map.id,r=this.depExports,n=this.exports,o=this.factory;if(this.inited){if(this.error)this.emit("error",this.error);else if(!this.defining){if(this.defining=!0,this.depCount<1&&!this.defined){if(isFunction(o)){if(this.events.error&&this.map.isDefine||req.onError!==defaultOnError)try{n=l.execCb(i,o,r,n)}catch(e){t=e}else n=l.execCb(i,o,r,n);if(this.map.isDefine&&void 0===n&&((e=this.module)?n=e.exports:this.usingExports&&(n=this.exports)),t)return t.requireMap=this.map,t.requireModules=this.map.isDefine?[this.map.id]:null,t.requireType=this.map.isDefine?"define":"require",O(this.error=t)}else n=o;if(this.exports=n,this.map.isDefine&&!this.ignore&&(m[i]=n,req.onResourceLoad)){var a=[];each(this.depMaps,function(e){a.push(e.normalizedMap||e)}),req.onResourceLoad(l,this.map,a)}P(i),this.defined=!0}this.defining=!1,this.defined&&!this.defineEmitted&&(this.defineEmitted=!0,this.emit("defined",this.exports),this.defineEmitComplete=!0)}}else hasProp(l.defQueueMap,i)||this.fetch()}},callPlugin:function(){var u=this.map,c=u.id,e=S(u.prefix);this.depMaps.push(e),M(e,"defined",bind(this,function(e){var o,t,i,r=getOwn(v,this.map.id),n=this.map.name,a=this.map.parentMap?this.map.parentMap.name:null,s=l.makeRequire(u.parentMap,{enableBuildCallback:!0});return this.map.unnormalized?(e.normalize&&(n=e.normalize(n,function(e){return q(e,a,!0)})||""),M(t=S(u.prefix+"!"+n,this.map.parentMap,!0),"defined",bind(this,function(e){this.map.normalizedMap=t,this.init([],function(){return e},null,{enabled:!0,ignore:!0})})),void((i=getOwn(p,t.id))&&(this.depMaps.push(t),this.events.error&&i.on("error",bind(this,function(e){this.emit("error",e)})),i.enable()))):r?(this.map.url=l.nameToUrl(r),void this.load()):((o=bind(this,function(e){this.init([],function(){return e},null,{enabled:!0})})).error=bind(this,function(e){this.inited=!0,(this.error=e).requireModules=[c],eachProp(p,function(e){0===e.map.id.indexOf(c+"_unnormalized")&&P(e.map.id)}),O(e)}),o.fromText=bind(this,function(e,t){var i=u.name,r=S(i),n=useInteractive;t&&(e=t),n&&(useInteractive=!1),k(r),hasProp(g.config,c)&&(g.config[i]=g.config[c]);try{req.exec(e)}catch(e){return O(makeError("fromtexteval","fromText eval for "+c+" failed: "+e,e,[c]))}n&&(useInteractive=!0),this.depMaps.push(r),l.completeLoad(i),s([i],o)}),void e.load(u.name,s,o,g))})),l.enable(e,this),this.pluginMaps[e.id]=e},enable:function(){(f[this.map.id]=this).enabled=!0,this.enabling=!0,each(this.depMaps,bind(this,function(e,t){var i,r,n;if("string"==typeof e){if(e=S(e,this.map.isDefine?this.map:this.map.parentMap,!1,!this.skipMap),this.depMaps[t]=e,n=getOwn(c,e.id))return void(this.depExports[t]=n(this));this.depCount+=1,M(e,"defined",bind(this,function(e){this.undefed||(this.defineDep(t,e),this.check())})),this.errback?M(e,"error",bind(this,this.errback)):this.events.error&&M(e,"error",bind(this,function(e){this.emit("error",e)}))}i=e.id,r=p[i],hasProp(c,i)||!r||r.enabled||l.enable(e,this)})),eachProp(this.pluginMaps,bind(this,function(e){var t=getOwn(p,e.id);t&&!t.enabled&&l.enable(e,this)})),this.enabling=!1,this.check()},on:function(e,t){var i=this.events[e];i||(i=this.events[e]=[]),i.push(t)},emit:function(e,t){each(this.events[e],function(e){e(t)}),"error"===e&&delete this.events[e]}},(l={config:g,contextName:u,registry:p,defined:m,urlFetched:n,defQueue:h,defQueueMap:{},Module:e,makeModuleMap:S,nextTick:req.nextTick,onError:O,configure:function(e){if(e.baseUrl&&"/"!==e.baseUrl.charAt(e.baseUrl.length-1)&&(e.baseUrl+="/"),"string"==typeof e.urlArgs){var i=e.urlArgs;e.urlArgs=function(e,t){return(-1===t.indexOf("?")?"?":"&")+i}}var r=g.shim,n={paths:!0,bundles:!0,config:!0,map:!0};eachProp(e,function(e,t){n[t]?(g[t]||(g[t]={}),mixin(g[t],e,!0,!0)):g[t]=e}),e.bundles&&eachProp(e.bundles,function(e,t){each(e,function(e){e!==t&&(v[e]=t)})}),e.shim&&(eachProp(e.shim,function(e,t){isArray(e)&&(e={deps:e}),!e.exports&&!e.init||e.exportsFn||(e.exportsFn=l.makeShimExports(e)),r[t]=e}),g.shim=r),e.packages&&each(e.packages,function(e){var t;t=(e="string"==typeof e?{name:e}:e).name,e.location&&(g.paths[t]=e.location),g.pkgs[t]=e.name+"/"+(e.main||"main").replace(currDirRegExp,"").replace(jsSuffixRegExp,"")}),eachProp(p,function(e,t){e.inited||e.map.unnormalized||(e.map=S(t,null,!0))}),(e.deps||e.callback)&&l.require(e.deps||[],e.callback)},makeShimExports:function(t){return function(){var e;return t.init&&(e=t.init.apply(global,arguments)),e||t.exports&&getGlobal(t.exports)}},makeRequire:function(o,a){function s(e,t,i){var r,n;return a.enableBuildCallback&&t&&isFunction(t)&&(t.__requireJsBuild=!0),"string"==typeof e?isFunction(t)?O(makeError("requireargs","Invalid require call"),i):o&&hasProp(c,e)?c[e](p[o.id]):req.get?req.get(l,e,o,s):(r=S(e,o,!1,!0).id,hasProp(m,r)?m[r]:O(makeError("notloaded",'Module name "'+r+'" has not been loaded yet for context: '+u+(o?"":". Use require([])")))):(T(),l.nextTick(function(){T(),(n=k(S(null,o))).skipMap=a.skipMap,n.init(e,t,i,{enabled:!0}),R()}),s)}return a=a||{},mixin(s,{isBrowser:isBrowser,toUrl:function(e){var t,i=e.lastIndexOf("."),r=e.split("/")[0];return-1!==i&&(!("."===r||".."===r)||1<i)&&(t=e.substring(i,e.length),e=e.substring(0,i)),l.nameToUrl(q(e,o&&o.id,!0),t,!0)},defined:function(e){return hasProp(m,S(e,o,!1,!0).id)},specified:function(e){return e=S(e,o,!1,!0).id,hasProp(m,e)||hasProp(p,e)}}),o||(s.undef=function(i){j();var e=S(i,o,!0),t=getOwn(p,i);t.undefed=!0,E(i),delete m[i],delete n[e.url],delete r[i],eachReverse(h,function(e,t){e[0]===i&&h.splice(t,1)}),delete l.defQueueMap[i],t&&(t.events.defined&&(r[i]=t.events),P(i))}),s},enable:function(e){getOwn(p,e.id)&&k(e).enable()},completeLoad:function(e){var t,i,r,n=getOwn(g.shim,e)||{},o=n.exports;for(j();h.length;){if(null===(i=h.shift())[0]){if(i[0]=e,t)break;t=!0}else i[0]===e&&(t=!0);a(i)}if(l.defQueueMap={},r=getOwn(p,e),!t&&!hasProp(m,e)&&r&&!r.inited){if(!(!g.enforceDefine||o&&getGlobal(o)))return w(e)?void 0:O(makeError("nodefine","No define call for "+e,null,[e]));a([e,n.deps||[],n.exportsFn])}R()},nameToUrl:function(e,t,i){var r,n,o,a,s,u,c=getOwn(g.pkgs,e);if(c&&(e=c),u=getOwn(v,e))return l.nameToUrl(u,t,i);if(req.jsExtRegExp.test(e))a=e+(t||"");else{for(r=g.paths,o=(n=e.split("/")).length;0<o;o-=1)if(s=getOwn(r,n.slice(0,o).join("/"))){isArray(s)&&(s=s[0]),n.splice(0,o,s);break}a=n.join("/"),a=("/"===(a+=t||(/^data\:|^blob\:|\?/.test(a)||i?"":".js")).charAt(0)||a.match(/^[\w\+\.\-]+:/)?"":g.baseUrl)+a}return g.urlArgs&&!/^blob\:/.test(a)?a+g.urlArgs(e,a):a},load:function(e,t){req.load(l,e,t)},execCb:function(e,t,i,r){return t.apply(r,i)},onScriptLoad:function(e){if("load"===e.type||readyRegExp.test((e.currentTarget||e.srcElement).readyState)){interactiveScript=null;var t=s(e);l.completeLoad(t.id)}},onScriptError:function(e){var i=s(e);if(!w(i.id)){var r=[];return eachProp(p,function(e,t){0!==t.indexOf("_@r")&&each(e.depMaps,function(e){if(e.id===i.id)return r.push(t),!0})}),O(makeError("scripterror",'Script error for "'+i.id+(r.length?'", needed by: '+r.join(", "):'"'),e,[i.id]))}}}).require=l.makeRequire(),l}function getInteractiveScript(){return interactiveScript&&"interactive"===interactiveScript.readyState||eachReverse(scripts(),function(e){if("interactive"===e.readyState)return interactiveScript=e}),interactiveScript}}(this,"undefined"==typeof setTimeout?void 0:setTimeout); \ No newline at end of file
diff --git a/interface/js/lib/solid.min.js b/interface/js/lib/solid.min.js
new file mode 100644
index 0000000..dbe8cb1
--- /dev/null
+++ b/interface/js/lib/solid.min.js
@@ -0,0 +1,5 @@
+/*!
+ * Font Awesome Free 5.13.1 by @fontawesome - https://fontawesome.com
+ * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+ */
+!function(){"use strict";var c={},h={};try{"undefined"!=typeof window&&(c=window),"undefined"!=typeof document&&(h=document)}catch(c){}var l=(c.navigator||{}).userAgent,v=void 0===l?"":l,z=c,a=h,M=(z.document,!!a.documentElement&&!!a.head&&"function"==typeof a.addEventListener&&a.createElement,~v.indexOf("MSIE")||v.indexOf("Trident/"),"___FONT_AWESOME___"),m=function(){try{return!0}catch(c){return!1}}();var H=z||{};H[M]||(H[M]={}),H[M].styles||(H[M].styles={}),H[M].hooks||(H[M].hooks={}),H[M].shims||(H[M].shims=[]);var s=H[M];function V(c,v){var h=(2<arguments.length&&void 0!==arguments[2]?arguments[2]:{}).skipHooks,l=void 0!==h&&h,z=Object.keys(v).reduce(function(c,h){var l=v[h];return!!l.icon?c[l.iconName]=l.icon:c[h]=l,c},{});"function"!=typeof s.hooks.addPack||l?s.styles[c]=function(z){for(var c=1;c<arguments.length;c++){var a=null!=arguments[c]?arguments[c]:{},h=Object.keys(a);"function"==typeof Object.getOwnPropertySymbols&&(h=h.concat(Object.getOwnPropertySymbols(a).filter(function(c){return Object.getOwnPropertyDescriptor(a,c).enumerable}))),h.forEach(function(c){var h,l,v;h=z,v=a[l=c],l in h?Object.defineProperty(h,l,{value:v,enumerable:!0,configurable:!0,writable:!0}):h[l]=v})}return z}({},s.styles[c]||{},z):s.hooks.addPack(c,z),"fas"===c&&V("fa",v)}var L={ad:[512,512,[],"f641","M157.52 272h36.96L176 218.78 157.52 272zM352 256c-13.23 0-24 10.77-24 24s10.77 24 24 24 24-10.77 24-24-10.77-24-24-24zM464 64H48C21.5 64 0 85.5 0 112v288c0 26.5 21.5 48 48 48h416c26.5 0 48-21.5 48-48V112c0-26.5-21.5-48-48-48zM250.58 352h-16.94c-6.81 0-12.88-4.32-15.12-10.75L211.15 320h-70.29l-7.38 21.25A16 16 0 0 1 118.36 352h-16.94c-11.01 0-18.73-10.85-15.12-21.25L140 176.12A23.995 23.995 0 0 1 162.67 160h26.66A23.99 23.99 0 0 1 212 176.13l53.69 154.62c3.61 10.4-4.11 21.25-15.11 21.25zM424 336c0 8.84-7.16 16-16 16h-16c-4.85 0-9.04-2.27-11.98-5.68-8.62 3.66-18.09 5.68-28.02 5.68-39.7 0-72-32.3-72-72s32.3-72 72-72c8.46 0 16.46 1.73 24 4.42V176c0-8.84 7.16-16 16-16h16c8.84 0 16 7.16 16 16v160z"],"address-book":[448,512,[],"f2b9","M436 160c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-20V48c0-26.5-21.5-48-48-48H48C21.5 0 0 21.5 0 48v416c0 26.5 21.5 48 48 48h320c26.5 0 48-21.5 48-48v-48h20c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-20v-64h20c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-20v-64h20zm-228-32c35.3 0 64 28.7 64 64s-28.7 64-64 64-64-28.7-64-64 28.7-64 64-64zm112 236.8c0 10.6-10 19.2-22.4 19.2H118.4C106 384 96 375.4 96 364.8v-19.2c0-31.8 30.1-57.6 67.2-57.6h5c12.3 5.1 25.7 8 39.8 8s27.6-2.9 39.8-8h5c37.1 0 67.2 25.8 67.2 57.6v19.2z"],"address-card":[576,512,[],"f2bb","M528 32H48C21.5 32 0 53.5 0 80v352c0 26.5 21.5 48 48 48h480c26.5 0 48-21.5 48-48V80c0-26.5-21.5-48-48-48zm-352 96c35.3 0 64 28.7 64 64s-28.7 64-64 64-64-28.7-64-64 28.7-64 64-64zm112 236.8c0 10.6-10 19.2-22.4 19.2H86.4C74 384 64 375.4 64 364.8v-19.2c0-31.8 30.1-57.6 67.2-57.6h5c12.3 5.1 25.7 8 39.8 8s27.6-2.9 39.8-8h5c37.1 0 67.2 25.8 67.2 57.6v19.2zM512 312c0 4.4-3.6 8-8 8H360c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h144c4.4 0 8 3.6 8 8v16zm0-64c0 4.4-3.6 8-8 8H360c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h144c4.4 0 8 3.6 8 8v16zm0-64c0 4.4-3.6 8-8 8H360c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h144c4.4 0 8 3.6 8 8v16z"],adjust:[512,512,[],"f042","M8 256c0 136.966 111.033 248 248 248s248-111.034 248-248S392.966 8 256 8 8 119.033 8 256zm248 184V72c101.705 0 184 82.311 184 184 0 101.705-82.311 184-184 184z"],"air-freshener":[384,512,[],"f5d0","M378.94 321.41L284.7 224h49.22c15.3 0 23.66-16.6 13.86-27.53L234.45 69.96c3.43-6.61 5.55-14 5.55-21.96 0-26.51-21.49-48-48-48s-48 21.49-48 48c0 7.96 2.12 15.35 5.55 21.96L36.22 196.47C26.42 207.4 34.78 224 50.08 224H99.3L5.06 321.41C-6.69 333.56 3.34 352 21.7 352H160v32H48c-8.84 0-16 7.16-16 16v96c0 8.84 7.16 16 16 16h288c8.84 0 16-7.16 16-16v-96c0-8.84-7.16-16-16-16H224v-32h138.3c18.36 0 28.39-18.44 16.64-30.59zM192 31.98c8.85 0 16.02 7.17 16.02 16.02 0 8.84-7.17 16.02-16.02 16.02S175.98 56.84 175.98 48c0-8.85 7.17-16.02 16.02-16.02zM304 432v32H80v-32h224z"],"align-center":[448,512,[],"f037","M432 160H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0 256H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zM108.1 96h231.81A12.09 12.09 0 0 0 352 83.9V44.09A12.09 12.09 0 0 0 339.91 32H108.1A12.09 12.09 0 0 0 96 44.09V83.9A12.1 12.1 0 0 0 108.1 96zm231.81 256A12.09 12.09 0 0 0 352 339.9v-39.81A12.09 12.09 0 0 0 339.91 288H108.1A12.09 12.09 0 0 0 96 300.09v39.81a12.1 12.1 0 0 0 12.1 12.1z"],"align-justify":[448,512,[],"f039","M432 416H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-128H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-128H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-128H16A16 16 0 0 0 0 48v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z"],"align-left":[448,512,[],"f036","M12.83 352h262.34A12.82 12.82 0 0 0 288 339.17v-38.34A12.82 12.82 0 0 0 275.17 288H12.83A12.82 12.82 0 0 0 0 300.83v38.34A12.82 12.82 0 0 0 12.83 352zm0-256h262.34A12.82 12.82 0 0 0 288 83.17V44.83A12.82 12.82 0 0 0 275.17 32H12.83A12.82 12.82 0 0 0 0 44.83v38.34A12.82 12.82 0 0 0 12.83 96zM432 160H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0 256H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"],"align-right":[448,512,[],"f038","M16 224h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16zm416 192H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm3.17-384H172.83A12.82 12.82 0 0 0 160 44.83v38.34A12.82 12.82 0 0 0 172.83 96h262.34A12.82 12.82 0 0 0 448 83.17V44.83A12.82 12.82 0 0 0 435.17 32zm0 256H172.83A12.82 12.82 0 0 0 160 300.83v38.34A12.82 12.82 0 0 0 172.83 352h262.34A12.82 12.82 0 0 0 448 339.17v-38.34A12.82 12.82 0 0 0 435.17 288z"],allergies:[448,512,[],"f461","M416 112c-17.6 0-32 14.4-32 32v72c0 4.4-3.6 8-8 8h-16c-4.4 0-8-3.6-8-8V64c0-17.6-14.4-32-32-32s-32 14.4-32 32v152c0 4.4-3.6 8-8 8h-16c-4.4 0-8-3.6-8-8V32c0-17.6-14.4-32-32-32s-32 14.4-32 32v184c0 4.4-3.6 8-8 8h-16c-4.4 0-8-3.6-8-8V64c0-17.6-14.4-32-32-32S96 46.4 96 64v241l-23.6-32.5c-13-17.9-38-21.8-55.9-8.8s-21.8 38-8.8 55.9l125.6 172.7c9 12.4 23.5 19.8 38.8 19.8h197.6c22.3 0 41.6-15.3 46.7-37l26.5-112.7c3.2-13.7 4.9-28.3 5.1-42.3V144c0-17.6-14.4-32-32-32zM176 416c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zm0-96c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zm64 128c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zm0-96c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zm64 32c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zm32 64c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zm32-128c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16z"],ambulance:[640,512,[],"f0f9","M624 352h-16V243.9c0-12.7-5.1-24.9-14.1-33.9L494 110.1c-9-9-21.2-14.1-33.9-14.1H416V48c0-26.5-21.5-48-48-48H48C21.5 0 0 21.5 0 48v320c0 26.5 21.5 48 48 48h16c0 53 43 96 96 96s96-43 96-96h128c0 53 43 96 96 96s96-43 96-96h48c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zM160 464c-26.5 0-48-21.5-48-48s21.5-48 48-48 48 21.5 48 48-21.5 48-48 48zm144-248c0 4.4-3.6 8-8 8h-56v56c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8v-56h-56c-4.4 0-8-3.6-8-8v-48c0-4.4 3.6-8 8-8h56v-56c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v56h56c4.4 0 8 3.6 8 8v48zm176 248c-26.5 0-48-21.5-48-48s21.5-48 48-48 48 21.5 48 48-21.5 48-48 48zm80-208H416V144h44.1l99.9 99.9V256z"],"american-sign-language-interpreting":[640,512,[],"f2a3","M290.547 189.039c-20.295-10.149-44.147-11.199-64.739-3.89 42.606 0 71.208 20.475 85.578 50.576 8.576 17.899-5.148 38.071-23.617 38.071 18.429 0 32.211 20.136 23.617 38.071-14.725 30.846-46.123 50.854-80.298 50.854-.557 0-94.471-8.615-94.471-8.615l-66.406 33.347c-9.384 4.693-19.815.379-23.895-7.781L1.86 290.747c-4.167-8.615-1.111-18.897 6.946-23.621l58.072-33.069L108 159.861c6.39-57.245 34.731-109.767 79.743-146.726 11.391-9.448 28.341-7.781 37.51 3.613 9.446 11.394 7.78 28.067-3.612 37.516-12.503 10.559-23.618 22.509-32.509 35.57 21.672-14.729 46.679-24.732 74.186-28.067 14.725-1.945 28.063 8.336 29.73 23.065 1.945 14.728-8.336 28.067-23.062 29.734-16.116 1.945-31.12 7.503-44.178 15.284 26.114-5.713 58.712-3.138 88.079 11.115 13.336 6.669 18.893 22.509 12.224 35.848-6.389 13.06-22.504 18.617-35.564 12.226zm-27.229 69.472c-6.112-12.505-18.338-20.286-32.231-20.286a35.46 35.46 0 0 0-35.565 35.57c0 21.428 17.808 35.57 35.565 35.57 13.893 0 26.119-7.781 32.231-20.286 4.446-9.449 13.614-15.006 23.339-15.284-9.725-.277-18.893-5.835-23.339-15.284zm374.821-37.237c4.168 8.615 1.111 18.897-6.946 23.621l-58.071 33.069L532 352.16c-6.39 57.245-34.731 109.767-79.743 146.726-10.932 9.112-27.799 8.144-37.51-3.613-9.446-11.394-7.78-28.067 3.613-37.516 12.503-10.559 23.617-22.509 32.508-35.57-21.672 14.729-46.679 24.732-74.186 28.067-10.021 2.506-27.552-5.643-29.73-23.065-1.945-14.728 8.336-28.067 23.062-29.734 16.116-1.946 31.12-7.503 44.178-15.284-26.114 5.713-58.712 3.138-88.079-11.115-13.336-6.669-18.893-22.509-12.224-35.848 6.389-13.061 22.505-18.619 35.565-12.227 20.295 10.149 44.147 11.199 64.739 3.89-42.606 0-71.208-20.475-85.578-50.576-8.576-17.899 5.148-38.071 23.617-38.071-18.429 0-32.211-20.136-23.617-38.071 14.033-29.396 44.039-50.887 81.966-50.854l92.803 8.615 66.406-33.347c9.408-4.704 19.828-.354 23.894 7.781l44.455 88.926zm-229.227-18.618c-13.893 0-26.119 7.781-32.231 20.286-4.446 9.449-13.614 15.006-23.339 15.284 9.725.278 18.893 5.836 23.339 15.284 6.112 12.505 18.338 20.286 32.231 20.286a35.46 35.46 0 0 0 35.565-35.57c0-21.429-17.808-35.57-35.565-35.57z"],anchor:[576,512,[],"f13d","M12.971 352h32.394C67.172 454.735 181.944 512 288 512c106.229 0 220.853-57.38 242.635-160h32.394c10.691 0 16.045-12.926 8.485-20.485l-67.029-67.029c-4.686-4.686-12.284-4.686-16.971 0l-67.029 67.029c-7.56 7.56-2.206 20.485 8.485 20.485h35.146c-20.29 54.317-84.963 86.588-144.117 94.015V256h52c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12h-52v-5.47c37.281-13.178 63.995-48.725 64-90.518C384.005 43.772 341.605.738 289.37.01 235.723-.739 192 42.525 192 96c0 41.798 26.716 77.35 64 90.53V192h-52c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h52v190.015c-58.936-7.399-123.82-39.679-144.117-94.015h35.146c10.691 0 16.045-12.926 8.485-20.485l-67.029-67.029c-4.686-4.686-12.284-4.686-16.971 0L4.485 331.515C-3.074 339.074 2.28 352 12.971 352zM288 64c17.645 0 32 14.355 32 32s-14.355 32-32 32-32-14.355-32-32 14.355-32 32-32z"],"angle-double-down":[320,512,[],"f103","M143 256.3L7 120.3c-9.4-9.4-9.4-24.6 0-33.9l22.6-22.6c9.4-9.4 24.6-9.4 33.9 0l96.4 96.4 96.4-96.4c9.4-9.4 24.6-9.4 33.9 0L313 86.3c9.4 9.4 9.4 24.6 0 33.9l-136 136c-9.4 9.5-24.6 9.5-34 .1zm34 192l136-136c9.4-9.4 9.4-24.6 0-33.9l-22.6-22.6c-9.4-9.4-24.6-9.4-33.9 0L160 352.1l-96.4-96.4c-9.4-9.4-24.6-9.4-33.9 0L7 278.3c-9.4 9.4-9.4 24.6 0 33.9l136 136c9.4 9.5 24.6 9.5 34 .1z"],"angle-double-left":[448,512,[],"f100","M223.7 239l136-136c9.4-9.4 24.6-9.4 33.9 0l22.6 22.6c9.4 9.4 9.4 24.6 0 33.9L319.9 256l96.4 96.4c9.4 9.4 9.4 24.6 0 33.9L393.7 409c-9.4 9.4-24.6 9.4-33.9 0l-136-136c-9.5-9.4-9.5-24.6-.1-34zm-192 34l136 136c9.4 9.4 24.6 9.4 33.9 0l22.6-22.6c9.4-9.4 9.4-24.6 0-33.9L127.9 256l96.4-96.4c9.4-9.4 9.4-24.6 0-33.9L201.7 103c-9.4-9.4-24.6-9.4-33.9 0l-136 136c-9.5 9.4-9.5 24.6-.1 34z"],"angle-double-right":[448,512,[],"f101","M224.3 273l-136 136c-9.4 9.4-24.6 9.4-33.9 0l-22.6-22.6c-9.4-9.4-9.4-24.6 0-33.9l96.4-96.4-96.4-96.4c-9.4-9.4-9.4-24.6 0-33.9L54.3 103c9.4-9.4 24.6-9.4 33.9 0l136 136c9.5 9.4 9.5 24.6.1 34zm192-34l-136-136c-9.4-9.4-24.6-9.4-33.9 0l-22.6 22.6c-9.4 9.4-9.4 24.6 0 33.9l96.4 96.4-96.4 96.4c-9.4 9.4-9.4 24.6 0 33.9l22.6 22.6c9.4 9.4 24.6 9.4 33.9 0l136-136c9.4-9.2 9.4-24.4 0-33.8z"],"angle-double-up":[320,512,[],"f102","M177 255.7l136 136c9.4 9.4 9.4 24.6 0 33.9l-22.6 22.6c-9.4 9.4-24.6 9.4-33.9 0L160 351.9l-96.4 96.4c-9.4 9.4-24.6 9.4-33.9 0L7 425.7c-9.4-9.4-9.4-24.6 0-33.9l136-136c9.4-9.5 24.6-9.5 34-.1zm-34-192L7 199.7c-9.4 9.4-9.4 24.6 0 33.9l22.6 22.6c9.4 9.4 24.6 9.4 33.9 0l96.4-96.4 96.4 96.4c9.4 9.4 24.6 9.4 33.9 0l22.6-22.6c9.4-9.4 9.4-24.6 0-33.9l-136-136c-9.2-9.4-24.4-9.4-33.8 0z"],"angle-down":[320,512,[],"f107","M143 352.3L7 216.3c-9.4-9.4-9.4-24.6 0-33.9l22.6-22.6c9.4-9.4 24.6-9.4 33.9 0l96.4 96.4 96.4-96.4c9.4-9.4 24.6-9.4 33.9 0l22.6 22.6c9.4 9.4 9.4 24.6 0 33.9l-136 136c-9.2 9.4-24.4 9.4-33.8 0z"],"angle-left":[256,512,[],"f104","M31.7 239l136-136c9.4-9.4 24.6-9.4 33.9 0l22.6 22.6c9.4 9.4 9.4 24.6 0 33.9L127.9 256l96.4 96.4c9.4 9.4 9.4 24.6 0 33.9L201.7 409c-9.4 9.4-24.6 9.4-33.9 0l-136-136c-9.5-9.4-9.5-24.6-.1-34z"],"angle-right":[256,512,[],"f105","M224.3 273l-136 136c-9.4 9.4-24.6 9.4-33.9 0l-22.6-22.6c-9.4-9.4-9.4-24.6 0-33.9l96.4-96.4-96.4-96.4c-9.4-9.4-9.4-24.6 0-33.9L54.3 103c9.4-9.4 24.6-9.4 33.9 0l136 136c9.5 9.4 9.5 24.6.1 34z"],"angle-up":[320,512,[],"f106","M177 159.7l136 136c9.4 9.4 9.4 24.6 0 33.9l-22.6 22.6c-9.4 9.4-24.6 9.4-33.9 0L160 255.9l-96.4 96.4c-9.4 9.4-24.6 9.4-33.9 0L7 329.7c-9.4-9.4-9.4-24.6 0-33.9l136-136c9.4-9.5 24.6-9.5 34-.1z"],angry:[496,512,[],"f556","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zM136 240c0-9.3 4.1-17.5 10.5-23.4l-31-9.3c-8.5-2.5-13.3-11.5-10.7-19.9 2.5-8.5 11.4-13.2 19.9-10.7l80 24c8.5 2.5 13.3 11.5 10.7 19.9-2.1 6.9-8.4 11.4-15.3 11.4-.5 0-1.1-.2-1.7-.2.7 2.7 1.7 5.3 1.7 8.2 0 17.7-14.3 32-32 32S136 257.7 136 240zm168 154.2c-27.8-33.4-84.2-33.4-112.1 0-13.5 16.3-38.2-4.2-24.6-20.5 20-24 49.4-37.8 80.6-37.8s60.6 13.8 80.6 37.8c13.8 16.5-11.1 36.6-24.5 20.5zm76.6-186.9l-31 9.3c6.3 5.8 10.5 14.1 10.5 23.4 0 17.7-14.3 32-32 32s-32-14.3-32-32c0-2.9.9-5.6 1.7-8.2-.6.1-1.1.2-1.7.2-6.9 0-13.2-4.5-15.3-11.4-2.5-8.5 2.3-17.4 10.7-19.9l80-24c8.4-2.5 17.4 2.3 19.9 10.7 2.5 8.5-2.3 17.4-10.8 19.9z"],ankh:[320,512,[],"f644","M296 256h-44.62C272.46 222.01 288 181.65 288 144 288 55.63 230.69 0 160 0S32 55.63 32 144c0 37.65 15.54 78.01 36.62 112H24c-13.25 0-24 10.74-24 24v32c0 13.25 10.75 24 24 24h96v152c0 13.25 10.75 24 24 24h32c13.25 0 24-10.75 24-24V336h96c13.25 0 24-10.75 24-24v-32c0-13.26-10.75-24-24-24zM160 80c29.61 0 48 24.52 48 64 0 34.66-27.14 78.14-48 100.87-20.86-22.72-48-66.21-48-100.87 0-39.48 18.39-64 48-64z"],"apple-alt":[448,512,[],"f5d1","M350.85 129c25.97 4.67 47.27 18.67 63.92 42 14.65 20.67 24.64 46.67 29.96 78 4.67 28.67 4.32 57.33-1 86-7.99 47.33-23.97 87-47.94 119-28.64 38.67-64.59 58-107.87 58-10.66 0-22.3-3.33-34.96-10-8.66-5.33-18.31-8-28.97-8s-20.3 2.67-28.97 8c-12.66 6.67-24.3 10-34.96 10-43.28 0-79.23-19.33-107.87-58-23.97-32-39.95-71.67-47.94-119-5.32-28.67-5.67-57.33-1-86 5.32-31.33 15.31-57.33 29.96-78 16.65-23.33 37.95-37.33 63.92-42 15.98-2.67 37.95-.33 65.92 7 23.97 6.67 44.28 14.67 60.93 24 16.65-9.33 36.96-17.33 60.93-24 27.98-7.33 49.96-9.67 65.94-7zm-54.94-41c-9.32 8.67-21.65 15-36.96 19-10.66 3.33-22.3 5-34.96 5l-14.98-1c-1.33-9.33-1.33-20 0-32 2.67-24 10.32-42.33 22.97-55 9.32-8.67 21.65-15 36.96-19 10.66-3.33 22.3-5 34.96-5l14.98 1 1 15c0 12.67-1.67 24.33-4.99 35-3.99 15.33-10.31 27.67-18.98 37z"],archive:[512,512,[],"f187","M32 448c0 17.7 14.3 32 32 32h384c17.7 0 32-14.3 32-32V160H32v288zm160-212c0-6.6 5.4-12 12-12h104c6.6 0 12 5.4 12 12v8c0 6.6-5.4 12-12 12H204c-6.6 0-12-5.4-12-12v-8zM480 32H32C14.3 32 0 46.3 0 64v48c0 8.8 7.2 16 16 16h480c8.8 0 16-7.2 16-16V64c0-17.7-14.3-32-32-32z"],archway:[576,512,[],"f557","M560 448h-16V96H32v352H16.02c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16H176c8.84 0 16-7.16 16-16V320c0-53.02 42.98-96 96-96s96 42.98 96 96l.02 160v16c0 8.84 7.16 16 16 16H560c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zm0-448H16C7.16 0 0 7.16 0 16v32c0 8.84 7.16 16 16 16h544c8.84 0 16-7.16 16-16V16c0-8.84-7.16-16-16-16z"],"arrow-alt-circle-down":[512,512,[],"f358","M504 256c0 137-111 248-248 248S8 393 8 256 119 8 256 8s248 111 248 248zM212 140v116h-70.9c-10.7 0-16.1 13-8.5 20.5l114.9 114.3c4.7 4.7 12.2 4.7 16.9 0l114.9-114.3c7.6-7.6 2.2-20.5-8.5-20.5H300V140c0-6.6-5.4-12-12-12h-64c-6.6 0-12 5.4-12 12z"],"arrow-alt-circle-left":[512,512,[],"f359","M256 504C119 504 8 393 8 256S119 8 256 8s248 111 248 248-111 248-248 248zm116-292H256v-70.9c0-10.7-13-16.1-20.5-8.5L121.2 247.5c-4.7 4.7-4.7 12.2 0 16.9l114.3 114.9c7.6 7.6 20.5 2.2 20.5-8.5V300h116c6.6 0 12-5.4 12-12v-64c0-6.6-5.4-12-12-12z"],"arrow-alt-circle-right":[512,512,[],"f35a","M256 8c137 0 248 111 248 248S393 504 256 504 8 393 8 256 119 8 256 8zM140 300h116v70.9c0 10.7 13 16.1 20.5 8.5l114.3-114.9c4.7-4.7 4.7-12.2 0-16.9l-114.3-115c-7.6-7.6-20.5-2.2-20.5 8.5V212H140c-6.6 0-12 5.4-12 12v64c0 6.6 5.4 12 12 12z"],"arrow-alt-circle-up":[512,512,[],"f35b","M8 256C8 119 119 8 256 8s248 111 248 248-111 248-248 248S8 393 8 256zm292 116V256h70.9c10.7 0 16.1-13 8.5-20.5L264.5 121.2c-4.7-4.7-12.2-4.7-16.9 0l-115 114.3c-7.6 7.6-2.2 20.5 8.5 20.5H212v116c0 6.6 5.4 12 12 12h64c6.6 0 12-5.4 12-12z"],"arrow-circle-down":[512,512,[],"f0ab","M504 256c0 137-111 248-248 248S8 393 8 256 119 8 256 8s248 111 248 248zm-143.6-28.9L288 302.6V120c0-13.3-10.7-24-24-24h-16c-13.3 0-24 10.7-24 24v182.6l-72.4-75.5c-9.3-9.7-24.8-9.9-34.3-.4l-10.9 11c-9.4 9.4-9.4 24.6 0 33.9L239 404.3c9.4 9.4 24.6 9.4 33.9 0l132.7-132.7c9.4-9.4 9.4-24.6 0-33.9l-10.9-11c-9.5-9.5-25-9.3-34.3.4z"],"arrow-circle-left":[512,512,[],"f0a8","M256 504C119 504 8 393 8 256S119 8 256 8s248 111 248 248-111 248-248 248zm28.9-143.6L209.4 288H392c13.3 0 24-10.7 24-24v-16c0-13.3-10.7-24-24-24H209.4l75.5-72.4c9.7-9.3 9.9-24.8.4-34.3l-11-10.9c-9.4-9.4-24.6-9.4-33.9 0L107.7 239c-9.4 9.4-9.4 24.6 0 33.9l132.7 132.7c9.4 9.4 24.6 9.4 33.9 0l11-10.9c9.5-9.5 9.3-25-.4-34.3z"],"arrow-circle-right":[512,512,[],"f0a9","M256 8c137 0 248 111 248 248S393 504 256 504 8 393 8 256 119 8 256 8zm-28.9 143.6l75.5 72.4H120c-13.3 0-24 10.7-24 24v16c0 13.3 10.7 24 24 24h182.6l-75.5 72.4c-9.7 9.3-9.9 24.8-.4 34.3l11 10.9c9.4 9.4 24.6 9.4 33.9 0L404.3 273c9.4-9.4 9.4-24.6 0-33.9L271.6 106.3c-9.4-9.4-24.6-9.4-33.9 0l-11 10.9c-9.5 9.6-9.3 25.1.4 34.4z"],"arrow-circle-up":[512,512,[],"f0aa","M8 256C8 119 119 8 256 8s248 111 248 248-111 248-248 248S8 393 8 256zm143.6 28.9l72.4-75.5V392c0 13.3 10.7 24 24 24h16c13.3 0 24-10.7 24-24V209.4l72.4 75.5c9.3 9.7 24.8 9.9 34.3.4l10.9-11c9.4-9.4 9.4-24.6 0-33.9L273 107.7c-9.4-9.4-24.6-9.4-33.9 0L106.3 240.4c-9.4 9.4-9.4 24.6 0 33.9l10.9 11c9.6 9.5 25.1 9.3 34.4-.4z"],"arrow-down":[448,512,[],"f063","M413.1 222.5l22.2 22.2c9.4 9.4 9.4 24.6 0 33.9L241 473c-9.4 9.4-24.6 9.4-33.9 0L12.7 278.6c-9.4-9.4-9.4-24.6 0-33.9l22.2-22.2c9.5-9.5 25-9.3 34.3.4L184 343.4V56c0-13.3 10.7-24 24-24h32c13.3 0 24 10.7 24 24v287.4l114.8-120.5c9.3-9.8 24.8-10 34.3-.4z"],"arrow-left":[448,512,[],"f060","M257.5 445.1l-22.2 22.2c-9.4 9.4-24.6 9.4-33.9 0L7 273c-9.4-9.4-9.4-24.6 0-33.9L201.4 44.7c9.4-9.4 24.6-9.4 33.9 0l22.2 22.2c9.5 9.5 9.3 25-.4 34.3L136.6 216H424c13.3 0 24 10.7 24 24v32c0 13.3-10.7 24-24 24H136.6l120.5 114.8c9.8 9.3 10 24.8.4 34.3z"],"arrow-right":[448,512,[],"f061","M190.5 66.9l22.2-22.2c9.4-9.4 24.6-9.4 33.9 0L441 239c9.4 9.4 9.4 24.6 0 33.9L246.6 467.3c-9.4 9.4-24.6 9.4-33.9 0l-22.2-22.2c-9.5-9.5-9.3-25 .4-34.3L311.4 296H24c-13.3 0-24-10.7-24-24v-32c0-13.3 10.7-24 24-24h287.4L190.9 101.2c-9.8-9.3-10-24.8-.4-34.3z"],"arrow-up":[448,512,[],"f062","M34.9 289.5l-22.2-22.2c-9.4-9.4-9.4-24.6 0-33.9L207 39c9.4-9.4 24.6-9.4 33.9 0l194.3 194.3c9.4 9.4 9.4 24.6 0 33.9L413 289.4c-9.5 9.5-25 9.3-34.3-.4L264 168.6V456c0 13.3-10.7 24-24 24h-32c-13.3 0-24-10.7-24-24V168.6L69.2 289.1c-9.3 9.8-24.8 10-34.3.4z"],"arrows-alt":[512,512,[],"f0b2","M352.201 425.775l-79.196 79.196c-9.373 9.373-24.568 9.373-33.941 0l-79.196-79.196c-15.119-15.119-4.411-40.971 16.971-40.97h51.162L228 284H127.196v51.162c0 21.382-25.851 32.09-40.971 16.971L7.029 272.937c-9.373-9.373-9.373-24.569 0-33.941L86.225 159.8c15.119-15.119 40.971-4.411 40.971 16.971V228H228V127.196h-51.23c-21.382 0-32.09-25.851-16.971-40.971l79.196-79.196c9.373-9.373 24.568-9.373 33.941 0l79.196 79.196c15.119 15.119 4.411 40.971-16.971 40.971h-51.162V228h100.804v-51.162c0-21.382 25.851-32.09 40.97-16.971l79.196 79.196c9.373 9.373 9.373 24.569 0 33.941L425.773 352.2c-15.119 15.119-40.971 4.411-40.97-16.971V284H284v100.804h51.23c21.382 0 32.09 25.851 16.971 40.971z"],"arrows-alt-h":[512,512,[],"f337","M377.941 169.941V216H134.059v-46.059c0-21.382-25.851-32.09-40.971-16.971L7.029 239.029c-9.373 9.373-9.373 24.568 0 33.941l86.059 86.059c15.119 15.119 40.971 4.411 40.971-16.971V296h243.882v46.059c0 21.382 25.851 32.09 40.971 16.971l86.059-86.059c9.373-9.373 9.373-24.568 0-33.941l-86.059-86.059c-15.119-15.12-40.971-4.412-40.971 16.97z"],"arrows-alt-v":[256,512,[],"f338","M214.059 377.941H168V134.059h46.059c21.382 0 32.09-25.851 16.971-40.971L144.971 7.029c-9.373-9.373-24.568-9.373-33.941 0L24.971 93.088c-15.119 15.119-4.411 40.971 16.971 40.971H88v243.882H41.941c-21.382 0-32.09 25.851-16.971 40.971l86.059 86.059c9.373 9.373 24.568 9.373 33.941 0l86.059-86.059c15.12-15.119 4.412-40.971-16.97-40.971z"],"assistive-listening-systems":[512,512,[],"f2a2","M216 260c0 15.464-12.536 28-28 28s-28-12.536-28-28c0-44.112 35.888-80 80-80s80 35.888 80 80c0 15.464-12.536 28-28 28s-28-12.536-28-28c0-13.234-10.767-24-24-24s-24 10.766-24 24zm24-176c-97.047 0-176 78.953-176 176 0 15.464 12.536 28 28 28s28-12.536 28-28c0-66.168 53.832-120 120-120s120 53.832 120 120c0 75.164-71.009 70.311-71.997 143.622L288 404c0 28.673-23.327 52-52 52-15.464 0-28 12.536-28 28s12.536 28 28 28c59.475 0 107.876-48.328 108-107.774.595-34.428 72-48.24 72-144.226 0-97.047-78.953-176-176-176zm-80 236c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32zM32 448c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32zm480-187.993c0-1.518-.012-3.025-.045-4.531C510.076 140.525 436.157 38.47 327.994 1.511c-14.633-4.998-30.549 2.809-35.55 17.442-5 14.633 2.81 30.549 17.442 35.55 85.906 29.354 144.61 110.513 146.077 201.953l.003.188c.026 1.118.033 2.236.033 3.363 0 15.464 12.536 28 28 28s28.001-12.536 28.001-28zM152.971 439.029l-80-80L39.03 392.97l80 80 33.941-33.941z"],asterisk:[512,512,[],"f069","M478.21 334.093L336 256l142.21-78.093c11.795-6.477 15.961-21.384 9.232-33.037l-19.48-33.741c-6.728-11.653-21.72-15.499-33.227-8.523L296 186.718l3.475-162.204C299.763 11.061 288.937 0 275.48 0h-38.96c-13.456 0-24.283 11.061-23.994 24.514L216 186.718 77.265 102.607c-11.506-6.976-26.499-3.13-33.227 8.523l-19.48 33.741c-6.728 11.653-2.562 26.56 9.233 33.037L176 256 33.79 334.093c-11.795 6.477-15.961 21.384-9.232 33.037l19.48 33.741c6.728 11.653 21.721 15.499 33.227 8.523L216 325.282l-3.475 162.204C212.237 500.939 223.064 512 236.52 512h38.961c13.456 0 24.283-11.061 23.995-24.514L296 325.282l138.735 84.111c11.506 6.976 26.499 3.13 33.227-8.523l19.48-33.741c6.728-11.653 2.563-26.559-9.232-33.036z"],at:[512,512,[],"f1fa","M256 8C118.941 8 8 118.919 8 256c0 137.059 110.919 248 248 248 48.154 0 95.342-14.14 135.408-40.223 12.005-7.815 14.625-24.288 5.552-35.372l-10.177-12.433c-7.671-9.371-21.179-11.667-31.373-5.129C325.92 429.757 291.314 440 256 440c-101.458 0-184-82.542-184-184S154.542 72 256 72c100.139 0 184 57.619 184 160 0 38.786-21.093 79.742-58.17 83.693-17.349-.454-16.91-12.857-13.476-30.024l23.433-121.11C394.653 149.75 383.308 136 368.225 136h-44.981a13.518 13.518 0 0 0-13.432 11.993l-.01.092c-14.697-17.901-40.448-21.775-59.971-21.775-74.58 0-137.831 62.234-137.831 151.46 0 65.303 36.785 105.87 96 105.87 26.984 0 57.369-15.637 74.991-38.333 9.522 34.104 40.613 34.103 70.71 34.103C462.609 379.41 504 307.798 504 232 504 95.653 394.023 8 256 8zm-21.68 304.43c-22.249 0-36.07-15.623-36.07-40.771 0-44.993 30.779-72.729 58.63-72.729 22.292 0 35.601 15.241 35.601 40.77 0 45.061-33.875 72.73-58.161 72.73z"],atlas:[448,512,[],"f558","M318.38 208h-39.09c-1.49 27.03-6.54 51.35-14.21 70.41 27.71-13.24 48.02-39.19 53.3-70.41zm0-32c-5.29-31.22-25.59-57.17-53.3-70.41 7.68 19.06 12.72 43.38 14.21 70.41h39.09zM224 97.31c-7.69 7.45-20.77 34.42-23.43 78.69h46.87c-2.67-44.26-15.75-71.24-23.44-78.69zm-41.08 8.28c-27.71 13.24-48.02 39.19-53.3 70.41h39.09c1.49-27.03 6.53-51.35 14.21-70.41zm0 172.82c-7.68-19.06-12.72-43.38-14.21-70.41h-39.09c5.28 31.22 25.59 57.17 53.3 70.41zM247.43 208h-46.87c2.66 44.26 15.74 71.24 23.43 78.69 7.7-7.45 20.78-34.43 23.44-78.69zM448 358.4V25.6c0-16-9.6-25.6-25.6-25.6H96C41.6 0 0 41.6 0 96v320c0 54.4 41.6 96 96 96h326.4c12.8 0 25.6-9.6 25.6-25.6v-16c0-6.4-3.2-12.8-9.6-19.2-3.2-16-3.2-60.8 0-73.6 6.4-3.2 9.6-9.6 9.6-19.2zM224 64c70.69 0 128 57.31 128 128s-57.31 128-128 128S96 262.69 96 192 153.31 64 224 64zm160 384H96c-19.2 0-32-12.8-32-32s16-32 32-32h288v64z"],atom:[448,512,[],"f5d2","M223.99908,224a32,32,0,1,0,32.00782,32A32.06431,32.06431,0,0,0,223.99908,224Zm214.172-96c-10.877-19.5-40.50979-50.75-116.27544-41.875C300.39168,34.875,267.63386,0,223.99908,0s-76.39066,34.875-97.89653,86.125C50.3369,77.375,20.706,108.5,9.82907,128-6.54984,157.375-5.17484,201.125,34.958,256-5.17484,310.875-6.54984,354.625,9.82907,384c29.13087,52.375,101.64652,43.625,116.27348,41.875C147.60842,477.125,180.36429,512,223.99908,512s76.3926-34.875,97.89652-86.125c14.62891,1.75,87.14456,10.5,116.27544-41.875C454.55,354.625,453.175,310.875,413.04017,256,453.175,201.125,454.55,157.375,438.171,128ZM63.33886,352c-4-7.25-.125-24.75,15.00391-48.25,6.87695,6.5,14.12891,12.875,21.88087,19.125,1.625,13.75,4,27.125,6.75,40.125C82.34472,363.875,67.09081,358.625,63.33886,352Zm36.88478-162.875c-7.752,6.25-15.00392,12.625-21.88087,19.125-15.12891-23.5-19.00392-41-15.00391-48.25,3.377-6.125,16.37891-11.5,37.88478-11.5,1.75,0,3.875.375,5.75.375C104.09864,162.25,101.84864,175.625,100.22364,189.125ZM223.99908,64c9.50195,0,22.25586,13.5,33.88282,37.25-11.252,3.75-22.50391,8-33.88282,12.875-11.377-4.875-22.62892-9.125-33.88283-12.875C201.74516,77.5,214.49712,64,223.99908,64Zm0,384c-9.502,0-22.25392-13.5-33.88283-37.25,11.25391-3.75,22.50587-8,33.88283-12.875C235.378,402.75,246.62994,407,257.8819,410.75,246.25494,434.5,233.501,448,223.99908,448Zm0-112a80,80,0,1,1,80-80A80.00023,80.00023,0,0,1,223.99908,336ZM384.6593,352c-3.625,6.625-19.00392,11.875-43.63479,11,2.752-13,5.127-26.375,6.752-40.125,7.75195-6.25,15.00391-12.625,21.87891-19.125C384.7843,327.25,388.6593,344.75,384.6593,352ZM369.65538,208.25c-6.875-6.5-14.127-12.875-21.87891-19.125-1.625-13.5-3.875-26.875-6.752-40.25,1.875,0,4.002-.375,5.752-.375,21.50391,0,34.50782,5.375,37.88283,11.5C388.6593,167.25,384.7843,184.75,369.65538,208.25Z"],"audio-description":[512,512,[],"f29e","M162.925 238.709l8.822 30.655h-25.606l9.041-30.652c1.277-4.421 2.651-9.994 3.872-15.245 1.22 5.251 2.594 10.823 3.871 15.242zm166.474-32.099h-14.523v98.781h14.523c29.776 0 46.175-17.678 46.175-49.776 0-32.239-17.49-49.005-46.175-49.005zM512 112v288c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48V112c0-26.51 21.49-48 48-48h416c26.51 0 48 21.49 48 48zM245.459 336.139l-57.097-168A12.001 12.001 0 0 0 177 160h-35.894a12.001 12.001 0 0 0-11.362 8.139l-57.097 168C70.003 343.922 75.789 352 84.009 352h29.133a12 12 0 0 0 11.535-8.693l8.574-29.906h51.367l8.793 29.977A12 12 0 0 0 204.926 352h29.172c8.22 0 14.006-8.078 11.361-15.861zm184.701-80.525c0-58.977-37.919-95.614-98.96-95.614h-57.366c-6.627 0-12 5.373-12 12v168c0 6.627 5.373 12 12 12H331.2c61.041 0 98.96-36.933 98.96-96.386z"],award:[384,512,[],"f559","M97.12 362.63c-8.69-8.69-4.16-6.24-25.12-11.85-9.51-2.55-17.87-7.45-25.43-13.32L1.2 448.7c-4.39 10.77 3.81 22.47 15.43 22.03l52.69-2.01L105.56 507c8 8.44 22.04 5.81 26.43-4.96l52.05-127.62c-10.84 6.04-22.87 9.58-35.31 9.58-19.5 0-37.82-7.59-51.61-21.37zM382.8 448.7l-45.37-111.24c-7.56 5.88-15.92 10.77-25.43 13.32-21.07 5.64-16.45 3.18-25.12 11.85-13.79 13.78-32.12 21.37-51.62 21.37-12.44 0-24.47-3.55-35.31-9.58L252 502.04c4.39 10.77 18.44 13.4 26.43 4.96l36.25-38.28 52.69 2.01c11.62.44 19.82-11.27 15.43-22.03zM263 340c15.28-15.55 17.03-14.21 38.79-20.14 13.89-3.79 24.75-14.84 28.47-28.98 7.48-28.4 5.54-24.97 25.95-45.75 10.17-10.35 14.14-25.44 10.42-39.58-7.47-28.38-7.48-24.42 0-52.83 3.72-14.14-.25-29.23-10.42-39.58-20.41-20.78-18.47-17.36-25.95-45.75-3.72-14.14-14.58-25.19-28.47-28.98-27.88-7.61-24.52-5.62-44.95-26.41-10.17-10.35-25-14.4-38.89-10.61-27.87 7.6-23.98 7.61-51.9 0-13.89-3.79-28.72.25-38.89 10.61-20.41 20.78-17.05 18.8-44.94 26.41-13.89 3.79-24.75 14.84-28.47 28.98-7.47 28.39-5.54 24.97-25.95 45.75-10.17 10.35-14.15 25.44-10.42 39.58 7.47 28.36 7.48 24.4 0 52.82-3.72 14.14.25 29.23 10.42 39.59 20.41 20.78 18.47 17.35 25.95 45.75 3.72 14.14 14.58 25.19 28.47 28.98C104.6 325.96 106.27 325 121 340c13.23 13.47 33.84 15.88 49.74 5.82a39.676 39.676 0 0 1 42.53 0c15.89 10.06 36.5 7.65 49.73-5.82zM97.66 175.96c0-53.03 42.24-96.02 94.34-96.02s94.34 42.99 94.34 96.02-42.24 96.02-94.34 96.02-94.34-42.99-94.34-96.02z"],baby:[384,512,[],"f77c","M192 160c44.2 0 80-35.8 80-80S236.2 0 192 0s-80 35.8-80 80 35.8 80 80 80zm-53.4 248.8l25.6-32-61.5-51.2L56.8 383c-11.4 14.2-11.7 34.4-.8 49l48 64c7.9 10.5 19.9 16 32 16 8.3 0 16.8-2.6 24-8 17.7-13.2 21.2-38.3 8-56l-29.4-39.2zm142.7-83.2l-61.5 51.2 25.6 32L216 448c-13.2 17.7-9.7 42.8 8 56 7.2 5.4 15.6 8 24 8 12.2 0 24.2-5.5 32-16l48-64c10.9-14.6 10.6-34.8-.8-49l-45.9-57.4zM376.7 145c-12.7-18.1-37.6-22.4-55.7-9.8l-40.6 28.5c-52.7 37-124.2 37-176.8 0L63 135.3C44.9 122.6 20 127 7.3 145-5.4 163.1-1 188 17 200.7l40.6 28.5c17 11.9 35.4 20.9 54.4 27.9V288h160v-30.8c19-7 37.4-16 54.4-27.9l40.6-28.5c18.1-12.8 22.4-37.7 9.7-55.8z"],"baby-carriage":[512,512,[],"f77d","M144.8 17c-11.3-17.8-37.2-22.8-54-9.4C35.3 51.9 0 118 0 192h256L144.8 17zM496 96h-48c-35.3 0-64 28.7-64 64v64H0c0 50.6 23 96.4 60.3 130.7C25.7 363.6 0 394.7 0 432c0 44.2 35.8 80 80 80s80-35.8 80-80c0-8.9-1.8-17.2-4.4-25.2 21.6 5.9 44.6 9.2 68.4 9.2s46.9-3.3 68.4-9.2c-2.7 8-4.4 16.3-4.4 25.2 0 44.2 35.8 80 80 80s80-35.8 80-80c0-37.3-25.7-68.4-60.3-77.3C425 320.4 448 274.6 448 224v-64h48c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zM80 464c-17.6 0-32-14.4-32-32s14.4-32 32-32 32 14.4 32 32-14.4 32-32 32zm320-32c0 17.6-14.4 32-32 32s-32-14.4-32-32 14.4-32 32-32 32 14.4 32 32z"],backspace:[640,512,[],"f55a","M576 64H205.26A63.97 63.97 0 0 0 160 82.75L9.37 233.37c-12.5 12.5-12.5 32.76 0 45.25L160 429.25c12 12 28.28 18.75 45.25 18.75H576c35.35 0 64-28.65 64-64V128c0-35.35-28.65-64-64-64zm-84.69 254.06c6.25 6.25 6.25 16.38 0 22.63l-22.62 22.62c-6.25 6.25-16.38 6.25-22.63 0L384 301.25l-62.06 62.06c-6.25 6.25-16.38 6.25-22.63 0l-22.62-22.62c-6.25-6.25-6.25-16.38 0-22.63L338.75 256l-62.06-62.06c-6.25-6.25-6.25-16.38 0-22.63l22.62-22.62c6.25-6.25 16.38-6.25 22.63 0L384 210.75l62.06-62.06c6.25-6.25 16.38-6.25 22.63 0l22.62 22.62c6.25 6.25 6.25 16.38 0 22.63L429.25 256l62.06 62.06z"],backward:[512,512,[],"f04a","M11.5 280.6l192 160c20.6 17.2 52.5 2.8 52.5-24.6V96c0-27.4-31.9-41.8-52.5-24.6l-192 160c-15.3 12.8-15.3 36.4 0 49.2zm256 0l192 160c20.6 17.2 52.5 2.8 52.5-24.6V96c0-27.4-31.9-41.8-52.5-24.6l-192 160c-15.3 12.8-15.3 36.4 0 49.2z"],bacon:[576,512,[],"f7e5","M218.92 336.39c34.89-34.89 44.2-59.7 54.05-86 10.61-28.29 21.59-57.54 61.37-97.34s69.05-50.77 97.35-61.38c23.88-9 46.64-17.68 76.79-45.37L470.81 8.91a31 31 0 0 0-40.18-2.83c-13.64 10.1-25.15 14.39-41 20.3C247 79.52 209.26 191.29 200.65 214.1c-29.75 78.83-89.55 94.68-98.72 98.09-24.86 9.26-54.73 20.38-91.07 50.36C-3 374-3.63 395 9.07 407.61l35.76 35.51C80 410.52 107 400.15 133 390.39c26.27-9.84 51.06-19.12 85.92-54zm348-232l-35.75-35.51c-35.19 32.63-62.18 43-88.25 52.79-26.26 9.85-51.06 19.16-85.95 54s-44.19 59.69-54 86C292.33 290 281.34 319.22 241.55 359s-69 50.73-97.3 61.32c-23.86 9-46.61 17.66-76.72 45.33l37.68 37.43a31 31 0 0 0 40.18 2.82c13.6-10.06 25.09-14.34 40.94-20.24 142.2-53 180-164.1 188.94-187.69C405 219.18 464.8 203.3 474 199.86c24.87-9.27 54.74-20.4 91.11-50.41 13.89-11.4 14.52-32.45 1.82-45.05z"],bacteria:[640,512,[],"f959","M272.35,226.4A17.71,17.71,0,0,0,281.46,203l-4-9.08a121.29,121.29,0,0,1,12.36-3.08A83.34,83.34,0,0,0,323.57,177l10,9a17.76,17.76,0,1,0,23.92-26.27l-9.72-8.76a83.12,83.12,0,0,0,11.65-48.18l11.85-3.51a17.73,17.73,0,1,0-10.15-34l-11.34,3.36a84,84,0,0,0-36.38-35.57l2.84-10.85a17.8,17.8,0,0,0-34.47-8.93l-2.82,10.78a83.25,83.25,0,0,0-16.74,1.1C250.83,27,240,30.22,229.1,33.39l-3.38-9.46a17.8,17.8,0,0,0-33.56,11.89l3.49,9.8a286.74,286.74,0,0,0-43.94,23.57l-6.32-8.43a17.9,17.9,0,0,0-24.94-3.6A17.69,17.69,0,0,0,116.84,82l6.45,8.61a286.59,286.59,0,0,0-34.95,35.33l-8.82-6.42a17.84,17.84,0,0,0-24.89,3.86,17.66,17.66,0,0,0,3.88,24.77l8.88,6.47a286.6,286.6,0,0,0-23,43.91l-10.48-3.59a17.73,17.73,0,1,0-11.59,33.52L32.67,232c-2.79,10-5.79,19.84-7.52,30.22a83.16,83.16,0,0,0-.82,19l-11.58,3.43a17.73,17.73,0,1,0,10.13,34l11.27-3.33a83.51,83.51,0,0,0,36.39,35.43l-2.88,11.06a17.81,17.81,0,0,0,34.48,8.92l2.87-11c1,0,2.07.26,3.1.26a83.39,83.39,0,0,0,45.65-13.88l8.59,8.8a17.77,17.77,0,0,0,25.56-24.7l-9.14-9.37a83.41,83.41,0,0,0,12.08-31.05,119.08,119.08,0,0,1,3.87-15.53l9,4.22a17.74,17.74,0,1,0,15.15-32.09l-8.8-4.11c.67-1,1.2-2.08,1.9-3.05a119.89,119.89,0,0,1,7.87-9.41,121.73,121.73,0,0,1,11.65-11.4,119.49,119.49,0,0,1,9.94-7.82c1.12-.77,2.32-1.42,3.47-2.15l3.92,8.85a17.86,17.86,0,0,0,16.32,10.58A18.14,18.14,0,0,0,272.35,226.4ZM128,256a32,32,0,1,1,32-32A32,32,0,0,1,128,256Zm80-96a16,16,0,1,1,16-16A16,16,0,0,1,208,160Zm431.26,45.3a17.79,17.79,0,0,0-17.06-12.69,17.55,17.55,0,0,0-5.08.74l-11.27,3.33a83.61,83.61,0,0,0-36.39-35.43l2.88-11.06a17.81,17.81,0,0,0-34.48-8.91l-2.87,11c-1,0-2.07-.26-3.1-.26a83.32,83.32,0,0,0-45.65,13.89l-8.59-8.81a17.77,17.77,0,0,0-25.56,24.7l9.14,9.37a83.28,83.28,0,0,0-12.08,31.06,119.34,119.34,0,0,1-3.87,15.52l-9-4.22a17.74,17.74,0,1,0-15.15,32.09l8.8,4.11c-.67,1-1.2,2.08-1.89,3.05a117.71,117.71,0,0,1-7.94,9.47,119,119,0,0,1-11.57,11.33,121.59,121.59,0,0,1-10,7.83c-1.12.77-2.32,1.42-3.47,2.15l-3.92-8.85a17.86,17.86,0,0,0-16.32-10.58,18.14,18.14,0,0,0-7.18,1.5A17.71,17.71,0,0,0,358.54,309l4,9.08a118.71,118.71,0,0,1-12.36,3.08,83.34,83.34,0,0,0-33.77,13.9l-10-9a17.77,17.77,0,1,0-23.92,26.28l9.72,8.75a83.12,83.12,0,0,0-11.65,48.18l-11.86,3.51a17.73,17.73,0,1,0,10.16,34l11.34-3.36A84,84,0,0,0,326.61,479l-2.84,10.85a17.8,17.8,0,0,0,34.47,8.93L361.06,488a83.3,83.3,0,0,0,16.74-1.1c11.37-1.89,22.24-5.07,33.1-8.24l3.38,9.46a17.8,17.8,0,0,0,33.56-11.89l-3.49-9.79a287.66,287.66,0,0,0,43.94-23.58l6.32,8.43a17.88,17.88,0,0,0,24.93,3.6A17.67,17.67,0,0,0,523.16,430l-6.45-8.61a287.37,287.37,0,0,0,34.95-35.34l8.82,6.42a17.76,17.76,0,1,0,21-28.63l-8.88-6.46a287.17,287.17,0,0,0,23-43.92l10.48,3.59a17.73,17.73,0,1,0,11.59-33.52L607.33,280c2.79-10,5.79-19.84,7.52-30.21a83.27,83.27,0,0,0,.82-19.05l11.58-3.43A17.7,17.7,0,0,0,639.26,205.3ZM416,416a32,32,0,1,1,32-32A32,32,0,0,1,416,416Z"],bacterium:[512,512,[],"f95a","M511,102.93A23.76,23.76,0,0,0,481.47,87l-15.12,4.48a111.85,111.85,0,0,0-48.5-47.42l3.79-14.47a23.74,23.74,0,0,0-46-11.91l-3.76,14.37a111.94,111.94,0,0,0-22.33,1.47,386.74,386.74,0,0,0-44.33,10.41l-4.3-12a23.74,23.74,0,0,0-44.75,15.85l4.3,12.05a383.4,383.4,0,0,0-58.69,31.83l-8-10.63a23.85,23.85,0,0,0-33.24-4.8,23.57,23.57,0,0,0-4.83,33.09l8,10.63a386.14,386.14,0,0,0-46.7,47.44l-11-8a23.68,23.68,0,1,0-28,38.17l11.09,8.06a383.45,383.45,0,0,0-30.92,58.75l-12.93-4.43a23.65,23.65,0,1,0-15.47,44.69l13,4.48a385.81,385.81,0,0,0-9.3,40.53A111.58,111.58,0,0,0,32.44,375L17,379.56a23.64,23.64,0,0,0,13.51,45.31l15-4.44a111.49,111.49,0,0,0,48.53,47.24l-3.85,14.75a23.66,23.66,0,0,0,17,28.83,24.7,24.7,0,0,0,6,.75,23.73,23.73,0,0,0,23-17.7L140,479.67c1.37.05,2.77.35,4.13.35A111.22,111.22,0,0,0,205,461.5l11.45,11.74a23.7,23.7,0,0,0,34.08-32.93l-12.19-12.5a111,111,0,0,0,16.11-41.4,158.69,158.69,0,0,1,5.16-20.71l12,5.64a23.66,23.66,0,1,0,20.19-42.79l-11.72-5.49c.89-1.32,1.59-2.77,2.52-4.06a157.86,157.86,0,0,1,10.46-12.49,159.5,159.5,0,0,1,15.59-15.28,162.18,162.18,0,0,1,13.23-10.4c1.5-1,3.1-1.89,4.63-2.87l5.23,11.8a23.74,23.74,0,0,0,43.48-19.08l-5.36-12.11a158.87,158.87,0,0,1,16.49-4.1,111,111,0,0,0,45-18.54l13.33,12a23.69,23.69,0,1,0,31.88-35l-12.94-11.67A110.83,110.83,0,0,0,479.21,137L495,132.32A23.61,23.61,0,0,0,511,102.93ZM160,368a48,48,0,1,1,48-48A48,48,0,0,1,160,368Zm80-136a24,24,0,1,1,24-24A24,24,0,0,1,240,232Z"],bahai:[512,512,[],"f666","M496.25 202.52l-110-15.44 41.82-104.34c6.67-16.64-11.6-32.18-26.59-22.63L307.44 120 273.35 12.82C270.64 4.27 263.32 0 256 0c-7.32 0-14.64 4.27-17.35 12.82l-34.09 107.19-94.04-59.89c-14.99-9.55-33.25 5.99-26.59 22.63l41.82 104.34-110 15.43c-17.54 2.46-21.68 26.27-6.03 34.67l98.16 52.66-74.48 83.54c-10.92 12.25-1.72 30.93 13.29 30.93 1.31 0 2.67-.14 4.07-.45l108.57-23.65-4.11 112.55c-.43 11.65 8.87 19.22 18.41 19.22 5.15 0 10.39-2.21 14.2-7.18l68.18-88.9 68.18 88.9c3.81 4.97 9.04 7.18 14.2 7.18 9.54 0 18.84-7.57 18.41-19.22l-4.11-112.55 108.57 23.65c17.36 3.76 29.21-17.2 17.35-30.49l-74.48-83.54 98.16-52.66c15.64-8.39 11.5-32.2-6.04-34.66zM338.51 311.68l-51.89-11.3 1.97 53.79L256 311.68l-32.59 42.49 1.96-53.79-51.89 11.3 35.6-39.93-46.92-25.17 52.57-7.38-19.99-49.87 44.95 28.62L256 166.72l16.29 51.23 44.95-28.62-19.99 49.87 52.57 7.38-46.92 25.17 35.61 39.93z"],"balance-scale":[640,512,[],"f24e","M256 336h-.02c0-16.18 1.34-8.73-85.05-181.51-17.65-35.29-68.19-35.36-85.87 0C-2.06 328.75.02 320.33.02 336H0c0 44.18 57.31 80 128 80s128-35.82 128-80zM128 176l72 144H56l72-144zm511.98 160c0-16.18 1.34-8.73-85.05-181.51-17.65-35.29-68.19-35.36-85.87 0-87.12 174.26-85.04 165.84-85.04 181.51H384c0 44.18 57.31 80 128 80s128-35.82 128-80h-.02zM440 320l72-144 72 144H440zm88 128H352V153.25c23.51-10.29 41.16-31.48 46.39-57.25H528c8.84 0 16-7.16 16-16V48c0-8.84-7.16-16-16-16H383.64C369.04 12.68 346.09 0 320 0s-49.04 12.68-63.64 32H112c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h129.61c5.23 25.76 22.87 46.96 46.39 57.25V448H112c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h416c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16z"],"balance-scale-left":[640,512,[],"f515","M528 448H352V153.25c20.42-8.94 36.1-26.22 43.38-47.47l132-44.26c8.38-2.81 12.89-11.88 10.08-20.26l-10.17-30.34C524.48 2.54 515.41-1.97 507.03.84L389.11 40.37C375.3 16.36 349.69 0 320 0c-44.18 0-80 35.82-80 80 0 3.43.59 6.71 1.01 10.03l-128.39 43.05c-8.38 2.81-12.89 11.88-10.08 20.26l10.17 30.34c2.81 8.38 11.88 12.89 20.26 10.08l142.05-47.63c4.07 2.77 8.43 5.12 12.99 7.12V496c0 8.84 7.16 16 16 16h224c8.84 0 16-7.16 16-16v-32c-.01-8.84-7.17-16-16.01-16zm111.98-144c0-16.18 1.34-8.73-85.05-181.51-17.65-35.29-68.19-35.36-85.87 0-87.12 174.26-85.04 165.84-85.04 181.51H384c0 44.18 57.31 80 128 80s128-35.82 128-80h-.02zM440 288l72-144 72 144H440zm-269.07-37.51c-17.65-35.29-68.19-35.36-85.87 0C-2.06 424.75.02 416.33.02 432H0c0 44.18 57.31 80 128 80s128-35.82 128-80h-.02c0-16.18 1.34-8.73-85.05-181.51zM56 416l72-144 72 144H56z"],"balance-scale-right":[640,512,[],"f516","M96 464v32c0 8.84 7.16 16 16 16h224c8.84 0 16-7.16 16-16V153.25c4.56-2 8.92-4.35 12.99-7.12l142.05 47.63c8.38 2.81 17.45-1.71 20.26-10.08l10.17-30.34c2.81-8.38-1.71-17.45-10.08-20.26l-128.4-43.05c.42-3.32 1.01-6.6 1.01-10.03 0-44.18-35.82-80-80-80-29.69 0-55.3 16.36-69.11 40.37L132.96.83c-8.38-2.81-17.45 1.71-20.26 10.08l-10.17 30.34c-2.81 8.38 1.71 17.45 10.08 20.26l132 44.26c7.28 21.25 22.96 38.54 43.38 47.47V448H112c-8.84 0-16 7.16-16 16zM0 304c0 44.18 57.31 80 128 80s128-35.82 128-80h-.02c0-15.67 2.08-7.25-85.05-181.51-17.68-35.36-68.22-35.29-85.87 0C-1.32 295.27.02 287.82.02 304H0zm56-16l72-144 72 144H56zm328.02 144H384c0 44.18 57.31 80 128 80s128-35.82 128-80h-.02c0-15.67 2.08-7.25-85.05-181.51-17.68-35.36-68.22-35.29-85.87 0-86.38 172.78-85.04 165.33-85.04 181.51zM440 416l72-144 72 144H440z"],ban:[512,512,[],"f05e","M256 8C119.034 8 8 119.033 8 256s111.034 248 248 248 248-111.034 248-248S392.967 8 256 8zm130.108 117.892c65.448 65.448 70 165.481 20.677 235.637L150.47 105.216c70.204-49.356 170.226-44.735 235.638 20.676zM125.892 386.108c-65.448-65.448-70-165.481-20.677-235.637L361.53 406.784c-70.203 49.356-170.226 44.736-235.638-20.676z"],"band-aid":[640,512,[],"f462","M0 160v192c0 35.3 28.7 64 64 64h96V96H64c-35.3 0-64 28.7-64 64zm576-64h-96v320h96c35.3 0 64-28.7 64-64V160c0-35.3-28.7-64-64-64zM192 416h256V96H192v320zm176-232c13.3 0 24 10.7 24 24s-10.7 24-24 24-24-10.7-24-24 10.7-24 24-24zm0 96c13.3 0 24 10.7 24 24s-10.7 24-24 24-24-10.7-24-24 10.7-24 24-24zm-96-96c13.3 0 24 10.7 24 24s-10.7 24-24 24-24-10.7-24-24 10.7-24 24-24zm0 96c13.3 0 24 10.7 24 24s-10.7 24-24 24-24-10.7-24-24 10.7-24 24-24z"],barcode:[512,512,[],"f02a","M0 448V64h18v384H0zm26.857-.273V64H36v383.727h-9.143zm27.143 0V64h8.857v383.727H54zm44.857 0V64h8.857v383.727h-8.857zm36 0V64h17.714v383.727h-17.714zm44.857 0V64h8.857v383.727h-8.857zm18 0V64h8.857v383.727h-8.857zm18 0V64h8.857v383.727h-8.857zm35.715 0V64h18v383.727h-18zm44.857 0V64h18v383.727h-18zm35.999 0V64h18.001v383.727h-18.001zm36.001 0V64h18.001v383.727h-18.001zm26.857 0V64h18v383.727h-18zm45.143 0V64h26.857v383.727h-26.857zm35.714 0V64h9.143v383.727H476zm18 .273V64h18v384h-18z"],bars:[448,512,[],"f0c9","M16 132h416c8.837 0 16-7.163 16-16V76c0-8.837-7.163-16-16-16H16C7.163 60 0 67.163 0 76v40c0 8.837 7.163 16 16 16zm0 160h416c8.837 0 16-7.163 16-16v-40c0-8.837-7.163-16-16-16H16c-8.837 0-16 7.163-16 16v40c0 8.837 7.163 16 16 16zm0 160h416c8.837 0 16-7.163 16-16v-40c0-8.837-7.163-16-16-16H16c-8.837 0-16 7.163-16 16v40c0 8.837 7.163 16 16 16z"],"baseball-ball":[496,512,[],"f433","M368.5 363.9l28.8-13.9c11.1 22.9 26 43.2 44.1 60.9 34-42.5 54.5-96.3 54.5-154.9 0-58.5-20.4-112.2-54.2-154.6-17.8 17.3-32.6 37.1-43.6 59.5l-28.7-14.1c12.8-26 30-49 50.8-69C375.6 34.7 315 8 248 8 181.1 8 120.5 34.6 75.9 77.7c20.7 19.9 37.9 42.9 50.7 68.8l-28.7 14.1c-11-22.3-25.7-42.1-43.5-59.4C20.4 143.7 0 197.4 0 256c0 58.6 20.4 112.3 54.4 154.7 18.2-17.7 33.2-38 44.3-61l28.8 13.9c-12.9 26.7-30.3 50.3-51.5 70.7 44.5 43.1 105.1 69.7 172 69.7 66.8 0 127.3-26.5 171.9-69.5-21.1-20.4-38.5-43.9-51.4-70.6zm-228.3-32l-30.5-9.8c14.9-46.4 12.7-93.8-.6-134l30.4-10c15 45.6 18 99.9.7 153.8zm216.3-153.4l30.4 10c-13.2 40.1-15.5 87.5-.6 134l-30.5 9.8c-17.3-54-14.3-108.3.7-153.8z"],"basketball-ball":[496,512,[],"f434","M212.3 10.3c-43.8 6.3-86.2 24.1-122.2 53.8l77.4 77.4c27.8-35.8 43.3-81.2 44.8-131.2zM248 222L405.9 64.1c-42.4-35-93.6-53.5-145.5-56.1-1.2 63.9-21.5 122.3-58.7 167.7L248 222zM56.1 98.1c-29.7 36-47.5 78.4-53.8 122.2 50-1.5 95.5-17 131.2-44.8L56.1 98.1zm272.2 204.2c45.3-37.1 103.7-57.4 167.7-58.7-2.6-51.9-21.1-103.1-56.1-145.5L282 256l46.3 46.3zM248 290L90.1 447.9c42.4 34.9 93.6 53.5 145.5 56.1 1.3-64 21.6-122.4 58.7-167.7L248 290zm191.9 123.9c29.7-36 47.5-78.4 53.8-122.2-50.1 1.6-95.5 17.1-131.2 44.8l77.4 77.4zM167.7 209.7C122.3 246.9 63.9 267.3 0 268.4c2.6 51.9 21.1 103.1 56.1 145.5L214 256l-46.3-46.3zm116 292c43.8-6.3 86.2-24.1 122.2-53.8l-77.4-77.4c-27.7 35.7-43.2 81.2-44.8 131.2z"],bath:[512,512,[],"f2cd","M32,384a95.4,95.4,0,0,0,32,71.09V496a16,16,0,0,0,16,16h32a16,16,0,0,0,16-16V480H384v16a16,16,0,0,0,16,16h32a16,16,0,0,0,16-16V455.09A95.4,95.4,0,0,0,480,384V336H32ZM496,256H80V69.25a21.26,21.26,0,0,1,36.28-15l19.27,19.26c-13.13,29.88-7.61,59.11,8.62,79.73l-.17.17A16,16,0,0,0,144,176l11.31,11.31a16,16,0,0,0,22.63,0L283.31,81.94a16,16,0,0,0,0-22.63L272,48a16,16,0,0,0-22.62,0l-.17.17c-20.62-16.23-49.83-21.75-79.73-8.62L150.22,20.28A69.25,69.25,0,0,0,32,69.25V256H16A16,16,0,0,0,0,272v16a16,16,0,0,0,16,16H496a16,16,0,0,0,16-16V272A16,16,0,0,0,496,256Z"],"battery-empty":[640,512,[],"f244","M544 160v64h32v64h-32v64H64V160h480m16-64H48c-26.51 0-48 21.49-48 48v224c0 26.51 21.49 48 48 48h512c26.51 0 48-21.49 48-48v-16h8c13.255 0 24-10.745 24-24V184c0-13.255-10.745-24-24-24h-8v-16c0-26.51-21.49-48-48-48z"],"battery-full":[640,512,[],"f240","M544 160v64h32v64h-32v64H64V160h480m16-64H48c-26.51 0-48 21.49-48 48v224c0 26.51 21.49 48 48 48h512c26.51 0 48-21.49 48-48v-16h8c13.255 0 24-10.745 24-24V184c0-13.255-10.745-24-24-24h-8v-16c0-26.51-21.49-48-48-48zm-48 96H96v128h416V192z"],"battery-half":[640,512,[],"f242","M544 160v64h32v64h-32v64H64V160h480m16-64H48c-26.51 0-48 21.49-48 48v224c0 26.51 21.49 48 48 48h512c26.51 0 48-21.49 48-48v-16h8c13.255 0 24-10.745 24-24V184c0-13.255-10.745-24-24-24h-8v-16c0-26.51-21.49-48-48-48zm-240 96H96v128h224V192z"],"battery-quarter":[640,512,[],"f243","M544 160v64h32v64h-32v64H64V160h480m16-64H48c-26.51 0-48 21.49-48 48v224c0 26.51 21.49 48 48 48h512c26.51 0 48-21.49 48-48v-16h8c13.255 0 24-10.745 24-24V184c0-13.255-10.745-24-24-24h-8v-16c0-26.51-21.49-48-48-48zm-336 96H96v128h128V192z"],"battery-three-quarters":[640,512,[],"f241","M544 160v64h32v64h-32v64H64V160h480m16-64H48c-26.51 0-48 21.49-48 48v224c0 26.51 21.49 48 48 48h512c26.51 0 48-21.49 48-48v-16h8c13.255 0 24-10.745 24-24V184c0-13.255-10.745-24-24-24h-8v-16c0-26.51-21.49-48-48-48zm-144 96H96v128h320V192z"],bed:[640,512,[],"f236","M176 256c44.11 0 80-35.89 80-80s-35.89-80-80-80-80 35.89-80 80 35.89 80 80 80zm352-128H304c-8.84 0-16 7.16-16 16v144H64V80c0-8.84-7.16-16-16-16H16C7.16 64 0 71.16 0 80v352c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-48h512v48c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16V240c0-61.86-50.14-112-112-112z"],beer:[448,512,[],"f0fc","M368 96h-48V56c0-13.255-10.745-24-24-24H24C10.745 32 0 42.745 0 56v400c0 13.255 10.745 24 24 24h272c13.255 0 24-10.745 24-24v-42.11l80.606-35.977C429.396 365.063 448 336.388 448 304.86V176c0-44.112-35.888-80-80-80zm16 208.86a16.018 16.018 0 0 1-9.479 14.611L320 343.805V160h48c8.822 0 16 7.178 16 16v128.86zM208 384c-8.836 0-16-7.164-16-16V144c0-8.836 7.164-16 16-16s16 7.164 16 16v224c0 8.836-7.164 16-16 16zm-96 0c-8.836 0-16-7.164-16-16V144c0-8.836 7.164-16 16-16s16 7.164 16 16v224c0 8.836-7.164 16-16 16z"],bell:[448,512,[],"f0f3","M224 512c35.32 0 63.97-28.65 63.97-64H160.03c0 35.35 28.65 64 63.97 64zm215.39-149.71c-19.32-20.76-55.47-51.99-55.47-154.29 0-77.7-54.48-139.9-127.94-155.16V32c0-17.67-14.32-32-31.98-32s-31.98 14.33-31.98 32v20.84C118.56 68.1 64.08 130.3 64.08 208c0 102.3-36.15 133.53-55.47 154.29-6 6.45-8.66 14.16-8.61 21.71.11 16.4 12.98 32 32.1 32h383.8c19.12 0 32-15.6 32.1-32 .05-7.55-2.61-15.27-8.61-21.71z"],"bell-slash":[640,512,[],"f1f6","M633.82 458.1l-90.62-70.05c.19-1.38.8-2.66.8-4.06.05-7.55-2.61-15.27-8.61-21.71-19.32-20.76-55.47-51.99-55.47-154.29 0-77.7-54.48-139.9-127.94-155.16V32c0-17.67-14.32-32-31.98-32s-31.98 14.33-31.98 32v20.84c-40.33 8.38-74.66 31.07-97.59 62.57L45.47 3.37C38.49-2.05 28.43-.8 23.01 6.18L3.37 31.45C-2.05 38.42-.8 48.47 6.18 53.9l588.35 454.73c6.98 5.43 17.03 4.17 22.46-2.81l19.64-25.27c5.42-6.97 4.17-17.02-2.81-22.45zM157.23 251.54c-8.61 67.96-36.41 93.33-52.62 110.75-6 6.45-8.66 14.16-8.61 21.71.11 16.4 12.98 32 32.1 32h241.92L157.23 251.54zM320 512c35.32 0 63.97-28.65 63.97-64H256.03c0 35.35 28.65 64 63.97 64z"],"bezier-curve":[640,512,[],"f55b","M368 32h-96c-17.67 0-32 14.33-32 32v96c0 17.67 14.33 32 32 32h96c17.67 0 32-14.33 32-32V64c0-17.67-14.33-32-32-32zM208 88h-84.75C113.75 64.56 90.84 48 64 48 28.66 48 0 76.65 0 112s28.66 64 64 64c26.84 0 49.75-16.56 59.25-40h79.73c-55.37 32.52-95.86 87.32-109.54 152h49.4c11.3-41.61 36.77-77.21 71.04-101.56-3.7-8.08-5.88-16.99-5.88-26.44V88zm-48 232H64c-17.67 0-32 14.33-32 32v96c0 17.67 14.33 32 32 32h96c17.67 0 32-14.33 32-32v-96c0-17.67-14.33-32-32-32zM576 48c-26.84 0-49.75 16.56-59.25 40H432v72c0 9.45-2.19 18.36-5.88 26.44 34.27 24.35 59.74 59.95 71.04 101.56h49.4c-13.68-64.68-54.17-119.48-109.54-152h79.73c9.5 23.44 32.41 40 59.25 40 35.34 0 64-28.65 64-64s-28.66-64-64-64zm0 272h-96c-17.67 0-32 14.33-32 32v96c0 17.67 14.33 32 32 32h96c17.67 0 32-14.33 32-32v-96c0-17.67-14.33-32-32-32z"],bible:[448,512,[],"f647","M448 358.4V25.6c0-16-9.6-25.6-25.6-25.6H96C41.6 0 0 41.6 0 96v320c0 54.4 41.6 96 96 96h326.4c12.8 0 25.6-9.6 25.6-25.6v-16c0-6.4-3.2-12.8-9.6-19.2-3.2-16-3.2-60.8 0-73.6 6.4-3.2 9.6-9.6 9.6-19.2zM144 144c0-8.84 7.16-16 16-16h48V80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v48h48c8.84 0 16 7.16 16 16v32c0 8.84-7.16 16-16 16h-48v112c0 8.84-7.16 16-16 16h-32c-8.84 0-16-7.16-16-16V192h-48c-8.84 0-16-7.16-16-16v-32zm236.8 304H96c-19.2 0-32-12.8-32-32s16-32 32-32h284.8v64z"],bicycle:[640,512,[],"f206","M512.509 192.001c-16.373-.064-32.03 2.955-46.436 8.495l-77.68-125.153A24 24 0 0 0 368.001 64h-64c-8.837 0-16 7.163-16 16v16c0 8.837 7.163 16 16 16h50.649l14.896 24H256.002v-16c0-8.837-7.163-16-16-16h-87.459c-13.441 0-24.777 10.999-24.536 24.437.232 13.044 10.876 23.563 23.995 23.563h48.726l-29.417 47.52c-13.433-4.83-27.904-7.483-42.992-7.52C58.094 191.83.412 249.012.002 319.236-.413 390.279 57.055 448 128.002 448c59.642 0 109.758-40.793 123.967-96h52.033a24 24 0 0 0 20.406-11.367L410.37 201.77l14.938 24.067c-25.455 23.448-41.385 57.081-41.307 94.437.145 68.833 57.899 127.051 126.729 127.719 70.606.685 128.181-55.803 129.255-125.996 1.086-70.941-56.526-129.72-127.476-129.996zM186.75 265.772c9.727 10.529 16.673 23.661 19.642 38.228h-43.306l23.664-38.228zM128.002 400c-44.112 0-80-35.888-80-80s35.888-80 80-80c5.869 0 11.586.653 17.099 1.859l-45.505 73.509C89.715 331.327 101.213 352 120.002 352h81.3c-12.37 28.225-40.562 48-73.3 48zm162.63-96h-35.624c-3.96-31.756-19.556-59.894-42.383-80.026L237.371 184h127.547l-74.286 120zm217.057 95.886c-41.036-2.165-74.049-35.692-75.627-76.755-.812-21.121 6.633-40.518 19.335-55.263l44.433 71.586c4.66 7.508 14.524 9.816 22.032 5.156l13.594-8.437c7.508-4.66 9.817-14.524 5.156-22.032l-44.468-71.643a79.901 79.901 0 0 1 19.858-2.497c44.112 0 80 35.888 80 80-.001 45.54-38.252 82.316-84.313 79.885z"],biking:[640,512,[],"f84a","M400 96a48 48 0 1 0-48-48 48 48 0 0 0 48 48zm-4 121a31.9 31.9 0 0 0 20 7h64a32 32 0 0 0 0-64h-52.78L356 103a31.94 31.94 0 0 0-40.81.68l-112 96a32 32 0 0 0 3.08 50.92L288 305.12V416a32 32 0 0 0 64 0V288a32 32 0 0 0-14.25-26.62l-41.36-27.57 58.25-49.92zm116 39a128 128 0 1 0 128 128 128 128 0 0 0-128-128zm0 192a64 64 0 1 1 64-64 64 64 0 0 1-64 64zM128 256a128 128 0 1 0 128 128 128 128 0 0 0-128-128zm0 192a64 64 0 1 1 64-64 64 64 0 0 1-64 64z"],binoculars:[512,512,[],"f1e5","M416 48c0-8.84-7.16-16-16-16h-64c-8.84 0-16 7.16-16 16v48h96V48zM63.91 159.99C61.4 253.84 3.46 274.22 0 404v44c0 17.67 14.33 32 32 32h96c17.67 0 32-14.33 32-32V288h32V128H95.84c-17.63 0-31.45 14.37-31.93 31.99zm384.18 0c-.48-17.62-14.3-31.99-31.93-31.99H320v160h32v160c0 17.67 14.33 32 32 32h96c17.67 0 32-14.33 32-32v-44c-3.46-129.78-61.4-150.16-63.91-244.01zM176 32h-64c-8.84 0-16 7.16-16 16v48h96V48c0-8.84-7.16-16-16-16zm48 256h64V128h-64v160z"],biohazard:[576,512,[],"f780","M287.9 112c18.6 0 36.2 3.8 52.8 9.6 13.3-10.3 23.6-24.3 29.5-40.7-25.2-10.9-53-17-82.2-17-29.1 0-56.9 6-82.1 16.9 5.9 16.4 16.2 30.4 29.5 40.7 16.5-5.7 34-9.5 52.5-9.5zM163.6 438.7c12-11.8 20.4-26.4 24.5-42.4-32.9-26.4-54.8-65.3-58.9-109.6-8.5-2.8-17.2-4.6-26.4-4.6-7.6 0-15.2 1-22.5 3.1 4.1 62.8 35.8 118 83.3 153.5zm224.2-42.6c4.1 16 12.5 30.7 24.5 42.5 47.4-35.5 79.1-90.7 83-153.5-7.2-2-14.7-3-22.2-3-9.2 0-18 1.9-26.6 4.7-4.1 44.2-26 82.9-58.7 109.3zm113.5-205c-17.6-10.4-36.3-16.6-55.3-19.9 6-17.7 10-36.4 10-56.2 0-41-14.5-80.8-41-112.2-2.5-3-6.6-3.7-10-1.8-3.3 1.9-4.8 6-3.6 9.7 4.5 13.8 6.6 26.3 6.6 38.5 0 67.8-53.8 122.9-120 122.9S168 117 168 49.2c0-12.1 2.2-24.7 6.6-38.5 1.2-3.7-.3-7.8-3.6-9.7-3.4-1.9-7.5-1.2-10 1.8C134.6 34.2 120 74 120 115c0 19.8 3.9 38.5 10 56.2-18.9 3.3-37.7 9.5-55.3 19.9-34.6 20.5-61 53.3-74.3 92.4-1.3 3.7.2 7.7 3.5 9.8 3.3 2 7.5 1.3 10-1.6 9.4-10.8 19-19.1 29.2-25.1 57.3-33.9 130.8-13.7 163.9 45 33.1 58.7 13.4 134-43.9 167.9-10.2 6.1-22 10.4-35.8 13.4-3.7.8-6.4 4.2-6.4 8.1.1 4 2.7 7.3 6.5 8 39.7 7.8 80.6.8 115.2-19.7 18-10.6 32.9-24.5 45.3-40.1 12.4 15.6 27.3 29.5 45.3 40.1 34.6 20.5 75.5 27.5 115.2 19.7 3.8-.7 6.4-4 6.5-8 0-3.9-2.6-7.3-6.4-8.1-13.9-2.9-25.6-7.3-35.8-13.4-57.3-33.9-77-109.2-43.9-167.9s106.6-78.9 163.9-45c10.2 6.1 19.8 14.3 29.2 25.1 2.5 2.9 6.7 3.6 10 1.6s4.8-6.1 3.5-9.8c-13.1-39.1-39.5-72-74.1-92.4zm-213.4 129c-26.5 0-48-21.5-48-48s21.5-48 48-48 48 21.5 48 48-21.5 48-48 48z"],"birthday-cake":[448,512,[],"f1fd","M448 384c-28.02 0-31.26-32-74.5-32-43.43 0-46.825 32-74.75 32-27.695 0-31.454-32-74.75-32-42.842 0-47.218 32-74.5 32-28.148 0-31.202-32-74.75-32-43.547 0-46.653 32-74.75 32v-80c0-26.5 21.5-48 48-48h16V112h64v144h64V112h64v144h64V112h64v144h16c26.5 0 48 21.5 48 48v80zm0 128H0v-96c43.356 0 46.767-32 74.75-32 27.951 0 31.253 32 74.75 32 42.843 0 47.217-32 74.5-32 28.148 0 31.201 32 74.75 32 43.357 0 46.767-32 74.75-32 27.488 0 31.252 32 74.5 32v96zM96 96c-17.75 0-32-14.25-32-32 0-31 32-23 32-64 12 0 32 29.5 32 56s-14.25 40-32 40zm128 0c-17.75 0-32-14.25-32-32 0-31 32-23 32-64 12 0 32 29.5 32 56s-14.25 40-32 40zm128 0c-17.75 0-32-14.25-32-32 0-31 32-23 32-64 12 0 32 29.5 32 56s-14.25 40-32 40z"],blender:[512,512,[],"f517","M416 384H160c-35.35 0-64 28.65-64 64v32c0 17.67 14.33 32 32 32h320c17.67 0 32-14.33 32-32v-32c0-35.35-28.65-64-64-64zm-128 96c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm40-416h166.54L512 0H48C21.49 0 0 21.49 0 48v160c0 26.51 21.49 48 48 48h103.27l8.73 96h256l17.46-64H328c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h114.18l17.46-64H328c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h140.36l17.46-64H328c-4.42 0-8-3.58-8-8V72c0-4.42 3.58-8 8-8zM64 192V64h69.82l11.64 128H64z"],"blender-phone":[576,512,[],"f6b6","M392 64h166.54L576 0H192v352h288l17.46-64H392c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h114.18l17.46-64H392c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h140.36l17.46-64H392c-4.42 0-8-3.58-8-8V72c0-4.42 3.58-8 8-8zM158.8 335.01l-25.78-63.26c-2.78-6.81-9.8-10.99-17.24-10.26l-45.03 4.42c-17.28-46.94-17.65-99.78 0-147.72l45.03 4.42c7.43.73 14.46-3.46 17.24-10.26l25.78-63.26c3.02-7.39.2-15.85-6.68-20.07l-39.28-24.1C98.51-3.87 80.09-.5 68.95 11.97c-92.57 103.6-92 259.55 2.1 362.49 9.87 10.8 29.12 12.48 41.65 4.8l39.41-24.18c6.89-4.22 9.7-12.67 6.69-20.07zM480 384H192c-35.35 0-64 28.65-64 64v32c0 17.67 14.33 32 32 32h352c17.67 0 32-14.33 32-32v-32c0-35.35-28.65-64-64-64zm-144 96c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],blind:[384,512,[],"f29d","M380.15 510.837a8 8 0 0 1-10.989-2.687l-125.33-206.427a31.923 31.923 0 0 0 12.958-9.485l126.048 207.608a8 8 0 0 1-2.687 10.991zM142.803 314.338l-32.54 89.485 36.12 88.285c6.693 16.36 25.377 24.192 41.733 17.501 16.357-6.692 24.193-25.376 17.501-41.734l-62.814-153.537zM96 88c24.301 0 44-19.699 44-44S120.301 0 96 0 52 19.699 52 44s19.699 44 44 44zm154.837 169.128l-120-152c-4.733-5.995-11.75-9.108-18.837-9.112V96H80v.026c-7.146.003-14.217 3.161-18.944 9.24L0 183.766v95.694c0 13.455 11.011 24.791 24.464 24.536C37.505 303.748 48 293.1 48 280v-79.766l16-20.571v140.698L9.927 469.055c-6.04 16.609 2.528 34.969 19.138 41.009 16.602 6.039 34.968-2.524 41.009-19.138L136 309.638V202.441l-31.406-39.816a4 4 0 1 1 6.269-4.971l102.3 129.217c9.145 11.584 24.368 11.339 33.708 3.965 10.41-8.216 12.159-23.334 3.966-33.708z"],blog:[512,512,[],"f781","M172.2 226.8c-14.6-2.9-28.2 8.9-28.2 23.8V301c0 10.2 7.1 18.4 16.7 22 18.2 6.8 31.3 24.4 31.3 45 0 26.5-21.5 48-48 48s-48-21.5-48-48V120c0-13.3-10.7-24-24-24H24c-13.3 0-24 10.7-24 24v248c0 89.5 82.1 160.2 175 140.7 54.4-11.4 98.3-55.4 109.7-109.7 17.4-82.9-37-157.2-112.5-172.2zM209 0c-9.2-.5-17 6.8-17 16v31.6c0 8.5 6.6 15.5 15 15.9 129.4 7 233.4 112 240.9 241.5.5 8.4 7.5 15 15.9 15h32.1c9.2 0 16.5-7.8 16-17C503.4 139.8 372.2 8.6 209 0zm.3 96c-9.3-.7-17.3 6.7-17.3 16.1v32.1c0 8.4 6.5 15.3 14.8 15.9 76.8 6.3 138 68.2 144.9 145.2.8 8.3 7.6 14.7 15.9 14.7h32.2c9.3 0 16.8-8 16.1-17.3-8.4-110.1-96.5-198.2-206.6-206.7z"],bold:[384,512,[],"f032","M333.49 238a122 122 0 0 0 27-65.21C367.87 96.49 308 32 233.42 32H34a16 16 0 0 0-16 16v48a16 16 0 0 0 16 16h31.87v288H34a16 16 0 0 0-16 16v48a16 16 0 0 0 16 16h209.32c70.8 0 134.14-51.75 141-122.4 4.74-48.45-16.39-92.06-50.83-119.6zM145.66 112h87.76a48 48 0 0 1 0 96h-87.76zm87.76 288h-87.76V288h87.76a56 56 0 0 1 0 112z"],bolt:[320,512,[],"f0e7","M296 160H180.6l42.6-129.8C227.2 15 215.7 0 200 0H56C44 0 33.8 8.9 32.2 20.8l-32 240C-1.7 275.2 9.5 288 24 288h118.7L96.6 482.5c-3.6 15.2 8 29.5 23.3 29.5 8.4 0 16.4-4.4 20.8-12l176-304c9.3-15.9-2.2-36-20.7-36z"],bomb:[512,512,[],"f1e2","M440.5 88.5l-52 52L415 167c9.4 9.4 9.4 24.6 0 33.9l-17.4 17.4c11.8 26.1 18.4 55.1 18.4 85.6 0 114.9-93.1 208-208 208S0 418.9 0 304 93.1 96 208 96c30.5 0 59.5 6.6 85.6 18.4L311 97c9.4-9.4 24.6-9.4 33.9 0l26.5 26.5 52-52 17.1 17zM500 60h-24c-6.6 0-12 5.4-12 12s5.4 12 12 12h24c6.6 0 12-5.4 12-12s-5.4-12-12-12zM440 0c-6.6 0-12 5.4-12 12v24c0 6.6 5.4 12 12 12s12-5.4 12-12V12c0-6.6-5.4-12-12-12zm33.9 55l17-17c4.7-4.7 4.7-12.3 0-17-4.7-4.7-12.3-4.7-17 0l-17 17c-4.7 4.7-4.7 12.3 0 17 4.8 4.7 12.4 4.7 17 0zm-67.8 0c4.7 4.7 12.3 4.7 17 0 4.7-4.7 4.7-12.3 0-17l-17-17c-4.7-4.7-12.3-4.7-17 0-4.7 4.7-4.7 12.3 0 17l17 17zm67.8 34c-4.7-4.7-12.3-4.7-17 0-4.7 4.7-4.7 12.3 0 17l17 17c4.7 4.7 12.3 4.7 17 0 4.7-4.7 4.7-12.3 0-17l-17-17zM112 272c0-35.3 28.7-64 64-64 8.8 0 16-7.2 16-16s-7.2-16-16-16c-52.9 0-96 43.1-96 96 0 8.8 7.2 16 16 16s16-7.2 16-16z"],bone:[640,512,[],"f5d7","M598.88 244.56c25.2-12.6 41.12-38.36 41.12-66.53v-7.64C640 129.3 606.7 96 565.61 96c-32.02 0-60.44 20.49-70.57 50.86-7.68 23.03-11.6 45.14-38.11 45.14H183.06c-27.38 0-31.58-25.54-38.11-45.14C134.83 116.49 106.4 96 74.39 96 33.3 96 0 129.3 0 170.39v7.64c0 28.17 15.92 53.93 41.12 66.53 9.43 4.71 9.43 18.17 0 22.88C15.92 280.04 0 305.8 0 333.97v7.64C0 382.7 33.3 416 74.38 416c32.02 0 60.44-20.49 70.57-50.86 7.68-23.03 11.6-45.14 38.11-45.14h273.87c27.38 0 31.58 25.54 38.11 45.14C505.17 395.51 533.6 416 565.61 416c41.08 0 74.38-33.3 74.38-74.39v-7.64c0-28.18-15.92-53.93-41.12-66.53-9.42-4.71-9.42-18.17.01-22.88z"],bong:[448,512,[],"f55c","M302.5 512c23.18 0 44.43-12.58 56-32.66C374.69 451.26 384 418.75 384 384c0-36.12-10.08-69.81-27.44-98.62L400 241.94l9.38 9.38c6.25 6.25 16.38 6.25 22.63 0l11.3-11.32c6.25-6.25 6.25-16.38 0-22.63l-52.69-52.69c-6.25-6.25-16.38-6.25-22.63 0l-11.31 11.31c-6.25 6.25-6.25 16.38 0 22.63l9.38 9.38-39.41 39.41c-11.56-11.37-24.53-21.33-38.65-29.51V63.74l15.97-.02c8.82-.01 15.97-7.16 15.98-15.98l.04-31.72C320 7.17 312.82-.01 303.97 0L80.03.26c-8.82.01-15.97 7.16-15.98 15.98l-.04 31.73c-.01 8.85 7.17 16.02 16.02 16.01L96 63.96v153.93C38.67 251.1 0 312.97 0 384c0 34.75 9.31 67.27 25.5 95.34C37.08 499.42 58.33 512 81.5 512h221zM120.06 259.43L144 245.56V63.91l96-.11v181.76l23.94 13.87c24.81 14.37 44.12 35.73 56.56 60.57h-257c12.45-24.84 31.75-46.2 56.56-60.57z"],book:[448,512,[],"f02d","M448 360V24c0-13.3-10.7-24-24-24H96C43 0 0 43 0 96v320c0 53 43 96 96 96h328c13.3 0 24-10.7 24-24v-16c0-7.5-3.5-14.3-8.9-18.7-4.2-15.4-4.2-59.3 0-74.7 5.4-4.3 8.9-11.1 8.9-18.6zM128 134c0-3.3 2.7-6 6-6h212c3.3 0 6 2.7 6 6v20c0 3.3-2.7 6-6 6H134c-3.3 0-6-2.7-6-6v-20zm0 64c0-3.3 2.7-6 6-6h212c3.3 0 6 2.7 6 6v20c0 3.3-2.7 6-6 6H134c-3.3 0-6-2.7-6-6v-20zm253.4 250H96c-17.7 0-32-14.3-32-32 0-17.6 14.4-32 32-32h285.4c-1.9 17.1-1.9 46.9 0 64z"],"book-dead":[448,512,[],"f6b7","M272 136c8.8 0 16-7.2 16-16s-7.2-16-16-16-16 7.2-16 16 7.2 16 16 16zm176 222.4V25.6c0-16-9.6-25.6-25.6-25.6H96C41.6 0 0 41.6 0 96v320c0 54.4 41.6 96 96 96h326.4c12.8 0 25.6-9.6 25.6-25.6v-16c0-6.4-3.2-12.8-9.6-19.2-3.2-16-3.2-60.8 0-73.6 6.4-3.2 9.6-9.6 9.6-19.2zM240 56c44.2 0 80 28.7 80 64 0 20.9-12.7 39.2-32 50.9V184c0 8.8-7.2 16-16 16h-64c-8.8 0-16-7.2-16-16v-13.1c-19.3-11.7-32-30-32-50.9 0-35.3 35.8-64 80-64zM124.8 223.3l6.3-14.7c1.7-4.1 6.4-5.9 10.5-4.2l98.3 42.1 98.4-42.1c4.1-1.7 8.8.1 10.5 4.2l6.3 14.7c1.7 4.1-.1 8.8-4.2 10.5L280.6 264l70.3 30.1c4.1 1.7 5.9 6.4 4.2 10.5l-6.3 14.7c-1.7 4.1-6.4 5.9-10.5 4.2L240 281.4l-98.3 42.2c-4.1 1.7-8.8-.1-10.5-4.2l-6.3-14.7c-1.7-4.1.1-8.8 4.2-10.5l70.4-30.1-70.5-30.3c-4.1-1.7-5.9-6.4-4.2-10.5zm256 224.7H96c-19.2 0-32-12.8-32-32s16-32 32-32h284.8zM208 136c8.8 0 16-7.2 16-16s-7.2-16-16-16-16 7.2-16 16 7.2 16 16 16z"],"book-medical":[448,512,[],"f7e6","M448 358.4V25.6c0-16-9.6-25.6-25.6-25.6H96C41.6 0 0 41.6 0 96v320c0 54.4 41.6 96 96 96h326.4c12.8 0 25.6-9.6 25.6-25.6v-16q0-9.6-9.6-19.2c-3.2-16-3.2-60.8 0-73.6q9.6-4.8 9.6-19.2zM144 168a8 8 0 0 1 8-8h56v-56a8 8 0 0 1 8-8h48a8 8 0 0 1 8 8v56h56a8 8 0 0 1 8 8v48a8 8 0 0 1-8 8h-56v56a8 8 0 0 1-8 8h-48a8 8 0 0 1-8-8v-56h-56a8 8 0 0 1-8-8zm236.8 280H96c-19.2 0-32-12.8-32-32s16-32 32-32h284.8z"],"book-open":[576,512,[],"f518","M542.22 32.05c-54.8 3.11-163.72 14.43-230.96 55.59-4.64 2.84-7.27 7.89-7.27 13.17v363.87c0 11.55 12.63 18.85 23.28 13.49 69.18-34.82 169.23-44.32 218.7-46.92 16.89-.89 30.02-14.43 30.02-30.66V62.75c.01-17.71-15.35-31.74-33.77-30.7zM264.73 87.64C197.5 46.48 88.58 35.17 33.78 32.05 15.36 31.01 0 45.04 0 62.75V400.6c0 16.24 13.13 29.78 30.02 30.66 49.49 2.6 149.59 12.11 218.77 46.95 10.62 5.35 23.21-1.94 23.21-13.46V100.63c0-5.29-2.62-10.14-7.27-12.99z"],"book-reader":[512,512,[],"f5da","M352 96c0-53.02-42.98-96-96-96s-96 42.98-96 96 42.98 96 96 96 96-42.98 96-96zM233.59 241.1c-59.33-36.32-155.43-46.3-203.79-49.05C13.55 191.13 0 203.51 0 219.14v222.8c0 14.33 11.59 26.28 26.49 27.05 43.66 2.29 131.99 10.68 193.04 41.43 9.37 4.72 20.48-1.71 20.48-11.87V252.56c-.01-4.67-2.32-8.95-6.42-11.46zm248.61-49.05c-48.35 2.74-144.46 12.73-203.78 49.05-4.1 2.51-6.41 6.96-6.41 11.63v245.79c0 10.19 11.14 16.63 20.54 11.9 61.04-30.72 149.32-39.11 192.97-41.4 14.9-.78 26.49-12.73 26.49-27.06V219.14c-.01-15.63-13.56-28.01-29.81-27.09z"],bookmark:[384,512,[],"f02e","M0 512V48C0 21.49 21.49 0 48 0h288c26.51 0 48 21.49 48 48v464L192 400 0 512z"],"border-all":[448,512,[],"f84c","M416 32H32A32 32 0 0 0 0 64v384a32 32 0 0 0 32 32h384a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32zm-32 64v128H256V96zm-192 0v128H64V96zM64 416V288h128v128zm192 0V288h128v128z"],"border-none":[448,512,[],"f850","M240 224h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm96 0h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm96 0h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm-288 0h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm96 192h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm96 0h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm96 0h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-96h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-192h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zM240 320h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-192h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm-96 288h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm96-384h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16zm96 0h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16zm96 0h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16zM48 224H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0 192H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-96H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-192H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-96H16A16 16 0 0 0 0 48v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16zm96 0h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z"],"border-style":[448,512,[],"f853","M240 416h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm-96 0h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm192 0h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm96-192h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0 96h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0 96h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-288h-32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-96H32A32 32 0 0 0 0 64v400a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V96h368a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z"],"bowling-ball":[496,512,[],"f436","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zM120 192c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm64-96c0-17.7 14.3-32 32-32s32 14.3 32 32-14.3 32-32 32-32-14.3-32-32zm48 144c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],box:[512,512,[],"f466","M509.5 184.6L458.9 32.8C452.4 13.2 434.1 0 413.4 0H272v192h238.7c-.4-2.5-.4-5-1.2-7.4zM240 0H98.6c-20.7 0-39 13.2-45.5 32.8L2.5 184.6c-.8 2.4-.8 4.9-1.2 7.4H240V0zM0 224v240c0 26.5 21.5 48 48 48h416c26.5 0 48-21.5 48-48V224H0z"],"box-open":[640,512,[],"f49e","M425.7 256c-16.9 0-32.8-9-41.4-23.4L320 126l-64.2 106.6c-8.7 14.5-24.6 23.5-41.5 23.5-4.5 0-9-.6-13.3-1.9L64 215v178c0 14.7 10 27.5 24.2 31l216.2 54.1c10.2 2.5 20.9 2.5 31 0L551.8 424c14.2-3.6 24.2-16.4 24.2-31V215l-137 39.1c-4.3 1.3-8.8 1.9-13.3 1.9zm212.6-112.2L586.8 41c-3.1-6.2-9.8-9.8-16.7-8.9L320 64l91.7 152.1c3.8 6.3 11.4 9.3 18.5 7.3l197.9-56.5c9.9-2.9 14.7-13.9 10.2-23.1zM53.2 41L1.7 143.8c-4.6 9.2.3 20.2 10.1 23l197.9 56.5c7.1 2 14.7-1 18.5-7.3L320 64 69.8 32.1c-6.9-.8-13.5 2.7-16.6 8.9z"],"box-tissue":[512,512,[],"f95b","M383.88,287.82l64-192H338.47a70.2,70.2,0,0,1-66.59-48,70.21,70.21,0,0,0-66.6-48H63.88l64,288Zm-384,192a32,32,0,0,0,32,32h448a32,32,0,0,0,32-32v-64H-.12Zm480-256H438.94l-21.33,64h14.27a16,16,0,0,1,0,32h-352a16,16,0,1,1,0-32H95.09l-14.22-64h-49a32,32,0,0,0-32,32v128h512v-128A32,32,0,0,0,479.88,223.82Z"],boxes:[576,512,[],"f468","M560 288h-80v96l-32-21.3-32 21.3v-96h-80c-8.8 0-16 7.2-16 16v192c0 8.8 7.2 16 16 16h224c8.8 0 16-7.2 16-16V304c0-8.8-7.2-16-16-16zm-384-64h224c8.8 0 16-7.2 16-16V16c0-8.8-7.2-16-16-16h-80v96l-32-21.3L256 96V0h-80c-8.8 0-16 7.2-16 16v192c0 8.8 7.2 16 16 16zm64 64h-80v96l-32-21.3L96 384v-96H16c-8.8 0-16 7.2-16 16v192c0 8.8 7.2 16 16 16h224c8.8 0 16-7.2 16-16V304c0-8.8-7.2-16-16-16z"],braille:[640,512,[],"f2a1","M128 256c0 35.346-28.654 64-64 64S0 291.346 0 256s28.654-64 64-64 64 28.654 64 64zM64 384c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32zm0-352C28.654 32 0 60.654 0 96s28.654 64 64 64 64-28.654 64-64-28.654-64-64-64zm160 192c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32zm0 160c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32zm0-352c-35.346 0-64 28.654-64 64s28.654 64 64 64 64-28.654 64-64-28.654-64-64-64zm224 192c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32zm0 160c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32zm0-352c-35.346 0-64 28.654-64 64s28.654 64 64 64 64-28.654 64-64-28.654-64-64-64zm160 192c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32zm0 160c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32zm0-320c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32z"],brain:[576,512,[],"f5dc","M208 0c-29.9 0-54.7 20.5-61.8 48.2-.8 0-1.4-.2-2.2-.2-35.3 0-64 28.7-64 64 0 4.8.6 9.5 1.7 14C52.5 138 32 166.6 32 200c0 12.6 3.2 24.3 8.3 34.9C16.3 248.7 0 274.3 0 304c0 33.3 20.4 61.9 49.4 73.9-.9 4.6-1.4 9.3-1.4 14.1 0 39.8 32.2 72 72 72 4.1 0 8.1-.5 12-1.2 9.6 28.5 36.2 49.2 68 49.2 39.8 0 72-32.2 72-72V64c0-35.3-28.7-64-64-64zm368 304c0-29.7-16.3-55.3-40.3-69.1 5.2-10.6 8.3-22.3 8.3-34.9 0-33.4-20.5-62-49.7-74 1-4.5 1.7-9.2 1.7-14 0-35.3-28.7-64-64-64-.8 0-1.5.2-2.2.2C422.7 20.5 397.9 0 368 0c-35.3 0-64 28.6-64 64v376c0 39.8 32.2 72 72 72 31.8 0 58.4-20.7 68-49.2 3.9.7 7.9 1.2 12 1.2 39.8 0 72-32.2 72-72 0-4.8-.5-9.5-1.4-14.1 29-12 49.4-40.6 49.4-73.9z"],"bread-slice":[576,512,[],"f7ec","M288 0C108 0 0 93.4 0 169.14 0 199.44 24.24 224 64 224v256c0 17.67 16.12 32 36 32h376c19.88 0 36-14.33 36-32V224c39.76 0 64-24.56 64-54.86C576 93.4 468 0 288 0z"],briefcase:[512,512,[],"f0b1","M320 336c0 8.84-7.16 16-16 16h-96c-8.84 0-16-7.16-16-16v-48H0v144c0 25.6 22.4 48 48 48h416c25.6 0 48-22.4 48-48V288H320v48zm144-208h-80V80c0-25.6-22.4-48-48-48H176c-25.6 0-48 22.4-48 48v48H48c-25.6 0-48 22.4-48 48v80h512v-80c0-25.6-22.4-48-48-48zm-144 0H192V96h128v32z"],"briefcase-medical":[512,512,[],"f469","M464 128h-80V80c0-26.5-21.5-48-48-48H176c-26.5 0-48 21.5-48 48v48H48c-26.5 0-48 21.5-48 48v288c0 26.5 21.5 48 48 48h416c26.5 0 48-21.5 48-48V176c0-26.5-21.5-48-48-48zM192 96h128v32H192V96zm160 248c0 4.4-3.6 8-8 8h-56v56c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8v-56h-56c-4.4 0-8-3.6-8-8v-48c0-4.4 3.6-8 8-8h56v-56c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v56h56c4.4 0 8 3.6 8 8v48z"],"broadcast-tower":[640,512,[],"f519","M150.94 192h33.73c11.01 0 18.61-10.83 14.86-21.18-4.93-13.58-7.55-27.98-7.55-42.82s2.62-29.24 7.55-42.82C203.29 74.83 195.68 64 184.67 64h-33.73c-7.01 0-13.46 4.49-15.41 11.23C130.64 92.21 128 109.88 128 128c0 18.12 2.64 35.79 7.54 52.76 1.94 6.74 8.39 11.24 15.4 11.24zM89.92 23.34C95.56 12.72 87.97 0 75.96 0H40.63c-6.27 0-12.14 3.59-14.74 9.31C9.4 45.54 0 85.65 0 128c0 24.75 3.12 68.33 26.69 118.86 2.62 5.63 8.42 9.14 14.61 9.14h34.84c12.02 0 19.61-12.74 13.95-23.37-49.78-93.32-16.71-178.15-.17-209.29zM614.06 9.29C611.46 3.58 605.6 0 599.33 0h-35.42c-11.98 0-19.66 12.66-14.02 23.25 18.27 34.29 48.42 119.42.28 209.23-5.72 10.68 1.8 23.52 13.91 23.52h35.23c6.27 0 12.13-3.58 14.73-9.29C630.57 210.48 640 170.36 640 128s-9.42-82.48-25.94-118.71zM489.06 64h-33.73c-11.01 0-18.61 10.83-14.86 21.18 4.93 13.58 7.55 27.98 7.55 42.82s-2.62 29.24-7.55 42.82c-3.76 10.35 3.85 21.18 14.86 21.18h33.73c7.02 0 13.46-4.49 15.41-11.24 4.9-16.97 7.53-34.64 7.53-52.76 0-18.12-2.64-35.79-7.54-52.76-1.94-6.75-8.39-11.24-15.4-11.24zm-116.3 100.12c7.05-10.29 11.2-22.71 11.2-36.12 0-35.35-28.63-64-63.96-64-35.32 0-63.96 28.65-63.96 64 0 13.41 4.15 25.83 11.2 36.12l-130.5 313.41c-3.4 8.15.46 17.52 8.61 20.92l29.51 12.31c8.15 3.4 17.52-.46 20.91-8.61L244.96 384h150.07l49.2 118.15c3.4 8.16 12.76 12.01 20.91 8.61l29.51-12.31c8.15-3.4 12-12.77 8.61-20.92l-130.5-313.41zM271.62 320L320 203.81 368.38 320h-96.76z"],broom:[640,512,[],"f51a","M256.47 216.77l86.73 109.18s-16.6 102.36-76.57 150.12C206.66 523.85 0 510.19 0 510.19s3.8-23.14 11-55.43l94.62-112.17c3.97-4.7-.87-11.62-6.65-9.5l-60.4 22.09c14.44-41.66 32.72-80.04 54.6-97.47 59.97-47.76 163.3-40.94 163.3-40.94zM636.53 31.03l-19.86-25c-5.49-6.9-15.52-8.05-22.41-2.56l-232.48 177.8-34.14-42.97c-5.09-6.41-15.14-5.21-18.59 2.21l-25.33 54.55 86.73 109.18 58.8-12.45c8-1.69 11.42-11.2 6.34-17.6l-34.09-42.92 232.48-177.8c6.89-5.48 8.04-15.53 2.55-22.44z"],brush:[384,512,[],"f55d","M352 0H32C14.33 0 0 14.33 0 32v224h384V32c0-17.67-14.33-32-32-32zM0 320c0 35.35 28.66 64 64 64h64v64c0 35.35 28.66 64 64 64s64-28.65 64-64v-64h64c35.34 0 64-28.65 64-64v-32H0v32zm192 104c13.25 0 24 10.74 24 24 0 13.25-10.75 24-24 24s-24-10.75-24-24c0-13.26 10.75-24 24-24z"],bug:[512,512,[],"f188","M511.988 288.9c-.478 17.43-15.217 31.1-32.653 31.1H424v16c0 21.864-4.882 42.584-13.6 61.145l60.228 60.228c12.496 12.497 12.496 32.758 0 45.255-12.498 12.497-32.759 12.496-45.256 0l-54.736-54.736C345.886 467.965 314.351 480 280 480V236c0-6.627-5.373-12-12-12h-24c-6.627 0-12 5.373-12 12v244c-34.351 0-65.886-12.035-90.636-32.108l-54.736 54.736c-12.498 12.497-32.759 12.496-45.256 0-12.496-12.497-12.496-32.758 0-45.255l60.228-60.228C92.882 378.584 88 357.864 88 336v-16H32.666C15.23 320 .491 306.33.013 288.9-.484 270.816 14.028 256 32 256h56v-58.745l-46.628-46.628c-12.496-12.497-12.496-32.758 0-45.255 12.498-12.497 32.758-12.497 45.256 0L141.255 160h229.489l54.627-54.627c12.498-12.497 32.758-12.497 45.256 0 12.496 12.497 12.496 32.758 0 45.255L424 197.255V256h56c17.972 0 32.484 14.816 31.988 32.9zM257 0c-61.856 0-112 50.144-112 112h224C369 50.144 318.856 0 257 0z"],building:[448,512,[],"f1ad","M436 480h-20V24c0-13.255-10.745-24-24-24H56C42.745 0 32 10.745 32 24v456H12c-6.627 0-12 5.373-12 12v20h448v-20c0-6.627-5.373-12-12-12zM128 76c0-6.627 5.373-12 12-12h40c6.627 0 12 5.373 12 12v40c0 6.627-5.373 12-12 12h-40c-6.627 0-12-5.373-12-12V76zm0 96c0-6.627 5.373-12 12-12h40c6.627 0 12 5.373 12 12v40c0 6.627-5.373 12-12 12h-40c-6.627 0-12-5.373-12-12v-40zm52 148h-40c-6.627 0-12-5.373-12-12v-40c0-6.627 5.373-12 12-12h40c6.627 0 12 5.373 12 12v40c0 6.627-5.373 12-12 12zm76 160h-64v-84c0-6.627 5.373-12 12-12h40c6.627 0 12 5.373 12 12v84zm64-172c0 6.627-5.373 12-12 12h-40c-6.627 0-12-5.373-12-12v-40c0-6.627 5.373-12 12-12h40c6.627 0 12 5.373 12 12v40zm0-96c0 6.627-5.373 12-12 12h-40c-6.627 0-12-5.373-12-12v-40c0-6.627 5.373-12 12-12h40c6.627 0 12 5.373 12 12v40zm0-96c0 6.627-5.373 12-12 12h-40c-6.627 0-12-5.373-12-12V76c0-6.627 5.373-12 12-12h40c6.627 0 12 5.373 12 12v40z"],bullhorn:[576,512,[],"f0a1","M576 240c0-23.63-12.95-44.04-32-55.12V32.01C544 23.26 537.02 0 512 0c-7.12 0-14.19 2.38-19.98 7.02l-85.03 68.03C364.28 109.19 310.66 128 256 128H64c-35.35 0-64 28.65-64 64v96c0 35.35 28.65 64 64 64h33.7c-1.39 10.48-2.18 21.14-2.18 32 0 39.77 9.26 77.35 25.56 110.94 5.19 10.69 16.52 17.06 28.4 17.06h74.28c26.05 0 41.69-29.84 25.9-50.56-16.4-21.52-26.15-48.36-26.15-77.44 0-11.11 1.62-21.79 4.41-32H256c54.66 0 108.28 18.81 150.98 52.95l85.03 68.03a32.023 32.023 0 0 0 19.98 7.02c24.92 0 32-22.78 32-32V295.13C563.05 284.04 576 263.63 576 240zm-96 141.42l-33.05-26.44C392.95 311.78 325.12 288 256 288v-96c69.12 0 136.95-23.78 190.95-66.98L480 98.58v282.84z"],bullseye:[496,512,[],"f140","M248 8C111.03 8 0 119.03 0 256s111.03 248 248 248 248-111.03 248-248S384.97 8 248 8zm0 432c-101.69 0-184-82.29-184-184 0-101.69 82.29-184 184-184 101.69 0 184 82.29 184 184 0 101.69-82.29 184-184 184zm0-312c-70.69 0-128 57.31-128 128s57.31 128 128 128 128-57.31 128-128-57.31-128-128-128zm0 192c-35.29 0-64-28.71-64-64s28.71-64 64-64 64 28.71 64 64-28.71 64-64 64z"],burn:[384,512,[],"f46a","M192 0C79.7 101.3 0 220.9 0 300.5 0 425 79 512 192 512s192-87 192-211.5c0-79.9-80.2-199.6-192-300.5zm0 448c-56.5 0-96-39-96-94.8 0-13.5 4.6-61.5 96-161.2 91.4 99.7 96 147.7 96 161.2 0 55.8-39.5 94.8-96 94.8z"],bus:[512,512,[],"f207","M488 128h-8V80c0-44.8-99.2-80-224-80S32 35.2 32 80v48h-8c-13.25 0-24 10.74-24 24v80c0 13.25 10.75 24 24 24h8v160c0 17.67 14.33 32 32 32v32c0 17.67 14.33 32 32 32h32c17.67 0 32-14.33 32-32v-32h192v32c0 17.67 14.33 32 32 32h32c17.67 0 32-14.33 32-32v-32h6.4c16 0 25.6-12.8 25.6-25.6V256h8c13.25 0 24-10.75 24-24v-80c0-13.26-10.75-24-24-24zM112 400c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm16-112c-17.67 0-32-14.33-32-32V128c0-17.67 14.33-32 32-32h256c17.67 0 32 14.33 32 32v128c0 17.67-14.33 32-32 32H128zm272 112c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],"bus-alt":[512,512,[],"f55e","M488 128h-8V80c0-44.8-99.2-80-224-80S32 35.2 32 80v48h-8c-13.25 0-24 10.74-24 24v80c0 13.25 10.75 24 24 24h8v160c0 17.67 14.33 32 32 32v32c0 17.67 14.33 32 32 32h32c17.67 0 32-14.33 32-32v-32h192v32c0 17.67 14.33 32 32 32h32c17.67 0 32-14.33 32-32v-32h6.4c16 0 25.6-12.8 25.6-25.6V256h8c13.25 0 24-10.75 24-24v-80c0-13.26-10.75-24-24-24zM160 72c0-4.42 3.58-8 8-8h176c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H168c-4.42 0-8-3.58-8-8V72zm-48 328c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm128-112H128c-17.67 0-32-14.33-32-32v-96c0-17.67 14.33-32 32-32h112v160zm32 0V128h112c17.67 0 32 14.33 32 32v96c0 17.67-14.33 32-32 32H272zm128 112c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],"business-time":[640,512,[],"f64a","M496 224c-79.59 0-144 64.41-144 144s64.41 144 144 144 144-64.41 144-144-64.41-144-144-144zm64 150.29c0 5.34-4.37 9.71-9.71 9.71h-60.57c-5.34 0-9.71-4.37-9.71-9.71v-76.57c0-5.34 4.37-9.71 9.71-9.71h12.57c5.34 0 9.71 4.37 9.71 9.71V352h38.29c5.34 0 9.71 4.37 9.71 9.71v12.58zM496 192c5.4 0 10.72.33 16 .81V144c0-25.6-22.4-48-48-48h-80V48c0-25.6-22.4-48-48-48H176c-25.6 0-48 22.4-48 48v48H48c-25.6 0-48 22.4-48 48v80h395.12c28.6-20.09 63.35-32 100.88-32zM320 96H192V64h128v32zm6.82 224H208c-8.84 0-16-7.16-16-16v-48H0v144c0 25.6 22.4 48 48 48h291.43C327.1 423.96 320 396.82 320 368c0-16.66 2.48-32.72 6.82-48z"],calculator:[448,512,[],"f1ec","M400 0H48C22.4 0 0 22.4 0 48v416c0 25.6 22.4 48 48 48h352c25.6 0 48-22.4 48-48V48c0-25.6-22.4-48-48-48zM128 435.2c0 6.4-6.4 12.8-12.8 12.8H76.8c-6.4 0-12.8-6.4-12.8-12.8v-38.4c0-6.4 6.4-12.8 12.8-12.8h38.4c6.4 0 12.8 6.4 12.8 12.8v38.4zm0-128c0 6.4-6.4 12.8-12.8 12.8H76.8c-6.4 0-12.8-6.4-12.8-12.8v-38.4c0-6.4 6.4-12.8 12.8-12.8h38.4c6.4 0 12.8 6.4 12.8 12.8v38.4zm128 128c0 6.4-6.4 12.8-12.8 12.8h-38.4c-6.4 0-12.8-6.4-12.8-12.8v-38.4c0-6.4 6.4-12.8 12.8-12.8h38.4c6.4 0 12.8 6.4 12.8 12.8v38.4zm0-128c0 6.4-6.4 12.8-12.8 12.8h-38.4c-6.4 0-12.8-6.4-12.8-12.8v-38.4c0-6.4 6.4-12.8 12.8-12.8h38.4c6.4 0 12.8 6.4 12.8 12.8v38.4zm128 128c0 6.4-6.4 12.8-12.8 12.8h-38.4c-6.4 0-12.8-6.4-12.8-12.8V268.8c0-6.4 6.4-12.8 12.8-12.8h38.4c6.4 0 12.8 6.4 12.8 12.8v166.4zm0-256c0 6.4-6.4 12.8-12.8 12.8H76.8c-6.4 0-12.8-6.4-12.8-12.8V76.8C64 70.4 70.4 64 76.8 64h294.4c6.4 0 12.8 6.4 12.8 12.8v102.4z"],calendar:[448,512,[],"f133","M12 192h424c6.6 0 12 5.4 12 12v260c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48V204c0-6.6 5.4-12 12-12zm436-44v-36c0-26.5-21.5-48-48-48h-48V12c0-6.6-5.4-12-12-12h-40c-6.6 0-12 5.4-12 12v52H160V12c0-6.6-5.4-12-12-12h-40c-6.6 0-12 5.4-12 12v52H48C21.5 64 0 85.5 0 112v36c0 6.6 5.4 12 12 12h424c6.6 0 12-5.4 12-12z"],"calendar-alt":[448,512,[],"f073","M0 464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V192H0v272zm320-196c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40zm0 128c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40zM192 268c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40zm0 128c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40zM64 268c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12H76c-6.6 0-12-5.4-12-12v-40zm0 128c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12H76c-6.6 0-12-5.4-12-12v-40zM400 64h-48V16c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v48H160V16c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v48H48C21.5 64 0 85.5 0 112v48h448v-48c0-26.5-21.5-48-48-48z"],"calendar-check":[448,512,[],"f274","M436 160H12c-6.627 0-12-5.373-12-12v-36c0-26.51 21.49-48 48-48h48V12c0-6.627 5.373-12 12-12h40c6.627 0 12 5.373 12 12v52h128V12c0-6.627 5.373-12 12-12h40c6.627 0 12 5.373 12 12v52h48c26.51 0 48 21.49 48 48v36c0 6.627-5.373 12-12 12zM12 192h424c6.627 0 12 5.373 12 12v260c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48V204c0-6.627 5.373-12 12-12zm333.296 95.947l-28.169-28.398c-4.667-4.705-12.265-4.736-16.97-.068L194.12 364.665l-45.98-46.352c-4.667-4.705-12.266-4.736-16.971-.068l-28.397 28.17c-4.705 4.667-4.736 12.265-.068 16.97l82.601 83.269c4.667 4.705 12.265 4.736 16.97.068l142.953-141.805c4.705-4.667 4.736-12.265.068-16.97z"],"calendar-day":[448,512,[],"f783","M0 464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V192H0v272zm64-192c0-8.8 7.2-16 16-16h96c8.8 0 16 7.2 16 16v96c0 8.8-7.2 16-16 16H80c-8.8 0-16-7.2-16-16v-96zM400 64h-48V16c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v48H160V16c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v48H48C21.5 64 0 85.5 0 112v48h448v-48c0-26.5-21.5-48-48-48z"],"calendar-minus":[448,512,[],"f272","M436 160H12c-6.6 0-12-5.4-12-12v-36c0-26.5 21.5-48 48-48h48V12c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v52h128V12c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v52h48c26.5 0 48 21.5 48 48v36c0 6.6-5.4 12-12 12zM12 192h424c6.6 0 12 5.4 12 12v260c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48V204c0-6.6 5.4-12 12-12zm304 192c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12H132c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h184z"],"calendar-plus":[448,512,[],"f271","M436 160H12c-6.6 0-12-5.4-12-12v-36c0-26.5 21.5-48 48-48h48V12c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v52h128V12c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v52h48c26.5 0 48 21.5 48 48v36c0 6.6-5.4 12-12 12zM12 192h424c6.6 0 12 5.4 12 12v260c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48V204c0-6.6 5.4-12 12-12zm316 140c0-6.6-5.4-12-12-12h-60v-60c0-6.6-5.4-12-12-12h-40c-6.6 0-12 5.4-12 12v60h-60c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h60v60c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12v-60h60c6.6 0 12-5.4 12-12v-40z"],"calendar-times":[448,512,[],"f273","M436 160H12c-6.6 0-12-5.4-12-12v-36c0-26.5 21.5-48 48-48h48V12c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v52h128V12c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v52h48c26.5 0 48 21.5 48 48v36c0 6.6-5.4 12-12 12zM12 192h424c6.6 0 12 5.4 12 12v260c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48V204c0-6.6 5.4-12 12-12zm257.3 160l48.1-48.1c4.7-4.7 4.7-12.3 0-17l-28.3-28.3c-4.7-4.7-12.3-4.7-17 0L224 306.7l-48.1-48.1c-4.7-4.7-12.3-4.7-17 0l-28.3 28.3c-4.7 4.7-4.7 12.3 0 17l48.1 48.1-48.1 48.1c-4.7 4.7-4.7 12.3 0 17l28.3 28.3c4.7 4.7 12.3 4.7 17 0l48.1-48.1 48.1 48.1c4.7 4.7 12.3 4.7 17 0l28.3-28.3c4.7-4.7 4.7-12.3 0-17L269.3 352z"],"calendar-week":[448,512,[],"f784","M0 464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V192H0v272zm64-192c0-8.8 7.2-16 16-16h288c8.8 0 16 7.2 16 16v64c0 8.8-7.2 16-16 16H80c-8.8 0-16-7.2-16-16v-64zM400 64h-48V16c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v48H160V16c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v48H48C21.5 64 0 85.5 0 112v48h448v-48c0-26.5-21.5-48-48-48z"],camera:[512,512,[],"f030","M512 144v288c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48V144c0-26.5 21.5-48 48-48h88l12.3-32.9c7-18.7 24.9-31.1 44.9-31.1h125.5c20 0 37.9 12.4 44.9 31.1L376 96h88c26.5 0 48 21.5 48 48zM376 288c0-66.2-53.8-120-120-120s-120 53.8-120 120 53.8 120 120 120 120-53.8 120-120zm-32 0c0 48.5-39.5 88-88 88s-88-39.5-88-88 39.5-88 88-88 88 39.5 88 88z"],"camera-retro":[512,512,[],"f083","M48 32C21.5 32 0 53.5 0 80v352c0 26.5 21.5 48 48 48h416c26.5 0 48-21.5 48-48V80c0-26.5-21.5-48-48-48H48zm0 32h106c3.3 0 6 2.7 6 6v20c0 3.3-2.7 6-6 6H38c-3.3 0-6-2.7-6-6V80c0-8.8 7.2-16 16-16zm426 96H38c-3.3 0-6-2.7-6-6v-36c0-3.3 2.7-6 6-6h138l30.2-45.3c1.1-1.7 3-2.7 5-2.7H464c8.8 0 16 7.2 16 16v74c0 3.3-2.7 6-6 6zM256 424c-66.2 0-120-53.8-120-120s53.8-120 120-120 120 53.8 120 120-53.8 120-120 120zm0-208c-48.5 0-88 39.5-88 88s39.5 88 88 88 88-39.5 88-88-39.5-88-88-88zm-48 104c-8.8 0-16-7.2-16-16 0-35.3 28.7-64 64-64 8.8 0 16 7.2 16 16s-7.2 16-16 16c-17.6 0-32 14.4-32 32 0 8.8-7.2 16-16 16z"],campground:[640,512,[],"f6bb","M624 448h-24.68L359.54 117.75l53.41-73.55c5.19-7.15 3.61-17.16-3.54-22.35l-25.9-18.79c-7.15-5.19-17.15-3.61-22.35 3.55L320 63.3 278.83 6.6c-5.19-7.15-15.2-8.74-22.35-3.55l-25.88 18.8c-7.15 5.19-8.74 15.2-3.54 22.35l53.41 73.55L40.68 448H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h608c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zM320 288l116.36 160H203.64L320 288z"],"candy-cane":[512,512,[],"f786","M497.5 92C469.6 33.1 411.8 0 352.4 0c-27.9 0-56.2 7.3-81.8 22.6L243.1 39c-15.2 9.1-20.1 28.7-11 43.9l32.8 54.9c6 10 16.6 15.6 27.5 15.6 5.6 0 11.2-1.5 16.4-4.5l27.5-16.4c5.1-3.1 10.8-4.5 16.4-4.5 10.9 0 21.5 5.6 27.5 15.6 9.1 15.1 4.1 34.8-11 43.9L15.6 397.6c-15.2 9.1-20.1 28.7-11 43.9l32.8 54.9c6 10 16.6 15.6 27.5 15.6 5.6 0 11.2-1.5 16.4-4.5L428.6 301c71.7-42.9 104.6-133.5 68.9-209zm-177.7 13l-2.5 1.5L296.8 45c9.7-4.7 19.8-8.1 30.3-10.2l20.6 61.8c-9.8.8-19.4 3.3-27.9 8.4zM145.9 431.8l-60.5-38.5 30.8-18.3 60.5 38.5-30.8 18.3zm107.5-63.9l-60.5-38.5 30.8-18.3 60.5 38.5-30.8 18.3zM364.3 302l-60.5-38.5 30.8-18.3 60.5 38.5-30.8 18.3zm20.4-197.3l46-46c8.4 6.5 16 14.1 22.6 22.6L407.6 127c-5.7-9.3-13.7-16.9-22.9-22.3zm82.1 107.8l-59.5-19.8c3.2-5.3 5.8-10.9 7.4-17.1 1.1-4.5 1.7-9.1 1.8-13.6l60.4 20.1c-2.1 10.4-5.5 20.6-10.1 30.4z"],cannabis:[512,512,[],"f55f","M503.47 360.25c-1.56-.82-32.39-16.89-76.78-25.81 64.25-75.12 84.05-161.67 84.93-165.64 1.18-5.33-.44-10.9-4.3-14.77-3.03-3.04-7.12-4.7-11.32-4.7-1.14 0-2.29.12-3.44.38-3.88.85-86.54 19.59-160.58 79.76.01-1.46.01-2.93.01-4.4 0-118.79-59.98-213.72-62.53-217.7A15.973 15.973 0 0 0 256 0c-5.45 0-10.53 2.78-13.47 7.37-2.55 3.98-62.53 98.91-62.53 217.7 0 1.47.01 2.94.01 4.4-74.03-60.16-156.69-78.9-160.58-79.76-1.14-.25-2.29-.38-3.44-.38-4.2 0-8.29 1.66-11.32 4.7A15.986 15.986 0 0 0 .38 168.8c.88 3.97 20.68 90.52 84.93 165.64-44.39 8.92-75.21 24.99-76.78 25.81a16.003 16.003 0 0 0-.02 28.29c2.45 1.29 60.76 31.72 133.49 31.72 6.14 0 11.96-.1 17.5-.31-11.37 22.23-16.52 38.31-16.81 39.22-1.8 5.68-.29 11.89 3.91 16.11a16.019 16.019 0 0 0 16.1 3.99c1.83-.57 37.72-11.99 77.3-39.29V504c0 4.42 3.58 8 8 8h16c4.42 0 8-3.58 8-8v-64.01c39.58 27.3 75.47 38.71 77.3 39.29a16.019 16.019 0 0 0 16.1-3.99c4.2-4.22 5.71-10.43 3.91-16.11-.29-.91-5.45-16.99-16.81-39.22 5.54.21 11.37.31 17.5.31 72.72 0 131.04-30.43 133.49-31.72 5.24-2.78 8.52-8.22 8.51-14.15-.01-5.94-3.29-11.39-8.53-14.15z"],capsules:[576,512,[],"f46b","M555.3 300.1L424.2 112.8C401.9 81 366.4 64 330.4 64c-22.6 0-45.5 6.7-65.5 20.7-19.7 13.8-33.7 32.8-41.5 53.8C220.5 79.2 172 32 112 32 50.1 32 0 82.1 0 144v224c0 61.9 50.1 112 112 112s112-50.1 112-112V218.9c3.3 8.6 7.3 17.1 12.8 25L368 431.2c22.2 31.8 57.7 48.8 93.8 48.8 22.7 0 45.5-6.7 65.5-20.7 51.7-36.2 64.2-107.5 28-159.2zM160 256H64V144c0-26.5 21.5-48 48-48s48 21.5 48 48v112zm194.8 44.9l-65.6-93.7c-7.7-11-10.7-24.4-8.3-37.6 2.3-13.2 9.7-24.8 20.7-32.5 8.5-6 18.5-9.1 28.8-9.1 16.5 0 31.9 8 41.3 21.5l65.6 93.7-82.5 57.7z"],car:[512,512,[],"f1b9","M499.99 176h-59.87l-16.64-41.6C406.38 91.63 365.57 64 319.5 64h-127c-46.06 0-86.88 27.63-103.99 70.4L71.87 176H12.01C4.2 176-1.53 183.34.37 190.91l6 24C7.7 220.25 12.5 224 18.01 224h20.07C24.65 235.73 16 252.78 16 272v48c0 16.12 6.16 30.67 16 41.93V416c0 17.67 14.33 32 32 32h32c17.67 0 32-14.33 32-32v-32h256v32c0 17.67 14.33 32 32 32h32c17.67 0 32-14.33 32-32v-54.07c9.84-11.25 16-25.8 16-41.93v-48c0-19.22-8.65-36.27-22.07-48H494c5.51 0 10.31-3.75 11.64-9.09l6-24c1.89-7.57-3.84-14.91-11.65-14.91zm-352.06-17.83c7.29-18.22 24.94-30.17 44.57-30.17h127c19.63 0 37.28 11.95 44.57 30.17L384 208H128l19.93-49.83zM96 319.8c-19.2 0-32-12.76-32-31.9S76.8 256 96 256s48 28.71 48 47.85-28.8 15.95-48 15.95zm320 0c-19.2 0-48 3.19-48-15.95S396.8 256 416 256s32 12.76 32 31.9-12.8 31.9-32 31.9z"],"car-alt":[480,512,[],"f5de","M438.66 212.33l-11.24-28.1-19.93-49.83C390.38 91.63 349.57 64 303.5 64h-127c-46.06 0-86.88 27.63-103.99 70.4l-19.93 49.83-11.24 28.1C17.22 221.5 0 244.66 0 272v48c0 16.12 6.16 30.67 16 41.93V416c0 17.67 14.33 32 32 32h32c17.67 0 32-14.33 32-32v-32h256v32c0 17.67 14.33 32 32 32h32c17.67 0 32-14.33 32-32v-54.07c9.84-11.25 16-25.8 16-41.93v-48c0-27.34-17.22-50.5-41.34-59.67zm-306.73-54.16c7.29-18.22 24.94-30.17 44.57-30.17h127c19.63 0 37.28 11.95 44.57 30.17L368 208H112l19.93-49.83zM80 319.8c-19.2 0-32-12.76-32-31.9S60.8 256 80 256s48 28.71 48 47.85-28.8 15.95-48 15.95zm320 0c-19.2 0-48 3.19-48-15.95S380.8 256 400 256s32 12.76 32 31.9-12.8 31.9-32 31.9z"],"car-battery":[512,512,[],"f5df","M480 128h-32V80c0-8.84-7.16-16-16-16h-96c-8.84 0-16 7.16-16 16v48H192V80c0-8.84-7.16-16-16-16H80c-8.84 0-16 7.16-16 16v48H32c-17.67 0-32 14.33-32 32v256c0 17.67 14.33 32 32 32h448c17.67 0 32-14.33 32-32V160c0-17.67-14.33-32-32-32zM192 264c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h112c4.42 0 8 3.58 8 8v16zm256 0c0 4.42-3.58 8-8 8h-40v40c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8v-40h-40c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h40v-40c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v40h40c4.42 0 8 3.58 8 8v16z"],"car-crash":[640,512,[],"f5e1","M143.25 220.81l-12.42 46.37c-3.01 11.25-3.63 22.89-2.41 34.39l-35.2 28.98c-6.57 5.41-16.31-.43-14.62-8.77l15.44-76.68c1.06-5.26-2.66-10.28-8-10.79l-77.86-7.55c-8.47-.82-11.23-11.83-4.14-16.54l65.15-43.3c4.46-2.97 5.38-9.15 1.98-13.29L21.46 93.22c-5.41-6.57.43-16.3 8.78-14.62l76.68 15.44c5.26 1.06 10.28-2.66 10.8-8l7.55-77.86c.82-8.48 11.83-11.23 16.55-4.14l43.3 65.14c2.97 4.46 9.15 5.38 13.29 1.98l60.4-49.71c6.57-5.41 16.3.43 14.62 8.77L262.1 86.38c-2.71 3.05-5.43 6.09-7.91 9.4l-32.15 42.97-10.71 14.32c-32.73 8.76-59.18 34.53-68.08 67.74zm494.57 132.51l-12.42 46.36c-3.13 11.68-9.38 21.61-17.55 29.36a66.876 66.876 0 0 1-8.76 7l-13.99 52.23c-1.14 4.27-3.1 8.1-5.65 11.38-7.67 9.84-20.74 14.68-33.54 11.25L515 502.62c-17.07-4.57-27.2-22.12-22.63-39.19l8.28-30.91-247.28-66.26-8.28 30.91c-4.57 17.07-22.12 27.2-39.19 22.63l-30.91-8.28c-12.8-3.43-21.7-14.16-23.42-26.51-.57-4.12-.35-8.42.79-12.68l13.99-52.23a66.62 66.62 0 0 1-4.09-10.45c-3.2-10.79-3.65-22.52-.52-34.2l12.42-46.37c5.31-19.8 19.36-34.83 36.89-42.21a64.336 64.336 0 0 1 18.49-4.72l18.13-24.23 32.15-42.97c3.45-4.61 7.19-8.9 11.2-12.84 8-7.89 17.03-14.44 26.74-19.51 4.86-2.54 9.89-4.71 15.05-6.49 10.33-3.58 21.19-5.63 32.24-6.04 11.05-.41 22.31.82 33.43 3.8l122.68 32.87c11.12 2.98 21.48 7.54 30.85 13.43a111.11 111.11 0 0 1 34.69 34.5c8.82 13.88 14.64 29.84 16.68 46.99l6.36 53.29 3.59 30.05a64.49 64.49 0 0 1 22.74 29.93c4.39 11.88 5.29 25.19 1.75 38.39zM255.58 234.34c-18.55-4.97-34.21 4.04-39.17 22.53-4.96 18.49 4.11 34.12 22.65 39.09 18.55 4.97 45.54 15.51 50.49-2.98 4.96-18.49-15.43-53.67-33.97-58.64zm290.61 28.17l-6.36-53.29c-.58-4.87-1.89-9.53-3.82-13.86-5.8-12.99-17.2-23.01-31.42-26.82l-122.68-32.87a48.008 48.008 0 0 0-50.86 17.61l-32.15 42.97 172 46.08 75.29 20.18zm18.49 54.65c-18.55-4.97-53.8 15.31-58.75 33.79-4.95 18.49 23.69 22.86 42.24 27.83 18.55 4.97 34.21-4.04 39.17-22.53 4.95-18.48-4.11-34.12-22.66-39.09z"],"car-side":[640,512,[],"f5e4","M544 192h-16L419.22 56.02A64.025 64.025 0 0 0 369.24 32H155.33c-26.17 0-49.7 15.93-59.42 40.23L48 194.26C20.44 201.4 0 226.21 0 256v112c0 8.84 7.16 16 16 16h48c0 53.02 42.98 96 96 96s96-42.98 96-96h128c0 53.02 42.98 96 96 96s96-42.98 96-96h48c8.84 0 16-7.16 16-16v-80c0-53.02-42.98-96-96-96zM160 432c-26.47 0-48-21.53-48-48s21.53-48 48-48 48 21.53 48 48-21.53 48-48 48zm72-240H116.93l38.4-96H232v96zm48 0V96h89.24l76.8 96H280zm200 240c-26.47 0-48-21.53-48-48s21.53-48 48-48 48 21.53 48 48-21.53 48-48 48z"],caravan:[640,512,[],"f8ff","M416,208a16,16,0,1,0,16,16A16,16,0,0,0,416,208ZM624,320H576V160A160,160,0,0,0,416,0H64A64,64,0,0,0,0,64V320a64,64,0,0,0,64,64H96a96,96,0,0,0,192,0H624a16,16,0,0,0,16-16V336A16,16,0,0,0,624,320ZM192,432a48,48,0,1,1,48-48A48.05,48.05,0,0,1,192,432Zm64-240a32,32,0,0,1-32,32H96a32,32,0,0,1-32-32V128A32,32,0,0,1,96,96H224a32,32,0,0,1,32,32ZM448,320H320V128a32,32,0,0,1,32-32h64a32,32,0,0,1,32,32Z"],"caret-down":[320,512,[],"f0d7","M31.3 192h257.3c17.8 0 26.7 21.5 14.1 34.1L174.1 354.8c-7.8 7.8-20.5 7.8-28.3 0L17.2 226.1C4.6 213.5 13.5 192 31.3 192z"],"caret-left":[192,512,[],"f0d9","M192 127.338v257.324c0 17.818-21.543 26.741-34.142 14.142L29.196 270.142c-7.81-7.81-7.81-20.474 0-28.284l128.662-128.662c12.599-12.6 34.142-3.676 34.142 14.142z"],"caret-right":[192,512,[],"f0da","M0 384.662V127.338c0-17.818 21.543-26.741 34.142-14.142l128.662 128.662c7.81 7.81 7.81 20.474 0 28.284L34.142 398.804C21.543 411.404 0 402.48 0 384.662z"],"caret-square-down":[448,512,[],"f150","M448 80v352c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48V80c0-26.5 21.5-48 48-48h352c26.5 0 48 21.5 48 48zM92.5 220.5l123 123c4.7 4.7 12.3 4.7 17 0l123-123c7.6-7.6 2.2-20.5-8.5-20.5H101c-10.7 0-16.1 12.9-8.5 20.5z"],"caret-square-left":[448,512,[],"f191","M400 480H48c-26.51 0-48-21.49-48-48V80c0-26.51 21.49-48 48-48h352c26.51 0 48 21.49 48 48v352c0 26.51-21.49 48-48 48zM259.515 124.485l-123.03 123.03c-4.686 4.686-4.686 12.284 0 16.971l123.029 123.029c7.56 7.56 20.485 2.206 20.485-8.485V132.971c.001-10.691-12.925-16.045-20.484-8.486z"],"caret-square-right":[448,512,[],"f152","M48 32h352c26.51 0 48 21.49 48 48v352c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48V80c0-26.51 21.49-48 48-48zm140.485 355.515l123.029-123.029c4.686-4.686 4.686-12.284 0-16.971l-123.029-123.03c-7.56-7.56-20.485-2.206-20.485 8.485v246.059c0 10.691 12.926 16.045 20.485 8.486z"],"caret-square-up":[448,512,[],"f151","M0 432V80c0-26.51 21.49-48 48-48h352c26.51 0 48 21.49 48 48v352c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48zm355.515-140.485l-123.03-123.03c-4.686-4.686-12.284-4.686-16.971 0L92.485 291.515c-7.56 7.56-2.206 20.485 8.485 20.485h246.059c10.691 0 16.045-12.926 8.486-20.485z"],"caret-up":[320,512,[],"f0d8","M288.662 352H31.338c-17.818 0-26.741-21.543-14.142-34.142l128.662-128.662c7.81-7.81 20.474-7.81 28.284 0l128.662 128.662c12.6 12.599 3.676 34.142-14.142 34.142z"],carrot:[512,512,[],"f787","M298.2 156.6c-52.7-25.7-114.5-10.5-150.2 32.8l55.2 55.2c6.3 6.3 6.3 16.4 0 22.6-3.1 3.1-7.2 4.7-11.3 4.7s-8.2-1.6-11.3-4.7L130.4 217 2.3 479.7c-2.9 6-3.1 13.3 0 19.7 5.4 11.1 18.9 15.7 30 10.3l133.6-65.2-49.2-49.2c-6.3-6.2-6.3-16.4 0-22.6 6.3-6.2 16.4-6.2 22.6 0l57 57 102-49.8c24-11.7 44.5-31.3 57.1-57.1 30.1-61.7 4.5-136.1-57.2-166.2zm92.1-34.9C409.8 81 399.7 32.9 360 0c-50.3 41.7-52.5 107.5-7.9 151.9l8 8c44.4 44.6 110.3 42.4 151.9-7.9-32.9-39.7-81-49.8-121.7-30.3z"],"cart-arrow-down":[576,512,[],"f218","M504.717 320H211.572l6.545 32h268.418c15.401 0 26.816 14.301 23.403 29.319l-5.517 24.276C523.112 414.668 536 433.828 536 456c0 31.202-25.519 56.444-56.824 55.994-29.823-.429-54.35-24.631-55.155-54.447-.44-16.287 6.085-31.049 16.803-41.548H231.176C241.553 426.165 248 440.326 248 456c0 31.813-26.528 57.431-58.67 55.938-28.54-1.325-51.751-24.385-53.251-52.917-1.158-22.034 10.436-41.455 28.051-51.586L93.883 64H24C10.745 64 0 53.255 0 40V24C0 10.745 10.745 0 24 0h102.529c11.401 0 21.228 8.021 23.513 19.19L159.208 64H551.99c15.401 0 26.816 14.301 23.403 29.319l-47.273 208C525.637 312.246 515.923 320 504.717 320zM403.029 192H360v-60c0-6.627-5.373-12-12-12h-24c-6.627 0-12 5.373-12 12v60h-43.029c-10.691 0-16.045 12.926-8.485 20.485l67.029 67.029c4.686 4.686 12.284 4.686 16.971 0l67.029-67.029c7.559-7.559 2.205-20.485-8.486-20.485z"],"cart-plus":[576,512,[],"f217","M504.717 320H211.572l6.545 32h268.418c15.401 0 26.816 14.301 23.403 29.319l-5.517 24.276C523.112 414.668 536 433.828 536 456c0 31.202-25.519 56.444-56.824 55.994-29.823-.429-54.35-24.631-55.155-54.447-.44-16.287 6.085-31.049 16.803-41.548H231.176C241.553 426.165 248 440.326 248 456c0 31.813-26.528 57.431-58.67 55.938-28.54-1.325-51.751-24.385-53.251-52.917-1.158-22.034 10.436-41.455 28.051-51.586L93.883 64H24C10.745 64 0 53.255 0 40V24C0 10.745 10.745 0 24 0h102.529c11.401 0 21.228 8.021 23.513 19.19L159.208 64H551.99c15.401 0 26.816 14.301 23.403 29.319l-47.273 208C525.637 312.246 515.923 320 504.717 320zM408 168h-48v-40c0-8.837-7.163-16-16-16h-16c-8.837 0-16 7.163-16 16v40h-48c-8.837 0-16 7.163-16 16v16c0 8.837 7.163 16 16 16h48v40c0 8.837 7.163 16 16 16h16c8.837 0 16-7.163 16-16v-40h48c8.837 0 16-7.163 16-16v-16c0-8.837-7.163-16-16-16z"],"cash-register":[512,512,[],"f788","M511.1 378.8l-26.7-160c-2.6-15.4-15.9-26.7-31.6-26.7H208v-64h96c8.8 0 16-7.2 16-16V16c0-8.8-7.2-16-16-16H48c-8.8 0-16 7.2-16 16v96c0 8.8 7.2 16 16 16h96v64H59.1c-15.6 0-29 11.3-31.6 26.7L.8 378.7c-.6 3.5-.9 7-.9 10.5V480c0 17.7 14.3 32 32 32h448c17.7 0 32-14.3 32-32v-90.7c.1-3.5-.2-7-.8-10.5zM280 248c0-8.8 7.2-16 16-16h16c8.8 0 16 7.2 16 16v16c0 8.8-7.2 16-16 16h-16c-8.8 0-16-7.2-16-16v-16zm-32 64h16c8.8 0 16 7.2 16 16v16c0 8.8-7.2 16-16 16h-16c-8.8 0-16-7.2-16-16v-16c0-8.8 7.2-16 16-16zm-32-80c8.8 0 16 7.2 16 16v16c0 8.8-7.2 16-16 16h-16c-8.8 0-16-7.2-16-16v-16c0-8.8 7.2-16 16-16h16zM80 80V48h192v32H80zm40 200h-16c-8.8 0-16-7.2-16-16v-16c0-8.8 7.2-16 16-16h16c8.8 0 16 7.2 16 16v16c0 8.8-7.2 16-16 16zm16 64v-16c0-8.8 7.2-16 16-16h16c8.8 0 16 7.2 16 16v16c0 8.8-7.2 16-16 16h-16c-8.8 0-16-7.2-16-16zm216 112c0 4.4-3.6 8-8 8H168c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h176c4.4 0 8 3.6 8 8v16zm24-112c0 8.8-7.2 16-16 16h-16c-8.8 0-16-7.2-16-16v-16c0-8.8 7.2-16 16-16h16c8.8 0 16 7.2 16 16v16zm48-80c0 8.8-7.2 16-16 16h-16c-8.8 0-16-7.2-16-16v-16c0-8.8 7.2-16 16-16h16c8.8 0 16 7.2 16 16v16z"],cat:[512,512,[],"f6be","M290.59 192c-20.18 0-106.82 1.98-162.59 85.95V192c0-52.94-43.06-96-96-96-17.67 0-32 14.33-32 32s14.33 32 32 32c17.64 0 32 14.36 32 32v256c0 35.3 28.7 64 64 64h176c8.84 0 16-7.16 16-16v-16c0-17.67-14.33-32-32-32h-32l128-96v144c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16V289.86c-10.29 2.67-20.89 4.54-32 4.54-61.81 0-113.52-44.05-125.41-102.4zM448 96h-64l-64-64v134.4c0 53.02 42.98 96 96 96s96-42.98 96-96V32l-64 64zm-72 80c-8.84 0-16-7.16-16-16s7.16-16 16-16 16 7.16 16 16-7.16 16-16 16zm80 0c-8.84 0-16-7.16-16-16s7.16-16 16-16 16 7.16 16 16-7.16 16-16 16z"],certificate:[512,512,[],"f0a3","M458.622 255.92l45.985-45.005c13.708-12.977 7.316-36.039-10.664-40.339l-62.65-15.99 17.661-62.015c4.991-17.838-11.829-34.663-29.661-29.671l-61.994 17.667-15.984-62.671C337.085.197 313.765-6.276 300.99 7.228L256 53.57 211.011 7.229c-12.63-13.351-36.047-7.234-40.325 10.668l-15.984 62.671-61.995-17.667C74.87 57.907 58.056 74.738 63.046 92.572l17.661 62.015-62.65 15.99C.069 174.878-6.31 197.944 7.392 210.915l45.985 45.005-45.985 45.004c-13.708 12.977-7.316 36.039 10.664 40.339l62.65 15.99-17.661 62.015c-4.991 17.838 11.829 34.663 29.661 29.671l61.994-17.667 15.984 62.671c4.439 18.575 27.696 24.018 40.325 10.668L256 458.61l44.989 46.001c12.5 13.488 35.987 7.486 40.325-10.668l15.984-62.671 61.994 17.667c17.836 4.994 34.651-11.837 29.661-29.671l-17.661-62.015 62.65-15.99c17.987-4.302 24.366-27.367 10.664-40.339l-45.984-45.004z"],chair:[448,512,[],"f6c0","M112 128c0-29.5 16.2-55 40-68.9V256h48V48h48v208h48V59.1c23.8 13.9 40 39.4 40 68.9v128h48V128C384 57.3 326.7 0 256 0h-64C121.3 0 64 57.3 64 128v128h48zm334.3 213.9l-10.7-32c-4.4-13.1-16.6-21.9-30.4-21.9H42.7c-13.8 0-26 8.8-30.4 21.9l-10.7 32C-5.2 362.6 10.2 384 32 384v112c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V384h256v112c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V384c21.8 0 37.2-21.4 30.3-42.1z"],chalkboard:[640,512,[],"f51b","M96 64h448v352h64V40c0-22.06-17.94-40-40-40H72C49.94 0 32 17.94 32 40v376h64V64zm528 384H480v-64H288v64H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h608c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16z"],"chalkboard-teacher":[640,512,[],"f51c","M208 352c-2.39 0-4.78.35-7.06 1.09C187.98 357.3 174.35 360 160 360c-14.35 0-27.98-2.7-40.95-6.91-2.28-.74-4.66-1.09-7.05-1.09C49.94 352-.33 402.48 0 464.62.14 490.88 21.73 512 48 512h224c26.27 0 47.86-21.12 48-47.38.33-62.14-49.94-112.62-112-112.62zm-48-32c53.02 0 96-42.98 96-96s-42.98-96-96-96-96 42.98-96 96 42.98 96 96 96zM592 0H208c-26.47 0-48 22.25-48 49.59V96c23.42 0 45.1 6.78 64 17.8V64h352v288h-64v-64H384v64h-76.24c19.1 16.69 33.12 38.73 39.69 64H592c26.47 0 48-22.25 48-49.59V49.59C640 22.25 618.47 0 592 0z"],"charging-station":[576,512,[],"f5e7","M336 448H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h320c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zm208-320V80c0-8.84-7.16-16-16-16s-16 7.16-16 16v48h-32V80c0-8.84-7.16-16-16-16s-16 7.16-16 16v48h-16c-8.84 0-16 7.16-16 16v32c0 35.76 23.62 65.69 56 75.93v118.49c0 13.95-9.5 26.92-23.26 29.19C431.22 402.5 416 388.99 416 372v-28c0-48.6-39.4-88-88-88h-8V64c0-35.35-28.65-64-64-64H96C60.65 0 32 28.65 32 64v352h288V304h8c22.09 0 40 17.91 40 40v24.61c0 39.67 28.92 75.16 68.41 79.01C481.71 452.05 520 416.41 520 372V251.93c32.38-10.24 56-40.17 56-75.93v-32c0-8.84-7.16-16-16-16h-16zm-283.91 47.76l-93.7 139c-2.2 3.33-6.21 5.24-10.39 5.24-7.67 0-13.47-6.28-11.67-12.92L167.35 224H108c-7.25 0-12.85-5.59-11.89-11.89l16-107C112.9 99.9 117.98 96 124 96h68c7.88 0 13.62 6.54 11.6 13.21L192 160h57.7c9.24 0 15.01 8.78 10.39 15.76z"],"chart-area":[512,512,[],"f1fe","M500 384c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12H12c-6.6 0-12-5.4-12-12V76c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v308h436zM372.7 159.5L288 216l-85.3-113.7c-5.1-6.8-15.5-6.3-19.9 1L96 248v104h384l-89.9-187.8c-3.2-6.5-11.4-8.7-17.4-4.7z"],"chart-bar":[512,512,[],"f080","M332.8 320h38.4c6.4 0 12.8-6.4 12.8-12.8V172.8c0-6.4-6.4-12.8-12.8-12.8h-38.4c-6.4 0-12.8 6.4-12.8 12.8v134.4c0 6.4 6.4 12.8 12.8 12.8zm96 0h38.4c6.4 0 12.8-6.4 12.8-12.8V76.8c0-6.4-6.4-12.8-12.8-12.8h-38.4c-6.4 0-12.8 6.4-12.8 12.8v230.4c0 6.4 6.4 12.8 12.8 12.8zm-288 0h38.4c6.4 0 12.8-6.4 12.8-12.8v-70.4c0-6.4-6.4-12.8-12.8-12.8h-38.4c-6.4 0-12.8 6.4-12.8 12.8v70.4c0 6.4 6.4 12.8 12.8 12.8zm96 0h38.4c6.4 0 12.8-6.4 12.8-12.8V108.8c0-6.4-6.4-12.8-12.8-12.8h-38.4c-6.4 0-12.8 6.4-12.8 12.8v198.4c0 6.4 6.4 12.8 12.8 12.8zM496 384H64V80c0-8.84-7.16-16-16-16H16C7.16 64 0 71.16 0 80v336c0 17.67 14.33 32 32 32h464c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16z"],"chart-line":[512,512,[],"f201","M496 384H64V80c0-8.84-7.16-16-16-16H16C7.16 64 0 71.16 0 80v336c0 17.67 14.33 32 32 32h464c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zM464 96H345.94c-21.38 0-32.09 25.85-16.97 40.97l32.4 32.4L288 242.75l-73.37-73.37c-12.5-12.5-32.76-12.5-45.25 0l-68.69 68.69c-6.25 6.25-6.25 16.38 0 22.63l22.62 22.62c6.25 6.25 16.38 6.25 22.63 0L192 237.25l73.37 73.37c12.5 12.5 32.76 12.5 45.25 0l96-96 32.4 32.4c15.12 15.12 40.97 4.41 40.97-16.97V112c.01-8.84-7.15-16-15.99-16z"],"chart-pie":[544,512,[],"f200","M527.79 288H290.5l158.03 158.03c6.04 6.04 15.98 6.53 22.19.68 38.7-36.46 65.32-85.61 73.13-140.86 1.34-9.46-6.51-17.85-16.06-17.85zm-15.83-64.8C503.72 103.74 408.26 8.28 288.8.04 279.68-.59 272 7.1 272 16.24V240h223.77c9.14 0 16.82-7.68 16.19-16.8zM224 288V50.71c0-9.55-8.39-17.4-17.84-16.06C86.99 51.49-4.1 155.6.14 280.37 4.5 408.51 114.83 513.59 243.03 511.98c50.4-.63 96.97-16.87 135.26-44.03 7.9-5.6 8.42-17.23 1.57-24.08L224 288z"],check:[512,512,[],"f00c","M173.898 439.404l-166.4-166.4c-9.997-9.997-9.997-26.206 0-36.204l36.203-36.204c9.997-9.998 26.207-9.998 36.204 0L192 312.69 432.095 72.596c9.997-9.997 26.207-9.997 36.204 0l36.203 36.204c9.997 9.997 9.997 26.206 0 36.204l-294.4 294.401c-9.998 9.997-26.207 9.997-36.204-.001z"],"check-circle":[512,512,[],"f058","M504 256c0 136.967-111.033 248-248 248S8 392.967 8 256 119.033 8 256 8s248 111.033 248 248zM227.314 387.314l184-184c6.248-6.248 6.248-16.379 0-22.627l-22.627-22.627c-6.248-6.249-16.379-6.249-22.628 0L216 308.118l-70.059-70.059c-6.248-6.248-16.379-6.248-22.628 0l-22.627 22.627c-6.248 6.248-6.248 16.379 0 22.627l104 104c6.249 6.249 16.379 6.249 22.628.001z"],"check-double":[512,512,[],"f560","M505 174.8l-39.6-39.6c-9.4-9.4-24.6-9.4-33.9 0L192 374.7 80.6 263.2c-9.4-9.4-24.6-9.4-33.9 0L7 302.9c-9.4 9.4-9.4 24.6 0 34L175 505c9.4 9.4 24.6 9.4 33.9 0l296-296.2c9.4-9.5 9.4-24.7.1-34zm-324.3 106c6.2 6.3 16.4 6.3 22.6 0l208-208.2c6.2-6.3 6.2-16.4 0-22.6L366.1 4.7c-6.2-6.3-16.4-6.3-22.6 0L192 156.2l-55.4-55.5c-6.2-6.3-16.4-6.3-22.6 0L68.7 146c-6.2 6.3-6.2 16.4 0 22.6l112 112.2z"],"check-square":[448,512,[],"f14a","M400 480H48c-26.51 0-48-21.49-48-48V80c0-26.51 21.49-48 48-48h352c26.51 0 48 21.49 48 48v352c0 26.51-21.49 48-48 48zm-204.686-98.059l184-184c6.248-6.248 6.248-16.379 0-22.627l-22.627-22.627c-6.248-6.248-16.379-6.249-22.628 0L184 302.745l-70.059-70.059c-6.248-6.248-16.379-6.248-22.628 0l-22.627 22.627c-6.248 6.248-6.248 16.379 0 22.627l104 104c6.249 6.25 16.379 6.25 22.628.001z"],cheese:[512,512,[],"f7ef","M0 288v160a32 32 0 0 0 32 32h448a32 32 0 0 0 32-32V288zM299.83 32a32 32 0 0 0-21.13 7L0 256h512c0-119.89-94-217.8-212.17-224z"],chess:[512,512,[],"f439","M74 208H64a16 16 0 0 0-16 16v16a16 16 0 0 0 16 16h15.94A535.78 535.78 0 0 1 64 384h128a535.78 535.78 0 0 1-15.94-128H192a16 16 0 0 0 16-16v-16a16 16 0 0 0-16-16h-10l33.89-90.38a16 16 0 0 0-15-21.62H144V64h24a8 8 0 0 0 8-8V40a8 8 0 0 0-8-8h-24V8a8 8 0 0 0-8-8h-16a8 8 0 0 0-8 8v24H88a8 8 0 0 0-8 8v16a8 8 0 0 0 8 8h24v32H55.09a16 16 0 0 0-15 21.62zm173.16 251.58L224 448v-16a16 16 0 0 0-16-16H48a16 16 0 0 0-16 16v16L8.85 459.58A16 16 0 0 0 0 473.89V496a16 16 0 0 0 16 16h224a16 16 0 0 0 16-16v-22.11a16 16 0 0 0-8.84-14.31zm92.77-157.78l-3.29 82.2h126.72l-3.29-82.21 24.6-20.79A32 32 0 0 0 496 256.54V198a6 6 0 0 0-6-6h-26.38a6 6 0 0 0-6 6v26h-24.71v-26a6 6 0 0 0-6-6H373.1a6 6 0 0 0-6 6v26h-24.71v-26a6 6 0 0 0-6-6H310a6 6 0 0 0-6 6v58.6a32 32 0 0 0 11.36 24.4zM384 304a16 16 0 0 1 32 0v32h-32zm119.16 155.58L480 448v-16a16 16 0 0 0-16-16H336a16 16 0 0 0-16 16v16l-23.15 11.58a16 16 0 0 0-8.85 14.31V496a16 16 0 0 0 16 16h192a16 16 0 0 0 16-16v-22.11a16 16 0 0 0-8.84-14.31z"],"chess-bishop":[320,512,[],"f43a","M8 287.88c0 51.64 22.14 73.83 56 84.6V416h192v-43.52c33.86-10.77 56-33 56-84.6 0-30.61-10.73-67.1-26.69-102.56L185 285.65a8 8 0 0 1-11.31 0l-11.31-11.31a8 8 0 0 1 0-11.31L270.27 155.1c-20.8-37.91-46.47-72.1-70.87-92.59C213.4 59.09 224 47.05 224 32a32 32 0 0 0-32-32h-64a32 32 0 0 0-32 32c0 15 10.6 27.09 24.6 30.51C67.81 106.8 8 214.5 8 287.88zM304 448H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h288a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"],"chess-board":[512,512,[],"f43c","M255.9.2h-64v64h64zM0 64.17v64h64v-64zM128 .2H64v64h64zm64 255.9v64h64v-64zM0 192.12v64h64v-64zM383.85.2h-64v64h64zm128 0h-64v64h64zM128 256.1H64v64h64zM511.8 448v-64h-64v64zm0-128v-64h-64v64zM383.85 512h64v-64h-64zm128-319.88v-64h-64v64zM128 512h64v-64h-64zM0 512h64v-64H0zm255.9 0h64v-64h-64zM0 320.07v64h64v-64zm319.88-191.92v-64h-64v64zm-64 128h64v-64h-64zm-64 128v64h64v-64zm128-64h64v-64h-64zm0-127.95h64v-64h-64zm0 191.93v64h64v-64zM64 384.05v64h64v-64zm128-255.9v-64h-64v64zm191.92 255.9h64v-64h-64zm-128-191.93v-64h-64v64zm128-127.95v64h64v-64zm-128 255.9v64h64v-64zm-64-127.95H128v64h64zm191.92 64h64v-64h-64zM128 128.15H64v64h64zm0 191.92v64h64v-64z"],"chess-king":[448,512,[],"f43f","M400 448H48a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h352a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm16-288H256v-48h40a8 8 0 0 0 8-8V56a8 8 0 0 0-8-8h-40V8a8 8 0 0 0-8-8h-48a8 8 0 0 0-8 8v40h-40a8 8 0 0 0-8 8v48a8 8 0 0 0 8 8h40v48H32a32 32 0 0 0-30.52 41.54L74.56 416h298.88l73.08-214.46A32 32 0 0 0 416 160z"],"chess-knight":[384,512,[],"f441","M19 272.47l40.63 18.06a32 32 0 0 0 24.88.47l12.78-5.12a32 32 0 0 0 18.76-20.5l9.22-30.65a24 24 0 0 1 12.55-15.65L159.94 208v50.33a48 48 0 0 1-26.53 42.94l-57.22 28.65A80 80 0 0 0 32 401.48V416h319.86V224c0-106-85.92-192-191.92-192H12A12 12 0 0 0 0 44a16.9 16.9 0 0 0 1.79 7.58L16 80l-9 9a24 24 0 0 0-7 17v137.21a32 32 0 0 0 19 29.26zM52 128a20 20 0 1 1-20 20 20 20 0 0 1 20-20zm316 320H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h352a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"],"chess-pawn":[320,512,[],"f443","M105.1 224H80a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h16v5.49c0 44-4.14 86.6-24 122.51h176c-19.89-35.91-24-78.51-24-122.51V288h16a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16h-25.1c29.39-18.38 49.1-50.78 49.1-88a104 104 0 0 0-208 0c0 37.22 19.71 69.62 49.1 88zM304 448H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h288a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"],"chess-queen":[512,512,[],"f445","M256 112a56 56 0 1 0-56-56 56 56 0 0 0 56 56zm176 336H80a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h352a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm72.87-263.84l-28.51-15.92c-7.44-5-16.91-2.46-22.29 4.68a47.59 47.59 0 0 1-47.23 18.23C383.7 186.86 368 164.93 368 141.4a13.4 13.4 0 0 0-13.4-13.4h-38.77c-6 0-11.61 4-12.86 9.91a48 48 0 0 1-93.94 0c-1.25-5.92-6.82-9.91-12.86-9.91H157.4a13.4 13.4 0 0 0-13.4 13.4c0 25.69-19 48.75-44.67 50.49a47.5 47.5 0 0 1-41.54-19.15c-5.28-7.09-14.73-9.45-22.09-4.54l-28.57 16a16 16 0 0 0-5.44 20.47L104.24 416h303.52l102.55-211.37a16 16 0 0 0-5.44-20.47z"],"chess-rook":[384,512,[],"f447","M368 32h-56a16 16 0 0 0-16 16v48h-48V48a16 16 0 0 0-16-16h-80a16 16 0 0 0-16 16v48H88.1V48a16 16 0 0 0-16-16H16A16 16 0 0 0 0 48v176l64 32c0 48.33-1.54 95-13.21 160h282.42C321.54 351 320 303.72 320 256l64-32V48a16 16 0 0 0-16-16zM224 320h-64v-64a32 32 0 0 1 64 0zm144 128H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h352a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"],"chevron-circle-down":[512,512,[],"f13a","M504 256c0 137-111 248-248 248S8 393 8 256 119 8 256 8s248 111 248 248zM273 369.9l135.5-135.5c9.4-9.4 9.4-24.6 0-33.9l-17-17c-9.4-9.4-24.6-9.4-33.9 0L256 285.1 154.4 183.5c-9.4-9.4-24.6-9.4-33.9 0l-17 17c-9.4 9.4-9.4 24.6 0 33.9L239 369.9c9.4 9.4 24.6 9.4 34 0z"],"chevron-circle-left":[512,512,[],"f137","M256 504C119 504 8 393 8 256S119 8 256 8s248 111 248 248-111 248-248 248zM142.1 273l135.5 135.5c9.4 9.4 24.6 9.4 33.9 0l17-17c9.4-9.4 9.4-24.6 0-33.9L226.9 256l101.6-101.6c9.4-9.4 9.4-24.6 0-33.9l-17-17c-9.4-9.4-24.6-9.4-33.9 0L142.1 239c-9.4 9.4-9.4 24.6 0 34z"],"chevron-circle-right":[512,512,[],"f138","M256 8c137 0 248 111 248 248S393 504 256 504 8 393 8 256 119 8 256 8zm113.9 231L234.4 103.5c-9.4-9.4-24.6-9.4-33.9 0l-17 17c-9.4 9.4-9.4 24.6 0 33.9L285.1 256 183.5 357.6c-9.4 9.4-9.4 24.6 0 33.9l17 17c9.4 9.4 24.6 9.4 33.9 0L369.9 273c9.4-9.4 9.4-24.6 0-34z"],"chevron-circle-up":[512,512,[],"f139","M8 256C8 119 119 8 256 8s248 111 248 248-111 248-248 248S8 393 8 256zm231-113.9L103.5 277.6c-9.4 9.4-9.4 24.6 0 33.9l17 17c9.4 9.4 24.6 9.4 33.9 0L256 226.9l101.6 101.6c9.4 9.4 24.6 9.4 33.9 0l17-17c9.4-9.4 9.4-24.6 0-33.9L273 142.1c-9.4-9.4-24.6-9.4-34 0z"],"chevron-down":[448,512,[],"f078","M207.029 381.476L12.686 187.132c-9.373-9.373-9.373-24.569 0-33.941l22.667-22.667c9.357-9.357 24.522-9.375 33.901-.04L224 284.505l154.745-154.021c9.379-9.335 24.544-9.317 33.901.04l22.667 22.667c9.373 9.373 9.373 24.569 0 33.941L240.971 381.476c-9.373 9.372-24.569 9.372-33.942 0z"],"chevron-left":[320,512,[],"f053","M34.52 239.03L228.87 44.69c9.37-9.37 24.57-9.37 33.94 0l22.67 22.67c9.36 9.36 9.37 24.52.04 33.9L131.49 256l154.02 154.75c9.34 9.38 9.32 24.54-.04 33.9l-22.67 22.67c-9.37 9.37-24.57 9.37-33.94 0L34.52 272.97c-9.37-9.37-9.37-24.57 0-33.94z"],"chevron-right":[320,512,[],"f054","M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z"],"chevron-up":[448,512,[],"f077","M240.971 130.524l194.343 194.343c9.373 9.373 9.373 24.569 0 33.941l-22.667 22.667c-9.357 9.357-24.522 9.375-33.901.04L224 227.495 69.255 381.516c-9.379 9.335-24.544 9.317-33.901-.04l-22.667-22.667c-9.373-9.373-9.373-24.569 0-33.941L207.03 130.525c9.372-9.373 24.568-9.373 33.941-.001z"],child:[384,512,[],"f1ae","M120 72c0-39.765 32.235-72 72-72s72 32.235 72 72c0 39.764-32.235 72-72 72s-72-32.236-72-72zm254.627 1.373c-12.496-12.497-32.758-12.497-45.254 0L242.745 160H141.254L54.627 73.373c-12.496-12.497-32.758-12.497-45.254 0-12.497 12.497-12.497 32.758 0 45.255L104 213.254V480c0 17.673 14.327 32 32 32h16c17.673 0 32-14.327 32-32V368h16v112c0 17.673 14.327 32 32 32h16c17.673 0 32-14.327 32-32V213.254l94.627-94.627c12.497-12.497 12.497-32.757 0-45.254z"],church:[640,512,[],"f51d","M464.46 246.68L352 179.2V128h48c8.84 0 16-7.16 16-16V80c0-8.84-7.16-16-16-16h-48V16c0-8.84-7.16-16-16-16h-32c-8.84 0-16 7.16-16 16v48h-48c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h48v51.2l-112.46 67.48A31.997 31.997 0 0 0 160 274.12V512h96v-96c0-35.35 28.65-64 64-64s64 28.65 64 64v96h96V274.12c0-11.24-5.9-21.66-15.54-27.44zM0 395.96V496c0 8.84 7.16 16 16 16h112V320L19.39 366.54A32.024 32.024 0 0 0 0 395.96zm620.61-29.42L512 320v192h112c8.84 0 16-7.16 16-16V395.96c0-12.8-7.63-24.37-19.39-29.42z"],circle:[512,512,[],"f111","M256 8C119 8 8 119 8 256s111 248 248 248 248-111 248-248S393 8 256 8z"],"circle-notch":[512,512,[],"f1ce","M288 39.056v16.659c0 10.804 7.281 20.159 17.686 23.066C383.204 100.434 440 171.518 440 256c0 101.689-82.295 184-184 184-101.689 0-184-82.295-184-184 0-84.47 56.786-155.564 134.312-177.219C216.719 75.874 224 66.517 224 55.712V39.064c0-15.709-14.834-27.153-30.046-23.234C86.603 43.482 7.394 141.206 8.003 257.332c.72 137.052 111.477 246.956 248.531 246.667C393.255 503.711 504 392.788 504 256c0-115.633-79.14-212.779-186.211-240.236C302.678 11.889 288 23.456 288 39.056z"],city:[640,512,[],"f64f","M616 192H480V24c0-13.26-10.74-24-24-24H312c-13.26 0-24 10.74-24 24v72h-64V16c0-8.84-7.16-16-16-16h-16c-8.84 0-16 7.16-16 16v80h-64V16c0-8.84-7.16-16-16-16H80c-8.84 0-16 7.16-16 16v80H24c-13.26 0-24 10.74-24 24v360c0 17.67 14.33 32 32 32h576c17.67 0 32-14.33 32-32V216c0-13.26-10.75-24-24-24zM128 404c0 6.63-5.37 12-12 12H76c-6.63 0-12-5.37-12-12v-40c0-6.63 5.37-12 12-12h40c6.63 0 12 5.37 12 12v40zm0-96c0 6.63-5.37 12-12 12H76c-6.63 0-12-5.37-12-12v-40c0-6.63 5.37-12 12-12h40c6.63 0 12 5.37 12 12v40zm0-96c0 6.63-5.37 12-12 12H76c-6.63 0-12-5.37-12-12v-40c0-6.63 5.37-12 12-12h40c6.63 0 12 5.37 12 12v40zm128 192c0 6.63-5.37 12-12 12h-40c-6.63 0-12-5.37-12-12v-40c0-6.63 5.37-12 12-12h40c6.63 0 12 5.37 12 12v40zm0-96c0 6.63-5.37 12-12 12h-40c-6.63 0-12-5.37-12-12v-40c0-6.63 5.37-12 12-12h40c6.63 0 12 5.37 12 12v40zm0-96c0 6.63-5.37 12-12 12h-40c-6.63 0-12-5.37-12-12v-40c0-6.63 5.37-12 12-12h40c6.63 0 12 5.37 12 12v40zm160 96c0 6.63-5.37 12-12 12h-40c-6.63 0-12-5.37-12-12v-40c0-6.63 5.37-12 12-12h40c6.63 0 12 5.37 12 12v40zm0-96c0 6.63-5.37 12-12 12h-40c-6.63 0-12-5.37-12-12v-40c0-6.63 5.37-12 12-12h40c6.63 0 12 5.37 12 12v40zm0-96c0 6.63-5.37 12-12 12h-40c-6.63 0-12-5.37-12-12V76c0-6.63 5.37-12 12-12h40c6.63 0 12 5.37 12 12v40zm160 288c0 6.63-5.37 12-12 12h-40c-6.63 0-12-5.37-12-12v-40c0-6.63 5.37-12 12-12h40c6.63 0 12 5.37 12 12v40zm0-96c0 6.63-5.37 12-12 12h-40c-6.63 0-12-5.37-12-12v-40c0-6.63 5.37-12 12-12h40c6.63 0 12 5.37 12 12v40z"],"clinic-medical":[576,512,[],"f7f2","M288 115L69.47 307.71c-1.62 1.46-3.69 2.14-5.47 3.35V496a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16V311.1c-1.7-1.16-3.72-1.82-5.26-3.2zm96 261a8 8 0 0 1-8 8h-56v56a8 8 0 0 1-8 8h-48a8 8 0 0 1-8-8v-56h-56a8 8 0 0 1-8-8v-48a8 8 0 0 1 8-8h56v-56a8 8 0 0 1 8-8h48a8 8 0 0 1 8 8v56h56a8 8 0 0 1 8 8zm186.69-139.72l-255.94-226a39.85 39.85 0 0 0-53.45 0l-256 226a16 16 0 0 0-1.21 22.6L25.5 282.7a16 16 0 0 0 22.6 1.21L277.42 81.63a16 16 0 0 1 21.17 0L527.91 283.9a16 16 0 0 0 22.6-1.21l21.4-23.82a16 16 0 0 0-1.22-22.59z"],clipboard:[384,512,[],"f328","M384 112v352c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48V112c0-26.51 21.49-48 48-48h80c0-35.29 28.71-64 64-64s64 28.71 64 64h80c26.51 0 48 21.49 48 48zM192 40c-13.255 0-24 10.745-24 24s10.745 24 24 24 24-10.745 24-24-10.745-24-24-24m96 114v-20a6 6 0 0 0-6-6H102a6 6 0 0 0-6 6v20a6 6 0 0 0 6 6h180a6 6 0 0 0 6-6z"],"clipboard-check":[384,512,[],"f46c","M336 64h-80c0-35.3-28.7-64-64-64s-64 28.7-64 64H48C21.5 64 0 85.5 0 112v352c0 26.5 21.5 48 48 48h288c26.5 0 48-21.5 48-48V112c0-26.5-21.5-48-48-48zM192 40c13.3 0 24 10.7 24 24s-10.7 24-24 24-24-10.7-24-24 10.7-24 24-24zm121.2 231.8l-143 141.8c-4.7 4.7-12.3 4.6-17-.1l-82.6-83.3c-4.7-4.7-4.6-12.3.1-17L99.1 285c4.7-4.7 12.3-4.6 17 .1l46 46.4 106-105.2c4.7-4.7 12.3-4.6 17 .1l28.2 28.4c4.7 4.8 4.6 12.3-.1 17z"],"clipboard-list":[384,512,[],"f46d","M336 64h-80c0-35.3-28.7-64-64-64s-64 28.7-64 64H48C21.5 64 0 85.5 0 112v352c0 26.5 21.5 48 48 48h288c26.5 0 48-21.5 48-48V112c0-26.5-21.5-48-48-48zM96 424c-13.3 0-24-10.7-24-24s10.7-24 24-24 24 10.7 24 24-10.7 24-24 24zm0-96c-13.3 0-24-10.7-24-24s10.7-24 24-24 24 10.7 24 24-10.7 24-24 24zm0-96c-13.3 0-24-10.7-24-24s10.7-24 24-24 24 10.7 24 24-10.7 24-24 24zm96-192c13.3 0 24 10.7 24 24s-10.7 24-24 24-24-10.7-24-24 10.7-24 24-24zm128 368c0 4.4-3.6 8-8 8H168c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h144c4.4 0 8 3.6 8 8v16zm0-96c0 4.4-3.6 8-8 8H168c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h144c4.4 0 8 3.6 8 8v16zm0-96c0 4.4-3.6 8-8 8H168c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h144c4.4 0 8 3.6 8 8v16z"],clock:[512,512,[],"f017","M256,8C119,8,8,119,8,256S119,504,256,504,504,393,504,256,393,8,256,8Zm92.49,313h0l-20,25a16,16,0,0,1-22.49,2.5h0l-67-49.72a40,40,0,0,1-15-31.23V112a16,16,0,0,1,16-16h32a16,16,0,0,1,16,16V256l58,42.5A16,16,0,0,1,348.49,321Z"],clone:[512,512,[],"f24d","M464 0c26.51 0 48 21.49 48 48v288c0 26.51-21.49 48-48 48H176c-26.51 0-48-21.49-48-48V48c0-26.51 21.49-48 48-48h288M176 416c-44.112 0-80-35.888-80-80V128H48c-26.51 0-48 21.49-48 48v288c0 26.51 21.49 48 48 48h288c26.51 0 48-21.49 48-48v-48H176z"],"closed-captioning":[512,512,[],"f20a","M464 64H48C21.5 64 0 85.5 0 112v288c0 26.5 21.5 48 48 48h416c26.5 0 48-21.5 48-48V112c0-26.5-21.5-48-48-48zM218.1 287.7c2.8-2.5 7.1-2.1 9.2.9l19.5 27.7c1.7 2.4 1.5 5.6-.5 7.7-53.6 56.8-172.8 32.1-172.8-67.9 0-97.3 121.7-119.5 172.5-70.1 2.1 2 2.5 3.2 1 5.7l-17.5 30.5c-1.9 3.1-6.2 4-9.1 1.7-40.8-32-94.6-14.9-94.6 31.2.1 48 51.1 70.5 92.3 32.6zm190.4 0c2.8-2.5 7.1-2.1 9.2.9l19.5 27.7c1.7 2.4 1.5 5.6-.5 7.7-53.5 56.9-172.7 32.1-172.7-67.9 0-97.3 121.7-119.5 172.5-70.1 2.1 2 2.5 3.2 1 5.7L420 222.2c-1.9 3.1-6.2 4-9.1 1.7-40.8-32-94.6-14.9-94.6 31.2 0 48 51 70.5 92.2 32.6z"],cloud:[640,512,[],"f0c2","M537.6 226.6c4.1-10.7 6.4-22.4 6.4-34.6 0-53-43-96-96-96-19.7 0-38.1 6-53.3 16.2C367 64.2 315.3 32 256 32c-88.4 0-160 71.6-160 160 0 2.7.1 5.4.2 8.1C40.2 219.8 0 273.2 0 336c0 79.5 64.5 144 144 144h368c70.7 0 128-57.3 128-128 0-61.9-44-113.6-102.4-125.4z"],"cloud-download-alt":[640,512,[],"f381","M537.6 226.6c4.1-10.7 6.4-22.4 6.4-34.6 0-53-43-96-96-96-19.7 0-38.1 6-53.3 16.2C367 64.2 315.3 32 256 32c-88.4 0-160 71.6-160 160 0 2.7.1 5.4.2 8.1C40.2 219.8 0 273.2 0 336c0 79.5 64.5 144 144 144h368c70.7 0 128-57.3 128-128 0-61.9-44-113.6-102.4-125.4zm-132.9 88.7L299.3 420.7c-6.2 6.2-16.4 6.2-22.6 0L171.3 315.3c-10.1-10.1-2.9-27.3 11.3-27.3H248V176c0-8.8 7.2-16 16-16h48c8.8 0 16 7.2 16 16v112h65.4c14.2 0 21.4 17.2 11.3 27.3z"],"cloud-meatball":[512,512,[],"f73b","M48 352c-26.5 0-48 21.5-48 48s21.5 48 48 48 48-21.5 48-48-21.5-48-48-48zm416 0c-26.5 0-48 21.5-48 48s21.5 48 48 48 48-21.5 48-48-21.5-48-48-48zm-119 11.1c4.6-14.5 1.6-30.8-9.8-42.3-11.5-11.5-27.8-14.4-42.3-9.9-7-13.5-20.7-23-36.9-23s-29.9 9.5-36.9 23c-14.5-4.6-30.8-1.6-42.3 9.9-11.5 11.5-14.4 27.8-9.9 42.3-13.5 7-23 20.7-23 36.9s9.5 29.9 23 36.9c-4.6 14.5-1.6 30.8 9.9 42.3 8.2 8.2 18.9 12.3 29.7 12.3 4.3 0 8.5-1.1 12.6-2.5 7 13.5 20.7 23 36.9 23s29.9-9.5 36.9-23c4.1 1.3 8.3 2.5 12.6 2.5 10.8 0 21.5-4.1 29.7-12.3 11.5-11.5 14.4-27.8 9.8-42.3 13.5-7 23-20.7 23-36.9s-9.5-29.9-23-36.9zM512 224c0-53-43-96-96-96-.6 0-1.1.2-1.6.2 1.1-5.2 1.6-10.6 1.6-16.2 0-44.2-35.8-80-80-80-24.6 0-46.3 11.3-61 28.8C256.4 24.8 219.3 0 176 0 114.1 0 64 50.1 64 112c0 7.3.8 14.3 2.1 21.2C27.8 145.8 0 181.5 0 224c0 53 43 96 96 96h43.4c3.6-8 8.4-15.4 14.8-21.8 13.5-13.5 31.5-21.1 50.8-21.3 13.5-13.2 31.7-20.9 51-20.9s37.5 7.7 51 20.9c19.3.2 37.3 7.8 50.8 21.3 6.4 6.4 11.3 13.8 14.8 21.8H416c53 0 96-43 96-96z"],"cloud-moon":[576,512,[],"f6c3","M342.8 352.7c5.7-9.6 9.2-20.7 9.2-32.7 0-35.3-28.7-64-64-64-17.2 0-32.8 6.9-44.3 17.9-16.3-29.6-47.5-49.9-83.7-49.9-53 0-96 43-96 96 0 2 .5 3.8.6 5.7C27.1 338.8 0 374.1 0 416c0 53 43 96 96 96h240c44.2 0 80-35.8 80-80 0-41.9-32.3-75.8-73.2-79.3zm222.5-54.3c-93.1 17.7-178.5-53.7-178.5-147.7 0-54.2 29-104 76.1-130.8 7.3-4.1 5.4-15.1-2.8-16.7C448.4 1.1 436.7 0 425 0 319.1 0 233.1 85.9 233.1 192c0 8.5.7 16.8 1.8 25 5.9 4.3 11.6 8.9 16.7 14.2 11.4-4.7 23.7-7.2 36.4-7.2 52.9 0 96 43.1 96 96 0 3.6-.2 7.2-.6 10.7 23.6 10.8 42.4 29.5 53.5 52.6 54.4-3.4 103.7-29.3 137.1-70.4 5.3-6.5-.5-16.1-8.7-14.5z"],"cloud-moon-rain":[576,512,[],"f73c","M350.5 225.5c-6.9-37.2-39.3-65.5-78.5-65.5-12.3 0-23.9 3-34.3 8-17.4-24.1-45.6-40-77.7-40-53 0-96 43-96 96 0 .5.2 1.1.2 1.6C27.6 232.9 0 265.2 0 304c0 44.2 35.8 80 80 80h256c44.2 0 80-35.8 80-80 0-39.2-28.2-71.7-65.5-78.5zm217.4-1.7c-70.4 13.3-135-40.3-135-110.8 0-40.6 21.9-78 57.5-98.1 5.5-3.1 4.1-11.4-2.1-12.5C479.6.8 470.7 0 461.8 0c-77.9 0-141.1 61.2-144.4 137.9 26.7 11.9 48.2 33.8 58.9 61.7 37.1 14.3 64 47.4 70.2 86.8 5.1.5 10 1.5 15.2 1.5 44.7 0 85.6-20.2 112.6-53.3 4.2-4.8-.2-12-6.4-10.8zM364.5 418.1c-7.6-4.3-17.4-1.8-21.8 6l-36.6 64c-4.4 7.7-1.7 17.4 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l36.6-64c4.3-7.7 1.7-17.4-6-21.8zm-96 0c-7.6-4.3-17.4-1.8-21.8 6l-36.6 64c-4.4 7.7-1.7 17.4 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l36.6-64c4.3-7.7 1.7-17.4-6-21.8zm-96 0c-7.6-4.3-17.4-1.8-21.8 6l-36.6 64c-4.4 7.7-1.7 17.4 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l36.6-64c4.3-7.7 1.7-17.4-6-21.8zm-96 0c-7.6-4.3-17.4-1.8-21.8 6l-36.6 64c-4.4 7.7-1.7 17.4 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l36.6-64c4.3-7.7 1.7-17.4-6-21.8z"],"cloud-rain":[512,512,[],"f73d","M416 128c-.6 0-1.1.2-1.6.2 1.1-5.2 1.6-10.6 1.6-16.2 0-44.2-35.8-80-80-80-24.6 0-46.3 11.3-61 28.8C256.4 24.8 219.3 0 176 0 114.1 0 64 50.1 64 112c0 7.3.8 14.3 2.1 21.2C27.8 145.8 0 181.5 0 224c0 53 43 96 96 96h320c53 0 96-43 96-96s-43-96-96-96zM88 374.2c-12.8 44.4-40 56.4-40 87.7 0 27.7 21.5 50.1 48 50.1s48-22.4 48-50.1c0-31.4-27.2-43.1-40-87.7-2.2-8.1-13.5-8.5-16 0zm160 0c-12.8 44.4-40 56.4-40 87.7 0 27.7 21.5 50.1 48 50.1s48-22.4 48-50.1c0-31.4-27.2-43.1-40-87.7-2.2-8.1-13.5-8.5-16 0zm160 0c-12.8 44.4-40 56.4-40 87.7 0 27.7 21.5 50.1 48 50.1s48-22.4 48-50.1c0-31.4-27.2-43.1-40-87.7-2.2-8.1-13.5-8.5-16 0z"],"cloud-showers-heavy":[512,512,[],"f740","M183.9 370.1c-7.6-4.4-17.4-1.8-21.8 6l-64 112c-4.4 7.7-1.7 17.5 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l64-112c4.4-7.6 1.7-17.4-6-21.8zm96 0c-7.6-4.4-17.4-1.8-21.8 6l-64 112c-4.4 7.7-1.7 17.5 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l64-112c4.4-7.6 1.7-17.4-6-21.8zm-192 0c-7.6-4.4-17.4-1.8-21.8 6l-64 112c-4.4 7.7-1.7 17.5 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l64-112c4.4-7.6 1.7-17.4-6-21.8zm384 0c-7.6-4.4-17.4-1.8-21.8 6l-64 112c-4.4 7.7-1.7 17.5 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l64-112c4.4-7.6 1.7-17.4-6-21.8zm-96 0c-7.6-4.4-17.4-1.8-21.8 6l-64 112c-4.4 7.7-1.7 17.5 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l64-112c4.4-7.6 1.7-17.4-6-21.8zM416 128c-.6 0-1.1.2-1.6.2 1.1-5.2 1.6-10.6 1.6-16.2 0-44.2-35.8-80-80-80-24.6 0-46.3 11.3-61 28.8C256.4 24.8 219.3 0 176 0 114.2 0 64 50.1 64 112c0 7.3.8 14.3 2.1 21.2C27.8 145.8 0 181.5 0 224c0 53 43 96 96 96h320c53 0 96-43 96-96s-43-96-96-96z"],"cloud-sun":[640,512,[],"f6c4","M575.2 325.7c.2-1.9.8-3.7.8-5.6 0-35.3-28.7-64-64-64-12.6 0-24.2 3.8-34.1 10-17.6-38.8-56.5-66-101.9-66-61.8 0-112 50.1-112 112 0 3 .7 5.8.9 8.7-49.6 3.7-88.9 44.7-88.9 95.3 0 53 43 96 96 96h272c53 0 96-43 96-96 0-42.1-27.2-77.4-64.8-90.4zm-430.4-22.6c-43.7-43.7-43.7-114.7 0-158.3 43.7-43.7 114.7-43.7 158.4 0 9.7 9.7 16.9 20.9 22.3 32.7 9.8-3.7 20.1-6 30.7-7.5L386 81.1c4-11.9-7.3-23.1-19.2-19.2L279 91.2 237.5 8.4C232-2.8 216-2.8 210.4 8.4L169 91.2 81.1 61.9C69.3 58 58 69.3 61.9 81.1l29.3 87.8-82.8 41.5c-11.2 5.6-11.2 21.5 0 27.1l82.8 41.4-29.3 87.8c-4 11.9 7.3 23.1 19.2 19.2l76.1-25.3c6.1-12.4 14-23.7 23.6-33.5-13.1-5.4-25.4-13.4-36-24zm-4.8-79.2c0 40.8 29.3 74.8 67.9 82.3 8-4.7 16.3-8.8 25.2-11.7 5.4-44.3 31-82.5 67.4-105C287.3 160.4 258 140 224 140c-46.3 0-84 37.6-84 83.9z"],"cloud-sun-rain":[576,512,[],"f743","M510.5 225.5c-6.9-37.2-39.3-65.5-78.5-65.5-12.3 0-23.9 3-34.3 8-17.4-24.1-45.6-40-77.7-40-53 0-96 43-96 96 0 .5.2 1.1.2 1.6C187.6 233 160 265.2 160 304c0 44.2 35.8 80 80 80h256c44.2 0 80-35.8 80-80 0-39.2-28.2-71.7-65.5-78.5zm-386.4 34.4c-37.4-37.4-37.4-98.3 0-135.8 34.6-34.6 89.1-36.8 126.7-7.4 20-12.9 43.6-20.7 69.2-20.7.7 0 1.3.2 2 .2l8.9-26.7c3.4-10.2-6.3-19.8-16.5-16.4l-75.3 25.1-35.5-71c-4.8-9.6-18.5-9.6-23.3 0l-35.5 71-75.3-25.1c-10.2-3.4-19.8 6.3-16.4 16.5l25.1 75.3-71 35.5c-9.6 4.8-9.6 18.5 0 23.3l71 35.5-25.1 75.3c-3.4 10.2 6.3 19.8 16.5 16.5l59.2-19.7c-.2-2.4-.7-4.7-.7-7.2 0-12.5 2.3-24.5 6.2-35.9-3.6-2.7-7.1-5.2-10.2-8.3zm69.8-58c4.3-24.5 15.8-46.4 31.9-64-9.8-6.2-21.4-9.9-33.8-9.9-35.3 0-64 28.7-64 64 0 18.7 8.2 35.4 21.1 47.1 11.3-15.9 26.6-28.9 44.8-37.2zm330.6 216.2c-7.6-4.3-17.4-1.8-21.8 6l-36.6 64c-4.4 7.7-1.7 17.4 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l36.6-64c4.3-7.7 1.7-17.4-6-21.8zm-96 0c-7.6-4.3-17.4-1.8-21.8 6l-36.6 64c-4.4 7.7-1.7 17.4 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l36.6-64c4.3-7.7 1.7-17.4-6-21.8zm-96 0c-7.6-4.3-17.4-1.8-21.8 6l-36.6 64c-4.4 7.7-1.7 17.4 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l36.6-64c4.3-7.7 1.7-17.4-6-21.8zm-96 0c-7.6-4.3-17.4-1.8-21.8 6l-36.6 64c-4.4 7.7-1.7 17.4 6 21.8 2.5 1.4 5.2 2.1 7.9 2.1 5.5 0 10.9-2.9 13.9-8.1l36.6-64c4.3-7.7 1.7-17.4-6-21.8z"],"cloud-upload-alt":[640,512,[],"f382","M537.6 226.6c4.1-10.7 6.4-22.4 6.4-34.6 0-53-43-96-96-96-19.7 0-38.1 6-53.3 16.2C367 64.2 315.3 32 256 32c-88.4 0-160 71.6-160 160 0 2.7.1 5.4.2 8.1C40.2 219.8 0 273.2 0 336c0 79.5 64.5 144 144 144h368c70.7 0 128-57.3 128-128 0-61.9-44-113.6-102.4-125.4zM393.4 288H328v112c0 8.8-7.2 16-16 16h-48c-8.8 0-16-7.2-16-16V288h-65.4c-14.3 0-21.4-17.2-11.3-27.3l105.4-105.4c6.2-6.2 16.4-6.2 22.6 0l105.4 105.4c10.1 10.1 2.9 27.3-11.3 27.3z"],cocktail:[576,512,[],"f561","M296 464h-56V338.78l168.74-168.73c15.52-15.52 4.53-42.05-17.42-42.05H24.68c-21.95 0-32.94 26.53-17.42 42.05L176 338.78V464h-56c-22.09 0-40 17.91-40 40 0 4.42 3.58 8 8 8h240c4.42 0 8-3.58 8-8 0-22.09-17.91-40-40-40zM432 0c-62.61 0-115.35 40.2-135.18 96h52.54c16.65-28.55 47.27-48 82.64-48 52.93 0 96 43.06 96 96s-43.07 96-96 96c-14.04 0-27.29-3.2-39.32-8.64l-35.26 35.26C379.23 279.92 404.59 288 432 288c79.53 0 144-64.47 144-144S511.53 0 432 0z"],code:[640,512,[],"f121","M278.9 511.5l-61-17.7c-6.4-1.8-10-8.5-8.2-14.9L346.2 8.7c1.8-6.4 8.5-10 14.9-8.2l61 17.7c6.4 1.8 10 8.5 8.2 14.9L293.8 503.3c-1.9 6.4-8.5 10.1-14.9 8.2zm-114-112.2l43.5-46.4c4.6-4.9 4.3-12.7-.8-17.2L117 256l90.6-79.7c5.1-4.5 5.5-12.3.8-17.2l-43.5-46.4c-4.5-4.8-12.1-5.1-17-.5L3.8 247.2c-5.1 4.7-5.1 12.8 0 17.5l144.1 135.1c4.9 4.6 12.5 4.4 17-.5zm327.2.6l144.1-135.1c5.1-4.7 5.1-12.8 0-17.5L492.1 112.1c-4.8-4.5-12.4-4.3-17 .5L431.6 159c-4.6 4.9-4.3 12.7.8 17.2L523 256l-90.6 79.7c-5.1 4.5-5.5 12.3-.8 17.2l43.5 46.4c4.5 4.9 12.1 5.1 17 .6z"],"code-branch":[384,512,[],"f126","M384 144c0-44.2-35.8-80-80-80s-80 35.8-80 80c0 36.4 24.3 67.1 57.5 76.8-.6 16.1-4.2 28.5-11 36.9-15.4 19.2-49.3 22.4-85.2 25.7-28.2 2.6-57.4 5.4-81.3 16.9v-144c32.5-10.2 56-40.5 56-76.3 0-44.2-35.8-80-80-80S0 35.8 0 80c0 35.8 23.5 66.1 56 76.3v199.3C23.5 365.9 0 396.2 0 432c0 44.2 35.8 80 80 80s80-35.8 80-80c0-34-21.2-63.1-51.2-74.6 3.1-5.2 7.8-9.8 14.9-13.4 16.2-8.2 40.4-10.4 66.1-12.8 42.2-3.9 90-8.4 118.2-43.4 14-17.4 21.1-39.8 21.6-67.9 31.6-10.8 54.4-40.7 54.4-75.9zM80 64c8.8 0 16 7.2 16 16s-7.2 16-16 16-16-7.2-16-16 7.2-16 16-16zm0 384c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zm224-320c8.8 0 16 7.2 16 16s-7.2 16-16 16-16-7.2-16-16 7.2-16 16-16z"],coffee:[640,512,[],"f0f4","M192 384h192c53 0 96-43 96-96h32c70.6 0 128-57.4 128-128S582.6 32 512 32H120c-13.3 0-24 10.7-24 24v232c0 53 43 96 96 96zM512 96c35.3 0 64 28.7 64 64s-28.7 64-64 64h-32V96h32zm47.7 384H48.3c-47.6 0-61-64-36-64h583.3c25 0 11.8 64-35.9 64z"],cog:[512,512,[],"f013","M487.4 315.7l-42.6-24.6c4.3-23.2 4.3-47 0-70.2l42.6-24.6c4.9-2.8 7.1-8.6 5.5-14-11.1-35.6-30-67.8-54.7-94.6-3.8-4.1-10-5.1-14.8-2.3L380.8 110c-17.9-15.4-38.5-27.3-60.8-35.1V25.8c0-5.6-3.9-10.5-9.4-11.7-36.7-8.2-74.3-7.8-109.2 0-5.5 1.2-9.4 6.1-9.4 11.7V75c-22.2 7.9-42.8 19.8-60.8 35.1L88.7 85.5c-4.9-2.8-11-1.9-14.8 2.3-24.7 26.7-43.6 58.9-54.7 94.6-1.7 5.4.6 11.2 5.5 14L67.3 221c-4.3 23.2-4.3 47 0 70.2l-42.6 24.6c-4.9 2.8-7.1 8.6-5.5 14 11.1 35.6 30 67.8 54.7 94.6 3.8 4.1 10 5.1 14.8 2.3l42.6-24.6c17.9 15.4 38.5 27.3 60.8 35.1v49.2c0 5.6 3.9 10.5 9.4 11.7 36.7 8.2 74.3 7.8 109.2 0 5.5-1.2 9.4-6.1 9.4-11.7v-49.2c22.2-7.9 42.8-19.8 60.8-35.1l42.6 24.6c4.9 2.8 11 1.9 14.8-2.3 24.7-26.7 43.6-58.9 54.7-94.6 1.5-5.5-.7-11.3-5.6-14.1zM256 336c-44.1 0-80-35.9-80-80s35.9-80 80-80 80 35.9 80 80-35.9 80-80 80z"],cogs:[640,512,[],"f085","M512.1 191l-8.2 14.3c-3 5.3-9.4 7.5-15.1 5.4-11.8-4.4-22.6-10.7-32.1-18.6-4.6-3.8-5.8-10.5-2.8-15.7l8.2-14.3c-6.9-8-12.3-17.3-15.9-27.4h-16.5c-6 0-11.2-4.3-12.2-10.3-2-12-2.1-24.6 0-37.1 1-6 6.2-10.4 12.2-10.4h16.5c3.6-10.1 9-19.4 15.9-27.4l-8.2-14.3c-3-5.2-1.9-11.9 2.8-15.7 9.5-7.9 20.4-14.2 32.1-18.6 5.7-2.1 12.1.1 15.1 5.4l8.2 14.3c10.5-1.9 21.2-1.9 31.7 0L552 6.3c3-5.3 9.4-7.5 15.1-5.4 11.8 4.4 22.6 10.7 32.1 18.6 4.6 3.8 5.8 10.5 2.8 15.7l-8.2 14.3c6.9 8 12.3 17.3 15.9 27.4h16.5c6 0 11.2 4.3 12.2 10.3 2 12 2.1 24.6 0 37.1-1 6-6.2 10.4-12.2 10.4h-16.5c-3.6 10.1-9 19.4-15.9 27.4l8.2 14.3c3 5.2 1.9 11.9-2.8 15.7-9.5 7.9-20.4 14.2-32.1 18.6-5.7 2.1-12.1-.1-15.1-5.4l-8.2-14.3c-10.4 1.9-21.2 1.9-31.7 0zm-10.5-58.8c38.5 29.6 82.4-14.3 52.8-52.8-38.5-29.7-82.4 14.3-52.8 52.8zM386.3 286.1l33.7 16.8c10.1 5.8 14.5 18.1 10.5 29.1-8.9 24.2-26.4 46.4-42.6 65.8-7.4 8.9-20.2 11.1-30.3 5.3l-29.1-16.8c-16 13.7-34.6 24.6-54.9 31.7v33.6c0 11.6-8.3 21.6-19.7 23.6-24.6 4.2-50.4 4.4-75.9 0-11.5-2-20-11.9-20-23.6V418c-20.3-7.2-38.9-18-54.9-31.7L74 403c-10 5.8-22.9 3.6-30.3-5.3-16.2-19.4-33.3-41.6-42.2-65.7-4-10.9.4-23.2 10.5-29.1l33.3-16.8c-3.9-20.9-3.9-42.4 0-63.4L12 205.8c-10.1-5.8-14.6-18.1-10.5-29 8.9-24.2 26-46.4 42.2-65.8 7.4-8.9 20.2-11.1 30.3-5.3l29.1 16.8c16-13.7 34.6-24.6 54.9-31.7V57.1c0-11.5 8.2-21.5 19.6-23.5 24.6-4.2 50.5-4.4 76-.1 11.5 2 20 11.9 20 23.6v33.6c20.3 7.2 38.9 18 54.9 31.7l29.1-16.8c10-5.8 22.9-3.6 30.3 5.3 16.2 19.4 33.2 41.6 42.1 65.8 4 10.9.1 23.2-10 29.1l-33.7 16.8c3.9 21 3.9 42.5 0 63.5zm-117.6 21.1c59.2-77-28.7-164.9-105.7-105.7-59.2 77 28.7 164.9 105.7 105.7zm243.4 182.7l-8.2 14.3c-3 5.3-9.4 7.5-15.1 5.4-11.8-4.4-22.6-10.7-32.1-18.6-4.6-3.8-5.8-10.5-2.8-15.7l8.2-14.3c-6.9-8-12.3-17.3-15.9-27.4h-16.5c-6 0-11.2-4.3-12.2-10.3-2-12-2.1-24.6 0-37.1 1-6 6.2-10.4 12.2-10.4h16.5c3.6-10.1 9-19.4 15.9-27.4l-8.2-14.3c-3-5.2-1.9-11.9 2.8-15.7 9.5-7.9 20.4-14.2 32.1-18.6 5.7-2.1 12.1.1 15.1 5.4l8.2 14.3c10.5-1.9 21.2-1.9 31.7 0l8.2-14.3c3-5.3 9.4-7.5 15.1-5.4 11.8 4.4 22.6 10.7 32.1 18.6 4.6 3.8 5.8 10.5 2.8 15.7l-8.2 14.3c6.9 8 12.3 17.3 15.9 27.4h16.5c6 0 11.2 4.3 12.2 10.3 2 12 2.1 24.6 0 37.1-1 6-6.2 10.4-12.2 10.4h-16.5c-3.6 10.1-9 19.4-15.9 27.4l8.2 14.3c3 5.2 1.9 11.9-2.8 15.7-9.5 7.9-20.4 14.2-32.1 18.6-5.7 2.1-12.1-.1-15.1-5.4l-8.2-14.3c-10.4 1.9-21.2 1.9-31.7 0zM501.6 431c38.5 29.6 82.4-14.3 52.8-52.8-38.5-29.6-82.4 14.3-52.8 52.8z"],coins:[512,512,[],"f51e","M0 405.3V448c0 35.3 86 64 192 64s192-28.7 192-64v-42.7C342.7 434.4 267.2 448 192 448S41.3 434.4 0 405.3zM320 128c106 0 192-28.7 192-64S426 0 320 0 128 28.7 128 64s86 64 192 64zM0 300.4V352c0 35.3 86 64 192 64s192-28.7 192-64v-51.6c-41.3 34-116.9 51.6-192 51.6S41.3 334.4 0 300.4zm416 11c57.3-11.1 96-31.7 96-55.4v-42.7c-23.2 16.4-57.3 27.6-96 34.5v63.6zM192 160C86 160 0 195.8 0 240s86 80 192 80 192-35.8 192-80-86-80-192-80zm219.3 56.3c60-10.8 100.7-32 100.7-56.3v-42.7c-35.5 25.1-96.5 38.6-160.7 41.8 29.5 14.3 51.2 33.5 60 57.2z"],columns:[512,512,[],"f0db","M464 32H48C21.49 32 0 53.49 0 80v352c0 26.51 21.49 48 48 48h416c26.51 0 48-21.49 48-48V80c0-26.51-21.49-48-48-48zM224 416H64V160h160v256zm224 0H288V160h160v256z"],comment:[512,512,[],"f075","M256 32C114.6 32 0 125.1 0 240c0 49.6 21.4 95 57 130.7C44.5 421.1 2.7 466 2.2 466.5c-2.2 2.3-2.8 5.7-1.5 8.7S4.8 480 8 480c66.3 0 116-31.8 140.6-51.4 32.7 12.3 69 19.4 107.4 19.4 141.4 0 256-93.1 256-208S397.4 32 256 32z"],"comment-alt":[512,512,[],"f27a","M448 0H64C28.7 0 0 28.7 0 64v288c0 35.3 28.7 64 64 64h96v84c0 9.8 11.2 15.5 19.1 9.7L304 416h144c35.3 0 64-28.7 64-64V64c0-35.3-28.7-64-64-64z"],"comment-dollar":[512,512,[],"f651","M256 32C114.62 32 0 125.12 0 240c0 49.56 21.41 95.01 57.02 130.74C44.46 421.05 2.7 465.97 2.2 466.5A7.995 7.995 0 0 0 8 480c66.26 0 115.99-31.75 140.6-51.38C181.29 440.93 217.59 448 256 448c141.38 0 256-93.12 256-208S397.38 32 256 32zm24 302.44V352c0 8.84-7.16 16-16 16h-16c-8.84 0-16-7.16-16-16v-17.73c-11.42-1.35-22.28-5.19-31.78-11.46-6.22-4.11-6.82-13.11-1.55-18.38l17.52-17.52c3.74-3.74 9.31-4.24 14.11-2.03 3.18 1.46 6.66 2.22 10.26 2.22h32.78c4.66 0 8.44-3.78 8.44-8.42 0-3.75-2.52-7.08-6.12-8.11l-50.07-14.3c-22.25-6.35-40.01-24.71-42.91-47.67-4.05-32.07 19.03-59.43 49.32-63.05V128c0-8.84 7.16-16 16-16h16c8.84 0 16 7.16 16 16v17.73c11.42 1.35 22.28 5.19 31.78 11.46 6.22 4.11 6.82 13.11 1.55 18.38l-17.52 17.52c-3.74 3.74-9.31 4.24-14.11 2.03a24.516 24.516 0 0 0-10.26-2.22h-32.78c-4.66 0-8.44 3.78-8.44 8.42 0 3.75 2.52 7.08 6.12 8.11l50.07 14.3c22.25 6.36 40.01 24.71 42.91 47.67 4.05 32.06-19.03 59.42-49.32 63.04z"],"comment-dots":[512,512,[],"f4ad","M256 32C114.6 32 0 125.1 0 240c0 49.6 21.4 95 57 130.7C44.5 421.1 2.7 466 2.2 466.5c-2.2 2.3-2.8 5.7-1.5 8.7S4.8 480 8 480c66.3 0 116-31.8 140.6-51.4 32.7 12.3 69 19.4 107.4 19.4 141.4 0 256-93.1 256-208S397.4 32 256 32zM128 272c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm128 0c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm128 0c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],"comment-medical":[512,512,[],"f7f5","M256 32C114.62 32 0 125.12 0 240c0 49.56 21.41 95 57 130.74C44.46 421.05 2.7 466 2.2 466.5A8 8 0 0 0 8 480c66.26 0 116-31.75 140.6-51.38A304.66 304.66 0 0 0 256 448c141.39 0 256-93.12 256-208S397.39 32 256 32zm96 232a8 8 0 0 1-8 8h-56v56a8 8 0 0 1-8 8h-48a8 8 0 0 1-8-8v-56h-56a8 8 0 0 1-8-8v-48a8 8 0 0 1 8-8h56v-56a8 8 0 0 1 8-8h48a8 8 0 0 1 8 8v56h56a8 8 0 0 1 8 8z"],"comment-slash":[640,512,[],"f4b3","M64 240c0 49.6 21.4 95 57 130.7-12.6 50.3-54.3 95.2-54.8 95.8-2.2 2.3-2.8 5.7-1.5 8.7 1.3 2.9 4.1 4.8 7.3 4.8 66.3 0 116-31.8 140.6-51.4 32.7 12.3 69 19.4 107.4 19.4 27.4 0 53.7-3.6 78.4-10L72.9 186.4c-5.6 17.1-8.9 35-8.9 53.6zm569.8 218.1l-114.4-88.4C554.6 334.1 576 289.2 576 240c0-114.9-114.6-208-256-208-65.1 0-124.2 20.1-169.4 52.7L45.5 3.4C38.5-2 28.5-.8 23 6.2L3.4 31.4c-5.4 7-4.2 17 2.8 22.4l588.4 454.7c7 5.4 17 4.2 22.5-2.8l19.6-25.3c5.4-6.8 4.1-16.9-2.9-22.3z"],comments:[576,512,[],"f086","M416 192c0-88.4-93.1-160-208-160S0 103.6 0 192c0 34.3 14.1 65.9 38 92-13.4 30.2-35.5 54.2-35.8 54.5-2.2 2.3-2.8 5.7-1.5 8.7S4.8 352 8 352c36.6 0 66.9-12.3 88.7-25 32.2 15.7 70.3 25 111.3 25 114.9 0 208-71.6 208-160zm122 220c23.9-26 38-57.7 38-92 0-66.9-53.5-124.2-129.3-148.1.9 6.6 1.3 13.3 1.3 20.1 0 105.9-107.7 192-240 192-10.8 0-21.3-.8-31.7-1.9C207.8 439.6 281.8 480 368 480c41 0 79.1-9.2 111.3-25 21.8 12.7 52.1 25 88.7 25 3.2 0 6.1-1.9 7.3-4.8 1.3-2.9.7-6.3-1.5-8.7-.3-.3-22.4-24.2-35.8-54.5z"],"comments-dollar":[576,512,[],"f653","M416 192c0-88.37-93.12-160-208-160S0 103.63 0 192c0 34.27 14.13 65.95 37.97 91.98C24.61 314.22 2.52 338.16 2.2 338.5A7.995 7.995 0 0 0 8 352c36.58 0 66.93-12.25 88.73-24.98C128.93 342.76 167.02 352 208 352c114.88 0 208-71.63 208-160zm-224 96v-16.29c-11.29-.58-22.27-4.52-31.37-11.35-3.9-2.93-4.1-8.77-.57-12.14l11.75-11.21c2.77-2.64 6.89-2.76 10.13-.73 3.87 2.42 8.26 3.72 12.82 3.72h28.11c6.5 0 11.8-5.92 11.8-13.19 0-5.95-3.61-11.19-8.77-12.73l-45-13.5c-18.59-5.58-31.58-23.42-31.58-43.39 0-24.52 19.05-44.44 42.67-45.07V96c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v16.29c11.29.58 22.27 4.51 31.37 11.35 3.9 2.93 4.1 8.77.57 12.14l-11.75 11.21c-2.77 2.64-6.89 2.76-10.13.73-3.87-2.43-8.26-3.72-12.82-3.72h-28.11c-6.5 0-11.8 5.92-11.8 13.19 0 5.95 3.61 11.19 8.77 12.73l45 13.5c18.59 5.58 31.58 23.42 31.58 43.39 0 24.53-19.05 44.44-42.67 45.07V288c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8zm346.01 123.99C561.87 385.96 576 354.27 576 320c0-66.94-53.49-124.2-129.33-148.07.86 6.6 1.33 13.29 1.33 20.07 0 105.87-107.66 192-240 192-10.78 0-21.32-.77-31.73-1.88C207.8 439.63 281.77 480 368 480c40.98 0 79.07-9.24 111.27-24.98C501.07 467.75 531.42 480 568 480c3.2 0 6.09-1.91 7.34-4.84 1.27-2.94.66-6.34-1.55-8.67-.31-.33-22.42-24.24-35.78-54.5z"],"compact-disc":[496,512,[],"f51f","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zM88 256H56c0-105.9 86.1-192 192-192v32c-88.2 0-160 71.8-160 160zm160 96c-53 0-96-43-96-96s43-96 96-96 96 43 96 96-43 96-96 96zm0-128c-17.7 0-32 14.3-32 32s14.3 32 32 32 32-14.3 32-32-14.3-32-32-32z"],compass:[496,512,[],"f14e","M225.38 233.37c-12.5 12.5-12.5 32.76 0 45.25 12.49 12.5 32.76 12.5 45.25 0 12.5-12.5 12.5-32.76 0-45.25-12.5-12.49-32.76-12.49-45.25 0zM248 8C111.03 8 0 119.03 0 256s111.03 248 248 248 248-111.03 248-248S384.97 8 248 8zm126.14 148.05L308.17 300.4a31.938 31.938 0 0 1-15.77 15.77l-144.34 65.97c-16.65 7.61-33.81-9.55-26.2-26.2l65.98-144.35a31.938 31.938 0 0 1 15.77-15.77l144.34-65.97c16.65-7.6 33.8 9.55 26.19 26.2z"],compress:[448,512,[],"f066","M436 192H312c-13.3 0-24-10.7-24-24V44c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v84h84c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12zm-276-24V44c0-6.6-5.4-12-12-12h-40c-6.6 0-12 5.4-12 12v84H12c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h124c13.3 0 24-10.7 24-24zm0 300V344c0-13.3-10.7-24-24-24H12c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h84v84c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12zm192 0v-84h84c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12H312c-13.3 0-24 10.7-24 24v124c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12z"],"compress-alt":[448,512,[],"f422","M4.686 427.314L104 328l-32.922-31.029C55.958 281.851 66.666 256 88.048 256h112C213.303 256 224 266.745 224 280v112c0 21.382-25.803 32.09-40.922 16.971L152 376l-99.314 99.314c-6.248 6.248-16.379 6.248-22.627 0L4.686 449.941c-6.248-6.248-6.248-16.379 0-22.627zM443.314 84.686L344 184l32.922 31.029c15.12 15.12 4.412 40.971-16.97 40.971h-112C234.697 256 224 245.255 224 232V120c0-21.382 25.803-32.09 40.922-16.971L296 136l99.314-99.314c6.248-6.248 16.379-6.248 22.627 0l25.373 25.373c6.248 6.248 6.248 16.379 0 22.627z"],"compress-arrows-alt":[512,512,[],"f78c","M200 288H88c-21.4 0-32.1 25.8-17 41l32.9 31-99.2 99.3c-6.2 6.2-6.2 16.4 0 22.6l25.4 25.4c6.2 6.2 16.4 6.2 22.6 0L152 408l31.1 33c15.1 15.1 40.9 4.4 40.9-17V312c0-13.3-10.7-24-24-24zm112-64h112c21.4 0 32.1-25.9 17-41l-33-31 99.3-99.3c6.2-6.2 6.2-16.4 0-22.6L481.9 4.7c-6.2-6.2-16.4-6.2-22.6 0L360 104l-31.1-33C313.8 55.9 288 66.6 288 88v112c0 13.3 10.7 24 24 24zm96 136l33-31.1c15.1-15.1 4.4-40.9-17-40.9H312c-13.3 0-24 10.7-24 24v112c0 21.4 25.9 32.1 41 17l31-32.9 99.3 99.3c6.2 6.2 16.4 6.2 22.6 0l25.4-25.4c6.2-6.2 6.2-16.4 0-22.6L408 360zM183 71.1L152 104 52.7 4.7c-6.2-6.2-16.4-6.2-22.6 0L4.7 30.1c-6.2 6.2-6.2 16.4 0 22.6L104 152l-33 31.1C55.9 198.2 66.6 224 88 224h112c13.3 0 24-10.7 24-24V88c0-21.3-25.9-32-41-16.9z"],"concierge-bell":[512,512,[],"f562","M288 130.54V112h16c8.84 0 16-7.16 16-16V80c0-8.84-7.16-16-16-16h-96c-8.84 0-16 7.16-16 16v16c0 8.84 7.16 16 16 16h16v18.54C115.49 146.11 32 239.18 32 352h448c0-112.82-83.49-205.89-192-221.46zM496 384H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h480c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16z"],cookie:[512,512,[],"f563","M510.37 254.79l-12.08-76.26a132.493 132.493 0 0 0-37.16-72.95l-54.76-54.75c-19.73-19.72-45.18-32.7-72.71-37.05l-76.7-12.15c-27.51-4.36-55.69.11-80.52 12.76L107.32 49.6a132.25 132.25 0 0 0-57.79 57.8l-35.1 68.88a132.602 132.602 0 0 0-12.82 80.94l12.08 76.27a132.493 132.493 0 0 0 37.16 72.95l54.76 54.75a132.087 132.087 0 0 0 72.71 37.05l76.7 12.14c27.51 4.36 55.69-.11 80.52-12.75l69.12-35.21a132.302 132.302 0 0 0 57.79-57.8l35.1-68.87c12.71-24.96 17.2-53.3 12.82-80.96zM176 368c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm32-160c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm160 128c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],"cookie-bite":[512,512,[],"f564","M510.52 255.82c-69.97-.85-126.47-57.69-126.47-127.86-70.17 0-127-56.49-127.86-126.45-27.26-4.14-55.13.3-79.72 12.82l-69.13 35.22a132.221 132.221 0 0 0-57.79 57.81l-35.1 68.88a132.645 132.645 0 0 0-12.82 80.95l12.08 76.27a132.521 132.521 0 0 0 37.16 72.96l54.77 54.76a132.036 132.036 0 0 0 72.71 37.06l76.71 12.15c27.51 4.36 55.7-.11 80.53-12.76l69.13-35.21a132.273 132.273 0 0 0 57.79-57.81l35.1-68.88c12.56-24.64 17.01-52.58 12.91-79.91zM176 368c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm32-160c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm160 128c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],copy:[448,512,[],"f0c5","M320 448v40c0 13.255-10.745 24-24 24H24c-13.255 0-24-10.745-24-24V120c0-13.255 10.745-24 24-24h72v296c0 30.879 25.121 56 56 56h168zm0-344V0H152c-13.255 0-24 10.745-24 24v368c0 13.255 10.745 24 24 24h272c13.255 0 24-10.745 24-24V128H344c-13.2 0-24-10.8-24-24zm120.971-31.029L375.029 7.029A24 24 0 0 0 358.059 0H352v96h96v-6.059a24 24 0 0 0-7.029-16.97z"],copyright:[512,512,[],"f1f9","M256 8C119.033 8 8 119.033 8 256s111.033 248 248 248 248-111.033 248-248S392.967 8 256 8zm117.134 346.753c-1.592 1.867-39.776 45.731-109.851 45.731-84.692 0-144.484-63.26-144.484-145.567 0-81.303 62.004-143.401 143.762-143.401 66.957 0 101.965 37.315 103.422 38.904a12 12 0 0 1 1.238 14.623l-22.38 34.655c-4.049 6.267-12.774 7.351-18.234 2.295-.233-.214-26.529-23.88-61.88-23.88-46.116 0-73.916 33.575-73.916 76.082 0 39.602 25.514 79.692 74.277 79.692 38.697 0 65.28-28.338 65.544-28.625 5.132-5.565 14.059-5.033 18.508 1.053l24.547 33.572a12.001 12.001 0 0 1-.553 14.866z"],couch:[640,512,[],"f4b8","M160 224v64h320v-64c0-35.3 28.7-64 64-64h32c0-53-43-96-96-96H160c-53 0-96 43-96 96h32c35.3 0 64 28.7 64 64zm416-32h-32c-17.7 0-32 14.3-32 32v96H128v-96c0-17.7-14.3-32-32-32H64c-35.3 0-64 28.7-64 64 0 23.6 13 44 32 55.1V432c0 8.8 7.2 16 16 16h64c8.8 0 16-7.2 16-16v-16h384v16c0 8.8 7.2 16 16 16h64c8.8 0 16-7.2 16-16V311.1c19-11.1 32-31.5 32-55.1 0-35.3-28.7-64-64-64z"],"credit-card":[576,512,[],"f09d","M0 432c0 26.5 21.5 48 48 48h480c26.5 0 48-21.5 48-48V256H0v176zm192-68c0-6.6 5.4-12 12-12h136c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12H204c-6.6 0-12-5.4-12-12v-40zm-128 0c0-6.6 5.4-12 12-12h72c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12H76c-6.6 0-12-5.4-12-12v-40zM576 80v48H0V80c0-26.5 21.5-48 48-48h480c26.5 0 48 21.5 48 48z"],crop:[512,512,[],"f125","M488 352h-40V109.25l59.31-59.31c6.25-6.25 6.25-16.38 0-22.63L484.69 4.69c-6.25-6.25-16.38-6.25-22.63 0L402.75 64H192v96h114.75L160 306.75V24c0-13.26-10.75-24-24-24H88C74.75 0 64 10.74 64 24v40H24C10.75 64 0 74.74 0 88v48c0 13.25 10.75 24 24 24h40v264c0 13.25 10.75 24 24 24h232v-96H205.25L352 205.25V488c0 13.25 10.75 24 24 24h48c13.25 0 24-10.75 24-24v-40h40c13.25 0 24-10.75 24-24v-48c0-13.26-10.75-24-24-24z"],"crop-alt":[512,512,[],"f565","M488 352h-40V96c0-17.67-14.33-32-32-32H192v96h160v328c0 13.25 10.75 24 24 24h48c13.25 0 24-10.75 24-24v-40h40c13.25 0 24-10.75 24-24v-48c0-13.26-10.75-24-24-24zM160 24c0-13.26-10.75-24-24-24H88C74.75 0 64 10.74 64 24v40H24C10.75 64 0 74.74 0 88v48c0 13.25 10.75 24 24 24h40v256c0 17.67 14.33 32 32 32h224v-96H160V24z"],cross:[384,512,[],"f654","M352 128h-96V32c0-17.67-14.33-32-32-32h-64c-17.67 0-32 14.33-32 32v96H32c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h96v224c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32V256h96c17.67 0 32-14.33 32-32v-64c0-17.67-14.33-32-32-32z"],crosshairs:[512,512,[],"f05b","M500 224h-30.364C455.724 130.325 381.675 56.276 288 42.364V12c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v30.364C130.325 56.276 56.276 130.325 42.364 224H12c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h30.364C56.276 381.675 130.325 455.724 224 469.636V500c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12v-30.364C381.675 455.724 455.724 381.675 469.636 288H500c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12zM288 404.634V364c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40.634C165.826 392.232 119.783 346.243 107.366 288H148c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12h-40.634C119.768 165.826 165.757 119.783 224 107.366V148c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12v-40.634C346.174 119.768 392.217 165.757 404.634 224H364c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40.634C392.232 346.174 346.243 392.217 288 404.634zM288 256c0 17.673-14.327 32-32 32s-32-14.327-32-32c0-17.673 14.327-32 32-32s32 14.327 32 32z"],crow:[640,512,[],"f520","M544 32h-16.36C513.04 12.68 490.09 0 464 0c-44.18 0-80 35.82-80 80v20.98L12.09 393.57A30.216 30.216 0 0 0 0 417.74c0 22.46 23.64 37.07 43.73 27.03L165.27 384h96.49l44.41 120.1c2.27 6.23 9.15 9.44 15.38 7.17l22.55-8.21c6.23-2.27 9.44-9.15 7.17-15.38L312.94 384H352c1.91 0 3.76-.23 5.66-.29l44.51 120.38c2.27 6.23 9.15 9.44 15.38 7.17l22.55-8.21c6.23-2.27 9.44-9.15 7.17-15.38l-41.24-111.53C485.74 352.8 544 279.26 544 192v-80l96-16c0-35.35-42.98-64-96-64zm-80 72c-13.25 0-24-10.75-24-24 0-13.26 10.75-24 24-24s24 10.74 24 24c0 13.25-10.75 24-24 24z"],crown:[640,512,[],"f521","M528 448H112c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h416c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zm64-320c-26.5 0-48 21.5-48 48 0 7.1 1.6 13.7 4.4 19.8L476 239.2c-15.4 9.2-35.3 4-44.2-11.6L350.3 85C361 76.2 368 63 368 48c0-26.5-21.5-48-48-48s-48 21.5-48 48c0 15 7 28.2 17.7 37l-81.5 142.6c-8.9 15.6-28.9 20.8-44.2 11.6l-72.3-43.4c2.7-6 4.4-12.7 4.4-19.8 0-26.5-21.5-48-48-48S0 149.5 0 176s21.5 48 48 48c2.6 0 5.2-.4 7.7-.8L128 416h384l72.3-192.8c2.5.4 5.1.8 7.7.8 26.5 0 48-21.5 48-48s-21.5-48-48-48z"],crutch:[512,512,[],"f7f7","M507.31 185.71l-181-181a16 16 0 0 0-22.62 0L281 27.31a16 16 0 0 0 0 22.63l181 181a16 16 0 0 0 22.63 0l22.62-22.63a16 16 0 0 0 .06-22.6zm-179.54 66.41l-67.89-67.89 55.1-55.1-45.25-45.25-109.67 109.67a96.08 96.08 0 0 0-25.67 46.29L106.65 360.1l-102 102a16 16 0 0 0 0 22.63l22.62 22.62a16 16 0 0 0 22.63 0l102-102 120.25-27.75a95.88 95.88 0 0 0 46.29-25.65l109.68-109.68L382.87 197zm-54.57 54.57a32 32 0 0 1-15.45 8.54l-79.3 18.32 18.3-79.3a32.22 32.22 0 0 1 8.56-15.45l9.31-9.31 67.89 67.89z"],cube:[512,512,[],"f1b2","M239.1 6.3l-208 78c-18.7 7-31.1 25-31.1 45v225.1c0 18.2 10.3 34.8 26.5 42.9l208 104c13.5 6.8 29.4 6.8 42.9 0l208-104c16.3-8.1 26.5-24.8 26.5-42.9V129.3c0-20-12.4-37.9-31.1-44.9l-208-78C262 2.2 250 2.2 239.1 6.3zM256 68.4l192 72v1.1l-192 78-192-78v-1.1l192-72zm32 356V275.5l160-65v133.9l-160 80z"],cubes:[512,512,[],"f1b3","M488.6 250.2L392 214V105.5c0-15-9.3-28.4-23.4-33.7l-100-37.5c-8.1-3.1-17.1-3.1-25.3 0l-100 37.5c-14.1 5.3-23.4 18.7-23.4 33.7V214l-96.6 36.2C9.3 255.5 0 268.9 0 283.9V394c0 13.6 7.7 26.1 19.9 32.2l100 50c10.1 5.1 22.1 5.1 32.2 0l103.9-52 103.9 52c10.1 5.1 22.1 5.1 32.2 0l100-50c12.2-6.1 19.9-18.6 19.9-32.2V283.9c0-15-9.3-28.4-23.4-33.7zM358 214.8l-85 31.9v-68.2l85-37v73.3zM154 104.1l102-38.2 102 38.2v.6l-102 41.4-102-41.4v-.6zm84 291.1l-85 42.5v-79.1l85-38.8v75.4zm0-112l-102 41.4-102-41.4v-.6l102-38.2 102 38.2v.6zm240 112l-85 42.5v-79.1l85-38.8v75.4zm0-112l-102 41.4-102-41.4v-.6l102-38.2 102 38.2v.6z"],cut:[448,512,[],"f0c4","M278.06 256L444.48 89.57c4.69-4.69 4.69-12.29 0-16.97-32.8-32.8-85.99-32.8-118.79 0L210.18 188.12l-24.86-24.86c4.31-10.92 6.68-22.81 6.68-35.26 0-53.02-42.98-96-96-96S0 74.98 0 128s42.98 96 96 96c4.54 0 8.99-.32 13.36-.93L142.29 256l-32.93 32.93c-4.37-.61-8.83-.93-13.36-.93-53.02 0-96 42.98-96 96s42.98 96 96 96 96-42.98 96-96c0-12.45-2.37-24.34-6.68-35.26l24.86-24.86L325.69 439.4c32.8 32.8 85.99 32.8 118.79 0 4.69-4.68 4.69-12.28 0-16.97L278.06 256zM96 160c-17.64 0-32-14.36-32-32s14.36-32 32-32 32 14.36 32 32-14.36 32-32 32zm0 256c-17.64 0-32-14.36-32-32s14.36-32 32-32 32 14.36 32 32-14.36 32-32 32z"],database:[448,512,[],"f1c0","M448 73.143v45.714C448 159.143 347.667 192 224 192S0 159.143 0 118.857V73.143C0 32.857 100.333 0 224 0s224 32.857 224 73.143zM448 176v102.857C448 319.143 347.667 352 224 352S0 319.143 0 278.857V176c48.125 33.143 136.208 48.572 224 48.572S399.874 209.143 448 176zm0 160v102.857C448 479.143 347.667 512 224 512S0 479.143 0 438.857V336c48.125 33.143 136.208 48.572 224 48.572S399.874 369.143 448 336z"],deaf:[512,512,[],"f2a4","M216 260c0 15.464-12.536 28-28 28s-28-12.536-28-28c0-44.112 35.888-80 80-80s80 35.888 80 80c0 15.464-12.536 28-28 28s-28-12.536-28-28c0-13.234-10.767-24-24-24s-24 10.766-24 24zm24-176c-97.047 0-176 78.953-176 176 0 15.464 12.536 28 28 28s28-12.536 28-28c0-66.168 53.832-120 120-120s120 53.832 120 120c0 75.164-71.009 70.311-71.997 143.622L288 404c0 28.673-23.327 52-52 52-15.464 0-28 12.536-28 28s12.536 28 28 28c59.475 0 107.876-48.328 108-107.774.595-34.428 72-48.24 72-144.226 0-97.047-78.953-176-176-176zm268.485-52.201L480.2 3.515c-4.687-4.686-12.284-4.686-16.971 0L376.2 90.544c-4.686 4.686-4.686 12.284 0 16.971l28.285 28.285c4.686 4.686 12.284 4.686 16.97 0l87.03-87.029c4.687-4.688 4.687-12.286 0-16.972zM168.97 314.745c-4.686-4.686-12.284-4.686-16.97 0L3.515 463.23c-4.686 4.686-4.686 12.284 0 16.971L31.8 508.485c4.687 4.686 12.284 4.686 16.971 0L197.256 360c4.686-4.686 4.686-12.284 0-16.971l-28.286-28.284z"],democrat:[640,512,[],"f747","M637.3 256.9l-19.6-29.4c-28.2-42.3-75.3-67.5-126.1-67.5H256l-81.2-81.2c20.1-20.1 22.6-51.1 7.5-73.9-3.4-5.2-10.8-5.9-15.2-1.5l-41.8 41.8L82.4 2.4c-3.6-3.6-9.6-3-12.4 1.2-12.3 18.6-10.3 44 6.1 60.4 3.3 3.3 7.3 5.3 11.3 7.5-2.2 1.7-4.7 3.1-6.4 5.4L6.4 176.2c-7.3 9.7-8.4 22.7-3 33.5l14.3 28.6c5.4 10.8 16.5 17.7 28.6 17.7h31c8.5 0 16.6-3.4 22.6-9.4L138 212l54 108h352v-77.8c16.2 12.2 18.3 17.6 40.1 50.3 4.9 7.4 14.8 9.3 22.2 4.4l26.6-17.7c7.3-5 9.3-14.9 4.4-22.3zm-341.1-13.6l-16.5 16.1 3.9 22.7c.7 4.1-3.6 7.2-7.2 5.3L256 276.7l-20.4 10.7c-3.6 1.9-7.9-1.2-7.2-5.3l3.9-22.7-16.5-16.1c-3-2.9-1.3-7.9 2.8-8.5l22.8-3.3 10.2-20.7c1.8-3.7 7.1-3.7 9 0l10.2 20.7 22.8 3.3c4 .6 5.6 5.6 2.6 8.5zm112 0l-16.5 16.1 3.9 22.7c.7 4.1-3.6 7.2-7.2 5.3L368 276.7l-20.4 10.7c-3.6 1.9-7.9-1.2-7.2-5.3l3.9-22.7-16.5-16.1c-3-2.9-1.3-7.9 2.8-8.5l22.8-3.3 10.2-20.7c1.8-3.7 7.1-3.7 9 0l10.2 20.7 22.8 3.3c4 .6 5.6 5.6 2.6 8.5zm112 0l-16.5 16.1 3.9 22.7c.7 4.1-3.6 7.2-7.2 5.3L480 276.7l-20.4 10.7c-3.6 1.9-7.9-1.2-7.2-5.3l3.9-22.7-16.5-16.1c-3-2.9-1.3-7.9 2.8-8.5l22.8-3.3 10.2-20.7c1.8-3.7 7.1-3.7 9 0l10.2 20.7 22.8 3.3c4 .6 5.6 5.6 2.6 8.5zM192 496c0 8.8 7.2 16 16 16h64c8.8 0 16-7.2 16-16v-80h160v80c0 8.8 7.2 16 16 16h64c8.8 0 16-7.2 16-16V352H192v144z"],desktop:[576,512,[],"f108","M528 0H48C21.5 0 0 21.5 0 48v320c0 26.5 21.5 48 48 48h192l-16 48h-72c-13.3 0-24 10.7-24 24s10.7 24 24 24h272c13.3 0 24-10.7 24-24s-10.7-24-24-24h-72l-16-48h192c26.5 0 48-21.5 48-48V48c0-26.5-21.5-48-48-48zm-16 352H64V64h448v288z"],dharmachakra:[512,512,[],"f655","M495 225.06l-17.22 1.08c-5.27-39.49-20.79-75.64-43.86-105.84l12.95-11.43c6.92-6.11 7.25-16.79.73-23.31L426.44 64.4c-6.53-6.53-17.21-6.19-23.31.73L391.7 78.07c-30.2-23.06-66.35-38.58-105.83-43.86L286.94 17c.58-9.21-6.74-17-15.97-17h-29.94c-9.23 0-16.54 7.79-15.97 17l1.08 17.22c-39.49 5.27-75.64 20.79-105.83 43.86l-11.43-12.95c-6.11-6.92-16.79-7.25-23.31-.73L64.4 85.56c-6.53 6.53-6.19 17.21.73 23.31l12.95 11.43c-23.06 30.2-38.58 66.35-43.86 105.84L17 225.06c-9.21-.58-17 6.74-17 15.97v29.94c0 9.23 7.79 16.54 17 15.97l17.22-1.08c5.27 39.49 20.79 75.64 43.86 105.83l-12.95 11.43c-6.92 6.11-7.25 16.79-.73 23.31l21.17 21.17c6.53 6.53 17.21 6.19 23.31-.73l11.43-12.95c30.2 23.06 66.35 38.58 105.84 43.86L225.06 495c-.58 9.21 6.74 17 15.97 17h29.94c9.23 0 16.54-7.79 15.97-17l-1.08-17.22c39.49-5.27 75.64-20.79 105.84-43.86l11.43 12.95c6.11 6.92 16.79 7.25 23.31.73l21.17-21.17c6.53-6.53 6.19-17.21-.73-23.31l-12.95-11.43c23.06-30.2 38.58-66.35 43.86-105.83l17.22 1.08c9.21.58 17-6.74 17-15.97v-29.94c-.01-9.23-7.8-16.54-17.01-15.97zM281.84 98.61c24.81 4.07 47.63 13.66 67.23 27.78l-42.62 48.29c-8.73-5.44-18.32-9.54-28.62-11.95l4.01-64.12zm-51.68 0l4.01 64.12c-10.29 2.41-19.89 6.52-28.62 11.95l-42.62-48.29c19.6-14.12 42.42-23.71 67.23-27.78zm-103.77 64.33l48.3 42.61c-5.44 8.73-9.54 18.33-11.96 28.62l-64.12-4.01c4.07-24.81 13.66-47.62 27.78-67.22zm-27.78 118.9l64.12-4.01c2.41 10.29 6.52 19.89 11.95 28.62l-48.29 42.62c-14.12-19.6-23.71-42.42-27.78-67.23zm131.55 131.55c-24.81-4.07-47.63-13.66-67.23-27.78l42.61-48.3c8.73 5.44 18.33 9.54 28.62 11.96l-4 64.12zM256 288c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm25.84 125.39l-4.01-64.12c10.29-2.41 19.89-6.52 28.62-11.96l42.61 48.3c-19.6 14.12-42.41 23.71-67.22 27.78zm103.77-64.33l-48.29-42.62c5.44-8.73 9.54-18.32 11.95-28.62l64.12 4.01c-4.07 24.82-13.66 47.64-27.78 67.23zm-36.34-114.89c-2.41-10.29-6.52-19.89-11.96-28.62l48.3-42.61c14.12 19.6 23.71 42.42 27.78 67.23l-64.12 4z"],diagnoses:[640,512,[],"f470","M496 256c8.8 0 16-7.2 16-16s-7.2-16-16-16-16 7.2-16 16 7.2 16 16 16zm-176-80c48.5 0 88-39.5 88-88S368.5 0 320 0s-88 39.5-88 88 39.5 88 88 88zM59.8 364c10.2 15.3 29.3 17.8 42.9 9.8 16.2-9.6 56.2-31.7 105.3-48.6V416h224v-90.7c49.1 16.8 89.1 39 105.3 48.6 13.6 8 32.7 5.3 42.9-9.8l17.8-26.7c8.8-13.2 7.6-34.6-10-45.1-11.9-7.1-29.7-17-51.1-27.4-28.1 46.1-99.4 17.8-87.7-35.1C409.3 217.2 365.1 208 320 208c-57 0-112.9 14.5-160 32.2-.2 40.2-47.6 63.3-79.2 36-11.2 6-21.3 11.6-28.7 16-17.6 10.5-18.8 31.8-10 45.1L59.8 364zM368 344c13.3 0 24 10.7 24 24s-10.7 24-24 24-24-10.7-24-24 10.7-24 24-24zm-96-96c13.3 0 24 10.7 24 24s-10.7 24-24 24-24-10.7-24-24 10.7-24 24-24zm-160 8c8.8 0 16-7.2 16-16s-7.2-16-16-16-16 7.2-16 16 7.2 16 16 16zm512 192H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h608c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16z"],dice:[640,512,[],"f522","M592 192H473.26c12.69 29.59 7.12 65.2-17 89.32L320 417.58V464c0 26.51 21.49 48 48 48h224c26.51 0 48-21.49 48-48V240c0-26.51-21.49-48-48-48zM480 376c-13.25 0-24-10.75-24-24 0-13.26 10.75-24 24-24s24 10.74 24 24c0 13.25-10.75 24-24 24zm-46.37-186.7L258.7 14.37c-19.16-19.16-50.23-19.16-69.39 0L14.37 189.3c-19.16 19.16-19.16 50.23 0 69.39L189.3 433.63c19.16 19.16 50.23 19.16 69.39 0L433.63 258.7c19.16-19.17 19.16-50.24 0-69.4zM96 248c-13.25 0-24-10.75-24-24 0-13.26 10.75-24 24-24s24 10.74 24 24c0 13.25-10.75 24-24 24zm128 128c-13.25 0-24-10.75-24-24 0-13.26 10.75-24 24-24s24 10.74 24 24c0 13.25-10.75 24-24 24zm0-128c-13.25 0-24-10.75-24-24 0-13.26 10.75-24 24-24s24 10.74 24 24c0 13.25-10.75 24-24 24zm0-128c-13.25 0-24-10.75-24-24 0-13.26 10.75-24 24-24s24 10.74 24 24c0 13.25-10.75 24-24 24zm128 128c-13.25 0-24-10.75-24-24 0-13.26 10.75-24 24-24s24 10.74 24 24c0 13.25-10.75 24-24 24z"],"dice-d20":[480,512,[],"f6cf","M106.75 215.06L1.2 370.95c-3.08 5 .1 11.5 5.93 12.14l208.26 22.07-108.64-190.1zM7.41 315.43L82.7 193.08 6.06 147.1c-2.67-1.6-6.06.32-6.06 3.43v162.81c0 4.03 5.29 5.53 7.41 2.09zM18.25 423.6l194.4 87.66c5.3 2.45 11.35-1.43 11.35-7.26v-65.67l-203.55-22.3c-4.45-.5-6.23 5.59-2.2 7.57zm81.22-257.78L179.4 22.88c4.34-7.06-3.59-15.25-10.78-11.14L17.81 110.35c-2.47 1.62-2.39 5.26.13 6.78l81.53 48.69zM240 176h109.21L253.63 7.62C250.5 2.54 245.25 0 240 0s-10.5 2.54-13.63 7.62L130.79 176H240zm233.94-28.9l-76.64 45.99 75.29 122.35c2.11 3.44 7.41 1.94 7.41-2.1V150.53c0-3.11-3.39-5.03-6.06-3.43zm-93.41 18.72l81.53-48.7c2.53-1.52 2.6-5.16.13-6.78l-150.81-98.6c-7.19-4.11-15.12 4.08-10.78 11.14l79.93 142.94zm79.02 250.21L256 438.32v65.67c0 5.84 6.05 9.71 11.35 7.26l194.4-87.66c4.03-1.97 2.25-8.06-2.2-7.56zm-86.3-200.97l-108.63 190.1 208.26-22.07c5.83-.65 9.01-7.14 5.93-12.14L373.25 215.06zM240 208H139.57L240 383.75 340.43 208H240z"],"dice-d6":[448,512,[],"f6d1","M422.19 109.95L256.21 9.07c-19.91-12.1-44.52-12.1-64.43 0L25.81 109.95c-5.32 3.23-5.29 11.27.06 14.46L224 242.55l198.14-118.14c5.35-3.19 5.38-11.22.05-14.46zm13.84 44.63L240 271.46v223.82c0 12.88 13.39 20.91 24.05 14.43l152.16-92.48c19.68-11.96 31.79-33.94 31.79-57.7v-197.7c0-6.41-6.64-10.43-11.97-7.25zM0 161.83v197.7c0 23.77 12.11 45.74 31.79 57.7l152.16 92.47c10.67 6.48 24.05-1.54 24.05-14.43V271.46L11.97 154.58C6.64 151.4 0 155.42 0 161.83z"],"dice-five":[448,512,[],"f523","M384 32H64C28.65 32 0 60.65 0 96v320c0 35.35 28.65 64 64 64h320c35.35 0 64-28.65 64-64V96c0-35.35-28.65-64-64-64zM128 384c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm0-192c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm96 96c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm96 96c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm0-192c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],"dice-four":[448,512,[],"f524","M384 32H64C28.65 32 0 60.65 0 96v320c0 35.35 28.65 64 64 64h320c35.35 0 64-28.65 64-64V96c0-35.35-28.65-64-64-64zM128 384c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm0-192c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm192 192c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm0-192c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],"dice-one":[448,512,[],"f525","M384 32H64C28.65 32 0 60.65 0 96v320c0 35.35 28.65 64 64 64h320c35.35 0 64-28.65 64-64V96c0-35.35-28.65-64-64-64zM224 288c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],"dice-six":[448,512,[],"f526","M384 32H64C28.65 32 0 60.65 0 96v320c0 35.35 28.65 64 64 64h320c35.35 0 64-28.65 64-64V96c0-35.35-28.65-64-64-64zM128 384c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm0-96c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm0-96c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm192 192c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm0-96c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm0-96c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],"dice-three":[448,512,[],"f527","M384 32H64C28.65 32 0 60.65 0 96v320c0 35.35 28.65 64 64 64h320c35.35 0 64-28.65 64-64V96c0-35.35-28.65-64-64-64zM128 192c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm96 96c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm96 96c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],"dice-two":[448,512,[],"f528","M384 32H64C28.65 32 0 60.65 0 96v320c0 35.35 28.65 64 64 64h320c35.35 0 64-28.65 64-64V96c0-35.35-28.65-64-64-64zM128 192c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm192 192c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],"digital-tachograph":[640,512,[],"f566","M608 96H32c-17.67 0-32 14.33-32 32v256c0 17.67 14.33 32 32 32h576c17.67 0 32-14.33 32-32V128c0-17.67-14.33-32-32-32zM304 352c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8v-8c0-4.42 3.58-8 8-8h224c4.42 0 8 3.58 8 8v8zM72 288v-16c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H80c-4.42 0-8-3.58-8-8zm64 0v-16c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8zm64 0v-16c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8zm64 0v-16c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8zm40-64c0 8.84-7.16 16-16 16H80c-8.84 0-16-7.16-16-16v-48c0-8.84 7.16-16 16-16h208c8.84 0 16 7.16 16 16v48zm272 128c0 4.42-3.58 8-8 8H344c-4.42 0-8-3.58-8-8v-8c0-4.42 3.58-8 8-8h224c4.42 0 8 3.58 8 8v8z"],directions:[512,512,[],"f5eb","M502.61 233.32L278.68 9.39c-12.52-12.52-32.83-12.52-45.36 0L9.39 233.32c-12.52 12.53-12.52 32.83 0 45.36l223.93 223.93c12.52 12.53 32.83 12.53 45.36 0l223.93-223.93c12.52-12.53 12.52-32.83 0-45.36zm-100.98 12.56l-84.21 77.73c-5.12 4.73-13.43 1.1-13.43-5.88V264h-96v64c0 4.42-3.58 8-8 8h-32c-4.42 0-8-3.58-8-8v-80c0-17.67 14.33-32 32-32h112v-53.73c0-6.97 8.3-10.61 13.43-5.88l84.21 77.73c3.43 3.17 3.43 8.59 0 11.76z"],disease:[512,512,[],"f7fa","M472.29 195.9l-67.06-23c-19.28-6.6-33.54-20.92-38.14-38.31l-16-60.45c-11.58-43.77-76.57-57.13-110-22.62L195 99.24c-13.26 13.71-33.54 20.93-54.2 19.31l-71.9-5.62c-52-4.07-86.93 44.89-59 82.84l38.54 52.42c11.08 15.07 12.82 33.86 4.64 50.24l-28.43 57C4 396.67 47.46 440.29 98.11 429.23l70-15.28c20.11-4.39 41.45 0 57.07 11.73l54.32 40.83c39.32 29.56 101 7.57 104.45-37.22l4.7-61.86c1.35-17.8 12.8-33.87 30.63-43l62-31.74c44.84-22.96 39.55-80.17-8.99-96.79zM160 256a32 32 0 1 1 32-32 32 32 0 0 1-32 32zm128 96a32 32 0 1 1 32-32 32 32 0 0 1-32 32zm16-128a16 16 0 1 1 16-16 16 16 0 0 1-16 16z"],divide:[448,512,[],"f529","M224 352c-35.35 0-64 28.65-64 64s28.65 64 64 64 64-28.65 64-64-28.65-64-64-64zm0-192c35.35 0 64-28.65 64-64s-28.65-64-64-64-64 28.65-64 64 28.65 64 64 64zm192 48H32c-17.67 0-32 14.33-32 32v32c0 17.67 14.33 32 32 32h384c17.67 0 32-14.33 32-32v-32c0-17.67-14.33-32-32-32z"],dizzy:[496,512,[],"f567","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm-96 206.6l-28.7 28.7c-14.8 14.8-37.8-7.5-22.6-22.6l28.7-28.7-28.7-28.7c-15-15 7.7-37.6 22.6-22.6l28.7 28.7 28.7-28.7c15-15 37.6 7.7 22.6 22.6L174.6 192l28.7 28.7c15.2 15.2-7.9 37.4-22.6 22.6L152 214.6zM248 416c-35.3 0-64-28.7-64-64s28.7-64 64-64 64 28.7 64 64-28.7 64-64 64zm147.3-195.3c15.2 15.2-7.9 37.4-22.6 22.6L344 214.6l-28.7 28.7c-14.8 14.8-37.8-7.5-22.6-22.6l28.7-28.7-28.7-28.7c-15-15 7.7-37.6 22.6-22.6l28.7 28.7 28.7-28.7c15-15 37.6 7.7 22.6 22.6L366.6 192l28.7 28.7z"],dna:[448,512,[],"f471","M.1 494.1c-1.1 9.5 6.3 17.8 15.9 17.8l32.3.1c8.1 0 14.9-5.9 16-13.9.7-4.9 1.8-11.1 3.4-18.1H380c1.6 6.9 2.9 13.2 3.5 18.1 1.1 8 7.9 14 16 13.9l32.3-.1c9.6 0 17.1-8.3 15.9-17.8-4.6-37.9-25.6-129-118.9-207.7-17.6 12.4-37.1 24.2-58.5 35.4 6.2 4.6 11.4 9.4 17 14.2H159.7c21.3-18.1 47-35.6 78.7-51.4C410.5 199.1 442.1 65.8 447.9 17.9 449 8.4 441.6.1 432 .1L399.6 0c-8.1 0-14.9 5.9-16 13.9-.7 4.9-1.8 11.1-3.4 18.1H67.8c-1.6-7-2.7-13.1-3.4-18.1-1.1-8-7.9-14-16-13.9L16.1.1C6.5.1-1 8.4.1 17.9 5.3 60.8 31.4 171.8 160 256 31.5 340.2 5.3 451.2.1 494.1zM224 219.6c-25.1-13.7-46.4-28.4-64.3-43.6h128.5c-17.8 15.2-39.1 30-64.2 43.6zM355.1 96c-5.8 10.4-12.8 21.1-21 32H114c-8.3-10.9-15.3-21.6-21-32h262.1zM92.9 416c5.8-10.4 12.8-21.1 21-32h219.4c8.3 10.9 15.4 21.6 21.2 32H92.9z"],dog:[576,512,[],"f6d3","M298.06,224,448,277.55V496a16,16,0,0,1-16,16H368a16,16,0,0,1-16-16V384H192V496a16,16,0,0,1-16,16H112a16,16,0,0,1-16-16V282.09C58.84,268.84,32,233.66,32,192a32,32,0,0,1,64,0,32.06,32.06,0,0,0,32,32ZM544,112v32a64,64,0,0,1-64,64H448v35.58L320,197.87V48c0-14.25,17.22-21.39,27.31-11.31L374.59,64h53.63c10.91,0,23.75,7.92,28.62,17.69L464,96h64A16,16,0,0,1,544,112Zm-112,0a16,16,0,1,0-16,16A16,16,0,0,0,432,112Z"],"dollar-sign":[288,512,[],"f155","M209.2 233.4l-108-31.6C88.7 198.2 80 186.5 80 173.5c0-16.3 13.2-29.5 29.5-29.5h66.3c12.2 0 24.2 3.7 34.2 10.5 6.1 4.1 14.3 3.1 19.5-2l34.8-34c7.1-6.9 6.1-18.4-1.8-24.5C238 74.8 207.4 64.1 176 64V16c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v48h-2.5C45.8 64-5.4 118.7.5 183.6c4.2 46.1 39.4 83.6 83.8 96.6l102.5 30c12.5 3.7 21.2 15.3 21.2 28.3 0 16.3-13.2 29.5-29.5 29.5h-66.3C100 368 88 364.3 78 357.5c-6.1-4.1-14.3-3.1-19.5 2l-34.8 34c-7.1 6.9-6.1 18.4 1.8 24.5 24.5 19.2 55.1 29.9 86.5 30v48c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16v-48.2c46.6-.9 90.3-28.6 105.7-72.7 21.5-61.6-14.6-124.8-72.5-141.7z"],dolly:[576,512,[],"f472","M294.2 277.7c18 5 34.7 13.4 49.5 24.7l161.5-53.8c8.4-2.8 12.9-11.9 10.1-20.2L454.9 47.2c-2.8-8.4-11.9-12.9-20.2-10.1l-61.1 20.4 33.1 99.4L346 177l-33.1-99.4-61.6 20.5c-8.4 2.8-12.9 11.9-10.1 20.2l53 159.4zm281 48.7L565 296c-2.8-8.4-11.9-12.9-20.2-10.1l-213.5 71.2c-17.2-22-43.6-36.4-73.5-37L158.4 21.9C154 8.8 141.8 0 128 0H16C7.2 0 0 7.2 0 16v32c0 8.8 7.2 16 16 16h88.9l92.2 276.7c-26.1 20.4-41.7 53.6-36 90.5 6.1 39.4 37.9 72.3 77.3 79.2 60.2 10.7 112.3-34.8 113.4-92.6l213.3-71.2c8.3-2.8 12.9-11.8 10.1-20.2zM256 464c-26.5 0-48-21.5-48-48s21.5-48 48-48 48 21.5 48 48-21.5 48-48 48z"],"dolly-flatbed":[640,512,[],"f474","M208 320h384c8.8 0 16-7.2 16-16V48c0-8.8-7.2-16-16-16H448v128l-48-32-48 32V32H208c-8.8 0-16 7.2-16 16v256c0 8.8 7.2 16 16 16zm416 64H128V16c0-8.8-7.2-16-16-16H16C7.2 0 0 7.2 0 16v32c0 8.8 7.2 16 16 16h48v368c0 8.8 7.2 16 16 16h82.9c-1.8 5-2.9 10.4-2.9 16 0 26.5 21.5 48 48 48s48-21.5 48-48c0-5.6-1.2-11-2.9-16H451c-1.8 5-2.9 10.4-2.9 16 0 26.5 21.5 48 48 48s48-21.5 48-48c0-5.6-1.2-11-2.9-16H624c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16z"],donate:[512,512,[],"f4b9","M256 416c114.9 0 208-93.1 208-208S370.9 0 256 0 48 93.1 48 208s93.1 208 208 208zM233.8 97.4V80.6c0-9.2 7.4-16.6 16.6-16.6h11.1c9.2 0 16.6 7.4 16.6 16.6v17c15.5.8 30.5 6.1 43 15.4 5.6 4.1 6.2 12.3 1.2 17.1L306 145.6c-3.8 3.7-9.5 3.8-14 1-5.4-3.4-11.4-5.1-17.8-5.1h-38.9c-9 0-16.3 8.2-16.3 18.3 0 8.2 5 15.5 12.1 17.6l62.3 18.7c25.7 7.7 43.7 32.4 43.7 60.1 0 34-26.4 61.5-59.1 62.4v16.8c0 9.2-7.4 16.6-16.6 16.6h-11.1c-9.2 0-16.6-7.4-16.6-16.6v-17c-15.5-.8-30.5-6.1-43-15.4-5.6-4.1-6.2-12.3-1.2-17.1l16.3-15.5c3.8-3.7 9.5-3.8 14-1 5.4 3.4 11.4 5.1 17.8 5.1h38.9c9 0 16.3-8.2 16.3-18.3 0-8.2-5-15.5-12.1-17.6l-62.3-18.7c-25.7-7.7-43.7-32.4-43.7-60.1.1-34 26.4-61.5 59.1-62.4zM480 352h-32.5c-19.6 26-44.6 47.7-73 64h63.8c5.3 0 9.6 3.6 9.6 8v16c0 4.4-4.3 8-9.6 8H73.6c-5.3 0-9.6-3.6-9.6-8v-16c0-4.4 4.3-8 9.6-8h63.8c-28.4-16.3-53.3-38-73-64H32c-17.7 0-32 14.3-32 32v96c0 17.7 14.3 32 32 32h448c17.7 0 32-14.3 32-32v-96c0-17.7-14.3-32-32-32z"],"door-closed":[640,512,[],"f52a","M624 448H512V50.8C512 22.78 490.47 0 464 0H175.99c-26.47 0-48 22.78-48 50.8V448H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h608c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zM415.99 288c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32c.01 17.67-14.32 32-32 32z"],"door-open":[640,512,[],"f52b","M624 448h-80V113.45C544 86.19 522.47 64 496 64H384v64h96v384h144c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zM312.24 1.01l-192 49.74C105.99 54.44 96 67.7 96 82.92V448H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h336V33.18c0-21.58-19.56-37.41-39.76-32.17zM264 288c-13.25 0-24-14.33-24-32s10.75-32 24-32 24 14.33 24 32-10.75 32-24 32z"],"dot-circle":[512,512,[],"f192","M256 8C119.033 8 8 119.033 8 256s111.033 248 248 248 248-111.033 248-248S392.967 8 256 8zm80 248c0 44.112-35.888 80-80 80s-80-35.888-80-80 35.888-80 80-80 80 35.888 80 80z"],dove:[512,512,[],"f4ba","M288 167.2v-28.1c-28.2-36.3-47.1-79.3-54.1-125.2-2.1-13.5-19-18.8-27.8-8.3-21.1 24.9-37.7 54.1-48.9 86.5 34.2 38.3 80 64.6 130.8 75.1zM400 64c-44.2 0-80 35.9-80 80.1v59.4C215.6 197.3 127 133 87 41.8c-5.5-12.5-23.2-13.2-29-.9C41.4 76 32 115.2 32 156.6c0 70.8 34.1 136.9 85.1 185.9 13.2 12.7 26.1 23.2 38.9 32.8l-143.9 36C1.4 414-3.4 426.4 2.6 435.7 20 462.6 63 508.2 155.8 512c8 .3 16-2.6 22.1-7.9l65.2-56.1H320c88.4 0 160-71.5 160-159.9V128l32-64H400zm0 96.1c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16z"],download:[512,512,[],"f019","M216 0h80c13.3 0 24 10.7 24 24v168h87.7c17.8 0 26.7 21.5 14.1 34.1L269.7 378.3c-7.5 7.5-19.8 7.5-27.3 0L90.1 226.1c-12.6-12.6-3.7-34.1 14.1-34.1H192V24c0-13.3 10.7-24 24-24zm296 376v112c0 13.3-10.7 24-24 24H24c-13.3 0-24-10.7-24-24V376c0-13.3 10.7-24 24-24h146.7l49 49c20.1 20.1 52.5 20.1 72.6 0l49-49H488c13.3 0 24 10.7 24 24zm-124 88c0-11-9-20-20-20s-20 9-20 20 9 20 20 20 20-9 20-20zm64 0c0-11-9-20-20-20s-20 9-20 20 9 20 20 20 20-9 20-20z"],"drafting-compass":[512,512,[],"f568","M457.01 344.42c-25.05 20.33-52.63 37.18-82.54 49.05l54.38 94.19 53.95 23.04c9.81 4.19 20.89-2.21 22.17-12.8l7.02-58.25-54.98-95.23zm42.49-94.56c4.86-7.67 1.89-17.99-6.05-22.39l-28.07-15.57c-7.48-4.15-16.61-1.46-21.26 5.72C403.01 281.15 332.25 320 256 320c-23.93 0-47.23-4.25-69.41-11.53l67.36-116.68c.7.02 1.34.21 2.04.21s1.35-.19 2.04-.21l51.09 88.5c31.23-8.96 59.56-25.75 82.61-48.92l-51.79-89.71C347.39 128.03 352 112.63 352 96c0-53.02-42.98-96-96-96s-96 42.98-96 96c0 16.63 4.61 32.03 12.05 45.66l-68.3 118.31c-12.55-11.61-23.96-24.59-33.68-39-4.79-7.1-13.97-9.62-21.38-5.33l-27.75 16.07c-7.85 4.54-10.63 14.9-5.64 22.47 15.57 23.64 34.69 44.21 55.98 62.02L0 439.66l7.02 58.25c1.28 10.59 12.36 16.99 22.17 12.8l53.95-23.04 70.8-122.63C186.13 377.28 220.62 384 256 384c99.05 0 190.88-51.01 243.5-134.14zM256 64c17.67 0 32 14.33 32 32s-14.33 32-32 32-32-14.33-32-32 14.33-32 32-32z"],dragon:[640,512,[],"f6d5","M18.32 255.78L192 223.96l-91.28 68.69c-10.08 10.08-2.94 27.31 11.31 27.31h222.7c-9.44-26.4-14.73-54.47-14.73-83.38v-42.27l-119.73-87.6c-23.82-15.88-55.29-14.01-77.06 4.59L5.81 227.64c-12.38 10.33-3.45 30.42 12.51 28.14zm556.87 34.1l-100.66-50.31A47.992 47.992 0 0 1 448 196.65v-36.69h64l28.09 22.63c6 6 14.14 9.37 22.63 9.37h30.97a32 32 0 0 0 28.62-17.69l14.31-28.62a32.005 32.005 0 0 0-3.02-33.51l-74.53-99.38C553.02 4.7 543.54 0 533.47 0H296.02c-7.13 0-10.7 8.57-5.66 13.61L352 63.96 292.42 88.8c-5.9 2.95-5.9 11.36 0 14.31L352 127.96v108.62c0 72.08 36.03 139.39 96 179.38-195.59 6.81-344.56 41.01-434.1 60.91C5.78 478.67 0 485.88 0 494.2 0 504 7.95 512 17.76 512h499.08c63.29.01 119.61-47.56 122.99-110.76 2.52-47.28-22.73-90.4-64.64-111.36zM489.18 66.25l45.65 11.41c-2.75 10.91-12.47 18.89-24.13 18.26-12.96-.71-25.85-12.53-21.52-29.67z"],"draw-polygon":[448,512,[],"f5ee","M384 352c-.35 0-.67.1-1.02.1l-39.2-65.32c5.07-9.17 8.22-19.56 8.22-30.78s-3.14-21.61-8.22-30.78l39.2-65.32c.35.01.67.1 1.02.1 35.35 0 64-28.65 64-64s-28.65-64-64-64c-23.63 0-44.04 12.95-55.12 32H119.12C108.04 44.95 87.63 32 64 32 28.65 32 0 60.65 0 96c0 23.63 12.95 44.04 32 55.12v209.75C12.95 371.96 0 392.37 0 416c0 35.35 28.65 64 64 64 23.63 0 44.04-12.95 55.12-32h209.75c11.09 19.05 31.49 32 55.12 32 35.35 0 64-28.65 64-64 .01-35.35-28.64-64-63.99-64zm-288 8.88V151.12A63.825 63.825 0 0 0 119.12 128h208.36l-38.46 64.1c-.35-.01-.67-.1-1.02-.1-35.35 0-64 28.65-64 64s28.65 64 64 64c.35 0 .67-.1 1.02-.1l38.46 64.1H119.12A63.748 63.748 0 0 0 96 360.88zM272 256c0-8.82 7.18-16 16-16s16 7.18 16 16-7.18 16-16 16-16-7.18-16-16zM400 96c0 8.82-7.18 16-16 16s-16-7.18-16-16 7.18-16 16-16 16 7.18 16 16zM64 80c8.82 0 16 7.18 16 16s-7.18 16-16 16-16-7.18-16-16 7.18-16 16-16zM48 416c0-8.82 7.18-16 16-16s16 7.18 16 16-7.18 16-16 16-16-7.18-16-16zm336 16c-8.82 0-16-7.18-16-16s7.18-16 16-16 16 7.18 16 16-7.18 16-16 16z"],drum:[512,512,[],"f569","M431.34 122.05l73.53-47.42a16 16 0 0 0 4.44-22.19l-8.87-13.31a16 16 0 0 0-22.19-4.44l-110.06 71C318.43 96.91 271.22 96 256 96 219.55 96 0 100.55 0 208.15v160.23c0 30.27 27.5 57.68 72 77.86v-101.9a24 24 0 1 1 48 0v118.93c33.05 9.11 71.07 15.06 112 16.73V376.39a24 24 0 1 1 48 0V480c40.93-1.67 78.95-7.62 112-16.73V344.34a24 24 0 1 1 48 0v101.9c44.5-20.18 72-47.59 72-77.86V208.15c0-43.32-35.76-69.76-80.66-86.1zM256 272.24c-114.88 0-208-28.69-208-64.09s93.12-64.08 208-64.08c17.15 0 33.73.71 49.68 1.91l-72.81 47a16 16 0 0 0-4.43 22.19l8.87 13.31a16 16 0 0 0 22.19 4.44l118.64-76.52C430.09 168 464 186.84 464 208.15c0 35.4-93.13 64.09-208 64.09z"],"drum-steelpan":[576,512,[],"f56a","M288 32C128.94 32 0 89.31 0 160v192c0 70.69 128.94 128 288 128s288-57.31 288-128V160c0-70.69-128.94-128-288-128zm-82.99 158.36c-4.45 16.61-14.54 30.57-28.31 40.48C100.23 217.46 48 190.78 48 160c0-30.16 50.11-56.39 124.04-70.03l25.6 44.34c9.86 17.09 12.48 36.99 7.37 56.05zM288 240c-21.08 0-41.41-1-60.89-2.7 8.06-26.13 32.15-45.3 60.89-45.3s52.83 19.17 60.89 45.3C329.41 239 309.08 240 288 240zm64-144c0 35.29-28.71 64-64 64s-64-28.71-64-64V82.96c20.4-1.88 41.8-2.96 64-2.96s43.6 1.08 64 2.96V96zm46.93 134.9c-13.81-9.91-23.94-23.9-28.4-40.54-5.11-19.06-2.49-38.96 7.38-56.04l25.65-44.42C477.72 103.5 528 129.79 528 160c0 30.83-52.4 57.54-129.07 70.9z"],"drumstick-bite":[512,512,[],"f6d7","M462.8 49.57a169.44 169.44 0 0 0-239.5 0C187.82 85 160.13 128 160.13 192v85.83l-40.62 40.59c-9.7 9.69-24 11.07-36.78 6a60.33 60.33 0 0 0-65 98.72C33 438.39 54.24 442.7 73.85 438.21c-4.5 19.6-.18 40.83 15.1 56.1a60.35 60.35 0 0 0 98.8-65c-5.09-12.73-3.72-27 6-36.75L234.36 352h85.89a187.87 187.87 0 0 0 61.89-10c-39.64-43.89-39.83-110.23 1.05-151.07 34.38-34.36 86.76-39.46 128.74-16.8 1.3-44.96-14.81-90.28-49.13-124.56z"],dumbbell:[640,512,[],"f44b","M104 96H56c-13.3 0-24 10.7-24 24v104H8c-4.4 0-8 3.6-8 8v48c0 4.4 3.6 8 8 8h24v104c0 13.3 10.7 24 24 24h48c13.3 0 24-10.7 24-24V120c0-13.3-10.7-24-24-24zm528 128h-24V120c0-13.3-10.7-24-24-24h-48c-13.3 0-24 10.7-24 24v272c0 13.3 10.7 24 24 24h48c13.3 0 24-10.7 24-24V288h24c4.4 0 8-3.6 8-8v-48c0-4.4-3.6-8-8-8zM456 32h-48c-13.3 0-24 10.7-24 24v168H256V56c0-13.3-10.7-24-24-24h-48c-13.3 0-24 10.7-24 24v400c0 13.3 10.7 24 24 24h48c13.3 0 24-10.7 24-24V288h128v168c0 13.3 10.7 24 24 24h48c13.3 0 24-10.7 24-24V56c0-13.3-10.7-24-24-24z"],dumpster:[576,512,[],"f793","M560 160c10.4 0 18-9.8 15.5-19.9l-24-96C549.7 37 543.3 32 536 32h-98.9l25.6 128H560zM272 32H171.5l-25.6 128H272V32zm132.5 0H304v128h126.1L404.5 32zM16 160h97.3l25.6-128H40c-7.3 0-13.7 5-15.5 12.1l-24 96C-2 150.2 5.6 160 16 160zm544 64h-20l4-32H32l4 32H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h28l20 160v16c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16v-16h320v16c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16v-16l20-160h28c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16z"],"dumpster-fire":[640,512,[],"f794","M418.7 104.1l.2-.2-14.4-72H304v128h60.8c16.2-19.3 34.2-38.2 53.9-55.8zM272 32H171.5l-25.6 128H272V32zm189.3 72.1c18.2 16.3 35.5 33.7 51.1 51.5 5.7-5.6 11.4-11.1 17.3-16.3l21.3-19 21.3 19c1.1.9 2.1 2.1 3.1 3.1-.1-.8.2-1.5 0-2.3l-24-96C549.7 37 543.3 32 536 32h-98.9l12.3 61.5 11.9 10.6zM16 160h97.3l25.6-128H40c-7.3 0-13.7 5-15.5 12.1l-24 96C-2 150.2 5.6 160 16 160zm324.6 32H32l4 32H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h28l20 160v16c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16v-16h208.8c-30.2-33.7-48.8-77.9-48.8-126.4 0-35.9 19.9-82.9 52.6-129.6zm210.5-28.8c-14.9 13.3-28.3 27.2-40.2 41.2-19.5-25.8-43.6-52-71-76.4-70.2 62.7-120 144.3-120 193.6 0 87.5 71.6 158.4 160 158.4s160-70.9 160-158.4c.1-36.6-37-112.2-88.8-158.4zm-18.6 229.4c-14.7 10.7-32.9 17-52.5 17-49 0-88.9-33.5-88.9-88 0-27.1 16.5-51 49.4-91.9 4.7 5.6 67.1 88.1 67.1 88.1l39.8-47c2.8 4.8 5.4 9.5 7.7 14 18.6 36.7 10.8 83.6-22.6 107.8z"],dungeon:[512,512,[],"f6d9","M128.73 195.32l-82.81-51.76c-8.04-5.02-18.99-2.17-22.93 6.45A254.19 254.19 0 0 0 .54 239.28C-.05 248.37 7.59 256 16.69 256h97.13c7.96 0 14.08-6.25 15.01-14.16 1.09-9.33 3.24-18.33 6.24-26.94 2.56-7.34.25-15.46-6.34-19.58zM319.03 8C298.86 2.82 277.77 0 256 0s-42.86 2.82-63.03 8c-9.17 2.35-13.91 12.6-10.39 21.39l37.47 104.03A16.003 16.003 0 0 0 235.1 144h41.8c6.75 0 12.77-4.23 15.05-10.58l37.47-104.03c3.52-8.79-1.22-19.03-10.39-21.39zM112 288H16c-8.84 0-16 7.16-16 16v64c0 8.84 7.16 16 16 16h96c8.84 0 16-7.16 16-16v-64c0-8.84-7.16-16-16-16zm0 128H16c-8.84 0-16 7.16-16 16v64c0 8.84 7.16 16 16 16h96c8.84 0 16-7.16 16-16v-64c0-8.84-7.16-16-16-16zm77.31-283.67l-36.32-90.8c-3.53-8.83-14.13-12.99-22.42-8.31a257.308 257.308 0 0 0-71.61 59.89c-6.06 7.32-3.85 18.48 4.22 23.52l82.93 51.83c6.51 4.07 14.66 2.62 20.11-2.79 5.18-5.15 10.79-9.85 16.79-14.05 6.28-4.41 9.15-12.17 6.3-19.29zM398.18 256h97.13c9.1 0 16.74-7.63 16.15-16.72a254.135 254.135 0 0 0-22.45-89.27c-3.94-8.62-14.89-11.47-22.93-6.45l-82.81 51.76c-6.59 4.12-8.9 12.24-6.34 19.58 3.01 8.61 5.15 17.62 6.24 26.94.93 7.91 7.05 14.16 15.01 14.16zm54.85-162.89a257.308 257.308 0 0 0-71.61-59.89c-8.28-4.68-18.88-.52-22.42 8.31l-36.32 90.8c-2.85 7.12.02 14.88 6.3 19.28 6 4.2 11.61 8.9 16.79 14.05 5.44 5.41 13.6 6.86 20.11 2.79l82.93-51.83c8.07-5.03 10.29-16.19 4.22-23.51zM496 288h-96c-8.84 0-16 7.16-16 16v64c0 8.84 7.16 16 16 16h96c8.84 0 16-7.16 16-16v-64c0-8.84-7.16-16-16-16zm0 128h-96c-8.84 0-16 7.16-16 16v64c0 8.84 7.16 16 16 16h96c8.84 0 16-7.16 16-16v-64c0-8.84-7.16-16-16-16zM240 177.62V472c0 4.42 3.58 8 8 8h16c4.42 0 8-3.58 8-8V177.62c-5.23-.89-10.52-1.62-16-1.62s-10.77.73-16 1.62zm-64 41.51V472c0 4.42 3.58 8 8 8h16c4.42 0 8-3.58 8-8V189.36c-12.78 7.45-23.84 17.47-32 29.77zm128-29.77V472c0 4.42 3.58 8 8 8h16c4.42 0 8-3.58 8-8V219.13c-8.16-12.3-19.22-22.32-32-29.77z"],edit:[576,512,[],"f044","M402.6 83.2l90.2 90.2c3.8 3.8 3.8 10 0 13.8L274.4 405.6l-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8L388.8 83.2c3.8-3.8 10-3.8 13.8 0zm162-22.9l-48.8-48.8c-15.2-15.2-39.9-15.2-55.2 0l-35.4 35.4c-3.8 3.8-3.8 10 0 13.8l90.2 90.2c3.8 3.8 10 3.8 13.8 0l35.4-35.4c15.2-15.3 15.2-40 0-55.2zM384 346.2V448H64V128h229.8c3.2 0 6.2-1.3 8.5-3.5l40-40c7.6-7.6 2.2-20.5-8.5-20.5H48C21.5 64 0 85.5 0 112v352c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V306.2c0-10.7-12.9-16-20.5-8.5l-40 40c-2.2 2.3-3.5 5.3-3.5 8.5z"],egg:[384,512,[],"f7fb","M192 0C86 0 0 214 0 320s86 192 192 192 192-86 192-192S298 0 192 0z"],eject:[448,512,[],"f052","M448 384v64c0 17.673-14.327 32-32 32H32c-17.673 0-32-14.327-32-32v-64c0-17.673 14.327-32 32-32h384c17.673 0 32 14.327 32 32zM48.053 320h351.886c41.651 0 63.581-49.674 35.383-80.435L259.383 47.558c-19.014-20.743-51.751-20.744-70.767 0L12.67 239.565C-15.475 270.268 6.324 320 48.053 320z"],"ellipsis-h":[512,512,[],"f141","M328 256c0 39.8-32.2 72-72 72s-72-32.2-72-72 32.2-72 72-72 72 32.2 72 72zm104-72c-39.8 0-72 32.2-72 72s32.2 72 72 72 72-32.2 72-72-32.2-72-72-72zm-352 0c-39.8 0-72 32.2-72 72s32.2 72 72 72 72-32.2 72-72-32.2-72-72-72z"],"ellipsis-v":[192,512,[],"f142","M96 184c39.8 0 72 32.2 72 72s-32.2 72-72 72-72-32.2-72-72 32.2-72 72-72zM24 80c0 39.8 32.2 72 72 72s72-32.2 72-72S135.8 8 96 8 24 40.2 24 80zm0 352c0 39.8 32.2 72 72 72s72-32.2 72-72-32.2-72-72-72-72 32.2-72 72z"],envelope:[512,512,[],"f0e0","M502.3 190.8c3.9-3.1 9.7-.2 9.7 4.7V400c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48V195.6c0-5 5.7-7.8 9.7-4.7 22.4 17.4 52.1 39.5 154.1 113.6 21.1 15.4 56.7 47.8 92.2 47.6 35.7.3 72-32.8 92.3-47.6 102-74.1 131.6-96.3 154-113.7zM256 320c23.2.4 56.6-29.2 73.4-41.4 132.7-96.3 142.8-104.7 173.4-128.7 5.8-4.5 9.2-11.5 9.2-18.9v-19c0-26.5-21.5-48-48-48H48C21.5 64 0 85.5 0 112v19c0 7.4 3.4 14.3 9.2 18.9 30.6 23.9 40.7 32.4 173.4 128.7 16.8 12.2 50.2 41.8 73.4 41.4z"],"envelope-open":[512,512,[],"f2b6","M512 464c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48V200.724a48 48 0 0 1 18.387-37.776c24.913-19.529 45.501-35.365 164.2-121.511C199.412 29.17 232.797-.347 256 .003c23.198-.354 56.596 29.172 73.413 41.433 118.687 86.137 139.303 101.995 164.2 121.512A48 48 0 0 1 512 200.724V464zm-65.666-196.605c-2.563-3.728-7.7-4.595-11.339-1.907-22.845 16.873-55.462 40.705-105.582 77.079-16.825 12.266-50.21 41.781-73.413 41.43-23.211.344-56.559-29.143-73.413-41.43-50.114-36.37-82.734-60.204-105.582-77.079-3.639-2.688-8.776-1.821-11.339 1.907l-9.072 13.196a7.998 7.998 0 0 0 1.839 10.967c22.887 16.899 55.454 40.69 105.303 76.868 20.274 14.781 56.524 47.813 92.264 47.573 35.724.242 71.961-32.771 92.263-47.573 49.85-36.179 82.418-59.97 105.303-76.868a7.998 7.998 0 0 0 1.839-10.967l-9.071-13.196z"],"envelope-open-text":[512,512,[],"f658","M176 216h160c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16H176c-8.84 0-16 7.16-16 16v16c0 8.84 7.16 16 16 16zm-16 80c0 8.84 7.16 16 16 16h160c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16H176c-8.84 0-16 7.16-16 16v16zm96 121.13c-16.42 0-32.84-5.06-46.86-15.19L0 250.86V464c0 26.51 21.49 48 48 48h416c26.51 0 48-21.49 48-48V250.86L302.86 401.94c-14.02 10.12-30.44 15.19-46.86 15.19zm237.61-254.18c-8.85-6.94-17.24-13.47-29.61-22.81V96c0-26.51-21.49-48-48-48h-77.55c-3.04-2.2-5.87-4.26-9.04-6.56C312.6 29.17 279.2-.35 256 0c-23.2-.35-56.59 29.17-73.41 41.44-3.17 2.3-6 4.36-9.04 6.56H96c-26.51 0-48 21.49-48 48v44.14c-12.37 9.33-20.76 15.87-29.61 22.81A47.995 47.995 0 0 0 0 200.72v10.65l96 69.35V96h320v184.72l96-69.35v-10.65c0-14.74-6.78-28.67-18.39-37.77z"],"envelope-square":[448,512,[],"f199","M400 32H48C21.49 32 0 53.49 0 80v352c0 26.51 21.49 48 48 48h352c26.51 0 48-21.49 48-48V80c0-26.51-21.49-48-48-48zM178.117 262.104C87.429 196.287 88.353 196.121 64 177.167V152c0-13.255 10.745-24 24-24h272c13.255 0 24 10.745 24 24v25.167c-24.371 18.969-23.434 19.124-114.117 84.938-10.5 7.655-31.392 26.12-45.883 25.894-14.503.218-35.367-18.227-45.883-25.895zM384 217.775V360c0 13.255-10.745 24-24 24H88c-13.255 0-24-10.745-24-24V217.775c13.958 10.794 33.329 25.236 95.303 70.214 14.162 10.341 37.975 32.145 64.694 32.01 26.887.134 51.037-22.041 64.72-32.025 61.958-44.965 81.325-59.406 95.283-70.199z"],equals:[448,512,[],"f52c","M416 304H32c-17.67 0-32 14.33-32 32v32c0 17.67 14.33 32 32 32h384c17.67 0 32-14.33 32-32v-32c0-17.67-14.33-32-32-32zm0-192H32c-17.67 0-32 14.33-32 32v32c0 17.67 14.33 32 32 32h384c17.67 0 32-14.33 32-32v-32c0-17.67-14.33-32-32-32z"],eraser:[512,512,[],"f12d","M497.941 273.941c18.745-18.745 18.745-49.137 0-67.882l-160-160c-18.745-18.745-49.136-18.746-67.883 0l-256 256c-18.745 18.745-18.745 49.137 0 67.882l96 96A48.004 48.004 0 0 0 144 480h356c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12H355.883l142.058-142.059zm-302.627-62.627l137.373 137.373L265.373 416H150.628l-80-80 124.686-124.686z"],ethernet:[512,512,[],"f796","M496 192h-48v-48c0-8.8-7.2-16-16-16h-48V80c0-8.8-7.2-16-16-16H144c-8.8 0-16 7.2-16 16v48H80c-8.8 0-16 7.2-16 16v48H16c-8.8 0-16 7.2-16 16v224c0 8.8 7.2 16 16 16h80V320h32v128h64V320h32v128h64V320h32v128h64V320h32v128h80c8.8 0 16-7.2 16-16V208c0-8.8-7.2-16-16-16z"],"euro-sign":[320,512,[],"f153","M310.706 413.765c-1.314-6.63-7.835-10.872-14.424-9.369-10.692 2.439-27.422 5.413-45.426 5.413-56.763 0-101.929-34.79-121.461-85.449h113.689a12 12 0 0 0 11.708-9.369l6.373-28.36c1.686-7.502-4.019-14.631-11.708-14.631H115.22c-1.21-14.328-1.414-28.287.137-42.245H261.95a12 12 0 0 0 11.723-9.434l6.512-29.755c1.638-7.484-4.061-14.566-11.723-14.566H130.184c20.633-44.991 62.69-75.03 117.619-75.03 14.486 0 28.564 2.25 37.851 4.145 6.216 1.268 12.347-2.498 14.002-8.623l11.991-44.368c1.822-6.741-2.465-13.616-9.326-14.917C290.217 34.912 270.71 32 249.635 32 152.451 32 74.03 92.252 45.075 176H12c-6.627 0-12 5.373-12 12v29.755c0 6.627 5.373 12 12 12h21.569c-1.009 13.607-1.181 29.287-.181 42.245H12c-6.627 0-12 5.373-12 12v28.36c0 6.627 5.373 12 12 12h30.114C67.139 414.692 145.264 480 249.635 480c26.301 0 48.562-4.544 61.101-7.788 6.167-1.595 10.027-7.708 8.788-13.957l-8.818-44.49z"],"exchange-alt":[512,512,[],"f362","M0 168v-16c0-13.255 10.745-24 24-24h360V80c0-21.367 25.899-32.042 40.971-16.971l80 80c9.372 9.373 9.372 24.569 0 33.941l-80 80C409.956 271.982 384 261.456 384 240v-48H24c-13.255 0-24-10.745-24-24zm488 152H128v-48c0-21.314-25.862-32.08-40.971-16.971l-80 80c-9.372 9.373-9.372 24.569 0 33.941l80 80C102.057 463.997 128 453.437 128 432v-48h360c13.255 0 24-10.745 24-24v-16c0-13.255-10.745-24-24-24z"],exclamation:[192,512,[],"f12a","M176 432c0 44.112-35.888 80-80 80s-80-35.888-80-80 35.888-80 80-80 80 35.888 80 80zM25.26 25.199l13.6 272C39.499 309.972 50.041 320 62.83 320h66.34c12.789 0 23.331-10.028 23.97-22.801l13.6-272C167.425 11.49 156.496 0 142.77 0H49.23C35.504 0 24.575 11.49 25.26 25.199z"],"exclamation-circle":[512,512,[],"f06a","M504 256c0 136.997-111.043 248-248 248S8 392.997 8 256C8 119.083 119.043 8 256 8s248 111.083 248 248zm-248 50c-25.405 0-46 20.595-46 46s20.595 46 46 46 46-20.595 46-46-20.595-46-46-46zm-43.673-165.346l7.418 136c.347 6.364 5.609 11.346 11.982 11.346h48.546c6.373 0 11.635-4.982 11.982-11.346l7.418-136c.375-6.874-5.098-12.654-11.982-12.654h-63.383c-6.884 0-12.356 5.78-11.981 12.654z"],"exclamation-triangle":[576,512,[],"f071","M569.517 440.013C587.975 472.007 564.806 512 527.94 512H48.054c-36.937 0-59.999-40.055-41.577-71.987L246.423 23.985c18.467-32.009 64.72-31.951 83.154 0l239.94 416.028zM288 354c-25.405 0-46 20.595-46 46s20.595 46 46 46 46-20.595 46-46-20.595-46-46-46zm-43.673-165.346l7.418 136c.347 6.364 5.609 11.346 11.982 11.346h48.546c6.373 0 11.635-4.982 11.982-11.346l7.418-136c.375-6.874-5.098-12.654-11.982-12.654h-63.383c-6.884 0-12.356 5.78-11.981 12.654z"],expand:[448,512,[],"f065","M0 180V56c0-13.3 10.7-24 24-24h124c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12H64v84c0 6.6-5.4 12-12 12H12c-6.6 0-12-5.4-12-12zM288 44v40c0 6.6 5.4 12 12 12h84v84c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12V56c0-13.3-10.7-24-24-24H300c-6.6 0-12 5.4-12 12zm148 276h-40c-6.6 0-12 5.4-12 12v84h-84c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h124c13.3 0 24-10.7 24-24V332c0-6.6-5.4-12-12-12zM160 468v-40c0-6.6-5.4-12-12-12H64v-84c0-6.6-5.4-12-12-12H12c-6.6 0-12 5.4-12 12v124c0 13.3 10.7 24 24 24h124c6.6 0 12-5.4 12-12z"],"expand-alt":[448,512,[],"f424","M212.686 315.314L120 408l32.922 31.029c15.12 15.12 4.412 40.971-16.97 40.971h-112C10.697 480 0 469.255 0 456V344c0-21.382 25.803-32.09 40.922-16.971L72 360l92.686-92.686c6.248-6.248 16.379-6.248 22.627 0l25.373 25.373c6.249 6.248 6.249 16.378 0 22.627zm22.628-118.628L328 104l-32.922-31.029C279.958 57.851 290.666 32 312.048 32h112C437.303 32 448 42.745 448 56v112c0 21.382-25.803 32.09-40.922 16.971L376 152l-92.686 92.686c-6.248 6.248-16.379 6.248-22.627 0l-25.373-25.373c-6.249-6.248-6.249-16.378 0-22.627z"],"expand-arrows-alt":[448,512,[],"f31e","M448 344v112a23.94 23.94 0 0 1-24 24H312c-21.39 0-32.09-25.9-17-41l36.2-36.2L224 295.6 116.77 402.9 153 439c15.09 15.1 4.39 41-17 41H24a23.94 23.94 0 0 1-24-24V344c0-21.4 25.89-32.1 41-17l36.19 36.2L184.46 256 77.18 148.7 41 185c-15.1 15.1-41 4.4-41-17V56a23.94 23.94 0 0 1 24-24h112c21.39 0 32.09 25.9 17 41l-36.2 36.2L224 216.4l107.23-107.3L295 73c-15.09-15.1-4.39-41 17-41h112a23.94 23.94 0 0 1 24 24v112c0 21.4-25.89 32.1-41 17l-36.19-36.2L263.54 256l107.28 107.3L407 327.1c15.1-15.2 41-4.5 41 16.9z"],"external-link-alt":[512,512,[],"f35d","M432,320H400a16,16,0,0,0-16,16V448H64V128H208a16,16,0,0,0,16-16V80a16,16,0,0,0-16-16H48A48,48,0,0,0,0,112V464a48,48,0,0,0,48,48H400a48,48,0,0,0,48-48V336A16,16,0,0,0,432,320ZM488,0h-128c-21.37,0-32.05,25.91-17,41l35.73,35.73L135,320.37a24,24,0,0,0,0,34L157.67,377a24,24,0,0,0,34,0L435.28,133.32,471,169c15,15,41,4.5,41-17V24A24,24,0,0,0,488,0Z"],"external-link-square-alt":[448,512,[],"f360","M448 80v352c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48V80c0-26.51 21.49-48 48-48h352c26.51 0 48 21.49 48 48zm-88 16H248.029c-21.313 0-32.08 25.861-16.971 40.971l31.984 31.987L67.515 364.485c-4.686 4.686-4.686 12.284 0 16.971l31.029 31.029c4.687 4.686 12.285 4.686 16.971 0l195.526-195.526 31.988 31.991C358.058 263.977 384 253.425 384 231.979V120c0-13.255-10.745-24-24-24z"],eye:[576,512,[],"f06e","M572.52 241.4C518.29 135.59 410.93 64 288 64S57.68 135.64 3.48 241.41a32.35 32.35 0 0 0 0 29.19C57.71 376.41 165.07 448 288 448s230.32-71.64 284.52-177.41a32.35 32.35 0 0 0 0-29.19zM288 400a144 144 0 1 1 144-144 143.93 143.93 0 0 1-144 144zm0-240a95.31 95.31 0 0 0-25.31 3.79 47.85 47.85 0 0 1-66.9 66.9A95.78 95.78 0 1 0 288 160z"],"eye-dropper":[512,512,[],"f1fb","M50.75 333.25c-12 12-18.75 28.28-18.75 45.26V424L0 480l32 32 56-32h45.49c16.97 0 33.25-6.74 45.25-18.74l126.64-126.62-128-128L50.75 333.25zM483.88 28.12c-37.47-37.5-98.28-37.5-135.75 0l-77.09 77.09-13.1-13.1c-9.44-9.44-24.65-9.31-33.94 0l-40.97 40.97c-9.37 9.37-9.37 24.57 0 33.94l161.94 161.94c9.44 9.44 24.65 9.31 33.94 0L419.88 288c9.37-9.37 9.37-24.57 0-33.94l-13.1-13.1 77.09-77.09c37.51-37.48 37.51-98.26.01-135.75z"],"eye-slash":[640,512,[],"f070","M320 400c-75.85 0-137.25-58.71-142.9-133.11L72.2 185.82c-13.79 17.3-26.48 35.59-36.72 55.59a32.35 32.35 0 0 0 0 29.19C89.71 376.41 197.07 448 320 448c26.91 0 52.87-4 77.89-10.46L346 397.39a144.13 144.13 0 0 1-26 2.61zm313.82 58.1l-110.55-85.44a331.25 331.25 0 0 0 81.25-102.07 32.35 32.35 0 0 0 0-29.19C550.29 135.59 442.93 64 320 64a308.15 308.15 0 0 0-147.32 37.7L45.46 3.37A16 16 0 0 0 23 6.18L3.37 31.45A16 16 0 0 0 6.18 53.9l588.36 454.73a16 16 0 0 0 22.46-2.81l19.64-25.27a16 16 0 0 0-2.82-22.45zm-183.72-142l-39.3-30.38A94.75 94.75 0 0 0 416 256a94.76 94.76 0 0 0-121.31-92.21A47.65 47.65 0 0 1 304 192a46.64 46.64 0 0 1-1.54 10l-73.61-56.89A142.31 142.31 0 0 1 320 112a143.92 143.92 0 0 1 144 144c0 21.63-5.29 41.79-13.9 60.11z"],fan:[512,512,[],"f863","M352.57 128c-28.09 0-54.09 4.52-77.06 12.86l12.41-123.11C289 7.31 279.81-1.18 269.33.13 189.63 10.13 128 77.64 128 159.43c0 28.09 4.52 54.09 12.86 77.06L17.75 224.08C7.31 223-1.18 232.19.13 242.67c10 79.7 77.51 141.33 159.3 141.33 28.09 0 54.09-4.52 77.06-12.86l-12.41 123.11c-1.05 10.43 8.11 18.93 18.59 17.62 79.7-10 141.33-77.51 141.33-159.3 0-28.09-4.52-54.09-12.86-77.06l123.11 12.41c10.44 1.05 18.93-8.11 17.62-18.59-10-79.7-77.51-141.33-159.3-141.33zM256 288a32 32 0 1 1 32-32 32 32 0 0 1-32 32z"],"fast-backward":[512,512,[],"f049","M0 436V76c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v151.9L235.5 71.4C256.1 54.3 288 68.6 288 96v131.9L459.5 71.4C480.1 54.3 512 68.6 512 96v320c0 27.4-31.9 41.7-52.5 24.6L288 285.3V416c0 27.4-31.9 41.7-52.5 24.6L64 285.3V436c0 6.6-5.4 12-12 12H12c-6.6 0-12-5.4-12-12z"],"fast-forward":[512,512,[],"f050","M512 76v360c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12V284.1L276.5 440.6c-20.6 17.2-52.5 2.8-52.5-24.6V284.1L52.5 440.6C31.9 457.8 0 443.4 0 416V96c0-27.4 31.9-41.7 52.5-24.6L224 226.8V96c0-27.4 31.9-41.7 52.5-24.6L448 226.8V76c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12z"],faucet:[512,512,[],"f905","M352,256H313.39c-15.71-13.44-35.46-23.07-57.39-28V180.44l-32-3.38-32,3.38V228c-21.93,5-41.68,14.6-57.39,28H16A16,16,0,0,0,0,272v96a16,16,0,0,0,16,16h92.79C129.38,421.73,173,448,224,448s94.62-26.27,115.21-64H352a32,32,0,0,1,32,32,32,32,0,0,0,32,32h64a32,32,0,0,0,32-32A160,160,0,0,0,352,256ZM81.59,159.91l142.41-15,142.41,15c9.42,1,17.59-6.81,17.59-16.8V112.89c0-10-8.17-17.8-17.59-16.81L256,107.74V80a16,16,0,0,0-16-16H208a16,16,0,0,0-16,16v27.74L81.59,96.08C72.17,95.09,64,102.9,64,112.89v30.22C64,153.1,72.17,160.91,81.59,159.91Z"],fax:[512,512,[],"f1ac","M480 160V77.25a32 32 0 0 0-9.38-22.63L425.37 9.37A32 32 0 0 0 402.75 0H160a32 32 0 0 0-32 32v448a32 32 0 0 0 32 32h320a32 32 0 0 0 32-32V192a32 32 0 0 0-32-32zM288 432a16 16 0 0 1-16 16h-32a16 16 0 0 1-16-16v-32a16 16 0 0 1 16-16h32a16 16 0 0 1 16 16zm0-128a16 16 0 0 1-16 16h-32a16 16 0 0 1-16-16v-32a16 16 0 0 1 16-16h32a16 16 0 0 1 16 16zm128 128a16 16 0 0 1-16 16h-32a16 16 0 0 1-16-16v-32a16 16 0 0 1 16-16h32a16 16 0 0 1 16 16zm0-128a16 16 0 0 1-16 16h-32a16 16 0 0 1-16-16v-32a16 16 0 0 1 16-16h32a16 16 0 0 1 16 16zm0-112H192V64h160v48a16 16 0 0 0 16 16h48zM64 128H32a32 32 0 0 0-32 32v320a32 32 0 0 0 32 32h32a32 32 0 0 0 32-32V160a32 32 0 0 0-32-32z"],feather:[512,512,[],"f52d","M467.14 44.84c-62.55-62.48-161.67-64.78-252.28 25.73-78.61 78.52-60.98 60.92-85.75 85.66-60.46 60.39-70.39 150.83-63.64 211.17l178.44-178.25c6.26-6.25 16.4-6.25 22.65 0s6.25 16.38 0 22.63L7.04 471.03c-9.38 9.37-9.38 24.57 0 33.94 9.38 9.37 24.6 9.37 33.98 0l66.1-66.03C159.42 454.65 279 457.11 353.95 384h-98.19l147.57-49.14c49.99-49.93 36.38-36.18 46.31-46.86h-97.78l131.54-43.8c45.44-74.46 34.31-148.84-16.26-199.36z"],"feather-alt":[512,512,[],"f56b","M512 0C460.22 3.56 96.44 38.2 71.01 287.61c-3.09 26.66-4.84 53.44-5.99 80.24l178.87-178.69c6.25-6.25 16.4-6.25 22.65 0s6.25 16.38 0 22.63L7.04 471.03c-9.38 9.37-9.38 24.57 0 33.94 9.38 9.37 24.59 9.37 33.98 0l57.13-57.07c42.09-.14 84.15-2.53 125.96-7.36 53.48-5.44 97.02-26.47 132.58-56.54H255.74l146.79-48.88c11.25-14.89 21.37-30.71 30.45-47.12h-81.14l106.54-53.21C500.29 132.86 510.19 26.26 512 0z"],female:[256,512,[],"f182","M128 0c35.346 0 64 28.654 64 64s-28.654 64-64 64c-35.346 0-64-28.654-64-64S92.654 0 128 0m119.283 354.179l-48-192A24 24 0 0 0 176 144h-11.36c-22.711 10.443-49.59 10.894-73.28 0H80a24 24 0 0 0-23.283 18.179l-48 192C4.935 369.305 16.383 384 32 384h56v104c0 13.255 10.745 24 24 24h32c13.255 0 24-10.745 24-24V384h56c15.591 0 27.071-14.671 23.283-29.821z"],"fighter-jet":[640,512,[],"f0fb","M544 224l-128-16-48-16h-24L227.158 44h39.509C278.333 44 288 41.375 288 38s-9.667-6-21.333-6H152v12h16v164h-48l-66.667-80H18.667L8 138.667V208h8v16h48v2.666l-64 8v42.667l64 8V288H16v16H8v69.333L18.667 384h34.667L120 304h48v164h-16v12h114.667c11.667 0 21.333-2.625 21.333-6s-9.667-6-21.333-6h-39.509L344 320h24l48-16 128-16c96-21.333 96-26.583 96-32 0-5.417 0-10.667-96-32z"],file:[384,512,[],"f15b","M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm160-14.1v6.1H256V0h6.1c6.4 0 12.5 2.5 17 7l97.9 98c4.5 4.5 7 10.6 7 16.9z"],"file-alt":[384,512,[],"f15c","M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm64 236c0 6.6-5.4 12-12 12H108c-6.6 0-12-5.4-12-12v-8c0-6.6 5.4-12 12-12h168c6.6 0 12 5.4 12 12v8zm0-64c0 6.6-5.4 12-12 12H108c-6.6 0-12-5.4-12-12v-8c0-6.6 5.4-12 12-12h168c6.6 0 12 5.4 12 12v8zm0-72v8c0 6.6-5.4 12-12 12H108c-6.6 0-12-5.4-12-12v-8c0-6.6 5.4-12 12-12h168c6.6 0 12 5.4 12 12zm96-114.1v6.1H256V0h6.1c6.4 0 12.5 2.5 17 7l97.9 98c4.5 4.5 7 10.6 7 16.9z"],"file-archive":[384,512,[],"f1c6","M377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9zM128.4 336c-17.9 0-32.4 12.1-32.4 27 0 15 14.6 27 32.5 27s32.4-12.1 32.4-27-14.6-27-32.5-27zM224 136V0h-63.6v32h-32V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zM95.9 32h32v32h-32zm32.3 384c-33.2 0-58-30.4-51.4-62.9L96.4 256v-32h32v-32h-32v-32h32v-32h-32V96h32V64h32v32h-32v32h32v32h-32v32h32v32h-32v32h22.1c5.7 0 10.7 4.1 11.8 9.7l17.3 87.7c6.4 32.4-18.4 62.6-51.4 62.6z"],"file-audio":[384,512,[],"f1c7","M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm-64 268c0 10.7-12.9 16-20.5 8.5L104 376H76c-6.6 0-12-5.4-12-12v-56c0-6.6 5.4-12 12-12h28l35.5-36.5c7.6-7.6 20.5-2.2 20.5 8.5v136zm33.2-47.6c9.1-9.3 9.1-24.1 0-33.4-22.1-22.8 12.2-56.2 34.4-33.5 27.2 27.9 27.2 72.4 0 100.4-21.8 22.3-56.9-10.4-34.4-33.5zm86-117.1c54.4 55.9 54.4 144.8 0 200.8-21.8 22.4-57-10.3-34.4-33.5 36.2-37.2 36.3-96.5 0-133.8-22.1-22.8 12.3-56.3 34.4-33.5zM384 121.9v6.1H256V0h6.1c6.4 0 12.5 2.5 17 7l97.9 98c4.5 4.5 7 10.6 7 16.9z"],"file-code":[384,512,[],"f1c9","M384 121.941V128H256V0h6.059c6.365 0 12.47 2.529 16.971 7.029l97.941 97.941A24.005 24.005 0 0 1 384 121.941zM248 160c-13.2 0-24-10.8-24-24V0H24C10.745 0 0 10.745 0 24v464c0 13.255 10.745 24 24 24h336c13.255 0 24-10.745 24-24V160H248zM123.206 400.505a5.4 5.4 0 0 1-7.633.246l-64.866-60.812a5.4 5.4 0 0 1 0-7.879l64.866-60.812a5.4 5.4 0 0 1 7.633.246l19.579 20.885a5.4 5.4 0 0 1-.372 7.747L101.65 336l40.763 35.874a5.4 5.4 0 0 1 .372 7.747l-19.579 20.884zm51.295 50.479l-27.453-7.97a5.402 5.402 0 0 1-3.681-6.692l61.44-211.626a5.402 5.402 0 0 1 6.692-3.681l27.452 7.97a5.4 5.4 0 0 1 3.68 6.692l-61.44 211.626a5.397 5.397 0 0 1-6.69 3.681zm160.792-111.045l-64.866 60.812a5.4 5.4 0 0 1-7.633-.246l-19.58-20.885a5.4 5.4 0 0 1 .372-7.747L284.35 336l-40.763-35.874a5.4 5.4 0 0 1-.372-7.747l19.58-20.885a5.4 5.4 0 0 1 7.633-.246l64.866 60.812a5.4 5.4 0 0 1-.001 7.879z"],"file-contract":[384,512,[],"f56c","M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zM64 72c0-4.42 3.58-8 8-8h80c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8V72zm0 64c0-4.42 3.58-8 8-8h80c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8v-16zm192.81 248H304c8.84 0 16 7.16 16 16s-7.16 16-16 16h-47.19c-16.45 0-31.27-9.14-38.64-23.86-2.95-5.92-8.09-6.52-10.17-6.52s-7.22.59-10.02 6.19l-7.67 15.34a15.986 15.986 0 0 1-14.31 8.84c-.38 0-.75-.02-1.14-.05-6.45-.45-12-4.75-14.03-10.89L144 354.59l-10.61 31.88c-5.89 17.66-22.38 29.53-41 29.53H80c-8.84 0-16-7.16-16-16s7.16-16 16-16h12.39c4.83 0 9.11-3.08 10.64-7.66l18.19-54.64c3.3-9.81 12.44-16.41 22.78-16.41s19.48 6.59 22.77 16.41l13.88 41.64c19.77-16.19 54.05-9.7 66 14.16 2.02 4.06 5.96 6.5 10.16 6.5zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"],"file-csv":[384,512,[],"f6dd","M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm-96 144c0 4.42-3.58 8-8 8h-8c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h8c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8h-8c-26.51 0-48-21.49-48-48v-32c0-26.51 21.49-48 48-48h8c4.42 0 8 3.58 8 8v16zm44.27 104H160c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h12.27c5.95 0 10.41-3.5 10.41-6.62 0-1.3-.75-2.66-2.12-3.84l-21.89-18.77c-8.47-7.22-13.33-17.48-13.33-28.14 0-21.3 19.02-38.62 42.41-38.62H200c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8h-12.27c-5.95 0-10.41 3.5-10.41 6.62 0 1.3.75 2.66 2.12 3.84l21.89 18.77c8.47 7.22 13.33 17.48 13.33 28.14.01 21.29-19 38.62-42.39 38.62zM256 264v20.8c0 20.27 5.7 40.17 16 56.88 10.3-16.7 16-36.61 16-56.88V264c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v20.8c0 35.48-12.88 68.89-36.28 94.09-3.02 3.25-7.27 5.11-11.72 5.11s-8.7-1.86-11.72-5.11c-23.4-25.2-36.28-58.61-36.28-94.09V264c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8zm121-159L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"],"file-download":[384,512,[],"f56d","M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"],"file-excel":[384,512,[],"f1c3","M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm60.1 106.5L224 336l60.1 93.5c5.1 8-.6 18.5-10.1 18.5h-34.9c-4.4 0-8.5-2.4-10.6-6.3C208.9 405.5 192 373 192 373c-6.4 14.8-10 20-36.6 68.8-2.1 3.9-6.1 6.3-10.5 6.3H110c-9.5 0-15.2-10.5-10.1-18.5l60.3-93.5-60.3-93.5c-5.2-8 .6-18.5 10.1-18.5h34.8c4.4 0 8.5 2.4 10.6 6.3 26.1 48.8 20 33.6 36.6 68.5 0 0 6.1-11.7 36.6-68.5 2.1-3.9 6.2-6.3 10.6-6.3H274c9.5-.1 15.2 10.4 10.1 18.4zM384 121.9v6.1H256V0h6.1c6.4 0 12.5 2.5 17 7l97.9 98c4.5 4.5 7 10.6 7 16.9z"],"file-export":[576,512,[],"f56e","M384 121.9c0-6.3-2.5-12.4-7-16.9L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128zM571 308l-95.7-96.4c-10.1-10.1-27.4-3-27.4 11.3V288h-64v64h64v65.2c0 14.3 17.3 21.4 27.4 11.3L571 332c6.6-6.6 6.6-17.4 0-24zm-379 28v-32c0-8.8 7.2-16 16-16h176V160H248c-13.2 0-24-10.8-24-24V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V352H208c-8.8 0-16-7.2-16-16z"],"file-image":[384,512,[],"f1c5","M384 121.941V128H256V0h6.059a24 24 0 0 1 16.97 7.029l97.941 97.941a24.002 24.002 0 0 1 7.03 16.971zM248 160c-13.2 0-24-10.8-24-24V0H24C10.745 0 0 10.745 0 24v464c0 13.255 10.745 24 24 24h336c13.255 0 24-10.745 24-24V160H248zm-135.455 16c26.51 0 48 21.49 48 48s-21.49 48-48 48-48-21.49-48-48 21.491-48 48-48zm208 240h-256l.485-48.485L104.545 328c4.686-4.686 11.799-4.201 16.485.485L160.545 368 264.06 264.485c4.686-4.686 12.284-4.686 16.971 0L320.545 304v112z"],"file-import":[512,512,[],"f56f","M16 288c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h112v-64zm489-183L407.1 7c-4.5-4.5-10.6-7-17-7H384v128h128v-6.1c0-6.3-2.5-12.4-7-16.9zm-153 31V0H152c-13.3 0-24 10.7-24 24v264h128v-65.2c0-14.3 17.3-21.4 27.4-11.3L379 308c6.6 6.7 6.6 17.4 0 24l-95.7 96.4c-10.1 10.1-27.4 3-27.4-11.3V352H128v136c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H376c-13.2 0-24-10.8-24-24z"],"file-invoice":[384,512,[],"f570","M288 256H96v64h192v-64zm89-151L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9zm-153 31V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zM64 72c0-4.42 3.58-8 8-8h80c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8V72zm0 64c0-4.42 3.58-8 8-8h80c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8v-16zm256 304c0 4.42-3.58 8-8 8h-80c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h80c4.42 0 8 3.58 8 8v16zm0-200v96c0 8.84-7.16 16-16 16H80c-8.84 0-16-7.16-16-16v-96c0-8.84 7.16-16 16-16h224c8.84 0 16 7.16 16 16z"],"file-invoice-dollar":[384,512,[],"f571","M377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9zm-153 31V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zM64 72c0-4.42 3.58-8 8-8h80c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8V72zm0 80v-16c0-4.42 3.58-8 8-8h80c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8zm144 263.88V440c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8v-24.29c-11.29-.58-22.27-4.52-31.37-11.35-3.9-2.93-4.1-8.77-.57-12.14l11.75-11.21c2.77-2.64 6.89-2.76 10.13-.73 3.87 2.42 8.26 3.72 12.82 3.72h28.11c6.5 0 11.8-5.92 11.8-13.19 0-5.95-3.61-11.19-8.77-12.73l-45-13.5c-18.59-5.58-31.58-23.42-31.58-43.39 0-24.52 19.05-44.44 42.67-45.07V232c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v24.29c11.29.58 22.27 4.51 31.37 11.35 3.9 2.93 4.1 8.77.57 12.14l-11.75 11.21c-2.77 2.64-6.89 2.76-10.13.73-3.87-2.43-8.26-3.72-12.82-3.72h-28.11c-6.5 0-11.8 5.92-11.8 13.19 0 5.95 3.61 11.19 8.77 12.73l45 13.5c18.59 5.58 31.58 23.42 31.58 43.39 0 24.53-19.05 44.44-42.67 45.07z"],"file-medical":[384,512,[],"f477","M377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9zm-153 31V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm64 160v48c0 4.4-3.6 8-8 8h-56v56c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8v-56h-56c-4.4 0-8-3.6-8-8v-48c0-4.4 3.6-8 8-8h56v-56c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v56h56c4.4 0 8 3.6 8 8z"],"file-medical-alt":[448,512,[],"f478","M288 136V0H88C74.7 0 64 10.7 64 24v232H8c-4.4 0-8 3.6-8 8v16c0 4.4 3.6 8 8 8h140.9c3 0 5.8 1.7 7.2 4.4l19.9 39.8 56.8-113.7c2.9-5.9 11.4-5.9 14.3 0l34.7 69.5H352c8.8 0 16 7.2 16 16s-7.2 16-16 16h-89.9L240 275.8l-56.8 113.7c-2.9 5.9-11.4 5.9-14.3 0L134.1 320H64v168c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H312c-13.2 0-24-10.8-24-24zm153-31L343.1 7c-4.5-4.5-10.6-7-17-7H320v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"],"file-pdf":[384,512,[],"f1c1","M181.9 256.1c-5-16-4.9-46.9-2-46.9 8.4 0 7.6 36.9 2 46.9zm-1.7 47.2c-7.7 20.2-17.3 43.3-28.4 62.7 18.3-7 39-17.2 62.9-21.9-12.7-9.6-24.9-23.4-34.5-40.8zM86.1 428.1c0 .8 13.2-5.4 34.9-40.2-6.7 6.3-29.1 24.5-34.9 40.2zM248 160h136v328c0 13.3-10.7 24-24 24H24c-13.3 0-24-10.7-24-24V24C0 10.7 10.7 0 24 0h200v136c0 13.2 10.8 24 24 24zm-8 171.8c-20-12.2-33.3-29-42.7-53.8 4.5-18.5 11.6-46.6 6.2-64.2-4.7-29.4-42.4-26.5-47.8-6.8-5 18.3-.4 44.1 8.1 77-11.6 27.6-28.7 64.6-40.8 85.8-.1 0-.1.1-.2.1-27.1 13.9-73.6 44.5-54.5 68 5.6 6.9 16 10 21.5 10 17.9 0 35.7-18 61.1-61.8 25.8-8.5 54.1-19.1 79-23.2 21.7 11.8 47.1 19.5 64 19.5 29.2 0 31.2-32 19.7-43.4-13.9-13.6-54.3-9.7-73.6-7.2zM377 105L279 7c-4.5-4.5-10.6-7-17-7h-6v128h128v-6.1c0-6.3-2.5-12.4-7-16.9zm-74.1 255.3c4.1-2.7-2.5-11.9-42.8-9 37.1 15.8 42.8 9 42.8 9z"],"file-powerpoint":[384,512,[],"f1c4","M193.7 271.2c8.8 0 15.5 2.7 20.3 8.1 9.6 10.9 9.8 32.7-.2 44.1-4.9 5.6-11.9 8.5-21.1 8.5h-26.9v-60.7h27.9zM377 105L279 7c-4.5-4.5-10.6-7-17-7h-6v128h128v-6.1c0-6.3-2.5-12.4-7-16.9zm-153 31V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm53 165.2c0 90.3-88.8 77.6-111.1 77.6V436c0 6.6-5.4 12-12 12h-30.8c-6.6 0-12-5.4-12-12V236.2c0-6.6 5.4-12 12-12h81c44.5 0 72.9 32.8 72.9 77z"],"file-prescription":[384,512,[],"f572","M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm68.53 179.48l11.31 11.31c6.25 6.25 6.25 16.38 0 22.63l-29.9 29.9L304 409.38c6.25 6.25 6.25 16.38 0 22.63l-11.31 11.31c-6.25 6.25-16.38 6.25-22.63 0L240 413.25l-30.06 30.06c-6.25 6.25-16.38 6.25-22.63 0L176 432c-6.25-6.25-6.25-16.38 0-22.63l30.06-30.06L146.74 320H128v48c0 8.84-7.16 16-16 16H96c-8.84 0-16-7.16-16-16V208c0-8.84 7.16-16 16-16h80c35.35 0 64 28.65 64 64 0 24.22-13.62 45.05-33.46 55.92L240 345.38l29.9-29.9c6.25-6.25 16.38-6.25 22.63 0zM176 272h-48v-32h48c8.82 0 16 7.18 16 16s-7.18 16-16 16zm208-150.1v6.1H256V0h6.1c6.4 0 12.5 2.5 17 7l97.9 98c4.5 4.5 7 10.6 7 16.9z"],"file-signature":[576,512,[],"f573","M218.17 424.14c-2.95-5.92-8.09-6.52-10.17-6.52s-7.22.59-10.02 6.19l-7.67 15.34c-6.37 12.78-25.03 11.37-29.48-2.09L144 386.59l-10.61 31.88c-5.89 17.66-22.38 29.53-41 29.53H80c-8.84 0-16-7.16-16-16s7.16-16 16-16h12.39c4.83 0 9.11-3.08 10.64-7.66l18.19-54.64c3.3-9.81 12.44-16.41 22.78-16.41s19.48 6.59 22.77 16.41l13.88 41.64c19.75-16.19 54.06-9.7 66 14.16 1.89 3.78 5.49 5.95 9.36 6.26v-82.12l128-127.09V160H248c-13.2 0-24-10.8-24-24V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24v-40l-128-.11c-16.12-.31-30.58-9.28-37.83-23.75zM384 121.9c0-6.3-2.5-12.4-7-16.9L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1zm-96 225.06V416h68.99l161.68-162.78-67.88-67.88L288 346.96zm280.54-179.63l-31.87-31.87c-9.94-9.94-26.07-9.94-36.01 0l-27.25 27.25 67.88 67.88 27.25-27.25c9.95-9.94 9.95-26.07 0-36.01z"],"file-upload":[384,512,[],"f574","M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm65.18 216.01H224v80c0 8.84-7.16 16-16 16h-32c-8.84 0-16-7.16-16-16v-80H94.82c-14.28 0-21.41-17.29-11.27-27.36l96.42-95.7c6.65-6.61 17.39-6.61 24.04 0l96.42 95.7c10.15 10.07 3.03 27.36-11.25 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"],"file-video":[384,512,[],"f1c8","M384 121.941V128H256V0h6.059c6.365 0 12.47 2.529 16.971 7.029l97.941 97.941A24.005 24.005 0 0 1 384 121.941zM224 136V0H24C10.745 0 0 10.745 0 24v464c0 13.255 10.745 24 24 24h336c13.255 0 24-10.745 24-24V160H248c-13.2 0-24-10.8-24-24zm96 144.016v111.963c0 21.445-25.943 31.998-40.971 16.971L224 353.941V392c0 13.255-10.745 24-24 24H88c-13.255 0-24-10.745-24-24V280c0-13.255 10.745-24 24-24h112c13.255 0 24 10.745 24 24v38.059l55.029-55.013c15.011-15.01 40.971-4.491 40.971 16.97z"],"file-word":[384,512,[],"f1c2","M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm57.1 120H305c7.7 0 13.4 7.1 11.7 14.7l-38 168c-1.2 5.5-6.1 9.3-11.7 9.3h-38c-5.5 0-10.3-3.8-11.6-9.1-25.8-103.5-20.8-81.2-25.6-110.5h-.5c-1.1 14.3-2.4 17.4-25.6 110.5-1.3 5.3-6.1 9.1-11.6 9.1H117c-5.6 0-10.5-3.9-11.7-9.4l-37.8-168c-1.7-7.5 4-14.6 11.7-14.6h24.5c5.7 0 10.7 4 11.8 9.7 15.6 78 20.1 109.5 21 122.2 1.6-10.2 7.3-32.7 29.4-122.7 1.3-5.4 6.1-9.1 11.7-9.1h29.1c5.6 0 10.4 3.8 11.7 9.2 24 100.4 28.8 124 29.6 129.4-.2-11.2-2.6-17.8 21.6-129.2 1-5.6 5.9-9.5 11.5-9.5zM384 121.9v6.1H256V0h6.1c6.4 0 12.5 2.5 17 7l97.9 98c4.5 4.5 7 10.6 7 16.9z"],fill:[512,512,[],"f575","M502.63 217.06L294.94 9.37C288.69 3.12 280.5 0 272.31 0s-16.38 3.12-22.62 9.37l-81.58 81.58L81.93 4.77c-6.24-6.25-16.38-6.25-22.62 0L36.69 27.38c-6.24 6.25-6.24 16.38 0 22.63l86.19 86.18-94.76 94.76c-37.49 37.49-37.49 98.26 0 135.75l117.19 117.19c18.75 18.74 43.31 28.12 67.87 28.12 24.57 0 49.13-9.37 67.88-28.12l221.57-221.57c12.49-12.5 12.49-32.76 0-45.26zm-116.22 70.97H65.93c1.36-3.84 3.57-7.98 7.43-11.83l13.15-13.15 81.61-81.61 58.61 58.6c12.49 12.49 32.75 12.49 45.24 0 12.49-12.49 12.49-32.75 0-45.24l-58.61-58.6 58.95-58.95 162.45 162.44-48.35 48.34z"],"fill-drip":[576,512,[],"f576","M512 320s-64 92.65-64 128c0 35.35 28.66 64 64 64s64-28.65 64-64-64-128-64-128zm-9.37-102.94L294.94 9.37C288.69 3.12 280.5 0 272.31 0s-16.38 3.12-22.62 9.37l-81.58 81.58L81.93 4.76c-6.25-6.25-16.38-6.25-22.62 0L36.69 27.38c-6.24 6.25-6.24 16.38 0 22.62l86.19 86.18-94.76 94.76c-37.49 37.48-37.49 98.26 0 135.75l117.19 117.19c18.74 18.74 43.31 28.12 67.87 28.12 24.57 0 49.13-9.37 67.87-28.12l221.57-221.57c12.5-12.5 12.5-32.75.01-45.25zm-116.22 70.97H65.93c1.36-3.84 3.57-7.98 7.43-11.83l13.15-13.15 81.61-81.61 58.6 58.6c12.49 12.49 32.75 12.49 45.24 0s12.49-32.75 0-45.24l-58.6-58.6 58.95-58.95 162.44 162.44-48.34 48.34z"],film:[512,512,[],"f008","M488 64h-8v20c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12V64H96v20c0 6.6-5.4 12-12 12H44c-6.6 0-12-5.4-12-12V64h-8C10.7 64 0 74.7 0 88v336c0 13.3 10.7 24 24 24h8v-20c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v20h320v-20c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v20h8c13.3 0 24-10.7 24-24V88c0-13.3-10.7-24-24-24zM96 372c0 6.6-5.4 12-12 12H44c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40zm0-96c0 6.6-5.4 12-12 12H44c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40zm0-96c0 6.6-5.4 12-12 12H44c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40zm272 208c0 6.6-5.4 12-12 12H156c-6.6 0-12-5.4-12-12v-96c0-6.6 5.4-12 12-12h200c6.6 0 12 5.4 12 12v96zm0-168c0 6.6-5.4 12-12 12H156c-6.6 0-12-5.4-12-12v-96c0-6.6 5.4-12 12-12h200c6.6 0 12 5.4 12 12v96zm112 152c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40zm0-96c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40zm0-96c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40z"],filter:[512,512,[],"f0b0","M487.976 0H24.028C2.71 0-8.047 25.866 7.058 40.971L192 225.941V432c0 7.831 3.821 15.17 10.237 19.662l80 55.98C298.02 518.69 320 507.493 320 487.98V225.941l184.947-184.97C520.021 25.896 509.338 0 487.976 0z"],fingerprint:[512,512,[],"f577","M256.12 245.96c-13.25 0-24 10.74-24 24 1.14 72.25-8.14 141.9-27.7 211.55-2.73 9.72 2.15 30.49 23.12 30.49 10.48 0 20.11-6.92 23.09-17.52 13.53-47.91 31.04-125.41 29.48-224.52.01-13.25-10.73-24-23.99-24zm-.86-81.73C194 164.16 151.25 211.3 152.1 265.32c.75 47.94-3.75 95.91-13.37 142.55-2.69 12.98 5.67 25.69 18.64 28.36 13.05 2.67 25.67-5.66 28.36-18.64 10.34-50.09 15.17-101.58 14.37-153.02-.41-25.95 19.92-52.49 54.45-52.34 31.31.47 57.15 25.34 57.62 55.47.77 48.05-2.81 96.33-10.61 143.55-2.17 13.06 6.69 25.42 19.76 27.58 19.97 3.33 26.81-15.1 27.58-19.77 8.28-50.03 12.06-101.21 11.27-152.11-.88-55.8-47.94-101.88-104.91-102.72zm-110.69-19.78c-10.3-8.34-25.37-6.8-33.76 3.48-25.62 31.5-39.39 71.28-38.75 112 .59 37.58-2.47 75.27-9.11 112.05-2.34 13.05 6.31 25.53 19.36 27.89 20.11 3.5 27.07-14.81 27.89-19.36 7.19-39.84 10.5-80.66 9.86-121.33-.47-29.88 9.2-57.88 28-80.97 8.35-10.28 6.79-25.39-3.49-33.76zm109.47-62.33c-15.41-.41-30.87 1.44-45.78 4.97-12.89 3.06-20.87 15.98-17.83 28.89 3.06 12.89 16 20.83 28.89 17.83 11.05-2.61 22.47-3.77 34-3.69 75.43 1.13 137.73 61.5 138.88 134.58.59 37.88-1.28 76.11-5.58 113.63-1.5 13.17 7.95 25.08 21.11 26.58 16.72 1.95 25.51-11.88 26.58-21.11a929.06 929.06 0 0 0 5.89-119.85c-1.56-98.75-85.07-180.33-186.16-181.83zm252.07 121.45c-2.86-12.92-15.51-21.2-28.61-18.27-12.94 2.86-21.12 15.66-18.26 28.61 4.71 21.41 4.91 37.41 4.7 61.6-.11 13.27 10.55 24.09 23.8 24.2h.2c13.17 0 23.89-10.61 24-23.8.18-22.18.4-44.11-5.83-72.34zm-40.12-90.72C417.29 43.46 337.6 1.29 252.81.02 183.02-.82 118.47 24.91 70.46 72.94 24.09 119.37-.9 181.04.14 246.65l-.12 21.47c-.39 13.25 10.03 24.31 23.28 24.69.23.02.48.02.72.02 12.92 0 23.59-10.3 23.97-23.3l.16-23.64c-.83-52.5 19.16-101.86 56.28-139 38.76-38.8 91.34-59.67 147.68-58.86 69.45 1.03 134.73 35.56 174.62 92.39 7.61 10.86 22.56 13.45 33.42 5.86 10.84-7.62 13.46-22.59 5.84-33.43z"],fire:[384,512,[],"f06d","M216 23.86c0-23.8-30.65-32.77-44.15-13.04C48 191.85 224 200 224 288c0 35.63-29.11 64.46-64.85 63.99-35.17-.45-63.15-29.77-63.15-64.94v-85.51c0-21.7-26.47-32.23-41.43-16.5C27.8 213.16 0 261.33 0 320c0 105.87 86.13 192 192 192s192-86.13 192-192c0-170.29-168-193-168-296.14z"],"fire-alt":[448,512,[],"f7e4","M323.56 51.2c-20.8 19.3-39.58 39.59-56.22 59.97C240.08 73.62 206.28 35.53 168 0 69.74 91.17 0 209.96 0 281.6 0 408.85 100.29 512 224 512s224-103.15 224-230.4c0-53.27-51.98-163.14-124.44-230.4zm-19.47 340.65C282.43 407.01 255.72 416 226.86 416 154.71 416 96 368.26 96 290.75c0-38.61 24.31-72.63 72.79-130.75 6.93 7.98 98.83 125.34 98.83 125.34l58.63-66.88c4.14 6.85 7.91 13.55 11.27 19.97 27.35 52.19 15.81 118.97-33.43 153.42z"],"fire-extinguisher":[448,512,[],"f134","M434.027 26.329l-168 28C254.693 56.218 256 67.8 256 72h-58.332C208.353 36.108 181.446 0 144 0c-39.435 0-66.368 39.676-52.228 76.203-52.039 13.051-75.381 54.213-90.049 90.884-4.923 12.307 1.063 26.274 13.37 31.197 12.317 4.926 26.279-1.075 31.196-13.37C75.058 112.99 106.964 120 168 120v27.076c-41.543 10.862-72 49.235-72 94.129V488c0 13.255 10.745 24 24 24h144c13.255 0 24-10.745 24-24V240c0-44.731-30.596-82.312-72-92.97V120h40c0 2.974-1.703 15.716 10.027 17.671l168 28C441.342 166.89 448 161.25 448 153.834V38.166c0-7.416-6.658-13.056-13.973-11.837zM144 72c-8.822 0-16-7.178-16-16s7.178-16 16-16 16 7.178 16 16-7.178 16-16 16z"],"first-aid":[576,512,[],"f479","M0 80v352c0 26.5 21.5 48 48 48h48V32H48C21.5 32 0 53.5 0 80zm128 400h320V32H128v448zm64-248c0-4.4 3.6-8 8-8h56v-56c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v56h56c4.4 0 8 3.6 8 8v48c0 4.4-3.6 8-8 8h-56v56c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8v-56h-56c-4.4 0-8-3.6-8-8v-48zM528 32h-48v448h48c26.5 0 48-21.5 48-48V80c0-26.5-21.5-48-48-48z"],fish:[576,512,[],"f578","M327.1 96c-89.97 0-168.54 54.77-212.27 101.63L27.5 131.58c-12.13-9.18-30.24.6-27.14 14.66L24.54 256 .35 365.77c-3.1 14.06 15.01 23.83 27.14 14.66l87.33-66.05C158.55 361.23 237.13 416 327.1 416 464.56 416 576 288 576 256S464.56 96 327.1 96zm87.43 184c-13.25 0-24-10.75-24-24 0-13.26 10.75-24 24-24 13.26 0 24 10.74 24 24 0 13.25-10.75 24-24 24z"],"fist-raised":[384,512,[],"f6de","M255.98 160V16c0-8.84-7.16-16-16-16h-32c-8.84 0-16 7.16-16 16v146.93c5.02-1.78 10.34-2.93 15.97-2.93h48.03zm128 95.99c-.01-35.34-28.66-63.99-63.99-63.99H207.85c-8.78 0-15.9 7.07-15.9 15.85v.56c0 26.27 21.3 47.59 47.57 47.59h35.26c9.68 0 13.2 3.58 13.2 8v16.2c0 4.29-3.59 7.78-7.88 8-44.52 2.28-64.16 24.71-96.05 72.55l-6.31 9.47a7.994 7.994 0 0 1-11.09 2.22l-13.31-8.88a7.994 7.994 0 0 1-2.22-11.09l6.31-9.47c15.73-23.6 30.2-43.26 47.31-58.08-17.27-5.51-31.4-18.12-38.87-34.45-6.59 3.41-13.96 5.52-21.87 5.52h-32c-12.34 0-23.49-4.81-32-12.48C71.48 251.19 60.33 256 48 256H16c-5.64 0-10.97-1.15-16-2.95v77.93c0 33.95 13.48 66.5 37.49 90.51L63.99 448v64h255.98v-63.96l35.91-35.92A96.035 96.035 0 0 0 384 344.21l-.02-88.22zm-32.01-90.09V48c0-8.84-7.16-16-16-16h-32c-8.84 0-16 7.16-16 16v112h32c11.28 0 21.94 2.31 32 5.9zM16 224h32c8.84 0 16-7.16 16-16V80c0-8.84-7.16-16-16-16H16C7.16 64 0 71.16 0 80v128c0 8.84 7.16 16 16 16zm95.99 0h32c8.84 0 16-7.16 16-16V48c0-8.84-7.16-16-16-16h-32c-8.84 0-16 7.16-16 16v160c0 8.84 7.16 16 16 16z"],flag:[512,512,[],"f024","M349.565 98.783C295.978 98.783 251.721 64 184.348 64c-24.955 0-47.309 4.384-68.045 12.013a55.947 55.947 0 0 0 3.586-23.562C118.117 24.015 94.806 1.206 66.338.048 34.345-1.254 8 24.296 8 56c0 19.026 9.497 35.825 24 45.945V488c0 13.255 10.745 24 24 24h16c13.255 0 24-10.745 24-24v-94.4c28.311-12.064 63.582-22.122 114.435-22.122 53.588 0 97.844 34.783 165.217 34.783 48.169 0 86.667-16.294 122.505-40.858C506.84 359.452 512 349.571 512 339.045v-243.1c0-23.393-24.269-38.87-45.485-29.016-34.338 15.948-76.454 31.854-116.95 31.854z"],"flag-checkered":[512,512,[],"f11e","M243.2 189.9V258c26.1 5.9 49.3 15.6 73.6 22.3v-68.2c-26-5.8-49.4-15.5-73.6-22.2zm223.3-123c-34.3 15.9-76.5 31.9-117 31.9C296 98.8 251.7 64 184.3 64c-25 0-47.3 4.4-68 12 2.8-7.3 4.1-15.2 3.6-23.6C118.1 24 94.8 1.2 66.3 0 34.3-1.3 8 24.3 8 56c0 19 9.5 35.8 24 45.9V488c0 13.3 10.7 24 24 24h16c13.3 0 24-10.7 24-24v-94.4c28.3-12.1 63.6-22.1 114.4-22.1 53.6 0 97.8 34.8 165.2 34.8 48.2 0 86.7-16.3 122.5-40.9 8.7-6 13.8-15.8 13.8-26.4V95.9c.1-23.3-24.2-38.8-45.4-29zM169.6 325.5c-25.8 2.7-50 8.2-73.6 16.6v-70.5c26.2-9.3 47.5-15 73.6-17.4zM464 191c-23.6 9.8-46.3 19.5-73.6 23.9V286c24.8-3.4 51.4-11.8 73.6-26v70.5c-25.1 16.1-48.5 24.7-73.6 27.1V286c-27 3.7-47.9 1.5-73.6-5.6v67.4c-23.9-7.4-47.3-16.7-73.6-21.3V258c-19.7-4.4-40.8-6.8-73.6-3.8v-70c-22.4 3.1-44.6 10.2-73.6 20.9v-70.5c33.2-12.2 50.1-19.8 73.6-22v71.6c27-3.7 48.4-1.3 73.6 5.7v-67.4c23.7 7.4 47.2 16.7 73.6 21.3v68.4c23.7 5.3 47.6 6.9 73.6 2.7V143c27-4.8 52.3-13.6 73.6-22.5z"],"flag-usa":[512,512,[],"f74d","M32 0C14.3 0 0 14.3 0 32v464c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V32C64 14.3 49.7 0 32 0zm267.9 303.6c-57.2-15.1-111.7-28.8-203.9 11.1V384c185.7-92.2 221.7 53.3 397.5-23.1 11.4-5 18.5-16.5 18.5-28.8v-36c-43.6 17.3-80.2 24.1-112.1 24.1-37.4-.1-68.9-8.4-100-16.6zm0-96c-57.2-15.1-111.7-28.8-203.9 11.1v61.5c94.8-37.6 154.6-22.7 212.1-7.6 57.2 15.1 111.7 28.8 203.9-11.1V200c-43.6 17.3-80.2 24.1-112.1 24.1-37.4 0-68.9-8.3-100-16.5zm9.5-125.9c51.8 15.6 97.4 29 202.6-20.1V30.8c0-25.1-26.8-38.1-49.4-26.6C291.3 91.5 305.4-62.2 96 32.4v151.9c94.8-37.5 154.6-22.7 212.1-7.6 57.2 15 111.7 28.7 203.9-11.1V96.7c-53.6 23.5-93.3 31.4-126.1 31.4s-59-7.8-85.7-15.9c-4-1.2-8.1-2.4-12.1-3.5V75.5c7.2 2 14.3 4.1 21.3 6.2zM160 128.1c-8.8 0-16-7.1-16-16 0-8.8 7.2-16 16-16s16 7.1 16 16-7.2 16-16 16zm0-55.8c-8.8 0-16-7.1-16-16 0-8.8 7.2-16 16-16s16 7.1 16 16c0 8.8-7.2 16-16 16zm64 47.9c-8.8 0-16-7.1-16-16 0-8.8 7.2-16 16-16s16 7.1 16 16c0 8.8-7.2 16-16 16zm0-55.9c-8.8 0-16-7.1-16-16 0-8.8 7.2-16 16-16s16 7.1 16 16c0 8.8-7.2 16-16 16z"],flask:[448,512,[],"f0c3","M437.2 403.5L320 215V64h8c13.3 0 24-10.7 24-24V24c0-13.3-10.7-24-24-24H120c-13.3 0-24 10.7-24 24v16c0 13.3 10.7 24 24 24h8v151L10.8 403.5C-18.5 450.6 15.3 512 70.9 512h306.2c55.7 0 89.4-61.5 60.1-108.5zM137.9 320l48.2-77.6c3.7-5.2 5.8-11.6 5.8-18.4V64h64v160c0 6.9 2.2 13.2 5.8 18.4l48.2 77.6h-172z"],flushed:[496,512,[],"f579","M344 200c-13.3 0-24 10.7-24 24s10.7 24 24 24 24-10.7 24-24-10.7-24-24-24zm-192 0c-13.3 0-24 10.7-24 24s10.7 24 24 24 24-10.7 24-24-10.7-24-24-24zM248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zM80 224c0-39.8 32.2-72 72-72s72 32.2 72 72-32.2 72-72 72-72-32.2-72-72zm232 176H184c-21.2 0-21.2-32 0-32h128c21.2 0 21.2 32 0 32zm32-104c-39.8 0-72-32.2-72-72s32.2-72 72-72 72 32.2 72 72-32.2 72-72 72z"],folder:[512,512,[],"f07b","M464 128H272l-64-64H48C21.49 64 0 85.49 0 112v288c0 26.51 21.49 48 48 48h416c26.51 0 48-21.49 48-48V176c0-26.51-21.49-48-48-48z"],"folder-minus":[512,512,[],"f65d","M464 128H272l-64-64H48C21.49 64 0 85.49 0 112v288c0 26.51 21.49 48 48 48h416c26.51 0 48-21.49 48-48V176c0-26.51-21.49-48-48-48zm-96 168c0 8.84-7.16 16-16 16H160c-8.84 0-16-7.16-16-16v-16c0-8.84 7.16-16 16-16h192c8.84 0 16 7.16 16 16v16z"],"folder-open":[576,512,[],"f07c","M572.694 292.093L500.27 416.248A63.997 63.997 0 0 1 444.989 448H45.025c-18.523 0-30.064-20.093-20.731-36.093l72.424-124.155A64 64 0 0 1 152 256h399.964c18.523 0 30.064 20.093 20.73 36.093zM152 224h328v-48c0-26.51-21.49-48-48-48H272l-64-64H48C21.49 64 0 85.49 0 112v278.046l69.077-118.418C86.214 242.25 117.989 224 152 224z"],"folder-plus":[512,512,[],"f65e","M464,128H272L208,64H48A48,48,0,0,0,0,112V400a48,48,0,0,0,48,48H464a48,48,0,0,0,48-48V176A48,48,0,0,0,464,128ZM359.5,296a16,16,0,0,1-16,16h-64v64a16,16,0,0,1-16,16h-16a16,16,0,0,1-16-16V312h-64a16,16,0,0,1-16-16V280a16,16,0,0,1,16-16h64V200a16,16,0,0,1,16-16h16a16,16,0,0,1,16,16v64h64a16,16,0,0,1,16,16Z"],font:[448,512,[],"f031","M432 416h-23.41L277.88 53.69A32 32 0 0 0 247.58 32h-47.16a32 32 0 0 0-30.3 21.69L39.41 416H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16h-19.58l23.3-64h152.56l23.3 64H304a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zM176.85 272L224 142.51 271.15 272z"],"font-awesome-logo-full":[3992,512,["Font Awesome"],"f4e6","M454.6 0H57.4C25.9 0 0 25.9 0 57.4v397.3C0 486.1 25.9 512 57.4 512h397.3c31.4 0 57.4-25.9 57.4-57.4V57.4C512 25.9 486.1 0 454.6 0zm-58.9 324.9c0 4.8-4.1 6.9-8.9 8.9-19.2 8.1-39.7 15.7-61.5 15.7-40.5 0-68.7-44.8-163.2 2.5v51.8c0 30.3-45.7 30.2-45.7 0v-250c-9-7-15-17.9-15-30.3 0-21 17.1-38.2 38.2-38.2 21 0 38.2 17.1 38.2 38.2 0 12.2-5.8 23.2-14.9 30.2v21c37.1-12 65.5-34.4 146.1-3.4 26.6 11.4 68.7-15.7 76.5-15.7 5.5 0 10.3 4.1 10.3 8.9v160.4zm432.9-174.2h-137v70.1H825c39.8 0 40.4 62.2 0 62.2H691.6v105.6c0 45.5-70.7 46.4-70.7 0V128.3c0-22 18-39.8 39.8-39.8h167.8c39.6 0 40.5 62.2.1 62.2zm191.1 23.4c-169.3 0-169.1 252.4 0 252.4 169.9 0 169.9-252.4 0-252.4zm0 196.1c-81.6 0-82.1-139.8 0-139.8 82.5 0 82.4 139.8 0 139.8zm372.4 53.4c-17.5 0-31.4-13.9-31.4-31.4v-117c0-62.4-72.6-52.5-99.1-16.4v133.4c0 41.5-63.3 41.8-63.3 0V208c0-40 63.1-41.6 63.1 0v3.4c43.3-51.6 162.4-60.4 162.4 39.3v141.5c.3 30.4-31.5 31.4-31.7 31.4zm179.7 2.9c-44.3 0-68.3-22.9-68.3-65.8V235.2H1488c-35.6 0-36.7-55.3 0-55.3h15.5v-37.3c0-41.3 63.8-42.1 63.8 0v37.5h24.9c35.4 0 35.7 55.3 0 55.3h-24.9v108.5c0 29.6 26.1 26.3 27.4 26.3 31.4 0 52.6 56.3-22.9 56.3zM1992 123c-19.5-50.2-95.5-50-114.5 0-107.3 275.7-99.5 252.7-99.5 262.8 0 42.8 58.3 51.2 72.1 14.4l13.5-35.9H2006l13 35.9c14.2 37.7 72.1 27.2 72.1-14.4 0-10.1 5.3 6.8-99.1-262.8zm-108.9 179.1l51.7-142.9 51.8 142.9h-103.5zm591.3-85.6l-53.7 176.3c-12.4 41.2-72 41-84 0l-42.3-135.9-42.3 135.9c-12.4 40.9-72 41.2-84.5 0l-54.2-176.3c-12.5-39.4 49.8-56.1 60.2-16.9L2213 342l45.3-139.5c10.9-32.7 59.6-34.7 71.2 0l45.3 139.5 39.3-142.4c10.3-38.3 72.6-23.8 60.3 16.9zm275.4 75.1c0-42.4-33.9-117.5-119.5-117.5-73.2 0-124.4 56.3-124.4 126 0 77.2 55.3 126.4 128.5 126.4 31.7 0 93-11.5 93-39.8 0-18.3-21.1-31.5-39.3-22.4-49.4 26.2-109 8.4-115.9-43.8h148.3c16.3 0 29.3-13.4 29.3-28.9zM2571 277.7c9.5-73.4 113.9-68.6 118.6 0H2571zm316.7 148.8c-31.4 0-81.6-10.5-96.6-31.9-12.4-17 2.5-39.8 21.8-39.8 16.3 0 36.8 22.9 77.7 22.9 27.4 0 40.4-11 40.4-25.8 0-39.8-142.9-7.4-142.9-102 0-40.4 35.3-75.7 98.6-75.7 31.4 0 74.1 9.9 87.6 29.4 10.8 14.8-1.4 36.2-20.9 36.2-15.1 0-26.7-17.3-66.2-17.3-22.9 0-37.8 10.5-37.8 23.8 0 35.9 142.4 6 142.4 103.1-.1 43.7-37.4 77.1-104.1 77.1zm266.8-252.4c-169.3 0-169.1 252.4 0 252.4 170.1 0 169.6-252.4 0-252.4zm0 196.1c-81.8 0-82-139.8 0-139.8 82.5 0 82.4 139.8 0 139.8zm476.9 22V268.7c0-53.8-61.4-45.8-85.7-10.5v134c0 41.3-63.8 42.1-63.8 0V268.7c0-52.1-59.5-47.4-85.7-10.1v133.6c0 41.5-63.3 41.8-63.3 0V208c0-40 63.1-41.6 63.1 0v3.4c9.9-14.4 41.8-37.3 78.6-37.3 35.3 0 57.7 16.4 66.7 43.8 13.9-21.8 45.8-43.8 82.6-43.8 44.3 0 70.7 23.4 70.7 72.7v145.3c.5 17.3-13.5 31.4-31.9 31.4 3.5.1-31.3 1.1-31.3-31.3zM3992 291.6c0-42.4-32.4-117.5-117.9-117.5-73.2 0-127.5 56.3-127.5 126 0 77.2 58.3 126.4 131.6 126.4 31.7 0 91.5-11.5 91.5-39.8 0-18.3-21.1-31.5-39.3-22.4-49.4 26.2-110.5 8.4-117.5-43.8h149.8c16.3 0 29.1-13.4 29.3-28.9zm-180.5-13.9c9.7-74.4 115.9-68.3 120.1 0h-120.1z"],"football-ball":[496,512,[],"f44e","M481.5 60.3c-4.8-18.2-19.1-32.5-37.3-37.4C420.3 16.5 383 8.9 339.4 8L496 164.8c-.8-43.5-8.2-80.6-14.5-104.5zm-467 391.4c4.8 18.2 19.1 32.5 37.3 37.4 23.9 6.4 61.2 14 104.8 14.9L0 347.2c.8 43.5 8.2 80.6 14.5 104.5zM4.2 283.4L220.4 500c132.5-19.4 248.8-118.7 271.5-271.4L275.6 12C143.1 31.4 26.8 130.7 4.2 283.4zm317.3-123.6c3.1-3.1 8.2-3.1 11.3 0l11.3 11.3c3.1 3.1 3.1 8.2 0 11.3l-28.3 28.3 28.3 28.3c3.1 3.1 3.1 8.2 0 11.3l-11.3 11.3c-3.1 3.1-8.2 3.1-11.3 0l-28.3-28.3-22.6 22.7 28.3 28.3c3.1 3.1 3.1 8.2 0 11.3l-11.3 11.3c-3.1 3.1-8.2 3.1-11.3 0L248 278.6l-22.6 22.6 28.3 28.3c3.1 3.1 3.1 8.2 0 11.3l-11.3 11.3c-3.1 3.1-8.2 3.1-11.3 0l-28.3-28.3-28.3 28.3c-3.1 3.1-8.2 3.1-11.3 0l-11.3-11.3c-3.1-3.1-3.1-8.2 0-11.3l28.3-28.3-28.3-28.2c-3.1-3.1-3.1-8.2 0-11.3l11.3-11.3c3.1-3.1 8.2-3.1 11.3 0l28.3 28.3 22.6-22.6-28.3-28.3c-3.1-3.1-3.1-8.2 0-11.3l11.3-11.3c3.1-3.1 8.2-3.1 11.3 0l28.3 28.3 22.6-22.6-28.3-28.3c-3.1-3.1-3.1-8.2 0-11.3l11.3-11.3c3.1-3.1 8.2-3.1 11.3 0l28.3 28.3 28.3-28.5z"],forward:[512,512,[],"f04e","M500.5 231.4l-192-160C287.9 54.3 256 68.6 256 96v320c0 27.4 31.9 41.8 52.5 24.6l192-160c15.3-12.8 15.3-36.4 0-49.2zm-256 0l-192-160C31.9 54.3 0 68.6 0 96v320c0 27.4 31.9 41.8 52.5 24.6l192-160c15.3-12.8 15.3-36.4 0-49.2z"],frog:[576,512,[],"f52e","M446.53 97.43C439.67 60.23 407.19 32 368 32c-39.23 0-71.72 28.29-78.54 65.54C126.75 112.96-.5 250.12 0 416.98.11 451.9 29.08 480 64 480h304c8.84 0 16-7.16 16-16 0-17.67-14.33-32-32-32h-79.49l35.8-48.33c24.14-36.23 10.35-88.28-33.71-106.6-23.89-9.93-51.55-4.65-72.24 10.88l-32.76 24.59c-7.06 5.31-17.09 3.91-22.41-3.19-5.3-7.08-3.88-17.11 3.19-22.41l34.78-26.09c36.84-27.66 88.28-27.62 125.13 0 10.87 8.15 45.87 39.06 40.8 93.21L469.62 480H560c8.84 0 16-7.16 16-16 0-17.67-14.33-32-32-32h-53.63l-98.52-104.68 154.44-86.65A58.16 58.16 0 0 0 576 189.94c0-21.4-11.72-40.95-30.48-51.23-40.56-22.22-98.99-41.28-98.99-41.28zM368 136c-13.26 0-24-10.75-24-24 0-13.26 10.74-24 24-24 13.25 0 24 10.74 24 24 0 13.25-10.75 24-24 24z"],frown:[496,512,[],"f119","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm80 168c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zm-160 0c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zm170.2 218.2C315.8 367.4 282.9 352 248 352s-67.8 15.4-90.2 42.2c-13.5 16.3-38.1-4.2-24.6-20.5C161.7 339.6 203.6 320 248 320s86.3 19.6 114.7 53.8c13.6 16.2-11 36.7-24.5 20.4z"],"frown-open":[496,512,[],"f57a","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zM136 208c0-17.7 14.3-32 32-32s32 14.3 32 32-14.3 32-32 32-32-14.3-32-32zm187.3 183.3c-31.2-9.6-59.4-15.3-75.3-15.3s-44.1 5.7-75.3 15.3c-11.5 3.5-22.5-6.3-20.5-18.1 7-40 60.1-61.2 95.8-61.2s88.8 21.3 95.8 61.2c2 11.9-9.1 21.6-20.5 18.1zM328 240c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],"funnel-dollar":[640,512,[],"f662","M433.46 165.94l101.2-111.87C554.61 34.12 540.48 0 512.26 0H31.74C3.52 0-10.61 34.12 9.34 54.07L192 256v155.92c0 12.59 5.93 24.44 16 32l79.99 60c20.86 15.64 48.47 6.97 59.22-13.57C310.8 455.38 288 406.35 288 352c0-89.79 62.05-165.17 145.46-186.06zM480 192c-88.37 0-160 71.63-160 160s71.63 160 160 160 160-71.63 160-160-71.63-160-160-160zm16 239.88V448c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8v-16.29c-11.29-.58-22.27-4.52-31.37-11.35-3.9-2.93-4.1-8.77-.57-12.14l11.75-11.21c2.77-2.64 6.89-2.76 10.13-.73 3.87 2.42 8.26 3.72 12.82 3.72h28.11c6.5 0 11.8-5.92 11.8-13.19 0-5.95-3.61-11.19-8.77-12.73l-45-13.5c-18.59-5.58-31.58-23.42-31.58-43.39 0-24.52 19.05-44.44 42.67-45.07V256c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v16.29c11.29.58 22.27 4.51 31.37 11.35 3.9 2.93 4.1 8.77.57 12.14l-11.75 11.21c-2.77 2.64-6.89 2.76-10.13.73-3.87-2.43-8.26-3.72-12.82-3.72h-28.11c-6.5 0-11.8 5.92-11.8 13.19 0 5.95 3.61 11.19 8.77 12.73l45 13.5c18.59 5.58 31.58 23.42 31.58 43.39 0 24.53-19.04 44.44-42.67 45.07z"],futbol:[512,512,[],"f1e3","M504 256c0 136.967-111.033 248-248 248S8 392.967 8 256 119.033 8 256 8s248 111.033 248 248zm-48 0l-.003-.282-26.064 22.741-62.679-58.5 16.454-84.355 34.303 3.072c-24.889-34.216-60.004-60.089-100.709-73.141l13.651 31.939L256 139l-74.953-41.525 13.651-31.939c-40.631 13.028-75.78 38.87-100.709 73.141l34.565-3.073 16.192 84.355-62.678 58.5-26.064-22.741-.003.282c0 43.015 13.497 83.952 38.472 117.991l7.704-33.897 85.138 10.447 36.301 77.826-29.902 17.786c40.202 13.122 84.29 13.148 124.572 0l-29.902-17.786 36.301-77.826 85.138-10.447 7.704 33.897C442.503 339.952 456 299.015 456 256zm-248.102 69.571l-29.894-91.312L256 177.732l77.996 56.527-29.622 91.312h-96.476z"],gamepad:[640,512,[],"f11b","M480.07 96H160a160 160 0 1 0 114.24 272h91.52A160 160 0 1 0 480.07 96zM248 268a12 12 0 0 1-12 12h-52v52a12 12 0 0 1-12 12h-24a12 12 0 0 1-12-12v-52H84a12 12 0 0 1-12-12v-24a12 12 0 0 1 12-12h52v-52a12 12 0 0 1 12-12h24a12 12 0 0 1 12 12v52h52a12 12 0 0 1 12 12zm216 76a40 40 0 1 1 40-40 40 40 0 0 1-40 40zm64-96a40 40 0 1 1 40-40 40 40 0 0 1-40 40z"],"gas-pump":[512,512,[],"f52f","M336 448H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h320c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zm157.2-340.7l-81-81c-6.2-6.2-16.4-6.2-22.6 0l-11.3 11.3c-6.2 6.2-6.2 16.4 0 22.6L416 97.9V160c0 28.1 20.9 51.3 48 55.2V376c0 13.2-10.8 24-24 24s-24-10.8-24-24v-32c0-48.6-39.4-88-88-88h-8V64c0-35.3-28.7-64-64-64H96C60.7 0 32 28.7 32 64v352h288V304h8c22.1 0 40 17.9 40 40v27.8c0 37.7 27 72 64.5 75.9 43 4.3 79.5-29.5 79.5-71.7V152.6c0-17-6.8-33.3-18.8-45.3zM256 192H96V64h160v128z"],gavel:[512,512,[],"f0e3","M504.971 199.362l-22.627-22.627c-9.373-9.373-24.569-9.373-33.941 0l-5.657 5.657L329.608 69.255l5.657-5.657c9.373-9.373 9.373-24.569 0-33.941L312.638 7.029c-9.373-9.373-24.569-9.373-33.941 0L154.246 131.48c-9.373 9.373-9.373 24.569 0 33.941l22.627 22.627c9.373 9.373 24.569 9.373 33.941 0l5.657-5.657 39.598 39.598-81.04 81.04-5.657-5.657c-12.497-12.497-32.758-12.497-45.255 0L9.373 412.118c-12.497 12.497-12.497 32.758 0 45.255l45.255 45.255c12.497 12.497 32.758 12.497 45.255 0l114.745-114.745c12.497-12.497 12.497-32.758 0-45.255l-5.657-5.657 81.04-81.04 39.598 39.598-5.657 5.657c-9.373 9.373-9.373 24.569 0 33.941l22.627 22.627c9.373 9.373 24.569 9.373 33.941 0l124.451-124.451c9.372-9.372 9.372-24.568 0-33.941z"],gem:[576,512,[],"f3a5","M485.5 0L576 160H474.9L405.7 0h79.8zm-128 0l69.2 160H149.3L218.5 0h139zm-267 0h79.8l-69.2 160H0L90.5 0zM0 192h100.7l123 251.7c1.5 3.1-2.7 5.9-5 3.3L0 192zm148.2 0h279.6l-137 318.2c-1 2.4-4.5 2.4-5.5 0L148.2 192zm204.1 251.7l123-251.7H576L357.3 446.9c-2.3 2.7-6.5-.1-5-3.2z"],genderless:[288,512,[],"f22d","M144 176c44.1 0 80 35.9 80 80s-35.9 80-80 80-80-35.9-80-80 35.9-80 80-80m0-64C64.5 112 0 176.5 0 256s64.5 144 144 144 144-64.5 144-144-64.5-144-144-144z"],ghost:[384,512,[],"f6e2","M186.1.09C81.01 3.24 0 94.92 0 200.05v263.92c0 14.26 17.23 21.39 27.31 11.31l24.92-18.53c6.66-4.95 16-3.99 21.51 2.21l42.95 48.35c6.25 6.25 16.38 6.25 22.63 0l40.72-45.85c6.37-7.17 17.56-7.17 23.92 0l40.72 45.85c6.25 6.25 16.38 6.25 22.63 0l42.95-48.35c5.51-6.2 14.85-7.17 21.51-2.21l24.92 18.53c10.08 10.08 27.31 2.94 27.31-11.31V192C384 84 294.83-3.17 186.1.09zM128 224c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm128 0c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],gift:[512,512,[],"f06b","M32 448c0 17.7 14.3 32 32 32h160V320H32v128zm256 32h160c17.7 0 32-14.3 32-32V320H288v160zm192-320h-42.1c6.2-12.1 10.1-25.5 10.1-40 0-48.5-39.5-88-88-88-41.6 0-68.5 21.3-103 68.3-34.5-47-61.4-68.3-103-68.3-48.5 0-88 39.5-88 88 0 14.5 3.8 27.9 10.1 40H32c-17.7 0-32 14.3-32 32v80c0 8.8 7.2 16 16 16h480c8.8 0 16-7.2 16-16v-80c0-17.7-14.3-32-32-32zm-326.1 0c-22.1 0-40-17.9-40-40s17.9-40 40-40c19.9 0 34.6 3.3 86.1 80h-86.1zm206.1 0h-86.1c51.4-76.5 65.7-80 86.1-80 22.1 0 40 17.9 40 40s-17.9 40-40 40z"],gifts:[640,512,[],"f79c","M240.6 194.1c1.9-30.8 17.3-61.2 44-79.8C279.4 103.5 268.7 96 256 96h-29.4l30.7-22c7.2-5.1 8.9-15.1 3.7-22.3l-9.3-13c-5.1-7.2-15.1-8.9-22.3-3.7l-32 22.9 11.5-30.6c3.1-8.3-1.1-17.5-9.4-20.6l-15-5.6c-8.3-3.1-17.5 1.1-20.6 9.4l-19.9 53-19.9-53.1C121 2.1 111.8-2.1 103.5 1l-15 5.6C80.2 9.7 76 19 79.2 27.2l11.5 30.6L58.6 35c-7.2-5.1-17.2-3.5-22.3 3.7l-9.3 13c-5.1 7.2-3.5 17.2 3.7 22.3l30.7 22H32c-17.7 0-32 14.3-32 32v352c0 17.7 14.3 32 32 32h168.9c-5.5-9.5-8.9-20.3-8.9-32V256c0-29.9 20.8-55 48.6-61.9zM224 480c0 17.7 14.3 32 32 32h160V384H224v96zm224 32h160c17.7 0 32-14.3 32-32v-96H448v128zm160-288h-20.4c2.6-7.6 4.4-15.5 4.4-23.8 0-35.5-27-72.2-72.1-72.2-48.1 0-75.9 47.7-87.9 75.3-12.1-27.6-39.9-75.3-87.9-75.3-45.1 0-72.1 36.7-72.1 72.2 0 8.3 1.7 16.2 4.4 23.8H256c-17.7 0-32 14.3-32 32v96h192V224h15.3l.7-.2.7.2H448v128h192v-96c0-17.7-14.3-32-32-32zm-272 0c-2.7-1.4-5.1-3-7.2-4.8-7.3-6.4-8.8-13.8-8.8-19 0-9.7 6.4-24.2 24.1-24.2 18.7 0 35.6 27.4 44.5 48H336zm199.2-4.8c-2.1 1.8-4.5 3.4-7.2 4.8h-52.6c8.8-20.3 25.8-48 44.5-48 17.7 0 24.1 14.5 24.1 24.2 0 5.2-1.5 12.6-8.8 19z"],"glass-cheers":[640,512,[],"f79f","M639.4 433.6c-8.4-20.4-31.8-30.1-52.2-21.6l-22.1 9.2-38.7-101.9c47.9-35 64.8-100.3 34.5-152.8L474.3 16c-8-13.9-25.1-19.7-40-13.6L320 49.8 205.7 2.4c-14.9-6.2-32-.3-40 13.6L79.1 166.5C48.9 219 65.7 284.3 113.6 319.2L74.9 421.1l-22.1-9.2c-20.4-8.5-43.7 1.2-52.2 21.6-1.7 4.1.2 8.8 4.3 10.5l162.3 67.4c4.1 1.7 8.7-.2 10.4-4.3 8.4-20.4-1.2-43.8-21.6-52.3l-22.1-9.2L173.3 342c4.4.5 8.8 1.3 13.1 1.3 51.7 0 99.4-33.1 113.4-85.3l20.2-75.4 20.2 75.4c14 52.2 61.7 85.3 113.4 85.3 4.3 0 8.7-.8 13.1-1.3L506 445.6l-22.1 9.2c-20.4 8.5-30.1 31.9-21.6 52.3 1.7 4.1 6.4 6 10.4 4.3L635.1 444c4-1.7 6-6.3 4.3-10.4zM275.9 162.1l-112.1-46.5 36.5-63.4 94.5 39.2-18.9 70.7zm88.2 0l-18.9-70.7 94.5-39.2 36.5 63.4-112.1 46.5z"],"glass-martini":[512,512,[],"f000","M502.05 57.6C523.3 36.34 508.25 0 478.2 0H33.8C3.75 0-11.3 36.34 9.95 57.6L224 271.64V464h-56c-22.09 0-40 17.91-40 40 0 4.42 3.58 8 8 8h240c4.42 0 8-3.58 8-8 0-22.09-17.91-40-40-40h-56V271.64L502.05 57.6z"],"glass-martini-alt":[512,512,[],"f57b","M502.05 57.6C523.3 36.34 508.25 0 478.2 0H33.8C3.75 0-11.3 36.34 9.95 57.6L224 271.64V464h-56c-22.09 0-40 17.91-40 40 0 4.42 3.58 8 8 8h240c4.42 0 8-3.58 8-8 0-22.09-17.91-40-40-40h-56V271.64L502.05 57.6zM443.77 48l-48 48H116.24l-48-48h375.53z"],"glass-whiskey":[512,512,[],"f7a0","M480 32H32C12.5 32-2.4 49.2.3 68.5l56 356.5c4.5 31.5 31.5 54.9 63.4 54.9h273c31.8 0 58.9-23.4 63.4-54.9l55.6-356.5C514.4 49.2 499.5 32 480 32zm-37.4 64l-30 192h-313L69.4 96h373.2z"],glasses:[576,512,[],"f530","M574.1 280.37L528.75 98.66c-5.91-23.7-21.59-44.05-43-55.81-21.44-11.73-46.97-14.11-70.19-6.33l-15.25 5.08c-8.39 2.79-12.92 11.86-10.12 20.24l5.06 15.18c2.79 8.38 11.85 12.91 20.23 10.12l13.18-4.39c10.87-3.62 23-3.57 33.16 1.73 10.29 5.37 17.57 14.56 20.37 25.82l38.46 153.82c-22.19-6.81-49.79-12.46-81.2-12.46-34.77 0-73.98 7.02-114.85 26.74h-73.18c-40.87-19.74-80.08-26.75-114.86-26.75-31.42 0-59.02 5.65-81.21 12.46l38.46-153.83c2.79-11.25 10.09-20.45 20.38-25.81 10.16-5.3 22.28-5.35 33.15-1.73l13.17 4.39c8.38 2.79 17.44-1.74 20.23-10.12l5.06-15.18c2.8-8.38-1.73-17.45-10.12-20.24l-15.25-5.08c-23.22-7.78-48.75-5.41-70.19 6.33-21.41 11.77-37.09 32.11-43 55.8L1.9 280.37A64.218 64.218 0 0 0 0 295.86v70.25C0 429.01 51.58 480 115.2 480h37.12c60.28 0 110.37-45.94 114.88-105.37l2.93-38.63h35.75l2.93 38.63C313.31 434.06 363.4 480 423.68 480h37.12c63.62 0 115.2-50.99 115.2-113.88v-70.25c0-5.23-.64-10.43-1.9-15.5zm-370.72 89.42c-1.97 25.91-24.4 46.21-51.06 46.21H115.2C86.97 416 64 393.62 64 366.11v-37.54c18.12-6.49 43.42-12.92 72.58-12.92 23.86 0 47.26 4.33 69.93 12.92l-3.13 41.22zM512 366.12c0 27.51-22.97 49.88-51.2 49.88h-37.12c-26.67 0-49.1-20.3-51.06-46.21l-3.13-41.22c22.67-8.59 46.08-12.92 69.95-12.92 29.12 0 54.43 6.44 72.55 12.93v37.54z"],globe:[496,512,[],"f0ac","M336.5 160C322 70.7 287.8 8 248 8s-74 62.7-88.5 152h177zM152 256c0 22.2 1.2 43.5 3.3 64h185.3c2.1-20.5 3.3-41.8 3.3-64s-1.2-43.5-3.3-64H155.3c-2.1 20.5-3.3 41.8-3.3 64zm324.7-96c-28.6-67.9-86.5-120.4-158-141.6 24.4 33.8 41.2 84.7 50 141.6h108zM177.2 18.4C105.8 39.6 47.8 92.1 19.3 160h108c8.7-56.9 25.5-107.8 49.9-141.6zM487.4 192H372.7c2.1 21 3.3 42.5 3.3 64s-1.2 43-3.3 64h114.6c5.5-20.5 8.6-41.8 8.6-64s-3.1-43.5-8.5-64zM120 256c0-21.5 1.2-43 3.3-64H8.6C3.2 212.5 0 233.8 0 256s3.2 43.5 8.6 64h114.6c-2-21-3.2-42.5-3.2-64zm39.5 96c14.5 89.3 48.7 152 88.5 152s74-62.7 88.5-152h-177zm159.3 141.6c71.4-21.2 129.4-73.7 158-141.6h-108c-8.8 56.9-25.6 107.8-50 141.6zM19.3 352c28.6 67.9 86.5 120.4 158 141.6-24.4-33.8-41.2-84.7-50-141.6h-108z"],"globe-africa":[496,512,[],"f57c","M248 8C111.03 8 0 119.03 0 256s111.03 248 248 248 248-111.03 248-248S384.97 8 248 8zm160 215.5v6.93c0 5.87-3.32 11.24-8.57 13.86l-15.39 7.7a15.485 15.485 0 0 1-15.53-.97l-18.21-12.14a15.52 15.52 0 0 0-13.5-1.81l-2.65.88c-9.7 3.23-13.66 14.79-7.99 23.3l13.24 19.86c2.87 4.31 7.71 6.9 12.89 6.9h8.21c8.56 0 15.5 6.94 15.5 15.5v11.34c0 3.35-1.09 6.62-3.1 9.3l-18.74 24.98c-1.42 1.9-2.39 4.1-2.83 6.43l-4.3 22.83c-.62 3.29-2.29 6.29-4.76 8.56a159.608 159.608 0 0 0-25 29.16l-13.03 19.55a27.756 27.756 0 0 1-23.09 12.36c-10.51 0-20.12-5.94-24.82-15.34a78.902 78.902 0 0 1-8.33-35.29V367.5c0-8.56-6.94-15.5-15.5-15.5h-25.88c-14.49 0-28.38-5.76-38.63-16a54.659 54.659 0 0 1-16-38.63v-14.06c0-17.19 8.1-33.38 21.85-43.7l27.58-20.69a54.663 54.663 0 0 1 32.78-10.93h.89c8.48 0 16.85 1.97 24.43 5.77l14.72 7.36c3.68 1.84 7.93 2.14 11.83.84l47.31-15.77c6.33-2.11 10.6-8.03 10.6-14.7 0-8.56-6.94-15.5-15.5-15.5h-10.09c-4.11 0-8.05-1.63-10.96-4.54l-6.92-6.92a15.493 15.493 0 0 0-10.96-4.54H199.5c-8.56 0-15.5-6.94-15.5-15.5v-4.4c0-7.11 4.84-13.31 11.74-15.04l14.45-3.61c3.74-.94 7-3.23 9.14-6.44l8.08-12.11c2.87-4.31 7.71-6.9 12.89-6.9h24.21c8.56 0 15.5-6.94 15.5-15.5v-21.7C359.23 71.63 422.86 131.02 441.93 208H423.5c-8.56 0-15.5 6.94-15.5 15.5z"],"globe-americas":[496,512,[],"f57d","M248 8C111.03 8 0 119.03 0 256s111.03 248 248 248 248-111.03 248-248S384.97 8 248 8zm82.29 357.6c-3.9 3.88-7.99 7.95-11.31 11.28-2.99 3-5.1 6.7-6.17 10.71-1.51 5.66-2.73 11.38-4.77 16.87l-17.39 46.85c-13.76 3-28 4.69-42.65 4.69v-27.38c1.69-12.62-7.64-36.26-22.63-51.25-6-6-9.37-14.14-9.37-22.63v-32.01c0-11.64-6.27-22.34-16.46-27.97-14.37-7.95-34.81-19.06-48.81-26.11-11.48-5.78-22.1-13.14-31.65-21.75l-.8-.72a114.792 114.792 0 0 1-18.06-20.74c-9.38-13.77-24.66-36.42-34.59-51.14 20.47-45.5 57.36-82.04 103.2-101.89l24.01 12.01C203.48 89.74 216 82.01 216 70.11v-11.3c7.99-1.29 16.12-2.11 24.39-2.42l28.3 28.3c6.25 6.25 6.25 16.38 0 22.63L264 112l-10.34 10.34c-3.12 3.12-3.12 8.19 0 11.31l4.69 4.69c3.12 3.12 3.12 8.19 0 11.31l-8 8a8.008 8.008 0 0 1-5.66 2.34h-8.99c-2.08 0-4.08.81-5.58 2.27l-9.92 9.65a8.008 8.008 0 0 0-1.58 9.31l15.59 31.19c2.66 5.32-1.21 11.58-7.15 11.58h-5.64c-1.93 0-3.79-.7-5.24-1.96l-9.28-8.06a16.017 16.017 0 0 0-15.55-3.1l-31.17 10.39a11.95 11.95 0 0 0-8.17 11.34c0 4.53 2.56 8.66 6.61 10.69l11.08 5.54c9.41 4.71 19.79 7.16 30.31 7.16s22.59 27.29 32 32h66.75c8.49 0 16.62 3.37 22.63 9.37l13.69 13.69a30.503 30.503 0 0 1 8.93 21.57 46.536 46.536 0 0 1-13.72 32.98zM417 274.25c-5.79-1.45-10.84-5-14.15-9.97l-17.98-26.97a23.97 23.97 0 0 1 0-26.62l19.59-29.38c2.32-3.47 5.5-6.29 9.24-8.15l12.98-6.49C440.2 193.59 448 223.87 448 256c0 8.67-.74 17.16-1.82 25.54L417 274.25z"],"globe-asia":[496,512,[],"f57e","M248 8C111.03 8 0 119.03 0 256s111.03 248 248 248 248-111.03 248-248S384.97 8 248 8zm-11.34 240.23c-2.89 4.82-8.1 7.77-13.72 7.77h-.31c-4.24 0-8.31 1.69-11.31 4.69l-5.66 5.66c-3.12 3.12-3.12 8.19 0 11.31l5.66 5.66c3 3 4.69 7.07 4.69 11.31V304c0 8.84-7.16 16-16 16h-6.11c-6.06 0-11.6-3.42-14.31-8.85l-22.62-45.23c-2.44-4.88-8.95-5.94-12.81-2.08l-19.47 19.46c-3 3-7.07 4.69-11.31 4.69H50.81C49.12 277.55 48 266.92 48 256c0-110.28 89.72-200 200-200 21.51 0 42.2 3.51 61.63 9.82l-50.16 38.53c-5.11 3.41-4.63 11.06.86 13.81l10.83 5.41c5.42 2.71 8.84 8.25 8.84 14.31V216c0 4.42-3.58 8-8 8h-3.06c-3.03 0-5.8-1.71-7.15-4.42-1.56-3.12-5.96-3.29-7.76-.3l-17.37 28.95zM408 358.43c0 4.24-1.69 8.31-4.69 11.31l-9.57 9.57c-3 3-7.07 4.69-11.31 4.69h-15.16c-4.24 0-8.31-1.69-11.31-4.69l-13.01-13.01a26.767 26.767 0 0 0-25.42-7.04l-21.27 5.32c-1.27.32-2.57.48-3.88.48h-10.34c-4.24 0-8.31-1.69-11.31-4.69l-11.91-11.91a8.008 8.008 0 0 1-2.34-5.66v-10.2c0-3.27 1.99-6.21 5.03-7.43l39.34-15.74c1.98-.79 3.86-1.82 5.59-3.05l23.71-16.89a7.978 7.978 0 0 1 4.64-1.48h12.09c3.23 0 6.15 1.94 7.39 4.93l5.35 12.85a4 4 0 0 0 3.69 2.46h3.8c1.78 0 3.35-1.18 3.84-2.88l4.2-14.47c.5-1.71 2.06-2.88 3.84-2.88h6.06c2.21 0 4 1.79 4 4v12.93c0 2.12.84 4.16 2.34 5.66l11.91 11.91c3 3 4.69 7.07 4.69 11.31v24.6z"],"globe-europe":[496,512,[],"f7a2","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm200 248c0 22.5-3.9 44.2-10.8 64.4h-20.3c-4.3 0-8.4-1.7-11.4-4.8l-32-32.6c-4.5-4.6-4.5-12.1.1-16.7l12.5-12.5v-8.7c0-3-1.2-5.9-3.3-8l-9.4-9.4c-2.1-2.1-5-3.3-8-3.3h-16c-6.2 0-11.3-5.1-11.3-11.3 0-3 1.2-5.9 3.3-8l9.4-9.4c2.1-2.1 5-3.3 8-3.3h32c6.2 0 11.3-5.1 11.3-11.3v-9.4c0-6.2-5.1-11.3-11.3-11.3h-36.7c-8.8 0-16 7.2-16 16v4.5c0 6.9-4.4 13-10.9 15.2l-31.6 10.5c-3.3 1.1-5.5 4.1-5.5 7.6v2.2c0 4.4-3.6 8-8 8h-16c-4.4 0-8-3.6-8-8s-3.6-8-8-8H247c-3 0-5.8 1.7-7.2 4.4l-9.4 18.7c-2.7 5.4-8.2 8.8-14.3 8.8H194c-8.8 0-16-7.2-16-16V199c0-4.2 1.7-8.3 4.7-11.3l20.1-20.1c4.6-4.6 7.2-10.9 7.2-17.5 0-3.4 2.2-6.5 5.5-7.6l40-13.3c1.7-.6 3.2-1.5 4.4-2.7l26.8-26.8c2.1-2.1 3.3-5 3.3-8 0-6.2-5.1-11.3-11.3-11.3H258l-16 16v8c0 4.4-3.6 8-8 8h-16c-4.4 0-8-3.6-8-8v-20c0-2.5 1.2-4.9 3.2-6.4l28.9-21.7c1.9-.1 3.8-.3 5.7-.3C358.3 56 448 145.7 448 256zM130.1 149.1c0-3 1.2-5.9 3.3-8l25.4-25.4c2.1-2.1 5-3.3 8-3.3 6.2 0 11.3 5.1 11.3 11.3v16c0 3-1.2 5.9-3.3 8l-9.4 9.4c-2.1 2.1-5 3.3-8 3.3h-16c-6.2 0-11.3-5.1-11.3-11.3zm128 306.4v-7.1c0-8.8-7.2-16-16-16h-20.2c-10.8 0-26.7-5.3-35.4-11.8l-22.2-16.7c-11.5-8.6-18.2-22.1-18.2-36.4v-23.9c0-16 8.4-30.8 22.1-39l42.9-25.7c7.1-4.2 15.2-6.5 23.4-6.5h31.2c10.9 0 21.4 3.9 29.6 10.9l43.2 37.1h18.3c8.5 0 16.6 3.4 22.6 9.4l17.3 17.3c3.4 3.4 8.1 5.3 12.9 5.3H423c-32.4 58.9-93.8 99.5-164.9 103.1z"],"golf-ball":[416,512,[],"f450","M96 416h224c0 17.7-14.3 32-32 32h-16c-17.7 0-32 14.3-32 32v20c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-20c0-17.7-14.3-32-32-32h-16c-17.7 0-32-14.3-32-32zm320-208c0 74.2-39 139.2-97.5 176h-221C39 347.2 0 282.2 0 208 0 93.1 93.1 0 208 0s208 93.1 208 208zm-180.1 43.9c18.3 0 33.1-14.8 33.1-33.1 0-14.4-9.3-26.3-22.1-30.9 9.6 26.8-15.6 51.3-41.9 41.9 4.6 12.8 16.5 22.1 30.9 22.1zm49.1 46.9c0-14.4-9.3-26.3-22.1-30.9 9.6 26.8-15.6 51.3-41.9 41.9 4.6 12.8 16.5 22.1 30.9 22.1 18.3 0 33.1-14.9 33.1-33.1zm64-64c0-14.4-9.3-26.3-22.1-30.9 9.6 26.8-15.6 51.3-41.9 41.9 4.6 12.8 16.5 22.1 30.9 22.1 18.3 0 33.1-14.9 33.1-33.1z"],gopuram:[512,512,[],"f664","M496 352h-16V240c0-8.8-7.2-16-16-16h-16v-80c0-8.8-7.2-16-16-16h-16V16c0-8.8-7.2-16-16-16s-16 7.2-16 16v16h-64V16c0-8.8-7.2-16-16-16s-16 7.2-16 16v16h-64V16c0-8.8-7.2-16-16-16s-16 7.2-16 16v16h-64V16c0-8.8-7.2-16-16-16S96 7.2 96 16v112H80c-8.8 0-16 7.2-16 16v80H48c-8.8 0-16 7.2-16 16v112H16c-8.8 0-16 7.2-16 16v128c0 8.8 7.2 16 16 16h80V352h32V224h32v-96h32v96h-32v128h-32v160h80v-80c0-8.8 7.2-16 16-16h64c8.8 0 16 7.2 16 16v80h80V352h-32V224h-32v-96h32v96h32v128h32v160h80c8.8 0 16-7.2 16-16V368c0-8.8-7.2-16-16-16zM232 176c0-8.8 7.2-16 16-16h16c8.8 0 16 7.2 16 16v48h-48zm56 176h-64v-64c0-8.8 7.2-16 16-16h32c8.8 0 16 7.2 16 16z"],"graduation-cap":[640,512,[],"f19d","M622.34 153.2L343.4 67.5c-15.2-4.67-31.6-4.67-46.79 0L17.66 153.2c-23.54 7.23-23.54 38.36 0 45.59l48.63 14.94c-10.67 13.19-17.23 29.28-17.88 46.9C38.78 266.15 32 276.11 32 288c0 10.78 5.68 19.85 13.86 25.65L20.33 428.53C18.11 438.52 25.71 448 35.94 448h56.11c10.24 0 17.84-9.48 15.62-19.47L82.14 313.65C90.32 307.85 96 298.78 96 288c0-11.57-6.47-21.25-15.66-26.87.76-15.02 8.44-28.3 20.69-36.72L296.6 284.5c9.06 2.78 26.44 6.25 46.79 0l278.95-85.7c23.55-7.24 23.55-38.36 0-45.6zM352.79 315.09c-28.53 8.76-52.84 3.92-65.59 0l-145.02-44.55L128 384c0 35.35 85.96 64 192 64s192-28.65 192-64l-14.18-113.47-145.03 44.56z"],"greater-than":[384,512,[],"f531","M365.52 209.85L59.22 67.01c-16.06-7.49-35.15-.54-42.64 15.52L3.01 111.61c-7.49 16.06-.54 35.15 15.52 42.64L236.96 256.1 18.49 357.99C2.47 365.46-4.46 384.5 3.01 400.52l13.52 29C24 445.54 43.04 452.47 59.06 445l306.47-142.91a32.003 32.003 0 0 0 18.48-29v-34.23c-.01-12.45-7.21-23.76-18.49-29.01z"],"greater-than-equal":[448,512,[],"f532","M55.22 107.69l175.56 68.09-175.44 68.05c-18.39 6.03-27.88 24.39-21.2 41l12.09 30.08c6.68 16.61 26.99 25.19 45.38 19.15L393.02 214.2c13.77-4.52 22.98-16.61 22.98-30.17v-15.96c0-13.56-9.21-25.65-22.98-30.17L91.3 17.92c-18.29-6-38.51 2.53-45.15 19.06L34.12 66.9c-6.64 16.53 2.81 34.79 21.1 40.79zM424 400H24c-13.25 0-24 10.74-24 24v48c0 13.25 10.75 24 24 24h400c13.25 0 24-10.75 24-24v-48c0-13.26-10.75-24-24-24z"],grimace:[496,512,[],"f57f","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zM144 400h-8c-17.7 0-32-14.3-32-32v-8h40v40zm0-56h-40v-8c0-17.7 14.3-32 32-32h8v40zm-8-136c0-17.7 14.3-32 32-32s32 14.3 32 32-14.3 32-32 32-32-14.3-32-32zm72 192h-48v-40h48v40zm0-56h-48v-40h48v40zm64 56h-48v-40h48v40zm0-56h-48v-40h48v40zm64 56h-48v-40h48v40zm0-56h-48v-40h48v40zm-8-104c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm64 128c0 17.7-14.3 32-32 32h-8v-40h40v8zm0-24h-40v-40h8c17.7 0 32 14.3 32 32v8z"],grin:[496,512,[],"f580","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm80 168c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zm-160 0c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zm80 256c-60.6 0-134.5-38.3-143.8-93.3-2-11.8 9.3-21.6 20.7-17.9C155.1 330.5 200 336 248 336s92.9-5.5 123.1-15.2c11.3-3.7 22.6 6.1 20.7 17.9-9.3 55-83.2 93.3-143.8 93.3z"],"grin-alt":[496,512,[],"f581","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm63.7 128.7c7.6-11.4 24.7-11.7 32.7 0 12.4 18.4 15.1 36.9 15.7 55.3-.5 18.4-3.3 36.9-15.7 55.3-7.6 11.4-24.7 11.7-32.7 0-12.4-18.4-15.1-36.9-15.7-55.3.5-18.4 3.3-36.9 15.7-55.3zm-160 0c7.6-11.4 24.7-11.7 32.7 0 12.4 18.4 15.1 36.9 15.7 55.3-.5 18.4-3.3 36.9-15.7 55.3-7.6 11.4-24.7 11.7-32.7 0-12.4-18.4-15.1-36.9-15.7-55.3.5-18.4 3.3-36.9 15.7-55.3zM248 432c-60.6 0-134.5-38.3-143.8-93.3-2-11.8 9.3-21.6 20.7-17.9C155.1 330.5 200 336 248 336s92.9-5.5 123.1-15.2c11.4-3.7 22.6 6.1 20.7 17.9-9.3 55-83.2 93.3-143.8 93.3z"],"grin-beam":[496,512,[],"f582","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm80 144c23.8 0 52.7 29.3 56 71.4.7 8.6-10.8 11.9-14.9 4.5l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.1 7.3-15.6 4-14.9-4.5 3.1-42.1 32-71.4 55.8-71.4zm-160 0c23.8 0 52.7 29.3 56 71.4.7 8.6-10.8 11.9-14.9 4.5l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.2 7.4-15.6 4-14.9-4.5 3.1-42.1 32-71.4 55.8-71.4zm80 280c-60.6 0-134.5-38.3-143.8-93.3-2-11.9 9.4-21.6 20.7-17.9C155.1 330.5 200 336 248 336s92.9-5.5 123.1-15.2c11.4-3.7 22.6 6.1 20.7 17.9-9.3 55-83.2 93.3-143.8 93.3z"],"grin-beam-sweat":[504,512,[],"f583","M456 128c26.5 0 48-21 48-47 0-20-28.5-60.4-41.6-77.8-3.2-4.3-9.6-4.3-12.8 0C436.5 20.6 408 61 408 81c0 26 21.5 47 48 47zm0 32c-44.1 0-80-35.4-80-79 0-4.4.3-14.2 8.1-32.2C345 23.1 298.3 8 248 8 111 8 0 119 0 256s111 248 248 248 248-111 248-248c0-35.1-7.4-68.4-20.5-98.6-6.3 1.5-12.7 2.6-19.5 2.6zm-128-8c23.8 0 52.7 29.3 56 71.4.7 8.6-10.8 12-14.9 4.5l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.1 7.4-15.6 4-14.9-4.5 3.1-42.1 32-71.4 55.8-71.4zm-160 0c23.8 0 52.7 29.3 56 71.4.7 8.6-10.8 12-14.9 4.5l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.2 7.4-15.6 4-14.9-4.5 3.1-42.1 32-71.4 55.8-71.4zm80 280c-60.6 0-134.5-38.3-143.8-93.3-2-11.8 9.3-21.6 20.7-17.9C155.1 330.5 200 336 248 336s92.9-5.5 123.1-15.2c11.5-3.7 22.6 6.2 20.7 17.9-9.3 55-83.2 93.3-143.8 93.3z"],"grin-hearts":[496,512,[],"f584","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zM90.4 183.6c6.7-17.6 26.7-26.7 44.9-21.9l7.1 1.9 2-7.1c5-18.1 22.8-30.9 41.5-27.9 21.4 3.4 34.4 24.2 28.8 44.5L195.3 243c-1.2 4.5-5.9 7.2-10.5 6l-70.2-18.2c-20.4-5.4-31.9-27-24.2-47.2zM248 432c-60.6 0-134.5-38.3-143.8-93.3-2-11.8 9.2-21.5 20.7-17.9C155.1 330.5 200 336 248 336s92.9-5.5 123.1-15.2c11.4-3.6 22.6 6.1 20.7 17.9-9.3 55-83.2 93.3-143.8 93.3zm133.4-201.3l-70.2 18.2c-4.5 1.2-9.2-1.5-10.5-6L281.3 173c-5.6-20.3 7.4-41.1 28.8-44.5 18.6-3 36.4 9.8 41.5 27.9l2 7.1 7.1-1.9c18.2-4.7 38.2 4.3 44.9 21.9 7.7 20.3-3.8 41.9-24.2 47.2z"],"grin-squint":[496,512,[],"f585","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm33.8 189.7l80-48c11.6-6.9 24 7.7 15.4 18L343.6 208l33.6 40.3c8.7 10.4-3.9 24.8-15.4 18l-80-48c-7.7-4.7-7.7-15.9 0-20.6zm-163-30c-8.6-10.3 3.8-24.9 15.4-18l80 48c7.8 4.7 7.8 15.9 0 20.6l-80 48c-11.5 6.8-24-7.6-15.4-18l33.6-40.3-33.6-40.3zM248 432c-60.6 0-134.5-38.3-143.8-93.3-2-11.9 9.4-21.6 20.7-17.9C155.1 330.5 200 336 248 336s92.9-5.5 123.1-15.2c11.5-3.7 22.6 6.2 20.7 17.9-9.3 55-83.2 93.3-143.8 93.3z"],"grin-squint-tears":[512,512,[],"f586","M409.6 111.9c22.6-3.2 73.5-12 88.3-26.8 19.2-19.2 18.9-50.6-.7-70.2S446-5 426.9 14.2c-14.8 14.8-23.5 65.7-26.8 88.3-.8 5.5 3.9 10.2 9.5 9.4zM102.4 400.1c-22.6 3.2-73.5 12-88.3 26.8-19.1 19.1-18.8 50.6.8 70.2s51 19.9 70.2.7c14.8-14.8 23.5-65.7 26.8-88.3.8-5.5-3.9-10.2-9.5-9.4zm311.7-256.5c-33 3.9-48.6-25.1-45.7-45.7 3.4-24 7.4-42.1 11.5-56.5C285.1-13.4 161.8-.5 80.6 80.6-.5 161.7-13.4 285 41.4 379.9c14.4-4.1 32.4-8 56.5-11.5 33.2-3.9 48.6 25.2 45.7 45.7-3.4 24-7.4 42.1-11.5 56.5 94.8 54.8 218.1 41.9 299.3-39.2s94-204.4 39.2-299.3c-14.4 4.1-32.5 8-56.5 11.5zM255.7 106c3.3-13.2 22.4-11.5 23.6 1.8l4.8 52.3 52.3 4.8c13.4 1.2 14.9 20.3 1.8 23.6l-90.5 22.6c-8.9 2.2-16.7-5.9-14.5-14.5l22.5-90.6zm-90.9 230.3L160 284l-52.3-4.8c-13.4-1.2-14.9-20.3-1.8-23.6l90.5-22.6c8.8-2.2 16.7 5.8 14.5 14.5L188.3 338c-3.1 13.2-22.2 11.7-23.5-1.7zm215.7 44.2c-29.3 29.3-75.7 50.4-116.7 50.4-18.9 0-36.6-4.5-51-14.7-9.8-6.9-8.7-21.8 2-27.2 28.3-14.6 63.9-42.4 97.8-76.3s61.7-69.6 76.3-97.8c5.4-10.5 20.2-11.9 27.3-2 32.3 45.3 7.1 124.7-35.7 167.6z"],"grin-stars":[496,512,[],"f587","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zM94.6 168.9l34.9-5 15.5-31.6c2.9-5.8 11-5.8 13.9 0l15.5 31.6 34.9 5c6.2 1 8.9 8.6 4.3 13.2l-25.4 24.6 6 34.9c1 6.2-5.3 11-11 7.9L152 233.3l-31.3 16.3c-5.7 3.1-12-1.7-11-7.9l6-34.9-25.4-24.6c-4.6-4.7-1.9-12.3 4.3-13.3zM248 432c-60.6 0-134.5-38.3-143.8-93.3-2-11.8 9.3-21.5 20.7-17.9C155.1 330.5 200 336 248 336s92.9-5.5 123.1-15.2c11.5-3.7 22.6 6.1 20.7 17.9-9.3 55-83.2 93.3-143.8 93.3zm157.7-249.9l-25.4 24.6 6 34.9c1 6.2-5.3 11-11 7.9L344 233.3l-31.3 16.3c-5.7 3.1-12-1.7-11-7.9l6-34.9-25.4-24.6c-4.5-4.6-1.9-12.2 4.3-13.2l34.9-5 15.5-31.6c2.9-5.8 11-5.8 13.9 0l15.5 31.6 34.9 5c6.3.9 9 8.5 4.4 13.1z"],"grin-tears":[640,512,[],"f588","M102.4 256.1c-22.6 3.2-73.5 12-88.3 26.8-19.1 19.1-18.8 50.6.8 70.2s51 19.9 70.2.7c14.8-14.8 23.5-65.7 26.8-88.3.8-5.5-3.9-10.2-9.5-9.4zm523.4 26.8c-14.8-14.8-65.7-23.5-88.3-26.8-5.5-.8-10.3 3.9-9.5 9.5 3.2 22.6 12 73.5 26.8 88.3 19.2 19.2 50.6 18.9 70.2-.7s20-51.2.8-70.3zm-129.4-12.8c-3.8-26.6 19.1-49.5 45.7-45.7 8.9 1.3 16.8 2.7 24.3 4.1C552.7 104.5 447.7 8 320 8S87.3 104.5 73.6 228.5c7.5-1.4 15.4-2.8 24.3-4.1 33.2-3.9 48.6 25.3 45.7 45.7-11.8 82.3-29.9 100.4-35.8 106.4-.9.9-2 1.6-3 2.5 42.7 74.6 123 125 215.2 125s172.5-50.4 215.2-125.1c-1-.9-2.1-1.5-3-2.5-5.9-5.9-24-24-35.8-106.3zM400 152c23.8 0 52.7 29.3 56 71.4.7 8.6-10.8 12-14.9 4.5l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.2 7.4-15.6 4-14.9-4.5 3.1-42.1 32-71.4 55.8-71.4zm-160 0c23.8 0 52.7 29.3 56 71.4.7 8.6-10.8 12-14.9 4.5l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.2 7.4-15.6 4-14.9-4.5 3.1-42.1 32-71.4 55.8-71.4zm80 280c-60.6 0-134.5-38.3-143.8-93.3-2-11.7 9.2-21.6 20.7-17.9C227.1 330.5 272 336 320 336s92.9-5.5 123.1-15.2c11.4-3.7 22.6 6.1 20.7 17.9-9.3 55-83.2 93.3-143.8 93.3z"],"grin-tongue":[496,512,[],"f589","M248 8C111 8 0 119 0 256c0 106.3 67 196.7 161 232-5.6-12.2-9-25.7-9-40v-45.5c-24.7-16.2-43.5-38.1-47.8-63.8-2-11.8 9.3-21.5 20.7-17.9C155.1 330.5 200 336 248 336s92.9-5.5 123.1-15.2c11.4-3.6 22.6 6.1 20.7 17.9-4.3 25.7-23.1 47.6-47.8 63.8V448c0 14.3-3.4 27.8-9 40 94-35.3 161-125.7 161-232C496 119 385 8 248 8zm-80 232c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm160 0c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm-34.9 134.6c-14.4-6.5-31.1 2.2-34.6 17.6l-1.8 7.8c-2.1 9.2-15.2 9.2-17.3 0l-1.8-7.8c-3.5-15.4-20.2-24.1-34.6-17.6-.9.4.3-.2-18.9 9.4v63c0 35.2 28 64.5 63.1 64.9 35.7.5 64.9-28.4 64.9-64v-64c-19.5-9.6-18.2-8.9-19-9.3z"],"grin-tongue-squint":[496,512,[],"f58a","M293.1 374.6c-14.4-6.5-31.1 2.2-34.6 17.6l-1.8 7.8c-2.1 9.2-15.2 9.2-17.3 0l-1.8-7.8c-3.5-15.4-20.2-24.1-34.6-17.6-.9.4.3-.2-18.9 9.4v63c0 35.2 28 64.5 63.1 64.9 35.7.5 64.9-28.4 64.9-64v-64c-19.5-9.6-18.2-8.9-19-9.3zM248 8C111 8 0 119 0 256c0 106.3 67 196.7 161 232-5.6-12.2-9-25.7-9-40v-45.5c-24.7-16.2-43.5-38.1-47.8-63.8-2-11.8 9.2-21.5 20.7-17.9C155.1 330.5 200 336 248 336s92.9-5.5 123.1-15.2c11.4-3.7 22.6 6.1 20.7 17.9-4.3 25.7-23.1 47.6-47.8 63.8V448c0 14.3-3.4 27.8-9 40 94-35.3 161-125.7 161-232C496 119 385 8 248 8zm-33.8 210.3l-80 48c-11.5 6.8-24-7.6-15.4-18l33.6-40.3-33.6-40.3c-8.6-10.3 3.8-24.9 15.4-18l80 48c7.7 4.7 7.7 15.9 0 20.6zm163 30c8.7 10.4-3.9 24.8-15.4 18l-80-48c-7.8-4.7-7.8-15.9 0-20.6l80-48c11.7-6.9 23.9 7.7 15.4 18L343.6 208l33.6 40.3z"],"grin-tongue-wink":[496,512,[],"f58b","M344 184c-13.3 0-24 10.7-24 24s10.7 24 24 24 24-10.7 24-24-10.7-24-24-24zM248 8C111 8 0 119 0 256c0 106.3 67 196.7 161 232-5.6-12.2-9-25.7-9-40v-45.5c-24.7-16.2-43.5-38.1-47.8-63.8-2-11.8 9.3-21.5 20.7-17.9C155.1 330.5 200 336 248 336s92.9-5.5 123.1-15.2c11.5-3.7 22.6 6.1 20.7 17.9-4.3 25.7-23.1 47.6-47.8 63.8V448c0 14.3-3.4 27.8-9 40 94-35.3 161-125.7 161-232C496 119 385 8 248 8zm-56 225l-9.5-8.5c-14.8-13.2-46.2-13.2-61 0L112 233c-8.5 7.4-21.6.3-19.8-10.8 4-25.2 34.2-42.1 59.9-42.1S208 197 212 222.2c1.6 11.1-11.6 18.2-20 10.8zm152 39c-35.3 0-64-28.7-64-64s28.7-64 64-64 64 28.7 64 64-28.7 64-64 64zm-50.9 102.6c-14.4-6.5-31.1 2.2-34.6 17.6l-1.8 7.8c-2.1 9.2-15.2 9.2-17.3 0l-1.8-7.8c-3.5-15.4-20.2-24.1-34.6-17.6-.9.4.3-.2-18.9 9.4v63c0 35.2 28 64.5 63.1 64.9 35.7.5 64.9-28.4 64.9-64v-64c-19.5-9.6-18.2-8.9-19-9.3z"],"grin-wink":[496,512,[],"f58c","M0 256c0 137 111 248 248 248s248-111 248-248S385 8 248 8 0 119 0 256zm200-48c0 17.7-14.3 32-32 32s-32-14.3-32-32 14.3-32 32-32 32 14.3 32 32zm168 25l-9.5-8.5c-14.8-13.2-46.2-13.2-61 0L288 233c-8.3 7.4-21.6.4-19.8-10.8 4-25.2 34.2-42.1 59.9-42.1S384 197 388 222.2c1.6 11-11.5 18.2-20 10.8zm-243.1 87.8C155.1 330.5 200 336 248 336s92.9-5.5 123.1-15.2c11.3-3.7 22.6 6 20.7 17.9-9.2 55-83.2 93.3-143.8 93.3s-134.5-38.3-143.8-93.3c-2-11.9 9.3-21.6 20.7-17.9z"],"grip-horizontal":[448,512,[],"f58d","M96 288H32c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32v-64c0-17.67-14.33-32-32-32zm160 0h-64c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32v-64c0-17.67-14.33-32-32-32zm160 0h-64c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32v-64c0-17.67-14.33-32-32-32zM96 96H32c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32v-64c0-17.67-14.33-32-32-32zm160 0h-64c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32v-64c0-17.67-14.33-32-32-32zm160 0h-64c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32v-64c0-17.67-14.33-32-32-32z"],"grip-lines":[512,512,[],"f7a4","M496 288H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h480c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zm0-128H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h480c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16z"],"grip-lines-vertical":[256,512,[],"f7a5","M96 496V16c0-8.8-7.2-16-16-16H48c-8.8 0-16 7.2-16 16v480c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16zm128 0V16c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v480c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16z"],"grip-vertical":[320,512,[],"f58e","M96 32H32C14.33 32 0 46.33 0 64v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32V64c0-17.67-14.33-32-32-32zm0 160H32c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32v-64c0-17.67-14.33-32-32-32zm0 160H32c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32v-64c0-17.67-14.33-32-32-32zM288 32h-64c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32V64c0-17.67-14.33-32-32-32zm0 160h-64c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32v-64c0-17.67-14.33-32-32-32zm0 160h-64c-17.67 0-32 14.33-32 32v64c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32v-64c0-17.67-14.33-32-32-32z"],guitar:[512,512,[],"f7a6","M502.63 39L473 9.37a32 32 0 0 0-45.26 0L381.46 55.7a35.14 35.14 0 0 0-8.53 13.79L360.77 106l-76.26 76.26c-12.16-8.76-25.5-15.74-40.1-19.14-33.45-7.78-67-.88-89.88 22a82.45 82.45 0 0 0-20.24 33.47c-6 18.56-23.21 32.69-42.15 34.46-23.7 2.27-45.73 11.45-62.61 28.44C-16.11 327-7.9 409 47.58 464.45S185 528 230.56 482.52c17-16.88 26.16-38.9 28.45-62.71 1.76-18.85 15.89-36.13 34.43-42.14a82.6 82.6 0 0 0 33.48-20.25c22.87-22.88 29.74-56.36 22-89.75-3.39-14.64-10.37-28-19.16-40.2L406 151.23l36.48-12.16a35.14 35.14 0 0 0 13.79-8.53l46.33-46.32a32 32 0 0 0 .03-45.22zM208 352a48 48 0 1 1 48-48 48 48 0 0 1-48 48z"],"h-square":[448,512,[],"f0fd","M448 80v352c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48V80c0-26.51 21.49-48 48-48h352c26.51 0 48 21.49 48 48zm-112 48h-32c-8.837 0-16 7.163-16 16v80H160v-80c0-8.837-7.163-16-16-16h-32c-8.837 0-16 7.163-16 16v224c0 8.837 7.163 16 16 16h32c8.837 0 16-7.163 16-16v-80h128v80c0 8.837 7.163 16 16 16h32c8.837 0 16-7.163 16-16V144c0-8.837-7.163-16-16-16z"],hamburger:[512,512,[],"f805","M464 256H48a48 48 0 0 0 0 96h416a48 48 0 0 0 0-96zm16 128H32a16 16 0 0 0-16 16v16a64 64 0 0 0 64 64h352a64 64 0 0 0 64-64v-16a16 16 0 0 0-16-16zM58.64 224h394.72c34.57 0 54.62-43.9 34.82-75.88C448 83.2 359.55 32.1 256 32c-103.54.1-192 51.2-232.18 116.11C4 180.09 24.07 224 58.64 224zM384 112a16 16 0 1 1-16 16 16 16 0 0 1 16-16zM256 80a16 16 0 1 1-16 16 16 16 0 0 1 16-16zm-128 32a16 16 0 1 1-16 16 16 16 0 0 1 16-16z"],hammer:[576,512,[],"f6e3","M571.31 193.94l-22.63-22.63c-6.25-6.25-16.38-6.25-22.63 0l-11.31 11.31-28.9-28.9c5.63-21.31.36-44.9-16.35-61.61l-45.25-45.25c-62.48-62.48-163.79-62.48-226.28 0l90.51 45.25v18.75c0 16.97 6.74 33.25 18.75 45.25l49.14 49.14c16.71 16.71 40.3 21.98 61.61 16.35l28.9 28.9-11.31 11.31c-6.25 6.25-6.25 16.38 0 22.63l22.63 22.63c6.25 6.25 16.38 6.25 22.63 0l90.51-90.51c6.23-6.24 6.23-16.37-.02-22.62zm-286.72-15.2c-3.7-3.7-6.84-7.79-9.85-11.95L19.64 404.96c-25.57 23.88-26.26 64.19-1.53 88.93s65.05 24.05 88.93-1.53l238.13-255.07c-3.96-2.91-7.9-5.87-11.44-9.41l-49.14-49.14z"],hamsa:[512,512,[],"f665","M509.34 307.25C504.28 295.56 492.75 288 480 288h-64V80c0-22-18-40-40-40s-40 18-40 40v134c0 5.52-4.48 10-10 10h-20c-5.52 0-10-4.48-10-10V40c0-22-18-40-40-40s-40 18-40 40v174c0 5.52-4.48 10-10 10h-20c-5.52 0-10-4.48-10-10V80c0-22-18-40-40-40S96 58 96 80v208H32c-12.75 0-24.28 7.56-29.34 19.25a31.966 31.966 0 0 0 5.94 34.58l102.69 110.03C146.97 490.08 199.69 512 256 512s109.03-21.92 144.72-60.14L503.4 341.83a31.966 31.966 0 0 0 5.94-34.58zM256 416c-53.02 0-96-64-96-64s42.98-64 96-64 96 64 96 64-42.98 64-96 64zm0-96c-17.67 0-32 14.33-32 32s14.33 32 32 32 32-14.33 32-32-14.33-32-32-32z"],"hand-holding":[576,512,[],"f4bd","M565.3 328.1c-11.8-10.7-30.2-10-42.6 0L430.3 402c-11.3 9.1-25.4 14-40 14H272c-8.8 0-16-7.2-16-16s7.2-16 16-16h78.3c15.9 0 30.7-10.9 33.3-26.6 3.3-20-12.1-37.4-31.6-37.4H192c-27 0-53.1 9.3-74.1 26.3L71.4 384H16c-8.8 0-16 7.2-16 16v96c0 8.8 7.2 16 16 16h356.8c14.5 0 28.6-4.9 40-14L564 377c15.2-12.1 16.4-35.3 1.3-48.9z"],"hand-holding-heart":[576,512,[],"f4be","M275.3 250.5c7 7.4 18.4 7.4 25.5 0l108.9-114.2c31.6-33.2 29.8-88.2-5.6-118.8-30.8-26.7-76.7-21.9-104.9 7.7L288 36.9l-11.1-11.6C248.7-4.4 202.8-9.2 172 17.5c-35.3 30.6-37.2 85.6-5.6 118.8l108.9 114.2zm290 77.6c-11.8-10.7-30.2-10-42.6 0L430.3 402c-11.3 9.1-25.4 14-40 14H272c-8.8 0-16-7.2-16-16s7.2-16 16-16h78.3c15.9 0 30.7-10.9 33.3-26.6 3.3-20-12.1-37.4-31.6-37.4H192c-27 0-53.1 9.3-74.1 26.3L71.4 384H16c-8.8 0-16 7.2-16 16v96c0 8.8 7.2 16 16 16h356.8c14.5 0 28.6-4.9 40-14L564 377c15.2-12.1 16.4-35.3 1.3-48.9z"],"hand-holding-medical":[576,512,[],"f95c","M159.88,175.82h64v64a16,16,0,0,0,16,16h64a16,16,0,0,0,16-16v-64h64a16,16,0,0,0,16-16v-64a16,16,0,0,0-16-16h-64v-64a16,16,0,0,0-16-16h-64a16,16,0,0,0-16,16v64h-64a16,16,0,0,0-16,16v64A16,16,0,0,0,159.88,175.82ZM568.07,336.13a39.91,39.91,0,0,0-55.93-8.47L392.47,415.84H271.86a16,16,0,0,1,0-32H350.1c16,0,30.75-10.87,33.37-26.61a32.06,32.06,0,0,0-31.62-37.38h-160a117.7,117.7,0,0,0-74.12,26.25l-46.5,37.74H15.87a16.11,16.11,0,0,0-16,16v96a16.11,16.11,0,0,0,16,16h347a104.8,104.8,0,0,0,61.7-20.27L559.6,392A40,40,0,0,0,568.07,336.13Z"],"hand-holding-usd":[576,512,[],"f4c0","M271.06,144.3l54.27,14.3a8.59,8.59,0,0,1,6.63,8.1c0,4.6-4.09,8.4-9.12,8.4h-35.6a30,30,0,0,1-11.19-2.2c-5.24-2.2-11.28-1.7-15.3,2l-19,17.5a11.68,11.68,0,0,0-2.25,2.66,11.42,11.42,0,0,0,3.88,15.74,83.77,83.77,0,0,0,34.51,11.5V240c0,8.8,7.83,16,17.37,16h17.37c9.55,0,17.38-7.2,17.38-16V222.4c32.93-3.6,57.84-31,53.5-63-3.15-23-22.46-41.3-46.56-47.7L282.68,97.4a8.59,8.59,0,0,1-6.63-8.1c0-4.6,4.09-8.4,9.12-8.4h35.6A30,30,0,0,1,332,83.1c5.23,2.2,11.28,1.7,15.3-2l19-17.5A11.31,11.31,0,0,0,368.47,61a11.43,11.43,0,0,0-3.84-15.78,83.82,83.82,0,0,0-34.52-11.5V16c0-8.8-7.82-16-17.37-16H295.37C285.82,0,278,7.2,278,16V33.6c-32.89,3.6-57.85,31-53.51,63C227.63,119.6,247,137.9,271.06,144.3ZM565.27,328.1c-11.8-10.7-30.2-10-42.6,0L430.27,402a63.64,63.64,0,0,1-40,14H272a16,16,0,0,1,0-32h78.29c15.9,0,30.71-10.9,33.25-26.6a31.2,31.2,0,0,0,.46-5.46A32,32,0,0,0,352,320H192a117.66,117.66,0,0,0-74.1,26.29L71.4,384H16A16,16,0,0,0,0,400v96a16,16,0,0,0,16,16H372.77a64,64,0,0,0,40-14L564,377a32,32,0,0,0,1.28-48.9Z"],"hand-holding-water":[576,512,[],"f4c1","M288 256c53 0 96-42.1 96-94 0-40-57.1-120.7-83.2-155.6-6.4-8.5-19.2-8.5-25.6 0C249.1 41.3 192 122 192 162c0 51.9 43 94 96 94zm277.3 72.1c-11.8-10.7-30.2-10-42.6 0L430.3 402c-11.3 9.1-25.4 14-40 14H272c-8.8 0-16-7.2-16-16s7.2-16 16-16h78.3c15.9 0 30.7-10.9 33.3-26.6 3.3-20-12.1-37.4-31.6-37.4H192c-27 0-53.1 9.3-74.1 26.3L71.4 384H16c-8.8 0-16 7.2-16 16v96c0 8.8 7.2 16 16 16h356.8c14.5 0 28.6-4.9 40-14L564 377c15.2-12.1 16.4-35.3 1.3-48.9z"],"hand-lizard":[576,512,[],"f258","M384 480h192V363.778a95.998 95.998 0 0 0-14.833-51.263L398.127 54.368A48 48 0 0 0 357.544 32H24C10.745 32 0 42.745 0 56v16c0 30.928 25.072 56 56 56h229.981c12.844 0 21.556 13.067 16.615 24.923l-21.41 51.385A32 32 0 0 1 251.648 224H128c-35.346 0-64 28.654-64 64v8c0 13.255 10.745 24 24 24h147.406a47.995 47.995 0 0 1 25.692 7.455l111.748 70.811A24.001 24.001 0 0 1 384 418.539V480z"],"hand-middle-finger":[512,512,[],"f806","M479.93 317.12a37.33 37.33 0 0 0-28.28-36.19L416 272v-49.59c0-11.44-9.69-21.29-23.15-23.54l-38.4-6.4C336.63 189.5 320 200.86 320 216v32a8 8 0 0 1-16 0V50c0-26.28-20.25-49.2-46.52-50A48 48 0 0 0 208 48v200a8 8 0 0 1-16 0v-32c0-15.15-16.63-26.51-34.45-23.54l-30.68 5.12c-18 3-30.87 16.12-30.87 31.38V376a8 8 0 0 1-16 0v-76l-27.36 15A37.34 37.34 0 0 0 32 348.4v73.47a37.31 37.31 0 0 0 10.93 26.39l30.93 30.93A112 112 0 0 0 153.05 512h215A112 112 0 0 0 480 400z"],"hand-paper":[448,512,[],"f256","M408.781 128.007C386.356 127.578 368 146.36 368 168.79V256h-8V79.79c0-22.43-18.356-41.212-40.781-40.783C297.488 39.423 280 57.169 280 79v177h-8V40.79C272 18.36 253.644-.422 231.219.007 209.488.423 192 18.169 192 40v216h-8V80.79c0-22.43-18.356-41.212-40.781-40.783C121.488 40.423 104 58.169 104 80v235.992l-31.648-43.519c-12.993-17.866-38.009-21.817-55.877-8.823-17.865 12.994-21.815 38.01-8.822 55.877l125.601 172.705A48 48 0 0 0 172.073 512h197.59c22.274 0 41.622-15.324 46.724-37.006l26.508-112.66a192.011 192.011 0 0 0 5.104-43.975V168c.001-21.831-17.487-39.577-39.218-39.993z"],"hand-peace":[448,512,[],"f25b","M408 216c-22.092 0-40 17.909-40 40h-8v-32c0-22.091-17.908-40-40-40s-40 17.909-40 40v32h-8V48c0-26.51-21.49-48-48-48s-48 21.49-48 48v208h-13.572L92.688 78.449C82.994 53.774 55.134 41.63 30.461 51.324 5.787 61.017-6.356 88.877 3.337 113.551l74.765 190.342-31.09 24.872c-15.381 12.306-19.515 33.978-9.741 51.081l64 112A39.998 39.998 0 0 0 136 512h240c18.562 0 34.686-12.77 38.937-30.838l32-136A39.97 39.97 0 0 0 448 336v-80c0-22.091-17.908-40-40-40z"],"hand-point-down":[384,512,[],"f0a7","M91.826 467.2V317.966c-8.248 5.841-16.558 10.57-24.918 14.153C35.098 345.752-.014 322.222 0 288c.008-18.616 10.897-32.203 29.092-40 28.286-12.122 64.329-78.648 77.323-107.534 7.956-17.857 25.479-28.453 43.845-28.464l.001-.002h171.526c11.812 0 21.897 8.596 23.703 20.269 7.25 46.837 38.483 61.76 38.315 123.731-.007 2.724.195 13.254.195 16 0 50.654-22.122 81.574-71.263 72.6-9.297 18.597-39.486 30.738-62.315 16.45-21.177 24.645-53.896 22.639-70.944 6.299V467.2c0 24.15-20.201 44.8-43.826 44.8-23.283 0-43.826-21.35-43.826-44.8zM112 72V24c0-13.255 10.745-24 24-24h192c13.255 0 24 10.745 24 24v48c0 13.255-10.745 24-24 24H136c-13.255 0-24-10.745-24-24zm212-24c0-11.046-8.954-20-20-20s-20 8.954-20 20 8.954 20 20 20 20-8.954 20-20z"],"hand-point-left":[512,512,[],"f0a5","M44.8 155.826h149.234c-5.841-8.248-10.57-16.558-14.153-24.918C166.248 99.098 189.778 63.986 224 64c18.616.008 32.203 10.897 40 29.092 12.122 28.286 78.648 64.329 107.534 77.323 17.857 7.956 28.453 25.479 28.464 43.845l.002.001v171.526c0 11.812-8.596 21.897-20.269 23.703-46.837 7.25-61.76 38.483-123.731 38.315-2.724-.007-13.254.195-16 .195-50.654 0-81.574-22.122-72.6-71.263-18.597-9.297-30.738-39.486-16.45-62.315-24.645-21.177-22.639-53.896-6.299-70.944H44.8c-24.15 0-44.8-20.201-44.8-43.826 0-23.283 21.35-43.826 44.8-43.826zM440 176h48c13.255 0 24 10.745 24 24v192c0 13.255-10.745 24-24 24h-48c-13.255 0-24-10.745-24-24V200c0-13.255 10.745-24 24-24zm24 212c11.046 0 20-8.954 20-20s-8.954-20-20-20-20 8.954-20 20 8.954 20 20 20z"],"hand-point-right":[512,512,[],"f0a4","M512 199.652c0 23.625-20.65 43.826-44.8 43.826h-99.851c16.34 17.048 18.346 49.766-6.299 70.944 14.288 22.829 2.147 53.017-16.45 62.315C353.574 425.878 322.654 448 272 448c-2.746 0-13.276-.203-16-.195-61.971.168-76.894-31.065-123.731-38.315C120.596 407.683 112 397.599 112 385.786V214.261l.002-.001c.011-18.366 10.607-35.889 28.464-43.845 28.886-12.994 95.413-49.038 107.534-77.323 7.797-18.194 21.384-29.084 40-29.092 34.222-.014 57.752 35.098 44.119 66.908-3.583 8.359-8.312 16.67-14.153 24.918H467.2c23.45 0 44.8 20.543 44.8 43.826zM96 200v192c0 13.255-10.745 24-24 24H24c-13.255 0-24-10.745-24-24V200c0-13.255 10.745-24 24-24h48c13.255 0 24 10.745 24 24zM68 368c0-11.046-8.954-20-20-20s-20 8.954-20 20 8.954 20 20 20 20-8.954 20-20z"],"hand-point-up":[384,512,[],"f0a6","M135.652 0c23.625 0 43.826 20.65 43.826 44.8v99.851c17.048-16.34 49.766-18.346 70.944 6.299 22.829-14.288 53.017-2.147 62.315 16.45C361.878 158.426 384 189.346 384 240c0 2.746-.203 13.276-.195 16 .168 61.971-31.065 76.894-38.315 123.731C343.683 391.404 333.599 400 321.786 400H150.261l-.001-.002c-18.366-.011-35.889-10.607-43.845-28.464C93.421 342.648 57.377 276.122 29.092 264 10.897 256.203.008 242.616 0 224c-.014-34.222 35.098-57.752 66.908-44.119 8.359 3.583 16.67 8.312 24.918 14.153V44.8c0-23.45 20.543-44.8 43.826-44.8zM136 416h192c13.255 0 24 10.745 24 24v48c0 13.255-10.745 24-24 24H136c-13.255 0-24-10.745-24-24v-48c0-13.255 10.745-24 24-24zm168 28c-11.046 0-20 8.954-20 20s8.954 20 20 20 20-8.954 20-20-8.954-20-20-20z"],"hand-pointer":[448,512,[],"f25a","M448 240v96c0 3.084-.356 6.159-1.063 9.162l-32 136C410.686 499.23 394.562 512 376 512H168a40.004 40.004 0 0 1-32.35-16.473l-127.997-176c-12.993-17.866-9.043-42.883 8.822-55.876 17.867-12.994 42.884-9.043 55.877 8.823L104 315.992V40c0-22.091 17.908-40 40-40s40 17.909 40 40v200h8v-40c0-22.091 17.908-40 40-40s40 17.909 40 40v40h8v-24c0-22.091 17.908-40 40-40s40 17.909 40 40v24h8c0-22.091 17.908-40 40-40s40 17.909 40 40zm-256 80h-8v96h8v-96zm88 0h-8v96h8v-96zm88 0h-8v96h8v-96z"],"hand-rock":[512,512,[],"f255","M464.8 80c-26.9-.4-48.8 21.2-48.8 48h-8V96.8c0-26.3-20.9-48.3-47.2-48.8-26.9-.4-48.8 21.2-48.8 48v32h-8V80.8c0-26.3-20.9-48.3-47.2-48.8-26.9-.4-48.8 21.2-48.8 48v48h-8V96.8c0-26.3-20.9-48.3-47.2-48.8-26.9-.4-48.8 21.2-48.8 48v136l-8-7.1v-48.1c0-26.3-20.9-48.3-47.2-48.8C21.9 127.6 0 149.2 0 176v66.4c0 27.4 11.7 53.5 32.2 71.8l111.7 99.3c10.2 9.1 16.1 22.2 16.1 35.9v6.7c0 13.3 10.7 24 24 24h240c13.3 0 24-10.7 24-24v-2.9c0-12.8 2.6-25.5 7.5-37.3l49-116.3c5-11.8 7.5-24.5 7.5-37.3V128.8c0-26.3-20.9-48.4-47.2-48.8z"],"hand-scissors":[512,512,[],"f257","M216 440c0-22.092 17.909-40 40-40v-8h-32c-22.091 0-40-17.908-40-40s17.909-40 40-40h32v-8H48c-26.51 0-48-21.49-48-48s21.49-48 48-48h208v-13.572l-177.551-69.74c-24.674-9.694-36.818-37.555-27.125-62.228 9.693-24.674 37.554-36.817 62.228-27.124l190.342 74.765 24.872-31.09c12.306-15.381 33.978-19.515 51.081-9.741l112 64A40.002 40.002 0 0 1 512 168v240c0 18.562-12.77 34.686-30.838 38.937l-136 32A39.982 39.982 0 0 1 336 480h-80c-22.091 0-40-17.908-40-40z"],"hand-sparkles":[640,512,[],"f95d","M106.66,170.64l.09,0,49.55-20.65a7.32,7.32,0,0,0,3.68-6h0a7.29,7.29,0,0,0-3.68-6l-49.57-20.67-.07,0L86,67.68a6.66,6.66,0,0,0-11.92,0l-20.7,49.63-.05,0L3.7,138A7.29,7.29,0,0,0,0,144H0a7.32,7.32,0,0,0,3.68,6L53.27,170.6l.07,0L74,220.26a6.65,6.65,0,0,0,11.92,0l20.69-49.62ZM471.38,467.41l-1-.42-1-.5a38.67,38.67,0,0,1,0-69.14l1-.49,1-.43,37.49-15.63,15.63-37.48.41-1,.47-.95c3.85-7.74,10.58-13.63,18.35-17.34,0-1.33.25-2.69.27-4V144a32,32,0,0,0-64,0v72a8,8,0,0,1-8,8H456a8,8,0,0,1-8-8V64a32,32,0,0,0-64,0V216a8,8,0,0,1-8,8H360a8,8,0,0,1-8-8V32a32,32,0,0,0-64,0V216a8,8,0,0,1-8,8H264a8,8,0,0,1-8-8V64a32,32,0,0,0-64,0v241l-23.59-32.49a40,40,0,0,0-64.71,47.09L229.3,492.21A48.07,48.07,0,0,0,268.09,512H465.7c19.24,0,35.65-11.73,43.24-28.79l-.07-.17ZM349.79,339.52,320,351.93l-12.42,29.78a4,4,0,0,1-7.15,0L288,351.93l-29.79-12.41a4,4,0,0,1,0-7.16L288,319.94l12.42-29.78a4,4,0,0,1,7.15,0L320,319.94l29.79,12.42a4,4,0,0,1,0,7.16ZM640,431.91a7.28,7.28,0,0,0-3.68-6l-49.57-20.67-.07,0L566,355.63a6.66,6.66,0,0,0-11.92,0l-20.7,49.63-.05,0L483.7,426a7.28,7.28,0,0,0-3.68,6h0a7.29,7.29,0,0,0,3.68,5.95l49.57,20.67.07,0L554,508.21a6.65,6.65,0,0,0,11.92,0l20.69-49.62h0l.09,0,49.55-20.66a7.29,7.29,0,0,0,3.68-5.95h0Z"],"hand-spock":[512,512,[],"f259","M510.9005,145.27027,442.604,432.09391A103.99507,103.99507,0,0,1,341.43745,512H214.074a135.96968,135.96968,0,0,1-93.18489-36.95291L12.59072,373.12723a39.992,39.992,0,0,1,54.8122-58.24988l60.59342,57.02528v0a283.24849,283.24849,0,0,0-11.6703-80.46734L73.63726,147.36011a40.00575,40.00575,0,1,1,76.71833-22.7187l37.15458,125.39477a8.33113,8.33113,0,0,0,16.05656-4.4414L153.26183,49.95406A39.99638,39.99638,0,1,1,230.73015,30.0166l56.09491,218.15825a10.42047,10.42047,0,0,0,20.30018-.501L344.80766,63.96966a40.052,40.052,0,0,1,51.30245-30.0893c19.86073,6.2998,30.86262,27.67378,26.67564,48.08487l-33.83869,164.966a7.55172,7.55172,0,0,0,14.74406,3.2666l29.3973-123.45874a39.99414,39.99414,0,1,1,77.81208,18.53121Z"],hands:[640,512,[],"f4c2","M204.8 230.4c-10.6-14.1-30.7-17-44.8-6.4-14.1 10.6-17 30.7-6.4 44.8l38.1 50.8c4.8 6.4 4.1 15.3-1.5 20.9l-12.8 12.8c-6.7 6.7-17.6 6.2-23.6-1.1L64 244.4V96c0-17.7-14.3-32-32-32S0 78.3 0 96v218.4c0 10.9 3.7 21.5 10.5 30l104.1 134.3c5 6.5 8.4 13.9 10.4 21.7 1.8 6.9 8.1 11.6 15.3 11.6H272c8.8 0 16-7.2 16-16V384c0-27.7-9-54.6-25.6-76.8l-57.6-76.8zM608 64c-17.7 0-32 14.3-32 32v148.4l-89.8 107.8c-6 7.2-17 7.7-23.6 1.1l-12.8-12.8c-5.6-5.6-6.3-14.5-1.5-20.9l38.1-50.8c10.6-14.1 7.7-34.2-6.4-44.8-14.1-10.6-34.2-7.7-44.8 6.4l-57.6 76.8C361 329.4 352 356.3 352 384v112c0 8.8 7.2 16 16 16h131.7c7.1 0 13.5-4.7 15.3-11.6 2-7.8 5.4-15.2 10.4-21.7l104.1-134.3c6.8-8.5 10.5-19.1 10.5-30V96c0-17.7-14.3-32-32-32z"],"hands-helping":[640,512,[],"f4c4","M488 192H336v56c0 39.7-32.3 72-72 72s-72-32.3-72-72V126.4l-64.9 39C107.8 176.9 96 197.8 96 220.2v47.3l-80 46.2C.7 322.5-4.6 342.1 4.3 357.4l80 138.6c8.8 15.3 28.4 20.5 43.7 11.7L231.4 448H368c35.3 0 64-28.7 64-64h16c17.7 0 32-14.3 32-32v-64h8c13.3 0 24-10.7 24-24v-48c0-13.3-10.7-24-24-24zm147.7-37.4L555.7 16C546.9.7 527.3-4.5 512 4.3L408.6 64H306.4c-12 0-23.7 3.4-33.9 9.7L239 94.6c-9.4 5.8-15 16.1-15 27.1V248c0 22.1 17.9 40 40 40s40-17.9 40-40v-88h184c30.9 0 56 25.1 56 56v28.5l80-46.2c15.3-8.9 20.5-28.4 11.7-43.7z"],"hands-wash":[576,512,[],"f95e","M496,224a48,48,0,1,0-48-48A48,48,0,0,0,496,224ZM311.47,178.45A56.77,56.77,0,0,1,328,176a56,56,0,0,1,19,3.49l15.35-48.61A24,24,0,0,0,342,99.74c-11.53-1.35-22.21,6.44-25.71,17.51l-20.9,66.17ZM93.65,386.33c.8-.19,1.54-.54,2.35-.71V359.93a156,156,0,0,1,107.06-148l73.7-22.76L310.92,81.05a24,24,0,0,0-20.33-31.11c-11.53-1.34-22.22,6.45-25.72,17.52L231.42,173.88a8,8,0,0,1-15.26-4.83L259.53,31.26A24,24,0,0,0,239.2.15C227.67-1.19,217,6.6,213.49,17.66L165.56,169.37a8,8,0,1,1-15.26-4.82l38.56-122a24,24,0,0,0-20.33-31.11C157,10,146.32,17.83,142.82,28.9l-60,189.85L80.76,168.7A24,24,0,0,0,56.9,144.55c-13.23-.05-24.72,10.54-24.9,23.86V281.14A123.69,123.69,0,0,0,93.65,386.33ZM519.1,336H360a8,8,0,0,1,0-16H488a24,24,0,0,0,23.54-28.76C509.35,279.84,498.71,272,487.1,272H288l47.09-17.06a24,24,0,0,0-14.18-45.88L213.19,242.31A123.88,123.88,0,0,0,128,360v25.65a79.78,79.78,0,0,1,58,108.63A118.9,118.9,0,0,0,248,512H456a24,24,0,0,0,23.54-28.76C477.35,471.84,466.71,464,455.1,464H360a8,8,0,0,1,0-16H488a24,24,0,0,0,23.54-28.76C509.35,407.84,498.71,400,487.1,400H360a8,8,0,0,1,0-16H520a24,24,0,0,0,23.54-28.76C541.35,343.84,530.71,336,519.1,336ZM416,64a32,32,0,1,0-32-32A32,32,0,0,0,416,64ZM112,416a48,48,0,1,0,48,48A48,48,0,0,0,112,416Z"],handshake:[640,512,[],"f2b5","M434.7 64h-85.9c-8 0-15.7 3-21.6 8.4l-98.3 90c-.1.1-.2.3-.3.4-16.6 15.6-16.3 40.5-2.1 56 12.7 13.9 39.4 17.6 56.1 2.7.1-.1.3-.1.4-.2l79.9-73.2c6.5-5.9 16.7-5.5 22.6 1 6 6.5 5.5 16.6-1 22.6l-26.1 23.9L504 313.8c2.9 2.4 5.5 5 7.9 7.7V128l-54.6-54.6c-5.9-6-14.1-9.4-22.6-9.4zM544 128.2v223.9c0 17.7 14.3 32 32 32h64V128.2h-96zm48 223.9c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zM0 384h64c17.7 0 32-14.3 32-32V128.2H0V384zm48-63.9c8.8 0 16 7.2 16 16s-7.2 16-16 16-16-7.2-16-16c0-8.9 7.2-16 16-16zm435.9 18.6L334.6 217.5l-30 27.5c-29.7 27.1-75.2 24.5-101.7-4.4-26.9-29.4-24.8-74.9 4.4-101.7L289.1 64h-83.8c-8.5 0-16.6 3.4-22.6 9.4L128 128v223.9h18.3l90.5 81.9c27.4 22.3 67.7 18.1 90-9.3l.2-.2 17.9 15.5c15.9 13 39.4 10.5 52.3-5.4l31.4-38.6 5.4 4.4c13.7 11.1 33.9 9.1 45-4.7l9.5-11.7c11.2-13.8 9.1-33.9-4.6-45.1z"],"handshake-alt-slash":[640,512,[],"f95f","M358.59,195.6,504.2,313.8a63.4,63.4,0,0,1,22.21,37.91H624a16.05,16.05,0,0,0,16-16V143.91A16,16,0,0,0,624,128H512L457.41,73.41A32,32,0,0,0,434.8,64H348.91a32,32,0,0,0-21.61,8.41l-88.12,80.68-25.69-19.85L289.09,64H205.3a32,32,0,0,0-22.6,9.41l-20.34,20.3L45.47,3.38A16,16,0,0,0,23,6.19L3.38,31.46A16,16,0,0,0,6.19,53.91L594.54,508.63A16,16,0,0,0,617,505.82l19.64-25.27a16,16,0,0,0-2.81-22.45L303.4,202.72l32.69-29.92,27-24.7a16,16,0,0,1,21.61,23.61ZM16,128A16.05,16.05,0,0,0,0,144V335.91a16,16,0,0,0,16,16H146.3l90.5,81.89a64,64,0,0,0,90-9.3l.2-.2,17.91,15.5a37.16,37.16,0,0,0,52.29-5.39l8.8-10.82L23.56,128Z"],"handshake-slash":[640,512,[],"f960","M0,128.21V384H64a32,32,0,0,0,32-32V184L23.83,128.21ZM48,320.1a16,16,0,1,1-16,16A16,16,0,0,1,48,320.1Zm80,31.81h18.3l90.5,81.89a64,64,0,0,0,90-9.3l.2-.2,17.91,15.5a37.16,37.16,0,0,0,52.29-5.39l8.8-10.82L128,208.72Zm416-223.7V352.1a32,32,0,0,0,32,32h64V128.21ZM592,352.1a16,16,0,1,1,16-16A16,16,0,0,1,592,352.1ZM303.33,202.67l59.58-54.57a16,16,0,0,1,21.59,23.61L358.41,195.6,504,313.8a73.08,73.08,0,0,1,7.91,7.7V128L457.3,73.41A31.76,31.76,0,0,0,434.7,64H348.8a31.93,31.93,0,0,0-21.6,8.41l-88.07,80.64-25.64-19.81L289.09,64H205.3a32,32,0,0,0-22.6,9.41L162.36,93.72,45.47,3.38A16,16,0,0,0,23,6.19L3.38,31.46A16,16,0,0,0,6.19,53.91L594.53,508.63A16,16,0,0,0,617,505.82l19.65-25.27a16,16,0,0,0-2.82-22.45Z"],hanukiah:[640,512,[],"f6e6","M232 160c-4.42 0-8 3.58-8 8v120h32V168c0-4.42-3.58-8-8-8h-16zm-64 0c-4.42 0-8 3.58-8 8v120h32V168c0-4.42-3.58-8-8-8h-16zm224 0c-4.42 0-8 3.58-8 8v120h32V168c0-4.42-3.58-8-8-8h-16zm64 0c-4.42 0-8 3.58-8 8v120h32V168c0-4.42-3.58-8-8-8h-16zm88 8c0-4.42-3.58-8-8-8h-16c-4.42 0-8 3.58-8 8v120h32V168zm-440-8c-4.42 0-8 3.58-8 8v120h32V168c0-4.42-3.58-8-8-8h-16zm520 0h-32c-8.84 0-16 7.16-16 16v112c0 17.67-14.33 32-32 32H352V128c0-8.84-7.16-16-16-16h-32c-8.84 0-16 7.16-16 16v192H96c-17.67 0-32-14.33-32-32V176c0-8.84-7.16-16-16-16H16c-8.84 0-16 7.16-16 16v112c0 53.02 42.98 96 96 96h192v64H112c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h416c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16H352v-64h192c53.02 0 96-42.98 96-96V176c0-8.84-7.16-16-16-16zm-16-32c13.25 0 24-11.94 24-26.67S608 48 608 48s-24 38.61-24 53.33S594.75 128 608 128zm-576 0c13.25 0 24-11.94 24-26.67S32 48 32 48 8 86.61 8 101.33 18.75 128 32 128zm288-48c13.25 0 24-11.94 24-26.67S320 0 320 0s-24 38.61-24 53.33S306.75 80 320 80zm-208 48c13.25 0 24-11.94 24-26.67S112 48 112 48s-24 38.61-24 53.33S98.75 128 112 128zm64 0c13.25 0 24-11.94 24-26.67S176 48 176 48s-24 38.61-24 53.33S162.75 128 176 128zm64 0c13.25 0 24-11.94 24-26.67S240 48 240 48s-24 38.61-24 53.33S226.75 128 240 128zm160 0c13.25 0 24-11.94 24-26.67S400 48 400 48s-24 38.61-24 53.33S386.75 128 400 128zm64 0c13.25 0 24-11.94 24-26.67S464 48 464 48s-24 38.61-24 53.33S450.75 128 464 128zm64 0c13.25 0 24-11.94 24-26.67S528 48 528 48s-24 38.61-24 53.33S514.75 128 528 128z"],"hard-hat":[512,512,[],"f807","M480 288c0-80.25-49.28-148.92-119.19-177.62L320 192V80a16 16 0 0 0-16-16h-96a16 16 0 0 0-16 16v112l-40.81-81.62C81.28 139.08 32 207.75 32 288v64h448zm16 96H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h480a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"],hashtag:[448,512,[],"f292","M440.667 182.109l7.143-40c1.313-7.355-4.342-14.109-11.813-14.109h-74.81l14.623-81.891C377.123 38.754 371.468 32 363.997 32h-40.632a12 12 0 0 0-11.813 9.891L296.175 128H197.54l14.623-81.891C213.477 38.754 207.822 32 200.35 32h-40.632a12 12 0 0 0-11.813 9.891L132.528 128H53.432a12 12 0 0 0-11.813 9.891l-7.143 40C33.163 185.246 38.818 192 46.289 192h74.81L98.242 320H19.146a12 12 0 0 0-11.813 9.891l-7.143 40C-1.123 377.246 4.532 384 12.003 384h74.81L72.19 465.891C70.877 473.246 76.532 480 84.003 480h40.632a12 12 0 0 0 11.813-9.891L151.826 384h98.634l-14.623 81.891C234.523 473.246 240.178 480 247.65 480h40.632a12 12 0 0 0 11.813-9.891L315.472 384h79.096a12 12 0 0 0 11.813-9.891l7.143-40c1.313-7.355-4.342-14.109-11.813-14.109h-74.81l22.857-128h79.096a12 12 0 0 0 11.813-9.891zM261.889 320h-98.634l22.857-128h98.634l-22.857 128z"],"hat-cowboy":[640,512,[],"f8c0","M490 296.9C480.51 239.51 450.51 64 392.3 64c-14 0-26.49 5.93-37 14a58.21 58.21 0 0 1-70.58 0c-10.51-8-23-14-37-14-58.2 0-88.2 175.47-97.71 232.88C188.81 309.47 243.73 320 320 320s131.23-10.51 170-23.1zm142.9-37.18a16 16 0 0 0-19.75 1.5c-1 .9-101.27 90.78-293.16 90.78-190.82 0-292.22-89.94-293.24-90.84A16 16 0 0 0 1 278.53C1.73 280.55 78.32 480 320 480s318.27-199.45 319-201.47a16 16 0 0 0-6.09-18.81z"],"hat-cowboy-side":[640,512,[],"f8c1","M260.8 291.06c-28.63-22.94-62-35.06-96.4-35.06C87 256 21.47 318.72 1.43 412.06c-3.55 16.6-.43 33.83 8.57 47.3C18.75 472.47 31.83 480 45.88 480H592c-103.21 0-155-37.07-233.19-104.46zm234.65-18.29L468.4 116.2A64 64 0 0 0 392 64.41L200.85 105a64 64 0 0 0-50.35 55.79L143.61 226c6.9-.83 13.7-2 20.79-2 41.79 0 82 14.55 117.29 42.82l98 84.48C450.76 412.54 494.9 448 592 448a48 48 0 0 0 48-48c0-25.39-29.6-119.33-144.55-127.23z"],"hat-wizard":[512,512,[],"f6e8","M496 448H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h480c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zm-304-64l-64-32 64-32 32-64 32 64 64 32-64 32-16 32h208l-86.41-201.63a63.955 63.955 0 0 1-1.89-45.45L416 0 228.42 107.19a127.989 127.989 0 0 0-53.46 59.15L64 416h144l-16-32zm64-224l16-32 16 32 32 16-32 16-16 32-16-32-32-16 32-16z"],hdd:[576,512,[],"f0a0","M576 304v96c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48v-96c0-26.51 21.49-48 48-48h480c26.51 0 48 21.49 48 48zm-48-80a79.557 79.557 0 0 1 30.777 6.165L462.25 85.374A48.003 48.003 0 0 0 422.311 64H153.689a48 48 0 0 0-39.938 21.374L17.223 230.165A79.557 79.557 0 0 1 48 224h480zm-48 96c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32zm-96 0c-17.673 0-32 14.327-32 32s14.327 32 32 32 32-14.327 32-32-14.327-32-32-32z"],"head-side-cough":[640,512,[],"f961","M616,304a24,24,0,1,0-24-24A24,24,0,0,0,616,304ZM552,416a24,24,0,1,0,24,24A24,24,0,0,0,552,416Zm-64-56a24,24,0,1,0,24,24A24,24,0,0,0,488,360ZM616,464a24,24,0,1,0,24,24A24,24,0,0,0,616,464Zm0-104a24,24,0,1,0,24,24A24,24,0,0,0,616,360Zm-64-40a24,24,0,1,0,24,24A24,24,0,0,0,552,320Zm-74.78-45c-21-47.12-48.5-151.75-73.12-186.75A208.13,208.13,0,0,0,234.1,0H192C86,0,0,86,0,192c0,56.75,24.75,107.62,64,142.88V512H288V480h64a64,64,0,0,0,64-64H320a32,32,0,0,1,0-64h96V320h32A32,32,0,0,0,477.22,275ZM288,224a32,32,0,1,1,32-32A32.07,32.07,0,0,1,288,224Z"],"head-side-cough-slash":[640,512,[],"f962","M454.11,319.21c19.56-3.81,31.62-25,23.11-44.21-21-47.12-48.5-151.75-73.12-186.75A208.13,208.13,0,0,0,234.1,0H192A190.64,190.64,0,0,0,84.18,33.3L45.46,3.38A16,16,0,0,0,23,6.19L3.37,31.46A16,16,0,0,0,6.18,53.91L594.53,508.63A16,16,0,0,0,617,505.82l19.64-25.27a16,16,0,0,0-2.81-22.45ZM313.39,210.45,263.61,172c5.88-7.14,14.43-12,24.36-12a32.06,32.06,0,0,1,32,32C320,199,317.24,205.17,313.39,210.45ZM616,304a24,24,0,1,0-24-24A24,24,0,0,0,616,304Zm-64,64a24,24,0,1,0-24-24A24,24,0,0,0,552,368ZM288,384a32,32,0,0,1,32-32h19.54L20.73,105.59A190.86,190.86,0,0,0,0,192c0,56.75,24.75,107.62,64,142.88V512H288V480h64a64,64,0,0,0,64-64H320A32,32,0,0,1,288,384Zm328-24a24,24,0,1,0,24,24A24,24,0,0,0,616,360Z"],"head-side-mask":[512,512,[],"f963","M.15,184.42C-2.17,244.21,23,298.06,64,334.88V512H224V316.51L3.67,156.25A182.28,182.28,0,0,0,.15,184.42ZM509.22,275c-21-47.12-48.5-151.75-73.12-186.75A208.11,208.11,0,0,0,266.11,0H200C117,0,42.48,50.57,13.25,123.65L239.21,288H511.76A31.35,31.35,0,0,0,509.22,275ZM320,224a32,32,0,1,1,32-32A32.07,32.07,0,0,1,320,224Zm16,144H496l16-48H256V512H401.88a64,64,0,0,0,60.71-43.76L464,464H336a16,16,0,0,1,0-32H474.67l10.67-32H336a16,16,0,0,1,0-32Z"],"head-side-virus":[512,512,[],"f964","M272,240a16,16,0,1,0,16,16A16,16,0,0,0,272,240Zm-64-64a16,16,0,1,0,16,16A16,16,0,0,0,208,176Zm301.2,99c-20.93-47.12-48.43-151.73-73.07-186.75A207.9,207.9,0,0,0,266.09,0H192C86,0,0,86,0,192A191.23,191.23,0,0,0,64,334.81V512H320V448h64a64,64,0,0,0,64-64V320H480A32,32,0,0,0,509.2,275ZM368,240H355.88c-28.51,0-42.79,34.47-22.63,54.63l8.58,8.57a16,16,0,1,1-22.63,22.63l-8.57-8.58C290.47,297.09,256,311.37,256,339.88V352a16,16,0,0,1-32,0V339.88c0-28.51-34.47-42.79-54.63-22.63l-8.57,8.58a16,16,0,0,1-22.63-22.63l8.58-8.57c20.16-20.16,5.88-54.63-22.63-54.63H112a16,16,0,0,1,0-32h12.12c28.51,0,42.79-34.47,22.63-54.63l-8.58-8.57a16,16,0,0,1,22.63-22.63l8.57,8.58c20.16,20.16,54.63,5.88,54.63-22.63V96a16,16,0,0,1,32,0v12.12c0,28.51,34.47,42.79,54.63,22.63l8.57-8.58a16,16,0,0,1,22.63,22.63l-8.58,8.57C313.09,173.53,327.37,208,355.88,208H368a16,16,0,0,1,0,32Z"],heading:[512,512,[],"f1dc","M448 96v320h32a16 16 0 0 1 16 16v32a16 16 0 0 1-16 16H320a16 16 0 0 1-16-16v-32a16 16 0 0 1 16-16h32V288H160v128h32a16 16 0 0 1 16 16v32a16 16 0 0 1-16 16H32a16 16 0 0 1-16-16v-32a16 16 0 0 1 16-16h32V96H32a16 16 0 0 1-16-16V48a16 16 0 0 1 16-16h160a16 16 0 0 1 16 16v32a16 16 0 0 1-16 16h-32v128h192V96h-32a16 16 0 0 1-16-16V48a16 16 0 0 1 16-16h160a16 16 0 0 1 16 16v32a16 16 0 0 1-16 16z"],headphones:[512,512,[],"f025","M256 32C114.52 32 0 146.496 0 288v48a32 32 0 0 0 17.689 28.622l14.383 7.191C34.083 431.903 83.421 480 144 480h24c13.255 0 24-10.745 24-24V280c0-13.255-10.745-24-24-24h-24c-31.342 0-59.671 12.879-80 33.627V288c0-105.869 86.131-192 192-192s192 86.131 192 192v1.627C427.671 268.879 399.342 256 368 256h-24c-13.255 0-24 10.745-24 24v176c0 13.255 10.745 24 24 24h24c60.579 0 109.917-48.098 111.928-108.187l14.382-7.191A32 32 0 0 0 512 336v-48c0-141.479-114.496-256-256-256z"],"headphones-alt":[512,512,[],"f58f","M160 288h-16c-35.35 0-64 28.7-64 64.12v63.76c0 35.41 28.65 64.12 64 64.12h16c17.67 0 32-14.36 32-32.06V320.06c0-17.71-14.33-32.06-32-32.06zm208 0h-16c-17.67 0-32 14.35-32 32.06v127.88c0 17.7 14.33 32.06 32 32.06h16c35.35 0 64-28.71 64-64.12v-63.76c0-35.41-28.65-64.12-64-64.12zM256 32C112.91 32 4.57 151.13 0 288v112c0 8.84 7.16 16 16 16h16c8.84 0 16-7.16 16-16V288c0-114.67 93.33-207.8 208-207.82 114.67.02 208 93.15 208 207.82v112c0 8.84 7.16 16 16 16h16c8.84 0 16-7.16 16-16V288C507.43 151.13 399.09 32 256 32z"],headset:[512,512,[],"f590","M192 208c0-17.67-14.33-32-32-32h-16c-35.35 0-64 28.65-64 64v48c0 35.35 28.65 64 64 64h16c17.67 0 32-14.33 32-32V208zm176 144c35.35 0 64-28.65 64-64v-48c0-35.35-28.65-64-64-64h-16c-17.67 0-32 14.33-32 32v112c0 17.67 14.33 32 32 32h16zM256 0C113.18 0 4.58 118.83 0 256v16c0 8.84 7.16 16 16 16h16c8.84 0 16-7.16 16-16v-16c0-114.69 93.31-208 208-208s208 93.31 208 208h-.12c.08 2.43.12 165.72.12 165.72 0 23.35-18.93 42.28-42.28 42.28H320c0-26.51-21.49-48-48-48h-32c-26.51 0-48 21.49-48 48s21.49 48 48 48h181.72c49.86 0 90.28-40.42 90.28-90.28V256C507.42 118.83 398.82 0 256 0z"],heart:[512,512,[],"f004","M462.3 62.6C407.5 15.9 326 24.3 275.7 76.2L256 96.5l-19.7-20.3C186.1 24.3 104.5 15.9 49.7 62.6c-62.8 53.6-66.1 149.8-9.9 207.9l193.5 199.8c12.5 12.9 32.8 12.9 45.3 0l193.5-199.8c56.3-58.1 53-154.3-9.8-207.9z"],"heart-broken":[512,512,[],"f7a9","M473.7 73.8l-2.4-2.5c-46-47-118-51.7-169.6-14.8L336 159.9l-96 64 48 128-144-144 96-64-28.6-86.5C159.7 19.6 87 24 40.7 71.4l-2.4 2.4C-10.4 123.6-12.5 202.9 31 256l212.1 218.6c7.1 7.3 18.6 7.3 25.7 0L481 255.9c43.5-53 41.4-132.3-7.3-182.1z"],heartbeat:[512,512,[],"f21e","M320.2 243.8l-49.7 99.4c-6 12.1-23.4 11.7-28.9-.6l-56.9-126.3-30 71.7H60.6l182.5 186.5c7.1 7.3 18.6 7.3 25.7 0L451.4 288H342.3l-22.1-44.2zM473.7 73.9l-2.4-2.5c-51.5-52.6-135.8-52.6-187.4 0L256 100l-27.9-28.5c-51.5-52.7-135.9-52.7-187.4 0l-2.4 2.4C-10.4 123.7-12.5 203 31 256h102.4l35.9-86.2c5.4-12.9 23.6-13.2 29.4-.4l58.2 129.3 49-97.9c5.9-11.8 22.7-11.8 28.6 0l27.6 55.2H481c43.5-53 41.4-132.3-7.3-182.1z"],helicopter:[640,512,[],"f533","M304 384h272c17.67 0 32-14.33 32-32 0-123.71-100.29-224-224-224V64h176c8.84 0 16-7.16 16-16V16c0-8.84-7.16-16-16-16H144c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h176v64H112L68.8 70.4C65.78 66.37 61.03 64 56 64H16.01C5.6 64-2.04 73.78.49 83.88L32 192l160 64 86.4 115.2A31.992 31.992 0 0 0 304 384zm112-188.49C478.55 208.3 528.03 257.44 540.79 320H416V195.51zm219.37 263.3l-22.15-22.2c-6.25-6.26-16.24-6.1-22.64.01-7.09 6.77-13.84 11.25-24.64 11.25H240c-8.84 0-16 7.18-16 16.03v32.06c0 8.85 7.16 16.03 16 16.03h325.94c14.88 0 35.3-.47 68.45-29.52 7.02-6.14 7.57-17.05.98-23.66z"],highlighter:[544,512,[],"f591","M0 479.98L99.92 512l35.45-35.45-67.04-67.04L0 479.98zm124.61-240.01a36.592 36.592 0 0 0-10.79 38.1l13.05 42.83-50.93 50.94 96.23 96.23 50.86-50.86 42.74 13.08c13.73 4.2 28.65-.01 38.15-10.78l35.55-41.64-173.34-173.34-41.52 35.44zm403.31-160.7l-63.2-63.2c-20.49-20.49-53.38-21.52-75.12-2.35L190.55 183.68l169.77 169.78L530.27 154.4c19.18-21.74 18.15-54.63-2.35-75.13z"],hiking:[384,512,[],"f6ec","M80.95 472.23c-4.28 17.16 6.14 34.53 23.28 38.81 2.61.66 5.22.95 7.8.95 14.33 0 27.37-9.7 31.02-24.23l25.24-100.97-52.78-52.78-34.56 138.22zm14.89-196.12L137 117c2.19-8.42-3.14-16.95-11.92-19.06-43.88-10.52-88.35 15.07-99.32 57.17L.49 253.24c-2.19 8.42 3.14 16.95 11.92 19.06l63.56 15.25c8.79 2.1 17.68-3.02 19.87-11.44zM368 160h-16c-8.84 0-16 7.16-16 16v16h-34.75l-46.78-46.78C243.38 134.11 228.61 128 212.91 128c-27.02 0-50.47 18.3-57.03 44.52l-26.92 107.72a32.012 32.012 0 0 0 8.42 30.39L224 397.25V480c0 17.67 14.33 32 32 32s32-14.33 32-32v-82.75c0-17.09-6.66-33.16-18.75-45.25l-46.82-46.82c.15-.5.49-.89.62-1.41l19.89-79.57 22.43 22.43c6 6 14.14 9.38 22.62 9.38h48v240c0 8.84 7.16 16 16 16h16c8.84 0 16-7.16 16-16V176c.01-8.84-7.15-16-15.99-16zM240 96c26.51 0 48-21.49 48-48S266.51 0 240 0s-48 21.49-48 48 21.49 48 48 48z"],hippo:[640,512,[],"f6ed","M581.12 96.2c-27.67-.15-52.5 17.58-76.6 26.62C489.98 88.27 455.83 64 416 64c-11.28 0-21.95 2.3-32 5.88V56c0-13.26-10.75-24-24-24h-16c-13.25 0-24 10.74-24 24v48.98C286.01 79.58 241.24 64 192 64 85.96 64 0 135.64 0 224v240c0 8.84 7.16 16 16 16h64c8.84 0 16-7.16 16-16v-70.79C128.35 407.57 166.72 416 208 416s79.65-8.43 112-22.79V464c0 8.84 7.16 16 16 16h64c8.84 0 16-7.16 16-16V288h128v32c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-32c17.67 0 32-14.33 32-32v-92.02c0-34.09-24.79-67.59-58.88-67.78zM448 176c-8.84 0-16-7.16-16-16s7.16-16 16-16 16 7.16 16 16-7.16 16-16 16z"],history:[512,512,[],"f1da","M504 255.531c.253 136.64-111.18 248.372-247.82 248.468-59.015.042-113.223-20.53-155.822-54.911-11.077-8.94-11.905-25.541-1.839-35.607l11.267-11.267c8.609-8.609 22.353-9.551 31.891-1.984C173.062 425.135 212.781 440 256 440c101.705 0 184-82.311 184-184 0-101.705-82.311-184-184-184-48.814 0-93.149 18.969-126.068 49.932l50.754 50.754c10.08 10.08 2.941 27.314-11.313 27.314H24c-8.837 0-16-7.163-16-16V38.627c0-14.254 17.234-21.393 27.314-11.314l49.372 49.372C129.209 34.136 189.552 8 256 8c136.81 0 247.747 110.78 248 247.531zm-180.912 78.784l9.823-12.63c8.138-10.463 6.253-25.542-4.21-33.679L288 256.349V152c0-13.255-10.745-24-24-24h-16c-13.255 0-24 10.745-24 24v135.651l65.409 50.874c10.463 8.137 25.541 6.253 33.679-4.21z"],"hockey-puck":[512,512,[],"f453","M0 160c0-53 114.6-96 256-96s256 43 256 96-114.6 96-256 96S0 213 0 160zm0 82.2V352c0 53 114.6 96 256 96s256-43 256-96V242.2c-113.4 82.3-398.5 82.4-512 0z"],"holly-berry":[448,512,[],"f7aa","M144 192c26.5 0 48-21.5 48-48s-21.5-48-48-48-48 21.5-48 48 21.5 48 48 48zm112-48c0 26.5 21.5 48 48 48s48-21.5 48-48-21.5-48-48-48-48 21.5-48 48zm-32-48c26.5 0 48-21.5 48-48S250.5 0 224 0s-48 21.5-48 48 21.5 48 48 48zm-16.2 139.1c.1-12.4-13.1-20.1-23.8-13.7-34.3 20.3-71.4 32.7-108.7 36.2-9.7.9-15.6 11.3-11.6 20.2 6.2 13.9 11.1 28.6 14.7 43.8 3.6 15.2-5.3 30.6-20.2 35.1-14.9 4.5-30.1 7.6-45.3 9.1-9.7 1-15.7 11.3-11.7 20.2 15 32.8 22.9 69.5 23 107.7.1 14.4 15.2 23.1 27.6 16 33.2-19 68.9-30.5 104.8-33.9 9.7-.9 15.6-11.3 11.6-20.2-6.2-13.9-11.1-28.6-14.7-43.8-3.6-15.2 5.3-30.6 20.2-35.1 14.9-4.5 30.1-7.6 45.3-9.1 9.7-1 15.7-11.3 11.7-20.2-15.5-34.2-23.3-72.5-22.9-112.3zM435 365.6c-15.2-1.6-30.3-4.7-45.3-9.1-14.9-4.5-23.8-19.9-20.2-35.1 3.6-15.2 8.5-29.8 14.7-43.8 4-8.9-1.9-19.3-11.6-20.2-37.3-3.5-74.4-15.9-108.7-36.2-10.7-6.3-23.9 1.4-23.8 13.7 0 1.6-.2 3.2-.2 4.9.2 33.3 7 65.7 19.9 94 5.7 12.4 5.2 26.6-.6 38.9 4.9 1.2 9.9 2.2 14.8 3.7 14.9 4.5 23.8 19.9 20.2 35.1-3.6 15.2-8.5 29.8-14.7 43.8-4 8.9 1.9 19.3 11.6 20.2 35.9 3.4 71.6 14.9 104.8 33.9 12.5 7.1 27.6-1.6 27.6-16 .2-38.2 8-75 23-107.7 4.3-8.7-1.8-19.1-11.5-20.1z"],home:[576,512,[],"f015","M280.37 148.26L96 300.11V464a16 16 0 0 0 16 16l112.06-.29a16 16 0 0 0 15.92-16V368a16 16 0 0 1 16-16h64a16 16 0 0 1 16 16v95.64a16 16 0 0 0 16 16.05L464 480a16 16 0 0 0 16-16V300L295.67 148.26a12.19 12.19 0 0 0-15.3 0zM571.6 251.47L488 182.56V44.05a12 12 0 0 0-12-12h-56a12 12 0 0 0-12 12v72.61L318.47 43a48 48 0 0 0-61 0L4.34 251.47a12 12 0 0 0-1.6 16.9l25.5 31A12 12 0 0 0 45.15 301l235.22-193.74a12.19 12.19 0 0 1 15.3 0L530.9 301a12 12 0 0 0 16.9-1.6l25.5-31a12 12 0 0 0-1.7-16.93z"],horse:[576,512,[],"f6f0","M575.92 76.6c-.01-8.13-3.02-15.87-8.58-21.8-3.78-4.03-8.58-9.12-13.69-14.5 11.06-6.84 19.5-17.49 22.18-30.66C576.85 4.68 572.96 0 567.9 0H447.92c-70.69 0-128 57.31-128 128H160c-28.84 0-54.4 12.98-72 33.11V160c-48.53 0-88 39.47-88 88v56c0 8.84 7.16 16 16 16h16c8.84 0 16-7.16 16-16v-56c0-13.22 6.87-24.39 16.78-31.68-.21 2.58-.78 5.05-.78 7.68 0 27.64 11.84 52.36 30.54 69.88l-25.72 68.6a63.945 63.945 0 0 0-2.16 37.99l24.85 99.41A15.982 15.982 0 0 0 107.02 512h65.96c10.41 0 18.05-9.78 15.52-19.88l-26.31-105.26 23.84-63.59L320 345.6V496c0 8.84 7.16 16 16 16h64c8.84 0 16-7.16 16-16V318.22c19.74-20.19 32-47.75 32-78.22 0-.22-.07-.42-.08-.64V136.89l16 7.11 18.9 37.7c7.45 14.87 25.05 21.55 40.49 15.37l32.55-13.02a31.997 31.997 0 0 0 20.12-29.74l-.06-77.71zm-64 19.4c-8.84 0-16-7.16-16-16s7.16-16 16-16 16 7.16 16 16-7.16 16-16 16z"],"horse-head":[512,512,[],"f7ab","M509.8 332.5l-69.9-164.3c-14.9-41.2-50.4-71-93-79.2 18-10.6 46.3-35.9 34.2-82.3-1.3-5-7.1-7.9-12-6.1L166.9 76.3C35.9 123.4 0 238.9 0 398.8V480c0 17.7 14.3 32 32 32h236.2c23.8 0 39.3-25 28.6-46.3L256 384v-.7c-45.6-3.5-84.6-30.7-104.3-69.6-1.6-3.1-.9-6.9 1.6-9.3l12.1-12.1c3.9-3.9 10.6-2.7 12.9 2.4 14.8 33.7 48.2 57.4 87.4 57.4 17.2 0 33-5.1 46.8-13.2l46 63.9c6 8.4 15.7 13.3 26 13.3h50.3c8.5 0 16.6-3.4 22.6-9.4l45.3-39.8c8.9-9.1 11.7-22.6 7.1-34.4zM328 224c-13.3 0-24-10.7-24-24s10.7-24 24-24 24 10.7 24 24-10.7 24-24 24z"],hospital:[448,512,[],"f0f8","M448 492v20H0v-20c0-6.627 5.373-12 12-12h20V120c0-13.255 10.745-24 24-24h88V24c0-13.255 10.745-24 24-24h112c13.255 0 24 10.745 24 24v72h88c13.255 0 24 10.745 24 24v360h20c6.627 0 12 5.373 12 12zM308 192h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12zm-168 64h40c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12zm104 128h-40c-6.627 0-12 5.373-12 12v84h64v-84c0-6.627-5.373-12-12-12zm64-96h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12zm-116 12c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12v-40zM182 96h26v26a6 6 0 0 0 6 6h20a6 6 0 0 0 6-6V96h26a6 6 0 0 0 6-6V70a6 6 0 0 0-6-6h-26V38a6 6 0 0 0-6-6h-20a6 6 0 0 0-6 6v26h-26a6 6 0 0 0-6 6v20a6 6 0 0 0 6 6z"],"hospital-alt":[576,512,[],"f47d","M544 96H416V32c0-17.7-14.3-32-32-32H192c-17.7 0-32 14.3-32 32v64H32c-17.7 0-32 14.3-32 32v368c0 8.8 7.2 16 16 16h544c8.8 0 16-7.2 16-16V128c0-17.7-14.3-32-32-32zM160 436c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40zm0-128c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40zm160 128c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40zm0-128c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40zm16-170c0 3.3-2.7 6-6 6h-26v26c0 3.3-2.7 6-6 6h-20c-3.3 0-6-2.7-6-6v-26h-26c-3.3 0-6-2.7-6-6v-20c0-3.3 2.7-6 6-6h26V86c0-3.3 2.7-6 6-6h20c3.3 0 6 2.7 6 6v26h26c3.3 0 6 2.7 6 6v20zm144 298c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40zm0-128c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h40c6.6 0 12 5.4 12 12v40z"],"hospital-symbol":[512,512,[],"f47e","M256 0C114.6 0 0 114.6 0 256s114.6 256 256 256 256-114.6 256-256S397.4 0 256 0zm112 376c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8v-88h-96v88c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8V136c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v88h96v-88c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v240z"],"hospital-user":[640,512,[],"f80d","M480 320a96 96 0 1 0-96-96 96 96 0 0 0 96 96zm48 32a22.88 22.88 0 0 0-7.06 1.09 124.76 124.76 0 0 1-81.89 0A22.82 22.82 0 0 0 432 352a112 112 0 0 0-112 112.62c.14 26.26 21.73 47.38 48 47.38h224c26.27 0 47.86-21.12 48-47.38A112 112 0 0 0 528 352zm-198.09 10.45A145.19 145.19 0 0 1 352 344.62V128a32 32 0 0 0-32-32h-32V32a32 32 0 0 0-32-32H96a32 32 0 0 0-32 32v64H32a32 32 0 0 0-32 32v368a16 16 0 0 0 16 16h288.31A78.62 78.62 0 0 1 288 464.79a143.06 143.06 0 0 1 41.91-102.34zM144 404a12 12 0 0 1-12 12H92a12 12 0 0 1-12-12v-40a12 12 0 0 1 12-12h40a12 12 0 0 1 12 12zm0-128a12 12 0 0 1-12 12H92a12 12 0 0 1-12-12v-40a12 12 0 0 1 12-12h40a12 12 0 0 1 12 12zm48-122a6 6 0 0 1-6 6h-20a6 6 0 0 1-6-6v-26h-26a6 6 0 0 1-6-6v-20a6 6 0 0 1 6-6h26V70a6 6 0 0 1 6-6h20a6 6 0 0 1 6 6v26h26a6 6 0 0 1 6 6v20a6 6 0 0 1-6 6h-26zm80 250a12 12 0 0 1-12 12h-40a12 12 0 0 1-12-12v-40a12 12 0 0 1 12-12h40a12 12 0 0 1 12 12zm0-128a12 12 0 0 1-12 12h-40a12 12 0 0 1-12-12v-40a12 12 0 0 1 12-12h40a12 12 0 0 1 12 12z"],"hot-tub":[512,512,[],"f593","M414.21 177.65c1.02 8.21 7.75 14.35 15.75 14.35h16.12c9.51 0 17.08-8.57 16-18.35-4.34-39.11-22.4-74.53-50.13-97.16-17.37-14.17-28.82-36.75-31.98-62.15C378.96 6.14 372.22 0 364.23 0h-16.12c-9.51 0-17.09 8.57-16 18.35 4.34 39.11 22.4 74.53 50.13 97.16 17.36 14.17 28.82 36.75 31.97 62.14zm-108 0c1.02 8.21 7.75 14.35 15.75 14.35h16.12c9.51 0 17.08-8.57 16-18.35-4.34-39.11-22.4-74.53-50.13-97.16-17.37-14.17-28.82-36.75-31.98-62.15C270.96 6.14 264.22 0 256.23 0h-16.12c-9.51 0-17.09 8.57-16 18.35 4.34 39.11 22.4 74.53 50.13 97.16 17.36 14.17 28.82 36.75 31.97 62.14zM480 256H256l-110.93-83.2a63.99 63.99 0 0 0-38.4-12.8H64c-35.35 0-64 28.65-64 64v224c0 35.35 28.65 64 64 64h384c35.35 0 64-28.65 64-64V288c0-17.67-14.33-32-32-32zM128 440c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8V328c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v112zm96 0c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8V328c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v112zm96 0c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8V328c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v112zm96 0c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8V328c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v112zM64 128c35.35 0 64-28.65 64-64S99.35 0 64 0 0 28.65 0 64s28.65 64 64 64z"],hotdog:[512,512,[],"f80f","M488.56 23.44a80 80 0 0 0-113.12 0l-352 352a80 80 0 1 0 113.12 113.12l352-352a80 80 0 0 0 0-113.12zm-49.93 95.19c-19.6 19.59-37.52 22.67-51.93 25.14C373.76 146 364.4 147.6 352 160s-14 21.76-16.23 34.71c-2.48 14.4-5.55 32.33-25.15 51.92s-37.52 22.67-51.92 25.15C245.75 274 236.4 275.6 224 288s-14 21.75-16.23 34.7c-2.47 14.4-5.54 32.33-25.14 51.92s-37.53 22.68-51.93 25.15C117.76 402 108.4 403.6 96 416a16 16 0 0 1-22.63-22.63c19.6-19.59 37.52-22.67 51.92-25.14 13-2.22 22.3-3.82 34.71-16.23s14-21.75 16.22-34.7c2.48-14.4 5.55-32.33 25.15-51.92s37.52-22.67 51.92-25.14c13-2.22 22.3-3.83 34.7-16.23s14-21.76 16.24-34.71c2.47-14.4 5.54-32.33 25.14-51.92s37.52-22.68 51.92-25.15C394.24 110 403.59 108.41 416 96a16 16 0 0 1 22.63 22.63zM31.44 322.18L322.18 31.44l-11.54-11.55c-25-25-63.85-26.66-86.79-3.72L16.17 223.85c-22.94 22.94-21.27 61.79 3.72 86.78zm449.12-132.36L189.82 480.56l11.54 11.55c25 25 63.85 26.66 86.79 3.72l207.68-207.68c22.94-22.94 21.27-61.79-3.72-86.79z"],hotel:[576,512,[],"f594","M560 64c8.84 0 16-7.16 16-16V16c0-8.84-7.16-16-16-16H16C7.16 0 0 7.16 0 16v32c0 8.84 7.16 16 16 16h15.98v384H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h240v-80c0-8.8 7.2-16 16-16h32c8.8 0 16 7.2 16 16v80h240c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16h-16V64h16zm-304 44.8c0-6.4 6.4-12.8 12.8-12.8h38.4c6.4 0 12.8 6.4 12.8 12.8v38.4c0 6.4-6.4 12.8-12.8 12.8h-38.4c-6.4 0-12.8-6.4-12.8-12.8v-38.4zm0 96c0-6.4 6.4-12.8 12.8-12.8h38.4c6.4 0 12.8 6.4 12.8 12.8v38.4c0 6.4-6.4 12.8-12.8 12.8h-38.4c-6.4 0-12.8-6.4-12.8-12.8v-38.4zm-128-96c0-6.4 6.4-12.8 12.8-12.8h38.4c6.4 0 12.8 6.4 12.8 12.8v38.4c0 6.4-6.4 12.8-12.8 12.8h-38.4c-6.4 0-12.8-6.4-12.8-12.8v-38.4zM179.2 256h-38.4c-6.4 0-12.8-6.4-12.8-12.8v-38.4c0-6.4 6.4-12.8 12.8-12.8h38.4c6.4 0 12.8 6.4 12.8 12.8v38.4c0 6.4-6.4 12.8-12.8 12.8zM192 384c0-53.02 42.98-96 96-96s96 42.98 96 96H192zm256-140.8c0 6.4-6.4 12.8-12.8 12.8h-38.4c-6.4 0-12.8-6.4-12.8-12.8v-38.4c0-6.4 6.4-12.8 12.8-12.8h38.4c6.4 0 12.8 6.4 12.8 12.8v38.4zm0-96c0 6.4-6.4 12.8-12.8 12.8h-38.4c-6.4 0-12.8-6.4-12.8-12.8v-38.4c0-6.4 6.4-12.8 12.8-12.8h38.4c6.4 0 12.8 6.4 12.8 12.8v38.4z"],hourglass:[384,512,[],"f254","M360 64c13.255 0 24-10.745 24-24V24c0-13.255-10.745-24-24-24H24C10.745 0 0 10.745 0 24v16c0 13.255 10.745 24 24 24 0 90.965 51.016 167.734 120.842 192C75.016 280.266 24 357.035 24 448c-13.255 0-24 10.745-24 24v16c0 13.255 10.745 24 24 24h336c13.255 0 24-10.745 24-24v-16c0-13.255-10.745-24-24-24 0-90.965-51.016-167.734-120.842-192C308.984 231.734 360 154.965 360 64z"],"hourglass-end":[384,512,[],"f253","M360 64c13.255 0 24-10.745 24-24V24c0-13.255-10.745-24-24-24H24C10.745 0 0 10.745 0 24v16c0 13.255 10.745 24 24 24 0 90.965 51.016 167.734 120.842 192C75.016 280.266 24 357.035 24 448c-13.255 0-24 10.745-24 24v16c0 13.255 10.745 24 24 24h336c13.255 0 24-10.745 24-24v-16c0-13.255-10.745-24-24-24 0-90.965-51.016-167.734-120.842-192C308.984 231.734 360 154.965 360 64zM192 208c-57.787 0-104-66.518-104-144h208c0 77.945-46.51 144-104 144z"],"hourglass-half":[384,512,[],"f252","M360 0H24C10.745 0 0 10.745 0 24v16c0 13.255 10.745 24 24 24 0 90.965 51.016 167.734 120.842 192C75.016 280.266 24 357.035 24 448c-13.255 0-24 10.745-24 24v16c0 13.255 10.745 24 24 24h336c13.255 0 24-10.745 24-24v-16c0-13.255-10.745-24-24-24 0-90.965-51.016-167.734-120.842-192C308.984 231.734 360 154.965 360 64c13.255 0 24-10.745 24-24V24c0-13.255-10.745-24-24-24zm-75.078 384H99.08c17.059-46.797 52.096-80 92.92-80 40.821 0 75.862 33.196 92.922 80zm.019-256H99.078C91.988 108.548 88 86.748 88 64h208c0 22.805-3.987 44.587-11.059 64z"],"hourglass-start":[384,512,[],"f251","M360 0H24C10.745 0 0 10.745 0 24v16c0 13.255 10.745 24 24 24 0 90.965 51.016 167.734 120.842 192C75.016 280.266 24 357.035 24 448c-13.255 0-24 10.745-24 24v16c0 13.255 10.745 24 24 24h336c13.255 0 24-10.745 24-24v-16c0-13.255-10.745-24-24-24 0-90.965-51.016-167.734-120.842-192C308.984 231.734 360 154.965 360 64c13.255 0 24-10.745 24-24V24c0-13.255-10.745-24-24-24zm-64 448H88c0-77.458 46.204-144 104-144 57.786 0 104 66.517 104 144z"],"house-damage":[576,512,[],"f6f1","M288 114.96L69.47 307.71c-1.62 1.46-3.69 2.14-5.47 3.35V496c0 8.84 7.16 16 16 16h149.23L192 439.19l104.11-64-60.16-119.22L384 392.75l-104.11 64L319.81 512H496c8.84 0 16-7.16 16-16V311.1c-1.7-1.16-3.72-1.82-5.26-3.2L288 114.96zm282.69 121.32L512 184.45V48c0-8.84-7.16-16-16-16h-64c-8.84 0-16 7.16-16 16v51.69L314.75 10.31C307.12 3.45 297.56.01 288 0s-19.1 3.41-26.7 10.27L5.31 236.28c-6.57 5.91-7.12 16.02-1.21 22.6l21.4 23.82c5.9 6.57 16.02 7.12 22.6 1.21L277.42 81.63c6.05-5.33 15.12-5.33 21.17 0L527.91 283.9c6.57 5.9 16.69 5.36 22.6-1.21l21.4-23.82c5.9-6.57 5.36-16.69-1.22-22.59z"],"house-user":[576,512,[],"f965","M570.69,236.27,512,184.44V48a16,16,0,0,0-16-16H432a16,16,0,0,0-16,16V99.67L314.78,10.3C308.5,4.61,296.53,0,288,0s-20.46,4.61-26.74,10.3l-256,226A18.27,18.27,0,0,0,0,248.2a18.64,18.64,0,0,0,4.09,10.71L25.5,282.7a21.14,21.14,0,0,0,12,5.3,21.67,21.67,0,0,0,10.69-4.11l15.9-14V480a32,32,0,0,0,32,32H480a32,32,0,0,0,32-32V269.88l15.91,14A21.94,21.94,0,0,0,538.63,288a20.89,20.89,0,0,0,11.87-5.31l21.41-23.81A21.64,21.64,0,0,0,576,248.19,21,21,0,0,0,570.69,236.27ZM288,176a64,64,0,1,1-64,64A64,64,0,0,1,288,176ZM400,448H176a16,16,0,0,1-16-16,96,96,0,0,1,96-96h64a96,96,0,0,1,96,96A16,16,0,0,1,400,448Z"],hryvnia:[384,512,[],"f6f2","M368 240c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16h-41.86c13.41-28.63 13.74-63.33-4.13-94.05C303.34 49.84 267.1 32 229.96 32h-78.82c-24.32 0-47.86 8.53-66.54 24.09L72.83 65.9c-10.18 8.49-11.56 23.62-3.07 33.8l20.49 24.59c8.49 10.19 23.62 11.56 33.81 3.07l11.73-9.78c4.32-3.6 9.77-5.57 15.39-5.57h83.62c11.69 0 21.2 9.52 21.2 21.2 0 5.91-2.48 11.58-6.81 15.58L219.7 176H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h134.37l-34.67 32H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h41.86c-13.41 28.63-13.74 63.33 4.13 94.05C80.66 462.15 116.9 480 154.04 480h78.82c24.32 0 47.86-8.53 66.54-24.09l11.77-9.81c10.18-8.49 11.56-23.62 3.07-33.8l-20.49-24.59c-8.49-10.19-23.62-11.56-33.81-3.07l-11.75 9.8a23.992 23.992 0 0 1-15.36 5.56H149.2c-11.69 0-21.2-9.52-21.2-21.2 0-5.91 2.48-11.58 6.81-15.58L164.3 336H368c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16H233.63l34.67-32H368z"],"i-cursor":[256,512,[],"f246","M256 52.048V12.065C256 5.496 250.726.148 244.158.066 211.621-.344 166.469.011 128 37.959 90.266.736 46.979-.114 11.913.114 5.318.157 0 5.519 0 12.114v39.645c0 6.687 5.458 12.078 12.145 11.998C38.111 63.447 96 67.243 96 112.182V224H60c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h36v112c0 44.932-56.075 48.031-83.95 47.959C5.404 447.942 0 453.306 0 459.952v39.983c0 6.569 5.274 11.917 11.842 11.999 32.537.409 77.689.054 116.158-37.894 37.734 37.223 81.021 38.073 116.087 37.845 6.595-.043 11.913-5.405 11.913-12V460.24c0-6.687-5.458-12.078-12.145-11.998C217.889 448.553 160 444.939 160 400V288h36c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12h-36V112.182c0-44.932 56.075-48.213 83.95-48.142 6.646.018 12.05-5.346 12.05-11.992z"],"ice-cream":[448,512,[],"f810","M368 160h-.94a144 144 0 1 0-286.12 0H80a48 48 0 0 0 0 96h288a48 48 0 0 0 0-96zM195.38 493.69a31.52 31.52 0 0 0 57.24 0L352 288H96z"],icicles:[512,512,[],"f7ad","M511.4 37.9C515.1 18.2 500 0 480 0H32C10.6 0-4.8 20.7 1.4 41.2l87.1 273.4c2.5 7.2 12.7 7.2 15.1 0L140 190.5l44.2 187.3c1.9 8.3 13.7 8.3 15.6 0l46.5-196.9 34.1 133.4c2.3 7.6 13 7.6 15.3 0l45.8-172.5 66.7 363.8c1.7 8.6 14 8.6 15.7 0l87.5-467.7z"],icons:[512,512,[],"f86d","M116.65 219.35a15.68 15.68 0 0 0 22.65 0l96.75-99.83c28.15-29 26.5-77.1-4.91-103.88C203.75-7.7 163-3.5 137.86 22.44L128 32.58l-9.85-10.14C93.05-3.5 52.25-7.7 24.86 15.64c-31.41 26.78-33 74.85-5 103.88zm143.92 100.49h-48l-7.08-14.24a27.39 27.39 0 0 0-25.66-17.78h-71.71a27.39 27.39 0 0 0-25.66 17.78l-7 14.24h-48A27.45 27.45 0 0 0 0 347.3v137.25A27.44 27.44 0 0 0 27.43 512h233.14A27.45 27.45 0 0 0 288 484.55V347.3a27.45 27.45 0 0 0-27.43-27.46zM144 468a52 52 0 1 1 52-52 52 52 0 0 1-52 52zm355.4-115.9h-60.58l22.36-50.75c2.1-6.65-3.93-13.21-12.18-13.21h-75.59c-6.3 0-11.66 3.9-12.5 9.1l-16.8 106.93c-1 6.3 4.88 11.89 12.5 11.89h62.31l-24.2 83c-1.89 6.65 4.2 12.9 12.23 12.9a13.26 13.26 0 0 0 10.92-5.25l92.4-138.91c4.88-6.91-1.16-15.7-10.87-15.7zM478.08.33L329.51 23.17C314.87 25.42 304 38.92 304 54.83V161.6a83.25 83.25 0 0 0-16-1.7c-35.35 0-64 21.48-64 48s28.65 48 64 48c35.2 0 63.73-21.32 64-47.66V99.66l112-17.22v47.18a83.25 83.25 0 0 0-16-1.7c-35.35 0-64 21.48-64 48s28.65 48 64 48c35.2 0 63.73-21.32 64-47.66V32c0-19.48-16-34.42-33.92-31.67z"],"id-badge":[384,512,[],"f2c1","M336 0H48C21.5 0 0 21.5 0 48v416c0 26.5 21.5 48 48 48h288c26.5 0 48-21.5 48-48V48c0-26.5-21.5-48-48-48zM144 32h96c8.8 0 16 7.2 16 16s-7.2 16-16 16h-96c-8.8 0-16-7.2-16-16s7.2-16 16-16zm48 128c35.3 0 64 28.7 64 64s-28.7 64-64 64-64-28.7-64-64 28.7-64 64-64zm112 236.8c0 10.6-10 19.2-22.4 19.2H102.4C90 416 80 407.4 80 396.8v-19.2c0-31.8 30.1-57.6 67.2-57.6h5c12.3 5.1 25.7 8 39.8 8s27.6-2.9 39.8-8h5c37.1 0 67.2 25.8 67.2 57.6v19.2z"],"id-card":[576,512,[],"f2c2","M528 32H48C21.5 32 0 53.5 0 80v16h576V80c0-26.5-21.5-48-48-48zM0 432c0 26.5 21.5 48 48 48h480c26.5 0 48-21.5 48-48V128H0v304zm352-232c0-4.4 3.6-8 8-8h144c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H360c-4.4 0-8-3.6-8-8v-16zm0 64c0-4.4 3.6-8 8-8h144c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H360c-4.4 0-8-3.6-8-8v-16zm0 64c0-4.4 3.6-8 8-8h144c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H360c-4.4 0-8-3.6-8-8v-16zM176 192c35.3 0 64 28.7 64 64s-28.7 64-64 64-64-28.7-64-64 28.7-64 64-64zM67.1 396.2C75.5 370.5 99.6 352 128 352h8.2c12.3 5.1 25.7 8 39.8 8s27.6-2.9 39.8-8h8.2c28.4 0 52.5 18.5 60.9 44.2 3.2 9.9-5.2 19.8-15.6 19.8H82.7c-10.4 0-18.8-10-15.6-19.8z"],"id-card-alt":[576,512,[],"f47f","M528 64H384v96H192V64H48C21.5 64 0 85.5 0 112v352c0 26.5 21.5 48 48 48h480c26.5 0 48-21.5 48-48V112c0-26.5-21.5-48-48-48zM288 224c35.3 0 64 28.7 64 64s-28.7 64-64 64-64-28.7-64-64 28.7-64 64-64zm93.3 224H194.7c-10.4 0-18.8-10-15.6-19.8 8.3-25.6 32.4-44.2 60.9-44.2h8.2c12.3 5.1 25.7 8 39.8 8s27.6-2.9 39.8-8h8.2c28.4 0 52.5 18.5 60.9 44.2 3.2 9.8-5.2 19.8-15.6 19.8zM352 32c0-17.7-14.3-32-32-32h-64c-17.7 0-32 14.3-32 32v96h128V32z"],igloo:[576,512,[],"f7ae","M320 33.9c-10.5-1.2-21.2-1.9-32-1.9-99.8 0-187.8 50.8-239.4 128H320V33.9zM96 192H30.3C11.1 230.6 0 274 0 320h96V192zM352 39.4V160h175.4C487.2 99.9 424.8 55.9 352 39.4zM480 320h96c0-46-11.1-89.4-30.3-128H480v128zm-64 64v96h128c17.7 0 32-14.3 32-32v-96H411.5c2.6 10.3 4.5 20.9 4.5 32zm32-192H128v128h49.8c22.2-38.1 63-64 110.2-64s88 25.9 110.2 64H448V192zM0 448c0 17.7 14.3 32 32 32h128v-96c0-11.1 1.9-21.7 4.5-32H0v96zm288-160c-53 0-96 43-96 96v96h192v-96c0-53-43-96-96-96z"],image:[512,512,[],"f03e","M464 448H48c-26.51 0-48-21.49-48-48V112c0-26.51 21.49-48 48-48h416c26.51 0 48 21.49 48 48v288c0 26.51-21.49 48-48 48zM112 120c-30.928 0-56 25.072-56 56s25.072 56 56 56 56-25.072 56-56-25.072-56-56-56zM64 384h384V272l-87.515-87.515c-4.686-4.686-12.284-4.686-16.971 0L208 320l-55.515-55.515c-4.686-4.686-12.284-4.686-16.971 0L64 336v48z"],images:[576,512,[],"f302","M480 416v16c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48V176c0-26.51 21.49-48 48-48h16v208c0 44.112 35.888 80 80 80h336zm96-80V80c0-26.51-21.49-48-48-48H144c-26.51 0-48 21.49-48 48v256c0 26.51 21.49 48 48 48h384c26.51 0 48-21.49 48-48zM256 128c0 26.51-21.49 48-48 48s-48-21.49-48-48 21.49-48 48-48 48 21.49 48 48zm-96 144l55.515-55.515c4.686-4.686 12.284-4.686 16.971 0L272 256l135.515-135.515c4.686-4.686 12.284-4.686 16.971 0L512 208v112H160v-48z"],inbox:[576,512,[],"f01c","M567.938 243.908L462.25 85.374A48.003 48.003 0 0 0 422.311 64H153.689a48 48 0 0 0-39.938 21.374L8.062 243.908A47.994 47.994 0 0 0 0 270.533V400c0 26.51 21.49 48 48 48h480c26.51 0 48-21.49 48-48V270.533a47.994 47.994 0 0 0-8.062-26.625zM162.252 128h251.497l85.333 128H376l-32 64H232l-32-64H76.918l85.334-128z"],indent:[448,512,[],"f03c","M27.31 363.3l96-96a16 16 0 0 0 0-22.62l-96-96C17.27 138.66 0 145.78 0 160v192c0 14.31 17.33 21.3 27.31 11.3zM432 416H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm3.17-128H204.83A12.82 12.82 0 0 0 192 300.83v38.34A12.82 12.82 0 0 0 204.83 352h230.34A12.82 12.82 0 0 0 448 339.17v-38.34A12.82 12.82 0 0 0 435.17 288zm0-128H204.83A12.82 12.82 0 0 0 192 172.83v38.34A12.82 12.82 0 0 0 204.83 224h230.34A12.82 12.82 0 0 0 448 211.17v-38.34A12.82 12.82 0 0 0 435.17 160zM432 32H16A16 16 0 0 0 0 48v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z"],industry:[512,512,[],"f275","M475.115 163.781L336 252.309v-68.28c0-18.916-20.931-30.399-36.885-20.248L160 252.309V56c0-13.255-10.745-24-24-24H24C10.745 32 0 42.745 0 56v400c0 13.255 10.745 24 24 24h464c13.255 0 24-10.745 24-24V184.029c0-18.917-20.931-30.399-36.885-20.248z"],infinity:[640,512,[],"f534","M471.1 96C405 96 353.3 137.3 320 174.6 286.7 137.3 235 96 168.9 96 75.8 96 0 167.8 0 256s75.8 160 168.9 160c66.1 0 117.8-41.3 151.1-78.6 33.3 37.3 85 78.6 151.1 78.6 93.1 0 168.9-71.8 168.9-160S564.2 96 471.1 96zM168.9 320c-40.2 0-72.9-28.7-72.9-64s32.7-64 72.9-64c38.2 0 73.4 36.1 94 64-20.4 27.6-55.9 64-94 64zm302.2 0c-38.2 0-73.4-36.1-94-64 20.4-27.6 55.9-64 94-64 40.2 0 72.9 28.7 72.9 64s-32.7 64-72.9 64z"],info:[192,512,[],"f129","M20 424.229h20V279.771H20c-11.046 0-20-8.954-20-20V212c0-11.046 8.954-20 20-20h112c11.046 0 20 8.954 20 20v212.229h20c11.046 0 20 8.954 20 20V492c0 11.046-8.954 20-20 20H20c-11.046 0-20-8.954-20-20v-47.771c0-11.046 8.954-20 20-20zM96 0C56.235 0 24 32.235 24 72s32.235 72 72 72 72-32.235 72-72S135.764 0 96 0z"],"info-circle":[512,512,[],"f05a","M256 8C119.043 8 8 119.083 8 256c0 136.997 111.043 248 248 248s248-111.003 248-248C504 119.083 392.957 8 256 8zm0 110c23.196 0 42 18.804 42 42s-18.804 42-42 42-42-18.804-42-42 18.804-42 42-42zm56 254c0 6.627-5.373 12-12 12h-88c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h12v-64h-12c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h64c6.627 0 12 5.373 12 12v100h12c6.627 0 12 5.373 12 12v24z"],italic:[320,512,[],"f033","M320 48v32a16 16 0 0 1-16 16h-62.76l-80 320H208a16 16 0 0 1 16 16v32a16 16 0 0 1-16 16H16a16 16 0 0 1-16-16v-32a16 16 0 0 1 16-16h62.76l80-320H112a16 16 0 0 1-16-16V48a16 16 0 0 1 16-16h192a16 16 0 0 1 16 16z"],jedi:[576,512,[],"f669","M535.95308,352c-42.64069,94.17188-137.64086,160-247.9848,160q-6.39844,0-12.84377-.29688C171.15558,506.9375,81.26481,442.23438,40.01474,352H79.93668L21.3272,293.40625a264.82522,264.82522,0,0,1-5.10938-39.42187,273.6653,273.6653,0,0,1,.5-29.98438H63.93665L22.546,182.625A269.79782,269.79782,0,0,1,130.51489,20.54688a16.06393,16.06393,0,0,1,9.28127-3,16.36332,16.36332,0,0,1,13.5,7.25,16.02739,16.02739,0,0,1,1.625,15.09374,138.387,138.387,0,0,0-9.84376,51.26563c0,45.10937,21.04691,86.57813,57.71884,113.73437a16.29989,16.29989,0,0,1,1.20313,25.39063c-26.54692,23.98437-41.17194,56.5-41.17194,91.57813,0,60.03124,42.95319,110.28124,99.89079,121.92187l2.5-65.26563L238.062,397a8.33911,8.33911,0,0,1-10-.75,8.025,8.025,0,0,1-1.39063-9.9375l20.125-33.76562-42.06257-8.73438a7.9898,7.9898,0,0,1,0-15.65625l42.06257-8.71875-20.10941-33.73438a7.99122,7.99122,0,0,1,11.35939-10.71874L268.437,295.64062,279.95265,7.67188a7.97138,7.97138,0,0,1,8-7.67188h.04687a8.02064,8.02064,0,0,1,7.95314,7.70312L307.48394,295.625l30.39068-20.67188a8.08327,8.08327,0,0,1,10,.8125,7.99866,7.99866,0,0,1,1.39062,9.90626L329.12461,319.4375l42.07819,8.73438a7.99373,7.99373,0,0,1,0,15.65624l-42.07819,8.71876,20.1094,33.73437a7.97791,7.97791,0,0,1-1.32812,9.92187A8.25739,8.25739,0,0,1,337.87462,397L310.7027,378.53125l2.5,65.34375c48.48446-9.40625,87.57828-48.15625,97.31267-96.5A123.52652,123.52652,0,0,0,371.9528,230.29688a16.30634,16.30634,0,0,1,1.20313-25.42188c36.65631-27.17188,57.6876-68.60938,57.6876-113.73438a138.01689,138.01689,0,0,0-9.85939-51.3125,15.98132,15.98132,0,0,1,1.60937-15.09374,16.36914,16.36914,0,0,1,13.5-7.23438,16.02453,16.02453,0,0,1,9.25,2.98438A271.26947,271.26947,0,0,1,553.25,182.76562L511.99992,224h46.9532C559.3125,229.76562,560,235.45312,560,241.26562a270.092,270.092,0,0,1-5.125,51.85938L495.98427,352Z"],joint:[640,512,[],"f595","M444.34 181.1c22.38 15.68 35.66 41.16 35.66 68.59V280c0 4.42 3.58 8 8 8h48c4.42 0 8-3.58 8-8v-30.31c0-43.24-21.01-83.41-56.34-108.06C463.85 125.02 448 99.34 448 70.31V8c0-4.42-3.58-8-8-8h-48c-4.42 0-8 3.58-8 8v66.4c0 43.69 24.56 81.63 60.34 106.7zM194.97 358.98C126.03 370.07 59.69 394.69 0 432c83.65 52.28 180.3 80 278.94 80h88.57L254.79 380.49c-14.74-17.2-37.45-25.11-59.82-21.51zM553.28 87.09c-5.67-3.8-9.28-9.96-9.28-16.78V8c0-4.42-3.58-8-8-8h-48c-4.42 0-8 3.58-8 8v62.31c0 22.02 10.17 43.41 28.64 55.39C550.79 153.04 576 199.54 576 249.69V280c0 4.42 3.58 8 8 8h48c4.42 0 8-3.58 8-8v-30.31c0-65.44-32.41-126.19-86.72-162.6zM360.89 352.05c-34.4.06-86.81.15-88.21.17l117.8 137.43A63.987 63.987 0 0 0 439.07 512h88.45L409.57 374.4a63.955 63.955 0 0 0-48.68-22.35zM616 352H432l117.99 137.65A63.987 63.987 0 0 0 598.58 512H616c13.25 0 24-10.75 24-24V376c0-13.26-10.75-24-24-24z"],"journal-whills":[448,512,[],"f66a","M438.40625,377.59375c-3.20313,12.8125-3.20313,57.60937,0,73.60937Q447.9922,460.78907,448,470.40625v16c0,16-12.79688,25.59375-25.59375,25.59375H96c-54.40625,0-96-41.59375-96-96V96C0,41.59375,41.59375,0,96,0H422.40625C438.40625,0,448,9.59375,448,25.59375v332.8125Q448,372.79688,438.40625,377.59375ZM380.79688,384H96c-16,0-32,12.79688-32,32s12.79688,32,32,32H380.79688ZM128.01562,176.01562c0,.51563.14063.98438.14063,1.5l37.10937,32.46876A7.99954,7.99954,0,0,1,160,224h-.01562a9.17678,9.17678,0,0,1-5.25-1.98438L131.14062,201.375C142.6875,250.95312,186.90625,288,240,288s97.3125-37.04688,108.875-86.625l-23.59375,20.64062a8.02516,8.02516,0,0,1-5.26563,1.96876H320a9.14641,9.14641,0,0,1-6.01562-2.71876A9.26508,9.26508,0,0,1,312,216a9.097,9.097,0,0,1,2.73438-6.01562l37.10937-32.46876c.01563-.53124.15625-1,.15625-1.51562,0-11.04688-2.09375-21.51562-5.06251-31.59375l-21.26562,21.25a8.00467,8.00467,0,0,1-11.32812-11.3125l26.42187-26.40625a111.81517,111.81517,0,0,0-46.35937-49.26562,63.02336,63.02336,0,0,1-14.0625,82.64062A55.83846,55.83846,0,0,1,251.625,254.73438l-1.42188-34.28126,12.67188,8.625a3.967,3.967,0,0,0,2.25.6875,3.98059,3.98059,0,0,0,3.43749-6.03124l-8.53124-14.3125,17.90625-3.71876a4.00647,4.00647,0,0,0,0-7.84374l-17.90625-3.71876,8.53124-14.3125a3.98059,3.98059,0,0,0-3.43749-6.03124,4.726,4.726,0,0,0-2.25.67187L248.6875,184.125,244,71.82812a4.00386,4.00386,0,0,0-8,0l-4.625,110.8125-12-8.15624a4.003,4.003,0,0,0-5.68751,5.35937l8.53126,14.3125L204.3125,197.875a3.99686,3.99686,0,0,0,0,7.82812l17.90625,3.73438-8.53126,14.29688a4.72469,4.72469,0,0,0-.56249,2.04687,4.59547,4.59547,0,0,0,1.25,2.90625,4.01059,4.01059,0,0,0,2.75,1.09375,4.09016,4.09016,0,0,0,2.25-.6875l10.35937-7.04687L228.375,254.76562a55.86414,55.86414,0,0,1-28.71875-93.45312,63.01119,63.01119,0,0,1-14.04688-82.65625,111.93158,111.93158,0,0,0-46.375,49.26563l26.42187,26.42187a7.99917,7.99917,0,0,1-11.3125,11.3125l-21.26563-21.26563C130.09375,154.48438,128,164.95312,128.01562,176.01562Z"],kaaba:[576,512,[],"f66b","M554.12 83.51L318.36 4.93a95.962 95.962 0 0 0-60.71 0L21.88 83.51A32.006 32.006 0 0 0 0 113.87v49.01l265.02-79.51c15.03-4.5 30.92-4.5 45.98 0l265 79.51v-49.01c0-13.77-8.81-26-21.88-30.36zm-279.9 30.52L0 196.3v228.38c0 15 10.42 27.98 25.06 31.24l242.12 53.8a95.937 95.937 0 0 0 41.65 0l242.12-53.8c14.64-3.25 25.06-16.24 25.06-31.24V196.29l-274.2-82.26c-9.04-2.72-18.59-2.72-27.59 0zM128 230.11c0 3.61-2.41 6.77-5.89 7.72l-80 21.82C37.02 261.03 32 257.2 32 251.93v-16.58c0-3.61 2.41-6.77 5.89-7.72l80-21.82c5.09-1.39 10.11 2.44 10.11 7.72v16.58zm144-39.28c0 3.61-2.41 6.77-5.89 7.72l-96 26.18c-5.09 1.39-10.11-2.44-10.11-7.72v-16.58c0-3.61 2.41-6.77 5.89-7.72l96-26.18c5.09-1.39 10.11 2.44 10.11 7.72v16.58zm176 22.7c0-5.28 5.02-9.11 10.11-7.72l80 21.82c3.48.95 5.89 4.11 5.89 7.72v16.58c0 5.28-5.02 9.11-10.11 7.72l-80-21.82a7.997 7.997 0 0 1-5.89-7.72v-16.58zm-144-39.27c0-5.28 5.02-9.11 10.11-7.72l96 26.18c3.48.95 5.89 4.11 5.89 7.72v16.58c0 5.28-5.02 9.11-10.11 7.72l-96-26.18a7.997 7.997 0 0 1-5.89-7.72v-16.58z"],key:[512,512,[],"f084","M512 176.001C512 273.203 433.202 352 336 352c-11.22 0-22.19-1.062-32.827-3.069l-24.012 27.014A23.999 23.999 0 0 1 261.223 384H224v40c0 13.255-10.745 24-24 24h-40v40c0 13.255-10.745 24-24 24H24c-13.255 0-24-10.745-24-24v-78.059c0-6.365 2.529-12.47 7.029-16.971l161.802-161.802C163.108 213.814 160 195.271 160 176 160 78.798 238.797.001 335.999 0 433.488-.001 512 78.511 512 176.001zM336 128c0 26.51 21.49 48 48 48s48-21.49 48-48-21.49-48-48-48-48 21.49-48 48z"],keyboard:[576,512,[],"f11c","M528 448H48c-26.51 0-48-21.49-48-48V112c0-26.51 21.49-48 48-48h480c26.51 0 48 21.49 48 48v288c0 26.51-21.49 48-48 48zM128 180v-40c0-6.627-5.373-12-12-12H76c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm-336 96v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm-336 96v-40c0-6.627-5.373-12-12-12H76c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm288 0v-40c0-6.627-5.373-12-12-12H172c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h232c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12z"],khanda:[512,512,[],"f66d","M415.81 66c-6.37-3.5-14.37-2.33-19.36 3.02a15.974 15.974 0 0 0-1.91 19.52c16.49 26.16 25.2 56.39 25.2 87.41-.19 53.25-26.77 102.69-71.27 132.41l-76.63 53.35v-20.1l44.05-36.09c3.92-4.2 5-10.09 2.81-15.28L310.85 273c33.84-19.26 56.94-55.25 56.94-96.99 0-40.79-22.02-76.13-54.59-95.71l5.22-11.44c2.34-5.53.93-11.83-3.57-16.04L255.86 0l-58.99 52.81c-4.5 4.21-5.9 10.51-3.57 16.04l5.22 11.44c-32.57 19.58-54.59 54.93-54.59 95.72 0 41.75 23.09 77.73 56.94 96.99l-7.85 17.24c-2.19 5.18-1.1 11.07 2.81 15.28l44.05 36.09v19.9l-76.59-53.33C119.02 278.62 92.44 229.19 92.26 176c0-31.08 8.71-61.31 25.2-87.47 3.87-6.16 2.4-13.77-2.59-19.08-5-5.34-13.68-6.2-20.02-2.7C16.32 109.6-22.3 205.3 13.36 295.99c7.07 17.99 17.89 34.38 30.46 49.06l55.97 65.36c4.87 5.69 13.04 7.24 19.65 3.72l79.35-42.23L228 392.23l-47.08 32.78c-1.67-.37-3.23-1.01-5.01-1.01-13.25 0-23.99 10.74-23.99 24 0 13.25 10.74 24 23.99 24 12.1 0 21.69-9.11 23.33-20.76l40.63-28.28v29.95c-9.39 5.57-15.99 15.38-15.99 27.1 0 17.67 14.32 32 31.98 32s31.98-14.33 31.98-32c0-11.71-6.61-21.52-15.99-27.1v-30.15l40.91 28.48C314.41 462.89 324 472 336.09 472c13.25 0 23.99-10.75 23.99-24 0-13.26-10.74-24-23.99-24-1.78 0-3.34.64-5.01 1.01L284 392.23l29.21-20.34 79.35 42.23c6.61 3.52 14.78 1.97 19.65-3.71l52.51-61.31c18.87-22.02 34-47.5 41.25-75.59 21.62-83.66-16.45-167.27-90.16-207.51zm-95.99 110c0 22.3-11.49 41.92-28.83 53.38l-5.65-12.41c-8.75-24.52-8.75-51.04 0-75.56l7.83-17.18c16.07 11.65 26.65 30.45 26.65 51.77zm-127.93 0c0-21.32 10.58-40.12 26.66-51.76l7.83 17.18c8.75 24.52 8.75 51.03 0 75.56l-5.65 12.41c-17.34-11.46-28.84-31.09-28.84-53.39z"],kiss:[496,512,[],"f596","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm-80 232c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm136 156c0 19.2-28.7 41.5-71.5 44-8.5.8-12.1-11.8-3.6-15.4l17-7.2c13-5.5 20.8-13.5 20.8-21.5s-7.8-16-20.8-21.5l-17-7.2c-6-2.5-6.1-12.2 0-14.8l17-7.2c13-5.5 20.8-13.5 20.8-21.5s-7.8-16-20.8-21.5l-17-7.2c-8.6-3.6-4.8-16.5 3.6-15.4 42.8 2.5 71.5 24.8 71.5 44 0 13-13.4 27.3-35.2 36C290.6 368.7 304 383 304 396zm24-156c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],"kiss-beam":[496,512,[],"f597","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm-39 219.9l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.2 7.4-15.6 4-14.9-4.5 3.3-42.1 32.2-71.4 56-71.4s52.7 29.3 56 71.4c.5 8.5-10.9 12-15.1 4.5zM304 396c0 19.2-28.7 41.5-71.5 44-8.5.8-12.1-11.8-3.6-15.4l17-7.2c13-5.5 20.8-13.5 20.8-21.5s-7.8-16-20.8-21.5l-17-7.2c-6-2.5-6.1-12.2 0-14.8l17-7.2c13-5.5 20.8-13.5 20.8-21.5s-7.8-16-20.8-21.5l-17-7.2c-8.6-3.6-4.8-16.5 3.6-15.4 42.8 2.5 71.5 24.8 71.5 44 0 13-13.4 27.3-35.2 36C290.6 368.7 304 383 304 396zm65-168.1l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.1 7.3-15.6 4-14.9-4.5 3.3-42.1 32.2-71.4 56-71.4s52.7 29.3 56 71.4c.5 8.5-10.9 12-15.1 4.5z"],"kiss-wink-heart":[504,512,[],"f598","M501.1 402.5c-8-20.8-31.5-31.5-53.1-25.9l-8.4 2.2-2.3-8.4c-5.9-21.4-27-36.5-49-33-25.2 4-40.6 28.6-34 52.6l22.9 82.6c1.5 5.3 7 8.5 12.4 7.1l83-21.5c24.1-6.3 37.7-31.8 28.5-55.7zm-177.6-4c-5.6-20.3-2.3-42 9-59.7 29.7-46.3 98.7-45.5 127.8 4.3 6.4.1 12.6 1.4 18.6 2.9 10.9-27.9 17.1-58.2 17.1-90C496 119 385 8 248 8S0 119 0 256s111 248 248 248c35.4 0 68.9-7.5 99.4-20.9-.3-.7-23.9-84.6-23.9-84.6zM168 240c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm120 156c0 19.2-28.7 41.5-71.5 44-8.5.8-12.1-11.8-3.6-15.4l17-7.2c13-5.5 20.8-13.5 20.8-21.5s-7.8-16-20.8-21.5l-17-7.2c-6-2.5-5.7-12.3 0-14.8l17-7.2c13-5.5 20.8-13.5 20.8-21.5s-7.8-16-20.8-21.5l-17-7.2c-8.8-3.7-4.6-16.6 3.6-15.4 42.8 2.5 71.5 24.8 71.5 44 0 13-13.4 27.3-35.2 36C274.6 368.7 288 383 288 396zm16-179c-8.3 7.4-21.6.4-19.8-10.8 4-25.2 34.2-42.1 59.9-42.1S400 181 404 206.2c1.7 11.1-11.3 18.3-19.8 10.8l-9.5-8.5c-14.8-13.2-46.2-13.2-61 0L304 217z"],"kiwi-bird":[576,512,[],"f535","M575.81 217.98C572.64 157.41 518.28 112 457.63 112h-9.37c-52.82 0-104.25-16.25-147.74-46.24-41.99-28.96-96.04-41.62-153.21-28.7C129.3 41.12-.08 78.24 0 224c.04 70.95 38.68 132.8 95.99 166.01V464c0 8.84 7.16 16 16 16h16c8.84 0 16-7.16 16-16v-54.26c15.36 3.96 31.4 6.26 48 6.26 5.44 0 10.68-.73 16-1.18V464c0 8.84 7.16 16 16 16h16c8.84 0 16-7.16 16-16v-59.43c14.24-5.06 27.88-11.39 40.34-19.51C342.07 355.25 393.86 336 448.46 336c25.48 0 16.01-.31 23.05-.78l74.41 136.44c2.86 5.23 8.3 8.34 14.05 8.34 1.31 0 2.64-.16 3.95-.5 7.09-1.8 12.05-8.19 12.05-15.5 0 0 .14-240.24-.16-246.02zM463.97 248c-13.25 0-24-10.75-24-24 0-13.26 10.75-24 24-24s24 10.74 24 24c0 13.25-10.75 24-24 24zm80 153.25l-39.86-73.08c15.12-5.83 28.73-14.6 39.86-25.98v99.06z"],landmark:[512,512,[],"f66f","M501.62 92.11L267.24 2.04a31.958 31.958 0 0 0-22.47 0L10.38 92.11A16.001 16.001 0 0 0 0 107.09V144c0 8.84 7.16 16 16 16h480c8.84 0 16-7.16 16-16v-36.91c0-6.67-4.14-12.64-10.38-14.98zM64 192v160H48c-8.84 0-16 7.16-16 16v48h448v-48c0-8.84-7.16-16-16-16h-16V192h-64v160h-96V192h-64v160h-96V192H64zm432 256H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h480c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16z"],language:[640,512,[],"f1ab","M152.1 236.2c-3.5-12.1-7.8-33.2-7.8-33.2h-.5s-4.3 21.1-7.8 33.2l-11.1 37.5H163zM616 96H336v320h280c13.3 0 24-10.7 24-24V120c0-13.3-10.7-24-24-24zm-24 120c0 6.6-5.4 12-12 12h-11.4c-6.9 23.6-21.7 47.4-42.7 69.9 8.4 6.4 17.1 12.5 26.1 18 5.5 3.4 7.3 10.5 4.1 16.2l-7.9 13.9c-3.4 5.9-10.9 7.8-16.7 4.3-12.6-7.8-24.5-16.1-35.4-24.9-10.9 8.7-22.7 17.1-35.4 24.9-5.8 3.5-13.3 1.6-16.7-4.3l-7.9-13.9c-3.2-5.6-1.4-12.8 4.2-16.2 9.3-5.7 18-11.7 26.1-18-7.9-8.4-14.9-17-21-25.7-4-5.7-2.2-13.6 3.7-17.1l6.5-3.9 7.3-4.3c5.4-3.2 12.4-1.7 16 3.4 5 7 10.8 14 17.4 20.9 13.5-14.2 23.8-28.9 30-43.2H412c-6.6 0-12-5.4-12-12v-16c0-6.6 5.4-12 12-12h64v-16c0-6.6 5.4-12 12-12h16c6.6 0 12 5.4 12 12v16h64c6.6 0 12 5.4 12 12zM0 120v272c0 13.3 10.7 24 24 24h280V96H24c-13.3 0-24 10.7-24 24zm58.9 216.1L116.4 167c1.7-4.9 6.2-8.1 11.4-8.1h32.5c5.1 0 9.7 3.3 11.4 8.1l57.5 169.1c2.6 7.8-3.1 15.9-11.4 15.9h-22.9a12 12 0 0 1-11.5-8.6l-9.4-31.9h-60.2l-9.1 31.8c-1.5 5.1-6.2 8.7-11.5 8.7H70.3c-8.2 0-14-8.1-11.4-15.9z"],laptop:[640,512,[],"f109","M624 416H381.54c-.74 19.81-14.71 32-32.74 32H288c-18.69 0-33.02-17.47-32.77-32H16c-8.8 0-16 7.2-16 16v16c0 35.2 28.8 64 64 64h512c35.2 0 64-28.8 64-64v-16c0-8.8-7.2-16-16-16zM576 48c0-26.4-21.6-48-48-48H112C85.6 0 64 21.6 64 48v336h512V48zm-64 272H128V64h384v256z"],"laptop-code":[640,512,[],"f5fc","M255.03 261.65c6.25 6.25 16.38 6.25 22.63 0l11.31-11.31c6.25-6.25 6.25-16.38 0-22.63L253.25 192l35.71-35.72c6.25-6.25 6.25-16.38 0-22.63l-11.31-11.31c-6.25-6.25-16.38-6.25-22.63 0l-58.34 58.34c-6.25 6.25-6.25 16.38 0 22.63l58.35 58.34zm96.01-11.3l11.31 11.31c6.25 6.25 16.38 6.25 22.63 0l58.34-58.34c6.25-6.25 6.25-16.38 0-22.63l-58.34-58.34c-6.25-6.25-16.38-6.25-22.63 0l-11.31 11.31c-6.25 6.25-6.25 16.38 0 22.63L386.75 192l-35.71 35.72c-6.25 6.25-6.25 16.38 0 22.63zM624 416H381.54c-.74 19.81-14.71 32-32.74 32H288c-18.69 0-33.02-17.47-32.77-32H16c-8.8 0-16 7.2-16 16v16c0 35.2 28.8 64 64 64h512c35.2 0 64-28.8 64-64v-16c0-8.8-7.2-16-16-16zM576 48c0-26.4-21.6-48-48-48H112C85.6 0 64 21.6 64 48v336h512V48zm-64 272H128V64h384v256z"],"laptop-house":[640,512,[],"f966","M272,288H208a16,16,0,0,1-16-16V208a16,16,0,0,1,16-16h64a16,16,0,0,1,16,16v37.12C299.11,232.24,315,224,332.8,224H469.74l6.65-7.53A16.51,16.51,0,0,0,480,207a16.31,16.31,0,0,0-4.75-10.61L416,144V48a16,16,0,0,0-16-16H368a16,16,0,0,0-16,16V87.3L263.5,8.92C258,4,247.45,0,240.05,0s-17.93,4-23.47,8.92L4.78,196.42A16.15,16.15,0,0,0,0,207a16.4,16.4,0,0,0,3.55,9.39L22.34,237.7A16.22,16.22,0,0,0,33,242.48,16.51,16.51,0,0,0,42.34,239L64,219.88V384a32,32,0,0,0,32,32H272ZM629.33,448H592V288c0-17.67-12.89-32-28.8-32H332.8c-15.91,0-28.8,14.33-28.8,32V448H266.67A10.67,10.67,0,0,0,256,458.67v10.66A42.82,42.82,0,0,0,298.6,512H597.4A42.82,42.82,0,0,0,640,469.33V458.67A10.67,10.67,0,0,0,629.33,448ZM544,448H352V304H544Z"],"laptop-medical":[640,512,[],"f812","M232 224h56v56a8 8 0 0 0 8 8h48a8 8 0 0 0 8-8v-56h56a8 8 0 0 0 8-8v-48a8 8 0 0 0-8-8h-56v-56a8 8 0 0 0-8-8h-48a8 8 0 0 0-8 8v56h-56a8 8 0 0 0-8 8v48a8 8 0 0 0 8 8zM576 48a48.14 48.14 0 0 0-48-48H112a48.14 48.14 0 0 0-48 48v336h512zm-64 272H128V64h384zm112 96H381.54c-.74 19.81-14.71 32-32.74 32H288c-18.69 0-33-17.47-32.77-32H16a16 16 0 0 0-16 16v16a64.19 64.19 0 0 0 64 64h512a64.19 64.19 0 0 0 64-64v-16a16 16 0 0 0-16-16z"],laugh:[496,512,[],"f599","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm80 152c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zm-160 0c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zm88 272h-16c-73.4 0-134-55-142.9-126-1.2-9.5 6.3-18 15.9-18h270c9.6 0 17.1 8.4 15.9 18-8.9 71-69.5 126-142.9 126z"],"laugh-beam":[496,512,[],"f59a","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm24 199.4c3.3-42.1 32.2-71.4 56-71.4s52.7 29.3 56 71.4c.7 8.6-10.8 11.9-14.9 4.5l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.2 7.4-15.8 4.1-15.1-4.5zm-160 0c3.3-42.1 32.2-71.4 56-71.4s52.7 29.3 56 71.4c.7 8.6-10.8 11.9-14.9 4.5l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.3 7.4-15.8 4-15.1-4.5zM398.9 306C390 377 329.4 432 256 432h-16c-73.4 0-134-55-142.9-126-1.2-9.5 6.3-18 15.9-18h270c9.6 0 17.1 8.4 15.9 18z"],"laugh-squint":[496,512,[],"f59b","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm33.8 161.7l80-48c11.6-6.9 24 7.7 15.4 18L343.6 180l33.6 40.3c8.7 10.4-3.9 24.8-15.4 18l-80-48c-7.7-4.7-7.7-15.9 0-20.6zm-163-30c-8.6-10.3 3.8-24.9 15.4-18l80 48c7.8 4.7 7.8 15.9 0 20.6l-80 48c-11.5 6.8-24-7.6-15.4-18l33.6-40.3-33.6-40.3zM398.9 306C390 377 329.4 432 256 432h-16c-73.4 0-134-55-142.9-126-1.2-9.5 6.3-18 15.9-18h270c9.6 0 17.1 8.4 15.9 18z"],"laugh-wink":[496,512,[],"f59c","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm20.1 198.1c4-25.2 34.2-42.1 59.9-42.1s55.9 16.9 59.9 42.1c1.7 11.1-11.4 18.3-19.8 10.8l-9.5-8.5c-14.8-13.2-46.2-13.2-61 0L288 217c-8.4 7.4-21.6.3-19.9-10.9zM168 160c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zm230.9 146C390 377 329.4 432 256 432h-16c-73.4 0-134-55-142.9-126-1.2-9.5 6.3-18 15.9-18h270c9.6 0 17.1 8.4 15.9 18z"],"layer-group":[512,512,[],"f5fd","M12.41 148.02l232.94 105.67c6.8 3.09 14.49 3.09 21.29 0l232.94-105.67c16.55-7.51 16.55-32.52 0-40.03L266.65 2.31a25.607 25.607 0 0 0-21.29 0L12.41 107.98c-16.55 7.51-16.55 32.53 0 40.04zm487.18 88.28l-58.09-26.33-161.64 73.27c-7.56 3.43-15.59 5.17-23.86 5.17s-16.29-1.74-23.86-5.17L70.51 209.97l-58.1 26.33c-16.55 7.5-16.55 32.5 0 40l232.94 105.59c6.8 3.08 14.49 3.08 21.29 0L499.59 276.3c16.55-7.5 16.55-32.5 0-40zm0 127.8l-57.87-26.23-161.86 73.37c-7.56 3.43-15.59 5.17-23.86 5.17s-16.29-1.74-23.86-5.17L70.29 337.87 12.41 364.1c-16.55 7.5-16.55 32.5 0 40l232.94 105.59c6.8 3.08 14.49 3.08 21.29 0L499.59 404.1c16.55-7.5 16.55-32.5 0-40z"],leaf:[576,512,[],"f06c","M546.2 9.7c-5.6-12.5-21.6-13-28.3-1.2C486.9 62.4 431.4 96 368 96h-80C182 96 96 182 96 288c0 7 .8 13.7 1.5 20.5C161.3 262.8 253.4 224 384 224c8.8 0 16 7.2 16 16s-7.2 16-16 16C132.6 256 26 410.1 2.4 468c-6.6 16.3 1.2 34.9 17.5 41.6 16.4 6.8 35-1.1 41.8-17.3 1.5-3.6 20.9-47.9 71.9-90.6 32.4 43.9 94 85.8 174.9 77.2C465.5 467.5 576 326.7 576 154.3c0-50.2-10.8-102.2-29.8-144.6z"],lemon:[512,512,[],"f094","M489.038 22.963C465.944-.13 434.648-5.93 413.947 6.129c-58.906 34.312-181.25-53.077-321.073 86.746S40.441 355.041 6.129 413.945c-12.059 20.702-6.26 51.999 16.833 75.093 23.095 23.095 54.392 28.891 75.095 16.832 58.901-34.31 181.246 53.079 321.068-86.743S471.56 156.96 505.871 98.056c12.059-20.702 6.261-51.999-16.833-75.093zM243.881 95.522c-58.189 14.547-133.808 90.155-148.358 148.358-1.817 7.27-8.342 12.124-15.511 12.124-1.284 0-2.59-.156-3.893-.481-8.572-2.144-13.784-10.83-11.642-19.403C81.901 166.427 166.316 81.93 236.119 64.478c8.575-2.143 17.261 3.069 19.403 11.642s-3.069 17.259-11.641 19.402z"],"less-than":[384,512,[],"f536","M365.46 357.74L147.04 255.89l218.47-101.88c16.02-7.47 22.95-26.51 15.48-42.53l-13.52-29C360 66.46 340.96 59.53 324.94 67L18.48 209.91a32.014 32.014 0 0 0-18.48 29v34.24c0 12.44 7.21 23.75 18.48 29l306.31 142.83c16.06 7.49 35.15.54 42.64-15.52l13.56-29.08c7.49-16.06.54-35.15-15.53-42.64z"],"less-than-equal":[448,512,[],"f537","M54.98 214.2l301.41 119.87c18.39 6.03 38.71-2.54 45.38-19.15l12.09-30.08c6.68-16.61-2.82-34.97-21.21-41l-175.44-68.05 175.56-68.09c18.29-6 27.74-24.27 21.1-40.79l-12.03-29.92c-6.64-16.53-26.86-25.06-45.15-19.06L54.98 137.89C41.21 142.41 32 154.5 32 168.07v15.96c0 13.56 9.21 25.65 22.98 30.17zM424 400H24c-13.25 0-24 10.74-24 24v48c0 13.25 10.75 24 24 24h400c13.25 0 24-10.75 24-24v-48c0-13.26-10.75-24-24-24z"],"level-down-alt":[320,512,[],"f3be","M313.553 392.331L209.587 504.334c-9.485 10.214-25.676 10.229-35.174 0L70.438 392.331C56.232 377.031 67.062 352 88.025 352H152V80H68.024a11.996 11.996 0 0 1-8.485-3.515l-56-56C-4.021 12.926 1.333 0 12.024 0H208c13.255 0 24 10.745 24 24v328h63.966c20.878 0 31.851 24.969 17.587 40.331z"],"level-up-alt":[320,512,[],"f3bf","M313.553 119.669L209.587 7.666c-9.485-10.214-25.676-10.229-35.174 0L70.438 119.669C56.232 134.969 67.062 160 88.025 160H152v272H68.024a11.996 11.996 0 0 0-8.485 3.515l-56 56C-4.021 499.074 1.333 512 12.024 512H208c13.255 0 24-10.745 24-24V160h63.966c20.878 0 31.851-24.969 17.587-40.331z"],"life-ring":[512,512,[],"f1cd","M256 8C119.033 8 8 119.033 8 256s111.033 248 248 248 248-111.033 248-248S392.967 8 256 8zm173.696 119.559l-63.399 63.399c-10.987-18.559-26.67-34.252-45.255-45.255l63.399-63.399a218.396 218.396 0 0 1 45.255 45.255zM256 352c-53.019 0-96-42.981-96-96s42.981-96 96-96 96 42.981 96 96-42.981 96-96 96zM127.559 82.304l63.399 63.399c-18.559 10.987-34.252 26.67-45.255 45.255l-63.399-63.399a218.372 218.372 0 0 1 45.255-45.255zM82.304 384.441l63.399-63.399c10.987 18.559 26.67 34.252 45.255 45.255l-63.399 63.399a218.396 218.396 0 0 1-45.255-45.255zm302.137 45.255l-63.399-63.399c18.559-10.987 34.252-26.67 45.255-45.255l63.399 63.399a218.403 218.403 0 0 1-45.255 45.255z"],lightbulb:[352,512,[],"f0eb","M96.06 454.35c.01 6.29 1.87 12.45 5.36 17.69l17.09 25.69a31.99 31.99 0 0 0 26.64 14.28h61.71a31.99 31.99 0 0 0 26.64-14.28l17.09-25.69a31.989 31.989 0 0 0 5.36-17.69l.04-38.35H96.01l.05 38.35zM0 176c0 44.37 16.45 84.85 43.56 115.78 16.52 18.85 42.36 58.23 52.21 91.45.04.26.07.52.11.78h160.24c.04-.26.07-.51.11-.78 9.85-33.22 35.69-72.6 52.21-91.45C335.55 260.85 352 220.37 352 176 352 78.61 272.91-.3 175.45 0 73.44.31 0 82.97 0 176zm176-80c-44.11 0-80 35.89-80 80 0 8.84-7.16 16-16 16s-16-7.16-16-16c0-61.76 50.24-112 112-112 8.84 0 16 7.16 16 16s-7.16 16-16 16z"],link:[512,512,[],"f0c1","M326.612 185.391c59.747 59.809 58.927 155.698.36 214.59-.11.12-.24.25-.36.37l-67.2 67.2c-59.27 59.27-155.699 59.262-214.96 0-59.27-59.26-59.27-155.7 0-214.96l37.106-37.106c9.84-9.84 26.786-3.3 27.294 10.606.648 17.722 3.826 35.527 9.69 52.721 1.986 5.822.567 12.262-3.783 16.612l-13.087 13.087c-28.026 28.026-28.905 73.66-1.155 101.96 28.024 28.579 74.086 28.749 102.325.51l67.2-67.19c28.191-28.191 28.073-73.757 0-101.83-3.701-3.694-7.429-6.564-10.341-8.569a16.037 16.037 0 0 1-6.947-12.606c-.396-10.567 3.348-21.456 11.698-29.806l21.054-21.055c5.521-5.521 14.182-6.199 20.584-1.731a152.482 152.482 0 0 1 20.522 17.197zM467.547 44.449c-59.261-59.262-155.69-59.27-214.96 0l-67.2 67.2c-.12.12-.25.25-.36.37-58.566 58.892-59.387 154.781.36 214.59a152.454 152.454 0 0 0 20.521 17.196c6.402 4.468 15.064 3.789 20.584-1.731l21.054-21.055c8.35-8.35 12.094-19.239 11.698-29.806a16.037 16.037 0 0 0-6.947-12.606c-2.912-2.005-6.64-4.875-10.341-8.569-28.073-28.073-28.191-73.639 0-101.83l67.2-67.19c28.239-28.239 74.3-28.069 102.325.51 27.75 28.3 26.872 73.934-1.155 101.96l-13.087 13.087c-4.35 4.35-5.769 10.79-3.783 16.612 5.864 17.194 9.042 34.999 9.69 52.721.509 13.906 17.454 20.446 27.294 10.606l37.106-37.106c59.271-59.259 59.271-155.699.001-214.959z"],"lira-sign":[384,512,[],"f195","M371.994 256h-48.019C317.64 256 312 260.912 312 267.246 312 368 230.179 416 144 416V256.781l134.603-29.912A12 12 0 0 0 288 215.155v-40.976c0-7.677-7.109-13.38-14.603-11.714L144 191.219V160.78l134.603-29.912A12 12 0 0 0 288 119.154V78.179c0-7.677-7.109-13.38-14.603-11.714L144 95.219V44c0-6.627-5.373-12-12-12H76c-6.627 0-12 5.373-12 12v68.997L9.397 125.131A12 12 0 0 0 0 136.845v40.976c0 7.677 7.109 13.38 14.603 11.714L64 178.558v30.439L9.397 221.131A12 12 0 0 0 0 232.845v40.976c0 7.677 7.109 13.38 14.603 11.714L64 274.558V468c0 6.627 5.373 12 12 12h79.583c134.091 0 223.255-77.834 228.408-211.592.261-6.782-5.211-12.408-11.997-12.408z"],list:[512,512,[],"f03a","M80 368H16a16 16 0 0 0-16 16v64a16 16 0 0 0 16 16h64a16 16 0 0 0 16-16v-64a16 16 0 0 0-16-16zm0-320H16A16 16 0 0 0 0 64v64a16 16 0 0 0 16 16h64a16 16 0 0 0 16-16V64a16 16 0 0 0-16-16zm0 160H16a16 16 0 0 0-16 16v64a16 16 0 0 0 16 16h64a16 16 0 0 0 16-16v-64a16 16 0 0 0-16-16zm416 176H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-320H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16V80a16 16 0 0 0-16-16zm0 160H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"],"list-alt":[512,512,[],"f022","M464 480H48c-26.51 0-48-21.49-48-48V80c0-26.51 21.49-48 48-48h416c26.51 0 48 21.49 48 48v352c0 26.51-21.49 48-48 48zM128 120c-22.091 0-40 17.909-40 40s17.909 40 40 40 40-17.909 40-40-17.909-40-40-40zm0 96c-22.091 0-40 17.909-40 40s17.909 40 40 40 40-17.909 40-40-17.909-40-40-40zm0 96c-22.091 0-40 17.909-40 40s17.909 40 40 40 40-17.909 40-40-17.909-40-40-40zm288-136v-32c0-6.627-5.373-12-12-12H204c-6.627 0-12 5.373-12 12v32c0 6.627 5.373 12 12 12h200c6.627 0 12-5.373 12-12zm0 96v-32c0-6.627-5.373-12-12-12H204c-6.627 0-12 5.373-12 12v32c0 6.627 5.373 12 12 12h200c6.627 0 12-5.373 12-12zm0 96v-32c0-6.627-5.373-12-12-12H204c-6.627 0-12 5.373-12 12v32c0 6.627 5.373 12 12 12h200c6.627 0 12-5.373 12-12z"],"list-ol":[512,512,[],"f0cb","M61.77 401l17.5-20.15a19.92 19.92 0 0 0 5.07-14.19v-3.31C84.34 356 80.5 352 73 352H16a8 8 0 0 0-8 8v16a8 8 0 0 0 8 8h22.83a157.41 157.41 0 0 0-11 12.31l-5.61 7c-4 5.07-5.25 10.13-2.8 14.88l1.05 1.93c3 5.76 6.29 7.88 12.25 7.88h4.73c10.33 0 15.94 2.44 15.94 9.09 0 4.72-4.2 8.22-14.36 8.22a41.54 41.54 0 0 1-15.47-3.12c-6.49-3.88-11.74-3.5-15.6 3.12l-5.59 9.31c-3.72 6.13-3.19 11.72 2.63 15.94 7.71 4.69 20.38 9.44 37 9.44 34.16 0 48.5-22.75 48.5-44.12-.03-14.38-9.12-29.76-28.73-34.88zM496 224H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-160H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16V80a16 16 0 0 0-16-16zm0 320H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zM16 160h64a8 8 0 0 0 8-8v-16a8 8 0 0 0-8-8H64V40a8 8 0 0 0-8-8H32a8 8 0 0 0-7.14 4.42l-8 16A8 8 0 0 0 24 64h8v64H16a8 8 0 0 0-8 8v16a8 8 0 0 0 8 8zm-3.91 160H80a8 8 0 0 0 8-8v-16a8 8 0 0 0-8-8H41.32c3.29-10.29 48.34-18.68 48.34-56.44 0-29.06-25-39.56-44.47-39.56-21.36 0-33.8 10-40.46 18.75-4.37 5.59-3 10.84 2.8 15.37l8.58 6.88c5.61 4.56 11 2.47 16.12-2.44a13.44 13.44 0 0 1 9.46-3.84c3.33 0 9.28 1.56 9.28 8.75C51 248.19 0 257.31 0 304.59v4C0 316 5.08 320 12.09 320z"],"list-ul":[512,512,[],"f0ca","M48 48a48 48 0 1 0 48 48 48 48 0 0 0-48-48zm0 160a48 48 0 1 0 48 48 48 48 0 0 0-48-48zm0 160a48 48 0 1 0 48 48 48 48 0 0 0-48-48zm448 16H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-320H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16V80a16 16 0 0 0-16-16zm0 160H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"],"location-arrow":[512,512,[],"f124","M444.52 3.52L28.74 195.42c-47.97 22.39-31.98 92.75 19.19 92.75h175.91v175.91c0 51.17 70.36 67.17 92.75 19.19l191.9-415.78c15.99-38.39-25.59-79.97-63.97-63.97z"],lock:[448,512,[],"f023","M400 224h-24v-72C376 68.2 307.8 0 224 0S72 68.2 72 152v72H48c-26.5 0-48 21.5-48 48v192c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V272c0-26.5-21.5-48-48-48zm-104 0H152v-72c0-39.7 32.3-72 72-72s72 32.3 72 72v72z"],"lock-open":[576,512,[],"f3c1","M423.5 0C339.5.3 272 69.5 272 153.5V224H48c-26.5 0-48 21.5-48 48v192c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V272c0-26.5-21.5-48-48-48h-48v-71.1c0-39.6 31.7-72.5 71.3-72.9 40-.4 72.7 32.1 72.7 72v80c0 13.3 10.7 24 24 24h32c13.3 0 24-10.7 24-24v-80C576 68 507.5-.3 423.5 0z"],"long-arrow-alt-down":[256,512,[],"f309","M168 345.941V44c0-6.627-5.373-12-12-12h-56c-6.627 0-12 5.373-12 12v301.941H41.941c-21.382 0-32.09 25.851-16.971 40.971l86.059 86.059c9.373 9.373 24.569 9.373 33.941 0l86.059-86.059c15.119-15.119 4.411-40.971-16.971-40.971H168z"],"long-arrow-alt-left":[448,512,[],"f30a","M134.059 296H436c6.627 0 12-5.373 12-12v-56c0-6.627-5.373-12-12-12H134.059v-46.059c0-21.382-25.851-32.09-40.971-16.971L7.029 239.029c-9.373 9.373-9.373 24.569 0 33.941l86.059 86.059c15.119 15.119 40.971 4.411 40.971-16.971V296z"],"long-arrow-alt-right":[448,512,[],"f30b","M313.941 216H12c-6.627 0-12 5.373-12 12v56c0 6.627 5.373 12 12 12h301.941v46.059c0 21.382 25.851 32.09 40.971 16.971l86.059-86.059c9.373-9.373 9.373-24.569 0-33.941l-86.059-86.059c-15.119-15.119-40.971-4.411-40.971 16.971V216z"],"long-arrow-alt-up":[256,512,[],"f30c","M88 166.059V468c0 6.627 5.373 12 12 12h56c6.627 0 12-5.373 12-12V166.059h46.059c21.382 0 32.09-25.851 16.971-40.971l-86.059-86.059c-9.373-9.373-24.569-9.373-33.941 0l-86.059 86.059c-15.119 15.119-4.411 40.971 16.971 40.971H88z"],"low-vision":[576,512,[],"f2a8","M569.344 231.631C512.96 135.949 407.81 72 288 72c-28.468 0-56.102 3.619-82.451 10.409L152.778 10.24c-7.601-10.858-22.564-13.5-33.423-5.9l-13.114 9.178c-10.86 7.601-13.502 22.566-5.9 33.426l43.131 58.395C89.449 131.73 40.228 174.683 6.682 231.581c-.01.017-.023.033-.034.05-8.765 14.875-8.964 33.528 0 48.739 38.5 65.332 99.742 115.862 172.859 141.349L55.316 244.302A272.194 272.194 0 0 1 83.61 208.39l119.4 170.58h.01l40.63 58.04a330.055 330.055 0 0 0 78.94 1.17l-189.98-271.4a277.628 277.628 0 0 1 38.777-21.563l251.836 356.544c7.601 10.858 22.564 13.499 33.423 5.9l13.114-9.178c10.86-7.601 13.502-22.567 5.9-33.426l-43.12-58.377-.007-.009c57.161-27.978 104.835-72.04 136.81-126.301a47.938 47.938 0 0 0 .001-48.739zM390.026 345.94l-19.066-27.23c24.682-32.567 27.711-76.353 8.8-111.68v.03c0 23.65-19.17 42.82-42.82 42.82-23.828 0-42.82-19.349-42.82-42.82 0-23.65 19.17-42.82 42.82-42.82h.03c-24.75-13.249-53.522-15.643-79.51-7.68l-19.068-27.237C253.758 123.306 270.488 120 288 120c75.162 0 136 60.826 136 136 0 34.504-12.833 65.975-33.974 89.94z"],"luggage-cart":[640,512,[],"f59d","M224 320h32V96h-32c-17.67 0-32 14.33-32 32v160c0 17.67 14.33 32 32 32zm352-32V128c0-17.67-14.33-32-32-32h-32v224h32c17.67 0 32-14.33 32-32zm48 96H128V16c0-8.84-7.16-16-16-16H16C7.16 0 0 7.16 0 16v32c0 8.84 7.16 16 16 16h48v368c0 8.84 7.16 16 16 16h82.94c-1.79 5.03-2.94 10.36-2.94 16 0 26.51 21.49 48 48 48s48-21.49 48-48c0-5.64-1.15-10.97-2.94-16h197.88c-1.79 5.03-2.94 10.36-2.94 16 0 26.51 21.49 48 48 48s48-21.49 48-48c0-5.64-1.15-10.97-2.94-16H624c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zM480 96V48c0-26.51-21.49-48-48-48h-96c-26.51 0-48 21.49-48 48v272h192V96zm-48 0h-96V48h96v48z"],lungs:[640,512,[],"f604","M636.11 390.15C614.44 308.85 580.07 231 534.1 159.13 511.98 124.56 498.03 96 454.05 96 415.36 96 384 125.42 384 161.71v60.11l-32.88-21.92a15.996 15.996 0 0 1-7.12-13.31V16c0-8.84-7.16-16-16-16h-16c-8.84 0-16 7.16-16 16v170.59c0 5.35-2.67 10.34-7.12 13.31L256 221.82v-60.11C256 125.42 224.64 96 185.95 96c-43.98 0-57.93 28.56-80.05 63.13C59.93 231 25.56 308.85 3.89 390.15 1.3 399.84 0 409.79 0 419.78c0 61.23 62.48 105.44 125.24 88.62l59.5-15.95c42.18-11.3 71.26-47.47 71.26-88.62v-87.49l-85.84 57.23a7.992 7.992 0 0 1-11.09-2.22l-8.88-13.31a7.992 7.992 0 0 1 2.22-11.09L320 235.23l167.59 111.72a7.994 7.994 0 0 1 2.22 11.09l-8.88 13.31a7.994 7.994 0 0 1-11.09 2.22L384 316.34v87.49c0 41.15 29.08 77.31 71.26 88.62l59.5 15.95C577.52 525.22 640 481.01 640 419.78c0-9.99-1.3-19.94-3.89-29.63z"],"lungs-virus":[640,512,[],"f967","M344,150.68V16A16,16,0,0,0,328,0H312a16,16,0,0,0-16,16V150.68a46.45,46.45,0,0,1,48,0ZM195.54,444.46a48.06,48.06,0,0,1,0-67.88l8.58-8.58H192a48,48,0,0,1,0-96h12.12l-8.58-8.57a48,48,0,0,1,60.46-74V161.75C256,125.38,224.62,96,186,96c-44,0-58,28.5-80.12,63.13a819.52,819.52,0,0,0-102,231A113.16,113.16,0,0,0,0,419.75C0,481,62.5,525.26,125.25,508.38l59.5-15.87a98.51,98.51,0,0,0,52.5-34.75,46.49,46.49,0,0,1-41.71-13.3Zm226.29-22.63a16,16,0,0,0,0-22.62l-8.58-8.58C393.09,370.47,407.37,336,435.88,336H448a16,16,0,0,0,0-32H435.88c-28.51,0-42.79-34.47-22.63-54.62l8.58-8.58a16,16,0,0,0-22.63-22.63l-8.57,8.58C370.47,246.91,336,232.63,336,204.12V192a16,16,0,0,0-32,0v12.12c0,28.51-34.47,42.79-54.63,22.63l-8.57-8.58a16,16,0,0,0-22.63,22.63l8.58,8.58c20.16,20.15,5.88,54.62-22.63,54.62H192a16,16,0,0,0,0,32h12.12c28.51,0,42.79,34.47,22.63,54.63l-8.58,8.58a16,16,0,1,0,22.63,22.62l8.57-8.57C269.53,393.1,304,407.38,304,435.88V448a16,16,0,0,0,32,0V435.88c0-28.5,34.47-42.78,54.63-22.62l8.57,8.57a16,16,0,0,0,22.63,0ZM288,304a16,16,0,1,1,16-16A16,16,0,0,1,288,304Zm64,64a16,16,0,1,1,16-16A16,16,0,0,1,352,368Zm284.12,22.13a819.52,819.52,0,0,0-102-231C512,124.5,498,96,454,96c-38.62,0-70,29.38-70,65.75v27.72a48,48,0,0,1,60.46,74L435.88,272H448a48,48,0,0,1,0,96H435.88l8.58,8.58a47.7,47.7,0,0,1-41.71,81.18,98.51,98.51,0,0,0,52.5,34.75l59.5,15.87C577.5,525.26,640,481,640,419.75A113.16,113.16,0,0,0,636.12,390.13Z"],magic:[512,512,[],"f0d0","M224 96l16-32 32-16-32-16-16-32-16 32-32 16 32 16 16 32zM80 160l26.66-53.33L160 80l-53.34-26.67L80 0 53.34 53.33 0 80l53.34 26.67L80 160zm352 128l-26.66 53.33L352 368l53.34 26.67L432 448l26.66-53.33L512 368l-53.34-26.67L432 288zm70.62-193.77L417.77 9.38C411.53 3.12 403.34 0 395.15 0c-8.19 0-16.38 3.12-22.63 9.38L9.38 372.52c-12.5 12.5-12.5 32.76 0 45.25l84.85 84.85c6.25 6.25 14.44 9.37 22.62 9.37 8.19 0 16.38-3.12 22.63-9.37l363.14-363.15c12.5-12.48 12.5-32.75 0-45.24zM359.45 203.46l-50.91-50.91 86.6-86.6 50.91 50.91-86.6 86.6z"],magnet:[512,512,[],"f076","M164.07 148.1H12a12 12 0 0 1-12-12v-80a36 36 0 0 1 36-36h104a36 36 0 0 1 36 36v80a11.89 11.89 0 0 1-11.93 12zm347.93-12V56a36 36 0 0 0-36-36H372a36 36 0 0 0-36 36v80a12 12 0 0 0 12 12h152a11.89 11.89 0 0 0 12-11.9zm-164 44a12 12 0 0 0-12 12v52c0 128.1-160 127.9-160 0v-52a12 12 0 0 0-12-12H12.1a12 12 0 0 0-12 12.1c.1 21.4.6 40.3 0 53.3 0 150.6 136.17 246.6 256.75 246.6s255-96 255-246.7c-.6-12.8-.2-33 0-53.2a12 12 0 0 0-12-12.1z"],"mail-bulk":[576,512,[],"f674","M160 448c-25.6 0-51.2-22.4-64-32-64-44.8-83.2-60.8-96-70.4V480c0 17.67 14.33 32 32 32h256c17.67 0 32-14.33 32-32V345.6c-12.8 9.6-32 25.6-96 70.4-12.8 9.6-38.4 32-64 32zm128-192H32c-17.67 0-32 14.33-32 32v16c25.6 19.2 22.4 19.2 115.2 86.4 9.6 6.4 28.8 25.6 44.8 25.6s35.2-19.2 44.8-22.4c92.8-67.2 89.6-67.2 115.2-86.4V288c0-17.67-14.33-32-32-32zm256-96H224c-17.67 0-32 14.33-32 32v32h96c33.21 0 60.59 25.42 63.71 57.82l.29-.22V416h192c17.67 0 32-14.33 32-32V192c0-17.67-14.33-32-32-32zm-32 128h-64v-64h64v64zm-352-96c0-35.29 28.71-64 64-64h224V32c0-17.67-14.33-32-32-32H96C78.33 0 64 14.33 64 32v192h96v-32z"],male:[192,512,[],"f183","M96 0c35.346 0 64 28.654 64 64s-28.654 64-64 64-64-28.654-64-64S60.654 0 96 0m48 144h-11.36c-22.711 10.443-49.59 10.894-73.28 0H48c-26.51 0-48 21.49-48 48v136c0 13.255 10.745 24 24 24h16v136c0 13.255 10.745 24 24 24h64c13.255 0 24-10.745 24-24V352h16c13.255 0 24-10.745 24-24V192c0-26.51-21.49-48-48-48z"],map:[576,512,[],"f279","M0 117.66v346.32c0 11.32 11.43 19.06 21.94 14.86L160 416V32L20.12 87.95A32.006 32.006 0 0 0 0 117.66zM192 416l192 64V96L192 32v384zM554.06 33.16L416 96v384l139.88-55.95A31.996 31.996 0 0 0 576 394.34V48.02c0-11.32-11.43-19.06-21.94-14.86z"],"map-marked":[576,512,[],"f59f","M288 0c-69.59 0-126 56.41-126 126 0 56.26 82.35 158.8 113.9 196.02 6.39 7.54 17.82 7.54 24.2 0C331.65 284.8 414 182.26 414 126 414 56.41 357.59 0 288 0zM20.12 215.95A32.006 32.006 0 0 0 0 245.66v250.32c0 11.32 11.43 19.06 21.94 14.86L160 448V214.92c-8.84-15.98-16.07-31.54-21.25-46.42L20.12 215.95zM288 359.67c-14.07 0-27.38-6.18-36.51-16.96-19.66-23.2-40.57-49.62-59.49-76.72v182l192 64V266c-18.92 27.09-39.82 53.52-59.49 76.72-9.13 10.77-22.44 16.95-36.51 16.95zm266.06-198.51L416 224v288l139.88-55.95A31.996 31.996 0 0 0 576 426.34V176.02c0-11.32-11.43-19.06-21.94-14.86z"],"map-marked-alt":[576,512,[],"f5a0","M288 0c-69.59 0-126 56.41-126 126 0 56.26 82.35 158.8 113.9 196.02 6.39 7.54 17.82 7.54 24.2 0C331.65 284.8 414 182.26 414 126 414 56.41 357.59 0 288 0zm0 168c-23.2 0-42-18.8-42-42s18.8-42 42-42 42 18.8 42 42-18.8 42-42 42zM20.12 215.95A32.006 32.006 0 0 0 0 245.66v250.32c0 11.32 11.43 19.06 21.94 14.86L160 448V214.92c-8.84-15.98-16.07-31.54-21.25-46.42L20.12 215.95zM288 359.67c-14.07 0-27.38-6.18-36.51-16.96-19.66-23.2-40.57-49.62-59.49-76.72v182l192 64V266c-18.92 27.09-39.82 53.52-59.49 76.72-9.13 10.77-22.44 16.95-36.51 16.95zm266.06-198.51L416 224v288l139.88-55.95A31.996 31.996 0 0 0 576 426.34V176.02c0-11.32-11.43-19.06-21.94-14.86z"],"map-marker":[384,512,[],"f041","M172.268 501.67C26.97 291.031 0 269.413 0 192 0 85.961 85.961 0 192 0s192 85.961 192 192c0 77.413-26.97 99.031-172.268 309.67-9.535 13.774-29.93 13.773-39.464 0z"],"map-marker-alt":[384,512,[],"f3c5","M172.268 501.67C26.97 291.031 0 269.413 0 192 0 85.961 85.961 0 192 0s192 85.961 192 192c0 77.413-26.97 99.031-172.268 309.67-9.535 13.774-29.93 13.773-39.464 0zM192 272c44.183 0 80-35.817 80-80s-35.817-80-80-80-80 35.817-80 80 35.817 80 80 80z"],"map-pin":[288,512,[],"f276","M112 316.94v156.69l22.02 33.02c4.75 7.12 15.22 7.12 19.97 0L176 473.63V316.94c-10.39 1.92-21.06 3.06-32 3.06s-21.61-1.14-32-3.06zM144 0C64.47 0 0 64.47 0 144s64.47 144 144 144 144-64.47 144-144S223.53 0 144 0zm0 76c-37.5 0-68 30.5-68 68 0 6.62-5.38 12-12 12s-12-5.38-12-12c0-50.73 41.28-92 92-92 6.62 0 12 5.38 12 12s-5.38 12-12 12z"],"map-signs":[512,512,[],"f277","M507.31 84.69L464 41.37c-6-6-14.14-9.37-22.63-9.37H288V16c0-8.84-7.16-16-16-16h-32c-8.84 0-16 7.16-16 16v16H56c-13.25 0-24 10.75-24 24v80c0 13.25 10.75 24 24 24h385.37c8.49 0 16.62-3.37 22.63-9.37l43.31-43.31c6.25-6.26 6.25-16.38 0-22.63zM224 496c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16V384h-64v112zm232-272H288v-32h-64v32H70.63c-8.49 0-16.62 3.37-22.63 9.37L4.69 276.69c-6.25 6.25-6.25 16.38 0 22.63L48 342.63c6 6 14.14 9.37 22.63 9.37H456c13.25 0 24-10.75 24-24v-80c0-13.25-10.75-24-24-24z"],marker:[512,512,[],"f5a1","M93.95 290.03A327.038 327.038 0 0 0 .17 485.11l-.03.23c-1.7 15.28 11.21 28.2 26.49 26.51a327.02 327.02 0 0 0 195.34-93.8l75.4-75.4-128.02-128.02-75.4 75.4zM485.49 26.51c-35.35-35.35-92.67-35.35-128.02 0l-21.76 21.76-36.56-36.55c-15.62-15.62-40.95-15.62-56.56 0L138.47 115.84c-6.25 6.25-6.25 16.38 0 22.63l22.62 22.62c6.25 6.25 16.38 6.25 22.63 0l87.15-87.15 19.59 19.59L191.98 192 320 320.02l165.49-165.49c35.35-35.35 35.35-92.66 0-128.02z"],mars:[384,512,[],"f222","M372 64h-79c-10.7 0-16 12.9-8.5 20.5l16.9 16.9-80.7 80.7c-22.2-14-48.5-22.1-76.7-22.1C64.5 160 0 224.5 0 304s64.5 144 144 144 144-64.5 144-144c0-28.2-8.1-54.5-22.1-76.7l80.7-80.7 16.9 16.9c7.6 7.6 20.5 2.2 20.5-8.5V76c0-6.6-5.4-12-12-12zM144 384c-44.1 0-80-35.9-80-80s35.9-80 80-80 80 35.9 80 80-35.9 80-80 80z"],"mars-double":[512,512,[],"f227","M340 0h-79c-10.7 0-16 12.9-8.5 20.5l16.9 16.9-48.7 48.7C198.5 72.1 172.2 64 144 64 64.5 64 0 128.5 0 208s64.5 144 144 144 144-64.5 144-144c0-28.2-8.1-54.5-22.1-76.7l48.7-48.7 16.9 16.9c2.4 2.4 5.5 3.5 8.4 3.5 6.2 0 12.1-4.8 12.1-12V12c0-6.6-5.4-12-12-12zM144 288c-44.1 0-80-35.9-80-80s35.9-80 80-80 80 35.9 80 80-35.9 80-80 80zm356-128.1h-79c-10.7 0-16 12.9-8.5 20.5l16.9 16.9-48.7 48.7c-18.2-11.4-39-18.9-61.5-21.3-2.1 21.8-8.2 43.3-18.4 63.3 1.1 0 2.2-.1 3.2-.1 44.1 0 80 35.9 80 80s-35.9 80-80 80-80-35.9-80-80c0-1.1 0-2.2.1-3.2-20 10.2-41.5 16.4-63.3 18.4C168.4 455.6 229.6 512 304 512c79.5 0 144-64.5 144-144 0-28.2-8.1-54.5-22.1-76.7l48.7-48.7 16.9 16.9c2.4 2.4 5.4 3.5 8.4 3.5 6.2 0 12.1-4.8 12.1-12v-79c0-6.7-5.4-12.1-12-12.1z"],"mars-stroke":[384,512,[],"f229","M372 64h-79c-10.7 0-16 12.9-8.5 20.5l16.9 16.9-17.5 17.5-14.1-14.1c-4.7-4.7-12.3-4.7-17 0L224.5 133c-4.7 4.7-4.7 12.3 0 17l14.1 14.1-18 18c-22.2-14-48.5-22.1-76.7-22.1C64.5 160 0 224.5 0 304s64.5 144 144 144 144-64.5 144-144c0-28.2-8.1-54.5-22.1-76.7l18-18 14.1 14.1c4.7 4.7 12.3 4.7 17 0l28.3-28.3c4.7-4.7 4.7-12.3 0-17L329.2 164l17.5-17.5 16.9 16.9c7.6 7.6 20.5 2.2 20.5-8.5V76c-.1-6.6-5.5-12-12.1-12zM144 384c-44.1 0-80-35.9-80-80s35.9-80 80-80 80 35.9 80 80-35.9 80-80 80z"],"mars-stroke-h":[480,512,[],"f22b","M476.2 247.5l-55.9-55.9c-7.6-7.6-20.5-2.2-20.5 8.5V224H376v-20c0-6.6-5.4-12-12-12h-40c-6.6 0-12 5.4-12 12v20h-27.6c-5.8-25.6-18.7-49.9-38.6-69.8C189.6 98 98.4 98 42.2 154.2c-56.2 56.2-56.2 147.4 0 203.6 56.2 56.2 147.4 56.2 203.6 0 19.9-19.9 32.8-44.2 38.6-69.8H312v20c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12v-20h23.9v23.9c0 10.7 12.9 16 20.5 8.5l55.9-55.9c4.6-4.7 4.6-12.3-.1-17zm-275.6 65.1c-31.2 31.2-81.9 31.2-113.1 0-31.2-31.2-31.2-81.9 0-113.1 31.2-31.2 81.9-31.2 113.1 0 31.2 31.1 31.2 81.9 0 113.1z"],"mars-stroke-v":[288,512,[],"f22a","M245.8 234.2c-19.9-19.9-44.2-32.8-69.8-38.6v-25.4h20c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-20V81.4h23.9c10.7 0 16-12.9 8.5-20.5L152.5 5.1c-4.7-4.7-12.3-4.7-17 0L79.6 61c-7.6 7.6-2.2 20.5 8.5 20.5H112v24.7H92c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h20v25.4c-25.6 5.8-49.9 18.7-69.8 38.6-56.2 56.2-56.2 147.4 0 203.6 56.2 56.2 147.4 56.2 203.6 0 56.3-56.2 56.3-147.4 0-203.6zm-45.2 158.4c-31.2 31.2-81.9 31.2-113.1 0-31.2-31.2-31.2-81.9 0-113.1 31.2-31.2 81.9-31.2 113.1 0 31.2 31.1 31.2 81.9 0 113.1z"],mask:[640,512,[],"f6fa","M320.67 64c-442.6 0-357.57 384-158.46 384 39.9 0 77.47-20.69 101.42-55.86l25.73-37.79c15.66-22.99 46.97-22.99 62.63 0l25.73 37.79C401.66 427.31 439.23 448 479.13 448c189.86 0 290.63-384-158.46-384zM184 308.36c-41.06 0-67.76-25.66-80.08-41.05-5.23-6.53-5.23-16.09 0-22.63 12.32-15.4 39.01-41.05 80.08-41.05s67.76 25.66 80.08 41.05c5.23 6.53 5.23 16.09 0 22.63-12.32 15.4-39.02 41.05-80.08 41.05zm272 0c-41.06 0-67.76-25.66-80.08-41.05-5.23-6.53-5.23-16.09 0-22.63 12.32-15.4 39.01-41.05 80.08-41.05s67.76 25.66 80.08 41.05c5.23 6.53 5.23 16.09 0 22.63-12.32 15.4-39.02 41.05-80.08 41.05z"],medal:[512,512,[],"f5a2","M223.75 130.75L154.62 15.54A31.997 31.997 0 0 0 127.18 0H16.03C3.08 0-4.5 14.57 2.92 25.18l111.27 158.96c29.72-27.77 67.52-46.83 109.56-53.39zM495.97 0H384.82c-11.24 0-21.66 5.9-27.44 15.54l-69.13 115.21c42.04 6.56 79.84 25.62 109.56 53.38L509.08 25.18C516.5 14.57 508.92 0 495.97 0zM256 160c-97.2 0-176 78.8-176 176s78.8 176 176 176 176-78.8 176-176-78.8-176-176-176zm92.52 157.26l-37.93 36.96 8.97 52.22c1.6 9.36-8.26 16.51-16.65 12.09L256 393.88l-46.9 24.65c-8.4 4.45-18.25-2.74-16.65-12.09l8.97-52.22-37.93-36.96c-6.82-6.64-3.05-18.23 6.35-19.59l52.43-7.64 23.43-47.52c2.11-4.28 6.19-6.39 10.28-6.39 4.11 0 8.22 2.14 10.33 6.39l23.43 47.52 52.43 7.64c9.4 1.36 13.17 12.95 6.35 19.59z"],medkit:[512,512,[],"f0fa","M96 480h320V128h-32V80c0-26.51-21.49-48-48-48H176c-26.51 0-48 21.49-48 48v48H96v352zm96-384h128v32H192V96zm320 80v256c0 26.51-21.49 48-48 48h-16V128h16c26.51 0 48 21.49 48 48zM64 480H48c-26.51 0-48-21.49-48-48V176c0-26.51 21.49-48 48-48h16v352zm288-208v32c0 8.837-7.163 16-16 16h-48v48c0 8.837-7.163 16-16 16h-32c-8.837 0-16-7.163-16-16v-48h-48c-8.837 0-16-7.163-16-16v-32c0-8.837 7.163-16 16-16h48v-48c0-8.837 7.163-16 16-16h32c8.837 0 16 7.163 16 16v48h48c8.837 0 16 7.163 16 16z"],meh:[496,512,[],"f11a","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm-80 168c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zm176 192H152c-21.2 0-21.2-32 0-32h192c21.2 0 21.2 32 0 32zm-16-128c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],"meh-blank":[496,512,[],"f5a4","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm-80 232c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm160 0c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],"meh-rolling-eyes":[496,512,[],"f5a5","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zM88 224c0-24.3 13.7-45.2 33.6-56-.7 2.6-1.6 5.2-1.6 8 0 17.7 14.3 32 32 32s32-14.3 32-32c0-2.8-.9-5.4-1.6-8 19.9 10.8 33.6 31.7 33.6 56 0 35.3-28.7 64-64 64s-64-28.7-64-64zm224 176H184c-21.2 0-21.2-32 0-32h128c21.2 0 21.2 32 0 32zm32-112c-35.3 0-64-28.7-64-64 0-24.3 13.7-45.2 33.6-56-.7 2.6-1.6 5.2-1.6 8 0 17.7 14.3 32 32 32s32-14.3 32-32c0-2.8-.9-5.4-1.6-8 19.9 10.8 33.6 31.7 33.6 56 0 35.3-28.7 64-64 64z"],memory:[640,512,[],"f538","M640 130.94V96c0-17.67-14.33-32-32-32H32C14.33 64 0 78.33 0 96v34.94c18.6 6.61 32 24.19 32 45.06s-13.4 38.45-32 45.06V320h640v-98.94c-18.6-6.61-32-24.19-32-45.06s13.4-38.45 32-45.06zM224 256h-64V128h64v128zm128 0h-64V128h64v128zm128 0h-64V128h64v128zM0 448h64v-26.67c0-8.84 7.16-16 16-16s16 7.16 16 16V448h128v-26.67c0-8.84 7.16-16 16-16s16 7.16 16 16V448h128v-26.67c0-8.84 7.16-16 16-16s16 7.16 16 16V448h128v-26.67c0-8.84 7.16-16 16-16s16 7.16 16 16V448h64v-96H0v96z"],menorah:[640,512,[],"f676","M144 128h-32c-8.84 0-16 7.16-16 16v144h64V144c0-8.84-7.16-16-16-16zm96 0h-32c-8.84 0-16 7.16-16 16v144h64V144c0-8.84-7.16-16-16-16zm192 0h-32c-8.84 0-16 7.16-16 16v144h64V144c0-8.84-7.16-16-16-16zm96 0h-32c-8.84 0-16 7.16-16 16v144h64V144c0-8.84-7.16-16-16-16zm80-32c17.67 0 32-14.33 32-32S608 0 608 0s-32 46.33-32 64 14.33 32 32 32zm-96 0c17.67 0 32-14.33 32-32S512 0 512 0s-32 46.33-32 64 14.33 32 32 32zm-96 0c17.67 0 32-14.33 32-32S416 0 416 0s-32 46.33-32 64 14.33 32 32 32zm-96 0c17.67 0 32-14.33 32-32S320 0 320 0s-32 46.33-32 64 14.33 32 32 32zm-96 0c17.67 0 32-14.33 32-32S224 0 224 0s-32 46.33-32 64 14.33 32 32 32zm-96 0c17.67 0 32-14.33 32-32S128 0 128 0 96 46.33 96 64s14.33 32 32 32zm-96 0c17.67 0 32-14.33 32-32S32 0 32 0 0 46.33 0 64s14.33 32 32 32zm544 192c0 17.67-14.33 32-32 32H352V144c0-8.84-7.16-16-16-16h-32c-8.84 0-16 7.16-16 16v176H96c-17.67 0-32-14.33-32-32V144c0-8.84-7.16-16-16-16H16c-8.84 0-16 7.16-16 16v144c0 53.02 42.98 96 96 96h192v64H112c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h416c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16H352v-64h192c53.02 0 96-42.98 96-96V144c0-8.84-7.16-16-16-16h-32c-8.84 0-16 7.16-16 16v144z"],mercury:[288,512,[],"f223","M288 208c0-44.2-19.9-83.7-51.2-110.1 2.5-1.8 4.9-3.8 7.2-5.8 24.7-21.2 39.8-48.8 43.2-78.8.9-7.1-4.7-13.3-11.9-13.3h-40.5C229 0 224.1 4.1 223 9.8c-2.4 12.5-9.6 24.3-20.7 33.8C187 56.8 166.3 64 144 64s-43-7.2-58.4-20.4C74.5 34.1 67.4 22.3 64.9 9.8 63.8 4.1 58.9 0 53.2 0H12.7C5.5 0-.1 6.2.8 13.3 4.2 43.4 19.2 71 44 92.2c2.3 2 4.7 3.9 7.2 5.8C19.9 124.3 0 163.8 0 208c0 68.5 47.9 125.9 112 140.4V400H76c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h36v36c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12v-36h36c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-36v-51.6c64.1-14.5 112-71.9 112-140.4zm-224 0c0-44.1 35.9-80 80-80s80 35.9 80 80-35.9 80-80 80-80-35.9-80-80z"],meteor:[512,512,[],"f753","M511.328,20.8027c-11.60759,38.70264-34.30724,111.70173-61.30311,187.70077,6.99893,2.09372,13.4042,4,18.60653,5.59368a16.06158,16.06158,0,0,1,9.49854,22.906c-22.106,42.29635-82.69047,152.795-142.47819,214.40356-.99984,1.09373-1.99969,2.5-2.99954,3.49995A194.83046,194.83046,0,1,1,57.085,179.41009c.99985-1,2.40588-2,3.49947-3,61.59994-59.90549,171.97367-120.40473,214.37343-142.4982a16.058,16.058,0,0,1,22.90274,9.49988c1.59351,5.09368,3.49947,11.5936,5.5929,18.59351C379.34818,35.00565,452.43074,12.30281,491.12794.70921A16.18325,16.18325,0,0,1,511.328,20.8027ZM319.951,320.00207A127.98041,127.98041,0,1,0,191.97061,448.00046,127.97573,127.97573,0,0,0,319.951,320.00207Zm-127.98041-31.9996a31.9951,31.9951,0,1,1-31.9951-31.9996A31.959,31.959,0,0,1,191.97061,288.00247Zm31.9951,79.999a15.99755,15.99755,0,1,1-15.99755-15.9998A16.04975,16.04975,0,0,1,223.96571,368.00147Z"],microchip:[512,512,[],"f2db","M416 48v416c0 26.51-21.49 48-48 48H144c-26.51 0-48-21.49-48-48V48c0-26.51 21.49-48 48-48h224c26.51 0 48 21.49 48 48zm96 58v12a6 6 0 0 1-6 6h-18v6a6 6 0 0 1-6 6h-42V88h42a6 6 0 0 1 6 6v6h18a6 6 0 0 1 6 6zm0 96v12a6 6 0 0 1-6 6h-18v6a6 6 0 0 1-6 6h-42v-48h42a6 6 0 0 1 6 6v6h18a6 6 0 0 1 6 6zm0 96v12a6 6 0 0 1-6 6h-18v6a6 6 0 0 1-6 6h-42v-48h42a6 6 0 0 1 6 6v6h18a6 6 0 0 1 6 6zm0 96v12a6 6 0 0 1-6 6h-18v6a6 6 0 0 1-6 6h-42v-48h42a6 6 0 0 1 6 6v6h18a6 6 0 0 1 6 6zM30 376h42v48H30a6 6 0 0 1-6-6v-6H6a6 6 0 0 1-6-6v-12a6 6 0 0 1 6-6h18v-6a6 6 0 0 1 6-6zm0-96h42v48H30a6 6 0 0 1-6-6v-6H6a6 6 0 0 1-6-6v-12a6 6 0 0 1 6-6h18v-6a6 6 0 0 1 6-6zm0-96h42v48H30a6 6 0 0 1-6-6v-6H6a6 6 0 0 1-6-6v-12a6 6 0 0 1 6-6h18v-6a6 6 0 0 1 6-6zm0-96h42v48H30a6 6 0 0 1-6-6v-6H6a6 6 0 0 1-6-6v-12a6 6 0 0 1 6-6h18v-6a6 6 0 0 1 6-6z"],microphone:[352,512,[],"f130","M176 352c53.02 0 96-42.98 96-96V96c0-53.02-42.98-96-96-96S80 42.98 80 96v160c0 53.02 42.98 96 96 96zm160-160h-16c-8.84 0-16 7.16-16 16v48c0 74.8-64.49 134.82-140.79 127.38C96.71 376.89 48 317.11 48 250.3V208c0-8.84-7.16-16-16-16H16c-8.84 0-16 7.16-16 16v40.16c0 89.64 63.97 169.55 152 181.69V464H96c-8.84 0-16 7.16-16 16v16c0 8.84 7.16 16 16 16h160c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16h-56v-33.77C285.71 418.47 352 344.9 352 256v-48c0-8.84-7.16-16-16-16z"],"microphone-alt":[352,512,[],"f3c9","M336 192h-16c-8.84 0-16 7.16-16 16v48c0 74.8-64.49 134.82-140.79 127.38C96.71 376.89 48 317.11 48 250.3V208c0-8.84-7.16-16-16-16H16c-8.84 0-16 7.16-16 16v40.16c0 89.64 63.97 169.55 152 181.69V464H96c-8.84 0-16 7.16-16 16v16c0 8.84 7.16 16 16 16h160c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16h-56v-33.77C285.71 418.47 352 344.9 352 256v-48c0-8.84-7.16-16-16-16zM176 352c53.02 0 96-42.98 96-96h-85.33c-5.89 0-10.67-3.58-10.67-8v-16c0-4.42 4.78-8 10.67-8H272v-32h-85.33c-5.89 0-10.67-3.58-10.67-8v-16c0-4.42 4.78-8 10.67-8H272v-32h-85.33c-5.89 0-10.67-3.58-10.67-8v-16c0-4.42 4.78-8 10.67-8H272c0-53.02-42.98-96-96-96S80 42.98 80 96v160c0 53.02 42.98 96 96 96z"],"microphone-alt-slash":[640,512,[],"f539","M633.82 458.1L476.26 336.33C488.74 312.21 496 284.98 496 256v-48c0-8.84-7.16-16-16-16h-16c-8.84 0-16 7.16-16 16v48c0 17.92-3.96 34.8-10.72 50.2l-26.55-20.52c3.1-9.4 5.28-19.22 5.28-29.67h-43.67l-41.4-32H416v-32h-85.33c-5.89 0-10.67-3.58-10.67-8v-16c0-4.42 4.78-8 10.67-8H416v-32h-85.33c-5.89 0-10.67-3.58-10.67-8v-16c0-4.42 4.78-8 10.67-8H416c0-53.02-42.98-96-96-96s-96 42.98-96 96v45.36L45.47 3.37C38.49-2.05 28.43-.8 23.01 6.18L3.37 31.45C-2.05 38.42-.8 48.47 6.18 53.9l588.36 454.73c6.98 5.43 17.03 4.17 22.46-2.81l19.64-25.27c5.41-6.97 4.16-17.02-2.82-22.45zM400 464h-56v-33.78c11.71-1.62 23.1-4.28 33.96-8.08l-50.4-38.96c-6.71.4-13.41.87-20.35.2-55.85-5.45-98.74-48.63-111.18-101.85L144 241.31v6.85c0 89.64 63.97 169.55 152 181.69V464h-56c-8.84 0-16 7.16-16 16v16c0 8.84 7.16 16 16 16h160c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16z"],"microphone-slash":[640,512,[],"f131","M633.82 458.1l-157.8-121.96C488.61 312.13 496 285.01 496 256v-48c0-8.84-7.16-16-16-16h-16c-8.84 0-16 7.16-16 16v48c0 17.92-3.96 34.8-10.72 50.2l-26.55-20.52c3.1-9.4 5.28-19.22 5.28-29.67V96c0-53.02-42.98-96-96-96s-96 42.98-96 96v45.36L45.47 3.37C38.49-2.05 28.43-.8 23.01 6.18L3.37 31.45C-2.05 38.42-.8 48.47 6.18 53.9l588.36 454.73c6.98 5.43 17.03 4.17 22.46-2.81l19.64-25.27c5.41-6.97 4.16-17.02-2.82-22.45zM400 464h-56v-33.77c11.66-1.6 22.85-4.54 33.67-8.31l-50.11-38.73c-6.71.4-13.41.87-20.35.2-55.85-5.45-98.74-48.63-111.18-101.85L144 241.31v6.85c0 89.64 63.97 169.55 152 181.69V464h-56c-8.84 0-16 7.16-16 16v16c0 8.84 7.16 16 16 16h160c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16z"],microscope:[512,512,[],"f610","M160 320h12v16c0 8.84 7.16 16 16 16h40c8.84 0 16-7.16 16-16v-16h12c17.67 0 32-14.33 32-32V64c0-17.67-14.33-32-32-32V16c0-8.84-7.16-16-16-16h-64c-8.84 0-16 7.16-16 16v16c-17.67 0-32 14.33-32 32v224c0 17.67 14.33 32 32 32zm304 128h-1.29C493.24 413.99 512 369.2 512 320c0-105.88-86.12-192-192-192v64c70.58 0 128 57.42 128 128s-57.42 128-128 128H48c-26.51 0-48 21.49-48 48 0 8.84 7.16 16 16 16h480c8.84 0 16-7.16 16-16 0-26.51-21.49-48-48-48zm-360-32h208c4.42 0 8-3.58 8-8v-16c0-4.42-3.58-8-8-8H104c-4.42 0-8 3.58-8 8v16c0 4.42 3.58 8 8 8z"],minus:[448,512,[],"f068","M416 208H32c-17.67 0-32 14.33-32 32v32c0 17.67 14.33 32 32 32h384c17.67 0 32-14.33 32-32v-32c0-17.67-14.33-32-32-32z"],"minus-circle":[512,512,[],"f056","M256 8C119 8 8 119 8 256s111 248 248 248 248-111 248-248S393 8 256 8zM124 296c-6.6 0-12-5.4-12-12v-56c0-6.6 5.4-12 12-12h264c6.6 0 12 5.4 12 12v56c0 6.6-5.4 12-12 12H124z"],"minus-square":[448,512,[],"f146","M400 32H48C21.5 32 0 53.5 0 80v352c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V80c0-26.5-21.5-48-48-48zM92 296c-6.6 0-12-5.4-12-12v-56c0-6.6 5.4-12 12-12h264c6.6 0 12 5.4 12 12v56c0 6.6-5.4 12-12 12H92z"],mitten:[448,512,[],"f7b5","M368 416H48c-8.8 0-16 7.2-16 16v64c0 8.8 7.2 16 16 16h320c8.8 0 16-7.2 16-16v-64c0-8.8-7.2-16-16-16zm57-209.1c-27.2-22.6-67.5-19-90.1 8.2l-20.9 25-29.6-128.4c-18-77.5-95.4-125.9-172.8-108C34.2 21.6-14.2 98.9 3.7 176.4L51.6 384h309l72.5-87c22.7-27.2 19-67.5-8.1-90.1z"],mobile:[320,512,[],"f10b","M272 0H48C21.5 0 0 21.5 0 48v416c0 26.5 21.5 48 48 48h224c26.5 0 48-21.5 48-48V48c0-26.5-21.5-48-48-48zM160 480c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],"mobile-alt":[320,512,[],"f3cd","M272 0H48C21.5 0 0 21.5 0 48v416c0 26.5 21.5 48 48 48h224c26.5 0 48-21.5 48-48V48c0-26.5-21.5-48-48-48zM160 480c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm112-108c0 6.6-5.4 12-12 12H60c-6.6 0-12-5.4-12-12V60c0-6.6 5.4-12 12-12h200c6.6 0 12 5.4 12 12v312z"],"money-bill":[640,512,[],"f0d6","M608 64H32C14.33 64 0 78.33 0 96v320c0 17.67 14.33 32 32 32h576c17.67 0 32-14.33 32-32V96c0-17.67-14.33-32-32-32zM48 400v-64c35.35 0 64 28.65 64 64H48zm0-224v-64h64c0 35.35-28.65 64-64 64zm272 176c-44.19 0-80-42.99-80-96 0-53.02 35.82-96 80-96s80 42.98 80 96c0 53.03-35.83 96-80 96zm272 48h-64c0-35.35 28.65-64 64-64v64zm0-224c-35.35 0-64-28.65-64-64h64v64z"],"money-bill-alt":[640,512,[],"f3d1","M352 288h-16v-88c0-4.42-3.58-8-8-8h-13.58c-4.74 0-9.37 1.4-13.31 4.03l-15.33 10.22a7.994 7.994 0 0 0-2.22 11.09l8.88 13.31a7.994 7.994 0 0 0 11.09 2.22l.47-.31V288h-16c-4.42 0-8 3.58-8 8v16c0 4.42 3.58 8 8 8h64c4.42 0 8-3.58 8-8v-16c0-4.42-3.58-8-8-8zM608 64H32C14.33 64 0 78.33 0 96v320c0 17.67 14.33 32 32 32h576c17.67 0 32-14.33 32-32V96c0-17.67-14.33-32-32-32zM48 400v-64c35.35 0 64 28.65 64 64H48zm0-224v-64h64c0 35.35-28.65 64-64 64zm272 192c-53.02 0-96-50.15-96-112 0-61.86 42.98-112 96-112s96 50.14 96 112c0 61.87-43 112-96 112zm272 32h-64c0-35.35 28.65-64 64-64v64zm0-224c-35.35 0-64-28.65-64-64h64v64z"],"money-bill-wave":[640,512,[],"f53a","M621.16 54.46C582.37 38.19 543.55 32 504.75 32c-123.17-.01-246.33 62.34-369.5 62.34-30.89 0-61.76-3.92-92.65-13.72-3.47-1.1-6.95-1.62-10.35-1.62C15.04 79 0 92.32 0 110.81v317.26c0 12.63 7.23 24.6 18.84 29.46C57.63 473.81 96.45 480 135.25 480c123.17 0 246.34-62.35 369.51-62.35 30.89 0 61.76 3.92 92.65 13.72 3.47 1.1 6.95 1.62 10.35 1.62 17.21 0 32.25-13.32 32.25-31.81V83.93c-.01-12.64-7.24-24.6-18.85-29.47zM48 132.22c20.12 5.04 41.12 7.57 62.72 8.93C104.84 170.54 79 192.69 48 192.69v-60.47zm0 285v-47.78c34.37 0 62.18 27.27 63.71 61.4-22.53-1.81-43.59-6.31-63.71-13.62zM320 352c-44.19 0-80-42.99-80-96 0-53.02 35.82-96 80-96s80 42.98 80 96c0 53.03-35.83 96-80 96zm272 27.78c-17.52-4.39-35.71-6.85-54.32-8.44 5.87-26.08 27.5-45.88 54.32-49.28v57.72zm0-236.11c-30.89-3.91-54.86-29.7-55.81-61.55 19.54 2.17 38.09 6.23 55.81 12.66v48.89z"],"money-bill-wave-alt":[640,512,[],"f53b","M621.16 54.46C582.37 38.19 543.55 32 504.75 32c-123.17-.01-246.33 62.34-369.5 62.34-30.89 0-61.76-3.92-92.65-13.72-3.47-1.1-6.95-1.62-10.35-1.62C15.04 79 0 92.32 0 110.81v317.26c0 12.63 7.23 24.6 18.84 29.46C57.63 473.81 96.45 480 135.25 480c123.17 0 246.34-62.35 369.51-62.35 30.89 0 61.76 3.92 92.65 13.72 3.47 1.1 6.95 1.62 10.35 1.62 17.21 0 32.25-13.32 32.25-31.81V83.93c-.01-12.64-7.24-24.6-18.85-29.47zM320 352c-44.19 0-80-42.99-80-96 0-53.02 35.82-96 80-96s80 42.98 80 96c0 53.03-35.83 96-80 96z"],"money-check":[640,512,[],"f53c","M0 448c0 17.67 14.33 32 32 32h576c17.67 0 32-14.33 32-32V128H0v320zm448-208c0-8.84 7.16-16 16-16h96c8.84 0 16 7.16 16 16v32c0 8.84-7.16 16-16 16h-96c-8.84 0-16-7.16-16-16v-32zm0 120c0-4.42 3.58-8 8-8h112c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H456c-4.42 0-8-3.58-8-8v-16zM64 264c0-4.42 3.58-8 8-8h304c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8v-16zm0 96c0-4.42 3.58-8 8-8h176c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8v-16zM624 32H16C7.16 32 0 39.16 0 48v48h640V48c0-8.84-7.16-16-16-16z"],"money-check-alt":[640,512,[],"f53d","M608 32H32C14.33 32 0 46.33 0 64v384c0 17.67 14.33 32 32 32h576c17.67 0 32-14.33 32-32V64c0-17.67-14.33-32-32-32zM176 327.88V344c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8v-16.29c-11.29-.58-22.27-4.52-31.37-11.35-3.9-2.93-4.1-8.77-.57-12.14l11.75-11.21c2.77-2.64 6.89-2.76 10.13-.73 3.87 2.42 8.26 3.72 12.82 3.72h28.11c6.5 0 11.8-5.92 11.8-13.19 0-5.95-3.61-11.19-8.77-12.73l-45-13.5c-18.59-5.58-31.58-23.42-31.58-43.39 0-24.52 19.05-44.44 42.67-45.07V152c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v16.29c11.29.58 22.27 4.51 31.37 11.35 3.9 2.93 4.1 8.77.57 12.14l-11.75 11.21c-2.77 2.64-6.89 2.76-10.13.73-3.87-2.43-8.26-3.72-12.82-3.72h-28.11c-6.5 0-11.8 5.92-11.8 13.19 0 5.95 3.61 11.19 8.77 12.73l45 13.5c18.59 5.58 31.58 23.42 31.58 43.39 0 24.53-19.05 44.44-42.67 45.07zM416 312c0 4.42-3.58 8-8 8H296c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h112c4.42 0 8 3.58 8 8v16zm160 0c0 4.42-3.58 8-8 8h-80c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h80c4.42 0 8 3.58 8 8v16zm0-96c0 4.42-3.58 8-8 8H296c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h272c4.42 0 8 3.58 8 8v16z"],monument:[384,512,[],"f5a6","M368 448H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h352c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zm-78.86-347.26a31.97 31.97 0 0 0-9.21-19.44L203.31 4.69c-6.25-6.25-16.38-6.25-22.63 0l-76.6 76.61a31.97 31.97 0 0 0-9.21 19.44L64 416h256l-30.86-315.26zM240 307.2c0 6.4-6.4 12.8-12.8 12.8h-70.4c-6.4 0-12.8-6.4-12.8-12.8v-38.4c0-6.4 6.4-12.8 12.8-12.8h70.4c6.4 0 12.8 6.4 12.8 12.8v38.4z"],moon:[512,512,[],"f186","M283.211 512c78.962 0 151.079-35.925 198.857-94.792 7.068-8.708-.639-21.43-11.562-19.35-124.203 23.654-238.262-71.576-238.262-196.954 0-72.222 38.662-138.635 101.498-174.394 9.686-5.512 7.25-20.197-3.756-22.23A258.156 258.156 0 0 0 283.211 0c-141.309 0-256 114.511-256 256 0 141.309 114.511 256 256 256z"],"mortar-pestle":[512,512,[],"f5a7","M501.54 60.91c17.22-17.22 12.51-46.25-9.27-57.14a35.696 35.696 0 0 0-37.37 3.37L251.09 160h151.37l99.08-99.09zM496 192H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h16c0 80.98 50.2 150.11 121.13 178.32-12.76 16.87-21.72 36.8-24.95 58.69-1.46 9.92 6.04 18.98 16.07 18.98h223.5c10.03 0 17.53-9.06 16.07-18.98-3.22-21.89-12.18-41.82-24.95-58.69C429.8 406.11 480 336.98 480 256h16c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16z"],mosque:[640,512,[],"f678","M0 480c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32V160H0v320zm579.16-192c17.86-17.39 28.84-37.34 28.84-58.91 0-52.86-41.79-93.79-87.92-122.9-41.94-26.47-80.63-57.77-111.96-96.22L400 0l-8.12 9.97c-31.33 38.45-70.01 69.76-111.96 96.22C233.79 135.3 192 176.23 192 229.09c0 21.57 10.98 41.52 28.84 58.91h358.32zM608 320H192c-17.67 0-32 14.33-32 32v128c0 17.67 14.33 32 32 32h32v-64c0-17.67 14.33-32 32-32s32 14.33 32 32v64h64v-72c0-48 48-72 48-72s48 24 48 72v72h64v-64c0-17.67 14.33-32 32-32s32 14.33 32 32v64h32c17.67 0 32-14.33 32-32V352c0-17.67-14.33-32-32-32zM64 0S0 32 0 96v32h128V96c0-64-64-96-64-96z"],motorcycle:[640,512,[],"f21c","M512.9 192c-14.9-.1-29.1 2.3-42.4 6.9L437.6 144H520c13.3 0 24-10.7 24-24V88c0-13.3-10.7-24-24-24h-45.3c-6.8 0-13.3 2.9-17.8 7.9l-37.5 41.7-22.8-38C392.2 68.4 384.4 64 376 64h-80c-8.8 0-16 7.2-16 16v16c0 8.8 7.2 16 16 16h66.4l19.2 32H227.9c-17.7-23.1-44.9-40-99.9-40H72.5C59 104 47.7 115 48 128.5c.2 13 10.9 23.5 24 23.5h56c24.5 0 38.7 10.9 47.8 24.8l-11.3 20.5c-13-3.9-26.9-5.7-41.3-5.2C55.9 194.5 1.6 249.6 0 317c-1.6 72.1 56.3 131 128 131 59.6 0 109.7-40.8 124-96h84.2c13.7 0 24.6-11.4 24-25.1-2.1-47.1 17.5-93.7 56.2-125l12.5 20.8c-27.6 23.7-45.1 58.9-44.8 98.2.5 69.6 57.2 126.5 126.8 127.1 71.6.7 129.8-57.5 129.2-129.1-.7-69.6-57.6-126.4-127.2-126.9zM128 400c-44.1 0-80-35.9-80-80s35.9-80 80-80c4.2 0 8.4.3 12.5 1L99 316.4c-8.8 16 2.8 35.6 21 35.6h81.3c-12.4 28.2-40.6 48-73.3 48zm463.9-75.6c-2.2 40.6-35 73.4-75.5 75.5-46.1 2.5-84.4-34.3-84.4-79.9 0-21.4 8.4-40.8 22.1-55.1l49.4 82.4c4.5 7.6 14.4 10 22 5.5l13.7-8.2c7.6-4.5 10-14.4 5.5-22l-48.6-80.9c5.2-1.1 10.5-1.6 15.9-1.6 45.6-.1 82.3 38.2 79.9 84.3z"],mountain:[640,512,[],"f6fc","M634.92 462.7l-288-448C341.03 5.54 330.89 0 320 0s-21.03 5.54-26.92 14.7l-288 448a32.001 32.001 0 0 0-1.17 32.64A32.004 32.004 0 0 0 32 512h576c11.71 0 22.48-6.39 28.09-16.67a31.983 31.983 0 0 0-1.17-32.63zM320 91.18L405.39 224H320l-64 64-38.06-38.06L320 91.18z"],mouse:[384,512,[],"f8cc","M0 352a160 160 0 0 0 160 160h64a160 160 0 0 0 160-160V224H0zM176 0h-16A160 160 0 0 0 0 160v32h176zm48 0h-16v192h176v-32A160 160 0 0 0 224 0z"],"mouse-pointer":[320,512,[],"f245","M302.189 329.126H196.105l55.831 135.993c3.889 9.428-.555 19.999-9.444 23.999l-49.165 21.427c-9.165 4-19.443-.571-23.332-9.714l-53.053-129.136-86.664 89.138C18.729 472.71 0 463.554 0 447.977V18.299C0 1.899 19.921-6.096 30.277 5.443l284.412 292.542c11.472 11.179 3.007 31.141-12.5 31.141z"],"mug-hot":[512,512,[],"f7b6","M127.1 146.5c1.3 7.7 8 13.5 16 13.5h16.5c9.8 0 17.6-8.5 16.3-18-3.8-28.2-16.4-54.2-36.6-74.7-14.4-14.7-23.6-33.3-26.4-53.5C111.8 5.9 105 0 96.8 0H80.4C70.6 0 63 8.5 64.1 18c3.9 31.9 18 61.3 40.6 84.4 12 12.2 19.7 27.5 22.4 44.1zm112 0c1.3 7.7 8 13.5 16 13.5h16.5c9.8 0 17.6-8.5 16.3-18-3.8-28.2-16.4-54.2-36.6-74.7-14.4-14.7-23.6-33.3-26.4-53.5C223.8 5.9 217 0 208.8 0h-16.4c-9.8 0-17.5 8.5-16.3 18 3.9 31.9 18 61.3 40.6 84.4 12 12.2 19.7 27.5 22.4 44.1zM400 192H32c-17.7 0-32 14.3-32 32v192c0 53 43 96 96 96h192c53 0 96-43 96-96h16c61.8 0 112-50.2 112-112s-50.2-112-112-112zm0 160h-16v-96h16c26.5 0 48 21.5 48 48s-21.5 48-48 48z"],music:[512,512,[],"f001","M470.38 1.51L150.41 96A32 32 0 0 0 128 126.51v261.41A139 139 0 0 0 96 384c-53 0-96 28.66-96 64s43 64 96 64 96-28.66 96-64V214.32l256-75v184.61a138.4 138.4 0 0 0-32-3.93c-53 0-96 28.66-96 64s43 64 96 64 96-28.65 96-64V32a32 32 0 0 0-41.62-30.49z"],"network-wired":[640,512,[],"f6ff","M640 264v-16c0-8.84-7.16-16-16-16H344v-40h72c17.67 0 32-14.33 32-32V32c0-17.67-14.33-32-32-32H224c-17.67 0-32 14.33-32 32v128c0 17.67 14.33 32 32 32h72v40H16c-8.84 0-16 7.16-16 16v16c0 8.84 7.16 16 16 16h104v40H64c-17.67 0-32 14.33-32 32v128c0 17.67 14.33 32 32 32h160c17.67 0 32-14.33 32-32V352c0-17.67-14.33-32-32-32h-56v-40h304v40h-56c-17.67 0-32 14.33-32 32v128c0 17.67 14.33 32 32 32h160c17.67 0 32-14.33 32-32V352c0-17.67-14.33-32-32-32h-56v-40h104c8.84 0 16-7.16 16-16zM256 128V64h128v64H256zm-64 320H96v-64h96v64zm352 0h-96v-64h96v64z"],neuter:[288,512,[],"f22c","M288 176c0-79.5-64.5-144-144-144S0 96.5 0 176c0 68.5 47.9 125.9 112 140.4V468c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12V316.4c64.1-14.5 112-71.9 112-140.4zm-144 80c-44.1 0-80-35.9-80-80s35.9-80 80-80 80 35.9 80 80-35.9 80-80 80z"],newspaper:[576,512,[],"f1ea","M552 64H88c-13.255 0-24 10.745-24 24v8H24c-13.255 0-24 10.745-24 24v272c0 30.928 25.072 56 56 56h472c26.51 0 48-21.49 48-48V88c0-13.255-10.745-24-24-24zM56 400a8 8 0 0 1-8-8V144h16v248a8 8 0 0 1-8 8zm236-16H140c-6.627 0-12-5.373-12-12v-8c0-6.627 5.373-12 12-12h152c6.627 0 12 5.373 12 12v8c0 6.627-5.373 12-12 12zm208 0H348c-6.627 0-12-5.373-12-12v-8c0-6.627 5.373-12 12-12h152c6.627 0 12 5.373 12 12v8c0 6.627-5.373 12-12 12zm-208-96H140c-6.627 0-12-5.373-12-12v-8c0-6.627 5.373-12 12-12h152c6.627 0 12 5.373 12 12v8c0 6.627-5.373 12-12 12zm208 0H348c-6.627 0-12-5.373-12-12v-8c0-6.627 5.373-12 12-12h152c6.627 0 12 5.373 12 12v8c0 6.627-5.373 12-12 12zm0-96H140c-6.627 0-12-5.373-12-12v-40c0-6.627 5.373-12 12-12h360c6.627 0 12 5.373 12 12v40c0 6.627-5.373 12-12 12z"],"not-equal":[448,512,[],"f53e","M416 208c17.67 0 32-14.33 32-32v-32c0-17.67-14.33-32-32-32h-23.88l51.87-66.81c5.37-7.02 4.04-17.06-2.97-22.43L415.61 3.3c-7.02-5.38-17.06-4.04-22.44 2.97L311.09 112H32c-17.67 0-32 14.33-32 32v32c0 17.67 14.33 32 32 32h204.56l-74.53 96H32c-17.67 0-32 14.33-32 32v32c0 17.67 14.33 32 32 32h55.49l-51.87 66.81c-5.37 7.01-4.04 17.05 2.97 22.43L64 508.7c7.02 5.38 17.06 4.04 22.43-2.97L168.52 400H416c17.67 0 32-14.33 32-32v-32c0-17.67-14.33-32-32-32H243.05l74.53-96H416z"],"notes-medical":[384,512,[],"f481","M336 64h-80c0-35.3-28.7-64-64-64s-64 28.7-64 64H48C21.5 64 0 85.5 0 112v352c0 26.5 21.5 48 48 48h288c26.5 0 48-21.5 48-48V112c0-26.5-21.5-48-48-48zM192 40c13.3 0 24 10.7 24 24s-10.7 24-24 24-24-10.7-24-24 10.7-24 24-24zm96 304c0 4.4-3.6 8-8 8h-56v56c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8v-56h-56c-4.4 0-8-3.6-8-8v-48c0-4.4 3.6-8 8-8h56v-56c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v56h56c4.4 0 8 3.6 8 8v48zm0-192c0 4.4-3.6 8-8 8H104c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h176c4.4 0 8 3.6 8 8v16z"],"object-group":[512,512,[],"f247","M480 128V96h20c6.627 0 12-5.373 12-12V44c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v20H64V44c0-6.627-5.373-12-12-12H12C5.373 32 0 37.373 0 44v40c0 6.627 5.373 12 12 12h20v320H12c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12v-20h384v20c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12h-20V128zM96 276V140c0-6.627 5.373-12 12-12h168c6.627 0 12 5.373 12 12v136c0 6.627-5.373 12-12 12H108c-6.627 0-12-5.373-12-12zm320 96c0 6.627-5.373 12-12 12H236c-6.627 0-12-5.373-12-12v-52h72c13.255 0 24-10.745 24-24v-72h84c6.627 0 12 5.373 12 12v136z"],"object-ungroup":[576,512,[],"f248","M64 320v26a6 6 0 0 1-6 6H6a6 6 0 0 1-6-6v-52a6 6 0 0 1 6-6h26V96H6a6 6 0 0 1-6-6V38a6 6 0 0 1 6-6h52a6 6 0 0 1 6 6v26h288V38a6 6 0 0 1 6-6h52a6 6 0 0 1 6 6v52a6 6 0 0 1-6 6h-26v192h26a6 6 0 0 1 6 6v52a6 6 0 0 1-6 6h-52a6 6 0 0 1-6-6v-26H64zm480-64v-32h26a6 6 0 0 0 6-6v-52a6 6 0 0 0-6-6h-52a6 6 0 0 0-6 6v26H408v72h8c13.255 0 24 10.745 24 24v64c0 13.255-10.745 24-24 24h-64c-13.255 0-24-10.745-24-24v-8H192v72h-26a6 6 0 0 0-6 6v52a6 6 0 0 0 6 6h52a6 6 0 0 0 6-6v-26h288v26a6 6 0 0 0 6 6h52a6 6 0 0 0 6-6v-52a6 6 0 0 0-6-6h-26V256z"],"oil-can":[640,512,[],"f613","M629.8 160.31L416 224l-50.49-25.24a64.07 64.07 0 0 0-28.62-6.76H280v-48h56c8.84 0 16-7.16 16-16v-16c0-8.84-7.16-16-16-16H176c-8.84 0-16 7.16-16 16v16c0 8.84 7.16 16 16 16h56v48h-56L37.72 166.86a31.9 31.9 0 0 0-5.79-.53C14.67 166.33 0 180.36 0 198.34v94.95c0 15.46 11.06 28.72 26.28 31.48L96 337.46V384c0 17.67 14.33 32 32 32h274.63c8.55 0 16.75-3.42 22.76-9.51l212.26-214.75c1.5-1.5 2.34-3.54 2.34-5.66V168c.01-5.31-5.08-9.15-10.19-7.69zM96 288.67l-48-8.73v-62.43l48 8.73v62.43zm453.33 84.66c0 23.56 19.1 42.67 42.67 42.67s42.67-19.1 42.67-42.67S592 288 592 288s-42.67 61.77-42.67 85.33z"],om:[512,512,[],"f679","M360.6 60.94a10.43 10.43 0 0 0 14.76 0l21.57-21.56a10.43 10.43 0 0 0 0-14.76L375.35 3.06c-4.08-4.07-10.68-4.07-14.76 0l-21.57 21.56a10.43 10.43 0 0 0 0 14.76l21.58 21.56zM412.11 192c-26.69 0-51.77 10.39-70.64 29.25l-24.25 24.25c-6.78 6.77-15.78 10.5-25.38 10.5H245c10.54-22.1 14.17-48.11 7.73-75.23-10.1-42.55-46.36-76.11-89.52-83.19-36.15-5.93-70.9 5.04-96.01 28.78-7.36 6.96-6.97 18.85 1.12 24.93l26.15 19.63c5.72 4.3 13.66 4.32 19.2-.21 8.45-6.9 19.02-10.71 30.27-10.71 26.47 0 48.01 21.53 48.01 48s-21.54 48-48.01 48h-31.9c-11.96 0-19.74 12.58-14.39 23.28l16.09 32.17c2.53 5.06 7.6 8.1 13.17 8.55h33.03c35.3 0 64.01 28.7 64.01 64s-28.71 64-64.01 64c-96.02 0-122.35-54.02-145.15-92.03-4.53-7.55-14.77-3.58-14.79 5.22C-.09 416 41.13 512 159.94 512c70.59 0 128.02-57.42 128.02-128 0-23.42-6.78-45.1-17.81-64h21.69c26.69 0 51.77-10.39 70.64-29.25l24.25-24.25c6.78-6.77 15.78-10.5 25.38-10.5 19.78 0 35.88 16.09 35.88 35.88V392c0 13.23-18.77 24-32.01 24-39.4 0-66.67-24.24-81.82-42.89-4.77-5.87-14.2-2.54-14.2 5.02V416s0 64 96.02 64c48.54 0 96.02-39.47 96.02-88V291.88c0-55.08-44.8-99.88-99.89-99.88zm42.18-124.73c-85.55 65.12-169.05 2.75-172.58.05-6.02-4.62-14.44-4.38-20.14.55-5.74 4.92-7.27 13.17-3.66 19.8 1.61 2.95 40.37 72.34 118.8 72.34 79.92 0 98.78-31.36 101.75-37.66 1.02-2.12 1.53-4.47 1.53-6.83V80c0-13.22-15.14-20.69-25.7-12.73z"],otter:[640,512,[],"f700","M608 32h-32l-13.25-13.25A63.97 63.97 0 0 0 517.49 0H497c-11.14 0-22.08 2.91-31.75 8.43L312 96h-56C149.96 96 64 181.96 64 288v1.61c0 32.75-16 62.14-39.56 84.89-18.19 17.58-28.1 43.68-23.19 71.8 6.76 38.8 42.9 65.7 82.28 65.7H192c17.67 0 32-14.33 32-32s-14.33-32-32-32H80c-8.83 0-16-7.17-16-16s7.17-16 16-16h224c8.84 0 16-7.16 16-16v-16c0-17.67-14.33-32-32-32h-64l149.49-80.5L448 416h80c8.84 0 16-7.16 16-16v-16c0-17.67-14.33-32-32-32h-28.22l-55.11-110.21L521.14 192H544c53.02 0 96-42.98 96-96V64c0-17.67-14.33-32-32-32zm-96 16c8.84 0 16 7.16 16 16s-7.16 16-16 16-16-7.16-16-16 7.16-16 16-16zm32 96h-34.96L407.2 198.84l-13.77-27.55L512 112h77.05c-6.62 18.58-24.22 32-45.05 32z"],outdent:[448,512,[],"f03b","M100.69 363.29c10 10 27.31 2.93 27.31-11.31V160c0-14.32-17.33-21.31-27.31-11.31l-96 96a16 16 0 0 0 0 22.62zM432 416H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm3.17-128H204.83A12.82 12.82 0 0 0 192 300.83v38.34A12.82 12.82 0 0 0 204.83 352h230.34A12.82 12.82 0 0 0 448 339.17v-38.34A12.82 12.82 0 0 0 435.17 288zm0-128H204.83A12.82 12.82 0 0 0 192 172.83v38.34A12.82 12.82 0 0 0 204.83 224h230.34A12.82 12.82 0 0 0 448 211.17v-38.34A12.82 12.82 0 0 0 435.17 160zM432 32H16A16 16 0 0 0 0 48v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z"],pager:[512,512,[],"f815","M448 64H64a64 64 0 0 0-64 64v256a64 64 0 0 0 64 64h384a64 64 0 0 0 64-64V128a64 64 0 0 0-64-64zM160 368H80a16 16 0 0 1-16-16v-16a16 16 0 0 1 16-16h80zm128-16a16 16 0 0 1-16 16h-80v-48h80a16 16 0 0 1 16 16zm160-128a32 32 0 0 1-32 32H96a32 32 0 0 1-32-32v-64a32 32 0 0 1 32-32h320a32 32 0 0 1 32 32z"],"paint-brush":[512,512,[],"f1fc","M167.02 309.34c-40.12 2.58-76.53 17.86-97.19 72.3-2.35 6.21-8 9.98-14.59 9.98-11.11 0-45.46-27.67-55.25-34.35C0 439.62 37.93 512 128 512c75.86 0 128-43.77 128-120.19 0-3.11-.65-6.08-.97-9.13l-88.01-73.34zM457.89 0c-15.16 0-29.37 6.71-40.21 16.45C213.27 199.05 192 203.34 192 257.09c0 13.7 3.25 26.76 8.73 38.7l63.82 53.18c7.21 1.8 14.64 3.03 22.39 3.03 62.11 0 98.11-45.47 211.16-256.46 7.38-14.35 13.9-29.85 13.9-45.99C512 20.64 486 0 457.89 0z"],"paint-roller":[512,512,[],"f5aa","M416 128V32c0-17.67-14.33-32-32-32H32C14.33 0 0 14.33 0 32v96c0 17.67 14.33 32 32 32h352c17.67 0 32-14.33 32-32zm32-64v128c0 17.67-14.33 32-32 32H256c-35.35 0-64 28.65-64 64v32c-17.67 0-32 14.33-32 32v128c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32V352c0-17.67-14.33-32-32-32v-32h160c53.02 0 96-42.98 96-96v-64c0-35.35-28.65-64-64-64z"],palette:[512,512,[],"f53f","M204.3 5C104.9 24.4 24.8 104.3 5.2 203.4c-37 187 131.7 326.4 258.8 306.7 41.2-6.4 61.4-54.6 42.5-91.7-23.1-45.4 9.9-98.4 60.9-98.4h79.7c35.8 0 64.8-29.6 64.9-65.3C511.5 97.1 368.1-26.9 204.3 5zM96 320c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm32-128c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm128-64c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm128 64c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],pallet:[640,512,[],"f482","M144 256h352c8.8 0 16-7.2 16-16V16c0-8.8-7.2-16-16-16H384v128l-64-32-64 32V0H144c-8.8 0-16 7.2-16 16v224c0 8.8 7.2 16 16 16zm480 128c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h48v64H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h608c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16h-48v-64h48zm-336 64H128v-64h160v64zm224 0H352v-64h160v64z"],"paper-plane":[512,512,[],"f1d8","M476 3.2L12.5 270.6c-18.1 10.4-15.8 35.6 2.2 43.2L121 358.4l287.3-253.2c5.5-4.9 13.3 2.6 8.6 8.3L176 407v80.5c0 23.6 28.5 32.9 42.5 15.8L282 426l124.6 52.2c14.2 6 30.4-2.9 33-18.2l72-432C515 7.8 493.3-6.8 476 3.2z"],paperclip:[448,512,[],"f0c6","M43.246 466.142c-58.43-60.289-57.341-157.511 1.386-217.581L254.392 34c44.316-45.332 116.351-45.336 160.671 0 43.89 44.894 43.943 117.329 0 162.276L232.214 383.128c-29.855 30.537-78.633 30.111-107.982-.998-28.275-29.97-27.368-77.473 1.452-106.953l143.743-146.835c6.182-6.314 16.312-6.422 22.626-.241l22.861 22.379c6.315 6.182 6.422 16.312.241 22.626L171.427 319.927c-4.932 5.045-5.236 13.428-.648 18.292 4.372 4.634 11.245 4.711 15.688.165l182.849-186.851c19.613-20.062 19.613-52.725-.011-72.798-19.189-19.627-49.957-19.637-69.154 0L90.39 293.295c-34.763 35.56-35.299 93.12-1.191 128.313 34.01 35.093 88.985 35.137 123.058.286l172.06-175.999c6.177-6.319 16.307-6.433 22.626-.256l22.877 22.364c6.319 6.177 6.434 16.307.256 22.626l-172.06 175.998c-59.576 60.938-155.943 60.216-214.77-.485z"],"parachute-box":[512,512,[],"f4cd","M511.9 175c-9.1-75.6-78.4-132.4-158.3-158.7C390 55.7 416 116.9 416 192h28.1L327.5 321.5c-2.5-.6-4.8-1.5-7.5-1.5h-48V192h112C384 76.8 315.1 0 256 0S128 76.8 128 192h112v128h-48c-2.7 0-5 .9-7.5 1.5L67.9 192H96c0-75.1 26-136.3 62.4-175.7C78.5 42.7 9.2 99.5.1 175c-1.1 9.1 6.8 17 16 17h8.7l136.7 151.9c-.7 2.6-1.6 5.2-1.6 8.1v128c0 17.7 14.3 32 32 32h128c17.7 0 32-14.3 32-32V352c0-2.9-.9-5.4-1.6-8.1L487.1 192h8.7c9.3 0 17.2-7.8 16.1-17z"],paragraph:[448,512,[],"f1dd","M448 48v32a16 16 0 0 1-16 16h-48v368a16 16 0 0 1-16 16h-32a16 16 0 0 1-16-16V96h-32v368a16 16 0 0 1-16 16h-32a16 16 0 0 1-16-16V352h-32a160 160 0 0 1 0-320h240a16 16 0 0 1 16 16z"],parking:[448,512,[],"f540","M400 32H48C21.5 32 0 53.5 0 80v352c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V80c0-26.5-21.5-48-48-48zM240 320h-48v48c0 8.8-7.2 16-16 16h-32c-8.8 0-16-7.2-16-16V144c0-8.8 7.2-16 16-16h96c52.9 0 96 43.1 96 96s-43.1 96-96 96zm0-128h-48v64h48c17.6 0 32-14.4 32-32s-14.4-32-32-32z"],passport:[448,512,[],"f5ab","M129.62 176h39.09c1.49-27.03 6.54-51.35 14.21-70.41-27.71 13.24-48.02 39.19-53.3 70.41zm0 32c5.29 31.22 25.59 57.17 53.3 70.41-7.68-19.06-12.72-43.38-14.21-70.41h-39.09zM224 286.69c7.69-7.45 20.77-34.42 23.43-78.69h-46.87c2.67 44.26 15.75 71.24 23.44 78.69zM200.57 176h46.87c-2.66-44.26-15.74-71.24-23.43-78.69-7.7 7.45-20.78 34.43-23.44 78.69zm64.51 102.41c27.71-13.24 48.02-39.19 53.3-70.41h-39.09c-1.49 27.03-6.53 51.35-14.21 70.41zM416 0H64C28.65 0 0 28.65 0 64v384c0 35.35 28.65 64 64 64h352c17.67 0 32-14.33 32-32V32c0-17.67-14.33-32-32-32zm-80 416H112c-8.8 0-16-7.2-16-16s7.2-16 16-16h224c8.8 0 16 7.2 16 16s-7.2 16-16 16zm-112-96c-70.69 0-128-57.31-128-128S153.31 64 224 64s128 57.31 128 128-57.31 128-128 128zm41.08-214.41c7.68 19.06 12.72 43.38 14.21 70.41h39.09c-5.28-31.22-25.59-57.17-53.3-70.41z"],pastafarianism:[640,512,[],"f67b","M624.54 347.67c-32.7-12.52-57.36 4.25-75.37 16.45-17.06 11.53-23.25 14.42-31.41 11.36-8.12-3.09-10.83-9.38-15.89-29.38-3.33-13.15-7.44-29.32-17.95-42.65 2.24-2.91 4.43-5.79 6.38-8.57C500.47 304.45 513.71 312 532 312c33.95 0 50.87-25.78 62.06-42.83 10.59-16.14 15-21.17 21.94-21.17 13.25 0 24-10.75 24-24s-10.75-24-24-24c-33.95 0-50.87 25.78-62.06 42.83-10.6 16.14-15 21.17-21.94 21.17-17.31 0-37.48-61.43-97.26-101.91l17.25-34.5C485.43 125.5 512 97.98 512 64c0-35.35-28.65-64-64-64s-64 28.65-64 64c0 13.02 3.94 25.1 10.62 35.21l-18.15 36.3c-16.98-4.6-35.6-7.51-56.46-7.51s-39.49 2.91-56.46 7.51l-18.15-36.3C252.06 89.1 256 77.02 256 64c0-35.35-28.65-64-64-64s-64 28.65-64 64c0 33.98 26.56 61.5 60.02 63.6l17.25 34.5C145.68 202.44 125.15 264 108 264c-6.94 0-11.34-5.03-21.94-21.17C74.88 225.78 57.96 200 24 200c-13.25 0-24 10.75-24 24s10.75 24 24 24c6.94 0 11.34 5.03 21.94 21.17C57.13 286.22 74.05 312 108 312c18.29 0 31.53-7.55 41.7-17.11 1.95 2.79 4.14 5.66 6.38 8.57-10.51 13.33-14.62 29.5-17.95 42.65-5.06 20-7.77 26.28-15.89 29.38-8.11 3.06-14.33.17-31.41-11.36-18.03-12.2-42.72-28.92-75.37-16.45-12.39 4.72-18.59 18.58-13.87 30.97 4.72 12.41 18.61 18.61 30.97 13.88 8.16-3.09 14.34-.19 31.39 11.36 13.55 9.16 30.83 20.86 52.42 20.84 7.17 0 14.83-1.28 22.97-4.39 32.66-12.44 39.98-41.33 45.33-62.44 2.21-8.72 3.99-14.49 5.95-18.87 16.62 13.61 36.95 25.88 61.64 34.17-9.96 37-32.18 90.8-60.26 90.8-13.25 0-24 10.75-24 24s10.75 24 24 24c66.74 0 97.05-88.63 107.42-129.14 6.69.6 13.42 1.14 20.58 1.14s13.89-.54 20.58-1.14C350.95 423.37 381.26 512 448 512c13.25 0 24-10.75 24-24s-10.75-24-24-24c-27.94 0-50.21-53.81-60.22-90.81 24.69-8.29 45-20.56 61.62-34.16 1.96 4.38 3.74 10.15 5.95 18.87 5.34 21.11 12.67 50 45.33 62.44 8.14 3.11 15.8 4.39 22.97 4.39 21.59 0 38.87-11.69 52.42-20.84 17.05-11.55 23.28-14.45 31.39-11.36 12.39 4.75 26.27-1.47 30.97-13.88 4.71-12.4-1.49-26.26-13.89-30.98zM448 48c8.82 0 16 7.18 16 16s-7.18 16-16 16-16-7.18-16-16 7.18-16 16-16zm-256 0c8.82 0 16 7.18 16 16s-7.18 16-16 16-16-7.18-16-16 7.18-16 16-16z"],paste:[448,512,[],"f0ea","M128 184c0-30.879 25.122-56 56-56h136V56c0-13.255-10.745-24-24-24h-80.61C204.306 12.89 183.637 0 160 0s-44.306 12.89-55.39 32H24C10.745 32 0 42.745 0 56v336c0 13.255 10.745 24 24 24h104V184zm32-144c13.255 0 24 10.745 24 24s-10.745 24-24 24-24-10.745-24-24 10.745-24 24-24zm184 248h104v200c0 13.255-10.745 24-24 24H184c-13.255 0-24-10.745-24-24V184c0-13.255 10.745-24 24-24h136v104c0 13.2 10.8 24 24 24zm104-38.059V256h-96v-96h6.059a24 24 0 0 1 16.97 7.029l65.941 65.941a24.002 24.002 0 0 1 7.03 16.971z"],pause:[448,512,[],"f04c","M144 479H48c-26.5 0-48-21.5-48-48V79c0-26.5 21.5-48 48-48h96c26.5 0 48 21.5 48 48v352c0 26.5-21.5 48-48 48zm304-48V79c0-26.5-21.5-48-48-48h-96c-26.5 0-48 21.5-48 48v352c0 26.5 21.5 48 48 48h96c26.5 0 48-21.5 48-48z"],"pause-circle":[512,512,[],"f28b","M256 8C119 8 8 119 8 256s111 248 248 248 248-111 248-248S393 8 256 8zm-16 328c0 8.8-7.2 16-16 16h-48c-8.8 0-16-7.2-16-16V176c0-8.8 7.2-16 16-16h48c8.8 0 16 7.2 16 16v160zm112 0c0 8.8-7.2 16-16 16h-48c-8.8 0-16-7.2-16-16V176c0-8.8 7.2-16 16-16h48c8.8 0 16 7.2 16 16v160z"],paw:[512,512,[],"f1b0","M256 224c-79.41 0-192 122.76-192 200.25 0 34.9 26.81 55.75 71.74 55.75 48.84 0 81.09-25.08 120.26-25.08 39.51 0 71.85 25.08 120.26 25.08 44.93 0 71.74-20.85 71.74-55.75C448 346.76 335.41 224 256 224zm-147.28-12.61c-10.4-34.65-42.44-57.09-71.56-50.13-29.12 6.96-44.29 40.69-33.89 75.34 10.4 34.65 42.44 57.09 71.56 50.13 29.12-6.96 44.29-40.69 33.89-75.34zm84.72-20.78c30.94-8.14 46.42-49.94 34.58-93.36s-46.52-72.01-77.46-63.87-46.42 49.94-34.58 93.36c11.84 43.42 46.53 72.02 77.46 63.87zm281.39-29.34c-29.12-6.96-61.15 15.48-71.56 50.13-10.4 34.65 4.77 68.38 33.89 75.34 29.12 6.96 61.15-15.48 71.56-50.13 10.4-34.65-4.77-68.38-33.89-75.34zm-156.27 29.34c30.94 8.14 65.62-20.45 77.46-63.87 11.84-43.42-3.64-85.21-34.58-93.36s-65.62 20.45-77.46 63.87c-11.84 43.42 3.64 85.22 34.58 93.36z"],peace:[496,512,[],"f67c","M248 8C111.03 8 0 119.03 0 256s111.03 248 248 248 248-111.03 248-248S384.97 8 248 8zm184 248c0 31.93-8.2 61.97-22.57 88.17L280 240.63V74.97c86.23 15.21 152 90.5 152 181.03zM216 437.03c-33.86-5.97-64.49-21.2-89.29-43.02L216 322.57v114.46zm64-114.46L369.29 394c-24.8 21.82-55.43 37.05-89.29 43.02V322.57zm-64-247.6v165.66L86.57 344.17C72.2 317.97 64 287.93 64 256c0-90.53 65.77-165.82 152-181.03z"],pen:[512,512,[],"f304","M290.74 93.24l128.02 128.02-277.99 277.99-114.14 12.6C11.35 513.54-1.56 500.62.14 485.34l12.7-114.22 277.9-277.88zm207.2-19.06l-60.11-60.11c-18.75-18.75-49.16-18.75-67.91 0l-56.55 56.55 128.02 128.02 56.55-56.55c18.75-18.76 18.75-49.16 0-67.91z"],"pen-alt":[512,512,[],"f305","M497.94 74.17l-60.11-60.11c-18.75-18.75-49.16-18.75-67.91 0l-56.55 56.55 128.02 128.02 56.55-56.55c18.75-18.75 18.75-49.15 0-67.91zm-246.8-20.53c-15.62-15.62-40.94-15.62-56.56 0L75.8 172.43c-6.25 6.25-6.25 16.38 0 22.62l22.63 22.63c6.25 6.25 16.38 6.25 22.63 0l101.82-101.82 22.63 22.62L93.95 290.03A327.038 327.038 0 0 0 .17 485.11l-.03.23c-1.7 15.28 11.21 28.2 26.49 26.51a327.02 327.02 0 0 0 195.34-93.8l196.79-196.79-82.77-82.77-84.85-84.85z"],"pen-fancy":[512,512,[],"f5ac","M79.18 282.94a32.005 32.005 0 0 0-20.24 20.24L0 480l4.69 4.69 92.89-92.89c-.66-2.56-1.57-5.03-1.57-7.8 0-17.67 14.33-32 32-32s32 14.33 32 32-14.33 32-32 32c-2.77 0-5.24-.91-7.8-1.57l-92.89 92.89L32 512l176.82-58.94a31.983 31.983 0 0 0 20.24-20.24l33.07-84.07-98.88-98.88-84.07 33.07zM369.25 28.32L186.14 227.81l97.85 97.85 199.49-183.11C568.4 67.48 443.73-55.94 369.25 28.32z"],"pen-nib":[512,512,[],"f5ad","M136.6 138.79a64.003 64.003 0 0 0-43.31 41.35L0 460l14.69 14.69L164.8 324.58c-2.99-6.26-4.8-13.18-4.8-20.58 0-26.51 21.49-48 48-48s48 21.49 48 48-21.49 48-48 48c-7.4 0-14.32-1.81-20.58-4.8L37.31 497.31 52 512l279.86-93.29a64.003 64.003 0 0 0 41.35-43.31L416 224 288 96l-151.4 42.79zm361.34-64.62l-60.11-60.11c-18.75-18.75-49.16-18.75-67.91 0l-56.55 56.55 128.02 128.02 56.55-56.55c18.75-18.75 18.75-49.15 0-67.91z"],"pen-square":[448,512,[],"f14b","M400 480H48c-26.5 0-48-21.5-48-48V80c0-26.5 21.5-48 48-48h352c26.5 0 48 21.5 48 48v352c0 26.5-21.5 48-48 48zM238.1 177.9L102.4 313.6l-6.3 57.1c-.8 7.6 5.6 14.1 13.3 13.3l57.1-6.3L302.2 242c2.3-2.3 2.3-6.1 0-8.5L246.7 178c-2.5-2.4-6.3-2.4-8.6-.1zM345 165.1L314.9 135c-9.4-9.4-24.6-9.4-33.9 0l-23.1 23.1c-2.3 2.3-2.3 6.1 0 8.5l55.5 55.5c2.3 2.3 6.1 2.3 8.5 0L345 199c9.3-9.3 9.3-24.5 0-33.9z"],"pencil-alt":[512,512,[],"f303","M497.9 142.1l-46.1 46.1c-4.7 4.7-12.3 4.7-17 0l-111-111c-4.7-4.7-4.7-12.3 0-17l46.1-46.1c18.7-18.7 49.1-18.7 67.9 0l60.1 60.1c18.8 18.7 18.8 49.1 0 67.9zM284.2 99.8L21.6 362.4.4 483.9c-2.9 16.4 11.4 30.6 27.8 27.8l121.5-21.3 262.6-262.6c4.7-4.7 4.7-12.3 0-17l-111-111c-4.8-4.7-12.4-4.7-17.1 0zM124.1 339.9c-5.5-5.5-5.5-14.3 0-19.8l154-154c5.5-5.5 14.3-5.5 19.8 0s5.5 14.3 0 19.8l-154 154c-5.5 5.5-14.3 5.5-19.8 0zM88 424h48v36.3l-64.5 11.3-31.1-31.1L51.7 376H88v48z"],"pencil-ruler":[512,512,[],"f5ae","M109.46 244.04l134.58-134.56-44.12-44.12-61.68 61.68a7.919 7.919 0 0 1-11.21 0l-11.21-11.21c-3.1-3.1-3.1-8.12 0-11.21l61.68-61.68-33.64-33.65C131.47-3.1 111.39-3.1 99 9.29L9.29 99c-12.38 12.39-12.39 32.47 0 44.86l100.17 100.18zm388.47-116.8c18.76-18.76 18.75-49.17 0-67.93l-45.25-45.25c-18.76-18.76-49.18-18.76-67.95 0l-46.02 46.01 113.2 113.2 46.02-46.03zM316.08 82.71l-297 296.96L.32 487.11c-2.53 14.49 10.09 27.11 24.59 24.56l107.45-18.84L429.28 195.9 316.08 82.71zm186.63 285.43l-33.64-33.64-61.68 61.68c-3.1 3.1-8.12 3.1-11.21 0l-11.21-11.21c-3.09-3.1-3.09-8.12 0-11.21l61.68-61.68-44.14-44.14L267.93 402.5l100.21 100.2c12.39 12.39 32.47 12.39 44.86 0l89.71-89.7c12.39-12.39 12.39-32.47 0-44.86z"],"people-arrows":[576,512,[],"f968","M96,128A64,64,0,1,0,32,64,64,64,0,0,0,96,128Zm0,176.08a44.11,44.11,0,0,1,13.64-32L181.77,204c1.65-1.55,3.77-2.31,5.61-3.57A63.91,63.91,0,0,0,128,160H64A64,64,0,0,0,0,224v96a32,32,0,0,0,32,32V480a32,32,0,0,0,32,32h64a32,32,0,0,0,32-32V383.61l-50.36-47.53A44.08,44.08,0,0,1,96,304.08ZM480,128a64,64,0,1,0-64-64A64,64,0,0,0,480,128Zm32,32H448a63.91,63.91,0,0,0-59.38,40.42c1.84,1.27,4,2,5.62,3.59l72.12,68.06a44.37,44.37,0,0,1,0,64L416,383.62V480a32,32,0,0,0,32,32h64a32,32,0,0,0,32-32V352a32,32,0,0,0,32-32V224A64,64,0,0,0,512,160ZM444.4,295.34l-72.12-68.06A12,12,0,0,0,352,236v36H224V236a12,12,0,0,0-20.28-8.73L131.6,295.34a12.4,12.4,0,0,0,0,17.47l72.12,68.07A12,12,0,0,0,224,372.14V336H352v36.14a12,12,0,0,0,20.28,8.74l72.12-68.07A12.4,12.4,0,0,0,444.4,295.34Z"],"people-carry":[640,512,[],"f4ce","M128 96c26.5 0 48-21.5 48-48S154.5 0 128 0 80 21.5 80 48s21.5 48 48 48zm384 0c26.5 0 48-21.5 48-48S538.5 0 512 0s-48 21.5-48 48 21.5 48 48 48zm125.7 372.1l-44-110-41.1 46.4-2 18.2 27.7 69.2c5 12.5 17 20.1 29.7 20.1 4 0 8-.7 11.9-2.3 16.4-6.6 24.4-25.2 17.8-41.6zm-34.2-209.8L585 178.1c-4.6-20-18.6-36.8-37.5-44.9-18.5-8-39-6.7-56.1 3.3-22.7 13.4-39.7 34.5-48.1 59.4L432 229.8 416 240v-96c0-8.8-7.2-16-16-16H240c-8.8 0-16 7.2-16 16v96l-16.1-10.2-11.3-33.9c-8.3-25-25.4-46-48.1-59.4-17.2-10-37.6-11.3-56.1-3.3-18.9 8.1-32.9 24.9-37.5 44.9l-18.4 80.2c-4.6 20 .7 41.2 14.4 56.7l67.2 75.9 10.1 92.6C130 499.8 143.8 512 160 512c1.2 0 2.3-.1 3.5-.2 17.6-1.9 30.2-17.7 28.3-35.3l-10.1-92.8c-1.5-13-6.9-25.1-15.6-35l-43.3-49 17.6-70.3 6.8 20.4c4.1 12.5 11.9 23.4 24.5 32.6l51.1 32.5c4.6 2.9 12.1 4.6 17.2 5h160c5.1-.4 12.6-2.1 17.2-5l51.1-32.5c12.6-9.2 20.4-20 24.5-32.6l6.8-20.4 17.6 70.3-43.3 49c-8.7 9.9-14.1 22-15.6 35l-10.1 92.8c-1.9 17.6 10.8 33.4 28.3 35.3 1.2.1 2.3.2 3.5.2 16.1 0 30-12.1 31.8-28.5l10.1-92.6 67.2-75.9c13.6-15.5 19-36.7 14.4-56.7zM46.3 358.1l-44 110c-6.6 16.4 1.4 35 17.8 41.6 16.8 6.6 35.1-1.7 41.6-17.8l27.7-69.2-2-18.2-41.1-46.4z"],"pepper-hot":[512,512,[],"f816","M330.67 263.12V173.4l-52.75-24.22C219.44 218.76 197.58 400 56 400a56 56 0 0 0 0 112c212.64 0 370.65-122.87 419.18-210.34l-37.05-38.54zm131.09-128.37C493.92 74.91 477.18 26.48 458.62 3a8 8 0 0 0-11.93-.59l-22.9 23a8.06 8.06 0 0 0-.89 10.23c6.86 10.36 17.05 35.1-1.4 72.32A142.85 142.85 0 0 0 364.34 96c-28 0-54 8.54-76.34 22.59l74.67 34.29v78.24h89.09L506.44 288c3.26-12.62 5.56-25.63 5.56-39.31a154 154 0 0 0-50.24-113.94z"],percent:[448,512,[],"f295","M112 224c61.9 0 112-50.1 112-112S173.9 0 112 0 0 50.1 0 112s50.1 112 112 112zm0-160c26.5 0 48 21.5 48 48s-21.5 48-48 48-48-21.5-48-48 21.5-48 48-48zm224 224c-61.9 0-112 50.1-112 112s50.1 112 112 112 112-50.1 112-112-50.1-112-112-112zm0 160c-26.5 0-48-21.5-48-48s21.5-48 48-48 48 21.5 48 48-21.5 48-48 48zM392.3.2l31.6-.1c19.4-.1 30.9 21.8 19.7 37.8L77.4 501.6a23.95 23.95 0 0 1-19.6 10.2l-33.4.1c-19.5 0-30.9-21.9-19.7-37.8l368-463.7C377.2 4 384.5.2 392.3.2z"],percentage:[384,512,[],"f541","M109.25 173.25c24.99-24.99 24.99-65.52 0-90.51-24.99-24.99-65.52-24.99-90.51 0-24.99 24.99-24.99 65.52 0 90.51 25 25 65.52 25 90.51 0zm256 165.49c-24.99-24.99-65.52-24.99-90.51 0-24.99 24.99-24.99 65.52 0 90.51 24.99 24.99 65.52 24.99 90.51 0 25-24.99 25-65.51 0-90.51zm-1.94-231.43l-22.62-22.62c-12.5-12.5-32.76-12.5-45.25 0L20.69 359.44c-12.5 12.5-12.5 32.76 0 45.25l22.62 22.62c12.5 12.5 32.76 12.5 45.25 0l274.75-274.75c12.5-12.49 12.5-32.75 0-45.25z"],"person-booth":[576,512,[],"f756","M192 496c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V320h-64v176zm32-272h-50.9l-45.2-45.3C115.8 166.6 99.7 160 82.7 160H64c-17.1 0-33.2 6.7-45.3 18.8C6.7 190.9 0 207 0 224.1L.2 320 0 480c0 17.7 14.3 32 31.9 32 17.6 0 32-14.3 32-32l.1-100.7c.9.5 1.6 1.3 2.5 1.7l29.1 43v56c0 17.7 14.3 32 32 32s32-14.3 32-32v-56.5c0-9.9-2.3-19.8-6.7-28.6l-41.2-61.3V253l20.9 20.9c9.1 9.1 21.1 14.1 33.9 14.1H224c17.7 0 32-14.3 32-32s-14.3-32-32-32zM64 128c26.5 0 48-21.5 48-48S90.5 32 64 32 16 53.5 16 80s21.5 48 48 48zm224-96l31.5 223.1-30.9 154.6c-4.3 21.6 13 38.3 31.4 38.3 15.2 0 28-9.1 32.3-30.4.9 16.9 14.6 30.4 31.7 30.4 17.7 0 32-14.3 32-32 0 17.7 14.3 32 32 32s32-14.3 32-32V0H288v32zm-96 0v160h64V0h-32c-17.7 0-32 14.3-32 32zM544 0h-32v496c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V32c0-17.7-14.3-32-32-32z"],phone:[512,512,[],"f095","M493.4 24.6l-104-24c-11.3-2.6-22.9 3.3-27.5 13.9l-48 112c-4.2 9.8-1.4 21.3 6.9 28l60.6 49.6c-36 76.7-98.9 140.5-177.2 177.2l-49.6-60.6c-6.8-8.3-18.2-11.1-28-6.9l-112 48C3.9 366.5-2 378.1.6 389.4l24 104C27.1 504.2 36.7 512 48 512c256.1 0 464-207.5 464-464 0-11.2-7.7-20.9-18.6-23.4z"],"phone-alt":[512,512,[],"f879","M497.39 361.8l-112-48a24 24 0 0 0-28 6.9l-49.6 60.6A370.66 370.66 0 0 1 130.6 204.11l60.6-49.6a23.94 23.94 0 0 0 6.9-28l-48-112A24.16 24.16 0 0 0 122.6.61l-104 24A24 24 0 0 0 0 48c0 256.5 207.9 464 464 464a24 24 0 0 0 23.4-18.6l24-104a24.29 24.29 0 0 0-14.01-27.6z"],"phone-slash":[640,512,[],"f3dd","M268.2 381.4l-49.6-60.6c-6.8-8.3-18.2-11.1-28-6.9l-112 48c-10.7 4.6-16.5 16.1-13.9 27.5l24 104c2.5 10.8 12.1 18.6 23.4 18.6 100.7 0 193.7-32.4 269.7-86.9l-80-61.8c-10.9 6.5-22.1 12.7-33.6 18.1zm365.6 76.7L475.1 335.5C537.9 256.4 576 156.9 576 48c0-11.2-7.7-20.9-18.6-23.4l-104-24c-11.3-2.6-22.9 3.3-27.5 13.9l-48 112c-4.2 9.8-1.4 21.3 6.9 28l60.6 49.6c-12.2 26.1-27.9 50.3-46 72.8L45.5 3.4C38.5-2 28.5-.8 23 6.2L3.4 31.4c-5.4 7-4.2 17 2.8 22.4l588.4 454.7c7 5.4 17 4.2 22.5-2.8l19.6-25.3c5.4-6.8 4.1-16.9-2.9-22.3z"],"phone-square":[448,512,[],"f098","M400 32H48C21.49 32 0 53.49 0 80v352c0 26.51 21.49 48 48 48h352c26.51 0 48-21.49 48-48V80c0-26.51-21.49-48-48-48zM94 416c-7.033 0-13.057-4.873-14.616-11.627l-14.998-65a15 15 0 0 1 8.707-17.16l69.998-29.999a15 15 0 0 1 17.518 4.289l30.997 37.885c48.944-22.963 88.297-62.858 110.781-110.78l-37.886-30.997a15.001 15.001 0 0 1-4.289-17.518l30-69.998a15 15 0 0 1 17.16-8.707l65 14.998A14.997 14.997 0 0 1 384 126c0 160.292-129.945 290-290 290z"],"phone-square-alt":[448,512,[],"f87b","M400 32H48A48 48 0 0 0 0 80v352a48 48 0 0 0 48 48h352a48 48 0 0 0 48-48V80a48 48 0 0 0-48-48zm-16.39 307.37l-15 65A15 15 0 0 1 354 416C194 416 64 286.29 64 126a15.7 15.7 0 0 1 11.63-14.61l65-15A18.23 18.23 0 0 1 144 96a16.27 16.27 0 0 1 13.79 9.09l30 70A17.9 17.9 0 0 1 189 181a17 17 0 0 1-5.5 11.61l-37.89 31a231.91 231.91 0 0 0 110.78 110.78l31-37.89A17 17 0 0 1 299 291a17.85 17.85 0 0 1 5.91 1.21l70 30A16.25 16.25 0 0 1 384 336a17.41 17.41 0 0 1-.39 3.37z"],"phone-volume":[384,512,[],"f2a0","M97.333 506.966c-129.874-129.874-129.681-340.252 0-469.933 5.698-5.698 14.527-6.632 21.263-2.422l64.817 40.513a17.187 17.187 0 0 1 6.849 20.958l-32.408 81.021a17.188 17.188 0 0 1-17.669 10.719l-55.81-5.58c-21.051 58.261-20.612 122.471 0 179.515l55.811-5.581a17.188 17.188 0 0 1 17.669 10.719l32.408 81.022a17.188 17.188 0 0 1-6.849 20.958l-64.817 40.513a17.19 17.19 0 0 1-21.264-2.422zM247.126 95.473c11.832 20.047 11.832 45.008 0 65.055-3.95 6.693-13.108 7.959-18.718 2.581l-5.975-5.726c-3.911-3.748-4.793-9.622-2.261-14.41a32.063 32.063 0 0 0 0-29.945c-2.533-4.788-1.65-10.662 2.261-14.41l5.975-5.726c5.61-5.378 14.768-4.112 18.718 2.581zm91.787-91.187c60.14 71.604 60.092 175.882 0 247.428-4.474 5.327-12.53 5.746-17.552.933l-5.798-5.557c-4.56-4.371-4.977-11.529-.93-16.379 49.687-59.538 49.646-145.933 0-205.422-4.047-4.85-3.631-12.008.93-16.379l5.798-5.557c5.022-4.813 13.078-4.394 17.552.933zm-45.972 44.941c36.05 46.322 36.108 111.149 0 157.546-4.39 5.641-12.697 6.251-17.856 1.304l-5.818-5.579c-4.4-4.219-4.998-11.095-1.285-15.931 26.536-34.564 26.534-82.572 0-117.134-3.713-4.836-3.115-11.711 1.285-15.931l5.818-5.579c5.159-4.947 13.466-4.337 17.856 1.304z"],"photo-video":[640,512,[],"f87c","M608 0H160a32 32 0 0 0-32 32v96h160V64h192v320h128a32 32 0 0 0 32-32V32a32 32 0 0 0-32-32zM232 103a9 9 0 0 1-9 9h-30a9 9 0 0 1-9-9V73a9 9 0 0 1 9-9h30a9 9 0 0 1 9 9zm352 208a9 9 0 0 1-9 9h-30a9 9 0 0 1-9-9v-30a9 9 0 0 1 9-9h30a9 9 0 0 1 9 9zm0-104a9 9 0 0 1-9 9h-30a9 9 0 0 1-9-9v-30a9 9 0 0 1 9-9h30a9 9 0 0 1 9 9zm0-104a9 9 0 0 1-9 9h-30a9 9 0 0 1-9-9V73a9 9 0 0 1 9-9h30a9 9 0 0 1 9 9zm-168 57H32a32 32 0 0 0-32 32v288a32 32 0 0 0 32 32h384a32 32 0 0 0 32-32V192a32 32 0 0 0-32-32zM96 224a32 32 0 1 1-32 32 32 32 0 0 1 32-32zm288 224H64v-32l64-64 32 32 128-128 96 96z"],"piggy-bank":[576,512,[],"f4d3","M560 224h-29.5c-8.8-20-21.6-37.7-37.4-52.5L512 96h-32c-29.4 0-55.4 13.5-73 34.3-7.6-1.1-15.1-2.3-23-2.3H256c-77.4 0-141.9 55-156.8 128H56c-14.8 0-26.5-13.5-23.5-28.8C34.7 215.8 45.4 208 57 208h1c3.3 0 6-2.7 6-6v-20c0-3.3-2.7-6-6-6-28.5 0-53.9 20.4-57.5 48.6C-3.9 258.8 22.7 288 56 288h40c0 52.2 25.4 98.1 64 127.3V496c0 8.8 7.2 16 16 16h64c8.8 0 16-7.2 16-16v-48h128v48c0 8.8 7.2 16 16 16h64c8.8 0 16-7.2 16-16v-80.7c11.8-8.9 22.3-19.4 31.3-31.3H560c8.8 0 16-7.2 16-16V240c0-8.8-7.2-16-16-16zm-128 64c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zM256 96h128c5.4 0 10.7.4 15.9.8 0-.3.1-.5.1-.8 0-53-43-96-96-96s-96 43-96 96c0 2.1.5 4.1.6 6.2 15.2-3.9 31-6.2 47.4-6.2z"],pills:[576,512,[],"f484","M112 32C50.1 32 0 82.1 0 144v224c0 61.9 50.1 112 112 112s112-50.1 112-112V144c0-61.9-50.1-112-112-112zm48 224H64V144c0-26.5 21.5-48 48-48s48 21.5 48 48v112zm139.7-29.7c-3.5-3.5-9.4-3.1-12.3.8-45.3 62.5-40.4 150.1 15.9 206.4 56.3 56.3 143.9 61.2 206.4 15.9 4-2.9 4.3-8.8.8-12.3L299.7 226.3zm229.8-19c-56.3-56.3-143.9-61.2-206.4-15.9-4 2.9-4.3 8.8-.8 12.3l210.8 210.8c3.5 3.5 9.4 3.1 12.3-.8 45.3-62.6 40.5-150.1-15.9-206.4z"],"pizza-slice":[512,512,[],"f818","M158.87.15c-16.16-1.52-31.2 8.42-35.33 24.12l-14.81 56.27c187.62 5.49 314.54 130.61 322.48 317l56.94-15.78c15.72-4.36 25.49-19.68 23.62-35.9C490.89 165.08 340.78 17.32 158.87.15zm-58.47 112L.55 491.64a16.21 16.21 0 0 0 20 19.75l379-105.1c-4.27-174.89-123.08-292.14-299.15-294.1zM128 416a32 32 0 1 1 32-32 32 32 0 0 1-32 32zm48-152a32 32 0 1 1 32-32 32 32 0 0 1-32 32zm104 104a32 32 0 1 1 32-32 32 32 0 0 1-32 32z"],"place-of-worship":[640,512,[],"f67f","M620.61 366.55L512 320v192h112c8.84 0 16-7.16 16-16V395.96a32 32 0 0 0-19.39-29.41zM0 395.96V496c0 8.84 7.16 16 16 16h112V320L19.39 366.55A32 32 0 0 0 0 395.96zm464.46-149.28L416 217.6V102.63c0-8.49-3.37-16.62-9.38-22.63L331.31 4.69c-6.25-6.25-16.38-6.25-22.62 0L233.38 80c-6 6-9.38 14.14-9.38 22.63V217.6l-48.46 29.08A31.997 31.997 0 0 0 160 274.12V512h96v-96c0-35.35 28.66-64 64-64s64 28.65 64 64v96h96V274.12c0-11.24-5.9-21.66-15.54-27.44z"],plane:[576,512,[],"f072","M480 192H365.71L260.61 8.06A16.014 16.014 0 0 0 246.71 0h-65.5c-10.63 0-18.3 10.17-15.38 20.39L214.86 192H112l-43.2-57.6c-3.02-4.03-7.77-6.4-12.8-6.4H16.01C5.6 128-2.04 137.78.49 147.88L32 256 .49 364.12C-2.04 374.22 5.6 384 16.01 384H56c5.04 0 9.78-2.37 12.8-6.4L112 320h102.86l-49.03 171.6c-2.92 10.22 4.75 20.4 15.38 20.4h65.5c5.74 0 11.04-3.08 13.89-8.06L365.71 320H480c35.35 0 96-28.65 96-64s-60.65-64-96-64z"],"plane-arrival":[640,512,[],"f5af","M624 448H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h608c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zM44.81 205.66l88.74 80a62.607 62.607 0 0 0 25.47 13.93l287.6 78.35c26.48 7.21 54.56 8.72 81 1.36 29.67-8.27 43.44-21.21 47.25-35.71 3.83-14.5-1.73-32.71-23.37-54.96-19.28-19.82-44.35-32.79-70.83-40l-97.51-26.56L282.8 30.22c-1.51-5.81-5.95-10.35-11.66-11.91L206.05.58c-10.56-2.88-20.9 5.32-20.71 16.44l47.92 164.21-102.2-27.84-27.59-67.88c-1.93-4.89-6.01-8.57-11.02-9.93L52.72 64.75c-10.34-2.82-20.53 5-20.72 15.88l.23 101.78c.19 8.91 6.03 17.34 12.58 23.25z"],"plane-departure":[640,512,[],"f5b0","M624 448H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h608c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zM80.55 341.27c6.28 6.84 15.1 10.72 24.33 10.71l130.54-.18a65.62 65.62 0 0 0 29.64-7.12l290.96-147.65c26.74-13.57 50.71-32.94 67.02-58.31 18.31-28.48 20.3-49.09 13.07-63.65-7.21-14.57-24.74-25.27-58.25-27.45-29.85-1.94-59.54 5.92-86.28 19.48l-98.51 49.99-218.7-82.06a17.799 17.799 0 0 0-18-1.11L90.62 67.29c-10.67 5.41-13.25 19.65-5.17 28.53l156.22 98.1-103.21 52.38-72.35-36.47a17.804 17.804 0 0 0-16.07.02L9.91 230.22c-10.44 5.3-13.19 19.12-5.57 28.08l76.21 82.97z"],"plane-slash":[640,512,[],"f969","M32.48,147.88,64,256,32.48,364.13A16,16,0,0,0,48,384H88a16,16,0,0,0,12.8-6.41L144,320H246.85l-49,171.59A16,16,0,0,0,213.2,512h65.5a16,16,0,0,0,13.89-8.06l66.6-116.54L34.35,136.34A15.47,15.47,0,0,0,32.48,147.88ZM633.82,458.09,455.14,320H512c35.34,0,96-28.66,96-64s-60.66-64-96-64H397.7L292.61,8.06C290.06,3.61,283.84,0,278.71,0H213.2a16,16,0,0,0-15.38,20.39l36.94,129.29L45.46,3.38A16,16,0,0,0,23,6.19L3.37,31.45A16,16,0,0,0,6.18,53.91L594.54,508.63A16,16,0,0,0,617,505.81l19.64-25.26A16,16,0,0,0,633.82,458.09Z"],play:[448,512,[],"f04b","M424.4 214.7L72.4 6.6C43.8-10.3 0 6.1 0 47.9V464c0 37.5 40.7 60.1 72.4 41.3l352-208c31.4-18.5 31.5-64.1 0-82.6z"],"play-circle":[512,512,[],"f144","M256 8C119 8 8 119 8 256s111 248 248 248 248-111 248-248S393 8 256 8zm115.7 272l-176 101c-15.8 8.8-35.7-2.5-35.7-21V152c0-18.4 19.8-29.8 35.7-21l176 107c16.4 9.2 16.4 32.9 0 42z"],plug:[384,512,[],"f1e6","M320,32a32,32,0,0,0-64,0v96h64Zm48,128H16A16,16,0,0,0,0,176v32a16,16,0,0,0,16,16H32v32A160.07,160.07,0,0,0,160,412.8V512h64V412.8A160.07,160.07,0,0,0,352,256V224h16a16,16,0,0,0,16-16V176A16,16,0,0,0,368,160ZM128,32a32,32,0,0,0-64,0v96h64Z"],plus:[448,512,[],"f067","M416 208H272V64c0-17.67-14.33-32-32-32h-32c-17.67 0-32 14.33-32 32v144H32c-17.67 0-32 14.33-32 32v32c0 17.67 14.33 32 32 32h144v144c0 17.67 14.33 32 32 32h32c17.67 0 32-14.33 32-32V304h144c17.67 0 32-14.33 32-32v-32c0-17.67-14.33-32-32-32z"],"plus-circle":[512,512,[],"f055","M256 8C119 8 8 119 8 256s111 248 248 248 248-111 248-248S393 8 256 8zm144 276c0 6.6-5.4 12-12 12h-92v92c0 6.6-5.4 12-12 12h-56c-6.6 0-12-5.4-12-12v-92h-92c-6.6 0-12-5.4-12-12v-56c0-6.6 5.4-12 12-12h92v-92c0-6.6 5.4-12 12-12h56c6.6 0 12 5.4 12 12v92h92c6.6 0 12 5.4 12 12v56z"],"plus-square":[448,512,[],"f0fe","M400 32H48C21.5 32 0 53.5 0 80v352c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V80c0-26.5-21.5-48-48-48zm-32 252c0 6.6-5.4 12-12 12h-92v92c0 6.6-5.4 12-12 12h-56c-6.6 0-12-5.4-12-12v-92H92c-6.6 0-12-5.4-12-12v-56c0-6.6 5.4-12 12-12h92v-92c0-6.6 5.4-12 12-12h56c6.6 0 12 5.4 12 12v92h92c6.6 0 12 5.4 12 12v56z"],podcast:[448,512,[],"f2ce","M267.429 488.563C262.286 507.573 242.858 512 224 512c-18.857 0-38.286-4.427-43.428-23.437C172.927 460.134 160 388.898 160 355.75c0-35.156 31.142-43.75 64-43.75s64 8.594 64 43.75c0 32.949-12.871 104.179-20.571 132.813zM156.867 288.554c-18.693-18.308-29.958-44.173-28.784-72.599 2.054-49.724 42.395-89.956 92.124-91.881C274.862 121.958 320 165.807 320 220c0 26.827-11.064 51.116-28.866 68.552-2.675 2.62-2.401 6.986.628 9.187 9.312 6.765 16.46 15.343 21.234 25.363 1.741 3.654 6.497 4.66 9.449 1.891 28.826-27.043 46.553-65.783 45.511-108.565-1.855-76.206-63.595-138.208-139.793-140.369C146.869 73.753 80 139.215 80 220c0 41.361 17.532 78.7 45.55 104.989 2.953 2.771 7.711 1.77 9.453-1.887 4.774-10.021 11.923-18.598 21.235-25.363 3.029-2.2 3.304-6.566.629-9.185zM224 0C100.204 0 0 100.185 0 224c0 89.992 52.602 165.647 125.739 201.408 4.333 2.118 9.267-1.544 8.535-6.31-2.382-15.512-4.342-30.946-5.406-44.339-.146-1.836-1.149-3.486-2.678-4.512-47.4-31.806-78.564-86.016-78.187-147.347.592-96.237 79.29-174.648 175.529-174.899C320.793 47.747 400 126.797 400 224c0 61.932-32.158 116.49-80.65 147.867-.999 14.037-3.069 30.588-5.624 47.23-.732 4.767 4.203 8.429 8.535 6.31C395.227 389.727 448 314.187 448 224 448 100.205 347.815 0 224 0zm0 160c-35.346 0-64 28.654-64 64s28.654 64 64 64 64-28.654 64-64-28.654-64-64-64z"],poll:[448,512,[],"f681","M400 32H48C21.5 32 0 53.5 0 80v352c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V80c0-26.5-21.5-48-48-48zM160 368c0 8.84-7.16 16-16 16h-32c-8.84 0-16-7.16-16-16V240c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v128zm96 0c0 8.84-7.16 16-16 16h-32c-8.84 0-16-7.16-16-16V144c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v224zm96 0c0 8.84-7.16 16-16 16h-32c-8.84 0-16-7.16-16-16v-64c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v64z"],"poll-h":[448,512,[],"f682","M448 432V80c0-26.5-21.5-48-48-48H48C21.5 32 0 53.5 0 80v352c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48zM112 192c-8.84 0-16-7.16-16-16v-32c0-8.84 7.16-16 16-16h128c8.84 0 16 7.16 16 16v32c0 8.84-7.16 16-16 16H112zm0 96c-8.84 0-16-7.16-16-16v-32c0-8.84 7.16-16 16-16h224c8.84 0 16 7.16 16 16v32c0 8.84-7.16 16-16 16H112zm0 96c-8.84 0-16-7.16-16-16v-32c0-8.84 7.16-16 16-16h64c8.84 0 16 7.16 16 16v32c0 8.84-7.16 16-16 16h-64z"],poo:[512,512,[],"f2fe","M451.4 369.1C468.7 356 480 335.4 480 312c0-39.8-32.2-72-72-72h-14.1c13.4-11.7 22.1-28.8 22.1-48 0-35.3-28.7-64-64-64h-5.9c3.6-10.1 5.9-20.7 5.9-32 0-53-43-96-96-96-5.2 0-10.2.7-15.1 1.5C250.3 14.6 256 30.6 256 48c0 44.2-35.8 80-80 80h-16c-35.3 0-64 28.7-64 64 0 19.2 8.7 36.3 22.1 48H104c-39.8 0-72 32.2-72 72 0 23.4 11.3 44 28.6 57.1C26.3 374.6 0 404.1 0 440c0 39.8 32.2 72 72 72h368c39.8 0 72-32.2 72-72 0-35.9-26.3-65.4-60.6-70.9zM192 256c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zm159.5 139C341 422.9 293 448 256 448s-85-25.1-95.5-53c-2-5.3 2-11 7.8-11h175.4c5.8 0 9.8 5.7 7.8 11zM320 320c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],"poo-storm":[448,512,[],"f75a","M308 336h-57.7l17.3-64.9c2-7.6-3.7-15.1-11.6-15.1h-68c-6 0-11.1 4.5-11.9 10.4l-16 120c-1 7.2 4.6 13.6 11.9 13.6h59.3l-23 97.2c-1.8 7.6 4 14.8 11.7 14.8 4.2 0 8.2-2.2 10.4-6l88-152c4.6-8-1.2-18-10.4-18zm66.4-111.3c5.9-9.6 9.6-20.6 9.6-32.7 0-35.3-28.7-64-64-64h-5.9c3.6-10.1 5.9-20.7 5.9-32 0-53-43-96-96-96-5.2 0-10.2.7-15.1 1.5C218.3 14.6 224 30.6 224 48c0 44.2-35.8 80-80 80h-16c-35.3 0-64 28.7-64 64 0 12.1 3.7 23.1 9.6 32.7C32.6 228 0 262.2 0 304c0 44 36 80 80 80h48.3c.1-.6 0-1.2 0-1.8l16-120c3-21.8 21.7-38.2 43.7-38.2h68c13.8 0 26.5 6.3 34.9 17.2s11.2 24.8 7.6 38.1l-6.6 24.7h16c15.7 0 30.3 8.4 38.1 22 7.8 13.6 7.8 30.5 0 44l-8.1 14h30c44 0 80-36 80-80 .1-41.8-32.5-76-73.5-79.3z"],poop:[512,512,[],"f619","M451.36 369.14C468.66 355.99 480 335.41 480 312c0-39.77-32.24-72-72-72h-14.07c13.42-11.73 22.07-28.78 22.07-48 0-35.35-28.65-64-64-64h-5.88c3.57-10.05 5.88-20.72 5.88-32 0-53.02-42.98-96-96-96-5.17 0-10.15.74-15.11 1.52C250.31 14.64 256 30.62 256 48c0 44.18-35.82 80-80 80h-16c-35.35 0-64 28.65-64 64 0 19.22 8.65 36.27 22.07 48H104c-39.76 0-72 32.23-72 72 0 23.41 11.34 43.99 28.64 57.14C26.31 374.62 0 404.12 0 440c0 39.76 32.24 72 72 72h368c39.76 0 72-32.24 72-72 0-35.88-26.31-65.38-60.64-70.86z"],portrait:[384,512,[],"f3e0","M336 0H48C21.5 0 0 21.5 0 48v416c0 26.5 21.5 48 48 48h288c26.5 0 48-21.5 48-48V48c0-26.5-21.5-48-48-48zM192 128c35.3 0 64 28.7 64 64s-28.7 64-64 64-64-28.7-64-64 28.7-64 64-64zm112 236.8c0 10.6-10 19.2-22.4 19.2H102.4C90 384 80 375.4 80 364.8v-19.2c0-31.8 30.1-57.6 67.2-57.6h5c12.3 5.1 25.7 8 39.8 8s27.6-2.9 39.8-8h5c37.1 0 67.2 25.8 67.2 57.6v19.2z"],"pound-sign":[320,512,[],"f154","M308 352h-45.495c-6.627 0-12 5.373-12 12v50.848H128V288h84c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12h-84v-63.556c0-32.266 24.562-57.086 61.792-57.086 23.658 0 45.878 11.505 57.652 18.849 5.151 3.213 11.888 2.051 15.688-2.685l28.493-35.513c4.233-5.276 3.279-13.005-2.119-17.081C273.124 54.56 236.576 32 187.931 32 106.026 32 48 84.742 48 157.961V224H20c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h28v128H12c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h296c6.627 0 12-5.373 12-12V364c0-6.627-5.373-12-12-12z"],"power-off":[512,512,[],"f011","M400 54.1c63 45 104 118.6 104 201.9 0 136.8-110.8 247.7-247.5 248C120 504.3 8.2 393 8 256.4 7.9 173.1 48.9 99.3 111.8 54.2c11.7-8.3 28-4.8 35 7.7L162.6 90c5.9 10.5 3.1 23.8-6.6 31-41.5 30.8-68 79.6-68 134.9-.1 92.3 74.5 168.1 168 168.1 91.6 0 168.6-74.2 168-169.1-.3-51.8-24.7-101.8-68.1-134-9.7-7.2-12.4-20.5-6.5-30.9l15.8-28.1c7-12.4 23.2-16.1 34.8-7.8zM296 264V24c0-13.3-10.7-24-24-24h-32c-13.3 0-24 10.7-24 24v240c0 13.3 10.7 24 24 24h32c13.3 0 24-10.7 24-24z"],pray:[384,512,[],"f683","M256 128c35.35 0 64-28.65 64-64S291.35 0 256 0s-64 28.65-64 64 28.65 64 64 64zm-30.63 169.75c14.06 16.72 39 19.09 55.97 5.22l88-72.02c17.09-13.98 19.59-39.19 5.62-56.28-13.97-17.11-39.19-19.59-56.31-5.62l-57.44 47-38.91-46.31c-15.44-18.39-39.22-27.92-64-25.33-24.19 2.48-45.25 16.27-56.37 36.92l-49.37 92.03c-23.4 43.64-8.69 96.37 34.19 123.75L131.56 432H40c-22.09 0-40 17.91-40 40s17.91 40 40 40h208c34.08 0 53.77-42.79 28.28-68.28L166.42 333.86l34.8-64.87 24.15 28.76z"],"praying-hands":[640,512,[],"f684","M272 191.91c-17.6 0-32 14.4-32 32v80c0 8.84-7.16 16-16 16s-16-7.16-16-16v-76.55c0-17.39 4.72-34.47 13.69-49.39l77.75-129.59c9.09-15.16 4.19-34.81-10.97-43.91-14.45-8.67-32.72-4.3-42.3 9.21-.2.23-.62.21-.79.48l-117.26 175.9C117.56 205.9 112 224.31 112 243.29v80.23l-90.12 30.04A31.974 31.974 0 0 0 0 383.91v96c0 10.82 8.52 32 32 32 2.69 0 5.41-.34 8.06-1.03l179.19-46.62C269.16 449.99 304 403.8 304 351.91v-128c0-17.6-14.4-32-32-32zm346.12 161.73L528 323.6v-80.23c0-18.98-5.56-37.39-16.12-53.23L394.62 14.25c-.18-.27-.59-.24-.79-.48-9.58-13.51-27.85-17.88-42.3-9.21-15.16 9.09-20.06 28.75-10.97 43.91l77.75 129.59c8.97 14.92 13.69 32 13.69 49.39V304c0 8.84-7.16 16-16 16s-16-7.16-16-16v-80c0-17.6-14.4-32-32-32s-32 14.4-32 32v128c0 51.89 34.84 98.08 84.75 112.34l179.19 46.62c2.66.69 5.38 1.03 8.06 1.03 23.48 0 32-21.18 32-32v-96c0-13.77-8.81-25.99-21.88-30.35z"],prescription:[384,512,[],"f5b1","M301.26 352l78.06-78.06c6.25-6.25 6.25-16.38 0-22.63l-22.63-22.63c-6.25-6.25-16.38-6.25-22.63 0L256 306.74l-83.96-83.96C219.31 216.8 256 176.89 256 128c0-53.02-42.98-96-96-96H16C7.16 32 0 39.16 0 48v256c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-80h18.75l128 128-78.06 78.06c-6.25 6.25-6.25 16.38 0 22.63l22.63 22.63c6.25 6.25 16.38 6.25 22.63 0L256 397.25l78.06 78.06c6.25 6.25 16.38 6.25 22.63 0l22.63-22.63c6.25-6.25 6.25-16.38 0-22.63L301.26 352zM64 96h96c17.64 0 32 14.36 32 32s-14.36 32-32 32H64V96z"],"prescription-bottle":[384,512,[],"f485","M32 192h120c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H32v64h120c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H32v64h120c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H32v64c0 17.6 14.4 32 32 32h256c17.6 0 32-14.4 32-32V128H32v64zM360 0H24C10.8 0 0 10.8 0 24v48c0 13.2 10.8 24 24 24h336c13.2 0 24-10.8 24-24V24c0-13.2-10.8-24-24-24z"],"prescription-bottle-alt":[384,512,[],"f486","M360 0H24C10.8 0 0 10.8 0 24v48c0 13.2 10.8 24 24 24h336c13.2 0 24-10.8 24-24V24c0-13.2-10.8-24-24-24zM32 480c0 17.6 14.4 32 32 32h256c17.6 0 32-14.4 32-32V128H32v352zm64-184c0-4.4 3.6-8 8-8h56v-56c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v56h56c4.4 0 8 3.6 8 8v48c0 4.4-3.6 8-8 8h-56v56c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8v-56h-56c-4.4 0-8-3.6-8-8v-48z"],print:[512,512,[],"f02f","M448 192V77.25c0-8.49-3.37-16.62-9.37-22.63L393.37 9.37c-6-6-14.14-9.37-22.63-9.37H96C78.33 0 64 14.33 64 32v160c-35.35 0-64 28.65-64 64v112c0 8.84 7.16 16 16 16h48v96c0 17.67 14.33 32 32 32h320c17.67 0 32-14.33 32-32v-96h48c8.84 0 16-7.16 16-16V256c0-35.35-28.65-64-64-64zm-64 256H128v-96h256v96zm0-224H128V64h192v48c0 8.84 7.16 16 16 16h48v96zm48 72c-13.25 0-24-10.75-24-24 0-13.26 10.75-24 24-24s24 10.74 24 24c0 13.25-10.75 24-24 24z"],procedures:[640,512,[],"f487","M528 224H272c-8.8 0-16 7.2-16 16v144H64V144c0-8.8-7.2-16-16-16H16c-8.8 0-16 7.2-16 16v352c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16v-48h512v48c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V336c0-61.9-50.1-112-112-112zM136 96h126.1l27.6 55.2c5.9 11.8 22.7 11.8 28.6 0L368 51.8 390.1 96H512c8.8 0 16-7.2 16-16s-7.2-16-16-16H409.9L382.3 8.8C376.4-3 359.6-3 353.7 8.8L304 108.2l-19.9-39.8c-1.4-2.7-4.1-4.4-7.2-4.4H136c-4.4 0-8 3.6-8 8v16c0 4.4 3.6 8 8 8zm24 256c35.3 0 64-28.7 64-64s-28.7-64-64-64-64 28.7-64 64 28.7 64 64 64z"],"project-diagram":[640,512,[],"f542","M384 320H256c-17.67 0-32 14.33-32 32v128c0 17.67 14.33 32 32 32h128c17.67 0 32-14.33 32-32V352c0-17.67-14.33-32-32-32zM192 32c0-17.67-14.33-32-32-32H32C14.33 0 0 14.33 0 32v128c0 17.67 14.33 32 32 32h95.72l73.16 128.04C211.98 300.98 232.4 288 256 288h.28L192 175.51V128h224V64H192V32zM608 0H480c-17.67 0-32 14.33-32 32v128c0 17.67 14.33 32 32 32h128c17.67 0 32-14.33 32-32V32c0-17.67-14.33-32-32-32z"],"pump-medical":[384,512,[],"f96a","M235.51,159.82H84.24A64,64,0,0,0,20.51,218L.14,442a64,64,0,0,0,63.74,69.8h192A64,64,0,0,0,319.61,442L299.24,218A64,64,0,0,0,235.51,159.82Zm4.37,173.33a13.35,13.35,0,0,1-13.34,13.34h-40v40a13.33,13.33,0,0,1-13.33,13.33H146.54a13.33,13.33,0,0,1-13.33-13.33v-40h-40a13.34,13.34,0,0,1-13.33-13.34V306.49a13.33,13.33,0,0,1,13.33-13.34h40v-40a13.33,13.33,0,0,1,13.33-13.33h26.67a13.33,13.33,0,0,1,13.33,13.33v40h40a13.34,13.34,0,0,1,13.34,13.34ZM379.19,93.88,335.87,50.56a64,64,0,0,0-45.24-18.74H223.88a32,32,0,0,0-32-32h-64a32,32,0,0,0-32,32v96h128v-32h66.75l43.31,43.31a16,16,0,0,0,22.63,0l22.62-22.62A16,16,0,0,0,379.19,93.88Z"],"pump-soap":[384,512,[],"f96b","M235.63,160H84.37a64,64,0,0,0-63.74,58.21L.27,442.21A64,64,0,0,0,64,512H256a64,64,0,0,0,63.74-69.79l-20.36-224A64,64,0,0,0,235.63,160ZM160,416c-33.12,0-60-26.33-60-58.75,0-25,35.7-75.47,52-97.27A10,10,0,0,1,168,260c16.33,21.8,52,72.27,52,97.27C220,389.67,193.12,416,160,416ZM379.31,94.06,336,50.74A64,64,0,0,0,290.75,32H224A32,32,0,0,0,192,0H128A32,32,0,0,0,96,32v96H224V96h66.75l43.31,43.31a16,16,0,0,0,22.63,0l22.62-22.62A16,16,0,0,0,379.31,94.06Z"],"puzzle-piece":[576,512,[],"f12e","M519.442 288.651c-41.519 0-59.5 31.593-82.058 31.593C377.409 320.244 432 144 432 144s-196.288 80-196.288-3.297c0-35.827 36.288-46.25 36.288-85.985C272 19.216 243.885 0 210.539 0c-34.654 0-66.366 18.891-66.366 56.346 0 41.364 31.711 59.277 31.711 81.75C175.885 207.719 0 166.758 0 166.758v333.237s178.635 41.047 178.635-28.662c0-22.473-40-40.107-40-81.471 0-37.456 29.25-56.346 63.577-56.346 33.673 0 61.788 19.216 61.788 54.717 0 39.735-36.288 50.158-36.288 85.985 0 60.803 129.675 25.73 181.23 25.73 0 0-34.725-120.101 25.827-120.101 35.962 0 46.423 36.152 86.308 36.152C556.712 416 576 387.99 576 354.443c0-34.199-18.962-65.792-56.558-65.792z"],qrcode:[448,512,[],"f029","M0 224h192V32H0v192zM64 96h64v64H64V96zm192-64v192h192V32H256zm128 128h-64V96h64v64zM0 480h192V288H0v192zm64-128h64v64H64v-64zm352-64h32v128h-96v-32h-32v96h-64V288h96v32h64v-32zm0 160h32v32h-32v-32zm-64 0h32v32h-32v-32z"],question:[384,512,[],"f128","M202.021 0C122.202 0 70.503 32.703 29.914 91.026c-7.363 10.58-5.093 25.086 5.178 32.874l43.138 32.709c10.373 7.865 25.132 6.026 33.253-4.148 25.049-31.381 43.63-49.449 82.757-49.449 30.764 0 68.816 19.799 68.816 49.631 0 22.552-18.617 34.134-48.993 51.164-35.423 19.86-82.299 44.576-82.299 106.405V320c0 13.255 10.745 24 24 24h72.471c13.255 0 24-10.745 24-24v-5.773c0-42.86 125.268-44.645 125.268-160.627C377.504 66.256 286.902 0 202.021 0zM192 373.459c-38.196 0-69.271 31.075-69.271 69.271 0 38.195 31.075 69.27 69.271 69.27s69.271-31.075 69.271-69.271-31.075-69.27-69.271-69.27z"],"question-circle":[512,512,[],"f059","M504 256c0 136.997-111.043 248-248 248S8 392.997 8 256C8 119.083 119.043 8 256 8s248 111.083 248 248zM262.655 90c-54.497 0-89.255 22.957-116.549 63.758-3.536 5.286-2.353 12.415 2.715 16.258l34.699 26.31c5.205 3.947 12.621 3.008 16.665-2.122 17.864-22.658 30.113-35.797 57.303-35.797 20.429 0 45.698 13.148 45.698 32.958 0 14.976-12.363 22.667-32.534 33.976C247.128 238.528 216 254.941 216 296v4c0 6.627 5.373 12 12 12h56c6.627 0 12-5.373 12-12v-1.333c0-28.462 83.186-29.647 83.186-106.667 0-58.002-60.165-102-116.531-102zM256 338c-25.365 0-46 20.635-46 46 0 25.364 20.635 46 46 46s46-20.636 46-46c0-25.365-20.635-46-46-46z"],quidditch:[640,512,[],"f458","M256.5 216.8L343.2 326s-16.6 102.4-76.6 150.1C206.7 523.8 0 510.2 0 510.2s3.8-23.1 11-55.4l94.6-112.2c4-4.7-.9-11.6-6.6-9.5l-60.4 22.1c14.4-41.7 32.7-80 54.6-97.5 59.9-47.8 163.3-40.9 163.3-40.9zm238 135c-44 0-79.8 35.8-79.8 79.9 0 44.1 35.7 79.9 79.8 79.9 44.1 0 79.8-35.8 79.8-79.9 0-44.2-35.8-79.9-79.8-79.9zM636.5 31L616.7 6c-5.5-6.9-15.5-8-22.4-2.6L361.8 181.3l-34.1-43c-5.1-6.4-15.1-5.2-18.6 2.2l-25.3 54.6 86.7 109.2 58.8-12.4c8-1.7 11.4-11.2 6.3-17.6l-34.1-42.9L634 53.5c6.9-5.5 8-15.6 2.5-22.5z"],"quote-left":[512,512,[],"f10d","M464 256h-80v-64c0-35.3 28.7-64 64-64h8c13.3 0 24-10.7 24-24V56c0-13.3-10.7-24-24-24h-8c-88.4 0-160 71.6-160 160v240c0 26.5 21.5 48 48 48h128c26.5 0 48-21.5 48-48V304c0-26.5-21.5-48-48-48zm-288 0H96v-64c0-35.3 28.7-64 64-64h8c13.3 0 24-10.7 24-24V56c0-13.3-10.7-24-24-24h-8C71.6 32 0 103.6 0 192v240c0 26.5 21.5 48 48 48h128c26.5 0 48-21.5 48-48V304c0-26.5-21.5-48-48-48z"],"quote-right":[512,512,[],"f10e","M464 32H336c-26.5 0-48 21.5-48 48v128c0 26.5 21.5 48 48 48h80v64c0 35.3-28.7 64-64 64h-8c-13.3 0-24 10.7-24 24v48c0 13.3 10.7 24 24 24h8c88.4 0 160-71.6 160-160V80c0-26.5-21.5-48-48-48zm-288 0H48C21.5 32 0 53.5 0 80v128c0 26.5 21.5 48 48 48h80v64c0 35.3-28.7 64-64 64h-8c-13.3 0-24 10.7-24 24v48c0 13.3 10.7 24 24 24h8c88.4 0 160-71.6 160-160V80c0-26.5-21.5-48-48-48z"],quran:[448,512,[],"f687","M448 358.4V25.6c0-16-9.6-25.6-25.6-25.6H96C41.6 0 0 41.6 0 96v320c0 54.4 41.6 96 96 96h326.4c12.8 0 25.6-9.6 25.6-25.6v-16c0-6.4-3.2-12.8-9.6-19.2-3.2-16-3.2-60.8 0-73.6 6.4-3.2 9.6-9.6 9.6-19.2zM301.08 145.82c.6-1.21 1.76-1.82 2.92-1.82s2.32.61 2.92 1.82l11.18 22.65 25 3.63c2.67.39 3.74 3.67 1.81 5.56l-18.09 17.63 4.27 24.89c.36 2.11-1.31 3.82-3.21 3.82-.5 0-1.02-.12-1.52-.38L304 211.87l-22.36 11.75c-.5.26-1.02.38-1.52.38-1.9 0-3.57-1.71-3.21-3.82l4.27-24.89-18.09-17.63c-1.94-1.89-.87-5.17 1.81-5.56l24.99-3.63 11.19-22.65zm-57.89-69.01c13.67 0 27.26 2.49 40.38 7.41a6.775 6.775 0 1 1-2.38 13.12c-.67 0-3.09-.21-4.13-.21-52.31 0-94.86 42.55-94.86 94.86 0 52.3 42.55 94.86 94.86 94.86 1.03 0 3.48-.21 4.13-.21 3.93 0 6.8 3.14 6.8 6.78 0 2.98-1.94 5.51-4.62 6.42-13.07 4.87-26.59 7.34-40.19 7.34C179.67 307.19 128 255.51 128 192c0-63.52 51.67-115.19 115.19-115.19zM380.8 448H96c-19.2 0-32-12.8-32-32s16-32 32-32h284.8v64z"],radiation:[496,512,[],"f7b9","M328.2 255.8h151.6c9.1 0 16.8-7.7 16.2-16.8-5.1-75.8-44.4-142.2-102.5-184.2-7.4-5.3-17.9-2.9-22.7 4.8L290.4 188c22.6 14.3 37.8 39.2 37.8 67.8zm-37.8 67.7c-12.3 7.7-26.8 12.4-42.4 12.4-15.6 0-30-4.7-42.4-12.4L125.2 452c-4.8 7.7-2.4 18.1 5.6 22.4C165.7 493.2 205.6 504 248 504s82.3-10.8 117.2-29.6c8-4.3 10.4-14.8 5.6-22.4l-80.4-128.5zM248 303.8c26.5 0 48-21.5 48-48s-21.5-48-48-48-48 21.5-48 48 21.5 48 48 48zm-231.8-48h151.6c0-28.6 15.2-53.5 37.8-67.7L125.2 59.7c-4.8-7.7-15.3-10.2-22.7-4.8C44.4 96.9 5.1 163.3 0 239.1c-.6 9 7.1 16.7 16.2 16.7z"],"radiation-alt":[496,512,[],"f7ba","M312 256h79.1c9.2 0 16.9-7.7 16-16.8-4.6-43.6-27-81.8-59.5-107.8-7.6-6.1-18.8-4.5-24 3.8L281.9 202c18 11.2 30.1 31.2 30.1 54zm-97.8 54.1L172.4 377c-4.9 7.8-2.4 18.4 5.8 22.5 21.1 10.4 44.7 16.5 69.8 16.5s48.7-6.1 69.9-16.5c8.2-4.1 10.6-14.7 5.8-22.5l-41.8-66.9c-9.8 6.2-21.4 9.9-33.8 9.9s-24.1-3.7-33.9-9.9zM104.9 256H184c0-22.8 12.1-42.8 30.2-54.1l-41.7-66.8c-5.2-8.3-16.4-9.9-24-3.8-32.6 26-54.9 64.2-59.5 107.8-1.1 9.2 6.7 16.9 15.9 16.9zM248 504c137 0 248-111 248-248S385 8 248 8 0 119 0 256s111 248 248 248zm0-432c101.5 0 184 82.5 184 184s-82.5 184-184 184S64 357.5 64 256 146.5 72 248 72zm0 216c17.7 0 32-14.3 32-32s-14.3-32-32-32-32 14.3-32 32 14.3 32 32 32z"],rainbow:[576,512,[],"f75b","M268.3 32.7C115.4 42.9 0 176.9 0 330.2V464c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V320C64 186.8 180.9 80.3 317.5 97.9 430.4 112.4 512 214 512 327.8V464c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V320c0-165.3-140-298.6-307.7-287.3zm-5.6 96.9C166 142 96 229.1 96 326.7V464c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V320c0-74.8 64.5-134.8 140.8-127.4 66.5 6.5 115.2 66.2 115.2 133.1V464c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V320c0-114.2-100.2-205.4-217.3-190.4zm6.2 96.3c-45.6 8.9-76.9 51.5-76.9 97.9V464c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V320c0-17.6 14.3-32 32-32s32 14.4 32 32v144c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V320c0-59.2-53.8-106-115.1-94.1z"],random:[512,512,[],"f074","M504.971 359.029c9.373 9.373 9.373 24.569 0 33.941l-80 79.984c-15.01 15.01-40.971 4.49-40.971-16.971V416h-58.785a12.004 12.004 0 0 1-8.773-3.812l-70.556-75.596 53.333-57.143L352 336h32v-39.981c0-21.438 25.943-31.998 40.971-16.971l80 79.981zM12 176h84l52.781 56.551 53.333-57.143-70.556-75.596A11.999 11.999 0 0 0 122.785 96H12c-6.627 0-12 5.373-12 12v56c0 6.627 5.373 12 12 12zm372 0v39.984c0 21.46 25.961 31.98 40.971 16.971l80-79.984c9.373-9.373 9.373-24.569 0-33.941l-80-79.981C409.943 24.021 384 34.582 384 56.019V96h-58.785a12.004 12.004 0 0 0-8.773 3.812L96 336H12c-6.627 0-12 5.373-12 12v56c0 6.627 5.373 12 12 12h110.785c3.326 0 6.503-1.381 8.773-3.812L352 176h32z"],receipt:[384,512,[],"f543","M358.4 3.2L320 48 265.6 3.2a15.9 15.9 0 0 0-19.2 0L192 48 137.6 3.2a15.9 15.9 0 0 0-19.2 0L64 48 25.6 3.2C15-4.7 0 2.8 0 16v480c0 13.2 15 20.7 25.6 12.8L64 464l54.4 44.8a15.9 15.9 0 0 0 19.2 0L192 464l54.4 44.8a15.9 15.9 0 0 0 19.2 0L320 464l38.4 44.8c10.5 7.9 25.6.4 25.6-12.8V16c0-13.2-15-20.7-25.6-12.8zM320 360c0 4.4-3.6 8-8 8H72c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h240c4.4 0 8 3.6 8 8v16zm0-96c0 4.4-3.6 8-8 8H72c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h240c4.4 0 8 3.6 8 8v16zm0-96c0 4.4-3.6 8-8 8H72c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h240c4.4 0 8 3.6 8 8v16z"],"record-vinyl":[512,512,[],"f8d9","M256 152a104 104 0 1 0 104 104 104 104 0 0 0-104-104zm0 128a24 24 0 1 1 24-24 24 24 0 0 1-24 24zm0-272C119 8 8 119 8 256s111 248 248 248 248-111 248-248S393 8 256 8zm0 376a128 128 0 1 1 128-128 128 128 0 0 1-128 128z"],recycle:[512,512,[],"f1b8","M184.561 261.903c3.232 13.997-12.123 24.635-24.068 17.168l-40.736-25.455-50.867 81.402C55.606 356.273 70.96 384 96.012 384H148c6.627 0 12 5.373 12 12v40c0 6.627-5.373 12-12 12H96.115c-75.334 0-121.302-83.048-81.408-146.88l50.822-81.388-40.725-25.448c-12.081-7.547-8.966-25.961 4.879-29.158l110.237-25.45c8.611-1.988 17.201 3.381 19.189 11.99l25.452 110.237zm98.561-182.915l41.289 66.076-40.74 25.457c-12.051 7.528-9 25.953 4.879 29.158l110.237 25.45c8.672 1.999 17.215-3.438 19.189-11.99l25.45-110.237c3.197-13.844-11.99-24.719-24.068-17.168l-40.687 25.424-41.263-66.082c-37.521-60.033-125.209-60.171-162.816 0l-17.963 28.766c-3.51 5.62-1.8 13.021 3.82 16.533l33.919 21.195c5.62 3.512 13.024 1.803 16.536-3.817l17.961-28.743c12.712-20.341 41.973-19.676 54.257-.022zM497.288 301.12l-27.515-44.065c-3.511-5.623-10.916-7.334-16.538-3.821l-33.861 21.159c-5.62 3.512-7.33 10.915-3.818 16.536l27.564 44.112c13.257 21.211-2.057 48.96-27.136 48.96H320V336.02c0-14.213-17.242-21.383-27.313-11.313l-80 79.981c-6.249 6.248-6.249 16.379 0 22.627l80 79.989C302.689 517.308 320 510.3 320 495.989V448h95.88c75.274 0 121.335-82.997 81.408-146.88z"],redo:[512,512,[],"f01e","M500.33 0h-47.41a12 12 0 0 0-12 12.57l4 82.76A247.42 247.42 0 0 0 256 8C119.34 8 7.9 119.53 8 256.19 8.1 393.07 119.1 504 256 504a247.1 247.1 0 0 0 166.18-63.91 12 12 0 0 0 .48-17.43l-34-34a12 12 0 0 0-16.38-.55A176 176 0 1 1 402.1 157.8l-101.53-4.87a12 12 0 0 0-12.57 12v47.41a12 12 0 0 0 12 12h200.33a12 12 0 0 0 12-12V12a12 12 0 0 0-12-12z"],"redo-alt":[512,512,[],"f2f9","M256.455 8c66.269.119 126.437 26.233 170.859 68.685l35.715-35.715C478.149 25.851 504 36.559 504 57.941V192c0 13.255-10.745 24-24 24H345.941c-21.382 0-32.09-25.851-16.971-40.971l41.75-41.75c-30.864-28.899-70.801-44.907-113.23-45.273-92.398-.798-170.283 73.977-169.484 169.442C88.764 348.009 162.184 424 256 424c41.127 0 79.997-14.678 110.629-41.556 4.743-4.161 11.906-3.908 16.368.553l39.662 39.662c4.872 4.872 4.631 12.815-.482 17.433C378.202 479.813 319.926 504 256 504 119.034 504 8.001 392.967 8 256.002 7.999 119.193 119.646 7.755 256.455 8z"],registered:[512,512,[],"f25d","M285.363 207.475c0 18.6-9.831 28.431-28.431 28.431h-29.876v-56.14h23.378c28.668 0 34.929 8.773 34.929 27.709zM504 256c0 136.967-111.033 248-248 248S8 392.967 8 256 119.033 8 256 8s248 111.033 248 248zM363.411 360.414c-46.729-84.825-43.299-78.636-44.702-80.98 23.432-15.172 37.945-42.979 37.945-74.486 0-54.244-31.5-89.252-105.498-89.252h-70.667c-13.255 0-24 10.745-24 24V372c0 13.255 10.745 24 24 24h22.567c13.255 0 24-10.745 24-24v-71.663h25.556l44.129 82.937a24.001 24.001 0 0 0 21.188 12.727h24.464c18.261-.001 29.829-19.591 21.018-35.587z"],"remove-format":[640,512,[],"f87d","M336 416h-11.17l9.26-27.77L267 336.4 240.49 416H208a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm297.82 42.1L377 259.59 426.17 112H544v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16H176a16 16 0 0 0-16 16v43.9L45.46 3.38A16 16 0 0 0 23 6.19L3.37 31.46a16 16 0 0 0 2.81 22.45l588.36 454.72a16 16 0 0 0 22.46-2.81l19.64-25.27a16 16 0 0 0-2.82-22.45zM309.91 207.76L224 141.36V112h117.83z"],reply:[512,512,[],"f3e5","M8.309 189.836L184.313 37.851C199.719 24.546 224 35.347 224 56.015v80.053c160.629 1.839 288 34.032 288 186.258 0 61.441-39.581 122.309-83.333 154.132-13.653 9.931-33.111-2.533-28.077-18.631 45.344-145.012-21.507-183.51-176.59-185.742V360c0 20.7-24.3 31.453-39.687 18.164l-176.004-152c-11.071-9.562-11.086-26.753 0-36.328z"],"reply-all":[576,512,[],"f122","M136.309 189.836L312.313 37.851C327.72 24.546 352 35.348 352 56.015v82.763c129.182 10.231 224 52.212 224 183.548 0 61.441-39.582 122.309-83.333 154.132-13.653 9.931-33.111-2.533-28.077-18.631 38.512-123.162-3.922-169.482-112.59-182.015v84.175c0 20.701-24.3 31.453-39.687 18.164L136.309 226.164c-11.071-9.561-11.086-26.753 0-36.328zm-128 36.328L184.313 378.15C199.7 391.439 224 380.687 224 359.986v-15.818l-108.606-93.785A55.96 55.96 0 0 1 96 207.998a55.953 55.953 0 0 1 19.393-42.38L224 71.832V56.015c0-20.667-24.28-31.469-39.687-18.164L8.309 189.836c-11.086 9.575-11.071 26.767 0 36.328z"],republican:[640,512,[],"f75e","M544 192c0-88.4-71.6-160-160-160H160C71.6 32 0 103.6 0 192v64h544v-64zm-367.7-21.6l-19.8 19.3 4.7 27.3c.8 4.9-4.3 8.6-8.7 6.3L128 210.4l-24.5 12.9c-4.3 2.3-9.5-1.4-8.7-6.3l4.7-27.3-19.8-19.3c-3.6-3.5-1.6-9.5 3.3-10.2l27.4-4 12.2-24.8c2.2-4.5 8.6-4.4 10.7 0l12.2 24.8 27.4 4c5 .7 6.9 6.7 3.4 10.2zm144 0l-19.8 19.3 4.7 27.3c.8 4.9-4.3 8.6-8.7 6.3L272 210.4l-24.5 12.9c-4.3 2.3-9.5-1.4-8.7-6.3l4.7-27.3-19.8-19.3c-3.6-3.5-1.6-9.5 3.3-10.2l27.4-4 12.2-24.8c2.2-4.5 8.6-4.4 10.7 0l12.2 24.8 27.4 4c5 .7 6.9 6.7 3.4 10.2zm144 0l-19.8 19.3 4.7 27.3c.8 4.9-4.3 8.6-8.7 6.3L416 210.4l-24.5 12.9c-4.3 2.3-9.5-1.4-8.7-6.3l4.7-27.3-19.8-19.3c-3.6-3.5-1.6-9.5 3.3-10.2l27.4-4 12.2-24.8c2.2-4.5 8.6-4.4 10.7 0l12.2 24.8 27.4 4c5 .7 6.9 6.7 3.4 10.2zM624 320h-32c-8.8 0-16 7.2-16 16v64c0 8.8-7.2 16-16 16s-16-7.2-16-16V288H0v176c0 8.8 7.2 16 16 16h96c8.8 0 16-7.2 16-16v-80h192v80c0 8.8 7.2 16 16 16h96c8.8 0 16-7.2 16-16V352h32v43.3c0 41.8 30 80.1 71.6 84.3 47.8 4.9 88.4-32.7 88.4-79.6v-64c0-8.8-7.2-16-16-16z"],restroom:[640,512,[],"f7bd","M128 128c35.3 0 64-28.7 64-64S163.3 0 128 0 64 28.7 64 64s28.7 64 64 64zm384 0c35.3 0 64-28.7 64-64S547.3 0 512 0s-64 28.7-64 64 28.7 64 64 64zm127.3 226.5l-45.6-185.8c-3.3-13.5-15.5-23-29.8-24.2-15 9.7-32.8 15.5-52 15.5-19.2 0-37-5.8-52-15.5-14.3 1.2-26.5 10.7-29.8 24.2l-45.6 185.8C381 369.6 393 384 409.2 384H464v104c0 13.3 10.7 24 24 24h48c13.3 0 24-10.7 24-24V384h54.8c16.2 0 28.2-14.4 24.5-29.5zM336 0h-32c-8.8 0-16 7.2-16 16v480c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V16c0-8.8-7.2-16-16-16zM180.1 144.4c-15 9.8-32.9 15.6-52.1 15.6-19.2 0-37.1-5.8-52.1-15.6C51.3 146.5 32 166.9 32 192v136c0 13.3 10.7 24 24 24h8v136c0 13.3 10.7 24 24 24h80c13.3 0 24-10.7 24-24V352h8c13.3 0 24-10.7 24-24V192c0-25.1-19.3-45.5-43.9-47.6z"],retweet:[640,512,[],"f079","M629.657 343.598L528.971 444.284c-9.373 9.372-24.568 9.372-33.941 0L394.343 343.598c-9.373-9.373-9.373-24.569 0-33.941l10.823-10.823c9.562-9.562 25.133-9.34 34.419.492L480 342.118V160H292.451a24.005 24.005 0 0 1-16.971-7.029l-16-16C244.361 121.851 255.069 96 276.451 96H520c13.255 0 24 10.745 24 24v222.118l40.416-42.792c9.285-9.831 24.856-10.054 34.419-.492l10.823 10.823c9.372 9.372 9.372 24.569-.001 33.941zm-265.138 15.431A23.999 23.999 0 0 0 347.548 352H160V169.881l40.416 42.792c9.286 9.831 24.856 10.054 34.419.491l10.822-10.822c9.373-9.373 9.373-24.569 0-33.941L144.971 67.716c-9.373-9.373-24.569-9.373-33.941 0L10.343 168.402c-9.373 9.373-9.373 24.569 0 33.941l10.822 10.822c9.562 9.562 25.133 9.34 34.419-.491L96 169.881V392c0 13.255 10.745 24 24 24h243.549c21.382 0 32.09-25.851 16.971-40.971l-16.001-16z"],ribbon:[448,512,[],"f4d6","M6.1 444.3c-9.6 10.8-7.5 27.6 4.5 35.7l68.8 27.9c9.9 6.7 23.3 5 31.3-3.8l91.8-101.9-79.2-87.9-117.2 130zm435.8 0s-292-324.6-295.4-330.1c15.4-8.4 40.2-17.9 77.5-17.9s62.1 9.5 77.5 17.9c-3.3 5.6-56 64.6-56 64.6l79.1 87.7 34.2-38c28.7-31.9 33.3-78.6 11.4-115.5l-43.7-73.5c-4.3-7.2-9.9-13.3-16.8-18-40.7-27.6-127.4-29.7-171.4 0-6.9 4.7-12.5 10.8-16.8 18l-43.6 73.2c-1.5 2.5-37.1 62.2 11.5 116L337.5 504c8 8.9 21.4 10.5 31.3 3.8l68.8-27.9c11.9-8 14-24.8 4.3-35.6z"],ring:[512,512,[],"f70b","M256 64C110.06 64 0 125.91 0 208v98.13C0 384.48 114.62 448 256 448s256-63.52 256-141.87V208c0-82.09-110.06-144-256-144zm0 64c106.04 0 192 35.82 192 80 0 9.26-3.97 18.12-10.91 26.39C392.15 208.21 328.23 192 256 192s-136.15 16.21-181.09 42.39C67.97 226.12 64 217.26 64 208c0-44.18 85.96-80 192-80zM120.43 264.64C155.04 249.93 201.64 240 256 240s100.96 9.93 135.57 24.64C356.84 279.07 308.93 288 256 288s-100.84-8.93-135.57-23.36z"],road:[576,512,[],"f018","M573.19 402.67l-139.79-320C428.43 71.29 417.6 64 405.68 64h-97.59l2.45 23.16c.5 4.72-3.21 8.84-7.96 8.84h-29.16c-4.75 0-8.46-4.12-7.96-8.84L267.91 64h-97.59c-11.93 0-22.76 7.29-27.73 18.67L2.8 402.67C-6.45 423.86 8.31 448 30.54 448h196.84l10.31-97.68c.86-8.14 7.72-14.32 15.91-14.32h68.8c8.19 0 15.05 6.18 15.91 14.32L348.62 448h196.84c22.23 0 36.99-24.14 27.73-45.33zM260.4 135.16a8 8 0 0 1 7.96-7.16h39.29c4.09 0 7.53 3.09 7.96 7.16l4.6 43.58c.75 7.09-4.81 13.26-11.93 13.26h-40.54c-7.13 0-12.68-6.17-11.93-13.26l4.59-43.58zM315.64 304h-55.29c-9.5 0-16.91-8.23-15.91-17.68l5.07-48c.86-8.14 7.72-14.32 15.91-14.32h45.15c8.19 0 15.05 6.18 15.91 14.32l5.07 48c1 9.45-6.41 17.68-15.91 17.68z"],robot:[640,512,[],"f544","M32,224H64V416H32A31.96166,31.96166,0,0,1,0,384V256A31.96166,31.96166,0,0,1,32,224Zm512-48V448a64.06328,64.06328,0,0,1-64,64H160a64.06328,64.06328,0,0,1-64-64V176a79.974,79.974,0,0,1,80-80H288V32a32,32,0,0,1,64,0V96H464A79.974,79.974,0,0,1,544,176ZM264,256a40,40,0,1,0-40,40A39.997,39.997,0,0,0,264,256Zm-8,128H192v32h64Zm96,0H288v32h64ZM456,256a40,40,0,1,0-40,40A39.997,39.997,0,0,0,456,256Zm-8,128H384v32h64ZM640,256V384a31.96166,31.96166,0,0,1-32,32H576V224h32A31.96166,31.96166,0,0,1,640,256Z"],rocket:[512,512,[],"f135","M505.12019,19.09375c-1.18945-5.53125-6.65819-11-12.207-12.1875C460.716,0,435.507,0,410.40747,0,307.17523,0,245.26909,55.20312,199.05238,128H94.83772c-16.34763.01562-35.55658,11.875-42.88664,26.48438L2.51562,253.29688A28.4,28.4,0,0,0,0,264a24.00867,24.00867,0,0,0,24.00582,24H127.81618l-22.47457,22.46875c-11.36521,11.36133-12.99607,32.25781,0,45.25L156.24582,406.625c11.15623,11.1875,32.15619,13.15625,45.27726,0l22.47457-22.46875V488a24.00867,24.00867,0,0,0,24.00581,24,28.55934,28.55934,0,0,0,10.707-2.51562l98.72834-49.39063c14.62888-7.29687,26.50776-26.5,26.50776-42.85937V312.79688c72.59753-46.3125,128.03493-108.40626,128.03493-211.09376C512.07526,76.5,512.07526,51.29688,505.12019,19.09375ZM384.04033,168A40,40,0,1,1,424.05,128,40.02322,40.02322,0,0,1,384.04033,168Z"],route:[512,512,[],"f4d7","M416 320h-96c-17.6 0-32-14.4-32-32s14.4-32 32-32h96s96-107 96-160-43-96-96-96-96 43-96 96c0 25.5 22.2 63.4 45.3 96H320c-52.9 0-96 43.1-96 96s43.1 96 96 96h96c17.6 0 32 14.4 32 32s-14.4 32-32 32H185.5c-16 24.8-33.8 47.7-47.3 64H416c52.9 0 96-43.1 96-96s-43.1-96-96-96zm0-256c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zM96 256c-53 0-96 43-96 96s96 160 96 160 96-107 96-160-43-96-96-96zm0 128c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],rss:[448,512,[],"f09e","M128.081 415.959c0 35.369-28.672 64.041-64.041 64.041S0 451.328 0 415.959s28.672-64.041 64.041-64.041 64.04 28.673 64.04 64.041zm175.66 47.25c-8.354-154.6-132.185-278.587-286.95-286.95C7.656 175.765 0 183.105 0 192.253v48.069c0 8.415 6.49 15.472 14.887 16.018 111.832 7.284 201.473 96.702 208.772 208.772.547 8.397 7.604 14.887 16.018 14.887h48.069c9.149.001 16.489-7.655 15.995-16.79zm144.249.288C439.596 229.677 251.465 40.445 16.503 32.01 7.473 31.686 0 38.981 0 48.016v48.068c0 8.625 6.835 15.645 15.453 15.999 191.179 7.839 344.627 161.316 352.465 352.465.353 8.618 7.373 15.453 15.999 15.453h48.068c9.034-.001 16.329-7.474 16.005-16.504z"],"rss-square":[448,512,[],"f143","M400 32H48C21.49 32 0 53.49 0 80v352c0 26.51 21.49 48 48 48h352c26.51 0 48-21.49 48-48V80c0-26.51-21.49-48-48-48zM112 416c-26.51 0-48-21.49-48-48s21.49-48 48-48 48 21.49 48 48-21.49 48-48 48zm157.533 0h-34.335c-6.011 0-11.051-4.636-11.442-10.634-5.214-80.05-69.243-143.92-149.123-149.123-5.997-.39-10.633-5.431-10.633-11.441v-34.335c0-6.535 5.468-11.777 11.994-11.425 110.546 5.974 198.997 94.536 204.964 204.964.352 6.526-4.89 11.994-11.425 11.994zm103.027 0h-34.334c-6.161 0-11.175-4.882-11.427-11.038-5.598-136.535-115.204-246.161-251.76-251.76C68.882 152.949 64 147.935 64 141.774V107.44c0-6.454 5.338-11.664 11.787-11.432 167.83 6.025 302.21 141.191 308.205 308.205.232 6.449-4.978 11.787-11.432 11.787z"],"ruble-sign":[384,512,[],"f158","M239.36 320C324.48 320 384 260.542 384 175.071S324.48 32 239.36 32H76c-6.627 0-12 5.373-12 12v206.632H12c-6.627 0-12 5.373-12 12V308c0 6.627 5.373 12 12 12h52v32H12c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h52v52c0 6.627 5.373 12 12 12h58.56c6.627 0 12-5.373 12-12v-52H308c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12H146.56v-32h92.8zm-92.8-219.252h78.72c46.72 0 74.88 29.11 74.88 74.323 0 45.832-28.16 75.561-76.16 75.561h-77.44V100.748z"],ruler:[640,512,[],"f545","M635.7 167.2L556.1 31.7c-8.8-15-28.3-20.1-43.5-11.5l-69 39.1L503.3 161c2.2 3.8.9 8.5-2.9 10.7l-13.8 7.8c-3.8 2.2-8.7.9-10.9-2.9L416 75l-55.2 31.3 27.9 47.4c2.2 3.8.9 8.5-2.9 10.7l-13.8 7.8c-3.8 2.2-8.7.9-10.9-2.9L333.2 122 278 153.3 337.8 255c2.2 3.7.9 8.5-2.9 10.7l-13.8 7.8c-3.8 2.2-8.7.9-10.9-2.9l-59.7-101.7-55.2 31.3 27.9 47.4c2.2 3.8.9 8.5-2.9 10.7l-13.8 7.8c-3.8 2.2-8.7.9-10.9-2.9l-27.9-47.5-55.2 31.3 59.7 101.7c2.2 3.7.9 8.5-2.9 10.7l-13.8 7.8c-3.8 2.2-8.7.9-10.9-2.9L84.9 262.9l-69 39.1C.7 310.7-4.6 329.8 4.2 344.8l79.6 135.6c8.8 15 28.3 20.1 43.5 11.5L624.1 210c15.2-8.6 20.4-27.8 11.6-42.8z"],"ruler-combined":[512,512,[],"f546","M160 288h-56c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h56v-64h-56c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h56V96h-56c-4.42 0-8-3.58-8-8V72c0-4.42 3.58-8 8-8h56V32c0-17.67-14.33-32-32-32H32C14.33 0 0 14.33 0 32v448c0 2.77.91 5.24 1.57 7.8L160 329.38V288zm320 64h-32v56c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8v-56h-64v56c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8v-56h-64v56c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8v-56h-41.37L24.2 510.43c2.56.66 5.04 1.57 7.8 1.57h448c17.67 0 32-14.33 32-32v-96c0-17.67-14.33-32-32-32z"],"ruler-horizontal":[576,512,[],"f547","M544 128h-48v88c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8v-88h-64v88c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8v-88h-64v88c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8v-88h-64v88c0 4.42-3.58 8-8 8h-16c-4.42 0-8-3.58-8-8v-88h-64v88c0 4.42-3.58 8-8 8H88c-4.42 0-8-3.58-8-8v-88H32c-17.67 0-32 14.33-32 32v192c0 17.67 14.33 32 32 32h512c17.67 0 32-14.33 32-32V160c0-17.67-14.33-32-32-32z"],"ruler-vertical":[256,512,[],"f548","M168 416c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h88v-64h-88c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h88v-64h-88c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h88v-64h-88c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h88V32c0-17.67-14.33-32-32-32H32C14.33 0 0 14.33 0 32v448c0 17.67 14.33 32 32 32h192c17.67 0 32-14.33 32-32v-64h-88z"],running:[416,512,[],"f70c","M272 96c26.51 0 48-21.49 48-48S298.51 0 272 0s-48 21.49-48 48 21.49 48 48 48zM113.69 317.47l-14.8 34.52H32c-17.67 0-32 14.33-32 32s14.33 32 32 32h77.45c19.25 0 36.58-11.44 44.11-29.09l8.79-20.52-10.67-6.3c-17.32-10.23-30.06-25.37-37.99-42.61zM384 223.99h-44.03l-26.06-53.25c-12.5-25.55-35.45-44.23-61.78-50.94l-71.08-21.14c-28.3-6.8-57.77-.55-80.84 17.14l-39.67 30.41c-14.03 10.75-16.69 30.83-5.92 44.86s30.84 16.66 44.86 5.92l39.69-30.41c7.67-5.89 17.44-8 25.27-6.14l14.7 4.37-37.46 87.39c-12.62 29.48-1.31 64.01 26.3 80.31l84.98 50.17-27.47 87.73c-5.28 16.86 4.11 34.81 20.97 40.09 3.19 1 6.41 1.48 9.58 1.48 13.61 0 26.23-8.77 30.52-22.45l31.64-101.06c5.91-20.77-2.89-43.08-21.64-54.39l-61.24-36.14 31.31-78.28 20.27 41.43c8 16.34 24.92 26.89 43.11 26.89H384c17.67 0 32-14.33 32-32s-14.33-31.99-32-31.99z"],"rupee-sign":[320,512,[],"f156","M308 96c6.627 0 12-5.373 12-12V44c0-6.627-5.373-12-12-12H12C5.373 32 0 37.373 0 44v44.748c0 6.627 5.373 12 12 12h85.28c27.308 0 48.261 9.958 60.97 27.252H12c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h158.757c-6.217 36.086-32.961 58.632-74.757 58.632H12c-6.627 0-12 5.373-12 12v53.012c0 3.349 1.4 6.546 3.861 8.818l165.052 152.356a12.001 12.001 0 0 0 8.139 3.182h82.562c10.924 0 16.166-13.408 8.139-20.818L116.871 319.906c76.499-2.34 131.144-53.395 138.318-127.906H308c6.627 0 12-5.373 12-12v-40c0-6.627-5.373-12-12-12h-58.69c-3.486-11.541-8.28-22.246-14.252-32H308z"],"sad-cry":[496,512,[],"f5b3","M248 8C111 8 0 119 0 256c0 90.1 48.2 168.7 120 212.1V288c0-8.8 7.2-16 16-16s16 7.2 16 16v196.7c29.5 12.4 62 19.3 96 19.3s66.5-6.9 96-19.3V288c0-8.8 7.2-16 16-16s16 7.2 16 16v180.1C447.8 424.7 496 346 496 256 496 119 385 8 248 8zm-65.5 216.5c-14.8-13.2-46.2-13.2-61 0L112 233c-3.8 3.3-9.3 4-13.7 1.6-4.4-2.4-6.9-7.4-6.1-12.4 4-25.2 34.2-42.1 59.9-42.1S208 197 212 222.2c.8 5-1.7 10-6.1 12.4-5.8 3.1-11.2.7-13.7-1.6l-9.7-8.5zM248 416c-26.5 0-48-28.7-48-64s21.5-64 48-64 48 28.7 48 64-21.5 64-48 64zm149.8-181.5c-5.8 3.1-11.2.7-13.7-1.6l-9.5-8.5c-14.8-13.2-46.2-13.2-61 0L304 233c-3.8 3.3-9.3 4-13.7 1.6-4.4-2.4-6.9-7.4-6.1-12.4 4-25.2 34.2-42.1 59.9-42.1S400 197 404 222.2c.6 4.9-1.8 9.9-6.2 12.3z"],"sad-tear":[496,512,[],"f5b4","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm80 168c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zM152 416c-26.5 0-48-21-48-47 0-20 28.5-60.4 41.6-77.8 3.2-4.3 9.6-4.3 12.8 0C171.5 308.6 200 349 200 369c0 26-21.5 47-48 47zm16-176c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm170.2 154.2C315.8 367.4 282.9 352 248 352c-21.2 0-21.2-32 0-32 44.4 0 86.3 19.6 114.7 53.8 13.8 16.4-11.2 36.5-24.5 20.4z"],satellite:[512,512,[],"f7bf","M502.60969,310.04206l-96.70393,96.71625a31.88151,31.88151,0,0,1-45.00765,0L280.572,326.34115l-9.89231,9.90759a190.56343,190.56343,0,0,1-5.40716,168.52287c-4.50077,8.50115-16.39342,9.59505-23.20707,2.79725L134.54715,400.05428l-17.7999,17.79929c.70324,2.60972,1.60965,5.00067,1.60965,7.79793a32.00544,32.00544,0,1,1-32.00544-32.00434c2.79735,0,5.18838.90637,7.7982,1.60959l17.7999-17.79929L4.43129,269.94287c-6.798-6.81342-5.70409-18.6119,2.79735-23.20627a190.58161,190.58161,0,0,1,168.52864-5.407l9.79854-9.79821-80.31053-80.41716a32.002,32.002,0,0,1,0-45.09987L201.96474,9.29814A31.62639,31.62639,0,0,1,224.46868,0a31.99951,31.99951,0,0,1,22.59759,9.29814l80.32615,80.30777,47.805-47.89713a33.6075,33.6075,0,0,1,47.50808,0l47.50807,47.50645a33.63308,33.63308,0,0,1,0,47.50644l-47.805,47.89713L502.71908,265.036A31.78938,31.78938,0,0,1,502.60969,310.04206ZM219.56159,197.433l73.82505-73.82252-68.918-68.9-73.80942,73.80689Zm237.74352,90.106-68.90233-68.9156-73.825,73.82252,68.918,68.9Z"],"satellite-dish":[512,512,[],"f7c0","M305.44954,462.59c7.39157,7.29792,6.18829,20.09661-3.00038,25.00356-77.713,41.80281-176.72559,29.9105-242.34331-35.7082C-5.49624,386.28227-17.404,287.362,24.41381,209.554c4.89125-9.095,17.68975-10.29834,25.00318-3.00043L166.22872,323.36708l27.39411-27.39452c-.68759-2.60974-1.594-5.00071-1.594-7.81361a32.00407,32.00407,0,1,1,32.00407,32.00455c-2.79723,0-5.20378-.89075-7.79786-1.594l-27.40974,27.41015ZM511.9758,303.06732a16.10336,16.10336,0,0,1-16.002,17.00242H463.86031a15.96956,15.96956,0,0,1-15.89265-15.00213C440.46671,175.5492,336.45348,70.53427,207.03078,63.53328a15.84486,15.84486,0,0,1-15.00191-15.90852V16.02652A16.09389,16.09389,0,0,1,209.031.02425C372.25491,8.61922,503.47472,139.841,511.9758,303.06732Zm-96.01221-.29692a16.21093,16.21093,0,0,1-16.11142,17.29934H367.645a16.06862,16.06862,0,0,1-15.89265-14.70522c-6.90712-77.01094-68.118-138.91037-144.92467-145.22376a15.94,15.94,0,0,1-14.79876-15.89289V112.13393a16.134,16.134,0,0,1,17.29908-16.096C319.45132,104.5391,407.55627,192.64538,415.96359,302.7704Z"],save:[448,512,[],"f0c7","M433.941 129.941l-83.882-83.882A48 48 0 0 0 316.118 32H48C21.49 32 0 53.49 0 80v352c0 26.51 21.49 48 48 48h352c26.51 0 48-21.49 48-48V163.882a48 48 0 0 0-14.059-33.941zM224 416c-35.346 0-64-28.654-64-64 0-35.346 28.654-64 64-64s64 28.654 64 64c0 35.346-28.654 64-64 64zm96-304.52V212c0 6.627-5.373 12-12 12H76c-6.627 0-12-5.373-12-12V108c0-6.627 5.373-12 12-12h228.52c3.183 0 6.235 1.264 8.485 3.515l3.48 3.48A11.996 11.996 0 0 1 320 111.48z"],school:[640,512,[],"f549","M0 224v272c0 8.84 7.16 16 16 16h80V192H32c-17.67 0-32 14.33-32 32zm360-48h-24v-40c0-4.42-3.58-8-8-8h-16c-4.42 0-8 3.58-8 8v64c0 4.42 3.58 8 8 8h48c4.42 0 8-3.58 8-8v-16c0-4.42-3.58-8-8-8zm137.75-63.96l-160-106.67a32.02 32.02 0 0 0-35.5 0l-160 106.67A32.002 32.002 0 0 0 128 138.66V512h128V368c0-8.84 7.16-16 16-16h96c8.84 0 16 7.16 16 16v144h128V138.67c0-10.7-5.35-20.7-14.25-26.63zM320 256c-44.18 0-80-35.82-80-80s35.82-80 80-80 80 35.82 80 80-35.82 80-80 80zm288-64h-64v320h80c8.84 0 16-7.16 16-16V224c0-17.67-14.33-32-32-32z"],screwdriver:[512,512,[],"f54a","M448 0L320 96v62.06l-83.03 83.03c6.79 4.25 13.27 9.06 19.07 14.87 5.8 5.8 10.62 12.28 14.87 19.07L353.94 192H416l96-128-64-64zM128 278.59L10.92 395.67c-14.55 14.55-14.55 38.15 0 52.71l52.7 52.7c14.56 14.56 38.15 14.56 52.71 0L233.41 384c29.11-29.11 29.11-76.3 0-105.41s-76.3-29.11-105.41 0z"],scroll:[640,512,[],"f70e","M48 0C21.53 0 0 21.53 0 48v64c0 8.84 7.16 16 16 16h80V48C96 21.53 74.47 0 48 0zm208 412.57V352h288V96c0-52.94-43.06-96-96-96H111.59C121.74 13.41 128 29.92 128 48v368c0 38.87 34.65 69.65 74.75 63.12C234.22 474 256 444.46 256 412.57zM288 384v32c0 52.93-43.06 96-96 96h336c61.86 0 112-50.14 112-112 0-8.84-7.16-16-16-16H288z"],"sd-card":[384,512,[],"f7c2","M320 0H128L0 128v320c0 35.3 28.7 64 64 64h256c35.3 0 64-28.7 64-64V64c0-35.3-28.7-64-64-64zM160 160h-48V64h48v96zm80 0h-48V64h48v96zm80 0h-48V64h48v96z"],search:[512,512,[],"f002","M505 442.7L405.3 343c-4.5-4.5-10.6-7-17-7H372c27.6-35.3 44-79.7 44-128C416 93.1 322.9 0 208 0S0 93.1 0 208s93.1 208 208 208c48.3 0 92.7-16.4 128-44v16.3c0 6.4 2.5 12.5 7 17l99.7 99.7c9.4 9.4 24.6 9.4 33.9 0l28.3-28.3c9.4-9.4 9.4-24.6.1-34zM208 336c-70.7 0-128-57.2-128-128 0-70.7 57.2-128 128-128 70.7 0 128 57.2 128 128 0 70.7-57.2 128-128 128z"],"search-dollar":[512,512,[],"f688","M505.04 442.66l-99.71-99.69c-4.5-4.5-10.6-7-17-7h-16.3c27.6-35.3 44-79.69 44-127.99C416.03 93.09 322.92 0 208.02 0S0 93.09 0 207.98s93.11 207.98 208.02 207.98c48.3 0 92.71-16.4 128.01-44v16.3c0 6.4 2.5 12.5 7 17l99.71 99.69c9.4 9.4 24.6 9.4 33.9 0l28.3-28.3c9.4-9.4 9.4-24.59.1-33.99zm-297.02-90.7c-79.54 0-144-64.34-144-143.98 0-79.53 64.35-143.98 144-143.98 79.54 0 144 64.34 144 143.98 0 79.53-64.35 143.98-144 143.98zm27.11-152.54l-45.01-13.5c-5.16-1.55-8.77-6.78-8.77-12.73 0-7.27 5.3-13.19 11.8-13.19h28.11c4.56 0 8.96 1.29 12.82 3.72 3.24 2.03 7.36 1.91 10.13-.73l11.75-11.21c3.53-3.37 3.33-9.21-.57-12.14-9.1-6.83-20.08-10.77-31.37-11.35V112c0-4.42-3.58-8-8-8h-16c-4.42 0-8 3.58-8 8v16.12c-23.63.63-42.68 20.55-42.68 45.07 0 19.97 12.99 37.81 31.58 43.39l45.01 13.5c5.16 1.55 8.77 6.78 8.77 12.73 0 7.27-5.3 13.19-11.8 13.19h-28.1c-4.56 0-8.96-1.29-12.82-3.72-3.24-2.03-7.36-1.91-10.13.73l-11.75 11.21c-3.53 3.37-3.33 9.21.57 12.14 9.1 6.83 20.08 10.77 31.37 11.35V304c0 4.42 3.58 8 8 8h16c4.42 0 8-3.58 8-8v-16.12c23.63-.63 42.68-20.54 42.68-45.07 0-19.97-12.99-37.81-31.59-43.39z"],"search-location":[512,512,[],"f689","M505.04 442.66l-99.71-99.69c-4.5-4.5-10.6-7-17-7h-16.3c27.6-35.3 44-79.69 44-127.99C416.03 93.09 322.92 0 208.02 0S0 93.09 0 207.98s93.11 207.98 208.02 207.98c48.3 0 92.71-16.4 128.01-44v16.3c0 6.4 2.5 12.5 7 17l99.71 99.69c9.4 9.4 24.6 9.4 33.9 0l28.3-28.3c9.4-9.4 9.4-24.59.1-33.99zm-297.02-90.7c-79.54 0-144-64.34-144-143.98 0-79.53 64.35-143.98 144-143.98 79.54 0 144 64.34 144 143.98 0 79.53-64.35 143.98-144 143.98zm.02-239.96c-40.78 0-73.84 33.05-73.84 73.83 0 32.96 48.26 93.05 66.75 114.86a9.24 9.24 0 0 0 14.18 0c18.49-21.81 66.75-81.89 66.75-114.86 0-40.78-33.06-73.83-73.84-73.83zm0 96c-13.26 0-24-10.75-24-24 0-13.26 10.75-24 24-24s24 10.74 24 24c0 13.25-10.75 24-24 24z"],"search-minus":[512,512,[],"f010","M304 192v32c0 6.6-5.4 12-12 12H124c-6.6 0-12-5.4-12-12v-32c0-6.6 5.4-12 12-12h168c6.6 0 12 5.4 12 12zm201 284.7L476.7 505c-9.4 9.4-24.6 9.4-33.9 0L343 405.3c-4.5-4.5-7-10.6-7-17V372c-35.3 27.6-79.7 44-128 44C93.1 416 0 322.9 0 208S93.1 0 208 0s208 93.1 208 208c0 48.3-16.4 92.7-44 128h16.3c6.4 0 12.5 2.5 17 7l99.7 99.7c9.3 9.4 9.3 24.6 0 34zM344 208c0-75.2-60.8-136-136-136S72 132.8 72 208s60.8 136 136 136 136-60.8 136-136z"],"search-plus":[512,512,[],"f00e","M304 192v32c0 6.6-5.4 12-12 12h-56v56c0 6.6-5.4 12-12 12h-32c-6.6 0-12-5.4-12-12v-56h-56c-6.6 0-12-5.4-12-12v-32c0-6.6 5.4-12 12-12h56v-56c0-6.6 5.4-12 12-12h32c6.6 0 12 5.4 12 12v56h56c6.6 0 12 5.4 12 12zm201 284.7L476.7 505c-9.4 9.4-24.6 9.4-33.9 0L343 405.3c-4.5-4.5-7-10.6-7-17V372c-35.3 27.6-79.7 44-128 44C93.1 416 0 322.9 0 208S93.1 0 208 0s208 93.1 208 208c0 48.3-16.4 92.7-44 128h16.3c6.4 0 12.5 2.5 17 7l99.7 99.7c9.3 9.4 9.3 24.6 0 34zM344 208c0-75.2-60.8-136-136-136S72 132.8 72 208s60.8 136 136 136 136-60.8 136-136z"],seedling:[512,512,[],"f4d8","M64 96H0c0 123.7 100.3 224 224 224v144c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V320C288 196.3 187.7 96 64 96zm384-64c-84.2 0-157.4 46.5-195.7 115.2 27.7 30.2 48.2 66.9 59 107.6C424 243.1 512 147.9 512 32h-64z"],server:[512,512,[],"f233","M480 160H32c-17.673 0-32-14.327-32-32V64c0-17.673 14.327-32 32-32h448c17.673 0 32 14.327 32 32v64c0 17.673-14.327 32-32 32zm-48-88c-13.255 0-24 10.745-24 24s10.745 24 24 24 24-10.745 24-24-10.745-24-24-24zm-64 0c-13.255 0-24 10.745-24 24s10.745 24 24 24 24-10.745 24-24-10.745-24-24-24zm112 248H32c-17.673 0-32-14.327-32-32v-64c0-17.673 14.327-32 32-32h448c17.673 0 32 14.327 32 32v64c0 17.673-14.327 32-32 32zm-48-88c-13.255 0-24 10.745-24 24s10.745 24 24 24 24-10.745 24-24-10.745-24-24-24zm-64 0c-13.255 0-24 10.745-24 24s10.745 24 24 24 24-10.745 24-24-10.745-24-24-24zm112 248H32c-17.673 0-32-14.327-32-32v-64c0-17.673 14.327-32 32-32h448c17.673 0 32 14.327 32 32v64c0 17.673-14.327 32-32 32zm-48-88c-13.255 0-24 10.745-24 24s10.745 24 24 24 24-10.745 24-24-10.745-24-24-24zm-64 0c-13.255 0-24 10.745-24 24s10.745 24 24 24 24-10.745 24-24-10.745-24-24-24z"],shapes:[512,512,[],"f61f","M128,256A128,128,0,1,0,256,384,128,128,0,0,0,128,256Zm379-54.86L400.07,18.29a37.26,37.26,0,0,0-64.14,0L229,201.14C214.76,225.52,232.58,256,261.09,256H474.91C503.42,256,521.24,225.52,507,201.14ZM480,288H320a32,32,0,0,0-32,32V480a32,32,0,0,0,32,32H480a32,32,0,0,0,32-32V320A32,32,0,0,0,480,288Z"],share:[512,512,[],"f064","M503.691 189.836L327.687 37.851C312.281 24.546 288 35.347 288 56.015v80.053C127.371 137.907 0 170.1 0 322.326c0 61.441 39.581 122.309 83.333 154.132 13.653 9.931 33.111-2.533 28.077-18.631C66.066 312.814 132.917 274.316 288 272.085V360c0 20.7 24.3 31.453 39.687 18.164l176.004-152c11.071-9.562 11.086-26.753 0-36.328z"],"share-alt":[448,512,[],"f1e0","M352 320c-22.608 0-43.387 7.819-59.79 20.895l-102.486-64.054a96.551 96.551 0 0 0 0-41.683l102.486-64.054C308.613 184.181 329.392 192 352 192c53.019 0 96-42.981 96-96S405.019 0 352 0s-96 42.981-96 96c0 7.158.79 14.13 2.276 20.841L155.79 180.895C139.387 167.819 118.608 160 96 160c-53.019 0-96 42.981-96 96s42.981 96 96 96c22.608 0 43.387-7.819 59.79-20.895l102.486 64.054A96.301 96.301 0 0 0 256 416c0 53.019 42.981 96 96 96s96-42.981 96-96-42.981-96-96-96z"],"share-alt-square":[448,512,[],"f1e1","M448 80v352c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48V80c0-26.51 21.49-48 48-48h352c26.51 0 48 21.49 48 48zM304 296c-14.562 0-27.823 5.561-37.783 14.671l-67.958-40.775a56.339 56.339 0 0 0 0-27.793l67.958-40.775C276.177 210.439 289.438 216 304 216c30.928 0 56-25.072 56-56s-25.072-56-56-56-56 25.072-56 56c0 4.797.605 9.453 1.74 13.897l-67.958 40.775C171.823 205.561 158.562 200 144 200c-30.928 0-56 25.072-56 56s25.072 56 56 56c14.562 0 27.823-5.561 37.783-14.671l67.958 40.775a56.088 56.088 0 0 0-1.74 13.897c0 30.928 25.072 56 56 56s56-25.072 56-56C360 321.072 334.928 296 304 296z"],"share-square":[576,512,[],"f14d","M568.482 177.448L424.479 313.433C409.3 327.768 384 317.14 384 295.985v-71.963c-144.575.97-205.566 35.113-164.775 171.353 4.483 14.973-12.846 26.567-25.006 17.33C155.252 383.105 120 326.488 120 269.339c0-143.937 117.599-172.5 264-173.312V24.012c0-21.174 25.317-31.768 40.479-17.448l144.003 135.988c10.02 9.463 10.028 25.425 0 34.896zM384 379.128V448H64V128h50.916a11.99 11.99 0 0 0 8.648-3.693c14.953-15.568 32.237-27.89 51.014-37.676C185.708 80.83 181.584 64 169.033 64H48C21.49 64 0 85.49 0 112v352c0 26.51 21.49 48 48 48h352c26.51 0 48-21.49 48-48v-88.806c0-8.288-8.197-14.066-16.011-11.302a71.83 71.83 0 0 1-34.189 3.377c-7.27-1.046-13.8 4.514-13.8 11.859z"],"shekel-sign":[448,512,[],"f20b","M248 168v168c0 8.84 7.16 16 16 16h48c8.84 0 16-7.16 16-16V168c0-75.11-60.89-136-136-136H24C10.75 32 0 42.74 0 56v408c0 8.84 7.16 16 16 16h48c8.84 0 16-7.16 16-16V112h112c30.93 0 56 25.07 56 56zM432 32h-48c-8.84 0-16 7.16-16 16v296c0 30.93-25.07 56-56 56H200V176c0-8.84-7.16-16-16-16h-48c-8.84 0-16 7.16-16 16v280c0 13.25 10.75 24 24 24h168c75.11 0 136-60.89 136-136V48c0-8.84-7.16-16-16-16z"],"shield-alt":[512,512,[],"f3ed","M466.5 83.7l-192-80a48.15 48.15 0 0 0-36.9 0l-192 80C27.7 91.1 16 108.6 16 128c0 198.5 114.5 335.7 221.5 380.3 11.8 4.9 25.1 4.9 36.9 0C360.1 472.6 496 349.3 496 128c0-19.4-11.7-36.9-29.5-44.3zM256.1 446.3l-.1-381 175.9 73.3c-3.3 151.4-82.1 261.1-175.8 307.7z"],"shield-virus":[512,512,[],"f96c","M224,192a16,16,0,1,0,16,16A16,16,0,0,0,224,192ZM466.5,83.68l-192-80A57.4,57.4,0,0,0,256.05,0a57.4,57.4,0,0,0-18.46,3.67l-192,80A47.93,47.93,0,0,0,16,128C16,326.5,130.5,463.72,237.5,508.32a48.09,48.09,0,0,0,36.91,0C360.09,472.61,496,349.3,496,128A48,48,0,0,0,466.5,83.68ZM384,256H371.88c-28.51,0-42.79,34.47-22.63,54.63l8.58,8.57a16,16,0,1,1-22.63,22.63l-8.57-8.58C306.47,313.09,272,327.37,272,355.88V368a16,16,0,0,1-32,0V355.88c0-28.51-34.47-42.79-54.63-22.63l-8.57,8.58a16,16,0,0,1-22.63-22.63l8.58-8.57c20.16-20.16,5.88-54.63-22.63-54.63H128a16,16,0,0,1,0-32h12.12c28.51,0,42.79-34.47,22.63-54.63l-8.58-8.57a16,16,0,0,1,22.63-22.63l8.57,8.58c20.16,20.16,54.63,5.88,54.63-22.63V112a16,16,0,0,1,32,0v12.12c0,28.51,34.47,42.79,54.63,22.63l8.57-8.58a16,16,0,0,1,22.63,22.63l-8.58,8.57C329.09,189.53,343.37,224,371.88,224H384a16,16,0,0,1,0,32Zm-96,0a16,16,0,1,0,16,16A16,16,0,0,0,288,256Z"],ship:[640,512,[],"f21a","M496.616 372.639l70.012-70.012c16.899-16.9 9.942-45.771-12.836-53.092L512 236.102V96c0-17.673-14.327-32-32-32h-64V24c0-13.255-10.745-24-24-24H248c-13.255 0-24 10.745-24 24v40h-64c-17.673 0-32 14.327-32 32v140.102l-41.792 13.433c-22.753 7.313-29.754 36.173-12.836 53.092l70.012 70.012C125.828 416.287 85.587 448 24 448c-13.255 0-24 10.745-24 24v16c0 13.255 10.745 24 24 24 61.023 0 107.499-20.61 143.258-59.396C181.677 487.432 216.021 512 256 512h128c39.979 0 74.323-24.568 88.742-59.396C508.495 491.384 554.968 512 616 512c13.255 0 24-10.745 24-24v-16c0-13.255-10.745-24-24-24-60.817 0-101.542-31.001-119.384-75.361zM192 128h256v87.531l-118.208-37.995a31.995 31.995 0 0 0-19.584 0L192 215.531V128z"],"shipping-fast":[640,512,[],"f48b","M624 352h-16V243.9c0-12.7-5.1-24.9-14.1-33.9L494 110.1c-9-9-21.2-14.1-33.9-14.1H416V48c0-26.5-21.5-48-48-48H112C85.5 0 64 21.5 64 48v48H8c-4.4 0-8 3.6-8 8v16c0 4.4 3.6 8 8 8h272c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H40c-4.4 0-8 3.6-8 8v16c0 4.4 3.6 8 8 8h208c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H8c-4.4 0-8 3.6-8 8v16c0 4.4 3.6 8 8 8h208c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H64v128c0 53 43 96 96 96s96-43 96-96h128c0 53 43 96 96 96s96-43 96-96h48c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zM160 464c-26.5 0-48-21.5-48-48s21.5-48 48-48 48 21.5 48 48-21.5 48-48 48zm320 0c-26.5 0-48-21.5-48-48s21.5-48 48-48 48 21.5 48 48-21.5 48-48 48zm80-208H416V144h44.1l99.9 99.9V256z"],"shoe-prints":[640,512,[],"f54b","M192 160h32V32h-32c-35.35 0-64 28.65-64 64s28.65 64 64 64zM0 416c0 35.35 28.65 64 64 64h32V352H64c-35.35 0-64 28.65-64 64zm337.46-128c-34.91 0-76.16 13.12-104.73 32-24.79 16.38-44.52 32-104.73 32v128l57.53 15.97c26.21 7.28 53.01 13.12 80.31 15.05 32.69 2.31 65.6.67 97.58-6.2C472.9 481.3 512 429.22 512 384c0-64-84.18-96-174.54-96zM491.42 7.19C459.44.32 426.53-1.33 393.84.99c-27.3 1.93-54.1 7.77-80.31 15.04L256 32v128c60.2 0 79.94 15.62 104.73 32 28.57 18.88 69.82 32 104.73 32C555.82 224 640 192 640 128c0-45.22-39.1-97.3-148.58-120.81z"],"shopping-bag":[448,512,[],"f290","M352 160v-32C352 57.42 294.579 0 224 0 153.42 0 96 57.42 96 128v32H0v272c0 44.183 35.817 80 80 80h288c44.183 0 80-35.817 80-80V160h-96zm-192-32c0-35.29 28.71-64 64-64s64 28.71 64 64v32H160v-32zm160 120c-13.255 0-24-10.745-24-24s10.745-24 24-24 24 10.745 24 24-10.745 24-24 24zm-192 0c-13.255 0-24-10.745-24-24s10.745-24 24-24 24 10.745 24 24-10.745 24-24 24z"],"shopping-basket":[576,512,[],"f291","M576 216v16c0 13.255-10.745 24-24 24h-8l-26.113 182.788C514.509 462.435 494.257 480 470.37 480H105.63c-23.887 0-44.139-17.565-47.518-41.212L32 256h-8c-13.255 0-24-10.745-24-24v-16c0-13.255 10.745-24 24-24h67.341l106.78-146.821c10.395-14.292 30.407-17.453 44.701-7.058 14.293 10.395 17.453 30.408 7.058 44.701L170.477 192h235.046L326.12 82.821c-10.395-14.292-7.234-34.306 7.059-44.701 14.291-10.395 34.306-7.235 44.701 7.058L484.659 192H552c13.255 0 24 10.745 24 24zM312 392V280c0-13.255-10.745-24-24-24s-24 10.745-24 24v112c0 13.255 10.745 24 24 24s24-10.745 24-24zm112 0V280c0-13.255-10.745-24-24-24s-24 10.745-24 24v112c0 13.255 10.745 24 24 24s24-10.745 24-24zm-224 0V280c0-13.255-10.745-24-24-24s-24 10.745-24 24v112c0 13.255 10.745 24 24 24s24-10.745 24-24z"],"shopping-cart":[576,512,[],"f07a","M528.12 301.319l47.273-208C578.806 78.301 567.391 64 551.99 64H159.208l-9.166-44.81C147.758 8.021 137.93 0 126.529 0H24C10.745 0 0 10.745 0 24v16c0 13.255 10.745 24 24 24h69.883l70.248 343.435C147.325 417.1 136 435.222 136 456c0 30.928 25.072 56 56 56s56-25.072 56-56c0-15.674-6.447-29.835-16.824-40h209.647C430.447 426.165 424 440.326 424 456c0 30.928 25.072 56 56 56s56-25.072 56-56c0-22.172-12.888-41.332-31.579-50.405l5.517-24.276c3.413-15.018-8.002-29.319-23.403-29.319H218.117l-6.545-32h293.145c11.206 0 20.92-7.754 23.403-18.681z"],shower:[512,512,[],"f2cc","M304,320a16,16,0,1,0,16,16A16,16,0,0,0,304,320Zm32-96a16,16,0,1,0,16,16A16,16,0,0,0,336,224Zm32,64a16,16,0,1,0-16-16A16,16,0,0,0,368,288Zm-32,32a16,16,0,1,0-16-16A16,16,0,0,0,336,320Zm-32-64a16,16,0,1,0,16,16A16,16,0,0,0,304,256Zm128-32a16,16,0,1,0-16-16A16,16,0,0,0,432,224Zm-48,16a16,16,0,1,0,16-16A16,16,0,0,0,384,240Zm-16-48a16,16,0,1,0,16,16A16,16,0,0,0,368,192Zm96,32a16,16,0,1,0,16,16A16,16,0,0,0,464,224Zm32-32a16,16,0,1,0,16,16A16,16,0,0,0,496,192Zm-64,64a16,16,0,1,0,16,16A16,16,0,0,0,432,256Zm-32,32a16,16,0,1,0,16,16A16,16,0,0,0,400,288Zm-64,64a16,16,0,1,0,16,16A16,16,0,0,0,336,352Zm-32,32a16,16,0,1,0,16,16A16,16,0,0,0,304,384Zm64-64a16,16,0,1,0,16,16A16,16,0,0,0,368,320Zm21.65-218.35-11.3-11.31a16,16,0,0,0-22.63,0L350.05,96A111.19,111.19,0,0,0,272,64c-19.24,0-37.08,5.3-52.9,13.85l-10-10A121.72,121.72,0,0,0,123.44,32C55.49,31.5,0,92.91,0,160.85V464a16,16,0,0,0,16,16H48a16,16,0,0,0,16-16V158.4c0-30.15,21-58.2,51-61.93a58.38,58.38,0,0,1,48.93,16.67l10,10C165.3,138.92,160,156.76,160,176a111.23,111.23,0,0,0,32,78.05l-5.66,5.67a16,16,0,0,0,0,22.62l11.3,11.31a16,16,0,0,0,22.63,0L389.65,124.28A16,16,0,0,0,389.65,101.65Z"],"shuttle-van":[640,512,[],"f5b6","M628.88 210.65L494.39 49.27A48.01 48.01 0 0 0 457.52 32H32C14.33 32 0 46.33 0 64v288c0 17.67 14.33 32 32 32h32c0 53.02 42.98 96 96 96s96-42.98 96-96h128c0 53.02 42.98 96 96 96s96-42.98 96-96h32c17.67 0 32-14.33 32-32V241.38c0-11.23-3.94-22.1-11.12-30.73zM64 192V96h96v96H64zm96 240c-26.51 0-48-21.49-48-48s21.49-48 48-48 48 21.49 48 48-21.49 48-48 48zm160-240h-96V96h96v96zm160 240c-26.51 0-48-21.49-48-48s21.49-48 48-48 48 21.49 48 48-21.49 48-48 48zm-96-240V96h66.02l80 96H384z"],sign:[512,512,[],"f4d9","M496 64H128V16c0-8.8-7.2-16-16-16H80c-8.8 0-16 7.2-16 16v48H16C7.2 64 0 71.2 0 80v32c0 8.8 7.2 16 16 16h48v368c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V128h368c8.8 0 16-7.2 16-16V80c0-8.8-7.2-16-16-16zM160 384h320V160H160v224z"],"sign-in-alt":[512,512,[],"f2f6","M416 448h-84c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h84c17.7 0 32-14.3 32-32V160c0-17.7-14.3-32-32-32h-84c-6.6 0-12-5.4-12-12V76c0-6.6 5.4-12 12-12h84c53 0 96 43 96 96v192c0 53-43 96-96 96zm-47-201L201 79c-15-15-41-4.5-41 17v96H24c-13.3 0-24 10.7-24 24v96c0 13.3 10.7 24 24 24h136v96c0 21.5 26 32 41 17l168-168c9.3-9.4 9.3-24.6 0-34z"],"sign-language":[448,512,[],"f2a7","M91.434 483.987c-.307-16.018 13.109-29.129 29.13-29.129h62.293v-5.714H56.993c-16.021 0-29.437-13.111-29.13-29.129C28.16 404.491 40.835 392 56.428 392h126.429v-5.714H29.136c-16.021 0-29.437-13.111-29.13-29.129.297-15.522 12.973-28.013 28.566-28.013h154.286v-5.714H57.707c-16.021 0-29.437-13.111-29.13-29.129.297-15.522 12.973-28.013 28.566-28.013h168.566l-31.085-22.606c-12.762-9.281-15.583-27.149-6.302-39.912 9.281-12.761 27.15-15.582 39.912-6.302l123.361 89.715a34.287 34.287 0 0 1 14.12 27.728v141.136c0 15.91-10.946 29.73-26.433 33.374l-80.471 18.934a137.16 137.16 0 0 1-31.411 3.646H120c-15.593-.001-28.269-12.492-28.566-28.014zm73.249-225.701h36.423l-11.187-8.136c-18.579-13.511-20.313-40.887-3.17-56.536l-13.004-16.7c-9.843-12.641-28.43-15.171-40.88-5.088-12.065 9.771-14.133 27.447-4.553 39.75l36.371 46.71zm283.298-2.103l-5.003-152.452c-.518-15.771-13.722-28.136-29.493-27.619-15.773.518-28.137 13.722-27.619 29.493l1.262 38.415L283.565 11.019c-9.58-12.303-27.223-14.63-39.653-5.328-12.827 9.599-14.929 28.24-5.086 40.881l76.889 98.745-4.509 3.511-94.79-121.734c-9.58-12.303-27.223-14.63-39.653-5.328-12.827 9.599-14.929 28.24-5.086 40.881l94.443 121.288-4.509 3.511-77.675-99.754c-9.58-12.303-27.223-14.63-39.653-5.328-12.827 9.599-14.929 28.24-5.086 40.881l52.053 66.849c12.497-8.257 29.055-8.285 41.69.904l123.36 89.714c10.904 7.93 17.415 20.715 17.415 34.198v16.999l61.064-47.549a34.285 34.285 0 0 0 13.202-28.177z"],"sign-out-alt":[512,512,[],"f2f5","M497 273L329 441c-15 15-41 4.5-41-17v-96H152c-13.3 0-24-10.7-24-24v-96c0-13.3 10.7-24 24-24h136V88c0-21.4 25.9-32 41-17l168 168c9.3 9.4 9.3 24.6 0 34zM192 436v-40c0-6.6-5.4-12-12-12H96c-17.7 0-32-14.3-32-32V160c0-17.7 14.3-32 32-32h84c6.6 0 12-5.4 12-12V76c0-6.6-5.4-12-12-12H96c-53 0-96 43-96 96v192c0 53 43 96 96 96h84c6.6 0 12-5.4 12-12z"],signal:[640,512,[],"f012","M216 288h-48c-8.84 0-16 7.16-16 16v192c0 8.84 7.16 16 16 16h48c8.84 0 16-7.16 16-16V304c0-8.84-7.16-16-16-16zM88 384H40c-8.84 0-16 7.16-16 16v96c0 8.84 7.16 16 16 16h48c8.84 0 16-7.16 16-16v-96c0-8.84-7.16-16-16-16zm256-192h-48c-8.84 0-16 7.16-16 16v288c0 8.84 7.16 16 16 16h48c8.84 0 16-7.16 16-16V208c0-8.84-7.16-16-16-16zm128-96h-48c-8.84 0-16 7.16-16 16v384c0 8.84 7.16 16 16 16h48c8.84 0 16-7.16 16-16V112c0-8.84-7.16-16-16-16zM600 0h-48c-8.84 0-16 7.16-16 16v480c0 8.84 7.16 16 16 16h48c8.84 0 16-7.16 16-16V16c0-8.84-7.16-16-16-16z"],signature:[640,512,[],"f5b7","M623.2 192c-51.8 3.5-125.7 54.7-163.1 71.5-29.1 13.1-54.2 24.4-76.1 24.4-22.6 0-26-16.2-21.3-51.9 1.1-8 11.7-79.2-42.7-76.1-25.1 1.5-64.3 24.8-169.5 126L192 182.2c30.4-75.9-53.2-151.5-129.7-102.8L7.4 116.3C0 121-2.2 130.9 2.5 138.4l17.2 27c4.7 7.5 14.6 9.7 22.1 4.9l58-38.9c18.4-11.7 40.7 7.2 32.7 27.1L34.3 404.1C27.5 421 37 448 64 448c8.3 0 16.5-3.2 22.6-9.4 42.2-42.2 154.7-150.7 211.2-195.8-2.2 28.5-2.1 58.9 20.6 83.8 15.3 16.8 37.3 25.3 65.5 25.3 35.6 0 68-14.6 102.3-30 33-14.8 99-62.6 138.4-65.8 8.5-.7 15.2-7.3 15.2-15.8v-32.1c.2-9.1-7.5-16.8-16.6-16.2z"],"sim-card":[384,512,[],"f7c4","M0 64v384c0 35.3 28.7 64 64 64h256c35.3 0 64-28.7 64-64V128L256 0H64C28.7 0 0 28.7 0 64zm224 192h-64v-64h64v64zm96 0h-64v-64h32c17.7 0 32 14.3 32 32v32zm-64 128h64v32c0 17.7-14.3 32-32 32h-32v-64zm-96 0h64v64h-64v-64zm-96 0h64v64H96c-17.7 0-32-14.3-32-32v-32zm0-96h256v64H64v-64zm0-64c0-17.7 14.3-32 32-32h32v64H64v-32z"],sink:[512,512,[],"f96d","M32,416a96,96,0,0,0,96,96H384a96,96,0,0,0,96-96V384H32ZM496,288H400V256h64a16,16,0,0,0,16-16V224a16,16,0,0,0-16-16H384a32,32,0,0,0-32,32v48H288V96a32,32,0,0,1,64,0v16a16,16,0,0,0,16,16h32a16,16,0,0,0,16-16V96A96.16,96.16,0,0,0,300.87,1.86C255.29,10.71,224,53.36,224,99.79V288H160V240a32,32,0,0,0-32-32H48a16,16,0,0,0-16,16v16a16,16,0,0,0,16,16h64v32H16A16,16,0,0,0,0,304v32a16,16,0,0,0,16,16H496a16,16,0,0,0,16-16V304A16,16,0,0,0,496,288Z"],sitemap:[640,512,[],"f0e8","M128 352H32c-17.67 0-32 14.33-32 32v96c0 17.67 14.33 32 32 32h96c17.67 0 32-14.33 32-32v-96c0-17.67-14.33-32-32-32zm-24-80h192v48h48v-48h192v48h48v-57.59c0-21.17-17.23-38.41-38.41-38.41H344v-64h40c17.67 0 32-14.33 32-32V32c0-17.67-14.33-32-32-32H256c-17.67 0-32 14.33-32 32v96c0 17.67 14.33 32 32 32h40v64H94.41C73.23 224 56 241.23 56 262.41V320h48v-48zm264 80h-96c-17.67 0-32 14.33-32 32v96c0 17.67 14.33 32 32 32h96c17.67 0 32-14.33 32-32v-96c0-17.67-14.33-32-32-32zm240 0h-96c-17.67 0-32 14.33-32 32v96c0 17.67 14.33 32 32 32h96c17.67 0 32-14.33 32-32v-96c0-17.67-14.33-32-32-32z"],skating:[448,512,[],"f7c5","M400 0c-26.5 0-48 21.5-48 48s21.5 48 48 48 48-21.5 48-48-21.5-48-48-48zm0 448c-8.8 0-16 7.2-16 16s-7.2 16-16 16h-96c-8.8 0-16 7.2-16 16s7.2 16 16 16h96c26.5 0 48-21.5 48-48 0-8.8-7.2-16-16-16zm-282.2 8.6c-6.2 6.2-16.4 6.3-22.6 0l-67.9-67.9c-6.2-6.2-16.4-6.2-22.6 0s-6.2 16.4 0 22.6l67.9 67.9c9.4 9.4 21.7 14 34 14s24.6-4.7 33.9-14c6.2-6.2 6.2-16.4 0-22.6s-16.4-6.3-22.7 0zm56.1-179.8l-93.7 93.7c-12.5 12.5-12.5 32.8 0 45.2 6.2 6.2 14.4 9.4 22.6 9.4s16.4-3.1 22.6-9.4l91.9-91.9-30.2-30.2c-5-5-9.4-10.7-13.2-16.8zM128 160h105.5l-20.1 17.2c-13.5 11.5-21.6 28.4-22.3 46.1-.7 17.8 6.1 35.2 18.7 47.7l78.2 78.2V432c0 17.7 14.3 32 32 32s32-14.3 32-32v-89.4c0-12.6-5.1-25-14.1-33.9l-61-61c.5-.4 1.2-.6 1.7-1.1l82.3-82.3c11.5-11.5 14.9-28.6 8.7-43.6-6.2-15-20.7-24.7-37-24.7H128c-17.7 0-32 14.3-32 32s14.3 32 32 32z"],skiing:[512,512,[],"f7c9","M432 96c26.5 0 48-21.5 48-48S458.5 0 432 0s-48 21.5-48 48 21.5 48 48 48zm73 356.1c-9.4-9.4-24.6-9.4-33.9 0-12.1 12.1-30.5 15.4-45.1 8.7l-135.8-70.2 49.2-73.8c12.7-19 10.2-44.5-6-60.6L293 215.7l-107-53.1c-2.9 19.9 3.4 40 17.7 54.4l75.1 75.2-45.9 68.8L35 258.7c-11.7-6-26.2-1.5-32.3 10.3-6.1 11.8-1.5 26.3 10.3 32.3l391.9 202.5c11.9 5.5 24.5 8.1 37.1 8.1 23.2 0 46-9 63-26 9.3-9.3 9.3-24.5 0-33.8zM120 91.6l-11.5 22.5c14.4 7.3 31.2 4.9 42.8-4.8l47.2 23.4c-.1.1-.1.2-.2.3l114.5 56.8 32.4-13 6.4 19.1c4 12.1 12.6 22 24 27.7l58.1 29c15.9 7.9 35 1.5 42.9-14.3 7.9-15.8 1.5-35-14.3-42.9l-52.1-26.1-17.1-51.2c-8.1-24.2-40.9-56.6-84.5-39.2l-81.2 32.5-62.5-31c.3-14.5-7.2-28.6-20.9-35.6l-11.1 21.7h-.2l-34.4-7c-1.8-.4-3.7.2-5 1.7-1.9 2.2-1.7 5.5.5 7.4l26.2 23z"],"skiing-nordic":[576,512,[],"f7ca","M336 96c26.5 0 48-21.5 48-48S362.5 0 336 0s-48 21.5-48 48 21.5 48 48 48zm216 320c-13.2 0-24 10.7-24 24 0 13.2-10.8 24-24 24h-69.5L460 285.6c11.7-4.7 20.1-16.2 20.1-29.6 0-17.7-14.3-32-32-32h-44L378 170.8c-12.5-25.5-35.5-44.2-61.8-50.9L245 98.7c-28.3-6.8-57.8-.5-80.8 17.1l-39.7 30.4c-14 10.7-16.7 30.8-5.9 44.9.7.9 1.7 1.3 2.4 2.1L66.9 464H24c-13.2 0-24 10.7-24 24s10.8 24 24 24h480c39.7 0 72-32.3 72-72 0-13.2-10.8-24-24-24zm-260.5 48h-96.9l43.1-91-22-13c-12.1-7.2-21.9-16.9-29.5-27.8L123.7 464H99.5l52.3-261.4c4.1-1 8.1-2.9 11.7-5.6l39.7-30.4c7.7-5.9 17.4-8 25.3-6.1l14.7 4.4-37.5 87.4c-12.6 29.5-1.3 64 26.3 80.3l85 50.2-25.5 81.2zm110.6 0h-43.6l23.6-75.5c5.9-20.8-2.9-43.1-21.6-54.4L299.3 298l31.3-78.3 20.3 41.4c8 16.3 24.9 26.9 43.1 26.9h33.3l-25.2 176z"],skull:[512,512,[],"f54c","M256 0C114.6 0 0 100.3 0 224c0 70.1 36.9 132.6 94.5 173.7 9.6 6.9 15.2 18.1 13.5 29.9l-9.4 66.2c-1.4 9.6 6 18.2 15.7 18.2H192v-56c0-4.4 3.6-8 8-8h16c4.4 0 8 3.6 8 8v56h64v-56c0-4.4 3.6-8 8-8h16c4.4 0 8 3.6 8 8v56h77.7c9.7 0 17.1-8.6 15.7-18.2l-9.4-66.2c-1.7-11.7 3.8-23 13.5-29.9C475.1 356.6 512 294.1 512 224 512 100.3 397.4 0 256 0zm-96 320c-35.3 0-64-28.7-64-64s28.7-64 64-64 64 28.7 64 64-28.7 64-64 64zm192 0c-35.3 0-64-28.7-64-64s28.7-64 64-64 64 28.7 64 64-28.7 64-64 64z"],"skull-crossbones":[448,512,[],"f714","M439.15 453.06L297.17 384l141.99-69.06c7.9-3.95 11.11-13.56 7.15-21.46L432 264.85c-3.95-7.9-13.56-11.11-21.47-7.16L224 348.41 37.47 257.69c-7.9-3.95-17.51-.75-21.47 7.16L1.69 293.48c-3.95 7.9-.75 17.51 7.15 21.46L150.83 384 8.85 453.06c-7.9 3.95-11.11 13.56-7.15 21.47l14.31 28.63c3.95 7.9 13.56 11.11 21.47 7.15L224 419.59l186.53 90.72c7.9 3.95 17.51.75 21.47-7.15l14.31-28.63c3.95-7.91.74-17.52-7.16-21.47zM150 237.28l-5.48 25.87c-2.67 12.62 5.42 24.85 16.45 24.85h126.08c11.03 0 19.12-12.23 16.45-24.85l-5.5-25.87c41.78-22.41 70-62.75 70-109.28C368 57.31 303.53 0 224 0S80 57.31 80 128c0 46.53 28.22 86.87 70 109.28zM280 112c17.65 0 32 14.35 32 32s-14.35 32-32 32-32-14.35-32-32 14.35-32 32-32zm-112 0c17.65 0 32 14.35 32 32s-14.35 32-32 32-32-14.35-32-32 14.35-32 32-32z"],slash:[640,512,[],"f715","M594.53 508.63L6.18 53.9c-6.97-5.42-8.23-15.47-2.81-22.45L23.01 6.18C28.43-.8 38.49-2.06 45.47 3.37L633.82 458.1c6.97 5.42 8.23 15.47 2.81 22.45l-19.64 25.27c-5.42 6.98-15.48 8.23-22.46 2.81z"],sleigh:[640,512,[],"f7cc","M612.7 350.7l-9.3-7.4c-6.9-5.5-17-4.4-22.5 2.5l-10 12.5c-5.5 6.9-4.4 17 2.5 22.5l9.3 7.4c5.9 4.7 9.2 11.7 9.2 19.2 0 13.6-11 24.6-24.6 24.6H48c-8.8 0-16 7.2-16 16v16c0 8.8 7.2 16 16 16h516c39 0 73.7-29.3 75.9-68.3 1.4-23.8-8.7-46.3-27.2-61zM32 224c0 59.6 40.9 109.2 96 123.5V400h64v-48h192v48h64v-48c53 0 96-43 96-96v-96c17.7 0 32-14.3 32-32s-14.3-32-32-32h-96v64c0 35.3-28.7 64-64 64h-20.7c-65.8 0-125.9-37.2-155.3-96-29.4-58.8-89.6-96-155.3-96H32C14.3 32 0 46.3 0 64s14.3 32 32 32v128z"],"sliders-h":[512,512,[],"f1de","M496 384H160v-16c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v16H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h80v16c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16v-16h336c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zm0-160h-80v-16c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v16H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h336v16c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16v-16h80c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zm0-160H288V48c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v16H16C7.2 64 0 71.2 0 80v32c0 8.8 7.2 16 16 16h208v16c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16v-16h208c8.8 0 16-7.2 16-16V80c0-8.8-7.2-16-16-16z"],smile:[496,512,[],"f118","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm80 168c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zm-160 0c17.7 0 32 14.3 32 32s-14.3 32-32 32-32-14.3-32-32 14.3-32 32-32zm194.8 170.2C334.3 380.4 292.5 400 248 400s-86.3-19.6-114.8-53.8c-13.6-16.3 11-36.7 24.6-20.5 22.4 26.9 55.2 42.2 90.2 42.2s67.8-15.4 90.2-42.2c13.4-16.2 38.1 4.2 24.6 20.5z"],"smile-beam":[496,512,[],"f5b8","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zM112 223.4c3.3-42.1 32.2-71.4 56-71.4s52.7 29.3 56 71.4c.7 8.6-10.8 11.9-14.9 4.5l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.3 7.4-15.8 4-15.1-4.5zm250.8 122.8C334.3 380.4 292.5 400 248 400s-86.3-19.6-114.8-53.8c-13.5-16.3 11-36.7 24.6-20.5 22.4 26.9 55.2 42.2 90.2 42.2s67.8-15.4 90.2-42.2c13.6-16.2 38.1 4.3 24.6 20.5zm6.2-118.3l-9.5-17c-7.7-13.7-19.2-21.6-31.5-21.6s-23.8 7.9-31.5 21.6l-9.5 17c-4.1 7.3-15.6 4-14.9-4.5 3.3-42.1 32.2-71.4 56-71.4s52.7 29.3 56 71.4c.6 8.6-11 11.9-15.1 4.5z"],"smile-wink":[496,512,[],"f4da","M0 256c0 137 111 248 248 248s248-111 248-248S385 8 248 8 0 119 0 256zm200-48c0 17.7-14.3 32-32 32s-32-14.3-32-32 14.3-32 32-32 32 14.3 32 32zm158.5 16.5c-14.8-13.2-46.2-13.2-61 0L288 233c-8.3 7.4-21.6.4-19.8-10.8 4-25.2 34.2-42.1 59.9-42.1S384 197 388 222.2c1.7 11.1-11.4 18.3-19.8 10.8l-9.7-8.5zM157.8 325.8C180.2 352.7 213 368 248 368s67.8-15.4 90.2-42.2c13.6-16.2 38.1 4.2 24.6 20.5C334.3 380.4 292.5 400 248 400s-86.3-19.6-114.8-53.8c-13.5-16.3 11.2-36.7 24.6-20.4z"],smog:[640,512,[],"f75f","M624 368H80c-8.8 0-16 7.2-16 16v16c0 8.8 7.2 16 16 16h544c8.8 0 16-7.2 16-16v-16c0-8.8-7.2-16-16-16zm-480 96H16c-8.8 0-16 7.2-16 16v16c0 8.8 7.2 16 16 16h128c8.8 0 16-7.2 16-16v-16c0-8.8-7.2-16-16-16zm416 0H224c-8.8 0-16 7.2-16 16v16c0 8.8 7.2 16 16 16h336c8.8 0 16-7.2 16-16v-16c0-8.8-7.2-16-16-16zM144 288h156.1c22.5 19.7 51.6 32 83.9 32s61.3-12.3 83.9-32H528c61.9 0 112-50.1 112-112S589.9 64 528 64c-18 0-34.7 4.6-49.7 12.1C454 31 406.8 0 352 0c-41 0-77.8 17.3-104 44.8C221.8 17.3 185 0 144 0 64.5 0 0 64.5 0 144s64.5 144 144 144z"],smoking:[640,512,[],"f48d","M632 352h-48c-4.4 0-8 3.6-8 8v144c0 4.4 3.6 8 8 8h48c4.4 0 8-3.6 8-8V360c0-4.4-3.6-8-8-8zM553.3 87.1c-5.7-3.8-9.3-10-9.3-16.8V8c0-4.4-3.6-8-8-8h-48c-4.4 0-8 3.6-8 8v62.3c0 22 10.2 43.4 28.6 55.4 42.2 27.3 67.4 73.8 67.4 124V280c0 4.4 3.6 8 8 8h48c4.4 0 8-3.6 8-8v-30.3c0-65.5-32.4-126.2-86.7-162.6zM432 352H48c-26.5 0-48 21.5-48 48v64c0 26.5 21.5 48 48 48h384c8.8 0 16-7.2 16-16V368c0-8.8-7.2-16-16-16zm-32 112H224v-64h176v64zm87.7-322.4C463.8 125 448 99.3 448 70.3V8c0-4.4-3.6-8-8-8h-48c-4.4 0-8 3.6-8 8v66.4c0 43.7 24.6 81.6 60.3 106.7 22.4 15.7 35.7 41.2 35.7 68.6V280c0 4.4 3.6 8 8 8h48c4.4 0 8-3.6 8-8v-30.3c0-43.3-21-83.4-56.3-108.1zM536 352h-48c-4.4 0-8 3.6-8 8v144c0 4.4 3.6 8 8 8h48c4.4 0 8-3.6 8-8V360c0-4.4-3.6-8-8-8z"],"smoking-ban":[512,512,[],"f54d","M96 304c0 8.8 7.2 16 16 16h117.5l-96-96H112c-8.8 0-16 7.2-16 16v64zM256 0C114.6 0 0 114.6 0 256s114.6 256 256 256 256-114.6 256-256S397.4 0 256 0zm0 448c-105.9 0-192-86.1-192-192 0-41.4 13.3-79.7 35.7-111.1l267.4 267.4C335.7 434.7 297.4 448 256 448zm45.2-192H384v32h-50.8l-32-32zm111.1 111.1L365.2 320H400c8.8 0 16-7.2 16-16v-64c0-8.8-7.2-16-16-16H269.2L144.9 99.7C176.3 77.3 214.6 64 256 64c105.9 0 192 86.1 192 192 0 41.4-13.3 79.7-35.7 111.1zM320.6 128c-15.6 0-28.6-11.2-31.4-25.9-.7-3.6-4-6.1-7.7-6.1h-16.2c-5 0-8.7 4.5-8 9.4 4.6 30.9 31.2 54.6 63.3 54.6 15.6 0 28.6 11.2 31.4 25.9.7 3.6 4 6.1 7.7 6.1h16.2c5 0 8.7-4.5 8-9.4-4.6-30.9-31.2-54.6-63.3-54.6z"],sms:[512,512,[],"f7cd","M256 32C114.6 32 0 125.1 0 240c0 49.6 21.4 95 57 130.7C44.5 421.1 2.7 466 2.2 466.5c-2.2 2.3-2.8 5.7-1.5 8.7 1.3 3 4.1 4.8 7.3 4.8 66.3 0 116-31.8 140.6-51.4 32.7 12.3 69 19.4 107.4 19.4 141.4 0 256-93.1 256-208S397.4 32 256 32zM128.2 304H116c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h12.3c6 0 10.4-3.5 10.4-6.6 0-1.3-.8-2.7-2.1-3.8l-21.9-18.8c-8.5-7.2-13.3-17.5-13.3-28.1 0-21.3 19-38.6 42.4-38.6H156c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8h-12.3c-6 0-10.4 3.5-10.4 6.6 0 1.3.8 2.7 2.1 3.8l21.9 18.8c8.5 7.2 13.3 17.5 13.3 28.1.1 21.3-19 38.6-42.4 38.6zm191.8-8c0 4.4-3.6 8-8 8h-16c-4.4 0-8-3.6-8-8v-68.2l-24.8 55.8c-2.9 5.9-11.4 5.9-14.3 0L224 227.8V296c0 4.4-3.6 8-8 8h-16c-4.4 0-8-3.6-8-8V192c0-8.8 7.2-16 16-16h16c6.1 0 11.6 3.4 14.3 8.8l17.7 35.4 17.7-35.4c2.7-5.4 8.3-8.8 14.3-8.8h16c8.8 0 16 7.2 16 16v104zm48.3 8H356c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h12.3c6 0 10.4-3.5 10.4-6.6 0-1.3-.8-2.7-2.1-3.8l-21.9-18.8c-8.5-7.2-13.3-17.5-13.3-28.1 0-21.3 19-38.6 42.4-38.6H396c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8h-12.3c-6 0-10.4 3.5-10.4 6.6 0 1.3.8 2.7 2.1 3.8l21.9 18.8c8.5 7.2 13.3 17.5 13.3 28.1.1 21.3-18.9 38.6-42.3 38.6z"],snowboarding:[512,512,[],"f7ce","M432 96c26.5 0 48-21.5 48-48S458.5 0 432 0s-48 21.5-48 48 21.5 48 48 48zm28.8 153.6c5.8 4.3 12.5 6.4 19.2 6.4 9.7 0 19.3-4.4 25.6-12.8 10.6-14.1 7.8-34.2-6.4-44.8l-111.4-83.5c-13.8-10.3-29.1-18.4-45.4-23.8l-63.7-21.2-26.1-52.1C244.7 2 225.5-4.4 209.7 3.5c-15.8 7.9-22.2 27.1-14.3 42.9l29.1 58.1c5.7 11.4 15.6 19.9 27.7 24l16.4 5.5-41.2 20.6c-21.8 10.9-35.4 32.8-35.4 57.2v53.1l-74.1 24.7c-16.8 5.6-25.8 23.7-20.2 40.5 1.7 5.2 4.9 9.4 8.7 12.9l-38.7-14.1c-9.7-3.5-17.4-10.6-21.8-20-5.6-12-19.9-17.2-31.9-11.6s-17.2 19.9-11.6 31.9c9.8 21 27.1 36.9 48.9 44.8l364.8 132.7c9.7 3.5 19.7 5.3 29.7 5.3 12.5 0 24.9-2.7 36.5-8.2 12-5.6 17.2-19.9 11.6-31.9S474 454.7 462 460.3c-9.3 4.4-19.8 4.8-29.5 1.3l-90.8-33.1c8.7-4.1 15.6-11.8 17.8-21.9l21.9-102c3.9-18.2-3.2-37.2-18.1-48.4l-52-39 66-30.5 83.5 62.9zm-144.4 51.7l-19.7 92c-1.5 7.1-.1 13.9 2.8 20l-169.4-61.6c2.7-.2 5.4-.4 8-1.3l85-28.4c19.6-6.5 32.8-24.8 32.8-45.5V256l60.5 45.3z"],snowflake:[448,512,[],"f2dc","M440.3 345.2l-33.8-19.5 26-7c8.2-2.2 13.1-10.7 10.9-18.9l-4-14.9c-2.2-8.2-10.7-13.1-18.9-10.9l-70.8 19-63.9-37 63.8-36.9 70.8 19c8.2 2.2 16.7-2.7 18.9-10.9l4-14.9c2.2-8.2-2.7-16.7-10.9-18.9l-26-7 33.8-19.5c7.4-4.3 9.9-13.7 5.7-21.1L430.4 119c-4.3-7.4-13.7-9.9-21.1-5.7l-33.8 19.5 7-26c2.2-8.2-2.7-16.7-10.9-18.9l-14.9-4c-8.2-2.2-16.7 2.7-18.9 10.9l-19 70.8-62.8 36.2v-77.5l53.7-53.7c6.2-6.2 6.2-16.4 0-22.6l-11.3-11.3c-6.2-6.2-16.4-6.2-22.6 0L256 56.4V16c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v40.4l-19.7-19.7c-6.2-6.2-16.4-6.2-22.6 0L138.3 48c-6.3 6.2-6.3 16.4 0 22.6l53.7 53.7v77.5l-62.8-36.2-19-70.8c-2.2-8.2-10.7-13.1-18.9-10.9l-14.9 4c-8.2 2.2-13.1 10.7-10.9 18.9l7 26-33.8-19.5c-7.4-4.3-16.8-1.7-21.1 5.7L2.1 145.7c-4.3 7.4-1.7 16.8 5.7 21.1l33.8 19.5-26 7c-8.3 2.2-13.2 10.7-11 19l4 14.9c2.2 8.2 10.7 13.1 18.9 10.9l70.8-19 63.8 36.9-63.8 36.9-70.8-19c-8.2-2.2-16.7 2.7-18.9 10.9l-4 14.9c-2.2 8.2 2.7 16.7 10.9 18.9l26 7-33.8 19.6c-7.4 4.3-9.9 13.7-5.7 21.1l15.5 26.8c4.3 7.4 13.7 9.9 21.1 5.7l33.8-19.5-7 26c-2.2 8.2 2.7 16.7 10.9 18.9l14.9 4c8.2 2.2 16.7-2.7 18.9-10.9l19-70.8 62.8-36.2v77.5l-53.7 53.7c-6.3 6.2-6.3 16.4 0 22.6l11.3 11.3c6.2 6.2 16.4 6.2 22.6 0l19.7-19.7V496c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16v-40.4l19.7 19.7c6.2 6.2 16.4 6.2 22.6 0l11.3-11.3c6.2-6.2 6.2-16.4 0-22.6L256 387.7v-77.5l62.8 36.2 19 70.8c2.2 8.2 10.7 13.1 18.9 10.9l14.9-4c8.2-2.2 13.1-10.7 10.9-18.9l-7-26 33.8 19.5c7.4 4.3 16.8 1.7 21.1-5.7l15.5-26.8c4.3-7.3 1.8-16.8-5.6-21z"],snowman:[512,512,[],"f7d0","M510.9 152.3l-5.9-14.5c-3.3-8-12.6-11.9-20.8-8.7L456 140.6v-29c0-8.6-7.2-15.6-16-15.6h-16c-8.8 0-16 7-16 15.6v46.9c0 .5.3 1 .3 1.5l-56.4 23c-5.9-10-13.3-18.9-22-26.6 13.6-16.6 22-37.4 22-60.5 0-53-43-96-96-96s-96 43-96 96c0 23.1 8.5 43.9 22 60.5-8.7 7.7-16 16.6-22 26.6l-56.4-23c.1-.5.3-1 .3-1.5v-46.9C104 103 96.8 96 88 96H72c-8.8 0-16 7-16 15.6v29l-28.1-11.5c-8.2-3.2-17.5.7-20.8 8.7l-5.9 14.5c-3.3 8 .7 17.1 8.9 20.3l135.2 55.2c-.4 4-1.2 8-1.2 12.2 0 10.1 1.7 19.6 4.2 28.9C120.9 296.4 104 334.2 104 376c0 54 28.4 100.9 70.8 127.8 9.3 5.9 20.3 8.2 31.3 8.2h99.2c13.3 0 26.3-4.1 37.2-11.7 46.5-32.3 74.4-89.4 62.9-152.6-5.5-30.2-20.5-57.6-41.6-79 2.5-9.2 4.2-18.7 4.2-28.7 0-4.2-.8-8.1-1.2-12.2L502 172.6c8.1-3.1 12.1-12.2 8.9-20.3zM224 96c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zm32 272c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zm0-64c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zm0-64c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16zm0-88s-16-23.2-16-32 7.2-16 16-16 16 7.2 16 16-16 32-16 32zm32-56c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16z"],snowplow:[640,512,[],"f7d2","M120 376c-13.3 0-24 10.7-24 24s10.7 24 24 24 24-10.7 24-24-10.7-24-24-24zm80 0c-13.3 0-24 10.7-24 24s10.7 24 24 24 24-10.7 24-24-10.7-24-24-24zm80 0c-13.3 0-24 10.7-24 24s10.7 24 24 24 24-10.7 24-24-10.7-24-24-24zm80 0c-13.3 0-24 10.7-24 24s10.7 24 24 24 24-10.7 24-24-10.7-24-24-24zm238.6 49.4c-14.5-14.5-22.6-34.1-22.6-54.6V269.2c0-20.5 8.1-40.1 22.6-54.6l36.7-36.7c6.2-6.2 6.2-16.4 0-22.6l-22.6-22.6c-6.2-6.2-16.4-6.2-22.6 0l-36.7 36.7c-26.5 26.5-41.4 62.4-41.4 99.9V288h-64v-50.9c0-8.7-1.8-17.2-5.2-25.2L364.5 29.1C356.9 11.4 339.6 0 320.3 0H176c-26.5 0-48 21.5-48 48v112h-16c-26.5 0-48 21.5-48 48v91.2C26.3 317.2 0 355.4 0 400c0 61.9 50.1 112 112 112h256c61.9 0 112-50.1 112-112 0-17.3-4.2-33.4-11.2-48H512v18.7c0 37.5 14.9 73.4 41.4 99.9l36.7 36.7c6.2 6.2 16.4 6.2 22.6 0l22.6-22.6c6.2-6.2 6.2-16.4 0-22.6l-36.7-36.7zM192 64h117.8l68.6 160H256l-64-64V64zm176 384H112c-26.5 0-48-21.5-48-48s21.5-48 48-48h256c26.5 0 48 21.5 48 48s-21.5 48-48 48z"],soap:[512,512,[],"f96e","M416,192a95.42,95.42,0,0,1-30.94,70.21A95.8,95.8,0,0,1,352,448H160a96,96,0,0,1,0-192h88.91A95.3,95.3,0,0,1,224,192H96A96,96,0,0,0,0,288V416a96,96,0,0,0,96,96H416a96,96,0,0,0,96-96V288A96,96,0,0,0,416,192Zm-96,64a64,64,0,1,0-64-64A64,64,0,0,0,320,256ZM208,96a48,48,0,1,0-48-48A48,48,0,0,0,208,96ZM384,64a32,32,0,1,0-32-32A32,32,0,0,0,384,64ZM160,288a64,64,0,0,0,0,128H352a64,64,0,0,0,0-128Z"],socks:[512,512,[],"f696","M214.66 311.01L288 256V96H128v176l-86.65 64.61c-39.4 29.56-53.86 84.42-29.21 127.06C30.39 495.25 63.27 512 96.08 512c20.03 0 40.25-6.25 57.52-19.2l21.86-16.39c-29.85-55.38-13.54-125.84 39.2-165.4zM288 32c0-11.05 3.07-21.3 8.02-30.38C293.4.92 290.85 0 288 0H160c-17.67 0-32 14.33-32 32v32h160V32zM480 0H352c-17.67 0-32 14.33-32 32v32h192V32c0-17.67-14.33-32-32-32zM320 272l-86.13 64.61c-39.4 29.56-53.86 84.42-29.21 127.06 18.25 31.58 50.61 48.33 83.42 48.33 20.03 0 40.25-6.25 57.52-19.2l115.2-86.4A127.997 127.997 0 0 0 512 304V96H320v176z"],"solar-panel":[640,512,[],"f5ba","M431.98 448.01l-47.97.05V416h-128v32.21l-47.98.05c-8.82.01-15.97 7.16-15.98 15.99l-.05 31.73c-.01 8.85 7.17 16.03 16.02 16.02l223.96-.26c8.82-.01 15.97-7.16 15.98-15.98l.04-31.73c.01-8.85-7.17-16.03-16.02-16.02zM585.2 26.74C582.58 11.31 568.99 0 553.06 0H86.93C71 0 57.41 11.31 54.79 26.74-3.32 369.16.04 348.08.03 352c-.03 17.32 14.29 32 32.6 32h574.74c18.23 0 32.51-14.56 32.59-31.79.02-4.08 3.35 16.95-54.76-325.47zM259.83 64h120.33l9.77 96H250.06l9.77-96zm-75.17 256H71.09L90.1 208h105.97l-11.41 112zm16.29-160H98.24l16.29-96h96.19l-9.77 96zm32.82 160l11.4-112h149.65l11.4 112H233.77zm195.5-256h96.19l16.29 96H439.04l-9.77-96zm26.06 256l-11.4-112H549.9l19.01 112H455.33z"],sort:[320,512,[],"f0dc","M41 288h238c21.4 0 32.1 25.9 17 41L177 448c-9.4 9.4-24.6 9.4-33.9 0L24 329c-15.1-15.1-4.4-41 17-41zm255-105L177 64c-9.4-9.4-24.6-9.4-33.9 0L24 183c-15.1 15.1-4.4 41 17 41h238c21.4 0 32.1-25.9 17-41z"],"sort-alpha-down":[448,512,[],"f15d","M176 352h-48V48a16 16 0 0 0-16-16H80a16 16 0 0 0-16 16v304H16c-14.19 0-21.36 17.24-11.29 27.31l80 96a16 16 0 0 0 22.62 0l80-96C197.35 369.26 190.22 352 176 352zm240-64H288a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h56l-61.26 70.45A32 32 0 0 0 272 446.37V464a16 16 0 0 0 16 16h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16h-56l61.26-70.45A32 32 0 0 0 432 321.63V304a16 16 0 0 0-16-16zm31.06-85.38l-59.27-160A16 16 0 0 0 372.72 32h-41.44a16 16 0 0 0-15.07 10.62l-59.27 160A16 16 0 0 0 272 224h24.83a16 16 0 0 0 15.23-11.08l4.42-12.92h71l4.41 12.92A16 16 0 0 0 407.16 224H432a16 16 0 0 0 15.06-21.38zM335.61 144L352 96l16.39 48z"],"sort-alpha-down-alt":[448,512,[],"f881","M176 352h-48V48a16 16 0 0 0-16-16H80a16 16 0 0 0-16 16v304H16c-14.19 0-21.36 17.24-11.29 27.31l80 96a16 16 0 0 0 22.62 0l80-96C197.35 369.26 190.22 352 176 352zm112-128h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16h-56l61.26-70.45A32 32 0 0 0 432 65.63V48a16 16 0 0 0-16-16H288a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h56l-61.26 70.45A32 32 0 0 0 272 190.37V208a16 16 0 0 0 16 16zm159.06 234.62l-59.27-160A16 16 0 0 0 372.72 288h-41.44a16 16 0 0 0-15.07 10.62l-59.27 160A16 16 0 0 0 272 480h24.83a16 16 0 0 0 15.23-11.08l4.42-12.92h71l4.41 12.92A16 16 0 0 0 407.16 480H432a16 16 0 0 0 15.06-21.38zM335.61 400L352 352l16.39 48z"],"sort-alpha-up":[448,512,[],"f15e","M16 160h48v304a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V160h48c14.21 0 21.38-17.24 11.31-27.31l-80-96a16 16 0 0 0-22.62 0l-80 96C-5.35 142.74 1.78 160 16 160zm400 128H288a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h56l-61.26 70.45A32 32 0 0 0 272 446.37V464a16 16 0 0 0 16 16h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16h-56l61.26-70.45A32 32 0 0 0 432 321.63V304a16 16 0 0 0-16-16zm31.06-85.38l-59.27-160A16 16 0 0 0 372.72 32h-41.44a16 16 0 0 0-15.07 10.62l-59.27 160A16 16 0 0 0 272 224h24.83a16 16 0 0 0 15.23-11.08l4.42-12.92h71l4.41 12.92A16 16 0 0 0 407.16 224H432a16 16 0 0 0 15.06-21.38zM335.61 144L352 96l16.39 48z"],"sort-alpha-up-alt":[448,512,[],"f882","M16 160h48v304a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V160h48c14.21 0 21.38-17.24 11.31-27.31l-80-96a16 16 0 0 0-22.62 0l-80 96C-5.35 142.74 1.78 160 16 160zm272 64h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16h-56l61.26-70.45A32 32 0 0 0 432 65.63V48a16 16 0 0 0-16-16H288a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h56l-61.26 70.45A32 32 0 0 0 272 190.37V208a16 16 0 0 0 16 16zm159.06 234.62l-59.27-160A16 16 0 0 0 372.72 288h-41.44a16 16 0 0 0-15.07 10.62l-59.27 160A16 16 0 0 0 272 480h24.83a16 16 0 0 0 15.23-11.08l4.42-12.92h71l4.41 12.92A16 16 0 0 0 407.16 480H432a16 16 0 0 0 15.06-21.38zM335.61 400L352 352l16.39 48z"],"sort-amount-down":[512,512,[],"f160","M304 416h-64a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h64a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm-128-64h-48V48a16 16 0 0 0-16-16H80a16 16 0 0 0-16 16v304H16c-14.19 0-21.37 17.24-11.29 27.31l80 96a16 16 0 0 0 22.62 0l80-96C197.35 369.26 190.22 352 176 352zm256-192H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h192a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm-64 128H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zM496 32H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h256a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z"],"sort-amount-down-alt":[512,512,[],"f884","M240 96h64a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16h-64a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16zm0 128h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16zm256 192H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h256a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm-256-64h192a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16zm-64 0h-48V48a16 16 0 0 0-16-16H80a16 16 0 0 0-16 16v304H16c-14.19 0-21.37 17.24-11.29 27.31l80 96a16 16 0 0 0 22.62 0l80-96C197.35 369.26 190.22 352 176 352z"],"sort-amount-up":[512,512,[],"f161","M304 416h-64a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h64a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zM16 160h48v304a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V160h48c14.21 0 21.38-17.24 11.31-27.31l-80-96a16 16 0 0 0-22.62 0l-80 96C-5.35 142.74 1.77 160 16 160zm416 0H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h192a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm-64 128H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zM496 32H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h256a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z"],"sort-amount-up-alt":[512,512,[],"f885","M240 96h64a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16h-64a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16zm0 128h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16zm256 192H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h256a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm-256-64h192a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16H240a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16zM16 160h48v304a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V160h48c14.21 0 21.39-17.24 11.31-27.31l-80-96a16 16 0 0 0-22.62 0l-80 96C-5.35 142.74 1.78 160 16 160z"],"sort-down":[320,512,[],"f0dd","M41 288h238c21.4 0 32.1 25.9 17 41L177 448c-9.4 9.4-24.6 9.4-33.9 0L24 329c-15.1-15.1-4.4-41 17-41z"],"sort-numeric-down":[448,512,[],"f162","M304 96h16v64h-16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h96a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16h-16V48a16 16 0 0 0-16-16h-48a16 16 0 0 0-14.29 8.83l-16 32A16 16 0 0 0 304 96zm26.15 162.91a79 79 0 0 0-55 54.17c-14.25 51.05 21.21 97.77 68.85 102.53a84.07 84.07 0 0 1-20.85 12.91c-7.57 3.4-10.8 12.47-8.18 20.34l9.9 20c2.87 8.63 12.53 13.49 20.9 9.91 58-24.76 86.25-61.61 86.25-132V336c-.02-51.21-48.4-91.34-101.85-77.09zM352 356a20 20 0 1 1 20-20 20 20 0 0 1-20 20zm-176-4h-48V48a16 16 0 0 0-16-16H80a16 16 0 0 0-16 16v304H16c-14.19 0-21.36 17.24-11.29 27.31l80 96a16 16 0 0 0 22.62 0l80-96C197.35 369.26 190.22 352 176 352z"],"sort-numeric-down-alt":[448,512,[],"f886","M176 352h-48V48a16 16 0 0 0-16-16H80a16 16 0 0 0-16 16v304H16c-14.19 0-21.36 17.24-11.29 27.31l80 96a16 16 0 0 0 22.62 0l80-96C197.35 369.26 190.22 352 176 352zm224 64h-16V304a16 16 0 0 0-16-16h-48a16 16 0 0 0-14.29 8.83l-16 32A16 16 0 0 0 304 352h16v64h-16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h96a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zM330.17 34.91a79 79 0 0 0-55 54.17c-14.27 51.05 21.19 97.77 68.83 102.53a84.07 84.07 0 0 1-20.85 12.91c-7.57 3.4-10.8 12.47-8.18 20.34l9.9 20c2.87 8.63 12.53 13.49 20.9 9.91 58-24.77 86.25-61.61 86.25-132V112c-.02-51.21-48.4-91.34-101.85-77.09zM352 132a20 20 0 1 1 20-20 20 20 0 0 1-20 20z"],"sort-numeric-up":[448,512,[],"f163","M330.17 258.91a79 79 0 0 0-55 54.17c-14.27 51.05 21.19 97.77 68.83 102.53a84.07 84.07 0 0 1-20.85 12.91c-7.57 3.4-10.8 12.47-8.18 20.34l9.9 20c2.87 8.63 12.53 13.49 20.9 9.91 58-24.76 86.25-61.61 86.25-132V336c-.02-51.21-48.4-91.34-101.85-77.09zM352 356a20 20 0 1 1 20-20 20 20 0 0 1-20 20zM304 96h16v64h-16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h96a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16h-16V48a16 16 0 0 0-16-16h-48a16 16 0 0 0-14.29 8.83l-16 32A16 16 0 0 0 304 96zM107.31 36.69a16 16 0 0 0-22.62 0l-80 96C-5.35 142.74 1.78 160 16 160h48v304a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V160h48c14.21 0 21.38-17.24 11.31-27.31z"],"sort-numeric-up-alt":[448,512,[],"f887","M107.31 36.69a16 16 0 0 0-22.62 0l-80 96C-5.35 142.74 1.78 160 16 160h48v304a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V160h48c14.21 0 21.38-17.24 11.31-27.31zM400 416h-16V304a16 16 0 0 0-16-16h-48a16 16 0 0 0-14.29 8.83l-16 32A16 16 0 0 0 304 352h16v64h-16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h96a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zM330.17 34.91a79 79 0 0 0-55 54.17c-14.27 51.05 21.19 97.77 68.83 102.53a84.07 84.07 0 0 1-20.85 12.91c-7.57 3.4-10.8 12.47-8.18 20.34l9.9 20c2.87 8.63 12.53 13.49 20.9 9.91 58-24.77 86.25-61.61 86.25-132V112c-.02-51.21-48.4-91.34-101.85-77.09zM352 132a20 20 0 1 1 20-20 20 20 0 0 1-20 20z"],"sort-up":[320,512,[],"f0de","M279 224H41c-21.4 0-32.1-25.9-17-41L143 64c9.4-9.4 24.6-9.4 33.9 0l119 119c15.2 15.1 4.5 41-16.9 41z"],spa:[576,512,[],"f5bb","M568.25 192c-29.04.13-135.01 6.16-213.84 83-33.12 29.63-53.36 63.3-66.41 94.86-13.05-31.56-33.29-65.23-66.41-94.86-78.83-76.84-184.8-82.87-213.84-83-4.41-.02-7.79 3.4-7.75 7.82.23 27.92 7.14 126.14 88.77 199.3C172.79 480.94 256 480 288 480s115.19.95 199.23-80.88c81.64-73.17 88.54-171.38 88.77-199.3.04-4.42-3.34-7.84-7.75-7.82zM287.98 302.6c12.82-18.85 27.6-35.78 44.09-50.52 19.09-18.61 39.58-33.3 60.26-45.18-16.44-70.5-51.72-133.05-96.73-172.22-4.11-3.58-11.02-3.58-15.14 0-44.99 39.14-80.27 101.63-96.74 172.07 20.37 11.7 40.5 26.14 59.22 44.39a282.768 282.768 0 0 1 45.04 51.46z"],"space-shuttle":[640,512,[],"f197","M592.604 208.244C559.735 192.836 515.777 184 472 184H186.327c-4.952-6.555-10.585-11.978-16.72-16H376C229.157 137.747 219.403 32 96.003 32H96v128H80V32c-26.51 0-48 28.654-48 64v64c-23.197 0-32 10.032-32 24v40c0 13.983 8.819 24 32 24v16c-23.197 0-32 10.032-32 24v40c0 13.983 8.819 24 32 24v64c0 35.346 21.49 64 48 64V352h16v128h.003c123.4 0 133.154-105.747 279.997-136H169.606c6.135-4.022 11.768-9.445 16.72-16H472c43.777 0 87.735-8.836 120.604-24.244C622.282 289.845 640 271.992 640 256s-17.718-33.845-47.396-47.756zM488 296a8 8 0 0 1-8-8v-64a8 8 0 0 1 8-8c31.909 0 31.942 80 0 80z"],"spell-check":[576,512,[],"f891","M272 256h91.36c43.2 0 82-32.2 84.51-75.34a79.82 79.82 0 0 0-25.26-63.07 79.81 79.81 0 0 0 9.06-44.91C427.9 30.57 389.3 0 347 0h-75a16 16 0 0 0-16 16v224a16 16 0 0 0 16 16zm40-200h40a24 24 0 0 1 0 48h-40zm0 96h56a24 24 0 0 1 0 48h-56zM155.12 22.25A32 32 0 0 0 124.64 0H99.36a32 32 0 0 0-30.48 22.25L.59 235.73A16 16 0 0 0 16 256h24.93a16 16 0 0 0 15.42-11.73L68.29 208h87.42l11.94 36.27A16 16 0 0 0 183.07 256H208a16 16 0 0 0 15.42-20.27zM89.37 144L112 75.3l22.63 68.7zm482 132.48l-45.21-45.3a15.88 15.88 0 0 0-22.59 0l-151.5 151.5-55.41-55.5a15.88 15.88 0 0 0-22.59 0l-45.3 45.3a16 16 0 0 0 0 22.59l112 112.21a15.89 15.89 0 0 0 22.6 0l208-208.21a16 16 0 0 0-.02-22.59z"],spider:[576,512,[],"f717","M151.17 167.35L177.1 176h4.67l5.22-26.12c.72-3.58 1.8-7.58 3.21-11.79l-20.29-40.58 23.8-71.39c2.79-8.38-1.73-17.44-10.12-20.24L168.42.82c-8.38-2.8-17.45 1.73-20.24 10.12l-25.89 77.68a32.04 32.04 0 0 0 1.73 24.43l27.15 54.3zm422.14 182.03l-52.75-79.12a32.002 32.002 0 0 0-26.62-14.25H416l68.99-24.36a32.03 32.03 0 0 0 16.51-12.61l53.6-80.41c4.9-7.35 2.91-17.29-4.44-22.19l-13.31-8.88c-7.35-4.9-17.29-2.91-22.19 4.44l-50.56 75.83L404.1 208H368l-10.37-51.85C355.44 145.18 340.26 96 288 96c-52.26 0-67.44 49.18-69.63 60.15L208 208h-36.1l-60.49-20.17L60.84 112c-4.9-7.35-14.83-9.34-22.19-4.44l-13.31 8.88c-7.35 4.9-9.34 14.83-4.44 22.19l53.6 80.41a32.03 32.03 0 0 0 16.51 12.61L160 256H82.06a32.02 32.02 0 0 0-26.63 14.25L2.69 349.38c-4.9 7.35-2.92 17.29 4.44 22.19l13.31 8.88c7.35 4.9 17.29 2.91 22.19-4.44l48-72h47.06l-60.83 97.33A31.988 31.988 0 0 0 72 418.3V496c0 8.84 7.16 16 16 16h16c8.84 0 16-7.16 16-16v-73.11l74.08-118.53c-1.01 14.05-2.08 28.11-2.08 42.21C192 399.64 232.76 448 288 448s96-48.36 96-101.43c0-14.1-1.08-28.16-2.08-42.21L456 422.89V496c0 8.84 7.16 16 16 16h16c8.84 0 16-7.16 16-16v-77.71c0-6-1.69-11.88-4.86-16.96L438.31 304h47.06l48 72c4.9 7.35 14.84 9.34 22.19 4.44l13.31-8.88c7.36-4.9 9.34-14.83 4.44-22.18zM406.09 97.51l-20.29 40.58c1.41 4.21 2.49 8.21 3.21 11.79l5.22 26.12h4.67l25.93-8.65 27.15-54.3a31.995 31.995 0 0 0 1.73-24.43l-25.89-77.68C425.03 2.56 415.96-1.98 407.58.82l-15.17 5.06c-8.38 2.8-12.91 11.86-10.12 20.24l23.8 71.39z"],spinner:[512,512,[],"f110","M304 48c0 26.51-21.49 48-48 48s-48-21.49-48-48 21.49-48 48-48 48 21.49 48 48zm-48 368c-26.51 0-48 21.49-48 48s21.49 48 48 48 48-21.49 48-48-21.49-48-48-48zm208-208c-26.51 0-48 21.49-48 48s21.49 48 48 48 48-21.49 48-48-21.49-48-48-48zM96 256c0-26.51-21.49-48-48-48S0 229.49 0 256s21.49 48 48 48 48-21.49 48-48zm12.922 99.078c-26.51 0-48 21.49-48 48s21.49 48 48 48 48-21.49 48-48c0-26.509-21.491-48-48-48zm294.156 0c-26.51 0-48 21.49-48 48s21.49 48 48 48 48-21.49 48-48c0-26.509-21.49-48-48-48zM108.922 60.922c-26.51 0-48 21.49-48 48s21.49 48 48 48 48-21.49 48-48-21.491-48-48-48z"],splotch:[512,512,[],"f5bc","M472.29 195.89l-67.06-22.95c-19.28-6.6-33.54-20.92-38.14-38.3L351.1 74.19c-11.58-43.77-76.57-57.13-109.98-22.62l-46.14 47.67c-13.26 13.71-33.54 20.93-54.2 19.31l-71.88-5.62c-52.05-4.07-86.93 44.88-59.03 82.83l38.54 52.42c11.08 15.07 12.82 33.86 4.64 50.24L24.62 355.4c-20.59 41.25 22.84 84.87 73.49 73.81l69.96-15.28c20.11-4.39 41.45 0 57.07 11.73l54.32 40.83c39.32 29.56 101.04 7.57 104.45-37.22l4.7-61.86c1.35-17.79 12.8-33.86 30.63-42.99l62-31.74c44.88-22.96 39.59-80.17-8.95-96.79z"],"spray-can":[512,512,[],"f5bd","M224 32c0-17.67-14.33-32-32-32h-64c-17.67 0-32 14.33-32 32v96h128V32zm256 96c-17.67 0-32 14.33-32 32s14.33 32 32 32 32-14.33 32-32-14.33-32-32-32zm-256 32H96c-53.02 0-96 42.98-96 96v224c0 17.67 14.33 32 32 32h256c17.67 0 32-14.33 32-32V256c0-53.02-42.98-96-96-96zm-64 256c-44.18 0-80-35.82-80-80s35.82-80 80-80 80 35.82 80 80-35.82 80-80 80zM480 96c17.67 0 32-14.33 32-32s-14.33-32-32-32-32 14.33-32 32 14.33 32 32 32zm-96 32c-17.67 0-32 14.33-32 32s14.33 32 32 32 32-14.33 32-32-14.33-32-32-32zm-96-96c-17.67 0-32 14.33-32 32s14.33 32 32 32 32-14.33 32-32-14.33-32-32-32zm96 0c-17.67 0-32 14.33-32 32s14.33 32 32 32 32-14.33 32-32-14.33-32-32-32zm96 192c-17.67 0-32 14.33-32 32s14.33 32 32 32 32-14.33 32-32-14.33-32-32-32z"],square:[448,512,[],"f0c8","M400 32H48C21.5 32 0 53.5 0 80v352c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V80c0-26.5-21.5-48-48-48z"],"square-full":[512,512,[],"f45c","M512 512H0V0h512v512z"],"square-root-alt":[576,512,[],"f698","M571.31 251.31l-22.62-22.62c-6.25-6.25-16.38-6.25-22.63 0L480 274.75l-46.06-46.06c-6.25-6.25-16.38-6.25-22.63 0l-22.62 22.62c-6.25 6.25-6.25 16.38 0 22.63L434.75 320l-46.06 46.06c-6.25 6.25-6.25 16.38 0 22.63l22.62 22.62c6.25 6.25 16.38 6.25 22.63 0L480 365.25l46.06 46.06c6.25 6.25 16.38 6.25 22.63 0l22.62-22.62c6.25-6.25 6.25-16.38 0-22.63L525.25 320l46.06-46.06c6.25-6.25 6.25-16.38 0-22.63zM552 0H307.65c-14.54 0-27.26 9.8-30.95 23.87l-84.79 322.8-58.41-106.1A32.008 32.008 0 0 0 105.47 224H24c-13.25 0-24 10.74-24 24v48c0 13.25 10.75 24 24 24h43.62l88.88 163.73C168.99 503.5 186.3 512 204.94 512c17.27 0 44.44-9 54.28-41.48L357.03 96H552c13.25 0 24-10.75 24-24V24c0-13.26-10.75-24-24-24z"],stamp:[512,512,[],"f5bf","M32 512h448v-64H32v64zm384-256h-66.56c-16.26 0-29.44-13.18-29.44-29.44v-9.46c0-27.37 8.88-53.41 21.46-77.72 9.11-17.61 12.9-38.39 9.05-60.42-6.77-38.78-38.47-70.7-77.26-77.45C212.62-9.04 160 37.33 160 96c0 14.16 3.12 27.54 8.69 39.58C182.02 164.43 192 194.7 192 226.49v.07c0 16.26-13.18 29.44-29.44 29.44H96c-53.02 0-96 42.98-96 96v32c0 17.67 14.33 32 32 32h448c17.67 0 32-14.33 32-32v-32c0-53.02-42.98-96-96-96z"],star:[576,512,[],"f005","M259.3 17.8L194 150.2 47.9 171.5c-26.2 3.8-36.7 36.1-17.7 54.6l105.7 103-25 145.5c-4.5 26.3 23.2 46 46.4 33.7L288 439.6l130.7 68.7c23.2 12.2 50.9-7.4 46.4-33.7l-25-145.5 105.7-103c19-18.5 8.5-50.8-17.7-54.6L382 150.2 316.7 17.8c-11.7-23.6-45.6-23.9-57.4 0z"],"star-and-crescent":[512,512,[],"f699","M340.47 466.36c-1.45 0-6.89.46-9.18.46-116.25 0-210.82-94.57-210.82-210.82S215.04 45.18 331.29 45.18c2.32 0 7.7.46 9.18.46 7.13 0 13.33-5.03 14.75-12.07 1.46-7.25-2.55-14.49-9.47-17.09C316.58 5.54 286.39 0 256 0 114.84 0 0 114.84 0 256s114.84 256 256 256c30.23 0 60.28-5.49 89.32-16.32 5.96-2.02 10.28-7.64 10.28-14.26 0-8.09-6.39-15.06-15.13-15.06zm162.99-252.5l-76.38-11.1-34.16-69.21c-1.83-3.7-5.38-5.55-8.93-5.55s-7.1 1.85-8.93 5.55l-34.16 69.21-76.38 11.1c-8.17 1.18-11.43 11.22-5.52 16.99l55.27 53.87-13.05 76.07c-1.11 6.44 4.01 11.66 9.81 11.66 1.53 0 3.11-.36 4.64-1.17L384 335.37l68.31 35.91c1.53.8 3.11 1.17 4.64 1.17 5.8 0 10.92-5.23 9.81-11.66l-13.05-76.07 55.27-53.87c5.91-5.77 2.65-15.81-5.52-16.99z"],"star-half":[576,512,[],"f089","M288 0c-11.4 0-22.8 5.9-28.7 17.8L194 150.2 47.9 171.4c-26.2 3.8-36.7 36.1-17.7 54.6l105.7 103-25 145.5c-4.5 26.1 23 46 46.4 33.7L288 439.6V0z"],"star-half-alt":[536,512,[],"f5c0","M508.55 171.51L362.18 150.2 296.77 17.81C290.89 5.98 279.42 0 267.95 0c-11.4 0-22.79 5.9-28.69 17.81l-65.43 132.38-146.38 21.29c-26.25 3.8-36.77 36.09-17.74 54.59l105.89 103-25.06 145.48C86.98 495.33 103.57 512 122.15 512c4.93 0 10-1.17 14.87-3.75l130.95-68.68 130.94 68.7c4.86 2.55 9.92 3.71 14.83 3.71 18.6 0 35.22-16.61 31.66-37.4l-25.03-145.49 105.91-102.98c19.04-18.5 8.52-50.8-17.73-54.6zm-121.74 123.2l-18.12 17.62 4.28 24.88 19.52 113.45-102.13-53.59-22.38-11.74.03-317.19 51.03 103.29 11.18 22.63 25.01 3.64 114.23 16.63-82.65 80.38z"],"star-of-david":[464,512,[],"f69a","M405.68 256l53.21-89.39C473.3 142.4 455.48 112 426.88 112H319.96l-55.95-93.98C256.86 6.01 244.43 0 232 0s-24.86 6.01-32.01 18.02L144.04 112H37.11c-28.6 0-46.42 30.4-32.01 54.61L58.32 256 5.1 345.39C-9.31 369.6 8.51 400 37.11 400h106.93l55.95 93.98C207.14 505.99 219.57 512 232 512s24.86-6.01 32.01-18.02L319.96 400h106.93c28.6 0 46.42-30.4 32.01-54.61L405.68 256zm-12.78-88l-19.8 33.26L353.3 168h39.6zm-52.39 88l-52.39 88H175.88l-52.39-88 52.38-88h112.25l52.39 88zM232 73.72L254.79 112h-45.57L232 73.72zM71.1 168h39.6l-19.8 33.26L71.1 168zm0 176l19.8-33.26L110.7 344H71.1zM232 438.28L209.21 400h45.57L232 438.28zM353.29 344l19.8-33.26L392.9 344h-39.61z"],"star-of-life":[480,512,[],"f621","M471.99 334.43L336.06 256l135.93-78.43c7.66-4.42 10.28-14.2 5.86-21.86l-32.02-55.43c-4.42-7.65-14.21-10.28-21.87-5.86l-135.93 78.43V16c0-8.84-7.17-16-16.01-16h-64.04c-8.84 0-16.01 7.16-16.01 16v156.86L56.04 94.43c-7.66-4.42-17.45-1.79-21.87 5.86L2.15 155.71c-4.42 7.65-1.8 17.44 5.86 21.86L143.94 256 8.01 334.43c-7.66 4.42-10.28 14.21-5.86 21.86l32.02 55.43c4.42 7.65 14.21 10.27 21.87 5.86l135.93-78.43V496c0 8.84 7.17 16 16.01 16h64.04c8.84 0 16.01-7.16 16.01-16V339.14l135.93 78.43c7.66 4.42 17.45 1.8 21.87-5.86l32.02-55.43c4.42-7.65 1.8-17.43-5.86-21.85z"],"step-backward":[448,512,[],"f048","M64 468V44c0-6.6 5.4-12 12-12h48c6.6 0 12 5.4 12 12v176.4l195.5-181C352.1 22.3 384 36.6 384 64v384c0 27.4-31.9 41.7-52.5 24.6L136 292.7V468c0 6.6-5.4 12-12 12H76c-6.6 0-12-5.4-12-12z"],"step-forward":[448,512,[],"f051","M384 44v424c0 6.6-5.4 12-12 12h-48c-6.6 0-12-5.4-12-12V291.6l-195.5 181C95.9 489.7 64 475.4 64 448V64c0-27.4 31.9-41.7 52.5-24.6L312 219.3V44c0-6.6 5.4-12 12-12h48c6.6 0 12 5.4 12 12z"],stethoscope:[512,512,[],"f0f1","M447.1 112c-34.2.5-62.3 28.4-63 62.6-.5 24.3 12.5 45.6 32 56.8V344c0 57.3-50.2 104-112 104-60 0-109.2-44.1-111.9-99.2C265 333.8 320 269.2 320 192V36.6c0-11.4-8.1-21.3-19.3-23.5L237.8.5c-13-2.6-25.6 5.8-28.2 18.8L206.4 35c-2.6 13 5.8 25.6 18.8 28.2l30.7 6.1v121.4c0 52.9-42.2 96.7-95.1 97.2-53.4.5-96.9-42.7-96.9-96V69.4l30.7-6.1c13-2.6 21.4-15.2 18.8-28.2l-3.1-15.7C107.7 6.4 95.1-2 82.1.6L19.3 13C8.1 15.3 0 25.1 0 36.6V192c0 77.3 55.1 142 128.1 156.8C130.7 439.2 208.6 512 304 512c97 0 176-75.4 176-168V231.4c19.1-11.1 32-31.7 32-55.4 0-35.7-29.2-64.5-64.9-64zm.9 80c-8.8 0-16-7.2-16-16s7.2-16 16-16 16 7.2 16 16-7.2 16-16 16z"],"sticky-note":[448,512,[],"f249","M312 320h136V56c0-13.3-10.7-24-24-24H24C10.7 32 0 42.7 0 56v400c0 13.3 10.7 24 24 24h264V344c0-13.2 10.8-24 24-24zm129 55l-98 98c-4.5 4.5-10.6 7-17 7h-6V352h128v6.1c0 6.3-2.5 12.4-7 16.9z"],stop:[448,512,[],"f04d","M400 32H48C21.5 32 0 53.5 0 80v352c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V80c0-26.5-21.5-48-48-48z"],"stop-circle":[512,512,[],"f28d","M256 8C119 8 8 119 8 256s111 248 248 248 248-111 248-248S393 8 256 8zm96 328c0 8.8-7.2 16-16 16H176c-8.8 0-16-7.2-16-16V176c0-8.8 7.2-16 16-16h160c8.8 0 16 7.2 16 16v160z"],stopwatch:[448,512,[],"f2f2","M432 304c0 114.9-93.1 208-208 208S16 418.9 16 304c0-104 76.3-190.2 176-205.5V64h-28c-6.6 0-12-5.4-12-12V12c0-6.6 5.4-12 12-12h120c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12h-28v34.5c37.5 5.8 71.7 21.6 99.7 44.6l27.5-27.5c4.7-4.7 12.3-4.7 17 0l28.3 28.3c4.7 4.7 4.7 12.3 0 17l-29.4 29.4-.6.6C419.7 223.3 432 262.2 432 304zm-176 36V188.5c0-6.6-5.4-12-12-12h-40c-6.6 0-12 5.4-12 12V340c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12z"],"stopwatch-20":[448,512,[],"f96f","M398.5,190.91l.59-.61,26.59-26.58a16,16,0,0,0,0-22.63L403,118.41a16,16,0,0,0-22.63,0l-24.68,24.68A206.68,206.68,0,0,0,256,98.5V64h32a16,16,0,0,0,16-16V16A16,16,0,0,0,288,0H160a16.05,16.05,0,0,0-16,16V48a16.05,16.05,0,0,0,16,16h32V98.5A207.92,207.92,0,0,0,16.09,297.57C12.64,411.5,106.76,510.22,220.72,512,337.13,513.77,432,420,432,304A206,206,0,0,0,398.5,190.91ZM204.37,377.55a8.2,8.2,0,0,1,8.32,8.07v22.31a8.2,8.2,0,0,1-8.32,8.07H121.52a16.46,16.46,0,0,1-16.61-17.62c2.78-35.22,14.67-57.41,38.45-91.37,20.42-29.19,27.1-37.32,27.1-62.34,0-16.92-1.79-24.27-12.21-24.27-9.39,0-12.69,7.4-12.69,22.68v5.23a8.2,8.2,0,0,1-8.33,8.07h-24.9a8.2,8.2,0,0,1-8.33-8.07v-4.07c0-27.3,8.48-60.24,56.43-60.24,43,0,55.57,25.85,55.57,61,0,35.58-12.44,51.21-34.35,81.31-11.56,15-24.61,35.57-26.41,51.2ZM344,352.32c0,35.16-12.3,63.68-57.23,63.68C243.19,416,232,386.48,232,352.55V247.22c0-40.73,19.58-63.22,56.2-63.22C325,184,344,206.64,344,245.3ZM287.87,221.73c-9.41,0-13.23,7.5-13.23,20V357.68c0,13.11,3.59,20.59,13.23,20.59s13-8,13-21.27V241.06C300.89,229.79,297.88,221.73,287.87,221.73Z"],store:[616,512,[],"f54e","M602 118.6L537.1 15C531.3 5.7 521 0 510 0H106C95 0 84.7 5.7 78.9 15L14 118.6c-33.5 53.5-3.8 127.9 58.8 136.4 4.5.6 9.1.9 13.7.9 29.6 0 55.8-13 73.8-33.1 18 20.1 44.3 33.1 73.8 33.1 29.6 0 55.8-13 73.8-33.1 18 20.1 44.3 33.1 73.8 33.1 29.6 0 55.8-13 73.8-33.1 18.1 20.1 44.3 33.1 73.8 33.1 4.7 0 9.2-.3 13.7-.9 62.8-8.4 92.6-82.8 59-136.4zM529.5 288c-10 0-19.9-1.5-29.5-3.8V384H116v-99.8c-9.6 2.2-19.5 3.8-29.5 3.8-6 0-12.1-.4-18-1.2-5.6-.8-11.1-2.1-16.4-3.6V480c0 17.7 14.3 32 32 32h448c17.7 0 32-14.3 32-32V283.2c-5.4 1.6-10.8 2.9-16.4 3.6-6.1.8-12.1 1.2-18.2 1.2z"],"store-alt":[640,512,[],"f54f","M320 384H128V224H64v256c0 17.7 14.3 32 32 32h256c17.7 0 32-14.3 32-32V224h-64v160zm314.6-241.8l-85.3-128c-6-8.9-16-14.2-26.7-14.2H117.4c-10.7 0-20.7 5.3-26.6 14.2l-85.3 128c-14.2 21.3 1 49.8 26.6 49.8H608c25.5 0 40.7-28.5 26.6-49.8zM512 496c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16V224h-64v272z"],"store-alt-slash":[640,512,[],"f970","M17.89,123.62,5.51,142.2c-14.2,21.3,1,49.8,26.59,49.8h74.26ZM576,413.42V224H512V364L384,265V224H330.92l-41.4-32H608c25.5,0,40.7-28.5,26.59-49.8l-85.29-128A32.18,32.18,0,0,0,522.6,0H117.42A31.87,31.87,0,0,0,90.81,14.2l-10.66,16L45.46,3.38A16,16,0,0,0,23,6.19L3.37,31.46A16,16,0,0,0,6.18,53.91L594.53,508.63A16,16,0,0,0,617,505.81l19.64-25.26a16,16,0,0,0-2.81-22.45ZM320,384H128V224H64V480a32,32,0,0,0,32,32H352a32,32,0,0,0,32-32V406.59l-64-49.47Z"],"store-slash":[640,512,[],"f971","M121.51,384V284.2a119.43,119.43,0,0,1-28,3.8,123.46,123.46,0,0,1-17.1-1.2,114.88,114.88,0,0,1-15.58-3.6V480c0,17.7,13.59,32,30.4,32H505.75L348.42,384Zm-28-128.09c25.1,0,47.29-10.72,64-27.24L24,120.05c-30.52,53.39-2.45,126.53,56.49,135A95.68,95.68,0,0,0,93.48,255.91ZM602.13,458.09,547.2,413.41V283.2a93.5,93.5,0,0,1-15.57,3.6,127.31,127.31,0,0,1-17.29,1.2,114.89,114.89,0,0,1-28-3.8v79.68L348.52,251.77a88.06,88.06,0,0,0,25.41,4.14c28.11,0,53-13,70.11-33.11,17.19,20.11,42.08,33.11,70.11,33.11a94.31,94.31,0,0,0,13-.91c59.66-8.41,88-82.8,56.06-136.4L521.55,15A30.1,30.1,0,0,0,495.81,0H112A30.11,30.11,0,0,0,86.27,15L76.88,30.78,43.19,3.38A14.68,14.68,0,0,0,21.86,6.19L3.2,31.45A16.58,16.58,0,0,0,5.87,53.91L564.81,508.63a14.69,14.69,0,0,0,21.33-2.82l18.66-25.26A16.58,16.58,0,0,0,602.13,458.09Z"],stream:[512,512,[],"f550","M16 128h416c8.84 0 16-7.16 16-16V48c0-8.84-7.16-16-16-16H16C7.16 32 0 39.16 0 48v64c0 8.84 7.16 16 16 16zm480 80H80c-8.84 0-16 7.16-16 16v64c0 8.84 7.16 16 16 16h416c8.84 0 16-7.16 16-16v-64c0-8.84-7.16-16-16-16zm-64 176H16c-8.84 0-16 7.16-16 16v64c0 8.84 7.16 16 16 16h416c8.84 0 16-7.16 16-16v-64c0-8.84-7.16-16-16-16z"],"street-view":[512,512,[],"f21d","M367.9 329.76c-4.62 5.3-9.78 10.1-15.9 13.65v22.94c66.52 9.34 112 28.05 112 49.65 0 30.93-93.12 56-208 56S48 446.93 48 416c0-21.6 45.48-40.3 112-49.65v-22.94c-6.12-3.55-11.28-8.35-15.9-13.65C58.87 345.34 0 378.05 0 416c0 53.02 114.62 96 256 96s256-42.98 256-96c0-37.95-58.87-70.66-144.1-86.24zM256 128c35.35 0 64-28.65 64-64S291.35 0 256 0s-64 28.65-64 64 28.65 64 64 64zm-64 192v96c0 17.67 14.33 32 32 32h64c17.67 0 32-14.33 32-32v-96c17.67 0 32-14.33 32-32v-96c0-26.51-21.49-48-48-48h-11.8c-11.07 5.03-23.26 8-36.2 8s-25.13-2.97-36.2-8H208c-26.51 0-48 21.49-48 48v96c0 17.67 14.33 32 32 32z"],strikethrough:[512,512,[],"f0cc","M496 224H293.9l-87.17-26.83A43.55 43.55 0 0 1 219.55 112h66.79A49.89 49.89 0 0 1 331 139.58a16 16 0 0 0 21.46 7.15l42.94-21.47a16 16 0 0 0 7.16-21.46l-.53-1A128 128 0 0 0 287.51 32h-68a123.68 123.68 0 0 0-123 135.64c2 20.89 10.1 39.83 21.78 56.36H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h480a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm-180.24 96A43 43 0 0 1 336 356.45 43.59 43.59 0 0 1 292.45 400h-66.79A49.89 49.89 0 0 1 181 372.42a16 16 0 0 0-21.46-7.15l-42.94 21.47a16 16 0 0 0-7.16 21.46l.53 1A128 128 0 0 0 224.49 480h68a123.68 123.68 0 0 0 123-135.64 114.25 114.25 0 0 0-5.34-24.36z"],stroopwafel:[512,512,[],"f551","M188.12 210.74L142.86 256l45.25 45.25L233.37 256l-45.25-45.26zm113.13-22.62L256 142.86l-45.25 45.25L256 233.37l45.25-45.25zm-90.5 135.76L256 369.14l45.26-45.26L256 278.63l-45.25 45.25zM256 0C114.62 0 0 114.62 0 256s114.62 256 256 256 256-114.62 256-256S397.38 0 256 0zm186.68 295.6l-11.31 11.31c-3.12 3.12-8.19 3.12-11.31 0l-28.29-28.29-45.25 45.25 33.94 33.94 16.97-16.97c3.12-3.12 8.19-3.12 11.31 0l11.31 11.31c3.12 3.12 3.12 8.19 0 11.31l-16.97 16.97 16.97 16.97c3.12 3.12 3.12 8.19 0 11.31l-11.31 11.31c-3.12 3.12-8.19 3.12-11.31 0l-16.97-16.97-16.97 16.97c-3.12 3.12-8.19 3.12-11.31 0l-11.31-11.31c-3.12-3.12-3.12-8.19 0-11.31l16.97-16.97-33.94-33.94-45.26 45.26 28.29 28.29c3.12 3.12 3.12 8.19 0 11.31l-11.31 11.31c-3.12 3.12-8.19 3.12-11.31 0L256 414.39l-28.29 28.29c-3.12 3.12-8.19 3.12-11.31 0l-11.31-11.31c-3.12-3.12-3.12-8.19 0-11.31l28.29-28.29-45.25-45.26-33.94 33.94 16.97 16.97c3.12 3.12 3.12 8.19 0 11.31l-11.31 11.31c-3.12 3.12-8.19 3.12-11.31 0l-16.97-16.97-16.97 16.97c-3.12 3.12-8.19 3.12-11.31 0l-11.31-11.31c-3.12-3.12-3.12-8.19 0-11.31l16.97-16.97-16.97-16.97c-3.12-3.12-3.12-8.19 0-11.31l11.31-11.31c3.12-3.12 8.19-3.12 11.31 0l16.97 16.97 33.94-33.94-45.25-45.25-28.29 28.29c-3.12 3.12-8.19 3.12-11.31 0L69.32 295.6c-3.12-3.12-3.12-8.19 0-11.31L97.61 256l-28.29-28.29c-3.12-3.12-3.12-8.19 0-11.31l11.31-11.31c3.12-3.12 8.19-3.12 11.31 0l28.29 28.29 45.25-45.26-33.94-33.94-16.97 16.97c-3.12 3.12-8.19 3.12-11.31 0l-11.31-11.31c-3.12-3.12-3.12-8.19 0-11.31l16.97-16.97-16.97-16.97c-3.12-3.12-3.12-8.19 0-11.31l11.31-11.31c3.12-3.12 8.19-3.12 11.31 0l16.97 16.97 16.97-16.97c3.12-3.12 8.19-3.12 11.31 0l11.31 11.31c3.12 3.12 3.12 8.19 0 11.31l-16.97 16.97 33.94 33.94 45.26-45.25-28.29-28.29c-3.12-3.12-3.12-8.19 0-11.31l11.31-11.31c3.12-3.12 8.19-3.12 11.31 0L256 97.61l28.29-28.29c3.12-3.12 8.19-3.12 11.31 0l11.31 11.31c3.12 3.12 3.12 8.19 0 11.31l-28.29 28.29 45.26 45.25 33.94-33.94-16.97-16.97c-3.12-3.12-3.12-8.19 0-11.31l11.31-11.31c3.12-3.12 8.19-3.12 11.31 0l16.97 16.97 16.97-16.97c3.12-3.12 8.19-3.12 11.31 0l11.31 11.31c3.12 3.12 3.12 8.19 0 11.31l-16.97 16.97 16.97 16.97c3.12 3.12 3.12 8.19 0 11.31l-11.31 11.31c-3.12 3.12-8.19 3.12-11.31 0l-16.97-16.97-33.94 33.94 45.25 45.26 28.29-28.29c3.12-3.12 8.19-3.12 11.31 0l11.31 11.31c3.12 3.12 3.12 8.19 0 11.31L414.39 256l28.29 28.28a8.015 8.015 0 0 1 0 11.32zM278.63 256l45.26 45.25L369.14 256l-45.25-45.26L278.63 256z"],subscript:[512,512,[],"f12c","M496 448h-16V304a16 16 0 0 0-16-16h-48a16 16 0 0 0-14.29 8.83l-16 32A16 16 0 0 0 400 352h16v96h-16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h96a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zM336 64h-67a16 16 0 0 0-13.14 6.87l-79.9 115-79.9-115A16 16 0 0 0 83 64H16A16 16 0 0 0 0 80v48a16 16 0 0 0 16 16h33.48l77.81 112-77.81 112H16a16 16 0 0 0-16 16v48a16 16 0 0 0 16 16h67a16 16 0 0 0 13.14-6.87l79.9-115 79.9 115A16 16 0 0 0 269 448h67a16 16 0 0 0 16-16v-48a16 16 0 0 0-16-16h-33.48l-77.81-112 77.81-112H336a16 16 0 0 0 16-16V80a16 16 0 0 0-16-16z"],subway:[448,512,[],"f239","M448 96v256c0 51.815-61.624 96-130.022 96l62.98 49.721C386.905 502.417 383.562 512 376 512H72c-7.578 0-10.892-9.594-4.957-14.279L130.022 448C61.82 448 0 403.954 0 352V96C0 42.981 64 0 128 0h192c65 0 128 42.981 128 96zM200 232V120c0-13.255-10.745-24-24-24H72c-13.255 0-24 10.745-24 24v112c0 13.255 10.745 24 24 24h104c13.255 0 24-10.745 24-24zm200 0V120c0-13.255-10.745-24-24-24H272c-13.255 0-24 10.745-24 24v112c0 13.255 10.745 24 24 24h104c13.255 0 24-10.745 24-24zm-48 56c-26.51 0-48 21.49-48 48s21.49 48 48 48 48-21.49 48-48-21.49-48-48-48zm-256 0c-26.51 0-48 21.49-48 48s21.49 48 48 48 48-21.49 48-48-21.49-48-48-48z"],suitcase:[512,512,[],"f0f2","M128 480h256V80c0-26.5-21.5-48-48-48H176c-26.5 0-48 21.5-48 48v400zm64-384h128v32H192V96zm320 80v256c0 26.5-21.5 48-48 48h-48V128h48c26.5 0 48 21.5 48 48zM96 480H48c-26.5 0-48-21.5-48-48V176c0-26.5 21.5-48 48-48h48v352z"],"suitcase-rolling":[384,512,[],"f5c1","M336 160H48c-26.51 0-48 21.49-48 48v224c0 26.51 21.49 48 48 48h16v16c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-16h128v16c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-16h16c26.51 0 48-21.49 48-48V208c0-26.51-21.49-48-48-48zm-16 216c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h240c4.42 0 8 3.58 8 8v16zm0-96c0 4.42-3.58 8-8 8H72c-4.42 0-8-3.58-8-8v-16c0-4.42 3.58-8 8-8h240c4.42 0 8 3.58 8 8v16zM144 48h96v80h48V48c0-26.51-21.49-48-48-48h-96c-26.51 0-48 21.49-48 48v80h48V48z"],sun:[512,512,[],"f185","M256 160c-52.9 0-96 43.1-96 96s43.1 96 96 96 96-43.1 96-96-43.1-96-96-96zm246.4 80.5l-94.7-47.3 33.5-100.4c4.5-13.6-8.4-26.5-21.9-21.9l-100.4 33.5-47.4-94.8c-6.4-12.8-24.6-12.8-31 0l-47.3 94.7L92.7 70.8c-13.6-4.5-26.5 8.4-21.9 21.9l33.5 100.4-94.7 47.4c-12.8 6.4-12.8 24.6 0 31l94.7 47.3-33.5 100.5c-4.5 13.6 8.4 26.5 21.9 21.9l100.4-33.5 47.3 94.7c6.4 12.8 24.6 12.8 31 0l47.3-94.7 100.4 33.5c13.6 4.5 26.5-8.4 21.9-21.9l-33.5-100.4 94.7-47.3c13-6.5 13-24.7.2-31.1zm-155.9 106c-49.9 49.9-131.1 49.9-181 0-49.9-49.9-49.9-131.1 0-181 49.9-49.9 131.1-49.9 181 0 49.9 49.9 49.9 131.1 0 181z"],superscript:[512,512,[],"f12b","M496 160h-16V16a16 16 0 0 0-16-16h-48a16 16 0 0 0-14.29 8.83l-16 32A16 16 0 0 0 400 64h16v96h-16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h96a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zM336 64h-67a16 16 0 0 0-13.14 6.87l-79.9 115-79.9-115A16 16 0 0 0 83 64H16A16 16 0 0 0 0 80v48a16 16 0 0 0 16 16h33.48l77.81 112-77.81 112H16a16 16 0 0 0-16 16v48a16 16 0 0 0 16 16h67a16 16 0 0 0 13.14-6.87l79.9-115 79.9 115A16 16 0 0 0 269 448h67a16 16 0 0 0 16-16v-48a16 16 0 0 0-16-16h-33.48l-77.81-112 77.81-112H336a16 16 0 0 0 16-16V80a16 16 0 0 0-16-16z"],surprise:[496,512,[],"f5c2","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zM136 208c0-17.7 14.3-32 32-32s32 14.3 32 32-14.3 32-32 32-32-14.3-32-32zm112 208c-35.3 0-64-28.7-64-64s28.7-64 64-64 64 28.7 64 64-28.7 64-64 64zm80-176c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],swatchbook:[512,512,[],"f5c3","M434.66,167.71h0L344.5,77.36a31.83,31.83,0,0,0-45-.07h0l-.07.07L224,152.88V424L434.66,212.9A32,32,0,0,0,434.66,167.71ZM480,320H373.09L186.68,506.51c-2.06,2.07-4.5,3.58-6.68,5.49H480a32,32,0,0,0,32-32V352A32,32,0,0,0,480,320ZM192,32A32,32,0,0,0,160,0H32A32,32,0,0,0,0,32V416a96,96,0,0,0,192,0ZM96,440a24,24,0,1,1,24-24A24,24,0,0,1,96,440Zm32-184H64V192h64Zm0-128H64V64h64Z"],swimmer:[640,512,[],"f5c4","M189.61 310.58c3.54 3.26 15.27 9.42 34.39 9.42s30.86-6.16 34.39-9.42c16.02-14.77 34.5-22.58 53.46-22.58h16.3c18.96 0 37.45 7.81 53.46 22.58 3.54 3.26 15.27 9.42 34.39 9.42s30.86-6.16 34.39-9.42c14.86-13.71 31.88-21.12 49.39-22.16l-112.84-80.6 18-12.86c3.64-2.58 8.28-3.52 12.62-2.61l100.35 21.53c25.91 5.53 51.44-10.97 57-36.88 5.55-25.92-10.95-51.44-36.88-57L437.68 98.47c-30.73-6.58-63.02.12-88.56 18.38l-80.02 57.17c-10.38 7.39-19.36 16.44-26.72 26.94L173.75 299c5.47 3.23 10.82 6.93 15.86 11.58zM624 352h-16c-26.04 0-45.8-8.42-56.09-17.9-8.9-8.21-19.66-14.1-31.77-14.1h-16.3c-12.11 0-22.87 5.89-31.77 14.1C461.8 343.58 442.04 352 416 352s-45.8-8.42-56.09-17.9c-8.9-8.21-19.66-14.1-31.77-14.1h-16.3c-12.11 0-22.87 5.89-31.77 14.1C269.8 343.58 250.04 352 224 352s-45.8-8.42-56.09-17.9c-8.9-8.21-19.66-14.1-31.77-14.1h-16.3c-12.11 0-22.87 5.89-31.77 14.1C77.8 343.58 58.04 352 32 352H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h16c38.62 0 72.72-12.19 96-31.84 23.28 19.66 57.38 31.84 96 31.84s72.72-12.19 96-31.84c23.28 19.66 57.38 31.84 96 31.84s72.72-12.19 96-31.84c23.28 19.66 57.38 31.84 96 31.84h16c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zm-512-96c44.18 0 80-35.82 80-80s-35.82-80-80-80-80 35.82-80 80 35.82 80 80 80z"],"swimming-pool":[640,512,[],"f5c5","M624 416h-16c-26.04 0-45.8-8.42-56.09-17.9-8.9-8.21-19.66-14.1-31.77-14.1h-16.3c-12.11 0-22.87 5.89-31.77 14.1C461.8 407.58 442.04 416 416 416s-45.8-8.42-56.09-17.9c-8.9-8.21-19.66-14.1-31.77-14.1h-16.3c-12.11 0-22.87 5.89-31.77 14.1C269.8 407.58 250.04 416 224 416s-45.8-8.42-56.09-17.9c-8.9-8.21-19.66-14.1-31.77-14.1h-16.3c-12.11 0-22.87 5.89-31.77 14.1C77.8 407.58 58.04 416 32 416H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h16c38.62 0 72.72-12.19 96-31.84 23.28 19.66 57.38 31.84 96 31.84s72.72-12.19 96-31.84c23.28 19.66 57.38 31.84 96 31.84s72.72-12.19 96-31.84c23.28 19.66 57.38 31.84 96 31.84h16c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zm-400-32v-96h192v96c19.12 0 30.86-6.16 34.39-9.42 9.17-8.46 19.2-14.34 29.61-18.07V128c0-17.64 14.36-32 32-32s32 14.36 32 32v16c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-16c0-52.94-43.06-96-96-96s-96 43.06-96 96v96H224v-96c0-17.64 14.36-32 32-32s32 14.36 32 32v16c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-16c0-52.94-43.06-96-96-96s-96 43.06-96 96v228.5c10.41 3.73 20.44 9.62 29.61 18.07 3.53 3.27 15.27 9.43 34.39 9.43z"],synagogue:[640,512,[],"f69b","M70 196.51L6.67 268.29A26.643 26.643 0 0 0 0 285.93V512h128V239.58l-38-43.07c-5.31-6.01-14.69-6.01-20 0zm563.33 71.78L570 196.51c-5.31-6.02-14.69-6.02-20 0l-38 43.07V512h128V285.93c0-6.5-2.37-12.77-6.67-17.64zM339.99 7.01c-11.69-9.35-28.29-9.35-39.98 0l-128 102.4A32.005 32.005 0 0 0 160 134.4V512h96v-92.57c0-31.88 21.78-61.43 53.25-66.55C349.34 346.35 384 377.13 384 416v96h96V134.4c0-9.72-4.42-18.92-12.01-24.99l-128-102.4zm52.07 215.55c1.98 3.15-.29 7.24-4 7.24h-38.94L324 269.79c-1.85 2.95-6.15 2.95-8 0l-25.12-39.98h-38.94c-3.72 0-5.98-4.09-4-7.24l19.2-30.56-19.2-30.56c-1.98-3.15.29-7.24 4-7.24h38.94l25.12-40c1.85-2.95 6.15-2.95 8 0l25.12 39.98h38.95c3.71 0 5.98 4.09 4 7.24L372.87 192l19.19 30.56z"],sync:[512,512,[],"f021","M440.65 12.57l4 82.77A247.16 247.16 0 0 0 255.83 8C134.73 8 33.91 94.92 12.29 209.82A12 12 0 0 0 24.09 224h49.05a12 12 0 0 0 11.67-9.26 175.91 175.91 0 0 1 317-56.94l-101.46-4.86a12 12 0 0 0-12.57 12v47.41a12 12 0 0 0 12 12H500a12 12 0 0 0 12-12V12a12 12 0 0 0-12-12h-47.37a12 12 0 0 0-11.98 12.57zM255.83 432a175.61 175.61 0 0 1-146-77.8l101.8 4.87a12 12 0 0 0 12.57-12v-47.4a12 12 0 0 0-12-12H12a12 12 0 0 0-12 12V500a12 12 0 0 0 12 12h47.35a12 12 0 0 0 12-12.6l-4.15-82.57A247.17 247.17 0 0 0 255.83 504c121.11 0 221.93-86.92 243.55-201.82a12 12 0 0 0-11.8-14.18h-49.05a12 12 0 0 0-11.67 9.26A175.86 175.86 0 0 1 255.83 432z"],"sync-alt":[512,512,[],"f2f1","M370.72 133.28C339.458 104.008 298.888 87.962 255.848 88c-77.458.068-144.328 53.178-162.791 126.85-1.344 5.363-6.122 9.15-11.651 9.15H24.103c-7.498 0-13.194-6.807-11.807-14.176C33.933 94.924 134.813 8 256 8c66.448 0 126.791 26.136 171.315 68.685L463.03 40.97C478.149 25.851 504 36.559 504 57.941V192c0 13.255-10.745 24-24 24H345.941c-21.382 0-32.09-25.851-16.971-40.971l41.75-41.749zM32 296h134.059c21.382 0 32.09 25.851 16.971 40.971l-41.75 41.75c31.262 29.273 71.835 45.319 114.876 45.28 77.418-.07 144.315-53.144 162.787-126.849 1.344-5.363 6.122-9.15 11.651-9.15h57.304c7.498 0 13.194 6.807 11.807 14.176C478.067 417.076 377.187 504 256 504c-66.448 0-126.791-26.136-171.315-68.685L48.97 471.03C33.851 486.149 8 475.441 8 454.059V320c0-13.255 10.745-24 24-24z"],syringe:[512,512,[],"f48e","M201.5 174.8l55.7 55.8c3.1 3.1 3.1 8.2 0 11.3l-11.3 11.3c-3.1 3.1-8.2 3.1-11.3 0l-55.7-55.8-45.3 45.3 55.8 55.8c3.1 3.1 3.1 8.2 0 11.3l-11.3 11.3c-3.1 3.1-8.2 3.1-11.3 0L111 265.2l-26.4 26.4c-17.3 17.3-25.6 41.1-23 65.4l7.1 63.6L2.3 487c-3.1 3.1-3.1 8.2 0 11.3l11.3 11.3c3.1 3.1 8.2 3.1 11.3 0l66.3-66.3 63.6 7.1c23.9 2.6 47.9-5.4 65.4-23l181.9-181.9-135.7-135.7-64.9 65zm308.2-93.3L430.5 2.3c-3.1-3.1-8.2-3.1-11.3 0l-11.3 11.3c-3.1 3.1-3.1 8.2 0 11.3l28.3 28.3-45.3 45.3-56.6-56.6-17-17c-3.1-3.1-8.2-3.1-11.3 0l-33.9 33.9c-3.1 3.1-3.1 8.2 0 11.3l17 17L424.8 223l17 17c3.1 3.1 8.2 3.1 11.3 0l33.9-34c3.1-3.1 3.1-8.2 0-11.3l-73.5-73.5 45.3-45.3 28.3 28.3c3.1 3.1 8.2 3.1 11.3 0l11.3-11.3c3.1-3.2 3.1-8.2 0-11.4z"],table:[512,512,[],"f0ce","M464 32H48C21.49 32 0 53.49 0 80v352c0 26.51 21.49 48 48 48h416c26.51 0 48-21.49 48-48V80c0-26.51-21.49-48-48-48zM224 416H64v-96h160v96zm0-160H64v-96h160v96zm224 160H288v-96h160v96zm0-160H288v-96h160v96z"],"table-tennis":[512,512,[],"f45d","M496.2 296.5C527.7 218.7 512 126.2 449 63.1 365.1-21 229-21 145.1 63.1l-56 56.1 211.5 211.5c46.1-62.1 131.5-77.4 195.6-34.2zm-217.9 79.7L57.9 155.9c-27.3 45.3-21.7 105 17.3 144.1l34.5 34.6L6.7 424c-8.6 7.5-9.1 20.7-1 28.8l53.4 53.5c8 8.1 21.2 7.6 28.7-1L177.1 402l35.7 35.7c19.7 19.7 44.6 30.5 70.3 33.3-7.1-17-11-35.6-11-55.1-.1-13.8 2.5-27 6.2-39.7zM416 320c-53 0-96 43-96 96s43 96 96 96 96-43 96-96-43-96-96-96z"],tablet:[448,512,[],"f10a","M400 0H48C21.5 0 0 21.5 0 48v416c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V48c0-26.5-21.5-48-48-48zM224 480c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z"],"tablet-alt":[448,512,[],"f3fa","M400 0H48C21.5 0 0 21.5 0 48v416c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V48c0-26.5-21.5-48-48-48zM224 480c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm176-108c0 6.6-5.4 12-12 12H60c-6.6 0-12-5.4-12-12V60c0-6.6 5.4-12 12-12h328c6.6 0 12 5.4 12 12v312z"],tablets:[640,512,[],"f490","M160 192C78.9 192 12.5 250.5.1 326.7c-.8 4.8 3.3 9.3 8.3 9.3h303.3c5 0 9.1-4.5 8.3-9.3C307.5 250.5 241.1 192 160 192zm151.6 176H8.4c-5 0-9.1 4.5-8.3 9.3C12.5 453.5 78.9 512 160 512s147.5-58.5 159.9-134.7c.8-4.8-3.3-9.3-8.3-9.3zM593.4 46.6c-56.5-56.5-144.2-61.4-206.9-16-4 2.9-4.3 8.9-.8 12.3L597 254.3c3.5 3.5 9.5 3.2 12.3-.8 45.5-62.7 40.6-150.4-15.9-206.9zM363 65.7c-3.5-3.5-9.5-3.2-12.3.8-45.4 62.7-40.5 150.4 15.9 206.9 56.5 56.5 144.2 61.4 206.9 15.9 4-2.9 4.3-8.9.8-12.3L363 65.7z"],"tachometer-alt":[576,512,[],"f3fd","M288 32C128.94 32 0 160.94 0 320c0 52.8 14.25 102.26 39.06 144.8 5.61 9.62 16.3 15.2 27.44 15.2h443c11.14 0 21.83-5.58 27.44-15.2C561.75 422.26 576 372.8 576 320c0-159.06-128.94-288-288-288zm0 64c14.71 0 26.58 10.13 30.32 23.65-1.11 2.26-2.64 4.23-3.45 6.67l-9.22 27.67c-5.13 3.49-10.97 6.01-17.64 6.01-17.67 0-32-14.33-32-32S270.33 96 288 96zM96 384c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm48-160c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm246.77-72.41l-61.33 184C343.13 347.33 352 364.54 352 384c0 11.72-3.38 22.55-8.88 32H232.88c-5.5-9.45-8.88-20.28-8.88-32 0-33.94 26.5-61.43 59.9-63.59l61.34-184.01c4.17-12.56 17.73-19.45 30.36-15.17 12.57 4.19 19.35 17.79 15.17 30.36zm14.66 57.2l15.52-46.55c3.47-1.29 7.13-2.23 11.05-2.23 17.67 0 32 14.33 32 32s-14.33 32-32 32c-11.38-.01-20.89-6.28-26.57-15.22zM480 384c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],tag:[512,512,[],"f02b","M0 252.118V48C0 21.49 21.49 0 48 0h204.118a48 48 0 0 1 33.941 14.059l211.882 211.882c18.745 18.745 18.745 49.137 0 67.882L293.823 497.941c-18.745 18.745-49.137 18.745-67.882 0L14.059 286.059A48 48 0 0 1 0 252.118zM112 64c-26.51 0-48 21.49-48 48s21.49 48 48 48 48-21.49 48-48-21.49-48-48-48z"],tags:[640,512,[],"f02c","M497.941 225.941L286.059 14.059A48 48 0 0 0 252.118 0H48C21.49 0 0 21.49 0 48v204.118a48 48 0 0 0 14.059 33.941l211.882 211.882c18.744 18.745 49.136 18.746 67.882 0l204.118-204.118c18.745-18.745 18.745-49.137 0-67.882zM112 160c-26.51 0-48-21.49-48-48s21.49-48 48-48 48 21.49 48 48-21.49 48-48 48zm513.941 133.823L421.823 497.941c-18.745 18.745-49.137 18.745-67.882 0l-.36-.36L527.64 323.522c16.999-16.999 26.36-39.6 26.36-63.64s-9.362-46.641-26.36-63.64L331.397 0h48.721a48 48 0 0 1 33.941 14.059l211.882 211.882c18.745 18.745 18.745 49.137 0 67.882z"],tape:[640,512,[],"f4db","M224 192c-35.3 0-64 28.7-64 64s28.7 64 64 64 64-28.7 64-64-28.7-64-64-64zm400 224H380.6c41.5-40.7 67.4-97.3 67.4-160 0-123.7-100.3-224-224-224S0 132.3 0 256s100.3 224 224 224h400c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zm-400-64c-53 0-96-43-96-96s43-96 96-96 96 43 96 96-43 96-96 96z"],tasks:[512,512,[],"f0ae","M139.61 35.5a12 12 0 0 0-17 0L58.93 98.81l-22.7-22.12a12 12 0 0 0-17 0L3.53 92.41a12 12 0 0 0 0 17l47.59 47.4a12.78 12.78 0 0 0 17.61 0l15.59-15.62L156.52 69a12.09 12.09 0 0 0 .09-17zm0 159.19a12 12 0 0 0-17 0l-63.68 63.72-22.7-22.1a12 12 0 0 0-17 0L3.53 252a12 12 0 0 0 0 17L51 316.5a12.77 12.77 0 0 0 17.6 0l15.7-15.69 72.2-72.22a12 12 0 0 0 .09-16.9zM64 368c-26.49 0-48.59 21.5-48.59 48S37.53 464 64 464a48 48 0 0 0 0-96zm432 16H208a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h288a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-320H208a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h288a16 16 0 0 0 16-16V80a16 16 0 0 0-16-16zm0 160H208a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h288a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"],taxi:[512,512,[],"f1ba","M462 241.64l-22-84.84c-9.6-35.2-41.6-60.8-76.8-60.8H352V64c0-17.67-14.33-32-32-32H192c-17.67 0-32 14.33-32 32v32h-11.2c-35.2 0-67.2 25.6-76.8 60.8l-22 84.84C21.41 248.04 0 273.47 0 304v48c0 23.63 12.95 44.04 32 55.12V448c0 17.67 14.33 32 32 32h32c17.67 0 32-14.33 32-32v-32h256v32c0 17.67 14.33 32 32 32h32c17.67 0 32-14.33 32-32v-40.88c19.05-11.09 32-31.5 32-55.12v-48c0-30.53-21.41-55.96-50-62.36zM96 352c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm20.55-112l17.2-66.36c2.23-8.16 9.59-13.64 15.06-13.64h214.4c5.47 0 12.83 5.48 14.85 12.86L395.45 240h-278.9zM416 352c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],teeth:[640,512,[],"f62e","M544 0H96C42.98 0 0 42.98 0 96v320c0 53.02 42.98 96 96 96h448c53.02 0 96-42.98 96-96V96c0-53.02-42.98-96-96-96zM160 368c0 26.51-21.49 48-48 48s-48-21.49-48-48v-64c0-8.84 7.16-16 16-16h64c8.84 0 16 7.16 16 16v64zm0-128c0 8.84-7.16 16-16 16H80c-8.84 0-16-7.16-16-16v-64c0-26.51 21.49-48 48-48s48 21.49 48 48v64zm144 120c0 30.93-25.07 56-56 56s-56-25.07-56-56v-56c0-8.84 7.16-16 16-16h80c8.84 0 16 7.16 16 16v56zm0-120c0 8.84-7.16 16-16 16h-80c-8.84 0-16-7.16-16-16v-88c0-30.93 25.07-56 56-56s56 25.07 56 56v88zm144 120c0 30.93-25.07 56-56 56s-56-25.07-56-56v-56c0-8.84 7.16-16 16-16h80c8.84 0 16 7.16 16 16v56zm0-120c0 8.84-7.16 16-16 16h-80c-8.84 0-16-7.16-16-16v-88c0-30.93 25.07-56 56-56s56 25.07 56 56v88zm128 128c0 26.51-21.49 48-48 48s-48-21.49-48-48v-64c0-8.84 7.16-16 16-16h64c8.84 0 16 7.16 16 16v64zm0-128c0 8.84-7.16 16-16 16h-64c-8.84 0-16-7.16-16-16v-64c0-26.51 21.49-48 48-48s48 21.49 48 48v64z"],"teeth-open":[640,512,[],"f62f","M544 0H96C42.98 0 0 42.98 0 96v64c0 35.35 28.66 64 64 64h512c35.34 0 64-28.65 64-64V96c0-53.02-42.98-96-96-96zM160 176c0 8.84-7.16 16-16 16H80c-8.84 0-16-7.16-16-16v-32c0-26.51 21.49-48 48-48s48 21.49 48 48v32zm144 0c0 8.84-7.16 16-16 16h-80c-8.84 0-16-7.16-16-16v-56c0-30.93 25.07-56 56-56s56 25.07 56 56v56zm144 0c0 8.84-7.16 16-16 16h-80c-8.84 0-16-7.16-16-16v-56c0-30.93 25.07-56 56-56s56 25.07 56 56v56zm128 0c0 8.84-7.16 16-16 16h-64c-8.84 0-16-7.16-16-16v-32c0-26.51 21.49-48 48-48s48 21.49 48 48v32zm0 144H64c-35.34 0-64 28.65-64 64v32c0 53.02 42.98 96 96 96h448c53.02 0 96-42.98 96-96v-32c0-35.35-28.66-64-64-64zm-416 80c0 26.51-21.49 48-48 48s-48-21.49-48-48v-32c0-8.84 7.16-16 16-16h64c8.84 0 16 7.16 16 16v32zm144-8c0 30.93-25.07 56-56 56s-56-25.07-56-56v-24c0-8.84 7.16-16 16-16h80c8.84 0 16 7.16 16 16v24zm144 0c0 30.93-25.07 56-56 56s-56-25.07-56-56v-24c0-8.84 7.16-16 16-16h80c8.84 0 16 7.16 16 16v24zm128 8c0 26.51-21.49 48-48 48s-48-21.49-48-48v-32c0-8.84 7.16-16 16-16h64c8.84 0 16 7.16 16 16v32z"],"temperature-high":[512,512,[],"f769","M416 0c-52.9 0-96 43.1-96 96s43.1 96 96 96 96-43.1 96-96-43.1-96-96-96zm0 128c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm-160-16C256 50.1 205.9 0 144 0S32 50.1 32 112v166.5C12.3 303.2 0 334 0 368c0 79.5 64.5 144 144 144s144-64.5 144-144c0-34-12.3-64.9-32-89.5V112zM144 448c-44.1 0-80-35.9-80-80 0-25.5 12.2-48.9 32-63.8V112c0-26.5 21.5-48 48-48s48 21.5 48 48v192.2c19.8 14.8 32 38.3 32 63.8 0 44.1-35.9 80-80 80zm16-125.1V112c0-8.8-7.2-16-16-16s-16 7.2-16 16v210.9c-18.6 6.6-32 24.2-32 45.1 0 26.5 21.5 48 48 48s48-21.5 48-48c0-20.9-13.4-38.5-32-45.1z"],"temperature-low":[512,512,[],"f76b","M416 0c-52.9 0-96 43.1-96 96s43.1 96 96 96 96-43.1 96-96-43.1-96-96-96zm0 128c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm-160-16C256 50.1 205.9 0 144 0S32 50.1 32 112v166.5C12.3 303.2 0 334 0 368c0 79.5 64.5 144 144 144s144-64.5 144-144c0-34-12.3-64.9-32-89.5V112zM144 448c-44.1 0-80-35.9-80-80 0-25.5 12.2-48.9 32-63.8V112c0-26.5 21.5-48 48-48s48 21.5 48 48v192.2c19.8 14.8 32 38.3 32 63.8 0 44.1-35.9 80-80 80zm16-125.1V304c0-8.8-7.2-16-16-16s-16 7.2-16 16v18.9c-18.6 6.6-32 24.2-32 45.1 0 26.5 21.5 48 48 48s48-21.5 48-48c0-20.9-13.4-38.5-32-45.1z"],tenge:[384,512,[],"f7d7","M372 160H12c-6.6 0-12 5.4-12 12v56c0 6.6 5.4 12 12 12h140v228c0 6.6 5.4 12 12 12h56c6.6 0 12-5.4 12-12V240h140c6.6 0 12-5.4 12-12v-56c0-6.6-5.4-12-12-12zm0-128H12C5.4 32 0 37.4 0 44v56c0 6.6 5.4 12 12 12h360c6.6 0 12-5.4 12-12V44c0-6.6-5.4-12-12-12z"],terminal:[640,512,[],"f120","M257.981 272.971L63.638 467.314c-9.373 9.373-24.569 9.373-33.941 0L7.029 444.647c-9.357-9.357-9.375-24.522-.04-33.901L161.011 256 6.99 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L257.981 239.03c9.373 9.372 9.373 24.568 0 33.941zM640 456v-32c0-13.255-10.745-24-24-24H312c-13.255 0-24 10.745-24 24v32c0 13.255 10.745 24 24 24h304c13.255 0 24-10.745 24-24z"],"text-height":[576,512,[],"f034","M304 32H16A16 16 0 0 0 0 48v96a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32h56v304H80a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h160a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16h-40V112h56v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16zm256 336h-48V144h48c14.31 0 21.33-17.31 11.31-27.31l-80-80a16 16 0 0 0-22.62 0l-80 80C379.36 126 384.36 144 400 144h48v224h-48c-14.31 0-21.32 17.31-11.31 27.31l80 80a16 16 0 0 0 22.62 0l80-80C580.64 386 575.64 368 560 368z"],"text-width":[448,512,[],"f035","M432 32H16A16 16 0 0 0 0 48v80a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-16h120v112h-24a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h128a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16h-24V112h120v16a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16zm-68.69 260.69C354 283.36 336 288.36 336 304v48H112v-48c0-14.31-17.31-21.32-27.31-11.31l-80 80a16 16 0 0 0 0 22.62l80 80C94 484.64 112 479.64 112 464v-48h224v48c0 14.31 17.31 21.33 27.31 11.31l80-80a16 16 0 0 0 0-22.62z"],th:[512,512,[],"f00a","M149.333 56v80c0 13.255-10.745 24-24 24H24c-13.255 0-24-10.745-24-24V56c0-13.255 10.745-24 24-24h101.333c13.255 0 24 10.745 24 24zm181.334 240v-80c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24h101.333c13.256 0 24.001-10.745 24.001-24zm32-240v80c0 13.255 10.745 24 24 24H488c13.255 0 24-10.745 24-24V56c0-13.255-10.745-24-24-24H386.667c-13.255 0-24 10.745-24 24zm-32 80V56c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24h101.333c13.256 0 24.001-10.745 24.001-24zm-205.334 56H24c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24h101.333c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24zM0 376v80c0 13.255 10.745 24 24 24h101.333c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H24c-13.255 0-24 10.745-24 24zm386.667-56H488c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H386.667c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24zm0 160H488c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H386.667c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24zM181.333 376v80c0 13.255 10.745 24 24 24h101.333c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24z"],"th-large":[512,512,[],"f009","M296 32h192c13.255 0 24 10.745 24 24v160c0 13.255-10.745 24-24 24H296c-13.255 0-24-10.745-24-24V56c0-13.255 10.745-24 24-24zm-80 0H24C10.745 32 0 42.745 0 56v160c0 13.255 10.745 24 24 24h192c13.255 0 24-10.745 24-24V56c0-13.255-10.745-24-24-24zM0 296v160c0 13.255 10.745 24 24 24h192c13.255 0 24-10.745 24-24V296c0-13.255-10.745-24-24-24H24c-13.255 0-24 10.745-24 24zm296 184h192c13.255 0 24-10.745 24-24V296c0-13.255-10.745-24-24-24H296c-13.255 0-24 10.745-24 24v160c0 13.255 10.745 24 24 24z"],"th-list":[512,512,[],"f00b","M149.333 216v80c0 13.255-10.745 24-24 24H24c-13.255 0-24-10.745-24-24v-80c0-13.255 10.745-24 24-24h101.333c13.255 0 24 10.745 24 24zM0 376v80c0 13.255 10.745 24 24 24h101.333c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H24c-13.255 0-24 10.745-24 24zM125.333 32H24C10.745 32 0 42.745 0 56v80c0 13.255 10.745 24 24 24h101.333c13.255 0 24-10.745 24-24V56c0-13.255-10.745-24-24-24zm80 448H488c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24zm-24-424v80c0 13.255 10.745 24 24 24H488c13.255 0 24-10.745 24-24V56c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24zm24 264H488c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24z"],"theater-masks":[640,512,[],"f630","M206.86 245.15c-35.88 10.45-59.95 41.2-57.53 74.1 11.4-12.72 28.81-23.7 49.9-30.92l7.63-43.18zM95.81 295L64.08 115.49c-.29-1.62.28-2.62.24-2.65 57.76-32.06 123.12-49.01 189.01-49.01 1.61 0 3.23.17 4.85.19 13.95-13.47 31.73-22.83 51.59-26 18.89-3.02 38.05-4.55 57.18-5.32-9.99-13.95-24.48-24.23-41.77-27C301.27 1.89 277.24 0 253.32 0 176.66 0 101.02 19.42 33.2 57.06 9.03 70.48-3.92 98.48 1.05 126.58l31.73 179.51c14.23 80.52 136.33 142.08 204.45 142.08 3.59 0 6.75-.46 10.01-.8-13.52-17.08-28.94-40.48-39.5-67.58-47.61-12.98-106.06-51.62-111.93-84.79zm97.55-137.46c-.73-4.12-2.23-7.87-4.07-11.4-8.25 8.91-20.67 15.75-35.32 18.32-14.65 2.58-28.67.4-39.48-5.17-.52 3.94-.64 7.98.09 12.1 3.84 21.7 24.58 36.19 46.34 32.37 21.75-3.82 36.28-24.52 32.44-46.22zM606.8 120.9c-88.98-49.38-191.43-67.41-291.98-51.35-27.31 4.36-49.08 26.26-54.04 54.36l-31.73 179.51c-15.39 87.05 95.28 196.27 158.31 207.35 63.03 11.09 204.47-53.79 219.86-140.84l31.73-179.51c4.97-28.11-7.98-56.11-32.15-69.52zm-273.24 96.8c3.84-21.7 24.58-36.19 46.34-32.36 21.76 3.83 36.28 24.52 32.45 46.22-.73 4.12-2.23 7.87-4.07 11.4-8.25-8.91-20.67-15.75-35.32-18.32-14.65-2.58-28.67-.4-39.48 5.17-.53-3.95-.65-7.99.08-12.11zm70.47 198.76c-55.68-9.79-93.52-59.27-89.04-112.9 20.6 25.54 56.21 46.17 99.49 53.78 43.28 7.61 83.82.37 111.93-16.6-14.18 51.94-66.71 85.51-122.38 75.72zm130.3-151.34c-8.25-8.91-20.68-15.75-35.33-18.32-14.65-2.58-28.67-.4-39.48 5.17-.52-3.94-.64-7.98.09-12.1 3.84-21.7 24.58-36.19 46.34-32.37 21.75 3.83 36.28 24.52 32.45 46.22-.73 4.13-2.23 7.88-4.07 11.4z"],thermometer:[512,512,[],"f491","M476.8 20.4c-37.5-30.7-95.5-26.3-131.9 10.2l-45.7 46 50.5 50.5c3.1 3.1 3.1 8.2 0 11.3l-11.3 11.3c-3.1 3.1-8.2 3.1-11.3 0l-50.4-50.5-45.1 45.4 50.3 50.4c3.1 3.1 3.1 8.2 0 11.3l-11.3 11.3c-3.1 3.1-8.2 3.1-11.3 0L209 167.4l-45.1 45.4L214 263c3.1 3.1 3.1 8.2 0 11.3l-11.3 11.3c-3.1 3.1-8.2 3.1-11.3 0l-50.1-50.2L96 281.1V382L7 471c-9.4 9.4-9.4 24.6 0 33.9 9.4 9.4 24.6 9.4 33.9 0l89-89h99.9L484 162.6c34.9-34.9 42.2-101.5-7.2-142.2z"],"thermometer-empty":[256,512,[],"f2cb","M192 384c0 35.346-28.654 64-64 64s-64-28.654-64-64c0-35.346 28.654-64 64-64s64 28.654 64 64zm32-84.653c19.912 22.563 32 52.194 32 84.653 0 70.696-57.303 128-128 128-.299 0-.609-.001-.909-.003C56.789 511.509-.357 453.636.002 383.333.166 351.135 12.225 321.755 32 299.347V96c0-53.019 42.981-96 96-96s96 42.981 96 96v203.347zM208 384c0-34.339-19.37-52.19-32-66.502V96c0-26.467-21.533-48-48-48S80 69.533 80 96v221.498c-12.732 14.428-31.825 32.1-31.999 66.08-.224 43.876 35.563 80.116 79.423 80.42L128 464c44.112 0 80-35.888 80-80z"],"thermometer-full":[256,512,[],"f2c7","M224 96c0-53.019-42.981-96-96-96S32 42.981 32 96v203.347C12.225 321.756.166 351.136.002 383.333c-.359 70.303 56.787 128.176 127.089 128.664.299.002.61.003.909.003 70.698 0 128-57.304 128-128 0-32.459-12.088-62.09-32-84.653V96zm-96 368l-.576-.002c-43.86-.304-79.647-36.544-79.423-80.42.173-33.98 19.266-51.652 31.999-66.08V96c0-26.467 21.533-48 48-48s48 21.533 48 48v221.498c12.63 14.312 32 32.164 32 66.502 0 44.112-35.888 80-80 80zm64-80c0 35.346-28.654 64-64 64s-64-28.654-64-64c0-23.685 12.876-44.349 32-55.417V96c0-17.673 14.327-32 32-32s32 14.327 32 32v232.583c19.124 11.068 32 31.732 32 55.417z"],"thermometer-half":[256,512,[],"f2c9","M192 384c0 35.346-28.654 64-64 64s-64-28.654-64-64c0-23.685 12.876-44.349 32-55.417V224c0-17.673 14.327-32 32-32s32 14.327 32 32v104.583c19.124 11.068 32 31.732 32 55.417zm32-84.653c19.912 22.563 32 52.194 32 84.653 0 70.696-57.303 128-128 128-.299 0-.609-.001-.909-.003C56.789 511.509-.357 453.636.002 383.333.166 351.135 12.225 321.755 32 299.347V96c0-53.019 42.981-96 96-96s96 42.981 96 96v203.347zM208 384c0-34.339-19.37-52.19-32-66.502V96c0-26.467-21.533-48-48-48S80 69.533 80 96v221.498c-12.732 14.428-31.825 32.1-31.999 66.08-.224 43.876 35.563 80.116 79.423 80.42L128 464c44.112 0 80-35.888 80-80z"],"thermometer-quarter":[256,512,[],"f2ca","M192 384c0 35.346-28.654 64-64 64s-64-28.654-64-64c0-23.685 12.876-44.349 32-55.417V288c0-17.673 14.327-32 32-32s32 14.327 32 32v40.583c19.124 11.068 32 31.732 32 55.417zm32-84.653c19.912 22.563 32 52.194 32 84.653 0 70.696-57.303 128-128 128-.299 0-.609-.001-.909-.003C56.789 511.509-.357 453.636.002 383.333.166 351.135 12.225 321.755 32 299.347V96c0-53.019 42.981-96 96-96s96 42.981 96 96v203.347zM208 384c0-34.339-19.37-52.19-32-66.502V96c0-26.467-21.533-48-48-48S80 69.533 80 96v221.498c-12.732 14.428-31.825 32.1-31.999 66.08-.224 43.876 35.563 80.116 79.423 80.42L128 464c44.112 0 80-35.888 80-80z"],"thermometer-three-quarters":[256,512,[],"f2c8","M192 384c0 35.346-28.654 64-64 64-35.346 0-64-28.654-64-64 0-23.685 12.876-44.349 32-55.417V160c0-17.673 14.327-32 32-32s32 14.327 32 32v168.583c19.124 11.068 32 31.732 32 55.417zm32-84.653c19.912 22.563 32 52.194 32 84.653 0 70.696-57.303 128-128 128-.299 0-.609-.001-.909-.003C56.789 511.509-.357 453.636.002 383.333.166 351.135 12.225 321.755 32 299.347V96c0-53.019 42.981-96 96-96s96 42.981 96 96v203.347zM208 384c0-34.339-19.37-52.19-32-66.502V96c0-26.467-21.533-48-48-48S80 69.533 80 96v221.498c-12.732 14.428-31.825 32.1-31.999 66.08-.224 43.876 35.563 80.116 79.423 80.42L128 464c44.112 0 80-35.888 80-80z"],"thumbs-down":[512,512,[],"f165","M0 56v240c0 13.255 10.745 24 24 24h80c13.255 0 24-10.745 24-24V56c0-13.255-10.745-24-24-24H24C10.745 32 0 42.745 0 56zm40 200c0-13.255 10.745-24 24-24s24 10.745 24 24-10.745 24-24 24-24-10.745-24-24zm272 256c-20.183 0-29.485-39.293-33.931-57.795-5.206-21.666-10.589-44.07-25.393-58.902-32.469-32.524-49.503-73.967-89.117-113.111a11.98 11.98 0 0 1-3.558-8.521V59.901c0-6.541 5.243-11.878 11.783-11.998 15.831-.29 36.694-9.079 52.651-16.178C256.189 17.598 295.709.017 343.995 0h2.844c42.777 0 93.363.413 113.774 29.737 8.392 12.057 10.446 27.034 6.148 44.632 16.312 17.053 25.063 48.863 16.382 74.757 17.544 23.432 19.143 56.132 9.308 79.469l.11.11c11.893 11.949 19.523 31.259 19.439 49.197-.156 30.352-26.157 58.098-59.553 58.098H350.723C358.03 364.34 384 388.132 384 430.548 384 504 336 512 312 512z"],"thumbs-up":[512,512,[],"f164","M104 224H24c-13.255 0-24 10.745-24 24v240c0 13.255 10.745 24 24 24h80c13.255 0 24-10.745 24-24V248c0-13.255-10.745-24-24-24zM64 472c-13.255 0-24-10.745-24-24s10.745-24 24-24 24 10.745 24 24-10.745 24-24 24zM384 81.452c0 42.416-25.97 66.208-33.277 94.548h101.723c33.397 0 59.397 27.746 59.553 58.098.084 17.938-7.546 37.249-19.439 49.197l-.11.11c9.836 23.337 8.237 56.037-9.308 79.469 8.681 25.895-.069 57.704-16.382 74.757 4.298 17.598 2.244 32.575-6.148 44.632C440.202 511.587 389.616 512 346.839 512l-2.845-.001c-48.287-.017-87.806-17.598-119.56-31.725-15.957-7.099-36.821-15.887-52.651-16.178-6.54-.12-11.783-5.457-11.783-11.998v-213.77c0-3.2 1.282-6.271 3.558-8.521 39.614-39.144 56.648-80.587 89.117-113.111 14.804-14.832 20.188-37.236 25.393-58.902C282.515 39.293 291.817 0 312 0c24 0 72 8 72 81.452z"],thumbtack:[384,512,[],"f08d","M298.028 214.267L285.793 96H328c13.255 0 24-10.745 24-24V24c0-13.255-10.745-24-24-24H56C42.745 0 32 10.745 32 24v48c0 13.255 10.745 24 24 24h42.207L85.972 214.267C37.465 236.82 0 277.261 0 328c0 13.255 10.745 24 24 24h136v104.007c0 1.242.289 2.467.845 3.578l24 48c2.941 5.882 11.364 5.893 14.311 0l24-48a8.008 8.008 0 0 0 .845-3.578V352h136c13.255 0 24-10.745 24-24-.001-51.183-37.983-91.42-85.973-113.733z"],"ticket-alt":[576,512,[],"f3ff","M128 160h320v192H128V160zm400 96c0 26.51 21.49 48 48 48v96c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48v-96c26.51 0 48-21.49 48-48s-21.49-48-48-48v-96c0-26.51 21.49-48 48-48h480c26.51 0 48 21.49 48 48v96c-26.51 0-48 21.49-48 48zm-48-104c0-13.255-10.745-24-24-24H120c-13.255 0-24 10.745-24 24v208c0 13.255 10.745 24 24 24h336c13.255 0 24-10.745 24-24V152z"],times:[352,512,[],"f00d","M242.72 256l100.07-100.07c12.28-12.28 12.28-32.19 0-44.48l-22.24-22.24c-12.28-12.28-32.19-12.28-44.48 0L176 189.28 75.93 89.21c-12.28-12.28-32.19-12.28-44.48 0L9.21 111.45c-12.28 12.28-12.28 32.19 0 44.48L109.28 256 9.21 356.07c-12.28 12.28-12.28 32.19 0 44.48l22.24 22.24c12.28 12.28 32.2 12.28 44.48 0L176 322.72l100.07 100.07c12.28 12.28 32.2 12.28 44.48 0l22.24-22.24c12.28-12.28 12.28-32.19 0-44.48L242.72 256z"],"times-circle":[512,512,[],"f057","M256 8C119 8 8 119 8 256s111 248 248 248 248-111 248-248S393 8 256 8zm121.6 313.1c4.7 4.7 4.7 12.3 0 17L338 377.6c-4.7 4.7-12.3 4.7-17 0L256 312l-65.1 65.6c-4.7 4.7-12.3 4.7-17 0L134.4 338c-4.7-4.7-4.7-12.3 0-17l65.6-65-65.6-65.1c-4.7-4.7-4.7-12.3 0-17l39.6-39.6c4.7-4.7 12.3-4.7 17 0l65 65.7 65.1-65.6c4.7-4.7 12.3-4.7 17 0l39.6 39.6c4.7 4.7 4.7 12.3 0 17L312 256l65.6 65.1z"],tint:[352,512,[],"f043","M205.22 22.09c-7.94-28.78-49.44-30.12-58.44 0C100.01 179.85 0 222.72 0 333.91 0 432.35 78.72 512 176 512s176-79.65 176-178.09c0-111.75-99.79-153.34-146.78-311.82zM176 448c-61.75 0-112-50.25-112-112 0-8.84 7.16-16 16-16s16 7.16 16 16c0 44.11 35.89 80 80 80 8.84 0 16 7.16 16 16s-7.16 16-16 16z"],"tint-slash":[640,512,[],"f5c7","M633.82 458.1L494.97 350.78c.52-5.57 1.03-11.16 1.03-16.87 0-111.76-99.79-153.34-146.78-311.82-7.94-28.78-49.44-30.12-58.44 0-15.52 52.34-36.87 91.96-58.49 125.68L45.47 3.37C38.49-2.05 28.43-.8 23.01 6.18L3.37 31.45C-2.05 38.42-.8 48.47 6.18 53.9l588.36 454.73c6.98 5.43 17.03 4.17 22.46-2.81l19.64-25.27c5.41-6.97 4.16-17.02-2.82-22.45zM144 333.91C144 432.35 222.72 512 320 512c44.71 0 85.37-16.96 116.4-44.7L162.72 255.78c-11.41 23.5-18.72 48.35-18.72 78.13z"],tired:[496,512,[],"f5c8","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm33.8 189.7l80-48c11.6-6.9 24 7.7 15.4 18L343.6 208l33.6 40.3c8.7 10.4-3.9 24.8-15.4 18l-80-48c-7.7-4.7-7.7-15.9 0-20.6zm-163-30c-8.6-10.3 3.8-24.9 15.4-18l80 48c7.8 4.7 7.8 15.9 0 20.6l-80 48c-11.5 6.8-24-7.6-15.4-18l33.6-40.3-33.6-40.3zM248 288c51.9 0 115.3 43.8 123.2 106.7 1.7 13.6-8 24.6-17.7 20.4-25.9-11.1-64.4-17.4-105.5-17.4s-79.6 6.3-105.5 17.4c-9.8 4.2-19.4-7-17.7-20.4C132.7 331.8 196.1 288 248 288z"],"toggle-off":[576,512,[],"f204","M384 64H192C85.961 64 0 149.961 0 256s85.961 192 192 192h192c106.039 0 192-85.961 192-192S490.039 64 384 64zM64 256c0-70.741 57.249-128 128-128 70.741 0 128 57.249 128 128 0 70.741-57.249 128-128 128-70.741 0-128-57.249-128-128zm320 128h-48.905c65.217-72.858 65.236-183.12 0-256H384c70.741 0 128 57.249 128 128 0 70.74-57.249 128-128 128z"],"toggle-on":[576,512,[],"f205","M384 64H192C86 64 0 150 0 256s86 192 192 192h192c106 0 192-86 192-192S490 64 384 64zm0 320c-70.8 0-128-57.3-128-128 0-70.8 57.3-128 128-128 70.8 0 128 57.3 128 128 0 70.8-57.3 128-128 128z"],toilet:[384,512,[],"f7d8","M368 48c8.8 0 16-7.2 16-16V16c0-8.8-7.2-16-16-16H16C7.2 0 0 7.2 0 16v16c0 8.8 7.2 16 16 16h16v156.7C11.8 214.8 0 226.9 0 240c0 67.2 34.6 126.2 86.8 160.5l-21.4 70.2C59.1 491.2 74.5 512 96 512h192c21.5 0 36.9-20.8 30.6-41.3l-21.4-70.2C349.4 366.2 384 307.2 384 240c0-13.1-11.8-25.2-32-35.3V48h16zM80 72c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H88c-4.4 0-8-3.6-8-8V72zm112 200c-77.1 0-139.6-14.3-139.6-32s62.5-32 139.6-32 139.6 14.3 139.6 32-62.5 32-139.6 32z"],"toilet-paper":[576,512,[],"f71e","M128 0C74.98 0 32 85.96 32 192v172.07c0 41.12-9.8 62.77-31.17 126.87C-2.62 501.3 5.09 512 16.01 512h280.92c13.77 0 26-8.81 30.36-21.88 12.83-38.48 24.71-72.4 24.71-126.05V192c0-83.6 23.67-153.52 60.44-192H128zM96 224c-8.84 0-16-7.16-16-16s7.16-16 16-16 16 7.16 16 16-7.16 16-16 16zm64 0c-8.84 0-16-7.16-16-16s7.16-16 16-16 16 7.16 16 16-7.16 16-16 16zm64 0c-8.84 0-16-7.16-16-16s7.16-16 16-16 16 7.16 16 16-7.16 16-16 16zm64 0c-8.84 0-16-7.16-16-16s7.16-16 16-16 16 7.16 16 16-7.16 16-16 16zM480 0c-53.02 0-96 85.96-96 192s42.98 192 96 192 96-85.96 96-192S533.02 0 480 0zm0 256c-17.67 0-32-28.65-32-64s14.33-64 32-64 32 28.65 32 64-14.33 64-32 64z"],"toilet-paper-slash":[640,512,[],"f972","M64,192V364.13c0,41.12-9.75,62.75-31.12,126.87A16,16,0,0,0,48,512H328.86a31.87,31.87,0,0,0,30.38-21.87c9.31-27.83,18-53.35,22.18-85.55l-316-244.25C64.53,170.66,64,181.19,64,192ZM633.82,458.09l-102-78.81C575.28,360.91,608,284.32,608,192,608,86,565,0,512,0s-96,86-96,192c0,42,7,80.4,18.43,112L384,265V192c0-83.62,23.63-153.5,60.5-192H160c-23.33,0-44.63,16.83-61.26,44.53L45.46,3.38A16,16,0,0,0,23,6.19L3.37,31.45A16,16,0,0,0,6.18,53.91L594.54,508.63A16,16,0,0,0,617,505.81l19.64-25.26A16,16,0,0,0,633.82,458.09ZM512,256c-17.63,0-32-28.62-32-64s14.37-64,32-64,32,28.63,32,64S529.62,256,512,256Z"],toolbox:[512,512,[],"f552","M502.63 214.63l-45.25-45.25c-6-6-14.14-9.37-22.63-9.37H384V80c0-26.51-21.49-48-48-48H176c-26.51 0-48 21.49-48 48v80H77.25c-8.49 0-16.62 3.37-22.63 9.37L9.37 214.63c-6 6-9.37 14.14-9.37 22.63V320h128v-16c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v16h128v-16c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v16h128v-82.75c0-8.48-3.37-16.62-9.37-22.62zM320 160H192V96h128v64zm64 208c0 8.84-7.16 16-16 16h-32c-8.84 0-16-7.16-16-16v-16H192v16c0 8.84-7.16 16-16 16h-32c-8.84 0-16-7.16-16-16v-16H0v96c0 17.67 14.33 32 32 32h448c17.67 0 32-14.33 32-32v-96H384v16z"],tools:[512,512,[],"f7d9","M501.1 395.7L384 278.6c-23.1-23.1-57.6-27.6-85.4-13.9L192 158.1V96L64 0 0 64l96 128h62.1l106.6 106.6c-13.6 27.8-9.2 62.3 13.9 85.4l117.1 117.1c14.6 14.6 38.2 14.6 52.7 0l52.7-52.7c14.5-14.6 14.5-38.2 0-52.7zM331.7 225c28.3 0 54.9 11 74.9 31l19.4 19.4c15.8-6.9 30.8-16.5 43.8-29.5 37.1-37.1 49.7-89.3 37.9-136.7-2.2-9-13.5-12.1-20.1-5.5l-74.4 74.4-67.9-11.3L334 98.9l74.4-74.4c6.6-6.6 3.4-17.9-5.7-20.2-47.4-11.7-99.6.9-136.6 37.9-28.5 28.5-41.9 66.1-41.2 103.6l82.1 82.1c8.1-1.9 16.5-2.9 24.7-2.9zm-103.9 82l-56.7-56.7L18.7 402.8c-25 25-25 65.5 0 90.5s65.5 25 90.5 0l123.6-123.6c-7.6-19.9-9.9-41.6-5-62.7zM64 472c-13.2 0-24-10.8-24-24 0-13.3 10.7-24 24-24s24 10.7 24 24c0 13.2-10.7 24-24 24z"],tooth:[448,512,[],"f5c9","M443.98 96.25c-11.01-45.22-47.11-82.06-92.01-93.72-32.19-8.36-63 5.1-89.14 24.33-3.25 2.39-6.96 3.73-10.5 5.48l28.32 18.21c7.42 4.77 9.58 14.67 4.8 22.11-4.46 6.95-14.27 9.86-22.11 4.8L162.83 12.84c-20.7-10.85-43.38-16.4-66.81-10.31-44.9 11.67-81 48.5-92.01 93.72-10.13 41.62-.42 80.81 21.5 110.43 23.36 31.57 32.68 68.66 36.29 107.35 4.4 47.16 10.33 94.16 20.94 140.32l7.8 33.95c3.19 13.87 15.49 23.7 29.67 23.7 13.97 0 26.15-9.55 29.54-23.16l34.47-138.42c4.56-18.32 20.96-31.16 39.76-31.16s35.2 12.85 39.76 31.16l34.47 138.42c3.39 13.61 15.57 23.16 29.54 23.16 14.18 0 26.48-9.83 29.67-23.7l7.8-33.95c10.61-46.15 16.53-93.16 20.94-140.32 3.61-38.7 12.93-75.78 36.29-107.35 21.95-29.61 31.66-68.8 21.53-110.43z"],torah:[640,512,[],"f6a0","M320.05 366.48l17.72-29.64h-35.46zm99.21-166H382.4l18.46 30.82zM48 0C21.49 0 0 14.33 0 32v448c0 17.67 21.49 32 48 32s48-14.33 48-32V32C96 14.33 74.51 0 48 0zm172.74 311.5h36.85l-18.46-30.82zm161.71 0h36.86l-18.45-30.8zM128 464h384V48H128zm66.77-278.13a21.22 21.22 0 0 1 18.48-10.71h59.45l29.13-48.71a21.13 21.13 0 0 1 18.22-10.37A20.76 20.76 0 0 1 338 126.29l29.25 48.86h59.52a21.12 21.12 0 0 1 18.1 32L415.63 256 445 305a20.69 20.69 0 0 1 .24 21.12 21.25 21.25 0 0 1-18.48 10.72h-59.47l-29.13 48.7a21.13 21.13 0 0 1-18.16 10.4 20.79 20.79 0 0 1-18-10.22l-29.25-48.88h-59.5a21.11 21.11 0 0 1-18.1-32L224.36 256 195 207a20.7 20.7 0 0 1-.23-21.13zM592 0c-26.51 0-48 14.33-48 32v448c0 17.67 21.49 32 48 32s48-14.33 48-32V32c0-17.67-21.49-32-48-32zM320 145.53l-17.78 29.62h35.46zm-62.45 55h-36.81l18.44 30.8zm29.58 111h65.79L386.09 256l-33.23-55.52h-65.79L253.9 256z"],"torii-gate":[512,512,[],"f6a1","M376.45 32h-240.9A303.17 303.17 0 0 1 0 0v96c0 17.67 14.33 32 32 32h32v64H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h48v240c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16V256h256v240c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16V256h48c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16h-48v-64h32c17.67 0 32-14.33 32-32V0a303.17 303.17 0 0 1-135.55 32zM128 128h96v64h-96v-64zm256 64h-96v-64h96v64z"],tractor:[640,512,[],"f722","M528 336c-48.6 0-88 39.4-88 88s39.4 88 88 88 88-39.4 88-88-39.4-88-88-88zm0 112c-13.23 0-24-10.77-24-24s10.77-24 24-24 24 10.77 24 24-10.77 24-24 24zm80-288h-64v-40.2c0-14.12 4.7-27.76 13.15-38.84 4.42-5.8 3.55-14.06-1.32-19.49L534.2 37.3c-6.66-7.45-18.32-6.92-24.7.78C490.58 60.9 480 89.81 480 119.8V160H377.67L321.58 29.14A47.914 47.914 0 0 0 277.45 0H144c-26.47 0-48 21.53-48 48v146.52c-8.63-6.73-20.96-6.46-28.89 1.47L36 227.1c-8.59 8.59-8.59 22.52 0 31.11l5.06 5.06c-4.99 9.26-8.96 18.82-11.91 28.72H22c-12.15 0-22 9.85-22 22v44c0 12.15 9.85 22 22 22h7.14c2.96 9.91 6.92 19.46 11.91 28.73l-5.06 5.06c-8.59 8.59-8.59 22.52 0 31.11L67.1 476c8.59 8.59 22.52 8.59 31.11 0l5.06-5.06c9.26 4.99 18.82 8.96 28.72 11.91V490c0 12.15 9.85 22 22 22h44c12.15 0 22-9.85 22-22v-7.14c9.9-2.95 19.46-6.92 28.72-11.91l5.06 5.06c8.59 8.59 22.52 8.59 31.11 0l31.11-31.11c8.59-8.59 8.59-22.52 0-31.11l-5.06-5.06c4.99-9.26 8.96-18.82 11.91-28.72H330c12.15 0 22-9.85 22-22v-6h80.54c21.91-28.99 56.32-48 95.46-48 18.64 0 36.07 4.61 51.8 12.2l50.82-50.82c6-6 9.37-14.14 9.37-22.63V192c.01-17.67-14.32-32-31.99-32zM176 416c-44.18 0-80-35.82-80-80s35.82-80 80-80 80 35.82 80 80-35.82 80-80 80zm22-256h-38V64h106.89l41.15 96H198z"],trademark:[640,512,[],"f25c","M260.6 96H12c-6.6 0-12 5.4-12 12v43.1c0 6.6 5.4 12 12 12h85.1V404c0 6.6 5.4 12 12 12h54.3c6.6 0 12-5.4 12-12V163.1h85.1c6.6 0 12-5.4 12-12V108c.1-6.6-5.3-12-11.9-12zM640 403l-24-296c-.5-6.2-5.7-11-12-11h-65.4c-5.1 0-9.7 3.3-11.3 8.1l-43.8 127.1c-7.2 20.6-16.1 52.8-16.1 52.8h-.9s-8.9-32.2-16.1-52.8l-43.8-127.1c-1.7-4.8-6.2-8.1-11.3-8.1h-65.4c-6.2 0-11.4 4.8-12 11l-24.4 296c-.6 7 4.9 13 12 13H360c6.3 0 11.5-4.9 12-11.2l9.1-132.9c1.8-24.2 0-53.7 0-53.7h.9s10.7 33.6 17.9 53.7l30.7 84.7c1.7 4.7 6.2 7.9 11.3 7.9h50.3c5.1 0 9.6-3.2 11.3-7.9l30.7-84.7c7.2-20.1 17.9-53.7 17.9-53.7h.9s-1.8 29.5 0 53.7l9.1 132.9c.4 6.3 5.7 11.2 12 11.2H628c7 0 12.5-6 12-13z"],"traffic-light":[384,512,[],"f637","M384 192h-64v-37.88c37.2-13.22 64-48.38 64-90.12h-64V32c0-17.67-14.33-32-32-32H96C78.33 0 64 14.33 64 32v32H0c0 41.74 26.8 76.9 64 90.12V192H0c0 41.74 26.8 76.9 64 90.12V320H0c0 42.84 28.25 78.69 66.99 91.05C79.42 468.72 130.6 512 192 512s112.58-43.28 125.01-100.95C355.75 398.69 384 362.84 384 320h-64v-37.88c37.2-13.22 64-48.38 64-90.12zM192 416c-26.51 0-48-21.49-48-48s21.49-48 48-48 48 21.49 48 48-21.49 48-48 48zm0-128c-26.51 0-48-21.49-48-48s21.49-48 48-48 48 21.49 48 48-21.49 48-48 48zm0-128c-26.51 0-48-21.49-48-48s21.49-48 48-48 48 21.49 48 48-21.49 48-48 48z"],trailer:[640,512,[],"f941","M624,320H544V80a16,16,0,0,0-16-16H16A16,16,0,0,0,0,80V368a16,16,0,0,0,16,16H65.61c7.83-54.21,54-96,110.39-96s102.56,41.79,110.39,96H624a16,16,0,0,0,16-16V336A16,16,0,0,0,624,320ZM96,243.68a176.29,176.29,0,0,0-32,20.71V136a8,8,0,0,1,8-8H88a8,8,0,0,1,8,8Zm96-18.54c-5.31-.49-10.57-1.14-16-1.14s-10.69.65-16,1.14V136a8,8,0,0,1,8-8h16a8,8,0,0,1,8,8Zm96,39.25a176.29,176.29,0,0,0-32-20.71V136a8,8,0,0,1,8-8h16a8,8,0,0,1,8,8ZM384,320H352V136a8,8,0,0,1,8-8h16a8,8,0,0,1,8,8Zm96,0H448V136a8,8,0,0,1,8-8h16a8,8,0,0,1,8,8Zm-304,0a80,80,0,1,0,80,80A80,80,0,0,0,176,320Zm0,112a32,32,0,1,1,32-32A32,32,0,0,1,176,432Z"],train:[448,512,[],"f238","M448 96v256c0 51.815-61.624 96-130.022 96l62.98 49.721C386.905 502.417 383.562 512 376 512H72c-7.578 0-10.892-9.594-4.957-14.279L130.022 448C61.82 448 0 403.954 0 352V96C0 42.981 64 0 128 0h192c65 0 128 42.981 128 96zm-48 136V120c0-13.255-10.745-24-24-24H72c-13.255 0-24 10.745-24 24v112c0 13.255 10.745 24 24 24h304c13.255 0 24-10.745 24-24zm-176 64c-30.928 0-56 25.072-56 56s25.072 56 56 56 56-25.072 56-56-25.072-56-56-56z"],tram:[512,512,[],"f7da","M288 64c17.7 0 32-14.3 32-32S305.7 0 288 0s-32 14.3-32 32 14.3 32 32 32zm223.5-12.1c-2.3-8.6-11-13.6-19.6-11.3l-480 128c-8.5 2.3-13.6 11-11.3 19.6C2.5 195.3 8.9 200 16 200c1.4 0 2.8-.2 4.1-.5L240 140.8V224H64c-17.7 0-32 14.3-32 32v224c0 17.7 14.3 32 32 32h384c17.7 0 32-14.3 32-32V256c0-17.7-14.3-32-32-32H272v-91.7l228.1-60.8c8.6-2.3 13.6-11.1 11.4-19.6zM176 384H80v-96h96v96zm160-96h96v96h-96v-96zm-32 0v96h-96v-96h96zM192 96c17.7 0 32-14.3 32-32s-14.3-32-32-32-32 14.3-32 32 14.3 32 32 32z"],transgender:[384,512,[],"f224","M372 0h-79c-10.7 0-16 12.9-8.5 20.5l16.9 16.9-80.7 80.7C198.5 104.1 172.2 96 144 96 64.5 96 0 160.5 0 240c0 68.5 47.9 125.9 112 140.4V408H76c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h36v28c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12v-28h36c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-36v-27.6c64.1-14.6 112-71.9 112-140.4 0-28.2-8.1-54.5-22.1-76.7l80.7-80.7 16.9 16.9c7.6 7.6 20.5 2.2 20.5-8.5V12c0-6.6-5.4-12-12-12zM144 320c-44.1 0-80-35.9-80-80s35.9-80 80-80 80 35.9 80 80-35.9 80-80 80z"],"transgender-alt":[480,512,[],"f225","M468 0h-79c-10.7 0-16 12.9-8.5 20.5l16.9 16.9-80.7 80.7C294.5 104.1 268.2 96 240 96c-28.2 0-54.5 8.1-76.7 22.1l-16.5-16.5 19.8-19.8c4.7-4.7 4.7-12.3 0-17l-28.3-28.3c-4.7-4.7-12.3-4.7-17 0l-19.8 19.8-19-19 16.9-16.9C107.1 12.9 101.7 0 91 0H12C5.4 0 0 5.4 0 12v79c0 10.7 12.9 16 20.5 8.5l16.9-16.9 19 19-19.8 19.8c-4.7 4.7-4.7 12.3 0 17l28.3 28.3c4.7 4.7 12.3 4.7 17 0l19.8-19.8 16.5 16.5C104.1 185.5 96 211.8 96 240c0 68.5 47.9 125.9 112 140.4V408h-36c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h36v28c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12v-28h36c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-36v-27.6c64.1-14.6 112-71.9 112-140.4 0-28.2-8.1-54.5-22.1-76.7l80.7-80.7 16.9 16.9c7.6 7.6 20.5 2.2 20.5-8.5V12c0-6.6-5.4-12-12-12zM240 320c-44.1 0-80-35.9-80-80s35.9-80 80-80 80 35.9 80 80-35.9 80-80 80z"],trash:[448,512,[],"f1f8","M432 32H312l-9.4-18.7A24 24 0 0 0 281.1 0H166.8a23.72 23.72 0 0 0-21.4 13.3L136 32H16A16 16 0 0 0 0 48v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16zM53.2 467a48 48 0 0 0 47.9 45h245.8a48 48 0 0 0 47.9-45L416 128H32z"],"trash-alt":[448,512,[],"f2ed","M32 464a48 48 0 0 0 48 48h288a48 48 0 0 0 48-48V128H32zm272-256a16 16 0 0 1 32 0v224a16 16 0 0 1-32 0zm-96 0a16 16 0 0 1 32 0v224a16 16 0 0 1-32 0zm-96 0a16 16 0 0 1 32 0v224a16 16 0 0 1-32 0zM432 32H312l-9.4-18.7A24 24 0 0 0 281.1 0H166.8a23.72 23.72 0 0 0-21.4 13.3L136 32H16A16 16 0 0 0 0 48v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z"],"trash-restore":[448,512,[],"f829","M53.2 467a48 48 0 0 0 47.9 45h245.8a48 48 0 0 0 47.9-45L416 128H32zm70.11-175.8l89.38-94.26a15.41 15.41 0 0 1 22.62 0l89.38 94.26c10.08 10.62 2.94 28.8-11.32 28.8H256v112a16 16 0 0 1-16 16h-32a16 16 0 0 1-16-16V320h-57.37c-14.26 0-21.4-18.18-11.32-28.8zM432 32H312l-9.4-18.7A24 24 0 0 0 281.1 0H166.8a23.72 23.72 0 0 0-21.4 13.3L136 32H16A16 16 0 0 0 0 48v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z"],"trash-restore-alt":[448,512,[],"f82a","M32 464a48 48 0 0 0 48 48h288a48 48 0 0 0 48-48V128H32zm91.31-172.8l89.38-94.26a15.41 15.41 0 0 1 22.62 0l89.38 94.26c10.08 10.62 2.94 28.8-11.32 28.8H256v112a16 16 0 0 1-16 16h-32a16 16 0 0 1-16-16V320h-57.37c-14.26 0-21.4-18.18-11.32-28.8zM432 32H312l-9.4-18.7A24 24 0 0 0 281.1 0H166.8a23.72 23.72 0 0 0-21.4 13.3L136 32H16A16 16 0 0 0 0 48v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z"],tree:[384,512,[],"f1bb","M378.31 378.49L298.42 288h30.63c9.01 0 16.98-5 20.78-13.06 3.8-8.04 2.55-17.26-3.28-24.05L268.42 160h28.89c9.1 0 17.3-5.35 20.86-13.61 3.52-8.13 1.86-17.59-4.24-24.08L203.66 4.83c-6.03-6.45-17.28-6.45-23.32 0L70.06 122.31c-6.1 6.49-7.75 15.95-4.24 24.08C69.38 154.65 77.59 160 86.69 160h28.89l-78.14 90.91c-5.81 6.78-7.06 15.99-3.27 24.04C37.97 283 45.93 288 54.95 288h30.63L5.69 378.49c-6 6.79-7.36 16.09-3.56 24.26 3.75 8.05 12 13.25 21.01 13.25H160v24.45l-30.29 48.4c-5.32 10.64 2.42 23.16 14.31 23.16h95.96c11.89 0 19.63-12.52 14.31-23.16L224 440.45V416h136.86c9.01 0 17.26-5.2 21.01-13.25 3.8-8.17 2.44-17.47-3.56-24.26z"],trophy:[576,512,[],"f091","M552 64H448V24c0-13.3-10.7-24-24-24H152c-13.3 0-24 10.7-24 24v40H24C10.7 64 0 74.7 0 88v56c0 35.7 22.5 72.4 61.9 100.7 31.5 22.7 69.8 37.1 110 41.7C203.3 338.5 240 360 240 360v72h-48c-35.3 0-64 20.7-64 56v12c0 6.6 5.4 12 12 12h296c6.6 0 12-5.4 12-12v-12c0-35.3-28.7-56-64-56h-48v-72s36.7-21.5 68.1-73.6c40.3-4.6 78.6-19 110-41.7 39.3-28.3 61.9-65 61.9-100.7V88c0-13.3-10.7-24-24-24zM99.3 192.8C74.9 175.2 64 155.6 64 144v-16h64.2c1 32.6 5.8 61.2 12.8 86.2-15.1-5.2-29.2-12.4-41.7-21.4zM512 144c0 16.1-17.7 36.1-35.3 48.8-12.5 9-26.7 16.2-41.8 21.4 7-25 11.8-53.6 12.8-86.2H512v16z"],truck:[640,512,[],"f0d1","M624 352h-16V243.9c0-12.7-5.1-24.9-14.1-33.9L494 110.1c-9-9-21.2-14.1-33.9-14.1H416V48c0-26.5-21.5-48-48-48H48C21.5 0 0 21.5 0 48v320c0 26.5 21.5 48 48 48h16c0 53 43 96 96 96s96-43 96-96h128c0 53 43 96 96 96s96-43 96-96h48c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zM160 464c-26.5 0-48-21.5-48-48s21.5-48 48-48 48 21.5 48 48-21.5 48-48 48zm320 0c-26.5 0-48-21.5-48-48s21.5-48 48-48 48 21.5 48 48-21.5 48-48 48zm80-208H416V144h44.1l99.9 99.9V256z"],"truck-loading":[640,512,[],"f4de","M50.2 375.6c2.3 8.5 11.1 13.6 19.6 11.3l216.4-58c8.5-2.3 13.6-11.1 11.3-19.6l-49.7-185.5c-2.3-8.5-11.1-13.6-19.6-11.3L151 133.3l24.8 92.7-61.8 16.5-24.8-92.7-77.3 20.7C3.4 172.8-1.7 181.6.6 190.1l49.6 185.5zM384 0c-17.7 0-32 14.3-32 32v323.6L5.9 450c-4.3 1.2-6.8 5.6-5.6 9.8l12.6 46.3c1.2 4.3 5.6 6.8 9.8 5.6l393.7-107.4C418.8 464.1 467.6 512 528 512c61.9 0 112-50.1 112-112V0H384zm144 448c-26.5 0-48-21.5-48-48s21.5-48 48-48 48 21.5 48 48-21.5 48-48 48z"],"truck-monster":[640,512,[],"f63b","M624 224h-16v-64c0-17.67-14.33-32-32-32h-73.6L419.22 24.02A64.025 64.025 0 0 0 369.24 0H256c-17.67 0-32 14.33-32 32v96H48c-8.84 0-16 7.16-16 16v80H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h16.72c29.21-38.65 75.1-64 127.28-64s98.07 25.35 127.28 64h65.45c29.21-38.65 75.1-64 127.28-64s98.07 25.35 127.28 64H624c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zm-336-96V64h81.24l51.2 64H288zm304 224h-5.2c-2.2-7.33-5.07-14.28-8.65-20.89l3.67-3.67c6.25-6.25 6.25-16.38 0-22.63l-22.63-22.63c-6.25-6.25-16.38-6.25-22.63 0l-3.67 3.67A110.85 110.85 0 0 0 512 277.2V272c0-8.84-7.16-16-16-16h-32c-8.84 0-16 7.16-16 16v5.2c-7.33 2.2-14.28 5.07-20.89 8.65l-3.67-3.67c-6.25-6.25-16.38-6.25-22.63 0l-22.63 22.63c-6.25 6.25-6.25 16.38 0 22.63l3.67 3.67A110.85 110.85 0 0 0 373.2 352H368c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h5.2c2.2 7.33 5.07 14.28 8.65 20.89l-3.67 3.67c-6.25 6.25-6.25 16.38 0 22.63l22.63 22.63c6.25 6.25 16.38 6.25 22.63 0l3.67-3.67c6.61 3.57 13.57 6.45 20.9 8.65v5.2c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-5.2c7.33-2.2 14.28-5.07 20.9-8.65l3.67 3.67c6.25 6.25 16.38 6.25 22.63 0l22.63-22.63c6.25-6.25 6.25-16.38 0-22.63l-3.67-3.67a110.85 110.85 0 0 0 8.65-20.89h5.2c8.84 0 16-7.16 16-16v-32c-.02-8.84-7.18-16-16.02-16zm-112 80c-26.51 0-48-21.49-48-48s21.49-48 48-48 48 21.49 48 48-21.49 48-48 48zm-208-80h-5.2c-2.2-7.33-5.07-14.28-8.65-20.89l3.67-3.67c6.25-6.25 6.25-16.38 0-22.63l-22.63-22.63c-6.25-6.25-16.38-6.25-22.63 0l-3.67 3.67A110.85 110.85 0 0 0 192 277.2V272c0-8.84-7.16-16-16-16h-32c-8.84 0-16 7.16-16 16v5.2c-7.33 2.2-14.28 5.07-20.89 8.65l-3.67-3.67c-6.25-6.25-16.38-6.25-22.63 0L58.18 304.8c-6.25 6.25-6.25 16.38 0 22.63l3.67 3.67a110.85 110.85 0 0 0-8.65 20.89H48c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h5.2c2.2 7.33 5.07 14.28 8.65 20.89l-3.67 3.67c-6.25 6.25-6.25 16.38 0 22.63l22.63 22.63c6.25 6.25 16.38 6.25 22.63 0l3.67-3.67c6.61 3.57 13.57 6.45 20.9 8.65v5.2c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-5.2c7.33-2.2 14.28-5.07 20.9-8.65l3.67 3.67c6.25 6.25 16.38 6.25 22.63 0l22.63-22.63c6.25-6.25 6.25-16.38 0-22.63l-3.67-3.67a110.85 110.85 0 0 0 8.65-20.89h5.2c8.84 0 16-7.16 16-16v-32C288 359.16 280.84 352 272 352zm-112 80c-26.51 0-48-21.49-48-48s21.49-48 48-48 48 21.49 48 48-21.49 48-48 48z"],"truck-moving":[640,512,[],"f4df","M621.3 237.3l-58.5-58.5c-12-12-28.3-18.7-45.3-18.7H480V64c0-17.7-14.3-32-32-32H32C14.3 32 0 46.3 0 64v336c0 44.2 35.8 80 80 80 26.3 0 49.4-12.9 64-32.4 14.6 19.6 37.7 32.4 64 32.4 44.2 0 80-35.8 80-80 0-5.5-.6-10.8-1.6-16h163.2c-1.1 5.2-1.6 10.5-1.6 16 0 44.2 35.8 80 80 80s80-35.8 80-80c0-5.5-.6-10.8-1.6-16H624c8.8 0 16-7.2 16-16v-85.5c0-17-6.7-33.2-18.7-45.2zM80 432c-17.6 0-32-14.4-32-32s14.4-32 32-32 32 14.4 32 32-14.4 32-32 32zm128 0c-17.6 0-32-14.4-32-32s14.4-32 32-32 32 14.4 32 32-14.4 32-32 32zm272-224h37.5c4.3 0 8.3 1.7 11.3 4.7l43.3 43.3H480v-48zm48 224c-17.6 0-32-14.4-32-32s14.4-32 32-32 32 14.4 32 32-14.4 32-32 32z"],"truck-pickup":[640,512,[],"f63c","M624 288h-16v-64c0-17.67-14.33-32-32-32h-48L419.22 56.02A64.025 64.025 0 0 0 369.24 32H256c-17.67 0-32 14.33-32 32v128H64c-17.67 0-32 14.33-32 32v64H16c-8.84 0-16 7.16-16 16v32c0 8.84 7.16 16 16 16h49.61c-.76 5.27-1.61 10.52-1.61 16 0 61.86 50.14 112 112 112s112-50.14 112-112c0-5.48-.85-10.73-1.61-16h67.23c-.76 5.27-1.61 10.52-1.61 16 0 61.86 50.14 112 112 112s112-50.14 112-112c0-5.48-.85-10.73-1.61-16H624c8.84 0 16-7.16 16-16v-32c0-8.84-7.16-16-16-16zM288 96h81.24l76.8 96H288V96zM176 416c-26.47 0-48-21.53-48-48s21.53-48 48-48 48 21.53 48 48-21.53 48-48 48zm288 0c-26.47 0-48-21.53-48-48s21.53-48 48-48 48 21.53 48 48-21.53 48-48 48z"],tshirt:[640,512,[],"f553","M631.2 96.5L436.5 0C416.4 27.8 371.9 47.2 320 47.2S223.6 27.8 203.5 0L8.8 96.5c-7.9 4-11.1 13.6-7.2 21.5l57.2 114.5c4 7.9 13.6 11.1 21.5 7.2l56.6-27.7c10.6-5.2 23 2.5 23 14.4V480c0 17.7 14.3 32 32 32h256c17.7 0 32-14.3 32-32V226.3c0-11.8 12.4-19.6 23-14.4l56.6 27.7c7.9 4 17.5.8 21.5-7.2L638.3 118c4-7.9.8-17.6-7.1-21.5z"],tty:[512,512,[],"f1e4","M5.37 103.822c138.532-138.532 362.936-138.326 501.262 0 6.078 6.078 7.074 15.496 2.583 22.681l-43.214 69.138a18.332 18.332 0 0 1-22.356 7.305l-86.422-34.569a18.335 18.335 0 0 1-11.434-18.846L351.741 90c-62.145-22.454-130.636-21.986-191.483 0l5.953 59.532a18.331 18.331 0 0 1-11.434 18.846l-86.423 34.568a18.334 18.334 0 0 1-22.356-7.305L2.787 126.502a18.333 18.333 0 0 1 2.583-22.68zM96 308v-40c0-6.627-5.373-12-12-12H44c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm-336 96v-40c0-6.627-5.373-12-12-12H92c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zM96 500v-40c0-6.627-5.373-12-12-12H44c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12zm288 0v-40c0-6.627-5.373-12-12-12H140c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h232c6.627 0 12-5.373 12-12zm96 0v-40c0-6.627-5.373-12-12-12h-40c-6.627 0-12 5.373-12 12v40c0 6.627 5.373 12 12 12h40c6.627 0 12-5.373 12-12z"],tv:[640,512,[],"f26c","M592 0H48A48 48 0 0 0 0 48v320a48 48 0 0 0 48 48h240v32H112a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16H352v-32h240a48 48 0 0 0 48-48V48a48 48 0 0 0-48-48zm-16 352H64V64h512z"],umbrella:[576,512,[],"f0e9","M575.7 280.8C547.1 144.5 437.3 62.6 320 49.9V32c0-17.7-14.3-32-32-32s-32 14.3-32 32v17.9C138.3 62.6 29.5 144.5.3 280.8c-2.2 10.1 8.5 21.3 18.7 11.4 52-55 107.7-52.4 158.6 37 5.3 9.5 14.9 8.6 19.7 0 20.2-35.4 44.9-73.2 90.7-73.2 58.5 0 88.2 68.8 90.7 73.2 4.8 8.6 14.4 9.5 19.7 0 51-89.5 107.1-91.4 158.6-37 10.3 10 20.9-1.3 18.7-11.4zM256 301.7V432c0 8.8-7.2 16-16 16-7.8 0-13.2-5.3-15.1-10.7-5.9-16.7-24.1-25.4-40.8-19.5-16.7 5.9-25.4 24.2-19.5 40.8 11.2 31.9 41.6 53.3 75.4 53.3 44.1 0 80-35.9 80-80V301.6c-9.1-7.9-19.8-13.6-32-13.6-12.3.1-22.4 4.8-32 13.7z"],"umbrella-beach":[640,512,[],"f5ca","M115.38 136.9l102.11 37.18c35.19-81.54 86.21-144.29 139-173.7-95.88-4.89-188.78 36.96-248.53 111.8-6.69 8.4-2.66 21.05 7.42 24.72zm132.25 48.16l238.48 86.83c35.76-121.38 18.7-231.66-42.63-253.98-7.4-2.7-15.13-4-23.09-4-58.02.01-128.27 69.17-172.76 171.15zM521.48 60.5c6.22 16.3 10.83 34.6 13.2 55.19 5.74 49.89-1.42 108.23-18.95 166.98l102.62 37.36c10.09 3.67 21.31-3.43 21.57-14.17 2.32-95.69-41.91-187.44-118.44-245.36zM560 447.98H321.06L386 269.5l-60.14-21.9-72.9 200.37H16c-8.84 0-16 7.16-16 16.01v32.01C0 504.83 7.16 512 16 512h544c8.84 0 16-7.17 16-16.01v-32.01c0-8.84-7.16-16-16-16z"],underline:[448,512,[],"f0cd","M32 64h32v160c0 88.22 71.78 160 160 160s160-71.78 160-160V64h32a16 16 0 0 0 16-16V16a16 16 0 0 0-16-16H272a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32v160a80 80 0 0 1-160 0V64h32a16 16 0 0 0 16-16V16a16 16 0 0 0-16-16H32a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16zm400 384H16a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"],undo:[512,512,[],"f0e2","M212.333 224.333H12c-6.627 0-12-5.373-12-12V12C0 5.373 5.373 0 12 0h48c6.627 0 12 5.373 12 12v78.112C117.773 39.279 184.26 7.47 258.175 8.007c136.906.994 246.448 111.623 246.157 248.532C504.041 393.258 393.12 504 256.333 504c-64.089 0-122.496-24.313-166.51-64.215-5.099-4.622-5.334-12.554-.467-17.42l33.967-33.967c4.474-4.474 11.662-4.717 16.401-.525C170.76 415.336 211.58 432 256.333 432c97.268 0 176-78.716 176-176 0-97.267-78.716-176-176-176-58.496 0-110.28 28.476-142.274 72.333h98.274c6.627 0 12 5.373 12 12v48c0 6.627-5.373 12-12 12z"],"undo-alt":[512,512,[],"f2ea","M255.545 8c-66.269.119-126.438 26.233-170.86 68.685L48.971 40.971C33.851 25.851 8 36.559 8 57.941V192c0 13.255 10.745 24 24 24h134.059c21.382 0 32.09-25.851 16.971-40.971l-41.75-41.75c30.864-28.899 70.801-44.907 113.23-45.273 92.398-.798 170.283 73.977 169.484 169.442C423.236 348.009 349.816 424 256 424c-41.127 0-79.997-14.678-110.63-41.556-4.743-4.161-11.906-3.908-16.368.553L89.34 422.659c-4.872 4.872-4.631 12.815.482 17.433C133.798 479.813 192.074 504 256 504c136.966 0 247.999-111.033 248-247.998C504.001 119.193 392.354 7.755 255.545 8z"],"universal-access":[512,512,[],"f29a","M256 48c114.953 0 208 93.029 208 208 0 114.953-93.029 208-208 208-114.953 0-208-93.029-208-208 0-114.953 93.029-208 208-208m0-40C119.033 8 8 119.033 8 256s111.033 248 248 248 248-111.033 248-248S392.967 8 256 8zm0 56C149.961 64 64 149.961 64 256s85.961 192 192 192 192-85.961 192-192S362.039 64 256 64zm0 44c19.882 0 36 16.118 36 36s-16.118 36-36 36-36-16.118-36-36 16.118-36 36-36zm117.741 98.023c-28.712 6.779-55.511 12.748-82.14 15.807.851 101.023 12.306 123.052 25.037 155.621 3.617 9.26-.957 19.698-10.217 23.315-9.261 3.617-19.699-.957-23.316-10.217-8.705-22.308-17.086-40.636-22.261-78.549h-9.686c-5.167 37.851-13.534 56.208-22.262 78.549-3.615 9.255-14.05 13.836-23.315 10.217-9.26-3.617-13.834-14.056-10.217-23.315 12.713-32.541 24.185-54.541 25.037-155.621-26.629-3.058-53.428-9.027-82.141-15.807-8.6-2.031-13.926-10.648-11.895-19.249s10.647-13.926 19.249-11.895c96.686 22.829 124.283 22.783 220.775 0 8.599-2.03 17.218 3.294 19.249 11.895 2.029 8.601-3.297 17.219-11.897 19.249z"],university:[512,512,[],"f19c","M496 128v16a8 8 0 0 1-8 8h-24v12c0 6.627-5.373 12-12 12H60c-6.627 0-12-5.373-12-12v-12H24a8 8 0 0 1-8-8v-16a8 8 0 0 1 4.941-7.392l232-88a7.996 7.996 0 0 1 6.118 0l232 88A8 8 0 0 1 496 128zm-24 304H40c-13.255 0-24 10.745-24 24v16a8 8 0 0 0 8 8h464a8 8 0 0 0 8-8v-16c0-13.255-10.745-24-24-24zM96 192v192H60c-6.627 0-12 5.373-12 12v20h416v-20c0-6.627-5.373-12-12-12h-36V192h-64v192h-64V192h-64v192h-64V192H96z"],unlink:[512,512,[],"f127","M304.083 405.907c4.686 4.686 4.686 12.284 0 16.971l-44.674 44.674c-59.263 59.262-155.693 59.266-214.961 0-59.264-59.265-59.264-155.696 0-214.96l44.675-44.675c4.686-4.686 12.284-4.686 16.971 0l39.598 39.598c4.686 4.686 4.686 12.284 0 16.971l-44.675 44.674c-28.072 28.073-28.072 73.75 0 101.823 28.072 28.072 73.75 28.073 101.824 0l44.674-44.674c4.686-4.686 12.284-4.686 16.971 0l39.597 39.598zm-56.568-260.216c4.686 4.686 12.284 4.686 16.971 0l44.674-44.674c28.072-28.075 73.75-28.073 101.824 0 28.072 28.073 28.072 73.75 0 101.823l-44.675 44.674c-4.686 4.686-4.686 12.284 0 16.971l39.598 39.598c4.686 4.686 12.284 4.686 16.971 0l44.675-44.675c59.265-59.265 59.265-155.695 0-214.96-59.266-59.264-155.695-59.264-214.961 0l-44.674 44.674c-4.686 4.686-4.686 12.284 0 16.971l39.597 39.598zm234.828 359.28l22.627-22.627c9.373-9.373 9.373-24.569 0-33.941L63.598 7.029c-9.373-9.373-24.569-9.373-33.941 0L7.029 29.657c-9.373 9.373-9.373 24.569 0 33.941l441.373 441.373c9.373 9.372 24.569 9.372 33.941 0z"],unlock:[448,512,[],"f09c","M400 256H152V152.9c0-39.6 31.7-72.5 71.3-72.9 40-.4 72.7 32.1 72.7 72v16c0 13.3 10.7 24 24 24h32c13.3 0 24-10.7 24-24v-16C376 68 307.5-.3 223.5 0 139.5.3 72 69.5 72 153.5V256H48c-26.5 0-48 21.5-48 48v160c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V304c0-26.5-21.5-48-48-48z"],"unlock-alt":[448,512,[],"f13e","M400 256H152V152.9c0-39.6 31.7-72.5 71.3-72.9 40-.4 72.7 32.1 72.7 72v16c0 13.3 10.7 24 24 24h32c13.3 0 24-10.7 24-24v-16C376 68 307.5-.3 223.5 0 139.5.3 72 69.5 72 153.5V256H48c-26.5 0-48 21.5-48 48v160c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V304c0-26.5-21.5-48-48-48zM264 408c0 22.1-17.9 40-40 40s-40-17.9-40-40v-48c0-22.1 17.9-40 40-40s40 17.9 40 40v48z"],upload:[512,512,[],"f093","M296 384h-80c-13.3 0-24-10.7-24-24V192h-87.7c-17.8 0-26.7-21.5-14.1-34.1L242.3 5.7c7.5-7.5 19.8-7.5 27.3 0l152.2 152.2c12.6 12.6 3.7 34.1-14.1 34.1H320v168c0 13.3-10.7 24-24 24zm216-8v112c0 13.3-10.7 24-24 24H24c-13.3 0-24-10.7-24-24V376c0-13.3 10.7-24 24-24h136v8c0 30.9 25.1 56 56 56h80c30.9 0 56-25.1 56-56v-8h136c13.3 0 24 10.7 24 24zm-124 88c0-11-9-20-20-20s-20 9-20 20 9 20 20 20 20-9 20-20zm64 0c0-11-9-20-20-20s-20 9-20 20 9 20 20 20 20-9 20-20z"],user:[448,512,[],"f007","M224 256c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm89.6 32h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48v-41.6c0-74.2-60.2-134.4-134.4-134.4z"],"user-alt":[512,512,[],"f406","M256 288c79.5 0 144-64.5 144-144S335.5 0 256 0 112 64.5 112 144s64.5 144 144 144zm128 32h-55.1c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16H128C57.3 320 0 377.3 0 448v16c0 26.5 21.5 48 48 48h416c26.5 0 48-21.5 48-48v-16c0-70.7-57.3-128-128-128z"],"user-alt-slash":[640,512,[],"f4fa","M633.8 458.1L389.6 269.3C433.8 244.7 464 198.1 464 144 464 64.5 399.5 0 320 0c-67.1 0-123 46.1-139 108.2L45.5 3.4C38.5-2 28.5-.8 23 6.2L3.4 31.4c-5.4 7-4.2 17 2.8 22.4l588.4 454.7c7 5.4 17 4.2 22.5-2.8l19.6-25.3c5.4-6.8 4.1-16.9-2.9-22.3zM198.4 320C124.2 320 64 380.2 64 454.4v9.6c0 26.5 21.5 48 48 48h382.2L245.8 320h-47.4z"],"user-astronaut":[448,512,[],"f4fb","M64 224h13.5c24.7 56.5 80.9 96 146.5 96s121.8-39.5 146.5-96H384c8.8 0 16-7.2 16-16v-96c0-8.8-7.2-16-16-16h-13.5C345.8 39.5 289.6 0 224 0S102.2 39.5 77.5 96H64c-8.8 0-16 7.2-16 16v96c0 8.8 7.2 16 16 16zm40-88c0-22.1 21.5-40 48-40h144c26.5 0 48 17.9 48 40v24c0 53-43 96-96 96h-48c-53 0-96-43-96-96v-24zm72 72l12-36 36-12-36-12-12-36-12 36-36 12 36 12 12 36zm151.6 113.4C297.7 340.7 262.2 352 224 352s-73.7-11.3-103.6-30.6C52.9 328.5 0 385 0 454.4v9.6c0 26.5 21.5 48 48 48h80v-64c0-17.7 14.3-32 32-32h128c17.7 0 32 14.3 32 32v64h80c26.5 0 48-21.5 48-48v-9.6c0-69.4-52.9-125.9-120.4-133zM272 448c-8.8 0-16 7.2-16 16s7.2 16 16 16 16-7.2 16-16-7.2-16-16-16zm-96 0c-8.8 0-16 7.2-16 16v48h32v-48c0-8.8-7.2-16-16-16z"],"user-check":[640,512,[],"f4fc","M224 256c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm89.6 32h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48v-41.6c0-74.2-60.2-134.4-134.4-134.4zm323-128.4l-27.8-28.1c-4.6-4.7-12.1-4.7-16.8-.1l-104.8 104-45.5-45.8c-4.6-4.7-12.1-4.7-16.8-.1l-28.1 27.9c-4.7 4.6-4.7 12.1-.1 16.8l81.7 82.3c4.6 4.7 12.1 4.7 16.8.1l141.3-140.2c4.6-4.7 4.7-12.2.1-16.8z"],"user-circle":[496,512,[],"f2bd","M248 8C111 8 0 119 0 256s111 248 248 248 248-111 248-248S385 8 248 8zm0 96c48.6 0 88 39.4 88 88s-39.4 88-88 88-88-39.4-88-88 39.4-88 88-88zm0 344c-58.7 0-111.3-26.6-146.5-68.2 18.8-35.4 55.6-59.8 98.5-59.8 2.4 0 4.8.4 7.1 1.1 13 4.2 26.6 6.9 40.9 6.9 14.3 0 28-2.7 40.9-6.9 2.3-.7 4.7-1.1 7.1-1.1 42.9 0 79.7 24.4 98.5 59.8C359.3 421.4 306.7 448 248 448z"],"user-clock":[640,512,[],"f4fd","M496 224c-79.6 0-144 64.4-144 144s64.4 144 144 144 144-64.4 144-144-64.4-144-144-144zm64 150.3c0 5.3-4.4 9.7-9.7 9.7h-60.6c-5.3 0-9.7-4.4-9.7-9.7v-76.6c0-5.3 4.4-9.7 9.7-9.7h12.6c5.3 0 9.7 4.4 9.7 9.7V352h38.3c5.3 0 9.7 4.4 9.7 9.7v12.6zM320 368c0-27.8 6.7-54.1 18.2-77.5-8-1.5-16.2-2.5-24.6-2.5h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h347.1c-45.3-31.9-75.1-84.5-75.1-144zm-96-112c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128z"],"user-cog":[640,512,[],"f4fe","M610.5 373.3c2.6-14.1 2.6-28.5 0-42.6l25.8-14.9c3-1.7 4.3-5.2 3.3-8.5-6.7-21.6-18.2-41.2-33.2-57.4-2.3-2.5-6-3.1-9-1.4l-25.8 14.9c-10.9-9.3-23.4-16.5-36.9-21.3v-29.8c0-3.4-2.4-6.4-5.7-7.1-22.3-5-45-4.8-66.2 0-3.3.7-5.7 3.7-5.7 7.1v29.8c-13.5 4.8-26 12-36.9 21.3l-25.8-14.9c-2.9-1.7-6.7-1.1-9 1.4-15 16.2-26.5 35.8-33.2 57.4-1 3.3.4 6.8 3.3 8.5l25.8 14.9c-2.6 14.1-2.6 28.5 0 42.6l-25.8 14.9c-3 1.7-4.3 5.2-3.3 8.5 6.7 21.6 18.2 41.1 33.2 57.4 2.3 2.5 6 3.1 9 1.4l25.8-14.9c10.9 9.3 23.4 16.5 36.9 21.3v29.8c0 3.4 2.4 6.4 5.7 7.1 22.3 5 45 4.8 66.2 0 3.3-.7 5.7-3.7 5.7-7.1v-29.8c13.5-4.8 26-12 36.9-21.3l25.8 14.9c2.9 1.7 6.7 1.1 9-1.4 15-16.2 26.5-35.8 33.2-57.4 1-3.3-.4-6.8-3.3-8.5l-25.8-14.9zM496 400.5c-26.8 0-48.5-21.8-48.5-48.5s21.8-48.5 48.5-48.5 48.5 21.8 48.5 48.5-21.7 48.5-48.5 48.5zM224 256c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm201.2 226.5c-2.3-1.2-4.6-2.6-6.8-3.9l-7.9 4.6c-6 3.4-12.8 5.3-19.6 5.3-10.9 0-21.4-4.6-28.9-12.6-18.3-19.8-32.3-43.9-40.2-69.6-5.5-17.7 1.9-36.4 17.9-45.7l7.9-4.6c-.1-2.6-.1-5.2 0-7.8l-7.9-4.6c-16-9.2-23.4-28-17.9-45.7.9-2.9 2.2-5.8 3.2-8.7-3.8-.3-7.5-1.2-11.4-1.2h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h352c10.1 0 19.5-3.2 27.2-8.5-1.2-3.8-2-7.7-2-11.8v-9.2z"],"user-edit":[640,512,[],"f4ff","M224 256c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm89.6 32h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h274.9c-2.4-6.8-3.4-14-2.6-21.3l6.8-60.9 1.2-11.1 7.9-7.9 77.3-77.3c-24.5-27.7-60-45.5-99.9-45.5zm45.3 145.3l-6.8 61c-1.1 10.2 7.5 18.8 17.6 17.6l60.9-6.8 137.9-137.9-71.7-71.7-137.9 137.8zM633 268.9L595.1 231c-9.3-9.3-24.5-9.3-33.8 0l-37.8 37.8-4.1 4.1 71.8 71.7 41.8-41.8c9.3-9.4 9.3-24.5 0-33.9z"],"user-friends":[640,512,[],"f500","M192 256c61.9 0 112-50.1 112-112S253.9 32 192 32 80 82.1 80 144s50.1 112 112 112zm76.8 32h-8.3c-20.8 10-43.9 16-68.5 16s-47.6-6-68.5-16h-8.3C51.6 288 0 339.6 0 403.2V432c0 26.5 21.5 48 48 48h288c26.5 0 48-21.5 48-48v-28.8c0-63.6-51.6-115.2-115.2-115.2zM480 256c53 0 96-43 96-96s-43-96-96-96-96 43-96 96 43 96 96 96zm48 32h-3.8c-13.9 4.8-28.6 8-44.2 8s-30.3-3.2-44.2-8H432c-20.4 0-39.2 5.9-55.7 15.4 24.4 26.3 39.7 61.2 39.7 99.8v38.4c0 2.2-.5 4.3-.6 6.4H592c26.5 0 48-21.5 48-48 0-61.9-50.1-112-112-112z"],"user-graduate":[448,512,[],"f501","M319.4 320.6L224 416l-95.4-95.4C57.1 323.7 0 382.2 0 454.4v9.6c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48v-9.6c0-72.2-57.1-130.7-128.6-133.8zM13.6 79.8l6.4 1.5v58.4c-7 4.2-12 11.5-12 20.3 0 8.4 4.6 15.4 11.1 19.7L3.5 242c-1.7 6.9 2.1 14 7.6 14h41.8c5.5 0 9.3-7.1 7.6-14l-15.6-62.3C51.4 175.4 56 168.4 56 160c0-8.8-5-16.1-12-20.3V87.1l66 15.9c-8.6 17.2-14 36.4-14 57 0 70.7 57.3 128 128 128s128-57.3 128-128c0-20.6-5.3-39.8-14-57l96.3-23.2c18.2-4.4 18.2-27.1 0-31.5l-190.4-46c-13-3.1-26.7-3.1-39.7 0L13.6 48.2c-18.1 4.4-18.1 27.2 0 31.6z"],"user-injured":[448,512,[],"f728","M277.37 11.98C261.08 4.47 243.11 0 224 0c-53.69 0-99.5 33.13-118.51 80h81.19l90.69-68.02zM342.51 80c-7.9-19.47-20.67-36.2-36.49-49.52L239.99 80h102.52zM224 256c70.69 0 128-57.31 128-128 0-5.48-.95-10.7-1.61-16H97.61c-.67 5.3-1.61 10.52-1.61 16 0 70.69 57.31 128 128 128zM80 299.7V512h128.26l-98.45-221.52A132.835 132.835 0 0 0 80 299.7zM0 464c0 26.51 21.49 48 48 48V320.24C18.88 344.89 0 381.26 0 422.4V464zm256-48h-55.38l42.67 96H256c26.47 0 48-21.53 48-48s-21.53-48-48-48zm57.6-128h-16.71c-22.24 10.18-46.88 16-72.89 16s-50.65-5.82-72.89-16h-7.37l42.67 96H256c44.11 0 80 35.89 80 80 0 18.08-6.26 34.59-16.41 48H400c26.51 0 48-21.49 48-48v-41.6c0-74.23-60.17-134.4-134.4-134.4z"],"user-lock":[640,512,[],"f502","M224 256A128 128 0 1 0 96 128a128 128 0 0 0 128 128zm96 64a63.08 63.08 0 0 1 8.1-30.5c-4.8-.5-9.5-1.5-14.5-1.5h-16.7a174.08 174.08 0 0 1-145.8 0h-16.7A134.43 134.43 0 0 0 0 422.4V464a48 48 0 0 0 48 48h280.9a63.54 63.54 0 0 1-8.9-32zm288-32h-32v-80a80 80 0 0 0-160 0v80h-32a32 32 0 0 0-32 32v160a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V320a32 32 0 0 0-32-32zM496 432a32 32 0 1 1 32-32 32 32 0 0 1-32 32zm32-144h-64v-80a32 32 0 0 1 64 0z"],"user-md":[448,512,[],"f0f0","M224 256c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zM104 424c0 13.3 10.7 24 24 24s24-10.7 24-24-10.7-24-24-24-24 10.7-24 24zm216-135.4v49c36.5 7.4 64 39.8 64 78.4v41.7c0 7.6-5.4 14.2-12.9 15.7l-32.2 6.4c-4.3.9-8.5-1.9-9.4-6.3l-3.1-15.7c-.9-4.3 1.9-8.6 6.3-9.4l19.3-3.9V416c0-62.8-96-65.1-96 1.9v26.7l19.3 3.9c4.3.9 7.1 5.1 6.3 9.4l-3.1 15.7c-.9 4.3-5.1 7.1-9.4 6.3l-31.2-4.2c-7.9-1.1-13.8-7.8-13.8-15.9V416c0-38.6 27.5-70.9 64-78.4v-45.2c-2.2.7-4.4 1.1-6.6 1.9-18 6.3-37.3 9.8-57.4 9.8s-39.4-3.5-57.4-9.8c-7.4-2.6-14.9-4.2-22.6-5.2v81.6c23.1 6.9 40 28.1 40 53.4 0 30.9-25.1 56-56 56s-56-25.1-56-56c0-25.3 16.9-46.5 40-53.4v-80.4C48.5 301 0 355.8 0 422.4v44.8C0 491.9 20.1 512 44.8 512h358.4c24.7 0 44.8-20.1 44.8-44.8v-44.8c0-72-56.8-130.3-128-133.8z"],"user-minus":[640,512,[],"f503","M624 208H432c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h192c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zm-400 48c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm89.6 32h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48v-41.6c0-74.2-60.2-134.4-134.4-134.4z"],"user-ninja":[448,512,[],"f504","M325.4 289.2L224 390.6 122.6 289.2C54 295.3 0 352.2 0 422.4V464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48v-41.6c0-70.2-54-127.1-122.6-133.2zM32 192c27.3 0 51.8-11.5 69.2-29.7 15.1 53.9 64 93.7 122.8 93.7 70.7 0 128-57.3 128-128S294.7 0 224 0c-50.4 0-93.6 29.4-114.5 71.8C92.1 47.8 64 32 32 32c0 33.4 17.1 62.8 43.1 80-26 17.2-43.1 46.6-43.1 80zm144-96h96c17.7 0 32 14.3 32 32H144c0-17.7 14.3-32 32-32z"],"user-nurse":[448,512,[],"f82f","M319.41,320,224,415.39,128.59,320C57.1,323.1,0,381.6,0,453.79A58.21,58.21,0,0,0,58.21,512H389.79A58.21,58.21,0,0,0,448,453.79C448,381.6,390.9,323.1,319.41,320ZM224,304A128,128,0,0,0,352,176V65.82a32,32,0,0,0-20.76-30L246.47,4.07a64,64,0,0,0-44.94,0L116.76,35.86A32,32,0,0,0,96,65.82V176A128,128,0,0,0,224,304ZM184,71.67a5,5,0,0,1,5-5h21.67V45a5,5,0,0,1,5-5h16.66a5,5,0,0,1,5,5V66.67H259a5,5,0,0,1,5,5V88.33a5,5,0,0,1-5,5H237.33V115a5,5,0,0,1-5,5H215.67a5,5,0,0,1-5-5V93.33H189a5,5,0,0,1-5-5ZM144,160H304v16a80,80,0,0,1-160,0Z"],"user-plus":[640,512,[],"f234","M624 208h-64v-64c0-8.8-7.2-16-16-16h-32c-8.8 0-16 7.2-16 16v64h-64c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h64v64c0 8.8 7.2 16 16 16h32c8.8 0 16-7.2 16-16v-64h64c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zm-400 48c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm89.6 32h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48v-41.6c0-74.2-60.2-134.4-134.4-134.4z"],"user-secret":[448,512,[],"f21b","M383.9 308.3l23.9-62.6c4-10.5-3.7-21.7-15-21.7h-58.5c11-18.9 17.8-40.6 17.8-64v-.3c39.2-7.8 64-19.1 64-31.7 0-13.3-27.3-25.1-70.1-33-9.2-32.8-27-65.8-40.6-82.8-9.5-11.9-25.9-15.6-39.5-8.8l-27.6 13.8c-9 4.5-19.6 4.5-28.6 0L182.1 3.4c-13.6-6.8-30-3.1-39.5 8.8-13.5 17-31.4 50-40.6 82.8-42.7 7.9-70 19.7-70 33 0 12.6 24.8 23.9 64 31.7v.3c0 23.4 6.8 45.1 17.8 64H56.3c-11.5 0-19.2 11.7-14.7 22.3l25.8 60.2C27.3 329.8 0 372.7 0 422.4v44.8C0 491.9 20.1 512 44.8 512h358.4c24.7 0 44.8-20.1 44.8-44.8v-44.8c0-48.4-25.8-90.4-64.1-114.1zM176 480l-41.6-192 49.6 32 24 40-32 120zm96 0l-32-120 24-40 49.6-32L272 480zm41.7-298.5c-3.9 11.9-7 24.6-16.5 33.4-10.1 9.3-48 22.4-64-25-2.8-8.4-15.4-8.4-18.3 0-17 50.2-56 32.4-64 25-9.5-8.8-12.7-21.5-16.5-33.4-.8-2.5-6.3-5.7-6.3-5.8v-10.8c28.3 3.6 61 5.8 96 5.8s67.7-2.1 96-5.8v10.8c-.1.1-5.6 3.2-6.4 5.8z"],"user-shield":[640,512,[],"f505","M622.3 271.1l-115.2-45c-4.1-1.6-12.6-3.7-22.2 0l-115.2 45c-10.7 4.2-17.7 14-17.7 24.9 0 111.6 68.7 188.8 132.9 213.9 9.6 3.7 18 1.6 22.2 0C558.4 489.9 640 420.5 640 296c0-10.9-7-20.7-17.7-24.9zM496 462.4V273.3l95.5 37.3c-5.6 87.1-60.9 135.4-95.5 151.8zM224 256c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm96 40c0-2.5.8-4.8 1.1-7.2-2.5-.1-4.9-.8-7.5-.8h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h352c6.8 0 13.3-1.5 19.2-4-54-42.9-99.2-116.7-99.2-212z"],"user-slash":[640,512,[],"f506","M633.8 458.1L362.3 248.3C412.1 230.7 448 183.8 448 128 448 57.3 390.7 0 320 0c-67.1 0-121.5 51.8-126.9 117.4L45.5 3.4C38.5-2 28.5-.8 23 6.2L3.4 31.4c-5.4 7-4.2 17 2.8 22.4l588.4 454.7c7 5.4 17 4.2 22.5-2.8l19.6-25.3c5.4-6.8 4.1-16.9-2.9-22.3zM96 422.4V464c0 26.5 21.5 48 48 48h350.2L207.4 290.3C144.2 301.3 96 356 96 422.4z"],"user-tag":[640,512,[],"f507","M630.6 364.9l-90.3-90.2c-12-12-28.3-18.7-45.3-18.7h-79.3c-17.7 0-32 14.3-32 32v79.2c0 17 6.7 33.2 18.7 45.2l90.3 90.2c12.5 12.5 32.8 12.5 45.3 0l92.5-92.5c12.6-12.5 12.6-32.7.1-45.2zm-182.8-21c-13.3 0-24-10.7-24-24s10.7-24 24-24 24 10.7 24 24c0 13.2-10.7 24-24 24zm-223.8-88c70.7 0 128-57.3 128-128C352 57.3 294.7 0 224 0S96 57.3 96 128c0 70.6 57.3 127.9 128 127.9zm127.8 111.2V294c-12.2-3.6-24.9-6.2-38.2-6.2h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 287.9 0 348.1 0 422.3v41.6c0 26.5 21.5 48 48 48h352c15.5 0 29.1-7.5 37.9-18.9l-58-58c-18.1-18.1-28.1-42.2-28.1-67.9z"],"user-tie":[448,512,[],"f508","M224 256c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm95.8 32.6L272 480l-32-136 32-56h-96l32 56-32 136-47.8-191.4C56.9 292 0 350.3 0 422.4V464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48v-41.6c0-72.1-56.9-130.4-128.2-133.8z"],"user-times":[640,512,[],"f235","M589.6 240l45.6-45.6c6.3-6.3 6.3-16.5 0-22.8l-22.8-22.8c-6.3-6.3-16.5-6.3-22.8 0L544 194.4l-45.6-45.6c-6.3-6.3-16.5-6.3-22.8 0l-22.8 22.8c-6.3 6.3-6.3 16.5 0 22.8l45.6 45.6-45.6 45.6c-6.3 6.3-6.3 16.5 0 22.8l22.8 22.8c6.3 6.3 16.5 6.3 22.8 0l45.6-45.6 45.6 45.6c6.3 6.3 16.5 6.3 22.8 0l22.8-22.8c6.3-6.3 6.3-16.5 0-22.8L589.6 240zM224 256c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm89.6 32h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48v-41.6c0-74.2-60.2-134.4-134.4-134.4z"],users:[640,512,[],"f0c0","M96 224c35.3 0 64-28.7 64-64s-28.7-64-64-64-64 28.7-64 64 28.7 64 64 64zm448 0c35.3 0 64-28.7 64-64s-28.7-64-64-64-64 28.7-64 64 28.7 64 64 64zm32 32h-64c-17.6 0-33.5 7.1-45.1 18.6 40.3 22.1 68.9 62 75.1 109.4h66c17.7 0 32-14.3 32-32v-32c0-35.3-28.7-64-64-64zm-256 0c61.9 0 112-50.1 112-112S381.9 32 320 32 208 82.1 208 144s50.1 112 112 112zm76.8 32h-8.3c-20.8 10-43.9 16-68.5 16s-47.6-6-68.5-16h-8.3C179.6 288 128 339.6 128 403.2V432c0 26.5 21.5 48 48 48h288c26.5 0 48-21.5 48-48v-28.8c0-63.6-51.6-115.2-115.2-115.2zm-223.7-13.4C161.5 263.1 145.6 256 128 256H64c-35.3 0-64 28.7-64 64v32c0 17.7 14.3 32 32 32h65.9c6.3-47.4 34.9-87.3 75.2-109.4z"],"users-cog":[640,512,[],"f509","M610.5 341.3c2.6-14.1 2.6-28.5 0-42.6l25.8-14.9c3-1.7 4.3-5.2 3.3-8.5-6.7-21.6-18.2-41.2-33.2-57.4-2.3-2.5-6-3.1-9-1.4l-25.8 14.9c-10.9-9.3-23.4-16.5-36.9-21.3v-29.8c0-3.4-2.4-6.4-5.7-7.1-22.3-5-45-4.8-66.2 0-3.3.7-5.7 3.7-5.7 7.1v29.8c-13.5 4.8-26 12-36.9 21.3l-25.8-14.9c-2.9-1.7-6.7-1.1-9 1.4-15 16.2-26.5 35.8-33.2 57.4-1 3.3.4 6.8 3.3 8.5l25.8 14.9c-2.6 14.1-2.6 28.5 0 42.6l-25.8 14.9c-3 1.7-4.3 5.2-3.3 8.5 6.7 21.6 18.2 41.1 33.2 57.4 2.3 2.5 6 3.1 9 1.4l25.8-14.9c10.9 9.3 23.4 16.5 36.9 21.3v29.8c0 3.4 2.4 6.4 5.7 7.1 22.3 5 45 4.8 66.2 0 3.3-.7 5.7-3.7 5.7-7.1v-29.8c13.5-4.8 26-12 36.9-21.3l25.8 14.9c2.9 1.7 6.7 1.1 9-1.4 15-16.2 26.5-35.8 33.2-57.4 1-3.3-.4-6.8-3.3-8.5l-25.8-14.9zM496 368.5c-26.8 0-48.5-21.8-48.5-48.5s21.8-48.5 48.5-48.5 48.5 21.8 48.5 48.5-21.7 48.5-48.5 48.5zM96 224c35.3 0 64-28.7 64-64s-28.7-64-64-64-64 28.7-64 64 28.7 64 64 64zm224 32c1.9 0 3.7-.5 5.6-.6 8.3-21.7 20.5-42.1 36.3-59.2 7.4-8 17.9-12.6 28.9-12.6 6.9 0 13.7 1.8 19.6 5.3l7.9 4.6c.8-.5 1.6-.9 2.4-1.4 7-14.6 11.2-30.8 11.2-48 0-61.9-50.1-112-112-112S208 82.1 208 144c0 61.9 50.1 112 112 112zm105.2 194.5c-2.3-1.2-4.6-2.6-6.8-3.9-8.2 4.8-15.3 9.8-27.5 9.8-10.9 0-21.4-4.6-28.9-12.6-18.3-19.8-32.3-43.9-40.2-69.6-10.7-34.5 24.9-49.7 25.8-50.3-.1-2.6-.1-5.2 0-7.8l-7.9-4.6c-3.8-2.2-7-5-9.8-8.1-3.3.2-6.5.6-9.8.6-24.6 0-47.6-6-68.5-16h-8.3C179.6 288 128 339.6 128 403.2V432c0 26.5 21.5 48 48 48h255.4c-3.7-6-6.2-12.8-6.2-20.3v-9.2zM173.1 274.6C161.5 263.1 145.6 256 128 256H64c-35.3 0-64 28.7-64 64v32c0 17.7 14.3 32 32 32h65.9c6.3-47.4 34.9-87.3 75.2-109.4z"],"users-slash":[640,512,[],"f973","M132.65,212.32,36.21,137.78A63.4,63.4,0,0,0,32,160a63.84,63.84,0,0,0,100.65,52.32Zm40.44,62.28A63.79,63.79,0,0,0,128,256H64A64.06,64.06,0,0,0,0,320v32a32,32,0,0,0,32,32H97.91A146.62,146.62,0,0,1,173.09,274.6ZM544,224a64,64,0,1,0-64-64A64.06,64.06,0,0,0,544,224ZM500.56,355.11a114.24,114.24,0,0,0-84.47-65.28L361,247.23c41.46-16.3,71-55.92,71-103.23A111.93,111.93,0,0,0,320,32c-57.14,0-103.69,42.83-110.6,98.08L45.46,3.38A16,16,0,0,0,23,6.19L3.37,31.46A16,16,0,0,0,6.18,53.91L594.53,508.63A16,16,0,0,0,617,505.82l19.64-25.27a16,16,0,0,0-2.81-22.45ZM128,403.21V432a48,48,0,0,0,48,48H464a47.45,47.45,0,0,0,12.57-1.87L232,289.13C173.74,294.83,128,343.42,128,403.21ZM576,256H512a63.79,63.79,0,0,0-45.09,18.6A146.29,146.29,0,0,1,542,384h66a32,32,0,0,0,32-32V320A64.06,64.06,0,0,0,576,256Z"],"utensil-spoon":[512,512,[],"f2e5","M480.1 31.9c-55-55.1-164.9-34.5-227.8 28.5-49.3 49.3-55.1 110-28.8 160.4L9 413.2c-11.6 10.5-12.1 28.5-1 39.5L59.3 504c11 11 29.1 10.5 39.5-1.1l192.4-214.4c50.4 26.3 111.1 20.5 160.4-28.8 63-62.9 83.6-172.8 28.5-227.8z"],utensils:[416,512,[],"f2e7","M207.9 15.2c.8 4.7 16.1 94.5 16.1 128.8 0 52.3-27.8 89.6-68.9 104.6L168 486.7c.7 13.7-10.2 25.3-24 25.3H80c-13.7 0-24.7-11.5-24-25.3l12.9-238.1C27.7 233.6 0 196.2 0 144 0 109.6 15.3 19.9 16.1 15.2 19.3-5.1 61.4-5.4 64 16.3v141.2c1.3 3.4 15.1 3.2 16 0 1.4-25.3 7.9-139.2 8-141.8 3.3-20.8 44.7-20.8 47.9 0 .2 2.7 6.6 116.5 8 141.8.9 3.2 14.8 3.4 16 0V16.3c2.6-21.6 44.8-21.4 48-1.1zm119.2 285.7l-15 185.1c-1.2 14 9.9 26 23.9 26h56c13.3 0 24-10.7 24-24V24c0-13.2-10.7-24-24-24-82.5 0-221.4 178.5-64.9 300.9z"],"vector-square":[512,512,[],"f5cb","M512 128V32c0-17.67-14.33-32-32-32h-96c-17.67 0-32 14.33-32 32H160c0-17.67-14.33-32-32-32H32C14.33 0 0 14.33 0 32v96c0 17.67 14.33 32 32 32v192c-17.67 0-32 14.33-32 32v96c0 17.67 14.33 32 32 32h96c17.67 0 32-14.33 32-32h192c0 17.67 14.33 32 32 32h96c17.67 0 32-14.33 32-32v-96c0-17.67-14.33-32-32-32V160c17.67 0 32-14.33 32-32zm-96-64h32v32h-32V64zM64 64h32v32H64V64zm32 384H64v-32h32v32zm352 0h-32v-32h32v32zm-32-96h-32c-17.67 0-32 14.33-32 32v32H160v-32c0-17.67-14.33-32-32-32H96V160h32c17.67 0 32-14.33 32-32V96h192v32c0 17.67 14.33 32 32 32h32v192z"],venus:[288,512,[],"f221","M288 176c0-79.5-64.5-144-144-144S0 96.5 0 176c0 68.5 47.9 125.9 112 140.4V368H76c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h36v36c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12v-36h36c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-36v-51.6c64.1-14.5 112-71.9 112-140.4zm-224 0c0-44.1 35.9-80 80-80s80 35.9 80 80-35.9 80-80 80-80-35.9-80-80z"],"venus-double":[512,512,[],"f226","M288 176c0-79.5-64.5-144-144-144S0 96.5 0 176c0 68.5 47.9 125.9 112 140.4V368H76c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h36v36c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12v-36h36c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-36v-51.6c64.1-14.5 112-71.9 112-140.4zm-224 0c0-44.1 35.9-80 80-80s80 35.9 80 80-35.9 80-80 80-80-35.9-80-80zm336 140.4V368h36c6.6 0 12 5.4 12 12v40c0 6.6-5.4 12-12 12h-36v36c0 6.6-5.4 12-12 12h-40c-6.6 0-12-5.4-12-12v-36h-36c-6.6 0-12-5.4-12-12v-40c0-6.6 5.4-12 12-12h36v-51.6c-21.2-4.8-40.6-14.3-57.2-27.3 14-16.7 25-36 32.1-57.1 14.5 14.8 34.7 24 57.1 24 44.1 0 80-35.9 80-80s-35.9-80-80-80c-22.3 0-42.6 9.2-57.1 24-7.1-21.1-18-40.4-32.1-57.1C303.4 43.6 334.3 32 368 32c79.5 0 144 64.5 144 144 0 68.5-47.9 125.9-112 140.4z"],"venus-mars":[576,512,[],"f228","M564 0h-79c-10.7 0-16 12.9-8.5 20.5l16.9 16.9-48.7 48.7C422.5 72.1 396.2 64 368 64c-33.7 0-64.6 11.6-89.2 30.9 14 16.7 25 36 32.1 57.1 14.5-14.8 34.7-24 57.1-24 44.1 0 80 35.9 80 80s-35.9 80-80 80c-22.3 0-42.6-9.2-57.1-24-7.1 21.1-18 40.4-32.1 57.1 24.5 19.4 55.5 30.9 89.2 30.9 79.5 0 144-64.5 144-144 0-28.2-8.1-54.5-22.1-76.7l48.7-48.7 16.9 16.9c2.4 2.4 5.4 3.5 8.4 3.5 6.2 0 12.1-4.8 12.1-12V12c0-6.6-5.4-12-12-12zM144 64C64.5 64 0 128.5 0 208c0 68.5 47.9 125.9 112 140.4V400H76c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h36v36c0 6.6 5.4 12 12 12h40c6.6 0 12-5.4 12-12v-36h36c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-36v-51.6c64.1-14.6 112-71.9 112-140.4 0-79.5-64.5-144-144-144zm0 224c-44.1 0-80-35.9-80-80s35.9-80 80-80 80 35.9 80 80-35.9 80-80 80z"],vial:[480,512,[],"f492","M477.7 186.1L309.5 18.3c-3.1-3.1-8.2-3.1-11.3 0l-34 33.9c-3.1 3.1-3.1 8.2 0 11.3l11.2 11.1L33 316.5c-38.8 38.7-45.1 102-9.4 143.5 20.6 24 49.5 36 78.4 35.9 26.4 0 52.8-10 72.9-30.1l246.3-245.7 11.2 11.1c3.1 3.1 8.2 3.1 11.3 0l34-33.9c3.1-3 3.1-8.1 0-11.2zM318 256H161l148-147.7 78.5 78.3L318 256z"],vials:[640,512,[],"f493","M72 64h24v240c0 44.1 35.9 80 80 80s80-35.9 80-80V64h24c4.4 0 8-3.6 8-8V8c0-4.4-3.6-8-8-8H72c-4.4 0-8 3.6-8 8v48c0 4.4 3.6 8 8 8zm72 0h64v96h-64V64zm480 384H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h608c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zM360 64h24v240c0 44.1 35.9 80 80 80s80-35.9 80-80V64h24c4.4 0 8-3.6 8-8V8c0-4.4-3.6-8-8-8H360c-4.4 0-8 3.6-8 8v48c0 4.4 3.6 8 8 8zm72 0h64v96h-64V64z"],video:[576,512,[],"f03d","M336.2 64H47.8C21.4 64 0 85.4 0 111.8v288.4C0 426.6 21.4 448 47.8 448h288.4c26.4 0 47.8-21.4 47.8-47.8V111.8c0-26.4-21.4-47.8-47.8-47.8zm189.4 37.7L416 177.3v157.4l109.6 75.5c21.2 14.6 50.4-.3 50.4-25.8V127.5c0-25.4-29.1-40.4-50.4-25.8z"],"video-slash":[640,512,[],"f4e2","M633.8 458.1l-55-42.5c15.4-1.4 29.2-13.7 29.2-31.1v-257c0-25.5-29.1-40.4-50.4-25.8L448 177.3v137.2l-32-24.7v-178c0-26.4-21.4-47.8-47.8-47.8H123.9L45.5 3.4C38.5-2 28.5-.8 23 6.2L3.4 31.4c-5.4 7-4.2 17 2.8 22.4L42.7 82 416 370.6l178.5 138c7 5.4 17 4.2 22.5-2.8l19.6-25.3c5.5-6.9 4.2-17-2.8-22.4zM32 400.2c0 26.4 21.4 47.8 47.8 47.8h288.4c11.2 0 21.4-4 29.6-10.5L32 154.7v245.5z"],vihara:[640,512,[],"f6a7","M632.88 400.71L544 352v-64l55.16-17.69c11.79-5.9 11.79-22.72 0-28.62L480 192v-64l27.31-16.3c7.72-7.72 5.61-20.74-4.16-25.62L320 0 136.85 86.07c-9.77 4.88-11.88 17.9-4.16 25.62L160 128v64L40.84 241.69c-11.79 5.9-11.79 22.72 0 28.62L96 288v64L7.12 400.71c-5.42 3.62-7.7 9.63-7 15.29.62 5.01 3.57 9.75 8.72 12.33L64 448v48c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-48h160v48c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-48h160v48c0 8.84 7.16 16 16 16h32c8.84 0 16-7.16 16-16v-48l55.15-19.67c5.16-2.58 8.1-7.32 8.72-12.33.71-5.67-1.57-11.68-6.99-15.29zM224 128h192v64H224v-64zm-64 224v-64h320v64H160z"],virus:[512,512,[],"f974","M483.55,227.55H462c-50.68,0-76.07-61.27-40.23-97.11L437,115.19A28.44,28.44,0,0,0,396.8,75L381.56,90.22c-35.84,35.83-97.11,10.45-97.11-40.23V28.44a28.45,28.45,0,0,0-56.9,0V50c0,50.68-61.27,76.06-97.11,40.23L115.2,75A28.44,28.44,0,0,0,75,115.19l15.25,15.25c35.84,35.84,10.45,97.11-40.23,97.11H28.45a28.45,28.45,0,1,0,0,56.89H50c50.68,0,76.07,61.28,40.23,97.12L75,396.8A28.45,28.45,0,0,0,115.2,437l15.24-15.25c35.84-35.84,97.11-10.45,97.11,40.23v21.54a28.45,28.45,0,0,0,56.9,0V462c0-50.68,61.27-76.07,97.11-40.23L396.8,437A28.45,28.45,0,0,0,437,396.8l-15.25-15.24c-35.84-35.84-10.45-97.12,40.23-97.12h21.54a28.45,28.45,0,1,0,0-56.89ZM224,272a48,48,0,1,1,48-48A48,48,0,0,1,224,272Zm80,56a24,24,0,1,1,24-24A24,24,0,0,1,304,328Z"],"virus-slash":[640,512,[],"f975","M114,227.56H92.44a28.44,28.44,0,0,0,0,56.88H114c50.68,0,76.06,61.28,40.23,97.12L139,396.81A28.44,28.44,0,1,0,179.19,437l15.25-15.25c35.84-35.84,97.11-10.45,97.11,40.23v21.54a28.45,28.45,0,0,0,56.9,0V462c0-26.61,17-45.91,38.22-53.37l-244.5-189A55.58,55.58,0,0,1,114,227.56ZM633.82,458.09,470.62,332c4.17-25.39,24.91-47.52,55.39-47.52h21.55a28.44,28.44,0,1,0,0-56.88H526c-50.68,0-76.06-61.28-40.23-97.12L501,115.19A28.44,28.44,0,0,0,460.81,75L445.56,90.22c-35.84,35.84-97.11,10.46-97.11-40.23V28.45a28.45,28.45,0,0,0-56.9,0V50c0,50.69-61.27,76.07-97.11,40.23L179.19,75A28.43,28.43,0,0,0,139,75c-.13.14-.15.32-.28.46L45.46,3.38A16,16,0,0,0,23,6.19L3.37,31.45A16,16,0,0,0,6.18,53.91L594.54,508.63A16,16,0,0,0,617,505.81l19.64-25.26A16,16,0,0,0,633.82,458.09ZM335.43,227.48l-62.87-48.59A46.55,46.55,0,0,1,288,176a48,48,0,0,1,48,48C336,225.22,335.52,226.29,335.43,227.48Z"],viruses:[640,512,[],"f976","M624,352H611.88c-28.51,0-42.79-34.47-22.63-54.63l8.58-8.57a16,16,0,1,0-22.63-22.63l-8.57,8.58C546.47,294.91,512,280.63,512,252.12V240a16,16,0,0,0-32,0v12.12c0,28.51-34.47,42.79-54.63,22.63l-8.57-8.58a16,16,0,0,0-22.63,22.63l8.58,8.57c20.16,20.16,5.88,54.63-22.63,54.63H368a16,16,0,0,0,0,32h12.12c28.51,0,42.79,34.47,22.63,54.63l-8.58,8.57a16,16,0,1,0,22.63,22.63l8.57-8.58c20.16-20.16,54.63-5.88,54.63,22.63V496a16,16,0,0,0,32,0V483.88c0-28.51,34.47-42.79,54.63-22.63l8.57,8.58a16,16,0,1,0,22.63-22.63l-8.58-8.57C569.09,418.47,583.37,384,611.88,384H624a16,16,0,0,0,0-32ZM480,384a32,32,0,1,1,32-32A32,32,0,0,1,480,384ZM346.51,213.33h16.16a21.33,21.33,0,0,0,0-42.66H346.51c-38,0-57.05-46-30.17-72.84l11.43-11.44A21.33,21.33,0,0,0,297.6,56.23L286.17,67.66c-26.88,26.88-72.84,7.85-72.84-30.17V21.33a21.33,21.33,0,0,0-42.66,0V37.49c0,38-46,57.05-72.84,30.17L86.4,56.23A21.33,21.33,0,0,0,56.23,86.39L67.66,97.83c26.88,26.88,7.85,72.84-30.17,72.84H21.33a21.33,21.33,0,0,0,0,42.66H37.49c38,0,57.05,46,30.17,72.84L56.23,297.6A21.33,21.33,0,1,0,86.4,327.77l11.43-11.43c26.88-26.88,72.84-7.85,72.84,30.17v16.16a21.33,21.33,0,0,0,42.66,0V346.51c0-38,46-57.05,72.84-30.17l11.43,11.43a21.33,21.33,0,0,0,30.17-30.17l-11.43-11.43C289.46,259.29,308.49,213.33,346.51,213.33ZM160,192a32,32,0,1,1,32-32A32,32,0,0,1,160,192Zm80,32a16,16,0,1,1,16-16A16,16,0,0,1,240,224Z"],voicemail:[640,512,[],"f897","M496 128a144 144 0 0 0-119.74 224H263.74A144 144 0 1 0 144 416h352a144 144 0 0 0 0-288zM64 272a80 80 0 1 1 80 80 80 80 0 0 1-80-80zm432 80a80 80 0 1 1 80-80 80 80 0 0 1-80 80z"],"volleyball-ball":[512,512,[],"f45f","M231.39 243.48a285.56 285.56 0 0 0-22.7-105.7c-90.8 42.4-157.5 122.4-180.3 216.8a249 249 0 0 0 56.9 81.1 333.87 333.87 0 0 1 146.1-192.2zm-36.9-134.4a284.23 284.23 0 0 0-57.4-70.7c-91 49.8-144.8 152.9-125 262.2 33.4-83.1 98.4-152 182.4-191.5zm187.6 165.1c8.6-99.8-27.3-197.5-97.5-264.4-14.7-1.7-51.6-5.5-98.9 8.5A333.87 333.87 0 0 1 279.19 241a285 285 0 0 0 102.9 33.18zm-124.7 9.5a286.33 286.33 0 0 0-80.2 72.6c82 57.3 184.5 75.1 277.5 47.8a247.15 247.15 0 0 0 42.2-89.9 336.1 336.1 0 0 1-80.9 10.4c-54.6-.1-108.9-14.1-158.6-40.9zm-98.3 99.7c-15.2 26-25.7 54.4-32.1 84.2a247.07 247.07 0 0 0 289-22.1c-112.9 16.1-203.3-24.8-256.9-62.1zm180.3-360.6c55.3 70.4 82.5 161.2 74.6 253.6a286.59 286.59 0 0 0 89.7-14.2c0-2 .3-4 .3-6 0-107.8-68.7-199.1-164.6-233.4z"],"volume-down":[384,512,[],"f027","M215.03 72.04L126.06 161H24c-13.26 0-24 10.74-24 24v144c0 13.25 10.74 24 24 24h102.06l88.97 88.95c15.03 15.03 40.97 4.47 40.97-16.97V89.02c0-21.47-25.96-31.98-40.97-16.98zm123.2 108.08c-11.58-6.33-26.19-2.16-32.61 9.45-6.39 11.61-2.16 26.2 9.45 32.61C327.98 229.28 336 242.62 336 257c0 14.38-8.02 27.72-20.92 34.81-11.61 6.41-15.84 21-9.45 32.61 6.43 11.66 21.05 15.8 32.61 9.45 28.23-15.55 45.77-45 45.77-76.88s-17.54-61.32-45.78-76.87z"],"volume-mute":[512,512,[],"f6a9","M215.03 71.05L126.06 160H24c-13.26 0-24 10.74-24 24v144c0 13.25 10.74 24 24 24h102.06l88.97 88.95c15.03 15.03 40.97 4.47 40.97-16.97V88.02c0-21.46-25.96-31.98-40.97-16.97zM461.64 256l45.64-45.64c6.3-6.3 6.3-16.52 0-22.82l-22.82-22.82c-6.3-6.3-16.52-6.3-22.82 0L416 210.36l-45.64-45.64c-6.3-6.3-16.52-6.3-22.82 0l-22.82 22.82c-6.3 6.3-6.3 16.52 0 22.82L370.36 256l-45.63 45.63c-6.3 6.3-6.3 16.52 0 22.82l22.82 22.82c6.3 6.3 16.52 6.3 22.82 0L416 301.64l45.64 45.64c6.3 6.3 16.52 6.3 22.82 0l22.82-22.82c6.3-6.3 6.3-16.52 0-22.82L461.64 256z"],"volume-off":[256,512,[],"f026","M215 71l-89 89H24a24 24 0 0 0-24 24v144a24 24 0 0 0 24 24h102.06L215 441c15 15 41 4.47 41-17V88c0-21.47-26-32-41-17z"],"volume-up":[576,512,[],"f028","M215.03 71.05L126.06 160H24c-13.26 0-24 10.74-24 24v144c0 13.25 10.74 24 24 24h102.06l88.97 88.95c15.03 15.03 40.97 4.47 40.97-16.97V88.02c0-21.46-25.96-31.98-40.97-16.97zm233.32-51.08c-11.17-7.33-26.18-4.24-33.51 6.95-7.34 11.17-4.22 26.18 6.95 33.51 66.27 43.49 105.82 116.6 105.82 195.58 0 78.98-39.55 152.09-105.82 195.58-11.17 7.32-14.29 22.34-6.95 33.5 7.04 10.71 21.93 14.56 33.51 6.95C528.27 439.58 576 351.33 576 256S528.27 72.43 448.35 19.97zM480 256c0-63.53-32.06-121.94-85.77-156.24-11.19-7.14-26.03-3.82-33.12 7.46s-3.78 26.21 7.41 33.36C408.27 165.97 432 209.11 432 256s-23.73 90.03-63.48 115.42c-11.19 7.14-14.5 22.07-7.41 33.36 6.51 10.36 21.12 15.14 33.12 7.46C447.94 377.94 480 319.54 480 256zm-141.77-76.87c-11.58-6.33-26.19-2.16-32.61 9.45-6.39 11.61-2.16 26.2 9.45 32.61C327.98 228.28 336 241.63 336 256c0 14.38-8.02 27.72-20.92 34.81-11.61 6.41-15.84 21-9.45 32.61 6.43 11.66 21.05 15.8 32.61 9.45 28.23-15.55 45.77-45 45.77-76.88s-17.54-61.32-45.78-76.86z"],"vote-yea":[640,512,[],"f772","M608 320h-64v64h22.4c5.3 0 9.6 3.6 9.6 8v16c0 4.4-4.3 8-9.6 8H73.6c-5.3 0-9.6-3.6-9.6-8v-16c0-4.4 4.3-8 9.6-8H96v-64H32c-17.7 0-32 14.3-32 32v96c0 17.7 14.3 32 32 32h576c17.7 0 32-14.3 32-32v-96c0-17.7-14.3-32-32-32zm-96 64V64.3c0-17.9-14.5-32.3-32.3-32.3H160.4C142.5 32 128 46.5 128 64.3V384h384zM211.2 202l25.5-25.3c4.2-4.2 11-4.2 15.2.1l41.3 41.6 95.2-94.4c4.2-4.2 11-4.2 15.2.1l25.3 25.5c4.2 4.2 4.2 11-.1 15.2L300.5 292c-4.2 4.2-11 4.2-15.2-.1l-74.1-74.7c-4.3-4.2-4.2-11 0-15.2z"],"vr-cardboard":[640,512,[],"f729","M608 64H32C14.33 64 0 78.33 0 96v320c0 17.67 14.33 32 32 32h160.22c25.19 0 48.03-14.77 58.36-37.74l27.74-61.64C286.21 331.08 302.35 320 320 320s33.79 11.08 41.68 28.62l27.74 61.64C399.75 433.23 422.6 448 447.78 448H608c17.67 0 32-14.33 32-32V96c0-17.67-14.33-32-32-32zM160 304c-35.35 0-64-28.65-64-64s28.65-64 64-64 64 28.65 64 64-28.65 64-64 64zm320 0c-35.35 0-64-28.65-64-64s28.65-64 64-64 64 28.65 64 64-28.65 64-64 64z"],walking:[320,512,[],"f554","M208 96c26.5 0 48-21.5 48-48S234.5 0 208 0s-48 21.5-48 48 21.5 48 48 48zm94.5 149.1l-23.3-11.8-9.7-29.4c-14.7-44.6-55.7-75.8-102.2-75.9-36-.1-55.9 10.1-93.3 25.2-21.6 8.7-39.3 25.2-49.7 46.2L17.6 213c-7.8 15.8-1.5 35 14.2 42.9 15.6 7.9 34.6 1.5 42.5-14.3L81 228c3.5-7 9.3-12.5 16.5-15.4l26.8-10.8-15.2 60.7c-5.2 20.8.4 42.9 14.9 58.8l59.9 65.4c7.2 7.9 12.3 17.4 14.9 27.7l18.3 73.3c4.3 17.1 21.7 27.6 38.8 23.3 17.1-4.3 27.6-21.7 23.3-38.8l-22.2-89c-2.6-10.3-7.7-19.9-14.9-27.7l-45.5-49.7 17.2-68.7 5.5 16.5c5.3 16.1 16.7 29.4 31.7 37l23.3 11.8c15.6 7.9 34.6 1.5 42.5-14.3 7.7-15.7 1.4-35.1-14.3-43zM73.6 385.8c-3.2 8.1-8 15.4-14.2 21.5l-50 50.1c-12.5 12.5-12.5 32.8 0 45.3s32.7 12.5 45.2 0l59.4-59.4c6.1-6.1 10.9-13.4 14.2-21.5l13.5-33.8c-55.3-60.3-38.7-41.8-47.4-53.7l-20.7 51.5z"],wallet:[512,512,[],"f555","M461.2 128H80c-8.84 0-16-7.16-16-16s7.16-16 16-16h384c8.84 0 16-7.16 16-16 0-26.51-21.49-48-48-48H64C28.65 32 0 60.65 0 96v320c0 35.35 28.65 64 64 64h397.2c28.02 0 50.8-21.53 50.8-48V176c0-26.47-22.78-48-50.8-48zM416 336c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32z"],warehouse:[640,512,[],"f494","M504 352H136.4c-4.4 0-8 3.6-8 8l-.1 48c0 4.4 3.6 8 8 8H504c4.4 0 8-3.6 8-8v-48c0-4.4-3.6-8-8-8zm0 96H136.1c-4.4 0-8 3.6-8 8l-.1 48c0 4.4 3.6 8 8 8h368c4.4 0 8-3.6 8-8v-48c0-4.4-3.6-8-8-8zm0-192H136.6c-4.4 0-8 3.6-8 8l-.1 48c0 4.4 3.6 8 8 8H504c4.4 0 8-3.6 8-8v-48c0-4.4-3.6-8-8-8zm106.5-139L338.4 3.7a48.15 48.15 0 0 0-36.9 0L29.5 117C11.7 124.5 0 141.9 0 161.3V504c0 4.4 3.6 8 8 8h80c4.4 0 8-3.6 8-8V256c0-17.6 14.6-32 32.6-32h382.8c18 0 32.6 14.4 32.6 32v248c0 4.4 3.6 8 8 8h80c4.4 0 8-3.6 8-8V161.3c0-19.4-11.7-36.8-29.5-44.3z"],water:[576,512,[],"f773","M562.1 383.9c-21.5-2.4-42.1-10.5-57.9-22.9-14.1-11.1-34.2-11.3-48.2 0-37.9 30.4-107.2 30.4-145.7-1.5-13.5-11.2-33-9.1-46.7 1.8-38 30.1-106.9 30-145.2-1.7-13.5-11.2-33.3-8.9-47.1 2-15.5 12.2-36 20.1-57.7 22.4-7.9.8-13.6 7.8-13.6 15.7v32.2c0 9.1 7.6 16.8 16.7 16 28.8-2.5 56.1-11.4 79.4-25.9 56.5 34.6 137 34.1 192 0 56.5 34.6 137 34.1 192 0 23.3 14.2 50.9 23.3 79.1 25.8 9.1.8 16.7-6.9 16.7-16v-31.6c.1-8-5.7-15.4-13.8-16.3zm0-144c-21.5-2.4-42.1-10.5-57.9-22.9-14.1-11.1-34.2-11.3-48.2 0-37.9 30.4-107.2 30.4-145.7-1.5-13.5-11.2-33-9.1-46.7 1.8-38 30.1-106.9 30-145.2-1.7-13.5-11.2-33.3-8.9-47.1 2-15.5 12.2-36 20.1-57.7 22.4-7.9.8-13.6 7.8-13.6 15.7v32.2c0 9.1 7.6 16.8 16.7 16 28.8-2.5 56.1-11.4 79.4-25.9 56.5 34.6 137 34.1 192 0 56.5 34.6 137 34.1 192 0 23.3 14.2 50.9 23.3 79.1 25.8 9.1.8 16.7-6.9 16.7-16v-31.6c.1-8-5.7-15.4-13.8-16.3zm0-144C540.6 93.4 520 85.4 504.2 73 490.1 61.9 470 61.7 456 73c-37.9 30.4-107.2 30.4-145.7-1.5-13.5-11.2-33-9.1-46.7 1.8-38 30.1-106.9 30-145.2-1.7-13.5-11.2-33.3-8.9-47.1 2-15.5 12.2-36 20.1-57.7 22.4-7.9.8-13.6 7.8-13.6 15.7v32.2c0 9.1 7.6 16.8 16.7 16 28.8-2.5 56.1-11.4 79.4-25.9 56.5 34.6 137 34.1 192 0 56.5 34.6 137 34.1 192 0 23.3 14.2 50.9 23.3 79.1 25.8 9.1.8 16.7-6.9 16.7-16v-31.6c.1-8-5.7-15.4-13.8-16.3z"],"wave-square":[640,512,[],"f83e","M476 480H324a36 36 0 0 1-36-36V96h-96v156a36 36 0 0 1-36 36H16a16 16 0 0 1-16-16v-32a16 16 0 0 1 16-16h112V68a36 36 0 0 1 36-36h152a36 36 0 0 1 36 36v348h96V260a36 36 0 0 1 36-36h140a16 16 0 0 1 16 16v32a16 16 0 0 1-16 16H512v156a36 36 0 0 1-36 36z"],weight:[512,512,[],"f496","M448 64h-25.98C438.44 92.28 448 125.01 448 160c0 105.87-86.13 192-192 192S64 265.87 64 160c0-34.99 9.56-67.72 25.98-96H64C28.71 64 0 92.71 0 128v320c0 35.29 28.71 64 64 64h384c35.29 0 64-28.71 64-64V128c0-35.29-28.71-64-64-64zM256 320c88.37 0 160-71.63 160-160S344.37 0 256 0 96 71.63 96 160s71.63 160 160 160zm-.3-151.94l33.58-78.36c3.5-8.17 12.94-11.92 21.03-8.41 8.12 3.48 11.88 12.89 8.41 21l-33.67 78.55C291.73 188 296 197.45 296 208c0 22.09-17.91 40-40 40s-40-17.91-40-40c0-21.98 17.76-39.77 39.7-39.94z"],"weight-hanging":[512,512,[],"f5cd","M510.28 445.86l-73.03-292.13c-3.8-15.19-16.44-25.72-30.87-25.72h-60.25c3.57-10.05 5.88-20.72 5.88-32 0-53.02-42.98-96-96-96s-96 42.98-96 96c0 11.28 2.3 21.95 5.88 32h-60.25c-14.43 0-27.08 10.54-30.87 25.72L1.72 445.86C-6.61 479.17 16.38 512 48.03 512h415.95c31.64 0 54.63-32.83 46.3-66.14zM256 128c-17.64 0-32-14.36-32-32s14.36-32 32-32 32 14.36 32 32-14.36 32-32 32z"],wheelchair:[512,512,[],"f193","M496.101 385.669l14.227 28.663c3.929 7.915.697 17.516-7.218 21.445l-65.465 32.886c-16.049 7.967-35.556 1.194-43.189-15.055L331.679 320H192c-15.925 0-29.426-11.71-31.679-27.475C126.433 55.308 128.38 70.044 128 64c0-36.358 30.318-65.635 67.052-63.929 33.271 1.545 60.048 28.905 60.925 62.201.868 32.933-23.152 60.423-54.608 65.039l4.67 32.69H336c8.837 0 16 7.163 16 16v32c0 8.837-7.163 16-16 16H215.182l4.572 32H352a32 32 0 0 1 28.962 18.392L438.477 396.8l36.178-18.349c7.915-3.929 17.517-.697 21.446 7.218zM311.358 352h-24.506c-7.788 54.204-54.528 96-110.852 96-61.757 0-112-50.243-112-112 0-41.505 22.694-77.809 56.324-97.156-3.712-25.965-6.844-47.86-9.488-66.333C45.956 198.464 0 261.963 0 336c0 97.047 78.953 176 176 176 71.87 0 133.806-43.308 161.11-105.192L311.358 352z"],wifi:[640,512,[],"f1eb","M634.91 154.88C457.74-8.99 182.19-8.93 5.09 154.88c-6.66 6.16-6.79 16.59-.35 22.98l34.24 33.97c6.14 6.1 16.02 6.23 22.4.38 145.92-133.68 371.3-133.71 517.25 0 6.38 5.85 16.26 5.71 22.4-.38l34.24-33.97c6.43-6.39 6.3-16.82-.36-22.98zM320 352c-35.35 0-64 28.65-64 64s28.65 64 64 64 64-28.65 64-64-28.65-64-64-64zm202.67-83.59c-115.26-101.93-290.21-101.82-405.34 0-6.9 6.1-7.12 16.69-.57 23.15l34.44 33.99c6 5.92 15.66 6.32 22.05.8 83.95-72.57 209.74-72.41 293.49 0 6.39 5.52 16.05 5.13 22.05-.8l34.44-33.99c6.56-6.46 6.33-17.06-.56-23.15z"],wind:[512,512,[],"f72e","M156.7 256H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h142.2c15.9 0 30.8 10.9 33.4 26.6 3.3 20-12.1 37.4-31.6 37.4-14.1 0-26.1-9.2-30.4-21.9-2.1-6.3-8.6-10.1-15.2-10.1H81.6c-9.8 0-17.7 8.8-15.9 18.4 8.6 44.1 47.6 77.6 94.2 77.6 57.1 0 102.7-50.1 95.2-108.6C249 291 205.4 256 156.7 256zM16 224h336c59.7 0 106.8-54.8 93.8-116.7-7.6-36.2-36.9-65.5-73.1-73.1-55.4-11.6-105.1 24.9-114.9 75.5-1.9 9.6 6.1 18.3 15.8 18.3h32.8c6.7 0 13.1-3.8 15.2-10.1C325.9 105.2 337.9 96 352 96c19.4 0 34.9 17.4 31.6 37.4-2.6 15.7-17.4 26.6-33.4 26.6H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16zm384 32H243.7c19.3 16.6 33.2 38.8 39.8 64H400c26.5 0 48 21.5 48 48s-21.5 48-48 48c-17.9 0-33.3-9.9-41.6-24.4-2.9-5-8.7-7.6-14.5-7.6h-33.8c-10.9 0-19 10.8-15.3 21.1 17.8 50.6 70.5 84.8 129.4 72.3 41.2-8.7 75.1-41.6 84.7-82.7C526 321.5 470.5 256 400 256z"],"window-close":[512,512,[],"f410","M464 32H48C21.5 32 0 53.5 0 80v352c0 26.5 21.5 48 48 48h416c26.5 0 48-21.5 48-48V80c0-26.5-21.5-48-48-48zm-83.6 290.5c4.8 4.8 4.8 12.6 0 17.4l-40.5 40.5c-4.8 4.8-12.6 4.8-17.4 0L256 313.3l-66.5 67.1c-4.8 4.8-12.6 4.8-17.4 0l-40.5-40.5c-4.8-4.8-4.8-12.6 0-17.4l67.1-66.5-67.1-66.5c-4.8-4.8-4.8-12.6 0-17.4l40.5-40.5c4.8-4.8 12.6-4.8 17.4 0l66.5 67.1 66.5-67.1c4.8-4.8 12.6-4.8 17.4 0l40.5 40.5c4.8 4.8 4.8 12.6 0 17.4L313.3 256l67.1 66.5z"],"window-maximize":[512,512,[],"f2d0","M464 32H48C21.5 32 0 53.5 0 80v352c0 26.5 21.5 48 48 48h416c26.5 0 48-21.5 48-48V80c0-26.5-21.5-48-48-48zm-16 160H64v-84c0-6.6 5.4-12 12-12h360c6.6 0 12 5.4 12 12v84z"],"window-minimize":[512,512,[],"f2d1","M464 352H48c-26.5 0-48 21.5-48 48v32c0 26.5 21.5 48 48 48h416c26.5 0 48-21.5 48-48v-32c0-26.5-21.5-48-48-48z"],"window-restore":[512,512,[],"f2d2","M512 48v288c0 26.5-21.5 48-48 48h-48V176c0-44.1-35.9-80-80-80H128V48c0-26.5 21.5-48 48-48h288c26.5 0 48 21.5 48 48zM384 176v288c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48V176c0-26.5 21.5-48 48-48h288c26.5 0 48 21.5 48 48zm-68 28c0-6.6-5.4-12-12-12H76c-6.6 0-12 5.4-12 12v52h252v-52z"],"wine-bottle":[512,512,[],"f72f","M507.31 72.57L439.43 4.69c-6.25-6.25-16.38-6.25-22.63 0l-22.63 22.63c-6.25 6.25-6.25 16.38 0 22.63l-76.67 76.67c-46.58-19.7-102.4-10.73-140.37 27.23L18.75 312.23c-24.99 24.99-24.99 65.52 0 90.51l90.51 90.51c24.99 24.99 65.52 24.99 90.51 0l158.39-158.39c37.96-37.96 46.93-93.79 27.23-140.37l76.67-76.67c6.25 6.25 16.38 6.25 22.63 0l22.63-22.63c6.24-6.24 6.24-16.37-.01-22.62zM179.22 423.29l-90.51-90.51 122.04-122.04 90.51 90.51-122.04 122.04z"],"wine-glass":[288,512,[],"f4e3","M216 464h-40V346.81c68.47-15.89 118.05-79.91 111.4-154.16l-15.95-178.1C270.71 6.31 263.9 0 255.74 0H32.26c-8.15 0-14.97 6.31-15.7 14.55L.6 192.66C-6.05 266.91 43.53 330.93 112 346.82V464H72c-22.09 0-40 17.91-40 40 0 4.42 3.58 8 8 8h208c4.42 0 8-3.58 8-8 0-22.09-17.91-40-40-40z"],"wine-glass-alt":[288,512,[],"f5ce","M216 464h-40V346.81c68.47-15.89 118.05-79.91 111.4-154.16l-15.95-178.1C270.71 6.31 263.9 0 255.74 0H32.26c-8.15 0-14.97 6.31-15.7 14.55L.6 192.66C-6.05 266.91 43.53 330.93 112 346.82V464H72c-22.09 0-40 17.91-40 40 0 4.42 3.58 8 8 8h208c4.42 0 8-3.58 8-8 0-22.09-17.91-40-40-40zM61.75 48h164.5l7.17 80H54.58l7.17-80z"],"won-sign":[576,512,[],"f159","M564 192c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-48l18.6-80.6c1.7-7.5-4-14.7-11.7-14.7h-46.1c-5.7 0-10.6 4-11.7 9.5L450.7 128H340.8l-19.7-86c-1.3-5.5-6.1-9.3-11.7-9.3h-44c-5.6 0-10.4 3.8-11.7 9.3l-20 86H125l-17.5-85.7c-1.1-5.6-6.1-9.6-11.8-9.6H53.6c-7.7 0-13.4 7.1-11.7 14.6L60 128H12c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h62.3l7.2 32H12c-6.6 0-12 5.4-12 12v40c0 6.6 5.4 12 12 12h83.9l40.9 182.6c1.2 5.5 6.1 9.4 11.7 9.4h56.8c5.6 0 10.4-3.9 11.7-9.3L259.3 288h55.1l42.4 182.7c1.3 5.4 6.1 9.3 11.7 9.3h56.8c5.6 0 10.4-3.9 11.7-9.3L479.1 288H564c6.6 0 12-5.4 12-12v-40c0-6.6-5.4-12-12-12h-70.1l7.4-32zM183.8 342c-6.2 25.8-6.8 47.2-7.3 47.2h-1.1s-1.7-22-6.8-47.2l-11-54h38.8zm27.5-118h-66.8l-6.5-32h80.8zm62.9 0l2-8.6c1.9-8 3.5-16 4.8-23.4h11.8c1.3 7.4 2.9 15.4 4.8 23.4l2 8.6zm130.9 118c-5.1 25.2-6.8 47.2-6.8 47.2h-1.1c-.6 0-1.1-21.4-7.3-47.2l-12.4-54h39.1zm25.2-118h-67.4l-7.3-32h81.6z"],wrench:[512,512,[],"f0ad","M507.73 109.1c-2.24-9.03-13.54-12.09-20.12-5.51l-74.36 74.36-67.88-11.31-11.31-67.88 74.36-74.36c6.62-6.62 3.43-17.9-5.66-20.16-47.38-11.74-99.55.91-136.58 37.93-39.64 39.64-50.55 97.1-34.05 147.2L18.74 402.76c-24.99 24.99-24.99 65.51 0 90.5 24.99 24.99 65.51 24.99 90.5 0l213.21-213.21c50.12 16.71 107.47 5.68 147.37-34.22 37.07-37.07 49.7-89.32 37.91-136.73zM64 472c-13.25 0-24-10.75-24-24 0-13.26 10.75-24 24-24s24 10.74 24 24c0 13.25-10.75 24-24 24z"],"x-ray":[640,512,[],"f497","M240 384c-8.8 0-16 7.2-16 16s7.2 16 16 16 16-7.2 16-16-7.2-16-16-16zm160 32c8.8 0 16-7.2 16-16s-7.2-16-16-16-16 7.2-16 16 7.2 16 16 16zM624 0H16C7.2 0 0 7.2 0 16v32c0 8.8 7.2 16 16 16h608c8.8 0 16-7.2 16-16V16c0-8.8-7.2-16-16-16zm0 448h-48V96H64v352H16c-8.8 0-16 7.2-16 16v32c0 8.8 7.2 16 16 16h608c8.8 0 16-7.2 16-16v-32c0-8.8-7.2-16-16-16zM480 248c0 4.4-3.6 8-8 8H336v32h104c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H336v32h64c26.5 0 48 21.5 48 48s-21.5 48-48 48-48-21.5-48-48v-16h-64v16c0 26.5-21.5 48-48 48s-48-21.5-48-48 21.5-48 48-48h64v-32H200c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h104v-32H168c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h136v-32H200c-4.4 0-8-3.6-8-8v-16c0-4.4 3.6-8 8-8h104v-24c0-4.4 3.6-8 8-8h16c4.4 0 8 3.6 8 8v24h104c4.4 0 8 3.6 8 8v16c0 4.4-3.6 8-8 8H336v32h136c4.4 0 8 3.6 8 8v16z"],"yen-sign":[384,512,[],"f157","M351.2 32h-65.3c-4.6 0-8.8 2.6-10.8 6.7l-55.4 113.2c-14.5 34.7-27.1 71.9-27.1 71.9h-1.3s-12.6-37.2-27.1-71.9L108.8 38.7c-2-4.1-6.2-6.7-10.8-6.7H32.8c-9.1 0-14.8 9.7-10.6 17.6L102.3 200H44c-6.6 0-12 5.4-12 12v32c0 6.6 5.4 12 12 12h88.2l19.8 37.2V320H44c-6.6 0-12 5.4-12 12v32c0 6.6 5.4 12 12 12h108v92c0 6.6 5.4 12 12 12h56c6.6 0 12-5.4 12-12v-92h108c6.6 0 12-5.4 12-12v-32c0-6.6-5.4-12-12-12H232v-26.8l19.8-37.2H340c6.6 0 12-5.4 12-12v-32c0-6.6-5.4-12-12-12h-58.3l80.1-150.4c4.3-7.9-1.5-17.6-10.6-17.6z"],"yin-yang":[496,512,[],"f6ad","M248 8C111.03 8 0 119.03 0 256s111.03 248 248 248 248-111.03 248-248S384.97 8 248 8zm0 376c-17.67 0-32-14.33-32-32s14.33-32 32-32 32 14.33 32 32-14.33 32-32 32zm0-128c-53.02 0-96 42.98-96 96s42.98 96 96 96c-106.04 0-192-85.96-192-192S141.96 64 248 64c53.02 0 96 42.98 96 96s-42.98 96-96 96zm0-128c-17.67 0-32 14.33-32 32s14.33 32 32 32 32-14.33 32-32-14.33-32-32-32z"]};!function(c){try{c()}catch(c){if(!m)throw c}}(function(){V("fas",L)})}(); \ No newline at end of file
diff --git a/interface/js/lib/visibility.min.js b/interface/js/lib/visibility.min.js
new file mode 100644
index 0000000..d3cec48
--- /dev/null
+++ b/interface/js/lib/visibility.min.js
@@ -0,0 +1,5 @@
+/*
+ * Visibility.js 2.0.2 (https://github.com/ai/visibilityjs)
+ * Copyright (c) 2011, Andrey Sitnik, MIT
+ */
+!function(e){var n=-1,r={onVisible:function(n){var e=r.isSupported();if(!e||!r.hidden())return n(),e;var t=r.change(function(e,i){r.hidden()||(r.unbind(t),n())});return t},change:function(e){if(!r.isSupported())return!1;var i=n+=1;return r._callbacks[i]=e,r._listen(),i},unbind:function(e){delete r._callbacks[e]},afterPrerendering:function(n){var e=r.isSupported(),t="prerender";if(!e||t!=r.state())return n(),e;var d=r.change(function(e,i){t!=i&&(r.unbind(d),n())});return d},hidden:function(){return!(!r._doc.hidden&&!r._doc.webkitHidden)},state:function(){return r._doc.visibilityState||r._doc.webkitVisibilityState||"visible"},isSupported:function(){return void 0!==r._doc.hidden||void 0!==r._doc.webkitHidden},_doc:document||{},_callbacks:{},_change:function(e){var i=r.state();for(var n in r._callbacks)r._callbacks[n].call(r._doc,e,i)},_listen:function(){if(!r._init){var e="visibilitychange";r._doc.webkitVisibilityState&&(e="webkit"+e);function i(){r._change.apply(r,arguments)}r._doc.addEventListener?r._doc.addEventListener(e,i):r._doc.attachEvent(e,i),r._init=!0}}};"undefined"!=typeof module&&module.exports?module.exports=r:e.Visibility=r}(this),function(a){function e(o){return o.every=function(e,i,n){o._time(),n||(n=i,i=null);var t=d+=1;return o._timers[t]={visible:e,hidden:i,callback:n},o._run(t,!1),o.isSupported()&&o._listen(),t},o.stop=function(e){return!!o._timers[e]&&(o._stop(e),delete o._timers[e],!0)},o._timers={},o._time=function(){o._timed||(o._timed=!0,o._wasHidden=o.hidden(),o.change(function(){o._stopRun(),o._wasHidden=o.hidden()}))},o._run=function(e,i){var n,t=o._timers[e];if(o.hidden()){if(null===t.hidden)return;n=t.hidden}else n=t.visible;function d(){t.last=new Date,t.callback.call(a)}if(i){var r=new Date-t.last;r<n?t.delay=setTimeout(function(){t.id=setInterval(d,n),d()},n-r):(t.id=setInterval(d,n),d())}else t.id=setInterval(d,n)},o._stop=function(e){var i=o._timers[e];clearInterval(i.id),clearTimeout(i.delay),delete i.id,delete i.delay},o._stopRun=function(e){var i=o.hidden(),n=o._wasHidden;if(i&&!n||!i&&n)for(var t in o._timers)o._stop(t),o._run(t,!i)},o}var d=-1;"undefined"!=typeof module&&module.exports?module.exports=e(require("./visibility.core")):e(a.Visibility||require("./visibility.core"))}(window);
diff --git a/interface/js/main.js b/interface/js/main.js
new file mode 100644
index 0000000..0e226a0
--- /dev/null
+++ b/interface/js/main.js
@@ -0,0 +1,68 @@
+/* global d3:writable, require, requirejs */ // eslint-disable-line no-unused-vars
+
+requirejs.config({
+ baseUrl: "js/lib",
+ paths: {
+ app: "../app",
+ jquery: "jquery-3.7.1.min",
+ visibility: "visibility.min",
+ bootstrap: "bootstrap.bundle.min",
+ codejar: "codejar.min",
+ d3: "d3.min",
+ d3evolution: "d3evolution.min",
+ d3pie: "d3pie.min",
+ fontawesome: "fontawesome.min",
+ fontawesome_solid: "solid.min",
+ footable: "footable.min",
+ linenumbers: "codejar-linenumbers.min",
+ nprogress: "nprogress.min",
+ prism: "prism",
+ stickytabs: "jquery.stickytabs.min"
+ },
+ shim: {
+ app: {deps: ["jquery"]},
+ codejar: {exports: "CodeJar", deps: ["linenumbers"]},
+ bootstrap: {exports: "bootstrap", deps: ["jquery"]}, // Popovers require jQuery
+ d3: {exports: "d3"},
+ d3evolution: {exports: "D3Evolution", deps: ["d3.global", "jquery"]},
+ d3pie: {exports: "D3Pie", deps: ["d3.global", "jquery"]},
+ fontawesome: {exports: "FontAwesome", deps: ["fontawesome_solid"]},
+ footable: {deps: ["bootstrap", "jquery"]},
+ linenumbers: {exports: "withLineNumbers", deps: ["prism"]},
+ prism: {exports: "Prism"},
+ stickytabs: {deps: ["jquery"]}
+ },
+ waitSeconds: 30,
+});
+
+document.title = window.location.hostname +
+ (window.location.port ? ":" + window.location.port : "") +
+ (window.location.pathname !== "/" ? window.location.pathname : "") +
+ " - Rspamd Web Interface";
+
+// Ugly hack to get d3pie work with requirejs
+define("d3.global", ["d3"], (d3global) => { // eslint-disable-line strict
+ d3 = d3global;
+});
+
+// Notify user on module loading failure
+requirejs.onError = function (e) {
+ "use strict";
+ document.getElementById("loading").classList.add("d-none");
+ document.getElementsByClassName("notification-area")[0].innerHTML =
+ "<div class=\"alert alert-error\">" +
+ "<strong>Module loading error: " + e.requireType + ", module: " + e.requireModules + "</strong>" +
+ "<button type=\"button\" class=\"btn btn-info btn-xs float-end\" " +
+ "onClick=\"window.location.reload(); this.parentNode.parentNode.removeChild(this.parentNode);\" " +
+ "title=\"Reload current page\">" +
+ "<i class=\"glyphicon glyphicon-repeat\"></i> Reload" +
+ "</button>" +
+ "</div>";
+ throw e;
+};
+
+// Load main UI
+require(["app/rspamd"], (rspamd) => {
+ "use strict";
+ rspamd.connect();
+});
diff --git a/interface/mstile-150x150.png b/interface/mstile-150x150.png
new file mode 100644
index 0000000..4317d45
--- /dev/null
+++ b/interface/mstile-150x150.png
Binary files differ
diff --git a/interface/safari-pinned-tab.svg b/interface/safari-pinned-tab.svg
new file mode 100644
index 0000000..cf37f8c
--- /dev/null
+++ b/interface/safari-pinned-tab.svg
@@ -0,0 +1 @@
+<svg version="1" xmlns="http://www.w3.org/2000/svg" width="346.667" height="346.667" viewBox="0 0 260.000000 260.000000"><path d="M0 16v16h199.5L169 62.5 138.5 93l14.3 14.2 14.2 14.3L197.5 91 228 60.5V178h32V0H0v16zM0 170.5V260h260v-33H61.5L92 196.5l30.5-30.5-14.3-14.2L94 137.5 63.5 168 33 198.5V81H0v89.5z"/></svg> \ No newline at end of file
diff --git a/lua_style.md b/lua_style.md
new file mode 100644
index 0000000..8ea7a81
--- /dev/null
+++ b/lua_style.md
@@ -0,0 +1,733 @@
+# Lua Style Guide
+
+This style guide contains a list of guidelines that we try to follow for Rspamd.
+
+This guide is forked from https://github.com/Olivine-Labs/lua-style-guide
+
+
+## <a name='TOC'>Table of Contents</a>
+
+ 1. [Types](#types)
+ 1. [Tables](#tables)
+ 1. [Strings](#strings)
+ 1. [Functions](#functions)
+ 1. [Properties](#properties)
+ 1. [Variables](#variables)
+ 1. [Conditional Expressions & Equality](#conditionals)
+ 1. [Blocks](#blocks)
+ 1. [Whitespace](#whitespace)
+ 1. [Commas](#commas)
+ 1. [Semicolons](#semicolons)
+ 1. [Type Casting & Coercion](#type-coercion)
+ 1. [Naming Conventions](#naming-conventions)
+ 1. [Accessors](#accessors)
+ 1. [Constructors](#constructors)
+ 1. [Modules](#modules)
+ 1. [Testing](#testing)
+ 1. [License](#license)
+
+## <a name='types'>Types</a>
+
+ - **Primitives**: When you access a primitive type you work directly on its value
+
+ + `string`
+ + `number`
+ + `boolean`
+ + `nil`
+
+ ```lua
+ local foo = 1
+ local bar = foo
+
+ bar = 9
+
+ print(foo, bar) -- => 1 9
+ ```
+
+ - **Complex**: When you access a complex type you work on a reference to its value
+
+ + `table`
+ + `function`
+ + `userdata`
+
+ ```lua
+ local foo = { 1, 2 }
+ local bar = foo
+
+ bar[0] = 9
+ foo[1] = 3
+
+ print(foo[0], bar[0]) -- => 9 9
+ print(foo[1], bar[1]) -- => 3 3
+ print(foo[2], bar[2]) -- => 2 2
+ ```
+
+ **[[⬆]](#TOC)**
+
+## <a name='tables'>Tables</a>
+
+ - Use the constructor syntax for table property creation where possible.
+
+ ```lua
+ -- bad
+ local player = {}
+ player.name = 'Jack'
+ player.class = 'Rogue'
+
+ -- good
+ local player = {
+ name = 'Jack',
+ class = 'Rogue'
+ }
+ ```
+
+ - Define functions externally to table definition.
+
+ ```lua
+ -- bad
+ local player = {
+ attack = function()
+ -- ...stuff...
+ end
+ }
+
+ -- good
+ local function attack()
+ end
+
+ local player = {
+ attack = attack
+ }
+ ```
+
+ **[[⬆]](#TOC)**
+
+## <a name='strings'>Strings</a>
+
+ - Use single quotes `''` for strings.
+
+ ```lua
+ -- bad
+ local name = "Bob Parr"
+
+ -- good
+ local name = 'Bob Parr'
+
+ -- bad
+ local fullName = "Bob " .. self.lastName
+
+ -- good
+ local fullName = 'Bob ' .. self.lastName
+ ```
+
+ - Strings longer than 80 characters should be written across multiple lines
+ using concatenation. This allows you to indent nicely.
+
+ ```lua
+ -- bad
+ local errorMessage = 'This is a super long error that was thrown because of Batman. When you stop to think about how Batman had anything to do with this, you would get nowhere fast.'
+
+ -- bad
+ local errorMessage = 'This is a super long error that \
+ was thrown because of Batman. \
+ When you stop to think about \
+ how Batman had anything to do \
+ with this, you would get nowhere \
+ fast.'
+
+
+ -- bad
+ local errorMessage = [[This is a super long error that
+ was thrown because of Batman.
+ When you stop to think about
+ how Batman had anything to do
+ with this, you would get nowhere
+ fast.]]
+
+ -- good
+ local errorMessage = 'This is a super long error that ' ..
+ 'was thrown because of Batman. ' ..
+ 'When you stop to think about ' ..
+ 'how Batman had anything to do ' ..
+ 'with this, you would get nowhere ' ..
+ 'fast.'
+ ```
+
+ **[[⬆]](#TOC)**
+
+
+## <a name='functions'>Functions</a>
+ - Prefer lots of small functions to large, complex functions. [Smalls Functions Are Good For The Universe](http://kikito.github.io/blog/2012/03/16/small-functions-are-good-for-the-universe/).
+
+ - Prefer function syntax over variable syntax. This helps differentiate
+ between named and anonymous functions.
+
+ ```lua
+ -- bad
+ local nope = function(name, options)
+ -- ...stuff...
+ end
+
+ -- good
+ local function yup(name, options)
+ -- ...stuff...
+ end
+ ```
+
+ - Never name a parameter `arg`, this will take precedence over the `arg` object that is given to every function scope in older versions of Lua.
+
+ ```lua
+ -- bad
+ local function nope(name, options, arg)
+ -- ...stuff...
+ end
+
+ -- good
+ local function yup(name, options, ...)
+ -- ...stuff...
+ end
+ ```
+
+ - Perform validation early and return as early as possible.
+
+ ```lua
+ -- bad
+ local is_good_name = function(name, options, arg)
+ local is_good = #name > 3
+ is_good = is_good and #name < 30
+
+ -- ...stuff...
+
+ return is_bad
+ end
+
+ -- good
+ local is_good_name = function(name, options, args)
+ if #name < 3 or #name > 30 then return false end
+
+ -- ...stuff...
+
+ return true
+ end
+ ```
+
+ **[[⬆]](#TOC)**
+
+
+## <a name='properties'>Properties</a>
+
+ - Use dot notation when accessing known properties.
+
+ ```lua
+ local luke = {
+ jedi = true,
+ age = 28
+ }
+
+ -- bad
+ local isJedi = luke['jedi']
+
+ -- good
+ local isJedi = luke.jedi
+ ```
+
+ - Use subscript notation `[]` when accessing properties with a variable
+ or if using a table as a list.
+
+ ```lua
+ local luke = {
+ jedi = true,
+ age = 28
+ }
+
+ local function getProp(prop)
+ return luke[prop]
+ end
+
+ local isJedi = getProp('jedi')
+ ```
+
+ **[[⬆]](#TOC)**
+
+
+## <a name='variables'>Variables</a>
+
+ - Always use `local` to declare variables. Not doing so will result in
+ global variables to avoid polluting the global namespace.
+
+ ```lua
+ -- bad
+ superPower = SuperPower()
+
+ -- good
+ local superPower = SuperPower()
+ ```
+
+ - Assign variables at the top of their scope where possible. This makes it
+ easier to check for existing variables.
+
+ ```lua
+ -- bad
+ local bad = function()
+ test()
+ print('doing stuff..')
+
+ //..other stuff..
+
+ local name = getName()
+
+ if name == 'test' then
+ return false
+ end
+
+ return name
+ end
+
+ -- good
+ local function good()
+ local name = getName()
+
+ test()
+ print('doing stuff..')
+
+ //..other stuff..
+
+ if name == 'test' then
+ return false
+ end
+
+ return name
+ end
+ ```
+
+ **[[⬆]](#TOC)**
+
+
+## <a name='conditionals'>Conditional Expressions & Equality</a>
+
+ - False and nil are *falsy* in conditional expressions. All else is true.
+
+ ```lua
+ local str = ''
+
+ if str then
+ -- true
+ end
+ ```
+
+ - Use shortcuts when you can, unless you need to know the difference between
+ false and nil.
+
+ ```lua
+ -- bad
+ if name ~= nil then
+ -- ...stuff...
+ end
+
+ -- good
+ if name then
+ -- ...stuff...
+ end
+ ```
+
+ - Prefer *true* statements over *false* statements where it makes sense.
+ Prioritize truthy conditions when writing multiple conditions.
+
+ ```lua
+ --bad
+ if not thing then
+ -- ...stuff...
+ else
+ -- ...stuff...
+ end
+
+ --good
+ if thing then
+ -- ...stuff...
+ else
+ -- ...stuff...
+ end
+ ```
+
+ - Prefer defaults to `else` statements where it makes sense. This results in
+ less complex and safer code at the expense of variable reassignment, so
+ situations may differ.
+
+ ```lua
+ --bad
+ local function full_name(first, last)
+ local name
+
+ if first and last then
+ name = first .. ' ' .. last
+ else
+ name = 'John Smith'
+ end
+
+ return name
+ end
+
+ --good
+ local function full_name(first, last)
+ local name = 'John Smith'
+
+ if first and last then
+ name = first .. ' ' .. last
+ end
+
+ return name
+ end
+ ```
+
+ - Short ternaries are okay.
+
+ ```lua
+ local function default_name(name)
+ -- return the default 'Waldo' if name is nil
+ return name or 'Waldo'
+ end
+
+ local function brew_coffee(machine)
+ return machine and machine.is_loaded and 'coffee brewing' or 'fill your water'
+ end
+ ```
+
+
+ **[[⬆]](#TOC)**
+
+
+## <a name='blocks'>Blocks</a>
+
+ - Single line blocks are okay for *small* statements. Try to keep lines to 80 characters.
+ Indent lines if they overflow past the limit.
+
+ ```lua
+ -- good
+ if test then return false end
+
+ -- good
+ if test then
+ return false
+ end
+
+ -- bad
+ if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == 10 then do_other_complicated_function()end
+
+ -- good
+ if test < 1 and do_complicated_function(test) == false or
+ seven == 8 and nine == 10 then
+
+ do_other_complicated_function()
+ return false
+ end
+ ```
+
+ **[[⬆]](#TOC)**
+
+
+## <a name='whitespace'>Whitespace</a>
+
+ - Use soft tabs set to 2 spaces.
+
+ ```lua
+ -- bad
+ function()
+ ∙∙∙∙local name
+ end
+
+ -- bad
+ function()
+ ∙local name
+ end
+
+ -- good
+ function()
+ ∙∙local name
+ end
+ ```
+
+ - Place 1 space before opening and closing braces. Place no spaces around parens.
+
+ ```lua
+ -- bad
+ local test = {one=1}
+
+ -- good
+ local test = { one = 1 }
+
+ -- bad
+ dog.set('attr',{
+ age = '1 year',
+ breed = 'Bernese Mountain Dog'
+ })
+
+ -- good
+ dog.set('attr', {
+ age = '1 year',
+ breed = 'Bernese Mountain Dog'
+ })
+ ```
+
+ - Place an empty newline at the end of the file.
+
+ ```lua
+ -- bad
+ (function(global)
+ -- ...stuff...
+ end)(self)
+ ```
+
+ ```lua
+ -- good
+ (function(global)
+ -- ...stuff...
+ end)(self)
+
+ ```
+
+ - Surround operators with spaces.
+
+ ```lua
+ -- bad
+ local thing=1
+ thing = thing-1
+ thing = thing*1
+ thing = 'string'..'s'
+
+ -- good
+ local thing = 1
+ thing = thing - 1
+ thing = thing * 1
+ thing = 'string' .. 's'
+ ```
+
+ - Use one space after commas.
+
+ ```lua
+ --bad
+ local thing = {1,2,3}
+ thing = {1 , 2 , 3}
+ thing = {1 ,2 ,3}
+
+ --good
+ local thing = {1, 2, 3}
+ ```
+
+ - Add a line break after multiline blocks.
+
+ ```lua
+ --bad
+ if thing then
+ -- ...stuff...
+ end
+ function derp()
+ -- ...stuff...
+ end
+ local wat = 7
+
+ --good
+ if thing then
+ -- ...stuff...
+ end
+
+ function derp()
+ -- ...stuff...
+ end
+
+ local wat = 7
+ ```
+
+ - Delete unnecessary whitespace at the end of lines.
+
+ **[[⬆]](#TOC)**
+
+## <a name='commas'>Commas</a>
+
+ - Leading commas aren't okay. An ending comma on the last item is okay but discouraged.
+
+ ```lua
+ -- bad
+ local thing = {
+ once = 1
+ , upon = 2
+ , aTime = 3
+ }
+
+ -- good
+ local thing = {
+ once = 1,
+ upon = 2,
+ aTime = 3
+ }
+
+ -- okay
+ local thing = {
+ once = 1,
+ upon = 2,
+ aTime = 3,
+ }
+ ```
+
+ **[[⬆]](#TOC)**
+
+
+## <a name='semicolons'>Semicolons</a>
+
+ - **Nope.** Separate statements onto multiple lines.
+
+ ```lua
+ -- bad
+ local whatever = 'sure';
+ a = 1; b = 2
+
+ -- good
+ local whatever = 'sure'
+ a = 1
+ b = 2
+ ```
+
+ **[[⬆]](#TOC)**
+
+
+## <a name='type-coercion'>Type Casting & Coercion</a>
+
+ - Perform type coercion at the beginning of the statement. Use the built-in functions. (`tostring`, `tonumber`, etc.)
+
+ - Use `tostring` for strings if you need to cast without string concatenation.
+
+ ```lua
+ -- bad
+ local totalScore = reviewScore .. ''
+
+ -- good
+ local totalScore = tostring(reviewScore)
+ ```
+
+ - Use `tonumber` for Numbers.
+
+ ```lua
+ local inputValue = '4'
+
+ -- bad
+ local val = inputValue * 1
+
+ -- good
+ local val = tonumber(inputValue)
+ ```
+
+ **[[⬆]](#TOC)**
+
+
+## <a name='naming-conventions'>Naming Conventions</a>
+
+ - Avoid single letter names. Be descriptive with your naming. You can get
+ away with single-letter names when they are variables in loops.
+
+ ```lua
+ -- bad
+ local function q()
+ -- ...stuff...
+ end
+
+ -- good
+ local function query()
+ -- ..stuff..
+ end
+ ```
+
+ - Use underscores for ignored variables in loops.
+
+ ```lua
+ --good
+ for _, name in pairs(names) do
+ -- ...stuff...
+ end
+ ```
+
+ - Use snake_case when naming objects, functions, and instances. Tend towards
+ verbosity if unsure about naming.
+
+ ```lua
+ -- bad
+ local OBJEcttsssss = {}
+ local thisIsMyObject = {}
+
+ local c = function()
+ -- ...stuff...
+ end
+
+ -- good
+ local this_is_my_object = {}
+
+ local function do_that_thing()
+ -- ...stuff...
+ end
+ ```
+
+ - Use PascalCase for factories.
+
+ ```lua
+ -- bad
+ local player = require('player')
+
+ -- good
+ local Player = require('player')
+ local me = Player({ name = 'Jack' })
+ ```
+
+ **[[⬆]](#TOC)**
+
+ - Use `is` or `has` for boolean-returning functions that are part of tables.
+
+ ```lua
+ --bad
+ local function evil(alignment)
+ return alignment < 100
+ end
+
+ --good
+ local function is_evil(alignment)
+ return alignment < 100
+ end
+ ```
+
+## <a name='modules'>Modules</a>
+
+ - The module should return a table or function.
+ - The module should not use the global namespace for anything ever. The
+ module should be a closure.
+ - The file should be named like the module.
+
+ ```lua
+ -- thing.lua
+ local thing = { }
+
+ local meta = {
+ __call = function(self, key, vars)
+ print key
+ end
+ }
+
+
+ return setmetatable(thing, meta)
+ ```
+
+ - Note that modules are [loaded as singletons](http://lua-users.org/wiki/TheEssenceOfLoadingCode)
+ and therefore should usually be factories (a function returning a new instance of a table)
+ unless static (like utility libraries.)
+
+ **[[⬆]](#TOC)**
+
+## <a name='testing'>Testing</a>
+
+ - Use [telescope](https://github.com/norman/telescope) for unit tests and Robot framework for functional testing.
+ Unit tests can rely on LuaJIT ffi module if C function testing is required.
+
+ **[[⬆]](#TOC)**
+
+## <a name='license'>License</a>
+
+ - Released under CC0 (Public Domain).
+ Information can be found at [http://creativecommons.org/publicdomain/zero/1.0/](http://creativecommons.org/publicdomain/zero/1.0/).
+
+**[[⬆]](#TOC)** \ No newline at end of file
diff --git a/lualib/ansicolors.lua b/lualib/ansicolors.lua
new file mode 100644
index 0000000..81783f6
--- /dev/null
+++ b/lualib/ansicolors.lua
@@ -0,0 +1,68 @@
+local colormt = {}
+local ansicolors = {}
+
+local rspamd_util = require "rspamd_util"
+local isatty = rspamd_util.isatty()
+
+function colormt:__tostring()
+ return self.value
+end
+
+function colormt:__concat(other)
+ return tostring(self) .. tostring(other)
+end
+
+function colormt:__call(s)
+ return self .. s .. ansicolors.reset
+end
+
+colormt.__metatable = {}
+local function makecolor(value)
+ if isatty then
+ return setmetatable({
+ value = string.char(27) .. '[' .. tostring(value) .. 'm'
+ }, colormt)
+ else
+ return setmetatable({
+ value = ''
+ }, colormt)
+ end
+end
+
+local colors = {
+ -- attributes
+ reset = 0,
+ clear = 0,
+ bright = 1,
+ dim = 2,
+ underscore = 4,
+ blink = 5,
+ reverse = 7,
+ hidden = 8,
+
+ -- foreground
+ black = 30,
+ red = 31,
+ green = 32,
+ yellow = 33,
+ blue = 34,
+ magenta = 35,
+ cyan = 36,
+ white = 37,
+
+ -- background
+ onblack = 40,
+ onred = 41,
+ ongreen = 42,
+ onyellow = 43,
+ onblue = 44,
+ onmagenta = 45,
+ oncyan = 46,
+ onwhite = 47,
+}
+
+for c, v in pairs(colors) do
+ ansicolors[c] = makecolor(v)
+end
+
+return ansicolors \ No newline at end of file
diff --git a/lualib/global_functions.lua b/lualib/global_functions.lua
new file mode 100644
index 0000000..45d4e84
--- /dev/null
+++ b/lualib/global_functions.lua
@@ -0,0 +1,56 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local lua_redis = require "lua_redis"
+local meta_functions = require "lua_meta"
+local maps = require "lua_maps"
+
+local exports = {}
+
+exports.rspamd_parse_redis_server = lua_redis.rspamd_parse_redis_server
+exports.parse_redis_server = lua_redis.rspamd_parse_redis_server
+exports.rspamd_redis_make_request = lua_redis.rspamd_redis_make_request
+exports.redis_make_request = lua_redis.rspamd_redis_make_request
+
+exports.rspamd_gen_metatokens = meta_functions.rspamd_gen_metatokens
+exports.rspamd_count_metatokens = meta_functions.rspamd_count_metatokens
+
+exports.rspamd_map_add = maps.rspamd_map_add
+
+exports.rspamd_str_split = lua_util.rspamd_str_split
+
+-- a special syntax sugar to export all functions to the global table
+setmetatable(exports, {
+ __call = function(t, override)
+ for k, v in pairs(t) do
+ if _G[k] ~= nil then
+ local msg = 'function ' .. k .. ' already exists in global scope.'
+ if override then
+ _G[k] = v
+ logger.errx('WARNING: ' .. msg .. ' Overwritten.')
+ else
+ logger.errx('NOTICE: ' .. msg .. ' Skipped.')
+ end
+ else
+ _G[k] = v
+ end
+ end
+ end,
+})
+
+return exports
diff --git a/lualib/lua_auth_results.lua b/lualib/lua_auth_results.lua
new file mode 100644
index 0000000..8c907d9
--- /dev/null
+++ b/lualib/lua_auth_results.lua
@@ -0,0 +1,301 @@
+--[[
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local rspamd_util = require "rspamd_util"
+local lua_util = require "lua_util"
+
+local default_settings = {
+ spf_symbols = {
+ pass = 'R_SPF_ALLOW',
+ fail = 'R_SPF_FAIL',
+ softfail = 'R_SPF_SOFTFAIL',
+ neutral = 'R_SPF_NEUTRAL',
+ temperror = 'R_SPF_DNSFAIL',
+ none = 'R_SPF_NA',
+ permerror = 'R_SPF_PERMFAIL',
+ },
+ dmarc_symbols = {
+ pass = 'DMARC_POLICY_ALLOW',
+ permerror = 'DMARC_BAD_POLICY',
+ temperror = 'DMARC_DNSFAIL',
+ none = 'DMARC_NA',
+ reject = 'DMARC_POLICY_REJECT',
+ softfail = 'DMARC_POLICY_SOFTFAIL',
+ quarantine = 'DMARC_POLICY_QUARANTINE',
+ },
+ arc_symbols = {
+ pass = 'ARC_ALLOW',
+ permerror = 'ARC_INVALID',
+ temperror = 'ARC_DNSFAIL',
+ none = 'ARC_NA',
+ reject = 'ARC_REJECT',
+ },
+ dkim_symbols = {
+ none = 'R_DKIM_NA',
+ },
+ add_smtp_user = true,
+}
+
+local exports = {
+ default_settings = default_settings
+}
+
+local local_hostname = rspamd_util.get_hostname()
+
+local function gen_auth_results(task, settings)
+ local auth_results, hdr_parts = {}, {}
+
+ if not settings then
+ settings = default_settings
+ end
+
+ local auth_types = {
+ dkim = settings.dkim_symbols,
+ dmarc = settings.dmarc_symbols,
+ spf = settings.spf_symbols,
+ arc = settings.arc_symbols,
+ }
+
+ local common = {
+ symbols = {}
+ }
+
+ local mta_hostname = task:get_request_header('MTA-Name') or
+ task:get_request_header('MTA-Tag')
+ if mta_hostname then
+ mta_hostname = tostring(mta_hostname)
+ else
+ mta_hostname = local_hostname
+ end
+
+ table.insert(hdr_parts, mta_hostname)
+
+ for auth_type, symbols in pairs(auth_types) do
+ for key, sym in pairs(symbols) do
+ if not common.symbols.sym then
+ local s = task:get_symbol(sym)
+ if not s then
+ common.symbols[sym] = false
+ else
+ common.symbols[sym] = s
+ if not auth_results[auth_type] then
+ auth_results[auth_type] = { key }
+ else
+ table.insert(auth_results[auth_type], key)
+ end
+
+ if auth_type ~= 'dkim' then
+ break
+ end
+ end
+ end
+ end
+ end
+
+ local dkim_results = task:get_dkim_results()
+ -- For each signature we set authentication results
+ -- dkim=neutral (body hash did not verify) header.d=example.com header.s=sel header.b=fA8VVvJ8;
+ -- dkim=neutral (body hash did not verify) header.d=example.com header.s=sel header.b=f8pM8o90;
+
+ for _, dres in ipairs(dkim_results) do
+ local ar_string = 'none'
+
+ if dres.result == 'reject' then
+ ar_string = 'fail' -- imply failure, not neutral
+ elseif dres.result == 'allow' then
+ ar_string = 'pass'
+ elseif dres.result == 'bad record' or dres.result == 'permerror' then
+ ar_string = 'permerror'
+ elseif dres.result == 'tempfail' then
+ ar_string = 'temperror'
+ end
+ local hdr = {}
+
+ hdr[1] = string.format('dkim=%s', ar_string)
+
+ if dres.fail_reason then
+ hdr[#hdr + 1] = string.format('(%s)', lua_util.maybe_smtp_quote_value(dres.fail_reason))
+ end
+
+ if dres.domain then
+ hdr[#hdr + 1] = string.format('header.d=%s', lua_util.maybe_smtp_quote_value(dres.domain))
+ end
+
+ if dres.selector then
+ hdr[#hdr + 1] = string.format('header.s=%s', lua_util.maybe_smtp_quote_value(dres.selector))
+ end
+
+ if dres.bhash then
+ hdr[#hdr + 1] = string.format('header.b=%s', lua_util.maybe_smtp_quote_value(dres.bhash))
+ end
+
+ table.insert(hdr_parts, table.concat(hdr, ' '))
+ end
+
+ if #dkim_results == 0 then
+ -- We have no dkim results, so check for DKIM_NA symbol
+ if common.symbols[settings.dkim_symbols.none] then
+ table.insert(hdr_parts, 'dkim=none')
+ end
+ end
+
+ for auth_type, keys in pairs(auth_results) do
+ for _, key in ipairs(keys) do
+ local hdr = ''
+ if auth_type == 'dmarc' then
+ local opts = common.symbols[auth_types['dmarc'][key]][1]['options'] or {}
+ hdr = hdr .. 'dmarc='
+ if key == 'reject' or key == 'quarantine' or key == 'softfail' then
+ hdr = hdr .. 'fail'
+ else
+ hdr = hdr .. lua_util.maybe_smtp_quote_value(key)
+ end
+ if key == 'pass' then
+ hdr = hdr .. ' (policy=' .. lua_util.maybe_smtp_quote_value(opts[2]) .. ')'
+ hdr = hdr .. ' header.from=' .. lua_util.maybe_smtp_quote_value(opts[1])
+ elseif key ~= 'none' then
+ local t = { opts[1]:match('^([^%s]+) : (.*)$') }
+ if #t > 0 then
+ local dom = t[1]
+ local rsn = t[2]
+ if rsn then
+ hdr = string.format('%s reason=%s', hdr, lua_util.maybe_smtp_quote_value(rsn))
+ end
+ hdr = string.format('%s header.from=%s', hdr, lua_util.maybe_smtp_quote_value(dom))
+ end
+ if key == 'softfail' then
+ hdr = hdr .. ' (policy=none)'
+ else
+ hdr = hdr .. ' (policy=' .. lua_util.maybe_smtp_quote_value(key) .. ')'
+ end
+ end
+ table.insert(hdr_parts, hdr)
+ elseif auth_type == 'arc' then
+ if common.symbols[auth_types['arc'][key]][1] then
+ local opts = common.symbols[auth_types['arc'][key]][1]['options'] or {}
+ for _, v in ipairs(opts) do
+ hdr = string.format('%s%s=%s (%s)', hdr, auth_type,
+ lua_util.maybe_smtp_quote_value(key), lua_util.maybe_smtp_quote_value(v))
+ table.insert(hdr_parts, hdr)
+ end
+ end
+ elseif auth_type == 'spf' then
+ -- Main type
+ local sender
+ local sender_type
+ local smtp_from = task:get_from({ 'smtp', 'orig' })
+
+ if smtp_from and
+ smtp_from[1] and
+ smtp_from[1]['addr'] ~= '' and
+ smtp_from[1]['addr'] ~= nil then
+ sender = lua_util.maybe_smtp_quote_value(smtp_from[1]['addr'])
+ sender_type = 'smtp.mailfrom'
+ else
+ local helo = task:get_helo()
+ if helo then
+ sender = lua_util.maybe_smtp_quote_value(helo)
+ sender_type = 'smtp.helo'
+ end
+ end
+
+ if sender and sender_type then
+ -- Comment line
+ local comment = ''
+ if key == 'pass' then
+ comment = string.format('%s: domain of %s designates %s as permitted sender',
+ mta_hostname, sender, tostring(task:get_from_ip() or 'unknown'))
+ elseif key == 'fail' then
+ comment = string.format('%s: domain of %s does not designate %s as permitted sender',
+ mta_hostname, sender, tostring(task:get_from_ip() or 'unknown'))
+ elseif key == 'neutral' or key == 'softfail' then
+ comment = string.format('%s: %s is neither permitted nor denied by domain of %s',
+ mta_hostname, tostring(task:get_from_ip() or 'unknown'), sender)
+ elseif key == 'permerror' then
+ comment = string.format('%s: domain of %s uses mechanism not recognized by this client',
+ mta_hostname, sender)
+ elseif key == 'temperror' then
+ comment = string.format('%s: error in processing during lookup of %s: DNS error',
+ mta_hostname, sender)
+ elseif key == 'none' then
+ comment = string.format('%s: domain of %s has no SPF policy when checking %s',
+ mta_hostname, sender, tostring(task:get_from_ip() or 'unknown'))
+ end
+ hdr = string.format('%s=%s (%s) %s=%s', auth_type, key,
+ comment, sender_type, sender)
+ else
+ hdr = string.format('%s=%s', auth_type, key)
+ end
+
+ table.insert(hdr_parts, hdr)
+ end
+ end
+ end
+
+ local u = task:get_user()
+ local smtp_from = task:get_from({ 'smtp', 'orig' })
+
+ if u and smtp_from then
+ local hdr = { [1] = 'auth=pass' }
+
+ if settings['add_smtp_user'] then
+ table.insert(hdr, 'smtp.auth=' .. lua_util.maybe_smtp_quote_value(u))
+ end
+ if smtp_from[1]['addr'] then
+ table.insert(hdr, 'smtp.mailfrom=' .. lua_util.maybe_smtp_quote_value(smtp_from[1]['addr']))
+ end
+
+ table.insert(hdr_parts, table.concat(hdr, ' '))
+ end
+
+ if #hdr_parts > 0 then
+ if #hdr_parts == 1 then
+ hdr_parts[2] = 'none'
+ end
+ return table.concat(hdr_parts, '; ')
+ end
+
+ return nil
+end
+
+exports.gen_auth_results = gen_auth_results
+
+local aar_elt_grammar
+-- This function parses an ar element to a table of kv pairs that represents different
+-- elements
+local function parse_ar_element(elt)
+
+ if not aar_elt_grammar then
+ -- Generate grammar
+ local lpeg = require "lpeg"
+ local P = lpeg.P
+ local S = lpeg.S
+ local V = lpeg.V
+ local C = lpeg.C
+ local space = S(" ") ^ 0
+ local doublequoted = space * P '"' * ((1 - S '"\r\n\f\\') + (P '\\' * 1)) ^ 0 * '"' * space
+ local comment = space * P { "(" * ((1 - S "()") + V(1)) ^ 0 * ")" } * space
+ local name = C((1 - S('=(" ')) ^ 1) * space
+ local pair = lpeg.Cg(name * "=" * space * name) * space
+ aar_elt_grammar = lpeg.Cf(lpeg.Ct("") * (pair + comment + doublequoted) ^ 1, rawset)
+ end
+
+ return aar_elt_grammar:match(elt)
+end
+exports.parse_ar_element = parse_ar_element
+
+return exports
diff --git a/lualib/lua_aws.lua b/lualib/lua_aws.lua
new file mode 100644
index 0000000..e6c4b29
--- /dev/null
+++ b/lualib/lua_aws.lua
@@ -0,0 +1,300 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+
+--[[[
+-- @module lua_aws
+-- This module contains Amazon AWS utility functions
+--]]
+
+local N = "aws"
+--local rspamd_logger = require "rspamd_logger"
+local ts = (require "tableshape").types
+local lua_util = require "lua_util"
+local fun = require "fun"
+local rspamd_crypto_hash = require "rspamd_cryptobox_hash"
+
+local exports = {}
+
+-- Returns a canonical representation of today date
+local function today_canonical()
+ return os.date('!%Y%m%d')
+end
+
+--[[[
+-- @function lua_aws.aws_date([date_str])
+-- Returns an aws date header corresponding to the specific date
+--]]
+local function aws_date(date_str)
+ if not date_str then
+ date_str = today_canonical()
+ end
+
+ return date_str .. os.date('!T%H%M%SZ')
+end
+
+exports.aws_date = aws_date
+
+
+-- Local cache of the keys to save resources
+local cached_keys = {}
+
+local function maybe_get_cached_key(date_str, secret_key, region, service, req_type)
+ local bucket = cached_keys[tonumber(date_str)]
+
+ if not bucket then
+ return nil
+ end
+
+ local elt = bucket[string.format('%s.%s.%s.%s', secret_key, region, service, req_type)]
+ if elt then
+ return elt
+ end
+end
+
+local function save_cached_key(date_str, secret_key, region, service, req_type, key)
+ local numdate = tonumber(date_str)
+ -- expire old buckets
+ for k, _ in pairs(cached_keys) do
+ if k < numdate then
+ cached_keys[k] = nil
+ end
+ end
+
+ local bucket = cached_keys[tonumber(date_str)]
+ local idx = string.format('%s.%s.%s.%s', secret_key, region, service, req_type)
+
+ if not bucket then
+ cached_keys[tonumber(date_str)] = {
+ idx = key
+ }
+ else
+ bucket[idx] = key
+ end
+end
+--[[[
+-- @function lua_aws.aws_signing_key([date_str], secret_key, region, [service='s3'], [req_type='aws4_request'])
+-- Returns a signing key for the specific parameters
+--]]
+local function aws_signing_key(date_str, secret_key, region, service, req_type)
+ if not date_str then
+ date_str = today_canonical()
+ end
+
+ if not service then
+ service = 's3'
+ end
+
+ if not req_type then
+ req_type = 'aws4_request'
+ end
+
+ assert(type(secret_key) == 'string')
+ assert(type(region) == 'string')
+
+ local maybe_cached = maybe_get_cached_key(date_str, secret_key, region, service, req_type)
+
+ if maybe_cached then
+ return maybe_cached
+ end
+
+ local hmac1 = rspamd_crypto_hash.create_specific_keyed("AWS4" .. secret_key, "sha256", date_str):bin()
+ local hmac2 = rspamd_crypto_hash.create_specific_keyed(hmac1, "sha256", region):bin()
+ local hmac3 = rspamd_crypto_hash.create_specific_keyed(hmac2, "sha256", service):bin()
+ local final_key = rspamd_crypto_hash.create_specific_keyed(hmac3, "sha256", req_type):bin()
+
+ save_cached_key(date_str, secret_key, region, service, req_type, final_key)
+
+ return final_key
+end
+
+exports.aws_signing_key = aws_signing_key
+
+--[[[
+-- @function lua_aws.aws_canon_request_hash(method, path, headers_to_sign, hex_hash)
+-- Returns a hash + list of headers as required to produce signature afterwards
+--]]
+local function aws_canon_request_hash(method, uri, headers_to_sign, hex_hash)
+ assert(type(method) == 'string')
+ assert(type(uri) == 'string')
+ assert(type(headers_to_sign) == 'table')
+
+ if not hex_hash then
+ hex_hash = headers_to_sign['x-amz-content-sha256']
+ end
+
+ assert(type(hex_hash) == 'string')
+
+ local sha_ctx = rspamd_crypto_hash.create_specific('sha256')
+
+ lua_util.debugm(N, 'update signature with the method %s',
+ method)
+ sha_ctx:update(method .. '\n')
+ lua_util.debugm(N, 'update signature with the uri %s',
+ uri)
+ sha_ctx:update(uri .. '\n')
+ -- XXX add query string canonicalisation
+ sha_ctx:update('\n')
+ -- Sort auth headers and canonicalise them as requested
+ local hdr_canon = fun.tomap(fun.map(function(k, v)
+ return k:lower(), lua_util.str_trim(v)
+ end, headers_to_sign))
+ local header_names = lua_util.keys(hdr_canon)
+ table.sort(header_names)
+ for _, hn in ipairs(header_names) do
+ local v = hdr_canon[hn]
+ lua_util.debugm(N, 'update signature with the header %s, %s',
+ hn, v)
+ sha_ctx:update(string.format('%s:%s\n', hn, v))
+ end
+ local hdrs_list = table.concat(header_names, ';')
+ lua_util.debugm(N, 'headers list to sign: %s', hdrs_list)
+ sha_ctx:update(string.format('\n%s\n%s', hdrs_list, hex_hash))
+
+ return sha_ctx:hex(), hdrs_list
+end
+
+exports.aws_canon_request_hash = aws_canon_request_hash
+
+local aws_authorization_hdr_args_schema = ts.shape {
+ date = ts.string + ts['nil'] / today_canonical,
+ secret_key = ts.string,
+ method = ts.string + ts['nil'] / function()
+ return 'GET'
+ end,
+ uri = ts.string,
+ region = ts.string,
+ service = ts.string + ts['nil'] / function()
+ return 's3'
+ end,
+ req_type = ts.string + ts['nil'] / function()
+ return 'aws4_request'
+ end,
+ headers = ts.map_of(ts.string, ts.string),
+ key_id = ts.string,
+}
+--[[[
+-- @function lua_aws.aws_authorization_hdr(params)
+-- Produces an authorization header as required by AWS
+-- Parameters schema is the following:
+ts.shape{
+ date = ts.string + ts['nil'] / today_canonical,
+ secret_key = ts.string,
+ method = ts.string + ts['nil'] / function() return 'GET' end,
+ uri = ts.string,
+ region = ts.string,
+ service = ts.string + ts['nil'] / function() return 's3' end,
+ req_type = ts.string + ts['nil'] / function() return 'aws4_request' end,
+ headers = ts.map_of(ts.string, ts.string),
+ key_id = ts.string,
+}
+--
+--]]
+local function aws_authorization_hdr(tbl, transformed)
+ local res, err
+ if not transformed then
+ res, err = aws_authorization_hdr_args_schema:transform(tbl)
+ assert(res, err)
+ else
+ res = tbl
+ end
+
+ local signing_key = aws_signing_key(res.date, res.secret_key, res.region, res.service,
+ res.req_type)
+ assert(signing_key ~= nil)
+ local signed_sha, signed_hdrs = aws_canon_request_hash(res.method, res.uri,
+ res.headers)
+
+ if not signed_sha then
+ return nil
+ end
+
+ local string_to_sign = string.format('AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s',
+ res.headers['x-amz-date'] or aws_date(),
+ res.date, res.region, res.service, res.req_type,
+ signed_sha)
+ lua_util.debugm(N, "string to sign: %s", string_to_sign)
+ local hmac = rspamd_crypto_hash.create_specific_keyed(signing_key, 'sha256', string_to_sign):hex()
+ lua_util.debugm(N, "hmac: %s", hmac)
+ local auth_hdr = string.format('AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/%s,' ..
+ 'SignedHeaders=%s,Signature=%s',
+ res.key_id, res.date, res.region, res.service, res.req_type,
+ signed_hdrs, hmac)
+
+ return auth_hdr
+end
+
+exports.aws_authorization_hdr = aws_authorization_hdr
+
+
+
+--[[[
+-- @function lua_aws.aws_request_enrich(params, content)
+-- Produces an authorization header as required by AWS
+-- Parameters schema is the following:
+ts.shape{
+ date = ts.string + ts['nil'] / today_canonical,
+ secret_key = ts.string,
+ method = ts.string + ts['nil'] / function() return 'GET' end,
+ uri = ts.string,
+ region = ts.string,
+ service = ts.string + ts['nil'] / function() return 's3' end,
+ req_type = ts.string + ts['nil'] / function() return 'aws4_request' end,
+ headers = ts.map_of(ts.string, ts.string),
+ key_id = ts.string,
+}
+This method returns new/modified in place table of the headers
+--
+--]]
+local function aws_request_enrich(tbl, content)
+ local res, err = aws_authorization_hdr_args_schema:transform(tbl)
+ assert(res, err)
+ local content_sha256 = rspamd_crypto_hash.create_specific('sha256', content):hex()
+ local hdrs = res.headers
+ hdrs['x-amz-content-sha256'] = content_sha256
+ if not hdrs['x-amz-date'] then
+ hdrs['x-amz-date'] = aws_date(res.date)
+ end
+ hdrs['Authorization'] = aws_authorization_hdr(res, true)
+
+ return hdrs
+end
+
+exports.aws_request_enrich = aws_request_enrich
+
+-- A simple tests according to AWS docs to check sanity
+local test_request_hdrs = {
+ ['Host'] = 'examplebucket.s3.amazonaws.com',
+ ['x-amz-date'] = '20130524T000000Z',
+ ['Range'] = 'bytes=0-9',
+ ['x-amz-content-sha256'] = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+}
+
+assert(aws_canon_request_hash('GET', '/test.txt', test_request_hdrs) ==
+ '7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972')
+
+assert(aws_authorization_hdr {
+ date = '20130524',
+ region = 'us-east-1',
+ headers = test_request_hdrs,
+ uri = '/test.txt',
+ key_id = 'AKIAIOSFODNN7EXAMPLE',
+ secret_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
+} == 'AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,' ..
+ 'SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,' ..
+ 'Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41')
+
+return exports
diff --git a/lualib/lua_bayes_learn.lua b/lualib/lua_bayes_learn.lua
new file mode 100644
index 0000000..ea97db6
--- /dev/null
+++ b/lualib/lua_bayes_learn.lua
@@ -0,0 +1,151 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- This file contains functions to simplify bayes classifier auto-learning
+
+local lua_util = require "lua_util"
+local lua_verdict = require "lua_verdict"
+local N = "lua_bayes"
+
+local exports = {}
+
+exports.can_learn = function(task, is_spam, is_unlearn)
+ local learn_type = task:get_request_header('Learn-Type')
+
+ if not (learn_type and tostring(learn_type) == 'bulk') then
+ local prob = task:get_mempool():get_variable('bayes_prob', 'double')
+
+ if prob then
+ local in_class = false
+ local cl
+ if is_spam then
+ cl = 'spam'
+ in_class = prob >= 0.95
+ else
+ cl = 'ham'
+ in_class = prob <= 0.05
+ end
+
+ if in_class then
+ return false, string.format(
+ 'already in class %s; probability %.2f%%',
+ cl, math.abs((prob - 0.5) * 200.0))
+ end
+ end
+ end
+
+ return true
+end
+
+exports.autolearn = function(task, conf)
+ local function log_can_autolearn(verdict, score, threshold)
+ local from = task:get_from('smtp')
+ local mime_rcpts = 'undef'
+ local mr = task:get_recipients('mime')
+ if mr then
+ for _, r in ipairs(mr) do
+ if mime_rcpts == 'undef' then
+ mime_rcpts = r.addr
+ else
+ mime_rcpts = mime_rcpts .. ',' .. r.addr
+ end
+ end
+ end
+
+ lua_util.debugm(N, task, 'id: %s, from: <%s>: can autolearn %s: score %s %s %s, mime_rcpts: <%s>',
+ task:get_header('Message-Id') or '<undef>',
+ from and from[1].addr or 'undef',
+ verdict,
+ string.format("%.2f", score),
+ verdict == 'ham' and '<=' or verdict == 'spam' and '>=' or '/',
+ threshold,
+ mime_rcpts)
+ end
+
+ -- We have autolearn config so let's figure out what is requested
+ local verdict, score = lua_verdict.get_specific_verdict("bayes", task)
+ local learn_spam, learn_ham = false, false
+
+ if verdict == 'passthrough' then
+ -- No need to autolearn
+ lua_util.debugm(N, task, 'no need to autolearn - verdict: %s',
+ verdict)
+ return
+ end
+
+ if conf.spam_threshold and conf.ham_threshold then
+ if verdict == 'spam' then
+ if conf.spam_threshold and score >= conf.spam_threshold then
+ log_can_autolearn(verdict, score, conf.spam_threshold)
+ learn_spam = true
+ end
+ elseif verdict == 'junk' then
+ if conf.junk_threshold and score >= conf.junk_threshold then
+ log_can_autolearn(verdict, score, conf.junk_threshold)
+ learn_spam = true
+ end
+ elseif verdict == 'ham' then
+ if conf.ham_threshold and score <= conf.ham_threshold then
+ log_can_autolearn(verdict, score, conf.ham_threshold)
+ learn_ham = true
+ end
+ end
+ elseif conf.learn_verdict then
+ if verdict == 'spam' or verdict == 'junk' then
+ learn_spam = true
+ elseif verdict == 'ham' then
+ learn_ham = true
+ end
+ end
+
+ if conf.check_balance then
+ -- Check balance of learns
+ local spam_learns = task:get_mempool():get_variable('spam_learns', 'int64') or 0
+ local ham_learns = task:get_mempool():get_variable('ham_learns', 'int64') or 0
+
+ local min_balance = 0.9
+ if conf.min_balance then
+ min_balance = conf.min_balance
+ end
+
+ if spam_learns > 0 or ham_learns > 0 then
+ local max_ratio = 1.0 / min_balance
+ local spam_learns_ratio = spam_learns / (ham_learns + 1)
+ if spam_learns_ratio > max_ratio and learn_spam then
+ lua_util.debugm(N, task,
+ 'skip learning spam, balance is not satisfied: %s < %s; %s spam learns; %s ham learns',
+ spam_learns_ratio, min_balance, spam_learns, ham_learns)
+ learn_spam = false
+ end
+
+ local ham_learns_ratio = ham_learns / (spam_learns + 1)
+ if ham_learns_ratio > max_ratio and learn_ham then
+ lua_util.debugm(N, task,
+ 'skip learning ham, balance is not satisfied: %s < %s; %s spam learns; %s ham learns',
+ ham_learns_ratio, min_balance, spam_learns, ham_learns)
+ learn_ham = false
+ end
+ end
+ end
+
+ if learn_spam then
+ return 'spam'
+ elseif learn_ham then
+ return 'ham'
+ end
+end
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_bayes_redis.lua b/lualib/lua_bayes_redis.lua
new file mode 100644
index 0000000..7533997
--- /dev/null
+++ b/lualib/lua_bayes_redis.lua
@@ -0,0 +1,244 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]
+
+-- This file contains functions to support Bayes statistics in Redis
+
+local exports = {}
+local lua_redis = require "lua_redis"
+local logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local ucl = require "ucl"
+
+local N = "bayes"
+
+local function gen_classify_functor(redis_params, classify_script_id)
+ return function(task, expanded_key, id, is_spam, stat_tokens, callback)
+
+ local function classify_redis_cb(err, data)
+ lua_util.debugm(N, task, 'classify redis cb: %s, %s', err, data)
+ if err then
+ callback(task, false, err)
+ else
+ callback(task, true, data[1], data[2], data[3], data[4])
+ end
+ end
+
+ lua_redis.exec_redis_script(classify_script_id,
+ { task = task, is_write = false, key = expanded_key },
+ classify_redis_cb, { expanded_key, stat_tokens })
+ end
+end
+
+local function gen_learn_functor(redis_params, learn_script_id)
+ return function(task, expanded_key, id, is_spam, symbol, is_unlearn, stat_tokens, callback, maybe_text_tokens)
+ local function learn_redis_cb(err, data)
+ lua_util.debugm(N, task, 'learn redis cb: %s, %s', err, data)
+ if err then
+ callback(task, false, err)
+ else
+ callback(task, true)
+ end
+ end
+
+ if maybe_text_tokens then
+ lua_redis.exec_redis_script(learn_script_id,
+ { task = task, is_write = true, key = expanded_key },
+ learn_redis_cb,
+ { expanded_key, tostring(is_spam), symbol, tostring(is_unlearn), stat_tokens, maybe_text_tokens })
+ else
+ lua_redis.exec_redis_script(learn_script_id,
+ { task = task, is_write = true, key = expanded_key },
+ learn_redis_cb, { expanded_key, tostring(is_spam), symbol, tostring(is_unlearn), stat_tokens })
+ end
+
+ end
+end
+
+local function load_redis_params(classifier_ucl, statfile_ucl)
+ local redis_params
+
+ -- Try load from statfile options
+ if statfile_ucl.redis then
+ redis_params = lua_redis.try_load_redis_servers(statfile_ucl.redis, rspamd_config, true)
+ end
+
+ if not redis_params then
+ if statfile_ucl then
+ redis_params = lua_redis.try_load_redis_servers(statfile_ucl, rspamd_config, true)
+ end
+ end
+
+ -- Try load from classifier config
+ if not redis_params and classifier_ucl.backend then
+ redis_params = lua_redis.try_load_redis_servers(classifier_ucl.backend, rspamd_config, true)
+ end
+
+ if not redis_params and classifier_ucl.redis then
+ redis_params = lua_redis.try_load_redis_servers(classifier_ucl.redis, rspamd_config, true)
+ end
+
+ if not redis_params then
+ redis_params = lua_redis.try_load_redis_servers(classifier_ucl, rspamd_config, true)
+ end
+
+ -- Try load global options
+ if not redis_params then
+ redis_params = lua_redis.try_load_redis_servers(rspamd_config:get_all_opt('redis'), rspamd_config, true)
+ end
+
+ if not redis_params then
+ logger.err(rspamd_config, "cannot load Redis parameters for the classifier")
+ return nil
+ end
+
+ return redis_params
+end
+
+---
+--- Init bayes classifier
+--- @param classifier_ucl ucl of the classifier config
+--- @param statfile_ucl ucl of the statfile config
+--- @return a pair of (classify_functor, learn_functor) or `nil` in case of error
+exports.lua_bayes_init_statfile = function(classifier_ucl, statfile_ucl, symbol, is_spam, ev_base, stat_periodic_cb)
+
+ local redis_params = load_redis_params(classifier_ucl, statfile_ucl)
+
+ if not redis_params then
+ return nil
+ end
+
+ local classify_script_id = lua_redis.load_redis_script_from_file("bayes_classify.lua", redis_params)
+ local learn_script_id = lua_redis.load_redis_script_from_file("bayes_learn.lua", redis_params)
+ local stat_script_id = lua_redis.load_redis_script_from_file("bayes_stat.lua", redis_params)
+ local max_users = classifier_ucl.max_users or 1000
+
+ local current_data = {
+ users = 0,
+ revision = 0,
+ }
+ local final_data = {
+ users = 0,
+ revision = 0, -- number of learns
+ }
+ local cursor = 0
+ rspamd_config:add_periodic(ev_base, 0.0, function(cfg, _)
+
+ local function stat_redis_cb(err, data)
+ lua_util.debugm(N, cfg, 'stat redis cb: %s, %s', err, data)
+
+ if err then
+ logger.warn(cfg, 'cannot get bayes statistics for %s: %s', symbol, err)
+ else
+ local new_cursor = data[1]
+ current_data.users = current_data.users + data[2]
+ current_data.revision = current_data.revision + data[3]
+ if new_cursor == 0 then
+ -- Done iteration
+ final_data = lua_util.shallowcopy(current_data)
+ current_data = {
+ users = 0,
+ revision = 0,
+ }
+ lua_util.debugm(N, cfg, 'final data: %s', final_data)
+ stat_periodic_cb(cfg, final_data)
+ end
+
+ cursor = new_cursor
+ end
+ end
+
+ lua_redis.exec_redis_script(stat_script_id,
+ { ev_base = ev_base, cfg = cfg, is_write = false },
+ stat_redis_cb, { tostring(cursor),
+ symbol,
+ is_spam and "learns_spam" or "learns_ham",
+ tostring(max_users) })
+ return statfile_ucl.monitor_timeout or classifier_ucl.monitor_timeout or 30.0
+ end)
+
+ return gen_classify_functor(redis_params, classify_script_id), gen_learn_functor(redis_params, learn_script_id)
+end
+
+local function gen_cache_check_functor(redis_params, check_script_id, conf)
+ local packed_conf = ucl.to_format(conf, 'msgpack')
+ return function(task, cache_id, callback)
+
+ local function classify_redis_cb(err, data)
+ lua_util.debugm(N, task, 'check cache redis cb: %s, %s (%s)', err, data, type(data))
+ if err then
+ callback(task, false, err)
+ else
+ if type(data) == 'number' then
+ callback(task, true, data)
+ else
+ callback(task, false, 'not found')
+ end
+ end
+ end
+
+ lua_util.debugm(N, task, 'checking cache: %s', cache_id)
+ lua_redis.exec_redis_script(check_script_id,
+ { task = task, is_write = false, key = cache_id },
+ classify_redis_cb, { cache_id, packed_conf })
+ end
+end
+
+local function gen_cache_learn_functor(redis_params, learn_script_id, conf)
+ local packed_conf = ucl.to_format(conf, 'msgpack')
+ return function(task, cache_id, is_spam)
+ local function learn_redis_cb(err, data)
+ lua_util.debugm(N, task, 'learn_cache redis cb: %s, %s', err, data)
+ end
+
+ lua_util.debugm(N, task, 'try to learn cache: %s', cache_id)
+ lua_redis.exec_redis_script(learn_script_id,
+ { task = task, is_write = true, key = cache_id },
+ learn_redis_cb,
+ { cache_id, is_spam and "1" or "0", packed_conf })
+
+ end
+end
+
+exports.lua_bayes_init_cache = function(classifier_ucl, statfile_ucl)
+ local redis_params = load_redis_params(classifier_ucl, statfile_ucl)
+
+ if not redis_params then
+ return nil
+ end
+
+ local default_conf = {
+ cache_prefix = "learned_ids",
+ cache_max_elt = 10000, -- Maximum number of elements in the cache key
+ cache_max_keys = 5, -- Maximum number of keys in the cache
+ cache_elt_len = 32, -- Length of the element in the cache (will trim id to that value)
+ }
+
+ local conf = lua_util.override_defaults(default_conf, classifier_ucl)
+ -- Clean all not known configurations
+ for k, _ in pairs(conf) do
+ if default_conf[k] == nil then
+ conf[k] = nil
+ end
+ end
+
+ local check_script_id = lua_redis.load_redis_script_from_file("bayes_cache_check.lua", redis_params)
+ local learn_script_id = lua_redis.load_redis_script_from_file("bayes_cache_learn.lua", redis_params)
+
+ return gen_cache_check_functor(redis_params, check_script_id, conf), gen_cache_learn_functor(redis_params,
+ learn_script_id, conf)
+end
+
+return exports
diff --git a/lualib/lua_cfg_transform.lua b/lualib/lua_cfg_transform.lua
new file mode 100644
index 0000000..d6243ad
--- /dev/null
+++ b/lualib/lua_cfg_transform.lua
@@ -0,0 +1,634 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local fun = require "fun"
+
+local function is_implicit(t)
+ local mt = getmetatable(t)
+
+ return mt and mt.class and mt.class == 'ucl.type.impl_array'
+end
+
+local function metric_pairs(t)
+ -- collect the keys
+ local keys = {}
+ local implicit_array = is_implicit(t)
+
+ local function gen_keys(tbl)
+ if implicit_array then
+ for _, v in ipairs(tbl) do
+ if v.name then
+ table.insert(keys, { v.name, v })
+ v.name = nil
+ else
+ -- Very tricky to distinguish:
+ -- group {name = "foo" ... } + group "blah" { ... }
+ for gr_name, gr in pairs(v) do
+ if type(gr_name) ~= 'number' then
+ -- We can also have implicit arrays here
+ local gr_implicit = is_implicit(gr)
+
+ if gr_implicit then
+ for _, gr_elt in ipairs(gr) do
+ table.insert(keys, { gr_name, gr_elt })
+ end
+ else
+ table.insert(keys, { gr_name, gr })
+ end
+ end
+ end
+ end
+ end
+ else
+ if tbl.name then
+ table.insert(keys, { tbl.name, tbl })
+ tbl.name = nil
+ else
+ for k, v in pairs(tbl) do
+ if type(k) ~= 'number' then
+ -- We can also have implicit arrays here
+ local sym_implicit = is_implicit(v)
+
+ if sym_implicit then
+ for _, elt in ipairs(v) do
+ table.insert(keys, { k, elt })
+ end
+ else
+ table.insert(keys, { k, v })
+ end
+ end
+ end
+ end
+ end
+ end
+
+ gen_keys(t)
+
+ -- return the iterator function
+ local i = 0
+ return function()
+ i = i + 1
+ if keys[i] then
+ return keys[i][1], keys[i][2]
+ end
+ end
+end
+
+local function group_transform(cfg, k, v)
+ if v.name then
+ k = v.name
+ end
+
+ local new_group = {
+ symbols = {}
+ }
+
+ if v.enabled then
+ new_group.enabled = v.enabled
+ end
+ if v.disabled then
+ new_group.disabled = v.disabled
+ end
+ if v.max_score then
+ new_group.max_score = v.max_score
+ end
+
+ if v.symbol then
+ for sk, sv in metric_pairs(v.symbol) do
+ if sv.name then
+ sk = sv.name
+ sv.name = nil -- Remove field
+ end
+
+ new_group.symbols[sk] = sv
+ end
+ end
+
+ if not cfg.group then
+ cfg.group = {}
+ end
+
+ if cfg.group[k] then
+ cfg.group[k] = lua_util.override_defaults(cfg.group[k], new_group)
+ else
+ cfg.group[k] = new_group
+ end
+
+ logger.infox("overriding group %s from the legacy metric settings", k)
+end
+
+local function symbol_transform(cfg, k, v)
+ -- first try to find any group where there is a definition of this symbol
+ for gr_n, gr in pairs(cfg.group) do
+ if gr.symbols and gr.symbols[k] then
+ -- We override group symbol with ungrouped symbol
+ logger.infox("overriding group symbol %s in the group %s", k, gr_n)
+ gr.symbols[k] = lua_util.override_defaults(gr.symbols[k], v)
+ return
+ end
+ end
+ -- Now check what Rspamd knows about this symbol
+ local sym = rspamd_config:get_symbol(k)
+
+ if not sym or not sym.group then
+ -- Otherwise we just use group 'ungrouped'
+ if not cfg.group.ungrouped then
+ cfg.group.ungrouped = {
+ symbols = {}
+ }
+ end
+
+ cfg.group.ungrouped.symbols[k] = v
+ logger.debugx("adding symbol %s to the group 'ungrouped'", k)
+ end
+end
+
+local function test_groups(groups)
+ for gr_name, gr in pairs(groups) do
+ if not gr.symbols then
+ local cnt = 0
+ for _, _ in pairs(gr) do
+ cnt = cnt + 1
+ end
+
+ if cnt == 0 then
+ logger.debugx('group %s is empty', gr_name)
+ else
+ logger.infox('group %s has no symbols', gr_name)
+ end
+ end
+ end
+end
+
+local function convert_metric(cfg, metric)
+ if metric.actions then
+ cfg.actions = lua_util.override_defaults(cfg.actions, metric.actions)
+ logger.infox("overriding actions from the legacy metric settings")
+ end
+ if metric.unknown_weight then
+ cfg.actions.unknown_weight = metric.unknown_weight
+ end
+
+ if metric.subject then
+ logger.infox("overriding subject from the legacy metric settings")
+ cfg.actions.subject = metric.subject
+ end
+
+ if metric.group then
+ for k, v in metric_pairs(metric.group) do
+ group_transform(cfg, k, v)
+ end
+ else
+ if not cfg.group then
+ cfg.group = {
+ ungrouped = {
+ symbols = {}
+ }
+ }
+ end
+ end
+
+ if metric.symbol then
+ for k, v in metric_pairs(metric.symbol) do
+ symbol_transform(cfg, k, v)
+ end
+ end
+
+ return cfg
+end
+
+-- Converts a table of groups indexed by number (implicit array) to a
+-- merged group definition
+local function merge_groups(groups)
+ local ret = {}
+ for k, gr in pairs(groups) do
+ if type(k) == 'number' then
+ for key, sec in pairs(gr) do
+ ret[key] = sec
+ end
+ else
+ ret[k] = gr
+ end
+ end
+
+ return ret
+end
+
+-- Checks configuration files for statistics
+local function check_statistics_sanity()
+ local local_conf = rspamd_paths['LOCAL_CONFDIR']
+ local local_stat = string.format('%s/local.d/%s', local_conf,
+ 'statistic.conf')
+ local local_bayes = string.format('%s/local.d/%s', local_conf,
+ 'classifier-bayes.conf')
+
+ if rspamd_util.file_exists(local_stat) and
+ rspamd_util.file_exists(local_bayes) then
+ logger.warnx(rspamd_config, 'conflicting files %s and %s are found: ' ..
+ 'Rspamd classifier configuration might be broken!', local_stat, local_bayes)
+ end
+end
+
+-- Converts surbl module config to rbl module
+local function surbl_section_convert(cfg, section)
+ local rbl_section = cfg.rbl.rbls
+ local wl = section.whitelist
+ for name, value in pairs(section.rules or {}) do
+ if rbl_section[name] then
+ logger.warnx(rspamd_config, 'conflicting names in surbl and rbl rules: %s, prefer surbl rule!',
+ name)
+ end
+ local converted = {
+ urls = true,
+ ignore_defaults = true,
+ }
+
+ if wl then
+ converted.whitelist = wl
+ end
+
+ for k, v in pairs(value) do
+ local skip = false
+ -- Rename
+ if k == 'suffix' then
+ k = 'rbl'
+ end
+ if k == 'ips' then
+ k = 'returncodes'
+ end
+ if k == 'bits' then
+ k = 'returnbits'
+ end
+ if k == 'noip' then
+ k = 'no_ip'
+ end
+ -- Crappy legacy
+ if k == 'options' then
+ if v == 'noip' or v == 'no_ip' then
+ converted.no_ip = true
+ skip = true
+ end
+ end
+ if k:match('check_') then
+ local n = k:match('check_(.*)')
+ k = n
+ end
+
+ if k == 'dkim' and v then
+ converted.dkim_domainonly = false
+ converted.dkim_match_from = true
+ end
+
+ if k == 'emails' and v then
+ -- To match surbl behaviour
+ converted.emails_domainonly = true
+ end
+
+ if not skip then
+ converted[k] = lua_util.deepcopy(v)
+ end
+ end
+ rbl_section[name] = lua_util.override_defaults(rbl_section[name], converted)
+ end
+end
+
+-- Converts surbl module config to rbl module
+local function emails_section_convert(cfg, section)
+ local rbl_section = cfg.rbl.rbls
+ local wl = section.whitelist
+ for name, value in pairs(section.rules or {}) do
+ if rbl_section[name] then
+ logger.warnx(rspamd_config, 'conflicting names in emails and rbl rules: %s, prefer emails rule!',
+ name)
+ end
+ local converted = {
+ emails = true,
+ ignore_defaults = true,
+ }
+
+ if wl then
+ converted.whitelist = wl
+ end
+
+ for k, v in pairs(value) do
+ local skip = false
+ -- Rename
+ if k == 'dnsbl' then
+ k = 'rbl'
+ end
+ if k == 'check_replyto' then
+ k = 'replyto'
+ end
+ if k == 'hashlen' then
+ k = 'hash_len'
+ end
+ if k == 'encoding' then
+ k = 'hash_format'
+ end
+ if k == 'domain_only' then
+ k = 'emails_domainonly'
+ end
+ if k == 'delimiter' then
+ k = 'emails_delimiter'
+ end
+ if k == 'skip_body' then
+ skip = true
+ if v then
+ -- Hack
+ converted.emails = false
+ converted.replyto = true
+ else
+ converted.emails = true
+ end
+ end
+ if k == 'expect_ip' then
+ -- Another stupid hack
+ if not converted.return_codes then
+ converted.returncodes = {}
+ end
+ local symbol = value.symbol or name
+ converted.returncodes[symbol] = { v }
+ skip = true
+ end
+
+ if not skip then
+ converted[k] = lua_util.deepcopy(v)
+ end
+ end
+ rbl_section[name] = lua_util.override_defaults(rbl_section[name], converted)
+ end
+end
+
+return function(cfg)
+ local ret = false
+
+ if cfg['metric'] then
+ for _, v in metric_pairs(cfg.metric) do
+ cfg = convert_metric(cfg, v)
+ end
+ ret = true
+ end
+
+ if cfg.symbols then
+ for k, v in metric_pairs(cfg.symbols) do
+ symbol_transform(cfg, k, v)
+ end
+ end
+
+ check_statistics_sanity()
+
+ if not cfg.actions then
+ logger.errx('no actions defined')
+ else
+ -- Perform sanity check for actions
+ local actions_defs = { 'no action', 'no_action', -- In case if that's added
+ 'greylist', 'add header', 'add_header',
+ 'rewrite subject', 'rewrite_subject', 'quarantine',
+ 'reject', 'discard' }
+
+ if not cfg.actions['no action'] and not cfg.actions['no_action'] and
+ not cfg.actions['accept'] then
+ for _, d in ipairs(actions_defs) do
+ if cfg.actions[d] then
+
+ local action_score = nil
+ if type(cfg.actions[d]) == 'number' then
+ action_score = cfg.actions[d]
+ elseif type(cfg.actions[d]) == 'table' and cfg.actions[d]['score'] then
+ action_score = cfg.actions[d]['score']
+ end
+
+ if type(cfg.actions[d]) ~= 'table' and not action_score then
+ cfg.actions[d] = nil
+ elseif type(action_score) == 'number' and action_score < 0 then
+ cfg.actions['no_action'] = cfg.actions[d] - 0.001
+ logger.infox(rspamd_config, 'set no_action score to: %s, as action %s has negative score',
+ cfg.actions['no_action'], d)
+ break
+ end
+ end
+ end
+ end
+
+ local actions_set = lua_util.list_to_hash(actions_defs)
+
+ -- Now check actions section for garbage
+ actions_set['unknown_weight'] = true
+ actions_set['grow_factor'] = true
+ actions_set['subject'] = true
+
+ for k, _ in pairs(cfg.actions) do
+ if not actions_set[k] then
+ logger.warnx(rspamd_config, 'unknown element in actions section: %s', k)
+ end
+ end
+
+ -- Performs thresholds sanity
+ -- We exclude greylist here as it can be set to whatever threshold in practice
+ local actions_order = {
+ 'no_action',
+ 'add_header',
+ 'rewrite_subject',
+ 'quarantine',
+ 'reject',
+ 'discard'
+ }
+ for i = 1, (#actions_order - 1) do
+ local act = actions_order[i]
+
+ if cfg.actions[act] and type(cfg.actions[act]) == 'number' then
+ local score = cfg.actions[act]
+
+ for j = i + 1, #actions_order do
+ local next_act = actions_order[j]
+ if cfg.actions[next_act] and type(cfg.actions[next_act]) == 'number' then
+ local next_score = cfg.actions[next_act]
+ if next_score <= score then
+ logger.errx(rspamd_config, 'invalid actions thresholds order: action %s (%s) must have lower ' ..
+ 'score than action %s (%s)', act, score, next_act, next_score)
+ ret = false
+ end
+ end
+ end
+ end
+ end
+ end
+
+ if not cfg.group then
+ logger.errx('no symbol groups defined')
+ else
+ if cfg.group[1] then
+ -- We need to merge groups
+ cfg.group = merge_groups(cfg.group)
+ ret = true
+ end
+ test_groups(cfg.group)
+ end
+
+ -- Deal with dkim settings
+ if not cfg.dkim then
+ cfg.dkim = {}
+ else
+ if cfg.dkim.sign_condition then
+ -- We have an obsoleted sign condition, so we need to either add dkim_signing and move it
+ -- there or just move sign condition there...
+ if not cfg.dkim_signing then
+ logger.warnx('obsoleted DKIM signing method used, converting it to "dkim_signing" module')
+ cfg.dkim_signing = {
+ sign_condition = cfg.dkim.sign_condition
+ }
+ else
+ if not cfg.dkim_signing.sign_condition then
+ logger.warnx('obsoleted DKIM signing method used, move it to "dkim_signing" module')
+ cfg.dkim_signing.sign_condition = cfg.dkim.sign_condition
+ else
+ logger.warnx('obsoleted DKIM signing method used, ignore it as "dkim_signing" also defines condition!')
+ end
+ end
+ end
+ end
+
+ -- Again: legacy stuff :(
+ if not cfg.dkim.sign_headers then
+ local sec = cfg.dkim_signing
+ if sec and sec[1] then
+ sec = cfg.dkim_signing[1]
+ end
+
+ if sec and sec.sign_headers then
+ cfg.dkim.sign_headers = sec.sign_headers
+ end
+ end
+
+ -- DKIM signing/ARC legacy
+ for _, mod in ipairs({ 'dkim_signing', 'arc' }) do
+ if cfg[mod] then
+ if cfg[mod].auth_only ~= nil then
+ if cfg[mod].sign_authenticated ~= nil then
+ logger.warnx(rspamd_config,
+ 'both auth_only (%s) and sign_authenticated (%s) for %s are specified, prefer auth_only',
+ cfg[mod].auth_only, cfg[mod].sign_authenticated, mod)
+ end
+ cfg[mod].sign_authenticated = cfg[mod].auth_only
+ end
+ end
+ end
+
+ if cfg.dkim and cfg.dkim.sign_headers and type(cfg.dkim.sign_headers) == 'table' then
+ -- Flatten
+ cfg.dkim.sign_headers = table.concat(cfg.dkim.sign_headers, ':')
+ end
+
+ -- Try to find some obvious issues with configuration
+ for k, v in pairs(cfg) do
+ if type(v) == 'table' and v[k] and type(v[k]) == 'table' then
+ logger.errx('nested section: %s { %s { ... } }, it is likely a configuration error',
+ k, k)
+ end
+ end
+
+ -- If neural network is enabled we MUST have `check_all_filters` flag
+ if cfg.neural then
+ if not cfg.options then
+ cfg.options = {}
+ end
+
+ if not cfg.options.check_all_filters then
+ logger.infox(rspamd_config, 'enable `options.check_all_filters` for neural network')
+ cfg.options.check_all_filters = true
+ end
+ end
+
+ -- Deal with IP_SCORE
+ if cfg.ip_score and (cfg.ip_score.servers or cfg.redis.servers) then
+ logger.warnx(rspamd_config, 'ip_score module is deprecated in honor of reputation module!')
+
+ if not cfg.reputation then
+ cfg.reputation = {
+ rules = {}
+ }
+ end
+
+ if not cfg.reputation.rules then
+ cfg.reputation.rules = {}
+ end
+
+ if not fun.any(function(_, v)
+ return v.selector and v.selector.ip
+ end,
+ cfg.reputation.rules) then
+ logger.infox(rspamd_config, 'attach ip reputation element to use it')
+
+ cfg.reputation.rules.ip_score = {
+ selector = {
+ ip = {},
+ },
+ backend = {
+ redis = {},
+ }
+ }
+
+ if cfg.ip_score.servers then
+ cfg.reputation.rules.ip_score.backend.redis.servers = cfg.ip_score.servers
+ end
+
+ if cfg.symbols and cfg.symbols['IP_SCORE'] then
+ local t = cfg.symbols['IP_SCORE']
+
+ if not cfg.symbols['SENDER_REP_SPAM'] then
+ cfg.symbols['SENDER_REP_SPAM'] = t
+ cfg.symbols['SENDER_REP_HAM'] = t
+ cfg.symbols['SENDER_REP_HAM'].weight = -(t.weight or 0)
+ end
+ end
+ else
+ logger.infox(rspamd_config, 'ip reputation already exists, do not do any IP_SCORE transforms')
+ end
+ end
+
+ if cfg.surbl then
+ if not cfg.rbl then
+ cfg.rbl = {
+ rbls = {}
+ }
+ end
+ if not cfg.rbl.rbls then
+ cfg.rbl.rbls = {}
+ end
+ surbl_section_convert(cfg, cfg.surbl)
+ logger.infox(rspamd_config, 'converted surbl rules to rbl rules')
+ cfg.surbl = {}
+ end
+
+ if cfg.emails then
+ if not cfg.rbl then
+ cfg.rbl = {
+ rbls = {}
+ }
+ end
+ if not cfg.rbl.rbls then
+ cfg.rbl.rbls = {}
+ end
+ emails_section_convert(cfg, cfg.emails)
+ logger.infox(rspamd_config, 'converted emails rules to rbl rules')
+ cfg.emails = {}
+ end
+
+ return ret, cfg
+end
diff --git a/lualib/lua_cfg_utils.lua b/lualib/lua_cfg_utils.lua
new file mode 100644
index 0000000..e07a3ae
--- /dev/null
+++ b/lualib/lua_cfg_utils.lua
@@ -0,0 +1,84 @@
+--[[
+Copyright (c) 2023, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_cfg_utils
+-- This module contains utility functions for configuration of Rspamd modules
+--]]
+
+local rspamd_logger = require "rspamd_logger"
+local exports = {}
+
+--[[[
+-- @function lua_util.disable_module(modname, how[, reason])
+-- Disables a plugin
+-- @param {string} modname name of plugin to disable
+-- @param {string} how 'redis' to disable redis, 'config' to disable startup
+-- @param {string} reason optional reason for failure
+--]]
+exports.disable_module = function(modname, how, reason)
+ if rspamd_plugins_state.enabled[modname] then
+ rspamd_plugins_state.enabled[modname] = nil
+ end
+
+ if how == 'redis' then
+ rspamd_plugins_state.disabled_redis[modname] = {}
+ elseif how == 'config' then
+ rspamd_plugins_state.disabled_unconfigured[modname] = {}
+ elseif how == 'experimental' then
+ rspamd_plugins_state.disabled_experimental[modname] = {}
+ elseif how == 'failed' then
+ rspamd_plugins_state.disabled_failed[modname] = { reason = reason }
+ else
+ rspamd_plugins_state.disabled_unknown[modname] = {}
+ end
+end
+
+--[[[
+-- @function lua_util.push_config_error(module, err)
+-- Pushes a configuration error to the state
+-- @param {string} module name of module
+-- @param {string} err error string
+--]]
+exports.push_config_error = function(module, err)
+ if not rspamd_plugins_state.config_errors then
+ rspamd_plugins_state.config_errors = {}
+ end
+
+ if not rspamd_plugins_state.config_errors[module] then
+ rspamd_plugins_state.config_errors[module] = {}
+ end
+
+ table.insert(rspamd_plugins_state.config_errors[module], err)
+end
+
+exports.check_configuration_errors = function()
+ local ret = true
+
+ if type(rspamd_plugins_state.config_errors) == 'table' then
+ -- We have some errors found during the configuration, so we need to show them
+ for m, errs in pairs(rspamd_plugins_state.config_errors) do
+ for _, err in ipairs(errs) do
+ rspamd_logger.errx(rspamd_config, 'configuration error: module %s: %s', m, err)
+ ret = false
+ end
+ end
+ end
+
+ return ret
+end
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_clickhouse.lua b/lualib/lua_clickhouse.lua
new file mode 100644
index 0000000..28366d2
--- /dev/null
+++ b/lualib/lua_clickhouse.lua
@@ -0,0 +1,547 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2018, Mikhail Galanin <mgalanin@mimecast.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_clickhouse
+-- This module contains Clickhouse access functions
+--]]
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_http = require "rspamd_http"
+local lua_util = require "lua_util"
+local rspamd_text = require "rspamd_text"
+
+local exports = {}
+local N = 'clickhouse'
+
+local default_timeout = 10.0
+
+local function escape_spaces(query)
+ return query:gsub('%s', '%%20')
+end
+
+local function ch_number(a)
+ if (a + 2 ^ 52) - 2 ^ 52 == a then
+ -- Integer
+ return tostring(math.floor(a))
+ end
+
+ return tostring(a)
+end
+
+local function clickhouse_quote(str)
+ if str then
+ return str:gsub('[\'\\\n\t\r]', {
+ ['\''] = [[\']],
+ ['\\'] = [[\\]],
+ ['\n'] = [[\n]],
+ ['\t'] = [[\t]],
+ ['\r'] = [[\r]],
+ })
+ end
+
+ return ''
+end
+
+-- Converts an array to a string suitable for clickhouse
+local function array_to_string(ar)
+ for i, elt in ipairs(ar) do
+ local t = type(elt)
+ if t == 'string' then
+ ar[i] = string.format('\'%s\'', clickhouse_quote(elt))
+ elseif t == 'userdata' then
+ ar[i] = string.format('\'%s\'', clickhouse_quote(tostring(elt)))
+ elseif t == 'number' then
+ ar[i] = ch_number(elt)
+ end
+ end
+
+ return table.concat(ar, ',')
+end
+
+-- Converts a row into TSV, taking extra care about arrays
+local function row_to_tsv(row)
+
+ for i, elt in ipairs(row) do
+ local t = type(elt)
+ if t == 'table' then
+ row[i] = '[' .. array_to_string(elt) .. ']'
+ elseif t == 'number' then
+ row[i] = ch_number(elt)
+ elseif t == 'userdata' then
+ row[i] = clickhouse_quote(tostring(elt))
+ else
+ row[i] = clickhouse_quote(elt)
+ end
+ end
+
+ return rspamd_text.fromtable(row, '\t')
+end
+
+exports.row_to_tsv = row_to_tsv
+
+-- Parses JSONEachRow reply from CH
+local function parse_clickhouse_response_json_eachrow(params, data, row_cb)
+ local ucl = require "ucl"
+
+ if data == nil then
+ -- clickhouse returned no data (i.e. empty result set): exiting
+ return {}
+ end
+
+ local function parse_string(s)
+ local parser = ucl.parser()
+ local res, err
+ if type(s) == 'string' then
+ res, err = parser:parse_string(s)
+ else
+ res, err = parser:parse_text(s)
+ end
+
+ if not res then
+ rspamd_logger.errx(params.log_obj, 'Parser error: %s', err)
+ return nil
+ end
+ return parser:get_object()
+ end
+
+ -- iterate over rows and parse
+ local parsed_rows = {}
+ for plain_row in data:lines() do
+ if plain_row and #plain_row > 1 then
+ local parsed_row = parse_string(plain_row)
+ if parsed_row then
+ if row_cb then
+ row_cb(parsed_row)
+ else
+ table.insert(parsed_rows, parsed_row)
+ end
+ end
+ end
+ end
+
+ return parsed_rows
+end
+
+-- Parses JSON reply from CH
+local function parse_clickhouse_response_json(params, data)
+ local ucl = require "ucl"
+
+ if data == nil then
+ -- clickhouse returned no data (i.e. empty result set) considered valid!
+ return nil, {}
+ end
+
+ local function parse_string(s)
+ local parser = ucl.parser()
+ local res, err
+
+ if type(s) == 'string' then
+ res, err = parser:parse_string(s)
+ else
+ res, err = parser:parse_text(s)
+ end
+
+ if not res then
+ rspamd_logger.errx(params.log_obj, 'Parser error: %s', err)
+ return nil
+ end
+ return parser:get_object()
+ end
+
+ local json = parse_string(data)
+
+ if not json then
+ return 'bad json', {}
+ end
+
+ return nil, json
+end
+
+-- Helper to generate HTTP closure
+local function mk_http_select_cb(upstream, params, ok_cb, fail_cb, row_cb)
+ local function http_cb(err_message, code, data, _)
+ if code ~= 200 or err_message then
+ if not err_message then
+ err_message = data
+ end
+ local ip_addr = upstream:get_addr():to_string(true)
+
+ if fail_cb then
+ fail_cb(params, err_message, data)
+ else
+ rspamd_logger.errx(params.log_obj,
+ "request failed on clickhouse server %s: %s",
+ ip_addr, err_message)
+ end
+ upstream:fail()
+ else
+ upstream:ok()
+ local rows = parse_clickhouse_response_json_eachrow(params, data, row_cb)
+
+ if rows then
+ if ok_cb then
+ ok_cb(params, rows)
+ else
+ lua_util.debugm(N, params.log_obj,
+ "http_select_cb ok: %s, %s, %s, %s", err_message, code,
+ data:gsub('[\n%s]+', ' '), _)
+ end
+ else
+ if fail_cb then
+ fail_cb(params, 'failed to parse reply', data)
+ else
+ local ip_addr = upstream:get_addr():to_string(true)
+ rspamd_logger.errx(params.log_obj,
+ "request failed on clickhouse server %s: %s",
+ ip_addr, 'failed to parse reply')
+ end
+ end
+ end
+ end
+
+ return http_cb
+end
+
+-- Helper to generate HTTP closure
+local function mk_http_insert_cb(upstream, params, ok_cb, fail_cb)
+ local function http_cb(err_message, code, data, _)
+ if code ~= 200 or err_message then
+ if not err_message then
+ err_message = data
+ end
+ local ip_addr = upstream:get_addr():to_string(true)
+
+ if fail_cb then
+ fail_cb(params, err_message, data)
+ else
+ rspamd_logger.errx(params.log_obj,
+ "request failed on clickhouse server %s: %s",
+ ip_addr, err_message)
+ end
+ upstream:fail()
+ else
+ upstream:ok()
+
+ if ok_cb then
+ local err, parsed = parse_clickhouse_response_json(data)
+
+ if err then
+ fail_cb(params, err, data)
+ else
+ ok_cb(params, parsed)
+ end
+
+ else
+ lua_util.debugm(N, params.log_obj,
+ "http_insert_cb ok: %s, %s, %s, %s", err_message, code,
+ data:gsub('[\n%s]+', ' '), _)
+ end
+ end
+ end
+
+ return http_cb
+end
+
+--[[[
+-- @function lua_clickhouse.select(upstream, settings, params, query,
+ ok_cb, fail_cb)
+-- Make select request to clickhouse
+-- @param {upstream} upstream clickhouse server upstream
+-- @param {table} settings global settings table:
+-- * use_gsip: use gzip compression
+-- * timeout: request timeout
+-- * no_ssl_verify: skip SSL verification
+-- * user: HTTP user
+-- * password: HTTP password
+-- @param {params} HTTP request params
+-- @param {string} query select query (passed in HTTP body)
+-- @param {function} ok_cb callback to be called in case of success
+-- @param {function} fail_cb callback to be called in case of some error
+-- @param {function} row_cb optional callback to be called on each parsed data row (instead of table insertion)
+-- @return {boolean} whether a connection was successful
+-- @example
+--
+--]]
+exports.select = function(upstream, settings, params, query, ok_cb, fail_cb, row_cb)
+ local http_params = {}
+
+ for k, v in pairs(params) do
+ http_params[k] = v
+ end
+
+ http_params.callback = mk_http_select_cb(upstream, http_params, ok_cb, fail_cb, row_cb)
+ http_params.gzip = settings.use_gzip
+ http_params.mime_type = 'text/plain'
+ http_params.timeout = settings.timeout or default_timeout
+ http_params.no_ssl_verify = settings.no_ssl_verify
+ http_params.user = settings.user
+ http_params.password = settings.password
+ http_params.body = query
+ http_params.log_obj = params.task or params.config
+ http_params.opaque_body = true
+
+ lua_util.debugm(N, http_params.log_obj, "clickhouse select request: %s", http_params.body)
+
+ if not http_params.url then
+ local connect_prefix = "http://"
+ if settings.use_https then
+ connect_prefix = 'https://'
+ end
+ local ip_addr = upstream:get_addr():to_string(true)
+ local database = settings.database or 'default'
+ http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow',
+ connect_prefix, ip_addr, escape_spaces(database))
+ end
+
+ return rspamd_http.request(http_params)
+end
+
+--[[[
+-- @function lua_clickhouse.select_sync(upstream, settings, params, query,
+ ok_cb, fail_cb, row_cb)
+-- Make select request to clickhouse
+-- @param {upstream} upstream clickhouse server upstream
+-- @param {table} settings global settings table:
+-- * use_gsip: use gzip compression
+-- * timeout: request timeout
+-- * no_ssl_verify: skip SSL verification
+-- * user: HTTP user
+-- * password: HTTP password
+-- @param {params} HTTP request params
+-- @param {string} query select query (passed in HTTP body)
+-- @param {function} ok_cb callback to be called in case of success
+-- @param {function} fail_cb callback to be called in case of some error
+-- @param {function} row_cb optional callback to be called on each parsed data row (instead of table insertion)
+-- @return
+-- {string} error message if exists
+-- nil | {rows} | {http_response}
+-- @example
+--
+--]]
+exports.select_sync = function(upstream, settings, params, query, row_cb)
+ local http_params = {}
+
+ for k, v in pairs(params) do
+ http_params[k] = v
+ end
+
+ http_params.gzip = settings.use_gzip
+ http_params.mime_type = 'text/plain'
+ http_params.timeout = settings.timeout or default_timeout
+ http_params.no_ssl_verify = settings.no_ssl_verify
+ http_params.user = settings.user
+ http_params.password = settings.password
+ http_params.body = query
+ http_params.log_obj = params.task or params.config
+ http_params.opaque_body = true
+
+ lua_util.debugm(N, http_params.log_obj, "clickhouse select request: %s", http_params.body)
+
+ if not http_params.url then
+ local connect_prefix = "http://"
+ if settings.use_https then
+ connect_prefix = 'https://'
+ end
+ local ip_addr = upstream:get_addr():to_string(true)
+ local database = settings.database or 'default'
+ http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow',
+ connect_prefix, ip_addr, escape_spaces(database))
+ end
+
+ local err, response = rspamd_http.request(http_params)
+
+ if err then
+ return err, nil
+ elseif response.code ~= 200 then
+ return response.content, response
+ else
+ lua_util.debugm(N, http_params.log_obj, "clickhouse select response: %1", response)
+ local rows = parse_clickhouse_response_json_eachrow(params, response.content, row_cb)
+ return nil, rows
+ end
+end
+
+--[[[
+-- @function lua_clickhouse.insert(upstream, settings, params, query, rows,
+ ok_cb, fail_cb)
+-- Insert data rows to clickhouse
+-- @param {upstream} upstream clickhouse server upstream
+-- @param {table} settings global settings table:
+-- * use_gsip: use gzip compression
+-- * timeout: request timeout
+-- * no_ssl_verify: skip SSL verification
+-- * user: HTTP user
+-- * password: HTTP password
+-- @param {params} HTTP request params
+-- @param {string} query select query (passed in `query` request element with spaces escaped)
+-- @param {table|mixed} rows mix of strings, numbers or tables (for arrays)
+-- @param {function} ok_cb callback to be called in case of success
+-- @param {function} fail_cb callback to be called in case of some error
+-- @return {boolean} whether a connection was successful
+-- @example
+--
+--]]
+exports.insert = function(upstream, settings, params, query, rows,
+ ok_cb, fail_cb)
+ local http_params = {}
+
+ for k, v in pairs(params) do
+ http_params[k] = v
+ end
+
+ http_params.callback = mk_http_insert_cb(upstream, http_params, ok_cb, fail_cb)
+ http_params.gzip = settings.use_gzip
+ http_params.mime_type = 'text/plain'
+ http_params.timeout = settings.timeout or default_timeout
+ http_params.no_ssl_verify = settings.no_ssl_verify
+ http_params.user = settings.user
+ http_params.password = settings.password
+ http_params.method = 'POST'
+ http_params.body = { rspamd_text.fromtable(rows, '\n'), '\n' }
+ http_params.log_obj = params.task or params.config
+
+ if not http_params.url then
+ local connect_prefix = "http://"
+ if settings.use_https then
+ connect_prefix = 'https://'
+ end
+ local ip_addr = upstream:get_addr():to_string(true)
+ local database = settings.database or 'default'
+ http_params.url = string.format('%s%s/?database=%s&query=%s%%20FORMAT%%20TabSeparated',
+ connect_prefix,
+ ip_addr,
+ escape_spaces(database),
+ escape_spaces(query))
+ end
+
+ return rspamd_http.request(http_params)
+end
+
+--[[[
+-- @function lua_clickhouse.generic(upstream, settings, params, query,
+ ok_cb, fail_cb)
+-- Make a generic request to Clickhouse (e.g. alter)
+-- @param {upstream} upstream clickhouse server upstream
+-- @param {table} settings global settings table:
+-- * use_gsip: use gzip compression
+-- * timeout: request timeout
+-- * no_ssl_verify: skip SSL verification
+-- * user: HTTP user
+-- * password: HTTP password
+-- @param {params} HTTP request params
+-- @param {string} query Clickhouse query (passed in `query` request element with spaces escaped)
+-- @param {function} ok_cb callback to be called in case of success
+-- @param {function} fail_cb callback to be called in case of some error
+-- @return {boolean} whether a connection was successful
+-- @example
+--
+--]]
+exports.generic = function(upstream, settings, params, query,
+ ok_cb, fail_cb)
+ local http_params = {}
+
+ for k, v in pairs(params) do
+ http_params[k] = v
+ end
+
+ http_params.callback = mk_http_insert_cb(upstream, http_params, ok_cb, fail_cb)
+ http_params.gzip = settings.use_gzip
+ http_params.mime_type = 'text/plain'
+ http_params.timeout = settings.timeout or default_timeout
+ http_params.no_ssl_verify = settings.no_ssl_verify
+ http_params.user = settings.user
+ http_params.password = settings.password
+ http_params.log_obj = params.task or params.config
+ http_params.body = query
+
+ if not http_params.url then
+ local connect_prefix = "http://"
+ if settings.use_https then
+ connect_prefix = 'https://'
+ end
+ local ip_addr = upstream:get_addr():to_string(true)
+ local database = settings.database or 'default'
+ http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow',
+ connect_prefix, ip_addr, escape_spaces(database))
+ end
+
+ return rspamd_http.request(http_params)
+end
+
+--[[[
+-- @function lua_clickhouse.generic_sync(upstream, settings, params, query,
+ ok_cb, fail_cb)
+-- Make a generic request to Clickhouse (e.g. alter)
+-- @param {upstream} upstream clickhouse server upstream
+-- @param {table} settings global settings table:
+-- * use_gsip: use gzip compression
+-- * timeout: request timeout
+-- * no_ssl_verify: skip SSL verification
+-- * user: HTTP user
+-- * password: HTTP password
+-- @param {params} HTTP request params
+-- @param {string} query Clickhouse query (passed in `query` request element with spaces escaped)
+-- @return {boolean} whether a connection was successful
+-- @example
+--
+--]]
+exports.generic_sync = function(upstream, settings, params, query)
+ local http_params = {}
+
+ for k, v in pairs(params) do
+ http_params[k] = v
+ end
+
+ http_params.gzip = settings.use_gzip
+ http_params.mime_type = 'text/plain'
+ http_params.timeout = settings.timeout or default_timeout
+ http_params.no_ssl_verify = settings.no_ssl_verify
+ http_params.user = settings.user
+ http_params.password = settings.password
+ http_params.log_obj = params.task or params.config
+ http_params.body = query
+
+ if not http_params.url then
+ local connect_prefix = "http://"
+ if settings.use_https then
+ connect_prefix = 'https://'
+ end
+ local ip_addr = upstream:get_addr():to_string(true)
+ local database = settings.database or 'default'
+ http_params.url = string.format('%s%s/?database=%s&default_format=JSON',
+ connect_prefix, ip_addr, escape_spaces(database))
+ end
+
+ local err, response = rspamd_http.request(http_params)
+
+ if err then
+ return err, nil
+ elseif response.code ~= 200 then
+ return response.content, response
+ else
+ lua_util.debugm(N, http_params.log_obj, "clickhouse generic response: %1", response)
+ local e, obj = parse_clickhouse_response_json(params, response.content)
+
+ if e then
+ return e, nil
+ end
+ return nil, obj
+ end
+end
+
+return exports
diff --git a/lualib/lua_content/ical.lua b/lualib/lua_content/ical.lua
new file mode 100644
index 0000000..d018a85
--- /dev/null
+++ b/lualib/lua_content/ical.lua
@@ -0,0 +1,105 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local l = require 'lpeg'
+local lua_util = require "lua_util"
+local N = "lua_content"
+
+local ical_grammar
+
+local function gen_grammar()
+ if not ical_grammar then
+ local wsp = l.S(" \t\v\f")
+ local crlf = (l.P "\r" ^ -1 * l.P "\n") + l.P "\r"
+ local eol = (crlf * #crlf) + (crlf - (crlf ^ -1 * wsp))
+ local name = l.C((l.P(1) - (l.P ":")) ^ 1) / function(v)
+ return (v:gsub("[\n\r]+%s", ""))
+ end
+ local value = l.C((l.P(1) - eol) ^ 0) / function(v)
+ return (v:gsub("[\n\r]+%s", ""))
+ end
+ ical_grammar = name * ":" * wsp ^ 0 * value * eol ^ -1
+ end
+
+ return ical_grammar
+end
+
+local exports = {}
+
+local function extract_text_data(specific)
+ local fun = require "fun"
+
+ local tbl = fun.totable(fun.map(function(e)
+ return e[2]:lower()
+ end, specific.elts))
+ return table.concat(tbl, '\n')
+end
+
+
+-- Keys that can have visible urls
+local url_keys = lua_util.list_to_hash {
+ 'description',
+ 'location',
+ 'summary',
+ 'organizer',
+ 'organiser',
+ 'attendee',
+ 'url'
+}
+
+local function process_ical(input, mpart, task)
+ local control = { n = '\n', r = '' }
+ local rspamd_url = require "rspamd_url"
+ local escaper = l.Ct((gen_grammar() / function(key, value)
+ value = value:gsub("\\(.)", control)
+ key = key:lower():match('^([^;]+)')
+
+ if key and url_keys[key] then
+ local local_urls = rspamd_url.all(task:get_mempool(), value)
+
+ if local_urls and #local_urls > 0 then
+ for _, u in ipairs(local_urls) do
+ lua_util.debugm(N, task, 'ical: found URL in ical key "%s": %s',
+ key, tostring(u))
+ task:inject_url(u, mpart)
+ end
+ end
+ end
+ lua_util.debugm(N, task, 'ical: ical key %s = "%s"',
+ key, value)
+ return { key, value }
+ end) ^ 1)
+
+ local elts = escaper:match(input)
+
+ if not elts then
+ return nil
+ end
+
+ return {
+ tag = 'ical',
+ extract_text = extract_text_data,
+ elts = elts
+ }
+end
+
+--[[[
+-- @function lua_ical.process(input)
+-- Returns all values from ical as a plain text. Names are completely ignored.
+--]]
+exports.process = process_ical
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_content/init.lua b/lualib/lua_content/init.lua
new file mode 100644
index 0000000..701d223
--- /dev/null
+++ b/lualib/lua_content/init.lua
@@ -0,0 +1,109 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_content
+-- This module contains content processing logic
+--]]
+
+
+local exports = {}
+local N = "lua_content"
+local lua_util = require "lua_util"
+
+local content_modules = {
+ ical = {
+ mime_type = { "text/calendar", "application/calendar" },
+ module = require "lua_content/ical",
+ extensions = { 'ics' },
+ output = "text"
+ },
+ vcf = {
+ mime_type = { "text/vcard", "application/vcard" },
+ module = require "lua_content/vcard",
+ extensions = { 'vcf' },
+ output = "text"
+ },
+ pdf = {
+ mime_type = "application/pdf",
+ module = require "lua_content/pdf",
+ extensions = { 'pdf' },
+ output = "table"
+ },
+}
+
+local modules_by_mime_type
+local modules_by_extension
+
+local function init()
+ modules_by_mime_type = {}
+ modules_by_extension = {}
+ for k, v in pairs(content_modules) do
+ if v.mime_type then
+ if type(v.mime_type) == 'table' then
+ for _, mt in ipairs(v.mime_type) do
+ modules_by_mime_type[mt] = { k, v }
+ end
+ else
+ modules_by_mime_type[v.mime_type] = { k, v }
+ end
+
+ end
+ if v.extensions then
+ for _, ext in ipairs(v.extensions) do
+ modules_by_extension[ext] = { k, v }
+ end
+ end
+ end
+end
+
+exports.maybe_process_mime_part = function(part, task)
+ if not modules_by_mime_type then
+ init()
+ end
+
+ local ctype, csubtype = part:get_type()
+ local mt = string.format("%s/%s", ctype or 'application',
+ csubtype or 'octet-stream')
+ local pair = modules_by_mime_type[mt]
+
+ if not pair then
+ local ext = part:get_detected_ext()
+
+ if ext then
+ pair = modules_by_extension[ext]
+ end
+ end
+
+ if pair then
+ lua_util.debugm(N, task, "found known content of type %s: %s",
+ mt, pair[1])
+
+ local data = pair[2].module.process(part:get_content(), part, task)
+
+ if data then
+ lua_util.debugm(N, task, "extracted content from %s: %s type",
+ pair[1], type(data))
+ part:set_specific(data)
+ else
+ lua_util.debugm(N, task, "failed to extract anything from %s",
+ pair[1])
+ end
+ end
+
+end
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_content/pdf.lua b/lualib/lua_content/pdf.lua
new file mode 100644
index 0000000..f6d5c0b
--- /dev/null
+++ b/lualib/lua_content/pdf.lua
@@ -0,0 +1,1424 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_content/pdf
+-- This module contains some heuristics for PDF files
+--]]
+
+local rspamd_trie = require "rspamd_trie"
+local rspamd_util = require "rspamd_util"
+local rspamd_text = require "rspamd_text"
+local rspamd_url = require "rspamd_url"
+local bit = require "bit"
+local N = "lua_content"
+local lua_util = require "lua_util"
+local rspamd_regexp = require "rspamd_regexp"
+local lpeg = require "lpeg"
+local pdf_patterns = {
+ trailer = {
+ patterns = {
+ [[\ntrailer\r?\n]]
+ }
+ },
+ suspicious = {
+ patterns = {
+ [[netsh\s]],
+ [[echo\s]],
+ [=[\/[A-Za-z]*#\d\d[#A-Za-z<>/\s]]=], -- Hex encode obfuscation
+ }
+ },
+ start_object = {
+ patterns = {
+ [=[[\r\n\0]\s*\d+\s+\d+\s+obj[\s<]]=]
+ }
+ },
+ end_object = {
+ patterns = {
+ [=[endobj[\r\n]]=]
+ }
+ },
+ start_stream = {
+ patterns = {
+ [=[>\s*stream[\r\n]]=],
+ }
+ },
+ end_stream = {
+ patterns = {
+ [=[endstream[\r\n]]=]
+ }
+ }
+}
+
+local pdf_text_patterns = {
+ start = {
+ patterns = {
+ [[\sBT\s]]
+ }
+ },
+ stop = {
+ patterns = {
+ [[\sET\b]]
+ }
+ }
+}
+
+local pdf_cmap_patterns = {
+ start = {
+ patterns = {
+ [[\d\s+beginbfchar\s]],
+ [[\d\s+beginbfrange\s]]
+ }
+ },
+ stop = {
+ patterns = {
+ [[\sendbfrange\b]],
+ [[\sendbchar\b]]
+ }
+ }
+}
+
+-- index[n] ->
+-- t[1] - pattern,
+-- t[2] - key in patterns table,
+-- t[3] - value in patterns table
+-- t[4] - local pattern index
+local pdf_indexes = {}
+local pdf_text_indexes = {}
+local pdf_cmap_indexes = {}
+
+local pdf_trie
+local pdf_text_trie
+local pdf_cmap_trie
+
+local exports = {}
+
+local config = {
+ max_extraction_size = 512 * 1024,
+ max_processing_size = 32 * 1024,
+ text_extraction = false, -- NYI feature
+ url_extraction = true,
+ enabled = true,
+ js_fuzzy = true, -- Generate fuzzy hashes from PDF javascripts
+ min_js_fuzzy = 256, -- Minimum size of js to be considered as a fuzzy
+ openaction_fuzzy_only = false, -- Generate fuzzy from all scripts
+ max_pdf_objects = 10000, -- Maximum number of objects to be considered
+ max_pdf_trailer = 10 * 1024 * 1024, -- Maximum trailer size (to avoid abuse)
+ max_pdf_trailer_lines = 100, -- Maximum number of lines in pdf trailer
+ pdf_process_timeout = 1.0, -- Timeout in seconds for processing
+}
+
+-- Used to process patterns found in PDF
+-- positions for functional processors should be a iter/table from trie matcher in form
+---- [{n1, pat_idx1}, ... {nn, pat_idxn}] where
+---- pat_idxn is pattern index and n1 ... nn are match positions
+local processors = {}
+-- PDF objects outer grammar in LPEG style (performing table captures)
+local pdf_outer_grammar
+local pdf_text_grammar
+
+-- Used to match objects
+local object_re = rspamd_regexp.create_cached([=[/(\d+)\s+(\d+)\s+obj\s*/]=])
+
+local function config_module()
+ local opts = rspamd_config:get_all_opt('lua_content')
+
+ if opts and opts.pdf then
+ config = lua_util.override_defaults(config, opts.pdf)
+ end
+end
+
+local function compile_tries()
+ local default_compile_flags = bit.bor(rspamd_trie.flags.re,
+ rspamd_trie.flags.dot_all,
+ rspamd_trie.flags.no_start)
+ local function compile_pats(patterns, indexes, compile_flags)
+ local strs = {}
+ for what, data in pairs(patterns) do
+ for i, pat in ipairs(data.patterns) do
+ strs[#strs + 1] = pat
+ indexes[#indexes + 1] = { what, data, pat, i }
+ end
+ end
+
+ return rspamd_trie.create(strs, compile_flags or default_compile_flags)
+ end
+
+ if not pdf_trie then
+ pdf_trie = compile_pats(pdf_patterns, pdf_indexes)
+ end
+ if not pdf_text_trie then
+ pdf_text_trie = compile_pats(pdf_text_patterns, pdf_text_indexes)
+ end
+ if not pdf_cmap_trie then
+ pdf_cmap_trie = compile_pats(pdf_cmap_patterns, pdf_cmap_indexes)
+ end
+end
+
+-- Returns a table with generic grammar elements for PDF
+local function generic_grammar_elts()
+ local P = lpeg.P
+ local R = lpeg.R
+ local S = lpeg.S
+ local V = lpeg.V
+ local C = lpeg.C
+ local D = R '09' -- Digits
+
+ local grammar_elts = {}
+
+ -- Helper functions
+ local function pdf_hexstring_unescape(s)
+ if #s % 2 == 0 then
+ -- Sane hex string
+ return lua_util.unhex(s)
+ end
+
+ -- WTF hex string
+ -- Append '0' to it and unescape...
+ return lua_util.unhex(s:sub(1, #s - 1)) .. lua_util.unhex((s:sub(#s) .. '0'))
+ end
+
+ local function pdf_string_unescape(s)
+ local function ue_single(cc)
+ if cc == '\\r' then
+ return '\r'
+ elseif cc == '\\n' then
+ return '\n'
+ else
+ return cc:gsub(2, 2)
+ end
+ end
+ -- simple unescape \char
+ s = s:gsub('\\[^%d]', ue_single)
+ -- unescape octal
+ local function ue_octal(cc)
+ -- Replace unknown stuff with '?'
+ return string.char(tonumber(cc:sub(2), 8) or 63)
+ end
+ s = s:gsub('\\%d%d?%d?', ue_octal)
+
+ return s
+ end
+
+ local function pdf_id_unescape(s)
+ return (s:gsub('#%d%d', function(cc)
+ return string.char(tonumber(cc:sub(2), 16))
+ end))
+ end
+
+ local delim = S '()<>[]{}/%'
+ grammar_elts.ws = S '\0 \r\n\t\f'
+ local hex = R 'af' + R 'AF' + D
+ -- Comments.
+ local eol = P '\r\n' + '\n'
+ local line = (1 - S '\r\n\f') ^ 0 * eol ^ -1
+ grammar_elts.comment = P '%' * line
+
+ -- Numbers.
+ local sign = S '+-' ^ -1
+ local decimal = D ^ 1
+ local float = D ^ 1 * P '.' * D ^ 0 + P '.' * D ^ 1
+ grammar_elts.number = C(sign * (float + decimal)) / tonumber
+
+ -- String
+ grammar_elts.str = P { "(" * C(((1 - S "()\\") + (P '\\' * 1) + V(1)) ^ 0) / pdf_string_unescape * ")" }
+ grammar_elts.hexstr = P { "<" * C(hex ^ 0) / pdf_hexstring_unescape * ">" }
+
+ -- Identifier
+ grammar_elts.id = P { '/' * C((1 - (delim + grammar_elts.ws)) ^ 1) / pdf_id_unescape }
+
+ -- Booleans (who care about them?)
+ grammar_elts.boolean = C(P("true") + P("false"))
+
+ -- Stupid references
+ grammar_elts.ref = lpeg.Ct { lpeg.Cc("%REF%") * C(D ^ 1) * " " * C(D ^ 1) * " " * "R" }
+
+ return grammar_elts
+end
+
+
+-- Generates a grammar to parse outer elements (external objects in PDF notation)
+local function gen_outer_grammar()
+ local V = lpeg.V
+ local gen = generic_grammar_elts()
+
+ return lpeg.P {
+ "EXPR";
+ EXPR = gen.ws ^ 0 * V("ELT") ^ 0 * gen.ws ^ 0,
+ ELT = V("ARRAY") + V("DICT") + V("ATOM"),
+ ATOM = gen.ws ^ 0 * (gen.comment + gen.boolean + gen.ref +
+ gen.number + V("STRING") + gen.id) * gen.ws ^ 0,
+ DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>",
+ KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ELT") * gen.ws ^ 0),
+ ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ELT") ^ 0) * gen.ws ^ 0 * "]",
+ STRING = lpeg.P { gen.str + gen.hexstr },
+ }
+end
+
+-- Graphic state in PDF
+local function gen_graphics_unary()
+ local P = lpeg.P
+ local S = lpeg.S
+
+ return P("q") + P("Q") + P("h")
+ + S("WSsFfBb") * P("*") ^ 0 + P("n")
+
+end
+local function gen_graphics_binary()
+ local P = lpeg.P
+ local S = lpeg.S
+
+ return S("gGwJjMi") +
+ P("M") + P("ri") + P("gs") +
+ P("CS") + P("cs") + P("sh")
+end
+local function gen_graphics_ternary()
+ local P = lpeg.P
+ local S = lpeg.S
+
+ return P("d") + P("m") + S("lm")
+end
+local function gen_graphics_nary()
+ local P = lpeg.P
+ local S = lpeg.S
+
+ return P("SC") + P("sc") + P("SCN") + P("scn") + P("k") + P("K") + P("re") + S("cvy") +
+ P("RG") + P("rg")
+end
+
+-- Generates a grammar to parse text blocks (between BT and ET)
+local function gen_text_grammar()
+ local V = lpeg.V
+ local P = lpeg.P
+ local C = lpeg.C
+ local gen = generic_grammar_elts()
+
+ local empty = ""
+ local unary_ops = C("T*") / "\n" +
+ C(gen_graphics_unary()) / empty
+ local binary_ops = P("Tc") + P("Tw") + P("Tz") + P("TL") + P("Tr") + P("Ts") +
+ gen_graphics_binary()
+ local ternary_ops = P("TD") + P("Td") + gen_graphics_ternary()
+ local nary_op = P("Tm") + gen_graphics_nary()
+ local text_binary_op = P("Tj") + P("TJ") + P("'")
+ local text_quote_op = P('"')
+ local font_op = P("Tf")
+
+ return lpeg.P {
+ "EXPR";
+ EXPR = gen.ws ^ 0 * lpeg.Ct(V("COMMAND") ^ 0),
+ COMMAND = (V("UNARY") + V("BINARY") + V("TERNARY") + V("NARY") + V("TEXT") +
+ V("FONT") + gen.comment) * gen.ws ^ 0,
+ UNARY = unary_ops,
+ BINARY = V("ARG") / empty * gen.ws ^ 1 * binary_ops,
+ TERNARY = V("ARG") / empty * gen.ws ^ 1 * V("ARG") / empty * gen.ws ^ 1 * ternary_ops,
+ NARY = (gen.number / 0 * gen.ws ^ 1) ^ 1 * (gen.id / empty * gen.ws ^ 0) ^ -1 * nary_op,
+ ARG = V("ARRAY") + V("DICT") + V("ATOM"),
+ ATOM = (gen.comment + gen.boolean + gen.ref +
+ gen.number + V("STRING") + gen.id),
+ DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>",
+ KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ARG") * gen.ws ^ 0),
+ ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ARG") ^ 0) * gen.ws ^ 0 * "]",
+ STRING = lpeg.P { gen.str + gen.hexstr },
+ TEXT = (V("TEXT_ARG") * gen.ws ^ 1 * text_binary_op) +
+ (V("ARG") / 0 * gen.ws ^ 1 * V("ARG") / 0 * gen.ws ^ 1 * V("TEXT_ARG") * gen.ws ^ 1 * text_quote_op),
+ FONT = (V("FONT_ARG") * gen.ws ^ 1 * (gen.number / 0) * gen.ws ^ 1 * font_op),
+ FONT_ARG = lpeg.Ct(lpeg.Cc("%font%") * gen.id),
+ TEXT_ARG = lpeg.Ct(V("STRING")) + V("TEXT_ARRAY"),
+ TEXT_ARRAY = "[" *
+ lpeg.Ct(((gen.ws ^ 0 * (gen.ws ^ 0 * (gen.number / 0) ^ 0 * gen.ws ^ 0 * (gen.str + gen.hexstr))) ^ 1)) * gen.ws ^ 0 * "]",
+ }
+end
+
+
+-- Call immediately on require
+compile_tries()
+config_module()
+pdf_outer_grammar = gen_outer_grammar()
+pdf_text_grammar = gen_text_grammar()
+
+local function extract_text_data(specific)
+ return nil -- NYI
+end
+
+-- Generates index for major/minor pair
+local function obj_ref(major, minor)
+ return major * 10.0 + 1.0 / (minor + 1.0)
+end
+
+-- Return indirect object reference (if needed)
+local function maybe_dereference_object(elt, pdf, task)
+ if type(elt) == 'table' and elt[1] == '%REF%' then
+ local ref = obj_ref(elt[2], elt[3])
+
+ if pdf.ref[ref] then
+ -- No recursion!
+ return pdf.ref[ref]
+ else
+ lua_util.debugm(N, task, 'cannot dereference %s:%s -> %s, no object',
+ elt[2], elt[3], obj_ref(elt[2], elt[3]))
+ return nil
+ end
+ end
+
+ return elt
+end
+
+-- Apply PDF stream filter
+local function apply_pdf_filter(input, filt)
+ if filt == 'FlateDecode' then
+ return rspamd_util.inflate(input, config.max_extraction_size)
+ end
+
+ return nil
+end
+
+-- Conditionally apply a pipeline of stream filters and return uncompressed data
+local function maybe_apply_filter(dict, data, pdf, task)
+ local uncompressed = data
+
+ if dict.Filter then
+ local filt = dict.Filter
+ if type(filt) == 'string' then
+ filt = { filt }
+ end
+
+ if dict.DecodeParms then
+ local decode_params = maybe_dereference_object(dict.DecodeParms, pdf, task)
+
+ if type(decode_params) == 'table' then
+ if decode_params.Predictor then
+ return nil, 'predictor exists'
+ end
+ end
+ end
+
+ for _, f in ipairs(filt) do
+ uncompressed = apply_pdf_filter(uncompressed, f)
+
+ if not uncompressed then
+ break
+ end
+ end
+ end
+
+ return uncompressed, nil
+end
+
+-- Conditionally extract stream data from object and attach it as obj.uncompressed
+local function maybe_extract_object_stream(obj, pdf, task)
+ if pdf.encrypted then
+ -- TODO add decryption some day
+ return nil
+ end
+ local dict = obj.dict
+ if dict.Length and type(obj.stream) == 'table' then
+ local len = math.min(obj.stream.len,
+ tonumber(maybe_dereference_object(dict.Length, pdf, task)) or 0)
+ if len > 0 then
+ local real_stream = obj.stream.data:span(1, len)
+
+ local uncompressed, filter_err = maybe_apply_filter(dict, real_stream, pdf, task)
+
+ if uncompressed then
+ obj.uncompressed = uncompressed
+ lua_util.debugm(N, task, 'extracted object %s:%s: (%s -> %s)',
+ obj.major, obj.minor, len, uncompressed:len())
+ return obj.uncompressed
+ else
+ lua_util.debugm(N, task, 'cannot extract object %s:%s; len = %s; filter = %s: %s',
+ obj.major, obj.minor, len, dict.Filter, filter_err)
+ end
+ else
+ lua_util.debugm(N, task, 'cannot extract object %s:%s; len = %s',
+ obj.major, obj.minor, len)
+ end
+ end
+end
+
+local function parse_object_grammar(obj, task, pdf)
+ -- Parse grammar
+ local obj_dict_span
+ if obj.stream then
+ obj_dict_span = obj.data:span(1, obj.stream.start - obj.start)
+ else
+ obj_dict_span = obj.data
+ end
+
+ if obj_dict_span:len() < config.max_processing_size then
+ local ret, obj_or_err = pcall(pdf_outer_grammar.match, pdf_outer_grammar, obj_dict_span)
+
+ if ret then
+ if obj.stream then
+ if type(obj_or_err) == 'table' then
+ obj.dict = obj_or_err
+ else
+ obj.dict = {}
+ end
+
+ lua_util.debugm(N, task, 'stream object %s:%s is parsed to: %s',
+ obj.major, obj.minor, obj_or_err)
+ else
+ -- Direct object
+ if type(obj_or_err) == 'table' then
+ obj.dict = obj_or_err
+ obj.uncompressed = obj_or_err
+ lua_util.debugm(N, task, 'direct object %s:%s is parsed to: %s',
+ obj.major, obj.minor, obj_or_err)
+ pdf.ref[obj_ref(obj.major, obj.minor)] = obj
+ else
+ lua_util.debugm(N, task, 'direct object %s:%s is parsed to raw data: %s',
+ obj.major, obj.minor, obj_or_err)
+ pdf.ref[obj_ref(obj.major, obj.minor)] = obj_or_err
+ obj.dict = {}
+ obj.uncompressed = obj_or_err
+ end
+ end
+ else
+ lua_util.debugm(N, task, 'object %s:%s cannot be parsed: %s',
+ obj.major, obj.minor, obj_or_err)
+ end
+ else
+ lua_util.debugm(N, task, 'object %s:%s cannot be parsed: too large %s',
+ obj.major, obj.minor, obj_dict_span:len())
+ end
+end
+
+-- Extracts font data and process /ToUnicode mappings
+-- NYI in fact as cmap is ridiculously stupid and complicated
+--[[
+local function process_font(task, pdf, font, fname)
+ local dict = font
+ if font.dict then
+ dict = font.dict
+ end
+
+ if type(dict) == 'table' and dict.ToUnicode then
+ local cmap = maybe_dereference_object(dict.ToUnicode, pdf, task)
+
+ if cmap and cmap.dict then
+ maybe_extract_object_stream(cmap, pdf, task)
+ lua_util.debugm(N, task, 'found cmap for font %s: %s',
+ fname, cmap.uncompressed)
+ end
+ end
+end
+--]]
+
+-- Forward declaration
+local process_dict
+
+-- This function processes javascript string and returns JS hash and JS rspamd_text
+local function process_javascript(task, pdf, js, obj)
+ local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash"
+ if type(js) == 'string' then
+ js = rspamd_text.fromstring(js):oneline()
+ elseif type(js) == 'userdata' then
+ js = js:oneline()
+ else
+ return nil
+ end
+
+ local hash = rspamd_cryptobox_hash.create(js)
+ local bin_hash = hash:bin()
+
+ if not pdf.scripts then
+ pdf.scripts = {}
+ end
+
+ if pdf.scripts[bin_hash] then
+ -- Duplicate
+ return pdf.scripts[bin_hash]
+ end
+
+ local njs = {
+ data = js,
+ hash = hash:hex(),
+ bin_hash = bin_hash,
+ object = obj,
+ }
+ pdf.scripts[bin_hash] = njs
+ return njs
+end
+
+-- Extract interesting stuff from /Action, e.g. javascript
+local function process_action(task, pdf, obj)
+ if not (obj.js or obj.launch) and (obj.dict and obj.dict.JS) then
+ local js = maybe_dereference_object(obj.dict.JS, pdf, task)
+
+ if js then
+ if type(js) == 'table' then
+ local extracted_js = maybe_extract_object_stream(js, pdf, task)
+
+ if not extracted_js then
+ lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
+ obj.major, obj.minor, js)
+ else
+ js = extracted_js
+ end
+ end
+
+ js = process_javascript(task, pdf, js, obj)
+ if js then
+ obj.js = js
+ lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s',
+ obj.major, obj.minor, obj.js.data)
+ else
+ lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
+ obj.major, obj.minor, js)
+ end
+ elseif obj.dict.F then
+ local launch = maybe_dereference_object(obj.dict.F, pdf, task)
+
+ if launch then
+ if type(launch) == 'string' then
+ obj.launch = rspamd_text.fromstring(launch):exclude_chars('%n%c')
+ lua_util.debugm(N, task, 'extracted launch from %s:%s: %s',
+ obj.major, obj.minor, obj.launch)
+ elseif type(launch) == 'userdata' then
+ obj.launch = launch:exclude_chars('%n%c')
+ lua_util.debugm(N, task, 'extracted launch from %s:%s: %s',
+ obj.major, obj.minor, obj.launch)
+ else
+ lua_util.debugm(N, task, 'invalid type for launch from %s:%s: %s',
+ obj.major, obj.minor, launch)
+ end
+ end
+ else
+
+ lua_util.debugm(N, task, 'no JS attribute in action %s:%s',
+ obj.major, obj.minor)
+ end
+ end
+end
+
+-- Extract interesting stuff from /Catalog, e.g. javascript in /OpenAction
+local function process_catalog(task, pdf, obj)
+ if obj.dict then
+ if obj.dict.OpenAction then
+ local action = maybe_dereference_object(obj.dict.OpenAction, pdf, task)
+
+ if action and type(action) == 'table' then
+ -- This also processes action js (if not already processed)
+ process_dict(task, pdf, action, action.dict)
+ if action.js then
+ lua_util.debugm(N, task, 'found openaction JS in %s:%s: %s',
+ obj.major, obj.minor, action.js)
+ pdf.openaction = action.js
+ action.js.object = obj
+ elseif action.launch then
+ lua_util.debugm(N, task, 'found openaction launch in %s:%s: %s',
+ obj.major, obj.minor, action.launch)
+ pdf.launch = action.launch
+ else
+ lua_util.debugm(N, task, 'no JS in openaction %s:%s: %s',
+ obj.major, obj.minor, action)
+ end
+ else
+ lua_util.debugm(N, task, 'cannot find openaction %s:%s: %s -> %s',
+ obj.major, obj.minor, obj.dict.OpenAction, action)
+ end
+ else
+ lua_util.debugm(N, task, 'no openaction in catalog %s:%s',
+ obj.major, obj.minor)
+ end
+ end
+end
+
+local function process_xref(task, pdf, obj)
+ if obj.dict then
+ if obj.dict.Encrypt then
+ local encrypt = maybe_dereference_object(obj.dict.Encrypt, pdf, task)
+ lua_util.debugm(N, task, 'found encrypt: %s in xref object %s:%s',
+ encrypt, obj.major, obj.minor)
+ pdf.encrypted = true
+ end
+ end
+end
+
+process_dict = function(task, pdf, obj, dict)
+ if not obj.type and type(dict) == 'table' then
+ if dict.Type and type(dict.Type) == 'string' then
+ -- Common stuff
+ obj.type = dict.Type
+ end
+
+ if not obj.type then
+
+ if obj.dict.S and obj.dict.JS then
+ obj.type = 'Javascript'
+ lua_util.debugm(N, task, 'implicit type for JavaScript object %s:%s',
+ obj.major, obj.minor)
+ else
+ lua_util.debugm(N, task, 'no type for %s:%s',
+ obj.major, obj.minor)
+ return
+ end
+ end
+
+ lua_util.debugm(N, task, 'processed stream dictionary for object %s:%s -> %s',
+ obj.major, obj.minor, obj.type)
+ local contents = dict.Contents
+ if contents and type(contents) == 'table' then
+ if contents[1] == '%REF%' then
+ -- Single reference
+ contents = { contents }
+ end
+ obj.contents = {}
+
+ for _, c in ipairs(contents) do
+ local cobj = maybe_dereference_object(c, pdf, task)
+ if cobj and type(cobj) == 'table' then
+ obj.contents[#obj.contents + 1] = cobj
+ cobj.parent = obj
+ cobj.type = 'content'
+ end
+ end
+
+ lua_util.debugm(N, task, 'found content objects for %s:%s -> %s',
+ obj.major, obj.minor, #obj.contents)
+ end
+
+ local resources = dict.Resources
+ if resources and type(resources) == 'table' then
+ local res_ref = maybe_dereference_object(resources, pdf, task)
+
+ if type(res_ref) ~= 'table' then
+ lua_util.debugm(N, task, 'cannot parse resources from pdf: %s',
+ resources)
+ obj.resources = {}
+ elseif res_ref.dict then
+ obj.resources = res_ref.dict
+ else
+ obj.resources = {}
+ end
+ else
+ -- Fucking pdf: we need to inherit from parent
+ resources = {}
+ if dict.Parent then
+ local parent = maybe_dereference_object(dict.Parent, pdf, task)
+
+ if parent and type(parent) == 'table' and parent.dict then
+ if parent.resources then
+ lua_util.debugm(N, task, 'propagated resources from %s:%s to %s:%s',
+ parent.major, parent.minor, obj.major, obj.minor)
+ resources = parent.resources
+ end
+ end
+ end
+
+ obj.resources = resources
+ end
+
+
+
+ --[[Disabled fonts extraction
+ local fonts = obj.resources.Font
+ if fonts and type(fonts) == 'table' then
+ obj.fonts = {}
+ for k,v in pairs(fonts) do
+ obj.fonts[k] = maybe_dereference_object(v, pdf, task)
+
+ if obj.fonts[k] then
+ local font = obj.fonts[k]
+
+ if config.text_extraction then
+ process_font(task, pdf, font, k)
+ lua_util.debugm(N, task, 'found font "%s" for object %s:%s -> %s',
+ k, obj.major, obj.minor, font)
+ end
+ end
+ end
+ end
+ ]]
+
+ lua_util.debugm(N, task, 'found resources for object %s:%s (%s): %s',
+ obj.major, obj.minor, obj.type, obj.resources)
+
+ if obj.type == 'Action' then
+ process_action(task, pdf, obj)
+ elseif obj.type == 'Catalog' then
+ process_catalog(task, pdf, obj)
+ elseif obj.type == 'XRef' then
+ -- XRef stream instead of trailer from PDF 1.5 (thanks Adobe)
+ process_xref(task, pdf, obj)
+ elseif obj.type == 'Javascript' then
+ local js = maybe_dereference_object(obj.dict.JS, pdf, task)
+
+ if js then
+ if type(js) == 'table' then
+ local extracted_js = maybe_extract_object_stream(js, pdf, task)
+
+ if not extracted_js then
+ lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
+ obj.major, obj.minor, js)
+ else
+ js = extracted_js
+ end
+ end
+
+ js = process_javascript(task, pdf, js, obj)
+ if js then
+ obj.js = js
+ lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s',
+ obj.major, obj.minor, obj.js.data)
+ else
+ lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
+ obj.major, obj.minor, js)
+ end
+ end
+ end
+ end -- Already processed dict (obj.type is not empty)
+end
+
+-- This function is intended to unpack objects from ObjStm crappy structure
+local compound_obj_grammar
+local function compound_obj_grammar_gen()
+ if not compound_obj_grammar then
+ local gen = generic_grammar_elts()
+ compound_obj_grammar = gen.ws ^ 0 * (gen.comment * gen.ws ^ 1) ^ 0 *
+ lpeg.Ct(lpeg.Ct(gen.number * gen.ws ^ 1 * gen.number * gen.ws ^ 0) ^ 1)
+ end
+
+ return compound_obj_grammar
+end
+local function pdf_compound_object_unpack(_, uncompressed, pdf, task, first)
+ -- First, we need to parse data line by line likely to find a line
+ -- that consists of pairs of numbers
+ compound_obj_grammar_gen()
+ local elts = compound_obj_grammar:match(uncompressed)
+ if elts and #elts > 0 then
+ lua_util.debugm(N, task, 'compound elts (chunk length %s): %s',
+ #uncompressed, elts)
+
+ for i, pair in ipairs(elts) do
+ local obj_number, offset = pair[1], pair[2]
+
+ offset = offset + first
+ if offset < #uncompressed then
+ local span_len
+ if i == #elts then
+ span_len = #uncompressed - offset
+ else
+ span_len = (elts[i + 1][2] + first) - offset
+ end
+
+ if span_len > 0 and offset + span_len <= #uncompressed then
+ local obj = {
+ major = obj_number,
+ minor = 0, -- Implicit
+ data = uncompressed:span(offset + 1, span_len),
+ ref = obj_ref(obj_number, 0)
+ }
+ parse_object_grammar(obj, task, pdf)
+
+ if obj.dict then
+ pdf.objects[#pdf.objects + 1] = obj
+ end
+ else
+ lua_util.debugm(N, task, 'invalid span_len for compound object %s:%s; offset = %s, len = %s',
+ pair[1], pair[2], offset + span_len, #uncompressed)
+ end
+ end
+ end
+ end
+end
+
+-- PDF 1.5 ObjStmt
+local function extract_pdf_compound_objects(task, pdf)
+ for i, obj in ipairs(pdf.objects or {}) do
+ if i > 0 and i % 100 == 0 then
+ local now = rspamd_util.get_ticks()
+
+ if now >= pdf.end_timestamp then
+ pdf.timeout_processing = now - pdf.start_timestamp
+
+ lua_util.debugm(N, task, 'pdf: timeout processing compound objects after spending %s seconds, ' ..
+ '%s elements processed',
+ pdf.timeout_processing, i)
+ break
+ end
+ end
+ if obj.stream and obj.dict and type(obj.dict) == 'table' then
+ local t = obj.dict.Type
+ if t and t == 'ObjStm' then
+ -- We are in troubles sir...
+ local nobjs = tonumber(maybe_dereference_object(obj.dict.N, pdf, task))
+ local first = tonumber(maybe_dereference_object(obj.dict.First, pdf, task))
+
+ if nobjs and first then
+ --local extend = maybe_dereference_object(obj.dict.Extends, pdf, task)
+ lua_util.debugm(N, task, 'extract ObjStm with %s objects (%s first) %s extend',
+ nobjs, first, obj.dict.Extends)
+
+ local uncompressed = maybe_extract_object_stream(obj, pdf, task)
+
+ if uncompressed then
+ pdf_compound_object_unpack(obj, uncompressed, pdf, task, first)
+ end
+ else
+ lua_util.debugm(N, task, 'ObjStm object %s:%s has bad dict: %s',
+ obj.major, obj.minor, obj.dict)
+ end
+ end
+ end
+ end
+end
+
+-- This function arranges starts and ends of all objects and process them into initial
+-- set of objects
+local function extract_outer_objects(task, input, pdf)
+ local start_pos, end_pos = 1, 1
+ local max_start_pos, max_end_pos
+ local obj_count = 0
+
+ max_start_pos = math.min(config.max_pdf_objects, #pdf.start_objects)
+ max_end_pos = math.min(config.max_pdf_objects, #pdf.end_objects)
+ lua_util.debugm(N, task, "pdf: extract objects from %s start positions and %s end positions",
+ max_start_pos, max_end_pos)
+
+ while start_pos <= max_start_pos and end_pos <= max_end_pos do
+ local first = pdf.start_objects[start_pos]
+ local last = pdf.end_objects[end_pos]
+
+ -- 7 is length of `endobj\n`
+ if first + 6 < last then
+ local len = last - first - 6
+
+ -- Also get the starting span and try to match it versus obj re to get numbers
+ local obj_line_potential = first - 32
+ if obj_line_potential < 1 then
+ obj_line_potential = 1
+ end
+ local prev_obj_end = pdf.end_objects[end_pos - 1]
+ if end_pos > 1 and prev_obj_end >= obj_line_potential and prev_obj_end < first then
+ obj_line_potential = prev_obj_end + 1
+ end
+
+ local obj_line_span = input:span(obj_line_potential, first - obj_line_potential + 1)
+ local matches = object_re:search(obj_line_span, true, true)
+
+ if matches and matches[1] then
+ local nobj = {
+ start = first,
+ len = len,
+ data = input:span(first, len),
+ major = tonumber(matches[1][2]),
+ minor = tonumber(matches[1][3]),
+ }
+ pdf.objects[obj_count + 1] = nobj
+ if nobj.major and nobj.minor then
+ -- Add reference
+ local ref = obj_ref(nobj.major, nobj.minor)
+ nobj.ref = ref -- Our internal reference
+ pdf.ref[ref] = nobj
+ end
+ end
+
+ obj_count = obj_count + 1
+ start_pos = start_pos + 1
+ end_pos = end_pos + 1
+ elseif first > last then
+ end_pos = end_pos + 1
+ else
+ start_pos = start_pos + 1
+ end_pos = end_pos + 1
+ end
+ end
+end
+
+-- This function attaches streams to objects and processes outer pdf grammar
+local function attach_pdf_streams(task, input, pdf)
+ if pdf.start_streams and pdf.end_streams then
+ local start_pos, end_pos = 1, 1
+ local max_start_pos, max_end_pos
+
+ max_start_pos = math.min(config.max_pdf_objects, #pdf.start_streams)
+ max_end_pos = math.min(config.max_pdf_objects, #pdf.end_streams)
+
+ for _, obj in ipairs(pdf.objects) do
+ while start_pos <= max_start_pos and end_pos <= max_end_pos do
+ local first = pdf.start_streams[start_pos]
+ local last = pdf.end_streams[end_pos]
+ last = last - 10 -- Exclude endstream\n pattern
+ lua_util.debugm(N, task, "start: %s, end: %s; obj: %s-%s",
+ first, last, obj.start, obj.start + obj.len)
+ if first > obj.start and last < obj.start + obj.len and last > first then
+ -- In case if we have fake endstream :(
+ while pdf.end_streams[end_pos + 1] and pdf.end_streams[end_pos + 1] < obj.start + obj.len do
+ end_pos = end_pos + 1
+ last = pdf.end_streams[end_pos]
+ end
+ -- Strip the first \n
+ while first < last do
+ local chr = input:byte(first)
+ if chr ~= 13 and chr ~= 10 then
+ break
+ end
+ first = first + 1
+ end
+ local len = last - first
+ obj.stream = {
+ start = first,
+ len = len,
+ data = input:span(first, len)
+ }
+ start_pos = start_pos + 1
+ end_pos = end_pos + 1
+ break
+ elseif first < obj.start then
+ start_pos = start_pos + 1
+ elseif last > obj.start + obj.len then
+ -- Not this object
+ break
+ else
+ start_pos = start_pos + 1
+ end_pos = end_pos + 1
+ end
+ end
+ if obj.stream then
+ lua_util.debugm(N, task, 'found object %s:%s %s start %s len, %s stream start, %s stream length',
+ obj.major, obj.minor, obj.start, obj.len, obj.stream.start, obj.stream.len)
+ else
+ lua_util.debugm(N, task, 'found object %s:%s %s start %s len, no stream',
+ obj.major, obj.minor, obj.start, obj.len)
+ end
+ end
+ end
+end
+
+-- Processes PDF objects: extracts streams, object numbers, process outer grammar,
+-- augment object types
+local function postprocess_pdf_objects(task, input, pdf)
+ pdf.objects = {} -- objects table
+ pdf.ref = {} -- references table
+ extract_outer_objects(task, input, pdf)
+
+ -- Now we have objects and we need to attach streams that are in bounds
+ attach_pdf_streams(task, input, pdf)
+ -- Parse grammar for outer objects
+ for i, obj in ipairs(pdf.objects) do
+ if i > 0 and i % 100 == 0 then
+ local now = rspamd_util.get_ticks()
+
+ if now >= pdf.end_timestamp then
+ pdf.timeout_processing = now - pdf.start_timestamp
+
+ lua_util.debugm(N, task, 'pdf: timeout processing grammars after spending %s seconds, ' ..
+ '%s elements processed',
+ pdf.timeout_processing, i)
+ break
+ end
+ end
+ if obj.ref then
+ parse_object_grammar(obj, task, pdf)
+
+ -- Special early handling
+ if obj.dict and obj.dict.Type and obj.dict.Type == 'XRef' then
+ process_xref(task, pdf, obj)
+ end
+ end
+ end
+
+ if not pdf.timeout_processing then
+ extract_pdf_compound_objects(task, pdf)
+ else
+ -- ENOTIME
+ return
+ end
+
+ -- Now we might probably have all objects being processed
+ for i, obj in ipairs(pdf.objects) do
+ if obj.dict then
+ -- Types processing
+ if i > 0 and i % 100 == 0 then
+ local now = rspamd_util.get_ticks()
+
+ if now >= pdf.end_timestamp then
+ pdf.timeout_processing = now - pdf.start_timestamp
+
+ lua_util.debugm(N, task, 'pdf: timeout processing dicts after spending %s seconds, ' ..
+ '%s elements processed',
+ pdf.timeout_processing, i)
+ break
+ end
+ end
+ process_dict(task, pdf, obj, obj.dict)
+ end
+ end
+end
+
+local function offsets_to_blocks(starts, ends, out)
+ local start_pos, end_pos = 1, 1
+
+ while start_pos <= #starts and end_pos <= #ends do
+ local first = starts[start_pos]
+ local last = ends[end_pos]
+
+ if first < last then
+ local len = last - first
+ out[#out + 1] = {
+ start = first,
+ len = len,
+ }
+ start_pos = start_pos + 1
+ end_pos = end_pos + 1
+ elseif first > last then
+ end_pos = end_pos + 1
+ else
+ -- Not ordered properly!
+ break
+ end
+ end
+end
+
+local function search_text(task, pdf)
+ for _, obj in ipairs(pdf.objects) do
+ if obj.type == 'Page' and obj.contents then
+ local text = {}
+ for _, tobj in ipairs(obj.contents) do
+ maybe_extract_object_stream(tobj, pdf, task)
+ local matches = pdf_text_trie:match(tobj.uncompressed or '')
+ if matches then
+ local text_blocks = {}
+ local starts = {}
+ local ends = {}
+
+ for npat, matched_positions in pairs(matches) do
+ if npat == 1 then
+ for _, pos in ipairs(matched_positions) do
+ starts[#starts + 1] = pos
+ end
+ else
+ for _, pos in ipairs(matched_positions) do
+ ends[#ends + 1] = pos
+ end
+ end
+ end
+
+ offsets_to_blocks(starts, ends, text_blocks)
+ for _, bl in ipairs(text_blocks) do
+ if bl.len > 2 then
+ -- To remove \s+ET\b pattern (it can leave trailing space or not but it doesn't matter)
+ bl.len = bl.len - 2
+ end
+
+ bl.data = tobj.uncompressed:span(bl.start, bl.len)
+ --lua_util.debugm(N, task, 'extracted text from object %s:%s: %s',
+ -- tobj.major, tobj.minor, bl.data)
+
+ if bl.len < config.max_processing_size then
+ local ret, obj_or_err = pcall(pdf_text_grammar.match, pdf_text_grammar,
+ bl.data)
+
+ if ret then
+ text[#text + 1] = obj_or_err
+ lua_util.debugm(N, task, 'attached %s from content object %s:%s to %s:%s',
+ obj_or_err, tobj.major, tobj.minor, obj.major, obj.minor)
+ else
+ lua_util.debugm(N, task, 'object %s:%s cannot be parsed: %s',
+ obj.major, obj.minor, obj_or_err)
+ end
+
+ end
+ end
+ end
+ end
+
+ -- Join all text data together
+ if #text > 0 then
+ obj.text = rspamd_text.fromtable(text)
+ lua_util.debugm(N, task, 'object %s:%s is parsed to: %s',
+ obj.major, obj.minor, obj.text)
+ end
+ end
+ end
+end
+
+-- This function searches objects for `/URI` key and parses it's content
+local function search_urls(task, pdf, mpart)
+ local function recursive_object_traverse(obj, dict, rec)
+ if rec > 10 then
+ lua_util.debugm(N, task, 'object %s:%s recurses too much',
+ obj.major, obj.minor)
+ return
+ end
+
+ for k, v in pairs(dict) do
+ if type(v) == 'table' then
+ recursive_object_traverse(obj, v, rec + 1)
+ elseif k == 'URI' then
+ v = maybe_dereference_object(v, pdf, task)
+ if type(v) == 'string' then
+ local url = rspamd_url.create(task:get_mempool(), v, { 'content' })
+
+ if url then
+ lua_util.debugm(N, task, 'found url %s in object %s:%s',
+ v, obj.major, obj.minor)
+ task:inject_url(url, mpart)
+ end
+ end
+ end
+ end
+ end
+
+ for _, obj in ipairs(pdf.objects) do
+ if obj.dict and type(obj.dict) == 'table' then
+ recursive_object_traverse(obj, obj.dict, 0)
+ end
+ end
+end
+
+local function process_pdf(input, mpart, task)
+
+ if not config.enabled then
+ -- Skip processing
+ return {}
+ end
+
+ local matches = pdf_trie:match(input)
+
+ if matches then
+ local start_ts = rspamd_util.get_ticks()
+ -- Temp object used to share data between pdf extraction methods
+ local pdf_object = {
+ tag = 'pdf',
+ extract_text = extract_text_data,
+ start_timestamp = start_ts,
+ end_timestamp = start_ts + config.pdf_process_timeout,
+ }
+ -- Output object that excludes all internal stuff
+ local pdf_output = lua_util.shallowcopy(pdf_object)
+ local grouped_processors = {}
+ for npat, matched_positions in pairs(matches) do
+ local index = pdf_indexes[npat]
+
+ local proc_key, loc_npat = index[1], index[4]
+
+ if not grouped_processors[proc_key] then
+ grouped_processors[proc_key] = {
+ processor_func = processors[proc_key],
+ offsets = {},
+ }
+ end
+ local proc = grouped_processors[proc_key]
+ -- Fill offsets
+ for _, pos in ipairs(matched_positions) do
+ proc.offsets[#proc.offsets + 1] = { pos, loc_npat }
+ end
+ end
+
+ for name, processor in pairs(grouped_processors) do
+ -- Sort by offset
+ lua_util.debugm(N, task, "pdf: process group %s with %s matches",
+ name, #processor.offsets)
+ table.sort(processor.offsets, function(e1, e2)
+ return e1[1] < e2[1]
+ end)
+ processor.processor_func(input, task, processor.offsets, pdf_object, pdf_output)
+ end
+
+ pdf_output.flags = {}
+
+ if pdf_object.start_objects and pdf_object.end_objects then
+ if #pdf_object.start_objects > config.max_pdf_objects then
+ pdf_output.many_objects = #pdf_object.start_objects
+ -- Trim
+ end
+
+ -- Postprocess objects
+ postprocess_pdf_objects(task, input, pdf_object)
+ if config.text_extraction then
+ search_text(task, pdf_object, pdf_output)
+ end
+ if config.url_extraction then
+ search_urls(task, pdf_object, mpart, pdf_output)
+ end
+
+ if config.js_fuzzy and pdf_object.scripts then
+ pdf_output.fuzzy_hashes = {}
+ if config.openaction_fuzzy_only then
+ -- OpenAction only
+ if pdf_object.openaction and pdf_object.openaction.bin_hash then
+ if config.min_js_fuzzy and #pdf_object.openaction.data >= config.min_js_fuzzy then
+ lua_util.debugm(N, task, "pdf: add fuzzy hash from openaction: %s; size = %s; object: %s:%s",
+ pdf_object.openaction.hash,
+ #pdf_object.openaction.data,
+ pdf_object.openaction.object.major, pdf_object.openaction.object.minor)
+ table.insert(pdf_output.fuzzy_hashes, pdf_object.openaction.bin_hash)
+ else
+ lua_util.debugm(N, task, "pdf: skip fuzzy hash from JavaScript: %s, too short: %s",
+ pdf_object.openaction.hash, #pdf_object.openaction.data)
+ end
+ end
+ else
+ -- All hashes
+ for h, sc in pairs(pdf_object.scripts) do
+ if config.min_js_fuzzy and #sc.data >= config.min_js_fuzzy then
+ lua_util.debugm(N, task, "pdf: add fuzzy hash from JavaScript: %s; size = %s; object: %s:%s",
+ sc.hash,
+ #sc.data,
+ sc.object.major, sc.object.minor)
+ table.insert(pdf_output.fuzzy_hashes, h)
+ else
+ lua_util.debugm(N, task, "pdf: skip fuzzy hash from JavaScript: %s, too short: %s",
+ sc.hash, #sc.data)
+ end
+ end
+
+ end
+ end
+ else
+ pdf_output.flags.no_objects = true
+ end
+
+ -- Propagate from object to output
+ if pdf_object.encrypted then
+ pdf_output.encrypted = true
+ end
+ if pdf_object.scripts then
+ pdf_output.scripts = true
+ end
+
+ return pdf_output
+ end
+end
+
+-- Processes the PDF trailer
+processors.trailer = function(input, task, positions, pdf_object, pdf_output)
+ local last_pos = positions[#positions]
+
+ lua_util.debugm(N, task, 'pdf: process trailer at position %s (%s total length)',
+ last_pos, #input)
+
+ if last_pos[1] > config.max_pdf_trailer then
+ pdf_output.long_trailer = #input - last_pos[1]
+ return
+ end
+
+ local last_span = input:span(last_pos[1])
+ local lines_checked = 0
+ for line in last_span:lines(true) do
+ if line:find('/Encrypt ') then
+ lua_util.debugm(N, task, "pdf: found encrypted line in trailer: %s",
+ line)
+ pdf_output.encrypted = true
+ pdf_object.encrypted = true
+ break
+ end
+ lines_checked = lines_checked + 1
+
+ if lines_checked > config.max_pdf_trailer_lines then
+ lua_util.debugm(N, task, "pdf: trailer has too many lines, stop checking")
+ pdf_output.long_trailer = #input - last_pos[1]
+ break
+ end
+ end
+end
+
+processors.suspicious = function(input, task, positions, pdf_object, pdf_output)
+ local suspicious_factor = 0.0
+ local nexec = 0
+ local nencoded = 0
+ local close_encoded = 0
+ local last_encoded
+ for _, match in ipairs(positions) do
+ if match[2] == 1 then
+ -- netsh
+ suspicious_factor = suspicious_factor + 0.5
+ elseif match[2] == 2 then
+ nexec = nexec + 1
+ elseif match[2] == 3 then
+ local enc_data = input:sub(match[1] - 2, match[1] - 1)
+ local legal_escape = false
+
+ if enc_data then
+ enc_data = enc_data:strtoul()
+
+ if enc_data then
+ -- Legit encode cases are non printable characters (e.g. spaces)
+ if enc_data < 0x21 or enc_data >= 0x7f then
+ legal_escape = true
+ end
+ end
+ end
+
+ if not legal_escape then
+ nencoded = nencoded + 1
+
+ if last_encoded then
+ if match[1] - last_encoded < 8 then
+ -- likely consecutive encoded chars, increase factor
+ close_encoded = close_encoded + 1
+ end
+ end
+ last_encoded = match[1]
+
+ end
+ end
+ end
+
+ if nencoded > 10 then
+ suspicious_factor = suspicious_factor + nencoded / 10
+ end
+ if nexec > 1 then
+ suspicious_factor = suspicious_factor + nexec / 2.0
+ end
+ if close_encoded > 4 and nencoded - close_encoded < 5 then
+ -- Too many close encoded comparing to the total number of encoded characters
+ suspicious_factor = suspicious_factor + 0.5
+ end
+
+ lua_util.debugm(N, task, 'pdf: found a suspicious patterns: %s exec, %s encoded (%s close), ' ..
+ '%s final factor',
+ nexec, nencoded, close_encoded, suspicious_factor)
+
+ if suspicious_factor > 1.0 then
+ suspicious_factor = 1.0
+ end
+
+ pdf_output.suspicious = suspicious_factor
+end
+
+local function generic_table_inserter(positions, pdf_object, output_key)
+ if not pdf_object[output_key] then
+ pdf_object[output_key] = {}
+ end
+ local shift = #pdf_object[output_key]
+ for i, pos in ipairs(positions) do
+ pdf_object[output_key][i + shift] = pos[1]
+ end
+end
+
+processors.start_object = function(_, task, positions, pdf_object)
+ generic_table_inserter(positions, pdf_object, 'start_objects')
+end
+
+processors.end_object = function(_, task, positions, pdf_object)
+ generic_table_inserter(positions, pdf_object, 'end_objects')
+end
+
+processors.start_stream = function(_, task, positions, pdf_object)
+ generic_table_inserter(positions, pdf_object, 'start_streams')
+end
+
+processors.end_stream = function(_, task, positions, pdf_object)
+ generic_table_inserter(positions, pdf_object, 'end_streams')
+end
+
+exports.process = process_pdf
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_content/vcard.lua b/lualib/lua_content/vcard.lua
new file mode 100644
index 0000000..ed14412
--- /dev/null
+++ b/lualib/lua_content/vcard.lua
@@ -0,0 +1,84 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local l = require 'lpeg'
+local lua_util = require "lua_util"
+local N = "lua_content"
+
+local vcard_grammar
+
+-- XXX: Currently it is a copy of ical grammar
+local function gen_grammar()
+ if not vcard_grammar then
+ local wsp = l.S(" \t\v\f")
+ local crlf = (l.P "\r" ^ -1 * l.P "\n") + l.P "\r"
+ local eol = (crlf * #crlf) + (crlf - (crlf ^ -1 * wsp))
+ local name = l.C((l.P(1) - (l.P ":")) ^ 1) / function(v)
+ return (v:gsub("[\n\r]+%s", ""))
+ end
+ local value = l.C((l.P(1) - eol) ^ 0) / function(v)
+ return (v:gsub("[\n\r]+%s", ""))
+ end
+ vcard_grammar = name * ":" * wsp ^ 0 * value * eol ^ -1
+ end
+
+ return vcard_grammar
+end
+
+local exports = {}
+
+local function process_vcard(input, mpart, task)
+ local control = { n = '\n', r = '' }
+ local rspamd_url = require "rspamd_url"
+ local escaper = l.Ct((gen_grammar() / function(key, value)
+ value = value:gsub("\\(.)", control)
+ key = key:lower()
+ local local_urls = rspamd_url.all(task:get_mempool(), value)
+
+ if local_urls and #local_urls > 0 then
+ for _, u in ipairs(local_urls) do
+ lua_util.debugm(N, task, 'vcard: found URL in vcard %s',
+ tostring(u))
+ task:inject_url(u, mpart)
+ end
+ end
+ lua_util.debugm(N, task, 'vcard: vcard key %s = "%s"',
+ key, value)
+ return { key, value }
+ end) ^ 1)
+
+ local elts = escaper:match(input)
+
+ if not elts then
+ return nil
+ end
+
+ return {
+ tag = 'vcard',
+ extract_text = function()
+ return nil
+ end, -- NYI
+ elts = elts
+ }
+end
+
+--[[[
+-- @function vcard.process(input)
+-- Returns all values from vcard as a plain text. Names are completely ignored.
+--]]
+exports.process = process_vcard
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_dkim_tools.lua b/lualib/lua_dkim_tools.lua
new file mode 100644
index 0000000..165ea8f
--- /dev/null
+++ b/lualib/lua_dkim_tools.lua
@@ -0,0 +1,742 @@
+--[[
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local exports = {}
+
+local E = {}
+local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local logger = require "rspamd_logger"
+local fun = require "fun"
+
+local function check_violation(N, task, domain)
+ -- Check for DKIM_REJECT
+ local sym_check = 'R_DKIM_REJECT'
+
+ if N == 'arc' then
+ sym_check = 'ARC_REJECT'
+ end
+ if task:has_symbol(sym_check) then
+ local sym = task:get_symbol(sym_check)[1]
+ logger.infox(task, 'skip signing for %s: violation %s found: %s',
+ domain, sym_check, sym.options)
+ return false
+ end
+
+ return true
+end
+
+local function insert_or_update_prop(N, task, p, prop, origin, data)
+ if #p == 0 then
+ local k = {}
+ k[prop] = data
+ table.insert(p, k)
+ lua_util.debugm(N, task, 'add %s "%s" using %s', prop, data, origin)
+ else
+ for _, k in ipairs(p) do
+ if not k[prop] then
+ k[prop] = data
+ lua_util.debugm(N, task, 'set %s to "%s" using %s', prop, data, origin)
+ end
+ end
+ end
+end
+
+local function get_mempool_selectors(N, task)
+ local p = {}
+ local key_var = "dkim_key"
+ local selector_var = "dkim_selector"
+ if N == "arc" then
+ key_var = "arc_key"
+ selector_var = "arc_selector"
+ end
+
+ p.key = task:get_mempool():get_variable(key_var)
+ p.selector = task:get_mempool():get_variable(selector_var)
+
+ if (not p.key or not p.selector) then
+ return false, {}
+ end
+
+ lua_util.debugm(N, task, 'override selector and key to %s:%s', p.key, p.selector)
+ return true, p
+end
+
+local function parse_dkim_http_headers(N, task, settings)
+ -- Configure headers
+ local headers = {
+ sign_header = settings.http_sign_header or "PerformDkimSign",
+ sign_on_reject_header = settings.http_sign_on_reject_header_header or 'SignOnAuthFailed',
+ domain_header = settings.http_domain_header or 'DkimDomain',
+ selector_header = settings.http_selector_header or 'DkimSelector',
+ key_header = settings.http_key_header or 'DkimPrivateKey'
+ }
+
+ if task:get_request_header(headers.sign_header) then
+ local domain = task:get_request_header(headers.domain_header)
+ local selector = task:get_request_header(headers.selector_header)
+ local key = task:get_request_header(headers.key_header)
+
+ if not (domain and selector and key) then
+
+ logger.errx(task, 'missing required headers to sign email')
+ return false, {}
+ end
+
+ -- Now check if we need to check the existing auth
+ local hdr = task:get_request_header(headers.sign_on_reject_header)
+ if not hdr or tostring(hdr) == '0' or tostring(hdr) == 'false' then
+ if not check_violation(N, task, domain, selector) then
+ return false, {}
+ end
+ end
+
+ local p = {}
+ local k = {
+ domain = tostring(domain),
+ rawkey = tostring(key),
+ selector = tostring(selector),
+ }
+ table.insert(p, k)
+ return true, p
+ end
+
+ lua_util.debugm(N, task, 'no sign header %s', headers.sign_header)
+ return false, {}
+end
+
+local function prepare_dkim_signing(N, task, settings)
+ local is_local, is_sign_networks, is_authed
+
+ if settings.use_http_headers then
+ local res, tbl = parse_dkim_http_headers(N, task, settings)
+
+ if not res then
+ if not settings.allow_headers_fallback then
+ return res, {}
+ else
+ lua_util.debugm(N, task, 'failed to read http headers, fallback to normal schema')
+ end
+ else
+ return res, tbl
+ end
+ end
+
+ if settings.sign_condition and type(settings.sign_condition) == 'function' then
+ -- Use sign condition only
+ local ret = settings.sign_condition(task)
+
+ if not ret then
+ return false, {}
+ end
+
+ if ret[1] then
+ return true, ret
+ else
+ return true, { ret }
+ end
+ end
+
+ local auser = task:get_user()
+ local ip = task:get_from_ip()
+
+ if ip and ip:is_local() then
+ is_local = true
+ end
+
+ local has_pre_result = task:has_pre_result()
+ if has_pre_result then
+ local metric_action = task:get_metric_action()
+
+ if metric_action == 'reject' or metric_action == 'drop' then
+ -- No need to sign what we are already rejecting/dropping
+ lua_util.debugm(N, task, 'task result is already %s, no need to sign', metric_action)
+ return false, {}
+ end
+
+ if metric_action == 'soft reject' then
+ -- Same here, we are going to delay an email, signing is just a waste of time
+ lua_util.debugm(N, task, 'task result is %s, skip signing', metric_action)
+ return false, {}
+ end
+
+ -- For spam actions, there is no clear distinction
+ if metric_action ~= 'no action' and type(settings.skip_spam_sign) == 'boolean' and settings.skip_spam_sign then
+ lua_util.debugm(N, task, 'task result is %s, no need to sign', metric_action)
+ return false, {}
+ end
+ end
+
+ if settings.sign_authenticated and auser then
+ lua_util.debugm(N, task, 'user is authenticated')
+ is_authed = true
+ elseif (settings.sign_networks and settings.sign_networks:get_key(ip)) then
+ is_sign_networks = true
+ lua_util.debugm(N, task, 'mail is from address in sign_networks')
+ elseif settings.sign_local and is_local then
+ lua_util.debugm(N, task, 'mail is from local address')
+ elseif settings.sign_inbound and not is_local and not auser then
+ lua_util.debugm(N, task, 'mail was sent to us')
+ else
+ lua_util.debugm(N, task, 'mail is ineligible for signing')
+ return false, {}
+ end
+
+ local efrom = task:get_from('smtp')
+ local empty_envelope = false
+ if #(((efrom or E)[1] or E).addr or '') == 0 then
+ if not settings.allow_envfrom_empty then
+ lua_util.debugm(N, task, 'empty envelope from not allowed')
+ return false, {}
+ else
+ empty_envelope = true
+ end
+ end
+
+ local hfrom = task:get_from('mime')
+ if not settings.allow_hdrfrom_multiple and (hfrom or E)[2] then
+ lua_util.debugm(N, task, 'multiple header from not allowed')
+ return false, {}
+ end
+
+ local eto = task:get_recipients(0)
+
+ local dkim_domain
+ local hdom = ((hfrom or E)[1] or E).domain
+ local edom = ((efrom or E)[1] or E).domain
+ local tdom = ((eto or E)[1] or E).domain
+ local udom = string.match(auser or '', '.*@(.*)')
+
+ local function get_dkim_domain(dtype)
+ if settings[dtype] == 'header' then
+ return hdom
+ elseif settings[dtype] == 'envelope' then
+ return edom
+ elseif settings[dtype] == 'auth' then
+ return udom
+ elseif settings[dtype] == 'recipient' then
+ return tdom
+ else
+ return settings[dtype]:lower()
+ end
+ end
+
+ local function is_skip_sign()
+ return not (settings.sign_networks and is_sign_networks) and
+ not (settings.sign_authenticated and is_authed) and
+ not (settings.sign_local and is_local)
+ end
+
+ if hdom then
+ hdom = hdom:lower()
+ end
+ if edom then
+ edom = edom:lower()
+ end
+ if udom then
+ udom = udom:lower()
+ end
+ if tdom then
+ tdom = tdom:lower()
+ end
+
+ if settings.signing_table and (settings.key_table or settings.use_vault) then
+ -- OpenDKIM style
+ if is_skip_sign() then
+ lua_util.debugm(N, task,
+ 'skip signing: is_sign_network: %s, is_authed: %s, is_local: %s',
+ is_sign_networks, is_authed, is_local)
+ return false, {}
+ end
+
+ if not hfrom or not hfrom[1] or not hfrom[1].addr then
+ lua_util.debugm(N, task,
+ 'signing_table: cannot get data when no header from is presented')
+ return false, {}
+ end
+ local sign_entry = settings.signing_table:get_key(hfrom[1].addr:lower())
+
+ if sign_entry then
+ -- Check opendkim style entries
+ lua_util.debugm(N, task,
+ 'signing_table: found entry for %s: %s', hfrom[1].addr, sign_entry)
+ if sign_entry == '%' then
+ sign_entry = hdom
+ end
+
+ if settings.key_table then
+ -- Now search in key table
+ local key_entry = settings.key_table:get_key(sign_entry)
+
+ if key_entry then
+ local parts = lua_util.str_split(key_entry, ':')
+
+ if #parts == 2 then
+ -- domain + key
+ local selector = settings.selector
+
+ if not selector then
+ logger.errx(task, 'no selector defined for sign_entry %s, key_entry %s',
+ sign_entry, key_entry)
+ return false, {}
+ end
+
+ local res = {
+ selector = selector,
+ domain = parts[1]:gsub('%%', hdom)
+ }
+
+ local st = parts[2]:sub(1, 2)
+
+ if st:sub(1, 1) == '/' or st == './' or st == '..' then
+ res.key = parts[2]:gsub('%%', hdom)
+ lua_util.debugm(N, task, 'perform dkim signing for %s, selector=%s, domain=%s, key file=%s',
+ hdom, selector, res.domain, res.key)
+ else
+ res.rawkey = parts[2] -- No sanity check here
+ lua_util.debugm(N, task, 'perform dkim signing for %s, selector=%s, domain=%s, raw key used',
+ hdom, selector, res.domain)
+ end
+
+ return true, { res }
+ elseif #parts == 3 then
+ -- domain, selector, key
+ local selector = parts[2]
+
+ local res = {
+ selector = selector,
+ domain = parts[1]:gsub('%%', hdom)
+ }
+
+ local st = parts[3]:sub(1, 2)
+
+ if st:sub(1, 1) == '/' or st == './' or st == '..' then
+ res.key = parts[3]:gsub('%%', hdom)
+ lua_util.debugm(N, task, 'perform dkim signing for %s, selector=%s, domain=%s, key file=%s',
+ hdom, selector, res.domain, res.key)
+ else
+ res.rawkey = parts[3] -- No sanity check here
+ lua_util.debugm(N, task, 'perform dkim signing for %s, selector=%s, domain=%s, raw key used',
+ hdom, selector, res.domain)
+ end
+
+ return true, { res }
+ else
+ logger.errx(task, 'invalid key entry for sign entry %s: %s; when signing %s domain',
+ sign_entry, key_entry, hdom)
+ return false, {}
+ end
+ elseif settings.use_vault then
+ -- Sign table is presented, the rest is covered by vault
+ lua_util.debugm(N, task, 'check vault for %s, by sign entry %s, key entry is missing',
+ hdom, sign_entry)
+ return true, {
+ domain = sign_entry,
+ vault = true
+ }
+ else
+ logger.errx(task, 'missing key entry for sign entry %s; when signing %s domain',
+ sign_entry, hdom)
+ return false, {}
+ end
+ else
+ logger.errx(task, 'cannot get key entry for signing entry %s, when signing %s domain',
+ sign_entry, hdom)
+ return false, {}
+ end
+ else
+ lua_util.debugm(N, task,
+ 'signing_table: no entry for %s', hfrom[1].addr)
+ return false, {}
+ end
+ else
+ if settings.use_domain_sign_networks and is_sign_networks then
+ dkim_domain = get_dkim_domain('use_domain_sign_networks')
+ lua_util.debugm(N, task,
+ 'sign_networks: use domain(%s) for signature: %s',
+ settings.use_domain_sign_networks, dkim_domain)
+ elseif settings.use_domain_sign_local and is_local then
+ dkim_domain = get_dkim_domain('use_domain_sign_local')
+ lua_util.debugm(N, task, 'local: use domain(%s) for signature: %s',
+ settings.use_domain_sign_local, dkim_domain)
+ elseif settings.use_domain_sign_inbound and not is_local and not auser then
+ dkim_domain = get_dkim_domain('use_domain_sign_inbound')
+ lua_util.debugm(N, task, 'inbound: use domain(%s) for signature: %s',
+ settings.use_domain_sign_inbound, dkim_domain)
+ elseif settings.use_domain_custom then
+ if type(settings.use_domain_custom) == 'string' then
+ -- Load custom function
+ local loadstring = loadstring or load
+ local ret, res_or_err = pcall(loadstring(settings.use_domain_custom))
+ if ret then
+ if type(res_or_err) == 'function' then
+ settings.use_domain_custom = res_or_err
+ dkim_domain = settings.use_domain_custom(task)
+ lua_util.debugm(N, task, 'use custom domain for signing: %s',
+ dkim_domain)
+ else
+ logger.errx(task, 'cannot load dkim domain custom script: invalid type: %s, expected function',
+ type(res_or_err))
+ settings.use_domain_custom = nil
+ end
+ else
+ logger.errx(task, 'cannot load dkim domain custom script: %s', res_or_err)
+ settings.use_domain_custom = nil
+ end
+ else
+ dkim_domain = settings.use_domain_custom(task)
+ lua_util.debugm(N, task, 'use custom domain for signing: %s',
+ dkim_domain)
+ end
+ else
+ dkim_domain = get_dkim_domain('use_domain')
+ lua_util.debugm(N, task, 'use domain(%s) for signature: %s',
+ settings.use_domain, dkim_domain)
+ end
+ end
+
+ if not dkim_domain then
+ lua_util.debugm(N, task, 'could not extract dkim domain')
+ return false, {}
+ end
+
+ if settings.use_esld then
+ dkim_domain = rspamd_util.get_tld(dkim_domain)
+ if hdom then
+ hdom = rspamd_util.get_tld(hdom)
+ end
+ if edom then
+ edom = rspamd_util.get_tld(edom)
+ end
+ end
+
+ lua_util.debugm(N, task, 'final DKIM domain: %s', dkim_domain)
+
+ -- Sanity checks
+ if edom and hdom and not settings.allow_hdrfrom_mismatch and hdom ~= edom then
+ if settings.allow_hdrfrom_mismatch_local and is_local then
+ lua_util.debugm(N, task, 'domain mismatch allowed for local IP: %1 != %2', hdom, edom)
+ elseif settings.allow_hdrfrom_mismatch_sign_networks and is_sign_networks then
+ lua_util.debugm(N, task, 'domain mismatch allowed for sign_networks: %1 != %2', hdom, edom)
+ else
+ if empty_envelope and hdom then
+ lua_util.debugm(N, task, 'domain mismatch allowed for empty envelope: %1 != %2', hdom, edom)
+ else
+ lua_util.debugm(N, task, 'domain mismatch not allowed: %1 != %2', hdom, edom)
+ return false, {}
+ end
+ end
+ end
+
+ if auser and not settings.allow_username_mismatch then
+ if not udom then
+ lua_util.debugm(N, task, 'couldnt find domain in username')
+ return false, {}
+ end
+ if settings.use_esld then
+ udom = rspamd_util.get_tld(udom)
+ end
+ if udom ~= dkim_domain then
+ lua_util.debugm(N, task, 'user domain mismatch')
+ return false, {}
+ end
+ end
+
+ local p = {}
+
+ if settings.use_vault then
+ if settings.vault_domains then
+ if settings.vault_domains:get_key(dkim_domain) then
+ return true, {
+ domain = dkim_domain,
+ vault = true,
+ }
+ else
+ lua_util.debugm(N, task, 'domain %s is not designated for vault',
+ dkim_domain)
+ return false, {}
+ end
+ else
+ -- TODO: try every domain in the vault
+ return true, {
+ domain = dkim_domain,
+ vault = true,
+ }
+ end
+ end
+
+ if settings.domain[dkim_domain] then
+ -- support old style selector/paths
+ if settings.domain[dkim_domain].selector or
+ settings.domain[dkim_domain].path then
+ local k = {}
+ k.selector = settings.domain[dkim_domain].selector
+ k.key = settings.domain[dkim_domain].path
+ table.insert(p, k)
+ end
+ for _, s in ipairs((settings.domain[dkim_domain].selectors or {})) do
+ lua_util.debugm(N, task, 'adding selector: %1', s)
+ local k = {}
+ k.selector = s.selector
+ k.key = s.path
+ table.insert(p, k)
+ end
+ end
+
+ if #p == 0 then
+ local ret, k = get_mempool_selectors(N, task)
+ if ret then
+ table.insert(p, k)
+ lua_util.debugm(N, task, 'using mempool selector %s with key %s',
+ k.selector, k.key)
+ end
+ end
+
+ if settings.selector_map then
+ local data = settings.selector_map:get_key(dkim_domain)
+ if data then
+ insert_or_update_prop(N, task, p, 'selector', 'selector_map', data)
+ else
+ lua_util.debugm(N, task, 'no selector in map for %s', dkim_domain)
+ end
+ end
+
+ if settings.path_map then
+ local data = settings.path_map:get_key(dkim_domain)
+ if data then
+ insert_or_update_prop(N, task, p, 'key', 'path_map', data)
+ else
+ lua_util.debugm(N, task, 'no key in map for %s', dkim_domain)
+ end
+ end
+
+ if #p == 0 and not settings.try_fallback then
+ lua_util.debugm(N, task, 'dkim unconfigured and fallback disabled')
+ return false, {}
+ end
+
+ if not settings.use_redis then
+ insert_or_update_prop(N, task, p, 'key',
+ 'default path', settings.path)
+ end
+
+ insert_or_update_prop(N, task, p, 'selector',
+ 'default selector', settings.selector)
+
+ if settings.check_violation then
+ if not check_violation(N, task, p.domain) then
+ return false, {}
+ end
+ end
+
+ insert_or_update_prop(N, task, p, 'domain', 'dkim_domain',
+ dkim_domain)
+
+ return true, p
+end
+
+exports.prepare_dkim_signing = prepare_dkim_signing
+
+exports.sign_using_redis = function(N, task, settings, selectors, sign_func, err_func)
+ local lua_redis = require "lua_redis"
+
+ local function try_redis_key(selector, p)
+ p.key = nil
+ p.selector = selector
+ local rk = string.format('%s.%s', p.selector, p.domain)
+ local function redis_key_cb(err, data)
+ if err then
+ err_func(string.format("cannot make request to load DKIM key for %s: %s",
+ rk, err))
+ elseif type(data) ~= 'string' then
+ lua_util.debugm(N, task, "missing DKIM key for %s", rk)
+ else
+ p.rawkey = data
+ lua_util.debugm(N, task, 'found and parsed key for %s:%s in Redis',
+ p.domain, p.selector)
+ sign_func(task, p)
+ end
+ end
+ local rret = lua_redis.redis_make_request(task,
+ settings.redis_params, -- connect params
+ rk, -- hash key
+ false, -- is write
+ redis_key_cb, --callback
+ 'HGET', -- command
+ { settings.key_prefix, rk } -- arguments
+ )
+ if not rret then
+ err_func(task,
+ string.format("cannot make request to load DKIM key for %s", rk))
+ end
+ end
+
+ for _, p in ipairs(selectors) do
+ if settings.selector_prefix then
+ logger.infox(task, "using selector prefix '%s' for domain '%s'",
+ settings.selector_prefix, p.domain);
+ local function redis_selector_cb(err, data)
+ if err or type(data) ~= 'string' then
+ err_func(task, string.format("cannot make request to load DKIM selector for domain %s: %s",
+ p.domain, err))
+ else
+ try_redis_key(data, p)
+ end
+ end
+ local rret = lua_redis.redis_make_request(task,
+ settings.redis_params, -- connect params
+ p.domain, -- hash key
+ false, -- is write
+ redis_selector_cb, --callback
+ 'HGET', -- command
+ { settings.selector_prefix, p.domain } -- arguments
+ )
+ if not rret then
+ err_func(task, string.format("cannot make Redis request to load DKIM selector for domain %s",
+ p.domain))
+ end
+ else
+ try_redis_key(p.selector, p)
+ end
+ end
+end
+
+exports.sign_using_vault = function(N, task, settings, selectors, sign_func, err_func)
+ local http = require "rspamd_http"
+ local ucl = require "ucl"
+
+ local full_url = string.format('%s/v1/%s/%s',
+ settings.vault_url, settings.vault_path or 'dkim', selectors.domain)
+ local upstream_list = lua_util.http_upstreams_by_url(rspamd_config:get_mempool(), settings.vault_url)
+
+ local function vault_callback(err, code, body, _)
+ if code ~= 200 then
+ err_func(task, string.format('cannot request data from the vault url: %s; %s (%s)',
+ full_url, err, body))
+ else
+ local parser = ucl.parser()
+ local res, parser_err = parser:parse_string(body)
+ if not res then
+ err_func(task, string.format('vault reply for %s (data=%s) cannot be parsed: %s',
+ full_url, body, parser_err))
+ else
+ local obj = parser:get_object()
+
+ if not obj or not obj.data then
+ err_func(task, string.format('vault reply for %s (data=%s) is invalid, no data',
+ full_url, body))
+ else
+ local elts = obj.data.selectors or {}
+
+ -- Filter selectors by time/sanity
+ local function is_selector_valid(p)
+ if not p.key or not p.selector then
+ return false
+ end
+
+ if p.valid_start then
+ -- Check start time
+ if rspamd_util.get_time() < tonumber(p.valid_start) then
+ return false
+ end
+ end
+
+ if p.valid_end then
+ if rspamd_util.get_time() >= tonumber(p.valid_end) then
+ return false
+ end
+ end
+
+ return true
+ end
+ fun.each(function(p)
+ local dkim_sign_data = {
+ rawkey = p.key,
+ selector = p.selector,
+ domain = p.domain or selectors.domain,
+ alg = p.alg,
+ }
+ lua_util.debugm(N, task, 'found and parsed key for %s:%s in Vault',
+ dkim_sign_data.domain, dkim_sign_data.selector)
+ sign_func(task, dkim_sign_data)
+ end, fun.filter(is_selector_valid, elts))
+ end
+ end
+ end
+ end
+
+ local ret = http.request {
+ task = task,
+ url = full_url,
+ callback = vault_callback,
+ timeout = settings.http_timeout or 5.0,
+ no_ssl_verify = settings.no_ssl_verify,
+ keepalive = true,
+ upstream = upstream_list and upstream_list:get_upstream_round_robin() or nil,
+ headers = {
+ ['X-Vault-Token'] = settings.vault_token,
+ },
+ }
+
+ if not ret then
+ err_func(task, string.format("cannot make HTTP request to load DKIM data domain %s",
+ selectors.domain))
+ end
+end
+
+exports.validate_signing_settings = function(settings)
+ return settings.use_redis or
+ settings.path or
+ settings.domain or
+ settings.path_map or
+ settings.selector_map or
+ settings.use_http_headers or
+ (settings.signing_table and settings.key_table) or
+ (settings.use_vault and settings.vault_url and settings.vault_token) or
+ settings.sign_condition
+end
+
+exports.process_signing_settings = function(N, settings, opts)
+ local lua_maps = require "lua_maps"
+ -- Used to convert plain options to the maps
+ local maps_opts = {
+ sign_networks = { 'radix', 'DKIM signing networks' },
+ path_map = { 'map', 'Paths to DKIM signing keys' },
+ selector_map = { 'map', 'DKIM selectors' },
+ signing_table = { 'glob', 'DKIM signing table' },
+ key_table = { 'glob', 'DKIM keys table' },
+ vault_domains = { 'glob', 'DKIM signing domains in vault' },
+ whitelisted_signers_map = { 'set', 'ARC trusted signers domains' }
+ }
+ for k, v in pairs(opts) do
+ local maybe_map = maps_opts[k]
+ if maybe_map then
+ settings[k] = lua_maps.map_add_from_ucl(v, maybe_map[1], maybe_map[2])
+ elseif k == 'sign_condition' then
+ local ret, f = lua_util.callback_from_string(v)
+ if ret then
+ settings[k] = f
+ else
+ logger.errx(rspamd_config, 'cannot load sign condition %s: %s', v, f)
+ end
+ else
+ settings[k] = v
+ end
+ end
+end
+
+return exports
diff --git a/lualib/lua_ffi/common.lua b/lualib/lua_ffi/common.lua
new file mode 100644
index 0000000..4076cfa
--- /dev/null
+++ b/lualib/lua_ffi/common.lua
@@ -0,0 +1,45 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_ffi/common
+-- Common ffi definitions
+--]]
+
+local ffi = require 'ffi'
+
+ffi.cdef [[
+struct GString {
+ char *str;
+ size_t len;
+ size_t allocated_len;
+};
+struct GArray {
+ char *data;
+ unsigned len;
+};
+typedef void (*ref_dtor_cb_t)(void *data);
+struct ref_entry_s {
+ unsigned int refcount;
+ ref_dtor_cb_t dtor;
+};
+
+void g_string_free (struct GString *st, int free_data);
+void g_free (void *p);
+long rspamd_snprintf (char *buf, long max, const char *fmt, ...);
+]]
+
+return {} \ No newline at end of file
diff --git a/lualib/lua_ffi/dkim.lua b/lualib/lua_ffi/dkim.lua
new file mode 100644
index 0000000..e4592c2
--- /dev/null
+++ b/lualib/lua_ffi/dkim.lua
@@ -0,0 +1,144 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_ffi/dkim
+-- This module contains ffi interfaces to DKIM
+--]]
+
+local ffi = require 'ffi'
+
+ffi.cdef [[
+struct rspamd_dkim_sign_context_s;
+struct rspamd_dkim_key_s;
+struct rspamd_task;
+enum rspamd_dkim_key_format {
+ RSPAMD_DKIM_KEY_FILE = 0,
+ RSPAMD_DKIM_KEY_PEM,
+ RSPAMD_DKIM_KEY_BASE64,
+ RSPAMD_DKIM_KEY_RAW,
+};
+enum rspamd_dkim_type {
+ RSPAMD_DKIM_NORMAL,
+ RSPAMD_DKIM_ARC_SIG,
+ RSPAMD_DKIM_ARC_SEAL
+};
+struct rspamd_dkim_sign_context_s*
+rspamd_create_dkim_sign_context (struct rspamd_task *task,
+ struct rspamd_dkim_key_s *priv_key,
+ int headers_canon,
+ int body_canon,
+ const char *dkim_headers,
+ enum rspamd_dkim_type type,
+ void *unused);
+struct rspamd_dkim_key_s* rspamd_dkim_sign_key_load (const char *what, size_t len,
+ enum rspamd_dkim_key_format,
+ void *err);
+void rspamd_dkim_key_unref (struct rspamd_dkim_key_s *k);
+struct GString *rspamd_dkim_sign (struct rspamd_task *task,
+ const char *selector,
+ const char *domain,
+ unsigned long expire,
+ size_t len,
+ unsigned int idx,
+ const char *arc_cv,
+ struct rspamd_dkim_sign_context_s *ctx);
+]]
+
+local function load_sign_key(what, format)
+ if not format then
+ format = ffi.C.RSPAMD_DKIM_KEY_PEM
+ else
+ if format == 'file' then
+ format = ffi.C.RSPAMD_DKIM_KEY_FILE
+ elseif format == 'base64' then
+ format = ffi.C.RSPAMD_DKIM_KEY_BASE64
+ elseif format == 'raw' then
+ format = ffi.C.RSPAMD_DKIM_KEY_RAW
+ else
+ return nil, 'unknown key format'
+ end
+ end
+
+ return ffi.C.rspamd_dkim_sign_key_load(what, #what, format, nil)
+end
+
+local default_dkim_headers = "(o)from:(o)sender:(o)reply-to:(o)subject:(o)date:(o)message-id:" ..
+ "(o)to:(o)cc:(o)mime-version:(o)content-type:(o)content-transfer-encoding:" ..
+ "resent-to:resent-cc:resent-from:resent-sender:resent-message-id:" ..
+ "(o)in-reply-to:(o)references:list-id:list-owner:list-unsubscribe:" ..
+ "list-subscribe:list-post:(o)openpgp:(o)autocrypt"
+
+local function create_sign_context(task, privkey, dkim_headers, sign_type)
+ if not task or not privkey then
+ return nil, 'invalid arguments'
+ end
+
+ if not dkim_headers then
+ dkim_headers = default_dkim_headers
+ end
+
+ if not sign_type then
+ sign_type = 'dkim'
+ end
+
+ if sign_type == 'dkim' then
+ sign_type = ffi.C.RSPAMD_DKIM_NORMAL
+ elseif sign_type == 'arc-sig' then
+ sign_type = ffi.C.RSPAMD_DKIM_ARC_SIG
+ elseif sign_type == 'arc-seal' then
+ sign_type = ffi.C.RSPAMD_DKIM_ARC_SEAL
+ else
+ return nil, 'invalid sign type'
+ end
+
+ return ffi.C.rspamd_create_dkim_sign_context(task:topointer(), privkey,
+ 1, 1, dkim_headers, sign_type, nil)
+end
+
+local function do_sign(task, sign_context, selector, domain,
+ expire, len, arc_idx)
+ if not task or not sign_context or not selector or not domain then
+ return nil, 'invalid arguments'
+ end
+
+ if not expire then
+ expire = 0
+ end
+ if not len then
+ len = 0
+ end
+ if not arc_idx then
+ arc_idx = 0
+ end
+
+ local gstring = ffi.C.rspamd_dkim_sign(task:topointer(), selector, domain, expire, len, arc_idx, nil, sign_context)
+
+ if not gstring then
+ return nil, 'cannot sign'
+ end
+
+ local ret = ffi.string(gstring.str, gstring.len)
+ ffi.C.g_string_free(gstring, true)
+
+ return ret
+end
+
+return {
+ load_sign_key = load_sign_key,
+ create_sign_context = create_sign_context,
+ do_sign = do_sign
+}
diff --git a/lualib/lua_ffi/init.lua b/lualib/lua_ffi/init.lua
new file mode 100644
index 0000000..efbbc7a
--- /dev/null
+++ b/lualib/lua_ffi/init.lua
@@ -0,0 +1,59 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_ffi
+-- This module contains ffi interfaces (requires luajit or lua-ffi)
+--]]
+
+local ffi
+
+local exports = {}
+
+if type(jit) == 'table' then
+ ffi = require "ffi"
+ local NULL = ffi.new 'void*'
+
+ exports.is_null = function(o)
+ return o ~= NULL
+ end
+else
+ local ret, result_or_err = pcall(require, 'ffi')
+
+ if not ret then
+ return {}
+ end
+
+ ffi = result_or_err
+ -- Lua ffi
+ local NULL = ffi.NULL or ffi.C.NULL
+ exports.is_null = function(o)
+ return o ~= NULL
+ end
+end
+
+pcall(ffi.load, "rspamd-server", true)
+exports.common = require "lua_ffi/common"
+exports.dkim = require "lua_ffi/dkim"
+exports.spf = require "lua_ffi/spf"
+exports.linalg = require "lua_ffi/linalg"
+
+for k, v in pairs(ffi) do
+ -- Preserve all stuff to use lua_ffi as ffi itself
+ exports[k] = v
+end
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_ffi/linalg.lua b/lualib/lua_ffi/linalg.lua
new file mode 100644
index 0000000..2df488a
--- /dev/null
+++ b/lualib/lua_ffi/linalg.lua
@@ -0,0 +1,87 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_ffi/linalg
+-- This module contains ffi interfaces to linear algebra routines
+--]]
+
+local ffi = require 'ffi'
+
+local exports = {}
+
+ffi.cdef [[
+ void kad_sgemm_simple(int trans_A, int trans_B, int M, int N, int K, const float *A, const float *B, float *C);
+ bool kad_ssyev_simple (int N, float *A, float *output);
+]]
+
+local function table_to_ffi(a, m, n)
+ local a_conv = ffi.new("float[?]", m * n)
+ for i = 1, m or #a do
+ for j = 1, n or #a[1] do
+ a_conv[(i - 1) * n + (j - 1)] = a[i][j]
+ end
+ end
+ return a_conv
+end
+
+local function ffi_to_table(a, m, n)
+ local res = {}
+
+ for i = 0, m - 1 do
+ res[i + 1] = {}
+ for j = 0, n - 1 do
+ res[i + 1][j + 1] = a[i * n + j]
+ end
+ end
+
+ return res
+end
+
+exports.sgemm = function(a, m, b, n, k, trans_a, trans_b)
+ if type(a) == 'table' then
+ -- Need to convert, slow!
+ a = table_to_ffi(a, m, k)
+ end
+ if type(b) == 'table' then
+ b = table_to_ffi(b, k, n)
+ end
+ local res = ffi.new("float[?]", m * n)
+ ffi.C.kad_sgemm_simple(trans_a or 0, trans_b or 0, m, n, k, ffi.cast('const float*', a),
+ ffi.cast('const float*', b), ffi.cast('float*', res))
+ return res
+end
+
+exports.eigen = function(a, n)
+ if type(a) == 'table' then
+ -- Need to convert, slow!
+ n = n or #a
+ a = table_to_ffi(a, n, n)
+ end
+
+ local res = ffi.new("float[?]", n)
+
+ if ffi.C.kad_ssyev_simple(n, ffi.cast('float*', a), res) then
+ return res, a
+ end
+
+ return nil
+end
+
+exports.ffi_to_table = ffi_to_table
+exports.table_to_ffi = table_to_ffi
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_ffi/spf.lua b/lualib/lua_ffi/spf.lua
new file mode 100644
index 0000000..0f982f2
--- /dev/null
+++ b/lualib/lua_ffi/spf.lua
@@ -0,0 +1,143 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_ffi/spf
+-- This module contains ffi interfaces to SPF
+--]]
+
+local ffi = require 'ffi'
+
+ffi.cdef [[
+enum spf_mech_e {
+ SPF_FAIL,
+ SPF_SOFT_FAIL,
+ SPF_PASS,
+ SPF_NEUTRAL
+};
+static const unsigned RSPAMD_SPF_FLAG_IPV6 = (1 << 0);
+static const unsigned RSPAMD_SPF_FLAG_IPV4 = (1 << 1);
+static const unsigned RSPAMD_SPF_FLAG_ANY = (1 << 3);
+struct spf_addr {
+ unsigned char addr6[16];
+ unsigned char addr4[4];
+ union {
+ struct {
+ uint16_t mask_v4;
+ uint16_t mask_v6;
+ } dual;
+ uint32_t idx;
+ } m;
+ unsigned flags;
+ enum spf_mech_e mech;
+ char *spf_string;
+ struct spf_addr *prev, *next;
+};
+
+struct spf_resolved {
+ char *domain;
+ unsigned ttl;
+ int temp_failed;
+ int na;
+ int perm_failed;
+ uint64_t digest;
+ struct GArray *elts;
+ struct ref_entry_s ref;
+};
+
+typedef void (*spf_cb_t)(struct spf_resolved *record,
+ struct rspamd_task *task, void *data);
+struct rspamd_task;
+int rspamd_spf_resolve(struct rspamd_task *task, spf_cb_t callback,
+ void *cbdata);
+const char * rspamd_spf_get_domain (struct rspamd_task *task);
+struct spf_resolved * spf_record_ref (struct spf_resolved *rec);
+void spf_record_unref (struct spf_resolved *rec);
+char * spf_addr_mask_to_string (struct spf_addr *addr);
+struct spf_addr * spf_addr_match_task (struct rspamd_task *task, struct spf_resolved *rec);
+]]
+
+local function convert_mech(mech)
+ if mech == ffi.C.SPF_FAIL then
+ return 'fail'
+ elseif mech == ffi.C.SPF_SOFT_FAIL then
+ return 'softfail'
+ elseif mech == ffi.C.SPF_PASS then
+ return 'pass'
+ elseif mech == ffi.C.SPF_NEUTRAL then
+ return 'neutral'
+ end
+end
+
+local NULL = ffi.new 'void*'
+
+local function spf_addr_tolua(ffi_spf_addr)
+ local ipstr = ffi.C.spf_addr_mask_to_string(ffi_spf_addr)
+ local ret = {
+ res = convert_mech(ffi_spf_addr.mech),
+ ipnet = ffi.string(ipstr),
+ }
+
+ if ffi_spf_addr.spf_string ~= NULL then
+ ret.spf_str = ffi.string(ffi_spf_addr.spf_string)
+ end
+
+ ffi.C.g_free(ipstr)
+ return ret
+end
+
+local function spf_resolve(task, cb)
+ local function spf_cb(rec, _, _)
+ if not rec then
+ cb(false, 'record is empty')
+ else
+ local nelts = rec.elts.len
+ local elts = ffi.cast("struct spf_addr *", rec.elts.data)
+ local res = {
+ addrs = {}
+ }
+ local digstr = ffi.new("char[64]")
+ ffi.C.rspamd_snprintf(digstr, 64, "0x%xuL", rec.digest)
+ res.digest = ffi.string(digstr)
+ for i = 1, nelts do
+ res.addrs[i] = spf_addr_tolua(elts[i - 1])
+ end
+
+ local matched = ffi.C.spf_addr_match_task(task:topointer(), rec)
+
+ if matched ~= NULL then
+ cb(true, res, spf_addr_tolua(matched))
+ else
+ cb(true, res, nil)
+ end
+ end
+ end
+
+ local ret = ffi.C.rspamd_spf_resolve(task:topointer(), spf_cb, nil)
+
+ if not ret then
+ cb(false, 'cannot perform resolving')
+ end
+end
+
+local function spf_unref(rec)
+ ffi.C.spf_record_unref(rec)
+end
+
+return {
+ spf_resolve = spf_resolve,
+ spf_unref = spf_unref
+} \ No newline at end of file
diff --git a/lualib/lua_fuzzy.lua b/lualib/lua_fuzzy.lua
new file mode 100644
index 0000000..986d1a0
--- /dev/null
+++ b/lualib/lua_fuzzy.lua
@@ -0,0 +1,355 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_fuzzy
+-- This module contains helper functions for supporting fuzzy check module
+--]]
+
+
+local N = "lua_fuzzy"
+local lua_util = require "lua_util"
+local rspamd_regexp = require "rspamd_regexp"
+local fun = require "fun"
+local rspamd_logger = require "rspamd_logger"
+local ts = require("tableshape").types
+
+-- Filled by C code, indexed by number in this table
+local rules = {}
+
+-- Pre-defined rules options
+local policies = {
+ recommended = {
+ min_bytes = 1024,
+ min_height = 500,
+ min_width = 500,
+ min_length = 64,
+ text_multiplier = 4.0, -- divide min_bytes by 4 for texts
+ mime_types = { "application/*" },
+ scan_archives = true,
+ short_text_direct_hash = true,
+ text_shingles = true,
+ skip_images = false,
+ }
+}
+
+local default_policy = policies.recommended
+
+local schema_fields = {
+ min_bytes = ts.number + ts.string / tonumber,
+ min_height = ts.number + ts.string / tonumber,
+ min_width = ts.number + ts.string / tonumber,
+ min_length = ts.number + ts.string / tonumber,
+ text_multiplier = ts.number,
+ mime_types = ts.array_of(ts.string),
+ scan_archives = ts.boolean,
+ short_text_direct_hash = ts.boolean,
+ text_shingles = ts.boolean,
+ skip_images = ts.boolean,
+}
+local policy_schema = ts.shape(schema_fields)
+
+local policy_schema_open = ts.shape(schema_fields, {
+ open = true,
+})
+
+local exports = {}
+
+
+--[[[
+-- @function lua_fuzzy.register_policy(name, policy)
+-- Adds a new policy with name `name`. Must be valid, checked using policy_schema
+--]]
+exports.register_policy = function(name, policy)
+ if policies[name] then
+ rspamd_logger.warnx(rspamd_config, "overriding policy %s", name)
+ end
+
+ local parsed_policy, err = policy_schema:transform(policy)
+
+ if not parsed_policy then
+ rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule policy %s: %s',
+ name, err)
+
+ return
+ else
+ policies.name = parsed_policy
+ end
+end
+
+--[[[
+-- @function lua_fuzzy.process_rule(rule)
+-- Processes fuzzy rule (applying policies or defaults if needed). Returns policy id
+--]]
+exports.process_rule = function(rule)
+ local processed_rule = lua_util.shallowcopy(rule)
+ local policy = default_policy
+
+ if processed_rule.policy then
+ policy = policies[processed_rule.policy]
+ end
+
+ if policy then
+ processed_rule = lua_util.override_defaults(policy, processed_rule)
+
+ local parsed_policy, err = policy_schema_open:transform(processed_rule)
+
+ if not parsed_policy then
+ rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule default fields: %s', err)
+ else
+ processed_rule = parsed_policy
+ end
+ else
+ rspamd_logger.warnx(rspamd_config, "unknown policy %s", processed_rule.policy)
+ end
+
+ if processed_rule.mime_types then
+ processed_rule.mime_types = fun.totable(fun.map(function(gl)
+ return rspamd_regexp.import_glob(gl, 'i')
+ end, processed_rule.mime_types))
+ end
+
+ table.insert(rules, processed_rule)
+ return #rules
+end
+
+local function check_length(task, part, rule)
+ local bytes = part:get_length()
+ local length_ok = bytes > 0
+
+ local id = part:get_id()
+ lua_util.debugm(N, task, 'check size of part %s', id)
+
+ if length_ok and rule.min_bytes > 0 then
+
+ local adjusted_bytes = bytes
+
+ if part:is_text() then
+ -- Fuzzy plugin uses stripped utf content to get an exact hash, that
+ -- corresponds to `get_content_oneline()`
+ -- However, in the case of empty parts this method returns `nil`, so extra
+ -- sanity check is required.
+ bytes = #(part:get_text():get_content_oneline() or '')
+
+ -- Short hashing algorithm also use subject unless explicitly denied
+ if not rule.no_subject then
+ local subject = task:get_subject() or ''
+ bytes = bytes + #subject
+ end
+
+ if rule.text_multiplier then
+ adjusted_bytes = bytes * rule.text_multiplier
+ end
+ end
+
+ if rule.min_bytes > adjusted_bytes then
+ lua_util.debugm(N, task, 'skip part of length %s (%s adjusted) ' ..
+ 'as it has less than %s bytes',
+ bytes, adjusted_bytes, rule.min_bytes)
+ length_ok = false
+ else
+ lua_util.debugm(N, task, 'allow part of length %s (%s adjusted)',
+ bytes, adjusted_bytes, rule.min_bytes)
+ end
+ else
+ lua_util.debugm(N, task, 'allow part %s, no length limits', id)
+ end
+
+ return length_ok
+end
+
+local function check_text_part(task, part, rule, text)
+ local allow_direct, allow_shingles = false, false
+
+ local id = part:get_id()
+ lua_util.debugm(N, task, 'check text part %s', id)
+ local wcnt = text:get_words_count()
+
+ if rule.text_shingles then
+ -- Check number of words
+ local min_words = rule.min_length or 0
+ if min_words < 32 then
+ min_words = 32 -- Minimum for shingles
+ end
+ if wcnt < min_words then
+ lua_util.debugm(N, task, 'text has less than %s words: %s; disable shingles',
+ rule.min_length, wcnt)
+ allow_shingles = false
+ else
+ lua_util.debugm(N, task, 'allow shingles in text %s, %s words',
+ id, wcnt)
+ allow_shingles = true
+ end
+
+ if not rule.short_text_direct_hash and not allow_shingles then
+ allow_direct = false
+ else
+ if not allow_shingles then
+ lua_util.debugm(N, task,
+ 'allow direct hash for short text %s, %s words',
+ id, wcnt)
+ allow_direct = check_length(task, part, rule)
+ else
+ allow_direct = wcnt > 0
+ end
+ end
+ else
+ lua_util.debugm(N, task,
+ 'disable shingles in text %s', id)
+ allow_direct = check_length(task, part, rule)
+ end
+
+ return allow_direct, allow_shingles
+end
+
+--local function has_sane_text_parts(task)
+-- local text_parts = task:get_text_parts() or {}
+-- return fun.any(function(tp) return tp:get_words_count() > 32 end, text_parts)
+--end
+
+local function check_image_part(task, part, rule, image)
+ if rule.skip_images then
+ lua_util.debugm(N, task, 'skip image part as images are disabled')
+ return false, false
+ end
+
+ local id = part:get_id()
+ lua_util.debugm(N, task, 'check image part %s', id)
+
+ if rule.min_width > 0 or rule.min_height > 0 then
+ -- Check dimensions
+ local min_width = rule.min_width or rule.min_height
+ local min_height = rule.min_height or rule.min_width
+ local height = image:get_height()
+ local width = image:get_width()
+
+ if height and width then
+ if height < min_height or width < min_width then
+ lua_util.debugm(N, task, 'skip image part %s as it does not meet minimum sizes: %sx%s < %sx%s',
+ id, width, height, min_width, min_height)
+ return false, false
+ else
+ lua_util.debugm(N, task, 'allow image part %s: %sx%s',
+ id, width, height)
+ end
+ end
+ end
+
+ return check_length(task, part, rule), false
+end
+
+local function mime_types_check(task, part, rule)
+ local t, st = part:get_type()
+
+ if not t then
+ return false, false
+ end
+
+ local ct = string.format('%s/%s', t, st)
+
+ local detected_ct
+ t, st = part:get_detected_type()
+ if t then
+ detected_ct = string.format('%s/%s', t, st)
+ else
+ detected_ct = ct
+ end
+
+ local id = part:get_id()
+ lua_util.debugm(N, task, 'check binary part %s: %s', id, ct)
+
+ -- For bad mime parts we implicitly enable fuzzy check
+ local mime_trace = (task:get_symbol('MIME_TRACE') or {})[1]
+ local opts = {}
+
+ if mime_trace then
+ opts = mime_trace.options or opts
+ end
+ opts = fun.tomap(fun.map(function(opt)
+ local elts = lua_util.str_split(opt, ':')
+ return elts[1], elts[2]
+ end, opts))
+
+ if opts[id] and opts[id] == '-' then
+ lua_util.debugm(N, task, 'explicitly check binary part %s: bad mime type %s', id, ct)
+ return check_length(task, part, rule), false
+ end
+
+ if rule.mime_types then
+
+ if fun.any(function(gl_re)
+ if gl_re:match(ct) or (detected_ct and gl_re:match(detected_ct)) then
+ return true
+ else
+ return false
+ end
+ end, rule.mime_types) then
+ lua_util.debugm(N, task, 'found mime type match for part %s: %s (%s detected)',
+ id, ct, detected_ct)
+ return check_length(task, part, rule), false
+ end
+
+ return false, false
+ end
+
+ return false, false
+end
+
+exports.check_mime_part = function(task, part, rule_id)
+ local rule = rules[rule_id]
+
+ if not rule then
+ rspamd_logger.errx(task, 'cannot find rule with id %s', rule_id)
+
+ return false, false
+ end
+
+ if part:is_text() then
+ return check_text_part(task, part, rule, part:get_text())
+ end
+
+ if part:is_image() then
+ return check_image_part(task, part, rule, part:get_image())
+ end
+
+ if part:is_archive() and rule.scan_archives then
+ -- Always send archives
+ lua_util.debugm(N, task, 'check archive part %s', part:get_id())
+
+ return true, false
+ end
+
+ if part:is_specific() then
+ local sp = part:get_specific()
+
+ if type(sp) == 'table' and sp.fuzzy_hashes then
+ lua_util.debugm(N, task, 'check specific part %s', part:get_id())
+ return true, false
+ end
+ end
+
+ if part:is_attachment() then
+ return mime_types_check(task, part, rule)
+ end
+
+ return false, false
+end
+
+exports.cleanup_rules = function()
+ rules = {}
+end
+
+return exports
diff --git a/lualib/lua_lexer.lua b/lualib/lua_lexer.lua
new file mode 100644
index 0000000..54bbd7c
--- /dev/null
+++ b/lualib/lua_lexer.lua
@@ -0,0 +1,163 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[ Lua LPEG grammar based on https://github.com/xolox/lua-lxsh/ ]]
+
+
+local lpeg = require "lpeg"
+
+local P = lpeg.P
+local R = lpeg.R
+local S = lpeg.S
+local D = R '09' -- Digits
+local I = R('AZ', 'az', '\127\255') + '_' -- Identifiers
+local B = -(I + D) -- Word boundary
+local EOS = -lpeg.P(1) -- end of string
+
+-- Pattern for long strings and long comments.
+local longstring = #(P '[[' + (P '[' * P '=' ^ 0 * '[')) * P(function(input, index)
+ local level = input:match('^%[(=*)%[', index)
+ if level then
+ local _, last = input:find(']' .. level .. ']', index, true)
+ if last then
+ return last + 1
+ end
+ end
+end)
+
+-- String literals.
+local singlequoted = P "'" * ((1 - S "'\r\n\f\\") + (P '\\' * 1)) ^ 0 * "'"
+local doublequoted = P '"' * ((1 - S '"\r\n\f\\') + (P '\\' * 1)) ^ 0 * '"'
+
+-- Comments.
+local eol = P '\r\n' + '\n'
+local line = (1 - S '\r\n\f') ^ 0 * eol ^ -1
+local singleline = P '--' * line
+local multiline = P '--' * longstring
+
+-- Numbers.
+local sign = S '+-' ^ -1
+local decimal = D ^ 1
+local hexadecimal = P '0' * S 'xX' * R('09', 'AF', 'af') ^ 1
+local float = D ^ 1 * P '.' * D ^ 0 + P '.' * D ^ 1
+local maybeexp = (float + decimal) * (S 'eE' * sign * D ^ 1) ^ -1
+
+local function compile_keywords(keywords)
+ local list = {}
+ for word in keywords:gmatch('%S+') do
+ list[#list + 1] = word
+ end
+ -- Sort by length
+ table.sort(list, function(a, b)
+ return #a > #b
+ end)
+
+ local pattern
+ for _, word in ipairs(list) do
+ local p = lpeg.P(word)
+ pattern = pattern and (pattern + p) or p
+ end
+
+ local AB = B + EOS -- ending boundary
+ return pattern * AB
+end
+
+-- Identifiers
+local ident = I * (I + D) ^ 0
+local expr = ('.' * ident) ^ 0
+
+local patterns = {
+ { 'whitespace', S '\r\n\f\t\v ' ^ 1 },
+ { 'constant', (P 'true' + 'false' + 'nil') * B },
+ { 'string', singlequoted + doublequoted + longstring },
+ { 'comment', multiline + singleline },
+ { 'number', hexadecimal + maybeexp },
+ { 'operator', P 'not' + '...' + 'and' + '..' + '~=' + '==' + '>=' + '<='
+ + 'or' + S ']{=>^[<;)*(%}+-:,/.#' },
+ { 'keyword', compile_keywords([[
+ break do else elseif end for function if in local repeat return then until while
+ ]]) },
+ { 'identifier', lpeg.Cmt(ident,
+ function(input, index)
+ return expr:match(input, index)
+ end)
+ },
+ { 'error', 1 },
+}
+
+local compiled
+
+local function compile_patterns()
+ if not compiled then
+ local function process(elt)
+ local n, grammar = elt[1], elt[2]
+ return lpeg.Cc(n) * lpeg.P(grammar) * lpeg.Cp()
+ end
+ local any = process(patterns[1])
+ for i = 2, #patterns do
+ any = any + process(patterns[i])
+ end
+ compiled = any
+ end
+
+ return compiled
+end
+
+local function sync(token, lnum, cnum)
+ local lastidx
+ lnum, cnum = lnum or 1, cnum or 1
+ if token:find '\n' then
+ for i in token:gmatch '()\n' do
+ lnum = lnum + 1
+ lastidx = i
+ end
+ cnum = #token - lastidx + 1
+ else
+ cnum = cnum + #token
+ end
+ return lnum, cnum
+end
+
+local exports = {}
+
+exports.gmatch = function(input)
+ local parser = compile_patterns()
+ local index, lnum, cnum = 1, 1, 1
+
+ return function()
+ local kind, after = parser:match(input, index)
+ if kind and after then
+ local text = input:sub(index, after - 1)
+ local oldlnum, oldcnum = lnum, cnum
+ index = after
+ lnum, cnum = sync(text, lnum, cnum)
+ return kind, text, oldlnum, oldcnum
+ end
+ end
+end
+
+exports.lex_to_table = function(input)
+ local out = {}
+
+ for kind, text, lnum, cnum in exports.gmatch(input) do
+ out[#out + 1] = { kind, text, lnum, cnum }
+ end
+
+ return out
+end
+
+return exports
+
diff --git a/lualib/lua_magic/heuristics.lua b/lualib/lua_magic/heuristics.lua
new file mode 100644
index 0000000..b8a1b41
--- /dev/null
+++ b/lualib/lua_magic/heuristics.lua
@@ -0,0 +1,605 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_magic/heuristics
+-- This module contains heuristics for some specific cases
+--]]
+
+local rspamd_trie = require "rspamd_trie"
+local rspamd_util = require "rspamd_util"
+local lua_util = require "lua_util"
+local bit = require "bit"
+local fun = require "fun"
+
+local N = "lua_magic"
+local msoffice_trie
+local msoffice_patterns = {
+ doc = { [[WordDocument]] },
+ xls = { [[Workbook]], [[Book]] },
+ ppt = { [[PowerPoint Document]], [[Current User]] },
+ vsd = { [[VisioDocument]] },
+}
+local msoffice_trie_clsid
+local msoffice_clsids = {
+ doc = { [[0609020000000000c000000000000046]] },
+ xls = { [[1008020000000000c000000000000046]], [[2008020000000000c000000000000046]] },
+ ppt = { [[108d81649b4fcf1186ea00aa00b929e8]] },
+ msg = { [[46f0060000000000c000000000000046]], [[0b0d020000000000c000000000000046]] },
+ msi = { [[84100c0000000000c000000000000046]] },
+}
+local zip_trie
+local zip_patterns = {
+ -- https://lists.oasis-open.org/archives/office/200505/msg00006.html
+ odt = {
+ [[mimetypeapplication/vnd\.oasis\.opendocument\.text]],
+ [[mimetypeapplication/vnd\.oasis\.opendocument\.image]],
+ [[mimetypeapplication/vnd\.oasis\.opendocument\.graphic]]
+ },
+ ods = {
+ [[mimetypeapplication/vnd\.oasis\.opendocument\.spreadsheet]],
+ [[mimetypeapplication/vnd\.oasis\.opendocument\.formula]],
+ [[mimetypeapplication/vnd\.oasis\.opendocument\.chart]]
+ },
+ odp = { [[mimetypeapplication/vnd\.oasis\.opendocument\.presentation]] },
+ epub = { [[epub\+zip]] },
+ asice = { [[mimetypeapplication/vnd\.etsi\.asic-e\+zipPK]] },
+ asics = { [[mimetypeapplication/vnd\.etsi\.asic-s\+zipPK]] },
+}
+
+local txt_trie
+local txt_patterns = {
+ html = {
+ { [=[(?i)<html[\s>]]=], 32 },
+ { [[(?i)<script\b]], 20 }, -- Commonly used by spammers
+ { [[<script\s+type="text\/javascript">]], 31 }, -- Another spammy pattern
+ { [[(?i)<\!DOCTYPE HTML\b]], 33 },
+ { [[(?i)<body\b]], 20 },
+ { [[(?i)<table\b]], 20 },
+ { [[(?i)<a\s]], 10 },
+ { [[(?i)<p\b]], 10 },
+ { [[(?i)<div\b]], 10 },
+ { [[(?i)<span\b]], 10 },
+ },
+ csv = {
+ { [[(?:[-a-zA-Z0-9_]+\s*,){2,}(?:[-a-zA-Z0-9_]+,?[ ]*[\r\n])]], 20 }
+ },
+ ics = {
+ { [[^BEGIN:VCALENDAR\r?\n]], 40 },
+ },
+ vcf = {
+ { [[^BEGIN:VCARD\r?\n]], 40 },
+ },
+ xml = {
+ { [[<\?xml\b.+\?>]], 31 },
+ }
+}
+
+-- Used to match pattern index and extension
+local msoffice_clsid_indexes = {}
+local msoffice_patterns_indexes = {}
+local zip_patterns_indexes = {}
+local txt_patterns_indexes = {}
+
+local exports = {}
+
+local function compile_tries()
+ local default_compile_flags = bit.bor(rspamd_trie.flags.re,
+ rspamd_trie.flags.dot_all,
+ rspamd_trie.flags.single_match,
+ rspamd_trie.flags.no_start)
+ local function compile_pats(patterns, indexes, transform_func, compile_flags)
+ local strs = {}
+ for ext, pats in pairs(patterns) do
+ for _, pat in ipairs(pats) do
+ -- These are utf16 strings in fact...
+ strs[#strs + 1] = transform_func(pat)
+ indexes[#indexes + 1] = { ext, pat }
+ end
+ end
+
+ return rspamd_trie.create(strs, compile_flags or default_compile_flags)
+ end
+
+ if not msoffice_trie then
+ -- Directory names
+ local function msoffice_pattern_transform(pat)
+ return '^' ..
+ table.concat(
+ fun.totable(
+ fun.map(function(c)
+ return c .. [[\x{00}]]
+ end,
+ fun.iter(pat))))
+ end
+ local function msoffice_clsid_transform(pat)
+ local hex_table = {}
+ for i = 1, #pat, 2 do
+ local subc = pat:sub(i, i + 1)
+ hex_table[#hex_table + 1] = string.format('\\x{%s}', subc)
+ end
+
+ return '^' .. table.concat(hex_table) .. '$'
+ end
+ -- Directory entries
+ msoffice_trie = compile_pats(msoffice_patterns, msoffice_patterns_indexes,
+ msoffice_pattern_transform)
+ -- Clsids
+ msoffice_trie_clsid = compile_pats(msoffice_clsids, msoffice_clsid_indexes,
+ msoffice_clsid_transform)
+ -- Misc zip patterns at the initial fragment
+ zip_trie = compile_pats(zip_patterns, zip_patterns_indexes,
+ function(pat)
+ return pat
+ end)
+ -- Text patterns at the initial fragment
+ txt_trie = compile_pats(txt_patterns, txt_patterns_indexes,
+ function(pat_tbl)
+ return pat_tbl[1]
+ end,
+ bit.bor(rspamd_trie.flags.re,
+ rspamd_trie.flags.dot_all,
+ rspamd_trie.flags.no_start))
+ end
+end
+
+-- Call immediately on require
+compile_tries()
+
+local function detect_ole_format(input, log_obj, _, part)
+ local inplen = #input
+ if inplen < 0x31 + 4 then
+ lua_util.debugm(N, log_obj, "short length: %s", inplen)
+ return nil
+ end
+
+ local bom, sec_size = rspamd_util.unpack('<I2<I2', input:span(29, 4))
+ if bom == 0xFFFE then
+ bom = '<'
+ else
+ lua_util.debugm(N, log_obj, "bom file!: %s", bom)
+ bom = '>';
+ sec_size = bit.bswap(sec_size)
+ end
+
+ if sec_size < 7 or sec_size > 31 then
+ lua_util.debugm(N, log_obj, "bad sec_size: %s", sec_size)
+ return nil
+ end
+
+ sec_size = 2 ^ sec_size
+
+ -- SecID of first sector of the directory stream
+ local directory_offset = (rspamd_util.unpack(bom .. 'I4', input:span(0x31, 4)))
+ * sec_size + 512 + 1
+ lua_util.debugm(N, log_obj, "directory: %s", directory_offset)
+
+ if inplen < directory_offset then
+ lua_util.debugm(N, log_obj, "short length: %s", inplen)
+ return nil
+ end
+
+ local function process_dir_entry(offset)
+ local dtype = input:byte(offset + 66)
+ lua_util.debugm(N, log_obj, "dtype: %s, offset: %s", dtype, offset)
+
+ if dtype then
+ if dtype == 5 then
+ -- Extract clsid
+ local matches = msoffice_trie_clsid:match(input:span(offset + 80, 16))
+ if matches then
+ for n, _ in pairs(matches) do
+ if msoffice_clsid_indexes[n] then
+ lua_util.debugm(N, log_obj, "found valid clsid for %s",
+ msoffice_clsid_indexes[n][1])
+ return true, msoffice_clsid_indexes[n][1]
+ end
+ end
+ end
+ return true, nil
+ elseif dtype == 2 then
+ local matches = msoffice_trie:match(input:span(offset, 64))
+ if matches then
+ for n, _ in pairs(matches) do
+ if msoffice_patterns_indexes[n] then
+ return true, msoffice_patterns_indexes[n][1]
+ end
+ end
+ end
+ return true, nil
+ elseif dtype >= 0 and dtype < 5 then
+ -- Bad type
+ return true, nil
+ end
+ end
+
+ return false, nil
+ end
+
+ repeat
+ local res, ext = process_dir_entry(directory_offset)
+
+ if res and ext then
+ return ext, 60
+ end
+
+ if not res then
+ break
+ end
+
+ directory_offset = directory_offset + 128
+ until directory_offset >= inplen
+end
+
+exports.ole_format_heuristic = detect_ole_format
+
+local function process_top_detected(res)
+ local extensions = lua_util.keys(res)
+
+ if #extensions > 0 then
+ table.sort(extensions, function(ex1, ex2)
+ return res[ex1] > res[ex2]
+ end)
+
+ return extensions[1], res[extensions[1]]
+ end
+
+ return nil
+end
+
+local function detect_archive_flaw(part, arch, log_obj, _)
+ local arch_type = arch:get_type()
+ local res = {
+ docx = 0,
+ xlsx = 0,
+ pptx = 0,
+ jar = 0,
+ odt = 0,
+ odp = 0,
+ ods = 0,
+ apk = 0,
+ } -- ext + confidence pairs
+
+ -- General msoffice patterns
+ local function add_msoffice_confidence(incr)
+ res.docx = res.docx + incr
+ res.xlsx = res.xlsx + incr
+ res.pptx = res.pptx + incr
+ end
+
+ if arch_type == 'zip' then
+ -- Find specific files/folders in zip file
+ local files = arch:get_files(100) or {}
+ for _, file in ipairs(files) do
+ if file == '[Content_Types].xml' then
+ add_msoffice_confidence(10)
+ elseif file:sub(1, 3) == 'xl/' then
+ res.xlsx = res.xlsx + 30
+ elseif file:sub(1, 5) == 'word/' then
+ res.docx = res.docx + 30
+ elseif file:sub(1, 4) == 'ppt/' then
+ res.pptx = res.pptx + 30
+ elseif file == 'META-INF/MANIFEST.MF' then
+ res.jar = res.jar + 40
+ elseif file == 'AndroidManifest.xml' then
+ res.apk = res.apk + 60
+ end
+ end
+
+ local ext, weight = process_top_detected(res)
+
+ if weight >= 40 then
+ return ext, weight
+ end
+
+ -- Apply misc Zip detection logic
+ local content = part:get_content()
+
+ if #content > 128 then
+ local start_span = content:span(1, 128)
+
+ local matches = zip_trie:match(start_span)
+ if matches then
+ for n, _ in pairs(matches) do
+ if zip_patterns_indexes[n] then
+ lua_util.debugm(N, log_obj, "found zip pattern for %s",
+ zip_patterns_indexes[n][1])
+ return zip_patterns_indexes[n][1], 40
+ end
+ end
+ end
+ end
+ end
+
+ return arch_type:lower(), 40
+end
+
+local csv_grammar
+-- Returns a grammar that will count commas
+local function get_csv_grammar()
+ if not csv_grammar then
+ local lpeg = require 'lpeg'
+
+ local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P '""' / '"') ^ 0) * '"' +
+ lpeg.C((1 - lpeg.S ',\n"') ^ 0)
+
+ csv_grammar = lpeg.Cf(lpeg.Cc(0) * field * lpeg.P((lpeg.P(',') +
+ lpeg.P('\t')) * field) ^ 1 * (lpeg.S '\r\n' + -1),
+ function(acc)
+ return acc + 1
+ end)
+ end
+
+ return csv_grammar
+end
+local function validate_csv(part, content, log_obj)
+ local max_chunk = 32768
+ local chunk = content:sub(1, max_chunk)
+
+ local expected_commas
+ local matched_lines = 0
+ local max_matched_lines = 10
+
+ lua_util.debugm(N, log_obj, "check for csv pattern")
+
+ for s in chunk:lines() do
+ local ncommas = get_csv_grammar():match(s)
+
+ if not ncommas then
+ lua_util.debugm(N, log_obj, "not a csv line at line number %s",
+ matched_lines)
+ return false
+ end
+
+ if expected_commas and ncommas ~= expected_commas then
+ -- Mismatched commas
+ lua_util.debugm(N, log_obj, "missmatched commas on line %s: %s != %s",
+ matched_lines, ncommas, expected_commas)
+ return false
+ elseif not expected_commas then
+ if ncommas == 0 then
+ lua_util.debugm(N, log_obj, "no commas in the first line")
+ return false
+ end
+ expected_commas = ncommas
+ end
+
+ matched_lines = matched_lines + 1
+
+ if matched_lines > max_matched_lines then
+ break
+ end
+ end
+
+ lua_util.debugm(N, log_obj, "csv content is sane: %s fields; %s lines checked",
+ expected_commas, matched_lines)
+
+ return true
+end
+
+exports.mime_part_heuristic = function(part, log_obj, _)
+ if part:is_archive() then
+ local arch = part:get_archive()
+ return detect_archive_flaw(part, arch, log_obj)
+ end
+
+ return nil
+end
+
+exports.text_part_heuristic = function(part, log_obj, _)
+ -- We get some span of data and check it
+ local function is_span_text(span)
+ -- We examine 8 bit content, and we assume it might be localized text
+ -- if it has more than 3 subsequent 8 bit characters
+ local function rough_8bit_check(bytes, idx, remain, len)
+ local b = bytes[idx]
+ local n8bit = 0
+
+ while b >= 127 and idx < len do
+ -- utf8 part
+ if bit.band(b, 0xe0) == 0xc0 and remain > 1 and
+ bit.band(bytes[idx + 1], 0xc0) == 0x80 then
+ return true, 1
+ elseif bit.band(b, 0xf0) == 0xe0 and remain > 2 and
+ bit.band(bytes[idx + 1], 0xc0) == 0x80 and
+ bit.band(bytes[idx + 2], 0xc0) == 0x80 then
+ return true, 2
+ elseif bit.band(b, 0xf8) == 0xf0 and remain > 3 and
+ bit.band(bytes[idx + 1], 0xc0) == 0x80 and
+ bit.band(bytes[idx + 2], 0xc0) == 0x80 and
+ bit.band(bytes[idx + 3], 0xc0) == 0x80 then
+ return true, 3
+ end
+
+ n8bit = n8bit + 1
+ idx = idx + 1
+ b = bytes[idx]
+ remain = remain - 1
+ end
+
+ if n8bit >= 3 then
+ return true, n8bit
+ end
+
+ return false, 0
+ end
+
+ -- Convert to string as LuaJIT can optimise string.sub (and fun.iter) but not C calls
+ local tlen = #span
+ local non_printable = 0
+ local bytes = span:bytes()
+ local i = 1
+ repeat
+ local b = bytes[i]
+
+ if (b < 0x20) and not (b == 0x0d or b == 0x0a or b == 0x09) then
+ non_printable = non_printable + 1
+ elseif b >= 127 then
+ local c, nskip = rough_8bit_check(bytes, i, tlen - i, tlen)
+
+ if not c then
+ non_printable = non_printable + 1
+ else
+ i = i + nskip
+ end
+ end
+ i = i + 1
+ until i > tlen
+
+ lua_util.debugm(N, log_obj, "text part check: %s printable, %s non-printable, %s total",
+ tlen - non_printable, non_printable, tlen)
+ if non_printable / tlen > 0.0078125 then
+ return false
+ end
+
+ return true
+ end
+
+ local parent = part:get_parent()
+
+ if parent then
+ local parent_type, parent_subtype = parent:get_type()
+
+ if parent_type == 'multipart' and parent_subtype == 'encrypted' then
+ -- Skip text heuristics for encrypted parts
+ lua_util.debugm(N, log_obj, "text part check: parent is encrypted, not a text part")
+
+ return false
+ end
+ end
+
+ local content = part:get_content()
+ local mtype, msubtype = part:get_type()
+ local clen = #content
+ local is_text
+
+ if clen > 0 then
+ if clen > 80 * 3 then
+ -- Use chunks
+ is_text = is_span_text(content:span(1, 160)) and is_span_text(content:span(clen - 80, 80))
+ else
+ is_text = is_span_text(content)
+ end
+
+ if is_text and mtype ~= 'message' then
+ -- Try patterns
+ local span_len = math.min(4096, clen)
+ local start_span = content:span(1, span_len)
+ local matches = txt_trie:match(start_span)
+ local res = {}
+ local fname = part:get_filename()
+
+ if matches then
+ -- Require at least 2 occurrences of those patterns
+ for n, positions in pairs(matches) do
+ local ext, weight = txt_patterns_indexes[n][1], txt_patterns_indexes[n][2][2]
+ if ext then
+ res[ext] = (res[ext] or 0) + weight * #positions
+ lua_util.debugm(N, log_obj, "found txt pattern for %s: %s, total: %s; %s/%s announced",
+ ext, weight * #positions, res[ext], mtype, msubtype)
+ end
+ end
+
+ if res.html and res.html >= 40 then
+ -- HTML has priority over something like js...
+ return 'html', res.html
+ end
+
+ local ext, weight = process_top_detected(res)
+
+ if weight then
+ if weight >= 40 then
+ -- Extra validation for csv extension
+ if ext ~= 'csv' or validate_csv(part, content, log_obj) then
+ return ext, weight
+ end
+ elseif fname and weight >= 20 then
+ return ext, weight
+ end
+ end
+ end
+
+ -- Content type stuff
+ if (mtype == 'text' or mtype == 'application') and
+ (msubtype == 'html' or msubtype == 'xhtml+xml') then
+ return 'html', 21
+ end
+
+ if msubtype:lower() == 'csv' then
+ if validate_csv(part, content, log_obj) then
+ return 'csv', 40
+ end
+ end
+
+ -- Extension stuff
+ local function has_extension(file, ext)
+ local ext_len = ext:len()
+ return file:len() > ext_len + 1
+ and file:sub(-ext_len):lower() == ext
+ and file:sub(-ext_len - 1, -ext_len - 1) == '.'
+ end
+
+ if fname and (has_extension(fname, 'htm') or has_extension(fname, 'html')) then
+ return 'html', 21
+ end
+
+ if mtype ~= 'text' then
+ -- Do not treat non text patterns as text
+ return nil
+ end
+
+ return 'txt', 40
+ end
+ end
+end
+
+exports.pdf_format_heuristic = function(input, log_obj, pos, part)
+ local weight = 10
+ local ext = string.match(part:get_filename() or '', '%.([^.]+)$')
+ -- If we found a pattern at the beginning
+ if pos <= 10 then
+ weight = weight + 30
+ end
+ -- If the announced extension is `pdf`
+ if ext and ext:lower() == 'pdf' then
+ weight = weight + 30
+ end
+
+ return 'pdf', weight
+end
+
+exports.pe_part_heuristic = function(input, log_obj, pos, part)
+ if not input then
+ return
+ end
+
+ -- pe header should start at the offset that is placed in msdos header at position 60..64
+ local pe_ptr_bin = input:sub(60, 64)
+ if #pe_ptr_bin ~= 4 then
+ return
+ end
+
+ -- it is an LE 32 bit integer
+ local pe_ptr = rspamd_util.unpack("<I4", pe_ptr_bin)
+ -- if pe header magic matches the offset, it is definitely a PE file
+ if pe_ptr ~= pos then
+ return
+ end
+
+ return 'exe', 30
+end
+
+return exports
diff --git a/lualib/lua_magic/init.lua b/lualib/lua_magic/init.lua
new file mode 100644
index 0000000..38bfddb
--- /dev/null
+++ b/lualib/lua_magic/init.lua
@@ -0,0 +1,388 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_magic
+-- This module contains file types detection logic
+--]]
+
+local patterns = require "lua_magic/patterns"
+local types = require "lua_magic/types"
+local heuristics = require "lua_magic/heuristics"
+local fun = require "fun"
+local lua_util = require "lua_util"
+
+local rspamd_text = require "rspamd_text"
+local rspamd_trie = require "rspamd_trie"
+
+local N = "lua_magic"
+local exports = {}
+-- trie objects
+local compiled_patterns
+local compiled_short_patterns
+local compiled_tail_patterns
+-- {<str>, <match_object>, <pattern_object>} indexed by pattern number
+local processed_patterns = {}
+local short_patterns = {}
+local tail_patterns = {}
+
+local short_match_limit = 128
+local max_short_offset = -1
+local min_tail_offset = math.huge
+
+local function process_patterns(log_obj)
+ -- Add pattern to either short patterns or to normal patterns
+ local function add_processed(str, match, pattern)
+ if match.position and type(match.position) == 'number' then
+ if match.tail then
+ -- Tail pattern
+ tail_patterns[#tail_patterns + 1] = {
+ str, match, pattern
+ }
+ if min_tail_offset > match.tail then
+ min_tail_offset = match.tail
+ end
+
+ lua_util.debugm(N, log_obj, 'add tail pattern %s for ext %s',
+ str, pattern.ext)
+ elseif match.position < short_match_limit then
+ short_patterns[#short_patterns + 1] = {
+ str, match, pattern
+ }
+ if str:sub(1, 1) == '^' then
+ lua_util.debugm(N, log_obj, 'add head pattern %s for ext %s',
+ str, pattern.ext)
+ else
+ lua_util.debugm(N, log_obj, 'add short pattern %s for ext %s',
+ str, pattern.ext)
+ end
+
+ if max_short_offset < match.position then
+ max_short_offset = match.position
+ end
+ else
+ processed_patterns[#processed_patterns + 1] = {
+ str, match, pattern
+ }
+
+ lua_util.debugm(N, log_obj, 'add long pattern %s for ext %s',
+ str, pattern.ext)
+ end
+ else
+ processed_patterns[#processed_patterns + 1] = {
+ str, match, pattern
+ }
+
+ lua_util.debugm(N, log_obj, 'add long pattern %s for ext %s',
+ str, pattern.ext)
+ end
+ end
+
+ if not compiled_patterns then
+ for ext, pattern in pairs(patterns) do
+ assert(types[ext], 'not found type: ' .. ext)
+ pattern.ext = ext
+ for _, match in ipairs(pattern.matches) do
+ if match.string then
+ if match.relative_position and not match.position then
+ match.position = match.relative_position + #match.string
+
+ if match.relative_position == 0 then
+ if match.string:sub(1, 1) ~= '^' then
+ match.string = '^' .. match.string
+ end
+ end
+ end
+ add_processed(match.string, match, pattern)
+ elseif match.hex then
+ local hex_table = {}
+
+ for i = 1, #match.hex, 2 do
+ local subc = match.hex:sub(i, i + 1)
+ hex_table[#hex_table + 1] = string.format('\\x{%s}', subc)
+ end
+
+ if match.relative_position and not match.position then
+ match.position = match.relative_position + #match.hex / 2
+ end
+ if match.relative_position == 0 then
+ table.insert(hex_table, 1, '^')
+ end
+ add_processed(table.concat(hex_table), match, pattern)
+ end
+ end
+ end
+ local bit = require "bit"
+ local compile_flags = bit.bor(rspamd_trie.flags.re, rspamd_trie.flags.dot_all)
+ compile_flags = bit.bor(compile_flags, rspamd_trie.flags.single_match)
+ compile_flags = bit.bor(compile_flags, rspamd_trie.flags.no_start)
+ compiled_patterns = rspamd_trie.create(fun.totable(
+ fun.map(function(t)
+ return t[1]
+ end, processed_patterns)),
+ compile_flags
+ )
+ compiled_short_patterns = rspamd_trie.create(fun.totable(
+ fun.map(function(t)
+ return t[1]
+ end, short_patterns)),
+ compile_flags
+ )
+ compiled_tail_patterns = rspamd_trie.create(fun.totable(
+ fun.map(function(t)
+ return t[1]
+ end, tail_patterns)),
+ compile_flags
+ )
+
+ lua_util.debugm(N, log_obj,
+ 'compiled %s (%s short; %s long; %s tail) patterns',
+ #processed_patterns + #short_patterns + #tail_patterns,
+ #short_patterns, #processed_patterns, #tail_patterns)
+ end
+end
+
+process_patterns(rspamd_config)
+
+local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_obj, res, part)
+ local matches = trie:match(chunk)
+
+ local last = tlen
+
+ local function add_result(weight, ext)
+ if not res[ext] then
+ res[ext] = 0
+ end
+ if weight then
+ res[ext] = res[ext] + weight
+ else
+ res[ext] = res[ext] + 1
+ end
+
+ lua_util.debugm(N, log_obj, 'add pattern for %s, weight %s, total weight %s',
+ ext, weight, res[ext])
+ end
+
+ local function match_position(pos, expected)
+ local cmp = function(a, b)
+ return a == b
+ end
+ if type(expected) == 'table' then
+ -- Something like {'>', 0}
+ if expected[1] == '>' then
+ cmp = function(a, b)
+ return a > b
+ end
+ elseif expected[1] == '>=' then
+ cmp = function(a, b)
+ return a >= b
+ end
+ elseif expected[1] == '<' then
+ cmp = function(a, b)
+ return a < b
+ end
+ elseif expected[1] == '<=' then
+ cmp = function(a, b)
+ return a <= b
+ end
+ elseif expected[1] == '!=' then
+ cmp = function(a, b)
+ return a ~= b
+ end
+ end
+ expected = expected[2]
+ end
+
+ -- Tail match
+ if expected < 0 then
+ expected = last + expected + 1
+ end
+ return cmp(pos, expected)
+ end
+
+ for npat, matched_positions in pairs(matches) do
+ local pat_data = processed_tbl[npat]
+ local pattern = pat_data[3]
+ local match = pat_data[2]
+
+ -- Single position
+ if match.position then
+ local position = match.position
+
+ for _, pos in ipairs(matched_positions) do
+ lua_util.debugm(N, log_obj, 'found match %s at offset %s(from %s)',
+ pattern.ext, pos, offset)
+ if match_position(pos + offset, position) then
+ if match.heuristic then
+ local ext, weight = match.heuristic(input, log_obj, pos + offset, part)
+
+ if ext then
+ add_result(weight, ext)
+ break
+ end
+ else
+ add_result(match.weight, pattern.ext)
+ break
+ end
+ end
+ end
+ elseif match.positions then
+ -- Match all positions
+ local all_right = true
+ local matched_pos = 0
+ for _, position in ipairs(match.positions) do
+ local matched = false
+ for _, pos in ipairs(matched_positions) do
+ lua_util.debugm(N, log_obj, 'found match %s at offset %s(from %s)',
+ pattern.ext, pos, offset)
+ if not match_position(pos + offset, position) then
+ matched = true
+ matched_pos = pos
+ break
+ end
+ end
+ if not matched then
+ all_right = false
+ break
+ end
+ end
+
+ if all_right then
+ if match.heuristic then
+ local ext, weight = match.heuristic(input, log_obj, matched_pos + offset, part)
+
+ if ext then
+ add_result(weight, ext)
+ break
+ end
+ else
+ add_result(match.weight, pattern.ext)
+ break
+ end
+ end
+ end
+ end
+
+end
+
+local function process_detected(res)
+ local extensions = lua_util.keys(res)
+
+ if #extensions > 0 then
+ table.sort(extensions, function(ex1, ex2)
+ return res[ex1] > res[ex2]
+ end)
+
+ return extensions, res[extensions[1]]
+ end
+
+ return nil
+end
+
+exports.detect = function(part, log_obj)
+ if not log_obj then
+ log_obj = rspamd_config
+ end
+ local input = part:get_content()
+
+ local res = {}
+
+ if type(input) == 'string' then
+ -- Convert to rspamd_text
+ input = rspamd_text.fromstring(input)
+ end
+
+ if type(input) == 'userdata' then
+ local inplen = #input
+
+ -- Check tail matches
+ if inplen > min_tail_offset then
+ local tail = input:span(inplen - min_tail_offset, min_tail_offset)
+ match_chunk(tail, input, inplen, inplen - min_tail_offset,
+ compiled_tail_patterns, tail_patterns, log_obj, res, part)
+ end
+
+ -- Try short match
+ local head = input:span(1, math.min(max_short_offset, inplen))
+ match_chunk(head, input, inplen, 0,
+ compiled_short_patterns, short_patterns, log_obj, res, part)
+
+ -- Check if we have enough data or go to long patterns
+ local extensions, confidence = process_detected(res)
+
+ if extensions and #extensions > 0 and confidence > 30 then
+ -- We are done on short patterns
+ return extensions[1], types[extensions[1]]
+ end
+
+ -- No way, let's check data in chunks or just the whole input if it is small enough
+ if #input > exports.chunk_size * 3 then
+ -- Chunked version as input is too long
+ local chunk1, chunk2 = input:span(1, exports.chunk_size * 2),
+ input:span(inplen - exports.chunk_size, exports.chunk_size)
+ local offset1, offset2 = 0, inplen - exports.chunk_size
+
+ match_chunk(chunk1, input, inplen,
+ offset1, compiled_patterns, processed_patterns, log_obj, res, part)
+ match_chunk(chunk2, input, inplen,
+ offset2, compiled_patterns, processed_patterns, log_obj, res, part)
+ else
+ -- Input is short enough to match it at all
+ match_chunk(input, input, inplen, 0,
+ compiled_patterns, processed_patterns, log_obj, res, part)
+ end
+ else
+ -- Table input is NYI
+ assert(0, 'table input for match')
+ end
+
+ local extensions = process_detected(res)
+
+ if extensions and #extensions > 0 then
+ return extensions[1], types[extensions[1]]
+ end
+
+ -- Nothing found
+ return nil
+end
+
+exports.detect_mime_part = function(part, log_obj)
+ local ext, weight = heuristics.mime_part_heuristic(part, log_obj)
+
+ if ext and weight and weight > 20 then
+ return ext, types[ext]
+ end
+
+ ext = exports.detect(part, log_obj)
+
+ if ext then
+ return ext, types[ext]
+ end
+
+ -- Text/html and other parts
+ ext, weight = heuristics.text_part_heuristic(part, log_obj)
+ if ext and weight and weight > 20 then
+ return ext, types[ext]
+ end
+end
+
+-- This parameter specifies how many bytes are checked in the input
+-- Rspamd checks 2 chunks at start and 1 chunk at the end
+exports.chunk_size = 32768
+
+exports.types = types
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_magic/patterns.lua b/lualib/lua_magic/patterns.lua
new file mode 100644
index 0000000..971ddd9
--- /dev/null
+++ b/lualib/lua_magic/patterns.lua
@@ -0,0 +1,471 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_magic/patterns
+-- This module contains most common patterns
+--]]
+
+local heuristics = require "lua_magic/heuristics"
+
+local patterns = {
+ pdf = {
+ -- These are alternatives
+ matches = {
+ {
+ string = [[%PDF-[12]\.\d]],
+ position = { '<=', 1024 },
+ weight = 60,
+ heuristic = heuristics.pdf_format_heuristic
+ },
+ {
+ string = [[%FDF-[12]\.\d]],
+ position = { '<=', 1024 },
+ weight = 60,
+ heuristic = heuristics.pdf_format_heuristic
+ },
+ },
+ },
+ ps = {
+ matches = {
+ {
+ string = [[%!PS-Adobe]],
+ relative_position = 0,
+ weight = 60,
+ },
+ },
+ },
+ -- RTF document
+ rtf = {
+ matches = {
+ {
+ string = [[^{\\rt]],
+ position = 4,
+ weight = 60,
+ }
+ }
+ },
+ chm = {
+ matches = {
+ {
+ string = [[ITSF]],
+ relative_position = 0,
+ weight = 60,
+ }
+ }
+ },
+ djvu = {
+ matches = {
+ {
+ string = [[AT&TFORM]],
+ relative_position = 0,
+ weight = 60,
+ },
+ {
+ string = [[DJVM]],
+ relative_position = 0x0c,
+ weight = 60,
+ }
+ }
+ },
+ -- MS Office format, needs heuristic
+ ole = {
+ matches = {
+ {
+ hex = [[d0cf11e0a1b11ae1]],
+ relative_position = 0,
+ weight = 60,
+ heuristic = heuristics.ole_format_heuristic
+ }
+ }
+ },
+ -- MS Exe file
+ exe = {
+ matches = {
+ {
+ string = [[MZ]],
+ relative_position = 0,
+ weight = 15,
+ },
+ -- PE part
+ {
+ string = [[PE\x{00}\x{00}]],
+ position = { '>=', 0x3c + 4 },
+ weight = 15,
+ heuristic = heuristics.pe_part_heuristic,
+ }
+ }
+ },
+ elf = {
+ matches = {
+ {
+ hex = [[7f454c46]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ lnk = {
+ matches = {
+ {
+ hex = [[4C0000000114020000000000C000000000000046]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ bat = {
+ matches = {
+ {
+ string = [[(?i)@\s*ECHO\s+OFF]],
+ position = { '>=', 0 },
+ weight = 60,
+ },
+ }
+ },
+ class = {
+ -- Technically, this also matches MachO files, but I don't care about
+ -- Apple and their mental health problems here: just consider Java files,
+ -- Mach object files and all other cafe babes as bad and block them!
+ matches = {
+ {
+ hex = [[cafebabe]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ ics = {
+ matches = {
+ {
+ string = [[BEGIN:VCALENDAR]],
+ weight = 60,
+ relative_position = 0,
+ }
+ }
+ },
+ vcf = {
+ matches = {
+ {
+ string = [[BEGIN:VCARD]],
+ weight = 60,
+ relative_position = 0,
+ }
+ }
+ },
+ -- Archives
+ arj = {
+ matches = {
+ {
+ hex = '60EA',
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ ace = {
+ matches = {
+ {
+ string = [[\*\*ACE\*\*]],
+ position = 14,
+ weight = 60,
+ },
+ }
+ },
+ cab = {
+ matches = {
+ {
+ hex = [[4d53434600000000]], -- Can be anywhere for SFX :(
+ position = { '>=', 8 },
+ weight = 60,
+ },
+ }
+ },
+ tar = {
+ matches = {
+ {
+ string = [[ustar]],
+ relative_position = 257,
+ weight = 60,
+ },
+ }
+ },
+ bz2 = {
+ matches = {
+ {
+ string = "^BZ[h0]",
+ position = 3,
+ weight = 60,
+ },
+ }
+ },
+ lz4 = {
+ matches = {
+ {
+ hex = "04224d18",
+ relative_position = 0,
+ weight = 60,
+ },
+ {
+ hex = "03214c18",
+ relative_position = 0,
+ weight = 60,
+ },
+ {
+ hex = "02214c18",
+ relative_position = 0,
+ weight = 60,
+ },
+ {
+ -- MozLZ4
+ hex = '6d6f7a4c7a343000',
+ relative_position = 0,
+ weight = 60,
+ }
+ }
+ },
+ zst = {
+ matches = {
+ {
+ string = [[^[\x{22}-\x{40}]\x{B5}\x{2F}\x{FD}]],
+ position = 4,
+ weight = 60,
+ },
+ }
+ },
+ zoo = {
+ matches = {
+ {
+ hex = [[dca7c4fd]],
+ relative_position = 20,
+ weight = 60,
+ },
+ }
+ },
+ xar = {
+ matches = {
+ {
+ string = [[xar!]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ iso = {
+ matches = {
+ {
+ string = [[\x{01}CD001\x{01}]],
+ position = { '>=', 0x8000 + 7 }, -- first 32k is unused
+ weight = 60,
+ },
+ }
+ },
+ egg = {
+ -- ALZip egg
+ matches = {
+ {
+ string = [[EGGA]],
+ weight = 60,
+ relative_position = 0,
+ },
+ }
+ },
+ alz = {
+ -- ALZip alz
+ matches = {
+ {
+ string = [[ALZ\x{01}]],
+ weight = 60,
+ relative_position = 0,
+ },
+ }
+ },
+ -- Apple is a 'special' child: this needs to be matched at the data tail...
+ dmg = {
+ matches = {
+ {
+ string = [[koly\x{00}\x{00}\x{00}\x{04}]],
+ position = -512 + 8,
+ weight = 61,
+ tail = 512,
+ },
+ }
+ },
+ szdd = {
+ matches = {
+ {
+ hex = [[535a4444]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ xz = {
+ matches = {
+ {
+ hex = [[FD377A585A00]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ -- Images
+ psd = {
+ matches = {
+ {
+ string = [[8BPS]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ ico = {
+ matches = {
+ {
+ hex = [[00000100]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ pcx = {
+ matches = {
+ {
+ hex = [[0A050108]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ pic = {
+ matches = {
+ {
+ hex = [[FF80C9C71A00]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ swf = {
+ matches = {
+ {
+ hex = [[5a5753]], -- LZMA
+ relative_position = 0,
+ weight = 60,
+ },
+ {
+ hex = [[435753]], -- Zlib
+ relative_position = 0,
+ weight = 60,
+ },
+ {
+ hex = [[465753]], -- Uncompressed
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ tiff = {
+ matches = {
+ {
+ hex = [[49492a00]], -- LE encoded
+ relative_position = 0,
+ weight = 60,
+ },
+ {
+ hex = [[4d4d]], -- BE tiff
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ -- Other
+ pgp = {
+ matches = {
+ {
+ hex = [[A803504750]],
+ relative_position = 0,
+ weight = 60,
+ },
+ {
+ hex = [[2D424547494E20504750204D4553534147452D]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ uue = {
+ matches = {
+ {
+ hex = [[626567696e20]],
+ relative_position = 0,
+ weight = 60,
+ },
+ }
+ },
+ dwg = {
+ matches = {
+ {
+ string = '^AC10[12][2-9]',
+ position = 6,
+ weight = 60,
+ }
+ }
+ },
+ jpg = {
+ matches = {
+ { -- JPEG2000
+ hex = [[0000000c6a5020200d0a870a]],
+ relative_position = 0,
+ weight = 60,
+ },
+ {
+ string = [[^\x{ff}\x{d8}\x{ff}]],
+ weight = 60,
+ position = 3,
+ },
+ },
+ },
+ png = {
+ matches = {
+ {
+ string = [[^\x{89}PNG\x{0d}\x{0a}\x{1a}\x{0a}]],
+ position = 8,
+ weight = 60,
+ },
+ }
+ },
+ gif = {
+ matches = {
+ {
+ string = [[^GIF8\d]],
+ position = 5,
+ weight = 60,
+ },
+ }
+ },
+ bmp = {
+ matches = {
+ {
+ string = [[^BM...\x{00}\x{00}\x{00}\x{00}]],
+ position = 9,
+ weight = 60,
+ },
+ }
+ },
+}
+
+return patterns
diff --git a/lualib/lua_magic/types.lua b/lualib/lua_magic/types.lua
new file mode 100644
index 0000000..3dce2e1
--- /dev/null
+++ b/lualib/lua_magic/types.lua
@@ -0,0 +1,327 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_magic/patterns
+-- This module contains types definitions
+--]]
+
+-- This table is indexed by msdos extension for convenience
+
+local types = {
+ -- exe
+ exe = {
+ ct = 'application/x-ms-application',
+ type = 'executable',
+ },
+ elf = {
+ ct = 'application/x-elf-executable',
+ type = 'executable',
+ },
+ lnk = {
+ ct = 'application/x-ms-application',
+ type = 'executable',
+ },
+ class = {
+ ct = 'application/x-java-applet',
+ type = 'executable',
+ },
+ jar = {
+ ct = 'application/java-archive',
+ type = 'archive',
+ },
+ apk = {
+ ct = 'application/vnd.android.package-archive',
+ type = 'archive',
+ },
+ bat = {
+ ct = 'application/x-bat',
+ type = 'executable',
+ },
+ -- text
+ rtf = {
+ ct = "application/rtf",
+ type = 'binary',
+ },
+ pdf = {
+ ct = 'application/pdf',
+ type = 'binary',
+ },
+ ps = {
+ ct = 'application/postscript',
+ type = 'binary',
+ },
+ chm = {
+ ct = 'application/x-chm',
+ type = 'binary',
+ },
+ djvu = {
+ ct = 'application/x-djvu',
+ type = 'binary',
+ },
+ -- archives
+ arj = {
+ ct = 'application/x-arj',
+ type = 'archive',
+ },
+ cab = {
+ ct = 'application/x-cab',
+ type = 'archive',
+ },
+ ace = {
+ ct = 'application/x-ace',
+ type = 'archive',
+ },
+ tar = {
+ ct = 'application/x-tar',
+ type = 'archive',
+ },
+ bz2 = {
+ ct = 'application/x-bzip',
+ type = 'archive',
+ },
+ xz = {
+ ct = 'application/x-xz',
+ type = 'archive',
+ },
+ lz4 = {
+ ct = 'application/x-lz4',
+ type = 'archive',
+ },
+ zst = {
+ ct = 'application/x-zstandard',
+ type = 'archive',
+ },
+ dmg = {
+ ct = 'application/x-dmg',
+ type = 'archive',
+ },
+ iso = {
+ ct = 'application/x-iso',
+ type = 'archive',
+ },
+ zoo = {
+ ct = 'application/x-zoo',
+ type = 'archive',
+ },
+ egg = {
+ ct = 'application/x-egg',
+ type = 'archive',
+ },
+ alz = {
+ ct = 'application/x-alz',
+ type = 'archive',
+ },
+ xar = {
+ ct = 'application/x-xar',
+ type = 'archive',
+ },
+ epub = {
+ ct = 'application/x-epub',
+ type = 'archive'
+ },
+ szdd = { -- in fact, their MSDOS extension is like FOO.TX_ or FOO.TX$
+ ct = 'application/x-compressed',
+ type = 'archive',
+ },
+ -- images
+ psd = {
+ ct = 'image/psd',
+ type = 'image',
+ av_check = false,
+ },
+ pcx = {
+ ct = 'image/pcx',
+ type = 'image',
+ av_check = false,
+ },
+ pic = {
+ ct = 'image/pic',
+ type = 'image',
+ av_check = false,
+ },
+ tiff = {
+ ct = 'image/tiff',
+ type = 'image',
+ av_check = false,
+ },
+ ico = {
+ ct = 'image/ico',
+ type = 'image',
+ av_check = false,
+ },
+ swf = {
+ ct = 'application/x-shockwave-flash',
+ type = 'image',
+ },
+ -- Ole files
+ ole = {
+ ct = 'application/octet-stream',
+ type = 'office'
+ },
+ doc = {
+ ct = 'application/msword',
+ type = 'office'
+ },
+ xls = {
+ ct = 'application/vnd.ms-excel',
+ type = 'office'
+ },
+ ppt = {
+ ct = 'application/vnd.ms-powerpoint',
+ type = 'office'
+ },
+ vsd = {
+ ct = 'application/vnd.visio',
+ type = 'office'
+ },
+ msi = {
+ ct = 'application/x-msi',
+ type = 'executable'
+ },
+ msg = {
+ ct = 'application/vnd.ms-outlook',
+ type = 'office'
+ },
+ -- newer office (2007+)
+ docx = {
+ ct = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+ type = 'office'
+ },
+ xlsx = {
+ ct = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+ type = 'office'
+ },
+ pptx = {
+ ct = 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+ type = 'office'
+ },
+ -- OpenOffice formats
+ odt = {
+ ct = 'application/vnd.oasis.opendocument.text',
+ type = 'office'
+ },
+ ods = {
+ ct = 'application/vnd.oasis.opendocument.spreadsheet',
+ type = 'office'
+ },
+ odp = {
+ ct = 'application/vnd.oasis.opendocument.presentation',
+ type = 'office'
+ },
+ -- https://en.wikipedia.org/wiki/Associated_Signature_Containers
+ asice = {
+ ct = 'application/vnd.etsi.asic-e+zip',
+ type = 'office'
+ },
+ asics = {
+ ct = 'application/vnd.etsi.asic-s+zip',
+ type = 'office'
+ },
+ -- other
+ pgp = {
+ ct = 'application/encrypted',
+ type = 'encrypted'
+ },
+ uue = {
+ ct = 'application/x-uuencoded',
+ type = 'binary',
+ },
+ -- Types that are detected by Rspamd itself
+ -- Archives
+ zip = {
+ ct = 'application/zip',
+ type = 'archive',
+ },
+ rar = {
+ ct = 'application/x-rar',
+ type = 'archive',
+ },
+ ['7z'] = {
+ ct = 'application/x-7z-compressed',
+ type = 'archive',
+ },
+ gz = {
+ ct = 'application/gzip',
+ type = 'archive',
+ },
+ -- Images
+ png = {
+ ct = 'image/png',
+ type = 'image',
+ av_check = false,
+ },
+ gif = {
+ ct = 'image/gif',
+ type = 'image',
+ av_check = false,
+ },
+ jpg = {
+ ct = 'image/jpeg',
+ type = 'image',
+ av_check = false,
+ },
+ bmp = {
+ type = 'image',
+ ct = 'image/bmp',
+ av_check = false,
+ },
+ dwg = {
+ type = 'image',
+ ct = 'image/vnd.dwg',
+ },
+ -- Text
+ xml = {
+ ct = 'application/xml',
+ type = 'text',
+ no_text = true,
+ },
+ txt = {
+ type = 'text',
+ ct = 'text/plain',
+ av_check = false,
+ },
+ html = {
+ type = 'text',
+ ct = 'text/html',
+ av_check = false,
+ },
+ csv = {
+ type = 'text',
+ ct = 'text/csv',
+ av_check = false,
+ no_text = true,
+ },
+ ics = {
+ type = 'text',
+ ct = 'text/calendar',
+ av_check = false,
+ no_text = true,
+ },
+ vcf = {
+ type = 'text',
+ ct = 'text/vcard',
+ av_check = false,
+ no_text = true,
+ },
+ eml = {
+ type = 'message',
+ ct = 'message/rfc822',
+ av_check = false,
+ },
+}
+
+return types \ No newline at end of file
diff --git a/lualib/lua_maps.lua b/lualib/lua_maps.lua
new file mode 100644
index 0000000..d357310
--- /dev/null
+++ b/lualib/lua_maps.lua
@@ -0,0 +1,612 @@
+--[[[
+-- @module lua_maps
+-- This module contains helper functions for managing rspamd maps
+--]]
+
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local rspamd_logger = require "rspamd_logger"
+local ts = require("tableshape").types
+local lua_util = require "lua_util"
+
+local exports = {}
+
+local maps_cache = {}
+
+local function map_hash_key(data, mtype)
+ local hash = require "rspamd_cryptobox_hash"
+ local st = hash.create_specific('xxh64')
+ st:update(data)
+ st:update(mtype)
+
+ return st:hex()
+end
+
+local function starts(where, st)
+ return string.sub(where, 1, string.len(st)) == st
+end
+
+local function cut_prefix(where, st)
+ return string.sub(where, #st + 1)
+end
+
+local function maybe_adjust_type(data, mtype)
+ local function check_prefix(prefix, t)
+ if starts(data, prefix) then
+ data = cut_prefix(data, prefix)
+ mtype = t
+
+ return true
+ end
+
+ return false
+ end
+
+ local known_types = {
+ { 'regexp;', 'regexp' },
+ { 're;', 'regexp' },
+ { 'regexp_multi;', 'regexp_multi' },
+ { 're_multi;', 'regexp_multi' },
+ { 'glob;', 'glob' },
+ { 'glob_multi;', 'glob_multi' },
+ { 'radix;', 'radix' },
+ { 'ipnet;', 'radix' },
+ { 'set;', 'set' },
+ { 'hash;', 'hash' },
+ { 'plain;', 'hash' },
+ { 'cdb;', 'cdb' },
+ { 'cdb:/', 'cdb' },
+ }
+
+ if mtype == 'callback' then
+ return mtype
+ end
+
+ for _, t in ipairs(known_types) do
+ if check_prefix(t[1], t[2]) then
+ return data, mtype
+ end
+ end
+
+ -- No change
+ return data, mtype
+end
+
+local external_map_schema = ts.shape {
+ external = ts.equivalent(true), -- must be true
+ backend = ts.string, -- where to get data, required
+ method = ts.one_of { "body", "header", "query" }, -- how to pass input
+ encode = ts.one_of { "json", "messagepack" }:is_optional(), -- how to encode input (if relevant)
+ timeout = (ts.number + ts.string / lua_util.parse_time_interval):is_optional(),
+}
+
+local rspamd_http = require "rspamd_http"
+local ucl = require "ucl"
+
+local function url_encode_string(str)
+ str = string.gsub(str, "([^%w _%%%-%.~])",
+ function(c)
+ return string.format("%%%02X", string.byte(c))
+ end)
+ str = string.gsub(str, " ", "+")
+ return str
+end
+
+assert(url_encode_string('上海+中國') == '%E4%B8%8A%E6%B5%B7%2B%E4%B8%AD%E5%9C%8B')
+assert(url_encode_string('? and the Mysterians') == '%3F+and+the+Mysterians')
+
+local function query_external_map(map_config, upstreams, key, callback, task)
+ local http_method = (map_config.method == 'body' or map_config.method == 'form') and 'POST' or 'GET'
+ local upstream = upstreams:get_upstream_round_robin()
+ local http_headers = {
+ ['Accept'] = '*/*'
+ }
+ local http_body = nil
+ local url = map_config.backend
+
+ if type(key) == 'string' or type(key) == 'userdata' then
+ if map_config.method == 'body' then
+ http_body = key
+ http_headers['Content-Type'] = 'text/plain'
+ elseif map_config.method == 'header' then
+ http_headers = {
+ key = key
+ }
+ elseif map_config.method == 'query' then
+ url = string.format('%s?key=%s', url, url_encode_string(tostring(key)))
+ end
+ elseif type(key) == 'table' then
+ if map_config.method == 'body' then
+ if map_config.encode == 'json' then
+ http_body = ucl.to_format(key, 'json-compact', true)
+ http_headers['Content-Type'] = 'application/json'
+ elseif map_config.encode == 'messagepack' then
+ http_body = ucl.to_format(key, 'messagepack', true)
+ http_headers['Content-Type'] = 'application/msgpack'
+ else
+ local caller = debug.getinfo(2) or {}
+ rspamd_logger.errx(task,
+ "requested external map key with a wrong combination body method and missing encode; caller: %s:%s",
+ caller.short_src, caller.currentline)
+ callback(false, 'invalid map usage', 500, task)
+ end
+ else
+ -- query/header and no encode
+ if map_config.method == 'query' then
+ local params_table = {}
+ for k, v in pairs(key) do
+ if type(v) == 'string' then
+ table.insert(params_table, string.format('%s=%s', url_encode_string(k), url_encode_string(v)))
+ end
+ end
+ url = string.format('%s?%s', url, table.concat(params_table, '&'))
+ elseif map_config.method == 'header' then
+ http_headers = key
+ else
+ local caller = debug.getinfo(2) or {}
+ rspamd_logger.errx(task,
+ "requested external map key with a wrong combination of encode and input; caller: %s:%s",
+ caller.short_src, caller.currentline)
+ callback(false, 'invalid map usage', 500, task)
+ return
+ end
+ end
+ end
+
+ local function map_callback(err, code, body, _)
+ if err then
+ callback(false, err, code, task)
+ elseif code == 200 then
+ callback(true, body, 200, task)
+ else
+ callback(false, err, code, task)
+ end
+ end
+
+ local ret = rspamd_http.request {
+ task = task,
+ url = url,
+ callback = map_callback,
+ timeout = map_config.timeout or 1.0,
+ keepalive = true,
+ upstream = upstream,
+ method = http_method,
+ headers = http_headers,
+ body = http_body,
+ }
+
+ if not ret then
+ callback(false, 'http request error', 500, task)
+ end
+end
+
+--[[[
+-- @function lua_maps.map_add_from_ucl(opt, mtype, description)
+-- Creates a map from static data
+-- Returns true if map was added or nil
+-- @param {string or table} opt data for map (or URL)
+-- @param {string} mtype type of map (`set`, `map`, `radix`, `regexp`)
+-- @param {string} description human-readable description of map
+-- @param {function} callback optional callback that will be called on map match (required for external maps)
+-- @return {bool} true on success, or `nil`
+--]]
+local function rspamd_map_add_from_ucl(opt, mtype, description, callback)
+ local ret = {
+ get_key = function(t, k, key_callback, task)
+ if t.__data then
+ local cb = key_callback or callback
+ if t.__external then
+ if not cb or not task then
+ local caller = debug.getinfo(2) or {}
+ rspamd_logger.errx(rspamd_config, "requested external map key without callback or task; caller: %s:%s",
+ caller.short_src, caller.currentline)
+ return nil
+ end
+ query_external_map(t.__data, t.__upstreams, k, cb, task)
+ else
+ local result = t.__data:get_key(k)
+ if cb then
+ if result then
+ cb(true, result, 200, task)
+ else
+ cb(false, 'not found', 404, task)
+ end
+ else
+ return result
+ end
+ end
+ end
+
+ return nil
+ end,
+ foreach = function(t, cb)
+ return t.__data:foreach(cb)
+ end,
+ on_load = function(t, cb)
+ t.__data:on_load(cb)
+ end
+ }
+ local ret_mt = {
+ __index = function(t, k, key_callback, task)
+ if t.__data then
+ return t.get_key(k, key_callback, task)
+ end
+
+ return nil
+ end
+ }
+
+ if not opt then
+ return nil
+ end
+
+ local function maybe_register_selector()
+ if opt.selector_alias then
+ local lua_selectors = require "lua_selectors"
+ lua_selectors.add_map(opt.selector_alias, ret)
+ end
+ end
+
+ if type(opt) == 'string' then
+ opt, mtype = maybe_adjust_type(opt, mtype)
+ local cache_key = map_hash_key(opt, mtype)
+ if not callback and maps_cache[cache_key] then
+ rspamd_logger.infox(rspamd_config, 'reuse url for %s(%s)',
+ opt, mtype)
+
+ return maps_cache[cache_key]
+ end
+ -- We have a single string, so we treat it as a map
+ local map = rspamd_config:add_map {
+ type = mtype,
+ description = description,
+ url = opt,
+ }
+
+ if map then
+ ret.__data = map
+ ret.hash = cache_key
+ setmetatable(ret, ret_mt)
+ maps_cache[cache_key] = ret
+ return ret
+ end
+ elseif type(opt) == 'table' then
+ local cache_key = lua_util.table_digest(opt)
+ if not callback and maps_cache[cache_key] then
+ rspamd_logger.infox(rspamd_config, 'reuse url for complex map definition %s: %s',
+ cache_key:sub(1, 8), description)
+
+ return maps_cache[cache_key]
+ end
+
+ if opt[1] then
+ -- Adjust each element if needed
+ local adjusted
+ for i, source in ipairs(opt) do
+ local nsrc, ntype = maybe_adjust_type(source, mtype)
+
+ if mtype ~= ntype then
+ if not adjusted then
+ mtype = ntype
+ end
+ adjusted = true
+ end
+ opt[i] = nsrc
+ end
+
+ if mtype == 'radix' then
+
+ if string.find(opt[1], '^%d') then
+ local map = rspamd_config:radix_from_ucl(opt)
+
+ if map then
+ ret.__data = map
+ setmetatable(ret, ret_mt)
+ maps_cache[cache_key] = ret
+ maybe_register_selector()
+
+ return ret
+ end
+ else
+ -- Plain table
+ local map = rspamd_config:add_map {
+ type = mtype,
+ description = description,
+ url = opt,
+ }
+ if map then
+ ret.__data = map
+ setmetatable(ret, ret_mt)
+ maps_cache[cache_key] = ret
+ maybe_register_selector()
+
+ return ret
+ end
+ end
+ elseif mtype == 'regexp' or mtype == 'glob' then
+ if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then
+ -- Plain table
+ local map = rspamd_config:add_map {
+ type = mtype,
+ description = description,
+ url = opt,
+ }
+ if map then
+ ret.__data = map
+ setmetatable(ret, ret_mt)
+ maps_cache[cache_key] = ret
+ maybe_register_selector()
+
+ return ret
+ end
+ else
+ local map = rspamd_config:add_map {
+ type = mtype,
+ description = description,
+ url = {
+ url = 'static',
+ data = opt,
+ }
+ }
+ if map then
+ ret.__data = map
+ setmetatable(ret, ret_mt)
+ maps_cache[cache_key] = ret
+ maybe_register_selector()
+
+ return ret
+ end
+ end
+ else
+ if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then
+ -- Plain table
+ local map = rspamd_config:add_map {
+ type = mtype,
+ description = description,
+ url = opt,
+ }
+ if map then
+ ret.__data = map
+ setmetatable(ret, ret_mt)
+ maps_cache[cache_key] = ret
+ maybe_register_selector()
+
+ return ret
+ end
+ else
+ local data = {}
+ local nelts = 0
+ -- Plain array of keys, count merely numeric elts
+ for _, elt in ipairs(opt) do
+ if type(elt) == 'string' then
+ -- Numeric table
+ if mtype == 'hash' then
+ -- Treat as KV pair
+ local pieces = lua_util.str_split(elt, ' ')
+ if #pieces > 1 then
+ local key = table.remove(pieces, 1)
+ data[key] = table.concat(pieces, ' ')
+ else
+ data[elt] = true
+ end
+ else
+ data[elt] = true
+ end
+
+ nelts = nelts + 1
+ end
+ end
+
+ if nelts > 0 then
+ -- Plain Lua table that is used as a map
+ ret.__data = data
+ ret.get_key = function(t, k)
+ if k ~= '__data' then
+ return t.__data[k]
+ end
+
+ return nil
+ end
+ ret.foreach = function(_, func)
+ for k, v in pairs(ret.__data) do
+ if not func(k, v) then
+ return false
+ end
+ end
+
+ return true
+ end
+ ret.on_load = function(_, cb)
+ rspamd_config:add_on_load(function(_, _, _)
+ cb()
+ end)
+ end
+
+ maps_cache[cache_key] = ret
+ maybe_register_selector()
+
+ return ret
+ else
+ -- Empty map, huh?
+ rspamd_logger.errx(rspamd_config, 'invalid map element: %s',
+ opt)
+ end
+ end
+ end
+ else
+ if opt.external then
+ -- External map definition, missing fields are handled by schema
+ local parse_res, parse_err = external_map_schema(opt)
+
+ if parse_res then
+ ret.__upstreams = lua_util.http_upstreams_by_url(rspamd_config:get_mempool(), opt.backend)
+ if ret.__upstreams then
+ ret.__data = opt
+ ret.__external = true
+ setmetatable(ret, ret_mt)
+ maybe_register_selector()
+
+ return ret
+ else
+ rspamd_logger.errx(rspamd_config, 'cannot parse external map upstreams: %s',
+ opt.backend)
+ end
+ else
+ rspamd_logger.errx(rspamd_config, 'cannot parse external map: %s',
+ parse_err)
+ end
+ else
+ -- Adjust lua specific augmentations in a trivial case
+ if type(opt.url) == 'string' then
+ local nsrc, ntype = maybe_adjust_type(opt.url, mtype)
+ if nsrc and ntype then
+ opt.url = nsrc
+ mtype = ntype
+ end
+ end
+ -- We have some non-trivial object so let C code to deal with it somehow...
+ local map = rspamd_config:add_map {
+ type = mtype,
+ description = description,
+ url = opt,
+ }
+ if map then
+ ret.__data = map
+ setmetatable(ret, ret_mt)
+ maps_cache[cache_key] = ret
+ maybe_register_selector()
+
+ return ret
+ end
+ end
+ end -- opt[1]
+ end
+
+ return nil
+end
+
+--[[[
+-- @function lua_maps.map_add(mname, optname, mtype, description)
+-- Creates a map from configuration elements (static data or URL)
+-- Returns true if map was added or nil
+-- @param {string} mname config section to use
+-- @param {string} optname option name to use
+-- @param {string} mtype type of map ('set', 'hash', 'radix', 'regexp', 'glob')
+-- @param {string} description human-readable description of map
+-- @param {function} callback optional callback that will be called on map match (required for external maps)
+-- @return {bool} true on success, or `nil`
+--]]
+
+local function rspamd_map_add(mname, optname, mtype, description, callback)
+ local opt = rspamd_config:get_module_opt(mname, optname)
+
+ return rspamd_map_add_from_ucl(opt, mtype, description, callback)
+end
+
+exports.rspamd_map_add = rspamd_map_add
+exports.map_add = rspamd_map_add
+exports.rspamd_map_add_from_ucl = rspamd_map_add_from_ucl
+exports.map_add_from_ucl = rspamd_map_add_from_ucl
+
+-- Check `what` for being lua_map name, otherwise just compares key with what
+local function rspamd_maybe_check_map(key, what)
+ local fun = require "fun"
+
+ if type(what) == "table" then
+ return fun.any(function(elt)
+ return rspamd_maybe_check_map(key, elt)
+ end, what)
+ end
+ if type(rspamd_maps) == "table" then
+ local mn
+ if starts(key, "map:") then
+ mn = string.sub(key, 5)
+ elseif starts(key, "map://") then
+ mn = string.sub(key, 7)
+ end
+
+ if mn and rspamd_maps[mn] then
+ return rspamd_maps[mn]:get_key(what)
+ end
+ end
+
+ return what:lower() == key
+end
+
+exports.rspamd_maybe_check_map = rspamd_maybe_check_map
+
+--[[[
+-- @function lua_maps.fill_config_maps(mname, options, defs)
+-- Fill maps that could be defined in defs, from the config in the options
+-- Defs is a table indexed by a map's parameter name and defining it's config,
+-- @example
+-- defs = {
+-- my_map = {
+-- type = 'map',
+-- description = 'my cool map',
+-- optional = true,
+-- }
+-- }
+-- -- Then this function will look for opts.my_map parameter and try to replace it with
+-- -- a map with the specific type, description but not failing if it was empty.
+-- -- It will also set options.my_map_orig to the original value defined in the map.
+--]]
+exports.fill_config_maps = function(mname, opts, map_defs)
+ assert(type(opts) == 'table')
+ assert(type(map_defs) == 'table')
+ for k, v in pairs(map_defs) do
+ if opts[k] then
+ local map = rspamd_map_add_from_ucl(opts[k], v.type or 'map', v.description)
+ if not map then
+ rspamd_logger.errx(rspamd_config, 'map add error %s for module %s', k, mname)
+ return false
+ end
+ opts[k .. '_orig'] = opts[k]
+ opts[k] = map
+ elseif not v.optional then
+ rspamd_logger.errx(rspamd_config, 'cannot find non optional map %s for module %s', k, mname)
+ return false
+ end
+ end
+
+ return true
+end
+
+local direct_map_schema = ts.shape { -- complex object
+ name = ts.string:is_optional(),
+ description = ts.string:is_optional(),
+ selector_alias = ts.string:is_optional(), -- an optional alias for the selectos framework
+ timeout = ts.number,
+ data = ts.array_of(ts.string):is_optional(),
+ -- Tableshape has no options support for something like key1 or key2?
+ upstreams = ts.one_of {
+ ts.string,
+ ts.array_of(ts.string),
+ } :is_optional(),
+ url = ts.one_of {
+ ts.string,
+ ts.array_of(ts.string),
+ } :is_optional(),
+}
+
+exports.map_schema = ts.one_of {
+ ts.string, -- 'http://some_map'
+ ts.array_of(ts.string), -- ['foo', 'bar']
+ ts.one_of { direct_map_schema, external_map_schema }
+}
+
+return exports
diff --git a/lualib/lua_maps_expressions.lua b/lualib/lua_maps_expressions.lua
new file mode 100644
index 0000000..996de99
--- /dev/null
+++ b/lualib/lua_maps_expressions.lua
@@ -0,0 +1,219 @@
+--[[[
+-- @module lua_maps_expressions
+-- This module contains routines to combine maps, selectors and expressions
+-- in a generic framework
+@example
+whitelist_ip_from = {
+ rules {
+ ip {
+ selector = "ip";
+ map = "/path/to/whitelist_ip.map";
+ }
+ from {
+ selector = "from(smtp)";
+ map = "/path/to/whitelist_from.map";
+ }
+ }
+ expression = "ip & from";
+}
+--]]
+
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local lua_selectors = require "lua_selectors"
+local lua_maps = require "lua_maps"
+local rspamd_expression = require "rspamd_expression"
+local rspamd_logger = require "rspamd_logger"
+local fun = require "fun"
+local ts = require("tableshape").types
+
+local exports = {}
+
+local function process_func(elt, task)
+ local matched = {}
+ local function process_atom(atom)
+ local rule = elt.rules[atom]
+ local res = 0
+
+ local function match_rule(val)
+ local map_match = rule.map:get_key(val)
+ if map_match then
+ res = 1.0
+ matched[rule.name] = {
+ matched = val,
+ value = map_match
+ }
+ end
+ end
+
+ local values = rule.selector(task)
+
+ if values then
+ if type(values) == 'table' then
+ for _, val in ipairs(values) do
+ if res == 0 then
+ match_rule(val)
+ end
+ end
+ else
+ match_rule(values)
+ end
+ end
+
+ return res
+ end
+
+ local res = elt.expr:process(process_atom)
+
+ if res > 0 then
+ return res, matched
+ end
+
+ return nil
+end
+
+exports.schema = ts.shape {
+ expression = ts.string,
+ rules = ts.array_of(
+ ts.shape {
+ selector = ts.string,
+ map = lua_maps.map_schema,
+ }
+ )
+}
+
+--[[[
+-- @function lua_maps_expression.create(config, object, module_name)
+-- Creates a new maps combination from `object` for `module_name`.
+-- The input should be table with the following fields:
+--
+-- * `rules` - kv map of rules where each rule has `map` and `selector` mandatory attribute, also `type` for map type, e.g. `regexp`
+-- * `expression` - Rspamd expression where elements are names from `rules` field, e.g. `ip & from`
+--
+-- This function returns an object with public method `process(task)` that checks
+-- a task for the conditions defined in `expression` and `rules` and returns 2 values:
+--
+-- 1. value returned by an expression (e.g. 1 or 0)
+-- 2. an map (rule_name -> table) of matches, where each element has the following fields:
+-- * `matched` - selector's value
+-- * `value` - map's result
+--
+-- In case if `expression` is false a `nil` value is returned.
+-- @param {rspamd_config} cfg rspamd config
+-- @param {table} obj configuration table
+--
+--]]
+local function create(cfg, obj, module_name)
+ if not module_name then
+ module_name = 'lua_maps_expressions'
+ end
+
+ if not obj or not obj.rules or not obj.expression then
+ rspamd_logger.errx(cfg, 'cannot add maps combination for module %s: required elements are missing',
+ module_name)
+ return nil
+ end
+
+ local ret = {
+ process = process_func,
+ rules = {},
+ module_name = module_name
+ }
+
+ for name, rule in pairs(obj.rules) do
+ local sel = lua_selectors.create_selector_closure(cfg, rule.selector)
+
+ if not sel then
+ rspamd_logger.errx(cfg, 'cannot add selector for element %s in module %s',
+ name, module_name)
+ end
+
+ if not rule.type then
+ -- Guess type
+ if name:find('ip') or name:find('ipnet') then
+ rule.type = 'radix'
+ elseif name:find('regexp') or name:find('re_') then
+ rule.type = 'regexp'
+ elseif name:find('glob') then
+ rule.type = 'regexp'
+ else
+ rule.type = 'set'
+ end
+ end
+ local map = lua_maps.map_add_from_ucl(rule.map, rule.type,
+ obj.description or module_name)
+ if not map then
+ rspamd_logger.errx(cfg, 'cannot add map for element %s in module %s',
+ name, module_name)
+ end
+
+ if sel and map then
+ ret.rules[name] = {
+ selector = sel,
+ map = map,
+ name = name,
+ }
+ else
+ return nil
+ end
+ end
+
+ -- Now process and parse expression
+ local function parse_atom(str)
+ local atom = table.concat(fun.totable(fun.take_while(function(c)
+ if string.find(', \t()><+!|&\n', c, 1, true) then
+ return false
+ end
+ return true
+ end, fun.iter(str))), '')
+
+ if ret.rules[atom] then
+ return atom
+ end
+
+ rspamd_logger.errx(cfg, 'use of undefined element "%s" when parsing maps expression for %s',
+ atom, module_name)
+
+ return nil
+ end
+ local expr = rspamd_expression.create(obj.expression, parse_atom,
+ rspamd_config:get_mempool())
+
+ if not expr then
+ rspamd_logger.errx(cfg, 'cannot add map expression for module %s',
+ module_name)
+ return nil
+ end
+
+ ret.expr = expr
+
+ if obj.symbol then
+ rspamd_config:register_symbol {
+ type = 'virtual,ghost',
+ name = obj.symbol,
+ score = 0.0,
+ }
+ end
+
+ ret.symbol = obj.symbol
+
+ return ret
+end
+
+exports.create = create
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_meta.lua b/lualib/lua_meta.lua
new file mode 100644
index 0000000..340d89e
--- /dev/null
+++ b/lualib/lua_meta.lua
@@ -0,0 +1,549 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local exports = {}
+
+local N = "metatokens"
+local ts = require("tableshape").types
+local logger = require "rspamd_logger"
+
+-- Metafunctions
+local function meta_size_function(task)
+ local sizes = {
+ 100,
+ 200,
+ 500,
+ 1000,
+ 2000,
+ 4000,
+ 10000,
+ 20000,
+ 30000,
+ 100000,
+ 200000,
+ 400000,
+ 800000,
+ 1000000,
+ 2000000,
+ 8000000,
+ }
+
+ local size = task:get_size()
+ for i = 1, #sizes do
+ if sizes[i] >= size then
+ return { (1.0 * i) / #sizes }
+ end
+ end
+
+ return { 0 }
+end
+
+local function meta_images_function(task)
+ local images = task:get_images()
+ local ntotal = 0
+ local njpg = 0
+ local npng = 0
+ local nlarge = 0
+ local nsmall = 0
+
+ if images then
+ for _, img in ipairs(images) do
+ if img:get_type() == 'png' then
+ npng = npng + 1
+ elseif img:get_type() == 'jpeg' then
+ njpg = njpg + 1
+ end
+
+ local w = img:get_width()
+ local h = img:get_height()
+
+ if w > 0 and h > 0 then
+ if w + h > 256 then
+ nlarge = nlarge + 1
+ else
+ nsmall = nsmall + 1
+ end
+ end
+
+ ntotal = ntotal + 1
+ end
+ end
+ if ntotal > 0 then
+ njpg = 1.0 * njpg / ntotal
+ npng = 1.0 * npng / ntotal
+ nlarge = 1.0 * nlarge / ntotal
+ nsmall = 1.0 * nsmall / ntotal
+ end
+ return { ntotal, njpg, npng, nlarge, nsmall }
+end
+
+local function meta_nparts_function(task)
+ local nattachments = 0
+ local ntextparts = 0
+ local totalparts = 1
+
+ local tp = task:get_text_parts()
+ if tp then
+ ntextparts = #tp
+ end
+
+ local parts = task:get_parts()
+
+ if parts then
+ for _, p in ipairs(parts) do
+ if p:is_attachment() then
+ nattachments = nattachments + 1
+ end
+ totalparts = totalparts + 1
+ end
+ end
+
+ return { (1.0 * ntextparts) / totalparts, (1.0 * nattachments) / totalparts }
+end
+
+local function meta_encoding_function(task)
+ local nutf = 0
+ local nother = 0
+
+ local tp = task:get_text_parts()
+ if tp and #tp > 0 then
+ for _, p in ipairs(tp) do
+ if p:is_utf() then
+ nutf = nutf + 1
+ else
+ nother = nother + 1
+ end
+ end
+
+ return { nutf / #tp, nother / #tp }
+ end
+
+ return { 0, 0 }
+end
+
+local function meta_recipients_function(task)
+ local nmime = 0
+ local nsmtp = 0
+
+ if task:has_recipients('mime') then
+ nmime = #(task:get_recipients('mime'))
+ end
+ if task:has_recipients('smtp') then
+ nsmtp = #(task:get_recipients('smtp'))
+ end
+
+ if nmime > 0 then
+ nmime = 1.0 / nmime
+ end
+ if nsmtp > 0 then
+ nsmtp = 1.0 / nsmtp
+ end
+
+ return { nmime, nsmtp }
+end
+
+local function meta_received_function(task)
+ local count_factor = 0
+ local invalid_factor = 0
+ local rh = task:get_received_headers()
+ local time_factor = 0
+ local secure_factor = 0
+ local fun = require "fun"
+
+ if rh and #rh > 0 then
+
+ local ntotal = 0.0
+ local init_time = 0
+
+ fun.each(function(rc)
+ ntotal = ntotal + 1.0
+
+ if not rc.by_hostname then
+ invalid_factor = invalid_factor + 1.0
+ end
+ if init_time == 0 and rc.timestamp then
+ init_time = rc.timestamp
+ elseif rc.timestamp then
+ time_factor = time_factor + math.abs(init_time - rc.timestamp)
+ init_time = rc.timestamp
+ end
+ if rc.flags and (rc.flags['ssl'] or rc.flags['authenticated']) then
+ secure_factor = secure_factor + 1.0
+ end
+ end,
+ fun.filter(function(rc)
+ return not rc.flags or not rc.flags['artificial']
+ end, rh))
+
+ if ntotal > 0 then
+ invalid_factor = invalid_factor / ntotal
+ secure_factor = secure_factor / ntotal
+ count_factor = 1.0 / ntotal
+ end
+
+ if time_factor ~= 0 then
+ time_factor = 1.0 / time_factor
+ end
+ end
+
+ return { count_factor, invalid_factor, time_factor, secure_factor }
+end
+
+local function meta_urls_function(task)
+ local has_urls, nurls = task:has_urls()
+ if has_urls and nurls > 0 then
+ return { 1.0 / nurls }
+ end
+
+ return { 0 }
+end
+
+local function meta_words_function(task)
+ local avg_len = task:get_mempool():get_variable("avg_words_len", "double") or 0.0
+ local short_words = task:get_mempool():get_variable("short_words_cnt", "double") or 0.0
+ local ret_len = 0
+
+ local lens = {
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9,
+ 10,
+ 15,
+ 20,
+ }
+
+ for i = 1, #lens do
+ if lens[i] >= avg_len then
+ ret_len = (1.0 * i) / #lens
+ break
+ end
+ end
+
+ local tp = task:get_text_parts()
+ local wres = {
+ 0, -- spaces rate
+ 0, -- double spaces rate
+ 0, -- non spaces rate
+ 0, -- ascii characters rate
+ 0, -- non-ascii characters rate
+ 0, -- capital characters rate
+ 0, -- numeric characters
+ }
+ for _, p in ipairs(tp) do
+ local stats = p:get_stats()
+ local len = p:get_length()
+
+ if len > 0 then
+ wres[1] = wres[1] + stats['spaces'] / len
+ wres[2] = wres[2] + stats['double_spaces'] / len
+ wres[3] = wres[3] + stats['non_spaces'] / len
+ wres[4] = wres[4] + stats['ascii_characters'] / len
+ wres[5] = wres[5] + stats['non_ascii_characters'] / len
+ wres[6] = wres[6] + stats['capital_letters'] / len
+ wres[7] = wres[7] + stats['numeric_characters'] / len
+ end
+ end
+
+ local ret = {
+ short_words,
+ ret_len,
+ }
+
+ local divisor = 1.0
+ if #tp > 0 then
+ divisor = #tp
+ end
+
+ for _, wr in ipairs(wres) do
+ table.insert(ret, wr / divisor)
+ end
+
+ return ret
+end
+
+local metafunctions = {
+ {
+ cb = meta_size_function,
+ ninputs = 1,
+ names = {
+ "size"
+ },
+ description = 'Describes size of the message',
+ },
+ {
+ cb = meta_images_function,
+ ninputs = 5,
+ -- 1 - number of images,
+ -- 2 - number of png images,
+ -- 3 - number of jpeg images
+ -- 4 - number of large images (> 128 x 128)
+ -- 5 - number of small images (< 128 x 128)
+ names = {
+ 'nimages',
+ 'npng_images',
+ 'njpeg_images',
+ 'nlarge_images',
+ 'nsmall_images'
+ },
+ description = [[Functions for images matching:
+ - number of images,
+ - number of png images,
+ - number of jpeg images
+ - number of large images (> 128 x 128)
+ - number of small images (< 128 x 128)
+]]
+ },
+ {
+ cb = meta_nparts_function,
+ ninputs = 2,
+ -- 1 - number of text parts
+ -- 2 - number of attachments
+ names = {
+ 'ntext_parts',
+ 'nattachments'
+ },
+ description = [[Functions for images matching:
+ - number of text parts
+ - number of attachments
+]]
+ },
+ {
+ cb = meta_encoding_function,
+ ninputs = 2,
+ -- 1 - number of utf parts
+ -- 2 - number of non-utf parts
+ names = {
+ 'nutf_parts',
+ 'nascii_parts'
+ },
+ description = [[Functions for encoding matching:
+ - number of utf parts
+ - number of non-utf parts
+]]
+ },
+ {
+ cb = meta_recipients_function,
+ ninputs = 2,
+ -- 1 - number of mime rcpt
+ -- 2 - number of smtp rcpt
+ names = {
+ 'nmime_rcpt',
+ 'nsmtp_rcpt'
+ },
+ description = [[Functions for recipients data matching:
+ - number of mime rcpt
+ - number of smtp rcpt
+]]
+ },
+ {
+ cb = meta_received_function,
+ ninputs = 4,
+ names = {
+ 'nreceived',
+ 'nreceived_invalid',
+ 'nreceived_bad_time',
+ 'nreceived_secure'
+ },
+ description = [[Functions for received headers data matching:
+ - number of received headers
+ - number of bad received headers
+ - number of skewed time received headers
+ - number of received via secured relays
+]]
+ },
+ {
+ cb = meta_urls_function,
+ ninputs = 1,
+ names = {
+ 'nurls'
+ },
+ description = [[Functions for urls data matching:
+ - number of urls
+]]
+ },
+ {
+ cb = meta_words_function,
+ ninputs = 9,
+ names = {
+ 'avg_words_len',
+ 'nshort_words',
+ 'spaces_rate',
+ 'double_spaces_rate',
+ 'non_spaces_rate',
+ 'ascii_characters_rate',
+ 'non_ascii_characters_rate',
+ 'capital_characters_rate',
+ 'numeric_characters'
+ },
+ description = [[Functions for words data matching:
+ - average length of the words
+ - number of short words
+ - rate of spaces in the text
+ - rate of multiple spaces
+ - rate of non space characters
+ - rate of ascii characters
+ - rate of non-ascii characters
+ - rate of capital letters
+ - rate of numbers
+]]
+ },
+}
+
+local meta_schema = ts.shape {
+ cb = ts.func,
+ ninputs = ts.number,
+ names = ts.array_of(ts.string),
+ description = ts.string:is_optional()
+}
+
+local metatokens_by_name = {}
+
+local function fill_metatokens_by_name()
+ metatokens_by_name = {}
+
+ for _, mt in ipairs(metafunctions) do
+ for i = 1, mt.ninputs do
+ local name = mt.names[i]
+
+ metatokens_by_name[name] = function(task)
+ local results = mt.cb(task)
+ return results[i]
+ end
+ end
+ end
+end
+
+local function calculate_digest()
+ local cr = require "rspamd_cryptobox_hash"
+
+ local h = cr.create()
+ for _, mt in ipairs(metafunctions) do
+ for i = 1, mt.ninputs do
+ local name = mt.names[i]
+ h:update(name)
+ end
+ end
+
+ exports.digest = h:hex()
+end
+
+local function rspamd_gen_metatokens(task, names)
+ local lua_util = require "lua_util"
+ local ipairs = ipairs
+ local metatokens = {}
+
+ if not names then
+ local cached = task:cache_get('metatokens')
+
+ if cached then
+ return cached
+ else
+ for _, mt in ipairs(metafunctions) do
+ local ct = mt.cb(task)
+ for i, tok in ipairs(ct) do
+ lua_util.debugm(N, task, "metatoken: %s = %s",
+ mt.names[i], tok)
+ if tok ~= tok or tok == math.huge then
+ logger.errx(task, 'metatoken %s returned %s; replace it with 0 for sanity',
+ mt.names[i], tok)
+ tok = 0.0
+ end
+ table.insert(metatokens, tok)
+ end
+ end
+
+ task:cache_set('metatokens', metatokens)
+ end
+
+ else
+ for _, n in ipairs(names) do
+ if metatokens_by_name[n] then
+ local tok = metatokens_by_name[n](task)
+ if tok ~= tok or tok == math.huge then
+ logger.errx(task, 'metatoken %s returned %s; replace it with 0 for sanity',
+ n, tok)
+ tok = 0.0
+ end
+ table.insert(metatokens, tok)
+ else
+ logger.errx(task, 'unknown metatoken: %s', n)
+ end
+ end
+ end
+
+ return metatokens
+end
+
+exports.rspamd_gen_metatokens = rspamd_gen_metatokens
+exports.gen_metatokens = rspamd_gen_metatokens
+
+local function rspamd_gen_metatokens_table(task)
+ local metatokens = {}
+
+ for _, mt in ipairs(metafunctions) do
+ local ct = mt.cb(task)
+ for i, tok in ipairs(ct) do
+ if tok ~= tok or tok == math.huge then
+ logger.errx(task, 'metatoken %s returned %s; replace it with 0 for sanity',
+ mt.names[i], tok)
+ tok = 0.0
+ end
+
+ metatokens[mt.names[i]] = tok
+ end
+ end
+
+ return metatokens
+end
+
+exports.rspamd_gen_metatokens_table = rspamd_gen_metatokens_table
+exports.gen_metatokens_table = rspamd_gen_metatokens_table
+
+local function rspamd_count_metatokens()
+ local ipairs = ipairs
+ local total = 0
+ for _, mt in ipairs(metafunctions) do
+ total = total + mt.ninputs
+ end
+
+ return total
+end
+
+exports.rspamd_count_metatokens = rspamd_count_metatokens
+exports.count_metatokens = rspamd_count_metatokens
+exports.version = 1 -- MUST be increased on each change of metatokens
+
+exports.add_metafunction = function(tbl)
+ local ret, err = meta_schema(tbl)
+
+ if not ret then
+ logger.errx('cannot add metafunction: %s', err)
+ else
+ table.insert(metafunctions, tbl)
+ fill_metatokens_by_name()
+ calculate_digest()
+ end
+end
+
+fill_metatokens_by_name()
+calculate_digest()
+
+return exports
diff --git a/lualib/lua_mime.lua b/lualib/lua_mime.lua
new file mode 100644
index 0000000..0f5aa75
--- /dev/null
+++ b/lualib/lua_mime.lua
@@ -0,0 +1,760 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_mime
+-- This module contains helper functions to modify mime parts
+--]]
+
+local logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local rspamd_text = require "rspamd_text"
+local ucl = require "ucl"
+
+local exports = {}
+
+local function newline(task)
+ local t = task:get_newlines_type()
+
+ if t == 'cr' then
+ return '\r'
+ elseif t == 'lf' then
+ return '\n'
+ end
+
+ return '\r\n'
+end
+
+local function do_append_footer(task, part, footer, is_multipart, out, state)
+ local tp = part:get_text()
+ local ct = 'text/plain'
+ local cte = 'quoted-printable'
+ local newline_s = state.newline_s
+
+ if tp:is_html() then
+ ct = 'text/html'
+ end
+
+ local encode_func = function(input)
+ return rspamd_util.encode_qp(input, 80, task:get_newlines_type())
+ end
+
+ if part:get_cte() == '7bit' then
+ cte = '7bit'
+ encode_func = function(input)
+ if type(input) == 'userdata' then
+ return input
+ else
+ return rspamd_text.fromstring(input)
+ end
+ end
+ end
+
+ if is_multipart then
+ out[#out + 1] = string.format('Content-Type: %s; charset=utf-8%s' ..
+ 'Content-Transfer-Encoding: %s',
+ ct, newline_s, cte)
+ out[#out + 1] = ''
+ else
+ state.new_cte = cte
+ end
+
+ local content = tp:get_content('raw_utf') or ''
+ local double_nline = newline_s .. newline_s
+ local nlen = #double_nline
+ -- Hack, if part ends with 2 newline, then we append it after footer
+ if content:sub(-(nlen), nlen + 1) == double_nline then
+ -- content without last newline
+ content = content:sub(-(#newline_s), #newline_s + 1) .. footer
+ out[#out + 1] = { encode_func(content), true }
+ out[#out + 1] = ''
+ else
+ content = content .. footer
+ out[#out + 1] = { encode_func(content), true }
+ out[#out + 1] = ''
+ end
+
+end
+
+--[[[
+-- @function lua_mime.add_text_footer(task, html_footer, text_footer)
+-- Adds a footer to all text parts in a message. It returns a table with the following
+-- fields:
+-- * out: new content (body only)
+-- * need_rewrite_ct: boolean field that means if we must rewrite content type
+-- * new_ct: new content type (type => string, subtype => string)
+-- * new_cte: new content-transfer encoding (string)
+--]]
+exports.add_text_footer = function(task, html_footer, text_footer)
+ local newline_s = newline(task)
+ local state = {
+ newline_s = newline_s
+ }
+ local out = {}
+ local text_parts = task:get_text_parts()
+
+ if not (html_footer or text_footer) or not (text_parts and #text_parts > 0) then
+ return false
+ end
+
+ if html_footer or text_footer then
+ -- We need to take extra care about content-type and cte
+ local ct = task:get_header('Content-Type')
+ if ct then
+ ct = rspamd_util.parse_content_type(ct, task:get_mempool())
+ end
+
+ if ct then
+ if ct.type and ct.type == 'text' then
+ if ct.subtype then
+ if html_footer and (ct.subtype == 'html' or ct.subtype == 'htm') then
+ state.need_rewrite_ct = true
+ elseif text_footer and ct.subtype == 'plain' then
+ state.need_rewrite_ct = true
+ end
+ else
+ if text_footer then
+ state.need_rewrite_ct = true
+ end
+ end
+
+ state.new_ct = ct
+ end
+ else
+
+ if text_parts then
+
+ if #text_parts == 1 then
+ state.need_rewrite_ct = true
+ state.new_ct = {
+ type = 'text',
+ subtype = 'plain'
+ }
+ elseif #text_parts > 1 then
+ -- XXX: in fact, it cannot be
+ state.new_ct = {
+ type = 'multipart',
+ subtype = 'mixed'
+ }
+ end
+ end
+ end
+ end
+
+ local boundaries = {}
+ local cur_boundary
+ for _, part in ipairs(task:get_parts()) do
+ local boundary = part:get_boundary()
+ if part:is_multipart() then
+ if cur_boundary then
+ out[#out + 1] = string.format('--%s',
+ boundaries[#boundaries])
+ end
+
+ boundaries[#boundaries + 1] = boundary or '--XXX'
+ cur_boundary = boundary
+
+ local rh = part:get_raw_headers()
+ if #rh > 0 then
+ out[#out + 1] = { rh, true }
+ end
+ elseif part:is_message() then
+ if boundary then
+ if cur_boundary and boundary ~= cur_boundary then
+ -- Need to close boundary
+ out[#out + 1] = string.format('--%s--%s',
+ boundaries[#boundaries], newline_s)
+ table.remove(boundaries)
+ cur_boundary = nil
+ end
+ out[#out + 1] = string.format('--%s',
+ boundary)
+ end
+
+ out[#out + 1] = { part:get_raw_headers(), true }
+ else
+ local append_footer = false
+ local skip_footer = part:is_attachment()
+
+ local parent = part:get_parent()
+ if parent then
+ local t, st = parent:get_type()
+
+ if t == 'multipart' and st == 'signed' then
+ -- Do not modify signed parts
+ skip_footer = true
+ end
+ end
+ if text_footer and part:is_text() then
+ local tp = part:get_text()
+
+ if not tp:is_html() then
+ append_footer = text_footer
+ end
+ end
+
+ if html_footer and part:is_text() then
+ local tp = part:get_text()
+
+ if tp:is_html() then
+ append_footer = html_footer
+ end
+ end
+
+ if boundary then
+ if cur_boundary and boundary ~= cur_boundary then
+ -- Need to close boundary
+ out[#out + 1] = string.format('--%s--%s',
+ boundaries[#boundaries], newline_s)
+ table.remove(boundaries)
+ cur_boundary = boundary
+ end
+ out[#out + 1] = string.format('--%s',
+ boundary)
+ end
+
+ if append_footer and not skip_footer then
+ do_append_footer(task, part, append_footer,
+ parent and parent:is_multipart(), out, state)
+ else
+ out[#out + 1] = { part:get_raw_headers(), true }
+ out[#out + 1] = { part:get_raw_content(), false }
+ end
+ end
+ end
+
+ -- Close remaining
+ local b = table.remove(boundaries)
+ while b do
+ out[#out + 1] = string.format('--%s--', b)
+ if #boundaries > 0 then
+ out[#out + 1] = ''
+ end
+ b = table.remove(boundaries)
+ end
+
+ state.out = out
+
+ return state
+end
+
+local function do_replacement (task, part, mp, replacements,
+ is_multipart, out, state)
+
+ local tp = part:get_text()
+ local ct = 'text/plain'
+ local cte = 'quoted-printable'
+ local newline_s = state.newline_s
+
+ if tp:is_html() then
+ ct = 'text/html'
+ end
+
+ local encode_func = function(input)
+ return rspamd_util.encode_qp(input, 80, task:get_newlines_type())
+ end
+
+ if part:get_cte() == '7bit' then
+ cte = '7bit'
+ encode_func = function(input)
+ if type(input) == 'userdata' then
+ return input
+ else
+ return rspamd_text.fromstring(input)
+ end
+ end
+ end
+
+ local content = tp:get_content('raw_utf') or rspamd_text.fromstring('')
+ local match_pos = mp:match(content, true)
+
+ if match_pos then
+ -- sort matches and form the table:
+ -- start .. end for inclusion position
+ local matches_flattened = {}
+ for npat, matches in pairs(match_pos) do
+ for _, m in ipairs(matches) do
+ table.insert(matches_flattened, { m, npat })
+ end
+ end
+
+ -- Handle the case of empty match
+ if #matches_flattened == 0 then
+ out[#out + 1] = { part:get_raw_headers(), true }
+ out[#out + 1] = { part:get_raw_content(), false }
+
+ return
+ end
+
+ if is_multipart then
+ out[#out + 1] = { string.format('Content-Type: %s; charset="utf-8"%s' ..
+ 'Content-Transfer-Encoding: %s',
+ ct, newline_s, cte), true }
+ out[#out + 1] = { '', true }
+ else
+ state.new_cte = cte
+ end
+
+ state.has_matches = true
+ -- now sort flattened by start of match and eliminate all overlaps
+ table.sort(matches_flattened, function(m1, m2)
+ return m1[1][1] < m2[1][1]
+ end)
+
+ for i = 1, #matches_flattened - 1 do
+ local st = matches_flattened[i][1][1] -- current start of match
+ local e = matches_flattened[i][1][2] -- current end of match
+ local max_npat = matches_flattened[i][2]
+ for j = i + 1, #matches_flattened do
+ if matches_flattened[j][1][1] == st then
+ -- overlap
+ if matches_flattened[j][1][2] > e then
+ -- larger exclusion and switch replacement
+ e = matches_flattened[j][1][2]
+ max_npat = matches_flattened[j][2]
+ end
+ else
+ break
+ end
+ end
+ -- Maximum overlap for all matches
+ for j = i, #matches_flattened do
+ if matches_flattened[j][1][1] == st then
+ if e > matches_flattened[j][1][2] then
+ matches_flattened[j][1][2] = e
+ matches_flattened[j][2] = max_npat
+ end
+ else
+ break
+ end
+ end
+ end
+ -- Off-by one: match returns 0 based positions while we use 1 based in Lua
+ for _, m in ipairs(matches_flattened) do
+ m[1][1] = m[1][1] + 1
+ m[1][2] = m[1][2] + 1
+ end
+
+ -- Now flattened match table is sorted by start pos and has the maximum overlapped pattern
+ -- Matches with the same start and end are covering the same replacement
+ -- e.g. we had something like [1 .. 2] -> replacement 1 and [1 .. 4] -> replacement 2
+ -- after flattening we should have [1 .. 4] -> 2 and [1 .. 4] -> 2
+ -- we can safely ignore those duplicates in the following code
+
+ local cur_start = 1
+ local fragments = {}
+ for _, m in ipairs(matches_flattened) do
+ if m[1][1] >= cur_start then
+ fragments[#fragments + 1] = content:sub(cur_start, m[1][1] - 1)
+ fragments[#fragments + 1] = replacements[m[2]]
+ cur_start = m[1][2] -- end of match
+ end
+ end
+
+ -- last part
+ if cur_start < #content then
+ fragments[#fragments + 1] = content:span(cur_start)
+ end
+
+ -- Final stuff
+ out[#out + 1] = { encode_func(rspamd_text.fromtable(fragments)), false }
+ else
+ -- No matches
+ out[#out + 1] = { part:get_raw_headers(), true }
+ out[#out + 1] = { part:get_raw_content(), false }
+ end
+end
+
+--[[[
+-- @function lua_mime.multipattern_text_replace(task, mp, replacements)
+-- Replaces text according to multipattern matches. It returns a table with the following
+-- fields:
+-- * out: new content (body only)
+-- * need_rewrite_ct: boolean field that means if we must rewrite content type
+-- * new_ct: new content type (type => string, subtype => string)
+-- * new_cte: new content-transfer encoding (string)
+--]]
+exports.multipattern_text_replace = function(task, mp, replacements)
+ local newline_s = newline(task)
+ local state = {
+ newline_s = newline_s
+ }
+ local out = {}
+ local text_parts = task:get_text_parts()
+
+ if not mp or not (text_parts and #text_parts > 0) then
+ return false
+ end
+
+ -- We need to take extra care about content-type and cte
+ local ct = task:get_header('Content-Type')
+ if ct then
+ ct = rspamd_util.parse_content_type(ct, task:get_mempool())
+ end
+
+ if ct then
+ if ct.type and ct.type == 'text' then
+ state.need_rewrite_ct = true
+ state.new_ct = ct
+ end
+ else
+ -- No explicit CT, need to guess
+ if text_parts then
+ if #text_parts == 1 then
+ state.need_rewrite_ct = true
+ state.new_ct = {
+ type = 'text',
+ subtype = 'plain'
+ }
+ elseif #text_parts > 1 then
+ -- XXX: in fact, it cannot be
+ state.new_ct = {
+ type = 'multipart',
+ subtype = 'mixed'
+ }
+ end
+ end
+ end
+
+ local boundaries = {}
+ local cur_boundary
+ for _, part in ipairs(task:get_parts()) do
+ local boundary = part:get_boundary()
+ if part:is_multipart() then
+ if cur_boundary then
+ out[#out + 1] = { string.format('--%s',
+ boundaries[#boundaries]), true }
+ end
+
+ boundaries[#boundaries + 1] = boundary or '--XXX'
+ cur_boundary = boundary
+
+ local rh = part:get_raw_headers()
+ if #rh > 0 then
+ out[#out + 1] = { rh, true }
+ end
+ elseif part:is_message() then
+ if boundary then
+ if cur_boundary and boundary ~= cur_boundary then
+ -- Need to close boundary
+ out[#out + 1] = { string.format('--%s--',
+ boundaries[#boundaries]), true }
+ table.remove(boundaries)
+ cur_boundary = nil
+ end
+ out[#out + 1] = { string.format('--%s',
+ boundary), true }
+ end
+
+ out[#out + 1] = { part:get_raw_headers(), true }
+ else
+ local skip_replacement = part:is_attachment()
+
+ local parent = part:get_parent()
+ if parent then
+ local t, st = parent:get_type()
+
+ if t == 'multipart' and st == 'signed' then
+ -- Do not modify signed parts
+ skip_replacement = true
+ end
+ end
+ if not part:is_text() then
+ skip_replacement = true
+ end
+
+ if boundary then
+ if cur_boundary and boundary ~= cur_boundary then
+ -- Need to close boundary
+ out[#out + 1] = { string.format('--%s--',
+ boundaries[#boundaries]), true }
+ table.remove(boundaries)
+ cur_boundary = boundary
+ end
+ out[#out + 1] = { string.format('--%s',
+ boundary), true }
+ end
+
+ if not skip_replacement then
+ do_replacement(task, part, mp, replacements,
+ parent and parent:is_multipart(), out, state)
+ else
+ -- Append as is
+ out[#out + 1] = { part:get_raw_headers(), true }
+ out[#out + 1] = { part:get_raw_content(), false }
+ end
+ end
+ end
+
+ -- Close remaining
+ local b = table.remove(boundaries)
+ while b do
+ out[#out + 1] = { string.format('--%s--', b), true }
+ if #boundaries > 0 then
+ out[#out + 1] = { '', true }
+ end
+ b = table.remove(boundaries)
+ end
+
+ state.out = out
+
+ return state
+end
+
+--[[[
+-- @function lua_mime.modify_headers(task, {add = {hname = {value = 'value', order = 1}}, remove = {hname = {1,2}}})
+-- Adds/removes headers both internal and in the milter reply
+-- Mode defines to be compatible with Rspamd <=3.2 and is the default (equal to 'compat')
+--]]
+exports.modify_headers = function(task, hdr_alterations, mode)
+ -- Assume default mode compatibility
+ if not mode then
+ mode = 'compat'
+ end
+ local add = hdr_alterations.add or {}
+ local remove = hdr_alterations.remove or {}
+
+ local add_headers = {} -- For Milter reply
+ local hdr_flattened = {} -- For C API
+
+ local function flatten_add_header(hname, hdr)
+ if not add_headers[hname] then
+ add_headers[hname] = {}
+ end
+ if not hdr_flattened[hname] then
+ hdr_flattened[hname] = { add = {} }
+ end
+ local add_tbl = hdr_flattened[hname].add
+ if hdr.value then
+ table.insert(add_headers[hname], {
+ order = (tonumber(hdr.order) or -1),
+ value = hdr.value,
+ })
+ table.insert(add_tbl, { tonumber(hdr.order) or -1, hdr.value })
+ elseif type(hdr) == 'table' then
+ for _, v in ipairs(hdr) do
+ flatten_add_header(hname, v)
+ end
+ elseif type(hdr) == 'string' then
+ table.insert(add_headers[hname], {
+ order = -1,
+ value = hdr,
+ })
+ table.insert(add_tbl, { -1, hdr })
+ else
+ logger.errx(task, 'invalid modification of header: %s', hdr)
+ end
+
+ if mode == 'compat' and #add_headers[hname] == 1 then
+ -- Switch to the compatibility mode
+ add_headers[hname] = add_headers[hname][1]
+ end
+ end
+ if hdr_alterations.order then
+ -- Get headers alterations ordered
+ for _, hname in ipairs(hdr_alterations.order) do
+ flatten_add_header(hname, add[hname])
+ end
+ else
+ for hname, hdr in pairs(add) do
+ flatten_add_header(hname, hdr)
+ end
+ end
+
+ for hname, hdr in pairs(remove) do
+ if not hdr_flattened[hname] then
+ hdr_flattened[hname] = { remove = {} }
+ end
+ if not hdr_flattened[hname].remove then
+ hdr_flattened[hname].remove = {}
+ end
+ local remove_tbl = hdr_flattened[hname].remove
+ if type(hdr) == 'number' then
+ table.insert(remove_tbl, hdr)
+ else
+ for _, num in ipairs(hdr) do
+ table.insert(remove_tbl, num)
+ end
+ end
+ end
+
+ if mode == 'compat' then
+ -- Clear empty alterations in the compat mode
+ if add_headers and not next(add_headers) then
+ add_headers = nil
+ end
+ if hdr_alterations.remove and not next(hdr_alterations.remove) then
+ hdr_alterations.remove = nil
+ end
+ end
+ task:set_milter_reply({
+ add_headers = add_headers,
+ remove_headers = hdr_alterations.remove
+ })
+
+ for hname, flat_rules in pairs(hdr_flattened) do
+ task:modify_header(hname, flat_rules)
+ end
+end
+
+--[[[
+-- @function lua_mime.message_to_ucl(task, [stringify_content])
+-- Exports a message to an ucl object
+--]]
+exports.message_to_ucl = function(task, stringify_content)
+ local E = {}
+
+ local maybe_stringify_f = stringify_content and
+ tostring or function(t)
+ return t
+ end
+ local result = {
+ size = task:get_size(),
+ digest = task:get_digest(),
+ newlines = task:get_newlines_type(),
+ headers = task:get_headers(true)
+ }
+
+ -- Utility to convert ip addr to a string or nil if invalid/absent
+ local function maybe_stringify_ip(addr)
+ if addr and addr:is_valid() then
+ return addr:to_string()
+ end
+
+ return nil
+ end
+
+ -- Envelope (smtp) information from email (nil if empty)
+ result.envelope = {
+ from_smtp = (task:get_from('smtp') or E)[1],
+ recipients_smtp = task:get_recipients('smtp'),
+ helo = task:get_helo(),
+ hostname = task:get_hostname(),
+ client_ip = maybe_stringify_ip(task:get_client_ip()),
+ from_ip = maybe_stringify_ip(task:get_from_ip()),
+ }
+ if not next(result.envelope) then
+ result.envelope = ucl.null
+ end
+
+ local parts = task:get_parts() or E
+ result.parts = {}
+ for _, part in ipairs(parts) do
+ if not part:is_multipart() and not part:is_message() then
+ local p = {
+ size = part:get_length(),
+ type = string.format('%s/%s', part:get_type()),
+ detected_type = string.format('%s/%s', part:get_detected_type()),
+ filename = part:get_filename(),
+ content = maybe_stringify_f(part:get_content()),
+ headers = part:get_headers(true) or E,
+ boundary = part:get_enclosing_boundary(),
+ }
+ table.insert(result.parts, p)
+ else
+ -- Service part: multipart container or message/rfc822
+ local p = {
+ type = string.format('%s/%s', part:get_type()),
+ headers = part:get_headers(true) or E,
+ boundary = part:get_enclosing_boundary(),
+ size = 0,
+ }
+
+ if part:is_multipart() then
+ p.multipart_boundary = part:get_boundary()
+ end
+
+ table.insert(result.parts, p)
+ end
+ end
+
+ return result
+end
+
+--[[[
+-- @function lua_mime.message_to_ucl_schema()
+-- Returns schema for a message to verify result/document fields
+--]]
+exports.message_to_ucl_schema = function()
+ local ts = require("tableshape").types
+
+ local function headers_schema()
+ return ts.shape {
+ order = ts.integer:describe('Header order in a message'),
+ raw = ts.string:describe('Raw header value'):is_optional(),
+ empty_separator = ts.boolean:describe('Whether header has an empty separator'),
+ separator = ts.string:describe('Separator between a header and a value'),
+ decoded = ts.string:describe('Decoded value'):is_optional(),
+ value = ts.string:describe('Decoded value'):is_optional(),
+ name = ts.string:describe('Header name'),
+ tab_separated = ts.boolean:describe('Whether header has tab as a separator')
+ }
+ end
+
+ local function part_schema()
+ return ts.shape {
+ content = ts.string:describe('Decoded content'):is_optional(),
+ multipart_boundary = ts.string:describe('Multipart service boundary'):is_optional(),
+ size = ts.integer:describe('Size of the part'),
+ type = ts.string:describe('Announced type'):is_optional(),
+ detected_type = ts.string:describe('Detected type'):is_optional(),
+ boundary = ts.string:describe('Eclosing boundary'):is_optional(),
+ filename = ts.string:describe('File name for attachments'):is_optional(),
+ headers = ts.array_of(headers_schema()):describe('Part headers'),
+ }
+ end
+
+ local function email_addr_schema()
+ return ts.shape {
+ addr = ts.string:describe('Parsed address'):is_optional(),
+ raw = ts.string:describe('Raw address'),
+ flags = ts.shape {
+ valid = ts.boolean:describe('Valid address'):is_optional(),
+ ip = ts.boolean:describe('IP like address'):is_optional(),
+ braced = ts.boolean:describe('Have braces around address'):is_optional(),
+ quoted = ts.boolean:describe('Have quotes around address'):is_optional(),
+ empty = ts.boolean:describe('Empty address'):is_optional(),
+ backslash = ts.boolean:describe('Backslash in address'):is_optional(),
+ ['8bit'] = ts.boolean:describe('8 bit characters in address'):is_optional(),
+ },
+ user = ts.string:describe('Parsed user part'):is_optional(),
+ name = ts.string:describe('Displayed name'):is_optional(),
+ domain = ts.string:describe('Parsed domain part'):is_optional(),
+ }
+ end
+ local function envelope_schema()
+ return ts.shape {
+ from_smtp = email_addr_schema():describe('SMTP from'):is_optional(),
+ recipients_smtp = ts.array_of(email_addr_schema()):describe('SMTP recipients'):is_optional(),
+ helo = ts.string:describe('SMTP Helo'):is_optional(),
+ hostname = ts.string:describe('Sender hostname'):is_optional(),
+ client_ip = ts.string:describe('Client ip'):is_optional(),
+ from_ip = ts.string:describe('Sender ip'):is_optional(),
+ }
+ end
+
+ return ts.shape {
+ headers = ts.array_of(headers_schema()),
+ parts = ts.array_of(part_schema()),
+ digest = ts.pattern(string.format('^%s$', string.rep('%x', 32)))
+ :describe('Message digest'),
+ newlines = ts.one_of({ "cr", "lf", "crlf" }):describe('Newlines type'),
+ size = ts.integer:describe('Size of the message in bytes'),
+ envelope = envelope_schema()
+ }
+end
+
+return exports
diff --git a/lualib/lua_mime_types.lua b/lualib/lua_mime_types.lua
new file mode 100644
index 0000000..ba55f97
--- /dev/null
+++ b/lualib/lua_mime_types.lua
@@ -0,0 +1,745 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_mime_types
+-- This module contains mime types list
+--]]
+
+local exports = {}
+
+-- All mime extensions with corresponding content types
+exports.full_extensions_map = {
+ { "323", "text/h323" },
+ { "3g2", "video/3gpp2" },
+ { "3gp", "video/3gpp" },
+ { "3gp2", "video/3gpp2" },
+ { "3gpp", "video/3gpp" },
+ { "7z", { "application/x-7z-compressed", "application/7z" } },
+ { "aa", "audio/audible" },
+ { "AAC", "audio/aac" },
+ { "aaf", "application/octet-stream" },
+ { "aax", "audio/vnd.audible.aax" },
+ { "ac3", "audio/ac3" },
+ { "aca", "application/octet-stream" },
+ { "accda", "application/msaccess.addin" },
+ { "accdb", "application/msaccess" },
+ { "accdc", "application/msaccess.cab" },
+ { "accde", "application/msaccess" },
+ { "accdr", "application/msaccess.runtime" },
+ { "accdt", "application/msaccess" },
+ { "accdw", "application/msaccess.webapplication" },
+ { "accft", "application/msaccess.ftemplate" },
+ { "acx", "application/internet-property-stream" },
+ { "AddIn", "text/xml" },
+ { "ade", "application/msaccess" },
+ { "adobebridge", "application/x-bridge-url" },
+ { "adp", "application/msaccess" },
+ { "ADT", "audio/vnd.dlna.adts" },
+ { "ADTS", "audio/aac" },
+ { "afm", "application/octet-stream" },
+ { "ai", "application/postscript" },
+ { "aif", "audio/aiff" },
+ { "aifc", "audio/aiff" },
+ { "aiff", "audio/aiff" },
+ { "air", "application/vnd.adobe.air-application-installer-package+zip" },
+ { "amc", "application/mpeg" },
+ { "anx", "application/annodex" },
+ { "apk", "application/vnd.android.package-archive" },
+ { "application", "application/x-ms-application" },
+ { "art", "image/x-jg" },
+ { "asa", "application/xml" },
+ { "asax", "application/xml" },
+ { "ascx", "application/xml" },
+ { "asd", "application/octet-stream" },
+ { "asf", "video/x-ms-asf" },
+ { "ashx", "application/xml" },
+ { "asi", "application/octet-stream" },
+ { "asm", "text/plain" },
+ { "asmx", "application/xml" },
+ { "aspx", "application/xml" },
+ { "asr", "video/x-ms-asf" },
+ { "asx", "video/x-ms-asf" },
+ { "atom", "application/atom+xml" },
+ { "au", "audio/basic" },
+ { "avi", "video/x-msvideo" },
+ { "axa", "audio/annodex" },
+ { "axs", "application/olescript" },
+ { "axv", "video/annodex" },
+ { "bas", "text/plain" },
+ { "bcpio", "application/x-bcpio" },
+ { "bin", "application/octet-stream" },
+ { "bmp", { "image/bmp", "image/x-ms-bmp" } },
+ { "c", "text/plain" },
+ { "cab", "application/octet-stream" },
+ { "caf", "audio/x-caf" },
+ { "calx", "application/vnd.ms-office.calx" },
+ { "cat", "application/vnd.ms-pki.seccat" },
+ { "cc", "text/plain" },
+ { "cd", "text/plain" },
+ { "cdda", "audio/aiff" },
+ { "cdf", "application/x-cdf" },
+ { "cer", "application/x-x509-ca-cert" },
+ { "cfg", "text/plain" },
+ { "chm", "application/octet-stream" },
+ { "class", "application/x-java-applet" },
+ { "clp", "application/x-msclip" },
+ { "cmd", "text/plain" },
+ { "cmx", "image/x-cmx" },
+ { "cnf", "text/plain" },
+ { "cod", "image/cis-cod" },
+ { "config", "application/xml" },
+ { "contact", "text/x-ms-contact" },
+ { "coverage", "application/xml" },
+ { "cpio", "application/x-cpio" },
+ { "cpp", "text/plain" },
+ { "crd", "application/x-mscardfile" },
+ { "crl", "application/pkix-crl" },
+ { "crt", "application/x-x509-ca-cert" },
+ { "cs", "text/plain" },
+ { "csdproj", "text/plain" },
+ { "csh", "application/x-csh" },
+ { "csproj", "text/plain" },
+ { "css", "text/css" },
+ { "csv", { "application/vnd.ms-excel", "text/csv", "text/plain" } },
+ { "cur", "application/octet-stream" },
+ { "cxx", "text/plain" },
+ { "dat", { "application/octet-stream", "application/ms-tnef" } },
+ { "datasource", "application/xml" },
+ { "dbproj", "text/plain" },
+ { "dcr", "application/x-director" },
+ { "def", "text/plain" },
+ { "deploy", "application/octet-stream" },
+ { "der", "application/x-x509-ca-cert" },
+ { "dgml", "application/xml" },
+ { "dib", "image/bmp" },
+ { "dif", "video/x-dv" },
+ { "dir", "application/x-director" },
+ { "disco", "text/xml" },
+ { "divx", "video/divx" },
+ { "dll", "application/x-msdownload" },
+ { "dll.config", "text/xml" },
+ { "dlm", "text/dlm" },
+ { "doc", "application/msword" },
+ { "docm", "application/vnd.ms-word.document.macroEnabled.12" },
+ { "docx", {
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/msword",
+ "application/vnd.ms-word.document.12",
+ "application/octet-stream",
+ } },
+ { "dot", "application/msword" },
+ { "dotm", "application/vnd.ms-word.template.macroEnabled.12" },
+ { "dotx", "application/vnd.openxmlformats-officedocument.wordprocessingml.template" },
+ { "dsp", "application/octet-stream" },
+ { "dsw", "text/plain" },
+ { "dtd", "text/xml" },
+ { "dtsConfig", "text/xml" },
+ { "dv", "video/x-dv" },
+ { "dvi", "application/x-dvi" },
+ { "dwf", "drawing/x-dwf" },
+ { "dwg", { "application/acad", "image/vnd.dwg" } },
+ { "dwp", "application/octet-stream" },
+ { "dxf", "application/x-dxf" },
+ { "dxr", "application/x-director" },
+ { "eml", "message/rfc822" },
+ { "emz", "application/octet-stream" },
+ { "eot", "application/vnd.ms-fontobject" },
+ { "eps", "application/postscript" },
+ { "etl", "application/etl" },
+ { "etx", "text/x-setext" },
+ { "evy", "application/envoy" },
+ { "exe", {
+ "application/x-dosexec",
+ "application/x-msdownload",
+ "application/x-executable",
+ } },
+ { "exe.config", "text/xml" },
+ { "fdf", "application/vnd.fdf" },
+ { "fif", "application/fractals" },
+ { "filters", "application/xml" },
+ { "fla", "application/octet-stream" },
+ { "flac", "audio/flac" },
+ { "flr", "x-world/x-vrml" },
+ { "flv", "video/x-flv" },
+ { "fsscript", "application/fsharp-script" },
+ { "fsx", "application/fsharp-script" },
+ { "generictest", "application/xml" },
+ { "gif", "image/gif" },
+ { "gpx", "application/gpx+xml" },
+ { "group", "text/x-ms-group" },
+ { "gsm", "audio/x-gsm" },
+ { "gtar", "application/x-gtar" },
+ { "gz", { "application/gzip", "application/x-gzip", "application/tlsrpt+gzip" } },
+ { "h", "text/plain" },
+ { "hdf", "application/x-hdf" },
+ { "hdml", "text/x-hdml" },
+ { "hhc", "application/x-oleobject" },
+ { "hhk", "application/octet-stream" },
+ { "hhp", "application/octet-stream" },
+ { "hlp", "application/winhlp" },
+ { "hpp", "text/plain" },
+ { "hqx", "application/mac-binhex40" },
+ { "hta", "application/hta" },
+ { "htc", "text/x-component" },
+ { "htm", "text/html" },
+ { "html", "text/html" },
+ { "htt", "text/webviewhtml" },
+ { "hxa", "application/xml" },
+ { "hxc", "application/xml" },
+ { "hxd", "application/octet-stream" },
+ { "hxe", "application/xml" },
+ { "hxf", "application/xml" },
+ { "hxh", "application/octet-stream" },
+ { "hxi", "application/octet-stream" },
+ { "hxk", "application/xml" },
+ { "hxq", "application/octet-stream" },
+ { "hxr", "application/octet-stream" },
+ { "hxs", "application/octet-stream" },
+ { "hxt", "text/html" },
+ { "hxv", "application/xml" },
+ { "hxw", "application/octet-stream" },
+ { "hxx", "text/plain" },
+ { "i", "text/plain" },
+ { "ico", "image/x-icon" },
+ { "ics", { "text/calendar", "application/ics", "application/octet-stream" } },
+ { "idl", "text/plain" },
+ { "ief", "image/ief" },
+ { "iii", "application/x-iphone" },
+ { "inc", "text/plain" },
+ { "inf", "application/octet-stream" },
+ { "ini", "text/plain" },
+ { "inl", "text/plain" },
+ { "ins", "application/x-internet-signup" },
+ { "ipa", "application/x-itunes-ipa" },
+ { "ipg", "application/x-itunes-ipg" },
+ { "ipproj", "text/plain" },
+ { "ipsw", "application/x-itunes-ipsw" },
+ { "iqy", "text/x-ms-iqy" },
+ { "isp", "application/x-internet-signup" },
+ { "ite", "application/x-itunes-ite" },
+ { "itlp", "application/x-itunes-itlp" },
+ { "itms", "application/x-itunes-itms" },
+ { "itpc", "application/x-itunes-itpc" },
+ { "IVF", "video/x-ivf" },
+ { "jar", "application/java-archive" },
+ { "java", "application/octet-stream" },
+ { "jck", "application/liquidmotion" },
+ { "jcz", "application/liquidmotion" },
+ { "jfif", { "image/jpeg", "image/pjpeg" } },
+ { "jnlp", "application/x-java-jnlp-file" },
+ { "jpb", "application/octet-stream" },
+ { "jpe", { "image/jpeg", "image/pjpeg" } },
+ { "jpeg", { "image/jpeg", "image/pjpeg" } },
+ { "jpg", { "image/jpeg", "image/pjpeg" } },
+ { "js", "application/javascript" },
+ { "json", "application/json" },
+ { "jsx", "text/jscript" },
+ { "jsxbin", "text/plain" },
+ { "latex", "application/x-latex" },
+ { "library-ms", "application/windows-library+xml" },
+ { "lit", "application/x-ms-reader" },
+ { "loadtest", "application/xml" },
+ { "lpk", "application/octet-stream" },
+ { "lsf", "video/x-la-asf" },
+ { "lst", "text/plain" },
+ { "lsx", "video/x-la-asf" },
+ { "lzh", "application/octet-stream" },
+ { "m13", "application/x-msmediaview" },
+ { "m14", "application/x-msmediaview" },
+ { "m1v", "video/mpeg" },
+ { "m2t", "video/vnd.dlna.mpeg-tts" },
+ { "m2ts", "video/vnd.dlna.mpeg-tts" },
+ { "m2v", "video/mpeg" },
+ { "m3u", "audio/x-mpegurl" },
+ { "m3u8", "audio/x-mpegurl" },
+ { "m4a", { "audio/m4a", "audio/x-m4a" } },
+ { "m4b", "audio/m4b" },
+ { "m4p", "audio/m4p" },
+ { "m4r", "audio/x-m4r" },
+ { "m4v", "video/x-m4v" },
+ { "mac", "image/x-macpaint" },
+ { "mak", "text/plain" },
+ { "man", "application/x-troff-man" },
+ { "manifest", "application/x-ms-manifest" },
+ { "map", "text/plain" },
+ { "master", "application/xml" },
+ { "mbox", "application/mbox" },
+ { "mda", "application/msaccess" },
+ { "mdb", "application/x-msaccess" },
+ { "mde", "application/msaccess" },
+ { "mdp", "application/octet-stream" },
+ { "me", "application/x-troff-me" },
+ { "mfp", "application/x-shockwave-flash" },
+ { "mht", "message/rfc822" },
+ { "mhtml", "message/rfc822" },
+ { "mid", "audio/mid" },
+ { "midi", "audio/mid" },
+ { "mix", "application/octet-stream" },
+ { "mk", "text/plain" },
+ { "mmf", "application/x-smaf" },
+ { "mno", "text/xml" },
+ { "mny", "application/x-msmoney" },
+ { "mod", "video/mpeg" },
+ { "mov", "video/quicktime" },
+ { "movie", "video/x-sgi-movie" },
+ { "mp2", "video/mpeg" },
+ { "mp2v", "video/mpeg" },
+ { "mp3", { "audio/mpeg", "audio/mpeg3", "audio/mp3", "audio/x-mpeg-3" } },
+ { "mp4", "video/mp4" },
+ { "mp4v", "video/mp4" },
+ { "mpa", "video/mpeg" },
+ { "mpe", "video/mpeg" },
+ { "mpeg", "video/mpeg" },
+ { "mpf", "application/vnd.ms-mediapackage" },
+ { "mpg", "video/mpeg" },
+ { "mpp", "application/vnd.ms-project" },
+ { "mpv2", "video/mpeg" },
+ { "mqv", "video/quicktime" },
+ { "ms", "application/x-troff-ms" },
+ { "msg", "application/vnd.ms-outlook" },
+ { "msi", { "application/x-msi", "application/octet-stream" } },
+ { "mso", "application/octet-stream" },
+ { "mts", "video/vnd.dlna.mpeg-tts" },
+ { "mtx", "application/xml" },
+ { "mvb", "application/x-msmediaview" },
+ { "mvc", "application/x-miva-compiled" },
+ { "mxp", "application/x-mmxp" },
+ { "nc", "application/x-netcdf" },
+ { "nsc", "video/x-ms-asf" },
+ { "nws", "message/rfc822" },
+ { "ocx", "application/octet-stream" },
+ { "oda", "application/oda" },
+ { "odb", "application/vnd.oasis.opendocument.database" },
+ { "odc", "application/vnd.oasis.opendocument.chart" },
+ { "odf", "application/vnd.oasis.opendocument.formula" },
+ { "odg", "application/vnd.oasis.opendocument.graphics" },
+ { "odh", "text/plain" },
+ { "odi", "application/vnd.oasis.opendocument.image" },
+ { "odl", "text/plain" },
+ { "odm", "application/vnd.oasis.opendocument.text-master" },
+ { "odp", "application/vnd.oasis.opendocument.presentation" },
+ { "ods", "application/vnd.oasis.opendocument.spreadsheet" },
+ { "odt", "application/vnd.oasis.opendocument.text" },
+ { "oga", "audio/ogg" },
+ { "ogg", "audio/ogg" },
+ { "ogv", "video/ogg" },
+ { "ogx", "application/ogg" },
+ { "one", "application/onenote" },
+ { "onea", "application/onenote" },
+ { "onepkg", "application/onenote" },
+ { "onetmp", "application/onenote" },
+ { "onetoc", "application/onenote" },
+ { "onetoc2", "application/onenote" },
+ { "opus", "audio/ogg" },
+ { "orderedtest", "application/xml" },
+ { "osdx", "application/opensearchdescription+xml" },
+ { "otf", "application/font-sfnt" },
+ { "otg", "application/vnd.oasis.opendocument.graphics-template" },
+ { "oth", "application/vnd.oasis.opendocument.text-web" },
+ { "otp", "application/vnd.oasis.opendocument.presentation-template" },
+ { "ots", "application/vnd.oasis.opendocument.spreadsheet-template" },
+ { "ott", "application/vnd.oasis.opendocument.text-template" },
+ { "oxt", "application/vnd.openofficeorg.extension" },
+ { "p10", "application/pkcs10" },
+ { "p12", "application/x-pkcs12" },
+ { "p7b", "application/x-pkcs7-certificates" },
+ { "p7c", "application/pkcs7-mime" },
+ { "p7m", "application/pkcs7-mime", "application/x-pkcs7-mime" },
+ { "p7r", "application/x-pkcs7-certreqresp" },
+ { "p7s", { "application/pkcs7-signature", "application/x-pkcs7-signature", "text/plain" } },
+ { "pbm", "image/x-portable-bitmap" },
+ { "pcast", "application/x-podcast" },
+ { "pct", "image/pict" },
+ { "pcx", "application/octet-stream" },
+ { "pcz", "application/octet-stream" },
+ { "pdf", "application/pdf" },
+ { "pfb", "application/octet-stream" },
+ { "pfm", "application/octet-stream" },
+ { "pfx", "application/x-pkcs12" },
+ { "pgm", "image/x-portable-graymap" },
+ { "pic", "image/pict" },
+ { "pict", "image/pict" },
+ { "pkgdef", "text/plain" },
+ { "pkgundef", "text/plain" },
+ { "pko", "application/vnd.ms-pki.pko" },
+ { "pls", "audio/scpls" },
+ { "pma", "application/x-perfmon" },
+ { "pmc", "application/x-perfmon" },
+ { "pml", "application/x-perfmon" },
+ { "pmr", "application/x-perfmon" },
+ { "pmw", "application/x-perfmon" },
+ { "png", "image/png" },
+ { "pnm", "image/x-portable-anymap" },
+ { "pnt", "image/x-macpaint" },
+ { "pntg", "image/x-macpaint" },
+ { "pnz", "image/png" },
+ { "pot", "application/vnd.ms-powerpoint" },
+ { "potm", "application/vnd.ms-powerpoint.template.macroEnabled.12" },
+ { "potx", "application/vnd.openxmlformats-officedocument.presentationml.template" },
+ { "ppa", "application/vnd.ms-powerpoint" },
+ { "ppam", "application/vnd.ms-powerpoint.addin.macroEnabled.12" },
+ { "ppm", "image/x-portable-pixmap" },
+ { "pps", "application/vnd.ms-powerpoint" },
+ { "ppsm", "application/vnd.ms-powerpoint.slideshow.macroEnabled.12" },
+ { "ppsx", "application/vnd.openxmlformats-officedocument.presentationml.slideshow" },
+ { "ppt", "application/vnd.ms-powerpoint" },
+ { "pptm", "application/vnd.ms-powerpoint.presentation.macroEnabled.12" },
+ { "pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation" },
+ { "prf", "application/pics-rules" },
+ { "prm", "application/octet-stream" },
+ { "prx", "application/octet-stream" },
+ { "ps", "application/postscript" },
+ { "psc1", "application/PowerShell" },
+ { "psd", "application/octet-stream" },
+ { "psess", "application/xml" },
+ { "psm", "application/octet-stream" },
+ { "psp", "application/octet-stream" },
+ { "pst", "application/vnd.ms-outlook" },
+ { "pub", "application/x-mspublisher" },
+ { "pwz", "application/vnd.ms-powerpoint" },
+ { "qht", "text/x-html-insertion" },
+ { "qhtm", "text/x-html-insertion" },
+ { "qt", "video/quicktime" },
+ { "qti", "image/x-quicktime" },
+ { "qtif", "image/x-quicktime" },
+ { "qtl", "application/x-quicktimeplayer" },
+ { "qxd", "application/octet-stream" },
+ { "ra", "audio/x-pn-realaudio" },
+ { "ram", "audio/x-pn-realaudio" },
+ { "rar", { "application/x-rar-compressed", "application/x-rar", "application/rar", "application/octet-stream" } },
+ { "ras", "image/x-cmu-raster" },
+ { "rat", "application/rat-file" },
+ { "rc", "text/plain" },
+ { "rc2", "text/plain" },
+ { "rct", "text/plain" },
+ { "rdlc", "application/xml" },
+ { "reg", "text/plain" },
+ { "resx", "application/xml" },
+ { "rf", "image/vnd.rn-realflash" },
+ { "rgb", "image/x-rgb" },
+ { "rgs", "text/plain" },
+ { "rm", "application/vnd.rn-realmedia" },
+ { "rmi", "audio/mid" },
+ { "rmp", "application/vnd.rn-rn_music_package" },
+ { "roff", "application/x-troff" },
+ { "rpm", "audio/x-pn-realaudio-plugin" },
+ { "rqy", "text/x-ms-rqy" },
+ { "rtf", { "application/rtf", "application/msword", "text/richtext", "text/rtf" } },
+ { "rtx", "text/richtext" },
+ { "rvt", "application/octet-stream" },
+ { "ruleset", "application/xml" },
+ { "s", "text/plain" },
+ { "safariextz", "application/x-safari-safariextz" },
+ { "scd", "application/x-msschedule" },
+ { "scr", "text/plain" },
+ { "sct", "text/scriptlet" },
+ { "sd2", "audio/x-sd2" },
+ { "sdp", "application/sdp" },
+ { "sea", "application/octet-stream" },
+ { "searchConnector-ms", "application/windows-search-connector+xml" },
+ { "setpay", "application/set-payment-initiation" },
+ { "setreg", "application/set-registration-initiation" },
+ { "settings", "application/xml" },
+ { "sgimb", "application/x-sgimb" },
+ { "sgml", "text/sgml" },
+ { "sh", "application/x-sh" },
+ { "shar", "application/x-shar" },
+ { "shtml", "text/html" },
+ { "sit", "application/x-stuffit" },
+ { "sitemap", "application/xml" },
+ { "skin", "application/xml" },
+ { "skp", "application/x-koan" },
+ { "sldm", "application/vnd.ms-powerpoint.slide.macroEnabled.12" },
+ { "sldx", "application/vnd.openxmlformats-officedocument.presentationml.slide" },
+ { "slk", "application/vnd.ms-excel" },
+ { "sln", "text/plain" },
+ { "slupkg-ms", "application/x-ms-license" },
+ { "smd", "audio/x-smd" },
+ { "smi", "application/octet-stream" },
+ { "smx", "audio/x-smd" },
+ { "smz", "audio/x-smd" },
+ { "snd", "audio/basic" },
+ { "snippet", "application/xml" },
+ { "snp", "application/octet-stream" },
+ { "sol", "text/plain" },
+ { "sor", "text/plain" },
+ { "spc", "application/x-pkcs7-certificates" },
+ { "spl", "application/futuresplash" },
+ { "spx", "audio/ogg" },
+ { "src", "application/x-wais-source" },
+ { "srf", "text/plain" },
+ { "SSISDeploymentManifest", "text/xml" },
+ { "ssm", "application/streamingmedia" },
+ { "sst", "application/vnd.ms-pki.certstore" },
+ { "stl", "application/vnd.ms-pki.stl" },
+ { "sv4cpio", "application/x-sv4cpio" },
+ { "sv4crc", "application/x-sv4crc" },
+ { "svc", "application/xml" },
+ { "svg", "image/svg+xml" },
+ { "swf", "application/x-shockwave-flash" },
+ { "step", "application/step" },
+ { "stp", "application/step" },
+ { "t", "application/x-troff" },
+ { "tar", "application/x-tar" },
+ { "tcl", "application/x-tcl" },
+ { "testrunconfig", "application/xml" },
+ { "testsettings", "application/xml" },
+ { "tex", "application/x-tex" },
+ { "texi", "application/x-texinfo" },
+ { "texinfo", "application/x-texinfo" },
+ { "tgz", "application/x-compressed" },
+ { "thmx", "application/vnd.ms-officetheme" },
+ { "thn", "application/octet-stream" },
+ { "tif", { "image/tiff", "application/octet-stream" } },
+ { "tiff", "image/tiff" },
+ { "tlh", "text/plain" },
+ { "tli", "text/plain" },
+ { "toc", "application/octet-stream" },
+ { "tr", "application/x-troff" },
+ { "trm", "application/x-msterminal" },
+ { "trx", "application/xml" },
+ { "ts", "video/vnd.dlna.mpeg-tts" },
+ { "tsv", "text/tab-separated-values" },
+ { "ttf", "application/font-sfnt" },
+ { "tts", "video/vnd.dlna.mpeg-tts" },
+ { "txt", "text/plain" },
+ { "u32", "application/octet-stream" },
+ { "uls", "text/iuls" },
+ { "user", "text/plain" },
+ { "ustar", "application/x-ustar" },
+ { "vb", "text/plain" },
+ { "vbdproj", "text/plain" },
+ { "vbk", "video/mpeg" },
+ { "vbproj", "text/plain" },
+ { "vbs", "text/vbscript" },
+ { "vcf", { "text/x-vcard", "text/vcard" } },
+ { "vcproj", "application/xml" },
+ { "vcs", "text/plain" },
+ { "vcxproj", "application/xml" },
+ { "vddproj", "text/plain" },
+ { "vdp", "text/plain" },
+ { "vdproj", "text/plain" },
+ { "vdx", "application/vnd.ms-visio.viewer" },
+ { "vml", "text/xml" },
+ { "vscontent", "application/xml" },
+ { "vsct", "text/xml" },
+ { "vsd", "application/vnd.visio" },
+ { "vsi", "application/ms-vsi" },
+ { "vsix", "application/vsix" },
+ { "vsixlangpack", "text/xml" },
+ { "vsixmanifest", "text/xml" },
+ { "vsmdi", "application/xml" },
+ { "vspscc", "text/plain" },
+ { "vss", "application/vnd.visio" },
+ { "vsscc", "text/plain" },
+ { "vssettings", "text/xml" },
+ { "vssscc", "text/plain" },
+ { "vst", "application/vnd.visio" },
+ { "vstemplate", "text/xml" },
+ { "vsto", "application/x-ms-vsto" },
+ { "vsw", "application/vnd.visio" },
+ { "vsx", "application/vnd.visio" },
+ { "vtx", "application/vnd.visio" },
+ { "wav", { "audio/wav", "audio/vnd.wave", "audio/x-wav" } },
+ { "wave", "audio/wav" },
+ { "wax", "audio/x-ms-wax" },
+ { "wbk", "application/msword" },
+ { "wbmp", "image/vnd.wap.wbmp" },
+ { "wcm", "application/vnd.ms-works" },
+ { "wdb", "application/vnd.ms-works" },
+ { "wdp", "image/vnd.ms-photo" },
+ { "webarchive", "application/x-safari-webarchive" },
+ { "webm", "video/webm" },
+ { "webp", "image/webp" },
+ { "webtest", "application/xml" },
+ { "wiq", "application/xml" },
+ { "wiz", "application/msword" },
+ { "wks", "application/vnd.ms-works" },
+ { "WLMP", "application/wlmoviemaker" },
+ { "wlpginstall", "application/x-wlpg-detect" },
+ { "wlpginstall3", "application/x-wlpg3-detect" },
+ { "wm", "video/x-ms-wm" },
+ { "wma", "audio/x-ms-wma" },
+ { "wmd", "application/x-ms-wmd" },
+ { "wmf", { "application/x-msmetafile", "image/wmf", "image/x-wmf" } },
+ { "wml", "text/vnd.wap.wml" },
+ { "wmlc", "application/vnd.wap.wmlc" },
+ { "wmls", "text/vnd.wap.wmlscript" },
+ { "wmlsc", "application/vnd.wap.wmlscriptc" },
+ { "wmp", "video/x-ms-wmp" },
+ { "wmv", "video/x-ms-wmv" },
+ { "wmx", "video/x-ms-wmx" },
+ { "wmz", "application/x-ms-wmz" },
+ { "woff", "application/font-woff" },
+ { "wpl", "application/vnd.ms-wpl" },
+ { "wps", "application/vnd.ms-works" },
+ { "wri", "application/x-mswrite" },
+ { "wrl", "x-world/x-vrml" },
+ { "wrz", "x-world/x-vrml" },
+ { "wsc", "text/scriptlet" },
+ { "wsdl", "text/xml" },
+ { "wvx", "video/x-ms-wvx" },
+ { "x", "application/directx" },
+ { "xaf", "x-world/x-vrml" },
+ { "xaml", "application/xaml+xml" },
+ { "xap", "application/x-silverlight-app" },
+ { "xbap", "application/x-ms-xbap" },
+ { "xbm", "image/x-xbitmap" },
+ { "xdr", "text/plain" },
+ { "xht", "application/xhtml+xml" },
+ { "xhtml", "application/xhtml+xml" },
+ { "xla", "application/vnd.ms-excel" },
+ { "xlam", "application/vnd.ms-excel.addin.macroEnabled.12" },
+ { "xlc", "application/vnd.ms-excel" },
+ { "xld", "application/vnd.ms-excel" },
+ { "xlk", "application/vnd.ms-excel" },
+ { "xll", "application/vnd.ms-excel" },
+ { "xlm", "application/vnd.ms-excel" },
+ { "xls", {
+ "application/excel",
+ "application/vnd.ms-excel",
+ "application/vnd.ms-office",
+ "application/x-excel",
+ "application/octet-stream"
+ } },
+ { "xlsb", "application/vnd.ms-excel.sheet.binary.macroEnabled.12" },
+ { "xlsm", "application/vnd.ms-excel.sheet.macroEnabled.12" },
+ { "xlsx", {
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ "application/vnd.ms-excel.12",
+ "application/octet-stream"
+ } },
+ { "xlt", "application/vnd.ms-excel" },
+ { "xltm", "application/vnd.ms-excel.template.macroEnabled.12" },
+ { "xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template" },
+ { "xlw", "application/vnd.ms-excel" },
+ { "xml", { "application/xml", "text/xml", "application/octet-stream" } },
+ { "xmp", "application/octet-stream" },
+ { "xmta", "application/xml" },
+ { "xof", "x-world/x-vrml" },
+ { "XOML", "text/plain" },
+ { "xpm", "image/x-xpixmap" },
+ { "xps", "application/vnd.ms-xpsdocument" },
+ { "xrm-ms", "text/xml" },
+ { "xsc", "application/xml" },
+ { "xsd", "text/xml" },
+ { "xsf", "text/xml" },
+ { "xsl", "text/xml" },
+ { "xslt", "text/xml" },
+ { "xsn", "application/octet-stream" },
+ { "xss", "application/xml" },
+ { "xspf", "application/xspf+xml" },
+ { "xtp", "application/octet-stream" },
+ { "xwd", "image/x-xwindowdump" },
+ { "z", "application/x-compress" },
+ { "zip", {
+ "application/zip",
+ "application/x-zip-compressed",
+ "application/octet-stream"
+ } },
+ { "zlib", "application/zlib" },
+}
+
+-- Used to match extension by content type
+exports.reversed_extensions_map = {
+ ["text/html"] = "html",
+ ["text/css"] = "css",
+ ["text/xml"] = "xml",
+ ["image/gif"] = "gif",
+ ["image/jpeg"] = "jpeg",
+ ["application/javascript"] = "js",
+ ["application/atom+xml"] = "atom",
+ ["application/rss+xml"] = "rss",
+ ["application/csv"] = "csv",
+ ["text/mathml"] = "mml",
+ ["text/plain"] = "txt",
+ ["text/vnd.sun.j2me.app-descriptor"] = "jad",
+ ["text/vnd.wap.wml"] = "wml",
+ ["text/x-component"] = "htc",
+ ["image/png"] = "png",
+ ["image/svg+xml"] = "svg",
+ ["image/tiff"] = "tiff",
+ ["image/vnd.wap.wbmp"] = "wbmp",
+ ["image/webp"] = "webp",
+ ["image/x-icon"] = "ico",
+ ["image/x-jng"] = "jng",
+ ["image/x-ms-bmp"] = "bmp",
+ ["font/woff"] = "woff",
+ ["font/woff2"] = "woff2",
+ ["application/java-archive"] = "jar",
+ ["application/json"] = "json",
+ ["application/mac-binhex40"] = "hqx",
+ ["application/msword"] = "doc",
+ ["application/pdf"] = "pdf",
+ ["application/postscript"] = "ps",
+ ["application/rtf"] = "rtf",
+ ["application/vnd.apple.mpegurl"] = "m3u8",
+ ["application/vnd.google-earth.kml+xml"] = "kml",
+ ["application/vnd.google-earth.kmz"] = "kmz",
+ ["application/vnd.ms-excel"] = "xls",
+ ["application/vnd.ms-fontobject"] = "eot",
+ ["application/vnd.ms-powerpoint"] = "ppt",
+ ["application/vnd.oasis.opendocument.graphics"] = "odg",
+ ["application/vnd.oasis.opendocument.presentation"] = "odp",
+ ["application/vnd.oasis.opendocument.spreadsheet"] = "ods",
+ ["application/vnd.oasis.opendocument.text"] = "odt",
+ ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = "pptx",
+ ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = "xlsx",
+ ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = "docx",
+ ["application/x-7z-compressed"] = "7z",
+ ["application/x-cocoa"] = "cco",
+ ["application/x-java-archive-diff"] = "jardiff",
+ ["application/x-java-jnlp-file"] = "jnlp",
+ ["application/x-makeself"] = "run",
+ ["application/x-perl"] = "pl",
+ ["application/x-pilot"] = "pdb",
+ ["application/x-rar-compressed"] = "rar",
+ ["application/x-redhat-package-manager"] = "rpm",
+ ["application/x-sea"] = "sea",
+ ["application/x-shockwave-flash"] = "swf",
+ ["application/x-stuffit"] = "sit",
+ ["application/x-tcl"] = "tcl",
+ ["application/x-x509-ca-cert"] = "crt",
+ ["application/x-xpinstall"] = "xpi",
+ ["application/xhtml+xml"] = "xhtml",
+ ["application/xspf+xml"] = "xspf",
+ ["application/zip"] = "zip",
+ ["application/x-dosexec"] = "exe",
+ ["application/x-msdownload"] = "exe",
+ ["application/x-executable"] = "exe",
+ ["text/x-msdos-batch"] = "bat",
+
+ ["audio/midi"] = "mid",
+ ["audio/mpeg"] = "mp3",
+ ["audio/ogg"] = "ogg",
+ ["audio/x-m4a"] = "m4a",
+ ["audio/x-realaudio"] = "ra",
+ ["video/3gpp"] = "3gpp",
+ ["video/mp2t"] = "ts",
+ ["video/mp4"] = "mp4",
+ ["video/mpeg"] = "mpeg",
+ ["video/quicktime"] = "mov",
+ ["video/webm"] = "webm",
+ ["video/x-flv"] = "flv",
+ ["video/x-m4v"] = "m4v",
+ ["video/x-mng"] = "mng",
+ ["video/x-ms-asf"] = "asx",
+ ["video/x-ms-wmv"] = "wmv",
+ ["video/x-msvideo"] = "avi",
+}
+
+return exports
diff --git a/lualib/lua_redis.lua b/lualib/lua_redis.lua
new file mode 100644
index 0000000..818d955
--- /dev/null
+++ b/lualib/lua_redis.lua
@@ -0,0 +1,1817 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local logger = require "rspamd_logger"
+local lutil = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local ts = require("tableshape").types
+
+local exports = {}
+
+local E = {}
+local N = "lua_redis"
+
+local common_schema = {
+ timeout = (ts.number + ts.string / lutil.parse_time_interval):is_optional():describe("Connection timeout"),
+ db = ts.string:is_optional():describe("Database number"),
+ database = ts.string:is_optional():describe("Database number"),
+ dbname = ts.string:is_optional():describe("Database number"),
+ prefix = ts.string:is_optional():describe("Key prefix"),
+ username = ts.string:is_optional():describe("Username"),
+ password = ts.string:is_optional():describe("Password"),
+ expand_keys = ts.boolean:is_optional():describe("Expand keys"),
+ sentinels = (ts.string + ts.array_of(ts.string)):is_optional():describe("Sentinel servers"),
+ sentinel_watch_time = (ts.number + ts.string / lutil.parse_time_interval):is_optional():describe("Sentinel watch time"),
+ sentinel_masters_pattern = ts.string:is_optional():describe("Sentinel masters pattern"),
+ sentinel_master_maxerrors = (ts.number + ts.string / tonumber):is_optional():describe("Sentinel master max errors"),
+ sentinel_username = ts.string:is_optional():describe("Sentinel username"),
+ sentinel_password = ts.string:is_optional():describe("Sentinel password"),
+}
+
+local read_schema = lutil.table_merge({
+ read_servers = ts.string + ts.array_of(ts.string),
+}, common_schema)
+
+local write_schema = lutil.table_merge({
+ write_servers = ts.string + ts.array_of(ts.string),
+}, common_schema)
+
+local rw_schema = lutil.table_merge({
+ read_servers = ts.string + ts.array_of(ts.string),
+ write_servers = ts.string + ts.array_of(ts.string),
+}, common_schema)
+
+local servers_schema = lutil.table_merge({
+ servers = ts.string + ts.array_of(ts.string),
+}, common_schema)
+
+local server_schema = lutil.table_merge({
+ server = ts.string + ts.array_of(ts.string),
+}, common_schema)
+
+local enrich_schema = function(external)
+ return ts.one_of {
+ ts.shape(external), -- no specific redis parameters
+ ts.shape(lutil.table_merge(read_schema, external)), -- read_servers specified
+ ts.shape(lutil.table_merge(write_schema, external)), -- write_servers specified
+ ts.shape(lutil.table_merge(rw_schema, external)), -- both read and write servers defined
+ ts.shape(lutil.table_merge(servers_schema, external)), -- just servers for both ops
+ ts.shape(lutil.table_merge(server_schema, external)), -- legacy `server` attribute
+ }
+end
+
+exports.enrich_schema = enrich_schema
+
+local function redis_query_sentinel(ev_base, params, initialised)
+ local function flatten_redis_table(tbl)
+ local res = {}
+ for i = 1, #tbl, 2 do
+ res[tbl[i]] = tbl[i + 1]
+ end
+
+ return res
+ end
+ -- Coroutines syntax
+ local rspamd_redis = require "rspamd_redis"
+ local sentinels = params.sentinels
+ local addr = sentinels:get_upstream_round_robin()
+
+ local host = addr:get_addr()
+ local masters = {}
+ local process_masters -- Function that is called to process masters data
+
+ local function masters_cb(err, result)
+ if not err and result and type(result) == 'table' then
+
+ local pending_subrequests = 0
+
+ for _, m in ipairs(result) do
+ local master = flatten_redis_table(m)
+
+ -- Wrap IPv6-addresses in brackets
+ if (master.ip:match(":")) then
+ master.ip = "[" .. master.ip .. "]"
+ end
+
+ if params.sentinel_masters_pattern then
+ if master.name:match(params.sentinel_masters_pattern) then
+ lutil.debugm(N, 'found master %s with ip %s and port %s',
+ master.name, master.ip, master.port)
+ masters[master.name] = master
+ else
+ lutil.debugm(N, 'skip master %s with ip %s and port %s, pattern %s',
+ master.name, master.ip, master.port, params.sentinel_masters_pattern)
+ end
+ else
+ lutil.debugm(N, 'found master %s with ip %s and port %s',
+ master.name, master.ip, master.port)
+ masters[master.name] = master
+ end
+ end
+
+ -- For each master we need to get a list of slaves
+ for k, v in pairs(masters) do
+ v.slaves = {}
+ local function slaves_cb(slave_err, slave_result)
+ if not slave_err and type(slave_result) == 'table' then
+ for _, s in ipairs(slave_result) do
+ local slave = flatten_redis_table(s)
+ lutil.debugm(N, rspamd_config,
+ 'found slave for master %s with ip %s and port %s',
+ v.name, slave.ip, slave.port)
+ -- Wrap IPv6-addresses in brackets
+ if (slave.ip:match(":")) then
+ slave.ip = "[" .. slave.ip .. "]"
+ end
+ v.slaves[#v.slaves + 1] = slave
+ end
+ else
+ logger.errx('cannot get slaves data from Redis Sentinel %s: %s',
+ host:to_string(true), slave_err)
+ addr:fail()
+ end
+
+ pending_subrequests = pending_subrequests - 1
+
+ if pending_subrequests == 0 then
+ -- Finalize masters and slaves
+ process_masters()
+ end
+ end
+
+ local ret = rspamd_redis.make_request {
+ host = addr:get_addr(),
+ timeout = params.timeout,
+ username = params.sentinel_username,
+ password = params.sentinel_password,
+ config = rspamd_config,
+ ev_base = ev_base,
+ cmd = 'SENTINEL',
+ args = { 'slaves', k },
+ no_pool = true,
+ callback = slaves_cb
+ }
+
+ if not ret then
+ logger.errx(rspamd_config, 'cannot connect sentinel when query slaves at address: %s',
+ host:to_string(true))
+ addr:fail()
+ else
+ pending_subrequests = pending_subrequests + 1
+ end
+ end
+
+ addr:ok()
+ else
+ logger.errx('cannot get masters data from Redis Sentinel %s: %s',
+ host:to_string(true), err)
+ addr:fail()
+ end
+ end
+
+ local ret = rspamd_redis.make_request {
+ host = addr:get_addr(),
+ timeout = params.timeout,
+ config = rspamd_config,
+ ev_base = ev_base,
+ username = params.sentinel_username,
+ password = params.sentinel_password,
+ cmd = 'SENTINEL',
+ args = { 'masters' },
+ no_pool = true,
+ callback = masters_cb,
+ }
+
+ if not ret then
+ logger.errx(rspamd_config, 'cannot connect sentinel at address: %s',
+ host:to_string(true))
+ addr:fail()
+ end
+
+ process_masters = function()
+ -- We now form new strings for masters and slaves
+ local read_servers_tbl, write_servers_tbl = {}, {}
+
+ for _, master in pairs(masters) do
+ write_servers_tbl[#write_servers_tbl + 1] = string.format(
+ '%s:%s', master.ip, master.port
+ )
+ read_servers_tbl[#read_servers_tbl + 1] = string.format(
+ '%s:%s', master.ip, master.port
+ )
+
+ for _, slave in ipairs(master.slaves) do
+ if slave['master-link-status'] == 'ok' then
+ read_servers_tbl[#read_servers_tbl + 1] = string.format(
+ '%s:%s', slave.ip, slave.port
+ )
+ end
+ end
+ end
+
+ table.sort(read_servers_tbl)
+ table.sort(write_servers_tbl)
+
+ local read_servers_str = table.concat(read_servers_tbl, ',')
+ local write_servers_str = table.concat(write_servers_tbl, ',')
+
+ lutil.debugm(N, rspamd_config,
+ 'new servers list: %s read; %s write',
+ read_servers_str,
+ write_servers_str)
+
+ if read_servers_str ~= params.read_servers_str then
+ local upstream_list = require "rspamd_upstream_list"
+
+ local read_upstreams = upstream_list.create(rspamd_config,
+ read_servers_str, 6379)
+
+ if read_upstreams then
+ logger.infox(rspamd_config, 'sentinel %s: replace read servers with new list: %s',
+ host:to_string(true), read_servers_str)
+ params.read_servers = read_upstreams
+ params.read_servers_str = read_servers_str
+ end
+ end
+
+ if write_servers_str ~= params.write_servers_str then
+ local upstream_list = require "rspamd_upstream_list"
+
+ local write_upstreams = upstream_list.create(rspamd_config,
+ write_servers_str, 6379)
+
+ if write_upstreams then
+ logger.infox(rspamd_config, 'sentinel %s: replace write servers with new list: %s',
+ host:to_string(true), write_servers_str)
+ params.write_servers = write_upstreams
+ params.write_servers_str = write_servers_str
+
+ local queried = false
+
+ local function monitor_failures(up, _, count)
+ if count > params.sentinel_master_maxerrors and not queried then
+ logger.infox(rspamd_config, 'sentinel: master with address %s, caused %s failures, try to query sentinel',
+ host:to_string(true), count)
+ queried = true -- Avoid multiple checks caused by this monitor
+ redis_query_sentinel(ev_base, params, true)
+ end
+ end
+
+ write_upstreams:add_watcher('failure', monitor_failures)
+ end
+ end
+ end
+
+end
+
+local function add_redis_sentinels(params)
+ local upstream_list = require "rspamd_upstream_list"
+
+ local upstreams_sentinels = upstream_list.create(rspamd_config,
+ params.sentinels, 5000)
+
+ if not upstreams_sentinels then
+ logger.errx(rspamd_config, 'cannot load redis sentinels string: %s',
+ params.sentinels)
+
+ return
+ end
+
+ params.sentinels = upstreams_sentinels
+
+ if not params.sentinel_watch_time then
+ params.sentinel_watch_time = 60 -- Each minute
+ end
+
+ if not params.sentinel_master_maxerrors then
+ params.sentinel_master_maxerrors = 2 -- Maximum number of errors before rechecking
+ end
+
+ rspamd_config:add_on_load(function(_, ev_base, worker)
+ local initialised = false
+ if worker:is_scanner() or worker:get_type() == 'fuzzy' then
+ rspamd_config:add_periodic(ev_base, 0.0, function()
+ redis_query_sentinel(ev_base, params, initialised)
+ initialised = true
+
+ return params.sentinel_watch_time
+ end, false)
+ end
+ end)
+end
+
+local cached_results = {}
+
+local function calculate_redis_hash(params)
+ local cr = require "rspamd_cryptobox_hash"
+
+ local h = cr.create()
+
+ local function rec_hash(k, v)
+ if type(v) == 'string' then
+ h:update(k)
+ h:update(v)
+ elseif type(v) == 'number' then
+ h:update(k)
+ h:update(tostring(v))
+ elseif type(v) == 'table' then
+ for kk, vv in pairs(v) do
+ rec_hash(kk, vv)
+ end
+ end
+ end
+
+ rec_hash('top', params)
+
+ return h:base32()
+end
+
+local function process_redis_opts(options, redis_params)
+ local default_timeout = 1.0
+ local default_expand_keys = false
+
+ if not redis_params['timeout'] or redis_params['timeout'] == default_timeout then
+ if options['timeout'] then
+ redis_params['timeout'] = tonumber(options['timeout'])
+ else
+ redis_params['timeout'] = default_timeout
+ end
+ end
+
+ if options['prefix'] and not redis_params['prefix'] then
+ redis_params['prefix'] = options['prefix']
+ end
+
+ if type(options['expand_keys']) == 'boolean' then
+ redis_params['expand_keys'] = options['expand_keys']
+ else
+ redis_params['expand_keys'] = default_expand_keys
+ end
+
+ if not redis_params['db'] then
+ if options['db'] then
+ redis_params['db'] = tostring(options['db'])
+ elseif options['dbname'] then
+ redis_params['db'] = tostring(options['dbname'])
+ elseif options['database'] then
+ redis_params['db'] = tostring(options['database'])
+ end
+ end
+ if options['username'] and not redis_params['username'] then
+ redis_params['username'] = options['username']
+ end
+ if options['password'] and not redis_params['password'] then
+ redis_params['password'] = options['password']
+ end
+
+ if not redis_params.sentinels and options.sentinels then
+ redis_params.sentinels = options.sentinels
+ end
+
+ if options['sentinel_masters_pattern'] and not redis_params['sentinel_masters_pattern'] then
+ redis_params['sentinel_masters_pattern'] = options['sentinel_masters_pattern']
+ end
+
+end
+
+local function enrich_defaults(rspamd_config, module, redis_params)
+ if rspamd_config then
+ local opts = rspamd_config:get_all_opt('redis')
+
+ if opts then
+ if module then
+ if opts[module] then
+ process_redis_opts(opts[module], redis_params)
+ end
+ end
+
+ process_redis_opts(opts, redis_params)
+ end
+ end
+end
+
+local function maybe_return_cached(redis_params)
+ local h = calculate_redis_hash(redis_params)
+
+ if cached_results[h] then
+ lutil.debugm(N, 'reused redis server: %s', redis_params)
+ return cached_results[h]
+ end
+
+ redis_params.hash = h
+ cached_results[h] = redis_params
+
+ if not redis_params.read_only and redis_params.sentinels then
+ add_redis_sentinels(redis_params)
+ end
+
+ lutil.debugm(N, 'loaded new redis server: %s', redis_params)
+ return redis_params
+end
+
+--[[[
+-- @module lua_redis
+-- This module contains helper functions for working with Redis
+--]]
+local function process_redis_options(options, rspamd_config, result)
+ local default_port = 6379
+ local upstream_list = require "rspamd_upstream_list"
+ local read_only = true
+
+ -- Try to get read servers:
+ local upstreams_read, upstreams_write
+
+ if options['read_servers'] then
+ if rspamd_config then
+ upstreams_read = upstream_list.create(rspamd_config,
+ options['read_servers'], default_port)
+ else
+ upstreams_read = upstream_list.create(options['read_servers'],
+ default_port)
+ end
+
+ result.read_servers_str = options['read_servers']
+ elseif options['servers'] then
+ if rspamd_config then
+ upstreams_read = upstream_list.create(rspamd_config,
+ options['servers'], default_port)
+ else
+ upstreams_read = upstream_list.create(options['servers'], default_port)
+ end
+
+ result.read_servers_str = options['servers']
+ read_only = false
+ elseif options['server'] then
+ if rspamd_config then
+ upstreams_read = upstream_list.create(rspamd_config,
+ options['server'], default_port)
+ else
+ upstreams_read = upstream_list.create(options['server'], default_port)
+ end
+
+ result.read_servers_str = options['server']
+ read_only = false
+ end
+
+ if upstreams_read then
+ if options['write_servers'] then
+ if rspamd_config then
+ upstreams_write = upstream_list.create(rspamd_config,
+ options['write_servers'], default_port)
+ else
+ upstreams_write = upstream_list.create(options['write_servers'],
+ default_port)
+ end
+ result.write_servers_str = options['write_servers']
+ read_only = false
+ elseif not read_only then
+ upstreams_write = upstreams_read
+ result.write_servers_str = result.read_servers_str
+ end
+ end
+
+ -- Store options
+ process_redis_opts(options, result)
+
+ if read_only and not upstreams_write then
+ result.read_only = true
+ elseif upstreams_write then
+ result.read_only = false
+ end
+
+ if upstreams_read then
+ result.read_servers = upstreams_read
+
+ if upstreams_write then
+ result.write_servers = upstreams_write
+ end
+
+ return true
+ end
+
+ lutil.debugm(N, rspamd_config,
+ 'cannot load redis server from obj: %s, processed to %s',
+ options, result)
+
+ return false
+end
+
+--[[[
+@function try_load_redis_servers(options, rspamd_config, no_fallback)
+Tries to load redis servers from the specified `options` object.
+Returns `redis_params` table or nil in case of failure
+
+--]]
+exports.try_load_redis_servers = function(options, rspamd_config, no_fallback, module_name)
+ local result = {}
+
+ if process_redis_options(options, rspamd_config, result) then
+ if not no_fallback then
+ enrich_defaults(rspamd_config, module_name, result)
+ end
+ return maybe_return_cached(result)
+ end
+end
+
+-- This function parses redis server definition using either
+-- specific server string for this module or global
+-- redis section
+local function rspamd_parse_redis_server(module_name, module_opts, no_fallback)
+ local result = {}
+
+ -- Try local options
+ local opts
+ lutil.debugm(N, rspamd_config, 'try load redis config for: %s', module_name)
+ if not module_opts then
+ opts = rspamd_config:get_all_opt(module_name)
+ else
+ opts = module_opts
+ end
+
+ if opts then
+ local ret
+
+ if opts.redis then
+ ret = process_redis_options(opts.redis, rspamd_config, result)
+
+ if ret then
+ if not no_fallback then
+ enrich_defaults(rspamd_config, module_name, result)
+ end
+ return maybe_return_cached(result)
+ end
+ end
+
+ ret = process_redis_options(opts, rspamd_config, result)
+
+ if ret then
+ if not no_fallback then
+ enrich_defaults(rspamd_config, module_name, result)
+ end
+ return maybe_return_cached(result)
+ end
+ end
+
+ if no_fallback then
+ logger.infox(rspamd_config, "cannot find Redis definitions for %s and fallback is disabled",
+ module_name)
+
+ return nil
+ end
+
+ -- Try global options
+ opts = rspamd_config:get_all_opt('redis')
+
+ if opts then
+ local ret
+
+ if opts[module_name] then
+ ret = process_redis_options(opts[module_name], rspamd_config, result)
+
+ if ret then
+ return maybe_return_cached(result)
+ end
+ else
+ ret = process_redis_options(opts, rspamd_config, result)
+
+ -- Exclude disabled
+ if opts['disabled_modules'] then
+ for _, v in ipairs(opts['disabled_modules']) do
+ if v == module_name then
+ logger.infox(rspamd_config, "NOT using default redis server for module %s: it is disabled",
+ module_name)
+
+ return nil
+ end
+ end
+ end
+
+ if ret then
+ logger.infox(rspamd_config, "use default Redis settings for %s",
+ module_name)
+ return maybe_return_cached(result)
+ end
+ end
+ end
+
+ if result.read_servers then
+ return maybe_return_cached(result)
+ end
+
+ return nil
+end
+
+--[[[
+-- @function lua_redis.parse_redis_server(module_name, module_opts, no_fallback)
+-- Extracts Redis server settings from configuration
+-- @param {string} module_name name of module to get settings for
+-- @param {table} module_opts settings for module or `nil` to fetch them from configuration
+-- @param {boolean} no_fallback should be `true` if global settings must not be used
+-- @return {table} redis server settings
+-- @example
+-- local rconfig = lua_redis.parse_redis_server('my_module')
+-- -- rconfig contains upstream_list objects in ['write_servers'] and ['read_servers']
+-- -- ['timeout'] contains timeout in seconds
+-- -- ['expand_keys'] if true tells that redis key expansion is enabled
+--]]
+
+exports.rspamd_parse_redis_server = rspamd_parse_redis_server
+exports.parse_redis_server = rspamd_parse_redis_server
+
+local process_cmd = {
+ bitop = function(args)
+ local idx_l = {}
+ for i = 2, #args do
+ table.insert(idx_l, i)
+ end
+ return idx_l
+ end,
+ blpop = function(args)
+ local idx_l = {}
+ for i = 1, #args - 1 do
+ table.insert(idx_l, i)
+ end
+ return idx_l
+ end,
+ eval = function(args)
+ local idx_l = {}
+ local numkeys = args[2]
+ if numkeys and tonumber(numkeys) >= 1 then
+ for i = 3, numkeys + 2 do
+ table.insert(idx_l, i)
+ end
+ end
+ return idx_l
+ end,
+ set = function(args)
+ return { 1 }
+ end,
+ mget = function(args)
+ local idx_l = {}
+ for i = 1, #args do
+ table.insert(idx_l, i)
+ end
+ return idx_l
+ end,
+ mset = function(args)
+ local idx_l = {}
+ for i = 1, #args, 2 do
+ table.insert(idx_l, i)
+ end
+ return idx_l
+ end,
+ sdiffstore = function(args)
+ local idx_l = {}
+ for i = 2, #args do
+ table.insert(idx_l, i)
+ end
+ return idx_l
+ end,
+ smove = function(args)
+ return { 1, 2 }
+ end,
+ script = function()
+ end
+}
+process_cmd.append = process_cmd.set
+process_cmd.auth = process_cmd.script
+process_cmd.bgrewriteaof = process_cmd.script
+process_cmd.bgsave = process_cmd.script
+process_cmd.bitcount = process_cmd.set
+process_cmd.bitfield = process_cmd.set
+process_cmd.bitpos = process_cmd.set
+process_cmd.brpop = process_cmd.blpop
+process_cmd.brpoplpush = process_cmd.blpop
+process_cmd.client = process_cmd.script
+process_cmd.cluster = process_cmd.script
+process_cmd.command = process_cmd.script
+process_cmd.config = process_cmd.script
+process_cmd.dbsize = process_cmd.script
+process_cmd.debug = process_cmd.script
+process_cmd.decr = process_cmd.set
+process_cmd.decrby = process_cmd.set
+process_cmd.del = process_cmd.mget
+process_cmd.discard = process_cmd.script
+process_cmd.dump = process_cmd.set
+process_cmd.echo = process_cmd.script
+process_cmd.evalsha = process_cmd.eval
+process_cmd.exec = process_cmd.script
+process_cmd.exists = process_cmd.mget
+process_cmd.expire = process_cmd.set
+process_cmd.expireat = process_cmd.set
+process_cmd.flushall = process_cmd.script
+process_cmd.flushdb = process_cmd.script
+process_cmd.geoadd = process_cmd.set
+process_cmd.geohash = process_cmd.set
+process_cmd.geopos = process_cmd.set
+process_cmd.geodist = process_cmd.set
+process_cmd.georadius = process_cmd.set
+process_cmd.georadiusbymember = process_cmd.set
+process_cmd.get = process_cmd.set
+process_cmd.getbit = process_cmd.set
+process_cmd.getrange = process_cmd.set
+process_cmd.getset = process_cmd.set
+process_cmd.hdel = process_cmd.set
+process_cmd.hexists = process_cmd.set
+process_cmd.hget = process_cmd.set
+process_cmd.hgetall = process_cmd.set
+process_cmd.hincrby = process_cmd.set
+process_cmd.hincrbyfloat = process_cmd.set
+process_cmd.hkeys = process_cmd.set
+process_cmd.hlen = process_cmd.set
+process_cmd.hmget = process_cmd.set
+process_cmd.hmset = process_cmd.set
+process_cmd.hscan = process_cmd.set
+process_cmd.hset = process_cmd.set
+process_cmd.hsetnx = process_cmd.set
+process_cmd.hstrlen = process_cmd.set
+process_cmd.hvals = process_cmd.set
+process_cmd.incr = process_cmd.set
+process_cmd.incrby = process_cmd.set
+process_cmd.incrbyfloat = process_cmd.set
+process_cmd.info = process_cmd.script
+process_cmd.keys = process_cmd.script
+process_cmd.lastsave = process_cmd.script
+process_cmd.lindex = process_cmd.set
+process_cmd.linsert = process_cmd.set
+process_cmd.llen = process_cmd.set
+process_cmd.lpop = process_cmd.set
+process_cmd.lpush = process_cmd.set
+process_cmd.lpushx = process_cmd.set
+process_cmd.lrange = process_cmd.set
+process_cmd.lrem = process_cmd.set
+process_cmd.lset = process_cmd.set
+process_cmd.ltrim = process_cmd.set
+process_cmd.migrate = process_cmd.script
+process_cmd.monitor = process_cmd.script
+process_cmd.move = process_cmd.set
+process_cmd.msetnx = process_cmd.mset
+process_cmd.multi = process_cmd.script
+process_cmd.object = process_cmd.script
+process_cmd.persist = process_cmd.set
+process_cmd.pexpire = process_cmd.set
+process_cmd.pexpireat = process_cmd.set
+process_cmd.pfadd = process_cmd.set
+process_cmd.pfcount = process_cmd.set
+process_cmd.pfmerge = process_cmd.mget
+process_cmd.ping = process_cmd.script
+process_cmd.psetex = process_cmd.set
+process_cmd.psubscribe = process_cmd.script
+process_cmd.pubsub = process_cmd.script
+process_cmd.pttl = process_cmd.set
+process_cmd.publish = process_cmd.script
+process_cmd.punsubscribe = process_cmd.script
+process_cmd.quit = process_cmd.script
+process_cmd.randomkey = process_cmd.script
+process_cmd.readonly = process_cmd.script
+process_cmd.readwrite = process_cmd.script
+process_cmd.rename = process_cmd.mget
+process_cmd.renamenx = process_cmd.mget
+process_cmd.restore = process_cmd.set
+process_cmd.role = process_cmd.script
+process_cmd.rpop = process_cmd.set
+process_cmd.rpoplpush = process_cmd.mget
+process_cmd.rpush = process_cmd.set
+process_cmd.rpushx = process_cmd.set
+process_cmd.sadd = process_cmd.set
+process_cmd.save = process_cmd.script
+process_cmd.scard = process_cmd.set
+process_cmd.sdiff = process_cmd.mget
+process_cmd.select = process_cmd.script
+process_cmd.setbit = process_cmd.set
+process_cmd.setex = process_cmd.set
+process_cmd.setnx = process_cmd.set
+process_cmd.sinterstore = process_cmd.sdiff
+process_cmd.sismember = process_cmd.set
+process_cmd.slaveof = process_cmd.script
+process_cmd.slowlog = process_cmd.script
+process_cmd.smembers = process_cmd.script
+process_cmd.sort = process_cmd.set
+process_cmd.spop = process_cmd.set
+process_cmd.srandmember = process_cmd.set
+process_cmd.srem = process_cmd.set
+process_cmd.strlen = process_cmd.set
+process_cmd.subscribe = process_cmd.script
+process_cmd.sunion = process_cmd.mget
+process_cmd.sunionstore = process_cmd.mget
+process_cmd.swapdb = process_cmd.script
+process_cmd.sync = process_cmd.script
+process_cmd.time = process_cmd.script
+process_cmd.touch = process_cmd.mget
+process_cmd.ttl = process_cmd.set
+process_cmd.type = process_cmd.set
+process_cmd.unsubscribe = process_cmd.script
+process_cmd.unlink = process_cmd.mget
+process_cmd.unwatch = process_cmd.script
+process_cmd.wait = process_cmd.script
+process_cmd.watch = process_cmd.mget
+process_cmd.zadd = process_cmd.set
+process_cmd.zcard = process_cmd.set
+process_cmd.zcount = process_cmd.set
+process_cmd.zincrby = process_cmd.set
+process_cmd.zinterstore = process_cmd.eval
+process_cmd.zlexcount = process_cmd.set
+process_cmd.zrange = process_cmd.set
+process_cmd.zrangebylex = process_cmd.set
+process_cmd.zrank = process_cmd.set
+process_cmd.zrem = process_cmd.set
+process_cmd.zrembylex = process_cmd.set
+process_cmd.zrembyrank = process_cmd.set
+process_cmd.zrembyscore = process_cmd.set
+process_cmd.zrevrange = process_cmd.set
+process_cmd.zrevrangebyscore = process_cmd.set
+process_cmd.zrevrank = process_cmd.set
+process_cmd.zscore = process_cmd.set
+process_cmd.zunionstore = process_cmd.eval
+process_cmd.scan = process_cmd.script
+process_cmd.sscan = process_cmd.set
+process_cmd.hscan = process_cmd.set
+process_cmd.zscan = process_cmd.set
+
+local function get_key_indexes(cmd, args)
+ local idx_l = {}
+ cmd = string.lower(cmd)
+ if process_cmd[cmd] then
+ idx_l = process_cmd[cmd](args)
+ else
+ logger.warnx(rspamd_config, "Don't know how to extract keys for %s Redis command", cmd)
+ end
+ return idx_l
+end
+
+local gen_meta = {
+ principal_recipient = function(task)
+ return task:get_principal_recipient()
+ end,
+ principal_recipient_domain = function(task)
+ local p = task:get_principal_recipient()
+ if not p then
+ return
+ end
+ return string.match(p, '.*@(.*)')
+ end,
+ ip = function(task)
+ local i = task:get_ip()
+ if i and i:is_valid() then
+ return i:to_string()
+ end
+ end,
+ from = function(task)
+ return ((task:get_from('smtp') or E)[1] or E)['addr']
+ end,
+ from_domain = function(task)
+ return ((task:get_from('smtp') or E)[1] or E)['domain']
+ end,
+ from_domain_or_helo_domain = function(task)
+ local d = ((task:get_from('smtp') or E)[1] or E)['domain']
+ if d and #d > 0 then
+ return d
+ end
+ return task:get_helo()
+ end,
+ mime_from = function(task)
+ return ((task:get_from('mime') or E)[1] or E)['addr']
+ end,
+ mime_from_domain = function(task)
+ return ((task:get_from('mime') or E)[1] or E)['domain']
+ end,
+}
+
+local function gen_get_esld(f)
+ return function(task)
+ local d = f(task)
+ if not d then
+ return
+ end
+ return rspamd_util.get_tld(d)
+ end
+end
+
+gen_meta.smtp_from = gen_meta.from
+gen_meta.smtp_from_domain = gen_meta.from_domain
+gen_meta.smtp_from_domain_or_helo_domain = gen_meta.from_domain_or_helo_domain
+gen_meta.esld_principal_recipient_domain = gen_get_esld(gen_meta.principal_recipient_domain)
+gen_meta.esld_from_domain = gen_get_esld(gen_meta.from_domain)
+gen_meta.esld_smtp_from_domain = gen_meta.esld_from_domain
+gen_meta.esld_mime_from_domain = gen_get_esld(gen_meta.mime_from_domain)
+gen_meta.esld_from_domain_or_helo_domain = gen_get_esld(gen_meta.from_domain_or_helo_domain)
+gen_meta.esld_smtp_from_domain_or_helo_domain = gen_meta.esld_from_domain_or_helo_domain
+
+local function get_key_expansion_metadata(task)
+
+ local md_mt = {
+ __index = function(self, k)
+ k = string.lower(k)
+ local v = rawget(self, k)
+ if v then
+ return v
+ end
+ if gen_meta[k] then
+ v = gen_meta[k](task)
+ rawset(self, k, v)
+ end
+ return v
+ end,
+ }
+
+ local lazy_meta = {}
+ setmetatable(lazy_meta, md_mt)
+ return lazy_meta
+
+end
+
+-- Performs async call to redis hiding all complexity inside function
+-- task - rspamd_task
+-- redis_params - valid params returned by rspamd_parse_redis_server
+-- key - key to select upstream or nil to select round-robin/master-slave
+-- is_write - true if need to write to redis server
+-- callback - function to be called upon request is completed
+-- command - redis command
+-- args - table of arguments
+-- extra_opts - table of optional request arguments
+local function rspamd_redis_make_request(task, redis_params, key, is_write,
+ callback, command, args, extra_opts)
+ local addr
+ local function rspamd_redis_make_request_cb(err, data)
+ if err then
+ addr:fail()
+ else
+ addr:ok()
+ end
+ if callback then
+ callback(err, data, addr)
+ end
+ end
+ if not task or not redis_params or not command then
+ return false, nil, nil
+ end
+
+ local rspamd_redis = require "rspamd_redis"
+
+ if key then
+ if is_write then
+ addr = redis_params['write_servers']:get_upstream_by_hash(key)
+ else
+ addr = redis_params['read_servers']:get_upstream_by_hash(key)
+ end
+ else
+ if is_write then
+ addr = redis_params['write_servers']:get_upstream_master_slave(key)
+ else
+ addr = redis_params['read_servers']:get_upstream_round_robin(key)
+ end
+ end
+
+ if not addr then
+ logger.errx(task, 'cannot select server to make redis request')
+ end
+
+ if redis_params['expand_keys'] then
+ local m = get_key_expansion_metadata(task)
+ local indexes = get_key_indexes(command, args)
+ for _, i in ipairs(indexes) do
+ args[i] = lutil.template(args[i], m)
+ end
+ end
+
+ local ip_addr = addr:get_addr()
+ local options = {
+ task = task,
+ callback = rspamd_redis_make_request_cb,
+ host = ip_addr,
+ timeout = redis_params['timeout'],
+ cmd = command,
+ args = args
+ }
+
+ if extra_opts then
+ for k, v in pairs(extra_opts) do
+ options[k] = v
+ end
+ end
+
+ if redis_params['username'] then
+ options['username'] = redis_params['username']
+ end
+
+ if redis_params['password'] then
+ options['password'] = redis_params['password']
+ end
+
+ if redis_params['db'] then
+ options['dbname'] = redis_params['db']
+ end
+
+ lutil.debugm(N, task, 'perform request to redis server' ..
+ ' (host=%s, timeout=%s): cmd: %s', ip_addr,
+ options.timeout, options.cmd)
+
+ local ret, conn = rspamd_redis.make_request(options)
+
+ if not ret then
+ addr:fail()
+ logger.warnx(task, "cannot make redis request to: %s", tostring(ip_addr))
+ end
+
+ return ret, conn, addr
+end
+
+--[[[
+-- @function lua_redis.redis_make_request(task, redis_params, key, is_write, callback, command, args)
+-- Sends a request to Redis
+-- @param {rspamd_task} task task object
+-- @param {table} redis_params redis configuration in format returned by lua_redis.parse_redis_server()
+-- @param {string} key key to use for sharding
+-- @param {boolean} is_write should be `true` if we are performing a write operating
+-- @param {function} callback callback function (first parameter is error if applicable, second is a 2D array (table))
+-- @param {string} command Redis command to run
+-- @param {table} args Numerically indexed table containing arguments for command
+--]]
+
+exports.rspamd_redis_make_request = rspamd_redis_make_request
+exports.redis_make_request = rspamd_redis_make_request
+
+local function redis_make_request_taskless(ev_base, cfg, redis_params, key,
+ is_write, callback, command, args, extra_opts)
+ if not ev_base or not redis_params or not command then
+ return false, nil, nil
+ end
+
+ local addr
+ local function rspamd_redis_make_request_cb(err, data)
+ if err then
+ addr:fail()
+ else
+ addr:ok()
+ end
+ if callback then
+ callback(err, data, addr)
+ end
+ end
+
+ local rspamd_redis = require "rspamd_redis"
+
+ if key then
+ if is_write then
+ addr = redis_params['write_servers']:get_upstream_by_hash(key)
+ else
+ addr = redis_params['read_servers']:get_upstream_by_hash(key)
+ end
+ else
+ if is_write then
+ addr = redis_params['write_servers']:get_upstream_master_slave(key)
+ else
+ addr = redis_params['read_servers']:get_upstream_round_robin(key)
+ end
+ end
+
+ if not addr then
+ logger.errx(cfg, 'cannot select server to make redis request')
+ end
+
+ local options = {
+ ev_base = ev_base,
+ config = cfg,
+ callback = rspamd_redis_make_request_cb,
+ host = addr:get_addr(),
+ timeout = redis_params['timeout'],
+ cmd = command,
+ args = args
+ }
+ if extra_opts then
+ for k, v in pairs(extra_opts) do
+ options[k] = v
+ end
+ end
+
+ if redis_params['username'] then
+ options['username'] = redis_params['username']
+ end
+
+ if redis_params['password'] then
+ options['password'] = redis_params['password']
+ end
+
+ if redis_params['db'] then
+ options['dbname'] = redis_params['db']
+ end
+
+ lutil.debugm(N, cfg, 'perform taskless request to redis server' ..
+ ' (host=%s, timeout=%s): cmd: %s', options.host:tostring(true),
+ options.timeout, options.cmd)
+ local ret, conn = rspamd_redis.make_request(options)
+ if not ret then
+ logger.errx('cannot execute redis request')
+ addr:fail()
+ end
+
+ return ret, conn, addr
+end
+
+--[[[
+-- @function lua_redis.redis_make_request_taskless(ev_base, redis_params, key, is_write, callback, command, args)
+-- Sends a request to Redis in context where `task` is not available for some specific use-cases
+-- Identical to redis_make_request() except in that first parameter is an `event base` object
+--]]
+
+exports.rspamd_redis_make_request_taskless = redis_make_request_taskless
+exports.redis_make_request_taskless = redis_make_request_taskless
+
+local redis_scripts = {
+}
+
+local function script_set_loaded(script)
+ if script.sha then
+ script.loaded = true
+ end
+
+ local wait_table = {}
+ for _, s in ipairs(script.waitq) do
+ table.insert(wait_table, s)
+ end
+
+ script.waitq = {}
+
+ for _, s in ipairs(wait_table) do
+ s(script.loaded)
+ end
+end
+
+local function prepare_redis_call(script)
+ local servers = {}
+ local options = {}
+
+ if script.redis_params.read_servers then
+ servers = lutil.table_merge(servers, script.redis_params.read_servers:all_upstreams())
+ end
+ if script.redis_params.write_servers then
+ servers = lutil.table_merge(servers, script.redis_params.write_servers:all_upstreams())
+ end
+
+ -- Call load script on each server, set loaded flag
+ script.in_flight = #servers
+ for _, s in ipairs(servers) do
+ local cur_opts = {
+ host = s:get_addr(),
+ timeout = script.redis_params['timeout'],
+ cmd = 'SCRIPT',
+ args = { 'LOAD', script.script },
+ upstream = s
+ }
+
+ if script.redis_params['username'] then
+ cur_opts['username'] = script.redis_params['username']
+ end
+
+ if script.redis_params['password'] then
+ cur_opts['password'] = script.redis_params['password']
+ end
+
+ if script.redis_params['db'] then
+ cur_opts['dbname'] = script.redis_params['db']
+ end
+
+ table.insert(options, cur_opts)
+ end
+
+ return options
+end
+
+local function load_script_task(script, task, is_write)
+ local rspamd_redis = require "rspamd_redis"
+ local opts = prepare_redis_call(script)
+
+ for _, opt in ipairs(opts) do
+ opt.task = task
+ opt.is_write = is_write
+ opt.callback = function(err, data)
+ if err then
+ logger.errx(task, 'cannot upload script to %s: %s; registered from: %s:%s',
+ opt.upstream:get_addr():to_string(true),
+ err, script.caller.short_src, script.caller.currentline)
+ opt.upstream:fail()
+ script.fatal_error = err
+ else
+ opt.upstream:ok()
+ logger.infox(task,
+ "uploaded redis script to %s %s %s, sha: %s",
+ opt.upstream:get_addr():to_string(true),
+ script.filename and "from file" or "with id", script.filename or script.id, data)
+ script.sha = data -- We assume that sha is the same on all servers
+ end
+ script.in_flight = script.in_flight - 1
+
+ if script.in_flight == 0 then
+ script_set_loaded(script)
+ end
+ end
+
+ local ret = rspamd_redis.make_request(opt)
+
+ if not ret then
+ logger.errx('cannot execute redis request to load script on %s',
+ opt.upstream:get_addr())
+ script.in_flight = script.in_flight - 1
+ opt.upstream:fail()
+ end
+
+ if script.in_flight == 0 then
+ script_set_loaded(script)
+ end
+ end
+end
+
+local function load_script_taskless(script, cfg, ev_base, is_write)
+ local rspamd_redis = require "rspamd_redis"
+ local opts = prepare_redis_call(script)
+
+ for _, opt in ipairs(opts) do
+ opt.config = cfg
+ opt.ev_base = ev_base
+ opt.is_write = is_write
+ opt.callback = function(err, data)
+ if err then
+ logger.errx(cfg, 'cannot upload script to %s: %s; registered from: %s:%s, filename: %s',
+ opt.upstream:get_addr():to_string(true),
+ err, script.caller.short_src, script.caller.currentline, script.filename)
+ opt.upstream:fail()
+ script.fatal_error = err
+ else
+ opt.upstream:ok()
+ logger.infox(cfg,
+ "uploaded redis script to %s %s %s, sha: %s",
+ opt.upstream:get_addr():to_string(true),
+ script.filename and "from file" or "with id", script.filename or script.id,
+ data)
+ script.sha = data -- We assume that sha is the same on all servers
+ script.fatal_error = nil
+ end
+ script.in_flight = script.in_flight - 1
+
+ if script.in_flight == 0 then
+ script_set_loaded(script)
+ end
+ end
+ local ret = rspamd_redis.make_request(opt)
+
+ if not ret then
+ logger.errx('cannot execute redis request to load script on %s',
+ opt.upstream:get_addr())
+ script.in_flight = script.in_flight - 1
+ opt.upstream:fail()
+ end
+
+ if script.in_flight == 0 then
+ script_set_loaded(script)
+ end
+ end
+end
+
+local function load_redis_script(script, cfg, ev_base, _)
+ if script.redis_params then
+ load_script_taskless(script, cfg, ev_base)
+ end
+end
+
+local function add_redis_script(script, redis_params, caller_level, maybe_filename)
+ if not caller_level then
+ caller_level = 2
+ end
+ local caller = debug.getinfo(caller_level) or debug.getinfo(caller_level - 1) or E
+
+ local new_script = {
+ caller = caller,
+ loaded = false,
+ redis_params = redis_params,
+ script = script,
+ waitq = {}, -- callbacks pending for script being loaded
+ id = #redis_scripts + 1,
+ filename = maybe_filename,
+ }
+
+ -- Register on load function
+ rspamd_config:add_on_load(function(cfg, ev_base, worker)
+ local mult = 0.0
+ rspamd_config:add_periodic(ev_base, 0.0, function()
+ if not new_script.sha then
+ load_redis_script(new_script, cfg, ev_base, worker)
+ mult = mult + 1
+ return 1.0 * mult -- Check one more time in one second
+ end
+
+ return false
+ end, false)
+ end)
+
+ table.insert(redis_scripts, new_script)
+
+ return #redis_scripts
+end
+exports.add_redis_script = add_redis_script
+
+-- Loads a Redis script from a file, strips comments, and passes the content to
+-- `add_redis_script` function.
+--
+-- @param filename The name of the file containing the Redis script.
+-- @param redis_params The Redis parameters to use for this script.
+-- @return The ID of the newly added Redis script.
+--
+local function load_redis_script_from_file(filename, redis_params, dir)
+ local lua_util = require "lua_util"
+ local rspamd_logger = require "rspamd_logger"
+
+ if not dir then
+ dir = rspamd_paths.LUALIBDIR
+ end
+ local path = filename
+ if filename:sub(1, 1) ~= package.config:sub(1, 1) then
+ -- Relative path
+ path = lua_util.join_path(dir, "redis_scripts", filename)
+ end
+ -- Read file contents
+ local file = io.open(path, "r")
+ if not file then
+ rspamd_logger.errx("failed to open Redis script file: %s", path)
+ return nil
+ end
+ local script = file:read("*all")
+ if not script then
+ rspamd_logger.errx("failed to load Redis script file: %s", path)
+ return nil
+ end
+ file:close()
+ script = lua_util.strip_lua_comments(script)
+
+ return add_redis_script(script, redis_params, 3, filename)
+end
+
+exports.load_redis_script_from_file = load_redis_script_from_file
+
+local function exec_redis_script(id, params, callback, keys, args)
+ local redis_args = {}
+
+ if not redis_scripts[id] then
+ logger.errx("cannot find registered script with id %s", id)
+ return false
+ end
+
+ local script = redis_scripts[id]
+
+ if script.fatal_error then
+ callback(script.fatal_error, nil)
+ return true
+ end
+
+ if not script.redis_params then
+ callback('no redis servers defined', nil)
+ return true
+ end
+
+ local function do_call(can_reload)
+ local function redis_cb(err, data)
+ if not err then
+ callback(err, data)
+ elseif string.match(err, 'NOSCRIPT') then
+ -- Schedule restart
+ script.sha = nil
+ if can_reload then
+ table.insert(script.waitq, do_call)
+ if script.in_flight == 0 then
+ -- Reload scripts if this has not been initiated yet
+ if params.task then
+ load_script_task(script, params.task)
+ else
+ load_script_taskless(script, rspamd_config, params.ev_base)
+ end
+ end
+ else
+ callback(err, data)
+ end
+ else
+ callback(err, data)
+ end
+ end
+
+ if #redis_args == 0 then
+ table.insert(redis_args, script.sha)
+ table.insert(redis_args, tostring(#keys))
+ for _, k in ipairs(keys) do
+ table.insert(redis_args, k)
+ end
+
+ if type(args) == 'table' then
+ for _, a in ipairs(args) do
+ table.insert(redis_args, a)
+ end
+ end
+ end
+
+ if params.task then
+ if not rspamd_redis_make_request(params.task, script.redis_params,
+ params.key, params.is_write, redis_cb, 'EVALSHA', redis_args) then
+ callback('Cannot make redis request', nil)
+ end
+ else
+ if not redis_make_request_taskless(params.ev_base, rspamd_config,
+ script.redis_params,
+ params.key, params.is_write, redis_cb, 'EVALSHA', redis_args) then
+ callback('Cannot make redis request', nil)
+ end
+ end
+ end
+
+ if script.loaded then
+ do_call(true)
+ else
+ -- Delayed until scripts are loaded
+ if not params.task then
+ table.insert(script.waitq, do_call)
+ else
+ -- TODO: fix taskfull requests
+ table.insert(script.waitq, function()
+ if script.loaded then
+ do_call(false)
+ else
+ callback('NOSCRIPT', nil)
+ end
+ end)
+ load_script_task(script, params.task, params.is_write)
+ end
+ end
+
+ return true
+end
+
+exports.exec_redis_script = exec_redis_script
+
+local function redis_connect_sync(redis_params, is_write, key, cfg, ev_base)
+ if not redis_params then
+ return false, nil
+ end
+
+ local rspamd_redis = require "rspamd_redis"
+ local addr
+
+ if key then
+ if is_write then
+ addr = redis_params['write_servers']:get_upstream_by_hash(key)
+ else
+ addr = redis_params['read_servers']:get_upstream_by_hash(key)
+ end
+ else
+ if is_write then
+ addr = redis_params['write_servers']:get_upstream_master_slave(key)
+ else
+ addr = redis_params['read_servers']:get_upstream_round_robin(key)
+ end
+ end
+
+ if not addr then
+ logger.errx(cfg, 'cannot select server to make redis request')
+ end
+
+ local options = {
+ host = addr:get_addr(),
+ timeout = redis_params['timeout'],
+ config = cfg or rspamd_config,
+ ev_base = ev_base or rspamadm_ev_base,
+ session = redis_params.session or rspamadm_session
+ }
+
+ for k, v in pairs(redis_params) do
+ options[k] = v
+ end
+
+ if not options.config then
+ logger.errx('config is not set')
+ return false, nil, addr
+ end
+
+ if not options.ev_base then
+ logger.errx('ev_base is not set')
+ return false, nil, addr
+ end
+
+ if not options.session then
+ logger.errx('session is not set')
+ return false, nil, addr
+ end
+
+ local ret, conn = rspamd_redis.connect_sync(options)
+ if not ret then
+ logger.errx('cannot create redis connection: %s', conn)
+ addr:fail()
+
+ return false, nil, addr
+ end
+
+ if conn then
+ local need_exec = false
+ if redis_params['username'] then
+ if redis_params['password'] then
+ conn:add_cmd('AUTH', { redis_params['username'], redis_params['password'] })
+ need_exec = true
+ else
+ logger.warnx('Redis requires a password when username is supplied')
+ return false, nil, addr
+ end
+ elseif redis_params['password'] then
+ conn:add_cmd('AUTH', { redis_params['password'] })
+ need_exec = true
+ end
+
+ if redis_params['db'] then
+ conn:add_cmd('SELECT', { tostring(redis_params['db']) })
+ need_exec = true
+ elseif redis_params['dbname'] then
+ conn:add_cmd('SELECT', { tostring(redis_params['dbname']) })
+ need_exec = true
+ end
+
+ if need_exec then
+ local exec_ret, res = conn:exec()
+
+ if not exec_ret then
+ logger.errx('cannot prepare redis connection (authentication or db selection failure): %s',
+ res)
+ addr:fail()
+ return false, nil, addr
+ end
+ end
+ end
+
+ return ret, conn, addr
+end
+
+exports.redis_connect_sync = redis_connect_sync
+
+--[[[
+-- @function lua_redis.request(redis_params, attrs, req)
+-- Sends a request to Redis synchronously with coroutines or asynchronously using
+-- a callback (modern API)
+-- @param redis_params a table of redis server parameters
+-- @param attrs a table of redis request attributes (e.g. task, or ev_base + cfg + session)
+-- @param req a table of request: a command + command options
+-- @return {result,data/connection,address} boolean result, connection object in case of async request and results if using coroutines, redis server address
+--]]
+
+exports.request = function(redis_params, attrs, req)
+ local lua_util = require "lua_util"
+
+ if not attrs or not redis_params or not req then
+ logger.errx('invalid arguments for redis request')
+ return false, nil, nil
+ end
+
+ if not (attrs.task or (attrs.config and attrs.ev_base)) then
+ logger.errx('invalid attributes for redis request')
+ return false, nil, nil
+ end
+
+ local opts = lua_util.shallowcopy(attrs)
+
+ local log_obj = opts.task or opts.config
+
+ local addr
+
+ if opts.callback then
+ -- Wrap callback
+ local callback = opts.callback
+ local function rspamd_redis_make_request_cb(err, data)
+ if err then
+ addr:fail()
+ else
+ addr:ok()
+ end
+ callback(err, data, addr)
+ end
+ opts.callback = rspamd_redis_make_request_cb
+ end
+
+ local rspamd_redis = require "rspamd_redis"
+ local is_write = opts.is_write
+
+ if opts.key then
+ if is_write then
+ addr = redis_params['write_servers']:get_upstream_by_hash(attrs.key)
+ else
+ addr = redis_params['read_servers']:get_upstream_by_hash(attrs.key)
+ end
+ else
+ if is_write then
+ addr = redis_params['write_servers']:get_upstream_master_slave(attrs.key)
+ else
+ addr = redis_params['read_servers']:get_upstream_round_robin(attrs.key)
+ end
+ end
+
+ if not addr then
+ logger.errx(log_obj, 'cannot select server to make redis request')
+ end
+
+ opts.host = addr:get_addr()
+ opts.timeout = redis_params.timeout
+
+ if type(req) == 'string' then
+ opts.cmd = req
+ else
+ -- XXX: modifies the input table
+ opts.cmd = table.remove(req, 1);
+ opts.args = req
+ end
+
+ if redis_params.username then
+ opts.username = redis_params.username
+ end
+
+ if redis_params.password then
+ opts.password = redis_params.password
+ end
+
+ if redis_params.db then
+ opts.dbname = redis_params.db
+ end
+
+ lutil.debugm(N, 'perform generic request to redis server' ..
+ ' (host=%s, timeout=%s): cmd: %s, arguments: %s', addr,
+ opts.timeout, opts.cmd, opts.args)
+
+ if opts.callback then
+ local ret, conn = rspamd_redis.make_request(opts)
+ if not ret then
+ logger.errx(log_obj, 'cannot execute redis request')
+ addr:fail()
+ end
+
+ return ret, conn, addr
+ else
+ -- Coroutines version
+ local ret, conn = rspamd_redis.connect_sync(opts)
+ if not ret then
+ logger.errx(log_obj, 'cannot execute redis request')
+ addr:fail()
+ else
+ conn:add_cmd(opts.cmd, opts.args)
+ return conn:exec()
+ end
+ return false, nil, addr
+ end
+end
+
+--[[[
+-- @function lua_redis.connect(redis_params, attrs)
+-- Connects to Redis synchronously with coroutines or asynchronously using a callback (modern API)
+-- @param redis_params a table of redis server parameters
+-- @param attrs a table of redis request attributes (e.g. task, or ev_base + cfg + session)
+-- @return {result,connection,address} boolean result, connection object, redis server address
+--]]
+
+exports.connect = function(redis_params, attrs)
+ local lua_util = require "lua_util"
+
+ if not attrs or not redis_params then
+ logger.errx('invalid arguments for redis connect')
+ return false, nil, nil
+ end
+
+ if not (attrs.task or (attrs.config and attrs.ev_base)) then
+ logger.errx('invalid attributes for redis connect')
+ return false, nil, nil
+ end
+
+ local opts = lua_util.shallowcopy(attrs)
+
+ local log_obj = opts.task or opts.config
+
+ local addr
+
+ if opts.callback then
+ -- Wrap callback
+ local callback = opts.callback
+ local function rspamd_redis_make_request_cb(err, data)
+ if err then
+ addr:fail()
+ else
+ addr:ok()
+ end
+ callback(err, data, addr)
+ end
+ opts.callback = rspamd_redis_make_request_cb
+ end
+
+ local rspamd_redis = require "rspamd_redis"
+ local is_write = opts.is_write
+
+ if opts.key then
+ if is_write then
+ addr = redis_params['write_servers']:get_upstream_by_hash(attrs.key)
+ else
+ addr = redis_params['read_servers']:get_upstream_by_hash(attrs.key)
+ end
+ else
+ if is_write then
+ addr = redis_params['write_servers']:get_upstream_master_slave(attrs.key)
+ else
+ addr = redis_params['read_servers']:get_upstream_round_robin(attrs.key)
+ end
+ end
+
+ if not addr then
+ logger.errx(log_obj, 'cannot select server to make redis connect')
+ end
+
+ opts.host = addr:get_addr()
+ opts.timeout = redis_params.timeout
+
+ if redis_params.username then
+ opts.username = redis_params.username
+ end
+
+ if redis_params.password then
+ opts.password = redis_params.password
+ end
+
+ if redis_params.db then
+ opts.dbname = redis_params.db
+ end
+
+ if opts.callback then
+ local ret, conn = rspamd_redis.connect(opts)
+ if not ret then
+ logger.errx(log_obj, 'cannot execute redis connect')
+ addr:fail()
+ end
+
+ return ret, conn, addr
+ else
+ -- Coroutines version
+ local ret, conn = rspamd_redis.connect_sync(opts)
+ if not ret then
+ logger.errx(log_obj, 'cannot execute redis connect')
+ addr:fail()
+ else
+ return true, conn, addr
+ end
+
+ return false, nil, addr
+ end
+end
+
+local redis_prefixes = {}
+
+--[[[
+-- @function lua_redis.register_prefix(prefix, module, description[, optional])
+-- Register new redis prefix for documentation purposes
+-- @param {string} prefix string prefix
+-- @param {string} module module name
+-- @param {string} description prefix description
+-- @param {table} optional optional kv pairs (e.g. pattern)
+--]]
+local function register_prefix(prefix, module, description, optional)
+ local pr = {
+ module = module,
+ description = description
+ }
+
+ if optional and type(optional) == 'table' then
+ for k, v in pairs(optional) do
+ pr[k] = v
+ end
+ end
+
+ redis_prefixes[prefix] = pr
+end
+
+exports.register_prefix = register_prefix
+
+--[[[
+-- @function lua_redis.prefixes([mname])
+-- Returns prefixes for specific module (or all prefixes). Returns a table prefix -> table
+--]]
+exports.prefixes = function(mname)
+ if not mname then
+ return redis_prefixes
+ else
+ local fun = require "fun"
+
+ return fun.totable(fun.filter(function(_, data)
+ return data.module == mname
+ end,
+ redis_prefixes))
+ end
+end
+
+return exports
diff --git a/lualib/lua_scanners/avast.lua b/lualib/lua_scanners/avast.lua
new file mode 100644
index 0000000..7e77897
--- /dev/null
+++ b/lualib/lua_scanners/avast.lua
@@ -0,0 +1,304 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module avast
+-- This module contains avast av access functions
+--]]
+
+local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_regexp = require "rspamd_regexp"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "avast"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function avast_config(opts)
+ local avast_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ timeout = 4.0, -- FIXME: this will break task_timeout!
+ log_clean = false,
+ detection_category = "virus",
+ retransmits = 1,
+ servers = nil, -- e.g. /var/run/avast/scan.sock
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ tmpdir = '/tmp',
+ }
+
+ avast_conf = lua_util.override_defaults(avast_conf, opts)
+
+ if not avast_conf.prefix then
+ avast_conf.prefix = 'rs_' .. avast_conf.name .. '_'
+ end
+
+ if not avast_conf.log_prefix then
+ if avast_conf.name:lower() == avast_conf.type:lower() then
+ avast_conf.log_prefix = avast_conf.name
+ else
+ avast_conf.log_prefix = avast_conf.name .. ' (' .. avast_conf.type .. ')'
+ end
+ end
+
+ if not avast_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers/unix socket defined')
+
+ return nil
+ end
+
+ avast_conf['upstreams'] = upstream_list.create(rspamd_config,
+ avast_conf['servers'],
+ 0)
+
+ if avast_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', avast_conf.name)
+ return avast_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ avast_conf['servers'])
+ return nil
+end
+
+local function avast_check(task, content, digest, rule, maybe_part)
+ local function avast_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local CRLF = '\r\n'
+
+ -- Common tcp options
+ local tcp_opts = {
+ stop_pattern = CRLF,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout,
+ task = task
+ }
+
+ -- Regexps to process reply from avast
+ local clean_re = rspamd_regexp.create_cached(
+ [=[(?!\\)\t\[\+\]]=]
+ )
+ local virus_re = rspamd_regexp.create_cached(
+ [[(?!\\)\t\[L\]\d\.\d\t\d\s(.*)]]
+ )
+ local error_re = rspamd_regexp.create_cached(
+ [[(?!\\)\t\[E\]\d+\.0\tError\s\d+\s(.*)]]
+ )
+
+ -- Used to make a dialog
+ local tcp_conn
+
+ -- Save content in file as avast can work with files only
+ local fname = string.format('%s/%s.avtmp',
+ rule.tmpdir, rspamd_util.random_hex(32))
+ local message_fd = rspamd_util.create_file(fname)
+
+ if not message_fd then
+ rspamd_logger.errx('cannot store file for avast scan: %s', fname)
+ return
+ end
+
+ if type(content) == 'string' then
+ -- Create rspamd_text
+ local rspamd_text = require "rspamd_text"
+ content = rspamd_text.fromstring(content)
+ end
+ content:save_in_file(message_fd)
+
+ -- Ensure file cleanup on task processed
+ task:get_mempool():add_destructor(function()
+ os.remove(fname)
+ rspamd_util.close_file(message_fd)
+ end)
+
+ -- Dialog stages closures
+ local avast_helo_cb
+ local avast_scan_cb
+ local avast_scan_done_cb
+
+ -- Utility closures
+ local function maybe_retransmit()
+ if retransmits > 0 then
+ retransmits = retransmits - 1
+ else
+ rspamd_logger.errx(task,
+ '%s [%s]: failed to scan, maximum retransmits exceed',
+ rule['symbol'], rule['type'])
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed',
+ 0.0, 'fail', maybe_part)
+
+ return
+ end
+
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+ tcp_opts.upstream = upstream
+ tcp_opts.callback = avast_helo_cb
+
+ local is_succ, err = tcp.request(tcp_opts)
+
+ if not is_succ then
+ rspamd_logger.infox(task, 'cannot create connection to avast server: %s (%s)',
+ addr:to_string(true), err)
+ else
+ lua_util.debugm(rule.log_prefix, task, 'established connection to %s; retransmits=%s',
+ addr:to_string(true), retransmits)
+ end
+ end
+
+ local function no_connection_error(err)
+ if err then
+ if tcp_conn then
+ tcp_conn:close()
+ tcp_conn = nil
+
+ rspamd_logger.infox(task, 'failed to request to avast (%s): %s',
+ addr:to_string(true), err)
+ maybe_retransmit()
+ end
+
+ return false
+ end
+
+ return true
+ end
+
+
+ -- Define callbacks
+ avast_helo_cb = function(merr, mdata, conn)
+ -- Called when we have established a connection but not read anything
+ tcp_conn = conn
+
+ if no_connection_error(merr) then
+ -- Check mdata to ensure that it starts with 220
+ if #mdata > 3 and tostring(mdata:span(1, 3)) == '220' then
+ tcp_conn:add_write(avast_scan_cb, string.format(
+ 'SCAN %s%s', fname, CRLF))
+ else
+ rspamd_logger.errx(task, 'Unhandled response: %s', mdata)
+ end
+ end
+ end
+
+ avast_scan_cb = function(merr)
+ -- Called when we have send request to avast and are waiting for reply
+ if no_connection_error(merr) then
+ tcp_conn:add_read(avast_scan_done_cb, CRLF)
+ end
+ end
+
+ avast_scan_done_cb = function(merr, mdata)
+ if no_connection_error(merr) then
+ lua_util.debugm(rule.log_prefix, task, 'got reply from avast: %s',
+ mdata)
+ if #mdata > 4 then
+ local beg = tostring(mdata:span(1, 3))
+
+ if beg == '210' then
+ -- Ignore 210, fire another read
+ if tcp_conn then
+ tcp_conn:add_read(avast_scan_done_cb, CRLF)
+ end
+ elseif beg == '200' then
+ -- Final line
+ if tcp_conn then
+ tcp_conn:close()
+ tcp_conn = nil
+ end
+ else
+ -- Check line using regular expressions
+ local cached
+ local ret = clean_re:search(mdata, false, true)
+
+ if ret then
+ cached = 'OK'
+ if rule.log_clean then
+ rspamd_logger.infox(task,
+ '%s [%s]: message or mime_part is clean',
+ rule.symbol, rule.type)
+ end
+ end
+
+ if not cached then
+ ret = virus_re:search(mdata, false, true)
+
+ if ret then
+ local vname = ret[1][2]
+
+ if vname then
+ vname = vname:gsub('\\ ', ' '):gsub('\\\\', '\\')
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ cached = vname
+ end
+ end
+ end
+
+ if not cached then
+ ret = error_re:search(mdata, false, true)
+
+ if ret then
+ rspamd_logger.errx(task, '%s: error: %s', rule.log_prefix, ret[1][2])
+ common.yield_result(task, rule, 'error:' .. ret[1][2],
+ 0.0, 'fail', maybe_part)
+ end
+ end
+
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ else
+ -- Unexpected reply
+ rspamd_logger.errx(task, '%s: unexpected reply: %s', rule.log_prefix, mdata)
+ end
+ -- Read more
+ if tcp_conn then
+ tcp_conn:add_read(avast_scan_done_cb, CRLF)
+ end
+ end
+ end
+ end
+ end
+
+ -- Send the real request
+ maybe_retransmit()
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ avast_check_uncached, maybe_part) then
+ return
+ else
+ avast_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'Avast antivirus',
+ configure = avast_config,
+ check = avast_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/clamav.lua b/lualib/lua_scanners/clamav.lua
new file mode 100644
index 0000000..fc99ab0
--- /dev/null
+++ b/lualib/lua_scanners/clamav.lua
@@ -0,0 +1,193 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module clamav
+-- This module contains clamav access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "clamav"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function clamav_config(opts)
+ local clamav_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 3310,
+ log_clean = false,
+ timeout = 5.0, -- FIXME: this will break task_timeout!
+ detection_category = "virus",
+ retransmits = 2,
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ }
+
+ clamav_conf = lua_util.override_defaults(clamav_conf, opts)
+
+ if not clamav_conf.prefix then
+ clamav_conf.prefix = 'rs_' .. clamav_conf.name .. '_'
+ end
+
+ if not clamav_conf.log_prefix then
+ if clamav_conf.name:lower() == clamav_conf.type:lower() then
+ clamav_conf.log_prefix = clamav_conf.name
+ else
+ clamav_conf.log_prefix = clamav_conf.name .. ' (' .. clamav_conf.type .. ')'
+ end
+ end
+
+ if not clamav_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ clamav_conf['upstreams'] = upstream_list.create(rspamd_config,
+ clamav_conf['servers'],
+ clamav_conf.default_port)
+
+ if clamav_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', clamav_conf.name)
+ return clamav_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ clamav_conf['servers'])
+ return nil
+end
+
+local function clamav_check(task, content, digest, rule, maybe_part)
+ local function clamav_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local header = rspamd_util.pack("c9 c1 >I4", "zINSTREAM", "\0",
+ #content)
+ local footer = rspamd_util.pack(">I4", 0)
+
+ local function clamav_callback(err, data)
+ if err then
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = clamav_callback,
+ data = { header, content, footer },
+ stop_pattern = '\0'
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed', rule.log_prefix)
+ common.yield_result(task, rule,
+ 'failed to scan and retransmits exceed', 0.0, 'fail',
+ maybe_part)
+ end
+
+ else
+ data = tostring(data)
+ local cached
+ lua_util.debugm(rule.name, task, '%s: got reply: %s',
+ rule.log_prefix, data)
+ if data == 'stream: OK' then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: message or mime_part is clean',
+ rule.log_prefix)
+ else
+ lua_util.debugm(rule.name, task, '%s: message or mime_part is clean', rule.log_prefix)
+ end
+ else
+ local vname = string.match(data, 'stream: (.+) FOUND')
+ if string.find(vname, '^Heuristics%.Encrypted') then
+ rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix)
+ common.yield_result(task, rule, 'File is encrypted: ' .. vname,
+ 0.0, 'encrypted', maybe_part)
+ cached = 'ENCRYPTED'
+ elseif string.find(vname, '^Heuristics%.OLE2%.ContainsMacros') then
+ rspamd_logger.errx(task, '%s: ClamAV Found an OLE2 Office Macro', rule.log_prefix)
+ common.yield_result(task, rule, vname, 0.0, 'macro', maybe_part)
+ cached = 'MACRO'
+ elseif string.find(vname, '^Heuristics%.Limits%.Exceeded') then
+ rspamd_logger.errx(task, '%s: ClamAV Limits Exceeded', rule.log_prefix)
+ common.yield_result(task, rule, 'Limits Exceeded: ' .. vname, 0.0,
+ 'fail', maybe_part)
+ elseif vname then
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ cached = vname
+ else
+ rspamd_logger.errx(task, '%s: unhandled response: %s', rule.log_prefix, data)
+ common.yield_result(task, rule, 'unhandled response:' .. vname, 0.0,
+ 'fail', maybe_part)
+ end
+ end
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ timeout = rule['timeout'],
+ callback = clamav_callback,
+ upstream = upstream,
+ data = { header, content, footer },
+ stop_pattern = '\0'
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ clamav_check_uncached, maybe_part) then
+ return
+ else
+ clamav_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'clamav antivirus',
+ configure = clamav_config,
+ check = clamav_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/cloudmark.lua b/lualib/lua_scanners/cloudmark.lua
new file mode 100644
index 0000000..b07f238
--- /dev/null
+++ b/lualib/lua_scanners/cloudmark.lua
@@ -0,0 +1,372 @@
+--[[
+Copyright (c) 2021, Alexander Moisseev <moiseev@mezonplus.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module cloudmark
+-- This module contains Cloudmark v2 interface
+--]]
+
+local lua_util = require "lua_util"
+local http = require "rspamd_http"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+local rspamd_util = require "rspamd_util"
+local common = require "lua_scanners/common"
+local fun = require "fun"
+local lua_mime = require "lua_mime"
+
+local N = 'cloudmark'
+-- Boundary for multipart transfers, generated on module init
+local static_boundary = rspamd_util.random_hex(32)
+
+local function cloudmark_url(rule, addr, maybe_url)
+ local url
+ local port = addr:get_port()
+
+ maybe_url = maybe_url or rule.url
+ if port == 0 then
+ port = rule.default_port
+ end
+ if rule.use_https then
+ url = string.format('https://%s:%d%s', tostring(addr),
+ port, maybe_url)
+ else
+ url = string.format('http://%s:%d%s', tostring(addr),
+ port, maybe_url)
+ end
+
+ return url
+end
+
+-- Detect cloudmark max size
+local function cloudmark_preload(rule, cfg, ev_base, _)
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local function max_message_size_cb(http_err, code, body, _)
+ if http_err then
+ rspamd_logger.errx(ev_base, 'HTTP error when getting max message size: %s',
+ http_err)
+ return
+ end
+ if code ~= 200 then
+ rspamd_logger.errx(ev_base, 'bad HTTP code when getting max message size: %s', code)
+ end
+ local parser = ucl.parser()
+ local ret, err = parser:parse_string(body)
+ if not ret then
+ rspamd_logger.errx(ev_base, 'could not parse response body [%s]: %s', body, err)
+ return
+ end
+ local obj = parser:get_object()
+ local ms = obj.maxMessageSize
+ if not ms then
+ rspamd_logger.errx(ev_base, 'missing maxMessageSize in the response body (JSON): %s', obj)
+ return
+ end
+
+ rule.max_size = ms
+ lua_util.debugm(N, cfg, 'set maximum message size set to %s bytes', ms)
+ end
+ http.request({
+ ev_base = ev_base,
+ config = cfg,
+ url = cloudmark_url(rule, addr, '/score/v2/max-message-size'),
+ callback = max_message_size_cb,
+ })
+end
+
+local function cloudmark_config(opts)
+
+ local cloudmark_conf = {
+ name = N,
+ default_port = 2713,
+ url = '/score/v2/message',
+ use_https = false,
+ timeout = 5.0,
+ log_clean = false,
+ retransmits = 1,
+ score_threshold = 90, -- minimum score to considerate reply
+ message = '${SCANNER}: spam message found: "${VIRUS}"',
+ max_message = 0,
+ detection_category = "hash",
+ default_score = 1,
+ action = false,
+ log_spamcause = true,
+ symbol_fail = 'CLOUDMARK_FAIL',
+ symbol = 'CLOUDMARK_CHECK',
+ symbol_spam = 'CLOUDMARK_SPAM',
+ add_headers = false, -- allow addition of the headers from Cloudmark
+ }
+
+ cloudmark_conf = lua_util.override_defaults(cloudmark_conf, opts)
+
+ if not cloudmark_conf.prefix then
+ cloudmark_conf.prefix = 'rs_' .. cloudmark_conf.name .. '_'
+ end
+
+ if not cloudmark_conf.log_prefix then
+ if cloudmark_conf.name:lower() == cloudmark_conf.type:lower() then
+ cloudmark_conf.log_prefix = cloudmark_conf.name
+ else
+ cloudmark_conf.log_prefix = cloudmark_conf.name .. ' (' .. cloudmark_conf.type .. ')'
+ end
+ end
+
+ if not cloudmark_conf.servers and cloudmark_conf.socket then
+ cloudmark_conf.servers = cloudmark_conf.socket
+ end
+
+ if not cloudmark_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ cloudmark_conf.upstreams = upstream_list.create(rspamd_config,
+ cloudmark_conf.servers,
+ cloudmark_conf.default_port)
+
+ if cloudmark_conf.upstreams then
+
+ cloudmark_conf.symbols = { { symbol = cloudmark_conf.symbol_spam, score = 5.0 } }
+ cloudmark_conf.preloads = { cloudmark_preload }
+ lua_util.add_debug_alias('external_services', cloudmark_conf.name)
+ return cloudmark_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ cloudmark_conf['servers'])
+ return nil
+end
+
+-- Converts a key-value map to the table representing multipart body, with the following values:
+-- `data`: data of the part
+-- `filename`: optional filename
+-- `content-type`: content type of the element (optional)
+-- `content-transfer-encoding`: optional CTE header
+local function table_to_multipart_body(tbl, boundary)
+ local seen_data = false
+ local out = {}
+
+ for k, v in pairs(tbl) do
+ if v.data then
+ seen_data = true
+ table.insert(out, string.format('--%s\r\n', boundary))
+ if v.filename then
+ table.insert(out,
+ string.format('Content-Disposition: form-data; name="%s"; filename="%s"\r\n',
+ k, v.filename))
+ else
+ table.insert(out,
+ string.format('Content-Disposition: form-data; name="%s"\r\n', k))
+ end
+ if v['content-type'] then
+ table.insert(out,
+ string.format('Content-Type: %s\r\n', v['content-type']))
+ else
+ table.insert(out, 'Content-Type: text/plain\r\n')
+ end
+ if v['content-transfer-encoding'] then
+ table.insert(out,
+ string.format('Content-Transfer-Encoding: %s\r\n',
+ v['content-transfer-encoding']))
+ else
+ table.insert(out, 'Content-Transfer-Encoding: binary\r\n')
+ end
+ table.insert(out, '\r\n')
+ table.insert(out, v.data)
+ table.insert(out, '\r\n')
+ end
+ end
+
+ if seen_data then
+ table.insert(out, string.format('--%s--\r\n', boundary))
+ end
+
+ return out
+end
+
+local function parse_cloudmark_reply(task, rule, body)
+ local parser = ucl.parser()
+ local ret, err = parser:parse_string(body)
+ if not ret then
+ rspamd_logger.errx(task, '%s: bad response body (raw): %s', N, body)
+ task:insert_result(rule.symbol_fail, 1.0, 'Parser error: ' .. err)
+ return
+ end
+ local obj = parser:get_object()
+ lua_util.debugm(N, task, 'cloudmark reply is: %s', obj)
+
+ if not obj.score then
+ rspamd_logger.errx(task, '%s: bad response body (raw): %s', N, body)
+ task:insert_result(rule.symbol_fail, 1.0, 'Parser error: no score')
+ return
+ end
+
+ if obj.analysis then
+ -- Report analysis string
+ rspamd_logger.infox(task, 'cloudmark report string: %s', obj.analysis)
+ end
+
+ local score = tonumber(obj.score) or 0
+ if score >= rule.score_threshold then
+ task:insert_result(rule.symbol_spam, 1.0, tostring(score))
+ end
+
+ if rule.add_headers and type(obj.appendHeaders) == 'table' then
+ local headers_add = fun.tomap(fun.map(function(h)
+ return h.headerField, {
+ order = 1, value = h.body
+ }
+ end, obj.appendHeaders))
+ lua_mime.modify_headers(task, {
+ add = headers_add
+ })
+ end
+
+end
+
+local function cloudmark_check(task, content, digest, rule, maybe_part)
+ local function cloudmark_check_uncached()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ local url = cloudmark_url(rule, addr)
+ local message_data = task:get_content()
+ if rule.max_message and rule.max_message > 0 and #message_data > rule.max_message then
+ task:insert_result(rule['symbol_fail'], 0.0, 'Message too large: ' .. #message_data)
+ return
+ end
+ local request = {
+ rfc822 = {
+ ['Content-Type'] = 'message/rfc822',
+ data = message_data,
+ }
+ }
+
+ local helo = task:get_helo()
+ if helo then
+ request['heloDomain'] = {
+ data = helo,
+ }
+ end
+ local mail_from = task:get_from('smtp') or {}
+ if mail_from[1] and #mail_from[1].addr > 1 then
+ request['mailFrom'] = {
+ data = mail_from[1].addr
+ }
+ end
+
+ local rcpt_to = task:get_recipients('smtp')
+ if rcpt_to then
+ request['rcptTo'] = {
+ data = table.concat(fun.totable(fun.map(function(r)
+ return r.addr
+ end, rcpt_to)), ',')
+ }
+ end
+
+ local fip = task:get_from_ip()
+ if fip and fip:is_valid() then
+ request['connIp'] = tostring(fip)
+ end
+
+ local hostname = task:get_hostname()
+ if hostname then
+ request['fromHost'] = hostname
+ end
+
+ local request_data = {
+ task = task,
+ url = url,
+ body = table_to_multipart_body(request, static_boundary),
+ headers = {
+ ['Content-Type'] = string.format('multipart/form-data; boundary="%s"', static_boundary)
+ },
+ timeout = rule.timeout,
+ }
+
+ local function cloudmark_callback(http_err, code, body, headers)
+
+ local function cloudmark_requery()
+ -- set current upstream to fail because an error occurred
+ upstream:fail()
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.name, task,
+ '%s: request Error: %s - retries left: %s',
+ rule.log_prefix, http_err, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+ url = cloudmark_url(rule, addr)
+
+ lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+ request_data.url = url
+
+ http.request(request_data)
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
+ task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' ..
+ 'retransmits exceed')
+ upstream:fail()
+ end
+ end
+
+ if http_err then
+ cloudmark_requery()
+ else
+ -- Parse the response
+ if upstream then
+ upstream:ok()
+ end
+ if code ~= 200 then
+ rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
+ return
+ end
+ parse_cloudmark_reply(task, rule, body)
+ end
+ end
+
+ request_data.callback = cloudmark_callback
+ http.request(request_data)
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ cloudmark_check_uncached, maybe_part) then
+ return
+ else
+ cloudmark_check_uncached()
+ end
+end
+
+return {
+ type = { 'cloudmark', 'scanner' },
+ description = 'Cloudmark cartridge interface',
+ configure = cloudmark_config,
+ check = cloudmark_check,
+ name = N,
+}
diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua
new file mode 100644
index 0000000..11f5e1f
--- /dev/null
+++ b/lualib/lua_scanners/common.lua
@@ -0,0 +1,539 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_scanners_common
+-- This module contains common external scanners functions
+--]]
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_regexp = require "rspamd_regexp"
+local lua_util = require "lua_util"
+local lua_redis = require "lua_redis"
+local lua_magic_types = require "lua_magic/types"
+local fun = require "fun"
+
+local exports = {}
+
+local function log_clean(task, rule, msg)
+
+ msg = msg or 'message or mime_part is clean'
+
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: %s', rule.log_prefix, msg)
+ else
+ lua_util.debugm(rule.name, task, '%s: %s', rule.log_prefix, msg)
+ end
+
+end
+
+local function match_patterns(default_sym, found, patterns, dyn_weight)
+ if type(patterns) ~= 'table' then
+ return default_sym, dyn_weight
+ end
+ if not patterns[1] then
+ for sym, pat in pairs(patterns) do
+ if pat:match(found) then
+ return sym, '1'
+ end
+ end
+ return default_sym, dyn_weight
+ else
+ for _, p in ipairs(patterns) do
+ for sym, pat in pairs(p) do
+ if pat:match(found) then
+ return sym, '1'
+ end
+ end
+ end
+ return default_sym, dyn_weight
+ end
+end
+
+local function yield_result(task, rule, vname, dyn_weight, is_fail, maybe_part)
+ local all_whitelisted = true
+ local patterns
+ local symbol
+ local threat_table
+ local threat_info
+ local flags
+
+ if type(vname) == 'string' then
+ threat_table = { vname }
+ elseif type(vname) == 'table' then
+ threat_table = vname
+ end
+
+
+ -- This should be more generic
+ if not is_fail then
+ patterns = rule.patterns
+ symbol = rule.symbol
+ threat_info = rule.detection_category .. 'found'
+ if not dyn_weight then
+ dyn_weight = 1.0
+ end
+ elseif is_fail == 'fail' then
+ patterns = rule.patterns_fail
+ symbol = rule.symbol_fail
+ threat_info = "FAILED with error"
+ dyn_weight = 0.0
+ elseif is_fail == 'encrypted' then
+ patterns = rule.patterns
+ symbol = rule.symbol_encrypted
+ threat_info = "Scan has returned that input was encrypted"
+ dyn_weight = 1.0
+ elseif is_fail == 'macro' then
+ patterns = rule.patterns
+ symbol = rule.symbol_macro
+ threat_info = "Scan has returned that input contains macros"
+ dyn_weight = 1.0
+ end
+
+ for _, tm in ipairs(threat_table) do
+ local symname, symscore = match_patterns(symbol, tm, patterns, dyn_weight)
+ if rule.whitelist and rule.whitelist:get_key(tm) then
+ rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, tm)
+ else
+ all_whitelisted = false
+ rspamd_logger.infox(task, '%s: result - %s: "%s - score: %s"',
+ rule.log_prefix, threat_info, tm, symscore)
+
+ if maybe_part and rule.show_attachments and maybe_part:get_filename() then
+ local fname = maybe_part:get_filename()
+ task:insert_result(symname, symscore, string.format("%s|%s",
+ tm, fname))
+ else
+ task:insert_result(symname, symscore, tm)
+ end
+
+ end
+ end
+
+ if rule.action and is_fail ~= 'fail' and not all_whitelisted then
+ threat_table = table.concat(threat_table, '; ')
+ if rule.action ~= 'reject' then
+ flags = 'least'
+ end
+ task:set_pre_result(rule.action,
+ lua_util.template(rule.message or 'Rejected', {
+ SCANNER = rule.name,
+ VIRUS = threat_table,
+ }), rule.name, nil, nil, flags)
+ end
+end
+
+local function message_not_too_large(task, content, rule)
+ local max_size = tonumber(rule.max_size)
+ if not max_size then
+ return true
+ end
+ if #content > max_size then
+ rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)",
+ rule.log_prefix, #content, max_size)
+ return false
+ end
+ return true
+end
+
+local function message_not_too_small(task, content, rule)
+ local min_size = tonumber(rule.min_size)
+ if not min_size then
+ return true
+ end
+ if #content < min_size then
+ rspamd_logger.infox(task, "skip %s check as it is too small: %s (%s is allowed)",
+ rule.log_prefix, #content, min_size)
+ return false
+ end
+ return true
+end
+
+local function message_min_words(task, rule)
+ if rule.text_part_min_words and tonumber(rule.text_part_min_words) > 0 then
+ local text_part_above_limit = false
+ local text_parts = task:get_text_parts()
+
+ local filter_func = function(p)
+ return p:get_words_count() >= tonumber(rule.text_part_min_words)
+ end
+
+ fun.each(function(p)
+ text_part_above_limit = true
+ end, fun.filter(filter_func, text_parts))
+
+ if not text_part_above_limit then
+ rspamd_logger.infox(task, '%s: #words in all text parts is below text_part_min_words limit: %s',
+ rule.log_prefix, rule.text_part_min_words)
+ end
+
+ return text_part_above_limit
+ else
+ return true
+ end
+end
+
+local function dynamic_scan(task, rule)
+ if rule.dynamic_scan then
+ if rule.action ~= 'reject' then
+ local metric_result = task:get_metric_score()
+ local metric_action = task:get_metric_action()
+ local has_pre_result = task:has_pre_result()
+ -- ToDo: needed?
+ -- Sometimes leads to FPs
+ --if rule.symbol_type == 'postfilter' and metric_action == 'reject' then
+ -- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, "result is already reject")
+ -- return false
+ --elseif metric_result[1] > metric_result[2]*2 then
+ if metric_result[1] > metric_result[2] * 2 then
+ rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'score > 2 * reject_level: ' .. metric_result[1])
+ return false
+ elseif has_pre_result and metric_action == 'reject' then
+ rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'pre_result reject is set')
+ return false
+ else
+ return true, 'undecided'
+ end
+ else
+ return true, 'dynamic_scan is not possible with config `action=reject;`'
+ end
+ else
+ return true
+ end
+end
+
+local function need_check(task, content, rule, digest, fn, maybe_part)
+
+ local uncached = true
+ local key = digest
+
+ local function redis_av_cb(err, data)
+ if data and type(data) == 'string' then
+ -- Cached
+ data = lua_util.str_split(data, '\t')
+ local threat_string = lua_util.str_split(data[1], '\v')
+ local score = data[2] or rule.default_score
+
+ if threat_string[1] ~= 'OK' then
+ if threat_string[1] == 'MACRO' then
+ yield_result(task, rule, 'File contains macros',
+ 0.0, 'macro', maybe_part)
+ elseif threat_string[1] == 'ENCRYPTED' then
+ yield_result(task, rule, 'File is encrypted',
+ 0.0, 'encrypted', maybe_part)
+ else
+ lua_util.debugm(rule.name, task, '%s: got cached threat result for %s: %s - score: %s',
+ rule.log_prefix, key, threat_string[1], score)
+ yield_result(task, rule, threat_string, score, false, maybe_part)
+ end
+
+ else
+ lua_util.debugm(rule.name, task, '%s: got cached negative result for %s: %s',
+ rule.log_prefix, key, threat_string[1])
+ end
+ uncached = false
+ else
+ if err then
+ rspamd_logger.errx(task, 'got error checking cache: %s', err)
+ end
+ end
+
+ local f_message_not_too_large = message_not_too_large(task, content, rule)
+ local f_message_not_too_small = message_not_too_small(task, content, rule)
+ local f_message_min_words = message_min_words(task, rule)
+ local f_dynamic_scan = dynamic_scan(task, rule)
+
+ if uncached and
+ f_message_not_too_large and
+ f_message_not_too_small and
+ f_message_min_words and
+ f_dynamic_scan then
+
+ fn()
+
+ end
+
+ end
+
+ if rule.redis_params and not rule.no_cache then
+
+ key = rule.prefix .. key
+
+ if lua_redis.redis_make_request(task,
+ rule.redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_av_cb, --callback
+ 'GET', -- command
+ { key } -- arguments)
+ ) then
+ return true
+ end
+ end
+
+ return false
+
+end
+
+local function save_cache(task, digest, rule, to_save, dyn_weight, maybe_part)
+ local key = digest
+ if not dyn_weight then
+ dyn_weight = 1.0
+ end
+
+ local function redis_set_cb(err)
+ -- Do nothing
+ if err then
+ rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s',
+ rule.detection_category, to_save, key, err)
+ else
+ lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s - ttl %s',
+ rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire)
+ end
+ end
+
+ if type(to_save) == 'table' then
+ to_save = table.concat(to_save, '\v')
+ end
+
+ local value_tbl = { to_save, dyn_weight }
+ if maybe_part and rule.show_attachments and maybe_part:get_filename() then
+ local fname = maybe_part:get_filename()
+ table.insert(value_tbl, fname)
+ end
+ local value = table.concat(value_tbl, '\t')
+
+ if rule.redis_params and rule.prefix then
+ key = rule.prefix .. key
+
+ lua_redis.redis_make_request(task,
+ rule.redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'SETEX', -- command
+ { key, rule.cache_expire or 0, value }
+ )
+ end
+
+ return false
+end
+
+local function create_regex_table(patterns)
+ local regex_table = {}
+ if patterns[1] then
+ for i, p in ipairs(patterns) do
+ if type(p) == 'table' then
+ local new_set = {}
+ for k, v in pairs(p) do
+ new_set[k] = rspamd_regexp.create_cached(v)
+ end
+ regex_table[i] = new_set
+ else
+ regex_table[i] = {}
+ end
+ end
+ else
+ for k, v in pairs(patterns) do
+ regex_table[k] = rspamd_regexp.create_cached(v)
+ end
+ end
+ return regex_table
+end
+
+local function match_filter(task, rule, found, patterns, pat_type)
+ if type(patterns) ~= 'table' or not found then
+ return false
+ end
+ if not patterns[1] then
+ for _, pat in pairs(patterns) do
+ if pat_type == 'ext' and tostring(pat) == tostring(found) then
+ return true
+ elseif pat_type == 'regex' and pat:match(found) then
+ return true
+ end
+ end
+ return false
+ else
+ for _, p in ipairs(patterns) do
+ for _, pat in ipairs(p) do
+ if pat_type == 'ext' and tostring(pat) == tostring(found) then
+ return true
+ elseif pat_type == 'regex' and pat:match(found) then
+ return true
+ end
+ end
+ end
+ return false
+ end
+end
+
+-- borrowed from mime_types.lua
+-- ext is the last extension, LOWERCASED
+-- ext2 is the one before last extension LOWERCASED
+local function gen_extension(fname)
+ local filename_parts = lua_util.str_split(fname, '.')
+
+ local ext = {}
+ for n = 1, 2 do
+ ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil
+ end
+ return ext[1], ext[2], filename_parts
+end
+
+local function check_parts_match(task, rule)
+
+ local filter_func = function(p)
+ local mtype, msubtype = p:get_type()
+ local detected_ext = p:get_detected_ext()
+ local fname = p:get_filename()
+ local ext, ext2
+
+ if rule.scan_all_mime_parts == false then
+ -- check file extension and filename regex matching
+ --lua_util.debugm(rule.name, task, '%s: filename: |%s|%s|', rule.log_prefix, fname)
+ if fname ~= nil then
+ ext, ext2 = gen_extension(fname)
+ --lua_util.debugm(rule.name, task, '%s: extension, fname: |%s|%s|%s|', rule.log_prefix, ext, ext2, fname)
+ if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext')
+ or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then
+ lua_util.debugm(rule.name, task, '%s: extension matched: |%s|%s|', rule.log_prefix, ext, ext2)
+ return true
+ elseif match_filter(task, rule, fname, rule.mime_parts_filter_regex, 'regex') then
+ lua_util.debugm(rule.name, task, '%s: filename regex matched', rule.log_prefix)
+ return true
+ end
+ end
+ -- check content type string regex matching
+ if mtype ~= nil and msubtype ~= nil then
+ local ct = string.format('%s/%s', mtype, msubtype):lower()
+ if match_filter(task, rule, ct, rule.mime_parts_filter_regex, 'regex') then
+ lua_util.debugm(rule.name, task, '%s: regex content-type: %s', rule.log_prefix, ct)
+ return true
+ end
+ end
+ -- check detected content type (libmagic) regex matching
+ if detected_ext then
+ local magic = lua_magic_types[detected_ext] or {}
+ if match_filter(task, rule, detected_ext, rule.mime_parts_filter_ext, 'ext') then
+ lua_util.debugm(rule.name, task, '%s: detected extension matched: |%s|', rule.log_prefix, detected_ext)
+ return true
+ elseif magic.ct and match_filter(task, rule, magic.ct, rule.mime_parts_filter_regex, 'regex') then
+ lua_util.debugm(rule.name, task, '%s: regex detected libmagic content-type: %s',
+ rule.log_prefix, magic.ct)
+ return true
+ end
+ end
+ -- check filenames in archives
+ if p:is_archive() then
+ local arch = p:get_archive()
+ local filelist = arch:get_files_full(1000)
+ for _, f in ipairs(filelist) do
+ ext, ext2 = gen_extension(f.name)
+ if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext')
+ or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then
+ lua_util.debugm(rule.name, task, '%s: extension matched in archive: |%s|%s|', rule.log_prefix, ext, ext2)
+ --lua_util.debugm(rule.name, task, '%s: extension matched in archive: %s', rule.log_prefix, ext)
+ return true
+ elseif match_filter(task, rule, f.name, rule.mime_parts_filter_regex, 'regex') then
+ lua_util.debugm(rule.name, task, '%s: filename regex matched in archive', rule.log_prefix)
+ return true
+ end
+ end
+ end
+ end
+
+ -- check text_part has more words than text_part_min_words_check
+ if rule.scan_text_mime and rule.text_part_min_words and p:is_text() and
+ p:get_words_count() >= tonumber(rule.text_part_min_words) then
+ return true
+ end
+
+ if rule.scan_image_mime and p:is_image() then
+ return true
+ end
+
+ if rule.scan_all_mime_parts ~= false then
+ local is_part_checkable = (p:is_attachment() and (not p:is_image() or rule.scan_image_mime))
+ if detected_ext then
+ -- We know what to scan!
+ local magic = lua_magic_types[detected_ext] or {}
+
+ if magic.av_check ~= false or is_part_checkable then
+ return true
+ end
+ elseif is_part_checkable then
+ -- Just rely on attachment property
+ return true
+ end
+ end
+
+ return false
+ end
+
+ return fun.filter(filter_func, task:get_parts())
+end
+
+local function check_metric_results(task, rule)
+
+ if rule.action ~= 'reject' then
+ local metric_result = task:get_metric_score()
+ local metric_action = task:get_metric_action()
+ local has_pre_result = task:has_pre_result()
+
+ if rule.symbol_type == 'postfilter' and metric_action == 'reject' then
+ return true, 'result is already reject'
+ elseif metric_result[1] > metric_result[2] * 2 then
+ return true, 'score > 2 * reject_level: ' .. metric_result[1]
+ elseif has_pre_result and metric_action == 'reject' then
+ return true, 'pre_result reject is set'
+ else
+ return false, 'undecided'
+ end
+ else
+ return false, 'dynamic_scan is not possible with config `action=reject;`'
+ end
+end
+
+exports.log_clean = log_clean
+exports.yield_result = yield_result
+exports.match_patterns = match_patterns
+exports.condition_check_and_continue = need_check
+exports.save_cache = save_cache
+exports.create_regex_table = create_regex_table
+exports.check_parts_match = check_parts_match
+exports.check_metric_results = check_metric_results
+
+setmetatable(exports, {
+ __call = function(t, override)
+ for k, v in pairs(t) do
+ if _G[k] ~= nil then
+ local msg = 'function ' .. k .. ' already exists in global scope.'
+ if override then
+ _G[k] = v
+ print('WARNING: ' .. msg .. ' Overwritten.')
+ else
+ print('NOTICE: ' .. msg .. ' Skipped.')
+ end
+ else
+ _G[k] = v
+ end
+ end
+ end,
+})
+
+return exports
diff --git a/lualib/lua_scanners/dcc.lua b/lualib/lua_scanners/dcc.lua
new file mode 100644
index 0000000..8d5e9e1
--- /dev/null
+++ b/lualib/lua_scanners/dcc.lua
@@ -0,0 +1,313 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module dcc
+-- This module contains dcc access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+local fun = require "fun"
+
+local N = 'dcc'
+
+local function dcc_config(opts)
+
+ local dcc_conf = {
+ name = N,
+ default_port = 10045,
+ timeout = 5.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 7200, -- expire redis in 2h
+ message = '${SCANNER}: bulk message found: "${VIRUS}"',
+ detection_category = "hash",
+ default_score = 1,
+ action = false,
+ client = '0.0.0.0',
+ symbol_fail = 'DCC_FAIL',
+ symbol = 'DCC_REJECT',
+ symbol_bulk = 'DCC_BULK',
+ body_max = 999999,
+ fuz1_max = 999999,
+ fuz2_max = 999999,
+ }
+
+ dcc_conf = lua_util.override_defaults(dcc_conf, opts)
+
+ if not dcc_conf.prefix then
+ dcc_conf.prefix = 'rs_' .. dcc_conf.name .. '_'
+ end
+
+ if not dcc_conf.log_prefix then
+ dcc_conf.log_prefix = dcc_conf.name
+ end
+
+ if not dcc_conf.servers and dcc_conf.socket then
+ dcc_conf.servers = dcc_conf.socket
+ end
+
+ if not dcc_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ dcc_conf.upstreams = upstream_list.create(rspamd_config,
+ dcc_conf.servers,
+ dcc_conf.default_port)
+
+ if dcc_conf.upstreams then
+ lua_util.add_debug_alias('external_services', dcc_conf.name)
+ return dcc_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ dcc_conf['servers'])
+ return nil
+end
+
+local function dcc_check(task, content, digest, rule)
+ local function dcc_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local client = rule.client
+
+ local client_ip = task:get_from_ip()
+ if client_ip and client_ip:is_valid() then
+ client = client_ip:to_string()
+ end
+ local client_host = task:get_hostname()
+ if client_host then
+ client = client .. "\r" .. client_host
+ end
+
+ -- HELO
+ local helo = task:get_helo() or ''
+
+ -- Envelope From
+ local ef = task:get_from()
+ local envfrom = 'test@example.com'
+ if ef and ef[1] then
+ envfrom = ef[1]['addr']
+ end
+
+ -- Envelope To
+ local envrcpt = 'test@example.com'
+ local rcpts = task:get_recipients();
+ if rcpts then
+ local dcc_recipients = table.concat(fun.totable(fun.map(function(rcpt)
+ return rcpt['addr']
+ end,
+ rcpts)), '\n')
+ if dcc_recipients then
+ envrcpt = dcc_recipients
+ end
+ end
+
+ -- Build the DCC query
+ -- https://www.dcc-servers.net/dcc/dcc-tree/dccifd.html#Protocol
+ local request_data = {
+ "header\n",
+ client .. "\n",
+ helo .. "\n",
+ envfrom .. "\n",
+ envrcpt .. "\n",
+ "\n",
+ content
+ }
+
+ local function dcc_callback(err, data, conn)
+
+ local function dcc_requery()
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ timeout = rule.timeout or 2.0,
+ upstream = upstream,
+ shutdown = true,
+ data = request_data,
+ callback = dcc_callback,
+ body_max = 999999,
+ fuz1_max = 999999,
+ fuz2_max = 999999,
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
+ end
+ end
+
+ if err then
+
+ dcc_requery()
+
+ else
+ -- Parse the response
+ local _, _, result, disposition, header = tostring(data):find("(.-)\n(.-)\n(.-)$")
+ lua_util.debugm(rule.name, task, 'DCC result=%1 disposition=%2 header="%3"',
+ result, disposition, header)
+
+ if header then
+ -- Unfold header
+ header = header:gsub('\r?\n%s*', ' ')
+ local _, _, info = header:find("; (.-)$")
+ if (result == 'R') then
+ -- Reject
+ common.yield_result(task, rule, info, rule.default_score)
+ common.save_cache(task, digest, rule, info, rule.default_score)
+ elseif (result == 'T') then
+ -- Temporary failure
+ rspamd_logger.warnx(task, 'DCC returned a temporary failure result: %s', result)
+ dcc_requery()
+ elseif result == 'A' then
+
+ local opts = {}
+ local score = 0.0
+ info = info:lower()
+ local rep = info:match('rep=([^=%s]+)')
+
+ -- Adjust reputation if available
+ if rep then
+ rep = tonumber(rep)
+ end
+ if not rep then
+ rep = 1.0
+ end
+
+ local function check_threshold(what, num, lim)
+ local rnum
+ if num == 'many' then
+ rnum = lim
+ else
+ rnum = tonumber(num)
+ end
+
+ if rnum and rnum >= lim then
+ opts[#opts + 1] = string.format('%s=%s', what, num)
+ score = score + rep / 3.0
+ end
+ end
+
+ info = info:lower()
+ local body = info:match('body=([^=%s]+)')
+
+ if body then
+ check_threshold('body', body, rule.body_max)
+ end
+
+ local fuz1 = info:match('fuz1=([^=%s]+)')
+
+ if fuz1 then
+ check_threshold('fuz1', fuz1, rule.fuz1_max)
+ end
+
+ local fuz2 = info:match('fuz2=([^=%s]+)')
+
+ if fuz2 then
+ check_threshold('fuz2', fuz2, rule.fuz2_max)
+ end
+
+ if #opts > 0 and score > 0 then
+ task:insert_result(rule.symbol_bulk,
+ score,
+ opts)
+ common.save_cache(task, digest, rule, opts, score)
+ else
+ common.save_cache(task, digest, rule, 'OK')
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
+ rule.log_prefix, info)
+ else
+ lua_util.debugm(rule.name, task, '%s: returned result A - info: %s',
+ rule.log_prefix, info)
+ end
+ end
+ elseif result == 'G' then
+ -- do nothing
+ common.save_cache(task, digest, rule, 'OK')
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: clean, returned result G - info: %s', rule.log_prefix, info)
+ else
+ lua_util.debugm(rule.name, task, '%s: returned result G - info: %s', rule.log_prefix, info)
+ end
+ elseif result == 'S' then
+ -- do nothing
+ common.save_cache(task, digest, rule, 'OK')
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: clean, returned result S - info: %s', rule.log_prefix, info)
+ else
+ lua_util.debugm(rule.name, task, '%s: returned result S - info: %s', rule.log_prefix, info)
+ end
+ else
+ -- Unknown result
+ rspamd_logger.warnx(task, '%s: result error: %1', rule.log_prefix, result);
+ common.yield_result(task, rule, 'error: ' .. result, 0.0, 'fail')
+ end
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ timeout = rule.timeout or 2.0,
+ shutdown = true,
+ upstream = upstream,
+ data = request_data,
+ callback = dcc_callback,
+ body_max = 999999,
+ fuz1_max = 999999,
+ fuz2_max = 999999,
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest, dcc_check_uncached) then
+ return
+ else
+ dcc_check_uncached()
+ end
+
+end
+
+return {
+ type = { 'dcc', 'bulk', 'hash', 'scanner' },
+ description = 'dcc bulk scanner',
+ configure = dcc_config,
+ check = dcc_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/fprot.lua b/lualib/lua_scanners/fprot.lua
new file mode 100644
index 0000000..5a469c3
--- /dev/null
+++ b/lualib/lua_scanners/fprot.lua
@@ -0,0 +1,181 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module fprot
+-- This module contains fprot access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "fprot"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function fprot_config(opts)
+ local fprot_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 10200,
+ timeout = 5.0, -- FIXME: this will break task_timeout!
+ log_clean = false,
+ detection_category = "virus",
+ retransmits = 2,
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ }
+
+ fprot_conf = lua_util.override_defaults(fprot_conf, opts)
+
+ if not fprot_conf.prefix then
+ fprot_conf.prefix = 'rs_' .. fprot_conf.name .. '_'
+ end
+
+ if not fprot_conf.log_prefix then
+ if fprot_conf.name:lower() == fprot_conf.type:lower() then
+ fprot_conf.log_prefix = fprot_conf.name
+ else
+ fprot_conf.log_prefix = fprot_conf.name .. ' (' .. fprot_conf.type .. ')'
+ end
+ end
+
+ if not fprot_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ fprot_conf['upstreams'] = upstream_list.create(rspamd_config,
+ fprot_conf['servers'],
+ fprot_conf.default_port)
+
+ if fprot_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', fprot_conf.name)
+ return fprot_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ fprot_conf['servers'])
+ return nil
+end
+
+local function fprot_check(task, content, digest, rule, maybe_part)
+ local function fprot_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local scan_id = task:get_queue_id()
+ if not scan_id then
+ scan_id = task:get_uid()
+ end
+ local header = string.format('SCAN STREAM %s SIZE %d\n', scan_id,
+ #content)
+ local footer = '\n'
+
+ local function fprot_callback(err, data)
+ if err then
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = fprot_callback,
+ data = { header, content, footer },
+ stop_pattern = '\n'
+ })
+ else
+ rspamd_logger.errx(task,
+ '%s [%s]: failed to scan, maximum retransmits exceed',
+ rule['symbol'], rule['type'])
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed',
+ 0.0, 'fail', maybe_part)
+ end
+ else
+ upstream:ok()
+ data = tostring(data)
+ local cached
+ local clean = string.match(data, '^0 <clean>')
+ if clean then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task,
+ '%s [%s]: message or mime_part is clean',
+ rule['symbol'], rule['type'])
+ end
+ else
+ -- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occurred
+ -- see http://www.f-prot.com/support/helpfiles/unix/appendix_c.html for more detail
+ local vname = string.match(data, '^[1-3] <[%w%s]-: (.-)>')
+ if not vname then
+ rspamd_logger.errx(task, 'Unhandled response: %s', data)
+ else
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ cached = vname
+ end
+ end
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = fprot_callback,
+ data = { header, content, footer },
+ stop_pattern = '\n'
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ fprot_check_uncached, maybe_part) then
+ return
+ else
+ fprot_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'fprot antivirus',
+ configure = fprot_config,
+ check = fprot_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/icap.lua b/lualib/lua_scanners/icap.lua
new file mode 100644
index 0000000..682562d
--- /dev/null
+++ b/lualib/lua_scanners/icap.lua
@@ -0,0 +1,713 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[
+@module icap
+This module contains icap access functions.
+Currently tested with
+ - C-ICAP Squidclamav / echo
+ - Checkpoint Sandblast
+ - F-Secure Internet Gatekeeper
+ - Kaspersky Web Traffic Security
+ - Kaspersky Scan Engine 2.0
+ - McAfee Web Gateway 9/10/11
+ - Sophos Savdi
+ - Symantec (Rspamd <3.2, >=3.2 untested)
+ - Trend Micro IWSVA 6.0
+ - Trend Micro Web Gateway
+
+@TODO
+ - Preview / Continue
+ - Reqmod URL's
+ - Content-Type / Filename
+]] --
+
+--[[
+Configuration Notes:
+
+C-ICAP Squidclamav
+ scheme = "squidclamav";
+
+Checkpoint Sandblast example:
+ scheme = "sandblast";
+
+ESET Gateway Security / Antivirus for Linux example:
+ scheme = "scan";
+
+F-Secure Internet Gatekeeper example:
+ scheme = "respmod";
+ x_client_header = true;
+ x_rcpt_header = true;
+ x_from_header = true;
+
+Kaspersky Web Traffic Security example:
+ scheme = "av/respmod";
+ x_client_header = true;
+
+Kaspersky Web Traffic Security (as configured in kavicapd.xml):
+ scheme = "resp";
+ x_client_header = true;
+
+McAfee Web Gateway 10/11 (Headers must be activated with personal extra Rules)
+ scheme = "respmod";
+ x_client_header = true;
+
+Sophos SAVDI example:
+ # scheme as configured in savdi.conf (name option in service section)
+ scheme = "respmod";
+
+Symantec example:
+ scheme = "avscan";
+
+Trend Micro IWSVA example (X-Virus-ID/X-Infection-Found headers must be activated):
+ scheme = "avscan";
+ x_client_header = true;
+
+Trend Micro Web Gateway example (X-Virus-ID/X-Infection-Found headers must be activated):
+ scheme = "interscan";
+ x_client_header = true;
+]] --
+
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+local rspamd_util = require "rspamd_util"
+local rspamd_version = rspamd_version
+
+local N = 'icap'
+
+local function icap_config(opts)
+
+ local icap_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_all_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ scheme = "scan",
+ default_port = 1344,
+ ssl = false,
+ no_ssl_verify = false,
+ timeout = 10.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 7200, -- expire redis in one hour
+ message = '${SCANNER}: threat found with icap scanner: "${VIRUS}"',
+ detection_category = "virus",
+ default_score = 1,
+ action = false,
+ dynamic_scan = false,
+ user_agent = "Rspamd",
+ x_client_header = false,
+ x_rcpt_header = false,
+ x_from_header = false,
+ req_headers_enabled = true,
+ req_fake_url = "http://127.0.0.1/mail",
+ http_headers_enabled = true,
+ use_http_result_header = true,
+ use_http_3xx_as_threat = false,
+ use_specific_content_type = false, -- Use content type from a part where possible
+ }
+
+ icap_conf = lua_util.override_defaults(icap_conf, opts)
+
+ if not icap_conf.prefix then
+ icap_conf.prefix = 'rs_' .. icap_conf.name .. '_'
+ end
+
+ if not icap_conf.log_prefix then
+ icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')'
+ end
+
+ if not icap_conf.log_prefix then
+ if icap_conf.name:lower() == icap_conf.type:lower() then
+ icap_conf.log_prefix = icap_conf.name
+ else
+ icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')'
+ end
+ end
+
+ if not icap_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ icap_conf.upstreams = upstream_list.create(rspamd_config,
+ icap_conf.servers,
+ icap_conf.default_port)
+
+ if icap_conf.upstreams then
+ lua_util.add_debug_alias('external_services', icap_conf.name)
+ return icap_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ icap_conf.servers)
+ return nil
+end
+
+local function icap_check(task, content, digest, rule, maybe_part)
+ local function icap_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local http_headers = {}
+ local req_headers = {}
+ local tcp_options = {}
+ local threat_table = {}
+
+ -- Build extended User Agent
+ if rule.user_agent == "extended" then
+ rule.user_agent = string.format("Rspamd/%s-%s (%s/%s)",
+ rspamd_version('main'),
+ rspamd_version('id'),
+ rspamd_util.get_hostname(),
+ string.sub(task:get_uid(), 1, 6))
+ end
+
+ -- Build the icap queries
+ local options_request = {
+ string.format("OPTIONS icap://%s/%s ICAP/1.0\r\n", addr:to_string(), rule.scheme),
+ string.format('Host: %s\r\n', addr:to_string()),
+ string.format("User-Agent: %s\r\n", rule.user_agent),
+ "Connection: keep-alive\r\n",
+ "Encapsulated: null-body=0\r\n\r\n",
+ }
+ if rule.user_agent == "none" then
+ table.remove(options_request, 3)
+ end
+
+ local respond_headers = {
+ -- Add main RESPMOD header before any other
+ string.format('RESPMOD icap://%s/%s ICAP/1.0\r\n', addr:to_string(), rule.scheme),
+ string.format('Host: %s\r\n', addr:to_string()),
+ }
+
+ local size = tonumber(#content)
+ local chunked_size = string.format("%x", size)
+
+ local function icap_callback(err, conn)
+
+ local function icap_requery(err_m, info)
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.name, task,
+ '%s: %s Request Error: %s - retries left: %s',
+ rule.log_prefix, info, err_m, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+
+ tcp_options.host = addr:to_string()
+ tcp_options.port = addr:get_port()
+ tcp_options.callback = icap_callback
+ tcp_options.data = options_request
+ tcp_options.upstream = upstream
+
+ tcp.request(tcp_options)
+
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed - error: %s', rule.log_prefix, err_m or '')
+ common.yield_result(task, rule, string.format('failed - error: %s', err_m),
+ 0.0, 'fail', maybe_part)
+ end
+ end
+
+ local function get_req_headers()
+
+ local in_client_ip = task:get_from_ip()
+ local req_hlen = 2
+ if maybe_part then
+ table.insert(req_headers,
+ string.format('GET http://%s/%s HTTP/1.0\r\n', in_client_ip, maybe_part:get_filename()))
+ if rule.use_specific_content_type then
+ table.insert(http_headers, string.format('Content-Type: %s/%s\r\n', maybe_part:get_detected_type()))
+ --else
+ -- To test: what content type is better for icap servers?
+ --table.insert(http_headers, 'Content-Type: text/plain\r\n')
+ end
+ else
+ table.insert(req_headers, string.format('GET %s HTTP/1.0\r\n', rule.req_fake_url))
+ table.insert(http_headers, string.format('Content-Type: application/octet-stream\r\n'))
+ end
+ table.insert(req_headers, string.format('Date: %s\r\n', rspamd_util.time_to_string(rspamd_util.get_time())))
+ if rule.user_agent ~= "none" then
+ table.insert(req_headers, string.format("User-Agent: %s\r\n", rule.user_agent))
+ end
+
+ for _, h in ipairs(req_headers) do
+ req_hlen = req_hlen + tonumber(#h)
+ end
+
+ return req_hlen, req_headers
+
+ end
+
+ local function get_http_headers()
+ local http_hlen = 2
+ table.insert(http_headers, 'HTTP/1.0 200 OK\r\n')
+ table.insert(http_headers, string.format('Date: %s\r\n', rspamd_util.time_to_string(rspamd_util.get_time())))
+ table.insert(http_headers, string.format('Server: %s\r\n', 'Apache/2.4'))
+ if rule.user_agent ~= "none" then
+ table.insert(http_headers, string.format("User-Agent: %s\r\n", rule.user_agent))
+ end
+ --table.insert(http_headers, string.format('Content-Type: %s\r\n', 'text/html'))
+ table.insert(http_headers, string.format('Content-Length: %s\r\n', size))
+
+ for _, h in ipairs(http_headers) do
+ http_hlen = http_hlen + tonumber(#h)
+ end
+
+ return http_hlen, http_headers
+
+ end
+
+ local function get_respond_query()
+ local req_hlen = 0
+ local resp_req_headers
+ local http_hlen = 0
+ local resp_http_headers
+
+ -- Append all extra headers
+ if rule.user_agent ~= "none" then
+ table.insert(respond_headers,
+ string.format("User-Agent: %s\r\n", rule.user_agent))
+ end
+
+ if rule.req_headers_enabled then
+ req_hlen, resp_req_headers = get_req_headers()
+ end
+ if rule.http_headers_enabled then
+ http_hlen, resp_http_headers = get_http_headers()
+ end
+
+ if rule.req_headers_enabled and rule.http_headers_enabled then
+ local res_body_hlen = req_hlen + http_hlen
+ table.insert(respond_headers,
+ string.format('Encapsulated: req-hdr=0, res-hdr=%s, res-body=%s\r\n',
+ req_hlen, res_body_hlen))
+ elseif rule.http_headers_enabled then
+ table.insert(respond_headers,
+ string.format('Encapsulated: res-hdr=0, res-body=%s\r\n',
+ http_hlen))
+ else
+ table.insert(respond_headers, 'Encapsulated: res-body=0\r\n')
+ end
+
+ table.insert(respond_headers, '\r\n')
+ for _, h in ipairs(resp_req_headers) do
+ table.insert(respond_headers, h)
+ end
+ table.insert(respond_headers, '\r\n')
+ for _, h in ipairs(resp_http_headers) do
+ table.insert(respond_headers, h)
+ end
+ table.insert(respond_headers, '\r\n')
+ table.insert(respond_headers, chunked_size .. '\r\n')
+ table.insert(respond_headers, content)
+ table.insert(respond_headers, '\r\n0\r\n\r\n')
+ return respond_headers
+ end
+
+ local function add_respond_header(name, value)
+ if name and value then
+ table.insert(respond_headers, string.format('%s: %s\r\n', name, value))
+ end
+ end
+
+ local function result_header_table(result)
+ local icap_headers = {}
+ for s in result:gmatch("[^\r\n]+") do
+ if string.find(s, '^ICAP') then
+ icap_headers['icap'] = tostring(s)
+ elseif string.find(s, '^HTTP') then
+ icap_headers['http'] = tostring(s)
+ elseif string.find(s, '[%a%d-+]-:') then
+ local _, _, key, value = tostring(s):find("([%a%d-+]-):%s?(.+)")
+ if key ~= nil then
+ icap_headers[key:lower()] = tostring(value)
+ end
+ end
+ end
+ lua_util.debugm(rule.name, task, '%s: icap_headers: %s',
+ rule.log_prefix, icap_headers)
+ return icap_headers
+ end
+
+ local function threat_table_add(icap_threat, maybe_split)
+
+ if maybe_split and string.find(icap_threat, ',') then
+ local threats = lua_util.str_split(string.gsub(icap_threat, "%s", ""), ',') or {}
+
+ for _, v in ipairs(threats) do
+ table.insert(threat_table, v)
+ end
+ else
+ table.insert(threat_table, icap_threat)
+ end
+ return true
+ end
+
+ local function icap_parse_result(headers)
+
+ --[[
+ @ToDo: handle type in response
+
+ Generic Strings:
+ icap: X-Infection-Found: Type=0; Resolution=2; Threat=Troj/DocDl-OYC;
+ icap: X-Infection-Found: Type=0; Resolution=2; Threat=W97M.Downloader;
+
+ Symantec String:
+ icap: X-Infection-Found: Type=2; Resolution=2; Threat=Container size violation
+ icap: X-Infection-Found: Type=2; Resolution=2; Threat=Encrypted container violation;
+
+ Sophos Strings:
+ icap: X-Virus-ID: Troj/DocDl-OYC
+ http: X-Blocked: Virus found during virus scan
+ http: X-Blocked-By: Sophos Anti-Virus
+
+ Kaspersky Web Traffic Security Strings:
+ icap: X-Virus-ID: HEUR:Backdoor.Java.QRat.gen
+ icap: X-Response-Info: blocked
+ icap: X-Virus-ID: no threats
+ icap: X-Response-Info: blocked
+ icap: X-Response-Info: passed
+ http: HTTP/1.1 403 Forbidden
+
+ Kaspersky Scan Engine 2.0 (ICAP mode)
+ icap: X-Virus-ID: EICAR-Test-File
+ http: HTTP/1.0 403 Forbidden
+
+ Trend Micro Strings:
+ icap: X-Virus-ID: Trojan.W97M.POWLOAD.SMTHF1
+ icap: X-Infection-Found: Type=0; Resolution=2; Threat=Trojan.W97M.POWLOAD.SMTHF1;
+ http: HTTP/1.1 403 Forbidden (TMWS Blocked)
+ http: HTTP/1.1 403 Forbidden
+
+ F-Secure Internet Gatekeeper Strings:
+ icap: X-FSecure-Scan-Result: infected
+ icap: X-FSecure-Infection-Name: "Malware.W97M/Agent.32584203"
+ icap: X-FSecure-Infected-Filename: "virus.doc"
+
+ ESET File Security for Linux 7.0
+ icap: X-Infection-Found: Type=0; Resolution=0; Threat=VBA/TrojanDownloader.Agent.JOA;
+ icap: X-Virus-ID: Trojaner
+ icap: X-Response-Info: Blocked
+
+ McAfee Web Gateway 10/11 (Headers must be activated with personal extra Rules)
+ icap: X-Virus-ID: EICAR test file
+ icap: X-Media-Type: text/plain
+ icap: X-Block-Result: 80
+ icap: X-Block-Reason: Malware found
+ icap: X-Block-Reason: Archive not supported
+ icap: X-Block-Reason: Media Type (Block List)
+ http: HTTP/1.0 403 VirusFound
+
+ C-ICAP Squidclamav
+ icap/http: X-Infection-Found: Type=0; Resolution=2; Threat={HEX}EICAR.TEST.3.UNOFFICIAL;
+ icap/http: X-Virus-ID: {HEX}EICAR.TEST.3.UNOFFICIAL
+ http: HTTP/1.0 307 Temporary Redirect
+ ]] --
+
+ -- Generic ICAP Headers
+ if headers['x-infection-found'] then
+ local _, _, icap_type, _, icap_threat = headers['x-infection-found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$")
+
+ -- Type=2 is typical for scan error returns
+ if icap_type and icap_type == '2' then
+ lua_util.debugm(rule.name, task,
+ '%s: icap error X-Infection-Found: %s', rule.log_prefix, icap_threat)
+ common.yield_result(task, rule, icap_threat, 0,
+ 'fail', maybe_part)
+ return true
+ elseif icap_threat ~= nil then
+ lua_util.debugm(rule.name, task,
+ '%s: icap X-Infection-Found: %s', rule.log_prefix, icap_threat)
+ threat_table_add(icap_threat, false)
+ -- stupid workaround for unuseable x-infection-found header
+ -- but also x-virus-name set (McAfee Web Gateway 9)
+ elseif not icap_threat and headers['x-virus-name'] then
+ threat_table_add(headers['x-virus-name'], true)
+ else
+ threat_table_add(headers['x-infection-found'], true)
+ end
+ elseif headers['x-virus-name'] and headers['x-virus-name'] ~= "no threats" then
+ lua_util.debugm(rule.name, task,
+ '%s: icap X-Virus-Name: %s', rule.log_prefix, headers['x-virus-name'])
+ threat_table_add(headers['x-virus-name'], true)
+ elseif headers['x-virus-id'] and headers['x-virus-id'] ~= "no threats" then
+ lua_util.debugm(rule.name, task,
+ '%s: icap X-Virus-ID: %s', rule.log_prefix, headers['x-virus-id'])
+ threat_table_add(headers['x-virus-id'], true)
+ -- FSecure X-Headers
+ elseif headers['x-fsecure-scan-result'] and headers['x-fsecure-scan-result'] ~= "clean" then
+
+ local infected_filename = ""
+ local infection_name = "-unknown-"
+
+ if headers['x-fsecure-infected-filename'] then
+ infected_filename = string.gsub(headers['x-fsecure-infected-filename'], '[%s"]', '')
+ end
+ if headers['x-fsecure-infection-name'] then
+ infection_name = string.gsub(headers['x-fsecure-infection-name'], '[%s"]', '')
+ end
+
+ lua_util.debugm(rule.name, task,
+ '%s: icap X-FSecure-Infection-Name (X-FSecure-Infected-Filename): %s (%s)',
+ rule.log_prefix, infection_name, infected_filename)
+
+ threat_table_add(infection_name, true)
+ -- McAfee Web Gateway manual extra headers
+ elseif headers['x-mwg-block-reason'] and headers['x-mwg-block-reason'] ~= "" then
+ threat_table_add(headers['x-mwg-block-reason'], false)
+ -- Sophos SAVDI special http headers
+ elseif headers['x-blocked'] and headers['x-blocked'] ~= "" then
+ threat_table_add(headers['x-blocked'], false)
+ elseif headers['x-block-reason'] and headers['x-block-reason'] ~= "" then
+ threat_table_add(headers['x-block-reason'], false)
+ -- last try HTTP [4]xx return
+ elseif headers.http and string.find(headers.http, '^HTTP%/[12]%.. [4]%d%d') then
+ threat_table_add(
+ string.format("pseudo-virus (blocked): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false)
+ elseif rule.use_http_3xx_as_threat and
+ headers.http and
+ string.find(headers.http, '^HTTP%/[12]%.. [3]%d%d')
+ then
+ threat_table_add(
+ string.format("pseudo-virus (redirect): %s",
+ string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false)
+ end
+
+ if #threat_table > 0 then
+ common.yield_result(task, rule, threat_table, rule.default_score, nil, maybe_part)
+ common.save_cache(task, digest, rule, threat_table, rule.default_score, maybe_part)
+ return true
+ else
+ return false
+ end
+ end
+
+ local function icap_r_respond_http_cb(err_m, data, connection)
+ if err_m or connection == nil then
+ icap_requery(err_m, "icap_r_respond_http_cb")
+ else
+ local result = tostring(data)
+
+ local icap_http_headers = result_header_table(result) or {}
+ -- Find HTTP/[12].x [234]xx response
+ if icap_http_headers.http and string.find(icap_http_headers.http, 'HTTP%/[12]%.. [234]%d%d') then
+ local icap_http_header_result = icap_parse_result(icap_http_headers)
+ if icap_http_header_result then
+ -- Threat found - close connection
+ connection:close()
+ else
+ common.save_cache(task, digest, rule, 'OK', 0, maybe_part)
+ common.log_clean(task, rule)
+ end
+ else
+ rspamd_logger.errx(task, '%s: unhandled response |%s|',
+ rule.log_prefix, string.gsub(result, "\r\n", ", "))
+ common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_http_headers.icap),
+ 0.0, 'fail', maybe_part)
+ end
+ end
+ end
+
+ local function icap_r_respond_cb(err_m, data, connection)
+ if err_m or connection == nil then
+ icap_requery(err_m, "icap_r_respond_cb")
+ else
+ local result = tostring(data)
+
+ local icap_headers = result_header_table(result) or {}
+ -- Find ICAP/1.x 2xx response
+ if icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then
+ local icap_header_result = icap_parse_result(icap_headers)
+ if icap_header_result then
+ -- Threat found - close connection
+ connection:close()
+ elseif not icap_header_result
+ and rule.use_http_result_header
+ and icap_headers.encapsulated
+ and not string.find(icap_headers.encapsulated, 'null%-body=0')
+ then
+ -- Try to read encapsulated HTTP Headers
+ lua_util.debugm(rule.name, task, '%s: no ICAP virus header found - try HTTP headers',
+ rule.log_prefix)
+ connection:add_read(icap_r_respond_http_cb, '\r\n\r\n')
+ else
+ connection:close()
+ common.save_cache(task, digest, rule, 'OK', 0, maybe_part)
+ common.log_clean(task, rule)
+ end
+ elseif icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. [45]%d%d') then
+ -- Find ICAP/1.x 5/4xx response
+ --[[
+ Symantec String:
+ ICAP/1.0 539 Aborted - No AV scanning license
+ SquidClamAV/C-ICAP:
+ ICAP/1.0 500 Server error
+ Eset:
+ ICAP/1.0 405 Forbidden
+ TrendMicro:
+ ICAP/1.0 400 Bad request
+ McAfee:
+ ICAP/1.0 418 Bad composition
+ ]]--
+ rspamd_logger.errx(task, '%s: ICAP ERROR: %s', rule.log_prefix, icap_headers.icap)
+ common.yield_result(task, rule, icap_headers.icap, 0.0,
+ 'fail', maybe_part)
+ return false
+ else
+ rspamd_logger.errx(task, '%s: unhandled response |%s|',
+ rule.log_prefix, string.gsub(result, "\r\n", ", "))
+ common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_headers.icap),
+ 0.0, 'fail', maybe_part)
+ end
+ end
+ end
+
+ local function icap_w_respond_cb(err_m, connection)
+ if err_m or connection == nil then
+ icap_requery(err_m, "icap_w_respond_cb")
+ else
+ connection:add_read(icap_r_respond_cb, '\r\n\r\n')
+ end
+ end
+
+ local function icap_r_options_cb(err_m, data, connection)
+ if err_m or connection == nil then
+ icap_requery(err_m, "icap_r_options_cb")
+ else
+ local icap_headers = result_header_table(tostring(data))
+
+ if icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then
+ if icap_headers['methods'] and string.find(icap_headers['methods'], 'RESPMOD') then
+ -- Allow "204 No Content" responses
+ -- https://datatracker.ietf.org/doc/html/rfc3507#section-4.6
+ if icap_headers['allow'] and string.find(icap_headers['allow'], '204') then
+ add_respond_header('Allow', '204')
+ end
+
+ if rule.x_client_header then
+ local client = task:get_from_ip()
+ if client then
+ add_respond_header('X-Client-IP', client:to_string())
+ end
+ end
+
+ -- F-Secure extra headers
+ if icap_headers['server'] and string.find(icap_headers['server'], 'f-secure icap server') then
+
+ if rule.x_rcpt_header then
+ local rcpt_to = task:get_principal_recipient()
+ if rcpt_to then
+ add_respond_header('X-Rcpt-To', rcpt_to)
+ end
+ end
+
+ if rule.x_from_header then
+ local mail_from = task:get_principal_recipient()
+ if mail_from and mail_from[1] then
+ add_respond_header('X-Rcpt-To', mail_from[1].addr)
+ end
+ end
+
+ end
+
+ if icap_headers.connection and icap_headers.connection:lower() == 'close' then
+ lua_util.debugm(rule.name, task, '%s: OPTIONS request Connection: %s - using new connection',
+ rule.log_prefix, icap_headers.connection)
+ connection:close()
+ tcp_options.callback = icap_w_respond_cb
+ tcp_options.data = get_respond_query()
+ tcp.request(tcp_options)
+ else
+ connection:add_write(icap_w_respond_cb, get_respond_query())
+ end
+
+ else
+ rspamd_logger.errx(task, '%s: RESPMOD method not advertised: Methods: %s',
+ rule.log_prefix, icap_headers['methods'])
+ common.yield_result(task, rule, 'NO RESPMOD', 0.0,
+ 'fail', maybe_part)
+ end
+ else
+ rspamd_logger.errx(task, '%s: OPTIONS query failed: %s',
+ rule.log_prefix, icap_headers.icap or "-")
+ common.yield_result(task, rule, 'OPTIONS query failed', 0.0,
+ 'fail', maybe_part)
+ end
+ end
+ end
+
+ if err or conn == nil then
+ icap_requery(err, "options_request")
+ else
+ conn:add_read(icap_r_options_cb, '\r\n\r\n')
+ end
+ end
+
+ tcp_options.task = task
+ tcp_options.stop_pattern = '\r\n'
+ tcp_options.read = false
+ tcp_options.timeout = rule.timeout
+ tcp_options.callback = icap_callback
+ tcp_options.data = options_request
+
+ if rule.ssl then
+ tcp_options.ssl = true
+ if rule.no_ssl_verify then
+ tcp_options.no_ssl_verify = true
+ end
+ end
+
+ tcp_options.host = addr:to_string()
+ tcp_options.port = addr:get_port()
+ tcp_options.upstream = upstream
+
+ tcp.request(tcp_options)
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ icap_check_uncached, maybe_part) then
+ return
+ else
+ icap_check_uncached()
+ end
+
+end
+
+return {
+ type = { N, 'virus', 'virus', 'scanner' },
+ description = 'generic icap antivirus',
+ configure = icap_config,
+ check = icap_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/init.lua b/lualib/lua_scanners/init.lua
new file mode 100644
index 0000000..e47cebe
--- /dev/null
+++ b/lualib/lua_scanners/init.lua
@@ -0,0 +1,75 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_scanners
+-- This module contains external scanners functions
+--]]
+
+local fun = require "fun"
+
+local exports = {
+}
+
+local function require_scanner(name)
+ local sc = require("lua_scanners/" .. name)
+
+ exports[sc.name or name] = sc
+end
+
+-- Antiviruses
+require_scanner('clamav')
+require_scanner('fprot')
+require_scanner('kaspersky_av')
+require_scanner('kaspersky_se')
+require_scanner('savapi')
+require_scanner('sophos')
+require_scanner('virustotal')
+require_scanner('avast')
+
+-- Other scanners
+require_scanner('dcc')
+require_scanner('oletools')
+require_scanner('icap')
+require_scanner('vadesecure')
+require_scanner('spamassassin')
+require_scanner('p0f')
+require_scanner('razor')
+require_scanner('pyzor')
+require_scanner('cloudmark')
+
+exports.add_scanner = function(name, t, conf_func, check_func)
+ assert(type(conf_func) == 'function' and type(check_func) == 'function',
+ 'bad arguments')
+ exports[name] = {
+ type = t,
+ configure = conf_func,
+ check = check_func,
+ }
+end
+
+exports.filter = function(t)
+ return fun.tomap(fun.filter(function(_, elt)
+ return type(elt) == 'table' and elt.type and (
+ (type(elt.type) == 'string' and elt.type == t) or
+ (type(elt.type) == 'table' and fun.any(function(tt)
+ return tt == t
+ end, elt.type))
+ )
+ end, exports))
+end
+
+return exports
diff --git a/lualib/lua_scanners/kaspersky_av.lua b/lualib/lua_scanners/kaspersky_av.lua
new file mode 100644
index 0000000..d52cef0
--- /dev/null
+++ b/lualib/lua_scanners/kaspersky_av.lua
@@ -0,0 +1,197 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module kaspersky
+-- This module contains kaspersky antivirus access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "kaspersky"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function kaspersky_config(opts)
+ local kaspersky_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ product_id = 0,
+ log_clean = false,
+ timeout = 5.0,
+ retransmits = 1, -- use local files, retransmits are useless
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ detection_category = "virus",
+ tmpdir = '/tmp',
+ }
+
+ kaspersky_conf = lua_util.override_defaults(kaspersky_conf, opts)
+
+ if not kaspersky_conf.prefix then
+ kaspersky_conf.prefix = 'rs_' .. kaspersky_conf.name .. '_'
+ end
+
+ if not kaspersky_conf.log_prefix then
+ if kaspersky_conf.name:lower() == kaspersky_conf.type:lower() then
+ kaspersky_conf.log_prefix = kaspersky_conf.name
+ else
+ kaspersky_conf.log_prefix = kaspersky_conf.name .. ' (' .. kaspersky_conf.type .. ')'
+ end
+ end
+
+ if not kaspersky_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ kaspersky_conf['upstreams'] = upstream_list.create(rspamd_config,
+ kaspersky_conf['servers'], 0)
+
+ if kaspersky_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', kaspersky_conf.name)
+ return kaspersky_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ kaspersky_conf['servers'])
+ return nil
+end
+
+local function kaspersky_check(task, content, digest, rule, maybe_part)
+ local function kaspersky_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local fname = string.format('%s/%s.tmp',
+ rule.tmpdir, rspamd_util.random_hex(32))
+ local message_fd = rspamd_util.create_file(fname)
+ local clamav_compat_cmd = string.format("nSCAN %s\n", fname)
+
+ if not message_fd then
+ rspamd_logger.errx('cannot store file for kaspersky scan: %s', fname)
+ return
+ end
+
+ if type(content) == 'string' then
+ -- Create rspamd_text
+ local rspamd_text = require "rspamd_text"
+ content = rspamd_text.fromstring(content)
+ end
+ content:save_in_file(message_fd)
+
+ -- Ensure file cleanup
+ task:get_mempool():add_destructor(function()
+ os.remove(fname)
+ rspamd_util.close_file(message_fd)
+ end)
+
+ local function kaspersky_callback(err, data)
+ if err then
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = kaspersky_callback,
+ data = { clamav_compat_cmd },
+ stop_pattern = '\n'
+ })
+ else
+ rspamd_logger.errx(task,
+ '%s [%s]: failed to scan, maximum retransmits exceed',
+ rule['symbol'], rule['type'])
+ common.yield_result(task, rule,
+ 'failed to scan and retransmits exceed', 0.0, 'fail',
+ maybe_part)
+ end
+
+ else
+ data = tostring(data)
+ local cached
+ lua_util.debugm(rule.name, task,
+ '%s [%s]: got reply: %s',
+ rule['symbol'], rule['type'], data)
+ if data == 'stream: OK' or data == fname .. ': OK' then
+ cached = 'OK'
+ common.log_clean(task, rule)
+ else
+ local vname = string.match(data, ': (.+) FOUND')
+ if vname then
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ cached = vname
+ else
+ rspamd_logger.errx(task, 'unhandled response: %s', data)
+ common.yield_result(task, rule, 'unhandled response',
+ 0.0, 'fail', maybe_part)
+ end
+ end
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = kaspersky_callback,
+ data = { clamav_compat_cmd },
+ stop_pattern = '\n'
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ kaspersky_check_uncached, maybe_part) then
+ return
+ else
+ kaspersky_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'kaspersky antivirus',
+ configure = kaspersky_config,
+ check = kaspersky_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/kaspersky_se.lua b/lualib/lua_scanners/kaspersky_se.lua
new file mode 100644
index 0000000..5e0f2ea
--- /dev/null
+++ b/lualib/lua_scanners/kaspersky_se.lua
@@ -0,0 +1,287 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module kaspersky_se
+-- This module contains Kaspersky Scan Engine integration support
+-- https://www.kaspersky.com/scan-engine
+--]]
+
+local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local http = require "rspamd_http"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'kaspersky_se'
+
+local function kaspersky_se_config(opts)
+
+ local default_conf = {
+ name = N,
+ default_port = 9999,
+ use_https = false,
+ use_files = false,
+ timeout = 5.0,
+ log_clean = false,
+ tmpdir = '/tmp',
+ retransmits = 1,
+ cache_expire = 7200, -- expire redis in 2h
+ message = '${SCANNER}: spam message found: "${VIRUS}"',
+ detection_category = "virus",
+ default_score = 1,
+ action = false,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ }
+
+ default_conf = lua_util.override_defaults(default_conf, opts)
+
+ if not default_conf.prefix then
+ default_conf.prefix = 'rs_' .. default_conf.name .. '_'
+ end
+
+ if not default_conf.log_prefix then
+ if default_conf.name:lower() == default_conf.type:lower() then
+ default_conf.log_prefix = default_conf.name
+ else
+ default_conf.log_prefix = default_conf.name .. ' (' .. default_conf.type .. ')'
+ end
+ end
+
+ if not default_conf.servers and default_conf.socket then
+ default_conf.servers = default_conf.socket
+ end
+
+ if not default_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ default_conf.upstreams = upstream_list.create(rspamd_config,
+ default_conf.servers,
+ default_conf.default_port)
+
+ if default_conf.upstreams then
+ lua_util.add_debug_alias('external_services', default_conf.name)
+ return default_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ default_conf['servers'])
+ return nil
+end
+
+local function kaspersky_se_check(task, content, digest, rule, maybe_part)
+ local function kaspersky_se_check_uncached()
+ local function make_url(addr)
+ local url
+ local suffix = '/scanmemory'
+
+ if rule.use_files then
+ suffix = '/scanfile'
+ end
+ if rule.use_https then
+ url = string.format('https://%s:%d%s', tostring(addr),
+ addr:get_port(), suffix)
+ else
+ url = string.format('http://%s:%d%s', tostring(addr),
+ addr:get_port(), suffix)
+ end
+
+ return url
+ end
+
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ local url = make_url(addr)
+ local hdrs = {
+ ['X-KAV-ProtocolVersion'] = '1',
+ ['X-KAV-Timeout'] = tostring(rule.timeout * 1000),
+ }
+
+ if task:has_from() then
+ hdrs['X-KAV-ObjectURL'] = string.format('[from:%s]', task:get_from()[1].addr)
+ end
+
+ local req_body
+
+ if rule.use_files then
+ local fname = string.format('%s/%s.tmp',
+ rule.tmpdir, rspamd_util.random_hex(32))
+ local message_fd = rspamd_util.create_file(fname)
+
+ if not message_fd then
+ rspamd_logger.errx('cannot store file for kaspersky_se scan: %s', fname)
+ return
+ end
+
+ if type(content) == 'string' then
+ -- Create rspamd_text
+ local rspamd_text = require "rspamd_text"
+ content = rspamd_text.fromstring(content)
+ end
+ content:save_in_file(message_fd)
+
+ -- Ensure cleanup
+ task:get_mempool():add_destructor(function()
+ os.remove(fname)
+ rspamd_util.close_file(message_fd)
+ end)
+
+ req_body = fname
+ else
+ req_body = content
+ end
+
+ local request_data = {
+ task = task,
+ url = url,
+ body = req_body,
+ headers = hdrs,
+ timeout = rule.timeout,
+ }
+
+ local function kas_callback(http_err, code, body, headers)
+
+ local function requery()
+ -- set current upstream to fail because an error occurred
+ upstream:fail()
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.name, task,
+ '%s: Request Error: %s - retries left: %s',
+ rule.log_prefix, http_err, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+ url = make_url(addr)
+
+ lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+ request_data.url = url
+ request_data.upstream = upstream
+
+ http.request(request_data)
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
+ task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' ..
+ 'retransmits exceed')
+ end
+ end
+
+ if http_err then
+ requery()
+ else
+ -- Parse the response
+ if upstream then
+ upstream:ok()
+ end
+ if code ~= 200 then
+ rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
+ return
+ end
+ local data = string.gsub(tostring(body), '[\r\n%s]$', '')
+ local cached
+ lua_util.debugm(rule.name, task, '%s: got reply data: "%s"',
+ rule.log_prefix, data)
+
+ if data:find('^CLEAN') then
+ -- Handle CLEAN replies
+ if data == 'CLEAN' then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: message or mime_part is clean',
+ rule.log_prefix)
+ else
+ lua_util.debugm(rule.name, task, '%s: message or mime_part is clean',
+ rule.log_prefix)
+ end
+ elseif data == 'CLEAN AND CONTAINS OFFICE MACRO' then
+ common.yield_result(task, rule, 'File contains macros',
+ 0.0, 'macro', maybe_part)
+ cached = 'MACRO'
+ else
+ rspamd_logger.errx(task, '%s: unhandled clean response: %s', rule.log_prefix, data)
+ common.yield_result(task, rule, 'unhandled response:' .. data,
+ 0.0, 'fail', maybe_part)
+ end
+ elseif data == 'SERVER_ERROR' then
+ rspamd_logger.errx(task, '%s: error: %s', rule.log_prefix, data)
+ common.yield_result(task, rule, 'error:' .. data,
+ 0.0, 'fail', maybe_part)
+ elseif string.match(data, 'DETECT (.+)') then
+ local vname = string.match(data, 'DETECT (.+)')
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ cached = vname
+ elseif string.match(data, 'NON_SCANNED %((.+)%)') then
+ local why = string.match(data, 'NON_SCANNED %((.+)%)')
+
+ if why == 'PASSWORD PROTECTED' then
+ rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix)
+ common.yield_result(task, rule, 'File is encrypted: ' .. why,
+ 0.0, 'encrypted', maybe_part)
+ cached = 'ENCRYPTED'
+ else
+ common.yield_result(task, rule, 'unhandled response:' .. data,
+ 0.0, 'fail', maybe_part)
+ end
+ else
+ rspamd_logger.errx(task, '%s: unhandled response: %s', rule.log_prefix, data)
+ common.yield_result(task, rule, 'unhandled response:' .. data,
+ 0.0, 'fail', maybe_part)
+ end
+
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ end
+
+ end
+ end
+
+ request_data.callback = kas_callback
+ http.request(request_data)
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ kaspersky_se_check_uncached, maybe_part) then
+ return
+ else
+
+ kaspersky_se_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'Kaspersky Scan Engine interface',
+ configure = kaspersky_se_config,
+ check = kaspersky_se_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/oletools.lua b/lualib/lua_scanners/oletools.lua
new file mode 100644
index 0000000..378e094
--- /dev/null
+++ b/lualib/lua_scanners/oletools.lua
@@ -0,0 +1,369 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module oletools
+-- This module contains oletools access functions.
+-- Olefy is needed: https://github.com/HeinleinSupport/olefy
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+local common = require "lua_scanners/common"
+
+local N = 'oletools'
+
+local function oletools_config(opts)
+
+ local oletools_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 10050,
+ timeout = 15.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 86400, -- expire redis in 1d
+ min_size = 500,
+ symbol = "OLETOOLS",
+ message = '${SCANNER}: Oletools threat message found: "${VIRUS}"',
+ detection_category = "office macro",
+ default_score = 1,
+ action = false,
+ extended = false,
+ symbol_type = 'postfilter',
+ dynamic_scan = true,
+ }
+
+ oletools_conf = lua_util.override_defaults(oletools_conf, opts)
+
+ if not oletools_conf.prefix then
+ oletools_conf.prefix = 'rs_' .. oletools_conf.name .. '_'
+ end
+
+ if not oletools_conf.log_prefix then
+ if oletools_conf.name:lower() == oletools_conf.type:lower() then
+ oletools_conf.log_prefix = oletools_conf.name
+ else
+ oletools_conf.log_prefix = oletools_conf.name .. ' (' .. oletools_conf.type .. ')'
+ end
+ end
+
+ if not oletools_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ oletools_conf.upstreams = upstream_list.create(rspamd_config,
+ oletools_conf.servers,
+ oletools_conf.default_port)
+
+ if oletools_conf.upstreams then
+ lua_util.add_debug_alias('external_services', oletools_conf.name)
+ return oletools_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ oletools_conf.servers)
+ return nil
+end
+
+local function oletools_check(task, content, digest, rule, maybe_part)
+ local function oletools_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local protocol = 'OLEFY/1.0\nMethod: oletools\nRspamd-ID: ' .. task:get_uid() .. '\n\n'
+ local json_response = ""
+
+ local function oletools_callback(err, data, conn)
+
+ local function oletools_requery(error)
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout,
+ shutdown = true,
+ data = { protocol, content },
+ callback = oletools_callback,
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed - err: %s', rule.log_prefix, error)
+ common.yield_result(task, rule,
+ 'failed to scan, maximum retransmits exceed - err: ' .. error,
+ 0.0, 'fail', maybe_part)
+ end
+ end
+
+ if err then
+
+ oletools_requery(err)
+
+ else
+ json_response = json_response .. tostring(data)
+
+ if not string.find(json_response, '\t\n\n\t') and #data == 8192 then
+ lua_util.debugm(rule.name, task, '%s: no stop word: add_read - #json: %s / current packet: %s',
+ rule.log_prefix, #json_response, #data)
+ conn:add_read(oletools_callback)
+
+ else
+ local ucl_parser = ucl.parser()
+ local ok, ucl_err = ucl_parser:parse_string(tostring(json_response))
+ if not ok then
+ rspamd_logger.errx(task, "%s: error parsing json response, retry: %s",
+ rule.log_prefix, ucl_err)
+ oletools_requery(ucl_err)
+ return
+ end
+
+ local result = ucl_parser:get_object()
+
+ local oletools_rc = {
+ [0] = 'RETURN_OK',
+ [1] = 'RETURN_WARNINGS',
+ [2] = 'RETURN_WRONG_ARGS',
+ [3] = 'RETURN_FILE_NOT_FOUND',
+ [4] = 'RETURN_XGLOB_ERR',
+ [5] = 'RETURN_OPEN_ERROR',
+ [6] = 'RETURN_PARSE_ERROR',
+ [7] = 'RETURN_SEVERAL_ERRS',
+ [8] = 'RETURN_UNEXPECTED',
+ [9] = 'RETURN_ENCRYPTED',
+ }
+
+ -- M=Macros, A=Auto-executable, S=Suspicious keywords, I=IOCs,
+ -- H=Hex strings, B=Base64 strings, D=Dridex strings, V=VBA strings
+ -- Keep sorted to avoid dragons
+ local analysis_cat_table = {
+ autoexec = '-',
+ base64 = '-',
+ dridex = '-',
+ hex = '-',
+ iocs = '-',
+ macro_exist = '-',
+ suspicious = '-',
+ vba = '-'
+ }
+ local analysis_keyword_table = {}
+
+ for _, v in ipairs(result) do
+
+ if v.error ~= nil and v.type ~= 'error' then
+ -- olefy, not oletools error
+ rspamd_logger.errx(task, '%s: ERROR found: %s', rule.log_prefix,
+ v.error)
+ if v.error == 'File too small' then
+ common.save_cache(task, digest, rule, 'OK', 1.0, maybe_part)
+ common.log_clean(task, rule, 'File too small to be scanned for macros')
+ return
+ else
+ oletools_requery(v.error)
+ end
+
+ elseif tostring(v.type) == "MetaInformation" and v.version ~= nil then
+ -- if MetaInformation section - check and print script and version
+
+ lua_util.debugm(N, task, '%s: version: %s %s', rule.log_prefix,
+ tostring(v.script_name), tostring(v.version))
+
+ elseif tostring(v.type) == "MetaInformation" and v.return_code ~= nil then
+ -- if MetaInformation section - check return_code
+
+ local oletools_rc_code = tonumber(v.return_code)
+ if oletools_rc_code == 9 then
+ rspamd_logger.warnx(task, '%s: File is encrypted.', rule.log_prefix)
+ common.yield_result(task, rule,
+ 'failed - err: ' .. oletools_rc[oletools_rc_code],
+ 0.0, 'encrypted', maybe_part)
+ common.save_cache(task, digest, rule, 'encrypted', 1.0, maybe_part)
+ return
+ elseif oletools_rc_code == 5 then
+ rspamd_logger.warnx(task, '%s: olefy could not open the file - error: %s', rule.log_prefix,
+ result[2]['message'])
+ common.yield_result(task, rule,
+ 'failed - err: ' .. oletools_rc[oletools_rc_code],
+ 0.0, 'fail', maybe_part)
+ return
+ elseif oletools_rc_code > 6 then
+ rspamd_logger.errx(task, '%s: MetaInfo section error code: %s',
+ rule.log_prefix, oletools_rc[oletools_rc_code])
+ rspamd_logger.errx(task, '%s: MetaInfo section message: %s',
+ rule.log_prefix, result[2]['message'])
+ common.yield_result(task, rule,
+ 'failed - err: ' .. oletools_rc[oletools_rc_code],
+ 0.0, 'fail', maybe_part)
+ return
+ elseif oletools_rc_code > 1 then
+ rspamd_logger.errx(task, '%s: Error message: %s',
+ rule.log_prefix, result[2]['message'])
+ oletools_requery(oletools_rc[oletools_rc_code])
+ end
+
+ elseif tostring(v.type) == "error" then
+ -- error section found - check message
+ rspamd_logger.errx(task, '%s: Error section error code: %s',
+ rule.log_prefix, v.error)
+ rspamd_logger.errx(task, '%s: Error section message: %s',
+ rule.log_prefix, v.message)
+ --common.yield_result(task, rule, 'failed - err: ' .. v.error, 0.0, 'fail')
+
+ elseif type(v.analysis) == 'table' and type(v.macros) == 'table' then
+ -- analysis + macro found - evaluate response
+
+ if type(v.analysis) == 'table' and #v.analysis == 0 and #v.macros == 0 then
+ rspamd_logger.warnx(task, '%s: maybe unhandled python or oletools error', rule.log_prefix)
+ oletools_requery('oletools unhandled error')
+
+ elseif #v.macros > 0 then
+
+ analysis_cat_table.macro_exist = 'M'
+
+ lua_util.debugm(rule.name, task,
+ '%s: filename: %s', rule.log_prefix, result[2]['file'])
+ lua_util.debugm(rule.name, task,
+ '%s: type: %s', rule.log_prefix, result[2]['type'])
+
+ for _, m in ipairs(v.macros) do
+ lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, ' ..
+ 'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename)
+ end
+
+ for _, a in ipairs(v.analysis) do
+ lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, ' ..
+ 'description: %s', rule.log_prefix, a.type, a.keyword, a.description)
+ if a.type == 'AutoExec' then
+ analysis_cat_table.autoexec = 'A'
+ table.insert(analysis_keyword_table, a.keyword)
+ elseif a.type == 'Suspicious' then
+ if rule.extended == true or
+ (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings')
+ then
+ analysis_cat_table.suspicious = 'S'
+ table.insert(analysis_keyword_table, a.keyword)
+ end
+ elseif a.type == 'IOC' then
+ analysis_cat_table.iocs = 'I'
+ elseif a.type == 'Hex strings' then
+ analysis_cat_table.hex = 'H'
+ elseif a.type == 'Base64 strings' then
+ analysis_cat_table.base64 = 'B'
+ elseif a.type == 'Dridex strings' then
+ analysis_cat_table.dridex = 'D'
+ elseif a.type == 'VBA strings' then
+ analysis_cat_table.vba = 'V'
+ end
+ end
+ end
+ end
+ end
+
+ lua_util.debugm(N, task, '%s: analysis_keyword_table: %s', rule.log_prefix, analysis_keyword_table)
+ lua_util.debugm(N, task, '%s: analysis_cat_table: %s', rule.log_prefix, analysis_cat_table)
+
+ if rule.extended == false and analysis_cat_table.autoexec == 'A' and analysis_cat_table.suspicious == 'S' then
+ -- use single string as virus name
+ local threat = 'AutoExec + Suspicious (' .. table.concat(analysis_keyword_table, ',') .. ')'
+ lua_util.debugm(rule.name, task, '%s: threat result: %s', rule.log_prefix, threat)
+ common.yield_result(task, rule, threat, rule.default_score, nil, maybe_part)
+ common.save_cache(task, digest, rule, threat, rule.default_score, maybe_part)
+
+ elseif rule.extended == true and #analysis_keyword_table > 0 then
+ -- report any flags (types) and any most keywords as individual virus name
+ local analysis_cat_table_values_sorted = {}
+
+ -- see https://github.com/rspamd/rspamd/commit/6bd3e2b9f49d1de3ab882aeca9c30bc7d526ac9d#commitcomment-40130493
+ -- for details
+ local analysis_cat_table_keys_sorted = lua_util.keys(analysis_cat_table)
+ table.sort(analysis_cat_table_keys_sorted)
+
+ for _, v in ipairs(analysis_cat_table_keys_sorted) do
+ table.insert(analysis_cat_table_values_sorted, analysis_cat_table[v])
+ end
+
+ table.insert(analysis_keyword_table, 1, table.concat(analysis_cat_table_values_sorted))
+
+ lua_util.debugm(rule.name, task, '%s: extended threat result: %s',
+ rule.log_prefix, table.concat(analysis_keyword_table, ','))
+
+ common.yield_result(task, rule, analysis_keyword_table,
+ rule.default_score, nil, maybe_part)
+ common.save_cache(task, digest, rule, analysis_keyword_table,
+ rule.default_score, maybe_part)
+
+ elseif analysis_cat_table.macro_exist == '-' and #analysis_keyword_table == 0 then
+ common.save_cache(task, digest, rule, 'OK', 1.0, maybe_part)
+ common.log_clean(task, rule, 'No macro found')
+
+ else
+ common.save_cache(task, digest, rule, 'OK', 1.0, maybe_part)
+ common.log_clean(task, rule, 'Scanned Macro is OK')
+ end
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout,
+ shutdown = true,
+ data = { protocol, content },
+ callback = oletools_callback,
+ })
+
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ oletools_check_uncached, maybe_part) then
+ return
+ else
+ oletools_check_uncached()
+ end
+
+end
+
+return {
+ type = { N, 'attachment scanner', 'hash', 'scanner' },
+ description = 'oletools office macro scanner',
+ configure = oletools_config,
+ check = oletools_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/p0f.lua b/lualib/lua_scanners/p0f.lua
new file mode 100644
index 0000000..7785f83
--- /dev/null
+++ b/lualib/lua_scanners/p0f.lua
@@ -0,0 +1,227 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2019, Denis Paavilainen <denpa@denpa.pro>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module p0f
+-- This module contains p0f access functions
+--]]
+
+local tcp = require "rspamd_tcp"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local lua_redis = require "lua_redis"
+local lua_util = require "lua_util"
+local common = require "lua_scanners/common"
+
+-- SEE: https://github.com/p0f/p0f/blob/v3.06b/docs/README#L317
+local S = {
+ BAD_QUERY = 0x0,
+ OK = 0x10,
+ NO_MATCH = 0x20
+}
+
+local N = 'p0f'
+
+local function p0f_check(task, ip, rule)
+
+ local function ip2bin(addr)
+ addr = addr:to_table()
+
+ for k, v in ipairs(addr) do
+ addr[k] = rspamd_util.pack('B', v)
+ end
+
+ return table.concat(addr)
+ end
+
+ local function trim(...)
+ local vars = { ... }
+
+ for k, v in ipairs(vars) do
+ -- skip numbers, trim only strings
+ if tonumber(vars[k]) == nil then
+ vars[k] = string.gsub(v, '[^%w-_\\.\\(\\) ]', '')
+ end
+ end
+
+ return lua_util.unpack(vars)
+ end
+
+ local function parse_p0f_response(data)
+ --[[
+ p0f_api_response[232]: magic, status, first_seen, last_seen, total_conn,
+ uptime_min, up_mod_days, last_nat, last_chg, distance, bad_sw, os_match_q,
+ os_name, os_flavor, http_name, http_flavor, link_type, language
+ ]]--
+
+ data = tostring(data)
+
+ -- API response must be 232 bytes long
+ if #data ~= 232 then
+ rspamd_logger.errx(task, 'malformed response from p0f on %s, %s bytes',
+ rule.socket, #data)
+
+ common.yield_result(task, rule, 'Malformed Response: ' .. rule.socket,
+ 0.0, 'fail')
+ return
+ end
+
+ local _, status, _, _, _, uptime_min, _, _, _, distance, _, _, os_name,
+ os_flavor, _, _, link_type, _ = trim(rspamd_util.unpack(
+ 'I4I4I4I4I4I4I4I4I4hbbc32c32c32c32c32c32', data))
+
+ if status ~= S.OK then
+ if status == S.BAD_QUERY then
+ rspamd_logger.errx(task, 'malformed p0f query on %s', rule.socket)
+ common.yield_result(task, rule, 'Malformed Query: ' .. rule.socket,
+ 0.0, 'fail')
+ end
+
+ return
+ end
+
+ local os_string = #os_name == 0 and 'unknown' or os_name .. ' ' .. os_flavor
+
+ task:get_mempool():set_variable('os_fingerprint', os_string, link_type,
+ uptime_min, distance)
+
+ if link_type and #link_type > 0 then
+ common.yield_result(task, rule, {
+ os_string,
+ 'link=' .. link_type,
+ 'distance=' .. distance },
+ 0.0)
+ else
+ common.yield_result(task, rule, {
+ os_string,
+ 'link=unknown',
+ 'distance=' .. distance },
+ 0.0)
+ end
+
+ return data
+ end
+
+ local function make_p0f_request()
+
+ local function check_p0f_cb(err, data)
+
+ local function redis_set_cb(redis_set_err)
+ if redis_set_err then
+ rspamd_logger.errx(task, 'redis received an error: %s', redis_set_err)
+ end
+ end
+
+ if err then
+ rspamd_logger.errx(task, 'p0f received an error: %s', err)
+ common.yield_result(task, rule, 'Error getting result: ' .. err,
+ 0.0, 'fail')
+ return
+ end
+
+ data = parse_p0f_response(data)
+
+ if rule.redis_params and data then
+ local key = rule.prefix .. ip:to_string()
+ local ret = lua_redis.redis_make_request(task,
+ rule.redis_params,
+ key,
+ true,
+ redis_set_cb,
+ 'SETEX',
+ { key, tostring(rule.expire), data }
+ )
+
+ if not ret then
+ rspamd_logger.warnx(task, 'error connecting to redis')
+ end
+ end
+ end
+
+ local query = rspamd_util.pack('I4 I1 c16', 0x50304601,
+ ip:get_version(), ip2bin(ip))
+
+ tcp.request({
+ host = rule.socket,
+ callback = check_p0f_cb,
+ data = { query },
+ task = task,
+ timeout = rule.timeout
+ })
+ end
+
+ local function redis_get_cb(err, data)
+ if err or type(data) ~= 'string' then
+ make_p0f_request()
+ else
+ parse_p0f_response(data)
+ end
+ end
+
+ local ret = nil
+ if rule.redis_params then
+ local key = rule.prefix .. ip:to_string()
+ ret = lua_redis.redis_make_request(task,
+ rule.redis_params,
+ key,
+ false,
+ redis_get_cb,
+ 'GET',
+ { key }
+ )
+ end
+
+ if not ret then
+ make_p0f_request() -- fallback to directly querying p0f
+ end
+end
+
+local function p0f_config(opts)
+ local p0f_conf = {
+ name = N,
+ timeout = 5,
+ symbol = 'P0F',
+ symbol_fail = 'P0F_FAIL',
+ patterns = {},
+ expire = 7200,
+ prefix = 'p0f',
+ detection_category = 'fingerprint',
+ message = '${SCANNER}: fingerprint matched: "${VIRUS}"'
+ }
+
+ p0f_conf = lua_util.override_defaults(p0f_conf, opts)
+ p0f_conf.patterns = common.create_regex_table(p0f_conf.patterns)
+
+ if not p0f_conf.log_prefix then
+ p0f_conf.log_prefix = p0f_conf.name
+ end
+
+ if not p0f_conf.socket then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+ return nil
+ end
+
+ return p0f_conf
+end
+
+return {
+ type = { N, 'fingerprint', 'scanner' },
+ description = 'passive OS fingerprinter',
+ configure = p0f_config,
+ check = p0f_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/pyzor.lua b/lualib/lua_scanners/pyzor.lua
new file mode 100644
index 0000000..75c1b4a
--- /dev/null
+++ b/lualib/lua_scanners/pyzor.lua
@@ -0,0 +1,206 @@
+--[[
+Copyright (c) 2021, defkev <defkev@gmail.com>
+Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module pyzor
+-- This module contains pyzor access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'pyzor'
+local categories = { 'pyzor', 'bulk', 'hash', 'scanner' }
+
+local function pyzor_config(opts)
+
+ local pyzor_conf = {
+ text_part_min_words = 2,
+ default_port = 5953,
+ timeout = 15.0,
+ log_clean = false,
+ retransmits = 2,
+ detection_category = "hash",
+ cache_expire = 7200, -- expire redis in one hour
+ message = '${SCANNER}: Pyzor bulk message found: "${VIRUS}"',
+ default_score = 1.5,
+ action = false,
+ }
+
+ pyzor_conf = lua_util.override_defaults(pyzor_conf, opts)
+
+ if not pyzor_conf.prefix then
+ pyzor_conf.prefix = 'rext_' .. N .. '_'
+ end
+
+ if not pyzor_conf.log_prefix then
+ pyzor_conf.log_prefix = N .. ' (' .. pyzor_conf.detection_category .. ')'
+ end
+
+ if not pyzor_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ pyzor_conf['upstreams'] = upstream_list.create(rspamd_config,
+ pyzor_conf['servers'],
+ pyzor_conf.default_port)
+
+ if pyzor_conf['upstreams'] then
+ lua_util.add_debug_alias('external_services', N)
+ return pyzor_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ pyzor_conf['servers'])
+ return nil
+end
+
+local function pyzor_check(task, content, digest, rule)
+ local function pyzor_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ local function pyzor_callback(err, data, conn)
+
+ if err then
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(N, task, '%s: retry IP: %s:%s err: %s',
+ rule.log_prefix, addr, addr:get_port(), err)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ shutdown = true,
+ data = content,
+ callback = pyzor_callback,
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed',
+ rule['symbol'], rule['type'])
+ task:insert_result(rule['symbol_fail'], 0.0,
+ 'failed to scan and retransmits exceed')
+ end
+ else
+ -- pyzor output is unicode (\x09 -> tab, \0a -> newline)
+ -- public.pyzor.org:24441 (200, 'OK') 21285091 206759
+ -- server:port Code Diag Count WL-Count
+ local str_data = tostring(data)
+ lua_util.debugm(N, task, '%s: returned data: %s',
+ rule.log_prefix, str_data)
+ -- If pyzor would return JSON this wouldn't be necessary
+ local resp = {}
+ for v in string.gmatch(str_data, '[^\t]+') do
+ table.insert(resp, v)
+ end
+ -- rspamd_logger.infox(task, 'resp: %s', resp)
+ if resp[2] ~= [[(200, 'OK')]] then
+ rspamd_logger.errx(task, "error parsing response: %s", str_data)
+ return
+ end
+
+ local whitelisted = tonumber(resp[4])
+ local reported = tonumber(resp[3])
+
+ --rspamd_logger.infox(task, "%s - count=%s wl=%s", addr:to_string(), reported, whitelisted)
+
+ --[[
+ Weight is Count - WL-Count of rule.default_score in percent, e.g.
+ SPAM:
+ Count: 100 (100%)
+ WL-Count: 1 (1%)
+ rule.default_score: 1
+ Weight: 0.99
+ HAM:
+ Count: 10 (100%)
+ WL-Count: 10 (100%)
+ rule.default_score: 1
+ Weight: 0
+ ]]
+ local weight = tonumber(string.format("%.2f",
+ rule.default_score * (reported - whitelisted) / (reported + whitelisted)))
+ local info = string.format("count=%d wl=%d", reported, whitelisted)
+ local threat_string = string.format("bl_%d_wl_%d",
+ reported, whitelisted)
+
+ if weight > 0 then
+ lua_util.debugm(N, task, '%s: returned result is spam - info: %s',
+ rule.log_prefix, info)
+ common.yield_result(task, rule, threat_string, weight)
+ common.save_cache(task, digest, rule, threat_string, weight)
+ else
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: clean, returned result is ham - info: %s',
+ rule.log_prefix, info)
+ else
+ lua_util.debugm(N, task, '%s: returned result is ham - info: %s',
+ rule.log_prefix, info)
+ end
+ common.save_cache(task, digest, rule, 'OK', weight)
+ end
+
+ end
+ end
+
+ if digest == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' then
+ rspamd_logger.infox(task, '%s: not checking default digest', rule.log_prefix)
+ return
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout,
+ shutdown = true,
+ data = content,
+ callback = pyzor_callback,
+ })
+ end
+ if common.condition_check_and_continue(task, content, rule, digest, pyzor_check_uncached) then
+ return
+ else
+ pyzor_check_uncached()
+ end
+end
+
+return {
+ type = categories,
+ description = 'pyzor bulk scanner',
+ configure = pyzor_config,
+ check = pyzor_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/razor.lua b/lualib/lua_scanners/razor.lua
new file mode 100644
index 0000000..fcc0a8e
--- /dev/null
+++ b/lualib/lua_scanners/razor.lua
@@ -0,0 +1,181 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module razor
+-- This module contains razor access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'razor'
+
+local function razor_config(opts)
+
+ local razor_conf = {
+ name = N,
+ default_port = 11342,
+ timeout = 5.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 7200, -- expire redis in 2h
+ message = '${SCANNER}: spam message found: "${VIRUS}"',
+ detection_category = "hash",
+ default_score = 1,
+ action = false,
+ dynamic_scan = false,
+ symbol_fail = 'RAZOR_FAIL',
+ symbol = 'RAZOR',
+ }
+
+ razor_conf = lua_util.override_defaults(razor_conf, opts)
+
+ if not razor_conf.prefix then
+ razor_conf.prefix = 'rs_' .. razor_conf.name .. '_'
+ end
+
+ if not razor_conf.log_prefix then
+ razor_conf.log_prefix = razor_conf.name
+ end
+
+ if not razor_conf.servers and razor_conf.socket then
+ razor_conf.servers = razor_conf.socket
+ end
+
+ if not razor_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ razor_conf.upstreams = upstream_list.create(rspamd_config,
+ razor_conf.servers,
+ razor_conf.default_port)
+
+ if razor_conf.upstreams then
+ lua_util.add_debug_alias('external_services', razor_conf.name)
+ return razor_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ razor_conf['servers'])
+ return nil
+end
+
+local function razor_check(task, content, digest, rule)
+ local function razor_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ local function razor_callback(err, data, conn)
+
+ local function razor_requery()
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.name, task, '%s: Request Error: %s - retries left: %s',
+ rule.log_prefix, err, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout or 2.0,
+ shutdown = true,
+ data = content,
+ callback = razor_callback,
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
+ end
+ end
+
+ if err then
+
+ razor_requery()
+
+ else
+ --[[
+ @todo: Razorsocket currently only returns ham or spam. When the wrapper is fixed we should add dynamic scores here.
+ Maybe check spamassassin implementation.
+
+ This implementation is based on https://github.com/cgt/rspamd-plugins
+ Thanks @cgt!
+ ]] --
+
+ local threat_string = tostring(data)
+ if threat_string == "spam" then
+ lua_util.debugm(N, task, '%s: returned result is spam', rule['symbol'], rule['type'])
+ common.yield_result(task, rule, threat_string, rule.default_score)
+ common.save_cache(task, digest, rule, threat_string, rule.default_score)
+ elseif threat_string == "ham" then
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: returned result is ham', rule['symbol'], rule['type'])
+ else
+ lua_util.debugm(N, task, '%s: returned result is ham', rule['symbol'], rule['type'])
+ end
+ common.save_cache(task, digest, rule, 'OK', rule.default_score)
+ else
+ rspamd_logger.errx(task, "%s - unknown response from razorfy: %s", addr:to_string(), threat_string)
+ end
+
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule.timeout or 2.0,
+ shutdown = true,
+ data = content,
+ callback = razor_callback,
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest, razor_check_uncached) then
+ return
+ else
+ razor_check_uncached()
+ end
+end
+
+return {
+ type = { 'razor', 'spam', 'hash', 'scanner' },
+ description = 'razor bulk scanner',
+ configure = razor_config,
+ check = razor_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/savapi.lua b/lualib/lua_scanners/savapi.lua
new file mode 100644
index 0000000..08f7b66
--- /dev/null
+++ b/lualib/lua_scanners/savapi.lua
@@ -0,0 +1,261 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module savapi
+-- This module contains avira savapi antivirus access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "savapi"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function savapi_config(opts)
+ local savapi_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 4444, -- note: You must set ListenAddress in savapi.conf
+ product_id = 0,
+ log_clean = false,
+ timeout = 15.0, -- FIXME: this will break task_timeout!
+ retransmits = 1, -- FIXME: useless, for local files
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ detection_category = "virus",
+ tmpdir = '/tmp',
+ }
+
+ savapi_conf = lua_util.override_defaults(savapi_conf, opts)
+
+ if not savapi_conf.prefix then
+ savapi_conf.prefix = 'rs_' .. savapi_conf.name .. '_'
+ end
+
+ if not savapi_conf.log_prefix then
+ if savapi_conf.name:lower() == savapi_conf.type:lower() then
+ savapi_conf.log_prefix = savapi_conf.name
+ else
+ savapi_conf.log_prefix = savapi_conf.name .. ' (' .. savapi_conf.type .. ')'
+ end
+ end
+
+ if not savapi_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ savapi_conf['upstreams'] = upstream_list.create(rspamd_config,
+ savapi_conf['servers'],
+ savapi_conf.default_port)
+
+ if savapi_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', savapi_conf.name)
+ return savapi_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ savapi_conf['servers'])
+ return nil
+end
+
+local function savapi_check(task, content, digest, rule)
+ local function savapi_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local fname = string.format('%s/%s.tmp',
+ rule.tmpdir, rspamd_util.random_hex(32))
+ local message_fd = rspamd_util.create_file(fname)
+
+ if not message_fd then
+ rspamd_logger.errx('cannot store file for savapi scan: %s', fname)
+ return
+ end
+
+ if type(content) == 'string' then
+ -- Create rspamd_text
+ local rspamd_text = require "rspamd_text"
+ content = rspamd_text.fromstring(content)
+ end
+ content:save_in_file(message_fd)
+
+ -- Ensure cleanup
+ task:get_mempool():add_destructor(function()
+ os.remove(fname)
+ rspamd_util.close_file(message_fd)
+ end)
+
+ local vnames = {}
+
+ -- Forward declaration for recursive calls
+ local savapi_scan1_cb
+
+ local function savapi_fin_cb(err, conn)
+ local vnames_reordered = {}
+ -- Swap table
+ for virus, _ in pairs(vnames) do
+ table.insert(vnames_reordered, virus)
+ end
+ lua_util.debugm(rule.name, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered)
+ if #vnames_reordered > 0 then
+ local vname = {}
+ for _, virus in ipairs(vnames_reordered) do
+ table.insert(vname, virus)
+ end
+
+ common.yield_result(task, rule, vname)
+ common.save_cache(task, digest, rule, vname)
+ end
+ if conn then
+ conn:close()
+ end
+ end
+
+ local function savapi_scan2_cb(err, data, conn)
+ local result = tostring(data)
+ lua_util.debugm(rule.name, task, "%s: got reply: %s",
+ rule.type, result)
+
+ -- Terminal response - clean
+ if string.find(result, '200') or string.find(result, '210') then
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: message or mime_part is clean', rule['type'])
+ end
+ common.save_cache(task, digest, rule, 'OK')
+ conn:add_write(savapi_fin_cb, 'QUIT\n')
+
+ -- Terminal response - infected
+ elseif string.find(result, '319') then
+ conn:add_write(savapi_fin_cb, 'QUIT\n')
+
+ -- Non-terminal response
+ elseif string.find(result, '310') then
+ local virus
+ virus = result:match "310.*<<<%s(.*)%s+;.*;.*"
+ if not virus then
+ virus = result:match "310%s(.*)%s+;.*;.*"
+ if not virus then
+ rspamd_logger.errx(task, "%s: virus result unparseable: %s",
+ rule['type'], result)
+ common.yield_result(task, rule, 'virus result unparseable: ' .. result, 0.0, 'fail')
+ return
+ end
+ end
+ -- Store unique virus names
+ vnames[virus] = 1
+ -- More content is expected
+ conn:add_write(savapi_scan1_cb, '\n')
+ end
+ end
+
+ savapi_scan1_cb = function(err, conn)
+ conn:add_read(savapi_scan2_cb, '\n')
+ end
+
+ -- 100 PRODUCT:xyz
+ local function savapi_greet2_cb(err, data, conn)
+ local result = tostring(data)
+ if string.find(result, '100 PRODUCT') then
+ lua_util.debugm(rule.name, task, "%s: scanning file: %s",
+ rule['type'], fname)
+ conn:add_write(savapi_scan1_cb, { string.format('SCAN %s\n',
+ fname) })
+ else
+ rspamd_logger.errx(task, '%s: invalid product id %s', rule['type'],
+ rule['product_id'])
+ common.yield_result(task, rule, 'invalid product id: ' .. result, 0.0, 'fail')
+ conn:add_write(savapi_fin_cb, 'QUIT\n')
+ end
+ end
+
+ local function savapi_greet1_cb(err, conn)
+ conn:add_read(savapi_greet2_cb, '\n')
+ end
+
+ local function savapi_callback_init(err, data, conn)
+ if err then
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = savapi_callback_init,
+ stop_pattern = { '\n' },
+ })
+ else
+ rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
+ end
+ else
+ local result = tostring(data)
+
+ -- 100 SAVAPI:4.0 greeting
+ if string.find(result, '100') then
+ conn:add_write(savapi_greet1_cb, { string.format('SET PRODUCT %s\n', rule['product_id']) })
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = savapi_callback_init,
+ stop_pattern = { '\n' },
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest, savapi_check_uncached) then
+ return
+ else
+ savapi_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'savapi avira antivirus',
+ configure = savapi_config,
+ check = savapi_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/sophos.lua b/lualib/lua_scanners/sophos.lua
new file mode 100644
index 0000000..d9b64f1
--- /dev/null
+++ b/lualib/lua_scanners/sophos.lua
@@ -0,0 +1,192 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module savapi
+-- This module contains avira savapi antivirus access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "sophos"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function sophos_config(opts)
+ local sophos_conf = {
+ name = N,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 4010,
+ timeout = 15.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ detection_category = "virus",
+ }
+
+ sophos_conf = lua_util.override_defaults(sophos_conf, opts)
+
+ if not sophos_conf.prefix then
+ sophos_conf.prefix = 'rs_' .. sophos_conf.name .. '_'
+ end
+
+ if not sophos_conf.log_prefix then
+ if sophos_conf.name:lower() == sophos_conf.type:lower() then
+ sophos_conf.log_prefix = sophos_conf.name
+ else
+ sophos_conf.log_prefix = sophos_conf.name .. ' (' .. sophos_conf.type .. ')'
+ end
+ end
+
+ if not sophos_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ sophos_conf['upstreams'] = upstream_list.create(rspamd_config,
+ sophos_conf['servers'],
+ sophos_conf.default_port)
+
+ if sophos_conf['upstreams'] then
+ lua_util.add_debug_alias('antivirus', sophos_conf.name)
+ return sophos_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ sophos_conf['servers'])
+ return nil
+end
+
+local function sophos_check(task, content, digest, rule, maybe_part)
+ local function sophos_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local protocol = 'SSSP/1.0\n'
+ local streamsize = string.format('SCANDATA %d\n', #content)
+ local bye = 'BYE\n'
+
+ local function sophos_callback(err, data, conn)
+
+ if err then
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.name, task, '%s: error: %s; retry IP: %s; retries left: %s',
+ rule.log_prefix, err, addr, retransmits)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = sophos_callback,
+ data = { protocol, streamsize, content, bye }
+ })
+ else
+ rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed',
+ 0.0, 'fail', maybe_part)
+ end
+ else
+ data = tostring(data)
+ lua_util.debugm(rule.name, task,
+ '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data)
+ local vname = string.match(data, 'VIRUS (%S+) ')
+ local cached
+ if vname then
+ common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
+ common.save_cache(task, digest, rule, vname, 1.0, maybe_part)
+ else
+ if string.find(data, 'DONE OK') then
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: message or mime_part is clean', rule.log_prefix)
+ else
+ lua_util.debugm(rule.name, task,
+ '%s: message or mime_part is clean', rule.log_prefix)
+ end
+ cached = 'OK'
+ -- not finished - continue
+ elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then
+ conn:add_read(sophos_callback)
+ elseif string.find(data, 'FAIL 0212') then
+ rspamd_logger.warnx(task, 'Message is encrypted (FAIL 0212): %s', data)
+ common.yield_result(task, rule, 'SAVDI: Message is encrypted (FAIL 0212)',
+ 0.0, 'encrypted', maybe_part)
+ cached = 'ENCRYPTED'
+ elseif string.find(data, 'REJ 4') then
+ rspamd_logger.warnx(task, 'Message is oversized (REJ 4): %s', data)
+ common.yield_result(task, rule, 'SAVDI: Message oversized (REJ 4)',
+ 0.0, 'fail', maybe_part)
+ -- explicitly set REJ1 message when SAVDIreports a protocol error
+ elseif string.find(data, 'REJ 1') then
+ rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data)
+ common.yield_result(task, rule, 'SAVDI: Protocol error (REJ 1)',
+ 0.0, 'fail', maybe_part)
+ else
+ rspamd_logger.errx(task, 'unhandled response: %s', data)
+ common.yield_result(task, rule, 'unhandled response: ' .. data,
+ 0.0, 'fail', maybe_part)
+ end
+ if cached then
+ common.save_cache(task, digest, rule, cached, 1.0, maybe_part)
+ end
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ callback = sophos_callback,
+ data = { protocol, streamsize, content, bye }
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ sophos_check_uncached, maybe_part) then
+ return
+ else
+ sophos_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'sophos antivirus',
+ configure = sophos_config,
+ check = sophos_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/spamassassin.lua b/lualib/lua_scanners/spamassassin.lua
new file mode 100644
index 0000000..f425924
--- /dev/null
+++ b/lualib/lua_scanners/spamassassin.lua
@@ -0,0 +1,213 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module spamassassin
+-- This module contains spamd access functions.
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'spamassassin'
+
+local function spamassassin_config(opts)
+
+ local spamassassin_conf = {
+ N = N,
+ scan_mime_parts = false,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ default_port = 783,
+ timeout = 15.0,
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 3600, -- expire redis in one hour
+ symbol = "SPAMD",
+ message = '${SCANNER}: Spamassassin bulk message found: "${VIRUS}"',
+ detection_category = "spam",
+ default_score = 1,
+ action = false,
+ extended = false,
+ symbol_type = 'postfilter',
+ dynamic_scan = true,
+ }
+
+ spamassassin_conf = lua_util.override_defaults(spamassassin_conf, opts)
+
+ if not spamassassin_conf.prefix then
+ spamassassin_conf.prefix = 'rs_' .. spamassassin_conf.name .. '_'
+ end
+
+ if not spamassassin_conf.log_prefix then
+ if spamassassin_conf.name:lower() == spamassassin_conf.type:lower() then
+ spamassassin_conf.log_prefix = spamassassin_conf.name
+ else
+ spamassassin_conf.log_prefix = spamassassin_conf.name .. ' (' .. spamassassin_conf.type .. ')'
+ end
+ end
+
+ if not spamassassin_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ spamassassin_conf.upstreams = upstream_list.create(rspamd_config,
+ spamassassin_conf.servers,
+ spamassassin_conf.default_port)
+
+ if spamassassin_conf.upstreams then
+ lua_util.add_debug_alias('external_services', spamassassin_conf.N)
+ return spamassassin_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ spamassassin_conf.servers)
+ return nil
+end
+
+local function spamassassin_check(task, content, digest, rule)
+ local function spamassassin_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ -- Build the spamd query
+ -- https://svn.apache.org/repos/asf/spamassassin/trunk/spamd/PROTOCOL
+ local request_data = {
+ "HEADERS SPAMC/1.5\r\n",
+ "User: root\r\n",
+ "Content-length: " .. #content .. "\r\n",
+ "\r\n",
+ content,
+ }
+
+ local function spamassassin_callback(err, data)
+
+ local function spamassassin_requery(error)
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s',
+ rule.log_prefix, error, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ data = request_data,
+ callback = spamassassin_callback,
+ })
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed - err: %s', rule.log_prefix, error)
+ common.yield_result(task, rule, 'failed to scan and retransmits exceed: ' .. error, 0.0, 'fail')
+ end
+ end
+
+ if err then
+
+ spamassassin_requery(err)
+
+ else
+ --lua_util.debugm(rule.N, task, '%s: returned result: %s', rule.log_prefix, data)
+
+ --[[
+ patterns tested against Spamassassin 3.4.6
+
+ X-Spam-Status: No, score=1.1 required=5.0 tests=HTML_MESSAGE,MIME_HTML_ONLY,
+ TVD_RCVD_SPACE_BRACKET,UNPARSEABLE_RELAY autolearn=no
+ autolearn_force=no version=3.4.6
+ ]] --
+ local header = string.gsub(tostring(data), "[\r\n]+[\t ]", " ")
+ --lua_util.debugm(rule.N, task, '%s: returned header: %s', rule.log_prefix, header)
+
+ local symbols = ""
+ local spam_score = 0
+ for s in header:gmatch("[^\r\n]+") do
+ if string.find(s, 'X%-Spam%-Status: %S+, score') then
+ local pattern_symbols = "X%-Spam%-Status: %S+, score%=([%-%d%.]+)%s.*tests%=(.*,?)(%s*%S+)%sautolearn.*"
+ spam_score = string.gsub(s, pattern_symbols, "%1")
+ symbols = string.gsub(s, pattern_symbols, "%2%3")
+ symbols = string.gsub(symbols, "%s", "")
+ end
+ end
+
+ lua_util.debugm(rule.N, task, '%s: spam_score: %s, symbols: %s, int spam_score: |%s|, type spam_score: |%s|',
+ rule.log_prefix, spam_score, symbols, tonumber(spam_score), type(spam_score))
+
+ if tonumber(spam_score) > 0 and #symbols > 0 and symbols ~= "none" then
+
+ if rule.extended == false then
+ common.yield_result(task, rule, symbols, spam_score)
+ common.save_cache(task, digest, rule, symbols, spam_score)
+ else
+ local symbols_table = lua_util.str_split(symbols, ",")
+ lua_util.debugm(rule.N, task, '%s: returned symbols as table: %s', rule.log_prefix, symbols_table)
+
+ common.yield_result(task, rule, symbols_table, spam_score)
+ common.save_cache(task, digest, rule, symbols_table, spam_score)
+ end
+ else
+ common.save_cache(task, digest, rule, 'OK')
+ common.log_clean(task, rule, 'no spam detected - spam score: ' .. spam_score .. ', symbols: ' .. symbols)
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ upstream = upstream,
+ timeout = rule['timeout'],
+ data = request_data,
+ callback = spamassassin_callback,
+ })
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest, spamassassin_check_uncached) then
+ return
+ else
+ spamassassin_check_uncached()
+ end
+
+end
+
+return {
+ type = { N, 'spam', 'scanner' },
+ description = 'spamassassin spam scanner',
+ configure = spamassassin_config,
+ check = spamassassin_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/vadesecure.lua b/lualib/lua_scanners/vadesecure.lua
new file mode 100644
index 0000000..826573a
--- /dev/null
+++ b/lualib/lua_scanners/vadesecure.lua
@@ -0,0 +1,351 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module vadesecure
+-- This module contains Vadesecure Filterd interface
+--]]
+
+local lua_util = require "lua_util"
+local http = require "rspamd_http"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+local common = require "lua_scanners/common"
+
+local N = 'vadesecure'
+
+local function vade_config(opts)
+
+ local vade_conf = {
+ name = N,
+ default_port = 23808,
+ url = '/api/v1/scan',
+ use_https = false,
+ timeout = 5.0,
+ log_clean = false,
+ retransmits = 1,
+ cache_expire = 7200, -- expire redis in 2h
+ message = '${SCANNER}: spam message found: "${VIRUS}"',
+ detection_category = "hash",
+ default_score = 1,
+ action = false,
+ log_spamcause = true,
+ symbol_fail = 'VADE_FAIL',
+ symbol = 'VADE_CHECK',
+ settings_outbound = nil, -- Set when there is a settings id for outbound messages
+ symbols = {
+ clean = {
+ symbol = 'VADE_CLEAN',
+ score = -0.5,
+ description = 'VadeSecure decided message to be clean'
+ },
+ spam = {
+ high = {
+ symbol = 'VADE_SPAM_HIGH',
+ score = 8.0,
+ description = 'VadeSecure decided message to be clearly spam'
+ },
+ medium = {
+ symbol = 'VADE_SPAM_MEDIUM',
+ score = 5.0,
+ description = 'VadeSecure decided message to be highly likely spam'
+ },
+ low = {
+ symbol = 'VADE_SPAM_LOW',
+ score = 2.0,
+ description = 'VadeSecure decided message to be likely spam'
+ },
+ },
+ malware = {
+ symbol = 'VADE_MALWARE',
+ score = 8.0,
+ description = 'VadeSecure decided message to be malware'
+ },
+ scam = {
+ symbol = 'VADE_SCAM',
+ score = 7.0,
+ description = 'VadeSecure decided message to be scam'
+ },
+ phishing = {
+ symbol = 'VADE_PHISHING',
+ score = 8.0,
+ description = 'VadeSecure decided message to be phishing'
+ },
+ commercial = {
+ symbol = 'VADE_COMMERCIAL',
+ score = 0.0,
+ description = 'VadeSecure decided message to be commercial message'
+ },
+ community = {
+ symbol = 'VADE_COMMUNITY',
+ score = 0.0,
+ description = 'VadeSecure decided message to be community message'
+ },
+ transactional = {
+ symbol = 'VADE_TRANSACTIONAL',
+ score = 0.0,
+ description = 'VadeSecure decided message to be transactional message'
+ },
+ suspect = {
+ symbol = 'VADE_SUSPECT',
+ score = 3.0,
+ description = 'VadeSecure decided message to be suspicious message'
+ },
+ bounce = {
+ symbol = 'VADE_BOUNCE',
+ score = 0.0,
+ description = 'VadeSecure decided message to be bounce message'
+ },
+ other = 'VADE_OTHER',
+ }
+ }
+
+ vade_conf = lua_util.override_defaults(vade_conf, opts)
+
+ if not vade_conf.prefix then
+ vade_conf.prefix = 'rs_' .. vade_conf.name .. '_'
+ end
+
+ if not vade_conf.log_prefix then
+ if vade_conf.name:lower() == vade_conf.type:lower() then
+ vade_conf.log_prefix = vade_conf.name
+ else
+ vade_conf.log_prefix = vade_conf.name .. ' (' .. vade_conf.type .. ')'
+ end
+ end
+
+ if not vade_conf.servers and vade_conf.socket then
+ vade_conf.servers = vade_conf.socket
+ end
+
+ if not vade_conf.servers then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ vade_conf.upstreams = upstream_list.create(rspamd_config,
+ vade_conf.servers,
+ vade_conf.default_port)
+
+ if vade_conf.upstreams then
+ lua_util.add_debug_alias('external_services', vade_conf.name)
+ return vade_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ vade_conf['servers'])
+ return nil
+end
+
+local function vade_check(task, content, digest, rule, maybe_part)
+ local function vade_check_uncached()
+ local function vade_url(addr)
+ local url
+ if rule.use_https then
+ url = string.format('https://%s:%d%s', tostring(addr),
+ rule.default_port, rule.url)
+ else
+ url = string.format('http://%s:%d%s', tostring(addr),
+ rule.default_port, rule.url)
+ end
+
+ return url
+ end
+
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+
+ local url = vade_url(addr)
+ local hdrs = {}
+
+ local helo = task:get_helo()
+ if helo then
+ hdrs['X-Helo'] = helo
+ end
+ local mail_from = task:get_from('smtp') or {}
+ if mail_from[1] and #mail_from[1].addr > 1 then
+ hdrs['X-Mailfrom'] = mail_from[1].addr
+ end
+
+ local rcpt_to = task:get_recipients('smtp')
+ if rcpt_to then
+ hdrs['X-Rcptto'] = {}
+ for _, r in ipairs(rcpt_to) do
+ table.insert(hdrs['X-Rcptto'], r.addr)
+ end
+ end
+
+ local fip = task:get_from_ip()
+ if fip and fip:is_valid() then
+ hdrs['X-Inet'] = tostring(fip)
+ end
+
+ if rule.settings_outbound then
+ local settings_id = task:get_settings_id()
+
+ if settings_id then
+ local lua_settings = require "lua_settings"
+ -- Convert to string
+ settings_id = lua_settings.settings_by_id(settings_id)
+
+ if settings_id then
+ settings_id = settings_id.name or ''
+
+ if settings_id == rule.settings_outbound then
+ lua_util.debugm(rule.name, task, '%s settings has matched outbound',
+ settings_id)
+ hdrs['X-Params'] = 'mode=smtpout'
+ end
+ end
+ end
+ end
+
+ local request_data = {
+ task = task,
+ url = url,
+ body = task:get_content(),
+ headers = hdrs,
+ timeout = rule.timeout,
+ }
+
+ local function vade_callback(http_err, code, body, headers)
+
+ local function vade_requery()
+ -- set current upstream to fail because an error occurred
+ upstream:fail()
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ lua_util.debugm(rule.name, task,
+ '%s: Request Error: %s - retries left: %s',
+ rule.log_prefix, http_err, retransmits)
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+ url = vade_url(addr)
+
+ lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+ rule.log_prefix, addr, addr:get_port())
+ request_data.url = url
+
+ http.request(request_data)
+ else
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
+ task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' ..
+ 'retransmits exceed')
+ end
+ end
+
+ if http_err then
+ vade_requery()
+ else
+ -- Parse the response
+ if upstream then
+ upstream:ok()
+ end
+ if code ~= 200 then
+ rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
+ return
+ end
+ local parser = ucl.parser()
+ local ret, err = parser:parse_string(body)
+ if not ret then
+ rspamd_logger.errx(task, 'vade: bad response body (raw): %s', body)
+ task:insert_result(rule.symbol_fail, 1.0, 'Parser error: ' .. err)
+ return
+ end
+ local obj = parser:get_object()
+ local verdict = obj.verdict
+ if not verdict then
+ rspamd_logger.errx(task, 'vade: bad response JSON (no verdict): %s', obj)
+ task:insert_result(rule.symbol_fail, 1.0, 'No verdict/unknown verdict')
+ return
+ end
+ local vparts = lua_util.str_split(verdict, ":")
+ verdict = table.remove(vparts, 1) or verdict
+
+ local sym = rule.symbols[verdict]
+ if not sym then
+ sym = rule.symbols.other
+ end
+
+ if not sym.symbol then
+ -- Subcategory match
+ local lvl = 'low'
+ if vparts and vparts[1] then
+ lvl = vparts[1]
+ end
+
+ if sym[lvl] then
+ sym = sym[lvl]
+ else
+ sym = rule.symbols.other
+ end
+ end
+
+ local opts = {}
+ if obj.score then
+ table.insert(opts, 'score=' .. obj.score)
+ end
+ if obj.elapsed then
+ table.insert(opts, 'elapsed=' .. obj.elapsed)
+ end
+
+ if rule.log_spamcause and obj.spamcause then
+ rspamd_logger.infox(task, 'vadesecure verdict="%s", score=%s, spamcause="%s", message-id="%s"',
+ verdict, obj.score, obj.spamcause, task:get_message_id())
+ else
+ lua_util.debugm(rule.name, task, 'vadesecure returned verdict="%s", score=%s, spamcause="%s"',
+ verdict, obj.score, obj.spamcause)
+ end
+
+ if #vparts > 0 then
+ table.insert(opts, 'verdict=' .. verdict .. ';' .. table.concat(vparts, ':'))
+ end
+
+ task:insert_result(sym.symbol, 1.0, opts)
+ end
+ end
+
+ request_data.callback = vade_callback
+ http.request(request_data)
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ vade_check_uncached, maybe_part) then
+ return
+ else
+ vade_check_uncached()
+ end
+
+end
+
+return {
+ type = { 'vadesecure', 'scanner' },
+ description = 'VadeSecure Filterd interface',
+ configure = vade_config,
+ check = vade_check,
+ name = N
+}
diff --git a/lualib/lua_scanners/virustotal.lua b/lualib/lua_scanners/virustotal.lua
new file mode 100644
index 0000000..d937c41
--- /dev/null
+++ b/lualib/lua_scanners/virustotal.lua
@@ -0,0 +1,214 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module virustotal
+-- This module contains Virustotal integration support
+-- https://www.virustotal.com/
+--]]
+
+local lua_util = require "lua_util"
+local http = require "rspamd_http"
+local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'virustotal'
+
+local function virustotal_config(opts)
+
+ local default_conf = {
+ name = N,
+ url = 'https://www.virustotal.com/vtapi/v2/file',
+ timeout = 5.0,
+ log_clean = false,
+ retransmits = 1,
+ cache_expire = 7200, -- expire redis in 2h
+ message = '${SCANNER}: spam message found: "${VIRUS}"',
+ detection_category = "virus",
+ default_score = 1,
+ action = false,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ apikey = nil, -- Required to set by user
+ -- Specific for virustotal
+ minimum_engines = 3, -- Minimum required to get scored
+ full_score_engines = 7, -- After this number we set max score
+ }
+
+ default_conf = lua_util.override_defaults(default_conf, opts)
+
+ if not default_conf.prefix then
+ default_conf.prefix = 'rs_' .. default_conf.name .. '_'
+ end
+
+ if not default_conf.log_prefix then
+ if default_conf.name:lower() == default_conf.type:lower() then
+ default_conf.log_prefix = default_conf.name
+ else
+ default_conf.log_prefix = default_conf.name .. ' (' .. default_conf.type .. ')'
+ end
+ end
+
+ if not default_conf.apikey then
+ rspamd_logger.errx(rspamd_config, 'no apikey defined for virustotal, disable checks')
+
+ return nil
+ end
+
+ lua_util.add_debug_alias('external_services', default_conf.name)
+ return default_conf
+end
+
+local function virustotal_check(task, content, digest, rule, maybe_part)
+ local function virustotal_check_uncached()
+ local function make_url(hash)
+ return string.format('%s/report?apikey=%s&resource=%s',
+ rule.url, rule.apikey, hash)
+ end
+
+ local hash = rspamd_cryptobox_hash.create_specific('md5')
+ hash:update(content)
+ hash = hash:hex()
+
+ local url = make_url(hash)
+ lua_util.debugm(N, task, "send request %s", url)
+ local request_data = {
+ task = task,
+ url = url,
+ timeout = rule.timeout,
+ }
+
+ local function vt_http_callback(http_err, code, body, headers)
+ if http_err then
+ rspamd_logger.errx(task, 'HTTP error: %s, body: %s, headers: %s', http_err, body, headers)
+ else
+ local cached
+ local dyn_score
+ -- Parse the response
+ if code ~= 200 then
+ if code == 404 then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: hash %s clean (not found)',
+ rule.log_prefix, hash)
+ else
+ lua_util.debugm(rule.name, task, '%s: hash %s clean (not found)',
+ rule.log_prefix, hash)
+ end
+ elseif code == 204 then
+ -- Request rate limit exceeded
+ rspamd_logger.infox(task, 'virustotal request rate limit exceeded')
+ task:insert_result(rule.symbol_fail, 1.0, 'rate limit exceeded')
+ return
+ else
+ rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
+ return
+ end
+ else
+ local ucl = require "ucl"
+ local parser = ucl.parser()
+ local res, json_err = parser:parse_string(body)
+
+ lua_util.debugm(rule.name, task, '%s: got reply data: "%s"',
+ rule.log_prefix, body)
+
+ if res then
+ local obj = parser:get_object()
+ if not obj.positives or type(obj.positives) ~= 'number' then
+ if obj.response_code then
+ if obj.response_code == 0 then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: hash %s clean (not found)',
+ rule.log_prefix, hash)
+ else
+ lua_util.debugm(rule.name, task, '%s: hash %s clean (not found)',
+ rule.log_prefix, hash)
+ end
+ else
+ rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s',
+ 'bad response code: ' .. tostring(obj.response_code), body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element')
+ return
+ end
+ else
+ rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s',
+ 'no response_code', body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element')
+ return
+ end
+ else
+ if obj.positives < rule.minimum_engines then
+ lua_util.debugm(rule.name, task, '%s: hash %s has not enough hits: %s where %s is min',
+ rule.log_prefix, obj.positives, rule.minimum_engines)
+ -- TODO: add proper hashing!
+ cached = 'OK'
+ else
+ if obj.positives > rule.full_score_engines then
+ dyn_score = 1.0
+ else
+ local norm_pos = obj.positives - rule.minimum_engines
+ dyn_score = norm_pos / (rule.full_score_engines - rule.minimum_engines)
+ end
+
+ if dyn_score < 0 or dyn_score > 1 then
+ dyn_score = 1.0
+ end
+ local sopt = string.format("%s:%s/%s",
+ hash, obj.positives, obj.total)
+ common.yield_result(task, rule, sopt, dyn_score, nil, maybe_part)
+ cached = sopt
+ end
+ end
+ else
+ -- not res
+ rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s',
+ json_err, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: ' .. json_err)
+ return
+ end
+ end
+
+ if cached then
+ common.save_cache(task, digest, rule, cached, dyn_score, maybe_part)
+ end
+ end
+ end
+
+ request_data.callback = vt_http_callback
+ http.request(request_data)
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ virustotal_check_uncached) then
+ return
+ else
+
+ virustotal_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'Virustotal integration',
+ configure = virustotal_config,
+ check = virustotal_check,
+ name = N
+}
diff --git a/lualib/lua_selectors/common.lua b/lualib/lua_selectors/common.lua
new file mode 100644
index 0000000..7b2372d
--- /dev/null
+++ b/lualib/lua_selectors/common.lua
@@ -0,0 +1,95 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local ts = require("tableshape").types
+local exports = {}
+local cr_hash = require 'rspamd_cryptobox_hash'
+
+local blake2b_key = cr_hash.create_specific('blake2'):update('rspamd'):bin()
+
+local function digest_schema()
+ return { ts.one_of { 'hex', 'base32', 'bleach32', 'rbase32', 'base64' }:is_optional(),
+ ts.one_of { 'blake2', 'sha256', 'sha1', 'sha512', 'md5' }:is_optional() }
+end
+
+exports.digest_schema = digest_schema
+
+local function create_raw_digest(data, args)
+ local ht = args[2] or 'blake2'
+
+ local h
+
+ if ht == 'blake2' then
+ -- Hack to be compatible with various 'get_digest' methods
+ h = cr_hash.create_keyed(blake2b_key):update(data)
+ else
+ h = cr_hash.create_specific(ht):update(data)
+ end
+
+ return h
+end
+
+local function encode_digest(h, args)
+ local encoding = args[1] or 'hex'
+
+ local s
+ if encoding == 'hex' then
+ s = h:hex()
+ elseif encoding == 'base32' then
+ s = h:base32()
+ elseif encoding == 'bleach32' then
+ s = h:base32('bleach')
+ elseif encoding == 'rbase32' then
+ s = h:base32('rfc')
+ elseif encoding == 'base64' then
+ s = h:base64()
+ end
+
+ return s
+end
+
+local function create_digest(data, args)
+ local h = create_raw_digest(data, args)
+ return encode_digest(h, args)
+end
+
+local function get_cached_or_raw_digest(task, idx, mime_part, args)
+ if #args == 0 then
+ -- Optimise as we already have this hash in the API
+ return mime_part:get_digest()
+ end
+
+ local ht = args[2] or 'blake2'
+ local cache_key = 'mp_digest_' .. ht .. tostring(idx)
+
+ local cached = task:cache_get(cache_key)
+
+ if cached then
+ return encode_digest(cached, args)
+ end
+
+ local h = create_raw_digest(mime_part:get_content('raw_parsed'), args)
+ task:cache_set(cache_key, h)
+
+ return encode_digest(h, args)
+end
+
+exports.create_digest = create_digest
+exports.create_raw_digest = create_raw_digest
+exports.get_cached_or_raw_digest = get_cached_or_raw_digest
+exports.encode_digest = encode_digest
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_selectors/extractors.lua b/lualib/lua_selectors/extractors.lua
new file mode 100644
index 0000000..81dfa9d
--- /dev/null
+++ b/lualib/lua_selectors/extractors.lua
@@ -0,0 +1,565 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local fun = require 'fun'
+local meta_functions = require "lua_meta"
+local lua_util = require "lua_util"
+local rspamd_url = require "rspamd_url"
+local common = require "lua_selectors/common"
+local ts = require("tableshape").types
+local maps = require "lua_selectors/maps"
+local E = {}
+local M = "selectors"
+
+local url_flags_ts = ts.array_of(ts.one_of(lua_util.keys(rspamd_url.flags))):is_optional()
+
+local function gen_exclude_flags_filter(exclude_flags)
+ return function(u)
+ local got_flags = u:get_flags()
+ for _, flag in ipairs(exclude_flags) do
+ if got_flags[flag] then
+ return false
+ end
+ end
+ return true
+ end
+end
+
+local extractors = {
+ -- Plain id function
+ ['id'] = {
+ ['get_value'] = function(_, args)
+ if args[1] then
+ return args[1], 'string'
+ end
+
+ return '', 'string'
+ end,
+ ['description'] = [[Return value from function's argument or an empty string,
+For example, `id('Something')` returns a string 'Something']],
+ ['args_schema'] = { ts.string:is_optional() }
+ },
+ -- Similar but for making lists
+ ['list'] = {
+ ['get_value'] = function(_, args)
+ if args[1] then
+ return fun.map(tostring, args), 'string_list'
+ end
+
+ return {}, 'string_list'
+ end,
+ ['description'] = [[Return a list from function's arguments or an empty list,
+For example, `list('foo', 'bar')` returns a list {'foo', 'bar'}]],
+ },
+ -- Get source IP address
+ ['ip'] = {
+ ['get_value'] = function(task)
+ local ip = task:get_ip()
+ if ip and ip:is_valid() then
+ return ip, 'userdata'
+ end
+ return nil
+ end,
+ ['description'] = [[Get source IP address]],
+ },
+ -- Get MIME from
+ ['from'] = {
+ ['get_value'] = function(task, args)
+ local from
+ if type(args) == 'table' then
+ from = task:get_from(args)
+ else
+ from = task:get_from(0)
+ end
+ if ((from or E)[1] or E).addr then
+ return from[1], 'table'
+ end
+ return nil
+ end,
+ ['description'] = [[Get MIME or SMTP from (e.g. `from('smtp')` or `from('mime')`,
+uses any type by default)]],
+ },
+ ['rcpts'] = {
+ ['get_value'] = function(task, args)
+ local rcpts
+ if type(args) == 'table' then
+ rcpts = task:get_recipients(args)
+ else
+ rcpts = task:get_recipients(0)
+ end
+ if ((rcpts or E)[1] or E).addr then
+ return rcpts, 'table_list'
+ end
+ return nil
+ end,
+ ['description'] = [[Get MIME or SMTP rcpts (e.g. `rcpts('smtp')` or `rcpts('mime')`,
+uses any type by default)]],
+ },
+ -- Get country (ASN module must be executed first)
+ ['country'] = {
+ ['get_value'] = function(task)
+ local country = task:get_mempool():get_variable('country')
+ if not country then
+ return nil
+ else
+ return country, 'string'
+ end
+ end,
+ ['description'] = [[Get country (ASN module must be executed first)]],
+ },
+ -- Get ASN number
+ ['asn'] = {
+ ['type'] = 'string',
+ ['get_value'] = function(task)
+ local asn = task:get_mempool():get_variable('asn')
+ if not asn then
+ return nil
+ else
+ return asn, 'string'
+ end
+ end,
+ ['description'] = [[Get AS number (ASN module must be executed first)]],
+ },
+ -- Get authenticated username
+ ['user'] = {
+ ['get_value'] = function(task)
+ local auser = task:get_user()
+ if not auser then
+ return nil
+ else
+ return auser, 'string'
+ end
+ end,
+ ['description'] = 'Get authenticated user name',
+ },
+ -- Get principal recipient
+ ['to'] = {
+ ['get_value'] = function(task)
+ return task:get_principal_recipient(), 'string'
+ end,
+ ['description'] = 'Get principal recipient',
+ },
+ -- Get content digest
+ ['digest'] = {
+ ['get_value'] = function(task)
+ return task:get_digest(), 'string'
+ end,
+ ['description'] = 'Get content digest',
+ },
+ -- Get list of all attachments digests
+ ['attachments'] = {
+ ['get_value'] = function(task, args)
+ local parts = task:get_parts() or E
+ local digests = {}
+ for i, p in ipairs(parts) do
+ if p:is_attachment() then
+ table.insert(digests, common.get_cached_or_raw_digest(task, i, p, args))
+ end
+ end
+
+ if #digests > 0 then
+ return digests, 'string_list'
+ end
+
+ return nil
+ end,
+ ['description'] = [[Get list of all attachments digests.
+The first optional argument is encoding (`hex`, `base32` (and forms `bleach32`, `rbase32`), `base64`),
+the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, `md5`)]],
+ ['args_schema'] = common.digest_schema()
+
+ },
+ -- Get all attachments files
+ ['files'] = {
+ ['get_value'] = function(task)
+ local parts = task:get_parts() or E
+ local files = {}
+
+ for _, p in ipairs(parts) do
+ local fname = p:get_filename()
+ if fname then
+ table.insert(files, fname)
+ end
+ end
+
+ if #files > 0 then
+ return files, 'string_list'
+ end
+
+ return nil
+ end,
+ ['description'] = 'Get all attachments files',
+ },
+ -- Get languages for text parts
+ ['languages'] = {
+ ['get_value'] = function(task)
+ local text_parts = task:get_text_parts() or E
+ local languages = {}
+
+ for _, p in ipairs(text_parts) do
+ local lang = p:get_language()
+ if lang then
+ table.insert(languages, lang)
+ end
+ end
+
+ if #languages > 0 then
+ return languages, 'string_list'
+ end
+
+ return nil
+ end,
+ ['description'] = 'Get languages for text parts',
+ },
+ -- Get helo value
+ ['helo'] = {
+ ['get_value'] = function(task)
+ return task:get_helo(), 'string'
+ end,
+ ['description'] = 'Get helo value',
+ },
+ -- Get header with the name that is expected as an argument. Returns list of
+ -- headers with this name
+ ['header'] = {
+ ['get_value'] = function(task, args)
+ local strong = false
+ if args[2] then
+ if args[2]:match('strong') then
+ strong = true
+ end
+
+ if args[2]:match('full') then
+ return task:get_header_full(args[1], strong), 'table_list'
+ end
+
+ return task:get_header(args[1], strong), 'string'
+ else
+ return task:get_header(args[1]), 'string'
+ end
+ end,
+ ['description'] = [[Get header with the name that is expected as an argument.
+The optional second argument accepts list of flags:
+ - `full`: returns all headers with this name with all data (like task:get_header_full())
+ - `strong`: use case sensitive match when matching header's name]],
+ ['args_schema'] = { ts.string,
+ (ts.pattern("strong") + ts.pattern("full")):is_optional() }
+ },
+ -- Get list of received headers (returns list of tables)
+ ['received'] = {
+ ['get_value'] = function(task, args)
+ local rh = task:get_received_headers()
+ if not rh[1] then
+ return nil
+ end
+ if args[1] then
+ return fun.map(function(r)
+ return r[args[1]]
+ end, rh), 'string_list'
+ end
+
+ return rh, 'table_list'
+ end,
+ ['description'] = [[Get list of received headers.
+If no arguments specified, returns list of tables. Otherwise, selects a specific element,
+e.g. `by_hostname`]],
+ },
+ -- Get all urls
+ ['urls'] = {
+ ['get_value'] = function(task, args)
+ local urls = task:get_urls()
+ if not urls[1] then
+ return nil
+ end
+ if args[1] then
+ return fun.map(function(r)
+ return r[args[1]](r)
+ end, urls), 'string_list'
+ end
+ return urls, 'userdata_list'
+ end,
+ ['description'] = [[Get list of all urls.
+If no arguments specified, returns list of url objects. Otherwise, calls a specific method,
+e.g. `get_tld`]],
+ },
+ -- Get specific urls
+ ['specific_urls'] = {
+ ['get_value'] = function(task, args)
+ local params = args[1] or {}
+ params.task = task
+ params.no_cache = true
+ if params.exclude_flags then
+ params.filter = gen_exclude_flags_filter(params.exclude_flags)
+ end
+ local urls = lua_util.extract_specific_urls(params)
+ if not urls[1] then
+ return nil
+ end
+ return urls, 'userdata_list'
+ end,
+ ['description'] = [[Get most specific urls. Arguments are equal to the Lua API function]],
+ ['args_schema'] = { ts.shape {
+ limit = ts.number + ts.string / tonumber,
+ esld_limit = (ts.number + ts.string / tonumber):is_optional(),
+ exclude_flags = url_flags_ts,
+ flags = url_flags_ts,
+ flags_mode = ts.one_of { 'explicit' }:is_optional(),
+ prefix = ts.string:is_optional(),
+ need_content = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
+ need_emails = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
+ need_images = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
+ ignore_redirected = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
+ } }
+ },
+ ['specific_urls_filter_map'] = {
+ ['get_value'] = function(task, args)
+ local map = maps[args[1]]
+ if not map then
+ lua_util.debugm(M, "invalid/unknown map: %s", args[1])
+ end
+ local params = args[2] or {}
+ params.task = task
+ params.no_cache = true
+ if params.exclude_flags then
+ params.filter = gen_exclude_flags_filter(params.exclude_flags)
+ end
+ local urls = lua_util.extract_specific_urls(params)
+ if not urls[1] then
+ return nil
+ end
+ return fun.filter(function(u)
+ return map:get_key(tostring(u))
+ end, urls), 'userdata_list'
+ end,
+ ['description'] = [[Get most specific urls, filtered by some map. Arguments are equal to the Lua API function]],
+ ['args_schema'] = { ts.string, ts.shape {
+ limit = ts.number + ts.string / tonumber,
+ esld_limit = (ts.number + ts.string / tonumber):is_optional(),
+ exclude_flags = url_flags_ts,
+ flags = url_flags_ts,
+ flags_mode = ts.one_of { 'explicit' }:is_optional(),
+ prefix = ts.string:is_optional(),
+ need_content = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
+ need_emails = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
+ need_images = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
+ ignore_redirected = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
+ } }
+ },
+ -- URLs filtered by flags
+ ['urls_filtered'] = {
+ ['get_value'] = function(task, args)
+ local urls = task:get_urls_filtered(args[1], args[2])
+ if not urls[1] then
+ return nil
+ end
+ return urls, 'userdata_list'
+ end,
+ ['description'] = [[Get list of all urls filtered by flags_include/exclude
+(see rspamd_task:get_urls_filtered for description)]],
+ ['args_schema'] = { ts.array_of {
+ url_flags_ts:is_optional(), url_flags_ts:is_optional()
+ } }
+ },
+ -- Get all emails
+ ['emails'] = {
+ ['get_value'] = function(task, args)
+ local urls = task:get_emails()
+ if not urls[1] then
+ return nil
+ end
+ if args[1] then
+ return fun.map(function(r)
+ return r[args[1]](r)
+ end, urls), 'string_list'
+ end
+ return urls, 'userdata_list'
+ end,
+ ['description'] = [[Get list of all emails.
+If no arguments specified, returns list of url objects. Otherwise, calls a specific method,
+e.g. `get_user`]],
+ },
+ -- Get specific pool var. The first argument must be variable name,
+ -- the second argument is optional and defines the type (string by default)
+ ['pool_var'] = {
+ ['get_value'] = function(task, args)
+ local type = args[2] or 'string'
+ return task:get_mempool():get_variable(args[1], type), (type)
+ end,
+ ['description'] = [[Get specific pool var. The first argument must be variable name,
+the second argument is optional and defines the type (string by default)]],
+ ['args_schema'] = { ts.string, ts.string:is_optional() }
+ },
+ -- Get value of specific key from task cache
+ ['task_cache'] = {
+ ['get_value'] = function(task, args)
+ local val = task:cache_get(args[1])
+ if not val then
+ return
+ end
+ if type(val) == 'table' then
+ if not val[1] then
+ return
+ end
+ return val, 'string_list'
+ end
+ return val, 'string'
+ end,
+ ['description'] = [[Get value of specific key from task cache. The first argument must be
+the key name]],
+ ['args_schema'] = { ts.string }
+ },
+ -- Get specific HTTP request header. The first argument must be header name.
+ ['request_header'] = {
+ ['get_value'] = function(task, args)
+ local hdr = task:get_request_header(args[1])
+ if hdr then
+ return hdr, 'string'
+ end
+
+ return nil
+ end,
+ ['description'] = [[Get specific HTTP request header.
+The first argument must be header name.]],
+ ['args_schema'] = { ts.string }
+ },
+ -- Get task date, optionally formatted
+ ['time'] = {
+ ['get_value'] = function(task, args)
+ local what = args[1] or 'message'
+ local dt = task:get_date { format = what, gmt = true }
+
+ if dt then
+ if args[2] then
+ -- Should be in format !xxx, as dt is in GMT
+ return os.date(args[2], dt), 'string'
+ end
+
+ return tostring(dt), 'string'
+ end
+
+ return nil
+ end,
+ ['description'] = [[Get task timestamp. The first argument is type:
+ - `connect`: connection timestamp (default)
+ - `message`: timestamp as defined by `Date` header
+
+ The second argument is optional time format, see [os.date](http://pgl.yoyo.org/luai/i/os.date) description]],
+ ['args_schema'] = { ts.one_of { 'connect', 'message' }:is_optional(),
+ ts.string:is_optional() }
+ },
+ -- Get text words from a message
+ ['words'] = {
+ ['get_value'] = function(task, args)
+ local how = args[1] or 'stem'
+ local tp = task:get_text_parts()
+
+ if tp then
+ local rtype = 'string_list'
+ if how == 'full' then
+ rtype = 'table_list'
+ end
+
+ return lua_util.flatten(
+ fun.map(function(p)
+ return p:get_words(how)
+ end, tp)), rtype
+ end
+
+ return nil
+ end,
+ ['description'] = [[Get words from text parts
+ - `stem`: stemmed words (default)
+ - `raw`: raw words
+ - `norm`: normalised words (lowercased)
+ - `full`: list of tables
+ ]],
+ ['args_schema'] = { ts.one_of { 'stem', 'raw', 'norm', 'full' }:is_optional() },
+ },
+ -- Get queue ID
+ ['queueid'] = {
+ ['get_value'] = function(task)
+ local queueid = task:get_queue_id()
+ if queueid then
+ return queueid, 'string'
+ end
+ return nil
+ end,
+ ['description'] = [[Get queue ID]],
+ },
+ -- Get ID of the task being processed
+ ['uid'] = {
+ ['get_value'] = function(task)
+ local uid = task:get_uid()
+ if uid then
+ return uid, 'string'
+ end
+ return nil
+ end,
+ ['description'] = [[Get ID of the task being processed]],
+ },
+ -- Get message ID of the task being processed
+ ['messageid'] = {
+ ['get_value'] = function(task)
+ local mid = task:get_message_id()
+ if mid then
+ return mid, 'string'
+ end
+ return nil
+ end,
+ ['description'] = [[Get message ID]],
+ },
+ -- Get specific symbol
+ ['symbol'] = {
+ ['get_value'] = function(task, args)
+ local symbol = task:get_symbol(args[1], args[2])
+ if symbol then
+ return symbol[1], 'table'
+ end
+ end,
+ ['description'] = 'Get specific symbol. The first argument must be the symbol name. ' ..
+ 'The second argument is an optional shadow result name. ' ..
+ 'Returns the symbol table. See task:get_symbol()',
+ ['args_schema'] = { ts.string, ts.string:is_optional() }
+ },
+ -- Get full scan result
+ ['scan_result'] = {
+ ['get_value'] = function(task, args)
+ local res = task:get_metric_result(args[1])
+ if res then
+ return res, 'table'
+ end
+ end,
+ ['description'] = 'Get full scan result (either default or shadow if shadow result name is specified)' ..
+ 'Returns the result table. See task:get_metric_result()',
+ ['args_schema'] = { ts.string:is_optional() }
+ },
+ -- Get list of metatokens as strings
+ ['metatokens'] = {
+ ['get_value'] = function(task)
+ local tokens = meta_functions.gen_metatokens(task)
+ if not tokens[1] then
+ return nil
+ end
+ local res = {}
+ for _, t in ipairs(tokens) do
+ table.insert(res, tostring(t))
+ end
+ return res, 'string_list'
+ end,
+ ['description'] = 'Get metatokens for a message as strings',
+ },
+}
+
+return extractors
diff --git a/lualib/lua_selectors/init.lua b/lualib/lua_selectors/init.lua
new file mode 100644
index 0000000..5fcdb38
--- /dev/null
+++ b/lualib/lua_selectors/init.lua
@@ -0,0 +1,668 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- This module contains 'selectors' implementation: code to extract data
+-- from Rspamd tasks and compose those together
+--
+-- Read more at https://rspamd.com/doc/configuration/selectors.html
+
+--[[[
+-- @module lua_selectors
+-- This module contains 'selectors' implementation: code to extract data
+-- from Rspamd tasks and compose those together.
+-- Typical selector looks like this: header(User).lower.substring(1, 2):ip
+--]]
+
+local exports = {
+ maps = require "lua_selectors/maps"
+}
+
+local logger = require 'rspamd_logger'
+local fun = require 'fun'
+local lua_util = require "lua_util"
+local M = "selectors"
+local rspamd_text = require "rspamd_text"
+local unpack_function = table.unpack or unpack
+local E = {}
+
+local extractors = require "lua_selectors/extractors"
+local transform_function = require "lua_selectors/transforms"
+
+local text_cookie = rspamd_text.cookie
+
+local function pure_type(ltype)
+ return ltype:match('^(.*)_list$')
+end
+
+local function implicit_tostring(t, ud_or_table)
+ if t == 'table' then
+ -- Table (very special)
+ if ud_or_table.value then
+ return ud_or_table.value, 'string'
+ elseif ud_or_table.addr then
+ return ud_or_table.addr, 'string'
+ end
+
+ return logger.slog("%s", ud_or_table), 'string'
+ elseif (t == 'string' or t == 'text') and type(ud_or_table) == 'userdata' then
+ if ud_or_table.cookie and ud_or_table.cookie == text_cookie then
+ -- Preserve opaque
+ return ud_or_table, 'string'
+ else
+ return tostring(ud_or_table), 'string'
+ end
+ elseif t ~= 'nil' then
+ return tostring(ud_or_table), 'string'
+ end
+
+ return nil
+end
+
+local function process_selector(task, sel)
+ local function allowed_type(t)
+ if t == 'string' or t == 'string_list' then
+ return true
+ end
+
+ return false
+ end
+
+ local function list_type(t)
+ return pure_type(t)
+ end
+
+ local input, etype = sel.selector.get_value(task, sel.selector.args)
+
+ if not input then
+ lua_util.debugm(M, task, 'no value extracted for %s', sel.selector.name)
+ return nil
+ end
+
+ lua_util.debugm(M, task, 'extracted %s, type %s',
+ sel.selector.name, etype)
+
+ local pipe = sel.processor_pipe or E
+ local first_elt = pipe[1]
+
+ if first_elt and (first_elt.method or
+ fun.any(function(t)
+ return t == 'userdata' or t == 'table'
+ end, first_elt.types)) then
+ -- Explicit conversion
+ local meth = first_elt
+
+ if meth.types[etype] then
+ lua_util.debugm(M, task, 'apply method `%s` to %s',
+ meth.name, etype)
+ input, etype = meth.process(input, etype, meth.args)
+ else
+ local pt = pure_type(etype)
+
+ if meth.types[pt] then
+ lua_util.debugm(M, task, 'map method `%s` to list of %s',
+ meth.name, pt)
+ -- Map method to a list of inputs, excluding empty elements
+ -- We need to fold it down here to get a proper type resolution
+ input = fun.totable(fun.filter(function(map_elt, _)
+ return map_elt
+ end,
+ fun.map(function(list_elt)
+ local ret, ty = meth.process(list_elt, pt, meth.args)
+ if ret then
+ etype = ty
+ end
+ return ret
+ end, input)))
+ if input and etype then
+ etype = etype .. "_list"
+ else
+ input = nil
+ end
+ end
+ end
+ -- Remove method from the pipeline
+ pipe = fun.drop_n(1, pipe)
+ elseif etype:match('^userdata') or etype:match('^table') then
+ -- Implicit conversion
+ local pt = pure_type(etype)
+
+ if not pt then
+ lua_util.debugm(M, task, 'apply implicit conversion %s->string', etype)
+ input = implicit_tostring(etype, input)
+ etype = 'string'
+ else
+ lua_util.debugm(M, task, 'apply implicit map %s->string', pt)
+ input = fun.filter(function(map_elt)
+ return map_elt
+ end,
+ fun.map(function(list_elt)
+ local ret = implicit_tostring(pt, list_elt)
+ return ret
+ end, input))
+ etype = 'string_list'
+ end
+ else
+ lua_util.debugm(M, task, 'avoid implicit conversion as the transformer accepts complex input')
+ end
+
+ -- Now we fold elements using left fold
+ local function fold_function(acc, x)
+ if acc == nil or acc[1] == nil then
+ lua_util.debugm(M, task, 'do not apply %s, accumulator is nil', x.name)
+ return nil
+ end
+
+ local value = acc[1]
+ local t = acc[2]
+
+ if not x.types[t] then
+ local pt = pure_type(t)
+
+ if pt and x.types['list'] then
+ -- Generic list processor
+ lua_util.debugm(M, task, 'apply list function `%s` to %s', x.name, t)
+ return { x.process(value, t, x.args) }
+ elseif pt and x.map_type and x.types[pt] then
+ local map_type = x.map_type .. '_list'
+ lua_util.debugm(M, task, 'map `%s` to list of %s resulting %s',
+ x.name, pt, map_type)
+ -- Apply map, filtering empty values
+ return {
+ fun.filter(function(map_elt)
+ return map_elt
+ end,
+ fun.map(function(list_elt)
+ if not list_elt then
+ return nil
+ end
+ local ret, _ = x.process(list_elt, pt, x.args)
+ return ret
+ end, value)),
+ map_type -- Returned type
+ }
+ end
+ logger.errx(task, 'cannot apply transform %s for type %s', x.name, t)
+ return nil
+ end
+
+ lua_util.debugm(M, task, 'apply %s to %s', x.name, t)
+ return { x.process(value, t, x.args) }
+ end
+
+ local res = fun.foldl(fold_function,
+ { input, etype },
+ pipe)
+
+ if not res or not res[1] then
+ return nil
+ end -- Pipeline failed
+
+ if not allowed_type(res[2]) then
+ -- Search for implicit conversion
+ local pt = pure_type(res[2])
+
+ if pt then
+ lua_util.debugm(M, task, 'apply implicit map %s->string_list', pt)
+ res[1] = fun.map(function(e)
+ return implicit_tostring(pt, e)
+ end, res[1])
+ res[2] = 'string_list'
+ else
+ res[1] = implicit_tostring(res[2], res[1])
+ res[2] = 'string'
+ end
+ end
+
+ if list_type(res[2]) then
+ -- Convert to table as it might have a functional form
+ res[1] = fun.totable(res[1])
+ end
+
+ lua_util.debugm(M, task, 'final selector type: %s, value: %s', res[2], res[1])
+
+ return res[1]
+end
+
+local function make_grammar()
+ local l = require "lpeg"
+ local spc = l.S(" \t\n") ^ 0
+ local cont = l.R("\128\191") -- continuation byte
+ local utf8_high = l.R("\194\223") * cont
+ + l.R("\224\239") * cont * cont
+ + l.R("\240\244") * cont * cont * cont
+ local atom_start = (l.R("az") + l.R("AZ") + l.R("09") + utf8_high + l.S "-") ^ 1
+ local atom_end = (l.R("az") + l.R("AZ") + l.R("09") + l.S "-_" + utf8_high) ^ 1
+ local atom_mid = (1 - l.S("'\r\n\f\\,)(}{= " .. '"')) ^ 1
+ local atom_argument = l.C(atom_start * atom_mid ^ 0 * atom_end ^ 0) -- We allow more characters for the arguments
+ local atom = l.C(atom_start * atom_end ^ 0) -- We are more strict about selector names itself
+ local singlequoted_string = l.P "'" * l.C(((1 - l.S "'\r\n\f\\") + (l.P '\\' * 1)) ^ 0) * "'"
+ local doublequoted_string = l.P '"' * l.C(((1 - l.S '"\r\n\f\\') + (l.P '\\' * 1)) ^ 0) * '"'
+ local argument = atom_argument + singlequoted_string + doublequoted_string
+ local dot = l.P(".")
+ local semicolon = l.P(":")
+ local obrace = "(" * spc
+ local tbl_obrace = "{" * spc
+ local eqsign = spc * "=" * spc
+ local tbl_ebrace = spc * "}"
+ local ebrace = spc * ")"
+ local comma = spc * "," * spc
+ local sel_separator = spc * l.S ";*" * spc
+
+ return l.P {
+ "LIST";
+ LIST = l.Ct(l.V("EXPR")) * (sel_separator * l.Ct(l.V("EXPR"))) ^ 0,
+ EXPR = l.V("FUNCTION") * (semicolon * l.V("METHOD")) ^ -1 * (dot * l.V("PROCESSOR")) ^ 0,
+ PROCESSOR = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0),
+ FUNCTION = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0),
+ METHOD = l.Ct(atom / function(e)
+ return '__' .. e
+ end * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0),
+ ARG_LIST = l.Ct((l.V("ARG") * comma ^ 0) ^ 0),
+ ARG = l.Cf(tbl_obrace * l.V("NAMED_ARG") * tbl_ebrace, rawset) + argument + l.V("LIST_ARGS"),
+ NAMED_ARG = (l.Ct("") * l.Cg(argument * eqsign * (argument + l.V("LIST_ARGS")) * comma ^ 0) ^ 0),
+ LIST_ARGS = l.Ct(tbl_obrace * l.V("LIST_ARG") * tbl_ebrace),
+ LIST_ARG = l.Cg(argument * comma ^ 0) ^ 0,
+ }
+end
+
+local parser = make_grammar()
+
+--[[[
+-- @function lua_selectors.parse_selector(cfg, str)
+--]]
+exports.parse_selector = function(cfg, str)
+ local parsed = { parser:match(str) }
+ local output = {}
+
+ if not parsed or not parsed[1] then
+ return nil
+ end
+
+ local function check_args(name, schema, args)
+ if schema then
+ if getmetatable(schema) then
+ -- Schema covers all arguments
+ local res, err = schema:transform(args)
+ if not res then
+ logger.errx(rspamd_config, 'invalid arguments for %s: %s', name, err)
+ return false
+ else
+ for i, elt in ipairs(res) do
+ args[i] = elt
+ end
+ end
+ else
+ for i, selt in ipairs(schema) do
+ local res, err = selt:transform(args[i])
+
+ if err then
+ logger.errx(rspamd_config, 'invalid arguments for %s: argument number: %s, error: %s', name, i, err)
+ return false
+ else
+ args[i] = res
+ end
+ end
+ end
+ end
+
+ return true
+ end
+
+ -- Output AST format is the following:
+ -- table of individual selectors
+ -- each selector: list of functions
+ -- each function: function name + optional list of arguments
+ for _, sel in ipairs(parsed) do
+ local res = {
+ selector = {},
+ processor_pipe = {},
+ }
+
+ local selector_tbl = sel[1]
+ if not selector_tbl then
+ logger.errx(cfg, 'no selector represented')
+ return nil
+ end
+ if not extractors[selector_tbl[1]] then
+ logger.errx(cfg, 'selector %s is unknown', selector_tbl[1])
+ return nil
+ end
+
+ res.selector = lua_util.shallowcopy(extractors[selector_tbl[1]])
+ res.selector.name = selector_tbl[1]
+ res.selector.args = selector_tbl[2] or E
+
+ if not check_args(res.selector.name,
+ res.selector.args_schema,
+ res.selector.args) then
+ return nil
+ end
+
+ lua_util.debugm(M, cfg, 'processed selector %s, args: %s',
+ res.selector.name, res.selector.args)
+
+ local pipeline_error = false
+ -- Now process processors pipe
+ fun.each(function(proc_tbl)
+ local proc_name = proc_tbl[1]
+
+ if proc_name:match('^__') then
+ -- Special case - method
+ local method_name = proc_name:match('^__(.*)$')
+ -- Check array indexing...
+ if tonumber(method_name) then
+ method_name = tonumber(method_name)
+ end
+ local processor = {
+ name = tostring(method_name),
+ method = true,
+ args = proc_tbl[2] or E,
+ types = {
+ userdata = true,
+ table = true,
+ string = true,
+ },
+ map_type = 'string',
+ process = function(inp, t, args)
+ local ret
+ if t == 'table' then
+ -- Plain table field
+ ret = inp[method_name]
+ else
+ -- We call method unpacking arguments and dropping all but the first result returned
+ ret = (inp[method_name](inp, unpack_function(args or E)))
+ end
+
+ local ret_type = type(ret)
+
+ if ret_type == 'nil' then
+ return nil
+ end
+ -- Now apply types heuristic
+ if ret_type == 'string' then
+ return ret, 'string'
+ elseif ret_type == 'table' then
+ -- TODO: we need to ensure that 1) table is numeric 2) table has merely strings
+ return ret, 'string_list'
+ else
+ return implicit_tostring(ret_type, ret)
+ end
+ end,
+ }
+ lua_util.debugm(M, cfg, 'attached method %s to selector %s, args: %s',
+ proc_name, res.selector.name, processor.args)
+ table.insert(res.processor_pipe, processor)
+ else
+
+ if not transform_function[proc_name] then
+ logger.errx(cfg, 'processor %s is unknown', proc_name)
+ pipeline_error = proc_name
+ return nil
+ end
+ local processor = lua_util.shallowcopy(transform_function[proc_name])
+ processor.name = proc_name
+ processor.args = proc_tbl[2] or E
+
+ if not check_args(processor.name, processor.args_schema, processor.args) then
+ pipeline_error = 'args schema for ' .. proc_name
+ return nil
+ end
+
+ lua_util.debugm(M, cfg, 'attached processor %s to selector %s, args: %s',
+ proc_name, res.selector.name, processor.args)
+ table.insert(res.processor_pipe, processor)
+ end
+ end, fun.tail(sel))
+
+ if pipeline_error then
+ logger.errx(cfg, 'unknown or invalid processor used: "%s", exiting', pipeline_error)
+ return nil
+ end
+
+ table.insert(output, res)
+ end
+
+ return output
+end
+
+--[[[
+-- @function lua_selectors.register_extractor(cfg, name, selector)
+--]]
+exports.register_extractor = function(cfg, name, selector)
+ if selector.get_value then
+ if extractors[name] then
+ logger.warnx(cfg, 'redefining selector %s', name)
+ end
+ extractors[name] = selector
+
+ return true
+ end
+
+ logger.errx(cfg, 'bad selector %s', name)
+ return false
+end
+
+--[[[
+-- @function lua_selectors.register_transform(cfg, name, transform)
+--]]
+exports.register_transform = function(cfg, name, transform)
+ if transform.process and transform.types then
+ if transform_function[name] then
+ logger.warnx(cfg, 'redefining transform function %s', name)
+ end
+ transform_function[name] = transform
+
+ return true
+ end
+
+ logger.errx(cfg, 'bad transform function %s', name)
+ return false
+end
+
+--[[[
+-- @function lua_selectors.process_selectors(task, selectors_pipe)
+--]]
+exports.process_selectors = function(task, selectors_pipe)
+ local ret = {}
+
+ for _, sel in ipairs(selectors_pipe) do
+ local r = process_selector(task, sel)
+
+ -- If any element is nil, then the whole selector is nil
+ if not r then
+ return nil
+ end
+ table.insert(ret, r)
+ end
+
+ return ret
+end
+
+--[[[
+-- @function lua_selectors.combine_selectors(task, selectors, delimiter)
+--]]
+exports.combine_selectors = function(_, selectors, delimiter)
+ if not delimiter then
+ delimiter = ''
+ end
+
+ if not selectors then
+ return nil
+ end
+
+ local have_tables, have_userdata
+
+ for _, s in ipairs(selectors) do
+ if type(s) == 'table' then
+ have_tables = true
+ elseif type(s) == 'userdata' then
+ have_userdata = true
+ end
+ end
+
+ if not have_tables then
+ if not have_userdata then
+ return table.concat(selectors, delimiter)
+ else
+ return rspamd_text.fromtable(selectors, delimiter)
+ end
+ else
+ -- We need to do a spill on each table selector and make a cortesian product
+ -- e.g. s:tbl:s -> s:telt1:s + s:telt2:s ...
+ local tbl = {}
+ local res = {}
+
+ for i, s in ipairs(selectors) do
+ if type(s) == 'string' then
+ rawset(tbl, i, fun.duplicate(s))
+ elseif type(s) == 'userdata' then
+ rawset(tbl, i, fun.duplicate(tostring(s)))
+ else
+ -- Raw table
+ rawset(tbl, i, fun.map(tostring, s))
+ end
+ end
+
+ fun.each(function(...)
+ table.insert(res, table.concat({ ... }, delimiter))
+ end, fun.zip(lua_util.unpack(tbl)))
+
+ return res
+ end
+end
+
+--[[[
+-- @function lua_selectors.flatten_selectors(selectors)
+-- Convert selectors to a flat table of elements
+--]]
+exports.flatten_selectors = function(_, selectors, _)
+ local res = {}
+
+ local function fill(tbl)
+ for _, s in ipairs(tbl) do
+ if type(s) == 'string' then
+ rawset(res, #res + 1, s)
+ elseif type(s) == 'userdata' then
+ rawset(res, #res + 1, tostring(s))
+ else
+ fill(s)
+ end
+ end
+ end
+
+ fill(selectors)
+
+ return res
+end
+
+--[[[
+-- @function lua_selectors.kv_table_from_pairs(selectors)
+-- Convert selectors to a table where the odd elements are keys and even are elements
+-- Similarly to make a map from (k, v) pairs list
+-- To specify the concrete constant keys, one can use the `id` extractor
+--]]
+exports.kv_table_from_pairs = function(log_obj, selectors, _)
+ local res = {}
+ local rspamd_logger = require "rspamd_logger"
+
+ local function fill(tbl)
+ local tbl_len = #tbl
+ if tbl_len % 2 ~= 0 or tbl_len == 0 then
+ rspamd_logger.errx(log_obj, "invalid invocation of the `kv_table_from_pairs`: table length is invalid %s",
+ tbl_len)
+ return
+ end
+ for i = 1, tbl_len, 2 do
+ local k = tostring(tbl[i])
+ local v = tbl[i + 1]
+ if type(v) == 'string' then
+ res[k] = v
+ elseif type(v) == 'userdata' then
+ res[k] = tostring(v)
+ else
+ res[k] = fun.totable(fun.map(function(elt)
+ return tostring(elt)
+ end, v))
+ end
+ end
+ end
+
+ fill(selectors)
+
+ return res
+end
+
+
+--[[[
+-- @function lua_selectors.create_closure(log_obj, cfg, selector_str, delimiter, fn)
+-- Creates a closure from a string selector, using the specific combinator function
+--]]
+exports.create_selector_closure_fn = function(log_obj, cfg, selector_str, delimiter, fn)
+ local selector = exports.parse_selector(cfg, selector_str)
+
+ if not selector then
+ return nil
+ end
+
+ return function(task)
+ local res = exports.process_selectors(task, selector)
+
+ if res then
+ return fn(log_obj, res, delimiter)
+ end
+
+ return nil
+ end
+end
+
+--[[[
+-- @function lua_selectors.create_closure(cfg, selector_str, delimiter='', flatten=false)
+-- Creates a closure from a string selector
+--]]
+exports.create_selector_closure = function(cfg, selector_str, delimiter, flatten)
+ local combinator_fn = flatten and exports.flatten_selectors or exports.combine_selectors
+
+ return exports.create_selector_closure_fn(nil, cfg, selector_str, delimiter, combinator_fn)
+end
+
+local function display_selectors(tbl)
+ return fun.tomap(fun.map(function(k, v)
+ return k, fun.tomap(fun.filter(function(kk, vv)
+ return type(vv) ~= 'function'
+ end, v))
+ end, tbl))
+end
+
+exports.list_extractors = function()
+ return display_selectors(extractors)
+end
+
+exports.list_transforms = function()
+ return display_selectors(transform_function)
+end
+
+exports.add_map = function(name, map)
+ if not exports.maps[name] then
+ exports.maps[name] = map
+ else
+ logger.errx(rspamd_config, "duplicate map redefinition for the selectors: %s", name)
+ end
+end
+
+-- Publish log target
+exports.M = M
+
+return exports
diff --git a/lualib/lua_selectors/maps.lua b/lualib/lua_selectors/maps.lua
new file mode 100644
index 0000000..85b54a6
--- /dev/null
+++ b/lualib/lua_selectors/maps.lua
@@ -0,0 +1,19 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local maps = {} -- Shared within selectors, indexed by name
+
+return maps \ No newline at end of file
diff --git a/lualib/lua_selectors/transforms.lua b/lualib/lua_selectors/transforms.lua
new file mode 100644
index 0000000..6c6bc71
--- /dev/null
+++ b/lualib/lua_selectors/transforms.lua
@@ -0,0 +1,571 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local fun = require 'fun'
+local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local ts = require("tableshape").types
+local logger = require 'rspamd_logger'
+local common = require "lua_selectors/common"
+local M = "selectors"
+
+local maps = require "lua_selectors/maps"
+
+local function pure_type(ltype)
+ return ltype:match('^(.*)_list$')
+end
+
+local transform_function = {
+ -- Returns the lowercased string
+ ['lower'] = {
+ ['types'] = {
+ ['string'] = true,
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _)
+ return inp:lower(), 'string'
+ end,
+ ['description'] = 'Returns the lowercased string',
+ },
+ -- Returns the lowercased utf8 string
+ ['lower_utf8'] = {
+ ['types'] = {
+ ['string'] = true,
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, t)
+ return rspamd_util.lower_utf8(inp), t
+ end,
+ ['description'] = 'Returns the lowercased utf8 string',
+ },
+ -- Returns the first element
+ ['first'] = {
+ ['types'] = {
+ ['list'] = true,
+ },
+ ['process'] = function(inp, t)
+ return fun.head(inp), pure_type(t)
+ end,
+ ['description'] = 'Returns the first element',
+ },
+ -- Returns the last element
+ ['last'] = {
+ ['types'] = {
+ ['list'] = true,
+ },
+ ['process'] = function(inp, t)
+ return fun.nth(fun.length(inp), inp), pure_type(t)
+ end,
+ ['description'] = 'Returns the last element',
+ },
+ -- Returns the nth element
+ ['nth'] = {
+ ['types'] = {
+ ['list'] = true,
+ },
+ ['process'] = function(inp, t, args)
+ return fun.nth(args[1] or 1, inp), pure_type(t)
+ end,
+ ['description'] = 'Returns the nth element',
+ ['args_schema'] = { ts.number + ts.string / tonumber }
+ },
+ ['take_n'] = {
+ ['types'] = {
+ ['list'] = true,
+ },
+ ['process'] = function(inp, t, args)
+ return fun.take_n(args[1] or 1, inp), t
+ end,
+ ['description'] = 'Returns the n first elements',
+ ['args_schema'] = { ts.number + ts.string / tonumber }
+ },
+ ['drop_n'] = {
+ ['types'] = {
+ ['list'] = true,
+ },
+ ['process'] = function(inp, t, args)
+ return fun.drop_n(args[1] or 1, inp), t
+ end,
+ ['description'] = 'Returns list without the first n elements',
+ ['args_schema'] = { ts.number + ts.string / tonumber }
+ },
+ -- Joins strings into a single string using separator in the argument
+ ['join'] = {
+ ['types'] = {
+ ['string_list'] = true
+ },
+ ['process'] = function(inp, _, args)
+ return table.concat(fun.totable(inp), args[1] or ''), 'string'
+ end,
+ ['description'] = 'Joins strings into a single string using separator in the argument',
+ ['args_schema'] = { ts.string:is_optional() }
+ },
+ -- Joins strings into a set of strings using N elements and a separator in the argument
+ ['join_nth'] = {
+ ['types'] = {
+ ['string_list'] = true
+ },
+ ['process'] = function(inp, _, args)
+ local step = args[1]
+ local sep = args[2] or ''
+ local inp_t = fun.totable(inp)
+ local res = {}
+
+ for i = 1, #inp_t, step do
+ table.insert(res, table.concat(inp_t, sep, i, i + step))
+ end
+ return res, 'string_list'
+ end,
+ ['description'] = 'Joins strings into a set of strings using N elements and a separator in the argument',
+ ['args_schema'] = { ts.number + ts.string / tonumber, ts.string:is_optional() }
+ },
+ -- Joins tables into a table of strings
+ ['join_tables'] = {
+ ['types'] = {
+ ['list'] = true
+ },
+ ['process'] = function(inp, _, args)
+ local sep = args[1] or ''
+ return fun.map(function(t)
+ return table.concat(t, sep)
+ end, inp), 'string_list'
+ end,
+ ['description'] = 'Joins tables into a table of strings',
+ ['args_schema'] = { ts.string:is_optional() }
+ },
+ -- Sort strings
+ ['sort'] = {
+ ['types'] = {
+ ['list'] = true
+ },
+ ['process'] = function(inp, t, _)
+ table.sort(inp)
+ return inp, t
+ end,
+ ['description'] = 'Sort strings lexicographically',
+ },
+ -- Return unique elements based on hashing (can work without sorting)
+ ['uniq'] = {
+ ['types'] = {
+ ['list'] = true
+ },
+ ['process'] = function(inp, t, _)
+ local tmp = {}
+ fun.each(function(val)
+ tmp[val] = true
+ end, inp)
+
+ return fun.map(function(k, _)
+ return k
+ end, tmp), t
+ end,
+ ['description'] = 'Returns a list of unique elements (using a hash table)',
+ },
+ -- Create a digest from string or a list of strings
+ ['digest'] = {
+ ['types'] = {
+ ['string'] = true
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _, args)
+ return common.create_digest(inp, args), 'string'
+ end,
+ ['description'] = [[Create a digest from a string.
+The first argument is encoding (`hex`, `base32` (and forms `bleach32`, `rbase32`), `base64`),
+the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, `md5`)]],
+ ['args_schema'] = common.digest_schema()
+ },
+ -- Extracts substring
+ ['substring'] = {
+ ['types'] = {
+ ['string'] = true
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _, args)
+ local start_pos = args[1] or 1
+ local end_pos = args[2] or -1
+
+ return inp:sub(start_pos, end_pos), 'string'
+ end,
+ ['description'] = 'Extracts substring; the first argument is start, the second is the last (like in Lua)',
+ ['args_schema'] = { (ts.number + ts.string / tonumber):is_optional(),
+ (ts.number + ts.string / tonumber):is_optional() }
+ },
+ -- Prepends a string or a strings list
+ ['prepend'] = {
+ ['types'] = {
+ ['string'] = true
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _, args)
+ local prepend = table.concat(args, '')
+
+ return prepend .. inp, 'string'
+ end,
+ ['description'] = 'Prepends a string or a strings list',
+ },
+ -- Appends a string or a strings list
+ ['append'] = {
+ ['types'] = {
+ ['string'] = true
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _, args)
+ local append = table.concat(args, '')
+
+ return inp .. append, 'string'
+ end,
+ ['description'] = 'Appends a string or a strings list',
+ },
+ -- Regexp matching
+ ['regexp'] = {
+ ['types'] = {
+ ['string'] = true
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _, args)
+ local rspamd_regexp = require "rspamd_regexp"
+
+ local re = rspamd_regexp.create_cached(args[1])
+
+ if not re then
+ logger.errx('invalid regexp: %s', args[1])
+ return nil
+ end
+
+ local res = re:search(inp, false, true)
+
+ if res then
+ -- Map all results in a single list
+ local flattened_table = {}
+ local function flatten_table(tbl)
+ for _, v in ipairs(tbl) do
+ if type(v) == 'table' then
+ flatten_table(v)
+ else
+ table.insert(flattened_table, v)
+ end
+ end
+ end
+ flatten_table(res)
+ return flattened_table, 'string_list'
+ end
+
+ return nil
+ end,
+ ['description'] = 'Regexp matching, returns all matches flattened in a single list',
+ ['args_schema'] = { ts.string }
+ },
+ -- Returns a value if it exists in some map (or acts like a `filter` function)
+ ['filter_map'] = {
+ ['types'] = {
+ ['string'] = true
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, t, args)
+ local map = maps[args[1]]
+
+ if not map then
+ logger.errx('invalid map name: %s', args[1])
+ return nil
+ end
+
+ local res = map:get_key(inp)
+
+ if res then
+ return inp, t
+ end
+
+ return nil
+ end,
+ ['description'] = 'Returns a value if it exists in some map (or acts like a `filter` function)',
+ ['args_schema'] = { ts.string }
+ },
+ -- Returns a value if it exists in some map (or acts like a `filter` function)
+ ['except_map'] = {
+ ['types'] = {
+ ['string'] = true
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, t, args)
+ local map = maps[args[1]]
+
+ if not map then
+ logger.errx('invalid map name: %s', args[1])
+ return nil
+ end
+
+ local res = map:get_key(inp)
+
+ if not res then
+ return inp, t
+ end
+
+ return nil
+ end,
+ ['description'] = 'Returns a value if it does not exists in some map (or acts like a `except` function)',
+ ['args_schema'] = { ts.string }
+ },
+ -- Returns a value from some map corresponding to some key (or acts like a `map` function)
+ ['apply_map'] = {
+ ['types'] = {
+ ['string'] = true
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, t, args)
+ local map = maps[args[1]]
+
+ if not map then
+ logger.errx('invalid map name: %s', args[1])
+ return nil
+ end
+
+ local res = map:get_key(inp)
+
+ if res then
+ return res, t
+ end
+
+ return nil
+ end,
+ ['description'] = 'Returns a value from some map corresponding to some key (or acts like a `map` function)',
+ ['args_schema'] = { ts.string }
+ },
+ -- Drops input value and return values from function's arguments or an empty string
+ ['id'] = {
+ ['types'] = {
+ ['string'] = true,
+ ['list'] = true,
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(_, _, args)
+ if args[1] and args[2] then
+ return fun.map(tostring, args), 'string_list'
+ elseif args[1] then
+ return args[1], 'string'
+ end
+
+ return '', 'string'
+ end,
+ ['description'] = 'Drops input value and return values from function\'s arguments or an empty string',
+ ['args_schema'] = (ts.string + ts.array_of(ts.string)):is_optional()
+ },
+ ['equal'] = {
+ ['types'] = {
+ ['string'] = true,
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _, args)
+ if inp == args[1] then
+ return inp, 'string'
+ end
+
+ return nil
+ end,
+ ['description'] = [[Boolean function equal.
+Returns either nil or its argument if input is equal to argument]],
+ ['args_schema'] = { ts.string }
+ },
+ -- Boolean function in, returns either nil or its input if input is in args list
+ ['in'] = {
+ ['types'] = {
+ ['string'] = true,
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, t, args)
+ for _, a in ipairs(args) do
+ if a == inp then
+ return inp, t
+ end
+ end
+ return nil
+ end,
+ ['description'] = [[Boolean function in.
+Returns either nil or its input if input is in args list]],
+ ['args_schema'] = ts.array_of(ts.string)
+ },
+ ['not_in'] = {
+ ['types'] = {
+ ['string'] = true,
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, t, args)
+ for _, a in ipairs(args) do
+ if a == inp then
+ return nil
+ end
+ end
+ return inp, t
+ end,
+ ['description'] = [[Boolean function not in.
+Returns either nil or its input if input is not in args list]],
+ ['args_schema'] = ts.array_of(ts.string)
+ },
+ ['inverse'] = {
+ ['types'] = {
+ ['string'] = true,
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _, args)
+ if inp then
+ return nil
+ else
+ return (args[1] or 'true'), 'string'
+ end
+ end,
+ ['description'] = [[Inverses input.
+Empty string comes the first argument or 'true', non-empty string comes nil]],
+ ['args_schema'] = { ts.string:is_optional() }
+ },
+ ['ipmask'] = {
+ ['types'] = {
+ ['string'] = true,
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _, args)
+ local rspamd_ip = require "rspamd_ip"
+ -- Non optimal: convert string to an IP address
+ local ip = rspamd_ip.from_string(inp)
+
+ if not ip or not ip:is_valid() then
+ lua_util.debugm(M, "cannot convert %s to IP", inp)
+ return nil
+ end
+
+ if ip:get_version() == 4 then
+ local mask = tonumber(args[1])
+
+ return ip:apply_mask(mask):to_string(), 'string'
+ else
+ -- IPv6 takes the second argument or the first one...
+ local mask_str = args[2] or args[1]
+ local mask = tonumber(mask_str)
+
+ return ip:apply_mask(mask):to_string(), 'string'
+ end
+ end,
+ ['description'] = 'Applies mask to IP address.' ..
+ ' The first argument is the mask for IPv4 addresses, the second is the mask for IPv6 addresses.',
+ ['args_schema'] = { (ts.number + ts.string / tonumber),
+ (ts.number + ts.string / tonumber):is_optional() }
+ },
+ -- Returns the string(s) with all non ascii chars replaced
+ ['to_ascii'] = {
+ ['types'] = {
+ ['string'] = true,
+ ['list'] = true,
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _, args)
+ if type(inp) == 'table' then
+ return fun.map(
+ function(s)
+ return string.gsub(tostring(s), '[\128-\255]', args[1] or '?')
+ end, inp), 'string_list'
+ else
+ return string.gsub(tostring(inp), '[\128-\255]', '?'), 'string'
+ end
+ end,
+ ['description'] = 'Returns the string with all non-ascii bytes replaced with the character ' ..
+ 'given as second argument or `?`',
+ ['args_schema'] = { ts.string:is_optional() }
+ },
+ -- Extracts tld from a hostname
+ ['get_tld'] = {
+ ['types'] = {
+ ['string'] = true
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _, _)
+ return rspamd_util.get_tld(inp), 'string'
+ end,
+ ['description'] = 'Extracts tld from a hostname represented as a string',
+ ['args_schema'] = {}
+ },
+ -- Converts list of strings to numbers and returns a packed string
+ ['pack_numbers'] = {
+ ['types'] = {
+ ['string_list'] = true
+ },
+ ['map_type'] = 'string',
+ ['process'] = function(inp, _, args)
+ local fmt = args[1] or 'f'
+ local res = {}
+ for _, s in ipairs(inp) do
+ table.insert(res, tonumber(s))
+ end
+ return rspamd_util.pack(string.rep(fmt, #res), lua_util.unpack(res)), 'string'
+ end,
+ ['description'] = 'Converts a list of strings to numbers & returns a packed string',
+ ['args_schema'] = { ts.string:is_optional() }
+ },
+ -- Filter nils from a list
+ ['filter_string_nils'] = {
+ ['types'] = {
+ ['string_list'] = true
+ },
+ ['process'] = function(inp, _, _)
+ return fun.filter(function(val)
+ return type(val) == 'string' and val ~= 'nil'
+ end, inp), 'string_list'
+ end,
+ ['description'] = 'Removes all nils from a list of strings (when converted implicitly)',
+ ['args_schema'] = {}
+ },
+ -- Call a set of methods on a userdata object
+ ['apply_methods'] = {
+ ['types'] = {
+ ['userdata'] = true,
+ },
+ ['process'] = function(inp, _, args)
+ local res = {}
+ for _, arg in ipairs(args) do
+ local meth = inp[arg]
+ local ret = meth(inp)
+ if ret then
+ table.insert(res, tostring(ret))
+ end
+ end
+ return res, 'string_list'
+ end,
+ ['description'] = 'Apply a list of method calls to the userdata object',
+ },
+ -- Apply method to list of userdata and use it as a filter, excluding elements for which method returns false/nil
+ ['filter_method'] = {
+ ['types'] = {
+ ['userdata_list'] = true
+ },
+ ['process'] = function(inp, t, args)
+ local meth = args[1]
+
+ if not meth then
+ logger.errx('invalid method name: %s', args[1])
+ return nil
+ end
+
+ return fun.filter(function(val)
+ return val[meth](val)
+ end, inp), 'userdata_list'
+ end,
+ ['description'] = 'Apply method to list of userdata and use it as a filter,' ..
+ ' excluding elements for which method returns false/nil',
+ ['args_schema'] = { ts.string }
+ },
+}
+
+transform_function.match = transform_function.regexp
+
+return transform_function
diff --git a/lualib/lua_settings.lua b/lualib/lua_settings.lua
new file mode 100644
index 0000000..d6d24d6
--- /dev/null
+++ b/lualib/lua_settings.lua
@@ -0,0 +1,309 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_settings
+-- This module contains internal helpers for the settings infrastructure in Rspamd
+-- More details at https://rspamd.com/doc/configuration/settings.html
+--]]
+
+local exports = {}
+local known_ids = {}
+local post_init_added = false
+local post_init_performed = false
+local all_symbols
+local default_symbols
+
+local fun = require "fun"
+local lua_util = require "lua_util"
+local rspamd_logger = require "rspamd_logger"
+
+local function register_settings_cb(from_postload)
+ if not post_init_performed then
+ all_symbols = rspamd_config:get_symbols()
+
+ default_symbols = fun.totable(fun.filter(function(_, v)
+ return not v.allowed_ids or #v.allowed_ids == 0 or v.flags.explicit_disable
+ end, all_symbols))
+
+ local explicit_symbols = lua_util.keys(fun.filter(function(k, v)
+ return v.flags.explicit_disable
+ end, all_symbols))
+
+ local symnames = lua_util.list_to_hash(lua_util.keys(all_symbols))
+
+ for _, set in pairs(known_ids) do
+ local s = set.settings.apply or {}
+ set.symbols = lua_util.shallowcopy(symnames)
+ local enabled_symbols = {}
+ local seen_enabled = false
+ local disabled_symbols = {}
+ local seen_disabled = false
+
+ -- Enabled map
+ if s.symbols_enabled then
+ -- Remove all symbols from set.symbols aside of explicit_disable symbols
+ set.symbols = lua_util.list_to_hash(explicit_symbols)
+ seen_enabled = true
+ for _, sym in ipairs(s.symbols_enabled) do
+ enabled_symbols[sym] = true
+ set.symbols[sym] = true
+ end
+ end
+ if s.groups_enabled then
+ seen_enabled = true
+ for _, gr in ipairs(s.groups_enabled) do
+ local syms = rspamd_config:get_group_symbols(gr)
+
+ if syms then
+ for _, sym in ipairs(syms) do
+ enabled_symbols[sym] = true
+ set.symbols[sym] = true
+ end
+ end
+ end
+ end
+
+ -- Disabled map
+ if s.symbols_disabled then
+ seen_disabled = true
+ for _, sym in ipairs(s.symbols_disabled) do
+ disabled_symbols[sym] = true
+ set.symbols[sym] = false
+ end
+ end
+ if s.groups_disabled then
+ seen_disabled = true
+ for _, gr in ipairs(s.groups_disabled) do
+ local syms = rspamd_config:get_group_symbols(gr)
+
+ if syms then
+ for _, sym in ipairs(syms) do
+ disabled_symbols[sym] = true
+ set.symbols[sym] = false
+ end
+ end
+ end
+ end
+
+ -- Deal with complexity to avoid mess in C
+ if not seen_enabled then
+ enabled_symbols = nil
+ end
+ if not seen_disabled then
+ disabled_symbols = nil
+ end
+
+ if enabled_symbols or disabled_symbols then
+ -- Specify what symbols are really enabled for this settings id
+ set.has_specific_symbols = true
+ end
+
+ rspamd_config:register_settings_id(set.name, enabled_symbols, disabled_symbols)
+
+ -- Remove to avoid clash
+ s.symbols_disabled = nil
+ s.symbols_enabled = nil
+ s.groups_enabled = nil
+ s.groups_disabled = nil
+ end
+
+ -- We now iterate over all symbols and check for allowed_ids/forbidden_ids
+ for k, v in pairs(all_symbols) do
+ if v.allowed_ids and not v.flags.explicit_disable then
+ for _, id in ipairs(v.allowed_ids) do
+ if known_ids[id] then
+ local set = known_ids[id]
+ if not set.has_specific_symbols then
+ set.has_specific_symbols = true
+ end
+ set.symbols[k] = true
+ else
+ rspamd_logger.errx(rspamd_config, 'symbol %s is allowed at unknown settings id %s',
+ k, id)
+ end
+ end
+ end
+ if v.forbidden_ids then
+ for _, id in ipairs(v.forbidden_ids) do
+ if known_ids[id] then
+ local set = known_ids[id]
+ if not set.has_specific_symbols then
+ set.has_specific_symbols = true
+ end
+ set.symbols[k] = false
+ else
+ rspamd_logger.errx(rspamd_config, 'symbol %s is denied at unknown settings id %s',
+ k, id)
+ end
+ end
+ end
+ end
+
+ -- Now we create lists of symbols for each settings and digest
+ for _, set in pairs(known_ids) do
+ set.symbols = lua_util.keys(fun.filter(function(_, v)
+ return v
+ end, set.symbols))
+ table.sort(set.symbols)
+ set.digest = lua_util.table_digest(set.symbols)
+ end
+
+ post_init_performed = true
+ end
+end
+
+-- Returns numeric representation of the settings id
+local function numeric_settings_id(str)
+ local cr = require "rspamd_cryptobox_hash"
+ local util = require "rspamd_util"
+ local ret = util.unpack("I4",
+ cr.create_specific('xxh64'):update(str):bin())
+
+ return ret
+end
+
+exports.numeric_settings_id = numeric_settings_id
+
+-- Used to do the following:
+-- If there is a group of symbols_allowed, it checks if that is an array
+-- If that is a hash table then we transform it to a normal list, probably adding symbols to adjust scores
+local function transform_settings_maybe(settings, name)
+ if settings.apply then
+ local apply = settings.apply
+
+ if apply.symbols_enabled then
+ local senabled = apply.symbols_enabled
+
+ if not senabled[1] then
+ -- Transform map to a list
+ local nlist = {}
+ if not apply.scores then
+ apply.scores = {}
+ end
+ for k, v in pairs(senabled) do
+ if tonumber(v) then
+ -- Move to symbols as well
+ apply.scores[k] = tonumber(v)
+ lua_util.debugm('settings', rspamd_config,
+ 'set symbol %s -> %s for settings %s', k, v, name)
+ end
+ nlist[#nlist + 1] = k
+ end
+ -- Convert
+ apply.symbols_enabled = nlist
+ end
+
+ local symhash = lua_util.list_to_hash(apply.symbols_enabled)
+
+ if apply.symbols then
+ -- Check if added symbols are enabled
+ for k, v in pairs(apply.symbols) do
+ local s
+ -- Check if we have ["sym1", "sym2" ...] or {"sym1": xx, "sym2": yy}
+ if type(k) == 'string' then
+ s = k
+ else
+ s = v
+ end
+ if not symhash[s] then
+ lua_util.debugm('settings', rspamd_config,
+ 'added symbol %s to symbols_enabled for %s', s, name)
+ apply.symbols_enabled[#apply.symbols_enabled + 1] = s
+ end
+ end
+ end
+ end
+ end
+
+ return settings
+end
+
+local function register_settings_id(str, settings, from_postload)
+ local numeric_id = numeric_settings_id(str)
+
+ if known_ids[numeric_id] then
+ -- Might be either rewrite or a collision
+ if known_ids[numeric_id].name ~= str then
+ local logger = require "rspamd_logger"
+
+ logger.errx(rspamd_config, 'settings ID clash! id %s maps to %s and conflicts with %s',
+ numeric_id, known_ids[numeric_id].name, str)
+
+ return nil
+ end
+ else
+ known_ids[numeric_id] = {
+ name = str,
+ id = numeric_id,
+ settings = transform_settings_maybe(settings, str),
+ symbols = {}
+ }
+ end
+
+ if not from_postload and not post_init_added then
+ -- Use high priority to ensure that settings are initialised early but not before all
+ -- plugins are loaded
+ rspamd_config:add_post_init(function()
+ register_settings_cb(true)
+ end, 150)
+ rspamd_config:add_config_unload(function()
+ if post_init_added then
+ known_ids = {}
+ post_init_added = false
+ end
+ post_init_performed = false
+ end)
+
+ post_init_added = true
+ end
+
+ return numeric_id
+end
+
+exports.register_settings_id = register_settings_id
+
+local function settings_by_id(id)
+ if not post_init_performed then
+ register_settings_cb(false)
+ end
+ return known_ids[id]
+end
+
+exports.settings_by_id = settings_by_id
+exports.all_settings = function()
+ if not post_init_performed then
+ register_settings_cb(false)
+ end
+ return known_ids
+end
+exports.all_symbols = function()
+ if not post_init_performed then
+ register_settings_cb(false)
+ end
+ return all_symbols
+end
+-- What is enabled when no settings are there
+exports.default_symbols = function()
+ if not post_init_performed then
+ register_settings_cb(false)
+ end
+ return default_symbols
+end
+
+exports.load_all_settings = register_settings_cb
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_smtp.lua b/lualib/lua_smtp.lua
new file mode 100644
index 0000000..3c40349
--- /dev/null
+++ b/lualib/lua_smtp.lua
@@ -0,0 +1,201 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local rspamd_tcp = require "rspamd_tcp"
+local lua_util = require "lua_util"
+
+local exports = {}
+
+local CRLF = '\r\n'
+local default_timeout = 10.0
+
+--[[[
+-- @function lua_smtp.sendmail(task, message, opts, callback)
+--]]
+local function sendmail(opts, message, callback)
+ local stage = 'connect'
+
+ local function mail_cb(err, data, conn)
+ local function no_error_write(merr)
+ if merr then
+ callback(false, string.format('error on stage %s: %s',
+ stage, merr))
+ if conn then
+ conn:close()
+ end
+
+ return false
+ end
+
+ return true
+ end
+
+ local function no_error_read(merr, mdata, wantcode)
+ wantcode = wantcode or '2'
+ if merr then
+ callback(false, string.format('error on stage %s: %s',
+ stage, merr))
+ if conn then
+ conn:close()
+ end
+
+ return false
+ end
+ if mdata then
+ if type(mdata) ~= 'string' then
+ mdata = tostring(mdata)
+ end
+ if string.sub(mdata, 1, 1) ~= wantcode then
+ callback(false, string.format('bad smtp response on stage %s: "%s" when "%s" expected',
+ stage, mdata, wantcode))
+ if conn then
+ conn:close()
+ end
+ return false
+ end
+ else
+ callback(false, string.format('no data on stage %s',
+ stage))
+ if conn then
+ conn:close()
+ end
+ return false
+ end
+ return true
+ end
+
+ -- After quit
+ local function all_done_cb(merr, mdata)
+ if conn then
+ conn:close()
+ end
+
+ callback(true, nil)
+
+ return true
+ end
+
+ -- QUIT stage
+ local function quit_done_cb(_, _)
+ conn:add_read(all_done_cb, CRLF)
+ end
+ local function quit_cb(merr, mdata)
+ if no_error_read(merr, mdata) then
+ conn:add_write(quit_done_cb, 'QUIT' .. CRLF)
+ end
+ end
+ local function pre_quit_cb(merr, _)
+ if no_error_write(merr) then
+ stage = 'quit'
+ conn:add_read(quit_cb, CRLF)
+ end
+ end
+
+ -- DATA stage
+ local function data_done_cb(merr, mdata)
+ if no_error_read(merr, mdata, '3') then
+ if type(message) == 'string' or type(message) == 'userdata' then
+ conn:add_write(pre_quit_cb, { message, CRLF .. '.' .. CRLF })
+ else
+ table.insert(message, CRLF .. '.' .. CRLF)
+ conn:add_write(pre_quit_cb, message)
+ end
+ end
+ end
+ local function data_cb(merr, _)
+ if no_error_write(merr) then
+ conn:add_read(data_done_cb, CRLF)
+ end
+ end
+
+ -- RCPT phase
+ local next_recipient
+ local function rcpt_done_cb_gen(i)
+ return function(merr, mdata)
+ if no_error_read(merr, mdata) then
+ if i == #opts.recipients then
+ conn:add_write(data_cb, 'DATA' .. CRLF)
+ else
+ next_recipient(i + 1)
+ end
+ end
+ end
+ end
+
+ local function rcpt_cb_gen(i)
+ return function(merr, _)
+ if no_error_write(merr, '2') then
+ conn:add_read(rcpt_done_cb_gen(i), CRLF)
+ end
+ end
+ end
+
+ next_recipient = function(i)
+ conn:add_write(rcpt_cb_gen(i),
+ string.format('RCPT TO: <%s>%s', opts.recipients[i], CRLF))
+ end
+
+ -- FROM stage
+ local function from_done_cb(merr, mdata)
+ -- We need to iterate over recipients sequentially
+ if no_error_read(merr, mdata, '2') then
+ stage = 'rcpt'
+ next_recipient(1)
+ end
+ end
+ local function from_cb(merr, _)
+ if no_error_write(merr) then
+ conn:add_read(from_done_cb, CRLF)
+ end
+ end
+ local function hello_done_cb(merr, mdata)
+ if no_error_read(merr, mdata) then
+ stage = 'from'
+ conn:add_write(from_cb, string.format(
+ 'MAIL FROM: <%s>%s', opts.from, CRLF))
+ end
+ end
+
+ -- HELO stage
+ local function hello_cb(merr)
+ if no_error_write(merr) then
+ conn:add_read(hello_done_cb, CRLF)
+ end
+ end
+ if no_error_read(err, data) then
+ stage = 'helo'
+ conn:add_write(hello_cb, string.format('HELO %s%s',
+ opts.helo, CRLF))
+ end
+ end
+
+ if type(opts.recipients) == 'string' then
+ opts.recipients = { opts.recipients }
+ end
+
+ local tcp_opts = lua_util.shallowcopy(opts)
+ tcp_opts.stop_pattern = CRLF
+ tcp_opts.timeout = opts.timeout or default_timeout
+ tcp_opts.callback = mail_cb
+
+ if not rspamd_tcp.request(tcp_opts) then
+ callback(false, 'cannot make a TCP connection')
+ end
+end
+
+exports.sendmail = sendmail
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_stat.lua b/lualib/lua_stat.lua
new file mode 100644
index 0000000..a0f3303
--- /dev/null
+++ b/lualib/lua_stat.lua
@@ -0,0 +1,869 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_stat
+-- This module contains helper functions for supporting statistics
+--]]
+
+local logger = require "rspamd_logger"
+local sqlite3 = require "rspamd_sqlite3"
+local util = require "rspamd_util"
+local lua_redis = require "lua_redis"
+local lua_util = require "lua_util"
+local exports = {}
+
+local N = "stat_tools" -- luacheck: ignore (maybe unused)
+
+-- Performs synchronous conversion of redis schema
+local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expire)
+
+ -- Old schema is the following one:
+ -- Keys are named <symbol>[<user>]
+ -- Elements are placed within hash:
+ -- BAYES_SPAM -> {<id1>: <num_hits>, <id2>: <num_hits> ...}
+ -- In new schema it is changed to a more extensible schema:
+ -- Keys are named RS[<user>]_<id> -> {'H': <ham_hits>, 'S': <spam_hits>}
+ -- So we can expire individual records, measure most popular elements by zranges,
+ -- add new fields, such as tokens etc
+
+ local res, conn = lua_redis.redis_connect_sync(redis_params, true)
+
+ if not res then
+ logger.errx("cannot connect to redis server")
+ return false
+ end
+
+ -- KEYS[1]: key to check (e.g. 'BAYES_SPAM')
+ -- KEYS[2]: hash key ('S' or 'H')
+ -- KEYS[3]: expire
+ local lua_script = [[
+redis.replicate_commands()
+local keys = redis.call('SMEMBERS', KEYS[1]..'_keys')
+local nconverted = 0
+for _,k in ipairs(keys) do
+ local cursor = redis.call('HSCAN', k, 0)
+ local neutral_prefix = string.gsub(k, KEYS[1], 'RS')
+ local elts
+ while cursor[1] ~= "0" do
+ elts = cursor[2]
+ cursor = redis.call('HSCAN', k, cursor[1])
+ local real_key
+ for i,v in ipairs(elts) do
+ if i % 2 ~= 0 then
+ real_key = v
+ else
+ local nkey = string.format('%s_%s', neutral_prefix, real_key)
+ redis.call('HSET', nkey, KEYS[2], v)
+ if KEYS[3] and tonumber(KEYS[3]) > 0 then
+ redis.call('EXPIRE', nkey, KEYS[3])
+ end
+ nconverted = nconverted + 1
+ end
+ end
+ end
+end
+return nconverted
+]]
+
+ conn:add_cmd('EVAL', { lua_script, '3', symbol_spam, 'S', tostring(expire) })
+ local ret
+ ret, res = conn:exec()
+
+ if not ret then
+ logger.errx('error converting symbol %s: %s', symbol_spam, res)
+ return false
+ else
+ logger.messagex('converted %s elements from symbol %s', res, symbol_spam)
+ end
+
+ conn:add_cmd('EVAL', { lua_script, '3', symbol_ham, 'H', tostring(expire) })
+ ret, res = conn:exec()
+
+ if not ret then
+ logger.errx('error converting symbol %s: %s', symbol_ham, res)
+ return false
+ else
+ logger.messagex('converted %s elements from symbol %s', res, symbol_ham)
+ end
+
+ -- We can now convert metadata: set + learned + version
+ -- KEYS[1]: key to check (e.g. 'BAYES_SPAM')
+ -- KEYS[2]: learn key (e.g. 'learns_spam' or 'learns_ham')
+ lua_script = [[
+local keys = redis.call('SMEMBERS', KEYS[1]..'_keys')
+
+for _,k in ipairs(keys) do
+ local learns = redis.call('HGET', k, 'learns') or 0
+ local neutral_prefix = string.gsub(k, KEYS[1], 'RS')
+
+ redis.call('HSET', neutral_prefix, KEYS[2], learns)
+ redis.call('SADD', KEYS[1]..'_keys', neutral_prefix)
+ redis.call('SREM', KEYS[1]..'_keys', k)
+ redis.call('DEL', KEYS[1])
+ redis.call('SET', k ..'_version', '2')
+end
+]]
+
+ conn:add_cmd('EVAL', { lua_script, '2', symbol_spam, 'learns_spam' })
+ ret, res = conn:exec()
+
+ if not ret then
+ logger.errx('error converting metadata for symbol %s: %s', symbol_spam, res)
+ return false
+ end
+
+ conn:add_cmd('EVAL', { lua_script, '2', symbol_ham, 'learns_ham' })
+ ret, res = conn:exec()
+
+ if not ret then
+ logger.errx('error converting metadata for symbol %s', symbol_ham, res)
+ return false
+ end
+
+ return true
+end
+
+exports.convert_bayes_schema = convert_bayes_schema
+
+-- It now accepts both ham and spam databases
+-- parameters:
+-- redis_params - how do we connect to a redis server
+-- sqlite_db_spam - name for sqlite database with spam tokens
+-- sqlite_db_ham - name for sqlite database with ham tokens
+-- symbol_ham - name for symbol representing spam, e.g. BAYES_SPAM
+-- symbol_spam - name for symbol representing ham, e.g. BAYES_HAM
+-- learn_cache_spam - name for sqlite database with spam learn cache
+-- learn_cache_ham - name for sqlite database with ham learn cache
+-- reset_previous - if true, then the old database is flushed (slow)
+local function convert_sqlite_to_redis(redis_params,
+ sqlite_db_spam, sqlite_db_ham, symbol_spam, symbol_ham,
+ learn_cache_db, expire, reset_previous)
+ local nusers = 0
+ local lim = 1000 -- Update each 1000 tokens
+ local users_map = {}
+ local converted = 0
+
+ local db_spam = sqlite3.open(sqlite_db_spam)
+ if not db_spam then
+ logger.errx('Cannot open source db: %s', sqlite_db_spam)
+ return false
+ end
+ local db_ham = sqlite3.open(sqlite_db_ham)
+ if not db_ham then
+ logger.errx('Cannot open source db: %s', sqlite_db_ham)
+ return false
+ end
+
+ local res, conn = lua_redis.redis_connect_sync(redis_params, true)
+
+ if not res then
+ logger.errx("cannot connect to redis server")
+ return false
+ end
+
+ if reset_previous then
+ -- Do a more complicated cleanup
+ -- execute a lua script that cleans up data
+ local script = [[
+local members = redis.call('SMEMBERS', KEYS[1]..'_keys')
+
+for _,prefix in ipairs(members) do
+ local keys = redis.call('KEYS', prefix..'*')
+ redis.call('DEL', keys)
+end
+]]
+ -- Common keys
+ for _, sym in ipairs({ symbol_spam, symbol_ham }) do
+ logger.messagex('Cleaning up old data for %s', sym)
+ conn:add_cmd('EVAL', { script, '1', sym })
+ conn:exec()
+ conn:add_cmd('DEL', { sym .. "_version" })
+ conn:add_cmd('DEL', { sym .. "_keys" })
+ conn:exec()
+ end
+
+ if learn_cache_db then
+ -- Cleanup learned_cache
+ logger.messagex('Cleaning up old data learned cache')
+ conn:add_cmd('DEL', { "learned_ids" })
+ conn:exec()
+ end
+ end
+
+ local function convert_db(db, is_spam)
+ -- Map users and languages
+ local what = 'ham'
+ if is_spam then
+ what = 'spam'
+ end
+
+ local learns = {}
+ db:sql('BEGIN;')
+ -- Fill users mapping
+ for row in db:rows('SELECT * FROM users;') do
+ if row.id == '0' then
+ users_map[row.id] = ''
+ else
+ users_map[row.id] = row.name
+ end
+ learns[row.id] = row.learns
+ nusers = nusers + 1
+ end
+
+ -- Workaround for old databases
+ for row in db:rows('SELECT * FROM languages') do
+ if learns['0'] then
+ learns['0'] = learns['0'] + row.learns
+ else
+ learns['0'] = row.learns
+ end
+ end
+
+ local function send_batch(tokens, prefix)
+ -- We use the new schema: RS[user]_token -> H=ham count
+ -- S=spam count
+ local hash_key = 'H'
+ if is_spam then
+ hash_key = 'S'
+ end
+ for _, tok in ipairs(tokens) do
+ -- tok schema:
+ -- tok[1] = token_id (uint64 represented as a string)
+ -- tok[2] = token value (number)
+ -- tok[3] = user_map[user_id] or ''
+ local rkey = string.format('%s%s_%s', prefix, tok[3], tok[1])
+ conn:add_cmd('HINCRBYFLOAT', { rkey, hash_key, tostring(tok[2]) })
+
+ if expire and expire ~= 0 then
+ conn:add_cmd('EXPIRE', { rkey, tostring(expire) })
+ end
+ end
+
+ return conn:exec()
+ end
+ -- Fill tokens, sending data to redis each `lim` records
+
+ local ntokens = db:query('SELECT count(*) as c FROM tokens')['c']
+ local tokens = {}
+ local num = 0
+ local total = 0
+
+ for row in db:rows('SELECT token,value,user FROM tokens;') do
+ local user = ''
+ if row.user ~= 0 and users_map[row.user] then
+ user = users_map[row.user]
+ end
+
+ table.insert(tokens, { row.token, row.value, user })
+ num = num + 1
+ total = total + 1
+ if num > lim then
+ -- TODO: we use the default 'RS' prefix, it can be false in case of
+ -- classifiers with labels
+ local ret, err_str = send_batch(tokens, 'RS')
+ if not ret then
+ logger.errx('Cannot send tokens to the redis server: ' .. err_str)
+ db:sql('COMMIT;')
+ return false
+ end
+
+ num = 0
+ tokens = {}
+ end
+
+ io.write(string.format('Processed batch %s: %s/%s\r', what, total, ntokens))
+ end
+ -- Last batch
+ if #tokens > 0 then
+ local ret, err_str = send_batch(tokens, 'RS')
+ if not ret then
+ logger.errx('Cannot send tokens to the redis server: ' .. err_str)
+ db:sql('COMMIT;')
+ return false
+ end
+
+ io.write(string.format('Processed batch %s: %s/%s\r', what, total, ntokens))
+ end
+ io.write('\n')
+
+ converted = converted + total
+
+ -- Close DB
+ db:sql('COMMIT;')
+ local symbol = symbol_ham
+ local learns_elt = "learns_ham"
+
+ if is_spam then
+ symbol = symbol_spam
+ learns_elt = "learns_spam"
+ end
+
+ for id, learned in pairs(learns) do
+ local user = users_map[id]
+ if not conn:add_cmd('HSET', { 'RS' .. user, learns_elt, learned }) then
+ logger.errx('Cannot update learns for user: ' .. user)
+ return false
+ end
+ if not conn:add_cmd('SADD', { symbol .. '_keys', 'RS' .. user }) then
+ logger.errx('Cannot update learns for user: ' .. user)
+ return false
+ end
+ end
+ -- Set version
+ conn:add_cmd('SET', { symbol .. '_version', '2' })
+ return conn:exec()
+ end
+
+ logger.messagex('Convert spam tokens')
+ if not convert_db(db_spam, true) then
+ return false
+ end
+
+ logger.messagex('Convert ham tokens')
+ if not convert_db(db_ham, false) then
+ return false
+ end
+
+ if learn_cache_db then
+ logger.messagex('Convert learned ids from %s', learn_cache_db)
+ local db = sqlite3.open(learn_cache_db)
+ local ret = true
+ local total = 0
+
+ if not db then
+ logger.errx('Cannot open cache database: ' .. learn_cache_db)
+ return false
+ end
+
+ db:sql('BEGIN;')
+
+ for row in db:rows('SELECT * FROM learns;') do
+ local is_spam
+ local digest = tostring(util.encode_base32(row.digest))
+
+ if row.flag == '0' then
+ is_spam = '-1'
+ else
+ is_spam = '1'
+ end
+
+ if not conn:add_cmd('HSET', { 'learned_ids', digest, is_spam }) then
+ logger.errx('Cannot add hash: ' .. digest)
+ ret = false
+ else
+ total = total + 1
+ end
+ end
+ db:sql('COMMIT;')
+
+ if ret then
+ conn:exec()
+ end
+
+ if ret then
+ logger.messagex('Converted %s cached items from sqlite3 learned cache to redis',
+ total)
+ else
+ logger.errx('Error occurred during sending data to redis')
+ end
+ end
+
+ logger.messagex('Migrated %s tokens for %s users for symbols (%s, %s)',
+ converted, nusers, symbol_spam, symbol_ham)
+ return true
+end
+
+exports.convert_sqlite_to_redis = convert_sqlite_to_redis
+
+-- Loads sqlite3 based classifiers and output data in form of array of objects:
+-- [
+-- {
+-- symbol_spam = XXX
+-- symbol_ham = YYY
+-- db_spam = XXX.sqlite
+-- db_ham = YYY.sqlite
+-- learn_cache = ZZZ.sqlite
+-- per_user = true/false
+-- label = str
+-- }
+-- ]
+local function load_sqlite_config(cfg)
+ local result = {}
+
+ local function parse_classifier(cls)
+ local tbl = {}
+ if cls.cache then
+ local cache = cls.cache
+ if cache.type == 'sqlite3' and (cache.file or cache.path) then
+ tbl.learn_cache = (cache.file or cache.path)
+ end
+ end
+
+ if cls.per_user then
+ tbl.per_user = cls.per_user
+ end
+
+ if cls.label then
+ tbl.label = cls.label
+ end
+
+ local statfiles = cls.statfile
+ for _, stf in ipairs(statfiles) do
+ local path = (stf.file or stf.path or stf.db or stf.dbname)
+ local symbol = stf.symbol or 'undefined'
+
+ if not path then
+ logger.errx('no path defined for statfile %s', symbol)
+ else
+
+ local spam
+ if stf.spam then
+ spam = stf.spam
+ else
+ if string.match(symbol:upper(), 'SPAM') then
+ spam = true
+ else
+ spam = false
+ end
+ end
+
+ if spam then
+ tbl.symbol_spam = symbol
+ tbl.db_spam = path
+ else
+ tbl.symbol_ham = symbol
+ tbl.db_ham = path
+ end
+ end
+ end
+
+ if tbl.symbol_spam and tbl.symbol_ham and tbl.db_ham and tbl.db_spam then
+ table.insert(result, tbl)
+ end
+ end
+
+ local classifier = cfg.classifier
+
+ if classifier then
+ if classifier[1] then
+ for _, cls in ipairs(classifier) do
+ if cls.bayes then
+ cls = cls.bayes
+ end
+ if cls.backend and cls.backend == 'sqlite3' then
+ parse_classifier(cls)
+ end
+ end
+ else
+ if classifier.bayes then
+ classifier = classifier.bayes
+ if classifier[1] then
+ for _, cls in ipairs(classifier) do
+ if cls.backend and cls.backend == 'sqlite3' then
+ parse_classifier(cls)
+ end
+ end
+ else
+ if classifier.backend and classifier.backend == 'sqlite3' then
+ parse_classifier(classifier)
+ end
+ end
+ end
+ end
+ end
+
+ return result
+end
+
+exports.load_sqlite_config = load_sqlite_config
+
+-- A helper method that suggests a user how to configure Redis based
+-- classifier based on the existing sqlite classifier
+local function redis_classifier_from_sqlite(sqlite_classifier, expire)
+ local result = {
+ new_schema = true,
+ backend = 'redis',
+ cache = {
+ backend = 'redis'
+ },
+ statfile = {
+ [sqlite_classifier.symbol_spam] = {
+ spam = true
+ },
+ [sqlite_classifier.symbol_ham] = {
+ spam = false
+ }
+ }
+ }
+
+ if expire then
+ result.expire = expire
+ end
+
+ return { classifier = { bayes = result } }
+end
+
+exports.redis_classifier_from_sqlite = redis_classifier_from_sqlite
+
+-- Reads statistics config and return preprocessed table
+local function process_stat_config(cfg)
+ local opts_section = cfg:get_all_opt('options') or {}
+
+ -- Check if we have a dedicated section for statistics
+ if opts_section.statistics then
+ opts_section = opts_section.statistics
+ end
+
+ -- Default
+ local res_config = {
+ classify_headers = {
+ "User-Agent",
+ "X-Mailer",
+ "Content-Type",
+ "X-MimeOLE",
+ "Organization",
+ "Organisation"
+ },
+ classify_images = true,
+ classify_mime_info = true,
+ classify_urls = true,
+ classify_meta = true,
+ classify_max_tlds = 10,
+ }
+
+ res_config = lua_util.override_defaults(res_config, opts_section)
+
+ -- Postprocess classify_headers
+ local classify_headers_parsed = {}
+
+ for _, v in ipairs(res_config.classify_headers) do
+ local s1, s2 = v:match("^([A-Z])[^%-]+%-([A-Z]).*$")
+
+ local hname
+ if s1 and s2 then
+ hname = string.format('%s-%s', s1, s2)
+ else
+ s1 = v:match("^X%-([A-Z].*)$")
+
+ if s1 then
+ hname = string.format('x%s', s1:sub(1, 3):lower())
+ else
+ hname = string.format('%s', v:sub(1, 3):lower())
+ end
+ end
+
+ if classify_headers_parsed[hname] then
+ table.insert(classify_headers_parsed[hname], v)
+ else
+ classify_headers_parsed[hname] = { v }
+ end
+ end
+
+ res_config.classify_headers_parsed = classify_headers_parsed
+
+ return res_config
+end
+
+local function get_mime_stat_tokens(task, res, i)
+ local parts = task:get_parts() or {}
+ local seen_multipart = false
+ local seen_plain = false
+ local seen_html = false
+ local empty_plain = false
+ local empty_html = false
+ local online_text = false
+
+ for _, part in ipairs(parts) do
+ local fname = part:get_filename()
+
+ local sz = part:get_length()
+
+ if sz > 0 then
+ rawset(res, i, string.format("#ps:%d",
+ math.floor(math.log(sz))))
+ lua_util.debugm("bayes", task, "part size: %s",
+ res[i])
+ i = i + 1
+ end
+
+ if fname then
+ rawset(res, i, "#f:" .. fname)
+ i = i + 1
+
+ lua_util.debugm("bayes", task, "added attachment: #f:%s",
+ fname)
+ end
+
+ if part:is_text() then
+ local tp = part:get_text()
+
+ if tp:is_html() then
+ seen_html = true
+
+ if tp:get_length() == 0 then
+ empty_html = true
+ end
+ else
+ seen_plain = true
+
+ if tp:get_length() == 0 then
+ empty_plain = true
+ end
+ end
+
+ if tp:get_lines_count() < 2 then
+ online_text = true
+ end
+
+ rawset(res, i, "#lang:" .. (tp:get_language() or 'unk'))
+ lua_util.debugm("bayes", task, "added language: %s",
+ res[i])
+ i = i + 1
+
+ rawset(res, i, "#cs:" .. (tp:get_charset() or 'unk'))
+ lua_util.debugm("bayes", task, "added charset: %s",
+ res[i])
+ i = i + 1
+
+ elseif part:is_multipart() then
+ seen_multipart = true;
+ end
+ end
+
+ -- Create a special token depending on parts structure
+ local st_tok = "#unk"
+ if seen_multipart and seen_html and seen_plain then
+ st_tok = '#mpth'
+ end
+
+ if seen_html and not seen_plain then
+ st_tok = "#ho"
+ end
+
+ if seen_plain and not seen_html then
+ st_tok = "#to"
+ end
+
+ local spec_tok = ""
+ if online_text then
+ spec_tok = "#ot"
+ end
+
+ if empty_plain then
+ spec_tok = spec_tok .. "#ep"
+ end
+
+ if empty_html then
+ spec_tok = spec_tok .. "#eh"
+ end
+
+ rawset(res, i, string.format("#m:%s%s", st_tok, spec_tok))
+ lua_util.debugm("bayes", task, "added mime token: %s",
+ res[i])
+ i = i + 1
+
+ return i
+end
+
+local function get_headers_stat_tokens(task, cf, res, i)
+ --[[
+ -- As discussed with Alexander Moisseev, this feature can skew statistics
+ -- especially when learning is separated from scanning, so learning
+ -- has a different set of tokens where this token can have too high weight
+ local hdrs_cksum = task:get_mempool():get_variable("headers_hash")
+
+ if hdrs_cksum then
+ rawset(res, i, string.format("#hh:%s", hdrs_cksum:sub(1, 7)))
+ lua_util.debugm("bayes", task, "added hdrs hash token: %s",
+ res[i])
+ i = i + 1
+ end
+ ]]--
+
+ for k, hdrs in pairs(cf.classify_headers_parsed) do
+ for _, hname in ipairs(hdrs) do
+ local value = task:get_header(hname)
+
+ if value then
+ rawset(res, i, string.format("#h:%s:%s", k, value))
+ lua_util.debugm("bayes", task, "added hdrs token: %s",
+ res[i])
+ i = i + 1
+ end
+ end
+ end
+
+ local from = (task:get_from('mime') or {})[1]
+
+ if from and from.name then
+ rawset(res, i, string.format("#F:%s", from.name))
+ lua_util.debugm("bayes", task, "added from name token: %s",
+ res[i])
+ i = i + 1
+ end
+
+ return i
+end
+
+local function get_meta_stat_tokens(task, res, i)
+ local day_and_hour = os.date('%u:%H',
+ task:get_date { format = 'message', gmt = true })
+ rawset(res, i, string.format("#dt:%s", day_and_hour))
+ lua_util.debugm("bayes", task, "added day_of_week token: %s",
+ res[i])
+ i = i + 1
+
+ local pol = {}
+
+ -- Authentication results
+ if task:has_symbol('DKIM_TRACE') then
+ -- Autolearn or scan
+ if task:has_symbol('R_SPF_ALLOW') then
+ table.insert(pol, 's=pass')
+ end
+
+ local trace = task:get_symbol('DKIM_TRACE')
+ local dkim_opts = trace[1]['options']
+ if dkim_opts then
+ for _, o in ipairs(dkim_opts) do
+ local check_res = string.sub(o, -1)
+ local domain = string.sub(o, 1, -3)
+
+ if check_res == '+' then
+ table.insert(pol, string.format('d=%s:%s', "pass", domain))
+ end
+ end
+ end
+ else
+ -- Offline learn
+ local aur = task:get_header('Authentication-Results')
+
+ if aur then
+ local spf = aur:match('spf=([a-z]+)')
+ local dkim, dkim_domain = aur:match('dkim=([a-z]+) header.d=([a-z.%-]+)')
+
+ if spf then
+ table.insert(pol, 's=' .. spf)
+ end
+ if dkim and dkim_domain then
+ table.insert(pol, string.format('d=%s:%s', dkim, dkim_domain))
+ end
+ end
+ end
+
+ if #pol > 0 then
+ rawset(res, i, string.format("#aur:%s", table.concat(pol, ',')))
+ lua_util.debugm("bayes", task, "added policies token: %s",
+ res[i])
+ i = i + 1
+ end
+
+ --[[
+ -- Disabled.
+ -- 1. Depending on the source the message has a different set of Received
+ -- headers as the receiving MTA adds another Received header.
+ -- 2. The usefulness of the Received tokens is questionable.
+ local rh = task:get_received_headers()
+
+ if rh and #rh > 0 then
+ local lim = math.min(5, #rh)
+ for j =1,lim do
+ local rcvd = rh[j]
+ local ip = rcvd.real_ip
+ if ip and ip:is_valid() and ip:get_version() == 4 then
+ local masked = ip:apply_mask(24)
+
+ rawset(res, i, string.format("#rcv:%s:%s", tostring(masked),
+ rcvd.proto))
+ lua_util.debugm("bayes", task, "added received token: %s",
+ res[i])
+ i = i + 1
+ end
+ end
+ end
+ ]]--
+
+ return i
+end
+
+local function get_stat_tokens(task, cf)
+ local res = {}
+ local E = {}
+ local i = 1
+
+ if cf.classify_images then
+ local images = task:get_images() or E
+
+ for _, img in ipairs(images) do
+ rawset(res, i, "image")
+ i = i + 1
+ rawset(res, i, tostring(img:get_height()))
+ i = i + 1
+ rawset(res, i, tostring(img:get_width()))
+ i = i + 1
+ rawset(res, i, tostring(img:get_type()))
+ i = i + 1
+
+ local fname = img:get_filename()
+
+ if fname then
+ rawset(res, i, tostring(img:get_filename()))
+ i = i + 1
+ end
+
+ lua_util.debugm("bayes", task, "added image: %s",
+ fname)
+ end
+ end
+
+ if cf.classify_mime_info then
+ i = get_mime_stat_tokens(task, res, i)
+ end
+
+ if cf.classify_headers and #cf.classify_headers > 0 then
+ i = get_headers_stat_tokens(task, cf, res, i)
+ end
+
+ if cf.classify_urls then
+ local urls = lua_util.extract_specific_urls { task = task, limit = 5, esld_limit = 1 }
+
+ if urls then
+ for _, u in ipairs(urls) do
+ rawset(res, i, string.format("#u:%s", u:get_tld()))
+ lua_util.debugm("bayes", task, "added url token: %s",
+ res[i])
+ i = i + 1
+ end
+ end
+ end
+
+ if cf.classify_meta then
+ i = get_meta_stat_tokens(task, res, i)
+ end
+
+ return res
+end
+
+exports.gen_stat_tokens = function(cfg)
+ local stat_config = process_stat_config(cfg)
+
+ return function(task)
+ return get_stat_tokens(task, stat_config)
+ end
+end
+
+return exports
diff --git a/lualib/lua_tcp_sync.lua b/lualib/lua_tcp_sync.lua
new file mode 100644
index 0000000..f8e6044
--- /dev/null
+++ b/lualib/lua_tcp_sync.lua
@@ -0,0 +1,213 @@
+local rspamd_tcp = require "rspamd_tcp"
+local lua_util = require "lua_util"
+
+local exports = {}
+local N = 'tcp_sync'
+
+local tcp_sync = { _conn = nil, _data = '', _eof = false, _addr = '' }
+local metatable = {
+ __tostring = function(self)
+ return "class {tcp_sync connect to: " .. self._addr .. "}"
+ end
+}
+
+function tcp_sync.new(connection)
+ local self = {}
+
+ for name, method in pairs(tcp_sync) do
+ if name ~= 'new' then
+ self[name] = method
+ end
+ end
+
+ self._conn = connection
+
+ setmetatable(self, metatable)
+
+ return self
+end
+
+--[[[
+-- @method tcp_sync.read_once()
+--
+-- Acts exactly like low-level tcp_sync.read_once()
+-- the only exception is that if there is some pending data,
+-- it's returned immediately and no underlying call is performed
+--
+-- @return
+-- true, {data} if everything is fine
+-- false, {error message} otherwise
+--
+--]]
+function tcp_sync:read_once()
+ local is_ok, data
+ if self._data:len() > 0 then
+ data = self._data
+ self._data = nil
+ return true, data
+ end
+
+ is_ok, data = self._conn:read_once()
+
+ return is_ok, data
+end
+
+--[[[
+-- @method tcp_sync.read_until(pattern)
+--
+-- Reads data from the connection until pattern is found
+-- returns all bytes before the pattern
+--
+-- @param {pattern} Read data until pattern is found
+-- @return
+-- true, {data} if everything is fine
+-- false, {error message} otherwise
+-- @example
+--
+--]]
+function tcp_sync:read_until(pattern)
+ repeat
+ local pos_start, pos_end = self._data:find(pattern, 1, true)
+ if pos_start then
+ local data = self._data:sub(1, pos_start - 1)
+ self._data = self._data:sub(pos_end + 1)
+ return true, data
+ end
+
+ local is_ok, more_data = self._conn:read_once()
+ if not is_ok then
+ return is_ok, more_data
+ end
+
+ self._data = self._data .. more_data
+ until false
+end
+
+--[[[
+-- @method tcp_sync.read_bytes(n)
+--
+-- Reads {n} bytes from the stream
+--
+-- @param {n} Number of bytes to read
+-- @return
+-- true, {data} if everything is fine
+-- false, {error message} otherwise
+--
+--]]
+function tcp_sync:read_bytes(n)
+ repeat
+ if self._data:len() >= n then
+ local data = self._data:sub(1, n)
+ self._data = self._data:sub(n + 1)
+ return true, data
+ end
+
+ local is_ok, more_data = self._conn:read_once()
+ if not is_ok then
+ return is_ok, more_data
+ end
+
+ self._data = self._data .. more_data
+ until false
+end
+
+--[[[
+-- @method tcp_sync.read_until_eof(n)
+--
+-- Reads stream until EOF is reached
+--
+-- @return
+-- true, {data} if everything is fine
+-- false, {error message} otherwise
+--
+--]]
+function tcp_sync:read_until_eof()
+ while not self:eof() do
+ local is_ok, more_data = self._conn:read_once()
+ if not is_ok then
+ if self:eof() then
+ -- this error is EOF (connection terminated)
+ -- exactly what we were waiting for
+ break
+ end
+ return is_ok, more_data
+ end
+ self._data = self._data .. more_data
+ end
+
+ local data = self._data
+ self._data = ''
+ return true, data
+end
+
+--[[[
+-- @method tcp_sync.write(n)
+--
+-- Writes data into the stream.
+--
+-- @return
+-- true if everything is fine
+-- false, {error message} otherwise
+--
+--]]
+function tcp_sync:write(data)
+ return self._conn:write(data)
+end
+
+--[[[
+-- @method tcp_sync.close()
+--
+-- Closes the connection. If the connection was created with task,
+-- this method is called automatically as soon as the task is done
+-- Calling this method helps to prevent connections leak.
+-- The object is finally destroyed by garbage collector.
+--
+-- @return
+--
+--]]
+function tcp_sync:close()
+ return self._conn:close()
+end
+
+--[[[
+-- @method tcp_sync.eof()
+--
+-- @return
+-- true if last "read" operation ended with EOF
+-- false otherwise
+--
+--]]
+function tcp_sync:eof()
+ if not self._eof and self._conn:eof() then
+ self._eof = true
+ end
+ return self._eof
+end
+
+--[[[
+-- @function tcp_sync.shutdown(n)
+--
+-- half-close socket
+--
+-- @return
+--
+--]]
+function tcp_sync:shutdown()
+ return self._conn:shutdown()
+end
+
+exports.connect = function(args)
+ local is_ok, connection = rspamd_tcp.connect_sync(args)
+ if not is_ok then
+ return is_ok, connection
+ end
+
+ local instance = tcp_sync.new(connection)
+ instance._addr = string.format("%s:%s", tostring(args.host), tostring(args.port))
+
+ lua_util.debugm(N, args.task, 'Connected to %s', instance._addr)
+
+ return true, instance
+end
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_urls_compose.lua b/lualib/lua_urls_compose.lua
new file mode 100644
index 0000000..1113421
--- /dev/null
+++ b/lualib/lua_urls_compose.lua
@@ -0,0 +1,286 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_urls_compose
+-- This module contains functions to compose urls queries from hostname
+-- to TLD part
+--]]
+
+local N = "lua_urls_compose"
+local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local bit = require "bit"
+local rspamd_trie = require "rspamd_trie"
+local fun = require "fun"
+local rspamd_regexp = require "rspamd_regexp"
+
+local maps_cache = {}
+
+local exports = {}
+
+local function process_url(self, log_obj, url_tld, url_host)
+ local tld_elt = self.tlds[url_tld]
+
+ if tld_elt then
+ lua_util.debugm(N, log_obj, 'found compose tld for %s (host = %s)',
+ url_tld, url_host)
+
+ for _, excl in ipairs(tld_elt.except_rules) do
+ local matched, ret = excl[2](url_tld, url_host)
+ if matched then
+ lua_util.debugm(N, log_obj, 'found compose exclusion for %s (%s) -> %s',
+ url_host, excl[1], ret)
+
+ return ret
+ end
+ end
+
+ if tld_elt.multipattern_compose_rules then
+ local matches = tld_elt.multipattern_compose_rules:match(url_host)
+
+ if matches then
+ local lua_pat_idx = math.huge
+
+ for m, _ in pairs(matches) do
+ if m < lua_pat_idx then
+ lua_pat_idx = m
+ end
+ end
+
+ if #tld_elt.compose_rules >= lua_pat_idx then
+ local lua_pat = tld_elt.compose_rules[lua_pat_idx]
+ local matched, ret = lua_pat[2](url_tld, url_host)
+
+ if not matched then
+ lua_util.debugm(N, log_obj, 'NOT found compose inclusion for %s (%s) -> %s',
+ url_host, lua_pat[1], url_tld)
+
+ return url_tld
+ else
+ lua_util.debugm(N, log_obj, 'found compose inclusion for %s (%s) -> %s',
+ url_host, lua_pat[1], ret)
+
+ return ret
+ end
+ else
+ lua_util.debugm(N, log_obj, 'NOT found compose inclusion for %s (%s) -> %s',
+ url_host, lua_pat_idx, url_tld)
+
+ return url_tld
+ end
+ end
+ else
+ -- Match one by one
+ for _, lua_pat in ipairs(tld_elt.compose_rules) do
+ local matched, ret = lua_pat[2](url_tld, url_host)
+ if matched then
+ lua_util.debugm(N, log_obj, 'found compose inclusion for %s (%s) -> %s',
+ url_host, lua_pat[1], ret)
+
+ return ret
+ end
+ end
+ end
+
+ lua_util.debugm(N, log_obj, 'not found compose inclusion for %s in %s -> %s',
+ url_host, url_tld, url_tld)
+ else
+ lua_util.debugm(N, log_obj, 'not found compose tld for %s in %s -> %s',
+ url_host, url_tld, url_tld)
+ end
+
+ return url_tld
+end
+
+local function tld_pattern_transform(tld_pat)
+ -- Convert tld like pattern to a lua match pattern
+ -- blah -> %.blah
+ -- *.blah -> .*%.blah
+ local ret
+ if tld_pat:sub(1, 2) == '*.' then
+ ret = string.format('^((?:[^.]+\\.)*%s)$', tld_pat:sub(3))
+ else
+ ret = string.format('(?:^|\\.)((?:[^.]+\\.)?%s)$', tld_pat)
+ end
+
+ lua_util.debugm(N, nil, 'added pattern %s -> %s',
+ tld_pat, ret)
+
+ return ret
+end
+
+local function include_elt_gen(pat)
+ pat = rspamd_regexp.create(tld_pattern_transform(pat), 'i')
+ return function(_, host)
+ local matches = pat:search(host, false, true)
+ if matches then
+ return true, matches[1][2]
+ end
+
+ return false
+ end
+end
+
+local function exclude_elt_gen(pat)
+ pat = rspamd_regexp.create(tld_pattern_transform(pat))
+ return function(tld, host)
+ if pat:search(host) then
+ return true, tld
+ end
+
+ return false
+ end
+end
+
+local function compose_map_cb(self, map_text)
+ local lpeg = require "lpeg"
+
+ local singleline_comment = lpeg.P '#' * (1 - lpeg.S '\r\n\f') ^ 0
+ local comments_strip_grammar = lpeg.C((1 - lpeg.P '#') ^ 1) * lpeg.S(' \t') ^ 0 * singleline_comment ^ 0
+
+ local function process_tld_rule(tld_elt, l)
+ if l:sub(1, 1) == '!' then
+ -- Exclusion elt
+ table.insert(tld_elt.except_rules, { l, exclude_elt_gen(l:sub(2)) })
+ else
+ table.insert(tld_elt.compose_rules, { l, include_elt_gen(l) })
+ end
+ end
+
+ local function process_map_line(l)
+ -- Skip empty lines and comments
+ if #l == 0 then
+ return
+ end
+ l = comments_strip_grammar:match(l)
+ if not l or #l == 0 then
+ return
+ end
+
+ -- Get TLD
+ local tld = rspamd_util.get_tld(l)
+
+ if tld then
+ local tld_elt = self.tlds[tld]
+
+ if not tld_elt then
+ tld_elt = {
+ compose_rules = {},
+ except_rules = {},
+ multipattern_compose_rules = nil
+ }
+
+ lua_util.debugm(N, rspamd_config, 'processed new tld rule for %s', tld)
+ self.tlds[tld] = tld_elt
+ end
+
+ process_tld_rule(tld_elt, l)
+ else
+ lua_util.debugm(N, rspamd_config, 'cannot read tld from compose map line: %s', l)
+ end
+ end
+
+ for line in map_text:lines() do
+ process_map_line(line)
+ end
+
+ local multipattern_threshold = 1
+ for tld, tld_elt in pairs(self.tlds) do
+ -- Sort patterns to have longest labels before shortest ones,
+ -- so we can ensure that they match before
+ table.sort(tld_elt.compose_rules, function(e1, e2)
+ local _, ndots1 = string.gsub(e1[1], '(%.)', '')
+ local _, ndots2 = string.gsub(e2[1], '(%.)', '')
+
+ return ndots1 > ndots2
+ end)
+ if rspamd_trie.has_hyperscan() and #tld_elt.compose_rules >= multipattern_threshold then
+ lua_util.debugm(N, rspamd_config, 'tld %s has %s rules, apply multipattern',
+ tld, #tld_elt.compose_rules)
+ local flags = bit.bor(rspamd_trie.flags.re,
+ rspamd_trie.flags.dot_all,
+ rspamd_trie.flags.no_start,
+ rspamd_trie.flags.icase)
+
+
+ -- We now convert our internal patterns to multipattern patterns
+ local mp_table = fun.totable(fun.map(function(pat_elt)
+ return tld_pattern_transform(pat_elt[1])
+ end, tld_elt.compose_rules))
+ tld_elt.multipattern_compose_rules = rspamd_trie.create(mp_table, flags)
+ end
+ end
+end
+
+exports.add_composition_map = function(cfg, map_obj)
+ local hash_key = map_obj
+ if type(map_obj) == 'table' then
+ hash_key = lua_util.table_digest(map_obj)
+ end
+
+ local map = maps_cache[hash_key]
+
+ if not map then
+ local ret = {
+ process_url = process_url,
+ hash = hash_key,
+ tlds = {},
+ }
+
+ map = cfg:add_map {
+ type = 'callback',
+ description = 'URL compose map',
+ url = map_obj,
+ callback = function(input)
+ compose_map_cb(ret, input)
+ end,
+ opaque_data = true,
+ }
+
+ ret.map = map
+ maps_cache[hash_key] = ret
+ map = ret
+ end
+
+ return map
+end
+
+exports.inject_composition_rules = function(cfg, rules)
+ local hash_key = rules
+ local rspamd_text = require "rspamd_text"
+ if type(rules) == 'table' then
+ hash_key = lua_util.table_digest(rules)
+ end
+
+ local map = maps_cache[hash_key]
+
+ if not map then
+ local ret = {
+ process_url = process_url,
+ hash = hash_key,
+ tlds = {},
+ }
+
+ compose_map_cb(ret, rspamd_text.fromtable(rules, '\n'))
+ maps_cache[hash_key] = ret
+ map = ret
+ end
+
+ return map
+end
+
+return exports \ No newline at end of file
diff --git a/lualib/lua_util.lua b/lualib/lua_util.lua
new file mode 100644
index 0000000..6964b0f
--- /dev/null
+++ b/lualib/lua_util.lua
@@ -0,0 +1,1639 @@
+--[[
+Copyright (c) 2023, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_util
+-- This module contains utility functions for working with Lua and/or Rspamd
+--]]
+
+local exports = {}
+local lpeg = require 'lpeg'
+local rspamd_util = require "rspamd_util"
+local fun = require "fun"
+local lupa = require "lupa"
+
+local split_grammar = {}
+local spaces_split_grammar
+local space = lpeg.S ' \t\n\v\f\r'
+local nospace = 1 - space
+local ptrim = space ^ 0 * lpeg.C((space ^ 0 * nospace ^ 1) ^ 0)
+local match = lpeg.match
+
+lupa.configure('{%', '%}', '{=', '=}', '{#', '#}', {
+ keep_trailing_newline = true,
+ autoescape = false,
+})
+
+lupa.filters.pbkdf = function(s)
+ local cr = require "rspamd_cryptobox"
+ return cr.pbkdf(s)
+end
+
+local function rspamd_str_split(s, sep)
+ local gr
+ if not sep then
+ if not spaces_split_grammar then
+ local _sep = space
+ local elem = lpeg.C((1 - _sep) ^ 0)
+ local p = lpeg.Ct(elem * (_sep * elem) ^ 0)
+ spaces_split_grammar = p
+ end
+
+ gr = spaces_split_grammar
+ else
+ gr = split_grammar[sep]
+
+ if not gr then
+ local _sep
+ if type(sep) == 'string' then
+ _sep = lpeg.S(sep) -- Assume set
+ else
+ _sep = sep -- Assume lpeg object
+ end
+ local elem = lpeg.C((1 - _sep) ^ 0)
+ local p = lpeg.Ct(elem * (_sep * elem) ^ 0)
+ gr = p
+ split_grammar[sep] = gr
+ end
+ end
+
+ return gr:match(s)
+end
+
+--[[[
+-- @function lua_util.str_split(text, delimiter)
+-- Splits text into a numeric table by delimiter
+-- @param {string} text delimited text
+-- @param {string} delimiter the delimiter
+-- @return {table} numeric table containing string parts
+--]]
+
+exports.rspamd_str_split = rspamd_str_split
+exports.str_split = rspamd_str_split
+
+local function rspamd_str_trim(s)
+ return match(ptrim, s)
+end
+exports.rspamd_str_trim = rspamd_str_trim
+--[[[
+-- @function lua_util.str_trim(text)
+-- Returns a string with no trailing and leading spaces
+-- @param {string} text input text
+-- @return {string} string with no trailing and leading spaces
+--]]
+exports.str_trim = rspamd_str_trim
+
+--[[[
+-- @function lua_util.str_startswith(text, prefix)
+-- @param {string} text
+-- @param {string} prefix
+-- @return {boolean} true if text starts with the specified prefix, false otherwise
+--]]
+exports.str_startswith = function(s, prefix)
+ return s:sub(1, prefix:len()) == prefix
+end
+
+--[[[
+-- @function lua_util.str_endswith(text, suffix)
+-- @param {string} text
+-- @param {string} suffix
+-- @return {boolean} true if text ends with the specified suffix, false otherwise
+--]]
+exports.str_endswith = function(s, suffix)
+ return s:find(suffix, -suffix:len(), true) ~= nil
+end
+
+--[[[
+-- @function lua_util.round(number, decimalPlaces)
+-- Round number to fixed number of decimal points
+-- @param {number} number number to round
+-- @param {number} decimalPlaces number of decimal points
+-- @return {number} rounded number
+--]]
+
+-- modified version from Robert Jay Gould http://lua-users.org/wiki/SimpleRound
+exports.round = function(num, numDecimalPlaces)
+ local mult = 10 ^ (numDecimalPlaces or 0)
+ if num >= 0 then
+ return math.floor(num * mult + 0.5) / mult
+ else
+ return math.ceil(num * mult - 0.5) / mult
+ end
+end
+
+--[[[
+-- @function lua_util.template(text, replacements)
+-- Replaces values in a text template
+-- Variable names can contain letters, numbers and underscores, are prefixed with `$` and may or not use curly braces.
+-- @param {string} text text containing variables
+-- @param {table} replacements key/value pairs for replacements
+-- @return {string} string containing replaced values
+-- @example
+-- local goop = lua_util.template("HELLO $FOO ${BAR}!", {['FOO'] = 'LUA', ['BAR'] = 'WORLD'})
+-- -- goop contains "HELLO LUA WORLD!"
+--]]
+
+exports.template = function(tmpl, keys)
+ local var_lit = lpeg.P { lpeg.R("az") + lpeg.R("AZ") + lpeg.R("09") + "_" }
+ local var = lpeg.P { (lpeg.P("$") / "") * ((var_lit ^ 1) / keys) }
+ local var_braced = lpeg.P { (lpeg.P("${") / "") * ((var_lit ^ 1) / keys) * (lpeg.P("}") / "") }
+
+ local template_grammar = lpeg.Cs((var + var_braced + 1) ^ 0)
+
+ return lpeg.match(template_grammar, tmpl)
+end
+
+local function enrich_template_with_globals(env)
+ local newenv = exports.shallowcopy(env)
+ newenv.paths = rspamd_paths
+ newenv.env = rspamd_env
+
+ return newenv
+end
+--[[[
+-- @function lua_util.jinja_template(text, env[, skip_global_env])
+-- Replaces values in a text template according to jinja2 syntax
+-- @param {string} text text containing variables
+-- @param {table} replacements key/value pairs for replacements
+-- @param {boolean} skip_global_env don't export Rspamd superglobals
+-- @return {string} string containing replaced values
+-- @example
+-- lua_util.jinja_template("HELLO {{FOO}} {{BAR}}!", {['FOO'] = 'LUA', ['BAR'] = 'WORLD'})
+-- "HELLO LUA WORLD!"
+--]]
+exports.jinja_template = function(text, env, skip_global_env)
+ if not skip_global_env then
+ env = enrich_template_with_globals(env)
+ end
+
+ return lupa.expand(text, env)
+end
+
+--[[[
+-- @function lua_util.jinja_file(filename, env[, skip_global_env])
+-- Replaces values in a text template according to jinja2 syntax
+-- @param {string} filename name of file to expand
+-- @param {table} replacements key/value pairs for replacements
+-- @param {boolean} skip_global_env don't export Rspamd superglobals
+-- @return {string} string containing replaced values
+-- @example
+-- lua_util.jinja_template("HELLO {{FOO}} {{BAR}}!", {['FOO'] = 'LUA', ['BAR'] = 'WORLD'})
+-- "HELLO LUA WORLD!"
+--]]
+exports.jinja_template_file = function(filename, env, skip_global_env)
+ if not skip_global_env then
+ env = enrich_template_with_globals(env)
+ end
+
+ return lupa.expand_file(filename, env)
+end
+
+exports.remove_email_aliases = function(email_addr)
+ local function check_gmail_user(addr)
+ -- Remove all points
+ local no_dots_user = string.gsub(addr.user, '%.', '')
+ local cap, pluses = string.match(no_dots_user, '^([^%+][^%+]*)(%+.*)$')
+ if cap then
+ return cap, rspamd_str_split(pluses, '+'), nil
+ elseif no_dots_user ~= addr.user then
+ return no_dots_user, {}, nil
+ end
+
+ return nil
+ end
+
+ local function check_address(addr)
+ if addr.user then
+ local cap, pluses = string.match(addr.user, '^([^%+][^%+]*)(%+.*)$')
+ if cap then
+ return cap, rspamd_str_split(pluses, '+'), nil
+ end
+ end
+
+ return nil
+ end
+
+ local function set_addr(addr, new_user, new_domain)
+ if new_user then
+ addr.user = new_user
+ end
+ if new_domain then
+ addr.domain = new_domain
+ end
+
+ if addr.domain then
+ addr.addr = string.format('%s@%s', addr.user, addr.domain)
+ else
+ addr.addr = string.format('%s@', addr.user)
+ end
+
+ if addr.name and #addr.name > 0 then
+ addr.raw = string.format('"%s" <%s>', addr.name, addr.addr)
+ else
+ addr.raw = string.format('<%s>', addr.addr)
+ end
+ end
+
+ local function check_gmail(addr)
+ local nu, tags, nd = check_gmail_user(addr)
+
+ if nu then
+ return nu, tags, nd
+ end
+
+ return nil
+ end
+
+ local function check_googlemail(addr)
+ local nd = 'gmail.com'
+ local nu, tags = check_gmail_user(addr)
+
+ if nu then
+ return nu, tags, nd
+ end
+
+ return nil, nil, nd
+ end
+
+ local specific_domains = {
+ ['gmail.com'] = check_gmail,
+ ['googlemail.com'] = check_googlemail,
+ }
+
+ if email_addr then
+ if email_addr.domain and specific_domains[email_addr.domain] then
+ local nu, tags, nd = specific_domains[email_addr.domain](email_addr)
+ if nu or nd then
+ set_addr(email_addr, nu, nd)
+
+ return nu, tags
+ end
+ else
+ local nu, tags, nd = check_address(email_addr)
+ if nu or nd then
+ set_addr(email_addr, nu, nd)
+
+ return nu, tags
+ end
+ end
+
+ return nil
+ end
+end
+
+exports.is_rspamc_or_controller = function(task)
+ local ua = task:get_request_header('User-Agent') or ''
+ local pwd = task:get_request_header('Password')
+ local is_rspamc = false
+ if tostring(ua) == 'rspamc' or pwd then
+ is_rspamc = true
+ end
+
+ return is_rspamc
+end
+
+--[[[
+-- @function lua_util.unpack(table)
+-- Converts numeric table to varargs
+-- This is `unpack` on Lua 5.1/5.2/LuaJIT and `table.unpack` on Lua 5.3
+-- @param {table} table numerically indexed table to unpack
+-- @return {varargs} unpacked table elements
+--]]
+
+local unpack_function = table.unpack or unpack
+exports.unpack = function(t)
+ return unpack_function(t)
+end
+
+--[[[
+-- @function lua_util.flatten(table)
+-- Flatten underlying tables in a single table
+-- @param {table} table table of tables
+-- @return {table} flattened table
+--]]
+exports.flatten = function(t)
+ local res = {}
+ for _, e in fun.iter(t) do
+ for _, v in fun.iter(e) do
+ res[#res + 1] = v
+ end
+ end
+
+ return res
+end
+
+--[[[
+-- @function lua_util.spairs(table)
+-- Like `pairs` but keys are sorted lexicographically
+-- @param {table} table table containing key/value pairs
+-- @return {function} generator function returning key/value pairs
+--]]
+
+-- Sorted iteration:
+-- for k,v in spairs(t) do ... end
+--
+-- or with custom comparison:
+-- for k, v in spairs(t, function(t, a, b) return t[a] < t[b] end)
+--
+-- optional limit is also available (e.g. return top X elements)
+local function spairs(t, order, lim)
+ -- collect the keys
+ local keys = {}
+ for k in pairs(t) do
+ keys[#keys + 1] = k
+ end
+
+ -- if order function given, sort by it by passing the table and keys a, b,
+ -- otherwise just sort the keys
+ if order then
+ table.sort(keys, function(a, b)
+ return order(t, a, b)
+ end)
+ else
+ table.sort(keys)
+ end
+
+ -- return the iterator function
+ local i = 0
+ return function()
+ i = i + 1
+ if not lim or i <= lim then
+ if keys[i] then
+ return keys[i], t[keys[i]]
+ end
+ end
+ end
+end
+
+exports.spairs = spairs
+
+local lua_cfg_utils = require "lua_cfg_utils"
+
+exports.config_utils = lua_cfg_utils
+exports.disable_module = lua_cfg_utils.disable_module
+
+--[[[
+-- @function lua_util.disable_module(modname)
+-- Checks experimental plugins state and disable if needed
+-- @param {string} modname name of plugin to check
+-- @return {boolean} true if plugin should be enabled, false otherwise
+--]]
+local function check_experimental(modname)
+ if rspamd_config:experimental_enabled() then
+ return true
+ else
+ lua_cfg_utils.disable_module(modname, 'experimental')
+ end
+
+ return false
+end
+
+exports.check_experimental = check_experimental
+
+--[[[
+-- @function lua_util.list_to_hash(list)
+-- Converts numerically-indexed table to table indexed by values
+-- @param {table} list numerically-indexed table or string, which is treated as a one-element list
+-- @return {table} table indexed by values
+-- @example
+-- local h = lua_util.list_to_hash({"a", "b"})
+-- -- h contains {a = true, b = true}
+--]]
+local function list_to_hash(list)
+ if type(list) == 'table' then
+ if list[1] then
+ local h = {}
+ for _, e in ipairs(list) do
+ h[e] = true
+ end
+ return h
+ else
+ return list
+ end
+ elseif type(list) == 'string' then
+ local h = {}
+ h[list] = true
+ return h
+ end
+end
+
+exports.list_to_hash = list_to_hash
+
+--[[[
+-- @function lua_util.nkeys(table|gen, param, state)
+-- Returns number of keys in a table (i.e. from both the array and hash parts combined)
+-- @param {table} list numerically-indexed table or string, which is treated as a one-element list
+-- @return {number} number of keys
+-- @example
+-- print(lua_util.nkeys({})) -- 0
+-- print(lua_util.nkeys({ "a", nil, "b" })) -- 2
+-- print(lua_util.nkeys({ dog = 3, cat = 4, bird = nil })) -- 2
+-- print(lua_util.nkeys({ "a", dog = 3, cat = 4 })) -- 3
+--
+--]]
+local function nkeys(gen, param, state)
+ local n = 0
+ if not param then
+ for _, _ in pairs(gen) do
+ n = n + 1
+ end
+ else
+ for _, _ in fun.iter(gen, param, state) do
+ n = n + 1
+ end
+ end
+ return n
+end
+
+exports.nkeys = nkeys
+
+--[[[
+-- @function lua_util.parse_time_interval(str)
+-- Parses human readable time interval
+-- Accepts 's' for seconds, 'm' for minutes, 'h' for hours, 'd' for days,
+-- 'w' for weeks, 'y' for years
+-- @param {string} str input string
+-- @return {number|nil} parsed interval as seconds (might be fractional)
+--]]
+local function parse_time_interval(str)
+ local function parse_time_suffix(s)
+ if s == 's' then
+ return 1
+ elseif s == 'm' then
+ return 60
+ elseif s == 'h' then
+ return 3600
+ elseif s == 'd' then
+ return 86400
+ elseif s == 'w' then
+ return 86400 * 7
+ elseif s == 'y' then
+ return 365 * 86400;
+ end
+ end
+
+ local digit = lpeg.R("09")
+ local parser = {}
+ parser.integer = (lpeg.S("+-") ^ -1) *
+ (digit ^ 1)
+ parser.fractional = (lpeg.P(".")) *
+ (digit ^ 1)
+ parser.number = (parser.integer *
+ (parser.fractional ^ -1)) +
+ (lpeg.S("+-") * parser.fractional)
+ parser.time = lpeg.Cf(lpeg.Cc(1) *
+ (parser.number / tonumber) *
+ ((lpeg.S("smhdwy") / parse_time_suffix) ^ -1),
+ function(acc, val)
+ return acc * val
+ end)
+
+ local t = lpeg.match(parser.time, str)
+
+ return t
+end
+
+exports.parse_time_interval = parse_time_interval
+
+--[[[
+-- @function lua_util.dehumanize_number(str)
+-- Parses human readable number
+-- Accepts 'k' for thousands, 'm' for millions, 'g' for billions, 'b' suffix for 1024 multiplier,
+-- e.g. `10mb` equal to `10 * 1024 * 1024`
+-- @param {string} str input string
+-- @return {number|nil} parsed number
+--]]
+local function dehumanize_number(str)
+ local function parse_suffix(s)
+ if s == 'k' then
+ return 1000
+ elseif s == 'm' then
+ return 1000000
+ elseif s == 'g' then
+ return 1e9
+ elseif s == 'kb' then
+ return 1024
+ elseif s == 'mb' then
+ return 1024 * 1024
+ elseif s == 'gb' then
+ return 1024 * 1024;
+ end
+ end
+
+ local digit = lpeg.R("09")
+ local parser = {}
+ parser.integer = (lpeg.S("+-") ^ -1) *
+ (digit ^ 1)
+ parser.fractional = (lpeg.P(".")) *
+ (digit ^ 1)
+ parser.number = (parser.integer *
+ (parser.fractional ^ -1)) +
+ (lpeg.S("+-") * parser.fractional)
+ parser.humanized_number = lpeg.Cf(lpeg.Cc(1) *
+ (parser.number / tonumber) *
+ (((lpeg.S("kmg") * (lpeg.P("b") ^ -1)) / parse_suffix) ^ -1),
+ function(acc, val)
+ return acc * val
+ end)
+
+ local t = lpeg.match(parser.humanized_number, str)
+
+ return t
+end
+
+exports.dehumanize_number = dehumanize_number
+
+--[[[
+-- @function lua_util.table_cmp(t1, t2)
+-- Compare two tables deeply
+--]]
+local function table_cmp(table1, table2)
+ local avoid_loops = {}
+ local function recurse(t1, t2)
+ if type(t1) ~= type(t2) then
+ return false
+ end
+ if type(t1) ~= "table" then
+ return t1 == t2
+ end
+
+ if avoid_loops[t1] then
+ return avoid_loops[t1] == t2
+ end
+ avoid_loops[t1] = t2
+ -- Copy keys from t2
+ local t2keys = {}
+ local t2tablekeys = {}
+ for k, _ in pairs(t2) do
+ if type(k) == "table" then
+ table.insert(t2tablekeys, k)
+ end
+ t2keys[k] = true
+ end
+ -- Let's iterate keys from t1
+ for k1, v1 in pairs(t1) do
+ local v2 = t2[k1]
+ if type(k1) == "table" then
+ -- if key is a table, we need to find an equivalent one.
+ local ok = false
+ for i, tk in ipairs(t2tablekeys) do
+ if table_cmp(k1, tk) and recurse(v1, t2[tk]) then
+ table.remove(t2tablekeys, i)
+ t2keys[tk] = nil
+ ok = true
+ break
+ end
+ end
+ if not ok then
+ return false
+ end
+ else
+ -- t1 has a key which t2 doesn't have, fail.
+ if v2 == nil then
+ return false
+ end
+ t2keys[k1] = nil
+ if not recurse(v1, v2) then
+ return false
+ end
+ end
+ end
+ -- if t2 has a key which t1 doesn't have, fail.
+ if next(t2keys) then
+ return false
+ end
+ return true
+ end
+ return recurse(table1, table2)
+end
+
+exports.table_cmp = table_cmp
+
+--[[[
+-- @function lua_util.table_merge(t1, t2)
+-- Merge two tables
+--]]
+local function table_merge(t1, t2)
+ local res = {}
+ local nidx = 1 -- for numeric indicies
+ local it_func = function(k, v)
+ if type(k) == 'number' then
+ res[nidx] = v
+ nidx = nidx + 1
+ else
+ res[k] = v
+ end
+ end
+ for k, v in pairs(t1) do
+ it_func(k, v)
+ end
+ for k, v in pairs(t2) do
+ it_func(k, v)
+ end
+ return res
+end
+
+exports.table_merge = table_merge
+
+--[[[
+-- @function lua_util.table_cmp(task, name, value, stop_chars)
+-- Performs header folding
+--]]
+exports.fold_header = function(task, name, value, stop_chars)
+
+ local how
+
+ if task:has_flag("milter") then
+ how = "lf"
+ else
+ how = task:get_newlines_type()
+ end
+
+ return rspamd_util.fold_header(name, value, how, stop_chars)
+end
+
+--[[[
+-- @function lua_util.override_defaults(defaults, override)
+-- Overrides values from defaults with override
+--]]
+local function override_defaults(def, override)
+ -- Corner cases
+ if not override or type(override) ~= 'table' then
+ return def
+ end
+ if not def or type(def) ~= 'table' then
+ return override
+ end
+
+ local res = {}
+
+ for k, v in pairs(override) do
+ if type(v) == 'table' then
+ if def[k] and type(def[k]) == 'table' then
+ -- Recursively override elements
+ res[k] = override_defaults(def[k], v)
+ else
+ res[k] = v
+ end
+ else
+ res[k] = v
+ end
+ end
+
+ for k, v in pairs(def) do
+ if type(res[k]) == 'nil' then
+ res[k] = v
+ end
+ end
+
+ return res
+end
+
+exports.override_defaults = override_defaults
+
+--[[[
+-- @function lua_util.filter_specific_urls(urls, params)
+-- params: {
+- - task - if needed to save in the cache
+- - limit <int> (default = 9999)
+- - esld_limit <int> (default = 9999) n domains per eSLD (effective second level domain)
+ works only if number of unique eSLD less than `limit`
+- - need_emails <bool> (default = false)
+- - filter <callback> (default = nil)
+- - prefix <string> cache prefix (default = nil)
+-- }
+-- Apply heuristic in extracting of urls from `urls` table, this function
+-- tries its best to extract specific number of urls from a task based on
+-- their characteristics
+--]]
+exports.filter_specific_urls = function(urls, params)
+ local cache_key
+
+ if params.task and not params.no_cache then
+ if params.prefix then
+ cache_key = params.prefix
+ else
+ cache_key = string.format('sp_urls_%d%s%s%s', params.limit,
+ tostring(params.need_emails or false),
+ tostring(params.need_images or false),
+ tostring(params.need_content or false))
+ end
+ local cached = params.task:cache_get(cache_key)
+
+ if cached then
+ return cached
+ end
+ end
+
+ if not urls then
+ return {}
+ end
+
+ if params.filter then
+ urls = fun.totable(fun.filter(params.filter, urls))
+ end
+
+ -- Filter by tld:
+ local tlds = {}
+ local eslds = {}
+ local ntlds, neslds = 0, 0
+
+ local res = {}
+ local nres = 0
+
+ local function insert_url(str, u)
+ if not res[str] then
+ res[str] = u
+ nres = nres + 1
+
+ return true
+ end
+
+ return false
+ end
+
+ local function process_single_url(u, default_priority)
+ local priority = default_priority or 1 -- Normal priority
+ local flags = u:get_flags()
+ if params.ignore_ip and flags.numeric then
+ return
+ end
+
+ if flags.redirected then
+ local redir = u:get_redirected() -- get the real url
+
+ if params.ignore_redirected then
+ -- Replace `u` with redir
+ u = redir
+ priority = 2
+ else
+ -- Process both redirected url and the original one
+ process_single_url(redir, 2)
+ end
+ end
+
+ if flags.image then
+ if not params.need_images then
+ -- Ignore url
+ return
+ else
+ -- Penalise images in urls
+ priority = 0
+ end
+ end
+
+ local esld = u:get_tld()
+ local str_hash = tostring(u)
+
+ if esld then
+ -- Special cases
+ if (u:get_protocol() ~= 'mailto') and (not flags.html_displayed) then
+ if flags.obscured then
+ priority = 3
+ else
+ if (flags.has_user or flags.has_port) then
+ priority = 2
+ elseif (flags.subject or flags.phished) then
+ priority = 2
+ end
+ end
+ elseif flags.html_displayed then
+ priority = 0
+ end
+
+ if not eslds[esld] then
+ eslds[esld] = { { str_hash, u, priority } }
+ neslds = neslds + 1
+ else
+ if #eslds[esld] < params.esld_limit then
+ table.insert(eslds[esld], { str_hash, u, priority })
+ end
+ end
+
+
+ -- eSLD - 1 part => tld
+ local parts = rspamd_str_split(esld, '.')
+ local tld = table.concat(fun.totable(fun.tail(parts)), '.')
+
+ if not tlds[tld] then
+ tlds[tld] = { { str_hash, u, priority } }
+ ntlds = ntlds + 1
+ else
+ table.insert(tlds[tld], { str_hash, u, priority })
+ end
+ end
+ end
+
+ for _, u in ipairs(urls) do
+ process_single_url(u)
+ end
+
+ local limit = params.limit
+ limit = limit - nres
+ if limit < 0 then
+ limit = 0
+ end
+
+ if limit == 0 then
+ res = exports.values(res)
+ if params.task and not params.no_cache then
+ params.task:cache_set(cache_key, res)
+ end
+ return res
+ end
+
+ -- Sort eSLDs and tlds
+ local function sort_stuff(tbl)
+ -- Sort according to max priority
+ table.sort(tbl, function(e1, e2)
+ -- Sort by priority so max priority is at the end
+ table.sort(e1, function(tr1, tr2)
+ return tr1[3] < tr2[3]
+ end)
+ table.sort(e2, function(tr1, tr2)
+ return tr1[3] < tr2[3]
+ end)
+
+ if e1[#e1][3] ~= e2[#e2][3] then
+ -- Sort by priority so max priority is at the beginning
+ return e1[#e1][3] > e2[#e2][3]
+ else
+ -- Prefer less urls to more urls per esld
+ return #e1 < #e2
+ end
+
+ end)
+
+ return tbl
+ end
+
+ eslds = sort_stuff(exports.values(eslds))
+ neslds = #eslds
+
+ if neslds <= limit then
+ -- Number of eslds < limit
+ repeat
+ local item_found = false
+
+ for _, lurls in ipairs(eslds) do
+ if #lurls > 0 then
+ local last = table.remove(lurls)
+ insert_url(last[1], last[2])
+ limit = limit - 1
+ item_found = true
+ end
+ end
+
+ until limit <= 0 or not item_found
+
+ res = exports.values(res)
+ if params.task and not params.no_cache then
+ params.task:cache_set(cache_key, res)
+ end
+ return res
+ end
+
+ tlds = sort_stuff(exports.values(tlds))
+ ntlds = #tlds
+
+ -- Number of tlds < limit
+ while limit > 0 do
+ for _, lurls in ipairs(tlds) do
+ if #lurls > 0 then
+ local last = table.remove(lurls)
+ insert_url(last[1], last[2])
+ limit = limit - 1
+ end
+ if limit == 0 then
+ break
+ end
+ end
+ end
+
+ res = exports.values(res)
+ if params.task and not params.no_cache then
+ params.task:cache_set(cache_key, res)
+ end
+ return res
+end
+
+--[[[
+-- @function lua_util.extract_specific_urls(params)
+-- params: {
+- - task
+- - limit <int> (default = 9999)
+- - esld_limit <int> (default = 9999) n domains per eSLD (effective second level domain)
+ works only if number of unique eSLD less than `limit`
+- - need_emails <bool> (default = false)
+- - filter <callback> (default = nil)
+- - prefix <string> cache prefix (default = nil)
+- - ignore_redirected <bool> (default = false)
+- - need_images <bool> (default = false)
+- - need_content <bool> (default = false)
+-- }
+-- Apply heuristic in extracting of urls from task, this function
+-- tries its best to extract specific number of urls from a task based on
+-- their characteristics
+--]]
+-- exports.extract_specific_urls = function(params_or_task, limit, need_emails, filter, prefix)
+exports.extract_specific_urls = function(params_or_task, lim, need_emails, filter, prefix)
+ local default_params = {
+ limit = 9999,
+ esld_limit = 9999,
+ need_emails = false,
+ need_images = false,
+ need_content = false,
+ filter = nil,
+ prefix = nil,
+ ignore_ip = false,
+ ignore_redirected = false,
+ no_cache = false,
+ }
+
+ local params
+ if type(params_or_task) == 'table' and type(lim) == 'nil' then
+ params = params_or_task
+ else
+ -- Deprecated call
+ params = {
+ task = params_or_task,
+ limit = lim,
+ need_emails = need_emails,
+ filter = filter,
+ prefix = prefix
+ }
+ end
+ for k, v in pairs(default_params) do
+ if type(params[k]) == 'nil' and v ~= nil then
+ params[k] = v
+ end
+ end
+ local url_params = {
+ emails = params.need_emails,
+ images = params.need_images,
+ content = params.need_content,
+ flags = params.flags, -- maybe nil
+ flags_mode = params.flags_mode, -- maybe nil
+ }
+
+ -- Shortcut for cached stuff
+ if params.task and not params.no_cache then
+ local cache_key
+ if params.prefix then
+ cache_key = params.prefix
+ else
+ local cache_key_suffix
+ if params.flags then
+ cache_key_suffix = table.concat(params.flags) .. (params.flags_mode or '')
+ else
+ cache_key_suffix = string.format('%s%s%s',
+ tostring(params.need_emails or false),
+ tostring(params.need_images or false),
+ tostring(params.need_content or false))
+ end
+ cache_key = string.format('sp_urls_%d%s', params.limit, cache_key_suffix)
+ end
+ local cached = params.task:cache_get(cache_key)
+
+ if cached then
+ return cached
+ end
+ end
+
+ -- No cache version
+ local urls = params.task:get_urls(url_params)
+
+ return exports.filter_specific_urls(urls, params)
+end
+
+--[[[
+-- @function lua_util.deepcopy(table)
+-- params: {
+- - table
+-- }
+-- Performs deep copy of the table. Including metatables
+--]]
+local function deepcopy(orig)
+ local orig_type = type(orig)
+ local copy
+ if orig_type == 'table' then
+ copy = {}
+ for orig_key, orig_value in next, orig, nil do
+ copy[deepcopy(orig_key)] = deepcopy(orig_value)
+ end
+ if getmetatable(orig) then
+ setmetatable(copy, deepcopy(getmetatable(orig)))
+ end
+ else
+ -- number, string, boolean, etc
+ copy = orig
+ end
+ return copy
+end
+
+exports.deepcopy = deepcopy
+
+--[[[
+-- @function lua_util.deepsort(table)
+-- params: {
+- - table
+-- }
+-- Performs recursive in-place sort of a table
+--]]
+local function default_sort_cmp(e1, e2)
+ if type(e1) == type(e2) then
+ return e1 < e2
+ else
+ return type(e1) < type(e2)
+ end
+end
+
+local function deepsort(tbl, sort_func)
+ local orig_type = type(tbl)
+ if orig_type == 'table' then
+ table.sort(tbl, sort_func or default_sort_cmp)
+ for _, orig_value in next, tbl, nil do
+ deepsort(orig_value)
+ end
+ end
+end
+
+exports.deepsort = deepsort
+
+--[[[
+-- @function lua_util.shallowcopy(tbl)
+-- Performs shallow (and fast) copy of a table or another Lua type
+--]]
+exports.shallowcopy = function(orig)
+ local orig_type = type(orig)
+ local copy
+ if orig_type == 'table' then
+ copy = {}
+ for orig_key, orig_value in pairs(orig) do
+ copy[orig_key] = orig_value
+ end
+ else
+ copy = orig
+ end
+ return copy
+end
+
+-- Debugging support
+local logger = require "rspamd_logger"
+local unconditional_debug = logger.log_level() == 'debug'
+local debug_modules = {}
+local debug_aliases = {}
+local log_level = 384 -- debug + forced (1 << 7 | 1 << 8)
+
+
+exports.init_debug_logging = function(config)
+ -- Fill debug modules from the config
+ if not unconditional_debug then
+ local log_config = config:get_all_opt('logging')
+ if log_config then
+ local log_level_str = log_config.level
+ if log_level_str then
+ if log_level_str == 'debug' then
+ unconditional_debug = true
+ end
+ end
+ if log_config.debug_modules then
+ for _, m in ipairs(log_config.debug_modules) do
+ debug_modules[m] = true
+ logger.infox(config, 'enable debug for Lua module %s', m)
+ end
+ end
+
+ if #debug_aliases > 0 then
+ for alias, mod in pairs(debug_aliases) do
+ if debug_modules[mod] then
+ debug_modules[alias] = true
+ logger.infox(config, 'enable debug for Lua module %s (%s aliased)',
+ alias, mod)
+ end
+ end
+ end
+ end
+ end
+end
+
+exports.enable_debug_logging = function()
+ unconditional_debug = true
+end
+
+exports.enable_debug_modules = function(...)
+ for _, m in ipairs({ ... }) do
+ debug_modules[m] = true
+ end
+end
+
+exports.disable_debug_logging = function()
+ unconditional_debug = false
+end
+
+--[[[
+-- @function lua_util.debugm(module, [log_object], format, ...)
+-- Performs fast debug log for a specific module
+--]]
+exports.debugm = function(mod, obj_or_fmt, fmt_or_something, ...)
+ if unconditional_debug or debug_modules[mod] then
+ if type(obj_or_fmt) == 'string' then
+ logger.logx(log_level, mod, '', 2, obj_or_fmt, fmt_or_something, ...)
+ else
+ logger.logx(log_level, mod, obj_or_fmt, 2, fmt_or_something, ...)
+ end
+ end
+end
+
+--[[[
+-- @function lua_util.add_debug_alias(mod, alias)
+-- Add debugging alias so logging to `alias` will be treated as logging to `mod`
+--]]
+exports.add_debug_alias = function(mod, alias)
+ debug_aliases[alias] = mod
+
+ if debug_modules[mod] then
+ debug_modules[alias] = true
+ logger.infox(rspamd_config, 'enable debug for Lua module %s (%s aliased)',
+ alias, mod)
+ end
+end
+---[[[
+-- @function lua_util.get_task_verdict(task)
+-- Returns verdict for a task + score if certain, must be called from idempotent filters only
+-- Returns string:
+-- * `spam`: if message have over reject threshold and has more than one positive rule
+-- * `junk`: if a message has between score between [add_header/rewrite subject] to reject thresholds and has more than two positive rules
+-- * `passthrough`: if a message has been passed through some short-circuit rule
+-- * `ham`: if a message has overall score below junk level **and** more than three negative rule, or negative total score
+-- * `uncertain`: all other cases
+--]]
+exports.get_task_verdict = function(task)
+ local lua_verdict = require "lua_verdict"
+
+ return lua_verdict.get_default_verdict(task)
+end
+
+---[[[
+-- @function lua_util.maybe_obfuscate_string(subject, settings, prefix)
+-- Obfuscate string if enabled in settings. Also checks utf8 validity - if
+-- string is not valid utf8 then '???' is returned. Empty string returned as is.
+-- Supported settings:
+-- * <prefix>_privacy = false - subject privacy is off
+-- * <prefix>_privacy_alg = 'blake2' - default hash-algorithm to obfuscate subject
+-- * <prefix>_privacy_prefix = 'obf' - prefix to show it's obfuscated
+-- * <prefix>_privacy_length = 16 - cut the length of the hash; if 0 or fasle full hash is returned
+-- @return obfuscated or validated subject
+--]]
+
+exports.maybe_obfuscate_string = function(subject, settings, prefix)
+ local hash = require 'rspamd_cryptobox_hash'
+ if not subject or subject == '' then
+ return subject
+ elseif not rspamd_util.is_valid_utf8(subject) then
+ subject = '???'
+ elseif settings[prefix .. '_privacy'] then
+ local hash_alg = settings[prefix .. '_privacy_alg'] or 'blake2'
+ local subject_hash = hash.create_specific(hash_alg, subject)
+
+ local strip_len = settings[prefix .. '_privacy_length']
+ if strip_len and strip_len > 0 then
+ subject = subject_hash:hex():sub(1, strip_len)
+ else
+ subject = subject_hash:hex()
+ end
+
+ local privacy_prefix = settings[prefix .. '_privacy_prefix']
+ if privacy_prefix and #privacy_prefix > 0 then
+ subject = privacy_prefix .. ':' .. subject
+ end
+ end
+
+ return subject
+end
+
+---[[[
+-- @function lua_util.callback_from_string(str)
+-- Converts a string like `return function(...) end` to lua function and return true and this function
+-- or returns false + error message
+-- @return status code and function object or an error message
+--]]]
+exports.callback_from_string = function(s)
+ local loadstring = loadstring or load
+
+ if not s or #s == 0 then
+ return false, 'invalid or empty string'
+ end
+
+ s = exports.rspamd_str_trim(s)
+ local inp
+
+ if s:match('^return%s*function') then
+ -- 'return function', can be evaluated directly
+ inp = s
+ elseif s:match('^function%s*%(') then
+ inp = 'return ' .. s
+ else
+ -- Just a plain sequence
+ inp = 'return function(...)\n' .. s .. '; end'
+ end
+
+ local ret, res_or_err = pcall(loadstring(inp))
+
+ if not ret or type(res_or_err) ~= 'function' then
+ return false, res_or_err
+ end
+
+ return ret, res_or_err
+end
+
+---[[[
+-- @function lua_util.keys(t)
+-- Returns all keys from a specific table
+-- @param {table} t input table (or iterator triplet)
+-- @return array of keys
+--]]]
+exports.keys = function(gen, param, state)
+ local keys = {}
+ local i = 1
+
+ if param then
+ for k, _ in fun.iter(gen, param, state) do
+ rawset(keys, i, k)
+ i = i + 1
+ end
+ else
+ for k, _ in pairs(gen) do
+ rawset(keys, i, k)
+ i = i + 1
+ end
+ end
+
+ return keys
+end
+
+---[[[
+-- @function lua_util.values(t)
+-- Returns all values from a specific table
+-- @param {table} t input table
+-- @return array of values
+--]]]
+exports.values = function(gen, param, state)
+ local values = {}
+ local i = 1
+
+ if param then
+ for _, v in fun.iter(gen, param, state) do
+ rawset(values, i, v)
+ i = i + 1
+ end
+ else
+ for _, v in pairs(gen) do
+ rawset(values, i, v)
+ i = i + 1
+ end
+ end
+
+ return values
+end
+
+---[[[
+-- @function lua_util.distance_sorted(t1, t2)
+-- Returns distance between two sorted tables t1 and t2
+-- @param {table} t1 input table
+-- @param {table} t2 input table
+-- @return distance between `t1` and `t2`
+--]]]
+exports.distance_sorted = function(t1, t2)
+ local ncomp = #t1
+ local ndiff = 0
+ local i, j = 1, 1
+
+ if ncomp < #t2 then
+ ncomp = #t2
+ end
+
+ for _ = 1, ncomp do
+ if j > #t2 then
+ ndiff = ndiff + ncomp - #t2
+ if i > j then
+ ndiff = ndiff - (i - j)
+ end
+ break
+ elseif i > #t1 then
+ ndiff = ndiff + ncomp - #t1
+ if j > i then
+ ndiff = ndiff - (j - i)
+ end
+ break
+ end
+
+ if t1[i] == t2[j] then
+ i = i + 1
+ j = j + 1
+ elseif t1[i] < t2[j] then
+ i = i + 1
+ ndiff = ndiff + 1
+ else
+ j = j + 1
+ ndiff = ndiff + 1
+ end
+ end
+
+ return ndiff
+end
+
+---[[[
+-- @function lua_util.table_digest(t)
+-- Returns hash of all values if t[1] is string or all keys/values otherwise
+-- @param {table} t input array or map
+-- @return {string} base32 representation of blake2b hash of all strings
+--]]]
+local function table_digest(t)
+ local cr = require "rspamd_cryptobox_hash"
+ local h = cr.create()
+
+ if t[1] then
+ for _, e in ipairs(t) do
+ if type(e) == 'table' then
+ h:update(table_digest(e))
+ else
+ h:update(tostring(e))
+ end
+ end
+ else
+ for k, v in pairs(t) do
+ h:update(tostring(k))
+
+ if type(v) == 'string' then
+ h:update(v)
+ elseif type(v) == 'table' then
+ h:update(table_digest(v))
+ end
+ end
+ end
+ return h:base32()
+end
+
+exports.table_digest = table_digest
+
+---[[[
+-- @function lua_util.toboolean(v)
+-- Converts a string or a number to boolean
+-- @param {string|number} v
+-- @return {boolean} v converted to boolean
+--]]]
+exports.toboolean = function(v)
+ local true_t = {
+ ['1'] = true,
+ ['true'] = true,
+ ['TRUE'] = true,
+ ['True'] = true,
+ };
+ local false_t = {
+ ['0'] = false,
+ ['false'] = false,
+ ['FALSE'] = false,
+ ['False'] = false,
+ };
+
+ if type(v) == 'string' then
+ if true_t[v] == true then
+ return true;
+ elseif false_t[v] == false then
+ return false;
+ else
+ return false, string.format('cannot convert %q to boolean', v);
+ end
+ elseif type(v) == 'number' then
+ return v ~= 0
+ else
+ return false, string.format('cannot convert %q to boolean', v);
+ end
+end
+
+---[[[
+-- @function lua_util.config_check_local_or_authed(config, modname)
+-- Reads check_local and check_authed from the config as this is used in many modules
+-- @param {rspamd_config} config `rspamd_config` global
+-- @param {name} module name
+-- @return {boolean} v converted to boolean
+--]]]
+exports.config_check_local_or_authed = function(rspamd_config, modname, def_local, def_authed)
+ local check_local = def_local or false
+ local check_authed = def_authed or false
+
+ local function try_section(where)
+ local ret = false
+ local opts = rspamd_config:get_all_opt(where)
+ if type(opts) == 'table' then
+ if type(opts['check_local']) == 'boolean' then
+ check_local = opts['check_local']
+ ret = true
+ end
+ if type(opts['check_authed']) == 'boolean' then
+ check_authed = opts['check_authed']
+ ret = true
+ end
+ end
+
+ return ret
+ end
+
+ if not try_section(modname) then
+ try_section('options')
+ end
+
+ return { check_local, check_authed }
+end
+
+---[[[
+-- @function lua_util.is_skip_local_or_authed(task, conf[, ip])
+-- Returns `true` if local or authenticated task should be skipped for this module
+-- @param {rspamd_task} task
+-- @param {table} conf table returned from `config_check_local_or_authed`
+-- @param {rspamd_ip} ip optional ip address (can be obtained from a task)
+-- @return {boolean} true if check should be skipped
+--]]]
+exports.is_skip_local_or_authed = function(task, conf, ip)
+ if not ip then
+ ip = task:get_from_ip()
+ end
+ if not conf then
+ conf = { false, false }
+ end
+ if ((not conf[2] and task:get_user()) or
+ (not conf[1] and type(ip) == 'userdata' and ip:is_local())) then
+ return true
+ end
+
+ return false
+end
+
+---[[[
+-- @function lua_util.maybe_smtp_quote_value(str)
+-- Checks string for the forbidden elements (tspecials in RFC and quote string if needed)
+-- @param {string} str input string
+-- @return {string} original or quoted string
+--]]]
+local tspecial = lpeg.S "()<>,;:\\\"/[]?= \t\v"
+local special_match = lpeg.P((1 - tspecial) ^ 0 * tspecial ^ 1)
+exports.maybe_smtp_quote_value = function(str)
+ if special_match:match(str) then
+ return string.format('"%s"', str:gsub('"', '\\"'))
+ end
+
+ return str
+end
+
+---[[[
+-- @function lua_util.shuffle(table)
+-- Performs in-place shuffling of a table
+-- @param {table} tbl table to shuffle
+-- @return {table} same table
+--]]]
+exports.shuffle = function(tbl)
+ local size = #tbl
+ for i = size, 1, -1 do
+ local rand = math.random(size)
+ tbl[i], tbl[rand] = tbl[rand], tbl[i]
+ end
+ return tbl
+end
+
+--
+local hex_table = {}
+for idx = 0, 255 do
+ hex_table[("%02X"):format(idx)] = string.char(idx)
+ hex_table[("%02x"):format(idx)] = string.char(idx)
+end
+
+---[[[
+-- @function lua_util.unhex(str)
+-- Decode hex encoded string
+-- @param {string} str string to decode
+-- @return {string} hex decoded string (valid hex pairs are decoded, everything else is printed as is)
+--]]]
+exports.unhex = function(str)
+ return str:gsub('(..)', hex_table)
+end
+
+local http_upstream_lists = {}
+local function http_upstreams_by_url(pool, url)
+ local rspamd_url = require "rspamd_url"
+
+ local cached = http_upstream_lists[url]
+ if cached then
+ return cached
+ end
+
+ local real_url = rspamd_url.create(pool, url)
+
+ if not real_url then
+ return nil
+ end
+
+ local host = real_url:get_host()
+ local proto = real_url:get_protocol() or 'http'
+ local port = real_url:get_port() or (proto == 'https' and 443 or 80)
+ local upstream_list = require "rspamd_upstream_list"
+ local upstreams = upstream_list.create(host, port)
+
+ if upstreams then
+ http_upstream_lists[url] = upstreams
+ return upstreams
+ end
+
+ return nil
+end
+---[[[
+-- @function lua_util.http_upstreams_by_url(pool, url)
+-- Returns a cached or new upstreams list that corresponds to the specific url
+-- @param {mempool} pool memory pool to use (typically static pool from rspamd_config)
+-- @param {string} url full url
+-- @return {upstreams_list} object to get upstream from an url
+--]]]
+exports.http_upstreams_by_url = http_upstreams_by_url
+
+---[[[
+-- @function lua_util.dns_timeout_augmentation(cfg)
+-- Returns an augmentation suitable to define DNS timeout for a module
+-- @return {string} a string in format 'timeout=x' where `x` is a number of seconds for DNS timeout
+--]]]
+local function dns_timeout_augmentation(cfg)
+ return string.format('timeout=%f', cfg:get_dns_timeout() or 0.0)
+end
+
+exports.dns_timeout_augmentation = dns_timeout_augmentation
+
+---[[[
+--- @function lua_util.strip_lua_comments(lua_code)
+-- Strips single-line and multi-line comments from a given Lua code string and removes
+-- any extra spaces or newlines.
+--
+-- @param lua_code The Lua code string to strip comments from.
+-- @return The resulting Lua code string with comments and extra spaces removed.
+--
+---]]]
+local function strip_lua_comments(lua_code)
+ -- Remove single-line comments
+ lua_code = lua_code:gsub("%-%-[^\r\n]*", "")
+
+ -- Remove multi-line comments
+ lua_code = lua_code:gsub("%-%-%[%[.-%]%]", "")
+
+ -- Remove extra spaces and newlines
+ lua_code = lua_code:gsub("%s+", " ")
+
+ return lua_code
+end
+
+exports.strip_lua_comments = strip_lua_comments
+
+---[[[
+-- @function lua_util.join_path(...)
+-- Joins path components into a single path string using the appropriate separator
+-- for the current operating system.
+--
+-- @param ... Any number of path components to join together.
+-- @return A single path string, with components separated by the appropriate separator.
+--
+---]]]
+local path_sep = package.config:sub(1, 1) or '/'
+local function join_path(...)
+ local components = { ... }
+
+ -- Join components using separator
+ return table.concat(components, path_sep)
+end
+exports.join_path = join_path
+
+-- Short unit test for sanity
+if path_sep == '/' then
+ assert(join_path('/path', 'to', 'file') == '/path/to/file')
+else
+ assert(join_path('C:', 'path', 'to', 'file') == 'C:\\path\\to\\file')
+end
+
+-- Defines symbols priorities for common usage in prefilters/postfilters
+exports.symbols_priorities = {
+ top = 10, -- Symbols must be executed first (or last), such as settings
+ high = 9, -- Example: asn
+ medium = 5, -- Everything should use this as default
+ low = 0,
+}
+
+return exports
diff --git a/lualib/lua_verdict.lua b/lualib/lua_verdict.lua
new file mode 100644
index 0000000..6ce99e6
--- /dev/null
+++ b/lualib/lua_verdict.lua
@@ -0,0 +1,208 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local exports = {}
+
+---[[[
+-- @function lua_verdict.get_default_verdict(task)
+-- Returns verdict for a task + score if certain, must be called from idempotent filters only
+-- Returns string:
+-- * `spam`: if message have over reject threshold and has more than one positive rule
+-- * `junk`: if a message has between score between [add_header/rewrite subject] to reject thresholds and has more than two positive rules
+-- * `passthrough`: if a message has been passed through some short-circuit rule
+-- * `ham`: if a message has overall score below junk level **and** more than three negative rule, or negative total score
+-- * `uncertain`: all other cases
+--]]
+local function default_verdict_function(task)
+ local result = task:get_metric_result()
+
+ if result then
+
+ if result.passthrough then
+ return 'passthrough', nil
+ end
+
+ local score = result.score
+
+ local action = result.action
+
+ if action == 'reject' and result.npositive > 1 then
+ return 'spam', score
+ elseif action == 'no action' then
+ if score < 0 or result.nnegative > 3 then
+ return 'ham', score
+ end
+ else
+ -- All colors of junk
+ if action == 'add header' or action == 'rewrite subject' then
+ if result.npositive > 2 then
+ return 'junk', score
+ end
+ end
+ end
+
+ return 'uncertain', score
+ end
+end
+
+local default_possible_verdicts = {
+ passthrough = {
+ can_learn = false,
+ description = 'message has passthrough result',
+ },
+ spam = {
+ can_learn = 'spam',
+ description = 'message is likely spam',
+ },
+ junk = {
+ can_learn = 'spam',
+ description = 'message is likely possible spam',
+ },
+ ham = {
+ can_learn = 'ham',
+ description = 'message is likely ham',
+ },
+ uncertain = {
+ can_learn = false,
+ description = 'not certainty in verdict'
+ }
+}
+
+-- Verdict functions specific for modules
+local specific_verdicts = {
+ default = {
+ callback = default_verdict_function,
+ possible_verdicts = default_possible_verdicts
+ }
+}
+
+local default_verdict = specific_verdicts.default
+
+exports.get_default_verdict = default_verdict.callback
+exports.set_verdict_function = function(func, what)
+ assert(type(func) == 'function')
+ if not what then
+ -- Default verdict
+ local existing = specific_verdicts.default.callback
+ specific_verdicts.default.callback = func
+ exports.get_default_verdict = func
+
+ return existing
+ else
+ local existing = specific_verdicts[what]
+
+ if not existing then
+ specific_verdicts[what] = {
+ callback = func,
+ possible_verdicts = default_possible_verdicts
+ }
+ else
+ existing = existing.callback
+ end
+
+ specific_verdicts[what].callback = func
+ return existing
+ end
+end
+
+exports.set_verdict_table = function(verdict_tbl, what)
+ assert(type(verdict_tbl) == 'table' and
+ type(verdict_tbl.callback) == 'function' and
+ type(verdict_tbl.possible_verdicts) == 'table')
+
+ if not what then
+ -- Default verdict
+ local existing = specific_verdicts.default
+ specific_verdicts.default = verdict_tbl
+ exports.get_default_verdict = specific_verdicts.default.callback
+
+ return existing
+ else
+ local existing = specific_verdicts[what]
+ specific_verdicts[what] = verdict_tbl
+ return existing
+ end
+end
+
+exports.get_specific_verdict = function(what, task)
+ if specific_verdicts[what] then
+ return specific_verdicts[what].callback(task)
+ end
+
+ return exports.get_default_verdict(task)
+end
+
+exports.get_possible_verdicts = function(what)
+ local lua_util = require "lua_util"
+ if what then
+ if specific_verdicts[what] then
+ return lua_util.keys(specific_verdicts[what].possible_verdicts)
+ end
+ else
+ return lua_util.keys(specific_verdicts.default.possible_verdicts)
+ end
+
+ return nil
+end
+
+exports.can_learn = function(verdict, what)
+ if what then
+ if specific_verdicts[what] and specific_verdicts[what].possible_verdicts[verdict] then
+ return specific_verdicts[what].possible_verdicts[verdict].can_learn
+ end
+ else
+ if specific_verdicts.default.possible_verdicts[verdict] then
+ return specific_verdicts.default.possible_verdicts[verdict].can_learn
+ end
+ end
+
+ return nil -- To distinguish from `false` that could happen in can_learn
+end
+
+exports.describe = function(verdict, what)
+ if what then
+ if specific_verdicts[what] and specific_verdicts[what].possible_verdicts[verdict] then
+ return specific_verdicts[what].possible_verdicts[verdict].description
+ end
+ else
+ if specific_verdicts.default.possible_verdicts[verdict] then
+ return specific_verdicts.default.possible_verdicts[verdict].description
+ end
+ end
+
+ return nil
+end
+
+---[[[
+-- @function lua_verdict.adjust_passthrough_action(task)
+-- If an action is `soft reject` then this function extracts a module that has set this action
+-- and returns an adjusted action (e.g. 'greylist' or 'ratelimit').
+-- Otherwise an action is returned as is.
+--]]
+exports.adjust_passthrough_action = function(task)
+ local action = task:get_metric_action()
+ if action == 'soft reject' then
+ local has_pr, _, _, module = task:has_pre_result()
+
+ if has_pr and module then
+ action = module
+ end
+ end
+
+ return action
+end
+
+return exports \ No newline at end of file
diff --git a/lualib/plugins/dmarc.lua b/lualib/plugins/dmarc.lua
new file mode 100644
index 0000000..7791f4e
--- /dev/null
+++ b/lualib/plugins/dmarc.lua
@@ -0,0 +1,359 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2015-2016, Andrew Lewis <nerf@judo.za.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Common dmarc stuff
+local rspamd_logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local N = "dmarc"
+
+local exports = {}
+
+exports.default_settings = {
+ auth_and_local_conf = false,
+ symbols = {
+ spf_allow_symbol = 'R_SPF_ALLOW',
+ spf_deny_symbol = 'R_SPF_FAIL',
+ spf_softfail_symbol = 'R_SPF_SOFTFAIL',
+ spf_neutral_symbol = 'R_SPF_NEUTRAL',
+ spf_tempfail_symbol = 'R_SPF_DNSFAIL',
+ spf_permfail_symbol = 'R_SPF_PERMFAIL',
+ spf_na_symbol = 'R_SPF_NA',
+
+ dkim_allow_symbol = 'R_DKIM_ALLOW',
+ dkim_deny_symbol = 'R_DKIM_REJECT',
+ dkim_tempfail_symbol = 'R_DKIM_TEMPFAIL',
+ dkim_na_symbol = 'R_DKIM_NA',
+ dkim_permfail_symbol = 'R_DKIM_PERMFAIL',
+
+ -- DMARC symbols
+ allow = 'DMARC_POLICY_ALLOW',
+ badpolicy = 'DMARC_BAD_POLICY',
+ dnsfail = 'DMARC_DNSFAIL',
+ na = 'DMARC_NA',
+ reject = 'DMARC_POLICY_REJECT',
+ softfail = 'DMARC_POLICY_SOFTFAIL',
+ quarantine = 'DMARC_POLICY_QUARANTINE',
+ },
+ no_sampling_domains = nil,
+ no_reporting_domains = nil,
+ reporting = {
+ report_local_controller = false, -- Store reports for local/controller scans (for testing only)
+ redis_keys = {
+ index_prefix = 'dmarc_idx',
+ report_prefix = 'dmarc_rpt',
+ join_char = ';',
+ },
+ helo = 'rspamd.localhost',
+ smtp = '127.0.0.1',
+ smtp_port = 25,
+ retries = 2,
+ from_name = 'Rspamd',
+ msgid_from = 'rspamd',
+ enabled = false,
+ max_entries = 1000,
+ keys_expire = 172800,
+ only_domains = nil,
+ },
+ actions = {},
+}
+
+
+-- Returns a key used to be inserted into dmarc report sample
+exports.dmarc_report = function(task, settings, data)
+ local rspamd_lua_utils = require "lua_util"
+ local E = {}
+
+ local ip = task:get_from_ip()
+ if not ip or not ip:is_valid() then
+ rspamd_logger.infox(task, 'cannot store dmarc report for %s: no valid source IP',
+ data.domain)
+ return nil
+ end
+
+ ip = ip:to_string()
+
+ if rspamd_lua_utils.is_rspamc_or_controller(task) and not settings.reporting.report_local_controller then
+ rspamd_logger.infox(task, 'cannot store dmarc report for %s from IP %s: has come from controller/rspamc',
+ data.domain, ip)
+ return
+ end
+
+ local dkim_pass = table.concat(data.dkim_results.pass or E, '|')
+ local dkim_fail = table.concat(data.dkim_results.fail or E, '|')
+ local dkim_temperror = table.concat(data.dkim_results.temperror or E, '|')
+ local dkim_permerror = table.concat(data.dkim_results.permerror or E, '|')
+ local disposition_to_return = data.disposition
+ local res = table.concat({
+ ip, data.spf_ok, data.dkim_ok,
+ disposition_to_return, (data.sampled_out and 'sampled_out' or ''), data.domain,
+ dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, data.spf_domain, data.spf_result }, ',')
+
+ return res
+end
+
+exports.gen_munging_callback = function(munging_opts, settings)
+ local rspamd_util = require "rspamd_util"
+ local lua_mime = require "lua_mime"
+ return function(task)
+ if munging_opts.mitigate_allow_only then
+ if not task:has_symbol(settings.symbols.allow) then
+ lua_util.debugm(N, task, 'skip munging, no %s symbol',
+ settings.symbols.allow)
+ -- Excepted
+ return
+ end
+ else
+ local has_dmarc = task:has_symbol(settings.symbols.allow) or
+ task:has_symbol(settings.symbols.quarantine) or
+ task:has_symbol(settings.symbols.reject) or
+ task:has_symbol(settings.symbols.softfail)
+
+ if not has_dmarc then
+ lua_util.debugm(N, task, 'skip munging, no %s symbol',
+ settings.symbols.allow)
+ -- Excepted
+ return
+ end
+ end
+ if munging_opts.mitigate_strict_only then
+ local s = task:get_symbol(settings.symbols.allow) or { [1] = {} }
+ local sopts = s[1].options or {}
+
+ local seen_strict
+ for _, o in ipairs(sopts) do
+ if o == 'reject' or o == 'quarantine' then
+ seen_strict = true
+ break
+ end
+ end
+
+ if not seen_strict then
+ lua_util.debugm(N, task, 'skip munging, no strict policy found in %s',
+ settings.symbols.allow)
+ -- Excepted
+ return
+ end
+ end
+ if munging_opts.munge_map_condition then
+ local accepted, trace = munging_opts.munge_map_condition:process(task)
+ if not accepted then
+ lua_util.debugm(N, task, 'skip munging, maps condition not satisfied: (%s)',
+ trace)
+ -- Excepted
+ return
+ end
+ end
+ -- Now, look for domain for munging
+ local mr = task:get_recipients({ 'mime', 'orig' })
+ local rcpt_found
+ if mr then
+ for _, r in ipairs(mr) do
+ if r.domain and munging_opts.list_map:get_key(r.addr) then
+ rcpt_found = r
+ break
+ end
+ end
+ end
+
+ if not rcpt_found then
+ lua_util.debugm(N, task, 'skip munging, recipients are not in list_map')
+ -- Excepted
+ return
+ end
+
+ local from = task:get_from({ 'mime', 'orig' })
+
+ if not from or not from[1] then
+ lua_util.debugm(N, task, 'skip munging, from is bad')
+ -- Excepted
+ return
+ end
+
+ from = from[1]
+ local via_user = rcpt_found.user
+ local via_addr = rcpt_found.addr
+ local via_name
+
+ if from.name == "" then
+ via_name = string.format('%s via %s', from.user or 'unknown', via_user)
+ else
+ via_name = string.format('%s via %s', from.name, via_user)
+ end
+
+ local hdr_encoded = rspamd_util.fold_header('From',
+ rspamd_util.mime_header_encode(string.format('%s <%s>',
+ via_name, via_addr)), task:get_newlines_type())
+ local orig_from_encoded = rspamd_util.fold_header('X-Original-From',
+ rspamd_util.mime_header_encode(string.format('%s <%s>',
+ from.name or '', from.addr)), task:get_newlines_type())
+ local add_hdrs = {
+ ['From'] = { order = 1, value = hdr_encoded },
+ }
+ local remove_hdrs = { ['From'] = 0 }
+
+ local nreply = from.addr
+ if munging_opts.reply_goes_to_list then
+ -- Reply-to goes to the list
+ nreply = via_addr
+ end
+
+ if task:has_header('Reply-To') then
+ -- If we have reply-to header, then we need to insert an additional
+ -- address there
+ local orig_reply = task:get_header_full('Reply-To')[1]
+ if orig_reply.value then
+ nreply = string.format('%s, %s', orig_reply.value, nreply)
+ end
+ remove_hdrs['Reply-To'] = 1
+ end
+
+ add_hdrs['Reply-To'] = { order = 0, value = nreply }
+
+ add_hdrs['X-Original-From'] = { order = 0, value = orig_from_encoded }
+ lua_mime.modify_headers(task, {
+ remove = remove_hdrs,
+ add = add_hdrs
+ })
+ lua_util.debugm(N, task, 'munged DMARC header for %s: %s -> %s',
+ from.domain, hdr_encoded, from.addr)
+ rspamd_logger.infox(task, 'munged DMARC header for %s', from.addr)
+ task:insert_result('DMARC_MUNGED', 1.0, from.addr)
+ end
+end
+
+local function gen_dmarc_grammar()
+ local lpeg = require "lpeg"
+ lpeg.locale(lpeg)
+ local space = lpeg.space ^ 0
+ local name = lpeg.C(lpeg.alpha ^ 1) * space
+ local sep = space * (lpeg.S("\\;") * space) + (lpeg.space ^ 1)
+ local value = lpeg.C(lpeg.P(lpeg.graph - sep) ^ 1)
+ local pair = lpeg.Cg(name * "=" * space * value) * sep ^ -1
+ local list = lpeg.Cf(lpeg.Ct("") * pair ^ 0, rawset)
+ local version = lpeg.P("v") * space * lpeg.P("=") * space * lpeg.P("DMARC1")
+ local record = version * sep * list
+
+ return record
+end
+
+local dmarc_grammar = gen_dmarc_grammar()
+
+local function dmarc_key_value_case(elts)
+ if type(elts) ~= "table" then
+ return elts
+ end
+ local result = {}
+ for k, v in pairs(elts) do
+ k = k:lower()
+ if k ~= "v" then
+ v = v:lower()
+ end
+
+ result[k] = v
+ end
+
+ return result
+end
+
+--[[
+-- Used to check dmarc record, check elements and produce dmarc policy processed
+-- result.
+-- Returns:
+-- false,false - record is garbage
+-- false,error_message - record is invalid
+-- true,policy_table - record is valid and parsed
+]]
+local function dmarc_check_record(log_obj, record, is_tld)
+ local failed_policy
+ local result = {
+ dmarc_policy = 'none'
+ }
+
+ local elts = dmarc_grammar:match(record)
+ lua_util.debugm(N, log_obj, "got DMARC record: %s, tld_flag=%s, processed=%s",
+ record, is_tld, elts)
+
+ if elts then
+ elts = dmarc_key_value_case(elts)
+
+ local dkim_pol = elts['adkim']
+ if dkim_pol then
+ if dkim_pol == 's' then
+ result.strict_dkim = true
+ elseif dkim_pol ~= 'r' then
+ failed_policy = 'adkim tag has invalid value: ' .. dkim_pol
+ return false, failed_policy
+ end
+ end
+
+ local spf_pol = elts['aspf']
+ if spf_pol then
+ if spf_pol == 's' then
+ result.strict_spf = true
+ elseif spf_pol ~= 'r' then
+ failed_policy = 'aspf tag has invalid value: ' .. spf_pol
+ return false, failed_policy
+ end
+ end
+
+ local policy = elts['p']
+ if policy then
+ if (policy == 'reject') then
+ result.dmarc_policy = 'reject'
+ elseif (policy == 'quarantine') then
+ result.dmarc_policy = 'quarantine'
+ elseif (policy ~= 'none') then
+ failed_policy = 'p tag has invalid value: ' .. policy
+ return false, failed_policy
+ end
+ end
+
+ -- Adjust policy if we are in tld mode
+ local subdomain_policy = elts['sp']
+ if elts['sp'] and is_tld then
+ result.subdomain_policy = elts['sp']
+
+ if (subdomain_policy == 'reject') then
+ result.dmarc_policy = 'reject'
+ elseif (subdomain_policy == 'quarantine') then
+ result.dmarc_policy = 'quarantine'
+ elseif (subdomain_policy == 'none') then
+ result.dmarc_policy = 'none'
+ elseif (subdomain_policy ~= 'none') then
+ failed_policy = 'sp tag has invalid value: ' .. subdomain_policy
+ return false, failed_policy
+ end
+ end
+ result.pct = elts['pct']
+ if result.pct then
+ result.pct = tonumber(result.pct)
+ end
+
+ if elts.rua then
+ result.rua = elts['rua']
+ end
+ result.raw_elts = elts
+ else
+ return false, false -- Ignore garbage
+ end
+
+ return true, result
+end
+
+exports.dmarc_check_record = dmarc_check_record
+
+return exports
diff --git a/lualib/plugins/neural.lua b/lualib/plugins/neural.lua
new file mode 100644
index 0000000..6e88ef2
--- /dev/null
+++ b/lualib/plugins/neural.lua
@@ -0,0 +1,892 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local fun = require "fun"
+local lua_redis = require "lua_redis"
+local lua_settings = require "lua_settings"
+local lua_util = require "lua_util"
+local meta_functions = require "lua_meta"
+local rspamd_kann = require "rspamd_kann"
+local rspamd_logger = require "rspamd_logger"
+local rspamd_tensor = require "rspamd_tensor"
+local rspamd_util = require "rspamd_util"
+local ucl = require "ucl"
+
+local N = 'neural'
+
+-- Used in prefix to avoid wrong ANN to be loaded
+local plugin_ver = '2'
+
+-- Module vars
+local default_options = {
+ train = {
+ max_trains = 1000,
+ max_epoch = 1000,
+ max_usages = 10,
+ max_iterations = 25, -- Torch style
+ mse = 0.001,
+ autotrain = true,
+ train_prob = 1.0,
+ learn_threads = 1,
+ learn_mode = 'balanced', -- Possible values: balanced, proportional
+ learning_rate = 0.01,
+ classes_bias = 0.0, -- balanced mode: what difference is allowed between classes (1:1 proportion means 0 bias)
+ spam_skip_prob = 0.0, -- proportional mode: spam skip probability (0-1)
+ ham_skip_prob = 0.0, -- proportional mode: ham skip probability
+ store_pool_only = false, -- store tokens in cache only (disables autotrain);
+ -- neural_vec_mpack stores vector of training data in messagepack neural_profile_digest stores profile digest
+ },
+ watch_interval = 60.0,
+ lock_expire = 600,
+ learning_spawned = false,
+ ann_expire = 60 * 60 * 24 * 2, -- 2 days
+ hidden_layer_mult = 1.5, -- number of neurons in the hidden layer
+ roc_enabled = false, -- Use ROC to find the best possible thresholds for ham and spam. If spam_score_threshold or ham_score_threshold is defined, it takes precedence over ROC thresholds.
+ roc_misclassification_cost = 0.5, -- Cost of misclassifying a spam message (must be 0..1).
+ spam_score_threshold = nil, -- neural score threshold for spam (must be 0..1 or nil to disable)
+ ham_score_threshold = nil, -- neural score threshold for ham (must be 0..1 or nil to disable)
+ flat_threshold_curve = false, -- use binary classification 0/1 when threshold is reached
+ symbol_spam = 'NEURAL_SPAM',
+ symbol_ham = 'NEURAL_HAM',
+ max_inputs = nil, -- when PCA is used
+ blacklisted_symbols = {}, -- list of symbols skipped in neural processing
+}
+
+-- Rule structure:
+-- * static config fields (see `default_options`)
+-- * prefix - name or defined prefix
+-- * settings - table of settings indexed by settings id, -1 is used when no settings defined
+
+-- Rule settings element defines elements for specific settings id:
+-- * symbols - static symbols profile (defined by config or extracted from symcache)
+-- * name - name of settings id
+-- * digest - digest of all symbols
+-- * ann - dynamic ANN configuration loaded from Redis
+-- * train - train data for ANN (e.g. the currently trained ANN)
+
+-- Settings ANN table is loaded from Redis and represents dynamic profile for ANN
+-- Some elements are directly stored in Redis, ANN is, in turn loaded dynamically
+-- * version - version of ANN loaded from redis
+-- * redis_key - name of ANN key in Redis
+-- * symbols - symbols in THIS PARTICULAR ANN (might be different from set.symbols)
+-- * distance - distance between set.symbols and set.ann.symbols
+-- * ann - kann object
+
+local settings = {
+ rules = {},
+ prefix = 'rn', -- Neural network default prefix
+ max_profiles = 3, -- Maximum number of NN profiles stored
+}
+
+-- Get module & Redis configuration
+local module_config = rspamd_config:get_all_opt(N)
+settings = lua_util.override_defaults(settings, module_config)
+local redis_params = lua_redis.parse_redis_server('neural')
+
+local redis_lua_script_vectors_len = "neural_train_size.lua"
+local redis_lua_script_maybe_invalidate = "neural_maybe_invalidate.lua"
+local redis_lua_script_maybe_lock = "neural_maybe_lock.lua"
+local redis_lua_script_save_unlock = "neural_save_unlock.lua"
+
+local redis_script_id = {}
+
+local function load_scripts()
+ redis_script_id.vectors_len = lua_redis.load_redis_script_from_file(redis_lua_script_vectors_len,
+ redis_params)
+ redis_script_id.maybe_invalidate = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_invalidate,
+ redis_params)
+ redis_script_id.maybe_lock = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_lock,
+ redis_params)
+ redis_script_id.save_unlock = lua_redis.load_redis_script_from_file(redis_lua_script_save_unlock,
+ redis_params)
+end
+
+local function create_ann(n, nlayers, rule)
+ -- We ignore number of layers so far when using kann
+ local nhidden = math.floor(n * (rule.hidden_layer_mult or 1.0) + 1.0)
+ local t = rspamd_kann.layer.input(n)
+ t = rspamd_kann.transform.relu(t)
+ t = rspamd_kann.layer.dense(t, nhidden);
+ t = rspamd_kann.layer.cost(t, 1, rspamd_kann.cost.ceb_neg)
+ return rspamd_kann.new.kann(t)
+end
+
+-- Fills ANN data for a specific settings element
+local function fill_set_ann(set, ann_key)
+ if not set.ann then
+ set.ann = {
+ symbols = set.symbols,
+ distance = 0,
+ digest = set.digest,
+ redis_key = ann_key,
+ version = 0,
+ }
+ end
+end
+
+-- This function takes all inputs, applies PCA transformation and returns the final
+-- PCA matrix as rspamd_tensor
+local function learn_pca(inputs, max_inputs)
+ local scatter_matrix = rspamd_tensor.scatter_matrix(rspamd_tensor.fromtable(inputs))
+ local eigenvals = scatter_matrix:eigen()
+ -- scatter matrix is not filled with eigenvectors
+ lua_util.debugm(N, 'eigenvalues: %s', eigenvals)
+ local w = rspamd_tensor.new(2, max_inputs, #scatter_matrix[1])
+ for i = 1, max_inputs do
+ w[i] = scatter_matrix[#scatter_matrix - i + 1]
+ end
+
+ lua_util.debugm(N, 'pca matrix: %s', w)
+
+ return w
+end
+
+-- This function computes optimal threshold using ROC for the given set of inputs.
+-- Returns a threshold that minimizes:
+-- alpha * (false_positive_rate) + beta * (false_negative_rate)
+-- Where alpha is cost of false positive result
+-- beta is cost of false negative result
+local function get_roc_thresholds(ann, inputs, outputs, alpha, beta)
+
+ -- Sorts list x and list y based on the values in list x.
+ local sort_relative = function(x, y)
+
+ local r = {}
+
+ assert(#x == #y)
+ local n = #x
+
+ local a = {}
+ local b = {}
+ for i = 1, n do
+ r[i] = i
+ end
+
+ local cmp = function(p, q)
+ return p < q
+ end
+
+ table.sort(r, function(p, q)
+ return cmp(x[p], x[q])
+ end)
+
+ for i = 1, n do
+ a[i] = x[r[i]]
+ b[i] = y[r[i]]
+ end
+
+ return a, b
+ end
+
+ local function get_scores(nn, input_vectors)
+ local scores = {}
+ for i = 1, #inputs do
+ local score = nn:apply1(input_vectors[i], nn.pca)[1]
+ scores[#scores + 1] = score
+ end
+
+ return scores
+ end
+
+ local fpr = {}
+ local fnr = {}
+ local scores = get_scores(ann, inputs)
+
+ scores, outputs = sort_relative(scores, outputs)
+
+ local n_samples = #outputs
+ local n_spam = 0
+ local n_ham = 0
+ local ham_count_ahead = {}
+ local spam_count_ahead = {}
+ local ham_count_behind = {}
+ local spam_count_behind = {}
+
+ ham_count_ahead[n_samples + 1] = 0
+ spam_count_ahead[n_samples + 1] = 0
+
+ for i = n_samples, 1, -1 do
+
+ if outputs[i][1] == 0 then
+ n_ham = n_ham + 1
+ ham_count_ahead[i] = 1
+ spam_count_ahead[i] = 0
+ else
+ n_spam = n_spam + 1
+ ham_count_ahead[i] = 0
+ spam_count_ahead[i] = 1
+ end
+
+ ham_count_ahead[i] = ham_count_ahead[i] + ham_count_ahead[i + 1]
+ spam_count_ahead[i] = spam_count_ahead[i] + spam_count_ahead[i + 1]
+ end
+
+ for i = 1, n_samples do
+ if outputs[i][1] == 0 then
+ ham_count_behind[i] = 1
+ spam_count_behind[i] = 0
+ else
+ ham_count_behind[i] = 0
+ spam_count_behind[i] = 1
+ end
+
+ if i ~= 1 then
+ ham_count_behind[i] = ham_count_behind[i] + ham_count_behind[i - 1]
+ spam_count_behind[i] = spam_count_behind[i] + spam_count_behind[i - 1]
+ end
+ end
+
+ for i = 1, n_samples do
+ fpr[i] = 0
+ fnr[i] = 0
+
+ if (ham_count_ahead[i + 1] + ham_count_behind[i]) ~= 0 then
+ fpr[i] = ham_count_ahead[i + 1] / (ham_count_ahead[i + 1] + ham_count_behind[i])
+ end
+
+ if (spam_count_behind[i] + spam_count_ahead[i + 1]) ~= 0 then
+ fnr[i] = spam_count_behind[i] / (spam_count_behind[i] + spam_count_ahead[i + 1])
+ end
+ end
+
+ local p = n_spam / (n_spam + n_ham)
+
+ local cost = {}
+ local min_cost_idx = 0
+ local min_cost = math.huge
+ for i = 1, n_samples do
+ cost[i] = ((1 - p) * alpha * fpr[i]) + (p * beta * fnr[i])
+ if min_cost >= cost[i] then
+ min_cost = cost[i]
+ min_cost_idx = i
+ end
+ end
+
+ return scores[min_cost_idx]
+end
+
+-- This function is intended to extend lock for ANN during training
+-- It registers periodic that increases locked key each 30 seconds unless
+-- `set.learning_spawned` is set to `true`
+local function register_lock_extender(rule, set, ev_base, ann_key)
+ rspamd_config:add_periodic(ev_base, 30.0,
+ function()
+ local function redis_lock_extend_cb(err, _)
+ if err then
+ rspamd_logger.errx(rspamd_config, 'cannot lock ANN %s from redis: %s',
+ ann_key, err)
+ else
+ rspamd_logger.infox(rspamd_config, 'extend lock for ANN %s for 30 seconds',
+ ann_key)
+ end
+ end
+
+ if set.learning_spawned then
+ lua_redis.redis_make_request_taskless(ev_base,
+ rspamd_config,
+ rule.redis,
+ nil,
+ true, -- is write
+ redis_lock_extend_cb, --callback
+ 'HINCRBY', -- command
+ { ann_key, 'lock', '30' }
+ )
+ else
+ lua_util.debugm(N, rspamd_config, "stop lock extension as learning_spawned is false")
+ return false -- do not plan any more updates
+ end
+
+ return true
+ end
+ )
+end
+
+local function can_push_train_vector(rule, task, learn_type, nspam, nham)
+ local train_opts = rule.train
+ local coin = math.random()
+
+ if train_opts.train_prob and coin < 1.0 - train_opts.train_prob then
+ rspamd_logger.infox(task, 'probabilistically skip sample: %s', coin)
+ return false
+ end
+
+ if train_opts.learn_mode == 'balanced' then
+ -- Keep balanced training set based on number of spam and ham samples
+ if learn_type == 'spam' then
+ if nspam <= train_opts.max_trains then
+ if nspam > nham then
+ -- Apply sampling
+ local skip_rate = 1.0 - nham / (nspam + 1)
+ if coin < skip_rate - train_opts.classes_bias then
+ rspamd_logger.infox(task,
+ 'skip %s sample to keep spam/ham balance; probability %s; %s spam and %s ham vectors stored',
+ learn_type,
+ skip_rate - train_opts.classes_bias,
+ nspam, nham)
+ return false
+ end
+ end
+ return true
+ else
+ -- Enough learns
+ rspamd_logger.infox(task, 'skip %s sample to keep spam/ham balance; too many spam samples: %s',
+ learn_type,
+ nspam)
+ end
+ else
+ if nham <= train_opts.max_trains then
+ if nham > nspam then
+ -- Apply sampling
+ local skip_rate = 1.0 - nspam / (nham + 1)
+ if coin < skip_rate - train_opts.classes_bias then
+ rspamd_logger.infox(task,
+ 'skip %s sample to keep spam/ham balance; probability %s; %s spam and %s ham vectors stored',
+ learn_type,
+ skip_rate - train_opts.classes_bias,
+ nspam, nham)
+ return false
+ end
+ end
+ return true
+ else
+ rspamd_logger.infox(task, 'skip %s sample to keep spam/ham balance; too many ham samples: %s', learn_type,
+ nham)
+ end
+ end
+ else
+ -- Probabilistic learn mode, we just skip learn if we already have enough samples or
+ -- if our coin drop is less than desired probability
+ if learn_type == 'spam' then
+ if nspam <= train_opts.max_trains then
+ if train_opts.spam_skip_prob then
+ if coin <= train_opts.spam_skip_prob then
+ rspamd_logger.infox(task, 'skip %s sample probabilistically; probability %s (%s skip chance)', learn_type,
+ coin, train_opts.spam_skip_prob)
+ return false
+ end
+
+ return true
+ end
+ else
+ rspamd_logger.infox(task, 'skip %s sample; too many spam samples: %s (%s limit)', learn_type,
+ nspam, train_opts.max_trains)
+ end
+ else
+ if nham <= train_opts.max_trains then
+ if train_opts.ham_skip_prob then
+ if coin <= train_opts.ham_skip_prob then
+ rspamd_logger.infox(task, 'skip %s sample probabilistically; probability %s (%s skip chance)', learn_type,
+ coin, train_opts.ham_skip_prob)
+ return false
+ end
+
+ return true
+ end
+ else
+ rspamd_logger.infox(task, 'skip %s sample; too many ham samples: %s (%s limit)', learn_type,
+ nham, train_opts.max_trains)
+ end
+ end
+ end
+
+ return false
+end
+
+-- Closure generator for unlock function
+local function gen_unlock_cb(rule, set, ann_key)
+ return function(err)
+ if err then
+ rspamd_logger.errx(rspamd_config, 'cannot unlock ANN %s:%s at %s from redis: %s',
+ rule.prefix, set.name, ann_key, err)
+ else
+ lua_util.debugm(N, rspamd_config, 'unlocked ANN %s:%s at %s',
+ rule.prefix, set.name, ann_key)
+ end
+ end
+end
+
+-- Used to generate new ANN key for specific profile
+local function new_ann_key(rule, set, version)
+ local ann_key = string.format('%s_%s_%s_%s_%s', settings.prefix,
+ rule.prefix, set.name, set.digest:sub(1, 8), tostring(version))
+
+ return ann_key
+end
+
+local function redis_ann_prefix(rule, settings_name)
+ -- We also need to count metatokens:
+ local n = meta_functions.version
+ return string.format('%s%d_%s_%d_%s',
+ settings.prefix, plugin_ver, rule.prefix, n, settings_name)
+end
+
+-- This function receives training vectors, checks them, spawn learning and saves ANN in Redis
+local function spawn_train(params)
+ -- Check training data sanity
+ -- Now we need to join inputs and create the appropriate test vectors
+ local n = #params.set.symbols +
+ meta_functions.rspamd_count_metatokens()
+
+ -- Now we can train ann
+ local train_ann = create_ann(params.rule.max_inputs or n, 3, params.rule)
+
+ if #params.ham_vec + #params.spam_vec < params.rule.train.max_trains / 2 then
+ -- Invalidate ANN as it is definitely invalid
+ -- TODO: add invalidation
+ assert(false)
+ else
+ local inputs, outputs = {}, {}
+
+ -- Used to show parsed vectors in a convenient format (for debugging only)
+ local function debug_vec(t)
+ local ret = {}
+ for i, v in ipairs(t) do
+ if v ~= 0 then
+ ret[#ret + 1] = string.format('%d=%.2f', i, v)
+ end
+ end
+
+ return ret
+ end
+
+ -- Make training set by joining vectors
+ -- KANN automatically shuffles those samples
+ -- 1.0 is used for spam and -1.0 is used for ham
+ -- It implies that output layer can express that (e.g. tanh output)
+ for _, e in ipairs(params.spam_vec) do
+ inputs[#inputs + 1] = e
+ outputs[#outputs + 1] = { 1.0 }
+ --rspamd_logger.debugm(N, rspamd_config, 'spam vector: %s', debug_vec(e))
+ end
+ for _, e in ipairs(params.ham_vec) do
+ inputs[#inputs + 1] = e
+ outputs[#outputs + 1] = { -1.0 }
+ --rspamd_logger.debugm(N, rspamd_config, 'ham vector: %s', debug_vec(e))
+ end
+
+ -- Called in child process
+ local function train()
+ local log_thresh = params.rule.train.max_iterations / 10
+ local seen_nan = false
+
+ local function train_cb(iter, train_cost, value_cost)
+ if (iter * (params.rule.train.max_iterations / log_thresh)) % (params.rule.train.max_iterations) == 0 then
+ if train_cost ~= train_cost and not seen_nan then
+ -- We have nan :( try to log lot's of stuff to dig into a problem
+ seen_nan = true
+ rspamd_logger.errx(rspamd_config, 'ANN %s:%s: train error: observed nan in error cost!; value cost = %s',
+ params.rule.prefix, params.set.name,
+ value_cost)
+ for i, e in ipairs(inputs) do
+ lua_util.debugm(N, rspamd_config, 'train vector %s -> %s',
+ debug_vec(e), outputs[i][1])
+ end
+ end
+
+ rspamd_logger.infox(rspamd_config,
+ "ANN %s:%s: learned from %s redis key in %s iterations, error: %s, value cost: %s",
+ params.rule.prefix, params.set.name,
+ params.ann_key,
+ iter,
+ train_cost,
+ value_cost)
+ end
+ end
+
+ lua_util.debugm(N, rspamd_config, "subprocess to learn ANN %s:%s has been started",
+ params.rule.prefix, params.set.name)
+
+ local pca
+ if params.rule.max_inputs then
+ -- Train PCA in the main process, presumably it is not that long
+ lua_util.debugm(N, rspamd_config, "start PCA train for ANN %s:%s",
+ params.rule.prefix, params.set.name)
+ pca = learn_pca(inputs, params.rule.max_inputs)
+ end
+
+ lua_util.debugm(N, rspamd_config, "start neural train for ANN %s:%s",
+ params.rule.prefix, params.set.name)
+ local ret, err = pcall(train_ann.train1, train_ann,
+ inputs, outputs, {
+ lr = params.rule.train.learning_rate,
+ max_epoch = params.rule.train.max_iterations,
+ cb = train_cb,
+ pca = pca
+ })
+
+ if not ret then
+ rspamd_logger.errx(rspamd_config, "cannot train ann %s:%s: %s",
+ params.rule.prefix, params.set.name, err)
+
+ return nil
+ else
+ lua_util.debugm(N, rspamd_config, "finished neural train for ANN %s:%s",
+ params.rule.prefix, params.set.name)
+ end
+
+ local roc_thresholds = {}
+ if params.rule.roc_enabled then
+ local spam_threshold = get_roc_thresholds(train_ann,
+ inputs,
+ outputs,
+ 1 - params.rule.roc_misclassification_cost,
+ params.rule.roc_misclassification_cost)
+ local ham_threshold = get_roc_thresholds(train_ann,
+ inputs,
+ outputs,
+ params.rule.roc_misclassification_cost,
+ 1 - params.rule.roc_misclassification_cost)
+ roc_thresholds = { spam_threshold, ham_threshold }
+
+ rspamd_logger.messagex("ROC thresholds: (spam_threshold: %s, ham_threshold: %s)",
+ roc_thresholds[1], roc_thresholds[2])
+ end
+
+ if not seen_nan then
+ -- Convert to strings as ucl cannot rspamd_text properly
+ local pca_data
+ if pca then
+ pca_data = tostring(pca:save())
+ end
+ local out = {
+ ann_data = tostring(train_ann:save()),
+ pca_data = pca_data,
+ roc_thresholds = roc_thresholds,
+ }
+
+ local final_data = ucl.to_format(out, 'msgpack')
+ lua_util.debugm(N, rspamd_config, "subprocess for ANN %s:%s returned %s bytes",
+ params.rule.prefix, params.set.name, #final_data)
+ return final_data
+ else
+ return nil
+ end
+ end
+
+ params.set.learning_spawned = true
+
+ local function redis_save_cb(err)
+ if err then
+ rspamd_logger.errx(rspamd_config, 'cannot save ANN %s:%s to redis key %s: %s',
+ params.rule.prefix, params.set.name, params.ann_key, err)
+ lua_redis.redis_make_request_taskless(params.ev_base,
+ rspamd_config,
+ params.rule.redis,
+ nil,
+ false, -- is write
+ gen_unlock_cb(params.rule, params.set, params.ann_key), --callback
+ 'HDEL', -- command
+ { params.ann_key, 'lock' }
+ )
+ else
+ rspamd_logger.infox(rspamd_config, 'saved ANN %s:%s to redis: %s',
+ params.rule.prefix, params.set.name, params.set.ann.redis_key)
+ end
+ end
+
+ local function ann_trained(err, data)
+ params.set.learning_spawned = false
+ if err then
+ rspamd_logger.errx(rspamd_config, 'cannot train ANN %s:%s : %s',
+ params.rule.prefix, params.set.name, err)
+ lua_redis.redis_make_request_taskless(params.ev_base,
+ rspamd_config,
+ params.rule.redis,
+ nil,
+ true, -- is write
+ gen_unlock_cb(params.rule, params.set, params.ann_key), --callback
+ 'HDEL', -- command
+ { params.ann_key, 'lock' }
+ )
+ else
+ local parser = ucl.parser()
+ local ok, parse_err = parser:parse_text(data, 'msgpack')
+ assert(ok, parse_err)
+ local parsed = parser:get_object()
+ local ann_data = rspamd_util.zstd_compress(parsed.ann_data)
+ local pca_data = parsed.pca_data
+ local roc_thresholds = parsed.roc_thresholds
+
+ fill_set_ann(params.set, params.ann_key)
+ if pca_data then
+ params.set.ann.pca = rspamd_tensor.load(pca_data)
+ pca_data = rspamd_util.zstd_compress(pca_data)
+ end
+
+ if roc_thresholds then
+ params.set.ann.roc_thresholds = roc_thresholds
+ end
+
+
+ -- Deserialise ANN from the child process
+ ann_trained = rspamd_kann.load(parsed.ann_data)
+ local version = (params.set.ann.version or 0) + 1
+ params.set.ann.version = version
+ params.set.ann.ann = ann_trained
+ params.set.ann.symbols = params.set.symbols
+ params.set.ann.redis_key = new_ann_key(params.rule, params.set, version)
+
+ local profile = {
+ symbols = params.set.symbols,
+ digest = params.set.digest,
+ redis_key = params.set.ann.redis_key,
+ version = version
+ }
+
+ local profile_serialized = ucl.to_format(profile, 'json-compact', true)
+ local roc_thresholds_serialized = ucl.to_format(roc_thresholds, 'json-compact', true)
+
+ rspamd_logger.infox(rspamd_config,
+ 'trained ANN %s:%s, %s bytes (%s compressed); %s rows in pca (%sb compressed); redis key: %s (old key %s)',
+ params.rule.prefix, params.set.name,
+ #data, #ann_data,
+ #(params.set.ann.pca or {}), #(pca_data or {}),
+ params.set.ann.redis_key, params.ann_key)
+
+ lua_redis.exec_redis_script(redis_script_id.save_unlock,
+ { ev_base = params.ev_base, is_write = true },
+ redis_save_cb,
+ { profile.redis_key,
+ redis_ann_prefix(params.rule, params.set.name),
+ ann_data,
+ profile_serialized,
+ tostring(params.rule.ann_expire),
+ tostring(os.time()),
+ params.ann_key, -- old key to unlock...
+ roc_thresholds_serialized,
+ pca_data,
+ })
+ end
+ end
+
+ if params.rule.max_inputs then
+ fill_set_ann(params.set, params.ann_key)
+ end
+
+ params.worker:spawn_process {
+ func = train,
+ on_complete = ann_trained,
+ proctitle = string.format("ANN train for %s/%s", params.rule.prefix, params.set.name),
+ }
+ -- Spawn learn and register lock extension
+ params.set.learning_spawned = true
+ register_lock_extender(params.rule, params.set, params.ev_base, params.ann_key)
+ return
+
+ end
+end
+
+-- This function is used to adjust profiles and allowed setting ids for each rule
+-- It must be called when all settings are already registered (e.g. at post-init for config)
+local function process_rules_settings()
+ local function process_settings_elt(rule, selt)
+ local profile = rule.profile[selt.name]
+ if profile then
+ -- Use static user defined profile
+ -- Ensure that we have an array...
+ lua_util.debugm(N, rspamd_config, "use static profile for %s (%s): %s",
+ rule.prefix, selt.name, profile)
+ if not profile[1] then
+ profile = lua_util.keys(profile)
+ end
+ selt.symbols = profile
+ else
+ lua_util.debugm(N, rspamd_config, "use dynamic cfg based profile for %s (%s)",
+ rule.prefix, selt.name)
+ end
+
+ local function filter_symbols_predicate(sname)
+ if settings.blacklisted_symbols and settings.blacklisted_symbols[sname] then
+ return false
+ end
+ local fl = rspamd_config:get_symbol_flags(sname)
+ if fl then
+ fl = lua_util.list_to_hash(fl)
+
+ return not (fl.nostat or fl.idempotent or fl.skip or fl.composite)
+ end
+
+ return false
+ end
+
+ -- Generic stuff
+ if not profile then
+ -- Do filtering merely if we are using a dynamic profile
+ selt.symbols = fun.totable(fun.filter(filter_symbols_predicate, selt.symbols))
+ end
+
+ table.sort(selt.symbols)
+
+ selt.digest = lua_util.table_digest(selt.symbols)
+ selt.prefix = redis_ann_prefix(rule, selt.name)
+
+ rspamd_logger.messagex(rspamd_config,
+ 'use NN prefix for rule %s; settings id "%s"; symbols digest: "%s"',
+ selt.prefix, selt.name, selt.digest)
+
+ lua_redis.register_prefix(selt.prefix, N,
+ string.format('NN prefix for rule "%s"; settings id "%s"',
+ selt.prefix, selt.name), {
+ persistent = true,
+ type = 'zlist',
+ })
+ -- Versions
+ lua_redis.register_prefix(selt.prefix .. '_\\d+', N,
+ string.format('NN storage for rule "%s"; settings id "%s"',
+ selt.prefix, selt.name), {
+ persistent = true,
+ type = 'hash',
+ })
+ lua_redis.register_prefix(selt.prefix .. '_\\d+_spam_set', N,
+ string.format('NN learning set (spam) for rule "%s"; settings id "%s"',
+ selt.prefix, selt.name), {
+ persistent = true,
+ type = 'set',
+ })
+ lua_redis.register_prefix(selt.prefix .. '_\\d+_ham_set', N,
+ string.format('NN learning set (spam) for rule "%s"; settings id "%s"',
+ rule.prefix, selt.name), {
+ persistent = true,
+ type = 'set',
+ })
+ end
+
+ for k, rule in pairs(settings.rules) do
+ if not rule.allowed_settings then
+ rule.allowed_settings = {}
+ elseif rule.allowed_settings == 'all' then
+ -- Extract all settings ids
+ rule.allowed_settings = lua_util.keys(lua_settings.all_settings())
+ end
+
+ -- Convert to a map <setting_id> -> true
+ rule.allowed_settings = lua_util.list_to_hash(rule.allowed_settings)
+
+ -- Check if we can work without settings
+ if k == 'default' or type(rule.default) ~= 'boolean' then
+ rule.default = true
+ end
+
+ rule.settings = {}
+
+ if rule.default then
+ local default_settings = {
+ symbols = lua_settings.default_symbols(),
+ name = 'default'
+ }
+
+ process_settings_elt(rule, default_settings)
+ rule.settings[-1] = default_settings -- Magic constant, but OK as settings are positive int32
+ end
+
+ -- Now, for each allowed settings, we store sorted symbols + digest
+ -- We set table rule.settings[id] -> { name = name, symbols = symbols, digest = digest }
+ for s, _ in pairs(rule.allowed_settings) do
+ -- Here, we have a name, set of symbols and
+ local settings_id = s
+ if type(settings_id) ~= 'number' then
+ settings_id = lua_settings.numeric_settings_id(s)
+ end
+ local selt = lua_settings.settings_by_id(settings_id)
+
+ local nelt = {
+ symbols = selt.symbols, -- Already sorted
+ name = selt.name
+ }
+
+ process_settings_elt(rule, nelt)
+ for id, ex in pairs(rule.settings) do
+ if type(ex) == 'table' then
+ if nelt and lua_util.distance_sorted(ex.symbols, nelt.symbols) == 0 then
+ -- Equal symbols, add reference
+ lua_util.debugm(N, rspamd_config,
+ 'added reference from settings id %s to %s; same symbols',
+ nelt.name, ex.name)
+ rule.settings[settings_id] = id
+ nelt = nil
+ end
+ end
+ end
+
+ if nelt then
+ rule.settings[settings_id] = nelt
+ lua_util.debugm(N, rspamd_config, 'added new settings id %s(%s) to %s',
+ nelt.name, settings_id, rule.prefix)
+ end
+ end
+ end
+end
+
+-- Extract settings element for a specific settings id
+local function get_rule_settings(task, rule)
+ local sid = task:get_settings_id() or -1
+ local set = rule.settings[sid]
+
+ if not set then
+ return nil
+ end
+
+ while type(set) == 'number' do
+ -- Reference to another settings!
+ set = rule.settings[set]
+ end
+
+ return set
+end
+
+local function result_to_vector(task, profile)
+ if not profile.zeros then
+ -- Fill zeros vector
+ local zeros = {}
+ for i = 1, meta_functions.count_metatokens() do
+ zeros[i] = 0.0
+ end
+ for _, _ in ipairs(profile.symbols) do
+ zeros[#zeros + 1] = 0.0
+ end
+ profile.zeros = zeros
+ end
+
+ local vec = lua_util.shallowcopy(profile.zeros)
+ local mt = meta_functions.rspamd_gen_metatokens(task)
+
+ for i, v in ipairs(mt) do
+ vec[i] = v
+ end
+
+ task:process_ann_tokens(profile.symbols, vec, #mt, 0.1)
+
+ return vec
+end
+
+return {
+ can_push_train_vector = can_push_train_vector,
+ create_ann = create_ann,
+ default_options = default_options,
+ gen_unlock_cb = gen_unlock_cb,
+ get_rule_settings = get_rule_settings,
+ load_scripts = load_scripts,
+ module_config = module_config,
+ new_ann_key = new_ann_key,
+ plugin_ver = plugin_ver,
+ process_rules_settings = process_rules_settings,
+ redis_ann_prefix = redis_ann_prefix,
+ redis_params = redis_params,
+ redis_script_id = redis_script_id,
+ result_to_vector = result_to_vector,
+ settings = settings,
+ spawn_train = spawn_train,
+}
diff --git a/lualib/plugins/rbl.lua b/lualib/plugins/rbl.lua
new file mode 100644
index 0000000..af5d6bd
--- /dev/null
+++ b/lualib/plugins/rbl.lua
@@ -0,0 +1,232 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local ts = require("tableshape").types
+local lua_maps = require "lua_maps"
+local lua_util = require "lua_util"
+
+-- Common RBL plugin definitions
+
+local check_types = {
+ from = {
+ connfilter = true,
+ },
+ received = {},
+ helo = {
+ connfilter = true,
+ },
+ urls = {},
+ content_urls = {},
+ numeric_urls = {},
+ emails = {},
+ replyto = {},
+ dkim = {},
+ rdns = {
+ connfilter = true,
+ },
+ selector = {
+ require_argument = true,
+ },
+}
+
+local default_options = {
+ ['default_enabled'] = true,
+ ['default_ipv4'] = true,
+ ['default_ipv6'] = true,
+ ['default_unknown'] = false,
+ ['default_dkim_domainonly'] = true,
+ ['default_emails_domainonly'] = false,
+ ['default_exclude_users'] = false,
+ ['default_exclude_local'] = true,
+ ['default_no_ip'] = false,
+ ['default_dkim_match_from'] = false,
+ ['default_selector_flatten'] = true,
+}
+
+local return_codes_schema = ts.map_of(
+ ts.string / string.upper, -- Symbol name
+ (
+ ts.array_of(ts.string) +
+ (ts.string / function(s)
+ return { s }
+ end) -- List of IP patterns
+ )
+)
+local return_bits_schema = ts.map_of(
+ ts.string / string.upper, -- Symbol name
+ (
+ ts.array_of(ts.number + ts.string / tonumber) +
+ (ts.string / function(s)
+ return { tonumber(s) }
+ end) +
+ (ts.number / function(s)
+ return { s }
+ end)
+ )
+)
+
+local rule_schema_tbl = {
+ content_urls = ts.boolean:is_optional(),
+ disable_monitoring = ts.boolean:is_optional(),
+ disabled = ts.boolean:is_optional(),
+ dkim = ts.boolean:is_optional(),
+ dkim_domainonly = ts.boolean:is_optional(),
+ dkim_match_from = ts.boolean:is_optional(),
+ emails = ts.boolean:is_optional(),
+ emails_delimiter = ts.string:is_optional(),
+ emails_domainonly = ts.boolean:is_optional(),
+ enabled = ts.boolean:is_optional(),
+ exclude_local = ts.boolean:is_optional(),
+ exclude_users = ts.boolean:is_optional(),
+ from = ts.boolean:is_optional(),
+ hash = ts.one_of { "sha1", "sha256", "sha384", "sha512", "md5", "blake2" }:is_optional(),
+ hash_format = ts.one_of { "hex", "base32", "base64" }:is_optional(),
+ hash_len = (ts.integer + ts.string / tonumber):is_optional(),
+ helo = ts.boolean:is_optional(),
+ ignore_default = ts.boolean:is_optional(), -- alias
+ ignore_defaults = ts.boolean:is_optional(),
+ ignore_url_whitelist = ts.boolean:is_optional(),
+ ignore_whitelist = ts.boolean:is_optional(),
+ ignore_whitelists = ts.boolean:is_optional(), -- alias
+ images = ts.boolean:is_optional(),
+ ipv4 = ts.boolean:is_optional(),
+ ipv6 = ts.boolean:is_optional(),
+ is_whitelist = ts.boolean:is_optional(),
+ local_exclude_ip_map = ts.string:is_optional(),
+ monitored_address = ts.string:is_optional(),
+ no_ip = ts.boolean:is_optional(),
+ process_script = ts.string:is_optional(),
+ random_monitored = ts.boolean:is_optional(),
+ rbl = ts.string,
+ rdns = ts.boolean:is_optional(),
+ received = ts.boolean:is_optional(),
+ received_flags = ts.array_of(ts.string):is_optional(),
+ received_max_pos = ts.number:is_optional(),
+ received_min_pos = ts.number:is_optional(),
+ received_nflags = ts.array_of(ts.string):is_optional(),
+ replyto = ts.boolean:is_optional(),
+ requests_limit = (ts.integer + ts.string / tonumber):is_optional(),
+ require_symbols = (
+ ts.array_of(ts.string) + (ts.string / function(s)
+ return { s }
+ end)
+ ):is_optional(),
+ resolve_ip = ts.boolean:is_optional(),
+ return_bits = return_bits_schema:is_optional(),
+ return_codes = return_codes_schema:is_optional(),
+ returnbits = return_bits_schema:is_optional(),
+ returncodes = return_codes_schema:is_optional(),
+ returncodes_matcher = ts.one_of { "equality", "glob", "luapattern", "radix", "regexp" }:is_optional(),
+ selector = ts.one_of { ts.string, ts.table }:is_optional(),
+ selector_flatten = ts.boolean:is_optional(),
+ symbol = ts.string:is_optional(),
+ symbols_prefixes = ts.map_of(ts.string, ts.string):is_optional(),
+ unknown = ts.boolean:is_optional(),
+ url_compose_map = lua_maps.map_schema:is_optional(),
+ url_full_hostname = ts.boolean:is_optional(),
+ url_whitelist = lua_maps.map_schema:is_optional(),
+ urls = ts.boolean:is_optional(),
+ whitelist = lua_maps.map_schema:is_optional(),
+ whitelist_exception = (
+ ts.array_of(ts.string) + (ts.string / function(s)
+ return { s }
+ end)
+ ):is_optional(),
+ checks = ts.array_of(ts.one_of(lua_util.keys(check_types))):is_optional(),
+ exclude_checks = ts.array_of(ts.one_of(lua_util.keys(check_types))):is_optional(),
+}
+
+local function convert_checks(rule, name)
+ local rspamd_logger = require "rspamd_logger"
+ if rule.checks then
+ local all_connfilter = true
+ local exclude_checks = lua_util.list_to_hash(rule.exclude_checks or {})
+ for _, check in ipairs(rule.checks) do
+ if not exclude_checks[check] then
+ local check_type = check_types[check]
+ if check_type.require_argument then
+ if not rule[check] then
+ rspamd_logger.errx(rspamd_config, 'rbl rule %s has check %s which requires an argument',
+ name, check)
+ return nil
+ end
+ end
+
+ rule[check] = check_type
+
+ if not check_type.connfilter then
+ all_connfilter = false
+ end
+
+ if not check_type then
+ rspamd_logger.errx(rspamd_config, 'rbl rule %s has invalid check type: %s',
+ name, check)
+ return nil
+ end
+ else
+ rspamd_logger.infox(rspamd_config, 'disable check %s in %s: excluded explicitly',
+ check, name)
+ end
+ end
+ rule.connfilter = all_connfilter
+ end
+
+ -- Now check if we have any check enabled at all
+ local check_found = false
+ for k, _ in pairs(check_types) do
+ if type(rule[k]) ~= 'nil' then
+ check_found = true
+ break
+ end
+ end
+
+ if not check_found then
+ -- Enable implicit `from` check to allow upgrade
+ rspamd_logger.warnx(rspamd_config, 'rbl rule %s has no check enabled, enable default `from` check',
+ name)
+ rule.from = true
+ end
+
+ if rule.returncodes and not rule.returncodes_matcher then
+ for _, v in pairs(rule.returncodes) do
+ for _, e in ipairs(v) do
+ if e:find('[%%%[]') then
+ rspamd_logger.warn(rspamd_config, 'implicitly enabling luapattern returncodes_matcher for rule %s', name)
+ rule.returncodes_matcher = 'luapattern'
+ break
+ end
+ end
+ if rule.returncodes_matcher then
+ break
+ end
+ end
+ end
+
+ return rule
+end
+
+
+-- Add default boolean flags to the schema
+for def_k, _ in pairs(default_options) do
+ rule_schema_tbl[def_k:sub(#('default_') + 1)] = ts.boolean:is_optional()
+end
+
+return {
+ check_types = check_types,
+ rule_schema = ts.shape(rule_schema_tbl),
+ default_options = default_options,
+ convert_checks = convert_checks,
+}
diff --git a/lualib/plugins_stats.lua b/lualib/plugins_stats.lua
new file mode 100644
index 0000000..2497fb9
--- /dev/null
+++ b/lualib/plugins_stats.lua
@@ -0,0 +1,48 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local ansicolors = require "ansicolors"
+
+local function printf(fmt, ...)
+ print(string.format(fmt, ...))
+end
+
+local function highlight(str)
+ return ansicolors.white .. str .. ansicolors.reset
+end
+
+local function print_plugins_table(tbl, what)
+ local mods = {}
+ for k, _ in pairs(tbl) do
+ table.insert(mods, k)
+ end
+
+ printf("Modules %s: %s", highlight(what), table.concat(mods, ", "))
+end
+
+return function(args, _)
+ print_plugins_table(rspamd_plugins_state.enabled, "enabled")
+ print_plugins_table(rspamd_plugins_state.disabled_explicitly,
+ "disabled (explicitly)")
+ print_plugins_table(rspamd_plugins_state.disabled_unconfigured,
+ "disabled (unconfigured)")
+ print_plugins_table(rspamd_plugins_state.disabled_redis,
+ "disabled (no Redis)")
+ print_plugins_table(rspamd_plugins_state.disabled_experimental,
+ "disabled (experimental)")
+ print_plugins_table(rspamd_plugins_state.disabled_failed,
+ "disabled (failed)")
+end \ No newline at end of file
diff --git a/lualib/redis_scripts/bayes_cache_check.lua b/lualib/redis_scripts/bayes_cache_check.lua
new file mode 100644
index 0000000..f1ffc2b
--- /dev/null
+++ b/lualib/redis_scripts/bayes_cache_check.lua
@@ -0,0 +1,20 @@
+-- Lua script to perform cache checking for bayes classification
+-- This script accepts the following parameters:
+-- key1 - cache id
+-- key2 - configuration table in message pack
+
+local cache_id = KEYS[1]
+local conf = cmsgpack.unpack(KEYS[2])
+cache_id = string.sub(cache_id, 1, conf.cache_elt_len)
+
+-- Try each prefix that is in Redis
+for i = 0, conf.cache_max_keys do
+ local prefix = conf.cache_prefix .. string.rep("X", i)
+ local have = redis.call('HGET', prefix, cache_id)
+
+ if have then
+ return tonumber(have)
+ end
+end
+
+return nil
diff --git a/lualib/redis_scripts/bayes_cache_learn.lua b/lualib/redis_scripts/bayes_cache_learn.lua
new file mode 100644
index 0000000..8811f3c
--- /dev/null
+++ b/lualib/redis_scripts/bayes_cache_learn.lua
@@ -0,0 +1,61 @@
+-- Lua script to perform cache checking for bayes classification
+-- This script accepts the following parameters:
+-- key1 - cache id
+-- key3 - is spam (1 or 0)
+-- key3 - configuration table in message pack
+
+local cache_id = KEYS[1]
+local is_spam = KEYS[2]
+local conf = cmsgpack.unpack(KEYS[3])
+cache_id = string.sub(cache_id, 1, conf.cache_elt_len)
+
+-- Try each prefix that is in Redis (as some other instance might have set it)
+for i = 0, conf.cache_max_keys do
+ local prefix = conf.cache_prefix .. string.rep("X", i)
+ local have = redis.call('HGET', prefix, cache_id)
+
+ if have then
+ -- Already in cache
+ return false
+ end
+end
+
+local added = false
+local lim = conf.cache_max_elt
+for i = 0, conf.cache_max_keys do
+ if not added then
+ local prefix = conf.cache_prefix .. string.rep("X", i)
+ local count = redis.call('HLEN', prefix)
+
+ if count < lim then
+ -- We can add it to this prefix
+ redis.call('HSET', prefix, cache_id, is_spam)
+ added = true
+ end
+ end
+end
+
+if not added then
+ -- Need to expire some keys
+ local expired = false
+ for i = 0, conf.cache_max_keys do
+ local prefix = conf.cache_prefix .. string.rep("X", i)
+ local exists = redis.call('EXISTS', prefix)
+
+ if exists then
+ if expired then
+ redis.call('DEL', prefix)
+ redis.call('HSET', prefix, cache_id, is_spam)
+
+ -- Do not expire anything else
+ expired = true
+ elseif i > 0 then
+ -- Move key to a shorter prefix, so we will rotate them eventually from lower to upper
+ local new_prefix = conf.cache_prefix .. string.rep("X", i - 1)
+ redis.call('RENAME', prefix, new_prefix)
+ end
+ end
+ end
+end
+
+return true \ No newline at end of file
diff --git a/lualib/redis_scripts/bayes_classify.lua b/lualib/redis_scripts/bayes_classify.lua
new file mode 100644
index 0000000..e94f645
--- /dev/null
+++ b/lualib/redis_scripts/bayes_classify.lua
@@ -0,0 +1,37 @@
+-- Lua script to perform bayes classification
+-- This script accepts the following parameters:
+-- key1 - prefix for bayes tokens (e.g. for per-user classification)
+-- key2 - set of tokens encoded in messagepack array of strings
+
+local prefix = KEYS[1]
+local output_spam = {}
+local output_ham = {}
+
+local learned_ham = tonumber(redis.call('HGET', prefix, 'learns_ham')) or 0
+local learned_spam = tonumber(redis.call('HGET', prefix, 'learns_spam')) or 0
+
+-- Output is a set of pairs (token_index, token_count), tokens that are not
+-- found are not filled.
+-- This optimisation will save a lot of space for sparse tokens, and in Bayes that assumption is normally held
+
+if learned_ham > 0 and learned_spam > 0 then
+ local input_tokens = cmsgpack.unpack(KEYS[2])
+ for i, token in ipairs(input_tokens) do
+ local token_data = redis.call('HMGET', token, 'H', 'S')
+
+ if token_data then
+ local ham_count = token_data[1]
+ local spam_count = token_data[2]
+
+ if ham_count then
+ table.insert(output_ham, { i, tonumber(ham_count) })
+ end
+
+ if spam_count then
+ table.insert(output_spam, { i, tonumber(spam_count) })
+ end
+ end
+ end
+end
+
+return { learned_ham, learned_spam, output_ham, output_spam } \ No newline at end of file
diff --git a/lualib/redis_scripts/bayes_learn.lua b/lualib/redis_scripts/bayes_learn.lua
new file mode 100644
index 0000000..80d86d8
--- /dev/null
+++ b/lualib/redis_scripts/bayes_learn.lua
@@ -0,0 +1,44 @@
+-- Lua script to perform bayes learning
+-- This script accepts the following parameters:
+-- key1 - prefix for bayes tokens (e.g. for per-user classification)
+-- key2 - boolean is_spam
+-- key3 - string symbol
+-- key4 - boolean is_unlearn
+-- key5 - set of tokens encoded in messagepack array of strings
+-- key6 - set of text tokens (if any) encoded in messagepack array of strings (size must be twice of `KEYS[5]`)
+
+local prefix = KEYS[1]
+local is_spam = KEYS[2] == 'true' and true or false
+local symbol = KEYS[3]
+local is_unlearn = KEYS[4] == 'true' and true or false
+local input_tokens = cmsgpack.unpack(KEYS[5])
+local text_tokens
+
+if KEYS[6] then
+ text_tokens = cmsgpack.unpack(KEYS[6])
+end
+
+local hash_key = is_spam and 'S' or 'H'
+local learned_key = is_spam and 'learns_spam' or 'learns_ham'
+
+redis.call('SADD', symbol .. '_keys', prefix)
+redis.call('HSET', prefix, 'version', '2') -- new schema
+redis.call('HINCRBY', prefix, learned_key, is_unlearn and -1 or 1) -- increase or decrease learned count
+
+for i, token in ipairs(input_tokens) do
+ redis.call('HINCRBY', token, hash_key, 1)
+ if text_tokens then
+ local tok1 = text_tokens[i * 2 - 1]
+ local tok2 = text_tokens[i * 2]
+
+ if tok1 then
+ if tok2 then
+ redis.call('HSET', token, 'tokens', string.format('%s:%s', tok1, tok2))
+ else
+ redis.call('HSET', token, 'tokens', tok1)
+ end
+
+ redis.call('ZINCRBY', prefix .. '_z', is_unlearn and -1 or 1, token)
+ end
+ end
+end \ No newline at end of file
diff --git a/lualib/redis_scripts/bayes_stat.lua b/lualib/redis_scripts/bayes_stat.lua
new file mode 100644
index 0000000..31e5128
--- /dev/null
+++ b/lualib/redis_scripts/bayes_stat.lua
@@ -0,0 +1,19 @@
+-- Lua script to perform bayes stats
+-- This script accepts the following parameters:
+-- key1 - current cursor
+-- key2 - symbol to examine
+-- key3 - learn key (e.g. learns_ham or learns_spam)
+-- key4 - max users
+
+local cursor = tonumber(KEYS[1])
+
+local ret = redis.call('SSCAN', KEYS[2] .. '_keys', cursor, 'COUNT', tonumber(KEYS[4]))
+
+local new_cursor = tonumber(ret[1])
+local nkeys = #ret[2]
+local learns = 0
+for _, key in ipairs(ret[2]) do
+ learns = learns + (tonumber(redis.call('HGET', key, KEYS[3])) or 0)
+end
+
+return { new_cursor, nkeys, learns } \ No newline at end of file
diff --git a/lualib/redis_scripts/neural_maybe_invalidate.lua b/lualib/redis_scripts/neural_maybe_invalidate.lua
new file mode 100644
index 0000000..517fa01
--- /dev/null
+++ b/lualib/redis_scripts/neural_maybe_invalidate.lua
@@ -0,0 +1,25 @@
+-- Lua script to invalidate ANNs by rank
+-- Uses the following keys
+-- key1 - prefix for keys
+-- key2 - number of elements to leave
+
+local card = redis.call('ZCARD', KEYS[1])
+local lim = tonumber(KEYS[2])
+if card > lim then
+ local to_delete = redis.call('ZRANGE', KEYS[1], 0, card - lim - 1)
+ if to_delete then
+ for _, k in ipairs(to_delete) do
+ local tb = cjson.decode(k)
+ if type(tb) == 'table' and type(tb.redis_key) == 'string' then
+ redis.call('DEL', tb.redis_key)
+ -- Also train vectors
+ redis.call('DEL', tb.redis_key .. '_spam_set')
+ redis.call('DEL', tb.redis_key .. '_ham_set')
+ end
+ end
+ end
+ redis.call('ZREMRANGEBYRANK', KEYS[1], 0, card - lim - 1)
+ return to_delete
+else
+ return {}
+end \ No newline at end of file
diff --git a/lualib/redis_scripts/neural_maybe_lock.lua b/lualib/redis_scripts/neural_maybe_lock.lua
new file mode 100644
index 0000000..f705115
--- /dev/null
+++ b/lualib/redis_scripts/neural_maybe_lock.lua
@@ -0,0 +1,19 @@
+-- Lua script lock ANN for learning
+-- Uses the following keys
+-- key1 - prefix for keys
+-- key2 - current time
+-- key3 - key expire
+-- key4 - hostname
+
+local locked = redis.call('HGET', KEYS[1], 'lock')
+local now = tonumber(KEYS[2])
+if locked then
+ locked = tonumber(locked)
+ local expire = tonumber(KEYS[3])
+ if now > locked and (now - locked) < expire then
+ return { tostring(locked), redis.call('HGET', KEYS[1], 'hostname') or 'unknown' }
+ end
+end
+redis.call('HSET', KEYS[1], 'lock', tostring(now))
+redis.call('HSET', KEYS[1], 'hostname', KEYS[4])
+return 1 \ No newline at end of file
diff --git a/lualib/redis_scripts/neural_save_unlock.lua b/lualib/redis_scripts/neural_save_unlock.lua
new file mode 100644
index 0000000..5af1ddc
--- /dev/null
+++ b/lualib/redis_scripts/neural_save_unlock.lua
@@ -0,0 +1,24 @@
+-- Lua script to save and unlock ANN in redis
+-- Uses the following keys
+-- key1 - prefix for ANN
+-- key2 - prefix for profile
+-- key3 - compressed ANN
+-- key4 - profile as JSON
+-- key5 - expire in seconds
+-- key6 - current time
+-- key7 - old key
+-- key8 - ROC Thresholds
+-- key9 - optional PCA
+local now = tonumber(KEYS[6])
+redis.call('ZADD', KEYS[2], now, KEYS[4])
+redis.call('HSET', KEYS[1], 'ann', KEYS[3])
+redis.call('DEL', KEYS[1] .. '_spam_set')
+redis.call('DEL', KEYS[1] .. '_ham_set')
+redis.call('HDEL', KEYS[1], 'lock')
+redis.call('HDEL', KEYS[7], 'lock')
+redis.call('EXPIRE', KEYS[1], tonumber(KEYS[5]))
+redis.call('HSET', KEYS[1], 'roc_thresholds', KEYS[8])
+if KEYS[9] then
+ redis.call('HSET', KEYS[1], 'pca', KEYS[9])
+end
+return 1 \ No newline at end of file
diff --git a/lualib/redis_scripts/neural_train_size.lua b/lualib/redis_scripts/neural_train_size.lua
new file mode 100644
index 0000000..45ad6a9
--- /dev/null
+++ b/lualib/redis_scripts/neural_train_size.lua
@@ -0,0 +1,24 @@
+-- Lua script that checks if we can store a new training vector
+-- Uses the following keys:
+-- key1 - ann key
+-- returns nspam,nham (or nil if locked)
+
+local prefix = KEYS[1]
+local locked = redis.call('HGET', prefix, 'lock')
+if locked then
+ local host = redis.call('HGET', prefix, 'hostname') or 'unknown'
+ return string.format('%s:%s', host, locked)
+end
+local nspam = 0
+local nham = 0
+
+local ret = redis.call('SCARD', prefix .. '_spam_set')
+if ret then
+ nspam = tonumber(ret)
+end
+ret = redis.call('SCARD', prefix .. '_ham_set')
+if ret then
+ nham = tonumber(ret)
+end
+
+return { nspam, nham } \ No newline at end of file
diff --git a/lualib/redis_scripts/ratelimit_check.lua b/lualib/redis_scripts/ratelimit_check.lua
new file mode 100644
index 0000000..d39cdf1
--- /dev/null
+++ b/lualib/redis_scripts/ratelimit_check.lua
@@ -0,0 +1,85 @@
+-- This Lua script is a rate limiter for Redis using the token bucket algorithm.
+-- The script checks if a message should be rate-limited and updates the bucket status accordingly.
+-- Input keys:
+-- KEYS[1]: A prefix for the Redis keys, e.g., RL_<triplet>_<seconds>
+-- KEYS[2]: The current time in milliseconds
+-- KEYS[3]: The bucket leak rate (messages per millisecond)
+-- KEYS[4]: The maximum allowed burst
+-- KEYS[5]: The expiration time for a bucket
+-- KEYS[6]: The number of recipients for the message
+
+-- Redis keys used:
+-- l: Last hit (time in milliseconds)
+-- b: Current burst (number of tokens in the bucket)
+-- p: Pending messages (number of messages in processing)
+-- dr: Current dynamic rate multiplier (*10000)
+-- db: Current dynamic burst multiplier (*10000)
+
+-- Returns:
+-- An array containing:
+-- 1. if the message should be rate-limited or 0 if not
+-- 2. The current burst value after processing the message
+-- 3. The dynamic rate multiplier
+-- 4. The dynamic burst multiplier
+-- 5. The number of tokens leaked during processing
+
+local last = redis.call('HGET', KEYS[1], 'l')
+local now = tonumber(KEYS[2])
+local nrcpt = tonumber(KEYS[6])
+local leak_rate = tonumber(KEYS[3])
+local max_burst = tonumber(KEYS[4])
+local prefix = KEYS[1]
+local dynr, dynb, leaked = 0, 0, 0
+if not last then
+ -- New bucket
+ redis.call('HMSET', prefix, 'l', tostring(now), 'b', '0', 'dr', '10000', 'db', '10000', 'p', tostring(nrcpt))
+ redis.call('EXPIRE', prefix, KEYS[5])
+ return { 0, '0', '1', '1', '0' }
+end
+last = tonumber(last)
+
+local burst, pending = unpack(redis.call('HMGET', prefix, 'b', 'p'))
+burst, pending = tonumber(burst or '0'), tonumber(pending or '0')
+-- Sanity to avoid races
+if burst < 0 then
+ burst = 0
+end
+if pending < 0 then
+ pending = 0
+end
+pending = pending + nrcpt -- this message
+-- Perform leak
+if burst + pending > 0 then
+ -- If we have any time passed
+ if burst > 0 and last < now then
+ dynr = tonumber(redis.call('HGET', prefix, 'dr')) / 10000.0
+ if dynr == 0 then
+ dynr = 0.0001
+ end
+ leak_rate = leak_rate * dynr
+ leaked = ((now - last) * leak_rate)
+ if leaked > burst then
+ leaked = burst
+ end
+ burst = burst - leaked
+ redis.call('HINCRBYFLOAT', prefix, 'b', -(leaked))
+ redis.call('HSET', prefix, 'l', tostring(now))
+ end
+
+ dynb = tonumber(redis.call('HGET', prefix, 'db')) / 10000.0
+ if dynb == 0 then
+ dynb = 0.0001
+ end
+
+ burst = burst + pending
+ if burst > 0 and burst > max_burst * dynb then
+ return { 1, tostring(burst - pending), tostring(dynr), tostring(dynb), tostring(leaked) }
+ end
+ -- Increase pending if we allow ratelimit
+ redis.call('HINCRBY', prefix, 'p', nrcpt)
+else
+ burst = 0
+ redis.call('HMSET', prefix, 'b', '0', 'p', tostring(nrcpt))
+end
+
+return { 0, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked) } \ No newline at end of file
diff --git a/lualib/redis_scripts/ratelimit_cleanup_pending.lua b/lualib/redis_scripts/ratelimit_cleanup_pending.lua
new file mode 100644
index 0000000..698a3ec
--- /dev/null
+++ b/lualib/redis_scripts/ratelimit_cleanup_pending.lua
@@ -0,0 +1,33 @@
+-- This script cleans up the pending requests in Redis.
+
+-- KEYS: Input parameters
+-- KEYS[1] - prefix: The Redis key prefix used to store the bucket information.
+-- KEYS[2] - now: The current time in milliseconds.
+-- KEYS[3] - expire: The expiration time for the Redis key storing the bucket information, in seconds.
+-- KEYS[4] - number_of_recipients: The number of requests to be allowed (or the increase rate).
+
+-- 1. Retrieve the last hit time and initialize variables
+local prefix = KEYS[1]
+local last = redis.call('HGET', prefix, 'l')
+local nrcpt = tonumber(KEYS[4])
+if not last then
+ -- No bucket, no cleanup
+ return 0
+end
+
+
+-- 2. Update the pending values based on the number of recipients (requests)
+local pending = redis.call('HGET', prefix, 'p')
+pending = tonumber(pending or '0')
+if pending < nrcpt then
+ pending = 0
+else
+ pending = pending - nrcpt
+end
+
+-- 3. Set the updated values back to Redis and update the expiration time for the bucket
+redis.call('HMSET', prefix, 'p', tostring(pending), 'l', KEYS[2])
+redis.call('EXPIRE', prefix, KEYS[3])
+
+-- 4. Return the updated pending value
+return pending \ No newline at end of file
diff --git a/lualib/redis_scripts/ratelimit_update.lua b/lualib/redis_scripts/ratelimit_update.lua
new file mode 100644
index 0000000..caee8fb
--- /dev/null
+++ b/lualib/redis_scripts/ratelimit_update.lua
@@ -0,0 +1,93 @@
+-- This script updates a token bucket rate limiter with dynamic rate and burst multipliers in Redis.
+
+-- KEYS: Input parameters
+-- KEYS[1] - prefix: The Redis key prefix used to store the bucket information.
+-- KEYS[2] - now: The current time in milliseconds.
+-- KEYS[3] - dynamic_rate_multiplier: A multiplier to adjust the rate limit dynamically.
+-- KEYS[4] - dynamic_burst_multiplier: A multiplier to adjust the burst limit dynamically.
+-- KEYS[5] - max_dyn_rate: The maximum allowed value for the dynamic rate multiplier.
+-- KEYS[6] - max_burst_rate: The maximum allowed value for the dynamic burst multiplier.
+-- KEYS[7] - expire: The expiration time for the Redis key storing the bucket information, in seconds.
+-- KEYS[8] - number_of_recipients: The number of requests to be allowed (or the increase rate).
+
+-- 1. Retrieve the last hit time and initialize variables
+local prefix = KEYS[1]
+local last = redis.call('HGET', prefix, 'l')
+local now = tonumber(KEYS[2])
+local nrcpt = tonumber(KEYS[8])
+if not last then
+ -- 2. Initialize a new bucket if the last hit time is not found (must not happen)
+ redis.call('HMSET', prefix, 'l', tostring(now), 'b', tostring(nrcpt), 'dr', '10000', 'db', '10000', 'p', '0')
+ redis.call('EXPIRE', prefix, KEYS[7])
+ return { 1, 1, 1 }
+end
+
+-- 3. Update the dynamic rate multiplier based on input parameters
+local dr, db = 1.0, 1.0
+
+local max_dr = tonumber(KEYS[5])
+
+if max_dr > 1 then
+ local rate_mult = tonumber(KEYS[3])
+ dr = tonumber(redis.call('HGET', prefix, 'dr')) / 10000
+
+ if rate_mult > 1.0 and dr < max_dr then
+ dr = dr * rate_mult
+ if dr > 0.0001 then
+ redis.call('HSET', prefix, 'dr', tostring(math.floor(dr * 10000)))
+ else
+ redis.call('HSET', prefix, 'dr', '1')
+ end
+ elseif rate_mult < 1.0 and dr > (1.0 / max_dr) then
+ dr = dr * rate_mult
+ if dr > 0.0001 then
+ redis.call('HSET', prefix, 'dr', tostring(math.floor(dr * 10000)))
+ else
+ redis.call('HSET', prefix, 'dr', '1')
+ end
+ end
+end
+
+-- 4. Update the dynamic burst multiplier based on input parameters
+local max_db = tonumber(KEYS[6])
+if max_db > 1 then
+ local rate_mult = tonumber(KEYS[4])
+ db = tonumber(redis.call('HGET', prefix, 'db')) / 10000
+
+ if rate_mult > 1.0 and db < max_db then
+ db = db * rate_mult
+ if db > 0.0001 then
+ redis.call('HSET', prefix, 'db', tostring(math.floor(db * 10000)))
+ else
+ redis.call('HSET', prefix, 'db', '1')
+ end
+ elseif rate_mult < 1.0 and db > (1.0 / max_db) then
+ db = db * rate_mult
+ if db > 0.0001 then
+ redis.call('HSET', prefix, 'db', tostring(math.floor(db * 10000)))
+ else
+ redis.call('HSET', prefix, 'db', '1')
+ end
+ end
+end
+
+-- 5. Update the burst and pending values based on the number of recipients (requests)
+local burst, pending = unpack(redis.call('HMGET', prefix, 'b', 'p'))
+burst, pending = tonumber(burst or '0'), tonumber(pending or '0')
+if burst < 0 then
+ burst = nrcpt
+else
+ burst = burst + nrcpt
+end
+if pending < nrcpt then
+ pending = 0
+else
+ pending = pending - nrcpt
+end
+
+-- 6. Set the updated values back to Redis and update the expiration time for the bucket
+redis.call('HMSET', prefix, 'b', tostring(burst), 'p', tostring(pending), 'l', KEYS[2])
+redis.call('EXPIRE', prefix, KEYS[7])
+
+-- 7. Return the updated burst value, dynamic rate multiplier, and dynamic burst multiplier
+return { tostring(burst), tostring(dr), tostring(db) } \ No newline at end of file
diff --git a/lualib/rspamadm/clickhouse.lua b/lualib/rspamadm/clickhouse.lua
new file mode 100644
index 0000000..b22d800
--- /dev/null
+++ b/lualib/rspamadm/clickhouse.lua
@@ -0,0 +1,528 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+local lua_clickhouse = require "lua_clickhouse"
+local lua_util = require "lua_util"
+local rspamd_http = require "rspamd_http"
+local rspamd_upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+
+local E = {}
+
+-- Define command line options
+local parser = argparse()
+ :name 'rspamadm clickhouse'
+ :description 'Retrieve information from Clickhouse'
+ :help_description_margin(30)
+ :command_target('command')
+ :require_command(true)
+
+parser:option '-c --config'
+ :description 'Path to config file'
+ :argname('config_file')
+ :default(rspamd_paths['CONFDIR'] .. '/rspamd.conf')
+parser:option '-d --database'
+ :description 'Name of Clickhouse database to use'
+ :argname('database')
+ :default('default')
+parser:flag '--no-ssl-verify'
+ :description 'Disable SSL verification'
+ :argname('no_ssl_verify')
+parser:mutex(
+ parser:option '-p --password'
+ :description 'Password to use for Clickhouse'
+ :argname('password'),
+ parser:flag '-a --ask-password'
+ :description 'Ask password from the terminal'
+ :argname('ask_password')
+)
+parser:option '-s --server'
+ :description 'Address[:port] to connect to Clickhouse with'
+ :argname('server')
+parser:option '-u --user'
+ :description 'Username to use for Clickhouse'
+ :argname('user')
+parser:option '--use-gzip'
+ :description 'Use Gzip with Clickhouse'
+ :argname('use_gzip')
+ :default(true)
+parser:flag '--use-https'
+ :description 'Use HTTPS with Clickhouse'
+ :argname('use_https')
+
+local neural_profile = parser:command 'neural_profile'
+ :description 'Generate symbols profile using data from Clickhouse'
+neural_profile:option '-w --where'
+ :description 'WHERE clause for Clickhouse query'
+ :argname('where')
+neural_profile:flag '-j --json'
+ :description 'Write output as JSON'
+ :argname('json')
+neural_profile:option '--days'
+ :description 'Number of days to collect stats for'
+ :argname('days')
+ :default('7')
+neural_profile:option '--limit -l'
+ :description 'Maximum rows to fetch per day'
+ :argname('limit')
+neural_profile:option '--settings-id'
+ :description 'Settings ID to query'
+ :argname('settings_id')
+ :default('')
+
+local neural_train = parser:command 'neural_train'
+ :description 'Train neural using data from Clickhouse'
+neural_train:option '--days'
+ :description 'Number of days to query data for'
+ :argname('days')
+ :default('7')
+neural_train:option '--column-name-digest'
+ :description 'Name of neural profile digest column in Clickhouse'
+ :argname('column_name_digest')
+ :default('NeuralDigest')
+neural_train:option '--column-name-vector'
+ :description 'Name of neural training vector column in Clickhouse'
+ :argname('column_name_vector')
+ :default('NeuralMpack')
+neural_train:option '--limit -l'
+ :description 'Maximum rows to fetch per day'
+ :argname('limit')
+neural_train:option '--profile -p'
+ :description 'Profile to use for training'
+ :argname('profile')
+ :default('default')
+neural_train:option '--rule -r'
+ :description 'Rule to train'
+ :argname('rule')
+ :default('default')
+neural_train:option '--spam -s'
+ :description 'WHERE clause to use for spam'
+ :argname('spam')
+ :default("Action == 'reject'")
+neural_train:option '--ham -h'
+ :description 'WHERE clause to use for ham'
+ :argname('ham')
+ :default('Score < 0')
+neural_train:option '--url -u'
+ :description 'URL to use for training'
+ :argname('url')
+ :default('http://127.0.0.1:11334/plugins/neural/learn')
+
+local http_params = {
+ config = rspamd_config,
+ ev_base = rspamadm_ev_base,
+ session = rspamadm_session,
+ resolver = rspamadm_dns_resolver,
+}
+
+local function load_config(config_file)
+ local _r, err = rspamd_config:load_ucl(config_file)
+
+ if not _r then
+ rspamd_logger.errx('cannot load %s: %s', config_file, err)
+ os.exit(1)
+ end
+
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
+ if not _r then
+ rspamd_logger.errx('cannot process %s: %s', config_file, err)
+ os.exit(1)
+ end
+
+ if not rspamd_config:init_modules() then
+ rspamd_logger.errx('cannot init modules when parsing %s', config_file)
+ os.exit(1)
+ end
+
+ rspamd_config:init_subsystem('symcache')
+end
+
+local function days_list(days)
+ -- Create list of days to query starting with yesterday
+ local query_days = {}
+ local previous_date = os.time() - 86400
+ local num_days = tonumber(days)
+ for _ = 1, num_days do
+ table.insert(query_days, os.date('%Y-%m-%d', previous_date))
+ previous_date = previous_date - 86400
+ end
+ return query_days
+end
+
+local function get_excluded_symbols(known_symbols, correlations, seen_total)
+ -- Walk results once to collect all symbols & count occurrences
+
+ local remove = {}
+ local known_symbols_list = {}
+ local composites = rspamd_config:get_all_opt('composites')
+ local all_symbols = rspamd_config:get_symbols()
+ local skip_flags = {
+ nostat = true,
+ skip = true,
+ idempotent = true,
+ composite = true,
+ }
+ for k, v in pairs(known_symbols) do
+ local lower_count, higher_count
+ if v.seen_spam > v.seen_ham then
+ lower_count = v.seen_ham
+ higher_count = v.seen_spam
+ else
+ lower_count = v.seen_spam
+ higher_count = v.seen_ham
+ end
+
+ if composites[k] then
+ remove[k] = 'composite symbol'
+ elseif lower_count / higher_count >= 0.95 then
+ remove[k] = 'weak ham/spam correlation'
+ elseif v.seen / seen_total >= 0.9 then
+ remove[k] = 'omnipresent symbol'
+ elseif not all_symbols[k] then
+ remove[k] = 'nonexistent symbol'
+ else
+ for fl, _ in pairs(all_symbols[k].flags or {}) do
+ if skip_flags[fl] then
+ remove[k] = fl .. ' symbol'
+ break
+ end
+ end
+ end
+ known_symbols_list[v.id] = {
+ seen = v.seen,
+ name = k,
+ }
+ end
+
+ -- Walk correlation matrix and check total counts
+ for sym_id, row in pairs(correlations) do
+ for inner_sym_id, count in pairs(row) do
+ local known = known_symbols_list[sym_id]
+ local inner = known_symbols_list[inner_sym_id]
+ if known and count == known.seen and not remove[inner.name] and not remove[known.name] then
+ remove[known.name] = string.format("overlapped by %s",
+ known_symbols_list[inner_sym_id].name)
+ end
+ end
+ end
+
+ return remove
+end
+
+local function handle_neural_profile(args)
+
+ local known_symbols, correlations = {}, {}
+ local symbols_count, seen_total = 0, 0
+
+ local function process_row(r)
+ local is_spam = true
+ if r['Action'] == 'no action' or r['Action'] == 'greylist' then
+ is_spam = false
+ end
+ seen_total = seen_total + 1
+
+ local nsym = #r['Symbols.Names']
+
+ for i = 1, nsym do
+ local sym = r['Symbols.Names'][i]
+ local t = known_symbols[sym]
+ if not t then
+ local spam_count, ham_count = 0, 0
+ if is_spam then
+ spam_count = spam_count + 1
+ else
+ ham_count = ham_count + 1
+ end
+ known_symbols[sym] = {
+ id = symbols_count,
+ seen = 1,
+ seen_ham = ham_count,
+ seen_spam = spam_count,
+ }
+ symbols_count = symbols_count + 1
+ else
+ known_symbols[sym].seen = known_symbols[sym].seen + 1
+ if is_spam then
+ known_symbols[sym].seen_spam = known_symbols[sym].seen_spam + 1
+ else
+ known_symbols[sym].seen_ham = known_symbols[sym].seen_ham + 1
+ end
+ end
+ end
+
+ -- Fill correlations
+ for i = 1, nsym do
+ for j = 1, nsym do
+ if i ~= j then
+ local sym = r['Symbols.Names'][i]
+ local inner_sym_name = r['Symbols.Names'][j]
+ local known_sym = known_symbols[sym]
+ local inner_sym = known_symbols[inner_sym_name]
+ if known_sym and inner_sym then
+ if not correlations[known_sym.id] then
+ correlations[known_sym.id] = {}
+ end
+ local n = correlations[known_sym.id][inner_sym.id] or 0
+ n = n + 1
+ correlations[known_sym.id][inner_sym.id] = n
+ end
+ end
+ end
+ end
+ end
+
+ local query_days = days_list(args.days)
+ local conditions = {}
+ table.insert(conditions, string.format("SettingsId = '%s'", args.settings_id))
+ local limit = ''
+ local num_limit = tonumber(args.limit)
+ if num_limit then
+ limit = string.format(' LIMIT %d', num_limit) -- Contains leading space
+ end
+ if args.where then
+ table.insert(conditions, args.where)
+ end
+
+ local query_fmt = 'SELECT Action, Symbols.Names FROM rspamd WHERE %s%s'
+ for _, query_day in ipairs(query_days) do
+ -- Date should be the last condition
+ table.insert(conditions, string.format("Date = '%s'", query_day))
+ local query = string.format(query_fmt, table.concat(conditions, ' AND '), limit)
+ local upstream = args.upstream:get_upstream_round_robin()
+ local err = lua_clickhouse.select_sync(upstream, args, http_params, query, process_row)
+ if err ~= nil then
+ io.stderr:write(string.format('Error querying Clickhouse: %s\n', err))
+ os.exit(1)
+ end
+ conditions[#conditions] = nil -- remove Date condition
+ end
+
+ local remove = get_excluded_symbols(known_symbols, correlations, seen_total)
+ if not args.json then
+ for k in pairs(known_symbols) do
+ if not remove[k] then
+ io.stdout:write(string.format('%s\n', k))
+ end
+ end
+ os.exit(0)
+ end
+
+ local json_output = {
+ all_symbols = {},
+ removed_symbols = {},
+ used_symbols = {},
+ }
+ for k in pairs(known_symbols) do
+ table.insert(json_output.all_symbols, k)
+ local why_removed = remove[k]
+ if why_removed then
+ json_output.removed_symbols[k] = why_removed
+ else
+ table.insert(json_output.used_symbols, k)
+ end
+ end
+ io.stdout:write(ucl.to_format(json_output, 'json'))
+end
+
+local function post_neural_training(url, rule, spam_rows, ham_rows)
+ -- Prepare JSON payload
+ local payload = ucl.to_format(
+ {
+ ham_vec = ham_rows,
+ rule = rule,
+ spam_vec = spam_rows,
+ }, 'json')
+
+ -- POST the payload
+ local err, response = rspamd_http.request({
+ body = payload,
+ config = rspamd_config,
+ ev_base = rspamadm_ev_base,
+ log_obj = rspamd_config,
+ resolver = rspamadm_dns_resolver,
+ session = rspamadm_session,
+ url = url,
+ })
+
+ if err then
+ io.stderr:write(string.format('HTTP error: %s\n', err))
+ os.exit(1)
+ end
+ if response.code ~= 200 then
+ io.stderr:write(string.format('bad HTTP code: %d\n', response.code))
+ os.exit(1)
+ end
+ io.stdout:write(string.format('%s\n', response.content))
+end
+
+local function handle_neural_train(args)
+
+ local this_where -- which class of messages are we collecting data for
+ local ham_rows, spam_rows = {}, {}
+ local want_spam, want_ham = true, true -- keep collecting while true
+
+ -- Try find profile in config
+ local neural_opts = rspamd_config:get_all_opt('neural')
+ local symbols_profile = ((((neural_opts or E).rules or E)[args.rule] or E).profile or E)[args.profile]
+ if not symbols_profile then
+ io.stderr:write(string.format("Couldn't find profile %s in rule %s\n", args.profile, args.rule))
+ os.exit(1)
+ end
+ -- Try find max_trains
+ local max_trains = (neural_opts.rules[args.rule].train or E).max_trains or 1000
+
+ -- Callback used to process rows from Clickhouse
+ local function process_row(r)
+ local destination -- which table to collect this information in
+ if this_where == args.ham then
+ destination = ham_rows
+ if #destination >= max_trains then
+ want_ham = false
+ return
+ end
+ else
+ destination = spam_rows
+ if #destination >= max_trains then
+ want_spam = false
+ return
+ end
+ end
+ local ucl_parser = ucl.parser()
+ local ok, err = ucl_parser:parse_string(r[args.column_name_vector], 'msgpack')
+ if not ok then
+ io.stderr:write(string.format("Couldn't parse [%s]: %s", r[args.column_name_vector], err))
+ os.exit(1)
+ end
+ table.insert(destination, ucl_parser:get_object())
+ end
+
+ -- Generate symbols digest
+ table.sort(symbols_profile)
+ local symbols_digest = lua_util.table_digest(symbols_profile)
+ -- Create list of days to query data for
+ local query_days = days_list(args.days)
+ -- Set value for limit
+ local limit = ''
+ local num_limit = tonumber(args.limit)
+ if num_limit then
+ limit = string.format(' LIMIT %d', num_limit) -- Contains leading space
+ end
+ -- Prepare query elements
+ local conditions = { string.format("%s = '%s'", args.column_name_digest, symbols_digest) }
+ local query_fmt = 'SELECT %s FROM rspamd WHERE %s%s'
+
+ -- Run queries
+ for _, the_where in ipairs({ args.ham, args.spam }) do
+ -- Inform callback which group of vectors we're collecting
+ this_where = the_where
+ table.insert(conditions, the_where) -- should be 2nd from last condition
+ -- Loop over days and try collect data
+ for _, query_day in ipairs(query_days) do
+ -- Break the loop if we have enough data already
+ if this_where == args.ham then
+ if not want_ham then
+ break
+ end
+ else
+ if not want_spam then
+ break
+ end
+ end
+ -- Date should be the last condition
+ table.insert(conditions, string.format("Date = '%s'", query_day))
+ local query = string.format(query_fmt, args.column_name_vector, table.concat(conditions, ' AND '), limit)
+ local upstream = args.upstream:get_upstream_round_robin()
+ local err = lua_clickhouse.select_sync(upstream, args, http_params, query, process_row)
+ if err ~= nil then
+ io.stderr:write(string.format('Error querying Clickhouse: %s\n', err))
+ os.exit(1)
+ end
+ conditions[#conditions] = nil -- remove Date condition
+ end
+ conditions[#conditions] = nil -- remove spam/ham condition
+ end
+
+ -- Make sure we collected enough data for training
+ if #ham_rows < max_trains then
+ io.stderr:write(string.format('Insufficient ham rows: %d/%d\n', #ham_rows, max_trains))
+ os.exit(1)
+ end
+ if #spam_rows < max_trains then
+ io.stderr:write(string.format('Insufficient spam rows: %d/%d\n', #spam_rows, max_trains))
+ os.exit(1)
+ end
+
+ return post_neural_training(args.url, args.rule, spam_rows, ham_rows)
+end
+
+local command_handlers = {
+ neural_profile = handle_neural_profile,
+ neural_train = handle_neural_train,
+}
+
+local function handler(args)
+ local cmd_opts = parser:parse(args)
+
+ load_config(cmd_opts.config_file)
+ local cfg_opts = rspamd_config:get_all_opt('clickhouse')
+
+ if cmd_opts.ask_password then
+ local rspamd_util = require "rspamd_util"
+
+ io.write('Password: ')
+ cmd_opts.password = rspamd_util.readpassphrase()
+ end
+
+ local function override_settings(params)
+ for _, which in ipairs(params) do
+ if cmd_opts[which] == nil then
+ cmd_opts[which] = cfg_opts[which]
+ end
+ end
+ end
+
+ override_settings({
+ 'database', 'no_ssl_verify', 'password', 'server',
+ 'use_gzip', 'use_https', 'user',
+ })
+
+ local servers = cmd_opts['server'] or cmd_opts['servers']
+ if not servers then
+ parser:error("server(s) unspecified & couldn't be fetched from config")
+ end
+
+ cmd_opts.upstream = rspamd_upstream_list.create(rspamd_config, servers, 8123)
+
+ if not cmd_opts.upstream then
+ io.stderr:write(string.format("can't parse clickhouse address: %s\n", servers))
+ os.exit(1)
+ end
+
+ local f = command_handlers[cmd_opts.command]
+ if not f then
+ parser:error(string.format("command isn't implemented: %s",
+ cmd_opts.command))
+ end
+ f(cmd_opts)
+end
+
+return {
+ handler = handler,
+ description = parser._description,
+ name = 'clickhouse'
+}
diff --git a/lualib/rspamadm/configgraph.lua b/lualib/rspamadm/configgraph.lua
new file mode 100644
index 0000000..07f14a9
--- /dev/null
+++ b/lualib/rspamadm/configgraph.lua
@@ -0,0 +1,172 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local rspamd_regexp = require "rspamd_regexp"
+local argparse = require "argparse"
+
+-- Define command line options
+local parser = argparse()
+ :name "rspamadm configgraph"
+ :description "Produces graph of Rspamd includes"
+ :help_description_margin(30)
+parser:option "-c --config"
+ :description "Path to config file"
+ :argname("<file>")
+ :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
+parser:flag "-a --all"
+ :description('Show all nodes, not just existing ones')
+
+local function process_filename(fname)
+ local cdir = rspamd_paths['CONFDIR'] .. '/'
+ fname = fname:gsub(cdir, '')
+ return fname
+end
+
+local function output_dot(opts, nodes, adjacency)
+ rspamd_logger.messagex("digraph rspamd {")
+ for k, node in pairs(nodes) do
+ local attrs = { "shape=box" }
+ local skip = false
+ if node.exists then
+ if node.priority >= 10 then
+ attrs[#attrs + 1] = "color=red"
+ elseif node.priority > 0 then
+ attrs[#attrs + 1] = "color=blue"
+ end
+ else
+ if opts.all then
+ attrs[#attrs + 1] = "style=dotted"
+ else
+ skip = true
+ end
+ end
+
+ if not skip then
+ rspamd_logger.messagex("\"%s\" [%s];", process_filename(k),
+ table.concat(attrs, ','))
+ end
+ end
+ for _, adj in ipairs(adjacency) do
+ local attrs = {}
+ local skip = false
+
+ if adj.to.exists then
+ if adj.to.merge then
+ attrs[#attrs + 1] = "arrowhead=diamond"
+ attrs[#attrs + 1] = "label=\"+\""
+ elseif adj.to.priority > 1 then
+ attrs[#attrs + 1] = "color=red"
+ end
+ else
+ if opts.all then
+ attrs[#attrs + 1] = "style=dotted"
+ else
+ skip = true
+ end
+ end
+
+ if not skip then
+ rspamd_logger.messagex("\"%s\" -> \"%s\" [%s];", process_filename(adj.from),
+ adj.to.short_path, table.concat(attrs, ','))
+ end
+ end
+ rspamd_logger.messagex("}")
+end
+
+local function load_config_traced(opts)
+ local glob_traces = {}
+ local adjacency = {}
+ local nodes = {}
+
+ local function maybe_match_glob(file)
+ for _, gl in ipairs(glob_traces) do
+ if gl.re:match(file) then
+ return gl
+ end
+ end
+
+ return nil
+ end
+
+ local function add_dep(from, node, args)
+ adjacency[#adjacency + 1] = {
+ from = from,
+ to = node,
+ args = args
+ }
+ end
+
+ local function process_node(fname, args)
+ local node = nodes[fname]
+ if not node then
+ node = {
+ path = fname,
+ short_path = process_filename(fname),
+ exists = rspamd_util.file_exists(fname),
+ merge = args.duplicate and args.duplicate == 'merge',
+ priority = args.priority or 0,
+ glob = args.glob,
+ try = args.try,
+ }
+ nodes[fname] = node
+ end
+
+ return node
+ end
+
+ local function trace_func(cur_file, included_file, args, parent)
+ if args.glob then
+ glob_traces[#glob_traces + 1] = {
+ re = rspamd_regexp.import_glob(included_file, ''),
+ parent = cur_file,
+ args = args,
+ seen = {},
+ }
+ else
+ local node = process_node(included_file, args)
+ if opts.all or node.exists then
+ local gl_parent = maybe_match_glob(included_file)
+ if gl_parent and not gl_parent.seen[cur_file] then
+ add_dep(gl_parent.parent, nodes[cur_file], gl_parent.args)
+ gl_parent.seen[cur_file] = true
+ end
+ add_dep(cur_file, node, args)
+ end
+ end
+ end
+
+ local _r, err = rspamd_config:load_ucl(opts['config'], trace_func)
+ if not _r then
+ rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ output_dot(opts, nodes, adjacency)
+end
+
+local function handler(args)
+ local res = parser:parse(args)
+
+ load_config_traced(res)
+end
+
+return {
+ handler = handler,
+ description = parser._description,
+ name = 'configgraph'
+} \ No newline at end of file
diff --git a/lualib/rspamadm/confighelp.lua b/lualib/rspamadm/confighelp.lua
new file mode 100644
index 0000000..38b26b6
--- /dev/null
+++ b/lualib/rspamadm/confighelp.lua
@@ -0,0 +1,123 @@
+local opts
+local known_attrs = {
+ data = 1,
+ example = 1,
+ type = 1,
+ required = 1,
+ default = 1,
+}
+local argparse = require "argparse"
+local ansicolors = require "ansicolors"
+
+local parser = argparse()
+ :name "rspamadm confighelp"
+ :description "Shows help for the specified configuration options"
+ :help_description_margin(32)
+parser:argument "path":args "*"
+ :description('Optional config paths')
+parser:flag "--no-color"
+ :description "Disable coloured output"
+parser:flag "--short"
+ :description "Show only option names"
+parser:flag "--no-examples"
+ :description "Do not show examples (implied by --short)"
+
+local function maybe_print_color(key)
+ if not opts['no-color'] then
+ return ansicolors.white .. key .. ansicolors.reset
+ else
+ return key
+ end
+end
+
+local function sort_values(tbl)
+ local res = {}
+ for k, v in pairs(tbl) do
+ table.insert(res, { key = k, value = v })
+ end
+
+ -- Sort order
+ local order = {
+ options = 1,
+ dns = 2,
+ upstream = 3,
+ logging = 4,
+ metric = 5,
+ composite = 6,
+ classifier = 7,
+ modules = 8,
+ lua = 9,
+ worker = 10,
+ workers = 11,
+ }
+
+ table.sort(res, function(a, b)
+ local oa = order[a['key']]
+ local ob = order[b['key']]
+
+ if oa and ob then
+ return oa < ob
+ elseif oa then
+ return -1 < 0
+ elseif ob then
+ return 1 < 0
+ else
+ return a['key'] < b['key']
+ end
+
+ end)
+
+ return res
+end
+
+local function print_help(key, value, tabs)
+ print(string.format('%sConfiguration element: %s', tabs, maybe_print_color(key)))
+
+ if not opts['short'] then
+ if value['data'] then
+ local nv = string.match(value['data'], '^#%s*(.*)%s*$') or value.data
+ print(string.format('%s\tDescription: %s', tabs, nv))
+ end
+ if type(value['type']) == 'string' then
+ print(string.format('%s\tType: %s', tabs, value['type']))
+ end
+ if type(value['required']) == 'boolean' then
+ if value['required'] then
+ print(string.format('%s\tRequired: %s', tabs,
+ maybe_print_color(tostring(value['required']))))
+ else
+ print(string.format('%s\tRequired: %s', tabs,
+ tostring(value['required'])))
+ end
+ end
+ if value['default'] then
+ print(string.format('%s\tDefault: %s', tabs, value['default']))
+ end
+ if not opts['no-examples'] and value['example'] then
+ local nv = string.match(value['example'], '^%s*(.*[^%s])%s*$') or value.example
+ print(string.format('%s\tExample:\n%s', tabs, nv))
+ end
+ if value.type and value.type == 'object' then
+ print('')
+ end
+ end
+
+ local sorted = sort_values(value)
+ for _, v in ipairs(sorted) do
+ if not known_attrs[v['key']] then
+ -- We need to go deeper
+ print_help(v['key'], v['value'], tabs .. '\t')
+ end
+ end
+end
+
+return function(args, res)
+ opts = parser:parse(args)
+
+ local sorted = sort_values(res)
+
+ for _, v in ipairs(sorted) do
+ print_help(v['key'], v['value'], '')
+ print('')
+ end
+end
diff --git a/lualib/rspamadm/configwizard.lua b/lualib/rspamadm/configwizard.lua
new file mode 100644
index 0000000..2637036
--- /dev/null
+++ b/lualib/rspamadm/configwizard.lua
@@ -0,0 +1,849 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local ansicolors = require "ansicolors"
+local local_conf = rspamd_paths['CONFDIR']
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local lua_stat_tools = require "lua_stat"
+local lua_redis = require "lua_redis"
+local ucl = require "ucl"
+local argparse = require "argparse"
+local fun = require "fun"
+
+local plugins_stat = require "plugins_stats"
+
+local rspamd_logo = [[
+ ____ _
+ | _ \ ___ _ __ __ _ _ __ ___ __| |
+ | |_) |/ __|| '_ \ / _` || '_ ` _ \ / _` |
+ | _ < \__ \| |_) || (_| || | | | | || (_| |
+ |_| \_\|___/| .__/ \__,_||_| |_| |_| \__,_|
+ |_|
+]]
+
+local parser = argparse()
+ :name "rspamadm configwizard"
+ :description "Perform guided configuration for Rspamd daemon"
+ :help_description_margin(32)
+parser:option "-c --config"
+ :description "Path to config file"
+ :argname("<file>")
+ :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
+parser:argument "checks"
+ :description "Checks to do (or 'list')"
+ :argname("<checks>")
+ :args "*"
+
+local redis_params
+
+local function printf(fmt, ...)
+ if fmt then
+ io.write(string.format(fmt, ...))
+ end
+ io.write('\n')
+end
+
+local function highlight(str)
+ return ansicolors.white .. str .. ansicolors.reset
+end
+
+local function ask_yes_no(greet, default)
+ local def_str
+ if default then
+ greet = greet .. "[Y/n]: "
+ def_str = "yes"
+ else
+ greet = greet .. "[y/N]: "
+ def_str = "no"
+ end
+
+ local reply = rspamd_util.readline(greet)
+
+ if not reply then
+ os.exit(0)
+ end
+ if #reply == 0 then
+ reply = def_str
+ end
+ reply = reply:lower()
+ if reply == 'y' or reply == 'yes' then
+ return true
+ end
+
+ return false
+end
+
+local function readline_default(greet, def_value)
+ local reply = rspamd_util.readline(greet)
+ if not reply then
+ os.exit(0)
+ end
+
+ if #reply == 0 then
+ return def_value
+ end
+
+ return reply
+end
+
+local function readline_expire()
+ local expire = '100d'
+ repeat
+ expire = readline_default("Expire time for new tokens [" .. expire .. "]: ",
+ expire)
+ expire = lua_util.parse_time_interval(expire)
+
+ if not expire then
+ expire = '100d'
+ elseif expire > 2147483647 then
+ printf("The maximum possible value is 2147483647 (about 68y)")
+ expire = '68y'
+ elseif expire < -1 then
+ printf("The value must be a non-negative integer or -1")
+ expire = -1
+ elseif expire ~= math.floor(expire) then
+ printf("The value must be an integer")
+ expire = math.floor(expire)
+ else
+ return expire
+ end
+ until false
+end
+
+local function print_changes(changes)
+ local function print_change(k, c, where)
+ printf('File: %s, changes list:', highlight(local_conf .. '/'
+ .. where .. '/' .. k))
+
+ for ek, ev in pairs(c) do
+ printf("%s => %s", highlight(ek), rspamd_logger.slog("%s", ev))
+ end
+ end
+ for k, v in pairs(changes.l) do
+ print_change(k, v, 'local.d')
+ if changes.o[k] then
+ v = changes.o[k]
+ print_change(k, v, 'override.d')
+ end
+ print()
+ end
+end
+
+local function apply_changes(changes)
+ local function dirname(fname)
+ if fname:match(".-/.-") then
+ return string.gsub(fname, "(.*/)(.*)", "%1")
+ else
+ return nil
+ end
+ end
+
+ local function apply_change(k, c, where)
+ local fname = local_conf .. '/' .. where .. '/' .. k
+
+ if not rspamd_util.file_exists(fname) then
+ printf("Create file %s", highlight(fname))
+
+ local dname = dirname(fname)
+
+ if dname then
+ local ret, err = rspamd_util.mkdir(dname, true)
+
+ if not ret then
+ printf("Cannot make directory %s: %s", dname, highlight(err))
+ os.exit(1)
+ end
+ end
+ end
+
+ local f = io.open(fname, "a+")
+
+ if not f then
+ printf("Cannot open file %s, aborting", highlight(fname))
+ os.exit(1)
+ end
+
+ f:write(ucl.to_config(c))
+
+ f:close()
+ end
+ for k, v in pairs(changes.l) do
+ apply_change(k, v, 'local.d')
+ if changes.o[k] then
+ v = changes.o[k]
+ apply_change(k, v, 'override.d')
+ end
+ end
+end
+
+local function setup_controller(controller, changes)
+ printf("Setup %s and controller worker:", highlight("WebUI"))
+
+ if not controller.password or controller.password == 'q1' then
+ if ask_yes_no("Controller password is not set, do you want to set one?", true) then
+ local pw_encrypted = rspamadm.pw_encrypt()
+ if pw_encrypted then
+ printf("Set encrypted password to: %s", highlight(pw_encrypted))
+ changes.l['worker-controller.inc'] = {
+ password = pw_encrypted
+ }
+ end
+ end
+ end
+end
+
+local function setup_redis(cfg, changes)
+ local function parse_servers(servers)
+ local ls = lua_util.rspamd_str_split(servers, ",")
+
+ return ls
+ end
+
+ printf("%s servers are not set:", highlight("Redis"))
+ printf("The following modules will be enabled if you add Redis servers:")
+
+ for k, _ in pairs(rspamd_plugins_state.disabled_redis) do
+ printf("\t* %s", highlight(k))
+ end
+
+ if ask_yes_no("Do you wish to set Redis servers?", true) then
+ local read_servers = readline_default("Input read only servers separated by `,` [default: localhost]: ",
+ "localhost")
+
+ local rs = parse_servers(read_servers)
+ if rs and #rs > 0 then
+ changes.l['redis.conf'] = {
+ read_servers = table.concat(rs, ",")
+ }
+ end
+ local write_servers = readline_default("Input write only servers separated by `,` [default: "
+ .. read_servers .. "]: ", read_servers)
+
+ if not write_servers or #write_servers == 0 then
+ printf("Use read servers %s as write servers", highlight(table.concat(rs, ",")))
+ write_servers = read_servers
+ end
+
+ redis_params = {
+ read_servers = rs,
+ }
+
+ local ws = parse_servers(write_servers)
+ if ws and #ws > 0 then
+ changes.l['redis.conf']['write_servers'] = table.concat(ws, ",")
+ redis_params['write_servers'] = ws
+ end
+
+ if ask_yes_no('Do you have any username set for your Redis (ACL SETUSER and Redis 6.0+)') then
+ local username = readline_default("Enter Redis username:", nil)
+
+ if username then
+ changes.l['redis.conf'].username = username
+ redis_params.username = username
+ end
+
+ local passwd = readline_default("Enter Redis password:", nil)
+
+ if passwd then
+ changes.l['redis.conf']['password'] = passwd
+ redis_params['password'] = passwd
+ end
+ elseif ask_yes_no('Do you have any password set for your Redis?') then
+ local passwd = readline_default("Enter Redis password:", nil)
+
+ if passwd then
+ changes.l['redis.conf']['password'] = passwd
+ redis_params['password'] = passwd
+ end
+ end
+
+ if ask_yes_no('Do you have any specific database for your Redis?') then
+ local db = readline_default("Enter Redis database:", nil)
+
+ if db then
+ changes.l['redis.conf']['db'] = db
+ redis_params['db'] = db
+ end
+ end
+ end
+end
+
+local function setup_dkim_signing(cfg, changes)
+ -- Remove the trailing slash of a pathname, if present.
+ local function remove_trailing_slash(path)
+ if string.sub(path, -1) ~= "/" then
+ return path
+ end
+ return string.sub(path, 1, string.len(path) - 1)
+ end
+
+ printf('How would you like to set up DKIM signing?')
+ printf('1. Use domain from %s for sign', highlight('mime from header'))
+ printf('2. Use domain from %s for sign', highlight('SMTP envelope from'))
+ printf('3. Use domain from %s for sign', highlight('authenticated user'))
+ printf('4. Sign all mail from %s', highlight('specific networks'))
+ printf()
+
+ local sign_type = readline_default('Enter your choice (1, 2, 3, 4) [default: 1]: ', '1')
+ local sign_networks
+ local allow_mismatch
+ local sign_authenticated
+ local use_esld
+ local sign_domain = 'pet luacheck'
+
+ local defined_auth_types = { 'header', 'envelope', 'auth', 'recipient' }
+
+ if sign_type == '4' then
+ repeat
+ sign_networks = readline_default('Enter list of networks to perform dkim signing: ',
+ '')
+ until #sign_networks ~= 0
+
+ sign_networks = fun.totable(fun.map(lua_util.rspamd_str_trim,
+ lua_util.str_split(sign_networks, ',; ')))
+ printf('What domain would you like to use for signing?')
+ printf('* %s to use mime from domain', highlight('header'))
+ printf('* %s to use SMTP from domain', highlight('envelope'))
+ printf('* %s to use domain from SMTP auth', highlight('auth'))
+ printf('* %s to use domain from SMTP recipient', highlight('recipient'))
+ printf('* anything else to use as a %s domain (e.g. `example.com`)', highlight('static'))
+ printf()
+
+ sign_domain = readline_default('Enter your choice [default: header]: ', 'header')
+ else
+ if sign_type == '1' then
+ sign_domain = 'header'
+ elseif sign_type == '2' then
+ sign_domain = 'envelope'
+ else
+ sign_domain = 'auth'
+ end
+ end
+
+ if sign_type ~= '3' then
+ sign_authenticated = ask_yes_no(
+ string.format('Do you want to sign mail from %s? ',
+ highlight('authenticated users')), true)
+ else
+ sign_authenticated = true
+ end
+
+ if fun.any(function(s)
+ return s == sign_domain
+ end, defined_auth_types) then
+ -- Allow mismatch
+ allow_mismatch = ask_yes_no(
+ string.format('Allow data %s, e.g. if mime from domain is not equal to authenticated user domain? ',
+ highlight('mismatch')), true)
+ -- ESLD check
+ use_esld = ask_yes_no(
+ string.format('Do you want to use %s domain (e.g. example.com instead of foo.example.com)? ',
+ highlight('effective')), true)
+ else
+ allow_mismatch = true
+ end
+
+ local domains = {}
+ local has_domains = false
+
+ local dkim_keys_dir = rspamd_paths["DBDIR"] .. "/dkim/"
+
+ local prompt = string.format("Enter output directory for the keys [default: %s]: ",
+ highlight(dkim_keys_dir))
+ dkim_keys_dir = remove_trailing_slash(readline_default(prompt, dkim_keys_dir))
+
+ local ret, err = rspamd_util.mkdir(dkim_keys_dir, true)
+
+ if not ret then
+ printf("Cannot make directory %s: %s", dkim_keys_dir, highlight(err))
+ os.exit(1)
+ end
+
+ local function print_domains()
+ printf("Domains configured:")
+ for k, v in pairs(domains) do
+ printf("Domain: %s, selector: %s, privkey: %s", highlight(k),
+ v.selector, v.privkey)
+ end
+ printf("--")
+ end
+ local function print_public_key(pk)
+ local base64_pk = tostring(rspamd_util.encode_base64(pk))
+ printf('v=DKIM1; k=rsa; p=%s\n', base64_pk)
+ end
+ repeat
+ if has_domains then
+ print_domains()
+ end
+
+ local domain
+ repeat
+ domain = rspamd_util.readline("Enter domain to sign: ")
+ if not domain then
+ os.exit(1)
+ end
+ until #domain ~= 0
+
+ local selector = readline_default("Enter selector [default: dkim]: ", 'dkim')
+ if not selector then
+ selector = 'dkim'
+ end
+
+ local privkey_file = string.format("%s/%s.%s.key", dkim_keys_dir, domain,
+ selector)
+ if not rspamd_util.file_exists(privkey_file) then
+ if ask_yes_no("Do you want to create privkey " .. highlight(privkey_file),
+ true) then
+ local rsa = require "rspamd_rsa"
+ local sk, pk = rsa.keypair(2048)
+ sk:save(privkey_file, 'pem')
+ print("You need to chown private key file to rspamd user!!")
+ print("To make dkim signing working, to place the following record in your DNS zone:")
+ print_public_key(tostring(pk))
+ end
+ end
+
+ domains[domain] = {
+ selector = selector,
+ path = privkey_file,
+ }
+ until not ask_yes_no("Do you wish to add another DKIM domain?")
+
+ changes.l['dkim_signing.conf'] = { domain = domains }
+ local res_tbl = changes.l['dkim_signing.conf']
+
+ if sign_networks then
+ res_tbl.sign_networks = sign_networks
+ res_tbl.use_domain_sign_networks = sign_domain
+ else
+ res_tbl.use_domain = sign_domain
+ end
+
+ if allow_mismatch then
+ res_tbl.allow_hdrfrom_mismatch = true
+ res_tbl.allow_hdrfrom_mismatch_sign_networks = true
+ res_tbl.allow_username_mismatch = true
+ end
+
+ res_tbl.use_esld = use_esld
+ res_tbl.sign_authenticated = sign_authenticated
+end
+
+local function check_redis_classifier(cls, changes)
+ local symbol_spam, symbol_ham
+ -- Load symbols from statfiles
+ local statfiles = cls.statfile
+ for _, stf in ipairs(statfiles) do
+ local symbol = stf.symbol or 'undefined'
+
+ local spam
+ if stf.spam then
+ spam = stf.spam
+ else
+ if string.match(symbol:upper(), 'SPAM') then
+ spam = true
+ else
+ spam = false
+ end
+ end
+
+ if spam then
+ symbol_spam = symbol
+ else
+ symbol_ham = symbol
+ end
+ end
+
+ if not symbol_spam or not symbol_ham then
+ printf("Classifier has no symbols defined")
+ return
+ end
+
+ local parsed_redis = lua_redis.try_load_redis_servers(cls, nil)
+
+ if not parsed_redis and redis_params then
+ parsed_redis = lua_redis.try_load_redis_servers(redis_params, nil)
+ if not parsed_redis then
+ printf("Cannot parse Redis params")
+ return
+ end
+ end
+
+ local function try_convert(update_config)
+ if ask_yes_no("Do you wish to convert data to the new schema?", true) then
+ local expire = readline_expire()
+ if not lua_stat_tools.convert_bayes_schema(parsed_redis, symbol_spam,
+ symbol_ham, expire) then
+ printf("Conversion failed")
+ else
+ printf("Conversion succeed")
+ if update_config then
+ changes.l['classifier-bayes.conf'] = {
+ new_schema = true,
+ }
+
+ if expire then
+ changes.l['classifier-bayes.conf'].expire = expire
+ end
+ end
+ end
+ end
+ end
+
+ local function get_version(conn)
+ conn:add_cmd("SMEMBERS", { "RS_keys" })
+
+ local ret, members = conn:exec()
+
+ -- Empty db
+ if not ret or #members == 0 then
+ return false, 0
+ end
+
+ -- We still need to check versions
+ local lua_script = [[
+local ver = 0
+
+local tst = redis.call('GET', KEYS[1]..'_version')
+if tst then
+ ver = tonumber(tst) or 0
+end
+
+return ver
+]]
+ conn:add_cmd('EVAL', { lua_script, '1', 'RS' })
+ local _, ver = conn:exec()
+
+ return true, tonumber(ver)
+ end
+
+ local function check_expire(conn)
+ -- We still need to check versions
+ local lua_script = [[
+local ttl = 0
+
+local sc = redis.call('SCAN', 0, 'MATCH', 'RS*_*', 'COUNT', 1)
+local _,key = sc[1], sc[2]
+
+if key and key[1] then
+ ttl = redis.call('TTL', key[1])
+end
+
+return ttl
+]]
+ conn:add_cmd('EVAL', { lua_script, '0' })
+ local _, ttl = conn:exec()
+
+ return tonumber(ttl)
+ end
+
+ local res, conn = lua_redis.redis_connect_sync(parsed_redis, true)
+ if not res then
+ printf("Cannot connect to Redis server")
+ return false
+ end
+
+ if not cls.new_schema then
+ local r, ver = get_version(conn)
+ if not r then
+ return false
+ end
+ if ver ~= 2 then
+ if not ver then
+ printf('Key "RS_version" has not been found in Redis for %s/%s',
+ symbol_ham, symbol_spam)
+ else
+ printf("You are using an old schema version: %s for %s/%s",
+ ver, symbol_ham, symbol_spam)
+ end
+ try_convert(true)
+ else
+ printf("You have configured an old schema for %s/%s but your data has new layout",
+ symbol_ham, symbol_spam)
+
+ if ask_yes_no("Switch config to the new schema?", true) then
+ changes.l['classifier-bayes.conf'] = {
+ new_schema = true,
+ }
+
+ local expire = check_expire(conn)
+ if expire then
+ changes.l['classifier-bayes.conf'].expire = expire
+ end
+ end
+ end
+ else
+ local r, ver = get_version(conn)
+ if not r then
+ return false
+ end
+ if ver ~= 2 then
+ printf("You have configured new schema for %s/%s but your DB has old version: %s",
+ symbol_spam, symbol_ham, ver)
+ try_convert(false)
+ else
+ printf(
+ 'You have configured new schema for %s/%s and your DB already has new layout (v. %s).' ..
+ ' DB conversion is not needed.',
+ symbol_spam, symbol_ham, ver)
+ end
+ end
+end
+
+local function setup_statistic(cfg, changes)
+ local sqlite_configs = lua_stat_tools.load_sqlite_config(cfg)
+
+ if #sqlite_configs > 0 then
+
+ if not redis_params then
+ printf('You have %d sqlite classifiers, but you have no Redis servers being set',
+ #sqlite_configs)
+ return false
+ end
+
+ local parsed_redis = lua_redis.try_load_redis_servers(redis_params, nil)
+ if parsed_redis then
+ printf('You have %d sqlite classifiers', #sqlite_configs)
+ local expire = readline_expire()
+
+ local reset_previous = ask_yes_no("Reset previous data?")
+ if ask_yes_no('Do you wish to convert them to Redis?', true) then
+
+ for _, cls in ipairs(sqlite_configs) do
+ if rspamd_util.file_exists(cls.db_spam) and rspamd_util.file_exists(cls.db_ham) then
+ if not lua_stat_tools.convert_sqlite_to_redis(parsed_redis, cls.db_spam,
+ cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, expire,
+ reset_previous) then
+ rspamd_logger.errx('conversion failed')
+
+ return false
+ end
+ else
+ rspamd_logger.messagex('cannot find %s and %s, skip conversion',
+ cls.db_spam, cls.db_ham)
+ end
+
+ rspamd_logger.messagex('Converted classifier to the from sqlite to redis')
+ changes.l['classifier-bayes.conf'] = {
+ backend = 'redis',
+ new_schema = true,
+ }
+
+ if expire then
+ changes.l['classifier-bayes.conf'].expire = expire
+ end
+
+ if cls.learn_cache then
+ changes.l['classifier-bayes.conf'].cache = {
+ backend = 'redis'
+ }
+ end
+ end
+ end
+ end
+ else
+ -- Check sanity for the existing Redis classifiers
+ local classifier = cfg.classifier
+
+ if classifier then
+ if classifier[1] then
+ for _, cls in ipairs(classifier) do
+ if cls.bayes then
+ cls = cls.bayes
+ end
+ if cls.backend and cls.backend == 'redis' then
+ check_redis_classifier(cls, changes)
+ end
+ end
+ else
+ if classifier.bayes then
+
+ classifier = classifier.bayes
+ if classifier[1] then
+ for _, cls in ipairs(classifier) do
+ if cls.backend and cls.backend == 'redis' then
+ check_redis_classifier(cls, changes)
+ end
+ end
+ else
+ if classifier.backend and classifier.backend == 'redis' then
+ check_redis_classifier(classifier, changes)
+ end
+ end
+ end
+ end
+ end
+ end
+end
+
+local function find_worker(cfg, wtype)
+ if cfg.worker then
+ for k, s in pairs(cfg.worker) do
+ if type(k) == 'number' and type(s) == 'table' then
+ if s[wtype] then
+ return s[wtype]
+ end
+ end
+ if type(s) == 'table' and s.type and s.type == wtype then
+ return s
+ end
+ if type(k) == 'string' and k == wtype then
+ return s
+ end
+ end
+ end
+
+ return nil
+end
+
+return {
+ handler = function(cmd_args)
+ local changes = {
+ l = {}, -- local changes
+ o = {}, -- override changes
+ }
+
+ local interactive_start = true
+ local checks = {}
+ local all_checks = {
+ 'controller',
+ 'redis',
+ 'dkim',
+ 'statistic',
+ }
+
+ local opts = parser:parse(cmd_args)
+ local args = opts['checks'] or {}
+
+ local _r, err = rspamd_config:load_ucl(opts['config'])
+
+ if not _r then
+ rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
+ if not _r then
+ rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ local cfg = rspamd_config:get_ucl()
+
+ if not rspamd_config:init_modules() then
+ rspamd_logger.errx('cannot init modules when parsing %s', opts['config'])
+ os.exit(1)
+ end
+
+ if #args > 0 then
+ interactive_start = false
+
+ for _, arg in ipairs(args) do
+ if arg == 'all' then
+ checks = all_checks
+ elseif arg == 'list' then
+ printf(highlight(rspamd_logo))
+ printf('Available modules')
+ for _, c in ipairs(all_checks) do
+ printf('- %s', c)
+ end
+ return
+ else
+ table.insert(checks, arg)
+ end
+ end
+ else
+ checks = all_checks
+ end
+
+ local function has_check(check)
+ for _, c in ipairs(checks) do
+ if c == check then
+ return true
+ end
+ end
+
+ return false
+ end
+
+ rspamd_util.umask('022')
+ if interactive_start then
+ printf(highlight(rspamd_logo))
+ printf("Welcome to the configuration tool")
+ printf("We use %s configuration file, writing results to %s",
+ highlight(opts['config']), highlight(local_conf))
+ plugins_stat(nil, nil)
+ end
+
+ if not interactive_start or
+ ask_yes_no("Do you wish to continue?", true) then
+
+ if has_check('controller') then
+ local controller = find_worker(cfg, 'controller')
+ if controller then
+ setup_controller(controller, changes)
+ end
+ end
+
+ if has_check('redis') then
+ if not cfg.redis or (not cfg.redis.servers and not cfg.redis.read_servers) then
+ setup_redis(cfg, changes)
+ else
+ redis_params = cfg.redis
+ end
+ else
+ redis_params = cfg.redis
+ end
+
+ if has_check('dkim') then
+ if cfg.dkim_signing and not cfg.dkim_signing.domain then
+ if ask_yes_no('Do you want to setup dkim signing feature?') then
+ setup_dkim_signing(cfg, changes)
+ end
+ end
+ end
+
+ if has_check('statistic') or has_check('statistics') then
+ setup_statistic(cfg, changes)
+ end
+
+ local nchanges = 0
+ for _, _ in pairs(changes.l) do
+ nchanges = nchanges + 1
+ end
+ for _, _ in pairs(changes.o) do
+ nchanges = nchanges + 1
+ end
+
+ if nchanges > 0 then
+ print_changes(changes)
+ if ask_yes_no("Apply changes?", true) then
+ apply_changes(changes)
+ printf("%d changes applied, the wizard is finished now", nchanges)
+ printf("*** Please reload the Rspamd configuration ***")
+ else
+ printf("No changes applied, the wizard is finished now")
+ end
+ else
+ printf("No changes found, the wizard is finished now")
+ end
+ end
+ end,
+ name = 'configwizard',
+ description = parser._description,
+}
diff --git a/lualib/rspamadm/cookie.lua b/lualib/rspamadm/cookie.lua
new file mode 100644
index 0000000..7e0526a
--- /dev/null
+++ b/lualib/rspamadm/cookie.lua
@@ -0,0 +1,125 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+
+
+-- Define command line options
+local parser = argparse()
+ :name "rspamadm cookie"
+ :description "Produces cookies or message ids"
+ :help_description_margin(30)
+
+parser:mutex(
+ parser:option "-k --key"
+ :description('Key to load')
+ :argname "<32hex>",
+ parser:flag "-K --new-key"
+ :description('Generates a new key')
+)
+
+parser:option "-d --domain"
+ :description('Use specified domain and generate full message id')
+ :argname "<domain>"
+parser:flag "-D --decrypt"
+ :description('Decrypt cookie instead of encrypting one')
+parser:flag "-t --timestamp"
+ :description('Show cookie timestamp (valid for decrypting only)')
+parser:argument "cookie":args "?"
+ :description('Use specified cookie')
+
+local function gen_cookie(args, key)
+ local cr = require "rspamd_cryptobox"
+
+ if not args.cookie then
+ return
+ end
+
+ local function encrypt()
+ if #args.cookie > 31 then
+ print('cookie too long (>31 characters), cannot encrypt')
+ os.exit(1)
+ end
+
+ local enc_cookie = cr.encrypt_cookie(key, args.cookie)
+ if args.domain then
+ print(string.format('<%s@%s>', enc_cookie, args.domain))
+ else
+ print(enc_cookie)
+ end
+ end
+
+ local function decrypt()
+ local extracted_cookie = args.cookie:match('^%<?([^@]+)@.*$')
+ if not extracted_cookie then
+ -- Assume full message id as a cookie
+ extracted_cookie = args.cookie
+ end
+
+ local dec_cookie, ts = cr.decrypt_cookie(key, extracted_cookie)
+
+ if dec_cookie then
+ if args.timestamp then
+ print(string.format('%s %s', dec_cookie, ts))
+ else
+ print(dec_cookie)
+ end
+ else
+ print('cannot decrypt cookie')
+ os.exit(1)
+ end
+ end
+
+ if args.decrypt then
+ decrypt()
+ else
+ encrypt()
+ end
+end
+
+local function handler(args)
+ local res = parser:parse(args)
+
+ if not (res.key or res['new_key']) then
+ parser:error('--key or --new-key must be specified')
+ end
+
+ if res.key then
+ local pattern = { '^' }
+ for i = 1, 32 do
+ pattern[i + 1] = '[a-zA-Z0-9]'
+ end
+ pattern[34] = '$'
+
+ if not res.key:match(table.concat(pattern, '')) then
+ parser:error('invalid key: ' .. res.key)
+ end
+
+ gen_cookie(res, res.key)
+ else
+ local util = require "rspamd_util"
+ local key = util.random_hex(32)
+
+ print(key)
+ gen_cookie(res, res.key)
+ end
+end
+
+return {
+ handler = handler,
+ description = parser._description,
+ name = 'cookie'
+} \ No newline at end of file
diff --git a/lualib/rspamadm/corpus_test.lua b/lualib/rspamadm/corpus_test.lua
new file mode 100644
index 0000000..0e63f9f
--- /dev/null
+++ b/lualib/rspamadm/corpus_test.lua
@@ -0,0 +1,185 @@
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+local lua_util = require "lua_util"
+local argparse = require "argparse"
+
+local parser = argparse()
+ :name "rspamadm corpus_test"
+ :description "Create logs files from email corpus"
+ :help_description_margin(32)
+
+parser:option "-H --ham"
+ :description("Ham directory")
+ :argname("<dir>")
+parser:option "-S --spam"
+ :description("Spam directory")
+ :argname("<dir>")
+parser:option "-n --conns"
+ :description("Number of parallel connections")
+ :argname("<N>")
+ :convert(tonumber)
+ :default(10)
+parser:option "-o --output"
+ :description("Output file")
+ :argname("<file>")
+ :default('results.log')
+parser:option "-t --timeout"
+ :description("Timeout for client connections")
+ :argname("<sec>")
+ :convert(tonumber)
+ :default(60)
+parser:option "-c --connect"
+ :description("Connect to specific host")
+ :argname("<host>")
+ :default('localhost:11334')
+parser:option "-r --rspamc"
+ :description("Use specific rspamc path")
+ :argname("<path>")
+ :default('rspamc')
+
+local HAM = "HAM"
+local SPAM = "SPAM"
+local opts
+
+local function scan_email(n_parallel, path, timeout)
+
+ local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s",
+ opts.rspamc, opts.connect, n_parallel, timeout, path)
+ local result = assert(io.popen(rspamc_command))
+ result = result:read("*all")
+ return result
+end
+
+local function write_results(results, file)
+
+ local f = io.open(file, 'w')
+
+ for _, result in pairs(results) do
+ local log_line = string.format("%s %.2f %s",
+ result.type, result.score, result.action)
+
+ for _, sym in pairs(result.symbols) do
+ log_line = log_line .. " " .. sym
+ end
+
+ log_line = log_line .. " " .. result.scan_time .. " " .. file .. ':' .. result.filename
+
+ log_line = log_line .. "\r\n"
+
+ f:write(log_line)
+ end
+
+ f:close()
+end
+
+local function encoded_json_to_log(result)
+ -- Returns table containing score, action, list of symbols
+
+ local filtered_result = {}
+ local ucl_parser = ucl.parser()
+
+ local is_good, err = ucl_parser:parse_string(result)
+
+ if not is_good then
+ rspamd_logger.errx("Parser error: %1", err)
+ return nil
+ end
+
+ result = ucl_parser:get_object()
+
+ filtered_result.score = result.score
+ if not result.action then
+ rspamd_logger.errx("Bad JSON: %1", result)
+ return nil
+ end
+ local action = result.action:gsub("%s+", "_")
+ filtered_result.action = action
+
+ filtered_result.symbols = {}
+
+ for sym, _ in pairs(result.symbols) do
+ table.insert(filtered_result.symbols, sym)
+ end
+
+ filtered_result.filename = result.filename
+ filtered_result.scan_time = result.scan_time
+
+ return filtered_result
+end
+
+local function scan_results_to_logs(results, actual_email_type)
+
+ local logs = {}
+
+ results = lua_util.rspamd_str_split(results, "\n")
+
+ if results[#results] == "" then
+ results[#results] = nil
+ end
+
+ for _, result in pairs(results) do
+ result = encoded_json_to_log(result)
+ if result then
+ result['type'] = actual_email_type
+ table.insert(logs, result)
+ end
+ end
+
+ return logs
+end
+
+local function handler(args)
+ opts = parser:parse(args)
+ local ham_directory = opts['ham']
+ local spam_directory = opts['spam']
+ local connections = opts["conns"]
+ local output = opts["output"]
+
+ local results = {}
+
+ local start_time = os.time()
+ local no_of_ham = 0
+ local no_of_spam = 0
+
+ if ham_directory then
+ rspamd_logger.messagex("Scanning ham corpus...")
+ local ham_results = scan_email(connections, ham_directory, opts["timeout"])
+ ham_results = scan_results_to_logs(ham_results, HAM)
+
+ no_of_ham = #ham_results
+
+ for _, result in pairs(ham_results) do
+ table.insert(results, result)
+ end
+ end
+
+ if spam_directory then
+ rspamd_logger.messagex("Scanning spam corpus...")
+ local spam_results = scan_email(connections, spam_directory, opts.timeout)
+ spam_results = scan_results_to_logs(spam_results, SPAM)
+
+ no_of_spam = #spam_results
+
+ for _, result in pairs(spam_results) do
+ table.insert(results, result)
+ end
+ end
+
+ rspamd_logger.messagex("Writing results to %s", output)
+ write_results(results, output)
+
+ rspamd_logger.messagex("Stats: ")
+ local elapsed_time = os.time() - start_time
+ local total_msgs = no_of_ham + no_of_spam
+ rspamd_logger.messagex("Elapsed time: %ss", elapsed_time)
+ rspamd_logger.messagex("No of ham: %s", no_of_ham)
+ rspamd_logger.messagex("No of spam: %s", no_of_spam)
+ rspamd_logger.messagex("Messages/sec: %s", (total_msgs / elapsed_time))
+end
+
+return {
+ name = 'corpustest',
+ aliases = { 'corpus_test', 'corpus' },
+ handler = handler,
+ description = parser._description
+}
diff --git a/lualib/rspamadm/dkim_keygen.lua b/lualib/rspamadm/dkim_keygen.lua
new file mode 100644
index 0000000..211094d
--- /dev/null
+++ b/lualib/rspamadm/dkim_keygen.lua
@@ -0,0 +1,178 @@
+--[[
+Copyright (c) 2023, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+local rspamd_util = require "rspamd_util"
+local rspamd_cryptobox = require "rspamd_cryptobox"
+
+local parser = argparse()
+ :name 'rspamadm dkim_keygen'
+ :description 'Create key pairs for dkim signing'
+ :help_description_margin(30)
+parser:option '-d --domain'
+ :description 'Create a key for a specific domain'
+ :default "example.com"
+parser:option '-s --selector'
+ :description 'Create a key for a specific DKIM selector'
+ :default "mail"
+parser:option '-k --privkey'
+ :description 'Save private key to file instead of printing it to stdout'
+parser:option '-b --bits'
+ :convert(function(input)
+ local n = tonumber(input)
+ if not n or n < 512 or n > 4096 then
+ return nil
+ end
+ return n
+end)
+ :description 'Generate an RSA key with the specified number of bits (512 to 4096)'
+ :default(1024)
+parser:option '-t --type'
+ :description 'Key type: RSA, ED25519 or ED25119-seed'
+ :convert {
+ ['rsa'] = 'rsa',
+ ['RSA'] = 'rsa',
+ ['ed25519'] = 'ed25519',
+ ['ED25519'] = 'ed25519',
+ ['ed25519-seed'] = 'ed25519-seed',
+ ['ED25519-seed'] = 'ed25519-seed',
+}
+ :default 'rsa'
+parser:option '-o --output'
+ :description 'Output public key in the following format: dns, dnskey or plain'
+ :convert {
+ ['dns'] = 'dns',
+ ['plain'] = 'plain',
+ ['dnskey'] = 'dnskey',
+}
+ :default 'dns'
+parser:option '--priv-output'
+ :description 'Output private key in the following format: PEM or DER (for RSA)'
+ :convert {
+ ['pem'] = 'pem',
+ ['der'] = 'der',
+}
+ :default 'pem'
+parser:flag '-f --force'
+ :description 'Force overwrite of existing files'
+
+local function split_string(input, max_length)
+ max_length = max_length or 253
+ local pieces = {}
+ local index = 1
+
+ while index <= #input do
+ local piece = input:sub(index, index + max_length - 1)
+ table.insert(pieces, piece)
+ index = index + max_length
+ end
+
+ return pieces
+end
+
+local function print_public_key_dns(opts, base64_pk)
+ local key_type = opts.type == 'rsa' and 'rsa' or 'ed25519'
+ if #base64_pk < 255 - 2 then
+ io.write(string.format('%s._domainkey IN TXT ( "v=DKIM1; k=%s;" \n\t"p=%s" ) ;\n',
+ opts.selector, key_type, base64_pk))
+ else
+ -- Split it by parts
+ local parts = split_string(base64_pk)
+ io.write(string.format('%s._domainkey IN TXT ( "v=DKIM1; k=%s; "\n', opts.selector, key_type))
+ for i, part in ipairs(parts) do
+ if i == 1 then
+ io.write(string.format('\t"p=%s"\n', part))
+ else
+ io.write(string.format('\t"%s"\n', part))
+ end
+ end
+ io.write(") ; \n")
+ end
+
+end
+
+local function print_public_key(opts, pk, need_base64)
+ local key_type = opts.type == 'rsa' and 'rsa' or 'ed25519'
+ local base64_pk = need_base64 and tostring(rspamd_util.encode_base64(pk)) or tostring(pk)
+ if opts.output == 'plain' then
+ io.write(base64_pk)
+ io.write("\n")
+ elseif opts.output == 'dns' then
+ print_public_key_dns(opts, base64_pk, false)
+ elseif opts.output == 'dnskey' then
+ io.write(string.format('v=DKIM1; k=%s; p=%s\n', key_type, base64_pk))
+ end
+end
+
+local function gen_rsa_key(opts)
+ local rsa = require "rspamd_rsa"
+
+ local sk, pk = rsa.keypair(opts.bits or 1024)
+ if opts.privkey then
+ if opts.force then
+ os.remove(opts.privkey)
+ end
+ sk:save(opts.privkey, opts.priv_output)
+ else
+ sk:save("-", opts.priv_output)
+ end
+
+ -- We generate key directly via lua_rsa and it returns unencoded raw data
+ print_public_key(opts, tostring(pk), true)
+end
+
+local function gen_eddsa_key(opts)
+ local sk, pk = rspamd_cryptobox.gen_dkim_keypair(opts.type)
+
+ if opts.privkey and opts.force then
+ os.remove(opts.privkey)
+ end
+ if not sk:save_in_file(opts.privkey, tonumber('0600', 8)) then
+ io.stderr:write('cannot save private key to ' .. (opts.privkey or 'stdout') .. '\n')
+ os.exit(1)
+ end
+
+ if not opts.privkey then
+ io.write("\n")
+ io.flush()
+ end
+
+ -- gen_dkim_keypair function returns everything encoded in base64, so no need to do anything
+ print_public_key(opts, tostring(pk), false)
+end
+
+local function handler(args)
+ local opts = parser:parse(args)
+
+ if not opts then
+ os.exit(1)
+ end
+
+ if opts.type == 'rsa' then
+ gen_rsa_key(opts)
+ else
+ gen_eddsa_key(opts)
+ end
+end
+
+return {
+ name = 'dkim_keygen',
+ aliases = { 'dkimkeygen' },
+ handler = handler,
+ description = parser._description
+}
+
+
diff --git a/lualib/rspamadm/dmarc_report.lua b/lualib/rspamadm/dmarc_report.lua
new file mode 100644
index 0000000..42c801e
--- /dev/null
+++ b/lualib/rspamadm/dmarc_report.lua
@@ -0,0 +1,737 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+local lua_util = require "lua_util"
+local logger = require "rspamd_logger"
+local lua_redis = require "lua_redis"
+local dmarc_common = require "plugins/dmarc"
+local lupa = require "lupa"
+local rspamd_mempool = require "rspamd_mempool"
+local rspamd_url = require "rspamd_url"
+local rspamd_text = require "rspamd_text"
+local rspamd_util = require "rspamd_util"
+local rspamd_dns = require "rspamd_dns"
+
+local N = 'dmarc_report'
+
+-- Define command line options
+local parser = argparse()
+ :name "rspamadm dmarc_report"
+ :description "Dmarc reports sending tool"
+ :help_description_margin(30)
+
+parser:option "-c --config"
+ :description "Path to config file"
+ :argname("<cfg>")
+ :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
+
+parser:flag "-v --verbose"
+ :description "Enable dmarc specific logging"
+
+parser:flag "-n --no-opt"
+ :description "Do not reset reporting data/send reports"
+
+parser:argument "date"
+ :description "Date to process (today by default)"
+ :argname "<YYYYMMDD>"
+ :args "*"
+parser:option "-b --batch-size"
+ :description "Send reports in batches up to <batch-size> messages"
+ :argname "<number>"
+ :convert(tonumber)
+ :default "10"
+
+local report_template = [[From: "{= from_name =}" <{= from_addr =}>
+To: {= rcpt =}
+{%+ if is_string(bcc) %}Bcc: {= bcc =}{%- endif %}
+Subject: Report Domain: {= reporting_domain =}
+ Submitter: {= submitter =}
+ Report-ID: {= report_id =}
+Date: {= report_date =}
+MIME-Version: 1.0
+Message-ID: <{= message_id =}>
+Content-Type: multipart/mixed;
+ boundary="----=_NextPart_{= uuid =}"
+
+This is a multipart message in MIME format.
+
+------=_NextPart_{= uuid =}
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+This is an aggregate report from {= submitter =}.
+
+Report domain: {= reporting_domain =}
+Submitter: {= submitter =}
+Report ID: {= report_id =}
+
+------=_NextPart_{= uuid =}
+Content-Type: application/gzip
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment;
+ filename="{= submitter =}!{= reporting_domain =}!{= report_start =}!{= report_end =}.xml.gz"
+
+]]
+local report_footer = [[
+
+------=_NextPart_{= uuid =}--]]
+
+local dmarc_settings = {}
+local redis_params
+local redis_attrs = {
+ config = rspamd_config,
+ ev_base = rspamadm_ev_base,
+ session = rspamadm_session,
+ log_obj = rspamd_config,
+ resolver = rspamadm_dns_resolver,
+}
+local pool
+
+local function load_config(opts)
+ local _r, err = rspamd_config:load_ucl(opts['config'])
+
+ if not _r then
+ logger.errx('cannot parse %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
+ if not _r then
+ logger.errx('cannot process %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+end
+
+-- Concat elements using redis_keys.join_char
+local function redis_prefix(...)
+ return table.concat({ ... }, dmarc_settings.reporting.redis_keys.join_char)
+end
+
+local function get_rua(rep_key)
+ local parts = lua_util.str_split(rep_key, dmarc_settings.reporting.redis_keys.join_char)
+
+ if #parts >= 3 then
+ return parts[3]
+ end
+
+ return nil
+end
+
+local function get_domain(rep_key)
+ local parts = lua_util.str_split(rep_key, dmarc_settings.reporting.redis_keys.join_char)
+
+ if #parts >= 3 then
+ return parts[2]
+ end
+
+ return nil
+end
+
+local function gen_uuid()
+ local template = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'
+ return string.gsub(template, '[xy]', function(c)
+ local v = (c == 'x') and math.random(0, 0xf) or math.random(8, 0xb)
+ return string.format('%x', v)
+ end)
+end
+
+local function gen_xml_grammar()
+ local lpeg = require 'lpeg'
+ local lt = lpeg.P('<') / '&lt;'
+ local gt = lpeg.P('>') / '&gt;'
+ local amp = lpeg.P('&') / '&amp;'
+ local quot = lpeg.P('"') / '&quot;'
+ local apos = lpeg.P("'") / '&apos;'
+ local special = lt + gt + amp + quot + apos
+ local grammar = lpeg.Cs((special + 1) ^ 0)
+ return grammar
+end
+
+local xml_grammar = gen_xml_grammar()
+
+local function escape_xml(input)
+ if type(input) == 'string' or type(input) == 'userdata' then
+ return xml_grammar:match(input)
+ else
+ input = tostring(input)
+
+ if input then
+ return xml_grammar:match(input)
+ end
+ end
+
+ return ''
+end
+-- Enable xml escaping in lupa templates
+lupa.filters.escape_xml = escape_xml
+
+-- Creates report XML header
+local function report_header(reporting_domain, report_start, report_end, domain_policy)
+ local report_id = string.format('%s.%d.%d',
+ reporting_domain, report_start, report_end)
+ local xml_template = [[
+<?xml version="1.0" encoding="UTF-8" ?>
+<feedback>
+ <report_metadata>
+ <org_name>{= report_settings.org_name | escape_xml =}</org_name>
+ <email>{= report_settings.email | escape_xml =}</email>
+ <report_id>{= report_id =}</report_id>
+ <date_range>
+ <begin>{= report_start =}</begin>
+ <end>{= report_end =}</end>
+ </date_range>
+ </report_metadata>
+ <policy_published>
+ <domain>{= reporting_domain | escape_xml =}</domain>
+ <adkim>{= domain_policy.adkim | escape_xml =}</adkim>
+ <aspf>{= domain_policy.aspf | escape_xml =}</aspf>
+ <p>{= domain_policy.p | escape_xml =}</p>
+ <sp>{= domain_policy.sp | escape_xml =}</sp>
+ <pct>{= domain_policy.pct | escape_xml =}</pct>
+ </policy_published>
+]]
+ return lua_util.jinja_template(xml_template, {
+ report_settings = dmarc_settings.reporting,
+ report_id = report_id,
+ report_start = report_start,
+ report_end = report_end,
+ domain_policy = domain_policy,
+ reporting_domain = reporting_domain,
+ }, true)
+end
+
+-- Generate xml entry for a preprocessed redis row
+local function entry_to_xml(data)
+ local xml_template = [[<record>
+ <row>
+ <source_ip>{= data.ip =}</source_ip>
+ <count>{= data.count =}</count>
+ <policy_evaluated>
+ <disposition>{= data.disposition =}</disposition>
+ <dkim>{= data.dkim_disposition =}</dkim>
+ <spf>{= data.spf_disposition =}</spf>
+ {% if data.override and data.override ~= '' -%}
+ <reason><type>{= data.override =}</type></reason>
+ {%- endif %}
+ </policy_evaluated>
+ </row>
+ <identifiers>
+ <header_from>{= data.header_from =}</header_from>
+ </identifiers>
+ <auth_results>
+ {% if data.dkim_results[1] -%}
+ {% for d in data.dkim_results -%}
+ <dkim>
+ <domain>{= d.domain =}</domain>
+ <result>{= d.result =}</result>
+ </dkim>
+ {%- endfor %}
+ {%- endif %}
+ <spf>
+ <domain>{= data.spf_domain =}</domain>
+ <result>{= data.spf_result =}</result>
+ </spf>
+ </auth_results>
+</record>
+]]
+ return lua_util.jinja_template(xml_template, { data = data }, true)
+end
+
+-- Process a report entry stored in Redis splitting it to a lua table
+local function process_report_entry(data, score)
+ local split = lua_util.str_split(data, ',')
+ local row = {
+ ip = split[1],
+ spf_disposition = split[2],
+ dkim_disposition = split[3],
+ disposition = split[4],
+ override = split[5],
+ header_from = split[6],
+ dkim_results = {},
+ spf_domain = split[11],
+ spf_result = split[12],
+ count = tonumber(score),
+ }
+ -- Process dkim entries
+ local function dkim_entries_process(dkim_data, result)
+ if dkim_data and dkim_data ~= '' then
+ local dkim_elts = lua_util.str_split(dkim_data, '|')
+ for _, d in ipairs(dkim_elts) do
+ table.insert(row.dkim_results, { domain = d, result = result })
+ end
+ end
+ end
+ dkim_entries_process(split[7], 'pass')
+ dkim_entries_process(split[8], 'fail')
+ dkim_entries_process(split[9], 'temperror')
+ dkim_entries_process(split[9], 'permerror')
+
+ return row
+end
+
+-- Process a single rua entry, validating in DNS if needed
+local function process_rua(dmarc_domain, rua)
+ local parts = lua_util.str_split(rua, ',')
+
+ -- Remove size limitation, as we don't care about them
+ local addrs = {}
+ for _, rua_part in ipairs(parts) do
+ local u = rspamd_url.create(pool, rua_part:gsub('!%d+[kmg]?$', ''))
+ local u2 = rspamd_url.create(pool, dmarc_domain)
+ if u and (u:get_protocol() or '') == 'mailto' and u:get_user() then
+ -- Check each address for sanity
+ if u:get_tld() == u2:get_tld() then
+ -- Same eSLD - always include
+ table.insert(addrs, u)
+ else
+ -- We need to check authority
+ local resolve_str = string.format('%s._report._dmarc.%s',
+ dmarc_domain, u:get_host())
+ local is_ok, results = rspamd_dns.request({
+ config = rspamd_config,
+ session = rspamadm_session,
+ type = 'txt',
+ name = resolve_str,
+ })
+
+ if not is_ok then
+ logger.errx('cannot resolve %s: %s; exclude %s', resolve_str, results, rua_part)
+ else
+ local found = false
+ for _, t in ipairs(results) do
+ if string.match(t, 'v=DMARC1') then
+ found = true
+ break
+ end
+ end
+
+ if not found then
+ logger.errx('%s is not authorized to process reports on %s', dmarc_domain, u:get_host())
+ else
+ -- All good
+ table.insert(addrs, u)
+ end
+ end
+ end
+ else
+ logger.errx('invalid rua url: "%s""', tostring(u or 'null'))
+ end
+ end
+
+ if #addrs > 0 then
+ return addrs
+ end
+
+ return nil
+end
+
+-- Validate reporting domain, extracting rua and checking 3rd party report domains
+-- This function returns a full dmarc record processed + rua as a list of url objects
+local function validate_reporting_domain(reporting_domain)
+ local is_ok, results = rspamd_dns.request({
+ config = rspamd_config,
+ session = rspamadm_session,
+ type = 'txt',
+ name = '_dmarc.' .. reporting_domain,
+ })
+
+ if not is_ok or not results then
+ logger.errx('cannot resolve _dmarc.%s: %s', reporting_domain, results)
+ return nil
+ end
+
+ for _, r in ipairs(results) do
+ local processed, rec = dmarc_common.dmarc_check_record(rspamd_config, r, false)
+ if processed and rec.rua then
+ -- We need to check or alter rua if needed
+ local processed_rua = process_rua(reporting_domain, rec.rua)
+ if processed_rua then
+ rec = rec.raw_elts
+ rec.rua = processed_rua
+
+ -- Fill defaults in a record to avoid nils in a report
+ rec['pct'] = rec['pct'] or 100
+ rec['adkim'] = rec['adkim'] or 'r'
+ rec['aspf'] = rec['aspf'] or 'r'
+ rec['p'] = rec['p'] or 'none'
+ rec['sp'] = rec['sp'] or 'none'
+ return rec
+ end
+ return nil
+ end
+ end
+
+ return nil
+end
+
+-- Returns a list of recipients from a table as a string processing elements if needed
+local function rcpt_list(tbl, func)
+ local res = {}
+ for _, r in ipairs(tbl) do
+ if func then
+ table.insert(res, func(r))
+ else
+ table.insert(res, r)
+ end
+ end
+
+ return table.concat(res, ',')
+end
+
+-- Synchronous smtp send function
+local function send_reports_by_smtp(opts, reports, finish_cb)
+ local lua_smtp = require "lua_smtp"
+ local reports_failed = 0
+ local reports_sent = 0
+ local report_settings = dmarc_settings.reporting
+
+ local function gen_sendmail_cb(report, args)
+ return function(ret, err)
+ -- We modify this from all callbacks
+ args.nreports = args.nreports - 1
+ if not ret then
+ logger.errx("Couldn't send mail for %s: %s", report.reporting_domain, err)
+ reports_failed = reports_failed + 1
+ else
+ reports_sent = reports_sent + 1
+ lua_util.debugm(N, 'successfully sent a report for %s: %s bytes sent',
+ report.reporting_domain, #report.message)
+ end
+
+ -- Tail call to the next batch or to the final function
+ if args.nreports == 0 then
+ if args.next_start > #reports then
+ finish_cb(reports_sent, reports_failed)
+ else
+ args.cont_func(args.next_start)
+ end
+ end
+ end
+ end
+
+ local function send_data_in_batches(cur_batch)
+ local nreports = math.min(#reports - cur_batch + 1, opts.batch_size)
+ local next_start = cur_batch + nreports
+ lua_util.debugm(N, 'send data for %s domains (from %s to %s)',
+ nreports, cur_batch, next_start - 1)
+ -- Shared across all closures
+ local gen_args = {
+ cont_func = send_data_in_batches,
+ nreports = nreports,
+ next_start = next_start
+ }
+ for i = cur_batch, next_start - 1 do
+ local report = reports[i]
+ local send_opts = {
+ ev_base = rspamadm_ev_base,
+ session = rspamadm_session,
+ config = rspamd_config,
+ host = report_settings.smtp,
+ port = report_settings.smtp_port or 25,
+ resolver = rspamadm_dns_resolver,
+ from = report_settings.email,
+ recipients = report.rcpts,
+ helo = report_settings.helo or 'rspamd.localhost',
+ }
+
+ lua_smtp.sendmail(send_opts,
+ report.message,
+ gen_sendmail_cb(report, gen_args))
+ end
+ end
+
+ send_data_in_batches(1)
+end
+
+local function prepare_report(opts, start_time, end_time, rep_key)
+ local rua = get_rua(rep_key)
+ local reporting_domain = get_domain(rep_key)
+
+ if not rua then
+ logger.errx('report %s has no valid rua, skip it', rep_key)
+ return nil
+ end
+ if not reporting_domain then
+ logger.errx('report %s has no valid reporting_domain, skip it', rep_key)
+ return nil
+ end
+
+ local ret, results = lua_redis.request(redis_params, redis_attrs,
+ { 'EXISTS', rep_key })
+
+ if not ret or not results or results == 0 then
+ return nil
+ end
+
+ -- Rename report key to avoid races
+ if not opts.no_opt then
+ lua_redis.request(redis_params, redis_attrs,
+ { 'RENAME', rep_key, rep_key .. '_processing' })
+ rep_key = rep_key .. '_processing'
+ end
+
+ local dmarc_record = validate_reporting_domain(reporting_domain)
+ lua_util.debugm(N, 'process reporting domain %s: %s', reporting_domain, dmarc_record)
+
+ if not dmarc_record then
+ if not opts.no_opt then
+ lua_redis.request(redis_params, redis_attrs,
+ { 'DEL', rep_key })
+ end
+ logger.messagex('Cannot process reports for domain %s; invalid dmarc record', reporting_domain)
+ return nil
+ end
+
+ -- Get all reports for a domain
+ ret, results = lua_redis.request(redis_params, redis_attrs,
+ { 'ZRANGE', rep_key, '0', '-1', 'WITHSCORES' })
+ local report_entries = {}
+ table.insert(report_entries,
+ report_header(reporting_domain, start_time, end_time, dmarc_record))
+ for i = 1, #results, 2 do
+ local xml_record = entry_to_xml(process_report_entry(results[i], results[i + 1]))
+ table.insert(report_entries, xml_record)
+ end
+ table.insert(report_entries, '</feedback>')
+ local xml_to_compress = rspamd_text.fromtable(report_entries)
+ lua_util.debugm(N, 'got xml: %s', xml_to_compress)
+
+ -- Prepare SMTP message
+ local report_settings = dmarc_settings.reporting
+ local rcpt_string = rcpt_list(dmarc_record.rua, function(rua_elt)
+ return string.format('%s@%s', rua_elt:get_user(), rua_elt:get_host())
+ end)
+ local bcc_string
+ if report_settings.bcc_addrs then
+ bcc_string = rcpt_list(report_settings.bcc_addrs)
+ end
+ local uuid = gen_uuid()
+ local rhead = lua_util.jinja_template(report_template, {
+ from_name = report_settings.from_name,
+ from_addr = report_settings.email,
+ rcpt = rcpt_string,
+ bcc = bcc_string,
+ uuid = uuid,
+ reporting_domain = reporting_domain,
+ submitter = report_settings.domain,
+ report_id = string.format('%s.%d.%d', reporting_domain, start_time,
+ end_time),
+ report_date = rspamd_util.time_to_string(rspamd_util.get_time()),
+ message_id = rspamd_util.random_hex(16) .. '@' .. report_settings.msgid_from,
+ report_start = start_time,
+ report_end = end_time
+ }, true)
+ local rfooter = lua_util.jinja_template(report_footer, {
+ uuid = uuid,
+ }, true)
+ local message = rspamd_text.fromtable {
+ (rhead:gsub("\n", "\r\n")),
+ rspamd_util.encode_base64(rspamd_util.gzip_compress(xml_to_compress), 73),
+ rfooter:gsub("\n", "\r\n"),
+ }
+
+ lua_util.debugm(N, 'got final message: %s', message)
+
+ if not opts.no_opt then
+ lua_redis.request(redis_params, redis_attrs,
+ { 'DEL', rep_key })
+ end
+
+ local report_rcpts = lua_util.str_split(rcpt_string, ',')
+
+ if report_settings.bcc_addrs then
+ for _, b in ipairs(report_settings.bcc_addrs) do
+ table.insert(report_rcpts, b)
+ end
+ end
+
+ return {
+ message = message,
+ rcpts = report_rcpts,
+ reporting_domain = reporting_domain
+ }
+end
+
+local function process_report_date(opts, start_time, end_time, date)
+ local idx_key = redis_prefix(dmarc_settings.reporting.redis_keys.index_prefix, date)
+ local ret, results = lua_redis.request(redis_params, redis_attrs,
+ { 'EXISTS', idx_key })
+
+ if not ret or not results or results == 0 then
+ logger.messagex('No reports for %s', date)
+ return {}
+ end
+
+ -- Rename index key to avoid races
+ if not opts.no_opt then
+ lua_redis.request(redis_params, redis_attrs,
+ { 'RENAME', idx_key, idx_key .. '_processing' })
+ idx_key = idx_key .. '_processing'
+ end
+ ret, results = lua_redis.request(redis_params, redis_attrs,
+ { 'SMEMBERS', idx_key })
+
+ if not ret or not results then
+ -- Remove bad key
+ if not opts.no_opt then
+ lua_redis.request(redis_params, redis_attrs,
+ { 'DEL', idx_key })
+ end
+ logger.messagex('Cannot get reports for %s', date)
+ return {}
+ end
+
+ local reports = {}
+ for _, rep in ipairs(results) do
+ local report = prepare_report(opts, start_time, end_time, rep)
+
+ if report then
+ table.insert(reports, report)
+ end
+ end
+
+ -- Shuffle reports to make sending more fair
+ lua_util.shuffle(reports)
+ -- Remove processed key
+ if not opts.no_opt then
+ lua_redis.request(redis_params, redis_attrs,
+ { 'DEL', idx_key })
+ end
+
+ return reports
+end
+
+
+-- Returns a day before today at 00:00 as unix seconds
+local function yesterday_midnight()
+ local piecewise_time = os.date("*t")
+ piecewise_time.day = piecewise_time.day - 1 -- Lua allows negative values here
+ piecewise_time.hour = 0
+ piecewise_time.sec = 0
+ piecewise_time.min = 0
+ return os.time(piecewise_time)
+end
+
+-- Returns today time at 00:00 as unix seconds
+local function today_midnight()
+ local piecewise_time = os.date("*t")
+ piecewise_time.hour = 0
+ piecewise_time.sec = 0
+ piecewise_time.min = 0
+ return os.time(piecewise_time)
+end
+
+local function handler(args)
+ local start_time
+ -- Preserve start time as report sending might take some time
+ local start_collection = today_midnight()
+
+ local opts = parser:parse(args)
+
+ pool = rspamd_mempool.create()
+ load_config(opts)
+ rspamd_url.init(rspamd_config:get_tld_path())
+
+ if opts.verbose then
+ lua_util.enable_debug_modules('dmarc', N)
+ end
+
+ dmarc_settings = rspamd_config:get_all_opt('dmarc')
+ if not dmarc_settings or not dmarc_settings.reporting or not dmarc_settings.reporting.enabled then
+ logger.errx('dmarc reporting is not enabled, exiting')
+ os.exit(1)
+ end
+
+ dmarc_settings = lua_util.override_defaults(dmarc_common.default_settings, dmarc_settings)
+ redis_params = lua_redis.parse_redis_server('dmarc', dmarc_settings)
+
+ if not redis_params then
+ logger.errx('Redis is not configured, exiting')
+ os.exit(1)
+ end
+
+ for _, e in ipairs({ 'email', 'domain', 'org_name' }) do
+ if not dmarc_settings.reporting[e] then
+ logger.errx('Missing required setting: dmarc.reporting.%s', e)
+ return
+ end
+ end
+
+ local ret, results = lua_redis.request(redis_params, redis_attrs, {
+ 'GET', 'rspamd_dmarc_last_collection'
+ })
+
+ if not ret or not tonumber(results) then
+ start_time = yesterday_midnight()
+ else
+ start_time = tonumber(results)
+ end
+
+ lua_util.debugm(N, 'previous last report date is %s', start_time)
+
+ if not opts.date or #opts.date == 0 then
+ opts.date = {}
+ table.insert(opts.date, os.date('%Y%m%d', yesterday_midnight()))
+ end
+
+ local ndates = 0
+ local nreports = 0
+ local all_reports = {}
+ for _, date in ipairs(opts.date) do
+ lua_util.debugm(N, 'Process date %s', date)
+ local reports_for_date = process_report_date(opts, start_time, start_collection, date)
+ if #reports_for_date > 0 then
+ ndates = ndates + 1
+ nreports = nreports + #reports_for_date
+
+ for _, r in ipairs(reports_for_date) do
+ table.insert(all_reports, r)
+ end
+ end
+ end
+
+ local function finish_cb(nsuccess, nfail)
+ if not opts.no_opt then
+ lua_util.debugm(N, 'set last report date to %s', start_collection)
+ -- Hack to avoid coroutines + async functions mess: we use async redis call here
+ redis_attrs.callback = function()
+ logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed',
+ ndates, nreports, nsuccess, nfail)
+ end
+ lua_redis.request(redis_params, redis_attrs,
+ { 'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2,
+ tostring(start_collection) })
+ else
+ logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed',
+ ndates, nreports, nsuccess, nfail)
+ end
+
+ pool:destroy()
+ end
+ if not opts.no_opt then
+ send_reports_by_smtp(opts, all_reports, finish_cb)
+ else
+ logger.messagex('Skip sending mails due to -n / --no-opt option')
+ end
+end
+
+return {
+ name = 'dmarc_report',
+ aliases = { 'dmarc_reporting' },
+ handler = handler,
+ description = parser._description
+}
diff --git a/lualib/rspamadm/dns_tool.lua b/lualib/rspamadm/dns_tool.lua
new file mode 100644
index 0000000..3eb09a8
--- /dev/null
+++ b/lualib/rspamadm/dns_tool.lua
@@ -0,0 +1,232 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+
+local argparse = require "argparse"
+local rspamd_logger = require "rspamd_logger"
+local ansicolors = require "ansicolors"
+local bit = require "bit"
+
+local parser = argparse()
+ :name "rspamadm dnstool"
+ :description "DNS tools provided by Rspamd"
+ :help_description_margin(30)
+ :command_target("command")
+ :require_command(true)
+
+parser:option "-c --config"
+ :description "Path to config file"
+ :argname("<cfg>")
+ :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
+
+local spf = parser:command "spf"
+ :description "Extracts spf records"
+spf:mutex(
+ spf:option "-d --domain"
+ :description "Domain to use"
+ :argname("<domain>"),
+ spf:option "-f --from"
+ :description "SMTP from to use"
+ :argname("<from>")
+)
+
+spf:option "-i --ip"
+ :description "Source IP address to use"
+ :argname("<ip>")
+spf:flag "-a --all"
+ :description "Print all records"
+
+local function printf(fmt, ...)
+ if fmt then
+ io.write(string.format(fmt, ...))
+ end
+ io.write('\n')
+end
+
+local function highlight(str)
+ return ansicolors.white .. str .. ansicolors.reset
+end
+
+local function green(str)
+ return ansicolors.green .. str .. ansicolors.reset
+end
+
+local function red(str)
+ return ansicolors.red .. str .. ansicolors.reset
+end
+
+local function load_config(opts)
+ local _r, err = rspamd_config:load_ucl(opts['config'])
+
+ if not _r then
+ rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
+ if not _r then
+ rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+end
+
+local function spf_handler(opts)
+ local rspamd_spf = require "rspamd_spf"
+ local rspamd_task = require "rspamd_task"
+ local rspamd_ip = require "rspamd_ip"
+
+ local task = rspamd_task:create(rspamd_config, rspamadm_ev_base)
+ task:set_session(rspamadm_session)
+ task:set_resolver(rspamadm_dns_resolver)
+
+ if opts.ip then
+ opts.ip = rspamd_ip.fromstring(opts.ip)
+ task:set_from_ip(opts.ip)
+ else
+ opts.all = true
+ end
+
+ if opts.from then
+ local rspamd_parsers = require "rspamd_parsers"
+ local addr_parsed = rspamd_parsers.parse_mail_address(opts.from)
+ if addr_parsed then
+ task:set_from('smtp', addr_parsed[1])
+ else
+ io.stderr:write('Invalid from addr\n')
+ os.exit(1)
+ end
+ elseif opts.domain then
+ task:set_from('smtp', { user = 'user', domain = opts.domain })
+ else
+ io.stderr:write('Neither domain nor from specified\n')
+ os.exit(1)
+ end
+
+ local function flag_to_str(fl)
+ if bit.band(fl, rspamd_spf.flags.temp_fail) ~= 0 then
+ return "temporary failure"
+ elseif bit.band(fl, rspamd_spf.flags.perm_fail) ~= 0 then
+ return "permanent failure"
+ elseif bit.band(fl, rspamd_spf.flags.na) ~= 0 then
+ return "no spf record"
+ end
+
+ return "unknown flag: " .. tostring(fl)
+ end
+
+ local function display_spf_results(elt, colored)
+ local dec = function(e)
+ return e
+ end
+ local policy_decode = function(e)
+ if e == rspamd_spf.policy.fail then
+ return 'reject'
+ elseif e == rspamd_spf.policy.pass then
+ return 'pass'
+ elseif e == rspamd_spf.policy.soft_fail then
+ return 'soft fail'
+ elseif e == rspamd_spf.policy.neutral then
+ return 'neutral'
+ end
+
+ return 'unknown'
+ end
+
+ if colored then
+ dec = function(e)
+ return highlight(e)
+ end
+
+ if elt.result == rspamd_spf.policy.pass then
+ dec = function(e)
+ return green(e)
+ end
+ elseif elt.result == rspamd_spf.policy.fail then
+ dec = function(e)
+ return red(e)
+ end
+ end
+
+ end
+ printf('%s: %s', highlight('Policy'), dec(policy_decode(elt.result)))
+ printf('%s: %s', highlight('Network'), dec(elt.addr))
+
+ if elt.str then
+ printf('%s: %s', highlight('Original'), elt.str)
+ end
+ end
+
+ local function cb(record, flags, err)
+ if record then
+ local result, flag_or_policy, error_or_addr
+ if opts.ip then
+ result, flag_or_policy, error_or_addr = record:check_ip(opts.ip)
+ elseif opts.all then
+ result = true
+ end
+ if opts.ip and not opts.all then
+ if result then
+ display_spf_results(error_or_addr, true)
+ else
+ printf('Not matched: %s', error_or_addr)
+ end
+
+ os.exit(0)
+ end
+
+ if result then
+ printf('SPF record for %s; digest: %s',
+ highlight(opts.domain or opts.from), highlight(record:get_digest()))
+ for _, elt in ipairs(record:get_elts()) do
+ if result and error_or_addr and elt.str and elt.str == error_or_addr.str then
+ printf("%s", highlight('*** Matched ***'))
+ display_spf_results(elt, true)
+ printf('------')
+ else
+ display_spf_results(elt, false)
+ printf('------')
+ end
+ end
+ else
+ printf('Error getting SPF record: %s (%s flag)', err,
+ flag_to_str(flag_or_policy or flags))
+ end
+ else
+ printf('Cannot get SPF record: %s', err)
+ end
+ end
+ rspamd_spf.resolve(task, cb)
+end
+
+local function handler(args)
+ local opts = parser:parse(args)
+ load_config(opts)
+
+ local command = opts.command
+
+ if command == 'spf' then
+ spf_handler(opts)
+ else
+ parser:error('command %s is not implemented', command)
+ end
+end
+
+return {
+ name = 'dnstool',
+ aliases = { 'dns', 'dns_tool' },
+ handler = handler,
+ description = parser._description
+} \ No newline at end of file
diff --git a/lualib/rspamadm/fuzzy_convert.lua b/lualib/rspamadm/fuzzy_convert.lua
new file mode 100644
index 0000000..fab3995
--- /dev/null
+++ b/lualib/rspamadm/fuzzy_convert.lua
@@ -0,0 +1,208 @@
+local sqlite3 = require "rspamd_sqlite3"
+local redis = require "rspamd_redis"
+local util = require "rspamd_util"
+
+local function connect_redis(server, username, password, db)
+ local ret
+ local conn, err = redis.connect_sync({
+ host = server,
+ })
+
+ if not conn then
+ return nil, 'Cannot connect: ' .. err
+ end
+
+ if username then
+ if password then
+ ret = conn:add_cmd('AUTH', { username, password })
+ if not ret then
+ return nil, 'Cannot queue command'
+ end
+ else
+ return nil, 'Redis requires a password when username is supplied'
+ end
+ elseif password then
+ ret = conn:add_cmd('AUTH', { password })
+ if not ret then
+ return nil, 'Cannot queue command'
+ end
+ end
+ if db then
+ ret = conn:add_cmd('SELECT', { db })
+ if not ret then
+ return nil, 'Cannot queue command'
+ end
+ end
+
+ return conn, nil
+end
+
+local function send_digests(digests, redis_host, redis_username, redis_password, redis_db)
+ local conn, err = connect_redis(redis_host, redis_username, redis_password, redis_db)
+ if err then
+ print(err)
+ return false
+ end
+ local ret
+ for _, v in ipairs(digests) do
+ ret = conn:add_cmd('HMSET', {
+ 'fuzzy' .. v[1],
+ 'F', v[2],
+ 'V', v[3],
+ })
+ if not ret then
+ print('Cannot batch command')
+ return false
+ end
+ ret = conn:add_cmd('EXPIRE', {
+ 'fuzzy' .. v[1],
+ tostring(v[4]),
+ })
+ if not ret then
+ print('Cannot batch command')
+ return false
+ end
+ end
+ ret, err = conn:exec()
+ if not ret then
+ print('Cannot execute batched commands: ' .. err)
+ return false
+ end
+ return true
+end
+
+local function send_shingles(shingles, redis_host, redis_username, redis_password, redis_db)
+ local conn, err = connect_redis(redis_host, redis_username, redis_password, redis_db)
+ if err then
+ print("Redis error: " .. err)
+ return false
+ end
+ local ret
+ for _, v in ipairs(shingles) do
+ ret = conn:add_cmd('SET', {
+ 'fuzzy_' .. v[2] .. '_' .. v[1],
+ v[4],
+ })
+ if not ret then
+ print('Cannot batch SET command: ' .. err)
+ return false
+ end
+ ret = conn:add_cmd('EXPIRE', {
+ 'fuzzy_' .. v[2] .. '_' .. v[1],
+ tostring(v[3]),
+ })
+ if not ret then
+ print('Cannot batch command')
+ return false
+ end
+ end
+ ret, err = conn:exec()
+ if not ret then
+ print('Cannot execute batched commands: ' .. err)
+ return false
+ end
+ return true
+end
+
+local function update_counters(total, redis_host, redis_username, redis_password, redis_db)
+ local conn, err = connect_redis(redis_host, redis_username, redis_password, redis_db)
+ if err then
+ print(err)
+ return false
+ end
+ local ret
+ ret = conn:add_cmd('SET', {
+ 'fuzzylocal',
+ total,
+ })
+ if not ret then
+ print('Cannot batch command')
+ return false
+ end
+ ret = conn:add_cmd('SET', {
+ 'fuzzy_count',
+ total,
+ })
+ if not ret then
+ print('Cannot batch command')
+ return false
+ end
+ ret, err = conn:exec()
+ if not ret then
+ print('Cannot execute batched commands: ' .. err)
+ return false
+ end
+ return true
+end
+
+return function(_, res)
+ local db = sqlite3.open(res['source_db'])
+ local shingles = {}
+ local digests = {}
+ local num_batch_digests = 0
+ local num_batch_shingles = 0
+ local total_digests = 0
+ local total_shingles = 0
+ local lim_batch = 1000 -- Update each 1000 entries
+ local redis_username = res['redis_username']
+ local redis_password = res['redis_password']
+ local redis_db = nil
+
+ if res['redis_db'] then
+ redis_db = tostring(res['redis_db'])
+ end
+
+ if not db then
+ print('Cannot open source db: ' .. res['source_db'])
+ return
+ end
+
+ local now = util.get_time()
+ for row in db:rows('SELECT id, flag, digest, value, time FROM digests') do
+
+ local expire_in = math.floor(now - row.time + res['expiry'])
+ if expire_in >= 1 then
+ table.insert(digests, { row.digest, row.flag, row.value, expire_in })
+ num_batch_digests = num_batch_digests + 1
+ total_digests = total_digests + 1
+ for srow in db:rows('SELECT value, number FROM shingles WHERE digest_id = ' .. row.id) do
+ table.insert(shingles, { srow.value, srow.number, expire_in, row.digest })
+ total_shingles = total_shingles + 1
+ num_batch_shingles = num_batch_shingles + 1
+ end
+ end
+ if num_batch_digests >= lim_batch then
+ if not send_digests(digests, res['redis_host'], redis_username, redis_password, redis_db) then
+ return
+ end
+ num_batch_digests = 0
+ digests = {}
+ end
+ if num_batch_shingles >= lim_batch then
+ if not send_shingles(shingles, res['redis_host'], redis_username, redis_password, redis_db) then
+ return
+ end
+ num_batch_shingles = 0
+ shingles = {}
+ end
+ end
+ if digests[1] then
+ if not send_digests(digests, res['redis_host'], redis_username, redis_password, redis_db) then
+ return
+ end
+ end
+ if shingles[1] then
+ if not send_shingles(shingles, res['redis_host'], redis_username, redis_password, redis_db) then
+ return
+ end
+ end
+
+ local message = string.format(
+ 'Migrated %d digests and %d shingles',
+ total_digests, total_shingles
+ )
+ if not update_counters(total_digests, res['redis_host'], redis_username, redis_password, redis_db) then
+ message = message .. ' but failed to update counters'
+ end
+ print(message)
+end
diff --git a/lualib/rspamadm/fuzzy_ping.lua b/lualib/rspamadm/fuzzy_ping.lua
new file mode 100644
index 0000000..e0345da
--- /dev/null
+++ b/lualib/rspamadm/fuzzy_ping.lua
@@ -0,0 +1,259 @@
+--[[
+Copyright (c) 2023, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+local ansicolors = require "ansicolors"
+local rspamd_logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+
+local E = {}
+
+local parser = argparse()
+ :name 'rspamadm fuzzy_ping'
+ :description 'Pings fuzzy storage'
+ :help_description_margin(30)
+parser:option "-c --config"
+ :description "Path to config file"
+ :argname("<cfg>")
+ :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
+parser:option "-r --rule"
+ :description "Storage to ping (must be configured in Rspamd configuration)"
+ :argname("<name>")
+ :default("rspamd.com")
+parser:flag "-f --flood"
+ :description "Flood mode (no waiting for replies)"
+parser:flag "-S --silent"
+ :description "Silent mode (statistics only)"
+parser:option "-t --timeout"
+ :description "Timeout for requests"
+ :argname("<timeout>")
+ :convert(tonumber)
+ :default(5)
+parser:option "-s --server"
+ :description "Override server to ping"
+ :argname("<name>")
+parser:option "-n --number"
+ :description "Timeout for requests"
+ :argname("<number>")
+ :convert(tonumber)
+ :default(5)
+parser:flag "-l --list"
+ :description "List configured storages"
+
+local function load_config(opts)
+ local _r, err = rspamd_config:load_ucl(opts['config'])
+
+ if not _r then
+ rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ -- Init the real structure excluding logging and workers
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
+ if not _r then
+ rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ _r, err = rspamd_config:init_modules()
+ if not _r then
+ rspamd_logger.errx('cannot init modules from %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+end
+
+local function highlight(fmt, ...)
+ return ansicolors.white .. string.format(fmt, ...) .. ansicolors.reset
+end
+
+local function highlight_err(fmt, ...)
+ return ansicolors.red .. string.format(fmt, ...) .. ansicolors.reset
+end
+
+local function print_storages(rules)
+ for n, rule in pairs(rules) do
+ print(highlight('Rule: %s', n))
+ print(string.format("\tRead only: %s", rule.read_only))
+ print(string.format("\tServers: %s", table.concat(lua_util.values(rule.servers), ',')))
+ print("\tFlags:")
+
+ for fl, id in pairs(rule.flags or E) do
+ print(string.format("\t\t%s: %s", fl, id))
+ end
+ end
+end
+
+local function std_mean(tbl)
+ local function mean()
+ local sum = 0
+ local count = 0
+
+ for _, v in ipairs(tbl) do
+ sum = sum + v
+ count = count + 1
+ end
+
+ return (sum / count)
+ end
+
+ local m
+ local vm
+ local sum = 0
+ local count = 0
+ local result
+
+ m = mean(tbl)
+
+ for _, v in ipairs(tbl) do
+ vm = v - m
+ sum = sum + (vm * vm)
+ count = count + 1
+ end
+
+ result = math.sqrt(sum / (count - 1))
+
+ return result, m
+end
+
+local function maxmin(tbl)
+ local max = -math.huge
+ local min = math.huge
+
+ for _, v in ipairs(tbl) do
+ max = math.max(max, v)
+ min = math.min(min, v)
+ end
+
+ return max, min
+end
+
+local function print_results(results)
+ local servers = {}
+ local err_servers = {}
+ for _, res in ipairs(results) do
+ if res.success then
+ if servers[res.server] then
+ table.insert(servers[res.server], res.latency)
+ else
+ servers[res.server] = { res.latency }
+ end
+ else
+ if err_servers[res.server] then
+ err_servers[res.server] = err_servers[res.server] + 1
+ else
+ err_servers[res.server] = 1
+ end
+ -- For the case if no successful replies are detected
+ if not servers[res.server] then
+ servers[res.server] = {}
+ end
+ end
+ end
+
+ for s, l in pairs(servers) do
+ local total = #l + (err_servers[s] or 0)
+ print(highlight('Summary for %s: %d packets transmitted, %d packets received, %.1f%% packet loss',
+ s, total, #l, (total - #l) * 100.0 / total))
+ local mean, std = std_mean(l)
+ local max, min = maxmin(l)
+ print(string.format('round-trip min/avg/max/std-dev = %.2f/%.2f/%.2f/%.2f ms',
+ min, mean,
+ max, std))
+ end
+end
+
+local function handler(args)
+ local opts = parser:parse(args)
+
+ load_config(opts)
+
+ if opts.list then
+ print_storages(rspamd_plugins.fuzzy_check.list_storages(rspamd_config))
+ os.exit(0)
+ end
+
+ -- Perform ping using a fake task from async stuff provided by rspamadm
+ local rspamd_task = require "rspamd_task"
+
+ -- TODO: this task is not cleared at the end, do something about it some day
+ local task = rspamd_task.create(rspamd_config, rspamadm_ev_base)
+ task:set_session(rspamadm_session)
+ task:set_resolver(rspamadm_dns_resolver)
+
+ local replied = 0
+ local results = {}
+ local ping_fuzzy
+
+ local function gen_ping_fuzzy_cb(num)
+ return function(success, server, latency_or_err)
+ if not success then
+ if not opts.silent then
+ print(highlight_err('error from %s: %s', server, latency_or_err))
+ end
+ results[num] = {
+ success = false,
+ error = latency_or_err,
+ server = tostring(server),
+ }
+ else
+ if not opts.silent then
+ local adjusted_latency = math.floor(latency_or_err * 1000) * 1.0 / 1000;
+ print(highlight('reply from %s: %s ms', server, adjusted_latency))
+
+ end
+ results[num] = {
+ success = true,
+ latency = latency_or_err,
+ server = tostring(server),
+ }
+ end
+
+ if replied == opts.number - 1 then
+ print_results(results)
+ else
+ replied = replied + 1
+ if not opts.flood then
+ ping_fuzzy(replied + 1)
+ end
+ end
+ end
+ end
+
+ ping_fuzzy = function(num)
+ local ret, err = rspamd_plugins.fuzzy_check.ping_storage(task, gen_ping_fuzzy_cb(num),
+ opts.rule, opts.timeout, opts.server)
+
+ if not ret then
+ print(highlight_err('error from %s: %s', opts.server, err))
+ opts.number = opts.number - 1 -- To avoid issues with waiting for other replies
+ end
+ end
+
+ if opts.flood then
+ for i = 1, opts.number do
+ ping_fuzzy(i)
+ end
+ else
+ ping_fuzzy(1)
+ end
+end
+
+return {
+ name = 'fuzzy_ping',
+ aliases = { 'fuzzyping' },
+ handler = handler,
+ description = parser._description
+} \ No newline at end of file
diff --git a/lualib/rspamadm/fuzzy_stat.lua b/lualib/rspamadm/fuzzy_stat.lua
new file mode 100644
index 0000000..ef8a5de
--- /dev/null
+++ b/lualib/rspamadm/fuzzy_stat.lua
@@ -0,0 +1,366 @@
+local rspamd_util = require "rspamd_util"
+local lua_util = require "lua_util"
+local opts = {}
+
+local argparse = require "argparse"
+local parser = argparse()
+ :name "rspamadm control fuzzystat"
+ :description "Shows help for the specified configuration options"
+ :help_description_margin(32)
+parser:flag "--no-ips"
+ :description "No IPs stats"
+parser:flag "--no-keys"
+ :description "No keys stats"
+parser:flag "--short"
+ :description "Short output mode"
+parser:flag "-n --number"
+ :description "Disable numbers humanization"
+parser:option "--sort"
+ :description "Sort order"
+ :convert {
+ checked = "checked",
+ matched = "matched",
+ errors = "errors",
+ name = "name"
+}
+
+local function add_data(target, src)
+ for k, v in pairs(src) do
+ if type(v) == 'number' then
+ if target[k] then
+ target[k] = target[k] + v
+ else
+ target[k] = v
+ end
+ elseif k == 'ips' then
+ if not target['ips'] then
+ target['ips'] = {}
+ end
+ -- Iterate over IPs
+ for ip, st in pairs(v) do
+ if not target['ips'][ip] then
+ target['ips'][ip] = {}
+ end
+ add_data(target['ips'][ip], st)
+ end
+ elseif k == 'flags' then
+ if not target['flags'] then
+ target['flags'] = {}
+ end
+ -- Iterate over Flags
+ for flag, st in pairs(v) do
+ if not target['flags'][flag] then
+ target['flags'][flag] = {}
+ end
+ add_data(target['flags'][flag], st)
+ end
+ elseif k == 'keypair' then
+ if type(v.extensions) == 'table' then
+ if type(v.extensions.name) == 'string' then
+ target.name = v.extensions.name
+ end
+ end
+ end
+ end
+end
+
+local function print_num(num)
+ if num then
+ if opts['n'] or opts['number'] then
+ return tostring(num)
+ else
+ return rspamd_util.humanize_number(num)
+ end
+ else
+ return 'na'
+ end
+end
+
+local function print_stat(st, tabs)
+ if st['checked'] then
+ if st.checked_per_hour then
+ print(string.format('%sChecked: %s (%s per hour in average)', tabs,
+ print_num(st['checked']), print_num(st['checked_per_hour'])))
+ else
+ print(string.format('%sChecked: %s', tabs, print_num(st['checked'])))
+ end
+ end
+ if st['matched'] then
+ if st.checked and st.checked > 0 and st.checked <= st.matched then
+ local percentage = st.matched / st.checked * 100.0
+ if st.matched_per_hour then
+ print(string.format('%sMatched: %s - %s percent (%s per hour in average)', tabs,
+ print_num(st['matched']), percentage, print_num(st['matched_per_hour'])))
+ else
+ print(string.format('%sMatched: %s - %s percent', tabs, print_num(st['matched']), percentage))
+ end
+ else
+ if st.matched_per_hour then
+ print(string.format('%sMatched: %s (%s per hour in average)', tabs,
+ print_num(st['matched']), print_num(st['matched_per_hour'])))
+ else
+ print(string.format('%sMatched: %s', tabs, print_num(st['matched'])))
+ end
+ end
+ end
+ if st['errors'] then
+ print(string.format('%sErrors: %s', tabs, print_num(st['errors'])))
+ end
+ if st['added'] then
+ print(string.format('%sAdded: %s', tabs, print_num(st['added'])))
+ end
+ if st['deleted'] then
+ print(string.format('%sDeleted: %s', tabs, print_num(st['deleted'])))
+ end
+end
+
+-- Sort by checked
+local function sort_hash_table(tbl, sort_opts, key_key)
+ local res = {}
+ for k, v in pairs(tbl) do
+ table.insert(res, { [key_key] = k, data = v })
+ end
+
+ local function sort_order(elt)
+ local key = 'checked'
+ local sort_res = 0
+
+ if sort_opts['sort'] then
+ if sort_opts['sort'] == 'matched' then
+ key = 'matched'
+ elseif sort_opts['sort'] == 'errors' then
+ key = 'errors'
+ elseif sort_opts['sort'] == 'name' then
+ return elt[key_key]
+ end
+ end
+
+ if elt.data[key] then
+ sort_res = elt.data[key]
+ end
+
+ return sort_res
+ end
+
+ table.sort(res, function(a, b)
+ return sort_order(a) > sort_order(b)
+ end)
+
+ return res
+end
+
+local function add_result(dst, src, k)
+ if type(src) == 'table' then
+ if type(dst) == 'number' then
+ -- Convert dst to table
+ dst = { dst }
+ elseif type(dst) == 'nil' then
+ dst = {}
+ end
+
+ for i, v in ipairs(src) do
+ if dst[i] and k ~= 'fuzzy_stored' then
+ dst[i] = dst[i] + v
+ else
+ dst[i] = v
+ end
+ end
+ else
+ if type(dst) == 'table' then
+ if k ~= 'fuzzy_stored' then
+ dst[1] = dst[1] + src
+ else
+ dst[1] = src
+ end
+ else
+ if dst and k ~= 'fuzzy_stored' then
+ dst = dst + src
+ else
+ dst = src
+ end
+ end
+ end
+
+ return dst
+end
+
+local function print_result(r)
+ local function num_to_epoch(num)
+ if num == 1 then
+ return 'v0.6'
+ elseif num == 2 then
+ return 'v0.8'
+ elseif num == 3 then
+ return 'v0.9'
+ elseif num == 4 then
+ return 'v1.0+'
+ elseif num == 5 then
+ return 'v1.7+'
+ end
+ return '???'
+ end
+ if type(r) == 'table' then
+ local res = {}
+ for i, num in ipairs(r) do
+ res[i] = string.format('(%s: %s)', num_to_epoch(i), print_num(num))
+ end
+
+ return table.concat(res, ', ')
+ end
+
+ return print_num(r)
+end
+
+return function(args, res)
+ local res_ips = {}
+ local res_databases = {}
+ local wrk = res['workers']
+ opts = parser:parse(args)
+
+ if wrk then
+ for _, pr in pairs(wrk) do
+ -- processes cycle
+ if pr['data'] then
+ local id = pr['id']
+
+ if id then
+ local res_db = res_databases[id]
+ if not res_db then
+ res_db = {
+ keys = {}
+ }
+ res_databases[id] = res_db
+ end
+
+ -- General stats
+ for k, v in pairs(pr['data']) do
+ if k ~= 'keys' and k ~= 'errors_ips' then
+ res_db[k] = add_result(res_db[k], v, k)
+ elseif k == 'errors_ips' then
+ -- Errors ips
+ if not res_db['errors_ips'] then
+ res_db['errors_ips'] = {}
+ end
+ for ip, nerrors in pairs(v) do
+ if not res_db['errors_ips'][ip] then
+ res_db['errors_ips'][ip] = nerrors
+ else
+ res_db['errors_ips'][ip] = nerrors + res_db['errors_ips'][ip]
+ end
+ end
+ end
+ end
+
+ if pr['data']['keys'] then
+ local res_keys = res_db['keys']
+ if not res_keys then
+ res_keys = {}
+ res_db['keys'] = res_keys
+ end
+ -- Go through keys in input
+ for k, elts in pairs(pr['data']['keys']) do
+ -- keys cycle
+ if not res_keys[k] then
+ res_keys[k] = {}
+ end
+
+ add_data(res_keys[k], elts)
+
+ if elts['ips'] then
+ for ip, v in pairs(elts['ips']) do
+ if not res_ips[ip] then
+ res_ips[ip] = {}
+ end
+ add_data(res_ips[ip], v)
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+
+ -- General stats
+ for db, st in pairs(res_databases) do
+ print(string.format('Statistics for storage %s', db))
+
+ for k, v in pairs(st) do
+ if k ~= 'keys' and k ~= 'errors_ips' then
+ print(string.format('%s: %s', k, print_result(v)))
+ end
+ end
+ print('')
+
+ local res_keys = st['keys']
+ if res_keys and not opts['no_keys'] and not opts['short'] then
+ print('Keys statistics:')
+ -- Convert into an array to allow sorting
+ local sorted_keys = sort_hash_table(res_keys, opts, 'key')
+
+ for _, key in ipairs(sorted_keys) do
+ local key_stat = key.data
+ if key_stat.name then
+ print(string.format('Key id: %s, name: %s', key.key, key_stat.name))
+ else
+ print(string.format('Key id: %s', key.key))
+ end
+
+ print_stat(key_stat, '\t')
+
+ if key_stat['ips'] and not opts['no_ips'] then
+ print('')
+ print('\tIPs stat:')
+ local sorted_ips = sort_hash_table(key_stat['ips'], opts, 'ip')
+
+ for _, v in ipairs(sorted_ips) do
+ print(string.format('\t%s', v['ip']))
+ print_stat(v['data'], '\t\t')
+ print('')
+ end
+ end
+
+ if key_stat.flags then
+ print('')
+ print('\tFlags stat:')
+ for flag, v in pairs(key_stat.flags) do
+ print(string.format('\t[%s]:', flag))
+ -- Remove irrelevant fields
+ v.checked = nil
+ print_stat(v, '\t\t')
+ print('')
+ end
+ end
+
+ print('')
+ end
+ end
+ if st['errors_ips'] and not opts['no_ips'] and not opts['short'] then
+ print('')
+ print('Errors IPs statistics:')
+ local ip_stat = st['errors_ips']
+ local ips = lua_util.keys(ip_stat)
+ -- Reverse sort by number of errors
+ table.sort(ips, function(a, b)
+ return ip_stat[a] > ip_stat[b]
+ end)
+ for _, ip in ipairs(ips) do
+ print(string.format('%s: %s', ip, print_result(ip_stat[ip])))
+ end
+ print('')
+ end
+ end
+
+ if not opts['no_ips'] and not opts['short'] then
+ print('')
+ print('IPs statistics:')
+
+ local sorted_ips = sort_hash_table(res_ips, opts, 'ip')
+ for _, v in ipairs(sorted_ips) do
+ print(string.format('%s', v['ip']))
+ print_stat(v['data'], '\t')
+ print('')
+ end
+ end
+end
+
diff --git a/lualib/rspamadm/grep.lua b/lualib/rspamadm/grep.lua
new file mode 100644
index 0000000..6ed0569
--- /dev/null
+++ b/lualib/rspamadm/grep.lua
@@ -0,0 +1,174 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+
+
+-- Define command line options
+local parser = argparse()
+ :name "rspamadm grep"
+ :description "Search for patterns in rspamd logs"
+ :help_description_margin(30)
+parser:mutex(
+ parser:option "-s --string"
+ :description('Plain string to search (case-insensitive)')
+ :argname "<str>",
+ parser:option "-p --pattern"
+ :description('Pattern to search for (regex)')
+ :argname "<re>"
+)
+parser:flag "-l --lua"
+ :description('Use Lua patterns in string search')
+
+parser:argument "input":args "*"
+ :description('Process specified inputs')
+ :default("stdin")
+parser:flag "-S --sensitive"
+ :description('Enable case-sensitivity in string search')
+parser:flag "-o --orphans"
+ :description('Print orphaned logs')
+parser:flag "-P --partial"
+ :description('Print partial logs')
+
+local function handler(args)
+
+ local rspamd_regexp = require 'rspamd_regexp'
+ local res = parser:parse(args)
+
+ if not res['string'] and not res['pattern'] then
+ parser:error('string or pattern options must be specified')
+ end
+
+ if res['string'] and res['pattern'] then
+ parser:error('string and pattern options are mutually exclusive')
+ end
+
+ local buffer = {}
+ local matches = {}
+
+ local pattern = res['pattern']
+ local re
+ if pattern then
+ re = rspamd_regexp.create(pattern)
+ if not re then
+ io.stderr:write("Couldn't compile regex: " .. pattern .. '\n')
+ os.exit(1)
+ end
+ end
+
+ local plainm = true
+ if res['lua'] then
+ plainm = false
+ end
+ local orphans = res['orphans']
+ local search_str = res['string']
+ local sensitive = res['sensitive']
+ local partial = res['partial']
+ if search_str and not sensitive then
+ search_str = string.lower(search_str)
+ end
+ local inputs = res['input'] or { 'stdin' }
+
+ for _, n in ipairs(inputs) do
+ local h, err
+ if string.match(n, '%.xz$') then
+ h, err = io.popen('xzcat ' .. n, 'r')
+ elseif string.match(n, '%.bz2$') then
+ h, err = io.popen('bzcat ' .. n, 'r')
+ elseif string.match(n, '%.gz$') then
+ h, err = io.popen('zcat ' .. n, 'r')
+ elseif string.match(n, '%.zst$') then
+ h, err = io.popen('zstdcat ' .. n, 'r')
+ elseif n == 'stdin' then
+ h = io.input()
+ else
+ h, err = io.open(n, 'r')
+ end
+ if not h then
+ if err then
+ io.stderr:write("Couldn't open file (" .. n .. '): ' .. err .. '\n')
+ else
+ io.stderr:write("Couldn't open file (" .. n .. '): no error\n')
+ end
+ else
+ for line in h:lines() do
+ local hash = string.match(line, '<(%x+)>')
+ local already_matching = false
+ if hash then
+ if matches[hash] then
+ table.insert(matches[hash], line)
+ already_matching = true
+ else
+ if buffer[hash] then
+ table.insert(buffer[hash], line)
+ else
+ buffer[hash] = { line }
+ end
+ end
+ end
+ local ismatch = false
+ if re then
+ ismatch = re:match(line)
+ elseif sensitive and search_str then
+ ismatch = string.find(line, search_str, 1, plainm)
+ elseif search_str then
+ local lwr = string.lower(line)
+ ismatch = string.find(lwr, search_str, 1, plainm)
+ end
+ if ismatch then
+ if not hash then
+ if orphans then
+ print('*** orphaned ***')
+ print(line)
+ print()
+ end
+ elseif not already_matching then
+ matches[hash] = buffer[hash]
+ end
+ end
+ local is_end = string.match(line, '<%x+>; task; rspamd_protocol_http_reply:')
+ if is_end then
+ buffer[hash] = nil
+ if matches[hash] then
+ for _, v in ipairs(matches[hash]) do
+ print(v)
+ end
+ print()
+ matches[hash] = nil
+ end
+ end
+ end
+ if partial then
+ for k, v in pairs(matches) do
+ print('*** partial ***')
+ for _, vv in ipairs(v) do
+ print(vv)
+ end
+ print()
+ matches[k] = nil
+ end
+ else
+ matches = {}
+ end
+ end
+ end
+end
+
+return {
+ handler = handler,
+ description = parser._description,
+ name = 'grep'
+} \ No newline at end of file
diff --git a/lualib/rspamadm/keypair.lua b/lualib/rspamadm/keypair.lua
new file mode 100644
index 0000000..f0716a2
--- /dev/null
+++ b/lualib/rspamadm/keypair.lua
@@ -0,0 +1,508 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+local rspamd_keypair = require "rspamd_cryptobox_keypair"
+local rspamd_pubkey = require "rspamd_cryptobox_pubkey"
+local rspamd_signature = require "rspamd_cryptobox_signature"
+local rspamd_crypto = require "rspamd_cryptobox"
+local rspamd_util = require "rspamd_util"
+local ucl = require "ucl"
+local logger = require "rspamd_logger"
+
+-- Define command line options
+local parser = argparse()
+ :name "rspamadm keypair"
+ :description "Manages keypairs for Rspamd"
+ :help_description_margin(30)
+ :command_target("command")
+ :require_command(false)
+
+-- Generate subcommand
+local generate = parser:command "generate gen g"
+ :description "Creates a new keypair"
+generate:flag "-s --sign"
+ :description "Generates a sign keypair instead of the encryption one"
+generate:flag "-n --nist"
+ :description "Uses nist encryption algorithm"
+generate:option "-o --output"
+ :description "Write keypair to file"
+ :argname "<file>"
+generate:mutex(
+ generate:flag "-j --json"
+ :description "Output JSON instead of UCL",
+ generate:flag "-u --ucl"
+ :description "Output UCL"
+ :default(true)
+)
+generate:option "--name"
+ :description "Adds name extension"
+ :argname "<name>"
+
+-- Sign subcommand
+
+local sign = parser:command "sign sig s"
+ :description "Signs a file using keypair"
+sign:option "-k --keypair"
+ :description "Keypair to use"
+ :argname "<file>"
+sign:option "-s --suffix"
+ :description "Suffix for signature"
+ :argname "<suffix>"
+ :default("sig")
+sign:argument "file"
+ :description "File to sign"
+ :argname "<file>"
+ :args "*"
+
+-- Verify subcommand
+
+local verify = parser:command "verify ver v"
+ :description "Verifies a file using keypair or a public key"
+verify:mutex(
+ verify:option "-p --pubkey"
+ :description "Load pubkey from the specified file"
+ :argname "<file>",
+ verify:option "-P --pubstring"
+ :description "Load pubkey from the base32 encoded string"
+ :argname "<base32>",
+ verify:option "-k --keypair"
+ :description "Get pubkey from the keypair file"
+ :argname "<file>"
+)
+verify:argument "file"
+ :description "File to verify"
+ :argname "<file>"
+ :args "*"
+verify:flag "-n --nist"
+ :description "Uses nistp curves (P256)"
+verify:option "-s --suffix"
+ :description "Suffix for signature"
+ :argname "<suffix>"
+ :default("sig")
+
+-- Encrypt subcommand
+
+local encrypt = parser:command "encrypt crypt enc e"
+ :description "Encrypts a file using keypair (or a pubkey)"
+encrypt:mutex(
+ encrypt:option "-p --pubkey"
+ :description "Load pubkey from the specified file"
+ :argname "<file>",
+ encrypt:option "-P --pubstring"
+ :description "Load pubkey from the base32 encoded string"
+ :argname "<base32>",
+ encrypt:option "-k --keypair"
+ :description "Get pubkey from the keypair file"
+ :argname "<file>"
+)
+encrypt:option "-s --suffix"
+ :description "Suffix for encrypted file"
+ :argname "<suffix>"
+ :default("enc")
+encrypt:argument "file"
+ :description "File to encrypt"
+ :argname "<file>"
+ :args "*"
+encrypt:flag "-r --rm"
+ :description "Remove unencrypted file"
+encrypt:flag "-f --force"
+ :description "Remove destination file if it exists"
+
+-- Decrypt subcommand
+
+local decrypt = parser:command "decrypt dec d"
+ :description "Decrypts a file using keypair"
+decrypt:option "-k --keypair"
+ :description "Get pubkey from the keypair file"
+ :argname "<file>"
+decrypt:flag "-S --keep-suffix"
+ :description "Preserve suffix for decrypted file (overwrite encrypted)"
+decrypt:argument "file"
+ :description "File to encrypt"
+ :argname "<file>"
+ :args "*"
+decrypt:flag "-f --force"
+ :description "Remove destination file if it exists (implied with -S)"
+decrypt:flag "-r --rm"
+ :description "Remove encrypted file"
+
+-- Default command is generate, so duplicate options to be compatible
+
+parser:flag "-s --sign"
+ :description "Generates a sign keypair instead of the encryption one"
+parser:flag "-n --nist"
+ :description "Uses nistp curves (P256)"
+parser:mutex(
+ parser:flag "-j --json"
+ :description "Output JSON instead of UCL",
+ parser:flag "-u --ucl"
+ :description "Output UCL"
+ :default(true)
+)
+parser:option "-o --output"
+ :description "Write keypair to file"
+ :argname "<file>"
+
+local function fatal(...)
+ logger.errx(...)
+ os.exit(1)
+end
+
+local function ask_yes_no(greet, default)
+ local def_str
+ if default then
+ greet = greet .. "[Y/n]: "
+ def_str = "yes"
+ else
+ greet = greet .. "[y/N]: "
+ def_str = "no"
+ end
+
+ local reply = rspamd_util.readline(greet)
+
+ if not reply then
+ os.exit(0)
+ end
+ if #reply == 0 then
+ reply = def_str
+ end
+ reply = reply:lower()
+ if reply == 'y' or reply == 'yes' then
+ return true
+ end
+
+ return false
+end
+
+local function generate_handler(opts)
+ local mode = 'encryption'
+ if opts.sign then
+ mode = 'sign'
+ end
+ local alg = 'curve25519'
+ if opts.nist then
+ alg = 'nist'
+ end
+ -- TODO: probably, do it in a more safe way
+ local kp = rspamd_keypair.create(mode, alg):totable()
+
+ if opts.name then
+ kp.keypair.extensions = {
+ name = opts.name
+ }
+ end
+
+ local format = 'ucl'
+
+ if opts.json then
+ format = 'json'
+ end
+
+ if opts.output then
+ local out = io.open(opts.output, 'w')
+ if not out then
+ fatal('cannot open output to write: ' .. opts.output)
+ end
+ out:write(ucl.to_format(kp, format))
+ out:close()
+ else
+ io.write(ucl.to_format(kp, format))
+ end
+end
+
+local function sign_handler(opts)
+ if opts.file then
+ if type(opts.file) == 'string' then
+ opts.file = { opts.file }
+ end
+ else
+ parser:error('no files to sign')
+ end
+ if not opts.keypair then
+ parser:error("no keypair specified")
+ end
+
+ local ucl_parser = ucl.parser()
+ local res, err = ucl_parser:parse_file(opts.keypair)
+
+ if not res then
+ fatal(string.format('cannot load %s: %s', opts.keypair, err))
+ end
+
+ local kp = rspamd_keypair.load(ucl_parser:get_object())
+
+ if not kp then
+ fatal("cannot load keypair: " .. opts.keypair)
+ end
+
+ for _, fname in ipairs(opts.file) do
+ local sig = rspamd_crypto.sign_file(kp, fname)
+
+ if not sig then
+ fatal(string.format("cannot sign %s\n", fname))
+ end
+
+ local out = string.format('%s.%s', fname, opts.suffix or 'sig')
+ local of = io.open(out, 'w')
+ if not of then
+ fatal('cannot open output to write: ' .. out)
+ end
+ of:write(sig:bin())
+ of:close()
+ io.write(string.format('signed %s -> %s (%s)\n', fname, out, sig:hex()))
+ end
+end
+
+local function verify_handler(opts)
+ if opts.file then
+ if type(opts.file) == 'string' then
+ opts.file = { opts.file }
+ end
+ else
+ parser:error('no files to verify')
+ end
+
+ local pk
+ local alg = 'curve25519'
+
+ if opts.keypair then
+ local ucl_parser = ucl.parser()
+ local res, err = ucl_parser:parse_file(opts.keypair)
+
+ if not res then
+ fatal(string.format('cannot load %s: %s', opts.keypair, err))
+ end
+
+ local kp = rspamd_keypair.load(ucl_parser:get_object())
+
+ if not kp then
+ fatal("cannot load keypair: " .. opts.keypair)
+ end
+
+ pk = kp:pk()
+ alg = kp:alg()
+ elseif opts.pubkey then
+ if opts.nist then
+ alg = 'nist'
+ end
+ pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg)
+ elseif opts.pubstr then
+ if opts.nist then
+ alg = 'nist'
+ end
+ pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg)
+ end
+
+ if not pk then
+ fatal("cannot create pubkey")
+ end
+
+ local valid = true
+
+ for _, fname in ipairs(opts.file) do
+
+ local sig_fname = string.format('%s.%s', fname, opts.suffix or 'sig')
+ local sig = rspamd_signature.load(sig_fname, alg)
+
+ if not sig then
+ fatal(string.format("cannot load signature for %s -> %s",
+ fname, sig_fname))
+ end
+
+ if rspamd_crypto.verify_file(pk, sig, fname, alg) then
+ io.write(string.format('verified %s -> %s (%s)\n', fname, sig_fname, sig:hex()))
+ else
+ valid = false
+ io.write(string.format('FAILED to verify %s -> %s (%s)\n', fname,
+ sig_fname, sig:hex()))
+ end
+ end
+
+ if not valid then
+ os.exit(1)
+ end
+end
+
+local function encrypt_handler(opts)
+ if opts.file then
+ if type(opts.file) == 'string' then
+ opts.file = { opts.file }
+ end
+ else
+ parser:error('no files to sign')
+ end
+
+ local pk
+ local alg = 'curve25519'
+
+ if opts.keypair then
+ local ucl_parser = ucl.parser()
+ local res, err = ucl_parser:parse_file(opts.keypair)
+
+ if not res then
+ fatal(string.format('cannot load %s: %s', opts.keypair, err))
+ end
+
+ local kp = rspamd_keypair.load(ucl_parser:get_object())
+
+ if not kp then
+ fatal("cannot load keypair: " .. opts.keypair)
+ end
+
+ pk = kp:pk()
+ alg = kp:alg()
+ elseif opts.pubkey then
+ if opts.nist then
+ alg = 'nist'
+ end
+ pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg)
+ elseif opts.pubstr then
+ if opts.nist then
+ alg = 'nist'
+ end
+ pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg)
+ end
+
+ if not pk then
+ fatal("cannot load keypair: " .. opts.keypair)
+ end
+
+ for _, fname in ipairs(opts.file) do
+ local enc = rspamd_crypto.encrypt_file(pk, fname, alg)
+
+ if not enc then
+ fatal(string.format("cannot encrypt %s\n", fname))
+ end
+
+ local out
+ if opts.suffix and #opts.suffix > 0 then
+ out = string.format('%s.%s', fname, opts.suffix)
+ else
+ out = string.format('%s', fname)
+ end
+
+ if rspamd_util.file_exists(out) then
+ if opts.force or ask_yes_no(string.format('File %s already exists, overwrite?',
+ out), true) then
+ os.remove(out)
+ else
+ os.exit(1)
+ end
+ end
+
+ enc:save_in_file(out)
+
+ if opts.rm then
+ os.remove(fname)
+ io.write(string.format('encrypted %s (deleted) -> %s\n', fname, out))
+ else
+ io.write(string.format('encrypted %s -> %s\n', fname, out))
+ end
+ end
+end
+
+local function decrypt_handler(opts)
+ if opts.file then
+ if type(opts.file) == 'string' then
+ opts.file = { opts.file }
+ end
+ else
+ parser:error('no files to decrypt')
+ end
+ if not opts.keypair then
+ parser:error("no keypair specified")
+ end
+
+ local ucl_parser = ucl.parser()
+ local res, err = ucl_parser:parse_file(opts.keypair)
+
+ if not res then
+ fatal(string.format('cannot load %s: %s', opts.keypair, err))
+ end
+
+ local kp = rspamd_keypair.load(ucl_parser:get_object())
+
+ if not kp then
+ fatal("cannot load keypair: " .. opts.keypair)
+ end
+
+ for _, fname in ipairs(opts.file) do
+ local decrypted = rspamd_crypto.decrypt_file(kp, fname)
+
+ if not decrypted then
+ fatal(string.format("cannot decrypt %s\n", fname))
+ end
+
+ local out
+ if not opts['keep-suffix'] then
+ -- Strip the last suffix
+ out = fname:match("^(.+)%..+$")
+ else
+ out = fname
+ end
+
+ local removed = false
+
+ if rspamd_util.file_exists(out) then
+ if (opts.force or opts['keep-suffix'])
+ or ask_yes_no(string.format('File %s already exists, overwrite?', out), true) then
+ os.remove(out)
+ removed = true
+ else
+ os.exit(1)
+ end
+ end
+
+ if opts.rm then
+ os.remove(fname)
+ removed = true
+ end
+
+ if removed then
+ io.write(string.format('decrypted %s (removed) -> %s\n', fname, out))
+ else
+ io.write(string.format('decrypted %s -> %s\n', fname, out))
+ end
+ end
+end
+
+local function handler(args)
+ local opts = parser:parse(args)
+
+ local command = opts.command or "generate"
+
+ if command == 'generate' then
+ generate_handler(opts)
+ elseif command == 'sign' then
+ sign_handler(opts)
+ elseif command == 'verify' then
+ verify_handler(opts)
+ elseif command == 'encrypt' then
+ encrypt_handler(opts)
+ elseif command == 'decrypt' then
+ decrypt_handler(opts)
+ else
+ parser:error('command %s is not implemented', command)
+ end
+end
+
+return {
+ name = 'keypair',
+ aliases = { 'kp', 'key' },
+ handler = handler,
+ description = parser._description
+} \ No newline at end of file
diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua
new file mode 100644
index 0000000..6a589d6
--- /dev/null
+++ b/lualib/rspamadm/mime.lua
@@ -0,0 +1,1012 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+local ansicolors = require "ansicolors"
+local rspamd_util = require "rspamd_util"
+local rspamd_task = require "rspamd_task"
+local rspamd_text = require "rspamd_text"
+local rspamd_logger = require "rspamd_logger"
+local lua_meta = require "lua_meta"
+local rspamd_url = require "rspamd_url"
+local lua_util = require "lua_util"
+local lua_mime = require "lua_mime"
+local ucl = require "ucl"
+
+-- Define command line options
+local parser = argparse()
+ :name "rspamadm mime"
+ :description "Mime manipulations provided by Rspamd"
+ :help_description_margin(30)
+ :command_target("command")
+ :require_command(true)
+
+parser:option "-c --config"
+ :description "Path to config file"
+ :argname("<cfg>")
+ :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
+parser:mutex(
+ parser:flag "-j --json"
+ :description "JSON output",
+ parser:flag "-U --ucl"
+ :description "UCL output",
+ parser:flag "-M --messagepack"
+ :description "MessagePack output"
+)
+parser:flag "-C --compact"
+ :description "Use compact format"
+parser:flag "--no-file"
+ :description "Do not print filename"
+
+-- Extract subcommand
+local extract = parser:command "extract ex e"
+ :description "Extracts data from MIME messages"
+extract:argument "file"
+ :description "File to process"
+ :argname "<file>"
+ :args "+"
+
+extract:flag "-t --text"
+ :description "Extracts plain text data from a message"
+extract:flag "-H --html"
+ :description "Extracts htm data from a message"
+extract:option "-o --output"
+ :description "Output format ('raw', 'content', 'oneline', 'decoded', 'decoded_utf')"
+ :argname("<type>")
+ :convert {
+ raw = "raw",
+ content = "content",
+ oneline = "content_oneline",
+ decoded = "raw_parsed",
+ decoded_utf = "raw_utf"
+}
+ :default "content"
+extract:flag "-w --words"
+ :description "Extracts words"
+extract:flag "-p --part"
+ :description "Show part info"
+extract:flag "-s --structure"
+ :description "Show structure info (e.g. HTML tags)"
+extract:flag "-i --invisible"
+ :description "Show invisible content for HTML parts"
+extract:option "-F --words-format"
+ :description "Words format ('stem', 'norm', 'raw', 'full')"
+ :argname("<type>")
+ :convert {
+ stem = "stem",
+ norm = "norm",
+ raw = "raw",
+ full = "full",
+}
+ :default "stem"
+
+local stat = parser:command "stat st s"
+ :description "Extracts statistical data from MIME messages"
+stat:argument "file"
+ :description "File to process"
+ :argname "<file>"
+ :args "+"
+stat:mutex(
+ stat:flag "-m --meta"
+ :description "Lua metatokens",
+ stat:flag "-b --bayes"
+ :description "Bayes tokens",
+ stat:flag "-F --fuzzy"
+ :description "Fuzzy hashes"
+)
+stat:flag "-s --shingles"
+ :description "Show shingles for fuzzy hashes"
+
+local urls = parser:command "urls url u"
+ :description "Extracts URLs from MIME messages"
+urls:argument "file"
+ :description "File to process"
+ :argname "<file>"
+ :args "+"
+urls:mutex(
+ urls:flag "-t --tld"
+ :description "Get TLDs only",
+ urls:flag "-H --host"
+ :description "Get hosts only",
+ urls:flag "-f --full"
+ :description "Show piecewise urls as processed by Rspamd"
+)
+
+urls:flag "-u --unique"
+ :description "Print only unique urls"
+urls:flag "-s --sort"
+ :description "Sort output"
+urls:flag "--count"
+ :description "Print count of each printed element"
+urls:flag "-r --reverse"
+ :description "Reverse sort order"
+
+local modify = parser:command "modify mod m"
+ :description "Modifies MIME message"
+modify:argument "file"
+ :description "File to process"
+ :argname "<file>"
+ :args "+"
+
+modify:option "-a --add-header"
+ :description "Adds specific header"
+ :argname "<header=value>"
+ :count "*"
+modify:option "-r --remove-header"
+ :description "Removes specific header (all occurrences)"
+ :argname "<header>"
+ :count "*"
+modify:option "-R --rewrite-header"
+ :description "Rewrites specific header, uses Lua string.format pattern"
+ :argname "<header=pattern>"
+ :count "*"
+modify:option "-t --text-footer"
+ :description "Adds footer to text/plain parts from a specific file"
+ :argname "<file>"
+modify:option "-H --html-footer"
+ :description "Adds footer to text/html parts from a specific file"
+ :argname "<file>"
+
+local sign = parser:command "sign"
+ :description "Performs DKIM signing"
+sign:argument "file"
+ :description "File to process"
+ :argname "<file>"
+ :args "+"
+
+sign:option "-d --domain"
+ :description "Use specific domain"
+ :argname "<domain>"
+ :count "1"
+sign:option "-s --selector"
+ :description "Use specific selector"
+ :argname "<selector>"
+ :count "1"
+sign:option "-k --key"
+ :description "Use specific key of file"
+ :argname "<key>"
+ :count "1"
+sign:option "-t --type"
+ :description "ARC or DKIM signing"
+ :argname("<arc|dkim>")
+ :convert {
+ ['arc'] = 'arc',
+ ['dkim'] = 'dkim',
+}
+ :default 'dkim'
+sign:option "-o --output"
+ :description "Output format"
+ :argname("<message|signature>")
+ :convert {
+ ['message'] = 'message',
+ ['signature'] = 'signature',
+}
+ :default 'message'
+
+local dump = parser:command "dump"
+ :description "Dumps a raw message in different formats"
+dump:argument "file"
+ :description "File to process"
+ :argname "<file>"
+ :args "+"
+-- Duplicate format for convenience
+dump:mutex(
+ parser:flag "-j --json"
+ :description "JSON output",
+ parser:flag "-U --ucl"
+ :description "UCL output",
+ parser:flag "-M --messagepack"
+ :description "MessagePack output"
+)
+dump:flag "-s --split"
+ :description "Split the output file contents such that no content is embedded"
+
+dump:option "-o --outdir"
+ :description "Output directory"
+ :argname("<directory>")
+
+local function load_config(opts)
+ local _r, err = rspamd_config:load_ucl(opts['config'])
+
+ if not _r then
+ rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
+ if not _r then
+ rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+end
+
+local function load_task(opts, fname)
+ if not fname then
+ fname = '-'
+ end
+
+ local res, task = rspamd_task.load_from_file(fname, rspamd_config)
+
+ if not res then
+ parser:error(string.format('cannot read message from %s: %s', fname,
+ task))
+ end
+
+ if not task:process_message() then
+ parser:error(string.format('cannot read message from %s: %s', fname,
+ 'failed to parse'))
+ end
+
+ return task
+end
+
+local function highlight(fmt, ...)
+ return ansicolors.white .. string.format(fmt, ...) .. ansicolors.reset
+end
+
+local function maybe_print_fname(opts, fname)
+ if not opts.json and not opts['no-file'] then
+ rspamd_logger.messagex(highlight('File: %s', fname))
+ end
+end
+
+local function output_fmt(opts)
+ local fmt = 'json'
+ if opts.compact then
+ fmt = 'json-compact'
+ end
+ if opts.ucl then
+ fmt = 'ucl'
+ end
+ if opts.messagepack then
+ fmt = 'msgpack'
+ end
+
+ return fmt
+end
+
+-- Print elements in form
+-- filename -> table of elements
+local function print_elts(elts, opts, func)
+ local fun = require "fun"
+
+ if opts.json or opts.ucl then
+ io.write(ucl.to_format(elts, output_fmt(opts)))
+ else
+ fun.each(function(fname, elt)
+
+ if not opts.json and not opts.ucl then
+ if func then
+ elt = fun.map(func, elt)
+ end
+ maybe_print_fname(opts, fname)
+ fun.each(function(e)
+ io.write(e)
+ io.write("\n")
+ end, elt)
+ end
+ end, elts)
+ end
+end
+
+local function extract_handler(opts)
+ local out_elts = {}
+ local tasks = {}
+ local process_func
+
+ if opts.words then
+ -- Enable stemming and urls detection
+ load_config(opts)
+ rspamd_url.init(rspamd_config:get_tld_path())
+ rspamd_config:init_subsystem('langdet')
+ end
+
+ local function maybe_print_text_part_info(part, out)
+ local fun = require "fun"
+ if opts.part then
+ local t = 'plain text'
+ if part:is_html() then
+ t = 'html'
+ end
+
+ if not opts.json and not opts.ucl then
+ table.insert(out,
+ rspamd_logger.slog('Part: %s: %s, language: %s, size: %s (%s raw), words: %s',
+ part:get_mimepart():get_digest():sub(1, 8),
+ t,
+ part:get_language(),
+ part:get_length(), part:get_raw_length(),
+ part:get_words_count()))
+ table.insert(out,
+ rspamd_logger.slog('Stats: %s',
+ fun.foldl(function(acc, k, v)
+ if acc ~= '' then
+ return string.format('%s, %s:%s', acc, k, v)
+ else
+ return string.format('%s:%s', k, v)
+ end
+ end, '', part:get_stats())))
+ end
+ end
+ end
+
+ local function maybe_print_mime_part_info(part, out)
+ if opts.part then
+
+ if not opts.json and not opts.ucl then
+ local mtype, msubtype = part:get_type()
+ local det_mtype, det_msubtype = part:get_detected_type()
+ table.insert(out,
+ rspamd_logger.slog('Mime Part: %s: %s/%s (%s/%s detected), filename: %s (%s detected ext), size: %s',
+ part:get_digest():sub(1, 8),
+ mtype, msubtype,
+ det_mtype, det_msubtype,
+ part:get_filename(),
+ part:get_detected_ext(),
+ part:get_length()))
+ end
+ end
+ end
+
+ local function print_words(words, full)
+ local fun = require "fun"
+
+ if not full then
+ return table.concat(words, ' ')
+ else
+ return table.concat(
+ fun.totable(
+ fun.map(function(w)
+ -- [1] - stemmed word
+ -- [2] - normalised word
+ -- [3] - raw word
+ -- [4] - flags (table of strings)
+ return string.format('%s|%s|%s(%s)',
+ w[3], w[2], w[1], table.concat(w[4], ','))
+ end, words)
+ ),
+ ' '
+ )
+ end
+ end
+
+ for _, fname in ipairs(opts.file) do
+ local task = load_task(opts, fname)
+ out_elts[fname] = {}
+
+ if not opts.text and not opts.html then
+ opts.text = true
+ opts.html = true
+ end
+
+ if opts.words then
+ local how_words = opts['words_format'] or 'stem'
+ table.insert(out_elts[fname], 'meta_words: ' ..
+ print_words(task:get_meta_words(how_words), how_words == 'full'))
+ end
+
+ if opts.text or opts.html then
+ local mp = task:get_parts() or {}
+
+ for _, mime_part in ipairs(mp) do
+ local how = opts.output
+ local part
+ if mime_part:is_text() then
+ part = mime_part:get_text()
+ end
+
+ if part and opts.text and not part:is_html() then
+ maybe_print_text_part_info(part, out_elts[fname])
+ maybe_print_mime_part_info(mime_part, out_elts[fname])
+ if not opts.json and not opts.ucl then
+ table.insert(out_elts[fname], '\n')
+ end
+
+ if opts.words then
+ local how_words = opts['words_format'] or 'stem'
+ table.insert(out_elts[fname], print_words(part:get_words(how_words),
+ how_words == 'full'))
+ else
+ table.insert(out_elts[fname], tostring(part:get_content(how)))
+ end
+ elseif part and opts.html and part:is_html() then
+ maybe_print_text_part_info(part, out_elts[fname])
+ maybe_print_mime_part_info(mime_part, out_elts[fname])
+ if not opts.json and not opts.ucl then
+ table.insert(out_elts[fname], '\n')
+ end
+
+ if opts.words then
+ local how_words = opts['words_format'] or 'stem'
+ table.insert(out_elts[fname], print_words(part:get_words(how_words),
+ how_words == 'full'))
+ else
+ if opts.structure then
+ local hc = part:get_html()
+ local res = {}
+ process_func = function(elt)
+ local fun = require "fun"
+ if type(elt) == 'table' then
+ return table.concat(fun.totable(
+ fun.map(
+ function(t)
+ return rspamd_logger.slog("%s", t)
+ end,
+ elt)), '\n')
+ else
+ return rspamd_logger.slog("%s", elt)
+ end
+ end
+
+ hc:foreach_tag('any', function(tag)
+ local elt = {}
+ local ex = tag:get_extra()
+ elt.tag = tag:get_type()
+ if ex then
+ elt.extra = ex
+ end
+ local content = tag:get_content()
+ if content then
+ elt.content = tostring(content)
+ end
+ local style = tag:get_style()
+ if style then
+ elt.style = style
+ end
+ table.insert(res, elt)
+ end)
+ table.insert(out_elts[fname], res)
+ else
+ -- opts.structure
+ table.insert(out_elts[fname], tostring(part:get_content(how)))
+ end
+ if opts.invisible then
+ local hc = part:get_html()
+ table.insert(out_elts[fname], string.format('invisible content: %s',
+ tostring(hc:get_invisible())))
+ end
+ end
+ end
+
+ if not part then
+ maybe_print_mime_part_info(mime_part, out_elts[fname])
+ end
+ end
+ end
+
+ table.insert(out_elts[fname], "")
+ table.insert(tasks, task)
+ end
+
+ print_elts(out_elts, opts, process_func)
+ -- To avoid use after free we postpone tasks destruction
+ for _, task in ipairs(tasks) do
+ task:destroy()
+ end
+end
+
+local function stat_handler(opts)
+ local fun = require "fun"
+ local out_elts = {}
+
+ load_config(opts)
+ rspamd_url.init(rspamd_config:get_tld_path())
+ rspamd_config:init_subsystem('langdet,stat') -- Needed to gen stat tokens
+
+ local process_func
+
+ for _, fname in ipairs(opts.file) do
+ local task = load_task(opts, fname)
+ out_elts[fname] = {}
+
+ if opts.meta then
+ local mt = lua_meta.gen_metatokens_table(task)
+ out_elts[fname] = mt
+ process_func = function(k, v)
+ return string.format("%s = %s", k, v)
+ end
+ elseif opts.bayes then
+ local bt = task:get_stat_tokens()
+ out_elts[fname] = bt
+ process_func = function(e)
+ return string.format('%s (%d): "%s"+"%s", [%s]', e.data, e.win, e.t1 or "",
+ e.t2 or "", table.concat(fun.totable(
+ fun.map(function(k)
+ return k
+ end, e.flags)), ","))
+ end
+ elseif opts.fuzzy then
+ local parts = task:get_parts() or {}
+ out_elts[fname] = {}
+ process_func = function(e)
+ local ret = string.format('part: %s(%s): %s', e.type, e.file or "", e.digest)
+ if opts.shingles and e.shingles then
+ local sgl = {}
+ for _, s in ipairs(e.shingles) do
+ table.insert(sgl, string.format('%s: %s+%s+%s', s[1], s[2], s[3], s[4]))
+ end
+
+ ret = ret .. '\n' .. table.concat(sgl, '\n')
+ end
+ return ret
+ end
+ for _, part in ipairs(parts) do
+ if not part:is_multipart() then
+ local text = part:get_text()
+
+ if text then
+ local digest, shingles = text:get_fuzzy_hashes(task:get_mempool())
+ table.insert(out_elts[fname], {
+ digest = digest,
+ shingles = shingles,
+ type = string.format('%s/%s',
+ ({ part:get_type() })[1],
+ ({ part:get_type() })[2])
+ })
+ else
+ table.insert(out_elts[fname], {
+ digest = part:get_digest(),
+ file = part:get_filename(),
+ type = string.format('%s/%s',
+ ({ part:get_type() })[1],
+ ({ part:get_type() })[2])
+ })
+ end
+ end
+ end
+ end
+
+ task:destroy() -- No automatic dtor
+ end
+
+ print_elts(out_elts, opts, process_func)
+end
+
+local function urls_handler(opts)
+ load_config(opts)
+ rspamd_url.init(rspamd_config:get_tld_path())
+ local out_elts = {}
+
+ if opts.json then
+ rspamd_logger.messagex('[')
+ end
+
+ for _, fname in ipairs(opts.file) do
+ out_elts[fname] = {}
+ local task = load_task(opts, fname)
+ local elts = {}
+
+ local function process_url(u)
+ local s
+ if opts.tld then
+ s = u:get_tld()
+ elseif opts.host then
+ s = u:get_host()
+ elseif opts.full then
+ s = rspamd_logger.slog('%s: %s', u:get_text(), u:to_table())
+ else
+ s = u:get_text()
+ end
+
+ if opts.unique then
+ if elts[s] then
+ elts[s].count = elts[s].count + 1
+ else
+ elts[s] = {
+ count = 1,
+ url = u:to_table()
+ }
+ end
+ else
+ if opts.json then
+ table.insert(elts, u)
+ else
+ table.insert(elts, s)
+ end
+ end
+ end
+
+ for _, u in ipairs(task:get_urls(true)) do
+ process_url(u)
+ end
+
+ local json_elts = {}
+
+ local function process_elt(s, u)
+ if opts.unique then
+ -- s is string, u is {url = url, count = count }
+ if not opts.json then
+ if opts.count then
+ table.insert(json_elts, string.format('%s : %s', s, u.count))
+ else
+ table.insert(json_elts, s)
+ end
+ else
+ local tb = u.url
+ tb.count = u.count
+ table.insert(json_elts, tb)
+ end
+ else
+ -- s is index, u is url or string
+ if opts.json then
+ table.insert(json_elts, u)
+ else
+ table.insert(json_elts, u)
+ end
+ end
+ end
+
+ if opts.sort then
+ local sfunc
+ if opts.unique then
+ sfunc = function(t, a, b)
+ if t[a].count ~= t[b].count then
+ if opts.reverse then
+ return t[a].count > t[b].count
+ else
+ return t[a].count < t[b].count
+ end
+ else
+ -- Sort lexicography
+ if opts.reverse then
+ return a > b
+ else
+ return a < b
+ end
+ end
+ end
+ else
+ sfunc = function(t, a, b)
+ local va, vb
+ if opts.json then
+ va = t[a]:get_text()
+ vb = t[b]:get_text()
+ else
+ va = t[a]
+ vb = t[b]
+ end
+ if opts.reverse then
+ return va > vb
+ else
+ return va < vb
+ end
+ end
+ end
+
+ for s, u in lua_util.spairs(elts, sfunc) do
+ process_elt(s, u)
+ end
+ else
+ for s, u in pairs(elts) do
+ process_elt(s, u)
+ end
+ end
+
+ out_elts[fname] = json_elts
+
+ task:destroy() -- No automatic dtor
+ end
+
+ print_elts(out_elts, opts)
+end
+
+local function newline(task)
+ local t = task:get_newlines_type()
+
+ if t == 'cr' then
+ return '\r'
+ elseif t == 'lf' then
+ return '\n'
+ end
+
+ return '\r\n'
+end
+
+local function modify_handler(opts)
+ load_config(opts)
+ rspamd_url.init(rspamd_config:get_tld_path())
+
+ local function read_file(file)
+ local f = assert(io.open(file, "rb"))
+ local content = f:read("*all")
+ f:close()
+ return content
+ end
+
+ local text_footer, html_footer
+
+ if opts['text_footer'] then
+ text_footer = read_file(opts['text_footer'])
+ end
+
+ if opts['html_footer'] then
+ html_footer = read_file(opts['html_footer'])
+ end
+
+ for _, fname in ipairs(opts.file) do
+ local task = load_task(opts, fname)
+ local newline_s = newline(task)
+ local seen_cte
+
+ local rewrite = lua_mime.add_text_footer(task, html_footer, text_footer) or {}
+ local out = {} -- Start with headers
+
+ local function process_headers_cb(name, hdr)
+ for _, h in ipairs(opts['remove_header']) do
+ if name:match(h) then
+ return
+ end
+ end
+
+ for _, h in ipairs(opts['rewrite_header']) do
+ local hname, hpattern = h:match('^([^=]+)=(.+)$')
+ if hname == name then
+ local new_value = string.format(hpattern, hdr.decoded)
+ new_value = string.format('%s:%s%s',
+ name, hdr.separator,
+ rspamd_util.fold_header(name,
+ rspamd_util.mime_header_encode(new_value),
+ task:get_newlines_type()))
+ out[#out + 1] = new_value
+ return
+ end
+ end
+
+ if rewrite.need_rewrite_ct then
+ if name:lower() == 'content-type' then
+ local nct = string.format('%s: %s/%s; charset=utf-8',
+ 'Content-Type', rewrite.new_ct.type, rewrite.new_ct.subtype)
+ out[#out + 1] = nct
+ return
+ elseif name:lower() == 'content-transfer-encoding' then
+ out[#out + 1] = string.format('%s: %s',
+ 'Content-Transfer-Encoding', rewrite.new_cte or 'quoted-printable')
+ seen_cte = true
+ return
+ end
+ end
+
+ out[#out + 1] = hdr.raw:gsub('\r?\n?$', '')
+ end
+
+ task:headers_foreach(process_headers_cb, { full = true })
+
+ for _, h in ipairs(opts['add_header']) do
+ local hname, hvalue = h:match('^([^=]+)=(.+)$')
+
+ if hname and hvalue then
+ out[#out + 1] = string.format('%s: %s', hname,
+ rspamd_util.fold_header(hname, hvalue, task:get_newlines_type()))
+ end
+ end
+
+ if not seen_cte and rewrite.need_rewrite_ct then
+ out[#out + 1] = string.format('%s: %s',
+ 'Content-Transfer-Encoding', rewrite.new_cte or 'quoted-printable')
+ end
+
+ -- End of headers
+ out[#out + 1] = ''
+
+ if rewrite.out then
+ for _, o in ipairs(rewrite.out) do
+ out[#out + 1] = o
+ end
+ else
+ out[#out + 1] = { task:get_rawbody(), false }
+ end
+
+ for _, o in ipairs(out) do
+ if type(o) == 'string' then
+ io.write(o)
+ io.write(newline_s)
+ elseif type(o) == 'table' then
+ io.flush()
+ if type(o[1]) == 'string' then
+ io.write(o[1])
+ else
+ o[1]:save_in_file(1)
+ end
+
+ if o[2] then
+ io.write(newline_s)
+ end
+ else
+ o:save_in_file(1)
+ io.write(newline_s)
+ end
+ end
+
+ task:destroy() -- No automatic dtor
+ end
+end
+
+local function sign_handler(opts)
+ load_config(opts)
+ rspamd_url.init(rspamd_config:get_tld_path())
+
+ local lua_dkim = require("lua_ffi").dkim
+
+ if not lua_dkim then
+ io.stderr:write('FFI support is required: please use LuaJIT or install lua-ffi')
+ os.exit(1)
+ end
+
+ local sign_key
+ if rspamd_util.file_exists(opts.key) then
+ sign_key = lua_dkim.load_sign_key(opts.key, 'file')
+ else
+ sign_key = lua_dkim.load_sign_key(opts.key, 'base64')
+ end
+
+ if not sign_key then
+ io.stderr:write('Cannot load key: ' .. opts.key .. '\n')
+ os.exit(1)
+ end
+
+ for _, fname in ipairs(opts.file) do
+ local task = load_task(opts, fname)
+ local ctx = lua_dkim.create_sign_context(task, sign_key, nil, opts.algorithm)
+
+ if not ctx then
+ io.stderr:write('Cannot init signing\n')
+ os.exit(1)
+ end
+
+ local sig = lua_dkim.do_sign(task, ctx, opts.selector, opts.domain)
+
+ if not sig then
+ io.stderr:write('Cannot create signature\n')
+ os.exit(1)
+ end
+
+ if opts.output == 'signature' then
+ io.write(sig)
+ io.write('\n')
+ io.flush()
+ else
+ local dkim_hdr = string.format('%s: %s%s',
+ 'DKIM-Signature',
+ rspamd_util.fold_header('DKIM-Signature',
+ rspamd_util.mime_header_encode(sig),
+ task:get_newlines_type()),
+ newline(task))
+ io.write(dkim_hdr)
+ io.flush()
+ task:get_content():save_in_file(1)
+ end
+
+ task:destroy() -- No automatic dtor
+ end
+end
+
+-- Strips directories and .extensions (if present) from a filepath
+local function filename_only(filepath)
+ local filename = filepath:match(".*%/([^%.]+)")
+ if not filename then
+ filename = filepath:match("([^%.]+)")
+ end
+ return filename
+end
+
+assert(filename_only("very_simple") == "very_simple")
+assert(filename_only("/home/very_simple.eml") == "very_simple")
+assert(filename_only("very_simple.eml") == "very_simple")
+assert(filename_only("very_simple.example.eml") == "very_simple")
+assert(filename_only("/home/very_simple") == "very_simple")
+assert(filename_only("home/very_simple") == "very_simple")
+assert(filename_only("./home/very_simple") == "very_simple")
+assert(filename_only("../home/very_simple.eml") == "very_simple")
+assert(filename_only("/home/dir.with.dots/very_simple.eml") == "very_simple")
+
+--Write the dump content to file or standard out
+local function write_dump_content(dump_content, fname, extension, outdir)
+ if type(dump_content) == "string" then
+ dump_content = rspamd_text.fromstring(dump_content)
+ end
+
+ local wrote_filepath = nil
+ if outdir then
+ if outdir:sub(-1) ~= "/" then
+ outdir = outdir .. "/"
+ end
+
+ local outpath = string.format("%s%s.%s", outdir, filename_only(fname), extension)
+ if rspamd_util.file_exists(outpath) then
+ os.remove(outpath)
+ end
+ if dump_content:save_in_file(outpath) then
+ wrote_filepath = outpath
+ io.write(wrote_filepath .. "\n")
+ else
+ io.stderr:write(string.format("Unable to save dump content to file: %s\n", outpath))
+ end
+ else
+ dump_content:save_in_file(1)
+ end
+ return wrote_filepath
+end
+
+-- Get the formatted ucl (split or unsplit) or the raw task content
+local function get_dump_content(task, opts, fname)
+ if opts.ucl or opts.json or opts.messagepack then
+ local ucl_object = lua_mime.message_to_ucl(task)
+ -- Split out the content field into separate raws and update the ucl
+ if opts.split then
+ for i, part in ipairs(ucl_object.parts) do
+ if part.content then
+ local part_filename = string.format("%s-part%d", filename_only(fname), i)
+ local part_path = write_dump_content(part.content, part_filename, "raw", opts.outdir)
+ if part_path then
+ part.content = ucl.null
+ part.content_path = part_path
+ end
+ end
+ end
+ end
+ local extension = output_fmt(opts)
+ return ucl.to_format(ucl_object, extension), extension
+ end
+ return task:get_content(), "mime"
+end
+
+local function dump_handler(opts)
+ load_config(opts)
+ rspamd_url.init(rspamd_config:get_tld_path())
+
+ for _, fname in ipairs(opts.file) do
+ local task = load_task(opts, fname)
+ local data, extension = get_dump_content(task, opts, fname)
+ write_dump_content(data, fname, extension, opts.outdir)
+
+ task:destroy() -- No automatic dtor
+ end
+end
+
+local function handler(args)
+ local opts = parser:parse(args)
+
+ local command = opts.command
+
+ if type(opts.file) == 'string' then
+ opts.file = { opts.file }
+ elseif type(opts.file) == 'none' then
+ opts.file = {}
+ end
+
+ if command == 'extract' then
+ extract_handler(opts)
+ elseif command == 'stat' then
+ stat_handler(opts)
+ elseif command == 'urls' then
+ urls_handler(opts)
+ elseif command == 'modify' then
+ modify_handler(opts)
+ elseif command == 'sign' then
+ sign_handler(opts)
+ elseif command == 'dump' then
+ dump_handler(opts)
+ else
+ parser:error('command %s is not implemented', command)
+ end
+end
+
+return {
+ name = 'mime',
+ aliases = { 'mime_tool' },
+ handler = handler,
+ description = parser._description
+}
diff --git a/lualib/rspamadm/neural_test.lua b/lualib/rspamadm/neural_test.lua
new file mode 100644
index 0000000..31d21a9
--- /dev/null
+++ b/lualib/rspamadm/neural_test.lua
@@ -0,0 +1,228 @@
+local rspamd_logger = require "rspamd_logger"
+local argparse = require "argparse"
+local lua_util = require "lua_util"
+local ucl = require "ucl"
+
+local parser = argparse()
+ :name "rspamadm neural_test"
+ :description "Test the neural network with labelled dataset"
+ :help_description_margin(32)
+
+parser:option "-c --config"
+ :description "Path to config file"
+ :argname("<cfg>")
+ :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
+parser:option "-H --hamdir"
+ :description("Ham directory")
+ :argname("<dir>")
+parser:option "-S --spamdir"
+ :description("Spam directory")
+ :argname("<dir>")
+parser:option "-t --timeout"
+ :description("Timeout for client connections")
+ :argname("<sec>")
+ :convert(tonumber)
+ :default(60)
+parser:option "-n --conns"
+ :description("Number of parallel connections")
+ :argname("<N>")
+ :convert(tonumber)
+ :default(10)
+parser:option "-c --connect"
+ :description("Connect to specific host")
+ :argname("<host>")
+ :default('localhost:11334')
+parser:option "-r --rspamc"
+ :description("Use specific rspamc path")
+ :argname("<path>")
+ :default('rspamc')
+parser:option '--rule'
+ :description 'Rule to test'
+ :argname('<rule>')
+
+local HAM = "HAM"
+local SPAM = "SPAM"
+
+local function load_config(opts)
+ local _r, err = rspamd_config:load_ucl(opts['config'])
+
+ if not _r then
+ rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
+ if not _r then
+ rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+end
+
+local function scan_email(rspamc_path, host, n_parallel, path, timeout)
+
+ local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s",
+ rspamc_path, host, n_parallel, timeout, path)
+ local result = assert(io.popen(rspamc_command))
+ result = result:read("*all")
+ return result
+end
+
+local function encoded_json_to_log(result)
+ -- Returns table containing score, action, list of symbols
+
+ local filtered_result = {}
+ local ucl_parser = ucl.parser()
+
+ local is_good, err = ucl_parser:parse_string(result)
+
+ if not is_good then
+ rspamd_logger.errx("Parser error: %1", err)
+ return nil
+ end
+
+ result = ucl_parser:get_object()
+
+ filtered_result.score = result.score
+ if not result.action then
+ rspamd_logger.errx("Bad JSON: %1", result)
+ return nil
+ end
+ local action = result.action:gsub("%s+", "_")
+ filtered_result.action = action
+
+ filtered_result.symbols = {}
+
+ for sym, _ in pairs(result.symbols) do
+ table.insert(filtered_result.symbols, sym)
+ end
+
+ filtered_result.filename = result.filename
+ filtered_result.scan_time = result.scan_time
+
+ return filtered_result
+end
+
+local function filter_scan_results(results, actual_email_type)
+
+ local logs = {}
+
+ results = lua_util.rspamd_str_split(results, "\n")
+
+ if results[#results] == "" then
+ results[#results] = nil
+ end
+
+ for _, result in pairs(results) do
+ result = encoded_json_to_log(result)
+ if result then
+ result['type'] = actual_email_type
+ table.insert(logs, result)
+ end
+ end
+
+ return logs
+end
+
+local function get_stats_from_scan_results(results, rules)
+
+ local rule_stats = {}
+ for rule, _ in pairs(rules) do
+ rule_stats[rule] = { tp = 0, tn = 0, fp = 0, fn = 0 }
+ end
+
+ for _, result in ipairs(results) do
+ for _, symbol in ipairs(result["symbols"]) do
+ for name, rule in pairs(rules) do
+ if rule.symbol_spam and rule.symbol_spam == symbol then
+ if result.type == HAM then
+ rule_stats[name].fp = rule_stats[name].fp + 1
+ elseif result.type == SPAM then
+ rule_stats[name].tp = rule_stats[name].tp + 1
+ end
+ elseif rule.symbol_ham and rule.symbol_ham == symbol then
+ if result.type == HAM then
+ rule_stats[name].tn = rule_stats[name].tn + 1
+ elseif result.type == SPAM then
+ rule_stats[name].fn = rule_stats[name].fn + 1
+ end
+ end
+ end
+ end
+ end
+
+ for rule, _ in pairs(rules) do
+ rule_stats[rule].fpr = rule_stats[rule].fp / (rule_stats[rule].fp + rule_stats[rule].tn)
+ rule_stats[rule].fnr = rule_stats[rule].fn / (rule_stats[rule].fn + rule_stats[rule].tp)
+ end
+
+ return rule_stats
+end
+
+local function print_neural_stats(neural_stats)
+ for rule, stats in pairs(neural_stats) do
+ rspamd_logger.messagex("\nStats for rule: %s", rule)
+ rspamd_logger.messagex("False positive rate: %s%%", stats.fpr * 100)
+ rspamd_logger.messagex("False negative rate: %s%%", stats.fnr * 100)
+ end
+end
+
+local function handler(args)
+ local opts = parser:parse(args)
+
+ local ham_directory = opts['hamdir']
+ local spam_directory = opts['spamdir']
+ local connections = opts["conns"]
+
+ load_config(opts)
+
+ local neural_opts = rspamd_config:get_all_opt('neural')
+
+ if opts["rule"] then
+ local found = false
+ for rule_name, _ in pairs(neural_opts.rules) do
+ if string.lower(rule_name) == string.lower(opts["rule"]) then
+ found = true
+ else
+ neural_opts.rules[rule_name] = nil
+ end
+ end
+
+ if not found then
+ rspamd_logger.errx("Couldn't find the rule %s", opts["rule"])
+ return
+ end
+ end
+
+ local results = {}
+
+ if ham_directory then
+ rspamd_logger.messagex("Scanning ham corpus...")
+ local ham_results = scan_email(opts.rspamc, opts.connect, connections, ham_directory, opts.timeout)
+ ham_results = filter_scan_results(ham_results, HAM)
+
+ for _, result in pairs(ham_results) do
+ table.insert(results, result)
+ end
+ end
+
+ if spam_directory then
+ rspamd_logger.messagex("Scanning spam corpus...")
+ local spam_results = scan_email(opts.rspamc, opts.connect, connections, spam_directory, opts.timeout)
+ spam_results = filter_scan_results(spam_results, SPAM)
+
+ for _, result in pairs(spam_results) do
+ table.insert(results, result)
+ end
+ end
+
+ local neural_stats = get_stats_from_scan_results(results, neural_opts.rules)
+ print_neural_stats(neural_stats)
+
+end
+
+return {
+ name = "neuraltest",
+ aliases = { "neural_test" },
+ handler = handler,
+ description = parser._description
+} \ No newline at end of file
diff --git a/lualib/rspamadm/publicsuffix.lua b/lualib/rspamadm/publicsuffix.lua
new file mode 100644
index 0000000..96bf069
--- /dev/null
+++ b/lualib/rspamadm/publicsuffix.lua
@@ -0,0 +1,82 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+local rspamd_logger = require "rspamd_logger"
+
+-- Define command line options
+local parser = argparse()
+ :name 'rspamadm publicsuffix'
+ :description 'Do manipulations with the publicsuffix list'
+ :help_description_margin(30)
+ :command_target('command')
+ :require_command(true)
+
+parser:option '-c --config'
+ :description 'Path to config file'
+ :argname('config_file')
+ :default(rspamd_paths['CONFDIR'] .. '/rspamd.conf')
+
+parser:command 'compile'
+ :description 'Compile publicsuffix list if needed'
+
+local function load_config(config_file)
+ local _r, err = rspamd_config:load_ucl(config_file)
+
+ if not _r then
+ rspamd_logger.errx('cannot load %s: %s', config_file, err)
+ os.exit(1)
+ end
+
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
+ if not _r then
+ rspamd_logger.errx('cannot process %s: %s', config_file, err)
+ os.exit(1)
+ end
+end
+
+local function compile_handler(_)
+ local rspamd_url = require "rspamd_url"
+ local tld_file = rspamd_config:get_tld_path()
+
+ if not tld_file then
+ rspamd_logger.errx('missing `url_tld` option, cannot continue')
+ os.exit(1)
+ end
+
+ rspamd_logger.messagex('loading public suffix file from %s', tld_file)
+ rspamd_url.init(tld_file)
+ rspamd_logger.messagex('public suffix file has been loaded')
+end
+
+local function handler(args)
+ local cmd_opts = parser:parse(args)
+
+ load_config(cmd_opts.config_file)
+
+ if cmd_opts.command == 'compile' then
+ compile_handler(cmd_opts)
+ else
+ rspamd_logger.errx('unknown command: %s', cmd_opts.command)
+ os.exit(1)
+ end
+end
+
+return {
+ handler = handler,
+ description = parser._description,
+ name = 'publicsuffix'
+} \ No newline at end of file
diff --git a/lualib/rspamadm/stat_convert.lua b/lualib/rspamadm/stat_convert.lua
new file mode 100644
index 0000000..62a19a2
--- /dev/null
+++ b/lualib/rspamadm/stat_convert.lua
@@ -0,0 +1,38 @@
+local lua_redis = require "lua_redis"
+local stat_tools = require "lua_stat"
+local ucl = require "ucl"
+local logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+
+return function(_, res)
+ local redis_params = lua_redis.try_load_redis_servers(res.redis, nil)
+ if res.expire then
+ res.expire = lua_util.parse_time_interval(res.expire)
+ end
+ if not redis_params then
+ logger.errx('cannot load redis server definition')
+
+ return false
+ end
+
+ local sqlite_params = stat_tools.load_sqlite_config(res)
+
+ if #sqlite_params == 0 then
+ logger.errx('cannot load sqlite classifiers')
+ return false
+ end
+
+ for _, cls in ipairs(sqlite_params) do
+ if not stat_tools.convert_sqlite_to_redis(redis_params, cls.db_spam,
+ cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, res.expire,
+ res.reset_previous) then
+ logger.errx('conversion failed')
+
+ return false
+ end
+ logger.messagex('Converted classifier to the from sqlite to redis')
+ logger.messagex('Suggested configuration:')
+ logger.messagex(ucl.to_format(stat_tools.redis_classifier_from_sqlite(cls, res.expire),
+ 'config'))
+ end
+end
diff --git a/lualib/rspamadm/statistics_dump.lua b/lualib/rspamadm/statistics_dump.lua
new file mode 100644
index 0000000..6bc0458
--- /dev/null
+++ b/lualib/rspamadm/statistics_dump.lua
@@ -0,0 +1,544 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local lua_redis = require "lua_redis"
+local rspamd_logger = require "rspamd_logger"
+local argparse = require "argparse"
+local rspamd_zstd = require "rspamd_zstd"
+local rspamd_text = require "rspamd_text"
+local rspamd_util = require "rspamd_util"
+local rspamd_cdb = require "rspamd_cdb"
+local lua_util = require "lua_util"
+local rspamd_i64 = require "rspamd_int64"
+local ucl = require "ucl"
+
+local N = "statistics_dump"
+local E = {}
+local classifiers = {}
+
+-- Define command line options
+local parser = argparse()
+ :name "rspamadm statistics_dump"
+ :description "Dump/restore Rspamd statistics"
+ :help_description_margin(30)
+ :command_target("command")
+ :require_command(false)
+
+parser:option "-c --config"
+ :description "Path to config file"
+ :argname("<cfg>")
+ :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
+
+-- Extract subcommand
+local dump = parser:command "dump d"
+ :description "Dump bayes statistics"
+dump:mutex(
+ dump:flag "-j --json"
+ :description "Json output",
+ dump:flag "-C --cdb"
+ :description "CDB output"
+)
+dump:flag "-c --compress"
+ :description "Compress output"
+dump:option "-b --batch-size"
+ :description "Number of entires to process at once"
+ :argname("<elts>")
+ :convert(tonumber)
+ :default(1000)
+
+
+-- Restore
+local restore = parser:command "restore r"
+ :description "Restore bayes statistics"
+restore:argument "file"
+ :description "Input file to process"
+ :argname "<file>"
+ :args "*"
+restore:option "-b --batch-size"
+ :description "Number of entires to process at once"
+ :argname("<elts>")
+ :convert(tonumber)
+ :default(1000)
+restore:option "-m --mode"
+ :description "Number of entires to process at once"
+ :argname("<append|subtract|replace>")
+ :convert {
+ ['append'] = 'append',
+ ['subtract'] = 'subtract',
+ ['replace'] = 'replace',
+}
+ :default 'append'
+restore:flag "-n --no-operation"
+ :description "Only show redis commands to be issued"
+
+local function load_config(opts)
+ local _r, err = rspamd_config:load_ucl(opts['config'])
+
+ if not _r then
+ rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
+ if not _r then
+ rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
+ os.exit(1)
+ end
+end
+
+local function check_redis_classifier(cls, cfg)
+ -- Skip old classifiers
+ if cls.new_schema then
+ local symbol_spam, symbol_ham
+ -- Load symbols from statfiles
+
+ local function check_statfile_table(tbl, def_sym)
+ local symbol = tbl.symbol or def_sym
+
+ local spam
+ if tbl.spam then
+ spam = tbl.spam
+ else
+ if string.match(symbol:upper(), 'SPAM') then
+ spam = true
+ else
+ spam = false
+ end
+ end
+
+ if spam then
+ symbol_spam = symbol
+ else
+ symbol_ham = symbol
+ end
+ end
+
+ local statfiles = cls.statfile
+ if statfiles[1] then
+ for _, stf in ipairs(statfiles) do
+ if not stf.symbol then
+ for k, v in pairs(stf) do
+ check_statfile_table(v, k)
+ end
+ else
+ check_statfile_table(stf, 'undefined')
+ end
+ end
+ else
+ for stn, stf in pairs(statfiles) do
+ check_statfile_table(stf, stn)
+ end
+ end
+
+ local redis_params
+ redis_params = lua_redis.try_load_redis_servers(cls,
+ rspamd_config, false, 'bayes')
+ if not redis_params then
+ redis_params = lua_redis.try_load_redis_servers(cfg[N] or E,
+ rspamd_config, false, 'bayes')
+ if not redis_params then
+ redis_params = lua_redis.try_load_redis_servers(cfg[N] or E,
+ rspamd_config, true)
+ if not redis_params then
+ return false
+ end
+ end
+ end
+
+ table.insert(classifiers, {
+ symbol_spam = symbol_spam,
+ symbol_ham = symbol_ham,
+ redis_params = redis_params,
+ })
+ end
+end
+
+local function redis_map_zip(ar)
+ local data = {}
+ for j = 1, #ar, 2 do
+ data[ar[j]] = ar[j + 1]
+ end
+
+ return data
+end
+
+-- Used to clear tables
+local clear_fcn = table.clear or function(tbl)
+ local keys = lua_util.keys(tbl)
+ for _, k in ipairs(keys) do
+ tbl[k] = nil
+ end
+end
+
+local compress_ctx
+
+local function dump_out(out, opts, last)
+ if opts.compress and not compress_ctx then
+ compress_ctx = rspamd_zstd.compress_ctx()
+ end
+
+ if compress_ctx then
+ if last then
+ compress_ctx:stream(rspamd_text.fromtable(out), 'end'):write()
+ else
+ compress_ctx:stream(rspamd_text.fromtable(out), 'flush'):write()
+ end
+ else
+ for _, o in ipairs(out) do
+ io.write(o)
+ end
+ end
+end
+
+local function dump_cdb(out, opts, last, pattern)
+ local results = out[pattern]
+
+ if not out.cdb_builder then
+ -- First invocation
+ out.cdb_builder = rspamd_cdb.build(string.format('%s.cdb', pattern))
+ out.cdb_builder:add('_lrnspam', rspamd_i64.fromstring(results.learns_spam or '0'))
+ out.cdb_builder:add('_lrnham_', rspamd_i64.fromstring(results.learns_ham or '0'))
+ end
+
+ for _, o in ipairs(results.elts) do
+ out.cdb_builder:add(o.key, o.value)
+ end
+
+ if last then
+ out.cdb_builder:finalize()
+ out.cdb_builder = nil
+ end
+end
+
+local function dump_pattern(conn, pattern, opts, out, key)
+ local cursor = 0
+
+ repeat
+ conn:add_cmd('SCAN', { tostring(cursor),
+ 'MATCH', pattern,
+ 'COUNT', tostring(opts.batch_size) })
+ local ret, results = conn:exec()
+
+ if not ret then
+ rspamd_logger.errx("cannot connect execute scan command: %s", results)
+ os.exit(1)
+ end
+
+ cursor = tonumber(results[1])
+
+ local elts = results[2]
+ local tokens = {}
+
+ for _, e in ipairs(elts) do
+ conn:add_cmd('HGETALL', { e })
+ end
+ -- This function returns many results, each for each command
+ -- So if we have batch 1000, then we would have 1000 tables in form
+ -- [result, {hash_content}]
+ local all_results = { conn:exec() }
+
+ for i = 1, #all_results, 2 do
+ local r, hash_content = all_results[i], all_results[i + 1]
+
+ if r then
+ -- List to a hash map
+ local data = redis_map_zip(hash_content)
+ tokens[#tokens + 1] = { key = elts[(i + 1) / 2], data = data }
+ end
+ end
+
+ -- Output keeping track of the commas
+ for i, d in ipairs(tokens) do
+ if cursor == 0 and i == #tokens or not opts.json then
+ if opts.cdb then
+ table.insert(out[key].elts, {
+ key = rspamd_i64.fromstring(string.match(d.key, '%d+')),
+ value = rspamd_util.pack('ff', tonumber(d.data["S"] or '0') or 0,
+ tonumber(d.data["H"] or '0'))
+ })
+ else
+ out[#out + 1] = rspamd_logger.slog('"%s": %s\n', d.key,
+ ucl.to_format(d.data, "json-compact"))
+ end
+ else
+ out[#out + 1] = rspamd_logger.slog('"%s": %s,\n', d.key,
+ ucl.to_format(d.data, "json-compact"))
+ end
+
+ end
+
+ if opts.json and cursor == 0 then
+ out[#out + 1] = '}}\n'
+ end
+
+ -- Do not write the last chunk of out as it will be processed afterwards
+ if cursor ~= 0 then
+ if opts.cdb then
+ dump_out(out, opts, false)
+ clear_fcn(out)
+ else
+ dump_cdb(out, opts, false, key)
+ out[key].elts = {}
+ end
+ elseif opts.cdb then
+ dump_cdb(out, opts, true, key)
+ end
+
+ until cursor == 0
+end
+
+local function dump_handler(opts)
+ local patterns_seen = {}
+ for _, cls in ipairs(classifiers) do
+ local res, conn = lua_redis.redis_connect_sync(cls.redis_params, false)
+
+ if not res then
+ rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params)
+ os.exit(1)
+ end
+
+ local out = {}
+ local function check_keys(sym)
+ local sym_keys_pattern = string.format("%s_keys", sym)
+ conn:add_cmd('SMEMBERS', { sym_keys_pattern })
+ local ret, keys = conn:exec()
+
+ if not ret then
+ rspamd_logger.errx("cannot execute command to get keys: %s", keys)
+ os.exit(1)
+ end
+
+ if not opts.json then
+ out[#out + 1] = string.format('"%s": %s\n', sym_keys_pattern,
+ ucl.to_format(keys, 'json-compact'))
+ end
+ for _, k in ipairs(keys) do
+ local pat = string.format('%s*_*', k)
+ if not patterns_seen[pat] then
+ conn:add_cmd('HGETALL', { k })
+ local _ret, additional_keys = conn:exec()
+
+ if _ret then
+ if opts.json then
+ out[#out + 1] = string.format('{"pattern": "%s", "meta": %s, "elts": {\n',
+ k, ucl.to_format(redis_map_zip(additional_keys), 'json-compact'))
+ elseif opts.cdb then
+ out[k] = redis_map_zip(additional_keys)
+ out[k].elts = {}
+ else
+ out[#out + 1] = string.format('"%s": %s\n', k,
+ ucl.to_format(redis_map_zip(additional_keys), 'json-compact'))
+ end
+ dump_pattern(conn, pat, opts, out, k)
+ patterns_seen[pat] = true
+ end
+ end
+ end
+ end
+
+ check_keys(cls.symbol_spam)
+ check_keys(cls.symbol_ham)
+
+ if #out > 0 then
+ dump_out(out, opts, true)
+ end
+ end
+end
+
+local function obj_to_redis_arguments(obj, opts, cmd_pipe)
+ local key, value = next(obj)
+
+ if type(key) == 'string' then
+ if type(value) == 'table' then
+ if not value[1] then
+ if opts.mode == 'replace' then
+ local cmd = 'HMSET'
+ local params = { key }
+ for k, v in pairs(value) do
+ table.insert(params, k)
+ table.insert(params, v)
+ end
+ table.insert(cmd_pipe, { cmd, params })
+ else
+ local cmd = 'HINCRBYFLOAT'
+ local mult = 1.0
+ if opts.mode == 'subtract' then
+ mult = (-mult)
+ end
+
+ for k, v in pairs(value) do
+ if tonumber(v) then
+ v = tonumber(v)
+ table.insert(cmd_pipe, { cmd, { key, k, tostring(v * mult) } })
+ else
+ table.insert(cmd_pipe, { 'HSET', { key, k, v } })
+ end
+ end
+ end
+ else
+ -- Numeric table of elements (e.g. _keys) - it is actually a set in Redis
+ for _, elt in ipairs(value) do
+ table.insert(cmd_pipe, { 'SADD', { key, elt } })
+ end
+ end
+ end
+ end
+
+ return cmd_pipe
+end
+
+local function execute_batch(batch, conns, opts)
+ local cmd_pipe = {}
+
+ for _, cmd in ipairs(batch) do
+ obj_to_redis_arguments(cmd, opts, cmd_pipe)
+ end
+
+ if opts.no_operation then
+ for _, cmd in ipairs(cmd_pipe) do
+ rspamd_logger.messagex('%s %s', cmd[1], table.concat(cmd[2], ' '))
+ end
+ else
+ for _, conn in ipairs(conns) do
+ for _, cmd in ipairs(cmd_pipe) do
+ local is_ok, err = conn:add_cmd(cmd[1], cmd[2])
+
+ if not is_ok then
+ rspamd_logger.errx("cannot add command: %s with args: %s: %s", cmd[1], cmd[2], err)
+ end
+ end
+
+ conn:exec()
+ end
+ end
+end
+
+local function restore_handler(opts)
+ local files = opts.file or { '-' }
+ local conns = {}
+
+ for _, cls in ipairs(classifiers) do
+ local res, conn = lua_redis.redis_connect_sync(cls.redis_params, true)
+
+ if not res then
+ rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params)
+ os.exit(1)
+ end
+
+ table.insert(conns, conn)
+ end
+
+ local batch = {}
+
+ for _, f in ipairs(files) do
+ local fd
+ if f ~= '-' then
+ fd = io.open(f, 'r')
+ io.input(fd)
+ end
+
+ local cur_line = 1
+ for line in io.lines() do
+ local ucl_parser = ucl.parser()
+ local res, err
+ res, err = ucl_parser:parse_string(line)
+
+ if not res then
+ rspamd_logger.errx("%s: cannot read line %s: %s", f, cur_line, err)
+ os.exit(1)
+ end
+
+ table.insert(batch, ucl_parser:get_object())
+ cur_line = cur_line + 1
+
+ if cur_line % opts.batch_size == 0 then
+ execute_batch(batch, conns, opts)
+ batch = {}
+ end
+ end
+
+ if fd then
+ fd:close()
+ end
+ end
+
+ if #batch > 0 then
+ execute_batch(batch, conns, opts)
+ end
+end
+
+local function handler(args)
+ local opts = parser:parse(args)
+
+ local command = opts.command or 'dump'
+
+ load_config(opts)
+ rspamd_config:init_subsystem('stat')
+
+ local obj = rspamd_config:get_ucl()
+
+ local classifier = obj.classifier
+
+ if classifier then
+ if classifier[1] then
+ for _, cls in ipairs(classifier) do
+ if cls.bayes then
+ cls = cls.bayes
+ end
+ if cls.backend and cls.backend == 'redis' then
+ check_redis_classifier(cls, obj)
+ end
+ end
+ else
+ if classifier.bayes then
+
+ classifier = classifier.bayes
+ if classifier[1] then
+ for _, cls in ipairs(classifier) do
+ if cls.backend and cls.backend == 'redis' then
+ check_redis_classifier(cls, obj)
+ end
+ end
+ else
+ if classifier.backend and classifier.backend == 'redis' then
+ check_redis_classifier(classifier, obj)
+ end
+ end
+ end
+ end
+ end
+
+ if type(opts.file) == 'string' then
+ opts.file = { opts.file }
+ elseif type(opts.file) == 'none' then
+ opts.file = {}
+ end
+
+ if command == 'dump' then
+ dump_handler(opts)
+ elseif command == 'restore' then
+ restore_handler(opts)
+ else
+ parser:error('command %s is not implemented', command)
+ end
+end
+
+return {
+ name = 'statistics_dump',
+ aliases = { 'stat_dump', 'bayes_dump' },
+ handler = handler,
+ description = parser._description
+} \ No newline at end of file
diff --git a/lualib/rspamadm/template.lua b/lualib/rspamadm/template.lua
new file mode 100644
index 0000000..ca1779a
--- /dev/null
+++ b/lualib/rspamadm/template.lua
@@ -0,0 +1,131 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local argparse = require "argparse"
+
+
+-- Define command line options
+local parser = argparse()
+ :name "rspamadm template"
+ :description "Apply jinja templates for strings/files"
+ :help_description_margin(30)
+parser:argument "file"
+ :description "File to process"
+ :argname "<file>"
+ :args "*"
+
+parser:flag "-n --no-vars"
+ :description "Don't add Rspamd internal variables"
+parser:option "-e --env"
+ :description "Load additional environment vars from specific file (name=value)"
+ :argname "<filename>"
+ :count "*"
+parser:option "-l --lua-env"
+ :description "Load additional environment vars from specific file (lua source)"
+ :argname "<filename>"
+ :count "*"
+parser:mutex(
+ parser:option "-s --suffix"
+ :description "Store files with the new suffix"
+ :argname "<suffix>",
+ parser:flag "-i --inplace"
+ :description "Replace input file(s)"
+)
+
+local lua_util = require "lua_util"
+
+local function set_env(opts, env)
+ if opts.env then
+ for _, fname in ipairs(opts.env) do
+ for kv in assert(io.open(fname)):lines() do
+ if not kv:match('%s*#.*') then
+ local k, v = kv:match('([^=%s]+)%s*=%s*(.+)')
+
+ if k and v then
+ env[k] = v
+ else
+ io.write(string.format('invalid env line in %s: %s\n', fname, kv))
+ end
+ end
+ end
+ end
+ end
+
+ if opts.lua_env then
+ for _, fname in ipairs(opts.env) do
+ local ret, res_or_err = pcall(loadfile(fname))
+
+ if not ret then
+ io.write(string.format('cannot load %s: %s\n', fname, res_or_err))
+ else
+ if type(res_or_err) == 'table' then
+ for k, v in pairs(res_or_err) do
+ env[k] = lua_util.deepcopy(v)
+ end
+ else
+ io.write(string.format('cannot load %s: not a table\n', fname))
+ end
+ end
+ end
+ end
+end
+
+local function read_file(file)
+ local content
+ if file == '-' then
+ content = io.read("*all")
+ else
+ local f = assert(io.open(file, "rb"))
+ content = f:read("*all")
+ f:close()
+ end
+ return content
+end
+
+local function handler(args)
+ local opts = parser:parse(args)
+ local env = {}
+ set_env(opts, env)
+
+ if not opts.file or #opts.file == 0 then
+ opts.file = { '-' }
+ end
+ for _, fname in ipairs(opts.file) do
+ local content = read_file(fname)
+ local res = lua_util.jinja_template(content, env, opts.no_vars)
+
+ if opts.inplace then
+ local nfile = string.format('%s.new', fname)
+ local out = assert(io.open(nfile, 'w'))
+ out:write(content)
+ out:close()
+ os.rename(nfile, fname)
+ elseif opts.suffix then
+ local nfile = string.format('%s.%s', opts.suffix)
+ local out = assert(io.open(nfile, 'w'))
+ out:write(content)
+ out:close()
+ else
+ io.write(res)
+ end
+ end
+end
+
+return {
+ handler = handler,
+ description = parser._description,
+ name = 'template'
+} \ No newline at end of file
diff --git a/lualib/rspamadm/vault.lua b/lualib/rspamadm/vault.lua
new file mode 100644
index 0000000..840e504
--- /dev/null
+++ b/lualib/rspamadm/vault.lua
@@ -0,0 +1,579 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+
+local rspamd_logger = require "rspamd_logger"
+local ansicolors = require "ansicolors"
+local ucl = require "ucl"
+local argparse = require "argparse"
+local fun = require "fun"
+local rspamd_http = require "rspamd_http"
+local cr = require "rspamd_cryptobox"
+
+local parser = argparse()
+ :name "rspamadm vault"
+ :description "Perform Hashicorp Vault management"
+ :help_description_margin(32)
+ :command_target("command")
+ :require_command(true)
+
+parser:flag "-s --silent"
+ :description "Do not output extra information"
+parser:option "-a --addr"
+ :description "Vault address (if not defined in VAULT_ADDR env)"
+parser:option "-t --token"
+ :description "Vault token (not recommended, better define VAULT_TOKEN env)"
+parser:option "-p --path"
+ :description "Path to work with in the vault"
+ :default "dkim"
+parser:option "-o --output"
+ :description "Output format ('ucl', 'json', 'json-compact', 'yaml')"
+ :argname("<type>")
+ :convert {
+ ucl = "ucl",
+ json = "json",
+ ['json-compact'] = "json-compact",
+ yaml = "yaml",
+}
+ :default "ucl"
+
+parser:command "list ls l"
+ :description "List elements in the vault"
+
+local show = parser:command "show get"
+ :description "Extract element from the vault"
+show:argument "domain"
+ :description "Domain to create key for"
+ :args "+"
+
+local delete = parser:command "delete del rm remove"
+ :description "Delete element from the vault"
+delete:argument "domain"
+ :description "Domain to create delete key(s) for"
+ :args "+"
+
+local newkey = parser:command "newkey new create"
+ :description "Add new key to the vault"
+newkey:argument "domain"
+ :description "Domain to create key for"
+ :args "+"
+newkey:option "-s --selector"
+ :description "Selector to use"
+ :count "?"
+newkey:option "-A --algorithm"
+ :argname("<type>")
+ :convert {
+ rsa = "rsa",
+ ed25519 = "ed25519",
+ eddsa = "ed25519",
+}
+ :default "rsa"
+newkey:option "-b --bits"
+ :argname("<nbits>")
+ :convert(tonumber)
+ :default "1024"
+newkey:option "-x --expire"
+ :argname("<days>")
+ :convert(tonumber)
+newkey:flag "-r --rewrite"
+
+local roll = parser:command "roll rollover"
+ :description "Perform keys rollover"
+roll:argument "domain"
+ :description "Domain to roll key(s) for"
+ :args "+"
+roll:option "-T --ttl"
+ :description "Validity period for old keys (days)"
+ :convert(tonumber)
+ :default "1"
+roll:flag "-r --remove-expired"
+ :description "Remove expired keys"
+roll:option "-x --expire"
+ :argname("<days>")
+ :convert(tonumber)
+
+local function printf(fmt, ...)
+ if fmt then
+ io.write(rspamd_logger.slog(fmt, ...))
+ end
+ io.write('\n')
+end
+
+local function maybe_printf(opts, fmt, ...)
+ if not opts.silent then
+ printf(fmt, ...)
+ end
+end
+
+local function highlight(str, color)
+ return ansicolors[color or 'white'] .. str .. ansicolors.reset
+end
+
+local function vault_url(opts, path)
+ if path then
+ return string.format('%s/v1/%s/%s', opts.addr, opts.path, path)
+ end
+
+ return string.format('%s/v1/%s', opts.addr, opts.path)
+end
+
+local function is_http_error(err, data)
+ return err or (math.floor(data.code / 100) ~= 2)
+end
+
+local function parse_vault_reply(data)
+ local p = ucl.parser()
+ local res, parser_err = p:parse_string(data)
+
+ if not res then
+ return nil, parser_err
+ else
+ return p:get_object(), nil
+ end
+end
+
+local function maybe_print_vault_data(opts, data, func)
+ if data then
+ local res, parser_err = parse_vault_reply(data)
+
+ if not res then
+ printf('vault reply for cannot be parsed: %s', parser_err)
+ else
+ if func then
+ printf(ucl.to_format(func(res), opts.output))
+ else
+ printf(ucl.to_format(res, opts.output))
+ end
+ end
+ else
+ printf('no data received')
+ end
+end
+
+local function print_dkim_txt_record(b64, selector, alg)
+ local labels = {}
+ local prefix = string.format("v=DKIM1; k=%s; p=", alg)
+ b64 = prefix .. b64
+ if #b64 < 255 then
+ labels = { '"' .. b64 .. '"' }
+ else
+ for sl = 1, #b64, 256 do
+ table.insert(labels, '"' .. b64:sub(sl, sl + 255) .. '"')
+ end
+ end
+
+ printf("%s._domainkey IN TXT ( %s )", selector,
+ table.concat(labels, "\n\t"))
+end
+
+local function show_handler(opts, domain)
+ local uri = vault_url(opts, domain)
+ local err, data = rspamd_http.request {
+ config = rspamd_config,
+ ev_base = rspamadm_ev_base,
+ session = rspamadm_session,
+ resolver = rspamadm_dns_resolver,
+ url = uri,
+ headers = {
+ ['X-Vault-Token'] = opts.token
+ }
+ }
+
+ if is_http_error(err, data) then
+ printf('cannot get request to the vault (%s), HTTP error code %s', uri, data.code)
+ maybe_print_vault_data(opts, err)
+ os.exit(1)
+ else
+ maybe_print_vault_data(opts, data.content, function(obj)
+ return obj.data.selectors
+ end)
+ end
+end
+
+local function delete_handler(opts, domain)
+ local uri = vault_url(opts, domain)
+ local err, data = rspamd_http.request {
+ config = rspamd_config,
+ ev_base = rspamadm_ev_base,
+ session = rspamadm_session,
+ resolver = rspamadm_dns_resolver,
+ url = uri,
+ method = 'delete',
+ headers = {
+ ['X-Vault-Token'] = opts.token
+ }
+ }
+
+ if is_http_error(err, data) then
+ printf('cannot get request to the vault (%s), HTTP error code %s', uri, data.code)
+ maybe_print_vault_data(opts, err)
+ os.exit(1)
+ else
+ printf('deleted key(s) for %s', domain)
+ end
+end
+
+local function list_handler(opts)
+ local uri = vault_url(opts)
+ local err, data = rspamd_http.request {
+ config = rspamd_config,
+ ev_base = rspamadm_ev_base,
+ session = rspamadm_session,
+ resolver = rspamadm_dns_resolver,
+ url = uri .. '?list=true',
+ headers = {
+ ['X-Vault-Token'] = opts.token
+ }
+ }
+
+ if is_http_error(err, data) then
+ printf('cannot get request to the vault (%s), HTTP error code %s', uri, data.code)
+ maybe_print_vault_data(opts, err)
+ os.exit(1)
+ else
+ maybe_print_vault_data(opts, data.content, function(obj)
+ return obj.data.keys
+ end)
+ end
+end
+
+-- Returns pair privkey+pubkey
+local function genkey(opts)
+ return cr.gen_dkim_keypair(opts.algorithm, opts.bits)
+end
+
+local function create_and_push_key(opts, domain, existing)
+ local uri = vault_url(opts, domain)
+ local sk, pk = genkey(opts)
+
+ local res = {
+ selectors = {
+ [1] = {
+ selector = opts.selector,
+ domain = domain,
+ key = tostring(sk),
+ pubkey = tostring(pk),
+ alg = opts.algorithm,
+ bits = opts.bits or 0,
+ valid_start = os.time(),
+ }
+ }
+ }
+
+ for _, sel in ipairs(existing) do
+ res.selectors[#res.selectors + 1] = sel
+ end
+
+ if opts.expire then
+ res.selectors[1].valid_end = os.time() + opts.expire * 3600 * 24
+ end
+
+ local err, data = rspamd_http.request {
+ config = rspamd_config,
+ ev_base = rspamadm_ev_base,
+ session = rspamadm_session,
+ resolver = rspamadm_dns_resolver,
+ url = uri,
+ method = 'put',
+ headers = {
+ ['Content-Type'] = 'application/json',
+ ['X-Vault-Token'] = opts.token
+ },
+ body = {
+ ucl.to_format(res, 'json-compact')
+ },
+ }
+
+ if is_http_error(err, data) then
+ printf('cannot get request to the vault (%s), HTTP error code %s', uri, data.code)
+ maybe_print_vault_data(opts, data.content)
+ os.exit(1)
+ else
+ maybe_printf(opts, 'stored key for: %s, selector: %s', domain, opts.selector)
+ maybe_printf(opts, 'please place the corresponding public key as following:')
+
+ if opts.silent then
+ printf('%s', pk)
+ else
+ print_dkim_txt_record(tostring(pk), opts.selector, opts.algorithm)
+ end
+ end
+end
+
+local function newkey_handler(opts, domain)
+ local uri = vault_url(opts, domain)
+
+ if not opts.selector then
+ opts.selector = string.format('%s-%s', opts.algorithm,
+ os.date("!%Y%m%d"))
+ end
+
+ local err, data = rspamd_http.request {
+ config = rspamd_config,
+ ev_base = rspamadm_ev_base,
+ session = rspamadm_session,
+ resolver = rspamadm_dns_resolver,
+ url = uri,
+ method = 'get',
+ headers = {
+ ['X-Vault-Token'] = opts.token
+ }
+ }
+
+ if is_http_error(err, data) or not data.content then
+ create_and_push_key(opts, domain, {})
+ else
+ -- Key exists
+ local rep = parse_vault_reply(data.content)
+
+ if not rep or not rep.data then
+ printf('cannot parse reply for %s: %s', uri, data.content)
+ os.exit(1)
+ end
+
+ local elts = rep.data.selectors
+
+ if not elts then
+ create_and_push_key(opts, domain, {})
+ os.exit(0)
+ end
+
+ for _, sel in ipairs(elts) do
+ if sel.alg == opts.algorithm then
+ printf('key with the specific algorithm %s is already presented at %s selector for %s domain',
+ opts.algorithm, sel.selector, domain)
+ os.exit(1)
+ else
+ create_and_push_key(opts, domain, elts)
+ end
+ end
+ end
+end
+
+local function roll_handler(opts, domain)
+ local uri = vault_url(opts, domain)
+ local res = {
+ selectors = {}
+ }
+
+ local err, data = rspamd_http.request {
+ config = rspamd_config,
+ ev_base = rspamadm_ev_base,
+ session = rspamadm_session,
+ resolver = rspamadm_dns_resolver,
+ url = uri,
+ method = 'get',
+ headers = {
+ ['X-Vault-Token'] = opts.token
+ }
+ }
+
+ if is_http_error(err, data) or not data.content then
+ printf("No keys to roll for domain %s", domain)
+ os.exit(1)
+ else
+ local rep = parse_vault_reply(data.content)
+
+ if not rep or not rep.data then
+ printf('cannot parse reply for %s: %s', uri, data.content)
+ os.exit(1)
+ end
+
+ local elts = rep.data.selectors
+
+ if not elts then
+ printf("No keys to roll for domain %s", domain)
+ os.exit(1)
+ end
+
+ local nkeys = {} -- indexed by algorithm
+
+ local function insert_key(sel, add_expire)
+ if not nkeys[sel.alg] then
+ nkeys[sel.alg] = {}
+ end
+
+ if add_expire then
+ sel.valid_end = os.time() + opts.ttl * 3600 * 24
+ end
+
+ table.insert(nkeys[sel.alg], sel)
+ end
+
+ for _, sel in ipairs(elts) do
+ if sel.valid_end and sel.valid_end < os.time() then
+ if not opts.remove_expired then
+ insert_key(sel, false)
+ else
+ maybe_printf(opts, 'removed expired key for %s (selector %s, expire "%s"',
+ domain, sel.selector, os.date('%c', sel.valid_end))
+ end
+ else
+ insert_key(sel, true)
+ end
+ end
+
+ -- Now we need to ensure that all but one selectors have either expired or just a single key
+ for alg, keys in pairs(nkeys) do
+ table.sort(keys, function(k1, k2)
+ if k1.valid_end and k2.valid_end then
+ return k1.valid_end > k2.valid_end
+ elseif k1.valid_end then
+ return true
+ elseif k2.valid_end then
+ return false
+ end
+ return false
+ end)
+ -- Exclude the key with the highest expiration date and examine the rest
+ if not (#keys == 1 or fun.all(function(k)
+ return k.valid_end and k.valid_end < os.time()
+ end, fun.tail(keys))) then
+ printf('bad keys list for %s and %s algorithm', domain, alg)
+ fun.each(function(k)
+ if not k.valid_end then
+ printf('selector %s, algorithm %s has a key with no expire',
+ k.selector, k.alg)
+ elseif k.valid_end >= os.time() then
+ printf('selector %s, algorithm %s has a key that not yet expired: %s',
+ k.selector, k.alg, os.date('%c', k.valid_end))
+ end
+ end, fun.tail(keys))
+ os.exit(1)
+ end
+ -- Do not create new keys, if we only want to remove expired keys
+ if not opts.remove_expired then
+ -- OK to process
+ -- Insert keys for each algorithm in pairs <old_key(s)>, <new_key>
+ local sk, pk = genkey({ algorithm = alg, bits = keys[1].bits })
+ local selector = string.format('%s-%s', alg,
+ os.date("!%Y%m%d"))
+
+ if selector == keys[1].selector then
+ selector = selector .. '-1'
+ end
+ local nelt = {
+ selector = selector,
+ domain = domain,
+ key = tostring(sk),
+ pubkey = tostring(pk),
+ alg = alg,
+ bits = keys[1].bits,
+ valid_start = os.time(),
+ }
+
+ if opts.expire then
+ nelt.valid_end = os.time() + opts.expire * 3600 * 24
+ end
+
+ table.insert(res.selectors, nelt)
+ end
+ for _, k in ipairs(keys) do
+ table.insert(res.selectors, k)
+ end
+ end
+ end
+
+ -- We can now store res in the vault
+ err, data = rspamd_http.request {
+ config = rspamd_config,
+ ev_base = rspamadm_ev_base,
+ session = rspamadm_session,
+ resolver = rspamadm_dns_resolver,
+ url = uri,
+ method = 'put',
+ headers = {
+ ['Content-Type'] = 'application/json',
+ ['X-Vault-Token'] = opts.token
+ },
+ body = {
+ ucl.to_format(res, 'json-compact')
+ },
+ }
+
+ if is_http_error(err, data) then
+ printf('cannot put request to the vault (%s), HTTP error code %s', uri, data.code)
+ maybe_print_vault_data(opts, data.content)
+ os.exit(1)
+ else
+ for _, key in ipairs(res.selectors) do
+ if not key.valid_end or key.valid_end > os.time() + opts.ttl * 3600 * 24 then
+ maybe_printf(opts, 'rolled key for: %s, new selector: %s', domain, key.selector)
+ maybe_printf(opts, 'please place the corresponding public key as following:')
+
+ if opts.silent then
+ printf('%s', key.pubkey)
+ else
+ print_dkim_txt_record(key.pubkey, key.selector, key.alg)
+ end
+
+ end
+ end
+
+ maybe_printf(opts, 'your old keys will be valid until %s',
+ os.date('%c', os.time() + opts.ttl * 3600 * 24))
+ end
+end
+
+local function handler(args)
+ local opts = parser:parse(args)
+
+ if not opts.addr then
+ opts.addr = os.getenv('VAULT_ADDR')
+ end
+
+ if not opts.token then
+ opts.token = os.getenv('VAULT_TOKEN')
+ else
+ maybe_printf(opts, 'defining token via command line is insecure, define it via environment variable %s',
+ highlight('VAULT_TOKEN', 'red'))
+ end
+
+ if not opts.token or not opts.addr then
+ printf('no token or/and vault addr has been specified, exiting')
+ os.exit(1)
+ end
+
+ local command = opts.command
+
+ if command == 'list' then
+ list_handler(opts)
+ elseif command == 'show' then
+ fun.each(function(d)
+ show_handler(opts, d)
+ end, opts.domain)
+ elseif command == 'newkey' then
+ fun.each(function(d)
+ newkey_handler(opts, d)
+ end, opts.domain)
+ elseif command == 'roll' then
+ fun.each(function(d)
+ roll_handler(opts, d)
+ end, opts.domain)
+ elseif command == 'delete' then
+ fun.each(function(d)
+ delete_handler(opts, d)
+ end, opts.domain)
+ else
+ parser:error(string.format('command %s is not implemented', command))
+ end
+end
+
+return {
+ handler = handler,
+ description = parser._description,
+ name = 'vault'
+}
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..dd1bca3
--- /dev/null
+++ b/package.json
@@ -0,0 +1,14 @@
+{
+ "devDependencies": {
+ "eslint": "*",
+ "@stylistic/eslint-plugin": "*",
+ "stylelint": ">=13.6.0",
+ "stylelint-config-standard": "*",
+ "postcss-html": "*"
+ },
+ "eslintIgnore": [
+ "*.min.js",
+ "index.html",
+ "prism.js"
+ ]
+}
diff --git a/rpm/80-rspamd.preset b/rpm/80-rspamd.preset
new file mode 100644
index 0000000..0fcf159
--- /dev/null
+++ b/rpm/80-rspamd.preset
@@ -0,0 +1 @@
+enable rspamd.service
diff --git a/rpm/rspamd.logrotate b/rpm/rspamd.logrotate
new file mode 100644
index 0000000..ea1b02d
--- /dev/null
+++ b/rpm/rspamd.logrotate
@@ -0,0 +1,11 @@
+/var/log/rspamd/*log {
+ daily
+ rotate 10
+ missingok
+ notifempty
+ compress
+ sharedscripts
+ postrotate
+ systemctl --signal=USR1 --kill-who=main kill rspamd.service 2>/dev/null || :
+ endscript
+}
diff --git a/rpm/rspamd.spec b/rpm/rspamd.spec
new file mode 100644
index 0000000..7a1a15c
--- /dev/null
+++ b/rpm/rspamd.spec
@@ -0,0 +1,223 @@
+%if 0%{getenv:ASAN}
+Name: rspamd-asan
+Conflicts: rspamd
+%else
+Name: rspamd
+Conflicts: rspamd-asan
+%endif
+Provides: rspamd
+Version: 3.2
+Release: 1
+Summary: Rapid spam filtering system
+Group: System Environment/Daemons
+License: Apache-2.0
+URL: https://rspamd.com
+Source0: https://github.com/rspamd/rspamd/archive/%{version}/rspamd-%{version}.tar.gz
+Source1: rspamd.logrotate
+Source2: 80-rspamd.preset
+BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}
+%if 0%{?el7}
+BuildRequires: cmake3
+BuildRequires: devtoolset-10-gcc-c++
+%else
+BuildRequires: cmake
+%if 0%{?el8}
+BuildRequires: gcc-toolset-10-gcc-c++
+%endif
+%if 0%{?el9}
+BuildRequires: gcc-toolset-12-gcc-c++
+%endif
+%endif
+BuildRequires: file-devel
+BuildRequires: glib2-devel
+BuildRequires: lapack-devel
+BuildRequires: libicu-devel
+BuildRequires: libsodium-devel
+BuildRequires: libunwind-devel
+%if 0%{getenv:ASAN}
+%if 0%{?el7}
+BuildRequires: devtoolset-10-libasan-devel
+%else
+%if 0%{?el8}
+BuildRequires: gcc-toolset-10-libasan-devel
+%endif
+%if 0%{?el9}
+BuildRequires: gcc-toolset-12-libasan-devel
+%endif
+%endif
+%endif
+
+%ifarch x86_64 amd64
+%if 0%{?el8} || 0%{?fedora} > 10
+BuildRequires: hyperscan-devel
+%endif
+BuildRequires: jemalloc-devel
+%endif
+
+%if 0%{getenv:LUAJIT}
+BuildRequires: git
+%else
+BuildRequires: lua-devel
+%endif
+BuildRequires: openblas-devel
+BuildRequires: openssl-devel
+BuildRequires: pcre2-devel
+BuildRequires: ragel
+BuildRequires: sqlite-devel
+BuildRequires: systemd
+BuildRequires: binutils-devel
+Requires(pre): shadow-utils
+Requires(post): systemd
+Requires(preun): systemd
+Requires(postun): systemd
+
+%description
+Rspamd is a rapid, modular and lightweight spam filter. It is designed to work
+with big amount of mail and can be easily extended with own filters written in
+lua.
+
+%prep
+%setup -q -n %{name}-%{version}
+%if 0%{getenv:LUAJIT}
+rm -fr %{_builddir}/luajit-src || true
+rm -fr %{_builddir}/luajit-build || true
+git clone -b v2.1 https://luajit.org/git/luajit-2.0.git %{_builddir}/luajit-src
+%endif
+
+%build
+%if 0%{?el7}
+source /opt/rh/devtoolset-10/enable
+%else
+%if 0%{?el8}
+source /opt/rh/gcc-toolset-10/enable
+%endif
+%if 0%{?el9}
+source /opt/rh/gcc-toolset-12/enable
+%endif
+%endif
+
+%if 0%{getenv:LUAJIT}
+pushd %{_builddir}/luajit-src && make clean && make %{?_smp_mflags} CC="gcc -fPIC" PREFIX=%{_builddir}/luajit-build && make install PREFIX=%{_builddir}/luajit-build ; popd
+rm -f %{_builddir}/luajit-build/lib/*.so || true
+%endif
+%if 0%{?el7}
+%{cmake3} \
+%else
+%{cmake} \
+%endif
+ -B . \
+%if 0%{getenv:ASAN}
+ -DCMAKE_BUILD_TYPE=Debug \
+ -DSANITIZE=address \
+%else
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+ -DENABLE_LTO=ON \
+%endif
+ -DCMAKE_C_FLAGS_RELEASE="%{optflags}" \
+ -DCMAKE_CXX_FLAGS_RELEASE="%{optflags}" \
+%if 0%{?fedora} >= 36
+ -DLINKER_NAME=/usr/bin/ld.bfd \
+%endif
+%if 0%{?el8}
+ -DLINKER_NAME=ld.bfd \
+%endif
+ -DCMAKE_INSTALL_PREFIX=%{_prefix} \
+ -DCONFDIR=%{_sysconfdir}/rspamd \
+ -DMANDIR=%{_mandir} \
+ -DDBDIR=%{_localstatedir}/lib/rspamd \
+ -DRUNDIR=%{_localstatedir}/run/rspamd \
+ -DLOGDIR=%{_localstatedir}/log/rspamd \
+ -DEXAMPLESDIR=%{_datadir}/examples/rspamd \
+ -DSHAREDIR=%{_datadir}/rspamd \
+ -DLIBDIR=%{_libdir}/rspamd/ \
+ -DINCLUDEDIR=%{_includedir} \
+ -DRSPAMD_GROUP=_rspamd \
+ -DRSPAMD_USER=_rspamd \
+ -DSYSTEMDDIR=%{_unitdir} \
+ -DWANT_SYSTEMD_UNITS=ON \
+ -DNO_SHARED=ON \
+ -DDEBIAN_BUILD=1 \
+%ifarch x86_64 amd64 arm64 aarch64
+ -DENABLE_HYPERSCAN=ON \
+%endif
+%ifarch arm64 aarch64
+ -DHYPERSCAN_ROOT_DIR=/vectorscan \
+%endif
+%ifarch x86_64 amd64
+%if 0%{?el7}
+ -DHYPERSCAN_ROOT_DIR=/vectorscan \
+%endif
+%endif
+%ifarch x86_64 amd64
+ -DENABLE_JEMALLOC=ON \
+%endif
+%if 0%{getenv:LUAJIT}
+ -DENABLE_LUAJIT=ON \
+ -DLUA_ROOT=%{_builddir}/luajit-build \
+%else
+ -DENABLE_LUAJIT=OFF \
+%endif
+ -DENABLE_FASTTEXT=ON \
+ -DFASTTEXT_ROOT_DIR=/fasttext \
+ -DENABLE_BLAS=ON
+make %{?_smp_mflags}
+
+%install
+%make_install
+%{__install} -p -D -m 0644 %{SOURCE1} %{buildroot}%{_sysconfdir}/logrotate.d/rspamd
+%{__install} -p -D -m 0644 %{SOURCE2} %{buildroot}%{_presetdir}/80-rspamd.preset
+%{__install} -d -p -m 0755 %{buildroot}%{_localstatedir}/log/rspamd
+%{__install} -d -p -m 0755 %{buildroot}%{_localstatedir}/lib/rspamd
+%{__install} -p -D -d -m 0755 %{buildroot}%{_sysconfdir}/rspamd/local.d/
+%{__install} -p -D -d -m 0755 %{buildroot}%{_sysconfdir}/rspamd/override.d/
+
+%clean
+rm -rf %{buildroot}
+
+%pre
+%{_sbindir}/groupadd -r _rspamd 2>/dev/null || :
+%{_sbindir}/useradd -g _rspamd -c "Rspamd user" -s /bin/false -r -d %{_localstatedir}/lib/rspamd _rspamd 2>/dev/null || :
+
+%post
+%{__chown} -R _rspamd:_rspamd %{_localstatedir}/lib/rspamd
+%{__chown} _rspamd:_rspamd %{_localstatedir}/log/rspamd
+%if 0%{?el7}
+# We need to clean old hyperscan files on upgrade: see https://github.com/rspamd/rspamd/issues/4441
+rm -f %{_localstatedir}/lib/rspamd/*.hs*
+%endif
+systemctl --no-reload preset rspamd.service >/dev/null 2>&1 || :
+
+%preun
+%systemd_preun rspamd.service
+
+%postun
+%systemd_postun_with_restart rspamd.service
+
+%files
+%defattr(-,root,root,-)
+
+%dir %{_sysconfdir}/rspamd
+%config(noreplace) %{_sysconfdir}/rspamd/*
+
+%{_bindir}/rspamd
+%{_bindir}/rspamd_stats
+%{_bindir}/rspamc
+%{_bindir}/rspamadm
+
+%{_unitdir}/rspamd.service
+%{_presetdir}/80-rspamd.preset
+
+%dir %{_libdir}/rspamd
+%{_libdir}/rspamd/*
+
+%{_mandir}/man8/rspamd.*
+%{_mandir}/man1/rspamc.*
+%{_mandir}/man1/rspamadm.*
+
+%dir %{_datadir}/rspamd
+%{_datadir}/rspamd/*
+
+%config(noreplace) %{_sysconfdir}/logrotate.d/rspamd
+
+%attr(-, _rspamd, _rspamd) %dir %{_localstatedir}/lib/rspamd
+%dir %{_localstatedir}/log/rspamd
diff --git a/rspamd.service b/rspamd.service
new file mode 100644
index 0000000..1d11808
--- /dev/null
+++ b/rspamd.service
@@ -0,0 +1,17 @@
+[Unit]
+Description=rapid spam filtering system
+After=nss-lookup.target network-online.target
+Documentation=https://rspamd.com/doc/
+
+[Service]
+LimitNOFILE=1048576
+NonBlocking=true
+ExecStart=/usr/bin/rspamd -c /etc/rspamd/rspamd.conf -f
+ExecReload=/bin/kill -HUP $MAINPID
+User=_rspamd
+RuntimeDirectory=rspamd
+RuntimeDirectoryMode=0755
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
diff --git a/rules/bitcoin.lua b/rules/bitcoin.lua
new file mode 100644
index 0000000..6a70721
--- /dev/null
+++ b/rules/bitcoin.lua
@@ -0,0 +1,237 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Bitcoin filter rules
+
+local fun = require "fun"
+local bit = require "bit"
+local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local N = "bitcoin"
+
+local off = 0
+local base58_dec = fun.tomap(fun.map(
+ function(c)
+ off = off + 1
+ return c, (off - 1)
+ end,
+ "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"))
+
+local function is_traditional_btc_address(word)
+ local hash = require "rspamd_cryptobox_hash"
+
+ local bytes = {}
+ for i = 1, 25 do
+ bytes[i] = 0
+ end
+ -- Base58 decode loop
+ fun.each(function(ch)
+ local acc = base58_dec[ch] or 0
+ for i = 25, 1, -1 do
+ acc = acc + (58 * bytes[i]);
+ bytes[i] = acc % 256
+ acc = math.floor(acc / 256);
+ end
+ end, word)
+ -- Now create a validation tag
+ local sha256 = hash.create_specific('sha256')
+ for i = 1, 21 do
+ sha256:update(string.char(bytes[i]))
+ end
+ sha256 = hash.create_specific('sha256', sha256:bin()):bin()
+
+ -- Compare tags
+ local valid = true
+ for i = 1, 4 do
+ if string.sub(sha256, i, i) ~= string.char(bytes[21 + i]) then
+ valid = false
+ end
+ end
+
+ return valid
+end
+
+-- Beach32 checksum combiner
+local function polymod(...)
+ local chk = 1;
+ local gen = { 0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3 };
+ for _, t in ipairs({ ... }) do
+ for _, v in ipairs(t) do
+ local top = bit.rshift(chk, 25)
+
+ chk = bit.bxor(bit.lshift(bit.band(chk, 0x1ffffff), 5), v)
+ for i = 1, 5 do
+ if bit.band(bit.rshift(top, i - 1), 0x1) ~= 0 then
+ chk = bit.bxor(chk, gen[i])
+ end
+ end
+ end
+ end
+
+ return chk
+end
+
+-- Beach32 expansion function
+local function hrpExpand(hrp)
+ local ret = {}
+ fun.each(function(byte)
+ ret[#ret + 1] = bit.rshift(byte, 5)
+ end, fun.map(string.byte, fun.iter(hrp)))
+ ret[#ret + 1] = 0
+ fun.each(function(byte)
+ ret[#ret + 1] = bit.band(byte, 0x1f)
+ end, fun.map(string.byte, fun.iter(hrp)))
+
+ return ret
+end
+
+local function verify_beach32_cksum(hrp, elts)
+ return polymod(hrpExpand(hrp), elts) == 1
+end
+
+local function gen_bleach32_table(input)
+ local d = {}
+ local i = 1
+ local res = true
+ local charset = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l'
+
+ fun.each(function(byte)
+ if res then
+ local pos = charset:find(byte, 1, true)
+ if not pos then
+ res = false
+ else
+ d[i] = pos - 1
+ i = i + 1
+ end
+ end
+ end, fun.iter(input))
+
+ return res and d or nil
+end
+
+local function is_segwit_bech32_address(task, word)
+ local semicolon_pos = string.find(word, ':')
+ local address_part = word
+ if semicolon_pos then
+ address_part = string.sub(word, semicolon_pos + 1)
+ end
+
+ local prefix = address_part:sub(1, 3)
+
+ if prefix == 'bc1' or prefix:sub(1, 1) == '1' or prefix:sub(1, 1) == '3' then
+ -- Strip beach32 prefix in bitcoin
+ address_part = address_part:lower()
+ local last_one_pos = address_part:find('1[^1]*$')
+ if not last_one_pos or (last_one_pos < 1 or last_one_pos + 7 > #address_part) then
+ return false
+ end
+ local hrp = address_part:sub(1, last_one_pos - 1)
+ local addr = address_part:sub(last_one_pos + 1, -1)
+ local decoded = gen_bleach32_table(addr)
+
+ if decoded then
+ return verify_beach32_cksum(hrp, decoded)
+ end
+ else
+ -- Bitcoin cash address
+ -- https://www.bitcoincash.org/spec/cashaddr.html
+ local decoded = gen_bleach32_table(address_part)
+ lua_util.debugm(N, task, 'check %s, %s decoded', word, decoded)
+
+ if decoded and #decoded > 8 then
+ if semicolon_pos then
+ prefix = word:sub(1, semicolon_pos - 1)
+ else
+ prefix = 'bitcoincash'
+ end
+
+ local polymod_tbl = {}
+ fun.each(function(byte)
+ local b = bit.band(string.byte(byte), 0x1f)
+ table.insert(polymod_tbl, b)
+ end, fun.iter(prefix))
+
+ -- For semicolon
+ table.insert(polymod_tbl, 0)
+
+ fun.each(function(byte)
+ table.insert(polymod_tbl, byte)
+ end, decoded)
+ lua_util.debugm(N, task, 'final polymod table: %s', polymod_tbl)
+
+ return rspamd_util.btc_polymod(polymod_tbl)
+ end
+ end
+end
+
+local normal_wallet_re = [[/\b[13LM][1-9A-Za-z]{25,34}\b/AL{sa_body}]]
+local btc_bleach_re = [[/\b(?:(?:[a-zA-Z]\w+:)|(?:bc1))?[qpzry9x8gf2tvdw0s3jn54khce6mua7l]{14,}\b/AL{sa_body}]]
+
+config.regexp['BITCOIN_ADDR'] = {
+ description = 'Message has a valid bitcoin wallet address',
+ -- Use + operator to ensure that each expression is always evaluated
+ re = string.format('(%s) + (%s) > 0', normal_wallet_re, btc_bleach_re),
+ re_conditions = {
+ [normal_wallet_re] = function(task, txt, s, e)
+ local len = e - s
+ if len <= 2 or len > 1024 then
+ return false
+ end
+
+ local word = lua_util.str_trim(txt:sub(s + 1, e))
+ local valid = is_traditional_btc_address(word)
+
+ if valid then
+ -- To save option
+ task:insert_result('BITCOIN_ADDR', 1.0, word)
+ lua_util.debugm(N, task, 'found valid traditional bitcoin addr in the word: %s',
+ word)
+ return true
+ else
+ lua_util.debugm(N, task, 'found invalid bitcoin addr in the word: %s',
+ word)
+
+ return false
+ end
+ end,
+ [btc_bleach_re] = function(task, txt, s, e)
+ local len = e - s
+ if len <= 2 or len > 1024 then
+ return false
+ end
+
+ local word = tostring(lua_util.str_trim(txt:sub(s + 1, e)))
+ local valid = is_segwit_bech32_address(task, word)
+
+ if valid then
+ -- To save option
+ task:insert_result('BITCOIN_ADDR', 1.0, word)
+ lua_util.debugm(N, task, 'found valid bleach bitcoin addr in the word: %s',
+ word)
+ return true
+ else
+ lua_util.debugm(N, task, 'found invalid bitcoin addr in the word: %s',
+ word)
+
+ return false
+ end
+ end,
+ },
+ score = 0.0,
+ one_shot = true,
+ group = 'scams',
+}
diff --git a/rules/bounce.lua b/rules/bounce.lua
new file mode 100644
index 0000000..fb74b97
--- /dev/null
+++ b/rules/bounce.lua
@@ -0,0 +1,117 @@
+--[[
+Copyright (c) 2020, Anton Yuzhaninov <citrin@citrin.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Rule to detect bounces:
+-- RFC 3464 Delivery status notifications and most common non-standard ones
+
+local function make_subj_bounce_keywords_re()
+ -- Words and phrases commonly used in Subjects for bounces
+ -- We cannot practically test all localized Subjects, but luckily English is by far the most common here
+ local keywords = {
+ 'could not send message',
+ "couldn't be delivered",
+ 'delivery failed',
+ 'delivery failure',
+ 'delivery report',
+ 'delivery status',
+ 'delivery warning',
+ 'failure delivery',
+ 'failure notice',
+ "hasn't been delivered",
+ 'mail failure',
+ 'returned mail',
+ 'undeliverable',
+ 'undelivered',
+ }
+ return string.format([[Subject=/\b(%s)\b/i{header}]], table.concat(keywords, '|'))
+end
+
+config.regexp.SUBJ_BOUNCE_WORDS = {
+ re = make_subj_bounce_keywords_re(),
+ group = 'headers',
+ score = 0.0,
+ description = 'Words/phrases typical for DSN'
+}
+
+rspamd_config.BOUNCE = {
+ callback = function(task)
+ local from = task:get_from('smtp')
+ if from and from[1].addr ~= '' then
+ -- RFC 3464:
+ -- Whenever an SMTP transaction is used to send a DSN, the MAIL FROM
+ -- command MUST use a NULL return address, i.e., "MAIL FROM:<>"
+ -- In practise it is almost always the case for DSN
+ return false
+ end
+
+ local parts = task:get_parts()
+ local top_type, top_subtype, params = parts[1]:get_type_full()
+ -- RFC 3464, RFC 8098
+ if top_type == 'multipart' and top_subtype == 'report' and params and
+ (params['report-type'] == 'delivery-status' or params['report-type'] == 'disposition-notification') then
+ -- Assume that inner parts are OK, don't check them to save time
+ return true, 1.0, 'DSN'
+ end
+
+ -- Apply heuristics for non-standard bounces
+ local bounce_sender
+ local mime_from = task:get_from('mime')
+ if mime_from then
+ local from_user = mime_from[1].user:lower()
+ -- Check common bounce senders
+ if (from_user == 'postmaster' or from_user == 'mailer-daemon') then
+ bounce_sender = from_user
+ -- MDaemon >= 14.5 sends multipart/report (RFC 3464) DSN covered above,
+ -- but older versions send non-standard bounces with localized subjects and they
+ -- are still around
+ elseif from_user == 'mdaemon' and task:has_header('X-MDDSN-Message') then
+ return true, 1.0, 'MDaemon'
+ end
+ end
+
+ local subj_keywords = task:has_symbol('SUBJ_BOUNCE_WORDS')
+
+ if not (bounce_sender or subj_keywords) then
+ return false
+ end
+
+ if bounce_sender and subj_keywords then
+ return true, 0.5, bounce_sender .. '+subj'
+ end
+
+ -- Look for a message/rfc822(-headers) part inside
+ local rfc822_part
+ parts[10] = nil -- limit number of parts to check
+ for _, p in ipairs(parts) do
+ local mime_type, mime_subtype = p:get_type()
+ if (mime_subtype == 'rfc822' or mime_subtype == 'rfc822-headers') and
+ (mime_type == 'message' or mime_type == 'text') then
+ rfc822_part = mime_type .. '/' .. mime_subtype
+ break
+ end
+ end
+
+ if rfc822_part and bounce_sender then
+ return true, 0.5, bounce_sender .. '+' .. rfc822_part
+ elseif rfc822_part and subj_keywords then
+ return true, 0.2, rfc822_part .. '+subj'
+ end
+ end,
+ description = '(Non) Delivery Status Notification',
+ group = 'headers',
+}
+
+rspamd_config:register_dependency('BOUNCE', 'SUBJ_BOUNCE_WORDS')
diff --git a/rules/content.lua b/rules/content.lua
new file mode 100644
index 0000000..667b7ec
--- /dev/null
+++ b/rules/content.lua
@@ -0,0 +1,118 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local function process_pdf_specific(task, part, specific)
+ local suspicious_factor = 0
+ if specific.encrypted then
+ task:insert_result('PDF_ENCRYPTED', 1.0, part:get_filename() or 'unknown')
+ suspicious_factor = suspicious_factor + 0.1
+ if specific.openaction then
+ suspicious_factor = suspicious_factor + 0.5
+ end
+ end
+
+ if specific.scripts then
+ task:insert_result('PDF_JAVASCRIPT', 1.0, part:get_filename() or 'unknown')
+ suspicious_factor = suspicious_factor + 0.1
+ end
+
+ if specific.suspicious then
+ suspicious_factor = suspicious_factor + specific.suspicious
+ end
+
+ if suspicious_factor > 0.5 then
+ if suspicious_factor > 1.0 then
+ suspicious_factor = 1.0
+ end
+ task:insert_result('PDF_SUSPICIOUS', suspicious_factor, part:get_filename() or 'unknown')
+ end
+
+ if specific.long_trailer then
+ task:insert_result('PDF_LONG_TRAILER', 1.0, string.format('%s:%d',
+ part:get_filename() or 'unknown', specific.long_trailer))
+ end
+ if specific.many_objects then
+ task:insert_result('PDF_MANY_OBJECTS', 1.0, string.format('%s:%d',
+ part:get_filename() or 'unknown', specific.many_objects))
+ end
+ if specific.timeout_processing then
+ task:insert_result('PDF_TIMEOUT', 1.0, string.format('%s:%.3f',
+ part:get_filename() or 'unknown', specific.timeout_processing))
+ end
+end
+
+local tags_processors = {
+ pdf = process_pdf_specific
+}
+
+local function process_specific_cb(task)
+ local parts = task:get_parts() or {}
+
+ for _, p in ipairs(parts) do
+ if p:is_specific() then
+ local data = p:get_specific()
+
+ if data and type(data) == 'table' and data.tag then
+ if tags_processors[data.tag] then
+ tags_processors[data.tag](task, p, data)
+ end
+ end
+ end
+ end
+end
+
+local id = rspamd_config:register_symbol {
+ type = 'callback',
+ name = 'SPECIFIC_CONTENT_CHECK',
+ callback = process_specific_cb
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ name = 'PDF_ENCRYPTED',
+ parent = id,
+ groups = { "content", "pdf" },
+}
+rspamd_config:register_symbol {
+ type = 'virtual',
+ name = 'PDF_JAVASCRIPT',
+ parent = id,
+ groups = { "content", "pdf" },
+}
+rspamd_config:register_symbol {
+ type = 'virtual',
+ name = 'PDF_SUSPICIOUS',
+ parent = id,
+ groups = { "content", "pdf" },
+}
+rspamd_config:register_symbol {
+ type = 'virtual',
+ name = 'PDF_LONG_TRAILER',
+ parent = id,
+ groups = { "content", "pdf" },
+}
+rspamd_config:register_symbol {
+ type = 'virtual',
+ name = 'PDF_MANY_OBJECTS',
+ parent = id,
+ groups = { "content", "pdf" },
+}
+rspamd_config:register_symbol {
+ type = 'virtual',
+ name = 'PDF_TIMEOUT',
+ parent = id,
+ groups = { "content", "pdf" },
+}
diff --git a/rules/controller/fuzzy.lua b/rules/controller/fuzzy.lua
new file mode 100644
index 0000000..193e6fd
--- /dev/null
+++ b/rules/controller/fuzzy.lua
@@ -0,0 +1,46 @@
+--[[
+Copyright (c) 2023, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local function handle_gen_fuzzy(task, conn, req_params)
+ if type(rspamd_plugins.fuzzy_check) == 'table' then
+ local ret, hashes
+ task:process_message()
+ if req_params.rule then
+ ret, hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, req_params.rule)
+ elseif req_params.flag then
+ ret, hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, tonumber(req_params.flag))
+ else
+ conn:send_error(404, 'missing rule or flag')
+ return
+ end
+
+ if ret then
+ conn:send_ucl({ success = true, hashes = hashes })
+ else
+ conn:send_error(500, 'cannot generate hashes')
+ end
+ else
+ conn:send_error(404, 'fuzzy_check is not enabled')
+ end
+end
+
+return {
+ hashes = {
+ handler = handle_gen_fuzzy,
+ need_task = true,
+ enable = false
+ },
+} \ No newline at end of file
diff --git a/rules/controller/init.lua b/rules/controller/init.lua
new file mode 100644
index 0000000..17fbbfc
--- /dev/null
+++ b/rules/controller/init.lua
@@ -0,0 +1,67 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Controller endpoints
+
+local local_conf = rspamd_paths['LOCAL_CONFDIR']
+local local_rules = rspamd_paths['RULESDIR']
+local rspamd_util = require "rspamd_util"
+local lua_util = require "lua_util"
+local rspamd_logger = require "rspamd_logger"
+
+-- Define default controller paths, could be overridden in local.d/controller.lua
+
+local controller_plugin_paths = {
+ maps = dofile(local_rules .. "/controller/maps.lua"),
+ neural = dofile(local_rules .. "/controller/neural.lua"),
+ selectors = dofile(local_rules .. "/controller/selectors.lua"),
+ fuzzy = dofile(local_rules .. "/controller/fuzzy.lua"),
+}
+
+if rspamd_util.file_exists(local_conf .. '/controller.lua') then
+ local controller_overrides = dofile(local_conf .. '/controller.lua')
+
+ if controller_overrides and type(controller_overrides) == 'table' then
+ controller_plugin_paths = lua_util.override_defaults(controller_plugin_paths, controller_overrides)
+ end
+end
+
+for plug, paths in pairs(controller_plugin_paths) do
+ if not rspamd_plugins[plug] then
+ rspamd_plugins[plug] = {}
+ end
+ if not rspamd_plugins[plug].webui then
+ rspamd_plugins[plug].webui = {}
+ end
+
+ local webui = rspamd_plugins[plug].webui
+
+ for path, attrs in pairs(paths) do
+ if type(attrs) == 'table' then
+ if type(attrs.handler) ~= 'function' then
+ rspamd_logger.infox(rspamd_config, 'controller plugin %s; webui path %s has invalid handler: %s; ignore it',
+ plug, path, type(attrs.handler))
+ else
+ webui[path] = lua_util.shallowcopy(attrs)
+ rspamd_logger.infox(rspamd_config, 'controller plugin %s; register webui path %s',
+ plug, path)
+ end
+ else
+ rspamd_logger.infox(rspamd_config, 'controller plugin %s; webui path %s has invalid type: %s; ignore it',
+ plug, path, type(attrs))
+ end
+ end
+end
diff --git a/rules/controller/maps.lua b/rules/controller/maps.lua
new file mode 100644
index 0000000..718e292
--- /dev/null
+++ b/rules/controller/maps.lua
@@ -0,0 +1,220 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Controller maps plugin
+local maps_cache
+local maps_aliases
+local lua_util = require "lua_util"
+local ts = require("tableshape").types
+local ucl = require "ucl"
+
+local function maybe_fill_maps_cache()
+ if not maps_cache then
+ maps_cache = {}
+ maps_aliases = {}
+ local maps = rspamd_config:get_maps()
+ for _, m in ipairs(maps) do
+ -- We get the first url here and that's it
+ local url = m:get_uri()
+ if url ~= 'static' then
+ if not maps_cache[url] then
+ local alias = url:match('/([^/]+)$')
+ maps_cache[url] = m
+ if not maps_aliases[alias] then
+ maps_aliases[alias] = url
+ end
+ else
+ -- Do not override, as we don't care about duplicate maps that come from different
+ -- sources.
+ -- In theory, that should be cached but there are some exceptions even so far...
+ url = math.random() -- to shut luacheck about empty branch with a comment
+ end
+ end
+ end
+ end
+end
+
+local function check_specific_map(input, uri, m, results, report_misses)
+ local value = m:get_key(input)
+
+ if value then
+ local result = {
+ map = uri,
+ alias = uri:match('/([^/]+)$'),
+ value = value,
+ key = input,
+ hit = true,
+ }
+ table.insert(results, result)
+ elseif report_misses then
+ local result = {
+ map = uri,
+ alias = uri:match('/([^/]+)$'),
+ key = input,
+ hit = false,
+ }
+ table.insert(results, result)
+ end
+end
+
+local function handle_query_map(_, conn, req_params)
+ maybe_fill_maps_cache()
+ local keys_to_check = {}
+
+ if req_params.value and req_params.value ~= '' then
+ keys_to_check[1] = req_params.value
+ elseif req_params.values then
+ keys_to_check = lua_util.str_split(req_params.values, ',')
+ end
+
+ local results = {}
+ for _, key in ipairs(keys_to_check) do
+ for uri, m in pairs(maps_cache) do
+ check_specific_map(key, uri, m, results, req_params.report_misses)
+ end
+ end
+ conn:send_ucl {
+ success = (#results > 0),
+ results = results
+ }
+end
+
+local function handle_query_specific_map(_, conn, req_params)
+ maybe_fill_maps_cache()
+ -- Fill keys to check
+ local keys_to_check = {}
+ if req_params.value and req_params.value ~= '' then
+ keys_to_check[1] = req_params.value
+ elseif req_params.values then
+ keys_to_check = lua_util.str_split(req_params.values, ',')
+ end
+ local maps_to_check = maps_cache
+ -- Fill maps to check
+ if req_params.maps then
+ local map_names = lua_util.str_split(req_params.maps, ',')
+ maps_to_check = {}
+ for _, mn in ipairs(map_names) do
+ if maps_cache[mn] then
+ maps_to_check[mn] = maps_cache[mn]
+ else
+ local alias = maps_aliases[mn]
+
+ if alias then
+ maps_to_check[alias] = maps_cache[alias]
+ else
+ conn:send_error(404, 'no such map: ' .. mn)
+ end
+ end
+ end
+ end
+
+ local results = {}
+ for _, key in ipairs(keys_to_check) do
+ for uri, m in pairs(maps_to_check) do
+ check_specific_map(key, uri, m, results, req_params.report_misses)
+ end
+ end
+
+ conn:send_ucl {
+ success = (#results > 0),
+ results = results
+ }
+end
+
+local function handle_list_maps(_, conn, _)
+ maybe_fill_maps_cache()
+ conn:send_ucl {
+ maps = lua_util.keys(maps_cache),
+ aliases = maps_aliases
+ }
+end
+
+local query_json_schema = ts.shape {
+ maps = ts.array_of(ts.string):is_optional(),
+ report_misses = ts.boolean:is_optional(),
+ values = ts.array_of(ts.string),
+}
+
+local function handle_query_json(task, conn)
+ maybe_fill_maps_cache()
+
+ local parser = ucl.parser()
+ local ok, err = parser:parse_text(task:get_rawbody())
+ if not ok then
+ conn:send_error(400, err)
+ return
+ end
+ local obj = parser:get_object()
+
+ ok, err = query_json_schema:transform(obj)
+ if not ok then
+ conn:send_error(400, err)
+ return
+ end
+
+ local maps_to_check = {}
+ local report_misses = obj.report_misses
+ local results = {}
+
+ if obj.maps then
+ for _, mn in ipairs(obj.maps) do
+ if maps_cache[mn] then
+ maps_to_check[mn] = maps_cache[mn]
+ else
+ local alias = maps_aliases[mn]
+
+ if alias then
+ maps_to_check[alias] = maps_cache[alias]
+ else
+ conn:send_error(400, 'no such map: ' .. mn)
+ return
+ end
+ end
+ end
+ else
+ maps_to_check = maps_cache
+ end
+
+ for _, key in ipairs(obj.values) do
+ for uri, m in pairs(maps_to_check) do
+ check_specific_map(key, uri, m, results, report_misses)
+ end
+ end
+ conn:send_ucl {
+ success = (#results > 0),
+ results = results
+ }
+end
+
+return {
+ query = {
+ handler = handle_query_map,
+ enable = false,
+ },
+ query_json = {
+ handler = handle_query_json,
+ enable = false,
+ need_task = true,
+ },
+ query_specific = {
+ handler = handle_query_specific_map,
+ enable = false,
+ },
+ list = {
+ handler = handle_list_maps,
+ enable = false,
+ },
+}
diff --git a/rules/controller/neural.lua b/rules/controller/neural.lua
new file mode 100644
index 0000000..aef1042
--- /dev/null
+++ b/rules/controller/neural.lua
@@ -0,0 +1,70 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local neural_common = require "plugins/neural"
+local ts = require("tableshape").types
+local ucl = require "ucl"
+
+local E = {}
+
+-- Controller neural plugin
+
+local learn_request_schema = ts.shape {
+ ham_vec = ts.array_of(ts.array_of(ts.number)),
+ rule = ts.string:is_optional(),
+ spam_vec = ts.array_of(ts.array_of(ts.number)),
+}
+
+local function handle_learn(task, conn)
+ local parser = ucl.parser()
+ local ok, err = parser:parse_text(task:get_rawbody())
+ if not ok then
+ conn:send_error(400, err)
+ return
+ end
+ local req_params = parser:get_object()
+
+ ok, err = learn_request_schema:transform(req_params)
+ if not ok then
+ conn:send_error(400, err)
+ return
+ end
+
+ local rule_name = req_params.rule or 'default'
+ local rule = neural_common.settings.rules[rule_name]
+ local set = neural_common.get_rule_settings(task, rule)
+ local version = ((set.ann or E).version or 0) + 1
+
+ neural_common.spawn_train {
+ ev_base = task:get_ev_base(),
+ ann_key = neural_common.new_ann_key(rule, set, version),
+ set = set,
+ rule = rule,
+ ham_vec = req_params.ham_vec,
+ spam_vec = req_params.spam_vec,
+ worker = task:get_worker(),
+ }
+
+ conn:send_string('{"success" : true}')
+end
+
+return {
+ learn = {
+ handler = handle_learn,
+ enable = true,
+ need_task = true,
+ },
+}
diff --git a/rules/controller/selectors.lua b/rules/controller/selectors.lua
new file mode 100644
index 0000000..7fc2894
--- /dev/null
+++ b/rules/controller/selectors.lua
@@ -0,0 +1,73 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local lua_selectors = require "lua_selectors"
+
+-- Controller selectors plugin
+
+local function handle_list_transforms(_, conn)
+ conn:send_ucl(lua_selectors.list_transforms())
+end
+
+local function handle_list_extractors(_, conn)
+ conn:send_ucl(lua_selectors.list_extractors())
+end
+
+local function handle_check_selector(_, conn, req_params)
+ if req_params.selector and req_params.selector ~= '' then
+ local selector = lua_selectors.create_selector_closure(rspamd_config,
+ req_params.selector, '', true)
+ conn:send_ucl({ success = selector and true })
+ else
+ conn:send_error(404, 'missing selector')
+ end
+end
+
+local function handle_check_message(task, conn, req_params)
+ if req_params.selector and req_params.selector ~= '' then
+ local selector = lua_selectors.create_selector_closure(rspamd_config,
+ req_params.selector, '', true)
+ if not selector then
+ conn:send_error(500, 'invalid selector')
+ else
+ task:process_message()
+ local elts = selector(task)
+ conn:send_ucl({ success = true, data = elts })
+ end
+ else
+ conn:send_error(404, 'missing selector')
+ end
+end
+
+return {
+ list_extractors = {
+ handler = handle_list_extractors,
+ enable = true,
+ },
+ list_transforms = {
+ handler = handle_list_transforms,
+ enable = true,
+ },
+ check_selector = {
+ handler = handle_check_selector,
+ enable = true,
+ },
+ check_message = {
+ handler = handle_check_message,
+ enable = true,
+ need_task = true,
+ }
+}
diff --git a/rules/forwarding.lua b/rules/forwarding.lua
new file mode 100644
index 0000000..a008c58
--- /dev/null
+++ b/rules/forwarding.lua
@@ -0,0 +1,163 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Rules to detect forwarding
+
+local rspamd_util = require "rspamd_util"
+
+rspamd_config.FWD_GOOGLE = {
+ callback = function(task)
+ if not (task:has_from(1) and task:has_recipients(1)) then
+ return false
+ end
+ local envfrom = task:get_from { 'smtp', 'orig' }
+ local envrcpts = task:get_recipients(1)
+ -- Forwarding will only be to a single recipient
+ if #envrcpts > 1 then
+ return false
+ end
+ -- Get recipient and compute VERP address
+ local rcpt = envrcpts[1].addr:lower()
+ local verp = rcpt:gsub('@', '=')
+ -- Get the user portion of the envfrom
+ local ef_user = envfrom[1].user:lower()
+ -- Check for a match
+ if ef_user:find('+caf_=' .. verp, 1, true) then
+ local _, _, user = ef_user:find('^(.+)+caf_=')
+ if user then
+ user = user .. '@' .. envfrom[1].domain
+ return true, user
+ end
+ end
+ return false
+ end,
+ score = 0.0,
+ description = "Message was forwarded by Google",
+ group = "forwarding"
+}
+
+rspamd_config.FWD_YANDEX = {
+ callback = function(task)
+ if not (task:has_from(1) and task:has_recipients(1)) then
+ return false
+ end
+ local hostname = task:get_hostname()
+ if hostname and hostname:lower():find('%.yandex%.[a-z]+$') then
+ return task:has_header('X-Yandex-Forward')
+ end
+ return false
+ end,
+ score = 0.0,
+ description = "Message was forwarded by Yandex",
+ group = "forwarding"
+}
+
+rspamd_config.FWD_MAILRU = {
+ callback = function(task)
+ if not (task:has_from(1) and task:has_recipients(1)) then
+ return false
+ end
+ local hostname = task:get_hostname()
+ if hostname and hostname:lower():find('%.mail%.ru$') then
+ return task:has_header('X-MailRu-Forward')
+ end
+ return false
+ end,
+ score = 0.0,
+ description = "Message was forwarded by Mail.ru",
+ group = "forwarding"
+}
+
+rspamd_config.FWD_SRS = {
+ callback = function(task)
+ if not (task:has_from(1) and task:has_recipients(1)) then
+ return false
+ end
+ local envfrom = task:get_from(1)
+ local envrcpts = task:get_recipients(1)
+ -- Forwarding is only to a single recipient
+ if #envrcpts > 1 then
+ return false
+ end
+ -- Get recipient and compute rewritten SRS address
+ local srs = '=' .. envrcpts[1].domain:lower() ..
+ '=' .. envrcpts[1].user:lower()
+ if envfrom[1].user:lower():find('^srs[01]=') and
+ envfrom[1].user:lower():find(srs, 1, false)
+ then
+ return true
+ end
+ return false
+ end,
+ score = 0.0,
+ description = "Message was forwarded using Sender Rewriting Scheme (SRS)",
+ group = "forwarding"
+}
+
+rspamd_config.FORWARDED = {
+ callback = function(task)
+ local function normalize_addr(addr)
+ addr = string.match(addr, '^<?([^>]*)>?$') or addr
+ local cap, _, domain = string.match(addr, '^([^%+][^%+]*)(%+[^@]*)@(.*)$')
+ if cap then
+ addr = string.format('%s@%s', cap, domain)
+ end
+
+ return addr
+ end
+
+ if not task:has_recipients(1) or not task:has_recipients(2) then
+ return false
+ end
+ local envrcpts = task:get_recipients(1)
+ -- Forwarding will only be for single recipient messages
+ if #envrcpts > 1 then
+ return false
+ end
+ -- Get any other headers we might need
+ local has_list_unsub = task:has_header('List-Unsubscribe')
+ local to = task:get_recipients(2)
+ local matches = 0
+ -- Retrieve and loop through all Received headers
+ local rcvds = task:get_received_headers()
+
+ if rcvds then
+ for _, rcvd in ipairs(rcvds) do
+ local addr = rcvd['for']
+ if addr then
+ addr = normalize_addr(addr)
+ matches = matches + 1
+ -- Check that it doesn't match the envrcpt
+ if not rspamd_util.strequal_caseless(addr, envrcpts[1].addr) then
+ -- Check for mailing-lists as they will have the same signature
+ if matches < 2 and has_list_unsub and to and rspamd_util.strequal_caseless(to[1].addr, addr) then
+ return false
+ else
+ return true, 1.0, addr
+ end
+ end
+ -- Prevent any other iterations as we only want
+ -- process the first matching Received header
+ return false
+ end
+ end
+ end
+ return false
+ end,
+ score = 0.0,
+ description = "Message was forwarded",
+ group = "forwarding"
+}
diff --git a/rules/headers_checks.lua b/rules/headers_checks.lua
new file mode 100644
index 0000000..92ebb0c
--- /dev/null
+++ b/rules/headers_checks.lua
@@ -0,0 +1,1174 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local util = require "rspamd_util"
+local ipairs = ipairs
+local pairs = pairs
+local table = table
+local tostring = tostring
+local tonumber = tonumber
+local fun = require "fun"
+local E = {}
+
+local rcvd_cb_id = rspamd_config:register_symbol {
+ name = 'CHECK_RECEIVED',
+ type = 'callback',
+ score = 0.0,
+ group = 'headers',
+ callback = function(task)
+ local cnts = {
+ [1] = 'ONE',
+ [2] = 'TWO',
+ [3] = 'THREE',
+ [5] = 'FIVE',
+ [7] = 'SEVEN',
+ [12] = 'TWELVE'
+ }
+ local def = 'ZERO'
+ local received = task:get_received_headers()
+ local nreceived = fun.reduce(function(acc, rcvd)
+ return acc + 1
+ end, 0, fun.filter(function(h)
+ return not h['flags']['artificial']
+ end, received))
+
+ for k, v in pairs(cnts) do
+ if nreceived >= tonumber(k) then
+ def = v
+ end
+ end
+
+ task:insert_result('RCVD_COUNT_' .. def, 1.0, tostring(nreceived))
+ end
+}
+
+rspamd_config:register_symbol {
+ name = 'RCVD_COUNT_ZERO',
+ score = 0.0,
+ parent = rcvd_cb_id,
+ type = 'virtual',
+ description = 'Message has no Received headers',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCVD_COUNT_ONE',
+ score = 0.0,
+ parent = rcvd_cb_id,
+ type = 'virtual',
+ description = 'Message has one Received header',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCVD_COUNT_TWO',
+ score = 0.0,
+ parent = rcvd_cb_id,
+ type = 'virtual',
+ description = 'Message has two Received headers',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCVD_COUNT_THREE',
+ score = 0.0,
+ parent = rcvd_cb_id,
+ type = 'virtual',
+ description = 'Message has 3-5 Received headers',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCVD_COUNT_FIVE',
+ score = 0.0,
+ parent = rcvd_cb_id,
+ type = 'virtual',
+ description = 'Message has 5-7 Received headers',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCVD_COUNT_SEVEN',
+ score = 0.0,
+ parent = rcvd_cb_id,
+ type = 'virtual',
+ description = 'Message has 7-11 Received headers',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCVD_COUNT_TWELVE',
+ score = 0.0,
+ parent = rcvd_cb_id,
+ type = 'virtual',
+ description = 'Message has 12 or more Received headers',
+ group = 'headers',
+}
+
+local prio_cb_id = rspamd_config:register_symbol {
+ name = 'HAS_X_PRIO',
+ type = 'callback',
+ description = 'X-Priority check callback rule',
+ score = 0.0,
+ group = 'headers',
+ callback = function(task)
+ local cnts = {
+ [1] = 'ONE',
+ [2] = 'TWO',
+ [3] = 'THREE',
+ [5] = 'FIVE',
+ }
+ local def = 'ZERO'
+ local xprio = task:get_header('X-Priority');
+ if not xprio then
+ return false
+ end
+ local _, _, x = xprio:find('^%s?(%d+)');
+ if (x) then
+ x = tonumber(x)
+ for k, v in pairs(cnts) do
+ if x >= tonumber(k) then
+ def = v
+ end
+ end
+ task:insert_result('HAS_X_PRIO_' .. def, 1.0, tostring(x))
+ end
+ end
+}
+rspamd_config:register_symbol {
+ name = 'HAS_X_PRIO_ZERO',
+ score = 0.0,
+ parent = prio_cb_id,
+ type = 'virtual',
+ description = 'Message has X-Priority header set to 0',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'HAS_X_PRIO_ONE',
+ score = 0.0,
+ parent = prio_cb_id,
+ type = 'virtual',
+ description = 'Message has X-Priority header set to 1',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'HAS_X_PRIO_TWO',
+ score = 0.0,
+ parent = prio_cb_id,
+ type = 'virtual',
+ description = 'Message has X-Priority header set to 2',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'HAS_X_PRIO_THREE',
+ score = 0.0,
+ parent = prio_cb_id,
+ type = 'virtual',
+ description = 'Message has X-Priority header set to 3 or 4',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'HAS_X_PRIO_FIVE',
+ score = 0.0,
+ parent = prio_cb_id,
+ type = 'virtual',
+ description = 'Message has X-Priority header set to 5 or higher',
+ group = 'headers',
+}
+
+local function get_raw_header(task, name)
+ return ((task:get_header_full(name) or {})[1] or {})['value']
+end
+
+local check_replyto_id = rspamd_config:register_symbol({
+ type = 'callback',
+ name = 'CHECK_REPLYTO',
+ score = 0.0,
+ group = 'headers',
+ callback = function(task)
+ local replyto = get_raw_header(task, 'Reply-To')
+ if not replyto then
+ return false
+ end
+ local rt = util.parse_mail_address(replyto, task:get_mempool())
+ if not (rt and rt[1] and (string.len(rt[1].addr) > 0)) then
+ task:insert_result('REPLYTO_UNPARSEABLE', 1.0)
+ return false
+ else
+ local rta = rt[1].addr
+ task:insert_result('HAS_REPLYTO', 1.0, rta)
+ -- Check if Reply-To address starts with title seen in display name
+ local sym = task:get_symbol('FROM_NAME_HAS_TITLE')
+ local title = (((sym or E)[1] or E).options or E)[1]
+ if title then
+ rta = rta:lower()
+ if rta:find('^' .. title) then
+ task:insert_result('REPLYTO_EMAIL_HAS_TITLE', 1.0)
+ end
+ end
+ end
+
+ -- See if Reply-To matches From in some way
+ local from = task:get_from { 'mime', 'orig' }
+ local from_h = get_raw_header(task, 'From')
+ if not (from and from[1]) then
+ return false
+ end
+ if (from_h and from_h == replyto) then
+ -- From and Reply-To are identical
+ task:insert_result('REPLYTO_EQ_FROM', 1.0)
+ else
+ if (from and from[1]) then
+ -- See if From and Reply-To addresses match
+ if (util.strequal_caseless(from[1].addr, rt[1].addr)) then
+ task:insert_result('REPLYTO_ADDR_EQ_FROM', 1.0)
+ elseif from[1].domain and rt[1].domain then
+ if (util.strequal_caseless(from[1].domain, rt[1].domain)) then
+ task:insert_result('REPLYTO_DOM_EQ_FROM_DOM', 1.0)
+ else
+ -- See if Reply-To matches the To address
+ local to = task:get_recipients(2)
+ if (to and to[1] and to[1].addr:lower() == rt[1].addr:lower()) then
+ -- Ignore this for mailing-lists and automatic submissions
+ if (not (task:get_header('List-Unsubscribe') or
+ task:get_header('X-To-Get-Off-This-List') or
+ task:get_header('X-List') or
+ task:get_header('Auto-Submitted')))
+ then
+ task:insert_result('REPLYTO_EQ_TO_ADDR', 1.0)
+ end
+ else
+ task:insert_result('REPLYTO_DOM_NEQ_FROM_DOM', 1.0)
+ end
+ end
+ end
+ -- See if the Display Names match
+ if (from[1].name and rt[1].name and
+ util.strequal_caseless(from[1].name, rt[1].name)) then
+ task:insert_result('REPLYTO_DN_EQ_FROM_DN', 1.0)
+ end
+ end
+ end
+ end
+})
+
+rspamd_config:register_symbol {
+ name = 'REPLYTO_UNPARSEABLE',
+ score = 1.0,
+ parent = check_replyto_id,
+ type = 'virtual',
+ description = 'Reply-To header could not be parsed',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'HAS_REPLYTO',
+ score = 0.0,
+ parent = check_replyto_id,
+ type = 'virtual',
+ description = 'Has Reply-To header',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'REPLYTO_EQ_FROM',
+ score = 0.0,
+ parent = check_replyto_id,
+ type = 'virtual',
+ description = 'Reply-To header is identical to From header',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'REPLYTO_ADDR_EQ_FROM',
+ score = 0.0,
+ parent = check_replyto_id,
+ type = 'virtual',
+ description = 'Reply-To header is identical to SMTP From',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'REPLYTO_DOM_EQ_FROM_DOM',
+ score = 0.0,
+ parent = check_replyto_id,
+ type = 'virtual',
+ description = 'Reply-To domain matches the From domain',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'REPLYTO_DOM_NEQ_FROM_DOM',
+ score = 0.0,
+ parent = check_replyto_id,
+ type = 'virtual',
+ description = 'Reply-To domain does not match the From domain',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'REPLYTO_DN_EQ_FROM_DN',
+ score = 0.0,
+ parent = check_replyto_id,
+ type = 'virtual',
+ description = 'Reply-To display name matches From',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'REPLYTO_EMAIL_HAS_TITLE',
+ score = 2.0,
+ parent = check_replyto_id,
+ type = 'virtual',
+ description = 'Reply-To header has title',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'REPLYTO_EQ_TO_ADDR',
+ score = 5.0,
+ parent = check_replyto_id,
+ type = 'virtual',
+ description = 'Reply-To is the same as the To address',
+ group = 'headers',
+}
+
+rspamd_config:register_dependency('CHECK_REPLYTO', 'CHECK_FROM')
+
+local check_mime_id = rspamd_config:register_symbol {
+ name = 'CHECK_MIME',
+ type = 'callback',
+ group = 'headers',
+ score = 0.0,
+ callback = function(task)
+ -- Check if there is a MIME-Version header
+ local missing_mime = false
+ if not task:has_header('MIME-Version') then
+ missing_mime = true
+ end
+
+ -- Check presence of MIME specific headers
+ local has_ct_header = task:has_header('Content-Type')
+ local has_cte_header = task:has_header('Content-Transfer-Encoding')
+
+ -- Add the symbol if we have MIME headers, but no MIME-Version
+ -- (do not add the symbol for RFC822 messages)
+ if (has_ct_header or has_cte_header) and missing_mime then
+ task:insert_result('MISSING_MIME_VERSION', 1.0)
+ end
+
+ local found_ma = false
+ local found_plain = false
+ local found_html = false
+
+ for _, p in ipairs(task:get_parts()) do
+ local mtype, subtype = p:get_type()
+ local ctype = mtype:lower() .. '/' .. subtype:lower()
+ if (ctype == 'multipart/alternative') then
+ found_ma = true
+ end
+ if (ctype == 'text/plain') then
+ found_plain = true
+ end
+ if (ctype == 'text/html') then
+ found_html = true
+ end
+ end
+
+ if (found_ma) then
+ if (not found_plain) then
+ task:insert_result('MIME_MA_MISSING_TEXT', 1.0)
+ end
+ if (not found_html) then
+ task:insert_result('MIME_MA_MISSING_HTML', 1.0)
+ end
+ end
+ end
+}
+
+rspamd_config:register_symbol {
+ name = 'MISSING_MIME_VERSION',
+ score = 2.0,
+ parent = check_mime_id,
+ type = 'virtual',
+ description = 'MIME-Version header is missing in MIME message',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'MIME_MA_MISSING_TEXT',
+ score = 2.0,
+ parent = check_mime_id,
+ type = 'virtual',
+ description = 'MIME multipart/alternative missing text/plain part',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'MIME_MA_MISSING_HTML',
+ score = 1.0,
+ parent = check_mime_id,
+ type = 'virtual',
+ description = 'MIME multipart/alternative missing text/html part',
+ group = 'headers',
+}
+
+-- Used to be called IS_LIST
+rspamd_config.PREVIOUSLY_DELIVERED = {
+ callback = function(task)
+ if not task:has_recipients(2) then
+ return false
+ end
+ local to = task:get_recipients(2)
+ local rcvds = task:get_header_full('Received')
+ if not rcvds then
+ return false
+ end
+ for _, rcvd in ipairs(rcvds) do
+ local _, _, addr = rcvd['decoded']:lower():find("%sfor%s<(.-)>")
+ if addr then
+ for _, toa in ipairs(to) do
+ if toa and toa.addr:lower() == addr then
+ return true, addr
+ end
+ end
+ return false
+ end
+ end
+ end,
+ description = 'Message either to a list or was forwarded',
+ group = 'headers',
+ score = 0.0
+}
+rspamd_config.BROKEN_HEADERS = {
+ callback = function(task)
+ return task:has_flag('broken_headers')
+ end,
+ score = 10.0,
+ group = 'headers',
+ description = 'Headers structure is likely broken'
+}
+
+rspamd_config.BROKEN_CONTENT_TYPE = {
+ callback = function(task)
+ return fun.any(function(p)
+ return p:is_broken()
+ end,
+ task:get_parts())
+ end,
+ score = 1.5,
+ group = 'headers',
+ description = 'Message has part with broken content type'
+}
+
+rspamd_config.HEADER_RCONFIRM_MISMATCH = {
+ callback = function(task)
+ local header_from = nil
+ local cread = task:get_header('X-Confirm-Reading-To')
+
+ if task:has_from('mime') then
+ header_from = task:get_from('mime')[1]
+ end
+
+ local header_cread = nil
+ if cread then
+ local headers_cread = util.parse_mail_address(cread, task:get_mempool())
+ if headers_cread then
+ header_cread = headers_cread[1]
+ end
+ end
+
+ if header_from and header_cread then
+ if not string.find(header_from['addr'], header_cread['addr']) then
+ return true
+ end
+ end
+
+ return false
+ end,
+
+ score = 2.0,
+ group = 'headers',
+ description = 'Read confirmation address is different to from address'
+}
+
+rspamd_config.HEADER_FORGED_MDN = {
+ callback = function(task)
+ local mdn = task:get_header('Disposition-Notification-To')
+ if not mdn then
+ return false
+ end
+ local header_rp = nil
+
+ if task:has_from('smtp') then
+ header_rp = task:get_from('smtp')[1]
+ end
+
+ -- Parse mail addr
+ local headers_mdn = util.parse_mail_address(mdn, task:get_mempool())
+
+ if headers_mdn and not header_rp then
+ return true
+ end
+ if header_rp and not headers_mdn then
+ return false
+ end
+ if not headers_mdn and not header_rp then
+ return false
+ end
+
+ local found_match = false
+ for _, h in ipairs(headers_mdn) do
+ if util.strequal_caseless(h['addr'], header_rp['addr']) then
+ found_match = true
+ break
+ end
+ end
+
+ return (not found_match)
+ end,
+
+ score = 2.0,
+ group = 'headers',
+ description = 'Read confirmation address is different to return path'
+}
+
+local headers_unique = {
+ ['Content-Type'] = 1.0,
+ ['Content-Transfer-Encoding'] = 1.0,
+ -- https://tools.ietf.org/html/rfc5322#section-3.6
+ ['Date'] = 0.1,
+ ['From'] = 1.0,
+ ['Sender'] = 1.0,
+ ['Reply-To'] = 1.0,
+ ['To'] = 0.2,
+ ['Cc'] = 0.1,
+ ['Bcc'] = 0.1,
+ ['Message-ID'] = 0.7,
+ ['In-Reply-To'] = 0.7,
+ ['References'] = 0.3,
+ ['Subject'] = 0.7
+}
+
+local multiple_unique_headers_id = rspamd_config:register_symbol {
+ name = 'MULTIPLE_UNIQUE_HEADERS',
+ callback = function(task)
+ local res = 0
+ local max_mult = 0.0
+ local res_tbl = {}
+ local found = 0
+
+ for hdr, mult in pairs(headers_unique) do
+ local hc = task:get_header_count(hdr)
+ found = found + hc
+
+ if hc > 1 then
+ res = res + 1
+ table.insert(res_tbl, hdr)
+ if max_mult < mult then
+ max_mult = mult
+ end
+ end
+ end
+
+ if res > 0 then
+ task:insert_result('MULTIPLE_UNIQUE_HEADERS', max_mult, table.concat(res_tbl, ','))
+ elseif found == 0 then
+ task:insert_result('MISSING_ESSENTIAL_HEADERS', 1.0)
+ end
+ end,
+
+ score = 7.0,
+ group = 'headers',
+ one_shot = true,
+ description = 'Repeated unique headers'
+}
+
+rspamd_config:register_symbol {
+ name = 'MISSING_ESSENTIAL_HEADERS',
+ score = 7.0,
+ group = 'blankspam',
+ parent = multiple_unique_headers_id,
+ type = 'virtual',
+ description = 'Common headers were entirely absent',
+}
+
+rspamd_config.MISSING_FROM = {
+ callback = function(task)
+ local from = task:get_header('From')
+ if from == nil or from == '' then
+ return true
+ end
+ return false
+ end,
+ score = 2.0,
+ group = 'headers',
+ description = 'Missing From header'
+}
+
+rspamd_config.MULTIPLE_FROM = {
+ callback = function(task)
+ local from = task:get_from('mime')
+ if from and from[2] then
+ return true, 1.0, fun.totable(fun.map(function(a)
+ return a.raw
+ end, from))
+ end
+ return false
+ end,
+ score = 8.0,
+ group = 'headers',
+ description = 'Multiple addresses in From header'
+}
+
+rspamd_config.MV_CASE = {
+ callback = function(task)
+ return task:has_header('Mime-Version', true)
+ end,
+ description = 'Mime-Version .vs. MIME-Version',
+ score = 0.5,
+ group = 'headers'
+}
+
+local check_from_id = rspamd_config:register_symbol {
+ name = 'CHECK_FROM',
+ type = 'callback',
+ score = 0.0,
+ group = 'headers',
+ callback = function(task)
+ local envfrom = task:get_from(1)
+ local from = task:get_from(2)
+ if (envfrom and envfrom[1] and not envfrom[1]["flags"]["valid"]) then
+ task:insert_result('ENVFROM_INVALID', 1.0)
+ end
+ if (from and from[1]) then
+ if not (from[1]["flags"]["valid"]) then
+ task:insert_result('FROM_INVALID', 1.0)
+ end
+ if (from[1].name == nil or from[1].name == '') then
+ task:insert_result('FROM_NO_DN', 1.0)
+ elseif (from[1].name and
+ util.strequal_caseless(from[1].name, from[1].addr)) then
+ task:insert_result('FROM_DN_EQ_ADDR', 1.0)
+ elseif (from[1].name and from[1].name ~= '') then
+ task:insert_result('FROM_HAS_DN', 1.0)
+ -- Look for Mr/Mrs/Dr titles
+ local n = from[1].name:lower()
+ local match, match_end
+ match, match_end = n:find('^mrs?[%.%s]')
+ if match then
+ task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end - 1))
+ end
+ match, match_end = n:find('^dr[%.%s]')
+ if match then
+ task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end - 1))
+ end
+ -- Check for excess spaces
+ if n:find('%s%s') then
+ task:insert_result('FROM_NAME_EXCESS_SPACE', 1.0)
+ end
+ end
+
+ if envfrom then
+ if util.strequal_caseless(envfrom[1].addr, from[1].addr) then
+ task:insert_result('FROM_EQ_ENVFROM', 1.0)
+ elseif envfrom[1].addr ~= '' then
+ task:insert_result('FROM_NEQ_ENVFROM', 1.0, from[1].addr, envfrom[1].addr)
+ end
+ end
+ end
+
+ local to = task:get_recipients(2)
+ if not (to and to[1] and #to == 1 and from and from[1]) then
+ return false
+ end
+ -- Check if FROM == TO
+ if (util.strequal_caseless(to[1].addr, from[1].addr)) then
+ task:insert_result('TO_EQ_FROM', 1.0)
+ elseif (to[1].domain and from[1].domain and
+ util.strequal_caseless(to[1].domain, from[1].domain))
+ then
+ task:insert_result('TO_DOM_EQ_FROM_DOM', 1.0)
+ end
+ end
+}
+
+rspamd_config:register_symbol {
+ name = 'ENVFROM_INVALID',
+ score = 2.0,
+ group = 'headers',
+ parent = check_from_id,
+ type = 'virtual',
+ description = 'Envelope from does not have a valid format',
+}
+rspamd_config:register_symbol {
+ name = 'FROM_INVALID',
+ score = 2.0,
+ group = 'headers',
+ parent = check_from_id,
+ type = 'virtual',
+ description = 'From header does not have a valid format',
+}
+rspamd_config:register_symbol {
+ name = 'FROM_NO_DN',
+ score = 0.0,
+ group = 'headers',
+ parent = check_from_id,
+ type = 'virtual',
+ description = 'From header does not have a display name',
+}
+rspamd_config:register_symbol {
+ name = 'FROM_DN_EQ_ADDR',
+ score = 1.0,
+ group = 'headers',
+ parent = check_from_id,
+ type = 'virtual',
+ description = 'From header display name is the same as the address',
+}
+rspamd_config:register_symbol {
+ name = 'FROM_HAS_DN',
+ score = 0.0,
+ group = 'headers',
+ parent = check_from_id,
+ type = 'virtual',
+ description = 'From header has a display name',
+}
+rspamd_config:register_symbol {
+ name = 'FROM_NAME_EXCESS_SPACE',
+ score = 1.0,
+ group = 'headers',
+ parent = check_from_id,
+ type = 'virtual',
+ description = 'From header display name contains excess whitespace',
+}
+rspamd_config:register_symbol {
+ name = 'FROM_NAME_HAS_TITLE',
+ score = 1.0,
+ group = 'headers',
+ parent = check_from_id,
+ type = 'virtual',
+ description = 'From header display name has a title (Mr/Mrs/Dr)',
+}
+rspamd_config:register_symbol {
+ name = 'FROM_EQ_ENVFROM',
+ score = 0.0,
+ group = 'headers',
+ parent = check_from_id,
+ type = 'virtual',
+ description = 'From address is the same as the envelope',
+}
+rspamd_config:register_symbol {
+ name = 'FROM_NEQ_ENVFROM',
+ score = 0.0,
+ group = 'headers',
+ parent = check_from_id,
+ type = 'virtual',
+ description = 'From address is different to the envelope',
+}
+rspamd_config:register_symbol {
+ name = 'TO_EQ_FROM',
+ score = 0.0,
+ group = 'headers',
+ parent = check_from_id,
+ type = 'virtual',
+ description = 'To address matches the From address',
+}
+rspamd_config:register_symbol {
+ name = 'TO_DOM_EQ_FROM_DOM',
+ score = 0.0,
+ group = 'headers',
+ parent = check_from_id,
+ type = 'virtual',
+ description = 'To domain is the same as the From domain',
+}
+
+local check_to_cc_id = rspamd_config:register_symbol {
+ name = 'CHECK_TO_CC',
+ type = 'callback',
+ score = 0.0,
+ group = 'headers,mime',
+ callback = function(task)
+ local rcpts = task:get_recipients(1)
+ local to = task:get_recipients(2)
+ local to_match_envrcpt = 0
+ local cnts = {
+ [1] = 'ONE',
+ [2] = 'TWO',
+ [3] = 'THREE',
+ [5] = 'FIVE',
+ [7] = 'SEVEN',
+ [12] = 'TWELVE',
+ [50] = 'GT_50'
+ }
+ local def = 'ZERO'
+ if (not to) then
+ return false
+ end
+ -- Add symbol for recipient count
+ local nrcpt = #to
+ for k, v in pairs(cnts) do
+ if nrcpt >= tonumber(k) then
+ def = v
+ end
+ end
+ task:insert_result('RCPT_COUNT_' .. def, 1.0, tostring(nrcpt))
+ -- Check for display names
+ local to_dn_count = 0
+ local to_dn_eq_addr_count = 0
+ for _, toa in ipairs(to) do
+ -- To: Recipients <noreply@dropbox.com>
+ if (toa['name'] and (toa['name']:lower() == 'recipient'
+ or toa['name']:lower() == 'recipients')) then
+ task:insert_result('TO_DN_RECIPIENTS', 1.0)
+ end
+ if (toa['name'] and util.strequal_caseless(toa['name'], toa['addr'])) then
+ to_dn_eq_addr_count = to_dn_eq_addr_count + 1
+ elseif (toa['name'] and toa['name'] ~= '') then
+ to_dn_count = to_dn_count + 1
+ end
+ -- See if header recipients match envrcpts
+ if (rcpts) then
+ for _, rcpt in ipairs(rcpts) do
+ if (toa and toa['addr'] and rcpt and rcpt['addr'] and
+ util.strequal_caseless(rcpt['addr'], toa['addr']))
+ then
+ to_match_envrcpt = to_match_envrcpt + 1
+ end
+ end
+ end
+ end
+ if (to_dn_count == 0 and to_dn_eq_addr_count == 0) then
+ task:insert_result('TO_DN_NONE', 1.0)
+ elseif (to_dn_count == #to) then
+ task:insert_result('TO_DN_ALL', 1.0)
+ elseif (to_dn_count > 0) then
+ task:insert_result('TO_DN_SOME', 1.0)
+ end
+ if (to_dn_eq_addr_count == #to) then
+ task:insert_result('TO_DN_EQ_ADDR_ALL', 1.0)
+ elseif (to_dn_eq_addr_count > 0) then
+ task:insert_result('TO_DN_EQ_ADDR_SOME', 1.0)
+ end
+
+ -- See if header recipients match envelope recipients
+ if (to_match_envrcpt == #to) then
+ task:insert_result('TO_MATCH_ENVRCPT_ALL', 1.0)
+ elseif (to_match_envrcpt > 0) then
+ task:insert_result('TO_MATCH_ENVRCPT_SOME', 1.0)
+ end
+ end
+}
+
+rspamd_config:register_symbol {
+ name = 'RCPT_COUNT_ZERO',
+ score = 0.0,
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = 'No recipients',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCPT_COUNT_ONE',
+ score = 0.0,
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = 'One recipient',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCPT_COUNT_TWO',
+ score = 0.0,
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = 'Two recipients',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCPT_COUNT_THREE',
+ score = 0.0,
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = '3-5 recipients',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCPT_COUNT_FIVE',
+ score = 0.0,
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = '5-7 recipients',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCPT_COUNT_SEVEN',
+ score = 0.0,
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = '7-11 recipients',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCPT_COUNT_TWELVE',
+ score = 0.0,
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = '12-50 recipients',
+ group = 'headers',
+}
+rspamd_config:register_symbol {
+ name = 'RCPT_COUNT_GT_50',
+ score = 0.0,
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = '50+ recipients',
+ group = 'headers',
+}
+
+rspamd_config:register_symbol {
+ name = 'TO_DN_RECIPIENTS',
+ score = 2.0,
+ group = 'headers',
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = 'To header display name is "Recipients"',
+}
+rspamd_config:register_symbol {
+ name = 'TO_DN_NONE',
+ score = 0.0,
+ group = 'headers',
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = 'None of the recipients have display names',
+}
+rspamd_config:register_symbol {
+ name = 'TO_DN_ALL',
+ score = 0.0,
+ group = 'headers',
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = 'All the recipients have display names',
+}
+rspamd_config:register_symbol {
+ name = 'TO_DN_SOME',
+ score = 0.0,
+ group = 'headers',
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = 'Some of the recipients have display names',
+}
+rspamd_config:register_symbol {
+ name = 'TO_DN_EQ_ADDR_ALL',
+ score = 0.0,
+ group = 'headers',
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = 'All of the recipients have display names that are the same as their address',
+}
+rspamd_config:register_symbol {
+ name = 'TO_DN_EQ_ADDR_SOME',
+ score = 0.0,
+ group = 'headers',
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = 'Some of the recipients have display names that are the same as their address',
+}
+rspamd_config:register_symbol {
+ name = 'TO_MATCH_ENVRCPT_ALL',
+ score = 0.0,
+ group = 'headers',
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = 'All of the recipients match the envelope',
+}
+rspamd_config:register_symbol {
+ name = 'TO_MATCH_ENVRCPT_SOME',
+ score = 0.0,
+ group = 'headers',
+ parent = check_to_cc_id,
+ type = 'virtual',
+ description = 'Some of the recipients match the envelope',
+}
+
+-- TODO: rewrite this rule, it should not touch headers directly
+rspamd_config.CTYPE_MISSING_DISPOSITION = {
+ callback = function(task)
+ local parts = task:get_parts()
+ if (not parts) or (parts and #parts < 1) then
+ return false
+ end
+ for _, p in ipairs(parts) do
+ local ct = p:get_header('Content-Type')
+ if (ct and ct:lower():match('^application/octet%-stream') ~= nil) then
+ local cd = p:get_header('Content-Disposition')
+ if (not cd) or (cd and cd:lower():find('^attachment') == nil) then
+ local ci = p:get_header('Content-ID')
+ if ci or (#parts > 1 and (cd and cd:find('filename=.+%.asc') ~= nil))
+ then
+ return false
+ end
+
+ local parent = p:get_parent()
+
+ if parent then
+ local t, st = parent:get_type()
+
+ if t == 'multipart' and st == 'encrypted' then
+ -- Special case
+ return false
+ end
+ end
+
+ return true
+ end
+ end
+ end
+ return false
+ end,
+ description = 'Binary content-type not specified as an attachment',
+ score = 4.0,
+ group = 'mime'
+}
+
+rspamd_config.CTYPE_MIXED_BOGUS = {
+ callback = function(task)
+ local ct = task:get_header('Content-Type')
+ if (not ct) then
+ return false
+ end
+ local parts = task:get_parts()
+ if (not parts) then
+ return false
+ end
+ if (not ct:lower():match('^multipart/mixed')) then
+ return false
+ end
+ local found = false
+ -- Check each part and look for a part that isn't multipart/* or text/plain or text/html
+ local ntext_parts = 0
+ for _, p in ipairs(parts) do
+ local mtype, _ = p:get_type()
+ if mtype then
+ if mtype == 'text' and not p:is_attachment() then
+ ntext_parts = ntext_parts + 1
+ if ntext_parts > 2 then
+ found = true
+ break
+ end
+ elseif mtype ~= 'multipart' then
+ found = true
+ break
+ end
+ end
+ end
+ if (not found) then
+ return true
+ end
+ return false
+ end,
+ description = 'multipart/mixed without non-textual part',
+ score = 1.0,
+ group = 'mime'
+}
+
+local function check_for_base64_text(part)
+ local ct = part:get_header('Content-Type')
+ if (not ct) then
+ return false
+ end
+ ct = ct:lower()
+ if (ct:match('^text')) then
+ -- Check encoding
+ local cte = part:get_header('Content-Transfer-Encoding')
+ if (cte and cte:lower():match('^base64')) then
+ return true
+ end
+ end
+ return false
+end
+
+rspamd_config.MIME_BASE64_TEXT = {
+ callback = function(task)
+ -- Check outer part
+ if (check_for_base64_text(task)) then
+ return true
+ else
+ local parts = task:get_parts()
+ if (not parts) then
+ return false
+ end
+ -- Check each part and look for base64 encoded text parts
+ for _, part in ipairs(parts) do
+ if (check_for_base64_text(part)) then
+ return true
+ end
+ end
+ end
+ return false
+ end,
+ description = 'Has text part encoded in base64',
+ score = 0.1,
+ group = 'mime'
+}
+
+rspamd_config.MIME_BASE64_TEXT_BOGUS = {
+ callback = function(task)
+ local parts = task:get_text_parts()
+ if (not parts) then
+ return false
+ end
+ -- Check each part and look for base64 encoded text parts
+ -- where the part does not have any 8bit characters within it
+ for _, part in ipairs(parts) do
+ local mimepart = part:get_mimepart();
+ if (check_for_base64_text(mimepart) and not part:has_8bit()) then
+ return true
+ end
+ end
+ return false
+ end,
+ description = 'Has text part encoded in base64 that does not contain any 8bit characters',
+ score = 1.0,
+ group = 'mime'
+}
+
+local function is_8bit_addr(addr)
+ if addr.flags and addr.flags['8bit'] then
+ return true
+ end
+
+ return false;
+end
+
+rspamd_config.INVALID_FROM_8BIT = {
+ callback = function(task)
+ local from = (task:get_from('mime') or {})[1] or {}
+ if is_8bit_addr(from) then
+ return true
+ end
+ return false
+ end,
+ description = 'Invalid 8bit character in From header',
+ score = 6.0,
+ group = 'headers'
+}
+
+rspamd_config.INVALID_RCPT_8BIT = {
+ callback = function(task)
+ local rcpts = task:get_recipients('mime') or {}
+ return fun.any(function(rcpt)
+ if is_8bit_addr(rcpt) then
+ return true
+ end
+ return false
+ end, rcpts)
+ end,
+ description = 'Invalid 8bit character in recipients headers',
+ score = 6.0,
+ group = 'headers'
+}
+
+rspamd_config.XM_CASE = {
+ callback = function(task)
+ return task:has_header('X-mailer', true)
+ end,
+ description = 'X-mailer .vs. X-Mailer',
+ score = 0.5,
+ group = 'headers'
+}
diff --git a/rules/html.lua b/rules/html.lua
new file mode 100644
index 0000000..7c352c2
--- /dev/null
+++ b/rules/html.lua
@@ -0,0 +1,462 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to you under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at:
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+local reconf = config['regexp']
+
+local rspamd_regexp = require "rspamd_regexp"
+
+-- Messages that have only HTML part
+reconf['MIME_HTML_ONLY'] = {
+ re = 'has_only_html_part()',
+ score = 0.2,
+ description = 'Message has only an HTML part',
+ group = 'headers'
+}
+
+local function has_anchor_parent(tag)
+ local parent = tag
+ repeat
+ parent = parent:get_parent()
+ if parent then
+ if parent:get_type() == 'a' then
+ return true
+ end
+ end
+ until not parent
+
+ return false
+end
+
+local function check_html_image(task, min, max)
+ local tp = task:get_text_parts()
+
+ for _, p in ipairs(tp) do
+ if p:is_html() then
+ local hc = p:get_html()
+ local len = p:get_length()
+
+ if hc and len >= min and len < max then
+ local images = hc:get_images()
+ if images then
+ for _, i in ipairs(images) do
+ local tag = i['tag']
+ if tag then
+ if has_anchor_parent(tag) then
+ -- do not trigger on small and unknown size images
+ if i['height'] + i['width'] >= 210 and i['embedded'] then
+ return true
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
+
+rspamd_config.HTML_SHORT_LINK_IMG_1 = {
+ callback = function(task)
+ return check_html_image(task, 0, 1024)
+ end,
+ score = 2.0,
+ group = 'html',
+ description = 'Short HTML part (0..1K) with a link to an image'
+}
+
+rspamd_config.HTML_SHORT_LINK_IMG_2 = {
+ callback = function(task)
+ return check_html_image(task, 1024, 1536)
+ end,
+ score = 1.0,
+ group = 'html',
+ description = 'Short HTML part (1K..1.5K) with a link to an image'
+}
+
+rspamd_config.HTML_SHORT_LINK_IMG_3 = {
+ callback = function(task)
+ return check_html_image(task, 1536, 2048)
+ end,
+ score = 0.5,
+ group = 'html',
+ description = 'Short HTML part (1.5K..2K) with a link to an image'
+}
+
+rspamd_config.R_EMPTY_IMAGE = {
+ callback = function(task)
+ local tp = task:get_text_parts() -- get text parts in a message
+
+ for _, p in ipairs(tp) do
+ -- iterate over text parts array using `ipairs`
+ if p:is_html() then
+ -- if the current part is html part
+ local hc = p:get_html() -- we get HTML context
+ local len = p:get_length() -- and part's length
+ if hc and len < 50 then
+ -- if we have a part that has less than 50 bytes of text
+ local images = hc:get_images() -- then we check for HTML images
+
+ if images then
+ -- if there are images
+ for _, i in ipairs(images) do
+ -- then iterate over images in the part
+ if i['height'] + i['width'] >= 400 then
+ -- if we have a large image
+ local tag = i['tag']
+ if tag then
+ if not has_anchor_parent(tag) then
+ return true
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end,
+
+ score = 2.0,
+ group = 'html',
+ description = 'Message contains empty parts and image'
+}
+
+rspamd_config.R_SUSPICIOUS_IMAGES = {
+ callback = function(task)
+ local tp = task:get_text_parts() -- get text parts in a message
+
+ for _, p in ipairs(tp) do
+ local h = p:get_html()
+
+ if h then
+ local l = p:get_words_count()
+ local img = h:get_images()
+ local pic_words = 0
+
+ if img then
+ for _, i in ipairs(img) do
+ local dim = i['width'] + i['height']
+ local tag = i['tag']
+
+ if tag then
+ if has_anchor_parent(tag) then
+ if dim > 100 and dim < 3000 then
+ -- We assume that a single picture 100x200 contains approx 3 words of text
+ pic_words = pic_words + dim / 100
+ end
+ end
+ end
+ end
+ end
+
+ if l + pic_words > 0 then
+ local rel = pic_words / (l + pic_words)
+
+ if rel > 0.5 then
+ return true, (rel - 0.5) * 2
+ end
+ end
+ end
+ end
+
+ return false
+ end,
+
+ score = 5.0,
+ group = 'html',
+ description = 'Message contains many suspicious messages'
+}
+
+local vis_check_id = rspamd_config:register_symbol {
+ name = 'HTML_VISIBLE_CHECKS',
+ type = 'callback',
+ group = 'html',
+ callback = function(task)
+ --local logger = require "rspamd_logger"
+ local tp = task:get_text_parts() -- get text parts in a message
+ local ret = false
+ local transp_rate = 0
+ local invisible_blocks = 0
+ local zero_size_blocks = 0
+ local arg
+
+ local normal_len = 0
+ local transp_len = 0
+
+ for _, p in ipairs(tp) do
+ -- iterate over text parts array using `ipairs`
+ normal_len = normal_len + p:get_length()
+ if p:is_html() and p:get_html() then
+ -- if the current part is html part
+ local hc = p:get_html() -- we get HTML context
+
+ hc:foreach_tag({ 'font', 'span', 'div', 'p', 'td' }, function(tag, clen, is_leaf)
+ local bl = tag:get_style()
+ if bl then
+ if not bl.visible and clen > 0 and is_leaf then
+ invisible_blocks = invisible_blocks + 1
+ end
+
+ if (bl.font_size or 12) == 0 and clen > 0 and is_leaf then
+ zero_size_blocks = zero_size_blocks + 1
+ end
+
+ if bl.transparent and is_leaf then
+ ret = true
+ invisible_blocks = invisible_blocks + 1 -- This block is invisible
+ transp_len = transp_len + clen
+ normal_len = normal_len - clen
+ local tr = transp_len / (normal_len + transp_len)
+ if tr > transp_rate then
+ transp_rate = tr
+ if not bl.color then
+ bl.color = { 0, 0, 0 }
+ end
+ if not bl.bgcolor then
+ bl.bgcolor = { 0, 0, 0 }
+ end
+ arg = string.format('%s color #%x%x%x bgcolor #%x%x%x',
+ tag:get_type(),
+ bl.color[1], bl.color[2], bl.color[3],
+ bl.bgcolor[1], bl.bgcolor[2], bl.bgcolor[3])
+ end
+ end
+ end
+
+ return false -- Continue search
+ end)
+
+ end
+ end
+
+ if ret then
+ transp_rate = transp_len / (normal_len + transp_len)
+
+ if transp_rate > 0.1 then
+ if transp_rate > 0.5 or transp_rate ~= transp_rate then
+ transp_rate = 0.5
+ end
+
+ task:insert_result('R_WHITE_ON_WHITE', (transp_rate * 2.0), arg)
+ end
+ end
+
+ if invisible_blocks > 0 then
+ if invisible_blocks > 10 then
+ invisible_blocks = 10
+ end
+ local rates = { -- From 1 to 10
+ 0.05,
+ 0.1,
+ 0.2,
+ 0.3,
+ 0.4,
+ 0.5,
+ 0.6,
+ 0.7,
+ 0.8,
+ 1.0,
+ }
+ task:insert_result('MANY_INVISIBLE_PARTS', rates[invisible_blocks],
+ tostring(invisible_blocks))
+ end
+
+ if zero_size_blocks > 0 then
+ if zero_size_blocks > 5 then
+ if zero_size_blocks > 10 then
+ -- Full score
+ task:insert_result('ZERO_FONT', 1.0,
+ tostring(zero_size_blocks))
+ else
+ zero_size_blocks = 5
+ end
+ end
+
+ if zero_size_blocks <= 5 then
+ local rates = { -- From 1 to 5
+ 0.1,
+ 0.2,
+ 0.2,
+ 0.3,
+ 0.5,
+ }
+ task:insert_result('ZERO_FONT', rates[zero_size_blocks],
+ tostring(zero_size_blocks))
+ end
+ end
+ end,
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = vis_check_id,
+ name = 'R_WHITE_ON_WHITE',
+ description = 'Message contains low contrast text',
+ score = 4.0,
+ group = 'html',
+ one_shot = true,
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = vis_check_id,
+ name = 'ZERO_FONT',
+ description = 'Zero sized font used',
+ score = 1.0, -- Reached if more than 5 elements have zero size
+ one_shot = true,
+ group = 'html'
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = vis_check_id,
+ name = 'MANY_INVISIBLE_PARTS',
+ description = 'Many parts are visually hidden',
+ score = 1.0, -- Reached if more than 10 elements are hidden
+ one_shot = true,
+ group = 'html'
+}
+
+rspamd_config.EXT_CSS = {
+ callback = function(task)
+ local regexp_lib = require "rspamd_regexp"
+ local re = regexp_lib.create_cached('/^.*\\.css(?:[?#].*)?$/i')
+ local tp = task:get_text_parts() -- get text parts in a message
+ local ret = false
+ for _, p in ipairs(tp) do
+ -- iterate over text parts array using `ipairs`
+ if p:is_html() and p:get_html() then
+ -- if the current part is html part
+ local hc = p:get_html() -- we get HTML context
+ hc:foreach_tag({ 'link' }, function(tag)
+ local bl = tag:get_extra()
+ if bl then
+ local s = tostring(bl)
+ if s and re:match(s) then
+ ret = true
+ end
+ end
+
+ return ret -- Continue search
+ end)
+
+ end
+ end
+
+ return ret
+ end,
+
+ score = 1.0,
+ group = 'html',
+ description = 'Message contains external CSS reference'
+}
+
+local https_re = rspamd_regexp.create_cached('/^https:/i')
+
+rspamd_config.HTTP_TO_HTTPS = {
+ callback = function(task)
+ local found_opts
+ local tp = task:get_text_parts() or {}
+
+ for _, p in ipairs(tp) do
+ if p:is_html() then
+ local hc = p:get_html()
+ if (not hc) then
+ return false
+ end
+
+ local found = false
+
+ hc:foreach_tag('a', function(tag, _)
+ -- Skip this loop if we already have a match
+ if (found) then
+ return true
+ end
+
+ local c = tag:get_content()
+ if (c) then
+ if (not https_re:match(c)) then
+ return false
+ end
+
+ local u = tag:get_extra()
+ if (not u) then
+ return false
+ end
+ local url_proto = u:get_protocol()
+
+ if url_proto ~= 'http' then
+ return false
+ end
+ -- Capture matches for http in href to https in visible part only
+ found = true
+ found_opts = u:get_host()
+ return true
+ end
+
+ return false
+ end)
+
+ if (found) then
+ return true, 1.0, found_opts
+ end
+
+ return false
+ end
+ end
+ return false
+ end,
+ description = 'The anchor text contains a distinct scheme compared to the target URL',
+ score = 0.5,
+ group = 'html'
+}
+
+rspamd_config.HTTP_TO_IP = {
+ callback = function(task)
+ local tp = task:get_text_parts()
+ if (not tp) then
+ return false
+ end
+ for _, p in ipairs(tp) do
+ if p:is_html() then
+ local hc = p:get_html()
+ if (not hc) then
+ return false
+ end
+ local found = false
+ hc:foreach_tag('a', function(tag, length)
+ if (found) then
+ return true
+ end
+ local u = tag:get_extra()
+ if (u) then
+ u = tostring(u):lower()
+ if (u:match('^https?://%d+%.%d+%.%d+%.%d+')) then
+ found = true
+ end
+ end
+ return false
+ end)
+ if found then
+ return true
+ end
+ return false
+ end
+ end
+ end,
+ description = 'HTML anchor points to an IP address',
+ score = 1.0,
+ group = 'html'
+}
diff --git a/rules/mid.lua b/rules/mid.lua
new file mode 100644
index 0000000..1bac26c
--- /dev/null
+++ b/rules/mid.lua
@@ -0,0 +1,131 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2016, Steve Freegard <steve@freegard.name>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+local rspamd_util = require "rspamd_util"
+local function mid_check_func(task)
+ local mid = task:get_header('Message-ID')
+ if not mid then
+ return false
+ end
+ -- Check for 'bare' IP addresses in RHS
+ if mid:find("@%d+%.%d+%.%d+%.%d+>$") then
+ task:insert_result('MID_BARE_IP', 1.0)
+ end
+ -- Check for non-FQDN RHS
+ if mid:find("@[^%.]+>?$") then
+ task:insert_result('MID_RHS_NOT_FQDN', 1.0)
+ end
+ -- Check for missing <>'s
+ if not mid:find('^<[^>]+>$') then
+ task:insert_result('MID_MISSING_BRACKETS', 1.0)
+ end
+ -- Check for IP literal in RHS
+ if mid:find("@%[%d+%.%d+%.%d+%.%d+%]") then
+ task:insert_result('MID_RHS_IP_LITERAL', 1.0)
+ end
+ -- Check From address attributes against MID
+ local from = task:get_from(2)
+ local fd
+ if (from and from[1] and from[1].domain and from[1].domain ~= '') then
+ fd = from[1].domain:lower()
+ local _, _, md = mid:find("@([^>]+)>?$")
+ -- See if all or part of the From address
+ -- can be found in the Message-ID
+ -- extract tld
+ local fdtld = nil
+ local mdtld = nil
+ if md then
+ fdtld = rspamd_util.get_tld(fd)
+ mdtld = rspamd_util.get_tld(md)
+ end
+ if (mid:lower():find(from[1].addr:lower(), 1, true)) then
+ task:insert_result('MID_CONTAINS_FROM', 1.0)
+ elseif (md and fd == md:lower()) then
+ task:insert_result('MID_RHS_MATCH_FROM', 1.0)
+ elseif (mdtld ~= nil and fdtld ~= nil and mdtld:lower() == fdtld) then
+ task:insert_result('MID_RHS_MATCH_FROMTLD', 1.0)
+ end
+ end
+ -- Check To address attributes against MID
+ local to = task:get_recipients(2)
+ if (to and to[1] and to[1].domain and to[1].domain ~= '') then
+ local td = to[1].domain:lower()
+ local _, _, md = mid:find("@([^>]+)>?$")
+ -- Skip if from domain == to domain
+ if ((fd and fd ~= td) or not fd) then
+ -- See if all or part of the To address
+ -- can be found in the Message-ID
+ if (mid:lower():find(to[1].addr:lower(), 1, true)) then
+ task:insert_result('MID_CONTAINS_TO', 1.0)
+ elseif (md and td == md:lower()) then
+ task:insert_result('MID_RHS_MATCH_TO', 1.0)
+ end
+ end
+ end
+end
+
+-- MID checks from Steve Freegard
+local check_mid_id = rspamd_config:register_symbol({
+ name = 'CHECK_MID',
+ score = 0.0,
+ group = 'mid',
+ type = 'callback,mime',
+ callback = mid_check_func
+})
+rspamd_config:register_virtual_symbol('MID_BARE_IP', 1.0, check_mid_id)
+rspamd_config:set_metric_symbol('MID_BARE_IP', 2.0, 'Message-ID RHS is a bare IP address', 'default', 'Message ID')
+rspamd_config:register_virtual_symbol('MID_RHS_NOT_FQDN', 1.0, check_mid_id)
+rspamd_config:set_metric_symbol('MID_RHS_NOT_FQDN', 0.5,
+ 'Message-ID RHS is not a fully-qualified domain name', 'default', 'Message ID')
+rspamd_config:register_virtual_symbol('MID_MISSING_BRACKETS', 1.0, check_mid_id)
+rspamd_config:set_metric_symbol('MID_MISSING_BRACKETS', 0.5, 'Message-ID is missing <>\'s', 'default', 'Message ID')
+rspamd_config:register_virtual_symbol('MID_RHS_IP_LITERAL', 1.0, check_mid_id)
+rspamd_config:set_metric_symbol('MID_RHS_IP_LITERAL', 0.5, 'Message-ID RHS is an IP-literal', 'default', 'Message ID')
+rspamd_config:register_virtual_symbol('MID_CONTAINS_FROM', 1.0, check_mid_id)
+rspamd_config:set_metric_symbol('MID_CONTAINS_FROM', 1.0, 'Message-ID contains From address', 'default', 'Message ID')
+rspamd_config:register_virtual_symbol('MID_RHS_MATCH_FROM', 1.0, check_mid_id)
+rspamd_config:set_metric_symbol('MID_RHS_MATCH_FROM', 0.0,
+ 'Message-ID RHS matches From domain', 'default', 'Message ID')
+rspamd_config:register_virtual_symbol('MID_RHS_MATCH_FROMTLD', 1.0, check_mid_id)
+rspamd_config:set_metric_symbol('MID_RHS_MATCH_FROMTLD', 0.0,
+ 'Message-ID RHS matches From domain tld', 'default', 'Message ID')
+rspamd_config:register_virtual_symbol('MID_CONTAINS_TO', 1.0, check_mid_id)
+rspamd_config:set_metric_symbol('MID_CONTAINS_TO', 1.0, 'Message-ID contains To address', 'default', 'Message ID')
+rspamd_config:register_virtual_symbol('MID_RHS_MATCH_TO', 1.0, check_mid_id)
+rspamd_config:set_metric_symbol('MID_RHS_MATCH_TO', 1.0, 'Message-ID RHS matches To domain', 'default', 'Message ID')
+
+-- Another check from https://github.com/rspamd/rspamd/issues/4299
+rspamd_config:register_symbol {
+ type = 'normal,mime',
+ group = 'mid',
+ name = 'MID_END_EQ_FROM_USER_PART',
+ description = 'Message-ID RHS (after @) and MIME from local part are the same',
+ score = 4.0,
+
+ callback = function(task)
+ local mid = task:get_header('Message-ID')
+ if not mid then
+ return
+ end
+ local mime_from = task:get_from('mime')
+ local _, _, mid_realm = mid:find("@([a-z]+)>?$")
+ if mid_realm and mime_from and mime_from[1] and mime_from[1].user then
+ if (mid_realm == mime_from[1].user) then
+ return true
+ end
+ end
+ end
+}
diff --git a/rules/misc.lua b/rules/misc.lua
new file mode 100644
index 0000000..faf4a8f
--- /dev/null
+++ b/rules/misc.lua
@@ -0,0 +1,864 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Misc rules
+
+local E = {}
+local fun = require "fun"
+local rspamd_util = require "rspamd_util"
+local rspamd_parsers = require "rspamd_parsers"
+local rspamd_regexp = require "rspamd_regexp"
+local lua_util = require "lua_util"
+local bit = require "bit"
+local rspamd_url = require "rspamd_url"
+local url_flags_tab = rspamd_url.flags
+
+-- Different text parts
+rspamd_config.R_PARTS_DIFFER = {
+ callback = function(task)
+ local distance = task:get_mempool():get_variable('parts_distance', 'double')
+
+ if distance then
+ local nd = tonumber(distance)
+ -- ND is relation of different words to total words
+ if nd >= 0.5 then
+ local tw = task:get_mempool():get_variable('total_words', 'int')
+
+ if tw then
+ local score
+ if tw > 30 then
+ -- We are confident about difference
+ score = (nd - 0.5) * 2.0
+ else
+ -- We are not so confident about difference
+ score = (nd - 0.5)
+ end
+ task:insert_result('R_PARTS_DIFFER', score,
+ string.format('%.1f%%', tostring(100.0 * nd)))
+ end
+ end
+ end
+ return false
+ end,
+ score = 1.0,
+ description = 'Text and HTML parts differ',
+ group = 'body'
+}
+
+-- Date issues
+local date_id = rspamd_config:register_symbol({
+ name = 'DATE_CB',
+ type = 'callback,mime',
+ callback = function(task)
+ local date_time = task:get_header('Date')
+ if date_time == nil or date_time == '' then
+ task:insert_result('MISSING_DATE', 1.0)
+ return
+ end
+
+ local dm, err = rspamd_parsers.parse_smtp_date(date_time)
+ if err then
+ task:insert_result('INVALID_DATE', 1.0)
+ return
+ end
+
+ local dt = task:get_date({ format = 'connect', gmt = true })
+ local date_diff = dt - dm
+
+ if date_diff > 86400 then
+ -- Older than a day
+ task:insert_result('DATE_IN_PAST', 1.0, tostring(math.floor(date_diff / 3600)))
+ elseif -date_diff > 7200 then
+ -- More than 2 hours in the future
+ task:insert_result('DATE_IN_FUTURE', 1.0, tostring(math.floor(-date_diff / 3600)))
+ end
+ end
+})
+
+rspamd_config:register_symbol({
+ name = 'MISSING_DATE',
+ score = 1.0,
+ description = 'Date header is missing',
+ group = 'headers',
+ type = 'virtual',
+ parent = date_id,
+})
+
+rspamd_config:register_symbol({
+ name = 'INVALID_DATE',
+ score = 1.5,
+ description = 'Malformed Date header',
+ group = 'headers',
+ type = 'virtual',
+ parent = date_id,
+})
+
+rspamd_config:register_symbol({
+ name = 'DATE_IN_FUTURE',
+ score = 4.0,
+ description = 'Message date is in the future',
+ group = 'headers',
+ type = 'virtual',
+ parent = date_id,
+})
+
+rspamd_config:register_symbol({
+ name = 'DATE_IN_PAST',
+ score = 1.0,
+ description = 'Message date is in the past',
+ group = 'headers',
+ type = 'virtual',
+ parent = date_id,
+})
+
+local obscured_id = rspamd_config:register_symbol {
+ callback = function(task)
+ local susp_urls = task:get_urls_filtered({ 'obscured', 'zw_spaces' })
+
+ if susp_urls and susp_urls[1] then
+ local obs_flag = url_flags_tab.obscured
+ local zw_flag = url_flags_tab.zw_spaces
+
+ for _, u in ipairs(susp_urls) do
+ local fl = u:get_flags_num()
+ if bit.band(fl, obs_flag) ~= 0 then
+ task:insert_result('R_SUSPICIOUS_URL', 1.0, u:get_host())
+ end
+ if bit.band(fl, zw_flag) ~= 0 then
+ task:insert_result('ZERO_WIDTH_SPACE_URL', 1.0, u:get_host())
+ end
+ end
+ end
+
+ return false
+ end,
+ name = 'R_SUSPICIOUS_URL',
+ score = 5.0,
+ one_shot = true,
+ description = 'A message has been identified to contain an obfuscated or suspicious URL',
+ group = 'url'
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ name = 'ZERO_WIDTH_SPACE_URL',
+ score = 7.0,
+ one_shot = true,
+ description = 'Zero width space in URL',
+ group = 'url',
+ parent = obscured_id,
+}
+
+rspamd_config.ENVFROM_PRVS = {
+ callback = function(task)
+ --[[
+ Detect PRVS/BATV addresses to avoid FORGED_SENDER
+ https://en.wikipedia.org/wiki/Bounce_Address_Tag_Validation
+
+ Signature syntax:
+
+ prvs=TAG=USER@example.com BATV draft (https://tools.ietf.org/html/draft-levine-smtp-batv-01)
+ prvs=USER=TAG@example.com
+ btv1==TAG==USER@example.com Barracuda appliance
+ msprvs1=TAG=USER@example.com Sparkpost email delivery service
+ ]]--
+ if not (task:has_from(1) and task:has_from(2)) then
+ return false
+ end
+ local envfrom = task:get_from(1)
+ local re_text = '^(?:(prvs|msprvs1)=([^=]+)=|btv1==[^=]+==)(.+@(.+))$'
+ local re = rspamd_regexp.create_cached(re_text)
+ local c = re:search(envfrom[1].addr:lower(), false, true)
+ if not c then
+ return false
+ end
+ local ef = c[1][4]
+ -- See if it matches the From header
+ local from = task:get_from(2)
+ if ef == from[1].addr:lower() then
+ return true
+ end
+ -- Check for prvs=USER=TAG@example.com
+ local t = c[1][2]
+ if t == 'prvs' then
+ local efr = c[1][3] .. '@' .. c[1][5]
+ if efr == from[1].addr:lower() then
+ return true
+ end
+ end
+ return false
+ end,
+ score = 0.0,
+ description = "Envelope From is a PRVS address that matches the From address",
+ group = 'headers',
+ type = 'mime',
+}
+
+rspamd_config.ENVFROM_VERP = {
+ callback = function(task)
+ if not (task:has_from(1) and task:has_recipients(1)) then
+ return false
+ end
+ local envfrom = task:get_from(1)
+ local envrcpts = task:get_recipients(1)
+ -- VERP only works for single recipient messages
+ if #envrcpts > 1 then
+ return false
+ end
+ -- Get recipient and compute VERP address
+ local rcpt = envrcpts[1].addr:lower()
+ local verp = rcpt:gsub('@', '=')
+ -- Get the user portion of the envfrom
+ local ef_user = envfrom[1].user:lower()
+ -- See if the VERP representation of the recipient appears in it
+ if ef_user:find(verp, 1, true)
+ and not ef_user:find('+caf_=' .. verp, 1, true) -- Google Forwarding
+ and not ef_user:find('^srs[01]=') -- SRS
+ then
+ return true
+ end
+ return false
+ end,
+ score = 0.0,
+ description = "Envelope From is a VERP address",
+ group = "headers",
+ type = 'mime',
+}
+
+local check_rcvd = rspamd_config:register_symbol {
+ name = 'CHECK_RCVD',
+ group = 'headers',
+ callback = function(task)
+ local rcvds = task:get_received_headers()
+ if not rcvds or #rcvds == 0 then
+ return false
+ end
+
+ local all_tls = fun.all(function(rc)
+ return rc.flags and rc.flags['ssl']
+ end, fun.filter(function(rc)
+ return rc.by_hostname and rc.by_hostname ~= 'localhost'
+ end, rcvds))
+
+ -- See if only the last hop was encrypted
+ if all_tls then
+ task:insert_result('RCVD_TLS_ALL', 1.0)
+ else
+ local rcvd = rcvds[1]
+ if rcvd.by_hostname and rcvd.by_hostname == 'localhost' then
+ -- Ignore artificial header from Rmilter
+ rcvd = rcvds[2] or {}
+ end
+ if rcvd.flags and rcvd.flags['ssl'] then
+ task:insert_result('RCVD_TLS_LAST', 1.0)
+ else
+ task:insert_result('RCVD_NO_TLS_LAST', 1.0)
+ end
+ end
+
+ local auth = fun.any(function(rc)
+ return rc.flags and rc.flags['authenticated']
+ end, rcvds)
+
+ if auth then
+ task:insert_result('RCVD_VIA_SMTP_AUTH', 1.0)
+ end
+ end,
+ type = 'callback,mime',
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = check_rcvd,
+ name = 'RCVD_TLS_ALL',
+ description = 'All hops used encrypted transports',
+ score = 0.0,
+ group = 'headers'
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = check_rcvd,
+ name = 'RCVD_TLS_LAST',
+ description = 'Last hop used encrypted transports',
+ score = 0.0,
+ group = 'headers'
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = check_rcvd,
+ name = 'RCVD_NO_TLS_LAST',
+ description = 'Last hop did not use encrypted transports',
+ score = 0.1,
+ group = 'headers'
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = check_rcvd,
+ name = 'RCVD_VIA_SMTP_AUTH',
+ -- NB This does not mean sender was authenticated; see task:get_user()
+ description = 'Authenticated hand-off was seen in Received headers',
+ score = 0.0,
+ group = 'headers'
+}
+
+rspamd_config.RCVD_HELO_USER = {
+ callback = function(task)
+ -- Check HELO argument from MTA
+ local helo = task:get_helo()
+ if (helo and helo:lower():find('^user$')) then
+ return true
+ end
+ -- Check Received headers
+ local rcvds = task:get_header_full('Received')
+ if not rcvds then
+ return false
+ end
+ for _, rcvd in ipairs(rcvds) do
+ local r = rcvd['decoded']:lower()
+ if (r:find("^%s*from%suser%s")) then
+ return true
+ end
+ if (r:find("helo[%s=]user[%s%)]")) then
+ return true
+ end
+ end
+ end,
+ description = 'HELO User spam pattern',
+ group = 'headers',
+ type = 'mime',
+ score = 3.0
+}
+
+rspamd_config.URI_COUNT_ODD = {
+ callback = function(task)
+ local ct = task:get_header('Content-Type')
+ if (ct and ct:lower():find('^multipart/alternative')) then
+ local urls = task:get_urls_filtered(nil, { 'subject', 'html_displayed', 'special' }) or {}
+ local nurls = fun.foldl(function(acc, val)
+ return acc + val:get_count()
+ end, 0, urls)
+
+ if nurls % 2 == 1 then
+ return true, 1.0, tostring(nurls)
+ end
+ end
+ end,
+ description = 'Odd number of URIs in multipart/alternative message',
+ score = 1.0,
+ group = 'url',
+}
+
+rspamd_config.HAS_ATTACHMENT = {
+ callback = function(task)
+ local parts = task:get_parts()
+ if parts and #parts > 1 then
+ for _, p in ipairs(parts) do
+ local cd = p:get_header('Content-Disposition')
+ if (cd and cd:lower():match('^attachment')) then
+ return true
+ end
+ end
+ end
+ end,
+ description = 'Message contains attachments',
+ group = 'body',
+}
+
+-- Requires freemail maps loaded in multimap
+local function freemail_reply_neq_from(task)
+ if not task:has_symbol('FREEMAIL_REPLYTO') or not task:has_symbol('FREEMAIL_FROM') then
+ return false
+ end
+ local frt = task:get_symbol('FREEMAIL_REPLYTO')
+ local ff = task:get_symbol('FREEMAIL_FROM')
+ local frt_opts = frt[1]['options']
+ local ff_opts = ff[1]['options']
+ return (frt_opts and ff_opts and frt_opts[1] ~= ff_opts[1])
+end
+
+rspamd_config:register_symbol({
+ name = 'FREEMAIL_REPLYTO_NEQ_FROM_DOM',
+ callback = freemail_reply_neq_from,
+ description = 'The From and Reply-To addresses in the email are from different freemail services',
+ score = 3.0,
+ group = 'headers',
+})
+rspamd_config:register_dependency('FREEMAIL_REPLYTO_NEQ_FROM_DOM', 'FREEMAIL_REPLYTO')
+rspamd_config:register_dependency('FREEMAIL_REPLYTO_NEQ_FROM_DOM', 'FREEMAIL_FROM')
+
+rspamd_config.OMOGRAPH_URL = {
+ callback = function(task)
+ local urls = task:get_urls()
+
+ if urls then
+ local bad_omographs = 0
+ local single_bad_omograps = 0
+ local bad_urls = {}
+ local seen = {}
+
+ fun.each(function(u)
+ if u:is_phished() then
+
+ local h1 = u:get_host()
+ local h2 = u:get_phished()
+ if h2 then
+ -- Due to changes of the phished flag in 2.8
+ h2 = h2:get_host()
+ end
+ if h1 and h2 then
+ local selt = string.format('%s->%s', h1, h2)
+ if not seen[selt] and rspamd_util.is_utf_spoofed(h1, h2) then
+ bad_urls[#bad_urls + 1] = selt
+ bad_omographs = bad_omographs + 1
+ end
+ seen[selt] = true
+ end
+ end
+ if not u:is_html_displayed() then
+ local h = u:get_tld()
+
+ if h then
+ if not seen[h] and rspamd_util.is_utf_spoofed(h) then
+ bad_urls[#bad_urls + 1] = h
+ single_bad_omograps = single_bad_omograps + 1
+ end
+ seen[h] = true
+ end
+ end
+ end, urls)
+
+ if bad_omographs > 0 then
+ return true, 1.0, bad_urls
+ elseif single_bad_omograps > 0 then
+ return true, 0.5, bad_urls
+ end
+ end
+
+ return false
+ end,
+ score = 5.0,
+ group = 'url',
+ description = 'URL contains both latin and non-latin characters'
+}
+
+rspamd_config.URL_IN_SUBJECT = {
+ callback = function(task)
+ local urls = task:get_urls()
+
+ if urls then
+ for _, u in ipairs(urls) do
+ local flags = u:get_flags()
+ if flags.subject then
+ if flags.schemaless then
+ return true, 0.1, u:get_host()
+ end
+ local subject = task:get_subject()
+
+ if subject then
+ if tostring(u) == subject then
+ return true, 1.0, u:get_host()
+ end
+ end
+ return true, 0.25, u:get_host()
+ end
+ end
+ end
+
+ return false
+ end,
+ score = 4.0,
+ group = 'subject',
+ type = 'mime',
+ description = 'Subject contains URL'
+}
+
+local aliases_id = rspamd_config:register_symbol {
+ type = 'prefilter',
+ name = 'EMAIL_PLUS_ALIASES',
+ callback = function(task)
+ local function check_from(type)
+ if task:has_from(type) then
+ local addr = task:get_from(type)[1]
+ local na, tags = lua_util.remove_email_aliases(addr)
+ if na then
+ task:set_from(type, addr, 'alias')
+ task:insert_result('TAGGED_FROM', 1.0, fun.totable(
+ fun.filter(function(t)
+ return t and #t > 0
+ end, tags)))
+ end
+ end
+ end
+
+ check_from('smtp')
+ check_from('mime')
+
+ local function check_rcpt(type)
+ if task:has_recipients(type) then
+ local modified = false
+ local all_tags = {}
+ local addrs = task:get_recipients(type)
+
+ for _, addr in ipairs(addrs) do
+ local na, tags = lua_util.remove_email_aliases(addr)
+ if na then
+ modified = true
+ fun.each(function(t)
+ table.insert(all_tags, t)
+ end,
+ fun.filter(function(t)
+ return t and #t > 0
+ end, tags))
+ end
+ end
+
+ if modified then
+ task:set_recipients(type, addrs, 'alias')
+ task:insert_result('TAGGED_RCPT', 1.0, all_tags)
+ end
+ end
+ end
+
+ check_rcpt('smtp')
+ check_rcpt('mime')
+ end,
+ priority = lua_util.symbols_priorities.top + 1,
+ description = 'Removes plus aliases from the email',
+ group = 'headers',
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = aliases_id,
+ name = 'TAGGED_RCPT',
+ description = 'SMTP recipients have plus tags',
+ group = 'headers',
+ score = 0.0,
+}
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = aliases_id,
+ name = 'TAGGED_FROM',
+ description = 'SMTP from has plus tags',
+ group = 'headers',
+ score = 0.0,
+}
+
+local check_from_display_name = rspamd_config:register_symbol {
+ type = 'callback,mime',
+ name = 'FROM_DISPLAY_CALLBACK',
+ callback = function(task)
+ local from = task:get_from(2)
+ if not (from and from[1] and from[1].name) then
+ return false
+ end
+ -- See if we can parse an email address from the name
+ local parsed = rspamd_parsers.parse_mail_address(from[1].name, task:get_mempool())
+ if not parsed then
+ return false
+ end
+ if not (parsed[1] and parsed[1]['addr']) then
+ return false
+ end
+ -- Make sure we did not mistake e.g. <something>@<name> for an email address
+ if not parsed[1]['domain'] or not parsed[1]['domain']:find('%.') then
+ return false
+ end
+ -- See if the parsed domains differ
+ if not rspamd_util.strequal_caseless(from[1]['domain'], parsed[1]['domain']) then
+ -- See if the destination domain is the same as the spoof
+ local mto = task:get_recipients(2)
+ local sto = task:get_recipients(1)
+ if mto then
+ for _, to in ipairs(mto) do
+ if to['domain'] ~= '' and rspamd_util.strequal_caseless(to['domain'], parsed[1]['domain']) then
+ task:insert_result('SPOOF_DISPLAY_NAME', 1.0, from[1]['domain'], parsed[1]['domain'])
+ return false
+ end
+ end
+ end
+ if sto then
+ for _, to in ipairs(sto) do
+ if to['domain'] ~= '' and rspamd_util.strequal_caseless(to['domain'], parsed[1]['domain']) then
+ task:insert_result('SPOOF_DISPLAY_NAME', 1.0, from[1]['domain'], parsed[1]['domain'])
+ return false
+ end
+ end
+ end
+ task:insert_result('FROM_NEQ_DISPLAY_NAME', 1.0, from[1]['domain'], parsed[1]['domain'])
+ end
+ return false
+ end,
+ group = 'headers',
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = check_from_display_name,
+ name = 'SPOOF_DISPLAY_NAME',
+ description = 'Display name is being used to spoof and trick the recipient',
+ group = 'headers',
+ score = 8.0,
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = check_from_display_name,
+ name = 'FROM_NEQ_DISPLAY_NAME',
+ group = 'headers',
+ description = 'Display name contains an email address different to the From address',
+ score = 4.0,
+}
+
+rspamd_config.SPOOF_REPLYTO = {
+ callback = function(task)
+ -- First check for a Reply-To header
+ local rt = task:get_header_full('Reply-To')
+ if not rt or not rt[1] then
+ return false
+ end
+ -- Get From and To headers
+ rt = rt[1]['value']
+ local from = task:get_from(2)
+ local to = task:get_recipients(2)
+ if not (from and from[1] and from[1].addr) then
+ return false
+ end
+ if (to and to[1] and to[1].addr) then
+ -- Handle common case for Web Contact forms of From = To
+ if rspamd_util.strequal_caseless(from[1].addr, to[1].addr) then
+ return false
+ end
+ end
+ -- SMTP recipients must contain From domain
+ to = task:get_recipients(1)
+ if not to then
+ return false
+ end
+ -- Try mitigate some possible FPs on mailing list posts
+ if #to == 1 and rspamd_util.strequal_caseless(to[1].addr, from[1].addr) then
+ return false
+ end
+ local found_fromdom = false
+ for _, t in ipairs(to) do
+ if rspamd_util.strequal_caseless(t.domain, from[1].domain) then
+ found_fromdom = true
+ break
+ end
+ end
+ if not found_fromdom then
+ return false
+ end
+ -- Parse Reply-To header
+ local parsed = ((rspamd_parsers.parse_mail_address(rt, task:get_mempool()) or E)[1] or E).domain
+ if not parsed then
+ return false
+ end
+ -- Reply-To domain must be different to From domain
+ if not rspamd_util.strequal_caseless(parsed, from[1].domain) then
+ return true, from[1].domain, parsed
+ end
+ return false
+ end,
+ group = 'headers',
+ type = 'mime',
+ description = 'Reply-To is being used to spoof and trick the recipient to send an off-domain reply',
+ score = 6.0
+}
+
+rspamd_config.INFO_TO_INFO_LU = {
+ callback = function(task)
+ if not task:has_header('List-Unsubscribe') then
+ return false
+ end
+ local from = task:get_from('mime')
+ if not (from and from[1] and rspamd_util.strequal_caseless(from[1].user, 'info')) then
+ return false
+ end
+ local to = task:get_recipients('smtp')
+ if not to then
+ return false
+ end
+ local found = false
+ for _, r in ipairs(to) do
+ if rspamd_util.strequal_caseless(r['user'], 'info') then
+ found = true
+ end
+ end
+ if found then
+ return true
+ end
+ return false
+ end,
+ description = 'info@ From/To address with List-Unsubscribe headers',
+ group = 'headers',
+ score = 2.0,
+ type = 'mime',
+}
+
+-- Detects bad content-transfer-encoding for text parts
+
+rspamd_config.R_BAD_CTE_7BIT = {
+ callback = function(task)
+ local tp = task:get_text_parts() or {}
+
+ for _, p in ipairs(tp) do
+ local cte = p:get_mimepart():get_cte() or ''
+ if cte ~= '8bit' and p:has_8bit_raw() then
+ local _, _, attrs = p:get_mimepart():get_type_full()
+ local mul = 1.0
+ local params = { cte }
+ if attrs then
+ if attrs.charset and attrs.charset:lower() == "utf-8" then
+ -- Penalise rule as people don't know that utf8 is surprisingly
+ -- eight bit encoding
+ mul = 0.3
+ table.insert(params, "utf8")
+ end
+ end
+
+ return true, mul, params
+ end
+ end
+
+ return false
+ end,
+ score = 3.5,
+ description = 'Detects bad Content-Transfer-Encoding for text parts',
+ group = 'headers',
+ type = 'mime',
+}
+
+local check_encrypted_name = rspamd_config:register_symbol {
+ name = 'BOGUS_ENCRYPTED_AND_TEXT',
+ callback = function(task)
+ local parts = task:get_parts() or {}
+ local seen_encrypted, seen_text
+ local opts = {}
+
+ local function check_part(part)
+ if part:is_multipart() then
+ local children = part:get_children() or {}
+ local text_kids = {}
+
+ for _, cld in ipairs(children) do
+ if cld:is_multipart() then
+ check_part(cld)
+ elseif cld:is_text() then
+ seen_text = true
+ text_kids[#text_kids + 1] = cld
+ else
+ local type, subtype, _ = cld:get_type_full()
+
+ if type:lower() == 'application' then
+ if string.find(subtype:lower(), 'pkcs7%-mime') then
+ -- S/MIME encrypted part
+ seen_encrypted = true
+ table.insert(opts, 'smime part')
+ task:insert_result('ENCRYPTED_SMIME', 1.0)
+ elseif string.find(subtype:lower(), 'pkcs7%-signature') then
+ task:insert_result('SIGNED_SMIME', 1.0)
+ elseif string.find(subtype:lower(), 'pgp%-encrypted') then
+ -- PGP/GnuPG encrypted part
+ seen_encrypted = true
+ table.insert(opts, 'pgp part')
+ task:insert_result('ENCRYPTED_PGP', 1.0)
+ elseif string.find(subtype:lower(), 'pgp%-signature') then
+ task:insert_result('SIGNED_PGP', 1.0)
+ end
+ end
+ end
+ if seen_text and seen_encrypted then
+ -- Ensure that our seen text is not really part of pgp #3205
+ for _, tp in ipairs(text_kids) do
+ local t, _ = tp:get_type()
+ seen_text = false -- reset temporary
+ if t and t == 'text' then
+ seen_text = true
+ break
+ end
+ end
+ end
+ end
+ end
+ end
+
+ for _, part in ipairs(parts) do
+ check_part(part)
+ end
+
+ if seen_text and seen_encrypted then
+ return true, 1.0, opts
+ end
+
+ return false
+ end,
+ score = 10.0,
+ description = 'Bogus mix of encrypted and text/html payloads',
+ group = 'mime_types',
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = check_encrypted_name,
+ name = 'ENCRYPTED_PGP',
+ description = 'Message is encrypted with PGP',
+ group = 'mime_types',
+ score = -0.5,
+ one_shot = true
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = check_encrypted_name,
+ name = 'ENCRYPTED_SMIME',
+ description = 'Message is encrypted with S/MIME',
+ group = 'mime_types',
+ score = -0.5,
+ one_shot = true
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = check_encrypted_name,
+ name = 'SIGNED_PGP',
+ description = 'Message is signed with PGP',
+ group = 'mime_types',
+ score = -2.0,
+ one_shot = true
+}
+
+rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = check_encrypted_name,
+ name = 'SIGNED_SMIME',
+ description = 'Message is signed with S/MIME',
+ group = 'mime_types',
+ score = -2.0,
+ one_shot = true
+}
+
+rspamd_config.COMPLETELY_EMPTY = {
+ callback = function(task)
+ return (task:get_size() == 0)
+ end,
+ flags = 'empty',
+ group = 'blankspam',
+ score = 15
+}
diff --git a/rules/parts.lua b/rules/parts.lua
new file mode 100644
index 0000000..2be9ff8
--- /dev/null
+++ b/rules/parts.lua
@@ -0,0 +1,11 @@
+rspamd_config.SINGLE_SHORT_PART = {
+ callback = function(task)
+ local parts = task:get_parts()
+ if #parts ~= 1 then return end
+ local text = parts[1]:get_text()
+ if not text then return end
+ if text:get_length() >= 64 then return end
+ return true
+ end,
+ score = 0.0,
+}
diff --git a/rules/regexp/compromised_hosts.lua b/rules/regexp/compromised_hosts.lua
new file mode 100644
index 0000000..e120b18
--- /dev/null
+++ b/rules/regexp/compromised_hosts.lua
@@ -0,0 +1,223 @@
+local reconf = config['regexp']
+local rspamd_regexp = require 'rspamd_regexp'
+local util = require 'rspamd_util'
+
+reconf['HAS_PHPMAILER_SIG'] = {
+ -- PHPMailer 6.0.0 and older used hex hash in boundary:
+ -- boundary="b1_2a45d5e29f78d3408e318878b049f474"
+ -- Since 6.0.1 it uses base64 (without =+/):
+ -- boundary="b1_uBN0UPD3n6RU04VPxI54tENiDgaCGoh15l9s73oFnlM"
+ -- boundary="b1_Ez5tmpb4bSqknyUZ1B1hIvLAfR1MlspDEKGioCOXc"
+ -- https://github.com/PHPMailer/PHPMailer/blob/v6.4.0/src/PHPMailer.php#L2660
+ re = [[X-Mailer=/^PHPMailer /H || Content-Type=/boundary="b1_[0-9a-zA-Z]+"/H]],
+ description = "PHPMailer signature",
+ group = "compromised_hosts"
+}
+
+reconf['PHP_SCRIPT_ROOT'] = {
+ re = "X-PHP-Originating-Script=/^0:/Hi",
+ description = "PHP Script executed by root UID",
+ score = 1.0,
+ group = "compromised_hosts"
+}
+
+reconf['HAS_X_POS'] = {
+ re = "header_exists('X-PHP-Originating-Script')",
+ description = "Has X-PHP-Originating-Script header",
+ group = "compromised_hosts"
+}
+
+reconf['HAS_X_PHP_SCRIPT'] = {
+ re = "header_exists('X-PHP-Script')",
+ description = "Has X-PHP-Script header",
+ group = "compromised_hosts"
+}
+
+-- X-Source:
+-- X-Source-Args: /usr/sbin/proxyexec -q -d -s /var/run/proxyexec/cagefs.sock/socket /bin/cagefs.server
+-- X-Source-Dir: silvianimberg.com:/public_html/wp-content/themes/ultimatum
+reconf['HAS_X_SOURCE'] = {
+ re = "header_exists('X-Source') || header_exists('X-Source-Args') || header_exists('X-Source-Dir')",
+ description = "Has X-Source headers",
+ group = "compromised_hosts"
+}
+
+-- X-Authenticated-Sender: accord.host-care.com: sales@cortaflex.si
+rspamd_config.HAS_X_AS = {
+ callback = function(task)
+ local xas = task:get_header('X-Authenticated-Sender')
+ if not xas then
+ return false
+ end
+ local _, _, auth = xas:find('[^:]+:%s(.+)$')
+ if auth then
+ -- TODO: see if we can parse an e-mail address from auth
+ -- and see if it matches the from address or not
+ return true, auth
+ else
+ return true
+ end
+ end,
+ description = 'Has X-Authenticated-Sender header',
+ group = "compromised_hosts",
+ score = 0.0
+}
+
+-- X-Get-Message-Sender-Via: accord.host-care.com: authenticated_id: sales@cortaflex.si
+rspamd_config.HAS_X_GMSV = {
+ callback = function(task)
+ local xgmsv = task:get_header('X-Get-Message-Sender-Via')
+ if not xgmsv then
+ return false
+ end
+ local _, _, auth = xgmsv:find('authenticated_id: (.+)$')
+ if auth then
+ -- TODO: see if we can parse an e-mail address from auth
+ -- and see if it matches the from address or not.
+ return true, auth
+ else
+ return true
+ end
+ end,
+ description = 'Has X-Get-Message-Sender-Via: header',
+ group = "compromised_hosts",
+ score = 0.0,
+}
+
+-- X-AntiAbuse: This header was added to track abuse, please include it with any abuse report
+-- X-AntiAbuse: Primary Hostname - accord.host-care.com
+-- X-AntiAbuse: Original Domain - swaney.com
+-- X-AntiAbuse: Originator/Caller UID/GID - [47 12] / [47 12]
+-- X-AntiAbuse: Sender Address Domain - dropbox.com
+reconf['HAS_X_ANTIABUSE'] = {
+ re = "header_exists('X-AntiAbuse')",
+ description = "Has X-AntiAbuse headers",
+ group = "compromised_hosts"
+}
+
+reconf['X_PHP_EVAL'] = {
+ re = [[X-PHP-Script=/eval\(\)'d code/H || X-PHP-Originating-Script=/eval\(\)'d code/H]],
+ description = "Message sent using eval'd PHP",
+ score = 4.0,
+ group = "compromised_hosts"
+}
+
+reconf['HAS_WP_URI'] = {
+ re = '/\\/wp-[^\\/]+\\//Ui',
+ description = "Contains WordPress URIs",
+ one_shot = true,
+ group = "compromised_hosts"
+}
+
+reconf['WP_COMPROMISED'] = {
+ re = '/\\/wp-(?:content|includes)[^\\/]+\\//Ui',
+ description = "URL that is pointing to a compromised WordPress installation",
+ one_shot = true,
+ group = "compromised_hosts"
+}
+
+reconf['PHP_XPS_PATTERN'] = {
+ re = 'X-PHP-Script=/^[^\\. ]+\\.[^\\.\\/ ]+\\/sendmail\\.php\\b/Hi',
+ description = "Message contains X-PHP-Script pattern",
+ group = "compromised_hosts"
+}
+
+reconf['HAS_XAW'] = {
+ re = "header_exists('X-Authentication-Warning')",
+ description = "Has X-Authentication-Warning header",
+ group = "compromised_hosts"
+}
+
+-- X-Authentication-Warning: localhost.localdomain: www-data set sender to info@globalstock.lv using -f
+reconf['XAW_SERVICE_ACCT'] = {
+ re = "X-Authentication-Warning=/\\b(?:www-data|anonymous|ftp|apache|nobody|guest|nginx|web|www) set sender to\\b/Hi",
+ description = "Message originally from a service account",
+ score = 1.0,
+ group = "compromised_hosts"
+}
+
+reconf['ENVFROM_SERVICE_ACCT'] = {
+ re = "check_smtp_data('from',/^(?:www-data|anonymous|ftp|apache|nobody|guest|nginx|web|www)@/i)",
+ description = "Envelope from is a service account",
+ score = 1.0,
+ group = "compromised_hosts"
+}
+
+reconf['HIDDEN_SOURCE_OBJ'] = {
+ re = "X-PHP-Script=/\\/\\..+/Hi || X-PHP-Originating-Script=/(?:^\\d+:|\\/)\\..+/Hi || X-Source-Args=/\\/\\..+/Hi",
+ description = "UNIX hidden file/directory in path",
+ score = 2.0,
+ group = "compromised_hosts"
+}
+
+local hidden_uri_re = rspamd_regexp.create_cached('/(?!\\/\\.well[-_]known\\/)(?:^\\.[A-Za-z0-9]|\\/' ..
+ '\\.[A-Za-z0-9]|\\/\\.\\.\\/)/i')
+rspamd_config.URI_HIDDEN_PATH = {
+ callback = function(task)
+ local urls = task:get_urls(false)
+ if (urls) then
+ for _, url in ipairs(urls) do
+ if (not (url:is_subject() and url:is_html_displayed())) then
+ local path = url:get_path()
+ if (hidden_uri_re:match(path)) then
+ -- TODO: need url:is_schemeless() to improve this
+ return true, 1.0, url:get_text()
+ end
+ end
+ end
+ end
+ end,
+ description = 'Message contains URI with a hidden path',
+ score = 1.0,
+ group = 'compromised_hosts',
+}
+
+reconf['MID_RHS_WWW'] = {
+ re = "Message-Id=/@www\\./Hi",
+ description = "Message-ID from www host",
+ score = 0.5,
+ group = "compromised_hosts"
+}
+
+rspamd_config.FROM_SERVICE_ACCT = {
+ callback = function(task)
+ local re = rspamd_regexp.create_cached('/^(?:www-data|anonymous|ftp|apache|nobody|guest|nginx|web|www)@/i');
+ -- From
+ local from = task:get_from(2)
+ if (from and from[1]) then
+ if (re:match(from[1].addr)) then
+ return true
+ end
+ end
+ -- Sender
+ local sender = task:get_header('Sender')
+ if sender then
+ local s = util.parse_mail_address(sender, task:get_mempool())
+ if (s and s[1]) then
+ if (re:match(s[1].addr)) then
+ return true
+ end
+ end
+ end
+ -- Reply-To
+ local replyto = task:get_header('Reply-To')
+ if replyto then
+ local rt = util.parse_mail_address(replyto, task:get_mempool())
+ if (rt and rt[1]) then
+ if (re:match(rt[1].addr)) then
+ return true
+ end
+ end
+ end
+ end,
+ description = "Sender/From/Reply-To is a service account",
+ score = 1.0,
+ group = "compromised_hosts"
+}
+
+reconf['WWW_DOT_DOMAIN'] = {
+ re = "From=/@www\\./Hi || Sender=/@www\\./Hi || Reply-To=/@www\\./Hi || check_smtp_data('from',/@www\\./i)",
+ description = "From/Sender/Reply-To or Envelope is @www.domain.com",
+ score = 0.5,
+ group = "compromised_hosts"
+}
diff --git a/rules/regexp/headers.lua b/rules/regexp/headers.lua
new file mode 100644
index 0000000..0624997
--- /dev/null
+++ b/rules/regexp/headers.lua
@@ -0,0 +1,1046 @@
+-- Actually these regular expressions were obtained from SpamAssassin project, so they are licensed by apache license:
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to you under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at:
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Definitions of header regexps
+
+local reconf = config['regexp']
+
+-- Subject needs encoding
+-- Define encodings types
+local subject_encoded_b64 = 'Subject=/=\\?\\S+\\?B\\?/iX'
+local subject_encoded_qp = 'Subject=/=\\?\\S+\\?Q\\?/iX'
+-- Define whether subject must be encoded (contains non-7bit characters)
+local subject_needs_mime = 'Subject=/[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f-\\xff]/X'
+-- Final rule
+reconf['SUBJECT_NEEDS_ENCODING'] = {
+ re = string.format('!(%s) & !(%s) & (%s)', subject_encoded_b64, subject_encoded_qp, subject_needs_mime),
+ score = 1.0,
+ mime_only = true,
+ description = 'Subject needs encoding',
+ group = 'headers'
+}
+
+local from_encoded_b64 = 'From=/=\\?\\S+\\?B\\?/iX'
+local from_encoded_qp = 'From=/=\\?\\S+\\?Q\\?/iX'
+local raw_from_needs_mime = 'From=/[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f-\\xff]/X'
+reconf['FROM_NEEDS_ENCODING'] = {
+ re = string.format('!(%s) & !(%s) & (%s)', from_encoded_b64, from_encoded_qp, raw_from_needs_mime),
+ score = 1.0,
+ mime_only = true,
+ description = 'From header needs encoding',
+ group = 'headers'
+}
+
+local to_encoded_b64 = 'To=/=\\?\\S+\\?B\\?/iX'
+local to_encoded_qp = 'To=/=\\?\\S+\\?Q\\?/iX'
+local raw_to_needs_mime = 'To=/[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f-\\xff]/X'
+reconf['TO_NEEDS_ENCODING'] = {
+ re = string.format('!(%s) & !(%s) & (%s)', to_encoded_b64, to_encoded_qp, raw_to_needs_mime),
+ score = 1.0,
+ mime_only = true,
+ description = 'To header needs encoding',
+ group = 'headers'
+}
+
+-- Detects that there is no space in From header (e.g. Some Name<some@host>)
+reconf['R_NO_SPACE_IN_FROM'] = {
+ re = 'From=/\\S<[-\\w\\.]+\\@[-\\w\\.]+>/X',
+ score = 1.0,
+ mime_only = true,
+ description = 'No space in From header',
+ group = 'headers'
+}
+
+reconf['TO_WRAPPED_IN_SPACES'] = {
+ re = [[To=/<\s[-.\w]+\@[-.\w]+\s>/X]],
+ score = 2.0,
+ mime_only = true,
+ description = 'To address is wrapped in spaces inside angle brackets (e.g. display-name < local-part@domain >)',
+ group = 'headers'
+}
+
+-- Detects missing Subject header
+reconf['MISSING_SUBJECT'] = {
+ re = '!raw_header_exists(Subject)',
+ score = 2.0,
+ mime_only = true,
+ description = 'Subject header is missing',
+ group = 'headers'
+}
+
+rspamd_config.EMPTY_SUBJECT = {
+ score = 1.0,
+ mime_only = true,
+ description = 'Subject header is empty',
+ group = 'headers',
+ callback = function(task)
+ local hdr = task:get_header('Subject')
+ if hdr and #hdr == 0 then
+ return true
+ end
+ return false
+ end
+}
+
+-- Detects missing To header
+reconf['MISSING_TO'] = {
+ re = '!raw_header_exists(To)',
+ score = 2.0,
+ description = 'To header is missing',
+ group = 'headers',
+ mime_only = true,
+}
+
+-- Detects undisclosed recipients
+reconf['R_UNDISC_RCPT'] = {
+ -- match:
+ -- To: undisclosed-recipients:;
+ -- To: Undisclosed recipients:;
+ -- To: undisclosed-recipients: ;
+ -- To: <Undisclosed-Recipient:;>
+ -- To: <"Undisclosed-Recipient:;">
+ -- To: "undisclosed-recipients (utajeni adresati)": ;
+ -- To: Undisclosed recipients:
+ -- but do not match:
+ -- Undisclosed Recipient <user@example.org>
+ re = [[To=/^<?"?undisclosed[- ]recipients?\b.*:/i{header}]],
+ score = 3.0,
+ description = 'Recipients are absent or undisclosed',
+ group = 'headers',
+ mime_only = true,
+}
+
+-- Detects missing Message-ID
+local has_mid = 'header_exists(Message-Id)'
+reconf['MISSING_MID'] = {
+ re = '!header_exists(Message-Id)',
+ score = 2.5,
+ description = 'Message-ID header is missing',
+ group = 'headers',
+ mime_only = true,
+}
+
+-- Received seems to be fake
+reconf['R_RCVD_SPAMBOTS'] = {
+ re = 'Received=/^from \\[\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\] by [-.\\w+]{5,255}; [SMTWF][a-z][a-z],' ..
+ ' [\\s\\d]?\\d [JFMAJSOND][a-z][a-z] \\d{4} \\d{2}:\\d{2}:\\d{2} [-+]\\d{4}$/mH',
+ score = 3.0,
+ description = 'Spambots signatures in received headers',
+ group = 'headers',
+ mime_only = true,
+}
+
+-- Charset is missing in message
+reconf['R_MISSING_CHARSET'] = {
+ re = string.format('!is_empty_body() & content_type_is_type(text) & content_type_is_subtype(plain) & !content_type_has_param(charset) & !%s',
+ 'compare_transfer_encoding(7bit)'),
+ score = 0.5,
+ description = 'Charset header is missing',
+ group = 'headers',
+ mime_only = true,
+}
+
+-- Find forged Outlook MUA
+-- Yahoo groups messages
+local yahoo_bulk = 'Received=/from \\[\\S+\\] by \\S+\\.(?:groups|scd|dcn)\\.yahoo\\.com with NNFMP/H'
+-- Outlook MUA
+local outlook_mua = 'X-Mailer=/^Microsoft Outlook\\b/H'
+local any_outlook_mua = 'X-Mailer=/^Microsoft Outlook\\b/H'
+reconf['FORGED_OUTLOOK_HTML'] = {
+ re = string.format('!%s & %s & %s', yahoo_bulk, outlook_mua, 'has_only_html_part()'),
+ score = 5.0,
+ description = 'Forged Outlook HTML signature',
+ group = 'headers',
+ mime_only = true,
+}
+
+-- Recipients seems to be likely with each other (only works when recipients count is more than 5 recipients)
+reconf['SUSPICIOUS_RECIPS'] = {
+ re = 'compare_recipients_distance(0.65)',
+ score = 1.5,
+ description = 'Recipients seems to be autogenerated (works if recipients count is more than 5)',
+ group = 'headers',
+ mime_only = true,
+}
+
+-- Recipients list seems to be sorted
+reconf['SORTED_RECIPS'] = {
+ re = 'is_recipients_sorted()',
+ score = 3.5,
+ description = 'Recipients list seems to be sorted',
+ group = 'headers',
+ mime_only = true,
+}
+
+-- Spam string at the end of message to make statistics faults
+reconf['TRACKER_ID'] = {
+ re = '/^[a-z0-9]{6,24}[-_a-z0-9]{12,36}[a-z0-9]{6,24}\\s*\\z/isPr',
+ score = 3.84,
+ description = 'Spam string at the end of message to make statistics fault',
+ group = 'headers',
+ mime_only = true,
+}
+
+-- From contains only 7bit characters (parsed headers are used)
+local from_needs_mime = 'From=/[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f-\\xff]/Hr'
+-- From that contains encoded characters while base 64 is not needed as all symbols are 7bit
+reconf['FROM_EXCESS_BASE64'] = {
+ re = string.format('%s & !%s', from_encoded_b64, from_needs_mime),
+ score = 1.5,
+ description = 'From header is unnecessarily encoded in base64',
+ group = 'excessb64',
+ mime_only = true,
+}
+
+-- From that contains encoded characters while quoted-printable is not needed as all symbols are 7bit
+reconf['FROM_EXCESS_QP'] = {
+ re = string.format('%s & !%s', from_encoded_qp, from_needs_mime),
+ score = 1.2,
+ description = 'From header is unnecessarily encoded in quoted-printable',
+ group = 'excessqp'
+}
+
+-- To contains only 7bit characters (parsed headers are used)
+local to_needs_mime = 'To=/[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f-\\xff]/Hr'
+-- To that contains encoded characters while base 64 is not needed as all symbols are 7bit
+reconf['TO_EXCESS_BASE64'] = {
+ re = string.format('%s & !%s', to_encoded_b64, to_needs_mime),
+ score = 1.5,
+ description = 'To header is unnecessarily encoded in base64',
+ group = 'excessb64'
+}
+
+-- To that contains encoded characters while quoted-printable is not needed as all symbols are 7bit
+-- Final rule
+reconf['TO_EXCESS_QP'] = {
+ re = string.format('%s & !%s', to_encoded_qp, to_needs_mime),
+ score = 1.2,
+ description = 'To header is unnecessarily encoded in quoted-printable',
+ group = 'excessqp'
+}
+
+-- Reply-To that contains encoded characters while base 64 is not needed as all symbols are 7bit
+-- Regexp that checks that Reply-To header is encoded with base64 (search in raw headers)
+local replyto_encoded_b64 = 'Reply-To=/\\=\\?\\S+\\?B\\?/iX'
+-- Reply-To contains only 7bit characters (parsed headers are used)
+local replyto_needs_mime = 'Reply-To=/[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f-\\xff]/Hr'
+-- Final rule
+reconf['REPLYTO_EXCESS_BASE64'] = {
+ re = string.format('%s & !%s', replyto_encoded_b64, replyto_needs_mime),
+ score = 1.5,
+ description = 'Reply-To header is unnecessarily encoded in base64',
+ group = 'excessb64'
+}
+
+-- Reply-To that contains encoded characters while quoted-printable is not needed as all symbols are 7bit
+-- Regexp that checks that Reply-To header is encoded with quoted-printable (search in raw headers)
+local replyto_encoded_qp = 'Reply-To=/\\=\\?\\S+\\?Q\\?/iX'
+-- Final rule
+reconf['REPLYTO_EXCESS_QP'] = {
+ re = string.format('%s & !%s', replyto_encoded_qp, replyto_needs_mime),
+ score = 1.2,
+ description = 'Reply-To header is unnecessarily encoded in quoted-printable',
+ group = 'excessqp'
+}
+
+-- Cc that contains encoded characters while base 64 is not needed as all symbols are 7bit
+-- Regexp that checks that Cc header is encoded with base64 (search in raw headers)
+local cc_encoded_b64 = 'Cc=/\\=\\?\\S+\\?B\\?/iX'
+-- Co contains only 7bit characters (parsed headers are used)
+local cc_needs_mime = 'Cc=/[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f-\\xff]/Hr'
+-- Final rule
+reconf['CC_EXCESS_BASE64'] = {
+ re = string.format('%s & !%s', cc_encoded_b64, cc_needs_mime),
+ score = 1.5,
+ description = 'Cc header is unnecessarily encoded in base64',
+ group = 'excessb64'
+}
+
+-- Cc that contains encoded characters while quoted-printable is not needed as all symbols are 7bit
+-- Regexp that checks that Cc header is encoded with quoted-printable (search in raw headers)
+local cc_encoded_qp = 'Cc=/\\=\\?\\S+\\?Q\\?/iX'
+-- Final rule
+reconf['CC_EXCESS_QP'] = {
+ re = string.format('%s & !%s', cc_encoded_qp, cc_needs_mime),
+ score = 1.2,
+ description = 'Cc header is unnecessarily encoded in quoted-printable',
+ group = 'excessqp'
+}
+
+local subj_encoded_b64 = 'Subject=/\\=\\?\\S+\\?B\\?/iX'
+local subj_needs_mime = 'Subject=/[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f-\\xff]/Hr'
+reconf['SUBJ_EXCESS_BASE64'] = {
+ re = string.format('%s & !%s', subj_encoded_b64, subj_needs_mime),
+ score = 1.5,
+ description = 'Subject header is unnecessarily encoded in base64',
+ group = 'excessb64'
+}
+
+local subj_encoded_qp = 'Subject=/\\=\\?\\S+\\?Q\\?/iX'
+reconf['SUBJ_EXCESS_QP'] = {
+ re = string.format('%s & !%s', subj_encoded_qp, subj_needs_mime),
+ score = 1.2,
+ description = 'Subject header is unnecessarily encoded in quoted-printable',
+ group = 'excessqp'
+}
+
+-- Detect forged outlook headers
+-- OE X-Mailer header
+local oe_mua = 'X-Mailer=/\\bOutlook Express [456]\\./H'
+-- OE Message ID format
+local oe_msgid_1 = 'Message-Id=/^<?[A-Za-z0-9-]{7}[A-Za-z0-9]{20}\\@hotmail\\.com>?$/mH'
+local oe_msgid_2 = 'Message-Id=/^<?(?:[0-9a-f]{8}|[0-9a-f]{12})\\$[0-9a-f]{8}\\$[0-9a-f]{8}\\@\\S+>?$/H'
+-- EZLM remail of message
+local lyris_ezml_remailer = 'List-Unsubscribe=/<mailto:(?:leave-\\S+|\\S+-unsubscribe)\\@\\S+>$/H'
+-- Header of wacky sendmail
+local wacky_sendmail_version = 'Received=/\\/CWT\\/DCE\\)/H'
+-- Iplanet received header
+local iplanet_messaging_server = 'Received=/iPlanet Messaging Server/H'
+-- Hotmail message id
+local hotmail_baydav_msgid = 'Message-Id=/^<?BAY\\d+-DAV\\d+[A-Z0-9]{25}\\@phx\\.gbl?>$/H'
+-- Sympatico message id
+local sympatico_msgid = 'Message-Id=/^<?BAYC\\d+-PASMTP\\d+[A-Z0-9]{25}\\@CEZ\\.ICE>?$/H'
+-- Mailman message id
+-- https://bazaar.launchpad.net/~mailman-coders/mailman/2.1/view/head:/Mailman/Utils.py#L811
+local mailman_msgid = [[Message-ID=/^<mailman\.\d+\.\d+\.\d+\.[-+.:=\w]+@[-a-zA-Z\d.]+>$/H]]
+-- Message id seems to be forged
+local unusable_msgid = string.format('(%s | %s | %s | %s | %s | %s)',
+ lyris_ezml_remailer, wacky_sendmail_version,
+ iplanet_messaging_server, hotmail_baydav_msgid, sympatico_msgid, mailman_msgid)
+-- Outlook express data seems to be forged
+local forged_oe = string.format('(%s & !%s & !%s & !%s)', oe_mua, oe_msgid_1, oe_msgid_2, unusable_msgid)
+-- Outlook specific headers
+local outlook_dollars_mua = 'X-Mailer=/^Microsoft Outlook(?: 8| CWS, Build 9|, Build 10)\\./H'
+local outlook_dollars_other = 'Message-Id=/^<?\\!\\~\\!>?/H'
+local vista_msgid = 'Message-Id=/^<?[A-F\\d]{32}\\@\\S+>?$/H'
+local ims_msgid = 'Message-Id=/^<?[A-F\\d]{36,40}\\@\\S+>?$/H'
+-- Forged outlook headers
+local forged_outlook_dollars = string.format('(%s & !%s & !%s & !%s & !%s & !%s)',
+ outlook_dollars_mua, oe_msgid_2, outlook_dollars_other, vista_msgid, ims_msgid, unusable_msgid)
+-- Outlook versions that should be excluded from summary rule
+local fmo_excl_o3416 = 'X-Mailer=/^Microsoft Outlook, Build 10.0.3416$/H'
+local fmo_excl_oe3790 = 'X-Mailer=/^Microsoft Outlook Express 6.00.3790.3959$/H'
+-- Summary rule for forged outlook
+reconf['FORGED_MUA_OUTLOOK'] = {
+ re = string.format('(%s | %s) & !%s & !%s & !%s',
+ forged_oe, forged_outlook_dollars, fmo_excl_o3416, fmo_excl_oe3790, vista_msgid),
+ score = 3.0,
+ description = 'Forged Outlook MUA',
+ group = 'mua'
+}
+
+-- HTML outlook signs
+local mime_html = 'content_type_is_type(text) & content_type_is_subtype(/.?html/)'
+local tag_exists_html = 'has_html_tag(html)'
+local tag_exists_head = 'has_html_tag(head)'
+local tag_exists_meta = 'has_html_tag(meta)'
+local tag_exists_body = 'has_html_tag(body)'
+reconf['FORGED_OUTLOOK_TAGS'] = {
+ re = string.format('!%s & %s & %s & !(%s & %s & %s & %s)',
+ yahoo_bulk, any_outlook_mua, mime_html, tag_exists_html, tag_exists_head,
+ tag_exists_meta, tag_exists_body),
+ score = 2.1,
+ description = "Message pretends to be send from Outlook but has 'strange' tags",
+ group = 'headers'
+}
+
+-- Forged OE/MSO boundary
+reconf['SUSPICIOUS_BOUNDARY'] = {
+ re = 'Content-Type=/^\\s*multipart.+boundary="----=_NextPart_000_[A-Z\\d]{4}_(00EBFFA4|0102FFA4|32C6FFA4|3302FFA4)\\.[A-Z\\d]{8}"[\\r\\n]*$/siX',
+ score = 5.0,
+ description = 'Suspicious boundary in Content-Type header',
+ group = 'mua'
+}
+-- Forged OE/MSO boundary
+reconf['SUSPICIOUS_BOUNDARY2'] = {
+ re = 'Content-Type=/^\\s*multipart.+boundary="----=_NextPart_000_[A-Z\\d]{4}_(01C6527E)\\.[A-Z\\d]{8}"[\\r\\n]*$/siX',
+ score = 4.0,
+ description = 'Suspicious boundary in Content-Type header',
+ group = 'mua'
+}
+-- Forged OE/MSO boundary
+reconf['SUSPICIOUS_BOUNDARY3'] = {
+ re = 'Content-Type=/^\\s*multipart.+boundary="-----000-00\\d\\d-01C[\\dA-F]{5}-[\\dA-F]{8}"[\\r\\n]*$/siX',
+ score = 3.0,
+ description = 'Suspicious boundary in Content-Type header',
+ group = 'mua'
+}
+-- Forged OE/MSO boundary
+local suspicious_boundary_01C4 = 'Content-Type=/^\\s*multipart.+boundary="----=_NextPart_000_[A-Z\\d]{4}_01C4[\\dA-F]{4}\\.[A-Z\\d]{8}"[\\r\\n]*$/siX'
+local suspicious_boundary_01C4_date = 'Date=/^\\s*\\w\\w\\w,\\s+\\d+\\s+\\w\\w\\w 20(0[56789]|1\\d)/'
+reconf['SUSPICIOUS_BOUNDARY4'] = {
+ re = string.format('(%s) & (%s)', suspicious_boundary_01C4, suspicious_boundary_01C4_date),
+ score = 4.0,
+ description = 'Suspicious boundary in Content-Type header',
+ group = 'mua'
+}
+
+-- Detect forged The Bat! headers
+-- The Bat! X-Mailer header
+local thebat_mua_any = 'X-Mailer=/^\\s*The Bat!/H'
+-- The Bat! common Message-ID template
+local thebat_msgid_common = 'Message-ID=/^<?\\d+\\.\\d+\\@\\S+>?$/mH'
+-- Correct The Bat! Message-ID template
+local thebat_msgid = 'Message-ID=/^<?\\d+\\.(19[789]\\d|20\\d\\d)(0\\d|1[012])([012]\\d|3[01])([0-5]\\d)([0-5]\\d)([0-5]\\d)\\@\\S+>?/mH'
+-- Summary rule for forged The Bat! Message-ID header
+reconf['FORGED_MUA_THEBAT_MSGID'] = {
+ re = string.format('(%s) & !(%s) & (%s) & !(%s)', thebat_mua_any, thebat_msgid, thebat_msgid_common, unusable_msgid),
+ score = 4.0,
+ description = 'Message pretends to be send from The Bat! but has forged Message-ID',
+ group = 'mua'
+}
+-- Summary rule for forged The Bat! Message-ID header with unknown template
+reconf['FORGED_MUA_THEBAT_MSGID_UNKNOWN'] = {
+ re = string.format('(%s) & !(%s) & !(%s) & !(%s)', thebat_mua_any, thebat_msgid, thebat_msgid_common, unusable_msgid),
+ score = 3.0,
+ description = 'Message pretends to be send from The Bat! but has forged Message-ID',
+ group = 'mua'
+}
+
+-- Detect forged KMail headers
+-- KMail User-Agent header
+local kmail_mua = 'User-Agent=/^\\s*KMail\\/1\\.\\d+\\.\\d+/H'
+-- KMail common Message-ID template
+local kmail_msgid_common = 'Message-Id=/^<?\\s*\\d+\\.\\d+\\.\\S+\\@\\S+>?$/mH'
+-- Summary rule for forged KMail Message-ID header with unknown template
+reconf['FORGED_MUA_KMAIL_MSGID_UNKNOWN'] = {
+ re = string.format('(%s) & !(%s) & !(%s)', kmail_mua, kmail_msgid_common, unusable_msgid),
+ score = 2.5,
+ description = 'Message pretends to be send from KMail but has forged Message-ID',
+ group = 'mua'
+}
+
+-- Detect forged Opera Mail headers
+-- Opera Mail User-Agent header
+local opera1x_mua = 'User-Agent=/^\\s*Opera Mail\\/1[01]\\.\\d+ /H'
+-- Opera Mail Message-ID template
+local opera1x_msgid = 'Message-ID=/^<?op\\.[a-z\\d]{14}\\@\\S+>?$/H'
+-- Rule for forged Opera Mail Message-ID header
+reconf['FORGED_MUA_OPERA_MSGID'] = {
+ re = string.format('(%s) & !(%s) & !(%s)', opera1x_mua, opera1x_msgid, unusable_msgid),
+ score = 4.0,
+ description = 'Message pretends to be send from Opera Mail but has forged Message-ID',
+ group = 'mua'
+}
+
+-- Detect forged Mozilla Mail/Thunderbird/Seamonkey/Postbox headers
+-- Mozilla based X-Mailer
+local user_agent_mozilla5 = 'User-Agent=/^\\s*Mozilla\\/5\\.0/H'
+local user_agent_thunderbird = 'User-Agent=/^\\s*(Thunderbird|Mozilla Thunderbird|Mozilla\\/.*Gecko\\/.*(Thunderbird|Betterbird|Icedove)\\/)/H'
+local user_agent_seamonkey = 'User-Agent=/^\\s*Mozilla\\/5\\.0\\s.+\\sSeaMonkey\\/\\d+\\.\\d+/H'
+local user_agent_postbox = [[User-Agent=/^\s*Mozilla\/5\.0\s\([^)]+\)\sGecko\/\d+\sPostboxApp\/\d+(?:\.\d+){2,3}$/H]]
+local user_agent_mozilla = string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_mozilla5, user_agent_thunderbird,
+ user_agent_seamonkey, user_agent_postbox)
+-- Mozilla based common Message-ID template
+local mozilla_msgid_common = 'Message-ID=/^\\s*<[\\dA-F]{8}\\.\\d{1,7}\\@([^>\\.]+\\.)+[^>\\.]+>$/H'
+local mozilla_msgid_common_sec = 'Message-ID=/^\\s*<[\\da-f]{8}-([\\da-f]{4}-){3}[\\da-f]{12}\\@([^>\\.]+\\.)+[^>\\.]+>$/H'
+local mozilla_msgid = 'Message-ID=/^\\s*<(3[3-9A-F]|[4-9A-F][\\dA-F])[\\dA-F]{6}\\.(\\d0){1,4}\\d\\@([^>\\.]+\\.)+[^>\\.]+>$/H'
+-- Summary rule for forged Mozilla Mail Message-ID header
+reconf['FORGED_MUA_MOZILLA_MAIL_MSGID'] = {
+ re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_mozilla, mozilla_msgid_common, mozilla_msgid,
+ unusable_msgid),
+ score = 4.0,
+ description = 'Message pretends to be send from Mozilla Mail but has forged Message-ID',
+ group = 'mua'
+}
+reconf['FORGED_MUA_MOZILLA_MAIL_MSGID_UNKNOWN'] = {
+ re = string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_mozilla, mozilla_msgid_common, mozilla_msgid,
+ unusable_msgid),
+ score = 2.5,
+ description = 'Message pretends to be send from Mozilla Mail but has forged Message-ID',
+ group = 'mua'
+}
+
+-- Summary rule for forged Thunderbird Message-ID header
+reconf['FORGED_MUA_THUNDERBIRD_MSGID'] = {
+ re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_thunderbird, mozilla_msgid_common, mozilla_msgid,
+ unusable_msgid),
+ score = 4.0,
+ description = 'Forged mail pretending to be from Mozilla Thunderbird but has forged Message-ID',
+ group = 'mua'
+}
+reconf['FORGED_MUA_THUNDERBIRD_MSGID_UNKNOWN'] = {
+ re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_thunderbird, mozilla_msgid_common,
+ mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid),
+ score = 2.5,
+ description = 'Forged mail pretending to be from Mozilla Thunderbird but has forged Message-ID',
+ group = 'mua'
+}
+-- Summary rule for forged Seamonkey Message-ID header
+reconf['FORGED_MUA_SEAMONKEY_MSGID'] = {
+ re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common, mozilla_msgid,
+ unusable_msgid),
+ score = 4.0,
+ description = 'Forged mail pretending to be from Mozilla Seamonkey but has forged Message-ID',
+ group = 'mua'
+}
+reconf['FORGED_MUA_SEAMONKEY_MSGID_UNKNOWN'] = {
+ re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common,
+ mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid),
+ score = 2.5,
+ description = 'Forged mail pretending to be from Mozilla Seamonkey but has forged Message-ID',
+ group = 'mua'
+}
+-- Summary rule for forged Postbox Message-ID header
+reconf['FORGED_MUA_POSTBOX_MSGID'] = {
+ re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_postbox, mozilla_msgid_common, mozilla_msgid,
+ unusable_msgid),
+ score = 4.0,
+ description = 'Forged mail pretending to be from Postbox but has forged Message-ID',
+ group = 'mua'
+}
+reconf['FORGED_MUA_POSTBOX_MSGID_UNKNOWN'] = {
+ re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_postbox, mozilla_msgid_common,
+ mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid),
+ score = 2.5,
+ description = 'Forged mail pretending to be from Postbox but has forged Message-ID',
+ group = 'mua'
+}
+
+-- Message id validity
+local sane_msgid = 'Message-Id=/^<?[^<>\\\\ \\t\\n\\r\\x0b\\x80-\\xff]+\\@[^<>\\\\ \\t\\n\\r\\x0b\\x80-\\xff]+>?\\s*$/H'
+local msgid_comment = 'Message-Id=/\\(.*\\)/H'
+reconf['INVALID_MSGID'] = {
+ re = string.format('(%s) & !((%s) | (%s))', has_mid, sane_msgid, msgid_comment),
+ score = 1.7,
+ description = 'Message-ID header is incorrect',
+ group = 'headers'
+}
+
+-- Only Content-Type header without other MIME headers
+local cd = 'header_exists(Content-Disposition)'
+local cte = 'header_exists(Content-Transfer-Encoding)'
+local ct = 'header_exists(Content-Type)'
+local mime_version = 'raw_header_exists(MIME-Version)'
+local ct_text_plain = 'content_type_is_type(text) & content_type_is_subtype(plain)'
+reconf['MIME_HEADER_CTYPE_ONLY'] = {
+ re = string.format('!(%s) & !(%s) & (%s) & !(%s) & !(%s)', cd, cte, ct, mime_version, ct_text_plain),
+ score = 2.0,
+ description = 'Only Content-Type header without other MIME headers',
+ group = 'headers'
+}
+
+-- Forged Exchange messages
+local msgid_dollars_ok = 'Message-Id=/[0-9a-f]{4,}\\$[0-9a-f]{4,}\\$[0-9a-f]{4,}\\@\\S+/H'
+local mimeole_ms = 'X-MimeOLE=/^Produced By Microsoft MimeOLE/H'
+local rcvd_with_exchange = 'Received=/with Microsoft Exchange Server/H'
+reconf['RATWARE_MS_HASH'] = {
+ re = string.format('(%s) & !(%s) & !(%s)', msgid_dollars_ok, mimeole_ms, rcvd_with_exchange),
+ score = 2.0,
+ description = 'Forged Exchange messages',
+ group = 'headers'
+}
+
+-- Reply-type in content-type
+reconf['STOX_REPLY_TYPE'] = {
+ re = 'Content-Type=/text\\/plain; .* reply-type=original/H',
+ score = 1.0,
+ description = 'Reply-type in Content-Type header',
+ group = 'headers'
+}
+
+-- Forged yahoo msgid
+local at_yahoo_msgid = 'Message-Id=/\\@yahoo\\.com\\b/iH'
+local from_yahoo_com = 'From=/\\@yahoo\\.com\\b/iH'
+reconf['FORGED_MSGID_YAHOO'] = {
+ re = string.format('(%s) & !(%s)', at_yahoo_msgid, from_yahoo_com),
+ score = 2.0,
+ description = 'Forged Yahoo Message-ID header',
+ group = 'headers'
+}
+
+-- Forged The Bat! MUA headers
+local thebat_mua_v1 = 'X-Mailer=/^The Bat! \\(v1\\./H'
+local ctype_has_boundary = 'Content-Type=/boundary/iH'
+local bat_boundary = 'Content-Type=/boundary=\\"?-{10}/H'
+local mailman_21 = 'X-Mailman-Version=/\\d/H'
+reconf['FORGED_MUA_THEBAT_BOUN'] = {
+ re = string.format('(%s) & (%s) & !(%s) & !(%s)', thebat_mua_v1, ctype_has_boundary, bat_boundary, mailman_21),
+ score = 2.0,
+ description = 'Forged The Bat! MUA headers',
+ group = 'headers'
+}
+
+-- Detect Mail.Ru web-mail
+local xm_mail_ru_mailer_1_0 = 'X-Mailer=/^Mail\\.Ru Mailer 1\\.0$/H'
+local rcvd_e_mail_ru = 'Received=/^(?:from \\[\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\] )?by e\\.mail\\.ru with HTTP;/mH'
+reconf['MAIL_RU_MAILER'] = {
+ re = string.format('(%s) & (%s)', xm_mail_ru_mailer_1_0, rcvd_e_mail_ru),
+ score = 0.0,
+ description = 'Sent with Mail.Ru webmail',
+ group = 'headers'
+}
+
+-- Detect yandex.ru web-mail
+local xm_yandex_ru_mailer_5_0 = 'X-Mailer=/^Yamail \\[ http:\\/\\/yandex\\.ru \\] 5\\.0$/H'
+local rcvd_web_yandex_ru = 'Received=/^by web\\d{1,2}[a-z]\\.yandex\\.ru with HTTP;/mH'
+reconf['YANDEX_RU_MAILER'] = {
+ re = string.format('(%s) & (%s)', xm_yandex_ru_mailer_5_0, rcvd_web_yandex_ru),
+ score = 0.0,
+ description = 'Sent with Yandex webmail',
+ group = 'headers'
+}
+
+-- Detect 1C v8.2 and v8.3 mailers
+reconf['MAILER_1C_8'] = {
+ re = 'X-Mailer=/^1C:Enterprise 8\\.[23]$/H',
+ score = 0.0,
+ description = 'Sent with 1C:Enterprise 8',
+ group = 'headers'
+}
+
+-- Detect rogue 'strongmail' MTA with IPv4 and '(-)' in Received line
+reconf['STRONGMAIL'] = {
+ re = [[Received=/^from\s+strongmail\s+\(\[\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\]\) by \S+ \(-\); /mH]],
+ score = 6.0,
+ description = 'Sent via rogue "strongmail" MTA',
+ group = 'headers'
+}
+
+-- Two received headers with ip addresses
+local double_ip_spam_1 = 'Received=/from \\[\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\] by \\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3} with/H'
+local double_ip_spam_2 = 'Received=/from\\s+\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s+by\\s+\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3};/H'
+reconf['RCVD_DOUBLE_IP_SPAM'] = {
+ re = string.format('(%s) | (%s)', double_ip_spam_1, double_ip_spam_2),
+ score = 2.0,
+ description = 'Has two Received headers containing bare IP addresses',
+ group = 'headers'
+}
+
+-- Quoted reply-to from yahoo (seems to be forged)
+local repto_quote = 'Reply-To=/\\".*\\"\\s*\\</H'
+reconf['REPTO_QUOTE_YAHOO'] = {
+ re = string.format('(%s) & ((%s) | (%s))', repto_quote, from_yahoo_com, at_yahoo_msgid),
+ score = 2.0,
+ description = 'Quoted Reply-To header from Yahoo (seems to be forged)',
+ group = 'headers'
+}
+
+reconf['FAKE_REPLY'] = {
+ re = [[Subject=/^re:/i{header} & !(header_exists(In-Reply-To) | header_exists(References))]],
+ description = 'Fake reply',
+ score = 1.0,
+ group = 'headers'
+}
+
+-- Mime-OLE is needed but absent (e.g. fake Outlook or fake Exchange)
+local has_msmail_pri = 'header_exists(X-MSMail-Priority)'
+local has_mimeole = 'header_exists(X-MimeOLE)'
+local has_squirrelmail_in_mailer = 'X-Mailer=/SquirrelMail\\b/H'
+local has_office_version_in_mailer = [[X-Mailer=/^Microsoft (?:Office )?Outlook [12]\d\.0/]]
+local has_x_android_message_id = 'header_exists(X-Android-Message-Id)'
+reconf['MISSING_MIMEOLE'] = {
+ re = string.format('(%s) & !(%s) & !(%s) & !(%s) & !(%s)',
+ has_msmail_pri,
+ has_mimeole,
+ has_squirrelmail_in_mailer,
+ has_office_version_in_mailer,
+ has_x_android_message_id),
+ score = 2.0,
+ description = 'Mime-OLE is needed but absent (e.g. fake Outlook or fake Exchange)',
+ group = 'headers'
+}
+
+-- Empty delimiters between header names and header values
+local function gen_check_header_delimiter_empty(header_name)
+ return function(task)
+ for _, rh in ipairs(task:get_header_full(header_name) or {}) do
+ if rh['empty_separator'] then
+ return true
+ end
+ end
+ return false
+ end
+end
+reconf['HEADER_FROM_EMPTY_DELIMITER'] = {
+ re = string.format('(%s)', 'lua:check_from_delim_empty'),
+ score = 1.0,
+ description = 'From header has no delimiter between header name and header value',
+ group = 'headers',
+ functions = {
+ check_from_delim_empty = gen_check_header_delimiter_empty('From')
+ }
+}
+reconf['HEADER_TO_EMPTY_DELIMITER'] = {
+ re = string.format('(%s)', 'lua:check_to_delim_empty'),
+ score = 1.0,
+ description = 'To header has no delimiter between header name and header value',
+ group = 'headers',
+ functions = {
+ check_to_delim_empty = gen_check_header_delimiter_empty('To')
+ }
+}
+reconf['HEADER_CC_EMPTY_DELIMITER'] = {
+ re = string.format('(%s)', 'lua:check_cc_delim_empty'),
+ score = 1.0,
+ description = 'Cc header has no delimiter between header name and header value',
+ group = 'headers',
+ functions = {
+ check_cc_delim_empty = gen_check_header_delimiter_empty('Cc')
+ }
+}
+reconf['HEADER_REPLYTO_EMPTY_DELIMITER'] = {
+ re = string.format('(%s)', 'lua:check_repto_delim_empty'),
+ score = 1.0,
+ description = 'Reply-To header has no delimiter between header name and header value',
+ group = 'headers',
+ functions = {
+ check_repto_delim_empty = gen_check_header_delimiter_empty('Reply-To')
+ }
+}
+reconf['HEADER_DATE_EMPTY_DELIMITER'] = {
+ re = string.format('(%s)', 'lua:check_date_delim_empty'),
+ score = 1.0,
+ description = 'Date header has no delimiter between header name and header value',
+ group = 'headers',
+ functions = {
+ check_date_delim_empty = gen_check_header_delimiter_empty('Date')
+ }
+}
+
+-- Definitions of received headers regexp
+reconf['RCVD_ILLEGAL_CHARS'] = {
+ re = 'Received=/[\\x80-\\xff]/X',
+ score = 4.0,
+ description = 'Received header has raw illegal character',
+ group = 'headers'
+}
+
+local MAIL_RU_Return_Path = 'Return-path=/^\\s*<.+\\@mail\\.ru>$/iX'
+local MAIL_RU_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@mail\\.ru>$/iX'
+local MAIL_RU_From = 'From=/\\@mail\\.ru>?$/iX'
+local MAIL_RU_Received = 'Received=/from mail\\.ru \\(/mH'
+
+reconf['FAKE_RECEIVED_mail_ru'] = {
+ re = string.format('(%s) & !(((%s) | (%s)) & (%s))',
+ MAIL_RU_Received, MAIL_RU_Return_Path, MAIL_RU_X_Envelope_From, MAIL_RU_From),
+ score = 4.0,
+ description = 'Fake HELO mail.ru in Received header from non-mail.ru sender address',
+ group = 'headers'
+}
+
+local GMAIL_COM_Return_Path = 'Return-path=/^\\s*<.+\\@gmail\\.com>$/iX'
+local GMAIL_COM_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@gmail\\.com>$/iX'
+local GMAIL_COM_From = 'From=/\\@gmail\\.com>?$/iX'
+
+local UKR_NET_Return_Path = 'Return-path=/^\\s*<.+\\@ukr\\.net>$/iX'
+local UKR_NET_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@ukr\\.net>$/iX'
+local UKR_NET_From = 'From=/\\@ukr\\.net>?$/iX'
+
+local RECEIVED_smtp_yandex_ru_1 = 'Received=/from \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] \\((port=\\d+ )?helo=smtp\\.yandex\\.ru\\)/iX'
+local RECEIVED_smtp_yandex_ru_2 = 'Received=/from \\[UNAVAILABLE\\] \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]:\\d+ helo=smtp\\.yandex\\.ru\\)/iX'
+local RECEIVED_smtp_yandex_ru_3 = 'Received=/from \\S+ \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]:\\d+ helo=smtp\\.yandex\\.ru\\)/iX'
+local RECEIVED_smtp_yandex_ru_4 = 'Received=/from \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] \\(account \\S+ HELO smtp\\.yandex\\.ru\\)/iX'
+local RECEIVED_smtp_yandex_ru_5 = 'Received=/from smtp\\.yandex\\.ru \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]\\)/iX'
+local RECEIVED_smtp_yandex_ru_6 = 'Received=/from smtp\\.yandex\\.ru \\(\\S+ \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]\\)/iX'
+local RECEIVED_smtp_yandex_ru_7 = 'Received=/from \\S+ \\(HELO smtp\\.yandex\\.ru\\) \\(\\S+\\@\\d+\\.\\d+\\.\\d+\\.\\d+\\)/iX'
+local RECEIVED_smtp_yandex_ru_8 = 'Received=/from \\S+ \\(HELO smtp\\.yandex\\.ru\\) \\(\\d+\\.\\d+\\.\\d+\\.\\d+\\)/iX'
+local RECEIVED_smtp_yandex_ru_9 = 'Received=/from \\S+ \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] helo=smtp\\.yandex\\.ru\\)/iX'
+
+reconf['FAKE_RECEIVED_smtp_yandex_ru'] = {
+ re = string.format('(((%s) & ((%s) | (%s))) | ((%s) & ((%s) | (%s))) ' ..
+ ' | ((%s) & ((%s) | (%s)))) & (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s)',
+ MAIL_RU_From, MAIL_RU_Return_Path, MAIL_RU_X_Envelope_From, GMAIL_COM_From,
+ GMAIL_COM_Return_Path, GMAIL_COM_X_Envelope_From, UKR_NET_From, UKR_NET_Return_Path,
+ UKR_NET_X_Envelope_From, RECEIVED_smtp_yandex_ru_1, RECEIVED_smtp_yandex_ru_2,
+ RECEIVED_smtp_yandex_ru_3, RECEIVED_smtp_yandex_ru_4, RECEIVED_smtp_yandex_ru_5,
+ RECEIVED_smtp_yandex_ru_6, RECEIVED_smtp_yandex_ru_7, RECEIVED_smtp_yandex_ru_8,
+ RECEIVED_smtp_yandex_ru_9),
+ score = 4.0,
+ description = 'Fake smtp.yandex.ru Received header',
+ group = 'headers'
+}
+
+reconf['FORGED_GENERIC_RECEIVED'] = {
+ re = 'Received=/^\\s*(.+\\n)*from \\[\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\] by (([\\w\\d-]+\\.)+[a-zA-Z]{2,6}|\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}); \\w{3}, \\d+ \\w{3} 20\\d\\d \\d\\d\\:\\d\\d\\:\\d\\d [+-]\\d\\d\\d0/X',
+ score = 3.6,
+ description = 'Forged generic Received header',
+ group = 'headers'
+}
+
+reconf['FORGED_GENERIC_RECEIVED2'] = {
+ re = 'Received=/^\\s*(.+\\n)*from \\[\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\] by ([\\w\\d-]+\\.)+[a-z]{2,6} id [\\w\\d]{12}; \\w{3}, \\d+ \\w{3} 20\\d\\d \\d\\d\\:\\d\\d\\:\\d\\d [+-]\\d\\d\\d0/X',
+ score = 3.6,
+ description = 'Forged generic Received header',
+ group = 'headers'
+}
+
+reconf['FORGED_GENERIC_RECEIVED3'] = {
+ re = 'Received=/^\\s*(.+\\n)*by \\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3} with SMTP id [a-zA-Z]{14}\\.\\d{13};[\\r\\n\\s]*\\w{3}, \\d+ \\w{3} 20\\d\\d \\d\\d\\:\\d\\d\\:\\d\\d [+-]\\d\\d\\d0 \\(GMT\\)/X',
+ score = 3.6,
+ description = 'Forged generic Received header',
+ group = 'headers'
+}
+
+reconf['FORGED_GENERIC_RECEIVED4'] = {
+ re = 'Received=/^\\s*(.+\\n)*from localhost by \\S+;\\s+\\w{3}, \\d+ \\w{3} 20\\d\\d \\d\\d\\:\\d\\d\\:\\d\\d [+-]\\d\\d\\d0[\\s\\r\\n]*$/X',
+ score = 3.6,
+ description = 'Forged generic Received header',
+ group = 'headers'
+}
+
+reconf['INVALID_POSTFIX_RECEIVED'] = {
+ re = 'Received=/ \\(Postfix\\) with ESMTP id [A-Z\\d]+([\\s\\r\\n]+for <\\S+?>)?;[\\s\\r\\n]*[A-Z][a-z]{2}, \\d{1,2} [A-Z][a-z]{2} \\d\\d\\d\\d \\d\\d:\\d\\d:\\d\\d [\\+\\-]\\d\\d\\d\\d$/X',
+ score = 3.0,
+ description = 'Invalid Postfix Received header',
+ group = 'headers'
+}
+
+reconf['X_PHP_FORGED_0X'] = {
+ re = "X-PHP-Originating-Script=/^0\\d/X",
+ score = 4.0,
+ description = "X-PHP-Originating-Script header appears forged",
+ group = 'headers'
+}
+
+reconf['GOOGLE_FORWARDING_MID_MISSING'] = {
+ re = "Message-ID=/SMTPIN_ADDED_MISSING\\@mx\\.google\\.com>$/X",
+ score = 2.5,
+ description = "Message was missing Message-ID pre-forwarding",
+ group = 'headers'
+}
+
+reconf['GOOGLE_FORWARDING_MID_BROKEN'] = {
+ re = "Message-ID=/SMTPIN_ADDED_BROKEN\\@mx\\.google\\.com>$/X",
+ score = 1.7,
+ description = "Message had invalid Message-ID pre-forwarding",
+ group = 'headers'
+}
+
+reconf['CTE_CASE'] = {
+ re = 'Content-Transfer-Encoding=/^[78]B/X',
+ description = '[78]Bit .vs. [78]bit',
+ score = 0.5,
+ group = 'headers'
+}
+
+reconf['HAS_INTERSPIRE_SIG'] = {
+ re = string.format('((%s) & (%s) & (%s) & (%s)) | (%s)',
+ 'header_exists(X-Mailer-LID)',
+ 'header_exists(X-Mailer-RecptId)',
+ 'header_exists(X-Mailer-SID)',
+ 'header_exists(X-Mailer-Sent-By)',
+ 'List-Unsubscribe=/\\/unsubscribe\\.php\\?M=[^&]+&C=[^&]+&L=[^&]+&N=[^>]+>$/Xi'),
+ description = "Has Interspire fingerprint",
+ score = 1.0,
+ group = 'headers'
+}
+
+reconf['CT_EXTRA_SEMI'] = {
+ re = 'Content-Type=/;$/X',
+ description = 'Content-Type header ends with a semi-colon',
+ score = 1.0,
+ group = 'headers'
+}
+
+reconf['SUBJECT_ENDS_EXCLAIM'] = {
+ re = 'Subject=/!\\s*$/H',
+ description = 'Subject ends with an exclamation mark',
+ score = 0.0,
+ group = 'headers'
+}
+
+reconf['SUBJECT_HAS_EXCLAIM'] = {
+ re = string.format('%s & !%s', 'Subject=/!/H', 'Subject=/!\\s*$/H'),
+ description = 'Subject contains an exclamation mark',
+ score = 0.0,
+ group = 'headers'
+}
+
+reconf['SUBJECT_ENDS_QUESTION'] = {
+ re = 'Subject=/\\?\\s*$/Hu',
+ description = 'Subject ends with a question mark',
+ score = 1.0,
+ group = 'headers'
+}
+
+reconf['SUBJECT_HAS_QUESTION'] = {
+ re = string.format('%s & !%s', 'Subject=/\\?/H', 'Subject=/\\?\\s*$/Hu'),
+ description = 'Subject contains a question mark',
+ score = 0.0,
+ group = 'headers'
+}
+
+reconf['SUBJECT_HAS_CURRENCY'] = {
+ re = 'Subject=/\\p{Sc}/Hu',
+ description = 'Subject contains currency',
+ score = 1.0,
+ group = 'headers'
+}
+
+reconf['SUBJECT_ENDS_SPACES'] = {
+ re = 'Subject=/\\s+$/H',
+ description = 'Subject ends with space characters',
+ score = 0.5,
+ group = 'headers'
+}
+
+reconf['HAS_ORG_HEADER'] = {
+ re = string.format('%s || %s', 'header_exists(Organization)', 'header_exists(Organisation)'),
+ description = 'Has Organization header',
+ score = 0.0,
+ group = 'headers'
+}
+
+reconf['X_PHPOS_FAKE'] = {
+ re = 'X-PHP-Originating-Script=/^\\d{7}:/Hi',
+ description = 'Fake X-PHP-Originating-Script header',
+ score = 3.0,
+ group = 'headers'
+}
+
+reconf['HAS_XOIP'] = {
+ re = "header_exists('X-Originating-IP')",
+ description = "Has X-Originating-IP header",
+ score = 0.0,
+ group = 'headers'
+}
+
+reconf['HAS_LIST_UNSUB'] = {
+ re = string.format('%s', 'header_exists(List-Unsubscribe)'),
+ description = 'Has List-Unsubscribe header',
+ score = -0.01,
+ group = 'headers'
+}
+
+reconf['HAS_GUC_PROXY_URI'] = {
+ re = '/\\.googleusercontent\\.com\\/proxy/{url}i',
+ description = 'Has googleusercontent.com proxy URL',
+ score = 1.0,
+ group = 'url'
+}
+
+reconf['HAS_GOOGLE_REDIR'] = {
+ re = '/\\.google\\.([a-z]{2,3}(|\\.[a-z]{2,3})|info|jobs)\\/(amp\\/s\\/|url\\?)/{url}i',
+ description = 'Has google.com/url or alike Google redirection URL',
+ score = 1.0,
+ group = 'url'
+}
+
+reconf['HAS_GOOGLE_FIREBASE_URL'] = {
+ re = '/\\.firebasestorage\\.googleapis\\.com\\//{url}i',
+ description = 'Contains firebasestorage.googleapis.com URL',
+ score = 2.0,
+ group = 'url'
+}
+
+reconf['XM_UA_NO_VERSION'] = {
+ re = string.format('(!%s && !%s) && (%s || %s)',
+ 'X-Mailer=/https?:/H',
+ 'User-Agent=/https?:/H',
+ 'X-Mailer=/^[^0-9]+$/H',
+ 'User-Agent=/^[^0-9]+$/H'),
+ description = 'X-Mailer/User-Agent header has no version number',
+ score = 0.01,
+ group = 'experimental'
+}
+
+-- Detects messages missing both X-Mailer and User-Agent header
+local has_ua = 'header_exists(User-Agent)'
+local has_xmailer = 'header_exists(X-Mailer)'
+reconf['MISSING_XM_UA'] = {
+ re = string.format('!%s && !%s', has_xmailer, has_ua),
+ score = 0.0,
+ description = 'Message has neither X-Mailer nor User-Agent header',
+ group = 'headers',
+}
+
+-- X-Mailer for old MUA versions which are forged by spammers
+local old_x_mailers = {
+ -- Outlook Express 6.0 was last included in Windows XP (EOL 2014). Windows
+ -- XP is still used (in 2020) by relatively small number of internet users,
+ -- but this header is widely abused by spammers.
+ 'Microsoft Outlook Express',
+ -- Qualcomm Eudora for Windows 7.1.0.9 was released in 2006
+ [[QUALCOMM Windows Eudora (Pro )?Version [1-6]\.]],
+ -- The Bat 3.0 was released in 2004
+ [[The Bat! \(v[12]\.]],
+ -- Can be found in public maillist archives, messages circa 2000
+ [[Microsoft Outlook IMO, Build 9\.0\.]],
+ -- Outlook 2002 (Office XP)
+ [[Microsoft Outlook, Build 10\.]],
+ -- Some old Apple iOS versions are used on old devices, match only very old
+ -- versions (iOS 4.3.5 buid 8L1 was supported until 2013) and less old
+ -- versions frequently seen in spam
+ [[i(Phone|Pad) Mail \((?:[1-8][A-L]|12H|13E)]],
+}
+
+reconf['OLD_X_MAILER'] = {
+ description = 'X-Mailer header has a very old MUA version',
+ re = string.format('X-Mailer=/^(?:%s)/{header}', table.concat(old_x_mailers, '|')),
+ score = 2.0,
+ group = 'headers',
+}
+
+-- Detect Apple Mail
+local apple_x_mailer = [[Apple Mail \((?:(?:Version )?[1-9]\d{0,2}\.\d{1,3}|[1-9]\d{0,2}\.\d{1,4}\.\d{1,4}\.\d{1,4})\)]]
+reconf['APPLE_MAILER'] = {
+ description = 'Sent with Apple Mail',
+ re = string.format('X-Mailer=/^%s/{header}', apple_x_mailer),
+ score = 0.0,
+ group = 'headers',
+}
+
+-- Detect Apple iPhone/iPad Mail
+-- Apple iPhone/iPad Mail X-Mailer contains iOS build number, e. g. 9B206, 16H5, 18G5023c
+-- https://en.wikipedia.org/wiki/IOS_version_history
+local apple_ios_x_mailer = [[i(?:Phone|Pad) Mail \(\d{1,2}[A-Z]\d{1,4}[a-z]?\)]]
+reconf['APPLE_IOS_MAILER'] = {
+ description = 'Sent with Apple iPhone/iPad Mail',
+ re = string.format('X-Mailer=/^%s/{header}', apple_ios_x_mailer),
+ score = 0.0,
+ group = 'headers',
+}
+
+-- X-Mailer header values which should not occur (in the modern mail) at all
+local bad_x_mailers = {
+ -- header name repeated in the header value
+ [[X-Mailer: ]],
+ -- Mozilla Thunderbird uses User-Agent header, not X-Mailer
+ -- Early Thunderbird had U-A like:
+ -- Mozilla Thunderbird 1.0.2 (Windows/20050317)
+ -- Thunderbird 2.0.0.23 (X11/20090812)
+ [[(?:Mozilla )?Thunderbird \d]],
+ -- Was used by Yahoo Groups in 2000s, no one expected to use this in 2020s
+ [[eGroups Message Poster]],
+ -- Regexp for genuine iOS X-Mailer is below, anything which doesn't match it,
+ -- but starts with 'iPhone Mail' or 'iPad Mail' is likely fake
+ [[i(?:Phone|Pad) Mail]],
+}
+
+reconf['FORGED_X_MAILER'] = {
+ description = 'Forged X-Mailer header',
+ re = string.format('X-Mailer=/^(?:%s)/{header} && !X-Mailer=/^%s/{header}',
+ table.concat(bad_x_mailers, '|'), apple_ios_x_mailer),
+ score = 4.5,
+ group = 'headers',
+}
+
+-- X-Mailer headers like: 'Internet Mail Service (5.5.2650.21)' are being
+-- forged by spammers, but MS Exchange 5.5 is still being used (in 2020) on
+-- some mail servers. Example of genuine headers (DC-EXMPL is a hostname which
+-- can be a FQDN):
+-- Received: by DC-EXMPL with Internet Mail Service (5.5.2656.59)
+-- id <HKH4BJQX>; Tue, 8 Dec 2020 07:10:54 -0600
+-- Message-ID: <E7209F9DB64FCC4BB1051420F0E955DD05C9D59F@DC-EXMPL>
+-- X-Mailer: Internet Mail Service (5.5.2656.59)
+reconf['FORGED_IMS'] = {
+ description = 'Forged X-Mailer: Internet Mail Service',
+ re = [[X-Mailer=/^Internet Mail Service \(5\./{header} & !Received=/^by \S+ with Internet Mail Service \(5\./{header}]],
+ score = 3.0,
+ group = 'headers',
+}
diff --git a/rules/regexp/misc.lua b/rules/regexp/misc.lua
new file mode 100644
index 0000000..d723f29
--- /dev/null
+++ b/rules/regexp/misc.lua
@@ -0,0 +1,117 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+
+local reconf = config['regexp']
+
+reconf['HTML_META_REFRESH_URL'] = {
+ -- Requires options { check_attachements = true; }
+ re = '/<meta\\s+http-equiv="refresh"\\s+content="\\d+\\s*;\\s*url=/{sa_raw_body}i',
+ description = "Has HTML Meta refresh URL",
+ score = 5.0,
+ one_shot = true,
+ group = 'HTML'
+}
+
+reconf['HAS_DATA_URI'] = {
+ -- Requires options { check_attachements = true; }
+ re = '/data:[^\\/]+\\/[^; ]+;base64,/{sa_raw_body}i',
+ description = "Has Data URI encoding",
+ group = 'HTML',
+ one_shot = true,
+}
+
+reconf['DATA_URI_OBFU'] = {
+ -- Requires options { check_attachements = true; }
+ re = '/data:text\\/(?:plain|html);base64,/{sa_raw_body}i',
+ description = "Uses Data URI encoding to obfuscate plain or HTML in base64",
+ group = 'HTML',
+ one_shot = true,
+ score = 2.0
+}
+
+reconf['INTRODUCTION'] = {
+ re = '/\\b(?:my name is\\b|(?:i am|this is)\\s+(?:mr|mrs|ms|miss|master|sir|prof(?:essor)?|d(?:octo)?r|rev(?:erend)?)(?:\\.|\\b))/{sa_body}i',
+ description = "Sender introduces themselves",
+ score = 2.0,
+ one_shot = true,
+ group = 'scams'
+}
+
+-- Message contains a link to a .onion URI (Tor hidden service)
+local onion_uri_v2 = '/[a-z0-9]{16}\\.onion?/{url}i'
+local onion_uri_v3 = '/[a-z0-9]{56}\\.onion?/{url}i'
+reconf['HAS_ONION_URI'] = {
+ re = string.format('(%s | %s)', onion_uri_v2, onion_uri_v3),
+ description = 'Contains .onion hidden service URI',
+ score = 0.0,
+ group = 'url'
+}
+
+local my_victim = [[/(?:victim|prey)/{words}]]
+local your_webcam = [[/webcam/{words}]]
+local your_onan = [[/(?:mast[ur]{2}bati(?:on|ng)|onanism|solitary)/{words}]]
+local password_in_words = [[/^pass(?:(?:word)|(?:phrase))$/i{words}]]
+local btc_wallet_address = [[has_symbol(BITCOIN_ADDR)]]
+local wallet_word = [[/^wallet$/{words}]]
+local broken_unicode = [[has_flag(bad_unicode)]]
+local list_unsub = [[header_exists(List-Unsubscribe)]]
+local x_php_origin = [[header_exists(X-PHP-Originating-Script)]]
+
+reconf['LEAKED_PASSWORD_SCAM_RE'] = {
+ re = string.format('%s & (%s | %s | %s | %s | %s | %s | %s | %s | %s)',
+ btc_wallet_address, password_in_words, wallet_word,
+ my_victim, your_webcam, your_onan,
+ broken_unicode, 'lua:check_data_images',
+ list_unsub, x_php_origin),
+ description = 'Contains BTC wallet address and malicious regexps',
+ functions = {
+ check_data_images = function(task)
+ local tp = task:get_text_parts() or {}
+
+ for _, p in ipairs(tp) do
+ if p:is_html() then
+ local hc = p:get_html()
+
+ if hc and hc:has_property('data_urls') then
+ return true
+ end
+ end
+ end
+
+ return false
+ end
+ },
+ score = 0.0,
+ group = 'scams'
+}
+
+rspamd_config:register_dependency('LEAKED_PASSWORD_SCAM', 'BITCOIN_ADDR')
+
+-- Heurististic for detecting InterPlanetary File System (IPFS) gateway URLs:
+-- These contain "ipfs" somewhere (either in the FQDN or the URL path) and a
+-- content identifier (CID), comprising of either "qm", followed by 44 alphanumerical
+-- characters (CIDv0), or a CIDv1 of an alphanumerical string of unspecified length,
+-- depending on the hash algorithm used, but starting with a multibase prefix.
+local ipfs_cid = '/(qm[a-z0-9]{44}|[079fvtbchkzmup][a-z0-9]{44,128})/{url}i'
+local ipfs_string = '/ipfs(\\.|-|_|\\/|\\?)/{url}i'
+reconf['HAS_IPFS_GATEWAY_URL'] = {
+ description = 'Message contains InterPlanetary File System (IPFS) gateway URL, likely malicious',
+ re = string.format('(%s & %s)', ipfs_cid, ipfs_string),
+ score = 6.0,
+ one_shot = true,
+ group = 'url',
+}
diff --git a/rules/regexp/upstream_spam_filters.lua b/rules/regexp/upstream_spam_filters.lua
new file mode 100644
index 0000000..b92f473
--- /dev/null
+++ b/rules/regexp/upstream_spam_filters.lua
@@ -0,0 +1,60 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Rules for upstream services that have already run spam checks
+
+local reconf = config['regexp']
+
+reconf['PRECEDENCE_BULK'] = {
+ re = 'Precedence=/bulk/Hi',
+ score = 0.0,
+ description = "Message marked as bulk",
+ group = 'upstream_spam_filters'
+}
+
+reconf['MICROSOFT_SPAM'] = {
+ -- https://technet.microsoft.com/en-us/library/dn205071(v=exchg.150).aspx
+ re = 'X-Forefront-Antispam-Report=/SFV:SPM/H',
+ score = 4.0,
+ description = "Microsoft says the message is spam",
+ group = 'upstream_spam_filters'
+}
+
+reconf['KLMS_SPAM'] = {
+ re = 'X-KLMS-AntiSpam-Status=/^spam/H',
+ score = 5.0,
+ description = "Kaspersky Security for Mail Server says this message is spam",
+ group = 'upstream_spam_filters'
+}
+
+reconf['SPAM_FLAG'] = {
+ re = string.format('%s || %s || %s',
+ 'X-Spam-Flag=/^(?:yes|true)/Hi',
+ 'X-Spam=/^(?:yes|true)/Hi',
+ 'X-Spam-Status=/^(?:yes|true)/Hi'),
+ score = 5.0,
+ description = "Message was already marked as spam",
+ group = 'upstream_spam_filters'
+}
+
+reconf['UNITEDINTERNET_SPAM'] = {
+ re = string.format('%s || %s',
+ 'X-UI-Filterresults=/^junk:/H',
+ 'X-UI-Out-Filterresults=/^junk:/H'),
+ score = 5.0,
+ description = "United Internet says this message is spam",
+ group = 'upstream_spam_filters'
+}
diff --git a/rules/rspamd.lua b/rules/rspamd.lua
new file mode 100644
index 0000000..6b2c1a5
--- /dev/null
+++ b/rules/rspamd.lua
@@ -0,0 +1,71 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- This is main lua config file for rspamd
+
+require "global_functions"()
+
+config['regexp'] = {}
+rspamd_maps = {} -- Global maps
+
+local local_conf = rspamd_paths['LOCAL_CONFDIR']
+local local_rules = rspamd_paths['RULESDIR']
+local rspamd_util = require "rspamd_util"
+
+dofile(local_rules .. '/regexp/headers.lua')
+dofile(local_rules .. '/regexp/misc.lua')
+dofile(local_rules .. '/regexp/upstream_spam_filters.lua')
+dofile(local_rules .. '/regexp/compromised_hosts.lua')
+dofile(local_rules .. '/html.lua')
+dofile(local_rules .. '/headers_checks.lua')
+dofile(local_rules .. '/subject_checks.lua')
+dofile(local_rules .. '/misc.lua')
+dofile(local_rules .. '/forwarding.lua')
+dofile(local_rules .. '/mid.lua')
+dofile(local_rules .. '/parts.lua')
+dofile(local_rules .. '/bitcoin.lua')
+dofile(local_rules .. '/bounce.lua')
+dofile(local_rules .. '/content.lua')
+dofile(local_rules .. '/controller/init.lua')
+
+if rspamd_util.file_exists(local_conf .. '/rspamd.local.lua') then
+ dofile(local_conf .. '/rspamd.local.lua')
+else
+ -- Legacy lua/rspamd.local.lua
+ if rspamd_util.file_exists(local_conf .. '/lua/rspamd.local.lua') then
+ dofile(local_conf .. '/lua/rspamd.local.lua')
+ end
+end
+
+if rspamd_util.file_exists(local_conf .. '/local.d/rspamd.lua') then
+ dofile(local_conf .. '/local.d/rspamd.lua')
+end
+
+local rmaps = rspamd_config:get_all_opt("lua_maps")
+if rmaps and type(rmaps) == 'table' then
+ local rspamd_logger = require "rspamd_logger"
+ for k, v in pairs(rmaps) do
+ local status, map_or_err = pcall(function()
+ return rspamd_config:add_map(v)
+ end)
+
+ if not status then
+ rspamd_logger.errx(rspamd_config, "cannot add map %s: %s", k, map_or_err)
+ else
+ rspamd_maps[k] = map_or_err
+ end
+ end
+end
diff --git a/rules/subject_checks.lua b/rules/subject_checks.lua
new file mode 100644
index 0000000..f781e1d
--- /dev/null
+++ b/rules/subject_checks.lua
@@ -0,0 +1,70 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local rspamd_regexp = require "rspamd_regexp"
+local util = require "rspamd_util"
+
+-- Uncategorized rules
+local subject_re = rspamd_regexp.create('/^(?:(?:Re|Fwd|Fw|Aw|Antwort|Sv):\\s*)+(.+)$/i')
+
+local function test_subject(task, check_function, rate)
+ local function normalize_linear(a, x)
+ local f = a * x
+ return true, ((f < 1) and f or 1), tostring(x)
+ end
+
+ local sbj = task:get_header('Subject')
+
+ if sbj then
+ local stripped_subject = subject_re:search(sbj, false, true)
+ if stripped_subject and stripped_subject[1] and stripped_subject[1][2] then
+ sbj = stripped_subject[1][2]
+ end
+
+ local l = util.strlen_utf8(sbj)
+ if check_function(sbj, l) then
+ return normalize_linear(rate, l)
+ end
+ end
+
+ return false
+end
+
+rspamd_config.SUBJ_ALL_CAPS = {
+ callback = function(task)
+ local caps_test = function(sbj)
+ return util.is_uppercase(sbj)
+ end
+ return test_subject(task, caps_test, 1.0 / 40.0)
+ end,
+ score = 3.0,
+ group = 'subject',
+ type = 'mime',
+ description = 'Subject contains mostly capital letters'
+}
+
+rspamd_config.LONG_SUBJ = {
+ callback = function(task)
+ local length_test = function(_, len)
+ return len > 200
+ end
+ return test_subject(task, length_test, 1.0 / 400.0)
+ end,
+ score = 3.0,
+ group = 'subject',
+ type = 'mime',
+ description = 'Subject is very long'
+}
diff --git a/set-version.sh b/set-version.sh
new file mode 100755
index 0000000..bd15b3a
--- /dev/null
+++ b/set-version.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# Update the version after a new release.
+version=$1
+if ! [[ "$version" =~ ^[0-9]+.[0-9]+$ ]]; then
+ echo "Usage: $0 <major>.<minor>"
+ exit 1
+fi
+
+read major minor <<< "${version/./ }"
+sed -e "s/^\\(SET(RSPAMD_VERSION_MAJOR \\)[0-9]*)/\\1$major)/" \
+ -e "s/^\\(SET(RSPAMD_VERSION_MINOR \\)[0-9]*)/\\1$minor)/" \
+ -i CMakeLists.txt
+sed -e "1s/([0-9.]*)/($version)/" -i debian/changelog
+sed -e "s/^\\(Version: *\\)[0-9.]*$/\\1$version/" -i centos/rspamd.spec
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
new file mode 100644
index 0000000..1739177
--- /dev/null
+++ b/src/CMakeLists.txt
@@ -0,0 +1,246 @@
+MACRO(_AddModulesForced MLIST)
+# Generate unique string for this build
+ SET(MODULES_C "${CMAKE_CURRENT_BINARY_DIR}/modules.c")
+ FILE(WRITE "${MODULES_C}"
+ "#include \"rspamd.h\"\n")
+
+ # Handle even old cmake
+ LIST(LENGTH ${MLIST} MLIST_COUNT)
+ MATH(EXPR MLIST_MAX ${MLIST_COUNT}-1)
+
+ FOREACH(MOD_IDX RANGE ${MLIST_MAX})
+ LIST(GET ${MLIST} ${MOD_IDX} MOD)
+ FILE(APPEND "${MODULES_C}" "extern module_t ${MOD}_module;\n")
+ ENDFOREACH(MOD_IDX RANGE ${MLIST_MAX})
+
+ FILE(APPEND "${MODULES_C}" "\n\nmodule_t *modules[] = {\n")
+
+ FOREACH(MOD_IDX RANGE ${MLIST_MAX})
+ LIST(GET ${MLIST} ${MOD_IDX} MOD)
+ FILE(APPEND "${MODULES_C}" "&${MOD}_module,\n")
+ ENDFOREACH(MOD_IDX RANGE ${MLIST_MAX})
+
+ FILE(APPEND "${MODULES_C}" "NULL\n};\n")
+ENDMACRO(_AddModulesForced MLIST)
+
+MACRO(_AddWorkersForced WLIST)
+ SET(WORKERS_C "${CMAKE_CURRENT_BINARY_DIR}/workers.c")
+ FILE(WRITE "${WORKERS_C}"
+ "#include \"rspamd.h\"\n")
+
+ # Handle even old cmake
+ LIST(LENGTH ${WLIST} WLIST_COUNT)
+ MATH(EXPR WLIST_MAX ${WLIST_COUNT}-1)
+ FOREACH(MOD_IDX RANGE ${WLIST_MAX})
+ LIST(GET ${WLIST} ${MOD_IDX} WRK)
+ FILE(APPEND "${WORKERS_C}" "extern worker_t ${WRK}_worker;\n")
+ ENDFOREACH(MOD_IDX RANGE ${WLIST_MAX})
+
+ FILE(APPEND "${WORKERS_C}" "\n\nworker_t *workers[] = {\n")
+
+ FOREACH(MOD_IDX RANGE ${WLIST_MAX})
+ LIST(GET ${WLIST} ${MOD_IDX} WRK)
+ FILE(APPEND "${WORKERS_C}" "&${WRK}_worker,\n")
+ ENDFOREACH(MOD_IDX RANGE ${WLIST_MAX})
+ FILE(APPEND "${WORKERS_C}" "NULL\n};\n")
+ENDMACRO(_AddWorkersForced WLIST)
+
+MACRO(AddModules MLIST WLIST)
+ _AddModulesForced(${MLIST})
+ _AddWorkersForced(${WLIST})
+ #IF(NOT EXISTS "modules.c")
+ # _AddModulesForced(${MLIST} ${WLIST})
+ #ELSE(NOT EXISTS "modules.c")
+ # FILE(STRINGS "modules.c" FILE_ID_RAW REGEX "^/.*[a-zA-Z0-9]+.*/$")
+ # STRING(REGEX MATCH "[a-zA-Z0-9]+" FILE_ID "${FILE_ID_RAW}")
+ # IF(NOT FILE_ID STREQUAL MODULES_ID)
+ # MESSAGE("Regenerate modules info")
+ # _AddModulesForced(${MLIST} ${WLIST})
+ # ENDIF(NOT FILE_ID STREQUAL MODULES_ID)
+ #ENDIF(NOT EXISTS "modules.c")
+ENDMACRO(AddModules MLIST WLIST)
+
+# Rspamd core components
+IF (ENABLE_CLANG_PLUGIN MATCHES "ON")
+ SET(CMAKE_C_FLAGS
+ "${CMAKE_C_FLAGS} -Xclang -load -Xclang ${CMAKE_CURRENT_BINARY_DIR}/../clang-plugin/librspamd-clang${CMAKE_SHARED_LIBRARY_SUFFIX} -Xclang -add-plugin -Xclang rspamd-ast")
+ IF(CLANG_EXTRA_PLUGINS_LIBS)
+ FOREACH(_lib ${CLANG_EXTRA_PLUGINS_LIBS})
+ SET(CMAKE_C_FLAGS
+ "${CMAKE_C_FLAGS} -Xclang -load -Xclang ${_lib}")
+ SET(CMAKE_CXX_FLAGS
+ "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${_lib}")
+ ENDFOREACH()
+ ENDIF()
+ IF(CLANG_EXTRA_PLUGINS)
+ FOREACH(_plug ${CLANG_EXTRA_PLUGINS})
+ SET(CMAKE_C_FLAGS
+ "${CMAKE_C_FLAGS} -Xclang -add-plugin -Xclang ${_plug}")
+ SET(CMAKE_CXX_FLAGS
+ "${CMAKE_C_FLAGS} -Xclang -add-plugin -Xclang ${_plug}")
+ ENDFOREACH()
+ ENDIF()
+ENDIF ()
+
+ADD_SUBDIRECTORY(lua)
+ADD_SUBDIRECTORY(libcryptobox)
+ADD_SUBDIRECTORY(libutil)
+ADD_SUBDIRECTORY(libserver)
+ADD_SUBDIRECTORY(libmime)
+ADD_SUBDIRECTORY(libstat)
+ADD_SUBDIRECTORY(client)
+ADD_SUBDIRECTORY(rspamadm)
+
+SET(RSPAMDSRC controller.c
+ fuzzy_storage.c
+ rspamd.c
+ worker.c
+ rspamd_proxy.c)
+
+SET(PLUGINSSRC plugins/regexp.c
+ plugins/chartable.cxx
+ plugins/fuzzy_check.c
+ plugins/dkim_check.c
+ libserver/rspamd_control.c)
+
+SET(MODULES_LIST regexp chartable fuzzy_check dkim)
+SET(WORKERS_LIST normal controller fuzzy rspamd_proxy)
+IF (ENABLE_HYPERSCAN MATCHES "ON")
+ LIST(APPEND WORKERS_LIST "hs_helper")
+ LIST(APPEND RSPAMDSRC "hs_helper.c")
+ENDIF()
+
+AddModules(MODULES_LIST WORKERS_LIST)
+LIST(LENGTH PLUGINSSRC RSPAMD_MODULES_NUM)
+
+SET(RAGEL_DEPENDS "${CMAKE_SOURCE_DIR}/src/ragel/smtp_address.rl"
+ "${CMAKE_SOURCE_DIR}/src/ragel/smtp_date.rl"
+ "${CMAKE_SOURCE_DIR}/src/ragel/smtp_ip.rl"
+ "${CMAKE_SOURCE_DIR}/src/ragel/smtp_base.rl"
+ "${CMAKE_SOURCE_DIR}/src/ragel/content_disposition.rl")
+RAGEL_TARGET(ragel_smtp_addr
+ INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_addr_parser.rl
+ DEPENDS ${RAGEL_DEPENDS}
+ COMPILE_FLAGS -T1
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/smtp_addr_parser.rl.c)
+RAGEL_TARGET(ragel_content_disposition
+ INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/content_disposition_parser.rl
+ DEPENDS ${RAGEL_DEPENDS}
+ COMPILE_FLAGS -G2
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/content_disposition.rl.c)
+RAGEL_TARGET(ragel_rfc2047
+ INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/rfc2047_parser.rl
+ DEPENDS ${RAGEL_DEPENDS}
+ COMPILE_FLAGS -G2
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/rfc2047.rl.c)
+RAGEL_TARGET(ragel_smtp_date
+ INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_date_parser.rl
+ DEPENDS ${RAGEL_DEPENDS}
+ COMPILE_FLAGS -G2
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/date_parser.rl.c)
+RAGEL_TARGET(ragel_smtp_ip
+ INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_ip_parser.rl
+ DEPENDS ${RAGEL_DEPENDS}
+ COMPILE_FLAGS -G2
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ip_parser.rl.c)
+# Fucking cmake...
+FOREACH(_GEN ${LIBSERVER_GENERATED})
+ set_source_files_properties(${_GEN} PROPERTIES GENERATED TRUE)
+ENDFOREACH()
+######################### LINK SECTION ###############################
+
+IF(ENABLE_STATIC MATCHES "ON")
+ ADD_LIBRARY(rspamd-server STATIC
+ ${RSPAMD_CRYPTOBOX}
+ ${RSPAMD_UTIL}
+ ${RSPAMD_LUA}
+ ${RSPAMD_SERVER}
+ ${RSPAMD_STAT}
+ ${RSPAMD_MIME}
+ ${CMAKE_CURRENT_BINARY_DIR}/modules.c
+ ${PLUGINSSRC}
+ "${RAGEL_ragel_smtp_addr_OUTPUTS}"
+ "${RAGEL_ragel_newlines_strip_OUTPUTS}"
+ "${RAGEL_ragel_content_type_OUTPUTS}"
+ "${RAGEL_ragel_content_disposition_OUTPUTS}"
+ "${RAGEL_ragel_rfc2047_OUTPUTS}"
+ "${RAGEL_ragel_smtp_date_OUTPUTS}"
+ "${RAGEL_ragel_smtp_ip_OUTPUTS}"
+ ${BACKWARD_ENABLE})
+ELSE()
+ ADD_LIBRARY(rspamd-server SHARED
+ ${RSPAMD_CRYPTOBOX}
+ ${RSPAMD_UTIL}
+ ${RSPAMD_SERVER}
+ ${RSPAMD_STAT}
+ ${RSPAMD_MIME}
+ ${RSPAMD_LUA}
+ ${CMAKE_CURRENT_BINARY_DIR}/modules.c
+ ${PLUGINSSRC}
+ "${RAGEL_ragel_smtp_addr_OUTPUTS}"
+ "${RAGEL_ragel_newlines_strip_OUTPUTS}"
+ "${RAGEL_ragel_content_type_OUTPUTS}"
+ "${RAGEL_ragel_content_disposition_OUTPUTS}"
+ "${RAGEL_ragel_rfc2047_OUTPUTS}"
+ "${RAGEL_ragel_smtp_date_OUTPUTS}"
+ "${RAGEL_ragel_smtp_ip_OUTPUTS}"
+ ${BACKWARD_ENABLE})
+ENDIF()
+
+FOREACH(_DEP ${LIBSERVER_DEPENDS})
+ ADD_DEPENDENCIES(rspamd-server "${_DEP}")
+ENDFOREACH()
+
+TARGET_LINK_LIBRARIES(rspamd-server rspamd-http-parser)
+TARGET_LINK_LIBRARIES(rspamd-server rspamd-fpconv)
+TARGET_LINK_LIBRARIES(rspamd-server rspamd-cdb)
+TARGET_LINK_LIBRARIES(rspamd-server rspamd-lpeg)
+TARGET_LINK_LIBRARIES(rspamd-server lcbtrie)
+IF(SYSTEM_ZSTD MATCHES "OFF")
+ TARGET_LINK_LIBRARIES(rspamd-server rspamd-zstd)
+ELSE()
+ TARGET_LINK_LIBRARIES(rspamd-server zstd)
+ENDIF()
+TARGET_LINK_LIBRARIES(rspamd-server rspamd-fastutf8)
+
+IF (ENABLE_CLANG_PLUGIN MATCHES "ON")
+ ADD_DEPENDENCIES(rspamd-server rspamd-clang)
+ENDIF()
+
+IF (NOT WITH_LUAJIT)
+ TARGET_LINK_LIBRARIES(rspamd-server rspamd-bit)
+ENDIF()
+
+IF (ENABLE_SNOWBALL MATCHES "ON")
+ TARGET_LINK_LIBRARIES(rspamd-server stemmer)
+ENDIF()
+TARGET_LINK_LIBRARIES(rspamd-server rspamd-hiredis)
+
+IF (ENABLE_FANN MATCHES "ON")
+ TARGET_LINK_LIBRARIES(rspamd-server fann)
+ENDIF ()
+
+IF (ENABLE_HYPERSCAN MATCHES "ON")
+ TARGET_LINK_LIBRARIES(rspamd-server hs)
+ENDIF()
+
+IF(WITH_BLAS)
+ TARGET_LINK_LIBRARIES(rspamd-server ${BLAS_REQUIRED_LIBRARIES})
+ENDIF()
+
+TARGET_LINK_LIBRARIES(rspamd-server ${RSPAMD_REQUIRED_LIBRARIES})
+ADD_BACKWARD(rspamd-server)
+
+ADD_EXECUTABLE(rspamd ${RSPAMDSRC} ${CMAKE_CURRENT_BINARY_DIR}/workers.c ${CMAKE_CURRENT_BINARY_DIR}/config.h)
+ADD_BACKWARD(rspamd)
+SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE CXX)
+SET_TARGET_PROPERTIES(rspamd-server PROPERTIES LINKER_LANGUAGE CXX)
+IF(NOT DEBIAN_BUILD)
+ SET_TARGET_PROPERTIES(rspamd PROPERTIES VERSION ${RSPAMD_VERSION})
+ENDIF(NOT DEBIAN_BUILD)
+
+#TARGET_LINK_LIBRARIES(rspamd ${RSPAMD_REQUIRED_LIBRARIES})
+TARGET_LINK_LIBRARIES(rspamd rspamd-server)
+
+INSTALL(TARGETS rspamd RUNTIME DESTINATION bin)
+INSTALL(TARGETS rspamd-server LIBRARY DESTINATION ${RSPAMD_LIBDIR}) \ No newline at end of file
diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt
new file mode 100644
index 0000000..edf3cc1
--- /dev/null
+++ b/src/client/CMakeLists.txt
@@ -0,0 +1,16 @@
+# Librspamdclient
+SET(LIBRSPAMDCLIENTSRC rspamdclient.c)
+
+# rspamc
+SET(RSPAMCSRC rspamc.cxx)
+
+ADD_EXECUTABLE(rspamc ${RSPAMCSRC} ${LIBRSPAMDCLIENTSRC})
+SET_TARGET_PROPERTIES(rspamc PROPERTIES COMPILE_FLAGS "-I${CMAKE_SOURCE_DIR}/lib")
+TARGET_LINK_LIBRARIES(rspamc rspamd-server)
+SET_TARGET_PROPERTIES(rspamc PROPERTIES LINKER_LANGUAGE CXX)
+
+IF(NOT DEBIAN_BUILD)
+ SET_TARGET_PROPERTIES(rspamc PROPERTIES VERSION ${RSPAMD_VERSION})
+ENDIF(NOT DEBIAN_BUILD)
+
+INSTALL(TARGETS rspamc RUNTIME DESTINATION bin)
diff --git a/src/client/rspamc.cxx b/src/client/rspamc.cxx
new file mode 100644
index 0000000..50e3cbd
--- /dev/null
+++ b/src/client/rspamc.cxx
@@ -0,0 +1,2380 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "libutil/util.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
+#include "libserver/cfg_file.h"
+#include "rspamdclient.h"
+#include "unix-std.h"
+
+#include <vector>
+#include <string>
+#include <optional>
+#include <algorithm>
+#include <functional>
+#include <cstdint>
+#include <cstdio>
+#include <cmath>
+
+#include "frozen/string.h"
+#include "frozen/unordered_map.h"
+#include "fmt/format.h"
+#include "fmt/color.h"
+#include "libutil/cxx/file_util.hxx"
+#include "libutil/cxx/util.hxx"
+
+#ifdef HAVE_SYS_WAIT_H
+
+#include <sys/wait.h>
+
+#endif
+
+#define DEFAULT_PORT 11333
+#define DEFAULT_CONTROL_PORT 11334
+
+static const char *connect_str = "localhost";
+static const char *password = nullptr;
+static const char *ip = nullptr;
+static const char *from = nullptr;
+static const char *deliver_to = nullptr;
+static const char **rcpts = nullptr;
+static const char *user = nullptr;
+static const char *helo = nullptr;
+static const char *hostname = nullptr;
+static const char *classifier = nullptr;
+static const char *local_addr = nullptr;
+static const char *execute = nullptr;
+static const char *sort = nullptr;
+static const char **http_headers = nullptr;
+static const char **exclude_patterns = nullptr;
+static int weight = 0;
+static int flag = 0;
+static const char *fuzzy_symbol = nullptr;
+static const char *dictionary = nullptr;
+static int max_requests = 8;
+static double timeout = 10.0;
+static gboolean pass_all;
+static gboolean tty = FALSE;
+static gboolean verbose = FALSE;
+static gboolean print_commands = FALSE;
+static gboolean humanreport = FALSE;
+static gboolean json = FALSE;
+static gboolean compact = FALSE;
+static gboolean headers = FALSE;
+static gboolean raw = FALSE;
+static gboolean ucl_reply = FALSE;
+static gboolean extended_urls = FALSE;
+static gboolean mime_output = FALSE;
+static gboolean empty_input = FALSE;
+static gboolean compressed = FALSE;
+static gboolean profile = FALSE;
+static gboolean skip_images = FALSE;
+static gboolean skip_attachments = FALSE;
+static const char *pubkey = nullptr;
+static const char *user_agent = "rspamc";
+
+std::vector<GPid> children;
+static GPatternSpec **exclude_compiled = nullptr;
+static struct rspamd_http_context *http_ctx;
+
+static gint retcode = EXIT_SUCCESS;
+
+static gboolean rspamc_password_callback(const gchar *option_name,
+ const gchar *value,
+ gpointer data,
+ GError **error);
+
+static GOptionEntry entries[] =
+ {
+ {"connect", 'h', 0, G_OPTION_ARG_STRING, &connect_str,
+ "Specify host and port", nullptr},
+ {"password", 'P', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
+ (void *) &rspamc_password_callback, "Specify control password", nullptr},
+ {"classifier", 'c', 0, G_OPTION_ARG_STRING, &classifier,
+ "Classifier to learn spam or ham", nullptr},
+ {"weight", 'w', 0, G_OPTION_ARG_INT, &weight,
+ "Weight for fuzzy operations", nullptr},
+ {"flag", 'f', 0, G_OPTION_ARG_INT, &flag, "Flag for fuzzy operations",
+ nullptr},
+ {"pass-all", 'p', 0, G_OPTION_ARG_NONE, &pass_all, "Pass all filters",
+ nullptr},
+ {"verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose, "More verbose output",
+ nullptr},
+ {"ip", 'i', 0, G_OPTION_ARG_STRING, &ip,
+ "Emulate that message was received from specified ip address",
+ nullptr},
+ {"user", 'u', 0, G_OPTION_ARG_STRING, &user,
+ "Emulate that message was received from specified authenticated user", nullptr},
+ {"deliver", 'd', 0, G_OPTION_ARG_STRING, &deliver_to,
+ "Emulate that message is delivered to specified user (for LDA/statistics)", nullptr},
+ {"from", 'F', 0, G_OPTION_ARG_STRING, &from,
+ "Emulate that message has specified SMTP FROM address", nullptr},
+ {"rcpt", 'r', 0, G_OPTION_ARG_STRING_ARRAY, &rcpts,
+ "Emulate that message has specified SMTP RCPT address", nullptr},
+ {"helo", 0, 0, G_OPTION_ARG_STRING, &helo,
+ "Imitate SMTP HELO passing from MTA", nullptr},
+ {"hostname", 0, 0, G_OPTION_ARG_STRING, &hostname,
+ "Imitate hostname passing from MTA", nullptr},
+ {"timeout", 't', 0, G_OPTION_ARG_DOUBLE, &timeout,
+ "Time in seconds to wait for a reply", nullptr},
+ {"bind", 'b', 0, G_OPTION_ARG_STRING, &local_addr,
+ "Bind to specified ip address", nullptr},
+ {"commands", 0, 0, G_OPTION_ARG_NONE, &print_commands,
+ "List available commands", nullptr},
+ {"human", 'R', 0, G_OPTION_ARG_NONE, &humanreport, "Output human readable report", nullptr},
+ {"json", 'j', 0, G_OPTION_ARG_NONE, &json, "Output json reply", nullptr},
+ {"compact", '\0', 0, G_OPTION_ARG_NONE, &compact, "Output compact json reply", nullptr},
+ {"headers", 0, 0, G_OPTION_ARG_NONE, &headers, "Output HTTP headers",
+ nullptr},
+ {"raw", 0, 0, G_OPTION_ARG_NONE, &raw, "Input is a raw file, not an email file",
+ nullptr},
+ {"ucl", 0, 0, G_OPTION_ARG_NONE, &ucl_reply, "Output ucl reply from rspamd",
+ nullptr},
+ {"max-requests", 'n', 0, G_OPTION_ARG_INT, &max_requests,
+ "Maximum count of parallel requests to rspamd", nullptr},
+ {"extended-urls", 0, 0, G_OPTION_ARG_NONE, &extended_urls,
+ "Output urls in extended format", nullptr},
+ {"key", 0, 0, G_OPTION_ARG_STRING, &pubkey,
+ "Use specified pubkey to encrypt request", nullptr},
+ {"exec", 'e', 0, G_OPTION_ARG_STRING, &execute,
+ "Execute the specified command and pass output to it", nullptr},
+ {"mime", 'm', 0, G_OPTION_ARG_NONE, &mime_output,
+ "Write mime body of message with headers instead of just a scan's result", nullptr},
+ {"header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &http_headers,
+ "Add custom HTTP header to query (can be repeated)", nullptr},
+ {"exclude", 0, 0, G_OPTION_ARG_STRING_ARRAY, &exclude_patterns,
+ "Exclude specific glob patterns in file names (can be repeated)", nullptr},
+ {"sort", 0, 0, G_OPTION_ARG_STRING, &sort,
+ "Sort output in a specific order (name, weight, frequency, hits)", nullptr},
+ {"empty", 'E', 0, G_OPTION_ARG_NONE, &empty_input,
+ "Allow empty input instead of reading from stdin", nullptr},
+ {"fuzzy-symbol", 'S', 0, G_OPTION_ARG_STRING, &fuzzy_symbol,
+ "Learn the specified fuzzy symbol", nullptr},
+ {"compressed", 'z', 0, G_OPTION_ARG_NONE, &compressed,
+ "Enable zstd compression", nullptr},
+ {"profile", '\0', 0, G_OPTION_ARG_NONE, &profile,
+ "Profile symbols execution time", nullptr},
+ {"dictionary", 'D', 0, G_OPTION_ARG_FILENAME, &dictionary,
+ "Use dictionary to compress data", nullptr},
+ {"skip-images", '\0', 0, G_OPTION_ARG_NONE, &skip_images,
+ "Skip images when learning/unlearning fuzzy", nullptr},
+ {"skip-attachments", '\0', 0, G_OPTION_ARG_NONE, &skip_attachments,
+ "Skip attachments when learning/unlearning fuzzy", nullptr},
+ {"user-agent", 'U', 0, G_OPTION_ARG_STRING, &user_agent,
+ "Use specific User-Agent instead of \"rspamc\"", nullptr},
+ {nullptr, 0, 0, G_OPTION_ARG_NONE, nullptr, nullptr, nullptr}};
+
+static void rspamc_symbols_output(FILE *out, ucl_object_t *obj);
+
+static void rspamc_uptime_output(FILE *out, ucl_object_t *obj);
+
+static void rspamc_counters_output(FILE *out, ucl_object_t *obj);
+
+static void rspamc_stat_output(FILE *out, ucl_object_t *obj);
+
+enum rspamc_command_type {
+ RSPAMC_COMMAND_UNKNOWN = 0,
+ RSPAMC_COMMAND_CHECK,
+ RSPAMC_COMMAND_SYMBOLS,
+ RSPAMC_COMMAND_LEARN_SPAM,
+ RSPAMC_COMMAND_LEARN_HAM,
+ RSPAMC_COMMAND_FUZZY_ADD,
+ RSPAMC_COMMAND_FUZZY_DEL,
+ RSPAMC_COMMAND_FUZZY_DELHASH,
+ RSPAMC_COMMAND_STAT,
+ RSPAMC_COMMAND_STAT_RESET,
+ RSPAMC_COMMAND_COUNTERS,
+ RSPAMC_COMMAND_UPTIME,
+ RSPAMC_COMMAND_ADD_SYMBOL,
+ RSPAMC_COMMAND_ADD_ACTION
+};
+
+struct rspamc_command {
+ enum rspamc_command_type cmd;
+ const char *name;
+ const char *path;
+ const char *description;
+ gboolean is_controller;
+ gboolean is_privileged;
+ gboolean need_input;
+
+ void (*command_output_func)(FILE *, ucl_object_t *obj);
+};
+
+static const constexpr auto rspamc_commands = rspamd::array_of(
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_SYMBOLS,
+ .name = "symbols",
+ .path = "checkv2",
+ .description = "scan message and show symbols (default command)",
+ .is_controller = FALSE,
+ .is_privileged = FALSE,
+ .need_input = TRUE,
+ .command_output_func = rspamc_symbols_output},
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_LEARN_SPAM,
+ .name = "learn_spam",
+ .path = "learnspam",
+ .description = "learn message as spam",
+ .is_controller = TRUE,
+ .is_privileged = TRUE,
+ .need_input = TRUE,
+ .command_output_func = nullptr},
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_LEARN_HAM,
+ .name = "learn_ham",
+ .path = "learnham",
+ .description = "learn message as ham",
+ .is_controller = TRUE,
+ .is_privileged = TRUE,
+ .need_input = TRUE,
+ .command_output_func = nullptr},
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_FUZZY_ADD,
+ .name = "fuzzy_add",
+ .path = "fuzzyadd",
+ .description =
+ "add hashes from a message to the fuzzy storage (check -f and -w options for this command)",
+ .is_controller = TRUE,
+ .is_privileged = TRUE,
+ .need_input = TRUE,
+ .command_output_func = nullptr},
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_FUZZY_DEL,
+ .name = "fuzzy_del",
+ .path = "fuzzydel",
+ .description =
+ "delete hashes from a message from the fuzzy storage (check -f option for this command)",
+ .is_controller = TRUE,
+ .is_privileged = TRUE,
+ .need_input = TRUE,
+ .command_output_func = nullptr},
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_FUZZY_DELHASH,
+ .name = "fuzzy_delhash",
+ .path = "fuzzydelhash",
+ .description =
+ "delete a hash from fuzzy storage (check -f option for this command)",
+ .is_controller = TRUE,
+ .is_privileged = TRUE,
+ .need_input = FALSE,
+ .command_output_func = nullptr},
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_STAT,
+ .name = "stat",
+ .path = "stat",
+ .description = "show rspamd statistics",
+ .is_controller = TRUE,
+ .is_privileged = FALSE,
+ .need_input = FALSE,
+ .command_output_func = rspamc_stat_output,
+ },
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_STAT_RESET,
+ .name = "stat_reset",
+ .path = "statreset",
+ .description = "show and reset rspamd statistics (useful for graphs)",
+ .is_controller = TRUE,
+ .is_privileged = TRUE,
+ .need_input = FALSE,
+ .command_output_func = rspamc_stat_output},
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_COUNTERS,
+ .name = "counters",
+ .path = "counters",
+ .description = "display rspamd symbols statistics",
+ .is_controller = TRUE,
+ .is_privileged = FALSE,
+ .need_input = FALSE,
+ .command_output_func = rspamc_counters_output},
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_UPTIME,
+ .name = "uptime",
+ .path = "auth",
+ .description = "show rspamd uptime",
+ .is_controller = TRUE,
+ .is_privileged = FALSE,
+ .need_input = FALSE,
+ .command_output_func = rspamc_uptime_output},
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_ADD_SYMBOL,
+ .name = "add_symbol",
+ .path = "addsymbol",
+ .description = "add or modify symbol settings in rspamd",
+ .is_controller = TRUE,
+ .is_privileged = TRUE,
+ .need_input = FALSE,
+ .command_output_func = nullptr},
+ rspamc_command{
+ .cmd = RSPAMC_COMMAND_ADD_ACTION,
+ .name = "add_action",
+ .path = "addaction",
+ .description = "add or modify action settings",
+ .is_controller = TRUE,
+ .is_privileged = TRUE,
+ .need_input = FALSE,
+ .command_output_func = nullptr});
+
+struct rspamc_callback_data {
+ struct rspamc_command cmd;
+ std::string filename;
+};
+
+template<typename T>
+static constexpr auto emphasis_argument(const T &arg) -> auto
+{
+ if (tty) {
+ return fmt::format(fmt::emphasis::bold, "{}", arg);
+ }
+
+ return fmt::format("{}", arg);
+}
+
+template<typename T, typename std::enable_if_t<std::is_floating_point_v<T>, bool> = false>
+static constexpr auto emphasis_argument(const T &arg, int precision) -> auto
+{
+ if (tty) {
+ return fmt::format(fmt::emphasis::bold, "{:.{}f}", arg, precision);
+ }
+
+ return fmt::format("{:.{}f}", arg, precision);
+}
+
+template<>
+struct fmt::formatter<rspamd_action_type> : fmt::formatter<string_view> {
+ auto format(rspamd_action_type c, format_context &ctx) const
+ {
+ return formatter<string_view>::format(std::string_view{rspamd_action_to_str(c)}, ctx);
+ }
+};
+
+template<typename... T>
+static inline void rspamc_print(std::FILE *f, fmt::format_string<T...> fmt, T &&...args)
+{
+ static auto try_print_exception = true;
+ auto wanna_die = false;
+
+ try {
+ fmt::print(f, fmt, std::forward<T>(args)...);
+ } catch (const fmt::format_error &err) {
+ if (try_print_exception) {
+ if (fprintf(stderr, "Format error: %s\n", err.what()) < 0) {
+ try_print_exception = false;
+ }
+ }
+ } catch (std::system_error &err) {
+ wanna_die = true;
+ if (try_print_exception) {
+ if (fprintf(stderr, "System error: %s\n", err.what()) < 0) {
+ try_print_exception = false;
+ }
+ }
+ } catch (...) {
+ wanna_die = true;
+ if (try_print_exception) {
+ if (fprintf(stderr, "Unknown format error\n") < 0) {
+ try_print_exception = false;
+ }
+ }
+ }
+
+ if (wanna_die) {
+ exit(EXIT_FAILURE);
+ }
+}
+
+using sort_lambda = std::function<int(const ucl_object_t *, const ucl_object_t *)>;
+static const auto sort_map = frozen::make_unordered_map<frozen::string, sort_lambda>({
+ {"name", [](const ucl_object_t *o1, const ucl_object_t *o2) -> int {
+ const auto *elt1 = ucl_object_lookup(o1, "symbol");
+ const auto *elt2 = ucl_object_lookup(o2, "symbol");
+
+ if (elt1 && elt2) {
+ return strcmp(ucl_object_tostring(elt1),
+ ucl_object_tostring(elt2));
+ }
+ else if (ucl_object_key(o1) != nullptr && ucl_object_key(o2) != nullptr) {
+ return strcmp(ucl_object_key(o1),
+ ucl_object_key(o2));
+ }
+ return 0;
+ }},
+ {"weight", [](const ucl_object_t *o1, const ucl_object_t *o2) -> int {
+ const auto *elt1 = ucl_object_lookup(o1, "weight");
+ const auto *elt2 = ucl_object_lookup(o2, "weight");
+
+ if (elt1 && elt2) {
+ return ucl_object_todouble(elt2) * 1000.0 - ucl_object_todouble(elt1) * 1000.0;
+ }
+ return 0;
+ }},
+ {"score", [](const ucl_object_t *o1, const ucl_object_t *o2) -> int {
+ const auto *elt1 = ucl_object_lookup(o1, "score");
+ const auto *elt2 = ucl_object_lookup(o2, "score");
+
+ if (elt1 && elt2) {
+ return std::fabs(ucl_object_todouble(elt2)) * 1000.0 -
+ std::fabs(ucl_object_todouble(elt1)) * 1000.0;
+ }
+ return 0;
+ }},
+ {"time", [](const ucl_object_t *o1, const ucl_object_t *o2) -> int {
+ const auto *elt1 = ucl_object_lookup(o1, "time");
+ const auto *elt2 = ucl_object_lookup(o2, "time");
+
+ if (elt1 && elt2) {
+ return ucl_object_todouble(elt2) * 1000.0 - ucl_object_todouble(elt1) * 1000.0;
+ }
+ return 0;
+ }},
+ {"frequency", [](const ucl_object_t *o1, const ucl_object_t *o2) -> int {
+ const auto *elt1 = ucl_object_lookup(o1, "frequency");
+ const auto *elt2 = ucl_object_lookup(o2, "frequency");
+
+ if (elt1 && elt2) {
+ return ucl_object_todouble(elt2) * 1000.0 - ucl_object_todouble(elt1) * 1000.0;
+ }
+ return 0;
+ }},
+ {"hits", [](const ucl_object_t *o1, const ucl_object_t *o2) -> int {
+ const auto *elt1 = ucl_object_lookup(o1, "hits");
+ const auto *elt2 = ucl_object_lookup(o2, "hits");
+
+ if (elt1 && elt2) {
+ return ucl_object_toint(elt2) - ucl_object_toint(elt1);
+ }
+ return 0;
+ }},
+});
+
+/* TODO: remove once migrate to C++20 standard */
+static constexpr auto
+sv_ends_with(std::string_view inp, std::string_view suffix) -> bool
+{
+ return inp.size() >= suffix.size() && inp.compare(inp.size() - suffix.size(), std::string_view::npos, suffix) == 0;
+}
+
+template<typename T>
+auto sort_ucl_container_with_default(T &cont, const char *default_sort,
+ typename std::enable_if<std::is_same_v<typename T::value_type, const ucl_object_t *>>::type * = 0) -> void
+{
+ auto real_sort = sort ? sort : default_sort;
+ if (real_sort) {
+ auto sort_view = std::string_view{real_sort};
+ auto inverse = false;
+
+ if (sv_ends_with(sort_view, ":asc")) {
+ inverse = true;
+ sort_view = std::string_view{sort, strlen(sort) - sizeof(":asc") + 1};
+ }
+
+ const auto sort_functor = sort_map.find(sort_view);
+ if (sort_functor != sort_map.end()) {
+ std::stable_sort(std::begin(cont), std::end(cont),
+ [&](const ucl_object_t *o1, const ucl_object_t *o2) -> int {
+ auto order = sort_functor->second(o1, o2);
+
+ return inverse ? order > 0 : order < 0;
+ });
+ }
+ }
+}
+
+
+static gboolean
+rspamc_password_callback(const gchar *option_name,
+ const gchar *value,
+ gpointer data,
+ GError **error)
+{
+ // Some efforts to keep password erased
+ static std::vector<char, rspamd::secure_mem_allocator<char>> processed_passwd;
+ processed_passwd.clear();
+
+ if (value != nullptr) {
+ std::string_view value_view{value};
+ if (value_view[0] == '/' || value_view[0] == '.') {
+ /* Try to open file */
+ auto locked_mmap = rspamd::util::raii_mmaped_file::mmap_shared(value, O_RDONLY, PROT_READ);
+
+ if (!locked_mmap.has_value() || locked_mmap.value().get_size() == 0) {
+ /* Just use it as a string */
+ processed_passwd.assign(std::begin(value_view), std::end(value_view));
+ processed_passwd.push_back('\0');
+ }
+ else {
+ /* Strip trailing spaces */
+ auto *map = (char *) locked_mmap.value().get_map();
+ auto *end = map + locked_mmap.value().get_size() - 1;
+
+ while (g_ascii_isspace(*end) && end > map) {
+ end--;
+ }
+
+ end++;
+ value_view = std::string_view{map, static_cast<std::size_t>(end - map + 1)};
+ processed_passwd.assign(std::begin(value_view), std::end(value_view));
+ processed_passwd.push_back('\0');
+ }
+ }
+ else {
+ processed_passwd.assign(std::begin(value_view), std::end(value_view));
+ processed_passwd.push_back('\0');
+ }
+ }
+ else {
+ /* Read password from console */
+ auto plen = 8192;
+ processed_passwd.resize(plen, '\0');
+ plen = rspamd_read_passphrase(processed_passwd.data(), plen, 0, nullptr);
+ if (plen == 0) {
+ rspamc_print(stderr, "Invalid password\n");
+ exit(EXIT_FAILURE);
+ }
+ processed_passwd.resize(plen);
+ processed_passwd.push_back('\0');
+ }
+
+ password = processed_passwd.data();
+
+ return TRUE;
+}
+
+/*
+ * Parse command line
+ */
+static void
+read_cmd_line(gint *argc, gchar ***argv)
+{
+ GError *error = nullptr;
+ GOptionContext *context;
+
+ /* Prepare parser */
+ context = g_option_context_new("- run rspamc client");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd client version " RVERSION "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, nullptr);
+
+ /* Parse options */
+ if (!g_option_context_parse(context, argc, argv, &error)) {
+ rspamc_print(stderr, "option parsing failed: {}\n", error->message);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ if (json || compact) {
+ ucl_reply = TRUE;
+ }
+ /* Argc and argv are shifted after this function */
+ g_option_context_free(context);
+}
+
+static auto
+add_client_header(GQueue *opts, const char *hn, const char *hv) -> void
+{
+ g_assert(hn != nullptr);
+ g_assert(hv != nullptr);
+ auto *nhdr = g_new(rspamd_http_client_header, 1);
+ nhdr->name = g_strdup(hn);
+ nhdr->value = g_strdup(hv);
+ g_queue_push_tail(opts, (void *) nhdr);
+}
+
+static auto
+add_client_header(GQueue *opts, std::string_view hn, std::string_view hv) -> void
+{
+ auto *nhdr = g_new(rspamd_http_client_header, 1);
+ nhdr->name = g_new(char, hn.size() + 1);
+ rspamd_strlcpy(nhdr->name, hn.data(), hn.size() + 1);
+ nhdr->value = g_new(char, hv.size() + 1);
+ rspamd_strlcpy(nhdr->value, hv.data(), hv.size() + 1);
+ g_queue_push_tail(opts, (void *) nhdr);
+}
+
+static auto
+rspamd_string_tolower(const char *inp) -> std::string
+{
+ std::string s{inp};
+ std::transform(std::begin(s), std::end(s), std::begin(s),
+ [](unsigned char c) { return std::tolower(c); });
+ return s;
+}
+
+static auto
+rspamd_action_from_str_rspamc(const char *data) -> std::optional<int>
+{
+ static constexpr const auto str_map = frozen::make_unordered_map<frozen::string, int>({
+ {"reject", METRIC_ACTION_REJECT},
+ {"greylist", METRIC_ACTION_GREYLIST},
+ {"add_header", METRIC_ACTION_ADD_HEADER},
+ {"add header", METRIC_ACTION_ADD_HEADER},
+ {"rewrite_subject", METRIC_ACTION_REWRITE_SUBJECT},
+ {"rewrite subject", METRIC_ACTION_REWRITE_SUBJECT},
+ {"soft_reject", METRIC_ACTION_SOFT_REJECT},
+ {"soft reject", METRIC_ACTION_SOFT_REJECT},
+ {"no_action", METRIC_ACTION_NOACTION},
+ {"no action", METRIC_ACTION_NOACTION},
+ });
+
+ auto st_lower = rspamd_string_tolower(data);
+ return rspamd::find_map(str_map, std::string_view{st_lower});
+}
+
+/*
+ * Check rspamc command from string (used for arguments parsing)
+ */
+static auto
+check_rspamc_command(const char *cmd) -> std::optional<rspamc_command>
+{
+ static constexpr const auto str_map = frozen::make_unordered_map<frozen::string, int>({
+ {"symbols", RSPAMC_COMMAND_SYMBOLS},
+ {"check", RSPAMC_COMMAND_SYMBOLS},
+ {"report", RSPAMC_COMMAND_SYMBOLS},
+ {"learn_spam", RSPAMC_COMMAND_LEARN_SPAM},
+ {"learn_ham", RSPAMC_COMMAND_LEARN_HAM},
+ {"fuzzy_add", RSPAMC_COMMAND_FUZZY_ADD},
+ {"fuzzy_del", RSPAMC_COMMAND_FUZZY_DEL},
+ {"fuzzy_delhash", RSPAMC_COMMAND_FUZZY_DELHASH},
+ {"stat", RSPAMC_COMMAND_STAT},
+ {"stat_reset", RSPAMC_COMMAND_STAT_RESET},
+ {"counters", RSPAMC_COMMAND_COUNTERS},
+ {"uptime", RSPAMC_COMMAND_UPTIME},
+ });
+
+ std::string cmd_lc = rspamd_string_tolower(cmd);
+ auto ct = rspamd::find_map(str_map, std::string_view{cmd_lc});
+
+ auto elt_it = std::find_if(rspamc_commands.begin(), rspamc_commands.end(), [&](const auto &item) {
+ return item.cmd == ct;
+ });
+
+ if (elt_it != std::end(rspamc_commands)) {
+ return *elt_it;
+ }
+
+ return std::nullopt;
+}
+
+static void
+print_commands_list()
+{
+ guint cmd_len = 0;
+
+ rspamc_print(stdout, "Rspamc commands summary:\n");
+
+ for (const auto &cmd: rspamc_commands) {
+ auto clen = strlen(cmd.name);
+
+ if (clen > cmd_len) {
+ cmd_len = clen;
+ }
+ }
+
+ for (const auto &cmd: rspamc_commands) {
+ rspamc_print(stdout,
+ " {:>{}} ({:7}{:1})\t{}\n",
+ cmd.name,
+ cmd_len,
+ cmd.is_controller ? "control" : "normal",
+ cmd.is_privileged ? "*" : "",
+ cmd.description);
+ }
+
+ rspamc_print(stdout,
+ "\n* is for privileged commands that may need password (see -P option)\n");
+ rspamc_print(stdout,
+ "control commands use port 11334 while normal use 11333 by default (see -h option)\n");
+}
+
+static void
+add_options(GQueue *opts)
+{
+ std::string flagbuf;
+
+ if (ip != nullptr) {
+ rspamd_inet_addr_t *addr = nullptr;
+
+ if (!rspamd_parse_inet_address(&addr, ip, strlen(ip),
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ /* Try to resolve */
+ struct addrinfo hints, *res, *cur;
+ int r;
+
+ memset(&hints, 0, sizeof(hints));
+ hints.ai_socktype = SOCK_STREAM; /* Type of the socket */
+#ifdef AI_IDN
+ hints.ai_flags = AI_NUMERICSERV | AI_IDN;
+#else
+ hints.ai_flags = AI_NUMERICSERV;
+#endif
+ hints.ai_family = AF_UNSPEC;
+
+ if ((r = getaddrinfo(ip, "25", &hints, &res)) == 0) {
+
+ cur = res;
+ while (cur) {
+ addr = rspamd_inet_address_from_sa(cur->ai_addr,
+ cur->ai_addrlen);
+
+ if (addr != nullptr) {
+ ip = g_strdup(rspamd_inet_address_to_string(addr));
+ rspamd_inet_address_free(addr);
+ break;
+ }
+
+ cur = cur->ai_next;
+ }
+
+ freeaddrinfo(res);
+ }
+ else {
+ rspamc_print(stderr, "address resolution for {} failed: {}\n",
+ ip,
+ gai_strerror(r));
+ }
+ }
+ else {
+ rspamd_inet_address_free(addr);
+ }
+
+ add_client_header(opts, "Ip", ip);
+ }
+
+ if (from != nullptr) {
+ add_client_header(opts, "From", from);
+ }
+
+ if (user != nullptr) {
+ add_client_header(opts, "User", user);
+ }
+
+ if (rcpts != nullptr) {
+ for (auto *rcpt = rcpts; *rcpt != nullptr; rcpt++) {
+ add_client_header(opts, "Rcpt", *rcpt);
+ }
+ }
+
+ if (deliver_to != nullptr) {
+ add_client_header(opts, "Deliver-To", deliver_to);
+ }
+
+ if (helo != nullptr) {
+ add_client_header(opts, "Helo", helo);
+ }
+
+ if (hostname != nullptr) {
+ add_client_header(opts, "Hostname", hostname);
+ }
+
+ if (password != nullptr) {
+ add_client_header(opts, "Password", password);
+ }
+
+ if (pass_all) {
+ flagbuf += "pass_all,";
+ }
+
+ if (raw) {
+ add_client_header(opts, "Raw", "yes");
+ }
+
+ if (classifier) {
+ add_client_header(opts, "Classifier", classifier);
+ }
+
+ if (weight != 0) {
+ auto nstr = fmt::format("{}", weight);
+ add_client_header(opts, "Weight", nstr.c_str());
+ }
+
+ if (fuzzy_symbol != nullptr) {
+ add_client_header(opts, "Symbol", fuzzy_symbol);
+ }
+
+ if (flag != 0) {
+ auto nstr = fmt::format("{}", flag);
+ add_client_header(opts, "Flag", nstr.c_str());
+ }
+
+ if (extended_urls) {
+ add_client_header(opts, "URL-Format", "extended");
+ }
+
+ if (profile) {
+ flagbuf += "profile,";
+ }
+
+ flagbuf += "body_block,";
+
+ if (skip_images) {
+ add_client_header(opts, "Skip-Images", "true");
+ }
+
+ if (skip_attachments) {
+ add_client_header(opts, "Skip-Attachments", "true");
+ }
+
+ auto hdr = http_headers;
+
+ while (hdr != nullptr && *hdr != nullptr) {
+ std::string_view hdr_view{*hdr};
+
+ auto delim_pos = std::find_if(std::begin(hdr_view), std::end(hdr_view), [](auto c) {
+ return c == ':' || c == '=';
+ });
+ if (delim_pos == std::end(hdr_view)) {
+ /* Just a header name with no value */
+ add_client_header(opts, *hdr, "");
+ }
+ else {
+ add_client_header(opts,
+ hdr_view.substr(0, std::distance(std::begin(hdr_view), delim_pos)),
+ hdr_view.substr(std::distance(std::begin(hdr_view), delim_pos) + 1));
+ }
+
+ hdr++;
+ }
+
+ if (!flagbuf.empty()) {
+ if (flagbuf.back() == ',') {
+ flagbuf.pop_back();
+ }
+
+ add_client_header(opts, "Flags", flagbuf.c_str());
+ }
+}
+
+template<std::size_t maxlen, std::size_t indent>
+static auto
+rspamc_print_indented_line(FILE *out, std::string_view line) -> void
+{
+ static_assert(maxlen > 0, "maxlen must be > 0");
+ static_assert(maxlen > indent, "maxlen must be more than indent");
+ using namespace std::literals;
+
+ constexpr const auto whitespace = " \f\n\r\t\v"sv;
+ constexpr const auto break_begin = " \f\n\r\t\v?,;<({[~!#$%^&*+:=/\\|"sv;
+ constexpr const auto break_end = " \f\n\r\t\v?,;]})>~!#$%^&*+:_=/\\|"sv;
+
+ for (size_t pos = 0; pos < line.size();) {
+ auto len = pos ? (maxlen - indent) : maxlen;
+ auto s = line.substr(pos, len);
+ if ((pos + s.size()) < line.size() && // reached EOL?
+ break_begin.find_first_of(line.at(pos + s.size())) == std::string_view::npos// new word next?
+ ) {
+ auto wrap_at = s.find_last_of(break_end);
+ if (wrap_at != std::string_view::npos) {
+ s = line.substr(pos, wrap_at + 1);
+ }
+ }
+ if (indent && pos) {
+ rspamc_print(out, "{:>{}}", " ", indent);
+ }
+ rspamc_print(out, "{}\n", s);
+ pos = line.find_first_not_of(whitespace, pos + s.size());//skip leading whitespace
+ }
+}
+
+static void
+rspamc_symbol_human_output(FILE *out, const ucl_object_t *obj)
+{
+ auto first = true;
+ auto score = 0.0;
+ const char *desc = nullptr;
+
+ const auto *key = ucl_object_key(obj);
+ const auto *val = ucl_object_lookup(obj, "score");
+ if (val != nullptr) {
+ score = ucl_object_todouble(val);
+ }
+
+ val = ucl_object_lookup(obj, "description");
+ if (val != nullptr) {
+ desc = ucl_object_tostring(val);
+ }
+
+ auto line = fmt::format("{:>4.1f} {:<22} ", score, key);
+ if (desc != nullptr) {
+ line += desc;
+ }
+
+ val = ucl_object_lookup(obj, "options");
+ if (val != nullptr && ucl_object_type(val) == UCL_ARRAY) {
+ ucl_object_iter_t it = nullptr;
+ const ucl_object_t *cur;
+
+ line += fmt::format("{}[", desc == nullptr ? "" : " ");
+
+ while ((cur = ucl_object_iterate(val, &it, true)) != nullptr) {
+ if (first) {
+ line += fmt::format("{}", ucl_object_tostring(cur));
+ first = false;
+ }
+ else {
+ line += fmt::format(",{}", ucl_object_tostring(cur));
+ }
+ }
+ line += ']';
+ }
+
+ rspamc_print_indented_line<78, 28>(out, line);
+}
+
+static void
+rspamc_symbol_output(FILE *out, const ucl_object_t *obj)
+{
+ auto first = true;
+
+ rspamc_print(out, "Symbol: {} ", ucl_object_key(obj));
+ const auto *val = ucl_object_lookup(obj, "score");
+
+ if (val != nullptr) {
+ rspamc_print(out, "({:.2f})", ucl_object_todouble(val));
+ }
+ val = ucl_object_lookup(obj, "options");
+ if (val != nullptr && ucl_object_type(val) == UCL_ARRAY) {
+ ucl_object_iter_t it = nullptr;
+ const ucl_object_t *cur;
+
+ rspamc_print(out, "[");
+
+ while ((cur = ucl_object_iterate(val, &it, true)) != nullptr) {
+ if (first) {
+ rspamc_print(out, "{}", ucl_object_tostring(cur));
+ first = false;
+ }
+ else {
+ rspamc_print(out, ", {}", ucl_object_tostring(cur));
+ }
+ }
+ rspamc_print(out, "]");
+ }
+ rspamc_print(out, "\n");
+}
+
+static void
+rspamc_metric_output(FILE *out, const ucl_object_t *obj)
+{
+ int got_scores = 0;
+ bool is_spam = false;
+ double score = 0, required_score = 0, greylist_score = 0, addheader_score = 0;
+
+ auto print_protocol_string = [&](const char *ucl_name, const char *output_message) {
+ auto *elt = ucl_object_lookup(obj, ucl_name);
+ if (elt) {
+ if (humanreport) {
+ rspamc_print(out, ",{}={}", output_message, emphasis_argument(ucl_object_tostring(elt)));
+ }
+ else {
+ rspamc_print(out, "{}: {}\n", output_message, emphasis_argument(ucl_object_tostring(elt)));
+ }
+ }
+ };
+
+ if (!humanreport) {
+ rspamc_print(out, "[Metric: default]\n");
+ }
+
+ const auto *elt = ucl_object_lookup(obj, "required_score");
+ if (elt) {
+ required_score = ucl_object_todouble(elt);
+ got_scores++;
+ }
+
+ elt = ucl_object_lookup(obj, "score");
+ if (elt) {
+ score = ucl_object_todouble(elt);
+ got_scores++;
+ }
+
+ const auto *thresholds_obj = ucl_object_lookup(obj, "thresholds");
+
+ if (thresholds_obj && ucl_object_type(thresholds_obj) == UCL_OBJECT) {
+ const auto *action_obj = ucl_object_lookup(thresholds_obj, "greylist");
+ if (action_obj) {
+ greylist_score = ucl_object_todouble(action_obj);
+ }
+
+ action_obj = ucl_object_lookup(thresholds_obj, "add header");
+ if (action_obj) {
+ addheader_score = ucl_object_todouble(action_obj);
+ }
+
+ action_obj = ucl_object_lookup(thresholds_obj, "reject");
+ if (action_obj) {
+ required_score = ucl_object_todouble(action_obj);
+ }
+ }
+
+
+ if (humanreport) {
+ rspamc_print(out,
+ "{}/{}/{}/{}",
+ emphasis_argument(score, 2),
+ emphasis_argument(greylist_score, 2),
+ emphasis_argument(addheader_score, 2),
+ emphasis_argument(required_score, 2));
+ }
+
+ elt = ucl_object_lookup(obj, "action");
+ if (elt) {
+ auto act = rspamd_action_from_str_rspamc(ucl_object_tostring(elt));
+
+ if (act.has_value()) {
+ if (!tty) {
+ if (humanreport) {
+ rspamc_print(out, ",action={}:{}", act.value(), ucl_object_tostring(elt));
+ }
+ else {
+ print_protocol_string("action", "Action");
+ }
+ }
+ else {
+ /* Colorize action type */
+ std::string colorized_action;
+ switch (act.value()) {
+ case METRIC_ACTION_REJECT:
+ colorized_action = fmt::format(fmt::fg(fmt::color::red), "reject");
+ break;
+ case METRIC_ACTION_NOACTION:
+ colorized_action = fmt::format(fmt::fg(fmt::color::green), "no action");
+ break;
+ case METRIC_ACTION_ADD_HEADER:
+ case METRIC_ACTION_REWRITE_SUBJECT:
+ colorized_action = fmt::format(fmt::fg(fmt::color::orange), ucl_object_tostring(elt));
+ break;
+ case METRIC_ACTION_GREYLIST:
+ case METRIC_ACTION_SOFT_REJECT:
+ colorized_action = fmt::format(fmt::fg(fmt::color::gray), ucl_object_tostring(elt));
+ break;
+ default:
+ colorized_action = fmt::format(fmt::emphasis::bold, ucl_object_tostring(elt));
+ break;
+ }
+
+ if (humanreport) {
+ rspamc_print(out, ",action={}:{}", act.value(), colorized_action);
+ }
+ else {
+ rspamc_print(out, "Action: {}\n", colorized_action);
+ }
+ }
+
+ is_spam = act.value() < METRIC_ACTION_GREYLIST ? true : false;
+ if (!humanreport) {
+ rspamc_print(out, "Spam: {}\n", is_spam ? "true" : "false");
+ }
+ }
+ else {
+ if (humanreport) {
+ rspamc_print(out, ",action={}:{}", METRIC_ACTION_NOACTION, ucl_object_tostring(elt));
+ }
+ else {
+ print_protocol_string("action", "Action");
+ }
+ }
+ }
+
+ if (!humanreport) {
+ print_protocol_string("subject", "Subject");
+ }
+
+ if (humanreport) {
+ auto is_skipped = 0;
+
+ elt = ucl_object_lookup(obj, "is_skipped");
+ if (elt && ucl_object_toboolean(elt)) {
+ is_skipped = 1;
+ }
+
+ rspamc_print(out, ",spam={},skipped={}\n", is_spam ? 1 : 0, is_skipped);
+ }
+ else if (got_scores == 2) {
+ rspamc_print(out,
+ "Score: {} / {}\n",
+ emphasis_argument(score, 2),
+ emphasis_argument(required_score, 2));
+ }
+
+ if (humanreport) {
+ rspamc_print(out, "Content analysis details: ({} points, {} required)\n\n",
+ emphasis_argument(score, 2),
+ emphasis_argument(addheader_score, 2));
+ rspamc_print(out, " pts rule name description\n");
+ rspamc_print(out, "---- ---------------------- --------------------------------------------------\n");
+ }
+
+ elt = ucl_object_lookup(obj, "symbols");
+
+ if (elt) {
+ std::vector<const ucl_object_t *> symbols;
+ ucl_object_iter_t it = nullptr;
+ const ucl_object_t *cur;
+
+ while ((cur = ucl_object_iterate(elt, &it, true)) != nullptr) {
+ symbols.push_back(cur);
+ }
+
+ sort_ucl_container_with_default(symbols, "name");
+
+ for (const auto *sym_obj: symbols) {
+ humanreport ? rspamc_symbol_human_output(out, sym_obj) : rspamc_symbol_output(out, sym_obj);
+ }
+ }
+ if (humanreport) {
+ rspamc_print(out, "\n");
+ }
+}
+
+static void
+rspamc_profile_output(FILE *out, const ucl_object_t *obj)
+{
+ ucl_object_iter_t it = nullptr;
+ const ucl_object_t *cur;
+
+ std::vector<const ucl_object_t *> ar;
+
+ while ((cur = ucl_object_iterate(obj, &it, true)) != nullptr) {
+ ar.push_back(cur);
+ }
+ std::stable_sort(std::begin(ar), std::end(ar),
+ [](const ucl_object_t *u1, const ucl_object_t *u2) -> int {
+ return ucl_object_compare(u1, u2);
+ });
+
+ for (const auto *cur_elt: ar) {
+ rspamc_print(out, "\t{}: {:3} usec\n",
+ ucl_object_key(cur_elt), ucl_object_todouble(cur_elt));
+ }
+}
+
+static void
+rspamc_symbols_output(FILE *out, ucl_object_t *obj)
+{
+ rspamc_metric_output(out, obj);
+
+ auto print_protocol_string = [&](const char *ucl_name, const char *output_message) {
+ auto *elt = ucl_object_lookup(obj, ucl_name);
+ if (elt) {
+ rspamc_print(out, "{}: {}\n", output_message, ucl_object_tostring(elt));
+ }
+ };
+
+ if (!humanreport) {
+ print_protocol_string("message-id", "Message-ID");
+ print_protocol_string("queue-id", "Queue-ID");
+ }
+
+ const auto *elt = ucl_object_lookup(obj, "urls");
+
+ if (elt) {
+ char *emitted;
+
+ if (!extended_urls || compact) {
+ emitted = (char *) ucl_object_emit(elt, UCL_EMIT_JSON_COMPACT);
+ }
+ else {
+ emitted = (char *) ucl_object_emit(elt, UCL_EMIT_JSON);
+ }
+
+ if (humanreport) {
+ if (emitted && strcmp(emitted, "[]") != 0) {
+ rspamc_print_indented_line<78, 4>(out, fmt::format("Domains found: {}", emitted));
+ }
+ }
+ else {
+ rspamc_print(out, "Urls: {}\n", emitted);
+ }
+ free(emitted);
+ }
+
+ elt = ucl_object_lookup(obj, "emails");
+
+ if (elt) {
+ char *emitted;
+ if (!extended_urls || compact) {
+ emitted = (char *) ucl_object_emit(elt, UCL_EMIT_JSON_COMPACT);
+ }
+ else {
+ emitted = (char *) ucl_object_emit(elt, UCL_EMIT_JSON);
+ }
+
+ if (humanreport) {
+ if (emitted && strcmp(emitted, "[]") != 0) {
+ rspamc_print_indented_line<78, 4>(out, fmt::format("Emails found: {}", emitted));
+ }
+ }
+ else {
+ rspamc_print(out, "Emails: {}\n", emitted);
+ }
+ free(emitted);
+ }
+
+ print_protocol_string("error", "Scan error");
+ if (humanreport) {
+ return;
+ }
+
+ elt = ucl_object_lookup(obj, "messages");
+ if (elt && elt->type == UCL_OBJECT) {
+ ucl_object_iter_t mit = nullptr;
+ const ucl_object_t *cmesg;
+
+ while ((cmesg = ucl_object_iterate(elt, &mit, true)) != nullptr) {
+ if (ucl_object_type(cmesg) == UCL_STRING) {
+ rspamc_print(out, "Message - {}: {}\n",
+ ucl_object_key(cmesg), ucl_object_tostring(cmesg));
+ }
+ else {
+ char *rendered_message;
+ rendered_message = (char *) ucl_object_emit(cmesg, UCL_EMIT_JSON_COMPACT);
+ rspamc_print(out, "Message - {}: {:.60}\n",
+ ucl_object_key(cmesg), rendered_message);
+ free(rendered_message);
+ }
+ }
+ }
+
+ elt = ucl_object_lookup(obj, "dkim-signature");
+ if (elt && elt->type == UCL_STRING) {
+ rspamc_print(out, "DKIM-Signature: {}\n", ucl_object_tostring(elt));
+ }
+ else if (elt && elt->type == UCL_ARRAY) {
+ ucl_object_iter_t it = nullptr;
+ const ucl_object_t *cur;
+
+ while ((cur = ucl_object_iterate(elt, &it, true)) != nullptr) {
+ rspamc_print(out, "DKIM-Signature: {}\n", ucl_object_tostring(cur));
+ }
+ }
+
+ elt = ucl_object_lookup(obj, "profile");
+
+ if (elt) {
+ rspamc_print(out, "Profile data:\n");
+ rspamc_profile_output(out, elt);
+ }
+}
+
+static void
+rspamc_uptime_output(FILE *out, ucl_object_t *obj)
+{
+ int64_t seconds, days, hours, minutes;
+
+ const auto *elt = ucl_object_lookup(obj, "version");
+ if (elt != nullptr) {
+ rspamc_print(out, "Rspamd version: {}\n", ucl_object_tostring(elt));
+ }
+
+ elt = ucl_object_lookup(obj, "uptime");
+ if (elt != nullptr) {
+ rspamc_print(out, "Uptime: ");
+ seconds = ucl_object_toint(elt);
+ if (seconds >= 2 * 3600) {
+ days = seconds / 86400;
+ hours = seconds / 3600 - days * 24;
+ minutes = seconds / 60 - hours * 60 - days * 1440;
+ rspamc_print(out, "{} day{} {} hour{} {} minute{}\n", days,
+ days > 1 ? "s" : "", hours, hours > 1 ? "s" : "",
+ minutes, minutes > 1 ? "s" : "");
+ }
+ /* If uptime is less than 1 minute print only seconds */
+ else if (seconds / 60 == 0) {
+ rspamc_print(out, "{} second{}\n", seconds,
+ seconds > 1 ? "s" : "");
+ }
+ /* Else print the minutes and seconds. */
+ else {
+ hours = seconds / 3600;
+ minutes = seconds / 60 - hours * 60;
+ seconds -= hours * 3600 + minutes * 60;
+ rspamc_print(out, "{} hour {} minute{} {} second{}\n", hours,
+ minutes, minutes > 1 ? "s" : "",
+ seconds, seconds > 1 ? "s" : "");
+ }
+ }
+}
+
+static void
+rspamc_counters_output(FILE *out, ucl_object_t *obj)
+{
+ if (obj->type != UCL_ARRAY) {
+ rspamc_print(out, "Bad output\n");
+ return;
+ }
+
+ std::vector<const ucl_object_t *> counters_vec;
+ auto max_len = sizeof("Symbol") - 1;
+
+ {
+ ucl_object_iter_t iter = nullptr;
+ const ucl_object_t *cur;
+
+ while ((cur = ucl_object_iterate(obj, &iter, true)) != nullptr) {
+ const auto *sym = ucl_object_lookup(cur, "symbol");
+ if (sym != nullptr) {
+ if (sym->len > max_len) {
+ max_len = sym->len;
+ }
+ }
+ counters_vec.push_back(cur);
+ }
+ }
+
+ sort_ucl_container_with_default(counters_vec, "name");
+
+ char dash_buf[82], sym_buf[82];
+ const int dashes = 44;
+
+ max_len = MIN(sizeof(dash_buf) - dashes - 1, max_len);
+ memset(dash_buf, '-', dashes + max_len);
+ dash_buf[dashes + max_len] = '\0';
+
+ rspamc_print(out, "Symbols cache\n");
+
+ rspamc_print(out, " {} \n", emphasis_argument(dash_buf));
+ rspamc_print(out,
+ "| {:<4} | {:<{}} | {:^7} | {:^13} | {:^7} |\n",
+ "Pri",
+ "Symbol",
+ max_len,
+ "Weight",
+ "Frequency",
+ "Hits");
+ rspamc_print(out, " {} \n", emphasis_argument(dash_buf));
+ rspamc_print(out, "| {:<4} | {:<{}} | {:^7} | {:^13} | {:^7} |\n", "",
+ "", max_len,
+ "", "hits/min", "");
+
+ for (const auto [i, cur]: rspamd::enumerate(counters_vec)) {
+ rspamc_print(out, " {} \n", dash_buf);
+ const auto *sym = ucl_object_lookup(cur, "symbol");
+ const auto *weight = ucl_object_lookup(cur, "weight");
+ const auto *freq = ucl_object_lookup(cur, "frequency");
+ const auto *freq_dev = ucl_object_lookup(cur, "frequency_stddev");
+ const auto *nhits = ucl_object_lookup(cur, "hits");
+
+ if (sym && weight && freq && nhits) {
+ const char *sym_name;
+
+ if (sym->len > max_len) {
+ rspamd_snprintf(sym_buf, sizeof(sym_buf), "%*s...",
+ (max_len - 3), ucl_object_tostring(sym));
+ sym_name = sym_buf;
+ }
+ else {
+ sym_name = ucl_object_tostring(sym);
+ }
+
+ rspamc_print(out, "| {:<4} | {:<{}} | {:^7.1f} | {:^6.3f}({:^5.3f}) | {:^7} |\n", i,
+ sym_name,
+ max_len,
+ ucl_object_todouble(weight),
+ ucl_object_todouble(freq) * 60.0,
+ ucl_object_todouble(freq_dev) * 60.0,
+ (std::uintmax_t) ucl_object_toint(nhits));
+ }
+ }
+ rspamc_print(out, " {} \n", dash_buf);
+}
+
+static void
+rspamc_stat_actions(ucl_object_t *obj, std::string &out, std::int64_t scanned)
+{
+ const ucl_object_t *actions = ucl_object_lookup(obj, "actions"), *cur;
+ ucl_object_iter_t iter = nullptr;
+
+ if (scanned > 0) {
+ if (actions && ucl_object_type(actions) == UCL_OBJECT) {
+ while ((cur = ucl_object_iterate(actions, &iter, true)) != nullptr) {
+ auto cnt = ucl_object_toint(cur);
+ fmt::format_to(std::back_inserter(out), "Messages with action {}: {}, {:.2f}%\n",
+ ucl_object_key(cur), emphasis_argument(cnt),
+ ((double) cnt / (double) scanned) * 100.);
+ }
+ }
+
+ auto spam = ucl_object_toint(ucl_object_lookup(obj, "spam_count"));
+ auto ham = ucl_object_toint(ucl_object_lookup(obj, "ham_count"));
+ fmt::format_to(std::back_inserter(out), "Messages treated as spam: {}, {:.2f}%\n",
+ emphasis_argument(spam),
+ ((double) spam / (double) scanned) * 100.);
+ fmt::format_to(std::back_inserter(out), "Messages treated as ham: {}, {:.2f}%\n",
+ emphasis_argument(ham),
+ ((double) ham / (double) scanned) * 100.);
+ }
+}
+
+static void
+rspamc_stat_statfile(const ucl_object_t *obj, std::string &out)
+{
+ auto version = ucl_object_toint(ucl_object_lookup(obj, "revision"));
+ auto size = ucl_object_toint(ucl_object_lookup(obj, "size"));
+ auto blocks = ucl_object_toint(ucl_object_lookup(obj, "total"));
+ auto used_blocks = ucl_object_toint(ucl_object_lookup(obj, "used"));
+ auto label = ucl_object_tostring(ucl_object_lookup(obj, "label"));
+ auto symbol = ucl_object_tostring(ucl_object_lookup(obj, "symbol"));
+ auto type = ucl_object_tostring(ucl_object_lookup(obj, "type"));
+ auto nlanguages = ucl_object_toint(ucl_object_lookup(obj, "languages"));
+ auto nusers = ucl_object_toint(ucl_object_lookup(obj, "users"));
+
+ if (label) {
+ fmt::format_to(std::back_inserter(out), "Statfile: {} <{}> type: {}; ", symbol,
+ label, type);
+ }
+ else {
+ fmt::format_to(std::back_inserter(out), "Statfile: {} type: {}; ", symbol, type);
+ }
+ fmt::format_to(std::back_inserter(out), "length: {}; free blocks: {}; total blocks: {}; "
+ "free: {:.2f}%; learned: {}; users: {}; languages: {}\n",
+ size,
+ blocks - used_blocks, blocks,
+ blocks > 0 ? (blocks - used_blocks) * 100.0 / (double) blocks : 0,
+ version,
+ nusers, nlanguages);
+}
+
+static void
+rspamc_stat_output(FILE *out, ucl_object_t *obj)
+{
+ std::string out_str;
+
+ out_str.reserve(8192);
+
+ auto scanned = ucl_object_toint(ucl_object_lookup(obj, "scanned"));
+ fmt::format_to(std::back_inserter(out_str), "Messages scanned: {}\n",
+ emphasis_argument(scanned));
+
+ rspamc_stat_actions(obj, out_str, scanned);
+
+ fmt::format_to(std::back_inserter(out_str), "Messages learned: {}\n",
+ emphasis_argument(ucl_object_toint(ucl_object_lookup(obj, "learned"))));
+ fmt::format_to(std::back_inserter(out_str), "Connections count: {}\n",
+ emphasis_argument(ucl_object_toint(ucl_object_lookup(obj, "connections"))));
+ fmt::format_to(std::back_inserter(out_str), "Control connections count: {}\n",
+ emphasis_argument(ucl_object_toint(ucl_object_lookup(obj, "control_connections"))));
+
+ const auto *avg_time_obj = ucl_object_lookup(obj, "scan_times");
+
+ if (avg_time_obj && ucl_object_type(avg_time_obj) == UCL_ARRAY) {
+ ucl_object_iter_t iter = nullptr;
+ const ucl_object_t *cur;
+ std::vector<float> nums;
+
+ while ((cur = ucl_object_iterate(avg_time_obj, &iter, true)) != nullptr) {
+ if (ucl_object_type(cur) == UCL_FLOAT || ucl_object_type(cur) == UCL_INT) {
+ nums.push_back(ucl_object_todouble(cur));
+ }
+ }
+
+ auto cnt = nums.size();
+
+ if (cnt > 0) {
+ auto sum = rspamd_sum_floats(nums.data(), &cnt);
+ fmt::format_to(std::back_inserter(out_str),
+ "Average scan time: {} sec\n",
+ emphasis_argument(sum / cnt, 3));
+ }
+ }
+
+ /* Pools */
+ fmt::format_to(std::back_inserter(out_str), "Pools allocated: {}\n",
+ ucl_object_toint(ucl_object_lookup(obj, "pools_allocated")));
+ fmt::format_to(std::back_inserter(out_str), "Pools freed: {}\n",
+ ucl_object_toint(ucl_object_lookup(obj, "pools_freed")));
+ fmt::format_to(std::back_inserter(out_str), "Bytes allocated: {}\n",
+ ucl_object_toint(ucl_object_lookup(obj, "bytes_allocated")));
+ fmt::format_to(std::back_inserter(out_str), "Memory chunks allocated: {}\n",
+ ucl_object_toint(ucl_object_lookup(obj, "chunks_allocated")));
+ fmt::format_to(std::back_inserter(out_str), "Shared chunks allocated: {}\n",
+ ucl_object_toint(ucl_object_lookup(obj, "shared_chunks_allocated")));
+ fmt::format_to(std::back_inserter(out_str), "Chunks freed: {}\n",
+ ucl_object_toint(ucl_object_lookup(obj, "chunks_freed")));
+ fmt::format_to(std::back_inserter(out_str), "Oversized chunks: {}\n",
+ ucl_object_toint(ucl_object_lookup(obj, "chunks_oversized")));
+ /* Fuzzy */
+
+ const auto *st = ucl_object_lookup(obj, "fuzzy_hashes");
+ if (st) {
+ ucl_object_iter_t it = nullptr;
+ const ucl_object_t *cur;
+ std::uint64_t stored = 0;
+
+ while ((cur = ucl_iterate_object(st, &it, true)) != nullptr) {
+ auto num = ucl_object_toint(cur);
+ fmt::format_to(std::back_inserter(out_str), "Fuzzy hashes in storage \"{}\": {}\n",
+ ucl_object_key(cur),
+ num);
+ stored += num;
+ }
+
+ fmt::format_to(std::back_inserter(out_str), "Fuzzy hashes stored: {}\n",
+ stored);
+ }
+
+ st = ucl_object_lookup(obj, "fuzzy_checked");
+ if (st != nullptr && ucl_object_type(st) == UCL_ARRAY) {
+ ucl_object_iter_t iter = nullptr;
+ const ucl_object_t *cur;
+
+ out_str += "Fuzzy hashes checked: ";
+
+ while ((cur = ucl_object_iterate(st, &iter, true)) != nullptr) {
+ fmt::format_to(std::back_inserter(out_str), "{} ", ucl_object_toint(cur));
+ }
+
+ out_str.push_back('\n');
+ }
+
+ st = ucl_object_lookup(obj, "fuzzy_found");
+ if (st != nullptr && ucl_object_type(st) == UCL_ARRAY) {
+ ucl_object_iter_t iter = nullptr;
+ const ucl_object_t *cur;
+
+ out_str += "Fuzzy hashes found: ";
+
+ while ((cur = ucl_object_iterate(st, &iter, true)) != nullptr) {
+ fmt::format_to(std::back_inserter(out_str), "{} ", ucl_object_toint(cur));
+ }
+
+ out_str.push_back('\n');
+ }
+
+ st = ucl_object_lookup(obj, "statfiles");
+ if (st != nullptr && ucl_object_type(st) == UCL_ARRAY) {
+ ucl_object_iter_t iter = nullptr;
+ const ucl_object_t *cur;
+
+ while ((cur = ucl_object_iterate(st, &iter, true)) != nullptr) {
+ rspamc_stat_statfile(cur, out_str);
+ }
+ }
+ fmt::format_to(std::back_inserter(out_str), "Total learns: {}\n",
+ ucl_object_toint(ucl_object_lookup(obj, "total_learns")));
+
+ rspamc_print(out, "{}", out_str.c_str());
+}
+
+static void
+rspamc_output_headers(FILE *out, struct rspamd_http_message *msg)
+{
+ struct rspamd_http_header *h;
+
+ kh_foreach_value(msg->headers, h, {
+ rspamc_print(out, "{}: {}\n", std::string_view{h->name.begin, h->name.len},
+ std::string_view{h->value.begin, h->value.len});
+ });
+
+ rspamc_print(out, "\n");
+}
+
+static void
+rspamc_mime_output(FILE *out, ucl_object_t *result, GString *input,
+ gdouble time, GError *err)
+{
+ const gchar *action = "no action", *line_end = "\r\n", *p;
+ gdouble score = 0.0, required_score = 0.0;
+ gboolean is_spam = FALSE;
+ auto nl_type = RSPAMD_TASK_NEWLINES_CRLF;
+
+ auto headers_pos = rspamd_string_find_eoh(input, nullptr);
+
+ if (headers_pos == -1) {
+ rspamc_print(stderr, "cannot find end of headers position");
+ return;
+ }
+
+ p = input->str + headers_pos;
+
+ if (headers_pos > 1 && *(p - 1) == '\n') {
+ if (headers_pos > 2 && *(p - 2) == '\r') {
+ line_end = "\r\n";
+ nl_type = RSPAMD_TASK_NEWLINES_CRLF;
+ }
+ else {
+ line_end = "\n";
+ nl_type = RSPAMD_TASK_NEWLINES_LF;
+ }
+ }
+ else if (headers_pos > 1 && *(p - 1) == '\r') {
+ line_end = "\r";
+ nl_type = RSPAMD_TASK_NEWLINES_CR;
+ }
+
+ std::string added_headers;
+
+ if (result) {
+ const auto *res = ucl_object_lookup(result, "action");
+
+ if (res) {
+ action = ucl_object_tostring(res);
+ }
+
+ res = ucl_object_lookup(result, "score");
+ if (res) {
+ score = ucl_object_todouble(res);
+ }
+
+ res = ucl_object_lookup(result, "required_score");
+ if (res) {
+ required_score = ucl_object_todouble(res);
+ }
+
+ auto act = rspamd_action_from_str_rspamc(action);
+
+ if (act.has_value() && act.value() < METRIC_ACTION_GREYLIST) {
+ is_spam = TRUE;
+ }
+
+ fmt::format_to(std::back_inserter(added_headers), "X-Spam-Scanner: {}{}",
+ "rspamc " RVERSION, line_end);
+ fmt::format_to(std::back_inserter(added_headers), "X-Spam-Scan-Time: {:.3}{}",
+ time, line_end);
+
+ /*
+ * TODO: add milter_headers support here
+ */
+ if (is_spam) {
+ fmt::format_to(std::back_inserter(added_headers), "X-Spam: yes{}", line_end);
+ }
+
+ fmt::format_to(std::back_inserter(added_headers), "X-Spam-Action: {}{}",
+ action, line_end);
+ fmt::format_to(std::back_inserter(added_headers), "X-Spam-Score: {:.2f} / {:.2f}{}",
+ score, required_score, line_end);
+
+ /* SA style stars header */
+ std::string scorebuf;
+ auto adjusted_score = std::min(score, 32.0);
+ while (adjusted_score > 0) {
+ scorebuf.push_back('*');
+ adjusted_score -= 1.0;
+ }
+
+ fmt::format_to(std::back_inserter(added_headers), "X-Spam-Level: {}{}",
+ scorebuf, line_end);
+
+ /* Short description of all symbols */
+ std::string symbuf;
+ const ucl_object_t *cur;
+ ucl_object_iter_t it = nullptr;
+ const auto *syms = ucl_object_lookup(result, "symbols");
+
+ while (syms && (cur = ucl_object_iterate(syms, &it, true)) != nullptr) {
+ if (ucl_object_type(cur) == UCL_OBJECT) {
+ fmt::format_to(std::back_inserter(symbuf), "{},", ucl_object_key(cur));
+ }
+ }
+ /* Trim the last comma */
+ if (symbuf.back() == ',') {
+ symbuf.pop_back();
+ }
+
+ auto *folded_symbuf = rspamd_header_value_fold("X-Spam-Symbols", strlen("X-Spam-Symbols"),
+ symbuf.data(), symbuf.size(),
+ 0, nl_type, ",");
+ fmt::format_to(std::back_inserter(added_headers), "X-Spam-Symbols: {}{}",
+ folded_symbuf->str, line_end);
+
+ g_string_free(folded_symbuf, TRUE);
+
+ res = ucl_object_lookup(result, "dkim-signature");
+ if (res && res->type == UCL_STRING) {
+ fmt::format_to(std::back_inserter(added_headers), "DKIM-Signature: {}{}",
+ ucl_object_tostring(res), line_end);
+ }
+ else if (res && res->type == UCL_ARRAY) {
+ it = nullptr;
+ while ((cur = ucl_object_iterate(res, &it, true)) != nullptr) {
+ fmt::format_to(std::back_inserter(added_headers), "DKIM-Signature: {}{}",
+ ucl_object_tostring(cur), line_end);
+ }
+ }
+
+ if (json || ucl_reply || compact) {
+ unsigned char *json_header;
+ /* We also append json data as a specific header */
+ if (json) {
+ json_header = ucl_object_emit(result,
+ compact ? UCL_EMIT_JSON_COMPACT : UCL_EMIT_JSON);
+ }
+ else {
+ json_header = ucl_object_emit(result,
+ compact ? UCL_EMIT_JSON_COMPACT : UCL_EMIT_CONFIG);
+ }
+
+ auto *json_header_encoded = rspamd_encode_base64_fold(json_header,
+ strlen((char *) json_header), 60, nullptr, nl_type);
+ free(json_header);
+ fmt::format_to(std::back_inserter(added_headers),
+ "X-Spam-Result: {}{}",
+ json_header_encoded, line_end);
+ g_free(json_header_encoded);
+ }
+
+ ucl_object_unref(result);
+ }
+ else {
+ fmt::format_to(std::back_inserter(added_headers), "X-Spam-Scanner: {}{}",
+ "rspamc " RVERSION, line_end);
+ fmt::format_to(std::back_inserter(added_headers), "X-Spam-Scan-Time: {:.3f}{}",
+ time, line_end);
+ fmt::format_to(std::back_inserter(added_headers), "X-Spam-Error: {}{}",
+ err->message, line_end);
+ }
+
+ /* Write message */
+ /* Original headers */
+ rspamc_print(out, "{}", std::string_view{input->str, (std::size_t) headers_pos});
+ /* Added headers */
+ rspamc_print(out, "{}", added_headers);
+ /* Message body */
+ rspamc_print(out, "{}", input->str + headers_pos);
+}
+
+static void
+rspamc_client_execute_cmd(const struct rspamc_command &cmd, ucl_object_t *result,
+ GString *input, gdouble time, GError *err)
+{
+ gchar **eargv;
+ gint eargc, infd, outfd, errfd;
+ GError *exec_err = nullptr;
+ GPid cld;
+
+ if (!g_shell_parse_argv(execute, &eargc, &eargv, &err)) {
+ rspamc_print(stderr, "Cannot execute {}: {}", execute, err->message);
+ g_error_free(err);
+
+ return;
+ }
+
+ if (!g_spawn_async_with_pipes(nullptr, eargv, nullptr,
+ static_cast<GSpawnFlags>(G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD), nullptr, nullptr, &cld,
+ &infd, &outfd, &errfd, &exec_err)) {
+
+ rspamc_print(stderr, "Cannot execute {}: {}", execute, exec_err->message);
+ g_error_free(exec_err);
+
+ exit(EXIT_FAILURE);
+ }
+ else {
+ children.push_back(cld);
+ auto *out = fdopen(infd, "w");
+
+ if (cmd.cmd == RSPAMC_COMMAND_SYMBOLS && mime_output && input) {
+ rspamc_mime_output(out, result, input, time, err);
+ }
+ else if (result) {
+ if (ucl_reply || cmd.command_output_func == nullptr) {
+ char *ucl_out;
+
+ if (json) {
+ ucl_out = (char *) ucl_object_emit(result,
+ compact ? UCL_EMIT_JSON_COMPACT : UCL_EMIT_JSON);
+ }
+ else {
+ ucl_out = (char *) ucl_object_emit(result,
+ compact ? UCL_EMIT_JSON_COMPACT : UCL_EMIT_CONFIG);
+ }
+ rspamc_print(out, "{}", ucl_out);
+ free(ucl_out);
+ }
+ else {
+ cmd.command_output_func(out, result);
+ }
+
+ ucl_object_unref(result);
+ }
+ else {
+ rspamc_print(out, "{}\n", err->message);
+ }
+
+ fflush(out);
+ fclose(out);
+ }
+
+ g_strfreev(eargv);
+}
+
+static void
+rspamc_client_cb(struct rspamd_client_connection *conn,
+ struct rspamd_http_message *msg,
+ const char *name, ucl_object_t *result, GString *input,
+ gpointer ud, gdouble start_time, gdouble send_time,
+ const char *body, gsize bodylen,
+ GError *err)
+{
+ struct rspamc_callback_data *cbdata = (struct rspamc_callback_data *) ud;
+ FILE *out = stdout;
+ gdouble finish = rspamd_get_ticks(FALSE), diff;
+
+ auto &cmd = cbdata->cmd;
+
+ if (send_time > 0) {
+ diff = finish - send_time;
+ }
+ else {
+ diff = finish - start_time;
+ }
+
+ if (execute) {
+ /* Pass all to the external command */
+ rspamc_client_execute_cmd(cmd, result, input, diff, err);
+ }
+ else {
+
+ if (cmd.cmd == RSPAMC_COMMAND_SYMBOLS && mime_output && input) {
+ if (body) {
+ GString tmp;
+
+ tmp.str = (char *) body;
+ tmp.len = bodylen;
+ rspamc_mime_output(out, result, &tmp, diff, err);
+ }
+ else {
+ rspamc_mime_output(out, result, input, diff, err);
+ }
+ }
+ else {
+ if (cmd.need_input && !json) {
+ if (!compact && !humanreport) {
+ rspamc_print(out, "Results for file: {} ({:.3} seconds)\n",
+ emphasis_argument(cbdata->filename), diff);
+ }
+ }
+ else {
+ if (!compact && !json && !humanreport) {
+ rspamc_print(out, "Results for command: {} ({:.3} seconds)\n",
+ emphasis_argument(cmd.name), diff);
+ }
+ }
+
+ if (result != nullptr) {
+ if (headers && msg != nullptr) {
+ rspamc_output_headers(out, msg);
+ }
+ if (ucl_reply || cmd.command_output_func == nullptr) {
+ if (cmd.need_input) {
+ ucl_object_insert_key(result,
+ ucl_object_fromstring(cbdata->filename.c_str()),
+ "filename", 0,
+ false);
+ }
+
+ ucl_object_insert_key(result,
+ ucl_object_fromdouble(diff),
+ "scan_time", 0,
+ false);
+
+ char *ucl_out;
+
+ if (json) {
+ ucl_out = (char *) ucl_object_emit(result,
+ compact ? UCL_EMIT_JSON_COMPACT : UCL_EMIT_JSON);
+ }
+ else {
+ ucl_out = (char *) ucl_object_emit(result,
+ compact ? UCL_EMIT_JSON_COMPACT : UCL_EMIT_CONFIG);
+ }
+
+ rspamc_print(out, "{}", ucl_out);
+ free(ucl_out);
+ }
+ else {
+ cmd.command_output_func(out, result);
+ }
+
+ if (body) {
+ rspamc_print(out, "\nNew body:\n{}\n",
+ std::string_view{body, bodylen});
+ }
+
+ ucl_object_unref(result);
+ }
+ else if (err != nullptr) {
+ rspamc_print(out, "{}\n", err->message);
+
+ if (json && msg != nullptr) {
+ gsize rawlen;
+
+ auto *raw_body = rspamd_http_message_get_body(msg, &rawlen);
+
+ if (raw_body) {
+ /* We can also output the resulting json */
+ rspamc_print(out, "{}\n", std::string_view{raw_body, (std::size_t)(rawlen - bodylen)});
+ }
+ }
+ }
+ rspamc_print(out, "\n");
+ }
+
+ fflush(out);
+ }
+
+ rspamd_client_destroy(conn);
+ delete cbdata;
+
+ if (err) {
+ retcode = EXIT_FAILURE;
+ }
+}
+
+static void
+rspamc_process_input(struct ev_loop *ev_base, const struct rspamc_command &cmd,
+ FILE *in, const std::string &name, GQueue *attrs)
+{
+ struct rspamd_client_connection *conn;
+ const char *p;
+ guint16 port;
+ GError *err = nullptr;
+ std::string hostbuf;
+
+ if (connect_str[0] == '[') {
+ p = strrchr(connect_str, ']');
+
+ if (p != nullptr) {
+ hostbuf.assign(connect_str + 1, (std::size_t)(p - connect_str - 1));
+ p++;
+ }
+ else {
+ p = connect_str;
+ }
+ }
+ else {
+ p = connect_str;
+ }
+
+ p = strrchr(p, ':');
+
+ if (hostbuf.empty()) {
+ if (p != nullptr) {
+ hostbuf.assign(connect_str, (std::size_t)(p - connect_str));
+ }
+ else {
+ hostbuf.assign(connect_str);
+ }
+ }
+
+ if (p != nullptr) {
+ port = strtoul(p + 1, nullptr, 10);
+ }
+ else {
+ /*
+ * If we connect to localhost, 127.0.0.1 or ::1, then try controller
+ * port first
+ */
+
+ if (hostbuf == "localhost" ||
+ hostbuf == "127.0.0.1" ||
+ hostbuf == "::1" ||
+ hostbuf == "[::1]") {
+ port = DEFAULT_CONTROL_PORT;
+ }
+ else {
+ port = cmd.is_controller ? DEFAULT_CONTROL_PORT : DEFAULT_PORT;
+ }
+ }
+
+ conn = rspamd_client_init(http_ctx, ev_base, hostbuf.c_str(), port, timeout, pubkey);
+
+ if (conn != nullptr) {
+ auto *cbdata = new rspamc_callback_data;
+ cbdata->cmd = cmd;
+ cbdata->filename = name;
+
+ if (cmd.need_input) {
+ rspamd_client_command(conn, cmd.path, attrs, in, rspamc_client_cb,
+ cbdata, compressed, dictionary, cbdata->filename.c_str(), &err);
+ }
+ else {
+ rspamd_client_command(conn,
+ cmd.path,
+ attrs,
+ nullptr,
+ rspamc_client_cb,
+ cbdata,
+ compressed,
+ dictionary,
+ cbdata->filename.c_str(),
+ &err);
+ }
+ }
+ else {
+ rspamc_print(stderr, "cannot connect to {}: {}\n", connect_str,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+}
+
+static gsize
+rspamd_dirent_size(DIR *dirp)
+{
+ goffset name_max;
+ gsize name_end;
+
+#if defined(HAVE_FPATHCONF) && defined(HAVE_DIRFD) && defined(_PC_NAME_MAX)
+ name_max = fpathconf(dirfd(dirp), _PC_NAME_MAX);
+
+
+#if defined(NAME_MAX)
+ if (name_max == -1) {
+ name_max = (NAME_MAX > 255) ? NAME_MAX : 255;
+ }
+#else
+ if (name_max == -1) {
+ return (size_t) (-1);
+ }
+#endif
+#else
+#if defined(NAME_MAX)
+ name_max = (NAME_MAX > 255) ? NAME_MAX : 255;
+#else
+#error "buffer size for readdir_r cannot be determined"
+#endif
+#endif
+
+ name_end = G_STRUCT_OFFSET(struct dirent, d_name) + name_max + 1;
+
+ return (name_end > sizeof(struct dirent) ? name_end : sizeof(struct dirent));
+}
+
+static void
+rspamc_process_dir(struct ev_loop *ev_base, const struct rspamc_command &cmd,
+ const std::string &name, GQueue *attrs)
+{
+ static auto cur_req = 0;
+ auto *d = opendir(name.c_str());
+
+ if (d != nullptr) {
+ struct dirent *pentry;
+ std::string fpath;
+
+ fpath.reserve(PATH_MAX);
+
+ while ((pentry = readdir(d)) != nullptr) {
+
+ if (pentry->d_name[0] == '.') {
+ continue;
+ }
+
+ fpath.clear();
+ fmt::format_to(std::back_inserter(fpath), "{}{}{}",
+ name, G_DIR_SEPARATOR,
+ pentry->d_name);
+
+ /* Check exclude */
+ auto **ex = exclude_compiled;
+ auto skip = false;
+ while (ex != nullptr && *ex != nullptr) {
+#if GLIB_MAJOR_VERSION >= 2 && GLIB_MINOR_VERSION >= 70
+ if (g_pattern_spec_match(*ex, fpath.size(), fpath.c_str(), nullptr)) {
+#else
+ if (g_pattern_match(*ex, fpath.size(), fpath.c_str(), nullptr)) {
+#endif
+ skip = true;
+ break;
+ }
+
+ ex++;
+ }
+
+ if (skip) {
+ continue;
+ }
+
+ auto is_reg = false;
+ auto is_dir = false;
+ struct stat st;
+
+#if (defined(_DIRENT_HAVE_D_TYPE) || defined(__APPLE__)) && defined(DT_UNKNOWN)
+ if (pentry->d_type == DT_UNKNOWN) {
+ /* Fallback to lstat */
+ if (lstat(fpath.c_str(), &st) == -1) {
+ rspamc_print(stderr, "cannot stat file {}: {}\n",
+ fpath, strerror(errno));
+ continue;
+ }
+
+ is_dir = S_ISDIR(st.st_mode);
+ is_reg = S_ISREG(st.st_mode);
+ }
+ else {
+ if (pentry->d_type == DT_REG) {
+ is_reg = true;
+ }
+ else if (pentry->d_type == DT_DIR) {
+ is_dir = true;
+ }
+ }
+#else
+ if (lstat(fpath.c_str(), &st) == -1) {
+ rspamc_print(stderr, "cannot stat file {}: {}\n",
+ fpath, strerror(errno));
+ continue;
+ }
+
+ is_dir = S_ISDIR(st.st_mode);
+ is_reg = S_ISREG(st.st_mode);
+#endif
+ if (is_dir) {
+ rspamc_process_dir(ev_base, cmd, fpath, attrs);
+ continue;
+ }
+ else if (is_reg) {
+ auto *in = fopen(fpath.c_str(), "r");
+ if (in == nullptr) {
+ rspamc_print(stderr, "cannot open file {}: {}\n",
+ fpath, strerror(errno));
+ continue;
+ }
+
+ rspamc_process_input(ev_base, cmd, in, fpath, attrs);
+ cur_req++;
+ fclose(in);
+
+ if (cur_req >= max_requests) {
+ cur_req = 0;
+ /* Wait for completion */
+ ev_loop(ev_base, 0);
+ }
+ }
+ }
+ }
+ else {
+ rspamc_print(stderr, "cannot open directory {}: {}\n", name, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ closedir(d);
+ ev_loop(ev_base, 0);
+}
+
+
+static void
+rspamc_kwattr_free(gpointer p)
+{
+ struct rspamd_http_client_header *h = (struct rspamd_http_client_header *) p;
+
+ g_free(h->value);
+ g_free(h->name);
+ g_free(h);
+}
+
+int main(int argc, char **argv, char **env)
+{
+ auto *kwattrs = g_queue_new();
+
+ read_cmd_line(&argc, &argv);
+ tty = isatty(STDOUT_FILENO);
+
+ if (print_commands) {
+ print_commands_list();
+ exit(EXIT_SUCCESS);
+ }
+
+ /* Deal with exclude patterns */
+ auto **exclude_pattern = exclude_patterns;
+ auto npatterns = 0;
+
+ while (exclude_pattern && *exclude_pattern) {
+ exclude_pattern++;
+ npatterns++;
+ }
+
+ if (npatterns > 0) {
+ exclude_compiled = g_new0(GPatternSpec *, (npatterns + 1));
+
+ for (auto i = 0; i < npatterns; i++) {
+ exclude_compiled[i] = g_pattern_spec_new(exclude_patterns[i]);
+
+ if (exclude_compiled[i] == nullptr) {
+ rspamc_print(stderr, "Invalid glob pattern: {}\n",
+ exclude_patterns[i]);
+ exit(EXIT_FAILURE);
+ }
+ }
+ }
+
+ auto *libs = rspamd_init_libs();
+ auto *event_loop = ev_loop_new(EVBACKEND_ALL);
+
+ struct rspamd_http_context_cfg http_config;
+ memset(&http_config, 0, sizeof(http_config));
+ http_config.kp_cache_size_client = 32;
+ http_config.kp_cache_size_server = 0;
+ http_config.user_agent = user_agent;
+ http_ctx = rspamd_http_context_create_config(&http_config,
+ event_loop, nullptr);
+
+ /* Ignore sigpipe */
+ struct sigaction sigpipe_act;
+ sigemptyset(&sigpipe_act.sa_mask);
+ sigaddset(&sigpipe_act.sa_mask, SIGPIPE);
+ sigpipe_act.sa_handler = SIG_IGN;
+ sigpipe_act.sa_flags = 0;
+ sigaction(SIGPIPE, &sigpipe_act, nullptr);
+
+ /* Now read other args from argc and argv */
+ FILE *in = nullptr;
+ std::optional<rspamc_command> maybe_cmd;
+ auto start_argc = 0;
+
+ if (argc == 1) {
+ start_argc = argc;
+ in = stdin;
+ maybe_cmd = check_rspamc_command("symbols");
+ }
+ else if (argc == 2) {
+ /* One argument is whether command or filename */
+ maybe_cmd = check_rspamc_command(argv[1]);
+
+ if (maybe_cmd.has_value()) {
+ start_argc = argc;
+ in = stdin;
+ }
+ else {
+ maybe_cmd = check_rspamc_command("symbols"); /* Symbols command */
+ start_argc = 1;
+ }
+ }
+ else {
+ maybe_cmd = check_rspamc_command(argv[1]);
+ if (maybe_cmd.has_value()) {
+ auto &cmd = maybe_cmd.value();
+ /* In case of command read arguments starting from 2 */
+ if (cmd.cmd == RSPAMC_COMMAND_ADD_SYMBOL || cmd.cmd == RSPAMC_COMMAND_ADD_ACTION) {
+ if (argc < 4 || argc > 5) {
+ rspamc_print(stderr, "invalid arguments\n");
+ exit(EXIT_FAILURE);
+ }
+ if (argc == 5) {
+ add_client_header(kwattrs, "metric", argv[2]);
+ add_client_header(kwattrs, "name", argv[3]);
+ add_client_header(kwattrs, "value", argv[4]);
+ }
+ else {
+ add_client_header(kwattrs, "name", argv[2]);
+ add_client_header(kwattrs, "value", argv[3]);
+ }
+ start_argc = argc;
+ }
+ else {
+ start_argc = 2;
+ }
+ }
+ else {
+ maybe_cmd = check_rspamc_command("symbols");
+ start_argc = 1;
+ }
+ }
+
+ if (!maybe_cmd.has_value()) {
+ rspamc_print(stderr, "invalid command\n");
+ exit(EXIT_FAILURE);
+ }
+
+ add_options(kwattrs);
+ auto cmd = maybe_cmd.value();
+
+ if (start_argc == argc) {
+ /* Do command without input or with stdin */
+ if (empty_input) {
+ rspamc_process_input(event_loop, cmd, nullptr, "empty", kwattrs);
+ }
+ else {
+ rspamc_process_input(event_loop, cmd, in, "stdin", kwattrs);
+ }
+ }
+ else {
+ auto cur_req = 0;
+
+ for (auto i = start_argc; i < argc; i++) {
+ if (cmd.cmd == RSPAMC_COMMAND_FUZZY_DELHASH) {
+ add_client_header(kwattrs, "Hash", argv[i]);
+ }
+ else {
+ struct stat st;
+
+ if (stat(argv[i], &st) == -1) {
+ rspamc_print(stderr, "cannot stat file {}\n", argv[i]);
+ exit(EXIT_FAILURE);
+ }
+ if (S_ISDIR(st.st_mode)) {
+ /* Directories are processed with a separate limit */
+ rspamc_process_dir(event_loop, cmd, argv[i], kwattrs);
+ cur_req = 0;
+ }
+ else {
+ in = fopen(argv[i], "r");
+ if (in == nullptr) {
+ rspamc_print(stderr, "cannot open file {}\n", argv[i]);
+ exit(EXIT_FAILURE);
+ }
+ rspamc_process_input(event_loop, cmd, in, argv[i], kwattrs);
+ cur_req++;
+ fclose(in);
+ }
+ if (cur_req >= max_requests) {
+ cur_req = 0;
+ /* Wait for completion */
+ ev_loop(event_loop, 0);
+ }
+ }
+ }
+
+ if (cmd.cmd == RSPAMC_COMMAND_FUZZY_DELHASH) {
+ rspamc_process_input(event_loop, cmd, nullptr, "hashes", kwattrs);
+ }
+ }
+
+ ev_loop(event_loop, 0);
+
+ g_queue_free_full(kwattrs, rspamc_kwattr_free);
+
+ /* Wait for children processes */
+ auto ret = 0;
+
+ for (auto cld: children) {
+ auto res = 0;
+ if (waitpid(cld, &res, 0) == -1) {
+ rspamc_print(stderr, "Cannot wait for {}: {}", cld,
+ strerror(errno));
+
+ ret = errno;
+ }
+
+ if (ret == 0) {
+ /* Check return code */
+ if (WIFSIGNALED(res)) {
+ ret = WTERMSIG(res);
+ }
+ else if (WIFEXITED(res)) {
+ ret = WEXITSTATUS(res);
+ }
+ }
+ }
+
+ for (auto i = 0; i < npatterns; i++) {
+ g_pattern_spec_free(exclude_compiled[i]);
+ }
+ g_free(exclude_compiled);
+
+ rspamd_deinit_libs(libs);
+
+ /* Mix retcode (return from Rspamd side) and ret (return from subprocess) */
+ return ret | retcode;
+}
diff --git a/src/client/rspamdclient.c b/src/client/rspamdclient.c
new file mode 100644
index 0000000..85f4749
--- /dev/null
+++ b/src/client/rspamdclient.c
@@ -0,0 +1,498 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "rspamdclient.h"
+#include "libutil/util.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
+#include "libserver/protocol_internal.h"
+#include "unix-std.h"
+
+#ifdef SYS_ZSTD
+#include "zstd.h"
+#else
+#include "contrib/zstd/zstd.h"
+#endif
+
+#ifdef HAVE_FETCH_H
+#include <fetch.h>
+#elif defined(CURL_FOUND)
+#include <curl/curl.h>
+#endif
+
+struct rspamd_client_request;
+
+/*
+ * Since rspamd uses untagged HTTP we can pass a single message per socket
+ */
+struct rspamd_client_connection {
+ gint fd;
+ GString *server_name;
+ struct rspamd_cryptobox_pubkey *key;
+ struct rspamd_cryptobox_keypair *keypair;
+ struct ev_loop *event_loop;
+ ev_tstamp timeout;
+ struct rspamd_http_connection *http_conn;
+ gboolean req_sent;
+ gdouble start_time;
+ gdouble send_time;
+ struct rspamd_client_request *req;
+ struct rspamd_keypair_cache *keys_cache;
+};
+
+struct rspamd_client_request {
+ struct rspamd_client_connection *conn;
+ struct rspamd_http_message *msg;
+ GString *input;
+ rspamd_client_callback cb;
+ gpointer ud;
+};
+
+#define RCLIENT_ERROR rspamd_client_error_quark()
+GQuark
+rspamd_client_error_quark(void)
+{
+ return g_quark_from_static_string("rspamd-client-error");
+}
+
+static void
+rspamd_client_request_free(struct rspamd_client_request *req)
+{
+ if (req != NULL) {
+ if (req->conn) {
+ req->conn->req = NULL;
+ }
+ if (req->input) {
+ g_string_free(req->input, TRUE);
+ }
+
+ g_free(req);
+ }
+}
+
+static gint
+rspamd_client_body_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *chunk, gsize len)
+{
+ /* Do nothing here */
+ return 0;
+}
+
+static void
+rspamd_client_error_handler(struct rspamd_http_connection *conn, GError *err)
+{
+ struct rspamd_client_request *req =
+ (struct rspamd_client_request *) conn->ud;
+ struct rspamd_client_connection *c;
+
+ c = req->conn;
+ req->cb(c, NULL, c->server_name->str, NULL,
+ req->input, req->ud,
+ c->start_time, c->send_time, NULL, 0, err);
+}
+
+static gint
+rspamd_client_finish_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_client_request *req =
+ (struct rspamd_client_request *) conn->ud;
+ struct rspamd_client_connection *c;
+ struct ucl_parser *parser;
+ GError *err;
+ const rspamd_ftok_t *tok;
+ const gchar *start, *body = NULL;
+ guchar *out = NULL;
+ gsize len, bodylen = 0;
+
+ c = req->conn;
+
+ if (!c->req_sent) {
+ c->req_sent = TRUE;
+ c->send_time = rspamd_get_ticks(FALSE);
+ rspamd_http_connection_reset(c->http_conn);
+ rspamd_http_connection_read_message(c->http_conn,
+ c->req,
+ c->timeout);
+
+ return 0;
+ }
+ else {
+ if (rspamd_http_message_get_body(msg, NULL) == NULL || msg->code / 100 != 2) {
+ err = g_error_new(RCLIENT_ERROR, msg->code, "HTTP error: %d, %.*s",
+ msg->code,
+ (gint) msg->status->len, msg->status->str);
+ req->cb(c, msg, c->server_name->str, NULL, req->input, req->ud,
+ c->start_time, c->send_time, body, bodylen, err);
+ g_error_free(err);
+
+ return 0;
+ }
+
+ tok = rspamd_http_message_find_header(msg, COMPRESSION_HEADER);
+
+ if (tok) {
+ /* Need to uncompress */
+ rspamd_ftok_t t;
+
+ t.begin = "zstd";
+ t.len = 4;
+
+ if (rspamd_ftok_casecmp(tok, &t) == 0) {
+ ZSTD_DStream *zstream;
+ ZSTD_inBuffer zin;
+ ZSTD_outBuffer zout;
+ gsize outlen, r;
+
+ zstream = ZSTD_createDStream();
+ ZSTD_initDStream(zstream);
+
+ zin.pos = 0;
+ zin.src = msg->body_buf.begin;
+ zin.size = msg->body_buf.len;
+
+ if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
+ outlen = ZSTD_DStreamOutSize();
+ }
+
+ out = g_malloc(outlen);
+ zout.dst = out;
+ zout.pos = 0;
+ zout.size = outlen;
+
+ while (zin.pos < zin.size) {
+ r = ZSTD_decompressStream(zstream, &zout, &zin);
+
+ if (ZSTD_isError(r)) {
+ err = g_error_new(RCLIENT_ERROR, 500,
+ "Decompression error: %s",
+ ZSTD_getErrorName(r));
+ req->cb(c, msg, c->server_name->str, NULL,
+ req->input, req->ud, c->start_time,
+ c->send_time, body, bodylen, err);
+ g_error_free(err);
+ ZSTD_freeDStream(zstream);
+
+ goto end;
+ }
+
+ if (zout.pos == zout.size) {
+ /* We need to extend output buffer */
+ zout.size = zout.size * 2;
+ zout.dst = g_realloc(zout.dst, zout.size);
+ }
+ }
+
+ ZSTD_freeDStream(zstream);
+
+ start = zout.dst;
+ len = zout.pos;
+ }
+ else {
+ err = g_error_new(RCLIENT_ERROR, 500,
+ "Invalid compression method");
+ req->cb(c, msg, c->server_name->str, NULL,
+ req->input, req->ud, c->start_time, c->send_time,
+ body, bodylen, err);
+ g_error_free(err);
+
+ return 0;
+ }
+ }
+ else {
+ start = msg->body_buf.begin;
+ len = msg->body_buf.len;
+ }
+
+ /* Deal with body */
+ tok = rspamd_http_message_find_header(msg, MESSAGE_OFFSET_HEADER);
+
+ if (tok) {
+ gulong value = 0;
+
+ if (rspamd_strtoul(tok->begin, tok->len, &value) &&
+ value < len) {
+ body = start + value;
+ bodylen = len - value;
+ len = value;
+ }
+ }
+
+ parser = ucl_parser_new(0);
+ if (!ucl_parser_add_chunk(parser, start, len)) {
+ err = g_error_new(RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s",
+ ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+ req->cb(c, msg, c->server_name->str, NULL,
+ req->input, req->ud,
+ c->start_time, c->send_time, body, bodylen, err);
+ g_error_free(err);
+
+ goto end;
+ }
+
+ req->cb(c, msg, c->server_name->str,
+ ucl_parser_get_object(parser),
+ req->input, req->ud,
+ c->start_time, c->send_time, body, bodylen, NULL);
+ ucl_parser_free(parser);
+ }
+
+end:
+ if (out) {
+ g_free(out);
+ }
+
+ return 0;
+}
+
+struct rspamd_client_connection *
+rspamd_client_init(struct rspamd_http_context *http_ctx,
+ struct ev_loop *ev_base, const gchar *name,
+ guint16 port, gdouble timeout, const gchar *key)
+{
+ struct rspamd_client_connection *conn;
+ gint fd;
+
+ fd = rspamd_socket(name, port, SOCK_STREAM, TRUE, FALSE, TRUE);
+
+ if (fd == -1) {
+ return NULL;
+ }
+
+ conn = g_malloc0(sizeof(struct rspamd_client_connection));
+ conn->event_loop = ev_base;
+ conn->fd = fd;
+ conn->req_sent = FALSE;
+ conn->http_conn = rspamd_http_connection_new_client_socket(http_ctx,
+ rspamd_client_body_handler,
+ rspamd_client_error_handler,
+ rspamd_client_finish_handler,
+ 0,
+ fd);
+
+ if (!conn->http_conn) {
+ rspamd_client_destroy(conn);
+ return NULL;
+ }
+
+ /* Pass socket ownership */
+ rspamd_http_connection_own_socket(conn->http_conn);
+ conn->server_name = g_string_new(name);
+
+ if (port != 0) {
+ rspamd_printf_gstring(conn->server_name, ":%d", (int) port);
+ }
+
+ conn->timeout = timeout;
+
+ if (key) {
+ conn->key = rspamd_pubkey_from_base32(key, 0, RSPAMD_KEYPAIR_KEX,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+
+ if (conn->key) {
+ conn->keypair = rspamd_keypair_new(RSPAMD_KEYPAIR_KEX,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+ rspamd_http_connection_set_key(conn->http_conn, conn->keypair);
+ }
+ else {
+ rspamd_client_destroy(conn);
+ return NULL;
+ }
+ }
+
+ return conn;
+}
+
+gboolean
+rspamd_client_command(struct rspamd_client_connection *conn,
+ const gchar *command, GQueue *attrs,
+ FILE *in, rspamd_client_callback cb,
+ gpointer ud, gboolean compressed,
+ const gchar *comp_dictionary,
+ const gchar *filename,
+ GError **err)
+{
+ struct rspamd_client_request *req;
+ struct rspamd_http_client_header *nh;
+ gchar *p;
+ gsize remain, old_len;
+ GList *cur;
+ GString *input = NULL;
+ rspamd_fstring_t *body;
+ guint dict_id = 0;
+ gsize dict_len = 0;
+ void *dict = NULL;
+ ZSTD_CCtx *zctx;
+ gboolean ret;
+
+ req = g_malloc0(sizeof(struct rspamd_client_request));
+ req->conn = conn;
+ req->cb = cb;
+ req->ud = ud;
+
+ req->msg = rspamd_http_new_message(HTTP_REQUEST);
+ if (conn->key) {
+ req->msg->peer_key = rspamd_pubkey_ref(conn->key);
+ }
+
+ if (in != NULL) {
+ /* Read input stream */
+ input = g_string_sized_new(BUFSIZ);
+
+ while (!feof(in)) {
+ p = input->str + input->len;
+ remain = input->allocated_len - input->len - 1;
+ if (remain == 0) {
+ old_len = input->len;
+ g_string_set_size(input, old_len * 2);
+ input->len = old_len;
+ continue;
+ }
+ remain = fread(p, 1, remain, in);
+ if (remain > 0) {
+ input->len += remain;
+ input->str[input->len] = '\0';
+ }
+ }
+ if (ferror(in) != 0) {
+ g_set_error(err, RCLIENT_ERROR, ferror(in), "input IO error: %s", strerror(ferror(in)));
+ g_free(req);
+ g_string_free(input, TRUE);
+ return FALSE;
+ }
+
+ if (!compressed) {
+ /* Detect zstd input */
+ if (input->len > 4 && memcmp(input->str, "\x28\xb5\x2f\xfd", 4) == 0) {
+ compressed = TRUE;
+ }
+ body = rspamd_fstring_new_init(input->str, input->len);
+ }
+ else {
+ if (comp_dictionary) {
+ dict = rspamd_file_xmap(comp_dictionary, PROT_READ, &dict_len,
+ TRUE);
+
+ if (dict == NULL) {
+ g_set_error(err, RCLIENT_ERROR, errno,
+ "cannot open dictionary %s: %s",
+ comp_dictionary,
+ strerror(errno));
+ g_free(req);
+ g_string_free(input, TRUE);
+
+ return FALSE;
+ }
+
+ dict_id = -1;
+ }
+
+ body = rspamd_fstring_sized_new(ZSTD_compressBound(input->len));
+ zctx = ZSTD_createCCtx();
+ body->len = ZSTD_compress_usingDict(zctx, body->str, body->allocated,
+ input->str, input->len,
+ dict, dict_len,
+ 1);
+
+ munmap(dict, dict_len);
+
+ if (ZSTD_isError(body->len)) {
+ g_set_error(err, RCLIENT_ERROR, ferror(in), "compression error");
+ g_free(req);
+ g_string_free(input, TRUE);
+ rspamd_fstring_free(body);
+ ZSTD_freeCCtx(zctx);
+
+ return FALSE;
+ }
+
+ ZSTD_freeCCtx(zctx);
+ }
+
+ rspamd_http_message_set_body_from_fstring_steal(req->msg, body);
+ req->input = input;
+ }
+ else {
+ req->input = NULL;
+ }
+
+ /* Convert headers */
+ cur = attrs->head;
+ while (cur != NULL) {
+ nh = cur->data;
+
+ rspamd_http_message_add_header(req->msg, nh->name, nh->value);
+ cur = g_list_next(cur);
+ }
+
+ if (compressed) {
+ rspamd_http_message_add_header(req->msg, COMPRESSION_HEADER, "zstd");
+
+ if (dict_id != 0) {
+ gchar dict_str[32];
+
+ rspamd_snprintf(dict_str, sizeof(dict_str), "%ud", dict_id);
+ rspamd_http_message_add_header(req->msg, "Dictionary", dict_str);
+ }
+ }
+
+ if (filename) {
+ rspamd_http_message_add_header(req->msg, "Filename", filename);
+ }
+
+ req->msg->url = rspamd_fstring_append(req->msg->url, "/", 1);
+ req->msg->url = rspamd_fstring_append(req->msg->url, command, strlen(command));
+
+ conn->req = req;
+ conn->start_time = rspamd_get_ticks(FALSE);
+
+ if (compressed) {
+ ret = rspamd_http_connection_write_message(conn->http_conn, req->msg,
+ NULL, "application/x-compressed", req,
+ conn->timeout);
+ }
+ else {
+ ret = rspamd_http_connection_write_message(conn->http_conn, req->msg,
+ NULL, "text/plain", req, conn->timeout);
+ }
+
+ return ret;
+}
+
+void rspamd_client_destroy(struct rspamd_client_connection *conn)
+{
+ if (conn != NULL) {
+ if (conn->http_conn) {
+ rspamd_http_connection_unref(conn->http_conn);
+ }
+
+ if (conn->req != NULL) {
+ rspamd_client_request_free(conn->req);
+ }
+
+ if (conn->key) {
+ rspamd_pubkey_unref(conn->key);
+ }
+
+ if (conn->keypair) {
+ rspamd_keypair_unref(conn->keypair);
+ }
+
+ g_string_free(conn->server_name, TRUE);
+ g_free(conn);
+ }
+}
diff --git a/src/client/rspamdclient.h b/src/client/rspamdclient.h
new file mode 100644
index 0000000..27597df
--- /dev/null
+++ b/src/client/rspamdclient.h
@@ -0,0 +1,106 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMDCLIENT_H_
+#define RSPAMDCLIENT_H_
+
+#include "config.h"
+#include "ucl.h"
+#include "contrib/libev/ev.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_client_connection;
+struct rspamd_http_message;
+
+struct rspamd_http_client_header {
+ gchar *name;
+ gchar *value;
+};
+
+/**
+ * Callback is called on client connection completed
+ * @param name name of server
+ * @param port port for server
+ * @param result result object
+ * @param ud opaque user data
+ * @param err error pointer
+ */
+typedef void (*rspamd_client_callback)(
+ struct rspamd_client_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *name,
+ ucl_object_t *result,
+ GString *input,
+ gpointer ud,
+ gdouble start_time,
+ gdouble send_time,
+ const gchar *body,
+ gsize body_len,
+ GError *err);
+
+struct rspamd_http_context;
+
+/**
+ * Start rspamd worker or controller command
+ * @param ev_base event base
+ * @param name server name (hostname or unix socket)
+ * @param port port number (in host order)
+ * @param timeout timeout in seconds
+ * @return
+ */
+struct rspamd_client_connection *rspamd_client_init(
+ struct rspamd_http_context *http_ctx,
+ struct ev_loop *ev_base,
+ const gchar *name,
+ guint16 port,
+ gdouble timeout,
+ const gchar *key);
+
+/**
+ *
+ * @param conn connection object
+ * @param command command name
+ * @param attrs additional attributes
+ * @param in input file or NULL if no input required
+ * @param cb callback to be called on command completion
+ * @param ud opaque user data
+ * @return
+ */
+gboolean rspamd_client_command(
+ struct rspamd_client_connection *conn,
+ const gchar *command,
+ GQueue *attrs,
+ FILE *in,
+ rspamd_client_callback cb,
+ gpointer ud,
+ gboolean compressed,
+ const gchar *comp_dictionary,
+ const gchar *filename,
+ GError **err);
+
+/**
+ * Destroy a connection to rspamd
+ * @param conn
+ */
+void rspamd_client_destroy(struct rspamd_client_connection *conn);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RSPAMDCLIENT_H_ */
diff --git a/src/controller.c b/src/controller.c
new file mode 100644
index 0000000..f05105e
--- /dev/null
+++ b/src/controller.c
@@ -0,0 +1,4288 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "libserver/dynamic_cfg.h"
+#include "libserver/cfg_file_private.h"
+#include "libutil/rrd.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
+#include "libserver/maps/map_private.h"
+#include "libserver/http/http_private.h"
+#include "libserver/http/http_router.h"
+#include "libstat/stat_api.h"
+#include "rspamd.h"
+#include "libserver/worker_util.h"
+#include "worker_private.h"
+#include "lua/lua_common.h"
+#include "cryptobox.h"
+#include "ottery.h"
+#include "fuzzy_wire.h"
+#include "unix-std.h"
+#include "utlist.h"
+#include "libmime/lang_detection.h"
+#include "mempool_vars_internal.h"
+#include <math.h>
+
+/* 60 seconds for worker's IO */
+#define DEFAULT_WORKER_IO_TIMEOUT 60000
+
+/* HTTP paths */
+#define PATH_AUTH "/auth"
+#define PATH_SYMBOLS "/symbols"
+#define PATH_ACTIONS "/actions"
+#define PATH_MAPS "/maps"
+#define PATH_GET_MAP "/getmap"
+#define PATH_GRAPH "/graph"
+#define PATH_PIE_CHART "/pie"
+#define PATH_HEALTHY "/healthy"
+#define PATH_HISTORY "/history"
+#define PATH_HISTORY_RESET "/historyreset"
+#define PATH_LEARN_SPAM "/learnspam"
+#define PATH_LEARN_HAM "/learnham"
+#define PATH_METRICS "/metrics"
+#define PATH_READY "/ready"
+#define PATH_SAVE_ACTIONS "/saveactions"
+#define PATH_SAVE_SYMBOLS "/savesymbols"
+#define PATH_SAVE_MAP "/savemap"
+#define PATH_SCAN "/scan"
+#define PATH_CHECK "/check"
+#define PATH_CHECKV2 "/checkv2"
+#define PATH_STAT "/stat"
+#define PATH_STAT_RESET "/statreset"
+#define PATH_COUNTERS "/counters"
+#define PATH_ERRORS "/errors"
+#define PATH_NEIGHBOURS "/neighbours"
+#define PATH_PLUGINS "/plugins"
+#define PATH_PING "/ping"
+
+#define msg_err_session(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ session->pool->tag.tagname, session->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_session(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ session->pool->tag.tagname, session->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_session(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ session->pool->tag.tagname, session->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_err_ctx(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "controller", ctx->cfg->cfg_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_ctx(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "controller", ctx->cfg->cfg_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_ctx(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "controller", ctx->cfg->cfg_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+#define msg_debug_session(...) rspamd_conditional_debug_fast(NULL, session->from_addr, \
+ rspamd_controller_log_id, "controller", session->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(controller)
+
+/* Graph colors */
+#define COLOR_CLEAN "#58A458"
+#define COLOR_PROBABLE_SPAM "#D67E7E"
+#define COLOR_GREYLIST "#A0A0A0"
+#define COLOR_REJECT "#CB4B4B"
+#define COLOR_TOTAL "#9440ED"
+
+static const guint64 rspamd_controller_ctx_magic = 0xf72697805e6941faULL;
+
+extern void fuzzy_stat_command(struct rspamd_task *task);
+
+gpointer init_controller_worker(struct rspamd_config *cfg);
+void start_controller_worker(struct rspamd_worker *worker);
+
+worker_t controller_worker = {
+ "controller", /* Name */
+ init_controller_worker, /* Init function */
+ start_controller_worker, /* Start function */
+ RSPAMD_WORKER_HAS_SOCKET | RSPAMD_WORKER_KILLABLE |
+ RSPAMD_WORKER_SCANNER | RSPAMD_WORKER_CONTROLLER,
+ RSPAMD_WORKER_SOCKET_TCP, /* TCP socket */
+ RSPAMD_WORKER_VER /* Version info */
+};
+/*
+ * Worker's context
+ */
+struct rspamd_controller_worker_ctx {
+ guint64 magic;
+ /* Events base */
+ struct ev_loop *event_loop;
+ /* DNS resolver */
+ struct rspamd_dns_resolver *resolver;
+ /* Config */
+ struct rspamd_config *cfg;
+ /* END OF COMMON PART */
+ ev_tstamp timeout;
+ /* Whether we use ssl for this server */
+ gboolean use_ssl;
+ /* Webui password */
+ gchar *password;
+ /* Privileged password */
+ gchar *enable_password;
+ /* Cached versions of the passwords */
+ rspamd_ftok_t cached_password;
+ rspamd_ftok_t cached_enable_password;
+ /* HTTP server */
+ struct rspamd_http_context *http_ctx;
+ struct rspamd_http_connection_router *http;
+ /* Server's start time */
+ ev_tstamp start_time;
+ /* Main server */
+ struct rspamd_main *srv;
+ /* SSL cert */
+ gchar *ssl_cert;
+ /* SSL private key */
+ gchar *ssl_key;
+ /* A map of secure IP */
+ const ucl_object_t *secure_ip;
+ struct rspamd_radix_map_helper *secure_map;
+
+ /* Static files dir */
+ gchar *static_files_dir;
+
+ /* Custom commands registered by plugins */
+ GHashTable *custom_commands;
+
+ /* Plugins registered from lua */
+ GHashTable *plugins;
+
+ /* Worker */
+ struct rspamd_worker *worker;
+
+ /* Local keypair */
+ gpointer key;
+
+ struct rspamd_rrd_file *rrd;
+ struct rspamd_lang_detector *lang_det;
+ gdouble task_timeout;
+
+ /* Health check stuff */
+ guint workers_count;
+ guint scanners_count;
+ guint workers_hb_lost;
+ ev_timer health_check_timer;
+};
+
+struct rspamd_controller_plugin_cbdata {
+ lua_State *L;
+ struct rspamd_controller_worker_ctx *ctx;
+ gchar *plugin;
+ struct ucl_lua_funcdata *handler;
+ ucl_object_t *obj;
+ gboolean is_enable;
+ gboolean need_task;
+ guint version;
+};
+
+static gboolean
+rspamd_is_encrypted_password(const gchar *password,
+ struct rspamd_controller_pbkdf const **pbkdf)
+{
+ const gchar *start, *end;
+ gint64 id;
+ gsize size, i;
+ gboolean ret = FALSE;
+ const struct rspamd_controller_pbkdf *p;
+
+ if (password[0] == '$') {
+ /* Parse id */
+ start = password + 1;
+ end = start;
+ size = 0;
+
+ while (*end != '\0' && g_ascii_isdigit(*end)) {
+ size++;
+ end++;
+ }
+
+ if (size > 0) {
+ gchar *endptr;
+ id = strtoul(start, &endptr, 10);
+
+ if ((endptr == NULL || *endptr == *end)) {
+ for (i = 0; i < RSPAMD_PBKDF_ID_MAX - 1; i++) {
+ p = &pbkdf_list[i];
+
+ if (p->id == id) {
+ ret = TRUE;
+ if (pbkdf != NULL) {
+ *pbkdf = &pbkdf_list[i];
+ }
+
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ return ret;
+}
+
+static const gchar *
+rspamd_encrypted_password_get_str(const gchar *password, gsize skip,
+ gsize *length)
+{
+ const gchar *str, *start, *end;
+ gsize size;
+
+ start = password + skip;
+ end = start;
+ size = 0;
+
+ while (*end != '\0' && g_ascii_isalnum(*end)) {
+ size++;
+ end++;
+ }
+
+ if (size) {
+ str = start;
+ *length = size;
+ }
+ else {
+ str = NULL;
+ }
+
+ return str;
+}
+
+static gboolean
+rspamd_check_encrypted_password(struct rspamd_controller_worker_ctx *ctx,
+ const rspamd_ftok_t *password, const gchar *check,
+ const struct rspamd_controller_pbkdf *pbkdf,
+ gboolean is_enable)
+{
+ const gchar *salt, *hash;
+ gchar *salt_decoded, *key_decoded;
+ gsize salt_len = 0, key_len = 0;
+ gboolean ret = TRUE;
+ guchar *local_key;
+ rspamd_ftok_t *cache;
+ gpointer m;
+
+ /* First of all check cached versions to save resources */
+ if (is_enable && ctx->cached_enable_password.len != 0) {
+ if (password->len != ctx->cached_enable_password.len ||
+ !rspamd_constant_memcmp(password->begin,
+ ctx->cached_enable_password.begin, password->len)) {
+ msg_info_ctx("incorrect or absent enable password has been specified");
+ return FALSE;
+ }
+
+ return TRUE;
+ }
+ else if (!is_enable && ctx->cached_password.len != 0) {
+ if (password->len != ctx->cached_password.len ||
+ !rspamd_constant_memcmp(password->begin,
+ ctx->cached_password.begin, password->len)) {
+ /* We still need to check enable password here */
+ if (ctx->cached_enable_password.len != 0) {
+ if (password->len != ctx->cached_enable_password.len ||
+ !rspamd_constant_memcmp(password->begin,
+ ctx->cached_enable_password.begin,
+ password->len)) {
+ msg_info_ctx(
+ "incorrect or absent password has been specified");
+
+ return FALSE;
+ }
+ else {
+ /* Cached matched */
+ return TRUE;
+ }
+ }
+ else {
+ /* We might want to check uncached version */
+ goto check_uncached;
+ }
+ }
+ else {
+ /* Cached matched */
+ return TRUE;
+ }
+ }
+
+check_uncached:
+ g_assert(pbkdf != NULL);
+ /* get salt */
+ salt = rspamd_encrypted_password_get_str(check, 3, &salt_len);
+ /* get hash */
+ hash = rspamd_encrypted_password_get_str(check, 3 + salt_len + 1,
+ &key_len);
+ if (salt != NULL && hash != NULL) {
+
+ /* decode salt */
+ salt_decoded = rspamd_decode_base32(salt, salt_len, &salt_len, RSPAMD_BASE32_DEFAULT);
+
+ if (salt_decoded == NULL || salt_len != pbkdf->salt_len) {
+ /* We have some unknown salt here */
+ msg_info_ctx("incorrect salt: %z, while %z expected",
+ salt_len, pbkdf->salt_len);
+ g_free(salt_decoded);
+
+ return FALSE;
+ }
+
+ key_decoded = rspamd_decode_base32(hash, key_len, &key_len, RSPAMD_BASE32_DEFAULT);
+
+ if (key_decoded == NULL || key_len != pbkdf->key_len) {
+ /* We have some unknown salt here */
+ msg_info_ctx("incorrect key: %z, while %z expected",
+ key_len, pbkdf->key_len);
+ g_free(salt_decoded);
+ g_free(key_decoded); /* valid even if key_decoded == NULL */
+
+ return FALSE;
+ }
+
+ local_key = g_alloca(pbkdf->key_len);
+ rspamd_cryptobox_pbkdf(password->begin, password->len,
+ salt_decoded, salt_len,
+ local_key, pbkdf->key_len, pbkdf->complexity,
+ pbkdf->type);
+
+ if (!rspamd_constant_memcmp(key_decoded, local_key, pbkdf->key_len)) {
+ msg_info_ctx("incorrect or absent password has been specified");
+ ret = FALSE;
+ }
+
+ g_free(salt_decoded);
+ g_free(key_decoded);
+ }
+
+ if (ret) {
+ /* Save cached version */
+ cache = is_enable ? &ctx->cached_enable_password : &ctx->cached_password;
+
+ if (cache->len == 0) {
+ /* Mmap region */
+#ifdef MAP_NOCORE
+ m = mmap(NULL, password->len, PROT_WRITE,
+ MAP_PRIVATE | MAP_ANON | MAP_NOCORE, -1, 0);
+#else
+ m = mmap(NULL, password->len, PROT_WRITE,
+ MAP_PRIVATE | MAP_ANON, -1, 0);
+#endif
+ if (m != MAP_FAILED) {
+ memcpy(m, password->begin, password->len);
+ (void) mprotect(m, password->len, PROT_READ);
+ (void) mlock(m, password->len);
+ cache->begin = m;
+ cache->len = password->len;
+ }
+ else {
+ msg_err_ctx("cannot store cached password, mmap failed: %s",
+ strerror(errno));
+ }
+ }
+ }
+
+ return ret;
+}
+
+/**
+ * Checks for X-Forwarded-For header and update client's address if needed
+ *
+ * This function is intended to be called for a trusted client to ensure that
+ * a request is not proxied through it
+ * @return 0 if no forwarded found, 1 if forwarded found and it is yet trusted
+ * and -1 if forwarded is denied
+ */
+static gint
+rspamd_controller_check_forwarded(struct rspamd_controller_session *session,
+ struct rspamd_http_message *msg,
+ struct rspamd_controller_worker_ctx *ctx)
+{
+ const rspamd_ftok_t *hdr;
+ const gchar *comma;
+ const char *hdr_name = "X-Forwarded-For", *alt_hdr_name = "X-Real-IP";
+ char ip_buf[INET6_ADDRSTRLEN + 1];
+ rspamd_inet_addr_t *addr = NULL;
+ gint ret = 0;
+
+ hdr = rspamd_http_message_find_header(msg, hdr_name);
+
+ if (hdr) {
+ /*
+ * We need to parse and update the header
+ * X-Forwarded-For: client, proxy1, proxy2
+ */
+ comma = rspamd_memrchr(hdr->begin, ',', hdr->len);
+ if (comma != NULL) {
+ while (comma < hdr->begin + hdr->len &&
+ (*comma == ',' || g_ascii_isspace(*comma))) {
+ comma++;
+ }
+ }
+ else {
+ comma = hdr->begin;
+ }
+ if (rspamd_parse_inet_address(&addr, comma,
+ (hdr->begin + hdr->len) - comma,
+ RSPAMD_INET_ADDRESS_PARSE_NO_UNIX)) {
+ /* We have addr now, so check if it is still trusted */
+ if (ctx->secure_map &&
+ rspamd_match_radix_map_addr(ctx->secure_map, addr) != NULL) {
+ /* rspamd_inet_address_to_string is not reentrant */
+ rspamd_strlcpy(ip_buf, rspamd_inet_address_to_string(addr),
+ sizeof(ip_buf));
+ msg_info_session("allow unauthorized proxied connection "
+ "from a trusted IP %s via %s",
+ ip_buf,
+ rspamd_inet_address_to_string(session->from_addr));
+ ret = 1;
+ }
+ else {
+ ret = -1;
+ }
+
+ rspamd_inet_address_free(addr);
+ }
+ else {
+ msg_warn_session("cannot parse forwarded IP: %T", hdr);
+ ret = -1;
+ }
+ }
+ else {
+ /* Try also X-Real-IP */
+ hdr = rspamd_http_message_find_header(msg, alt_hdr_name);
+
+ if (hdr) {
+ if (rspamd_parse_inet_address(&addr, hdr->begin, hdr->len,
+ RSPAMD_INET_ADDRESS_PARSE_NO_UNIX)) {
+ /* We have addr now, so check if it is still trusted */
+ if (ctx->secure_map &&
+ rspamd_match_radix_map_addr(ctx->secure_map, addr) != NULL) {
+ /* rspamd_inet_address_to_string is not reentrant */
+ rspamd_strlcpy(ip_buf, rspamd_inet_address_to_string(addr),
+ sizeof(ip_buf));
+ msg_info_session("allow unauthorized proxied connection "
+ "from a trusted IP %s via %s",
+ ip_buf,
+ rspamd_inet_address_to_string(session->from_addr));
+ ret = 1;
+ }
+ else {
+ ret = -1;
+ }
+
+ rspamd_inet_address_free(addr);
+ }
+ else {
+ msg_warn_session("cannot parse real IP: %T", hdr);
+ ret = -1;
+ }
+ }
+ }
+
+ return ret;
+}
+
+/* Check for password if it is required by configuration */
+static gboolean
+rspamd_controller_check_password(struct rspamd_http_connection_entry *entry,
+ struct rspamd_controller_session *session,
+ struct rspamd_http_message *msg, gboolean is_enable)
+{
+ const gchar *check;
+ const rspamd_ftok_t *password;
+ rspamd_ftok_t lookup;
+ GHashTable *query_args = NULL;
+ struct rspamd_controller_worker_ctx *ctx = session->ctx;
+ gboolean check_normal = FALSE, check_enable = FALSE, ret = TRUE,
+ use_enable = FALSE;
+ const struct rspamd_controller_pbkdf *pbkdf = NULL;
+
+ /* Fail-safety */
+ session->is_read_only = TRUE;
+
+ /* Access list logic */
+ if (rspamd_inet_address_get_af(session->from_addr) == AF_UNIX) {
+ ret = rspamd_controller_check_forwarded(session, msg, ctx);
+
+ if (ret == 1) {
+ session->is_read_only = FALSE;
+
+ return TRUE;
+ }
+ else if (ret == 0) {
+ /* No forwarded found */
+ msg_info_session("allow unauthorized connection from a unix socket");
+ session->is_read_only = FALSE;
+
+ return TRUE;
+ }
+ }
+ else if (ctx->secure_map && rspamd_match_radix_map_addr(ctx->secure_map, session->from_addr) != NULL) {
+ ret = rspamd_controller_check_forwarded(session, msg, ctx);
+
+ if (ret == 1) {
+ session->is_read_only = FALSE;
+
+ return TRUE;
+ }
+ else if (ret == 0) {
+ /* No forwarded found */
+ msg_info_session("allow unauthorized connection from a trusted IP %s",
+ rspamd_inet_address_to_string(session->from_addr));
+ session->is_read_only = FALSE;
+
+ return TRUE;
+ }
+ }
+
+ /* Password logic */
+ password = rspamd_http_message_find_header(msg, "Password");
+
+ if (password == NULL) {
+ /* Try to get password from query args */
+ query_args = rspamd_http_message_parse_query(msg);
+
+ lookup.begin = (gchar *) "password";
+ lookup.len = sizeof("password") - 1;
+
+ password = g_hash_table_lookup(query_args, &lookup);
+ }
+
+ if (password == NULL) {
+ if (ctx->secure_map == NULL) {
+ if (ctx->password == NULL && !is_enable) {
+ return TRUE;
+ }
+ else if (is_enable && (ctx->password == NULL &&
+ ctx->enable_password == NULL)) {
+ session->is_read_only = FALSE;
+ return TRUE;
+ }
+ }
+
+ msg_info_session("absent password has been specified; source ip: %s",
+ rspamd_inet_address_to_string_pretty(session->from_addr));
+ ret = FALSE;
+ }
+ else {
+ if (rspamd_ftok_cstr_equal(password, "q1", FALSE) ||
+ rspamd_ftok_cstr_equal(password, "q2", FALSE)) {
+ msg_info_session("deny default password for remote access; source ip: %s",
+ rspamd_inet_address_to_string_pretty(session->from_addr));
+ ret = FALSE;
+ goto end;
+ }
+
+ if (is_enable) {
+ /* For privileged commands we strictly require enable password */
+ if (ctx->enable_password != NULL) {
+ check = ctx->enable_password;
+ use_enable = TRUE;
+ }
+ else {
+ /* Use just a password (legacy mode) */
+ msg_info(
+ "using password as enable_password for a privileged command");
+ check = ctx->password;
+ }
+
+ if (check != NULL) {
+ if (!rspamd_is_encrypted_password(check, &pbkdf)) {
+ ret = FALSE;
+
+ if (strlen(check) == password->len) {
+ ret = rspamd_constant_memcmp(password->begin, check,
+ password->len);
+ }
+ }
+ else {
+ ret = rspamd_check_encrypted_password(ctx, password, check,
+ pbkdf, use_enable);
+ }
+
+ if (ret) {
+ check_enable = TRUE;
+ }
+ }
+ else {
+ msg_warn_session(
+ "no password to check while executing a privileged command; source ip: %s",
+ rspamd_inet_address_to_string_pretty(session->from_addr));
+ ret = FALSE;
+ }
+
+ if (ret) {
+ session->is_read_only = FALSE;
+ }
+ }
+ else {
+ /* Accept both normal and enable passwords */
+ if (ctx->password != NULL) {
+ check = ctx->password;
+
+ if (!rspamd_is_encrypted_password(check, &pbkdf)) {
+ check_normal = FALSE;
+
+ if (strlen(check) == password->len) {
+ check_normal = rspamd_constant_memcmp(password->begin,
+ check,
+ password->len);
+ }
+ }
+ else {
+ check_normal = rspamd_check_encrypted_password(ctx,
+ password,
+ check, pbkdf, FALSE);
+ }
+
+ if (check_normal) {
+ if (ctx->enable_password == NULL) {
+ /* We have passed password check and no enable password is specified */
+ session->is_read_only = FALSE;
+ }
+ else {
+ /*
+ * Even if we have passed normal password check, we don't really
+ * know if password == enable_password, so we need to check it
+ * as well, to decide if we are in read-only mode or not
+ */
+ check = ctx->enable_password;
+
+ if (!rspamd_is_encrypted_password(check, &pbkdf)) {
+ check_enable = FALSE;
+
+ if (strlen(check) == password->len) {
+ check_enable = rspamd_constant_memcmp(password->begin,
+ check,
+ password->len);
+ }
+ }
+ else {
+ check_enable = rspamd_check_encrypted_password(ctx,
+ password,
+ check, pbkdf, TRUE);
+ }
+ }
+ }
+ }
+
+ if ((!check_normal && !check_enable) && ctx->enable_password != NULL) {
+ check = ctx->enable_password;
+
+ if (!rspamd_is_encrypted_password(check, &pbkdf)) {
+ check_enable = FALSE;
+
+ if (strlen(check) == password->len) {
+ check_enable = rspamd_constant_memcmp(password->begin,
+ check,
+ password->len);
+ }
+ }
+ else {
+ check_enable = rspamd_check_encrypted_password(ctx,
+ password,
+ check, pbkdf, TRUE);
+ }
+ }
+
+ if (check_enable) {
+ /* We have passed enable password check, not a read-only mode */
+ session->is_read_only = FALSE;
+ }
+ }
+ }
+
+ if (check_normal == FALSE && check_enable == FALSE) {
+ msg_info("absent or incorrect password has been specified; source ip: %s",
+ rspamd_inet_address_to_string_pretty(session->from_addr));
+ ret = FALSE;
+ }
+
+end:
+ if (query_args != NULL) {
+ g_hash_table_unref(query_args);
+ }
+
+ if (!ret) {
+ rspamd_controller_send_error(entry, 401, "Unauthorized");
+ }
+
+ return ret;
+}
+
+/* Command handlers */
+
+/*
+ * Auth command handler:
+ * request: /auth
+ * headers: Password
+ * reply: json {"auth": "ok", "version": "0.5.2", "uptime": "some uptime", "error": "none"}
+ */
+static int
+rspamd_controller_handle_auth(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_stat st;
+ int64_t uptime;
+ gulong data[5];
+ ucl_object_t *obj;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ obj = ucl_object_typed_new(UCL_OBJECT);
+ memcpy(&st, session->ctx->srv->stat, sizeof(st));
+ data[0] = st.actions_stat[METRIC_ACTION_NOACTION];
+ data[1] = st.actions_stat[METRIC_ACTION_ADD_HEADER] +
+ st.actions_stat[METRIC_ACTION_REWRITE_SUBJECT];
+ data[2] = st.actions_stat[METRIC_ACTION_GREYLIST];
+ data[3] = st.actions_stat[METRIC_ACTION_REJECT];
+ data[4] = st.actions_stat[METRIC_ACTION_SOFT_REJECT];
+
+ /* Get uptime */
+ uptime = ev_time() - session->ctx->start_time;
+
+ ucl_object_insert_key(obj, ucl_object_fromstring(RVERSION), "version", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromstring("ok"), "auth", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromint(uptime), "uptime", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromint(data[0]), "clean", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromint(data[1]), "probable", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromint(data[2]), "greylist", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromint(data[3]), "reject", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromint(data[4]), "soft_reject", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromint(st.messages_scanned), "scanned", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromint(st.messages_learned), "learned", 0, false);
+ ucl_object_insert_key(obj, ucl_object_frombool(session->is_read_only),
+ "read_only", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromstring(session->ctx->cfg->checksum),
+ "config_id", 0, false);
+
+ rspamd_controller_send_ucl(conn_ent, obj);
+ ucl_object_unref(obj);
+
+ return 0;
+}
+
+/*
+ * Symbols command handler:
+ * request: /symbols
+ * reply: json [{
+ * "name": "group_name",
+ * "symbols": [
+ * {
+ * "name": "name",
+ * "weight": 0.1,
+ * "description": "description of symbol"
+ * },
+ * {...}
+ * },
+ * {...}]
+ */
+static int
+rspamd_controller_handle_symbols(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ GHashTableIter it, sit;
+ struct rspamd_symbols_group *gr;
+ struct rspamd_symbol *sym;
+ ucl_object_t *obj, *top, *sym_obj, *group_symbols;
+ gpointer k, v;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ top = ucl_object_typed_new(UCL_ARRAY);
+
+ /* Go through all symbols groups in the default metric */
+ g_hash_table_iter_init(&it, session->cfg->groups);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ gr = v;
+ obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj, ucl_object_fromstring(gr->name), "group", 0, false);
+ /* Iterate through all symbols */
+
+ g_hash_table_iter_init(&sit, gr->symbols);
+ group_symbols = ucl_object_typed_new(UCL_ARRAY);
+
+ while (g_hash_table_iter_next(&sit, &k, &v)) {
+ gdouble tm = 0.0, freq = 0, freq_dev = 0;
+
+ sym = v;
+ sym_obj = ucl_object_typed_new(UCL_OBJECT);
+
+ ucl_object_insert_key(sym_obj, ucl_object_fromstring(sym->name),
+ "symbol", 0, false);
+ ucl_object_insert_key(sym_obj,
+ ucl_object_fromdouble(*sym->weight_ptr),
+ "weight", 0, false);
+ if (sym->description) {
+ ucl_object_insert_key(sym_obj,
+ ucl_object_fromstring(sym->description),
+ "description", 0, false);
+ }
+
+ if (rspamd_symcache_stat_symbol(session->ctx->cfg->cache,
+ sym->name, &freq, &freq_dev, &tm, NULL)) {
+ ucl_object_insert_key(sym_obj,
+ ucl_object_fromdouble(freq),
+ "frequency", 0, false);
+ ucl_object_insert_key(sym_obj,
+ ucl_object_fromdouble(freq_dev),
+ "frequency_stddev", 0, false);
+ ucl_object_insert_key(sym_obj,
+ ucl_object_fromdouble(tm),
+ "time", 0, false);
+ }
+
+ ucl_array_append(group_symbols, sym_obj);
+ }
+
+ ucl_object_insert_key(obj, group_symbols, "rules", 0, false);
+ ucl_array_append(top, obj);
+ }
+
+ rspamd_controller_send_ucl(conn_ent, top);
+ ucl_object_unref(top);
+
+ return 0;
+}
+
+static void
+rspamd_controller_actions_cb(struct rspamd_action *act, void *cbd)
+{
+ ucl_object_t *top = (ucl_object_t *) cbd;
+ ucl_object_t *obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj,
+ ucl_object_fromstring(act->name),
+ "action", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(act->threshold),
+ "value", 0, false);
+ ucl_array_append(top, obj);
+}
+
+/*
+ * Actions command handler:
+ * request: /actions
+ * reply: json [{
+ * "action": "no action",
+ * "value": 1.1
+ * },
+ * {...}]
+ */
+static int
+rspamd_controller_handle_actions(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ ucl_object_t *top;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ top = ucl_object_typed_new(UCL_ARRAY);
+
+ rspamd_config_actions_foreach(session->cfg, rspamd_controller_actions_cb, top);
+ rspamd_controller_send_ucl(conn_ent, top);
+ ucl_object_unref(top);
+
+ return 0;
+}
+
+static gboolean
+rspamd_controller_can_edit_map(struct rspamd_map_backend *bk)
+{
+ gchar *fpath;
+
+ if (access(bk->uri, W_OK) == 0) {
+ return TRUE;
+ }
+ else if (access(bk->uri, R_OK) == -1 && errno == ENOENT) {
+ fpath = g_path_get_dirname(bk->uri);
+
+ if (fpath) {
+ if (access(fpath, W_OK) == 0) {
+ g_free(fpath);
+
+ return TRUE;
+ }
+
+ g_free(fpath);
+ }
+ }
+
+ return FALSE;
+}
+
+/*
+ * Maps command handler:
+ * request: /maps
+ * headers: Password
+ * reply: json [
+ * {
+ * "map": "name",
+ * "description": "description",
+ * "editable": true
+ * },
+ * {...}
+ * ]
+ */
+static int
+rspamd_controller_handle_maps(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ GList *cur;
+ struct rspamd_map *map;
+ struct rspamd_map_backend *bk;
+ guint i;
+ gboolean editable;
+ ucl_object_t *obj, *top;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ top = ucl_object_typed_new(UCL_ARRAY);
+
+ /* Iterate over all maps */
+ cur = session->ctx->cfg->maps;
+ while (cur) {
+ map = cur->data;
+
+ PTR_ARRAY_FOREACH(map->backends, i, bk)
+ {
+
+ if (bk->protocol == MAP_PROTO_FILE) {
+ editable = rspamd_controller_can_edit_map(bk);
+
+ if (!editable && access(bk->uri, R_OK) == -1) {
+ /* Skip unreadable and non-existing maps */
+ continue;
+ }
+
+ obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj, ucl_object_fromint(bk->id),
+ "map", 0, false);
+ if (map->description) {
+ ucl_object_insert_key(obj, ucl_object_fromstring(map->description),
+ "description", 0, false);
+ }
+ ucl_object_insert_key(obj, ucl_object_fromstring(bk->uri),
+ "uri", 0, false);
+ ucl_object_insert_key(obj, ucl_object_frombool(editable),
+ "editable", 0, false);
+ ucl_array_append(top, obj);
+ }
+ }
+ cur = g_list_next(cur);
+ }
+
+ rspamd_controller_send_ucl(conn_ent, top);
+ ucl_object_unref(top);
+
+ return 0;
+}
+
+/*
+ * Get map command handler:
+ * request: /getmap
+ * headers: Password, Map
+ * reply: plain-text
+ */
+static int
+rspamd_controller_handle_get_map(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ GList *cur;
+ struct rspamd_map *map;
+ struct rspamd_map_backend *bk = NULL;
+ const rspamd_ftok_t *idstr;
+ struct stat st;
+ gint fd;
+ gulong id, i;
+ gboolean found = FALSE;
+ struct rspamd_http_message *reply;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ idstr = rspamd_http_message_find_header(msg, "Map");
+
+ if (idstr == NULL) {
+ msg_info_session("absent map id");
+ rspamd_controller_send_error(conn_ent, 400, "Id header missing");
+ return 0;
+ }
+
+ if (!rspamd_strtoul(idstr->begin, idstr->len, &id)) {
+ msg_info_session("invalid map id");
+ rspamd_controller_send_error(conn_ent, 400, "Invalid map id");
+ return 0;
+ }
+
+ /* Now let's be sure that we have map defined in configuration */
+ cur = session->ctx->cfg->maps;
+ while (cur && !found) {
+ map = cur->data;
+
+ PTR_ARRAY_FOREACH(map->backends, i, bk)
+ {
+ if (bk->id == id && bk->protocol == MAP_PROTO_FILE) {
+ found = TRUE;
+ break;
+ }
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ if (!found || bk == NULL) {
+ msg_info_session("map not found");
+ rspamd_controller_send_error(conn_ent, 404, "Map not found");
+ return 0;
+ }
+
+ if (stat(bk->uri, &st) == -1 || (fd = open(bk->uri, O_RDONLY)) == -1) {
+ reply = rspamd_http_new_message(HTTP_RESPONSE);
+ reply->date = time(NULL);
+ reply->code = 200;
+ }
+ else {
+
+ reply = rspamd_http_new_message(HTTP_RESPONSE);
+ reply->date = time(NULL);
+ reply->code = 200;
+
+ if (st.st_size > 0) {
+ if (!rspamd_http_message_set_body_from_fd(reply, fd)) {
+ close(fd);
+ rspamd_http_message_unref(reply);
+ msg_err_session("cannot read map %s: %s", bk->uri, strerror(errno));
+ rspamd_controller_send_error(conn_ent, 500, "Map read error");
+ return 0;
+ }
+ }
+ else {
+ rspamd_fstring_t *empty_body = rspamd_fstring_new_init("", 0);
+ rspamd_http_message_set_body_from_fstring_steal(reply, empty_body);
+ }
+
+ close(fd);
+ }
+
+ rspamd_http_connection_reset(conn_ent->conn);
+ rspamd_http_router_insert_headers(conn_ent->rt, reply);
+ rspamd_http_connection_write_message(conn_ent->conn, reply, NULL,
+ "text/plain", conn_ent,
+ conn_ent->rt->timeout);
+ conn_ent->is_reply = TRUE;
+
+ return 0;
+}
+
+static ucl_object_t *
+rspamd_controller_pie_element(enum rspamd_action_type action,
+ const char *label, gdouble data)
+{
+ ucl_object_t *res = ucl_object_typed_new(UCL_OBJECT);
+ const char *colors[METRIC_ACTION_MAX] = {
+ [METRIC_ACTION_REJECT] = "#FF0000",
+ [METRIC_ACTION_SOFT_REJECT] = "#cc9966",
+ [METRIC_ACTION_REWRITE_SUBJECT] = "#ff6600",
+ [METRIC_ACTION_ADD_HEADER] = "#FFD700",
+ [METRIC_ACTION_GREYLIST] = "#436EEE",
+ [METRIC_ACTION_NOACTION] = "#66cc00"};
+
+ ucl_object_insert_key(res, ucl_object_fromstring(colors[action]),
+ "color", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromstring(label), "label", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromdouble(data), "data", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromdouble(data), "value", 0, false);
+
+ return res;
+}
+
+/*
+ * Pie chart command handler:
+ * request: /pie
+ * headers: Password
+ * reply: json [
+ * { label: "Foo", data: 11 },
+ * { label: "Bar", data: 20 },
+ * {...}
+ * ]
+ */
+static int
+rspamd_controller_handle_pie_chart(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_controller_worker_ctx *ctx;
+ gdouble data[5], total;
+ ucl_object_t *top;
+
+ ctx = session->ctx;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ top = ucl_object_typed_new(UCL_ARRAY);
+ total = ctx->srv->stat->messages_scanned;
+ if (total != 0) {
+
+ data[0] = ctx->srv->stat->actions_stat[METRIC_ACTION_NOACTION];
+ data[1] = ctx->srv->stat->actions_stat[METRIC_ACTION_SOFT_REJECT];
+ data[2] = (ctx->srv->stat->actions_stat[METRIC_ACTION_ADD_HEADER] +
+ ctx->srv->stat->actions_stat[METRIC_ACTION_REWRITE_SUBJECT]);
+ data[3] = ctx->srv->stat->actions_stat[METRIC_ACTION_GREYLIST];
+ data[4] = ctx->srv->stat->actions_stat[METRIC_ACTION_REJECT];
+ }
+ else {
+ memset(data, 0, sizeof(data));
+ }
+ ucl_array_append(top, rspamd_controller_pie_element(
+ METRIC_ACTION_NOACTION, "Clean", data[0]));
+ ucl_array_append(top, rspamd_controller_pie_element(
+ METRIC_ACTION_SOFT_REJECT, "Temporarily rejected", data[1]));
+ ucl_array_append(top, rspamd_controller_pie_element(
+ METRIC_ACTION_ADD_HEADER, "Probable spam", data[2]));
+ ucl_array_append(top, rspamd_controller_pie_element(
+ METRIC_ACTION_GREYLIST, "Greylisted", data[3]));
+ ucl_array_append(top, rspamd_controller_pie_element(
+ METRIC_ACTION_REJECT, "Rejected", data[4]));
+
+ rspamd_controller_send_ucl(conn_ent, top);
+ ucl_object_unref(top);
+
+ return 0;
+}
+
+void rspamd_controller_graph_point(gulong t, gulong step,
+ struct rspamd_rrd_query_result *rrd_result,
+ gdouble *acc,
+ ucl_object_t **elt)
+{
+ guint nan_cnt;
+ gdouble sum = 0.0, yval;
+ ucl_object_t *data_elt;
+ guint i, j;
+
+ for (i = 0; i < rrd_result->ds_count; i++) {
+ sum = 0.0;
+ nan_cnt = 0;
+ data_elt = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(data_elt, ucl_object_fromint(t), "x", 1, false);
+
+ for (j = 0; j < step; j++) {
+ yval = acc[i + j * rrd_result->ds_count];
+ if (!isfinite(yval)) {
+ nan_cnt++;
+ }
+ else {
+ sum += yval;
+ }
+ }
+ if (nan_cnt == step) {
+ ucl_object_insert_key(data_elt, ucl_object_typed_new(UCL_NULL),
+ "y", 1, false);
+ }
+ else {
+ ucl_object_insert_key(data_elt,
+ ucl_object_fromdouble(sum / (gdouble) step), "y", 1,
+ false);
+ }
+ ucl_array_append(elt[i], data_elt);
+ }
+}
+
+/*
+ * Graph command handler:
+ * request: /graph?type=<day|week|month|year>
+ * headers: Password
+ * reply: json [
+ * { label: "Foo", data: 11 },
+ * { label: "Bar", data: 20 },
+ * {...}
+ * ]
+ */
+static int
+rspamd_controller_handle_graph(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ GHashTable *query;
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_controller_worker_ctx *ctx;
+ rspamd_ftok_t srch, *value;
+ struct rspamd_rrd_query_result *rrd_result;
+ gulong i, k, start_row, cnt, t, ts, step;
+ gdouble *acc;
+ ucl_object_t *res, *elt[METRIC_ACTION_MAX];
+ enum {
+ rra_day = 0,
+ rra_week,
+ rra_month,
+ rra_year,
+ rra_invalid
+ } rra_num = rra_invalid;
+ /* How many points are we going to send to display */
+ static const guint desired_points = 500;
+
+ ctx = session->ctx;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ if (ctx->rrd == NULL) {
+ msg_err_session("no rrd configured");
+ rspamd_controller_send_error(conn_ent, 404, "No rrd configured for graphs");
+
+ return 0;
+ }
+
+ query = rspamd_http_message_parse_query(msg);
+ srch.begin = (gchar *) "type";
+ srch.len = 4;
+
+ if (query == NULL || (value = g_hash_table_lookup(query, &srch)) == NULL) {
+ msg_err_session("absent graph type query");
+ rspamd_controller_send_error(conn_ent, 400, "Absent graph type");
+
+ if (query) {
+ g_hash_table_unref(query);
+ }
+
+ return 0;
+ }
+
+ if (value->len == 3 && rspamd_lc_cmp(value->begin, "day", value->len) == 0) {
+ rra_num = rra_day;
+ }
+ else if (value->len == 4 && rspamd_lc_cmp(value->begin, "week", value->len) == 0) {
+ rra_num = rra_week;
+ }
+ else if (value->len == 5 && rspamd_lc_cmp(value->begin, "month", value->len) == 0) {
+ rra_num = rra_month;
+ }
+ else if (value->len == 4 && rspamd_lc_cmp(value->begin, "year", value->len) == 0) {
+ rra_num = rra_year;
+ }
+
+ g_hash_table_unref(query);
+
+ if (rra_num == rra_invalid) {
+ msg_err_session("invalid graph type query");
+ rspamd_controller_send_error(conn_ent, 400, "Invalid graph type");
+
+ return 0;
+ }
+
+ rrd_result = rspamd_rrd_query(ctx->rrd, rra_num);
+
+ if (rrd_result == NULL) {
+ msg_err_session("cannot query rrd");
+ rspamd_controller_send_error(conn_ent, 500, "Cannot query rrd");
+
+ return 0;
+ }
+
+ g_assert(rrd_result->ds_count == G_N_ELEMENTS(elt));
+
+ res = ucl_object_typed_new(UCL_ARRAY);
+ /* How much full updates happened since the last update */
+ ts = rrd_result->last_update / rrd_result->pdp_per_cdp - rrd_result->rra_rows;
+
+ for (i = 0; i < rrd_result->ds_count; i++) {
+ elt[i] = ucl_object_typed_new(UCL_ARRAY);
+ }
+
+ start_row = rrd_result->cur_row == rrd_result->rra_rows - 1 ? 0 : rrd_result->cur_row;
+ t = ts * rrd_result->pdp_per_cdp;
+ k = 0;
+
+ /* Create window */
+ step = ceil(((gdouble) rrd_result->rra_rows) / desired_points);
+ g_assert(step >= 1);
+ acc = g_malloc0(sizeof(double) * rrd_result->ds_count * step);
+
+ for (i = start_row, cnt = 0; cnt < rrd_result->rra_rows;
+ cnt++) {
+
+ memcpy(&acc[k * rrd_result->ds_count],
+ &rrd_result->data[i * rrd_result->ds_count],
+ sizeof(gdouble) * rrd_result->ds_count);
+
+ if (k < step - 1) {
+ k++;
+ }
+ else {
+ t = ts * rrd_result->pdp_per_cdp;
+
+ /* Need a fresh point */
+ rspamd_controller_graph_point(t, step, rrd_result, acc, elt);
+ k = 0;
+ }
+
+ if (i == rrd_result->rra_rows - 1) {
+ i = 0;
+ }
+ else {
+ i++;
+ }
+
+ ts++;
+ }
+
+ if (k > 0) {
+ rspamd_controller_graph_point(t, k, rrd_result, acc, elt);
+ }
+
+ for (i = 0; i < rrd_result->ds_count; i++) {
+ ucl_array_append(res, elt[i]);
+ }
+
+ rspamd_controller_send_ucl(conn_ent, res);
+ ucl_object_unref(res);
+ g_free(acc);
+ g_free(rrd_result);
+
+ return 0;
+}
+
+static void
+rspamd_controller_handle_legacy_history(
+ struct rspamd_controller_session *session,
+ struct rspamd_controller_worker_ctx *ctx,
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct roll_history_row *row, *copied_rows;
+ guint i, rows_proc, row_num;
+ struct tm tm;
+ gchar timebuf[32], **syms;
+ ucl_object_t *top, *obj;
+
+ top = ucl_object_typed_new(UCL_ARRAY);
+
+ /* Set lock on history */
+ copied_rows = g_malloc(sizeof(*copied_rows) * ctx->srv->history->nrows);
+ memcpy(copied_rows, ctx->srv->history->rows,
+ sizeof(*copied_rows) * ctx->srv->history->nrows);
+
+ /* Go through all rows */
+ row_num = ctx->srv->history->cur_row;
+
+ for (i = 0, rows_proc = 0; i < ctx->srv->history->nrows; i++, row_num++) {
+ if (row_num == ctx->srv->history->nrows) {
+ row_num = 0;
+ }
+ row = &copied_rows[row_num];
+ /* Get only completed rows */
+ if (row->completed) {
+ rspamd_localtime(row->timestamp, &tm);
+ strftime(timebuf, sizeof(timebuf) - 1, "%Y-%m-%d %H:%M:%S", &tm);
+ obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj, ucl_object_fromstring(timebuf), "time", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromint(row->timestamp), "unix_time", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromstring(row->message_id), "id", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromstring(row->from_addr),
+ "ip", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromstring(rspamd_action_to_str(
+ row->action)),
+ "action", 0, false);
+
+ if (!isnan(row->score)) {
+ ucl_object_insert_key(obj, ucl_object_fromdouble(row->score), "score", 0, false);
+ }
+ else {
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(0.0), "score", 0, false);
+ }
+
+ if (!isnan(row->required_score)) {
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(
+ row->required_score),
+ "required_score", 0, false);
+ }
+ else {
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(0.0), "required_score", 0, false);
+ }
+
+ syms = g_strsplit_set(row->symbols, ", ", -1);
+
+ if (syms) {
+ guint nelts = g_strv_length(syms);
+ ucl_object_t *syms_obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_reserve(syms_obj, nelts);
+
+ for (guint j = 0; j < nelts; j++) {
+ g_strstrip(syms[j]);
+
+ if (strlen(syms[j]) == 0) {
+ /* Empty garbage */
+ continue;
+ }
+
+ ucl_object_t *cur = ucl_object_typed_new(UCL_OBJECT);
+
+ ucl_object_insert_key(cur, ucl_object_fromdouble(0.0),
+ "score", 0, false);
+ ucl_object_insert_key(syms_obj, cur, syms[j], 0, true);
+ }
+
+ ucl_object_insert_key(obj, syms_obj, "symbols", 0, false);
+ g_strfreev(syms);
+ }
+
+ ucl_object_insert_key(obj, ucl_object_fromint(row->len),
+ "size", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(row->scan_time),
+ "scan_time", 0, false);
+
+ if (row->user[0] != '\0') {
+ ucl_object_insert_key(obj, ucl_object_fromstring(row->user),
+ "user", 0, false);
+ }
+ if (row->from_addr[0] != '\0') {
+ ucl_object_insert_key(obj, ucl_object_fromstring(row->from_addr), "from", 0, false);
+ }
+ ucl_array_append(top, obj);
+ rows_proc++;
+ }
+ }
+
+ rspamd_controller_send_ucl(conn_ent, top);
+ ucl_object_unref(top);
+ g_free(copied_rows);
+}
+
+static gboolean
+rspamd_controller_history_lua_fin_task(void *ud)
+{
+ return TRUE;
+}
+
+static void
+rspamd_controller_handle_lua_history(lua_State *L,
+ struct rspamd_controller_session *session,
+ struct rspamd_controller_worker_ctx *ctx,
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg,
+ gboolean reset)
+{
+ struct rspamd_task *task, **ptask;
+ struct rspamd_http_connection_entry **pconn_ent;
+ GHashTable *params;
+ rspamd_ftok_t srch, *found;
+ glong from = 0, to = -1;
+
+ params = rspamd_http_message_parse_query(msg);
+
+ if (params) {
+ /* Check from and to */
+ RSPAMD_FTOK_ASSIGN(&srch, "from");
+ found = g_hash_table_lookup(params, &srch);
+
+ if (found) {
+ rspamd_strtol(found->begin, found->len, &from);
+ }
+ RSPAMD_FTOK_ASSIGN(&srch, "to");
+ found = g_hash_table_lookup(params, &srch);
+
+ if (found) {
+ rspamd_strtol(found->begin, found->len, &to);
+ }
+
+ g_hash_table_unref(params);
+ }
+
+ lua_getglobal(L, "rspamd_plugins");
+
+ if (lua_istable(L, -1)) {
+ lua_pushstring(L, "history");
+ lua_gettable(L, -2);
+
+ if (lua_istable(L, -1)) {
+ lua_pushstring(L, "handler");
+ lua_gettable(L, -2);
+
+ if (lua_isfunction(L, -1)) {
+ task = rspamd_task_new(session->ctx->worker, session->cfg,
+ session->pool, ctx->lang_det, ctx->event_loop, FALSE);
+
+ task->resolver = ctx->resolver;
+ task->s = rspamd_session_create(session->pool,
+ rspamd_controller_history_lua_fin_task,
+ NULL,
+ (event_finalizer_t) rspamd_task_free,
+ task);
+ task->fin_arg = conn_ent;
+
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ pconn_ent = lua_newuserdata(L, sizeof(*pconn_ent));
+ *pconn_ent = conn_ent;
+ rspamd_lua_setclass(L, "rspamd{csession}", -1);
+ lua_pushinteger(L, from);
+ lua_pushinteger(L, to);
+ lua_pushboolean(L, reset);
+
+ if (lua_pcall(L, 5, 0, 0) != 0) {
+ msg_err_session("call to history function failed: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+ rspamd_task_free(task);
+
+ goto err;
+ }
+
+ task->http_conn = rspamd_http_connection_ref(conn_ent->conn);
+ ;
+ task->sock = -1;
+ session->task = task;
+
+ rspamd_session_pending(task->s);
+ }
+ else {
+ msg_err_session("rspamd_plugins.history.handler is not a function");
+ lua_settop(L, 0);
+ goto err;
+ }
+ }
+ else {
+ msg_err_session("rspamd_plugins.history is not a table");
+ lua_settop(L, 0);
+ goto err;
+ }
+ }
+ else {
+ msg_err_session("rspamd_plugins is absent or has incorrect type");
+ lua_settop(L, 0);
+ goto err;
+ }
+
+ lua_settop(L, 0);
+
+ return;
+err:
+ rspamd_controller_send_error(conn_ent, 500, "Internal error");
+}
+
+/*
+ * Healthy command handler:
+ * request: /healthy
+ * headers: Password
+ * reply: json {"success":true}
+ */
+static int
+rspamd_controller_handle_healthy(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ if (session->ctx->workers_hb_lost != 0) {
+ rspamd_controller_send_error(conn_ent, 500,
+ "%d workers are not responding", session->ctx->workers_hb_lost);
+ }
+ else {
+ rspamd_controller_send_string(conn_ent, "{\"success\":true}");
+ }
+
+ return 0;
+}
+
+/*
+ * Ready command handler:
+ * request: /ready
+ * headers: Password
+ * reply: json {"success":true} or {"error":"error message"}
+ */
+static int
+rspamd_controller_handle_ready(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ if (session->ctx->scanners_count == 0) {
+ rspamd_controller_send_error(conn_ent, 500, "no healthy scanner workers are running");
+ }
+ else {
+ rspamd_controller_send_string(conn_ent, "{\"success\":true}");
+ }
+
+ return 0;
+}
+
+/*
+ * History command handler:
+ * request: /history
+ * headers: Password
+ * reply: json [
+ * { label: "Foo", data: 11 },
+ * { label: "Bar", data: 20 },
+ * {...}
+ * ]
+ */
+static int
+rspamd_controller_handle_history(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_controller_worker_ctx *ctx;
+ lua_State *L;
+ ctx = session->ctx;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ L = ctx->cfg->lua_state;
+
+ if (!ctx->srv->history->disabled) {
+ rspamd_controller_handle_legacy_history(session, ctx, conn_ent, msg);
+ }
+ else {
+ rspamd_controller_handle_lua_history(L, session, ctx, conn_ent, msg,
+ FALSE);
+ }
+
+ return 0;
+}
+
+/*
+ * Errors command handler:
+ * request: /errors
+ * headers: Password
+ * reply: json [
+ * { ts: 100500, type: normal, pid: 100, module: lua, message: bad things },
+ * {...}
+ * ]
+ */
+static int
+rspamd_controller_handle_errors(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_controller_worker_ctx *ctx;
+ ucl_object_t *top;
+
+ ctx = session->ctx;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, TRUE)) {
+ return 0;
+ }
+
+ top = rspamd_log_errorbuf_export(ctx->worker->srv->logger);
+ rspamd_controller_send_ucl(conn_ent, top);
+ ucl_object_unref(top);
+
+ return 0;
+}
+
+/*
+ * Neighbours command handler:
+ * request: /neighbours
+ * headers: Password
+ * reply: json {name: {url: "http://...", host: "host"}}
+ */
+static int
+rspamd_controller_handle_neighbours(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_controller_worker_ctx *ctx;
+
+ ctx = session->ctx;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ rspamd_controller_send_ucl(conn_ent, ctx->cfg->neighbours);
+
+ return 0;
+}
+
+
+static int
+rspamd_controller_handle_history_reset(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_controller_worker_ctx *ctx;
+ struct roll_history_row *row;
+ guint completed_rows, i, t;
+ lua_State *L;
+
+ ctx = session->ctx;
+ L = ctx->cfg->lua_state;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, TRUE)) {
+ return 0;
+ }
+
+ if (!ctx->srv->history->disabled) {
+ /* Clean from start to the current row */
+ completed_rows = g_atomic_int_get(&ctx->srv->history->cur_row);
+
+ completed_rows = MIN(completed_rows, ctx->srv->history->nrows - 1);
+
+ for (i = 0; i <= completed_rows; i++) {
+ t = g_atomic_int_get(&ctx->srv->history->cur_row);
+
+ /* We somehow come to the race condition */
+ if (i > t) {
+ break;
+ }
+
+ row = &ctx->srv->history->rows[i];
+ memset(row, 0, sizeof(*row));
+ }
+
+ msg_info_session("<%s> cleared %d entries from history",
+ rspamd_inet_address_to_string(session->from_addr),
+ completed_rows);
+ rspamd_controller_send_string(conn_ent, "{\"success\":true}");
+ }
+ else {
+ rspamd_controller_handle_lua_history(L, session, ctx, conn_ent, msg,
+ TRUE);
+ }
+
+ return 0;
+}
+
+static gboolean
+rspamd_controller_lua_fin_task(void *ud)
+{
+ struct rspamd_task *task = ud;
+ struct rspamd_http_connection_entry *conn_ent;
+
+ conn_ent = task->fin_arg;
+
+ if (task->err != NULL) {
+ rspamd_controller_send_error(conn_ent, task->err->code, "%s",
+ task->err->message);
+ }
+
+ return TRUE;
+}
+
+static int
+rspamd_controller_handle_lua(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_task *task, **ptask;
+ struct rspamd_http_connection_entry **pconn;
+ struct rspamd_controller_worker_ctx *ctx;
+ gchar filebuf[PATH_MAX], realbuf[PATH_MAX];
+ struct http_parser_url u;
+ rspamd_ftok_t lookup;
+ struct stat st;
+ lua_State *L;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, TRUE)) {
+ return 0;
+ }
+
+ ctx = session->ctx;
+ L = ctx->cfg->lua_state;
+
+ /* Find lua script */
+ if (msg->url != NULL && msg->url->len != 0) {
+
+ http_parser_parse_url(RSPAMD_FSTRING_DATA(msg->url),
+ RSPAMD_FSTRING_LEN(msg->url), TRUE, &u);
+
+ if (u.field_set & (1 << UF_PATH)) {
+ lookup.begin = RSPAMD_FSTRING_DATA(msg->url) +
+ u.field_data[UF_PATH].off;
+ lookup.len = u.field_data[UF_PATH].len;
+ }
+ else {
+ lookup.begin = RSPAMD_FSTRING_DATA(msg->url);
+ lookup.len = RSPAMD_FSTRING_LEN(msg->url);
+ }
+
+ rspamd_snprintf(filebuf, sizeof(filebuf), "%s%c%T",
+ ctx->static_files_dir, G_DIR_SEPARATOR, &lookup);
+
+ if (realpath(filebuf, realbuf) == NULL ||
+ lstat(realbuf, &st) == -1) {
+ rspamd_controller_send_error(conn_ent, 404, "Cannot find path: %s",
+ strerror(errno));
+
+ return 0;
+ }
+
+ /* TODO: add caching here, should be trivial */
+ /* Now we load and execute the code fragment, which should return a function */
+ if (luaL_loadfile(L, realbuf) != 0) {
+ rspamd_controller_send_error(conn_ent, 500, "Cannot load path: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return 0;
+ }
+
+ if (lua_pcall(L, 0, 1, 0) != 0) {
+ rspamd_controller_send_error(conn_ent, 501, "Cannot run path: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return 0;
+ }
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ rspamd_controller_send_error(conn_ent, 502, "Bad return type: %s",
+ lua_typename(L, lua_type(L, -1)));
+ lua_settop(L, 0);
+
+ return 0;
+ }
+ }
+ else {
+ rspamd_controller_send_error(conn_ent, 404, "Empty path is not permitted");
+
+ return 0;
+ }
+
+ task = rspamd_task_new(session->ctx->worker, session->cfg, session->pool,
+ ctx->lang_det, ctx->event_loop, FALSE);
+
+ task->resolver = ctx->resolver;
+ task->s = rspamd_session_create(session->pool,
+ rspamd_controller_lua_fin_task,
+ NULL,
+ (event_finalizer_t) rspamd_task_free,
+ task);
+ task->fin_arg = conn_ent;
+ task->http_conn = rspamd_http_connection_ref(conn_ent->conn);
+ ;
+ task->sock = -1;
+ session->task = task;
+
+ if (msg->body_buf.len > 0) {
+ if (!rspamd_task_load_message(task, msg, msg->body_buf.begin, msg->body_buf.len)) {
+ rspamd_controller_send_error(conn_ent, task->err->code, "%s",
+ task->err->message);
+ return 0;
+ }
+ }
+
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+
+ pconn = lua_newuserdata(L, sizeof(*pconn));
+ rspamd_lua_setclass(L, "rspamd{csession}", -1);
+ *pconn = conn_ent;
+
+ if (lua_pcall(L, 2, 0, 0) != 0) {
+ rspamd_controller_send_error(conn_ent, 503, "Cannot run callback: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return 0;
+ }
+
+ rspamd_session_pending(task->s);
+
+ return 0;
+}
+
+static gboolean
+rspamd_controller_learn_fin_task(void *ud)
+{
+ struct rspamd_task *task = ud;
+ struct rspamd_controller_session *session;
+ struct rspamd_http_connection_entry *conn_ent;
+
+ conn_ent = task->fin_arg;
+ session = conn_ent->ud;
+
+ if (task->err != NULL) {
+ msg_info_session("cannot learn <%s>: %e",
+ MESSAGE_FIELD(task, message_id), task->err);
+ rspamd_controller_send_error(conn_ent, task->err->code, "%s",
+ task->err->message);
+
+ return TRUE;
+ }
+
+ if (RSPAMD_TASK_IS_PROCESSED(task)) {
+ /* Successful learn */
+ msg_info_task("<%s> learned message as %s: %s",
+ rspamd_inet_address_to_string(session->from_addr),
+ session->is_spam ? "spam" : "ham",
+ MESSAGE_FIELD(task, message_id));
+ rspamd_controller_send_string(conn_ent, "{\"success\":true}");
+ return TRUE;
+ }
+
+ if (!rspamd_task_process(task, RSPAMD_TASK_PROCESS_LEARN)) {
+ msg_info_task("cannot learn <%s>: %e",
+ MESSAGE_FIELD(task, message_id), task->err);
+
+ if (task->err) {
+ rspamd_controller_send_error(conn_ent, task->err->code, "%s",
+ task->err->message);
+ }
+ else {
+ rspamd_controller_send_error(conn_ent, 500,
+ "Internal error");
+ }
+ }
+
+ if (RSPAMD_TASK_IS_PROCESSED(task)) {
+ if (task->err) {
+ rspamd_controller_send_error(conn_ent, task->err->code, "%s",
+ task->err->message);
+ }
+ else {
+ msg_info_task("<%s> learned message as %s: %s",
+ rspamd_inet_address_to_string(session->from_addr),
+ session->is_spam ? "spam" : "ham",
+ MESSAGE_FIELD(task, message_id));
+ rspamd_controller_send_string(conn_ent, "{\"success\":true}");
+ }
+
+ return TRUE;
+ }
+
+ /* One more iteration */
+ return FALSE;
+}
+
+static void
+rspamd_controller_scan_reply(struct rspamd_task *task)
+{
+ struct rspamd_http_message *msg;
+ struct rspamd_http_connection_entry *conn_ent;
+
+ conn_ent = task->fin_arg;
+ msg = rspamd_http_new_message(HTTP_RESPONSE);
+ msg->date = time(NULL);
+ msg->code = 200;
+ rspamd_protocol_http_reply(msg, task, NULL);
+ rspamd_http_connection_reset(conn_ent->conn);
+ rspamd_http_router_insert_headers(conn_ent->rt, msg);
+ rspamd_http_connection_write_message(conn_ent->conn, msg, NULL,
+ "application/json", conn_ent, conn_ent->rt->timeout);
+ conn_ent->is_reply = TRUE;
+}
+
+static gboolean
+rspamd_controller_check_fin_task(void *ud)
+{
+ struct rspamd_task *task = ud;
+ struct rspamd_http_connection_entry *conn_ent;
+
+ msg_debug_task("finish task");
+ conn_ent = task->fin_arg;
+
+ if (task->err) {
+ msg_info_task("cannot check <%s>: %e",
+ MESSAGE_FIELD_CHECK(task, message_id), task->err);
+ rspamd_controller_send_error(conn_ent, task->err->code, "%s",
+ task->err->message);
+ return TRUE;
+ }
+
+ if (RSPAMD_TASK_IS_PROCESSED(task)) {
+ rspamd_controller_scan_reply(task);
+ return TRUE;
+ }
+
+ if (!rspamd_task_process(task, RSPAMD_TASK_PROCESS_ALL)) {
+ rspamd_controller_scan_reply(task);
+ return TRUE;
+ }
+
+ if (RSPAMD_TASK_IS_PROCESSED(task)) {
+ rspamd_controller_scan_reply(task);
+ return TRUE;
+ }
+
+ /* One more iteration */
+ return FALSE;
+}
+
+static int
+rspamd_controller_handle_learn_common(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg,
+ gboolean is_spam)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_controller_worker_ctx *ctx;
+ struct rspamd_task *task;
+ const rspamd_ftok_t *cl_header;
+
+ ctx = session->ctx;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, TRUE)) {
+ return 0;
+ }
+
+ if (rspamd_http_message_get_body(msg, NULL) == NULL) {
+ msg_err_session("got zero length body, cannot continue");
+ rspamd_controller_send_error(conn_ent,
+ 400,
+ "Empty body is not permitted");
+ return 0;
+ }
+
+ task = rspamd_task_new(session->ctx->worker, session->cfg, session->pool,
+ session->ctx->lang_det, ctx->event_loop, FALSE);
+
+ task->resolver = ctx->resolver;
+ task->s = rspamd_session_create(session->pool,
+ rspamd_controller_learn_fin_task,
+ NULL,
+ (event_finalizer_t) rspamd_task_free,
+ task);
+ task->fin_arg = conn_ent;
+ task->http_conn = rspamd_http_connection_ref(conn_ent->conn);
+ task->sock = -1;
+ session->task = task;
+
+ cl_header = rspamd_http_message_find_header(msg, "classifier");
+ if (cl_header) {
+ session->classifier = rspamd_mempool_ftokdup(session->pool, cl_header);
+ }
+ else {
+ session->classifier = NULL;
+ }
+
+ if (!rspamd_task_load_message(task, msg, msg->body_buf.begin, msg->body_buf.len)) {
+ goto end;
+ }
+
+ rspamd_learn_task_spam(task, is_spam, session->classifier, NULL);
+
+ if (!rspamd_task_process(task, RSPAMD_TASK_PROCESS_LEARN)) {
+ msg_warn_session("<%s> message cannot be processed",
+ MESSAGE_FIELD(task, message_id));
+ goto end;
+ }
+
+end:
+ session->is_spam = is_spam;
+ rspamd_session_pending(task->s);
+
+ return 0;
+}
+
+/*
+ * Learn spam command handler:
+ * request: /learnspam
+ * headers: Password
+ * input: plaintext data
+ * reply: json {"success":true} or {"error":"error message"}
+ */
+static int
+rspamd_controller_handle_learnspam(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ return rspamd_controller_handle_learn_common(conn_ent, msg, TRUE);
+}
+/*
+ * Learn ham command handler:
+ * request: /learnham
+ * headers: Password
+ * input: plaintext data
+ * reply: json {"success":true} or {"error":"error message"}
+ */
+static int
+rspamd_controller_handle_learnham(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ return rspamd_controller_handle_learn_common(conn_ent, msg, FALSE);
+}
+
+/*
+ * Scan command handler:
+ * request: /scan
+ * headers: Password
+ * input: plaintext data
+ * reply: json {scan data} or {"error":"error message"}
+ */
+static int
+rspamd_controller_handle_scan(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_controller_worker_ctx *ctx;
+ struct rspamd_task *task;
+
+ ctx = session->ctx;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ task = rspamd_task_new(session->ctx->worker, session->cfg, session->pool,
+ ctx->lang_det, ctx->event_loop, FALSE);
+
+ task->resolver = ctx->resolver;
+ task->s = rspamd_session_create(session->pool,
+ rspamd_controller_check_fin_task,
+ NULL,
+ (event_finalizer_t) rspamd_task_free,
+ task);
+ task->fin_arg = conn_ent;
+ task->http_conn = rspamd_http_connection_ref(conn_ent->conn);
+ task->sock = conn_ent->conn->fd;
+ task->flags |= RSPAMD_TASK_FLAG_MIME;
+ task->resolver = ctx->resolver;
+
+ if (!rspamd_protocol_handle_request(task, msg)) {
+ task->flags |= RSPAMD_TASK_FLAG_SKIP;
+ goto end;
+ }
+
+ if (!rspamd_task_load_message(task, msg, msg->body_buf.begin, msg->body_buf.len)) {
+ goto end;
+ }
+
+ if (!rspamd_task_process(task, RSPAMD_TASK_PROCESS_ALL)) {
+ goto end;
+ }
+
+ if (!isnan(ctx->task_timeout) && ctx->task_timeout > 0.0) {
+ task->timeout_ev.data = task;
+ ev_timer_init(&task->timeout_ev, rspamd_task_timeout,
+ ctx->task_timeout, ctx->task_timeout);
+ ev_timer_start(task->event_loop, &task->timeout_ev);
+ ev_set_priority(&task->timeout_ev, EV_MAXPRI);
+ }
+
+end:
+ session->task = task;
+ rspamd_session_pending(task->s);
+
+ return 0;
+}
+
+/*
+ * Save actions command handler:
+ * request: /saveactions
+ * headers: Password
+ * input: json array [<spam>,<probable spam>,<greylist>]
+ * reply: json {"success":true} or {"error":"error message"}
+ */
+static int
+rspamd_controller_handle_saveactions(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct ucl_parser *parser;
+ ucl_object_t *obj;
+ const ucl_object_t *cur;
+ struct rspamd_controller_worker_ctx *ctx;
+ const gchar *error;
+ gdouble score;
+ gint i, added = 0;
+ enum rspamd_action_type act;
+ ucl_object_iter_t it = NULL;
+
+ ctx = session->ctx;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, TRUE)) {
+ return 0;
+ }
+
+ if (rspamd_http_message_get_body(msg, NULL) == NULL) {
+ msg_err_session("got zero length body, cannot continue");
+ rspamd_controller_send_error(conn_ent,
+ 400,
+ "Empty body is not permitted");
+ return 0;
+ }
+
+ parser = ucl_parser_new(0);
+ if (!ucl_parser_add_chunk(parser, msg->body_buf.begin, msg->body_buf.len)) {
+ if ((error = ucl_parser_get_error(parser)) != NULL) {
+ msg_err_session("cannot parse input: %s", error);
+ rspamd_controller_send_error(conn_ent, 400, "Cannot parse input");
+ ucl_parser_free(parser);
+ return 0;
+ }
+
+ msg_err_session("cannot parse input: unknown error");
+ rspamd_controller_send_error(conn_ent, 400, "Cannot parse input");
+ ucl_parser_free(parser);
+ return 0;
+ }
+
+ obj = ucl_parser_get_object(parser);
+ ucl_parser_free(parser);
+
+ if (obj->type != UCL_ARRAY || obj->len != 4) {
+ msg_err_session("input is not an array of 4 elements");
+ rspamd_controller_send_error(conn_ent, 400, "Cannot parse input");
+ ucl_object_unref(obj);
+ return 0;
+ }
+
+ it = ucl_object_iterate_new(obj);
+
+ for (i = 0; i < 4; i++) {
+ cur = ucl_object_iterate_safe(it, TRUE);
+
+ switch (i) {
+ case 0:
+ default:
+ act = METRIC_ACTION_REJECT;
+ break;
+ case 1:
+ act = METRIC_ACTION_REWRITE_SUBJECT;
+ break;
+ case 2:
+ act = METRIC_ACTION_ADD_HEADER;
+ break;
+ case 3:
+ act = METRIC_ACTION_GREYLIST;
+ break;
+ }
+
+ if (ucl_object_type(cur) == UCL_NULL) {
+ /* Assume NaN */
+ score = NAN;
+ }
+ else {
+ score = ucl_object_todouble(cur);
+ }
+
+ struct rspamd_action *cfg_action = rspamd_config_get_action_by_type(ctx->cfg, act);
+
+ if (cfg_action && ((isnan(cfg_action->threshold) != isnan(score)) ||
+ (cfg_action->threshold != score))) {
+ add_dynamic_action(ctx->cfg, DEFAULT_METRIC, act, score);
+ added++;
+ }
+ else {
+ if (remove_dynamic_action(ctx->cfg, DEFAULT_METRIC, act)) {
+ added++;
+ }
+ }
+ }
+
+ ucl_object_iterate_free(it);
+
+ if (dump_dynamic_config(ctx->cfg)) {
+ msg_info_session("<%s> modified %d actions",
+ rspamd_inet_address_to_string(session->from_addr),
+ added);
+
+ rspamd_controller_send_string(conn_ent, "{\"success\":true}");
+ }
+ else {
+ rspamd_controller_send_error(conn_ent, 500, "Save error");
+ }
+
+ ucl_object_unref(obj);
+
+ return 0;
+}
+
+/*
+ * Save symbols command handler:
+ * request: /savesymbols
+ * headers: Password
+ * input: json data
+ * reply: json {"success":true} or {"error":"error message"}
+ */
+static int
+rspamd_controller_handle_savesymbols(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct ucl_parser *parser;
+ ucl_object_t *obj;
+ const ucl_object_t *cur, *jname, *jvalue;
+ ucl_object_iter_t iter = NULL;
+ struct rspamd_controller_worker_ctx *ctx;
+ const gchar *error;
+ gdouble val;
+ struct rspamd_symbol *sym;
+ int added = 0;
+
+ ctx = session->ctx;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, TRUE)) {
+ return 0;
+ }
+
+ if (rspamd_http_message_get_body(msg, NULL) == NULL) {
+ msg_err_session("got zero length body, cannot continue");
+ rspamd_controller_send_error(conn_ent,
+ 400,
+ "Empty body is not permitted");
+ return 0;
+ }
+
+ parser = ucl_parser_new(0);
+ if (!ucl_parser_add_chunk(parser, msg->body_buf.begin, msg->body_buf.len)) {
+ if ((error = ucl_parser_get_error(parser)) != NULL) {
+ msg_err_session("cannot parse input: %s", error);
+ rspamd_controller_send_error(conn_ent, 400, "Cannot parse input");
+ ucl_parser_free(parser);
+ return 0;
+ }
+
+ msg_err_session("cannot parse input: unknown error");
+ rspamd_controller_send_error(conn_ent, 400, "Cannot parse input");
+ ucl_parser_free(parser);
+ return 0;
+ }
+
+ obj = ucl_parser_get_object(parser);
+ ucl_parser_free(parser);
+
+ if (obj->type != UCL_ARRAY) {
+ msg_err_session("input is not an array");
+ rspamd_controller_send_error(conn_ent, 400, "Cannot parse input");
+ ucl_object_unref(obj);
+ return 0;
+ }
+
+ iter = ucl_object_iterate_new(obj);
+
+ while ((cur = ucl_object_iterate_safe(iter, true))) {
+ if (cur->type != UCL_OBJECT) {
+ msg_err_session("json array data error");
+ rspamd_controller_send_error(conn_ent, 400, "Cannot parse input");
+ ucl_object_unref(obj);
+ ucl_object_iterate_free(iter);
+
+ return 0;
+ }
+
+ jname = ucl_object_lookup(cur, "name");
+ jvalue = ucl_object_lookup(cur, "value");
+ val = ucl_object_todouble(jvalue);
+ sym = g_hash_table_lookup(session->cfg->symbols, ucl_object_tostring(jname));
+
+ if (sym && fabs(*sym->weight_ptr - val) > DBL_EPSILON) {
+ if (!add_dynamic_symbol(ctx->cfg, DEFAULT_METRIC,
+ ucl_object_tostring(jname), val)) {
+ msg_err_session("add symbol failed for %s",
+ ucl_object_tostring(jname));
+ rspamd_controller_send_error(conn_ent, 506,
+ "Add symbol failed");
+ ucl_object_unref(obj);
+ ucl_object_iterate_free(iter);
+
+ return 0;
+ }
+ added++;
+ }
+ else if (sym && ctx->cfg->dynamic_conf) {
+ if (remove_dynamic_symbol(ctx->cfg, DEFAULT_METRIC,
+ ucl_object_tostring(jname))) {
+ added++;
+ }
+ }
+ }
+
+ ucl_object_iterate_free(iter);
+
+ if (added > 0) {
+ if (ctx->cfg->dynamic_conf) {
+ if (dump_dynamic_config(ctx->cfg)) {
+ msg_info_session("<%s> modified %d symbols",
+ rspamd_inet_address_to_string(session->from_addr),
+ added);
+
+ rspamd_controller_send_string(conn_ent, "{\"success\":true}");
+ }
+ else {
+ rspamd_controller_send_error(conn_ent, 500, "Save error");
+ }
+ }
+ else {
+ rspamd_controller_send_string(conn_ent, "{\"success\":true}");
+ }
+ }
+ else {
+ msg_err_session("no symbols to save");
+ rspamd_controller_send_error(conn_ent, 404, "No symbols to save");
+ }
+
+ ucl_object_unref(obj);
+
+ return 0;
+}
+
+/*
+ * Save map command handler:
+ * request: /savemap
+ * headers: Password, Map
+ * input: plaintext data
+ * reply: json {"success":true} or {"error":"error message"}
+ */
+static int
+rspamd_controller_handle_savemap(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ GList *cur;
+ struct rspamd_map *map = NULL;
+ struct rspamd_map_backend *bk;
+ struct rspamd_controller_worker_ctx *ctx;
+ const rspamd_ftok_t *idstr;
+ gulong id, i;
+ gboolean found = FALSE;
+ gchar tempname[PATH_MAX];
+ gint fd;
+
+ ctx = session->ctx;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, TRUE)) {
+ return 0;
+ }
+
+ if (rspamd_http_message_get_body(msg, NULL) == NULL) {
+ msg_err_session("got zero length body, cannot continue");
+ rspamd_controller_send_error(conn_ent,
+ 400,
+ "Empty body is not permitted");
+ return 0;
+ }
+
+ idstr = rspamd_http_message_find_header(msg, "Map");
+
+ if (idstr == NULL) {
+ msg_info_session("absent map id");
+ rspamd_controller_send_error(conn_ent, 400, "Map id not specified");
+ return 0;
+ }
+
+ if (!rspamd_strtoul(idstr->begin, idstr->len, &id)) {
+ msg_info_session("invalid map id: %T", idstr);
+ rspamd_controller_send_error(conn_ent, 400, "Map id is invalid");
+ return 0;
+ }
+
+ /* Now let's be sure that we have map defined in configuration */
+ cur = ctx->cfg->maps;
+ while (cur && !found) {
+ map = cur->data;
+
+ PTR_ARRAY_FOREACH(map->backends, i, bk)
+ {
+ if (bk->id == id && bk->protocol == MAP_PROTO_FILE) {
+ found = TRUE;
+ break;
+ }
+ }
+ cur = g_list_next(cur);
+ }
+
+ if (!found) {
+ msg_info_session("map not found: %L", id);
+ rspamd_controller_send_error(conn_ent, 404, "Map id not found");
+ return 0;
+ }
+
+ rspamd_snprintf(tempname, sizeof(tempname), "%s.newXXXXXX", bk->uri);
+ fd = g_mkstemp_full(tempname, O_WRONLY, 00644);
+
+ if (fd == -1) {
+ msg_info_session("map %s open error: %s", tempname, strerror(errno));
+ rspamd_controller_send_error(conn_ent, 404,
+ "Cannot open map: %s",
+ strerror(errno));
+ return 0;
+ }
+
+ if (write(fd, msg->body_buf.begin, msg->body_buf.len) == -1) {
+ msg_info_session("map %s write error: %s", tempname, strerror(errno));
+ unlink(tempname);
+ close(fd);
+ rspamd_controller_send_error(conn_ent, 500, "Map write error: %s",
+ strerror(errno));
+ return 0;
+ }
+
+ /* Rename */
+ if (rename(tempname, bk->uri) == -1) {
+ msg_info_session("map %s rename error: %s", tempname, strerror(errno));
+ unlink(tempname);
+ close(fd);
+ rspamd_controller_send_error(conn_ent, 500, "Map rename error: %s",
+ strerror(errno));
+ return 0;
+ }
+
+ msg_info_session("<%s>, map %s saved",
+ rspamd_inet_address_to_string(session->from_addr),
+ bk->uri);
+ close(fd);
+
+ rspamd_controller_send_string(conn_ent, "{\"success\":true}");
+
+ return 0;
+}
+
+struct rspamd_stat_cbdata {
+ struct rspamd_http_connection_entry *conn_ent;
+ struct rspamd_controller_worker_ctx *ctx;
+ ucl_object_t *top;
+ ucl_object_t *stat;
+ struct rspamd_task *task;
+ guint64 learned;
+};
+
+static gboolean
+rspamd_controller_stat_fin_task(void *ud)
+{
+ struct rspamd_stat_cbdata *cbdata = ud;
+ struct rspamd_http_connection_entry *conn_ent;
+ ucl_object_t *top, *ar;
+ struct rspamd_fuzzy_stat_entry *entry;
+
+ conn_ent = cbdata->conn_ent;
+ top = cbdata->top;
+
+ ucl_object_insert_key(top,
+ ucl_object_fromint(cbdata->learned), "total_learns", 0, false);
+
+ if (cbdata->stat) {
+ ucl_object_insert_key(top, cbdata->stat, "statfiles", 0, false);
+ }
+
+ GHashTable *fuzzy_elts = rspamd_mempool_get_variable(cbdata->task->task_pool, RSPAMD_MEMPOOL_FUZZY_STAT);
+
+ if (fuzzy_elts) {
+ ar = ucl_object_typed_new(UCL_OBJECT);
+
+ GHashTableIter it;
+
+ g_hash_table_iter_init(&it, fuzzy_elts);
+ while (g_hash_table_iter_next(&it, NULL, (gpointer *) &entry)) {
+ if (entry->name) {
+ ucl_object_insert_key(ar, ucl_object_fromint(entry->fuzzy_cnt),
+ entry->name, 0, true);
+ }
+ }
+
+ ucl_object_insert_key(top, ar, "fuzzy_hashes", 0, false);
+ }
+
+ rspamd_controller_send_ucl(conn_ent, top);
+
+
+ return TRUE;
+}
+
+static void
+rspamd_controller_stat_cleanup_task(void *ud)
+{
+ struct rspamd_stat_cbdata *cbdata = ud;
+
+ rspamd_task_free(cbdata->task);
+ ucl_object_unref(cbdata->top);
+}
+
+/*
+ * Stat command handler:
+ * request: /stat (/resetstat)
+ * headers: Password
+ * reply: json data
+ */
+static int
+rspamd_controller_handle_stat_common(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg,
+ gboolean do_reset)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ ucl_object_t *top, *sub;
+ gint i;
+ int64_t uptime;
+ guint64 spam = 0, ham = 0;
+ rspamd_mempool_stat_t mem_st;
+ struct rspamd_stat *stat, stat_copy;
+ struct rspamd_controller_worker_ctx *ctx;
+ struct rspamd_task *task;
+ struct rspamd_stat_cbdata *cbdata;
+
+ memset(&mem_st, 0, sizeof(mem_st));
+ rspamd_mempool_stat(&mem_st);
+ memcpy(&stat_copy, session->ctx->worker->srv->stat, sizeof(stat_copy));
+ stat = &stat_copy;
+ ctx = session->ctx;
+
+ task = rspamd_task_new(session->ctx->worker, session->cfg, session->pool,
+ ctx->lang_det, ctx->event_loop, FALSE);
+ task->resolver = ctx->resolver;
+ cbdata = rspamd_mempool_alloc0(session->pool, sizeof(*cbdata));
+ cbdata->conn_ent = conn_ent;
+ cbdata->task = task;
+ cbdata->ctx = ctx;
+ top = ucl_object_typed_new(UCL_OBJECT);
+ cbdata->top = top;
+
+ task->s = rspamd_session_create(session->pool,
+ rspamd_controller_stat_fin_task,
+ NULL,
+ rspamd_controller_stat_cleanup_task,
+ cbdata);
+ task->fin_arg = cbdata;
+ task->http_conn = rspamd_http_connection_ref(conn_ent->conn);
+ ;
+ task->sock = conn_ent->conn->fd;
+
+ ucl_object_insert_key(top, ucl_object_fromstring(RVERSION), "version", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromstring(session->ctx->cfg->checksum), "config_id", 0, false);
+ uptime = ev_time() - session->ctx->start_time;
+ ucl_object_insert_key(top, ucl_object_fromint(uptime), "uptime", 0, false);
+ ucl_object_insert_key(top, ucl_object_frombool(session->is_read_only),
+ "read_only", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(stat->messages_scanned), "scanned", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(stat->messages_learned), "learned", 0, false);
+
+ sub = ucl_object_typed_new(UCL_OBJECT);
+ for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
+ ucl_object_insert_key(sub,
+ ucl_object_fromint(stat->actions_stat[i]),
+ rspamd_action_to_str(i), 0, false);
+ if (i < METRIC_ACTION_GREYLIST) {
+ spam += stat->actions_stat[i];
+ }
+ else {
+ ham += stat->actions_stat[i];
+ }
+ if (do_reset) {
+#ifndef HAVE_ATOMIC_BUILTINS
+ session->ctx->worker->srv->stat->actions_stat[i] = 0;
+#else
+ __atomic_store_n(&session->ctx->worker->srv->stat->actions_stat[i],
+ 0, __ATOMIC_RELEASE);
+#endif
+ }
+ }
+ ucl_object_insert_key(top, sub, "actions", 0, false);
+
+ sub = ucl_object_typed_new(UCL_ARRAY);
+ for (i = 0; i < MAX_AVG_TIME_SLOTS; i++) {
+ ucl_array_append(sub, ucl_object_fromdouble(stat->avg_time.avg_time[i]));
+ }
+ ucl_object_insert_key(top, sub, "scan_times", 0, false);
+
+ ucl_object_insert_key(top, ucl_object_fromint(spam), "spam_count", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(ham), "ham_count", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(stat->connections_count), "connections", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(stat->control_connections_count),
+ "control_connections", 0, false);
+
+
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.pools_allocated), "pools_allocated", 0,
+ false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.pools_freed), "pools_freed", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.bytes_allocated), "bytes_allocated", 0,
+ false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(
+ mem_st.chunks_allocated),
+ "chunks_allocated", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.shared_chunks_allocated),
+ "shared_chunks_allocated", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.chunks_freed), "chunks_freed", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(
+ mem_st.oversized_chunks),
+ "chunks_oversized", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.fragmented_size), "fragmented", 0, false);
+
+ if (do_reset) {
+ session->ctx->srv->stat->messages_scanned = 0;
+ session->ctx->srv->stat->messages_learned = 0;
+ session->ctx->srv->stat->connections_count = 0;
+ session->ctx->srv->stat->control_connections_count = 0;
+ rspamd_mempool_stat_reset();
+ }
+
+ fuzzy_stat_command(task);
+
+ /* Now write statistics for each statfile */
+ rspamd_stat_statistics(task, session->ctx->cfg, &cbdata->learned,
+ &cbdata->stat);
+ session->task = task;
+ rspamd_session_pending(task->s);
+
+ return 0;
+}
+
+static int
+rspamd_controller_handle_stat(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ return rspamd_controller_handle_stat_common(conn_ent, msg, FALSE);
+}
+
+static int
+rspamd_controller_handle_statreset(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, TRUE)) {
+ return 0;
+ }
+
+ msg_info_session("<%s> reset stat",
+ rspamd_inet_address_to_string(session->from_addr));
+ return rspamd_controller_handle_stat_common(conn_ent, msg, TRUE);
+}
+
+static inline void
+rspamd_controller_metrics_add_integer(rspamd_fstring_t **output,
+ const ucl_object_t *top,
+ const char *name,
+ const char *type,
+ const char *description,
+ const char *ucl_key)
+{
+ rspamd_printf_fstring(output, "# HELP %s %s\n", name, description);
+ rspamd_printf_fstring(output, "# TYPE %s %s\n", name, type);
+ rspamd_printf_fstring(output, "%s %L\n", name,
+ ucl_object_toint(ucl_object_lookup(top, ucl_key)));
+}
+
+/*
+ * Metrics command handler:
+ * request: /metrics
+ * headers: Password
+ * reply: OpenMetrics
+ */
+static gboolean
+rspamd_controller_metrics_fin_task(void *ud)
+{
+ struct rspamd_stat_cbdata *cbdata = ud;
+ struct rspamd_http_connection_entry *conn_ent;
+ ucl_object_t *top;
+ struct rspamd_fuzzy_stat_entry *entry;
+ rspamd_fstring_t *output;
+ gint i;
+
+ conn_ent = cbdata->conn_ent;
+ top = cbdata->top;
+
+ output = rspamd_fstring_sized_new(1024);
+ rspamd_printf_fstring(&output, "# HELP rspamd_build_info A metric with a constant '1' value "
+ "labeled by version from which rspamd was built.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_build_info gauge\n");
+ rspamd_printf_fstring(&output, "rspamd_build_info{version=\"%s\"} 1\n",
+ ucl_object_tostring(ucl_object_lookup(top, "version")));
+ rspamd_printf_fstring(&output, "# HELP rspamd_config A metric with a constant '1' value "
+ "labeled by id of the current config.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_config gauge\n");
+ rspamd_printf_fstring(&output, "rspamd_config{id=\"%s\"} 1\n",
+ ucl_object_tostring(ucl_object_lookup(top, "config_id")));
+
+ gsize cnt = MAX_AVG_TIME_SLOTS;
+ float sum = rspamd_sum_floats(cbdata->ctx->worker->srv->stat->avg_time.avg_time, &cnt);
+ rspamd_printf_fstring(&output, "# HELP rspamd_scan_time_average Average messages scan time.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_scan_time_average gauge\n");
+ rspamd_printf_fstring(&output, "rspamd_scan_time_average %f\n",
+ cnt > 0 ? (double) sum / cnt : 0.0);
+
+ rspamd_controller_metrics_add_integer(&output, top,
+ "process_start_time_seconds",
+ "gauge",
+ "Start time of the process since unix epoch in seconds.",
+ "start_time");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_read_only",
+ "gauge",
+ "Whether the rspamd instance is read-only.",
+ "read_only");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_scanned_total",
+ "counter",
+ "Scanned messages.",
+ "scanned");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_learned_total",
+ "counter",
+ "Learned messages.",
+ "learned");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_spam_total",
+ "counter",
+ "Messages classified as spam.",
+ "spam_count");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_ham_total",
+ "counter",
+ "Messages classified as spam.",
+ "ham_count");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_connections",
+ "gauge",
+ "Active connections.",
+ "connections");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_control_connections_total",
+ "gauge",
+ "Control connections.",
+ "control_connections");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_pools_allocated",
+ "gauge",
+ "Pools allocated.",
+ "pools_allocated");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_pools_freed",
+ "gauge",
+ "Pools freed.",
+ "pools_freed");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_allocated_bytes",
+ "gauge",
+ "Bytes allocated.",
+ "bytes_allocated");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_chunks_allocated",
+ "gauge",
+ "Memory pools: current chunks allocated.",
+ "chunks_allocated");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_shared_chunks_allocated",
+ "gauge",
+ "Memory pools: current shared chunks allocated.",
+ "shared_chunks_allocated");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_chunks_freed",
+ "gauge",
+ "Memory pools: current chunks freed.",
+ "chunks_freed");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_chunks_oversized",
+ "gauge",
+ "Memory pools: current chunks oversized (needs extra allocation/fragmentation).",
+ "chunks_oversized");
+ rspamd_controller_metrics_add_integer(&output, top,
+ "rspamd_fragmented",
+ "gauge",
+ "Memory pools: fragmented memory waste.",
+ "fragmented");
+
+ rspamd_printf_fstring(&output, "# HELP rspamd_learns_total Total learns.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_learns_total counter\n");
+ rspamd_printf_fstring(&output, "rspamd_learns_total %L\n", cbdata->learned);
+
+ const ucl_object_t *acts_obj = ucl_object_lookup(top, "actions");
+
+ if (acts_obj) {
+ rspamd_printf_fstring(&output, "# HELP rspamd_actions_total Actions labelled by action type.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_actions_total counter\n");
+ for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
+ const char *str_act = rspamd_action_to_str(i);
+ const ucl_object_t *act = ucl_object_lookup(acts_obj, str_act);
+
+ if (act) {
+ rspamd_printf_fstring(&output, "rspamd_actions_total{type=\"%s\"} %L\n",
+ str_act,
+ ucl_object_toint(act));
+ }
+ else {
+ rspamd_printf_fstring(&output, "rspamd_actions_total{type=\"%s\"} 0\n",
+ str_act);
+ }
+ }
+ }
+
+ if (cbdata->stat) {
+ const ucl_object_t *cur_elt;
+ ucl_object_iter_t it = NULL;
+ rspamd_fstring_t *revision;
+ rspamd_fstring_t *used;
+ rspamd_fstring_t *total;
+ rspamd_fstring_t *size;
+ rspamd_fstring_t *languages;
+ rspamd_fstring_t *users;
+
+ revision = rspamd_fstring_sized_new(16);
+ used = rspamd_fstring_sized_new(16);
+ total = rspamd_fstring_sized_new(16);
+ size = rspamd_fstring_sized_new(16);
+ languages = rspamd_fstring_sized_new(16);
+ users = rspamd_fstring_sized_new(16);
+
+ while ((cur_elt = ucl_object_iterate(cbdata->stat, &it, true))) {
+ const char *sym = ucl_object_tostring(ucl_object_lookup(cur_elt, "symbol"));
+ const char *type = ucl_object_tostring(ucl_object_lookup(cur_elt, "type"));
+
+ if (sym && type) {
+ rspamd_printf_fstring(&revision, "rspamd_statfiles_revision{symbol=\"%s\",type=\"%s\"} %L\n",
+ sym,
+ type,
+ ucl_object_toint(ucl_object_lookup(cur_elt, "revision")));
+ rspamd_printf_fstring(&used, "rspamd_statfiles_used{symbol=\"%s\",type=\"%s\"} %L\n",
+ sym,
+ type,
+ ucl_object_toint(ucl_object_lookup(cur_elt, "used")));
+ rspamd_printf_fstring(&total, "rspamd_statfiles_totals{symbol=\"%s\",type=\"%s\"} %L\n",
+ sym,
+ type,
+ ucl_object_toint(ucl_object_lookup(cur_elt, "total")));
+ rspamd_printf_fstring(&size, "rspamd_statfiles_size{symbol=\"%s\",type=\"%s\"} %L\n",
+ sym,
+ type,
+ ucl_object_toint(ucl_object_lookup(cur_elt, "size")));
+ rspamd_printf_fstring(&languages, "rspamd_statfiles_languages{symbol=\"%s\",type=\"%s\"} %L\n",
+ sym,
+ type,
+ ucl_object_toint(ucl_object_lookup(cur_elt, "languages")));
+ rspamd_printf_fstring(&users, "rspamd_statfiles_users{symbol=\"%s\",type=\"%s\"} %L\n",
+ sym,
+ type,
+ ucl_object_toint(ucl_object_lookup(cur_elt, "users")));
+ }
+ }
+
+ if (RSPAMD_FSTRING_LEN(revision) > 0) {
+ rspamd_printf_fstring(&output, "# HELP rspamd_statfiles_revision Stat files revision.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_statfiles_revision gauge\n");
+ output = rspamd_fstring_append(output,
+ RSPAMD_FSTRING_DATA(revision), RSPAMD_FSTRING_LEN(revision));
+ }
+ if (RSPAMD_FSTRING_LEN(used) > 0) {
+ rspamd_printf_fstring(&output, "# HELP rspamd_statfiles_used Stat files used.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_statfiles_used gauge\n");
+ output = rspamd_fstring_append(output,
+ RSPAMD_FSTRING_DATA(used), RSPAMD_FSTRING_LEN(used));
+ }
+ if (RSPAMD_FSTRING_LEN(total) > 0) {
+ rspamd_printf_fstring(&output, "# HELP rspamd_statfiles_totals Stat files total.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_statfiles_totals gauge\n");
+ output = rspamd_fstring_append(output,
+ RSPAMD_FSTRING_DATA(total), RSPAMD_FSTRING_LEN(total));
+ }
+ if (RSPAMD_FSTRING_LEN(size) > 0) {
+ rspamd_printf_fstring(&output, "# HELP rspamd_statfiles_size Stat files size.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_statfiles_size gauge\n");
+ output = rspamd_fstring_append(output,
+ RSPAMD_FSTRING_DATA(size), RSPAMD_FSTRING_LEN(size));
+ }
+ if (RSPAMD_FSTRING_LEN(languages) > 0) {
+ rspamd_printf_fstring(&output, "# HELP rspamd_statfiles_languages Stat files languages.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_statfiles_languages gauge\n");
+ output = rspamd_fstring_append(output,
+ RSPAMD_FSTRING_DATA(languages), RSPAMD_FSTRING_LEN(languages));
+ }
+ if (RSPAMD_FSTRING_LEN(users) > 0) {
+ rspamd_printf_fstring(&output, "# HELP rspamd_statfiles_users Stat files users.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_statfiles_users gauge\n");
+ output = rspamd_fstring_append(output,
+ RSPAMD_FSTRING_DATA(users), RSPAMD_FSTRING_LEN(users));
+ }
+
+ rspamd_fstring_free(revision);
+ rspamd_fstring_free(used);
+ rspamd_fstring_free(total);
+ rspamd_fstring_free(size);
+ rspamd_fstring_free(languages);
+ rspamd_fstring_free(users);
+ }
+
+ GHashTable *fuzzy_elts = rspamd_mempool_get_variable(cbdata->task->task_pool, RSPAMD_MEMPOOL_FUZZY_STAT);
+
+ if (fuzzy_elts) {
+ rspamd_printf_fstring(&output, "# HELP rspamd_fuzzy_stat Fuzzy stat labelled by storage.\n");
+ rspamd_printf_fstring(&output, "# TYPE rspamd_fuzzy_stat gauge\n");
+
+ GHashTableIter it;
+
+ g_hash_table_iter_init(&it, fuzzy_elts);
+ while (g_hash_table_iter_next(&it, NULL, (gpointer *) &entry)) {
+ if (entry->name) {
+ rspamd_printf_fstring(&output, "rspamd_fuzzy_stat{storage=\"%s\"} %uL\n",
+ entry->name, entry->fuzzy_cnt);
+ }
+ }
+ }
+
+ rspamd_printf_fstring(&output, "# EOF\n");
+
+ rspamd_controller_send_openmetrics(conn_ent, output);
+
+ return TRUE;
+}
+
+static int
+rspamd_controller_handle_metrics_common(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg,
+ gboolean do_reset)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ ucl_object_t *top, *sub;
+ gint i;
+ int64_t uptime;
+ guint64 spam = 0, ham = 0;
+ rspamd_mempool_stat_t mem_st;
+ struct rspamd_stat *stat, stat_copy;
+ struct rspamd_controller_worker_ctx *ctx;
+ struct rspamd_task *task;
+ struct rspamd_stat_cbdata *cbdata;
+
+ memset(&mem_st, 0, sizeof(mem_st));
+ rspamd_mempool_stat(&mem_st);
+ memcpy(&stat_copy, session->ctx->worker->srv->stat, sizeof(stat_copy));
+ stat = &stat_copy;
+ ctx = session->ctx;
+
+ task = rspamd_task_new(session->ctx->worker, session->cfg, session->pool,
+ ctx->lang_det, ctx->event_loop, FALSE);
+ task->resolver = ctx->resolver;
+ cbdata = rspamd_mempool_alloc0(session->pool, sizeof(*cbdata));
+ cbdata->conn_ent = conn_ent;
+ cbdata->task = task;
+ cbdata->ctx = ctx;
+ top = ucl_object_typed_new(UCL_OBJECT);
+ cbdata->top = top;
+
+ task->s = rspamd_session_create(session->pool,
+ rspamd_controller_metrics_fin_task,
+ NULL,
+ rspamd_controller_stat_cleanup_task,
+ cbdata);
+ task->fin_arg = cbdata;
+ task->http_conn = rspamd_http_connection_ref(conn_ent->conn);
+ ;
+ task->sock = conn_ent->conn->fd;
+
+ ucl_object_insert_key(top, ucl_object_fromstring(RVERSION), "version", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromstring(session->ctx->cfg->checksum), "config_id", 0, false);
+ uptime = ev_time() - session->ctx->start_time;
+ ucl_object_insert_key(top, ucl_object_fromint(uptime), "uptime", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(session->ctx->start_time), "start_time", 0, false);
+ ucl_object_insert_key(top, ucl_object_frombool(session->is_read_only),
+ "read_only", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(stat->messages_scanned), "scanned", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(stat->messages_learned), "learned", 0, false);
+
+ if (stat->messages_scanned > 0) {
+ sub = ucl_object_typed_new(UCL_OBJECT);
+ for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
+ ucl_object_insert_key(sub,
+ ucl_object_fromint(stat->actions_stat[i]),
+ rspamd_action_to_str(i), 0, false);
+ if (i < METRIC_ACTION_GREYLIST) {
+ spam += stat->actions_stat[i];
+ }
+ else {
+ ham += stat->actions_stat[i];
+ }
+ if (do_reset) {
+#ifndef HAVE_ATOMIC_BUILTINS
+ session->ctx->worker->srv->stat->actions_stat[i] = 0;
+#else
+ __atomic_store_n(&session->ctx->worker->srv->stat->actions_stat[i],
+ 0, __ATOMIC_RELEASE);
+#endif
+ }
+ }
+ ucl_object_insert_key(top, sub, "actions", 0, false);
+ }
+
+ ucl_object_insert_key(top, ucl_object_fromint(spam), "spam_count", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(ham), "ham_count", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(stat->connections_count), "connections", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(stat->control_connections_count),
+ "control_connections", 0, false);
+
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.pools_allocated), "pools_allocated", 0,
+ false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.pools_freed), "pools_freed", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.bytes_allocated), "bytes_allocated", 0,
+ false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(
+ mem_st.chunks_allocated),
+ "chunks_allocated", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.shared_chunks_allocated),
+ "shared_chunks_allocated", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.chunks_freed), "chunks_freed", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(
+ mem_st.oversized_chunks),
+ "chunks_oversized", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.fragmented_size), "fragmented", 0, false);
+
+ if (do_reset) {
+ session->ctx->srv->stat->messages_scanned = 0;
+ session->ctx->srv->stat->messages_learned = 0;
+ session->ctx->srv->stat->connections_count = 0;
+ session->ctx->srv->stat->control_connections_count = 0;
+ rspamd_mempool_stat_reset();
+ }
+
+ fuzzy_stat_command(task);
+
+ /* Now write statistics for each statfile */
+ rspamd_stat_statistics(task, session->ctx->cfg, &cbdata->learned,
+ &cbdata->stat);
+ session->task = task;
+
+ rspamd_session_pending(task->s);
+
+
+ return 0;
+}
+
+
+static int
+rspamd_controller_handle_metrics(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+ return rspamd_controller_handle_metrics_common(conn_ent, msg, FALSE);
+}
+
+
+/*
+ * Counters command handler:
+ * request: /counters
+ * headers: Password
+ * reply: json array of all counters
+ */
+static int
+rspamd_controller_handle_counters(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ ucl_object_t *top;
+ struct rspamd_symcache *cache;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg, FALSE)) {
+ return 0;
+ }
+
+ cache = session->ctx->cfg->cache;
+
+ if (cache != NULL) {
+ top = rspamd_symcache_counters(cache);
+ rspamd_controller_send_ucl(conn_ent, top);
+ ucl_object_unref(top);
+ }
+ else {
+ rspamd_controller_send_error(conn_ent, 500, "Invalid cache");
+ }
+
+ return 0;
+}
+
+static int
+rspamd_controller_handle_custom(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_custom_controller_command *cmd;
+ gchar *url_str;
+ struct http_parser_url u;
+ rspamd_ftok_t lookup;
+
+ http_parser_parse_url(msg->url->str, msg->url->len, TRUE, &u);
+
+ if (u.field_set & (1 << UF_PATH)) {
+ gsize unnorm_len;
+ lookup.begin = msg->url->str + u.field_data[UF_PATH].off;
+ lookup.len = u.field_data[UF_PATH].len;
+
+ rspamd_normalize_path_inplace((gchar *) lookup.begin,
+ lookup.len,
+ &unnorm_len);
+ lookup.len = unnorm_len;
+ }
+ else {
+ lookup.begin = msg->url->str;
+ lookup.len = msg->url->len;
+ }
+
+ url_str = rspamd_ftok_cstr(&lookup);
+ cmd = g_hash_table_lookup(session->ctx->custom_commands, url_str);
+ g_free(url_str);
+
+ if (cmd == NULL || cmd->handler == NULL) {
+ msg_err_session("custom command %T has not been found", &lookup);
+ rspamd_controller_send_error(conn_ent, 404, "No command associated");
+ return 0;
+ }
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg,
+ cmd->privileged)) {
+ return 0;
+ }
+ if (cmd->require_message && (rspamd_http_message_get_body(msg, NULL) == NULL)) {
+ msg_err_session("got zero length body, cannot continue");
+ rspamd_controller_send_error(conn_ent,
+ 400,
+ "Empty body is not permitted");
+ return 0;
+ }
+
+ /* Transfer query arguments to headers */
+ if (u.field_set & (1u << UF_QUERY)) {
+ GHashTable *query_args;
+ GHashTableIter it;
+ gpointer k, v;
+ rspamd_ftok_t *key, *value;
+
+ /* In case if we have a query, we need to store it somewhere */
+ query_args = rspamd_http_message_parse_query(msg);
+
+ /* Insert the rest of query params as HTTP headers */
+ g_hash_table_iter_init(&it, query_args);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ key = k;
+ value = v;
+ /* Steal strings */
+ g_hash_table_iter_steal(&it);
+ url_str = rspamd_ftok_cstr(key);
+ rspamd_http_message_add_header_len(msg, url_str,
+ value->begin, value->len);
+ g_free(url_str);
+ }
+
+ g_hash_table_unref(query_args);
+ }
+
+ return cmd->handler(conn_ent, msg, cmd->ctx);
+}
+
+static int
+rspamd_controller_handle_plugins(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_controller_plugin_cbdata *cbd;
+ GHashTableIter it;
+ gpointer k, v;
+ ucl_object_t *plugins;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg,
+ FALSE)) {
+ return 0;
+ }
+
+ plugins = ucl_object_typed_new(UCL_OBJECT);
+ g_hash_table_iter_init(&it, session->ctx->plugins);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ ucl_object_t *elt, *npath;
+
+ cbd = v;
+ elt = (ucl_object_t *) ucl_object_lookup(plugins, cbd->plugin);
+
+ if (elt == NULL) {
+ elt = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(elt, ucl_object_fromint(cbd->version),
+ "version", 0, false);
+ npath = ucl_object_typed_new(UCL_ARRAY);
+ ucl_object_insert_key(elt, npath, "paths", 0, false);
+ ucl_object_insert_key(plugins, elt, cbd->plugin, 0, false);
+ }
+ else {
+ npath = (ucl_object_t *) ucl_object_lookup(elt, "paths");
+ }
+
+ g_assert(npath != NULL);
+ rspamd_ftok_t *key_tok = (rspamd_ftok_t *) k;
+ ucl_array_append(npath, ucl_object_fromlstring(key_tok->begin, key_tok->len));
+ }
+
+ rspamd_controller_send_ucl(conn_ent, plugins);
+ ucl_object_unref(plugins);
+
+ return 0;
+}
+
+static int
+rspamd_controller_handle_ping(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_http_message *rep_msg;
+ rspamd_fstring_t *reply;
+
+ rep_msg = rspamd_http_new_message(HTTP_RESPONSE);
+ rep_msg->date = time(NULL);
+ rep_msg->code = 200;
+ rep_msg->status = rspamd_fstring_new_init("OK", 2);
+ reply = rspamd_fstring_new_init("pong" CRLF, strlen("pong" CRLF));
+ rspamd_http_message_set_body_from_fstring_steal(rep_msg, reply);
+ rspamd_http_connection_reset(conn_ent->conn);
+ rspamd_http_router_insert_headers(conn_ent->rt, rep_msg);
+ rspamd_http_connection_write_message(conn_ent->conn,
+ rep_msg,
+ NULL,
+ "text/plain",
+ conn_ent,
+ conn_ent->rt->timeout);
+ conn_ent->is_reply = TRUE;
+
+ return 0;
+}
+
+/*
+ * Called on unknown methods and is used to deal with CORS as per
+ * https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS
+ */
+static int
+rspamd_controller_handle_unknown(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_http_message *rep;
+
+ if (msg->method == HTTP_OPTIONS) {
+ /* Assume CORS request */
+
+ rep = rspamd_http_new_message(HTTP_RESPONSE);
+ rep->date = time(NULL);
+ rep->code = 200;
+ rep->status = rspamd_fstring_new_init("OK", 2);
+ rspamd_http_message_add_header(rep, "Access-Control-Allow-Methods",
+ "POST, GET, OPTIONS");
+ rspamd_http_message_add_header(rep, "Access-Control-Allow-Headers",
+ "Content-Type,Password,Map,Weight,Flag");
+ rspamd_http_connection_reset(conn_ent->conn);
+ rspamd_http_router_insert_headers(conn_ent->rt, rep);
+ rspamd_http_connection_write_message(conn_ent->conn,
+ rep,
+ NULL,
+ "text/plain",
+ conn_ent,
+ conn_ent->rt->timeout);
+ conn_ent->is_reply = TRUE;
+ }
+ else {
+ rep = rspamd_http_new_message(HTTP_RESPONSE);
+ rep->date = time(NULL);
+ rep->code = 500;
+ rep->status = rspamd_fstring_new_init("Invalid method",
+ strlen("Invalid method"));
+ rspamd_http_connection_reset(conn_ent->conn);
+ rspamd_http_router_insert_headers(conn_ent->rt, rep);
+ rspamd_http_connection_write_message(conn_ent->conn,
+ rep,
+ NULL,
+ "text/plain",
+ conn_ent,
+ conn_ent->rt->timeout);
+ conn_ent->is_reply = TRUE;
+ }
+
+ return 0;
+}
+
+static int
+rspamd_controller_handle_lua_plugin(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_controller_plugin_cbdata *cbd;
+ struct rspamd_task *task, **ptask;
+ struct rspamd_http_connection_entry **pconn;
+ struct rspamd_controller_worker_ctx *ctx;
+ lua_State *L;
+ struct http_parser_url u;
+ rspamd_ftok_t lookup;
+
+
+ http_parser_parse_url(msg->url->str, msg->url->len, TRUE, &u);
+
+ if (u.field_set & (1 << UF_PATH)) {
+ gsize unnorm_len;
+ lookup.begin = msg->url->str + u.field_data[UF_PATH].off;
+ lookup.len = u.field_data[UF_PATH].len;
+
+ rspamd_normalize_path_inplace((gchar *) lookup.begin,
+ lookup.len,
+ &unnorm_len);
+ lookup.len = unnorm_len;
+ }
+ else {
+ lookup.begin = msg->url->str;
+ lookup.len = msg->url->len;
+ }
+
+ cbd = g_hash_table_lookup(session->ctx->plugins, &lookup);
+
+ if (cbd == NULL || cbd->handler == NULL) {
+ msg_err_session("plugin handler %T has not been found", &lookup);
+ rspamd_controller_send_error(conn_ent, 404, "No command associated");
+ return 0;
+ }
+
+ L = cbd->L;
+ ctx = cbd->ctx;
+
+ if (!rspamd_controller_check_password(conn_ent, session, msg,
+ cbd->is_enable)) {
+ return 0;
+ }
+ if (cbd->need_task && (rspamd_http_message_get_body(msg, NULL) == NULL)) {
+ msg_err_session("got zero length body, cannot continue");
+ rspamd_controller_send_error(conn_ent,
+ 400,
+ "Empty body is not permitted");
+ return 0;
+ }
+
+ task = rspamd_task_new(session->ctx->worker, session->cfg, session->pool,
+ ctx->lang_det, ctx->event_loop, FALSE);
+
+ task->resolver = ctx->resolver;
+ task->s = rspamd_session_create(session->pool,
+ rspamd_controller_lua_fin_task,
+ NULL,
+ (event_finalizer_t) rspamd_task_free,
+ task);
+ task->fin_arg = conn_ent;
+ task->http_conn = rspamd_http_connection_ref(conn_ent->conn);
+ ;
+ task->sock = -1;
+ session->task = task;
+
+ if (msg->body_buf.len > 0) {
+ if (!rspamd_task_load_message(task, msg, msg->body_buf.begin, msg->body_buf.len)) {
+ rspamd_controller_send_error(conn_ent, task->err->code, "%s",
+ task->err->message);
+ return 0;
+ }
+ }
+
+ /* Callback */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->handler->idx);
+
+ /* Task */
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+
+ /* Connection */
+ pconn = lua_newuserdata(L, sizeof(*pconn));
+ rspamd_lua_setclass(L, "rspamd{csession}", -1);
+ *pconn = conn_ent;
+
+ /* Query arguments */
+ GHashTable *params;
+ GHashTableIter it;
+ gpointer k, v;
+
+ params = rspamd_http_message_parse_query(msg);
+ lua_createtable(L, g_hash_table_size(params), 0);
+ g_hash_table_iter_init(&it, params);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ rspamd_ftok_t *key_tok = (rspamd_ftok_t *) k,
+ *value_tok = (rspamd_ftok_t *) v;
+
+ lua_pushlstring(L, key_tok->begin, key_tok->len);
+ /* TODO: consider rspamd_text here */
+ lua_pushlstring(L, value_tok->begin, value_tok->len);
+ lua_settable(L, -3);
+ }
+
+ g_hash_table_unref(params);
+
+ if (lua_pcall(L, 3, 0, 0) != 0) {
+ rspamd_controller_send_error(conn_ent, 503, "Cannot run callback: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return 0;
+ }
+
+ rspamd_session_pending(task->s);
+
+ return 0;
+}
+
+
+static void
+rspamd_controller_error_handler(struct rspamd_http_connection_entry *conn_ent,
+ GError *err)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+
+ msg_err_session("http error occurred: %s", err->message);
+}
+
+static void
+rspamd_controller_finish_handler(struct rspamd_http_connection_entry *conn_ent)
+{
+ struct rspamd_controller_session *session = conn_ent->ud;
+
+ session->ctx->worker->srv->stat->control_connections_count++;
+
+ if (session->task != NULL) {
+ rspamd_session_destroy(session->task->s);
+ }
+
+ session->wrk->nconns--;
+ rspamd_inet_address_free(session->from_addr);
+ REF_RELEASE(session->cfg);
+
+ if (session->pool) {
+ msg_debug_session("destroy session %p", session);
+ rspamd_mempool_delete(session->pool);
+ }
+
+ g_free(session);
+}
+
+static void
+rspamd_controller_accept_socket(EV_P_ ev_io *w, int revents)
+{
+ struct rspamd_worker *worker = (struct rspamd_worker *) w->data;
+ struct rspamd_controller_worker_ctx *ctx;
+ struct rspamd_controller_session *session;
+ rspamd_inet_addr_t *addr = NULL;
+ gint nfd;
+
+ ctx = worker->ctx;
+
+ if ((nfd =
+ rspamd_accept_from_socket(w->fd, &addr,
+ rspamd_worker_throttle_accept_events, worker->accept_events)) == -1) {
+ msg_warn_ctx("accept failed: %s", strerror(errno));
+ return;
+ }
+ /* Check for EAGAIN */
+ if (nfd == 0) {
+ rspamd_inet_address_free(addr);
+ return;
+ }
+
+ session = g_malloc0(sizeof(struct rspamd_controller_session));
+ session->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "csession", 0);
+ session->ctx = ctx;
+ session->cfg = ctx->cfg;
+ session->lang_det = ctx->lang_det;
+ REF_RETAIN(session->cfg);
+
+ session->from_addr = addr;
+ session->wrk = worker;
+ worker->nconns++;
+
+ rspamd_http_router_handle_socket(ctx->http, nfd, session);
+}
+
+static void
+rspamd_controller_password_sane(struct rspamd_controller_worker_ctx *ctx,
+ const gchar *password, const gchar *type)
+{
+ const struct rspamd_controller_pbkdf *pbkdf = &pbkdf_list[0];
+
+ if (password == NULL) {
+ msg_warn_ctx("%s is not set, so you should filter controller "
+ "availability "
+ "by using of firewall or `secure_ip` option",
+ type);
+ return;
+ }
+
+ g_assert(pbkdf != NULL);
+
+ if (!rspamd_is_encrypted_password(password, NULL)) {
+ /* Suggest encryption to a user */
+
+ msg_warn_ctx("your %s is not encrypted, we strongly "
+ "recommend to replace it with the encrypted one",
+ type);
+ }
+}
+
+gpointer
+init_controller_worker(struct rspamd_config *cfg)
+{
+ struct rspamd_controller_worker_ctx *ctx;
+ GQuark type;
+
+ type = g_quark_try_string("controller");
+
+ ctx = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(struct rspamd_controller_worker_ctx));
+
+ ctx->magic = rspamd_controller_ctx_magic;
+ ctx->timeout = DEFAULT_WORKER_IO_TIMEOUT;
+ ctx->task_timeout = NAN;
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "password",
+ rspamd_rcl_parse_struct_string,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx, password),
+ 0,
+ "Password for read-only commands");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "enable_password",
+ rspamd_rcl_parse_struct_string,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx,
+ enable_password),
+ 0,
+ "Password for read and write commands");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "ssl",
+ rspamd_rcl_parse_struct_boolean,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx, use_ssl),
+ 0,
+ "Unimplemented");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "ssl_cert",
+ rspamd_rcl_parse_struct_string,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx, ssl_cert),
+ 0,
+ "Unimplemented");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "ssl_key",
+ rspamd_rcl_parse_struct_string,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx, ssl_key),
+ 0,
+ "Unimplemented");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "timeout",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx,
+ timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Protocol timeout");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "secure_ip",
+ rspamd_rcl_parse_struct_ucl,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx, secure_ip),
+ 0,
+ "List of IP addresses that are allowed for password-less access");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "trusted_ips",
+ rspamd_rcl_parse_struct_ucl,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx, secure_ip),
+ 0,
+ "List of IP addresses that are allowed for password-less access");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "static_dir",
+ rspamd_rcl_parse_struct_string,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx,
+ static_files_dir),
+ 0,
+ "Directory for static files served by controller's HTTP server");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "keypair",
+ rspamd_rcl_parse_struct_keypair,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx,
+ key),
+ 0,
+ "Encryption keypair");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "task_timeout",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx,
+ task_timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Maximum task processing time, default: 8.0 seconds");
+
+ return ctx;
+}
+
+/* Lua bindings */
+LUA_FUNCTION_DEF(csession, get_ev_base);
+LUA_FUNCTION_DEF(csession, get_cfg);
+LUA_FUNCTION_DEF(csession, send_ucl);
+LUA_FUNCTION_DEF(csession, send_string);
+LUA_FUNCTION_DEF(csession, send_error);
+
+static const struct luaL_reg lua_csessionlib_m[] = {
+ LUA_INTERFACE_DEF(csession, get_ev_base),
+ LUA_INTERFACE_DEF(csession, get_cfg),
+ LUA_INTERFACE_DEF(csession, send_ucl),
+ LUA_INTERFACE_DEF(csession, send_string),
+ LUA_INTERFACE_DEF(csession, send_error),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/* Basic functions of LUA API for worker object */
+static void
+luaopen_controller(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{csession}", lua_csessionlib_m);
+ lua_pop(L, 1);
+}
+
+struct rspamd_http_connection_entry *
+lua_check_controller_entry(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{csession}");
+ luaL_argcheck(L, ud != NULL, pos, "'csession' expected");
+ return ud ? *((struct rspamd_http_connection_entry **) ud) : NULL;
+}
+
+static int
+lua_csession_get_ev_base(lua_State *L)
+{
+ struct rspamd_http_connection_entry *c = lua_check_controller_entry(L, 1);
+ struct ev_loop **pbase;
+ struct rspamd_controller_session *s;
+
+ if (c) {
+ s = c->ud;
+ pbase = lua_newuserdata(L, sizeof(struct ev_loop *));
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+ *pbase = s->ctx->event_loop;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_csession_get_cfg(lua_State *L)
+{
+ struct rspamd_http_connection_entry *c = lua_check_controller_entry(L, 1);
+ struct rspamd_config **pcfg;
+ struct rspamd_controller_session *s;
+
+ if (c) {
+ s = c->ud;
+ pcfg = lua_newuserdata(L, sizeof(gpointer));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = s->ctx->cfg;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_csession_send_ucl(lua_State *L)
+{
+ struct rspamd_http_connection_entry *c = lua_check_controller_entry(L, 1);
+ ucl_object_t *obj = ucl_object_lua_import_escape(L, 2);
+
+ if (c) {
+ rspamd_controller_send_ucl(c, obj);
+ }
+ else {
+ ucl_object_unref(obj);
+ return luaL_error(L, "invalid arguments");
+ }
+
+ ucl_object_unref(obj);
+
+ return 0;
+}
+
+static int
+lua_csession_send_error(lua_State *L)
+{
+ struct rspamd_http_connection_entry *c = lua_check_controller_entry(L, 1);
+ guint err_code = lua_tonumber(L, 2);
+ const gchar *err_str = lua_tostring(L, 3);
+
+ if (c) {
+ rspamd_controller_send_error(c, err_code, "%s", err_str);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static int
+lua_csession_send_string(lua_State *L)
+{
+ struct rspamd_http_connection_entry *c = lua_check_controller_entry(L, 1);
+ const gchar *str = lua_tostring(L, 2);
+
+ if (c) {
+ rspamd_controller_send_string(c, str);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static void
+rspamd_plugin_cbdata_dtor(gpointer p)
+{
+ struct rspamd_controller_plugin_cbdata *cbd = p;
+
+ g_free(cbd->plugin);
+ ucl_object_unref(cbd->obj); /* This also releases lua references */
+ g_free(cbd);
+}
+
+static void
+rspamd_controller_register_plugin_path(lua_State *L,
+ struct rspamd_controller_worker_ctx *ctx,
+ const ucl_object_t *webui_data,
+ const ucl_object_t *handler,
+ const gchar *path,
+ const gchar *plugin_name)
+{
+ struct rspamd_controller_plugin_cbdata *cbd;
+ const ucl_object_t *elt;
+ rspamd_fstring_t *full_path;
+
+ cbd = g_malloc0(sizeof(*cbd));
+ cbd->L = L;
+ cbd->ctx = ctx;
+ cbd->handler = ucl_object_toclosure(handler);
+ cbd->plugin = g_strdup(plugin_name);
+ cbd->obj = ucl_object_ref(webui_data);
+
+ elt = ucl_object_lookup(webui_data, "version");
+
+ if (elt) {
+ cbd->version = ucl_object_toint(elt);
+ }
+
+ elt = ucl_object_lookup(webui_data, "enable");
+
+ if (elt && ucl_object_toboolean(elt)) {
+ cbd->is_enable = TRUE;
+ }
+
+ elt = ucl_object_lookup(webui_data, "need_task");
+
+ if (elt && !!ucl_object_toboolean(elt)) {
+ cbd->need_task = TRUE;
+ }
+
+ full_path = rspamd_fstring_new_init("/plugins/", sizeof("/plugins/") - 1);
+ /* Zero terminated */
+ rspamd_printf_fstring(&full_path, "%s/%s%c",
+ plugin_name, path, '\0');
+
+ rspamd_http_router_add_path(ctx->http,
+ full_path->str,
+ rspamd_controller_handle_lua_plugin);
+ rspamd_ftok_t *key_tok = rspamd_ftok_map(full_path);
+ /* Truncate stupid \0 symbol to enable lookup */
+ key_tok->len--;
+ g_hash_table_insert(ctx->plugins, key_tok, cbd);
+}
+
+static void
+rspamd_controller_register_plugins_paths(struct rspamd_controller_worker_ctx *ctx)
+{
+ lua_State *L = ctx->cfg->lua_state;
+ ucl_object_t *webui_data;
+ const ucl_object_t *handler_obj, *cur;
+ ucl_object_iter_t it = NULL;
+
+ lua_getglobal(L, "rspamd_plugins");
+
+ if (lua_istable(L, -1)) {
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 2)) {
+ lua_pushvalue(L, -2); /* Store key */
+
+ lua_pushstring(L, "webui");
+ lua_gettable(L, -3); /* value is at -3 index */
+
+ if (lua_istable(L, -1)) {
+ webui_data = ucl_object_lua_import_escape(L, -1);
+
+ while ((cur = ucl_object_iterate(webui_data, &it, true)) != NULL) {
+ handler_obj = ucl_object_lookup(cur, "handler");
+
+ if (handler_obj && ucl_object_key(cur)) {
+ rspamd_controller_register_plugin_path(L, ctx,
+ cur, handler_obj, ucl_object_key(cur),
+ lua_tostring(L, -2));
+ }
+ else {
+ msg_err_ctx("bad webui definition for plugin: %s",
+ lua_tostring(L, -2));
+ }
+ }
+
+ ucl_object_unref(webui_data);
+ }
+
+ lua_pop(L, 1); /* remove table value */
+ }
+ }
+
+ lua_pop(L, 1); /* rspamd_plugins global */
+}
+
+static void
+rspamd_controller_health_rep(struct rspamd_worker *worker,
+ struct rspamd_srv_reply *rep, gint rep_fd,
+ gpointer ud)
+{
+ struct rspamd_controller_worker_ctx *ctx = (struct rspamd_controller_worker_ctx *) ud;
+
+ ctx->workers_count = rep->reply.health.workers_count;
+ ctx->scanners_count = rep->reply.health.scanners_count;
+ ctx->workers_hb_lost = rep->reply.health.workers_hb_lost;
+
+ ev_timer_again(ctx->event_loop, &ctx->health_check_timer);
+}
+
+static void
+rspamd_controller_health_timer(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_controller_worker_ctx *ctx = (struct rspamd_controller_worker_ctx *) w->data;
+ struct rspamd_srv_command srv_cmd;
+
+ memset(&srv_cmd, 0, sizeof(srv_cmd));
+ srv_cmd.type = RSPAMD_SRV_HEALTH;
+ rspamd_srv_send_command(ctx->worker, ctx->event_loop, &srv_cmd, -1,
+ rspamd_controller_health_rep, ctx);
+ ev_timer_stop(EV_A_ w);
+}
+
+/*
+ * Start worker process
+ */
+__attribute__((noreturn)) void
+start_controller_worker(struct rspamd_worker *worker)
+{
+ struct rspamd_controller_worker_ctx *ctx = worker->ctx;
+ struct module_ctx *mctx;
+ GHashTableIter iter;
+ gpointer key, value;
+ guint i;
+ gpointer m;
+
+ g_assert(rspamd_worker_check_context(worker->ctx, rspamd_controller_ctx_magic));
+ ctx->event_loop = rspamd_prepare_worker(worker,
+ "controller",
+ rspamd_controller_accept_socket);
+
+ ctx->start_time = ev_time();
+ ctx->worker = worker;
+ ctx->cfg = worker->srv->cfg;
+ ctx->srv = worker->srv;
+ ctx->custom_commands = g_hash_table_new(rspamd_strcase_hash,
+ rspamd_strcase_equal);
+ ctx->plugins = g_hash_table_new_full(rspamd_ftok_icase_hash,
+ rspamd_ftok_icase_equal, rspamd_fstring_mapped_ftok_free,
+ rspamd_plugin_cbdata_dtor);
+
+ ctx->task_timeout = rspamd_worker_check_and_adjust_timeout(ctx->cfg, ctx->task_timeout);
+
+ if (ctx->secure_ip != NULL) {
+ rspamd_config_radix_from_ucl(ctx->cfg, ctx->secure_ip,
+ "Allow unauthenticated requests from these addresses",
+ &ctx->secure_map,
+ NULL,
+ worker, "controller secure ip");
+ }
+
+ ctx->lang_det = ctx->cfg->lang_det;
+
+ rspamd_controller_password_sane(ctx, ctx->password, "normal password");
+ rspamd_controller_password_sane(ctx, ctx->enable_password, "enable "
+ "password");
+
+ /* Accept event */
+ ctx->http_ctx = rspamd_http_context_create(ctx->cfg, ctx->event_loop,
+ ctx->cfg->ups_ctx);
+ rspamd_mempool_add_destructor(ctx->cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_http_context_free,
+ ctx->http_ctx);
+ ctx->http = rspamd_http_router_new(rspamd_controller_error_handler,
+ rspamd_controller_finish_handler, ctx->timeout,
+ ctx->static_files_dir, ctx->http_ctx);
+
+ /* Add callbacks for different methods */
+ rspamd_http_router_add_path(ctx->http,
+ PATH_AUTH,
+ rspamd_controller_handle_auth);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_SYMBOLS,
+ rspamd_controller_handle_symbols);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_ACTIONS,
+ rspamd_controller_handle_actions);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_MAPS,
+ rspamd_controller_handle_maps);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_GET_MAP,
+ rspamd_controller_handle_get_map);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_PIE_CHART,
+ rspamd_controller_handle_pie_chart);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_GRAPH,
+ rspamd_controller_handle_graph);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_HEALTHY,
+ rspamd_controller_handle_healthy);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_READY,
+ rspamd_controller_handle_ready);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_HISTORY,
+ rspamd_controller_handle_history);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_HISTORY_RESET,
+ rspamd_controller_handle_history_reset);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_LEARN_SPAM,
+ rspamd_controller_handle_learnspam);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_LEARN_HAM,
+ rspamd_controller_handle_learnham);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_METRICS,
+ rspamd_controller_handle_metrics);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_SAVE_ACTIONS,
+ rspamd_controller_handle_saveactions);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_SAVE_SYMBOLS,
+ rspamd_controller_handle_savesymbols);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_SAVE_MAP,
+ rspamd_controller_handle_savemap);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_SCAN,
+ rspamd_controller_handle_scan);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_CHECK,
+ rspamd_controller_handle_scan);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_CHECKV2,
+ rspamd_controller_handle_scan);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_STAT,
+ rspamd_controller_handle_stat);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_STAT_RESET,
+ rspamd_controller_handle_statreset);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_COUNTERS,
+ rspamd_controller_handle_counters);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_ERRORS,
+ rspamd_controller_handle_errors);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_NEIGHBOURS,
+ rspamd_controller_handle_neighbours);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_PLUGINS,
+ rspamd_controller_handle_plugins);
+ rspamd_http_router_add_path(ctx->http,
+ PATH_PING,
+ rspamd_controller_handle_ping);
+ rspamd_controller_register_plugins_paths(ctx);
+
+#if 0
+ rspamd_regexp_t *lua_re = rspamd_regexp_new ("^/.*/.*\\.lua$", NULL, NULL);
+ rspamd_http_router_add_regexp (ctx->http, lua_re,
+ rspamd_controller_handle_lua);
+ rspamd_regexp_unref (lua_re);
+#endif
+ luaopen_controller(ctx->cfg->lua_state);
+
+ if (ctx->key) {
+ rspamd_http_router_set_key(ctx->http, ctx->key);
+ }
+
+ PTR_ARRAY_FOREACH(ctx->cfg->c_modules, i, mctx)
+ {
+ if (mctx->mod->module_attach_controller_func != NULL) {
+ mctx->mod->module_attach_controller_func(mctx,
+ ctx->custom_commands);
+ }
+ }
+
+ g_hash_table_iter_init(&iter, ctx->custom_commands);
+ while (g_hash_table_iter_next(&iter, &key, &value)) {
+ rspamd_http_router_add_path(ctx->http,
+ key,
+ rspamd_controller_handle_custom);
+ }
+
+ if (worker->srv->cfg->neighbours && worker->srv->cfg->neighbours->len > 0) {
+ rspamd_http_router_add_header(ctx->http,
+ "Access-Control-Allow-Origin", "*");
+ }
+
+ /* Disable all results caching, see #3330 */
+ rspamd_http_router_add_header(ctx->http,
+ "Cache-Control", "no-store");
+
+ rspamd_http_router_set_unknown_handler(ctx->http,
+ rspamd_controller_handle_unknown);
+
+ ctx->resolver = rspamd_dns_resolver_init(worker->srv->logger,
+ ctx->event_loop,
+ worker->srv->cfg);
+
+ rspamd_upstreams_library_config(worker->srv->cfg, worker->srv->cfg->ups_ctx,
+ ctx->event_loop, ctx->resolver->r);
+ rspamd_symcache_start_refresh(worker->srv->cfg->cache, ctx->event_loop,
+ worker);
+ rspamd_stat_init(worker->srv->cfg, ctx->event_loop);
+ rspamd_worker_init_controller(worker, &ctx->rrd);
+ rspamd_lua_run_postloads(ctx->cfg->lua_state, ctx->cfg, ctx->event_loop, worker);
+
+ /* TODO: maybe make it configurable */
+ ev_timer_init(&ctx->health_check_timer, rspamd_controller_health_timer,
+ 1.0, 60.0);
+ ctx->health_check_timer.data = ctx;
+ ev_timer_start(ctx->event_loop, &ctx->health_check_timer);
+
+#ifdef WITH_HYPERSCAN
+ rspamd_control_worker_add_cmd_handler(worker,
+ RSPAMD_CONTROL_HYPERSCAN_LOADED,
+ rspamd_worker_hyperscan_ready,
+ NULL);
+#endif
+
+ /* Start event loop */
+ ev_loop(ctx->event_loop, 0);
+ rspamd_worker_block_signals();
+ rspamd_controller_on_terminate(worker, ctx->rrd);
+
+ rspamd_stat_close();
+ rspamd_http_router_free(ctx->http);
+
+ if (ctx->cached_password.len > 0) {
+ m = (gpointer) ctx->cached_password.begin;
+ munmap(m, ctx->cached_password.len);
+ }
+
+ if (ctx->cached_enable_password.len > 0) {
+ m = (gpointer) ctx->cached_enable_password.begin;
+ munmap(m, ctx->cached_enable_password.len);
+ }
+
+ g_hash_table_unref(ctx->plugins);
+ g_hash_table_unref(ctx->custom_commands);
+
+ REF_RELEASE(ctx->cfg);
+ rspamd_log_close(worker->srv->logger);
+ rspamd_unset_crash_handler(worker->srv);
+
+ exit(EXIT_SUCCESS);
+}
diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
new file mode 100644
index 0000000..d42dffd
--- /dev/null
+++ b/src/fuzzy_storage.c
@@ -0,0 +1,3287 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Rspamd fuzzy storage server
+ */
+
+#include "config.h"
+#include "libserver/fuzzy_wire.h"
+#include "util.h"
+#include "rspamd.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
+#include "libserver/fuzzy_backend/fuzzy_backend.h"
+#include "ottery.h"
+#include "ref.h"
+#include "xxhash.h"
+#include "libserver/worker_util.h"
+#include "libserver/rspamd_control.h"
+#include "libcryptobox/cryptobox.h"
+#include "libcryptobox/keypairs_cache.h"
+#include "libcryptobox/keypair.h"
+#include "libutil/hash.h"
+#include "libserver/maps/map_private.h"
+#include "contrib/uthash/utlist.h"
+#include "lua/lua_common.h"
+#include "unix-std.h"
+
+#include <math.h>
+
+/* Resync value in seconds */
+#define DEFAULT_SYNC_TIMEOUT 60.0
+#define DEFAULT_KEYPAIR_CACHE_SIZE 512
+#define DEFAULT_MASTER_TIMEOUT 10.0
+#define DEFAULT_UPDATES_MAXFAIL 3
+#define DEFAULT_MAX_BUCKETS 2000
+#define DEFAULT_BUCKET_TTL 3600
+#define DEFAULT_BUCKET_MASK 24
+/* Update stats on keys each 1 hour */
+#define KEY_STAT_INTERVAL 3600.0
+
+static const gchar *local_db_name = "local";
+
+/* Init functions */
+gpointer init_fuzzy(struct rspamd_config *cfg);
+void start_fuzzy(struct rspamd_worker *worker);
+
+worker_t fuzzy_worker = {
+ "fuzzy", /* Name */
+ init_fuzzy, /* Init function */
+ start_fuzzy, /* Start function */
+ RSPAMD_WORKER_HAS_SOCKET | RSPAMD_WORKER_NO_STRICT_CONFIG,
+ RSPAMD_WORKER_SOCKET_UDP, /* UDP socket */
+ RSPAMD_WORKER_VER /* Version info */
+};
+
+struct fuzzy_global_stat {
+ guint64 fuzzy_hashes;
+ /**< number of fuzzy hashes stored */
+ guint64 fuzzy_hashes_expired;
+ /**< number of fuzzy hashes expired */
+ guint64 fuzzy_hashes_checked[RSPAMD_FUZZY_EPOCH_MAX];
+ /**< amount of check requests for each epoch */
+ guint64 fuzzy_shingles_checked[RSPAMD_FUZZY_EPOCH_MAX];
+ /**< amount of shingle check requests for each epoch */
+ guint64 fuzzy_hashes_found[RSPAMD_FUZZY_EPOCH_MAX];
+ /**< amount of invalid requests */
+ guint64 invalid_requests;
+ /**< amount of delayed hashes found */
+ guint64 delayed_hashes;
+};
+
+struct fuzzy_key_stat {
+ guint64 checked;
+ guint64 matched;
+ guint64 added;
+ guint64 deleted;
+ guint64 errors;
+ /* Store averages for checked/matched per minute */
+ struct rspamd_counter_data checked_ctr;
+ struct rspamd_counter_data matched_ctr;
+ gdouble last_checked_time;
+ guint64 last_checked_count;
+ guint64 last_matched_count;
+ struct rspamd_cryptobox_keypair *keypair;
+ rspamd_lru_hash_t *last_ips;
+
+ ref_entry_t ref;
+};
+
+struct rspamd_leaky_bucket_elt {
+ rspamd_inet_addr_t *addr;
+ gdouble last;
+ gdouble cur;
+};
+
+static const guint64 rspamd_fuzzy_storage_magic = 0x291a3253eb1b3ea5ULL;
+KHASH_SET_INIT_INT(fuzzy_key_ids_set);
+
+struct rspamd_fuzzy_storage_ctx {
+ guint64 magic;
+ /* Events base */
+ struct ev_loop *event_loop;
+ /* DNS resolver */
+ struct rspamd_dns_resolver *resolver;
+ /* Config */
+ struct rspamd_config *cfg;
+ /* END OF COMMON PART */
+ struct fuzzy_global_stat stat;
+ gdouble expire;
+ gdouble sync_timeout;
+ gdouble delay;
+ struct rspamd_radix_map_helper *update_ips;
+ struct rspamd_hash_map_helper *update_keys;
+ struct rspamd_radix_map_helper *blocked_ips;
+ struct rspamd_radix_map_helper *ratelimit_whitelist;
+ struct rspamd_radix_map_helper *delay_whitelist;
+
+ const ucl_object_t *update_map;
+ const ucl_object_t *update_keys_map;
+ const ucl_object_t *delay_whitelist_map;
+ const ucl_object_t *blocked_map;
+ const ucl_object_t *ratelimit_whitelist_map;
+
+ guint keypair_cache_size;
+ ev_timer stat_ev;
+ ev_io peer_ev;
+
+ /* Local keypair */
+ struct rspamd_cryptobox_keypair *default_keypair; /* Bad clash, need for parse keypair */
+ struct fuzzy_key *default_key;
+ GHashTable *keys;
+ gboolean encrypted_only;
+ gboolean read_only;
+ gboolean dedicated_update_worker;
+ struct rspamd_keypair_cache *keypair_cache;
+ struct rspamd_http_context *http_ctx;
+ rspamd_lru_hash_t *errors_ips;
+ rspamd_lru_hash_t *ratelimit_buckets;
+ struct rspamd_fuzzy_backend *backend;
+ GArray *updates_pending;
+ guint updates_failed;
+ guint updates_maxfail;
+ /* Used to send data between workers */
+ gint peer_fd;
+
+ /* Ratelimits */
+ guint leaky_bucket_ttl;
+ guint leaky_bucket_mask;
+ guint max_buckets;
+ gboolean ratelimit_log_only;
+ gdouble leaky_bucket_burst;
+ gdouble leaky_bucket_rate;
+
+ struct rspamd_worker *worker;
+ const ucl_object_t *skip_map;
+ struct rspamd_hash_map_helper *skip_hashes;
+ gint lua_pre_handler_cbref;
+ gint lua_post_handler_cbref;
+ gint lua_blacklist_cbref;
+ khash_t(fuzzy_key_ids_set) * default_forbidden_ids;
+ /* Ids that should not override other ids */
+ khash_t(fuzzy_key_ids_set) * weak_ids;
+};
+
+enum fuzzy_cmd_type {
+ CMD_NORMAL,
+ CMD_SHINGLE,
+ CMD_ENCRYPTED_NORMAL,
+ CMD_ENCRYPTED_SHINGLE
+};
+
+struct fuzzy_session {
+ struct rspamd_worker *worker;
+ rspamd_inet_addr_t *addr;
+ struct rspamd_fuzzy_storage_ctx *ctx;
+
+ struct rspamd_fuzzy_shingle_cmd cmd; /* Can handle both shingles and non-shingles */
+ struct rspamd_fuzzy_encrypted_reply reply; /* Again: contains everything */
+ struct fuzzy_key_stat *ip_stat;
+
+ enum rspamd_fuzzy_epoch epoch;
+ enum fuzzy_cmd_type cmd_type;
+ gint fd;
+ ev_tstamp timestamp;
+ struct ev_io io;
+ ref_entry_t ref;
+ struct fuzzy_key *key;
+ struct rspamd_fuzzy_cmd_extension *extensions;
+ guchar nm[rspamd_cryptobox_MAX_NMBYTES];
+};
+
+struct fuzzy_peer_request {
+ ev_io io_ev;
+ struct fuzzy_peer_cmd cmd;
+};
+
+KHASH_INIT(fuzzy_key_flag_stat, int, struct fuzzy_key_stat, 1, kh_int_hash_func,
+ kh_int_hash_equal);
+struct fuzzy_key {
+ struct rspamd_cryptobox_keypair *key;
+ struct rspamd_cryptobox_pubkey *pk;
+ struct fuzzy_key_stat *stat;
+ khash_t(fuzzy_key_flag_stat) * flags_stat;
+ khash_t(fuzzy_key_ids_set) * forbidden_ids;
+};
+
+struct rspamd_updates_cbdata {
+ GArray *updates_pending;
+ struct rspamd_fuzzy_storage_ctx *ctx;
+ gchar *source;
+ gboolean final;
+};
+
+
+static void rspamd_fuzzy_write_reply(struct fuzzy_session *session);
+static gboolean rspamd_fuzzy_process_updates_queue(struct rspamd_fuzzy_storage_ctx *ctx,
+ const gchar *source, gboolean final);
+static gboolean rspamd_fuzzy_check_client(struct rspamd_fuzzy_storage_ctx *ctx,
+ rspamd_inet_addr_t *addr);
+static void rspamd_fuzzy_maybe_call_blacklisted(struct rspamd_fuzzy_storage_ctx *ctx,
+ rspamd_inet_addr_t *addr,
+ const gchar *reason);
+
+static gboolean
+rspamd_fuzzy_check_ratelimit(struct fuzzy_session *session)
+{
+ rspamd_inet_addr_t *masked;
+ struct rspamd_leaky_bucket_elt *elt;
+
+ if (!session->addr) {
+ return TRUE;
+ }
+
+ if (session->ctx->ratelimit_whitelist != NULL) {
+ if (rspamd_match_radix_map_addr(session->ctx->ratelimit_whitelist,
+ session->addr) != NULL) {
+ return TRUE;
+ }
+ }
+
+ /*
+ if (rspamd_inet_address_is_local (session->addr, TRUE)) {
+ return TRUE;
+ }
+ */
+
+ masked = rspamd_inet_address_copy(session->addr, NULL);
+
+ if (rspamd_inet_address_get_af(masked) == AF_INET) {
+ rspamd_inet_address_apply_mask(masked,
+ MIN(session->ctx->leaky_bucket_mask, 32));
+ }
+ else {
+ /* Must be at least /64 */
+ rspamd_inet_address_apply_mask(masked,
+ MIN(MAX(session->ctx->leaky_bucket_mask * 4, 64), 128));
+ }
+
+ elt = rspamd_lru_hash_lookup(session->ctx->ratelimit_buckets, masked,
+ (time_t) session->timestamp);
+
+ if (elt) {
+ gboolean ratelimited = FALSE, new_ratelimit = FALSE;
+
+ if (isnan(elt->cur)) {
+ /* There is an issue with the previous logic: the TTL is updated each time
+ * we see that new bucket. Hence, we need to check the `last` and act accordingly
+ */
+ if (elt->last < session->timestamp && session->timestamp - elt->last >= session->ctx->leaky_bucket_ttl) {
+ /*
+ * We reset bucket to it's 90% capacity to allow some requests
+ * This should cope with the issue when we block an IP network for some burst and never unblock it
+ */
+ elt->cur = session->ctx->leaky_bucket_burst * 0.9;
+ elt->last = session->timestamp;
+ }
+ else {
+ ratelimited = TRUE;
+ }
+ }
+ else {
+ /* Update bucket: leak some elements */
+ if (elt->last < session->timestamp) {
+ elt->cur -= session->ctx->leaky_bucket_rate * (session->timestamp - elt->last);
+ elt->last = session->timestamp;
+
+ if (elt->cur < 0) {
+ elt->cur = 0;
+ }
+ }
+ else {
+ elt->last = session->timestamp;
+ }
+
+ /* Check the bucket */
+ if (elt->cur >= session->ctx->leaky_bucket_burst) {
+
+ msg_info("ratelimiting %s (%s), %.1f max elts",
+ rspamd_inet_address_to_string(session->addr),
+ rspamd_inet_address_to_string(masked),
+ session->ctx->leaky_bucket_burst);
+ elt->cur = NAN;
+ new_ratelimit = TRUE;
+ ratelimited = TRUE;
+ }
+ else {
+ elt->cur++; /* Allow one more request */
+ }
+ }
+
+ if (ratelimited) {
+ rspamd_fuzzy_maybe_call_blacklisted(session->ctx, session->addr, "ratelimit");
+ }
+
+ if (new_ratelimit) {
+ struct rspamd_srv_command srv_cmd;
+
+ srv_cmd.type = RSPAMD_SRV_FUZZY_BLOCKED;
+ srv_cmd.cmd.fuzzy_blocked.af = rspamd_inet_address_get_af(masked);
+
+ if (srv_cmd.cmd.fuzzy_blocked.af == AF_INET || srv_cmd.cmd.fuzzy_blocked.af == AF_INET6) {
+ socklen_t slen;
+ struct sockaddr *sa = rspamd_inet_address_get_sa(masked, &slen);
+
+ if (slen <= sizeof(srv_cmd.cmd.fuzzy_blocked.addr)) {
+ memcpy(&srv_cmd.cmd.fuzzy_blocked.addr, sa, slen);
+ msg_debug("propagating blocked address to other workers");
+ rspamd_srv_send_command(session->worker, session->ctx->event_loop, &srv_cmd, -1, NULL, NULL);
+ }
+ else {
+ msg_err("bad address length: %d, expected to be %d", (int) slen, (int) sizeof(srv_cmd.cmd.fuzzy_blocked.addr));
+ }
+ }
+ }
+
+ rspamd_inet_address_free(masked);
+
+ return !ratelimited;
+ }
+ else {
+ /* New bucket */
+ elt = g_malloc(sizeof(*elt));
+ elt->addr = masked; /* transfer ownership */
+ elt->cur = 1;
+ elt->last = session->timestamp;
+
+ rspamd_lru_hash_insert(session->ctx->ratelimit_buckets,
+ masked,
+ elt,
+ session->timestamp,
+ session->ctx->leaky_bucket_ttl);
+ }
+
+ return TRUE;
+}
+
+static void
+rspamd_fuzzy_maybe_call_blacklisted(struct rspamd_fuzzy_storage_ctx *ctx,
+ rspamd_inet_addr_t *addr,
+ const gchar *reason)
+{
+ if (ctx->lua_blacklist_cbref != -1) {
+ lua_State *L = ctx->cfg->lua_state;
+ gint err_idx, ret;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, ctx->lua_blacklist_cbref);
+ /* client IP */
+ rspamd_lua_ip_push(L, addr);
+ /* block reason */
+ lua_pushstring(L, reason);
+
+ if ((ret = lua_pcall(L, 2, 0, err_idx)) != 0) {
+ msg_err("call to lua_blacklist_cbref "
+ "script failed (%d): %s",
+ ret, lua_tostring(L, -1));
+ }
+
+ lua_settop(L, 0);
+ }
+}
+
+static gboolean
+rspamd_fuzzy_check_client(struct rspamd_fuzzy_storage_ctx *ctx,
+ rspamd_inet_addr_t *addr)
+{
+ if (ctx->blocked_ips != NULL) {
+ if (rspamd_match_radix_map_addr(ctx->blocked_ips,
+ addr) != NULL) {
+
+ rspamd_fuzzy_maybe_call_blacklisted(ctx, addr, "blacklisted");
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_fuzzy_check_write(struct fuzzy_session *session)
+{
+ if (session->ctx->read_only) {
+ return FALSE;
+ }
+
+ if (session->ctx->update_ips != NULL && session->addr) {
+ if (rspamd_inet_address_get_af(session->addr) == AF_UNIX) {
+ return TRUE;
+ }
+ if (rspamd_match_radix_map_addr(session->ctx->update_ips,
+ session->addr) == NULL) {
+ return FALSE;
+ }
+ else {
+ return TRUE;
+ }
+ }
+
+ if (session->ctx->update_keys != NULL && session->key->stat && session->key->key) {
+ static gchar base32_buf[rspamd_cryptobox_HASHBYTES * 2 + 1];
+ guint raw_len;
+ const guchar *pk_raw = rspamd_keypair_component(session->key->key,
+ RSPAMD_KEYPAIR_COMPONENT_ID, &raw_len);
+ gint encoded_len = rspamd_encode_base32_buf(pk_raw, raw_len,
+ base32_buf, sizeof(base32_buf),
+ RSPAMD_BASE32_DEFAULT);
+
+ if (rspamd_match_hash_map(session->ctx->update_keys, base32_buf, encoded_len)) {
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+static void
+fuzzy_key_stat_dtor(gpointer p)
+{
+ struct fuzzy_key_stat *st = p;
+
+ if (st->last_ips) {
+ rspamd_lru_hash_destroy(st->last_ips);
+ }
+
+ if (st->keypair) {
+ rspamd_keypair_unref(st->keypair);
+ }
+
+ g_free(st);
+}
+
+static void
+fuzzy_key_stat_unref(gpointer p)
+{
+ struct fuzzy_key_stat *st = p;
+
+ REF_RELEASE(st);
+}
+
+static void
+fuzzy_key_dtor(gpointer p)
+{
+ struct fuzzy_key *key = p;
+
+ if (key) {
+ if (key->stat) {
+ REF_RELEASE(key->stat);
+ }
+
+ if (key->flags_stat) {
+ kh_destroy(fuzzy_key_flag_stat, key->flags_stat);
+ }
+
+ if (key->forbidden_ids) {
+ kh_destroy(fuzzy_key_ids_set, key->forbidden_ids);
+ }
+
+ g_free(key);
+ }
+}
+
+static void
+fuzzy_count_callback(guint64 count, void *ud)
+{
+ struct rspamd_fuzzy_storage_ctx *ctx = ud;
+
+ ctx->stat.fuzzy_hashes = count;
+}
+
+static void
+fuzzy_rl_bucket_free(gpointer p)
+{
+ struct rspamd_leaky_bucket_elt *elt = (struct rspamd_leaky_bucket_elt *) p;
+
+ rspamd_inet_address_free(elt->addr);
+ g_free(elt);
+}
+
+static void
+fuzzy_stat_count_callback(guint64 count, void *ud)
+{
+ struct rspamd_fuzzy_storage_ctx *ctx = ud;
+
+ ev_timer_again(ctx->event_loop, &ctx->stat_ev);
+ ctx->stat.fuzzy_hashes = count;
+}
+
+static void
+rspamd_fuzzy_stat_callback(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_fuzzy_storage_ctx *ctx =
+ (struct rspamd_fuzzy_storage_ctx *) w->data;
+ rspamd_fuzzy_backend_count(ctx->backend, fuzzy_stat_count_callback, ctx);
+}
+
+
+static void
+fuzzy_update_version_callback(guint64 ver, void *ud)
+{
+}
+
+static void
+rspamd_fuzzy_updates_cb(gboolean success,
+ guint nadded,
+ guint ndeleted,
+ guint nextended,
+ guint nignored,
+ void *ud)
+{
+ struct rspamd_updates_cbdata *cbdata = ud;
+ struct rspamd_fuzzy_storage_ctx *ctx;
+ const gchar *source;
+
+ ctx = cbdata->ctx;
+ source = cbdata->source;
+
+ if (success) {
+ rspamd_fuzzy_backend_count(ctx->backend, fuzzy_count_callback, ctx);
+
+ msg_info("successfully updated fuzzy storage %s: %d updates in queue; "
+ "%d pending currently; "
+ "%d added; %d deleted; %d extended; %d duplicates",
+ ctx->worker->cf->bind_conf ? ctx->worker->cf->bind_conf->bind_line : "unknown",
+ cbdata->updates_pending->len,
+ ctx->updates_pending->len,
+ nadded, ndeleted, nextended, nignored);
+ rspamd_fuzzy_backend_version(ctx->backend, source,
+ fuzzy_update_version_callback, NULL);
+ ctx->updates_failed = 0;
+
+ if (cbdata->final || ctx->worker->state != rspamd_worker_state_running) {
+ /* Plan exit */
+ ev_break(ctx->event_loop, EVBREAK_ALL);
+ }
+ }
+ else {
+ if (++ctx->updates_failed > ctx->updates_maxfail) {
+ msg_err("cannot commit update transaction to fuzzy backend %s, discard "
+ "%ud updates after %d retries",
+ ctx->worker->cf->bind_conf ? ctx->worker->cf->bind_conf->bind_line : "unknown",
+ cbdata->updates_pending->len,
+ ctx->updates_maxfail);
+ ctx->updates_failed = 0;
+
+ if (cbdata->final || ctx->worker->state != rspamd_worker_state_running) {
+ /* Plan exit */
+ ev_break(ctx->event_loop, EVBREAK_ALL);
+ }
+ }
+ else {
+ if (ctx->updates_pending) {
+ msg_err("cannot commit update transaction to fuzzy backend %s; "
+ "%ud updates are still left; %ud currently pending;"
+ " %d retries remaining",
+ ctx->worker->cf->bind_conf ? ctx->worker->cf->bind_conf->bind_line : "unknown",
+ cbdata->updates_pending->len,
+ ctx->updates_pending->len,
+ ctx->updates_maxfail - ctx->updates_failed);
+ /* Move the remaining updates to ctx queue */
+ g_array_append_vals(ctx->updates_pending,
+ cbdata->updates_pending->data,
+ cbdata->updates_pending->len);
+
+ if (cbdata->final) {
+ /* Try one more time */
+ rspamd_fuzzy_process_updates_queue(cbdata->ctx, cbdata->source,
+ cbdata->final);
+ }
+ }
+ else {
+ /* We have lost our ctx, so it is a race condition case */
+ msg_err("cannot commit update transaction to fuzzy backend %s; "
+ "%ud updates are still left; no more retries: a worker is terminated",
+ ctx->worker->cf->bind_conf ? ctx->worker->cf->bind_conf->bind_line : "unknown",
+ cbdata->updates_pending->len);
+ }
+ }
+ }
+
+ g_array_free(cbdata->updates_pending, TRUE);
+ g_free(cbdata->source);
+ g_free(cbdata);
+}
+
+static gboolean
+rspamd_fuzzy_process_updates_queue(struct rspamd_fuzzy_storage_ctx *ctx,
+ const gchar *source, gboolean final)
+{
+
+ struct rspamd_updates_cbdata *cbdata;
+
+ if (ctx->updates_pending->len > 0) {
+ cbdata = g_malloc(sizeof(*cbdata));
+ cbdata->ctx = ctx;
+ cbdata->final = final;
+ cbdata->updates_pending = ctx->updates_pending;
+ ctx->updates_pending = g_array_sized_new(FALSE, FALSE,
+ sizeof(struct fuzzy_peer_cmd),
+ MAX(cbdata->updates_pending->len, 1024));
+ cbdata->source = g_strdup(source);
+ rspamd_fuzzy_backend_process_updates(ctx->backend,
+ cbdata->updates_pending,
+ source, rspamd_fuzzy_updates_cb, cbdata);
+ return TRUE;
+ }
+ else if (final) {
+ /* No need to sync */
+ ev_break(ctx->event_loop, EVBREAK_ALL);
+ }
+
+ return FALSE;
+}
+
+static void
+rspamd_fuzzy_reply_io(EV_P_ ev_io *w, int revents)
+{
+ struct fuzzy_session *session = (struct fuzzy_session *) w->data;
+
+ ev_io_stop(EV_A_ w);
+ rspamd_fuzzy_write_reply(session);
+ REF_RELEASE(session);
+}
+
+static void
+rspamd_fuzzy_write_reply(struct fuzzy_session *session)
+{
+ gssize r;
+ gsize len;
+ gconstpointer data;
+
+ if (session->cmd_type == CMD_ENCRYPTED_NORMAL ||
+ session->cmd_type == CMD_ENCRYPTED_SHINGLE) {
+ /* Encrypted reply */
+ data = &session->reply;
+
+ if (session->epoch > RSPAMD_FUZZY_EPOCH10) {
+ len = sizeof(session->reply);
+ }
+ else {
+ len = sizeof(session->reply.hdr) + sizeof(session->reply.rep.v1);
+ }
+ }
+ else {
+ data = &session->reply.rep;
+
+ if (session->epoch > RSPAMD_FUZZY_EPOCH10) {
+ len = sizeof(session->reply.rep);
+ }
+ else {
+ len = sizeof(session->reply.rep.v1);
+ }
+ }
+
+ r = rspamd_inet_address_sendto(session->fd, data, len, 0,
+ session->addr);
+
+ if (r == -1) {
+ if (errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) {
+ /* Grab reference to avoid early destruction */
+ REF_RETAIN(session);
+ session->io.data = session;
+ ev_io_init(&session->io,
+ rspamd_fuzzy_reply_io, session->fd, EV_WRITE);
+ ev_io_start(session->ctx->event_loop, &session->io);
+ }
+ else {
+ msg_err("error while writing reply: %s", strerror(errno));
+ }
+ }
+}
+
+static void
+rspamd_fuzzy_update_key_stat(gboolean matched,
+ struct fuzzy_key_stat *key_stat,
+ guint cmd,
+ struct rspamd_fuzzy_reply *res,
+ ev_tstamp timestamp)
+{
+ if (!matched && res->v1.value != 0) {
+ key_stat->errors++;
+ }
+ else {
+ if (cmd == FUZZY_CHECK) {
+ key_stat->checked++;
+
+ if (matched) {
+ key_stat->matched++;
+ }
+
+ if (G_UNLIKELY(key_stat->last_checked_time == 0.0)) {
+ key_stat->last_checked_time = timestamp;
+ key_stat->last_checked_count = key_stat->checked;
+ key_stat->last_matched_count = key_stat->matched;
+ }
+ else if (G_UNLIKELY(timestamp > key_stat->last_checked_time + KEY_STAT_INTERVAL)) {
+ guint64 nchecked = key_stat->checked - key_stat->last_checked_count;
+ guint64 nmatched = key_stat->matched - key_stat->last_matched_count;
+
+ rspamd_set_counter_ema(&key_stat->checked_ctr, nchecked, 0.5f);
+ rspamd_set_counter_ema(&key_stat->matched_ctr, nmatched, 0.5f);
+ key_stat->last_checked_time = timestamp;
+ key_stat->last_checked_count = key_stat->checked;
+ key_stat->last_matched_count = key_stat->matched;
+ }
+ }
+ else if (cmd == FUZZY_WRITE) {
+ key_stat->added++;
+ }
+ else if (cmd == FUZZY_DEL) {
+ key_stat->deleted++;
+ }
+ }
+}
+
+static void
+rspamd_fuzzy_update_stats(struct rspamd_fuzzy_storage_ctx *ctx,
+ enum rspamd_fuzzy_epoch epoch,
+ gboolean matched,
+ gboolean is_shingle,
+ gboolean is_delayed,
+ struct fuzzy_key *key,
+ struct fuzzy_key_stat *ip_stat,
+ guint cmd,
+ struct rspamd_fuzzy_reply *res,
+ ev_tstamp timestamp)
+{
+ ctx->stat.fuzzy_hashes_checked[epoch]++;
+
+ if (matched) {
+ ctx->stat.fuzzy_hashes_found[epoch]++;
+ }
+ if (is_shingle) {
+ ctx->stat.fuzzy_shingles_checked[epoch]++;
+ }
+ if (is_delayed) {
+ ctx->stat.delayed_hashes++;
+ }
+
+ if (key) {
+ rspamd_fuzzy_update_key_stat(matched, key->stat, cmd, res, timestamp);
+
+ if (matched || ((cmd == FUZZY_WRITE || cmd == FUZZY_DEL) && res->v1.value == 0)) {
+ /* Update per flag stats */
+ khiter_t k;
+ struct fuzzy_key_stat *flag_stat;
+ k = kh_get(fuzzy_key_flag_stat, key->flags_stat, res->v1.flag);
+
+ if (k == kh_end(key->flags_stat)) {
+ /* Insert new flag */
+ int r;
+ k = kh_put(fuzzy_key_flag_stat, key->flags_stat, res->v1.flag, &r);
+ memset(&kh_value(key->flags_stat, k), 0, sizeof(struct fuzzy_key_stat));
+ }
+
+ flag_stat = &kh_value(key->flags_stat, k);
+ rspamd_fuzzy_update_key_stat(matched, flag_stat, cmd, res, timestamp);
+ }
+ }
+
+ if (ip_stat) {
+ if (!matched && res->v1.value != 0) {
+ ip_stat->errors++;
+ }
+ else {
+ if (cmd == FUZZY_CHECK) {
+ ip_stat->checked++;
+
+ if (matched) {
+ ip_stat->matched++;
+ }
+ }
+ else if (cmd == FUZZY_WRITE) {
+ ip_stat->added++;
+ }
+ else if (cmd == FUZZY_DEL) {
+ ip_stat->deleted++;
+ }
+ }
+ }
+}
+
+enum rspamd_fuzzy_reply_flags {
+ RSPAMD_FUZZY_REPLY_ENCRYPTED = 0x1u << 0u,
+ RSPAMD_FUZZY_REPLY_SHINGLE = 0x1u << 1u,
+ RSPAMD_FUZZY_REPLY_DELAY = 0x1u << 2u,
+};
+
+static void
+rspamd_fuzzy_make_reply(struct rspamd_fuzzy_cmd *cmd,
+ struct rspamd_fuzzy_reply *result,
+ struct fuzzy_session *session,
+ gint flags)
+{
+ gsize len;
+
+ if (cmd) {
+ result->v1.tag = cmd->tag;
+ memcpy(&session->reply.rep, result, sizeof(*result));
+
+ if (flags & RSPAMD_FUZZY_REPLY_DELAY) {
+ /* Hash is too fresh, need to delay it */
+ session->reply.rep.ts = 0;
+ session->reply.rep.v1.prob = 0.0f;
+ session->reply.rep.v1.value = 0;
+ }
+
+ bool default_disabled = false;
+
+ {
+ khiter_t k;
+
+ k = kh_get(fuzzy_key_ids_set, session->ctx->default_forbidden_ids, session->reply.rep.v1.flag);
+
+ if (k != kh_end(session->ctx->default_forbidden_ids)) {
+ /* Hash is from a forbidden flag by default */
+ default_disabled = true;
+ }
+ }
+
+ if (flags & RSPAMD_FUZZY_REPLY_ENCRYPTED) {
+
+ if (session->reply.rep.v1.prob > 0 && session->key && session->key->forbidden_ids) {
+ khiter_t k;
+
+ k = kh_get(fuzzy_key_ids_set, session->key->forbidden_ids, session->reply.rep.v1.flag);
+
+ if (k != kh_end(session->key->forbidden_ids)) {
+ /* Hash is from a forbidden flag for this key */
+ session->reply.rep.ts = 0;
+ session->reply.rep.v1.prob = 0.0f;
+ session->reply.rep.v1.value = 0;
+ session->reply.rep.v1.flag = 0;
+ }
+ }
+ else if (default_disabled) {
+ /* Hash is from a forbidden flag by default */
+ session->reply.rep.ts = 0;
+ session->reply.rep.v1.prob = 0.0f;
+ session->reply.rep.v1.value = 0;
+ session->reply.rep.v1.flag = 0;
+ }
+
+ /* We need also to encrypt reply */
+ ottery_rand_bytes(session->reply.hdr.nonce,
+ sizeof(session->reply.hdr.nonce));
+
+ /*
+ * For old replies we need to encrypt just old part, otherwise
+ * decryption would fail due to mac verification mistake
+ */
+
+ if (session->epoch > RSPAMD_FUZZY_EPOCH10) {
+ len = sizeof(session->reply.rep);
+ }
+ else {
+ len = sizeof(session->reply.rep.v1);
+ }
+
+ /* Update stats before encryption */
+ if (cmd->cmd != FUZZY_STAT && cmd->cmd <= FUZZY_CLIENT_MAX) {
+ rspamd_fuzzy_update_stats(session->ctx,
+ session->epoch,
+ session->reply.rep.v1.prob > 0.5f,
+ flags & RSPAMD_FUZZY_REPLY_SHINGLE,
+ flags & RSPAMD_FUZZY_REPLY_DELAY,
+ session->key,
+ session->ip_stat,
+ cmd->cmd,
+ &session->reply.rep,
+ session->timestamp);
+ }
+
+ rspamd_cryptobox_encrypt_nm_inplace((guchar *) &session->reply.rep,
+ len,
+ session->reply.hdr.nonce,
+ session->nm,
+ session->reply.hdr.mac,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+ }
+ else if (default_disabled) {
+ /* Hash is from a forbidden flag by default, and there is no encryption override */
+ session->reply.rep.ts = 0;
+ session->reply.rep.v1.prob = 0.0f;
+ session->reply.rep.v1.value = 0;
+ session->reply.rep.v1.flag = 0;
+ }
+ if (!(flags & RSPAMD_FUZZY_REPLY_ENCRYPTED)) {
+ if (cmd->cmd != FUZZY_STAT && cmd->cmd <= FUZZY_CLIENT_MAX) {
+ rspamd_fuzzy_update_stats(session->ctx,
+ session->epoch,
+ session->reply.rep.v1.prob > 0.5f,
+ flags & RSPAMD_FUZZY_REPLY_SHINGLE,
+ flags & RSPAMD_FUZZY_REPLY_DELAY,
+ session->key,
+ session->ip_stat,
+ cmd->cmd,
+ &session->reply.rep,
+ session->timestamp);
+ }
+ }
+ }
+
+ rspamd_fuzzy_write_reply(session);
+}
+
+static gboolean
+fuzzy_peer_try_send(gint fd, struct fuzzy_peer_request *up_req)
+{
+ gssize r;
+
+ r = write(fd, &up_req->cmd, sizeof(up_req->cmd));
+
+ if (r != sizeof(up_req->cmd)) {
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static void
+fuzzy_peer_send_io(EV_P_ ev_io *w, int revents)
+{
+ struct fuzzy_peer_request *up_req = (struct fuzzy_peer_request *) w->data;
+
+ if (!fuzzy_peer_try_send(w->fd, up_req)) {
+ msg_err("cannot send update request to the peer: %s", strerror(errno));
+ }
+
+ ev_io_stop(EV_A_ w);
+ g_free(up_req);
+}
+
+static void
+rspamd_fuzzy_extensions_tolua(lua_State *L,
+ struct fuzzy_session *session)
+{
+ struct rspamd_fuzzy_cmd_extension *ext;
+ rspamd_inet_addr_t *addr;
+
+ lua_createtable(L, 0, 0);
+
+ LL_FOREACH(session->extensions, ext)
+ {
+ switch (ext->ext) {
+ case RSPAMD_FUZZY_EXT_SOURCE_DOMAIN:
+ lua_pushlstring(L, ext->payload, ext->length);
+ lua_setfield(L, -2, "domain");
+ break;
+ case RSPAMD_FUZZY_EXT_SOURCE_IP4:
+ addr = rspamd_inet_address_new(AF_INET, ext->payload);
+ rspamd_lua_ip_push(L, addr);
+ rspamd_inet_address_free(addr);
+ lua_setfield(L, -2, "ip");
+ break;
+ case RSPAMD_FUZZY_EXT_SOURCE_IP6:
+ addr = rspamd_inet_address_new(AF_INET6, ext->payload);
+ rspamd_lua_ip_push(L, addr);
+ rspamd_inet_address_free(addr);
+ lua_setfield(L, -2, "ip");
+ break;
+ }
+ }
+}
+
+static void
+rspamd_fuzzy_check_callback(struct rspamd_fuzzy_reply *result, void *ud)
+{
+ struct fuzzy_session *session = ud;
+ gboolean is_shingle = FALSE, __attribute__((unused)) encrypted = FALSE;
+ struct rspamd_fuzzy_cmd *cmd = NULL;
+ const struct rspamd_shingle *shingle = NULL;
+ struct rspamd_shingle sgl_cpy;
+ gint send_flags = 0;
+
+ switch (session->cmd_type) {
+ case CMD_ENCRYPTED_NORMAL:
+ encrypted = TRUE;
+ send_flags |= RSPAMD_FUZZY_REPLY_ENCRYPTED;
+ /* Fallthrough */
+ case CMD_NORMAL:
+ cmd = &session->cmd.basic;
+ break;
+
+ case CMD_ENCRYPTED_SHINGLE:
+ encrypted = TRUE;
+ send_flags |= RSPAMD_FUZZY_REPLY_ENCRYPTED;
+ /* Fallthrough */
+ case CMD_SHINGLE:
+ cmd = &session->cmd.basic;
+ memcpy(&sgl_cpy, &session->cmd.sgl, sizeof(sgl_cpy));
+ shingle = &sgl_cpy;
+ is_shingle = TRUE;
+ send_flags |= RSPAMD_FUZZY_REPLY_SHINGLE;
+ break;
+ }
+
+ if (session->ctx->lua_post_handler_cbref != -1) {
+ /* Start lua post handler */
+ lua_State *L = session->ctx->cfg->lua_state;
+ gint err_idx, ret;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ /* Preallocate stack (small opt) */
+ lua_checkstack(L, err_idx + 9);
+ /* function */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, session->ctx->lua_post_handler_cbref);
+ /* client IP */
+ if (session->addr) {
+ rspamd_lua_ip_push(L, session->addr);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ /* client command */
+ lua_pushinteger(L, cmd->cmd);
+ /* command value (push as rspamd_text) */
+ (void) lua_new_text(L, result->digest, sizeof(result->digest), FALSE);
+ /* is shingle */
+ lua_pushboolean(L, is_shingle);
+ /* result value */
+ lua_pushinteger(L, result->v1.value);
+ /* result probability */
+ lua_pushnumber(L, result->v1.prob);
+ /* result flag */
+ lua_pushinteger(L, result->v1.flag);
+ /* result timestamp */
+ lua_pushinteger(L, result->ts);
+ /* TODO: add additional data maybe (encryption, pubkey, etc) */
+ rspamd_fuzzy_extensions_tolua(L, session);
+
+ if ((ret = lua_pcall(L, 9, LUA_MULTRET, err_idx)) != 0) {
+ msg_err("call to lua_post_handler lua "
+ "script failed (%d): %s",
+ ret, lua_tostring(L, -1));
+ }
+ else {
+ /* Return values order:
+ * the first reply will be on err_idx + 1
+ * if it is true, then we need to read the former ones:
+ * 2-nd will be reply code
+ * 3-rd will be probability (or 0.0 if missing)
+ * 4-th value is flag (or default flag if missing)
+ */
+ ret = lua_toboolean(L, err_idx + 1);
+
+ if (ret) {
+ /* Artificial reply */
+ result->v1.value = lua_tointeger(L, err_idx + 2);
+
+ if (lua_isnumber(L, err_idx + 3)) {
+ result->v1.prob = lua_tonumber(L, err_idx + 3);
+ }
+ else {
+ result->v1.prob = 0.0f;
+ }
+
+ if (lua_isnumber(L, err_idx + 4)) {
+ result->v1.flag = lua_tointeger(L, err_idx + 4);
+ }
+
+ lua_settop(L, 0);
+ rspamd_fuzzy_make_reply(cmd, result, session, send_flags);
+ REF_RELEASE(session);
+
+ return;
+ }
+ }
+
+ lua_settop(L, 0);
+ }
+
+ if (!isnan(session->ctx->delay) &&
+ rspamd_match_radix_map_addr(session->ctx->delay_whitelist,
+ session->addr) == NULL) {
+ gdouble hash_age = rspamd_get_calendar_ticks() - result->ts;
+ gdouble jittered_age = rspamd_time_jitter(session->ctx->delay,
+ session->ctx->delay / 2.0);
+
+ if (hash_age < jittered_age) {
+ send_flags |= RSPAMD_FUZZY_REPLY_DELAY;
+ }
+ }
+
+ /* Refresh hash if found with strong confidence */
+ if (result->v1.prob > 0.9 && !session->ctx->read_only) {
+ struct fuzzy_peer_cmd up_cmd;
+ struct fuzzy_peer_request *up_req;
+
+ if (session->worker->index == 0) {
+ /* Just add to the queue */
+ memset(&up_cmd, 0, sizeof(up_cmd));
+ up_cmd.is_shingle = is_shingle;
+ memcpy(up_cmd.cmd.normal.digest, result->digest,
+ sizeof(up_cmd.cmd.normal.digest));
+ up_cmd.cmd.normal.flag = result->v1.flag;
+ up_cmd.cmd.normal.cmd = FUZZY_REFRESH;
+ up_cmd.cmd.normal.shingles_count = cmd->shingles_count;
+
+ if (is_shingle && shingle) {
+ memcpy(&up_cmd.cmd.shingle.sgl, shingle,
+ sizeof(up_cmd.cmd.shingle.sgl));
+ }
+
+ g_array_append_val(session->ctx->updates_pending, up_cmd);
+ }
+ else {
+ /* We need to send request to the peer */
+ up_req = g_malloc0(sizeof(*up_req));
+ up_req->cmd.is_shingle = is_shingle;
+
+ memcpy(up_req->cmd.cmd.normal.digest, result->digest,
+ sizeof(up_req->cmd.cmd.normal.digest));
+ up_req->cmd.cmd.normal.flag = result->v1.flag;
+ up_req->cmd.cmd.normal.cmd = FUZZY_REFRESH;
+ up_req->cmd.cmd.normal.shingles_count = cmd->shingles_count;
+
+ if (is_shingle && shingle) {
+ memcpy(&up_req->cmd.cmd.shingle.sgl, shingle,
+ sizeof(up_req->cmd.cmd.shingle.sgl));
+ }
+
+ if (!fuzzy_peer_try_send(session->ctx->peer_fd, up_req)) {
+ up_req->io_ev.data = up_req;
+ ev_io_init(&up_req->io_ev, fuzzy_peer_send_io,
+ session->ctx->peer_fd, EV_WRITE);
+ ev_io_start(session->ctx->event_loop, &up_req->io_ev);
+ }
+ else {
+ g_free(up_req);
+ }
+ }
+ }
+
+ rspamd_fuzzy_make_reply(cmd, result, session, send_flags);
+
+ REF_RELEASE(session);
+}
+
+static void
+rspamd_fuzzy_process_command(struct fuzzy_session *session)
+{
+ gboolean is_shingle = FALSE, __attribute__((unused)) encrypted = FALSE;
+ struct rspamd_fuzzy_cmd *cmd = NULL;
+ struct rspamd_fuzzy_reply result;
+ struct fuzzy_peer_cmd up_cmd;
+ struct fuzzy_peer_request *up_req;
+ struct fuzzy_key_stat *ip_stat = NULL;
+ gchar hexbuf[rspamd_cryptobox_HASHBYTES * 2 + 1];
+ rspamd_inet_addr_t *naddr;
+ gpointer ptr;
+ gsize up_len = 0;
+ gint send_flags = 0;
+
+ cmd = &session->cmd.basic;
+
+ switch (session->cmd_type) {
+ case CMD_NORMAL:
+ up_len = sizeof(session->cmd.basic);
+ break;
+ case CMD_SHINGLE:
+ up_len = sizeof(session->cmd);
+ is_shingle = TRUE;
+ send_flags |= RSPAMD_FUZZY_REPLY_SHINGLE;
+ break;
+ case CMD_ENCRYPTED_NORMAL:
+ up_len = sizeof(session->cmd.basic);
+ encrypted = TRUE;
+ send_flags |= RSPAMD_FUZZY_REPLY_ENCRYPTED;
+ break;
+ case CMD_ENCRYPTED_SHINGLE:
+ up_len = sizeof(session->cmd);
+ encrypted = TRUE;
+ is_shingle = TRUE;
+ send_flags |= RSPAMD_FUZZY_REPLY_SHINGLE | RSPAMD_FUZZY_REPLY_ENCRYPTED;
+ break;
+ default:
+ msg_err("invalid command type: %d", session->cmd_type);
+ return;
+ }
+
+ memset(&result, 0, sizeof(result));
+ memcpy(result.digest, cmd->digest, sizeof(result.digest));
+ result.v1.flag = cmd->flag;
+ result.v1.tag = cmd->tag;
+
+ if (session->ctx->lua_pre_handler_cbref != -1) {
+ /* Start lua pre handler */
+ lua_State *L = session->ctx->cfg->lua_state;
+ gint err_idx, ret;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ /* Preallocate stack (small opt) */
+ lua_checkstack(L, err_idx + 5);
+ /* function */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, session->ctx->lua_pre_handler_cbref);
+ /* client IP */
+ rspamd_lua_ip_push(L, session->addr);
+ /* client command */
+ lua_pushinteger(L, cmd->cmd);
+ /* command value (push as rspamd_text) */
+ (void) lua_new_text(L, cmd->digest, sizeof(cmd->digest), FALSE);
+ /* is shingle */
+ lua_pushboolean(L, is_shingle);
+ /* TODO: add additional data maybe (encryption, pubkey, etc) */
+ rspamd_fuzzy_extensions_tolua(L, session);
+
+ if ((ret = lua_pcall(L, 5, LUA_MULTRET, err_idx)) != 0) {
+ msg_err("call to lua_pre_handler lua "
+ "script failed (%d): %s",
+ ret, lua_tostring(L, -1));
+ }
+ else {
+ /* Return values order:
+ * the first reply will be on err_idx + 1
+ * if it is true, then we need to read the former ones:
+ * 2-nd will be reply code
+ * 3-rd will be probability (or 0.0 if missing)
+ */
+ ret = lua_toboolean(L, err_idx + 1);
+
+ if (ret) {
+ /* Artificial reply */
+ result.v1.value = lua_tointeger(L, err_idx + 2);
+
+ if (lua_isnumber(L, err_idx + 3)) {
+ result.v1.prob = lua_tonumber(L, err_idx + 3);
+ }
+ else {
+ result.v1.prob = 0.0f;
+ }
+
+ lua_settop(L, 0);
+ rspamd_fuzzy_make_reply(cmd, &result, session, send_flags);
+
+ return;
+ }
+ }
+
+ lua_settop(L, 0);
+ }
+
+
+ if (G_UNLIKELY(cmd == NULL || up_len == 0)) {
+ result.v1.value = 500;
+ result.v1.prob = 0.0f;
+ rspamd_fuzzy_make_reply(cmd, &result, session, send_flags);
+ return;
+ }
+
+ if (session->ctx->encrypted_only && !encrypted) {
+ /* Do not accept unencrypted commands */
+ result.v1.value = 403;
+ result.v1.prob = 0.0f;
+ rspamd_fuzzy_make_reply(cmd, &result, session, send_flags);
+ return;
+ }
+
+ if (session->key && session->addr) {
+ ip_stat = rspamd_lru_hash_lookup(session->key->stat->last_ips,
+ session->addr, -1);
+
+ if (ip_stat == NULL) {
+ naddr = rspamd_inet_address_copy(session->addr, NULL);
+ ip_stat = g_malloc0(sizeof(*ip_stat));
+ REF_INIT_RETAIN(ip_stat, fuzzy_key_stat_dtor);
+ rspamd_lru_hash_insert(session->key->stat->last_ips,
+ naddr, ip_stat, -1, 0);
+ }
+
+ REF_RETAIN(ip_stat);
+ session->ip_stat = ip_stat;
+ }
+
+ if (cmd->cmd == FUZZY_CHECK) {
+ bool can_continue = true;
+
+ if (session->ctx->ratelimit_buckets) {
+ if (session->ctx->ratelimit_log_only) {
+ (void) rspamd_fuzzy_check_ratelimit(session); /* Check but ignore */
+ }
+ else {
+ can_continue = rspamd_fuzzy_check_ratelimit(session);
+ }
+ }
+
+ if (can_continue) {
+ REF_RETAIN(session);
+ rspamd_fuzzy_backend_check(session->ctx->backend, cmd,
+ rspamd_fuzzy_check_callback, session);
+ }
+ else {
+ result.v1.value = 403;
+ result.v1.prob = 0.0f;
+ result.v1.flag = 0;
+ rspamd_fuzzy_make_reply(cmd, &result, session, send_flags);
+ }
+ }
+ else if (cmd->cmd == FUZZY_STAT) {
+ /* Store approximation (if needed) */
+ result.v1.prob = session->ctx->stat.fuzzy_hashes;
+ /* Store high qword in value and low qword in flag */
+ result.v1.value = (gint32) ((guint64) session->ctx->stat.fuzzy_hashes >> 32);
+ result.v1.flag = (guint32) (session->ctx->stat.fuzzy_hashes & G_MAXUINT32);
+ rspamd_fuzzy_make_reply(cmd, &result, session, send_flags);
+ }
+ else if (cmd->cmd == FUZZY_PING) {
+ result.v1.prob = 1.0f;
+ result.v1.value = cmd->value;
+ rspamd_fuzzy_make_reply(cmd, &result, session, send_flags);
+ }
+ else {
+ if (rspamd_fuzzy_check_write(session)) {
+ /* Check whitelist */
+ if (session->ctx->skip_hashes && cmd->cmd == FUZZY_WRITE) {
+ rspamd_encode_hex_buf(cmd->digest, sizeof(cmd->digest),
+ hexbuf, sizeof(hexbuf) - 1);
+ hexbuf[sizeof(hexbuf) - 1] = '\0';
+
+ if (rspamd_match_hash_map(session->ctx->skip_hashes,
+ hexbuf, sizeof(hexbuf) - 1)) {
+ result.v1.value = 401;
+ result.v1.prob = 0.0f;
+
+ goto reply;
+ }
+ }
+
+ if (session->ctx->weak_ids && kh_get(fuzzy_key_ids_set, session->ctx->weak_ids, cmd->flag) != kh_end(session->ctx->weak_ids)) {
+ /* Flag command as weak */
+ cmd->version |= RSPAMD_FUZZY_FLAG_WEAK;
+ }
+
+ if (session->worker->index == 0 || session->ctx->peer_fd == -1) {
+ /* Just add to the queue */
+ up_cmd.is_shingle = is_shingle;
+ ptr = is_shingle ? (gpointer) &up_cmd.cmd.shingle : (gpointer) &up_cmd.cmd.normal;
+ memcpy(ptr, cmd, up_len);
+ g_array_append_val(session->ctx->updates_pending, up_cmd);
+ }
+ else {
+ /* We need to send request to the peer */
+ up_req = g_malloc0(sizeof(*up_req));
+ up_req->cmd.is_shingle = is_shingle;
+ ptr = is_shingle ? (gpointer) &up_req->cmd.cmd.shingle : (gpointer) &up_req->cmd.cmd.normal;
+ memcpy(ptr, cmd, up_len);
+
+ if (!fuzzy_peer_try_send(session->ctx->peer_fd, up_req)) {
+ up_req->io_ev.data = up_req;
+ ev_io_init(&up_req->io_ev, fuzzy_peer_send_io,
+ session->ctx->peer_fd, EV_WRITE);
+ ev_io_start(session->ctx->event_loop, &up_req->io_ev);
+ }
+ else {
+ g_free(up_req);
+ }
+ }
+
+ result.v1.value = 0;
+ result.v1.prob = 1.0f;
+ }
+ else {
+ result.v1.value = 403;
+ result.v1.prob = 0.0f;
+ }
+ reply:
+ rspamd_fuzzy_make_reply(cmd, &result, session, send_flags);
+ }
+}
+
+
+static enum rspamd_fuzzy_epoch
+rspamd_fuzzy_command_valid(struct rspamd_fuzzy_cmd *cmd, gint r)
+{
+ enum rspamd_fuzzy_epoch ret = RSPAMD_FUZZY_EPOCH_MAX;
+
+ switch (cmd->version & RSPAMD_FUZZY_VERSION_MASK) {
+ case 4:
+ if (cmd->shingles_count > 0) {
+ if (r >= sizeof(struct rspamd_fuzzy_shingle_cmd)) {
+ ret = RSPAMD_FUZZY_EPOCH11;
+ }
+ }
+ else {
+ if (r >= sizeof(*cmd)) {
+ ret = RSPAMD_FUZZY_EPOCH11;
+ }
+ }
+ break;
+ case 3:
+ if (cmd->shingles_count > 0) {
+ if (r == sizeof(struct rspamd_fuzzy_shingle_cmd)) {
+ ret = RSPAMD_FUZZY_EPOCH10;
+ }
+ }
+ else {
+ if (r == sizeof(*cmd)) {
+ ret = RSPAMD_FUZZY_EPOCH10;
+ }
+ }
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+static gboolean
+rspamd_fuzzy_decrypt_command(struct fuzzy_session *s, guchar *buf, gsize buflen)
+{
+ struct rspamd_fuzzy_encrypted_req_hdr hdr;
+ struct rspamd_cryptobox_pubkey *rk;
+ struct fuzzy_key *key;
+
+ if (s->ctx->default_key == NULL) {
+ msg_warn("received encrypted request when encryption is not enabled");
+ return FALSE;
+ }
+
+ if (buflen < sizeof(hdr)) {
+ msg_warn("XXX: should not be reached");
+ return FALSE;
+ }
+
+ memcpy(&hdr, buf, sizeof(hdr));
+ buf += sizeof(hdr);
+ buflen -= sizeof(hdr);
+
+ /* Try to find the desired key */
+ key = g_hash_table_lookup(s->ctx->keys, hdr.key_id);
+
+ if (key == NULL) {
+ /* Unknown key, assume default one */
+ key = s->ctx->default_key;
+ }
+
+ s->key = key;
+
+ /* Now process keypair */
+ rk = rspamd_pubkey_from_bin(hdr.pubkey, sizeof(hdr.pubkey),
+ RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
+
+ if (rk == NULL) {
+ msg_err("bad key; ip=%s",
+ rspamd_inet_address_to_string(s->addr));
+ return FALSE;
+ }
+
+ rspamd_keypair_cache_process(s->ctx->keypair_cache, key->key, rk);
+
+ /* Now decrypt request */
+ if (!rspamd_cryptobox_decrypt_nm_inplace(buf, buflen, hdr.nonce,
+ rspamd_pubkey_get_nm(rk, key->key),
+ hdr.mac, RSPAMD_CRYPTOBOX_MODE_25519)) {
+ msg_err("decryption failed; ip=%s",
+ rspamd_inet_address_to_string(s->addr));
+ rspamd_pubkey_unref(rk);
+
+ return FALSE;
+ }
+
+ memcpy(s->nm, rspamd_pubkey_get_nm(rk, key->key), sizeof(s->nm));
+ rspamd_pubkey_unref(rk);
+
+ return TRUE;
+}
+
+
+static gboolean
+rspamd_fuzzy_extensions_from_wire(struct fuzzy_session *s, guchar *buf, gsize buflen)
+{
+ struct rspamd_fuzzy_cmd_extension *ext, *prev_ext;
+ guchar *storage, *p = buf, *end = buf + buflen;
+ gsize st_len = 0, n_ext = 0;
+
+ /* Read number of extensions to allocate array */
+ while (p < end) {
+ guchar cmd = *p++;
+
+ if (p < end) {
+ if (cmd == RSPAMD_FUZZY_EXT_SOURCE_DOMAIN) {
+ /* Next byte is buf length */
+ guchar dom_len = *p++;
+
+ if (dom_len <= (end - p)) {
+ st_len += dom_len;
+ n_ext++;
+ }
+ else {
+ /* Truncation */
+ return FALSE;
+ }
+
+ p += dom_len;
+ }
+ else if (cmd == RSPAMD_FUZZY_EXT_SOURCE_IP4) {
+ if (end - p >= sizeof(in_addr_t)) {
+ n_ext++;
+ st_len += sizeof(in_addr_t);
+ }
+ else {
+ /* Truncation */
+ return FALSE;
+ }
+
+ p += sizeof(in_addr_t);
+ }
+ else if (cmd == RSPAMD_FUZZY_EXT_SOURCE_IP6) {
+ if (end - p >= sizeof(struct in6_addr)) {
+ n_ext++;
+ st_len += sizeof(struct in6_addr);
+ }
+ else {
+ /* Truncation */
+ return FALSE;
+ }
+
+ p += sizeof(struct in6_addr);
+ }
+ else {
+ /* Invalid command */
+ return FALSE;
+ }
+ }
+ else {
+ /* Truncated extension */
+ return FALSE;
+ }
+ }
+
+ if (n_ext > 0) {
+ p = buf;
+ /*
+ * Memory layout: n_ext of struct rspamd_fuzzy_cmd_extension
+ * payload for each extension in a continuous data segment
+ */
+ storage = g_malloc(n_ext * sizeof(struct rspamd_fuzzy_cmd_extension) +
+ st_len);
+
+ guchar *data_buf = storage +
+ n_ext * sizeof(struct rspamd_fuzzy_cmd_extension);
+ ext = (struct rspamd_fuzzy_cmd_extension *) storage;
+
+ /* All validation has been done, so we can just go further */
+ while (p < end) {
+ prev_ext = ext;
+ guchar cmd = *p++;
+
+ if (cmd == RSPAMD_FUZZY_EXT_SOURCE_DOMAIN) {
+ /* Next byte is buf length */
+ guchar dom_len = *p++;
+ guchar *dest = data_buf;
+
+ ext->ext = RSPAMD_FUZZY_EXT_SOURCE_DOMAIN;
+ ext->next = ext + 1;
+ ext->length = dom_len;
+ ext->payload = dest;
+ memcpy(dest, p, dom_len);
+ p += dom_len;
+ data_buf += dom_len;
+ ext = ext->next;
+ }
+ else if (cmd == RSPAMD_FUZZY_EXT_SOURCE_IP4) {
+ guchar *dest = data_buf;
+
+ ext->ext = RSPAMD_FUZZY_EXT_SOURCE_IP4;
+ ext->next = ext + 1;
+ ext->length = sizeof(in_addr_t);
+ ext->payload = dest;
+ memcpy(dest, p, sizeof(in_addr_t));
+ p += sizeof(in_addr_t);
+ data_buf += sizeof(in_addr_t);
+ ext = ext->next;
+ }
+ else if (cmd == RSPAMD_FUZZY_EXT_SOURCE_IP6) {
+ guchar *dest = data_buf;
+
+ ext->ext = RSPAMD_FUZZY_EXT_SOURCE_IP6;
+ ext->next = ext + 1;
+ ext->length = sizeof(struct in6_addr);
+ ext->payload = dest;
+ memcpy(dest, p, sizeof(struct in6_addr));
+ p += sizeof(struct in6_addr);
+ data_buf += sizeof(struct in6_addr);
+ ext = ext->next;
+ }
+ else {
+ g_assert_not_reached();
+ }
+ }
+
+ /* Last next should be NULL */
+ prev_ext->next = NULL;
+
+ /* Rewind to the begin */
+ ext = (struct rspamd_fuzzy_cmd_extension *) storage;
+ s->extensions = ext;
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_fuzzy_cmd_from_wire(guchar *buf, guint buflen, struct fuzzy_session *s)
+{
+ enum rspamd_fuzzy_epoch epoch;
+ gboolean encrypted = FALSE;
+
+ if (buflen < sizeof(struct rspamd_fuzzy_cmd)) {
+ msg_debug("truncated fuzzy command of size %d received", buflen);
+ return FALSE;
+ }
+
+ /* Now check encryption */
+
+ if (buflen >= sizeof(struct rspamd_fuzzy_encrypted_cmd)) {
+ if (memcmp(buf, fuzzy_encrypted_magic, sizeof(fuzzy_encrypted_magic)) == 0) {
+ /* Encrypted command */
+ encrypted = TRUE;
+ }
+ }
+
+ if (encrypted) {
+ /* Decrypt first */
+ if (!rspamd_fuzzy_decrypt_command(s, buf, buflen)) {
+ return FALSE;
+ }
+ else {
+ /*
+ * Advance buffer to skip encrypted header.
+ * Note that after rspamd_fuzzy_decrypt_command buf is unencrypted
+ */
+ buf += sizeof(struct rspamd_fuzzy_encrypted_req_hdr);
+ buflen -= sizeof(struct rspamd_fuzzy_encrypted_req_hdr);
+ }
+ }
+
+ /* Fill the normal command */
+ if (buflen < sizeof(s->cmd.basic)) {
+ msg_debug("truncated normal fuzzy command of size %d received", buflen);
+ return FALSE;
+ }
+
+ memcpy(&s->cmd.basic, buf, sizeof(s->cmd.basic));
+ epoch = rspamd_fuzzy_command_valid(&s->cmd.basic, buflen);
+
+ if (epoch == RSPAMD_FUZZY_EPOCH_MAX) {
+ msg_debug("invalid fuzzy command of size %d received", buflen);
+ return FALSE;
+ }
+
+ s->epoch = epoch;
+
+ /* Advance buf */
+ buf += sizeof(s->cmd.basic);
+ buflen -= sizeof(s->cmd.basic);
+
+ if (s->cmd.basic.shingles_count > 0) {
+ if (buflen >= sizeof(s->cmd.sgl)) {
+ /* Copy the shingles part */
+ memcpy(&s->cmd.sgl, buf, sizeof(s->cmd.sgl));
+ }
+ else {
+ /* Truncated stuff */
+ msg_debug("truncated fuzzy shingles command of size %d received", buflen);
+ return FALSE;
+ }
+
+ buf += sizeof(s->cmd.sgl);
+ buflen -= sizeof(s->cmd.sgl);
+
+ if (encrypted) {
+ s->cmd_type = CMD_ENCRYPTED_SHINGLE;
+ }
+ else {
+ s->cmd_type = CMD_SHINGLE;
+ }
+ }
+ else {
+ if (encrypted) {
+ s->cmd_type = CMD_ENCRYPTED_NORMAL;
+ }
+ else {
+ s->cmd_type = CMD_NORMAL;
+ }
+ }
+
+ if (buflen > 0) {
+ /* Process possible extensions */
+ if (!rspamd_fuzzy_extensions_from_wire(s, buf, buflen)) {
+ msg_debug("truncated fuzzy shingles command of size %d received", buflen);
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+
+static void
+fuzzy_session_destroy(gpointer d)
+{
+ struct fuzzy_session *session = d;
+
+ rspamd_inet_address_free(session->addr);
+ rspamd_explicit_memzero(session->nm, sizeof(session->nm));
+ session->worker->nconns--;
+
+ if (session->ip_stat) {
+ REF_RELEASE(session->ip_stat);
+ }
+
+ if (session->extensions) {
+ g_free(session->extensions);
+ }
+
+ g_free(session);
+}
+
+#define FUZZY_INPUT_BUFLEN 1024
+#ifdef HAVE_RECVMMSG
+#define MSGVEC_LEN 16
+#else
+#define MSGVEC_LEN 1
+#endif
+
+union sa_union {
+ struct sockaddr sa;
+ struct sockaddr_in s4;
+ struct sockaddr_in6 s6;
+ struct sockaddr_un su;
+ struct sockaddr_storage ss;
+};
+/*
+ * Accept new connection and construct task
+ */
+static void
+accept_fuzzy_socket(EV_P_ ev_io *w, int revents)
+{
+ struct rspamd_worker *worker = (struct rspamd_worker *) w->data;
+ struct rspamd_fuzzy_storage_ctx *ctx;
+ struct fuzzy_session *session;
+ gssize r, msg_len;
+ guint64 *nerrors;
+ struct iovec iovs[MSGVEC_LEN];
+ guint8 bufs[MSGVEC_LEN][FUZZY_INPUT_BUFLEN];
+ union sa_union peer_sa[MSGVEC_LEN];
+ socklen_t salen = sizeof(peer_sa[0]);
+#ifdef HAVE_RECVMMSG
+#define MSG_FIELD(msg, field) msg.msg_hdr.field
+ struct mmsghdr msg[MSGVEC_LEN];
+#else
+#define MSG_FIELD(msg, field) msg.field
+ struct msghdr msg[MSGVEC_LEN];
+#endif
+
+ memset(msg, 0, sizeof(*msg) * MSGVEC_LEN);
+ ctx = (struct rspamd_fuzzy_storage_ctx *) worker->ctx;
+
+ /* Prepare messages to receive */
+ for (int i = 0; i < MSGVEC_LEN; i++) {
+ /* Prepare msghdr structs */
+ iovs[i].iov_base = bufs[i];
+ iovs[i].iov_len = sizeof(bufs[i]);
+ MSG_FIELD(msg[i], msg_name) = (void *) &peer_sa[i];
+ MSG_FIELD(msg[i], msg_namelen) = salen;
+ MSG_FIELD(msg[i], msg_iov) = &iovs[i];
+ MSG_FIELD(msg[i], msg_iovlen) = 1;
+ }
+
+ /* Got some data */
+ if (revents == EV_READ) {
+ ev_now_update_if_cheap(ctx->event_loop);
+ for (;;) {
+#ifdef HAVE_RECVMMSG
+ r = recvmmsg(w->fd, msg, MSGVEC_LEN, 0, NULL);
+#else
+ r = recvmsg(w->fd, msg, 0);
+#endif
+
+ if (r == -1) {
+ if (errno == EINTR) {
+ continue;
+ }
+ else if (errno == EAGAIN || errno == EWOULDBLOCK) {
+
+ return;
+ }
+
+ msg_err("got error while reading from socket: %d, %s",
+ errno,
+ strerror(errno));
+ return;
+ }
+
+#ifndef HAVE_RECVMMSG
+ msg_len = r; /* Save real length in bytes here */
+ r = 1; /* Assume that we have received a single message */
+#endif
+
+ for (int i = 0; i < r; i++) {
+ rspamd_inet_addr_t *client_addr;
+
+ if (MSG_FIELD(msg[i], msg_namelen) >= sizeof(struct sockaddr)) {
+ client_addr = rspamd_inet_address_from_sa(MSG_FIELD(msg[i], msg_name),
+ MSG_FIELD(msg[i], msg_namelen));
+ if (!rspamd_fuzzy_check_client(worker->ctx, client_addr)) {
+ /* Disallow forbidden clients silently */
+ rspamd_inet_address_free(client_addr);
+ continue;
+ }
+ }
+ else {
+ client_addr = NULL;
+ }
+
+ session = g_malloc0(sizeof(*session));
+ REF_INIT_RETAIN(session, fuzzy_session_destroy);
+ session->worker = worker;
+ session->fd = w->fd;
+ session->ctx = ctx;
+ session->timestamp = ev_now(ctx->event_loop);
+ session->addr = client_addr;
+ worker->nconns++;
+
+ /* Each message can have its length in case of recvmmsg */
+#ifdef HAVE_RECVMMSG
+ msg_len = msg[i].msg_len;
+#endif
+
+ if (rspamd_fuzzy_cmd_from_wire(iovs[i].iov_base,
+ msg_len, session)) {
+ /* Check shingles count sanity */
+ rspamd_fuzzy_process_command(session);
+ }
+ else {
+ /* Discard input */
+ session->ctx->stat.invalid_requests++;
+ msg_debug("invalid fuzzy command of size %z received", r);
+
+ if (session->addr) {
+ nerrors = rspamd_lru_hash_lookup(session->ctx->errors_ips,
+ session->addr, -1);
+
+ if (nerrors == NULL) {
+ nerrors = g_malloc(sizeof(*nerrors));
+ *nerrors = 1;
+ rspamd_lru_hash_insert(session->ctx->errors_ips,
+ rspamd_inet_address_copy(session->addr, NULL),
+ nerrors, -1, -1);
+ }
+ else {
+ *nerrors = *nerrors + 1;
+ }
+ }
+ }
+
+ REF_RELEASE(session);
+ }
+#ifdef HAVE_RECVMMSG
+ /* Stop reading as we are using recvmmsg instead of recvmsg */
+ break;
+#endif
+ }
+ }
+}
+
+static gboolean
+rspamd_fuzzy_storage_periodic_callback(void *ud)
+{
+ struct rspamd_fuzzy_storage_ctx *ctx = ud;
+
+ if (ctx->updates_pending->len > 0) {
+ rspamd_fuzzy_process_updates_queue(ctx, local_db_name, FALSE);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_fuzzy_storage_sync(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker, gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud)
+{
+ struct rspamd_fuzzy_storage_ctx *ctx = ud;
+ struct rspamd_control_reply rep;
+
+ rep.reply.fuzzy_sync.status = 0;
+ rep.type = RSPAMD_CONTROL_FUZZY_SYNC;
+
+ if (ctx->backend && worker->index == 0) {
+ rspamd_fuzzy_process_updates_queue(ctx, local_db_name, FALSE);
+ rspamd_fuzzy_backend_start_update(ctx->backend, ctx->sync_timeout,
+ rspamd_fuzzy_storage_periodic_callback, ctx);
+ }
+
+ if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
+ msg_err("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_fuzzy_control_blocked(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker, gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud)
+{
+ struct rspamd_fuzzy_storage_ctx *ctx = (struct rspamd_fuzzy_storage_ctx *) ud;
+ struct rspamd_control_reply rep;
+ struct rspamd_leaky_bucket_elt *elt;
+ ev_tstamp now = ev_now(ctx->event_loop);
+ rspamd_inet_addr_t *addr = NULL;
+
+ rep.type = RSPAMD_CONTROL_FUZZY_BLOCKED;
+ rep.reply.fuzzy_blocked.status = 0;
+
+ if (cmd->cmd.fuzzy_blocked.af == AF_INET) {
+ addr = rspamd_inet_address_from_sa(&cmd->cmd.fuzzy_blocked.addr.sa,
+ sizeof(struct sockaddr_in));
+ }
+ else if (cmd->cmd.fuzzy_blocked.af == AF_INET6) {
+ addr = rspamd_inet_address_from_sa(&cmd->cmd.fuzzy_blocked.addr.sa,
+ sizeof(struct sockaddr_in6));
+ }
+ else {
+ msg_err("invalid address family: %d", cmd->cmd.fuzzy_blocked.af);
+ rep.reply.fuzzy_blocked.status = -1;
+ }
+
+ if (addr) {
+ elt = rspamd_lru_hash_lookup(ctx->ratelimit_buckets, addr,
+ (time_t) now);
+
+ if (elt) {
+ if (isnan(elt->cur)) {
+ /* Already ratelimited, ignore */
+ }
+ else {
+ elt->last = now;
+ elt->cur = NAN;
+
+ msg_info("propagating ratelimiting %s, %.1f max elts",
+ rspamd_inet_address_to_string(addr),
+ ctx->leaky_bucket_burst);
+ rspamd_fuzzy_maybe_call_blacklisted(ctx, addr, "ratelimit");
+ }
+
+ rspamd_inet_address_free(addr);
+ }
+ else {
+ /* New bucket */
+ elt = g_malloc(sizeof(*elt));
+ elt->addr = addr; /* transfer ownership */
+ elt->cur = NAN;
+ elt->last = now;
+
+ rspamd_lru_hash_insert(ctx->ratelimit_buckets,
+ addr,
+ elt,
+ (time_t) now,
+ ctx->leaky_bucket_ttl);
+ msg_info("propagating ratelimiting %s, %.1f max elts",
+ rspamd_inet_address_to_string(addr),
+ ctx->leaky_bucket_burst);
+ rspamd_fuzzy_maybe_call_blacklisted(ctx, addr, "ratelimit");
+ }
+ }
+
+ if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
+ msg_err("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_fuzzy_storage_reload(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker, gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud)
+{
+ struct rspamd_fuzzy_storage_ctx *ctx = ud;
+ GError *err = NULL;
+ struct rspamd_control_reply rep;
+
+ msg_info("reloading fuzzy storage after receiving reload command");
+
+ if (ctx->backend) {
+ /* Close backend and reopen it one more time */
+ rspamd_fuzzy_backend_close(ctx->backend);
+ }
+
+ memset(&rep, 0, sizeof(rep));
+ rep.type = RSPAMD_CONTROL_RELOAD;
+
+ if ((ctx->backend = rspamd_fuzzy_backend_create(ctx->event_loop,
+ worker->cf->options, rspamd_main->cfg,
+ &err)) == NULL) {
+ msg_err("cannot open backend after reload: %e", err);
+ rep.reply.reload.status = err->code;
+ g_error_free(err);
+ }
+ else {
+ rep.reply.reload.status = 0;
+ }
+
+ if (ctx->backend && worker->index == 0) {
+ rspamd_fuzzy_backend_start_update(ctx->backend, ctx->sync_timeout,
+ rspamd_fuzzy_storage_periodic_callback, ctx);
+ }
+
+ if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
+ msg_err("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ return TRUE;
+}
+
+static ucl_object_t *
+rspamd_fuzzy_storage_stat_key(const struct fuzzy_key_stat *key_stat)
+{
+ ucl_object_t *res;
+
+ res = ucl_object_typed_new(UCL_OBJECT);
+
+ ucl_object_insert_key(res, ucl_object_fromint(key_stat->checked),
+ "checked", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromdouble(key_stat->checked_ctr.mean),
+ "checked_per_hour", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromint(key_stat->matched),
+ "matched", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromdouble(key_stat->matched_ctr.mean),
+ "matched_per_hour", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromint(key_stat->added),
+ "added", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromint(key_stat->deleted),
+ "deleted", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromint(key_stat->errors),
+ "errors", 0, false);
+
+ return res;
+}
+
+static ucl_object_t *
+rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
+{
+ struct fuzzy_key_stat *key_stat;
+ GHashTableIter it;
+ struct fuzzy_key *fuzzy_key;
+ ucl_object_t *obj, *keys_obj, *elt, *ip_elt, *ip_cur;
+ gpointer k, v;
+ gint i;
+ gchar keyname[17];
+
+ obj = ucl_object_typed_new(UCL_OBJECT);
+
+ keys_obj = ucl_object_typed_new(UCL_OBJECT);
+ g_hash_table_iter_init(&it, ctx->keys);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ fuzzy_key = v;
+ key_stat = fuzzy_key->stat;
+
+ if (key_stat) {
+ rspamd_snprintf(keyname, sizeof(keyname), "%8bs", k);
+
+ elt = rspamd_fuzzy_storage_stat_key(key_stat);
+
+ if (key_stat->last_ips && ip_stat) {
+ i = 0;
+
+ ip_elt = ucl_object_typed_new(UCL_OBJECT);
+
+ while ((i = rspamd_lru_hash_foreach(key_stat->last_ips,
+ i, &k, &v)) != -1) {
+ ip_cur = rspamd_fuzzy_storage_stat_key(v);
+ ucl_object_insert_key(ip_elt, ip_cur,
+ rspamd_inet_address_to_string(k), 0, true);
+ }
+ ucl_object_insert_key(elt, ip_elt, "ips", 0, false);
+ }
+
+ int flag;
+ struct fuzzy_key_stat *flag_stat;
+ ucl_object_t *flags_ucl = ucl_object_typed_new(UCL_OBJECT);
+
+ kh_foreach_key_value_ptr(fuzzy_key->flags_stat, flag, flag_stat, {
+ char intbuf[16];
+ rspamd_snprintf(intbuf, sizeof(intbuf), "%d", flag);
+ ucl_object_insert_key(flags_ucl, rspamd_fuzzy_storage_stat_key(flag_stat),
+ intbuf, 0, true);
+ });
+
+ ucl_object_insert_key(elt, flags_ucl, "flags", 0, false);
+
+ ucl_object_insert_key(elt,
+ rspamd_keypair_to_ucl(fuzzy_key->key, RSPAMD_KEYPAIR_DUMP_NO_SECRET | RSPAMD_KEYPAIR_DUMP_FLATTENED),
+ "keypair", 0, false);
+ ucl_object_insert_key(keys_obj, elt, keyname, 0, true);
+ }
+ }
+
+ ucl_object_insert_key(obj, keys_obj, "keys", 0, false);
+
+ /* Now generic stats */
+ ucl_object_insert_key(obj,
+ ucl_object_fromint(ctx->stat.fuzzy_hashes),
+ "fuzzy_stored",
+ 0,
+ false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromint(ctx->stat.fuzzy_hashes_expired),
+ "fuzzy_expired",
+ 0,
+ false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromint(ctx->stat.invalid_requests),
+ "invalid_requests",
+ 0,
+ false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromint(ctx->stat.delayed_hashes),
+ "delayed_hashes",
+ 0,
+ false);
+
+ if (ctx->errors_ips && ip_stat) {
+ i = 0;
+
+ ip_elt = ucl_object_typed_new(UCL_OBJECT);
+
+ while ((i = rspamd_lru_hash_foreach(ctx->errors_ips, i, &k, &v)) != -1) {
+ ucl_object_insert_key(ip_elt,
+ ucl_object_fromint(*(guint64 *) v),
+ rspamd_inet_address_to_string(k), 0, true);
+ }
+
+ ucl_object_insert_key(obj,
+ ip_elt,
+ "errors_ips",
+ 0,
+ false);
+ }
+
+ /* Checked by epoch */
+ elt = ucl_object_typed_new(UCL_ARRAY);
+
+ for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
+ ucl_array_append(elt,
+ ucl_object_fromint(ctx->stat.fuzzy_hashes_checked[i]));
+ }
+
+ ucl_object_insert_key(obj, elt, "fuzzy_checked", 0, false);
+
+ /* Shingles by epoch */
+ elt = ucl_object_typed_new(UCL_ARRAY);
+
+ for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
+ ucl_array_append(elt,
+ ucl_object_fromint(ctx->stat.fuzzy_shingles_checked[i]));
+ }
+
+ ucl_object_insert_key(obj, elt, "fuzzy_shingles", 0, false);
+
+ /* Matched by epoch */
+ elt = ucl_object_typed_new(UCL_ARRAY);
+
+ for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
+ ucl_array_append(elt,
+ ucl_object_fromint(ctx->stat.fuzzy_hashes_found[i]));
+ }
+
+ ucl_object_insert_key(obj, elt, "fuzzy_found", 0, false);
+
+
+ return obj;
+}
+
+static void
+rspamd_fuzzy_maybe_load_ratelimits(struct rspamd_fuzzy_storage_ctx *ctx)
+{
+ char path[PATH_MAX];
+
+ rspamd_snprintf(path, sizeof(path), "%s" G_DIR_SEPARATOR_S "fuzzy_ratelimits.ucl",
+ RSPAMD_DBDIR);
+
+ if (access(path, R_OK) != -1) {
+ struct ucl_parser *parser = ucl_parser_new(UCL_PARSER_NO_IMPLICIT_ARRAYS | UCL_PARSER_DISABLE_MACRO);
+ if (ucl_parser_add_file(parser, path)) {
+ ucl_object_t *obj = ucl_parser_get_object(parser);
+ int loaded = 0;
+
+ if (ucl_object_type(obj) == UCL_ARRAY) {
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur;
+
+ while ((cur = ucl_object_iterate(obj, &it, true)) != NULL) {
+ const ucl_object_t *ip, *value, *last;
+ const gchar *ip_str;
+ double limit_val, last_val;
+
+ ip = ucl_object_find_key(cur, "ip");
+ value = ucl_object_find_key(cur, "value");
+ last = ucl_object_find_key(cur, "last");
+
+ if (ip == NULL || value == NULL || last == NULL) {
+ msg_err("invalid ratelimit object");
+ continue;
+ }
+
+ ip_str = ucl_object_tostring(ip);
+ limit_val = ucl_object_todouble(value);
+ last_val = ucl_object_todouble(last);
+
+ if (ip_str == NULL || isnan(last_val)) {
+ msg_err("invalid ratelimit object");
+ continue;
+ }
+
+ rspamd_inet_addr_t *addr;
+ if (rspamd_parse_inet_address(&addr, ip_str, strlen(ip_str),
+ RSPAMD_INET_ADDRESS_PARSE_NO_UNIX | RSPAMD_INET_ADDRESS_PARSE_NO_PORT)) {
+ struct rspamd_leaky_bucket_elt *elt = g_malloc(sizeof(*elt));
+
+ elt->cur = limit_val;
+ elt->last = last_val;
+ elt->addr = addr;
+ rspamd_lru_hash_insert(ctx->ratelimit_buckets, addr, elt, elt->last, ctx->leaky_bucket_ttl);
+ loaded++;
+ }
+ else {
+ msg_err("invalid ratelimit ip: %s", ip_str);
+ continue;
+ }
+ }
+
+ msg_info("loaded %d ratelimit objects", loaded);
+ }
+
+ ucl_object_unref(obj);
+ }
+
+ ucl_parser_free(parser);
+ }
+}
+
+static void
+rspamd_fuzzy_maybe_save_ratelimits(struct rspamd_fuzzy_storage_ctx *ctx)
+{
+ char path[PATH_MAX];
+
+ rspamd_snprintf(path, sizeof(path), "%s" G_DIR_SEPARATOR_S "fuzzy_ratelimits.ucl.new",
+ RSPAMD_DBDIR);
+ FILE *f = fopen(path, "w");
+
+ if (f != NULL) {
+ ucl_object_t *top = ucl_object_typed_new(UCL_ARRAY);
+ int it = 0;
+ gpointer k, v;
+
+ ucl_object_reserve(top, rspamd_lru_hash_size(ctx->ratelimit_buckets));
+
+ while ((it = rspamd_lru_hash_foreach(ctx->ratelimit_buckets, it, &k, &v)) != -1) {
+ ucl_object_t *cur = ucl_object_typed_new(UCL_OBJECT);
+ struct rspamd_leaky_bucket_elt *elt = (struct rspamd_leaky_bucket_elt *) v;
+
+ ucl_object_insert_key(cur, ucl_object_fromdouble(elt->cur), "value", 0, false);
+ ucl_object_insert_key(cur, ucl_object_fromdouble(elt->last), "last", 0, false);
+ ucl_object_insert_key(cur, ucl_object_fromstring(rspamd_inet_address_to_string(elt->addr)), "ip", 0, false);
+ ucl_array_append(top, cur);
+ }
+
+ if (ucl_object_emit_full(top, UCL_EMIT_JSON_COMPACT, ucl_object_emit_file_funcs(f), NULL)) {
+ char npath[PATH_MAX];
+
+ fflush(f);
+ rspamd_snprintf(npath, sizeof(npath), "%s" G_DIR_SEPARATOR_S "fuzzy_ratelimits.ucl",
+ RSPAMD_DBDIR);
+
+ if (rename(path, npath) == -1) {
+ msg_warn("cannot rename %s to %s: %s", path, npath, strerror(errno));
+ }
+ else {
+ msg_info("saved %d ratelimits in %s", rspamd_lru_hash_size(ctx->ratelimit_buckets), npath);
+ }
+ }
+ else {
+ msg_warn("cannot serialize ratelimit buckets to %s: %s", path, strerror(errno));
+ }
+
+ fclose(f);
+ ucl_object_unref(top);
+ }
+ else {
+ msg_warn("cannot save ratelimit buckets to %s: %s", path, strerror(errno));
+ }
+}
+
+static int
+lua_fuzzy_add_pre_handler(lua_State *L)
+{
+ struct rspamd_worker *wrk, **pwrk = (struct rspamd_worker **)
+ rspamd_lua_check_udata(L, 1, "rspamd{worker}");
+ struct rspamd_fuzzy_storage_ctx *ctx;
+
+ if (!pwrk) {
+ return luaL_error(L, "invalid arguments, worker + function are expected");
+ }
+
+ wrk = *pwrk;
+
+ if (wrk && lua_isfunction(L, 2)) {
+ ctx = (struct rspamd_fuzzy_storage_ctx *) wrk->ctx;
+
+ if (ctx->lua_pre_handler_cbref != -1) {
+ /* Should not happen */
+ luaL_unref(L, LUA_REGISTRYINDEX, ctx->lua_pre_handler_cbref);
+ }
+
+ lua_pushvalue(L, 2);
+ ctx->lua_pre_handler_cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, worker + function are expected");
+ }
+
+ return 0;
+}
+
+static int
+lua_fuzzy_add_post_handler(lua_State *L)
+{
+ struct rspamd_worker *wrk, **pwrk = (struct rspamd_worker **)
+ rspamd_lua_check_udata(L, 1, "rspamd{worker}");
+ struct rspamd_fuzzy_storage_ctx *ctx;
+
+ if (!pwrk) {
+ return luaL_error(L, "invalid arguments, worker + function are expected");
+ }
+
+ wrk = *pwrk;
+
+ if (wrk && lua_isfunction(L, 2)) {
+ ctx = (struct rspamd_fuzzy_storage_ctx *) wrk->ctx;
+
+ if (ctx->lua_post_handler_cbref != -1) {
+ /* Should not happen */
+ luaL_unref(L, LUA_REGISTRYINDEX, ctx->lua_post_handler_cbref);
+ }
+
+ lua_pushvalue(L, 2);
+ ctx->lua_post_handler_cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, worker + function are expected");
+ }
+
+ return 0;
+}
+
+static int
+lua_fuzzy_add_blacklist_handler(lua_State *L)
+{
+ struct rspamd_worker *wrk, **pwrk = (struct rspamd_worker **)
+ rspamd_lua_check_udata(L, 1, "rspamd{worker}");
+ struct rspamd_fuzzy_storage_ctx *ctx;
+
+ if (!pwrk) {
+ return luaL_error(L, "invalid arguments, worker + function are expected");
+ }
+
+ wrk = *pwrk;
+
+ if (wrk && lua_isfunction(L, 2)) {
+ ctx = (struct rspamd_fuzzy_storage_ctx *) wrk->ctx;
+
+ if (ctx->lua_blacklist_cbref != -1) {
+ /* Should not happen */
+ luaL_unref(L, LUA_REGISTRYINDEX, ctx->lua_blacklist_cbref);
+ }
+
+ lua_pushvalue(L, 2);
+ ctx->lua_blacklist_cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, worker + function are expected");
+ }
+
+ return 0;
+}
+
+static gboolean
+rspamd_fuzzy_storage_stat(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker, gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud)
+{
+ struct rspamd_fuzzy_storage_ctx *ctx = ud;
+ struct rspamd_control_reply rep;
+ ucl_object_t *obj;
+ struct ucl_emitter_functions *emit_subr;
+ guchar fdspace[CMSG_SPACE(sizeof(int))];
+ struct iovec iov;
+ struct msghdr msg;
+ struct cmsghdr *cmsg;
+
+ gint outfd = -1;
+ gchar tmppath[PATH_MAX];
+
+ memset(&rep, 0, sizeof(rep));
+ rep.type = RSPAMD_CONTROL_FUZZY_STAT;
+
+ rspamd_snprintf(tmppath, sizeof(tmppath), "%s%c%s-XXXXXXXXXX",
+ rspamd_main->cfg->temp_dir, G_DIR_SEPARATOR, "fuzzy-stat");
+
+ if ((outfd = mkstemp(tmppath)) == -1) {
+ rep.reply.fuzzy_stat.status = errno;
+ msg_info_main("cannot make temporary stat file for fuzzy stat: %s",
+ strerror(errno));
+ }
+ else {
+ rep.reply.fuzzy_stat.status = 0;
+
+ memcpy(rep.reply.fuzzy_stat.storage_id,
+ rspamd_fuzzy_backend_id(ctx->backend),
+ sizeof(rep.reply.fuzzy_stat.storage_id));
+
+ obj = rspamd_fuzzy_stat_to_ucl(ctx, TRUE);
+ emit_subr = ucl_object_emit_fd_funcs(outfd);
+ ucl_object_emit_full(obj, UCL_EMIT_JSON_COMPACT, emit_subr, NULL);
+ ucl_object_emit_funcs_free(emit_subr);
+ ucl_object_unref(obj);
+ /* Rewind output file */
+ close(outfd);
+ outfd = open(tmppath, O_RDONLY);
+ unlink(tmppath);
+ }
+
+ /* Now we can send outfd and status message */
+ memset(&msg, 0, sizeof(msg));
+
+ /* Attach fd to the message */
+ if (outfd != -1) {
+ memset(fdspace, 0, sizeof(fdspace));
+ msg.msg_control = fdspace;
+ msg.msg_controllen = sizeof(fdspace);
+ cmsg = CMSG_FIRSTHDR(&msg);
+
+ if (cmsg) {
+ cmsg->cmsg_level = SOL_SOCKET;
+ cmsg->cmsg_type = SCM_RIGHTS;
+ cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+ memcpy(CMSG_DATA(cmsg), &outfd, sizeof(int));
+ }
+ }
+
+ iov.iov_base = &rep;
+ iov.iov_len = sizeof(rep);
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+
+ if (sendmsg(fd, &msg, 0) == -1) {
+ msg_err_main("cannot send fuzzy stat: %s", strerror(errno));
+ }
+
+ if (outfd != -1) {
+ close(outfd);
+ }
+
+ return TRUE;
+}
+
+static gboolean
+fuzzy_parse_ids(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ struct rspamd_rcl_struct_parser *pd = (struct rspamd_rcl_struct_parser *) ud;
+ khash_t(fuzzy_key_ids_set) * target;
+
+ target = *(khash_t(fuzzy_key_ids_set) **) ((gchar *) pd->user_struct + pd->offset);
+
+ if (ucl_object_type(obj) == UCL_ARRAY) {
+ const ucl_object_t *cur;
+ ucl_object_iter_t it = NULL;
+ guint64 id;
+
+ while ((cur = ucl_object_iterate(obj, &it, true)) != NULL) {
+ if (ucl_object_toint_safe(cur, &id)) {
+ int r;
+
+ kh_put(fuzzy_key_ids_set, target, id, &r);
+ }
+ else {
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+ }
+ else if (ucl_object_type(obj) == UCL_INT) {
+ int r;
+ kh_put(fuzzy_key_ids_set, target, ucl_object_toint(obj), &r);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static gboolean
+fuzzy_parse_keypair(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ struct rspamd_rcl_struct_parser *pd = ud;
+ struct rspamd_fuzzy_storage_ctx *ctx;
+ struct rspamd_cryptobox_keypair *kp;
+ struct fuzzy_key_stat *keystat;
+ struct fuzzy_key *key;
+ const ucl_object_t *cur;
+ const guchar *pk;
+ ucl_object_iter_t it = NULL;
+ gboolean ret;
+
+ ctx = pd->user_struct;
+ pd->offset = G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, default_keypair);
+
+ /*
+ * Single key
+ */
+ if (ucl_object_type(obj) == UCL_STRING || ucl_object_type(obj) == UCL_OBJECT) {
+ ret = rspamd_rcl_parse_struct_keypair(pool, obj, pd, section, err);
+
+ if (!ret) {
+ return ret;
+ }
+
+ /* Insert key to the hash table */
+ kp = ctx->default_keypair;
+
+ if (kp == NULL) {
+ return FALSE;
+ }
+
+ if (rspamd_keypair_alg(kp) != RSPAMD_CRYPTOBOX_MODE_25519 ||
+ rspamd_keypair_type(kp) != RSPAMD_KEYPAIR_KEX) {
+ return FALSE;
+ }
+
+ key = g_malloc0(sizeof(*key));
+ key->key = kp;
+ keystat = g_malloc0(sizeof(*keystat));
+ REF_INIT_RETAIN(keystat, fuzzy_key_stat_dtor);
+ /* Hash of ip -> fuzzy_key_stat */
+ keystat->last_ips = rspamd_lru_hash_new_full(1024,
+ (GDestroyNotify) rspamd_inet_address_free,
+ fuzzy_key_stat_unref,
+ rspamd_inet_address_hash, rspamd_inet_address_equal);
+ key->stat = keystat;
+ key->flags_stat = kh_init(fuzzy_key_flag_stat);
+ /* Preallocate some space for flags */
+ kh_resize(fuzzy_key_flag_stat, key->flags_stat, 8);
+ pk = rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_PK,
+ NULL);
+ keystat->keypair = rspamd_keypair_ref(kp);
+ /* We map entries by pubkey in binary form for faster lookup */
+ g_hash_table_insert(ctx->keys, (gpointer) pk, key);
+ ctx->default_key = key;
+
+ const ucl_object_t *extensions = rspamd_keypair_get_extensions(kp);
+
+ if (extensions) {
+ const ucl_object_t *forbidden_ids = ucl_object_lookup(extensions, "forbidden_ids");
+
+ if (forbidden_ids && ucl_object_type(forbidden_ids) == UCL_ARRAY) {
+ key->forbidden_ids = kh_init(fuzzy_key_ids_set);
+ while ((cur = ucl_object_iterate(forbidden_ids, &it, true)) != NULL) {
+ if (ucl_object_type(cur) == UCL_INT || ucl_object_type(cur) == UCL_FLOAT) {
+ int id = ucl_object_toint(cur);
+ int r;
+
+ kh_put(fuzzy_key_ids_set, key->forbidden_ids, id, &r);
+ }
+ }
+ }
+ }
+
+ msg_debug_pool_check("loaded keypair %*xs", 8, pk);
+ }
+ else if (ucl_object_type(obj) == UCL_ARRAY) {
+ while ((cur = ucl_object_iterate(obj, &it, true)) != NULL) {
+ if (!fuzzy_parse_keypair(pool, cur, pd, section, err)) {
+ msg_err_pool("cannot parse keypair");
+ }
+ }
+ }
+
+ return TRUE;
+}
+
+static guint
+fuzzy_kp_hash(gconstpointer p)
+{
+ return *(guint *) p;
+}
+
+static gboolean
+fuzzy_kp_equal(gconstpointer a, gconstpointer b)
+{
+ const guchar *pa = a, *pb = b;
+
+ return (memcmp(pa, pb, RSPAMD_FUZZY_KEYLEN) == 0);
+}
+
+gpointer
+init_fuzzy(struct rspamd_config *cfg)
+{
+ struct rspamd_fuzzy_storage_ctx *ctx;
+ GQuark type;
+
+ type = g_quark_try_string("fuzzy");
+
+ ctx = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(struct rspamd_fuzzy_storage_ctx));
+
+ ctx->magic = rspamd_fuzzy_storage_magic;
+ ctx->sync_timeout = DEFAULT_SYNC_TIMEOUT;
+ ctx->keypair_cache_size = DEFAULT_KEYPAIR_CACHE_SIZE;
+ ctx->lua_pre_handler_cbref = -1;
+ ctx->lua_post_handler_cbref = -1;
+ ctx->lua_blacklist_cbref = -1;
+ ctx->keys = g_hash_table_new_full(fuzzy_kp_hash, fuzzy_kp_equal,
+ NULL, fuzzy_key_dtor);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) g_hash_table_unref, ctx->keys);
+ ctx->errors_ips = rspamd_lru_hash_new_full(1024,
+ (GDestroyNotify) rspamd_inet_address_free, g_free,
+ rspamd_inet_address_hash, rspamd_inet_address_equal);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_lru_hash_destroy, ctx->errors_ips);
+ ctx->cfg = cfg;
+ ctx->updates_maxfail = DEFAULT_UPDATES_MAXFAIL;
+ ctx->leaky_bucket_mask = DEFAULT_BUCKET_MASK;
+ ctx->leaky_bucket_ttl = DEFAULT_BUCKET_TTL;
+ ctx->max_buckets = DEFAULT_MAX_BUCKETS;
+ ctx->leaky_bucket_burst = NAN;
+ ctx->leaky_bucket_rate = NAN;
+ ctx->delay = NAN;
+ ctx->default_forbidden_ids = kh_init(fuzzy_key_ids_set);
+ ctx->weak_ids = kh_init(fuzzy_key_ids_set);
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "sync",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx,
+ sync_timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Time to perform database sync, default: " G_STRINGIFY(DEFAULT_SYNC_TIMEOUT) " seconds");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "expire",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx,
+ expire),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Default expire time for hashes, default: " G_STRINGIFY(DEFAULT_EXPIRE) " seconds");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "delay",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx,
+ delay),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Default delay time for hashes, default: not enabled");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "allow_update",
+ rspamd_rcl_parse_struct_ucl,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, update_map),
+ 0,
+ "Allow modifications from the following IP addresses");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "allow_update_keys",
+ rspamd_rcl_parse_struct_ucl,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, update_keys_map),
+ 0,
+ "Allow modifications for those using specific public keys");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "delay_whitelist",
+ rspamd_rcl_parse_struct_ucl,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, delay_whitelist_map),
+ 0,
+ "Disable delay check for the following IP addresses");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "keypair",
+ fuzzy_parse_keypair,
+ ctx,
+ 0,
+ RSPAMD_CL_FLAG_MULTIPLE,
+ "Encryption keypair (can be repeated for different keys)");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "forbidden_ids",
+ fuzzy_parse_ids,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, default_forbidden_ids),
+ 0,
+ "Deny specific flags by default");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "weak_ids",
+ fuzzy_parse_ids,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, weak_ids),
+ 0,
+ "Treat these flags as weak (i.e. they do not overwrite strong flags)");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "keypair_cache_size",
+ rspamd_rcl_parse_struct_integer,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx,
+ keypair_cache_size),
+ RSPAMD_CL_FLAG_UINT,
+ "Size of keypairs cache, default: " G_STRINGIFY(DEFAULT_KEYPAIR_CACHE_SIZE));
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "encrypted_only",
+ rspamd_rcl_parse_struct_boolean,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, encrypted_only),
+ 0,
+ "Allow encrypted requests only (and forbid all unknown keys or plaintext requests)");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "dedicated_update_worker",
+ rspamd_rcl_parse_struct_boolean,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, dedicated_update_worker),
+ 0,
+ "Use worker 0 for updates only");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "read_only",
+ rspamd_rcl_parse_struct_boolean,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, read_only),
+ 0,
+ "Work in read only mode");
+
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "blocked",
+ rspamd_rcl_parse_struct_ucl,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, blocked_map),
+ 0,
+ "Block requests from specific networks");
+
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "updates_maxfail",
+ rspamd_rcl_parse_struct_integer,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, updates_maxfail),
+ RSPAMD_CL_FLAG_UINT,
+ "Maximum number of updates to be failed before discarding");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "skip_hashes",
+ rspamd_rcl_parse_struct_ucl,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, skip_map),
+ 0,
+ "Skip specific hashes from the map");
+
+ /* Ratelimits */
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "ratelimit_whitelist",
+ rspamd_rcl_parse_struct_ucl,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, ratelimit_whitelist_map),
+ 0,
+ "Skip specific addresses from rate limiting");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "ratelimit_max_buckets",
+ rspamd_rcl_parse_struct_integer,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, max_buckets),
+ RSPAMD_CL_FLAG_UINT,
+ "Maximum number of leaky buckets (default: " G_STRINGIFY(DEFAULT_MAX_BUCKETS) ")");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "ratelimit_network_mask",
+ rspamd_rcl_parse_struct_integer,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, leaky_bucket_mask),
+ RSPAMD_CL_FLAG_UINT,
+ "Network mask to apply for IPv4 rate addresses (default: " G_STRINGIFY(DEFAULT_BUCKET_MASK) ")");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "ratelimit_bucket_ttl",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, leaky_bucket_ttl),
+ RSPAMD_CL_FLAG_TIME_INTEGER,
+ "Time to live for ratelimit element (default: " G_STRINGIFY(DEFAULT_BUCKET_TTL) ")");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "ratelimit_rate",
+ rspamd_rcl_parse_struct_double,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, leaky_bucket_rate),
+ 0,
+ "Leak rate in requests per second");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "ratelimit_burst",
+ rspamd_rcl_parse_struct_double,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, leaky_bucket_burst),
+ 0,
+ "Peak value for ratelimit bucket");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "ratelimit_log_only",
+ rspamd_rcl_parse_struct_boolean,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, ratelimit_log_only),
+ 0,
+ "Don't really ban on ratelimit reaching, just log");
+
+
+ return ctx;
+}
+
+static void
+rspamd_fuzzy_peer_io(EV_P_ ev_io *w, int revents)
+{
+ struct fuzzy_peer_cmd cmd;
+ struct rspamd_fuzzy_storage_ctx *ctx =
+ (struct rspamd_fuzzy_storage_ctx *) w->data;
+ gssize r;
+
+ for (;;) {
+ r = read(w->fd, &cmd, sizeof(cmd));
+
+ if (r != sizeof(cmd)) {
+ if (errno == EINTR) {
+ continue;
+ }
+ if (errno != EAGAIN) {
+ msg_err("cannot read command from peers: %s", strerror(errno));
+ }
+
+ break;
+ }
+ else {
+ g_array_append_val(ctx->updates_pending, cmd);
+ }
+ }
+}
+
+static void
+fuzzy_peer_rep(struct rspamd_worker *worker,
+ struct rspamd_srv_reply *rep, gint rep_fd,
+ gpointer ud)
+{
+ struct rspamd_fuzzy_storage_ctx *ctx = ud;
+ GList *cur;
+ struct rspamd_worker_listen_socket *ls;
+ struct rspamd_worker_accept_event *ac_ev;
+
+ ctx->peer_fd = rep_fd;
+
+ if (rep_fd == -1) {
+ msg_err("cannot receive peer fd from the main process");
+ exit(EXIT_FAILURE);
+ }
+ else {
+ rspamd_socket_nonblocking(rep_fd);
+ }
+
+ msg_info("got peer fd reply from the main process");
+
+ /* Start listening */
+ cur = worker->cf->listen_socks;
+ while (cur) {
+ ls = cur->data;
+
+ if (ls->fd != -1) {
+ msg_info("start listening on %s",
+ rspamd_inet_address_to_string_pretty(ls->addr));
+
+ if (ls->type == RSPAMD_WORKER_SOCKET_UDP) {
+ ac_ev = g_malloc0(sizeof(*ac_ev));
+ ac_ev->accept_ev.data = worker;
+ ac_ev->event_loop = ctx->event_loop;
+ ev_io_init(&ac_ev->accept_ev, accept_fuzzy_socket, ls->fd,
+ EV_READ);
+ ev_io_start(ctx->event_loop, &ac_ev->accept_ev);
+ DL_APPEND(worker->accept_events, ac_ev);
+ }
+ else {
+ /* We allow TCP listeners only for a update worker */
+ g_assert_not_reached();
+ }
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ if (ctx->peer_fd != -1) {
+ if (worker->index == 0) {
+ /* Listen for peer requests */
+ shutdown(ctx->peer_fd, SHUT_WR);
+ ctx->peer_ev.data = ctx;
+ ev_io_init(&ctx->peer_ev, rspamd_fuzzy_peer_io, ctx->peer_fd, EV_READ);
+ ev_io_start(ctx->event_loop, &ctx->peer_ev);
+ }
+ else {
+ shutdown(ctx->peer_fd, SHUT_RD);
+ }
+ }
+}
+
+/*
+ * Start worker process
+ */
+__attribute__((noreturn)) void
+start_fuzzy(struct rspamd_worker *worker)
+{
+ struct rspamd_fuzzy_storage_ctx *ctx = worker->ctx;
+ GError *err = NULL;
+ struct rspamd_srv_command srv_cmd;
+ struct rspamd_config *cfg = worker->srv->cfg;
+
+ g_assert(rspamd_worker_check_context(worker->ctx, rspamd_fuzzy_storage_magic));
+ ctx->event_loop = rspamd_prepare_worker(worker,
+ "fuzzy",
+ NULL);
+ ctx->peer_fd = -1;
+ ctx->worker = worker;
+ ctx->cfg = worker->srv->cfg;
+ ctx->resolver = rspamd_dns_resolver_init(worker->srv->logger,
+ ctx->event_loop,
+ worker->srv->cfg);
+ rspamd_upstreams_library_config(worker->srv->cfg, ctx->cfg->ups_ctx,
+ ctx->event_loop, ctx->resolver->r);
+ /* Since this worker uses maps it needs a valid HTTP context */
+ ctx->http_ctx = rspamd_http_context_create(ctx->cfg, ctx->event_loop,
+ ctx->cfg->ups_ctx);
+
+ if (ctx->keypair_cache_size > 0) {
+ /* Create keypairs cache */
+ ctx->keypair_cache = rspamd_keypair_cache_new(ctx->keypair_cache_size);
+ }
+
+
+ if ((ctx->backend = rspamd_fuzzy_backend_create(ctx->event_loop,
+ worker->cf->options, cfg, &err)) == NULL) {
+ msg_err("cannot open backend: %e", err);
+ if (err) {
+ g_error_free(err);
+ }
+ exit(EXIT_SUCCESS);
+ }
+
+ rspamd_fuzzy_backend_count(ctx->backend, fuzzy_count_callback, ctx);
+
+
+ if (worker->index == 0) {
+ ctx->updates_pending = g_array_sized_new(FALSE, FALSE,
+ sizeof(struct fuzzy_peer_cmd), 1024);
+ rspamd_fuzzy_backend_start_update(ctx->backend, ctx->sync_timeout,
+ rspamd_fuzzy_storage_periodic_callback, ctx);
+
+ if (ctx->dedicated_update_worker && worker->cf->count > 1) {
+ msg_info_config("stop serving clients request in dedicated update mode");
+ rspamd_worker_stop_accept(worker);
+
+ GList *cur = worker->cf->listen_socks;
+
+ while (cur) {
+ struct rspamd_worker_listen_socket *ls =
+ (struct rspamd_worker_listen_socket *) cur->data;
+
+ if (ls->fd != -1) {
+ close(ls->fd);
+ }
+
+ cur = g_list_next(cur);
+ }
+ }
+ }
+
+ ctx->stat_ev.data = ctx;
+ ev_timer_init(&ctx->stat_ev, rspamd_fuzzy_stat_callback, ctx->sync_timeout,
+ ctx->sync_timeout);
+ ev_timer_start(ctx->event_loop, &ctx->stat_ev);
+ /* Register custom reload and stat commands for the control socket */
+ rspamd_control_worker_add_cmd_handler(worker, RSPAMD_CONTROL_RELOAD,
+ rspamd_fuzzy_storage_reload, ctx);
+ rspamd_control_worker_add_cmd_handler(worker, RSPAMD_CONTROL_FUZZY_STAT,
+ rspamd_fuzzy_storage_stat, ctx);
+ rspamd_control_worker_add_cmd_handler(worker, RSPAMD_CONTROL_FUZZY_SYNC,
+ rspamd_fuzzy_storage_sync, ctx);
+ rspamd_control_worker_add_cmd_handler(worker, RSPAMD_CONTROL_FUZZY_BLOCKED,
+ rspamd_fuzzy_control_blocked, ctx);
+
+
+ if (ctx->update_map != NULL) {
+ rspamd_config_radix_from_ucl(worker->srv->cfg, ctx->update_map,
+ "Allow fuzzy updates from specified addresses",
+ &ctx->update_ips, NULL, worker, "fuzzy update");
+ }
+
+ if (ctx->update_keys_map != NULL) {
+ struct rspamd_map *m;
+
+ if ((m = rspamd_map_add_from_ucl(worker->srv->cfg, ctx->update_keys_map,
+ "Allow fuzzy updates from specified public keys",
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &ctx->update_keys, worker, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_warn_config("cannot load allow keys map from %s",
+ ucl_object_tostring(ctx->update_keys_map));
+ }
+ else {
+ m->active_http = TRUE;
+ }
+ }
+
+ if (ctx->skip_map != NULL) {
+ struct rspamd_map *m;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, ctx->skip_map,
+ "Skip hashes",
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &ctx->skip_hashes,
+ worker, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_warn_config("cannot load hashes list from %s",
+ ucl_object_tostring(ctx->skip_map));
+ }
+ else {
+ m->active_http = TRUE;
+ }
+ }
+
+ if (ctx->blocked_map != NULL) {
+ rspamd_config_radix_from_ucl(worker->srv->cfg, ctx->blocked_map,
+ "Block fuzzy requests from the specific IPs",
+ &ctx->blocked_ips,
+ NULL,
+ worker, "fuzzy blocked");
+ }
+
+ /* Create radix trees */
+ if (ctx->ratelimit_whitelist_map != NULL) {
+ rspamd_config_radix_from_ucl(worker->srv->cfg, ctx->ratelimit_whitelist_map,
+ "Skip ratelimits from specific ip addresses/networks",
+ &ctx->ratelimit_whitelist,
+ NULL,
+ worker, "fuzzy ratelimit whitelist");
+ }
+
+ if (!isnan(ctx->delay) && ctx->delay_whitelist_map != NULL) {
+ rspamd_config_radix_from_ucl(worker->srv->cfg, ctx->delay_whitelist_map,
+ "Skip delay from the following ips",
+ &ctx->delay_whitelist, NULL, worker,
+ "fuzzy delayed whitelist");
+ }
+
+ /* Ratelimits */
+ if (!isnan(ctx->leaky_bucket_rate) && !isnan(ctx->leaky_bucket_burst)) {
+ ctx->ratelimit_buckets = rspamd_lru_hash_new_full(ctx->max_buckets,
+ NULL, fuzzy_rl_bucket_free,
+ rspamd_inet_address_hash, rspamd_inet_address_equal);
+
+ rspamd_fuzzy_maybe_load_ratelimits(ctx);
+ }
+
+ /* Maps events */
+ ctx->resolver = rspamd_dns_resolver_init(worker->srv->logger,
+ ctx->event_loop,
+ worker->srv->cfg);
+ rspamd_map_watch(worker->srv->cfg, ctx->event_loop,
+ ctx->resolver, worker, RSPAMD_MAP_WATCH_WORKER);
+
+ /* Get peer pipe */
+ memset(&srv_cmd, 0, sizeof(srv_cmd));
+ srv_cmd.type = RSPAMD_SRV_SOCKETPAIR;
+ srv_cmd.cmd.spair.af = SOCK_DGRAM;
+ srv_cmd.cmd.spair.pair_num = worker->index;
+ memset(srv_cmd.cmd.spair.pair_id, 0, sizeof(srv_cmd.cmd.spair.pair_id));
+ /* 6 bytes of id (including \0) and bind_conf id */
+ G_STATIC_ASSERT(sizeof(srv_cmd.cmd.spair.pair_id) >=
+ sizeof("fuzzy") + sizeof(guint64));
+
+ memcpy(srv_cmd.cmd.spair.pair_id, "fuzzy", sizeof("fuzzy"));
+
+ /* Distinguish workers from each others... */
+ if (worker->cf->bind_conf && worker->cf->bind_conf->bind_line) {
+ guint64 bind_hash = rspamd_cryptobox_fast_hash(worker->cf->bind_conf->bind_line,
+ strlen(worker->cf->bind_conf->bind_line), 0xdeadbabe);
+
+ /* 8 more bytes */
+ memcpy(srv_cmd.cmd.spair.pair_id + sizeof("fuzzy"), &bind_hash,
+ sizeof(bind_hash));
+ }
+
+ rspamd_srv_send_command(worker, ctx->event_loop, &srv_cmd, -1,
+ fuzzy_peer_rep, ctx);
+
+ /*
+ * Extra fields available for this particular worker
+ */
+ luaL_Reg fuzzy_lua_reg = {
+ .name = "add_fuzzy_pre_handler",
+ .func = lua_fuzzy_add_pre_handler,
+ };
+ rspamd_lua_add_metamethod(ctx->cfg->lua_state, "rspamd{worker}", &fuzzy_lua_reg);
+ fuzzy_lua_reg = (luaL_Reg){
+ .name = "add_fuzzy_post_handler",
+ .func = lua_fuzzy_add_post_handler,
+ };
+ rspamd_lua_add_metamethod(ctx->cfg->lua_state, "rspamd{worker}", &fuzzy_lua_reg);
+ fuzzy_lua_reg = (luaL_Reg){
+ .name = "add_fuzzy_blacklist_handler",
+ .func = lua_fuzzy_add_blacklist_handler,
+ };
+ rspamd_lua_add_metamethod(ctx->cfg->lua_state, "rspamd{worker}", &fuzzy_lua_reg);
+
+ rspamd_lua_run_postloads(ctx->cfg->lua_state, ctx->cfg, ctx->event_loop,
+ worker);
+
+ ev_loop(ctx->event_loop, 0);
+ rspamd_worker_block_signals();
+
+ if (ctx->peer_fd != -1) {
+ if (worker->index == 0) {
+ ev_io_stop(ctx->event_loop, &ctx->peer_ev);
+ }
+ close(ctx->peer_fd);
+ }
+
+ if (worker->index == 0 && ctx->updates_pending->len > 0) {
+
+ msg_info_config("start another event loop to sync fuzzy storage");
+
+ if (rspamd_fuzzy_process_updates_queue(ctx, local_db_name, TRUE)) {
+ ev_loop(ctx->event_loop, 0);
+ msg_info_config("sync cycle is done");
+ }
+ else {
+ msg_info_config("no need to sync");
+ }
+ }
+
+ rspamd_fuzzy_backend_close(ctx->backend);
+
+ if (worker->index == 0) {
+ g_array_free(ctx->updates_pending, TRUE);
+ ctx->updates_pending = NULL;
+ }
+
+ if (ctx->keypair_cache) {
+ rspamd_keypair_cache_destroy(ctx->keypair_cache);
+ }
+
+ if (ctx->ratelimit_buckets) {
+ /* Try the best to save ratelimits from the proper worker */
+ if ((!ctx->dedicated_update_worker && worker->index == 0) ||
+ (ctx->dedicated_update_worker && worker->index == 1)) {
+ rspamd_fuzzy_maybe_save_ratelimits(ctx);
+ }
+
+ rspamd_lru_hash_destroy(ctx->ratelimit_buckets);
+ }
+
+ if (ctx->lua_pre_handler_cbref != -1) {
+ luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, ctx->lua_pre_handler_cbref);
+ }
+
+ if (ctx->lua_post_handler_cbref != -1) {
+ luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, ctx->lua_post_handler_cbref);
+ }
+
+ if (ctx->lua_blacklist_cbref != -1) {
+ luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, ctx->lua_blacklist_cbref);
+ }
+
+ if (ctx->default_forbidden_ids) {
+ kh_destroy(fuzzy_key_ids_set, ctx->default_forbidden_ids);
+ }
+
+ if (ctx->weak_ids) {
+ kh_destroy(fuzzy_key_ids_set, ctx->weak_ids);
+ }
+
+ REF_RELEASE(ctx->cfg);
+ rspamd_log_close(worker->srv->logger);
+ rspamd_unset_crash_handler(worker->srv);
+
+ exit(EXIT_SUCCESS);
+}
diff --git a/src/hs_helper.c b/src/hs_helper.c
new file mode 100644
index 0000000..438035e
--- /dev/null
+++ b/src/hs_helper.c
@@ -0,0 +1,426 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "libutil/util.h"
+#include "libserver/cfg_file.h"
+#include "libserver/cfg_rcl.h"
+#include "libserver/worker_util.h"
+#include "libserver/rspamd_control.h"
+#include "unix-std.h"
+
+#ifdef HAVE_GLOB_H
+#include <glob.h>
+#endif
+
+static gpointer init_hs_helper(struct rspamd_config *cfg);
+__attribute__((noreturn)) static void start_hs_helper(struct rspamd_worker *worker);
+
+worker_t hs_helper_worker = {
+ "hs_helper", /* Name */
+ init_hs_helper, /* Init function */
+ start_hs_helper, /* Start function */
+ RSPAMD_WORKER_UNIQUE | RSPAMD_WORKER_KILLABLE | RSPAMD_WORKER_ALWAYS_START | RSPAMD_WORKER_NO_TERMINATE_DELAY,
+ RSPAMD_WORKER_SOCKET_NONE,
+ RSPAMD_WORKER_VER /* Version info */
+};
+
+static const gdouble default_max_time = 1.0;
+static const gdouble default_recompile_time = 60.0;
+static const guint64 rspamd_hs_helper_magic = 0x22d310157a2288a0ULL;
+
+/*
+ * Worker's context
+ */
+struct hs_helper_ctx {
+ guint64 magic;
+ /* Events base */
+ struct ev_loop *event_loop;
+ /* DNS resolver */
+ struct rspamd_dns_resolver *resolver;
+ /* Config */
+ struct rspamd_config *cfg;
+ /* END OF COMMON PART */
+ gchar *hs_dir;
+ gboolean loaded;
+ gdouble max_time;
+ gdouble recompile_time;
+ ev_timer recompile_timer;
+};
+
+static gpointer
+init_hs_helper(struct rspamd_config *cfg)
+{
+ struct hs_helper_ctx *ctx;
+ GQuark type;
+
+ type = g_quark_try_string("hs_helper");
+ ctx = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*ctx));
+
+ ctx->magic = rspamd_hs_helper_magic;
+ ctx->cfg = cfg;
+ ctx->hs_dir = NULL;
+ ctx->max_time = default_max_time;
+ ctx->recompile_time = default_recompile_time;
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "cache_dir",
+ rspamd_rcl_parse_struct_string,
+ ctx,
+ G_STRUCT_OFFSET(struct hs_helper_ctx, hs_dir),
+ 0,
+ "Directory where to save hyperscan compiled expressions");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "max_time",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct hs_helper_ctx, max_time),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Maximum time to wait for compilation of a single expression");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "recompile",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct hs_helper_ctx, recompile_time),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Time between recompilation checks");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "timeout",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct hs_helper_ctx, max_time),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Maximum time to wait for compilation of a single expression");
+
+ return ctx;
+}
+
+/**
+ * Clean
+ */
+static gboolean
+rspamd_hs_helper_cleanup_dir(struct hs_helper_ctx *ctx, gboolean forced)
+{
+ struct stat st;
+ glob_t globbuf;
+ guint len, i;
+ gint rc;
+ gchar *pattern;
+ gboolean ret = TRUE;
+ pid_t our_pid = getpid();
+
+ if (getenv("RSPAMD_NO_CLEANUP")) {
+ /* Skip all cleanup */
+ return TRUE;
+ }
+
+ if (stat(ctx->hs_dir, &st) == -1) {
+ msg_err("cannot stat path %s, %s",
+ ctx->hs_dir,
+ strerror(errno));
+ return FALSE;
+ }
+
+ globbuf.gl_offs = 0;
+ /*
+ * We reuse this buffer for .new patterns as well, so allocate with some
+ * margin
+ */
+ len = strlen(ctx->hs_dir) + 1 + sizeof("*.hs") + sizeof(G_DIR_SEPARATOR);
+ pattern = g_malloc(len);
+ rspamd_snprintf(pattern, len, "%s%c%s", ctx->hs_dir, G_DIR_SEPARATOR, "*.hs");
+
+ if ((rc = glob(pattern, 0, NULL, &globbuf)) == 0) {
+ for (i = 0; i < globbuf.gl_pathc; i++) {
+ GError *err = NULL;
+
+ if (forced) {
+ g_set_error(&err, g_quark_from_static_string("re_cache"),
+ 0, "forced removal");
+ }
+
+ if (forced ||
+ !rspamd_re_cache_is_valid_hyperscan_file(ctx->cfg->re_cache,
+ globbuf.gl_pathv[i], TRUE, TRUE, &err)) {
+ if (unlink(globbuf.gl_pathv[i]) == -1) {
+ msg_err("cannot unlink %s: %s; reason for expiration: %e", globbuf.gl_pathv[i],
+ strerror(errno), err);
+ ret = FALSE;
+ }
+ else {
+ msg_notice("successfully removed outdated hyperscan file: %s; reason for expiration: %e",
+ globbuf.gl_pathv[i], err);
+ }
+ }
+
+ if (err) {
+ g_error_free(err);
+ }
+ }
+ }
+ else if (rc != GLOB_NOMATCH) {
+ msg_err("glob %s failed: %s", pattern, strerror(errno));
+ ret = FALSE;
+ }
+
+ globfree(&globbuf);
+
+ memset(&globbuf, 0, sizeof(globbuf));
+ rspamd_snprintf(pattern, len, "%s%c%s", ctx->hs_dir, G_DIR_SEPARATOR, "*.hs.new");
+ if ((rc = glob(pattern, 0, NULL, &globbuf)) == 0) {
+ for (i = 0; i < globbuf.gl_pathc; i++) {
+ /* Check if we have a pid in the filename */
+ const gchar *end_num = globbuf.gl_pathv[i] +
+ strlen(globbuf.gl_pathv[i]) - (sizeof(".hs.new") - 1);
+ const gchar *p = end_num - 1;
+ pid_t foreign_pid = -1;
+
+ while (p > globbuf.gl_pathv[i]) {
+ if (g_ascii_isdigit(*p)) {
+ p--;
+ }
+ else {
+ p++;
+ break;
+ }
+ }
+
+ gulong ul;
+ if (p < end_num && rspamd_strtoul(p, end_num - p, &ul)) {
+ foreign_pid = ul;
+ }
+
+ /*
+ * Remove only files that was left by us or some non-existing process
+ * There could be another race condition but it would just leave
+ * extra files which is relatively innocent?
+ */
+ if (foreign_pid == -1 || foreign_pid == our_pid || kill(foreign_pid, 0) == -1) {
+ if (unlink(globbuf.gl_pathv[i]) == -1) {
+ msg_err("cannot unlink %s: %s", globbuf.gl_pathv[i],
+ strerror(errno));
+ ret = FALSE;
+ }
+ else {
+ msg_notice("successfully removed outdated hyperscan temporary file: %s; "
+ "pid of the file creator process: %P",
+ globbuf.gl_pathv[i],
+ foreign_pid);
+ }
+ }
+ else {
+ msg_notice("skip removal of the hyperscan temporary file: %s; "
+ "pid of the file creator process: %P",
+ globbuf.gl_pathv[i],
+ foreign_pid);
+ }
+ }
+ }
+ else if (rc != GLOB_NOMATCH) {
+ msg_err("glob %s failed: %s", pattern, strerror(errno));
+ ret = FALSE;
+ }
+
+ globfree(&globbuf);
+ g_free(pattern);
+
+ return ret;
+}
+
+/* Bad hack, but who cares */
+static gboolean hack_global_forced;
+
+static void
+rspamd_rs_delayed_cb(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_worker *worker = (struct rspamd_worker *) w->data;
+ static struct rspamd_srv_command srv_cmd;
+ struct hs_helper_ctx *ctx;
+
+ ctx = (struct hs_helper_ctx *) worker->ctx;
+ memset(&srv_cmd, 0, sizeof(srv_cmd));
+ srv_cmd.type = RSPAMD_SRV_HYPERSCAN_LOADED;
+ rspamd_strlcpy(srv_cmd.cmd.hs_loaded.cache_dir, ctx->hs_dir,
+ sizeof(srv_cmd.cmd.hs_loaded.cache_dir));
+ srv_cmd.cmd.hs_loaded.forced = hack_global_forced;
+ hack_global_forced = FALSE;
+
+ rspamd_srv_send_command(worker,
+ ctx->event_loop, &srv_cmd, -1, NULL, NULL);
+ ev_timer_stop(EV_A_ w);
+ g_free(w);
+
+ ev_timer_again(EV_A_ & ctx->recompile_timer);
+}
+
+static void
+rspamd_rs_compile_cb(guint ncompiled, GError *err, void *cbd)
+{
+ struct rspamd_worker *worker = (struct rspamd_worker *) cbd;
+ ev_timer *tm;
+ ev_tstamp when = 0.0;
+ struct hs_helper_ctx *ctx;
+
+ ctx = (struct hs_helper_ctx *) worker->ctx;
+
+ if (err != NULL) {
+ /* Failed to compile: log and go out */
+ msg_err("cannot compile Hyperscan database: %e", err);
+
+ return;
+ }
+
+ if (ncompiled > 0) {
+ /* Enforce update for other workers */
+ hack_global_forced = TRUE;
+ }
+
+ /*
+ * Do not send notification unless all other workers are started
+ * XXX: now we just sleep for 1 seconds to ensure that
+ */
+ if (!ctx->loaded) {
+ when = 1.0; /* Postpone */
+ ctx->loaded = TRUE;
+ msg_info("compiled %d regular expressions to the hyperscan tree, "
+ "postpone loaded notification for %.0f seconds to avoid races",
+ ncompiled,
+ when);
+ }
+ else {
+ msg_info("compiled %d regular expressions to the hyperscan tree, "
+ "send loaded notification",
+ ncompiled);
+ }
+
+ tm = g_malloc0(sizeof(*tm));
+ tm->data = (void *) worker;
+ ev_timer_init(tm, rspamd_rs_delayed_cb, when, 0);
+ ev_timer_start(ctx->event_loop, tm);
+}
+
+static gboolean
+rspamd_rs_compile(struct hs_helper_ctx *ctx, struct rspamd_worker *worker,
+ gboolean forced)
+{
+#if !defined(__aarch64__) && !defined(__powerpc64__)
+ if (!(ctx->cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) {
+ msg_warn("CPU doesn't have SSSE3 instructions set "
+ "required for hyperscan, disable hyperscan compilation");
+ return FALSE;
+ }
+#endif
+
+ if (!rspamd_hs_helper_cleanup_dir(ctx, forced)) {
+ msg_warn("cannot cleanup cache dir '%s'", ctx->hs_dir);
+ }
+
+ hack_global_forced = forced; /* killmeplease */
+ rspamd_re_cache_compile_hyperscan(ctx->cfg->re_cache,
+ ctx->hs_dir, ctx->max_time, !forced,
+ ctx->event_loop,
+ rspamd_rs_compile_cb,
+ (void *) worker);
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_hs_helper_reload(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker, gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud)
+{
+ struct rspamd_control_reply rep;
+ struct hs_helper_ctx *ctx = ud;
+
+ msg_info("recompiling hyperscan expressions after receiving reload command");
+ memset(&rep, 0, sizeof(rep));
+ rep.type = RSPAMD_CONTROL_RECOMPILE;
+ rep.reply.recompile.status = 0;
+
+ /* We write reply before actual recompilation as it takes a lot of time */
+ if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
+ msg_err("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ /* Stop recompile */
+ ev_timer_stop(ctx->event_loop, &ctx->recompile_timer);
+ rspamd_rs_compile(ctx, worker, TRUE);
+
+ return TRUE;
+}
+
+static void
+rspamd_hs_helper_timer(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_worker *worker = (struct rspamd_worker *) w->data;
+ struct hs_helper_ctx *ctx;
+ double tim;
+
+ ctx = worker->ctx;
+ tim = rspamd_time_jitter(ctx->recompile_time, 0);
+ w->repeat = tim;
+ rspamd_rs_compile(ctx, worker, FALSE);
+}
+
+static void
+start_hs_helper(struct rspamd_worker *worker)
+{
+ struct hs_helper_ctx *ctx = worker->ctx;
+ double tim;
+
+ g_assert(rspamd_worker_check_context(worker->ctx, rspamd_hs_helper_magic));
+ ctx->cfg = worker->srv->cfg;
+
+ if (ctx->hs_dir == NULL) {
+ ctx->hs_dir = ctx->cfg->hs_cache_dir;
+ }
+ if (ctx->hs_dir == NULL) {
+ ctx->hs_dir = RSPAMD_DBDIR "/";
+ }
+
+ ctx->event_loop = rspamd_prepare_worker(worker,
+ "hs_helper",
+ NULL);
+
+ if (!rspamd_rs_compile(ctx, worker, FALSE)) {
+ /* Tell main not to respawn more workers */
+ exit(EXIT_SUCCESS);
+ }
+
+ rspamd_control_worker_add_cmd_handler(worker, RSPAMD_CONTROL_RECOMPILE,
+ rspamd_hs_helper_reload, ctx);
+
+ ctx->recompile_timer.data = worker;
+ tim = rspamd_time_jitter(ctx->recompile_time, 0);
+ ev_timer_init(&ctx->recompile_timer, rspamd_hs_helper_timer, tim, 0.0);
+ ev_timer_start(ctx->event_loop, &ctx->recompile_timer);
+
+ ev_loop(ctx->event_loop, 0);
+ rspamd_worker_block_signals();
+
+ rspamd_log_close(worker->srv->logger);
+ REF_RELEASE(ctx->cfg);
+ rspamd_unset_crash_handler(worker->srv);
+
+ exit(EXIT_SUCCESS);
+}
diff --git a/src/libcryptobox/AsmOpt.cmake b/src/libcryptobox/AsmOpt.cmake
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/libcryptobox/AsmOpt.cmake
diff --git a/src/libcryptobox/CMakeLists.txt b/src/libcryptobox/CMakeLists.txt
new file mode 100644
index 0000000..d1c8e3d
--- /dev/null
+++ b/src/libcryptobox/CMakeLists.txt
@@ -0,0 +1,41 @@
+SET(CHACHASRC ${CMAKE_CURRENT_SOURCE_DIR}/chacha20/chacha.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/chacha20/ref.c)
+
+SET(BASE64SRC ${CMAKE_CURRENT_SOURCE_DIR}/base64/ref.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/base64/base64.c)
+
+IF (HAVE_AVX2)
+ IF ("${ARCH}" STREQUAL "x86_64")
+ SET(CHACHASRC ${CHACHASRC} ${CMAKE_CURRENT_SOURCE_DIR}/chacha20/avx2.S)
+ MESSAGE(STATUS "Cryptobox: AVX2 support is added (chacha20)")
+ ENDIF ()
+ SET(BASE64SRC ${BASE64SRC} ${CMAKE_CURRENT_SOURCE_DIR}/base64/avx2.c)
+ MESSAGE(STATUS "Cryptobox: AVX2 support is added (base64)")
+ENDIF (HAVE_AVX2)
+IF (HAVE_AVX)
+ IF ("${ARCH}" STREQUAL "x86_64")
+ SET(CHACHASRC ${CHACHASRC} ${CMAKE_CURRENT_SOURCE_DIR}/chacha20/avx.S)
+ MESSAGE(STATUS "Cryptobox: AVX support is added (chacha20)")
+ ENDIF ()
+ENDIF (HAVE_AVX)
+IF (HAVE_SSE2)
+ IF ("${ARCH}" STREQUAL "x86_64")
+ SET(CHACHASRC ${CHACHASRC} ${CMAKE_CURRENT_SOURCE_DIR}/chacha20/sse2.S)
+ MESSAGE(STATUS "Cryptobox: SSE2 support is added (chacha20)")
+ ENDIF ()
+ENDIF (HAVE_SSE2)
+IF (HAVE_SSE42)
+ IF ("${ARCH}" STREQUAL "x86_64")
+ SET(BASE64SRC ${BASE64SRC} ${CMAKE_CURRENT_SOURCE_DIR}/base64/sse42.c)
+ MESSAGE(STATUS "Cryptobox: SSE42 support is added (base64)")
+ ENDIF ()
+ENDIF (HAVE_SSE42)
+
+CONFIGURE_FILE(platform_config.h.in platform_config.h)
+INCLUDE_DIRECTORIES("${CMAKE_CURRENT_BINARY_DIR}")
+SET(LIBCRYPTOBOXSRC ${CMAKE_CURRENT_SOURCE_DIR}/cryptobox.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/keypair.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/keypairs_cache.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/catena/catena.c)
+
+SET(RSPAMD_CRYPTOBOX ${LIBCRYPTOBOXSRC} ${CHACHASRC} ${BASE64SRC} PARENT_SCOPE)
diff --git a/src/libcryptobox/base64/avx2.c b/src/libcryptobox/base64/avx2.c
new file mode 100644
index 0000000..38abffc
--- /dev/null
+++ b/src/libcryptobox/base64/avx2.c
@@ -0,0 +1,287 @@
+/*-
+ * Copyright 2018 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*-
+Copyright (c) 2013-2015, Alfred Klomp
+Copyright (c) 2018, Vsevolod Stakhov
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "cryptobox.h"
+
+extern const uint8_t base64_table_dec[256];
+
+#ifdef RSPAMD_HAS_TARGET_ATTR
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC push_options
+#pragma GCC target("avx2")
+#endif
+#ifndef __SSE2__
+#define __SSE2__
+#endif
+#ifndef __SSE__
+#define __SSE__
+#endif
+#ifndef __SSE4_2__
+#define __SSE4_2__
+#endif
+#ifndef __SSE4_1__
+#define __SSE4_1__
+#endif
+#ifndef __SSEE3__
+#define __SSEE3__
+#endif
+#ifndef __AVX__
+#define __AVX__
+#endif
+#ifndef __AVX2__
+#define __AVX2__
+#endif
+
+#include <immintrin.h>
+
+#define CMPGT(s, n) _mm256_cmpgt_epi8((s), _mm256_set1_epi8(n))
+#define CMPEQ(s, n) _mm256_cmpeq_epi8((s), _mm256_set1_epi8(n))
+#define REPLACE(s, n) _mm256_and_si256((s), _mm256_set1_epi8(n))
+#define RANGE(s, a, b) _mm256_andnot_si256(CMPGT((s), (b)), CMPGT((s), (a) -1))
+
+static inline __m256i
+dec_reshuffle(__m256i in) __attribute__((__target__("avx2")));
+
+static inline __m256i
+dec_reshuffle(__m256i in)
+{
+ // in, lower lane, bits, upper case are most significant bits, lower case are least significant bits:
+ // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ
+ // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG
+ // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD
+ // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA
+
+ const __m256i merge_ab_and_bc = _mm256_maddubs_epi16(in, _mm256_set1_epi32(0x01400140));
+ // 0000kkkk LLllllll 0000JJJJ JJjjKKKK
+ // 0000hhhh IIiiiiii 0000GGGG GGggHHHH
+ // 0000eeee FFffffff 0000DDDD DDddEEEE
+ // 0000bbbb CCcccccc 0000AAAA AAaaBBBB
+
+ __m256i out = _mm256_madd_epi16(merge_ab_and_bc, _mm256_set1_epi32(0x00011000));
+ // 00000000 JJJJJJjj KKKKkkkk LLllllll
+ // 00000000 GGGGGGgg HHHHhhhh IIiiiiii
+ // 00000000 DDDDDDdd EEEEeeee FFffffff
+ // 00000000 AAAAAAaa BBBBbbbb CCcccccc
+
+ // Pack bytes together in each lane:
+ out = _mm256_shuffle_epi8(out, _mm256_setr_epi8(
+ 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1,
+ 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1));
+ // 00000000 00000000 00000000 00000000
+ // LLllllll KKKKkkkk JJJJJJjj IIiiiiii
+ // HHHHhhhh GGGGGGgg FFffffff EEEEeeee
+ // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa
+
+ // Pack lanes
+ return _mm256_permutevar8x32_epi32(out, _mm256_setr_epi32(0, 1, 2, 4, 5, 6, -1, -1));
+}
+
+
+#define INNER_LOOP_AVX2 \
+ while (inlen >= 45) { \
+ __m256i str = _mm256_loadu_si256((__m256i *) c); \
+ const __m256i lut_lo = _mm256_setr_epi8( \
+ 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, \
+ 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A, \
+ 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, \
+ 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A); \
+ const __m256i lut_hi = _mm256_setr_epi8( \
+ 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, \
+ 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, \
+ 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, \
+ 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10); \
+ const __m256i lut_roll = _mm256_setr_epi8( \
+ 0, 16, 19, 4, -65, -65, -71, -71, \
+ 0, 0, 0, 0, 0, 0, 0, 0, \
+ 0, 16, 19, 4, -65, -65, -71, -71, \
+ 0, 0, 0, 0, 0, 0, 0, 0); \
+ const __m256i mask_2F = _mm256_set1_epi8(0x2f); \
+ const __m256i hi_nibbles = _mm256_and_si256(_mm256_srli_epi32(str, 4), mask_2F); \
+ const __m256i lo_nibbles = _mm256_and_si256(str, mask_2F); \
+ const __m256i hi = _mm256_shuffle_epi8(lut_hi, hi_nibbles); \
+ const __m256i lo = _mm256_shuffle_epi8(lut_lo, lo_nibbles); \
+ const __m256i eq_2F = _mm256_cmpeq_epi8(str, mask_2F); \
+ const __m256i roll = _mm256_shuffle_epi8(lut_roll, _mm256_add_epi8(eq_2F, hi_nibbles)); \
+ if (!_mm256_testz_si256(lo, hi)) { \
+ seen_error = true; \
+ break; \
+ } \
+ str = _mm256_add_epi8(str, roll); \
+ str = dec_reshuffle(str); \
+ _mm256_storeu_si256((__m256i *) o, str); \
+ c += 32; \
+ o += 24; \
+ outl += 24; \
+ inlen -= 32; \
+ }
+
+int base64_decode_avx2(const char *in, size_t inlen,
+ unsigned char *out, size_t *outlen) __attribute__((__target__("avx2")));
+int base64_decode_avx2(const char *in, size_t inlen,
+ unsigned char *out, size_t *outlen)
+{
+ ssize_t ret = 0;
+ const uint8_t *c = (const uint8_t *) in;
+ uint8_t *o = (uint8_t *) out;
+ uint8_t q, carry;
+ size_t outl = 0;
+ size_t leftover = 0;
+ bool seen_error = false;
+
+repeat:
+ switch (leftover) {
+ for (;;) {
+ case 0:
+ if (G_LIKELY(!seen_error)) {
+ INNER_LOOP_AVX2
+ }
+
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ ret = 0;
+ break;
+ }
+ carry = q << 2;
+ leftover++;
+
+ case 1:
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ ret = 0;
+ break;
+ }
+ *o++ = carry | (q >> 4);
+ carry = q << 4;
+ leftover++;
+ outl++;
+
+ case 2:
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ leftover++;
+
+ if (q == 254) {
+ if (inlen-- != 0) {
+ leftover = 0;
+ q = base64_table_dec[*c++];
+ ret = ((q == 254) && (inlen == 0)) ? 1 : 0;
+ break;
+ }
+ else {
+ ret = 1;
+ break;
+ }
+ }
+ else {
+ leftover--;
+ }
+ /* If we get here, there was an error: */
+ break;
+ }
+ *o++ = carry | (q >> 2);
+ carry = q << 6;
+ leftover++;
+ outl++;
+
+ case 3:
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ /*
+ * When q == 254, the input char is '='. Return 1 and EOF.
+ * When q == 255, the input char is invalid. Return 0 and EOF.
+ */
+ if (q == 254 && inlen == 0) {
+ ret = 1;
+ leftover = 0;
+ }
+ else {
+ ret = 0;
+ }
+
+ break;
+ }
+
+ *o++ = carry | q;
+ carry = 0;
+ leftover = 0;
+ outl++;
+ }
+ }
+
+ if (!ret && inlen > 0) {
+ /* Skip to the next valid character in input */
+ while (inlen > 0 && base64_table_dec[*c] >= 254) {
+ c++;
+ inlen--;
+ }
+
+ if (inlen > 0) {
+ seen_error = false;
+ goto repeat;
+ }
+ }
+
+ *outlen = outl;
+
+ return ret;
+}
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC pop_options
+#endif
+#endif
diff --git a/src/libcryptobox/base64/base64.c b/src/libcryptobox/base64/base64.c
new file mode 100644
index 0000000..e868924
--- /dev/null
+++ b/src/libcryptobox/base64/base64.c
@@ -0,0 +1,445 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "cryptobox.h"
+#include "base64.h"
+#include "platform_config.h"
+#include "str_util.h"
+#include "util.h"
+#include "contrib/libottery/ottery.h"
+
+extern unsigned cpu_config;
+const uint8_t
+ base64_table_dec[256] =
+ {
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 62,
+ 255,
+ 255,
+ 255,
+ 63,
+ 52,
+ 53,
+ 54,
+ 55,
+ 56,
+ 57,
+ 58,
+ 59,
+ 60,
+ 61,
+ 255,
+ 255,
+ 255,
+ 254,
+ 255,
+ 255,
+ 255,
+ 0,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14,
+ 15,
+ 16,
+ 17,
+ 18,
+ 19,
+ 20,
+ 21,
+ 22,
+ 23,
+ 24,
+ 25,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 26,
+ 27,
+ 28,
+ 29,
+ 30,
+ 31,
+ 32,
+ 33,
+ 34,
+ 35,
+ 36,
+ 37,
+ 38,
+ 39,
+ 40,
+ 41,
+ 42,
+ 43,
+ 44,
+ 45,
+ 46,
+ 47,
+ 48,
+ 49,
+ 50,
+ 51,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+ 255,
+};
+
+static const char base64_alphabet[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+/";
+
+typedef struct base64_impl {
+ unsigned short enabled;
+ unsigned short min_len;
+ unsigned int cpu_flags;
+ const char *desc;
+ int (*decode)(const char *in, size_t inlen,
+ unsigned char *out, size_t *outlen);
+} base64_impl_t;
+
+#define BASE64_DECLARE(ext) \
+ int base64_decode_##ext(const char *in, size_t inlen, unsigned char *out, size_t *outlen);
+#define BASE64_IMPL(cpuflags, min_len, desc, ext) \
+ { \
+ 0, (min_len), (cpuflags), desc, base64_decode_##ext \
+ }
+
+BASE64_DECLARE(ref);
+#define BASE64_REF BASE64_IMPL(0, 0, "ref", ref)
+
+#ifdef RSPAMD_HAS_TARGET_ATTR
+#if defined(HAVE_SSE42) && defined(__x86_64__)
+int base64_decode_sse42(const char *in, size_t inlen,
+ unsigned char *out, size_t *outlen) __attribute__((__target__("sse4.2")));
+
+BASE64_DECLARE(sse42);
+#define BASE64_SSE42 BASE64_IMPL(CPUID_SSE42, 24, "sse42", sse42)
+#endif
+#endif
+
+#ifdef RSPAMD_HAS_TARGET_ATTR
+#if defined(HAVE_AVX2) && defined(__x86_64__)
+int base64_decode_avx2(const char *in, size_t inlen,
+ unsigned char *out, size_t *outlen) __attribute__((__target__("avx2")));
+
+BASE64_DECLARE(avx2);
+#define BASE64_AVX2 BASE64_IMPL(CPUID_AVX2, 128, "avx2", avx2)
+#endif
+#endif
+
+static base64_impl_t base64_list[] = {
+ BASE64_REF,
+#ifdef BASE64_SSE42
+ BASE64_SSE42,
+#endif
+#ifdef BASE64_AVX2
+ BASE64_AVX2,
+#endif
+};
+
+static const base64_impl_t *base64_ref = &base64_list[0];
+
+const char *
+base64_load(void)
+{
+ guint i;
+ const base64_impl_t *opt_impl = base64_ref;
+
+ /* Enable reference */
+ base64_list[0].enabled = true;
+
+ if (cpu_config != 0) {
+ for (i = 1; i < G_N_ELEMENTS(base64_list); i++) {
+ if (base64_list[i].cpu_flags & cpu_config) {
+ base64_list[i].enabled = true;
+ opt_impl = &base64_list[i];
+ }
+ }
+ }
+
+
+ return opt_impl->desc;
+}
+
+gboolean
+rspamd_cryptobox_base64_decode(const gchar *in, gsize inlen,
+ guchar *out, gsize *outlen)
+{
+ const base64_impl_t *opt_impl = base64_ref;
+
+ for (gint i = G_N_ELEMENTS(base64_list) - 1; i > 0; i--) {
+ if (base64_list[i].enabled && base64_list[i].min_len <= inlen) {
+ opt_impl = &base64_list[i];
+ break;
+ }
+ }
+
+ return opt_impl->decode(in, inlen, out, outlen);
+}
+
+double
+base64_test(bool generic, size_t niters, size_t len, size_t str_len)
+{
+ size_t cycles;
+ guchar *in, *out, *tmp;
+ gdouble t1, t2, total = 0;
+ gsize outlen;
+
+ g_assert(len > 0);
+ in = g_malloc(len);
+ tmp = g_malloc(len);
+ ottery_rand_bytes(in, len);
+
+ out = rspamd_encode_base64_fold(in, len, str_len, &outlen,
+ RSPAMD_TASK_NEWLINES_CRLF);
+
+ if (generic) {
+ base64_list[0].decode(out, outlen, tmp, &len);
+ }
+ else {
+ rspamd_cryptobox_base64_decode(out, outlen, tmp, &len);
+ }
+
+ g_assert(memcmp(in, tmp, len) == 0);
+
+ for (cycles = 0; cycles < niters; cycles++) {
+ t1 = rspamd_get_ticks(TRUE);
+ if (generic) {
+ base64_list[0].decode(out, outlen, tmp, &len);
+ }
+ else {
+ rspamd_cryptobox_base64_decode(out, outlen, tmp, &len);
+ }
+ t2 = rspamd_get_ticks(TRUE);
+ total += t2 - t1;
+ }
+
+ g_free(in);
+ g_free(tmp);
+ g_free(out);
+
+ return total;
+}
+
+
+gboolean
+rspamd_cryptobox_base64_is_valid(const gchar *in, gsize inlen)
+{
+ const guchar *p, *end;
+
+ if (inlen == 0) {
+ return FALSE;
+ }
+
+ p = in;
+ end = in + inlen;
+
+ while (p < end && *p != '=') {
+ if (!g_ascii_isspace(*p)) {
+ if (base64_table_dec[*p] == 255) {
+ return FALSE;
+ }
+ }
+ p++;
+ }
+
+ return TRUE;
+} \ No newline at end of file
diff --git a/src/libcryptobox/base64/base64.h b/src/libcryptobox/base64/base64.h
new file mode 100644
index 0000000..f53c80a
--- /dev/null
+++ b/src/libcryptobox/base64/base64.h
@@ -0,0 +1,31 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBCRYPTOBOX_BASE64_BASE64_H_
+#define SRC_LIBCRYPTOBOX_BASE64_BASE64_H_
+
+#include "config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+const char *base64_load(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBCRYPTOBOX_BASE64_BASE64_H_ */
diff --git a/src/libcryptobox/base64/ref.c b/src/libcryptobox/base64/ref.c
new file mode 100644
index 0000000..61df68e
--- /dev/null
+++ b/src/libcryptobox/base64/ref.c
@@ -0,0 +1,241 @@
+/*-
+Copyright (c) 2013-2015, Alfred Klomp
+Copyright (c) 2016, Vsevolod Stakhov
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "libutil/util.h"
+
+extern const uint8_t base64_table_dec[256];
+
+#define INNER_LOOP_64 \
+ do { \
+ uint64_t str, res, dec; \
+ bool aligned = rspamd_is_aligned_as(c, str); \
+ while (inlen >= 13) { \
+ if (aligned) { str = *(uint64_t *) c; } \
+ else { \
+ memcpy(&str, c, sizeof(str)); \
+ } \
+ str = GUINT64_TO_BE(str); \
+ if ((dec = base64_table_dec[str >> 56]) > 63) { \
+ break; \
+ } \
+ res = dec << 58; \
+ if ((dec = base64_table_dec[(str >> 48) & 0xFF]) > 63) { \
+ break; \
+ } \
+ res |= dec << 52; \
+ if ((dec = base64_table_dec[(str >> 40) & 0xFF]) > 63) { \
+ break; \
+ } \
+ res |= dec << 46; \
+ if ((dec = base64_table_dec[(str >> 32) & 0xFF]) > 63) { \
+ break; \
+ } \
+ res |= dec << 40; \
+ if ((dec = base64_table_dec[(str >> 24) & 0xFF]) > 63) { \
+ break; \
+ } \
+ res |= dec << 34; \
+ if ((dec = base64_table_dec[(str >> 16) & 0xFF]) > 63) { \
+ break; \
+ } \
+ res |= dec << 28; \
+ if ((dec = base64_table_dec[(str >> 8) & 0xFF]) > 63) { \
+ break; \
+ } \
+ res |= dec << 22; \
+ if ((dec = base64_table_dec[str & 0xFF]) > 63) { \
+ break; \
+ } \
+ res |= dec << 16; \
+ res = GUINT64_FROM_BE(res); \
+ memcpy(o, &res, sizeof(res)); \
+ c += 8; \
+ o += 6; \
+ outl += 6; \
+ inlen -= 8; \
+ } \
+ } while (0)
+
+#define INNER_LOOP_32 \
+ do { \
+ uint32_t str, res, dec; \
+ bool aligned = rspamd_is_aligned_as(c, str); \
+ while (inlen >= 8) { \
+ if (aligned) { str = *(uint32_t *) c; } \
+ else { \
+ memcpy(&str, c, sizeof(str)); \
+ } \
+ str = GUINT32_TO_BE(str); \
+ if ((dec = base64_table_dec[str >> 24]) > 63) { \
+ break; \
+ } \
+ res = dec << 26; \
+ if ((dec = base64_table_dec[(str >> 16) & 0xFF]) > 63) { \
+ break; \
+ } \
+ res |= dec << 20; \
+ if ((dec = base64_table_dec[(str >> 8) & 0xFF]) > 63) { \
+ break; \
+ } \
+ res |= dec << 14; \
+ if ((dec = base64_table_dec[str & 0xFF]) > 63) { \
+ break; \
+ } \
+ res |= dec << 8; \
+ res = GUINT32_FROM_BE(res); \
+ memcpy(o, &res, sizeof(res)); \
+ c += 4; \
+ o += 3; \
+ outl += 3; \
+ inlen -= 4; \
+ } \
+ } while (0)
+
+
+int base64_decode_ref(const char *in, size_t inlen,
+ unsigned char *out, size_t *outlen)
+{
+ ssize_t ret = 0;
+ const uint8_t *c = (const uint8_t *) in;
+ uint8_t *o = (uint8_t *) out;
+ uint8_t q, carry;
+ size_t outl = 0;
+ size_t leftover = 0;
+
+repeat:
+ switch (leftover) {
+ for (;;) {
+ case 0:
+#if defined(__LP64__)
+ INNER_LOOP_64;
+#else
+ INNER_LOOP_32;
+#endif
+
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ ret = 0;
+ break;
+ }
+ carry = (uint8_t) (q << 2);
+ leftover++;
+
+ case 1:
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ ret = 0;
+ break;
+ }
+ *o++ = carry | (q >> 4);
+ carry = (uint8_t) (q << 4);
+ leftover++;
+ outl++;
+
+ case 2:
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ leftover++;
+
+ if (q == 254) {
+ if (inlen-- != 0) {
+ leftover = 0;
+ q = base64_table_dec[*c++];
+ ret = ((q == 254) && (inlen == 0)) ? 1 : 0;
+ break;
+ }
+ else {
+ ret = 1;
+ break;
+ }
+ }
+ else {
+ leftover--;
+ }
+ /* If we get here, there was an error: */
+ break;
+ }
+ *o++ = carry | (q >> 2);
+ carry = (uint8_t) (q << 6);
+ leftover++;
+ outl++;
+
+ case 3:
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ /*
+ * When q == 254, the input char is '='. Return 1 and EOF.
+ * When q == 255, the input char is invalid. Return 0 and EOF.
+ */
+ if (q == 254 && inlen == 0) {
+ ret = 1;
+ leftover = 0;
+ }
+ else {
+ ret = 0;
+ }
+
+ break;
+ }
+
+ *o++ = carry | q;
+ carry = 0;
+ leftover = 0;
+ outl++;
+ }
+ }
+
+ if (!ret && inlen > 0) {
+ /* Skip to the next valid character in input */
+ while (inlen > 0 && base64_table_dec[*c] >= 254) {
+ c++;
+ inlen--;
+ }
+
+ if (inlen > 0) {
+ goto repeat;
+ }
+ }
+
+ *outlen = outl;
+
+ return ret;
+}
diff --git a/src/libcryptobox/base64/sse42.c b/src/libcryptobox/base64/sse42.c
new file mode 100644
index 0000000..36070ab
--- /dev/null
+++ b/src/libcryptobox/base64/sse42.c
@@ -0,0 +1,268 @@
+/*-
+ * Copyright 2017 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*-
+Copyright (c) 2013-2015, Alfred Klomp
+Copyright (c) 2016, Vsevolod Stakhov
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "cryptobox.h"
+
+extern const uint8_t base64_table_dec[256];
+
+#ifdef RSPAMD_HAS_TARGET_ATTR
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC push_options
+#pragma GCC target("sse4.2")
+#endif
+#ifndef __SSE2__
+#define __SSE2__
+#endif
+#ifndef __SSE__
+#define __SSE__
+#endif
+#ifndef __SSE4_2__
+#define __SSE4_2__
+#endif
+#ifndef __SSE4_1__
+#define __SSE4_1__
+#endif
+#ifndef __SSEE3__
+#define __SSEE3__
+#endif
+#include <xmmintrin.h>
+#include <nmmintrin.h>
+
+
+static inline __m128i
+dec_reshuffle(__m128i in) __attribute__((__target__("sse4.2")));
+
+static inline __m128i dec_reshuffle(__m128i in)
+{
+ // Mask in a single byte per shift:
+ const __m128i maskB2 = _mm_set1_epi32(0x003F0000);
+ const __m128i maskB1 = _mm_set1_epi32(0x00003F00);
+
+ // Pack bytes together:
+ __m128i out = _mm_srli_epi32(in, 16);
+
+ out = _mm_or_si128(out, _mm_srli_epi32(_mm_and_si128(in, maskB2), 2));
+
+ out = _mm_or_si128(out, _mm_slli_epi32(_mm_and_si128(in, maskB1), 12));
+
+ out = _mm_or_si128(out, _mm_slli_epi32(in, 26));
+
+ // Reshuffle and repack into 12-byte output format:
+ return _mm_shuffle_epi8(out, _mm_setr_epi8(
+ 3, 2, 1,
+ 7, 6, 5,
+ 11, 10, 9,
+ 15, 14, 13,
+ -1, -1, -1, -1));
+}
+
+#define CMPGT(s, n) _mm_cmpgt_epi8((s), _mm_set1_epi8(n))
+
+#define INNER_LOOP_SSE42 \
+ while (inlen >= 24) { \
+ __m128i str = _mm_loadu_si128((__m128i *) c); \
+ const __m128i lut = _mm_setr_epi8( \
+ 19, 16, 4, 4, \
+ 4, 4, 4, 4, \
+ 4, 4, 4, 4, \
+ 0, 0, -71, -65); \
+ const __m128i range = _mm_setr_epi8( \
+ '+', '+', \
+ '+', '+', \
+ '+', '+', \
+ '+', '+', \
+ '/', '/', \
+ '0', '9', \
+ 'A', 'Z', \
+ 'a', 'z'); \
+ if (_mm_cmpistrc(range, str, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | _SIDD_NEGATIVE_POLARITY)) { \
+ seen_error = true; \
+ break; \
+ } \
+ __m128i indices = _mm_subs_epu8(str, _mm_set1_epi8(46)); \
+ __m128i mask45 = CMPGT(str, 64); \
+ __m128i mask5 = CMPGT(str, 96); \
+ indices = _mm_andnot_si128(mask45, indices); \
+ mask45 = _mm_add_epi8(_mm_slli_epi16(_mm_abs_epi8(mask45), 4), mask45); \
+ indices = _mm_add_epi8(indices, mask45); \
+ indices = _mm_add_epi8(indices, mask5); \
+ __m128i delta = _mm_shuffle_epi8(lut, indices); \
+ str = _mm_add_epi8(str, delta); \
+ str = dec_reshuffle(str); \
+ _mm_storeu_si128((__m128i *) o, str); \
+ c += 16; \
+ o += 12; \
+ outl += 12; \
+ inlen -= 16; \
+ }
+
+int base64_decode_sse42(const char *in, size_t inlen,
+ unsigned char *out, size_t *outlen) __attribute__((__target__("sse4.2")));
+int base64_decode_sse42(const char *in, size_t inlen,
+ unsigned char *out, size_t *outlen)
+{
+ ssize_t ret = 0;
+ const uint8_t *c = (const uint8_t *) in;
+ uint8_t *o = (uint8_t *) out;
+ uint8_t q, carry;
+ size_t outl = 0;
+ size_t leftover = 0;
+ bool seen_error = false;
+
+repeat:
+ switch (leftover) {
+ for (;;) {
+ case 0:
+ if (G_LIKELY(!seen_error)) {
+ INNER_LOOP_SSE42
+ }
+
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ ret = 0;
+ break;
+ }
+ carry = q << 2;
+ leftover++;
+
+ case 1:
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ ret = 0;
+ break;
+ }
+ *o++ = carry | (q >> 4);
+ carry = q << 4;
+ leftover++;
+ outl++;
+
+ case 2:
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ leftover++;
+
+ if (q == 254) {
+ if (inlen-- != 0) {
+ leftover = 0;
+ q = base64_table_dec[*c++];
+ ret = ((q == 254) && (inlen == 0)) ? 1 : 0;
+ break;
+ }
+ else {
+ ret = 1;
+ break;
+ }
+ }
+ else {
+ leftover--;
+ }
+ /* If we get here, there was an error: */
+ break;
+ }
+ *o++ = carry | (q >> 2);
+ carry = q << 6;
+ leftover++;
+ outl++;
+
+ case 3:
+ if (inlen-- == 0) {
+ ret = 1;
+ break;
+ }
+ if ((q = base64_table_dec[*c++]) >= 254) {
+ /*
+ * When q == 254, the input char is '='. Return 1 and EOF.
+ * When q == 255, the input char is invalid. Return 0 and EOF.
+ */
+ if (q == 254 && inlen == 0) {
+ ret = 1;
+ leftover = 0;
+ }
+ else {
+ ret = 0;
+ }
+
+ break;
+ }
+
+ *o++ = carry | q;
+ carry = 0;
+ leftover = 0;
+ outl++;
+ }
+ }
+
+ if (!ret && inlen > 0) {
+ /* Skip to the next valid character in input */
+ while (inlen > 0 && base64_table_dec[*c] >= 254) {
+ c++;
+ inlen--;
+ }
+
+ if (inlen > 0) {
+ seen_error = false;
+ goto repeat;
+ }
+ }
+
+ *outlen = outl;
+
+ return ret;
+}
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC pop_options
+#endif
+#endif
diff --git a/src/libcryptobox/catena/LICENSE b/src/libcryptobox/catena/LICENSE
new file mode 100644
index 0000000..ff22dc8
--- /dev/null
+++ b/src/libcryptobox/catena/LICENSE
@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 cforler
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/src/libcryptobox/catena/README.md b/src/libcryptobox/catena/README.md
new file mode 100644
index 0000000..4a0f948
--- /dev/null
+++ b/src/libcryptobox/catena/README.md
@@ -0,0 +1,18 @@
+Catena
+======
+Catena is a memory-consuming password scrambler that excellently
+thwarts massively parallel attacks on cheap memory-constrained
+hardware, such as recent graphical processing units (GPUs).
+Furthermore, Catena provides resistance against cache-timing attacks, since
+its memory-access pattern is password-independent.
+
+Academic paper:
+<a href="http://www.uni-weimar.de/fileadmin/user/fak/medien/professuren/Mediensicherheit/Research/Publications/catena-v3.1.pdf">catena-v3.1.pdf</a>
+
+Rspamd specific
+---------------
+
+Rspamd implements Catena-Butterfly using full blake2b hash implemented in the
+cryptobox.
+
+Original code: https://github.com/medsec/catena \ No newline at end of file
diff --git a/src/libcryptobox/catena/catena.c b/src/libcryptobox/catena/catena.c
new file mode 100644
index 0000000..7e066dd
--- /dev/null
+++ b/src/libcryptobox/catena/catena.c
@@ -0,0 +1,444 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ * Copyright (c) 2014 cforler
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+ * the Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "config.h"
+#include "catena.h"
+
+#include <sodium.h>
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+#define TO_LITTLE_ENDIAN_64(n) (n)
+#define TO_LITTLE_ENDIAN_32(n) (n)
+#else
+#define TO_LITTLE_ENDIAN_64 GUINT64_SWAP_LE_BE
+#define TO_LITTLE_ENDIAN_32 GUINT32_SWAP_LE_BE
+#endif
+
+/* Recommended default values */
+#define H_LEN CATENA_HLEN
+#define KEY_LEN 16
+
+const uint8_t VERSION_ID[] = "Butterfly-Full";
+const uint8_t LAMBDA = 4;
+const uint8_t GARLIC = 16;
+const uint8_t MIN_GARLIC = 16;
+
+/*
+ * Hash part
+ */
+
+static inline void
+__Hash1(const uint8_t *input, const uint32_t inputlen,
+ uint8_t hash[H_LEN])
+{
+ crypto_generichash_blake2b_state ctx;
+ crypto_generichash_blake2b_init(&ctx, NULL, 0, H_LEN);
+ crypto_generichash_blake2b_update(&ctx, input, inputlen);
+ crypto_generichash_blake2b_final(&ctx, hash, H_LEN);
+}
+
+/***************************************************/
+
+static inline void __Hash2(const uint8_t *i1, const uint8_t i1len, const uint8_t *i2,
+ const uint8_t i2len, uint8_t hash[H_LEN])
+{
+ crypto_generichash_blake2b_state ctx;
+
+ crypto_generichash_blake2b_init(&ctx, NULL, 0, H_LEN);
+ crypto_generichash_blake2b_update(&ctx, i1, i1len);
+ crypto_generichash_blake2b_update(&ctx, i2, i2len);
+ crypto_generichash_blake2b_final(&ctx, hash, H_LEN);
+}
+
+/***************************************************/
+
+static inline void __Hash3(const uint8_t *i1, const uint8_t i1len, const uint8_t *i2,
+ const uint8_t i2len, const uint8_t *i3, const uint8_t i3len,
+ uint8_t hash[H_LEN])
+{
+ crypto_generichash_blake2b_state ctx;
+
+ crypto_generichash_blake2b_init(&ctx, NULL, 0, H_LEN);
+ crypto_generichash_blake2b_update(&ctx, i1, i1len);
+ crypto_generichash_blake2b_update(&ctx, i2, i2len);
+ crypto_generichash_blake2b_update(&ctx, i3, i3len);
+ crypto_generichash_blake2b_final(&ctx, hash, H_LEN);
+}
+
+/***************************************************/
+
+static inline void __Hash4(const uint8_t *i1, const uint8_t i1len, const uint8_t *i2,
+ const uint8_t i2len, const uint8_t *i3, const uint8_t i3len,
+ const uint8_t *i4, const uint8_t i4len, uint8_t hash[H_LEN])
+{
+ crypto_generichash_blake2b_state ctx;
+
+ crypto_generichash_blake2b_init(&ctx, NULL, 0, H_LEN);
+ crypto_generichash_blake2b_update(&ctx, i1, i1len);
+ crypto_generichash_blake2b_update(&ctx, i2, i2len);
+ crypto_generichash_blake2b_update(&ctx, i3, i3len);
+ crypto_generichash_blake2b_update(&ctx, i4, i4len);
+ crypto_generichash_blake2b_final(&ctx, hash, H_LEN);
+}
+
+/***************************************************/
+
+static inline void __Hash5(const uint8_t *i1, const uint8_t i1len, const uint8_t *i2,
+ const uint8_t i2len, const uint8_t *i3, const uint8_t i3len,
+ const uint8_t *i4, const uint8_t i4len, const uint8_t *i5,
+ const uint8_t i5len, uint8_t hash[H_LEN])
+{
+ crypto_generichash_blake2b_state ctx;
+
+ crypto_generichash_blake2b_init(&ctx, NULL, 0, H_LEN);
+ crypto_generichash_blake2b_update(&ctx, i1, i1len);
+ crypto_generichash_blake2b_update(&ctx, i2, i2len);
+ crypto_generichash_blake2b_update(&ctx, i3, i3len);
+ crypto_generichash_blake2b_update(&ctx, i4, i4len);
+ crypto_generichash_blake2b_update(&ctx, i5, i5len);
+ crypto_generichash_blake2b_final(&ctx, hash, H_LEN);
+}
+
+static inline void
+__HashFast(int vindex, const uint8_t *i1, const uint8_t *i2,
+ uint8_t hash[H_LEN])
+{
+ __Hash2(i1, H_LEN, i2, H_LEN, hash);
+}
+
+static void __ResetState(void)
+{
+}
+
+/*
+ * Misc utils
+ */
+const uint8_t ZERO8[H_LEN] = {0};
+
+/* see: http://en.wikipedia.org/wiki/Xorshift#Variations */
+static int p;
+static uint64_t s[16];
+
+static void
+initXSState(const uint8_t *a, const uint8_t *b)
+{
+ p = 0;
+
+ for (int i = 0; i < 8; i++) {
+ s[i] = UINT64_C(0);
+ s[i + 8] = UINT64_C(0);
+
+ for (int j = 0; j < 8; j++) {
+ s[i] |= ((uint64_t) a[i * 8 + j]) << j * 8;
+ s[i + 8] |= ((uint64_t) b[i * 8 + j]) << j * 8;
+ }
+ }
+}
+
+static uint64_t
+xorshift1024star(void)
+{
+ uint64_t s0 = s[p];
+ uint64_t s1 = s[p = (p + 1) & 15];
+ s1 ^= s1 << 31;
+ s1 ^= s1 >> 11;
+ s0 ^= s0 >> 30;
+ return (s[p] = s0 ^ s1) * UINT64_C(1181783497276652981);
+}
+
+static void
+H_INIT(const uint8_t *x, const uint16_t xlen, uint8_t *vm1, uint8_t *vm2)
+{
+ const uint8_t l = 2;
+ uint8_t *tmp = (uint8_t *) g_malloc(l * H_LEN);
+
+ for (uint8_t i = 0; i != l; ++i) {
+ __Hash2(&i, 1, x, xlen, tmp + i * H_LEN);
+ }
+
+ memcpy(vm1, tmp, H_LEN);
+ memcpy(vm2, tmp + (l / 2 * H_LEN), H_LEN);
+ g_free(tmp);
+}
+
+static void
+H_First(const uint8_t *i1, const uint8_t *i2, uint8_t *hash)
+{
+ uint8_t i = 0;
+ uint8_t *x = (uint8_t *) g_malloc(H_LEN);
+
+ __ResetState();
+ __Hash2(i1, H_LEN, i2, H_LEN, x);
+ __Hash2(&i, 1, x, H_LEN, hash);
+ g_free(x);
+}
+
+static inline void
+initmem(const uint8_t x[H_LEN], const uint64_t c, uint8_t *r)
+{
+ uint8_t *vm2 = (uint8_t *) g_malloc(H_LEN);
+ uint8_t *vm1 = (uint8_t *) g_malloc(H_LEN);
+
+ H_INIT(x, H_LEN, vm1, vm2);
+ __ResetState();
+ __HashFast(0, vm1, vm2, r);
+ __HashFast(1, r, vm1, r + H_LEN);
+
+ /* Top row */
+ for (uint64_t i = 2; i < c; i++) {
+ __HashFast(i, r + (i - 1) * H_LEN, r + (i - 2) * H_LEN, r + i * H_LEN);
+ }
+
+ g_free(vm2);
+ g_free(vm1);
+}
+
+static inline void
+catena_gamma(const uint8_t garlic, const uint8_t *salt,
+ const uint8_t saltlen, uint8_t *r)
+{
+ const uint64_t q = UINT64_C(1) << ((3 * garlic + 3) / 4);
+
+ uint64_t i, j, j2;
+ uint8_t *tmp = g_malloc(H_LEN);
+ uint8_t *tmp2 = g_malloc(H_LEN);
+
+ __Hash1(salt, saltlen, tmp);
+ __Hash1(tmp, H_LEN, tmp2);
+ initXSState(tmp, tmp2);
+
+ __ResetState();
+ for (i = 0; i < q; i++) {
+ j = xorshift1024star() >> (64 - garlic);
+ j2 = xorshift1024star() >> (64 - garlic);
+ __HashFast(i, r + j * H_LEN, r + j2 * H_LEN, r + j * H_LEN);
+ }
+
+ g_free(tmp);
+ g_free(tmp2);
+}
+
+static void
+XOR(const uint8_t *input1, const uint8_t *input2, uint8_t *output)
+{
+ uint32_t i;
+
+ for (i = 0; i < H_LEN; i++) {
+ output[i] = input1[i] ^ input2[i];
+ }
+}
+
+/*
+ * Butterfly part
+ */
+/*
+ * Sigma function that defines the diagonal connections of a DBG
+ * diagonal front: flip the (g-i)th bit (Inverse Butterfly Graph)
+ * diagonal back: flip the i-(g-1)th bit (Regular Butterfly Graph)
+ */
+static uint64_t
+sigma(const uint8_t g, const uint64_t i, const uint64_t j)
+{
+ if (i < g) {
+ return (j ^ (UINT64_C(1) << (g - 1 - i))); /* diagonal front */
+ }
+ else {
+ return (j ^ (UINT64_C(1) << (i - (g - 1)))); /* diagonal back */
+ }
+}
+
+/*calculate actual index from level and element index*/
+static uint64_t
+idx(uint64_t i, uint64_t j, uint8_t co, uint64_t c, uint64_t m)
+{
+ i += co;
+ if (i % 3 == 0) {
+ return j;
+ }
+ else if (i % 3 == 1) {
+ if (j < m) {
+ /* still fits in the array */
+ return j + c;
+ }
+ else {
+ /* start overwriting elements at the beginning */
+ return j - m;
+ }
+ }
+ /* i % 3 == 2 */
+ return j + m;
+}
+
+/*
+ * Computes the hash of x using a Double Butterfly Graph,
+ * that forms as (2^g,\lamba)-Superconcentrator
+ */
+static void
+Flap(const uint8_t x[H_LEN], const uint8_t lambda, const uint8_t garlic,
+ const uint8_t *salt, const uint8_t saltlen, uint8_t h[H_LEN])
+{
+ const uint64_t c = UINT64_C(1) << garlic;
+ const uint64_t m = UINT64_C(1) << (garlic - 1); /* 0.5 * 2^g */
+ const uint32_t l = 2 * garlic;
+
+ uint8_t *r = g_malloc((c + m) * H_LEN);
+ uint8_t *tmp = g_malloc(H_LEN);
+ uint64_t i, j;
+ uint8_t k;
+ uint8_t co = 0; /* carry over from last iteration */
+
+ /* Top row */
+ initmem(x, c, r);
+
+ /*Gamma Function*/
+ catena_gamma(garlic, salt, saltlen, r);
+
+ /* DBH */
+ for (k = 0; k < lambda; k++) {
+ for (i = 1; i < l; i++) {
+ XOR(r + idx(i - 1, c - 1, co, c, m) * H_LEN,
+ r + idx(i - 1, 0, co, c, m) * H_LEN, tmp);
+
+ /*
+ * r0 := H(tmp || vsigma(g,i-1,0) )
+ * __Hash2(tmp, H_LEN, r+idx(i-1,sigma(garlic,i-1,0),co,c,m) * H_LEN, H_LEN,
+ * r+idx(i,0,co,c,m) *H_LEN);
+ */
+ H_First(tmp,
+ r + idx(i - 1, sigma(garlic, i - 1, 0), co, c, m) * H_LEN,
+ r + idx(i, 0, co, c, m) * H_LEN);
+ __ResetState();
+
+ /* vertices */
+ for (j = 1; j < c; j++) {
+ /* tmp:= rj-1 XOR vj */
+ XOR(r + idx(i, j - 1, co, c, m) * H_LEN,
+ r + idx(i - 1, j, co, c, m) * H_LEN, tmp);
+ /* rj := H(tmp || vsigma(g,i-1,j)) */
+ __HashFast(j, tmp,
+ r + idx(i - 1, sigma(garlic, i - 1, j), co, c, m) * H_LEN,
+ r + idx(i, j, co, c, m) * H_LEN);
+ }
+ }
+ co = (co + (i - 1)) % 3;
+ }
+
+ memcpy(h, r + idx(0, c - 1, co, c, m) * H_LEN, H_LEN);
+ g_free(r);
+ g_free(tmp);
+}
+
+static int
+__Catena(const uint8_t *pwd, const uint32_t pwdlen,
+ const uint8_t *salt, const uint8_t saltlen, const uint8_t *data,
+ const uint32_t datalen, const uint8_t lambda, const uint8_t min_garlic,
+ const uint8_t garlic, const uint8_t hashlen, const uint8_t client,
+ const uint8_t tweak_id, uint8_t *hash)
+{
+ uint8_t x[H_LEN];
+ uint8_t hv[H_LEN];
+ uint8_t t[4];
+ uint8_t c;
+
+ if ((hashlen > H_LEN) || (garlic > 63) || (min_garlic > garlic) || (lambda == 0) || (min_garlic == 0)) {
+ return -1;
+ }
+
+ /*Compute H(V)*/
+ __Hash1(VERSION_ID, strlen((char *) VERSION_ID), hv);
+
+ /* Compute Tweak */
+ t[0] = tweak_id;
+ t[1] = lambda;
+ t[2] = hashlen;
+ t[3] = saltlen;
+
+ /* Compute H(AD) */
+ __Hash1((uint8_t *) data, datalen, x);
+
+ /* Compute the initial value to hash */
+ __Hash5(hv, H_LEN, t, 4, x, H_LEN, pwd, pwdlen, salt, saltlen, x);
+
+ /*Overwrite Password if enabled*/
+#ifdef OVERWRITE
+ erasepwd(pwd, pwdlen);
+#endif
+
+ Flap(x, lambda, (min_garlic + 1) / 2, salt, saltlen, x);
+
+ for (c = min_garlic; c <= garlic; c++) {
+ Flap(x, lambda, c, salt, saltlen, x);
+ if ((c == garlic) && (client == CLIENT)) {
+ memcpy(hash, x, H_LEN);
+ return 0;
+ }
+ __Hash2(&c, 1, x, H_LEN, x);
+ memset(x + hashlen, 0, H_LEN - hashlen);
+ }
+
+ memcpy(hash, x, hashlen);
+
+ return 0;
+}
+
+/***************************************************/
+
+int catena(const uint8_t *pwd, const uint32_t pwdlen, const uint8_t *salt,
+ const uint8_t saltlen, const uint8_t *data, const uint32_t datalen,
+ const uint8_t lambda, const uint8_t min_garlic, const uint8_t garlic,
+ const uint8_t hashlen, uint8_t *hash)
+{
+ return __Catena(pwd, pwdlen, salt, saltlen, data, datalen, lambda,
+ min_garlic, garlic, hashlen, REGULAR, PASSWORD_HASHING_MODE, hash);
+}
+
+int simple_catena(const uint8_t *pwd, const uint32_t pwdlen,
+ const uint8_t *salt, const uint8_t saltlen,
+ const uint8_t *data, const uint32_t datalen,
+ uint8_t hash[H_LEN])
+{
+ return __Catena(pwd, pwdlen, salt, saltlen, data, datalen,
+ LAMBDA, MIN_GARLIC, GARLIC, H_LEN,
+ REGULAR, PASSWORD_HASHING_MODE, hash);
+}
+
+int catena_test(void)
+{
+ /* From catena-v3.1 spec */
+ guint8 pw[] = {0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64};
+ guint8 salt[] = {0x73, 0x61, 0x6c, 0x74};
+ guint8 ad[] = {0x64, 0x61, 0x74, 0x61};
+ guint8 expected[] = {
+ 0x20, 0xc5, 0x91, 0x93, 0x8f, 0xc3, 0xaf, 0xcc, 0x3b, 0xba, 0x91, 0xd2, 0xfb,
+ 0x84, 0xbf, 0x7b, 0x44, 0x04, 0xf9, 0x4c, 0x45, 0xed, 0x4d, 0x11, 0xa7, 0xe2,
+ 0xb4, 0x12, 0x3e, 0xab, 0x0b, 0x77, 0x4a, 0x12, 0xb4, 0x22, 0xd0, 0xda, 0xb5,
+ 0x25, 0x29, 0x02, 0xfc, 0x54, 0x47, 0xea, 0x82, 0x63, 0x8c, 0x1a, 0xfb, 0xa7,
+ 0xa9, 0x94, 0x24, 0x13, 0x0e, 0x44, 0x36, 0x3b, 0x9d, 0x9f, 0xc9, 0x60};
+ guint8 real[H_LEN];
+
+ if (catena(pw, sizeof(pw), salt, sizeof(salt), ad, sizeof(ad),
+ 4, 10, 10, H_LEN, real) != 0) {
+ return -1;
+ }
+
+ return memcmp(real, expected, H_LEN);
+}
diff --git a/src/libcryptobox/catena/catena.h b/src/libcryptobox/catena/catena.h
new file mode 100644
index 0000000..1fcea21
--- /dev/null
+++ b/src/libcryptobox/catena/catena.h
@@ -0,0 +1,62 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBCRYPTOBOX_CATENA_CATENA_H_
+#define SRC_LIBCRYPTOBOX_CATENA_CATENA_H_
+
+/* Modes */
+#define PASSWORD_HASHING_MODE 0
+#define KEY_DERIVATION_MODE 1
+#define REGULAR 0
+#define CLIENT 1
+
+#define CATENA_HLEN 64
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int catena(const uint8_t *pwd, const uint32_t pwdlen,
+ const uint8_t *salt, const uint8_t saltlen,
+ const uint8_t *data, const uint32_t datalen,
+ const uint8_t lambda, const uint8_t min_garlic,
+ const uint8_t garlic, const uint8_t hashlen, uint8_t *hash);
+
+/**
+ * Simple interface for catena PBKDF
+ * @param pwd password
+ * @param pwdlen length of password
+ * @param salt salt
+ * @param saltlen length of salt
+ * @param data additional data
+ * @param datalen length of additional data
+ * @param hash output hash
+ * @return 0 if hash is generated, -1 in case of error
+ */
+int simple_catena(const uint8_t *pwd, const uint32_t pwdlen,
+ const uint8_t *salt, const uint8_t saltlen,
+ const uint8_t *data, const uint32_t datalen,
+ uint8_t hash[CATENA_HLEN]);
+
+/**
+ * Run a quick test on catena implementation
+ */
+int catena_test(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBCRYPTOBOX_CATENA_CATENA_H_ */
diff --git a/src/libcryptobox/chacha20/avx.S b/src/libcryptobox/chacha20/avx.S
new file mode 100644
index 0000000..7689b84
--- /dev/null
+++ b/src/libcryptobox/chacha20/avx.S
@@ -0,0 +1,614 @@
+#include "../macro.S"
+#include "constants.S"
+SECTION_TEXT
+
+GLOBAL_HIDDEN_FN chacha_blocks_avx
+chacha_blocks_avx_local:
+pushq %rbx
+pushq %rbp
+movq %rsp, %rbp
+andq $~63, %rsp
+subq $512, %rsp
+LOAD_VAR_PIC chacha_constants, %rax
+vmovdqa 0(%rax), %xmm8
+vmovdqa 16(%rax), %xmm6
+vmovdqa 32(%rax), %xmm7
+vmovdqu 0(%rdi), %xmm9
+vmovdqu 16(%rdi), %xmm10
+vmovdqu 32(%rdi), %xmm11
+movq 48(%rdi), %rax
+movq $1, %r9
+vmovdqa %xmm8, 0(%rsp)
+vmovdqa %xmm9, 16(%rsp)
+vmovdqa %xmm10, 32(%rsp)
+vmovdqa %xmm11, 48(%rsp)
+vmovdqa %xmm6, 80(%rsp)
+vmovdqa %xmm7, 96(%rsp)
+movq %rax, 64(%rsp)
+cmpq $256, %rcx
+jb chacha_blocks_avx_below256
+vpshufd $0x00, %xmm8, %xmm0
+vpshufd $0x55, %xmm8, %xmm1
+vpshufd $0xaa, %xmm8, %xmm2
+vpshufd $0xff, %xmm8, %xmm3
+vmovdqa %xmm0, 128(%rsp)
+vmovdqa %xmm1, 144(%rsp)
+vmovdqa %xmm2, 160(%rsp)
+vmovdqa %xmm3, 176(%rsp)
+vpshufd $0x00, %xmm9, %xmm0
+vpshufd $0x55, %xmm9, %xmm1
+vpshufd $0xaa, %xmm9, %xmm2
+vpshufd $0xff, %xmm9, %xmm3
+vmovdqa %xmm0, 192(%rsp)
+vmovdqa %xmm1, 208(%rsp)
+vmovdqa %xmm2, 224(%rsp)
+vmovdqa %xmm3, 240(%rsp)
+vpshufd $0x00, %xmm10, %xmm0
+vpshufd $0x55, %xmm10, %xmm1
+vpshufd $0xaa, %xmm10, %xmm2
+vpshufd $0xff, %xmm10, %xmm3
+vmovdqa %xmm0, 256(%rsp)
+vmovdqa %xmm1, 272(%rsp)
+vmovdqa %xmm2, 288(%rsp)
+vmovdqa %xmm3, 304(%rsp)
+vpshufd $0xaa, %xmm11, %xmm0
+vpshufd $0xff, %xmm11, %xmm1
+vmovdqa %xmm0, 352(%rsp)
+vmovdqa %xmm1, 368(%rsp)
+jmp chacha_blocks_avx_atleast256
+.p2align 6,,63
+nop
+nop
+nop
+nop
+nop
+chacha_blocks_avx_atleast256:
+movq 48(%rsp), %rax
+leaq 1(%rax), %r8
+leaq 2(%rax), %r9
+leaq 3(%rax), %r10
+leaq 4(%rax), %rbx
+movl %eax, 320(%rsp)
+movl %r8d, 4+320(%rsp)
+movl %r9d, 8+320(%rsp)
+movl %r10d, 12+320(%rsp)
+shrq $32, %rax
+shrq $32, %r8
+shrq $32, %r9
+shrq $32, %r10
+movl %eax, 336(%rsp)
+movl %r8d, 4+336(%rsp)
+movl %r9d, 8+336(%rsp)
+movl %r10d, 12+336(%rsp)
+movq %rbx, 48(%rsp)
+movq 64(%rsp), %rax
+vmovdqa 128(%rsp), %xmm0
+vmovdqa 144(%rsp), %xmm1
+vmovdqa 160(%rsp), %xmm2
+vmovdqa 176(%rsp), %xmm3
+vmovdqa 192(%rsp), %xmm4
+vmovdqa 208(%rsp), %xmm5
+vmovdqa 224(%rsp), %xmm6
+vmovdqa 240(%rsp), %xmm7
+vmovdqa 256(%rsp), %xmm8
+vmovdqa 272(%rsp), %xmm9
+vmovdqa 288(%rsp), %xmm10
+vmovdqa 304(%rsp), %xmm11
+vmovdqa 320(%rsp), %xmm12
+vmovdqa 336(%rsp), %xmm13
+vmovdqa 352(%rsp), %xmm14
+vmovdqa 368(%rsp), %xmm15
+chacha_blocks_avx_mainloop1:
+vpaddd %xmm0, %xmm4, %xmm0
+vpaddd %xmm1, %xmm5, %xmm1
+vpxor %xmm12, %xmm0, %xmm12
+vpxor %xmm13, %xmm1, %xmm13
+vpaddd %xmm2, %xmm6, %xmm2
+vpaddd %xmm3, %xmm7, %xmm3
+vpxor %xmm14, %xmm2, %xmm14
+vpxor %xmm15, %xmm3, %xmm15
+vpshufb 80(%rsp), %xmm12, %xmm12
+vpshufb 80(%rsp), %xmm13, %xmm13
+vpaddd %xmm8, %xmm12, %xmm8
+vpaddd %xmm9, %xmm13, %xmm9
+vpshufb 80(%rsp), %xmm14, %xmm14
+vpshufb 80(%rsp), %xmm15, %xmm15
+vpaddd %xmm10, %xmm14, %xmm10
+vpaddd %xmm11, %xmm15, %xmm11
+vmovdqa %xmm12, 112(%rsp)
+vpxor %xmm4, %xmm8, %xmm4
+vpxor %xmm5, %xmm9, %xmm5
+vpslld $ 12, %xmm4, %xmm12
+vpsrld $20, %xmm4, %xmm4
+vpxor %xmm4, %xmm12, %xmm4
+vpslld $ 12, %xmm5, %xmm12
+vpsrld $20, %xmm5, %xmm5
+vpxor %xmm5, %xmm12, %xmm5
+vpxor %xmm6, %xmm10, %xmm6
+vpxor %xmm7, %xmm11, %xmm7
+vpslld $ 12, %xmm6, %xmm12
+vpsrld $20, %xmm6, %xmm6
+vpxor %xmm6, %xmm12, %xmm6
+vpslld $ 12, %xmm7, %xmm12
+vpsrld $20, %xmm7, %xmm7
+vpxor %xmm7, %xmm12, %xmm7
+vpaddd %xmm0, %xmm4, %xmm0
+vpaddd %xmm1, %xmm5, %xmm1
+vpxor 112(%rsp), %xmm0, %xmm12
+vpxor %xmm13, %xmm1, %xmm13
+vpaddd %xmm2, %xmm6, %xmm2
+vpaddd %xmm3, %xmm7, %xmm3
+vpxor %xmm14, %xmm2, %xmm14
+vpxor %xmm15, %xmm3, %xmm15
+vpshufb 96(%rsp), %xmm12, %xmm12
+vpshufb 96(%rsp), %xmm13, %xmm13
+vpaddd %xmm8, %xmm12, %xmm8
+vpaddd %xmm9, %xmm13, %xmm9
+vpshufb 96(%rsp), %xmm14, %xmm14
+vpshufb 96(%rsp), %xmm15, %xmm15
+vpaddd %xmm10, %xmm14, %xmm10
+vpaddd %xmm11, %xmm15, %xmm11
+vmovdqa %xmm12, 112(%rsp)
+vpxor %xmm4, %xmm8, %xmm4
+vpxor %xmm5, %xmm9, %xmm5
+vpslld $ 7, %xmm4, %xmm12
+vpsrld $25, %xmm4, %xmm4
+vpxor %xmm4, %xmm12, %xmm4
+vpslld $ 7, %xmm5, %xmm12
+vpsrld $25, %xmm5, %xmm5
+vpxor %xmm5, %xmm12, %xmm5
+vpxor %xmm6, %xmm10, %xmm6
+vpxor %xmm7, %xmm11, %xmm7
+vpslld $ 7, %xmm6, %xmm12
+vpsrld $25, %xmm6, %xmm6
+vpxor %xmm6, %xmm12, %xmm6
+vpslld $ 7, %xmm7, %xmm12
+vpsrld $25, %xmm7, %xmm7
+vpxor %xmm7, %xmm12, %xmm7
+vpaddd %xmm0, %xmm5, %xmm0
+vpaddd %xmm1, %xmm6, %xmm1
+vpxor %xmm15, %xmm0, %xmm15
+vpxor 112(%rsp), %xmm1, %xmm12
+vpaddd %xmm2, %xmm7, %xmm2
+vpaddd %xmm3, %xmm4, %xmm3
+vpxor %xmm13, %xmm2, %xmm13
+vpxor %xmm14, %xmm3, %xmm14
+vpshufb 80(%rsp), %xmm15, %xmm15
+vpshufb 80(%rsp), %xmm12, %xmm12
+vpaddd %xmm10, %xmm15, %xmm10
+vpaddd %xmm11, %xmm12, %xmm11
+vpshufb 80(%rsp), %xmm13, %xmm13
+vpshufb 80(%rsp), %xmm14, %xmm14
+vpaddd %xmm8, %xmm13, %xmm8
+vpaddd %xmm9, %xmm14, %xmm9
+vmovdqa %xmm15, 112(%rsp)
+vpxor %xmm5, %xmm10, %xmm5
+vpxor %xmm6, %xmm11, %xmm6
+vpslld $ 12, %xmm5, %xmm15
+vpsrld $20, %xmm5, %xmm5
+vpxor %xmm5, %xmm15, %xmm5
+vpslld $ 12, %xmm6, %xmm15
+vpsrld $20, %xmm6, %xmm6
+vpxor %xmm6, %xmm15, %xmm6
+vpxor %xmm7, %xmm8, %xmm7
+vpxor %xmm4, %xmm9, %xmm4
+vpslld $ 12, %xmm7, %xmm15
+vpsrld $20, %xmm7, %xmm7
+vpxor %xmm7, %xmm15, %xmm7
+vpslld $ 12, %xmm4, %xmm15
+vpsrld $20, %xmm4, %xmm4
+vpxor %xmm4, %xmm15, %xmm4
+vpaddd %xmm0, %xmm5, %xmm0
+vpaddd %xmm1, %xmm6, %xmm1
+vpxor 112(%rsp), %xmm0, %xmm15
+vpxor %xmm12, %xmm1, %xmm12
+vpaddd %xmm2, %xmm7, %xmm2
+vpaddd %xmm3, %xmm4, %xmm3
+vpxor %xmm13, %xmm2, %xmm13
+vpxor %xmm14, %xmm3, %xmm14
+vpshufb 96(%rsp), %xmm15, %xmm15
+vpshufb 96(%rsp), %xmm12, %xmm12
+vpaddd %xmm10, %xmm15, %xmm10
+vpaddd %xmm11, %xmm12, %xmm11
+vpshufb 96(%rsp), %xmm13, %xmm13
+vpshufb 96(%rsp), %xmm14, %xmm14
+vpaddd %xmm8, %xmm13, %xmm8
+vpaddd %xmm9, %xmm14, %xmm9
+vmovdqa %xmm15, 112(%rsp)
+vpxor %xmm5, %xmm10, %xmm5
+vpxor %xmm6, %xmm11, %xmm6
+vpslld $ 7, %xmm5, %xmm15
+vpsrld $25, %xmm5, %xmm5
+vpxor %xmm5, %xmm15, %xmm5
+vpslld $ 7, %xmm6, %xmm15
+vpsrld $25, %xmm6, %xmm6
+vpxor %xmm6, %xmm15, %xmm6
+vpxor %xmm7, %xmm8, %xmm7
+vpxor %xmm4, %xmm9, %xmm4
+vpslld $ 7, %xmm7, %xmm15
+vpsrld $25, %xmm7, %xmm7
+vpxor %xmm7, %xmm15, %xmm7
+vpslld $ 7, %xmm4, %xmm15
+vpsrld $25, %xmm4, %xmm4
+vpxor %xmm4, %xmm15, %xmm4
+vmovdqa 112(%rsp), %xmm15
+subq $2, %rax
+jnz chacha_blocks_avx_mainloop1
+vpaddd 128(%rsp), %xmm0, %xmm0
+vpaddd 144(%rsp), %xmm1, %xmm1
+vpaddd 160(%rsp), %xmm2, %xmm2
+vpaddd 176(%rsp), %xmm3, %xmm3
+vpaddd 192(%rsp), %xmm4, %xmm4
+vpaddd 208(%rsp), %xmm5, %xmm5
+vpaddd 224(%rsp), %xmm6, %xmm6
+vpaddd 240(%rsp), %xmm7, %xmm7
+vpaddd 256(%rsp), %xmm8, %xmm8
+vpaddd 272(%rsp), %xmm9, %xmm9
+vpaddd 288(%rsp), %xmm10, %xmm10
+vpaddd 304(%rsp), %xmm11, %xmm11
+vpaddd 320(%rsp), %xmm12, %xmm12
+vpaddd 336(%rsp), %xmm13, %xmm13
+vpaddd 352(%rsp), %xmm14, %xmm14
+vpaddd 368(%rsp), %xmm15, %xmm15
+vmovdqa %xmm8, 384(%rsp)
+vmovdqa %xmm9, 400(%rsp)
+vmovdqa %xmm10, 416(%rsp)
+vmovdqa %xmm11, 432(%rsp)
+vmovdqa %xmm12, 448(%rsp)
+vmovdqa %xmm13, 464(%rsp)
+vmovdqa %xmm14, 480(%rsp)
+vmovdqa %xmm15, 496(%rsp)
+vpunpckldq %xmm1, %xmm0, %xmm8
+vpunpckldq %xmm3, %xmm2, %xmm9
+vpunpckhdq %xmm1, %xmm0, %xmm12
+vpunpckhdq %xmm3, %xmm2, %xmm13
+vpunpckldq %xmm5, %xmm4, %xmm10
+vpunpckldq %xmm7, %xmm6, %xmm11
+vpunpckhdq %xmm5, %xmm4, %xmm14
+vpunpckhdq %xmm7, %xmm6, %xmm15
+vpunpcklqdq %xmm9, %xmm8, %xmm0
+vpunpcklqdq %xmm11, %xmm10, %xmm1
+vpunpckhqdq %xmm9, %xmm8, %xmm2
+vpunpckhqdq %xmm11, %xmm10, %xmm3
+vpunpcklqdq %xmm13, %xmm12, %xmm4
+vpunpcklqdq %xmm15, %xmm14, %xmm5
+vpunpckhqdq %xmm13, %xmm12, %xmm6
+vpunpckhqdq %xmm15, %xmm14, %xmm7
+andq %rsi, %rsi
+jz chacha_blocks_avx_noinput1
+vpxor 0(%rsi), %xmm0, %xmm0
+vpxor 16(%rsi), %xmm1, %xmm1
+vpxor 64(%rsi), %xmm2, %xmm2
+vpxor 80(%rsi), %xmm3, %xmm3
+vpxor 128(%rsi), %xmm4, %xmm4
+vpxor 144(%rsi), %xmm5, %xmm5
+vpxor 192(%rsi), %xmm6, %xmm6
+vpxor 208(%rsi), %xmm7, %xmm7
+vmovdqu %xmm0, 0(%rdx)
+vmovdqu %xmm1, 16(%rdx)
+vmovdqu %xmm2, 64(%rdx)
+vmovdqu %xmm3, 80(%rdx)
+vmovdqu %xmm4, 128(%rdx)
+vmovdqu %xmm5, 144(%rdx)
+vmovdqu %xmm6, 192(%rdx)
+vmovdqu %xmm7, 208(%rdx)
+vmovdqa 384(%rsp), %xmm0
+vmovdqa 400(%rsp), %xmm1
+vmovdqa 416(%rsp), %xmm2
+vmovdqa 432(%rsp), %xmm3
+vmovdqa 448(%rsp), %xmm4
+vmovdqa 464(%rsp), %xmm5
+vmovdqa 480(%rsp), %xmm6
+vmovdqa 496(%rsp), %xmm7
+vpunpckldq %xmm1, %xmm0, %xmm8
+vpunpckldq %xmm3, %xmm2, %xmm9
+vpunpckhdq %xmm1, %xmm0, %xmm12
+vpunpckhdq %xmm3, %xmm2, %xmm13
+vpunpckldq %xmm5, %xmm4, %xmm10
+vpunpckldq %xmm7, %xmm6, %xmm11
+vpunpckhdq %xmm5, %xmm4, %xmm14
+vpunpckhdq %xmm7, %xmm6, %xmm15
+vpunpcklqdq %xmm9, %xmm8, %xmm0
+vpunpcklqdq %xmm11, %xmm10, %xmm1
+vpunpckhqdq %xmm9, %xmm8, %xmm2
+vpunpckhqdq %xmm11, %xmm10, %xmm3
+vpunpcklqdq %xmm13, %xmm12, %xmm4
+vpunpcklqdq %xmm15, %xmm14, %xmm5
+vpunpckhqdq %xmm13, %xmm12, %xmm6
+vpunpckhqdq %xmm15, %xmm14, %xmm7
+vpxor 32(%rsi), %xmm0, %xmm0
+vpxor 48(%rsi), %xmm1, %xmm1
+vpxor 96(%rsi), %xmm2, %xmm2
+vpxor 112(%rsi), %xmm3, %xmm3
+vpxor 160(%rsi), %xmm4, %xmm4
+vpxor 176(%rsi), %xmm5, %xmm5
+vpxor 224(%rsi), %xmm6, %xmm6
+vpxor 240(%rsi), %xmm7, %xmm7
+vmovdqu %xmm0, 32(%rdx)
+vmovdqu %xmm1, 48(%rdx)
+vmovdqu %xmm2, 96(%rdx)
+vmovdqu %xmm3, 112(%rdx)
+vmovdqu %xmm4, 160(%rdx)
+vmovdqu %xmm5, 176(%rdx)
+vmovdqu %xmm6, 224(%rdx)
+vmovdqu %xmm7, 240(%rdx)
+addq $256, %rsi
+jmp chacha_blocks_avx_mainloop_cont
+chacha_blocks_avx_noinput1:
+vmovdqu %xmm0, 0(%rdx)
+vmovdqu %xmm1, 16(%rdx)
+vmovdqu %xmm2, 64(%rdx)
+vmovdqu %xmm3, 80(%rdx)
+vmovdqu %xmm4, 128(%rdx)
+vmovdqu %xmm5, 144(%rdx)
+vmovdqu %xmm6, 192(%rdx)
+vmovdqu %xmm7, 208(%rdx)
+vmovdqa 384(%rsp), %xmm0
+vmovdqa 400(%rsp), %xmm1
+vmovdqa 416(%rsp), %xmm2
+vmovdqa 432(%rsp), %xmm3
+vmovdqa 448(%rsp), %xmm4
+vmovdqa 464(%rsp), %xmm5
+vmovdqa 480(%rsp), %xmm6
+vmovdqa 496(%rsp), %xmm7
+vpunpckldq %xmm1, %xmm0, %xmm8
+vpunpckldq %xmm3, %xmm2, %xmm9
+vpunpckhdq %xmm1, %xmm0, %xmm12
+vpunpckhdq %xmm3, %xmm2, %xmm13
+vpunpckldq %xmm5, %xmm4, %xmm10
+vpunpckldq %xmm7, %xmm6, %xmm11
+vpunpckhdq %xmm5, %xmm4, %xmm14
+vpunpckhdq %xmm7, %xmm6, %xmm15
+vpunpcklqdq %xmm9, %xmm8, %xmm0
+vpunpcklqdq %xmm11, %xmm10, %xmm1
+vpunpckhqdq %xmm9, %xmm8, %xmm2
+vpunpckhqdq %xmm11, %xmm10, %xmm3
+vpunpcklqdq %xmm13, %xmm12, %xmm4
+vpunpcklqdq %xmm15, %xmm14, %xmm5
+vpunpckhqdq %xmm13, %xmm12, %xmm6
+vpunpckhqdq %xmm15, %xmm14, %xmm7
+vmovdqu %xmm0, 32(%rdx)
+vmovdqu %xmm1, 48(%rdx)
+vmovdqu %xmm2, 96(%rdx)
+vmovdqu %xmm3, 112(%rdx)
+vmovdqu %xmm4, 160(%rdx)
+vmovdqu %xmm5, 176(%rdx)
+vmovdqu %xmm6, 224(%rdx)
+vmovdqu %xmm7, 240(%rdx)
+chacha_blocks_avx_mainloop_cont:
+addq $256, %rdx
+subq $256, %rcx
+cmp $256, %rcx
+jae chacha_blocks_avx_atleast256
+vmovdqa 80(%rsp), %xmm6
+vmovdqa 96(%rsp), %xmm7
+vmovdqa 0(%rsp), %xmm8
+vmovdqa 16(%rsp), %xmm9
+vmovdqa 32(%rsp), %xmm10
+vmovdqa 48(%rsp), %xmm11
+movq $1, %r9
+chacha_blocks_avx_below256:
+vmovq %r9, %xmm5
+andq %rcx, %rcx
+jz chacha_blocks_avx_done
+cmpq $64, %rcx
+jae chacha_blocks_avx_above63
+movq %rdx, %r9
+andq %rsi, %rsi
+jz chacha_blocks_avx_noinput2
+movq %rcx, %r10
+movq %rsp, %rdx
+addq %r10, %rsi
+addq %r10, %rdx
+negq %r10
+chacha_blocks_avx_copyinput:
+movb (%rsi, %r10), %al
+movb %al, (%rdx, %r10)
+incq %r10
+jnz chacha_blocks_avx_copyinput
+movq %rsp, %rsi
+chacha_blocks_avx_noinput2:
+movq %rsp, %rdx
+chacha_blocks_avx_above63:
+vmovdqa %xmm8, %xmm0
+vmovdqa %xmm9, %xmm1
+vmovdqa %xmm10, %xmm2
+vmovdqa %xmm11, %xmm3
+movq 64(%rsp), %rax
+chacha_blocks_avx_mainloop2:
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm6, %xmm3, %xmm3
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpslld $12, %xmm1, %xmm4
+vpsrld $20, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm7, %xmm3, %xmm3
+vpshufd $0x93, %xmm0, %xmm0
+vpaddd %xmm2, %xmm3, %xmm2
+vpshufd $0x4e, %xmm3, %xmm3
+vpxor %xmm1, %xmm2, %xmm1
+vpshufd $0x39, %xmm2, %xmm2
+vpslld $7, %xmm1, %xmm4
+vpsrld $25, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm6, %xmm3, %xmm3
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpslld $12, %xmm1, %xmm4
+vpsrld $20, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm7, %xmm3, %xmm3
+vpshufd $0x39, %xmm0, %xmm0
+vpaddd %xmm2, %xmm3, %xmm2
+vpshufd $0x4e, %xmm3, %xmm3
+vpxor %xmm1, %xmm2, %xmm1
+vpshufd $0x93, %xmm2, %xmm2
+vpslld $7, %xmm1, %xmm4
+vpsrld $25, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+subq $2, %rax
+jnz chacha_blocks_avx_mainloop2
+vpaddd %xmm0, %xmm8, %xmm0
+vpaddd %xmm1, %xmm9, %xmm1
+vpaddd %xmm2, %xmm10, %xmm2
+vpaddd %xmm3, %xmm11, %xmm3
+andq %rsi, %rsi
+jz chacha_blocks_avx_noinput3
+vpxor 0(%rsi), %xmm0, %xmm0
+vpxor 16(%rsi), %xmm1, %xmm1
+vpxor 32(%rsi), %xmm2, %xmm2
+vpxor 48(%rsi), %xmm3, %xmm3
+addq $64, %rsi
+chacha_blocks_avx_noinput3:
+vmovdqu %xmm0, 0(%rdx)
+vmovdqu %xmm1, 16(%rdx)
+vmovdqu %xmm2, 32(%rdx)
+vmovdqu %xmm3, 48(%rdx)
+vpaddq %xmm11, %xmm5, %xmm11
+cmpq $64, %rcx
+jbe chacha_blocks_avx_mainloop2_finishup
+addq $64, %rdx
+subq $64, %rcx
+jmp chacha_blocks_avx_below256
+chacha_blocks_avx_mainloop2_finishup:
+cmpq $64, %rcx
+je chacha_blocks_avx_done
+addq %rcx, %r9
+addq %rcx, %rdx
+negq %rcx
+chacha_blocks_avx_copyoutput:
+movb (%rdx, %rcx), %al
+movb %al, (%r9, %rcx)
+incq %rcx
+jnz chacha_blocks_avx_copyoutput
+chacha_blocks_avx_done:
+vmovdqu %xmm11, 32(%rdi)
+movq %rbp, %rsp
+popq %rbp
+popq %rbx
+ret
+FN_END chacha_blocks_avx
+
+GLOBAL_HIDDEN_FN hchacha_avx
+hchacha_avx_local:
+LOAD_VAR_PIC chacha_constants, %rax
+vmovdqa 0(%rax), %xmm0
+vmovdqa 16(%rax), %xmm6
+vmovdqa 32(%rax), %xmm5
+vmovdqu 0(%rdi), %xmm1
+vmovdqu 16(%rdi), %xmm2
+vmovdqu 0(%rsi), %xmm3
+hhacha_mainloop_avx:
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm6, %xmm3, %xmm3
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpslld $12, %xmm1, %xmm4
+vpsrld $20, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm5, %xmm3, %xmm3
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpslld $7, %xmm1, %xmm4
+vpsrld $25, %xmm1, %xmm1
+vpshufd $0x93, %xmm0, %xmm0
+vpxor %xmm1, %xmm4, %xmm1
+vpshufd $0x4e, %xmm3, %xmm3
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm6, %xmm3, %xmm3
+vpshufd $0x39, %xmm2, %xmm2
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpslld $12, %xmm1, %xmm4
+vpsrld $20, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm5, %xmm3, %xmm3
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpshufd $0x39, %xmm0, %xmm0
+vpslld $7, %xmm1, %xmm4
+vpshufd $0x4e, %xmm3, %xmm3
+vpsrld $25, %xmm1, %xmm1
+vpshufd $0x93, %xmm2, %xmm2
+vpxor %xmm1, %xmm4, %xmm1
+subl $2, %ecx
+jne hhacha_mainloop_avx
+vmovdqu %xmm0, (%rdx)
+vmovdqu %xmm3, 16(%rdx)
+ret
+FN_END hchacha_avx
+
+GLOBAL_HIDDEN_FN_EXT chacha_avx, 6, 16
+pushq %rbp
+movq %rsp, %rbp
+subq $64, %rsp
+andq $~63, %rsp
+vmovdqu 0(%rdi), %xmm0
+vmovdqu 16(%rdi), %xmm1
+vmovdqa %xmm0, 0(%rsp)
+vmovdqa %xmm1, 16(%rsp)
+xorq %rdi, %rdi
+movq %rdi, 32(%rsp)
+movq 0(%rsi), %rsi
+movq %rsi, 40(%rsp)
+movq %r9, 48(%rsp)
+movq %rsp, %rdi
+movq %rdx, %rsi
+movq %rcx, %rdx
+movq %r8, %rcx
+call chacha_blocks_avx_local
+vpxor %xmm0, %xmm0, %xmm0
+vmovdqa %xmm0, 0(%rsp)
+vmovdqa %xmm0, 16(%rsp)
+vmovdqa %xmm0, 32(%rsp)
+movq %rbp, %rsp
+popq %rbp
+ret
+FN_END chacha_avx
+
+GLOBAL_HIDDEN_FN_EXT xchacha_avx, 6, 16
+pushq %rbp
+pushq %rbx
+movq %rsp, %rbp
+subq $64, %rsp
+andq $~63, %rsp
+movq %rsp, %rbx
+xorq %rax, %rax
+movq %rax, 32(%rbx)
+movq 16(%rsi), %rax
+movq %rax, 40(%rbx)
+movq %r9, 48(%rbx)
+pushq %rdx
+pushq %rcx
+pushq %r8
+movq %rbx, %rdx
+movq %r9, %rcx
+call hchacha_avx_local
+movq %rbx, %rdi
+popq %rcx
+popq %rdx
+popq %rsi
+call chacha_blocks_avx_local
+vpxor %xmm0, %xmm0, %xmm0
+vmovdqa %xmm0, 0(%rbx)
+vmovdqa %xmm0, 16(%rbx)
+vmovdqa %xmm0, 32(%rbx)
+movq %rbp, %rsp
+popq %rbx
+popq %rbp
+ret
+FN_END xchacha_avx
diff --git a/src/libcryptobox/chacha20/avx2.S b/src/libcryptobox/chacha20/avx2.S
new file mode 100644
index 0000000..efd0f54
--- /dev/null
+++ b/src/libcryptobox/chacha20/avx2.S
@@ -0,0 +1,1018 @@
+#include "../macro.S"
+#include "constants.S"
+SECTION_TEXT
+
+GLOBAL_HIDDEN_FN chacha_blocks_avx2
+chacha_blocks_avx2_local:
+pushq %rbx
+pushq %rbp
+pushq %r12
+pushq %r13
+pushq %r14
+movq %rsp, %rbp
+andq $~63, %rsp
+subq $512, %rsp
+LOAD_VAR_PIC chacha_constants, %rax
+vmovdqa 0(%rax), %xmm8
+vmovdqa 16(%rax), %xmm6
+vmovdqa 32(%rax), %xmm7
+vmovdqu 0(%rdi), %xmm9
+vmovdqu 16(%rdi), %xmm10
+vmovdqu 32(%rdi), %xmm11
+movq 48(%rdi), %rax
+movq $1, %r9
+vmovdqa %xmm8, 0(%rsp)
+vmovdqa %xmm9, 16(%rsp)
+vmovdqa %xmm10, 32(%rsp)
+vmovdqa %xmm11, 48(%rsp)
+movq %rax, 64(%rsp)
+vmovdqa %xmm6, 448(%rsp)
+vmovdqa %xmm6, 464(%rsp)
+vmovdqa %xmm7, 480(%rsp)
+vmovdqa %xmm7, 496(%rsp)
+cmpq $512, %rcx
+jae chacha_blocks_avx2_atleast512
+cmp $256, %rcx
+jae chacha_blocks_avx2_atleast256
+jmp chacha_blocks_avx2_below256
+.p2align 6,,63
+chacha_blocks_avx2_atleast512:
+movq 48(%rsp), %rax
+leaq 1(%rax), %r8
+leaq 2(%rax), %r9
+leaq 3(%rax), %r10
+leaq 4(%rax), %rbx
+leaq 5(%rax), %r11
+leaq 6(%rax), %r12
+leaq 7(%rax), %r13
+leaq 8(%rax), %r14
+movl %eax, 128(%rsp)
+movl %r8d, 4+128(%rsp)
+movl %r9d, 8+128(%rsp)
+movl %r10d, 12+128(%rsp)
+movl %ebx, 16+128(%rsp)
+movl %r11d, 20+128(%rsp)
+movl %r12d, 24+128(%rsp)
+movl %r13d, 28+128(%rsp)
+shrq $32, %rax
+shrq $32, %r8
+shrq $32, %r9
+shrq $32, %r10
+shrq $32, %rbx
+shrq $32, %r11
+shrq $32, %r12
+shrq $32, %r13
+movl %eax, 160(%rsp)
+movl %r8d, 4+160(%rsp)
+movl %r9d, 8+160(%rsp)
+movl %r10d, 12+160(%rsp)
+movl %ebx, 16+160(%rsp)
+movl %r11d, 20+160(%rsp)
+movl %r12d, 24+160(%rsp)
+movl %r13d, 28+160(%rsp)
+movq %r14, 48(%rsp)
+movq 64(%rsp), %rax
+vpbroadcastd 0(%rsp), %ymm0
+vpbroadcastd 4+0(%rsp), %ymm1
+vpbroadcastd 8+0(%rsp), %ymm2
+vpbroadcastd 12+0(%rsp), %ymm3
+vpbroadcastd 16(%rsp), %ymm4
+vpbroadcastd 4+16(%rsp), %ymm5
+vpbroadcastd 8+16(%rsp), %ymm6
+vpbroadcastd 12+16(%rsp), %ymm7
+vpbroadcastd 32(%rsp), %ymm8
+vpbroadcastd 4+32(%rsp), %ymm9
+vpbroadcastd 8+32(%rsp), %ymm10
+vpbroadcastd 12+32(%rsp), %ymm11
+vpbroadcastd 8+48(%rsp), %ymm14
+vpbroadcastd 12+48(%rsp), %ymm15
+vmovdqa 128(%rsp), %ymm12
+vmovdqa 160(%rsp), %ymm13
+chacha_blocks_avx2_mainloop1:
+vpaddd %ymm0, %ymm4, %ymm0
+vpaddd %ymm1, %ymm5, %ymm1
+vpxor %ymm12, %ymm0, %ymm12
+vpxor %ymm13, %ymm1, %ymm13
+vpaddd %ymm2, %ymm6, %ymm2
+vpaddd %ymm3, %ymm7, %ymm3
+vpxor %ymm14, %ymm2, %ymm14
+vpxor %ymm15, %ymm3, %ymm15
+vpshufb 448(%rsp), %ymm12, %ymm12
+vpshufb 448(%rsp), %ymm13, %ymm13
+vpaddd %ymm8, %ymm12, %ymm8
+vpaddd %ymm9, %ymm13, %ymm9
+vpshufb 448(%rsp), %ymm14, %ymm14
+vpshufb 448(%rsp), %ymm15, %ymm15
+vpaddd %ymm10, %ymm14, %ymm10
+vpaddd %ymm11, %ymm15, %ymm11
+vmovdqa %ymm12, 96(%rsp)
+vpxor %ymm4, %ymm8, %ymm4
+vpxor %ymm5, %ymm9, %ymm5
+vpslld $ 12, %ymm4, %ymm12
+vpsrld $20, %ymm4, %ymm4
+vpxor %ymm4, %ymm12, %ymm4
+vpslld $ 12, %ymm5, %ymm12
+vpsrld $20, %ymm5, %ymm5
+vpxor %ymm5, %ymm12, %ymm5
+vpxor %ymm6, %ymm10, %ymm6
+vpxor %ymm7, %ymm11, %ymm7
+vpslld $ 12, %ymm6, %ymm12
+vpsrld $20, %ymm6, %ymm6
+vpxor %ymm6, %ymm12, %ymm6
+vpslld $ 12, %ymm7, %ymm12
+vpsrld $20, %ymm7, %ymm7
+vpxor %ymm7, %ymm12, %ymm7
+vpaddd %ymm0, %ymm4, %ymm0
+vpaddd %ymm1, %ymm5, %ymm1
+vpxor 96(%rsp), %ymm0, %ymm12
+vpxor %ymm13, %ymm1, %ymm13
+vpaddd %ymm2, %ymm6, %ymm2
+vpaddd %ymm3, %ymm7, %ymm3
+vpxor %ymm14, %ymm2, %ymm14
+vpxor %ymm15, %ymm3, %ymm15
+vpshufb 480(%rsp), %ymm12, %ymm12
+vpshufb 480(%rsp), %ymm13, %ymm13
+vpaddd %ymm8, %ymm12, %ymm8
+vpaddd %ymm9, %ymm13, %ymm9
+vpshufb 480(%rsp), %ymm14, %ymm14
+vpshufb 480(%rsp), %ymm15, %ymm15
+vpaddd %ymm10, %ymm14, %ymm10
+vpaddd %ymm11, %ymm15, %ymm11
+vmovdqa %ymm12, 96(%rsp)
+vpxor %ymm4, %ymm8, %ymm4
+vpxor %ymm5, %ymm9, %ymm5
+vpslld $ 7, %ymm4, %ymm12
+vpsrld $25, %ymm4, %ymm4
+vpxor %ymm4, %ymm12, %ymm4
+vpslld $ 7, %ymm5, %ymm12
+vpsrld $25, %ymm5, %ymm5
+vpxor %ymm5, %ymm12, %ymm5
+vpxor %ymm6, %ymm10, %ymm6
+vpxor %ymm7, %ymm11, %ymm7
+vpslld $ 7, %ymm6, %ymm12
+vpsrld $25, %ymm6, %ymm6
+vpxor %ymm6, %ymm12, %ymm6
+vpslld $ 7, %ymm7, %ymm12
+vpsrld $25, %ymm7, %ymm7
+vpxor %ymm7, %ymm12, %ymm7
+vpaddd %ymm0, %ymm5, %ymm0
+vpaddd %ymm1, %ymm6, %ymm1
+vpxor %ymm15, %ymm0, %ymm15
+vpxor 96(%rsp), %ymm1, %ymm12
+vpaddd %ymm2, %ymm7, %ymm2
+vpaddd %ymm3, %ymm4, %ymm3
+vpxor %ymm13, %ymm2, %ymm13
+vpxor %ymm14, %ymm3, %ymm14
+vpshufb 448(%rsp), %ymm15, %ymm15
+vpshufb 448(%rsp), %ymm12, %ymm12
+vpaddd %ymm10, %ymm15, %ymm10
+vpaddd %ymm11, %ymm12, %ymm11
+vpshufb 448(%rsp), %ymm13, %ymm13
+vpshufb 448(%rsp), %ymm14, %ymm14
+vpaddd %ymm8, %ymm13, %ymm8
+vpaddd %ymm9, %ymm14, %ymm9
+vmovdqa %ymm15, 96(%rsp)
+vpxor %ymm5, %ymm10, %ymm5
+vpxor %ymm6, %ymm11, %ymm6
+vpslld $ 12, %ymm5, %ymm15
+vpsrld $20, %ymm5, %ymm5
+vpxor %ymm5, %ymm15, %ymm5
+vpslld $ 12, %ymm6, %ymm15
+vpsrld $20, %ymm6, %ymm6
+vpxor %ymm6, %ymm15, %ymm6
+vpxor %ymm7, %ymm8, %ymm7
+vpxor %ymm4, %ymm9, %ymm4
+vpslld $ 12, %ymm7, %ymm15
+vpsrld $20, %ymm7, %ymm7
+vpxor %ymm7, %ymm15, %ymm7
+vpslld $ 12, %ymm4, %ymm15
+vpsrld $20, %ymm4, %ymm4
+vpxor %ymm4, %ymm15, %ymm4
+vpaddd %ymm0, %ymm5, %ymm0
+vpaddd %ymm1, %ymm6, %ymm1
+vpxor 96(%rsp), %ymm0, %ymm15
+vpxor %ymm12, %ymm1, %ymm12
+vpaddd %ymm2, %ymm7, %ymm2
+vpaddd %ymm3, %ymm4, %ymm3
+vpxor %ymm13, %ymm2, %ymm13
+vpxor %ymm14, %ymm3, %ymm14
+vpshufb 480(%rsp), %ymm15, %ymm15
+vpshufb 480(%rsp), %ymm12, %ymm12
+vpaddd %ymm10, %ymm15, %ymm10
+vpaddd %ymm11, %ymm12, %ymm11
+vpshufb 480(%rsp), %ymm13, %ymm13
+vpshufb 480(%rsp), %ymm14, %ymm14
+vpaddd %ymm8, %ymm13, %ymm8
+vpaddd %ymm9, %ymm14, %ymm9
+vmovdqa %ymm15, 96(%rsp)
+vpxor %ymm5, %ymm10, %ymm5
+vpxor %ymm6, %ymm11, %ymm6
+vpslld $ 7, %ymm5, %ymm15
+vpsrld $25, %ymm5, %ymm5
+vpxor %ymm5, %ymm15, %ymm5
+vpslld $ 7, %ymm6, %ymm15
+vpsrld $25, %ymm6, %ymm6
+vpxor %ymm6, %ymm15, %ymm6
+vpxor %ymm7, %ymm8, %ymm7
+vpxor %ymm4, %ymm9, %ymm4
+vpslld $ 7, %ymm7, %ymm15
+vpsrld $25, %ymm7, %ymm7
+vpxor %ymm7, %ymm15, %ymm7
+vpslld $ 7, %ymm4, %ymm15
+vpsrld $25, %ymm4, %ymm4
+vpxor %ymm4, %ymm15, %ymm4
+vmovdqa 96(%rsp), %ymm15
+subq $2, %rax
+jnz chacha_blocks_avx2_mainloop1
+vmovdqa %ymm8, 192(%rsp)
+vmovdqa %ymm9, 224(%rsp)
+vmovdqa %ymm10, 256(%rsp)
+vmovdqa %ymm11, 288(%rsp)
+vmovdqa %ymm12, 320(%rsp)
+vmovdqa %ymm13, 352(%rsp)
+vmovdqa %ymm14, 384(%rsp)
+vmovdqa %ymm15, 416(%rsp)
+vpbroadcastd 0(%rsp), %ymm8
+vpbroadcastd 4+0(%rsp), %ymm9
+vpbroadcastd 8+0(%rsp), %ymm10
+vpbroadcastd 12+0(%rsp), %ymm11
+vpbroadcastd 16(%rsp), %ymm12
+vpbroadcastd 4+16(%rsp), %ymm13
+vpbroadcastd 8+16(%rsp), %ymm14
+vpbroadcastd 12+16(%rsp), %ymm15
+vpaddd %ymm8, %ymm0, %ymm0
+vpaddd %ymm9, %ymm1, %ymm1
+vpaddd %ymm10, %ymm2, %ymm2
+vpaddd %ymm11, %ymm3, %ymm3
+vpaddd %ymm12, %ymm4, %ymm4
+vpaddd %ymm13, %ymm5, %ymm5
+vpaddd %ymm14, %ymm6, %ymm6
+vpaddd %ymm15, %ymm7, %ymm7
+vpunpckldq %ymm1, %ymm0, %ymm8
+vpunpckldq %ymm3, %ymm2, %ymm9
+vpunpckhdq %ymm1, %ymm0, %ymm12
+vpunpckhdq %ymm3, %ymm2, %ymm13
+vpunpckldq %ymm5, %ymm4, %ymm10
+vpunpckldq %ymm7, %ymm6, %ymm11
+vpunpckhdq %ymm5, %ymm4, %ymm14
+vpunpckhdq %ymm7, %ymm6, %ymm15
+vpunpcklqdq %ymm9, %ymm8, %ymm0
+vpunpcklqdq %ymm11, %ymm10, %ymm1
+vpunpckhqdq %ymm9, %ymm8, %ymm2
+vpunpckhqdq %ymm11, %ymm10, %ymm3
+vpunpcklqdq %ymm13, %ymm12, %ymm4
+vpunpcklqdq %ymm15, %ymm14, %ymm5
+vpunpckhqdq %ymm13, %ymm12, %ymm6
+vpunpckhqdq %ymm15, %ymm14, %ymm7
+vperm2i128 $0x20, %ymm1, %ymm0, %ymm8
+vperm2i128 $0x20, %ymm3, %ymm2, %ymm9
+vperm2i128 $0x31, %ymm1, %ymm0, %ymm12
+vperm2i128 $0x31, %ymm3, %ymm2, %ymm13
+vperm2i128 $0x20, %ymm5, %ymm4, %ymm10
+vperm2i128 $0x20, %ymm7, %ymm6, %ymm11
+vperm2i128 $0x31, %ymm5, %ymm4, %ymm14
+vperm2i128 $0x31, %ymm7, %ymm6, %ymm15
+andq %rsi, %rsi
+jz chacha_blocks_avx2_noinput1
+vpxor 0(%rsi), %ymm8, %ymm8
+vpxor 64(%rsi), %ymm9, %ymm9
+vpxor 128(%rsi), %ymm10, %ymm10
+vpxor 192(%rsi), %ymm11, %ymm11
+vpxor 256(%rsi), %ymm12, %ymm12
+vpxor 320(%rsi), %ymm13, %ymm13
+vpxor 384(%rsi), %ymm14, %ymm14
+vpxor 448(%rsi), %ymm15, %ymm15
+vmovdqu %ymm8, 0(%rdx)
+vmovdqu %ymm9, 64(%rdx)
+vmovdqu %ymm10, 128(%rdx)
+vmovdqu %ymm11, 192(%rdx)
+vmovdqu %ymm12, 256(%rdx)
+vmovdqu %ymm13, 320(%rdx)
+vmovdqu %ymm14, 384(%rdx)
+vmovdqu %ymm15, 448(%rdx)
+vmovdqa 192(%rsp), %ymm0
+vmovdqa 224(%rsp), %ymm1
+vmovdqa 256(%rsp), %ymm2
+vmovdqa 288(%rsp), %ymm3
+vmovdqa 320(%rsp), %ymm4
+vmovdqa 352(%rsp), %ymm5
+vmovdqa 384(%rsp), %ymm6
+vmovdqa 416(%rsp), %ymm7
+vpbroadcastd 32(%rsp), %ymm8
+vpbroadcastd 4+32(%rsp), %ymm9
+vpbroadcastd 8+32(%rsp), %ymm10
+vpbroadcastd 12+32(%rsp), %ymm11
+vmovdqa 128(%rsp), %ymm12
+vmovdqa 160(%rsp), %ymm13
+vpbroadcastd 8+48(%rsp), %ymm14
+vpbroadcastd 12+48(%rsp), %ymm15
+vpaddd %ymm8, %ymm0, %ymm0
+vpaddd %ymm9, %ymm1, %ymm1
+vpaddd %ymm10, %ymm2, %ymm2
+vpaddd %ymm11, %ymm3, %ymm3
+vpaddd %ymm12, %ymm4, %ymm4
+vpaddd %ymm13, %ymm5, %ymm5
+vpaddd %ymm14, %ymm6, %ymm6
+vpaddd %ymm15, %ymm7, %ymm7
+vpunpckldq %ymm1, %ymm0, %ymm8
+vpunpckldq %ymm3, %ymm2, %ymm9
+vpunpckhdq %ymm1, %ymm0, %ymm12
+vpunpckhdq %ymm3, %ymm2, %ymm13
+vpunpckldq %ymm5, %ymm4, %ymm10
+vpunpckldq %ymm7, %ymm6, %ymm11
+vpunpckhdq %ymm5, %ymm4, %ymm14
+vpunpckhdq %ymm7, %ymm6, %ymm15
+vpunpcklqdq %ymm9, %ymm8, %ymm0
+vpunpcklqdq %ymm11, %ymm10, %ymm1
+vpunpckhqdq %ymm9, %ymm8, %ymm2
+vpunpckhqdq %ymm11, %ymm10, %ymm3
+vpunpcklqdq %ymm13, %ymm12, %ymm4
+vpunpcklqdq %ymm15, %ymm14, %ymm5
+vpunpckhqdq %ymm13, %ymm12, %ymm6
+vpunpckhqdq %ymm15, %ymm14, %ymm7
+vperm2i128 $0x20, %ymm1, %ymm0, %ymm8
+vperm2i128 $0x20, %ymm3, %ymm2, %ymm9
+vperm2i128 $0x31, %ymm1, %ymm0, %ymm12
+vperm2i128 $0x31, %ymm3, %ymm2, %ymm13
+vperm2i128 $0x20, %ymm5, %ymm4, %ymm10
+vperm2i128 $0x20, %ymm7, %ymm6, %ymm11
+vperm2i128 $0x31, %ymm5, %ymm4, %ymm14
+vperm2i128 $0x31, %ymm7, %ymm6, %ymm15
+vpxor 32(%rsi), %ymm8, %ymm8
+vpxor 96(%rsi), %ymm9, %ymm9
+vpxor 160(%rsi), %ymm10, %ymm10
+vpxor 224(%rsi), %ymm11, %ymm11
+vpxor 288(%rsi), %ymm12, %ymm12
+vpxor 352(%rsi), %ymm13, %ymm13
+vpxor 416(%rsi), %ymm14, %ymm14
+vpxor 480(%rsi), %ymm15, %ymm15
+vmovdqu %ymm8, 32(%rdx)
+vmovdqu %ymm9, 96(%rdx)
+vmovdqu %ymm10, 160(%rdx)
+vmovdqu %ymm11, 224(%rdx)
+vmovdqu %ymm12, 288(%rdx)
+vmovdqu %ymm13, 352(%rdx)
+vmovdqu %ymm14, 416(%rdx)
+vmovdqu %ymm15, 480(%rdx)
+addq $512, %rsi
+jmp chacha_blocks_avx2_mainloop1_cont
+chacha_blocks_avx2_noinput1:
+vmovdqu %ymm8, 0(%rdx)
+vmovdqu %ymm9, 64(%rdx)
+vmovdqu %ymm10, 128(%rdx)
+vmovdqu %ymm11, 192(%rdx)
+vmovdqu %ymm12, 256(%rdx)
+vmovdqu %ymm13, 320(%rdx)
+vmovdqu %ymm14, 384(%rdx)
+vmovdqu %ymm15, 448(%rdx)
+vmovdqa 192(%rsp), %ymm0
+vmovdqa 224(%rsp), %ymm1
+vmovdqa 256(%rsp), %ymm2
+vmovdqa 288(%rsp), %ymm3
+vmovdqa 320(%rsp), %ymm4
+vmovdqa 352(%rsp), %ymm5
+vmovdqa 384(%rsp), %ymm6
+vmovdqa 416(%rsp), %ymm7
+vpbroadcastd 32(%rsp), %ymm8
+vpbroadcastd 4+32(%rsp), %ymm9
+vpbroadcastd 8+32(%rsp), %ymm10
+vpbroadcastd 12+32(%rsp), %ymm11
+vmovdqa 128(%rsp), %ymm12
+vmovdqa 160(%rsp), %ymm13
+vpbroadcastd 8+48(%rsp), %ymm14
+vpbroadcastd 12+48(%rsp), %ymm15
+vpaddd %ymm8, %ymm0, %ymm0
+vpaddd %ymm9, %ymm1, %ymm1
+vpaddd %ymm10, %ymm2, %ymm2
+vpaddd %ymm11, %ymm3, %ymm3
+vpaddd %ymm12, %ymm4, %ymm4
+vpaddd %ymm13, %ymm5, %ymm5
+vpaddd %ymm14, %ymm6, %ymm6
+vpaddd %ymm15, %ymm7, %ymm7
+vpunpckldq %ymm1, %ymm0, %ymm8
+vpunpckldq %ymm3, %ymm2, %ymm9
+vpunpckhdq %ymm1, %ymm0, %ymm12
+vpunpckhdq %ymm3, %ymm2, %ymm13
+vpunpckldq %ymm5, %ymm4, %ymm10
+vpunpckldq %ymm7, %ymm6, %ymm11
+vpunpckhdq %ymm5, %ymm4, %ymm14
+vpunpckhdq %ymm7, %ymm6, %ymm15
+vpunpcklqdq %ymm9, %ymm8, %ymm0
+vpunpcklqdq %ymm11, %ymm10, %ymm1
+vpunpckhqdq %ymm9, %ymm8, %ymm2
+vpunpckhqdq %ymm11, %ymm10, %ymm3
+vpunpcklqdq %ymm13, %ymm12, %ymm4
+vpunpcklqdq %ymm15, %ymm14, %ymm5
+vpunpckhqdq %ymm13, %ymm12, %ymm6
+vpunpckhqdq %ymm15, %ymm14, %ymm7
+vperm2i128 $0x20, %ymm1, %ymm0, %ymm8
+vperm2i128 $0x20, %ymm3, %ymm2, %ymm9
+vperm2i128 $0x31, %ymm1, %ymm0, %ymm12
+vperm2i128 $0x31, %ymm3, %ymm2, %ymm13
+vperm2i128 $0x20, %ymm5, %ymm4, %ymm10
+vperm2i128 $0x20, %ymm7, %ymm6, %ymm11
+vperm2i128 $0x31, %ymm5, %ymm4, %ymm14
+vperm2i128 $0x31, %ymm7, %ymm6, %ymm15
+vmovdqu %ymm8, 32(%rdx)
+vmovdqu %ymm9, 96(%rdx)
+vmovdqu %ymm10, 160(%rdx)
+vmovdqu %ymm11, 224(%rdx)
+vmovdqu %ymm12, 288(%rdx)
+vmovdqu %ymm13, 352(%rdx)
+vmovdqu %ymm14, 416(%rdx)
+vmovdqu %ymm15, 480(%rdx)
+chacha_blocks_avx2_mainloop1_cont:
+addq $512, %rdx
+subq $512, %rcx
+cmp $512, %rcx
+jae chacha_blocks_avx2_atleast512
+cmp $256, %rcx
+jb chacha_blocks_avx2_below256_fixup
+chacha_blocks_avx2_atleast256:
+movq 48(%rsp), %rax
+leaq 1(%rax), %r8
+leaq 2(%rax), %r9
+leaq 3(%rax), %r10
+leaq 4(%rax), %rbx
+movl %eax, 128(%rsp)
+movl %r8d, 4+128(%rsp)
+movl %r9d, 8+128(%rsp)
+movl %r10d, 12+128(%rsp)
+shrq $32, %rax
+shrq $32, %r8
+shrq $32, %r9
+shrq $32, %r10
+movl %eax, 160(%rsp)
+movl %r8d, 4+160(%rsp)
+movl %r9d, 8+160(%rsp)
+movl %r10d, 12+160(%rsp)
+movq %rbx, 48(%rsp)
+movq 64(%rsp), %rax
+vpbroadcastd 0(%rsp), %xmm0
+vpbroadcastd 4+0(%rsp), %xmm1
+vpbroadcastd 8+0(%rsp), %xmm2
+vpbroadcastd 12+0(%rsp), %xmm3
+vpbroadcastd 16(%rsp), %xmm4
+vpbroadcastd 4+16(%rsp), %xmm5
+vpbroadcastd 8+16(%rsp), %xmm6
+vpbroadcastd 12+16(%rsp), %xmm7
+vpbroadcastd 32(%rsp), %xmm8
+vpbroadcastd 4+32(%rsp), %xmm9
+vpbroadcastd 8+32(%rsp), %xmm10
+vpbroadcastd 12+32(%rsp), %xmm11
+vmovdqa 128(%rsp), %xmm12
+vmovdqa 160(%rsp), %xmm13
+vpbroadcastd 8+48(%rsp), %xmm14
+vpbroadcastd 12+48(%rsp), %xmm15
+chacha_blocks_avx2_mainloop2:
+vpaddd %xmm0, %xmm4, %xmm0
+vpaddd %xmm1, %xmm5, %xmm1
+vpxor %xmm12, %xmm0, %xmm12
+vpxor %xmm13, %xmm1, %xmm13
+vpaddd %xmm2, %xmm6, %xmm2
+vpaddd %xmm3, %xmm7, %xmm3
+vpxor %xmm14, %xmm2, %xmm14
+vpxor %xmm15, %xmm3, %xmm15
+vpshufb 448(%rsp), %xmm12, %xmm12
+vpshufb 448(%rsp), %xmm13, %xmm13
+vpaddd %xmm8, %xmm12, %xmm8
+vpaddd %xmm9, %xmm13, %xmm9
+vpshufb 448(%rsp), %xmm14, %xmm14
+vpshufb 448(%rsp), %xmm15, %xmm15
+vpaddd %xmm10, %xmm14, %xmm10
+vpaddd %xmm11, %xmm15, %xmm11
+vmovdqa %xmm12, 96(%rsp)
+vpxor %xmm4, %xmm8, %xmm4
+vpxor %xmm5, %xmm9, %xmm5
+vpslld $ 12, %xmm4, %xmm12
+vpsrld $20, %xmm4, %xmm4
+vpxor %xmm4, %xmm12, %xmm4
+vpslld $ 12, %xmm5, %xmm12
+vpsrld $20, %xmm5, %xmm5
+vpxor %xmm5, %xmm12, %xmm5
+vpxor %xmm6, %xmm10, %xmm6
+vpxor %xmm7, %xmm11, %xmm7
+vpslld $ 12, %xmm6, %xmm12
+vpsrld $20, %xmm6, %xmm6
+vpxor %xmm6, %xmm12, %xmm6
+vpslld $ 12, %xmm7, %xmm12
+vpsrld $20, %xmm7, %xmm7
+vpxor %xmm7, %xmm12, %xmm7
+vpaddd %xmm0, %xmm4, %xmm0
+vpaddd %xmm1, %xmm5, %xmm1
+vpxor 96(%rsp), %xmm0, %xmm12
+vpxor %xmm13, %xmm1, %xmm13
+vpaddd %xmm2, %xmm6, %xmm2
+vpaddd %xmm3, %xmm7, %xmm3
+vpxor %xmm14, %xmm2, %xmm14
+vpxor %xmm15, %xmm3, %xmm15
+vpshufb 480(%rsp), %xmm12, %xmm12
+vpshufb 480(%rsp), %xmm13, %xmm13
+vpaddd %xmm8, %xmm12, %xmm8
+vpaddd %xmm9, %xmm13, %xmm9
+vpshufb 480(%rsp), %xmm14, %xmm14
+vpshufb 480(%rsp), %xmm15, %xmm15
+vpaddd %xmm10, %xmm14, %xmm10
+vpaddd %xmm11, %xmm15, %xmm11
+vmovdqa %xmm12, 96(%rsp)
+vpxor %xmm4, %xmm8, %xmm4
+vpxor %xmm5, %xmm9, %xmm5
+vpslld $ 7, %xmm4, %xmm12
+vpsrld $25, %xmm4, %xmm4
+vpxor %xmm4, %xmm12, %xmm4
+vpslld $ 7, %xmm5, %xmm12
+vpsrld $25, %xmm5, %xmm5
+vpxor %xmm5, %xmm12, %xmm5
+vpxor %xmm6, %xmm10, %xmm6
+vpxor %xmm7, %xmm11, %xmm7
+vpslld $ 7, %xmm6, %xmm12
+vpsrld $25, %xmm6, %xmm6
+vpxor %xmm6, %xmm12, %xmm6
+vpslld $ 7, %xmm7, %xmm12
+vpsrld $25, %xmm7, %xmm7
+vpxor %xmm7, %xmm12, %xmm7
+vpaddd %xmm0, %xmm5, %xmm0
+vpaddd %xmm1, %xmm6, %xmm1
+vpxor %xmm15, %xmm0, %xmm15
+vpxor 96(%rsp), %xmm1, %xmm12
+vpaddd %xmm2, %xmm7, %xmm2
+vpaddd %xmm3, %xmm4, %xmm3
+vpxor %xmm13, %xmm2, %xmm13
+vpxor %xmm14, %xmm3, %xmm14
+vpshufb 448(%rsp), %xmm15, %xmm15
+vpshufb 448(%rsp), %xmm12, %xmm12
+vpaddd %xmm10, %xmm15, %xmm10
+vpaddd %xmm11, %xmm12, %xmm11
+vpshufb 448(%rsp), %xmm13, %xmm13
+vpshufb 448(%rsp), %xmm14, %xmm14
+vpaddd %xmm8, %xmm13, %xmm8
+vpaddd %xmm9, %xmm14, %xmm9
+vmovdqa %xmm15, 96(%rsp)
+vpxor %xmm5, %xmm10, %xmm5
+vpxor %xmm6, %xmm11, %xmm6
+vpslld $ 12, %xmm5, %xmm15
+vpsrld $20, %xmm5, %xmm5
+vpxor %xmm5, %xmm15, %xmm5
+vpslld $ 12, %xmm6, %xmm15
+vpsrld $20, %xmm6, %xmm6
+vpxor %xmm6, %xmm15, %xmm6
+vpxor %xmm7, %xmm8, %xmm7
+vpxor %xmm4, %xmm9, %xmm4
+vpslld $ 12, %xmm7, %xmm15
+vpsrld $20, %xmm7, %xmm7
+vpxor %xmm7, %xmm15, %xmm7
+vpslld $ 12, %xmm4, %xmm15
+vpsrld $20, %xmm4, %xmm4
+vpxor %xmm4, %xmm15, %xmm4
+vpaddd %xmm0, %xmm5, %xmm0
+vpaddd %xmm1, %xmm6, %xmm1
+vpxor 96(%rsp), %xmm0, %xmm15
+vpxor %xmm12, %xmm1, %xmm12
+vpaddd %xmm2, %xmm7, %xmm2
+vpaddd %xmm3, %xmm4, %xmm3
+vpxor %xmm13, %xmm2, %xmm13
+vpxor %xmm14, %xmm3, %xmm14
+vpshufb 480(%rsp), %xmm15, %xmm15
+vpshufb 480(%rsp), %xmm12, %xmm12
+vpaddd %xmm10, %xmm15, %xmm10
+vpaddd %xmm11, %xmm12, %xmm11
+vpshufb 480(%rsp), %xmm13, %xmm13
+vpshufb 480(%rsp), %xmm14, %xmm14
+vpaddd %xmm8, %xmm13, %xmm8
+vpaddd %xmm9, %xmm14, %xmm9
+vmovdqa %xmm15, 96(%rsp)
+vpxor %xmm5, %xmm10, %xmm5
+vpxor %xmm6, %xmm11, %xmm6
+vpslld $ 7, %xmm5, %xmm15
+vpsrld $25, %xmm5, %xmm5
+vpxor %xmm5, %xmm15, %xmm5
+vpslld $ 7, %xmm6, %xmm15
+vpsrld $25, %xmm6, %xmm6
+vpxor %xmm6, %xmm15, %xmm6
+vpxor %xmm7, %xmm8, %xmm7
+vpxor %xmm4, %xmm9, %xmm4
+vpslld $ 7, %xmm7, %xmm15
+vpsrld $25, %xmm7, %xmm7
+vpxor %xmm7, %xmm15, %xmm7
+vpslld $ 7, %xmm4, %xmm15
+vpsrld $25, %xmm4, %xmm4
+vpxor %xmm4, %xmm15, %xmm4
+vmovdqa 96(%rsp), %xmm15
+subq $2, %rax
+jnz chacha_blocks_avx2_mainloop2
+vmovdqa %xmm8, 192(%rsp)
+vmovdqa %xmm9, 208(%rsp)
+vmovdqa %xmm10, 224(%rsp)
+vmovdqa %xmm11, 240(%rsp)
+vmovdqa %xmm12, 256(%rsp)
+vmovdqa %xmm13, 272(%rsp)
+vmovdqa %xmm14, 288(%rsp)
+vmovdqa %xmm15, 304(%rsp)
+vpbroadcastd 0(%rsp), %xmm8
+vpbroadcastd 4+0(%rsp), %xmm9
+vpbroadcastd 8+0(%rsp), %xmm10
+vpbroadcastd 12+0(%rsp), %xmm11
+vpbroadcastd 16(%rsp), %xmm12
+vpbroadcastd 4+16(%rsp), %xmm13
+vpbroadcastd 8+16(%rsp), %xmm14
+vpbroadcastd 12+16(%rsp), %xmm15
+vpaddd %xmm8, %xmm0, %xmm0
+vpaddd %xmm9, %xmm1, %xmm1
+vpaddd %xmm10, %xmm2, %xmm2
+vpaddd %xmm11, %xmm3, %xmm3
+vpaddd %xmm12, %xmm4, %xmm4
+vpaddd %xmm13, %xmm5, %xmm5
+vpaddd %xmm14, %xmm6, %xmm6
+vpaddd %xmm15, %xmm7, %xmm7
+vpunpckldq %xmm1, %xmm0, %xmm8
+vpunpckldq %xmm3, %xmm2, %xmm9
+vpunpckhdq %xmm1, %xmm0, %xmm12
+vpunpckhdq %xmm3, %xmm2, %xmm13
+vpunpckldq %xmm5, %xmm4, %xmm10
+vpunpckldq %xmm7, %xmm6, %xmm11
+vpunpckhdq %xmm5, %xmm4, %xmm14
+vpunpckhdq %xmm7, %xmm6, %xmm15
+vpunpcklqdq %xmm9, %xmm8, %xmm0
+vpunpcklqdq %xmm11, %xmm10, %xmm1
+vpunpckhqdq %xmm9, %xmm8, %xmm2
+vpunpckhqdq %xmm11, %xmm10, %xmm3
+vpunpcklqdq %xmm13, %xmm12, %xmm4
+vpunpcklqdq %xmm15, %xmm14, %xmm5
+vpunpckhqdq %xmm13, %xmm12, %xmm6
+vpunpckhqdq %xmm15, %xmm14, %xmm7
+andq %rsi, %rsi
+jz chacha_blocks_avx2_noinput2
+vpxor 0(%rsi), %xmm0, %xmm0
+vpxor 16(%rsi), %xmm1, %xmm1
+vpxor 64(%rsi), %xmm2, %xmm2
+vpxor 80(%rsi), %xmm3, %xmm3
+vpxor 128(%rsi), %xmm4, %xmm4
+vpxor 144(%rsi), %xmm5, %xmm5
+vpxor 192(%rsi), %xmm6, %xmm6
+vpxor 208(%rsi), %xmm7, %xmm7
+vmovdqu %xmm0, 0(%rdx)
+vmovdqu %xmm1, 16(%rdx)
+vmovdqu %xmm2, 64(%rdx)
+vmovdqu %xmm3, 80(%rdx)
+vmovdqu %xmm4, 128(%rdx)
+vmovdqu %xmm5, 144(%rdx)
+vmovdqu %xmm6, 192(%rdx)
+vmovdqu %xmm7, 208(%rdx)
+vmovdqa 192(%rsp), %xmm0
+vmovdqa 208(%rsp), %xmm1
+vmovdqa 224(%rsp), %xmm2
+vmovdqa 240(%rsp), %xmm3
+vmovdqa 256(%rsp), %xmm4
+vmovdqa 272(%rsp), %xmm5
+vmovdqa 288(%rsp), %xmm6
+vmovdqa 304(%rsp), %xmm7
+vpbroadcastd 32(%rsp), %xmm8
+vpbroadcastd 4+32(%rsp), %xmm9
+vpbroadcastd 8+32(%rsp), %xmm10
+vpbroadcastd 12+32(%rsp), %xmm11
+vmovdqa 128(%rsp), %xmm12
+vmovdqa 160(%rsp), %xmm13
+vpbroadcastd 8+48(%rsp), %xmm14
+vpbroadcastd 12+48(%rsp), %xmm15
+vpaddd %xmm8, %xmm0, %xmm0
+vpaddd %xmm9, %xmm1, %xmm1
+vpaddd %xmm10, %xmm2, %xmm2
+vpaddd %xmm11, %xmm3, %xmm3
+vpaddd %xmm12, %xmm4, %xmm4
+vpaddd %xmm13, %xmm5, %xmm5
+vpaddd %xmm14, %xmm6, %xmm6
+vpaddd %xmm15, %xmm7, %xmm7
+vpunpckldq %xmm1, %xmm0, %xmm8
+vpunpckldq %xmm3, %xmm2, %xmm9
+vpunpckhdq %xmm1, %xmm0, %xmm12
+vpunpckhdq %xmm3, %xmm2, %xmm13
+vpunpckldq %xmm5, %xmm4, %xmm10
+vpunpckldq %xmm7, %xmm6, %xmm11
+vpunpckhdq %xmm5, %xmm4, %xmm14
+vpunpckhdq %xmm7, %xmm6, %xmm15
+vpunpcklqdq %xmm9, %xmm8, %xmm0
+vpunpcklqdq %xmm11, %xmm10, %xmm1
+vpunpckhqdq %xmm9, %xmm8, %xmm2
+vpunpckhqdq %xmm11, %xmm10, %xmm3
+vpunpcklqdq %xmm13, %xmm12, %xmm4
+vpunpcklqdq %xmm15, %xmm14, %xmm5
+vpunpckhqdq %xmm13, %xmm12, %xmm6
+vpunpckhqdq %xmm15, %xmm14, %xmm7
+vpxor 32(%rsi), %xmm0, %xmm0
+vpxor 48(%rsi), %xmm1, %xmm1
+vpxor 96(%rsi), %xmm2, %xmm2
+vpxor 112(%rsi), %xmm3, %xmm3
+vpxor 160(%rsi), %xmm4, %xmm4
+vpxor 176(%rsi), %xmm5, %xmm5
+vpxor 224(%rsi), %xmm6, %xmm6
+vpxor 240(%rsi), %xmm7, %xmm7
+vmovdqu %xmm0, 32(%rdx)
+vmovdqu %xmm1, 48(%rdx)
+vmovdqu %xmm2, 96(%rdx)
+vmovdqu %xmm3, 112(%rdx)
+vmovdqu %xmm4, 160(%rdx)
+vmovdqu %xmm5, 176(%rdx)
+vmovdqu %xmm6, 224(%rdx)
+vmovdqu %xmm7, 240(%rdx)
+addq $256, %rsi
+jmp chacha_blocks_avx2_mainloop2_cont
+chacha_blocks_avx2_noinput2:
+vmovdqu %xmm0, 0(%rdx)
+vmovdqu %xmm1, 16(%rdx)
+vmovdqu %xmm2, 64(%rdx)
+vmovdqu %xmm3, 80(%rdx)
+vmovdqu %xmm4, 128(%rdx)
+vmovdqu %xmm5, 144(%rdx)
+vmovdqu %xmm6, 192(%rdx)
+vmovdqu %xmm7, 208(%rdx)
+vmovdqa 192(%rsp), %xmm0
+vmovdqa 208(%rsp), %xmm1
+vmovdqa 224(%rsp), %xmm2
+vmovdqa 240(%rsp), %xmm3
+vmovdqa 256(%rsp), %xmm4
+vmovdqa 272(%rsp), %xmm5
+vmovdqa 288(%rsp), %xmm6
+vmovdqa 304(%rsp), %xmm7
+vpbroadcastd 32(%rsp), %xmm8
+vpbroadcastd 4+32(%rsp), %xmm9
+vpbroadcastd 8+32(%rsp), %xmm10
+vpbroadcastd 12+32(%rsp), %xmm11
+vmovdqa 128(%rsp), %xmm12
+vmovdqa 160(%rsp), %xmm13
+vpbroadcastd 8+48(%rsp), %xmm14
+vpbroadcastd 12+48(%rsp), %xmm15
+vpaddd %xmm8, %xmm0, %xmm0
+vpaddd %xmm9, %xmm1, %xmm1
+vpaddd %xmm10, %xmm2, %xmm2
+vpaddd %xmm11, %xmm3, %xmm3
+vpaddd %xmm12, %xmm4, %xmm4
+vpaddd %xmm13, %xmm5, %xmm5
+vpaddd %xmm14, %xmm6, %xmm6
+vpaddd %xmm15, %xmm7, %xmm7
+vpunpckldq %xmm1, %xmm0, %xmm8
+vpunpckldq %xmm3, %xmm2, %xmm9
+vpunpckhdq %xmm1, %xmm0, %xmm12
+vpunpckhdq %xmm3, %xmm2, %xmm13
+vpunpckldq %xmm5, %xmm4, %xmm10
+vpunpckldq %xmm7, %xmm6, %xmm11
+vpunpckhdq %xmm5, %xmm4, %xmm14
+vpunpckhdq %xmm7, %xmm6, %xmm15
+vpunpcklqdq %xmm9, %xmm8, %xmm0
+vpunpcklqdq %xmm11, %xmm10, %xmm1
+vpunpckhqdq %xmm9, %xmm8, %xmm2
+vpunpckhqdq %xmm11, %xmm10, %xmm3
+vpunpcklqdq %xmm13, %xmm12, %xmm4
+vpunpcklqdq %xmm15, %xmm14, %xmm5
+vpunpckhqdq %xmm13, %xmm12, %xmm6
+vpunpckhqdq %xmm15, %xmm14, %xmm7
+vmovdqu %xmm0, 32(%rdx)
+vmovdqu %xmm1, 48(%rdx)
+vmovdqu %xmm2, 96(%rdx)
+vmovdqu %xmm3, 112(%rdx)
+vmovdqu %xmm4, 160(%rdx)
+vmovdqu %xmm5, 176(%rdx)
+vmovdqu %xmm6, 224(%rdx)
+vmovdqu %xmm7, 240(%rdx)
+chacha_blocks_avx2_mainloop2_cont:
+addq $256, %rdx
+subq $256, %rcx
+cmp $256, %rcx
+jae chacha_blocks_avx2_atleast256
+chacha_blocks_avx2_below256_fixup:
+vmovdqa 448(%rsp), %xmm6
+vmovdqa 480(%rsp), %xmm7
+vmovdqa 0(%rsp), %xmm8
+vmovdqa 16(%rsp), %xmm9
+vmovdqa 32(%rsp), %xmm10
+vmovdqa 48(%rsp), %xmm11
+movq $1, %r9
+chacha_blocks_avx2_below256:
+vmovq %r9, %xmm5
+andq %rcx, %rcx
+jz chacha_blocks_avx2_done
+cmpq $64, %rcx
+jae chacha_blocks_avx2_above63
+movq %rdx, %r9
+andq %rsi, %rsi
+jz chacha_blocks_avx2_noinput3
+movq %rcx, %r10
+movq %rsp, %rdx
+addq %r10, %rsi
+addq %r10, %rdx
+negq %r10
+chacha_blocks_avx2_copyinput:
+movb (%rsi, %r10), %al
+movb %al, (%rdx, %r10)
+incq %r10
+jnz chacha_blocks_avx2_copyinput
+movq %rsp, %rsi
+chacha_blocks_avx2_noinput3:
+movq %rsp, %rdx
+chacha_blocks_avx2_above63:
+vmovdqa %xmm8, %xmm0
+vmovdqa %xmm9, %xmm1
+vmovdqa %xmm10, %xmm2
+vmovdqa %xmm11, %xmm3
+movq 64(%rsp), %rax
+chacha_blocks_avx2_mainloop3:
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm6, %xmm3, %xmm3
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpslld $12, %xmm1, %xmm4
+vpsrld $20, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm7, %xmm3, %xmm3
+vpshufd $0x93, %xmm0, %xmm0
+vpaddd %xmm2, %xmm3, %xmm2
+vpshufd $0x4e, %xmm3, %xmm3
+vpxor %xmm1, %xmm2, %xmm1
+vpshufd $0x39, %xmm2, %xmm2
+vpslld $7, %xmm1, %xmm4
+vpsrld $25, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm6, %xmm3, %xmm3
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpslld $12, %xmm1, %xmm4
+vpsrld $20, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm7, %xmm3, %xmm3
+vpshufd $0x39, %xmm0, %xmm0
+vpaddd %xmm2, %xmm3, %xmm2
+vpshufd $0x4e, %xmm3, %xmm3
+vpxor %xmm1, %xmm2, %xmm1
+vpshufd $0x93, %xmm2, %xmm2
+vpslld $7, %xmm1, %xmm4
+vpsrld $25, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+subq $2, %rax
+jnz chacha_blocks_avx2_mainloop3
+vpaddd %xmm0, %xmm8, %xmm0
+vpaddd %xmm1, %xmm9, %xmm1
+vpaddd %xmm2, %xmm10, %xmm2
+vpaddd %xmm3, %xmm11, %xmm3
+andq %rsi, %rsi
+jz chacha_blocks_avx2_noinput4
+vpxor 0(%rsi), %xmm0, %xmm0
+vpxor 16(%rsi), %xmm1, %xmm1
+vpxor 32(%rsi), %xmm2, %xmm2
+vpxor 48(%rsi), %xmm3, %xmm3
+addq $64, %rsi
+chacha_blocks_avx2_noinput4:
+vmovdqu %xmm0, 0(%rdx)
+vmovdqu %xmm1, 16(%rdx)
+vmovdqu %xmm2, 32(%rdx)
+vmovdqu %xmm3, 48(%rdx)
+vpaddq %xmm11, %xmm5, %xmm11
+cmpq $64, %rcx
+jbe chacha_blocks_avx2_mainloop3_finishup
+addq $64, %rdx
+subq $64, %rcx
+jmp chacha_blocks_avx2_below256
+chacha_blocks_avx2_mainloop3_finishup:
+cmpq $64, %rcx
+je chacha_blocks_avx2_done
+addq %rcx, %r9
+addq %rcx, %rdx
+negq %rcx
+chacha_blocks_avx2_copyoutput:
+movb (%rdx, %rcx), %al
+movb %al, (%r9, %rcx)
+incq %rcx
+jnz chacha_blocks_avx2_copyoutput
+chacha_blocks_avx2_done:
+vmovdqu %xmm11, 32(%rdi)
+movq %rbp, %rsp
+popq %r14
+popq %r13
+popq %r12
+popq %rbp
+popq %rbx
+vzeroupper
+ret
+FN_END chacha_blocks_avx2
+
+
+GLOBAL_HIDDEN_FN hchacha_avx2
+hchacha_avx2_local:
+LOAD_VAR_PIC chacha_constants, %rax
+vmovdqa 0(%rax), %xmm0
+vmovdqa 16(%rax), %xmm6
+vmovdqa 32(%rax), %xmm5
+vmovdqu 0(%rdi), %xmm1
+vmovdqu 16(%rdi), %xmm2
+vmovdqu 0(%rsi), %xmm3
+hhacha_mainloop_avx2:
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm6, %xmm3, %xmm3
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpslld $12, %xmm1, %xmm4
+vpsrld $20, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm5, %xmm3, %xmm3
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpslld $7, %xmm1, %xmm4
+vpsrld $25, %xmm1, %xmm1
+vpshufd $0x93, %xmm0, %xmm0
+vpxor %xmm1, %xmm4, %xmm1
+vpshufd $0x4e, %xmm3, %xmm3
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm6, %xmm3, %xmm3
+vpshufd $0x39, %xmm2, %xmm2
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpslld $12, %xmm1, %xmm4
+vpsrld $20, %xmm1, %xmm1
+vpxor %xmm1, %xmm4, %xmm1
+vpaddd %xmm0, %xmm1, %xmm0
+vpxor %xmm3, %xmm0, %xmm3
+vpshufb %xmm5, %xmm3, %xmm3
+vpaddd %xmm2, %xmm3, %xmm2
+vpxor %xmm1, %xmm2, %xmm1
+vpshufd $0x39, %xmm0, %xmm0
+vpslld $7, %xmm1, %xmm4
+vpshufd $0x4e, %xmm3, %xmm3
+vpsrld $25, %xmm1, %xmm1
+vpshufd $0x93, %xmm2, %xmm2
+vpxor %xmm1, %xmm4, %xmm1
+subl $2, %ecx
+jne hhacha_mainloop_avx2
+vmovdqu %xmm0, (%rdx)
+vmovdqu %xmm3, 16(%rdx)
+ret
+FN_END hchacha_avx2
+
+GLOBAL_HIDDEN_FN_EXT chacha_avx2, 6, 16
+pushq %rbp
+movq %rsp, %rbp
+subq $64, %rsp
+andq $~63, %rsp
+vmovdqu 0(%rdi), %xmm0
+vmovdqu 16(%rdi), %xmm1
+vmovdqa %xmm0, 0(%rsp)
+vmovdqa %xmm1, 16(%rsp)
+xorq %rdi, %rdi
+movq %rdi, 32(%rsp)
+movq 0(%rsi), %rsi
+movq %rsi, 40(%rsp)
+movq %r9, 48(%rsp)
+movq %rsp, %rdi
+movq %rdx, %rsi
+movq %rcx, %rdx
+movq %r8, %rcx
+call chacha_blocks_avx2_local
+vpxor %xmm0, %xmm0, %xmm0
+vmovdqa %xmm0, 0(%rsp)
+vmovdqa %xmm0, 16(%rsp)
+vmovdqa %xmm0, 32(%rsp)
+movq %rbp, %rsp
+popq %rbp
+ret
+FN_END chacha_avx2
+
+GLOBAL_HIDDEN_FN_EXT xchacha_avx2, 6, 16
+pushq %rbp
+pushq %rbx
+movq %rsp, %rbp
+subq $64, %rsp
+andq $~63, %rsp
+movq %rsp, %rbx
+xorq %rax, %rax
+movq %rax, 32(%rbx)
+movq 16(%rsi), %rax
+movq %rax, 40(%rbx)
+movq %r9, 48(%rbx)
+pushq %rdx
+pushq %rcx
+pushq %r8
+movq %rbx, %rdx
+movq %r9, %rcx
+call hchacha_avx2_local
+movq %rbx, %rdi
+popq %rcx
+popq %rdx
+popq %rsi
+call chacha_blocks_avx2_local
+vpxor %xmm0, %xmm0, %xmm0
+vmovdqa %xmm0, 0(%rbx)
+vmovdqa %xmm0, 16(%rbx)
+vmovdqa %xmm0, 32(%rbx)
+movq %rbp, %rsp
+popq %rbx
+popq %rbp
+ret
+FN_END xchacha_avx2
diff --git a/src/libcryptobox/chacha20/chacha.c b/src/libcryptobox/chacha20/chacha.c
new file mode 100644
index 0000000..0b471c8
--- /dev/null
+++ b/src/libcryptobox/chacha20/chacha.c
@@ -0,0 +1,262 @@
+/* Copyright (c) 2015, Vsevolod Stakhov
+ * Copyright (c) 2015, Andrew Moon
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "cryptobox.h"
+#include "chacha.h"
+#include "platform_config.h"
+
+extern unsigned cpu_config;
+
+typedef struct chacha_impl_t {
+ unsigned long cpu_flags;
+ const char *desc;
+ void (*chacha)(const chacha_key *key, const chacha_iv *iv,
+ const unsigned char *in, unsigned char *out, size_t inlen,
+ size_t rounds);
+ void (*xchacha)(const chacha_key *key, const chacha_iv24 *iv,
+ const unsigned char *in, unsigned char *out, size_t inlen,
+ size_t rounds);
+ void (*chacha_blocks)(chacha_state_internal *state,
+ const unsigned char *in, unsigned char *out, size_t bytes);
+ void (*hchacha)(const unsigned char key[32], const unsigned char iv[16],
+ unsigned char out[32], size_t rounds);
+} chacha_impl_t;
+
+#define CHACHA_DECLARE(ext) \
+ void chacha_##ext(const chacha_key *key, const chacha_iv *iv, const unsigned char *in, unsigned char *out, size_t inlen, size_t rounds); \
+ void xchacha_##ext(const chacha_key *key, const chacha_iv24 *iv, const unsigned char *in, unsigned char *out, size_t inlen, size_t rounds); \
+ void chacha_blocks_##ext(chacha_state_internal *state, const unsigned char *in, unsigned char *out, size_t bytes); \
+ void hchacha_##ext(const unsigned char key[32], const unsigned char iv[16], unsigned char out[32], size_t rounds);
+#define CHACHA_IMPL(cpuflags, desc, ext) \
+ { \
+ (cpuflags), desc, chacha_##ext, xchacha_##ext, chacha_blocks_##ext, hchacha_##ext \
+ }
+
+#if defined(HAVE_AVX2) && defined(__x86_64__)
+CHACHA_DECLARE(avx2)
+#define CHACHA_AVX2 CHACHA_IMPL(CPUID_AVX2, "avx2", avx2)
+#endif
+#if defined(HAVE_AVX) && defined(__x86_64__)
+CHACHA_DECLARE(avx)
+#define CHACHA_AVX CHACHA_IMPL(CPUID_AVX, "avx", avx)
+#endif
+#if defined(HAVE_SSE2) && defined(__x86_64__)
+CHACHA_DECLARE(sse2)
+#define CHACHA_SSE2 CHACHA_IMPL(CPUID_SSE2, "sse2", sse2)
+#endif
+
+CHACHA_DECLARE(ref)
+#define CHACHA_GENERIC CHACHA_IMPL(0, "generic", ref)
+
+static const chacha_impl_t chacha_list[] = {
+ CHACHA_GENERIC,
+#if defined(CHACHA_AVX2) && defined(__x86_64__)
+ CHACHA_AVX2,
+#endif
+#if defined(CHACHA_AVX) && defined(__x86_64__)
+ CHACHA_AVX,
+#endif
+#if defined(CHACHA_SSE2) && defined(__x86_64__)
+ CHACHA_SSE2
+#endif
+};
+
+static const chacha_impl_t *chacha_impl = &chacha_list[0];
+
+static int
+chacha_is_aligned(const void *p)
+{
+ return ((size_t) p & (sizeof(size_t) - 1)) == 0;
+}
+
+const char *
+chacha_load(void)
+{
+ guint i;
+
+ if (cpu_config != 0) {
+ for (i = 0; i < G_N_ELEMENTS(chacha_list); i++) {
+ if (chacha_list[i].cpu_flags & cpu_config) {
+ chacha_impl = &chacha_list[i];
+ break;
+ }
+ }
+ }
+
+ return chacha_impl->desc;
+}
+
+void chacha_init(chacha_state *S, const chacha_key *key,
+ const chacha_iv *iv, size_t rounds)
+{
+ chacha_state_internal *state = (chacha_state_internal *) S;
+ memcpy(state->s + 0, key, 32);
+ memset(state->s + 32, 0, 8);
+ memcpy(state->s + 40, iv, 8);
+ state->rounds = rounds;
+ state->leftover = 0;
+}
+
+/* processes inlen bytes (can do partial blocks), handling input/output alignment */
+static void
+chacha_consume(chacha_state_internal *state,
+ const unsigned char *in, unsigned char *out, size_t inlen)
+{
+ unsigned char buffer[16 * CHACHA_BLOCKBYTES];
+ int in_aligned, out_aligned;
+
+ /* it's ok to call with 0 bytes */
+ if (!inlen)
+ return;
+
+ /* if everything is aligned, handle directly */
+ in_aligned = chacha_is_aligned(in);
+ out_aligned = chacha_is_aligned(out);
+ if (in_aligned && out_aligned) {
+ chacha_impl->chacha_blocks(state, in, out, inlen);
+ return;
+ }
+
+ /* copy the unaligned data to an aligned buffer and process in chunks */
+ while (inlen) {
+ const size_t bytes = (inlen > sizeof(buffer)) ? sizeof(buffer) : inlen;
+ const unsigned char *src = in;
+ unsigned char *dst = (out_aligned) ? out : buffer;
+ if (!in_aligned) {
+ memcpy(buffer, in, bytes);
+ src = buffer;
+ }
+ chacha_impl->chacha_blocks(state, src, dst, bytes);
+ if (!out_aligned)
+ memcpy(out, buffer, bytes);
+ if (in)
+ in += bytes;
+ out += bytes;
+ inlen -= bytes;
+ }
+}
+
+/* hchacha */
+void hchacha(const unsigned char key[32],
+ const unsigned char iv[16], unsigned char out[32], size_t rounds)
+{
+ chacha_impl->hchacha(key, iv, out, rounds);
+}
+
+/* update, returns number of bytes written to out */
+size_t
+chacha_update(chacha_state *S, const unsigned char *in, unsigned char *out,
+ size_t inlen)
+{
+ chacha_state_internal *state = (chacha_state_internal *) S;
+ unsigned char *out_start = out;
+ size_t bytes;
+
+ /* enough for at least one block? */
+ while ((state->leftover + inlen) >= CHACHA_BLOCKBYTES) {
+ /* handle the previous data */
+ if (state->leftover) {
+ bytes = (CHACHA_BLOCKBYTES - state->leftover);
+ if (in) {
+ memcpy(state->buffer + state->leftover, in, bytes);
+ in += bytes;
+ }
+ chacha_consume(state, (in) ? state->buffer : NULL, out,
+ CHACHA_BLOCKBYTES);
+ inlen -= bytes;
+ out += CHACHA_BLOCKBYTES;
+ state->leftover = 0;
+ }
+
+ /* handle the direct data */
+ bytes = (inlen & ~(CHACHA_BLOCKBYTES - 1));
+ if (bytes) {
+ chacha_consume(state, in, out, bytes);
+ inlen -= bytes;
+ if (in)
+ in += bytes;
+ out += bytes;
+ }
+ }
+
+ /* handle leftover data */
+ if (inlen) {
+ if (in)
+ memcpy(state->buffer + state->leftover, in, inlen);
+ else
+ memset(state->buffer + state->leftover, 0, inlen);
+ state->leftover += inlen;
+ }
+
+ return out - out_start;
+}
+
+/* finalize, write out any leftover data */
+size_t
+chacha_final(chacha_state *S, unsigned char *out)
+{
+ chacha_state_internal *state = (chacha_state_internal *) S;
+ size_t leftover = state->leftover;
+ if (leftover) {
+ if (chacha_is_aligned(out)) {
+ chacha_impl->chacha_blocks(state, state->buffer, out, leftover);
+ }
+ else {
+ chacha_impl->chacha_blocks(state, state->buffer, state->buffer,
+ leftover);
+ memcpy(out, state->buffer, leftover);
+ }
+ }
+ rspamd_explicit_memzero(S, sizeof(chacha_state));
+ return leftover;
+}
+
+/* one-shot, input/output assumed to be word aligned */
+void chacha(const chacha_key *key, const chacha_iv *iv,
+ const unsigned char *in, unsigned char *out, size_t inlen,
+ size_t rounds)
+{
+ chacha_impl->chacha(key, iv, in, out, inlen, rounds);
+}
+
+/*
+ xchacha, chacha with a 192 bit nonce
+ */
+
+void xchacha_init(chacha_state *S, const chacha_key *key,
+ const chacha_iv24 *iv, size_t rounds)
+{
+ chacha_key subkey;
+ hchacha(key->b, iv->b, subkey.b, rounds);
+ chacha_init(S, &subkey, (chacha_iv *) (iv->b + 16), rounds);
+}
+
+/* one-shot, input/output assumed to be word aligned */
+void xchacha(const chacha_key *key, const chacha_iv24 *iv,
+ const unsigned char *in, unsigned char *out, size_t inlen,
+ size_t rounds)
+{
+ chacha_impl->xchacha(key, iv, in, out, inlen, rounds);
+}
diff --git a/src/libcryptobox/chacha20/chacha.h b/src/libcryptobox/chacha20/chacha.h
new file mode 100644
index 0000000..d05088a
--- /dev/null
+++ b/src/libcryptobox/chacha20/chacha.h
@@ -0,0 +1,87 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015 Andrew Moon, Vsevolod Stakhov
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+
+#ifndef CHACHA_H_
+#define CHACHA_H_
+
+
+#define CHACHA_BLOCKBYTES 64
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct chacha_state_internal_t {
+ unsigned char s[48];
+ size_t rounds;
+ size_t leftover;
+ unsigned char buffer[CHACHA_BLOCKBYTES];
+} chacha_state_internal;
+
+typedef struct chacha_state_t {
+ unsigned char opaque[128];
+} chacha_state;
+
+typedef struct chacha_key_t {
+ unsigned char b[32];
+} chacha_key;
+
+typedef struct chacha_iv_t {
+ unsigned char b[8];
+} chacha_iv;
+
+typedef struct chacha_iv24_t {
+ unsigned char b[24];
+} chacha_iv24;
+
+void hchacha(const unsigned char key[32], const unsigned char iv[16],
+ unsigned char out[32], size_t rounds);
+
+void chacha_init(chacha_state *S, const chacha_key *key, const chacha_iv *iv,
+ size_t rounds);
+
+void xchacha_init(chacha_state *S, const chacha_key *key,
+ const chacha_iv24 *iv, size_t rounds);
+
+size_t chacha_update(chacha_state *S, const unsigned char *in,
+ unsigned char *out, size_t inlen);
+
+size_t chacha_final(chacha_state *S, unsigned char *out);
+
+void chacha(const chacha_key *key, const chacha_iv *iv,
+ const unsigned char *in, unsigned char *out, size_t inlen,
+ size_t rounds);
+
+void xchacha(const chacha_key *key, const chacha_iv24 *iv,
+ const unsigned char *in, unsigned char *out, size_t inlen,
+ size_t rounds);
+
+const char *chacha_load(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CHACHA_H_ */
diff --git a/src/libcryptobox/chacha20/constants.S b/src/libcryptobox/chacha20/constants.S
new file mode 100644
index 0000000..ff109a3
--- /dev/null
+++ b/src/libcryptobox/chacha20/constants.S
@@ -0,0 +1,6 @@
+SECTION_RODATA
+.p2align 4,,15
+chacha_constants:
+.long 0x61707865,0x3320646e,0x79622d32,0x6b206574 /* "expand 32-byte k" */
+.byte 2,3,0,1,6,7,4,5,10,11,8,9,14,15,12,13 /* pshufb rotate by 16 */
+.byte 3,0,1,2,7,4,5,6,11,8,9,10,15,12,13,14 /* pshufb rotate by 8 */
diff --git a/src/libcryptobox/chacha20/ref.c b/src/libcryptobox/chacha20/ref.c
new file mode 100644
index 0000000..ee646db
--- /dev/null
+++ b/src/libcryptobox/chacha20/ref.c
@@ -0,0 +1,272 @@
+#include "config.h"
+#include "chacha.h"
+#include "cryptobox.h"
+
+#if defined(HAVE_INT32)
+typedef uint32_t chacha_int32;
+#else
+typedef guint32 chacha_int32;
+#endif
+
+/* interpret four 8 bit unsigned integers as a 32 bit unsigned integer in little endian */
+static chacha_int32
+U8TO32(const unsigned char *p)
+{
+ return (((chacha_int32) (p[0])) |
+ ((chacha_int32) (p[1]) << 8) |
+ ((chacha_int32) (p[2]) << 16) |
+ ((chacha_int32) (p[3]) << 24));
+}
+
+/* store a 32 bit unsigned integer as four 8 bit unsigned integers in little endian */
+static void
+U32TO8(unsigned char *p, chacha_int32 v)
+{
+ p[0] = (v) &0xff;
+ p[1] = (v >> 8) & 0xff;
+ p[2] = (v >> 16) & 0xff;
+ p[3] = (v >> 24) & 0xff;
+}
+
+/* 32 bit left rotate */
+static chacha_int32
+ROTL32(chacha_int32 x, int k)
+{
+ return ((x << k) | (x >> (32 - k))) & 0xffffffff;
+}
+
+/* "expand 32-byte k", as 4 little endian 32-bit unsigned integers */
+static const chacha_int32 chacha_constants[4] = {
+ 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574};
+
+void chacha_blocks_ref(chacha_state_internal *state, const unsigned char *in, unsigned char *out, size_t bytes)
+{
+ chacha_int32 x[16], j[12];
+ chacha_int32 t;
+ unsigned char *ctarget = out, tmp[64];
+ size_t i, r;
+
+ if (!bytes) return;
+
+ j[0] = U8TO32(state->s + 0);
+ j[1] = U8TO32(state->s + 4);
+ j[2] = U8TO32(state->s + 8);
+ j[3] = U8TO32(state->s + 12);
+ j[4] = U8TO32(state->s + 16);
+ j[5] = U8TO32(state->s + 20);
+ j[6] = U8TO32(state->s + 24);
+ j[7] = U8TO32(state->s + 28);
+ j[8] = U8TO32(state->s + 32);
+ j[9] = U8TO32(state->s + 36);
+ j[10] = U8TO32(state->s + 40);
+ j[11] = U8TO32(state->s + 44);
+
+ r = state->rounds;
+
+ for (;;) {
+ if (bytes < 64) {
+ if (in) {
+ for (i = 0; i < bytes; i++) tmp[i] = in[i];
+ in = tmp;
+ }
+ ctarget = out;
+ out = tmp;
+ }
+
+ x[0] = chacha_constants[0];
+ x[1] = chacha_constants[1];
+ x[2] = chacha_constants[2];
+ x[3] = chacha_constants[3];
+ x[4] = j[0];
+ x[5] = j[1];
+ x[6] = j[2];
+ x[7] = j[3];
+ x[8] = j[4];
+ x[9] = j[5];
+ x[10] = j[6];
+ x[11] = j[7];
+ x[12] = j[8];
+ x[13] = j[9];
+ x[14] = j[10];
+ x[15] = j[11];
+
+#define quarter(a, b, c, d) \
+ a += b; \
+ t = d ^ a; \
+ d = ROTL32(t, 16); \
+ c += d; \
+ t = b ^ c; \
+ b = ROTL32(t, 12); \
+ a += b; \
+ t = d ^ a; \
+ d = ROTL32(t, 8); \
+ c += d; \
+ t = b ^ c; \
+ b = ROTL32(t, 7);
+
+#define doubleround() \
+ quarter(x[0], x[4], x[8], x[12]) \
+ quarter(x[1], x[5], x[9], x[13]) \
+ quarter(x[2], x[6], x[10], x[14]) \
+ quarter(x[3], x[7], x[11], x[15]) \
+ quarter(x[0], x[5], x[10], x[15]) \
+ quarter(x[1], x[6], x[11], x[12]) \
+ quarter(x[2], x[7], x[8], x[13]) \
+ quarter(x[3], x[4], x[9], x[14])
+
+ i = r;
+ do {
+ doubleround()
+ i -= 2;
+ } while (i);
+
+ x[0] += chacha_constants[0];
+ x[1] += chacha_constants[1];
+ x[2] += chacha_constants[2];
+ x[3] += chacha_constants[3];
+ x[4] += j[0];
+ x[5] += j[1];
+ x[6] += j[2];
+ x[7] += j[3];
+ x[8] += j[4];
+ x[9] += j[5];
+ x[10] += j[6];
+ x[11] += j[7];
+ x[12] += j[8];
+ x[13] += j[9];
+ x[14] += j[10];
+ x[15] += j[11];
+
+ if (in) {
+ U32TO8(out + 0, x[0] ^ U8TO32(in + 0));
+ U32TO8(out + 4, x[1] ^ U8TO32(in + 4));
+ U32TO8(out + 8, x[2] ^ U8TO32(in + 8));
+ U32TO8(out + 12, x[3] ^ U8TO32(in + 12));
+ U32TO8(out + 16, x[4] ^ U8TO32(in + 16));
+ U32TO8(out + 20, x[5] ^ U8TO32(in + 20));
+ U32TO8(out + 24, x[6] ^ U8TO32(in + 24));
+ U32TO8(out + 28, x[7] ^ U8TO32(in + 28));
+ U32TO8(out + 32, x[8] ^ U8TO32(in + 32));
+ U32TO8(out + 36, x[9] ^ U8TO32(in + 36));
+ U32TO8(out + 40, x[10] ^ U8TO32(in + 40));
+ U32TO8(out + 44, x[11] ^ U8TO32(in + 44));
+ U32TO8(out + 48, x[12] ^ U8TO32(in + 48));
+ U32TO8(out + 52, x[13] ^ U8TO32(in + 52));
+ U32TO8(out + 56, x[14] ^ U8TO32(in + 56));
+ U32TO8(out + 60, x[15] ^ U8TO32(in + 60));
+ in += 64;
+ }
+ else {
+ U32TO8(out + 0, x[0]);
+ U32TO8(out + 4, x[1]);
+ U32TO8(out + 8, x[2]);
+ U32TO8(out + 12, x[3]);
+ U32TO8(out + 16, x[4]);
+ U32TO8(out + 20, x[5]);
+ U32TO8(out + 24, x[6]);
+ U32TO8(out + 28, x[7]);
+ U32TO8(out + 32, x[8]);
+ U32TO8(out + 36, x[9]);
+ U32TO8(out + 40, x[10]);
+ U32TO8(out + 44, x[11]);
+ U32TO8(out + 48, x[12]);
+ U32TO8(out + 52, x[13]);
+ U32TO8(out + 56, x[14]);
+ U32TO8(out + 60, x[15]);
+ }
+
+ /* increment the 64 bit counter, split in to two 32 bit halves */
+ j[8]++;
+ if (!j[8])
+ j[9]++;
+
+ if (bytes <= 64) {
+ if (bytes < 64)
+ for (i = 0; i < bytes; i++) ctarget[i] = out[i];
+
+ /* store the counter back to the state */
+ U32TO8(state->s + 32, j[8]);
+ U32TO8(state->s + 36, j[9]);
+ goto cleanup;
+ }
+ bytes -= 64;
+ out += 64;
+ }
+
+cleanup:
+ rspamd_explicit_memzero(j, sizeof(j));
+}
+
+void hchacha_ref(const unsigned char key[32], const unsigned char iv[16], unsigned char out[32], size_t rounds)
+{
+ chacha_int32 x[16];
+ chacha_int32 t;
+
+ x[0] = chacha_constants[0];
+ x[1] = chacha_constants[1];
+ x[2] = chacha_constants[2];
+ x[3] = chacha_constants[3];
+ x[4] = U8TO32(key + 0);
+ x[5] = U8TO32(key + 4);
+ x[6] = U8TO32(key + 8);
+ x[7] = U8TO32(key + 12);
+ x[8] = U8TO32(key + 16);
+ x[9] = U8TO32(key + 20);
+ x[10] = U8TO32(key + 24);
+ x[11] = U8TO32(key + 28);
+ x[12] = U8TO32(iv + 0);
+ x[13] = U8TO32(iv + 4);
+ x[14] = U8TO32(iv + 8);
+ x[15] = U8TO32(iv + 12);
+
+ do {
+ doubleround()
+ rounds -= 2;
+ } while (rounds);
+
+ /* indices for the chacha constant */
+ U32TO8(out + 0, x[0]);
+ U32TO8(out + 4, x[1]);
+ U32TO8(out + 8, x[2]);
+ U32TO8(out + 12, x[3]);
+
+ /* indices for the iv */
+ U32TO8(out + 16, x[12]);
+ U32TO8(out + 20, x[13]);
+ U32TO8(out + 24, x[14]);
+ U32TO8(out + 28, x[15]);
+}
+
+void chacha_clear_state_ref(chacha_state_internal *state)
+{
+ rspamd_explicit_memzero(state, 48);
+}
+
+void chacha_ref(const chacha_key *key, const chacha_iv *iv, const unsigned char *in, unsigned char *out, size_t inlen, size_t rounds)
+{
+ chacha_state_internal state;
+ size_t i;
+ for (i = 0; i < 32; i++)
+ state.s[i + 0] = key->b[i];
+ for (i = 0; i < 8; i++)
+ state.s[i + 32] = 0;
+ for (i = 0; i < 8; i++)
+ state.s[i + 40] = iv->b[i];
+ state.rounds = rounds;
+ chacha_blocks_ref(&state, in, out, inlen);
+ chacha_clear_state_ref(&state);
+}
+
+void xchacha_ref(const chacha_key *key, const chacha_iv24 *iv, const unsigned char *in, unsigned char *out, size_t inlen, size_t rounds)
+{
+ chacha_state_internal state;
+ size_t i;
+ hchacha_ref(key->b, iv->b, &state.s[0], rounds);
+ for (i = 0; i < 8; i++)
+ state.s[i + 32] = 0;
+ for (i = 0; i < 8; i++)
+ state.s[i + 40] = iv->b[i + 16];
+ state.rounds = rounds;
+ chacha_blocks_ref(&state, in, out, inlen);
+ chacha_clear_state_ref(&state);
+}
diff --git a/src/libcryptobox/chacha20/sse2.S b/src/libcryptobox/chacha20/sse2.S
new file mode 100644
index 0000000..a91d095
--- /dev/null
+++ b/src/libcryptobox/chacha20/sse2.S
@@ -0,0 +1,734 @@
+#include "../macro.S"
+#include "constants.S"
+SECTION_TEXT
+
+GLOBAL_HIDDEN_FN chacha_blocks_sse2
+chacha_blocks_sse2_local:
+pushq %rbx
+pushq %rbp
+movq %rsp, %rbp
+andq $~63, %rsp
+subq $512, %rsp
+movq $0x3320646e61707865, %rax
+movq $0x6b20657479622d32, %r8
+movd %rax, %xmm8
+movd %r8, %xmm14
+punpcklqdq %xmm14, %xmm8
+movdqu 0(%rdi), %xmm9
+movdqu 16(%rdi), %xmm10
+movdqu 32(%rdi), %xmm11
+movq 48(%rdi), %rax
+movq $1, %r9
+movdqa %xmm8, 0(%rsp)
+movdqa %xmm9, 16(%rsp)
+movdqa %xmm10, 32(%rsp)
+movdqa %xmm11, 48(%rsp)
+movq %rax, 64(%rsp)
+cmpq $256, %rcx
+jb chacha_blocks_sse2_below256
+pshufd $0x00, %xmm8, %xmm0
+pshufd $0x55, %xmm8, %xmm1
+pshufd $0xaa, %xmm8, %xmm2
+pshufd $0xff, %xmm8, %xmm3
+movdqa %xmm0, 128(%rsp)
+movdqa %xmm1, 144(%rsp)
+movdqa %xmm2, 160(%rsp)
+movdqa %xmm3, 176(%rsp)
+pshufd $0x00, %xmm9, %xmm0
+pshufd $0x55, %xmm9, %xmm1
+pshufd $0xaa, %xmm9, %xmm2
+pshufd $0xff, %xmm9, %xmm3
+movdqa %xmm0, 192(%rsp)
+movdqa %xmm1, 208(%rsp)
+movdqa %xmm2, 224(%rsp)
+movdqa %xmm3, 240(%rsp)
+pshufd $0x00, %xmm10, %xmm0
+pshufd $0x55, %xmm10, %xmm1
+pshufd $0xaa, %xmm10, %xmm2
+pshufd $0xff, %xmm10, %xmm3
+movdqa %xmm0, 256(%rsp)
+movdqa %xmm1, 272(%rsp)
+movdqa %xmm2, 288(%rsp)
+movdqa %xmm3, 304(%rsp)
+pshufd $0xaa, %xmm11, %xmm0
+pshufd $0xff, %xmm11, %xmm1
+movdqa %xmm0, 352(%rsp)
+movdqa %xmm1, 368(%rsp)
+jmp chacha_blocks_sse2_atleast256
+.p2align 6,,63
+chacha_blocks_sse2_atleast256:
+movq 48(%rsp), %rax
+leaq 1(%rax), %r8
+leaq 2(%rax), %r9
+leaq 3(%rax), %r10
+leaq 4(%rax), %rbx
+movl %eax, 320(%rsp)
+movl %r8d, 4+320(%rsp)
+movl %r9d, 8+320(%rsp)
+movl %r10d, 12+320(%rsp)
+shrq $32, %rax
+shrq $32, %r8
+shrq $32, %r9
+shrq $32, %r10
+movl %eax, 336(%rsp)
+movl %r8d, 4+336(%rsp)
+movl %r9d, 8+336(%rsp)
+movl %r10d, 12+336(%rsp)
+movq %rbx, 48(%rsp)
+movq 64(%rsp), %rax
+movdqa 128(%rsp), %xmm0
+movdqa 144(%rsp), %xmm1
+movdqa 160(%rsp), %xmm2
+movdqa 176(%rsp), %xmm3
+movdqa 192(%rsp), %xmm4
+movdqa 208(%rsp), %xmm5
+movdqa 224(%rsp), %xmm6
+movdqa 240(%rsp), %xmm7
+movdqa 256(%rsp), %xmm8
+movdqa 272(%rsp), %xmm9
+movdqa 288(%rsp), %xmm10
+movdqa 304(%rsp), %xmm11
+movdqa 320(%rsp), %xmm12
+movdqa 336(%rsp), %xmm13
+movdqa 352(%rsp), %xmm14
+movdqa 368(%rsp), %xmm15
+chacha_blocks_sse2_mainloop1:
+paddd %xmm4, %xmm0
+paddd %xmm5, %xmm1
+pxor %xmm0, %xmm12
+pxor %xmm1, %xmm13
+paddd %xmm6, %xmm2
+paddd %xmm7, %xmm3
+movdqa %xmm6, 96(%rsp)
+pxor %xmm2, %xmm14
+pxor %xmm3, %xmm15
+pshuflw $0xb1,%xmm12,%xmm12
+pshufhw $0xb1,%xmm12,%xmm12
+pshuflw $0xb1,%xmm13,%xmm13
+pshufhw $0xb1,%xmm13,%xmm13
+pshuflw $0xb1,%xmm14,%xmm14
+pshufhw $0xb1,%xmm14,%xmm14
+pshuflw $0xb1,%xmm15,%xmm15
+pshufhw $0xb1,%xmm15,%xmm15
+paddd %xmm12, %xmm8
+paddd %xmm13, %xmm9
+paddd %xmm14, %xmm10
+paddd %xmm15, %xmm11
+movdqa %xmm12, 112(%rsp)
+pxor %xmm8, %xmm4
+pxor %xmm9, %xmm5
+movdqa 96(%rsp), %xmm6
+movdqa %xmm4, %xmm12
+pslld $ 12, %xmm4
+psrld $20, %xmm12
+pxor %xmm12, %xmm4
+movdqa %xmm5, %xmm12
+pslld $ 12, %xmm5
+psrld $20, %xmm12
+pxor %xmm12, %xmm5
+pxor %xmm10, %xmm6
+pxor %xmm11, %xmm7
+movdqa %xmm6, %xmm12
+pslld $ 12, %xmm6
+psrld $20, %xmm12
+pxor %xmm12, %xmm6
+movdqa %xmm7, %xmm12
+pslld $ 12, %xmm7
+psrld $20, %xmm12
+pxor %xmm12, %xmm7
+movdqa 112(%rsp), %xmm12
+paddd %xmm4, %xmm0
+paddd %xmm5, %xmm1
+pxor %xmm0, %xmm12
+pxor %xmm1, %xmm13
+paddd %xmm6, %xmm2
+paddd %xmm7, %xmm3
+movdqa %xmm6, 96(%rsp)
+pxor %xmm2, %xmm14
+pxor %xmm3, %xmm15
+movdqa %xmm12, %xmm6
+pslld $ 8, %xmm12
+psrld $24, %xmm6
+pxor %xmm6, %xmm12
+movdqa %xmm13, %xmm6
+pslld $ 8, %xmm13
+psrld $24, %xmm6
+pxor %xmm6, %xmm13
+paddd %xmm12, %xmm8
+paddd %xmm13, %xmm9
+movdqa %xmm14, %xmm6
+pslld $ 8, %xmm14
+psrld $24, %xmm6
+pxor %xmm6, %xmm14
+movdqa %xmm15, %xmm6
+pslld $ 8, %xmm15
+psrld $24, %xmm6
+pxor %xmm6, %xmm15
+paddd %xmm14, %xmm10
+paddd %xmm15, %xmm11
+movdqa %xmm12, 112(%rsp)
+pxor %xmm8, %xmm4
+pxor %xmm9, %xmm5
+movdqa 96(%rsp), %xmm6
+movdqa %xmm4, %xmm12
+pslld $ 7, %xmm4
+psrld $25, %xmm12
+pxor %xmm12, %xmm4
+movdqa %xmm5, %xmm12
+pslld $ 7, %xmm5
+psrld $25, %xmm12
+pxor %xmm12, %xmm5
+pxor %xmm10, %xmm6
+pxor %xmm11, %xmm7
+movdqa %xmm6, %xmm12
+pslld $ 7, %xmm6
+psrld $25, %xmm12
+pxor %xmm12, %xmm6
+movdqa %xmm7, %xmm12
+pslld $ 7, %xmm7
+psrld $25, %xmm12
+pxor %xmm12, %xmm7
+movdqa 112(%rsp), %xmm12
+paddd %xmm5, %xmm0
+paddd %xmm6, %xmm1
+pxor %xmm0, %xmm15
+pxor %xmm1, %xmm12
+paddd %xmm7, %xmm2
+paddd %xmm4, %xmm3
+movdqa %xmm7, 96(%rsp)
+pxor %xmm2, %xmm13
+pxor %xmm3, %xmm14
+pshuflw $0xb1,%xmm15,%xmm15
+pshufhw $0xb1,%xmm15,%xmm15
+pshuflw $0xb1,%xmm12,%xmm12
+pshufhw $0xb1,%xmm12,%xmm12
+pshuflw $0xb1,%xmm13,%xmm13
+pshufhw $0xb1,%xmm13,%xmm13
+pshuflw $0xb1,%xmm14,%xmm14
+pshufhw $0xb1,%xmm14,%xmm14
+paddd %xmm15, %xmm10
+paddd %xmm12, %xmm11
+paddd %xmm13, %xmm8
+paddd %xmm14, %xmm9
+movdqa %xmm15, 112(%rsp)
+pxor %xmm10, %xmm5
+pxor %xmm11, %xmm6
+movdqa 96(%rsp), %xmm7
+movdqa %xmm5, %xmm15
+pslld $ 12, %xmm5
+psrld $20, %xmm15
+pxor %xmm15, %xmm5
+movdqa %xmm6, %xmm15
+pslld $ 12, %xmm6
+psrld $20, %xmm15
+pxor %xmm15, %xmm6
+pxor %xmm8, %xmm7
+pxor %xmm9, %xmm4
+movdqa %xmm7, %xmm15
+pslld $ 12, %xmm7
+psrld $20, %xmm15
+pxor %xmm15, %xmm7
+movdqa %xmm4, %xmm15
+pslld $ 12, %xmm4
+psrld $20, %xmm15
+pxor %xmm15, %xmm4
+movdqa 112(%rsp), %xmm15
+paddd %xmm5, %xmm0
+paddd %xmm6, %xmm1
+pxor %xmm0, %xmm15
+pxor %xmm1, %xmm12
+paddd %xmm7, %xmm2
+paddd %xmm4, %xmm3
+movdqa %xmm7, 96(%rsp)
+pxor %xmm2, %xmm13
+pxor %xmm3, %xmm14
+movdqa %xmm15, %xmm7
+pslld $ 8, %xmm15
+psrld $24, %xmm7
+pxor %xmm7, %xmm15
+movdqa %xmm12, %xmm7
+pslld $ 8, %xmm12
+psrld $24, %xmm7
+pxor %xmm7, %xmm12
+paddd %xmm15, %xmm10
+paddd %xmm12, %xmm11
+movdqa %xmm13, %xmm7
+pslld $ 8, %xmm13
+psrld $24, %xmm7
+pxor %xmm7, %xmm13
+movdqa %xmm14, %xmm7
+pslld $ 8, %xmm14
+psrld $24, %xmm7
+pxor %xmm7, %xmm14
+paddd %xmm13, %xmm8
+paddd %xmm14, %xmm9
+movdqa %xmm15, 112(%rsp)
+pxor %xmm10, %xmm5
+pxor %xmm11, %xmm6
+movdqa 96(%rsp), %xmm7
+movdqa %xmm5, %xmm15
+pslld $ 7, %xmm5
+psrld $25, %xmm15
+pxor %xmm15, %xmm5
+movdqa %xmm6, %xmm15
+pslld $ 7, %xmm6
+psrld $25, %xmm15
+pxor %xmm15, %xmm6
+pxor %xmm8, %xmm7
+pxor %xmm9, %xmm4
+movdqa %xmm7, %xmm15
+pslld $ 7, %xmm7
+psrld $25, %xmm15
+pxor %xmm15, %xmm7
+movdqa %xmm4, %xmm15
+pslld $ 7, %xmm4
+psrld $25, %xmm15
+pxor %xmm15, %xmm4
+movdqa 112(%rsp), %xmm15
+subq $2, %rax
+jnz chacha_blocks_sse2_mainloop1
+paddd 128(%rsp), %xmm0
+paddd 144(%rsp), %xmm1
+paddd 160(%rsp), %xmm2
+paddd 176(%rsp), %xmm3
+paddd 192(%rsp), %xmm4
+paddd 208(%rsp), %xmm5
+paddd 224(%rsp), %xmm6
+paddd 240(%rsp), %xmm7
+paddd 256(%rsp), %xmm8
+paddd 272(%rsp), %xmm9
+paddd 288(%rsp), %xmm10
+paddd 304(%rsp), %xmm11
+paddd 320(%rsp), %xmm12
+paddd 336(%rsp), %xmm13
+paddd 352(%rsp), %xmm14
+paddd 368(%rsp), %xmm15
+movdqa %xmm8, 384(%rsp)
+movdqa %xmm9, 400(%rsp)
+movdqa %xmm10, 416(%rsp)
+movdqa %xmm11, 432(%rsp)
+movdqa %xmm12, 448(%rsp)
+movdqa %xmm13, 464(%rsp)
+movdqa %xmm14, 480(%rsp)
+movdqa %xmm15, 496(%rsp)
+movdqa %xmm0, %xmm8
+movdqa %xmm2, %xmm9
+movdqa %xmm4, %xmm10
+movdqa %xmm6, %xmm11
+punpckhdq %xmm1, %xmm0
+punpckhdq %xmm3, %xmm2
+punpckhdq %xmm5, %xmm4
+punpckhdq %xmm7, %xmm6
+punpckldq %xmm1, %xmm8
+punpckldq %xmm3, %xmm9
+punpckldq %xmm5, %xmm10
+punpckldq %xmm7, %xmm11
+movdqa %xmm0, %xmm1
+movdqa %xmm4, %xmm3
+movdqa %xmm8, %xmm5
+movdqa %xmm10, %xmm7
+punpckhqdq %xmm2, %xmm0
+punpckhqdq %xmm6, %xmm4
+punpckhqdq %xmm9, %xmm8
+punpckhqdq %xmm11, %xmm10
+punpcklqdq %xmm2, %xmm1
+punpcklqdq %xmm6, %xmm3
+punpcklqdq %xmm9, %xmm5
+punpcklqdq %xmm11, %xmm7
+andq %rsi, %rsi
+jz chacha_blocks_sse2_noinput1
+movdqu 0(%rsi), %xmm2
+movdqu 16(%rsi), %xmm6
+movdqu 64(%rsi), %xmm9
+movdqu 80(%rsi), %xmm11
+movdqu 128(%rsi), %xmm12
+movdqu 144(%rsi), %xmm13
+movdqu 192(%rsi), %xmm14
+movdqu 208(%rsi), %xmm15
+pxor %xmm2, %xmm5
+pxor %xmm6, %xmm7
+pxor %xmm9, %xmm8
+pxor %xmm11, %xmm10
+pxor %xmm12, %xmm1
+pxor %xmm13, %xmm3
+pxor %xmm14, %xmm0
+pxor %xmm15, %xmm4
+movdqu %xmm5, 0(%rdx)
+movdqu %xmm7, 16(%rdx)
+movdqu %xmm8, 64(%rdx)
+movdqu %xmm10, 80(%rdx)
+movdqu %xmm1, 128(%rdx)
+movdqu %xmm3, 144(%rdx)
+movdqu %xmm0, 192(%rdx)
+movdqu %xmm4, 208(%rdx)
+movdqa 384(%rsp), %xmm0
+movdqa 400(%rsp), %xmm1
+movdqa 416(%rsp), %xmm2
+movdqa 432(%rsp), %xmm3
+movdqa 448(%rsp), %xmm4
+movdqa 464(%rsp), %xmm5
+movdqa 480(%rsp), %xmm6
+movdqa 496(%rsp), %xmm7
+movdqa %xmm0, %xmm8
+movdqa %xmm2, %xmm9
+movdqa %xmm4, %xmm10
+movdqa %xmm6, %xmm11
+punpckldq %xmm1, %xmm8
+punpckldq %xmm3, %xmm9
+punpckhdq %xmm1, %xmm0
+punpckhdq %xmm3, %xmm2
+punpckldq %xmm5, %xmm10
+punpckldq %xmm7, %xmm11
+punpckhdq %xmm5, %xmm4
+punpckhdq %xmm7, %xmm6
+movdqa %xmm8, %xmm1
+movdqa %xmm0, %xmm3
+movdqa %xmm10, %xmm5
+movdqa %xmm4, %xmm7
+punpcklqdq %xmm9, %xmm1
+punpcklqdq %xmm11, %xmm5
+punpckhqdq %xmm9, %xmm8
+punpckhqdq %xmm11, %xmm10
+punpcklqdq %xmm2, %xmm3
+punpcklqdq %xmm6, %xmm7
+punpckhqdq %xmm2, %xmm0
+punpckhqdq %xmm6, %xmm4
+movdqu 32(%rsi), %xmm2
+movdqu 48(%rsi), %xmm6
+movdqu 96(%rsi), %xmm9
+movdqu 112(%rsi), %xmm11
+movdqu 160(%rsi), %xmm12
+movdqu 176(%rsi), %xmm13
+movdqu 224(%rsi), %xmm14
+movdqu 240(%rsi), %xmm15
+pxor %xmm2, %xmm1
+pxor %xmm6, %xmm5
+pxor %xmm9, %xmm8
+pxor %xmm11, %xmm10
+pxor %xmm12, %xmm3
+pxor %xmm13, %xmm7
+pxor %xmm14, %xmm0
+pxor %xmm15, %xmm4
+movdqu %xmm1, 32(%rdx)
+movdqu %xmm5, 48(%rdx)
+movdqu %xmm8, 96(%rdx)
+movdqu %xmm10, 112(%rdx)
+movdqu %xmm3, 160(%rdx)
+movdqu %xmm7, 176(%rdx)
+movdqu %xmm0, 224(%rdx)
+movdqu %xmm4, 240(%rdx)
+addq $256, %rsi
+jmp chacha_blocks_sse2_mainloop_cont
+chacha_blocks_sse2_noinput1:
+movdqu %xmm5, 0(%rdx)
+movdqu %xmm7, 16(%rdx)
+movdqu %xmm8, 64(%rdx)
+movdqu %xmm10, 80(%rdx)
+movdqu %xmm1, 128(%rdx)
+movdqu %xmm3, 144(%rdx)
+movdqu %xmm0, 192(%rdx)
+movdqu %xmm4, 208(%rdx)
+movdqa 384(%rsp), %xmm0
+movdqa 400(%rsp), %xmm1
+movdqa 416(%rsp), %xmm2
+movdqa 432(%rsp), %xmm3
+movdqa 448(%rsp), %xmm4
+movdqa 464(%rsp), %xmm5
+movdqa 480(%rsp), %xmm6
+movdqa 496(%rsp), %xmm7
+movdqa %xmm0, %xmm8
+movdqa %xmm2, %xmm9
+movdqa %xmm4, %xmm10
+movdqa %xmm6, %xmm11
+punpckldq %xmm1, %xmm8
+punpckldq %xmm3, %xmm9
+punpckhdq %xmm1, %xmm0
+punpckhdq %xmm3, %xmm2
+punpckldq %xmm5, %xmm10
+punpckldq %xmm7, %xmm11
+punpckhdq %xmm5, %xmm4
+punpckhdq %xmm7, %xmm6
+movdqa %xmm8, %xmm1
+movdqa %xmm0, %xmm3
+movdqa %xmm10, %xmm5
+movdqa %xmm4, %xmm7
+punpcklqdq %xmm9, %xmm1
+punpcklqdq %xmm11, %xmm5
+punpckhqdq %xmm9, %xmm8
+punpckhqdq %xmm11, %xmm10
+punpcklqdq %xmm2, %xmm3
+punpcklqdq %xmm6, %xmm7
+punpckhqdq %xmm2, %xmm0
+punpckhqdq %xmm6, %xmm4
+movdqu %xmm1, 32(%rdx)
+movdqu %xmm5, 48(%rdx)
+movdqu %xmm8, 96(%rdx)
+movdqu %xmm10, 112(%rdx)
+movdqu %xmm3, 160(%rdx)
+movdqu %xmm7, 176(%rdx)
+movdqu %xmm0, 224(%rdx)
+movdqu %xmm4, 240(%rdx)
+chacha_blocks_sse2_mainloop_cont:
+addq $256, %rdx
+subq $256, %rcx
+cmp $256, %rcx
+jae chacha_blocks_sse2_atleast256
+movdqa 0(%rsp), %xmm8
+movdqa 16(%rsp), %xmm9
+movdqa 32(%rsp), %xmm10
+movdqa 48(%rsp), %xmm11
+movq $1, %r9
+chacha_blocks_sse2_below256:
+movq %r9, %xmm5
+andq %rcx, %rcx
+jz chacha_blocks_sse2_done
+cmpq $64, %rcx
+jae chacha_blocks_sse2_above63
+movq %rdx, %r9
+andq %rsi, %rsi
+jz chacha_blocks_sse2_noinput2
+movq %rcx, %r10
+movq %rsp, %rdx
+addq %r10, %rsi
+addq %r10, %rdx
+negq %r10
+chacha_blocks_sse2_copyinput:
+movb (%rsi, %r10), %al
+movb %al, (%rdx, %r10)
+incq %r10
+jnz chacha_blocks_sse2_copyinput
+movq %rsp, %rsi
+chacha_blocks_sse2_noinput2:
+movq %rsp, %rdx
+chacha_blocks_sse2_above63:
+movdqa %xmm8, %xmm0
+movdqa %xmm9, %xmm1
+movdqa %xmm10, %xmm2
+movdqa %xmm11, %xmm3
+movq 64(%rsp), %rax
+chacha_blocks_sse2_mainloop2:
+paddd %xmm1, %xmm0
+pxor %xmm0, %xmm3
+pshuflw $0xb1,%xmm3,%xmm3
+pshufhw $0xb1,%xmm3,%xmm3
+paddd %xmm3, %xmm2
+pxor %xmm2, %xmm1
+movdqa %xmm1,%xmm4
+pslld $12, %xmm1
+psrld $20, %xmm4
+pxor %xmm4, %xmm1
+paddd %xmm1, %xmm0
+pxor %xmm0, %xmm3
+movdqa %xmm3,%xmm4
+pslld $8, %xmm3
+psrld $24, %xmm4
+pshufd $0x93,%xmm0,%xmm0
+pxor %xmm4, %xmm3
+paddd %xmm3, %xmm2
+pshufd $0x4e,%xmm3,%xmm3
+pxor %xmm2, %xmm1
+pshufd $0x39,%xmm2,%xmm2
+movdqa %xmm1,%xmm4
+pslld $7, %xmm1
+psrld $25, %xmm4
+pxor %xmm4, %xmm1
+subq $2, %rax
+paddd %xmm1, %xmm0
+pxor %xmm0, %xmm3
+pshuflw $0xb1,%xmm3,%xmm3
+pshufhw $0xb1,%xmm3,%xmm3
+paddd %xmm3, %xmm2
+pxor %xmm2, %xmm1
+movdqa %xmm1,%xmm4
+pslld $12, %xmm1
+psrld $20, %xmm4
+pxor %xmm4, %xmm1
+paddd %xmm1, %xmm0
+pxor %xmm0, %xmm3
+movdqa %xmm3,%xmm4
+pslld $8, %xmm3
+psrld $24, %xmm4
+pshufd $0x39,%xmm0,%xmm0
+pxor %xmm4, %xmm3
+paddd %xmm3, %xmm2
+pshufd $0x4e,%xmm3,%xmm3
+pxor %xmm2, %xmm1
+pshufd $0x93,%xmm2,%xmm2
+movdqa %xmm1,%xmm4
+pslld $7, %xmm1
+psrld $25, %xmm4
+pxor %xmm4, %xmm1
+jnz chacha_blocks_sse2_mainloop2
+paddd %xmm8, %xmm0
+paddd %xmm9, %xmm1
+paddd %xmm10, %xmm2
+paddd %xmm11, %xmm3
+andq %rsi, %rsi
+jz chacha_blocks_sse2_noinput3
+movdqu 0(%rsi), %xmm12
+movdqu 16(%rsi), %xmm13
+movdqu 32(%rsi), %xmm14
+movdqu 48(%rsi), %xmm15
+pxor %xmm12, %xmm0
+pxor %xmm13, %xmm1
+pxor %xmm14, %xmm2
+pxor %xmm15, %xmm3
+addq $64, %rsi
+chacha_blocks_sse2_noinput3:
+movdqu %xmm0, 0(%rdx)
+movdqu %xmm1, 16(%rdx)
+movdqu %xmm2, 32(%rdx)
+movdqu %xmm3, 48(%rdx)
+paddq %xmm5, %xmm11
+cmpq $64, %rcx
+jbe chacha_blocks_sse2_mainloop2_finishup
+addq $64, %rdx
+subq $64, %rcx
+jmp chacha_blocks_sse2_below256
+chacha_blocks_sse2_mainloop2_finishup:
+cmpq $64, %rcx
+je chacha_blocks_sse2_done
+addq %rcx, %r9
+addq %rcx, %rdx
+negq %rcx
+chacha_blocks_sse2_copyoutput:
+movb (%rdx, %rcx), %al
+movb %al, (%r9, %rcx)
+incq %rcx
+jnz chacha_blocks_sse2_copyoutput
+chacha_blocks_sse2_done:
+movdqu %xmm11, 32(%rdi)
+movq %rbp, %rsp
+popq %rbp
+popq %rbx
+ret
+FN_END chacha_blocks_sse2
+
+GLOBAL_HIDDEN_FN hchacha_sse2
+hchacha_sse2_local:
+movq $0x3320646e61707865, %rax
+movq $0x6b20657479622d32, %r8
+movd %rax, %xmm0
+movd %r8, %xmm4
+punpcklqdq %xmm4, %xmm0
+movdqu 0(%rdi), %xmm1
+movdqu 16(%rdi), %xmm2
+movdqu 0(%rsi), %xmm3
+hchacha_sse2_mainloop:
+paddd %xmm1, %xmm0
+pxor %xmm0, %xmm3
+pshuflw $0xb1,%xmm3,%xmm3
+pshufhw $0xb1,%xmm3,%xmm3
+paddd %xmm3, %xmm2
+pxor %xmm2, %xmm1
+movdqa %xmm1,%xmm4
+pslld $12, %xmm1
+psrld $20, %xmm4
+pxor %xmm4, %xmm1
+paddd %xmm1, %xmm0
+pxor %xmm0, %xmm3
+movdqa %xmm3,%xmm4
+pslld $8, %xmm3
+psrld $24, %xmm4
+pshufd $0x93,%xmm0,%xmm0
+pxor %xmm4, %xmm3
+paddd %xmm3, %xmm2
+pshufd $0x4e,%xmm3,%xmm3
+pxor %xmm2, %xmm1
+pshufd $0x39,%xmm2,%xmm2
+movdqa %xmm1,%xmm4
+pslld $7, %xmm1
+psrld $25, %xmm4
+pxor %xmm4, %xmm1
+subq $2, %rcx
+paddd %xmm1, %xmm0
+pxor %xmm0, %xmm3
+pshuflw $0xb1,%xmm3,%xmm3
+pshufhw $0xb1,%xmm3,%xmm3
+paddd %xmm3, %xmm2
+pxor %xmm2, %xmm1
+movdqa %xmm1,%xmm4
+pslld $12, %xmm1
+psrld $20, %xmm4
+pxor %xmm4, %xmm1
+paddd %xmm1, %xmm0
+pxor %xmm0, %xmm3
+movdqa %xmm3,%xmm4
+pslld $8, %xmm3
+psrld $24, %xmm4
+pshufd $0x39,%xmm0,%xmm0
+pxor %xmm4, %xmm3
+paddd %xmm3, %xmm2
+pshufd $0x4e,%xmm3,%xmm3
+pxor %xmm2, %xmm1
+pshufd $0x93,%xmm2,%xmm2
+movdqa %xmm1,%xmm4
+pslld $7, %xmm1
+psrld $25, %xmm4
+pxor %xmm4, %xmm1
+ja hchacha_sse2_mainloop
+movdqu %xmm0, 0(%rdx)
+movdqu %xmm3, 16(%rdx)
+ret
+FN_END hchacha_sse2
+
+GLOBAL_HIDDEN_FN_EXT chacha_sse2, 6, 16
+pushq %rbp
+movq %rsp, %rbp
+subq $64, %rsp
+andq $~63, %rsp
+movdqu 0(%rdi), %xmm0
+movdqu 16(%rdi), %xmm1
+movdqa %xmm0, 0(%rsp)
+movdqa %xmm1, 16(%rsp)
+xorq %rdi, %rdi
+movq %rdi, 32(%rsp)
+movq 0(%rsi), %rsi
+movq %rsi, 40(%rsp)
+movq %r9, 48(%rsp)
+movq %rsp, %rdi
+movq %rdx, %rsi
+movq %rcx, %rdx
+movq %r8, %rcx
+call chacha_blocks_sse2_local
+pxor %xmm0, %xmm0
+movdqa %xmm0, 0(%rsp)
+movdqa %xmm0, 16(%rsp)
+movdqa %xmm0, 32(%rsp)
+movq %rbp, %rsp
+popq %rbp
+ret
+FN_END chacha_sse2
+
+GLOBAL_HIDDEN_FN_EXT xchacha_sse2, 6, 16
+pushq %rbp
+pushq %rbx
+movq %rsp, %rbp
+subq $64, %rsp
+andq $~63, %rsp
+movq %rsp, %rbx
+xorq %rax, %rax
+movq %rax, 32(%rbx)
+movq 16(%rsi), %rax
+movq %rax, 40(%rbx)
+movq %r9, 48(%rbx)
+pushq %rdx
+pushq %rcx
+pushq %r8
+movq %rbx, %rdx
+movq %r9, %rcx
+call hchacha_sse2_local
+movq %rbx, %rdi
+popq %rcx
+popq %rdx
+popq %rsi
+call chacha_blocks_sse2_local
+pxor %xmm0, %xmm0
+movdqa %xmm0, 0(%rbx)
+movdqa %xmm0, 16(%rbx)
+movdqa %xmm0, 32(%rbx)
+movq %rbp, %rsp
+popq %rbx
+popq %rbp
+ret
+FN_END xchacha_sse2
diff --git a/src/libcryptobox/cryptobox.c b/src/libcryptobox/cryptobox.c
new file mode 100644
index 0000000..e118c4a
--- /dev/null
+++ b/src/libcryptobox/cryptobox.c
@@ -0,0 +1,1778 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Workaround for memset_s */
+#ifdef __APPLE__
+#define __STDC_WANT_LIB_EXT1__ 1
+#include <string.h>
+#endif
+
+#include "config.h"
+#include "cryptobox.h"
+#include "platform_config.h"
+#include "chacha20/chacha.h"
+#include "catena/catena.h"
+#include "base64/base64.h"
+#include "ottery.h"
+#include "printf.h"
+#define XXH_INLINE_ALL
+#define XXH_PRIVATE_API
+#include "xxhash.h"
+#define MUM_TARGET_INDEPENDENT_HASH 1 /* For 32/64 bit equal hashes */
+#include "../../contrib/mumhash/mum.h"
+#include "../../contrib/t1ha/t1ha.h"
+#ifdef HAVE_CPUID_H
+#include <cpuid.h>
+#endif
+#ifdef HAVE_OPENSSL
+#include <openssl/opensslv.h>
+/* Openssl >= 1.0.1d is required for GCM verification */
+#if OPENSSL_VERSION_NUMBER >= 0x1000104fL
+#define HAVE_USABLE_OPENSSL 1
+#endif
+#endif
+
+#ifdef HAVE_USABLE_OPENSSL
+#include <openssl/evp.h>
+#include <openssl/ec.h>
+#include <openssl/ecdh.h>
+#include <openssl/ecdsa.h>
+#include <openssl/rand.h>
+#define CRYPTOBOX_CURVE_NID NID_X9_62_prime256v1
+#endif
+
+#include <signal.h>
+#include <setjmp.h>
+#include <stdalign.h>
+
+#include <sodium.h>
+
+unsigned cpu_config = 0;
+
+static gboolean cryptobox_loaded = FALSE;
+
+static const guchar n0[16] = {0};
+
+#define CRYPTOBOX_ALIGNMENT 16
+#define cryptobox_align_ptr(p, a) \
+ (void *) (((uintptr_t) (p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1))
+
+static void
+rspamd_cryptobox_cpuid(gint cpu[4], gint info)
+{
+ guint32 __attribute__((unused)) eax, __attribute__((unused)) ecx = 0, __attribute__((unused)) ebx = 0, __attribute__((unused)) edx = 0;
+
+ eax = info;
+#if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
+#if defined(__i386__) && defined(__PIC__)
+
+ /* in case of PIC under 32-bit EBX cannot be clobbered */
+
+ __asm__ volatile("movl %%ebx, %%edi \n\t cpuid \n\t xchgl %%ebx, %%edi"
+ : "=D"(ebx),
+ "+a"(eax), "+c"(ecx), "=d"(edx));
+#else
+ __asm__ volatile("cpuid"
+ : "+b"(ebx), "+a"(eax), "+c"(ecx), "=d"(edx));
+#endif
+
+ cpu[0] = eax;
+ cpu[1] = ebx;
+ cpu[2] = ecx;
+ cpu[3] = edx;
+#else
+ memset(cpu, 0, sizeof(gint) * 4);
+#endif
+}
+
+static sig_atomic_t ok = 0;
+static jmp_buf j;
+
+__attribute__((noreturn)) static void
+rspamd_cryptobox_ill_handler(int signo)
+{
+ ok = 0;
+ longjmp(j, -1);
+}
+
+static gboolean
+rspamd_cryptobox_test_instr(gint instr)
+{
+ void (*old_handler)(int);
+ guint32 rd;
+
+#if defined(__GNUC__)
+ ok = 1;
+ old_handler = signal(SIGILL, rspamd_cryptobox_ill_handler);
+
+ if (setjmp(j) != 0) {
+ signal(SIGILL, old_handler);
+
+ return FALSE;
+ }
+
+ switch (instr) {
+#if defined HAVE_SSE2 && defined(__x86_64__)
+ case CPUID_SSE2:
+ __asm__ volatile("psubb %xmm0, %xmm0");
+ break;
+ case CPUID_RDRAND:
+ /* Use byte code here for compatibility */
+ __asm__ volatile(".byte 0x0f,0xc7,0xf0; setc %1"
+ : "=a"(rd), "=qm"(ok)
+ :
+ : "edx");
+ break;
+#endif
+#ifdef HAVE_SSE3
+ case CPUID_SSE3:
+ __asm__ volatile("movshdup %xmm0, %xmm0");
+ break;
+#endif
+#ifdef HAVE_SSSE3
+ case CPUID_SSSE3:
+ __asm__ volatile("pshufb %xmm0, %xmm0");
+ break;
+#endif
+#ifdef HAVE_SSE41
+ case CPUID_SSE41:
+ __asm__ volatile("pcmpeqq %xmm0, %xmm0");
+ break;
+#endif
+#if defined HAVE_SSE42 && defined(__x86_64__)
+ case CPUID_SSE42:
+ __asm__ volatile("pushq %rax\n"
+ "xorq %rax, %rax\n"
+ "crc32 %rax, %rax\n"
+ "popq %rax");
+ break;
+#endif
+#ifdef HAVE_AVX
+ case CPUID_AVX:
+ __asm__ volatile("vpaddq %xmm0, %xmm0, %xmm0");
+ break;
+#endif
+#ifdef HAVE_AVX2
+ case CPUID_AVX2:
+ __asm__ volatile("vpaddq %ymm0, %ymm0, %ymm0");
+ break;
+#endif
+ default:
+ return FALSE;
+ break;
+ }
+
+ signal(SIGILL, old_handler);
+#endif
+
+ (void) rd; /* Silence warning */
+
+ /* We actually never return here if SIGILL has been caught */
+ return ok == 1;
+}
+
+struct rspamd_cryptobox_library_ctx *
+rspamd_cryptobox_init(void)
+{
+ gint cpu[4], nid;
+ const guint32 osxsave_mask = (1 << 27);
+ const guint32 fma_movbe_osxsave_mask = ((1 << 12) | (1 << 22) | (1 << 27));
+ const guint32 avx2_bmi12_mask = (1 << 5) | (1 << 3) | (1 << 8);
+ gulong bit;
+ static struct rspamd_cryptobox_library_ctx *ctx;
+ GString *buf;
+
+ if (cryptobox_loaded) {
+ /* Ignore reload attempts */
+ return ctx;
+ }
+
+ cryptobox_loaded = TRUE;
+ ctx = g_malloc0(sizeof(*ctx));
+
+ rspamd_cryptobox_cpuid(cpu, 0);
+ nid = cpu[0];
+ rspamd_cryptobox_cpuid(cpu, 1);
+
+ if (nid > 1) {
+ if ((cpu[3] & ((guint32) 1 << 26))) {
+ if (rspamd_cryptobox_test_instr(CPUID_SSE2)) {
+ cpu_config |= CPUID_SSE2;
+ }
+ }
+ if ((cpu[2] & ((guint32) 1 << 0))) {
+ if (rspamd_cryptobox_test_instr(CPUID_SSE3)) {
+ cpu_config |= CPUID_SSE3;
+ }
+ }
+ if ((cpu[2] & ((guint32) 1 << 9))) {
+ if (rspamd_cryptobox_test_instr(CPUID_SSSE3)) {
+ cpu_config |= CPUID_SSSE3;
+ }
+ }
+ if ((cpu[2] & ((guint32) 1 << 19))) {
+ if (rspamd_cryptobox_test_instr(CPUID_SSE41)) {
+ cpu_config |= CPUID_SSE41;
+ }
+ }
+ if ((cpu[2] & ((guint32) 1 << 20))) {
+ if (rspamd_cryptobox_test_instr(CPUID_SSE42)) {
+ cpu_config |= CPUID_SSE42;
+ }
+ }
+ if ((cpu[2] & ((guint32) 1 << 30))) {
+ if (rspamd_cryptobox_test_instr(CPUID_RDRAND)) {
+ cpu_config |= CPUID_RDRAND;
+ }
+ }
+
+ /* OSXSAVE */
+ if ((cpu[2] & osxsave_mask) == osxsave_mask) {
+ if ((cpu[2] & ((guint32) 1 << 28))) {
+ if (rspamd_cryptobox_test_instr(CPUID_AVX)) {
+ cpu_config |= CPUID_AVX;
+ }
+ }
+
+ if (nid >= 7 &&
+ (cpu[2] & fma_movbe_osxsave_mask) == fma_movbe_osxsave_mask) {
+ rspamd_cryptobox_cpuid(cpu, 7);
+
+ if ((cpu[1] & avx2_bmi12_mask) == avx2_bmi12_mask) {
+ if (rspamd_cryptobox_test_instr(CPUID_AVX2)) {
+ cpu_config |= CPUID_AVX2;
+ }
+ }
+ }
+ }
+ }
+
+ buf = g_string_new("");
+
+ for (bit = 0x1; bit != 0; bit <<= 1) {
+ if (cpu_config & bit) {
+ switch (bit) {
+ case CPUID_SSE2:
+ rspamd_printf_gstring(buf, "sse2, ");
+ break;
+ case CPUID_SSE3:
+ rspamd_printf_gstring(buf, "sse3, ");
+ break;
+ case CPUID_SSSE3:
+ rspamd_printf_gstring(buf, "ssse3, ");
+ break;
+ case CPUID_SSE41:
+ rspamd_printf_gstring(buf, "sse4.1, ");
+ break;
+ case CPUID_SSE42:
+ rspamd_printf_gstring(buf, "sse4.2, ");
+ break;
+ case CPUID_AVX:
+ rspamd_printf_gstring(buf, "avx, ");
+ break;
+ case CPUID_AVX2:
+ rspamd_printf_gstring(buf, "avx2, ");
+ break;
+ case CPUID_RDRAND:
+ rspamd_printf_gstring(buf, "rdrand, ");
+ break;
+ default:
+ break; /* Silence warning */
+ }
+ }
+ }
+
+ if (buf->len > 2) {
+ /* Trim last chars */
+ g_string_erase(buf, buf->len - 2, 2);
+ }
+
+ ctx->cpu_extensions = buf->str;
+ g_string_free(buf, FALSE);
+ ctx->cpu_config = cpu_config;
+ g_assert(sodium_init() != -1);
+
+ ctx->chacha20_impl = chacha_load();
+ ctx->base64_impl = base64_load();
+#if defined(HAVE_USABLE_OPENSSL) && (OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER))
+ /* Needed for old openssl api, not sure about LibreSSL */
+ ERR_load_EC_strings();
+ ERR_load_RAND_strings();
+ ERR_load_EVP_strings();
+#endif
+
+ return ctx;
+}
+
+void rspamd_cryptobox_deinit(struct rspamd_cryptobox_library_ctx *ctx)
+{
+ if (ctx) {
+ g_free(ctx->cpu_extensions);
+ g_free(ctx);
+ }
+}
+
+void rspamd_cryptobox_keypair(rspamd_pk_t pk, rspamd_sk_t sk,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ ottery_rand_bytes(sk, rspamd_cryptobox_MAX_SKBYTES);
+ sk[0] &= 248;
+ sk[31] &= 127;
+ sk[31] |= 64;
+
+ crypto_scalarmult_base(pk, sk);
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EC_KEY *ec_sec;
+ const BIGNUM *bn_sec;
+
+ const EC_POINT *ec_pub;
+ gsize len;
+
+ ec_sec = EC_KEY_new_by_curve_name(CRYPTOBOX_CURVE_NID);
+ g_assert(ec_sec != NULL);
+ g_assert(EC_KEY_generate_key(ec_sec) != 0);
+
+ bn_sec = EC_KEY_get0_private_key(ec_sec);
+ g_assert(bn_sec != NULL);
+ ec_pub = EC_KEY_get0_public_key(ec_sec);
+ g_assert(ec_pub != NULL);
+#if OPENSSL_VERSION_MAJOR >= 3
+ unsigned char *buf = NULL; /* Thanks openssl for this API (no) */
+ len = EC_POINT_point2buf(EC_KEY_get0_group(ec_sec), ec_pub,
+ POINT_CONVERSION_UNCOMPRESSED, &buf, NULL);
+ g_assert(len <= (gint) rspamd_cryptobox_pk_bytes(mode));
+ memcpy(pk, buf, len);
+ OPENSSL_free(buf);
+#else
+ BIGNUM *bn_pub;
+ bn_pub = EC_POINT_point2bn(EC_KEY_get0_group(ec_sec),
+ ec_pub, POINT_CONVERSION_UNCOMPRESSED, NULL, NULL);
+ len = BN_num_bytes(bn_pub);
+ g_assert(len <= (gint) rspamd_cryptobox_pk_bytes(mode));
+ BN_bn2bin(bn_pub, pk);
+ BN_free(bn_pub);
+#endif
+
+ len = BN_num_bytes(bn_sec);
+ g_assert(len <= (gint) sizeof(rspamd_sk_t));
+ BN_bn2bin(bn_sec, sk);
+
+ EC_KEY_free(ec_sec);
+#endif
+ }
+}
+
+void rspamd_cryptobox_keypair_sig(rspamd_sig_pk_t pk, rspamd_sig_sk_t sk,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ crypto_sign_keypair(pk, sk);
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EC_KEY *ec_sec;
+ const BIGNUM *bn_sec;
+ const EC_POINT *ec_pub;
+ gsize len;
+
+ ec_sec = EC_KEY_new_by_curve_name(CRYPTOBOX_CURVE_NID);
+ g_assert(ec_sec != NULL);
+ g_assert(EC_KEY_generate_key(ec_sec) != 0);
+
+ bn_sec = EC_KEY_get0_private_key(ec_sec);
+ g_assert(bn_sec != NULL);
+ ec_pub = EC_KEY_get0_public_key(ec_sec);
+ g_assert(ec_pub != NULL);
+
+#if OPENSSL_VERSION_MAJOR >= 3
+ unsigned char *buf = NULL; /* Thanks openssl for this API (no) */
+ len = EC_POINT_point2buf(EC_KEY_get0_group(ec_sec), ec_pub,
+ POINT_CONVERSION_UNCOMPRESSED, &buf, NULL);
+ g_assert(len <= (gint) rspamd_cryptobox_pk_bytes(mode));
+ memcpy(pk, buf, len);
+ OPENSSL_free(buf);
+#else
+ BIGNUM *bn_pub;
+ bn_pub = EC_POINT_point2bn(EC_KEY_get0_group(ec_sec),
+ ec_pub, POINT_CONVERSION_UNCOMPRESSED, NULL, NULL);
+ len = BN_num_bytes(bn_pub);
+ g_assert(len <= (gint) rspamd_cryptobox_pk_bytes(mode));
+ BN_bn2bin(bn_pub, pk);
+ BN_free(bn_pub);
+#endif
+
+ len = BN_num_bytes(bn_sec);
+ g_assert(len <= (gint) sizeof(rspamd_sk_t));
+ BN_bn2bin(bn_sec, sk);
+ EC_KEY_free(ec_sec);
+#endif
+ }
+}
+
+#if OPENSSL_VERSION_MAJOR >= 3
+/* Compatibility function for OpenSSL 3.0 - thanks for breaking all API one more time */
+EC_POINT *ec_point_bn2point_compat(const EC_GROUP *group,
+ const BIGNUM *bn, EC_POINT *point, BN_CTX *ctx)
+{
+ size_t buf_len = 0;
+ unsigned char *buf;
+ EC_POINT *ret;
+
+ if ((buf_len = BN_num_bytes(bn)) == 0)
+ buf_len = 1;
+ if ((buf = OPENSSL_malloc(buf_len)) == NULL) {
+ return NULL;
+ }
+
+ if (!BN_bn2binpad(bn, buf, buf_len)) {
+ OPENSSL_free(buf);
+ return NULL;
+ }
+
+ if (point == NULL) {
+ if ((ret = EC_POINT_new(group)) == NULL) {
+ OPENSSL_free(buf);
+ return NULL;
+ }
+ }
+ else
+ ret = point;
+
+ if (!EC_POINT_oct2point(group, ret, buf, buf_len, ctx)) {
+ if (ret != point)
+ EC_POINT_clear_free(ret);
+ OPENSSL_free(buf);
+ return NULL;
+ }
+
+ OPENSSL_free(buf);
+ return ret;
+}
+#else
+#define ec_point_bn2point_compat EC_POINT_bn2point
+#endif
+
+void rspamd_cryptobox_nm(rspamd_nm_t nm,
+ const rspamd_pk_t pk, const rspamd_sk_t sk,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ guchar s[32];
+ guchar e[32];
+
+ memcpy(e, sk, 32);
+ e[0] &= 248;
+ e[31] &= 127;
+ e[31] |= 64;
+
+ if (crypto_scalarmult(s, e, pk) != -1) {
+ hchacha(s, n0, nm, 20);
+ }
+
+ rspamd_explicit_memzero(e, 32);
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EC_KEY *lk;
+ EC_POINT *ec_pub;
+ BIGNUM *bn_pub, *bn_sec;
+ gint len;
+ guchar s[32];
+
+ lk = EC_KEY_new_by_curve_name(CRYPTOBOX_CURVE_NID);
+ g_assert(lk != NULL);
+
+ bn_pub = BN_bin2bn(pk, rspamd_cryptobox_pk_bytes(mode), NULL);
+ g_assert(bn_pub != NULL);
+ bn_sec = BN_bin2bn(sk, sizeof(rspamd_sk_t), NULL);
+ g_assert(bn_sec != NULL);
+
+ g_assert(EC_KEY_set_private_key(lk, bn_sec) == 1);
+ ec_pub = ec_point_bn2point_compat(EC_KEY_get0_group(lk), bn_pub, NULL, NULL);
+ g_assert(ec_pub != NULL);
+ len = ECDH_compute_key(s, sizeof(s), ec_pub, lk, NULL);
+ g_assert(len == sizeof(s));
+
+ /* Still do hchacha iteration since we are not using SHA1 KDF */
+ hchacha(s, n0, nm, 20);
+
+ EC_KEY_free(lk);
+ EC_POINT_free(ec_pub);
+ BN_free(bn_sec);
+ BN_free(bn_pub);
+#endif
+ }
+}
+
+void rspamd_cryptobox_sign(guchar *sig, unsigned long long *siglen_p,
+ const guchar *m, gsize mlen,
+ const rspamd_sk_t sk,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ crypto_sign_detached(sig, siglen_p, m, mlen, sk);
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EC_KEY *lk;
+ BIGNUM *bn_sec;
+ EVP_MD_CTX *sha_ctx;
+ unsigned char h[64];
+ guint diglen = rspamd_cryptobox_signature_bytes(mode);
+
+ /* Prehash */
+ sha_ctx = EVP_MD_CTX_create();
+ g_assert(EVP_DigestInit(sha_ctx, EVP_sha512()) == 1);
+ EVP_DigestUpdate(sha_ctx, m, mlen);
+ EVP_DigestFinal(sha_ctx, h, NULL);
+
+ /* Key setup */
+ lk = EC_KEY_new_by_curve_name(CRYPTOBOX_CURVE_NID);
+ g_assert(lk != NULL);
+ bn_sec = BN_bin2bn(sk, sizeof(rspamd_sk_t), NULL);
+ g_assert(bn_sec != NULL);
+ g_assert(EC_KEY_set_private_key(lk, bn_sec) == 1);
+
+ /* ECDSA */
+ g_assert(ECDSA_sign(0, h, sizeof(h), sig, &diglen, lk) == 1);
+ g_assert(diglen <= sizeof(rspamd_signature_t));
+
+ if (siglen_p) {
+ *siglen_p = diglen;
+ }
+
+ EC_KEY_free(lk);
+ EVP_MD_CTX_destroy(sha_ctx);
+ BN_free(bn_sec);
+#endif
+ }
+}
+
+bool rspamd_cryptobox_verify(const guchar *sig,
+ gsize siglen,
+ const guchar *m,
+ gsize mlen,
+ const rspamd_pk_t pk,
+ enum rspamd_cryptobox_mode mode)
+{
+ bool ret = false;
+
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ if (siglen == rspamd_cryptobox_signature_bytes(RSPAMD_CRYPTOBOX_MODE_25519)) {
+ ret = (crypto_sign_verify_detached(sig, m, mlen, pk) == 0);
+ }
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EC_KEY *lk;
+ EC_POINT *ec_pub;
+ BIGNUM *bn_pub;
+ EVP_MD_CTX *sha_ctx;
+ unsigned char h[64];
+
+ /* Prehash */
+ sha_ctx = EVP_MD_CTX_create();
+ g_assert(EVP_DigestInit(sha_ctx, EVP_sha512()) == 1);
+ EVP_DigestUpdate(sha_ctx, m, mlen);
+ EVP_DigestFinal(sha_ctx, h, NULL);
+
+ /* Key setup */
+ lk = EC_KEY_new_by_curve_name(CRYPTOBOX_CURVE_NID);
+ g_assert(lk != NULL);
+ bn_pub = BN_bin2bn(pk, rspamd_cryptobox_pk_bytes(mode), NULL);
+ g_assert(bn_pub != NULL);
+ ec_pub = ec_point_bn2point_compat(EC_KEY_get0_group(lk), bn_pub, NULL, NULL);
+ g_assert(ec_pub != NULL);
+ g_assert(EC_KEY_set_public_key(lk, ec_pub) == 1);
+
+ /* ECDSA */
+ ret = ECDSA_verify(0, h, sizeof(h), sig, siglen, lk) == 1;
+
+ EC_KEY_free(lk);
+ EVP_MD_CTX_destroy(sha_ctx);
+ BN_free(bn_pub);
+ EC_POINT_free(ec_pub);
+#endif
+ }
+
+ return ret;
+}
+
+static gsize
+rspamd_cryptobox_encrypt_ctx_len(enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ return sizeof(chacha_state) + CRYPTOBOX_ALIGNMENT;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ return sizeof(EVP_CIPHER_CTX *) + CRYPTOBOX_ALIGNMENT;
+#endif
+ }
+
+ return 0;
+}
+
+static gsize
+rspamd_cryptobox_auth_ctx_len(enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ return sizeof(crypto_onetimeauth_state) + RSPAMD_ALIGNOF(crypto_onetimeauth_state);
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ return sizeof(void *);
+#endif
+ }
+
+ return 0;
+}
+
+static void *
+rspamd_cryptobox_encrypt_init(void *enc_ctx, const rspamd_nonce_t nonce,
+ const rspamd_nm_t nm,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ chacha_state *s;
+
+ s = cryptobox_align_ptr(enc_ctx, CRYPTOBOX_ALIGNMENT);
+ xchacha_init(s,
+ (const chacha_key *) nm,
+ (const chacha_iv24 *) nonce,
+ 20);
+
+ return s;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EVP_CIPHER_CTX **s;
+
+ s = cryptobox_align_ptr(enc_ctx, CRYPTOBOX_ALIGNMENT);
+ memset(s, 0, sizeof(*s));
+ *s = EVP_CIPHER_CTX_new();
+ g_assert(EVP_EncryptInit_ex(*s, EVP_aes_256_gcm(), NULL, NULL, NULL) == 1);
+ g_assert(EVP_CIPHER_CTX_ctrl(*s, EVP_CTRL_GCM_SET_IVLEN,
+ rspamd_cryptobox_nonce_bytes(mode), NULL) == 1);
+ g_assert(EVP_EncryptInit_ex(*s, NULL, NULL, nm, nonce) == 1);
+
+ return s;
+#endif
+ }
+
+ return NULL;
+}
+
+static void *
+rspamd_cryptobox_auth_init(void *auth_ctx, void *enc_ctx,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ crypto_onetimeauth_state *mac_ctx;
+ guchar RSPAMD_ALIGNED(32) subkey[CHACHA_BLOCKBYTES];
+
+ mac_ctx = cryptobox_align_ptr(auth_ctx, CRYPTOBOX_ALIGNMENT);
+ memset(subkey, 0, sizeof(subkey));
+ chacha_update(enc_ctx, subkey, subkey, sizeof(subkey));
+ crypto_onetimeauth_init(mac_ctx, subkey);
+ rspamd_explicit_memzero(subkey, sizeof(subkey));
+
+ return mac_ctx;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ auth_ctx = enc_ctx;
+
+ return auth_ctx;
+#endif
+ }
+
+ return NULL;
+}
+
+static gboolean
+rspamd_cryptobox_encrypt_update(void *enc_ctx, const guchar *in, gsize inlen,
+ guchar *out, gsize *outlen,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ gsize r;
+ chacha_state *s;
+
+ s = cryptobox_align_ptr(enc_ctx, CRYPTOBOX_ALIGNMENT);
+
+ r = chacha_update(s, in, out, inlen);
+
+ if (outlen != NULL) {
+ *outlen = r;
+ }
+
+ return TRUE;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EVP_CIPHER_CTX **s = enc_ctx;
+ gint r;
+
+ r = inlen;
+ g_assert(EVP_EncryptUpdate(*s, out, &r, in, inlen) == 1);
+
+ if (outlen) {
+ *outlen = r;
+ }
+
+ return TRUE;
+#endif
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_cryptobox_auth_update(void *auth_ctx, const guchar *in, gsize inlen,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ crypto_onetimeauth_state *mac_ctx;
+
+ mac_ctx = cryptobox_align_ptr(auth_ctx, CRYPTOBOX_ALIGNMENT);
+ crypto_onetimeauth_update(mac_ctx, in, inlen);
+
+ return TRUE;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ return TRUE;
+#endif
+ }
+
+ return FALSE;
+}
+
+static gsize
+rspamd_cryptobox_encrypt_final(void *enc_ctx, guchar *out, gsize remain,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ chacha_state *s;
+
+ s = cryptobox_align_ptr(enc_ctx, CRYPTOBOX_ALIGNMENT);
+ return chacha_final(s, out);
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EVP_CIPHER_CTX **s = enc_ctx;
+ gint r = remain;
+
+ g_assert(EVP_EncryptFinal_ex(*s, out, &r) == 1);
+
+ return r;
+#endif
+ }
+
+ return 0;
+}
+
+static gboolean
+rspamd_cryptobox_auth_final(void *auth_ctx, rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ crypto_onetimeauth_state *mac_ctx;
+
+ mac_ctx = cryptobox_align_ptr(auth_ctx, CRYPTOBOX_ALIGNMENT);
+ crypto_onetimeauth_final(mac_ctx, sig);
+
+ return TRUE;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EVP_CIPHER_CTX **s = auth_ctx;
+
+ g_assert(EVP_CIPHER_CTX_ctrl(*s, EVP_CTRL_GCM_GET_TAG,
+ sizeof(rspamd_mac_t), sig) == 1);
+
+ return TRUE;
+#endif
+ }
+
+ return FALSE;
+}
+
+static void *
+rspamd_cryptobox_decrypt_init(void *enc_ctx, const rspamd_nonce_t nonce,
+ const rspamd_nm_t nm,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+
+ chacha_state *s;
+
+ s = cryptobox_align_ptr(enc_ctx, CRYPTOBOX_ALIGNMENT);
+ xchacha_init(s,
+ (const chacha_key *) nm,
+ (const chacha_iv24 *) nonce,
+ 20);
+
+ return s;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EVP_CIPHER_CTX **s;
+
+ s = cryptobox_align_ptr(enc_ctx, CRYPTOBOX_ALIGNMENT);
+ memset(s, 0, sizeof(*s));
+ *s = EVP_CIPHER_CTX_new();
+ g_assert(EVP_DecryptInit_ex(*s, EVP_aes_256_gcm(), NULL, NULL, NULL) == 1);
+ g_assert(EVP_CIPHER_CTX_ctrl(*s, EVP_CTRL_GCM_SET_IVLEN,
+ rspamd_cryptobox_nonce_bytes(mode), NULL) == 1);
+ g_assert(EVP_DecryptInit_ex(*s, NULL, NULL, nm, nonce) == 1);
+
+ return s;
+#endif
+ }
+
+ return NULL;
+}
+
+static void *
+rspamd_cryptobox_auth_verify_init(void *auth_ctx, void *enc_ctx,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ crypto_onetimeauth_state *mac_ctx;
+ guchar RSPAMD_ALIGNED(32) subkey[CHACHA_BLOCKBYTES];
+
+ mac_ctx = cryptobox_align_ptr(auth_ctx, CRYPTOBOX_ALIGNMENT);
+ memset(subkey, 0, sizeof(subkey));
+ chacha_update(enc_ctx, subkey, subkey, sizeof(subkey));
+ crypto_onetimeauth_init(mac_ctx, subkey);
+ rspamd_explicit_memzero(subkey, sizeof(subkey));
+
+ return mac_ctx;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ auth_ctx = enc_ctx;
+
+ return auth_ctx;
+#endif
+ }
+
+ return NULL;
+}
+
+static gboolean
+rspamd_cryptobox_decrypt_update(void *enc_ctx, const guchar *in, gsize inlen,
+ guchar *out, gsize *outlen,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ gsize r;
+ chacha_state *s;
+
+ s = cryptobox_align_ptr(enc_ctx, CRYPTOBOX_ALIGNMENT);
+ r = chacha_update(s, in, out, inlen);
+
+ if (outlen != NULL) {
+ *outlen = r;
+ }
+
+ return TRUE;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EVP_CIPHER_CTX **s = enc_ctx;
+ gint r;
+
+ r = outlen ? *outlen : inlen;
+ g_assert(EVP_DecryptUpdate(*s, out, &r, in, inlen) == 1);
+
+ if (outlen) {
+ *outlen = r;
+ }
+
+ return TRUE;
+#endif
+ }
+}
+
+static gboolean
+rspamd_cryptobox_auth_verify_update(void *auth_ctx,
+ const guchar *in, gsize inlen,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ crypto_onetimeauth_state *mac_ctx;
+
+ mac_ctx = cryptobox_align_ptr(auth_ctx, CRYPTOBOX_ALIGNMENT);
+ crypto_onetimeauth_update(mac_ctx, in, inlen);
+
+ return TRUE;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ /* We do not need to authenticate as a separate process */
+ return TRUE;
+#else
+#endif
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_cryptobox_decrypt_final(void *enc_ctx, guchar *out, gsize remain,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ chacha_state *s;
+
+ s = cryptobox_align_ptr(enc_ctx, CRYPTOBOX_ALIGNMENT);
+ chacha_final(s, out);
+
+ return TRUE;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EVP_CIPHER_CTX **s = enc_ctx;
+ gint r = remain;
+
+ if (EVP_DecryptFinal_ex(*s, out, &r) < 0) {
+ return FALSE;
+ }
+
+ return TRUE;
+#endif
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_cryptobox_auth_verify_final(void *auth_ctx, const rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ rspamd_mac_t mac;
+ crypto_onetimeauth_state *mac_ctx;
+
+ mac_ctx = cryptobox_align_ptr(auth_ctx, CRYPTOBOX_ALIGNMENT);
+ crypto_onetimeauth_final(mac_ctx, mac);
+
+ if (crypto_verify_16(mac, sig) != 0) {
+ return FALSE;
+ }
+
+ return TRUE;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EVP_CIPHER_CTX **s = auth_ctx;
+
+ if (EVP_CIPHER_CTX_ctrl(*s, EVP_CTRL_GCM_SET_TAG, 16, (guchar *) sig) != 1) {
+ return FALSE;
+ }
+
+ return TRUE;
+#endif
+ }
+
+ return FALSE;
+}
+
+
+static void
+rspamd_cryptobox_cleanup(void *enc_ctx, void *auth_ctx,
+ enum rspamd_cryptobox_mode mode)
+{
+ if (G_LIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ crypto_onetimeauth_state *mac_ctx;
+
+ mac_ctx = cryptobox_align_ptr(auth_ctx, CRYPTOBOX_ALIGNMENT);
+ rspamd_explicit_memzero(mac_ctx, sizeof(*mac_ctx));
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ EVP_CIPHER_CTX **s = enc_ctx;
+
+ EVP_CIPHER_CTX_cleanup(*s);
+ EVP_CIPHER_CTX_free(*s);
+#endif
+ }
+}
+
+void rspamd_cryptobox_encrypt_nm_inplace(guchar *data, gsize len,
+ const rspamd_nonce_t nonce,
+ const rspamd_nm_t nm,
+ rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode)
+{
+ gsize r;
+ void *enc_ctx, *auth_ctx;
+
+ enc_ctx = g_alloca(rspamd_cryptobox_encrypt_ctx_len(mode));
+ auth_ctx = g_alloca(rspamd_cryptobox_auth_ctx_len(mode));
+
+ enc_ctx = rspamd_cryptobox_encrypt_init(enc_ctx, nonce, nm, mode);
+ auth_ctx = rspamd_cryptobox_auth_init(auth_ctx, enc_ctx, mode);
+
+ rspamd_cryptobox_encrypt_update(enc_ctx, data, len, data, &r, mode);
+ rspamd_cryptobox_encrypt_final(enc_ctx, data + r, len - r, mode);
+
+ rspamd_cryptobox_auth_update(auth_ctx, data, len, mode);
+ rspamd_cryptobox_auth_final(auth_ctx, sig, mode);
+
+ rspamd_cryptobox_cleanup(enc_ctx, auth_ctx, mode);
+}
+
+static void
+rspamd_cryptobox_flush_outbuf(struct rspamd_cryptobox_segment *st,
+ const guchar *buf, gsize len, gsize offset)
+{
+ gsize cpy_len;
+
+ while (len > 0) {
+ cpy_len = MIN(len, st->len - offset);
+ memcpy(st->data + offset, buf, cpy_len);
+ st++;
+ buf += cpy_len;
+ len -= cpy_len;
+ offset = 0;
+ }
+}
+
+void rspamd_cryptobox_encryptv_nm_inplace(struct rspamd_cryptobox_segment *segments,
+ gsize cnt,
+ const rspamd_nonce_t nonce,
+ const rspamd_nm_t nm, rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode)
+{
+ struct rspamd_cryptobox_segment *cur = segments, *start_seg = segments;
+ guchar outbuf[CHACHA_BLOCKBYTES * 16];
+ void *enc_ctx, *auth_ctx;
+ guchar *out, *in;
+ gsize r, remain, inremain, seg_offset;
+
+ enc_ctx = g_alloca(rspamd_cryptobox_encrypt_ctx_len(mode));
+ auth_ctx = g_alloca(rspamd_cryptobox_auth_ctx_len(mode));
+
+ enc_ctx = rspamd_cryptobox_encrypt_init(enc_ctx, nonce, nm, mode);
+ auth_ctx = rspamd_cryptobox_auth_init(auth_ctx, enc_ctx, mode);
+
+ remain = sizeof(outbuf);
+ out = outbuf;
+ inremain = cur->len;
+ seg_offset = 0;
+
+ for (;;) {
+ if (cur - segments == (gint) cnt) {
+ break;
+ }
+
+ if (cur->len <= remain) {
+ memcpy(out, cur->data, cur->len);
+ remain -= cur->len;
+ out += cur->len;
+ cur++;
+
+ if (remain == 0) {
+ rspamd_cryptobox_encrypt_update(enc_ctx, outbuf, sizeof(outbuf),
+ outbuf, NULL, mode);
+ rspamd_cryptobox_auth_update(auth_ctx, outbuf, sizeof(outbuf),
+ mode);
+ rspamd_cryptobox_flush_outbuf(start_seg, outbuf,
+ sizeof(outbuf), seg_offset);
+ start_seg = cur;
+ seg_offset = 0;
+ remain = sizeof(outbuf);
+ out = outbuf;
+ }
+ }
+ else {
+ memcpy(out, cur->data, remain);
+ rspamd_cryptobox_encrypt_update(enc_ctx, outbuf, sizeof(outbuf),
+ outbuf, NULL, mode);
+ rspamd_cryptobox_auth_update(auth_ctx, outbuf, sizeof(outbuf),
+ mode);
+ rspamd_cryptobox_flush_outbuf(start_seg, outbuf, sizeof(outbuf),
+ seg_offset);
+ seg_offset = 0;
+
+ inremain = cur->len - remain;
+ in = cur->data + remain;
+ out = outbuf;
+ remain = 0;
+ start_seg = cur;
+
+ while (inremain > 0) {
+ if (sizeof(outbuf) <= inremain) {
+ memcpy(outbuf, in, sizeof(outbuf));
+ rspamd_cryptobox_encrypt_update(enc_ctx,
+ outbuf,
+ sizeof(outbuf),
+ outbuf,
+ NULL,
+ mode);
+ rspamd_cryptobox_auth_update(auth_ctx,
+ outbuf,
+ sizeof(outbuf),
+ mode);
+ memcpy(in, outbuf, sizeof(outbuf));
+ in += sizeof(outbuf);
+ inremain -= sizeof(outbuf);
+ remain = sizeof(outbuf);
+ }
+ else {
+ memcpy(outbuf, in, inremain);
+ remain = sizeof(outbuf) - inremain;
+ out = outbuf + inremain;
+ inremain = 0;
+ }
+ }
+
+ seg_offset = cur->len - (sizeof(outbuf) - remain);
+ cur++;
+ }
+ }
+
+ rspamd_cryptobox_encrypt_update(enc_ctx, outbuf, sizeof(outbuf) - remain,
+ outbuf, &r, mode);
+ out = outbuf + r;
+ rspamd_cryptobox_encrypt_final(enc_ctx, out, sizeof(outbuf) - remain - r,
+ mode);
+
+ rspamd_cryptobox_auth_update(auth_ctx, outbuf, sizeof(outbuf) - remain,
+ mode);
+ rspamd_cryptobox_auth_final(auth_ctx, sig, mode);
+
+ rspamd_cryptobox_flush_outbuf(start_seg, outbuf, sizeof(outbuf) - remain,
+ seg_offset);
+ rspamd_cryptobox_cleanup(enc_ctx, auth_ctx, mode);
+}
+
+gboolean
+rspamd_cryptobox_decrypt_nm_inplace(guchar *data, gsize len,
+ const rspamd_nonce_t nonce, const rspamd_nm_t nm,
+ const rspamd_mac_t sig, enum rspamd_cryptobox_mode mode)
+{
+ gsize r = 0;
+ gboolean ret = TRUE;
+ void *enc_ctx, *auth_ctx;
+
+ enc_ctx = g_alloca(rspamd_cryptobox_encrypt_ctx_len(mode));
+ auth_ctx = g_alloca(rspamd_cryptobox_auth_ctx_len(mode));
+
+ enc_ctx = rspamd_cryptobox_decrypt_init(enc_ctx, nonce, nm, mode);
+ auth_ctx = rspamd_cryptobox_auth_verify_init(auth_ctx, enc_ctx, mode);
+
+ rspamd_cryptobox_auth_verify_update(auth_ctx, data, len, mode);
+
+ if (!rspamd_cryptobox_auth_verify_final(auth_ctx, sig, mode)) {
+ ret = FALSE;
+ }
+ else {
+ rspamd_cryptobox_decrypt_update(enc_ctx, data, len, data, &r, mode);
+ ret = rspamd_cryptobox_decrypt_final(enc_ctx, data + r, len - r, mode);
+ }
+
+ rspamd_cryptobox_cleanup(enc_ctx, auth_ctx, mode);
+
+ return ret;
+}
+
+gboolean
+rspamd_cryptobox_decrypt_inplace(guchar *data, gsize len,
+ const rspamd_nonce_t nonce,
+ const rspamd_pk_t pk, const rspamd_sk_t sk,
+ const rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode)
+{
+ guchar nm[rspamd_cryptobox_MAX_NMBYTES];
+ gboolean ret;
+
+ rspamd_cryptobox_nm(nm, pk, sk, mode);
+ ret = rspamd_cryptobox_decrypt_nm_inplace(data, len, nonce, nm, sig, mode);
+
+ rspamd_explicit_memzero(nm, sizeof(nm));
+
+ return ret;
+}
+
+void rspamd_cryptobox_encrypt_inplace(guchar *data, gsize len,
+ const rspamd_nonce_t nonce,
+ const rspamd_pk_t pk, const rspamd_sk_t sk,
+ rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode)
+{
+ guchar nm[rspamd_cryptobox_MAX_NMBYTES];
+
+ rspamd_cryptobox_nm(nm, pk, sk, mode);
+ rspamd_cryptobox_encrypt_nm_inplace(data, len, nonce, nm, sig, mode);
+ rspamd_explicit_memzero(nm, sizeof(nm));
+}
+
+void rspamd_cryptobox_encryptv_inplace(struct rspamd_cryptobox_segment *segments,
+ gsize cnt,
+ const rspamd_nonce_t nonce,
+ const rspamd_pk_t pk, const rspamd_sk_t sk,
+ rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode)
+{
+ guchar nm[rspamd_cryptobox_MAX_NMBYTES];
+
+ rspamd_cryptobox_nm(nm, pk, sk, mode);
+ rspamd_cryptobox_encryptv_nm_inplace(segments, cnt, nonce, nm, sig, mode);
+ rspamd_explicit_memzero(nm, sizeof(nm));
+}
+
+
+void rspamd_cryptobox_siphash(unsigned char *out, const unsigned char *in,
+ unsigned long long inlen,
+ const rspamd_sipkey_t k)
+{
+ crypto_shorthash_siphash24(out, in, inlen, k);
+}
+
+/*
+ * Password-Based Key Derivation Function 2 (PKCS #5 v2.0).
+ * Code based on IEEE Std 802.11-2007, Annex H.4.2.
+ */
+static gboolean
+rspamd_cryptobox_pbkdf2(const char *pass, gsize pass_len,
+ const guint8 *salt, gsize salt_len, guint8 *key, gsize key_len,
+ unsigned int rounds)
+{
+ guint8 *asalt, obuf[crypto_generichash_blake2b_BYTES_MAX];
+ guint8 d1[crypto_generichash_blake2b_BYTES_MAX],
+ d2[crypto_generichash_blake2b_BYTES_MAX];
+ unsigned int i, j;
+ unsigned int count;
+ gsize r;
+
+ if (rounds < 1 || key_len == 0) {
+ return FALSE;
+ }
+ if (salt_len == 0 || salt_len > G_MAXSIZE - 4) {
+ return FALSE;
+ }
+
+ asalt = g_malloc(salt_len + 4);
+ memcpy(asalt, salt, salt_len);
+
+ for (count = 1; key_len > 0; count++) {
+ asalt[salt_len + 0] = (count >> 24) & 0xff;
+ asalt[salt_len + 1] = (count >> 16) & 0xff;
+ asalt[salt_len + 2] = (count >> 8) & 0xff;
+ asalt[salt_len + 3] = count & 0xff;
+
+ if (pass_len <= crypto_generichash_blake2b_KEYBYTES_MAX) {
+ crypto_generichash_blake2b(d1, sizeof(d1), asalt, salt_len + 4,
+ pass, pass_len);
+ }
+ else {
+ guint8 k[crypto_generichash_blake2b_BYTES_MAX];
+
+ /*
+ * We use additional blake2 iteration to store large key
+ * XXX: it is not compatible with the original implementation but safe
+ */
+ crypto_generichash_blake2b(k, sizeof(k), pass, pass_len,
+ NULL, 0);
+ crypto_generichash_blake2b(d1, sizeof(d1), asalt, salt_len + 4,
+ k, sizeof(k));
+ }
+
+ memcpy(obuf, d1, sizeof(obuf));
+
+ for (i = 1; i < rounds; i++) {
+ if (pass_len <= crypto_generichash_blake2b_KEYBYTES_MAX) {
+ crypto_generichash_blake2b(d2, sizeof(d2), d1, sizeof(d1),
+ pass, pass_len);
+ }
+ else {
+ guint8 k[crypto_generichash_blake2b_BYTES_MAX];
+
+ /*
+ * We use additional blake2 iteration to store large key
+ * XXX: it is not compatible with the original implementation but safe
+ */
+ crypto_generichash_blake2b(k, sizeof(k), pass, pass_len,
+ NULL, 0);
+ crypto_generichash_blake2b(d2, sizeof(d2), d1, sizeof(d1),
+ k, sizeof(k));
+ }
+
+ memcpy(d1, d2, sizeof(d1));
+
+ for (j = 0; j < sizeof(obuf); j++) {
+ obuf[j] ^= d1[j];
+ }
+ }
+
+ r = MIN(key_len, crypto_generichash_blake2b_BYTES_MAX);
+ memcpy(key, obuf, r);
+ key += r;
+ key_len -= r;
+ }
+
+ rspamd_explicit_memzero(asalt, salt_len + 4);
+ g_free(asalt);
+ rspamd_explicit_memzero(d1, sizeof(d1));
+ rspamd_explicit_memzero(d2, sizeof(d2));
+ rspamd_explicit_memzero(obuf, sizeof(obuf));
+
+ return TRUE;
+}
+
+gboolean
+rspamd_cryptobox_pbkdf(const char *pass, gsize pass_len,
+ const guint8 *salt, gsize salt_len, guint8 *key, gsize key_len,
+ unsigned int complexity, enum rspamd_cryptobox_pbkdf_type type)
+{
+ gboolean ret = FALSE;
+
+ switch (type) {
+ case RSPAMD_CRYPTOBOX_CATENA:
+ if (catena(pass, pass_len, salt, salt_len, "rspamd", 6,
+ 4, complexity, complexity, key_len, key) == 0) {
+ ret = TRUE;
+ }
+ break;
+ case RSPAMD_CRYPTOBOX_PBKDF2:
+ default:
+ ret = rspamd_cryptobox_pbkdf2(pass, pass_len, salt, salt_len, key,
+ key_len, complexity);
+ break;
+ }
+
+ return ret;
+}
+
+guint rspamd_cryptobox_pk_bytes(enum rspamd_cryptobox_mode mode)
+{
+ if (G_UNLIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ return 32;
+ }
+ else {
+ return 65;
+ }
+}
+
+guint rspamd_cryptobox_pk_sig_bytes(enum rspamd_cryptobox_mode mode)
+{
+ if (G_UNLIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ return 32;
+ }
+ else {
+ return 65;
+ }
+}
+
+guint rspamd_cryptobox_nonce_bytes(enum rspamd_cryptobox_mode mode)
+{
+ if (G_UNLIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ return 24;
+ }
+ else {
+ return 16;
+ }
+}
+
+
+guint rspamd_cryptobox_sk_bytes(enum rspamd_cryptobox_mode mode)
+{
+ return 32;
+}
+
+guint rspamd_cryptobox_sk_sig_bytes(enum rspamd_cryptobox_mode mode)
+{
+ if (G_UNLIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ return 64;
+ }
+ else {
+ return 32;
+ }
+}
+
+guint rspamd_cryptobox_signature_bytes(enum rspamd_cryptobox_mode mode)
+{
+ static guint ssl_keylen;
+
+ if (G_UNLIKELY(mode == RSPAMD_CRYPTOBOX_MODE_25519)) {
+ return 64;
+ }
+ else {
+#ifndef HAVE_USABLE_OPENSSL
+ g_assert(0);
+#else
+ if (ssl_keylen == 0) {
+ EC_KEY *lk;
+ lk = EC_KEY_new_by_curve_name(CRYPTOBOX_CURVE_NID);
+ ssl_keylen = ECDSA_size(lk);
+ EC_KEY_free(lk);
+ }
+#endif
+ return ssl_keylen;
+ }
+}
+
+guint rspamd_cryptobox_nm_bytes(enum rspamd_cryptobox_mode mode)
+{
+ return 32;
+}
+
+guint rspamd_cryptobox_mac_bytes(enum rspamd_cryptobox_mode mode)
+{
+ return 16;
+}
+
+void rspamd_cryptobox_hash_init(rspamd_cryptobox_hash_state_t *p, const guchar *key, gsize keylen)
+{
+ crypto_generichash_blake2b_state *st = cryptobox_align_ptr(p,
+ RSPAMD_ALIGNOF(crypto_generichash_blake2b_state));
+ crypto_generichash_blake2b_init(st, key, keylen,
+ crypto_generichash_blake2b_BYTES_MAX);
+}
+
+/**
+ * Update hash with data portion
+ */
+void rspamd_cryptobox_hash_update(rspamd_cryptobox_hash_state_t *p, const guchar *data, gsize len)
+{
+ crypto_generichash_blake2b_state *st = cryptobox_align_ptr(p,
+ RSPAMD_ALIGNOF(crypto_generichash_blake2b_state));
+ crypto_generichash_blake2b_update(st, data, len);
+}
+
+/**
+ * Output hash to the buffer of rspamd_cryptobox_HASHBYTES length
+ */
+void rspamd_cryptobox_hash_final(rspamd_cryptobox_hash_state_t *p, guchar *out)
+{
+ crypto_generichash_blake2b_state *st = cryptobox_align_ptr(p,
+ RSPAMD_ALIGNOF(crypto_generichash_blake2b_state));
+ crypto_generichash_blake2b_final(st, out, crypto_generichash_blake2b_BYTES_MAX);
+}
+
+/**
+ * One in all function
+ */
+void rspamd_cryptobox_hash(guchar *out,
+ const guchar *data,
+ gsize len,
+ const guchar *key,
+ gsize keylen)
+{
+ crypto_generichash_blake2b(out, crypto_generichash_blake2b_BYTES_MAX,
+ data, len, key, keylen);
+}
+
+G_STATIC_ASSERT(sizeof(t1ha_context_t) <=
+ sizeof(((rspamd_cryptobox_fast_hash_state_t *) NULL)->opaque));
+G_STATIC_ASSERT(sizeof(struct XXH3_state_s) <=
+ sizeof(((rspamd_cryptobox_fast_hash_state_t *) NULL)->opaque));
+
+
+struct RSPAMD_ALIGNED(16) _mum_iuf {
+ union {
+ gint64 ll;
+ unsigned char b[sizeof(guint64)];
+ } buf;
+ gint64 h;
+ unsigned rem;
+};
+
+rspamd_cryptobox_fast_hash_state_t *
+rspamd_cryptobox_fast_hash_new(void)
+{
+ rspamd_cryptobox_fast_hash_state_t *nst;
+ int ret = posix_memalign((void **) &nst, RSPAMD_ALIGNOF(rspamd_cryptobox_fast_hash_state_t),
+ sizeof(rspamd_cryptobox_fast_hash_state_t));
+
+ if (ret != 0) {
+ abort();
+ }
+
+ return nst;
+}
+
+void rspamd_cryptobox_fast_hash_free(rspamd_cryptobox_fast_hash_state_t *st)
+{
+ free(st);
+}
+
+void rspamd_cryptobox_fast_hash_init(rspamd_cryptobox_fast_hash_state_t *st,
+ guint64 seed)
+{
+ XXH3_state_t *xst = (XXH3_state_t *) st->opaque;
+ st->type = RSPAMD_CRYPTOBOX_XXHASH3;
+ XXH3_INITSTATE(xst);
+ XXH3_64bits_reset_withSeed(xst, seed);
+}
+
+void rspamd_cryptobox_fast_hash_init_specific(rspamd_cryptobox_fast_hash_state_t *st,
+ enum rspamd_cryptobox_fast_hash_type type,
+ guint64 seed)
+{
+ switch (type) {
+ case RSPAMD_CRYPTOBOX_T1HA:
+ case RSPAMD_CRYPTOBOX_HASHFAST:
+ case RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT: {
+ t1ha_context_t *rst = (t1ha_context_t *) st->opaque;
+ st->type = RSPAMD_CRYPTOBOX_T1HA;
+ t1ha2_init(rst, seed, 0);
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_XXHASH64: {
+ XXH64_state_t *xst = (XXH64_state_t *) st->opaque;
+ memset(xst, 0, sizeof(*xst));
+ st->type = RSPAMD_CRYPTOBOX_XXHASH64;
+ XXH64_reset(xst, seed);
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_XXHASH32: {
+ XXH32_state_t *xst = (XXH32_state_t *) st->opaque;
+ memset(xst, 0, sizeof(*xst));
+ st->type = RSPAMD_CRYPTOBOX_XXHASH32;
+ XXH32_reset(xst, seed);
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_XXHASH3: {
+ XXH3_state_t *xst = (XXH3_state_t *) st->opaque;
+ XXH3_INITSTATE(xst);
+ st->type = RSPAMD_CRYPTOBOX_XXHASH3;
+ XXH3_64bits_reset_withSeed(xst, seed);
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_MUMHASH: {
+ struct _mum_iuf *iuf = (struct _mum_iuf *) st->opaque;
+ st->type = RSPAMD_CRYPTOBOX_MUMHASH;
+ iuf->h = seed;
+ iuf->buf.ll = 0;
+ iuf->rem = 0;
+ break;
+ }
+ }
+}
+
+void rspamd_cryptobox_fast_hash_update(rspamd_cryptobox_fast_hash_state_t *st,
+ const void *data, gsize len)
+{
+ if (st->type == RSPAMD_CRYPTOBOX_T1HA) {
+ t1ha_context_t *rst = (t1ha_context_t *) st->opaque;
+ t1ha2_update(rst, data, len);
+ }
+ else {
+ switch (st->type) {
+ case RSPAMD_CRYPTOBOX_XXHASH64: {
+ XXH64_state_t *xst = (XXH64_state_t *) st->opaque;
+ XXH64_update(xst, data, len);
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_XXHASH32: {
+ XXH32_state_t *xst = (XXH32_state_t *) st->opaque;
+ XXH32_update(xst, data, len);
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_XXHASH3: {
+ XXH3_state_t *xst = (XXH3_state_t *) st->opaque;
+ XXH3_64bits_update(xst, data, len);
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_MUMHASH: {
+ struct _mum_iuf *iuf = (struct _mum_iuf *) st->opaque;
+ gsize drem = len;
+ const guchar *p = data;
+
+ if (iuf->rem > 0) {
+ /* Process remainder */
+ if (drem >= iuf->rem) {
+ memcpy(iuf->buf.b + sizeof(iuf->buf.ll) - iuf->rem,
+ p, iuf->rem);
+ drem -= iuf->rem;
+ p += iuf->rem;
+ iuf->h = mum_hash_step(iuf->h, iuf->buf.ll);
+ iuf->rem = 0;
+ }
+ else {
+ memcpy(iuf->buf.b + sizeof(iuf->buf.ll) - iuf->rem, p, drem);
+ iuf->rem -= drem;
+ drem = 0;
+ }
+ }
+
+ while (drem >= sizeof(iuf->buf.ll)) {
+ memcpy(iuf->buf.b, p, sizeof(iuf->buf.ll));
+ iuf->h = mum_hash_step(iuf->h, iuf->buf.ll);
+ drem -= sizeof(iuf->buf.ll);
+ p += sizeof(iuf->buf.ll);
+ }
+
+ /* Leftover */
+ if (drem > 0) {
+ iuf->rem = sizeof(guint64) - drem;
+ iuf->buf.ll = 0;
+ memcpy(iuf->buf.b, p, drem);
+ }
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_T1HA:
+ case RSPAMD_CRYPTOBOX_HASHFAST:
+ case RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT: {
+ t1ha_context_t *rst = (t1ha_context_t *) st->opaque;
+ t1ha2_update(rst, data, len);
+ break;
+ }
+ }
+ }
+}
+
+guint64
+rspamd_cryptobox_fast_hash_final(rspamd_cryptobox_fast_hash_state_t *st)
+{
+ guint64 ret;
+
+ if (st->type == RSPAMD_CRYPTOBOX_T1HA) {
+ t1ha_context_t *rst = (t1ha_context_t *) st->opaque;
+
+ return t1ha2_final(rst, NULL);
+ }
+ else {
+ switch (st->type) {
+ case RSPAMD_CRYPTOBOX_XXHASH64: {
+ XXH64_state_t *xst = (XXH64_state_t *) st->opaque;
+ ret = XXH64_digest(xst);
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_XXHASH32: {
+ XXH32_state_t *xst = (XXH32_state_t *) st->opaque;
+ ret = XXH32_digest(xst);
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_XXHASH3: {
+ XXH3_state_t *xst = (XXH3_state_t *) st->opaque;
+ ret = XXH3_64bits_digest(xst);
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_MUMHASH: {
+ struct _mum_iuf *iuf = (struct _mum_iuf *) st->opaque;
+ iuf->h = mum_hash_step(iuf->h, iuf->buf.ll);
+ ret = mum_hash_finish(iuf->h);
+ break;
+ }
+ case RSPAMD_CRYPTOBOX_T1HA:
+ case RSPAMD_CRYPTOBOX_HASHFAST:
+ case RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT: {
+ t1ha_context_t *rst = (t1ha_context_t *) st->opaque;
+
+ ret = t1ha2_final(rst, NULL);
+ break;
+ }
+ }
+ }
+
+ return ret;
+}
+
+/**
+ * One in all function
+ */
+static inline guint64
+rspamd_cryptobox_fast_hash_machdep(const void *data,
+ gsize len, guint64 seed)
+{
+ return XXH3_64bits_withSeed(data, len, seed);
+}
+
+static inline guint64
+rspamd_cryptobox_fast_hash_indep(const void *data,
+ gsize len, guint64 seed)
+{
+ return XXH3_64bits_withSeed(data, len, seed);
+}
+
+guint64
+rspamd_cryptobox_fast_hash(const void *data,
+ gsize len, guint64 seed)
+{
+ return rspamd_cryptobox_fast_hash_machdep(data, len, seed);
+}
+
+guint64
+rspamd_cryptobox_fast_hash_specific(
+ enum rspamd_cryptobox_fast_hash_type type,
+ const void *data,
+ gsize len, guint64 seed)
+{
+ switch (type) {
+ case RSPAMD_CRYPTOBOX_XXHASH32:
+ return XXH32(data, len, seed);
+ case RSPAMD_CRYPTOBOX_XXHASH3:
+ return XXH3_64bits_withSeed(data, len, seed);
+ case RSPAMD_CRYPTOBOX_XXHASH64:
+ return XXH64(data, len, seed);
+ case RSPAMD_CRYPTOBOX_MUMHASH:
+ return mum_hash(data, len, seed);
+ case RSPAMD_CRYPTOBOX_T1HA:
+ return t1ha2_atonce(data, len, seed);
+ case RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT:
+ return rspamd_cryptobox_fast_hash_indep(data, len, seed);
+ case RSPAMD_CRYPTOBOX_HASHFAST:
+ default:
+ return rspamd_cryptobox_fast_hash_machdep(data, len, seed);
+ }
+}
diff --git a/src/libcryptobox/cryptobox.h b/src/libcryptobox/cryptobox.h
new file mode 100644
index 0000000..8cd79bb
--- /dev/null
+++ b/src/libcryptobox/cryptobox.h
@@ -0,0 +1,437 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef CRYPTOBOX_H_
+#define CRYPTOBOX_H_
+
+#include "config.h"
+
+#include <sodium.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_cryptobox_segment {
+ guchar *data;
+ gsize len;
+};
+
+#if defined(__GNUC__) && \
+ ((defined(__clang__) && (__clang_major__ >= 4 || (__clang_major__ >= 3 && __clang_minor__ >= 8))) || \
+ ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 8) || (__GNUC__ > 4)))
+#define RSPAMD_HAS_TARGET_ATTR 1
+#endif
+
+#define rspamd_cryptobox_MAX_NONCEBYTES 24
+#define rspamd_cryptobox_MAX_PKBYTES 65
+#define rspamd_cryptobox_MAX_SKBYTES 32
+#define rspamd_cryptobox_MAX_MACBYTES 16
+#define rspamd_cryptobox_MAX_NMBYTES 32
+#define rspamd_cryptobox_SIPKEYBYTES 16
+#define rspamd_cryptobox_HASHBYTES 64
+#define rspamd_cryptobox_HASHKEYBYTES 64
+#define rspamd_cryptobox_HASHSTATEBYTES sizeof(crypto_generichash_blake2b_state) + 64
+#define rspamd_cryptobox_MAX_SIGSKBYTES 64
+#define rspamd_cryptobox_MAX_SIGPKBYTES 32
+#define rspamd_cryptobox_MAX_SIGBYTES 72
+
+#define CPUID_AVX2 0x1
+#define CPUID_AVX 0x2
+#define CPUID_SSE2 0x4
+#define CPUID_SSE3 0x8
+#define CPUID_SSSE3 0x10
+#define CPUID_SSE41 0x20
+#define CPUID_SSE42 0x40
+#define CPUID_RDRAND 0x80
+
+typedef guchar rspamd_pk_t[rspamd_cryptobox_MAX_PKBYTES];
+typedef guchar rspamd_sk_t[rspamd_cryptobox_MAX_SKBYTES];
+typedef guchar rspamd_mac_t[rspamd_cryptobox_MAX_MACBYTES];
+typedef guchar rspamd_nm_t[rspamd_cryptobox_MAX_NMBYTES];
+typedef guchar rspamd_nonce_t[rspamd_cryptobox_MAX_NONCEBYTES];
+typedef guchar rspamd_sipkey_t[rspamd_cryptobox_SIPKEYBYTES];
+typedef guchar rspamd_signature_t[rspamd_cryptobox_MAX_SIGBYTES];
+typedef guchar rspamd_sig_pk_t[rspamd_cryptobox_MAX_SIGPKBYTES];
+typedef guchar rspamd_sig_sk_t[rspamd_cryptobox_MAX_SIGSKBYTES];
+
+enum rspamd_cryptobox_mode {
+ RSPAMD_CRYPTOBOX_MODE_25519 = 0,
+ RSPAMD_CRYPTOBOX_MODE_NIST
+};
+
+struct rspamd_cryptobox_library_ctx {
+ gchar *cpu_extensions;
+ const gchar *chacha20_impl;
+ const gchar *base64_impl;
+ unsigned long cpu_config;
+};
+
+/**
+ * Init cryptobox library
+ */
+struct rspamd_cryptobox_library_ctx *rspamd_cryptobox_init(void);
+
+void rspamd_cryptobox_deinit(struct rspamd_cryptobox_library_ctx *);
+/**
+ * Generate new keypair
+ * @param pk public key buffer
+ * @param sk secret key buffer
+ */
+void rspamd_cryptobox_keypair(rspamd_pk_t pk, rspamd_sk_t sk,
+ enum rspamd_cryptobox_mode mode);
+
+/**
+ * Generate new keypair for signing
+ * @param pk public key buffer
+ * @param sk secret key buffer
+ */
+void rspamd_cryptobox_keypair_sig(rspamd_sig_pk_t pk, rspamd_sig_sk_t sk,
+ enum rspamd_cryptobox_mode mode);
+
+/**
+ * Encrypt data inplace adding signature to sig afterwards
+ * @param data input buffer
+ * @param pk remote pubkey
+ * @param sk local secret key
+ * @param sig output signature
+ */
+void rspamd_cryptobox_encrypt_inplace(guchar *data, gsize len,
+ const rspamd_nonce_t nonce,
+ const rspamd_pk_t pk, const rspamd_sk_t sk, rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode);
+
+/**
+ * Encrypt segments of data inplace adding signature to sig afterwards
+ * @param segments segments of data
+ * @param cnt count of segments
+ * @param pk remote pubkey
+ * @param sk local secret key
+ * @param sig output signature
+ */
+void rspamd_cryptobox_encryptv_inplace(struct rspamd_cryptobox_segment *segments,
+ gsize cnt,
+ const rspamd_nonce_t nonce,
+ const rspamd_pk_t pk, const rspamd_sk_t sk, rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode);
+
+
+/**
+ * Decrypt and verify data chunk inplace
+ * @param data data to decrypt
+ * @param len length of data
+ * @param pk remote pubkey
+ * @param sk local privkey
+ * @param sig signature input
+ * @return TRUE if input has been verified successfully
+ */
+gboolean rspamd_cryptobox_decrypt_inplace(guchar *data, gsize len,
+ const rspamd_nonce_t nonce,
+ const rspamd_pk_t pk, const rspamd_sk_t sk, const rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode);
+
+/**
+ * Encrypt segments of data inplace adding signature to sig afterwards
+ * @param segments segments of data
+ * @param cnt count of segments
+ * @param pk remote pubkey
+ * @param sk local secret key
+ * @param sig output signature
+ */
+void rspamd_cryptobox_encrypt_nm_inplace(guchar *data, gsize len,
+ const rspamd_nonce_t nonce,
+ const rspamd_nm_t nm, rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode);
+
+/**
+ * Encrypt segments of data inplace adding signature to sig afterwards
+ * @param segments segments of data
+ * @param cnt count of segments
+ * @param pk remote pubkey
+ * @param sk local secret key
+ * @param sig output signature
+ */
+void rspamd_cryptobox_encryptv_nm_inplace(struct rspamd_cryptobox_segment *segments,
+ gsize cnt,
+ const rspamd_nonce_t nonce,
+ const rspamd_nm_t nm, rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode);
+
+
+/**
+ * Decrypt and verify data chunk inplace
+ * @param data data to decrypt
+ * @param len length of data
+ * @param pk remote pubkey
+ * @param sk local privkey
+ * @param sig signature input
+ * @return TRUE if input has been verified successfully
+ */
+gboolean rspamd_cryptobox_decrypt_nm_inplace(guchar *data, gsize len,
+ const rspamd_nonce_t nonce,
+ const rspamd_nm_t nm, const rspamd_mac_t sig,
+ enum rspamd_cryptobox_mode mode);
+
+/**
+ * Generate shared secret from local sk and remote pk
+ * @param nm shared secret
+ * @param pk remote pubkey
+ * @param sk local privkey
+ */
+void rspamd_cryptobox_nm(rspamd_nm_t nm, const rspamd_pk_t pk,
+ const rspamd_sk_t sk, enum rspamd_cryptobox_mode mode);
+
+/**
+ * Create digital signature for the specified message and place result in `sig`
+ * @param sig signature target
+ * @param siglen_p pointer to signature length (might be NULL)
+ * @param m input message
+ * @param mlen input length
+ * @param sk secret key
+ */
+void rspamd_cryptobox_sign(guchar *sig, unsigned long long *siglen_p,
+ const guchar *m, gsize mlen,
+ const rspamd_sk_t sk,
+ enum rspamd_cryptobox_mode mode);
+
+/**
+ * Verifies digital signature for the specified message using the specified
+ * pubkey
+ * @param sig signature source
+ * @param m input message
+ * @param mlen message length
+ * @param pk public key for verification
+ * @return true if signature is valid, false otherwise
+ */
+bool rspamd_cryptobox_verify(const guchar *sig,
+ gsize siglen,
+ const guchar *m,
+ gsize mlen,
+ const rspamd_pk_t pk,
+ enum rspamd_cryptobox_mode mode);
+
+/**
+ * Securely clear the buffer specified
+ * @param buf buffer to zero
+ * @param buflen length of buffer
+ */
+
+#define rspamd_explicit_memzero sodium_memzero
+
+/**
+ * Constant time memcmp
+ * @param b1_
+ * @param b2_
+ * @param len
+ * @return
+ */
+#define rspamd_cryptobox_memcmp sodium_memcmp
+
+/**
+ * Calculates siphash-2-4 for a message
+ * @param out (8 bytes output)
+ * @param in
+ * @param inlen
+ * @param k key (must be 16 bytes)
+ */
+void rspamd_cryptobox_siphash(unsigned char *out, const unsigned char *in,
+ unsigned long long inlen,
+ const rspamd_sipkey_t k);
+
+enum rspamd_cryptobox_pbkdf_type {
+ RSPAMD_CRYPTOBOX_PBKDF2 = 0,
+ RSPAMD_CRYPTOBOX_CATENA
+};
+
+
+/**
+ * Derive key from password using the specified algorithm
+ * @param pass input password
+ * @param pass_len length of the password
+ * @param salt input salt
+ * @param salt_len length of salt
+ * @param key output key
+ * @param key_len size of the key
+ * @param complexity empiric number of complexity (rounds for pbkdf2 and garlic for catena)
+ * @return TRUE in case of success and FALSE if failed
+ */
+gboolean rspamd_cryptobox_pbkdf(const char *pass, gsize pass_len,
+ const guint8 *salt, gsize salt_len,
+ guint8 *key, gsize key_len,
+ unsigned int complexity,
+ enum rspamd_cryptobox_pbkdf_type type);
+
+
+/**
+ * Real size of rspamd cryptobox public key
+ */
+guint rspamd_cryptobox_pk_bytes(enum rspamd_cryptobox_mode mode);
+
+/**
+ * Real size of rspamd cryptobox signing public key
+ */
+guint rspamd_cryptobox_pk_sig_bytes(enum rspamd_cryptobox_mode mode);
+
+/**
+ * Real size of crypto nonce
+ */
+guint rspamd_cryptobox_nonce_bytes(enum rspamd_cryptobox_mode mode);
+
+/**
+ * Real size of rspamd cryptobox secret key
+ */
+guint rspamd_cryptobox_sk_bytes(enum rspamd_cryptobox_mode mode);
+
+/**
+ * Real size of rspamd cryptobox signing secret key
+ */
+guint rspamd_cryptobox_sk_sig_bytes(enum rspamd_cryptobox_mode mode);
+
+/**
+ * Real size of rspamd cryptobox shared key
+ */
+guint rspamd_cryptobox_nm_bytes(enum rspamd_cryptobox_mode mode);
+
+/**
+ * Real size of rspamd cryptobox MAC signature
+ */
+guint rspamd_cryptobox_mac_bytes(enum rspamd_cryptobox_mode mode);
+
+/**
+ * Real size of rspamd cryptobox digital signature
+ */
+guint rspamd_cryptobox_signature_bytes(enum rspamd_cryptobox_mode mode);
+
+/* Hash IUF interface */
+typedef crypto_generichash_blake2b_state rspamd_cryptobox_hash_state_t;
+
+/**
+ * Init cryptobox hash state using key if needed, `st` must point to the buffer
+ * with at least rspamd_cryptobox_HASHSTATEBYTES bytes length. If keylen == 0, then
+ * non-keyed hash is generated
+ */
+void rspamd_cryptobox_hash_init(rspamd_cryptobox_hash_state_t *st,
+ const guchar *key, gsize keylen);
+
+/**
+ * Update hash with data portion
+ */
+void rspamd_cryptobox_hash_update(rspamd_cryptobox_hash_state_t *st,
+ const guchar *data, gsize len);
+
+/**
+ * Output hash to the buffer of rspamd_cryptobox_HASHBYTES length
+ */
+void rspamd_cryptobox_hash_final(rspamd_cryptobox_hash_state_t *st, guchar *out);
+
+/**
+ * One in all function
+ */
+void rspamd_cryptobox_hash(guchar *out,
+ const guchar *data,
+ gsize len,
+ const guchar *key,
+ gsize keylen);
+
+enum rspamd_cryptobox_fast_hash_type {
+ RSPAMD_CRYPTOBOX_XXHASH64 = 0,
+ RSPAMD_CRYPTOBOX_XXHASH32,
+ RSPAMD_CRYPTOBOX_XXHASH3,
+ RSPAMD_CRYPTOBOX_MUMHASH,
+ RSPAMD_CRYPTOBOX_T1HA,
+ RSPAMD_CRYPTOBOX_HASHFAST,
+ RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT
+};
+
+/* Non crypto hash IUF interface */
+typedef struct CRYPTO_ALIGN(64) rspamd_cryptobox_fast_hash_state_s {
+ guchar opaque[576]; /* Required for xxhash3 */
+ enum rspamd_cryptobox_fast_hash_type type;
+} rspamd_cryptobox_fast_hash_state_t;
+
+
+/**
+ * Creates a new cryptobox state properly aligned
+ * @return
+ */
+rspamd_cryptobox_fast_hash_state_t *rspamd_cryptobox_fast_hash_new(void);
+void rspamd_cryptobox_fast_hash_free(rspamd_cryptobox_fast_hash_state_t *st);
+
+/**
+ * Init cryptobox hash state using key if needed, `st` must point to the buffer
+ * with at least rspamd_cryptobox_HASHSTATEBYTES bytes length. If keylen == 0, then
+ * non-keyed hash is generated
+ */
+void rspamd_cryptobox_fast_hash_init(rspamd_cryptobox_fast_hash_state_t *st,
+ guint64 seed);
+
+/**
+ * Init cryptobox hash state using key if needed, `st` must point to the buffer
+ * with at least rspamd_cryptobox_HASHSTATEBYTES bytes length. If keylen == 0, then
+ * non-keyed hash is generated
+ */
+void rspamd_cryptobox_fast_hash_init_specific(rspamd_cryptobox_fast_hash_state_t *st,
+ enum rspamd_cryptobox_fast_hash_type type,
+ guint64 seed);
+
+/**
+ * Update hash with data portion
+ */
+void rspamd_cryptobox_fast_hash_update(rspamd_cryptobox_fast_hash_state_t *st,
+ const void *data, gsize len);
+
+/**
+ * Output hash to the buffer of rspamd_cryptobox_HASHBYTES length
+ */
+guint64 rspamd_cryptobox_fast_hash_final(rspamd_cryptobox_fast_hash_state_t *st);
+
+/**
+ * One in all function
+ */
+guint64 rspamd_cryptobox_fast_hash(const void *data,
+ gsize len, guint64 seed);
+
+/**
+ * Platform independent version
+ */
+guint64 rspamd_cryptobox_fast_hash_specific(
+ enum rspamd_cryptobox_fast_hash_type type,
+ const void *data,
+ gsize len, guint64 seed);
+
+/**
+ * Decode base64 using platform optimized code
+ * @param in
+ * @param inlen
+ * @param out
+ * @param outlen
+ * @return
+ */
+gboolean rspamd_cryptobox_base64_decode(const gchar *in, gsize inlen,
+ guchar *out, gsize *outlen);
+
+/**
+ * Returns TRUE if data looks like a valid base64 string
+ * @param in
+ * @param inlen
+ * @return
+ */
+gboolean rspamd_cryptobox_base64_is_valid(const gchar *in, gsize inlen);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CRYPTOBOX_H_ */
diff --git a/src/libcryptobox/keypair.c b/src/libcryptobox/keypair.c
new file mode 100644
index 0000000..ec7490a
--- /dev/null
+++ b/src/libcryptobox/keypair.c
@@ -0,0 +1,1021 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "libcryptobox/keypair.h"
+#include "libcryptobox/keypair_private.h"
+#include "libutil/str_util.h"
+#include "libutil/printf.h"
+#include "contrib/libottery/ottery.h"
+
+const guchar encrypted_magic[7] = {'r', 'u', 'c', 'l', 'e', 'v', '1'};
+
+static GQuark
+rspamd_keypair_quark(void)
+{
+ return g_quark_from_static_string("rspamd-cryptobox-keypair");
+}
+
+/**
+ * Returns specific private key for different keypair types
+ */
+static void *
+rspamd_cryptobox_keypair_sk(struct rspamd_cryptobox_keypair *kp,
+ guint *len)
+{
+ g_assert(kp != NULL);
+
+ if (kp->alg == RSPAMD_CRYPTOBOX_MODE_25519) {
+ if (kp->type == RSPAMD_KEYPAIR_KEX) {
+ *len = 32;
+ return RSPAMD_CRYPTOBOX_KEYPAIR_25519(kp)->sk;
+ }
+ else {
+ *len = 64;
+ return RSPAMD_CRYPTOBOX_KEYPAIR_SIG_25519(kp)->sk;
+ }
+ }
+ else {
+ if (kp->type == RSPAMD_KEYPAIR_KEX) {
+ *len = 32;
+ return RSPAMD_CRYPTOBOX_KEYPAIR_NIST(kp)->sk;
+ }
+ else {
+ *len = 32;
+ return RSPAMD_CRYPTOBOX_KEYPAIR_SIG_NIST(kp)->sk;
+ }
+ }
+
+ /* Not reached */
+ return NULL;
+}
+
+static void *
+rspamd_cryptobox_keypair_pk(struct rspamd_cryptobox_keypair *kp,
+ guint *len)
+{
+ g_assert(kp != NULL);
+
+ if (kp->alg == RSPAMD_CRYPTOBOX_MODE_25519) {
+ if (kp->type == RSPAMD_KEYPAIR_KEX) {
+ *len = 32;
+ return RSPAMD_CRYPTOBOX_KEYPAIR_25519(kp)->pk;
+ }
+ else {
+ *len = 32;
+ return RSPAMD_CRYPTOBOX_KEYPAIR_SIG_25519(kp)->pk;
+ }
+ }
+ else {
+ if (kp->type == RSPAMD_KEYPAIR_KEX) {
+ *len = 65;
+ return RSPAMD_CRYPTOBOX_KEYPAIR_NIST(kp)->pk;
+ }
+ else {
+ *len = 65;
+ return RSPAMD_CRYPTOBOX_KEYPAIR_SIG_NIST(kp)->pk;
+ }
+ }
+
+ /* Not reached */
+ return NULL;
+}
+
+static void *
+rspamd_cryptobox_pubkey_pk(const struct rspamd_cryptobox_pubkey *kp,
+ guint *len)
+{
+ g_assert(kp != NULL);
+
+ if (kp->alg == RSPAMD_CRYPTOBOX_MODE_25519) {
+ if (kp->type == RSPAMD_KEYPAIR_KEX) {
+ *len = 32;
+ return RSPAMD_CRYPTOBOX_PUBKEY_25519(kp)->pk;
+ }
+ else {
+ *len = 32;
+ return RSPAMD_CRYPTOBOX_PUBKEY_SIG_25519(kp)->pk;
+ }
+ }
+ else {
+ if (kp->type == RSPAMD_KEYPAIR_KEX) {
+ *len = 65;
+ return RSPAMD_CRYPTOBOX_PUBKEY_NIST(kp)->pk;
+ }
+ else {
+ *len = 65;
+ return RSPAMD_CRYPTOBOX_PUBKEY_SIG_NIST(kp)->pk;
+ }
+ }
+
+ /* Not reached */
+ return NULL;
+}
+
+static struct rspamd_cryptobox_keypair *
+rspamd_cryptobox_keypair_alloc(enum rspamd_cryptobox_keypair_type type,
+ enum rspamd_cryptobox_mode alg)
+{
+ struct rspamd_cryptobox_keypair *kp;
+ guint size = 0;
+
+ if (alg == RSPAMD_CRYPTOBOX_MODE_25519) {
+ if (type == RSPAMD_KEYPAIR_KEX) {
+ size = sizeof(struct rspamd_cryptobox_keypair_25519);
+ }
+ else {
+ size = sizeof(struct rspamd_cryptobox_keypair_sig_25519);
+ }
+ }
+ else {
+ if (type == RSPAMD_KEYPAIR_KEX) {
+ size = sizeof(struct rspamd_cryptobox_keypair_nist);
+ }
+ else {
+ size = sizeof(struct rspamd_cryptobox_keypair_sig_nist);
+ }
+ }
+
+ g_assert(size >= sizeof(*kp));
+
+ if (posix_memalign((void **) &kp, 32, size) != 0) {
+ abort();
+ }
+
+ memset(kp, 0, size);
+
+ return kp;
+}
+
+static struct rspamd_cryptobox_pubkey *
+rspamd_cryptobox_pubkey_alloc(enum rspamd_cryptobox_keypair_type type,
+ enum rspamd_cryptobox_mode alg)
+{
+ struct rspamd_cryptobox_pubkey *pk;
+ guint size = 0;
+
+ if (alg == RSPAMD_CRYPTOBOX_MODE_25519) {
+ if (type == RSPAMD_KEYPAIR_KEX) {
+ size = sizeof(struct rspamd_cryptobox_pubkey_25519);
+ }
+ else {
+ size = sizeof(struct rspamd_cryptobox_pubkey_sig_25519);
+ }
+ }
+ else {
+ if (type == RSPAMD_KEYPAIR_KEX) {
+ size = sizeof(struct rspamd_cryptobox_pubkey_nist);
+ }
+ else {
+ size = sizeof(struct rspamd_cryptobox_pubkey_sig_nist);
+ }
+ }
+
+ g_assert(size >= sizeof(*pk));
+
+ if (posix_memalign((void **) &pk, 32, size) != 0) {
+ abort();
+ }
+
+ memset(pk, 0, size);
+
+ return pk;
+}
+
+
+void rspamd_cryptobox_nm_dtor(struct rspamd_cryptobox_nm *nm)
+{
+ rspamd_explicit_memzero(nm->nm, sizeof(nm->nm));
+ free(nm);
+}
+
+void rspamd_cryptobox_keypair_dtor(struct rspamd_cryptobox_keypair *kp)
+{
+ void *sk;
+ guint len = 0;
+
+ sk = rspamd_cryptobox_keypair_sk(kp, &len);
+ g_assert(sk != NULL && len > 0);
+ rspamd_explicit_memzero(sk, len);
+
+ if (kp->extensions) {
+ ucl_object_unref(kp->extensions);
+ }
+
+ /* Not g_free as kp is aligned using posix_memalign */
+ free(kp);
+}
+
+void rspamd_cryptobox_pubkey_dtor(struct rspamd_cryptobox_pubkey *p)
+{
+ if (p->nm) {
+ REF_RELEASE(p->nm);
+ }
+
+ /* Not g_free as p is aligned using posix_memalign */
+ free(p);
+}
+
+struct rspamd_cryptobox_keypair *
+rspamd_keypair_new(enum rspamd_cryptobox_keypair_type type,
+ enum rspamd_cryptobox_mode alg)
+{
+ struct rspamd_cryptobox_keypair *kp;
+ void *pk, *sk;
+ guint size;
+
+ kp = rspamd_cryptobox_keypair_alloc(type, alg);
+ kp->alg = alg;
+ kp->type = type;
+
+ sk = rspamd_cryptobox_keypair_sk(kp, &size);
+ pk = rspamd_cryptobox_keypair_pk(kp, &size);
+
+ if (type == RSPAMD_KEYPAIR_KEX) {
+ rspamd_cryptobox_keypair(pk, sk, alg);
+ }
+ else {
+ rspamd_cryptobox_keypair_sig(pk, sk, alg);
+ }
+
+ rspamd_cryptobox_hash(kp->id, pk, size, NULL, 0);
+
+ REF_INIT_RETAIN(kp, rspamd_cryptobox_keypair_dtor);
+
+ return kp;
+}
+
+
+struct rspamd_cryptobox_keypair *
+rspamd_keypair_ref(struct rspamd_cryptobox_keypair *kp)
+{
+ REF_RETAIN(kp);
+ return kp;
+}
+
+
+void rspamd_keypair_unref(struct rspamd_cryptobox_keypair *kp)
+{
+ REF_RELEASE(kp);
+}
+
+
+struct rspamd_cryptobox_pubkey *
+rspamd_pubkey_ref(struct rspamd_cryptobox_pubkey *kp)
+{
+ REF_RETAIN(kp);
+ return kp;
+}
+
+void rspamd_pubkey_unref(struct rspamd_cryptobox_pubkey *kp)
+{
+ REF_RELEASE(kp);
+}
+
+enum rspamd_cryptobox_keypair_type
+rspamd_keypair_type(struct rspamd_cryptobox_keypair *kp)
+{
+ g_assert(kp != NULL);
+
+ return kp->type;
+}
+
+enum rspamd_cryptobox_keypair_type
+rspamd_pubkey_type(struct rspamd_cryptobox_pubkey *p)
+{
+ g_assert(p != NULL);
+
+ return p->type;
+}
+
+
+enum rspamd_cryptobox_mode
+rspamd_keypair_alg(struct rspamd_cryptobox_keypair *kp)
+{
+ g_assert(kp != NULL);
+
+ return kp->alg;
+}
+
+enum rspamd_cryptobox_mode
+rspamd_pubkey_alg(struct rspamd_cryptobox_pubkey *p)
+{
+ g_assert(p != NULL);
+
+ return p->alg;
+}
+
+struct rspamd_cryptobox_pubkey *
+rspamd_pubkey_from_base32(const gchar *b32,
+ gsize len,
+ enum rspamd_cryptobox_keypair_type type,
+ enum rspamd_cryptobox_mode alg)
+{
+ guchar *decoded;
+ gsize dlen, expected_len;
+ guint pklen;
+ struct rspamd_cryptobox_pubkey *pk;
+ guchar *pk_data;
+
+ g_assert(b32 != NULL);
+
+ if (len == 0) {
+ len = strlen(b32);
+ }
+
+ decoded = rspamd_decode_base32(b32, len, &dlen, RSPAMD_BASE32_DEFAULT);
+
+ if (decoded == NULL) {
+ return NULL;
+ }
+
+ expected_len = (type == RSPAMD_KEYPAIR_KEX) ? rspamd_cryptobox_pk_bytes(alg) : rspamd_cryptobox_pk_sig_bytes(alg);
+
+ if (dlen != expected_len) {
+ g_free(decoded);
+ return NULL;
+ }
+
+ pk = rspamd_cryptobox_pubkey_alloc(type, alg);
+ REF_INIT_RETAIN(pk, rspamd_cryptobox_pubkey_dtor);
+ pk->alg = alg;
+ pk->type = type;
+ pk_data = rspamd_cryptobox_pubkey_pk(pk, &pklen);
+
+ memcpy(pk_data, decoded, pklen);
+ g_free(decoded);
+ rspamd_cryptobox_hash(pk->id, pk_data, pklen, NULL, 0);
+
+ return pk;
+}
+
+struct rspamd_cryptobox_pubkey *
+rspamd_pubkey_from_hex(const gchar *hex,
+ gsize len,
+ enum rspamd_cryptobox_keypair_type type,
+ enum rspamd_cryptobox_mode alg)
+{
+ guchar *decoded;
+ gsize dlen, expected_len;
+ guint pklen;
+ struct rspamd_cryptobox_pubkey *pk;
+ guchar *pk_data;
+
+ g_assert(hex != NULL);
+
+ if (len == 0) {
+ len = strlen(hex);
+ }
+
+ dlen = len / 2;
+
+ decoded = rspamd_decode_hex(hex, len);
+
+ if (decoded == NULL) {
+ return NULL;
+ }
+
+ expected_len = (type == RSPAMD_KEYPAIR_KEX) ? rspamd_cryptobox_pk_bytes(alg) : rspamd_cryptobox_pk_sig_bytes(alg);
+
+ if (dlen != expected_len) {
+ g_free(decoded);
+ return NULL;
+ }
+
+ pk = rspamd_cryptobox_pubkey_alloc(type, alg);
+ REF_INIT_RETAIN(pk, rspamd_cryptobox_pubkey_dtor);
+ pk->alg = alg;
+ pk->type = type;
+ pk_data = rspamd_cryptobox_pubkey_pk(pk, &pklen);
+
+ memcpy(pk_data, decoded, pklen);
+ g_free(decoded);
+ rspamd_cryptobox_hash(pk->id, pk_data, pklen, NULL, 0);
+
+ return pk;
+}
+
+struct rspamd_cryptobox_pubkey *
+rspamd_pubkey_from_bin(const guchar *raw,
+ gsize len,
+ enum rspamd_cryptobox_keypair_type type,
+ enum rspamd_cryptobox_mode alg)
+{
+ gsize expected_len;
+ guint pklen;
+ struct rspamd_cryptobox_pubkey *pk;
+ guchar *pk_data;
+
+ g_assert(raw != NULL && len > 0);
+
+ expected_len = (type == RSPAMD_KEYPAIR_KEX) ? rspamd_cryptobox_pk_bytes(alg) : rspamd_cryptobox_pk_sig_bytes(alg);
+
+ if (len != expected_len) {
+ return NULL;
+ }
+
+ pk = rspamd_cryptobox_pubkey_alloc(type, alg);
+ REF_INIT_RETAIN(pk, rspamd_cryptobox_pubkey_dtor);
+ pk->alg = alg;
+ pk->type = type;
+ pk_data = rspamd_cryptobox_pubkey_pk(pk, &pklen);
+
+ memcpy(pk_data, raw, pklen);
+ rspamd_cryptobox_hash(pk->id, pk_data, pklen, NULL, 0);
+
+ return pk;
+}
+
+
+const guchar *
+rspamd_pubkey_get_nm(struct rspamd_cryptobox_pubkey *p,
+ struct rspamd_cryptobox_keypair *kp)
+{
+ g_assert(p != NULL);
+
+ if (p->nm) {
+ if (memcmp(kp->id, (const guchar *) &p->nm->sk_id, sizeof(guint64)) == 0) {
+ return p->nm->nm;
+ }
+
+ /* Wrong ID, need to recalculate */
+ REF_RELEASE(p->nm);
+ p->nm = NULL;
+ }
+
+ return NULL;
+}
+
+const guchar *
+rspamd_pubkey_calculate_nm(struct rspamd_cryptobox_pubkey *p,
+ struct rspamd_cryptobox_keypair *kp)
+{
+ g_assert(kp->alg == p->alg);
+ g_assert(kp->type == p->type);
+ g_assert(p->type == RSPAMD_KEYPAIR_KEX);
+
+ if (p->nm == NULL) {
+ if (posix_memalign((void **) &p->nm, 32, sizeof(*p->nm)) != 0) {
+ abort();
+ }
+
+ memcpy(&p->nm->sk_id, kp->id, sizeof(guint64));
+ REF_INIT_RETAIN(p->nm, rspamd_cryptobox_nm_dtor);
+ }
+
+ if (kp->alg == RSPAMD_CRYPTOBOX_MODE_25519) {
+ struct rspamd_cryptobox_pubkey_25519 *rk_25519 =
+ RSPAMD_CRYPTOBOX_PUBKEY_25519(p);
+ struct rspamd_cryptobox_keypair_25519 *sk_25519 =
+ RSPAMD_CRYPTOBOX_KEYPAIR_25519(kp);
+
+ rspamd_cryptobox_nm(p->nm->nm, rk_25519->pk, sk_25519->sk, p->alg);
+ }
+ else {
+ struct rspamd_cryptobox_pubkey_nist *rk_nist =
+ RSPAMD_CRYPTOBOX_PUBKEY_NIST(p);
+ struct rspamd_cryptobox_keypair_nist *sk_nist =
+ RSPAMD_CRYPTOBOX_KEYPAIR_NIST(kp);
+
+ rspamd_cryptobox_nm(p->nm->nm, rk_nist->pk, sk_nist->sk, p->alg);
+ }
+
+ return p->nm->nm;
+}
+
+const guchar *
+rspamd_keypair_get_id(struct rspamd_cryptobox_keypair *kp)
+{
+ g_assert(kp != NULL);
+
+ return kp->id;
+}
+
+const ucl_object_t *
+rspamd_keypair_get_extensions(struct rspamd_cryptobox_keypair *kp)
+{
+ g_assert(kp != NULL);
+
+ return kp->extensions;
+}
+
+const guchar *
+rspamd_pubkey_get_id(struct rspamd_cryptobox_pubkey *pk)
+{
+ g_assert(pk != NULL);
+
+ return pk->id;
+}
+
+const guchar *
+rspamd_pubkey_get_pk(struct rspamd_cryptobox_pubkey *pk,
+ guint *len)
+{
+ guchar *ret = NULL;
+ guint rlen;
+
+ ret = rspamd_cryptobox_pubkey_pk(pk, &rlen);
+
+ if (len) {
+ *len = rlen;
+ }
+
+ return ret;
+}
+
+static void
+rspamd_keypair_print_component(guchar *data, gsize datalen,
+ GString *res, guint how, const gchar *description)
+{
+ gint olen, b32_len;
+
+ if (how & RSPAMD_KEYPAIR_HUMAN) {
+ rspamd_printf_gstring(res, "%s: ", description);
+ }
+
+ if (how & RSPAMD_KEYPAIR_BASE32) {
+ b32_len = (datalen * 8 / 5) + 2;
+ g_string_set_size(res, res->len + b32_len);
+ res->len -= b32_len;
+ olen = rspamd_encode_base32_buf(data, datalen, res->str + res->len,
+ res->len + b32_len - 1, RSPAMD_BASE32_DEFAULT);
+
+ if (olen > 0) {
+ res->len += olen;
+ res->str[res->len] = '\0';
+ }
+ }
+ else if (how & RSPAMD_KEYPAIR_HEX) {
+ rspamd_printf_gstring(res, "%*xs", (gint) datalen, data);
+ }
+ else {
+ g_string_append_len(res, data, datalen);
+ }
+
+ if (how & RSPAMD_KEYPAIR_HUMAN) {
+ g_string_append_c(res, '\n');
+ }
+}
+
+GString *
+rspamd_keypair_print(struct rspamd_cryptobox_keypair *kp, guint how)
+{
+ GString *res;
+ guint len;
+ gpointer p;
+
+ g_assert(kp != NULL);
+
+ res = g_string_sized_new(63);
+
+ if ((how & RSPAMD_KEYPAIR_PUBKEY)) {
+ p = rspamd_cryptobox_keypair_pk(kp, &len);
+ rspamd_keypair_print_component(p, len, res, how, "Public key");
+ }
+ if ((how & RSPAMD_KEYPAIR_PRIVKEY)) {
+ p = rspamd_cryptobox_keypair_sk(kp, &len);
+ rspamd_keypair_print_component(p, len, res, how, "Private key");
+ }
+ if ((how & RSPAMD_KEYPAIR_ID_SHORT)) {
+ rspamd_keypair_print_component(kp->id, RSPAMD_KEYPAIR_SHORT_ID_LEN,
+ res, how, "Short key ID");
+ }
+ if ((how & RSPAMD_KEYPAIR_ID)) {
+ rspamd_keypair_print_component(kp->id, sizeof(kp->id), res, how, "Key ID");
+ }
+
+ return res;
+}
+
+GString *
+rspamd_pubkey_print(struct rspamd_cryptobox_pubkey *pk, guint how)
+{
+ GString *res;
+ guint len;
+ gpointer p;
+
+ g_assert(pk != NULL);
+
+ res = g_string_sized_new(63);
+
+ if ((how & RSPAMD_KEYPAIR_PUBKEY)) {
+ p = rspamd_cryptobox_pubkey_pk(pk, &len);
+ rspamd_keypair_print_component(p, len, res, how, "Public key");
+ }
+ if ((how & RSPAMD_KEYPAIR_ID_SHORT)) {
+ rspamd_keypair_print_component(pk->id, RSPAMD_KEYPAIR_SHORT_ID_LEN,
+ res, how, "Short key ID");
+ }
+ if ((how & RSPAMD_KEYPAIR_ID)) {
+ rspamd_keypair_print_component(pk->id, sizeof(pk->id), res, how,
+ "Key ID");
+ }
+
+ return res;
+}
+
+const guchar *
+rspamd_keypair_component(struct rspamd_cryptobox_keypair *kp,
+ guint ncomp, guint *len)
+{
+ guint rlen = 0;
+ const guchar *ret = NULL;
+
+ g_assert(kp != NULL);
+
+ switch (ncomp) {
+ case RSPAMD_KEYPAIR_COMPONENT_ID:
+ rlen = sizeof(kp->id);
+ ret = kp->id;
+ break;
+ case RSPAMD_KEYPAIR_COMPONENT_PK:
+ ret = rspamd_cryptobox_keypair_pk(kp, &rlen);
+ break;
+ case RSPAMD_KEYPAIR_COMPONENT_SK:
+ ret = rspamd_cryptobox_keypair_sk(kp, &rlen);
+ break;
+ }
+
+ if (len) {
+ *len = rlen;
+ }
+
+ return ret;
+}
+
+struct rspamd_cryptobox_keypair *
+rspamd_keypair_from_ucl(const ucl_object_t *obj)
+{
+ const ucl_object_t *privkey, *pubkey, *elt;
+ const gchar *str;
+ enum rspamd_cryptobox_keypair_type type = RSPAMD_KEYPAIR_KEX;
+ enum rspamd_cryptobox_mode mode = RSPAMD_CRYPTOBOX_MODE_25519;
+ gboolean is_hex = FALSE;
+ struct rspamd_cryptobox_keypair *kp;
+ guint len;
+ gsize ucl_len;
+ gint dec_len;
+ gpointer target;
+
+ if (ucl_object_type(obj) != UCL_OBJECT) {
+ return NULL;
+ }
+
+ elt = ucl_object_lookup(obj, "keypair");
+ if (elt != NULL) {
+ obj = elt;
+ }
+
+ pubkey = ucl_object_lookup_any(obj, "pubkey", "public", "public_key",
+ NULL);
+ if (pubkey == NULL || ucl_object_type(pubkey) != UCL_STRING) {
+ return NULL;
+ }
+
+ privkey = ucl_object_lookup_any(obj, "privkey", "private", "private_key",
+ "secret", "secret_key", NULL);
+ if (privkey == NULL || ucl_object_type(privkey) != UCL_STRING) {
+ return NULL;
+ }
+
+ /* Optional fields */
+ elt = ucl_object_lookup(obj, "type");
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ str = ucl_object_tostring(elt);
+
+ if (g_ascii_strcasecmp(str, "kex") == 0) {
+ type = RSPAMD_KEYPAIR_KEX;
+ }
+ else if (g_ascii_strcasecmp(str, "sign") == 0) {
+ type = RSPAMD_KEYPAIR_SIGN;
+ }
+ /* TODO: handle errors */
+ }
+
+ elt = ucl_object_lookup(obj, "algorithm");
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ str = ucl_object_tostring(elt);
+
+ if (g_ascii_strcasecmp(str, "curve25519") == 0) {
+ mode = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else if (g_ascii_strcasecmp(str, "nistp256") == 0) {
+ mode = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ /* TODO: handle errors */
+ }
+
+ elt = ucl_object_lookup(obj, "encoding");
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ str = ucl_object_tostring(elt);
+
+ if (g_ascii_strcasecmp(str, "hex") == 0) {
+ is_hex = TRUE;
+ }
+ /* TODO: handle errors */
+ }
+
+ kp = rspamd_cryptobox_keypair_alloc(type, mode);
+ kp->type = type;
+ kp->alg = mode;
+ REF_INIT_RETAIN(kp, rspamd_cryptobox_keypair_dtor);
+ g_assert(kp != NULL);
+
+ target = rspamd_cryptobox_keypair_sk(kp, &len);
+ str = ucl_object_tolstring(privkey, &ucl_len);
+
+ if (is_hex) {
+ dec_len = rspamd_decode_hex_buf(str, ucl_len, target, len);
+ }
+ else {
+ dec_len = rspamd_decode_base32_buf(str, ucl_len, target, len, RSPAMD_BASE32_DEFAULT);
+ }
+
+ if (dec_len != (gint) len) {
+ rspamd_keypair_unref(kp);
+
+ return NULL;
+ }
+
+ target = rspamd_cryptobox_keypair_pk(kp, &len);
+ str = ucl_object_tolstring(pubkey, &ucl_len);
+
+ if (is_hex) {
+ dec_len = rspamd_decode_hex_buf(str, ucl_len, target, len);
+ }
+ else {
+ dec_len = rspamd_decode_base32_buf(str, ucl_len, target, len, RSPAMD_BASE32_DEFAULT);
+ }
+
+ if (dec_len != (gint) len) {
+ rspamd_keypair_unref(kp);
+
+ return NULL;
+ }
+
+ rspamd_cryptobox_hash(kp->id, target, len, NULL, 0);
+
+ elt = ucl_object_lookup(obj, "extensions");
+ if (elt && ucl_object_type(elt) == UCL_OBJECT) {
+ /* Use copy to avoid issues with the refcounts */
+ kp->extensions = ucl_object_copy(elt);
+ }
+
+ return kp;
+}
+
+ucl_object_t *
+rspamd_keypair_to_ucl(struct rspamd_cryptobox_keypair *kp,
+ enum rspamd_keypair_dump_flags flags)
+{
+ ucl_object_t *ucl_out, *elt;
+ gint how = 0;
+ GString *keypair_out;
+ const gchar *encoding;
+
+ g_assert(kp != NULL);
+
+ if (flags & RSPAMD_KEYPAIR_DUMP_HEX) {
+ how |= RSPAMD_KEYPAIR_HEX;
+ encoding = "hex";
+ }
+ else {
+ how |= RSPAMD_KEYPAIR_BASE32;
+ encoding = "base32";
+ }
+
+ if (flags & RSPAMD_KEYPAIR_DUMP_FLATTENED) {
+ ucl_out = ucl_object_typed_new(UCL_OBJECT);
+ elt = ucl_out;
+ }
+ else {
+ ucl_out = ucl_object_typed_new(UCL_OBJECT);
+ elt = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(ucl_out, elt, "keypair", 0, false);
+ }
+
+
+ /* pubkey part */
+ keypair_out = rspamd_keypair_print(kp,
+ RSPAMD_KEYPAIR_PUBKEY | how);
+ ucl_object_insert_key(elt,
+ ucl_object_fromlstring(keypair_out->str, keypair_out->len),
+ "pubkey", 0, false);
+ g_string_free(keypair_out, TRUE);
+
+ if (!(flags & RSPAMD_KEYPAIR_DUMP_NO_SECRET)) {
+ /* privkey part */
+ keypair_out = rspamd_keypair_print(kp,
+ RSPAMD_KEYPAIR_PRIVKEY | how);
+ ucl_object_insert_key(elt,
+ ucl_object_fromlstring(keypair_out->str, keypair_out->len),
+ "privkey", 0, false);
+ g_string_free(keypair_out, TRUE);
+ }
+
+ keypair_out = rspamd_keypair_print(kp,
+ RSPAMD_KEYPAIR_ID | how);
+ ucl_object_insert_key(elt,
+ ucl_object_fromlstring(keypair_out->str, keypair_out->len),
+ "id", 0, false);
+ g_string_free(keypair_out, TRUE);
+
+ ucl_object_insert_key(elt,
+ ucl_object_fromstring(encoding),
+ "encoding", 0, false);
+
+ ucl_object_insert_key(elt,
+ ucl_object_fromstring(
+ kp->alg == RSPAMD_CRYPTOBOX_MODE_NIST ? "nistp256" : "curve25519"),
+ "algorithm", 0, false);
+
+ ucl_object_insert_key(elt,
+ ucl_object_fromstring(
+ kp->type == RSPAMD_KEYPAIR_KEX ? "kex" : "sign"),
+ "type", 0, false);
+
+ if (kp->extensions) {
+ ucl_object_insert_key(elt, ucl_object_copy(kp->extensions),
+ "extensions", 0, false);
+ }
+
+ return ucl_out;
+}
+
+gboolean
+rspamd_keypair_decrypt(struct rspamd_cryptobox_keypair *kp,
+ const guchar *in, gsize inlen,
+ guchar **out, gsize *outlen,
+ GError **err)
+{
+ const guchar *nonce, *mac, *data, *pubkey;
+
+ g_assert(kp != NULL);
+ g_assert(in != NULL);
+
+ if (kp->type != RSPAMD_KEYPAIR_KEX) {
+ g_set_error(err, rspamd_keypair_quark(), EINVAL,
+ "invalid keypair type");
+
+ return FALSE;
+ }
+
+ if (inlen < sizeof(encrypted_magic) + rspamd_cryptobox_pk_bytes(kp->alg) +
+ rspamd_cryptobox_mac_bytes(kp->alg) +
+ rspamd_cryptobox_nonce_bytes(kp->alg)) {
+ g_set_error(err, rspamd_keypair_quark(), E2BIG, "invalid size: too small");
+
+ return FALSE;
+ }
+
+ if (memcmp(in, encrypted_magic, sizeof(encrypted_magic)) != 0) {
+ g_set_error(err, rspamd_keypair_quark(), EINVAL,
+ "invalid magic");
+
+ return FALSE;
+ }
+
+ /* Set pointers */
+ pubkey = in + sizeof(encrypted_magic);
+ mac = pubkey + rspamd_cryptobox_pk_bytes(kp->alg);
+ nonce = mac + rspamd_cryptobox_mac_bytes(kp->alg);
+ data = nonce + rspamd_cryptobox_nonce_bytes(kp->alg);
+
+ if (data - in >= inlen) {
+ g_set_error(err, rspamd_keypair_quark(), E2BIG, "invalid size: too small");
+
+ return FALSE;
+ }
+
+ inlen -= data - in;
+
+ /* Allocate memory for output */
+ *out = g_malloc(inlen);
+ memcpy(*out, data, inlen);
+
+ if (!rspamd_cryptobox_decrypt_inplace(*out, inlen, nonce, pubkey,
+ rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_SK, NULL),
+ mac, kp->alg)) {
+ g_set_error(err, rspamd_keypair_quark(), EPERM, "verification failed");
+ g_free(*out);
+
+ return FALSE;
+ }
+
+ if (outlen) {
+ *outlen = inlen;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_keypair_encrypt(struct rspamd_cryptobox_keypair *kp,
+ const guchar *in, gsize inlen,
+ guchar **out, gsize *outlen,
+ GError **err)
+{
+ guchar *nonce, *mac, *data, *pubkey;
+ struct rspamd_cryptobox_keypair *local;
+ gsize olen;
+
+ g_assert(kp != NULL);
+ g_assert(in != NULL);
+
+ if (kp->type != RSPAMD_KEYPAIR_KEX) {
+ g_set_error(err, rspamd_keypair_quark(), EINVAL,
+ "invalid keypair type");
+
+ return FALSE;
+ }
+
+ local = rspamd_keypair_new(kp->type, kp->alg);
+
+ olen = inlen + sizeof(encrypted_magic) +
+ rspamd_cryptobox_pk_bytes(kp->alg) +
+ rspamd_cryptobox_mac_bytes(kp->alg) +
+ rspamd_cryptobox_nonce_bytes(kp->alg);
+ *out = g_malloc(olen);
+ memcpy(*out, encrypted_magic, sizeof(encrypted_magic));
+ pubkey = *out + sizeof(encrypted_magic);
+ mac = pubkey + rspamd_cryptobox_pk_bytes(kp->alg);
+ nonce = mac + rspamd_cryptobox_mac_bytes(kp->alg);
+ data = nonce + rspamd_cryptobox_nonce_bytes(kp->alg);
+
+ ottery_rand_bytes(nonce, rspamd_cryptobox_nonce_bytes(kp->alg));
+ memcpy(data, in, inlen);
+ memcpy(pubkey, rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_PK, NULL),
+ rspamd_cryptobox_pk_bytes(kp->alg));
+ rspamd_cryptobox_encrypt_inplace(data, inlen, nonce, pubkey,
+ rspamd_keypair_component(local, RSPAMD_KEYPAIR_COMPONENT_SK, NULL),
+ mac, kp->alg);
+ rspamd_keypair_unref(local);
+
+ if (outlen) {
+ *outlen = olen;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_pubkey_encrypt(struct rspamd_cryptobox_pubkey *pk,
+ const guchar *in, gsize inlen,
+ guchar **out, gsize *outlen,
+ GError **err)
+{
+ guchar *nonce, *mac, *data, *pubkey;
+ struct rspamd_cryptobox_keypair *local;
+ gsize olen;
+
+ g_assert(pk != NULL);
+ g_assert(in != NULL);
+
+ if (pk->type != RSPAMD_KEYPAIR_KEX) {
+ g_set_error(err, rspamd_keypair_quark(), EINVAL,
+ "invalid pubkey type");
+
+ return FALSE;
+ }
+
+ local = rspamd_keypair_new(pk->type, pk->alg);
+
+ olen = inlen + sizeof(encrypted_magic) +
+ rspamd_cryptobox_pk_bytes(pk->alg) +
+ rspamd_cryptobox_mac_bytes(pk->alg) +
+ rspamd_cryptobox_nonce_bytes(pk->alg);
+ *out = g_malloc(olen);
+ memcpy(*out, encrypted_magic, sizeof(encrypted_magic));
+ pubkey = *out + sizeof(encrypted_magic);
+ mac = pubkey + rspamd_cryptobox_pk_bytes(pk->alg);
+ nonce = mac + rspamd_cryptobox_mac_bytes(pk->alg);
+ data = nonce + rspamd_cryptobox_nonce_bytes(pk->alg);
+
+ ottery_rand_bytes(nonce, rspamd_cryptobox_nonce_bytes(pk->alg));
+ memcpy(data, in, inlen);
+ memcpy(pubkey, rspamd_pubkey_get_pk(pk, NULL),
+ rspamd_cryptobox_pk_bytes(pk->alg));
+ rspamd_cryptobox_encrypt_inplace(data, inlen, nonce, pubkey,
+ rspamd_keypair_component(local, RSPAMD_KEYPAIR_COMPONENT_SK, NULL),
+ mac, pk->alg);
+ rspamd_keypair_unref(local);
+
+ if (outlen) {
+ *outlen = olen;
+ }
+
+ return TRUE;
+} \ No newline at end of file
diff --git a/src/libcryptobox/keypair.h b/src/libcryptobox/keypair.h
new file mode 100644
index 0000000..64461b7
--- /dev/null
+++ b/src/libcryptobox/keypair.h
@@ -0,0 +1,317 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBCRYPTOBOX_KEYPAIR_H_
+#define SRC_LIBCRYPTOBOX_KEYPAIR_H_
+
+#include "config.h"
+#include "cryptobox.h"
+#include "ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Keypair type
+ */
+enum rspamd_cryptobox_keypair_type {
+ RSPAMD_KEYPAIR_KEX = 0,
+ RSPAMD_KEYPAIR_SIGN
+};
+
+extern const guchar encrypted_magic[7];
+
+/**
+ * Opaque structure for the full (public + private) keypair
+ */
+struct rspamd_cryptobox_keypair;
+/**
+ * Opaque structure for public only keypair
+ */
+struct rspamd_cryptobox_pubkey;
+
+/**
+ * Creates new full keypair
+ * @param type type of the keypair
+ * @param alg algorithm for the keypair
+ * @return fresh keypair generated
+ */
+struct rspamd_cryptobox_keypair *rspamd_keypair_new(
+ enum rspamd_cryptobox_keypair_type type,
+ enum rspamd_cryptobox_mode alg);
+
+/**
+ * Increase refcount for the specific keypair
+ * @param kp
+ * @return
+ */
+struct rspamd_cryptobox_keypair *rspamd_keypair_ref(
+ struct rspamd_cryptobox_keypair *kp);
+
+/**
+ * Decrease refcount for the specific keypair (or destroy when refcount == 0)
+ * @param kp
+ */
+void rspamd_keypair_unref(struct rspamd_cryptobox_keypair *kp);
+
+/**
+ * Increase refcount for the specific pubkey
+ * @param kp
+ * @return
+ */
+struct rspamd_cryptobox_pubkey *rspamd_pubkey_ref(
+ struct rspamd_cryptobox_pubkey *kp);
+
+/**
+ * Load pubkey from base32 string
+ * @param b32 input string
+ * @param type type of key (signing or kex)
+ * @param alg algorithm of the key (nist or curve25519)
+ * @return new pubkey or NULL in case of error
+ */
+struct rspamd_cryptobox_pubkey *rspamd_pubkey_from_base32(const gchar *b32,
+ gsize len,
+ enum rspamd_cryptobox_keypair_type type,
+ enum rspamd_cryptobox_mode alg);
+
+/**
+ * Load pubkey from hex string
+ * @param hex input string
+ * @param type type of key (signing or kex)
+ * @param alg algorithm of the key (nist or curve25519)
+ * @return new pubkey or NULL in case of error
+ */
+struct rspamd_cryptobox_pubkey *rspamd_pubkey_from_hex(const gchar *hex,
+ gsize len,
+ enum rspamd_cryptobox_keypair_type type,
+ enum rspamd_cryptobox_mode alg);
+
+/**
+ * Load pubkey from raw chunk string
+ * @param hex input data
+ * @param type type of key (signing or kex)
+ * @param alg algorithm of the key (nist or curve25519)
+ * @return new pubkey or NULL in case of error
+ */
+struct rspamd_cryptobox_pubkey *rspamd_pubkey_from_bin(const guchar *raw,
+ gsize len,
+ enum rspamd_cryptobox_keypair_type type,
+ enum rspamd_cryptobox_mode alg);
+
+
+/**
+ * Decrease refcount for the specific pubkey (or destroy when refcount == 0)
+ * @param kp
+ */
+void rspamd_pubkey_unref(struct rspamd_cryptobox_pubkey *kp);
+
+/**
+ * Get type of keypair
+ */
+enum rspamd_cryptobox_keypair_type rspamd_keypair_type(
+ struct rspamd_cryptobox_keypair *kp);
+
+/**
+ * Get type of pubkey
+ */
+enum rspamd_cryptobox_keypair_type rspamd_pubkey_type(
+ struct rspamd_cryptobox_pubkey *p);
+
+/**
+ * Get algorithm of keypair
+ */
+enum rspamd_cryptobox_mode rspamd_keypair_alg(struct rspamd_cryptobox_keypair *kp);
+
+/**
+ * Get algorithm of pubkey
+ */
+enum rspamd_cryptobox_mode rspamd_pubkey_alg(struct rspamd_cryptobox_pubkey *p);
+
+/**
+ * Get cached NM for this specific pubkey
+ * @param p
+ * @return
+ */
+const guchar *rspamd_pubkey_get_nm(struct rspamd_cryptobox_pubkey *p,
+ struct rspamd_cryptobox_keypair *kp);
+
+/**
+ * Calculate and store nm value for the specified local key (performs ECDH)
+ * @param p
+ * @return
+ */
+const guchar *rspamd_pubkey_calculate_nm(struct rspamd_cryptobox_pubkey *p,
+ struct rspamd_cryptobox_keypair *kp);
+
+/**
+ * Get raw public key id for a specified keypair (rspamd_cryptobox_HASHBYTES)
+ * @param kp
+ * @return
+ */
+const guchar *rspamd_keypair_get_id(struct rspamd_cryptobox_keypair *kp);
+
+/**
+ * Returns keypair extensions if any
+ * @param kp
+ * @return
+ */
+const ucl_object_t *rspamd_keypair_get_extensions(struct rspamd_cryptobox_keypair *kp);
+
+/**
+ * Get raw public key id for a specified key (rspamd_cryptobox_HASHBYTES)
+ * @param kp
+ * @return
+ */
+const guchar *rspamd_pubkey_get_id(struct rspamd_cryptobox_pubkey *pk);
+
+/**
+ * Get raw public key from pubkey opaque structure
+ * @param pk
+ * @param len
+ * @return
+ */
+const guchar *rspamd_pubkey_get_pk(struct rspamd_cryptobox_pubkey *pk,
+ guint *len);
+
+/** Short ID characters count */
+#define RSPAMD_KEYPAIR_SHORT_ID_LEN 5
+/** Print pubkey */
+#define RSPAMD_KEYPAIR_PUBKEY 0x1
+/** Print secret key */
+#define RSPAMD_KEYPAIR_PRIVKEY 0x2
+/** Print key id */
+#define RSPAMD_KEYPAIR_ID 0x4
+/** Print short key id */
+#define RSPAMD_KEYPAIR_ID_SHORT 0x8
+/** Encode output with base 32 */
+#define RSPAMD_KEYPAIR_BASE32 0x10
+/** Human readable output */
+#define RSPAMD_KEYPAIR_HUMAN 0x20
+#define RSPAMD_KEYPAIR_HEX 0x40
+
+/**
+ * Print keypair encoding it if needed
+ * @param key key to print
+ * @param how flags that specifies printing behaviour
+ * @return newly allocated string with keypair
+ */
+GString *rspamd_keypair_print(struct rspamd_cryptobox_keypair *kp,
+ guint how);
+
+/**
+ * Print pubkey encoding it if needed
+ * @param key key to print
+ * @param how flags that specifies printing behaviour
+ * @return newly allocated string with keypair
+ */
+GString *rspamd_pubkey_print(struct rspamd_cryptobox_pubkey *pk,
+ guint how);
+
+/** Get keypair pubkey ID */
+#define RSPAMD_KEYPAIR_COMPONENT_ID 0
+/** Get keypair public key */
+#define RSPAMD_KEYPAIR_COMPONENT_PK 1
+/** Get keypair private key */
+#define RSPAMD_KEYPAIR_COMPONENT_SK 2
+
+/**
+ * Get specific component of a keypair
+ * @param kp keypair
+ * @param ncomp component number
+ * @param len length of input
+ * @return raw content of the component
+ */
+const guchar *rspamd_keypair_component(struct rspamd_cryptobox_keypair *kp,
+ guint ncomp, guint *len);
+
+/**
+ * Create a new keypair from ucl object
+ * @param obj object to load
+ * @return new structure or NULL if an object is invalid
+ */
+struct rspamd_cryptobox_keypair *rspamd_keypair_from_ucl(const ucl_object_t *obj);
+
+
+enum rspamd_keypair_dump_flags {
+ RSPAMD_KEYPAIR_DUMP_DEFAULT = 0,
+ RSPAMD_KEYPAIR_DUMP_HEX = 1u << 0u,
+ RSPAMD_KEYPAIR_DUMP_NO_SECRET = 1u << 1u,
+ RSPAMD_KEYPAIR_DUMP_FLATTENED = 1u << 2u,
+};
+
+/**
+ * Converts keypair to ucl object
+ * @param kp
+ * @return
+ */
+ucl_object_t *rspamd_keypair_to_ucl(struct rspamd_cryptobox_keypair *kp,
+ enum rspamd_keypair_dump_flags flags);
+
+
+/**
+ * Decrypts data using keypair and a pubkey stored in in, in must start from
+ * `encrypted_magic` constant
+ * @param kp keypair
+ * @param in raw input
+ * @param inlen input length
+ * @param out output (allocated internally using g_malloc)
+ * @param outlen output size
+ * @return TRUE if decryption is completed, out must be freed in this case
+ */
+gboolean rspamd_keypair_decrypt(struct rspamd_cryptobox_keypair *kp,
+ const guchar *in, gsize inlen,
+ guchar **out, gsize *outlen,
+ GError **err);
+
+/**
+ * Encrypts data usign specific keypair.
+ * This method actually generates ephemeral local keypair, use public key from
+ * the remote keypair and encrypts data
+ * @param kp keypair
+ * @param in raw input
+ * @param inlen input length
+ * @param out output (allocated internally using g_malloc)
+ * @param outlen output size
+ * @param err pointer to error
+ * @return TRUE if encryption has been completed, out must be freed in this case
+ */
+gboolean rspamd_keypair_encrypt(struct rspamd_cryptobox_keypair *kp,
+ const guchar *in, gsize inlen,
+ guchar **out, gsize *outlen,
+ GError **err);
+
+/**
+ * Encrypts data usign specific pubkey (must have KEX type).
+ * This method actually generates ephemeral local keypair, use public key from
+ * the remote keypair and encrypts data
+ * @param kp keypair
+ * @param in raw input
+ * @param inlen input length
+ * @param out output (allocated internally using g_malloc)
+ * @param outlen output size
+ * @param err pointer to error
+ * @return TRUE if encryption has been completed, out must be freed in this case
+ */
+gboolean rspamd_pubkey_encrypt(struct rspamd_cryptobox_pubkey *pk,
+ const guchar *in, gsize inlen,
+ guchar **out, gsize *outlen,
+ GError **err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBCRYPTOBOX_KEYPAIR_H_ */
diff --git a/src/libcryptobox/keypair_private.h b/src/libcryptobox/keypair_private.h
new file mode 100644
index 0000000..16e17e0
--- /dev/null
+++ b/src/libcryptobox/keypair_private.h
@@ -0,0 +1,143 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef KEYPAIR_PRIVATE_H_
+#define KEYPAIR_PRIVATE_H_
+
+#include "config.h"
+#include "ref.h"
+#include "cryptobox.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * KEX cached data
+ */
+struct rspamd_cryptobox_nm {
+ guchar nm[rspamd_cryptobox_MAX_NMBYTES];
+ guint64 sk_id; /* Used to store secret key id */
+ ref_entry_t ref;
+};
+
+/*
+ * Generic keypair
+ */
+struct rspamd_cryptobox_keypair {
+ guchar id[rspamd_cryptobox_HASHBYTES];
+ enum rspamd_cryptobox_keypair_type type;
+ enum rspamd_cryptobox_mode alg;
+ ucl_object_t *extensions;
+ ref_entry_t ref;
+};
+
+/*
+ * NIST p256 ecdh keypair
+ */
+#define RSPAMD_CRYPTOBOX_KEYPAIR_NIST(x) ((struct rspamd_cryptobox_keypair_nist *) (x))
+struct rspamd_cryptobox_keypair_nist {
+ struct rspamd_cryptobox_keypair parent;
+ guchar sk[32];
+ guchar pk[65];
+};
+
+/*
+ * Curve25519 ecdh keypair
+ */
+#define RSPAMD_CRYPTOBOX_KEYPAIR_25519(x) ((struct rspamd_cryptobox_keypair_25519 *) (x))
+struct rspamd_cryptobox_keypair_25519 {
+ struct rspamd_cryptobox_keypair parent;
+ guchar sk[32];
+ guchar pk[32];
+};
+
+/*
+ * NIST p256 ecdsa keypair
+ */
+#define RSPAMD_CRYPTOBOX_KEYPAIR_SIG_NIST(x) ((struct rspamd_cryptobox_keypair_sig_nist *) (x))
+struct rspamd_cryptobox_keypair_sig_nist {
+ struct rspamd_cryptobox_keypair parent;
+ guchar sk[32];
+ guchar pk[65];
+};
+
+/*
+ * Ed25519 keypair
+ */
+#define RSPAMD_CRYPTOBOX_KEYPAIR_SIG_25519(x) ((struct rspamd_cryptobox_keypair_sig_25519 *) (x))
+struct rspamd_cryptobox_keypair_sig_25519 {
+ struct rspamd_cryptobox_keypair parent;
+ guchar sk[64];
+ guchar pk[32];
+};
+
+/*
+ * Public component of the keypair
+ */
+struct rspamd_cryptobox_pubkey {
+ guchar id[rspamd_cryptobox_HASHBYTES];
+ struct rspamd_cryptobox_nm *nm;
+ enum rspamd_cryptobox_keypair_type type;
+ enum rspamd_cryptobox_mode alg;
+ ref_entry_t ref;
+};
+
+/*
+ * Public p256 ecdh
+ */
+#define RSPAMD_CRYPTOBOX_PUBKEY_NIST(x) ((struct rspamd_cryptobox_pubkey_nist *) (x))
+struct rspamd_cryptobox_pubkey_nist {
+ struct rspamd_cryptobox_pubkey parent;
+ guchar pk[65];
+};
+
+/*
+ * Public curve25519 ecdh
+ */
+#define RSPAMD_CRYPTOBOX_PUBKEY_25519(x) ((struct rspamd_cryptobox_pubkey_25519 *) (x))
+struct rspamd_cryptobox_pubkey_25519 {
+ struct rspamd_cryptobox_pubkey parent;
+ guchar pk[32];
+};
+
+/*
+ * Public p256 ecdsa
+ */
+#define RSPAMD_CRYPTOBOX_PUBKEY_SIG_NIST(x) ((struct rspamd_cryptobox_pubkey_sig_nist *) (x))
+struct rspamd_cryptobox_pubkey_sig_nist {
+ struct rspamd_cryptobox_pubkey parent;
+ guchar pk[65];
+};
+
+/*
+ * Public ed25519
+ */
+#define RSPAMD_CRYPTOBOX_PUBKEY_SIG_25519(x) ((struct rspamd_cryptobox_pubkey_sig_25519 *) (x))
+struct rspamd_cryptobox_pubkey_sig_25519 {
+ struct rspamd_cryptobox_pubkey parent;
+ guchar pk[32];
+};
+
+void rspamd_cryptobox_nm_dtor(struct rspamd_cryptobox_nm *nm);
+
+void rspamd_cryptobox_keypair_dtor(struct rspamd_cryptobox_keypair *kp);
+
+void rspamd_cryptobox_pubkey_dtor(struct rspamd_cryptobox_pubkey *p);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* KEYPAIR_PRIVATE_H_ */
diff --git a/src/libcryptobox/keypairs_cache.c b/src/libcryptobox/keypairs_cache.c
new file mode 100644
index 0000000..0616bb9
--- /dev/null
+++ b/src/libcryptobox/keypairs_cache.c
@@ -0,0 +1,141 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "keypairs_cache.h"
+#include "keypair_private.h"
+#include "libutil/util.h"
+#include "hash.h"
+
+struct rspamd_keypair_elt {
+ struct rspamd_cryptobox_nm *nm;
+ guchar pair[rspamd_cryptobox_HASHBYTES * 2];
+};
+
+struct rspamd_keypair_cache {
+ rspamd_lru_hash_t *hash;
+};
+
+static void
+rspamd_keypair_destroy(gpointer ptr)
+{
+ struct rspamd_keypair_elt *elt = (struct rspamd_keypair_elt *) ptr;
+
+ REF_RELEASE(elt->nm);
+ g_free(elt);
+}
+
+static guint
+rspamd_keypair_hash(gconstpointer ptr)
+{
+ struct rspamd_keypair_elt *elt = (struct rspamd_keypair_elt *) ptr;
+
+ return rspamd_cryptobox_fast_hash(elt->pair, sizeof(elt->pair),
+ rspamd_hash_seed());
+}
+
+static gboolean
+rspamd_keypair_equal(gconstpointer p1, gconstpointer p2)
+{
+ struct rspamd_keypair_elt *e1 = (struct rspamd_keypair_elt *) p1,
+ *e2 = (struct rspamd_keypair_elt *) p2;
+
+ return memcmp(e1->pair, e2->pair, sizeof(e1->pair)) == 0;
+}
+
+struct rspamd_keypair_cache *
+rspamd_keypair_cache_new(guint max_items)
+{
+ struct rspamd_keypair_cache *c;
+
+ g_assert(max_items > 0);
+
+ c = g_malloc0(sizeof(*c));
+ c->hash = rspamd_lru_hash_new_full(max_items, NULL,
+ rspamd_keypair_destroy, rspamd_keypair_hash, rspamd_keypair_equal);
+
+ return c;
+}
+
+void rspamd_keypair_cache_process(struct rspamd_keypair_cache *c,
+ struct rspamd_cryptobox_keypair *lk,
+ struct rspamd_cryptobox_pubkey *rk)
+{
+ struct rspamd_keypair_elt search, *new;
+
+ g_assert(lk != NULL);
+ g_assert(rk != NULL);
+ g_assert(rk->alg == lk->alg);
+ g_assert(rk->type == lk->type);
+ g_assert(rk->type == RSPAMD_KEYPAIR_KEX);
+
+ memset(&search, 0, sizeof(search));
+ memcpy(search.pair, rk->id, rspamd_cryptobox_HASHBYTES);
+ memcpy(&search.pair[rspamd_cryptobox_HASHBYTES], lk->id,
+ rspamd_cryptobox_HASHBYTES);
+ new = rspamd_lru_hash_lookup(c->hash, &search, time(NULL));
+
+ if (rk->nm) {
+ REF_RELEASE(rk->nm);
+ rk->nm = NULL;
+ }
+
+ if (new == NULL) {
+ new = g_malloc0(sizeof(*new));
+
+ if (posix_memalign((void **) &new->nm, 32, sizeof(*new->nm)) != 0) {
+ abort();
+ }
+
+ REF_INIT_RETAIN(new->nm, rspamd_cryptobox_nm_dtor);
+
+ memcpy(new->pair, rk->id, rspamd_cryptobox_HASHBYTES);
+ memcpy(&new->pair[rspamd_cryptobox_HASHBYTES], lk->id,
+ rspamd_cryptobox_HASHBYTES);
+ memcpy(&new->nm->sk_id, lk->id, sizeof(guint64));
+
+ if (rk->alg == RSPAMD_CRYPTOBOX_MODE_25519) {
+ struct rspamd_cryptobox_pubkey_25519 *rk_25519 =
+ RSPAMD_CRYPTOBOX_PUBKEY_25519(rk);
+ struct rspamd_cryptobox_keypair_25519 *sk_25519 =
+ RSPAMD_CRYPTOBOX_KEYPAIR_25519(lk);
+
+ rspamd_cryptobox_nm(new->nm->nm, rk_25519->pk, sk_25519->sk, rk->alg);
+ }
+ else {
+ struct rspamd_cryptobox_pubkey_nist *rk_nist =
+ RSPAMD_CRYPTOBOX_PUBKEY_NIST(rk);
+ struct rspamd_cryptobox_keypair_nist *sk_nist =
+ RSPAMD_CRYPTOBOX_KEYPAIR_NIST(lk);
+
+ rspamd_cryptobox_nm(new->nm->nm, rk_nist->pk, sk_nist->sk, rk->alg);
+ }
+
+ rspamd_lru_hash_insert(c->hash, new, new, time(NULL), -1);
+ }
+
+ g_assert(new != NULL);
+
+ rk->nm = new->nm;
+ REF_RETAIN(rk->nm);
+}
+
+void rspamd_keypair_cache_destroy(struct rspamd_keypair_cache *c)
+{
+ if (c != NULL) {
+ rspamd_lru_hash_destroy(c->hash);
+ g_free(c);
+ }
+}
diff --git a/src/libcryptobox/keypairs_cache.h b/src/libcryptobox/keypairs_cache.h
new file mode 100644
index 0000000..96e356a
--- /dev/null
+++ b/src/libcryptobox/keypairs_cache.h
@@ -0,0 +1,57 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef KEYPAIRS_CACHE_H_
+#define KEYPAIRS_CACHE_H_
+
+#include "config.h"
+#include "keypair.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_keypair_cache;
+
+/**
+ * Create new keypair cache of the specified size
+ * @param max_items defines maximum count of elements in the cache
+ * @return new cache
+ */
+struct rspamd_keypair_cache *rspamd_keypair_cache_new(guint max_items);
+
+
+/**
+ * Process local and remote keypair setting beforenm value as appropriate
+ * @param c cache of keypairs
+ * @param lk local key
+ * @param rk remote key
+ */
+void rspamd_keypair_cache_process(struct rspamd_keypair_cache *c,
+ struct rspamd_cryptobox_keypair *lk,
+ struct rspamd_cryptobox_pubkey *rk);
+
+/**
+ * Destroy old keypair cache
+ * @param c cache object
+ */
+void rspamd_keypair_cache_destroy(struct rspamd_keypair_cache *c);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* KEYPAIRS_CACHE_H_ */
diff --git a/src/libcryptobox/macro.S b/src/libcryptobox/macro.S
new file mode 100644
index 0000000..f213f15
--- /dev/null
+++ b/src/libcryptobox/macro.S
@@ -0,0 +1,176 @@
+#include "platform_config.h"
+
+#if !defined(HAVE_SLASHMACRO) && !defined(HAVE_DOLLARMACRO)
+ #error Unknown gnu as macro parameter convention! Run ./configure
+#endif
+
+#if defined(__MACH__)
+ .macro FN name
+ #if defined(HAVE_SLASHMACRO)
+ \name:
+ _\name:
+ #elif defined(HAVE_DOLLARMACRO)
+ $0:
+ _$0:
+ #endif
+ .endm
+
+ .macro FN_EXT name, args, xmmused
+ #if defined(HAVE_SLASHMACRO)
+ FN \name
+ #elif defined(HAVE_DOLLARMACRO)
+ FN $0
+ #endif
+ .endm
+
+ .macro FN_END name
+ .endm
+
+ .macro HIDDEN name
+ #if defined(HAVE_AS_PRIVATE_EXTERN)
+ #if defined(HAVE_SLASHMACRO)
+ .private_extern \name
+ .private_extern _\name
+ #elif defined(HAVE_DOLLARMACRO)
+ .private_extern $0
+ .private_extern _$0
+ #endif
+ #endif
+ .endm
+#else
+ .macro FN name
+ \name:
+ _\name:
+ .endm
+
+ .macro FN_EXT name, args, xmmused
+ FN \name
+ .endm
+
+ .macro FN_END name
+ .size \name, .-\name
+ .size _\name, .-_\name
+ .type \name, @function
+ .type _\name, @function
+ .endm
+
+ .macro HIDDEN name
+ #if defined(HAVE_AS_HIDDEN)
+ .hidden \name
+ .hidden _\name
+ #endif
+ .endm
+
+ /* set NX for stack */
+ .section .note.GNU-stack,"",@progbits
+#endif
+#if defined(__MACH__)
+ .macro SECTION_TEXT
+ .section __TEXT,__text,regular
+ .endm
+
+ .macro SECTION_RODATA
+ .section __TEXT,__text,regular
+ .endm
+#else
+ /* put everything in the code segment to simplify things */
+ .macro SECTION_TEXT
+ .text
+ .endm
+
+ .macro SECTION_RODATA
+ .text
+ .endm
+#endif
+
+/* declare a global function */
+.macro GLOBAL name
+#if defined(HAVE_SLASHMACRO)
+ .globl \name
+ .globl _\name
+#elif defined(HAVE_DOLLARMACRO)
+ .globl $0
+ .globl _$0
+#endif
+.endm
+
+.macro FN_LOCAL_PREFIX name
+#if defined(HAVE_SLASHMACRO)
+ FN LOCAL_PREFIX(\name)
+#elif defined(HAVE_DOLLARMACRO)
+ FN LOCAL_PREFIX($0)
+#endif
+.endm
+
+.macro FN_EXT_LOCAL_PREFIX name, args, xmmused
+#if defined(HAVE_SLASHMACRO)
+ FN_EXT LOCAL_PREFIX(\name), \args, \xmmused
+#elif defined(HAVE_DOLLARMACRO)
+ FN_EXT LOCAL_PREFIX($0), $1, $2
+#endif
+.endm
+
+.macro FN_END_LOCAL_PREFIX name
+#if defined(HAVE_SLASHMACRO)
+ FN_END LOCAL_PREFIX(\name)
+#elif defined(HAVE_DOLLARMACRO)
+ FN_END LOCAL_PREFIX($0)
+#endif
+.endm
+
+.macro GLOBAL_LOCAL_PREFIX name
+#if defined(HAVE_SLASHMACRO)
+ GLOBAL LOCAL_PREFIX(\name)
+ HIDDEN LOCAL_PREFIX(\name)
+#elif defined(HAVE_DOLLARMACRO)
+ GLOBAL LOCAL_PREFIX($0)
+ HIDDEN LOCAL_PREFIX($0)
+#endif
+.endm
+
+.macro GLOBAL_HIDDEN_FN name
+#if defined(HAVE_SLASHMACRO)
+ GLOBAL \name
+ HIDDEN \name
+ FN \name
+#elif defined(HAVE_DOLLARMACRO)
+ GLOBAL $0
+ HIDDEN $0
+ FN $0
+#endif
+.endm
+
+.macro GLOBAL_HIDDEN_FN_EXT name, args, xmmused
+#if defined(HAVE_SLASHMACRO)
+ GLOBAL \name
+ HIDDEN \name
+ FN_EXT \name, \args, \xmmused
+#elif defined(HAVE_DOLLARMACRO)
+ GLOBAL $0
+ HIDDEN $0
+ FN_EXT $0, $1, $2
+#endif
+.endm
+
+/* pic support */
+.macro LOAD_VAR_PIC var, reg
+#if !defined(__LP64__)
+ #if defined(HAVE_SLASHMACRO)
+ call 1f
+ 1:
+ popl \reg
+ leal \var - 1b(\reg), \reg
+ #elif defined(HAVE_DOLLARMACRO)
+ call 1f
+ 1:
+ popl $1
+ leal $0 - 1b($1), $1
+ #endif
+#else
+ #if defined(HAVE_SLASHMACRO)
+ leaq \var(%rip), \reg
+ #elif defined(HAVE_DOLLARMACRO)
+ leaq $0(%rip), $1
+ #endif
+#endif
+.endm
diff --git a/src/libcryptobox/platform_config.h.in b/src/libcryptobox/platform_config.h.in
new file mode 100644
index 0000000..7b7d17d
--- /dev/null
+++ b/src/libcryptobox/platform_config.h.in
@@ -0,0 +1,16 @@
+#ifndef PLATFORM_H_CONFIG
+#define PLATFORM_H_CONFIG
+
+#define ARCH "${ARCH}"
+#define CMAKE_ARCH_${ARCH} 1
+#cmakedefine HAVE_AVX2 1
+#cmakedefine HAVE_AVX 1
+#cmakedefine HAVE_SSE2 1
+#cmakedefine HAVE_SSE41 1
+#cmakedefine HAVE_SSE42 1
+#cmakedefine HAVE_SSE3 1
+#cmakedefine HAVE_SSSE3 1
+#cmakedefine HAVE_SLASHMACRO 1
+#cmakedefine HAVE_DOLLARMACRO 1
+
+#endif \ No newline at end of file
diff --git a/src/libmime/CMakeLists.txt b/src/libmime/CMakeLists.txt
new file mode 100644
index 0000000..09e5dbf
--- /dev/null
+++ b/src/libmime/CMakeLists.txt
@@ -0,0 +1,19 @@
+# Librspamd mime
+SET(LIBRSPAMDMIMESRC
+ ${CMAKE_CURRENT_SOURCE_DIR}/received.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/email_addr.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/mime_expressions.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/scan_result.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/images.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/message.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/archives.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/content_type.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/mime_headers.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/mime_parser.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/mime_encoding.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lang_detection.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lang_detection_fasttext.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/mime_string.cxx
+ )
+
+SET(RSPAMD_MIME ${LIBRSPAMDMIMESRC} PARENT_SCOPE) \ No newline at end of file
diff --git a/src/libmime/archives.c b/src/libmime/archives.c
new file mode 100644
index 0000000..ea0ea55
--- /dev/null
+++ b/src/libmime/archives.c
@@ -0,0 +1,2057 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "message.h"
+#include "task.h"
+#include "archives.h"
+#include "libmime/mime_encoding.h"
+#include <unicode/uchar.h>
+#include <unicode/utf8.h>
+#include <unicode/utf16.h>
+#include <unicode/ucnv.h>
+
+#define msg_debug_archive(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_archive_log_id, "archive", task->task_pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(archive)
+
+static void
+rspamd_archive_dtor(gpointer p)
+{
+ struct rspamd_archive *arch = p;
+ struct rspamd_archive_file *f;
+ guint i;
+
+ for (i = 0; i < arch->files->len; i++) {
+ f = g_ptr_array_index(arch->files, i);
+
+ if (f->fname) {
+ g_string_free(f->fname, TRUE);
+ }
+
+ g_free(f);
+ }
+
+ g_ptr_array_free(arch->files, TRUE);
+}
+
+static bool
+rspamd_archive_file_try_utf(struct rspamd_task *task,
+ struct rspamd_archive *arch,
+ struct rspamd_archive_file *fentry,
+ const gchar *in, gsize inlen)
+{
+ const gchar *charset = NULL, *p, *end;
+ GString *res;
+
+ charset = rspamd_mime_charset_find_by_content(in, inlen, TRUE);
+
+ if (charset) {
+ UChar *tmp;
+ UErrorCode uc_err = U_ZERO_ERROR;
+ gint32 r, clen, dlen;
+ struct rspamd_charset_converter *conv;
+ UConverter *utf8_converter;
+
+ conv = rspamd_mime_get_converter_cached(charset, task->task_pool,
+ TRUE, &uc_err);
+ utf8_converter = rspamd_get_utf8_converter();
+
+ if (conv == NULL) {
+ msg_info_task("cannot open converter for %s: %s",
+ charset, u_errorName(uc_err));
+ fentry->flags |= RSPAMD_ARCHIVE_FILE_OBFUSCATED;
+ fentry->fname = g_string_new_len(in, inlen);
+
+ return false;
+ }
+
+ tmp = g_malloc(sizeof(*tmp) * (inlen + 1));
+ r = rspamd_converter_to_uchars(conv, tmp, inlen + 1,
+ in, inlen, &uc_err);
+ if (!U_SUCCESS(uc_err)) {
+ msg_info_task("cannot convert data to unicode from %s: %s",
+ charset, u_errorName(uc_err));
+ g_free(tmp);
+
+ fentry->flags |= RSPAMD_ARCHIVE_FILE_OBFUSCATED;
+ fentry->fname = g_string_new_len(in, inlen);
+
+ return NULL;
+ }
+
+ int i = 0;
+
+ while (i < r) {
+ UChar32 uc;
+
+ U16_NEXT(tmp, i, r, uc);
+
+ if (IS_ZERO_WIDTH_SPACE(uc) || u_iscntrl(uc)) {
+ msg_info_task("control character in archive file name found: 0x%02xd "
+ "(filename=%T)",
+ uc, arch->archive_name);
+ fentry->flags |= RSPAMD_ARCHIVE_FILE_OBFUSCATED;
+ break;
+ }
+ }
+
+ clen = ucnv_getMaxCharSize(utf8_converter);
+ dlen = UCNV_GET_MAX_BYTES_FOR_STRING(r, clen);
+ res = g_string_sized_new(dlen);
+ r = ucnv_fromUChars(utf8_converter, res->str, dlen, tmp, r, &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ msg_info_task("cannot convert data from unicode from %s: %s",
+ charset, u_errorName(uc_err));
+ g_free(tmp);
+ g_string_free(res, TRUE);
+ fentry->flags |= RSPAMD_ARCHIVE_FILE_OBFUSCATED;
+ fentry->fname = g_string_new_len(in, inlen);
+
+ return NULL;
+ }
+
+ g_free(tmp);
+ res->len = r;
+
+ msg_debug_archive("converted from %s to UTF-8 inlen: %z, outlen: %d",
+ charset, inlen, r);
+ fentry->fname = res;
+ }
+ else {
+ /* Convert unsafe characters to '?' */
+ res = g_string_sized_new(inlen);
+ p = in;
+ end = in + inlen;
+
+ while (p < end) {
+ if (g_ascii_isgraph(*p)) {
+ g_string_append_c(res, *p);
+ }
+ else {
+ g_string_append_c(res, '?');
+
+ if (*p < 0x7f && (g_ascii_iscntrl(*p) || *p == '\0')) {
+ if (!(fentry->flags & RSPAMD_ARCHIVE_FILE_OBFUSCATED)) {
+ msg_info_task("suspicious character in archive file name found: 0x%02xd "
+ "(filename=%T)",
+ (int) *p, arch->archive_name);
+ fentry->flags |= RSPAMD_ARCHIVE_FILE_OBFUSCATED;
+ }
+ }
+ }
+
+ p++;
+ }
+ fentry->fname = res;
+ }
+
+ return true;
+}
+
+static void
+rspamd_archive_process_zip(struct rspamd_task *task,
+ struct rspamd_mime_part *part)
+{
+ const guchar *p, *start, *end, *eocd = NULL, *cd;
+ const guint32 eocd_magic = 0x06054b50, cd_basic_len = 46;
+ const guchar cd_magic[] = {0x50, 0x4b, 0x01, 0x02};
+ const guint max_processed = 1024;
+ guint32 cd_offset, cd_size, comp_size, uncomp_size, processed = 0;
+ guint16 extra_len, fname_len, comment_len;
+ struct rspamd_archive *arch;
+ struct rspamd_archive_file *f = NULL;
+
+ /* Zip files have interesting data at the end of archive */
+ p = part->parsed_data.begin + part->parsed_data.len - 1;
+ start = part->parsed_data.begin;
+ end = p;
+
+ /* Search for EOCD:
+ * 22 bytes is a typical size of eocd without a comment and
+ * end points one byte after the last character
+ */
+ p -= 21;
+
+ while (p > start + sizeof(guint32)) {
+ guint32 t;
+
+ if (processed > max_processed) {
+ break;
+ }
+
+ /* XXX: not an efficient approach */
+ memcpy(&t, p, sizeof(t));
+
+ if (GUINT32_FROM_LE(t) == eocd_magic) {
+ eocd = p;
+ break;
+ }
+
+ p--;
+ processed++;
+ }
+
+
+ if (eocd == NULL) {
+ /* Not a zip file */
+ msg_info_task("zip archive is invalid (no EOCD)");
+
+ return;
+ }
+
+ if (end - eocd < 21) {
+ msg_info_task("zip archive is invalid (short EOCD)");
+
+ return;
+ }
+
+
+ memcpy(&cd_size, eocd + 12, sizeof(cd_size));
+ cd_size = GUINT32_FROM_LE(cd_size);
+ memcpy(&cd_offset, eocd + 16, sizeof(cd_offset));
+ cd_offset = GUINT32_FROM_LE(cd_offset);
+
+ /* We need to check sanity as well */
+ if (cd_offset + cd_size > (guint) (eocd - start)) {
+ msg_info_task("zip archive is invalid (bad size/offset for CD)");
+
+ return;
+ }
+
+ cd = start + cd_offset;
+
+ arch = rspamd_mempool_alloc0(task->task_pool, sizeof(*arch));
+ arch->files = g_ptr_array_new();
+ arch->type = RSPAMD_ARCHIVE_ZIP;
+ if (part->cd) {
+ arch->archive_name = &part->cd->filename;
+ }
+ rspamd_mempool_add_destructor(task->task_pool, rspamd_archive_dtor,
+ arch);
+
+ while (cd < start + cd_offset + cd_size) {
+ guint16 flags;
+
+ /* Read central directory record */
+ if (eocd - cd < cd_basic_len ||
+ memcmp(cd, cd_magic, sizeof(cd_magic)) != 0) {
+ msg_info_task("zip archive is invalid (bad cd record)");
+
+ return;
+ }
+
+ memcpy(&flags, cd + 8, sizeof(guint16));
+ flags = GUINT16_FROM_LE(flags);
+ memcpy(&comp_size, cd + 20, sizeof(guint32));
+ comp_size = GUINT32_FROM_LE(comp_size);
+ memcpy(&uncomp_size, cd + 24, sizeof(guint32));
+ uncomp_size = GUINT32_FROM_LE(uncomp_size);
+ memcpy(&fname_len, cd + 28, sizeof(fname_len));
+ fname_len = GUINT16_FROM_LE(fname_len);
+ memcpy(&extra_len, cd + 30, sizeof(extra_len));
+ extra_len = GUINT16_FROM_LE(extra_len);
+ memcpy(&comment_len, cd + 32, sizeof(comment_len));
+ comment_len = GUINT16_FROM_LE(comment_len);
+
+ if (cd + fname_len + comment_len + extra_len + cd_basic_len > eocd) {
+ msg_info_task("zip archive is invalid (too large cd record)");
+
+ return;
+ }
+
+ f = g_malloc0(sizeof(*f));
+ rspamd_archive_file_try_utf(task, arch, f, cd + cd_basic_len, fname_len);
+
+ f->compressed_size = comp_size;
+ f->uncompressed_size = uncomp_size;
+
+ if (flags & 0x41u) {
+ f->flags |= RSPAMD_ARCHIVE_FILE_ENCRYPTED;
+ }
+
+ if (f->fname) {
+ if (f->flags & RSPAMD_ARCHIVE_FILE_OBFUSCATED) {
+ arch->flags |= RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES;
+ }
+
+ g_ptr_array_add(arch->files, f);
+ msg_debug_archive("found file in zip archive: %v", f->fname);
+ }
+ else {
+ g_free(f);
+
+ return;
+ }
+
+ /* Process extra fields */
+ const guchar *extra = cd + fname_len + cd_basic_len;
+ p = extra;
+
+ while (p + sizeof(guint16) * 2 < extra + extra_len) {
+ guint16 hid, hlen;
+
+ memcpy(&hid, p, sizeof(guint16));
+ hid = GUINT16_FROM_LE(hid);
+ memcpy(&hlen, p + sizeof(guint16), sizeof(guint16));
+ hlen = GUINT16_FROM_LE(hlen);
+
+ if (hid == 0x0017) {
+ f->flags |= RSPAMD_ARCHIVE_FILE_ENCRYPTED;
+ }
+
+ p += hlen + sizeof(guint16) * 2;
+ }
+
+ cd += fname_len + comment_len + extra_len + cd_basic_len;
+ }
+
+ part->part_type = RSPAMD_MIME_PART_ARCHIVE;
+ part->specific.arch = arch;
+
+ arch->size = part->parsed_data.len;
+}
+
+static inline gint
+rspamd_archive_rar_read_vint(const guchar *start, gsize remain, guint64 *res)
+{
+ /*
+ * From http://www.rarlab.com/technote.htm:
+ * Variable length integer. Can include one or more bytes, where
+ * lower 7 bits of every byte contain integer data and highest bit
+ * in every byte is the continuation flag.
+ * If highest bit is 0, this is the last byte in sequence.
+ * So first byte contains 7 least significant bits of integer and
+ * continuation flag. Second byte, if present, contains next 7 bits and so on.
+ */
+ guint64 t = 0;
+ guint shift = 0;
+ const guchar *p = start;
+
+ while (remain > 0 && shift <= 57) {
+ if (*p & 0x80) {
+ t |= ((guint64) (*p & 0x7f)) << shift;
+ }
+ else {
+ t |= ((guint64) (*p & 0x7f)) << shift;
+ p++;
+ break;
+ }
+
+ shift += 7;
+ p++;
+ remain--;
+ }
+
+ if (remain == 0 || shift > 64) {
+ return -1;
+ }
+
+ *res = GUINT64_FROM_LE(t);
+
+ return p - start;
+}
+
+#define RAR_SKIP_BYTES(n) \
+ do { \
+ if ((n) <= 0) { \
+ msg_debug_archive("rar archive is invalid (bad skip value)"); \
+ return; \
+ } \
+ if ((gsize) (end - p) < (n)) { \
+ msg_debug_archive("rar archive is invalid (truncated)"); \
+ return; \
+ } \
+ p += (n); \
+ } while (0)
+
+#define RAR_READ_VINT() \
+ do { \
+ r = rspamd_archive_rar_read_vint(p, end - p, &vint); \
+ if (r == -1) { \
+ msg_debug_archive("rar archive is invalid (bad vint)"); \
+ return; \
+ } \
+ else if (r == 0) { \
+ msg_debug_archive("rar archive is invalid (BAD vint offset)"); \
+ return; \
+ } \
+ } while (0)
+
+#define RAR_READ_VINT_SKIP() \
+ do { \
+ r = rspamd_archive_rar_read_vint(p, end - p, &vint); \
+ if (r == -1) { \
+ msg_debug_archive("rar archive is invalid (bad vint)"); \
+ return; \
+ } \
+ p += r; \
+ } while (0)
+
+#define RAR_READ_UINT16(n) \
+ do { \
+ if (end - p < (glong) sizeof(guint16)) { \
+ msg_debug_archive("rar archive is invalid (bad int16)"); \
+ return; \
+ } \
+ n = p[0] + (p[1] << 8); \
+ p += sizeof(guint16); \
+ } while (0)
+
+#define RAR_READ_UINT32(n) \
+ do { \
+ if (end - p < (glong) sizeof(guint32)) { \
+ msg_debug_archive("rar archive is invalid (bad int32)"); \
+ return; \
+ } \
+ n = (guint) p[0] + ((guint) p[1] << 8) + ((guint) p[2] << 16) + ((guint) p[3] << 24); \
+ p += sizeof(guint32); \
+ } while (0)
+
+static void
+rspamd_archive_process_rar_v4(struct rspamd_task *task, const guchar *start,
+ const guchar *end, struct rspamd_mime_part *part)
+{
+ const guchar *p = start, *start_section;
+ guint8 type;
+ guint flags;
+ guint64 sz, comp_sz = 0, uncomp_sz = 0;
+ struct rspamd_archive *arch;
+ struct rspamd_archive_file *f;
+
+ arch = rspamd_mempool_alloc0(task->task_pool, sizeof(*arch));
+ arch->files = g_ptr_array_new();
+ arch->type = RSPAMD_ARCHIVE_RAR;
+ if (part->cd) {
+ arch->archive_name = &part->cd->filename;
+ }
+ rspamd_mempool_add_destructor(task->task_pool, rspamd_archive_dtor,
+ arch);
+
+ while (p < end) {
+ /* Crc16 */
+ start_section = p;
+ RAR_SKIP_BYTES(sizeof(guint16));
+ type = *p;
+ p++;
+ RAR_READ_UINT16(flags);
+
+ if (type == 0x73) {
+ /* Main header, check for encryption */
+ if (flags & 0x80) {
+ arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
+ goto end;
+ }
+ }
+
+ RAR_READ_UINT16(sz);
+
+ if (flags & 0x8000) {
+ /* We also need to read ADD_SIZE element */
+ guint32 tmp;
+
+ RAR_READ_UINT32(tmp);
+ sz += tmp;
+ /* This is also used as PACK_SIZE */
+ comp_sz = tmp;
+ }
+
+ if (sz == 0) {
+ /* Zero sized block - error */
+ msg_debug_archive("rar archive is invalid (zero size block)");
+
+ return;
+ }
+
+ if (type == 0x74) {
+ guint fname_len;
+
+ /* File header */
+ /* Uncompressed size */
+ RAR_READ_UINT32(uncomp_sz);
+ /* Skip to NAME_SIZE element */
+ RAR_SKIP_BYTES(11);
+ RAR_READ_UINT16(fname_len);
+
+ if (fname_len == 0 || fname_len > (gsize) (end - p)) {
+ msg_debug_archive("rar archive is invalid (bad filename size: %d)",
+ fname_len);
+
+ return;
+ }
+
+ /* Attrs */
+ RAR_SKIP_BYTES(4);
+
+ if (flags & 0x100) {
+ /* We also need to read HIGH_PACK_SIZE */
+ guint32 tmp;
+
+ RAR_READ_UINT32(tmp);
+ sz += tmp;
+ comp_sz += tmp;
+ /* HIGH_UNP_SIZE */
+ RAR_READ_UINT32(tmp);
+ uncomp_sz += tmp;
+ }
+
+ f = g_malloc0(sizeof(*f));
+
+ if (flags & 0x200) {
+ /* We have unicode + normal version */
+ guchar *tmp;
+
+ tmp = memchr(p, '\0', fname_len);
+
+ if (tmp != NULL) {
+ /* Just use ASCII version */
+ rspamd_archive_file_try_utf(task, arch, f, p, tmp - p);
+ msg_debug_archive("found ascii filename in rarv4 archive: %v",
+ f->fname);
+ }
+ else {
+ /* We have UTF8 filename, use it as is */
+ rspamd_archive_file_try_utf(task, arch, f, p, fname_len);
+ msg_debug_archive("found utf filename in rarv4 archive: %v",
+ f->fname);
+ }
+ }
+ else {
+ rspamd_archive_file_try_utf(task, arch, f, p, fname_len);
+ msg_debug_archive("found ascii (old) filename in rarv4 archive: %v",
+ f->fname);
+ }
+
+ f->compressed_size = comp_sz;
+ f->uncompressed_size = uncomp_sz;
+
+ if (flags & 0x4) {
+ f->flags |= RSPAMD_ARCHIVE_FILE_ENCRYPTED;
+ }
+
+ if (f->fname) {
+ if (f->flags & RSPAMD_ARCHIVE_FILE_OBFUSCATED) {
+ arch->flags |= RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES;
+ }
+ g_ptr_array_add(arch->files, f);
+ }
+ else {
+ g_free(f);
+ }
+ }
+
+ p = start_section;
+ RAR_SKIP_BYTES(sz);
+ }
+
+end:
+ part->part_type = RSPAMD_MIME_PART_ARCHIVE;
+ part->specific.arch = arch;
+ arch->size = part->parsed_data.len;
+}
+
+static void
+rspamd_archive_process_rar(struct rspamd_task *task,
+ struct rspamd_mime_part *part)
+{
+ const guchar *p, *end, *section_start;
+ const guchar rar_v5_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00},
+ rar_v4_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00};
+ const guint rar_encrypted_header = 4, rar_main_header = 1,
+ rar_file_header = 2;
+ guint64 vint, sz, comp_sz = 0, uncomp_sz = 0, flags = 0, type = 0,
+ extra_sz = 0;
+ struct rspamd_archive *arch;
+ struct rspamd_archive_file *f;
+ gint r;
+
+ p = part->parsed_data.begin;
+ end = p + part->parsed_data.len;
+
+ if ((gsize) (end - p) <= sizeof(rar_v5_magic)) {
+ msg_debug_archive("rar archive is invalid (too small)");
+
+ return;
+ }
+
+ if (memcmp(p, rar_v5_magic, sizeof(rar_v5_magic)) == 0) {
+ p += sizeof(rar_v5_magic);
+ }
+ else if (memcmp(p, rar_v4_magic, sizeof(rar_v4_magic)) == 0) {
+ p += sizeof(rar_v4_magic);
+
+ rspamd_archive_process_rar_v4(task, p, end, part);
+ return;
+ }
+ else {
+ msg_debug_archive("rar archive is invalid (no rar magic)");
+
+ return;
+ }
+
+ /* Rar v5 format */
+ arch = rspamd_mempool_alloc0(task->task_pool, sizeof(*arch));
+ arch->files = g_ptr_array_new();
+ arch->type = RSPAMD_ARCHIVE_RAR;
+ if (part->cd) {
+ arch->archive_name = &part->cd->filename;
+ }
+ rspamd_mempool_add_destructor(task->task_pool, rspamd_archive_dtor,
+ arch);
+
+ /* Now we can have either encryption header or archive header */
+ /* Crc 32 */
+ RAR_SKIP_BYTES(sizeof(guint32));
+ /* Size */
+ RAR_READ_VINT_SKIP();
+ sz = vint;
+ /* Type */
+ section_start = p;
+ RAR_READ_VINT_SKIP();
+ type = vint;
+ /* Header flags */
+ RAR_READ_VINT_SKIP();
+ flags = vint;
+
+ if (flags & 0x1) {
+ /* Have extra zone */
+ RAR_READ_VINT_SKIP();
+ }
+ if (flags & 0x2) {
+ /* Data zone is presented */
+ RAR_READ_VINT_SKIP();
+ sz += vint;
+ }
+
+ if (type == rar_encrypted_header) {
+ /* We can't read any further information as archive is encrypted */
+ arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
+ goto end;
+ }
+ else if (type != rar_main_header) {
+ msg_debug_archive("rar archive is invalid (bad main header)");
+
+ return;
+ }
+
+ /* Nothing useful in main header */
+ p = section_start;
+ RAR_SKIP_BYTES(sz);
+
+ while (p < end) {
+ gboolean has_extra = FALSE;
+ /* Read the next header */
+ /* Crc 32 */
+ RAR_SKIP_BYTES(sizeof(guint32));
+ /* Size */
+ RAR_READ_VINT_SKIP();
+
+ sz = vint;
+ if (sz == 0) {
+ /* Zero sized block - error */
+ msg_debug_archive("rar archive is invalid (zero size block)");
+
+ return;
+ }
+
+ section_start = p;
+ /* Type */
+ RAR_READ_VINT_SKIP();
+ type = vint;
+ /* Header flags */
+ RAR_READ_VINT_SKIP();
+ flags = vint;
+
+ if (flags & 0x1) {
+ /* Have extra zone */
+ RAR_READ_VINT_SKIP();
+ extra_sz = vint;
+ has_extra = TRUE;
+ }
+
+ if (flags & 0x2) {
+ /* Data zone is presented */
+ RAR_READ_VINT_SKIP();
+ sz += vint;
+ comp_sz = vint;
+ }
+
+ if (type != rar_file_header) {
+ p = section_start;
+ RAR_SKIP_BYTES(sz);
+ }
+ else {
+ /* We have a file header, go forward */
+ guint64 fname_len;
+ bool is_directory = false;
+
+ /* File header specific flags */
+ RAR_READ_VINT_SKIP();
+ flags = vint;
+
+ /* Unpacked size */
+ RAR_READ_VINT_SKIP();
+ uncomp_sz = vint;
+ /* Attributes */
+ RAR_READ_VINT_SKIP();
+
+ if (flags & 0x2) {
+ /* Unix mtime */
+ RAR_SKIP_BYTES(sizeof(guint32));
+ }
+ if (flags & 0x4) {
+ /* Crc32 */
+ RAR_SKIP_BYTES(sizeof(guint32));
+ }
+ if (flags & 0x1) {
+ /* Ignore directories for sanity purposes */
+ is_directory = true;
+ msg_debug_archive("skip directory record in a rar archive");
+ }
+
+ if (!is_directory) {
+ /* Compression */
+ RAR_READ_VINT_SKIP();
+ /* Host OS */
+ RAR_READ_VINT_SKIP();
+ /* Filename length (finally!) */
+ RAR_READ_VINT_SKIP();
+ fname_len = vint;
+
+ if (fname_len == 0 || fname_len > (gsize) (end - p)) {
+ msg_debug_archive("rar archive is invalid (bad filename size)");
+
+ return;
+ }
+
+ f = g_malloc0(sizeof(*f));
+ f->uncompressed_size = uncomp_sz;
+ f->compressed_size = comp_sz;
+ rspamd_archive_file_try_utf(task, arch, f, p, fname_len);
+
+ if (f->fname) {
+ msg_debug_archive("added rarv5 file: %v", f->fname);
+ g_ptr_array_add(arch->files, f);
+ if (f->flags & RSPAMD_ARCHIVE_FILE_OBFUSCATED) {
+ arch->flags |= RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES;
+ }
+ }
+ else {
+ g_free(f);
+ f = NULL;
+ }
+
+ if (f && has_extra && extra_sz > 0 &&
+ p + fname_len + extra_sz < end) {
+ /* Try to find encryption record in extra field */
+ const guchar *ex = p + fname_len;
+
+ while (ex < p + extra_sz) {
+ const guchar *t;
+ gint64 cur_sz = 0, sec_type = 0;
+
+ r = rspamd_archive_rar_read_vint(ex, extra_sz, &cur_sz);
+ if (r == -1) {
+ msg_debug_archive("rar archive is invalid (bad vint)");
+ return;
+ }
+
+ t = ex + r;
+
+ r = rspamd_archive_rar_read_vint(t, extra_sz - r, &sec_type);
+ if (r == -1) {
+ msg_debug_archive("rar archive is invalid (bad vint)");
+ return;
+ }
+
+ if (sec_type == 0x01) {
+ f->flags |= RSPAMD_ARCHIVE_FILE_ENCRYPTED;
+ arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
+ break;
+ }
+
+ ex += cur_sz;
+ }
+ }
+ }
+
+ /* Restore p to the beginning of the header */
+ p = section_start;
+ RAR_SKIP_BYTES(sz);
+ }
+ }
+
+end:
+ part->part_type = RSPAMD_MIME_PART_ARCHIVE;
+ part->specific.arch = arch;
+ arch->size = part->parsed_data.len;
+}
+
+static inline gint
+rspamd_archive_7zip_read_vint(const guchar *start, gsize remain, guint64 *res)
+{
+ /*
+ * REAL_UINT64 means real UINT64.
+ * UINT64 means real UINT64 encoded with the following scheme:
+ *
+ * Size of encoding sequence depends from first byte:
+ * First_Byte Extra_Bytes Value
+ * (binary)
+ * 0xxxxxxx : ( xxxxxxx )
+ * 10xxxxxx BYTE y[1] : ( xxxxxx << (8 * 1)) + y
+ * 110xxxxx BYTE y[2] : ( xxxxx << (8 * 2)) + y
+ * ...
+ * 1111110x BYTE y[6] : ( x << (8 * 6)) + y
+ * 11111110 BYTE y[7] : y
+ * 11111111 BYTE y[8] : y
+ */
+ guchar t;
+
+ if (remain == 0) {
+ return -1;
+ }
+
+ t = *start;
+
+ if (!isset(&t, 7)) {
+ /* Trivial case */
+ *res = t;
+ return 1;
+ }
+ else if (t == 0xFF) {
+ if (remain >= sizeof(guint64) + 1) {
+ memcpy(res, start + 1, sizeof(guint64));
+ *res = GUINT64_FROM_LE(*res);
+
+ return sizeof(guint64) + 1;
+ }
+ }
+ else {
+ gint cur_bit = 6, intlen = 1;
+ const guchar bmask = 0xFF;
+ guint64 tgt;
+
+ while (cur_bit > 0) {
+ if (!isset(&t, cur_bit)) {
+ if (remain >= intlen + 1) {
+ memcpy(&tgt, start + 1, intlen);
+ tgt = GUINT64_FROM_LE(tgt);
+ /* Shift back */
+ tgt >>= sizeof(tgt) - NBBY * intlen;
+ /* Add masked value */
+ tgt += (guint64) (t & (bmask >> (NBBY - cur_bit)))
+ << (NBBY * intlen);
+ *res = tgt;
+
+ return intlen + 1;
+ }
+ }
+ cur_bit--;
+ intlen++;
+ }
+ }
+
+ return -1;
+}
+
+#define SZ_READ_VINT_SKIP() \
+ do { \
+ r = rspamd_archive_7zip_read_vint(p, end - p, &vint); \
+ if (r == -1) { \
+ msg_debug_archive("7z archive is invalid (bad vint)"); \
+ return; \
+ } \
+ p += r; \
+ } while (0)
+#define SZ_READ_VINT(var) \
+ do { \
+ int r; \
+ r = rspamd_archive_7zip_read_vint(p, end - p, &(var)); \
+ if (r == -1) { \
+ msg_debug_archive("7z archive is invalid (bad vint): %s", G_STRLOC); \
+ return NULL; \
+ } \
+ p += r; \
+ } while (0)
+
+#define SZ_READ_UINT64(n) \
+ do { \
+ if (end - p < (goffset) sizeof(guint64)) { \
+ msg_debug_archive("7zip archive is invalid (bad uint64): %s", G_STRLOC); \
+ return; \
+ } \
+ memcpy(&(n), p, sizeof(guint64)); \
+ n = GUINT64_FROM_LE(n); \
+ p += sizeof(guint64); \
+ } while (0)
+#define SZ_SKIP_BYTES(n) \
+ do { \
+ if (end - p >= (n)) { \
+ p += (n); \
+ } \
+ else { \
+ msg_debug_archive("7zip archive is invalid (truncated); wanted to read %d bytes, %d avail: %s", (gint) (n), (gint) (end - p), G_STRLOC); \
+ return NULL; \
+ } \
+ } while (0)
+
+enum rspamd_7zip_header_mark {
+ kEnd = 0x00,
+ kHeader = 0x01,
+ kArchiveProperties = 0x02,
+ kAdditionalStreamsInfo = 0x03,
+ kMainStreamsInfo = 0x04,
+ kFilesInfo = 0x05,
+ kPackInfo = 0x06,
+ kUnPackInfo = 0x07,
+ kSubStreamsInfo = 0x08,
+ kSize = 0x09,
+ kCRC = 0x0A,
+ kFolder = 0x0B,
+ kCodersUnPackSize = 0x0C,
+ kNumUnPackStream = 0x0D,
+ kEmptyStream = 0x0E,
+ kEmptyFile = 0x0F,
+ kAnti = 0x10,
+ kName = 0x11,
+ kCTime = 0x12,
+ kATime = 0x13,
+ kMTime = 0x14,
+ kWinAttributes = 0x15,
+ kComment = 0x16,
+ kEncodedHeader = 0x17,
+ kStartPos = 0x18,
+ kDummy = 0x19,
+};
+
+
+#define _7Z_CRYPTO_MAIN_ZIP 0x06F10101 /* Main Zip crypto algo */
+#define _7Z_CRYPTO_RAR_29 0x06F10303 /* Rar29 AES-128 + (modified SHA-1) */
+#define _7Z_CRYPTO_AES_256_SHA_256 0x06F10701 /* AES-256 + SHA-256 */
+
+#define IS_SZ_ENCRYPTED(codec_id) (((codec_id) == _7Z_CRYPTO_MAIN_ZIP) || \
+ ((codec_id) == _7Z_CRYPTO_RAR_29) || \
+ ((codec_id) == _7Z_CRYPTO_AES_256_SHA_256))
+
+static const guchar *
+rspamd_7zip_read_bits(struct rspamd_task *task,
+ const guchar *p, const guchar *end,
+ struct rspamd_archive *arch, guint nbits,
+ guint *pbits_set)
+{
+ unsigned mask = 0, avail = 0, i;
+ gboolean bit_set = 0;
+
+ for (i = 0; i < nbits; i++) {
+ if (mask == 0) {
+ avail = *p;
+ SZ_SKIP_BYTES(1);
+ mask = 0x80;
+ }
+
+ bit_set = (avail & mask) ? 1 : 0;
+
+ if (bit_set && pbits_set) {
+ (*pbits_set)++;
+ }
+
+ mask >>= 1;
+ }
+
+ return p;
+}
+
+static const guchar *
+rspamd_7zip_read_digest(struct rspamd_task *task,
+ const guchar *p, const guchar *end,
+ struct rspamd_archive *arch,
+ guint64 num_streams,
+ guint *pdigest_read)
+{
+ guchar all_defined = *p;
+ guint64 i;
+ guint num_defined = 0;
+ /*
+ * BYTE AllAreDefined
+ * if (AllAreDefined == 0)
+ * {
+ * for(NumStreams)
+ * BIT Defined
+ * }
+ * UINT32 CRCs[NumDefined]
+ */
+ SZ_SKIP_BYTES(1);
+
+ if (all_defined) {
+ num_defined = num_streams;
+ }
+ else {
+ if (num_streams > 8192) {
+ /* Gah */
+ return NULL;
+ }
+
+ p = rspamd_7zip_read_bits(task, p, end, arch, num_streams, &num_defined);
+
+ if (p == NULL) {
+ return NULL;
+ }
+ }
+
+ for (i = 0; i < num_defined; i++) {
+ SZ_SKIP_BYTES(sizeof(guint32));
+ }
+
+ if (pdigest_read) {
+ *pdigest_read = num_defined;
+ }
+
+ return p;
+}
+
+static const guchar *
+rspamd_7zip_read_pack_info(struct rspamd_task *task,
+ const guchar *p, const guchar *end,
+ struct rspamd_archive *arch)
+{
+ guint64 pack_pos = 0, pack_streams = 0, i, cur_sz;
+ guint num_digests = 0;
+ guchar t;
+ /*
+ * UINT64 PackPos
+ * UINT64 NumPackStreams
+ *
+ * []
+ * BYTE NID::kSize (0x09)
+ * UINT64 PackSizes[NumPackStreams]
+ * []
+ *
+ * []
+ * BYTE NID::kCRC (0x0A)
+ * PackStreamDigests[NumPackStreams]
+ * []
+ * BYTE NID::kEnd
+ */
+
+ SZ_READ_VINT(pack_pos);
+ SZ_READ_VINT(pack_streams);
+
+ while (p != NULL && p < end) {
+ t = *p;
+ SZ_SKIP_BYTES(1);
+ msg_debug_archive("7zip: read pack info %xc", t);
+
+ switch (t) {
+ case kSize:
+ /* We need to skip pack_streams VINTS */
+ for (i = 0; i < pack_streams; i++) {
+ SZ_READ_VINT(cur_sz);
+ }
+ break;
+ case kCRC:
+ /* CRCs are more complicated */
+ p = rspamd_7zip_read_digest(task, p, end, arch, pack_streams,
+ &num_digests);
+ break;
+ case kEnd:
+ goto end;
+ break;
+ default:
+ p = NULL;
+ msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+ goto end;
+ break;
+ }
+ }
+
+end:
+
+ return p;
+}
+
+static const guchar *
+rspamd_7zip_read_folder(struct rspamd_task *task,
+ const guchar *p, const guchar *end,
+ struct rspamd_archive *arch, guint *pnstreams, guint *ndigests)
+{
+ guint64 ncoders = 0, i, j, noutstreams = 0, ninstreams = 0;
+
+ SZ_READ_VINT(ncoders);
+
+ for (i = 0; i < ncoders && p != NULL && p < end; i++) {
+ guint64 sz, tmp;
+ guchar t;
+ /*
+ * BYTE
+ * {
+ * 0:3 CodecIdSize
+ * 4: Is Complex Coder
+ * 5: There Are Attributes
+ * 6: Reserved
+ * 7: There are more alternative methods. (Not used anymore, must be 0).
+ * }
+ * BYTE CodecId[CodecIdSize]
+ * if (Is Complex Coder)
+ * {
+ * UINT64 NumInStreams;
+ * UINT64 NumOutStreams;
+ * }
+ * if (There Are Attributes)
+ * {
+ * UINT64 PropertiesSize
+ * BYTE Properties[PropertiesSize]
+ * }
+ */
+ t = *p;
+ SZ_SKIP_BYTES(1);
+ sz = t & 0xF;
+ /* Codec ID */
+ tmp = 0;
+ for (j = 0; j < sz; j++) {
+ tmp <<= 8;
+ tmp += p[j];
+ }
+
+ msg_debug_archive("7zip: read codec id: %L", tmp);
+
+ if (IS_SZ_ENCRYPTED(tmp)) {
+ arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
+ }
+
+ SZ_SKIP_BYTES(sz);
+
+ if (t & (1u << 4)) {
+ /* Complex */
+ SZ_READ_VINT(tmp); /* InStreams */
+ ninstreams += tmp;
+ SZ_READ_VINT(tmp); /* OutStreams */
+ noutstreams += tmp;
+ }
+ else {
+ /* XXX: is it correct ? */
+ noutstreams++;
+ ninstreams++;
+ }
+ if (t & (1u << 5)) {
+ /* Attributes ... */
+ SZ_READ_VINT(tmp); /* Size of attrs */
+ SZ_SKIP_BYTES(tmp);
+ }
+ }
+
+ if (noutstreams > 1) {
+ /* BindPairs, WTF, huh */
+ for (i = 0; i < noutstreams - 1; i++) {
+ guint64 tmp;
+
+ SZ_READ_VINT(tmp);
+ SZ_READ_VINT(tmp);
+ }
+ }
+
+ gint64 npacked = (gint64) ninstreams - (gint64) noutstreams + 1;
+ msg_debug_archive("7zip: instreams=%L, outstreams=%L, packed=%L",
+ ninstreams, noutstreams, npacked);
+
+ if (npacked > 1) {
+ /* Gah... */
+ for (i = 0; i < npacked; i++) {
+ guint64 tmp;
+
+ SZ_READ_VINT(tmp);
+ }
+ }
+
+ *pnstreams = noutstreams;
+ (*ndigests) += npacked;
+
+ return p;
+}
+
+static const guchar *
+rspamd_7zip_read_coders_info(struct rspamd_task *task,
+ const guchar *p, const guchar *end,
+ struct rspamd_archive *arch,
+ guint *pnum_folders, guint *pnum_nodigest)
+{
+ guint64 num_folders = 0, i, tmp;
+ guchar t;
+ guint *folder_nstreams = NULL, num_digests = 0, digests_read = 0;
+
+ while (p != NULL && p < end) {
+ /*
+ * BYTE NID::kFolder (0x0B)
+ * UINT64 NumFolders
+ * BYTE External
+ * switch(External)
+ * {
+ * case 0:
+ * Folders[NumFolders]
+ * case 1:
+ * UINT64 DataStreamIndex
+ * }
+ * BYTE ID::kCodersUnPackSize (0x0C)
+ * for(Folders)
+ * for(Folder.NumOutStreams)
+ * UINT64 UnPackSize;
+ * []
+ * BYTE NID::kCRC (0x0A)
+ * UnPackDigests[NumFolders]
+ * []
+ * BYTE NID::kEnd
+ */
+
+ t = *p;
+ SZ_SKIP_BYTES(1);
+ msg_debug_archive("7zip: read coders info %xc", t);
+
+ switch (t) {
+ case kFolder:
+ SZ_READ_VINT(num_folders);
+ msg_debug_archive("7zip: nfolders=%L", num_folders);
+
+ if (*p != 0) {
+ /* External folders */
+ SZ_SKIP_BYTES(1);
+ SZ_READ_VINT(tmp);
+ }
+ else {
+ SZ_SKIP_BYTES(1);
+
+ if (num_folders > 8192) {
+ /* Gah */
+ return NULL;
+ }
+
+ if (folder_nstreams) {
+ g_free(folder_nstreams);
+ }
+
+ folder_nstreams = g_malloc(sizeof(int) * num_folders);
+
+ for (i = 0; i < num_folders && p != NULL && p < end; i++) {
+ p = rspamd_7zip_read_folder(task, p, end, arch,
+ &folder_nstreams[i], &num_digests);
+ }
+ }
+ break;
+ case kCodersUnPackSize:
+ for (i = 0; i < num_folders && p != NULL && p < end; i++) {
+ if (folder_nstreams) {
+ for (guint j = 0; j < folder_nstreams[i]; j++) {
+ SZ_READ_VINT(tmp); /* Unpacked size */
+ msg_debug_archive("7zip: unpacked size "
+ "(folder=%d, stream=%d) = %L",
+ (gint) i, j, tmp);
+ }
+ }
+ else {
+ msg_err_task("internal 7zip error");
+ }
+ }
+ break;
+ case kCRC:
+ /*
+ * Here are dragons. Spec tells that here there could be up
+ * to nfolders digests. However, according to the actual source
+ * code, in case of multiple out streams there should be digests
+ * for all out streams.
+ *
+ * In the real life (tm) it is even more idiotic: all these digests
+ * are in another section! But that section needs number of digests
+ * that are absent here. It is the most stupid thing I've ever seen
+ * in any file format.
+ *
+ * I hope there *WAS* some reason to do such shit...
+ */
+ p = rspamd_7zip_read_digest(task, p, end, arch, num_digests,
+ &digests_read);
+ break;
+ case kEnd:
+ goto end;
+ break;
+ default:
+ p = NULL;
+ msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+ goto end;
+ break;
+ }
+ }
+
+end:
+
+ if (pnum_nodigest) {
+ *pnum_nodigest = num_digests - digests_read;
+ }
+ if (pnum_folders) {
+ *pnum_folders = num_folders;
+ }
+
+ if (folder_nstreams) {
+ g_free(folder_nstreams);
+ }
+
+ return p;
+}
+
+static const guchar *
+rspamd_7zip_read_substreams_info(struct rspamd_task *task,
+ const guchar *p, const guchar *end,
+ struct rspamd_archive *arch,
+ guint num_folders, guint num_nodigest)
+{
+ guchar t;
+ guint i;
+ guint64 *folder_nstreams;
+
+ if (num_folders > 8192) {
+ /* Gah */
+ return NULL;
+ }
+
+ folder_nstreams = g_alloca(sizeof(guint64) * num_folders);
+ memset(folder_nstreams, 0, sizeof(guint64) * num_folders);
+
+ while (p != NULL && p < end) {
+ /*
+ * []
+ * BYTE NID::kNumUnPackStream; (0x0D)
+ * UINT64 NumUnPackStreamsInFolders[NumFolders];
+ * []
+ *
+ * []
+ * BYTE NID::kSize (0x09)
+ * UINT64 UnPackSizes[??]
+ * []
+ *
+ *
+ * []
+ * BYTE NID::kCRC (0x0A)
+ * Digests[Number of streams with unknown CRC]
+ * []
+
+ */
+ t = *p;
+ SZ_SKIP_BYTES(1);
+
+ msg_debug_archive("7zip: read substream info %xc", t);
+
+ switch (t) {
+ case kNumUnPackStream:
+ for (i = 0; i < num_folders; i++) {
+ guint64 tmp;
+
+ SZ_READ_VINT(tmp);
+ folder_nstreams[i] = tmp;
+ }
+ break;
+ case kCRC:
+ /*
+ * Read the comment in the rspamd_7zip_read_coders_info
+ */
+ p = rspamd_7zip_read_digest(task, p, end, arch, num_nodigest,
+ NULL);
+ break;
+ case kSize:
+ /*
+ * Another brain damaged logic, but we have to support it
+ * as there are no ways to proceed without it.
+ * In fact, it is just absent in the real life...
+ */
+ for (i = 0; i < num_folders; i++) {
+ for (guint j = 0; j < folder_nstreams[i]; j++) {
+ guint64 tmp;
+
+ SZ_READ_VINT(tmp); /* Who cares indeed */
+ }
+ }
+ break;
+ case kEnd:
+ goto end;
+ break;
+ default:
+ p = NULL;
+ msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+ goto end;
+ break;
+ }
+ }
+
+end:
+ return p;
+}
+
+static const guchar *
+rspamd_7zip_read_main_streams_info(struct rspamd_task *task,
+ const guchar *p, const guchar *end,
+ struct rspamd_archive *arch)
+{
+ guchar t;
+ guint num_folders = 0, unknown_digests = 0;
+
+ while (p != NULL && p < end) {
+ t = *p;
+ SZ_SKIP_BYTES(1);
+ msg_debug_archive("7zip: read main streams info %xc", t);
+
+ /*
+ *
+ * []
+ * PackInfo
+ * []
+
+ * []
+ * CodersInfo
+ * []
+ *
+ * []
+ * SubStreamsInfo
+ * []
+ *
+ * BYTE NID::kEnd
+ */
+ switch (t) {
+ case kPackInfo:
+ p = rspamd_7zip_read_pack_info(task, p, end, arch);
+ break;
+ case kUnPackInfo:
+ p = rspamd_7zip_read_coders_info(task, p, end, arch, &num_folders,
+ &unknown_digests);
+ break;
+ case kSubStreamsInfo:
+ p = rspamd_7zip_read_substreams_info(task, p, end, arch, num_folders,
+ unknown_digests);
+ break;
+ break;
+ case kEnd:
+ goto end;
+ break;
+ default:
+ p = NULL;
+ msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+ goto end;
+ break;
+ }
+ }
+
+end:
+ return p;
+}
+
+static const guchar *
+rspamd_7zip_read_archive_props(struct rspamd_task *task,
+ const guchar *p, const guchar *end,
+ struct rspamd_archive *arch)
+{
+ guchar proptype;
+ guint64 proplen;
+
+ /*
+ * for (;;)
+ * {
+ * BYTE PropertyType;
+ * if (aType == 0)
+ * break;
+ * UINT64 PropertySize;
+ * BYTE PropertyData[PropertySize];
+ * }
+ */
+
+ if (p != NULL) {
+ proptype = *p;
+ SZ_SKIP_BYTES(1);
+
+ while (proptype != 0) {
+ SZ_READ_VINT(proplen);
+
+ if (p + proplen < end) {
+ p += proplen;
+ }
+ else {
+ return NULL;
+ }
+
+ proptype = *p;
+ SZ_SKIP_BYTES(1);
+ }
+ }
+
+ return p;
+}
+
+static GString *
+rspamd_7zip_ucs2_to_utf8(struct rspamd_task *task, const guchar *p,
+ const guchar *end)
+{
+ GString *res;
+ goffset dest_pos = 0, src_pos = 0;
+ const gsize len = (end - p) / sizeof(guint16);
+ guint16 *up;
+ UChar32 wc;
+ UBool is_error = 0;
+
+ res = g_string_sized_new((end - p) * 3 / 2 + sizeof(wc) + 1);
+ up = (guint16 *) p;
+
+ while (src_pos < len) {
+ U16_NEXT(up, src_pos, len, wc);
+
+ if (wc > 0) {
+ U8_APPEND(res->str, dest_pos,
+ res->allocated_len - 1,
+ wc, is_error);
+ }
+
+ if (is_error) {
+ g_string_free(res, TRUE);
+
+ return NULL;
+ }
+ }
+
+ g_assert(dest_pos < res->allocated_len);
+
+ res->len = dest_pos;
+ res->str[dest_pos] = '\0';
+
+ return res;
+}
+
+static const guchar *
+rspamd_7zip_read_files_info(struct rspamd_task *task,
+ const guchar *p, const guchar *end,
+ struct rspamd_archive *arch)
+{
+ guint64 nfiles = 0, sz, i;
+ guchar t, b;
+ struct rspamd_archive_file *fentry;
+
+ SZ_READ_VINT(nfiles);
+
+ for (; p != NULL && p < end;) {
+ t = *p;
+ SZ_SKIP_BYTES(1);
+
+ msg_debug_archive("7zip: read file data type %xc", t);
+
+ if (t == kEnd) {
+ goto end;
+ }
+
+ /* This is SO SPECIAL, gah */
+ SZ_READ_VINT(sz);
+
+ switch (t) {
+ case kEmptyStream:
+ case kEmptyFile:
+ case kAnti: /* AntiFile, OMFG */
+ /* We don't care about these bits */
+ case kCTime:
+ case kATime:
+ case kMTime:
+ /* We don't care of these guys, but we still have to parse them, gah */
+ if (sz > 0) {
+ SZ_SKIP_BYTES(sz);
+ }
+ break;
+ case kName:
+ /* The most useful part in this whole bloody format */
+ b = *p; /* External flag */
+ SZ_SKIP_BYTES(1);
+
+ if (b) {
+ /* TODO: for the god sake, do something about external
+ * filenames...
+ */
+ guint64 tmp;
+
+ SZ_READ_VINT(tmp);
+ }
+ else {
+ for (i = 0; i < nfiles; i++) {
+ /* Zero terminated wchar_t: happy converting... */
+ /* First, find terminator */
+ const guchar *fend = NULL, *tp = p;
+ GString *res;
+
+ while (tp < end - 1) {
+ if (*tp == 0 && *(tp + 1) == 0) {
+ fend = tp;
+ break;
+ }
+
+ tp += 2;
+ }
+
+ if (fend == NULL || fend - p == 0) {
+ /* Crap instead of fname */
+ msg_debug_archive("bad 7zip name; %s", G_STRLOC);
+ goto end;
+ }
+
+ res = rspamd_7zip_ucs2_to_utf8(task, p, fend);
+
+ if (res != NULL) {
+ fentry = g_malloc0(sizeof(*fentry));
+ fentry->fname = res;
+ g_ptr_array_add(arch->files, fentry);
+ msg_debug_archive("7zip: found file %v", res);
+ }
+ else {
+ msg_debug_archive("bad 7zip name; %s", G_STRLOC);
+ }
+ /* Skip zero terminating character */
+ p = fend + 2;
+ }
+ }
+ break;
+ case kDummy:
+ case kWinAttributes:
+ if (sz > 0) {
+ SZ_SKIP_BYTES(sz);
+ }
+ break;
+ default:
+ p = NULL;
+ msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+ goto end;
+ break;
+ }
+ }
+
+end:
+ return p;
+}
+
+static const guchar *
+rspamd_7zip_read_next_section(struct rspamd_task *task,
+ const guchar *p, const guchar *end,
+ struct rspamd_archive *arch)
+{
+ guchar t = *p;
+
+ SZ_SKIP_BYTES(1);
+
+ msg_debug_archive("7zip: read section %xc", t);
+
+ switch (t) {
+ case kHeader:
+ /* We just skip byte and go further */
+ break;
+ case kEncodedHeader:
+ /*
+ * In fact, headers are just packed, but we assume it as
+ * encrypted to distinguish from the normal archives
+ */
+ msg_debug_archive("7zip: encoded header, needs to be uncompressed");
+ arch->flags |= RSPAMD_ARCHIVE_CANNOT_READ;
+ p = NULL; /* Cannot get anything useful */
+ break;
+ case kArchiveProperties:
+ p = rspamd_7zip_read_archive_props(task, p, end, arch);
+ break;
+ case kMainStreamsInfo:
+ p = rspamd_7zip_read_main_streams_info(task, p, end, arch);
+ break;
+ case kAdditionalStreamsInfo:
+ p = rspamd_7zip_read_main_streams_info(task, p, end, arch);
+ break;
+ case kFilesInfo:
+ p = rspamd_7zip_read_files_info(task, p, end, arch);
+ break;
+ case kEnd:
+ p = NULL;
+ msg_debug_archive("7zip: read final section");
+ break;
+ default:
+ p = NULL;
+ msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+ break;
+ }
+
+ return p;
+}
+
+static void
+rspamd_archive_process_7zip(struct rspamd_task *task,
+ struct rspamd_mime_part *part)
+{
+ struct rspamd_archive *arch;
+ const guchar *start, *p, *end;
+ const guchar sz_magic[] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C};
+ guint64 section_offset = 0, section_length = 0;
+
+ start = part->parsed_data.begin;
+ p = start;
+ end = p + part->parsed_data.len;
+
+ if (end - p <= sizeof(guint64) + sizeof(guint32) ||
+ memcmp(p, sz_magic, sizeof(sz_magic)) != 0) {
+ msg_debug_archive("7z archive is invalid (no 7z magic)");
+
+ return;
+ }
+
+ arch = rspamd_mempool_alloc0(task->task_pool, sizeof(*arch));
+ arch->files = g_ptr_array_new();
+ arch->type = RSPAMD_ARCHIVE_7ZIP;
+ rspamd_mempool_add_destructor(task->task_pool, rspamd_archive_dtor,
+ arch);
+
+ /* Magic (6 bytes) + version (2 bytes) + crc32 (4 bytes) */
+ p += sizeof(guint64) + sizeof(guint32);
+
+ SZ_READ_UINT64(section_offset);
+ SZ_READ_UINT64(section_length);
+
+ if (end - p > sizeof(guint32)) {
+ p += sizeof(guint32);
+ }
+ else {
+ msg_debug_archive("7z archive is invalid (truncated crc)");
+
+ return;
+ }
+
+ if (end - p > section_offset) {
+ p += section_offset;
+ }
+ else {
+ msg_debug_archive("7z archive is invalid (incorrect section offset)");
+
+ return;
+ }
+
+ while ((p = rspamd_7zip_read_next_section(task, p, end, arch)) != NULL)
+ ;
+
+ part->part_type = RSPAMD_MIME_PART_ARCHIVE;
+ part->specific.arch = arch;
+ if (part->cd != NULL) {
+ arch->archive_name = &part->cd->filename;
+ }
+ arch->size = part->parsed_data.len;
+}
+
+static void
+rspamd_archive_process_gzip(struct rspamd_task *task,
+ struct rspamd_mime_part *part)
+{
+ struct rspamd_archive *arch;
+ const guchar *start, *p, *end;
+ const guchar gz_magic[] = {0x1F, 0x8B};
+ guchar flags;
+
+ start = part->parsed_data.begin;
+ p = start;
+ end = p + part->parsed_data.len;
+
+ if (end - p <= 10 || memcmp(p, gz_magic, sizeof(gz_magic)) != 0) {
+ msg_debug_archive("gzip archive is invalid (no gzip magic)");
+
+ return;
+ }
+
+ arch = rspamd_mempool_alloc0(task->task_pool, sizeof(*arch));
+ arch->files = g_ptr_array_sized_new(1);
+ arch->type = RSPAMD_ARCHIVE_GZIP;
+ if (part->cd) {
+ arch->archive_name = &part->cd->filename;
+ }
+ rspamd_mempool_add_destructor(task->task_pool, rspamd_archive_dtor,
+ arch);
+
+ flags = p[3];
+
+ if (flags & (1u << 5)) {
+ arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
+ }
+
+ if (flags & (1u << 3)) {
+ /* We have file name presented in archive, try to use it */
+ if (flags & (1u << 1)) {
+ /* Multipart */
+ p += 12;
+ }
+ else {
+ p += 10;
+ }
+
+ if (flags & (1u << 2)) {
+ /* Optional section */
+ guint16 optlen = 0;
+
+ RAR_READ_UINT16(optlen);
+
+ if (end <= p + optlen) {
+ msg_debug_archive("gzip archive is invalid, bad extra length: %d",
+ (int) optlen);
+
+ return;
+ }
+
+ p += optlen;
+ }
+
+ /* Read file name */
+ const guchar *fname_start = p;
+
+ while (p < end) {
+ if (*p == '\0') {
+ if (p > fname_start) {
+ struct rspamd_archive_file *f;
+
+ f = g_malloc0(sizeof(*f));
+
+ rspamd_archive_file_try_utf(task, arch, f,
+ fname_start, p - fname_start);
+
+ if (f->fname) {
+ g_ptr_array_add(arch->files, f);
+
+ if (f->flags & RSPAMD_ARCHIVE_FILE_OBFUSCATED) {
+ arch->flags |= RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES;
+ }
+ }
+ else {
+ /* Invalid filename, skip */
+ g_free(f);
+ }
+
+ goto set;
+ }
+ }
+
+ p++;
+ }
+
+ /* Wrong filename, not zero terminated */
+ msg_debug_archive("gzip archive is invalid, bad filename at pos %d",
+ (int) (p - start));
+
+ return;
+ }
+
+ /* Fallback, we need to extract file name from archive name if possible */
+ if (part->cd && part->cd->filename.len > 0) {
+ const gchar *dot_pos, *slash_pos;
+
+ dot_pos = rspamd_memrchr(part->cd->filename.begin, '.',
+ part->cd->filename.len);
+
+ if (dot_pos) {
+ struct rspamd_archive_file *f;
+
+ slash_pos = rspamd_memrchr(part->cd->filename.begin, '/',
+ part->cd->filename.len);
+
+ if (slash_pos && slash_pos < dot_pos) {
+ f = g_malloc0(sizeof(*f));
+ f->fname = g_string_sized_new(dot_pos - slash_pos);
+ g_string_append_len(f->fname, slash_pos + 1,
+ dot_pos - slash_pos - 1);
+
+ msg_debug_archive("fallback to gzip filename based on cd: %v",
+ f->fname);
+
+ g_ptr_array_add(arch->files, f);
+
+ goto set;
+ }
+ else {
+ const gchar *fname_start = part->cd->filename.begin;
+
+ f = g_malloc0(sizeof(*f));
+
+ if (memchr(fname_start, '.', part->cd->filename.len) != dot_pos) {
+ /* Double dots, something like foo.exe.gz */
+ f->fname = g_string_sized_new(dot_pos - fname_start);
+ g_string_append_len(f->fname, fname_start,
+ dot_pos - fname_start);
+ }
+ else {
+ /* Single dot, something like foo.gzz */
+ f->fname = g_string_sized_new(part->cd->filename.len);
+ g_string_append_len(f->fname, fname_start,
+ part->cd->filename.len);
+ }
+
+ msg_debug_archive("fallback to gzip filename based on cd: %v",
+ f->fname);
+
+ g_ptr_array_add(arch->files, f);
+
+ goto set;
+ }
+ }
+ }
+
+ return;
+
+set:
+ /* Set archive data */
+ part->part_type = RSPAMD_MIME_PART_ARCHIVE;
+ part->specific.arch = arch;
+ arch->size = part->parsed_data.len;
+}
+
+static gboolean
+rspamd_archive_cheat_detect(struct rspamd_mime_part *part, const gchar *str,
+ const guchar *magic_start, gsize magic_len)
+{
+ struct rspamd_content_type *ct;
+ const gchar *p;
+ rspamd_ftok_t srch, *fname;
+
+ ct = part->ct;
+ RSPAMD_FTOK_ASSIGN(&srch, "application");
+
+ if (ct && ct->type.len && ct->subtype.len > 0 && rspamd_ftok_cmp(&ct->type, &srch) == 0) {
+ if (rspamd_substring_search_caseless(ct->subtype.begin, ct->subtype.len,
+ str, strlen(str)) != -1) {
+ /* We still need to check magic, see #1848 */
+ if (magic_start != NULL) {
+ if (part->parsed_data.len > magic_len &&
+ memcmp(part->parsed_data.begin,
+ magic_start, magic_len) == 0) {
+ return TRUE;
+ }
+ /* No magic, refuse this type of archive */
+ return FALSE;
+ }
+ else {
+ return TRUE;
+ }
+ }
+ }
+
+ if (part->cd) {
+ fname = &part->cd->filename;
+
+ if (fname && fname->len > strlen(str)) {
+ p = fname->begin + fname->len - strlen(str);
+
+ if (rspamd_lc_cmp(p, str, strlen(str)) == 0) {
+ if (*(p - 1) == '.') {
+ if (magic_start != NULL) {
+ if (part->parsed_data.len > magic_len &&
+ memcmp(part->parsed_data.begin,
+ magic_start, magic_len) == 0) {
+ return TRUE;
+ }
+ /* No magic, refuse this type of archive */
+ return FALSE;
+ }
+
+ return TRUE;
+ }
+ }
+ }
+
+ if (magic_start != NULL) {
+ if (part->parsed_data.len > magic_len &&
+ memcmp(part->parsed_data.begin, magic_start, magic_len) == 0) {
+ return TRUE;
+ }
+ }
+ }
+ else {
+ if (magic_start != NULL) {
+ if (part->parsed_data.len > magic_len &&
+ memcmp(part->parsed_data.begin, magic_start, magic_len) == 0) {
+ return TRUE;
+ }
+ }
+ }
+
+ return FALSE;
+}
+
+void rspamd_archives_process(struct rspamd_task *task)
+{
+ guint i;
+ struct rspamd_mime_part *part;
+ const guchar rar_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07};
+ const guchar zip_magic[] = {0x50, 0x4b, 0x03, 0x04};
+ const guchar sz_magic[] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C};
+ const guchar gz_magic[] = {0x1F, 0x8B, 0x08};
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ if (part->part_type == RSPAMD_MIME_PART_UNDEFINED) {
+ if (part->parsed_data.len > 0) {
+ if (rspamd_archive_cheat_detect(part, "zip",
+ zip_magic, sizeof(zip_magic))) {
+ rspamd_archive_process_zip(task, part);
+ }
+ else if (rspamd_archive_cheat_detect(part, "rar",
+ rar_magic, sizeof(rar_magic))) {
+ rspamd_archive_process_rar(task, part);
+ }
+ else if (rspamd_archive_cheat_detect(part, "7z",
+ sz_magic, sizeof(sz_magic))) {
+ rspamd_archive_process_7zip(task, part);
+ }
+ else if (rspamd_archive_cheat_detect(part, "gz",
+ gz_magic, sizeof(gz_magic))) {
+ rspamd_archive_process_gzip(task, part);
+ }
+
+ if (part->ct && (part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT) &&
+ part->part_type == RSPAMD_MIME_PART_ARCHIVE &&
+ part->specific.arch) {
+ struct rspamd_archive *arch = part->specific.arch;
+
+ msg_info_task("found %s archive with incorrect content-type: %T/%T",
+ rspamd_archive_type_str(arch->type),
+ &part->ct->type, &part->ct->subtype);
+
+ if (!(part->ct->flags & RSPAMD_CONTENT_TYPE_MISSING)) {
+ part->ct->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+ }
+ }
+ }
+ }
+ }
+}
+
+
+const gchar *
+rspamd_archive_type_str(enum rspamd_archive_type type)
+{
+ const gchar *ret = "unknown";
+
+ switch (type) {
+ case RSPAMD_ARCHIVE_ZIP:
+ ret = "zip";
+ break;
+ case RSPAMD_ARCHIVE_RAR:
+ ret = "rar";
+ break;
+ case RSPAMD_ARCHIVE_7ZIP:
+ ret = "7z";
+ break;
+ case RSPAMD_ARCHIVE_GZIP:
+ ret = "gz";
+ break;
+ }
+
+ return ret;
+}
diff --git a/src/libmime/archives.h b/src/libmime/archives.h
new file mode 100644
index 0000000..56beb62
--- /dev/null
+++ b/src/libmime/archives.h
@@ -0,0 +1,72 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_ARCHIVES_H_
+#define SRC_LIBMIME_ARCHIVES_H_
+
+#include "config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum rspamd_archive_type {
+ RSPAMD_ARCHIVE_ZIP,
+ RSPAMD_ARCHIVE_RAR,
+ RSPAMD_ARCHIVE_7ZIP,
+ RSPAMD_ARCHIVE_GZIP,
+};
+
+enum rspamd_archive_flags {
+ RSPAMD_ARCHIVE_ENCRYPTED = (1u << 0u),
+ RSPAMD_ARCHIVE_CANNOT_READ = (1u << 1u),
+ RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES = (1u << 2u),
+};
+
+enum rspamd_archive_file_flags {
+ RSPAMD_ARCHIVE_FILE_ENCRYPTED = (1u << 0u),
+ RSPAMD_ARCHIVE_FILE_OBFUSCATED = (1u << 1u),
+};
+
+struct rspamd_archive_file {
+ GString *fname;
+ gsize compressed_size;
+ gsize uncompressed_size;
+ enum rspamd_archive_file_flags flags;
+};
+
+struct rspamd_archive {
+ enum rspamd_archive_type type;
+ const rspamd_ftok_t *archive_name;
+ gsize size;
+ enum rspamd_archive_flags flags;
+ GPtrArray *files; /* Array of struct rspamd_archive_file */
+};
+
+/**
+ * Process archives from a worker task
+ */
+void rspamd_archives_process(struct rspamd_task *task);
+
+/**
+ * Get textual representation of an archive's type
+ */
+const gchar *rspamd_archive_type_str(enum rspamd_archive_type type);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_ARCHIVES_H_ */
diff --git a/src/libmime/content_type.c b/src/libmime/content_type.c
new file mode 100644
index 0000000..765cb87
--- /dev/null
+++ b/src/libmime/content_type.c
@@ -0,0 +1,884 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "libmime/content_type.h"
+#include "smtp_parsers.h"
+#include "utlist.h"
+#include "libserver/url.h"
+#include "libmime/mime_encoding.h"
+
+static gboolean
+rspamd_rfc2231_decode(rspamd_mempool_t *pool,
+ struct rspamd_content_type_param *param,
+ gchar *value_start, gchar *value_end)
+{
+ gchar *quote_pos;
+
+ quote_pos = memchr(value_start, '\'', value_end - value_start);
+
+ if (quote_pos == NULL) {
+ /* Plain percent encoding */
+ gsize r = rspamd_url_decode(value_start, value_start,
+ value_end - value_start);
+ param->value.begin = value_start;
+ param->value.len = r;
+ }
+ else {
+ /*
+ * We can have encoding'language'data, or
+ * encoding'data (in theory).
+ * Try to handle both...
+ */
+ const gchar *charset = NULL;
+ rspamd_ftok_t ctok;
+
+ ctok.begin = value_start;
+ ctok.len = quote_pos - value_start;
+
+ if (ctok.len > 0) {
+ charset = rspamd_mime_detect_charset(&ctok, pool);
+ }
+
+ /* Now, we can check for either next quote sign or, eh, ignore that */
+ value_start = quote_pos + 1;
+
+ quote_pos = memchr(value_start, '\'', value_end - value_start);
+
+ if (quote_pos) {
+ /* Ignore language */
+ value_start = quote_pos + 1;
+ }
+
+ /* Perform percent decoding */
+ gsize r = rspamd_url_decode(value_start, value_start,
+ value_end - value_start);
+ GError *err = NULL;
+
+ if (charset == NULL) {
+ /* Try heuristic */
+ charset = rspamd_mime_charset_find_by_content(value_start, r, TRUE);
+ }
+
+ if (charset == NULL) {
+ msg_warn_pool("cannot convert parameter from charset %T", &ctok);
+
+ return FALSE;
+ }
+
+ param->value.begin = rspamd_mime_text_to_utf8(pool,
+ value_start, r,
+ charset, &param->value.len, &err);
+
+ if (param->value.begin == NULL) {
+ msg_warn_pool("cannot convert parameter from charset %s: %e",
+ charset, err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return FALSE;
+ }
+ }
+
+ param->flags |= RSPAMD_CONTENT_PARAM_RFC2231;
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_param_maybe_rfc2231_process(rspamd_mempool_t *pool,
+ struct rspamd_content_type_param *param,
+ gchar *name_start, gchar *name_end,
+ gchar *value_start, gchar *value_end)
+{
+ const gchar *star_pos;
+
+ star_pos = memchr(name_start, '*', name_end - name_start);
+
+ if (star_pos == NULL) {
+ return FALSE;
+ }
+
+ /* We have three possibilities here:
+ * 1. name* (just name + 2231 encoding)
+ * 2. name*(\d+) (piecewise stuff but no rfc2231 encoding)
+ * 3. name*(\d+)* (piecewise stuff and rfc2231 encoding)
+ */
+
+ if (star_pos == name_end - 1) {
+ /* First */
+ if (rspamd_rfc2231_decode(pool, param, value_start, value_end)) {
+ param->name.begin = name_start;
+ param->name.len = name_end - name_start - 1;
+ }
+ }
+ else if (*(name_end - 1) == '*') {
+ /* Third */
+ /* Check number */
+ gulong tmp;
+
+ if (!rspamd_strtoul(star_pos + 1, name_end - star_pos - 2, &tmp)) {
+ return FALSE;
+ }
+
+ param->flags |= RSPAMD_CONTENT_PARAM_PIECEWISE | RSPAMD_CONTENT_PARAM_RFC2231;
+ param->rfc2231_id = tmp;
+ param->name.begin = name_start;
+ param->name.len = star_pos - name_start;
+ param->value.begin = value_start;
+ param->value.len = value_end - value_start;
+
+ /* Deal with that later... */
+ }
+ else {
+ /* Second case */
+ gulong tmp;
+
+ if (!rspamd_strtoul(star_pos + 1, name_end - star_pos - 1, &tmp)) {
+ return FALSE;
+ }
+
+ param->flags |= RSPAMD_CONTENT_PARAM_PIECEWISE;
+ param->rfc2231_id = tmp;
+ param->name.begin = name_start;
+ param->name.len = star_pos - name_start;
+ param->value.begin = value_start;
+ param->value.len = value_end - value_start;
+ }
+
+ return TRUE;
+}
+
+static gint32
+rspamd_cmp_pieces(struct rspamd_content_type_param *p1, struct rspamd_content_type_param *p2)
+{
+ return p1->rfc2231_id - p2->rfc2231_id;
+}
+
+static void
+rspamd_postprocess_ct_attributes(rspamd_mempool_t *pool,
+ GHashTable *htb,
+ void (*proc)(rspamd_mempool_t *, struct rspamd_content_type_param *, gpointer ud),
+ gpointer procd)
+{
+ GHashTableIter it;
+ gpointer k, v;
+ struct rspamd_content_type_param *param, *sorted, *cur;
+
+ if (htb == NULL) {
+ return;
+ }
+
+ g_hash_table_iter_init(&it, htb);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ param = (struct rspamd_content_type_param *) v;
+
+ if (param->flags & RSPAMD_CONTENT_PARAM_PIECEWISE) {
+ /* Reconstruct param */
+ gsize tlen = 0;
+ gchar *ndata, *pos;
+
+ sorted = param;
+ DL_SORT(sorted, rspamd_cmp_pieces);
+
+ DL_FOREACH(sorted, cur)
+ {
+ tlen += cur->value.len;
+ }
+
+ ndata = rspamd_mempool_alloc(pool, tlen);
+ pos = ndata;
+
+ DL_FOREACH(sorted, cur)
+ {
+ memcpy(pos, cur->value.begin, cur->value.len);
+ pos += cur->value.len;
+ }
+
+ if (param->flags & RSPAMD_CONTENT_PARAM_RFC2231) {
+ if (!rspamd_rfc2231_decode(pool, param,
+ ndata, pos)) {
+ param->flags |= RSPAMD_CONTENT_PARAM_BROKEN;
+ param->value.begin = ndata;
+ param->value.len = tlen;
+ }
+ }
+ else {
+ param->value.begin = ndata;
+ param->value.len = tlen;
+ }
+
+ /* Detach from list */
+ param->next = NULL;
+ param->prev = param;
+ }
+
+ gboolean invalid_utf = FALSE;
+
+ if (param->value.begin != NULL && param->value.len > 0) {
+ param->value.begin = rspamd_mime_header_decode(pool, param->value.begin,
+ param->value.len, &invalid_utf);
+ param->value.len = strlen(param->value.begin);
+ }
+
+ if (invalid_utf) {
+ param->flags |= RSPAMD_CONTENT_PARAM_BROKEN;
+ }
+
+ proc(pool, param, procd);
+ }
+}
+
+static void
+rspamd_content_type_postprocess(rspamd_mempool_t *pool,
+ struct rspamd_content_type_param *param,
+ gpointer ud)
+{
+ rspamd_ftok_t srch;
+ struct rspamd_content_type_param *found = NULL;
+
+ struct rspamd_content_type *ct = (struct rspamd_content_type *) ud;
+
+ RSPAMD_FTOK_ASSIGN(&srch, "charset");
+
+ if (rspamd_ftok_icase_equal(&param->name, &srch)) {
+ /* Adjust charset */
+ found = param;
+ ct->charset.begin = param->value.begin;
+ ct->charset.len = param->value.len;
+ }
+
+ RSPAMD_FTOK_ASSIGN(&srch, "boundary");
+
+ if (rspamd_ftok_icase_equal(&param->name, &srch)) {
+ found = param;
+ gchar *lc_boundary;
+ /* Adjust boundary */
+ lc_boundary = rspamd_mempool_alloc(pool, param->value.len);
+ memcpy(lc_boundary, param->value.begin, param->value.len);
+ rspamd_str_lc(lc_boundary, param->value.len);
+ ct->boundary.begin = lc_boundary;
+ ct->boundary.len = param->value.len;
+ /* Preserve original (case sensitive) boundary */
+ ct->orig_boundary.begin = param->value.begin;
+ ct->orig_boundary.len = param->value.len;
+ }
+
+ if (!found) {
+ RSPAMD_FTOK_ASSIGN(&srch, "name");
+ if (!rspamd_ftok_icase_equal(&param->name, &srch)) {
+ /* Just lowercase */
+ rspamd_str_lc_utf8((gchar *) param->value.begin, param->value.len);
+ }
+ }
+}
+
+static void
+rspamd_content_disposition_postprocess(rspamd_mempool_t *pool,
+ struct rspamd_content_type_param *param,
+ gpointer ud)
+{
+ rspamd_ftok_t srch;
+ struct rspamd_content_disposition *cd = (struct rspamd_content_disposition *) ud;
+
+ srch.begin = "filename";
+ srch.len = 8;
+
+ if (rspamd_ftok_icase_equal(&param->name, &srch)) {
+ /* Adjust filename */
+ cd->filename.begin = param->value.begin;
+ cd->filename.len = param->value.len;
+ }
+}
+
+void rspamd_content_type_add_param(rspamd_mempool_t *pool,
+ struct rspamd_content_type *ct,
+ gchar *name_start, gchar *name_end,
+ gchar *value_start, gchar *value_end)
+{
+ struct rspamd_content_type_param *nparam;
+ rspamd_ftok_t srch;
+ struct rspamd_content_type_param *found = NULL;
+
+ g_assert(ct != NULL);
+
+ nparam = rspamd_mempool_alloc0(pool, sizeof(*nparam));
+ rspamd_str_lc(name_start, name_end - name_start);
+
+ if (!rspamd_param_maybe_rfc2231_process(pool, nparam, name_start,
+ name_end, value_start, value_end)) {
+ nparam->name.begin = name_start;
+ nparam->name.len = name_end - name_start;
+ nparam->value.begin = value_start;
+ nparam->value.len = value_end - value_start;
+ }
+
+ srch.begin = nparam->name.begin;
+ srch.len = nparam->name.len;
+
+ if (ct->attrs) {
+ found = g_hash_table_lookup(ct->attrs, &srch);
+ }
+ else {
+ ct->attrs = g_hash_table_new(rspamd_ftok_icase_hash,
+ rspamd_ftok_icase_equal);
+ }
+
+ if (!found) {
+ DL_APPEND(found, nparam);
+ g_hash_table_insert(ct->attrs, &nparam->name, nparam);
+ }
+ else {
+ DL_APPEND(found, nparam);
+ }
+}
+
+static struct rspamd_content_type *
+rspamd_content_type_parser(gchar *in, gsize len, rspamd_mempool_t *pool)
+{
+ guint obraces = 0, ebraces = 0, qlen = 0;
+ gchar *p, *c, *end, *pname_start = NULL, *pname_end = NULL;
+ struct rspamd_content_type *res = NULL, val;
+ gboolean eqsign_seen = FALSE;
+ enum {
+ parse_type,
+ parse_subtype,
+ parse_after_subtype,
+ parse_param_name,
+ parse_param_after_name,
+ parse_param_value,
+ parse_param_value_after_quote,
+ parse_space,
+ parse_quoted,
+ parse_comment,
+ } state = parse_space,
+ next_state = parse_type;
+
+ p = in;
+ c = p;
+ end = p + len;
+ memset(&val, 0, sizeof(val));
+ val.cpy = in;
+
+ while (p < end) {
+ switch (state) {
+ case parse_type:
+ if (g_ascii_isspace(*p) || *p == ';') {
+ /* We have type without subtype */
+ val.type.begin = c;
+ val.type.len = p - c;
+ state = parse_after_subtype;
+ }
+ else if (*p == '/') {
+ val.type.begin = c;
+ val.type.len = p - c;
+ state = parse_space;
+ next_state = parse_subtype;
+ p++;
+ }
+ else {
+ p++;
+ }
+ break;
+ case parse_subtype:
+ if (g_ascii_isspace(*p) || *p == ';') {
+ val.subtype.begin = c;
+ val.subtype.len = p - c;
+ state = parse_after_subtype;
+ }
+ else {
+ p++;
+ }
+ break;
+ case parse_after_subtype:
+ if (*p == ';' || g_ascii_isspace(*p)) {
+ p++;
+ }
+ else if (*p == '(') {
+ c = p;
+ state = parse_comment;
+ next_state = parse_param_name;
+ obraces = 1;
+ ebraces = 0;
+ pname_start = NULL;
+ pname_end = NULL;
+ eqsign_seen = FALSE;
+ p++;
+ }
+ else {
+ c = p;
+ state = parse_param_name;
+ pname_start = NULL;
+ pname_end = NULL;
+ eqsign_seen = FALSE;
+ }
+ break;
+ case parse_param_name:
+ if (*p == '=') {
+ pname_start = c;
+ pname_end = p;
+ state = parse_param_after_name;
+ eqsign_seen = TRUE;
+ p++;
+ }
+ else if (g_ascii_isspace(*p)) {
+ pname_start = c;
+ pname_end = p;
+ state = parse_param_after_name;
+ }
+ else {
+ p++;
+ }
+ break;
+ case parse_param_after_name:
+ if (g_ascii_isspace(*p)) {
+ p++;
+ }
+ else if (*p == '=') {
+ if (eqsign_seen) {
+ /* Treat as value start */
+ c = p;
+ eqsign_seen = FALSE;
+ state = parse_param_value;
+ p++;
+ }
+ else {
+ eqsign_seen = TRUE;
+ p++;
+ }
+ }
+ else {
+ if (eqsign_seen) {
+ state = parse_param_value;
+ c = p;
+ }
+ else {
+ /* Invalid parameter without value */
+ c = p;
+ state = parse_param_name;
+ pname_start = NULL;
+ pname_end = NULL;
+ }
+ }
+ break;
+ case parse_param_value:
+ if (*p == '"') {
+ p++;
+ c = p;
+ state = parse_quoted;
+ next_state = parse_param_value_after_quote;
+ }
+ else if (g_ascii_isspace(*p)) {
+ if (pname_start && pname_end && pname_end > pname_start) {
+ rspamd_content_type_add_param(pool, &val, pname_start,
+ pname_end, c, p);
+ }
+
+ state = parse_space;
+ next_state = parse_param_name;
+ pname_start = NULL;
+ pname_end = NULL;
+ }
+ else if (*p == '(') {
+ if (pname_start && pname_end && pname_end > pname_start) {
+ rspamd_content_type_add_param(pool, &val, pname_start,
+ pname_end, c, p);
+ }
+
+ obraces = 1;
+ ebraces = 0;
+ p++;
+ state = parse_comment;
+ next_state = parse_param_name;
+ pname_start = NULL;
+ pname_end = NULL;
+ }
+ else if (*p == ';') {
+ if (pname_start && pname_end && pname_end > pname_start) {
+ rspamd_content_type_add_param(pool, &val, pname_start,
+ pname_end, c, p);
+ }
+
+ p++;
+ state = parse_space;
+ next_state = parse_param_name;
+ pname_start = NULL;
+ pname_end = NULL;
+ }
+ else {
+ p++;
+ }
+ break;
+ case parse_param_value_after_quote:
+ if (pname_start && pname_end && pname_end > pname_start) {
+ rspamd_content_type_add_param(pool, &val, pname_start,
+ pname_end, c, c + qlen);
+ }
+
+ if (*p == '"') {
+ p++;
+
+ if (p == end) {
+ /* Last quote: done... */
+ state = parse_space;
+ break;
+ }
+
+ if (*p == ';') {
+ p++;
+ state = parse_space;
+ next_state = parse_param_name;
+ pname_start = NULL;
+ pname_end = NULL;
+ continue;
+ }
+ }
+
+ /* We should not normally be here in fact */
+ if (g_ascii_isspace(*p)) {
+ state = parse_space;
+ next_state = parse_param_name;
+ pname_start = NULL;
+ pname_end = NULL;
+ }
+ else if (*p == '(') {
+ obraces = 1;
+ ebraces = 0;
+ p++;
+ state = parse_comment;
+ next_state = parse_param_name;
+ pname_start = NULL;
+ pname_end = NULL;
+ }
+ else {
+ state = parse_param_name;
+ pname_start = NULL;
+ pname_end = NULL;
+ c = p;
+ }
+ break;
+ case parse_quoted:
+ if (*p == '\\') {
+ /* Quoted pair */
+ if (p + 1 < end) {
+ p += 2;
+ }
+ else {
+ p++;
+ }
+ }
+ else if (*p == '"') {
+ qlen = p - c;
+ state = next_state;
+ }
+ else {
+ p++;
+ }
+ break;
+ case parse_comment:
+ if (*p == '(') {
+ obraces++;
+ p++;
+ }
+ else if (*p == ')') {
+ ebraces++;
+ p++;
+
+ if (ebraces == obraces && p < end) {
+ if (g_ascii_isspace(*p)) {
+ state = parse_space;
+ }
+ else {
+ c = p;
+ state = next_state;
+ }
+ }
+ }
+ else {
+ p++;
+ }
+ break;
+ case parse_space:
+ if (g_ascii_isspace(*p)) {
+ p++;
+ }
+ else if (*p == '(') {
+ obraces = 1;
+ ebraces = 0;
+ p++;
+ state = parse_comment;
+ }
+ else {
+ c = p;
+ state = next_state;
+ }
+ break;
+ }
+ }
+
+ /* Process leftover */
+ switch (state) {
+ case parse_type:
+ val.type.begin = c;
+ val.type.len = p - c;
+ break;
+ case parse_subtype:
+ val.subtype.begin = c;
+ val.subtype.len = p - c;
+ break;
+ case parse_param_value:
+ if (pname_start && pname_end && pname_end > pname_start) {
+ if (p > c && *(p - 1) == ';') {
+ p--;
+ }
+
+ rspamd_content_type_add_param(pool, &val, pname_start,
+ pname_end, c, p);
+ }
+ break;
+ case parse_param_value_after_quote:
+ if (pname_start && pname_end && pname_end > pname_start) {
+ rspamd_content_type_add_param(pool, &val, pname_start,
+ pname_end, c, c + qlen);
+ }
+ break;
+ default:
+ break;
+ }
+
+ if (val.type.len > 0) {
+ gchar *tmp;
+
+ res = rspamd_mempool_alloc(pool, sizeof(val));
+ memcpy(res, &val, sizeof(val));
+
+ /*
+ * Lowercase type and subtype as they are specified as case insensitive
+ * in rfc2045 section 5.1
+ */
+ tmp = rspamd_mempool_alloc(pool, val.type.len);
+ memcpy(tmp, val.type.begin, val.type.len);
+ rspamd_str_lc(tmp, val.type.len);
+ res->type.begin = tmp;
+
+ if (val.subtype.len > 0) {
+ tmp = rspamd_mempool_alloc(pool, val.subtype.len);
+ memcpy(tmp, val.subtype.begin, val.subtype.len);
+ rspamd_str_lc(tmp, val.subtype.len);
+ res->subtype.begin = tmp;
+ }
+ }
+
+ return res;
+}
+
+struct rspamd_content_type *
+rspamd_content_type_parse(const gchar *in,
+ gsize len, rspamd_mempool_t *pool)
+{
+ struct rspamd_content_type *res = NULL;
+ rspamd_ftok_t srch;
+ gchar *cpy;
+
+ cpy = rspamd_mempool_alloc(pool, len + 1);
+ rspamd_strlcpy(cpy, in, len + 1);
+
+ if ((res = rspamd_content_type_parser(cpy, len, pool)) != NULL) {
+ if (res->attrs) {
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) g_hash_table_unref, res->attrs);
+
+ rspamd_postprocess_ct_attributes(pool, res->attrs,
+ rspamd_content_type_postprocess, res);
+ }
+
+ /* Now do some hacks to work with broken content types */
+ if (res->subtype.len == 0) {
+ res->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+ RSPAMD_FTOK_ASSIGN(&srch, "text");
+
+ if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+ /* Workaround for Content-Type: text */
+ /* Assume text/plain */
+ RSPAMD_FTOK_ASSIGN(&srch, "plain");
+ }
+ else {
+ RSPAMD_FTOK_ASSIGN(&srch, "html");
+
+ if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+ /* Workaround for Content-Type: html */
+ RSPAMD_FTOK_ASSIGN(&res->type, "text");
+ RSPAMD_FTOK_ASSIGN(&res->subtype, "html");
+ }
+ else {
+ RSPAMD_FTOK_ASSIGN(&srch, "application");
+
+ if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+ RSPAMD_FTOK_ASSIGN(&res->subtype, "octet-stream");
+ }
+ }
+ }
+ }
+ else {
+ /* Common mistake done by retards */
+ RSPAMD_FTOK_ASSIGN(&srch, "alternate");
+
+ if (rspamd_ftok_casecmp(&res->subtype, &srch) == 0) {
+ res->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+ RSPAMD_FTOK_ASSIGN(&res->subtype, "alternative");
+ }
+
+ /* PKCS7 smime */
+ RSPAMD_FTOK_ASSIGN(&srch, "pkcs7-mime");
+ if (rspamd_substring_search(res->subtype.begin, res->subtype.len,
+ srch.begin, srch.len) != -1) {
+ res->flags |= RSPAMD_CONTENT_TYPE_SMIME;
+ }
+ }
+
+ RSPAMD_FTOK_ASSIGN(&srch, "multipart");
+
+ if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+ res->flags |= RSPAMD_CONTENT_TYPE_MULTIPART;
+
+ RSPAMD_FTOK_ASSIGN(&srch, "encrypted");
+ if (rspamd_ftok_casecmp(&res->subtype, &srch) == 0) {
+ res->flags |= RSPAMD_CONTENT_TYPE_ENCRYPTED;
+ }
+ }
+ else {
+ RSPAMD_FTOK_ASSIGN(&srch, "text");
+
+ if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+ res->flags |= RSPAMD_CONTENT_TYPE_TEXT;
+ }
+ else {
+ RSPAMD_FTOK_ASSIGN(&srch, "message");
+
+ if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+ RSPAMD_FTOK_ASSIGN(&srch, "delivery-status");
+
+ if (rspamd_ftok_casecmp(&res->subtype, &srch) == 0) {
+ res->flags |= RSPAMD_CONTENT_TYPE_TEXT | RSPAMD_CONTENT_TYPE_DSN;
+ }
+ else {
+ RSPAMD_FTOK_ASSIGN(&srch, "notification");
+
+ if (rspamd_substring_search_caseless(res->subtype.begin,
+ res->subtype.len, srch.begin, srch.len) != -1) {
+ res->flags |= RSPAMD_CONTENT_TYPE_TEXT |
+ RSPAMD_CONTENT_TYPE_DSN;
+ }
+ else {
+ res->flags |= RSPAMD_CONTENT_TYPE_MESSAGE;
+ }
+ }
+ }
+ }
+ }
+ }
+ else {
+ msg_warn_pool("cannot parse content type: %*s", (gint) len, cpy);
+ }
+
+ return res;
+}
+
+void rspamd_content_disposition_add_param(rspamd_mempool_t *pool,
+ struct rspamd_content_disposition *cd,
+ const gchar *name_start, const gchar *name_end,
+ const gchar *value_start, const gchar *value_end)
+{
+ rspamd_ftok_t srch;
+ gchar *name_cpy, *value_cpy, *name_cpy_end, *value_cpy_end;
+ struct rspamd_content_type_param *found = NULL, *nparam;
+
+ g_assert(cd != NULL);
+
+ name_cpy = rspamd_mempool_alloc(pool, name_end - name_start);
+ memcpy(name_cpy, name_start, name_end - name_start);
+ name_cpy_end = name_cpy + (name_end - name_start);
+
+ value_cpy = rspamd_mempool_alloc(pool, value_end - value_start);
+ memcpy(value_cpy, value_start, value_end - value_start);
+ value_cpy_end = value_cpy + (value_end - value_start);
+
+ nparam = rspamd_mempool_alloc0(pool, sizeof(*nparam));
+ rspamd_str_lc(name_cpy, name_cpy_end - name_cpy);
+
+ if (!rspamd_param_maybe_rfc2231_process(pool, nparam, name_cpy,
+ name_cpy_end, value_cpy, value_cpy_end)) {
+ nparam->name.begin = name_cpy;
+ nparam->name.len = name_cpy_end - name_cpy;
+ nparam->value.begin = value_cpy;
+ nparam->value.len = value_cpy_end - value_cpy;
+ }
+
+ srch.begin = nparam->name.begin;
+ srch.len = nparam->name.len;
+
+ if (cd->attrs) {
+ found = g_hash_table_lookup(cd->attrs, &srch);
+ }
+ else {
+ cd->attrs = g_hash_table_new(rspamd_ftok_icase_hash,
+ rspamd_ftok_icase_equal);
+ }
+
+ if (!found) {
+ DL_APPEND(found, nparam);
+ g_hash_table_insert(cd->attrs, &nparam->name, nparam);
+ }
+ else {
+ DL_APPEND(found, nparam);
+ }
+}
+
+struct rspamd_content_disposition *
+rspamd_content_disposition_parse(const gchar *in,
+ gsize len, rspamd_mempool_t *pool)
+{
+ struct rspamd_content_disposition *res = NULL, val;
+
+ if (rspamd_content_disposition_parser(in, len, &val, pool)) {
+
+ if (val.type == RSPAMD_CT_UNKNOWN) {
+ /* 'Fix' type to attachment as MUA does */
+ val.type = RSPAMD_CT_ATTACHMENT;
+ }
+
+ res = rspamd_mempool_alloc(pool, sizeof(val));
+ memcpy(res, &val, sizeof(val));
+ res->lc_data = rspamd_mempool_alloc(pool, len + 1);
+ rspamd_strlcpy(res->lc_data, in, len + 1);
+ rspamd_str_lc(res->lc_data, len);
+
+ if (res->attrs) {
+ rspamd_postprocess_ct_attributes(pool, res->attrs,
+ rspamd_content_disposition_postprocess, res);
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) g_hash_table_unref, res->attrs);
+ }
+ }
+ else {
+ msg_warn_pool("cannot parse content disposition: %*s",
+ (gint) len, in);
+ }
+
+ return res;
+}
diff --git a/src/libmime/content_type.h b/src/libmime/content_type.h
new file mode 100644
index 0000000..ac49bdc
--- /dev/null
+++ b/src/libmime/content_type.h
@@ -0,0 +1,130 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_CONTENT_TYPE_H_
+#define SRC_LIBMIME_CONTENT_TYPE_H_
+
+#include "config.h"
+#include "libutil/fstring.h"
+#include "libutil/mem_pool.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum rspamd_content_type_flags {
+ RSPAMD_CONTENT_TYPE_VALID = 0,
+ RSPAMD_CONTENT_TYPE_BROKEN = 1 << 0,
+ RSPAMD_CONTENT_TYPE_MULTIPART = 1 << 1,
+ RSPAMD_CONTENT_TYPE_TEXT = 1 << 2,
+ RSPAMD_CONTENT_TYPE_MESSAGE = 1 << 3,
+ RSPAMD_CONTENT_TYPE_DSN = 1 << 4,
+ RSPAMD_CONTENT_TYPE_MISSING = 1 << 5,
+ RSPAMD_CONTENT_TYPE_ENCRYPTED = 1 << 6,
+ RSPAMD_CONTENT_TYPE_SMIME = 1 << 7,
+};
+
+enum rspamd_content_param_flags {
+ RSPAMD_CONTENT_PARAM_NORMAL = 0,
+ RSPAMD_CONTENT_PARAM_RFC2231 = (1 << 0),
+ RSPAMD_CONTENT_PARAM_PIECEWISE = (1 << 1),
+ RSPAMD_CONTENT_PARAM_BROKEN = (1 << 2),
+};
+
+struct rspamd_content_type_param {
+ rspamd_ftok_t name;
+ rspamd_ftok_t value;
+ guint rfc2231_id;
+ enum rspamd_content_param_flags flags;
+ struct rspamd_content_type_param *prev, *next;
+};
+
+struct rspamd_content_type {
+ gchar *cpy;
+ rspamd_ftok_t type;
+ rspamd_ftok_t subtype;
+ rspamd_ftok_t charset;
+ rspamd_ftok_t boundary;
+ rspamd_ftok_t orig_boundary;
+ enum rspamd_content_type_flags flags;
+ GHashTable *attrs; /* Can be empty */
+};
+
+enum rspamd_content_disposition_type {
+ RSPAMD_CT_UNKNOWN = 0,
+ RSPAMD_CT_INLINE = 1,
+ RSPAMD_CT_ATTACHMENT = 2,
+};
+
+struct rspamd_content_disposition {
+ gchar *lc_data;
+ enum rspamd_content_disposition_type type;
+ rspamd_ftok_t filename;
+ GHashTable *attrs; /* Can be empty */
+};
+
+/**
+ * Adds new parameter to content type structure
+ * @param ct
+ * @param name_start (can be modified)
+ * @param name_end
+ * @param value_start (can be modified)
+ * @param value_end
+ */
+void rspamd_content_type_add_param(rspamd_mempool_t *pool,
+ struct rspamd_content_type *ct,
+ gchar *name_start, gchar *name_end,
+ gchar *value_start, gchar *value_end);
+
+/**
+ * Parse content type from the header (performs copy + lowercase)
+ * @param in
+ * @param len
+ * @param pool
+ * @return
+ */
+struct rspamd_content_type *rspamd_content_type_parse(const gchar *in,
+ gsize len, rspamd_mempool_t *pool);
+
+/**
+ * Adds new param for content disposition header
+ * @param pool
+ * @param cd
+ * @param name_start
+ * @param name_end
+ * @param value_start
+ * @param value_end
+ */
+void rspamd_content_disposition_add_param(rspamd_mempool_t *pool,
+ struct rspamd_content_disposition *cd,
+ const gchar *name_start, const gchar *name_end,
+ const gchar *value_start, const gchar *value_end);
+
+/**
+ * Parse content-disposition header
+ * @param in
+ * @param len
+ * @param pool
+ * @return
+ */
+struct rspamd_content_disposition *rspamd_content_disposition_parse(const gchar *in,
+ gsize len,
+ rspamd_mempool_t *pool);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_CONTENT_TYPE_H_ */
diff --git a/src/libmime/email_addr.c b/src/libmime/email_addr.c
new file mode 100644
index 0000000..0af7388
--- /dev/null
+++ b/src/libmime/email_addr.c
@@ -0,0 +1,563 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "email_addr.h"
+#include "message.h"
+#include "printf.h"
+#include "smtp_parsers.h"
+
+static void
+rspamd_email_address_unescape(struct rspamd_email_address *addr)
+{
+ const char *h, *end;
+ char *t, *d;
+
+ if (addr->user_len == 0) {
+ return;
+ }
+
+ d = g_malloc(addr->user_len);
+ t = d;
+ h = addr->user;
+ end = h + addr->user_len;
+
+ while (h < end) {
+ if (*h != '\\') {
+ *t++ = *h;
+ }
+ h++;
+ }
+
+ addr->user = d;
+ addr->user_len = t - d;
+ addr->flags |= RSPAMD_EMAIL_ADDR_USER_ALLOCATED;
+}
+
+struct rspamd_email_address *
+rspamd_email_address_from_smtp(const gchar *str, guint len)
+{
+ struct rspamd_email_address addr, *ret;
+ gsize nlen;
+
+ if (str == NULL || len == 0) {
+ return NULL;
+ }
+
+ rspamd_smtp_addr_parse(str, len, &addr);
+
+ if (addr.flags & RSPAMD_EMAIL_ADDR_VALID) {
+ ret = g_malloc(sizeof(*ret));
+ memcpy(ret, &addr, sizeof(addr));
+
+ if ((ret->flags & RSPAMD_EMAIL_ADDR_QUOTED) && ret->addr[0] == '"') {
+ if (ret->flags & RSPAMD_EMAIL_ADDR_HAS_BACKSLASH) {
+ /* We also need to unquote user */
+ rspamd_email_address_unescape(ret);
+ }
+
+ /* We need to unquote addr */
+ nlen = ret->domain_len + ret->user_len + 2;
+ ret->addr = g_malloc(nlen + 1);
+ ret->addr_len = rspamd_snprintf((char *) ret->addr, nlen, "%*s@%*s",
+ (gint) ret->user_len, ret->user,
+ (gint) ret->domain_len, ret->domain);
+ ret->flags |= RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED;
+ }
+
+ return ret;
+ }
+
+ return NULL;
+}
+
+void rspamd_email_address_free(struct rspamd_email_address *addr)
+{
+ if (addr) {
+ if (addr->flags & RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED) {
+ g_free((void *) addr->addr);
+ }
+
+ if (addr->flags & RSPAMD_EMAIL_ADDR_USER_ALLOCATED) {
+ g_free((void *) addr->user);
+ }
+
+ g_free(addr);
+ }
+}
+
+static inline void
+rspamd_email_address_add(rspamd_mempool_t *pool,
+ GPtrArray *ar,
+ struct rspamd_email_address *addr,
+ GString *name)
+{
+ struct rspamd_email_address *elt;
+ guint nlen;
+
+ elt = g_malloc0(sizeof(*elt));
+ rspamd_mempool_notify_alloc(pool, sizeof(*elt));
+
+ if (addr != NULL) {
+ memcpy(elt, addr, sizeof(*addr));
+ }
+ else {
+ elt->addr = "";
+ elt->domain = "";
+ elt->raw = "<>";
+ elt->raw_len = 2;
+ elt->user = "";
+ elt->flags |= RSPAMD_EMAIL_ADDR_EMPTY;
+ }
+
+ if ((elt->flags & RSPAMD_EMAIL_ADDR_QUOTED) && elt->addr[0] == '"') {
+ if (elt->flags & RSPAMD_EMAIL_ADDR_HAS_BACKSLASH) {
+ /* We also need to unquote user */
+ rspamd_email_address_unescape(elt);
+ }
+
+ /* We need to unquote addr */
+ nlen = elt->domain_len + elt->user_len + 2;
+ elt->addr = g_malloc(nlen + 1);
+ rspamd_mempool_notify_alloc(pool, nlen + 1);
+ elt->addr_len = rspamd_snprintf((char *) elt->addr, nlen, "%*s@%*s",
+ (gint) elt->user_len, elt->user,
+ (gint) elt->domain_len, elt->domain);
+ elt->flags |= RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED;
+ }
+
+ if (name->len > 0) {
+ rspamd_gstring_strip(name, " \t\v");
+ elt->name = rspamd_mime_header_decode(pool, name->str, name->len, NULL);
+ }
+
+ rspamd_mempool_notify_alloc(pool, name->len);
+ g_ptr_array_add(ar, elt);
+}
+
+/*
+ * Tries to parse an email address that doesn't conform RFC
+ */
+static gboolean
+rspamd_email_address_parse_heuristic(const char *data, size_t len,
+ struct rspamd_email_address *addr)
+{
+ const gchar *p = data, *at = NULL, *end = data + len;
+ gboolean ret = FALSE;
+
+ memset(addr, 0, sizeof(*addr));
+
+ if (*p == '<' && len > 1) {
+ /* Angled address */
+ addr->addr_len = rspamd_memcspn(p + 1, ">", len - 1);
+ addr->addr = p + 1;
+ addr->raw = p;
+ addr->raw_len = len;
+ ret = TRUE;
+
+ p = p + 1;
+ len = addr->addr_len;
+ end = p + len;
+ }
+ else if (len > 0) {
+ addr->addr = p;
+ addr->addr_len = len;
+ addr->raw = p;
+ addr->raw_len = len;
+ ret = TRUE;
+ }
+
+ if (ret) {
+ at = rspamd_memrchr(p, '@', len);
+
+ if (at != NULL && at + 1 < end) {
+ addr->domain = at + 1;
+ addr->domain_len = end - (at + 1);
+ addr->user = p;
+ addr->user_len = at - p;
+ }
+
+ if (rspamd_str_has_8bit(p, len)) {
+ addr->flags |= RSPAMD_EMAIL_ADDR_HAS_8BIT;
+ }
+ }
+
+ return ret;
+}
+
+static inline int
+rspamd_email_address_check_and_add(const gchar *start, gsize len,
+ GPtrArray *res,
+ rspamd_mempool_t *pool,
+ GString *ns,
+ gint max_elements)
+{
+ struct rspamd_email_address addr;
+
+ g_assert(res != NULL);
+
+ if (max_elements > 0 && res->len >= max_elements) {
+ msg_info_pool_check("reached maximum number of elements %d when adding %v",
+ max_elements,
+ ns);
+
+ return -1;
+ }
+
+ /* The whole email is likely address */
+ memset(&addr, 0, sizeof(addr));
+ rspamd_smtp_addr_parse(start, len, &addr);
+
+ if (addr.flags & RSPAMD_EMAIL_ADDR_VALID) {
+ rspamd_email_address_add(pool, res, &addr, ns);
+ }
+ else {
+ /* Try heuristic */
+ if (rspamd_email_address_parse_heuristic(start,
+ len, &addr)) {
+ rspamd_email_address_add(pool, res, &addr, ns);
+
+ return 1;
+ }
+ else {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+GPtrArray *
+rspamd_email_address_from_mime(rspamd_mempool_t *pool, const gchar *hdr,
+ guint len,
+ GPtrArray *src,
+ gint max_elements)
+{
+ GPtrArray *res = src;
+ gboolean seen_at = FALSE, seen_obrace = FALSE;
+
+ const gchar *p = hdr, *end = hdr + len, *c = hdr, *t;
+ GString *ns, *cpy;
+ gint obraces, ebraces;
+ enum {
+ parse_name = 0,
+ parse_quoted,
+ parse_addr,
+ skip_spaces
+ } state = parse_name,
+ next_state = parse_name;
+
+ if (res == NULL) {
+ res = g_ptr_array_sized_new(2);
+ rspamd_mempool_add_destructor(pool, rspamd_email_address_list_destroy,
+ res);
+ }
+ else if (max_elements > 0 && res->len >= max_elements) {
+ msg_info_pool_check("reached maximum number of elements %d", max_elements);
+
+ return res;
+ }
+
+ ns = g_string_sized_new(len);
+ cpy = g_string_sized_new(len);
+
+ rspamd_mempool_add_destructor(pool, rspamd_gstring_free_hard, cpy);
+
+ /* First, we need to remove all comments as they are terrible */
+ obraces = 0;
+ ebraces = 0;
+
+ while (p < end) {
+ if (state == parse_name) {
+ if (*p == '\\') {
+ if (obraces == 0) {
+ g_string_append_c(cpy, *p);
+ }
+
+ p++;
+ }
+ else {
+ if (*p == '"') {
+ state = parse_quoted;
+ }
+ else if (*p == '(') {
+ obraces++; /* To avoid ) itself being copied */
+ }
+ else if (*p == ')') {
+ ebraces++;
+ p++;
+ }
+
+ if (obraces == ebraces) {
+ obraces = 0;
+ ebraces = 0;
+ }
+ }
+
+ if (p < end && obraces == 0) {
+ g_string_append_c(cpy, *p);
+ }
+ }
+ else {
+ /* Quoted elt */
+ if (*p == '\\') {
+ g_string_append_c(cpy, *p);
+ p++;
+ }
+ else {
+ if (*p == '"') {
+ state = parse_name;
+ }
+ }
+
+ if (p < end) {
+ g_string_append_c(cpy, *p);
+ }
+ }
+
+ p++;
+ }
+
+ state = parse_name;
+
+ p = cpy->str;
+ c = p;
+ end = p + cpy->len;
+
+ while (p < end) {
+ switch (state) {
+ case parse_name:
+ if (*p == '"') {
+ /* We need to strip last spaces and update `ns` */
+ if (p > c) {
+ guint nspaces = 0;
+
+ t = p - 1;
+
+ while (t > c && g_ascii_isspace(*t)) {
+ t--;
+ nspaces++;
+ }
+
+ g_string_append_len(ns, c, t - c + 1);
+
+ if (nspaces > 0) {
+ g_string_append_c(ns, ' ');
+ }
+ }
+
+ state = parse_quoted;
+ c = p + 1;
+ }
+ else if (*p == '<') {
+ if (p > c) {
+ t = p - 1;
+
+ while (t > c && g_ascii_isspace(*t)) {
+ t--;
+ }
+
+ g_string_append_len(ns, c, t - c + 1);
+ }
+
+ c = p;
+ state = parse_addr;
+ }
+ else if (*p == ',') {
+ if (p > c && seen_at) {
+ /*
+ * Last token must be the address:
+ * e.g. Some name name@domain.com
+ */
+ t = p - 1;
+
+ while (t > c && g_ascii_isspace(*t)) {
+ t--;
+ }
+
+ int check = rspamd_email_address_check_and_add(c, t - c + 1,
+ res, pool, ns, max_elements);
+
+ if (check == 0 && res->len == 0) {
+ /* Insert fake address */
+ rspamd_email_address_add(pool, res, NULL, ns);
+ }
+ else if (check != 1) {
+ goto end;
+ }
+
+ /* Cleanup for the next use */
+ g_string_set_size(ns, 0);
+ seen_at = FALSE;
+ }
+
+ state = skip_spaces;
+ next_state = parse_name;
+ }
+ else if (*p == '@') {
+ seen_at = TRUE;
+ }
+
+ p++;
+ break;
+ case parse_quoted:
+ if (*p == '\\') {
+ if (p > c) {
+ g_string_append_len(ns, c, p - c);
+ }
+
+ p++;
+ c = p;
+ }
+ else if (*p == '"') {
+ if (p > c) {
+ g_string_append_len(ns, c, p - c);
+ }
+
+ if (p + 1 < end && g_ascii_isspace(p[1])) {
+ g_string_append_c(ns, ' ');
+ }
+
+ state = skip_spaces;
+ next_state = parse_name;
+ }
+ else if (*p == '@' && seen_obrace) {
+ seen_at = TRUE;
+ }
+ else if (*p == '<') {
+ seen_obrace = TRUE;
+ }
+ p++;
+ break;
+ case parse_addr:
+ if (*p == '>') {
+ int check = rspamd_email_address_check_and_add(c, p - c + 1,
+ res, pool, ns, max_elements);
+ if (check == 0 && res->len == 0) {
+ /* Insert a fake address */
+ rspamd_email_address_add(pool, res, NULL, ns);
+ }
+ else if (check != 1) {
+ goto end;
+ }
+
+ /* Cleanup for the next use */
+ g_string_set_size(ns, 0);
+ seen_at = FALSE;
+ state = skip_spaces;
+ next_state = parse_name;
+ }
+ else if (*p == '@') {
+ seen_at = TRUE;
+ }
+ p++;
+ break;
+ case skip_spaces:
+ if (!g_ascii_isspace(*p)) {
+ c = p;
+ state = next_state;
+ }
+ else {
+ p++;
+ }
+ break;
+ }
+ }
+
+ /* Handle leftover */
+ switch (state) {
+ case parse_name:
+ /* Assume the whole header as name (bad thing) */
+ if (p > c) {
+ while (p > c && g_ascii_isspace(*p)) {
+ p--;
+ }
+
+ if (p > c) {
+ if (seen_at) {
+ /* The whole email is likely address */
+ int check = rspamd_email_address_check_and_add(c, p - c,
+ res, pool, ns, max_elements);
+ if (check == 0 && res->len == 0) {
+ /* Insert a fake address */
+ rspamd_email_address_add(pool, res, NULL, ns);
+ }
+ else if (check != 1) {
+ goto end;
+ }
+ }
+ else {
+ /* No @ seen */
+ g_string_append_len(ns, c, p - c);
+
+ if (res->len == 0) {
+ rspamd_email_address_add(pool, res, NULL, ns);
+ }
+ }
+ }
+ else if (res->len == 0) {
+ rspamd_email_address_add(pool, res, NULL, ns);
+ }
+ }
+ break;
+ case parse_addr:
+ if (p > c) {
+ if (rspamd_email_address_check_and_add(c, p - c,
+ res, pool, ns, max_elements) == 0) {
+ if (res->len == 0) {
+ rspamd_email_address_add(pool, res, NULL, ns);
+ }
+ }
+ }
+ break;
+ case parse_quoted:
+ /* Unfinished quoted string or a comment */
+ /* If we have seen obrace + at, then we still can try to resolve address */
+ if (seen_at && seen_obrace) {
+ p = rspamd_memrchr(cpy->str, '<', cpy->len);
+ g_assert(p != NULL);
+ if (rspamd_email_address_check_and_add(p, end - p,
+ res, pool, ns, max_elements) == 0) {
+ if (res->len == 0) {
+ rspamd_email_address_add(pool, res, NULL, ns);
+ }
+ }
+ }
+ break;
+ default:
+ /* Do nothing */
+ break;
+ }
+end:
+ rspamd_mempool_notify_alloc(pool, cpy->len);
+ g_string_free(ns, TRUE);
+
+ return res;
+}
+
+void rspamd_email_address_list_destroy(gpointer ptr)
+{
+ GPtrArray *ar = ptr;
+ guint i;
+ struct rspamd_email_address *addr;
+
+ PTR_ARRAY_FOREACH(ar, i, addr)
+ {
+ rspamd_email_address_free(addr);
+ }
+
+ g_ptr_array_free(ar, TRUE);
+} \ No newline at end of file
diff --git a/src/libmime/email_addr.h b/src/libmime/email_addr.h
new file mode 100644
index 0000000..ed00722
--- /dev/null
+++ b/src/libmime/email_addr.h
@@ -0,0 +1,97 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_EMAIL_ADDR_H_
+#define SRC_LIBMIME_EMAIL_ADDR_H_
+
+#include "config.h"
+#include "libutil/mem_pool.h"
+#include "libutil/ref.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_mime_header;
+
+enum rspamd_email_address_flags {
+ RSPAMD_EMAIL_ADDR_VALID = (1 << 0),
+ RSPAMD_EMAIL_ADDR_IP = (1 << 1),
+ RSPAMD_EMAIL_ADDR_BRACED = (1 << 2),
+ RSPAMD_EMAIL_ADDR_QUOTED = (1 << 3),
+ RSPAMD_EMAIL_ADDR_EMPTY = (1 << 4),
+ RSPAMD_EMAIL_ADDR_HAS_BACKSLASH = (1 << 5),
+ RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED = (1 << 6),
+ RSPAMD_EMAIL_ADDR_USER_ALLOCATED = (1 << 7),
+ RSPAMD_EMAIL_ADDR_HAS_8BIT = (1 << 8),
+ RSPAMD_EMAIL_ADDR_ALIASED = (1 << 9),
+ RSPAMD_EMAIL_ADDR_ORIGINAL = (1 << 10),
+};
+
+/*
+ * Structure that represents email address in a convenient way
+ */
+struct rspamd_email_address {
+ const gchar *raw;
+ const gchar *addr;
+ const gchar *user;
+ const gchar *domain;
+ const gchar *name;
+
+ guint raw_len;
+ guint addr_len;
+ guint domain_len;
+ guint user_len;
+ guint flags;
+};
+
+struct rspamd_task;
+
+/**
+ * Create email address from a single rfc822 address (e.g. from mail from:)
+ * @param str string to use
+ * @param len length of string
+ * @return
+ */
+struct rspamd_email_address *rspamd_email_address_from_smtp(const gchar *str, guint len);
+
+/**
+ * Parses email address from the mime header, decodes names and return the array
+ * of `rspamd_email_address`. If `src` is NULL, then this function creates a new
+ * array and adds a destructor to remove elements when `pool` is destroyed.
+ * Otherwise, addresses are appended to `src`.
+ * @param hdr
+ * @param len
+ * @return
+ */
+GPtrArray *
+rspamd_email_address_from_mime(rspamd_mempool_t *pool, const gchar *hdr, guint len,
+ GPtrArray *src, gint max_elements);
+
+/**
+ * Destroys list of email addresses
+ * @param ptr
+ */
+void rspamd_email_address_list_destroy(gpointer ptr);
+
+void rspamd_email_address_free(struct rspamd_email_address *addr);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_EMAIL_ADDR_H_ */
diff --git a/src/libmime/images.c b/src/libmime/images.c
new file mode 100644
index 0000000..1344d91
--- /dev/null
+++ b/src/libmime/images.c
@@ -0,0 +1,718 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "images.h"
+#include "task.h"
+#include "message.h"
+#include "libserver/html/html.h"
+
+#define msg_debug_images(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_images_log_id, "images", task->task_pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(images)
+
+#ifdef USABLE_GD
+#include "gd.h"
+#include "hash.h"
+#include <math.h>
+
+#define RSPAMD_NORMALIZED_DIM 64
+
+static rspamd_lru_hash_t *images_hash = NULL;
+#endif
+
+static const guint8 png_signature[] = {137, 80, 78, 71, 13, 10, 26, 10};
+static const guint8 jpg_sig1[] = {0xff, 0xd8};
+static const guint8 jpg_sig_jfif[] = {0xff, 0xe0};
+static const guint8 jpg_sig_exif[] = {0xff, 0xe1};
+static const guint8 gif_signature[] = {'G', 'I', 'F', '8'};
+static const guint8 bmp_signature[] = {'B', 'M'};
+
+static bool process_image(struct rspamd_task *task, struct rspamd_mime_part *part);
+
+
+bool rspamd_images_process_mime_part_maybe(struct rspamd_task *task,
+ struct rspamd_mime_part *part)
+{
+ if (part->part_type == RSPAMD_MIME_PART_UNDEFINED) {
+ if (part->detected_type &&
+ strcmp(part->detected_type, "image") == 0 &&
+ part->parsed_data.len > 0) {
+
+ return process_image(task, part);
+ }
+ }
+
+ return false;
+}
+
+void rspamd_images_process(struct rspamd_task *task)
+{
+ guint i;
+ struct rspamd_mime_part *part;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ rspamd_images_process_mime_part_maybe(task, part);
+ }
+}
+
+static enum rspamd_image_type
+detect_image_type(rspamd_ftok_t *data)
+{
+ if (data->len > sizeof(png_signature) / sizeof(png_signature[0])) {
+ if (memcmp(data->begin, png_signature, sizeof(png_signature)) == 0) {
+ return IMAGE_TYPE_PNG;
+ }
+ }
+ if (data->len > 10) {
+ if (memcmp(data->begin, jpg_sig1, sizeof(jpg_sig1)) == 0) {
+ if (memcmp(data->begin + 2, jpg_sig_jfif, sizeof(jpg_sig_jfif)) == 0 ||
+ memcmp(data->begin + 2, jpg_sig_exif, sizeof(jpg_sig_exif)) == 0) {
+ return IMAGE_TYPE_JPG;
+ }
+ }
+ }
+ if (data->len > sizeof(gif_signature) / sizeof(gif_signature[0])) {
+ if (memcmp(data->begin, gif_signature, sizeof(gif_signature)) == 0) {
+ return IMAGE_TYPE_GIF;
+ }
+ }
+ if (data->len > sizeof(bmp_signature) / sizeof(bmp_signature[0])) {
+ if (memcmp(data->begin, bmp_signature, sizeof(bmp_signature)) == 0) {
+ return IMAGE_TYPE_BMP;
+ }
+ }
+
+ return IMAGE_TYPE_UNKNOWN;
+}
+
+
+static struct rspamd_image *
+process_png_image(rspamd_mempool_t *pool, rspamd_ftok_t *data)
+{
+ struct rspamd_image *img;
+ guint32 t;
+ const guint8 *p;
+
+ if (data->len < 24) {
+ msg_info_pool("bad png detected (maybe striped)");
+ return NULL;
+ }
+
+ /* In png we should find iHDR section and get data from it */
+ /* Skip signature and read header section */
+ p = data->begin + 12;
+ if (memcmp(p, "IHDR", 4) != 0) {
+ msg_info_pool("png doesn't begins with IHDR section");
+ return NULL;
+ }
+
+ img = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_image));
+ img->type = IMAGE_TYPE_PNG;
+ img->data = data;
+
+ p += 4;
+ memcpy(&t, p, sizeof(guint32));
+ img->width = ntohl(t);
+ p += 4;
+ memcpy(&t, p, sizeof(guint32));
+ img->height = ntohl(t);
+
+ return img;
+}
+
+static struct rspamd_image *
+process_jpg_image(rspamd_mempool_t *pool, rspamd_ftok_t *data)
+{
+ const guint8 *p, *end;
+ guint16 h, w;
+ struct rspamd_image *img;
+
+ img = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_image));
+ img->type = IMAGE_TYPE_JPG;
+ img->data = data;
+
+ p = data->begin;
+ end = p + data->len - 8;
+ p += 2;
+
+ while (p < end) {
+ if (p[0] == 0xFF && p[1] != 0xFF) {
+ guint len = p[2] * 256 + p[3];
+
+ p++;
+
+ if (*p == 0xc0 || *p == 0xc1 || *p == 0xc2 || *p == 0xc3 ||
+ *p == 0xc9 || *p == 0xca || *p == 0xcb) {
+ memcpy(&h, p + 4, sizeof(guint16));
+ h = p[4] * 0xff + p[5];
+ img->height = h;
+ w = p[6] * 0xff + p[7];
+ img->width = w;
+
+ return img;
+ }
+
+
+ p += len;
+ }
+ else {
+ p++;
+ }
+ }
+
+ return NULL;
+}
+
+static struct rspamd_image *
+process_gif_image(rspamd_mempool_t *pool, rspamd_ftok_t *data)
+{
+ struct rspamd_image *img;
+ const guint8 *p;
+ guint16 t;
+
+ if (data->len < 10) {
+ msg_info_pool("bad gif detected (maybe striped)");
+ return NULL;
+ }
+
+ img = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_image));
+ img->type = IMAGE_TYPE_GIF;
+ img->data = data;
+
+ p = data->begin + 6;
+ memcpy(&t, p, sizeof(guint16));
+ img->width = GUINT16_FROM_LE(t);
+ memcpy(&t, p + 2, sizeof(guint16));
+ img->height = GUINT16_FROM_LE(t);
+
+ return img;
+}
+
+static struct rspamd_image *
+process_bmp_image(rspamd_mempool_t *pool, rspamd_ftok_t *data)
+{
+ struct rspamd_image *img;
+ gint32 t;
+ const guint8 *p;
+
+ if (data->len < 28) {
+ msg_info_pool("bad bmp detected (maybe striped)");
+ return NULL;
+ }
+
+ img = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_image));
+ img->type = IMAGE_TYPE_BMP;
+ img->data = data;
+ p = data->begin + 18;
+ memcpy(&t, p, sizeof(guint32));
+ img->width = GUINT32_FROM_LE(t);
+ memcpy(&t, p + 4, sizeof(gint32));
+ img->height = GUINT32_FROM_LE(t);
+
+ return img;
+}
+
+#ifdef USABLE_GD
+/*
+ * DCT from Emil Mikulic.
+ * http://unix4lyfe.org/dct/
+ */
+static void
+rspamd_image_dct_block(gint pixels[8][8], gdouble *out)
+{
+ gint i;
+ gint rows[8][8];
+
+ static const gint c1 = 1004 /* cos(pi/16) << 10 */,
+ s1 = 200 /* sin(pi/16) */,
+ c3 = 851 /* cos(3pi/16) << 10 */,
+ s3 = 569 /* sin(3pi/16) << 10 */,
+ r2c6 = 554 /* sqrt(2)*cos(6pi/16) << 10 */,
+ r2s6 = 1337 /* sqrt(2)*sin(6pi/16) << 10 */,
+ r2 = 181; /* sqrt(2) << 7*/
+
+ gint x0, x1, x2, x3, x4, x5, x6, x7, x8;
+
+ /* transform rows */
+ for (i = 0; i < 8; i++) {
+ x0 = pixels[0][i];
+ x1 = pixels[1][i];
+ x2 = pixels[2][i];
+ x3 = pixels[3][i];
+ x4 = pixels[4][i];
+ x5 = pixels[5][i];
+ x6 = pixels[6][i];
+ x7 = pixels[7][i];
+
+ /* Stage 1 */
+ x8 = x7 + x0;
+ x0 -= x7;
+ x7 = x1 + x6;
+ x1 -= x6;
+ x6 = x2 + x5;
+ x2 -= x5;
+ x5 = x3 + x4;
+ x3 -= x4;
+
+ /* Stage 2 */
+ x4 = x8 + x5;
+ x8 -= x5;
+ x5 = x7 + x6;
+ x7 -= x6;
+ x6 = c1 * (x1 + x2);
+ x2 = (-s1 - c1) * x2 + x6;
+ x1 = (s1 - c1) * x1 + x6;
+ x6 = c3 * (x0 + x3);
+ x3 = (-s3 - c3) * x3 + x6;
+ x0 = (s3 - c3) * x0 + x6;
+
+ /* Stage 3 */
+ x6 = x4 + x5;
+ x4 -= x5;
+ x5 = r2c6 * (x7 + x8);
+ x7 = (-r2s6 - r2c6) * x7 + x5;
+ x8 = (r2s6 - r2c6) * x8 + x5;
+ x5 = x0 + x2;
+ x0 -= x2;
+ x2 = x3 + x1;
+ x3 -= x1;
+
+ /* Stage 4 and output */
+ rows[i][0] = x6;
+ rows[i][4] = x4;
+ rows[i][2] = x8 >> 10;
+ rows[i][6] = x7 >> 10;
+ rows[i][7] = (x2 - x5) >> 10;
+ rows[i][1] = (x2 + x5) >> 10;
+ rows[i][3] = (x3 * r2) >> 17;
+ rows[i][5] = (x0 * r2) >> 17;
+ }
+
+ /* transform columns */
+ for (i = 0; i < 8; i++) {
+ x0 = rows[0][i];
+ x1 = rows[1][i];
+ x2 = rows[2][i];
+ x3 = rows[3][i];
+ x4 = rows[4][i];
+ x5 = rows[5][i];
+ x6 = rows[6][i];
+ x7 = rows[7][i];
+
+ /* Stage 1 */
+ x8 = x7 + x0;
+ x0 -= x7;
+ x7 = x1 + x6;
+ x1 -= x6;
+ x6 = x2 + x5;
+ x2 -= x5;
+ x5 = x3 + x4;
+ x3 -= x4;
+
+ /* Stage 2 */
+ x4 = x8 + x5;
+ x8 -= x5;
+ x5 = x7 + x6;
+ x7 -= x6;
+ x6 = c1 * (x1 + x2);
+ x2 = (-s1 - c1) * x2 + x6;
+ x1 = (s1 - c1) * x1 + x6;
+ x6 = c3 * (x0 + x3);
+ x3 = (-s3 - c3) * x3 + x6;
+ x0 = (s3 - c3) * x0 + x6;
+
+ /* Stage 3 */
+ x6 = x4 + x5;
+ x4 -= x5;
+ x5 = r2c6 * (x7 + x8);
+ x7 = (-r2s6 - r2c6) * x7 + x5;
+ x8 = (r2s6 - r2c6) * x8 + x5;
+ x5 = x0 + x2;
+ x0 -= x2;
+ x2 = x3 + x1;
+ x3 -= x1;
+
+ /* Stage 4 and output */
+ out[i * 8] = (double) ((x6 + 16) >> 3);
+ out[i * 8 + 1] = (double) ((x4 + 16) >> 3);
+ out[i * 8 + 2] = (double) ((x8 + 16384) >> 13);
+ out[i * 8 + 3] = (double) ((x7 + 16384) >> 13);
+ out[i * 8 + 4] = (double) ((x2 - x5 + 16384) >> 13);
+ out[i * 8 + 5] = (double) ((x2 + x5 + 16384) >> 13);
+ out[i * 8 + 6] = (double) (((x3 >> 8) * r2 + 8192) >> 12);
+ out[i * 8 + 7] = (double) (((x0 >> 8) * r2 + 8192) >> 12);
+ }
+}
+
+struct rspamd_image_cache_entry {
+ guchar digest[64];
+ guchar dct[RSPAMD_DCT_LEN / NBBY];
+};
+
+static void
+rspamd_image_cache_entry_dtor(gpointer p)
+{
+ struct rspamd_image_cache_entry *entry = p;
+ g_free(entry);
+}
+
+static guint32
+rspamd_image_dct_hash(gconstpointer p)
+{
+ return rspamd_cryptobox_fast_hash(p, rspamd_cryptobox_HASHBYTES,
+ rspamd_hash_seed());
+}
+
+static gboolean
+rspamd_image_dct_equal(gconstpointer a, gconstpointer b)
+{
+ return memcmp(a, b, rspamd_cryptobox_HASHBYTES) == 0;
+}
+
+static void
+rspamd_image_create_cache(struct rspamd_config *cfg)
+{
+ images_hash = rspamd_lru_hash_new_full(cfg->images_cache_size, NULL,
+ rspamd_image_cache_entry_dtor,
+ rspamd_image_dct_hash, rspamd_image_dct_equal);
+}
+
+static gboolean
+rspamd_image_check_hash(struct rspamd_task *task, struct rspamd_image *img)
+{
+ struct rspamd_image_cache_entry *found;
+
+ if (images_hash == NULL) {
+ rspamd_image_create_cache(task->cfg);
+ }
+
+ found = rspamd_lru_hash_lookup(images_hash, img->parent->digest,
+ task->tv.tv_sec);
+
+ if (found) {
+ /* We need to decompress */
+ img->dct = g_malloc(RSPAMD_DCT_LEN / NBBY);
+ rspamd_mempool_add_destructor(task->task_pool, g_free,
+ img->dct);
+ /* Copy as found could be destroyed by LRU */
+ memcpy(img->dct, found->dct, RSPAMD_DCT_LEN / NBBY);
+ img->is_normalized = TRUE;
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static void
+rspamd_image_save_hash(struct rspamd_task *task, struct rspamd_image *img)
+{
+ struct rspamd_image_cache_entry *found;
+
+ if (img->is_normalized) {
+ found = rspamd_lru_hash_lookup(images_hash, img->parent->digest,
+ task->tv.tv_sec);
+
+ if (!found) {
+ found = g_malloc0(sizeof(*found));
+ memcpy(found->dct, img->dct, RSPAMD_DCT_LEN / NBBY);
+ memcpy(found->digest, img->parent->digest, sizeof(found->digest));
+
+ rspamd_lru_hash_insert(images_hash, found->digest, found,
+ task->tv.tv_sec, 0);
+ }
+ }
+}
+
+#endif
+
+void rspamd_image_normalize(struct rspamd_task *task, struct rspamd_image *img)
+{
+#ifdef USABLE_GD
+ gdImagePtr src = NULL, dst = NULL;
+ guint i, j, k, l;
+ gdouble *dct;
+
+ if (img->data->len == 0 || img->data->len > G_MAXINT32) {
+ return;
+ }
+
+ if (img->height <= RSPAMD_NORMALIZED_DIM ||
+ img->width <= RSPAMD_NORMALIZED_DIM) {
+ return;
+ }
+
+ if (img->data->len > task->cfg->max_pic_size) {
+ return;
+ }
+
+ if (rspamd_image_check_hash(task, img)) {
+ return;
+ }
+
+ switch (img->type) {
+ case IMAGE_TYPE_JPG:
+ src = gdImageCreateFromJpegPtr(img->data->len, (void *) img->data->begin);
+ break;
+ case IMAGE_TYPE_PNG:
+ src = gdImageCreateFromPngPtr(img->data->len, (void *) img->data->begin);
+ break;
+ case IMAGE_TYPE_GIF:
+ src = gdImageCreateFromGifPtr(img->data->len, (void *) img->data->begin);
+ break;
+ case IMAGE_TYPE_BMP:
+ src = gdImageCreateFromBmpPtr(img->data->len, (void *) img->data->begin);
+ break;
+ default:
+ return;
+ }
+
+ if (src == NULL) {
+ msg_info_task("cannot load image of type %s from %T",
+ rspamd_image_type_str(img->type), img->filename);
+ }
+ else {
+ gdImageSetInterpolationMethod(src, GD_BILINEAR_FIXED);
+
+ dst = gdImageScale(src, RSPAMD_NORMALIZED_DIM, RSPAMD_NORMALIZED_DIM);
+ gdImageGrayScale(dst);
+ gdImageDestroy(src);
+
+ img->is_normalized = TRUE;
+ dct = g_malloc0(sizeof(gdouble) * RSPAMD_DCT_LEN);
+ img->dct = g_malloc0(RSPAMD_DCT_LEN / NBBY);
+ rspamd_mempool_add_destructor(task->task_pool, g_free,
+ img->dct);
+
+ /*
+ * Split message into blocks:
+ *
+ * ****
+ * ****
+ *
+ * Get sum of saturation values, and set bit if sum is > avg
+ * Then go further
+ *
+ * ****
+ * ****
+ *
+ * and repeat this algorithm.
+ *
+ * So on each iteration we move by 16 pixels and calculate 2 elements of
+ * signature
+ */
+ for (i = 0; i < RSPAMD_NORMALIZED_DIM; i += 8) {
+ for (j = 0; j < RSPAMD_NORMALIZED_DIM; j += 8) {
+ gint p[8][8];
+
+ for (k = 0; k < 8; k++) {
+ p[k][0] = gdImageGetPixel(dst, i + k, j);
+ p[k][1] = gdImageGetPixel(dst, i + k, j + 1);
+ p[k][2] = gdImageGetPixel(dst, i + k, j + 2);
+ p[k][3] = gdImageGetPixel(dst, i + k, j + 3);
+ p[k][4] = gdImageGetPixel(dst, i + k, j + 4);
+ p[k][5] = gdImageGetPixel(dst, i + k, j + 5);
+ p[k][6] = gdImageGetPixel(dst, i + k, j + 6);
+ p[k][7] = gdImageGetPixel(dst, i + k, j + 7);
+ }
+
+ rspamd_image_dct_block(p,
+ dct + i * RSPAMD_NORMALIZED_DIM + j);
+
+ gdouble avg = 0.0;
+
+ for (k = 0; k < 8; k++) {
+ for (l = 0; l < 8; l++) {
+ gdouble x = *(dct +
+ i * RSPAMD_NORMALIZED_DIM + j + k * 8 + l);
+ avg += (x - avg) / (gdouble) (k * 8 + l + 1);
+ }
+ }
+
+
+ for (k = 0; k < 8; k++) {
+ for (l = 0; l < 8; l++) {
+ guint idx = i * RSPAMD_NORMALIZED_DIM + j + k * 8 + l;
+
+ if (dct[idx] >= avg) {
+ setbit(img->dct, idx);
+ }
+ }
+ }
+ }
+ }
+
+ gdImageDestroy(dst);
+ g_free(dct);
+ rspamd_image_save_hash(task, img);
+ }
+#endif
+}
+
+struct rspamd_image *
+rspamd_maybe_process_image(rspamd_mempool_t *pool,
+ rspamd_ftok_t *data)
+{
+ enum rspamd_image_type type;
+ struct rspamd_image *img = NULL;
+
+ if ((type = detect_image_type(data)) != IMAGE_TYPE_UNKNOWN) {
+ switch (type) {
+ case IMAGE_TYPE_PNG:
+ img = process_png_image(pool, data);
+ break;
+ case IMAGE_TYPE_JPG:
+ img = process_jpg_image(pool, data);
+ break;
+ case IMAGE_TYPE_GIF:
+ img = process_gif_image(pool, data);
+ break;
+ case IMAGE_TYPE_BMP:
+ img = process_bmp_image(pool, data);
+ break;
+ default:
+ img = NULL;
+ break;
+ }
+ }
+
+ return img;
+}
+
+static bool
+process_image(struct rspamd_task *task, struct rspamd_mime_part *part)
+{
+ struct rspamd_image *img;
+
+ img = rspamd_maybe_process_image(task->task_pool, &part->parsed_data);
+
+ if (img != NULL) {
+ msg_debug_images("detected %s image of size %ud x %ud",
+ rspamd_image_type_str(img->type),
+ img->width, img->height);
+
+ if (part->cd) {
+ img->filename = &part->cd->filename;
+ }
+
+ img->parent = part;
+
+ part->part_type = RSPAMD_MIME_PART_IMAGE;
+ part->specific.img = img;
+
+ return true;
+ }
+
+ return false;
+}
+
+const gchar *
+rspamd_image_type_str(enum rspamd_image_type type)
+{
+ switch (type) {
+ case IMAGE_TYPE_PNG:
+ return "PNG";
+ break;
+ case IMAGE_TYPE_JPG:
+ return "JPEG";
+ break;
+ case IMAGE_TYPE_GIF:
+ return "GIF";
+ break;
+ case IMAGE_TYPE_BMP:
+ return "BMP";
+ break;
+ default:
+ break;
+ }
+
+ return "unknown";
+}
+
+static void
+rspamd_image_process_part(struct rspamd_task *task, struct rspamd_mime_part *part)
+{
+ struct rspamd_mime_header *rh;
+ struct rspamd_mime_text_part *tp;
+ struct html_image *himg;
+ const gchar *cid;
+ guint cid_len, i;
+ struct rspamd_image *img;
+
+ img = (struct rspamd_image *) part->specific.img;
+
+ if (img) {
+ /* Check Content-Id */
+ rh = rspamd_message_get_header_from_hash(part->raw_headers,
+ "Content-Id", FALSE);
+
+ if (rh) {
+ cid = rh->decoded;
+
+ if (*cid == '<') {
+ cid++;
+ }
+
+ cid_len = strlen(cid);
+
+ if (cid_len > 0) {
+ if (cid[cid_len - 1] == '>') {
+ cid_len--;
+ }
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, tp)
+ {
+ if (IS_TEXT_PART_HTML(tp) && tp->html != NULL) {
+ himg = rspamd_html_find_embedded_image(tp->html, cid, cid_len);
+
+ if (himg != NULL) {
+ img->html_image = himg;
+ himg->embedded_image = img;
+
+ msg_debug_images("found linked image by cid: <%s>",
+ cid);
+
+ if (himg->height == 0) {
+ himg->height = img->height;
+ }
+
+ if (himg->width == 0) {
+ himg->width = img->width;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+void rspamd_images_link(struct rspamd_task *task)
+{
+ struct rspamd_mime_part *part;
+ guint i;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ if (part->part_type == RSPAMD_MIME_PART_IMAGE) {
+ rspamd_image_process_part(task, part);
+ }
+ }
+} \ No newline at end of file
diff --git a/src/libmime/images.h b/src/libmime/images.h
new file mode 100644
index 0000000..bf8b3be
--- /dev/null
+++ b/src/libmime/images.h
@@ -0,0 +1,76 @@
+#ifndef IMAGES_H_
+#define IMAGES_H_
+
+#include "config.h"
+#include "fstring.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct html_image;
+struct rspamd_task;
+struct rspamd_mime_part;
+
+#define RSPAMD_DCT_LEN (64 * 64)
+
+enum rspamd_image_type {
+ IMAGE_TYPE_PNG = 0,
+ IMAGE_TYPE_JPG,
+ IMAGE_TYPE_GIF,
+ IMAGE_TYPE_BMP,
+ IMAGE_TYPE_UNKNOWN
+};
+
+struct rspamd_image {
+ struct rspamd_mime_part *parent;
+ rspamd_ftok_t *data;
+ rspamd_ftok_t *filename;
+ struct html_image *html_image;
+ enum rspamd_image_type type;
+ guint32 width;
+ guint32 height;
+ gboolean is_normalized;
+ guchar *dct;
+};
+
+/*
+ * Process images from a worker task
+ */
+void rspamd_images_process(struct rspamd_task *task);
+
+/**
+ * Process image if possible in a single mime part
+ * @param task
+ * @param part
+ * @return
+ */
+bool rspamd_images_process_mime_part_maybe(struct rspamd_task *task,
+ struct rspamd_mime_part *part);
+
+/*
+ * Link embedded images to the HTML parts
+ */
+void rspamd_images_link(struct rspamd_task *task);
+
+/**
+ * Processes image in raw data
+ * @param task
+ * @param data
+ * @return
+ */
+struct rspamd_image *rspamd_maybe_process_image(rspamd_mempool_t *pool,
+ rspamd_ftok_t *data);
+
+/*
+ * Get textual representation of an image's type
+ */
+const gchar *rspamd_image_type_str(enum rspamd_image_type type);
+
+void rspamd_image_normalize(struct rspamd_task *task, struct rspamd_image *img);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* IMAGES_H_ */
diff --git a/src/libmime/lang_detection.c b/src/libmime/lang_detection.c
new file mode 100644
index 0000000..bdd0aad
--- /dev/null
+++ b/src/libmime/lang_detection.c
@@ -0,0 +1,2103 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lang_detection.h"
+#include "lang_detection_fasttext.h"
+#include "libserver/logger.h"
+#include "libcryptobox/cryptobox.h"
+#include "libutil/multipattern.h"
+#include "ucl.h"
+#include "khash.h"
+#include "libstemmer.h"
+
+#include <glob.h>
+#include <unicode/utf8.h>
+#include <unicode/utf16.h>
+#include <unicode/ucnv.h>
+#include <unicode/uchar.h>
+#include <unicode/ustring.h>
+#include <math.h>
+
+static const gsize default_short_text_limit = 10;
+static const gsize default_words = 80;
+static const gdouble update_prob = 0.6;
+static const gchar *default_languages_path = RSPAMD_SHAREDIR "/languages";
+
+#undef EXTRA_LANGDET_DEBUG
+
+struct rspamd_language_unicode_match {
+ const gchar *lang;
+ gint unicode_code;
+};
+
+/*
+ * List of languages detected by unicode scripts
+ */
+static const struct rspamd_language_unicode_match unicode_langs[] = {
+ {"el", RSPAMD_UNICODE_GREEK},
+ {"ml", RSPAMD_UNICODE_MALAYALAM},
+ {"te", RSPAMD_UNICODE_TELUGU},
+ {"ta", RSPAMD_UNICODE_TAMIL},
+ {"gu", RSPAMD_UNICODE_GUJARATI},
+ {"th", RSPAMD_UNICODE_THAI},
+ {"ka", RSPAMD_UNICODE_GEORGIAN},
+ {"si", RSPAMD_UNICODE_SINHALA},
+ {"hy", RSPAMD_UNICODE_ARMENIAN},
+ {"ja", RSPAMD_UNICODE_JP},
+ {"ko", RSPAMD_UNICODE_HANGUL},
+};
+
+/*
+ * Top languages
+ */
+static const gchar *tier0_langs[] = {
+ "en",
+};
+static const gchar *tier1_langs[] = {
+ "fr", "it", "de", "es", "nl",
+ "pt", "ru", "pl", "tk", "th", "ar"};
+
+enum rspamd_language_category {
+ RSPAMD_LANGUAGE_LATIN = 0,
+ RSPAMD_LANGUAGE_CYRILLIC,
+ RSPAMD_LANGUAGE_DEVANAGARI,
+ RSPAMD_LANGUAGE_ARAB,
+ RSPAMD_LANGUAGE_MAX,
+};
+
+struct rspamd_language_elt {
+ const gchar *name; /* e.g. "en" or "ru" */
+ gint flags; /* enum rspamd_language_elt_flags */
+ enum rspamd_language_category category;
+ guint trigrams_words;
+ guint stop_words;
+ gdouble mean;
+ gdouble std;
+ guint occurrences; /* total number of parts with this language */
+};
+
+struct rspamd_ngramm_elt {
+ struct rspamd_language_elt *elt;
+ gdouble prob;
+};
+
+struct rspamd_ngramm_chain {
+ GPtrArray *languages;
+ gdouble mean;
+ gdouble std;
+ gchar *utf;
+};
+
+struct rspamd_stop_word_range {
+ guint start;
+ guint stop;
+ struct rspamd_language_elt *elt;
+};
+
+struct rspamd_stop_word_elt {
+ struct rspamd_multipattern *mp;
+ GArray *ranges; /* of rspamd_stop_word_range */
+};
+
+#define msg_debug_lang_det(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_langdet_log_id, "langdet", task->task_pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_debug_lang_det_cfg(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_langdet_log_id, "langdet", cfg->cfg_pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE_PUBLIC(langdet)
+
+static const struct rspamd_language_unicode_match *
+rspamd_language_search_unicode_match(const gchar *key,
+ const struct rspamd_language_unicode_match *elts, size_t nelts)
+{
+ size_t i;
+
+ for (i = 0; i < nelts; i++) {
+ if (strcmp(elts[i].lang, key) == 0) {
+ return &elts[i];
+ }
+ }
+
+ return NULL;
+}
+
+static gboolean
+rspamd_language_search_str(const gchar *key, const gchar *elts[], size_t nelts)
+{
+ size_t i;
+
+ for (i = 0; i < nelts; i++) {
+ if (strcmp(elts[i], key) == 0) {
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static guint
+rspamd_trigram_hash_func(gconstpointer key)
+{
+ return rspamd_cryptobox_fast_hash(key, 3 * sizeof(UChar32),
+ rspamd_hash_seed());
+}
+
+static gboolean
+rspamd_trigram_equal_func(gconstpointer v, gconstpointer v2)
+{
+ return memcmp(v, v2, 3 * sizeof(UChar32)) == 0;
+}
+
+KHASH_INIT(rspamd_trigram_hash, const UChar32 *, struct rspamd_ngramm_chain, true,
+ rspamd_trigram_hash_func, rspamd_trigram_equal_func);
+KHASH_INIT(rspamd_candidates_hash, const gchar *,
+ struct rspamd_lang_detector_res *, true,
+ rspamd_str_hash, rspamd_str_equal);
+KHASH_INIT(rspamd_stopwords_hash, rspamd_ftok_t *,
+ char, false,
+ rspamd_ftok_hash, rspamd_ftok_equal);
+
+KHASH_INIT(rspamd_languages_hash, const gchar *, struct rspamd_language_elt *, true,
+ rspamd_str_hash, rspamd_str_equal);
+struct rspamd_lang_detector {
+ khash_t(rspamd_languages_hash) * languages;
+ khash_t(rspamd_trigram_hash) * trigrams[RSPAMD_LANGUAGE_MAX]; /* trigrams frequencies */
+ struct rspamd_stop_word_elt stop_words[RSPAMD_LANGUAGE_MAX];
+ khash_t(rspamd_stopwords_hash) * stop_words_norm;
+ UConverter *uchar_converter;
+ gsize short_text_limit;
+ bool prefer_fasttext;
+ gsize total_occurrences; /* number of all languages found */
+ gpointer fasttext_detector;
+ ref_entry_t ref;
+};
+
+static void
+rspamd_language_detector_ucs_lowercase(UChar32 *s, gsize len)
+{
+ gsize i;
+
+ for (i = 0; i < len; i++) {
+ s[i] = u_tolower(s[i]);
+ }
+}
+
+static gboolean
+rspamd_language_detector_ucs_is_latin(const UChar32 *s, gsize len)
+{
+ gsize i;
+ gboolean ret = TRUE;
+
+ for (i = 0; i < len; i++) {
+ if (s[i] >= 128 || !(g_ascii_isalnum(s[i]) || s[i] == ' ')) {
+ ret = FALSE;
+ break;
+ }
+ }
+
+ return ret;
+}
+
+struct rspamd_language_ucs_elt {
+ guint freq;
+ const gchar *utf;
+ UChar32 s[0];
+};
+
+static void
+rspamd_language_detector_init_ngramm(struct rspamd_config *cfg,
+ struct rspamd_lang_detector *d,
+ struct rspamd_language_elt *lelt,
+ struct rspamd_language_ucs_elt *ucs,
+ guint len,
+ guint freq,
+ guint total,
+ khash_t(rspamd_trigram_hash) * htb)
+{
+ struct rspamd_ngramm_chain *chain = NULL, st_chain;
+ struct rspamd_ngramm_elt *elt;
+ khiter_t k;
+ guint i;
+ gboolean found;
+
+ switch (len) {
+ case 1:
+ case 2:
+ g_assert_not_reached();
+ break;
+ case 3:
+ k = kh_get(rspamd_trigram_hash, htb, ucs->s);
+ if (k != kh_end(htb)) {
+ chain = &kh_value(htb, k);
+ }
+ break;
+ default:
+ g_assert_not_reached();
+ break;
+ }
+
+ if (chain == NULL) {
+ /* New element */
+ chain = &st_chain;
+ memset(chain, 0, sizeof(st_chain));
+ chain->languages = g_ptr_array_sized_new(32);
+ rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard,
+ chain->languages);
+ chain->utf = rspamd_mempool_strdup(cfg->cfg_pool, ucs->utf);
+ elt = rspamd_mempool_alloc(cfg->cfg_pool, sizeof(*elt));
+ elt->elt = lelt;
+ elt->prob = ((gdouble) freq) / ((gdouble) total);
+ g_ptr_array_add(chain->languages, elt);
+
+ k = kh_put(rspamd_trigram_hash, htb, ucs->s, &i);
+ kh_value(htb, k) = *chain;
+ }
+ else {
+ /* Check sanity */
+ found = FALSE;
+
+ PTR_ARRAY_FOREACH(chain->languages, i, elt)
+ {
+ if (strcmp(elt->elt->name, lelt->name) == 0) {
+ found = TRUE;
+ elt->prob += ((gdouble) freq) / ((gdouble) total);
+ break;
+ }
+ }
+
+ if (!found) {
+ elt = rspamd_mempool_alloc(cfg->cfg_pool, sizeof(*elt));
+ elt->elt = lelt;
+ elt->prob = ((gdouble) freq) / ((gdouble) total);
+ g_ptr_array_add(chain->languages, elt);
+ }
+ }
+}
+
+static inline enum rspamd_language_category
+rspamd_language_detector_get_category(guint uflags)
+{
+ enum rspamd_language_category cat = RSPAMD_LANGUAGE_LATIN;
+
+ if (uflags & RSPAMD_UNICODE_CYRILLIC) {
+ cat = RSPAMD_LANGUAGE_CYRILLIC;
+ }
+ else if (uflags & RSPAMD_UNICODE_DEVANAGARI) {
+ cat = RSPAMD_LANGUAGE_DEVANAGARI;
+ }
+ else if (uflags & RSPAMD_UNICODE_ARABIC) {
+ cat = RSPAMD_LANGUAGE_ARAB;
+ }
+
+ return cat;
+}
+
+static const gchar *
+rspamd_language_detector_print_flags(struct rspamd_language_elt *elt)
+{
+ static gchar flags_buf[256];
+ goffset r = 0;
+
+ if (elt->flags & RS_LANGUAGE_TIER1) {
+ r += rspamd_snprintf(flags_buf + r, sizeof(flags_buf) - r, "tier1,");
+ }
+ if (elt->flags & RS_LANGUAGE_TIER0) {
+ r += rspamd_snprintf(flags_buf + r, sizeof(flags_buf) - r, "tier0,");
+ }
+ if (elt->flags & RS_LANGUAGE_LATIN) {
+ r += rspamd_snprintf(flags_buf + r, sizeof(flags_buf) - r, "latin,");
+ }
+
+ if (r > 0) {
+ flags_buf[r - 1] = '\0';
+ }
+ else {
+ flags_buf[r] = '\0';
+ }
+
+ return flags_buf;
+}
+
+static gint
+rspamd_language_detector_cmp_ngramm(gconstpointer a, gconstpointer b)
+{
+ struct rspamd_language_ucs_elt *e1 = *(struct rspamd_language_ucs_elt **) a;
+ struct rspamd_language_ucs_elt *e2 = *(struct rspamd_language_ucs_elt **) b;
+
+ return (gint) e2->freq - (gint) e1->freq;
+}
+
+static void
+rspamd_language_detector_read_file(struct rspamd_config *cfg,
+ struct rspamd_lang_detector *d,
+ const gchar *path,
+ const ucl_object_t *stop_words)
+{
+ struct ucl_parser *parser;
+ ucl_object_t *top;
+ const ucl_object_t *freqs, *n_words, *cur, *type, *flags;
+ ucl_object_iter_t it = NULL;
+ UErrorCode uc_err = U_ZERO_ERROR;
+ struct rspamd_language_elt *nelt;
+ struct rspamd_language_ucs_elt *ucs_elt;
+ khash_t(rspamd_trigram_hash) *htb = NULL;
+ gchar *pos;
+ guint total = 0, total_latin = 0, total_ngramms = 0, i, skipped,
+ loaded;
+ gdouble mean = 0, std = 0, delta = 0, delta2 = 0, m2 = 0;
+ enum rspamd_language_category cat = RSPAMD_LANGUAGE_MAX;
+
+ parser = ucl_parser_new(UCL_PARSER_NO_FILEVARS);
+ if (!ucl_parser_add_file(parser, path)) {
+ msg_warn_config("cannot parse file %s: %s", path,
+ ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+
+ return;
+ }
+
+ top = ucl_parser_get_object(parser);
+ ucl_parser_free(parser);
+
+ freqs = ucl_object_lookup(top, "freq");
+
+ if (freqs == NULL) {
+ msg_warn_config("file %s has no 'freq' key", path);
+ ucl_object_unref(top);
+
+ return;
+ }
+
+ pos = strrchr(path, '/');
+ g_assert(pos != NULL);
+ nelt = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*nelt));
+ nelt->name = rspamd_mempool_strdup(cfg->cfg_pool, pos + 1);
+ /* Remove extension */
+ pos = strchr(nelt->name, '.');
+ g_assert(pos != NULL);
+ *pos = '\0';
+
+ n_words = ucl_object_lookup(top, "n_words");
+
+ if (n_words == NULL || ucl_object_type(n_words) != UCL_ARRAY ||
+ n_words->len != 3) {
+ msg_warn_config("cannot find n_words in language %s", nelt->name);
+ ucl_object_unref(top);
+
+ return;
+ }
+ else {
+ nelt->trigrams_words = ucl_object_toint(ucl_array_find_index(n_words,
+ 2));
+ }
+
+ type = ucl_object_lookup(top, "type");
+
+ if (type == NULL || ucl_object_type(type) != UCL_STRING) {
+ msg_debug_config("cannot find type in language %s", nelt->name);
+ ucl_object_unref(top);
+
+ return;
+ }
+ else {
+ const gchar *stype = ucl_object_tostring(type);
+
+ if (strcmp(stype, "latin") == 0) {
+ cat = RSPAMD_LANGUAGE_LATIN;
+ }
+ else if (strcmp(stype, "cyrillic") == 0) {
+ cat = RSPAMD_LANGUAGE_CYRILLIC;
+ }
+ else if (strcmp(stype, "arab") == 0) {
+ cat = RSPAMD_LANGUAGE_ARAB;
+ }
+ else if (strcmp(stype, "devanagari") == 0) {
+ cat = RSPAMD_LANGUAGE_DEVANAGARI;
+ }
+ else {
+ msg_debug_config("unknown type %s of language %s", stype, nelt->name);
+ ucl_object_unref(top);
+
+ return;
+ }
+ }
+
+ flags = ucl_object_lookup(top, "flags");
+
+ if (flags != NULL && ucl_object_type(flags) == UCL_ARRAY) {
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur;
+
+ while ((cur = ucl_object_iterate(flags, &it, true)) != NULL) {
+ const gchar *fl = ucl_object_tostring(cur);
+
+ if (cur) {
+ if (strcmp(fl, "diacritics") == 0) {
+ nelt->flags |= RS_LANGUAGE_DIACRITICS;
+ }
+ else if (strcmp(fl, "ascii") == 0) {
+ nelt->flags |= RS_LANGUAGE_ASCII;
+ }
+ else {
+ msg_debug_config("unknown flag %s of language %s", fl, nelt->name);
+ }
+ }
+ else {
+ msg_debug_config("unknown flags type of language %s", nelt->name);
+ }
+ }
+ }
+
+ if (stop_words) {
+ const ucl_object_t *specific_stop_words;
+
+ specific_stop_words = ucl_object_lookup(stop_words, nelt->name);
+
+ if (specific_stop_words) {
+ struct sb_stemmer *stem = NULL;
+ it = NULL;
+ const ucl_object_t *w;
+ guint start, stop;
+
+ stem = sb_stemmer_new(nelt->name, "UTF_8");
+ start = rspamd_multipattern_get_npatterns(d->stop_words[cat].mp);
+
+ while ((w = ucl_object_iterate(specific_stop_words, &it, true)) != NULL) {
+ gsize wlen;
+ const char *word = ucl_object_tolstring(w, &wlen);
+ const char *saved;
+ guint mp_flags = RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8;
+
+ if (rspamd_multipattern_has_hyperscan()) {
+ mp_flags |= RSPAMD_MULTIPATTERN_RE;
+ }
+
+ rspamd_multipattern_add_pattern_len(d->stop_words[cat].mp,
+ word, wlen,
+ mp_flags);
+ nelt->stop_words++;
+
+ /* Also lemmatise and store normalised */
+ if (stem) {
+ const char *nw = sb_stemmer_stem(stem, word, wlen);
+
+
+ if (nw) {
+ saved = nw;
+ wlen = strlen(nw);
+ }
+ else {
+ saved = word;
+ }
+ }
+ else {
+ saved = word;
+ }
+
+ if (saved) {
+ gint rc;
+ rspamd_ftok_t *tok;
+ gchar *dst;
+
+ tok = rspamd_mempool_alloc(cfg->cfg_pool,
+ sizeof(*tok) + wlen + 1);
+ dst = ((gchar *) tok) + sizeof(*tok);
+ rspamd_strlcpy(dst, saved, wlen + 1);
+ tok->begin = dst;
+ tok->len = wlen;
+
+ kh_put(rspamd_stopwords_hash, d->stop_words_norm,
+ tok, &rc);
+ }
+ }
+
+ if (stem) {
+ sb_stemmer_delete(stem);
+ }
+
+ stop = rspamd_multipattern_get_npatterns(d->stop_words[cat].mp);
+
+ struct rspamd_stop_word_range r;
+
+ r.start = start;
+ r.stop = stop;
+ r.elt = nelt;
+
+ g_array_append_val(d->stop_words[cat].ranges, r);
+ it = NULL;
+ }
+ }
+
+ nelt->category = cat;
+ htb = d->trigrams[cat];
+
+ GPtrArray *ngramms;
+ guint nsym;
+
+ if (rspamd_language_search_str(nelt->name, tier1_langs,
+ G_N_ELEMENTS(tier1_langs))) {
+ nelt->flags |= RS_LANGUAGE_TIER1;
+ }
+
+ if (rspamd_language_search_str(nelt->name, tier0_langs,
+ G_N_ELEMENTS(tier0_langs))) {
+ nelt->flags |= RS_LANGUAGE_TIER0;
+ }
+
+ it = NULL;
+ ngramms = g_ptr_array_sized_new(freqs->len);
+ i = 0;
+ skipped = 0;
+ loaded = 0;
+
+ while ((cur = ucl_object_iterate(freqs, &it, true)) != NULL) {
+ const gchar *key;
+ gsize keylen;
+ guint freq;
+
+ key = ucl_object_keyl(cur, &keylen);
+ freq = ucl_object_toint(cur);
+
+ i++;
+ delta = freq - mean;
+ mean += delta / i;
+ delta2 = freq - mean;
+ m2 += delta * delta2;
+
+ if (key != NULL) {
+ UChar32 *cur_ucs;
+ const char *end = key + keylen, *cur_utf = key;
+
+ ucs_elt = rspamd_mempool_alloc(cfg->cfg_pool,
+ sizeof(*ucs_elt) + (keylen + 1) * sizeof(UChar32));
+
+ cur_ucs = ucs_elt->s;
+ nsym = 0;
+ uc_err = U_ZERO_ERROR;
+
+ while (cur_utf < end) {
+ *cur_ucs++ = ucnv_getNextUChar(d->uchar_converter, &cur_utf,
+ end, &uc_err);
+ if (!U_SUCCESS(uc_err)) {
+ break;
+ }
+
+ nsym++;
+ }
+
+ if (!U_SUCCESS(uc_err)) {
+ msg_warn_config("cannot convert key %*s to unicode: %s",
+ (gint) keylen, key, u_errorName(uc_err));
+
+ continue;
+ }
+
+ ucs_elt->utf = key;
+ rspamd_language_detector_ucs_lowercase(ucs_elt->s, nsym);
+
+ if (nsym == 3) {
+ g_ptr_array_add(ngramms, ucs_elt);
+ }
+ else {
+ continue;
+ }
+
+ if (rspamd_language_detector_ucs_is_latin(ucs_elt->s, nsym)) {
+ total_latin++;
+ }
+
+ ucs_elt->freq = freq;
+
+ total_ngramms++;
+ }
+ }
+
+ std = sqrt(m2 / (i - 1));
+
+ if (total_latin >= total_ngramms / 3) {
+ nelt->flags |= RS_LANGUAGE_LATIN;
+ }
+
+ nsym = 3;
+
+ total = 0;
+ PTR_ARRAY_FOREACH(ngramms, i, ucs_elt)
+ {
+
+ if (!(nelt->flags & RS_LANGUAGE_LATIN) &&
+ rspamd_language_detector_ucs_is_latin(ucs_elt->s, nsym)) {
+ ucs_elt->freq = 0;
+ /* Skip latin ngramm for non-latin language to avoid garbage */
+ skipped++;
+ continue;
+ }
+
+ /* Now, discriminate low frequency ngramms */
+
+ total += ucs_elt->freq;
+ loaded++;
+ }
+
+ g_ptr_array_sort(ngramms, rspamd_language_detector_cmp_ngramm);
+
+ PTR_ARRAY_FOREACH(ngramms, i, ucs_elt)
+ {
+ if (ucs_elt->freq > 0) {
+ rspamd_language_detector_init_ngramm(cfg, d,
+ nelt, ucs_elt, nsym,
+ ucs_elt->freq, total, htb);
+ }
+ }
+
+#ifdef EXTRA_LANGDET_DEBUG
+ /* Useful for debug */
+ for (i = 0; i < 10; i++) {
+ ucs_elt = g_ptr_array_index(ngramms, i);
+
+ msg_debug_lang_det_cfg("%s -> %s: %d", nelt->name,
+ ucs_elt->utf, ucs_elt->freq);
+ }
+#endif
+
+ g_ptr_array_free(ngramms, TRUE);
+ nelt->mean = mean;
+ nelt->std = std;
+
+ msg_debug_lang_det_cfg("loaded %s language, %d trigrams, "
+ "%d ngramms loaded; "
+ "std=%.2f, mean=%.2f, skipped=%d, loaded=%d, stop_words=%d; "
+ "(%s)",
+ nelt->name,
+ (gint) nelt->trigrams_words,
+ total,
+ std, mean,
+ skipped, loaded, nelt->stop_words,
+ rspamd_language_detector_print_flags(nelt));
+
+ int ret;
+ khiter_t k = kh_put(rspamd_languages_hash, d->languages, nelt->name, &ret);
+ g_assert(ret > 0); /* must be unique */
+ kh_value(d->languages, k) = nelt;
+ ucl_object_unref(top);
+}
+
+static gboolean
+rspamd_ucl_array_find_str(const gchar *str, const ucl_object_t *ar)
+{
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur;
+
+ if (ar == NULL || ar->len == 0) {
+ return FALSE;
+ }
+
+ while ((cur = ucl_object_iterate(ar, &it, true)) != NULL) {
+ if (ucl_object_type(cur) == UCL_STRING && rspamd_strcase_equal(
+ ucl_object_tostring(cur), str)) {
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+static void
+rspamd_language_detector_process_chain(struct rspamd_config *cfg,
+ struct rspamd_ngramm_chain *chain)
+{
+ struct rspamd_ngramm_elt *elt;
+ guint i;
+ gdouble delta, mean = 0, delta2, m2 = 0, std;
+
+ if (chain->languages->len > 3) {
+ PTR_ARRAY_FOREACH(chain->languages, i, elt)
+ {
+ delta = elt->prob - mean;
+ mean += delta / (i + 1);
+ delta2 = elt->prob - mean;
+ m2 += delta * delta2;
+ }
+
+ std = sqrt(m2 / (i - 1));
+ chain->mean = mean;
+ chain->std = std;
+
+ /* Now, filter elements that are lower than mean */
+ PTR_ARRAY_FOREACH(chain->languages, i, elt)
+ {
+ if (elt->prob < mean) {
+ g_ptr_array_remove_index_fast(chain->languages, i);
+#ifdef EXTRA_LANGDET_DEBUG
+ msg_debug_lang_det_cfg("remove %s from %s; prob: %.4f; mean: %.4f, std: %.4f",
+ elt->elt->name, chain->utf, elt->prob, mean, std);
+#endif
+ }
+ }
+ }
+ else {
+ /* We have a unique ngramm, increase its weight */
+ PTR_ARRAY_FOREACH(chain->languages, i, elt)
+ {
+ elt->prob *= 4.0;
+#ifdef EXTRA_LANGDET_DEBUG
+ msg_debug_lang_det_cfg("increase weight of %s in %s; prob: %.4f",
+ elt->elt->name, chain->utf, elt->prob);
+#endif
+ }
+ }
+}
+
+static void
+rspamd_language_detector_dtor(struct rspamd_lang_detector *d)
+{
+ if (d) {
+ for (guint i = 0; i < RSPAMD_LANGUAGE_MAX; i++) {
+ kh_destroy(rspamd_trigram_hash, d->trigrams[i]);
+ rspamd_multipattern_destroy(d->stop_words[i].mp);
+ g_array_free(d->stop_words[i].ranges, TRUE);
+ }
+
+ if (d->languages) {
+ kh_destroy(rspamd_languages_hash, d->languages);
+ }
+
+ kh_destroy(rspamd_stopwords_hash, d->stop_words_norm);
+ rspamd_lang_detection_fasttext_destroy(d->fasttext_detector);
+ }
+}
+
+struct rspamd_lang_detector *
+rspamd_language_detector_init(struct rspamd_config *cfg)
+{
+ const ucl_object_t *section, *elt, *languages_enable = NULL,
+ *languages_disable = NULL;
+ const gchar *languages_path = default_languages_path;
+ glob_t gl;
+ size_t i, short_text_limit = default_short_text_limit, total = 0;
+ UErrorCode uc_err = U_ZERO_ERROR;
+ GString *languages_pattern;
+ struct rspamd_ngramm_chain *chain, schain;
+ gchar *fname;
+ struct rspamd_lang_detector *ret = NULL;
+ struct ucl_parser *parser;
+ ucl_object_t *stop_words;
+ bool prefer_fasttext = true;
+
+ section = ucl_object_lookup(cfg->cfg_ucl_obj, "lang_detection");
+
+ if (section != NULL) {
+ elt = ucl_object_lookup(section, "languages");
+
+ if (elt) {
+ languages_path = ucl_object_tostring(elt);
+ }
+
+ elt = ucl_object_lookup(section, "short_text_limit");
+
+ if (elt) {
+ short_text_limit = ucl_object_toint(elt);
+ }
+
+ languages_enable = ucl_object_lookup(section, "languages_enable");
+ languages_disable = ucl_object_lookup(section, "languages_disable");
+
+ elt = ucl_object_lookup(section, "prefer_fasttext");
+ if (elt) {
+ prefer_fasttext = ucl_object_toboolean(elt);
+ }
+ }
+
+ languages_pattern = g_string_sized_new(PATH_MAX);
+ rspamd_printf_gstring(languages_pattern, "%s/stop_words", languages_path);
+ parser = ucl_parser_new(UCL_PARSER_DEFAULT);
+
+ if (ucl_parser_add_file(parser, languages_pattern->str)) {
+ stop_words = ucl_parser_get_object(parser);
+ }
+ else {
+ msg_err_config("cannot read stop words from %s: %s",
+ languages_pattern->str,
+ ucl_parser_get_error(parser));
+ stop_words = NULL;
+ }
+
+ ucl_parser_free(parser);
+ languages_pattern->len = 0;
+
+ rspamd_printf_gstring(languages_pattern, "%s/*.json", languages_path);
+ memset(&gl, 0, sizeof(gl));
+
+ if (glob(languages_pattern->str, 0, NULL, &gl) != 0) {
+ msg_err_config("cannot read any files matching %v", languages_pattern);
+ goto end;
+ }
+
+ ret = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*ret));
+ ret->languages = kh_init(rspamd_languages_hash);
+ kh_resize(rspamd_languages_hash, ret->languages, gl.gl_pathc);
+ ret->uchar_converter = rspamd_get_utf8_converter();
+ ret->short_text_limit = short_text_limit;
+ ret->stop_words_norm = kh_init(rspamd_stopwords_hash);
+ ret->prefer_fasttext = prefer_fasttext;
+
+ /* Map from ngramm in ucs32 to GPtrArray of rspamd_language_elt */
+ for (i = 0; i < RSPAMD_LANGUAGE_MAX; i++) {
+ ret->trigrams[i] = kh_init(rspamd_trigram_hash);
+#ifdef WITH_HYPERSCAN
+ ret->stop_words[i].mp = rspamd_multipattern_create(
+ RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8 |
+ RSPAMD_MULTIPATTERN_RE);
+#else
+ ret->stop_words[i].mp = rspamd_multipattern_create(
+ RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+#endif
+
+ ret->stop_words[i].ranges = g_array_new(FALSE, FALSE,
+ sizeof(struct rspamd_stop_word_range));
+ }
+
+ g_assert(uc_err == U_ZERO_ERROR);
+
+ for (i = 0; i < gl.gl_pathc; i++) {
+ fname = g_path_get_basename(gl.gl_pathv[i]);
+
+ if (!rspamd_ucl_array_find_str(fname, languages_disable) ||
+ (languages_enable == NULL ||
+ rspamd_ucl_array_find_str(fname, languages_enable))) {
+ rspamd_language_detector_read_file(cfg, ret, gl.gl_pathv[i],
+ stop_words);
+ }
+ else {
+ msg_info_config("skip language file %s: disabled", fname);
+ }
+
+ g_free(fname);
+ }
+
+ for (i = 0; i < RSPAMD_LANGUAGE_MAX; i++) {
+ GError *err = NULL;
+
+ kh_foreach_value(ret->trigrams[i], schain, {
+ chain = &schain;
+ rspamd_language_detector_process_chain(cfg, chain);
+ });
+
+ if (!rspamd_multipattern_compile(ret->stop_words[i].mp, &err)) {
+ msg_err_config("cannot compile stop words for %z language group: %e",
+ i, err);
+ g_error_free(err);
+ }
+
+ total += kh_size(ret->trigrams[i]);
+ }
+
+ ret->fasttext_detector = rspamd_lang_detection_fasttext_init(cfg);
+ char *fasttext_status = rspamd_lang_detection_fasttext_show_info(ret->fasttext_detector);
+
+ msg_info_config("loaded %d languages, "
+ "%d trigrams; %s",
+ (gint) kh_size(ret->languages),
+ (gint) total, fasttext_status);
+ g_free(fasttext_status);
+
+ if (stop_words) {
+ ucl_object_unref(stop_words);
+ }
+
+ REF_INIT_RETAIN(ret, rspamd_language_detector_dtor);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_language_detector_unref,
+ ret);
+
+end:
+ if (gl.gl_pathc > 0) {
+ globfree(&gl);
+ }
+
+ g_string_free(languages_pattern, TRUE);
+
+ return ret;
+}
+
+static void
+rspamd_language_detector_random_select(GArray *ucs_tokens, guint nwords,
+ goffset *offsets_out,
+ guint64 *seed)
+{
+ guint step_len, remainder, i, out_idx;
+ guint64 coin, sel;
+ rspamd_stat_token_t *tok;
+
+ g_assert(nwords != 0);
+ g_assert(offsets_out != NULL);
+ g_assert(ucs_tokens->len >= nwords);
+ /*
+ * We split input array into `nwords` parts. For each part we randomly select
+ * an element from this particular split. Here is an example:
+ *
+ * nwords=2, input_len=5
+ *
+ * w1 w2 w3 w4 w5
+ * ^ ^
+ * part1 part2
+ * vv vv
+ * w2 w5
+ *
+ * So we have 2 output words from 5 input words selected randomly within
+ * their splits. It is not uniform distribution but it seems to be better
+ * to include words from different text parts
+ */
+ step_len = ucs_tokens->len / nwords;
+ remainder = ucs_tokens->len % nwords;
+
+ out_idx = 0;
+ coin = rspamd_random_uint64_fast_seed(seed);
+ sel = coin % (step_len + remainder);
+ offsets_out[out_idx] = sel;
+
+ for (i = step_len + remainder; i < ucs_tokens->len;
+ i += step_len, out_idx++) {
+ guint ntries = 0;
+ coin = rspamd_random_uint64_fast_seed(seed);
+ sel = (coin % step_len) + i;
+
+ for (;;) {
+ tok = &g_array_index(ucs_tokens, rspamd_stat_token_t, sel);
+ /* Filter bad tokens */
+
+ if (tok->unicode.len >= 2 &&
+ !(tok->flags & RSPAMD_STAT_TOKEN_FLAG_EXCEPTION) &&
+ u_isalpha(tok->unicode.begin[0]) &&
+ u_isalpha(tok->unicode.begin[tok->unicode.len - 1])) {
+ offsets_out[out_idx] = sel;
+ break;
+ }
+ else {
+ ntries++;
+ coin = rspamd_random_uint64_fast_seed(seed);
+
+ if (ntries < step_len) {
+ sel = (coin % step_len) + i;
+ }
+ else if (ntries < ucs_tokens->len) {
+ sel = coin % ucs_tokens->len;
+ }
+ else {
+ offsets_out[out_idx] = sel;
+ break;
+ }
+ }
+ }
+ }
+
+ /*
+ * Fisher-Yates algorithm:
+ * for i from 0 to n−2 do
+ * j ↠random integer such that i ≤ j < n
+ * exchange a[i] and a[j]
+ */
+#if 0
+ if (out_idx > 2) {
+ for (i = 0; i < out_idx - 2; i++) {
+ coin = rspamd_random_uint64_fast ();
+ sel = (coin % (out_idx - i)) + i;
+ /* swap */
+ tmp = offsets_out[i];
+ offsets_out[i] = offsets_out[sel];
+ offsets_out[sel] = tmp;
+ }
+ }
+#endif
+}
+
+static goffset
+rspamd_language_detector_next_ngramm(rspamd_stat_token_t *tok, UChar32 *window,
+ guint wlen, goffset cur_off)
+{
+ guint i;
+
+ if (wlen > 1) {
+ /* Deal with spaces at the beginning and ending */
+
+ if (cur_off == 0) {
+ window[0] = (UChar32) ' ';
+
+ for (i = 0; i < wlen - 1; i++) {
+ window[i + 1] = tok->unicode.begin[i];
+ }
+ }
+ else if (cur_off + wlen == tok->unicode.len + 1) {
+ /* Add trailing space */
+ for (i = 0; i < wlen - 1; i++) {
+ window[i] = tok->unicode.begin[cur_off + i];
+ }
+ window[wlen - 1] = (UChar32) ' ';
+ }
+ else if (cur_off + wlen > tok->unicode.len + 1) {
+ /* No more fun */
+ return -1;
+ }
+ else {
+ /* Normal case */
+ for (i = 0; i < wlen; i++) {
+ window[i] = tok->unicode.begin[cur_off + i];
+ }
+ }
+ }
+ else {
+ if (tok->normalized.len <= cur_off) {
+ return -1;
+ }
+
+ window[0] = tok->unicode.begin[cur_off];
+ }
+
+ return cur_off + 1;
+}
+
+/*
+ * Do full guess for a specific ngramm, checking all languages defined
+ */
+static void
+rspamd_language_detector_process_ngramm_full(struct rspamd_task *task,
+ struct rspamd_lang_detector *d,
+ UChar32 *window,
+ khash_t(rspamd_candidates_hash) * candidates,
+ khash_t(rspamd_trigram_hash) * trigrams)
+{
+ guint i;
+ gint ret;
+ struct rspamd_ngramm_chain *chain = NULL;
+ struct rspamd_ngramm_elt *elt;
+ struct rspamd_lang_detector_res *cand;
+ khiter_t k;
+ gdouble prob;
+
+ k = kh_get(rspamd_trigram_hash, trigrams, window);
+ if (k != kh_end(trigrams)) {
+ chain = &kh_value(trigrams, k);
+ }
+
+ if (chain) {
+ PTR_ARRAY_FOREACH(chain->languages, i, elt)
+ {
+ prob = elt->prob;
+
+ if (prob < chain->mean) {
+ continue;
+ }
+
+ k = kh_get(rspamd_candidates_hash, candidates, elt->elt->name);
+ if (k != kh_end(candidates)) {
+ cand = kh_value(candidates, k);
+ }
+ else {
+ cand = NULL;
+ }
+
+#ifdef NGRAMMS_DEBUG
+ msg_err("gramm: %s, lang: %s, prob: %.3f", chain->utf,
+ elt->elt->name, log2(elt->prob));
+#endif
+ if (cand == NULL) {
+ cand = rspamd_mempool_alloc(task->task_pool, sizeof(*cand));
+ cand->elt = elt->elt;
+ cand->lang = elt->elt->name;
+ cand->prob = prob;
+
+ k = kh_put(rspamd_candidates_hash, candidates, elt->elt->name,
+ &ret);
+ kh_value(candidates, k) = cand;
+ }
+ else {
+ /* Update guess */
+ cand->prob += prob;
+ }
+ }
+ }
+}
+
+static void
+rspamd_language_detector_detect_word(struct rspamd_task *task,
+ struct rspamd_lang_detector *d,
+ rspamd_stat_token_t *tok,
+ khash_t(rspamd_candidates_hash) * candidates,
+ khash_t(rspamd_trigram_hash) * trigrams)
+{
+ const guint wlen = 3;
+ UChar32 window[3];
+ goffset cur = 0;
+
+ /* Split words */
+ while ((cur = rspamd_language_detector_next_ngramm(tok, window, wlen, cur)) != -1) {
+ rspamd_language_detector_process_ngramm_full(task,
+ d, window, candidates, trigrams);
+ }
+}
+
+static const gdouble cutoff_limit = -8.0;
+/*
+ * Converts frequencies to log probabilities, filter those candidates who
+ * has the lowest probabilities
+ */
+
+static inline void
+rspamd_language_detector_filter_step1(struct rspamd_task *task,
+ struct rspamd_lang_detector_res *cand,
+ gdouble *max_prob, guint *filtered)
+{
+ if (!isnan(cand->prob)) {
+ if (cand->prob == 0) {
+ cand->prob = NAN;
+ msg_debug_lang_det(
+ "exclude language %s",
+ cand->lang);
+ (*filtered)++;
+ }
+ else {
+ cand->prob = log2(cand->prob);
+ if (cand->prob < cutoff_limit) {
+ msg_debug_lang_det(
+ "exclude language %s: %.3f, cutoff limit: %.3f",
+ cand->lang, cand->prob, cutoff_limit);
+ cand->prob = NAN;
+ (*filtered)++;
+ }
+ else if (cand->prob > *max_prob) {
+ *max_prob = cand->prob;
+ }
+ }
+ }
+}
+
+static inline void
+rspamd_language_detector_filter_step2(struct rspamd_task *task,
+ struct rspamd_lang_detector_res *cand,
+ gdouble max_prob, guint *filtered)
+{
+ /*
+ * Probabilities are logarithmic, so if prob1 - prob2 > 4, it means that
+ * prob2 is 2^4 less than prob1
+ */
+ if (!isnan(cand->prob) && max_prob - cand->prob > 1) {
+ msg_debug_lang_det("exclude language %s: %.3f (%.3f max)",
+ cand->lang, cand->prob, max_prob);
+ cand->prob = NAN;
+ (*filtered)++;
+ }
+}
+
+static void
+rspamd_language_detector_filter_negligible(struct rspamd_task *task,
+ khash_t(rspamd_candidates_hash) * candidates)
+{
+ struct rspamd_lang_detector_res *cand;
+ guint filtered = 0;
+ gdouble max_prob = -(G_MAXDOUBLE);
+
+ kh_foreach_value(candidates, cand,
+ rspamd_language_detector_filter_step1(task, cand, &max_prob, &filtered));
+ kh_foreach_value(candidates, cand,
+ rspamd_language_detector_filter_step2(task, cand, max_prob, &filtered));
+
+ msg_debug_lang_det("removed %d languages", filtered);
+}
+
+static void
+rspamd_language_detector_detect_type(struct rspamd_task *task,
+ guint nwords,
+ struct rspamd_lang_detector *d,
+ GArray *words,
+ enum rspamd_language_category cat,
+ khash_t(rspamd_candidates_hash) * candidates,
+ struct rspamd_mime_text_part *part)
+{
+ guint nparts = MIN(words->len, nwords);
+ goffset *selected_words;
+ rspamd_stat_token_t *tok;
+ guint i;
+ guint64 seed;
+
+ /* Seed PRNG with part digest to provide some sort of determinism */
+ memcpy(&seed, part->mime_part->digest, sizeof(seed));
+ selected_words = g_new0(goffset, nparts);
+ rspamd_language_detector_random_select(words, nparts, selected_words, &seed);
+ msg_debug_lang_det("randomly selected %d words", nparts);
+
+ for (i = 0; i < nparts; i++) {
+ tok = &g_array_index(words, rspamd_stat_token_t,
+ selected_words[i]);
+
+ if (tok->unicode.len >= 3) {
+ rspamd_language_detector_detect_word(task, d, tok, candidates,
+ d->trigrams[cat]);
+ }
+ }
+
+ /* Filter negligible candidates */
+ rspamd_language_detector_filter_negligible(task, candidates);
+ g_free(selected_words);
+}
+
+static gint
+rspamd_language_detector_cmp(gconstpointer a, gconstpointer b)
+{
+ const struct rspamd_lang_detector_res
+ *canda = *(const struct rspamd_lang_detector_res **) a,
+ *candb = *(const struct rspamd_lang_detector_res **) b;
+
+ if (canda->prob > candb->prob) {
+ return -1;
+ }
+ else if (candb->prob > canda->prob) {
+ return 1;
+ }
+
+ return 0;
+}
+
+enum rspamd_language_detected_type {
+ rs_detect_none = 0,
+ rs_detect_single,
+ rs_detect_multiple,
+};
+
+static enum rspamd_language_detected_type
+rspamd_language_detector_try_ngramm(struct rspamd_task *task,
+ guint nwords,
+ struct rspamd_lang_detector *d,
+ GArray *ucs_tokens,
+ enum rspamd_language_category cat,
+ khash_t(rspamd_candidates_hash) * candidates,
+ struct rspamd_mime_text_part *part)
+{
+ guint cand_len = 0;
+ struct rspamd_lang_detector_res *cand;
+
+ rspamd_language_detector_detect_type(task,
+ nwords,
+ d,
+ ucs_tokens,
+ cat,
+ candidates,
+ part);
+
+ kh_foreach_value(candidates, cand, {
+ if (!isnan(cand->prob)) {
+ cand_len++;
+ }
+ });
+
+ if (cand_len == 0) {
+ return rs_detect_none;
+ }
+ else if (cand_len == 1) {
+ return rs_detect_single;
+ }
+
+ return rs_detect_multiple;
+}
+
+enum rspamd_language_sort_flags {
+ RSPAMD_LANG_FLAG_DEFAULT = 0,
+ RSPAMD_LANG_FLAG_SHORT = 1 << 0,
+};
+
+struct rspamd_frequency_sort_cbdata {
+ struct rspamd_lang_detector *d;
+ enum rspamd_language_sort_flags flags;
+ gdouble std;
+ gdouble mean;
+};
+
+static const gdouble tier0_adjustment = 1.2;
+static const gdouble tier1_adjustment = 0.8;
+static const gdouble frequency_adjustment = 0.8;
+
+static gint
+rspamd_language_detector_cmp_heuristic(gconstpointer a, gconstpointer b,
+ gpointer ud)
+{
+ struct rspamd_frequency_sort_cbdata *cbd = ud;
+ struct rspamd_lang_detector_res
+ *canda = *(struct rspamd_lang_detector_res **) a,
+ *candb = *(struct rspamd_lang_detector_res **) b;
+ gdouble adj;
+ gdouble proba_adjusted, probb_adjusted, freqa, freqb;
+
+ if (cbd->d->total_occurrences == 0) {
+ /* Not enough data, compare directly */
+ return rspamd_language_detector_cmp(a, b);
+ }
+
+ freqa = ((gdouble) canda->elt->occurrences) /
+ (gdouble) cbd->d->total_occurrences;
+ freqb = ((gdouble) candb->elt->occurrences) /
+ (gdouble) cbd->d->total_occurrences;
+
+ proba_adjusted = canda->prob;
+ probb_adjusted = candb->prob;
+
+ if (isnormal(freqa) && isnormal(freqb)) {
+ proba_adjusted += cbd->std * (frequency_adjustment * freqa);
+ probb_adjusted += cbd->std * (frequency_adjustment * freqb);
+ }
+
+ if (cbd->flags & RSPAMD_LANG_FLAG_SHORT) {
+ adj = tier1_adjustment * 2.0;
+ }
+ else {
+ adj = tier1_adjustment;
+ }
+ if (canda->elt->flags & RS_LANGUAGE_TIER1) {
+ proba_adjusted += cbd->std * adj;
+ }
+
+ if (candb->elt->flags & RS_LANGUAGE_TIER1) {
+ probb_adjusted += cbd->std * adj;
+ }
+
+ if (cbd->flags & RSPAMD_LANG_FLAG_SHORT) {
+ adj = tier0_adjustment * 16.0;
+ }
+ else {
+ adj = tier0_adjustment;
+ }
+
+ if (canda->elt->flags & RS_LANGUAGE_TIER0) {
+ proba_adjusted += cbd->std * adj;
+ }
+
+ if (candb->elt->flags & RS_LANGUAGE_TIER0) {
+ probb_adjusted += cbd->std * adj;
+ }
+
+ /* Hack: adjust probability directly */
+ canda->prob = proba_adjusted;
+ candb->prob = probb_adjusted;
+
+ if (proba_adjusted > probb_adjusted) {
+ return -1;
+ }
+ else if (probb_adjusted > proba_adjusted) {
+ return 1;
+ }
+
+ return 0;
+}
+
+static void
+rspamd_language_detector_unicode_scripts(struct rspamd_task *task,
+ struct rspamd_mime_text_part *part,
+ guint *pchinese,
+ guint *pspecial)
+{
+ const gchar *p = part->utf_stripped_content->data, *end;
+ guint i = 0, cnt = 0;
+ end = p + part->utf_stripped_content->len;
+ gint32 uc, sc;
+ guint nlatin = 0, nchinese = 0, nspecial = 0;
+ const guint cutoff_limit = 32;
+
+ while (p + i < end) {
+ U8_NEXT(p, i, part->utf_stripped_content->len, uc);
+
+ if (((gint32) uc) < 0) {
+ break;
+ }
+
+ if (u_isalpha(uc)) {
+ sc = ublock_getCode(uc);
+ cnt++;
+
+ switch (sc) {
+ case UBLOCK_BASIC_LATIN:
+ case UBLOCK_LATIN_1_SUPPLEMENT:
+ part->unicode_scripts |= RSPAMD_UNICODE_LATIN;
+ nlatin++;
+ break;
+ case UBLOCK_HEBREW:
+ part->unicode_scripts |= RSPAMD_UNICODE_HEBREW;
+ nspecial++;
+ break;
+ case UBLOCK_GREEK:
+ part->unicode_scripts |= RSPAMD_UNICODE_GREEK;
+ nspecial++;
+ break;
+ case UBLOCK_CYRILLIC:
+ part->unicode_scripts |= RSPAMD_UNICODE_CYRILLIC;
+ nspecial++;
+ break;
+ case UBLOCK_CJK_UNIFIED_IDEOGRAPHS:
+ case UBLOCK_CJK_COMPATIBILITY:
+ case UBLOCK_CJK_RADICALS_SUPPLEMENT:
+ case UBLOCK_CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A:
+ case UBLOCK_CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B:
+ part->unicode_scripts |= RSPAMD_UNICODE_CJK;
+ nchinese++;
+ break;
+ case UBLOCK_HIRAGANA:
+ case UBLOCK_KATAKANA:
+ part->unicode_scripts |= RSPAMD_UNICODE_JP;
+ nspecial++;
+ break;
+ case UBLOCK_HANGUL_JAMO:
+ case UBLOCK_HANGUL_COMPATIBILITY_JAMO:
+ part->unicode_scripts |= RSPAMD_UNICODE_HANGUL;
+ nspecial++;
+ break;
+ case UBLOCK_ARABIC:
+ part->unicode_scripts |= RSPAMD_UNICODE_ARABIC;
+ nspecial++;
+ break;
+ case UBLOCK_DEVANAGARI:
+ part->unicode_scripts |= RSPAMD_UNICODE_DEVANAGARI;
+ nspecial++;
+ break;
+ case UBLOCK_ARMENIAN:
+ part->unicode_scripts |= RSPAMD_UNICODE_ARMENIAN;
+ nspecial++;
+ break;
+ case UBLOCK_GEORGIAN:
+ part->unicode_scripts |= RSPAMD_UNICODE_GEORGIAN;
+ nspecial++;
+ break;
+ case UBLOCK_GUJARATI:
+ part->unicode_scripts |= RSPAMD_UNICODE_GUJARATI;
+ nspecial++;
+ break;
+ case UBLOCK_TELUGU:
+ part->unicode_scripts |= RSPAMD_UNICODE_TELUGU;
+ nspecial++;
+ break;
+ case UBLOCK_TAMIL:
+ part->unicode_scripts |= RSPAMD_UNICODE_TAMIL;
+ nspecial++;
+ break;
+ case UBLOCK_THAI:
+ part->unicode_scripts |= RSPAMD_UNICODE_THAI;
+ nspecial++;
+ break;
+ case RSPAMD_UNICODE_MALAYALAM:
+ part->unicode_scripts |= RSPAMD_UNICODE_MALAYALAM;
+ nspecial++;
+ break;
+ case RSPAMD_UNICODE_SINHALA:
+ part->unicode_scripts |= RSPAMD_UNICODE_SINHALA;
+ nspecial++;
+ break;
+ }
+ }
+
+ if (nspecial > cutoff_limit && nspecial > nlatin) {
+ break;
+ }
+ else if (nchinese > cutoff_limit && nchinese > nlatin) {
+ if (nspecial > 0) {
+ /* Likely japanese */
+ break;
+ }
+ }
+ }
+
+ msg_debug_lang_det("stop after checking %d characters, "
+ "%d latin, %d special, %d chinese",
+ cnt, nlatin, nspecial, nchinese);
+
+ *pchinese = nchinese;
+ *pspecial = nspecial;
+}
+
+static inline void
+rspamd_language_detector_set_language(struct rspamd_task *task,
+ struct rspamd_mime_text_part *part,
+ const gchar *code,
+ struct rspamd_language_elt *elt)
+{
+ struct rspamd_lang_detector_res *r;
+
+ r = rspamd_mempool_alloc0(task->task_pool, sizeof(*r));
+ r->prob = 1.0;
+ r->lang = code;
+ r->elt = elt;
+
+ if (part->languages == NULL) {
+ part->languages = g_ptr_array_sized_new(1);
+ }
+
+ g_ptr_array_add(part->languages, r);
+ part->language = code;
+}
+
+static gboolean
+rspamd_language_detector_try_uniscript(struct rspamd_task *task,
+ struct rspamd_mime_text_part *part,
+ guint nchinese,
+ guint nspecial)
+{
+ guint i;
+
+ for (i = 0; i < G_N_ELEMENTS(unicode_langs); i++) {
+ if (unicode_langs[i].unicode_code & part->unicode_scripts) {
+
+ if (unicode_langs[i].unicode_code != RSPAMD_UNICODE_JP) {
+ msg_debug_lang_det("set language based on unicode script %s",
+ unicode_langs[i].lang);
+ rspamd_language_detector_set_language(task, part,
+ unicode_langs[i].lang, NULL);
+
+ return TRUE;
+ }
+ else {
+ /* Japanese <-> Chinese guess */
+
+ /*
+ * Typically there might be around 0-70% of kanji glyphs
+ * and the rest are Haragana/Katakana
+ *
+ * If we discover that Kanji is more than 80% then we consider
+ * it Chinese
+ */
+ if (nchinese <= 5 || nchinese < nspecial * 5) {
+ msg_debug_lang_det("set language based on unicode script %s",
+ unicode_langs[i].lang);
+ rspamd_language_detector_set_language(task, part,
+ unicode_langs[i].lang, NULL);
+
+ return TRUE;
+ }
+ }
+ }
+ }
+
+ if (part->unicode_scripts & RSPAMD_UNICODE_CJK) {
+ msg_debug_lang_det("guess chinese based on CJK characters: %d chinese, %d special",
+ nchinese, nspecial);
+ rspamd_language_detector_set_language(task, part,
+ "zh-CN", NULL);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static guint
+rspamd_langelt_hash_func(gconstpointer key)
+{
+ const struct rspamd_language_elt *elt = (const struct rspamd_language_elt *) key;
+ return rspamd_cryptobox_fast_hash(elt->name, strlen(elt->name),
+ rspamd_hash_seed());
+}
+
+static gboolean
+rspamd_langelt_equal_func(gconstpointer v, gconstpointer v2)
+{
+ const struct rspamd_language_elt *elt1 = (const struct rspamd_language_elt *) v,
+ *elt2 = (const struct rspamd_language_elt *) v2;
+ return strcmp(elt1->name, elt2->name) == 0;
+}
+
+/* This hash set stores a word index in the language to avoid duplicate stop words */
+KHASH_INIT(rspamd_sw_res_set, int, char, 0, kh_int_hash_func, kh_int_hash_equal);
+
+KHASH_INIT(rspamd_sw_hash, struct rspamd_language_elt *, khash_t(rspamd_sw_res_set) *, 1,
+ rspamd_langelt_hash_func, rspamd_langelt_equal_func);
+
+struct rspamd_sw_cbdata {
+ struct rspamd_task *task;
+ khash_t(rspamd_sw_hash) * res;
+ GArray *ranges;
+};
+
+static gint
+rspamd_ranges_cmp(const void *k, const void *memb)
+{
+ gint pos = GPOINTER_TO_INT(k);
+ const struct rspamd_stop_word_range *r = (struct rspamd_stop_word_range *) memb;
+
+ if (pos >= r->start && pos < r->stop) {
+ return 0;
+ }
+ else if (pos < r->start) {
+ return -1;
+ }
+
+ return 1;
+}
+
+static gint
+rspamd_language_detector_sw_cb(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint match_pos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ /* Check if boundary */
+ const gchar *prev = text, *next = text + len;
+ struct rspamd_stop_word_range *r;
+ struct rspamd_sw_cbdata *cbdata = (struct rspamd_sw_cbdata *) context;
+ khiter_t k;
+ static const gsize max_stop_words = 80;
+ struct rspamd_task *task;
+
+ if (match_start > 0) {
+ prev = text + match_start - 1;
+
+ if (!(g_ascii_isspace(*prev) || g_ascii_ispunct(*prev))) {
+ return 0;
+ }
+ }
+
+ if (match_pos < len) {
+ next = text + match_pos;
+
+ if (!(g_ascii_isspace(*next) || g_ascii_ispunct(*next))) {
+ return 0;
+ }
+ }
+
+ /* We have a word on the boundary, check range */
+ task = cbdata->task;
+ r = bsearch(GINT_TO_POINTER(strnum), cbdata->ranges->data,
+ cbdata->ranges->len, sizeof(*r), rspamd_ranges_cmp);
+
+ g_assert(r != NULL);
+
+ k = kh_get(rspamd_sw_hash, cbdata->res, r->elt);
+ gint nwords = 1;
+
+ if (k != kh_end(cbdata->res)) {
+ khiter_t set_k;
+ int tt;
+
+ set_k = kh_get(rspamd_sw_res_set, kh_value(cbdata->res, k), strnum);
+ nwords = kh_size(kh_value(cbdata->res, k));
+
+ if (set_k == kh_end(kh_value(cbdata->res, k))) {
+ /* New word */
+ set_k = kh_put(rspamd_sw_res_set, kh_value(cbdata->res, k), strnum, &tt);
+ msg_debug_lang_det("found new word %*s from %s language (%d stop words found so far)",
+ (int) (next - prev - 1), prev + 1, r->elt->name, nwords);
+ }
+
+ if (nwords > max_stop_words) {
+ return 1;
+ }
+ }
+ else {
+ gint tt;
+
+ k = kh_put(rspamd_sw_hash, cbdata->res, r->elt, &tt);
+ kh_value(cbdata->res, k) = kh_init(rspamd_sw_res_set);
+ kh_put(rspamd_sw_res_set, kh_value(cbdata->res, k), strnum, &tt);
+
+ msg_debug_lang_det("found new word %*s from %s language (%d stop words found so far)",
+ (int) (next - prev - 1), prev + 1, r->elt->name, nwords);
+ }
+
+ return 0;
+}
+
+static gboolean
+rspamd_language_detector_try_stop_words(struct rspamd_task *task,
+ struct rspamd_lang_detector *d,
+ struct rspamd_mime_text_part *part,
+ enum rspamd_language_category cat)
+{
+ struct rspamd_stop_word_elt *elt;
+ struct rspamd_sw_cbdata cbdata;
+ gboolean ret = FALSE;
+ static const int stop_words_threshold = 4, /* minimum stop words count */
+ strong_confidence_threshold = 10 /* we are sure that this is enough */;
+
+ elt = &d->stop_words[cat];
+ cbdata.res = kh_init(rspamd_sw_hash);
+ cbdata.ranges = elt->ranges;
+ cbdata.task = task;
+
+ rspamd_multipattern_lookup(elt->mp, part->utf_stripped_content->data,
+ part->utf_stripped_content->len, rspamd_language_detector_sw_cb,
+ &cbdata, NULL);
+
+ if (kh_size(cbdata.res) > 0) {
+ khash_t(rspamd_sw_res_set) * cur_res;
+ double max_rate = G_MINDOUBLE;
+ struct rspamd_language_elt *cur_lang, *sel = NULL;
+ gboolean ignore_ascii = FALSE, ignore_latin = FALSE;
+
+ again:
+ kh_foreach(cbdata.res, cur_lang, cur_res, {
+ int cur_matches = kh_size(cur_res);
+
+ if (!ignore_ascii && (cur_lang->flags & RS_LANGUAGE_DIACRITICS)) {
+ /* Restart matches */
+ ignore_ascii = TRUE;
+ sel = NULL;
+ max_rate = G_MINDOUBLE;
+ msg_debug_lang_det("ignore ascii after finding %d stop words from %s",
+ cur_matches, cur_lang->name);
+ goto again;
+ }
+
+ if (!ignore_latin && cur_lang->category != RSPAMD_LANGUAGE_LATIN) {
+ /* Restart matches */
+ ignore_latin = TRUE;
+ sel = NULL;
+ max_rate = G_MINDOUBLE;
+ msg_debug_lang_det("ignore latin after finding stop %d words from %s",
+ cur_matches, cur_lang->name);
+ goto again;
+ }
+
+ if (cur_matches < stop_words_threshold) {
+ continue;
+ }
+
+ if (cur_matches < strong_confidence_threshold) {
+ /* Ignore mixed languages when not enough confidence */
+ if (ignore_ascii && (cur_lang->flags & RS_LANGUAGE_ASCII)) {
+ continue;
+ }
+
+ if (ignore_latin && cur_lang->category == RSPAMD_LANGUAGE_LATIN) {
+ continue;
+ }
+ }
+
+ double rate = (double) cur_matches / (double) cur_lang->stop_words;
+
+ if (rate > max_rate) {
+ max_rate = rate;
+ sel = cur_lang;
+ }
+
+ msg_debug_lang_det("found %d stop words from %s: %3f rate",
+ cur_matches, cur_lang->name, rate);
+ });
+
+ /* Cleanup */
+ kh_foreach(cbdata.res, cur_lang, cur_res, {
+ kh_destroy(rspamd_sw_res_set, cur_res);
+ });
+
+ if (max_rate > 0 && sel) {
+ msg_debug_lang_det("set language based on stop words script %s, %.3f found",
+ sel->name, max_rate);
+ rspamd_language_detector_set_language(task, part,
+ sel->name, sel);
+
+ ret = TRUE;
+ }
+ }
+ else {
+ msg_debug_lang_det("found no stop words in a text");
+ }
+
+ kh_destroy(rspamd_sw_hash, cbdata.res);
+
+ return ret;
+}
+
+gboolean
+rspamd_language_detector_detect(struct rspamd_task *task,
+ struct rspamd_lang_detector *d,
+ struct rspamd_mime_text_part *part)
+{
+ khash_t(rspamd_candidates_hash) * candidates;
+ GPtrArray *result;
+ gdouble mean, std, start_ticks, end_ticks;
+ guint cand_len;
+ enum rspamd_language_category cat;
+ struct rspamd_lang_detector_res *cand;
+ enum rspamd_language_detected_type r;
+ struct rspamd_frequency_sort_cbdata cbd;
+ /* Check if we have sorted candidates based on frequency */
+ gboolean frequency_heuristic_applied = FALSE, ret = FALSE;
+
+ if (!part->utf_stripped_content) {
+ return FALSE;
+ }
+
+ start_ticks = rspamd_get_ticks(TRUE);
+
+ guint nchinese = 0, nspecial = 0;
+ rspamd_language_detector_unicode_scripts(task, part, &nchinese, &nspecial);
+
+ /* Disable internal language detection heuristics if we have fasttext */
+ if (!rspamd_lang_detection_fasttext_is_enabled(d->fasttext_detector) || !d->prefer_fasttext) {
+ /* Apply unicode scripts heuristic */
+ if (rspamd_language_detector_try_uniscript(task, part, nchinese, nspecial)) {
+ ret = TRUE;
+ }
+
+ cat = rspamd_language_detector_get_category(part->unicode_scripts);
+
+ if (!ret && rspamd_language_detector_try_stop_words(task, d, part, cat)) {
+ ret = TRUE;
+ }
+ }
+
+ if (!ret) {
+ unsigned ndetected = 0;
+ if (rspamd_lang_detection_fasttext_is_enabled(d->fasttext_detector)) {
+ rspamd_fasttext_predict_result_t fasttext_predict_result =
+ rspamd_lang_detection_fasttext_detect(d->fasttext_detector, task,
+ part->utf_words, 4);
+
+ ndetected = rspamd_lang_detection_fasttext_get_nlangs(fasttext_predict_result);
+
+ if (ndetected > 0) {
+ candidates = kh_init(rspamd_candidates_hash);
+ kh_resize(rspamd_candidates_hash, candidates, ndetected);
+
+ /* Now fill all results where probability is above threshold */
+ float max_prob = rspamd_lang_detection_fasttext_get_prob(fasttext_predict_result, 0);
+
+ for (unsigned int i = 0; i < ndetected; i++) {
+ float prob = rspamd_lang_detection_fasttext_get_prob(fasttext_predict_result, i);
+ if (prob > max_prob * 0.75) {
+ char *lang = rspamd_mempool_strdup(task->task_pool,
+ rspamd_lang_detection_fasttext_get_lang(fasttext_predict_result, i));
+ int tmp;
+ khiter_t k = kh_put(rspamd_candidates_hash, candidates, lang, &tmp);
+
+ kh_value(candidates, k) = rspamd_mempool_alloc0(task->task_pool, sizeof(*cand));
+ cand = kh_value(candidates, k);
+ cand->lang = lang;
+ cand->prob = rspamd_lang_detection_fasttext_get_prob(fasttext_predict_result, i);
+
+ /* Find the corresponding language elt */
+ k = kh_get(rspamd_languages_hash, d->languages, lang);
+ if (k != kh_end(d->languages)) {
+ cand->elt = kh_value(d->languages, k);
+ }
+ }
+ }
+
+ if (kh_size(candidates) == 1) {
+ r = rs_detect_single;
+ }
+ else if (kh_size(candidates) > 1) {
+ r = rs_detect_multiple;
+ }
+ else {
+ r = rs_detect_none;
+ }
+ }
+
+ rspamd_fasttext_predict_result_destroy(fasttext_predict_result);
+ }
+ if (ndetected == 0) {
+ if (part->utf_words->len < default_short_text_limit) {
+ r = rs_detect_none;
+ msg_debug_lang_det("text is too short for trigrams detection: "
+ "%d words; at least %d words required",
+ (int) part->utf_words->len,
+ (int) default_short_text_limit);
+ switch (cat) {
+ case RSPAMD_LANGUAGE_CYRILLIC:
+ rspamd_language_detector_set_language(task, part, "ru", NULL);
+ break;
+ case RSPAMD_LANGUAGE_DEVANAGARI:
+ rspamd_language_detector_set_language(task, part, "hi", NULL);
+ break;
+ case RSPAMD_LANGUAGE_ARAB:
+ rspamd_language_detector_set_language(task, part, "ar", NULL);
+ break;
+ default:
+ case RSPAMD_LANGUAGE_LATIN:
+ rspamd_language_detector_set_language(task, part, "en", NULL);
+ break;
+ }
+ msg_debug_lang_det("set %s language based on symbols category",
+ part->language);
+
+ candidates = kh_init(rspamd_candidates_hash);
+ }
+ else {
+ candidates = kh_init(rspamd_candidates_hash);
+ kh_resize(rspamd_candidates_hash, candidates, 32);
+
+ r = rspamd_language_detector_try_ngramm(task,
+ default_words,
+ d,
+ part->utf_words,
+ cat,
+ candidates,
+ part);
+
+ if (r == rs_detect_none) {
+ msg_debug_lang_det("no trigrams found, fallback to english");
+ rspamd_language_detector_set_language(task, part, "en", NULL);
+ }
+ else if (r == rs_detect_multiple) {
+ /* Check our guess */
+
+ mean = 0.0;
+ std = 0.0;
+ cand_len = 0;
+
+ /* Check distribution */
+ kh_foreach_value(candidates, cand, {
+ if (!isnan(cand->prob)) {
+ mean += cand->prob;
+ cand_len++;
+ }
+ });
+
+ if (cand_len > 0) {
+ mean /= cand_len;
+
+ kh_foreach_value(candidates, cand, {
+ gdouble err;
+ if (!isnan(cand->prob)) {
+ err = cand->prob - mean;
+ std += fabs(err);
+ }
+ });
+
+ std /= cand_len;
+ }
+
+ msg_debug_lang_det("trigrams checked, %d candidates, %.3f mean, %.4f stddev",
+ cand_len, mean, std);
+
+ if (cand_len > 0 && std / fabs(mean) < 0.25) {
+ msg_debug_lang_det("apply frequency heuristic sorting");
+ frequency_heuristic_applied = TRUE;
+ cbd.d = d;
+ cbd.mean = mean;
+ cbd.std = std;
+ cbd.flags = RSPAMD_LANG_FLAG_DEFAULT;
+
+ if (part->nwords < default_words / 2) {
+ cbd.flags |= RSPAMD_LANG_FLAG_SHORT;
+ }
+ }
+ }
+ }
+ }
+
+ /* Now, convert hash to array and sort it */
+ if (r != rs_detect_none && kh_size(candidates) > 0) {
+ result = g_ptr_array_sized_new(kh_size(candidates));
+
+ kh_foreach_value(candidates, cand, {
+ if (!isnan(cand->prob)) {
+ msg_debug_lang_det("pre-sorting probability %s -> %.2f", cand->lang,
+ cand->prob);
+ g_ptr_array_add(result, cand);
+ }
+ });
+
+ if (frequency_heuristic_applied) {
+ g_ptr_array_sort_with_data(result,
+ rspamd_language_detector_cmp_heuristic,
+ (gpointer) &cbd);
+ }
+ else {
+ g_ptr_array_sort(result, rspamd_language_detector_cmp);
+ }
+
+ int i;
+ PTR_ARRAY_FOREACH(result, i, cand)
+ {
+ msg_debug_lang_det("final probability %s -> %.2f", cand->lang,
+ cand->prob);
+ }
+
+ if (part->languages != NULL) {
+ g_ptr_array_unref(part->languages);
+ }
+
+ part->languages = result;
+ part->language = ((struct rspamd_lang_detector_res *) g_ptr_array_index(result, 0))->lang;
+ ret = TRUE;
+ }
+ else if (part->languages == NULL) {
+ rspamd_language_detector_set_language(task, part, "en", NULL);
+ }
+
+ kh_destroy(rspamd_candidates_hash, candidates);
+ }
+
+ /* Update internal stat */
+ if (part->languages != NULL && part->languages->len > 0 && !frequency_heuristic_applied) {
+ cand = g_ptr_array_index(part->languages, 0);
+ if (cand->elt) {
+ cand->elt->occurrences++;
+ d->total_occurrences++;
+
+ msg_debug_lang_det("updated stat for %s: %d occurrences, %z total detected",
+ cand->elt->name, cand->elt->occurrences,
+ d->total_occurrences);
+ }
+ }
+
+ end_ticks = rspamd_get_ticks(TRUE);
+ msg_debug_lang_det("detected languages in %.0f ticks",
+ (end_ticks - start_ticks));
+
+ return ret;
+}
+
+
+struct rspamd_lang_detector *
+rspamd_language_detector_ref(struct rspamd_lang_detector *d)
+{
+ REF_RETAIN(d);
+
+ return d;
+}
+
+void rspamd_language_detector_unref(struct rspamd_lang_detector *d)
+{
+ REF_RELEASE(d);
+}
+
+gboolean
+rspamd_language_detector_is_stop_word(struct rspamd_lang_detector *d,
+ const gchar *word, gsize wlen)
+{
+ khiter_t k;
+ rspamd_ftok_t search;
+
+ search.begin = word;
+ search.len = wlen;
+
+ k = kh_get(rspamd_stopwords_hash, d->stop_words_norm, &search);
+
+ if (k != kh_end(d->stop_words_norm)) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+gint rspamd_language_detector_elt_flags(const struct rspamd_language_elt *elt)
+{
+ if (elt) {
+ return elt->flags;
+ }
+
+ return 0;
+} \ No newline at end of file
diff --git a/src/libmime/lang_detection.h b/src/libmime/lang_detection.h
new file mode 100644
index 0000000..5423c13
--- /dev/null
+++ b/src/libmime/lang_detection.h
@@ -0,0 +1,110 @@
+/*-
+ * Copyright 2017 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_LANG_DETECTION_H
+#define RSPAMD_LANG_DETECTION_H
+
+#include "config.h"
+#include "libserver/cfg_file.h"
+#include "libstat/stat_api.h"
+#include "libmime/message.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_lang_detector;
+struct rspamd_language_elt;
+struct rspamd_task;
+
+enum rspamd_unicode_scripts {
+ RSPAMD_UNICODE_LATIN = (1 << 0),
+ RSPAMD_UNICODE_GREEK = (1 << 1),
+ RSPAMD_UNICODE_CYRILLIC = (1 << 2),
+ RSPAMD_UNICODE_HEBREW = (1 << 3),
+ RSPAMD_UNICODE_CJK = (1 << 4),
+ RSPAMD_UNICODE_JP = (1 << 5),
+ RSPAMD_UNICODE_ARABIC = (1 << 6),
+ RSPAMD_UNICODE_DEVANAGARI = (1 << 7),
+ RSPAMD_UNICODE_THAI = (1 << 8),
+ RSPAMD_UNICODE_ARMENIAN = (1 << 9),
+ RSPAMD_UNICODE_GEORGIAN = (1 << 10),
+ RSPAMD_UNICODE_GUJARATI = (1 << 11),
+ RSPAMD_UNICODE_TAMIL = (1 << 12),
+ RSPAMD_UNICODE_TELUGU = (1 << 13),
+ RSPAMD_UNICODE_MALAYALAM = (1 << 14),
+ RSPAMD_UNICODE_SINHALA = (1 << 15),
+ RSPAMD_UNICODE_HANGUL = (1 << 16),
+};
+
+enum rspamd_language_elt_flags {
+ RS_LANGUAGE_DEFAULT = 0,
+ RS_LANGUAGE_LATIN = (1 << 0),
+ RS_LANGUAGE_TIER1 = (1 << 3),
+ RS_LANGUAGE_TIER0 = (1 << 4),
+ RS_LANGUAGE_DIACRITICS = (1 << 5),
+ RS_LANGUAGE_ASCII = (1 << 6),
+};
+
+struct rspamd_lang_detector_res {
+ gdouble prob;
+ const gchar *lang;
+ struct rspamd_language_elt *elt;
+};
+
+/**
+ * Create new language detector object using configuration object
+ * @param cfg
+ * @return
+ */
+struct rspamd_lang_detector *rspamd_language_detector_init(struct rspamd_config *cfg);
+
+struct rspamd_lang_detector *rspamd_language_detector_ref(struct rspamd_lang_detector *d);
+
+void rspamd_language_detector_unref(struct rspamd_lang_detector *d);
+
+/**
+ * Try to detect language of words
+ * @param d
+ * @param ucs_tokens
+ * @param words_len
+ * @return array of struct rspamd_lang_detector_res sorted by freq descending
+ */
+gboolean rspamd_language_detector_detect(struct rspamd_task *task,
+ struct rspamd_lang_detector *d,
+ struct rspamd_mime_text_part *part);
+
+/**
+ * Returns TRUE if the specified word is known to be a stop word
+ * @param d
+ * @param word
+ * @param wlen
+ * @return
+ */
+gboolean rspamd_language_detector_is_stop_word(struct rspamd_lang_detector *d,
+ const gchar *word, gsize wlen);
+
+/**
+ * Return language flags for a specific language elt
+ * @param elt
+ * @return
+ */
+gint rspamd_language_detector_elt_flags(const struct rspamd_language_elt *elt);
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libmime/lang_detection_fasttext.cxx b/src/libmime/lang_detection_fasttext.cxx
new file mode 100644
index 0000000..c973ed7
--- /dev/null
+++ b/src/libmime/lang_detection_fasttext.cxx
@@ -0,0 +1,269 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lang_detection_fasttext.h"
+
+#ifdef WITH_FASTTEXT
+#include "fasttext/fasttext.h"
+#include "libserver/cfg_file.h"
+#include "libserver/logger.h"
+#include "fmt/core.h"
+#include "stat_api.h"
+#include <exception>
+#include <string_view>
+#include <vector>
+#endif
+
+#ifdef WITH_FASTTEXT
+
+EXTERN_LOG_MODULE_DEF(langdet);
+#define msg_debug_lang_det(...) rspamd_conditional_debug_fast(nullptr, nullptr, \
+ rspamd_langdet_log_id, "langdet", task->task_pool->tag.uid, \
+ __FUNCTION__, \
+ __VA_ARGS__)
+
+namespace rspamd::langdet {
+class fasttext_langdet {
+private:
+ fasttext::FastText ft;
+ std::string model_fname;
+ bool loaded = false;
+
+public:
+ explicit fasttext_langdet(struct rspamd_config *cfg)
+ {
+ const auto *ucl_obj = cfg->cfg_ucl_obj;
+ const auto *opts_section = ucl_object_find_key(ucl_obj, "lang_detection");
+
+ if (opts_section) {
+ const auto *model = ucl_object_find_key(opts_section, "fasttext_model");
+
+ if (model) {
+ try {
+ ft.loadModel(ucl_object_tostring(model));
+ loaded = true;
+ model_fname = std::string{ucl_object_tostring(model)};
+ } catch (std::exception &e) {
+ auto err_message = fmt::format("cannot load fasttext model: {}", e.what());
+ msg_err_config("%s", err_message.c_str());
+ loaded = false;
+ }
+ }
+ }
+ }
+
+ /* Disallow multiple initialisation */
+ fasttext_langdet() = delete;
+ fasttext_langdet(const fasttext_langdet &) = delete;
+ fasttext_langdet(fasttext_langdet &&) = delete;
+
+ ~fasttext_langdet() = default;
+
+ auto is_enabled() const -> bool
+ {
+ return loaded;
+ }
+ auto word2vec(const char *in, std::size_t len, std::vector<std::int32_t> &word_ngramms) const
+ {
+ if (!loaded) {
+ return;
+ }
+
+ std::string tok{in, len};
+ const auto &dic = ft.getDictionary();
+ auto h = dic->hash(tok);
+ auto wid = dic->getId(tok, h);
+ auto type = wid < 0 ? dic->getType(tok) : dic->getType(wid);
+
+ if (type == fasttext::entry_type::word) {
+ if (wid < 0) {
+ auto pipelined_word = fmt::format("{}{}{}", fasttext::Dictionary::BOW, tok, fasttext::Dictionary::EOW);
+ dic->computeSubwords(pipelined_word, word_ngramms);
+ }
+ else {
+ if (ft.getArgs().maxn <= 0) {
+ word_ngramms.push_back(wid);
+ }
+ else {
+ const auto ngrams = dic->getSubwords(wid);
+ word_ngramms.insert(word_ngramms.end(), ngrams.cbegin(), ngrams.cend());
+ }
+ }
+ }
+ }
+ auto detect_language(std::vector<std::int32_t> &words, int k)
+ -> std::vector<std::pair<fasttext::real, std::string>> *
+ {
+ if (!loaded) {
+ return nullptr;
+ }
+
+ auto predictions = new std::vector<std::pair<fasttext::real, std::string>>;
+ predictions->reserve(k);
+ fasttext::Predictions line_predictions;
+ line_predictions.reserve(k);
+ ft.predict(k, words, line_predictions, 0.0f);
+ const auto *dict = ft.getDictionary().get();
+
+ for (const auto &pred: line_predictions) {
+ predictions->push_back(std::make_pair(std::exp(pred.first), dict->getLabel(pred.second)));
+ }
+ return predictions;
+ }
+
+ auto model_info(void) const -> const std::string
+ {
+ if (!loaded) {
+ static const auto not_loaded = std::string{"fasttext model is not loaded"};
+ return not_loaded;
+ }
+ else {
+ return fmt::format("fasttext model {}: {} languages, {} tokens", model_fname,
+ ft.getDictionary()->nlabels(), ft.getDictionary()->ntokens());
+ }
+ }
+};
+}// namespace rspamd::langdet
+#endif
+
+/* C API part */
+G_BEGIN_DECLS
+
+#define FASTTEXT_MODEL_TO_C_API(p) reinterpret_cast<rspamd::langdet::fasttext_langdet *>(p)
+#define FASTTEXT_RESULT_TO_C_API(res) reinterpret_cast<std::vector<std::pair<fasttext::real, std::string>> *>(res)
+
+void *rspamd_lang_detection_fasttext_init(struct rspamd_config *cfg)
+{
+#ifndef WITH_FASTTEXT
+ return nullptr;
+#else
+ return (void *) new rspamd::langdet::fasttext_langdet(cfg);
+#endif
+}
+
+char *rspamd_lang_detection_fasttext_show_info(void *ud)
+{
+#ifndef WITH_FASTTEXT
+ return g_strdup("fasttext is not compiled in");
+#else
+ auto model_info = FASTTEXT_MODEL_TO_C_API(ud)->model_info();
+
+ return g_strdup(model_info.c_str());
+#endif
+}
+
+bool rspamd_lang_detection_fasttext_is_enabled(void *ud)
+{
+#ifdef WITH_FASTTEXT
+ auto *real_model = FASTTEXT_MODEL_TO_C_API(ud);
+
+ if (real_model) {
+ return real_model->is_enabled();
+ }
+#endif
+
+ return false;
+}
+
+rspamd_fasttext_predict_result_t rspamd_lang_detection_fasttext_detect(void *ud,
+ struct rspamd_task *task,
+ GArray *utf_words,
+ int k)
+{
+#ifndef WITH_FASTTEXT
+ return nullptr;
+#else
+ /* Avoid too long inputs */
+ static const guint max_fasttext_input_len = 1024 * 1024;
+ auto *real_model = FASTTEXT_MODEL_TO_C_API(ud);
+ std::vector<std::int32_t> words_vec;
+ words_vec.reserve(utf_words->len);
+
+ for (auto i = 0; i < std::min(utf_words->len, max_fasttext_input_len); i++) {
+ const auto *w = &g_array_index(utf_words, rspamd_stat_token_t, i);
+ if (w->original.len > 0) {
+ real_model->word2vec(w->original.begin, w->original.len, words_vec);
+ }
+ }
+
+ msg_debug_lang_det("fasttext: got %z word tokens from %ud words", words_vec.size(), utf_words->len);
+
+ auto *res = real_model->detect_language(words_vec, k);
+
+ return (rspamd_fasttext_predict_result_t) res;
+#endif
+}
+
+void rspamd_lang_detection_fasttext_destroy(void *ud)
+{
+#ifdef WITH_FASTTEXT
+ delete FASTTEXT_MODEL_TO_C_API(ud);
+#endif
+}
+
+
+guint rspamd_lang_detection_fasttext_get_nlangs(rspamd_fasttext_predict_result_t res)
+{
+#ifdef WITH_FASTTEXT
+ auto *real_res = FASTTEXT_RESULT_TO_C_API(res);
+
+ if (real_res) {
+ return real_res->size();
+ }
+#endif
+ return 0;
+}
+
+const char *
+rspamd_lang_detection_fasttext_get_lang(rspamd_fasttext_predict_result_t res, unsigned int idx)
+{
+#ifdef WITH_FASTTEXT
+ auto *real_res = FASTTEXT_RESULT_TO_C_API(res);
+
+ if (real_res && real_res->size() > idx) {
+ /* Fasttext returns result in form __label__<lang>, so we need to remove __label__ prefix */
+ auto lang = std::string_view{real_res->at(idx).second};
+ if (lang.size() > sizeof("__label__") && lang.substr(0, sizeof("__label__") - 1) == "__label__") {
+ lang.remove_prefix(sizeof("__label__") - 1);
+ }
+ return lang.data();
+ }
+#endif
+ return nullptr;
+}
+
+float rspamd_lang_detection_fasttext_get_prob(rspamd_fasttext_predict_result_t res, unsigned int idx)
+{
+#ifdef WITH_FASTTEXT
+ auto *real_res = FASTTEXT_RESULT_TO_C_API(res);
+
+ if (real_res && real_res->size() > idx) {
+ return real_res->at(idx).first;
+ }
+#endif
+ return 0.0f;
+}
+
+void rspamd_fasttext_predict_result_destroy(rspamd_fasttext_predict_result_t res)
+{
+#ifdef WITH_FASTTEXT
+ auto *real_res = FASTTEXT_RESULT_TO_C_API(res);
+
+ delete real_res;
+#endif
+}
+
+G_END_DECLS \ No newline at end of file
diff --git a/src/libmime/lang_detection_fasttext.h b/src/libmime/lang_detection_fasttext.h
new file mode 100644
index 0000000..c8710d3
--- /dev/null
+++ b/src/libmime/lang_detection_fasttext.h
@@ -0,0 +1,91 @@
+/*-
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_LANG_DETECTION_FASTTEXT_H
+#define RSPAMD_LANG_DETECTION_FASTTEXT_H
+
+#include "config.h"
+
+G_BEGIN_DECLS
+struct rspamd_config;
+struct rspamd_task; /* for logging */
+/**
+ * Initialize fasttext language detector
+ * @param cfg
+ * @return opaque pointer
+ */
+void *rspamd_lang_detection_fasttext_init(struct rspamd_config *cfg);
+
+/**
+ * Check if fasttext language detector is enabled
+ * @param ud
+ * @return
+ */
+bool rspamd_lang_detection_fasttext_is_enabled(void *ud);
+
+/**
+ * Show info about fasttext language detector
+ * @param ud
+ * @return
+ */
+char *rspamd_lang_detection_fasttext_show_info(void *ud);
+
+
+typedef void *rspamd_fasttext_predict_result_t;
+/**
+ * Detect language using fasttext
+ * @param ud opaque pointer
+ * @param in input text
+ * @param len length of input text
+ * @param k number of results to return
+ * @return TRUE if language is detected
+ */
+rspamd_fasttext_predict_result_t rspamd_lang_detection_fasttext_detect(void *ud,
+ struct rspamd_task *task, GArray *utf_words, int k);
+
+/**
+ * Get number of languages detected
+ * @param ud
+ * @return
+ */
+guint rspamd_lang_detection_fasttext_get_nlangs(rspamd_fasttext_predict_result_t ud);
+/**
+ * Get language from fasttext result
+ * @param res
+ * @return
+ */
+const char *rspamd_lang_detection_fasttext_get_lang(rspamd_fasttext_predict_result_t res, unsigned int idx);
+
+/**
+ * Get probability from fasttext result
+ * @param res
+ * @return
+ */
+float rspamd_lang_detection_fasttext_get_prob(rspamd_fasttext_predict_result_t res, unsigned int idx);
+
+/**
+ * Destroy fasttext result
+ * @param res
+ */
+void rspamd_fasttext_predict_result_destroy(rspamd_fasttext_predict_result_t res);
+
+/**
+ * Destroy fasttext language detector
+ */
+void rspamd_lang_detection_fasttext_destroy(void *ud);
+
+
+G_END_DECLS
+#endif /* RSPAMD_LANG_DETECTION_FASTTEXT_H */
diff --git a/src/libmime/message.c b/src/libmime/message.c
new file mode 100644
index 0000000..3acc935
--- /dev/null
+++ b/src/libmime/message.c
@@ -0,0 +1,1732 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "util.h"
+#include "rspamd.h"
+#include "message.h"
+#include "libserver/html/html.h"
+#include "images.h"
+#include "archives.h"
+#include "tokenizers/tokenizers.h"
+#include "smtp_parsers.h"
+#include "mime_parser.h"
+#include "mime_encoding.h"
+#include "lang_detection.h"
+#include "libutil/multipattern.h"
+#include "libserver/mempool_vars_internal.h"
+
+#ifdef WITH_SNOWBALL
+#include "libstemmer.h"
+#endif
+
+#include <math.h>
+#include <unicode/uchar.h>
+#include "sodium.h"
+#include "libserver/cfg_file_private.h"
+#include "lua/lua_common.h"
+#include "contrib/uthash/utlist.h"
+#include "contrib/t1ha/t1ha.h"
+#include "received.h"
+
+#define GTUBE_SYMBOL "GTUBE"
+
+#define SET_PART_RAW(part) ((part)->flags &= ~RSPAMD_MIME_TEXT_PART_FLAG_UTF)
+#define SET_PART_UTF(part) ((part)->flags |= RSPAMD_MIME_TEXT_PART_FLAG_UTF)
+
+static const gchar gtube_pattern_reject[] = "XJS*C4JDBQADN1.NSBN3*2IDNEN*"
+ "GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X";
+static const gchar gtube_pattern_add_header[] = "YJS*C4JDBQADN1.NSBN3*2IDNEN*"
+ "GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X";
+static const gchar gtube_pattern_rewrite_subject[] = "ZJS*C4JDBQADN1.NSBN3*2IDNEN*"
+ "GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X";
+static const gchar gtube_pattern_no_action[] = "AJS*C4JDBQADN1.NSBN3*2IDNEN*"
+ "GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X";
+struct rspamd_multipattern *gtube_matcher = NULL;
+static const guint64 words_hash_seed = 0xdeadbabe;
+
+static void
+free_byte_array_callback(void *pointer)
+{
+ GByteArray *arr = (GByteArray *) pointer;
+ g_byte_array_free(arr, TRUE);
+}
+
+static void
+rspamd_mime_part_extract_words(struct rspamd_task *task,
+ struct rspamd_mime_text_part *part)
+{
+ rspamd_stat_token_t *w;
+ guint i, total_len = 0, short_len = 0;
+
+ if (part->utf_words) {
+ rspamd_stem_words(part->utf_words, task->task_pool, part->language,
+ task->lang_det);
+
+ for (i = 0; i < part->utf_words->len; i++) {
+ guint64 h;
+
+ w = &g_array_index(part->utf_words, rspamd_stat_token_t, i);
+
+ if (w->stemmed.len > 0) {
+ /*
+ * We use static hash seed if we would want to use that in shingles
+ * computation in future
+ */
+ h = rspamd_cryptobox_fast_hash_specific(
+ RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT,
+ w->stemmed.begin, w->stemmed.len, words_hash_seed);
+ g_array_append_val(part->normalized_hashes, h);
+ total_len += w->stemmed.len;
+
+ if (w->stemmed.len <= 3) {
+ short_len++;
+ }
+
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT &&
+ !(w->flags & RSPAMD_STAT_TOKEN_FLAG_SKIPPED)) {
+ part->nwords++;
+ }
+ }
+
+ if (w->flags & (RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE |
+ RSPAMD_STAT_TOKEN_FLAG_NORMALISED |
+ RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES)) {
+ task->flags |= RSPAMD_TASK_FLAG_BAD_UNICODE;
+ }
+ }
+
+ if (part->utf_words->len) {
+ gdouble *avg_len_p, *short_len_p;
+
+ avg_len_p = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_AVG_WORDS_LEN);
+
+ if (avg_len_p == NULL) {
+ avg_len_p = rspamd_mempool_alloc(task->task_pool,
+ sizeof(double));
+ *avg_len_p = total_len;
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_AVG_WORDS_LEN, avg_len_p, NULL);
+ }
+ else {
+ *avg_len_p += total_len;
+ }
+
+ short_len_p = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_SHORT_WORDS_CNT);
+
+ if (short_len_p == NULL) {
+ short_len_p = rspamd_mempool_alloc(task->task_pool,
+ sizeof(double));
+ *short_len_p = short_len;
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_SHORT_WORDS_CNT, avg_len_p, NULL);
+ }
+ else {
+ *short_len_p += short_len;
+ }
+ }
+ }
+}
+
+static void
+rspamd_mime_part_create_words(struct rspamd_task *task,
+ struct rspamd_mime_text_part *part)
+{
+ enum rspamd_tokenize_type tok_type;
+
+ if (IS_TEXT_PART_UTF(part)) {
+
+#if U_ICU_VERSION_MAJOR_NUM < 50
+ /* Hack to prevent hang with Thai in old libicu */
+ const gchar *p = part->utf_stripped_content->data, *end;
+ guint i = 0;
+ end = p + part->utf_stripped_content->len;
+ gint32 uc, sc;
+
+ tok_type = RSPAMD_TOKENIZE_UTF;
+
+ while (p + i < end) {
+ U8_NEXT(p, i, part->utf_stripped_content->len, uc);
+
+ if (((gint32) uc) < 0) {
+ tok_type = RSPAMD_TOKENIZE_RAW;
+ break;
+ }
+
+ if (u_isalpha(uc)) {
+ sc = ublock_getCode(uc);
+
+ if (sc == UBLOCK_THAI) {
+ msg_info_task("enable workaround for Thai characters for old libicu");
+ tok_type = RSPAMD_TOKENIZE_RAW;
+ break;
+ }
+ }
+ }
+#else
+ tok_type = RSPAMD_TOKENIZE_UTF;
+#endif
+ }
+ else {
+ tok_type = RSPAMD_TOKENIZE_RAW;
+ }
+
+ part->utf_words = rspamd_tokenize_text(
+ part->utf_stripped_content->data,
+ part->utf_stripped_content->len,
+ &part->utf_stripped_text,
+ tok_type, task->cfg,
+ part->exceptions,
+ NULL,
+ NULL,
+ task->task_pool);
+
+
+ if (part->utf_words) {
+ part->normalized_hashes = g_array_sized_new(FALSE, FALSE,
+ sizeof(guint64), part->utf_words->len);
+ rspamd_normalize_words(part->utf_words, task->task_pool);
+ }
+}
+
+static void
+rspamd_mime_part_detect_language(struct rspamd_task *task,
+ struct rspamd_mime_text_part *part)
+{
+ struct rspamd_lang_detector_res *lang;
+
+ if (!IS_TEXT_PART_EMPTY(part) && part->utf_words && part->utf_words->len > 0 &&
+ task->lang_det) {
+ if (rspamd_language_detector_detect(task, task->lang_det, part)) {
+ lang = g_ptr_array_index(part->languages, 0);
+ part->language = lang->lang;
+
+ msg_info_task("detected part language: %s", part->language);
+ }
+ else {
+ part->language = "en"; /* Safe fallback */
+ }
+ }
+}
+
+static void
+rspamd_strip_newlines_parse(struct rspamd_task *task,
+ const gchar *begin, const gchar *pe,
+ struct rspamd_mime_text_part *part)
+{
+ const gchar *p = begin, *c = begin;
+ gboolean crlf_added = FALSE, is_utf = IS_TEXT_PART_UTF(part);
+ gboolean url_open_bracket = FALSE;
+ UChar32 uc;
+
+ enum {
+ normal_char,
+ seen_cr,
+ seen_lf,
+ } state = normal_char;
+
+ while (p < pe) {
+ if (U8_IS_LEAD(*p) && is_utf) {
+ gint32 off = p - begin;
+ U8_NEXT(begin, off, pe - begin, uc);
+
+ if (uc != -1) {
+ while (p < pe && off < (pe - begin)) {
+ if (IS_ZERO_WIDTH_SPACE(uc)) {
+ /* Invisible space ! */
+ task->flags |= RSPAMD_TASK_FLAG_BAD_UNICODE;
+ part->spaces++;
+
+ if (p > c) {
+ g_byte_array_append(part->utf_stripped_content,
+ (const guint8 *) c, p - c);
+ c = begin + off;
+ p = c;
+ }
+
+ U8_NEXT(begin, off, pe - begin, uc);
+
+ if (!IS_ZERO_WIDTH_SPACE(uc)) {
+ break;
+ }
+
+ part->double_spaces++;
+ p = begin + off;
+ c = p;
+ }
+ else {
+ break;
+ }
+ }
+ }
+ }
+
+ if (G_UNLIKELY(p >= pe)) {
+ /*
+ * This is reached when there is a utf8 part and we
+ * have zero width spaces at the end of the text
+ * So we just check overflow and refuse to access *p if it is
+ * after our real content.
+ */
+ break;
+ }
+ else if (*p == '\r') {
+ switch (state) {
+ case normal_char:
+ state = seen_cr;
+ if (p > c) {
+ g_byte_array_append(part->utf_stripped_content,
+ (const guint8 *) c, p - c);
+ }
+
+ crlf_added = FALSE;
+ c = p + 1;
+ break;
+ case seen_cr:
+ /* Double \r\r */
+ if (!crlf_added) {
+ g_byte_array_append(part->utf_stripped_content,
+ (const guint8 *) " ", 1);
+ crlf_added = TRUE;
+ g_ptr_array_add(part->newlines,
+ (((gpointer) (goffset) (part->utf_stripped_content->len))));
+ }
+
+ part->nlines++;
+ part->empty_lines++;
+ c = p + 1;
+ break;
+ case seen_lf:
+ /* Likely \r\n\r...*/
+ state = seen_cr;
+ c = p + 1;
+ break;
+ }
+
+ url_open_bracket = FALSE;
+
+ p++;
+ }
+ else if (*p == '\n') {
+ switch (state) {
+ case normal_char:
+ state = seen_lf;
+
+ if (p > c) {
+ g_byte_array_append(part->utf_stripped_content,
+ (const guint8 *) c, p - c);
+ }
+
+ c = p + 1;
+
+ if (IS_TEXT_PART_HTML(part) || !url_open_bracket) {
+ g_byte_array_append(part->utf_stripped_content,
+ (const guint8 *) " ", 1);
+ g_ptr_array_add(part->newlines,
+ (((gpointer) (goffset) (part->utf_stripped_content->len))));
+ crlf_added = TRUE;
+ }
+ else {
+ crlf_added = FALSE;
+ }
+
+ break;
+ case seen_cr:
+ /* \r\n */
+ if (!crlf_added) {
+ if (IS_TEXT_PART_HTML(part) || !url_open_bracket) {
+ g_byte_array_append(part->utf_stripped_content,
+ (const guint8 *) " ", 1);
+ crlf_added = TRUE;
+ }
+
+ g_ptr_array_add(part->newlines,
+ (((gpointer) (goffset) (part->utf_stripped_content->len))));
+ }
+
+ c = p + 1;
+ state = seen_lf;
+
+ break;
+ case seen_lf:
+ /* Double \n\n */
+ if (!crlf_added) {
+ g_byte_array_append(part->utf_stripped_content,
+ (const guint8 *) " ", 1);
+ crlf_added = TRUE;
+ g_ptr_array_add(part->newlines,
+ (((gpointer) (goffset) (part->utf_stripped_content->len))));
+ }
+
+ part->nlines++;
+ part->empty_lines++;
+
+ c = p + 1;
+ break;
+ }
+ url_open_bracket = FALSE;
+
+ p++;
+ }
+ else {
+ if ((*p) == '<') {
+ url_open_bracket = TRUE;
+ }
+ else if ((*p) == '>') {
+ url_open_bracket = FALSE;
+ }
+
+ switch (state) {
+ case normal_char:
+ if (*p == ' ') {
+ part->spaces++;
+
+ if (p > begin && *(p - 1) == ' ') {
+ part->double_spaces++;
+ }
+ }
+ else {
+ part->non_spaces++;
+
+ if ((*p) & 0x80) {
+ part->non_ascii_chars++;
+ }
+ else {
+ if (g_ascii_isupper(*p)) {
+ part->capital_letters++;
+ }
+ else if (g_ascii_isdigit(*p)) {
+ part->numeric_characters++;
+ }
+
+ part->ascii_chars++;
+ }
+ }
+ break;
+ case seen_cr:
+ case seen_lf:
+ part->nlines++;
+
+ if (!crlf_added) {
+ g_ptr_array_add(part->newlines,
+ (((gpointer) (goffset) (part->utf_stripped_content->len))));
+ }
+
+ /* Skip initial spaces */
+ if (*p == ' ') {
+ if (!crlf_added) {
+ g_byte_array_append(part->utf_stripped_content,
+ (const guint8 *) " ", 1);
+ }
+
+ while (p < pe && *p == ' ') {
+ p++;
+ c++;
+ part->spaces++;
+ }
+
+ if (p < pe && (*p == '\r' || *p == '\n')) {
+ part->empty_lines++;
+ }
+ }
+
+ state = normal_char;
+ continue;
+ }
+
+ p++;
+ }
+ }
+
+ /* Leftover */
+ if (p > c) {
+ if (p > pe) {
+ p = pe;
+ }
+
+ switch (state) {
+ case normal_char:
+ g_byte_array_append(part->utf_stripped_content,
+ (const guint8 *) c, p - c);
+
+ while (c < p) {
+ if (*c == ' ') {
+ part->spaces++;
+
+ if (c > begin && *(c - 1) == ' ') {
+ part->double_spaces++;
+ }
+ }
+ else {
+ part->non_spaces++;
+
+ if ((*c) & 0x80) {
+ part->non_ascii_chars++;
+ }
+ else {
+ part->ascii_chars++;
+ }
+ }
+
+ c++;
+ }
+ break;
+ default:
+
+ if (!crlf_added) {
+ g_byte_array_append(part->utf_stripped_content,
+ (const guint8 *) " ", 1);
+ g_ptr_array_add(part->newlines,
+ (((gpointer) (goffset) (part->utf_stripped_content->len))));
+ }
+
+ part->nlines++;
+ break;
+ }
+ }
+}
+
+static void
+rspamd_u_text_dtor(void *p)
+{
+ utext_close((UText *) p);
+}
+
+static void
+rspamd_normalize_text_part(struct rspamd_task *task,
+ struct rspamd_mime_text_part *part)
+{
+ const gchar *p, *end;
+ guint i;
+ goffset off;
+ struct rspamd_process_exception *ex;
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ part->newlines = g_ptr_array_sized_new(128);
+
+ if (IS_TEXT_PART_EMPTY(part)) {
+ part->utf_stripped_content = g_byte_array_new();
+ }
+ else {
+ part->utf_stripped_content = g_byte_array_sized_new(part->utf_content.len);
+
+ p = (const gchar *) part->utf_content.begin;
+ end = p + part->utf_content.len;
+
+ rspamd_strip_newlines_parse(task, p, end, part);
+
+ for (i = 0; i < part->newlines->len; i++) {
+ ex = rspamd_mempool_alloc(task->task_pool, sizeof(*ex));
+ off = (goffset) g_ptr_array_index(part->newlines, i);
+ g_ptr_array_index(part->newlines, i) = (gpointer) (goffset) (part->utf_stripped_content->data + off);
+ ex->pos = off;
+ ex->len = 0;
+ ex->type = RSPAMD_EXCEPTION_NEWLINE;
+ part->exceptions = g_list_prepend(part->exceptions, ex);
+ }
+ }
+
+ if (IS_TEXT_PART_UTF(part)) {
+ utext_openUTF8(&part->utf_stripped_text,
+ part->utf_stripped_content->data,
+ part->utf_stripped_content->len,
+ &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ msg_warn_task("cannot open text from utf content");
+ /* Probably, should be an assertion */
+ }
+ else {
+ rspamd_mempool_add_destructor(task->task_pool,
+ rspamd_u_text_dtor,
+ &part->utf_stripped_text);
+ }
+ }
+
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) free_byte_array_callback,
+ part->utf_stripped_content);
+ rspamd_mempool_notify_alloc(task->task_pool,
+ part->utf_stripped_content->len);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) rspamd_ptr_array_free_hard,
+ part->newlines);
+}
+
+#define MIN3(a, b, c) ((a) < (b) ? ((a) < (c) ? (a) : (c)) : ((b) < (c) ? (b) : (c)))
+
+static guint
+rspamd_words_levenshtein_distance(struct rspamd_task *task,
+ GArray *w1, GArray *w2)
+{
+ guint s1len, s2len, x, y, lastdiag, olddiag;
+ guint *column, ret;
+ guint64 h1, h2;
+ gint eq;
+ static const guint max_words = 8192;
+
+ s1len = w1->len;
+ s2len = w2->len;
+
+ if (s1len + s2len > max_words) {
+ msg_info_task("cannot direct compare multipart/alternative parts with more than %ud words in total: "
+ "(%ud words in one part and %ud in another)",
+ max_words, s1len, s2len);
+
+ /* Use approximate comparison of number of words */
+ if (s1len > s2len) {
+ return s1len - s2len;
+ }
+ else {
+ return s2len - s1len;
+ }
+ }
+
+ column = g_malloc0((s1len + 1) * sizeof(guint));
+
+ for (y = 1; y <= s1len; y++) {
+ column[y] = y;
+ }
+
+ for (x = 1; x <= s2len; x++) {
+ column[0] = x;
+
+ for (y = 1, lastdiag = x - 1; y <= s1len; y++) {
+ olddiag = column[y];
+ h1 = g_array_index(w1, guint64, y - 1);
+ h2 = g_array_index(w2, guint64, x - 1);
+ eq = (h1 == h2) ? 1 : 0;
+ /*
+ * Cost of replacement is twice higher than cost of add/delete
+ * to calculate percentage properly
+ */
+ column[y] = MIN3(column[y] + 1, column[y - 1] + 1,
+ lastdiag + (eq * 2));
+ lastdiag = olddiag;
+ }
+ }
+
+ ret = column[s1len];
+ g_free(column);
+
+ return ret;
+}
+
+static gint
+rspamd_multipattern_gtube_cb(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint match_pos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ struct rspamd_task *task = (struct rspamd_task *) context;
+
+ if (strnum > 0) {
+ if (task->cfg->gtube_patterns_policy == RSPAMD_GTUBE_ALL) {
+ return strnum + 1;
+ }
+
+ return 0;
+ }
+
+ return strnum + 1; /* To distinguish from zero */
+}
+
+static enum rspamd_action_type
+rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part)
+{
+ static const gsize max_check_size = 8 * 1024;
+ gint ret;
+ enum rspamd_action_type act = METRIC_ACTION_NOACTION;
+ enum rspamd_gtube_patterns_policy policy = task->cfg ? task->cfg->gtube_patterns_policy : RSPAMD_GTUBE_REJECT;
+ g_assert(part != NULL);
+
+ if (gtube_matcher == NULL && policy != RSPAMD_GTUBE_DISABLED) {
+ gtube_matcher = rspamd_multipattern_create(RSPAMD_MULTIPATTERN_DEFAULT);
+
+ rspamd_multipattern_add_pattern(gtube_matcher,
+ gtube_pattern_reject,
+ RSPAMD_MULTIPATTERN_DEFAULT);
+ rspamd_multipattern_add_pattern(gtube_matcher,
+ gtube_pattern_add_header,
+ RSPAMD_MULTIPATTERN_DEFAULT);
+ rspamd_multipattern_add_pattern(gtube_matcher,
+ gtube_pattern_rewrite_subject,
+ RSPAMD_MULTIPATTERN_DEFAULT);
+ rspamd_multipattern_add_pattern(gtube_matcher,
+ gtube_pattern_no_action,
+ RSPAMD_MULTIPATTERN_DEFAULT);
+
+ GError *err = NULL;
+ rspamd_multipattern_compile(gtube_matcher, &err);
+
+ if (err != NULL) {
+ /* It will be expensive, but I don't care, still better than to abort */
+ msg_err("cannot compile gtube matcher: %s", err->message);
+ g_error_free(err);
+ }
+ }
+
+ if (part->utf_content.len >= sizeof(gtube_pattern_reject) &&
+ part->utf_content.len <= max_check_size &&
+ policy != RSPAMD_GTUBE_DISABLED) {
+ if ((ret = rspamd_multipattern_lookup(gtube_matcher, part->utf_content.begin,
+ part->utf_content.len,
+ rspamd_multipattern_gtube_cb, task, NULL)) > 0) {
+
+ switch (ret) {
+ case 1:
+ act = METRIC_ACTION_REJECT;
+ break;
+ case 2:
+ act = METRIC_ACTION_ADD_HEADER;
+ break;
+ case 3:
+ act = METRIC_ACTION_REWRITE_SUBJECT;
+ break;
+ case 4:
+ act = METRIC_ACTION_NOACTION;
+ break;
+ }
+
+ if (ret != 0) {
+ task->flags |= RSPAMD_TASK_FLAG_SKIP;
+ task->flags |= RSPAMD_TASK_FLAG_GTUBE;
+ msg_info_task(
+ "gtube %s pattern has been found in part of length %uz",
+ rspamd_action_to_str(act),
+ part->utf_content.len);
+ }
+ }
+ }
+
+ return act;
+}
+
+static gint
+exceptions_compare_func(gconstpointer a, gconstpointer b)
+{
+ const struct rspamd_process_exception *ea = a, *eb = b;
+
+ return ea->pos - eb->pos;
+}
+
+static gboolean
+rspamd_message_process_plain_text_part(struct rspamd_task *task,
+ struct rspamd_mime_text_part *text_part)
+{
+ if (text_part->parsed.len == 0) {
+ text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_EMPTY;
+
+ return TRUE;
+ }
+
+ rspamd_mime_text_part_maybe_convert(task, text_part);
+
+ if (text_part->utf_raw_content != NULL) {
+ /* Just have the same content */
+ text_part->utf_content.begin = (const gchar *) text_part->utf_raw_content->data;
+ text_part->utf_content.len = text_part->utf_raw_content->len;
+ }
+ else {
+ /*
+ * We ignore unconverted parts from now as it is dangerous
+ * to treat them as text parts
+ */
+ text_part->utf_content.begin = NULL;
+ text_part->utf_content.len = 0;
+
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_message_process_html_text_part(struct rspamd_task *task,
+ struct rspamd_mime_text_part *text_part,
+ uint16_t *cur_url_order)
+{
+ text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_HTML;
+
+ if (text_part->parsed.len == 0) {
+ text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_EMPTY;
+
+ return TRUE;
+ }
+
+ rspamd_mime_text_part_maybe_convert(task, text_part);
+
+ if (text_part->utf_raw_content == NULL) {
+ return FALSE;
+ }
+
+
+ text_part->html = rspamd_html_process_part_full(
+ task,
+ text_part->utf_raw_content,
+ &text_part->exceptions,
+ MESSAGE_FIELD(task, urls),
+ text_part->mime_part->urls,
+ task->cfg ? task->cfg->enable_css_parser : true,
+ cur_url_order);
+ rspamd_html_get_parsed_content(text_part->html, &text_part->utf_content);
+
+ if (text_part->utf_content.len == 0) {
+ text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_EMPTY;
+ }
+
+ return TRUE;
+}
+
+enum rspamd_message_part_is_text_result {
+ RSPAMD_MESSAGE_PART_IS_TEXT_PLAIN = 0,
+ RSPAMD_MESSAGE_PART_IS_TEXT_HTML,
+ RSPAMD_MESSAGE_PART_IS_NOT_TEXT
+};
+
+static enum rspamd_message_part_is_text_result
+rspamd_message_part_can_be_parsed_as_text(struct rspamd_task *task,
+ struct rspamd_mime_part *mime_part)
+{
+ enum rspamd_message_part_is_text_result res = RSPAMD_MESSAGE_PART_IS_NOT_TEXT;
+
+ if ((mime_part->ct && (mime_part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT)) ||
+ (mime_part->detected_type && strcmp(mime_part->detected_type, "text") == 0)) {
+
+ res = RSPAMD_MESSAGE_PART_IS_TEXT_PLAIN;
+ rspamd_ftok_t html_tok, xhtml_tok;
+
+ html_tok.begin = "html";
+ html_tok.len = 4;
+ xhtml_tok.begin = "xhtml";
+ xhtml_tok.len = 5;
+
+ if (rspamd_ftok_casecmp(&mime_part->ct->subtype, &html_tok) == 0 ||
+ rspamd_ftok_casecmp(&mime_part->ct->subtype, &xhtml_tok) == 0 ||
+ (mime_part->detected_ext &&
+ strcmp(mime_part->detected_ext, "html") == 0)) {
+ res = RSPAMD_MESSAGE_PART_IS_TEXT_HTML;
+ }
+ }
+
+ /* Skip attachments */
+ if (res != RSPAMD_MESSAGE_PART_IS_NOT_TEXT &&
+ (mime_part->cd && mime_part->cd->type == RSPAMD_CT_ATTACHMENT)) {
+ if (!task->cfg->check_text_attachements) {
+ debug_task("skip attachments for checking as text parts");
+ return RSPAMD_MESSAGE_PART_IS_NOT_TEXT;
+ }
+ }
+
+ return res;
+}
+
+static gboolean
+rspamd_message_process_text_part_maybe(struct rspamd_task *task,
+ struct rspamd_mime_part *mime_part,
+ enum rspamd_message_part_is_text_result is_text,
+ uint16_t *cur_url_order)
+{
+ struct rspamd_mime_text_part *text_part;
+ guint flags = 0;
+ enum rspamd_action_type act;
+
+ /* Skip attachments */
+ if ((mime_part->cd && mime_part->cd->type == RSPAMD_CT_ATTACHMENT)) {
+ flags |= RSPAMD_MIME_TEXT_PART_ATTACHMENT;
+ }
+
+ text_part = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct rspamd_mime_text_part));
+ text_part->mime_part = mime_part;
+ text_part->raw.begin = mime_part->raw_data.begin;
+ text_part->raw.len = mime_part->raw_data.len;
+ text_part->parsed.begin = mime_part->parsed_data.begin;
+ text_part->parsed.len = mime_part->parsed_data.len;
+ text_part->utf_stripped_text = (UText) UTEXT_INITIALIZER;
+ text_part->flags |= flags;
+
+ if (is_text == RSPAMD_MESSAGE_PART_IS_TEXT_HTML) {
+ if (!rspamd_message_process_html_text_part(task, text_part, cur_url_order)) {
+ return FALSE;
+ }
+ }
+ else {
+ if (!rspamd_message_process_plain_text_part(task, text_part)) {
+ return FALSE;
+ }
+ }
+
+ g_ptr_array_add(MESSAGE_FIELD(task, text_parts), text_part);
+ mime_part->part_type = RSPAMD_MIME_PART_TEXT;
+ mime_part->specific.txt = text_part;
+
+ act = rspamd_check_gtube(task, text_part);
+ if (act != METRIC_ACTION_NOACTION) {
+ struct rspamd_action *action;
+ gdouble score = NAN;
+
+ action = rspamd_config_get_action_by_type(task->cfg, act);
+
+ if (action) {
+ score = action->threshold;
+
+ rspamd_add_passthrough_result(task, action,
+ RSPAMD_PASSTHROUGH_CRITICAL,
+ score, "Gtube pattern",
+ "GTUBE", 0, NULL);
+ }
+
+ rspamd_task_insert_result(task, GTUBE_SYMBOL, 0, NULL);
+
+ return TRUE;
+ }
+
+ /* Post process part */
+ rspamd_normalize_text_part(task, text_part);
+
+ if (!IS_TEXT_PART_HTML(text_part)) {
+ if (mime_part->parent_part) {
+ struct rspamd_mime_part *parent = mime_part->parent_part;
+
+ if (IS_PART_MULTIPART(parent) && parent->specific.mp->children->len == 2) {
+ /*
+ * Use strict extraction mode: we will extract missing urls from
+ * an html part if needed
+ */
+ rspamd_url_text_extract(task->task_pool, task, text_part, cur_url_order,
+ RSPAMD_URL_FIND_STRICT);
+ }
+ else {
+ /*
+ * Fall back to full text extraction using TLD patterns
+ */
+ rspamd_url_text_extract(task->task_pool, task, text_part, cur_url_order,
+ RSPAMD_URL_FIND_ALL);
+ }
+ }
+ else {
+ /*
+ * Fall back to full text extraction using TLD patterns
+ */
+ rspamd_url_text_extract(task->task_pool, task, text_part, cur_url_order,
+ RSPAMD_URL_FIND_ALL);
+ }
+ }
+ else {
+ rspamd_url_text_extract(task->task_pool, task, text_part, cur_url_order,
+ RSPAMD_URL_FIND_STRICT);
+ }
+
+ if (text_part->exceptions) {
+ text_part->exceptions = g_list_sort(text_part->exceptions,
+ exceptions_compare_func);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) g_list_free,
+ text_part->exceptions);
+ }
+
+ rspamd_mime_part_create_words(task, text_part);
+
+ return TRUE;
+}
+
+/* Creates message from various data using libmagic to detect type */
+static void
+rspamd_message_from_data(struct rspamd_task *task, const guchar *start,
+ gsize len)
+{
+ struct rspamd_content_type *ct = NULL;
+ struct rspamd_mime_part *part;
+ const char *mb = "application/octet-stream";
+ gchar *mid;
+ rspamd_ftok_t srch, *tok;
+ gchar cdbuf[1024];
+
+ g_assert(start != NULL);
+
+ part = rspamd_mempool_alloc0(task->task_pool, sizeof(*part));
+
+ part->raw_data.begin = start;
+ part->raw_data.len = len;
+ part->parsed_data.begin = start;
+ part->parsed_data.len = len;
+ part->part_number = MESSAGE_FIELD(task, parts)->len;
+ part->urls = g_ptr_array_new();
+ part->raw_headers = rspamd_message_headers_new();
+ part->headers_order = NULL;
+
+ tok = rspamd_task_get_request_header(task, "Content-Type");
+
+ if (tok) {
+ /* We have Content-Type defined */
+ ct = rspamd_content_type_parse(tok->begin, tok->len,
+ task->task_pool);
+ part->ct = ct;
+ }
+ else if (task->cfg && task->cfg->libs_ctx) {
+ lua_State *L = task->cfg->lua_state;
+
+ if (rspamd_lua_require_function(L,
+ "lua_magic", "detect_mime_part")) {
+
+ struct rspamd_mime_part **pmime;
+ struct rspamd_task **ptask;
+
+ pmime = lua_newuserdata(L, sizeof(struct rspamd_mime_part *));
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+ *pmime = part;
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+
+ if (lua_pcall(L, 2, 2, 0) != 0) {
+ msg_err_task("cannot detect type: %s", lua_tostring(L, -1));
+ }
+ else {
+ if (lua_istable(L, -1)) {
+ lua_pushstring(L, "ct");
+ lua_gettable(L, -2);
+
+ if (lua_isstring(L, -1)) {
+ mb = rspamd_mempool_strdup(task->task_pool,
+ lua_tostring(L, -1));
+ }
+ }
+ }
+
+ lua_settop(L, 0);
+ }
+ else {
+ msg_err_task("cannot require lua_magic.detect_mime_part");
+ }
+
+ if (mb) {
+ srch.begin = mb;
+ srch.len = strlen(mb);
+ ct = rspamd_content_type_parse(srch.begin, srch.len,
+ task->task_pool);
+
+ if (!part->ct) {
+ msg_info_task("construct fake mime of type: %s", mb);
+ part->ct = ct;
+ }
+ else {
+ /* Check sanity */
+ if (part->ct && (part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT)) {
+ RSPAMD_FTOK_FROM_STR(&srch, "application");
+
+ if (rspamd_ftok_cmp(&ct->type, &srch) == 0) {
+ msg_info_task("construct fake mime of type: %s", mb);
+ part->ct = ct;
+ }
+ }
+ else {
+ msg_info_task("construct fake mime of type: %T/%T, detected %s",
+ &part->ct->type, &part->ct->subtype, mb);
+ }
+ }
+
+ part->detected_ct = ct;
+ }
+ }
+
+
+ tok = rspamd_task_get_request_header(task, "Filename");
+
+ if (tok) {
+ rspamd_snprintf(cdbuf, sizeof(cdbuf), "inline; filename=\"%T\"", tok);
+ }
+ else {
+ rspamd_snprintf(cdbuf, sizeof(cdbuf), "inline");
+ }
+
+ part->cd = rspamd_content_disposition_parse(cdbuf, strlen(cdbuf),
+ task->task_pool);
+
+ g_ptr_array_add(MESSAGE_FIELD(task, parts), part);
+ rspamd_mime_parser_calc_digest(part);
+
+ /* Generate message ID */
+ mid = rspamd_mime_message_id_generate("localhost.localdomain");
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) g_free, mid);
+ MESSAGE_FIELD(task, message_id) = mid;
+ task->queue_id = mid;
+}
+
+static void
+rspamd_message_dtor(struct rspamd_message *msg)
+{
+ guint i;
+ struct rspamd_mime_part *p;
+ struct rspamd_mime_text_part *tp;
+
+
+ PTR_ARRAY_FOREACH(msg->parts, i, p)
+ {
+ if (p->raw_headers) {
+ rspamd_message_headers_unref(p->raw_headers);
+ }
+
+ if (IS_PART_MULTIPART(p)) {
+ if (p->specific.mp->children) {
+ g_ptr_array_free(p->specific.mp->children, TRUE);
+ }
+ }
+
+ if (p->part_type == RSPAMD_MIME_PART_CUSTOM_LUA &&
+ p->specific.lua_specific.cbref != -1) {
+ luaL_unref(msg->task->cfg->lua_state,
+ LUA_REGISTRYINDEX,
+ p->specific.lua_specific.cbref);
+ }
+
+ if (p->urls) {
+ g_ptr_array_unref(p->urls);
+ }
+ }
+
+ PTR_ARRAY_FOREACH(msg->text_parts, i, tp)
+ {
+ if (tp->utf_words) {
+ g_array_free(tp->utf_words, TRUE);
+ }
+ if (tp->normalized_hashes) {
+ g_array_free(tp->normalized_hashes, TRUE);
+ }
+ if (tp->languages) {
+ g_ptr_array_unref(tp->languages);
+ }
+ }
+
+ rspamd_message_headers_unref(msg->raw_headers);
+
+ g_ptr_array_unref(msg->text_parts);
+ g_ptr_array_unref(msg->parts);
+
+ kh_destroy(rspamd_url_hash, msg->urls);
+}
+
+struct rspamd_message *
+rspamd_message_new(struct rspamd_task *task)
+{
+ struct rspamd_message *msg;
+
+ msg = rspamd_mempool_alloc0(task->task_pool, sizeof(*msg));
+
+ msg->raw_headers = rspamd_message_headers_new();
+ msg->urls = kh_init(rspamd_url_hash);
+ msg->parts = g_ptr_array_sized_new(4);
+ msg->text_parts = g_ptr_array_sized_new(2);
+ msg->task = task;
+
+ REF_INIT_RETAIN(msg, rspamd_message_dtor);
+
+ return msg;
+}
+
+gboolean
+rspamd_message_parse(struct rspamd_task *task)
+{
+ const gchar *p;
+ gsize len;
+ guint i;
+ GError *err = NULL;
+ guint64 n[2], seed;
+
+ if (RSPAMD_TASK_IS_EMPTY(task)) {
+ /* Don't do anything with empty task */
+ task->flags |= RSPAMD_TASK_FLAG_SKIP_PROCESS;
+ return TRUE;
+ }
+
+ p = task->msg.begin;
+ len = task->msg.len;
+
+ /* Skip any space characters to avoid some bad messages to be unparsed */
+ while (len > 0 && g_ascii_isspace(*p)) {
+ p++;
+ len--;
+ }
+
+ /*
+ * Exim somehow uses mailbox format for messages being scanned:
+ * From xxx@xxx.com Fri May 13 19:08:48 2016
+ *
+ * So we check if a task has this line to avoid possible issues
+ */
+ if (len > sizeof("From ") - 1) {
+ if (memcmp(p, "From ", sizeof("From ") - 1) == 0) {
+ /* Skip to CRLF */
+ msg_info_task("mailbox input detected, enable workaround");
+ p += sizeof("From ") - 1;
+ len -= sizeof("From ") - 1;
+
+ while (len > 0 && *p != '\n') {
+ p++;
+ len--;
+ }
+ while (len > 0 && g_ascii_isspace(*p)) {
+ p++;
+ len--;
+ }
+ }
+ }
+
+ task->msg.begin = p;
+ task->msg.len = len;
+
+ /* Cleanup old message */
+ if (task->message) {
+ rspamd_message_unref(task->message);
+ }
+
+ task->message = rspamd_message_new(task);
+
+ if (task->flags & RSPAMD_TASK_FLAG_MIME) {
+ enum rspamd_mime_parse_error ret;
+
+ debug_task("construct mime parser from string length %d",
+ (gint) task->msg.len);
+ ret = rspamd_mime_parse_task(task, &err);
+
+ switch (ret) {
+ case RSPAMD_MIME_PARSE_FATAL:
+ msg_err_task("cannot construct mime from stream: %e", err);
+
+ if (task->cfg && (!task->cfg->allow_raw_input)) {
+ msg_err_task("cannot construct mime from stream");
+ if (err) {
+ task->err = err;
+ }
+
+ return FALSE;
+ }
+ else {
+ task->flags &= ~RSPAMD_TASK_FLAG_MIME;
+ rspamd_message_from_data(task, p, len);
+ }
+ break;
+ case RSPAMD_MIME_PARSE_NESTING:
+ msg_warn_task("cannot construct full mime from stream: %e", err);
+ task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
+ break;
+ case RSPAMD_MIME_PARSE_OK:
+ default:
+ break;
+ }
+
+ if (err) {
+ g_error_free(err);
+ }
+ }
+ else {
+ rspamd_message_from_data(task, p, len);
+ }
+
+
+ if (MESSAGE_FIELD(task, message_id) == NULL) {
+ MESSAGE_FIELD(task, message_id) = "undef";
+ }
+
+ debug_task("found %ud parts in message", MESSAGE_FIELD(task, parts)->len);
+ if (task->queue_id == NULL) {
+ task->queue_id = "undef";
+ }
+
+ rspamd_received_maybe_fix_task(task);
+
+ struct rspamd_mime_part *part;
+
+ /* Blake2b applied to string 'rspamd' */
+ static const guchar RSPAMD_ALIGNED(32) hash_key[] = {
+ 0xef,
+ 0x43,
+ 0xae,
+ 0x80,
+ 0xcc,
+ 0x8d,
+ 0xc3,
+ 0x4c,
+ 0x6f,
+ 0x1b,
+ 0xd6,
+ 0x18,
+ 0x1b,
+ 0xae,
+ 0x87,
+ 0x74,
+ 0x0c,
+ 0xca,
+ 0xf7,
+ 0x8e,
+ 0x5f,
+ 0x2e,
+ 0x54,
+ 0x32,
+ 0xf6,
+ 0x79,
+ 0xb9,
+ 0x27,
+ 0x26,
+ 0x96,
+ 0x20,
+ 0x92,
+ 0x70,
+ 0x07,
+ 0x85,
+ 0xeb,
+ 0x83,
+ 0xf7,
+ 0x89,
+ 0xe0,
+ 0xd7,
+ 0x32,
+ 0x2a,
+ 0xd2,
+ 0x1a,
+ 0x64,
+ 0x41,
+ 0xef,
+ 0x49,
+ 0xff,
+ 0xc3,
+ 0x8c,
+ 0x54,
+ 0xf9,
+ 0x67,
+ 0x74,
+ 0x30,
+ 0x1e,
+ 0x70,
+ 0x2e,
+ 0xb7,
+ 0x12,
+ 0x09,
+ 0xfe,
+ };
+
+ memcpy(&seed, hash_key, sizeof(seed));
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ n[0] = t1ha2_atonce128(&n[1],
+ part->digest, sizeof(part->digest),
+ seed);
+
+ seed = n[0] ^ n[1];
+ }
+
+ memcpy(MESSAGE_FIELD(task, digest), n, sizeof(n));
+
+ if (MESSAGE_FIELD(task, subject)) {
+ p = MESSAGE_FIELD(task, subject);
+ len = strlen(p);
+ n[0] = t1ha2_atonce128(&n[1],
+ p, len,
+ seed);
+ memcpy(MESSAGE_FIELD(task, digest), n, sizeof(n));
+ }
+
+ if (task->queue_id) {
+ msg_info_task("loaded message; id: <%s>; queue-id: <%s>; size: %z; "
+ "checksum: <%*xs>",
+ MESSAGE_FIELD(task, message_id), task->queue_id, task->msg.len,
+ (gint) sizeof(MESSAGE_FIELD(task, digest)), MESSAGE_FIELD(task, digest));
+ }
+ else {
+ msg_info_task("loaded message; id: <%s>; size: %z; "
+ "checksum: <%*xs>",
+ MESSAGE_FIELD(task, message_id), task->msg.len,
+ (gint) sizeof(MESSAGE_FIELD(task, digest)), MESSAGE_FIELD(task, digest));
+ }
+
+ return TRUE;
+}
+
+
+/*
+ * A helper structure to store text parts positions, if it was C++, I could just use std::pair,
+ * but here I have to make it all manually, sigh...
+ */
+struct rspamd_mime_part_text_position {
+ unsigned pos;
+ enum rspamd_message_part_is_text_result res;
+};
+
+/* Place html parts first during analysis */
+static int
+rspamd_mime_text_part_position_compare_func(const void *v1, const void *v2)
+{
+ const struct rspamd_mime_part_text_position *p1 = (const struct rspamd_mime_part_text_position *) v1;
+ const struct rspamd_mime_part_text_position *p2 = (const struct rspamd_mime_part_text_position *) v2;
+
+ if (p1->res == p2->res) {
+ return (int) p2->pos - (int) p1->pos;
+ }
+ else {
+ if (p1->res == RSPAMD_MESSAGE_PART_IS_TEXT_HTML) {
+ return -1;
+ }
+ else {
+ return 1;
+ }
+ }
+}
+
+void rspamd_message_process(struct rspamd_task *task)
+{
+ guint i;
+ struct rspamd_mime_text_part *p1, *p2;
+ gdouble diff, *pdiff;
+ guint tw, *ptw, dw;
+ struct rspamd_mime_part *part;
+ lua_State *L = NULL;
+ gint magic_func_pos = -1, content_func_pos = -1, old_top = -1, funcs_top = -1;
+
+ if (task->cfg) {
+ L = task->cfg->lua_state;
+ }
+
+ rspamd_archives_process(task);
+
+ if (L) {
+ old_top = lua_gettop(L);
+ }
+
+ if (L && rspamd_lua_require_function(L,
+ "lua_magic", "detect_mime_part")) {
+ magic_func_pos = lua_gettop(L);
+ }
+ else {
+ msg_err_task("cannot require lua_magic.detect_mime_part");
+ }
+
+ if (L && rspamd_lua_require_function(L,
+ "lua_content", "maybe_process_mime_part")) {
+ content_func_pos = lua_gettop(L);
+ }
+ else {
+ msg_err_task("cannot require lua_content.maybe_process_mime_part");
+ }
+
+ if (L) {
+ funcs_top = lua_gettop(L);
+ }
+
+ GArray *detected_text_parts = g_array_sized_new(FALSE, FALSE, sizeof(struct rspamd_mime_part_text_position), 2);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ if (magic_func_pos != -1 && part->parsed_data.len > 0) {
+ struct rspamd_mime_part **pmime;
+ struct rspamd_task **ptask;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(L);
+ lua_pushvalue(L, magic_func_pos);
+ pmime = lua_newuserdata(L, sizeof(struct rspamd_mime_part *));
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+ *pmime = part;
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+
+ if (lua_pcall(L, 2, 2, err_idx) != 0) {
+ msg_err_task("cannot detect type: %s", lua_tostring(L, -1));
+ }
+ else {
+ if (lua_istable(L, -1)) {
+ const gchar *mb;
+
+ /* First returned value */
+ part->detected_ext = rspamd_mempool_strdup(task->task_pool,
+ lua_tostring(L, -2));
+
+ lua_pushstring(L, "ct");
+ lua_gettable(L, -2);
+
+ if (lua_isstring(L, -1)) {
+ mb = lua_tostring(L, -1);
+
+ if (mb) {
+ rspamd_ftok_t srch;
+
+ srch.begin = mb;
+ srch.len = strlen(mb);
+ part->detected_ct = rspamd_content_type_parse(srch.begin,
+ srch.len,
+ task->task_pool);
+ }
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "type");
+ lua_gettable(L, -2);
+
+ if (lua_isstring(L, -1)) {
+ part->detected_type = rspamd_mempool_strdup(task->task_pool,
+ lua_tostring(L, -1));
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "no_text");
+ lua_gettable(L, -2);
+
+ if (lua_isboolean(L, -1)) {
+ if (!!lua_toboolean(L, -1)) {
+ part->flags |= RSPAMD_MIME_PART_NO_TEXT_EXTRACTION;
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+
+ lua_settop(L, funcs_top);
+ }
+
+ /* Now detect content */
+ if (content_func_pos != -1 && part->parsed_data.len > 0 &&
+ part->part_type == RSPAMD_MIME_PART_UNDEFINED) {
+ struct rspamd_mime_part **pmime;
+ struct rspamd_task **ptask;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(L);
+ lua_pushvalue(L, content_func_pos);
+ pmime = lua_newuserdata(L, sizeof(struct rspamd_mime_part *));
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+ *pmime = part;
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+
+ if (lua_pcall(L, 2, 0, err_idx) != 0) {
+ msg_err_task("cannot detect content: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, funcs_top);
+ }
+
+ /* Try to detect image before checking for text */
+ rspamd_images_process_mime_part_maybe(task, part);
+
+ if (part->part_type == RSPAMD_MIME_PART_UNDEFINED &&
+ !(part->flags & RSPAMD_MIME_PART_NO_TEXT_EXTRACTION)) {
+ enum rspamd_message_part_is_text_result res = rspamd_message_part_can_be_parsed_as_text(task, part);
+
+ if (res != RSPAMD_MESSAGE_PART_IS_NOT_TEXT) {
+ struct rspamd_mime_part_text_position p = {
+ .pos = i,
+ .res = res};
+ g_array_append_val(detected_text_parts, p);
+ }
+ }
+ }
+
+ uint16_t cur_url_order = 0;
+ g_array_sort(detected_text_parts, rspamd_mime_text_part_position_compare_func);
+ /* One more iteration to process text parts in a more specific order */
+ for (i = 0; i < detected_text_parts->len; i++) {
+ part = g_ptr_array_index(MESSAGE_FIELD(task, parts),
+ g_array_index(detected_text_parts, struct rspamd_mime_part_text_position, i).pos);
+ rspamd_message_process_text_part_maybe(task, part,
+ g_array_index(detected_text_parts, struct rspamd_mime_part_text_position, i).res, &cur_url_order);
+ }
+
+ g_array_free(detected_text_parts, TRUE);
+
+ if (old_top != -1) {
+ lua_settop(L, old_top);
+ }
+
+ /* Parse urls inside Subject header */
+ if (MESSAGE_FIELD(task, subject)) {
+ rspamd_url_find_multiple(task->task_pool, MESSAGE_FIELD(task, subject),
+ strlen(MESSAGE_FIELD(task, subject)),
+ RSPAMD_URL_FIND_STRICT, NULL,
+ rspamd_url_task_subject_callback,
+ task);
+ }
+
+ /* Calculate average words length and number of short words */
+ struct rspamd_mime_text_part *text_part;
+ gdouble *var;
+ guint total_words = 0;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, text_part)
+ {
+ if (!text_part->language) {
+ rspamd_mime_part_detect_language(task, text_part);
+ }
+
+ rspamd_mime_part_extract_words(task, text_part);
+
+ if (text_part->utf_words) {
+ total_words += text_part->nwords;
+ }
+ }
+
+ /* Calculate distance for 2-parts messages */
+ if (i == 2) {
+ p1 = g_ptr_array_index(MESSAGE_FIELD(task, text_parts), 0);
+ p2 = g_ptr_array_index(MESSAGE_FIELD(task, text_parts), 1);
+
+ /* First of all check parent object */
+ if (p1->mime_part->parent_part) {
+ rspamd_ftok_t srch;
+
+ srch.begin = "alternative";
+ srch.len = 11;
+
+ if (rspamd_ftok_cmp(&p1->mime_part->parent_part->ct->subtype, &srch) == 0) {
+ if (!IS_TEXT_PART_EMPTY(p1) && !IS_TEXT_PART_EMPTY(p2) &&
+ p1->normalized_hashes && p2->normalized_hashes) {
+ /*
+ * We also detect language on one part and propagate it to
+ * another one
+ */
+ struct rspamd_mime_text_part *sel;
+
+ /* Prefer HTML as text part is not displayed normally */
+ if (IS_TEXT_PART_HTML(p1)) {
+ sel = p1;
+ }
+ else if (IS_TEXT_PART_HTML(p2)) {
+ sel = p2;
+ }
+ else {
+ if (p1->utf_content.len > p2->utf_content.len) {
+ sel = p1;
+ }
+ else {
+ sel = p2;
+ }
+ }
+
+ if (sel->language && sel->language[0]) {
+ /* Propagate language */
+ if (sel == p1) {
+ if (p2->languages) {
+ g_ptr_array_unref(p2->languages);
+ }
+
+ p2->language = sel->language;
+ p2->languages = g_ptr_array_ref(sel->languages);
+ }
+ else {
+ if (p1->languages) {
+ g_ptr_array_unref(p1->languages);
+ }
+
+ p1->language = sel->language;
+ p1->languages = g_ptr_array_ref(sel->languages);
+ }
+ }
+
+ tw = p1->normalized_hashes->len + p2->normalized_hashes->len;
+
+ if (tw > 0) {
+ dw = rspamd_words_levenshtein_distance(task,
+ p1->normalized_hashes,
+ p2->normalized_hashes);
+ diff = dw / (gdouble) tw;
+
+ msg_debug_task(
+ "different words: %d, total words: %d, "
+ "got diff between parts of %.2f",
+ dw, tw,
+ diff);
+
+ pdiff = rspamd_mempool_alloc(task->task_pool,
+ sizeof(gdouble));
+ *pdiff = diff;
+ rspamd_mempool_set_variable(task->task_pool,
+ "parts_distance",
+ pdiff,
+ NULL);
+ ptw = rspamd_mempool_alloc(task->task_pool,
+ sizeof(gint));
+ *ptw = tw;
+ rspamd_mempool_set_variable(task->task_pool,
+ "total_words",
+ ptw,
+ NULL);
+ }
+ }
+ }
+ }
+ else {
+ debug_task(
+ "message contains two parts but they are in different multi-parts");
+ }
+ }
+
+ if (total_words > 0) {
+ var = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_AVG_WORDS_LEN);
+
+ if (var) {
+ *var /= (double) total_words;
+ }
+
+ var = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_SHORT_WORDS_CNT);
+
+ if (var) {
+ *var /= (double) total_words;
+ }
+ }
+
+ rspamd_images_link(task);
+ rspamd_tokenize_meta_words(task);
+}
+
+
+struct rspamd_message *
+rspamd_message_ref(struct rspamd_message *msg)
+{
+ REF_RETAIN(msg);
+
+ return msg;
+}
+
+void rspamd_message_unref(struct rspamd_message *msg)
+{
+ if (msg) {
+ REF_RELEASE(msg);
+ }
+}
+
+void rspamd_message_update_digest(struct rspamd_message *msg,
+ const void *input, gsize len)
+{
+ guint64 n[2];
+ /* Sanity */
+ G_STATIC_ASSERT(sizeof(n) == sizeof(msg->digest));
+
+ memcpy(n, msg->digest, sizeof(msg->digest));
+ n[0] = t1ha2_atonce128(&n[1], input, len, n[0]);
+ memcpy(msg->digest, n, sizeof(msg->digest));
+}
diff --git a/src/libmime/message.h b/src/libmime/message.h
new file mode 100644
index 0000000..52dedab
--- /dev/null
+++ b/src/libmime/message.h
@@ -0,0 +1,239 @@
+/**
+ * @file message.h
+ * Message processing functions and structures
+ */
+
+#ifndef RSPAMD_MESSAGE_H
+#define RSPAMD_MESSAGE_H
+
+#include "config.h"
+
+#include "libmime/email_addr.h"
+#include "libutil/addr.h"
+#include "libcryptobox/cryptobox.h"
+#include "libmime/mime_headers.h"
+#include "libmime/content_type.h"
+#include "libserver/url.h"
+#include "libutil/ref.h"
+#include "libutil/str_util.h"
+
+#include <unicode/uchar.h>
+#include <unicode/utext.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct controller_session;
+struct rspamd_image;
+struct rspamd_archive;
+
+enum rspamd_mime_part_flags {
+ RSPAMD_MIME_PART_ATTACHEMENT = (1u << 1u),
+ RSPAMD_MIME_PART_BAD_CTE = (1u << 4u),
+ RSPAMD_MIME_PART_MISSING_CTE = (1u << 5u),
+ RSPAMD_MIME_PART_NO_TEXT_EXTRACTION = (1u << 6u),
+};
+
+enum rspamd_mime_part_type {
+ RSPAMD_MIME_PART_UNDEFINED = 0,
+ RSPAMD_MIME_PART_MULTIPART,
+ RSPAMD_MIME_PART_MESSAGE,
+ RSPAMD_MIME_PART_TEXT,
+ RSPAMD_MIME_PART_ARCHIVE,
+ RSPAMD_MIME_PART_IMAGE,
+ RSPAMD_MIME_PART_CUSTOM_LUA
+};
+
+#define IS_PART_MULTIPART(part) ((part) && ((part)->part_type == RSPAMD_MIME_PART_MULTIPART))
+#define IS_PART_TEXT(part) ((part) && ((part)->part_type == RSPAMD_MIME_PART_TEXT))
+#define IS_PART_MESSAGE(part) ((part) && ((part)->part_type == RSPAMD_MIME_PART_MESSAGE))
+
+enum rspamd_cte {
+ RSPAMD_CTE_UNKNOWN = 0,
+ RSPAMD_CTE_7BIT = 1,
+ RSPAMD_CTE_8BIT = 2,
+ RSPAMD_CTE_QP = 3,
+ RSPAMD_CTE_B64 = 4,
+ RSPAMD_CTE_UUE = 5,
+};
+
+struct rspamd_mime_text_part;
+
+struct rspamd_mime_multipart {
+ GPtrArray *children;
+ rspamd_ftok_t boundary;
+};
+
+enum rspamd_lua_specific_type {
+ RSPAMD_LUA_PART_TEXT,
+ RSPAMD_LUA_PART_STRING,
+ RSPAMD_LUA_PART_TABLE,
+ RSPAMD_LUA_PART_FUNCTION,
+ RSPAMD_LUA_PART_UNKNOWN,
+};
+
+struct rspamd_lua_specific_part {
+ gint cbref;
+ enum rspamd_lua_specific_type type;
+};
+
+struct rspamd_mime_part {
+ struct rspamd_content_type *ct;
+ struct rspamd_content_type *detected_ct;
+ gchar *detected_type;
+ gchar *detected_ext;
+ struct rspamd_content_disposition *cd;
+ rspamd_ftok_t raw_data;
+ rspamd_ftok_t parsed_data;
+ struct rspamd_mime_part *parent_part;
+
+ struct rspamd_mime_header *headers_order;
+ struct rspamd_mime_headers_table *raw_headers;
+ GPtrArray *urls;
+
+ gchar *raw_headers_str;
+ gsize raw_headers_len;
+
+ enum rspamd_cte cte;
+ guint flags;
+ enum rspamd_mime_part_type part_type;
+ guint part_number;
+
+ union {
+ struct rspamd_mime_multipart *mp;
+ struct rspamd_mime_text_part *txt;
+ struct rspamd_image *img;
+ struct rspamd_archive *arch;
+ struct rspamd_lua_specific_part lua_specific;
+ } specific;
+
+ guchar digest[rspamd_cryptobox_HASHBYTES];
+};
+
+#define RSPAMD_MIME_TEXT_PART_FLAG_UTF (1 << 0)
+#define RSPAMD_MIME_TEXT_PART_FLAG_EMPTY (1 << 1)
+#define RSPAMD_MIME_TEXT_PART_FLAG_HTML (1 << 2)
+#define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_RAW (1 << 3)
+#define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED (1 << 4)
+#define RSPAMD_MIME_TEXT_PART_ATTACHMENT (1 << 5)
+
+#define IS_TEXT_PART_EMPTY(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_EMPTY)
+#define IS_TEXT_PART_UTF(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_UTF)
+#define IS_TEXT_PART_HTML(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_HTML)
+#define IS_TEXT_PART_ATTACHMENT(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_ATTACHMENT)
+
+
+struct rspamd_mime_text_part {
+ const gchar *language;
+ GPtrArray *languages;
+ const gchar *real_charset;
+
+ /* Raw data in native encoding */
+ rspamd_ftok_t raw;
+ rspamd_ftok_t parsed; /* decoded from mime encodings */
+
+ /* UTF8 content */
+ rspamd_ftok_t utf_content; /* utf8 encoded processed content */
+ GByteArray *utf_raw_content; /* utf raw content */
+ GByteArray *utf_stripped_content; /* utf content with no newlines */
+ GArray *normalized_hashes; /* Array of guint64 */
+ GArray *utf_words; /* Array of rspamd_stat_token_t */
+ UText utf_stripped_text; /* Used by libicu to represent the utf8 content */
+
+ GPtrArray *newlines; /**< positions of newlines in text, relative to content*/
+ void *html;
+ GList *exceptions; /**< list of offsets of urls */
+ struct rspamd_mime_part *mime_part;
+
+ guint flags;
+ guint nlines;
+ guint spaces;
+ guint nwords;
+ guint non_ascii_chars;
+ guint ascii_chars;
+ guint double_spaces;
+ guint non_spaces;
+ guint empty_lines;
+ guint capital_letters;
+ guint numeric_characters;
+ guint unicode_scripts;
+};
+
+struct rspamd_message_raw_headers_content {
+ const gchar *begin;
+ gsize len;
+ const gchar *body_start;
+};
+
+struct rspamd_message {
+ const gchar *message_id;
+ gchar *subject;
+
+ GPtrArray *parts; /**< list of parsed parts */
+ GPtrArray *text_parts; /**< list of text parts */
+ struct rspamd_message_raw_headers_content raw_headers_content;
+ void *received_headers; /**< list of received headers */
+ khash_t(rspamd_url_hash) * urls;
+ struct rspamd_mime_headers_table *raw_headers; /**< list of raw headers */
+ struct rspamd_mime_header *headers_order; /**< order of raw headers */
+ struct rspamd_task *task;
+ GPtrArray *rcpt_mime;
+ GPtrArray *from_mime;
+ guchar digest[16];
+ enum rspamd_newlines_type nlines_type; /**< type of newlines (detected on most of headers */
+ ref_entry_t ref;
+};
+
+#define MESSAGE_FIELD(task, field) ((task)->message->field)
+#define MESSAGE_FIELD_CHECK(task, field) ((task)->message ? (task)->message->field : (__typeof__((task)->message->field)) NULL)
+
+/**
+ * Parse and pre-process mime message
+ * @param task worker_task object
+ * @return
+ */
+gboolean rspamd_message_parse(struct rspamd_task *task);
+
+/**
+ * Process content in task (e.g. HTML parsing)
+ * @param task
+ */
+void rspamd_message_process(struct rspamd_task *task);
+
+
+/**
+ * Converts string to cte
+ * @param str
+ * @return
+ */
+enum rspamd_cte rspamd_cte_from_string(const gchar *str);
+
+/**
+ * Converts cte to string
+ * @param ct
+ * @return
+ */
+const gchar *rspamd_cte_to_string(enum rspamd_cte ct);
+
+struct rspamd_message *rspamd_message_new(struct rspamd_task *task);
+
+struct rspamd_message *rspamd_message_ref(struct rspamd_message *msg);
+
+void rspamd_message_unref(struct rspamd_message *msg);
+
+/**
+ * Updates digest of the message if modified
+ * @param msg
+ * @param input
+ * @param len
+ */
+void rspamd_message_update_digest(struct rspamd_message *msg,
+ const void *input, gsize len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libmime/mime_encoding.c b/src/libmime/mime_encoding.c
new file mode 100644
index 0000000..48a97a4
--- /dev/null
+++ b/src/libmime/mime_encoding.c
@@ -0,0 +1,864 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "libutil/mem_pool.h"
+#include "libutil/regexp.h"
+#include "libutil/hash.h"
+#include "libserver/cfg_file.h"
+#include "libserver/task.h"
+#include "mime_encoding.h"
+#include "message.h"
+#include "contrib/fastutf8/fastutf8.h"
+#include "contrib/google-ced/ced_c.h"
+#include <unicode/ucnv.h>
+#if U_ICU_VERSION_MAJOR_NUM >= 44
+#include <unicode/unorm2.h>
+#endif
+#include <math.h>
+
+#define UTF8_CHARSET "UTF-8"
+
+#define RSPAMD_CHARSET_FLAG_UTF (1 << 0)
+#define RSPAMD_CHARSET_FLAG_ASCII (1 << 1)
+
+#define RSPAMD_CHARSET_CACHE_SIZE 32
+#define RSPAMD_CHARSET_MAX_CONTENT 512
+
+#define SET_PART_RAW(part) ((part)->flags &= ~RSPAMD_MIME_TEXT_PART_FLAG_UTF)
+#define SET_PART_UTF(part) ((part)->flags |= RSPAMD_MIME_TEXT_PART_FLAG_UTF)
+
+static rspamd_regexp_t *utf_compatible_re = NULL;
+
+struct rspamd_charset_substitution {
+ const gchar *input;
+ const gchar *canon;
+ gint flags;
+};
+
+#include "mime_encoding_list.h"
+
+static GHashTable *sub_hash = NULL;
+
+static const UChar iso_8859_16_map[] = {
+ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
+ 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
+ 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
+ 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
+ 0x00A0, 0x0104, 0x0105, 0x0141, 0x20AC, 0x201E, 0x0160, 0x00A7,
+ 0x0161, 0x00A9, 0x0218, 0x00AB, 0x0179, 0x00AD, 0x017A, 0x017B,
+ 0x00B0, 0x00B1, 0x010C, 0x0142, 0x017D, 0x201D, 0x00B6, 0x00B7,
+ 0x017E, 0x010D, 0x0219, 0x00BB, 0x0152, 0x0153, 0x0178, 0x017C,
+ 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0106, 0x00C6, 0x00C7,
+ 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
+ 0x0110, 0x0143, 0x00D2, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x015A,
+ 0x0170, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0118, 0x021A, 0x00DF,
+ 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x0107, 0x00E6, 0x00E7,
+ 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
+ 0x0111, 0x0144, 0x00F2, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x015B,
+ 0x0171, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0119, 0x021B, 0x00FF};
+
+struct rspamd_charset_converter {
+ gchar *canon_name;
+ union {
+ UConverter *conv;
+ const UChar *cnv_table;
+ } d;
+ gboolean is_internal;
+};
+
+static GQuark
+rspamd_charset_conv_error_quark(void)
+{
+ return g_quark_from_static_string("charset conversion error");
+}
+
+static void
+rspamd_converter_dtor(gpointer p)
+{
+ struct rspamd_charset_converter *c = (struct rspamd_charset_converter *) p;
+
+ if (!c->is_internal) {
+ ucnv_close(c->d.conv);
+ }
+
+ g_free(c->canon_name);
+ g_free(c);
+}
+
+int32_t
+rspamd_converter_to_uchars(struct rspamd_charset_converter *cnv,
+ UChar *dest,
+ int32_t destCapacity,
+ const char *src,
+ int32_t srcLength,
+ UErrorCode *pErrorCode)
+{
+ if (!cnv->is_internal) {
+ return ucnv_toUChars(cnv->d.conv,
+ dest, destCapacity,
+ src, srcLength,
+ pErrorCode);
+ }
+ else {
+ UChar *d = dest, *dend = dest + destCapacity;
+ const guchar *p = src, *end = src + srcLength;
+
+ while (p < end && d < dend) {
+ if (*p <= 127) {
+ *d++ = (UChar) *p;
+ }
+ else {
+ *d++ = cnv->d.cnv_table[*p - 128];
+ }
+
+ p++;
+ }
+
+ return d - dest;
+ }
+}
+
+
+struct rspamd_charset_converter *
+rspamd_mime_get_converter_cached(const gchar *enc,
+ rspamd_mempool_t *pool,
+ gboolean is_canon,
+ UErrorCode *err)
+{
+ const gchar *canon_name;
+ static rspamd_lru_hash_t *cache;
+ struct rspamd_charset_converter *conv;
+
+ if (cache == NULL) {
+ cache = rspamd_lru_hash_new_full(RSPAMD_CHARSET_CACHE_SIZE, NULL,
+ rspamd_converter_dtor, rspamd_str_hash,
+ rspamd_str_equal);
+ }
+
+ if (enc == NULL) {
+ return NULL;
+ }
+
+ if (!is_canon) {
+ rspamd_ftok_t cset_tok;
+
+ RSPAMD_FTOK_FROM_STR(&cset_tok, enc);
+ canon_name = rspamd_mime_detect_charset(&cset_tok, pool);
+ }
+ else {
+ canon_name = enc;
+ }
+
+ if (canon_name == NULL) {
+ return NULL;
+ }
+
+ conv = rspamd_lru_hash_lookup(cache, (gpointer) canon_name, 0);
+
+ if (conv == NULL) {
+ if (!(strcmp(canon_name, "ISO-8859-16") == 0 ||
+ strcmp(canon_name, "latin10") == 0 ||
+ strcmp(canon_name, "iso-ir-226") == 0)) {
+ conv = g_malloc0(sizeof(*conv));
+ conv->d.conv = ucnv_open(canon_name, err);
+ conv->canon_name = g_strdup(canon_name);
+
+ if (conv->d.conv != NULL) {
+ ucnv_setToUCallBack(conv->d.conv,
+ UCNV_TO_U_CALLBACK_SUBSTITUTE,
+ NULL,
+ NULL,
+ NULL,
+ err);
+ rspamd_lru_hash_insert(cache, conv->canon_name, conv, 0, 0);
+ }
+ else {
+ g_free(conv);
+ conv = NULL;
+ }
+ }
+ else {
+ /* ISO-8859-16 */
+ conv = g_malloc0(sizeof(*conv));
+ conv->is_internal = TRUE;
+ conv->d.cnv_table = iso_8859_16_map;
+ conv->canon_name = g_strdup(canon_name);
+
+ rspamd_lru_hash_insert(cache, conv->canon_name, conv, 0, 0);
+ }
+ }
+
+ return conv;
+}
+
+static void
+rspamd_mime_encoding_substitute_init(void)
+{
+ guint i;
+
+ sub_hash = g_hash_table_new(rspamd_strcase_hash, rspamd_strcase_equal);
+
+ for (i = 0; i < G_N_ELEMENTS(sub); i++) {
+ g_hash_table_insert(sub_hash, (void *) sub[i].input, (void *) &sub[i]);
+ }
+}
+
+static void
+rspamd_charset_normalize(gchar *in)
+{
+ /*
+ * This is a simple routine to validate input charset
+ * we just check that charset starts with alphanumeric and ends
+ * with alphanumeric
+ */
+ gchar *begin, *end;
+ gboolean changed = FALSE;
+
+ begin = in;
+
+ while (*begin && !g_ascii_isalnum(*begin)) {
+ begin++;
+ changed = TRUE;
+ }
+
+ end = begin + strlen(begin) - 1;
+
+ while (end > begin && !g_ascii_isalnum(*end)) {
+ end--;
+ changed = TRUE;
+ }
+
+ if (changed) {
+ memmove(in, begin, end - begin + 2);
+ *(end + 1) = '\0';
+ }
+}
+
+const gchar *
+rspamd_mime_detect_charset(const rspamd_ftok_t *in, rspamd_mempool_t *pool)
+{
+ gchar *ret = NULL, *h, *t;
+ struct rspamd_charset_substitution *s;
+ const gchar *cset;
+ rspamd_ftok_t utf8_tok;
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ if (sub_hash == NULL) {
+ rspamd_mime_encoding_substitute_init();
+ }
+
+ /* Fast path */
+ RSPAMD_FTOK_ASSIGN(&utf8_tok, "utf-8");
+
+ if (rspamd_ftok_casecmp(in, &utf8_tok) == 0) {
+ return UTF8_CHARSET;
+ }
+
+ RSPAMD_FTOK_ASSIGN(&utf8_tok, "utf8");
+
+ if (rspamd_ftok_casecmp(in, &utf8_tok) == 0) {
+ return UTF8_CHARSET;
+ }
+
+ ret = rspamd_mempool_ftokdup(pool, in);
+ rspamd_charset_normalize(ret);
+
+ if ((in->len > 3 && rspamd_lc_cmp(in->begin, "cp-", 3) == 0) ||
+ (in->len > 4 && (rspamd_lc_cmp(in->begin, "ibm-", 4) == 0))) {
+ /* Try to remove '-' chars from encoding: e.g. CP-100 to CP100 */
+ h = ret;
+ t = ret;
+
+ while (*h != '\0') {
+ if (*h != '-') {
+ *t++ = *h;
+ }
+
+ h++;
+ }
+
+ *t = '\0';
+ }
+
+ s = g_hash_table_lookup(sub_hash, ret);
+
+ if (s) {
+ ret = (char *) s->canon;
+ }
+
+ /* Try different aliases */
+ cset = ucnv_getCanonicalName(ret, "MIME", &uc_err);
+
+ if (cset == NULL) {
+ uc_err = U_ZERO_ERROR;
+ cset = ucnv_getCanonicalName(ret, "IANA", &uc_err);
+ }
+
+ if (cset == NULL) {
+ uc_err = U_ZERO_ERROR;
+ cset = ucnv_getCanonicalName(ret, "", &uc_err);
+ }
+
+ if (cset == NULL) {
+ uc_err = U_ZERO_ERROR;
+ cset = ucnv_getAlias(ret, 0, &uc_err);
+ }
+
+ return cset;
+}
+
+gchar *
+rspamd_mime_text_to_utf8(rspamd_mempool_t *pool,
+ gchar *input, gsize len, const gchar *in_enc,
+ gsize *olen, GError **err)
+{
+ gchar *d;
+ gint32 r, clen, dlen;
+ UChar *tmp_buf;
+
+ UErrorCode uc_err = U_ZERO_ERROR;
+ UConverter *utf8_converter;
+ struct rspamd_charset_converter *conv;
+ rspamd_ftok_t cset_tok;
+
+ /* Check if already utf8 */
+ RSPAMD_FTOK_FROM_STR(&cset_tok, in_enc);
+
+ if (rspamd_mime_charset_utf_check(&cset_tok, input, len,
+ FALSE)) {
+ d = rspamd_mempool_alloc(pool, len);
+ memcpy(d, input, len);
+ if (olen) {
+ *olen = len;
+ }
+
+ return d;
+ }
+
+ conv = rspamd_mime_get_converter_cached(in_enc, pool, TRUE, &uc_err);
+ utf8_converter = rspamd_get_utf8_converter();
+
+ if (conv == NULL) {
+ g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+ "cannot open converter for %s: %s",
+ in_enc, u_errorName(uc_err));
+
+ return NULL;
+ }
+
+ tmp_buf = g_new(UChar, len + 1);
+ uc_err = U_ZERO_ERROR;
+ r = rspamd_converter_to_uchars(conv, tmp_buf, len + 1, input, len, &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+ "cannot convert data to unicode from %s: %s",
+ in_enc, u_errorName(uc_err));
+ g_free(tmp_buf);
+
+ return NULL;
+ }
+
+ /* Now, convert to utf8 */
+ clen = ucnv_getMaxCharSize(utf8_converter);
+ dlen = UCNV_GET_MAX_BYTES_FOR_STRING(r, clen);
+ d = rspamd_mempool_alloc(pool, dlen);
+ r = ucnv_fromUChars(utf8_converter, d, dlen, tmp_buf, r, &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+ "cannot convert data from unicode from %s: %s",
+ in_enc, u_errorName(uc_err));
+ g_free(tmp_buf);
+
+ return NULL;
+ }
+
+ msg_debug_pool("converted from %s to UTF-8 inlen: %z, outlen: %d",
+ in_enc, len, r);
+ g_free(tmp_buf);
+
+ if (olen) {
+ *olen = r;
+ }
+
+ return d;
+}
+
+static gboolean
+rspamd_mime_text_part_utf8_convert(struct rspamd_task *task,
+ struct rspamd_mime_text_part *text_part,
+ GByteArray *input,
+ const gchar *charset,
+ GError **err)
+{
+ gchar *d;
+ gint32 r, clen, dlen, uc_len;
+ UChar *tmp_buf;
+ UErrorCode uc_err = U_ZERO_ERROR;
+ UConverter *utf8_converter;
+ struct rspamd_charset_converter *conv;
+
+ conv = rspamd_mime_get_converter_cached(charset, task->task_pool,
+ TRUE, &uc_err);
+ utf8_converter = rspamd_get_utf8_converter();
+
+ if (conv == NULL) {
+ g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+ "cannot open converter for %s: %s",
+ charset, u_errorName(uc_err));
+
+ return FALSE;
+ }
+
+ tmp_buf = g_new(UChar, input->len + 1);
+ uc_err = U_ZERO_ERROR;
+ uc_len = rspamd_converter_to_uchars(conv,
+ tmp_buf,
+ input->len + 1,
+ input->data,
+ input->len,
+ &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+ "cannot convert data to unicode from %s: %s",
+ charset, u_errorName(uc_err));
+ g_free(tmp_buf);
+
+ return FALSE;
+ }
+
+ /* Now, convert to utf8 */
+ clen = ucnv_getMaxCharSize(utf8_converter);
+ dlen = UCNV_GET_MAX_BYTES_FOR_STRING(uc_len, clen);
+ d = rspamd_mempool_alloc(task->task_pool, dlen);
+ r = ucnv_fromUChars(utf8_converter, d, dlen,
+ tmp_buf, uc_len, &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+ "cannot convert data from unicode from %s: %s",
+ charset, u_errorName(uc_err));
+ g_free(tmp_buf);
+
+ return FALSE;
+ }
+
+ if (text_part->mime_part && text_part->mime_part->ct) {
+ msg_info_task("converted text part from %s ('%T' announced) to UTF-8 inlen: %d, outlen: %d (%d UTF16 chars)",
+ charset, &text_part->mime_part->ct->charset, input->len, r, uc_len);
+ }
+ else {
+ msg_info_task("converted text part from %s (no charset announced) to UTF-8 inlen: %d, "
+ "outlen: %d (%d UTF16 chars)",
+ charset, input->len, r, uc_len);
+ }
+
+ text_part->utf_raw_content = rspamd_mempool_alloc(task->task_pool,
+ sizeof(*text_part->utf_raw_content) + sizeof(gpointer) * 4);
+ text_part->utf_raw_content->data = d;
+ text_part->utf_raw_content->len = r;
+ g_free(tmp_buf);
+
+ return TRUE;
+}
+
+gboolean
+rspamd_mime_to_utf8_byte_array(GByteArray *in,
+ GByteArray *out,
+ rspamd_mempool_t *pool,
+ const gchar *enc)
+{
+ gint32 r, clen, dlen;
+ UChar *tmp_buf;
+ UErrorCode uc_err = U_ZERO_ERROR;
+ UConverter *utf8_converter;
+ struct rspamd_charset_converter *conv;
+ rspamd_ftok_t charset_tok;
+
+ if (in == NULL || in->len == 0) {
+ return FALSE;
+ }
+
+ if (enc == NULL) {
+ /* Assume utf ? */
+ if (rspamd_fast_utf8_validate(in->data, in->len) == 0) {
+ g_byte_array_set_size(out, in->len);
+ memcpy(out->data, in->data, out->len);
+
+ return TRUE;
+ }
+ else {
+ /* Bad stuff, keep out */
+ return FALSE;
+ }
+ }
+
+ RSPAMD_FTOK_FROM_STR(&charset_tok, enc);
+
+ if (rspamd_mime_charset_utf_check(&charset_tok, (gchar *) in->data, in->len,
+ FALSE)) {
+ g_byte_array_set_size(out, in->len);
+ memcpy(out->data, in->data, out->len);
+
+ return TRUE;
+ }
+
+ utf8_converter = rspamd_get_utf8_converter();
+ conv = rspamd_mime_get_converter_cached(enc, pool, TRUE, &uc_err);
+
+ if (conv == NULL) {
+ return FALSE;
+ }
+
+ tmp_buf = g_new(UChar, in->len + 1);
+ uc_err = U_ZERO_ERROR;
+ r = rspamd_converter_to_uchars(conv,
+ tmp_buf, in->len + 1,
+ in->data, in->len, &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ g_free(tmp_buf);
+
+ return FALSE;
+ }
+
+ /* Now, convert to utf8 */
+ clen = ucnv_getMaxCharSize(utf8_converter);
+ dlen = UCNV_GET_MAX_BYTES_FOR_STRING(r, clen);
+ g_byte_array_set_size(out, dlen);
+ r = ucnv_fromUChars(utf8_converter, out->data, dlen, tmp_buf, r, &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ g_free(tmp_buf);
+
+ return FALSE;
+ }
+
+ g_free(tmp_buf);
+ out->len = r;
+
+ return TRUE;
+}
+
+void rspamd_mime_charset_utf_enforce(gchar *in, gsize len)
+{
+ gchar *p, *end;
+ goffset err_offset;
+ UChar32 uc = 0;
+
+ /* Now we validate input and replace bad characters with '?' symbol */
+ p = in;
+ end = in + len;
+
+ while (p < end && len > 0 && (err_offset = rspamd_fast_utf8_validate(p, len)) > 0) {
+ err_offset--; /* As it returns it 1 indexed */
+ gint32 cur_offset = err_offset;
+
+ while (cur_offset < len) {
+ gint32 tmp = cur_offset;
+
+ U8_NEXT(p, cur_offset, len, uc);
+
+ if (uc > 0) {
+ /* Fill string between err_offset and tmp with `?` character */
+ memset(p + err_offset, '?', tmp - err_offset);
+ break;
+ }
+ }
+
+ if (uc < 0) {
+ /* Fill till the end */
+ memset(p + err_offset, '?', len - err_offset);
+ break;
+ }
+
+ p += cur_offset;
+ len = end - p;
+ }
+}
+
+const char *
+rspamd_mime_charset_find_by_content(const gchar *in, gsize inlen,
+ bool check_utf8)
+{
+ int nconsumed;
+ bool is_reliable;
+ const gchar *ced_name;
+
+ if (check_utf8) {
+ if (rspamd_fast_utf8_validate(in, inlen) == 0) {
+ return UTF8_CHARSET;
+ }
+ }
+
+
+ ced_name = ced_encoding_detect(in, inlen, NULL, NULL,
+ NULL, 0, CED_EMAIL_CORPUS,
+ false, &nconsumed, &is_reliable);
+
+ if (ced_name) {
+
+ return ced_name;
+ }
+
+ return NULL;
+}
+
+static const char *
+rspamd_mime_charset_find_by_content_maybe_split(const gchar *in, gsize inlen)
+{
+ if (inlen < RSPAMD_CHARSET_MAX_CONTENT * 3) {
+ return rspamd_mime_charset_find_by_content(in, inlen, false);
+ }
+ else {
+ const gchar *c1, *c2, *c3;
+
+ c1 = rspamd_mime_charset_find_by_content(in, RSPAMD_CHARSET_MAX_CONTENT, false);
+ c2 = rspamd_mime_charset_find_by_content(in + inlen / 2,
+ RSPAMD_CHARSET_MAX_CONTENT, false);
+ c3 = rspamd_mime_charset_find_by_content(in + inlen - RSPAMD_CHARSET_MAX_CONTENT,
+ RSPAMD_CHARSET_MAX_CONTENT, false);
+
+ /* 7bit stuff */
+ if (c1 && strcmp(c1, "US-ASCII") == 0) {
+ c1 = NULL; /* Invalid - we have 8 bit there */
+ }
+ if (c2 && strcmp(c2, "US-ASCII") == 0) {
+ c2 = NULL; /* Invalid - we have 8 bit there */
+ }
+ if (c3 && strcmp(c3, "US-ASCII") == 0) {
+ c3 = NULL; /* Invalid - we have 8 bit there */
+ }
+
+ if (!c1) {
+ c1 = c2 ? c2 : c3;
+ }
+ if (!c2) {
+ c2 = c3 ? c3 : c1;
+ }
+ if (!c3) {
+ c3 = c1 ? c2 : c1;
+ }
+
+ if (c1 && c2 && c3) {
+ /* Quorum */
+ if (c1 == c2) {
+ return c1;
+ }
+ else if (c2 == c3) {
+ return c2;
+ }
+ else if (c1 == c3) {
+ return c3;
+ }
+
+ /* All charsets are distinct. Use the one from the top */
+ return c1;
+ }
+
+ return NULL;
+ }
+}
+
+gboolean
+rspamd_mime_charset_utf_check(rspamd_ftok_t *charset,
+ gchar *in, gsize len, gboolean content_check)
+{
+ const gchar *real_charset;
+
+ if (utf_compatible_re == NULL) {
+ utf_compatible_re = rspamd_regexp_new(
+ "^(?:utf-?8.*)|(?:us-ascii)|(?:ascii)|(?:ansi.*)|(?:CSASCII)$",
+ "i", NULL);
+ }
+
+ if (charset->len == 0 ||
+ rspamd_regexp_match(utf_compatible_re,
+ charset->begin, charset->len, TRUE)) {
+ /*
+ * In case of UTF8 charset we still can check the content to find
+ * corner cases
+ */
+ if (content_check) {
+ if (rspamd_fast_utf8_validate(in, len) != 0) {
+ real_charset = rspamd_mime_charset_find_by_content_maybe_split(in, len);
+
+ if (real_charset) {
+
+ if (rspamd_regexp_match(utf_compatible_re,
+ real_charset, strlen(real_charset), TRUE)) {
+ RSPAMD_FTOK_ASSIGN(charset, UTF8_CHARSET);
+
+ return TRUE;
+ }
+ else {
+ charset->begin = real_charset;
+ charset->len = strlen(real_charset);
+
+ return FALSE;
+ }
+ }
+
+ rspamd_mime_charset_utf_enforce(in, len);
+ }
+ }
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+void rspamd_mime_text_part_maybe_convert(struct rspamd_task *task,
+ struct rspamd_mime_text_part *text_part)
+{
+ GError *err = NULL;
+ const gchar *charset = NULL;
+ gboolean checked = FALSE, need_charset_heuristic = TRUE, valid_utf8 = FALSE;
+ GByteArray *part_content;
+ rspamd_ftok_t charset_tok;
+ struct rspamd_mime_part *part = text_part->mime_part;
+
+ if (rspamd_str_has_8bit(text_part->raw.begin, text_part->raw.len)) {
+ text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_8BIT_RAW;
+ }
+
+ /* Allocate copy storage */
+ part_content = g_byte_array_sized_new(text_part->parsed.len);
+ memcpy(part_content->data, text_part->parsed.begin, text_part->parsed.len);
+ part_content->len = text_part->parsed.len;
+ rspamd_mempool_notify_alloc(task->task_pool,
+ part_content->len);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) g_byte_array_unref, part_content);
+
+ if (rspamd_str_has_8bit(text_part->parsed.begin, text_part->parsed.len)) {
+ if (rspamd_fast_utf8_validate(text_part->parsed.begin, text_part->parsed.len) == 0) {
+ /* Valid UTF, likely all good */
+ need_charset_heuristic = FALSE;
+ valid_utf8 = TRUE;
+ checked = TRUE;
+ }
+
+ text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED;
+ }
+ else {
+ /* All 7bit characters, assume it valid utf */
+ need_charset_heuristic = FALSE;
+ valid_utf8 = TRUE;
+ checked = TRUE; /* Already valid utf, no need in further checks */
+ }
+
+ if (part->ct->charset.len == 0) {
+ if (need_charset_heuristic) {
+ charset = rspamd_mime_charset_find_by_content_maybe_split(text_part->parsed.begin,
+ text_part->parsed.len);
+
+ if (charset != NULL) {
+ msg_info_task("detected charset %s", charset);
+ }
+
+ checked = TRUE;
+ text_part->real_charset = charset;
+ }
+ else if (valid_utf8) {
+ SET_PART_UTF(text_part);
+ text_part->utf_raw_content = part_content;
+ text_part->real_charset = UTF8_CHARSET;
+
+ return;
+ }
+ }
+ else {
+ charset = rspamd_mime_detect_charset(&part->ct->charset,
+ task->task_pool);
+
+ if (charset == NULL) {
+ /* We don't know the real charset but can try heuristic */
+ if (need_charset_heuristic) {
+ charset = rspamd_mime_charset_find_by_content_maybe_split(part_content->data,
+ part_content->len);
+ msg_info_task("detected charset: %s", charset);
+ checked = TRUE;
+ text_part->real_charset = charset;
+ }
+ else if (valid_utf8) {
+ /* We already know that the input is valid utf, so skip heuristic */
+ text_part->real_charset = UTF8_CHARSET;
+ }
+ }
+ else {
+ text_part->real_charset = charset;
+
+ if (strcmp(charset, UTF8_CHARSET) != 0) {
+ /*
+ * We have detected some charset, but we don't know which one,
+ * so we need to reset valid utf8 flag and enforce it later
+ */
+ valid_utf8 = FALSE;
+ }
+ }
+ }
+
+ if (text_part->real_charset == NULL) {
+ msg_info_task("<%s>: has invalid charset; original charset: %T; Content-Type: \"%s\"",
+ MESSAGE_FIELD_CHECK(task, message_id), &part->ct->charset,
+ part->ct->cpy);
+ SET_PART_RAW(text_part);
+ text_part->utf_raw_content = part_content;
+
+ return;
+ }
+
+ RSPAMD_FTOK_FROM_STR(&charset_tok, charset);
+
+ if (!valid_utf8) {
+ if (rspamd_mime_charset_utf_check(&charset_tok, part_content->data,
+ part_content->len, !checked)) {
+ SET_PART_UTF(text_part);
+ text_part->utf_raw_content = part_content;
+ text_part->real_charset = UTF8_CHARSET;
+
+ return;
+ }
+ else {
+ charset = charset_tok.begin;
+
+ if (!rspamd_mime_text_part_utf8_convert(task, text_part,
+ part_content, charset, &err)) {
+ msg_warn_task("<%s>: cannot convert from %s to utf8: %s",
+ MESSAGE_FIELD(task, message_id),
+ charset,
+ err ? err->message : "unknown problem");
+ SET_PART_RAW(text_part);
+ g_error_free(err);
+
+ text_part->utf_raw_content = part_content;
+ return;
+ }
+
+ SET_PART_UTF(text_part);
+ text_part->real_charset = charset;
+ }
+ }
+ else {
+ SET_PART_UTF(text_part);
+ text_part->utf_raw_content = part_content;
+ }
+}
diff --git a/src/libmime/mime_encoding.h b/src/libmime/mime_encoding.h
new file mode 100644
index 0000000..ff81292
--- /dev/null
+++ b/src/libmime/mime_encoding.h
@@ -0,0 +1,148 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_MIME_ENCODING_H_
+#define SRC_LIBMIME_MIME_ENCODING_H_
+
+#include "config.h"
+#include "mem_pool.h"
+#include "fstring.h"
+#include <unicode/uchar.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_mime_part;
+struct rspamd_mime_text_part;
+struct rspamd_charset_converter;
+
+/**
+ * Convert charset alias to a canonic charset name
+ * @param pool pool to store temporary data
+ * @param in
+ * @return
+ */
+const gchar *rspamd_mime_detect_charset(const rspamd_ftok_t *in,
+ rspamd_mempool_t *pool);
+
+/**
+ * Convert text chunk to utf-8. Input encoding is substituted using
+ * `rspamd_mime_detect_charset`.
+ * If input encoding is already utf, this function returns input pointer.
+ * Memory is allocated from pool if a conversion is needed
+ * @param pool
+ * @param input
+ * @param len
+ * @param in_enc canon charset
+ * @param olen
+ * @param err
+ * @return
+ */
+gchar *rspamd_mime_text_to_utf8(rspamd_mempool_t *pool,
+ gchar *input, gsize len, const gchar *in_enc,
+ gsize *olen, GError **err);
+
+/**
+ * Converts data from `in` to `out`,
+ * returns `FALSE` if `enc` is not a valid iconv charset
+ *
+ * This function, in fact, copies `in` from `out` replacing out content in
+ * total.
+ * @param in
+ * @param out
+ * @param enc validated canonical charset name. If NULL, then utf8 check is done only
+ * @return
+ */
+gboolean rspamd_mime_to_utf8_byte_array(GByteArray *in,
+ GByteArray *out,
+ rspamd_mempool_t *pool,
+ const gchar *enc);
+
+/**
+ * Maybe convert part to utf-8
+ * @param task
+ * @param text_part
+ * @return
+ */
+void rspamd_mime_text_part_maybe_convert(struct rspamd_task *task,
+ struct rspamd_mime_text_part *text_part);
+
+/**
+ * Checks utf8 charset and normalize/validate utf8 string
+ * @param charset
+ * @param in
+ * @param len
+ * @return
+ */
+gboolean rspamd_mime_charset_utf_check(rspamd_ftok_t *charset,
+ gchar *in, gsize len,
+ gboolean content_check);
+
+/**
+ * Ensure that all characters in string are valid utf8 chars or replace them
+ * with '?'
+ * @param in
+ * @param len
+ */
+void rspamd_mime_charset_utf_enforce(gchar *in, gsize len);
+
+/**
+ * Gets cached converter
+ * @param enc input encoding
+ * @param pool pool to use for temporary normalisation
+ * @param is_canon TRUE if normalisation is needed
+ * @param err output error
+ * @return converter
+ */
+struct rspamd_charset_converter *rspamd_mime_get_converter_cached(
+ const gchar *enc,
+ rspamd_mempool_t *pool,
+ gboolean is_canon,
+ UErrorCode *err);
+
+/**
+ * Performs charset->utf16 conversion
+ * @param cnv
+ * @param dest
+ * @param destCapacity
+ * @param src
+ * @param srcLength
+ * @param pErrorCode
+ * @return
+ */
+gint32
+rspamd_converter_to_uchars(struct rspamd_charset_converter *cnv,
+ UChar *dest,
+ gint32 destCapacity,
+ const char *src,
+ gint32 srcLength,
+ UErrorCode *pErrorCode);
+
+/**
+ * Detect charset in text
+ * @param in
+ * @param inlen
+ * @return detected charset name or NULL
+ */
+const char *rspamd_mime_charset_find_by_content(const gchar *in, gsize inlen,
+ bool check_utf8);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_MIME_ENCODING_H_ */
diff --git a/src/libmime/mime_encoding_list.h b/src/libmime/mime_encoding_list.h
new file mode 100644
index 0000000..b5fc5e1
--- /dev/null
+++ b/src/libmime/mime_encoding_list.h
@@ -0,0 +1,1577 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_MIME_ENCODING_LIST_H_
+#define SRC_LIBMIME_MIME_ENCODING_LIST_H_
+
+static const struct rspamd_charset_substitution sub[] = {
+ {
+ .input = "iso-646-us",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "ansi_x3.4-1968",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "iso-ir-6",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "iso_646.irv:1991",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "ascii",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "iso646-us",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "us",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "ibm367",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "cp367",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "csascii",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "ascii7",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "default",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "646",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "iso_646.irv:1983",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "iso969-us",
+ .canon = "ansi_x3.4-1986",
+ .flags = RSPAMD_CHARSET_FLAG_ASCII,
+ },
+ {
+ .input = "tw-big5",
+ .canon = "big5",
+ .flags = 0,
+ },
+ {
+ .input = "csbig5",
+ .canon = "big5",
+ .flags = 0,
+ },
+ {
+ .input = "hkscs-big5",
+ .canon = "big5-hkscs",
+ .flags = 0,
+ },
+ {
+ .input = "big5hk",
+ .canon = "big5-hkscs",
+ .flags = 0,
+ },
+ {
+ .input = "big5-hkscs:unicode",
+ .canon = "big5-hkscs",
+ .flags = 0,
+ },
+ {
+ .input = "extended_unix_code_packed_format_for_japanese",
+ .canon = "euc-jp",
+ .flags = 0,
+ },
+ {
+ .input = "cseucpkdfmtjapanese",
+ .canon = "euc-jp",
+ .flags = 0,
+ },
+ {
+ .input = "x-eucjp",
+ .canon = "euc-jp",
+ .flags = 0,
+ },
+ {
+ .input = "x-euc-jp",
+ .canon = "euc-jp",
+ .flags = 0,
+ },
+ {
+ .input = "unicode-1-1-utf-8",
+ .canon = "utf-8",
+ .flags = RSPAMD_CHARSET_FLAG_UTF,
+ },
+ {
+ .input = "cseuckr",
+ .canon = "euc-kr",
+ .flags = 0,
+ },
+ {
+ .input = "5601",
+ .canon = "euc-kr",
+ .flags = 0,
+ },
+ {
+ .input = "ksc-5601",
+ .canon = "euc-kr",
+ .flags = 0,
+ },
+ {
+ .input = "ksc-5601-1987",
+ .canon = "euc-kr",
+ .flags = 0,
+ },
+ {
+ .input = "ksc-5601_1987",
+ .canon = "euc-kr",
+ .flags = 0,
+ },
+ {
+ .input = "ksc5601",
+ .canon = "euc-kr",
+ .flags = 0,
+ },
+ {
+ .input = "cns11643",
+ .canon = "euc-tw",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-euctw",
+ .canon = "euc-tw",
+ .flags = 0,
+ },
+ {
+ .input = "gb-18030",
+ .canon = "gb18030",
+ .flags = 0,
+ },
+ {
+ .input = "ibm1392",
+ .canon = "gb18030",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-1392",
+ .canon = "gb18030",
+ .flags = 0,
+ },
+ {
+ .input = "gb18030-2000",
+ .canon = "gb18030",
+ .flags = 0,
+ },
+ {
+ .input = "gb-2312",
+ .canon = "gb2312",
+ .flags = 0,
+ },
+ {
+ .input = "csgb2312",
+ .canon = "gb2312",
+ .flags = 0,
+ },
+ {
+ .input = "euc_cn",
+ .canon = "gb2312",
+ .flags = 0,
+ },
+ {
+ .input = "euccn",
+ .canon = "gb2312",
+ .flags = 0,
+ },
+ {
+ .input = "euc-cn",
+ .canon = "gb2312",
+ .flags = 0,
+ },
+ {
+ .input = "gb-k",
+ .canon = "gbk",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-1:1987",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "iso-ir-100",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-1",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "latin1",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "l1",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "ibm819",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "cp819",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "csisolatin1",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "819",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "cp819",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859-1",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "8859-1",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859_1",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859_1",
+ .canon = "iso-8859-1",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-2:1987",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "iso-ir-101",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-2",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "latin2",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "l2",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "csisolatin2",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "912",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "cp912",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-912",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "ibm912",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859-2",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "8859-2",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859_2",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859_2",
+ .canon = "iso-8859-2",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-3:1988",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "iso-ir-109",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-3",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "latin3",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "l3",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "csisolatin3",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "913",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "cp913",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-913",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "ibm913",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859-3",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "8859-3",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859_3",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859_3",
+ .canon = "iso-8859-3",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-4:1988",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "iso-ir-110",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-4",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "latin4",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "l4",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "csisolatin4",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "914",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "cp914",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-914",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "ibm914",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859-4",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "8859-4",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859_4",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859_4",
+ .canon = "iso-8859-4",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-5:1988",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "iso-ir-144",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-5",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "cyrillic",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "csisolatincyrillic",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "915",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "cp915",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-915",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "ibm915",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859-5",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "8859-5",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859_5",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859_5",
+ .canon = "iso-8859-5",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-6:1987",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "iso-ir-127",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-6",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "ecma-114",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "asmo-708",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "arabic",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "csisolatinarabic",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "1089",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "cp1089",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-1089",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "ibm1089",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859-6",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "8859-6",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859_6",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859_6",
+ .canon = "iso-8859-6",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-7:1987",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "iso-ir-126",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-7",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "elot_928",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "ecma-118",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "greek",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "greek8",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "csisolatingreek",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "813",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "cp813",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-813",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "ibm813",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859-7",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "8859-7",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859_7",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859_7",
+ .canon = "iso-8859-7",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-8:1988",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "iso-ir-138",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-8",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "hebrew",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "csisolatinhebrew",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "916",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "cp916",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-916",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "ibm916",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859-8",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "8859-8",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859_8",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859_8",
+ .canon = "iso-8859-8",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-9:1989",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "iso-ir-148",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-9",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "latin5",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "l5",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "csisolatin5",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "920",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "cp920",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-920",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "ibm920",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859-9",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "8859-9",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859_9",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859_9",
+ .canon = "iso-8859-9",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-13",
+ .canon = "iso-8859-13",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859-13",
+ .canon = "iso-8859-13",
+ .flags = 0,
+ },
+ {
+ .input = "8859-13",
+ .canon = "iso-8859-13",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859_13",
+ .canon = "iso-8859-13",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859_13",
+ .canon = "iso-8859-13",
+ .flags = 0,
+ },
+ {
+ .input = "iso-ir-199",
+ .canon = "iso-8859-14",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-14:1998",
+ .canon = "iso-8859-14",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-14",
+ .canon = "iso-8859-14",
+ .flags = 0,
+ },
+ {
+ .input = "latin8",
+ .canon = "iso-8859-14",
+ .flags = 0,
+ },
+ {
+ .input = "iso-celtic",
+ .canon = "iso-8859-14",
+ .flags = 0,
+ },
+ {
+ .input = "l8",
+ .canon = "iso-8859-14",
+ .flags = 0,
+ },
+ {
+ .input = "csisolatin9",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "csisolatin0",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "latin9",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "latin0",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "923",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "cp923",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-923",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "ibm923",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "iso8859-15",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-15",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "8859-15",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "iso_8859-15_fdis",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "l9",
+ .canon = "iso-8859-15",
+ .flags = 0,
+ },
+ {
+ .input = "koi-8-r",
+ .canon = "koi8-r",
+ .flags = 0,
+ },
+ {
+ .input = "cskoi8r",
+ .canon = "koi8-r",
+ .flags = 0,
+ },
+ {
+ .input = "koi8",
+ .canon = "koi8-r",
+ .flags = 0,
+ },
+ {
+ .input = "koi-8-u",
+ .canon = "koi8-u",
+ .flags = 0,
+ },
+ {
+ .input = "koi-8-t",
+ .canon = "koi8-t",
+ .flags = 0,
+ },
+ {
+ .input = "shiftjis",
+ .canon = "shift_jis",
+ .flags = 0,
+ },
+ {
+ .input = "ms_kanji",
+ .canon = "shift_jis",
+ .flags = 0,
+ },
+ {
+ .input = "csshiftjis",
+ .canon = "shift_jis",
+ .flags = 0,
+ },
+ {
+ .input = "cp-437",
+ .canon = "ibm437",
+ .flags = 0,
+ },
+ {
+ .input = "cp437",
+ .canon = "ibm437",
+ .flags = 0,
+ },
+ {
+ .input = "437",
+ .canon = "ibm437",
+ .flags = 0,
+ },
+ {
+ .input = "cspc8codepage437437",
+ .canon = "ibm437",
+ .flags = 0,
+ },
+ {
+ .input = "cspc8codepage437",
+ .canon = "ibm437",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-437",
+ .canon = "ibm437",
+ .flags = 0,
+ },
+ {
+ .input = "cp-850",
+ .canon = "ibm850",
+ .flags = 0,
+ },
+ {
+ .input = "cp850",
+ .canon = "ibm850",
+ .flags = 0,
+ },
+ {
+ .input = "850",
+ .canon = "ibm850",
+ .flags = 0,
+ },
+ {
+ .input = "cspc850multilingual850",
+ .canon = "ibm850",
+ .flags = 0,
+ },
+ {
+ .input = "cspc850multilingual",
+ .canon = "ibm850",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-850",
+ .canon = "ibm850",
+ .flags = 0,
+ },
+ {
+ .input = "cp-851",
+ .canon = "ibm851",
+ .flags = 0,
+ },
+ {
+ .input = "cp851",
+ .canon = "ibm851",
+ .flags = 0,
+ },
+ {
+ .input = "851",
+ .canon = "ibm851",
+ .flags = 0,
+ },
+ {
+ .input = "csibm851",
+ .canon = "ibm851",
+ .flags = 0,
+ },
+ {
+ .input = "cp-852",
+ .canon = "ibm852",
+ .flags = 0,
+ },
+ {
+ .input = "cp852",
+ .canon = "ibm852",
+ .flags = 0,
+ },
+ {
+ .input = "852",
+ .canon = "ibm852",
+ .flags = 0,
+ },
+ {
+ .input = "cspcp852",
+ .canon = "ibm852",
+ .flags = 0,
+ },
+ {
+ .input = "852",
+ .canon = "ibm852",
+ .flags = 0,
+ },
+ {
+ .input = "cspcp852",
+ .canon = "ibm852",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-852",
+ .canon = "ibm852",
+ .flags = 0,
+ },
+ {
+ .input = "cp-855",
+ .canon = "ibm855",
+ .flags = 0,
+ },
+ {
+ .input = "cp855",
+ .canon = "ibm855",
+ .flags = 0,
+ },
+ {
+ .input = "855",
+ .canon = "ibm855",
+ .flags = 0,
+ },
+ {
+ .input = "csibm855",
+ .canon = "ibm855",
+ .flags = 0,
+ },
+ {
+ .input = "cspcp855",
+ .canon = "ibm855",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-855",
+ .canon = "ibm855",
+ .flags = 0,
+ },
+ {
+ .input = "cp-857",
+ .canon = "ibm857",
+ .flags = 0,
+ },
+ {
+ .input = "cp857",
+ .canon = "ibm857",
+ .flags = 0,
+ },
+ {
+ .input = "857",
+ .canon = "ibm857",
+ .flags = 0,
+ },
+ {
+ .input = "csibm857",
+ .canon = "ibm857",
+ .flags = 0,
+ },
+ {
+ .input = "857",
+ .canon = "ibm857",
+ .flags = 0,
+ },
+ {
+ .input = "csibm857",
+ .canon = "ibm857",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-857",
+ .canon = "ibm857",
+ .flags = 0,
+ },
+ {
+ .input = "cp-860",
+ .canon = "ibm860",
+ .flags = 0,
+ },
+ {
+ .input = "cp860",
+ .canon = "ibm860",
+ .flags = 0,
+ },
+ {
+ .input = "860",
+ .canon = "ibm860",
+ .flags = 0,
+ },
+ {
+ .input = "csibm860",
+ .canon = "ibm860",
+ .flags = 0,
+ },
+ {
+ .input = "860",
+ .canon = "ibm860",
+ .flags = 0,
+ },
+ {
+ .input = "csibm860",
+ .canon = "ibm860",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-860",
+ .canon = "ibm860",
+ .flags = 0,
+ },
+ {
+ .input = "cp-861",
+ .canon = "ibm861",
+ .flags = 0,
+ },
+ {
+ .input = "cp861",
+ .canon = "ibm861",
+ .flags = 0,
+ },
+ {
+ .input = "861",
+ .canon = "ibm861",
+ .flags = 0,
+ },
+ {
+ .input = "cp-is",
+ .canon = "ibm861",
+ .flags = 0,
+ },
+ {
+ .input = "csibm861",
+ .canon = "ibm861",
+ .flags = 0,
+ },
+ {
+ .input = "861",
+ .canon = "ibm861",
+ .flags = 0,
+ },
+ {
+ .input = "cp-is",
+ .canon = "ibm861",
+ .flags = 0,
+ },
+ {
+ .input = "csibm861",
+ .canon = "ibm861",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-861",
+ .canon = "ibm861",
+ .flags = 0,
+ },
+ {
+ .input = "cp-862",
+ .canon = "ibm862",
+ .flags = 0,
+ },
+ {
+ .input = "cp862",
+ .canon = "ibm862",
+ .flags = 0,
+ },
+ {
+ .input = "862",
+ .canon = "ibm862",
+ .flags = 0,
+ },
+ {
+ .input = "cspc862latinhebrew862",
+ .canon = "ibm862",
+ .flags = 0,
+ },
+ {
+ .input = "cspc862latinhebrew",
+ .canon = "ibm862",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-862",
+ .canon = "ibm862",
+ .flags = 0,
+ },
+ {
+ .input = "cp-863",
+ .canon = "ibm863",
+ .flags = 0,
+ },
+ {
+ .input = "cp863",
+ .canon = "ibm863",
+ .flags = 0,
+ },
+ {
+ .input = "863",
+ .canon = "ibm863",
+ .flags = 0,
+ },
+ {
+ .input = "csibm863",
+ .canon = "ibm863",
+ .flags = 0,
+ },
+ {
+ .input = "863",
+ .canon = "ibm863",
+ .flags = 0,
+ },
+ {
+ .input = "csibm863",
+ .canon = "ibm863",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-863",
+ .canon = "ibm863",
+ .flags = 0,
+ },
+ {
+ .input = "cp-864",
+ .canon = "ibm864",
+ .flags = 0,
+ },
+ {
+ .input = "cp864",
+ .canon = "ibm864",
+ .flags = 0,
+ },
+ {
+ .input = "csibm864",
+ .canon = "ibm864",
+ .flags = 0,
+ },
+ {
+ .input = "csibm864",
+ .canon = "ibm864",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-864",
+ .canon = "ibm864",
+ .flags = 0,
+ },
+ {
+ .input = "cp-865",
+ .canon = "ibm865",
+ .flags = 0,
+ },
+ {
+ .input = "cp865",
+ .canon = "ibm865",
+ .flags = 0,
+ },
+ {
+ .input = "865",
+ .canon = "ibm865",
+ .flags = 0,
+ },
+ {
+ .input = "csibm865",
+ .canon = "ibm865",
+ .flags = 0,
+ },
+ {
+ .input = "865",
+ .canon = "ibm865",
+ .flags = 0,
+ },
+ {
+ .input = "csibm865",
+ .canon = "ibm865",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-865",
+ .canon = "ibm865",
+ .flags = 0,
+ },
+ {
+ .input = "cp-866",
+ .canon = "ibm866",
+ .flags = 0,
+ },
+ {
+ .input = "cp866",
+ .canon = "ibm866",
+ .flags = 0,
+ },
+ {
+ .input = "866",
+ .canon = "ibm866",
+ .flags = 0,
+ },
+ {
+ .input = "csibm866",
+ .canon = "ibm866",
+ .flags = 0,
+ },
+ {
+ .input = "866",
+ .canon = "ibm866",
+ .flags = 0,
+ },
+ {
+ .input = "csibm866",
+ .canon = "ibm866",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-866",
+ .canon = "ibm866",
+ .flags = 0,
+ },
+ {
+ .input = "cp-868",
+ .canon = "ibm868",
+ .flags = 0,
+ },
+ {
+ .input = "cp868",
+ .canon = "ibm868",
+ .flags = 0,
+ },
+ {
+ .input = "cp-ar",
+ .canon = "ibm868",
+ .flags = 0,
+ },
+ {
+ .input = "csibm868",
+ .canon = "ibm868",
+ .flags = 0,
+ },
+ {
+ .input = "ibm-868",
+ .canon = "ibm868",
+ .flags = 0,
+ },
+ {
+ .input = "cp-869",
+ .canon = "ibm869",
+ .flags = 0,
+ },
+ {
+ .input = "cp869",
+ .canon = "ibm869",
+ .flags = 0,
+ },
+ {
+ .input = "869",
+ .canon = "ibm869",
+ .flags = 0,
+ },
+ {
+ .input = "cp-gr",
+ .canon = "ibm869",
+ .flags = 0,
+ },
+ {
+ .input = "csibm869",
+ .canon = "ibm869",
+ .flags = 0,
+ },
+ {
+ .input = "cp-891",
+ .canon = "ibm891",
+ .flags = 0,
+ },
+ {
+ .input = "cp891",
+ .canon = "ibm891",
+ .flags = 0,
+ },
+ {
+ .input = "csibm891",
+ .canon = "ibm891",
+ .flags = 0,
+ },
+ {
+ .input = "cp-903",
+ .canon = "ibm903",
+ .flags = 0,
+ },
+ {
+ .input = "cp903",
+ .canon = "ibm903",
+ .flags = 0,
+ },
+ {
+ .input = "csibm903",
+ .canon = "ibm903",
+ .flags = 0,
+ },
+ {
+ .input = "cp-904",
+ .canon = "ibm904",
+ .flags = 0,
+ },
+ {
+ .input = "cp904",
+ .canon = "ibm904",
+ .flags = 0,
+ },
+ {
+ .input = "904",
+ .canon = "ibm904",
+ .flags = 0,
+ },
+ {
+ .input = "csibm904",
+ .canon = "ibm904",
+ .flags = 0,
+ },
+ {
+ .input = "cp-1251",
+ .canon = "cp1251",
+ .flags = 0,
+ },
+ {
+ .input = "windows-1251",
+ .canon = "cp1251",
+ .flags = 0,
+ },
+ {
+ .input = "cp-1255",
+ .canon = "cp1255",
+ .flags = 0,
+ },
+ {
+ .input = "windows-1255",
+ .canon = "cp1255",
+ .flags = 0,
+ },
+ {
+ .input = "tis620.2533",
+ .canon = "tis-620",
+ .flags = 0,
+ },
+};
+
+#endif /* SRC_LIBMIME_MIME_ENCODING_LIST_H_ */
diff --git a/src/libmime/mime_expressions.c b/src/libmime/mime_expressions.c
new file mode 100644
index 0000000..e51539e
--- /dev/null
+++ b/src/libmime/mime_expressions.c
@@ -0,0 +1,2392 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <contrib/libucl/ucl.h>
+#include "config.h"
+#include "util.h"
+#include "cfg_file.h"
+#include "rspamd.h"
+#include "message.h"
+#include "mime_expressions.h"
+#include "libserver/html/html.h"
+#include "lua/lua_common.h"
+#include "utlist.h"
+
+gboolean rspamd_compare_encoding(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+gboolean rspamd_header_exists(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+gboolean rspamd_parts_distance(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+gboolean rspamd_recipients_distance(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+gboolean rspamd_has_only_html_part(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+gboolean rspamd_is_recipients_sorted(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+gboolean rspamd_compare_transfer_encoding(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+gboolean rspamd_is_html_balanced(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+gboolean rspamd_has_html_tag(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+gboolean rspamd_has_fake_html(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+static gboolean rspamd_raw_header_exists(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+static gboolean rspamd_check_smtp_data(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+static gboolean rspamd_content_type_is_type(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+static gboolean rspamd_content_type_is_subtype(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+static gboolean rspamd_content_type_has_param(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+static gboolean rspamd_content_type_compare_param(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+static gboolean rspamd_has_content_part(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+static gboolean rspamd_has_content_part_len(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+static gboolean rspamd_is_empty_body(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+static gboolean rspamd_has_flag_expr(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+static gboolean rspamd_has_symbol_expr(struct rspamd_task *task,
+ GArray *args,
+ void *unused);
+
+static rspamd_expression_atom_t *rspamd_mime_expr_parse(const gchar *line, gsize len,
+ rspamd_mempool_t *pool, gpointer ud, GError **err);
+static gdouble rspamd_mime_expr_process(void *ud, rspamd_expression_atom_t *atom);
+static gint rspamd_mime_expr_priority(rspamd_expression_atom_t *atom);
+static void rspamd_mime_expr_destroy(rspamd_expression_atom_t *atom);
+
+/**
+ * Regexp structure
+ */
+struct rspamd_regexp_atom {
+ enum rspamd_re_type type; /**< regexp type */
+ gchar *regexp_text; /**< regexp text representation */
+ rspamd_regexp_t *regexp; /**< regexp structure */
+ union {
+ const gchar *header; /**< header name for header regexps */
+ const gchar *selector; /**< selector name for lua selector regexp */
+ } extra;
+ gboolean is_test; /**< true if this expression must be tested */
+ gboolean is_strong; /**< true if headers search must be case sensitive */
+ gboolean is_multiple; /**< true if we need to match all inclusions of atom */
+};
+
+/**
+ * Rspamd expression function
+ */
+struct rspamd_function_atom {
+ gchar *name; /**< name of function */
+ GArray *args; /**< its args */
+};
+
+enum rspamd_mime_atom_type {
+ MIME_ATOM_REGEXP = 0,
+ MIME_ATOM_INTERNAL_FUNCTION,
+ MIME_ATOM_LUA_FUNCTION,
+ MIME_ATOM_LOCAL_LUA_FUNCTION, /* New style */
+};
+
+struct rspamd_mime_atom {
+ gchar *str;
+ union {
+ struct rspamd_regexp_atom *re;
+ struct rspamd_function_atom *func;
+ const gchar *lua_function;
+ gint lua_cbref;
+ } d;
+ enum rspamd_mime_atom_type type;
+};
+
+/*
+ * List of internal functions of rspamd
+ * Sorted by name to use bsearch
+ */
+static struct _fl {
+ const gchar *name;
+ rspamd_internal_func_t func;
+ void *user_data;
+} rspamd_functions_list[] = {
+ {"check_smtp_data", rspamd_check_smtp_data, NULL},
+ {"compare_encoding", rspamd_compare_encoding, NULL},
+ {"compare_parts_distance", rspamd_parts_distance, NULL},
+ {"compare_recipients_distance", rspamd_recipients_distance, NULL},
+ {"compare_transfer_encoding", rspamd_compare_transfer_encoding, NULL},
+ {"content_type_compare_param", rspamd_content_type_compare_param, NULL},
+ {"content_type_has_param", rspamd_content_type_has_param, NULL},
+ {"content_type_is_subtype", rspamd_content_type_is_subtype, NULL},
+ {"content_type_is_type", rspamd_content_type_is_type, NULL},
+ {"has_content_part", rspamd_has_content_part, NULL},
+ {"has_content_part_len", rspamd_has_content_part_len, NULL},
+ {"has_fake_html", rspamd_has_fake_html, NULL},
+ {"has_flag", rspamd_has_flag_expr, NULL},
+ {"has_html_tag", rspamd_has_html_tag, NULL},
+ {"has_only_html_part", rspamd_has_only_html_part, NULL},
+ {"has_symbol", rspamd_has_symbol_expr, NULL},
+ {"header_exists", rspamd_header_exists, NULL},
+ {"is_empty_body", rspamd_is_empty_body, NULL},
+ {"is_html_balanced", rspamd_is_html_balanced, NULL},
+ {"is_recipients_sorted", rspamd_is_recipients_sorted, NULL},
+ {"raw_header_exists", rspamd_raw_header_exists, NULL},
+};
+
+const struct rspamd_atom_subr mime_expr_subr = {
+ .parse = rspamd_mime_expr_parse,
+ .process = rspamd_mime_expr_process,
+ .priority = rspamd_mime_expr_priority,
+ .destroy = rspamd_mime_expr_destroy};
+
+static struct _fl *list_ptr = &rspamd_functions_list[0];
+static guint32 functions_number = sizeof(rspamd_functions_list) /
+ sizeof(struct _fl);
+static gboolean list_allocated = FALSE;
+
+/* Bsearch routine */
+static gint
+fl_cmp(const void *s1, const void *s2)
+{
+ struct _fl *fl1 = (struct _fl *) s1;
+ struct _fl *fl2 = (struct _fl *) s2;
+ return strcmp(fl1->name, fl2->name);
+}
+
+static GQuark
+rspamd_mime_expr_quark(void)
+{
+ return g_quark_from_static_string("mime-expressions");
+}
+
+#define TYPE_CHECK(str, type, len) (sizeof(type) - 1 == (len) && rspamd_lc_cmp((str), (type), (len)) == 0)
+static gboolean
+rspamd_parse_long_option(const gchar *start, gsize len,
+ struct rspamd_regexp_atom *a)
+{
+ gboolean ret = FALSE;
+
+ if (TYPE_CHECK(start, "body", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_BODY;
+ }
+ else if (TYPE_CHECK(start, "part", len) ||
+ TYPE_CHECK(start, "mime", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_MIME;
+ }
+ else if (TYPE_CHECK(start, "raw_part", len) ||
+ TYPE_CHECK(start, "raw_mime", len) ||
+ TYPE_CHECK(start, "mime_raw", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_RAWMIME;
+ }
+ else if (TYPE_CHECK(start, "header", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_HEADER;
+ }
+ else if (TYPE_CHECK(start, "mime_header", len) ||
+ TYPE_CHECK(start, "header_mime", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_MIMEHEADER;
+ }
+ else if (TYPE_CHECK(start, "raw_header", len) ||
+ TYPE_CHECK(start, "header_raw", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_RAWHEADER;
+ }
+ else if (TYPE_CHECK(start, "all_header", len) ||
+ TYPE_CHECK(start, "header_all", len) ||
+ TYPE_CHECK(start, "all_headers", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_ALLHEADER;
+ }
+ else if (TYPE_CHECK(start, "url", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_URL;
+ }
+ else if (TYPE_CHECK(start, "email", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_EMAIL;
+ }
+ else if (TYPE_CHECK(start, "sa_body", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_SABODY;
+ }
+ else if (TYPE_CHECK(start, "sa_raw_body", len) ||
+ TYPE_CHECK(start, "sa_body_raw", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_SARAWBODY;
+ }
+ else if (TYPE_CHECK(start, "words", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_WORDS;
+ }
+ else if (TYPE_CHECK(start, "raw_words", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_RAWWORDS;
+ }
+ else if (TYPE_CHECK(start, "stem_words", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_STEMWORDS;
+ }
+ else if (TYPE_CHECK(start, "selector", len)) {
+ ret = TRUE;
+ a->type = RSPAMD_RE_SELECTOR;
+ }
+
+ return ret;
+}
+
+/*
+ * Rspamd regexp utility functions
+ */
+static struct rspamd_regexp_atom *
+rspamd_mime_expr_parse_regexp_atom(rspamd_mempool_t *pool, const gchar *line,
+ struct rspamd_config *cfg)
+{
+ const gchar *begin, *end, *p, *src, *start, *brace;
+ gchar *dbegin, *dend, *extra = NULL;
+ struct rspamd_regexp_atom *result;
+ GError *err = NULL;
+ GString *re_flags;
+
+ if (line == NULL) {
+ msg_err_pool("cannot parse NULL line");
+ return NULL;
+ }
+
+ src = line;
+ result = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_regexp_atom));
+ /* Skip whitespaces */
+ while (g_ascii_isspace(*line)) {
+ line++;
+ }
+ if (*line == '\0') {
+ msg_warn_pool("got empty regexp");
+ return NULL;
+ }
+
+ result->type = RSPAMD_RE_MAX;
+
+ start = line;
+ /* First try to find header name */
+ begin = strchr(line, '/');
+ if (begin != NULL) {
+ p = begin;
+ end = NULL;
+ while (p != line) {
+ if (*p == '=') {
+ end = p;
+ break;
+ }
+ p--;
+ }
+
+ if (end) {
+ extra = rspamd_mempool_alloc(pool, end - line + 1);
+ rspamd_strlcpy(extra, line, end - line + 1);
+ line = end;
+ }
+ }
+ else {
+ extra = rspamd_mempool_strdup(pool, line);
+ result->type = RSPAMD_RE_MAX;
+ line = start;
+ }
+ /* Find begin of regexp */
+ while (*line && *line != '/') {
+ line++;
+ }
+ if (*line != '\0') {
+ begin = line + 1;
+ }
+ else if (extra == NULL) {
+ /* Assume that line without // is just a header name */
+ extra = rspamd_mempool_strdup(pool, line);
+ result->type = RSPAMD_RE_HEADER;
+ return result;
+ }
+ else {
+ /* We got header name earlier but have not found // expression, so it is invalid regexp */
+ msg_warn_pool(
+ "got no header name (eg. header=) but without corresponding regexp, %s",
+ src);
+ return NULL;
+ }
+ /* Find end */
+ end = begin;
+ while (*end && (*end != '/' || *(end - 1) == '\\')) {
+ end++;
+ }
+ if (end == begin || *end != '/') {
+ msg_warn_pool("no trailing / in regexp %s", src);
+ return NULL;
+ }
+ /* Parse flags */
+ p = end + 1;
+ re_flags = g_string_sized_new(32);
+
+ while (p != NULL) {
+ switch (*p) {
+ case 'i':
+ case 'm':
+ case 's':
+ case 'x':
+ case 'u':
+ case 'O':
+ case 'r':
+ case 'L':
+ /* Handled by rspamd_regexp_t */
+ g_string_append_c(re_flags, *p);
+ p++;
+ break;
+ case 'o':
+ p++;
+ break;
+ /* Type flags */
+ case 'H':
+ result->type = RSPAMD_RE_HEADER;
+ p++;
+ break;
+ case 'R':
+ result->type = RSPAMD_RE_ALLHEADER;
+ p++;
+ break;
+ case 'B':
+ result->type = RSPAMD_RE_MIMEHEADER;
+ p++;
+ break;
+ case 'C':
+ result->type = RSPAMD_RE_SABODY;
+ p++;
+ break;
+ case 'D':
+ result->type = RSPAMD_RE_SARAWBODY;
+ p++;
+ break;
+ case 'M':
+ result->type = RSPAMD_RE_BODY;
+ p++;
+ break;
+ case 'P':
+ result->type = RSPAMD_RE_MIME;
+ p++;
+ break;
+ case 'Q':
+ result->type = RSPAMD_RE_RAWMIME;
+ p++;
+ break;
+ case 'U':
+ result->type = RSPAMD_RE_URL;
+ p++;
+ break;
+ case 'X':
+ result->type = RSPAMD_RE_RAWHEADER;
+ p++;
+ break;
+ case '$':
+ result->type = RSPAMD_RE_SELECTOR;
+ p++;
+ break;
+ case '{':
+ /* Long definition */
+ if ((brace = strchr(p + 1, '}')) != NULL) {
+ if (!rspamd_parse_long_option(p + 1, brace - (p + 1), result)) {
+ msg_warn_pool("invalid long regexp type: %*s in '%s'",
+ (int) (brace - (p + 1)), p + 1, src);
+ p = NULL;
+ }
+ else {
+ p = brace + 1;
+ }
+ }
+ else {
+ p = NULL;
+ }
+ break;
+ /* Other flags */
+ case 'T':
+ result->is_test = TRUE;
+ p++;
+ break;
+ case 'S':
+ result->is_strong = TRUE;
+ p++;
+ break;
+ case 'A':
+ result->is_multiple = TRUE;
+ p++;
+ break;
+ /* Stop flags parsing */
+ default:
+ p = NULL;
+ break;
+ }
+ }
+
+ if (result->type >= RSPAMD_RE_MAX) {
+ if (extra) {
+ /* Assume header regexp */
+ result->extra.header = extra;
+ result->type = RSPAMD_RE_HEADER;
+ }
+ else {
+ msg_err_pool("could not read regexp: %s, unknown type", src);
+ return NULL;
+ }
+ }
+
+ if ((result->type == RSPAMD_RE_HEADER ||
+ result->type == RSPAMD_RE_RAWHEADER ||
+ result->type == RSPAMD_RE_MIMEHEADER)) {
+ if (extra == NULL) {
+ msg_err_pool("header regexp: '%s' has no header part", src);
+ return NULL;
+ }
+ else {
+ result->extra.header = extra;
+ }
+ }
+
+ if (result->type == RSPAMD_RE_SELECTOR) {
+ if (extra == NULL) {
+ msg_err_pool("selector regexp: '%s' has no selector part", src);
+ return NULL;
+ }
+ else {
+ result->extra.selector = extra;
+ }
+ }
+
+
+ result->regexp_text = rspamd_mempool_strdup(pool, start);
+ dbegin = result->regexp_text + (begin - start);
+ dend = result->regexp_text + (end - start);
+ *dend = '\0';
+
+ result->regexp = rspamd_regexp_new(dbegin, re_flags->str,
+ &err);
+
+ g_string_free(re_flags, TRUE);
+
+ if (result->regexp == NULL || err != NULL) {
+ msg_warn_pool("could not read regexp: %s while reading regexp %e",
+ src, err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return NULL;
+ }
+
+ if (result->is_multiple) {
+ rspamd_regexp_set_maxhits(result->regexp, 0);
+ }
+ else {
+ rspamd_regexp_set_maxhits(result->regexp, 1);
+ }
+
+ rspamd_regexp_set_ud(result->regexp, result);
+
+ *dend = '/';
+
+ return result;
+}
+
+struct rspamd_function_atom *
+rspamd_mime_expr_parse_function_atom(rspamd_mempool_t *pool, const gchar *input)
+{
+ const gchar *obrace, *ebrace, *p, *c;
+ gchar t, *databuf;
+ guint len;
+ struct rspamd_function_atom *res;
+ struct expression_argument arg;
+ GError *err = NULL;
+ enum {
+ start_read_argument = 0,
+ in_string,
+ in_regexp,
+ got_backslash,
+ got_comma
+ } state,
+ prev_state = 0;
+
+ obrace = strchr(input, '(');
+ ebrace = strrchr(input, ')');
+
+ g_assert(obrace != NULL && ebrace != NULL);
+
+ res = rspamd_mempool_alloc0(pool, sizeof(*res));
+ res->name = rspamd_mempool_alloc(pool, obrace - input + 1);
+ rspamd_strlcpy(res->name, input, obrace - input + 1);
+ res->args = g_array_new(FALSE, FALSE, sizeof(struct expression_argument));
+
+ p = obrace + 1;
+ c = p;
+ state = start_read_argument;
+
+ /* Read arguments */
+ while (p <= ebrace) {
+ t = *p;
+ switch (state) {
+ case start_read_argument:
+ if (t == '/') {
+ state = in_regexp;
+ c = p;
+ }
+ else if (!g_ascii_isspace(t)) {
+ state = in_string;
+
+ if (t == '\'' || t == '\"') {
+ c = p + 1;
+ }
+ else {
+ c = p;
+ }
+ }
+ p++;
+ break;
+ case in_regexp:
+ if (t == '\\') {
+ state = got_backslash;
+ prev_state = in_regexp;
+ }
+ else if (t == ',' || p == ebrace) {
+ len = p - c + 1;
+ databuf = rspamd_mempool_alloc(pool, len);
+ rspamd_strlcpy(databuf, c, len);
+ arg.type = EXPRESSION_ARGUMENT_REGEXP;
+ arg.data = rspamd_regexp_cache_create(NULL, databuf, NULL, &err);
+
+ if (arg.data == NULL) {
+ /* Fallback to string */
+ msg_warn("cannot parse slashed argument %s as regexp: %s",
+ databuf, err->message);
+ g_error_free(err);
+ arg.type = EXPRESSION_ARGUMENT_NORMAL;
+ arg.data = databuf;
+ }
+
+ g_array_append_val(res->args, arg);
+ state = got_comma;
+ }
+ p++;
+ break;
+ case in_string:
+ if (t == '\\') {
+ state = got_backslash;
+ prev_state = in_string;
+ }
+ else if (t == ',' || p == ebrace) {
+ if (*(p - 1) == '\'' || *(p - 1) == '\"') {
+ len = p - c;
+ }
+ else {
+ len = p - c + 1;
+ }
+
+ databuf = rspamd_mempool_alloc(pool, len);
+ rspamd_strlcpy(databuf, c, len);
+ arg.type = EXPRESSION_ARGUMENT_NORMAL;
+ arg.data = databuf;
+ g_array_append_val(res->args, arg);
+ state = got_comma;
+ }
+ p++;
+ break;
+ case got_backslash:
+ state = prev_state;
+ p++;
+ break;
+ case got_comma:
+ state = start_read_argument;
+ break;
+ }
+ }
+
+ return res;
+}
+
+static rspamd_expression_atom_t *
+rspamd_mime_expr_parse(const gchar *line, gsize len,
+ rspamd_mempool_t *pool, gpointer ud, GError **err)
+{
+ rspamd_expression_atom_t *a = NULL;
+ struct rspamd_mime_atom *mime_atom = NULL;
+ const gchar *p, *end, *c = NULL;
+ struct rspamd_mime_expr_ud *real_ud = (struct rspamd_mime_expr_ud *) ud;
+ struct rspamd_config *cfg;
+ rspamd_regexp_t *own_re;
+ gchar t;
+ gint type = MIME_ATOM_REGEXP, obraces = 0, ebraces = 0;
+ enum {
+ in_header = 0,
+ got_slash,
+ in_regexp,
+ got_backslash,
+ got_second_slash,
+ in_flags,
+ in_flags_brace,
+ got_obrace,
+ in_function,
+ in_local_function,
+ got_ebrace,
+ end_atom,
+ bad_atom
+ } state = 0,
+ prev_state = 0;
+
+ p = line;
+ end = p + len;
+ cfg = real_ud->cfg;
+
+ while (p < end) {
+ t = *p;
+
+ switch (state) {
+ case in_header:
+ if (t == '/') {
+ /* Regexp */
+ state = got_slash;
+ }
+ else if (t == '(') {
+ /* Function */
+ state = got_obrace;
+ }
+ else if (!g_ascii_isalnum(t) && t != '_' && t != '-' && t != '=') {
+ if (t == ':') {
+ if (p - line == 3 && memcmp(line, "lua", 3) == 0) {
+ type = MIME_ATOM_LOCAL_LUA_FUNCTION;
+ state = in_local_function;
+ c = p + 1;
+ }
+ }
+ else {
+ /* Likely lua function, identified by just a string */
+ type = MIME_ATOM_LUA_FUNCTION;
+ state = end_atom;
+ /* Do not increase p */
+ continue;
+ }
+ }
+ else if (g_ascii_isspace(t)) {
+ state = bad_atom;
+ }
+ p++;
+ break;
+ case got_slash:
+ state = in_regexp;
+ break;
+ case in_regexp:
+ if (t == '\\') {
+ state = got_backslash;
+ prev_state = in_regexp;
+ }
+ else if (t == '/') {
+ state = got_second_slash;
+ }
+ p++;
+ break;
+ case got_second_slash:
+ state = in_flags;
+ break;
+ case in_flags:
+ if (t == '{') {
+ state = in_flags_brace;
+ p++;
+ }
+ else if (!g_ascii_isalpha(t) && t != '$') {
+ state = end_atom;
+ }
+ else {
+ p++;
+ }
+ break;
+ case in_flags_brace:
+ if (t == '}') {
+ state = in_flags;
+ }
+ p++;
+ break;
+ case got_backslash:
+ state = prev_state;
+ p++;
+ break;
+ case got_obrace:
+ state = in_function;
+ type = MIME_ATOM_INTERNAL_FUNCTION;
+ obraces++;
+ break;
+ case in_function:
+ if (t == '\\') {
+ state = got_backslash;
+ prev_state = in_function;
+ }
+ else if (t == '(') {
+ obraces++;
+ }
+ else if (t == ')') {
+ ebraces++;
+ if (ebraces == obraces) {
+ state = got_ebrace;
+ }
+ }
+ p++;
+ break;
+ case in_local_function:
+ if (!(g_ascii_isalnum(t) || t == '-' || t == '_')) {
+ g_assert(c != NULL);
+ state = end_atom;
+ }
+ else {
+ p++;
+ }
+ break;
+ case got_ebrace:
+ state = end_atom;
+ break;
+ case bad_atom:
+ g_set_error(err, rspamd_mime_expr_quark(), 100, "cannot parse"
+ " mime atom '%s' when reading symbol '%c' at offset %d, "
+ "near %.*s",
+ line, t, (gint) (p - line),
+ (gint) MIN(end - p, 10), p);
+ return NULL;
+ case end_atom:
+ goto set;
+ }
+ }
+set:
+
+ if (p - line == 0 || (state != got_ebrace && state != got_second_slash &&
+ state != in_flags && state != end_atom)) {
+ g_set_error(err, rspamd_mime_expr_quark(), 200, "incomplete or empty"
+ " mime atom");
+ return NULL;
+ }
+
+ mime_atom = rspamd_mempool_alloc(pool, sizeof(*mime_atom));
+ mime_atom->type = type;
+ mime_atom->str = rspamd_mempool_alloc(pool, p - line + 1);
+ rspamd_strlcpy(mime_atom->str, line, p - line + 1);
+
+ if (type == MIME_ATOM_REGEXP) {
+ mime_atom->d.re = rspamd_mime_expr_parse_regexp_atom(pool,
+ mime_atom->str, cfg);
+ if (mime_atom->d.re == NULL) {
+ g_set_error(err, rspamd_mime_expr_quark(), 200,
+ "cannot parse regexp '%s'",
+ mime_atom->str);
+ goto err;
+ }
+ else {
+ gint lua_cbref = -1;
+
+ /* Check regexp condition */
+ if (real_ud->conf_obj != NULL) {
+ const ucl_object_t *re_conditions = ucl_object_lookup(real_ud->conf_obj,
+ "re_conditions");
+
+ if (re_conditions != NULL) {
+ if (ucl_object_type(re_conditions) != UCL_OBJECT) {
+ g_set_error(err, rspamd_mime_expr_quark(), 320,
+ "re_conditions is not a table for '%s'",
+ mime_atom->str);
+ rspamd_regexp_unref(mime_atom->d.re->regexp);
+ goto err;
+ }
+
+ const ucl_object_t *function_obj = ucl_object_lookup(re_conditions,
+ mime_atom->str);
+
+ if (function_obj != NULL) {
+ if (ucl_object_type(function_obj) != UCL_USERDATA) {
+ g_set_error(err, rspamd_mime_expr_quark(), 320,
+ "condition for '%s' is invalid, must be function",
+ mime_atom->str);
+ rspamd_regexp_unref(mime_atom->d.re->regexp);
+ goto err;
+ }
+
+ struct ucl_lua_funcdata *fd = function_obj->value.ud;
+
+ lua_cbref = fd->idx;
+ }
+ }
+ }
+
+ if (lua_cbref != -1) {
+ msg_info_config("added condition for regexp %s", mime_atom->str);
+ /* Add SOM_LEFTMOST_FLAG implicitly */
+ rspamd_regexp_set_flags(mime_atom->d.re->regexp, rspamd_regexp_get_flags(mime_atom->d.re->regexp) |
+ RSPAMD_REGEXP_FLAG_LEFTMOST);
+ }
+
+ /* Register new item in the cache */
+ if (mime_atom->d.re->type == RSPAMD_RE_HEADER ||
+ mime_atom->d.re->type == RSPAMD_RE_RAWHEADER ||
+ mime_atom->d.re->type == RSPAMD_RE_MIMEHEADER) {
+
+ if (mime_atom->d.re->extra.header != NULL) {
+ own_re = mime_atom->d.re->regexp;
+ mime_atom->d.re->regexp = rspamd_re_cache_add(cfg->re_cache,
+ mime_atom->d.re->regexp,
+ mime_atom->d.re->type,
+ mime_atom->d.re->extra.header,
+ strlen(mime_atom->d.re->extra.header) + 1,
+ lua_cbref);
+ /* Pass ownership to the cache */
+ rspamd_regexp_unref(own_re);
+ }
+ else {
+ /* We have header regexp, but no header name is detected */
+ g_set_error(err,
+ rspamd_mime_expr_quark(),
+ 200,
+ "no header name in header regexp: '%s'",
+ mime_atom->str);
+ rspamd_regexp_unref(mime_atom->d.re->regexp);
+ goto err;
+ }
+ }
+ else if (mime_atom->d.re->type == RSPAMD_RE_SELECTOR) {
+ if (mime_atom->d.re->extra.selector != NULL) {
+ own_re = mime_atom->d.re->regexp;
+ mime_atom->d.re->regexp = rspamd_re_cache_add(cfg->re_cache,
+ mime_atom->d.re->regexp,
+ mime_atom->d.re->type,
+ mime_atom->d.re->extra.selector,
+ strlen(mime_atom->d.re->extra.selector) + 1,
+ lua_cbref);
+ /* Pass ownership to the cache */
+ rspamd_regexp_unref(own_re);
+ }
+ else {
+ /* We have selector regexp, but no selector name is detected */
+ g_set_error(err,
+ rspamd_mime_expr_quark(),
+ 200,
+ "no selector name in selector regexp: '%s'",
+ mime_atom->str);
+ rspamd_regexp_unref(mime_atom->d.re->regexp);
+ goto err;
+ }
+ }
+ else {
+ own_re = mime_atom->d.re->regexp;
+ mime_atom->d.re->regexp = rspamd_re_cache_add(cfg->re_cache,
+ mime_atom->d.re->regexp,
+ mime_atom->d.re->type,
+ NULL,
+ 0,
+ lua_cbref);
+ /* Pass ownership to the cache */
+ rspamd_regexp_unref(own_re);
+ }
+ }
+ }
+ else if (type == MIME_ATOM_LUA_FUNCTION) {
+ mime_atom->d.lua_function = mime_atom->str;
+
+ lua_getglobal(cfg->lua_state, mime_atom->str);
+
+ if (lua_type(cfg->lua_state, -1) != LUA_TFUNCTION) {
+ g_set_error(err, rspamd_mime_expr_quark(), 200,
+ "no such lua function '%s'",
+ mime_atom->str);
+ lua_pop(cfg->lua_state, 1);
+
+ goto err;
+ }
+
+ lua_pop(cfg->lua_state, 1);
+ }
+ else if (type == MIME_ATOM_LOCAL_LUA_FUNCTION) {
+ /* p pointer is set to the start of Lua function name */
+
+ if (real_ud->conf_obj == NULL) {
+ g_set_error(err, rspamd_mime_expr_quark(), 300,
+ "no config object for '%s'",
+ mime_atom->str);
+ goto err;
+ }
+
+ const ucl_object_t *functions = ucl_object_lookup(real_ud->conf_obj,
+ "functions");
+
+ if (functions == NULL) {
+ g_set_error(err, rspamd_mime_expr_quark(), 310,
+ "no functions defined for '%s'",
+ mime_atom->str);
+ goto err;
+ }
+
+ if (ucl_object_type(functions) != UCL_OBJECT) {
+ g_set_error(err, rspamd_mime_expr_quark(), 320,
+ "functions is not a table for '%s'",
+ mime_atom->str);
+ goto err;
+ }
+
+ const ucl_object_t *function_obj;
+
+ function_obj = ucl_object_lookup_len(functions, c,
+ p - c);
+
+ if (function_obj == NULL) {
+ g_set_error(err, rspamd_mime_expr_quark(), 320,
+ "function %.*s is not found for '%s'",
+ (int) (p - c), c, mime_atom->str);
+ goto err;
+ }
+
+ if (ucl_object_type(function_obj) != UCL_USERDATA) {
+ g_set_error(err, rspamd_mime_expr_quark(), 320,
+ "function %.*s has invalid type for '%s'",
+ (int) (p - c), c, mime_atom->str);
+ goto err;
+ }
+
+ struct ucl_lua_funcdata *fd = function_obj->value.ud;
+
+ mime_atom->d.lua_cbref = fd->idx;
+ }
+ else {
+ mime_atom->d.func = rspamd_mime_expr_parse_function_atom(pool,
+ mime_atom->str);
+ if (mime_atom->d.func == NULL) {
+ g_set_error(err, rspamd_mime_expr_quark(), 200,
+ "cannot parse function '%s'",
+ mime_atom->str);
+ goto err;
+ }
+ }
+
+ a = rspamd_mempool_alloc0(pool, sizeof(*a));
+ a->len = p - line;
+ a->priority = 0;
+ a->data = mime_atom;
+
+ return a;
+
+err:
+
+ return NULL;
+}
+
+static gint
+rspamd_mime_expr_process_regexp(struct rspamd_regexp_atom *re,
+ struct rspamd_task *task)
+{
+ gint ret;
+
+ if (re == NULL) {
+ msg_info_task("invalid regexp passed");
+ return 0;
+ }
+
+ if (re->type == RSPAMD_RE_HEADER || re->type == RSPAMD_RE_RAWHEADER) {
+ ret = rspamd_re_cache_process(task,
+ re->regexp,
+ re->type,
+ re->extra.header,
+ strlen(re->extra.header),
+ re->is_strong);
+ }
+ else if (re->type == RSPAMD_RE_SELECTOR) {
+ ret = rspamd_re_cache_process(task,
+ re->regexp,
+ re->type,
+ re->extra.selector,
+ strlen(re->extra.selector),
+ re->is_strong);
+ }
+ else {
+ ret = rspamd_re_cache_process(task,
+ re->regexp,
+ re->type,
+ NULL,
+ 0,
+ re->is_strong);
+ }
+
+ if (re->is_test) {
+ msg_info_task("test %s regexp '%s' returned %d",
+ rspamd_re_cache_type_to_string(re->type),
+ re->regexp_text, ret);
+ }
+
+ return ret;
+}
+
+
+static gint
+rspamd_mime_expr_priority(rspamd_expression_atom_t *atom)
+{
+ struct rspamd_mime_atom *mime_atom = atom->data;
+ gint ret = 0;
+
+ switch (mime_atom->type) {
+ case MIME_ATOM_INTERNAL_FUNCTION:
+ /* Prioritize internal functions slightly */
+ ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 8;
+ break;
+ case MIME_ATOM_LUA_FUNCTION:
+ case MIME_ATOM_LOCAL_LUA_FUNCTION:
+ ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 4;
+ break;
+ case MIME_ATOM_REGEXP:
+ switch (mime_atom->d.re->type) {
+ case RSPAMD_RE_HEADER:
+ case RSPAMD_RE_RAWHEADER:
+ ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 16;
+ break;
+ case RSPAMD_RE_URL:
+ case RSPAMD_RE_EMAIL:
+ ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 8;
+ break;
+ case RSPAMD_RE_SELECTOR:
+ ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 8;
+ break;
+ case RSPAMD_RE_MIME:
+ case RSPAMD_RE_RAWMIME:
+ ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 2;
+ break;
+ case RSPAMD_RE_WORDS:
+ case RSPAMD_RE_RAWWORDS:
+ case RSPAMD_RE_STEMWORDS:
+ default:
+ /* For expensive regexps */
+ ret = 0;
+ break;
+ }
+ }
+
+ return ret;
+}
+
+static void
+rspamd_mime_expr_destroy(rspamd_expression_atom_t *atom)
+{
+ struct rspamd_mime_atom *mime_atom = atom->data;
+
+ if (mime_atom) {
+ if (mime_atom->type == MIME_ATOM_INTERNAL_FUNCTION) {
+ /* Need to cleanup arguments */
+ g_array_free(mime_atom->d.func->args, TRUE);
+ }
+ }
+}
+
+static gboolean
+rspamd_mime_expr_process_function(struct rspamd_function_atom *func,
+ struct rspamd_task *task,
+ lua_State *L)
+{
+ struct _fl *selected, key;
+
+ key.name = func->name;
+
+ selected = bsearch(&key,
+ list_ptr,
+ functions_number,
+ sizeof(struct _fl),
+ fl_cmp);
+ if (selected == NULL) {
+ /* Try to check lua function */
+ return FALSE;
+ }
+
+ return selected->func(task, func->args, selected->user_data);
+}
+
+static gdouble
+rspamd_mime_expr_process(void *ud, rspamd_expression_atom_t *atom)
+{
+ struct rspamd_task *task = (struct rspamd_task *) ud;
+ struct rspamd_mime_atom *mime_atom;
+ lua_State *L;
+ gdouble ret = 0;
+
+ g_assert(task != NULL);
+ g_assert(atom != NULL);
+
+ mime_atom = atom->data;
+
+ if (mime_atom->type == MIME_ATOM_REGEXP) {
+ ret = rspamd_mime_expr_process_regexp(mime_atom->d.re, task);
+ }
+ else if (mime_atom->type == MIME_ATOM_LUA_FUNCTION) {
+ L = task->cfg->lua_state;
+ lua_getglobal(L, mime_atom->d.lua_function);
+ rspamd_lua_task_push(L, task);
+
+ if (lua_pcall(L, 1, 1, 0) != 0) {
+ msg_info_task("lua call to global function '%s' for atom '%s' failed: %s",
+ mime_atom->d.lua_function,
+ mime_atom->str,
+ lua_tostring(L, -1));
+ lua_pop(L, 1);
+ }
+ else {
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ ret = lua_toboolean(L, -1);
+ }
+ else if (lua_type(L, -1) == LUA_TNUMBER) {
+ ret = lua_tonumber(L, 1);
+ }
+ else {
+ msg_err_task("%s returned wrong return type: %s",
+ mime_atom->str, lua_typename(L, lua_type(L, -1)));
+ }
+ /* Remove result */
+ lua_pop(L, 1);
+ }
+ }
+ else if (mime_atom->type == MIME_ATOM_LOCAL_LUA_FUNCTION) {
+ gint err_idx;
+
+ L = task->cfg->lua_state;
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, mime_atom->d.lua_cbref);
+ rspamd_lua_task_push(L, task);
+
+ if (lua_pcall(L, 1, 1, err_idx) != 0) {
+ msg_info_task("lua call to local function for atom '%s' failed: %s",
+ mime_atom->str,
+ lua_tostring(L, -1));
+ }
+ else {
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ ret = lua_toboolean(L, -1);
+ }
+ else if (lua_type(L, -1) == LUA_TNUMBER) {
+ ret = lua_tonumber(L, 1);
+ }
+ else {
+ msg_err_task("%s returned wrong return type: %s",
+ mime_atom->str, lua_typename(L, lua_type(L, -1)));
+ }
+ }
+
+ lua_settop(L, 0);
+ }
+ else {
+ ret = rspamd_mime_expr_process_function(mime_atom->d.func, task,
+ task->cfg->lua_state);
+ }
+
+ return ret;
+}
+
+void register_expression_function(const gchar *name,
+ rspamd_internal_func_t func,
+ void *user_data)
+{
+ static struct _fl *new;
+
+ functions_number++;
+
+ new = g_new(struct _fl, functions_number);
+ memcpy(new, list_ptr, (functions_number - 1) * sizeof(struct _fl));
+ if (list_allocated) {
+ g_free(list_ptr);
+ }
+
+ list_allocated = TRUE;
+ new[functions_number - 1].name = name;
+ new[functions_number - 1].func = func;
+ new[functions_number - 1].user_data = user_data;
+ qsort(new, functions_number, sizeof(struct _fl), fl_cmp);
+ list_ptr = new;
+}
+
+gboolean
+rspamd_compare_encoding(struct rspamd_task *task, GArray *args, void *unused)
+{
+ struct expression_argument *arg;
+
+ if (args == NULL || task == NULL) {
+ return FALSE;
+ }
+
+ arg = &g_array_index(args, struct expression_argument, 0);
+ if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+ msg_warn_task("invalid argument to function is passed");
+ return FALSE;
+ }
+
+ /* XXX: really write this function */
+ return TRUE;
+}
+
+gboolean
+rspamd_header_exists(struct rspamd_task *task, GArray *args, void *unused)
+{
+ struct expression_argument *arg;
+ struct rspamd_mime_header *rh;
+
+ if (args == NULL || task == NULL) {
+ return FALSE;
+ }
+
+ arg = &g_array_index(args, struct expression_argument, 0);
+ if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+ msg_warn_task("invalid argument to function is passed");
+ return FALSE;
+ }
+
+ rh = rspamd_message_get_header_array(task,
+ (gchar *) arg->data, FALSE);
+
+ debug_task("try to get header %s: %d", (gchar *) arg->data,
+ (rh != NULL));
+
+ if (rh) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+
+/*
+ * This function is designed to find difference between text/html and text/plain parts
+ * It takes one argument: difference threshold, if we have two text parts, compare
+ * its hashes and check for threshold, if value is greater than threshold, return TRUE
+ * and return FALSE otherwise.
+ */
+gboolean
+rspamd_parts_distance(struct rspamd_task *task, GArray *args, void *unused)
+{
+ gint threshold, threshold2 = -1;
+ struct expression_argument *arg;
+ gdouble *pdiff, diff;
+
+ if (args == NULL || args->len == 0) {
+ debug_task("no threshold is specified, assume it 100");
+ threshold = 100;
+ }
+ else {
+ errno = 0;
+ arg = &g_array_index(args, struct expression_argument, 0);
+ if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+ msg_warn_task("invalid argument to function is passed");
+ return FALSE;
+ }
+
+ threshold = strtoul((gchar *) arg->data, NULL, 10);
+ if (errno != 0) {
+ msg_info_task("bad numeric value for threshold \"%s\", assume it 100",
+ (gchar *) arg->data);
+ threshold = 100;
+ }
+ if (args->len >= 2) {
+ arg = &g_array_index(args, struct expression_argument, 1);
+ if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+ msg_warn_task("invalid argument to function is passed");
+ return FALSE;
+ }
+
+ errno = 0;
+ threshold2 = strtoul((gchar *) arg->data, NULL, 10);
+ if (errno != 0) {
+ msg_info_task("bad numeric value for threshold \"%s\", ignore it",
+ (gchar *) arg->data);
+ threshold2 = -1;
+ }
+ }
+ }
+
+ if ((pdiff =
+ rspamd_mempool_get_variable(task->task_pool,
+ "parts_distance")) != NULL) {
+ diff = (1.0 - (*pdiff)) * 100.0;
+
+ if (diff != -1) {
+ if (threshold2 > 0) {
+ if (diff >= MIN(threshold, threshold2) &&
+ diff < MAX(threshold, threshold2)) {
+
+ return TRUE;
+ }
+ }
+ else {
+ if (diff <= threshold) {
+ return TRUE;
+ }
+ }
+ return FALSE;
+ }
+ else {
+ return FALSE;
+ }
+ }
+
+ return FALSE;
+}
+
+struct addr_list {
+ const gchar *name;
+ guint namelen;
+ const gchar *addr;
+ guint addrlen;
+};
+
+static gint
+addr_list_cmp_func(const void *a, const void *b)
+{
+ const struct addr_list *addra = (struct addr_list *) a,
+ *addrb = (struct addr_list *) b;
+
+ if (addra->addrlen != addrb->addrlen) {
+ return addra->addrlen - addrb->addrlen;
+ }
+
+ return memcmp(addra->addr, addrb->addr, addra->addrlen);
+}
+
+#define COMPARE_RCPT_LEN 3
+#define MIN_RCPT_TO_COMPARE 7
+
+gboolean
+rspamd_recipients_distance(struct rspamd_task *task, GArray *args,
+ void *unused)
+{
+ struct expression_argument *arg;
+ struct rspamd_email_address *cur;
+ double threshold;
+ struct addr_list *ar;
+ gint num, i, hits = 0;
+
+ if (args == NULL) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+
+ arg = &g_array_index(args, struct expression_argument, 0);
+ if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+ msg_warn_task("invalid argument to function is passed");
+ return FALSE;
+ }
+
+ errno = 0;
+ threshold = strtod((gchar *) arg->data, NULL);
+
+ if (errno != 0) {
+ msg_warn_task("invalid numeric value '%s': %s",
+ (gchar *) arg->data,
+ strerror(errno));
+ return FALSE;
+ }
+
+ if (!MESSAGE_FIELD(task, rcpt_mime)) {
+ return FALSE;
+ }
+
+ num = MESSAGE_FIELD(task, rcpt_mime)->len;
+
+ if (num < MIN_RCPT_TO_COMPARE) {
+ return FALSE;
+ }
+
+ ar = rspamd_mempool_alloc0(task->task_pool, num * sizeof(struct addr_list));
+
+ /* Fill array */
+ num = 0;
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, rcpt_mime), i, cur)
+ {
+ if (cur->addr_len > COMPARE_RCPT_LEN) {
+ ar[num].name = cur->addr;
+ ar[num].namelen = cur->addr_len;
+ ar[num].addr = cur->domain;
+ ar[num].addrlen = cur->domain_len;
+ num++;
+ }
+ }
+
+ qsort(ar, num, sizeof(*ar), addr_list_cmp_func);
+
+ /* Cycle all elements in array */
+ for (i = 0; i < num; i++) {
+ if (i < num - 1) {
+ if (ar[i].namelen == ar[i + 1].namelen) {
+ if (rspamd_lc_cmp(ar[i].name, ar[i + 1].name, COMPARE_RCPT_LEN) == 0) {
+ hits++;
+ }
+ }
+ }
+ }
+
+ if ((hits * num / 2.) / (double) num >= threshold) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_has_only_html_part(struct rspamd_task *task, GArray *args,
+ void *unused)
+{
+ struct rspamd_mime_text_part *p;
+ guint i, cnt_html = 0, cnt_txt = 0;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, p)
+ {
+ if (!IS_TEXT_PART_ATTACHMENT(p)) {
+ if (IS_TEXT_PART_HTML(p)) {
+ cnt_html++;
+ }
+ else {
+ cnt_txt++;
+ }
+ }
+ }
+
+ return (cnt_html > 0 && cnt_txt == 0);
+}
+
+static gboolean
+is_recipient_list_sorted(GPtrArray *ar)
+{
+ struct rspamd_email_address *addr;
+ gboolean res = TRUE;
+ rspamd_ftok_t cur, prev;
+ gint i;
+
+ /* Do not check to short address lists */
+ if (ar == NULL || ar->len < MIN_RCPT_TO_COMPARE) {
+ return FALSE;
+ }
+
+ prev.len = 0;
+ prev.begin = NULL;
+
+ PTR_ARRAY_FOREACH(ar, i, addr)
+ {
+ cur.begin = addr->addr;
+ cur.len = addr->addr_len;
+
+ if (prev.len != 0) {
+ if (rspamd_ftok_casecmp(&cur, &prev) <= 0) {
+ res = FALSE;
+ break;
+ }
+ }
+
+ prev = cur;
+ }
+
+ return res;
+}
+
+gboolean
+rspamd_is_recipients_sorted(struct rspamd_task *task,
+ GArray *args,
+ void *unused)
+{
+ /* Check all types of addresses */
+
+ if (MESSAGE_FIELD(task, rcpt_mime)) {
+ return is_recipient_list_sorted(MESSAGE_FIELD(task, rcpt_mime));
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_compare_transfer_encoding(struct rspamd_task *task,
+ GArray *args,
+ void *unused)
+{
+ struct expression_argument *arg;
+ guint i;
+ struct rspamd_mime_part *part;
+ enum rspamd_cte cte;
+
+ if (args == NULL) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+
+ arg = &g_array_index(args, struct expression_argument, 0);
+ if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+ msg_warn_task("invalid argument to function is passed");
+ return FALSE;
+ }
+
+ cte = rspamd_cte_from_string(arg->data);
+
+ if (cte == RSPAMD_CTE_UNKNOWN) {
+ msg_warn_task("unknown cte: %s", arg->data);
+ return FALSE;
+ }
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ if (IS_PART_TEXT(part)) {
+ if (part->cte == cte) {
+ return TRUE;
+ }
+ }
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_is_html_balanced(struct rspamd_task *task, GArray *args, void *unused)
+{
+ /* Totally broken but seems to be never used */
+ return TRUE;
+}
+
+gboolean
+rspamd_has_html_tag(struct rspamd_task *task, GArray *args, void *unused)
+{
+ struct rspamd_mime_text_part *p;
+ struct expression_argument *arg;
+ guint i;
+ gboolean res = FALSE;
+
+ if (args == NULL) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+
+ arg = &g_array_index(args, struct expression_argument, 0);
+ if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+ msg_warn_task("invalid argument to function is passed");
+ return FALSE;
+ }
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, p)
+ {
+ if (IS_TEXT_PART_HTML(p) && p->html) {
+ res = rspamd_html_tag_seen(p->html, arg->data);
+ }
+
+ if (res) {
+ break;
+ }
+ }
+
+ return res;
+}
+
+gboolean
+rspamd_has_fake_html(struct rspamd_task *task, GArray *args, void *unused)
+{
+ struct rspamd_mime_text_part *p;
+ guint i;
+ gboolean res = FALSE;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, p)
+ {
+ if (IS_TEXT_PART_HTML(p) && (rspamd_html_get_tags_count(p->html) < 2)) {
+ res = TRUE;
+ }
+
+ if (res) {
+ break;
+ }
+ }
+
+ return res;
+}
+
+static gboolean
+rspamd_raw_header_exists(struct rspamd_task *task, GArray *args, void *unused)
+{
+ struct expression_argument *arg;
+
+ if (args == NULL || task == NULL) {
+ return FALSE;
+ }
+
+ arg = &g_array_index(args, struct expression_argument, 0);
+ if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+ msg_warn_task("invalid argument to function is passed");
+ return FALSE;
+ }
+
+ return rspamd_message_get_header_array(task, arg->data, FALSE) != NULL;
+}
+
+static gboolean
+match_smtp_data(struct rspamd_task *task,
+ struct expression_argument *arg,
+ const gchar *what, gsize len)
+{
+ rspamd_regexp_t *re;
+ gint r = 0;
+
+ if (arg->type == EXPRESSION_ARGUMENT_REGEXP) {
+ /* This is a regexp */
+ re = arg->data;
+ if (re == NULL) {
+ msg_warn_task("cannot compile regexp for function");
+ return FALSE;
+ }
+
+
+ if (len > 0) {
+ r = rspamd_regexp_search(re, what, len, NULL, NULL, FALSE, NULL);
+ }
+
+ return r;
+ }
+ else if (arg->type == EXPRESSION_ARGUMENT_NORMAL &&
+ g_ascii_strncasecmp(arg->data, what, len) == 0) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_check_smtp_data(struct rspamd_task *task, GArray *args, void *unused)
+{
+ struct expression_argument *arg;
+ struct rspamd_email_address *addr = NULL;
+ GPtrArray *rcpts = NULL;
+ const gchar *type, *str = NULL;
+ guint i;
+
+ if (args == NULL) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+
+ arg = &g_array_index(args, struct expression_argument, 0);
+
+ if (!arg || !arg->data || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+ else {
+ type = arg->data;
+ switch (*type) {
+ case 'f':
+ case 'F':
+ if (g_ascii_strcasecmp(type, "from") == 0) {
+ addr = rspamd_task_get_sender(task);
+ }
+ else {
+ msg_warn_task("bad argument to function: %s", type);
+ return FALSE;
+ }
+ break;
+ case 'h':
+ case 'H':
+ if (g_ascii_strcasecmp(type, "helo") == 0) {
+ str = task->helo;
+ }
+ else {
+ msg_warn_task("bad argument to function: %s", type);
+ return FALSE;
+ }
+ break;
+ case 'u':
+ case 'U':
+ if (g_ascii_strcasecmp(type, "user") == 0) {
+ str = task->auth_user;
+ }
+ else {
+ msg_warn_task("bad argument to function: %s", type);
+ return FALSE;
+ }
+ break;
+ case 's':
+ case 'S':
+ if (g_ascii_strcasecmp(type, "subject") == 0) {
+ str = MESSAGE_FIELD(task, subject);
+ }
+ else {
+ msg_warn_task("bad argument to function: %s", type);
+ return FALSE;
+ }
+ break;
+ case 'r':
+ case 'R':
+ if (g_ascii_strcasecmp(type, "rcpt") == 0) {
+ rcpts = task->rcpt_envelope;
+ }
+ else {
+ msg_warn_task("bad argument to function: %s", type);
+ return FALSE;
+ }
+ break;
+ default:
+ msg_warn_task("bad argument to function: %s", type);
+ return FALSE;
+ }
+ }
+
+ if (str == NULL && addr == NULL && rcpts == NULL) {
+ /* Not enough data so regexp would NOT be found anyway */
+ return FALSE;
+ }
+
+ /* We would process only one more argument, others are ignored */
+ if (args->len >= 2) {
+ arg = &g_array_index(args, struct expression_argument, 1);
+
+ if (arg) {
+ if (str != NULL) {
+ return match_smtp_data(task, arg, str, strlen(str));
+ }
+ else if (addr != NULL && addr->addr) {
+ return match_smtp_data(task, arg, addr->addr, addr->addr_len);
+ }
+ else {
+ if (rcpts != NULL) {
+ for (i = 0; i < rcpts->len; i++) {
+ addr = g_ptr_array_index(rcpts, i);
+
+ if (addr && addr->addr &&
+ match_smtp_data(task, arg,
+ addr->addr, addr->addr_len)) {
+ return TRUE;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ return FALSE;
+}
+
+static inline gboolean
+rspamd_check_ct_attr(const gchar *begin, gsize len,
+ struct expression_argument *arg_pattern)
+{
+ rspamd_regexp_t *re;
+ gboolean r = FALSE;
+
+ if (arg_pattern->type == EXPRESSION_ARGUMENT_REGEXP) {
+ re = arg_pattern->data;
+
+ if (len > 0) {
+ r = rspamd_regexp_search(re,
+ begin, len,
+ NULL, NULL, FALSE, NULL);
+ }
+
+ if (r) {
+ return TRUE;
+ }
+ }
+ else {
+ /* Just do strcasecmp */
+ gsize plen = strlen(arg_pattern->data);
+
+ if (plen == len &&
+ g_ascii_strncasecmp(arg_pattern->data, begin, len) == 0) {
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_content_type_compare_param(struct rspamd_task *task,
+ GArray *args,
+ void *unused)
+{
+
+ struct expression_argument *arg, *arg1, *arg_pattern;
+ gboolean recursive = FALSE;
+ struct rspamd_mime_part *cur_part;
+ guint i;
+ rspamd_ftok_t srch;
+ struct rspamd_content_type_param *found = NULL, *cur;
+ const gchar *param_name;
+
+ if (args == NULL || args->len < 2) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+
+ arg = &g_array_index(args, struct expression_argument, 0);
+ g_assert(arg->type == EXPRESSION_ARGUMENT_NORMAL);
+ param_name = arg->data;
+ arg_pattern = &g_array_index(args, struct expression_argument, 1);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, cur_part)
+ {
+ if (args->len >= 3) {
+ arg1 = &g_array_index(args, struct expression_argument, 2);
+ if (g_ascii_strncasecmp(arg1->data, "true",
+ sizeof("true") - 1) == 0) {
+ recursive = TRUE;
+ }
+ }
+ else {
+ /*
+ * If user did not specify argument, let's assume that he wants
+ * recursive search if mime part is multipart/mixed
+ */
+ if (IS_PART_MULTIPART(cur_part)) {
+ recursive = TRUE;
+ }
+ }
+
+ rspamd_ftok_t lit;
+ RSPAMD_FTOK_FROM_STR(&srch, param_name);
+ RSPAMD_FTOK_FROM_STR(&lit, "charset");
+
+ if (rspamd_ftok_equal(&srch, &lit)) {
+ if (rspamd_check_ct_attr(cur_part->ct->charset.begin,
+ cur_part->ct->charset.len, arg_pattern)) {
+ return TRUE;
+ }
+ }
+
+ RSPAMD_FTOK_FROM_STR(&lit, "boundary");
+ if (rspamd_ftok_equal(&srch, &lit)) {
+ if (rspamd_check_ct_attr(cur_part->ct->orig_boundary.begin,
+ cur_part->ct->orig_boundary.len, arg_pattern)) {
+ return TRUE;
+ }
+ }
+
+ if (cur_part->ct->attrs) {
+ found = g_hash_table_lookup(cur_part->ct->attrs, &srch);
+
+ if (found) {
+ DL_FOREACH(found, cur)
+ {
+ if (rspamd_check_ct_attr(cur->value.begin,
+ cur->value.len, arg_pattern)) {
+ return TRUE;
+ }
+ }
+ }
+ }
+
+ if (!recursive) {
+ break;
+ }
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_content_type_has_param(struct rspamd_task *task,
+ GArray *args,
+ void *unused)
+{
+ struct expression_argument *arg, *arg1;
+ gboolean recursive = FALSE;
+ struct rspamd_mime_part *cur_part;
+ guint i;
+ rspamd_ftok_t srch;
+ struct rspamd_content_type_param *found = NULL;
+ const gchar *param_name;
+
+ if (args == NULL || args->len < 1) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+
+ arg = &g_array_index(args, struct expression_argument, 0);
+ g_assert(arg->type == EXPRESSION_ARGUMENT_NORMAL);
+ param_name = arg->data;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, cur_part)
+ {
+ if (args->len >= 2) {
+ arg1 = &g_array_index(args, struct expression_argument, 1);
+ if (g_ascii_strncasecmp(arg1->data, "true",
+ sizeof("true") - 1) == 0) {
+ recursive = TRUE;
+ }
+ }
+ else {
+ /*
+ * If user did not specify argument, let's assume that he wants
+ * recursive search if mime part is multipart/mixed
+ */
+ if (IS_PART_MULTIPART(cur_part)) {
+ recursive = TRUE;
+ }
+ }
+
+
+ rspamd_ftok_t lit;
+ RSPAMD_FTOK_FROM_STR(&srch, param_name);
+ RSPAMD_FTOK_FROM_STR(&lit, "charset");
+
+ if (rspamd_ftok_equal(&srch, &lit)) {
+ if (cur_part->ct->charset.len > 0) {
+ return TRUE;
+ }
+ }
+
+ RSPAMD_FTOK_FROM_STR(&lit, "boundary");
+ if (rspamd_ftok_equal(&srch, &lit)) {
+ if (cur_part->ct->boundary.len > 0) {
+ return TRUE;
+ }
+ }
+
+ if (cur_part->ct->attrs) {
+ found = g_hash_table_lookup(cur_part->ct->attrs, &srch);
+
+ if (found) {
+ return TRUE;
+ }
+ }
+
+ if (!recursive) {
+ break;
+ }
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_content_type_check(struct rspamd_task *task,
+ GArray *args,
+ gboolean check_subtype)
+{
+ rspamd_ftok_t *param_data, srch;
+ rspamd_regexp_t *re;
+ struct expression_argument *arg1, *arg_pattern;
+ struct rspamd_content_type *ct;
+ gint r = 0;
+ guint i;
+ gboolean recursive = FALSE;
+ struct rspamd_mime_part *cur_part;
+
+ if (args == NULL || args->len < 1) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+
+ arg_pattern = &g_array_index(args, struct expression_argument, 0);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, cur_part)
+ {
+ ct = cur_part->ct;
+
+ if (args->len >= 2) {
+ arg1 = &g_array_index(args, struct expression_argument, 1);
+ if (g_ascii_strncasecmp(arg1->data, "true",
+ sizeof("true") - 1) == 0) {
+ recursive = TRUE;
+ }
+ }
+ else {
+ /*
+ * If user did not specify argument, let's assume that he wants
+ * recursive search if mime part is multipart/mixed
+ */
+ if (IS_PART_MULTIPART(cur_part)) {
+ recursive = TRUE;
+ }
+ }
+
+ if (check_subtype) {
+ param_data = &ct->subtype;
+ }
+ else {
+ param_data = &ct->type;
+ }
+
+ if (arg_pattern->type == EXPRESSION_ARGUMENT_REGEXP) {
+ re = arg_pattern->data;
+
+ if (param_data->len > 0) {
+ r = rspamd_regexp_search(re, param_data->begin, param_data->len,
+ NULL, NULL, FALSE, NULL);
+ }
+
+ if (r) {
+ return TRUE;
+ }
+ }
+ else {
+ /* Just do strcasecmp */
+ srch.begin = arg_pattern->data;
+ srch.len = strlen(arg_pattern->data);
+
+ if (rspamd_ftok_casecmp(param_data, &srch) == 0) {
+ return TRUE;
+ }
+ }
+
+ /* Get next part */
+ if (!recursive) {
+ break;
+ }
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_content_type_is_type(struct rspamd_task *task,
+ GArray *args,
+ void *unused)
+{
+ return rspamd_content_type_check(task, args, FALSE);
+}
+
+static gboolean
+rspamd_content_type_is_subtype(struct rspamd_task *task,
+ GArray *args,
+ void *unused)
+{
+ return rspamd_content_type_check(task, args, TRUE);
+}
+
+static gboolean
+compare_subtype(struct rspamd_task *task, struct rspamd_content_type *ct,
+ struct expression_argument *subtype)
+{
+ rspamd_regexp_t *re;
+ rspamd_ftok_t srch;
+ gint r = 0;
+
+ if (subtype == NULL || ct == NULL) {
+ msg_warn_task("invalid parameters passed");
+ return FALSE;
+ }
+ if (subtype->type == EXPRESSION_ARGUMENT_REGEXP) {
+ re = subtype->data;
+
+ if (ct->subtype.len > 0) {
+ r = rspamd_regexp_search(re, ct->subtype.begin, ct->subtype.len,
+ NULL, NULL, FALSE, NULL);
+ }
+ }
+ else {
+ srch.begin = subtype->data;
+ srch.len = strlen(subtype->data);
+
+ /* Just do strcasecmp */
+ if (rspamd_ftok_casecmp(&ct->subtype, &srch) == 0) {
+ return TRUE;
+ }
+ }
+
+ return r;
+}
+
+static gboolean
+compare_len(struct rspamd_mime_part *part, guint min, guint max)
+{
+ if (min == 0 && max == 0) {
+ return TRUE;
+ }
+
+ if (min == 0) {
+ return part->parsed_data.len <= max;
+ }
+ else if (max == 0) {
+ return part->parsed_data.len >= min;
+ }
+ else {
+ return part->parsed_data.len >= min && part->parsed_data.len <= max;
+ }
+}
+
+static gboolean
+common_has_content_part(struct rspamd_task *task,
+ struct expression_argument *param_type,
+ struct expression_argument *param_subtype,
+ gint min_len,
+ gint max_len)
+{
+ rspamd_regexp_t *re;
+ struct rspamd_mime_part *part;
+ struct rspamd_content_type *ct;
+ rspamd_ftok_t srch;
+ gint r = 0;
+ guint i;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ ct = part->ct;
+
+ if (ct == NULL) {
+ continue;
+ }
+
+ if (param_type->type == EXPRESSION_ARGUMENT_REGEXP) {
+ re = param_type->data;
+
+ if (ct->type.len > 0) {
+ r = rspamd_regexp_search(re, ct->type.begin, ct->type.len,
+ NULL, NULL, FALSE, NULL);
+ }
+
+ /* Also check subtype and length of the part */
+ if (r && param_subtype) {
+ r = compare_len(part, min_len, max_len) &&
+ compare_subtype(task, ct, param_subtype);
+
+ return r;
+ }
+ }
+ else {
+ /* Just do strcasecmp */
+ srch.begin = param_type->data;
+ srch.len = strlen(param_type->data);
+
+ if (rspamd_ftok_casecmp(&ct->type, &srch) == 0) {
+ if (param_subtype) {
+ if (compare_subtype(task, ct, param_subtype)) {
+ if (compare_len(part, min_len, max_len)) {
+ return TRUE;
+ }
+ }
+ }
+ else {
+ if (compare_len(part, min_len, max_len)) {
+ return TRUE;
+ }
+ }
+ }
+ }
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_has_content_part(struct rspamd_task *task, GArray *args, void *unused)
+{
+ struct expression_argument *param_type = NULL, *param_subtype = NULL;
+
+ if (args == NULL) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+
+ param_type = &g_array_index(args, struct expression_argument, 0);
+ if (args->len >= 2) {
+ param_subtype = &g_array_index(args, struct expression_argument, 1);
+ }
+
+ return common_has_content_part(task, param_type, param_subtype, 0, 0);
+}
+
+static gboolean
+rspamd_has_content_part_len(struct rspamd_task *task,
+ GArray *args,
+ void *unused)
+{
+ struct expression_argument *param_type = NULL, *param_subtype = NULL;
+ gint min = 0, max = 0;
+ struct expression_argument *arg;
+
+ if (args == NULL) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+
+ param_type = &g_array_index(args, struct expression_argument, 0);
+
+ if (args->len >= 2) {
+ param_subtype = &g_array_index(args, struct expression_argument, 1);
+
+ if (args->len >= 3) {
+ arg = &g_array_index(args, struct expression_argument, 2);
+ errno = 0;
+ min = strtoul(arg->data, NULL, 10);
+ g_assert(arg->type == EXPRESSION_ARGUMENT_NORMAL);
+
+ if (errno != 0) {
+ msg_warn_task("invalid numeric value '%s': %s",
+ (gchar *) arg->data,
+ strerror(errno));
+ return FALSE;
+ }
+
+ if (args->len >= 4) {
+ arg = &g_array_index(args, struct expression_argument, 3);
+ g_assert(arg->type == EXPRESSION_ARGUMENT_NORMAL);
+ max = strtoul(arg->data, NULL, 10);
+
+ if (errno != 0) {
+ msg_warn_task("invalid numeric value '%s': %s",
+ (gchar *) arg->data,
+ strerror(errno));
+ return FALSE;
+ }
+ }
+ }
+ }
+
+ return common_has_content_part(task, param_type, param_subtype, min, max);
+}
+
+static gboolean
+rspamd_is_empty_body(struct rspamd_task *task,
+ GArray *args,
+ void *unused)
+{
+ struct rspamd_mime_part *part;
+ guint i;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ if (part->parsed_data.len > 0) {
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+#define TASK_FLAG_READ(flag) \
+ do { \
+ result = !!(task->flags & (flag)); \
+ } while (0)
+
+#define TASK_GET_FLAG(flag, strname, macro) \
+ do { \
+ if (!found && strcmp((flag), strname) == 0) { \
+ TASK_FLAG_READ((macro)); \
+ found = TRUE; \
+ } \
+ } while (0)
+
+#define TASK_PROTOCOL_FLAG_READ(flag) \
+ do { \
+ result = !!(task->protocol_flags & (flag)); \
+ } while (0)
+
+#define TASK_GET_PROTOCOL_FLAG(flag, strname, macro) \
+ do { \
+ if (!found && strcmp((flag), strname) == 0) { \
+ TASK_PROTOCOL_FLAG_READ((macro)); \
+ found = TRUE; \
+ } \
+ } while (0)
+
+
+static gboolean
+rspamd_has_flag_expr(struct rspamd_task *task,
+ GArray *args,
+ void *unused)
+{
+ gboolean found = FALSE, result = FALSE;
+ struct expression_argument *flag_arg;
+ const gchar *flag_str;
+
+ if (args == NULL) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+
+ flag_arg = &g_array_index(args, struct expression_argument, 0);
+
+ if (flag_arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+ msg_warn_task("invalid parameter to function");
+ return FALSE;
+ }
+
+ flag_str = (const gchar *) flag_arg->data;
+
+ TASK_GET_FLAG(flag_str, "pass_all", RSPAMD_TASK_FLAG_PASS_ALL);
+ TASK_GET_FLAG(flag_str, "no_log", RSPAMD_TASK_FLAG_NO_LOG);
+ TASK_GET_FLAG(flag_str, "no_stat", RSPAMD_TASK_FLAG_NO_STAT);
+ TASK_GET_FLAG(flag_str, "skip", RSPAMD_TASK_FLAG_SKIP);
+ TASK_GET_PROTOCOL_FLAG(flag_str, "extended_urls",
+ RSPAMD_TASK_PROTOCOL_FLAG_EXT_URLS);
+ TASK_GET_FLAG(flag_str, "learn_spam", RSPAMD_TASK_FLAG_LEARN_SPAM);
+ TASK_GET_FLAG(flag_str, "learn_ham", RSPAMD_TASK_FLAG_LEARN_HAM);
+ TASK_GET_FLAG(flag_str, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED);
+ TASK_GET_FLAG(flag_str, "broken_headers",
+ RSPAMD_TASK_FLAG_BROKEN_HEADERS);
+ TASK_GET_FLAG(flag_str, "skip_process",
+ RSPAMD_TASK_FLAG_SKIP_PROCESS);
+ TASK_GET_PROTOCOL_FLAG(flag_str, "milter",
+ RSPAMD_TASK_PROTOCOL_FLAG_MILTER);
+ TASK_GET_FLAG(flag_str, "bad_unicode",
+ RSPAMD_TASK_FLAG_BAD_UNICODE);
+
+ if (!found) {
+ msg_warn_task("invalid flag name %s", flag_str);
+ return FALSE;
+ }
+
+ return result;
+}
+
+static gboolean
+rspamd_has_symbol_expr(struct rspamd_task *task,
+ GArray *args,
+ void *unused)
+{
+ struct expression_argument *sym_arg;
+ const gchar *symbol_str;
+
+ if (args == NULL) {
+ msg_warn_task("no parameters to function");
+ return FALSE;
+ }
+
+ sym_arg = &g_array_index(args, struct expression_argument, 0);
+
+ if (sym_arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+ msg_warn_task("invalid parameter to function");
+ return FALSE;
+ }
+
+ symbol_str = (const gchar *) sym_arg->data;
+
+ if (rspamd_task_find_symbol_result(task, symbol_str, NULL)) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
diff --git a/src/libmime/mime_expressions.h b/src/libmime/mime_expressions.h
new file mode 100644
index 0000000..a2ea3fe
--- /dev/null
+++ b/src/libmime/mime_expressions.h
@@ -0,0 +1,65 @@
+/**
+ * @file expressions.h
+ * Rspamd expressions API
+ */
+
+#ifndef RSPAMD_EXPRESSIONS_H
+#define RSPAMD_EXPRESSIONS_H
+
+#include "config.h"
+#include "expression.h"
+#include "contrib/libucl/ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_config;
+
+struct rspamd_mime_expr_ud {
+ struct rspamd_config *cfg;
+ const ucl_object_t *conf_obj;
+};
+
+extern const struct rspamd_atom_subr mime_expr_subr;
+
+/**
+ * Function's argument
+ */
+enum rspamd_expression_type {
+ EXPRESSION_ARGUMENT_NORMAL = 0,
+ EXPRESSION_ARGUMENT_BOOL,
+ EXPRESSION_ARGUMENT_REGEXP
+};
+struct expression_argument {
+ enum rspamd_expression_type type; /**< type of argument (text or other function) */
+ void *data; /**< pointer to its data */
+};
+
+
+typedef gboolean (*rspamd_internal_func_t)(struct rspamd_task *,
+ GArray *args, void *user_data);
+
+
+/**
+ * Register specified function to rspamd internal functions list
+ * @param name name of function
+ * @param func pointer to function
+ */
+void register_expression_function(const gchar *name,
+ rspamd_internal_func_t func,
+ void *user_data);
+
+/**
+ * Set global limit of regexp data size to be processed
+ * @param limit new limit in bytes
+ * @return old limit value
+ */
+guint rspamd_mime_expression_set_re_limit(guint limit);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libmime/mime_headers.c b/src/libmime/mime_headers.c
new file mode 100644
index 0000000..2bd559d
--- /dev/null
+++ b/src/libmime/mime_headers.c
@@ -0,0 +1,1441 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "mime_headers.h"
+#include "smtp_parsers.h"
+#include "mime_encoding.h"
+#include "received.h"
+#include "contrib/uthash/utlist.h"
+#include "libserver/mempool_vars_internal.h"
+#include "libserver/cfg_file.h"
+#include "libutil/util.h"
+#include <unicode/utf8.h>
+
+KHASH_INIT(rspamd_mime_headers_htb, gchar *,
+ struct rspamd_mime_header *, 1,
+ rspamd_strcase_hash, rspamd_strcase_equal);
+
+struct rspamd_mime_headers_table {
+ khash_t(rspamd_mime_headers_htb) htb;
+ ref_entry_t ref;
+};
+
+static void
+rspamd_mime_header_check_special(struct rspamd_task *task,
+ struct rspamd_mime_header *rh)
+{
+ guint64 h;
+ const gchar *p, *end;
+ gchar *id;
+ gint max_recipients = -1, len;
+
+ if (task->cfg) {
+ max_recipients = task->cfg->max_recipients;
+ }
+
+ h = rspamd_icase_hash(rh->name, strlen(rh->name), 0xdeadbabe);
+
+ switch (h) {
+ case 0x88705DC4D9D61ABULL: /* received */
+ if (rspamd_received_header_parse(task, rh->decoded, strlen(rh->decoded), rh)) {
+ rh->flags |= RSPAMD_HEADER_RECEIVED;
+ }
+ break;
+ case 0x76F31A09F4352521ULL: /* to */
+ MESSAGE_FIELD(task, rcpt_mime) = rspamd_email_address_from_mime(task->task_pool,
+ rh->value, strlen(rh->value),
+ MESSAGE_FIELD(task, rcpt_mime), max_recipients);
+ rh->flags |= RSPAMD_HEADER_TO | RSPAMD_HEADER_RCPT | RSPAMD_HEADER_UNIQUE;
+ break;
+ case 0x7EB117C1480B76ULL: /* cc */
+ MESSAGE_FIELD(task, rcpt_mime) = rspamd_email_address_from_mime(task->task_pool,
+ rh->value, strlen(rh->value),
+ MESSAGE_FIELD(task, rcpt_mime), max_recipients);
+ rh->flags |= RSPAMD_HEADER_CC | RSPAMD_HEADER_RCPT | RSPAMD_HEADER_UNIQUE;
+ break;
+ case 0xE4923E11C4989C8DULL: /* bcc */
+ MESSAGE_FIELD(task, rcpt_mime) = rspamd_email_address_from_mime(task->task_pool,
+ rh->value, strlen(rh->value),
+ MESSAGE_FIELD(task, rcpt_mime), max_recipients);
+ rh->flags |= RSPAMD_HEADER_BCC | RSPAMD_HEADER_RCPT | RSPAMD_HEADER_UNIQUE;
+ break;
+ case 0x41E1985EDC1CBDE4ULL: /* from */
+ MESSAGE_FIELD(task, from_mime) = rspamd_email_address_from_mime(task->task_pool,
+ rh->value, strlen(rh->value),
+ MESSAGE_FIELD(task, from_mime), max_recipients);
+ rh->flags |= RSPAMD_HEADER_FROM | RSPAMD_HEADER_SENDER | RSPAMD_HEADER_UNIQUE;
+ break;
+ case 0x43A558FC7C240226ULL: /* message-id */ {
+
+ rh->flags = RSPAMD_HEADER_MESSAGE_ID | RSPAMD_HEADER_UNIQUE;
+ p = rh->decoded;
+ len = rspamd_strip_smtp_comments_inplace(rh->decoded, strlen(p));
+ rh->decoded[len] = '\0'; /* Zero terminate after stripping */
+ /* Strip surrounding spaces */
+ rh->decoded = g_strstrip(rh->decoded);
+ end = p + len;
+
+ if (*p == '<') {
+ p++;
+ }
+
+ if (end > p) {
+ gchar *d;
+
+ if (*(end - 1) == '>') {
+ end--;
+ }
+
+ id = rspamd_mempool_alloc(task->task_pool, end - p + 1);
+ d = id;
+
+ while (p < end) {
+ if (g_ascii_isgraph(*p)) {
+ *d++ = *p++;
+ }
+ else {
+ *d++ = '?';
+ p++;
+ }
+ }
+
+ *d = '\0';
+
+ MESSAGE_FIELD(task, message_id) = id;
+ }
+
+ break;
+ }
+ case 0xB91D3910358E8212ULL: /* subject */
+ if (MESSAGE_FIELD(task, subject) == NULL) {
+ MESSAGE_FIELD(task, subject) = rh->decoded;
+ }
+ rh->flags = RSPAMD_HEADER_SUBJECT | RSPAMD_HEADER_UNIQUE;
+ break;
+ case 0xEE4AA2EAAC61D6F4ULL: /* return-path */
+ if (task->from_envelope == NULL) {
+ task->from_envelope = rspamd_email_address_from_smtp(rh->decoded,
+ strlen(rh->decoded));
+ }
+ rh->flags = RSPAMD_HEADER_RETURN_PATH | RSPAMD_HEADER_UNIQUE;
+ break;
+ case 0xB9EEFAD2E93C2161ULL: /* delivered-to */
+ if (task->deliver_to == NULL) {
+ task->deliver_to = rh->decoded;
+ }
+ rh->flags = RSPAMD_HEADER_DELIVERED_TO;
+ break;
+ case 0x2EC3BFF3C393FC10ULL: /* date */
+ case 0xAC0DDB1A1D214CAULL: /* sender */
+ case 0x54094572367AB695ULL: /* in-reply-to */
+ case 0x81CD9E9131AB6A9AULL: /* content-type */
+ case 0xC39BD9A75AA25B60ULL: /* content-transfer-encoding */
+ case 0xB3F6704CB3AD6589ULL: /* references */
+ rh->flags = RSPAMD_HEADER_UNIQUE;
+ break;
+ }
+}
+
+static void
+rspamd_mime_header_add(struct rspamd_task *task,
+ khash_t(rspamd_mime_headers_htb) * target,
+ struct rspamd_mime_header **order_ptr,
+ struct rspamd_mime_header *rh,
+ gboolean check_special)
+{
+ khiter_t k;
+ struct rspamd_mime_header *ex;
+ int res;
+
+ k = kh_put(rspamd_mime_headers_htb, target, rh->name, &res);
+
+ if (res == 0) {
+ ex = kh_value(target, k);
+ DL_APPEND(ex, rh);
+ msg_debug_task("append raw header %s: %s", rh->name, rh->value);
+ }
+ else {
+ kh_value(target, k) = rh;
+ rh->prev = rh;
+ rh->next = NULL;
+ msg_debug_task("add new raw header %s: %s", rh->name, rh->value);
+ }
+
+ LL_PREPEND2(*order_ptr, rh, ord_next);
+
+ if (check_special) {
+ rspamd_mime_header_check_special(task, rh);
+ }
+}
+
+
+/* Convert raw headers to a list of struct raw_header * */
+void rspamd_mime_headers_process(struct rspamd_task *task,
+ struct rspamd_mime_headers_table *target,
+ struct rspamd_mime_header **order_ptr,
+ const gchar *in, gsize len,
+ gboolean check_newlines)
+{
+ struct rspamd_mime_header *nh = NULL;
+ const gchar *p, *c, *end;
+ gchar *tmp, *tp;
+ gint state = 0, l, next_state = 100, err_state = 100, t_state;
+ gboolean valid_folding = FALSE, shift_by_one = FALSE;
+ guint nlines_count[RSPAMD_TASK_NEWLINES_MAX];
+ guint norder = 0;
+
+ p = in;
+ end = p + len;
+ c = p;
+ memset(nlines_count, 0, sizeof(nlines_count));
+ msg_debug_task("start processing headers");
+
+ while (p < end) {
+ /* FSM for processing headers */
+ switch (state) {
+ case 0:
+ /* Begin processing headers */
+ if (!g_ascii_isalpha(*p)) {
+ /* We have some garbage at the beginning of headers, skip this line */
+ state = 100;
+ next_state = 0;
+ }
+ else {
+ state = 1;
+ c = p;
+ }
+ break;
+ case 1:
+ /* We got something like header's name */
+ if (*p == ':') {
+ nh = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct rspamd_mime_header));
+ l = p - c;
+ tmp = rspamd_mempool_alloc(task->task_pool, l + 1);
+ rspamd_null_safe_copy(c, l, tmp, l + 1);
+ nh->name = tmp;
+ nh->flags |= RSPAMD_HEADER_EMPTY_SEPARATOR;
+ nh->raw_value = c;
+ nh->raw_len = p - c; /* Including trailing ':' */
+ p++;
+ state = 2;
+ c = p;
+ }
+ else if (g_ascii_isspace(*p)) {
+ /* Not header but some garbage */
+ if (target == MESSAGE_FIELD(task, raw_headers)) {
+ /* Do not propagate flag from the attachments */
+ task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
+ }
+ state = 100;
+ next_state = 0;
+ }
+ else {
+ p++;
+ }
+ break;
+ case 2:
+ /* We got header's name, so skip any \t or spaces */
+ if (*p == '\t') {
+ nh->flags &= ~RSPAMD_HEADER_EMPTY_SEPARATOR;
+ nh->flags |= RSPAMD_HEADER_TAB_SEPARATED;
+ p++;
+ }
+ else if (*p == ' ') {
+ nh->flags &= ~RSPAMD_HEADER_EMPTY_SEPARATOR;
+ p++;
+ }
+ else if (*p == '\n' || *p == '\r') {
+
+ if (check_newlines) {
+ if (*p == '\n') {
+ nlines_count[RSPAMD_TASK_NEWLINES_LF]++;
+ }
+ else if (p + 1 < end && *(p + 1) == '\n') {
+ nlines_count[RSPAMD_TASK_NEWLINES_CRLF]++;
+ }
+ else {
+ nlines_count[RSPAMD_TASK_NEWLINES_CR]++;
+ }
+ }
+
+ /* Process folding */
+ state = 99;
+ l = p - c;
+ if (l > 0) {
+ tmp = rspamd_mempool_alloc(task->task_pool, l + 1);
+ rspamd_null_safe_copy(c, l, tmp, l + 1);
+ nh->separator = tmp;
+ }
+ next_state = 3;
+ err_state = 5;
+ c = p;
+ }
+ else {
+ /* Process value */
+ l = p - c;
+ if (l >= 0) {
+ tmp = rspamd_mempool_alloc(task->task_pool, l + 1);
+ rspamd_null_safe_copy(c, l, tmp, l + 1);
+ nh->separator = tmp;
+ }
+ c = p;
+ state = 3;
+ }
+ break;
+ case 3:
+ if (*p == '\r' || *p == '\n') {
+ /* Hold folding */
+ if (check_newlines) {
+ if (*p == '\n') {
+ nlines_count[RSPAMD_TASK_NEWLINES_LF]++;
+ }
+ else if (p + 1 < end && *(p + 1) == '\n') {
+ nlines_count[RSPAMD_TASK_NEWLINES_CRLF]++;
+ }
+ else {
+ nlines_count[RSPAMD_TASK_NEWLINES_CR]++;
+ }
+ }
+ state = 99;
+ next_state = 3;
+ err_state = 4;
+ }
+ else if (p + 1 == end) {
+ state = 4;
+ }
+ else {
+ p++;
+ }
+ break;
+ case 4:
+ /* Copy header's value */
+
+ /*
+ * XXX:
+ * The original decision to use here null terminated
+ * strings was extremely poor!
+ */
+ l = p - c;
+ tmp = rspamd_mempool_alloc(task->task_pool, l + 1);
+ tp = tmp;
+ t_state = 0;
+ while (l--) {
+ if (t_state == 0) {
+ /* Before folding */
+ if (*c == '\n' || *c == '\r') {
+ t_state = 1;
+ c++;
+ *tp++ = ' ';
+ }
+ else {
+ if (*c != '\0') {
+ *tp++ = *c++;
+ }
+ else {
+ c++;
+ }
+ }
+ }
+ else if (t_state == 1) {
+ /* Inside folding */
+ if (g_ascii_isspace(*c)) {
+ c++;
+ }
+ else {
+ t_state = 0;
+ if (*c != '\0') {
+ *tp++ = *c++;
+ }
+ else {
+ c++;
+ }
+ }
+ }
+ }
+ /* Strip last space that can be added by \r\n parsing */
+ if (tp > tmp && *(tp - 1) == ' ') {
+ tp--;
+ }
+
+ *tp = '\0';
+ /* Strip the initial spaces that could also be added by folding */
+ while (*tmp != '\0' && g_ascii_isspace(*tmp)) {
+ tmp++;
+ }
+
+ if (p + 1 == end) {
+ nh->raw_len = end - nh->raw_value;
+ }
+ else {
+ nh->raw_len = p - nh->raw_value;
+ }
+
+ nh->value = tmp;
+
+ gboolean broken_utf = FALSE;
+
+ nh->decoded = rspamd_mime_header_decode(task->task_pool,
+ nh->value, strlen(tmp), &broken_utf);
+
+ if (broken_utf) {
+ task->flags |= RSPAMD_TASK_FLAG_BAD_UNICODE;
+ }
+
+ if (nh->decoded == NULL) {
+ /* As we strip comments in place... */
+ nh->decoded = rspamd_mempool_strdup(task->task_pool, "");
+ }
+
+ /* We also validate utf8 and replace all non-valid utf8 chars */
+ rspamd_mime_charset_utf_enforce(nh->decoded, strlen(nh->decoded));
+ nh->order = norder++;
+ rspamd_mime_header_add(task, &target->htb, order_ptr, nh, check_newlines);
+ nh = NULL;
+ state = 0;
+ break;
+ case 5:
+ /* Header has only name, no value */
+ nh->value = rspamd_mempool_strdup(task->task_pool, "");
+ nh->decoded = rspamd_mempool_strdup(task->task_pool, "");
+ nh->raw_len = p - nh->raw_value;
+ if (shift_by_one) {
+ nh->raw_len++;
+ }
+ nh->order = norder++;
+ rspamd_mime_header_add(task, &target->htb, order_ptr, nh, check_newlines);
+ nh = NULL;
+ state = 0;
+ break;
+ case 99:
+ /* Folding state */
+ if (p + 1 == end) {
+ state = err_state;
+ /* Include the last character into the next header */
+ shift_by_one = TRUE;
+ }
+ else {
+ if (*p == '\r' || *p == '\n') {
+ p++;
+ valid_folding = FALSE;
+ }
+ else if (*p == '\t' || *p == ' ') {
+ /* Valid folding */
+ p++;
+ valid_folding = TRUE;
+ }
+ else {
+ if (valid_folding) {
+ debug_task("go to state: %d->%d", state, next_state);
+ state = next_state;
+ }
+ else {
+ /* Fall back */
+ debug_task("go to state: %d->%d", state, err_state);
+ state = err_state;
+ }
+ }
+ }
+ break;
+ case 100:
+ /* Fail state, skip line */
+
+ if (*p == '\r') {
+ if (p + 1 < end && *(p + 1) == '\n') {
+ nlines_count[RSPAMD_TASK_NEWLINES_CRLF]++;
+ p++;
+ }
+ p++;
+ state = next_state;
+ }
+ else if (*p == '\n') {
+ nlines_count[RSPAMD_TASK_NEWLINES_LF]++;
+
+ if (p + 1 < end && *(p + 1) == '\r') {
+ p++;
+ }
+ p++;
+ state = next_state;
+ }
+ else if (p + 1 == end) {
+ state = next_state;
+ p++;
+ }
+ else {
+ p++;
+ }
+ break;
+ }
+ }
+
+ /* Since we have prepended headers, we need to reverse the list to get the actual order */
+ LL_REVERSE(*order_ptr);
+
+ if (check_newlines) {
+ guint max_cnt = 0;
+ gint sel = 0;
+ rspamd_cryptobox_hash_state_t hs;
+ guchar hout[rspamd_cryptobox_HASHBYTES], *hexout;
+
+ for (gint i = RSPAMD_TASK_NEWLINES_CR; i < RSPAMD_TASK_NEWLINES_MAX; i++) {
+ if (nlines_count[i] > max_cnt) {
+ max_cnt = nlines_count[i];
+ sel = i;
+ }
+ }
+
+ MESSAGE_FIELD(task, nlines_type) = sel;
+
+ rspamd_cryptobox_hash_init(&hs, NULL, 0);
+
+ LL_FOREACH(*order_ptr, nh)
+ {
+ if (nh->name && nh->flags != RSPAMD_HEADER_RECEIVED) {
+ rspamd_cryptobox_hash_update(&hs, nh->name, strlen(nh->name));
+ }
+ }
+
+ rspamd_cryptobox_hash_final(&hs, hout);
+ hexout = rspamd_mempool_alloc(task->task_pool, sizeof(hout) * 2 + 1);
+ hexout[sizeof(hout) * 2] = '\0';
+ rspamd_encode_hex_buf(hout, sizeof(hout), hexout,
+ sizeof(hout) * 2 + 1);
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_HEADERS_HASH,
+ hexout, NULL);
+ }
+}
+
+static void
+rspamd_mime_header_maybe_save_token(rspamd_mempool_t *pool,
+ GString *out,
+ GByteArray *token,
+ GByteArray *decoded_token,
+ rspamd_ftok_t *old_charset,
+ rspamd_ftok_t *new_charset)
+{
+ if (new_charset->len == 0) {
+ g_assert_not_reached();
+ }
+
+ if (old_charset->len > 0) {
+ if (rspamd_ftok_casecmp(new_charset, old_charset) == 0) {
+ rspamd_ftok_t srch;
+
+ /*
+ * Special case for iso-2022-jp:
+ * https://github.com/vstakhov/rspamd/issues/1669
+ */
+ RSPAMD_FTOK_ASSIGN(&srch, "iso-2022-jp");
+
+ if (rspamd_ftok_casecmp(new_charset, &srch) != 0) {
+ /* We can concatenate buffers, just return */
+ return;
+ }
+ }
+ }
+
+ /* We need to flush and decode old token to out string */
+ if (rspamd_mime_to_utf8_byte_array(token, decoded_token, pool,
+ rspamd_mime_detect_charset(new_charset, pool))) {
+ g_string_append_len(out, decoded_token->data, decoded_token->len);
+ }
+
+ /* We also reset buffer */
+ g_byte_array_set_size(token, 0);
+ /*
+ * Propagate charset
+ *
+ * Here are dragons: we save the original charset to allow buffers concat
+ * in the condition at the beginning of the function.
+ * However, it will likely cause unnecessary calls for
+ * `rspamd_mime_detect_charset` which could be relatively expensive.
+ * But we ignore that for now...
+ */
+ memcpy(old_charset, new_charset, sizeof(*old_charset));
+}
+
+static void
+rspamd_mime_header_sanity_check(GString *str)
+{
+ gsize i;
+ gchar t;
+
+ for (i = 0; i < str->len; i++) {
+ t = str->str[i];
+ if (!((t & 0x80) || g_ascii_isgraph(t))) {
+ if (g_ascii_isspace(t)) {
+ /* Replace spaces characters with plain space */
+ str->str[i] = ' ';
+ }
+ else {
+ str->str[i] = '?';
+ }
+ }
+ }
+}
+
+gchar *
+rspamd_mime_header_decode(rspamd_mempool_t *pool, const gchar *in,
+ gsize inlen, gboolean *invalid_utf)
+{
+ GString *out;
+ const guchar *c, *p, *end;
+ const gchar *tok_start = NULL;
+ gsize tok_len = 0, pos;
+ GByteArray *token = NULL, *decoded;
+ rspamd_ftok_t cur_charset = {0, NULL}, old_charset = {0, NULL};
+ gint encoding;
+ gssize r;
+ guint qmarks = 0;
+ gchar *ret;
+ enum {
+ parse_normal = 0,
+ got_eqsign,
+ got_encoded_start,
+ got_more_qmark,
+ skip_spaces,
+ } state = parse_normal;
+
+ g_assert(in != NULL);
+
+ c = in;
+ p = in;
+ end = in + inlen;
+ out = g_string_sized_new(inlen);
+ token = g_byte_array_sized_new(80);
+ decoded = g_byte_array_sized_new(122);
+
+ while (p < end) {
+ switch (state) {
+ case parse_normal:
+ if (*p == '=') {
+ g_string_append_len(out, c, p - c);
+ c = p;
+ state = got_eqsign;
+ }
+ else if (*p >= 128) {
+ gint off = 0;
+ UChar32 uc;
+ /* Unencoded character */
+ g_string_append_len(out, c, p - c);
+ /* Check if that's valid UTF8 */
+ U8_NEXT(p, off, end - p, uc);
+
+ if (uc <= 0) {
+ c = p + 1;
+ /* 0xFFFD in UTF8 */
+ g_string_append_len(out, " ", 3);
+ off = 0;
+ U8_APPEND_UNSAFE(out->str + out->len - 3,
+ off, 0xfffd);
+
+ if (invalid_utf) {
+ *invalid_utf = TRUE;
+ }
+ }
+ else {
+ c = p;
+ p = p + off;
+ continue; /* To avoid p ++ after this block */
+ }
+ }
+ p++;
+ break;
+ case got_eqsign:
+ if (*p == '?') {
+ state = got_encoded_start;
+ qmarks = 0;
+ }
+ else {
+ g_string_append_len(out, c, 1);
+ c = p;
+ state = parse_normal;
+ continue; /* Deal with == case */
+ }
+ p++;
+ break;
+ case got_encoded_start:
+ if (*p == '?') {
+ state = got_more_qmark;
+ qmarks++;
+
+ /* Skip multiple ? signs */
+ p++;
+ while (p < end && *p == '?') {
+ p++;
+ }
+
+ continue;
+ }
+ p++;
+ break;
+ case got_more_qmark:
+ if (*p == '=') {
+ if (qmarks < 3) {
+ state = got_encoded_start;
+ }
+ else {
+ /* Finished encoded boundary */
+ if (*c == '"') {
+ /* Quoted string, non-RFC conformant but used by retards */
+ c++;
+ }
+ if (rspamd_rfc2047_parser(c, p - c + 1, &encoding,
+ &cur_charset.begin, &cur_charset.len,
+ &tok_start, &tok_len)) {
+ /* We have a token, so we can decode it from `encoding` */
+ if (token->len > 0) {
+ if (old_charset.len == 0) {
+ memcpy(&old_charset, &cur_charset,
+ sizeof(old_charset));
+ }
+
+ rspamd_mime_header_maybe_save_token(pool, out,
+ token, decoded,
+ &old_charset, &cur_charset);
+ }
+
+ qmarks = 0;
+ pos = token->len;
+ g_byte_array_set_size(token, pos + tok_len);
+
+ if (encoding == RSPAMD_RFC2047_QP) {
+ r = rspamd_decode_qp2047_buf(tok_start, tok_len,
+ token->data + pos, tok_len);
+
+ if (r != -1) {
+ token->len = pos + r;
+ }
+ else {
+ /* Cannot decode qp */
+ token->len -= tok_len;
+ }
+ }
+ else {
+ if (rspamd_cryptobox_base64_decode(tok_start, tok_len,
+ token->data + pos, &tok_len)) {
+ token->len = pos + tok_len;
+ }
+ else {
+ /* Cannot decode */
+ token->len -= tok_len;
+ }
+ }
+
+ c = p + 1;
+ state = skip_spaces;
+ }
+ else {
+ /* Not encoded-word */
+ old_charset.len = 0;
+
+ if (token->len > 0) {
+ rspamd_mime_header_maybe_save_token(pool, out,
+ token, decoded,
+ &old_charset, &cur_charset);
+ }
+
+ g_string_append_len(out, c, p - c);
+ c = p;
+ state = parse_normal;
+ }
+ } /* qmarks >= 3 */
+ } /* p == '=' */
+ else {
+ state = got_encoded_start;
+ }
+ p++;
+ break;
+ case skip_spaces:
+ if (g_ascii_isspace(*p)) {
+ p++;
+ }
+ else if (*p == '=' && p < end - 1 && p[1] == '?') {
+ /* Next boundary, can glue */
+ c = p;
+ p += 2;
+ state = got_encoded_start;
+ }
+ else {
+ /* Need to save spaces and decoded token */
+ if (token->len > 0) {
+ old_charset.len = 0;
+ rspamd_mime_header_maybe_save_token(pool, out,
+ token, decoded,
+ &old_charset, &cur_charset);
+ }
+
+ g_string_append_len(out, c, p - c);
+ c = p;
+ state = parse_normal;
+ }
+ break;
+ }
+ }
+
+ /* Leftover */
+ switch (state) {
+ case skip_spaces:
+ if (token->len > 0 && cur_charset.len > 0) {
+ old_charset.len = 0;
+ rspamd_mime_header_maybe_save_token(pool, out,
+ token, decoded,
+ &old_charset, &cur_charset);
+ }
+ break;
+ default:
+ /* Just copy leftover */
+ if (p > c) {
+ g_string_append_len(out, c, p - c);
+ }
+ break;
+ }
+
+ g_byte_array_free(token, TRUE);
+ g_byte_array_free(decoded, TRUE);
+ rspamd_mime_header_sanity_check(out);
+ rspamd_mempool_notify_alloc(pool, out->len);
+ ret = g_string_free(out, FALSE);
+ rspamd_mempool_add_destructor(pool, g_free, ret);
+
+ return ret;
+}
+
+gchar *
+rspamd_mime_header_encode(const gchar *in, gsize len)
+{
+ const gchar *p = in, *end = in + len;
+ gchar *out, encode_buf[80 * sizeof(guint32)];
+ GString *res;
+ gboolean need_encoding = FALSE;
+
+ /* Check if we need to encode */
+ while (p < end) {
+ if ((((guchar) *p) & 0x80) != 0) {
+ need_encoding = TRUE;
+ break;
+ }
+ p++;
+ }
+
+ if (!need_encoding) {
+ out = g_malloc(len + 1);
+ rspamd_strlcpy(out, in, len + 1);
+ }
+ else {
+ /* Need encode */
+ gsize ulen, pos;
+ gint r;
+ const gchar *prev;
+ /* Choose step: =?UTF-8?Q?<qp>?= should be less than 76 chars */
+ guint step = (76 - 12) / 3 + 1;
+
+ ulen = g_utf8_strlen(in, len);
+ res = g_string_sized_new(len * 2 + 1);
+ pos = 0;
+ prev = in;
+ /* Adjust chunk size for unicode average length */
+ step *= 1.0 * ulen / (gdouble) len;
+
+ while (pos < ulen) {
+ p = g_utf8_offset_to_pointer(in, pos);
+
+ if (p > prev) {
+ /* Encode and print */
+ r = rspamd_encode_qp2047_buf(prev, p - prev,
+ encode_buf, sizeof(encode_buf));
+
+ if (r != -1) {
+ if (res->len > 0) {
+ rspamd_printf_gstring(res, " =?UTF-8?Q?%*s?=", r,
+ encode_buf);
+ }
+ else {
+ rspamd_printf_gstring(res, "=?UTF-8?Q?%*s?=", r,
+ encode_buf);
+ }
+ }
+ }
+
+ pos += MIN(step, ulen - pos);
+ prev = p;
+ }
+
+ /* Leftover */
+ if (prev < end) {
+ r = rspamd_encode_qp2047_buf(prev, end - prev,
+ encode_buf, sizeof(encode_buf));
+
+ if (r != -1) {
+ if (res->len > 0) {
+ rspamd_printf_gstring(res, " =?UTF-8?Q?%*s?=", r,
+ encode_buf);
+ }
+ else {
+ rspamd_printf_gstring(res, "=?UTF-8?Q?%*s?=", r,
+ encode_buf);
+ }
+ }
+ }
+
+ out = g_string_free(res, FALSE);
+ }
+
+ return out;
+}
+
+gchar *
+rspamd_mime_message_id_generate(const gchar *fqdn)
+{
+ GString *out;
+ guint64 rnd, clk;
+
+ out = g_string_sized_new(strlen(fqdn) + 22);
+ rnd = ottery_rand_uint64();
+ clk = rspamd_get_calendar_ticks() * 1e6;
+
+ rspamd_printf_gstring(out, "%*bs.%*bs@%s",
+ (gint) sizeof(guint64) - 3, (guchar *) &clk,
+ (gint) sizeof(guint64), (gchar *) &rnd,
+ fqdn);
+
+ return g_string_free(out, FALSE);
+}
+
+struct rspamd_mime_header *
+rspamd_message_get_header_from_hash(struct rspamd_mime_headers_table *hdrs,
+ const gchar *field,
+ gboolean need_modified)
+{
+ if (hdrs == NULL) {
+ return NULL;
+ }
+
+ khiter_t k;
+ khash_t(rspamd_mime_headers_htb) *htb = &hdrs->htb;
+ struct rspamd_mime_header *hdr;
+
+ if (htb) {
+ k = kh_get(rspamd_mime_headers_htb, htb, (gchar *) field);
+
+ if (k == kh_end(htb)) {
+ return NULL;
+ }
+
+ hdr = kh_value(htb, k);
+
+ if (!need_modified) {
+ if (hdr->flags & RSPAMD_HEADER_NON_EXISTING) {
+ return NULL;
+ }
+
+ return hdr;
+ }
+ else {
+ if (hdr->flags & RSPAMD_HEADER_MODIFIED) {
+ return hdr->modified_chain;
+ }
+
+ return hdr;
+ }
+ }
+
+ return NULL;
+}
+
+struct rspamd_mime_header *
+rspamd_message_get_header_array(struct rspamd_task *task, const gchar *field,
+ gboolean need_modified)
+{
+ return rspamd_message_get_header_from_hash(
+ MESSAGE_FIELD_CHECK(task, raw_headers),
+ field, need_modified);
+}
+
+gsize rspamd_mime_headers_count(struct rspamd_mime_headers_table *hdrs)
+{
+ if (hdrs) {
+ return kh_size(&hdrs->htb);
+ }
+
+ return 0;
+}
+
+bool rspamd_mime_headers_foreach(const struct rspamd_mime_headers_table *hdrs,
+ rspamd_hdr_traverse_func_t func, void *ud)
+{
+ const gchar *name;
+ struct rspamd_mime_header *hdr;
+
+ kh_foreach(&hdrs->htb, name, hdr, {
+ if (!func(name, hdr, ud)) {
+ return false;
+ }
+ });
+
+ return true;
+}
+
+static void
+rspamd_message_headers_dtor(struct rspamd_mime_headers_table *hdrs)
+{
+ if (hdrs) {
+ kfree(hdrs->htb.keys);
+ kfree(hdrs->htb.vals);
+ kfree(hdrs->htb.flags);
+ g_free(hdrs);
+ }
+}
+
+struct rspamd_mime_headers_table *
+rspamd_message_headers_ref(struct rspamd_mime_headers_table *hdrs)
+{
+ REF_RETAIN(hdrs);
+
+ return hdrs;
+}
+
+void rspamd_message_headers_unref(struct rspamd_mime_headers_table *hdrs)
+{
+ REF_RELEASE(hdrs);
+}
+
+struct rspamd_mime_headers_table *
+rspamd_message_headers_new(void)
+{
+ struct rspamd_mime_headers_table *nhdrs;
+
+ nhdrs = g_malloc0(sizeof(*nhdrs));
+ REF_INIT_RETAIN(nhdrs, rspamd_message_headers_dtor);
+
+ return nhdrs;
+}
+
+gsize rspamd_message_header_unfold_inplace(char *hdr, gsize len)
+{
+ /*
+ * t - tortoise (destination)
+ * h - hare (source)
+ */
+ char *t = hdr, *h = hdr, *end = (hdr + len);
+ enum {
+ copy_chars,
+ folding_cr,
+ folding_lf,
+ folding_ws,
+ } state = copy_chars;
+
+ while (h < end) {
+ switch (state) {
+ case copy_chars:
+ if (*h == '\r') {
+ state = folding_cr;
+ h++;
+ }
+ else if (*h == '\n') {
+ state = folding_lf;
+ h++;
+ }
+ else {
+ *t++ = *h++;
+ }
+ break;
+ case folding_cr:
+ if (*h == '\n') {
+ state = folding_lf;
+ h++;
+ }
+ else if (g_ascii_isspace(*h)) {
+ state = folding_ws;
+ h++;
+ }
+ else {
+ /* It is weird, not like a folding, so we need to revert back */
+ *t++ = '\r';
+ state = copy_chars;
+ }
+ break;
+ case folding_lf:
+ if (g_ascii_isspace(*h)) {
+ state = folding_ws;
+ h++;
+ }
+ else {
+ /* It is weird, not like a folding, so we need to revert back */
+ *t++ = '\n';
+ state = copy_chars;
+ }
+ break;
+ case folding_ws:
+ if (!g_ascii_isspace(*h)) {
+ *t++ = ' ';
+ state = copy_chars;
+ }
+ else {
+ h++;
+ }
+ break;
+ }
+ }
+
+ return t - hdr;
+}
+
+void rspamd_message_set_modified_header(struct rspamd_task *task,
+ struct rspamd_mime_headers_table *hdrs,
+ const gchar *hdr_name,
+ const ucl_object_t *obj,
+ struct rspamd_mime_header **order_ptr)
+{
+ khiter_t k;
+ khash_t(rspamd_mime_headers_htb) *htb = &hdrs->htb;
+ struct rspamd_mime_header *hdr_elt, *existing_chain;
+ int i;
+
+ if (htb) {
+ k = kh_get(rspamd_mime_headers_htb, htb, (gchar *) hdr_name);
+
+ if (k == kh_end(htb)) {
+ hdr_elt = rspamd_mempool_alloc0(task->task_pool, sizeof(*hdr_elt));
+
+ hdr_elt->flags |= RSPAMD_HEADER_MODIFIED | RSPAMD_HEADER_NON_EXISTING;
+ hdr_elt->name = rspamd_mempool_strdup(task->task_pool, hdr_name);
+
+ int r;
+ k = kh_put(rspamd_mime_headers_htb, htb, hdr_elt->name, &r);
+
+ kh_value(htb, k) = hdr_elt;
+
+ if (order_ptr) {
+ /*
+ * This iterates over all headers in O(N), but we have no other options here, as the
+ * list is already set.
+ */
+ LL_APPEND2(*order_ptr, hdr_elt, ord_next);
+ }
+ }
+ else {
+ hdr_elt = kh_value(htb, k);
+ }
+ }
+ else {
+ /* No hash, no modification */
+ msg_err_task("internal error: calling for set_modified_header for no headers");
+ return;
+ }
+
+ if (hdr_elt->flags & RSPAMD_HEADER_MODIFIED) {
+ existing_chain = hdr_elt->modified_chain;
+ }
+ else {
+ existing_chain = hdr_elt;
+ }
+
+ const ucl_object_t *elt, *cur;
+ ucl_object_iter_t it;
+
+ /* First, deal with removed headers, copying the relevant headers with remove flag */
+ elt = ucl_object_lookup(obj, "remove");
+
+ /*
+ * remove: {1, 2 ...}
+ * where number is the header's position starting from '1'
+ */
+ if (elt && ucl_object_type(elt) == UCL_ARRAY) {
+ /* First, use a temporary array to keep all headers */
+ GPtrArray *existing_ar = g_ptr_array_new();
+ struct rspamd_mime_header *cur_hdr;
+
+ /* Exclude removed headers */
+ LL_FOREACH(existing_chain, cur_hdr)
+ {
+ if (!(cur_hdr->flags & RSPAMD_HEADER_REMOVED)) {
+ g_ptr_array_add(existing_ar, cur_hdr);
+ }
+ }
+
+ it = NULL;
+
+ while ((cur = ucl_object_iterate(elt, &it, true)) != NULL) {
+ if (ucl_object_type(cur) == UCL_INT) {
+ int ord = ucl_object_toint(cur);
+
+ if (ord == 0) {
+ /* Remove all headers in the existing chain */
+ PTR_ARRAY_FOREACH(existing_ar, i, cur_hdr)
+ {
+ cur_hdr->flags |= RSPAMD_HEADER_MODIFIED | RSPAMD_HEADER_REMOVED;
+ }
+ }
+ else if (ord > 0) {
+ /* Start from the top */
+
+ if (ord <= existing_ar->len) {
+ cur_hdr = g_ptr_array_index(existing_ar, ord - 1);
+ cur_hdr->flags |= RSPAMD_HEADER_MODIFIED | RSPAMD_HEADER_REMOVED;
+ }
+ }
+ else {
+ /* Start from the bottom; ord < 0 */
+ if ((-ord) <= existing_ar->len) {
+ cur_hdr = g_ptr_array_index(existing_ar, existing_ar->len + ord);
+ cur_hdr->flags |= RSPAMD_HEADER_MODIFIED | RSPAMD_HEADER_REMOVED;
+ }
+ }
+ }
+ }
+
+ /*
+ * Next, we return all headers modified to the existing chain
+ * This implies an additional copy of all structures but is safe enough to
+ * deal with it
+ */
+ hdr_elt->flags |= RSPAMD_HEADER_MODIFIED;
+ hdr_elt->modified_chain = NULL;
+
+ PTR_ARRAY_FOREACH(existing_ar, i, cur_hdr)
+ {
+ if (!(cur_hdr->flags & RSPAMD_HEADER_REMOVED)) {
+ struct rspamd_mime_header *nhdr = rspamd_mempool_alloc(
+ task->task_pool, sizeof(*nhdr));
+ memcpy(nhdr, cur_hdr, sizeof(*nhdr));
+ nhdr->modified_chain = NULL;
+ nhdr->prev = NULL;
+ nhdr->next = NULL;
+ nhdr->ord_next = NULL;
+
+ DL_APPEND(hdr_elt->modified_chain, nhdr);
+ }
+ }
+
+ g_ptr_array_free(existing_ar, TRUE);
+
+ /* End of headers removal logic */
+ }
+
+ /* We can now deal with headers additions */
+ elt = ucl_object_lookup(obj, "add");
+ if (elt && ucl_object_type(elt) == UCL_ARRAY) {
+ if (!(hdr_elt->flags & RSPAMD_HEADER_MODIFIED)) {
+ /* Copy the header itself to the modified chain */
+ struct rspamd_mime_header *nhdr;
+ hdr_elt->flags |= RSPAMD_HEADER_MODIFIED;
+ nhdr = rspamd_mempool_alloc(
+ task->task_pool, sizeof(*nhdr));
+ memcpy(nhdr, hdr_elt, sizeof(*hdr_elt));
+ nhdr->modified_chain = NULL;
+ nhdr->next = NULL;
+ nhdr->ord_next = NULL;
+ nhdr->prev = nhdr;
+ hdr_elt->modified_chain = nhdr;
+ }
+
+ /*
+ * add: {{1, "foo"}, {-1, "bar"} ...}
+ * where number is the header's position starting from '1'
+ */
+ it = NULL;
+
+ while ((cur = ucl_object_iterate(elt, &it, true)) != NULL) {
+ if (ucl_object_type(cur) == UCL_ARRAY) {
+ const ucl_object_t *order = ucl_array_find_index(cur, 0),
+ *value = ucl_array_find_index(cur, 1);
+
+ if (order && value &&
+ (ucl_object_type(order) == UCL_INT &&
+ ucl_object_type(value) == UCL_STRING)) {
+ int ord = ucl_object_toint(order);
+ const char *raw_value;
+ gsize raw_len;
+
+ raw_value = ucl_object_tolstring(value, &raw_len);
+
+ if (raw_len == 0) {
+ continue;
+ }
+
+ struct rspamd_mime_header *nhdr = rspamd_mempool_alloc0(
+ task->task_pool, sizeof(*nhdr));
+
+ nhdr->flags |= RSPAMD_HEADER_ADDED;
+ nhdr->name = hdr_elt->name;
+ nhdr->value = rspamd_mempool_alloc(task->task_pool,
+ raw_len + 1);
+ /* Strlcpy will ensure that value will have no embedded \0 */
+ rspamd_strlcpy(nhdr->value, raw_value, raw_len + 1);
+ gsize value_len = rspamd_message_header_unfold_inplace(nhdr->value, raw_len);
+ nhdr->value[value_len] = '\0';
+
+ /* Deal with the raw value */
+ size_t namelen = strlen(hdr_elt->name);
+ char *rawbuf = rspamd_mempool_alloc(task->task_pool, namelen +
+ raw_len +
+ sizeof(": \r\n"));
+ /* Name: value<newline> */
+ nhdr->raw_value = rawbuf;
+ memcpy(rawbuf, hdr_elt->name, namelen);
+ rawbuf += namelen;
+ memcpy(rawbuf, ": ", sizeof(": ") - 1);
+ nhdr->separator = rspamd_mempool_strdup(task->task_pool, " ");
+ rawbuf += sizeof(": ") - 1;
+ memcpy(rawbuf, raw_value, raw_len);
+ nhdr->raw_len = raw_len;
+
+ if (MESSAGE_FIELD(task, nlines_type) == RSPAMD_TASK_NEWLINES_LF) {
+ rawbuf[raw_len++] = '\n';
+ }
+ else {
+ rawbuf[raw_len++] = '\r';
+
+ if (MESSAGE_FIELD(task, nlines_type) == RSPAMD_TASK_NEWLINES_CRLF) {
+ rawbuf[raw_len++] = '\n';
+ }
+ }
+
+ rawbuf[raw_len] = '\0';
+
+ nhdr->decoded = rspamd_mime_header_decode(task->task_pool,
+ raw_value, nhdr->raw_len,
+ NULL);
+
+ /* Now find a position to insert a value */
+ struct rspamd_mime_header **pos = &hdr_elt->modified_chain;
+
+ if (ord == 0) {
+ DL_PREPEND(hdr_elt->modified_chain, nhdr);
+ }
+ else if (ord == -1) {
+ DL_APPEND(hdr_elt->modified_chain, nhdr);
+ }
+ else if (ord > 0) {
+ while (ord > 0 && (*pos)) {
+ ord--;
+ pos = &((*pos)->next);
+ }
+ if (*pos) {
+ /* pos is &(elt)->next */
+ nhdr->next = (*pos);
+ nhdr->prev = (*pos)->prev;
+ (*pos)->prev = nhdr;
+ *pos = nhdr;
+ }
+ else {
+ /* Last element */
+ DL_APPEND(*pos, nhdr);
+ }
+ }
+ else {
+ /* NYI: negative order is not defined */
+ msg_err_task("internal error: calling for set_modified_header "
+ "with negative add order header");
+ }
+ }
+ else {
+ msg_err_task("internal error: calling for set_modified_header "
+ "with invalid header");
+ }
+ }
+ }
+ }
+}
+
+gsize rspamd_strip_smtp_comments_inplace(gchar *input, gsize len)
+{
+ enum parser_state {
+ parse_normal,
+ parse_obrace,
+ parse_comment,
+ parse_quoted_copy,
+ parse_quoted_ignore,
+ } state = parse_normal,
+ next_state = parse_normal;
+ gchar *d = input, *end = input + len, *start = input;
+ gchar t;
+ int obraces = 0, ebraces = 0;
+
+ while (input < end) {
+ t = *input;
+ switch (state) {
+ case parse_normal:
+ if (t == '(') {
+ state = parse_obrace;
+ }
+ else if (t == '\\') {
+ state = parse_quoted_copy;
+ next_state = parse_normal;
+ }
+ else {
+ *d++ = t;
+ }
+ input++;
+ break;
+ case parse_obrace:
+ obraces++;
+ if (t == '(') {
+ obraces++;
+ }
+ else if (t == ')') {
+ ebraces++;
+
+ if (obraces == ebraces) {
+ obraces = 0;
+ ebraces = 0;
+ state = parse_normal;
+ }
+ }
+ else if (t == '\\') {
+ state = parse_quoted_ignore;
+ next_state = parse_comment;
+ }
+ else {
+ state = parse_comment;
+ }
+ input++;
+ break;
+ case parse_comment:
+ if (t == '(') {
+ state = parse_obrace;
+ }
+ else if (t == ')') {
+ ebraces++;
+
+ if (obraces == ebraces) {
+ obraces = 0;
+ ebraces = 0;
+ state = parse_normal;
+ }
+ }
+ else if (t == '\\') {
+ state = parse_quoted_ignore;
+ next_state = parse_comment;
+ }
+ input++;
+ break;
+ case parse_quoted_copy:
+ *d++ = t;
+ state = next_state;
+ input++;
+ break;
+ case parse_quoted_ignore:
+ state = next_state;
+ input++;
+ break;
+ }
+ }
+
+ return (d - start);
+} \ No newline at end of file
diff --git a/src/libmime/mime_headers.h b/src/libmime/mime_headers.h
new file mode 100644
index 0000000..60015a2
--- /dev/null
+++ b/src/libmime/mime_headers.h
@@ -0,0 +1,200 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_MIME_HEADERS_H_
+#define SRC_LIBMIME_MIME_HEADERS_H_
+
+#include "config.h"
+#include "libutil/mem_pool.h"
+#include "libutil/addr.h"
+#include "khash.h"
+#include "contrib/libucl/ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+
+enum rspamd_rfc2047_encoding {
+ RSPAMD_RFC2047_QP = 0,
+ RSPAMD_RFC2047_BASE64,
+};
+
+enum rspamd_mime_header_flags {
+ RSPAMD_HEADER_GENERIC = 0u,
+ RSPAMD_HEADER_RECEIVED = 1u << 0u,
+ RSPAMD_HEADER_TO = 1u << 2u,
+ RSPAMD_HEADER_CC = 1u << 3u,
+ RSPAMD_HEADER_BCC = 1u << 4u,
+ RSPAMD_HEADER_FROM = 1u << 5u,
+ RSPAMD_HEADER_MESSAGE_ID = 1u << 6u,
+ RSPAMD_HEADER_SUBJECT = 1u << 7u,
+ RSPAMD_HEADER_RETURN_PATH = 1u << 8u,
+ RSPAMD_HEADER_DELIVERED_TO = 1u << 9u,
+ RSPAMD_HEADER_SENDER = 1u << 10u,
+ RSPAMD_HEADER_RCPT = 1u << 11u,
+ RSPAMD_HEADER_UNIQUE = 1u << 12u,
+ RSPAMD_HEADER_EMPTY_SEPARATOR = 1u << 13u,
+ RSPAMD_HEADER_TAB_SEPARATED = 1u << 14u,
+ RSPAMD_HEADER_MODIFIED = 1u << 15u, /* Means we need to check modified chain */
+ RSPAMD_HEADER_ADDED = 1u << 16u, /* A header has been artificially added */
+ RSPAMD_HEADER_REMOVED = 1u << 17u, /* A header has been artificially removed */
+ RSPAMD_HEADER_NON_EXISTING = 1u << 18u, /* Header was not in the original message */
+};
+
+struct rspamd_mime_header {
+ const gchar *raw_value; /* As it is in the message (unfolded and unparsed) */
+ gsize raw_len;
+ guint order;
+ int flags; /* see enum rspamd_mime_header_flags */
+ /* These are zero terminated (historically) */
+ gchar *name; /* Also used for key */
+ gchar *value;
+ gchar *separator;
+ gchar *decoded;
+ struct rspamd_mime_header *modified_chain; /* Headers modified during transform */
+ struct rspamd_mime_header *prev, *next; /* Headers with the same name */
+ struct rspamd_mime_header *ord_next; /* Overall order of headers, slist */
+};
+
+struct rspamd_mime_headers_table;
+
+/**
+ * Process headers and store them in `target`
+ * @param task
+ * @param target
+ * @param in
+ * @param len
+ * @param check_newlines
+ */
+void rspamd_mime_headers_process(struct rspamd_task *task,
+ struct rspamd_mime_headers_table *target,
+ struct rspamd_mime_header **order_ptr,
+ const gchar *in, gsize len,
+ gboolean check_newlines);
+
+/**
+ * Perform rfc2047 decoding of a header
+ * @param pool
+ * @param in
+ * @param inlen
+ * @return
+ */
+gchar *rspamd_mime_header_decode(rspamd_mempool_t *pool, const gchar *in,
+ gsize inlen, gboolean *invalid_utf);
+
+/**
+ * Encode mime header if needed
+ * @param in
+ * @param len
+ * @return newly allocated encoded header
+ */
+gchar *rspamd_mime_header_encode(const gchar *in, gsize len);
+
+/**
+ * Generate new unique message id
+ * @param fqdn
+ * @return
+ */
+gchar *rspamd_mime_message_id_generate(const gchar *fqdn);
+
+/**
+ * Get an array of header's values with specified header's name using raw headers
+ * @param task worker task structure
+ * @param field header's name
+ * @return An array of header's values or NULL. It is NOT permitted to free array or values.
+ */
+struct rspamd_mime_header *
+rspamd_message_get_header_array(struct rspamd_task *task,
+ const gchar *field,
+ gboolean need_modified);
+
+/**
+ * Get an array of header's values with specified header's name using raw headers
+ * @param htb hash table indexed by header name (caseless) with ptr arrays as elements
+ * @param field header's name
+ * @return An array of header's values or NULL. It is NOT permitted to free array or values.
+ */
+struct rspamd_mime_header *
+rspamd_message_get_header_from_hash(struct rspamd_mime_headers_table *hdrs,
+ const gchar *field,
+ gboolean need_modified);
+
+/**
+ * Modifies a header (or insert one if not found)
+ * @param hdrs
+ * @param hdr_name
+ * @param obj an array of modified values
+ *
+ */
+void rspamd_message_set_modified_header(struct rspamd_task *task,
+ struct rspamd_mime_headers_table *hdrs,
+ const gchar *hdr_name,
+ const ucl_object_t *obj,
+ struct rspamd_mime_header **order_ptr);
+
+/**
+ * Cleans up hash table of the headers
+ * @param htb
+ */
+void rspamd_message_headers_unref(struct rspamd_mime_headers_table *hdrs);
+
+struct rspamd_mime_headers_table *rspamd_message_headers_ref(struct rspamd_mime_headers_table *hdrs);
+
+/**
+ * Init headers hash
+ * @return
+ */
+struct rspamd_mime_headers_table *rspamd_message_headers_new(void);
+
+/**
+ * Returns size for a headers table
+ * @param hdrs
+ * @return
+ */
+gsize rspamd_mime_headers_count(struct rspamd_mime_headers_table *hdrs);
+
+typedef bool(rspamd_hdr_traverse_func_t)(const gchar *, const struct rspamd_mime_header *, void *);
+/**
+ * Traverse all headers in a table
+ * @param func
+ * @param ud
+ * @return
+ */
+bool rspamd_mime_headers_foreach(const struct rspamd_mime_headers_table *,
+ rspamd_hdr_traverse_func_t func, void *ud);
+
+/**
+ * Strip rfc822 CFWS sequences from a string in place
+ * @param input input
+ * @param len length of the input
+ * @return new length of the input
+ */
+gsize rspamd_strip_smtp_comments_inplace(gchar *input, gsize len);
+
+/**
+ * Unfold header in place
+ * @param hdr header value
+ * @param len length of the header
+ * @return new unfolded length
+ */
+gsize rspamd_message_header_unfold_inplace(char *hdr, gsize len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_MIME_HEADERS_H_ */
diff --git a/src/libmime/mime_parser.c b/src/libmime/mime_parser.c
new file mode 100644
index 0000000..217f0b8
--- /dev/null
+++ b/src/libmime/mime_parser.c
@@ -0,0 +1,1758 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include "config.h"
+#include "task.h"
+#include "mime_parser.h"
+#include "mime_headers.h"
+#include "message.h"
+#include "multipattern.h"
+#include "contrib/libottery/ottery.h"
+#include "contrib/uthash/utlist.h"
+#include <openssl/cms.h>
+#include <openssl/pkcs7.h>
+#include "contrib/fastutf8/fastutf8.h"
+
+struct rspamd_mime_parser_lib_ctx {
+ struct rspamd_multipattern *mp_boundary;
+ guchar hkey[rspamd_cryptobox_SIPKEYBYTES]; /* Key for hashing */
+ guint key_usages;
+};
+
+struct rspamd_mime_parser_lib_ctx *lib_ctx = NULL;
+
+static const guint max_nested = 64;
+static const guint max_key_usages = 10000;
+
+#define msg_debug_mime(...) rspamd_conditional_debug_fast(NULL, task->from_addr, \
+ rspamd_mime_log_id, "mime", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(mime)
+
+#define RSPAMD_MIME_BOUNDARY_FLAG_CLOSED (1 << 0)
+#define RSPAMD_BOUNDARY_IS_CLOSED(b) ((b)->flags & RSPAMD_MIME_BOUNDARY_FLAG_CLOSED)
+
+struct rspamd_mime_boundary {
+ goffset boundary;
+ goffset start;
+ guint64 hash;
+ guint64 closed_hash;
+ gint flags;
+};
+
+struct rspamd_mime_parser_ctx {
+ GPtrArray *stack; /* Stack of parts */
+ GArray *boundaries; /* Boundaries found in the whole message */
+ const gchar *start;
+ const gchar *pos;
+ const gchar *end;
+ struct rspamd_task *task;
+ guint nesting;
+};
+
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_multipart_part(struct rspamd_task *task,
+ struct rspamd_mime_part *part,
+ struct rspamd_mime_parser_ctx *st,
+ GError **err);
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_message(struct rspamd_task *task,
+ struct rspamd_mime_part *part,
+ struct rspamd_mime_parser_ctx *st,
+ GError **err);
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_normal_part(struct rspamd_task *task,
+ struct rspamd_mime_part *part,
+ struct rspamd_mime_parser_ctx *st,
+ struct rspamd_content_type *ct,
+ GError **err);
+
+static enum rspamd_mime_parse_error
+rspamd_mime_process_multipart_node(struct rspamd_task *task,
+ struct rspamd_mime_parser_ctx *st,
+ struct rspamd_mime_part *multipart,
+ const gchar *start, const gchar *end,
+ gboolean is_finished,
+ GError **err);
+
+
+#define RSPAMD_MIME_QUARK (rspamd_mime_parser_quark())
+static GQuark
+rspamd_mime_parser_quark(void)
+{
+ return g_quark_from_static_string("mime-parser");
+}
+
+const gchar *
+rspamd_cte_to_string(enum rspamd_cte ct)
+{
+ const gchar *ret = "unknown";
+
+ switch (ct) {
+ case RSPAMD_CTE_7BIT:
+ ret = "7bit";
+ break;
+ case RSPAMD_CTE_8BIT:
+ ret = "8bit";
+ break;
+ case RSPAMD_CTE_QP:
+ ret = "quoted-printable";
+ break;
+ case RSPAMD_CTE_B64:
+ ret = "base64";
+ break;
+ case RSPAMD_CTE_UUE:
+ ret = "X-uuencode";
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+enum rspamd_cte
+rspamd_cte_from_string(const gchar *str)
+{
+ enum rspamd_cte ret = RSPAMD_CTE_UNKNOWN;
+
+ g_assert(str != NULL);
+
+ if (strcmp(str, "7bit") == 0) {
+ ret = RSPAMD_CTE_7BIT;
+ }
+ else if (strcmp(str, "8bit") == 0) {
+ ret = RSPAMD_CTE_8BIT;
+ }
+ else if (strcmp(str, "quoted-printable") == 0) {
+ ret = RSPAMD_CTE_QP;
+ }
+ else if (strcmp(str, "base64") == 0) {
+ ret = RSPAMD_CTE_B64;
+ }
+ else if (strcmp(str, "X-uuencode") == 0) {
+ ret = RSPAMD_CTE_UUE;
+ }
+ else if (strcmp(str, "uuencode") == 0) {
+ ret = RSPAMD_CTE_UUE;
+ }
+ else if (strcmp(str, "X-uue") == 0) {
+ ret = RSPAMD_CTE_UUE;
+ }
+
+ return ret;
+}
+
+static void
+rspamd_mime_parser_init_lib(void)
+{
+ lib_ctx = g_malloc0(sizeof(*lib_ctx));
+ lib_ctx->mp_boundary = rspamd_multipattern_create(RSPAMD_MULTIPATTERN_DEFAULT);
+ g_assert(lib_ctx->mp_boundary != NULL);
+ rspamd_multipattern_add_pattern(lib_ctx->mp_boundary, "\r--", 0);
+ rspamd_multipattern_add_pattern(lib_ctx->mp_boundary, "\n--", 0);
+
+ GError *err = NULL;
+ if (!rspamd_multipattern_compile(lib_ctx->mp_boundary, &err)) {
+ msg_err("fatal error: cannot compile multipattern for mime parser boundaries: %e", err);
+ g_error_free(err);
+ g_abort();
+ }
+ ottery_rand_bytes(lib_ctx->hkey, sizeof(lib_ctx->hkey));
+}
+
+static enum rspamd_cte
+rspamd_mime_parse_cte(const gchar *in, gsize len)
+{
+ guint64 h;
+ enum rspamd_cte ret = RSPAMD_CTE_UNKNOWN;
+
+ in = rspamd_string_len_strip(in, &len, " \t;,.+-#!`~'");
+ h = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64,
+ in, len, 0xdeadbabe);
+
+ switch (h) {
+ case 0xCEDAA7056B4753F7ULL: /* 7bit */
+ ret = RSPAMD_CTE_7BIT;
+ break;
+ case 0x42E0745448B39FC1ULL: /* 8bit */
+ case 0x6B169E6B155BADC0ULL: /* binary */
+ ret = RSPAMD_CTE_8BIT;
+ break;
+ case 0x6D69A5BB02A633B0ULL: /* quoted-printable */
+ ret = RSPAMD_CTE_QP;
+ break;
+ case 0x96305588A76DC9A9ULL: /* base64 */
+ case 0x171029DE1B0423A9ULL: /* base-64 */
+ ret = RSPAMD_CTE_B64;
+ break;
+ case 0x420b54dc00d13cecULL: /* uuencode */
+ case 0x8df6700b8f6c4cf9ULL: /* x-uuencode */
+ case 0x41f725ec544356d3ULL: /* x-uue */
+ ret = RSPAMD_CTE_UUE;
+ break;
+ }
+
+ return ret;
+}
+
+static enum rspamd_cte
+rspamd_mime_part_get_cte_heuristic(struct rspamd_task *task,
+ struct rspamd_mime_part *part)
+{
+ const guint check_len = 128;
+ guint real_len, nspaces = 0, neqsign = 0, n8bit = 0, nqpencoded = 0,
+ padeqsign = 0, nupper = 0, nlower = 0;
+ gboolean b64_chars = TRUE;
+ const guchar *p, *end;
+ enum rspamd_cte ret = RSPAMD_CTE_UNKNOWN;
+
+ real_len = MIN(check_len, part->raw_data.len);
+ p = (const guchar *) part->raw_data.begin;
+ end = p + part->raw_data.len;
+
+ while (p < end && g_ascii_isspace(*p)) {
+ p++;
+ }
+
+ if (end - p > sizeof("begin-base64 ")) {
+ const guchar *uue_start;
+
+ if (memcmp(p, "begin ", sizeof("begin ") - 1) == 0) {
+ uue_start = p + sizeof("begin ") - 1;
+
+ while (uue_start < end && g_ascii_isspace(*uue_start)) {
+ uue_start++;
+ }
+
+ if (uue_start < end && g_ascii_isdigit(*uue_start)) {
+ return RSPAMD_CTE_UUE;
+ }
+ }
+ else if (memcmp(p, "begin-base64 ", sizeof("begin-base64 ") - 1) == 0) {
+ uue_start = p + sizeof("begin ") - 1;
+
+ while (uue_start < end && g_ascii_isspace(*uue_start)) {
+ uue_start++;
+ }
+
+ if (uue_start < end && g_ascii_isdigit(*uue_start)) {
+ return RSPAMD_CTE_UUE;
+ }
+ }
+ }
+
+ /* Skip trailing spaces */
+ while (end > p && g_ascii_isspace(*(end - 1))) {
+ end--;
+ }
+
+ if (end > p + 2) {
+ if (*(end - 1) == '=') {
+ padeqsign++;
+ end--;
+ }
+
+ if (*(end - 1) == '=') {
+ padeqsign++;
+ end--;
+ }
+ }
+
+ /* Adjust end to analyse only first characters */
+ if (end - p > real_len) {
+ end = p + real_len;
+ }
+
+ while (p < end) {
+ if (*p == ' ') {
+ nspaces++;
+ }
+ else if (*p == '=') {
+ b64_chars = FALSE; /* Eqsign must not be inside base64 */
+ neqsign++;
+ p++;
+
+ if (p + 2 < end && g_ascii_isxdigit(*p) && g_ascii_isxdigit(*(p + 1))) {
+ p++;
+ nqpencoded++;
+ }
+
+ continue;
+ }
+ else if (*p >= 0x80) {
+ n8bit++;
+ b64_chars = FALSE;
+ }
+ else if (!(g_ascii_isalnum(*p) || *p == '/' || *p == '+')) {
+ b64_chars = FALSE;
+ }
+ else if (g_ascii_isupper(*p)) {
+ nupper++;
+ }
+ else if (g_ascii_islower(*p)) {
+ nlower++;
+ }
+
+ p++;
+ }
+
+ if (b64_chars && neqsign <= 2 && nspaces == 0) {
+ /* Need more thinking */
+
+ if (part->raw_data.len > 80) {
+ if (padeqsign > 0) {
+ ret = RSPAMD_CTE_B64;
+ }
+ else {
+ /* We have a large piece of data with no spaces and base64
+ * symbols only, no padding is detected as well...
+ *
+ * There is a small chance that our first 128 characters
+ * are either some garbage or it is a base64 with no padding
+ * (e.g. when it is not needed)
+ */
+ if (nupper > 1 && nlower > 1) {
+ /*
+ * We have both uppercase and lowercase letters, so it can be
+ * base64
+ */
+ ret = RSPAMD_CTE_B64;
+ }
+ else {
+ ret = RSPAMD_CTE_7BIT;
+ }
+ }
+ }
+ else {
+
+ if (((end - (const guchar *) part->raw_data.begin) + padeqsign) % 4 == 0) {
+ if (padeqsign == 0) {
+ /*
+ * It can be either base64 or plain text, hard to say
+ * Let's assume that if we have > 1 uppercase it is
+ * likely base64
+ */
+ if (nupper > 1 && nlower > 1) {
+ ret = RSPAMD_CTE_B64;
+ }
+ else {
+ ret = RSPAMD_CTE_7BIT;
+ }
+ }
+ else {
+ ret = RSPAMD_CTE_B64;
+ }
+ }
+ else {
+ /* No way */
+ if (padeqsign == 1 || padeqsign == 2) {
+ ret = RSPAMD_CTE_B64;
+ }
+ else {
+ ret = RSPAMD_CTE_7BIT;
+ }
+ }
+ }
+ }
+ else if (n8bit == 0) {
+ if (neqsign > 2 && nqpencoded > 2) {
+ ret = RSPAMD_CTE_QP;
+ }
+ else {
+ ret = RSPAMD_CTE_7BIT;
+ }
+ }
+ else {
+ ret = RSPAMD_CTE_8BIT;
+ }
+
+ msg_debug_mime("detected cte: %s", rspamd_cte_to_string(ret));
+
+ return ret;
+}
+
+static void
+rspamd_mime_part_get_cte(struct rspamd_task *task,
+ struct rspamd_mime_headers_table *hdrs,
+ struct rspamd_mime_part *part,
+ gboolean apply_heuristic)
+{
+ struct rspamd_mime_header *hdr, *cur;
+ enum rspamd_cte cte = RSPAMD_CTE_UNKNOWN;
+ gboolean parent_propagated = FALSE;
+
+ hdr = rspamd_message_get_header_from_hash(hdrs, "Content-Transfer-Encoding", FALSE);
+
+ if (hdr == NULL) {
+ if (part->parent_part && part->parent_part->cte != RSPAMD_CTE_UNKNOWN &&
+ !(part->parent_part->flags & RSPAMD_MIME_PART_MISSING_CTE)) {
+ part->cte = part->parent_part->cte;
+ parent_propagated = TRUE;
+
+ goto check_cte;
+ }
+
+ if (apply_heuristic) {
+ part->cte = rspamd_mime_part_get_cte_heuristic(task, part);
+ msg_info_task("detected missing CTE for part as: %s",
+ rspamd_cte_to_string(part->cte));
+ }
+
+ part->flags |= RSPAMD_MIME_PART_MISSING_CTE;
+ }
+ else {
+ DL_FOREACH(hdr, cur)
+ {
+ gsize hlen;
+ gchar lc_buf[128];
+
+ hlen = rspamd_snprintf(lc_buf, sizeof(lc_buf), "%s", cur->value);
+ rspamd_str_lc(lc_buf, hlen);
+ cte = rspamd_mime_parse_cte(lc_buf, hlen);
+
+ if (cte != RSPAMD_CTE_UNKNOWN) {
+ part->cte = cte;
+ break;
+ }
+ }
+
+ check_cte:
+ if (apply_heuristic) {
+ if (part->cte == RSPAMD_CTE_UNKNOWN) {
+ part->cte = rspamd_mime_part_get_cte_heuristic(task, part);
+
+ msg_info_task("corrected bad CTE for part to: %s",
+ rspamd_cte_to_string(part->cte));
+ }
+ else if (part->cte == RSPAMD_CTE_B64 ||
+ part->cte == RSPAMD_CTE_QP) {
+ /* Additionally check sanity */
+ cte = rspamd_mime_part_get_cte_heuristic(task, part);
+
+ if (cte == RSPAMD_CTE_8BIT) {
+ msg_info_task(
+ "incorrect cte specified for part: %s, %s detected",
+ rspamd_cte_to_string(part->cte),
+ rspamd_cte_to_string(cte));
+ part->cte = cte;
+ part->flags |= RSPAMD_MIME_PART_BAD_CTE;
+ }
+ else if (cte != part->cte && parent_propagated) {
+ part->cte = cte;
+ msg_info_task("detected missing CTE for part as: %s",
+ rspamd_cte_to_string(part->cte));
+ }
+ }
+ else {
+ msg_debug_mime("processed cte: %s",
+ rspamd_cte_to_string(cte));
+ }
+ }
+ else {
+ msg_debug_mime("processed cte: %s", rspamd_cte_to_string(cte));
+ }
+ }
+}
+static void
+rspamd_mime_part_get_cd(struct rspamd_task *task, struct rspamd_mime_part *part)
+{
+ struct rspamd_mime_header *hdr, *cur;
+ struct rspamd_content_disposition *cd = NULL;
+ rspamd_ftok_t srch;
+ struct rspamd_content_type_param *found;
+
+ hdr = rspamd_message_get_header_from_hash(part->raw_headers,
+ "Content-Disposition", FALSE);
+
+
+ if (hdr == NULL) {
+ cd = rspamd_mempool_alloc0(task->task_pool, sizeof(*cd));
+ cd->type = RSPAMD_CT_INLINE;
+
+ /* We can also have content disposition definitions in Content-Type */
+ if (part->ct && part->ct->attrs) {
+ RSPAMD_FTOK_ASSIGN(&srch, "name");
+ found = g_hash_table_lookup(part->ct->attrs, &srch);
+
+ if (!found) {
+ RSPAMD_FTOK_ASSIGN(&srch, "filename");
+ found = g_hash_table_lookup(part->ct->attrs, &srch);
+ }
+
+ if (found) {
+ cd->type = RSPAMD_CT_ATTACHMENT;
+ memcpy(&cd->filename, &found->value, sizeof(cd->filename));
+ }
+ }
+ }
+ else {
+ DL_FOREACH(hdr, cur)
+ {
+ gsize hlen;
+ cd = NULL;
+
+ if (cur->value) {
+ hlen = strlen(cur->value);
+ cd = rspamd_content_disposition_parse(cur->value, hlen,
+ task->task_pool);
+ }
+
+ if (cd) {
+ /* We still need to check filename */
+ if (cd->filename.len == 0) {
+ if (part->ct && part->ct->attrs) {
+ RSPAMD_FTOK_ASSIGN(&srch, "name");
+ found = g_hash_table_lookup(part->ct->attrs, &srch);
+
+ if (!found) {
+ RSPAMD_FTOK_ASSIGN(&srch, "filename");
+ found = g_hash_table_lookup(part->ct->attrs, &srch);
+ }
+
+ if (found) {
+ cd->type = RSPAMD_CT_ATTACHMENT;
+ memcpy(&cd->filename, &found->value,
+ sizeof(cd->filename));
+ }
+ }
+ }
+
+ msg_debug_mime("processed content disposition: %s, file: \"%T\"",
+ cd->lc_data, &cd->filename);
+ break;
+ }
+ else if (part->ct) {
+ /*
+ * Even in case of malformed Content-Disposition, we can still
+ * fall back to Content-Type
+ */
+ cd = rspamd_mempool_alloc0(task->task_pool, sizeof(*cd));
+ cd->type = RSPAMD_CT_INLINE;
+
+ /* We can also have content disposition definitions in Content-Type */
+ if (part->ct->attrs) {
+ RSPAMD_FTOK_ASSIGN(&srch, "name");
+ found = g_hash_table_lookup(part->ct->attrs, &srch);
+
+ if (!found) {
+ RSPAMD_FTOK_ASSIGN(&srch, "filename");
+ found = g_hash_table_lookup(part->ct->attrs, &srch);
+ }
+
+ if (found) {
+ cd->type = RSPAMD_CT_ATTACHMENT;
+ memcpy(&cd->filename, &found->value, sizeof(cd->filename));
+ }
+ }
+ }
+ }
+ }
+
+ part->cd = cd;
+}
+
+void rspamd_mime_parser_calc_digest(struct rspamd_mime_part *part)
+{
+ /* Blake2b applied to string 'rspamd' */
+ static const guchar hash_key[] = {
+ 0xef,
+ 0x43,
+ 0xae,
+ 0x80,
+ 0xcc,
+ 0x8d,
+ 0xc3,
+ 0x4c,
+ 0x6f,
+ 0x1b,
+ 0xd6,
+ 0x18,
+ 0x1b,
+ 0xae,
+ 0x87,
+ 0x74,
+ 0x0c,
+ 0xca,
+ 0xf7,
+ 0x8e,
+ 0x5f,
+ 0x2e,
+ 0x54,
+ 0x32,
+ 0xf6,
+ 0x79,
+ 0xb9,
+ 0x27,
+ 0x26,
+ 0x96,
+ 0x20,
+ 0x92,
+ 0x70,
+ 0x07,
+ 0x85,
+ 0xeb,
+ 0x83,
+ 0xf7,
+ 0x89,
+ 0xe0,
+ 0xd7,
+ 0x32,
+ 0x2a,
+ 0xd2,
+ 0x1a,
+ 0x64,
+ 0x41,
+ 0xef,
+ 0x49,
+ 0xff,
+ 0xc3,
+ 0x8c,
+ 0x54,
+ 0xf9,
+ 0x67,
+ 0x74,
+ 0x30,
+ 0x1e,
+ 0x70,
+ 0x2e,
+ 0xb7,
+ 0x12,
+ 0x09,
+ 0xfe,
+ };
+
+ if (part->parsed_data.len > 0) {
+ rspamd_cryptobox_hash(part->digest,
+ part->parsed_data.begin, part->parsed_data.len,
+ hash_key, sizeof(hash_key));
+ }
+}
+
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_normal_part(struct rspamd_task *task,
+ struct rspamd_mime_part *part,
+ struct rspamd_mime_parser_ctx *st,
+ struct rspamd_content_type *ct,
+ GError **err)
+{
+ rspamd_fstring_t *parsed;
+ gssize r;
+
+ g_assert(part != NULL);
+
+ rspamd_mime_part_get_cte(task, part->raw_headers, part,
+ part->ct && !(part->ct->flags & RSPAMD_CONTENT_TYPE_MESSAGE));
+ rspamd_mime_part_get_cd(task, part);
+
+ switch (part->cte) {
+ case RSPAMD_CTE_7BIT:
+ case RSPAMD_CTE_8BIT:
+ case RSPAMD_CTE_UNKNOWN:
+ if (part->ct && (part->ct->flags & RSPAMD_CONTENT_TYPE_MISSING)) {
+ if (part->cte != RSPAMD_CTE_7BIT) {
+ /* We have something that has a missing content-type,
+ * but it has non-7bit characters.
+ *
+ * In theory, it is very unsafe to process it as a text part
+ * as we unlikely get some sane result
+ */
+
+ /*
+ * On the other hand, there is an evidence that some
+ * emails actually rely on that.
+ * So we apply an expensive hack here:
+ * if there are no 8bit characters -OR- the content is valid
+ * UTF8, we can still imply Content-Type == text/plain
+ */
+
+ if (rspamd_str_has_8bit(part->raw_data.begin, part->raw_data.len) &&
+ !rspamd_fast_utf8_validate(part->raw_data.begin, part->raw_data.len)) {
+ part->ct->flags &= ~RSPAMD_CONTENT_TYPE_TEXT;
+ part->ct->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+ }
+ }
+ }
+
+ if (part->ct && (part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT)) {
+ /* Need to copy text as we have couple of in-place change functions */
+ parsed = rspamd_fstring_sized_new(part->raw_data.len);
+ parsed->len = part->raw_data.len;
+ memcpy(parsed->str, part->raw_data.begin, parsed->len);
+ part->parsed_data.begin = parsed->str;
+ part->parsed_data.len = parsed->len;
+ rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) rspamd_fstring_free, parsed);
+ }
+ else {
+ part->parsed_data.begin = part->raw_data.begin;
+ part->parsed_data.len = part->raw_data.len;
+ }
+ break;
+ case RSPAMD_CTE_QP:
+ parsed = rspamd_fstring_sized_new(part->raw_data.len);
+ r = rspamd_decode_qp_buf(part->raw_data.begin, part->raw_data.len,
+ parsed->str, parsed->allocated);
+ if (r != -1) {
+ parsed->len = r;
+ part->parsed_data.begin = parsed->str;
+ part->parsed_data.len = parsed->len;
+ rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) rspamd_fstring_free, parsed);
+ }
+ else {
+ msg_err_task("invalid quoted-printable encoded part, assume 8bit");
+ if (part->ct) {
+ part->ct->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+ }
+ part->cte = RSPAMD_CTE_8BIT;
+ memcpy(parsed->str, part->raw_data.begin, part->raw_data.len);
+ parsed->len = part->raw_data.len;
+ part->parsed_data.begin = parsed->str;
+ part->parsed_data.len = parsed->len;
+ rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) rspamd_fstring_free, parsed);
+ }
+ break;
+ case RSPAMD_CTE_B64:
+ parsed = rspamd_fstring_sized_new(part->raw_data.len / 4 * 3 + 12);
+ rspamd_cryptobox_base64_decode(part->raw_data.begin,
+ part->raw_data.len,
+ parsed->str, &parsed->len);
+ part->parsed_data.begin = parsed->str;
+ part->parsed_data.len = parsed->len;
+ rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) rspamd_fstring_free, parsed);
+ break;
+ case RSPAMD_CTE_UUE:
+ parsed = rspamd_fstring_sized_new(part->raw_data.len / 4 * 3 + 12);
+ r = rspamd_decode_uue_buf(part->raw_data.begin, part->raw_data.len,
+ parsed->str, parsed->allocated);
+ rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) rspamd_fstring_free, parsed);
+ if (r != -1) {
+ parsed->len = r;
+ part->parsed_data.begin = parsed->str;
+ part->parsed_data.len = parsed->len;
+ }
+ else {
+ msg_err_task("invalid uuencoding in encoded part, assume 8bit");
+ if (part->ct) {
+ part->ct->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+ }
+ part->cte = RSPAMD_CTE_8BIT;
+ parsed->len = MIN(part->raw_data.len, parsed->allocated);
+ memcpy(parsed->str, part->raw_data.begin, parsed->len);
+ rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+ part->parsed_data.begin = parsed->str;
+ part->parsed_data.len = parsed->len;
+ }
+ break;
+ default:
+ g_assert_not_reached();
+ }
+
+ part->part_number = MESSAGE_FIELD(task, parts)->len;
+ part->urls = g_ptr_array_new();
+ g_ptr_array_add(MESSAGE_FIELD(task, parts), part);
+ msg_debug_mime("parsed data part %T/%T of length %z (%z orig), %s cte",
+ &part->ct->type, &part->ct->subtype, part->parsed_data.len,
+ part->raw_data.len, rspamd_cte_to_string(part->cte));
+ rspamd_mime_parser_calc_digest(part);
+
+ if (ct && (ct->flags & RSPAMD_CONTENT_TYPE_SMIME)) {
+ CMS_ContentInfo *cms;
+ const unsigned char *der_beg = part->parsed_data.begin;
+ cms = d2i_CMS_ContentInfo(NULL, &der_beg, part->parsed_data.len);
+
+ if (cms) {
+ const ASN1_OBJECT *asn_ct = CMS_get0_eContentType(cms);
+ int ct_nid = OBJ_obj2nid(asn_ct);
+
+ if (ct_nid == NID_pkcs7_data) {
+ BIO *bio = BIO_new_mem_buf(part->parsed_data.begin,
+ part->parsed_data.len);
+
+ PKCS7 *p7;
+ p7 = d2i_PKCS7_bio(bio, NULL);
+
+ if (p7) {
+ ct_nid = OBJ_obj2nid(p7->type);
+
+ if (ct_nid == NID_pkcs7_signed) {
+ PKCS7 *p7_signed_content = p7->d.sign->contents;
+
+ ct_nid = OBJ_obj2nid(p7_signed_content->type);
+
+ if (ct_nid == NID_pkcs7_data && p7_signed_content->d.data) {
+ int ret;
+
+ msg_debug_mime("found an additional part inside of "
+ "smime structure of type %T/%T; length=%d",
+ &ct->type, &ct->subtype, p7_signed_content->d.data->length);
+ /*
+ * Since ASN.1 structures are freed, we need to copy
+ * the content
+ */
+ gchar *cpy = rspamd_mempool_alloc(task->task_pool,
+ p7_signed_content->d.data->length);
+ memcpy(cpy, p7_signed_content->d.data->data,
+ p7_signed_content->d.data->length);
+ ret = rspamd_mime_process_multipart_node(task,
+ st, NULL,
+ cpy, cpy + p7_signed_content->d.data->length,
+ TRUE, err);
+
+ PKCS7_free(p7);
+ BIO_free(bio);
+ CMS_ContentInfo_free(cms);
+
+ return ret;
+ }
+ }
+
+ PKCS7_free(p7);
+ }
+
+ BIO_free(bio);
+ }
+
+ CMS_ContentInfo_free(cms);
+ }
+ }
+
+ return RSPAMD_MIME_PARSE_OK;
+}
+
+struct rspamd_mime_multipart_cbdata {
+ struct rspamd_task *task;
+ struct rspamd_mime_part *multipart;
+ struct rspamd_mime_parser_ctx *st;
+ const gchar *part_start;
+ rspamd_ftok_t *cur_boundary;
+ guint64 bhash;
+ GError **err;
+};
+
+static enum rspamd_mime_parse_error
+rspamd_mime_process_multipart_node(struct rspamd_task *task,
+ struct rspamd_mime_parser_ctx *st,
+ struct rspamd_mime_part *multipart,
+ const gchar *start, const gchar *end,
+ gboolean is_finished,
+ GError **err)
+{
+ struct rspamd_content_type *ct, *sel = NULL;
+ struct rspamd_mime_header *hdr = NULL, *cur;
+ struct rspamd_mime_part *npart;
+ GString str;
+ goffset hdr_pos, body_pos;
+ enum rspamd_mime_parse_error ret = RSPAMD_MIME_PARSE_FATAL;
+
+
+ str.str = (gchar *) start;
+ str.len = end - start;
+
+ if (*start == '\n' || *start == '\r') {
+ /*
+ * We have a part that starts from newline which means that
+ * there are completely no headers in this part,
+ * hence we assume it as a text part
+ */
+ hdr_pos = 0;
+ body_pos = 0;
+
+ if (!is_finished) {
+ /* Ignore garbage */
+ const gchar *p = start;
+ gboolean seen_something = FALSE;
+
+ while (p < end) {
+ if (g_ascii_isalnum(*p)) {
+ seen_something = TRUE;
+ break;
+ }
+ p++;
+ }
+
+ if (!seen_something) {
+ return RSPAMD_MIME_PARSE_NO_PART;
+ }
+ }
+ }
+ else {
+ hdr_pos = rspamd_string_find_eoh(&str, &body_pos);
+ }
+
+ npart = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct rspamd_mime_part));
+ npart->parent_part = multipart;
+ npart->raw_headers = rspamd_message_headers_new();
+ npart->headers_order = NULL;
+
+ if (multipart) {
+ if (multipart->specific.mp->children == NULL) {
+ multipart->specific.mp->children = g_ptr_array_sized_new(2);
+ }
+
+ g_ptr_array_add(multipart->specific.mp->children, npart);
+ }
+
+ if (hdr_pos > 0 && hdr_pos < str.len) {
+ npart->raw_headers_str = str.str;
+ npart->raw_headers_len = hdr_pos;
+ npart->raw_data.begin = start + body_pos;
+ npart->raw_data.len = (end - start) - body_pos;
+
+ if (npart->raw_headers_len > 0) {
+ rspamd_mime_headers_process(task, npart->raw_headers,
+ &npart->headers_order,
+ npart->raw_headers_str,
+ npart->raw_headers_len,
+ FALSE);
+
+ /* Preserve the natural order */
+ if (npart->headers_order) {
+ LL_REVERSE2(npart->headers_order, ord_next);
+ }
+ }
+
+ hdr = rspamd_message_get_header_from_hash(npart->raw_headers,
+ "Content-Type", FALSE);
+ }
+ else {
+ npart->raw_headers_str = 0;
+ npart->raw_headers_len = 0;
+ npart->raw_data.begin = start;
+ npart->raw_data.len = end - start;
+ }
+
+
+ if (hdr != NULL) {
+
+ DL_FOREACH(hdr, cur)
+ {
+ ct = rspamd_content_type_parse(cur->value, strlen(cur->value),
+ task->task_pool);
+
+ /* Here we prefer multipart content-type or any content-type */
+ if (ct) {
+ if (sel == NULL) {
+ sel = ct;
+ }
+ else if (ct->flags & RSPAMD_CONTENT_TYPE_MULTIPART) {
+ sel = ct;
+ }
+ }
+ }
+ }
+
+ if (sel == NULL) {
+ sel = rspamd_mempool_alloc0(task->task_pool, sizeof(*sel));
+ RSPAMD_FTOK_ASSIGN(&sel->type, "text");
+ RSPAMD_FTOK_ASSIGN(&sel->subtype, "plain");
+ }
+
+ npart->ct = sel;
+
+ if (sel->flags & RSPAMD_CONTENT_TYPE_MULTIPART) {
+ st->nesting++;
+ g_ptr_array_add(st->stack, npart);
+ npart->part_type = RSPAMD_MIME_PART_MULTIPART;
+ npart->specific.mp = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct rspamd_mime_multipart));
+ memcpy(&npart->specific.mp->boundary, &sel->orig_boundary,
+ sizeof(rspamd_ftok_t));
+ ret = rspamd_mime_parse_multipart_part(task, npart, st, err);
+ }
+ else if (sel->flags & RSPAMD_CONTENT_TYPE_MESSAGE) {
+ st->nesting++;
+ g_ptr_array_add(st->stack, npart);
+ npart->part_type = RSPAMD_MIME_PART_MESSAGE;
+
+ if ((ret = rspamd_mime_parse_normal_part(task, npart, st, sel, err)) == RSPAMD_MIME_PARSE_OK) {
+ ret = rspamd_mime_parse_message(task, npart, st, err);
+ }
+ }
+ else {
+ ret = rspamd_mime_parse_normal_part(task, npart, st, sel, err);
+ }
+
+ return ret;
+}
+
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_multipart_cb(struct rspamd_task *task,
+ struct rspamd_mime_part *multipart,
+ struct rspamd_mime_parser_ctx *st,
+ struct rspamd_mime_multipart_cbdata *cb,
+ struct rspamd_mime_boundary *b)
+{
+ const gchar *pos = st->start + b->boundary;
+ enum rspamd_mime_parse_error ret;
+
+ task = cb->task;
+
+ /* Now check boundary */
+ if (!cb->part_start) {
+ cb->part_start = st->start + b->start;
+ st->pos = cb->part_start;
+ }
+ else {
+ /*
+ * We have seen the start of the boundary,
+ * but it might be unsuitable (e.g. in broken headers)
+ */
+ if (cb->part_start < pos && cb->cur_boundary) {
+
+ if ((ret = rspamd_mime_process_multipart_node(task, cb->st,
+ cb->multipart, cb->part_start, pos, TRUE, cb->err)) != RSPAMD_MIME_PARSE_OK) {
+ return ret;
+ }
+
+ if (b->start > 0) {
+ /* Go towards the next part */
+ cb->part_start = st->start + b->start;
+ cb->st->pos = cb->part_start;
+ }
+ }
+ else {
+ /* We have an empty boundary, do nothing */
+ }
+ }
+
+ return RSPAMD_MIME_PARSE_OK;
+}
+
+static enum rspamd_mime_parse_error
+rspamd_multipart_boundaries_filter(struct rspamd_task *task,
+ struct rspamd_mime_part *multipart,
+ struct rspamd_mime_parser_ctx *st,
+ struct rspamd_mime_multipart_cbdata *cb)
+{
+ struct rspamd_mime_boundary *cur;
+ goffset last_offset;
+ guint i, sel = 0;
+ enum rspamd_mime_parse_error ret;
+
+ last_offset = (multipart->raw_data.begin - st->start) +
+ multipart->raw_data.len;
+
+ /* Find the first offset suitable for this part */
+ for (i = 0; i < st->boundaries->len; i++) {
+ cur = &g_array_index(st->boundaries, struct rspamd_mime_boundary, i);
+
+ if (cur->start >= multipart->raw_data.begin - st->start) {
+ if (cb->cur_boundary) {
+ /* Check boundary */
+ msg_debug_mime("compare %L and %L (and %L)",
+ cb->bhash, cur->hash, cur->closed_hash);
+
+ if (cb->bhash == cur->hash) {
+ sel = i;
+ break;
+ }
+ else if (cb->bhash == cur->closed_hash) {
+ /* Not a closing element in fact */
+ cur->flags &= ~(RSPAMD_MIME_BOUNDARY_FLAG_CLOSED);
+ cur->hash = cur->closed_hash;
+ sel = i;
+ break;
+ }
+ }
+ else {
+ /* Set current boundary */
+ cb->cur_boundary = rspamd_mempool_alloc(task->task_pool,
+ sizeof(rspamd_ftok_t));
+ cb->cur_boundary->begin = st->start + cur->boundary;
+ cb->cur_boundary->len = 0;
+ cb->bhash = cur->hash;
+ sel = i;
+ break;
+ }
+ }
+ }
+
+ /* Now we can go forward with boundaries that are same to what we have */
+ for (i = sel; i < st->boundaries->len; i++) {
+ cur = &g_array_index(st->boundaries, struct rspamd_mime_boundary, i);
+
+ if (cur->boundary > last_offset) {
+ break;
+ }
+
+ if (cur->hash == cb->bhash || cur->closed_hash == cb->bhash) {
+ if ((ret = rspamd_mime_parse_multipart_cb(task, multipart, st,
+ cb, cur)) != RSPAMD_MIME_PARSE_OK) {
+ return ret;
+ }
+
+ if (cur->closed_hash == cb->bhash) {
+ /* We have again fake closed hash */
+ cur->flags &= ~(RSPAMD_MIME_BOUNDARY_FLAG_CLOSED);
+ cur->hash = cur->closed_hash;
+ }
+
+ if (RSPAMD_BOUNDARY_IS_CLOSED(cur)) {
+ /* We also might check the next boundary... */
+ if (i < st->boundaries->len - 1) {
+ cur = &g_array_index(st->boundaries,
+ struct rspamd_mime_boundary, i + 1);
+
+ if (cur->hash == cb->bhash) {
+ continue;
+ }
+ else if (cur->closed_hash == cb->bhash) {
+ /* We have again fake closed hash */
+ cur->flags &= ~(RSPAMD_MIME_BOUNDARY_FLAG_CLOSED);
+ cur->hash = cur->closed_hash;
+ continue;
+ }
+ }
+
+ break;
+ }
+ }
+ }
+
+ if (i == st->boundaries->len && cb->cur_boundary) {
+ /* Process the last part */
+ struct rspamd_mime_boundary fb;
+
+ fb.boundary = last_offset;
+ fb.start = -1;
+
+ if ((ret = rspamd_mime_parse_multipart_cb(task, multipart, st,
+ cb, &fb)) != RSPAMD_MIME_PARSE_OK) {
+ return ret;
+ }
+ }
+
+ return RSPAMD_MIME_PARSE_OK;
+}
+
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_multipart_part(struct rspamd_task *task,
+ struct rspamd_mime_part *part,
+ struct rspamd_mime_parser_ctx *st,
+ GError **err)
+{
+ struct rspamd_mime_multipart_cbdata cbdata;
+ enum rspamd_mime_parse_error ret;
+
+ if (st->nesting > max_nested) {
+ g_set_error(err, RSPAMD_MIME_QUARK, E2BIG, "Nesting level is too high: %d",
+ st->nesting);
+ return RSPAMD_MIME_PARSE_NESTING;
+ }
+
+ part->part_number = MESSAGE_FIELD(task, parts)->len;
+ part->urls = g_ptr_array_new();
+ g_ptr_array_add(MESSAGE_FIELD(task, parts), part);
+ st->nesting++;
+ rspamd_mime_part_get_cte(task, part->raw_headers, part, FALSE);
+
+ st->pos = part->raw_data.begin;
+ cbdata.multipart = part;
+ cbdata.task = task;
+ cbdata.st = st;
+ cbdata.part_start = NULL;
+ cbdata.err = err;
+
+ if (part->ct->boundary.len > 0) {
+ /* We know our boundary */
+ cbdata.cur_boundary = &part->ct->boundary;
+ rspamd_cryptobox_siphash((guchar *) &cbdata.bhash,
+ cbdata.cur_boundary->begin, cbdata.cur_boundary->len,
+ lib_ctx->hkey);
+ msg_debug_mime("hash: %T -> %L", cbdata.cur_boundary, cbdata.bhash);
+ }
+ else {
+ /* Guess boundary */
+ cbdata.cur_boundary = NULL;
+ cbdata.bhash = 0;
+ }
+
+ ret = rspamd_multipart_boundaries_filter(task, part, st, &cbdata);
+ /* Cleanup stack */
+ st->nesting--;
+ g_ptr_array_remove_index_fast(st->stack, st->stack->len - 1);
+
+ return ret;
+}
+
+/* Process boundary like structures in a message */
+static gint
+rspamd_mime_preprocess_cb(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint match_pos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ const gchar *end = text + len, *p = text + match_pos, *bend;
+ gsize blen;
+ gboolean closing = FALSE;
+ struct rspamd_mime_boundary b;
+ struct rspamd_mime_parser_ctx *st = context;
+ struct rspamd_task *task;
+
+ task = st->task;
+
+ if (G_LIKELY(p < end)) {
+
+ blen = 0;
+
+ while (p < end) {
+ if (*p == '\r' || *p == '\n') {
+ break;
+ }
+
+ blen++;
+ p++;
+ }
+
+ if (blen > 0) {
+ /* We have found something like boundary */
+ p = text + match_pos;
+ bend = p + blen - 1;
+
+ if (*bend == '-') {
+ /* We need to verify last -- */
+ if (bend > p + 1 && *(bend - 1) == '-') {
+ closing = TRUE;
+ bend--;
+ blen -= 2;
+ }
+ else {
+ /* Not a closing boundary somehow, e.g. if a boundary=='-' */
+ bend++;
+ }
+ }
+ else {
+ bend++;
+ }
+
+ while (bend < end) {
+ if (*bend == '\r') {
+ bend++;
+
+ /* \r\n */
+ if (bend < end && *bend == '\n') {
+ bend++;
+ }
+ }
+ else if (*bend == '\n') {
+ /* \n */
+ bend++;
+ }
+ else if (g_ascii_isspace(*bend)) {
+ /* Spaces in the same line, skip them */
+ bend++;
+ continue;
+ }
+
+ break;
+ }
+
+ b.boundary = p - st->start - 2;
+ b.start = bend - st->start;
+
+ /* Small optimisation as boundaries are usually short strings */
+ gchar *lc_copy, lc_copy_buf[128];
+
+ if (blen + 2 < sizeof(lc_copy_buf)) {
+ lc_copy = lc_copy_buf;
+ }
+ else {
+ lc_copy = g_malloc(blen + 2);
+ }
+
+ if (closing) {
+ memcpy(lc_copy, p, blen + 2);
+ rspamd_str_lc(lc_copy, blen + 2);
+ }
+ else {
+ memcpy(lc_copy, p, blen);
+ rspamd_str_lc(lc_copy, blen);
+ }
+
+ rspamd_cryptobox_siphash((guchar *) &b.hash, lc_copy, blen,
+ lib_ctx->hkey);
+ msg_debug_mime("normal hash: %*s -> %L, %d boffset, %d data offset",
+ (gint) blen, lc_copy, b.hash, (int) b.boundary, (int) b.start);
+
+ if (closing) {
+ b.flags = RSPAMD_MIME_BOUNDARY_FLAG_CLOSED;
+ rspamd_cryptobox_siphash((guchar *) &b.closed_hash, lc_copy,
+ blen + 2,
+ lib_ctx->hkey);
+ msg_debug_mime("closing hash: %*s -> %L, %d boffset, %d data offset",
+ (gint) blen + 2, lc_copy,
+ b.closed_hash,
+ (int) b.boundary, (int) b.start);
+ }
+ else {
+ b.flags = 0;
+ b.closed_hash = 0;
+ }
+
+ /* Check if a string has been allocated on the heap */
+ if (blen + 2 >= sizeof(lc_copy_buf)) {
+ g_free(lc_copy);
+ }
+ g_array_append_val(st->boundaries, b);
+ }
+ }
+
+ return 0;
+}
+
+static goffset
+rspamd_mime_parser_headers_heuristic(GString *input, goffset *body_start)
+{
+ const gsize default_max_len = 76;
+ gsize max_len = MIN(input->len, default_max_len);
+ const gchar *p, *end;
+ enum {
+ st_before_colon = 0,
+ st_colon,
+ st_spaces_after_colon,
+ st_value,
+ st_error
+ } state = st_before_colon;
+
+ p = input->str;
+ end = p + max_len;
+
+ while (p < end) {
+ switch (state) {
+ case st_before_colon:
+ if (G_UNLIKELY(*p == ':')) {
+ state = st_colon;
+ }
+ else if (G_UNLIKELY(!g_ascii_isgraph(*p))) {
+ state = st_error;
+ }
+
+ p++;
+ break;
+ case st_colon:
+ if (g_ascii_isspace(*p)) {
+ state = st_spaces_after_colon;
+ }
+ else {
+ state = st_value;
+ }
+ p++;
+ break;
+ case st_spaces_after_colon:
+ if (!g_ascii_isspace(*p)) {
+ state = st_value;
+ }
+ p++;
+ break;
+ case st_value:
+ /* We accept any value */
+ goto end;
+ break;
+ case st_error:
+ return (-1);
+ break;
+ }
+ }
+
+end:
+ if (state == st_value) {
+ if (body_start) {
+ *body_start = input->len;
+ }
+
+ return input->len;
+ }
+
+ return (-1);
+}
+
+static void
+rspamd_mime_preprocess_message(struct rspamd_task *task,
+ struct rspamd_mime_part *top,
+ struct rspamd_mime_parser_ctx *st)
+{
+
+ if (top->raw_data.begin >= st->pos) {
+ rspamd_multipattern_lookup(lib_ctx->mp_boundary,
+ top->raw_data.begin - 1,
+ top->raw_data.len + 1,
+ rspamd_mime_preprocess_cb, st, NULL);
+ }
+ else {
+ rspamd_multipattern_lookup(lib_ctx->mp_boundary,
+ st->pos,
+ st->end - st->pos,
+ rspamd_mime_preprocess_cb, st, NULL);
+ }
+}
+
+static void
+rspamd_mime_parse_stack_free(struct rspamd_mime_parser_ctx *st)
+{
+ if (st) {
+ g_ptr_array_free(st->stack, TRUE);
+ g_array_free(st->boundaries, TRUE);
+ g_free(st);
+ }
+}
+
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_message(struct rspamd_task *task,
+ struct rspamd_mime_part *part,
+ struct rspamd_mime_parser_ctx *st,
+ GError **err)
+{
+ struct rspamd_content_type *ct, *sel = NULL;
+ struct rspamd_mime_header *hdr = NULL, *cur;
+ const gchar *pbegin, *p;
+ gsize plen, len;
+ struct rspamd_mime_part *npart;
+ goffset hdr_pos, body_pos;
+ guint i;
+ enum rspamd_mime_parse_error ret = RSPAMD_MIME_PARSE_OK;
+ GString str;
+ struct rspamd_mime_parser_ctx *nst = st;
+
+ if (st->nesting > max_nested) {
+ g_set_error(err, RSPAMD_MIME_QUARK, E2BIG, "Nesting level is too high: %d",
+ st->nesting);
+ return RSPAMD_MIME_PARSE_NESTING;
+ }
+
+ /* Allocate real part */
+ npart = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct rspamd_mime_part));
+
+ if (part == NULL) {
+ /* Top level message */
+ p = task->msg.begin;
+ len = task->msg.len;
+
+ str.str = (gchar *) p;
+ str.len = len;
+
+ hdr_pos = rspamd_string_find_eoh(&str, &body_pos);
+
+ if (hdr_pos > 0 && hdr_pos < str.len) {
+
+ MESSAGE_FIELD(task, raw_headers_content).begin = str.str;
+ MESSAGE_FIELD(task, raw_headers_content).len = hdr_pos;
+ MESSAGE_FIELD(task, raw_headers_content).body_start = str.str + body_pos;
+
+ if (MESSAGE_FIELD(task, raw_headers_content).len > 0) {
+ rspamd_mime_headers_process(task,
+ MESSAGE_FIELD(task, raw_headers),
+ &MESSAGE_FIELD(task, headers_order),
+ MESSAGE_FIELD(task, raw_headers_content).begin,
+ MESSAGE_FIELD(task, raw_headers_content).len,
+ TRUE);
+ npart->raw_headers = rspamd_message_headers_ref(
+ MESSAGE_FIELD(task, raw_headers));
+
+ /* Preserve the natural order */
+ if (MESSAGE_FIELD(task, headers_order)) {
+ LL_REVERSE2(MESSAGE_FIELD(task, headers_order), ord_next);
+ }
+ }
+
+ hdr = rspamd_message_get_header_from_hash(
+ MESSAGE_FIELD(task, raw_headers),
+ "Content-Type", FALSE);
+ }
+ else {
+ /* First apply heuristic, maybe we have just headers */
+ hdr_pos = rspamd_mime_parser_headers_heuristic(&str, &body_pos);
+
+ if (hdr_pos > 0 && hdr_pos <= str.len) {
+ MESSAGE_FIELD(task, raw_headers_content).begin = str.str;
+ MESSAGE_FIELD(task, raw_headers_content).len = hdr_pos;
+ MESSAGE_FIELD(task, raw_headers_content).body_start = str.str +
+ body_pos;
+
+ if (MESSAGE_FIELD(task, raw_headers_content).len > 0) {
+ rspamd_mime_headers_process(task,
+ MESSAGE_FIELD(task, raw_headers),
+ &MESSAGE_FIELD(task, headers_order),
+ MESSAGE_FIELD(task, raw_headers_content).begin,
+ MESSAGE_FIELD(task, raw_headers_content).len,
+ TRUE);
+ npart->raw_headers = rspamd_message_headers_ref(
+ MESSAGE_FIELD(task, raw_headers));
+
+ /* Preserve the natural order */
+ if (MESSAGE_FIELD(task, headers_order)) {
+ LL_REVERSE2(MESSAGE_FIELD(task, headers_order), ord_next);
+ }
+ }
+
+ hdr = rspamd_message_get_header_from_hash(
+ MESSAGE_FIELD(task, raw_headers),
+ "Content-Type", FALSE);
+ task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
+ }
+ else {
+ body_pos = 0;
+ }
+ }
+
+ pbegin = st->start + body_pos;
+ plen = st->end - pbegin;
+ npart->headers_order = NULL;
+ }
+ else {
+ /*
+ * Here are dragons:
+ * We allocate new parser context as we need to shift pointers
+ */
+ nst = g_malloc0(sizeof(*st));
+ nst->stack = g_ptr_array_sized_new(4);
+ nst->boundaries = g_array_sized_new(FALSE, FALSE,
+ sizeof(struct rspamd_mime_boundary), 8);
+ nst->start = part->parsed_data.begin;
+ nst->end = nst->start + part->parsed_data.len;
+ nst->pos = nst->start;
+ nst->task = st->task;
+ nst->nesting = st->nesting;
+ st->nesting++;
+
+ str.str = (gchar *) part->parsed_data.begin;
+ str.len = part->parsed_data.len;
+
+ hdr_pos = rspamd_string_find_eoh(&str, &body_pos);
+ npart->raw_headers = rspamd_message_headers_new();
+ npart->headers_order = NULL;
+
+ if (hdr_pos > 0 && hdr_pos < str.len) {
+ npart->raw_headers_str = str.str;
+ npart->raw_headers_len = hdr_pos;
+ npart->raw_data.begin = str.str + body_pos;
+
+ if (npart->raw_headers_len > 0) {
+ rspamd_mime_headers_process(task,
+ npart->raw_headers,
+ &npart->headers_order,
+ npart->raw_headers_str,
+ npart->raw_headers_len,
+ FALSE);
+
+ /* Preserve the natural order */
+ if (npart->headers_order) {
+ LL_REVERSE2(npart->headers_order, ord_next);
+ }
+ }
+
+ hdr = rspamd_message_get_header_from_hash(npart->raw_headers,
+ "Content-Type", FALSE);
+ }
+ else {
+ body_pos = 0;
+ }
+
+ pbegin = part->parsed_data.begin + body_pos;
+ plen = part->parsed_data.len - body_pos;
+ }
+
+ npart->raw_data.begin = pbegin;
+ npart->raw_data.len = plen;
+ npart->parent_part = part;
+
+ if (hdr == NULL) {
+ sel = NULL;
+ }
+ else {
+ DL_FOREACH(hdr, cur)
+ {
+ ct = rspamd_content_type_parse(cur->value, strlen(cur->value),
+ task->task_pool);
+
+ /* Here we prefer multipart content-type or any content-type */
+ if (ct) {
+ if (sel == NULL) {
+ sel = ct;
+ }
+ else if (ct->flags & RSPAMD_CONTENT_TYPE_MULTIPART) {
+ sel = ct;
+ }
+ }
+ }
+ }
+
+ if (sel == NULL) {
+ /* For messages we automatically assume plaintext */
+ msg_info_task("cannot find content-type for a message, assume text/plain");
+ sel = rspamd_mempool_alloc0(task->task_pool, sizeof(*sel));
+ sel->flags = RSPAMD_CONTENT_TYPE_TEXT | RSPAMD_CONTENT_TYPE_MISSING;
+ RSPAMD_FTOK_ASSIGN(&sel->type, "text");
+ RSPAMD_FTOK_ASSIGN(&sel->subtype, "plain");
+ }
+
+ npart->ct = sel;
+
+ if ((part == NULL || nst != st) &&
+ (sel->flags & (RSPAMD_CONTENT_TYPE_MULTIPART | RSPAMD_CONTENT_TYPE_MESSAGE))) {
+ /* Not a trivial message, need to preprocess */
+ rspamd_mime_preprocess_message(task, npart, nst);
+ }
+
+ if (sel->flags & RSPAMD_CONTENT_TYPE_MULTIPART) {
+ g_ptr_array_add(nst->stack, npart);
+ nst->nesting++;
+ npart->part_type = RSPAMD_MIME_PART_MULTIPART;
+ npart->specific.mp = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct rspamd_mime_multipart));
+ memcpy(&npart->specific.mp->boundary, &sel->orig_boundary,
+ sizeof(rspamd_ftok_t));
+ ret = rspamd_mime_parse_multipart_part(task, npart, nst, err);
+ }
+ else if (sel->flags & RSPAMD_CONTENT_TYPE_MESSAGE) {
+ if ((ret = rspamd_mime_parse_normal_part(task, npart, nst, sel, err)) == RSPAMD_MIME_PARSE_OK) {
+ npart->part_type = RSPAMD_MIME_PART_MESSAGE;
+ ret = rspamd_mime_parse_message(task, npart, nst, err);
+ }
+ }
+ else {
+ ret = rspamd_mime_parse_normal_part(task, npart, nst, sel, err);
+ }
+
+ if (ret != RSPAMD_MIME_PARSE_OK) {
+ return ret;
+ }
+
+ if (part && st->stack->len > 0) {
+ /* Remove message part from the parent stack */
+ g_ptr_array_remove_index_fast(st->stack, st->stack->len - 1);
+ st->nesting--;
+ }
+
+ /* Process leftovers for boundaries */
+ if (nst->boundaries) {
+ struct rspamd_mime_boundary *boundary, *start_boundary = NULL,
+ *end_boundary = NULL;
+ goffset cur_offset = nst->pos - nst->start,
+ end_offset = st->end - st->start;
+ guint sel_idx = 0;
+
+ for (;;) {
+ start_boundary = NULL;
+
+ for (i = sel_idx; i < nst->boundaries->len; i++) {
+ boundary = &g_array_index(nst->boundaries,
+ struct rspamd_mime_boundary, i);
+
+ if (boundary->start > cur_offset &&
+ boundary->boundary < end_offset &&
+ !RSPAMD_BOUNDARY_IS_CLOSED(boundary)) {
+ start_boundary = boundary;
+ sel_idx = i;
+ break;
+ }
+ }
+
+ if (start_boundary) {
+ const gchar *start, *end;
+
+ if (nst->boundaries->len > sel_idx + 1) {
+ end_boundary = &g_array_index(nst->boundaries,
+ struct rspamd_mime_boundary, sel_idx + 1);
+ end = nst->start + end_boundary->boundary;
+ }
+ else {
+ end = nst->end;
+ }
+
+ sel_idx++;
+
+ start = nst->start + start_boundary->start;
+
+ if (end > start &&
+ (ret = rspamd_mime_process_multipart_node(task, nst,
+ NULL, start, end, FALSE, err)) != RSPAMD_MIME_PARSE_OK) {
+
+ if (nst != st) {
+ rspamd_mime_parse_stack_free(nst);
+ }
+
+ if (ret == RSPAMD_MIME_PARSE_NO_PART) {
+ return RSPAMD_MIME_PARSE_OK;
+ }
+
+ return ret;
+ }
+ }
+ else {
+ break;
+ }
+ }
+ }
+
+ if (nst != st) {
+ rspamd_mime_parse_stack_free(nst);
+ }
+
+ return ret;
+}
+
+enum rspamd_mime_parse_error
+rspamd_mime_parse_task(struct rspamd_task *task, GError **err)
+{
+ struct rspamd_mime_parser_ctx *st;
+ enum rspamd_mime_parse_error ret = RSPAMD_MIME_PARSE_OK;
+
+ if (lib_ctx == NULL) {
+ rspamd_mime_parser_init_lib();
+ }
+
+ if (++lib_ctx->key_usages > max_key_usages) {
+ /* Regenerate siphash key */
+ ottery_rand_bytes(lib_ctx->hkey, sizeof(lib_ctx->hkey));
+ lib_ctx->key_usages = 0;
+ }
+
+ st = g_malloc0(sizeof(*st));
+ st->stack = g_ptr_array_sized_new(4);
+ st->pos = MESSAGE_FIELD(task, raw_headers_content).body_start;
+ st->end = task->msg.begin + task->msg.len;
+ st->boundaries = g_array_sized_new(FALSE, FALSE,
+ sizeof(struct rspamd_mime_boundary), 8);
+ st->task = task;
+
+ if (st->pos == NULL) {
+ st->pos = task->msg.begin;
+ }
+
+ st->start = task->msg.begin;
+ ret = rspamd_mime_parse_message(task, NULL, st, err);
+ rspamd_mime_parse_stack_free(st);
+
+ return ret;
+}
diff --git a/src/libmime/mime_parser.h b/src/libmime/mime_parser.h
new file mode 100644
index 0000000..aa77b2b
--- /dev/null
+++ b/src/libmime/mime_parser.h
@@ -0,0 +1,46 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_MIME_PARSER_H_
+#define SRC_LIBMIME_MIME_PARSER_H_
+
+#include "config.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_mime_part;
+
+enum rspamd_mime_parse_error {
+ RSPAMD_MIME_PARSE_OK = 0,
+ RSPAMD_MIME_PARSE_FATAL,
+ RSPAMD_MIME_PARSE_NESTING,
+ RSPAMD_MIME_PARSE_NO_PART,
+};
+
+enum rspamd_mime_parse_error rspamd_mime_parse_task(struct rspamd_task *task,
+ GError **err);
+
+void rspamd_mime_parser_calc_digest(struct rspamd_mime_part *part);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_MIME_PARSER_H_ */
diff --git a/src/libmime/mime_string.cxx b/src/libmime/mime_string.cxx
new file mode 100644
index 0000000..e818e64
--- /dev/null
+++ b/src/libmime/mime_string.cxx
@@ -0,0 +1,167 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+#include "mime_string.hxx"
+#include "unicode/uchar.h"
+
+TEST_SUITE("mime_string")
+{
+ using namespace rspamd::mime;
+ TEST_CASE("mime_string unfiltered ctors")
+ {
+ SUBCASE("empty")
+ {
+ mime_string st;
+ CHECK(st.size() == 0);
+ CHECK(st == "");
+ }
+ SUBCASE("unfiltered valid")
+ {
+ mime_string st{std::string_view("abcd")};
+ CHECK(st == "abcd");
+ }
+ SUBCASE("unfiltered zero character")
+ {
+ mime_string st{"abc\0d", 5};
+ CHECK(st.has_zeroes());
+ CHECK(st == "abcd");
+ }
+ SUBCASE("unfiltered invalid character - middle")
+ {
+ mime_string st{std::string("abc\234d")};
+ CHECK(st.has_invalid());
+ CHECK(st == "abc\uFFFDd");
+ }
+ SUBCASE("unfiltered invalid character - end")
+ {
+ mime_string st{std::string("abc\234")};
+ CHECK(st.has_invalid());
+ CHECK(st == "abc\uFFFD");
+ }
+ SUBCASE("unfiltered invalid character - start")
+ {
+ mime_string st{std::string("\234abc")};
+ CHECK(st.has_invalid());
+ CHECK(st == "\uFFFDabc");
+ }
+ }
+
+ TEST_CASE("mime_string filtered ctors")
+ {
+ auto print_filter = [](UChar32 inp) -> UChar32 {
+ if (!u_isprint(inp)) {
+ return 0;
+ }
+
+ return inp;
+ };
+
+ auto tolower_filter = [](UChar32 inp) -> UChar32 {
+ return u_tolower(inp);
+ };
+
+ SUBCASE("empty")
+ {
+ mime_string st{std::string_view(""), tolower_filter};
+ CHECK(st.size() == 0);
+ CHECK(st == "");
+ }
+ SUBCASE("filtered valid")
+ {
+ mime_string st{std::string("AbCdУ"), tolower_filter};
+ CHECK(st == "abcdу");
+ }
+ SUBCASE("filtered invalid + filtered")
+ {
+ mime_string st{std::string("abcd\234\1"), print_filter};
+ CHECK(st == "abcd\uFFFD");
+ }
+ }
+ TEST_CASE("mime_string assign")
+ {
+ SUBCASE("assign from valid")
+ {
+ mime_string st;
+
+ CHECK(st.assign_if_valid(std::string("test")));
+ CHECK(st == "test");
+ }
+ SUBCASE("assign from invalid")
+ {
+ mime_string st;
+
+ CHECK(!st.assign_if_valid(std::string("test\234t")));
+ CHECK(st == "");
+ }
+ }
+
+ TEST_CASE("mime_string iterators")
+ {
+
+ SUBCASE("unfiltered iterator ascii")
+ {
+ auto in = std::string("abcd");
+ mime_string st{in};
+ CHECK(st == "abcd");
+
+ int i = 0;
+ for (auto &&c: st) {
+ CHECK(c == in[i++]);
+ }
+ }
+
+ SUBCASE("unfiltered iterator utf8")
+ {
+ auto in = std::string("теÑÑ‚");
+ UChar32 ucs[4] = {1090, 1077, 1089, 1090};
+ mime_string st{in};
+ CHECK(st == "теÑÑ‚");
+
+ int i = 0;
+ for (auto &&c: st) {
+ CHECK(c == ucs[i++]);
+ }
+ CHECK(i == sizeof(ucs) / sizeof(ucs[0]));
+ }
+
+ SUBCASE("unfiltered raw iterator ascii")
+ {
+ auto in = std::string("abcd");
+ mime_string st{in};
+ CHECK(st == "abcd");
+
+ int i = 0;
+ for (auto it = st.raw_begin(); it != st.raw_end(); ++it) {
+ CHECK(*it == in[i++]);
+ }
+ }
+
+ SUBCASE("unfiltered raw iterator utf8")
+ {
+ auto in = std::string("теÑÑ‚");
+ mime_string st{in};
+ CHECK(st == "теÑÑ‚");
+
+ int i = 0;
+ for (auto it = st.raw_begin(); it != st.raw_end(); ++it) {
+ CHECK(*it == in[i++]);
+ }
+ CHECK(i == in.size());
+ }
+ }
+} \ No newline at end of file
diff --git a/src/libmime/mime_string.hxx b/src/libmime/mime_string.hxx
new file mode 100644
index 0000000..7476816
--- /dev/null
+++ b/src/libmime/mime_string.hxx
@@ -0,0 +1,670 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_MIME_STRING_HXX
+#define RSPAMD_MIME_STRING_HXX
+#pragma once
+
+#include <algorithm>
+#include <string>
+#include <string_view>
+#include <memory>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <iosfwd>
+#include "libutil/mem_pool.h"
+#include "function2/function2.hpp"
+#include "unicode/utf8.h"
+#include "contrib/fastutf8/fastutf8.h"
+
+namespace rspamd::mime {
+/*
+ * The motivation for another string is to have utf8 valid string replacing
+ * all bad things with FFFFD replacement character and filtering \0 and other
+ * strange stuff defined by policies.
+ * This string always exclude \0 characters and ignore them! This is how MUA acts,
+ * and we also store a flag about bad characters.
+ * Mime string iterators are always const, so the underlying storage should not
+ * be modified externally.
+ */
+template<class T = char, class Allocator = std::allocator<T>,
+ class Functor = fu2::function_view<UChar32(UChar32)>>
+class basic_mime_string;
+
+using mime_string = basic_mime_string<char>;
+using mime_pool_string = basic_mime_string<char, mempool_allocator<char>>;
+
+/* Helpers for type safe flags */
+enum class mime_string_flags : std::uint8_t {
+ MIME_STRING_DEFAULT = 0,
+ MIME_STRING_SEEN_ZEROES = 0x1 << 0,
+ MIME_STRING_SEEN_INVALID = 0x1 << 1,
+};
+
+constexpr mime_string_flags operator|(mime_string_flags lhs, mime_string_flags rhs)
+{
+ using ut = std::underlying_type<mime_string_flags>::type;
+ return static_cast<mime_string_flags>(static_cast<ut>(lhs) | static_cast<ut>(rhs));
+}
+
+constexpr mime_string_flags operator&(mime_string_flags lhs, mime_string_flags rhs)
+{
+ using ut = std::underlying_type<mime_string_flags>::type;
+ return static_cast<mime_string_flags>(static_cast<ut>(lhs) & static_cast<ut>(rhs));
+}
+
+constexpr bool operator!(mime_string_flags fl)
+{
+ return fl == mime_string_flags::MIME_STRING_DEFAULT;
+}
+
+// Codepoint iterator base class
+template<typename Container, bool Raw = false>
+struct iterator_base {
+ template<typename, typename, typename>
+ friend class basic_mime_string;
+
+public:
+ using value_type = typename Container::value_type;
+ using difference_type = typename Container::difference_type;
+ using codepoint_type = typename Container::codepoint_type;
+ using reference_type = codepoint_type;
+ using iterator_category = std::bidirectional_iterator_tag;
+
+ bool operator==(const iterator_base &it) const noexcept
+ {
+ return idx == it.idx;
+ }
+
+ bool operator!=(const iterator_base &it) const noexcept
+ {
+ return idx != it.idx;
+ }
+
+ iterator_base(difference_type index, Container *instance) noexcept
+ : idx(index), cont_instance(instance)
+ {
+ }
+ iterator_base() noexcept = default;
+ iterator_base(const iterator_base &) noexcept = default;
+
+ iterator_base &operator=(const iterator_base &) noexcept = default;
+
+ Container *get_instance() const noexcept
+ {
+ return cont_instance;
+ }
+
+ codepoint_type get_value() const noexcept
+ {
+ auto i = idx;
+ codepoint_type uc;
+ U8_NEXT_UNSAFE(cont_instance->data(), i, uc);
+ return uc;
+ }
+
+protected:
+ difference_type idx;
+ Container *cont_instance = nullptr;
+
+protected:
+ void advance(difference_type n) noexcept
+ {
+ if (n > 0) {
+ U8_FWD_N_UNSAFE(cont_instance->data(), idx, n);
+ }
+ else if (n < 0) {
+ U8_BACK_N_UNSAFE(cont_instance->data(), idx, (-n));
+ }
+ }
+ void increment() noexcept
+ {
+ codepoint_type uc;
+ U8_NEXT_UNSAFE(cont_instance->data(), idx, uc);
+ }
+
+ void decrement() noexcept
+ {
+ codepoint_type uc;
+ U8_PREV_UNSAFE(cont_instance->data(), idx, uc);
+ }
+};
+
+// Partial spec for raw Byte-based iterator base
+template<typename Container>
+struct iterator_base<Container, true> {
+ template<typename, typename, typename>
+ friend class basic_string;
+
+public:
+ using value_type = typename Container::value_type;
+ using difference_type = typename Container::difference_type;
+ using reference_type = value_type;
+ using iterator_category = std::bidirectional_iterator_tag;
+
+ bool operator==(const iterator_base &it) const noexcept
+ {
+ return idx == it.idx;
+ }
+ bool operator!=(const iterator_base &it) const noexcept
+ {
+ return idx != it.idx;
+ }
+
+ iterator_base(difference_type index, Container *instance) noexcept
+ : idx(index), cont_instance(instance)
+ {
+ }
+
+ iterator_base() noexcept = default;
+ iterator_base(const iterator_base &) noexcept = default;
+ iterator_base &operator=(const iterator_base &) noexcept = default;
+ Container *get_instance() const noexcept
+ {
+ return cont_instance;
+ }
+
+ value_type get_value() const noexcept
+ {
+ return cont_instance->get_storage().at(idx);
+ }
+
+protected:
+ difference_type idx;
+ Container *cont_instance = nullptr;
+
+protected:
+ //! Advance the iterator n times (negative values allowed!)
+ void advance(difference_type n) noexcept
+ {
+ idx += n;
+ }
+
+ void increment() noexcept
+ {
+ idx++;
+ }
+ void decrement() noexcept
+ {
+ idx--;
+ }
+};
+
+template<typename Container, bool Raw>
+struct iterator;
+template<typename Container, bool Raw>
+struct const_iterator;
+
+template<typename Container, bool Raw = false>
+struct iterator : iterator_base<Container, Raw> {
+ iterator(typename iterator_base<Container, Raw>::difference_type index, Container *instance) noexcept
+ : iterator_base<Container, Raw>(index, instance)
+ {
+ }
+ iterator() noexcept = default;
+ iterator(const iterator &) noexcept = default;
+
+ iterator &operator=(const iterator &) noexcept = default;
+ /* Disallow creating from const_iterator */
+ iterator(const const_iterator<Container, Raw> &) = delete;
+
+ /* Prefix */
+ iterator &operator++() noexcept
+ {
+ this->increment();
+ return *this;
+ }
+
+ /* Postfix */
+ iterator operator++(int) noexcept
+ {
+ iterator tmp{this->idx, this->cont_instance};
+ this->increment();
+ return tmp;
+ }
+
+ /* Prefix */
+ iterator &operator--() noexcept
+ {
+ this->decrement();
+ return *this;
+ }
+
+ /* Postfix */
+ iterator operator--(int) noexcept
+ {
+ iterator tmp{this->idx, this->cont_instance};
+ this->decrement();
+ return tmp;
+ }
+
+ iterator operator+(typename iterator_base<Container, Raw>::difference_type n) const noexcept
+ {
+ iterator it{*this};
+ it.advance(n);
+ return it;
+ }
+
+ iterator &operator+=(typename iterator_base<Container, Raw>::difference_type n) noexcept
+ {
+ this->advance(n);
+ return *this;
+ }
+
+ iterator operator-(typename iterator_base<Container, Raw>::difference_type n) const noexcept
+ {
+ iterator it{*this};
+ it.advance(-n);
+ return it;
+ }
+
+ iterator &operator-=(typename iterator_base<Container, Raw>::difference_type n) noexcept
+ {
+ this->advance(-n);
+ return *this;
+ }
+
+ typename iterator::reference_type operator*() const noexcept
+ {
+ return this->get_value();
+ }
+};
+
+template<class CharT, class Allocator, class Functor>
+class basic_mime_string : private Allocator {
+public:
+ using storage_type = std::basic_string<CharT, std::char_traits<CharT>, Allocator>;
+ using view_type = std::basic_string_view<CharT, std::char_traits<CharT>>;
+ using filter_type = Functor;
+ using codepoint_type = UChar32;
+ using value_type = CharT;
+ using difference_type = std::ptrdiff_t;
+ using iterator = rspamd::mime::iterator<basic_mime_string, false>;
+ using raw_iterator = rspamd::mime::iterator<basic_mime_string, true>;
+ /* Ctors */
+ basic_mime_string() noexcept
+ : Allocator()
+ {
+ }
+ explicit basic_mime_string(const Allocator &alloc) noexcept
+ : Allocator(alloc)
+ {
+ }
+ explicit basic_mime_string(filter_type &&filt, const Allocator &alloc = Allocator()) noexcept
+ : Allocator(alloc), filter_func(std::move(filt))
+ {
+ }
+
+ basic_mime_string(const CharT *str, std::size_t sz, const Allocator &alloc = Allocator()) noexcept
+ : Allocator(alloc)
+ {
+ append_c_string_unfiltered(str, sz);
+ }
+
+ basic_mime_string(const storage_type &st,
+ const Allocator &alloc = Allocator()) noexcept
+ : basic_mime_string(st.data(), st.size(), alloc)
+ {
+ }
+
+ basic_mime_string(const view_type &st,
+ const Allocator &alloc = Allocator()) noexcept
+ : basic_mime_string(st.data(), st.size(), alloc)
+ {
+ }
+ /* Explicit move ctor */
+ basic_mime_string(basic_mime_string &&other) noexcept
+ {
+ *this = std::move(other);
+ }
+
+
+ /**
+ * Creates a string with a filter function. It is calee responsibility to
+ * ensure that the filter functor survives long enough to work with a string
+ * @param str
+ * @param sz
+ * @param filt
+ * @param alloc
+ */
+ basic_mime_string(const CharT *str, std::size_t sz,
+ filter_type &&filt,
+ const Allocator &alloc = Allocator()) noexcept
+ : Allocator(alloc),
+ filter_func(std::move(filt))
+ {
+ append_c_string_filtered(str, sz);
+ }
+
+ basic_mime_string(const storage_type &st,
+ filter_type &&filt,
+ const Allocator &alloc = Allocator()) noexcept
+ : basic_mime_string(st.data(), st.size(), std::move(filt), alloc)
+ {
+ }
+ basic_mime_string(const view_type &st,
+ filter_type &&filt,
+ const Allocator &alloc = Allocator()) noexcept
+ : basic_mime_string(st.data(), st.size(), std::move(filt), alloc)
+ {
+ }
+
+ /* It seems some libc++ implementations still perform copy, this might fix them */
+ basic_mime_string &operator=(basic_mime_string &&other)
+ {
+ storage = std::move(other.storage);
+ filter_func = std::move(other.filter_func);
+
+ return *this;
+ }
+
+ constexpr auto size() const noexcept -> std::size_t
+ {
+ return storage.size();
+ }
+
+ constexpr auto data() const noexcept -> const CharT *
+ {
+ return storage.data();
+ }
+
+ constexpr auto has_zeroes() const noexcept -> bool
+ {
+ return !!(flags & mime_string_flags::MIME_STRING_SEEN_ZEROES);
+ }
+
+ constexpr auto has_invalid() const noexcept -> bool
+ {
+ return !!(flags & mime_string_flags::MIME_STRING_SEEN_INVALID);
+ }
+
+ /**
+ * Assign mime string from another string using move operation if a source string
+ * is utf8 valid.
+ * If this function returns false, then ownership has not been transferred
+ * and the `other` string is unmodified as well as the storage
+ * @param other
+ * @return
+ */
+ [[nodiscard]] auto assign_if_valid(storage_type &&other) -> bool
+ {
+ if (filter_func) {
+ /* No way */
+ return false;
+ }
+ if (rspamd_fast_utf8_validate((const unsigned char *) other.data(), other.size()) == 0) {
+ std::swap(storage, other);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * Copy to the internal storage discarding the contained value
+ * @param other
+ * @return
+ */
+ auto assign_copy(const view_type &other)
+ {
+ storage.clear();
+
+ if (filter_func) {
+ append_c_string_filtered(other.data(), other.size());
+ }
+ else {
+ append_c_string_unfiltered(other.data(), other.size());
+ }
+ }
+ auto assign_copy(const storage_type &other)
+ {
+ storage.clear();
+
+ if (filter_func) {
+ append_c_string_filtered(other.data(), other.size());
+ }
+ else {
+ append_c_string_unfiltered(other.data(), other.size());
+ }
+ }
+ auto assign_copy(const basic_mime_string &other)
+ {
+ storage.clear();
+
+ if (filter_func) {
+ append_c_string_filtered(other.data(), other.size());
+ }
+ else {
+ append_c_string_unfiltered(other.data(), other.size());
+ }
+ }
+
+ /* Mutators */
+ auto append(const CharT *str, std::size_t size) -> std::size_t
+ {
+ if (filter_func) {
+ return append_c_string_filtered(str, size);
+ }
+ else {
+ return append_c_string_unfiltered(str, size);
+ }
+ }
+ auto append(const storage_type &other) -> std::size_t
+ {
+ return append(other.data(), other.size());
+ }
+ auto append(const view_type &other) -> std::size_t
+ {
+ return append(other.data(), other.size());
+ }
+
+ auto ltrim(const view_type &what) -> void
+ {
+ auto it = std::find_if(storage.begin(), storage.end(),
+ [&what](CharT c) {
+ return !std::any_of(what.begin(), what.end(), [&c](CharT sc) { return sc == c; });
+ });
+ storage.erase(storage.begin(), it);
+ }
+
+ auto rtrim(const view_type &what) -> void
+ {
+ auto it = std::find_if(storage.rbegin(), storage.rend(),
+ [&what](CharT c) {
+ return !std::any_of(what.begin(), what.end(), [&c](CharT sc) { return sc == c; });
+ });
+ storage.erase(it.base(), storage.end());
+ }
+
+ auto trim(const view_type &what) -> void
+ {
+ ltrim(what);
+ rtrim(what);
+ }
+
+ /* Comparison */
+ auto operator==(const basic_mime_string &other)
+ {
+ return other.storage == storage;
+ }
+ auto operator==(const storage_type &other)
+ {
+ return other == storage;
+ }
+ auto operator==(const view_type &other)
+ {
+ return other == storage;
+ }
+ auto operator==(const CharT *other)
+ {
+ if (other == NULL) {
+ return false;
+ }
+ auto olen = strlen(other);
+ if (storage.size() == olen) {
+ return memcmp(storage.data(), other, olen) == 0;
+ }
+
+ return false;
+ }
+
+ /* Iterators */
+ inline auto begin() noexcept -> iterator
+ {
+ return {0, this};
+ }
+
+ inline auto raw_begin() noexcept -> raw_iterator
+ {
+ return {0, this};
+ }
+
+ inline auto end() noexcept -> iterator
+ {
+ return {(difference_type) size(), this};
+ }
+
+ inline auto raw_end() noexcept -> raw_iterator
+ {
+ return {(difference_type) size(), this};
+ }
+
+ /* Utility */
+ inline auto get_storage() const noexcept -> const storage_type &
+ {
+ return storage;
+ }
+
+ inline auto as_view() const noexcept -> view_type
+ {
+ return view_type{storage};
+ }
+
+ constexpr CharT operator[](std::size_t pos) const noexcept
+ {
+ return storage[pos];
+ }
+ constexpr CharT at(std::size_t pos) const
+ {
+ return storage.at(pos);
+ }
+ constexpr bool empty() const noexcept
+ {
+ return storage.empty();
+ }
+
+
+ /* For doctest stringify */
+ friend std::ostream &operator<<(std::ostream &os, const CharT &value)
+ {
+ os << value.storage;
+ return os;
+ }
+
+private:
+ mime_string_flags flags = mime_string_flags::MIME_STRING_DEFAULT;
+ storage_type storage;
+ filter_type filter_func;
+
+ auto append_c_string_unfiltered(const CharT *str, std::size_t len) -> std::size_t
+ {
+ /* This is fast path */
+ const auto *p = str;
+ const auto *end = str + len;
+ std::int32_t err_offset;// We have to use int32_t here as old libicu is brain-damaged
+ auto orig_size = storage.size();
+
+ storage.reserve(len + storage.size());
+
+ if (memchr(str, 0, len) != NULL) {
+ /* Fallback to slow path */
+ flags = flags | mime_string_flags::MIME_STRING_SEEN_ZEROES;
+ return append_c_string_filtered(str, len);
+ }
+
+ while (p < end && len > 0 &&
+ (err_offset = rspamd_fast_utf8_validate((const unsigned char *) p, len)) > 0) {
+ auto cur_offset = err_offset - 1;
+ storage.append(p, cur_offset);
+
+ while (cur_offset < len) {
+ auto tmp = cur_offset;
+ UChar32 uc;
+
+ U8_NEXT(p, cur_offset, len, uc);
+
+ if (uc < 0) {
+ storage.append("\uFFFD");
+ flags = flags | mime_string_flags::MIME_STRING_SEEN_INVALID;
+ }
+ else {
+ cur_offset = tmp;
+ break;
+ }
+ }
+
+ p += cur_offset;
+ len = end - p;
+ }
+
+ storage.append(p, len);
+ return storage.size() - orig_size;
+ }
+
+ auto append_c_string_filtered(const CharT *str, std::size_t len) -> std::size_t
+ {
+ std::int32_t i = 0;// We have to use int32_t here as old libicu is brain-damaged
+ UChar32 uc;
+ char tmp[4];
+ auto orig_size = storage.size();
+ /* Slow path */
+
+ storage.reserve(len + storage.size());
+
+ while (i < len) {
+ U8_NEXT(str, i, len, uc);
+
+ if (uc < 0) {
+ /* Replace with 0xFFFD */
+ storage.append("\uFFFD");
+ flags = flags | mime_string_flags::MIME_STRING_SEEN_INVALID;
+ }
+ else {
+ if (filter_func) {
+ uc = filter_func(uc);
+ }
+
+ if (uc == 0) {
+ /* Special case, ignore it */
+ flags = flags | mime_string_flags::MIME_STRING_SEEN_ZEROES;
+ }
+ else {
+ std::int32_t o = 0;
+ U8_APPEND_UNSAFE(tmp, o, uc);
+ storage.append(tmp, o);
+ }
+ }
+ }
+
+ return storage.size() - orig_size;
+ }
+};
+
+}// namespace rspamd::mime
+
+
+#endif//RSPAMD_MIME_STRING_HXX
diff --git a/src/libmime/received.cxx b/src/libmime/received.cxx
new file mode 100644
index 0000000..dc16d9b
--- /dev/null
+++ b/src/libmime/received.cxx
@@ -0,0 +1,1017 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "libserver/url.h"
+#include "lua/lua_common.h"
+#include "libserver/cfg_file.h"
+#include "libserver/mempool_vars_internal.h"
+#include "mime_string.hxx"
+#include "smtp_parsers.h"
+#include "message.h"
+#include "received.hxx"
+#include "frozen/string.h"
+#include "frozen/unordered_map.h"
+
+namespace rspamd::mime {
+
+enum class received_part_type {
+ RSPAMD_RECEIVED_PART_FROM,
+ RSPAMD_RECEIVED_PART_BY,
+ RSPAMD_RECEIVED_PART_FOR,
+ RSPAMD_RECEIVED_PART_WITH,
+ RSPAMD_RECEIVED_PART_ID,
+ RSPAMD_RECEIVED_PART_UNKNOWN,
+};
+
+struct received_part {
+ received_part_type type;
+ mime_string data;
+ std::vector<mime_string> comments;
+
+ explicit received_part(received_part_type t)
+ : type(t),
+ data(received_char_filter)
+ {
+ }
+};
+
+static inline auto
+received_part_set_or_append(const gchar *begin,
+ gsize len,
+ mime_string &dest) -> void
+{
+ if (len == 0) {
+ return;
+ }
+
+ dest.append(begin, len);
+ dest.trim(" \t");
+}
+
+static auto
+received_process_part(const std::string_view &data,
+ received_part_type type,
+ std::ptrdiff_t &last,
+ received_part &npart) -> bool
+{
+ auto obraces = 0, ebraces = 0;
+ auto seen_tcpinfo = false;
+ enum _parse_state {
+ skip_spaces,
+ in_comment,
+ read_data,
+ read_tcpinfo,
+ all_done
+ } state,
+ next_state;
+
+ /* In this function, we just process comments and data separately */
+ const auto *p = data.data();
+ const auto *end = p + data.size();
+ const auto *c = p;
+
+ state = skip_spaces;
+ next_state = read_data;
+
+ while (p < end) {
+ switch (state) {
+ case skip_spaces:
+ if (!g_ascii_isspace(*p)) {
+ c = p;
+ state = next_state;
+ }
+ else {
+ p++;
+ }
+ break;
+ case in_comment:
+ if (*p == '(') {
+ obraces++;
+ }
+ else if (*p == ')') {
+ ebraces++;
+
+ if (ebraces >= obraces) {
+ if (type != received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN) {
+ if (p > c) {
+ npart.comments.emplace_back(received_char_filter);
+ auto &comment = npart.comments.back();
+ received_part_set_or_append(c, p - c,
+ comment);
+ }
+ }
+
+ p++;
+ c = p;
+ state = skip_spaces;
+ next_state = read_data;
+
+ continue;
+ }
+ }
+
+ p++;
+ break;
+ case read_data:
+ if (*p == '(') {
+ if (p > c) {
+ if (type != received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN) {
+ received_part_set_or_append(c, p - c,
+ npart.data);
+ }
+ }
+
+ state = in_comment;
+ obraces = 1;
+ ebraces = 0;
+ p++;
+ c = p;
+ }
+ else if (g_ascii_isspace(*p)) {
+ if (p > c) {
+ if (type != received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN) {
+ received_part_set_or_append(c, p - c,
+ npart.data);
+ }
+ }
+
+ state = skip_spaces;
+ next_state = read_data;
+ c = p;
+ }
+ else if (*p == ';') {
+ /* It is actually delimiter of date part if not in the comments */
+ if (p > c) {
+ if (type != received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN) {
+ received_part_set_or_append(c, p - c,
+ npart.data);
+ }
+ }
+
+ state = all_done;
+ continue;
+ }
+ else if (npart.data.size() > 0) {
+ /* We have already received data and find something with no ( */
+ if (!seen_tcpinfo && type == received_part_type::RSPAMD_RECEIVED_PART_FROM) {
+ /* Check if we have something special here, such as TCPinfo */
+ if (*c == '[') {
+ state = read_tcpinfo;
+ p++;
+ }
+ else {
+ state = all_done;
+ continue;
+ }
+ }
+ else {
+ state = all_done;
+ continue;
+ }
+ }
+ else {
+ p++;
+ }
+ break;
+ case read_tcpinfo:
+ if (*p == ']') {
+ received_part_set_or_append(c, p - c + 1,
+ npart.data);
+ seen_tcpinfo = TRUE;
+ state = skip_spaces;
+ next_state = read_data;
+ c = p;
+ }
+ p++;
+ break;
+ case all_done:
+ if (p > data.data()) {
+ last = p - data.data();
+ return true;
+ }
+ else {
+ /* Empty element */
+ return false;
+ }
+ break;
+ }
+ }
+
+ /* Leftover */
+ switch (state) {
+ case read_data:
+ if (p > c) {
+ if (type != received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN) {
+ received_part_set_or_append(c, p - c,
+ npart.data);
+ }
+
+ last = p - data.data();
+
+ return true;
+ }
+ break;
+ case skip_spaces:
+ if (p > data.data()) {
+ last = p - data.data();
+
+ return true;
+ }
+ default:
+ break;
+ }
+
+ return false;
+}
+
+template<std::size_t N>
+constexpr auto lit_compare_lowercase(const char lit[N], const char *in) -> bool
+{
+ for (auto i = 0; i < N; i++) {
+ if (lc_map[(unsigned char) in[i]] != lit[i]) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static auto
+received_spill(const std::string_view &in,
+ std::ptrdiff_t &date_pos) -> std::vector<received_part>
+{
+ std::vector<received_part> parts;
+ std::ptrdiff_t pos = 0;
+ auto seen_from = false, seen_by = false;
+
+ const auto *p = in.data();
+ const auto *end = p + in.size();
+
+ auto skip_spaces = [&p, end]() {
+ while (p < end && g_ascii_isspace(*p)) {
+ p++;
+ }
+ };
+
+ skip_spaces();
+
+ /* Skip SMTP comments */
+ if (*p == '(') {
+ auto obraces = 0, ebraces = 0;
+
+ while (p < end) {
+ if (*p == ')') {
+ ebraces++;
+ }
+ else if (*p == '(') {
+ obraces++;
+ }
+
+ p++;
+
+ if (obraces == ebraces) {
+ /* Skip spaces after */
+ skip_spaces();
+ break;
+ }
+ }
+ }
+
+ auto len = end - p;
+
+ if (len == 0) {
+ return parts;
+ }
+
+ auto maybe_process_part = [&](received_part_type what) -> bool {
+ parts.emplace_back(what);
+ auto &rcvd_part = parts.back();
+ auto chunk = std::string_view{p, (std::size_t)(end - p)};
+
+ if (!received_process_part(chunk, what, pos, rcvd_part)) {
+ parts.pop_back();
+
+ return false;
+ }
+
+ return true;
+ };
+
+ if (len > 4 && lit_compare_lowercase<4>("from", p)) {
+ p += sizeof("from") - 1;
+
+ /* We can now store from part */
+ if (!maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_FROM)) {
+ /* Do not accept malformed from */
+ return {};
+ }
+
+ g_assert(pos != 0);
+ p += pos;
+ len = end > p ? end - p : 0;
+ seen_from = true;
+ }
+
+ if (len > 2 && lit_compare_lowercase<2>("by", p)) {
+ p += sizeof("by") - 1;
+
+ if (!maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_BY)) {
+ return {};
+ }
+
+ g_assert(pos != 0);
+ p += pos;
+ len = end > p ? end - p : 0;
+ seen_by = true;
+ }
+
+ if (!seen_from && !seen_by) {
+ /* Useless received */
+ return {};
+ }
+
+ while (p < end) {
+ bool got_part = false;
+ if (*p == ';') {
+ /* We are at the date separator, stop here */
+ date_pos = p - in.data() + 1;
+ break;
+ }
+ else {
+ if (len > sizeof("with") && lit_compare_lowercase<4>("with", p)) {
+ p += sizeof("with") - 1;
+
+ got_part = maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_WITH);
+ }
+ else if (len > sizeof("for") && lit_compare_lowercase<3>("for", p)) {
+ p += sizeof("for") - 1;
+ got_part = maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_FOR);
+ }
+ else if (len > sizeof("id") && lit_compare_lowercase<2>("id", p)) {
+ p += sizeof("id") - 1;
+ got_part = maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_ID);
+ }
+ else {
+ while (p < end) {
+ if (!(g_ascii_isspace(*p) || *p == '(' || *p == ';')) {
+ p++;
+ }
+ else {
+ break;
+ }
+ }
+
+ if (p == end) {
+ return {};
+ }
+ else if (*p == ';') {
+ date_pos = p - in.data() + 1;
+ break;
+ }
+ else {
+ got_part = maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN);
+ }
+ }
+
+ if (!got_part) {
+ p++;
+ len = end > p ? end - p : 0;
+ }
+ else {
+ g_assert(pos != 0);
+ p += pos;
+ len = end > p ? end - p : 0;
+ }
+ }
+ }
+
+ return parts;
+}
+
+#define RSPAMD_INET_ADDRESS_PARSE_RECEIVED \
+ (rspamd_inet_address_parse_flags)(RSPAMD_INET_ADDRESS_PARSE_REMOTE | RSPAMD_INET_ADDRESS_PARSE_NO_UNIX)
+
+static auto
+received_process_rdns(rspamd_mempool_t *pool,
+ const std::string_view &in,
+ mime_string &dest) -> bool
+{
+ auto seen_dot = false;
+
+ const auto *p = in.data();
+ const auto *end = p + in.size();
+
+ if (in.empty()) {
+ return false;
+ }
+
+ if (*p == '[' && *(end - 1) == ']' && in.size() > 2) {
+ /* We have enclosed ip address */
+ auto *addr = rspamd_parse_inet_address_pool(p + 1,
+ (end - p) - 2,
+ pool,
+ RSPAMD_INET_ADDRESS_PARSE_RECEIVED);
+
+ if (addr) {
+ const gchar *addr_str;
+
+ if (rspamd_inet_address_get_port(addr) != 0) {
+ addr_str = rspamd_inet_address_to_string_pretty(addr);
+ }
+ else {
+ addr_str = rspamd_inet_address_to_string(addr);
+ }
+
+ dest.assign_copy(std::string_view{addr_str});
+
+ return true;
+ }
+ }
+
+ auto hlen = 0u;
+
+ while (p < end) {
+ if (!g_ascii_isspace(*p) && rspamd_url_is_domain(*p)) {
+ if (*p == '.') {
+ seen_dot = true;
+ }
+
+ hlen++;
+ }
+ else {
+ break;
+ }
+
+ p++;
+ }
+
+ if (hlen > 0) {
+ if (p == end || (seen_dot && (g_ascii_isspace(*p) || *p == '[' || *p == '('))) {
+ /* All data looks like a hostname */
+ dest.assign_copy(std::string_view{in.data(), hlen});
+
+ return true;
+ }
+ }
+
+ return false;
+}
+
+static auto
+received_process_host_tcpinfo(rspamd_mempool_t *pool,
+ received_header &rh,
+ const std::string_view &in) -> bool
+{
+ rspamd_inet_addr_t *addr = nullptr;
+ auto ret = false;
+
+ if (in.empty()) {
+ return false;
+ }
+
+ if (in[0] == '[') {
+ /* Likely Exim version */
+
+ auto brace_pos = in.find(']');
+
+ if (brace_pos != std::string_view::npos) {
+ auto substr_addr = in.substr(1, brace_pos - 1);
+ addr = rspamd_parse_inet_address_pool(substr_addr.data(),
+ substr_addr.size(),
+ pool,
+ RSPAMD_INET_ADDRESS_PARSE_RECEIVED);
+
+ if (addr) {
+ rh.addr = addr;
+ rh.real_ip.assign_copy(std::string_view(rspamd_inet_address_to_string(addr)));
+ }
+ }
+ }
+ else {
+ if (g_ascii_isxdigit(in[0])) {
+ /* Try to parse IP address */
+ addr = rspamd_parse_inet_address_pool(in.data(),
+ in.size(), pool, RSPAMD_INET_ADDRESS_PARSE_RECEIVED);
+ if (addr) {
+ rh.addr = addr;
+ rh.real_ip.assign_copy(std::string_view(rspamd_inet_address_to_string(addr)));
+ }
+ }
+
+ if (!addr) {
+ /* Try canonical Postfix version: rdns [ip] */
+ auto obrace_pos = in.find('[');
+
+ if (obrace_pos != std::string_view::npos) {
+ auto ebrace_pos = in.rfind(']');
+
+ if (ebrace_pos != std::string_view::npos && ebrace_pos > obrace_pos) {
+ auto substr_addr = in.substr(obrace_pos + 1,
+ ebrace_pos - obrace_pos - 1);
+ addr = rspamd_parse_inet_address_pool(substr_addr.data(),
+ substr_addr.size(),
+ pool,
+ RSPAMD_INET_ADDRESS_PARSE_RECEIVED);
+
+ if (addr) {
+ rh.addr = addr;
+ rh.real_ip.assign_copy(std::string_view(rspamd_inet_address_to_string(addr)));
+
+ /* Process with rDNS */
+ auto rdns_substr = in.substr(0, obrace_pos);
+
+ if (received_process_rdns(pool, rdns_substr, rh.real_hostname)) {
+ ret = true;
+ }
+ }
+ }
+ }
+ else {
+ /* Hostname or some crap, sigh... */
+ if (received_process_rdns(pool, in, rh.real_hostname)) {
+ ret = true;
+ }
+ }
+ }
+ }
+
+ return ret;
+}
+
+static void
+received_process_from(rspamd_mempool_t *pool,
+ const received_part &rpart,
+ received_header &rh)
+{
+ if (rpart.data.size() > 0) {
+ /* We have seen multiple cases:
+ * - [ip] (hostname/unknown [real_ip])
+ * - helo (hostname/unknown [real_ip])
+ * - [ip]
+ * - hostname
+ * - hostname ([ip]:port helo=xxx)
+ * Maybe more...
+ */
+ auto seen_ip_in_data = false;
+
+ if (!rpart.comments.empty()) {
+ /* We can have info within comment as part of RFC */
+ received_process_host_tcpinfo(
+ pool, rh,
+ rpart.comments[0].as_view());
+ }
+
+ if (rh.real_ip.size() == 0) {
+ /* Try to do the same with data */
+ if (received_process_host_tcpinfo(
+ pool, rh,
+ rpart.data.as_view())) {
+ seen_ip_in_data = true;
+ }
+ }
+
+ if (!seen_ip_in_data) {
+ if (rh.real_ip.size() != 0) {
+ /* Get announced hostname (usually helo) */
+ received_process_rdns(pool,
+ rpart.data.as_view(),
+ rh.from_hostname);
+ }
+ else {
+ received_process_host_tcpinfo(pool,
+ rh, rpart.data.as_view());
+ }
+ }
+ }
+ else {
+ /* rpart->dlen = 0 */
+ if (!rpart.comments.empty()) {
+ received_process_host_tcpinfo(
+ pool, rh,
+ rpart.comments[0].as_view());
+ }
+ }
+}
+
+static auto
+received_header_parse(received_header_chain &chain, rspamd_mempool_t *pool,
+ const std::string_view &in,
+ struct rspamd_mime_header *hdr) -> bool
+{
+ std::ptrdiff_t date_pos = -1;
+
+ static constexpr const auto protos_map = frozen::make_unordered_map<frozen::string, received_flags>({{"smtp", received_flags::SMTP},
+ {"esmtp", received_flags::ESMTP},
+ {"esmtpa", received_flags::ESMTPA |
+ received_flags::AUTHENTICATED},
+ {"esmtpsa", received_flags::ESMTPSA |
+ received_flags::SSL |
+ received_flags::AUTHENTICATED},
+ {"esmtps", received_flags::ESMTPS |
+ received_flags::SSL},
+ {"lmtp", received_flags::LMTP},
+ {"imap", received_flags::IMAP},
+ {"imaps", received_flags::IMAP |
+ received_flags::SSL},
+ {"http", received_flags::HTTP},
+ {"https", received_flags::HTTP |
+ received_flags::SSL},
+ {"local", received_flags::LOCAL}});
+
+ auto parts = received_spill(in, date_pos);
+
+ if (parts.empty()) {
+ return false;
+ }
+
+ auto &rh = chain.new_received();
+
+ rh.flags = received_flags::UNKNOWN;
+ rh.hdr = hdr;
+
+ for (const auto &part: parts) {
+ switch (part.type) {
+ case received_part_type::RSPAMD_RECEIVED_PART_FROM:
+ received_process_from(pool, part, rh);
+ break;
+ case received_part_type::RSPAMD_RECEIVED_PART_BY:
+ received_process_rdns(pool,
+ part.data.as_view(),
+ rh.by_hostname);
+ break;
+ case received_part_type::RSPAMD_RECEIVED_PART_WITH:
+ if (part.data.size() > 0) {
+ auto proto_flag_it = protos_map.find(part.data.as_view());
+
+ if (proto_flag_it != protos_map.end()) {
+ rh.flags = proto_flag_it->second;
+ }
+ }
+ break;
+ case received_part_type::RSPAMD_RECEIVED_PART_FOR:
+ rh.for_mbox.assign_copy(part.data);
+ rh.for_addr = rspamd_email_address_from_smtp(rh.for_mbox.data(),
+ rh.for_mbox.size());
+ break;
+ default:
+ /* Do nothing */
+ break;
+ }
+ }
+
+ if (!rh.real_hostname.empty() && rh.from_hostname.empty()) {
+ rh.from_hostname.assign_copy(rh.real_hostname);
+ }
+
+ if (date_pos > 0 && date_pos < in.size()) {
+ auto date_sub = in.substr(date_pos);
+ rh.timestamp = rspamd_parse_smtp_date((const unsigned char *) date_sub.data(),
+ date_sub.size(), nullptr);
+ }
+
+ return true;
+}
+
+static auto
+received_maybe_fix_task(struct rspamd_task *task) -> bool
+{
+ auto *recv_chain_ptr = static_cast<received_header_chain *>(MESSAGE_FIELD(task, received_headers));
+
+ if (recv_chain_ptr) {
+ auto need_recv_correction = false;
+
+ auto top_recv_maybe = recv_chain_ptr->get_received(0);
+
+ if (top_recv_maybe.has_value()) {
+ auto &top_recv = top_recv_maybe.value().get();
+
+ const auto *raddr = top_recv.addr;
+ if (top_recv.real_ip.size() == 0 || (task->cfg && task->cfg->ignore_received)) {
+ need_recv_correction = true;
+ }
+ else if (!(task->flags & RSPAMD_TASK_FLAG_NO_IP) && task->from_addr) {
+ if (!raddr) {
+ need_recv_correction = true;
+ }
+ else {
+ if (rspamd_inet_address_compare(raddr, task->from_addr, FALSE) != 0) {
+ need_recv_correction = true;
+ }
+ }
+ }
+
+ if (need_recv_correction && !(task->flags & RSPAMD_TASK_FLAG_NO_IP) && task->from_addr) {
+ msg_debug_task("the first received seems to be"
+ " not ours, prepend it with fake one");
+
+ auto &trecv = recv_chain_ptr->new_received(received_header_chain::append_type::append_head);
+ trecv.flags |= received_flags::ARTIFICIAL;
+
+ if (task->flags & RSPAMD_TASK_FLAG_SSL) {
+ trecv.flags |= received_flags::SSL;
+ }
+
+ if (task->auth_user) {
+ trecv.flags |= received_flags::AUTHENTICATED;
+ }
+
+ trecv.real_ip.assign_copy(std::string_view(rspamd_inet_address_to_string(task->from_addr)));
+
+ const auto *mta_name = (const char *) rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_MTA_NAME);
+
+ if (mta_name) {
+ trecv.by_hostname.assign_copy(std::string_view(mta_name));
+ }
+ trecv.addr = rspamd_inet_address_copy(task->from_addr,
+ task->task_pool);
+
+ if (task->hostname) {
+ trecv.real_hostname.assign_copy(std::string_view(task->hostname));
+ trecv.from_hostname.assign_copy(trecv.real_hostname);
+ }
+
+ return true;
+ }
+
+ /* Extract data from received header if we were not given IP */
+ if (!need_recv_correction && (task->flags & RSPAMD_TASK_FLAG_NO_IP) &&
+ (task->cfg && !task->cfg->ignore_received)) {
+ if (!top_recv.real_ip.empty()) {
+ if (!rspamd_parse_inet_address(&task->from_addr,
+ top_recv.real_ip.data(),
+ top_recv.real_ip.size(),
+ RSPAMD_INET_ADDRESS_PARSE_NO_UNIX)) {
+ msg_warn_task("cannot get IP from received header: '%s'",
+ top_recv.real_ip.data());
+ task->from_addr = nullptr;
+ }
+ }
+ if (!top_recv.real_hostname.empty()) {
+ task->hostname = top_recv.real_hostname.data();
+ }
+
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
+static auto
+received_export_to_lua(received_header_chain *chain, lua_State *L) -> bool
+{
+ if (chain == nullptr) {
+ return false;
+ }
+
+ lua_createtable(L, chain->size(), 0);
+
+ auto push_flag = [L](const received_header &rh, received_flags fl, const char *name) {
+ lua_pushboolean(L, !!(rh.flags & fl));
+ lua_setfield(L, -2, name);
+ };
+
+ auto i = 1;
+
+ for (const auto &rh: chain->as_vector()) {
+ lua_createtable(L, 0, 10);
+
+ if (rh.hdr && rh.hdr->decoded) {
+ rspamd_lua_table_set(L, "raw", rh.hdr->decoded);
+ }
+
+ lua_createtable(L, 0, 3);
+ push_flag(rh, received_flags::ARTIFICIAL, "artificial");
+ push_flag(rh, received_flags::AUTHENTICATED, "authenticated");
+ push_flag(rh, received_flags::SSL, "ssl");
+ lua_setfield(L, -2, "flags");
+
+ auto push_nullable_string = [L](const mime_string &st, const char *field) {
+ if (st.empty()) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushlstring(L, st.data(), st.size());
+ }
+ lua_setfield(L, -2, field);
+ };
+
+ push_nullable_string(rh.from_hostname, "from_hostname");
+ push_nullable_string(rh.real_hostname, "real_hostname");
+ push_nullable_string(rh.real_ip, "from_ip");
+ push_nullable_string(rh.by_hostname, "by_hostname");
+ push_nullable_string(rh.for_mbox, "for");
+
+ if (rh.addr) {
+ rspamd_lua_ip_push(L, rh.addr);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ lua_setfield(L, -2, "real_ip");
+
+ lua_pushstring(L, received_protocol_to_string(rh.flags));
+ lua_setfield(L, -2, "proto");
+
+ lua_pushinteger(L, rh.timestamp);
+ lua_setfield(L, -2, "timestamp");
+
+ lua_rawseti(L, -2, i++);
+ }
+
+ return true;
+}
+
+}// namespace rspamd::mime
+
+bool rspamd_received_header_parse(struct rspamd_task *task,
+ const char *data, size_t sz,
+ struct rspamd_mime_header *hdr)
+{
+ auto *recv_chain_ptr = static_cast<rspamd::mime::received_header_chain *>(MESSAGE_FIELD(task, received_headers));
+
+ if (recv_chain_ptr == nullptr) {
+ /* This constructor automatically registers dtor in mempool */
+ recv_chain_ptr = new rspamd::mime::received_header_chain(task);
+ MESSAGE_FIELD(task, received_headers) = (void *) recv_chain_ptr;
+ }
+ return rspamd::mime::received_header_parse(*recv_chain_ptr, task->task_pool,
+ std::string_view{data, sz}, hdr);
+}
+
+bool rspamd_received_maybe_fix_task(struct rspamd_task *task)
+{
+ return rspamd::mime::received_maybe_fix_task(task);
+}
+
+bool rspamd_received_export_to_lua(struct rspamd_task *task, lua_State *L)
+{
+ return rspamd::mime::received_export_to_lua(
+ static_cast<rspamd::mime::received_header_chain *>(MESSAGE_FIELD(task, received_headers)),
+ L);
+}
+
+/* Tests part */
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+TEST_SUITE("received")
+{
+ TEST_CASE("parse received")
+ {
+ using namespace std::string_view_literals;
+ using map_type = ankerl::unordered_dense::map<std::string_view, std::string_view>;
+ std::vector<std::pair<std::string_view, map_type>> cases{
+ // Simple received
+ {"from smtp11.mailtrack.pl (smtp11.mailtrack.pl [185.243.30.90])"sv,
+ {{"real_ip", "185.243.30.90"},
+ {"real_hostname", "smtp11.mailtrack.pl"},
+ {"from_hostname", "smtp11.mailtrack.pl"}}},
+ // Real Postfix IPv6 received
+ {"from server.chat-met-vreemden.nl (unknown [IPv6:2a01:7c8:aab6:26d:5054:ff:fed1:1da2])\n"
+ "\t(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))\n"
+ "\t(Client did not present a certificate)\n"
+ "\tby mx1.freebsd.org (Postfix) with ESMTPS id CF0171862\n"
+ "\tfor <test@example.com>; Mon, 6 Jul 2015 09:01:20 +0000 (UTC)\n"
+ "\t(envelope-from upwest201diana@outlook.com)"sv,
+ {{"real_ip", "2a01:7c8:aab6:26d:5054:ff:fed1:1da2"},
+ {"from_hostname", "server.chat-met-vreemden.nl"},
+ {"by_hostname", "mx1.freebsd.org"},
+ {"for_mbox", "<test@example.com>"}}},
+ // Exim IPv4 received
+ {"from localhost ([127.0.0.1]:49019 helo=hummus.csx.cam.ac.uk)\n"
+ " by hummus.csx.cam.ac.uk with esmtp (Exim 4.91-pdpfix1)\n"
+ " (envelope-from <exim-dev-bounces@exim.org>)\n"
+ " id 1fZ55o-0006DP-3H\n"
+ " for <xxx@xxx.xxx>; Sat, 30 Jun 2018 02:54:28 +0100"sv,
+ {
+ {"from_hostname", "localhost"},
+ {"real_ip", "127.0.0.1"},
+ {"for_mbox", "<xxx@xxx.xxx>"},
+ {"by_hostname", "hummus.csx.cam.ac.uk"},
+ }},
+ // Exim IPv6 received
+ {"from smtp.spodhuis.org ([2a02:898:31:0:48:4558:736d:7470]:38689\n"
+ " helo=mx.spodhuis.org)\n"
+ " by hummus.csx.cam.ac.uk with esmtpsa (TLSv1.3:TLS_AES_256_GCM_SHA384:256)\n"
+ " (Exim 4.91-pdpfix1+cc) (envelope-from <xxx@exim.org>)\n"
+ " id 1fZ55k-0006CO-9M\n"
+ " for exim-dev@exim.org; Sat, 30 Jun 2018 02:54:24 +0100"sv,
+ {
+ {"from_hostname", "smtp.spodhuis.org"},
+ {"real_ip", "2a02:898:31:0:48:4558:736d:7470"},
+ {"for_mbox", "exim-dev@exim.org"},
+ {"by_hostname", "hummus.csx.cam.ac.uk"},
+ }},
+ // Haraka received
+ {"from aaa.cn ([1.1.1.1]) by localhost.localdomain (Haraka/2.8.18) with "
+ "ESMTPA id 349C9C2B-491A-4925-A687-3EF14038C344.1 envelope-from <huxin@xxx.com> "
+ "(authenticated bits=0); Tue, 03 Jul 2018 14:18:13 +0200"sv,
+ {
+ {"from_hostname", "aaa.cn"},
+ {"real_ip", "1.1.1.1"},
+ {"by_hostname", "localhost.localdomain"},
+ }},
+ // Invalid by
+ {"from [192.83.172.101] (HELLO 148.251.238.35) (148.251.238.35) "
+ "by guovswzqkvry051@sohu.com with gg login "
+ "by AOL 6.0 for Windows US sub 008 SMTP ; Tue, 03 Jul 2018 09:01:47 -0300"sv,
+ {
+ {"from_hostname", "192.83.172.101"},
+ {"real_ip", "192.83.172.101"},
+ }},
+ // Invalid hostinfo
+ {"from example.com ([]) by example.com with ESMTP id 2019091111 ;"
+ " Thu, 26 Sep 2019 11:19:07 +0200"sv,
+ {
+ {"by_hostname", "example.com"},
+ {"from_hostname", "example.com"},
+ {"real_hostname", "example.com"},
+ }},
+ // Different real and announced hostnames + broken crap
+ {"from 171-29.br (1-1-1-1.z.com.br [1.1.1.1]) by x.com.br (Postfix) "
+ "with;ESMTP id 44QShF6xj4z1X for <hey@y.br>; Thu, 21 Mar 2019 23:45:46 -0300 "
+ ": <g @yi.br>"sv,
+ {
+ {"real_ip", "1.1.1.1"},
+ {"from_hostname", "171-29.br"},
+ {"real_hostname", "1-1-1-1.z.com.br"},
+ {"by_hostname", "x.com.br"},
+ }},
+ // Different real and announced ips + no hostname
+ {"from [127.0.0.1] ([127.0.0.2]) by smtp.gmail.com with ESMTPSA id xxxololo"sv,
+ {
+ {"real_ip", "127.0.0.2"},
+ {"from_hostname", "127.0.0.1"},
+ {"by_hostname", "smtp.gmail.com"},
+ }},
+ // Different real and hostanes
+ {"from 185.118.166.127 (steven2.zhou01.pserver.ru [185.118.166.127]) "
+ "by mail.832zsu.cn (Postfix) with ESMTPA id AAD722133E34"sv,
+ {
+ {"real_ip", "185.118.166.127"},
+ {"from_hostname", "185.118.166.127"},
+ {"real_hostname", "steven2.zhou01.pserver.ru"},
+ {"by_hostname", "mail.832zsu.cn"},
+ }},
+ // \0 in received must be filtered
+ {"from smtp11.mailt\0rack.pl (smtp11.mail\0track.pl [1\085.243.30.90])"sv,
+ {{"real_ip", "185.243.30.90"},
+ {"real_hostname", "smtp11.mailtrack.pl"},
+ {"from_hostname", "smtp11.mailtrack.pl"}}},
+ // No from part
+ {"by mail.832zsu.cn (Postfix) with ESMTPA id AAD722133E34"sv,
+ {
+ {"by_hostname", "mail.832zsu.cn"},
+ }},
+ // From part is in the comment
+ {"(from asterisk@localhost)\n"
+ " by pbx.xxx.com (8.14.7/8.14.7/Submit) id 076Go4wD014562;\n"
+ " Thu, 6 Aug 2020 11:50:04 -0500"sv,
+ {
+ {"by_hostname", "pbx.xxx.com"},
+ }},
+ };
+ rspamd_mempool_t *pool = rspamd_mempool_new_default("rcvd test", 0);
+
+ for (auto &&c: cases) {
+ SUBCASE(c.first.data())
+ {
+ rspamd::mime::received_header_chain chain;
+ auto ret = rspamd::mime::received_header_parse(chain, pool,
+ c.first, nullptr);
+ CHECK(ret == true);
+ auto &&rh = chain.get_received(0);
+ CHECK(rh.has_value());
+ auto res = rh.value().get().as_map();
+
+ for (const auto &expected: c.second) {
+ CHECK_MESSAGE(res.contains(expected.first), expected.first.data());
+ CHECK(res[expected.first] == expected.second);
+ }
+ for (const auto &existing: res) {
+ CHECK_MESSAGE(c.second.contains(existing.first), existing.first.data());
+ CHECK(c.second[existing.first] == existing.second);
+ }
+ }
+ }
+
+ rspamd_mempool_delete(pool);
+ }
+} \ No newline at end of file
diff --git a/src/libmime/received.h b/src/libmime/received.h
new file mode 100644
index 0000000..46608a3
--- /dev/null
+++ b/src/libmime/received.h
@@ -0,0 +1,68 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#ifndef RSPAMD_RECEIVED_H
+#define RSPAMD_RECEIVED_H
+
+#include "config.h"
+#include "libutil/addr.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * C bindings for C++ received code
+ */
+
+struct rspamd_email_address;
+struct rspamd_received_header_chain;
+struct rspamd_mime_header;
+
+/**
+ * Parse received header from an input header data
+ * @param task
+ * @param data
+ * @param sz
+ * @param hdr
+ * @return
+ */
+bool rspamd_received_header_parse(struct rspamd_task *task,
+ const char *data, size_t sz, struct rspamd_mime_header *hdr);
+
+
+/**
+ * Process task data and the most top received and fix either part if needed
+ * @param task
+ * @return
+ */
+bool rspamd_received_maybe_fix_task(struct rspamd_task *task);
+
+struct lua_State;
+/**
+ * Push received headers chain to lua
+ * @param task
+ * @param L
+ * @return
+ */
+bool rspamd_received_export_to_lua(struct rspamd_task *task, struct lua_State *L);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif//RSPAMD_RECEIVED_H
diff --git a/src/libmime/received.hxx b/src/libmime/received.hxx
new file mode 100644
index 0000000..4f423f1
--- /dev/null
+++ b/src/libmime/received.hxx
@@ -0,0 +1,314 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#ifndef RSPAMD_RECEIVED_HXX
+#define RSPAMD_RECEIVED_HXX
+#pragma once
+
+#include "config.h"
+#include "received.h"
+#include "mime_string.hxx"
+#include "libmime/email_addr.h"
+#include "libserver/task.h"
+#include "contrib/ankerl/unordered_dense.h"
+#include <vector>
+#include <string_view>
+#include <utility>
+#include <optional>
+
+namespace rspamd::mime {
+
+static inline auto
+received_char_filter(UChar32 uc) -> UChar32
+{
+ if (u_isprint(uc)) {
+ return u_tolower(uc);
+ }
+
+ return 0;
+}
+
+enum class received_flags {
+ DEFAULT = 0,
+ SMTP = 1u << 0u,
+ ESMTP = 1u << 1u,
+ ESMTPA = 1u << 2u,
+ ESMTPS = 1u << 3u,
+ ESMTPSA = 1u << 4u,
+ LMTP = 1u << 5u,
+ IMAP = 1u << 6u,
+ LOCAL = 1u << 7u,
+ HTTP = 1u << 8u,
+ MAPI = 1u << 9u,
+ UNKNOWN = 1u << 10u,
+ ARTIFICIAL = (1u << 11u),
+ SSL = (1u << 12u),
+ AUTHENTICATED = (1u << 13u),
+};
+
+constexpr received_flags operator|(received_flags lhs, received_flags rhs)
+{
+ using ut = std::underlying_type<received_flags>::type;
+ return static_cast<received_flags>(static_cast<ut>(lhs) | static_cast<ut>(rhs));
+}
+
+constexpr received_flags operator|=(received_flags &lhs, const received_flags rhs)
+{
+ using ut = std::underlying_type<received_flags>::type;
+ lhs = static_cast<received_flags>(static_cast<ut>(lhs) | static_cast<ut>(rhs));
+ return lhs;
+}
+
+constexpr received_flags operator&(received_flags lhs, received_flags rhs)
+{
+ using ut = std::underlying_type<received_flags>::type;
+ return static_cast<received_flags>(static_cast<ut>(lhs) & static_cast<ut>(rhs));
+}
+
+constexpr bool operator!(received_flags fl)
+{
+ return fl == received_flags::DEFAULT;
+}
+
+constexpr received_flags received_type_apply_protocols_mask(received_flags fl)
+{
+ return fl & (received_flags::SMTP |
+ received_flags::ESMTP |
+ received_flags::ESMTPA |
+ received_flags::ESMTPS |
+ received_flags::ESMTPSA |
+ received_flags::IMAP |
+ received_flags::HTTP |
+ received_flags::LOCAL |
+ received_flags::MAPI |
+ received_flags::LMTP);
+}
+
+constexpr const char *received_protocol_to_string(received_flags fl)
+{
+ const auto *proto = "unknown";
+
+ switch (received_type_apply_protocols_mask(fl)) {
+ case received_flags::SMTP:
+ proto = "smtp";
+ break;
+ case received_flags::ESMTP:
+ proto = "esmtp";
+ break;
+ case received_flags::ESMTPS:
+ proto = "esmtps";
+ break;
+ case received_flags::ESMTPA:
+ proto = "esmtpa";
+ break;
+ case received_flags::ESMTPSA:
+ proto = "esmtpsa";
+ break;
+ case received_flags::LMTP:
+ proto = "lmtp";
+ break;
+ case received_flags::IMAP:
+ proto = "imap";
+ break;
+ case received_flags::HTTP:
+ proto = "http";
+ break;
+ case received_flags::LOCAL:
+ proto = "local";
+ break;
+ case received_flags::MAPI:
+ proto = "mapi";
+ break;
+ default:
+ break;
+ }
+
+ return proto;
+}
+
+struct received_header {
+ mime_string from_hostname;
+ mime_string real_hostname;
+ mime_string real_ip;
+ mime_string by_hostname;
+ mime_string for_mbox;
+ struct rspamd_email_address *for_addr = nullptr;
+ rspamd_inet_addr_t *addr = nullptr;
+ struct rspamd_mime_header *hdr = nullptr;
+ time_t timestamp = 0;
+ received_flags flags = received_flags::DEFAULT; /* See enum rspamd_received_type */
+
+ received_header() noexcept
+ : from_hostname(received_char_filter),
+ real_hostname(received_char_filter),
+ real_ip(received_char_filter),
+ by_hostname(received_char_filter),
+ for_mbox()
+ {
+ }
+ /* We have raw C pointers, so copy is explicitly disabled */
+ received_header(const received_header &other) = delete;
+ received_header(received_header &&other) noexcept
+ {
+ *this = std::move(other);
+ }
+
+ received_header &operator=(received_header &&other) noexcept
+ {
+ if (this != &other) {
+ from_hostname = std::move(other.from_hostname);
+ real_hostname = std::move(other.real_hostname);
+ real_ip = std::move(other.real_ip);
+ by_hostname = std::move(other.by_hostname);
+ for_mbox = std::move(other.for_mbox);
+ timestamp = other.timestamp;
+ flags = other.flags;
+ std::swap(for_addr, other.for_addr);
+ std::swap(addr, other.addr);
+ std::swap(hdr, other.hdr);
+ }
+ return *this;
+ }
+
+ /* Unit tests helper */
+ static auto from_map(const ankerl::unordered_dense::map<std::string_view, std::string_view> &map) -> received_header
+ {
+ using namespace std::string_view_literals;
+ received_header rh;
+
+ if (map.contains("from_hostname")) {
+ rh.from_hostname.assign_copy(map.at("from_hostname"sv));
+ }
+ if (map.contains("real_hostname")) {
+ rh.real_hostname.assign_copy(map.at("real_hostname"sv));
+ }
+ if (map.contains("by_hostname")) {
+ rh.by_hostname.assign_copy(map.at("by_hostname"sv));
+ }
+ if (map.contains("real_ip")) {
+ rh.real_ip.assign_copy(map.at("real_ip"sv));
+ }
+ if (map.contains("for_mbox")) {
+ rh.for_mbox.assign_copy(map.at("for_mbox"sv));
+ }
+
+ return rh;
+ }
+
+ auto as_map() const -> ankerl::unordered_dense::map<std::string_view, std::string_view>
+ {
+ ankerl::unordered_dense::map<std::string_view, std::string_view> map;
+
+ if (!from_hostname.empty()) {
+ map["from_hostname"] = from_hostname.as_view();
+ }
+ if (!real_hostname.empty()) {
+ map["real_hostname"] = real_hostname.as_view();
+ }
+ if (!by_hostname.empty()) {
+ map["by_hostname"] = by_hostname.as_view();
+ }
+ if (!real_ip.empty()) {
+ map["real_ip"] = real_ip.as_view();
+ }
+ if (!for_mbox.empty()) {
+ map["for_mbox"] = for_mbox.as_view();
+ }
+
+ return map;
+ }
+
+ ~received_header()
+ {
+ if (for_addr) {
+ rspamd_email_address_free(for_addr);
+ }
+ }
+};
+
+class received_header_chain {
+public:
+ explicit received_header_chain(struct rspamd_task *task)
+ {
+ headers.reserve(2);
+ rspamd_mempool_add_destructor(task->task_pool,
+ received_header_chain::received_header_chain_pool_dtor, this);
+ }
+ explicit received_header_chain()
+ {
+ headers.reserve(2);
+ }
+
+ enum class append_type {
+ append_tail,
+ append_head
+ };
+
+ auto new_received(append_type how = append_type::append_tail) -> received_header &
+ {
+ if (how == append_type::append_tail) {
+ headers.emplace_back();
+
+ return headers.back();
+ }
+ else {
+ headers.insert(std::begin(headers), received_header());
+
+ return headers.front();
+ }
+ }
+ auto new_received(received_header &&hdr, append_type how = append_type::append_tail) -> received_header &
+ {
+ if (how == append_type::append_tail) {
+ headers.emplace_back(std::move(hdr));
+
+ return headers.back();
+ }
+ else {
+ headers.insert(std::begin(headers), std::move(hdr));
+
+ return headers.front();
+ }
+ }
+ auto get_received(std::size_t nth) -> std::optional<std::reference_wrapper<received_header>>
+ {
+ if (nth < headers.size()) {
+ return headers[nth];
+ }
+
+ return std::nullopt;
+ }
+ auto size() const -> std::size_t
+ {
+ return headers.size();
+ }
+ constexpr auto as_vector() const -> const std::vector<received_header> &
+ {
+ return headers;
+ }
+
+private:
+ static auto received_header_chain_pool_dtor(void *ptr) -> void
+ {
+ delete static_cast<received_header_chain *>(ptr);
+ }
+ std::vector<received_header> headers;
+};
+
+}// namespace rspamd::mime
+
+#endif//RSPAMD_RECEIVED_HXX
diff --git a/src/libmime/scan_result.c b/src/libmime/scan_result.c
new file mode 100644
index 0000000..a6bc0cb
--- /dev/null
+++ b/src/libmime/scan_result.c
@@ -0,0 +1,1106 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "mem_pool.h"
+#include "scan_result.h"
+#include "rspamd.h"
+#include "message.h"
+#include "lua/lua_common.h"
+#include "libserver/cfg_file_private.h"
+#include "libmime/scan_result_private.h"
+#include "contrib/fastutf8/fastutf8.h"
+#include <math.h>
+#include "contrib/uthash/utlist.h"
+
+#define msg_debug_metric(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_metric_log_id, "metric", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(metric)
+
+/* Average symbols count to optimize hash allocation */
+static struct rspamd_counter_data symbols_count;
+
+static void
+rspamd_scan_result_dtor(gpointer d)
+{
+ struct rspamd_scan_result *r = (struct rspamd_scan_result *) d;
+ struct rspamd_symbol_result *sres;
+
+ rspamd_set_counter_ema(&symbols_count, kh_size(r->symbols), 0.5);
+
+ if (r->symbol_cbref != -1) {
+ luaL_unref(r->task->cfg->lua_state, LUA_REGISTRYINDEX, r->symbol_cbref);
+ }
+
+ kh_foreach_value(r->symbols, sres, {
+ if (sres->options) {
+ kh_destroy(rspamd_options_hash, sres->options);
+ }
+ });
+
+ kh_destroy(rspamd_symbols_hash, r->symbols);
+ kh_destroy(rspamd_symbols_group_hash, r->sym_groups);
+}
+
+static void
+rspamd_metric_actions_foreach_cb(int i, struct rspamd_action *act, void *cbd)
+{
+ struct rspamd_scan_result *metric_res = (struct rspamd_scan_result *) cbd;
+ metric_res->actions_config[i].flags = RSPAMD_ACTION_RESULT_DEFAULT;
+ if (!(act->flags & RSPAMD_ACTION_NO_THRESHOLD)) {
+ metric_res->actions_config[i].cur_limit = act->threshold;
+ }
+ else {
+ metric_res->actions_config[i].flags |= RSPAMD_ACTION_RESULT_NO_THRESHOLD;
+ }
+ metric_res->actions_config[i].action = act;
+}
+
+struct rspamd_scan_result *
+rspamd_create_metric_result(struct rspamd_task *task,
+ const gchar *name, gint lua_sym_cbref)
+{
+ struct rspamd_scan_result *metric_res;
+
+ metric_res = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct rspamd_scan_result));
+ metric_res->symbols = kh_init(rspamd_symbols_hash);
+ metric_res->sym_groups = kh_init(rspamd_symbols_group_hash);
+
+ if (name) {
+ metric_res->name = rspamd_mempool_strdup(task->task_pool, name);
+ }
+ else {
+ metric_res->name = NULL;
+ }
+
+ metric_res->symbol_cbref = lua_sym_cbref;
+ metric_res->task = task;
+
+ /* Optimize allocation */
+ kh_resize(rspamd_symbols_group_hash, metric_res->sym_groups, 4);
+
+ if (symbols_count.mean > 4) {
+ kh_resize(rspamd_symbols_hash, metric_res->symbols, symbols_count.mean);
+ }
+ else {
+ kh_resize(rspamd_symbols_hash, metric_res->symbols, 4);
+ }
+
+ if (task->cfg) {
+ size_t nact = rspamd_config_actions_size(task->cfg);
+ metric_res->actions_config = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct rspamd_action_config) * nact);
+ rspamd_config_actions_foreach_enumerate(task->cfg, rspamd_metric_actions_foreach_cb, metric_res);
+ metric_res->nactions = nact;
+ }
+
+ rspamd_mempool_add_destructor(task->task_pool,
+ rspamd_scan_result_dtor,
+ metric_res);
+ DL_APPEND(task->result, metric_res);
+
+ return metric_res;
+}
+
+static inline int
+rspamd_pr_sort(const struct rspamd_passthrough_result *pra,
+ const struct rspamd_passthrough_result *prb)
+{
+ return prb->priority - pra->priority;
+}
+
+bool rspamd_add_passthrough_result(struct rspamd_task *task,
+ struct rspamd_action *action,
+ guint priority,
+ double target_score,
+ const gchar *message,
+ const gchar *module,
+ uint flags,
+ struct rspamd_scan_result *scan_result)
+{
+ struct rspamd_passthrough_result *pr;
+
+ if (scan_result == NULL) {
+ scan_result = task->result;
+ }
+
+ /* Find the specific action config */
+ struct rspamd_action_config *action_config = NULL;
+
+ for (unsigned int i = 0; i < scan_result->nactions; i++) {
+ struct rspamd_action_config *cur = &scan_result->actions_config[i];
+
+ /* We assume that all action pointers are static */
+ if (cur->action == action) {
+ action_config = cur;
+ break;
+ }
+ }
+
+ if (action_config && (action_config->flags & RSPAMD_ACTION_RESULT_DISABLED)) {
+ msg_info_task("<%s>: NOT set pre-result to '%s' %s(%.2f): '%s' from %s(%d); action is disabled",
+ MESSAGE_FIELD_CHECK(task, message_id), action->name,
+ flags & RSPAMD_PASSTHROUGH_LEAST ? "*least " : "",
+ target_score,
+ message, module, priority);
+
+ return false;
+ }
+
+ pr = rspamd_mempool_alloc(task->task_pool, sizeof(*pr));
+ pr->action = action;
+ pr->priority = priority;
+ pr->message = message;
+ pr->module = module;
+ pr->target_score = target_score;
+ pr->flags = flags;
+
+ DL_APPEND(scan_result->passthrough_result, pr);
+ DL_SORT(scan_result->passthrough_result, rspamd_pr_sort);
+
+ if (!isnan(target_score)) {
+
+ msg_info_task("<%s>: set pre-result to '%s' %s(%.2f): '%s' from %s(%d)",
+ MESSAGE_FIELD_CHECK(task, message_id), action->name,
+ flags & RSPAMD_PASSTHROUGH_LEAST ? "*least " : "",
+ target_score,
+ message, module, priority);
+ }
+ else {
+ msg_info_task("<%s>: set pre-result to '%s' %s(no score): '%s' from %s(%d)",
+ MESSAGE_FIELD_CHECK(task, message_id), action->name,
+ flags & RSPAMD_PASSTHROUGH_LEAST ? "*least " : "",
+ message, module, priority);
+ }
+
+ scan_result->nresults++;
+
+ return true;
+}
+
+static inline gdouble
+rspamd_check_group_score(struct rspamd_task *task,
+ const gchar *symbol,
+ struct rspamd_symbols_group *gr,
+ gdouble *group_score,
+ gdouble w)
+{
+ if (gr != NULL && group_score && gr->max_score > 0.0 && w > 0.0) {
+ if (*group_score >= gr->max_score && w > 0) {
+ msg_info_task("maximum group score %.2f for group %s has been reached,"
+ " ignoring symbol %s with weight %.2f",
+ gr->max_score,
+ gr->name, symbol, w);
+ return NAN;
+ }
+ else if (*group_score + w > gr->max_score) {
+ w = gr->max_score - *group_score;
+ }
+ }
+
+ return w;
+}
+
+#ifndef DBL_EPSILON
+#define DBL_EPSILON 2.2204460492503131e-16
+#endif
+
+static struct rspamd_symbol_result *
+insert_metric_result(struct rspamd_task *task,
+ const gchar *symbol,
+ double weight,
+ const gchar *opt,
+ struct rspamd_scan_result *metric_res,
+ enum rspamd_symbol_insert_flags flags,
+ bool *new_sym)
+{
+ struct rspamd_symbol_result *symbol_result = NULL;
+ gdouble final_score, *gr_score = NULL, next_gf = 1.0, diff;
+ struct rspamd_symbol *sdef;
+ struct rspamd_symbols_group *gr = NULL;
+ const ucl_object_t *mobj, *sobj;
+ gint max_shots = G_MAXINT, ret;
+ guint i;
+ khiter_t k;
+ gboolean single = !!(flags & RSPAMD_SYMBOL_INSERT_SINGLE);
+ gchar *sym_cpy;
+
+ if (!isfinite(weight)) {
+ msg_warn_task("detected %s score for symbol %s, replace it with zero",
+ isnan(weight) ? "NaN" : "infinity", symbol);
+ weight = 0.0;
+ }
+
+ msg_debug_metric("want to insert symbol %s, initial weight %.2f",
+ symbol, weight);
+
+ sdef = g_hash_table_lookup(task->cfg->symbols, symbol);
+ if (sdef == NULL) {
+ if (flags & RSPAMD_SYMBOL_INSERT_ENFORCE) {
+ final_score = 1.0 * weight; /* Enforce static weight to 1.0 */
+ }
+ else {
+ final_score = 0.0;
+ }
+
+ msg_debug_metric("no symbol definition for %s; final multiplier %.2f",
+ symbol, final_score);
+ }
+ else {
+ if (sdef->cache_item) {
+ /* Check if we can insert this symbol at all */
+ if (!rspamd_symcache_is_item_allowed(task, sdef->cache_item, FALSE)) {
+ msg_debug_metric("symbol %s is not allowed to be inserted due to settings",
+ symbol);
+ return NULL;
+ }
+ }
+
+ final_score = (*sdef->weight_ptr) * weight;
+
+ PTR_ARRAY_FOREACH(sdef->groups, i, gr)
+ {
+ k = kh_get(rspamd_symbols_group_hash, metric_res->sym_groups, gr);
+
+ if (k == kh_end(metric_res->sym_groups)) {
+ k = kh_put(rspamd_symbols_group_hash, metric_res->sym_groups,
+ gr, &ret);
+ kh_value(metric_res->sym_groups, k) = 0;
+ }
+ }
+
+ msg_debug_metric("metric multiplier for %s is %.2f",
+ symbol, *sdef->weight_ptr);
+ }
+
+ if (task->settings) {
+ gdouble corr;
+ mobj = ucl_object_lookup(task->settings, "scores");
+
+ if (!mobj) {
+ /* Legacy */
+ mobj = task->settings;
+ }
+ else {
+ msg_debug_metric("found scores in the settings");
+ }
+
+ sobj = ucl_object_lookup(mobj, symbol);
+ if (sobj != NULL && ucl_object_todouble_safe(sobj, &corr)) {
+ msg_debug_metric("settings: changed weight of symbol %s from %.2f "
+ "to %.2f * %.2f",
+ symbol, final_score, corr, weight);
+ final_score = corr * weight;
+ }
+ }
+
+ k = kh_get(rspamd_symbols_hash, metric_res->symbols, symbol);
+ if (k != kh_end(metric_res->symbols)) {
+ /* Existing metric score */
+ symbol_result = kh_value(metric_res->symbols, k);
+ if (single) {
+ max_shots = 1;
+ }
+ else {
+ if (sdef) {
+ if (sdef->groups) {
+ PTR_ARRAY_FOREACH(sdef->groups, i, gr)
+ {
+ if (gr->flags & RSPAMD_SYMBOL_GROUP_ONE_SHOT) {
+ max_shots = 1;
+ }
+ }
+ }
+
+ max_shots = MIN(max_shots, sdef->nshots);
+ }
+ else {
+ max_shots = task->cfg->default_max_shots;
+ }
+ }
+
+ msg_debug_metric("nshots: %d for symbol %s", max_shots, symbol);
+
+ if (!single && (max_shots > 0 && (symbol_result->nshots >= max_shots))) {
+ single = TRUE;
+ }
+
+ symbol_result->nshots++;
+
+ if (opt) {
+ rspamd_task_add_result_option(task, symbol_result, opt, strlen(opt));
+ }
+
+ /* Adjust diff */
+ if (!single) {
+ diff = final_score;
+ msg_debug_metric("symbol %s can be inserted multiple times: %.2f weight",
+ symbol, diff);
+ }
+ else {
+ if (fabs(symbol_result->score) < fabs(final_score) &&
+ signbit(symbol_result->score) == signbit(final_score)) {
+ /* Replace less significant weight with a more significant one */
+ diff = final_score - symbol_result->score;
+ msg_debug_metric("symbol %s can be inserted single time;"
+ " weight adjusted %.2f + %.2f",
+ symbol, symbol_result->score, diff);
+ }
+ else {
+ diff = 0;
+ }
+ }
+
+ if (diff) {
+ /* Handle grow factor */
+ if (metric_res->grow_factor && diff > 0) {
+ diff *= metric_res->grow_factor;
+ next_gf *= task->cfg->grow_factor;
+ }
+ else if (diff > 0) {
+ next_gf = task->cfg->grow_factor;
+ }
+
+ msg_debug_metric("adjust grow factor to %.2f for symbol %s (%.2f final)",
+ next_gf, symbol, diff);
+
+ if (sdef) {
+ PTR_ARRAY_FOREACH(sdef->groups, i, gr)
+ {
+ gdouble cur_diff;
+
+ k = kh_get(rspamd_symbols_group_hash,
+ metric_res->sym_groups, gr);
+ g_assert(k != kh_end(metric_res->sym_groups));
+ gr_score = &kh_value(metric_res->sym_groups, k);
+ cur_diff = rspamd_check_group_score(task, symbol, gr,
+ gr_score, diff);
+
+ if (isnan(cur_diff)) {
+ /* Limit reached, do not add result */
+ msg_debug_metric(
+ "group limit %.2f is reached for %s when inserting symbol %s;"
+ " drop score %.2f",
+ *gr_score, gr->name, symbol, diff);
+
+ diff = NAN;
+ break;
+ }
+ else if (gr_score) {
+ *gr_score += cur_diff;
+
+ if (cur_diff < diff) {
+ /* Reduce */
+ msg_debug_metric(
+ "group limit %.2f is reached for %s when inserting symbol %s;"
+ " reduce score %.2f - %.2f",
+ *gr_score, gr->name, symbol, diff, cur_diff);
+ diff = cur_diff;
+ }
+ }
+ }
+ }
+
+ if (!isnan(diff)) {
+ metric_res->score += diff;
+ metric_res->grow_factor = next_gf;
+
+ if (single) {
+ msg_debug_metric("final score for single symbol %s = %.2f; %.2f diff",
+ symbol, final_score, diff);
+ symbol_result->score = final_score;
+ }
+ else {
+ msg_debug_metric("increase final score for multiple symbol %s += %.2f = %.2f",
+ symbol, symbol_result->score, diff);
+ symbol_result->score += diff;
+ }
+ }
+ }
+ }
+ else {
+ /* New result */
+ if (new_sym) {
+ *new_sym = true;
+ }
+
+ sym_cpy = rspamd_mempool_strdup(task->task_pool, symbol);
+ k = kh_put(rspamd_symbols_hash, metric_res->symbols,
+ sym_cpy, &ret);
+ g_assert(ret > 0);
+ symbol_result = rspamd_mempool_alloc0(task->task_pool, sizeof(*symbol_result));
+ kh_value(metric_res->symbols, k) = symbol_result;
+
+ /* Handle grow factor */
+ if (metric_res->grow_factor && final_score > 0) {
+ final_score *= metric_res->grow_factor;
+ next_gf *= task->cfg->grow_factor;
+ }
+ else if (final_score > 0) {
+ next_gf = task->cfg->grow_factor;
+ }
+
+ msg_debug_metric("adjust grow factor to %.2f for symbol %s (%.2f final)",
+ next_gf, symbol, final_score);
+
+ symbol_result->name = sym_cpy;
+ symbol_result->sym = sdef;
+ symbol_result->nshots = 1;
+
+ if (sdef) {
+ /* Check group limits */
+ PTR_ARRAY_FOREACH(sdef->groups, i, gr)
+ {
+ gdouble cur_score;
+
+ k = kh_get(rspamd_symbols_group_hash, metric_res->sym_groups, gr);
+ g_assert(k != kh_end(metric_res->sym_groups));
+ gr_score = &kh_value(metric_res->sym_groups, k);
+ cur_score = rspamd_check_group_score(task, symbol, gr,
+ gr_score, final_score);
+
+ if (isnan(cur_score)) {
+ /* Limit reached, do not add result */
+ msg_debug_metric(
+ "group limit %.2f is reached for %s when inserting symbol %s;"
+ " drop score %.2f",
+ *gr_score, gr->name, symbol, final_score);
+ final_score = NAN;
+ break;
+ }
+ else if (gr_score) {
+ *gr_score += cur_score;
+
+ if (cur_score < final_score) {
+ /* Reduce */
+ msg_debug_metric(
+ "group limit %.2f is reached for %s when inserting symbol %s;"
+ " reduce score %.2f - %.2f",
+ *gr_score, gr->name, symbol, final_score, cur_score);
+ final_score = cur_score;
+ }
+ }
+ }
+ }
+
+ if (!isnan(final_score)) {
+ const double epsilon = DBL_EPSILON;
+
+ metric_res->score += final_score;
+ metric_res->grow_factor = next_gf;
+ symbol_result->score = final_score;
+
+ if (final_score > epsilon) {
+ metric_res->npositive++;
+ metric_res->positive_score += final_score;
+ }
+ else if (final_score < -epsilon) {
+ metric_res->nnegative++;
+ metric_res->negative_score += fabs(final_score);
+ }
+ }
+ else {
+ symbol_result->score = 0;
+ }
+
+ if (opt) {
+ rspamd_task_add_result_option(task, symbol_result, opt, strlen(opt));
+ }
+ }
+
+ msg_debug_metric("final insertion for symbol %s, score %.2f, factor: %f",
+ symbol,
+ symbol_result->score,
+ final_score);
+ metric_res->nresults++;
+
+ return symbol_result;
+}
+
+struct rspamd_symbol_result *
+rspamd_task_insert_result_full(struct rspamd_task *task,
+ const gchar *symbol,
+ double weight,
+ const gchar *opt,
+ enum rspamd_symbol_insert_flags flags,
+ struct rspamd_scan_result *result)
+{
+ struct rspamd_symbol_result *symbol_result = NULL, *ret = NULL;
+ struct rspamd_scan_result *mres;
+
+ /*
+ * We allow symbols to be inserted for skipped tasks, as it might be a
+ * race condition before some symbol is finished and skip flag being set.
+ */
+ if (!RSPAMD_TASK_IS_SKIPPED(task) && (task->processed_stages & (RSPAMD_TASK_STAGE_IDEMPOTENT >> 1))) {
+ msg_err_task("cannot insert symbol %s on idempotent phase",
+ symbol);
+
+ return NULL;
+ }
+
+ if (result == NULL) {
+ /* Insert everywhere */
+ DL_FOREACH(task->result, mres)
+ {
+ if (mres->symbol_cbref != -1) {
+ /* Check if we can insert this symbol to this symbol result */
+ GError *err = NULL;
+ lua_State *L = (lua_State *) task->cfg->lua_state;
+
+ if (!rspamd_lua_universal_pcall(L, mres->symbol_cbref,
+ G_STRLOC, 1, "uss", &err,
+ "rspamd{task}", task, symbol, mres->name ? mres->name : "default")) {
+ msg_warn_task("cannot call for symbol_cbref for result %s: %e",
+ mres->name ? mres->name : "default", err);
+ g_error_free(err);
+
+ continue;
+ }
+ else {
+ if (!lua_toboolean(L, -1)) {
+ /* Skip symbol */
+ msg_debug_metric("skip symbol %s for result %s due to Lua return value",
+ symbol, mres->name);
+ lua_pop(L, 1); /* Remove result */
+
+ continue;
+ }
+
+ lua_pop(L, 1); /* Remove result */
+ }
+ }
+
+ bool new_symbol = false;
+
+ symbol_result = insert_metric_result(task,
+ symbol,
+ weight,
+ opt,
+ mres,
+ flags,
+ &new_symbol);
+
+ if (mres->name == NULL) {
+ /* Default result */
+ ret = symbol_result;
+
+ /* Process cache item */
+ if (symbol_result && task->cfg->cache && symbol_result->sym && symbol_result->nshots == 1) {
+ rspamd_symcache_inc_frequency(task->cfg->cache,
+ symbol_result->sym->cache_item,
+ symbol_result->sym->name);
+ }
+ }
+ else if (new_symbol) {
+ /* O(N) but we normally don't have any shadow results */
+ LL_APPEND(ret, symbol_result);
+ }
+ }
+ }
+ else {
+ /* Specific insertion */
+ symbol_result = insert_metric_result(task,
+ symbol,
+ weight,
+ opt,
+ result,
+ flags,
+ NULL);
+ ret = symbol_result;
+
+ if (result->name == NULL) {
+ /* Process cache item */
+ if (symbol_result && task->cfg->cache && symbol_result->sym && symbol_result->nshots == 1) {
+ rspamd_symcache_inc_frequency(task->cfg->cache,
+ symbol_result->sym->cache_item,
+ symbol_result->sym->name);
+ }
+ }
+ }
+
+ return ret;
+}
+
+static gchar *
+rspamd_task_option_safe_copy(struct rspamd_task *task,
+ const gchar *val,
+ gsize vlen,
+ gsize *outlen)
+{
+ const gchar *p, *end;
+
+ p = val;
+ end = val + vlen;
+ vlen = 0; /* Reuse */
+
+ while (p < end) {
+ if (*p & 0x80) {
+ UChar32 uc;
+ gint off = 0;
+
+ U8_NEXT(p, off, end - p, uc);
+
+ if (uc > 0) {
+ if (u_isprint(uc)) {
+ vlen += off;
+ }
+ else {
+ /* We will replace it with 0xFFFD */
+ vlen += MAX(off, 3);
+ }
+ }
+ else {
+ vlen += MAX(off, 3);
+ }
+
+ p += off;
+ }
+ else if (!g_ascii_isprint(*p)) {
+ /* Another 0xFFFD */
+ vlen += 3;
+ p++;
+ }
+ else {
+ p++;
+ vlen++;
+ }
+ }
+
+ gchar *dest, *d;
+
+ dest = rspamd_mempool_alloc(task->task_pool, vlen + 1);
+ d = dest;
+ p = val;
+
+ while (p < end) {
+ if (*p & 0x80) {
+ UChar32 uc;
+ gint off = 0;
+
+ U8_NEXT(p, off, end - p, uc);
+
+ if (uc > 0) {
+ if (u_isprint(uc)) {
+ memcpy(d, p, off);
+ d += off;
+ }
+ else {
+ /* We will replace it with 0xFFFD */
+ *d++ = '\357';
+ *d++ = '\277';
+ *d++ = '\275';
+ }
+ }
+ else {
+ *d++ = '\357';
+ *d++ = '\277';
+ *d++ = '\275';
+ }
+
+ p += off;
+ }
+ else if (!g_ascii_isprint(*p)) {
+ /* Another 0xFFFD */
+ *d++ = '\357';
+ *d++ = '\277';
+ *d++ = '\275';
+ p++;
+ }
+ else {
+ *d++ = *p++;
+ }
+ }
+
+ *d = '\0';
+ *(outlen) = d - dest;
+
+ return dest;
+}
+
+gboolean
+rspamd_task_add_result_option(struct rspamd_task *task,
+ struct rspamd_symbol_result *s,
+ const gchar *val,
+ gsize vlen)
+{
+ struct rspamd_symbol_option *opt, srch;
+ gboolean ret = FALSE;
+ gchar *opt_cpy = NULL;
+ gsize cpy_len;
+ khiter_t k;
+ gint r;
+ struct rspamd_symbol_result *cur;
+
+ if (s && val) {
+ /*
+ * Here we assume that this function is all the time called with the
+ * symbol from the default result, not some shadow result, or
+ * the option insertion will be wrong
+ */
+ LL_FOREACH(s, cur)
+ {
+ if (cur->opts_len < 0) {
+ /* Cannot add more options, give up */
+ msg_debug_task("cannot add more options to symbol %s when adding option %s",
+ cur->name, val);
+ ret = FALSE;
+ continue;
+ }
+
+ if (!cur->options) {
+ cur->options = kh_init(rspamd_options_hash);
+ }
+
+ if (vlen + cur->opts_len > task->cfg->max_opts_len) {
+ /* Add truncated option */
+ msg_info_task("cannot add more options to symbol %s when adding option %s",
+ cur->name, val);
+ val = "...";
+ vlen = 3;
+ cur->opts_len = -1;
+ }
+
+ if (!(cur->sym && (cur->sym->flags & RSPAMD_SYMBOL_FLAG_ONEPARAM))) {
+
+ srch.option = (gchar *) val;
+ srch.optlen = vlen;
+ k = kh_get(rspamd_options_hash, cur->options, &srch);
+
+ if (k == kh_end(cur->options)) {
+ opt_cpy = rspamd_task_option_safe_copy(task, val, vlen, &cpy_len);
+ if (cpy_len != vlen) {
+ srch.option = (gchar *) opt_cpy;
+ srch.optlen = cpy_len;
+ k = kh_get(rspamd_options_hash, cur->options, &srch);
+ }
+ /* Append new options */
+ if (k == kh_end(cur->options)) {
+ opt = rspamd_mempool_alloc0(task->task_pool, sizeof(*opt));
+ opt->optlen = cpy_len;
+ opt->option = opt_cpy;
+
+ kh_put(rspamd_options_hash, cur->options, opt, &r);
+ DL_APPEND(cur->opts_head, opt);
+
+ if (s == cur) {
+ ret = TRUE;
+ }
+ }
+ }
+ }
+ else {
+ /* Skip addition */
+ if (s == cur) {
+ ret = FALSE;
+ }
+ }
+
+ if (ret && cur->opts_len >= 0) {
+ cur->opts_len += vlen;
+ }
+ }
+ }
+ else if (!val) {
+ ret = TRUE;
+ }
+
+ task->result->nresults++;
+
+ return ret;
+}
+
+struct rspamd_action_config *
+rspamd_find_action_config_for_action(struct rspamd_scan_result *scan_result,
+ struct rspamd_action *act)
+{
+ for (unsigned int i = 0; i < scan_result->nactions; i++) {
+ struct rspamd_action_config *cur = &scan_result->actions_config[i];
+
+ if (act == cur->action) {
+ return cur;
+ }
+ }
+
+ return NULL;
+}
+
+struct rspamd_action *
+rspamd_check_action_metric(struct rspamd_task *task,
+ struct rspamd_passthrough_result **ppr,
+ struct rspamd_scan_result *scan_result)
+{
+ struct rspamd_action_config *action_lim,
+ *noaction = NULL;
+ struct rspamd_action *selected_action = NULL, *least_action = NULL;
+ struct rspamd_passthrough_result *pr, *sel_pr = NULL;
+ double max_score = -(G_MAXDOUBLE), sc;
+ gboolean seen_least = FALSE;
+
+ if (scan_result == NULL) {
+ scan_result = task->result;
+ }
+
+ if (scan_result->passthrough_result != NULL) {
+ DL_FOREACH(scan_result->passthrough_result, pr)
+ {
+ struct rspamd_action_config *act_config =
+ rspamd_find_action_config_for_action(scan_result, pr->action);
+
+ /* Skip disabled actions */
+ if (act_config && (act_config->flags & RSPAMD_ACTION_RESULT_DISABLED)) {
+ continue;
+ }
+
+ if (!seen_least || !(pr->flags & RSPAMD_PASSTHROUGH_LEAST)) {
+ sc = pr->target_score;
+ selected_action = pr->action;
+
+ if (!(pr->flags & RSPAMD_PASSTHROUGH_LEAST)) {
+ if (!isnan(sc)) {
+ if (pr->action->action_type == METRIC_ACTION_NOACTION) {
+ scan_result->score = MIN(sc, scan_result->score);
+ }
+ else {
+ scan_result->score = sc;
+ }
+ }
+
+ if (ppr) {
+ *ppr = pr;
+ }
+
+ return selected_action;
+ }
+ else {
+ seen_least = true;
+ least_action = selected_action;
+
+ if (isnan(sc)) {
+
+ if (selected_action->flags & RSPAMD_ACTION_NO_THRESHOLD) {
+ /*
+ * In this case, we have a passthrough action that
+ * is `least` action, however, there is no threshold
+ * on it.
+ *
+ * Hence, we imply the following logic:
+ *
+ * - we leave score unchanged
+ * - we apply passthrough no threshold action unless
+ * score based action *is not* reject, otherwise
+ * we apply reject action
+ */
+ }
+ else {
+ sc = selected_action->threshold;
+ max_score = sc;
+ sel_pr = pr;
+ }
+ }
+ else {
+ max_score = sc;
+ sel_pr = pr;
+ }
+ }
+ }
+ }
+ }
+
+ /*
+ * Select result by score
+ */
+ for (size_t i = scan_result->nactions - 1; i != (size_t) -1; i--) {
+ action_lim = &scan_result->actions_config[i];
+ sc = action_lim->cur_limit;
+
+ if (action_lim->action->action_type == METRIC_ACTION_NOACTION) {
+ noaction = action_lim;
+ }
+
+ if ((action_lim->flags & (RSPAMD_ACTION_RESULT_DISABLED | RSPAMD_ACTION_RESULT_NO_THRESHOLD))) {
+ continue;
+ }
+
+ if (isnan(sc) ||
+ (action_lim->action->flags & (RSPAMD_ACTION_NO_THRESHOLD | RSPAMD_ACTION_HAM))) {
+ continue;
+ }
+
+ if (scan_result->score >= sc && sc > max_score) {
+ selected_action = action_lim->action;
+ max_score = sc;
+ }
+ }
+
+ if (selected_action == NULL) {
+ selected_action = noaction->action;
+ }
+
+ if (selected_action) {
+
+ if (seen_least) {
+ /* Adjust least action */
+ if (least_action->flags & RSPAMD_ACTION_NO_THRESHOLD) {
+ if (selected_action->action_type != METRIC_ACTION_REJECT &&
+ selected_action->action_type != METRIC_ACTION_DISCARD) {
+ /* Override score based action with least action */
+ selected_action = least_action;
+
+ if (ppr) {
+ *ppr = sel_pr;
+ }
+ }
+ }
+ else {
+ /* Adjust score if needed */
+ if (max_score > scan_result->score) {
+ if (ppr) {
+ *ppr = sel_pr;
+ }
+
+ scan_result->score = max_score;
+ }
+ }
+ }
+
+ return selected_action;
+ }
+
+ if (ppr) {
+ *ppr = sel_pr;
+ }
+
+ return noaction->action;
+}
+
+struct rspamd_symbol_result *
+rspamd_task_find_symbol_result(struct rspamd_task *task, const char *sym,
+ struct rspamd_scan_result *result)
+{
+ struct rspamd_symbol_result *res = NULL;
+ khiter_t k;
+
+ if (result == NULL) {
+ /* Use default result */
+ result = task->result;
+ }
+
+ k = kh_get(rspamd_symbols_hash, result->symbols, sym);
+
+ if (k != kh_end(result->symbols)) {
+ res = kh_value(result->symbols, k);
+ }
+
+ return res;
+}
+
+struct rspamd_symbol_result *rspamd_task_remove_symbol_result(
+ struct rspamd_task *task,
+ const gchar *symbol,
+ struct rspamd_scan_result *result)
+{
+ struct rspamd_symbol_result *res = NULL;
+ khiter_t k;
+
+ if (result == NULL) {
+ /* Use default result */
+ result = task->result;
+ }
+
+ k = kh_get(rspamd_symbols_hash, result->symbols, symbol);
+
+ if (k != kh_end(result->symbols)) {
+ res = kh_value(result->symbols, k);
+
+ if (!isnan(res->score)) {
+ /* Remove score from the result */
+ result->score -= res->score;
+
+ /* Also check the group limit */
+ if (result->sym_groups && res->sym) {
+ struct rspamd_symbol_group *gr;
+ gint i;
+ khiter_t k_groups;
+
+ PTR_ARRAY_FOREACH(res->sym->groups, i, gr)
+ {
+ gdouble *gr_score;
+
+ k_groups = kh_get(rspamd_symbols_group_hash,
+ result->sym_groups, gr);
+
+ if (k_groups != kh_end(result->sym_groups)) {
+ gr_score = &kh_value(result->sym_groups, k_groups);
+
+ if (gr_score) {
+ *gr_score -= res->score;
+ }
+ }
+ }
+ }
+ }
+
+ kh_del(rspamd_symbols_hash, result->symbols, k);
+ }
+ else {
+ return NULL;
+ }
+
+ return res;
+}
+
+void rspamd_task_symbol_result_foreach(struct rspamd_task *task,
+ struct rspamd_scan_result *result, GHFunc func,
+ gpointer ud)
+{
+ const gchar *kk;
+ struct rspamd_symbol_result *res;
+
+ if (result == NULL) {
+ /* Use default result */
+ result = task->result;
+ }
+
+ if (func) {
+ kh_foreach(result->symbols, kk, res, {
+ func((gpointer) kk, (gpointer) res, ud);
+ });
+ }
+}
+
+struct rspamd_scan_result *
+rspamd_find_metric_result(struct rspamd_task *task,
+ const gchar *name)
+{
+ struct rspamd_scan_result *res;
+
+ if (name == NULL || strcmp(name, "default") == 0) {
+ return task->result;
+ }
+
+ DL_FOREACH(task->result, res)
+ {
+ if (res->name && strcmp(res->name, name) == 0) {
+ return res;
+ }
+ }
+
+ return NULL;
+}
diff --git a/src/libmime/scan_result.h b/src/libmime/scan_result.h
new file mode 100644
index 0000000..46c2de8
--- /dev/null
+++ b/src/libmime/scan_result.h
@@ -0,0 +1,250 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @file scan_result.h
+ * Scan result holder
+ */
+
+#ifndef RSPAMD_SCAN_RESULT_H
+#define RSPAMD_SCAN_RESULT_H
+
+#include "config.h"
+#include "rspamd_symcache.h"
+#include "task.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_settings;
+struct rspamd_classifier_config;
+
+struct rspamd_symbol_option {
+ gchar *option;
+ gsize optlen;
+ struct rspamd_symbol_option *prev, *next;
+};
+
+enum rspamd_symbol_result_flags {
+ RSPAMD_SYMBOL_RESULT_NORMAL = 0,
+ RSPAMD_SYMBOL_RESULT_IGNORED = (1 << 0)
+};
+
+struct kh_rspamd_options_hash_s;
+
+/**
+ * Rspamd symbol
+ */
+struct rspamd_symbol_result {
+ double score; /**< symbol's score */
+ struct kh_rspamd_options_hash_s *options; /**< list of symbol's options */
+ struct rspamd_symbol_option *opts_head; /**< head of linked list of options */
+ const gchar *name;
+ struct rspamd_symbol *sym; /**< symbol configuration */
+ gssize opts_len; /**< total size of all options (negative if truncated option is added) */
+ guint nshots;
+ int flags;
+ struct rspamd_symbol_result *next; /**< for shadow results */
+};
+
+
+#define RSPAMD_PASSTHROUGH_NORMAL 1
+#define RSPAMD_PASSTHROUGH_LOW 0
+#define RSPAMD_PASSTHROUGH_HIGH 2
+#define RSPAMD_PASSTHROUGH_CRITICAL 3
+
+#define RSPAMD_PASSTHROUGH_LEAST (1u << 0u)
+#define RSPAMD_PASSTHROUGH_NO_SMTP_MESSAGE (1u << 1u)
+#define RSPAMD_PASSTHROUGH_PROCESS_ALL (1u << 2u)
+
+struct rspamd_passthrough_result {
+ struct rspamd_action *action;
+ guint priority;
+ guint flags;
+ double target_score;
+ const gchar *message;
+ const gchar *module;
+ struct rspamd_passthrough_result *prev, *next;
+};
+
+
+enum rspamd_action_config_flags {
+ RSPAMD_ACTION_RESULT_DEFAULT = 0,
+ RSPAMD_ACTION_RESULT_NO_THRESHOLD = (1u << 0u),
+ RSPAMD_ACTION_RESULT_DISABLED = (1u << 1u),
+};
+struct rspamd_action_config {
+ gdouble cur_limit;
+ int flags;
+ struct rspamd_action *action;
+};
+
+struct kh_rspamd_symbols_hash_s;
+struct kh_rspamd_symbols_group_hash_s;
+
+
+struct rspamd_scan_result {
+ double score; /**< total score */
+ double grow_factor; /**< current grow factor */
+ struct rspamd_passthrough_result *passthrough_result;
+ double positive_score;
+ double negative_score;
+ struct kh_rspamd_symbols_hash_s *symbols; /**< symbols of metric */
+ struct kh_rspamd_symbols_group_hash_s *sym_groups; /**< groups of symbols */
+ struct rspamd_action_config *actions_config;
+ const gchar *name; /**< for named results, NULL is the default result */
+ struct rspamd_task *task; /**< back reference */
+ gint symbol_cbref; /**< lua function that defines if a symbol can be inserted, -1 if unused */
+ guint nactions;
+ guint npositive;
+ guint nnegative;
+ guint nresults; /**< all results: positive, negative, passthrough etc */
+ guint nresults_postfilters; /**< how many results are there before postfilters stage */
+ struct rspamd_scan_result *prev, *next; /**< double linked list of results */
+};
+
+/**
+ * Create or return existing result for the specified metric name
+ * @param task task object
+ * @return metric result or NULL if metric `name` has not been found
+ */
+struct rspamd_scan_result *rspamd_create_metric_result(struct rspamd_task *task,
+ const gchar *name, gint lua_sym_cbref);
+
+/**
+ * Find result with a specific name (NULL means the default result)
+ * @param task
+ * @param name
+ * @return
+ */
+struct rspamd_scan_result *rspamd_find_metric_result(struct rspamd_task *task,
+ const gchar *name);
+
+/**
+ * Adds a new passthrough result to a task
+ * @param task
+ * @param action
+ * @param priority
+ * @param target_score
+ * @param message
+ * @param module
+ */
+bool rspamd_add_passthrough_result(struct rspamd_task *task,
+ struct rspamd_action *action, guint priority,
+ double target_score, const gchar *message,
+ const gchar *module, guint flags,
+ struct rspamd_scan_result *scan_result);
+
+enum rspamd_symbol_insert_flags {
+ RSPAMD_SYMBOL_INSERT_DEFAULT = 0,
+ RSPAMD_SYMBOL_INSERT_SINGLE = (1 << 0),
+ RSPAMD_SYMBOL_INSERT_ENFORCE = (1 << 1),
+};
+
+/**
+ * Insert a result to task
+ * @param task worker's task that present message from user
+ * @param metric_name metric's name to which we need to insert result
+ * @param symbol symbol to insert
+ * @param weight numeric weight for symbol
+ * @param opts list of symbol's options
+ */
+struct rspamd_symbol_result *rspamd_task_insert_result_full(struct rspamd_task *task,
+ const gchar *symbol,
+ double weight,
+ const gchar *opts,
+ enum rspamd_symbol_insert_flags flags,
+ struct rspamd_scan_result *result);
+
+#define rspamd_task_insert_result_single(task, symbol, weight, opts) \
+ rspamd_task_insert_result_full((task), (symbol), (weight), (opts), RSPAMD_SYMBOL_INSERT_SINGLE, NULL)
+#define rspamd_task_insert_result(task, symbol, weight, opts) \
+ rspamd_task_insert_result_full((task), (symbol), (weight), (opts), RSPAMD_SYMBOL_INSERT_DEFAULT, NULL)
+
+/**
+ * Removes a symbol from a specific symbol result
+ * @param task
+ * @param symbol
+ * @param result
+ * @return
+ */
+struct rspamd_symbol_result *rspamd_task_remove_symbol_result(
+ struct rspamd_task *task,
+ const gchar *symbol,
+ struct rspamd_scan_result *result);
+/**
+ * Adds new option to symbol
+ * @param task
+ * @param s
+ * @param opt
+ */
+gboolean rspamd_task_add_result_option(struct rspamd_task *task,
+ struct rspamd_symbol_result *s,
+ const gchar *opt,
+ gsize vlen);
+
+/**
+ * Finds symbol result
+ * @param task
+ * @param sym
+ * @return
+ */
+struct rspamd_symbol_result *
+rspamd_task_find_symbol_result(struct rspamd_task *task, const char *sym,
+ struct rspamd_scan_result *result);
+
+/**
+ * Compatibility function to iterate on symbols hash
+ * @param task
+ * @param func
+ * @param ud
+ */
+void rspamd_task_symbol_result_foreach(struct rspamd_task *task,
+ struct rspamd_scan_result *result,
+ GHFunc func,
+ gpointer ud);
+
+/**
+ * Default consolidation function for metric, it get all symbols and multiply symbol
+ * weight by some factor that is specified in config. Default factor is 1.
+ * @param task worker's task that present message from user
+ * @param metric_name name of metric
+ * @return result metric weight
+ */
+double rspamd_factor_consolidation_func(struct rspamd_task *task,
+ const gchar *metric_name,
+ const gchar *unused);
+
+
+/**
+ * Check thresholds and return action for a task
+ * @param task
+ * @return
+ */
+struct rspamd_action *rspamd_check_action_metric(struct rspamd_task *task,
+ struct rspamd_passthrough_result **ppr,
+ struct rspamd_scan_result *scan_result);
+
+struct rspamd_action_config *rspamd_find_action_config_for_action(struct rspamd_scan_result *scan_result,
+ struct rspamd_action *act);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libmime/scan_result_private.h b/src/libmime/scan_result_private.h
new file mode 100644
index 0000000..cf0c0c5
--- /dev/null
+++ b/src/libmime/scan_result_private.h
@@ -0,0 +1,55 @@
+//
+// Created by Vsevolod Stakhov on 2019-01-14.
+//
+
+#ifndef RSPAMD_SCAN_RESULT_PRIVATE_H
+#define RSPAMD_SCAN_RESULT_PRIVATE_H
+
+#include "scan_result.h"
+#include "contrib/libucl/khash.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RSPAMD_OPTS_SEED 0x9f1f608628a4fefbULL
+#define rspamd_symopt_hash(opt) (rspamd_cryptobox_fast_hash( \
+ ((struct rspamd_symbol_option *) opt)->option, \
+ ((struct rspamd_symbol_option *) opt)->optlen, RSPAMD_OPTS_SEED))
+static inline bool
+rspamd_symopt_equal(const struct rspamd_symbol_option *o1,
+ const struct rspamd_symbol_option *o2)
+{
+ if (o1->optlen == o2->optlen) {
+ return (memcmp(o1->option, o2->option, o1->optlen) == 0);
+ }
+
+ return false;
+}
+
+KHASH_INIT(rspamd_options_hash, struct rspamd_symbol_option *, char,
+ 0, rspamd_symopt_hash, rspamd_symopt_equal);
+/**
+ * Result of metric processing
+ */
+KHASH_MAP_INIT_STR(rspamd_symbols_hash, struct rspamd_symbol_result *);
+#if UINTPTR_MAX <= UINT_MAX
+/* 32 bit */
+#define rspamd_ptr_hash_func(key) (khint32_t)(((uintptr_t) (key)) >> 1)
+#else
+/* likely 64 bit */
+#define rspamd_ptr_hash_func(key) (khint32_t)(((uintptr_t) (key)) >> 3)
+#endif
+#define rspamd_ptr_equal_func(a, b) ((a) == (b))
+KHASH_INIT(rspamd_symbols_group_hash,
+ void *,
+ double,
+ 1,
+ rspamd_ptr_hash_func,
+ rspamd_ptr_equal_func);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif//RSPAMD_SCAN_RESULT_PRIVATE_H
diff --git a/src/libmime/smtp_parsers.h b/src/libmime/smtp_parsers.h
new file mode 100644
index 0000000..e188b63
--- /dev/null
+++ b/src/libmime/smtp_parsers.h
@@ -0,0 +1,51 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_SMTP_PARSERS_H_
+#define SRC_LIBMIME_SMTP_PARSERS_H_
+
+#include "config.h"
+#include "email_addr.h"
+#include "content_type.h"
+#include "task.h"
+#include "message.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int rspamd_smtp_addr_parse(const char *data, size_t len,
+ struct rspamd_email_address *addr);
+
+gboolean rspamd_content_disposition_parser(const char *data, size_t len,
+ struct rspamd_content_disposition *cd,
+ rspamd_mempool_t *pool);
+
+gboolean
+rspamd_rfc2047_parser(const gchar *in, gsize len, gint *pencoding,
+ const gchar **charset, gsize *charset_len,
+ const gchar **encoded, gsize *encoded_len);
+
+rspamd_inet_addr_t *rspamd_parse_smtp_ip(const char *data, size_t len,
+ rspamd_mempool_t *pool);
+
+guint64 rspamd_parse_smtp_date(const unsigned char *data, size_t len, GError **err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_SMTP_PARSERS_H_ */
diff --git a/src/libserver/CMakeLists.txt b/src/libserver/CMakeLists.txt
new file mode 100644
index 0000000..dd17865
--- /dev/null
+++ b/src/libserver/CMakeLists.txt
@@ -0,0 +1,52 @@
+# Librspamdserver
+ADD_SUBDIRECTORY(css)
+SET(LIBRSPAMDSERVERSRC
+ ${CMAKE_CURRENT_SOURCE_DIR}/cfg_utils.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/cfg_rcl.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/composites/composites.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/composites/composites_manager.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/dkim.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/dns.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/dynamic_cfg.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/async_session.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend/fuzzy_backend.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend/fuzzy_backend_sqlite.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend/fuzzy_backend_redis.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/milter.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/monitored.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/protocol.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/re_cache.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/redis_pool.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/roll_history.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/spf.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/ssl_util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/symcache/symcache_impl.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/symcache/symcache_item.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/symcache/symcache_runtime.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/symcache/symcache_c.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/task.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/url.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/worker_util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/logger/logger.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/logger/logger_file.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/logger/logger_syslog.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/logger/logger_console.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http/http_util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http/http_message.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http/http_connection.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http/http_router.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http/http_context.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/maps/map.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/maps/map_helpers.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/html/html_entities.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/html/html_url.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/html/html.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/html/html_tests.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/hyperscan_tools.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/backtrace.cxx
+ ${LIBCSSSRC})
+
+# Librspamd-server
+SET(RSPAMD_SERVER ${LIBRSPAMDSERVERSRC} PARENT_SCOPE)
+SET(LIBSERVER_DEPENDS "${LIBCSS_DEPENDS}" PARENT_SCOPE)
+SET(LIBSERVER_GENERATED "${LIBCSS_GENERATED}" PARENT_SCOPE)
diff --git a/src/libserver/async_session.c b/src/libserver/async_session.c
new file mode 100644
index 0000000..baaee62
--- /dev/null
+++ b/src/libserver/async_session.c
@@ -0,0 +1,364 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "contrib/uthash/utlist.h"
+#include "contrib/libucl/khash.h"
+#include "async_session.h"
+#include "cryptobox.h"
+
+#define RSPAMD_SESSION_FLAG_DESTROYING (1 << 1)
+#define RSPAMD_SESSION_FLAG_CLEANUP (1 << 2)
+
+#define RSPAMD_SESSION_CAN_ADD_EVENT(s) (!((s)->flags & (RSPAMD_SESSION_FLAG_DESTROYING | RSPAMD_SESSION_FLAG_CLEANUP)))
+
+#define msg_err_session(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "events", session->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_warn_session(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "events", session->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_info_session(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "events", session->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_debug_session(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_events_log_id, "events", session->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(events)
+
+/* Average symbols count to optimize hash allocation */
+static struct rspamd_counter_data events_count;
+
+
+struct rspamd_async_event {
+ const gchar *subsystem;
+ const gchar *event_source;
+ event_finalizer_t fin;
+ void *user_data;
+};
+
+static inline bool
+rspamd_event_equal(const struct rspamd_async_event *ev1, const struct rspamd_async_event *ev2)
+{
+ return ev1->fin == ev2->fin && ev1->user_data == ev2->user_data;
+}
+
+static inline guint64
+rspamd_event_hash(const struct rspamd_async_event *ev)
+{
+ union _pointer_fp_thunk {
+ event_finalizer_t f;
+ gpointer p;
+ };
+ struct ev_storage {
+ union _pointer_fp_thunk p;
+ gpointer ud;
+ } st;
+
+ st.p.f = ev->fin;
+ st.ud = ev->user_data;
+
+ return rspamd_cryptobox_fast_hash(&st, sizeof(st), rspamd_hash_seed());
+}
+
+/* Define **SET** of events */
+KHASH_INIT(rspamd_events_hash,
+ struct rspamd_async_event *,
+ char,
+ false,
+ rspamd_event_hash,
+ rspamd_event_equal);
+
+struct rspamd_async_session {
+ session_finalizer_t fin;
+ event_finalizer_t restore;
+ event_finalizer_t cleanup;
+ khash_t(rspamd_events_hash) * events;
+ void *user_data;
+ rspamd_mempool_t *pool;
+ guint flags;
+};
+
+static void
+rspamd_session_dtor(gpointer d)
+{
+ struct rspamd_async_session *s = (struct rspamd_async_session *) d;
+
+ /* Events are usually empty at this point */
+ rspamd_set_counter_ema(&events_count, s->events->n_buckets, 0.5);
+ kh_destroy(rspamd_events_hash, s->events);
+}
+
+struct rspamd_async_session *
+rspamd_session_create(rspamd_mempool_t *pool,
+ session_finalizer_t fin,
+ event_finalizer_t restore,
+ event_finalizer_t cleanup,
+ void *user_data)
+{
+ struct rspamd_async_session *s;
+
+ s = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_async_session));
+ s->pool = pool;
+ s->fin = fin;
+ s->restore = restore;
+ s->cleanup = cleanup;
+ s->user_data = user_data;
+ s->events = kh_init(rspamd_events_hash);
+
+ kh_resize(rspamd_events_hash, s->events, MAX(4, events_count.mean));
+ rspamd_mempool_add_destructor(pool, rspamd_session_dtor, s);
+
+ return s;
+}
+
+struct rspamd_async_event *
+rspamd_session_add_event_full(struct rspamd_async_session *session,
+ event_finalizer_t fin,
+ gpointer user_data,
+ const gchar *subsystem,
+ const gchar *event_source)
+{
+ struct rspamd_async_event *new_event;
+ gint ret;
+
+ if (session == NULL) {
+ msg_err("session is NULL");
+ g_assert_not_reached();
+ }
+
+ if (!RSPAMD_SESSION_CAN_ADD_EVENT(session)) {
+ msg_debug_session("skip adding event subsystem: %s: "
+ "session is destroying/cleaning",
+ subsystem);
+
+ return NULL;
+ }
+
+ new_event = rspamd_mempool_alloc(session->pool,
+ sizeof(struct rspamd_async_event));
+ new_event->fin = fin;
+ new_event->user_data = user_data;
+ new_event->subsystem = subsystem;
+ new_event->event_source = event_source;
+
+ msg_debug_session("added event: %p, pending %d (+1) events, "
+ "subsystem: %s (%s)",
+ user_data,
+ kh_size(session->events),
+ subsystem,
+ event_source);
+
+ kh_put(rspamd_events_hash, session->events, new_event, &ret);
+ g_assert(ret > 0);
+
+ return new_event;
+}
+
+void rspamd_session_remove_event_full(struct rspamd_async_session *session,
+ event_finalizer_t fin,
+ void *ud,
+ const gchar *event_source)
+{
+ struct rspamd_async_event search_ev, *found_ev;
+ khiter_t k;
+
+ if (session == NULL) {
+ msg_err("session is NULL");
+ return;
+ }
+
+ if (!RSPAMD_SESSION_CAN_ADD_EVENT(session)) {
+ /* Session is already cleaned up, ignore this */
+ return;
+ }
+
+ /* Search for event */
+ search_ev.fin = fin;
+ search_ev.user_data = ud;
+ k = kh_get(rspamd_events_hash, session->events, &search_ev);
+ if (k == kh_end(session->events)) {
+
+ msg_err_session("cannot find event: %p(%p) from %s (%d total events)", fin, ud,
+ event_source, (int) kh_size(session->events));
+ kh_foreach_key(session->events, found_ev, {
+ msg_err_session("existing event %s (%s): %p(%p)",
+ found_ev->subsystem,
+ found_ev->event_source,
+ found_ev->fin,
+ found_ev->user_data);
+ });
+
+ g_assert_not_reached();
+ }
+
+ found_ev = kh_key(session->events, k);
+ msg_debug_session("removed event: %p, pending %d (-1) events, "
+ "subsystem: %s (%s), added at %s",
+ ud,
+ kh_size(session->events),
+ found_ev->subsystem,
+ event_source,
+ found_ev->event_source);
+ kh_del(rspamd_events_hash, session->events, k);
+
+ /* Remove event */
+ if (fin) {
+ fin(ud);
+ }
+
+ rspamd_session_pending(session);
+}
+
+gboolean
+rspamd_session_destroy(struct rspamd_async_session *session)
+{
+ if (session == NULL) {
+ msg_err("session is NULL");
+ return FALSE;
+ }
+
+ if (!rspamd_session_blocked(session)) {
+ session->flags |= RSPAMD_SESSION_FLAG_DESTROYING;
+ rspamd_session_cleanup(session, false);
+
+ if (session->cleanup != NULL) {
+ session->cleanup(session->user_data);
+ }
+ }
+
+ return TRUE;
+}
+
+void rspamd_session_cleanup(struct rspamd_async_session *session, bool forced_cleanup)
+{
+ struct rspamd_async_event *ev;
+
+ if (session == NULL) {
+ msg_err("session is NULL");
+ return;
+ }
+
+ session->flags |= RSPAMD_SESSION_FLAG_CLEANUP;
+ khash_t(rspamd_events_hash) *uncancellable_events = kh_init(rspamd_events_hash);
+
+ kh_foreach_key(session->events, ev, {
+ /* Call event's finalizer */
+ int ret;
+
+ if (ev->fin != NULL) {
+ if (forced_cleanup) {
+ msg_info_session("forced removed event on destroy: %p, subsystem: %s, scheduled from: %s",
+ ev->user_data,
+ ev->subsystem,
+ ev->event_source);
+ }
+ else {
+ msg_debug_session("removed event on destroy: %p, subsystem: %s",
+ ev->user_data,
+ ev->subsystem);
+ }
+ ev->fin(ev->user_data);
+ }
+ else {
+ if (forced_cleanup) {
+ msg_info_session("NOT forced removed event on destroy - uncancellable: "
+ "%p, subsystem: %s, scheduled from: %s",
+ ev->user_data,
+ ev->subsystem,
+ ev->event_source);
+ }
+ else {
+ msg_debug_session("NOT removed event on destroy - uncancellable: %p, subsystem: %s",
+ ev->user_data,
+ ev->subsystem);
+ }
+ /* Assume an event is uncancellable, move it to a new hash table */
+ kh_put(rspamd_events_hash, uncancellable_events, ev, &ret);
+ }
+ });
+
+ kh_destroy(rspamd_events_hash, session->events);
+ session->events = uncancellable_events;
+ if (forced_cleanup) {
+ msg_info_session("pending %d uncancellable events", kh_size(uncancellable_events));
+ }
+ else {
+ msg_debug_session("pending %d uncancellable events", kh_size(uncancellable_events));
+ }
+
+ session->flags &= ~RSPAMD_SESSION_FLAG_CLEANUP;
+}
+
+gboolean
+rspamd_session_pending(struct rspamd_async_session *session)
+{
+ gboolean ret = TRUE;
+
+ if (kh_size(session->events) == 0) {
+ if (session->fin != NULL) {
+ msg_debug_session("call fin handler, as no events are pending");
+
+ if (!session->fin(session->user_data)) {
+ /* Session finished incompletely, perform restoration */
+ msg_debug_session("restore incomplete session");
+ if (session->restore != NULL) {
+ session->restore(session->user_data);
+ }
+ }
+ else {
+ ret = FALSE;
+ }
+ }
+
+ ret = FALSE;
+ }
+
+ return ret;
+}
+
+guint rspamd_session_events_pending(struct rspamd_async_session *session)
+{
+ guint npending;
+
+ g_assert(session != NULL);
+
+ npending = kh_size(session->events);
+ msg_debug_session("pending %d events", npending);
+
+ return npending;
+}
+
+rspamd_mempool_t *
+rspamd_session_mempool(struct rspamd_async_session *session)
+{
+ g_assert(session != NULL);
+
+ return session->pool;
+}
+
+gboolean
+rspamd_session_blocked(struct rspamd_async_session *session)
+{
+ g_assert(session != NULL);
+
+ return !RSPAMD_SESSION_CAN_ADD_EVENT(session);
+} \ No newline at end of file
diff --git a/src/libserver/async_session.h b/src/libserver/async_session.h
new file mode 100644
index 0000000..4573545
--- /dev/null
+++ b/src/libserver/async_session.h
@@ -0,0 +1,121 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_ASYNC_SESSION_H
+#define RSPAMD_ASYNC_SESSION_H
+
+#include "config.h"
+#include "mem_pool.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_async_event;
+struct rspamd_async_session;
+
+typedef void (*event_finalizer_t)(gpointer ud);
+
+typedef gboolean (*session_finalizer_t)(gpointer user_data);
+
+/**
+ * Make new async session
+ * @param pool pool to alloc memory from
+ * @param fin a callback called when no events are found in session
+ * @param restore a callback is called to restore processing of session
+ * @param cleanup a callback called when session is forcefully destroyed
+ * @param user_data abstract user data
+ * @return
+ */
+struct rspamd_async_session *rspamd_session_create(rspamd_mempool_t *pool,
+ session_finalizer_t fin, event_finalizer_t restore,
+ event_finalizer_t cleanup, gpointer user_data);
+
+/**
+ * Insert new event to the session
+ * @param session session object
+ * @param fin finalizer callback
+ * @param user_data abstract user_data
+ * @param forced unused
+ */
+struct rspamd_async_event *
+rspamd_session_add_event_full(struct rspamd_async_session *session,
+ event_finalizer_t fin,
+ gpointer user_data,
+ const gchar *subsystem,
+ const gchar *event_source);
+
+#define rspamd_session_add_event(session, fin, user_data, subsystem) \
+ rspamd_session_add_event_full(session, fin, user_data, subsystem, G_STRLOC)
+
+/**
+ * Remove normal event
+ * @param session session object
+ * @param fin final callback
+ * @param ud user data object
+ */
+void rspamd_session_remove_event_full(struct rspamd_async_session *session,
+ event_finalizer_t fin,
+ gpointer ud,
+ const gchar *event_source);
+
+#define rspamd_session_remove_event(session, fin, user_data) \
+ rspamd_session_remove_event_full(session, fin, user_data, G_STRLOC)
+
+/**
+ * Must be called at the end of session, it calls fin functions for all non-forced callbacks
+ * @return true if the whole session was destroyed and false if there are forced events
+ */
+gboolean rspamd_session_destroy(struct rspamd_async_session *session);
+
+/**
+ * Try to remove all events pending
+ */
+void rspamd_session_cleanup(struct rspamd_async_session *session, bool forced_cleanup);
+
+/**
+ * Returns mempool associated with async session
+ * @param session
+ * @return
+ */
+rspamd_mempool_t *rspamd_session_mempool(struct rspamd_async_session *session);
+
+/**
+ * Check session for events pending and call fin callback if no events are pending
+ * @param session session object
+ * @return TRUE if session has pending events
+ */
+gboolean rspamd_session_pending(struct rspamd_async_session *session);
+
+/**
+ * Returns number of events pending
+ * @param session
+ * @return
+ */
+guint rspamd_session_events_pending(struct rspamd_async_session *session);
+
+
+/**
+ * Returns TRUE if an async session is currently destroying
+ * @param s
+ * @return
+ */
+gboolean rspamd_session_blocked(struct rspamd_async_session *s);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*RSPAMD_ASYNC_SESSION_H*/
diff --git a/src/libserver/backtrace.cxx b/src/libserver/backtrace.cxx
new file mode 100644
index 0000000..6507b96
--- /dev/null
+++ b/src/libserver/backtrace.cxx
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+
+#ifdef BACKWARD_ENABLE
+
+#include "contrib/backward-cpp/backward.hpp"
+#include "fmt/core.h"
+#include "logger.h"
+
+namespace rspamd {
+
+void log_backtrace(void)
+{
+ using namespace backward;
+ StackTrace st;
+ st.load_here(128);
+
+ TraceResolver tr;
+ tr.load_stacktrace(st);
+
+ for (auto i = 0ul; i < st.size(); ++i) {
+ auto trace = tr.resolve(st[i]);
+ auto trace_line = fmt::format("#{}: [{}]: ", i, trace.addr);
+
+ if (!trace.source.filename.empty()) {
+ trace_line += fmt::format("{}:{} in {}", trace.source.filename, trace.source.line, trace.source.function);
+ }
+ else {
+ trace_line += fmt::format("{} in {}", trace.object_filename, trace.object_function);
+ }
+
+ msg_err("%s", trace_line.c_str());
+ }
+}
+
+}// namespace rspamd
+#endif
+
+extern "C" void rspamd_print_crash(void);
+
+void rspamd_print_crash(void)
+{
+#ifdef BACKWARD_ENABLE
+ rspamd::log_backtrace();
+#endif
+} \ No newline at end of file
diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h
new file mode 100644
index 0000000..4cb87d9
--- /dev/null
+++ b/src/libserver/cfg_file.h
@@ -0,0 +1,889 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CFG_FILE_H
+#define CFG_FILE_H
+
+#include "config.h"
+#include "mem_pool.h"
+#include "upstream.h"
+#include "rspamd_symcache.h"
+#include "cfg_rcl.h"
+#include "ucl.h"
+#include "regexp.h"
+#include "libserver/re_cache.h"
+#include "libutil/ref.h"
+#include "libutil/radix.h"
+#include "monitored.h"
+#include "redis_pool.h"
+
+#define DEFAULT_BIND_PORT 11333
+#define DEFAULT_CONTROL_PORT 11334
+
+/* Default metric name */
+#define DEFAULT_METRIC "default"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct expression;
+struct tokenizer;
+struct rspamd_stat_classifier;
+struct module_s;
+struct worker_s;
+struct rspamd_external_libs_ctx;
+struct rspamd_cryptobox_pubkey;
+struct rspamd_dns_resolver;
+
+/**
+ * Logging type
+ */
+enum rspamd_log_type {
+ RSPAMD_LOG_CONSOLE,
+ RSPAMD_LOG_SYSLOG,
+ RSPAMD_LOG_FILE
+};
+
+enum rspamd_log_cfg_flags {
+ RSPAMD_LOG_FLAG_DEFAULT = 0u,
+ RSPAMD_LOG_FLAG_SYSTEMD = (1u << 0u),
+ RSPAMD_LOG_FLAG_COLOR = (1u << 1u),
+ RSPAMD_LOG_FLAG_RE_CACHE = (1u << 2u),
+ RSPAMD_LOG_FLAG_USEC = (1u << 3u),
+ RSPAMD_LOG_FLAG_RSPAMADM = (1u << 4u),
+ RSPAMD_LOG_FLAG_ENFORCED = (1u << 5u),
+ RSPAMD_LOG_FLAG_SEVERITY = (1u << 6u),
+ RSPAMD_LOG_FLAG_JSON = (1u << 7u),
+};
+
+struct rspamd_worker_log_pipe {
+ gint fd;
+ gint type;
+ struct rspamd_worker_log_pipe *prev, *next;
+};
+
+/**
+ * script module list item
+ */
+struct script_module {
+ gchar *name; /**< name of module */
+ gchar *path; /**< path to module */
+ gchar *digest;
+};
+
+enum rspamd_symbol_group_flags {
+ RSPAMD_SYMBOL_GROUP_NORMAL = 0u,
+ RSPAMD_SYMBOL_GROUP_DISABLED = (1u << 0u),
+ RSPAMD_SYMBOL_GROUP_ONE_SHOT = (1u << 1u),
+ RSPAMD_SYMBOL_GROUP_UNGROUPED = (1u << 2u),
+ RSPAMD_SYMBOL_GROUP_PUBLIC = (1u << 3u),
+};
+
+/**
+ * Symbols group
+ */
+struct rspamd_symbol;
+struct rspamd_symbols_group {
+ gchar *name;
+ gchar *description;
+ GHashTable *symbols;
+ gdouble max_score;
+ guint flags;
+};
+
+enum rspamd_symbol_flags {
+ RSPAMD_SYMBOL_FLAG_NORMAL = 0,
+ RSPAMD_SYMBOL_FLAG_IGNORE_METRIC = (1 << 1),
+ RSPAMD_SYMBOL_FLAG_ONEPARAM = (1 << 2),
+ RSPAMD_SYMBOL_FLAG_UNGROUPED = (1 << 3),
+ RSPAMD_SYMBOL_FLAG_DISABLED = (1 << 4),
+ RSPAMD_SYMBOL_FLAG_UNSCORED = (1 << 5),
+};
+
+/**
+ * Symbol config definition
+ */
+struct rspamd_symbol {
+ gchar *name;
+ gchar *description;
+ gdouble *weight_ptr;
+ gdouble score;
+ guint priority;
+ struct rspamd_symbols_group *gr; /* Main group */
+ GPtrArray *groups; /* Other groups */
+ guint flags;
+ void *cache_item;
+ gint nshots;
+};
+
+/**
+ * Statfile config definition
+ */
+struct rspamd_statfile_config {
+ gchar *symbol; /**< symbol of statfile */
+ gchar *label; /**< label of this statfile */
+ ucl_object_t *opts; /**< other options */
+ gboolean is_spam; /**< spam flag */
+ struct rspamd_classifier_config *clcf; /**< parent pointer of classifier configuration */
+ gpointer data; /**< opaque data */
+};
+
+struct rspamd_tokenizer_config {
+ const ucl_object_t *opts; /**< other options */
+ const gchar *name; /**< name of tokenizer */
+};
+
+
+/* Classifier has all integer values (e.g. bayes) */
+#define RSPAMD_FLAG_CLASSIFIER_INTEGER (1 << 0)
+/*
+ * Set if backend for a classifier is intended to increment and not set values
+ * (e.g. redis)
+ */
+#define RSPAMD_FLAG_CLASSIFIER_INCREMENTING_BACKEND (1 << 1)
+/*
+ * No backend required for classifier
+ */
+#define RSPAMD_FLAG_CLASSIFIER_NO_BACKEND (1 << 2)
+
+/**
+ * Classifier config definition
+ */
+struct rspamd_classifier_config {
+ GList *statfiles; /**< statfiles list */
+ GHashTable *labels; /**< statfiles with labels */
+ gchar *metric; /**< metric of this classifier */
+ gchar *classifier; /**< classifier interface */
+ struct rspamd_tokenizer_config *tokenizer; /**< tokenizer used for classifier */
+ const gchar *backend; /**< name of statfile's backend */
+ ucl_object_t *opts; /**< other options */
+ GList *learn_conditions; /**< list of learn condition callbacks */
+ GList *classify_conditions; /**< list of classify condition callbacks */
+ gchar *name; /**< unique name of classifier */
+ guint32 min_tokens; /**< minimal number of tokens to process classifier */
+ guint32 max_tokens; /**< maximum number of tokens */
+ guint min_token_hits; /**< minimum number of hits for a token to be considered */
+ gdouble min_prob_strength; /**< use only tokens with probability in [0.5 - MPS, 0.5 + MPS] */
+ guint min_learns; /**< minimum number of learns for each statfile */
+ guint flags;
+};
+
+struct rspamd_worker_bind_conf {
+ GPtrArray *addrs;
+ guint cnt;
+ gchar *name;
+ gchar *bind_line;
+ gboolean is_systemd;
+ struct rspamd_worker_bind_conf *next;
+};
+
+struct rspamd_worker_lua_script {
+ gint cbref;
+ struct rspamd_worker_lua_script *prev, *next;
+};
+
+/**
+ * Config params for rspamd worker
+ */
+struct rspamd_worker_conf {
+ struct worker_s *worker; /**< pointer to worker type */
+ GQuark type; /**< type of worker */
+ struct rspamd_worker_bind_conf *bind_conf; /**< bind configuration */
+ gint16 count; /**< number of workers */
+ GList *listen_socks; /**< listening sockets descriptors */
+ guint64 rlimit_nofile; /**< max files limit */
+ guint64 rlimit_maxcore; /**< maximum core file size */
+ GHashTable *params; /**< params for worker */
+ GQueue *active_workers; /**< linked list of spawned workers */
+ gpointer ctx; /**< worker's context */
+ ucl_object_t *options; /**< other worker's options */
+ struct rspamd_worker_lua_script *scripts; /**< registered lua scripts */
+ gboolean enabled;
+ ref_entry_t ref;
+};
+
+enum rspamd_log_format_type {
+ RSPAMD_LOG_STRING = 0,
+ RSPAMD_LOG_MID,
+ RSPAMD_LOG_QID,
+ RSPAMD_LOG_USER,
+ RSPAMD_LOG_ISSPAM,
+ RSPAMD_LOG_ACTION,
+ RSPAMD_LOG_SCORES,
+ RSPAMD_LOG_SYMBOLS,
+ RSPAMD_LOG_IP,
+ RSPAMD_LOG_LEN,
+ RSPAMD_LOG_DNS_REQ,
+ RSPAMD_LOG_SMTP_FROM,
+ RSPAMD_LOG_MIME_FROM,
+ RSPAMD_LOG_SMTP_RCPT,
+ RSPAMD_LOG_MIME_RCPT,
+ RSPAMD_LOG_SMTP_RCPTS,
+ RSPAMD_LOG_MIME_RCPTS,
+ RSPAMD_LOG_TIME_REAL,
+ RSPAMD_LOG_TIME_VIRTUAL,
+ RSPAMD_LOG_LUA,
+ RSPAMD_LOG_DIGEST,
+ RSPAMD_LOG_FILENAME,
+ RSPAMD_LOG_FORCED_ACTION,
+ RSPAMD_LOG_SETTINGS_ID,
+ RSPAMD_LOG_GROUPS,
+ RSPAMD_LOG_PUBLIC_GROUPS,
+ RSPAMD_LOG_MEMPOOL_SIZE,
+ RSPAMD_LOG_MEMPOOL_WASTE,
+};
+
+enum rspamd_log_format_flags {
+ RSPAMD_LOG_FMT_FLAG_DEFAULT = 0,
+ RSPAMD_LOG_FMT_FLAG_OPTIONAL = (1 << 0),
+ RSPAMD_LOG_FMT_FLAG_MIME_ALTERNATIVE = (1 << 1),
+ RSPAMD_LOG_FMT_FLAG_CONDITION = (1 << 2),
+ RSPAMD_LOG_FMT_FLAG_SYMBOLS_SCORES = (1 << 3),
+ RSPAMD_LOG_FMT_FLAG_SYMBOLS_PARAMS = (1 << 4)
+};
+
+struct rspamd_log_format {
+ enum rspamd_log_format_type type;
+ guint flags;
+ gsize len;
+ gpointer data;
+ struct rspamd_log_format *prev, *next;
+};
+
+/**
+ * Standard actions
+ */
+enum rspamd_action_type {
+ METRIC_ACTION_REJECT = 0,
+ METRIC_ACTION_SOFT_REJECT,
+ METRIC_ACTION_REWRITE_SUBJECT,
+ METRIC_ACTION_ADD_HEADER,
+ METRIC_ACTION_GREYLIST,
+ METRIC_ACTION_NOACTION,
+ METRIC_ACTION_MAX,
+ METRIC_ACTION_CUSTOM = 999,
+ METRIC_ACTION_DISCARD,
+ METRIC_ACTION_QUARANTINE
+};
+
+enum rspamd_action_flags {
+ RSPAMD_ACTION_NORMAL = 0u,
+ RSPAMD_ACTION_NO_THRESHOLD = (1u << 0u),
+ RSPAMD_ACTION_THRESHOLD_ONLY = (1u << 1u),
+ RSPAMD_ACTION_HAM = (1u << 2u),
+ RSPAMD_ACTION_MILTER = (1u << 3u),
+};
+
+
+struct rspamd_action;
+
+struct rspamd_config_cfg_lua_script {
+ gint cbref;
+ gint priority;
+ gchar *lua_src_pos;
+ struct rspamd_config_cfg_lua_script *prev, *next;
+};
+
+struct rspamd_config_post_init_script {
+ gint cbref;
+ struct rspamd_config_post_init_script *prev, *next;
+};
+
+struct rspamd_lang_detector;
+struct rspamd_rcl_sections_map;
+
+enum rspamd_config_settings_policy {
+ RSPAMD_SETTINGS_POLICY_DEFAULT = 0,
+ RSPAMD_SETTINGS_POLICY_IMPLICIT_ALLOW = 1,
+ RSPAMD_SETTINGS_POLICY_IMPLICIT_DENY = 2,
+};
+
+enum rspamd_gtube_patterns_policy {
+ RSPAMD_GTUBE_DISABLED = 0, /* Disabled */
+ RSPAMD_GTUBE_REJECT, /* Reject message with GTUBE pattern */
+ RSPAMD_GTUBE_ALL /* Check all GTUBE like patterns */
+};
+
+struct rspamd_config_settings_elt {
+ guint32 id;
+ enum rspamd_config_settings_policy policy;
+ const gchar *name;
+ ucl_object_t *symbols_enabled;
+ ucl_object_t *symbols_disabled;
+ struct rspamd_config_settings_elt *prev, *next;
+ ref_entry_t ref;
+};
+
+/**
+ * Structure that stores all config data
+ */
+struct rspamd_config {
+ gchar *rspamd_user; /**< user to run as */
+ gchar *rspamd_group; /**< group to run as */
+ rspamd_mempool_t *cfg_pool; /**< memory pool for config */
+ gchar *cfg_name; /**< name of config file */
+ gchar *pid_file; /**< name of pid file */
+ gchar *temp_dir; /**< dir for temp files */
+ gchar *control_socket_path; /**< path to the control socket */
+ const ucl_object_t *local_addrs; /**< tree of local addresses */
+#ifdef WITH_GPERF_TOOLS
+ gchar *profile_path;
+#endif
+ gdouble unknown_weight; /**< weight of unknown symbols */
+ gdouble grow_factor; /**< grow factor for metric */
+ GHashTable *symbols; /**< weights of symbols in metric */
+ const gchar *subject; /**< subject rewrite string */
+ GHashTable *groups; /**< groups of symbols */
+ void *actions; /**< all actions of the metric (opaque type) */
+
+ gboolean one_shot_mode; /**< rules add only one symbol */
+ gboolean check_text_attachements; /**< check text attachements as text */
+ gboolean check_all_filters; /**< check all filters */
+ gboolean allow_raw_input; /**< scan messages with invalid mime */
+ gboolean disable_hyperscan; /**< disable hyperscan usage */
+ gboolean vectorized_hyperscan; /**< use vectorized hyperscan matching */
+ gboolean enable_shutdown_workaround; /**< enable workaround for legacy SA clients (exim) */
+ gboolean ignore_received; /**< Ignore data from the first received header */
+ gboolean enable_sessions_cache; /**< Enable session cache for debug */
+ gboolean enable_experimental; /**< Enable experimental plugins */
+ gboolean disable_pcre_jit; /**< Disable pcre JIT */
+ gboolean own_lua_state; /**< True if we have created lua_state internally */
+ gboolean soft_reject_on_timeout; /**< If true emit soft reject on task timeout (if not reject) */
+ gboolean public_groups_only; /**< Output merely public groups everywhere */
+ enum rspamd_gtube_patterns_policy gtube_patterns_policy; /**< Enable test patterns */
+ gboolean enable_css_parser; /**< Enable css parsing in HTML */
+
+ gsize max_cores_size; /**< maximum size occupied by rspamd core files */
+ gsize max_cores_count; /**< maximum number of core files */
+ gchar *cores_dir; /**< directory for core files */
+ gsize max_message; /**< maximum size for messages */
+ gsize max_pic_size; /**< maximum size for a picture to process */
+ gsize images_cache_size; /**< size of LRU cache for DCT data from images */
+ gdouble task_timeout; /**< maximum message processing time */
+ gint default_max_shots; /**< default maximum count of symbols hits permitted (-1 for unlimited) */
+ gint32 heartbeats_loss_max; /**< number of heartbeats lost to consider worker's termination */
+ gdouble heartbeat_interval; /**< interval for heartbeats for workers */
+
+ enum rspamd_log_type log_type; /**< log type */
+ gint log_facility; /**< log facility in case of syslog */
+ gint log_level; /**< log level trigger */
+ gchar *log_file; /**< path to logfile in case of file logging */
+ gboolean log_buffered; /**< whether logging is buffered */
+ gboolean log_silent_workers; /**< silence info messages from workers */
+ guint32 log_buf_size; /**< length of log buffer */
+ const ucl_object_t *debug_ip_map; /**< turn on debugging for specified ip addresses */
+ gboolean log_urls; /**< whether we should log URLs */
+ GHashTable *debug_modules; /**< logging modules to debug */
+ struct rspamd_cryptobox_pubkey *log_encryption_key; /**< encryption key for logs */
+ guint log_flags; /**< logging flags */
+ guint log_error_elts; /**< number of elements in error logbuf */
+ guint log_error_elt_maxlen; /**< maximum size of error log element */
+ guint log_task_max_elts; /**< maximum number of elements in task logging */
+ struct rspamd_worker_log_pipe *log_pipes;
+
+ gboolean compat_messages; /**< use old messages in the protocol (array) */
+
+ GPtrArray *script_modules; /**< a list of script modules to load */
+ GHashTable *explicit_modules; /**< modules that should be always loaded */
+
+ GList *filters; /**< linked list of all filters */
+ GList *workers; /**< linked list of all workers params */
+ struct rspamd_rcl_sections_map *rcl_top_section; /**< top section for RCL config */
+ ucl_object_t *cfg_ucl_obj; /**< ucl object */
+ ucl_object_t *config_comments; /**< comments saved from the config */
+ ucl_object_t *doc_strings; /**< documentation strings for config options */
+ GPtrArray *c_modules; /**< list of C modules */
+ void *composites_manager; /**< hash of composite symbols indexed by its name */
+ GList *classifiers; /**< list of all classifiers defined */
+ GList *statfiles; /**< list of all statfiles in config file order */
+ GHashTable *classifiers_symbols; /**< hashtable indexed by symbol name of classifiers */
+ GHashTable *cfg_params; /**< all cfg params indexed by its name in this structure */
+ gchar *dynamic_conf; /**< path to dynamic configuration */
+ ucl_object_t *current_dynamic_conf; /**< currently loaded dynamic configuration */
+ gint clock_res; /**< resolution of clock used */
+
+ GList *maps; /**< maps active */
+ gdouble map_timeout; /**< maps watch timeout */
+ gdouble map_file_watch_multiplier; /**< multiplier for watch timeout when maps are files */
+ gchar *maps_cache_dir; /**< where to save HTTP cached data */
+
+ gdouble monitored_interval; /**< interval between monitored checks */
+ gboolean disable_monitored; /**< disable monitoring completely */
+ gboolean fips_mode; /**< turn on fips mode for openssl */
+
+ struct rspamd_symcache *cache; /**< symbols cache object */
+ gchar *cache_filename; /**< filename of cache file */
+ gdouble cache_reload_time; /**< how often cache reload should be performed */
+ gchar *checksum; /**< real checksum of config file */
+ gpointer lua_state; /**< pointer to lua state */
+ gpointer lua_thread_pool; /**< pointer to lua thread (coroutine) pool */
+
+ gchar *rrd_file; /**< rrd file to store statistics */
+ gchar *history_file; /**< file to save rolling history */
+ gchar *stats_file; /**< file to save stats */
+ gchar *tld_file; /**< file to load effective tld list from */
+ gchar *hs_cache_dir; /**< directory to save hyperscan databases */
+ gchar *events_backend; /**< string representation of the events backend used */
+
+ gdouble dns_timeout; /**< timeout in milliseconds for waiting for dns reply */
+ guint32 dns_retransmits; /**< maximum retransmits count */
+ guint32 dns_io_per_server; /**< number of sockets per DNS server */
+ const ucl_object_t *nameservers; /**< list of nameservers or NULL to parse resolv.conf */
+ guint32 dns_max_requests; /**< limit of DNS requests per task */
+ gboolean enable_dnssec; /**< enable dnssec stub resolver */
+
+ guint upstream_max_errors; /**< upstream max errors before shutting off */
+ gdouble upstream_error_time; /**< rate of upstream errors */
+ gdouble upstream_revive_time; /**< revive timeout for upstreams */
+ gdouble upstream_lazy_resolve_time; /**< lazy resolve time for upstreams */
+ struct upstream_ctx *ups_ctx; /**< upstream context */
+ struct rspamd_dns_resolver *dns_resolver; /**< dns resolver if loaded */
+
+ guint min_word_len; /**< minimum length of the word to be considered */
+ guint max_word_len; /**< maximum length of the word to be considered */
+ guint words_decay; /**< limit for words for starting adaptive ignoring */
+ guint history_rows; /**< number of history rows stored */
+ guint max_sessions_cache; /**< maximum number of sessions cache elts */
+ guint lua_gc_step; /**< lua gc step */
+ guint lua_gc_pause; /**< lua gc pause */
+ guint full_gc_iters; /**< iterations between full gc cycle */
+ guint max_lua_urls; /**< maximum number of urls to be passed to Lua */
+ guint max_urls; /**< maximum number of urls to be processed in general */
+ gint max_recipients; /**< maximum number of recipients to be processed */
+ guint max_blas_threads; /**< maximum threads for openblas when learning ANN */
+ guint max_opts_len; /**< maximum length for all options for a symbol */
+ gsize max_html_len; /**< maximum length of HTML document */
+
+ struct module_s **compiled_modules; /**< list of compiled C modules */
+ struct worker_s **compiled_workers; /**< list of compiled C modules */
+ struct rspamd_log_format *log_format; /**< parsed log format */
+ gchar *log_format_str; /**< raw log format string */
+
+ struct rspamd_external_libs_ctx *libs_ctx; /**< context for external libraries */
+ struct rspamd_monitored_ctx *monitored_ctx; /**< context for monitored resources */
+ void *redis_pool; /**< redis connection pool */
+
+ struct rspamd_re_cache *re_cache; /**< static regexp cache */
+
+ GHashTable *trusted_keys; /**< list of trusted public keys */
+
+ struct rspamd_config_cfg_lua_script *on_load_scripts; /**< list of scripts executed on workers load */
+ struct rspamd_config_cfg_lua_script *post_init_scripts; /**< list of scripts executed on config being fully loaded */
+ struct rspamd_config_cfg_lua_script *on_term_scripts; /**< list of callbacks called on worker's termination */
+ struct rspamd_config_cfg_lua_script *config_unload_scripts; /**< list of scripts executed on config unload */
+
+ gchar *ssl_ca_path; /**< path to CA certs */
+ gchar *ssl_ciphers; /**< set of preferred ciphers */
+ gchar *zstd_input_dictionary; /**< path to zstd input dictionary */
+ gchar *zstd_output_dictionary; /**< path to zstd output dictionary */
+ ucl_object_t *neighbours; /**< other servers in the cluster */
+
+ struct rspamd_config_settings_elt *setting_ids; /**< preprocessed settings ids */
+ struct rspamd_lang_detector *lang_det; /**< language detector */
+ struct rspamd_worker *cur_worker; /**< set dynamically by each worker */
+
+ ref_entry_t ref; /**< reference counter */
+};
+
+
+/**
+ * Parse bind credits
+ * @param cf config file to use
+ * @param str line that presents bind line
+ * @param type type of credits
+ * @return 1 if line was successfully parsed and 0 in case of error
+ */
+gboolean rspamd_parse_bind_line(struct rspamd_config *cfg,
+ struct rspamd_worker_conf *cf, const gchar *str);
+
+
+enum rspamd_config_init_flags {
+ RSPAMD_CONFIG_INIT_DEFAULT = 0u,
+ RSPAMD_CONFIG_INIT_SKIP_LUA = (1u << 0u),
+ RSPAMD_CONFIG_INIT_WIPE_LUA_MEM = (1u << 1u),
+};
+
+/**
+ * Init default values
+ * @param cfg config file
+ */
+struct rspamd_config *rspamd_config_new(enum rspamd_config_init_flags flags);
+
+/**
+ * Free memory used by config structure
+ * @param cfg config file
+ */
+void rspamd_config_free(struct rspamd_config *cfg);
+
+/**
+ * Gets module option with specified name
+ * @param cfg config file
+ * @param module_name name of module
+ * @param opt_name name of option to get
+ * @return module value or NULL if option does not defined
+ */
+const ucl_object_t *rspamd_config_get_module_opt(struct rspamd_config *cfg,
+ const gchar *module_name,
+ const gchar *opt_name) G_GNUC_WARN_UNUSED_RESULT;
+
+
+/**
+ * Parse flag
+ * @param str string representation of flag (eg. 'on')
+ * @return numeric value of flag (0 or 1)
+ */
+gint rspamd_config_parse_flag(const gchar *str, guint len);
+
+enum rspamd_post_load_options {
+ RSPAMD_CONFIG_INIT_URL = 1 << 0,
+ RSPAMD_CONFIG_INIT_LIBS = 1 << 1,
+ RSPAMD_CONFIG_INIT_SYMCACHE = 1 << 2,
+ RSPAMD_CONFIG_INIT_VALIDATE = 1 << 3,
+ RSPAMD_CONFIG_INIT_NO_TLD = 1 << 4,
+ RSPAMD_CONFIG_INIT_PRELOAD_MAPS = 1 << 5,
+ RSPAMD_CONFIG_INIT_POST_LOAD_LUA = 1 << 6,
+};
+
+#define RSPAMD_CONFIG_LOAD_ALL (RSPAMD_CONFIG_INIT_URL | \
+ RSPAMD_CONFIG_INIT_LIBS | \
+ RSPAMD_CONFIG_INIT_SYMCACHE | \
+ RSPAMD_CONFIG_INIT_VALIDATE | \
+ RSPAMD_CONFIG_INIT_PRELOAD_MAPS | \
+ RSPAMD_CONFIG_INIT_POST_LOAD_LUA)
+
+/**
+ * Do post load actions for config
+ * @param cfg config file
+ */
+gboolean rspamd_config_post_load(struct rspamd_config *cfg,
+ enum rspamd_post_load_options opts);
+
+/*
+ * Return a new classifier_config structure, setting default and non-conflicting attributes
+ */
+struct rspamd_classifier_config *rspamd_config_new_classifier(
+ struct rspamd_config *cfg,
+ struct rspamd_classifier_config *c);
+
+/*
+ * Return a new worker_conf structure, setting default and non-conflicting attributes
+ */
+struct rspamd_worker_conf *rspamd_config_new_worker(struct rspamd_config *cfg,
+ struct rspamd_worker_conf *c);
+
+/*
+ * Return a new metric structure, setting default and non-conflicting attributes
+ */
+void rspamd_config_init_metric(struct rspamd_config *cfg);
+
+/*
+ * Return new symbols group definition
+ */
+struct rspamd_symbols_group *rspamd_config_new_group(
+ struct rspamd_config *cfg,
+ const gchar *name);
+
+/*
+ * Return a new statfile structure, setting default and non-conflicting attributes
+ */
+struct rspamd_statfile_config *rspamd_config_new_statfile(
+ struct rspamd_config *cfg,
+ struct rspamd_statfile_config *c);
+
+/*
+ * Register symbols of classifiers inside metrics
+ */
+void rspamd_config_insert_classify_symbols(struct rspamd_config *cfg);
+
+/*
+ * Check statfiles inside a classifier
+ */
+gboolean rspamd_config_check_statfiles(struct rspamd_classifier_config *cf);
+
+/*
+ * Find classifier config by name
+ */
+struct rspamd_classifier_config *rspamd_config_find_classifier(
+ struct rspamd_config *cfg,
+ const gchar *name);
+
+void rspamd_ucl_add_conf_macros(struct ucl_parser *parser,
+ struct rspamd_config *cfg);
+
+void rspamd_ucl_add_conf_variables(struct ucl_parser *parser, GHashTable *vars);
+
+/**
+ * Initialize rspamd filtering system (lua and C filters)
+ * @param cfg
+ * @param reconfig
+ * @return
+ */
+gboolean rspamd_init_filters(struct rspamd_config *cfg, bool reconfig, bool strict);
+
+/**
+ * Add new symbol to the metric
+ * @param cfg
+ * @param metric metric's name (or NULL for the default metric)
+ * @param symbol symbol's name
+ * @param score symbol's score
+ * @param description optional description
+ * @param group optional group name
+ * @param one_shot TRUE if symbol can add its score once
+ * @param rewrite_existing TRUE if we need to rewrite the existing symbol
+ * @param priority use the following priority for a symbol
+ * @param nshots means maximum number of hits for a symbol in metric (-1 for unlimited)
+ * @return TRUE if symbol has been inserted or FALSE if symbol already exists with higher priority
+ */
+gboolean rspamd_config_add_symbol(struct rspamd_config *cfg,
+ const gchar *symbol,
+ gdouble score,
+ const gchar *description,
+ const gchar *group,
+ guint flags,
+ guint priority,
+ gint nshots);
+
+/**
+ * Adds new group for a symbol
+ * @param cfg
+ * @param symbol
+ * @param group
+ * @return
+ */
+gboolean rspamd_config_add_symbol_group(struct rspamd_config *cfg,
+ const gchar *symbol,
+ const gchar *group);
+
+/**
+ * Sets action score for a specified metric with the specified priority
+ * @param cfg config file
+ * @param metric metric name (or NULL for default metric)
+ * @param action_name symbolic name of action
+ * @param obj data to set for action
+ * @return TRUE if symbol has been inserted or FALSE if action already exists with higher priority
+ */
+gboolean rspamd_config_set_action_score(struct rspamd_config *cfg,
+ const gchar *action_name,
+ const ucl_object_t *obj);
+
+/**
+ * Check priority and maybe disable action completely
+ * @param cfg
+ * @param action_name
+ * @param priority
+ * @return
+ */
+gboolean rspamd_config_maybe_disable_action(struct rspamd_config *cfg,
+ const gchar *action_name,
+ guint priority);
+
+/**
+ * Checks if a specified C or lua module is enabled or disabled in the config.
+ * The logic of check is the following:
+ *
+ * - For C modules, we check `filters` line and enable module only if it is found there
+ * - For LUA modules we check the corresponding configuration section:
+ * - if section exists, then we check `enabled` key and check its value
+ * - if section is absent, we consider module as disabled
+ * - For both C and LUA modules we check if the group with the module name is disabled in the default metric
+ * @param cfg config file
+ * @param module_name module name
+ * @return TRUE if a module is enabled
+ */
+gboolean rspamd_config_is_module_enabled(struct rspamd_config *cfg,
+ const gchar *module_name);
+
+/**
+ * Verifies enabled/disabled combination in the specified object
+ * @param obj
+ * @return TRUE if there is no explicit disable in the object found
+ */
+gboolean rspamd_config_is_enabled_from_ucl(rspamd_mempool_t *pool,
+ const ucl_object_t *obj);
+
+/*
+ * Get action from a string
+ */
+gboolean rspamd_action_from_str(const gchar *data, enum rspamd_action_type *result);
+
+/*
+ * Return textual representation of action enumeration
+ */
+const gchar *rspamd_action_to_str(enum rspamd_action_type action);
+
+const gchar *rspamd_action_to_str_alt(enum rspamd_action_type action);
+
+/**
+ * Parse radix tree or radix map from ucl object
+ * @param cfg configuration object
+ * @param obj ucl object with parameter
+ * @param target target radix tree
+ * @param err error pointer
+ * @return
+ */
+struct rspamd_radix_map_helper;
+
+gboolean rspamd_config_radix_from_ucl(struct rspamd_config *cfg, const ucl_object_t *obj, const gchar *description,
+ struct rspamd_radix_map_helper **target, GError **err,
+ struct rspamd_worker *worker, const gchar *map_name);
+
+/**
+ * Adds new settings id to be preprocessed
+ * @param cfg
+ * @param name
+ * @param symbols_enabled (ownership is transferred to callee)
+ * @param symbols_disabled (ownership is transferred to callee)
+ */
+void rspamd_config_register_settings_id(struct rspamd_config *cfg,
+ const gchar *name,
+ ucl_object_t *symbols_enabled,
+ ucl_object_t *symbols_disabled,
+ enum rspamd_config_settings_policy policy);
+
+/**
+ * Convert settings name to settings id
+ * @param name
+ * @param namelen
+ * @return
+ */
+guint32 rspamd_config_name_to_id(const gchar *name, gsize namelen);
+
+/**
+ * Finds settings id element and obtain reference count (must be unrefed by caller)
+ * @param cfg
+ * @param id
+ * @return
+ */
+struct rspamd_config_settings_elt *rspamd_config_find_settings_id_ref(
+ struct rspamd_config *cfg,
+ guint32 id);
+
+/**
+ * Finds settings id element and obtain reference count (must be unrefed by callee)
+ * @param cfg
+ * @param id
+ * @return
+ */
+struct rspamd_config_settings_elt *rspamd_config_find_settings_name_ref(
+ struct rspamd_config *cfg,
+ const gchar *name, gsize namelen);
+
+/**
+ * Returns action object by name
+ * @param cfg
+ * @param name
+ * @return
+ */
+struct rspamd_action *rspamd_config_get_action(struct rspamd_config *cfg,
+ const gchar *name);
+
+struct rspamd_action *rspamd_config_get_action_by_type(struct rspamd_config *cfg,
+ enum rspamd_action_type type);
+
+/**
+ * Iterate over all actions
+ * @param cfg
+ * @param func
+ * @param data
+ */
+void rspamd_config_actions_foreach(struct rspamd_config *cfg,
+ void (*func)(struct rspamd_action *act, void *d),
+ void *data);
+/**
+ * Iterate over all actions with index
+ * @param cfg
+ * @param func
+ * @param data
+ */
+void rspamd_config_actions_foreach_enumerate(struct rspamd_config *cfg,
+ void (*func)(int idx, struct rspamd_action *act, void *d),
+ void *data);
+
+/**
+ * Returns number of actions defined in the config
+ * @param cfg
+ * @return
+ */
+gsize rspamd_config_actions_size(struct rspamd_config *cfg);
+
+int rspamd_config_ev_backend_get(struct rspamd_config *cfg);
+const gchar *rspamd_config_ev_backend_to_string(int ev_backend, gboolean *effective);
+
+struct rspamd_external_libs_ctx;
+
+/**
+ * Initialize rspamd libraries
+ */
+struct rspamd_external_libs_ctx *rspamd_init_libs(void);
+
+/**
+ * Reset and initialize decompressor
+ * @param ctx
+ */
+gboolean rspamd_libs_reset_decompression(struct rspamd_external_libs_ctx *ctx);
+
+/**
+ * Reset and initialize compressor
+ * @param ctx
+ */
+gboolean rspamd_libs_reset_compression(struct rspamd_external_libs_ctx *ctx);
+
+/**
+ * Destroy external libraries context
+ */
+void rspamd_deinit_libs(struct rspamd_external_libs_ctx *ctx);
+
+/**
+ * Returns TRUE if an address belongs to some local address
+ */
+gboolean rspamd_ip_is_local_cfg(struct rspamd_config *cfg,
+ const rspamd_inet_addr_t *addr);
+
+/**
+ * Configure libraries
+ */
+gboolean rspamd_config_libs(struct rspamd_external_libs_ctx *ctx,
+ struct rspamd_config *cfg);
+
+
+#define msg_err_config(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ cfg->cfg_pool->tag.tagname, cfg->checksum, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_err_config_forced(...) rspamd_default_log_function((gint) G_LOG_LEVEL_CRITICAL | (gint) RSPAMD_LOG_FORCED, \
+ cfg->cfg_pool->tag.tagname, cfg->checksum, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_config(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ cfg->cfg_pool->tag.tagname, cfg->checksum, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_config(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ cfg->cfg_pool->tag.tagname, cfg->checksum, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+extern guint rspamd_config_log_id;
+#define msg_debug_config(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_config_log_id, "config", cfg->checksum, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ifdef CFG_FILE_H */
diff --git a/src/libserver/cfg_file_private.h b/src/libserver/cfg_file_private.h
new file mode 100644
index 0000000..8c9fc65
--- /dev/null
+++ b/src/libserver/cfg_file_private.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_CFG_FILE_PRIVATE_H
+#define RSPAMD_CFG_FILE_PRIVATE_H
+
+#include "cfg_file.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Action config definition
+ */
+struct rspamd_action {
+ enum rspamd_action_type action_type;
+ int flags; /* enum rspamd_action_flags */
+ guint priority;
+ gdouble threshold;
+ gchar *name;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/cfg_rcl.cxx b/src/libserver/cfg_rcl.cxx
new file mode 100644
index 0000000..3ac7560
--- /dev/null
+++ b/src/libserver/cfg_rcl.cxx
@@ -0,0 +1,4110 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua/lua_common.h"
+#include "cfg_rcl.h"
+#include "rspamd.h"
+#include "cfg_file_private.h"
+#include "utlist.h"
+#include "cfg_file.h"
+#include "expression.h"
+#include "src/libserver/composites/composites.h"
+#include "libserver/worker_util.h"
+#include "unix-std.h"
+#include "cryptobox.h"
+#include "libutil/multipattern.h"
+#include "libmime/email_addr.h"
+#include "libmime/lang_detection.h"
+
+#include <string>
+#include <filesystem>
+#include <algorithm>// for std::transform
+#include <memory>
+#include "contrib/ankerl/unordered_dense.h"
+#include "fmt/core.h"
+#include "libutil/cxx/util.hxx"
+#include "libutil/cxx/file_util.hxx"
+#include "frozen/unordered_set.h"
+#include "frozen/string.h"
+
+#ifdef HAVE_SYSLOG_H
+#include <syslog.h>
+#endif
+
+#include <cmath>
+
+struct rspamd_rcl_default_handler_data {
+ struct rspamd_rcl_struct_parser pd;
+ std::string key;
+ rspamd_rcl_default_handler_t handler;
+};
+
+struct rspamd_rcl_sections_map;
+
+struct rspamd_rcl_section {
+ struct rspamd_rcl_sections_map *top{};
+ std::string name; /**< name of section */
+ std::optional<std::string> key_attr;
+ std::optional<std::string> default_key;
+ rspamd_rcl_handler_t handler{}; /**< handler of section attributes */
+ enum ucl_type type; /**< type of attribute */
+ bool required{}; /**< whether this param is required */
+ bool strict_type{}; /**< whether we need strict type */
+ mutable bool processed{}; /**< whether this section was processed */
+ ankerl::unordered_dense::map<std::string, std::shared_ptr<struct rspamd_rcl_section>> subsections;
+ ankerl::unordered_dense::map<std::string, struct rspamd_rcl_default_handler_data> default_parser; /**< generic parsing fields */
+ rspamd_rcl_section_fin_t fin{}; /** called at the end of section parsing */
+ gpointer fin_ud{};
+ ucl_object_t *doc_ref{}; /**< reference to the section's documentation */
+
+ virtual ~rspamd_rcl_section()
+ {
+ if (doc_ref) {
+ ucl_object_unref(doc_ref);
+ }
+ }
+};
+
+struct rspamd_worker_param_parser {
+ rspamd_rcl_default_handler_t handler; /**< handler function */
+ struct rspamd_rcl_struct_parser parser; /**< parser attributes */
+};
+
+struct rspamd_worker_cfg_parser {
+ struct pair_hash {
+ using is_avalanching = void;
+ template<class T1, class T2>
+ std::size_t operator()(const std::pair<T1, T2> &pair) const
+ {
+ return ankerl::unordered_dense::hash<T1>()(pair.first) ^ ankerl::unordered_dense::hash<T2>()(pair.second);
+ }
+ };
+ ankerl::unordered_dense::map<std::pair<std::string, gpointer>,
+ rspamd_worker_param_parser, pair_hash>
+ parsers; /**< parsers hash */
+ gint type; /**< workers quark */
+ gboolean (*def_obj_parser)(ucl_object_t *obj, gpointer ud); /**< default object parser */
+ gpointer def_ud;
+};
+
+struct rspamd_rcl_sections_map {
+ ankerl::unordered_dense::map<std::string, std::shared_ptr<struct rspamd_rcl_section>> sections;
+ std::vector<std::shared_ptr<struct rspamd_rcl_section>> sections_order;
+ ankerl::unordered_dense::map<int, struct rspamd_worker_cfg_parser> workers_parser;
+ ankerl::unordered_dense::set<std::string> lua_modules_seen;
+};
+
+static bool rspamd_rcl_process_section(struct rspamd_config *cfg,
+ const struct rspamd_rcl_section &sec,
+ gpointer ptr, const ucl_object_t *obj, rspamd_mempool_t *pool,
+ GError **err);
+static bool
+rspamd_rcl_section_parse_defaults(struct rspamd_config *cfg,
+ const struct rspamd_rcl_section &section,
+ rspamd_mempool_t *pool, const ucl_object_t *obj, gpointer ptr,
+ GError **err);
+
+/*
+ * Common section handlers
+ */
+static gboolean
+rspamd_rcl_logging_handler(rspamd_mempool_t *pool, const ucl_object_t *obj,
+ const gchar *key, gpointer ud, struct rspamd_rcl_section *section,
+ GError **err)
+{
+ const ucl_object_t *val;
+ const gchar *facility = nullptr, *log_type = nullptr, *log_level = nullptr;
+ auto *cfg = (struct rspamd_config *) ud;
+
+ val = ucl_object_lookup(obj, "type");
+ if (val != nullptr && ucl_object_tostring_safe(val, &log_type)) {
+ if (g_ascii_strcasecmp(log_type, "file") == 0) {
+ /* Need to get filename */
+ val = ucl_object_lookup(obj, "filename");
+ if (val == nullptr || val->type != UCL_STRING) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ ENOENT,
+ "filename attribute must be specified for file logging type");
+ return FALSE;
+ }
+ cfg->log_type = RSPAMD_LOG_FILE;
+ cfg->log_file = rspamd_mempool_strdup(cfg->cfg_pool,
+ ucl_object_tostring(val));
+ }
+ else if (g_ascii_strcasecmp(log_type, "syslog") == 0) {
+ /* Need to get facility */
+#ifdef HAVE_SYSLOG_H
+ cfg->log_facility = LOG_DAEMON;
+ cfg->log_type = RSPAMD_LOG_SYSLOG;
+ val = ucl_object_lookup(obj, "facility");
+ if (val != nullptr && ucl_object_tostring_safe(val, &facility)) {
+ if (g_ascii_strcasecmp(facility, "LOG_AUTH") == 0 ||
+ g_ascii_strcasecmp(facility, "auth") == 0) {
+ cfg->log_facility = LOG_AUTH;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_CRON") == 0 ||
+ g_ascii_strcasecmp(facility, "cron") == 0) {
+ cfg->log_facility = LOG_CRON;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_DAEMON") == 0 ||
+ g_ascii_strcasecmp(facility, "daemon") == 0) {
+ cfg->log_facility = LOG_DAEMON;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_MAIL") == 0 ||
+ g_ascii_strcasecmp(facility, "mail") == 0) {
+ cfg->log_facility = LOG_MAIL;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_USER") == 0 ||
+ g_ascii_strcasecmp(facility, "user") == 0) {
+ cfg->log_facility = LOG_USER;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_LOCAL0") == 0 ||
+ g_ascii_strcasecmp(facility, "local0") == 0) {
+ cfg->log_facility = LOG_LOCAL0;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_LOCAL1") == 0 ||
+ g_ascii_strcasecmp(facility, "local1") == 0) {
+ cfg->log_facility = LOG_LOCAL1;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_LOCAL2") == 0 ||
+ g_ascii_strcasecmp(facility, "local2") == 0) {
+ cfg->log_facility = LOG_LOCAL2;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_LOCAL3") == 0 ||
+ g_ascii_strcasecmp(facility, "local3") == 0) {
+ cfg->log_facility = LOG_LOCAL3;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_LOCAL4") == 0 ||
+ g_ascii_strcasecmp(facility, "local4") == 0) {
+ cfg->log_facility = LOG_LOCAL4;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_LOCAL5") == 0 ||
+ g_ascii_strcasecmp(facility, "local5") == 0) {
+ cfg->log_facility = LOG_LOCAL5;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_LOCAL6") == 0 ||
+ g_ascii_strcasecmp(facility, "local6") == 0) {
+ cfg->log_facility = LOG_LOCAL6;
+ }
+ else if (g_ascii_strcasecmp(facility, "LOG_LOCAL7") == 0 ||
+ g_ascii_strcasecmp(facility, "local7") == 0) {
+ cfg->log_facility = LOG_LOCAL7;
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "invalid log facility: %s",
+ facility);
+ return FALSE;
+ }
+ }
+#endif
+ }
+ else if (g_ascii_strcasecmp(log_type,
+ "stderr") == 0 ||
+ g_ascii_strcasecmp(log_type, "console") == 0) {
+ cfg->log_type = RSPAMD_LOG_CONSOLE;
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "invalid log type: %s",
+ log_type);
+ return FALSE;
+ }
+ }
+ else {
+ /* No type specified */
+ msg_warn_config(
+ "logging type is not specified correctly, log output to the console");
+ }
+
+ /* Handle log level */
+ val = ucl_object_lookup(obj, "level");
+ if (val != nullptr && ucl_object_tostring_safe(val, &log_level)) {
+ if (g_ascii_strcasecmp(log_level, "error") == 0) {
+ cfg->log_level = G_LOG_LEVEL_ERROR | G_LOG_LEVEL_CRITICAL;
+ }
+ else if (g_ascii_strcasecmp(log_level, "warning") == 0) {
+ cfg->log_level = G_LOG_LEVEL_WARNING;
+ }
+ else if (g_ascii_strcasecmp(log_level, "info") == 0) {
+ cfg->log_level = G_LOG_LEVEL_INFO | G_LOG_LEVEL_MESSAGE;
+ }
+ else if (g_ascii_strcasecmp(log_level, "message") == 0 ||
+ g_ascii_strcasecmp(log_level, "notice") == 0) {
+ cfg->log_level = G_LOG_LEVEL_MESSAGE;
+ }
+ else if (g_ascii_strcasecmp(log_level, "silent") == 0) {
+ cfg->log_level = G_LOG_LEVEL_MESSAGE | G_LOG_LEVEL_INFO;
+ cfg->log_silent_workers = TRUE;
+ }
+ else if (g_ascii_strcasecmp(log_level, "debug") == 0) {
+ cfg->log_level = G_LOG_LEVEL_DEBUG;
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "invalid log level: %s",
+ log_level);
+ return FALSE;
+ }
+ }
+
+ /* Handle flags */
+ val = ucl_object_lookup_any(obj, "color", "log_color", nullptr);
+ if (val && ucl_object_toboolean(val)) {
+ cfg->log_flags |= RSPAMD_LOG_FLAG_COLOR;
+ }
+
+ val = ucl_object_lookup_any(obj, "severity", "log_severity", nullptr);
+ if (val && ucl_object_toboolean(val)) {
+ cfg->log_flags |= RSPAMD_LOG_FLAG_SEVERITY;
+ }
+
+ val = ucl_object_lookup_any(obj, "systemd", "log_systemd", nullptr);
+ if (val && ucl_object_toboolean(val)) {
+ cfg->log_flags |= RSPAMD_LOG_FLAG_SYSTEMD;
+ }
+
+ val = ucl_object_lookup_any(obj, "json", "log_json", nullptr);
+ if (val && ucl_object_toboolean(val)) {
+ cfg->log_flags |= RSPAMD_LOG_FLAG_JSON;
+ }
+
+ val = ucl_object_lookup(obj, "log_re_cache");
+ if (val && ucl_object_toboolean(val)) {
+ cfg->log_flags |= RSPAMD_LOG_FLAG_RE_CACHE;
+ }
+
+ val = ucl_object_lookup_any(obj, "usec", "log_usec", nullptr);
+ if (val && ucl_object_toboolean(val)) {
+ cfg->log_flags |= RSPAMD_LOG_FLAG_USEC;
+ }
+
+ return rspamd_rcl_section_parse_defaults(cfg, *section, cfg->cfg_pool, obj,
+ (void *) cfg, err);
+}
+
+static gboolean
+rspamd_rcl_options_handler(rspamd_mempool_t *pool, const ucl_object_t *obj,
+ const gchar *key, gpointer ud,
+ struct rspamd_rcl_section *section, GError **err)
+{
+ const ucl_object_t *dns, *upstream, *neighbours;
+ auto *cfg = (struct rspamd_config *) ud;
+
+ auto maybe_subsection = rspamd::find_map(section->subsections, "dns");
+
+ dns = ucl_object_lookup(obj, "dns");
+ if (maybe_subsection && dns != nullptr) {
+ if (!rspamd_rcl_section_parse_defaults(cfg,
+ *maybe_subsection.value().get(), cfg->cfg_pool, dns,
+ cfg, err)) {
+ return FALSE;
+ }
+ }
+
+ maybe_subsection = rspamd::find_map(section->subsections, "upstream");
+
+ upstream = ucl_object_lookup_any(obj, "upstream", "upstreams", nullptr);
+ if (maybe_subsection && upstream != nullptr) {
+ if (!rspamd_rcl_section_parse_defaults(cfg,
+ *maybe_subsection.value().get(), cfg->cfg_pool,
+ upstream, cfg, err)) {
+ return FALSE;
+ }
+ }
+
+ maybe_subsection = rspamd::find_map(section->subsections, "neighbours");
+
+ neighbours = ucl_object_lookup(obj, "neighbours");
+ if (maybe_subsection && neighbours != nullptr) {
+ const ucl_object_t *cur;
+
+ LL_FOREACH(neighbours, cur)
+ {
+ if (!rspamd_rcl_process_section(cfg, *maybe_subsection.value().get(), cfg, cur,
+ pool, err)) {
+ return FALSE;
+ }
+ }
+ }
+
+ const auto *gtube_patterns = ucl_object_lookup(obj, "gtube_patterns");
+ if (gtube_patterns != nullptr && ucl_object_type(gtube_patterns) == UCL_STRING) {
+ auto gtube_st = std::string{ucl_object_tostring(gtube_patterns)};
+ std::transform(gtube_st.begin(), gtube_st.end(), gtube_st.begin(), [](const auto c) -> int {
+ if (c <= 'Z' && c >= 'A')
+ return c - ('Z' - 'z');
+ return c;
+ });
+
+
+ if (gtube_st == "all") {
+ cfg->gtube_patterns_policy = RSPAMD_GTUBE_ALL;
+ }
+ else if (gtube_st == "reject") {
+ cfg->gtube_patterns_policy = RSPAMD_GTUBE_REJECT;
+ }
+ else if (gtube_st == "disabled" || gtube_st == "disable") {
+ cfg->gtube_patterns_policy = RSPAMD_GTUBE_DISABLED;
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "invalid GTUBE patterns policy: %s",
+ gtube_st.c_str());
+ return FALSE;
+ }
+ }
+ else if (auto *enable_test_patterns = ucl_object_lookup(obj, "enable_test_patterns"); enable_test_patterns != nullptr) {
+ /* Legacy setting */
+ if (!!ucl_object_toboolean(enable_test_patterns)) {
+ cfg->gtube_patterns_policy = RSPAMD_GTUBE_ALL;
+ }
+ }
+
+ if (rspamd_rcl_section_parse_defaults(cfg,
+ *section, cfg->cfg_pool, obj,
+ cfg, err)) {
+ /* We need to init this early */
+ rspamd_multipattern_library_init(cfg->hs_cache_dir);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+struct rspamd_rcl_symbol_data {
+ struct rspamd_symbols_group *gr;
+ struct rspamd_config *cfg;
+};
+
+static gboolean
+rspamd_rcl_group_handler(rspamd_mempool_t *pool, const ucl_object_t *obj,
+ const gchar *key, gpointer ud,
+ struct rspamd_rcl_section *section, GError **err)
+{
+ auto *cfg = static_cast<rspamd_config *>(ud);
+
+ g_assert(key != nullptr);
+
+ auto *gr = static_cast<rspamd_symbols_group *>(g_hash_table_lookup(cfg->groups, key));
+
+ if (gr == nullptr) {
+ gr = rspamd_config_new_group(cfg, key);
+ }
+
+ if (!rspamd_rcl_section_parse_defaults(cfg, *section, pool, obj,
+ gr, err)) {
+ return FALSE;
+ }
+
+ if (const auto *elt = ucl_object_lookup(obj, "one_shot"); elt != nullptr) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "one_shot attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+ if (ucl_object_toboolean(elt)) {
+ gr->flags |= RSPAMD_SYMBOL_GROUP_ONE_SHOT;
+ }
+ }
+
+ if (const auto *elt = ucl_object_lookup(obj, "disabled"); elt != nullptr) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "disabled attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+ if (ucl_object_toboolean(elt)) {
+ gr->flags |= RSPAMD_SYMBOL_GROUP_DISABLED;
+ }
+ }
+
+ if (const auto *elt = ucl_object_lookup(obj, "enabled"); elt != nullptr) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "enabled attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+ if (!ucl_object_toboolean(elt)) {
+ gr->flags |= RSPAMD_SYMBOL_GROUP_DISABLED;
+ }
+ }
+
+ if (const auto *elt = ucl_object_lookup(obj, "public"); elt != nullptr) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "public attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+ if (ucl_object_toboolean(elt)) {
+ gr->flags |= RSPAMD_SYMBOL_GROUP_PUBLIC;
+ }
+ }
+
+ if (const auto *elt = ucl_object_lookup(obj, "private"); elt != nullptr) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "private attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+ if (!ucl_object_toboolean(elt)) {
+ gr->flags |= RSPAMD_SYMBOL_GROUP_PUBLIC;
+ }
+ }
+
+
+ if (const auto *elt = ucl_object_lookup(obj, "description"); elt != nullptr) {
+ gr->description = rspamd_mempool_strdup(cfg->cfg_pool,
+ ucl_object_tostring(elt));
+ }
+
+ struct rspamd_rcl_symbol_data sd = {
+ .gr = gr,
+ .cfg = cfg,
+ };
+
+ /* Handle symbols */
+ if (const auto *val = ucl_object_lookup(obj, "symbols"); val != nullptr && ucl_object_type(val) == UCL_OBJECT) {
+ auto subsection = rspamd::find_map(section->subsections, "symbols");
+
+ g_assert(subsection.has_value());
+ if (!rspamd_rcl_process_section(cfg, *subsection.value().get(), &sd, val,
+ pool, err)) {
+
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_rcl_symbol_handler(rspamd_mempool_t *pool, const ucl_object_t *obj,
+ const gchar *key, gpointer ud,
+ struct rspamd_rcl_section *section, GError **err)
+{
+ auto *sd = static_cast<rspamd_rcl_symbol_data *>(ud);
+ struct rspamd_config *cfg;
+ const ucl_object_t *elt;
+ const gchar *description = nullptr;
+ gdouble score = NAN;
+ guint priority = 1, flags = 0;
+ gint nshots = 0;
+
+ g_assert(key != nullptr);
+ cfg = sd->cfg;
+
+ if ((elt = ucl_object_lookup(obj, "one_shot")) != nullptr) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "one_shot attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+ if (ucl_object_toboolean(elt)) {
+ nshots = 1;
+ }
+ }
+
+ if ((elt = ucl_object_lookup(obj, "any_shot")) != nullptr) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "any_shot attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+ if (ucl_object_toboolean(elt)) {
+ nshots = -1;
+ }
+ }
+
+ if ((elt = ucl_object_lookup(obj, "one_param")) != nullptr) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "one_param attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+
+ if (ucl_object_toboolean(elt)) {
+ flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM;
+ }
+ }
+
+ if ((elt = ucl_object_lookup(obj, "ignore")) != nullptr) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "ignore attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+
+ if (ucl_object_toboolean(elt)) {
+ flags |= RSPAMD_SYMBOL_FLAG_IGNORE_METRIC;
+ }
+ }
+
+ if ((elt = ucl_object_lookup(obj, "enabled")) != nullptr) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "enabled attribute is not boolean for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+
+ if (!ucl_object_toboolean(elt)) {
+ flags |= RSPAMD_SYMBOL_FLAG_DISABLED;
+ }
+ }
+
+ if ((elt = ucl_object_lookup(obj, "nshots")) != nullptr) {
+ if (ucl_object_type(elt) != UCL_FLOAT && ucl_object_type(elt) != UCL_INT) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "nshots attribute is not numeric for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+
+ nshots = ucl_object_toint(elt);
+ }
+
+ elt = ucl_object_lookup_any(obj, "score", "weight", nullptr);
+ if (elt) {
+ if (ucl_object_type(elt) != UCL_FLOAT && ucl_object_type(elt) != UCL_INT) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "score attribute is not numeric for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+
+ score = ucl_object_todouble(elt);
+ }
+
+ elt = ucl_object_lookup(obj, "priority");
+ if (elt) {
+ if (ucl_object_type(elt) != UCL_FLOAT && ucl_object_type(elt) != UCL_INT) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "priority attribute is not numeric for symbol: '%s'",
+ key);
+
+ return FALSE;
+ }
+
+ priority = ucl_object_toint(elt);
+ }
+ else {
+ priority = ucl_object_get_priority(obj) + 1;
+ }
+
+ elt = ucl_object_lookup(obj, "description");
+ if (elt) {
+ description = ucl_object_tostring(elt);
+ }
+
+ if (sd->gr) {
+ rspamd_config_add_symbol(cfg, key, score,
+ description, sd->gr->name, flags, priority, nshots);
+ }
+ else {
+ rspamd_config_add_symbol(cfg, key, score,
+ description, nullptr, flags, priority, nshots);
+ }
+
+ elt = ucl_object_lookup(obj, "groups");
+
+ if (elt) {
+ ucl_object_iter_t gr_it;
+ const ucl_object_t *cur_gr;
+
+ gr_it = ucl_object_iterate_new(elt);
+
+ while ((cur_gr = ucl_object_iterate_safe(gr_it, true)) != nullptr) {
+ rspamd_config_add_symbol_group(cfg, key,
+ ucl_object_tostring(cur_gr));
+ }
+
+ ucl_object_iterate_free(gr_it);
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_rcl_actions_handler(rspamd_mempool_t *pool, const ucl_object_t *obj,
+ const gchar *key, gpointer ud,
+ struct rspamd_rcl_section *section, GError **err)
+{
+ auto *cfg = static_cast<rspamd_config *>(ud);
+ const ucl_object_t *cur;
+ ucl_object_iter_t it;
+
+ it = ucl_object_iterate_new(obj);
+
+ while ((cur = ucl_object_iterate_safe(it, true)) != nullptr) {
+ gint type = ucl_object_type(cur);
+
+ if (type == UCL_NULL) {
+ rspamd_config_maybe_disable_action(cfg, ucl_object_key(cur),
+ ucl_object_get_priority(cur));
+ }
+ else if (type == UCL_OBJECT || type == UCL_FLOAT || type == UCL_INT) {
+ /* Exceptions */
+ auto default_elt = false;
+
+ for (const auto &[name, def_elt]: section->default_parser) {
+ if (def_elt.key == ucl_object_key(cur)) {
+ default_elt = true;
+ break;
+ }
+ }
+
+ if (default_elt) {
+ continue;
+ }
+
+ /* Something non-default */
+ if (!rspamd_config_set_action_score(cfg,
+ ucl_object_key(cur),
+ cur)) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "invalid action definition for: '%s'",
+ ucl_object_key(cur));
+ ucl_object_iterate_free(it);
+
+ return FALSE;
+ }
+ }
+ }
+
+ ucl_object_iterate_free(it);
+
+ return rspamd_rcl_section_parse_defaults(cfg, *section, pool, obj, cfg, err);
+}
+constexpr const auto known_worker_attributes = frozen::make_unordered_set<frozen::string>({
+ "bind_socket",
+ "listen",
+ "bind",
+ "count",
+ "max_files",
+ "max_core",
+ "enabled",
+});
+static gboolean
+rspamd_rcl_worker_handler(rspamd_mempool_t *pool, const ucl_object_t *obj,
+ const gchar *key, gpointer ud,
+ struct rspamd_rcl_section *section, GError **err)
+{
+ auto *cfg = static_cast<rspamd_config *>(ud);
+
+ g_assert(key != nullptr);
+ const auto *worker_type = key;
+
+ auto qtype = g_quark_try_string(worker_type);
+ if (qtype == 0) {
+ msg_err_config("unknown worker type: %s", worker_type);
+ return FALSE;
+ }
+
+ auto *wrk = rspamd_config_new_worker(cfg, nullptr);
+ wrk->options = ucl_object_copy(obj);
+ wrk->worker = rspamd_get_worker_by_type(cfg, qtype);
+
+ if (wrk->worker == nullptr) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "unknown worker type: %s",
+ worker_type);
+ return FALSE;
+ }
+
+ wrk->type = qtype;
+
+ if (wrk->worker->worker_init_func) {
+ wrk->ctx = wrk->worker->worker_init_func(cfg);
+ }
+
+ const auto *val = ucl_object_lookup_any(obj, "bind_socket", "listen", "bind", nullptr);
+ /* This name is more logical */
+ if (val != nullptr) {
+ auto it = ucl_object_iterate_new(val);
+ const ucl_object_t *cur;
+ const char *worker_bind = nullptr;
+
+ while ((cur = ucl_object_iterate_safe(it, true)) != nullptr) {
+ if (!ucl_object_tostring_safe(cur, &worker_bind)) {
+ continue;
+ }
+ if (!rspamd_parse_bind_line(cfg, wrk, worker_bind)) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot parse bind line: %s",
+ worker_bind);
+ ucl_object_iterate_free(it);
+ return FALSE;
+ }
+ }
+
+ ucl_object_iterate_free(it);
+ }
+
+ if (!rspamd_rcl_section_parse_defaults(cfg, *section, cfg->cfg_pool, obj,
+ wrk, err)) {
+ return FALSE;
+ }
+
+ /* Parse other attributes */
+ auto maybe_wparser = rspamd::find_map(section->top->workers_parser, wrk->type);
+
+ if (maybe_wparser && obj->type == UCL_OBJECT) {
+ auto &wparser = maybe_wparser.value().get();
+ auto it = ucl_object_iterate_new(obj);
+ const ucl_object_t *cur;
+
+ while ((cur = ucl_object_iterate_full(it, UCL_ITERATE_EXPLICIT)) != nullptr) {
+ auto srch = std::make_pair(ucl_object_key(cur), (gpointer) wrk->ctx);
+ auto maybe_specific = rspamd::find_map(wparser.parsers, srch);
+
+ if (maybe_specific) {
+ auto &whandler = maybe_specific.value().get();
+ const ucl_object_t *cur_obj;
+
+ LL_FOREACH(cur, cur_obj)
+ {
+ if (!whandler.handler(cfg->cfg_pool,
+ cur_obj,
+ (void *) &whandler.parser,
+ section,
+ err)) {
+
+ ucl_object_iterate_free(it);
+ return FALSE;
+ }
+
+ if (!(whandler.parser.flags & RSPAMD_CL_FLAG_MULTIPLE)) {
+ break;
+ }
+ }
+ }
+ else if (!(wrk->worker->flags & RSPAMD_WORKER_NO_STRICT_CONFIG) &&
+ known_worker_attributes.find(std::string_view{ucl_object_key(cur)}) == known_worker_attributes.end()) {
+ msg_warn_config("unknown worker attribute: %s; worker type: %s", ucl_object_key(cur), worker_type);
+ }
+ }
+
+ ucl_object_iterate_free(it);
+
+ if (wparser.def_obj_parser != nullptr) {
+ auto *robj = ucl_object_ref(obj);
+
+ if (!wparser.def_obj_parser(robj, wparser.def_ud)) {
+ ucl_object_unref(robj);
+
+ return FALSE;
+ }
+
+ ucl_object_unref(robj);
+ }
+ }
+
+ cfg->workers = g_list_prepend(cfg->workers, wrk);
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_rcl_lua_handler(rspamd_mempool_t *pool, const ucl_object_t *obj,
+ const gchar *key, gpointer ud,
+ struct rspamd_rcl_section *section, GError **err)
+{
+ namespace fs = std::filesystem;
+ auto *cfg = static_cast<rspamd_config *>(ud);
+ auto lua_src = fs::path{ucl_object_tostring(obj)};
+ auto *L = RSPAMD_LUA_CFG_STATE(cfg);
+ std::error_code ec1;
+
+ auto lua_dir = fs::weakly_canonical(lua_src.parent_path(), ec1);
+ auto lua_file = lua_src.filename();
+
+ if (!ec1 && !lua_dir.empty() && !lua_file.empty()) {
+ auto cur_dir = fs::current_path(ec1);
+ if (!ec1 && !cur_dir.empty() && ::chdir(lua_dir.c_str()) != -1) {
+ /* Push traceback function */
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ auto err_idx = lua_gettop(L);
+
+ /* Load file */
+ if (luaL_loadfile(L, lua_file.c_str()) != 0) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot load lua file %s: %s",
+ lua_src.c_str(),
+ lua_tostring(L, -1));
+ if (::chdir(cur_dir.c_str()) == -1) {
+ msg_err_config("cannot chdir to %s: %s", cur_dir.c_str(),
+ strerror(errno));
+ }
+
+ return FALSE;
+ }
+
+ /* Now do it */
+ if (lua_pcall(L, 0, 0, err_idx) != 0) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot init lua file %s: %s",
+ lua_src.c_str(),
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ if (::chdir(cur_dir.c_str()) == -1) {
+ msg_err_config("cannot chdir to %s: %s", cur_dir.c_str(),
+ strerror(errno));
+ }
+
+ return FALSE;
+ }
+
+ lua_pop(L, 1);
+ }
+ else {
+ g_set_error(err, CFG_RCL_ERROR, ENOENT, "cannot chdir to %s: %s",
+ lua_dir.c_str(), strerror(errno));
+ if (::chdir(cur_dir.c_str()) == -1) {
+ msg_err_config("cannot chdir back to %s: %s", cur_dir.c_str(), strerror(errno));
+ }
+
+ return FALSE;
+ }
+ if (::chdir(cur_dir.c_str()) == -1) {
+ msg_err_config("cannot chdir back to %s: %s", cur_dir.c_str(), strerror(errno));
+ }
+ }
+ else {
+
+ g_set_error(err, CFG_RCL_ERROR, ENOENT, "cannot find to %s: %s",
+ lua_src.c_str(), strerror(errno));
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static int
+rspamd_lua_mod_sort_fn(gconstpointer a, gconstpointer b)
+{
+ auto *m1 = *(const script_module **) a;
+ auto *m2 = *(const script_module **) b;
+
+ return strcmp(m1->name, m2->name);
+}
+
+gboolean
+rspamd_rcl_add_lua_plugins_path(struct rspamd_rcl_sections_map *sections,
+ struct rspamd_config *cfg,
+ const gchar *path,
+ gboolean main_path,
+ GError **err)
+{
+ namespace fs = std::filesystem;
+ auto dir = fs::path{path};
+ std::error_code ec;
+
+ auto add_single_file = [&](const fs::path &fpath) -> bool {
+ auto fname = fpath.filename();
+ auto modname = fname.string();
+
+ if (fname.has_extension()) {
+ modname = modname.substr(0, modname.size() - fname.extension().native().size());
+ }
+ auto *cur_mod = rspamd_mempool_alloc_type(cfg->cfg_pool,
+ struct script_module);
+ cur_mod->path = rspamd_mempool_strdup(cfg->cfg_pool, fpath.c_str());
+ cur_mod->name = rspamd_mempool_strdup(cfg->cfg_pool, modname.c_str());
+
+ if (sections->lua_modules_seen.contains(modname)) {
+ msg_info_config("already seen module %s, skip %s",
+ cur_mod->name, cur_mod->path);
+ return false;
+ }
+
+ g_ptr_array_add(cfg->script_modules, cur_mod);
+ sections->lua_modules_seen.insert(fname.string());
+
+ return true;
+ };
+
+ if (fs::is_regular_file(dir, ec) && dir.has_extension() && dir.extension() == ".lua") {
+ add_single_file(dir);
+ }
+ else if (!fs::is_directory(dir, ec)) {
+ if (!fs::exists(dir) && !main_path) {
+ msg_debug_config("optional plugins path %s is absent, skip it", path);
+
+ return TRUE;
+ }
+
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ errno,
+ "invalid lua path spec %s, %s",
+ path,
+ ec.message().c_str());
+ return FALSE;
+ }
+ else {
+ /* Handle directory */
+ for (const auto &p: fs::recursive_directory_iterator(dir, ec)) {
+ auto fpath = p.path().string();
+ if (p.is_regular_file() && fpath.ends_with(".lua")) {
+ add_single_file(p.path());
+ }
+ }
+ }
+
+ g_ptr_array_sort(cfg->script_modules, rspamd_lua_mod_sort_fn);
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_rcl_modules_handler(rspamd_mempool_t *pool, const ucl_object_t *obj,
+ const gchar *key, gpointer ud,
+ struct rspamd_rcl_section *section, GError **err)
+{
+ auto *cfg = static_cast<rspamd_config *>(ud);
+ const char *data;
+
+ if (obj->type == UCL_OBJECT) {
+ const auto *val = ucl_object_lookup(obj, "path");
+
+ if (val) {
+ const auto *cur = val;
+ LL_FOREACH(val, cur)
+ {
+ if (ucl_object_tostring_safe(cur, &data)) {
+ if (!rspamd_rcl_add_lua_plugins_path(section->top,
+ cfg,
+ data,
+ TRUE,
+ err)) {
+ return FALSE;
+ }
+ }
+ }
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "path attribute is missing");
+
+ return FALSE;
+ }
+
+ val = ucl_object_lookup(obj, "fallback_path");
+
+ if (val) {
+ const auto *cur = val;
+ LL_FOREACH(val, cur)
+ {
+ if (ucl_object_tostring_safe(cur, &data)) {
+ if (!rspamd_rcl_add_lua_plugins_path(section->top,
+ cfg,
+ data,
+ FALSE,
+ err)) {
+
+ return FALSE;
+ }
+ }
+ }
+ }
+
+ val = ucl_object_lookup(obj, "try_path");
+
+ if (val) {
+ const auto *cur = val;
+ LL_FOREACH(val, cur)
+ {
+ if (ucl_object_tostring_safe(cur, &data)) {
+ if (!rspamd_rcl_add_lua_plugins_path(section->top,
+ cfg,
+ data,
+ FALSE,
+ err)) {
+
+ return FALSE;
+ }
+ }
+ }
+ }
+ }
+ else if (ucl_object_tostring_safe(obj, &data)) {
+ if (!rspamd_rcl_add_lua_plugins_path(section->top, cfg, data, TRUE, err)) {
+ return FALSE;
+ }
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "module parameter has wrong type (must be an object or a string)");
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+struct statfile_parser_data {
+ struct rspamd_config *cfg;
+ struct rspamd_classifier_config *ccf;
+};
+
+static gboolean
+rspamd_rcl_statfile_handler(rspamd_mempool_t *pool, const ucl_object_t *obj,
+ const gchar *key, gpointer ud,
+ struct rspamd_rcl_section *section, GError **err)
+{
+ auto *stud = (struct statfile_parser_data *) ud;
+ GList *labels;
+
+ g_assert(key != nullptr);
+
+ auto *cfg = stud->cfg;
+ auto *ccf = stud->ccf;
+
+ auto *st = rspamd_config_new_statfile(cfg, nullptr);
+ st->symbol = rspamd_mempool_strdup(cfg->cfg_pool, key);
+
+ if (rspamd_rcl_section_parse_defaults(cfg, *section, pool, obj, st, err)) {
+ ccf->statfiles = rspamd_mempool_glist_prepend(pool, ccf->statfiles, st);
+
+ if (st->label != nullptr) {
+ labels = (GList *) g_hash_table_lookup(ccf->labels, st->label);
+ if (labels != nullptr) {
+ /* Must use append to preserve the head stored in the hash table */
+ labels = g_list_append(labels, st);
+ }
+ else {
+ g_hash_table_insert(ccf->labels, st->label,
+ g_list_prepend(nullptr, st));
+ }
+ }
+
+ if (st->symbol != nullptr) {
+ g_hash_table_insert(cfg->classifiers_symbols, st->symbol, st);
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "statfile must have a symbol defined");
+ return FALSE;
+ }
+
+ st->opts = (ucl_object_t *) obj;
+ st->clcf = ccf;
+
+ const auto *val = ucl_object_lookup(obj, "spam");
+ if (val == nullptr) {
+ msg_info_config(
+ "statfile %s has no explicit 'spam' setting, trying to guess by symbol",
+ st->symbol);
+ if (rspamd_substring_search_caseless(st->symbol,
+ strlen(st->symbol), "spam", 4) != -1) {
+ st->is_spam = TRUE;
+ }
+ else if (rspamd_substring_search_caseless(st->symbol,
+ strlen(st->symbol), "ham", 3) != -1) {
+ st->is_spam = FALSE;
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot guess spam setting from %s",
+ st->symbol);
+ return FALSE;
+ }
+ msg_info_config("guessed that statfile with symbol %s is %s",
+ st->symbol,
+ st->is_spam ? "spam" : "ham");
+ }
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_rcl_classifier_handler(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ const gchar *key,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *cfg = static_cast<rspamd_config *>(ud);
+
+ g_assert(key != nullptr);
+ auto *ccf = rspamd_config_new_classifier(cfg, nullptr);
+ auto *tkcf = (rspamd_tokenizer_config *) nullptr;
+
+ ccf->classifier = rspamd_mempool_strdup(cfg->cfg_pool, key);
+
+ if (rspamd_rcl_section_parse_defaults(cfg, *section, cfg->cfg_pool, obj,
+ ccf, err)) {
+
+ auto stat_section = rspamd::find_map(section->subsections, "statfile");
+
+ if (ccf->classifier == nullptr) {
+ ccf->classifier = rspamd_mempool_strdup(cfg->cfg_pool, "bayes");
+ }
+
+ if (ccf->name == nullptr) {
+ ccf->name = ccf->classifier;
+ }
+
+ auto it = ucl_object_iterate_new(obj);
+ const auto *val = obj;
+ auto res = TRUE;
+
+ while ((val = ucl_object_iterate_safe(it, true)) != nullptr && res) {
+ const auto *st_key = ucl_object_key(val);
+
+ if (st_key != nullptr) {
+ if (g_ascii_strcasecmp(st_key, "statfile") == 0) {
+ const auto *cur = val;
+ LL_FOREACH(val, cur)
+ {
+ struct statfile_parser_data stud = {.cfg = cfg, .ccf = ccf};
+ res = rspamd_rcl_process_section(cfg, *stat_section.value().get(), &stud,
+ cur, cfg->cfg_pool, err);
+
+ if (!res) {
+ ucl_object_iterate_free(it);
+
+ return FALSE;
+ }
+ }
+ }
+ else if (g_ascii_strcasecmp(st_key, "tokenizer") == 0) {
+ tkcf = rspamd_mempool_alloc0_type(cfg->cfg_pool, rspamd_tokenizer_config);
+
+ if (ucl_object_type(val) == UCL_STRING) {
+ tkcf->name = ucl_object_tostring(val);
+ }
+ else if (ucl_object_type(val) == UCL_OBJECT) {
+ const auto *cur = ucl_object_lookup(val, "name");
+ if (cur != nullptr) {
+ tkcf->name = ucl_object_tostring(cur);
+ tkcf->opts = val;
+ }
+ else {
+ cur = ucl_object_lookup(val, "type");
+ if (cur != nullptr) {
+ tkcf->name = ucl_object_tostring(cur);
+ tkcf->opts = val;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ ucl_object_iterate_free(it);
+ }
+ else {
+ msg_err_config("fatal configuration error, cannot parse statfile definition");
+ }
+
+ if (tkcf == nullptr) {
+ tkcf = rspamd_mempool_alloc0_type(cfg->cfg_pool, rspamd_tokenizer_config);
+ tkcf->name = nullptr;
+ }
+
+ ccf->tokenizer = tkcf;
+
+ /* Handle lua conditions */
+ const auto *val = ucl_object_lookup_any(obj, "learn_condition", nullptr);
+
+ if (val) {
+ const auto *cur = val;
+ LL_FOREACH(val, cur)
+ {
+ if (ucl_object_type(cur) == UCL_STRING) {
+ const gchar *lua_script;
+ gsize slen;
+ gint ref_idx;
+
+ lua_script = ucl_object_tolstring(cur, &slen);
+ ref_idx = rspamd_lua_function_ref_from_str(RSPAMD_LUA_CFG_STATE(cfg),
+ lua_script, slen, "learn_condition", err);
+
+ if (ref_idx == LUA_NOREF) {
+ return FALSE;
+ }
+
+ rspamd_lua_add_ref_dtor(RSPAMD_LUA_CFG_STATE(cfg), cfg->cfg_pool, ref_idx);
+ ccf->learn_conditions = rspamd_mempool_glist_append(
+ cfg->cfg_pool,
+ ccf->learn_conditions,
+ GINT_TO_POINTER(ref_idx));
+ }
+ }
+ }
+
+ val = ucl_object_lookup_any(obj, "classify_condition", nullptr);
+
+ if (val) {
+ const auto *cur = val;
+ LL_FOREACH(val, cur)
+ {
+ if (ucl_object_type(cur) == UCL_STRING) {
+ const gchar *lua_script;
+ gsize slen;
+ gint ref_idx;
+
+ lua_script = ucl_object_tolstring(cur, &slen);
+ ref_idx = rspamd_lua_function_ref_from_str(RSPAMD_LUA_CFG_STATE(cfg),
+ lua_script, slen, "classify_condition", err);
+
+ if (ref_idx == LUA_NOREF) {
+ return FALSE;
+ }
+
+ rspamd_lua_add_ref_dtor(RSPAMD_LUA_CFG_STATE(cfg), cfg->cfg_pool, ref_idx);
+ ccf->classify_conditions = rspamd_mempool_glist_append(
+ cfg->cfg_pool,
+ ccf->classify_conditions,
+ GINT_TO_POINTER(ref_idx));
+ }
+ }
+ }
+
+ ccf->opts = (ucl_object_t *) obj;
+ cfg->classifiers = g_list_prepend(cfg->classifiers, ccf);
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_rcl_composite_handler(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ const gchar *key,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *cfg = static_cast<rspamd_config *>(ud);
+ void *composite;
+ const gchar *composite_name;
+
+ g_assert(key != nullptr);
+
+ composite_name = key;
+
+ const auto *val = ucl_object_lookup(obj, "enabled");
+ if (val != nullptr && !ucl_object_toboolean(val)) {
+ msg_info_config("composite %s is disabled", composite_name);
+ return TRUE;
+ }
+
+ if ((composite = rspamd_composites_manager_add_from_ucl(cfg->composites_manager,
+ composite_name, obj)) != nullptr) {
+ rspamd_symcache_add_symbol(cfg->cache, composite_name, 0,
+ nullptr, composite, SYMBOL_TYPE_COMPOSITE, -1);
+ }
+
+ return composite != nullptr;
+}
+
+static gboolean
+rspamd_rcl_composites_handler(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ const gchar *key,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto success = TRUE;
+
+ auto it = ucl_object_iterate_new(obj);
+ const auto *cur = obj;
+
+ while ((cur = ucl_object_iterate_safe(it, true))) {
+ success = rspamd_rcl_composite_handler(pool, cur,
+ ucl_object_key(cur), ud, section, err);
+ if (!success) {
+ break;
+ }
+ }
+
+ ucl_object_iterate_free(it);
+
+ return success;
+}
+
+static gboolean
+rspamd_rcl_neighbours_handler(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ const gchar *key,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *cfg = static_cast<rspamd_config *>(ud);
+ auto has_port = FALSE, has_proto = FALSE;
+ const gchar *p;
+
+ if (key == nullptr) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "missing name for neighbour");
+ return FALSE;
+ }
+
+ const auto *hostval = ucl_object_lookup(obj, "host");
+
+ if (hostval == nullptr || ucl_object_type(hostval) != UCL_STRING) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "missing host for neighbour: %s", ucl_object_key(obj));
+ return FALSE;
+ }
+
+ auto *neigh = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(neigh, ucl_object_copy(hostval), "host", 0, false);
+
+ if ((p = strrchr(ucl_object_tostring(hostval), ':')) != nullptr) {
+ if (g_ascii_isdigit(p[1])) {
+ has_port = TRUE;
+ }
+ }
+
+ if (strstr(ucl_object_tostring(hostval), "://") != nullptr) {
+ has_proto = TRUE;
+ }
+
+ /* Now make url */
+ auto urlstr = std::string{};
+ const auto *pathval = ucl_object_lookup(obj, "path");
+
+ if (!has_proto) {
+ urlstr += "http://";
+ }
+
+ urlstr += ucl_object_tostring(hostval);
+
+ if (!has_port) {
+ urlstr += ":11334";
+ }
+
+ if (pathval == nullptr) {
+ urlstr += "/";
+ }
+ else {
+ urlstr += ucl_object_tostring(pathval);
+ }
+
+ ucl_object_insert_key(neigh,
+ ucl_object_fromlstring(urlstr.data(), urlstr.size()),
+ "url", 0, false);
+ ucl_object_insert_key(cfg->neighbours, neigh, key, 0, true);
+
+ return TRUE;
+}
+
+
+struct rspamd_rcl_section *
+rspamd_rcl_add_section(struct rspamd_rcl_sections_map **top,
+ struct rspamd_rcl_section *parent_section,
+ const gchar *name, const gchar *key_attr, rspamd_rcl_handler_t handler,
+ enum ucl_type type, gboolean required, gboolean strict_type)
+{
+ return rspamd_rcl_add_section_doc(top, parent_section, name, key_attr, handler,
+ type, required, strict_type, nullptr, nullptr);
+}
+
+struct rspamd_rcl_section *
+rspamd_rcl_add_section_doc(struct rspamd_rcl_sections_map **top,
+ struct rspamd_rcl_section *parent_section,
+ const gchar *name, const gchar *key_attr, rspamd_rcl_handler_t handler,
+ enum ucl_type type, gboolean required, gboolean strict_type,
+ ucl_object_t *doc_target,
+ const gchar *doc_string)
+{
+ if (top == nullptr) {
+ g_error("invalid arguments to rspamd_rcl_add_section");
+ return nullptr;
+ }
+ if (*top == nullptr) {
+ *top = new rspamd_rcl_sections_map;
+ }
+
+ auto fill_section = [&](struct rspamd_rcl_section *section) {
+ section->name = name;
+ if (key_attr) {
+ section->key_attr = std::string{key_attr};
+ }
+ section->handler = handler;
+ section->type = type;
+ section->strict_type = strict_type;
+
+ if (doc_target == nullptr) {
+ if (parent_section && parent_section->doc_ref) {
+ section->doc_ref = ucl_object_ref(rspamd_rcl_add_doc_obj(parent_section->doc_ref,
+ doc_string,
+ name,
+ type,
+ nullptr,
+ 0,
+ nullptr,
+ 0));
+ }
+ else {
+ section->doc_ref = nullptr;
+ }
+ }
+ else {
+ section->doc_ref = ucl_object_ref(rspamd_rcl_add_doc_obj(doc_target,
+ doc_string,
+ name,
+ type,
+ nullptr,
+ 0,
+ nullptr,
+ 0));
+ }
+ section->top = *top;
+ };
+
+ /* Select the appropriate container and insert section inside it */
+ if (parent_section) {
+ auto it = parent_section->subsections.insert(std::make_pair(std::string{name},
+ std::make_shared<rspamd_rcl_section>()));
+ if (!it.second) {
+ g_error("invalid arguments to rspamd_rcl_add_section");
+ return nullptr;
+ }
+
+ fill_section(it.first->second.get());
+ return it.first->second.get();
+ }
+ else {
+ auto it = (*top)->sections.insert(std::make_pair(std::string{name},
+ std::make_shared<rspamd_rcl_section>()));
+ if (!it.second) {
+ g_error("invalid arguments to rspamd_rcl_add_section");
+ return nullptr;
+ }
+
+ (*top)->sections_order.push_back(it.first->second);
+ fill_section(it.first->second.get());
+ return it.first->second.get();
+ }
+}
+
+struct rspamd_rcl_default_handler_data *
+rspamd_rcl_add_default_handler(struct rspamd_rcl_section *section,
+ const gchar *name,
+ rspamd_rcl_default_handler_t handler,
+ goffset offset,
+ gint flags,
+ const gchar *doc_string)
+{
+ auto it = section->default_parser.emplace(std::make_pair(std::string{name}, rspamd_rcl_default_handler_data{}));
+
+ auto &nhandler = it.first->second;
+ nhandler.key = name;
+ nhandler.handler = handler;
+ nhandler.pd.offset = offset;
+ nhandler.pd.flags = flags;
+
+ if (section->doc_ref != nullptr) {
+ rspamd_rcl_add_doc_obj(section->doc_ref,
+ doc_string,
+ name,
+ UCL_NULL,
+ handler,
+ flags,
+ nullptr,
+ 0);
+ }
+
+ return &nhandler;
+}
+
+struct rspamd_rcl_sections_map *
+rspamd_rcl_config_init(struct rspamd_config *cfg, GHashTable *skip_sections)
+{
+ auto *top = new rspamd_rcl_sections_map;
+ /*
+ * Important notice:
+ * the order of parsing is equal to order of this initialization, therefore
+ * it is possible to init some portions of config prior to others
+ */
+
+ /**
+ * Logging section
+ */
+ if (!(skip_sections && g_hash_table_lookup(skip_sections, "logging"))) {
+ auto *sub = rspamd_rcl_add_section_doc(&top, nullptr,
+ "logging", nullptr,
+ rspamd_rcl_logging_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Configure rspamd logging");
+ /* Default handlers */
+ rspamd_rcl_add_default_handler(sub,
+ "log_buffer",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, log_buf_size),
+ RSPAMD_CL_FLAG_INT_32,
+ "Size of log buffer in bytes (for file logging)");
+ rspamd_rcl_add_default_handler(sub,
+ "log_urls",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, log_urls),
+ 0,
+ "Write each URL found in a message to the log file");
+ rspamd_rcl_add_default_handler(sub,
+ "debug_ip",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET(struct rspamd_config, debug_ip_map),
+ 0,
+ "Enable debugging log for the specified IP addresses");
+ rspamd_rcl_add_default_handler(sub,
+ "debug_modules",
+ rspamd_rcl_parse_struct_string_list,
+ G_STRUCT_OFFSET(struct rspamd_config, debug_modules),
+ RSPAMD_CL_FLAG_STRING_LIST_HASH,
+ "Enable debugging for the specified modules");
+ rspamd_rcl_add_default_handler(sub,
+ "log_format",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, log_format_str),
+ 0,
+ "Specify format string for the task logging output "
+ "(https://rspamd.com/doc/configuration/logging.html "
+ "for details)");
+ rspamd_rcl_add_default_handler(sub,
+ "encryption_key",
+ rspamd_rcl_parse_struct_pubkey,
+ G_STRUCT_OFFSET(struct rspamd_config, log_encryption_key),
+ 0,
+ "Encrypt sensitive information in logs using this pubkey");
+ rspamd_rcl_add_default_handler(sub,
+ "error_elts",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, log_error_elts),
+ RSPAMD_CL_FLAG_UINT,
+ "Size of circular buffer for last errors (10 by default)");
+ rspamd_rcl_add_default_handler(sub,
+ "error_maxlen",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, log_error_elt_maxlen),
+ RSPAMD_CL_FLAG_UINT,
+ "Size of each element in error log buffer (1000 by default)");
+ rspamd_rcl_add_default_handler(sub,
+ "task_max_elts",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, log_task_max_elts),
+ RSPAMD_CL_FLAG_UINT,
+ "Maximum number of elements in task log entry (7 by default)");
+
+ /* Documentation only options, handled in log_handler to map flags */
+ rspamd_rcl_add_doc_by_path(cfg,
+ "logging",
+ "Enable colored output (for console logging)",
+ "log_color",
+ UCL_BOOLEAN,
+ nullptr,
+ 0,
+ nullptr,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "logging",
+ "Enable severity logging output (e.g. [error] or [warning])",
+ "log_severity",
+ UCL_BOOLEAN,
+ nullptr,
+ 0,
+ nullptr,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "logging",
+ "Enable systemd compatible logging",
+ "systemd",
+ UCL_BOOLEAN,
+ nullptr,
+ 0,
+ nullptr,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "logging",
+ "Write statistics of regexp processing to log (useful for hyperscan)",
+ "log_re_cache",
+ UCL_BOOLEAN,
+ nullptr,
+ 0,
+ nullptr,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "logging",
+ "Use microseconds resolution for timestamps",
+ "log_usec",
+ UCL_BOOLEAN,
+ nullptr,
+ 0,
+ nullptr,
+ 0);
+ }
+ if (!(skip_sections && g_hash_table_lookup(skip_sections, "options"))) {
+ /**
+ * Options section
+ */
+ auto *sub = rspamd_rcl_add_section_doc(&top, nullptr,
+ "options", nullptr,
+ rspamd_rcl_options_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Global rspamd options");
+ rspamd_rcl_add_default_handler(sub,
+ "cache_file",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, cache_filename),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to the cache file");
+ rspamd_rcl_add_default_handler(sub,
+ "cache_reload",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET(struct rspamd_config, cache_reload_time),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "How often cache reload should be performed");
+
+ /* Old DNS configuration */
+ rspamd_rcl_add_default_handler(sub,
+ "dns_nameserver",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET(struct rspamd_config, nameservers),
+ 0,
+ "Legacy option for DNS servers used");
+ rspamd_rcl_add_default_handler(sub,
+ "dns_timeout",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET(struct rspamd_config, dns_timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Legacy option for DNS request timeout");
+ rspamd_rcl_add_default_handler(sub,
+ "dns_retransmits",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, dns_retransmits),
+ RSPAMD_CL_FLAG_INT_32,
+ "Legacy option for DNS retransmits count");
+ rspamd_rcl_add_default_handler(sub,
+ "dns_sockets",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, dns_io_per_server),
+ RSPAMD_CL_FLAG_INT_32,
+ "Legacy option for DNS sockets per server count");
+ rspamd_rcl_add_default_handler(sub,
+ "dns_max_requests",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, dns_max_requests),
+ RSPAMD_CL_FLAG_INT_32,
+ "Maximum DNS requests per task (default: 64)");
+ rspamd_rcl_add_default_handler(sub,
+ "control_socket",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, control_socket_path),
+ 0,
+ "Path to the control socket");
+ rspamd_rcl_add_default_handler(sub,
+ "explicit_modules",
+ rspamd_rcl_parse_struct_string_list,
+ G_STRUCT_OFFSET(struct rspamd_config, explicit_modules),
+ RSPAMD_CL_FLAG_STRING_LIST_HASH,
+ "Always load these modules even if they are not configured explicitly");
+ rspamd_rcl_add_default_handler(sub,
+ "allow_raw_input",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, allow_raw_input),
+ 0,
+ "Allow non MIME input for rspamd");
+ rspamd_rcl_add_default_handler(sub,
+ "one_shot",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, one_shot_mode),
+ 0,
+ "Add all symbols only once per message");
+ rspamd_rcl_add_default_handler(sub,
+ "check_attachements",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, check_text_attachements),
+ 0,
+ "Treat text attachments as normal text parts");
+ rspamd_rcl_add_default_handler(sub,
+ "tempdir",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, temp_dir),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Directory for temporary files");
+ rspamd_rcl_add_default_handler(sub,
+ "pidfile",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, pid_file),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to the pid file");
+ rspamd_rcl_add_default_handler(sub,
+ "filters",
+ rspamd_rcl_parse_struct_string_list,
+ G_STRUCT_OFFSET(struct rspamd_config, filters),
+ 0,
+ "List of internal filters enabled");
+ rspamd_rcl_add_default_handler(sub,
+ "map_watch_interval",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET(struct rspamd_config, map_timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Interval for checking maps");
+ rspamd_rcl_add_default_handler(sub,
+ "map_file_watch_multiplier",
+ rspamd_rcl_parse_struct_double,
+ G_STRUCT_OFFSET(struct rspamd_config, map_file_watch_multiplier),
+ 0,
+ "Multiplier for map watch interval when map is file");
+ rspamd_rcl_add_default_handler(sub,
+ "maps_cache_dir",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, maps_cache_dir),
+ 0,
+ "Directory to save maps cached data (default: $DBDIR)");
+ rspamd_rcl_add_default_handler(sub,
+ "monitoring_watch_interval",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET(struct rspamd_config, monitored_interval),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Interval for checking monitored instances");
+ rspamd_rcl_add_default_handler(sub,
+ "disable_monitoring",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, disable_monitored),
+ 0,
+ "Disable monitoring completely");
+ rspamd_rcl_add_default_handler(sub,
+ "fips_mode",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, fips_mode),
+ 0,
+ "Enable FIPS 140-2 mode in OpenSSL");
+ rspamd_rcl_add_default_handler(sub,
+ "dynamic_conf",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, dynamic_conf),
+ 0,
+ "Path to the dynamic configuration");
+ rspamd_rcl_add_default_handler(sub,
+ "rrd",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, rrd_file),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to RRD file");
+ rspamd_rcl_add_default_handler(sub,
+ "stats_file",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, stats_file),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to stats file");
+ rspamd_rcl_add_default_handler(sub,
+ "history_file",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, history_file),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to history file");
+ rspamd_rcl_add_default_handler(sub,
+ "check_all_filters",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, check_all_filters),
+ 0,
+ "Always check all filters");
+ rspamd_rcl_add_default_handler(sub,
+ "public_groups_only",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, public_groups_only),
+ 0,
+ "Output merely public groups everywhere");
+ rspamd_rcl_add_default_handler(sub,
+ "enable_css_parser",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, enable_css_parser),
+ 0,
+ "Enable CSS parser (experimental)");
+ rspamd_rcl_add_default_handler(sub,
+ "enable_experimental",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, enable_experimental),
+ 0,
+ "Enable experimental plugins");
+ rspamd_rcl_add_default_handler(sub,
+ "disable_pcre_jit",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, disable_pcre_jit),
+ 0,
+ "Disable PCRE JIT");
+ rspamd_rcl_add_default_handler(sub,
+ "min_word_len",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, min_word_len),
+ RSPAMD_CL_FLAG_UINT,
+ "Minimum length of the word to be considered in statistics/fuzzy");
+ rspamd_rcl_add_default_handler(sub,
+ "max_word_len",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_word_len),
+ RSPAMD_CL_FLAG_UINT,
+ "Maximum length of the word to be considered in statistics/fuzzy");
+ rspamd_rcl_add_default_handler(sub,
+ "max_html_len",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_word_len),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Maximum length of the html part to be parsed");
+ rspamd_rcl_add_default_handler(sub,
+ "words_decay",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, words_decay),
+ RSPAMD_CL_FLAG_UINT,
+ "Start skipping words at this amount");
+ rspamd_rcl_add_default_handler(sub,
+ "url_tld",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, tld_file),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to the TLD file for urls detector");
+ rspamd_rcl_add_default_handler(sub,
+ "tld",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, tld_file),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to the TLD file for urls detector");
+ rspamd_rcl_add_default_handler(sub,
+ "hs_cache_dir",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, hs_cache_dir),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path directory where rspamd would save hyperscan cache");
+ rspamd_rcl_add_default_handler(sub,
+ "history_rows",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, history_rows),
+ RSPAMD_CL_FLAG_UINT,
+ "Number of records in the history file");
+ rspamd_rcl_add_default_handler(sub,
+ "disable_hyperscan",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, disable_hyperscan),
+ 0,
+ "Disable hyperscan optimizations for regular expressions");
+ rspamd_rcl_add_default_handler(sub,
+ "vectorized_hyperscan",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, vectorized_hyperscan),
+ 0,
+ "Use hyperscan in vectorized mode (obsoleted, do not use)");
+ rspamd_rcl_add_default_handler(sub,
+ "cores_dir",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, cores_dir),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to the directory where rspamd core files are intended to be dumped");
+ rspamd_rcl_add_default_handler(sub,
+ "max_cores_size",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_cores_size),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Limit of joint size of all files in `cores_dir`");
+ rspamd_rcl_add_default_handler(sub,
+ "max_cores_count",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_cores_count),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Limit of files count in `cores_dir`");
+ rspamd_rcl_add_default_handler(sub,
+ "local_addrs",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET(struct rspamd_config, local_addrs),
+ 0,
+ "Use the specified addresses as local ones");
+ rspamd_rcl_add_default_handler(sub,
+ "local_networks",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET(struct rspamd_config, local_addrs),
+ 0,
+ "Use the specified addresses as local ones (alias for `local_addrs`)");
+ rspamd_rcl_add_default_handler(sub,
+ "trusted_keys",
+ rspamd_rcl_parse_struct_string_list,
+ G_STRUCT_OFFSET(struct rspamd_config, trusted_keys),
+ RSPAMD_CL_FLAG_STRING_LIST_HASH,
+ "List of trusted public keys used for signatures in base32 encoding");
+ rspamd_rcl_add_default_handler(sub,
+ "enable_shutdown_workaround",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, enable_shutdown_workaround),
+ 0,
+ "Enable workaround for legacy clients");
+ rspamd_rcl_add_default_handler(sub,
+ "ignore_received",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, ignore_received),
+ 0,
+ "Ignore data from the first received header");
+ rspamd_rcl_add_default_handler(sub,
+ "ssl_ca_path",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, ssl_ca_path),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Path to ssl CA file");
+ rspamd_rcl_add_default_handler(sub,
+ "ssl_ciphers",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, ssl_ciphers),
+ 0,
+ "List of ssl ciphers (e.g. HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4)");
+ rspamd_rcl_add_default_handler(sub,
+ "max_message",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_message),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Maximum size of the message to be scanned (50Mb by default)");
+ rspamd_rcl_add_default_handler(sub,
+ "max_pic",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_pic_size),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Maximum size of the picture to be normalized (1Mb by default)");
+ rspamd_rcl_add_default_handler(sub,
+ "images_cache",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_pic_size),
+ RSPAMD_CL_FLAG_INT_SIZE,
+ "Size of DCT data cache for images (256 elements by default)");
+ rspamd_rcl_add_default_handler(sub,
+ "zstd_input_dictionary",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, zstd_input_dictionary),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Dictionary for zstd inbound protocol compression");
+ rspamd_rcl_add_default_handler(sub,
+ "zstd_output_dictionary",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, zstd_output_dictionary),
+ RSPAMD_CL_FLAG_STRING_PATH,
+ "Dictionary for outbound zstd compression");
+ rspamd_rcl_add_default_handler(sub,
+ "compat_messages",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, compat_messages),
+ 0,
+ "Use pre 1.4 style of messages in the protocol");
+ rspamd_rcl_add_default_handler(sub,
+ "max_shots",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, default_max_shots),
+ 0,
+ "Maximum number of hits per a single symbol (default: 100)");
+ rspamd_rcl_add_default_handler(sub,
+ "sessions_cache",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, enable_sessions_cache),
+ 0,
+ "Enable sessions cache to debug dangling sessions");
+ rspamd_rcl_add_default_handler(sub,
+ "max_sessions_cache",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_sessions_cache),
+ 0,
+ "Maximum number of sessions in cache before warning (default: 100)");
+ rspamd_rcl_add_default_handler(sub,
+ "task_timeout",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET(struct rspamd_config, task_timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Maximum time for checking a message");
+ rspamd_rcl_add_default_handler(sub,
+ "soft_reject_on_timeout",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, soft_reject_on_timeout),
+ 0,
+ "Emit soft reject if task timeout takes place");
+ rspamd_rcl_add_default_handler(sub,
+ "check_timeout",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET(struct rspamd_config, task_timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Maximum time for checking a message (alias for task_timeout)");
+ rspamd_rcl_add_default_handler(sub,
+ "lua_gc_step",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, lua_gc_step),
+ RSPAMD_CL_FLAG_UINT,
+ "Lua garbage-collector step (default: 200)");
+ rspamd_rcl_add_default_handler(sub,
+ "lua_gc_pause",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, lua_gc_pause),
+ RSPAMD_CL_FLAG_UINT,
+ "Lua garbage-collector pause (default: 200)");
+ rspamd_rcl_add_default_handler(sub,
+ "full_gc_iters",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, full_gc_iters),
+ RSPAMD_CL_FLAG_UINT,
+ "Task scanned before memory gc is performed (default: 0 - disabled)");
+ rspamd_rcl_add_default_handler(sub,
+ "heartbeat_interval",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET(struct rspamd_config, heartbeat_interval),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Time between workers heartbeats");
+ rspamd_rcl_add_default_handler(sub,
+ "heartbeats_loss_max",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, heartbeats_loss_max),
+ RSPAMD_CL_FLAG_INT_32,
+ "Maximum count of heartbeats to be lost before trying to "
+ "terminate a worker (default: 0 - disabled)");
+ rspamd_rcl_add_default_handler(sub,
+ "max_lua_urls",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_lua_urls),
+ RSPAMD_CL_FLAG_INT_32,
+ "Maximum count of URLs to pass to Lua to avoid DoS (default: 1024)");
+ rspamd_rcl_add_default_handler(sub,
+ "max_urls",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_urls),
+ RSPAMD_CL_FLAG_INT_32,
+ "Maximum count of URLs to process to avoid DoS (default: 10240)");
+ rspamd_rcl_add_default_handler(sub,
+ "max_recipients",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_recipients),
+ RSPAMD_CL_FLAG_INT_32,
+ "Maximum count of recipients to process to avoid DoS (default: 1024)");
+ rspamd_rcl_add_default_handler(sub,
+ "max_blas_threads",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_blas_threads),
+ RSPAMD_CL_FLAG_INT_32,
+ "Maximum number of Blas threads for learning neural networks (default: 1)");
+ rspamd_rcl_add_default_handler(sub,
+ "max_opts_len",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, max_opts_len),
+ RSPAMD_CL_FLAG_INT_32,
+ "Maximum size of all options for a single symbol (default: 4096)");
+ rspamd_rcl_add_default_handler(sub,
+ "events_backend",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, events_backend),
+ 0,
+ "Events backend to use: kqueue, epoll, select, poll or auto (default: auto)");
+
+ rspamd_rcl_add_doc_by_path(cfg,
+ "options",
+ "Swtich mode of gtube patterns: disable, reject, all",
+ "gtube_patterns",
+ UCL_STRING,
+ nullptr,
+ 0,
+ "reject",
+ 0);
+
+ /* Neighbours configuration */
+ rspamd_rcl_add_section_doc(&top, sub, "neighbours", "name",
+ rspamd_rcl_neighbours_handler,
+ UCL_OBJECT, FALSE, TRUE,
+ cfg->doc_strings,
+ "List of members of Rspamd cluster");
+
+ /* New DNS configuration */
+ auto *ssub = rspamd_rcl_add_section_doc(&top, sub, "dns", nullptr, nullptr,
+ UCL_OBJECT, FALSE, TRUE,
+ cfg->doc_strings,
+ "Options for DNS resolver");
+ rspamd_rcl_add_default_handler(ssub,
+ "nameserver",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET(struct rspamd_config, nameservers),
+ 0,
+ "List of DNS servers");
+ rspamd_rcl_add_default_handler(ssub,
+ "server",
+ rspamd_rcl_parse_struct_ucl,
+ G_STRUCT_OFFSET(struct rspamd_config, nameservers),
+ 0,
+ "List of DNS servers");
+ rspamd_rcl_add_default_handler(ssub,
+ "timeout",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET(struct rspamd_config, dns_timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "DNS request timeout");
+ rspamd_rcl_add_default_handler(ssub,
+ "retransmits",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, dns_retransmits),
+ RSPAMD_CL_FLAG_INT_32,
+ "DNS request retransmits");
+ rspamd_rcl_add_default_handler(ssub,
+ "sockets",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, dns_io_per_server),
+ RSPAMD_CL_FLAG_INT_32,
+ "Number of sockets per DNS server");
+ rspamd_rcl_add_default_handler(ssub,
+ "connections",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, dns_io_per_server),
+ RSPAMD_CL_FLAG_INT_32,
+ "Number of sockets per DNS server");
+ rspamd_rcl_add_default_handler(ssub,
+ "enable_dnssec",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_config, enable_dnssec),
+ 0,
+ "Enable DNSSEC support in Rspamd");
+
+
+ /* New upstreams configuration */
+ ssub = rspamd_rcl_add_section_doc(&top, sub, "upstream", nullptr, nullptr,
+ UCL_OBJECT, FALSE, TRUE,
+ cfg->doc_strings,
+ "Upstreams configuration parameters");
+ rspamd_rcl_add_default_handler(ssub,
+ "max_errors",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_config, upstream_max_errors),
+ RSPAMD_CL_FLAG_UINT,
+ "Maximum number of errors during `error_time` to consider upstream down");
+ rspamd_rcl_add_default_handler(ssub,
+ "error_time",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET(struct rspamd_config, upstream_error_time),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Time frame to check errors");
+ rspamd_rcl_add_default_handler(ssub,
+ "revive_time",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET(struct rspamd_config, upstream_revive_time),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Time before attempting to recover upstream after an error");
+ rspamd_rcl_add_default_handler(ssub,
+ "lazy_resolve_time",
+ rspamd_rcl_parse_struct_time,
+ G_STRUCT_OFFSET(struct rspamd_config, upstream_lazy_resolve_time),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Time to resolve upstreams addresses in lazy mode");
+ }
+
+ if (!(skip_sections && g_hash_table_lookup(skip_sections, "actions"))) {
+ /**
+ * Symbols and actions sections
+ */
+ auto *sub = rspamd_rcl_add_section_doc(&top, nullptr,
+ "actions", nullptr,
+ rspamd_rcl_actions_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Actions configuration");
+ rspamd_rcl_add_default_handler(sub,
+ "unknown_weight",
+ rspamd_rcl_parse_struct_double,
+ G_STRUCT_OFFSET(struct rspamd_config, unknown_weight),
+ 0,
+ "Accept unknown symbols with the specified weight");
+ rspamd_rcl_add_default_handler(sub,
+ "grow_factor",
+ rspamd_rcl_parse_struct_double,
+ G_STRUCT_OFFSET(struct rspamd_config, grow_factor),
+ 0,
+ "Multiply the subsequent symbols by this number "
+ "(does not affect symbols with score less or "
+ "equal to zero)");
+ rspamd_rcl_add_default_handler(sub,
+ "subject",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_config, subject),
+ 0,
+ "Rewrite subject with this value");
+ }
+
+ if (!(skip_sections && g_hash_table_lookup(skip_sections, "group"))) {
+ auto *sub = rspamd_rcl_add_section_doc(&top, nullptr,
+ "group", "name",
+ rspamd_rcl_group_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Symbol groups configuration");
+ rspamd_rcl_add_section_doc(&top, sub, "symbols", "name",
+ rspamd_rcl_symbol_handler,
+ UCL_OBJECT, FALSE, TRUE,
+ cfg->doc_strings,
+ "Symbols configuration");
+
+ /* Group part */
+ rspamd_rcl_add_default_handler(sub,
+ "max_score",
+ rspamd_rcl_parse_struct_double,
+ G_STRUCT_OFFSET(struct rspamd_symbols_group, max_score),
+ 0,
+ "Maximum score that could be reached by this symbols group");
+ }
+
+ if (!(skip_sections && g_hash_table_lookup(skip_sections, "worker"))) {
+ /**
+ * Worker section
+ */
+ auto *sub = rspamd_rcl_add_section_doc(&top, nullptr, "worker", "type",
+ rspamd_rcl_worker_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Workers common options");
+ rspamd_rcl_add_default_handler(sub,
+ "count",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_worker_conf, count),
+ RSPAMD_CL_FLAG_INT_16,
+ "Number of workers to spawn");
+ rspamd_rcl_add_default_handler(sub,
+ "max_files",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_worker_conf, rlimit_nofile),
+ RSPAMD_CL_FLAG_INT_64,
+ "Maximum number of opened files per worker");
+ rspamd_rcl_add_default_handler(sub,
+ "max_core",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_worker_conf, rlimit_maxcore),
+ RSPAMD_CL_FLAG_INT_64,
+ "Max size of core file in bytes");
+ rspamd_rcl_add_default_handler(sub,
+ "enabled",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_worker_conf, enabled),
+ 0,
+ "Enable or disable a worker (true by default)");
+ }
+
+ if (!(skip_sections && g_hash_table_lookup(skip_sections, "modules"))) {
+ /**
+ * Modules handler
+ */
+ rspamd_rcl_add_section_doc(&top, nullptr,
+ "modules", nullptr,
+ rspamd_rcl_modules_handler,
+ UCL_OBJECT,
+ FALSE,
+ FALSE,
+ cfg->doc_strings,
+ "Lua plugins to load");
+ }
+
+ if (!(skip_sections && g_hash_table_lookup(skip_sections, "classifier"))) {
+ /**
+ * Classifiers handler
+ */
+ auto *sub = rspamd_rcl_add_section_doc(&top, nullptr,
+ "classifier", "type",
+ rspamd_rcl_classifier_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "CLassifier options");
+ /* Default classifier is 'bayes' for now */
+ sub->default_key = "bayes";
+
+ rspamd_rcl_add_default_handler(sub,
+ "min_tokens",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_classifier_config, min_tokens),
+ RSPAMD_CL_FLAG_INT_32,
+ "Minimum count of tokens (words) to be considered for statistics");
+ rspamd_rcl_add_default_handler(sub,
+ "min_token_hits",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_classifier_config, min_token_hits),
+ RSPAMD_CL_FLAG_UINT,
+ "Minimum number of hits for a token to be considered");
+ rspamd_rcl_add_default_handler(sub,
+ "min_prob_strength",
+ rspamd_rcl_parse_struct_double,
+ G_STRUCT_OFFSET(struct rspamd_classifier_config, min_token_hits),
+ 0,
+ "Use only tokens with probability in [0.5 - MPS, 0.5 + MPS]");
+ rspamd_rcl_add_default_handler(sub,
+ "max_tokens",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_classifier_config, max_tokens),
+ RSPAMD_CL_FLAG_INT_32,
+ "Maximum count of tokens (words) to be considered for statistics");
+ rspamd_rcl_add_default_handler(sub,
+ "min_learns",
+ rspamd_rcl_parse_struct_integer,
+ G_STRUCT_OFFSET(struct rspamd_classifier_config, min_learns),
+ RSPAMD_CL_FLAG_UINT,
+ "Minimum number of learns for each statfile to use this classifier");
+ rspamd_rcl_add_default_handler(sub,
+ "backend",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_classifier_config, backend),
+ 0,
+ "Statfiles engine");
+ rspamd_rcl_add_default_handler(sub,
+ "name",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_classifier_config, name),
+ 0,
+ "Name of classifier");
+
+ /*
+ * Statfile defaults
+ */
+ auto *ssub = rspamd_rcl_add_section_doc(&top, sub,
+ "statfile", "symbol",
+ rspamd_rcl_statfile_handler,
+ UCL_OBJECT,
+ TRUE,
+ TRUE,
+ sub->doc_ref,
+ "Statfiles options");
+ rspamd_rcl_add_default_handler(ssub,
+ "label",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_statfile_config, label),
+ 0,
+ "Statfile unique label");
+ rspamd_rcl_add_default_handler(ssub,
+ "spam",
+ rspamd_rcl_parse_struct_boolean,
+ G_STRUCT_OFFSET(struct rspamd_statfile_config, is_spam),
+ 0,
+ "Sets if this statfile contains spam samples");
+ }
+
+ if (!(skip_sections && g_hash_table_lookup(skip_sections, "composite"))) {
+ /**
+ * Composites handlers
+ */
+ rspamd_rcl_add_section_doc(&top, nullptr,
+ "composite", "name",
+ rspamd_rcl_composite_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Rspamd composite symbols");
+ rspamd_rcl_add_section_doc(&top, nullptr,
+ "composites", nullptr,
+ rspamd_rcl_composites_handler,
+ UCL_OBJECT,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Rspamd composite symbols");
+ }
+
+ if (!(skip_sections && g_hash_table_lookup(skip_sections, "lua"))) {
+ /**
+ * Lua handler
+ */
+ rspamd_rcl_add_section_doc(&top, nullptr,
+ "lua", nullptr,
+ rspamd_rcl_lua_handler,
+ UCL_STRING,
+ FALSE,
+ TRUE,
+ cfg->doc_strings,
+ "Lua files to load");
+ }
+
+ cfg->rcl_top_section = top;
+
+ return top;
+}
+
+static bool
+rspamd_rcl_process_section(struct rspamd_config *cfg,
+ const struct rspamd_rcl_section &sec,
+ gpointer ptr, const ucl_object_t *obj, rspamd_mempool_t *pool,
+ GError **err)
+{
+ ucl_object_iter_t it;
+ const ucl_object_t *cur;
+ auto is_nested = true;
+ const gchar *key = nullptr;
+
+ if (sec.processed) {
+ /* Section has been already processed */
+ return TRUE;
+ }
+
+ g_assert(obj != nullptr);
+ g_assert(sec.handler != nullptr);
+
+ if (sec.key_attr) {
+ it = ucl_object_iterate_new(obj);
+
+ while ((cur = ucl_object_iterate_full(it, UCL_ITERATE_EXPLICIT)) != nullptr) {
+ if (ucl_object_type(cur) != UCL_OBJECT) {
+ is_nested = false;
+ break;
+ }
+ }
+
+ ucl_object_iterate_free(it);
+ }
+ else {
+ is_nested = false;
+ }
+
+ if (is_nested) {
+ /* Just reiterate on all subobjects */
+ it = ucl_object_iterate_new(obj);
+
+ while ((cur = ucl_object_iterate_full(it, UCL_ITERATE_EXPLICIT)) != nullptr) {
+ if (!sec.handler(pool, cur, ucl_object_key(cur), ptr, const_cast<rspamd_rcl_section *>(&sec), err)) {
+ ucl_object_iterate_free(it);
+
+ return false;
+ }
+ }
+
+ ucl_object_iterate_free(it);
+
+ return true;
+ }
+ else {
+ if (sec.key_attr) {
+ /* First of all search for required attribute and use it as a key */
+ cur = ucl_object_lookup(obj, sec.key_attr.value().c_str());
+
+ if (cur == nullptr) {
+ if (!sec.default_key) {
+ g_set_error(err, CFG_RCL_ERROR, EINVAL, "required attribute "
+ "'%s' is missing for section '%s', current key: %s",
+ sec.key_attr.value().c_str(),
+ sec.name.c_str(),
+ ucl_object_key(obj));
+
+ return false;
+ }
+ else {
+ msg_info("using default key '%s' for mandatory field '%s' "
+ "for section '%s'",
+ sec.default_key.value().c_str(), sec.key_attr.value().c_str(),
+ sec.name.c_str());
+ key = sec.default_key.value().c_str();
+ }
+ }
+ else if (ucl_object_type(cur) != UCL_STRING) {
+ g_set_error(err, CFG_RCL_ERROR, EINVAL, "required attribute %s"
+ " is not a string for section %s",
+ sec.key_attr.value().c_str(), sec.name.c_str());
+
+ return false;
+ }
+ else {
+ key = ucl_object_tostring(cur);
+ }
+ }
+ }
+
+ return sec.handler(pool, obj, key, ptr, const_cast<rspamd_rcl_section *>(&sec), err);
+}
+
+gboolean
+rspamd_rcl_parse(struct rspamd_rcl_sections_map *top,
+ struct rspamd_config *cfg,
+ gpointer ptr, rspamd_mempool_t *pool,
+ const ucl_object_t *obj, GError **err)
+{
+ if (obj->type != UCL_OBJECT) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "top configuration must be an object");
+ return FALSE;
+ }
+
+ /* Iterate over known sections and ignore unknown ones */
+ for (const auto &sec_ptr: top->sections_order) {
+ if (sec_ptr->name == "*") {
+ /* Default section handler */
+ const auto *cur_obj = obj;
+ LL_FOREACH(obj, cur_obj)
+ {
+ if (!top->sections.contains(ucl_object_key(cur_obj))) {
+ if (sec_ptr->handler != nullptr) {
+ if (!rspamd_rcl_process_section(cfg, *sec_ptr, ptr, cur_obj,
+ pool, err)) {
+ return FALSE;
+ }
+ }
+ else {
+ rspamd_rcl_section_parse_defaults(cfg,
+ *sec_ptr,
+ pool,
+ cur_obj,
+ ptr,
+ err);
+ }
+ }
+ }
+ }
+ else {
+ const auto *found = ucl_object_lookup(obj, sec_ptr->name.c_str());
+ if (found == nullptr) {
+ if (sec_ptr->required) {
+ g_set_error(err, CFG_RCL_ERROR, ENOENT,
+ "required section %s is missing", sec_ptr->name.c_str());
+ return FALSE;
+ }
+ }
+ else {
+ /* Check type */
+ if (sec_ptr->strict_type) {
+ if (sec_ptr->type != found->type) {
+ g_set_error(err, CFG_RCL_ERROR, EINVAL,
+ "object in section %s has invalid type", sec_ptr->name.c_str());
+ return FALSE;
+ }
+ }
+
+ const auto *cur_obj = found;
+ LL_FOREACH(found, cur_obj)
+ {
+ if (sec_ptr->handler != nullptr) {
+ if (!rspamd_rcl_process_section(cfg, *sec_ptr, ptr, cur_obj,
+ pool, err)) {
+ return FALSE;
+ }
+ }
+ else {
+ rspamd_rcl_section_parse_defaults(cfg, *sec_ptr,
+ pool,
+ cur_obj,
+ ptr,
+ err);
+ }
+ }
+ }
+ }
+ if (sec_ptr->fin) {
+ sec_ptr->fin(pool, sec_ptr->fin_ud);
+ }
+ }
+
+ return TRUE;
+}
+
+static bool
+rspamd_rcl_section_parse_defaults(struct rspamd_config *cfg,
+ const struct rspamd_rcl_section &section,
+ rspamd_mempool_t *pool, const ucl_object_t *obj, gpointer ptr,
+ GError **err)
+{
+
+ if (obj->type != UCL_OBJECT) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "default configuration must be an object for section %s "
+ "(actual type is %s)",
+ section.name.c_str(), ucl_object_type_to_string(ucl_object_type(obj)));
+ return FALSE;
+ }
+
+ for (const auto &cur: section.default_parser) {
+ const auto *found = ucl_object_lookup(obj, cur.first.c_str());
+ if (found != nullptr) {
+ auto new_pd = cur.second.pd;
+ new_pd.user_struct = ptr;
+ new_pd.cfg = cfg;
+ const auto *cur_obj = found;
+
+ LL_FOREACH(found, cur_obj)
+ {
+ if (!cur.second.handler(pool, cur_obj, &new_pd, const_cast<rspamd_rcl_section *>(&section), err)) {
+ return FALSE;
+ }
+
+ if (!(new_pd.flags & RSPAMD_CL_FLAG_MULTIPLE)) {
+ break;
+ }
+ }
+ }
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_rcl_parse_struct_string(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *pd = (struct rspamd_rcl_struct_parser *) ud;
+ const gsize num_str_len = 32;
+
+ auto target = (gchar **) (((gchar *) pd->user_struct) + pd->offset);
+ switch (obj->type) {
+ case UCL_STRING:
+ *target =
+ rspamd_mempool_strdup(pool, ucl_copy_value_trash(obj));
+ break;
+ case UCL_INT:
+ *target = (gchar *) rspamd_mempool_alloc(pool, num_str_len);
+ rspamd_snprintf(*target, num_str_len, "%L", obj->value.iv);
+ break;
+ case UCL_FLOAT:
+ *target = (gchar *) rspamd_mempool_alloc(pool, num_str_len);
+ rspamd_snprintf(*target, num_str_len, "%f", obj->value.dv);
+ break;
+ case UCL_BOOLEAN:
+ *target = (gchar *) rspamd_mempool_alloc(pool, num_str_len);
+ rspamd_snprintf(*target, num_str_len, "%s",
+ ((gboolean) obj->value.iv) ? "true" : "false");
+ break;
+ case UCL_NULL:
+ /* String is enforced to be null */
+ *target = nullptr;
+ break;
+ default:
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to string in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_rcl_parse_struct_integer(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *pd = (struct rspamd_rcl_struct_parser *) ud;
+ union {
+ gint *ip;
+ gint32 *i32p;
+ gint16 *i16p;
+ gint64 *i64p;
+ guint *up;
+ gsize *sp;
+ } target;
+ int64_t val;
+
+ if (pd->flags == RSPAMD_CL_FLAG_INT_32) {
+ target.i32p = (gint32 *) (((gchar *) pd->user_struct) + pd->offset);
+ if (!ucl_object_toint_safe(obj, &val)) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to integer in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+ *target.i32p = val;
+ }
+ else if (pd->flags == RSPAMD_CL_FLAG_INT_64) {
+ target.i64p = (gint64 *) (((gchar *) pd->user_struct) + pd->offset);
+ if (!ucl_object_toint_safe(obj, &val)) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to integer in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+ *target.i64p = val;
+ }
+ else if (pd->flags == RSPAMD_CL_FLAG_INT_SIZE) {
+ target.sp = (gsize *) (((gchar *) pd->user_struct) + pd->offset);
+ if (!ucl_object_toint_safe(obj, &val)) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to integer in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+ *target.sp = val;
+ }
+ else if (pd->flags == RSPAMD_CL_FLAG_INT_16) {
+ target.i16p = (gint16 *) (((gchar *) pd->user_struct) + pd->offset);
+ if (!ucl_object_toint_safe(obj, &val)) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to integer in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+ *target.i16p = val;
+ }
+ else if (pd->flags == RSPAMD_CL_FLAG_UINT) {
+ target.up = (guint *) (((gchar *) pd->user_struct) + pd->offset);
+ if (!ucl_object_toint_safe(obj, &val)) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to integer in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+ *target.up = val;
+ }
+ else {
+ target.ip = (gint *) (((gchar *) pd->user_struct) + pd->offset);
+ if (!ucl_object_toint_safe(obj, &val)) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to integer in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+ *target.ip = val;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_rcl_parse_struct_double(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *pd = (struct rspamd_rcl_struct_parser *) ud;
+ gdouble *target;
+
+ target = (gdouble *) (((gchar *) pd->user_struct) + pd->offset);
+
+ if (!ucl_object_todouble_safe(obj, target)) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to double in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_rcl_parse_struct_time(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *pd = (struct rspamd_rcl_struct_parser *) ud;
+ union {
+ gint *psec;
+ guint32 *pu32;
+ gdouble *pdv;
+ struct timeval *ptv;
+ struct timespec *pts;
+ } target;
+ gdouble val;
+
+ if (!ucl_object_todouble_safe(obj, &val)) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to double in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+
+ if (pd->flags == RSPAMD_CL_FLAG_TIME_TIMEVAL) {
+ target.ptv =
+ (struct timeval *) (((gchar *) pd->user_struct) + pd->offset);
+ target.ptv->tv_sec = (glong) val;
+ target.ptv->tv_usec = (val - (glong) val) * 1000000;
+ }
+ else if (pd->flags == RSPAMD_CL_FLAG_TIME_TIMESPEC) {
+ target.pts =
+ (struct timespec *) (((gchar *) pd->user_struct) + pd->offset);
+ target.pts->tv_sec = (glong) val;
+ target.pts->tv_nsec = (val - (glong) val) * 1000000000000LL;
+ }
+ else if (pd->flags == RSPAMD_CL_FLAG_TIME_FLOAT) {
+ target.pdv = (double *) (((gchar *) pd->user_struct) + pd->offset);
+ *target.pdv = val;
+ }
+ else if (pd->flags == RSPAMD_CL_FLAG_TIME_INTEGER) {
+ target.psec = (gint *) (((gchar *) pd->user_struct) + pd->offset);
+ *target.psec = val * 1000;
+ }
+ else if (pd->flags == RSPAMD_CL_FLAG_TIME_UINT_32) {
+ target.pu32 = (guint32 *) (((gchar *) pd->user_struct) + pd->offset);
+ *target.pu32 = val * 1000;
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to time in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_rcl_parse_struct_keypair(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *pd = (struct rspamd_rcl_struct_parser *) ud;
+ struct rspamd_cryptobox_keypair **target, *kp;
+
+ target = (struct rspamd_cryptobox_keypair **) (((gchar *) pd->user_struct) +
+ pd->offset);
+ if (obj->type == UCL_OBJECT) {
+ kp = rspamd_keypair_from_ucl(obj);
+
+ if (kp != nullptr) {
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) rspamd_keypair_unref, kp);
+ *target = kp;
+ }
+ else {
+ gchar *dump = (char *) ucl_object_emit(obj, UCL_EMIT_JSON_COMPACT);
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot load the keypair specified: %s; section: %s; value: %s",
+ ucl_object_key(obj), section->name.c_str(), dump);
+ free(dump);
+
+ return FALSE;
+ }
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "no sane pubkey or privkey found in the keypair: %s",
+ ucl_object_key(obj));
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_rcl_parse_struct_pubkey(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *pd = (struct rspamd_rcl_struct_parser *) ud;
+ struct rspamd_cryptobox_pubkey **target, *pk;
+ gsize len;
+ const gchar *str;
+ rspamd_cryptobox_keypair_type keypair_type = RSPAMD_KEYPAIR_KEX;
+ rspamd_cryptobox_mode keypair_mode = RSPAMD_CRYPTOBOX_MODE_25519;
+
+ if (pd->flags & RSPAMD_CL_FLAG_SIGNKEY) {
+ keypair_type = RSPAMD_KEYPAIR_SIGN;
+ }
+ if (pd->flags & RSPAMD_CL_FLAG_NISTKEY) {
+ keypair_mode = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+
+ target = (struct rspamd_cryptobox_pubkey **) (((gchar *) pd->user_struct) +
+ pd->offset);
+ if (obj->type == UCL_STRING) {
+ str = ucl_object_tolstring(obj, &len);
+ pk = rspamd_pubkey_from_base32(str, len, keypair_type,
+ keypair_mode);
+
+ if (pk != nullptr) {
+ *target = pk;
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot load the pubkey specified: %s",
+ ucl_object_key(obj));
+ return FALSE;
+ }
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "no sane pubkey found in the element: %s",
+ ucl_object_key(obj));
+ return FALSE;
+ }
+
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) rspamd_pubkey_unref, pk);
+
+ return TRUE;
+}
+
+static void
+rspamd_rcl_insert_string_list_item(gpointer *target, rspamd_mempool_t *pool,
+ std::string_view elt, gboolean is_hash)
+{
+ union {
+ GHashTable *hv;
+ GList *lv;
+ gpointer p;
+ } d;
+ gchar *val;
+
+ d.p = *target;
+
+ if (is_hash) {
+ if (d.hv == nullptr) {
+ d.hv = g_hash_table_new(rspamd_str_hash, rspamd_str_equal);
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) g_hash_table_unref, d.hv);
+ }
+
+ val = rspamd_mempool_strdup_len(pool, elt.data(), elt.size());
+ g_hash_table_insert(d.hv, val, val);
+ }
+ else {
+ val = rspamd_mempool_strdup_len(pool, elt.data(), elt.size());
+ d.lv = g_list_prepend(d.lv, val);
+ }
+
+ *target = d.p;
+}
+
+gboolean
+rspamd_rcl_parse_struct_string_list(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *pd = (struct rspamd_rcl_struct_parser *) ud;
+ constexpr const auto num_str_len = 32;
+ auto need_destructor = true;
+
+
+ auto is_hash = pd->flags & RSPAMD_CL_FLAG_STRING_LIST_HASH;
+ auto *target = (gpointer *) (((gchar *) pd->user_struct) + pd->offset);
+
+ if (!is_hash && *target != nullptr) {
+ need_destructor = FALSE;
+ }
+
+ auto iter = ucl_object_iterate_new(obj);
+ const auto *cur = obj;
+
+ while ((cur = ucl_object_iterate_safe(iter, true)) != nullptr) {
+ switch (cur->type) {
+ case UCL_STRING: {
+ rspamd::string_foreach_delim(ucl_object_tostring(cur), ", ", [&](const auto &elt) {
+ rspamd_rcl_insert_string_list_item(target, pool, elt, is_hash);
+ });
+
+ /* Go to the next object */
+ continue;
+ }
+ case UCL_INT: {
+ auto *val = (gchar *) rspamd_mempool_alloc(pool, num_str_len);
+ rspamd_snprintf(val, num_str_len, "%L", cur->value.iv);
+ rspamd_rcl_insert_string_list_item(target, pool, val, is_hash);
+ break;
+ }
+ case UCL_FLOAT: {
+ auto *val = (gchar *) rspamd_mempool_alloc(pool, num_str_len);
+ rspamd_snprintf(val, num_str_len, "%f", cur->value.dv);
+ rspamd_rcl_insert_string_list_item(target, pool, val, is_hash);
+ break;
+ }
+ case UCL_BOOLEAN: {
+ auto *val = (gchar *) rspamd_mempool_alloc(pool, num_str_len);
+ rspamd_snprintf(val, num_str_len, "%s",
+ ((gboolean) cur->value.iv) ? "true" : "false");
+ rspamd_rcl_insert_string_list_item(target, pool, val, is_hash);
+ break;
+ }
+ default:
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to a string list in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ ucl_object_iterate_free(iter);
+
+ return FALSE;
+ }
+ }
+
+ ucl_object_iterate_free(iter);
+
+#if 0
+ /* WTF: why don't we allow empty list here?? */
+ if (*target == nullptr) {
+ g_set_error (err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "non-empty array of strings is expected: %s, "
+ "got: %s, of length: %d",
+ ucl_object_key (obj), ucl_object_type_to_string (obj->type),
+ obj->len);
+ return FALSE;
+ }
+#endif
+
+ if (!is_hash && *target != nullptr) {
+ *target = g_list_reverse(*(GList **) target);
+
+ if (need_destructor) {
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) g_list_free,
+ *target);
+ }
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_rcl_parse_struct_ucl(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *pd = (struct rspamd_rcl_struct_parser *) ud;
+ const ucl_object_t **target;
+
+ target = (const ucl_object_t **) (((gchar *) pd->user_struct) + pd->offset);
+
+ *target = obj;
+
+ return TRUE;
+}
+
+
+gboolean
+rspamd_rcl_parse_struct_boolean(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *pd = (struct rspamd_rcl_struct_parser *) ud;
+ gboolean *target;
+
+ target = (gboolean *) (((gchar *) pd->user_struct) + pd->offset);
+
+ if (obj->type == UCL_BOOLEAN) {
+ *target = obj->value.iv;
+ }
+ else if (obj->type == UCL_INT) {
+ *target = obj->value.iv;
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to boolean in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+
+ if (pd->flags & RSPAMD_CL_FLAG_BOOLEAN_INVERSE) {
+ *target = !*target;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_rcl_parse_struct_addr(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *pd = (struct rspamd_rcl_struct_parser *) ud;
+ rspamd_inet_addr_t **target;
+ const gchar *val;
+ gsize size;
+
+ target = (rspamd_inet_addr_t **) (((gchar *) pd->user_struct) + pd->offset);
+
+ if (ucl_object_type(obj) == UCL_STRING) {
+ val = ucl_object_tolstring(obj, &size);
+
+ if (!rspamd_parse_inet_address(target, val, size,
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot parse inet address: %s", val);
+ return FALSE;
+ }
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot convert %s to inet address in option %s",
+ ucl_object_type_to_string(ucl_object_type(obj)),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_rcl_parse_struct_mime_addr(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ auto *pd = (struct rspamd_rcl_struct_parser *) ud;
+ GPtrArray **target, *tmp_addr = nullptr;
+ const gchar *val;
+ ucl_object_iter_t it;
+ const ucl_object_t *cur;
+
+ target = (GPtrArray **) (((gchar *) pd->user_struct) + pd->offset);
+ it = ucl_object_iterate_new(obj);
+
+ while ((cur = ucl_object_iterate_safe(it, true)) != nullptr) {
+ if (ucl_object_type(cur) == UCL_STRING) {
+ val = ucl_object_tostring(obj);
+ tmp_addr = rspamd_email_address_from_mime(pool, val,
+ strlen(val), tmp_addr, -1);
+ }
+ else {
+ g_set_error(err,
+ CFG_RCL_ERROR,
+ EINVAL,
+ "cannot get inet address from ucl object in %s",
+ ucl_object_key(obj));
+ ucl_object_iterate_free(it);
+
+ return FALSE;
+ }
+ }
+
+ ucl_object_iterate_free(it);
+ *target = tmp_addr;
+
+ return TRUE;
+}
+
+void rspamd_rcl_register_worker_option(struct rspamd_config *cfg,
+ GQuark type,
+ const gchar *name,
+ rspamd_rcl_default_handler_t handler,
+ gpointer target,
+ glong offset,
+ gint flags,
+ const gchar *doc_string)
+{
+ auto parser_it = cfg->rcl_top_section->workers_parser.try_emplace(type, rspamd_worker_cfg_parser{});
+ auto &parser = parser_it.first->second;
+ auto handler_it = parser.parsers.try_emplace(std::make_pair(std::string{name}, target), rspamd_worker_param_parser{});
+
+ if (!handler_it.second) {
+ msg_warn_config(
+ "handler for parameter %s is already registered for worker type %s",
+ name,
+ g_quark_to_string(type));
+ return;
+ }
+
+ auto &nhandler = handler_it.first->second;
+ nhandler.parser.flags = flags;
+ nhandler.parser.offset = offset;
+ nhandler.parser.user_struct = target;
+ nhandler.handler = handler;
+
+ const auto *doc_workers = ucl_object_lookup(cfg->doc_strings, "workers");
+
+ if (doc_workers == nullptr) {
+ auto *doc_obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(cfg->doc_strings, doc_obj, "workers", 0, false);
+ doc_workers = doc_obj;
+ }
+
+ const auto *doc_target = ucl_object_lookup(doc_workers, g_quark_to_string(type));
+
+ if (doc_target == nullptr) {
+ auto *doc_obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key((ucl_object_t *) doc_workers, doc_obj,
+ g_quark_to_string(type), 0, true);
+ doc_target = doc_obj;
+ }
+
+ rspamd_rcl_add_doc_obj((ucl_object_t *) doc_target,
+ doc_string,
+ name,
+ UCL_NULL,
+ handler,
+ flags,
+ nullptr,
+ 0);
+}
+
+/* Checksum functions */
+static int
+rspamd_rcl_emitter_append_c(unsigned char c, size_t nchars, void *ud)
+{
+ auto *hs = (rspamd_cryptobox_hash_state_t *) ud;
+ guint64 d[2];
+
+ d[0] = nchars;
+ d[1] = c;
+
+ rspamd_cryptobox_hash_update(hs, (const guchar *) d, sizeof(d));
+
+ return 0;
+}
+
+static int
+rspamd_rcl_emitter_append_len(unsigned const char *str, size_t len, void *ud)
+{
+ auto *hs = (rspamd_cryptobox_hash_state_t *) ud;
+
+ rspamd_cryptobox_hash_update(hs, str, len);
+
+ return 0;
+}
+static int
+rspamd_rcl_emitter_append_int(int64_t elt, void *ud)
+{
+ auto *hs = (rspamd_cryptobox_hash_state_t *) ud;
+
+ rspamd_cryptobox_hash_update(hs, (const guchar *) &elt, sizeof(elt));
+
+ return 0;
+}
+
+static int
+rspamd_rcl_emitter_append_double(double elt, void *ud)
+{
+ auto *hs = (rspamd_cryptobox_hash_state_t *) ud;
+
+ rspamd_cryptobox_hash_update(hs, (const guchar *) &elt, sizeof(elt));
+
+ return 0;
+}
+
+void rspamd_rcl_sections_free(struct rspamd_rcl_sections_map *sections)
+{
+ delete sections;
+}
+
+/**
+ * Calls for an external lua function to apply potential config transformations
+ * if needed. This function can change the cfg->rcl_obj.
+ *
+ * Example of transformation function:
+ *
+ * function(obj)
+ * if obj.something == 'foo' then
+ * obj.something = "bla"
+ * return true, obj
+ * end
+ *
+ * return false, nil
+ * end
+ *
+ * If function returns 'false' then rcl_obj is not touched. Otherwise,
+ * it is changed, then rcl_obj is imported from lua. Old config is dereferenced.
+ * @param cfg
+ */
+void rspamd_rcl_maybe_apply_lua_transform(struct rspamd_config *cfg)
+{
+ auto *L = RSPAMD_LUA_CFG_STATE(cfg);
+ static const char *transform_script = "lua_cfg_transform";
+
+ g_assert(L != nullptr);
+
+ if (!rspamd_lua_require_function(L, transform_script, nullptr)) {
+ /* No function defined */
+ msg_warn_config("cannot execute lua script %s: %s",
+ transform_script, lua_tostring(L, -1));
+
+ return;
+ }
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ auto err_idx = lua_gettop(L);
+
+ /* Push function */
+ lua_pushvalue(L, -2);
+
+ /* Push the existing config */
+ ucl_object_push_lua(L, cfg->cfg_ucl_obj, true);
+
+ if (auto ret = lua_pcall(L, 1, 2, err_idx); ret != 0) {
+ msg_err("call to rspamadm lua script failed (%d): %s", ret,
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return;
+ }
+
+ if (lua_toboolean(L, -2) && lua_type(L, -1) == LUA_TTABLE) {
+ ucl_object_t *old_cfg = cfg->cfg_ucl_obj;
+
+ msg_info_config("configuration has been transformed in Lua");
+ cfg->cfg_ucl_obj = ucl_object_lua_import(L, -1);
+ ucl_object_unref(old_cfg);
+ }
+
+ /* error function */
+ lua_settop(L, 0);
+}
+
+static bool
+rspamd_rcl_decrypt_handler(struct ucl_parser *parser,
+ const unsigned char *source, size_t source_len,
+ unsigned char **destination, size_t *dest_len,
+ void *user_data)
+{
+ GError *err = nullptr;
+ auto *kp = (struct rspamd_cryptobox_keypair *) user_data;
+
+ if (!rspamd_keypair_decrypt(kp, source, source_len,
+ destination, dest_len, &err)) {
+ msg_err("cannot decrypt file: %e", err);
+ g_error_free(err);
+
+ return false;
+ }
+
+ return true;
+}
+
+static bool
+rspamd_rcl_jinja_handler(struct ucl_parser *parser,
+ const unsigned char *source, size_t source_len,
+ unsigned char **destination, size_t *dest_len,
+ void *user_data)
+{
+ auto *cfg = (struct rspamd_config *) user_data;
+ auto *L = RSPAMD_LUA_CFG_STATE(cfg);
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ auto err_idx = lua_gettop(L);
+
+ /* Obtain function */
+ if (!rspamd_lua_require_function(L, "lua_util", "jinja_template")) {
+ msg_err_config("cannot require lua_util.jinja_template");
+ lua_settop(L, err_idx - 1);
+
+ return false;
+ }
+
+ lua_pushlstring(L, (const char *) source, source_len);
+ lua_getglobal(L, "rspamd_env");
+ lua_pushboolean(L, false);
+
+ if (lua_pcall(L, 3, 1, err_idx) != 0) {
+ msg_err_config("cannot call lua jinja_template script: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+
+ return false;
+ }
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const char *ndata;
+ gsize nsize;
+
+ ndata = lua_tolstring(L, -1, &nsize);
+ *destination = (unsigned char *) UCL_ALLOC(nsize);
+ memcpy(*destination, ndata, nsize);
+ *dest_len = nsize;
+ }
+ else {
+ msg_err_config("invalid return type when templating jinja %s",
+ lua_typename(L, lua_type(L, -1)));
+ lua_settop(L, err_idx - 1);
+
+ return false;
+ }
+
+ lua_settop(L, err_idx - 1);
+
+ return true;
+}
+
+static void
+rspamd_rcl_decrypt_free(unsigned char *data, size_t len, void *user_data)
+{
+ g_free(data);
+}
+
+void rspamd_config_calculate_cksum(struct rspamd_config *cfg)
+{
+ rspamd_cryptobox_hash_state_t hs;
+ unsigned char cksumbuf[rspamd_cryptobox_HASHBYTES];
+ struct ucl_emitter_functions f;
+
+ /* Calculate checksum */
+ rspamd_cryptobox_hash_init(&hs, nullptr, 0);
+ f.ucl_emitter_append_character = rspamd_rcl_emitter_append_c;
+ f.ucl_emitter_append_double = rspamd_rcl_emitter_append_double;
+ f.ucl_emitter_append_int = rspamd_rcl_emitter_append_int;
+ f.ucl_emitter_append_len = rspamd_rcl_emitter_append_len;
+ f.ucl_emitter_free_func = nullptr;
+ f.ud = &hs;
+ ucl_object_emit_full(cfg->cfg_ucl_obj, UCL_EMIT_MSGPACK,
+ &f, cfg->config_comments);
+ rspamd_cryptobox_hash_final(&hs, cksumbuf);
+ cfg->checksum = rspamd_encode_base32(cksumbuf, sizeof(cksumbuf), RSPAMD_BASE32_DEFAULT);
+ /* Also change the tag of cfg pool to be equal to the checksum */
+ rspamd_strlcpy(cfg->cfg_pool->tag.uid, cfg->checksum,
+ MIN(sizeof(cfg->cfg_pool->tag.uid), strlen(cfg->checksum)));
+}
+
+gboolean
+rspamd_config_parse_ucl(struct rspamd_config *cfg,
+ const gchar *filename,
+ GHashTable *vars,
+ ucl_include_trace_func_t inc_trace,
+ void *trace_data,
+ gboolean skip_jinja,
+ GError **err)
+{
+ struct rspamd_cryptobox_keypair *decrypt_keypair = nullptr;
+ auto cfg_file_maybe = rspamd::util::raii_mmaped_file::mmap_shared(filename, O_RDONLY, PROT_READ, 0);
+
+ if (!cfg_file_maybe) {
+ g_set_error(err, cfg_rcl_error_quark(), errno,
+ "cannot open %s: %*s", filename, (int) cfg_file_maybe.error().error_message.size(),
+ cfg_file_maybe.error().error_message.data());
+ return FALSE;
+ }
+
+ auto &cfg_file = cfg_file_maybe.value();
+
+ /* Try to load keyfile if available */
+ rspamd::util::raii_file::open(fmt::format("{}.key", filename), O_RDONLY).map([&](const auto &keyfile) {
+ auto *kp_parser = ucl_parser_new(0);
+ if (ucl_parser_add_fd(kp_parser, keyfile.get_fd())) {
+ auto *kp_obj = ucl_parser_get_object(kp_parser);
+
+ g_assert(kp_obj != nullptr);
+ decrypt_keypair = rspamd_keypair_from_ucl(kp_obj);
+
+ if (decrypt_keypair == nullptr) {
+ msg_err_config_forced("cannot load keypair from %s.key: invalid keypair",
+ filename);
+ }
+ else {
+ /* Add decryption support to UCL */
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_keypair_unref,
+ decrypt_keypair);
+ }
+
+ ucl_object_unref(kp_obj);
+ }
+ else {
+ msg_err_config_forced("cannot load keypair from %s.key: %s",
+ filename, ucl_parser_get_error(kp_parser));
+ }
+ ucl_parser_free(kp_parser);
+ });
+
+ auto parser = std::shared_ptr<ucl_parser>(ucl_parser_new(UCL_PARSER_SAVE_COMMENTS), ucl_parser_free);
+ rspamd_ucl_add_conf_variables(parser.get(), vars);
+ rspamd_ucl_add_conf_macros(parser.get(), cfg);
+ ucl_parser_set_filevars(parser.get(), filename, true);
+
+ if (inc_trace) {
+ ucl_parser_set_include_tracer(parser.get(), inc_trace, trace_data);
+ }
+
+ if (decrypt_keypair) {
+ auto *decrypt_handler = rspamd_mempool_alloc0_type(cfg->cfg_pool,
+ struct ucl_parser_special_handler);
+ decrypt_handler->user_data = decrypt_keypair;
+ decrypt_handler->magic = encrypted_magic;
+ decrypt_handler->magic_len = sizeof(encrypted_magic);
+ decrypt_handler->handler = rspamd_rcl_decrypt_handler;
+ decrypt_handler->free_function = rspamd_rcl_decrypt_free;
+
+ ucl_parser_add_special_handler(parser.get(), decrypt_handler);
+ }
+
+ if (!skip_jinja) {
+ auto *jinja_handler = rspamd_mempool_alloc0_type(cfg->cfg_pool,
+ struct ucl_parser_special_handler);
+ jinja_handler->user_data = cfg;
+ jinja_handler->flags = UCL_SPECIAL_HANDLER_PREPROCESS_ALL;
+ jinja_handler->handler = rspamd_rcl_jinja_handler;
+
+ ucl_parser_add_special_handler(parser.get(), jinja_handler);
+ }
+
+ if (!ucl_parser_add_chunk(parser.get(), (unsigned char *) cfg_file.get_map(), cfg_file.get_size())) {
+ g_set_error(err, cfg_rcl_error_quark(), errno,
+ "ucl parser error: %s", ucl_parser_get_error(parser.get()));
+
+ return FALSE;
+ }
+
+ cfg->cfg_ucl_obj = ucl_parser_get_object(parser.get());
+ cfg->config_comments = ucl_object_ref(ucl_parser_get_comments(parser.get()));
+
+ return TRUE;
+}
+
+gboolean
+rspamd_config_read(struct rspamd_config *cfg,
+ const gchar *filename,
+ rspamd_rcl_section_fin_t logger_fin,
+ gpointer logger_ud,
+ GHashTable *vars,
+ gboolean skip_jinja,
+ gchar **lua_env)
+{
+ GError *err = nullptr;
+
+ rspamd_lua_set_path(RSPAMD_LUA_CFG_STATE(cfg), nullptr, vars);
+
+ if (!rspamd_lua_set_env(RSPAMD_LUA_CFG_STATE(cfg), vars, lua_env, &err)) {
+ msg_err_config_forced("failed to set up environment: %e", err);
+ g_error_free(err);
+
+ return FALSE;
+ }
+
+ if (!rspamd_config_parse_ucl(cfg, filename, vars, nullptr, nullptr, skip_jinja, &err)) {
+ msg_err_config_forced("failed to load config: %e", err);
+ g_error_free(err);
+
+ return FALSE;
+ }
+
+ auto *top = rspamd_rcl_config_init(cfg, nullptr);
+ cfg->rcl_top_section = top;
+ /* Add new paths if defined in options */
+ rspamd_lua_set_path(RSPAMD_LUA_CFG_STATE(cfg), cfg->cfg_ucl_obj, vars);
+ rspamd_lua_set_globals(cfg, RSPAMD_LUA_CFG_STATE(cfg));
+ rspamd_mempool_add_destructor(cfg->cfg_pool, (rspamd_mempool_destruct_t) rspamd_rcl_sections_free, top);
+ err = nullptr;
+
+ /* Pre-init logging if possible */
+ if (logger_fin != nullptr) {
+ auto logging_section_maybe = rspamd::find_map(top->sections, "logging");
+
+ if (logging_section_maybe) {
+ const auto *logger_obj = ucl_object_lookup_any(cfg->cfg_ucl_obj, "logging",
+ "logger", nullptr);
+
+ if (logger_obj == nullptr) {
+ logger_fin(cfg->cfg_pool, logger_ud);
+ }
+ else {
+ if (!rspamd_rcl_process_section(cfg, *logging_section_maybe.value().get().get(), cfg,
+ logger_obj, cfg->cfg_pool, &err)) {
+ msg_err_config_forced("cannot init logger: %e", err);
+ g_error_free(err);
+
+ return FALSE;
+ }
+ else {
+ logger_fin(cfg->cfg_pool, logger_ud);
+ }
+
+ /* Init lua logging */
+ lua_pushcfunction(RSPAMD_LUA_CFG_STATE(cfg), &rspamd_lua_traceback);
+ auto err_idx = lua_gettop(RSPAMD_LUA_CFG_STATE(cfg));
+
+ /* Obtain function */
+ if (!rspamd_lua_require_function(RSPAMD_LUA_CFG_STATE(cfg), "lua_util",
+ "init_debug_logging")) {
+ msg_err_config("cannot require lua_util.init_debug_logging");
+ lua_settop(RSPAMD_LUA_CFG_STATE(cfg), err_idx - 1);
+
+ return FALSE;
+ }
+
+ void *pcfg = lua_newuserdata(RSPAMD_LUA_CFG_STATE(cfg), sizeof(void *));
+ memcpy(pcfg, &cfg, sizeof(void *));
+ rspamd_lua_setclass(RSPAMD_LUA_CFG_STATE(cfg), "rspamd{config}", -1);
+
+ if (lua_pcall(RSPAMD_LUA_CFG_STATE(cfg), 1, 0, err_idx) != 0) {
+ msg_err_config("cannot call lua init_debug_logging script: %s",
+ lua_tostring(RSPAMD_LUA_CFG_STATE(cfg), -1));
+ lua_settop(RSPAMD_LUA_CFG_STATE(cfg), err_idx - 1);
+
+ return FALSE;
+ }
+
+ lua_settop(RSPAMD_LUA_CFG_STATE(cfg), err_idx - 1);
+ }
+ }
+ }
+
+ /* Transform config if needed */
+ rspamd_rcl_maybe_apply_lua_transform(cfg);
+ rspamd_config_calculate_cksum(cfg);
+
+ if (!rspamd_rcl_parse(top, cfg, cfg, cfg->cfg_pool, cfg->cfg_ucl_obj, &err)) {
+ msg_err_config("rcl parse error: %e", err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return FALSE;
+ }
+
+ cfg->lang_det = rspamd_language_detector_init(cfg);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_language_detector_unref,
+ cfg->lang_det);
+
+ return TRUE;
+}
+
+static void
+rspamd_rcl_doc_obj_from_handler(ucl_object_t *doc_obj,
+ rspamd_rcl_default_handler_t handler,
+ gint flags)
+{
+ auto has_example = ucl_object_lookup(doc_obj, "example") != nullptr;
+ auto has_type = ucl_object_lookup(doc_obj, "type") != nullptr;
+
+ if (handler == rspamd_rcl_parse_struct_string) {
+ if (!has_type) {
+ ucl_object_insert_key(doc_obj, ucl_object_fromstring("string"),
+ "type", 0, false);
+ }
+ }
+ else if (handler == rspamd_rcl_parse_struct_integer) {
+ auto *type = "int";
+
+ if (flags & RSPAMD_CL_FLAG_INT_16) {
+ type = "int16";
+ }
+ else if (flags & RSPAMD_CL_FLAG_INT_32) {
+ type = "int32";
+ }
+ else if (flags & RSPAMD_CL_FLAG_INT_64) {
+ type = "int64";
+ }
+ else if (flags & RSPAMD_CL_FLAG_INT_SIZE) {
+ type = "size";
+ }
+ else if (flags & RSPAMD_CL_FLAG_UINT) {
+ type = "uint";
+ }
+
+ if (!has_type) {
+ ucl_object_insert_key(doc_obj, ucl_object_fromstring(type),
+ "type", 0, false);
+ }
+ }
+ else if (handler == rspamd_rcl_parse_struct_double) {
+ if (!has_type) {
+ ucl_object_insert_key(doc_obj, ucl_object_fromstring("double"),
+ "type", 0, false);
+ }
+ }
+ else if (handler == rspamd_rcl_parse_struct_time) {
+ auto *type = "time";
+
+ if (!has_type) {
+ ucl_object_insert_key(doc_obj, ucl_object_fromstring(type),
+ "type", 0, false);
+ }
+ }
+ else if (handler == rspamd_rcl_parse_struct_string_list) {
+ if (!has_type) {
+ ucl_object_insert_key(doc_obj, ucl_object_fromstring("string list"),
+ "type", 0, false);
+ }
+ if (!has_example) {
+ ucl_object_insert_key(doc_obj,
+ ucl_object_fromstring_common("param = \"str1, str2, str3\" OR "
+ "param = [\"str1\", \"str2\", \"str3\"]",
+ 0, static_cast<ucl_string_flags>(0)),
+ "example",
+ 0,
+ false);
+ }
+ }
+ else if (handler == rspamd_rcl_parse_struct_boolean) {
+ if (!has_type) {
+ ucl_object_insert_key(doc_obj,
+ ucl_object_fromstring("bool"),
+ "type",
+ 0,
+ false);
+ }
+ }
+ else if (handler == rspamd_rcl_parse_struct_keypair) {
+ if (!has_type) {
+ ucl_object_insert_key(doc_obj,
+ ucl_object_fromstring("keypair"),
+ "type",
+ 0,
+ false);
+ }
+ if (!has_example) {
+ ucl_object_insert_key(doc_obj,
+ ucl_object_fromstring("keypair { "
+ "pubkey = <base32_string>;"
+ " privkey = <base32_string>; "
+ "}"),
+ "example",
+ 0,
+ false);
+ }
+ }
+ else if (handler == rspamd_rcl_parse_struct_addr) {
+ if (!has_type) {
+ ucl_object_insert_key(doc_obj,
+ ucl_object_fromstring("socket address"),
+ "type",
+ 0,
+ false);
+ }
+ }
+ else if (handler == rspamd_rcl_parse_struct_mime_addr) {
+ if (!has_type) {
+ ucl_object_insert_key(doc_obj,
+ ucl_object_fromstring("email address"),
+ "type",
+ 0,
+ false);
+ }
+ }
+}
+
+ucl_object_t *
+rspamd_rcl_add_doc_obj(ucl_object_t *doc_target,
+ const char *doc_string,
+ const char *doc_name,
+ ucl_type_t type,
+ rspamd_rcl_default_handler_t handler,
+ gint flags,
+ const char *default_value,
+ gboolean required)
+{
+ ucl_object_t *doc_obj;
+
+ if (doc_target == nullptr || doc_name == nullptr) {
+ return nullptr;
+ }
+
+ doc_obj = ucl_object_typed_new(UCL_OBJECT);
+
+ /* Insert doc string itself */
+ if (doc_string) {
+ ucl_object_insert_key(doc_obj,
+ ucl_object_fromstring_common(doc_string, 0, static_cast<ucl_string_flags>(0)),
+ "data", 0, false);
+ }
+ else {
+ ucl_object_insert_key(doc_obj, ucl_object_fromstring("undocumented"),
+ "data", 0, false);
+ }
+
+ if (type != UCL_NULL) {
+ ucl_object_insert_key(doc_obj,
+ ucl_object_fromstring(ucl_object_type_to_string(type)),
+ "type", 0, false);
+ }
+
+ rspamd_rcl_doc_obj_from_handler(doc_obj, handler, flags);
+
+ ucl_object_insert_key(doc_obj,
+ ucl_object_frombool(required),
+ "required", 0, false);
+
+ if (default_value) {
+ ucl_object_insert_key(doc_obj,
+ ucl_object_fromstring_common(default_value, 0, static_cast<ucl_string_flags>(0)),
+ "default", 0, false);
+ }
+
+ ucl_object_insert_key(doc_target, doc_obj, doc_name, 0, true);
+
+ return doc_obj;
+}
+
+ucl_object_t *
+rspamd_rcl_add_doc_by_path(struct rspamd_config *cfg,
+ const gchar *doc_path,
+ const char *doc_string,
+ const char *doc_name,
+ ucl_type_t type,
+ rspamd_rcl_default_handler_t handler,
+ gint flags,
+ const char *default_value,
+ gboolean required)
+{
+ const auto *cur = cfg->doc_strings;
+
+ if (doc_path == nullptr) {
+ /* Assume top object */
+ return rspamd_rcl_add_doc_obj(cfg->doc_strings,
+ doc_string,
+ doc_name,
+ type,
+ handler,
+ flags,
+ default_value,
+ required);
+ }
+ else {
+ const auto *found = ucl_object_lookup_path(cfg->doc_strings, doc_path);
+
+ if (found != nullptr) {
+ return rspamd_rcl_add_doc_obj((ucl_object_t *) found,
+ doc_string,
+ doc_name,
+ type,
+ handler,
+ flags,
+ default_value,
+ required);
+ }
+
+ /* Otherwise we need to insert all components of the path */
+ rspamd::string_foreach_delim(doc_path, ".", [&](const std::string_view &elt) {
+ if (ucl_object_type(cur) != UCL_OBJECT) {
+ msg_err_config("Bad path while lookup for '%s' at %*s",
+ doc_path, (int) elt.size(), elt.data());
+ }
+ const auto *found = ucl_object_lookup_len(cur, elt.data(), elt.size());
+ if (found == nullptr) {
+ auto *obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key((ucl_object_t *) cur,
+ obj,
+ elt.data(),
+ elt.size(),
+ true);
+ cur = obj;
+ }
+ else {
+ cur = found;
+ }
+ });
+ }
+
+ return rspamd_rcl_add_doc_obj(ucl_object_ref(cur),
+ doc_string,
+ doc_name,
+ type,
+ handler,
+ flags,
+ default_value,
+ required);
+}
+
+static void
+rspamd_rcl_add_doc_from_comments(struct rspamd_config *cfg,
+ ucl_object_t *top_doc, const ucl_object_t *obj,
+ const ucl_object_t *comments, gboolean is_top)
+{
+ ucl_object_iter_t it = nullptr;
+ const ucl_object_t *cur, *cmt;
+ ucl_object_t *cur_doc;
+
+ if (ucl_object_type(obj) == UCL_OBJECT) {
+ while ((cur = ucl_object_iterate(obj, &it, true)) != nullptr) {
+ cur_doc = nullptr;
+
+ if ((cmt = ucl_comments_find(comments, cur)) != nullptr) {
+ cur_doc = rspamd_rcl_add_doc_obj(top_doc,
+ ucl_object_tostring(cmt), ucl_object_key(cur),
+ ucl_object_type(cur), nullptr, 0, nullptr, FALSE);
+ }
+
+ if (ucl_object_type(cur) == UCL_OBJECT) {
+ if (cur_doc) {
+ rspamd_rcl_add_doc_from_comments(cfg, cur_doc, cur,
+ comments,
+ FALSE);
+ }
+ else {
+ rspamd_rcl_add_doc_from_comments(cfg, top_doc, cur,
+ comments,
+ FALSE);
+ }
+ }
+ }
+ }
+ else if (!is_top) {
+ if ((cmt = ucl_comments_find(comments, obj)) != nullptr) {
+ rspamd_rcl_add_doc_obj(top_doc,
+ ucl_object_tostring(cmt), ucl_object_key(obj),
+ ucl_object_type(obj), nullptr, 0, nullptr, FALSE);
+ }
+ }
+}
+
+ucl_object_t *
+rspamd_rcl_add_doc_by_example(struct rspamd_config *cfg,
+ const gchar *root_path,
+ const gchar *doc_string,
+ const gchar *doc_name,
+ const gchar *example_data, gsize example_len)
+{
+ auto parser = std::shared_ptr<ucl_parser>(ucl_parser_new(UCL_PARSER_NO_FILEVARS | UCL_PARSER_SAVE_COMMENTS), ucl_parser_free);
+
+ if (!ucl_parser_add_chunk(parser.get(), reinterpret_cast<const unsigned char *>(example_data), example_len)) {
+ msg_err_config("cannot parse example: %s",
+ ucl_parser_get_error(parser.get()));
+
+ return nullptr;
+ }
+
+ auto *top = ucl_parser_get_object(parser.get());
+ const auto *comments = ucl_parser_get_comments(parser.get());
+
+ /* Add top object */
+ auto *top_doc = rspamd_rcl_add_doc_by_path(cfg, root_path, doc_string,
+ doc_name, ucl_object_type(top), nullptr, 0, nullptr, FALSE);
+ ucl_object_insert_key(top_doc,
+ ucl_object_fromstring_common(example_data, example_len, static_cast<ucl_string_flags>(0)),
+ "example", 0, false);
+
+ rspamd_rcl_add_doc_from_comments(cfg, top_doc, top, comments, TRUE);
+
+ return top_doc;
+}
diff --git a/src/libserver/cfg_rcl.h b/src/libserver/cfg_rcl.h
new file mode 100644
index 0000000..766c55e
--- /dev/null
+++ b/src/libserver/cfg_rcl.h
@@ -0,0 +1,476 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef CFG_RCL_H_
+#define CFG_RCL_H_
+
+#include "config.h"
+#include "cfg_file.h"
+#include "ucl.h"
+#include "mem_pool.h"
+
+#define CFG_RCL_ERROR cfg_rcl_error_quark()
+static inline GQuark
+cfg_rcl_error_quark(void)
+{
+ return g_quark_from_static_string("cfg-rcl-error-quark");
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_rcl_section;
+struct rspamd_rcl_sections_map;
+struct rspamd_config;
+struct rspamd_rcl_default_handler_data;
+
+enum rspamd_rcl_flag {
+ RSPAMD_CL_FLAG_TIME_FLOAT = 0x1 << 0,
+ RSPAMD_CL_FLAG_TIME_TIMEVAL = 0x1 << 1,
+ RSPAMD_CL_FLAG_TIME_TIMESPEC = 0x1 << 2,
+ RSPAMD_CL_FLAG_TIME_INTEGER = 0x1 << 3,
+ RSPAMD_CL_FLAG_TIME_UINT_32 = 0x1 << 4,
+ RSPAMD_CL_FLAG_INT_16 = 0x1 << 5,
+ RSPAMD_CL_FLAG_INT_32 = 0x1 << 6,
+ RSPAMD_CL_FLAG_INT_64 = 0x1 << 7,
+ RSPAMD_CL_FLAG_UINT = 0x1 << 8,
+ RSPAMD_CL_FLAG_INT_SIZE = 0x1 << 9,
+ RSPAMD_CL_FLAG_STRING_PATH = 0x1 << 10,
+ RSPAMD_CL_FLAG_BOOLEAN_INVERSE = 0x1 << 11,
+ RSPAMD_CL_FLAG_STRING_LIST_HASH = 0x1 << 12,
+ RSPAMD_CL_FLAG_MULTIPLE = 0x1 << 13,
+ RSPAMD_CL_FLAG_SIGNKEY = 0x1 << 14,
+ RSPAMD_CL_FLAG_NISTKEY = 0x1 << 15,
+};
+
+struct rspamd_rcl_struct_parser {
+ struct rspamd_config *cfg;
+ gpointer user_struct;
+ goffset offset;
+ int flags; /* enum rspamd_rcl_flag */
+};
+
+
+/**
+ * Common handler type
+ * @param cfg configuration
+ * @param obj object to parse
+ * @param ud user data (depends on section)
+ * @param err error object
+ * @return TRUE if a section has been parsed
+ */
+typedef gboolean (*rspamd_rcl_handler_t)(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ const gchar *key,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+typedef gboolean (*rspamd_rcl_default_handler_t)(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+/**
+ * A handler type that is called at the end of section parsing
+ * @param cfg configuration
+ * @param ud user data
+ */
+typedef void (*rspamd_rcl_section_fin_t)(rspamd_mempool_t *pool, gpointer ud);
+
+/**
+ * Add a default handler for a section
+ * @param section section pointer
+ * @param name name of param
+ * @param handler handler of param
+ * @param offset offset in a structure
+ * @param flags flags for the parser
+ * @return newly created structure
+ */
+struct rspamd_rcl_default_handler_data *rspamd_rcl_add_default_handler(
+ struct rspamd_rcl_section *section,
+ const gchar *name,
+ rspamd_rcl_default_handler_t handler,
+ goffset offset,
+ gint flags,
+ const gchar *doc_string);
+
+/**
+ * Add new section to the configuration
+ * @param top top section
+ * @param name the name of the section
+ * @param key_attr name of the attribute that should be used as key attribute
+ * @param handler handler function for all attributes
+ * @param type type of object handled by a handler
+ * @param required whether at least one of these sections is required
+ * @param strict_type turn on strict check for types for this section
+ * @return newly created structure
+ */
+struct rspamd_rcl_section *rspamd_rcl_add_section(
+ struct rspamd_rcl_sections_map **top,
+ struct rspamd_rcl_section *parent_section,
+ const gchar *name,
+ const gchar *key_attr,
+ rspamd_rcl_handler_t handler,
+ enum ucl_type type,
+ gboolean required,
+ gboolean strict_type);
+
+struct rspamd_rcl_section *rspamd_rcl_add_section_doc(
+ struct rspamd_rcl_sections_map **top,
+ struct rspamd_rcl_section *parent_section,
+ const gchar *name, const gchar *key_attr,
+ rspamd_rcl_handler_t handler,
+ enum ucl_type type, gboolean required,
+ gboolean strict_type,
+ ucl_object_t *doc_target,
+ const gchar *doc_string);
+
+/**
+ * Init common sections known to rspamd
+ * @return top section
+ */
+struct rspamd_rcl_sections_map *rspamd_rcl_config_init(struct rspamd_config *cfg,
+ GHashTable *skip_sections);
+
+/**
+ * Parse configuration
+ * @param top top section
+ * @param cfg rspamd configuration
+ * @param ptr pointer to the target
+ * @param pool pool object
+ * @param obj ucl object to parse
+ * @param err error pointer
+ * @return
+ */
+gboolean rspamd_rcl_parse(struct rspamd_rcl_sections_map *top,
+ struct rspamd_config *cfg,
+ gpointer ptr, rspamd_mempool_t *pool,
+ const ucl_object_t *obj, GError **err);
+
+/**
+ * Here is a section of common handlers that accepts rcl_struct_parser
+ * which itself contains a struct pointer and the offset of a member in a
+ * specific structure
+ */
+
+/**
+ * Parse a string field of a structure
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a string value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_string(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+/**
+ * Parse an integer field of a structure
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_integer(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+
+/**
+ * Parse a float field of a structure
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_double(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+/**
+ * Parse a time field of a structure
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure (flags mean the exact structure used)
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_time(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+/**
+ * Parse a string list field of a structure presented by a GList* object
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure (flags mean the exact structure used)
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_string_list(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+/**
+ * Parse a boolean field of a structure
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure (flags mean the exact structure used)
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_boolean(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+/**
+ * Parse a keypair field of a structure
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure (flags mean the exact structure used)
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_keypair(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+/**
+ * Parse a pubkey field of a structure
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure (flags mean the exact structure used)
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_pubkey(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+/**
+ * Parse a inet addr field of a structure
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure (flags mean the exact structure used)
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_addr(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+/**
+ * Parse a gmime inet address field of a structure
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure (flags mean the exact structure used)
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_mime_addr(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+/**
+ * Parse a raw ucl object
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure (flags mean the exact structure used)
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_ucl(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err);
+
+
+/**
+ * Utility functions
+ */
+
+/**
+ * Register new parser for a worker type of an option with the specified name
+ * @param cfg config structure
+ * @param type type of worker (GQuark)
+ * @param name name of option
+ * @param handler handler of option
+ * @param target opaque target structure, note it **MUST** be worker ctx due to some reasons I don't really remember
+ * @param offset offset inside a structure
+ */
+void rspamd_rcl_register_worker_option(struct rspamd_config *cfg,
+ GQuark type,
+ const gchar *name,
+ rspamd_rcl_default_handler_t handler,
+ gpointer target,
+ glong offset,
+ gint flags,
+ const gchar *doc_string);
+
+/**
+ * Adds new documentation object to the configuration
+ * @param doc_target target object where to insert documentation (top object is used if this is NULL)
+ * @param doc_object documentation object to insert
+ */
+ucl_object_t *rspamd_rcl_add_doc_obj(ucl_object_t *doc_target,
+ const char *doc_string,
+ const char *doc_name,
+ ucl_type_t type,
+ rspamd_rcl_default_handler_t handler,
+ gint flags,
+ const char *default_value,
+ gboolean required);
+
+/**
+ * Adds new documentation option specified by path `doc_path` that should be
+ * split by dots
+ */
+ucl_object_t *rspamd_rcl_add_doc_by_path(struct rspamd_config *cfg,
+ const gchar *doc_path,
+ const char *doc_string,
+ const char *doc_name,
+ ucl_type_t type,
+ rspamd_rcl_default_handler_t handler,
+ gint flags,
+ const char *default_value,
+ gboolean required);
+
+
+/**
+ * Parses example and adds documentation according to the example:
+ *
+ * ```
+ * section {
+ * param1 = value; # explanation
+ * param2 = value; # explanation
+ * }
+ * ```
+ *
+ * will produce the following documentation strings:
+ * section ->
+ * section.param1 : explanation
+ * section.param2 : explanation
+ *
+ * @param cfg
+ * @param root_path
+ * @param example_data
+ * @param example_len
+ * @return
+ */
+ucl_object_t *rspamd_rcl_add_doc_by_example(struct rspamd_config *cfg,
+ const gchar *root_path,
+ const gchar *doc_string,
+ const gchar *doc_name,
+ const gchar *example_data, gsize example_len);
+
+/**
+ * Add lua modules path
+ * @param cfg
+ * @param path
+ * @param err
+ * @return
+ */
+gboolean rspamd_rcl_add_lua_plugins_path(struct rspamd_rcl_sections_map *sections,
+ struct rspamd_config *cfg,
+ const gchar *path,
+ gboolean main_path,
+ GError **err);
+
+
+/**
+ * Calls for an external lua function to apply potential config transformations
+ * if needed. This function can change the cfg->rcl_obj.
+ *
+ * Example of transformation function:
+ *
+ * function(obj)
+ * if obj.something == 'foo' then
+ * obj.something = "bla"
+ * return true, obj
+ * end
+ *
+ * return false, nil
+ * end
+ *
+ * If function returns 'false' then rcl_obj is not touched. Otherwise,
+ * it is changed, then rcl_obj is imported from lua. Old config is dereferenced.
+ * @param cfg
+ */
+void rspamd_rcl_maybe_apply_lua_transform(struct rspamd_config *cfg);
+void rspamd_rcl_sections_free(struct rspamd_rcl_sections_map *sections);
+
+void rspamd_config_calculate_cksum(struct rspamd_config *cfg);
+
+/*
+ * Read configuration file
+ */
+gboolean rspamd_config_parse_ucl(struct rspamd_config *cfg,
+ const gchar *filename,
+ GHashTable *vars,
+ ucl_include_trace_func_t inc_trace,
+ void *trace_data,
+ gboolean skip_jinja,
+ GError **err);
+gboolean rspamd_config_read(struct rspamd_config *cfg,
+ const gchar *filename,
+ rspamd_rcl_section_fin_t logger_fin,
+ gpointer logger_ud,
+ GHashTable *vars,
+ gboolean skip_jinja,
+ gchar **lua_env);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CFG_RCL_H_ */
diff --git a/src/libserver/cfg_utils.cxx b/src/libserver/cfg_utils.cxx
new file mode 100644
index 0000000..3a94b47
--- /dev/null
+++ b/src/libserver/cfg_utils.cxx
@@ -0,0 +1,2955 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+
+#include "lua/lua_common.h"
+#include "lua/lua_thread_pool.h"
+
+#include "cfg_file.h"
+#include "rspamd.h"
+#include "cfg_file_private.h"
+
+#include "maps/map.h"
+#include "maps/map_helpers.h"
+#include "maps/map_private.h"
+#include "dynamic_cfg.h"
+#include "utlist.h"
+#include "stat_api.h"
+#include "unix-std.h"
+#include "libutil/multipattern.h"
+#include "monitored.h"
+#include "ref.h"
+#include "cryptobox.h"
+#include "ssl_util.h"
+#include "contrib/libottery/ottery.h"
+#include "contrib/fastutf8/fastutf8.h"
+
+#ifdef SYS_ZSTD
+#include "zstd.h"
+#else
+#define ZSTD_STATIC_LINKING_ONLY
+#include "contrib/zstd/zstd.h"
+#endif
+
+#ifdef HAVE_OPENSSL
+#include <openssl/rand.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/ssl.h>
+#include <openssl/conf.h>
+#endif
+#ifdef HAVE_LOCALE_H
+#include <locale.h>
+#endif
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
+#include <math.h>
+#include "libserver/composites/composites.h"
+
+#include "blas-config.h"
+
+#include <string>
+#include <string_view>
+#include <vector>
+#include "fmt/core.h"
+#include "cxx/util.hxx"
+#include "frozen/unordered_map.h"
+#include "frozen/string.h"
+#include "contrib/ankerl/unordered_dense.h"
+
+#define DEFAULT_SCORE 10.0
+
+#define DEFAULT_RLIMIT_NOFILE 2048
+#define DEFAULT_RLIMIT_MAXCORE 0
+#define DEFAULT_MAP_TIMEOUT 60.0 * 5
+#define DEFAULT_MAP_FILE_WATCH_MULTIPLIER 1
+#define DEFAULT_MIN_WORD 0
+#define DEFAULT_MAX_WORD 40
+#define DEFAULT_WORDS_DECAY 600
+#define DEFAULT_MAX_MESSAGE (50 * 1024 * 1024)
+#define DEFAULT_MAX_PIC (1 * 1024 * 1024)
+#define DEFAULT_MAX_SHOTS 100
+#define DEFAULT_MAX_SESSIONS 100
+#define DEFAULT_MAX_WORKERS 4
+#define DEFAULT_MAX_HTML_SIZE DEFAULT_MAX_MESSAGE / 5 /* 10 Mb */
+/* Timeout for task processing */
+#define DEFAULT_TASK_TIMEOUT 8.0
+#define DEFAULT_LUA_GC_STEP 200
+#define DEFAULT_LUA_GC_PAUSE 200
+#define DEFAULT_GC_MAXITERS 0
+
+struct rspamd_ucl_map_cbdata {
+ struct rspamd_config *cfg;
+ std::string buf;
+
+ explicit rspamd_ucl_map_cbdata(struct rspamd_config *cfg)
+ : cfg(cfg)
+ {
+ }
+};
+static gchar *rspamd_ucl_read_cb(gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+static void rspamd_ucl_fin_cb(struct map_cb_data *data, void **target);
+static void rspamd_ucl_dtor_cb(struct map_cb_data *data);
+
+guint rspamd_config_log_id = (guint) -1;
+RSPAMD_CONSTRUCTOR(rspamd_config_log_init)
+{
+ rspamd_config_log_id = rspamd_logger_add_debug_module("config");
+}
+
+struct rspamd_actions_list {
+ using action_ptr = std::shared_ptr<rspamd_action>;
+ std::vector<action_ptr> actions;
+ ankerl::unordered_dense::map<std::string_view, action_ptr> actions_by_name;
+
+ explicit rspamd_actions_list()
+ {
+ actions.reserve(METRIC_ACTION_MAX + 2);
+ actions_by_name.reserve(METRIC_ACTION_MAX + 2);
+ }
+
+ void add_action(action_ptr action)
+ {
+ actions.push_back(action);
+ actions_by_name[action->name] = action;
+ sort();
+ }
+
+ void sort()
+ {
+ std::sort(actions.begin(), actions.end(), [](const action_ptr &a1, const action_ptr &a2) -> bool {
+ if (!isnan(a1->threshold) && !isnan(a2->threshold)) {
+ return a1->threshold < a2->threshold;
+ }
+
+ if (isnan(a1->threshold) && isnan(a2->threshold)) {
+ return false;
+ }
+ else if (isnan(a1->threshold)) {
+ return true;
+ }
+
+ return false;
+ });
+ }
+
+ void clear()
+ {
+ actions.clear();
+ actions_by_name.clear();
+ }
+};
+
+#define RSPAMD_CFG_ACTIONS(cfg) (reinterpret_cast<rspamd_actions_list *>((cfg)->actions))
+
+gboolean
+rspamd_parse_bind_line(struct rspamd_config *cfg,
+ struct rspamd_worker_conf *cf,
+ const gchar *str)
+{
+ struct rspamd_worker_bind_conf *cnf;
+ const gchar *fdname;
+ gboolean ret = TRUE;
+
+ if (str == nullptr) {
+ return FALSE;
+ }
+
+ cnf = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_worker_bind_conf);
+
+ cnf->cnt = 1024;
+ cnf->bind_line = rspamd_mempool_strdup(cfg->cfg_pool, str);
+
+ auto bind_line = std::string_view{cnf->bind_line};
+
+ if (bind_line.starts_with("systemd:")) {
+ /* The actual socket will be passed by systemd environment */
+ fdname = str + sizeof("systemd:") - 1;
+ cnf->is_systemd = TRUE;
+ cnf->addrs = g_ptr_array_new_full(1, nullptr);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ rspamd_ptr_array_free_hard, cnf->addrs);
+
+ if (fdname[0]) {
+ g_ptr_array_add(cnf->addrs, rspamd_mempool_strdup(cfg->cfg_pool, fdname));
+ cnf->cnt = cnf->addrs->len;
+ cnf->name = rspamd_mempool_strdup(cfg->cfg_pool, str);
+ LL_PREPEND(cf->bind_conf, cnf);
+ }
+ else {
+ msg_err_config("cannot parse bind line: %s", str);
+ ret = FALSE;
+ }
+ }
+ else {
+ if (rspamd_parse_host_port_priority(str, &cnf->addrs,
+ nullptr, &cnf->name, DEFAULT_BIND_PORT, TRUE, cfg->cfg_pool) == RSPAMD_PARSE_ADDR_FAIL) {
+ msg_err_config("cannot parse bind line: %s", str);
+ ret = FALSE;
+ }
+ else {
+ cnf->cnt = cnf->addrs->len;
+ LL_PREPEND(cf->bind_conf, cnf);
+ }
+ }
+
+ return ret;
+}
+
+struct rspamd_config *
+rspamd_config_new(enum rspamd_config_init_flags flags)
+{
+ struct rspamd_config *cfg;
+ rspamd_mempool_t *pool;
+
+ pool = rspamd_mempool_new(8 * 1024 * 1024, "cfg", 0);
+ cfg = rspamd_mempool_alloc0_type(pool, struct rspamd_config);
+ /* Allocate larger pool for cfg */
+ cfg->cfg_pool = pool;
+ cfg->dns_timeout = 1.0;
+ cfg->dns_retransmits = 5;
+ /* 16 sockets per DNS server */
+ cfg->dns_io_per_server = 16;
+ cfg->unknown_weight = NAN;
+
+ cfg->actions = (void *) new rspamd_actions_list();
+
+ /* Add all internal actions to keep compatibility */
+ for (int i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) {
+
+ auto &&action = std::make_shared<rspamd_action>();
+ action->threshold = NAN;
+ action->name = rspamd_mempool_strdup(cfg->cfg_pool,
+ rspamd_action_to_str(static_cast<rspamd_action_type>(i)));
+ action->action_type = static_cast<rspamd_action_type>(i);
+
+ if (i == METRIC_ACTION_SOFT_REJECT) {
+ action->flags |= RSPAMD_ACTION_NO_THRESHOLD | RSPAMD_ACTION_HAM;
+ }
+ else if (i == METRIC_ACTION_GREYLIST) {
+ action->flags |= RSPAMD_ACTION_THRESHOLD_ONLY | RSPAMD_ACTION_HAM;
+ }
+ else if (i == METRIC_ACTION_NOACTION) {
+ action->flags |= RSPAMD_ACTION_HAM;
+ }
+
+ RSPAMD_CFG_ACTIONS(cfg)->add_action(std::move(action));
+ }
+
+ /* Disable timeout */
+ cfg->task_timeout = DEFAULT_TASK_TIMEOUT;
+
+
+ rspamd_config_init_metric(cfg);
+ cfg->composites_manager = rspamd_composites_manager_create(cfg);
+ cfg->classifiers_symbols = g_hash_table_new(rspamd_str_hash,
+ rspamd_str_equal);
+ cfg->cfg_params = g_hash_table_new(rspamd_str_hash, rspamd_str_equal);
+ cfg->debug_modules = g_hash_table_new(rspamd_str_hash, rspamd_str_equal);
+ cfg->explicit_modules = g_hash_table_new(rspamd_str_hash, rspamd_str_equal);
+ cfg->trusted_keys = g_hash_table_new(rspamd_str_hash,
+ rspamd_str_equal);
+
+ cfg->map_timeout = DEFAULT_MAP_TIMEOUT;
+ cfg->map_file_watch_multiplier = DEFAULT_MAP_FILE_WATCH_MULTIPLIER;
+
+ cfg->log_level = G_LOG_LEVEL_WARNING;
+ cfg->log_flags = RSPAMD_LOG_FLAG_DEFAULT;
+
+ cfg->check_text_attachements = TRUE;
+
+ cfg->dns_max_requests = 64;
+ cfg->history_rows = 200;
+ cfg->log_error_elts = 10;
+ cfg->log_error_elt_maxlen = 1000;
+ cfg->log_task_max_elts = 7;
+ cfg->cache_reload_time = 30.0;
+ cfg->max_lua_urls = 1024;
+ cfg->max_urls = cfg->max_lua_urls * 10;
+ cfg->max_recipients = 1024;
+ cfg->max_blas_threads = 1;
+ cfg->max_opts_len = 4096;
+ cfg->gtube_patterns_policy = RSPAMD_GTUBE_REJECT;
+
+ /* Default log line */
+ cfg->log_format_str = rspamd_mempool_strdup(cfg->cfg_pool,
+ "id: <$mid>,$if_qid{ qid: <$>,}$if_ip{ ip: $,}"
+ "$if_user{ user: $,}$if_smtp_from{ from: <$>,} (default: $is_spam "
+ "($action): [$scores] [$symbols_scores_params]), len: $len, time: $time_real, "
+ "dns req: $dns_req, digest: <$digest>"
+ "$if_smtp_rcpts{ rcpts: <$>, }$if_mime_rcpt{ mime_rcpt: <$>, }");
+ /* Allow non-mime input by default */
+ cfg->allow_raw_input = TRUE;
+ /* Default maximum words processed */
+ cfg->words_decay = DEFAULT_WORDS_DECAY;
+ cfg->min_word_len = DEFAULT_MIN_WORD;
+ cfg->max_word_len = DEFAULT_MAX_WORD;
+ cfg->max_html_len = DEFAULT_MAX_HTML_SIZE;
+
+ /* GC limits */
+ cfg->lua_gc_pause = DEFAULT_LUA_GC_PAUSE;
+ cfg->lua_gc_step = DEFAULT_LUA_GC_STEP;
+ cfg->full_gc_iters = DEFAULT_GC_MAXITERS;
+
+ /* Default hyperscan cache */
+ cfg->hs_cache_dir = rspamd_mempool_strdup(cfg->cfg_pool, RSPAMD_DBDIR "/");
+
+ if (!(flags & RSPAMD_CONFIG_INIT_SKIP_LUA)) {
+ cfg->lua_state = (void *) rspamd_lua_init(flags & RSPAMD_CONFIG_INIT_WIPE_LUA_MEM);
+ cfg->own_lua_state = TRUE;
+ cfg->lua_thread_pool = (void *) lua_thread_pool_new(RSPAMD_LUA_CFG_STATE(cfg));
+ }
+
+ cfg->cache = rspamd_symcache_new(cfg);
+ cfg->ups_ctx = rspamd_upstreams_library_init();
+ cfg->re_cache = rspamd_re_cache_new();
+ cfg->doc_strings = ucl_object_typed_new(UCL_OBJECT);
+ /*
+ * Unless exim is fixed
+ */
+ cfg->enable_shutdown_workaround = TRUE;
+
+ cfg->ssl_ciphers = rspamd_mempool_strdup(cfg->cfg_pool, "HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4");
+ cfg->max_message = DEFAULT_MAX_MESSAGE;
+ cfg->max_pic_size = DEFAULT_MAX_PIC;
+ cfg->images_cache_size = 256;
+ cfg->monitored_ctx = rspamd_monitored_ctx_init();
+ cfg->neighbours = ucl_object_typed_new(UCL_OBJECT);
+ cfg->redis_pool = rspamd_redis_pool_init();
+ cfg->default_max_shots = DEFAULT_MAX_SHOTS;
+ cfg->max_sessions_cache = DEFAULT_MAX_SESSIONS;
+ cfg->maps_cache_dir = rspamd_mempool_strdup(cfg->cfg_pool, RSPAMD_DBDIR);
+ cfg->c_modules = g_ptr_array_new();
+ cfg->heartbeat_interval = 10.0;
+
+ cfg->enable_css_parser = true;
+ cfg->script_modules = g_ptr_array_new();
+
+ REF_INIT_RETAIN(cfg, rspamd_config_free);
+
+ return cfg;
+}
+
+void rspamd_config_free(struct rspamd_config *cfg)
+{
+ struct rspamd_config_cfg_lua_script *sc, *sctmp;
+ struct rspamd_config_settings_elt *set, *stmp;
+ struct rspamd_worker_log_pipe *lp, *ltmp;
+
+ rspamd_lua_run_config_unload(RSPAMD_LUA_CFG_STATE(cfg), cfg);
+
+ /* Scripts part */
+ DL_FOREACH_SAFE(cfg->on_term_scripts, sc, sctmp)
+ {
+ luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref);
+ }
+
+ DL_FOREACH_SAFE(cfg->on_load_scripts, sc, sctmp)
+ {
+ luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref);
+ }
+
+ DL_FOREACH_SAFE(cfg->post_init_scripts, sc, sctmp)
+ {
+ luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref);
+ }
+
+ DL_FOREACH_SAFE(cfg->config_unload_scripts, sc, sctmp)
+ {
+ luaL_unref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX, sc->cbref);
+ }
+
+ DL_FOREACH_SAFE(cfg->setting_ids, set, stmp)
+ {
+ REF_RELEASE(set);
+ }
+
+ rspamd_map_remove_all(cfg);
+ rspamd_mempool_destructors_enforce(cfg->cfg_pool);
+
+ g_list_free(cfg->classifiers);
+ g_list_free(cfg->workers);
+ rspamd_symcache_destroy(cfg->cache);
+ ucl_object_unref(cfg->cfg_ucl_obj);
+ ucl_object_unref(cfg->config_comments);
+ ucl_object_unref(cfg->doc_strings);
+ ucl_object_unref(cfg->neighbours);
+ g_hash_table_remove_all(cfg->cfg_params);
+ g_hash_table_unref(cfg->cfg_params);
+ g_hash_table_unref(cfg->classifiers_symbols);
+ g_hash_table_unref(cfg->debug_modules);
+ g_hash_table_unref(cfg->explicit_modules);
+ g_hash_table_unref(cfg->trusted_keys);
+
+ rspamd_re_cache_unref(cfg->re_cache);
+ g_ptr_array_free(cfg->c_modules, TRUE);
+ g_ptr_array_free(cfg->script_modules, TRUE);
+
+ if (cfg->monitored_ctx) {
+ rspamd_monitored_ctx_destroy(cfg->monitored_ctx);
+ }
+
+ if (RSPAMD_LUA_CFG_STATE(cfg) && cfg->own_lua_state) {
+ lua_thread_pool_free((struct lua_thread_pool *) cfg->lua_thread_pool);
+ rspamd_lua_close(RSPAMD_LUA_CFG_STATE(cfg));
+ }
+
+ if (cfg->redis_pool) {
+ rspamd_redis_pool_destroy(cfg->redis_pool);
+ }
+
+ rspamd_upstreams_library_unref(cfg->ups_ctx);
+ delete RSPAMD_CFG_ACTIONS(cfg);
+
+ rspamd_mempool_destructors_enforce(cfg->cfg_pool);
+
+ if (cfg->checksum) {
+ g_free(cfg->checksum);
+ }
+
+ REF_RELEASE(cfg->libs_ctx);
+
+ DL_FOREACH_SAFE(cfg->log_pipes, lp, ltmp)
+ {
+ close(lp->fd);
+ g_free(lp);
+ }
+
+ rspamd_mempool_delete(cfg->cfg_pool);
+}
+
+const ucl_object_t *
+rspamd_config_get_module_opt(struct rspamd_config *cfg,
+ const gchar *module_name,
+ const gchar *opt_name)
+{
+ const ucl_object_t *res = nullptr, *sec;
+
+ sec = ucl_obj_get_key(cfg->cfg_ucl_obj, module_name);
+ if (sec != nullptr) {
+ res = ucl_obj_get_key(sec, opt_name);
+ }
+
+ return res;
+}
+
+gint rspamd_config_parse_flag(const gchar *str, guint len)
+{
+ gint c;
+
+ if (!str || !*str) {
+ return -1;
+ }
+
+ if (len == 0) {
+ len = strlen(str);
+ }
+
+ switch (len) {
+ case 1:
+ c = g_ascii_tolower(*str);
+ if (c == 'y' || c == '1') {
+ return 1;
+ }
+ else if (c == 'n' || c == '0') {
+ return 0;
+ }
+ break;
+ case 2:
+ if (g_ascii_strncasecmp(str, "no", len) == 0) {
+ return 0;
+ }
+ else if (g_ascii_strncasecmp(str, "on", len) == 0) {
+ return 1;
+ }
+ break;
+ case 3:
+ if (g_ascii_strncasecmp(str, "yes", len) == 0) {
+ return 1;
+ }
+ else if (g_ascii_strncasecmp(str, "off", len) == 0) {
+ return 0;
+ }
+ break;
+ case 4:
+ if (g_ascii_strncasecmp(str, "true", len) == 0) {
+ return 1;
+ }
+ break;
+ case 5:
+ if (g_ascii_strncasecmp(str, "false", len) == 0) {
+ return 0;
+ }
+ break;
+ }
+
+ return -1;
+}
+
+// A mapping between names and log format types + flags
+constexpr const auto config_vars = frozen::make_unordered_map<frozen::string, std::pair<rspamd_log_format_type, int>>({
+ {"mid", {RSPAMD_LOG_MID, 0}},
+ {"qid", {RSPAMD_LOG_QID, 0}},
+ {"user", {RSPAMD_LOG_USER, 0}},
+ {"ip", {RSPAMD_LOG_IP, 0}},
+ {"len", {RSPAMD_LOG_LEN, 0}},
+ {"dns_req", {RSPAMD_LOG_DNS_REQ, 0}},
+ {"smtp_from", {RSPAMD_LOG_SMTP_FROM, 0}},
+ {"mime_from", {RSPAMD_LOG_MIME_FROM, 0}},
+ {"smtp_rcpt", {RSPAMD_LOG_SMTP_RCPT, 0}},
+ {"mime_rcpt", {RSPAMD_LOG_MIME_RCPT, 0}},
+ {"smtp_rcpts", {RSPAMD_LOG_SMTP_RCPTS, 0}},
+ {"mime_rcpts", {RSPAMD_LOG_MIME_RCPTS, 0}},
+ {"time_real", {RSPAMD_LOG_TIME_REAL, 0}},
+ {"time_virtual", {RSPAMD_LOG_TIME_VIRTUAL, 0}},
+ {"lua", {RSPAMD_LOG_LUA, 0}},
+ {"digest", {RSPAMD_LOG_DIGEST, 0}},
+ {"checksum", {RSPAMD_LOG_DIGEST, 0}},
+ {"filename", {RSPAMD_LOG_FILENAME, 0}},
+ {"forced_action", {RSPAMD_LOG_FORCED_ACTION, 0}},
+ {"settings_id", {RSPAMD_LOG_SETTINGS_ID, 0}},
+ {"mempool_size", {RSPAMD_LOG_MEMPOOL_SIZE, 0}},
+ {"mempool_waste", {RSPAMD_LOG_MEMPOOL_WASTE, 0}},
+ {"action", {RSPAMD_LOG_ACTION, 0}},
+ {"scores", {RSPAMD_LOG_SCORES, 0}},
+ {"symbols", {RSPAMD_LOG_SYMBOLS, 0}},
+ {"symbols_scores", {RSPAMD_LOG_SYMBOLS, RSPAMD_LOG_FMT_FLAG_SYMBOLS_SCORES}},
+ {"symbols_params", {RSPAMD_LOG_SYMBOLS, RSPAMD_LOG_FMT_FLAG_SYMBOLS_PARAMS}},
+ {"symbols_scores_params", {RSPAMD_LOG_SYMBOLS, RSPAMD_LOG_FMT_FLAG_SYMBOLS_PARAMS | RSPAMD_LOG_FMT_FLAG_SYMBOLS_SCORES}},
+ {"groups", {RSPAMD_LOG_GROUPS, 0}},
+ {"public_groups", {RSPAMD_LOG_PUBLIC_GROUPS, 0}},
+ {"is_spam", {RSPAMD_LOG_ISSPAM, 0}},
+});
+
+static gboolean
+rspamd_config_process_var(struct rspamd_config *cfg, const rspamd_ftok_t *var,
+ const rspamd_ftok_t *content)
+{
+ g_assert(var != nullptr);
+
+ auto flags = 0;
+ auto lc_var = std::string{var->begin, var->len};
+ std::transform(lc_var.begin(), lc_var.end(), lc_var.begin(), g_ascii_tolower);
+ auto tok = std::string_view{lc_var};
+
+ if (var->len > 3 && tok.starts_with("if_")) {
+ flags |= RSPAMD_LOG_FMT_FLAG_CONDITION;
+ tok = tok.substr(3);
+ }
+
+ auto maybe_fmt_var = rspamd::find_map(config_vars, tok);
+
+ if (maybe_fmt_var) {
+ auto &fmt_var = maybe_fmt_var.value().get();
+ auto *log_format = rspamd_mempool_alloc0_type(cfg->cfg_pool, rspamd_log_format);
+
+ log_format->type = fmt_var.first;
+ log_format->flags = fmt_var.second | flags;
+
+ if (log_format->type != RSPAMD_LOG_LUA) {
+ if (content && content->len > 0) {
+ log_format->data = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(rspamd_ftok_t));
+ memcpy(log_format->data, content, sizeof(*content));
+ log_format->len = sizeof(*content);
+ }
+ }
+ else {
+ /* Load lua code and ensure that we have function ref returned */
+ if (!content || content->len == 0) {
+ msg_err_config("lua variable needs content: %T", &tok);
+ return FALSE;
+ }
+
+ if (luaL_loadbuffer(RSPAMD_LUA_CFG_STATE(cfg), content->begin, content->len,
+ "lua log variable") != 0) {
+ msg_err_config("error loading lua code: '%T': %s", content,
+ lua_tostring(RSPAMD_LUA_CFG_STATE(cfg), -1));
+ return FALSE;
+ }
+ if (lua_pcall(RSPAMD_LUA_CFG_STATE(cfg), 0, 1, 0) != 0) {
+ msg_err_config("error executing lua code: '%T': %s", content,
+ lua_tostring(RSPAMD_LUA_CFG_STATE(cfg), -1));
+ lua_pop(RSPAMD_LUA_CFG_STATE(cfg), 1);
+
+ return FALSE;
+ }
+
+ if (lua_type(RSPAMD_LUA_CFG_STATE(cfg), -1) != LUA_TFUNCTION) {
+ msg_err_config("lua variable should return function: %T", content);
+ lua_pop(RSPAMD_LUA_CFG_STATE(cfg), 1);
+ return FALSE;
+ }
+
+ auto id = luaL_ref(RSPAMD_LUA_CFG_STATE(cfg), LUA_REGISTRYINDEX);
+ log_format->data = GINT_TO_POINTER(id);
+ log_format->len = 0;
+ }
+
+ DL_APPEND(cfg->log_format, log_format);
+ }
+ else {
+ std::string known_formats;
+
+ for (const auto &v: config_vars) {
+ known_formats += std::string_view{v.first.data(), v.first.size()};
+ known_formats += ", ";
+ }
+
+ if (known_formats.size() > 2) {
+ // Remove last comma
+ known_formats.resize(known_formats.size() - 2);
+ }
+ msg_err_config("unknown log variable: %T, known vars are: \"%s\"", var, known_formats.c_str());
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_config_parse_log_format(struct rspamd_config *cfg)
+{
+ const gchar *p, *c, *end, *s;
+ gchar *d;
+ struct rspamd_log_format *lf = nullptr;
+ rspamd_ftok_t var, var_content;
+ enum {
+ parse_str,
+ parse_dollar,
+ parse_var_name,
+ parse_var_content,
+ } state = parse_str;
+ gint braces = 0;
+
+ g_assert(cfg != nullptr);
+ c = cfg->log_format_str;
+
+ if (c == nullptr) {
+ return FALSE;
+ }
+
+ p = c;
+ end = p + strlen(p);
+
+ while (p < end) {
+ switch (state) {
+ case parse_str:
+ if (*p == '$') {
+ state = parse_dollar;
+ }
+ else {
+ p++;
+ }
+ break;
+ case parse_dollar:
+ if (p > c) {
+ /* We have string element that we need to store */
+ lf = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_log_format);
+ lf->type = RSPAMD_LOG_STRING;
+ lf->data = rspamd_mempool_alloc(cfg->cfg_pool, p - c + 1);
+ /* Filter \r\n from the destination */
+ s = c;
+ d = (char *) lf->data;
+
+ while (s < p) {
+ if (*s != '\r' && *s != '\n') {
+ *d++ = *s++;
+ }
+ else {
+ *d++ = ' ';
+ s++;
+ }
+ }
+ *d = '\0';
+
+ lf->len = d - (char *) lf->data;
+ DL_APPEND(cfg->log_format, lf);
+ lf = nullptr;
+ }
+ p++;
+ c = p;
+ state = parse_var_name;
+ break;
+ case parse_var_name:
+ if (*p == '{') {
+ var.begin = c;
+ var.len = p - c;
+ p++;
+ c = p;
+ state = parse_var_content;
+ braces = 1;
+ }
+ else if (*p != '_' && *p != '-' && !g_ascii_isalnum(*p)) {
+ /* Variable with no content */
+ var.begin = c;
+ var.len = p - c;
+ c = p;
+
+ if (!rspamd_config_process_var(cfg, &var, nullptr)) {
+ return FALSE;
+ }
+
+ state = parse_str;
+ }
+ else {
+ p++;
+ }
+ break;
+ case parse_var_content:
+ if (*p == '}' && --braces == 0) {
+ var_content.begin = c;
+ var_content.len = p - c;
+ p++;
+ c = p;
+
+ if (!rspamd_config_process_var(cfg, &var, &var_content)) {
+ return FALSE;
+ }
+
+ state = parse_str;
+ }
+ else if (*p == '{') {
+ braces++;
+ p++;
+ }
+ else {
+ p++;
+ }
+ break;
+ }
+ }
+
+ /* Last state */
+ switch (state) {
+ case parse_str:
+ if (p > c) {
+ /* We have string element that we need to store */
+ lf = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_log_format);
+ lf->type = RSPAMD_LOG_STRING;
+ lf->data = rspamd_mempool_alloc(cfg->cfg_pool, p - c + 1);
+ /* Filter \r\n from the destination */
+ s = c;
+ d = (char *) lf->data;
+
+ while (s < p) {
+ if (*s != '\r' && *s != '\n') {
+ *d++ = *s++;
+ }
+ else {
+ *d++ = ' ';
+ s++;
+ }
+ }
+ *d = '\0';
+
+ lf->len = d - (char *) lf->data;
+ DL_APPEND(cfg->log_format, lf);
+ lf = nullptr;
+ }
+ break;
+
+ case parse_var_name:
+ var.begin = c;
+ var.len = p - c;
+
+ if (!rspamd_config_process_var(cfg, &var, nullptr)) {
+ return FALSE;
+ }
+ break;
+ case parse_dollar:
+ case parse_var_content:
+ msg_err_config("cannot parse log format %s: incomplete string",
+ cfg->log_format_str);
+ return FALSE;
+ break;
+ }
+
+ return TRUE;
+}
+
+static void
+rspamd_urls_config_dtor(gpointer _unused)
+{
+ rspamd_url_deinit();
+}
+
+static void
+rspamd_adjust_clocks_resolution(struct rspamd_config *cfg)
+{
+#ifdef HAVE_CLOCK_GETTIME
+ struct timespec ts;
+#endif
+
+#ifdef HAVE_CLOCK_GETTIME
+#ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
+ clock_getres(CLOCK_PROCESS_CPUTIME_ID, &ts);
+#elif defined(HAVE_CLOCK_VIRTUAL)
+ clock_getres(CLOCK_VIRTUAL, &ts);
+#else
+ clock_getres(CLOCK_REALTIME, &ts);
+#endif
+ cfg->clock_res = log10(1000000. / ts.tv_nsec);
+ if (cfg->clock_res < 0) {
+ cfg->clock_res = 0;
+ }
+ if (cfg->clock_res > 3) {
+ cfg->clock_res = 3;
+ }
+#else
+ /* For gettimeofday */
+ cfg->clock_res = 1;
+#endif
+}
+
+/*
+ * Perform post load actions
+ */
+gboolean
+rspamd_config_post_load(struct rspamd_config *cfg,
+ enum rspamd_post_load_options opts)
+{
+
+ auto ret = TRUE;
+
+ rspamd_adjust_clocks_resolution(cfg);
+ rspamd_logger_configure_modules(cfg->debug_modules);
+
+ if (cfg->one_shot_mode) {
+ msg_info_config("enabling one shot mode (was %d max shots)",
+ cfg->default_max_shots);
+ cfg->default_max_shots = 1;
+ }
+
+#if defined(WITH_HYPERSCAN) && !defined(__aarch64__) && !defined(__powerpc64__)
+ if (!cfg->disable_hyperscan) {
+ if (!(cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) {
+ msg_warn_config("CPU doesn't have SSSE3 instructions set "
+ "required for hyperscan, disable it");
+ cfg->disable_hyperscan = TRUE;
+ }
+ }
+#endif
+
+ rspamd_regexp_library_init(cfg);
+ rspamd_multipattern_library_init(cfg->hs_cache_dir);
+
+ if (opts & RSPAMD_CONFIG_INIT_URL) {
+ if (cfg->tld_file == nullptr) {
+ /* Try to guess tld file */
+ auto fpath = fmt::format("{0}{1}{2}", RSPAMD_SHAREDIR,
+ G_DIR_SEPARATOR, "effective_tld_names.dat");
+
+ if (access(fpath.c_str(), R_OK) != -1) {
+ msg_debug_config("url_tld option is not specified but %s is available,"
+ " therefore this file is assumed as TLD file for URL"
+ " extraction",
+ fpath.c_str());
+ cfg->tld_file = rspamd_mempool_strdup(cfg->cfg_pool, fpath.c_str());
+ }
+ else {
+ if (opts & RSPAMD_CONFIG_INIT_VALIDATE) {
+ msg_err_config("no url_tld option has been specified");
+ ret = FALSE;
+ }
+ }
+ }
+ else {
+ if (access(cfg->tld_file, R_OK) == -1) {
+ if (opts & RSPAMD_CONFIG_INIT_VALIDATE) {
+ ret = FALSE;
+ msg_err_config("cannot access tld file %s: %s", cfg->tld_file,
+ strerror(errno));
+ }
+ else {
+ msg_debug_config("cannot access tld file %s: %s", cfg->tld_file,
+ strerror(errno));
+ cfg->tld_file = nullptr;
+ }
+ }
+ }
+
+ if (opts & RSPAMD_CONFIG_INIT_NO_TLD) {
+ rspamd_url_init(nullptr);
+ }
+ else {
+ rspamd_url_init(cfg->tld_file);
+ }
+
+ rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_urls_config_dtor,
+ nullptr);
+ }
+
+ init_dynamic_config(cfg);
+ /* Insert classifiers symbols */
+ rspamd_config_insert_classify_symbols(cfg);
+
+ /* Parse format string that we have */
+ if (!rspamd_config_parse_log_format(cfg)) {
+ msg_err_config("cannot parse log format, task logging will not be available");
+ if (opts & RSPAMD_CONFIG_INIT_VALIDATE) {
+ ret = FALSE;
+ }
+ }
+
+ if (opts & RSPAMD_CONFIG_INIT_SYMCACHE) {
+ /* Init config cache */
+ ret = rspamd_symcache_init(cfg->cache) && ret;
+
+ /* Init re cache */
+ rspamd_re_cache_init(cfg->re_cache, cfg);
+
+ /* Try load Hypersan */
+ auto hs_ret = rspamd_re_cache_load_hyperscan(cfg->re_cache,
+ cfg->hs_cache_dir ? cfg->hs_cache_dir : RSPAMD_DBDIR "/",
+ true);
+
+ if (hs_ret == RSPAMD_HYPERSCAN_LOAD_ERROR) {
+ msg_debug_config("cannot load hyperscan database, disable it");
+ }
+ }
+
+ if (opts & RSPAMD_CONFIG_INIT_LIBS) {
+ /* Config other libraries */
+ ret = rspamd_config_libs(cfg->libs_ctx, cfg) && ret;
+
+ if (!ret) {
+ msg_err_config("cannot configure libraries, fatal error");
+ return FALSE;
+ }
+ }
+
+ /* Validate cache */
+ if (opts & RSPAMD_CONFIG_INIT_VALIDATE) {
+ /* Check for actions sanity */
+ auto seen_controller = FALSE;
+
+ auto *cur = cfg->workers;
+ while (cur) {
+ auto *wcf = (struct rspamd_worker_conf *) cur->data;
+
+ if (wcf->type == g_quark_from_static_string("controller")) {
+ seen_controller = TRUE;
+ break;
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ if (!seen_controller) {
+ msg_warn_config("controller worker is unconfigured: learning,"
+ " periodic scripts, maps watching and many other"
+ " Rspamd features will be broken");
+ }
+
+ ret = rspamd_symcache_validate(cfg->cache, cfg, FALSE) && ret;
+ }
+
+ if (opts & RSPAMD_CONFIG_INIT_POST_LOAD_LUA) {
+ rspamd_lua_run_config_post_init(RSPAMD_LUA_CFG_STATE(cfg), cfg);
+ }
+
+ if (opts & RSPAMD_CONFIG_INIT_PRELOAD_MAPS) {
+ rspamd_map_preload(cfg);
+ }
+
+ return ret;
+}
+
+struct rspamd_classifier_config *
+rspamd_config_new_classifier(struct rspamd_config *cfg,
+ struct rspamd_classifier_config *c)
+{
+ if (c == nullptr) {
+ c =
+ rspamd_mempool_alloc0_type(cfg->cfg_pool,
+ struct rspamd_classifier_config);
+ c->min_prob_strength = 0.05;
+ c->min_token_hits = 2;
+ }
+
+ if (c->labels == nullptr) {
+ c->labels = g_hash_table_new_full(rspamd_str_hash,
+ rspamd_str_equal,
+ nullptr,
+ (GDestroyNotify) g_list_free);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) g_hash_table_destroy,
+ c->labels);
+ }
+
+ return c;
+}
+
+struct rspamd_statfile_config *
+rspamd_config_new_statfile(struct rspamd_config *cfg,
+ struct rspamd_statfile_config *c)
+{
+ if (c == nullptr) {
+ c =
+ rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_statfile_config);
+ }
+
+ return c;
+}
+
+void rspamd_config_init_metric(struct rspamd_config *cfg)
+{
+ cfg->grow_factor = 1.0;
+ cfg->symbols = g_hash_table_new(rspamd_str_hash, rspamd_str_equal);
+ cfg->groups = g_hash_table_new(rspamd_strcase_hash, rspamd_strcase_equal);
+
+ cfg->subject = SPAM_SUBJECT;
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) g_hash_table_unref,
+ cfg->symbols);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) g_hash_table_unref,
+ cfg->groups);
+}
+
+struct rspamd_symbols_group *
+rspamd_config_new_group(struct rspamd_config *cfg, const gchar *name)
+{
+ struct rspamd_symbols_group *gr;
+
+ gr = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_symbols_group);
+ gr->symbols = g_hash_table_new(rspamd_strcase_hash,
+ rspamd_strcase_equal);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) g_hash_table_unref, gr->symbols);
+ gr->name = rspamd_mempool_strdup(cfg->cfg_pool, name);
+
+ if (strcmp(gr->name, "ungrouped") == 0) {
+ gr->flags |= RSPAMD_SYMBOL_GROUP_UNGROUPED;
+ }
+
+ g_hash_table_insert(cfg->groups, gr->name, gr);
+
+ return gr;
+}
+
+static void
+rspamd_worker_conf_dtor(struct rspamd_worker_conf *wcf)
+{
+ if (wcf) {
+ ucl_object_unref(wcf->options);
+ g_queue_free(wcf->active_workers);
+ g_hash_table_unref(wcf->params);
+ g_free(wcf);
+ }
+}
+
+static void
+rspamd_worker_conf_cfg_fin(gpointer d)
+{
+ auto *wcf = (struct rspamd_worker_conf *) d;
+
+ REF_RELEASE(wcf);
+}
+
+struct rspamd_worker_conf *
+rspamd_config_new_worker(struct rspamd_config *cfg,
+ struct rspamd_worker_conf *c)
+{
+ if (c == nullptr) {
+ c = g_new0(struct rspamd_worker_conf, 1);
+ c->params = g_hash_table_new(rspamd_str_hash, rspamd_str_equal);
+ c->active_workers = g_queue_new();
+#ifdef HAVE_SC_NPROCESSORS_ONLN
+ auto nproc = sysconf(_SC_NPROCESSORS_ONLN);
+ c->count = MIN(DEFAULT_MAX_WORKERS, MAX(1, nproc - 2));
+#else
+ c->count = DEFAULT_MAX_WORKERS;
+#endif
+ c->rlimit_nofile = 0;
+ c->rlimit_maxcore = 0;
+ c->enabled = TRUE;
+
+ REF_INIT_RETAIN(c, rspamd_worker_conf_dtor);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ rspamd_worker_conf_cfg_fin, c);
+ }
+
+ return c;
+}
+
+
+static bool
+rspamd_include_map_handler(const guchar *data, gsize len,
+ const ucl_object_t *args, void *ud)
+{
+ auto *cfg = (struct rspamd_config *) ud;
+
+ auto ftok = rspamd_ftok_t{.len = len + 1, .begin = (char *) data};
+ auto *map_line = rspamd_mempool_ftokdup(cfg->cfg_pool, &ftok);
+
+ auto *cbdata = new rspamd_ucl_map_cbdata{cfg};
+ auto **pcbdata = new rspamd_ucl_map_cbdata *(cbdata);
+
+ return rspamd_map_add(cfg,
+ map_line,
+ "ucl include",
+ rspamd_ucl_read_cb,
+ rspamd_ucl_fin_cb,
+ rspamd_ucl_dtor_cb,
+ (void **) pcbdata,
+ nullptr, RSPAMD_MAP_DEFAULT) != nullptr;
+}
+
+/*
+ * Variables:
+ * $CONFDIR - configuration directory
+ * $LOCAL_CONFDIR - local configuration directory
+ * $RUNDIR - local states directory
+ * $DBDIR - databases dir
+ * $LOGDIR - logs dir
+ * $PLUGINSDIR - plugins dir
+ * $PREFIX - installation prefix
+ * $VERSION - rspamd version
+ */
+
+#define RSPAMD_CONFDIR_MACRO "CONFDIR"
+#define RSPAMD_LOCAL_CONFDIR_MACRO "LOCAL_CONFDIR"
+#define RSPAMD_RUNDIR_MACRO "RUNDIR"
+#define RSPAMD_DBDIR_MACRO "DBDIR"
+#define RSPAMD_LOGDIR_MACRO "LOGDIR"
+#define RSPAMD_PLUGINSDIR_MACRO "PLUGINSDIR"
+#define RSPAMD_SHAREDIR_MACRO "SHAREDIR"
+#define RSPAMD_RULESDIR_MACRO "RULESDIR"
+#define RSPAMD_WWWDIR_MACRO "WWWDIR"
+#define RSPAMD_PREFIX_MACRO "PREFIX"
+#define RSPAMD_VERSION_MACRO "VERSION"
+#define RSPAMD_VERSION_MAJOR_MACRO "VERSION_MAJOR"
+#define RSPAMD_VERSION_MINOR_MACRO "VERSION_MINOR"
+#define RSPAMD_BRANCH_VERSION_MACRO "BRANCH_VERSION"
+#define RSPAMD_HOSTNAME_MACRO "HOSTNAME"
+
+void rspamd_ucl_add_conf_variables(struct ucl_parser *parser, GHashTable *vars)
+{
+ GHashTableIter it;
+ gpointer k, v;
+
+ ucl_parser_register_variable(parser,
+ RSPAMD_CONFDIR_MACRO,
+ RSPAMD_CONFDIR);
+ ucl_parser_register_variable(parser,
+ RSPAMD_LOCAL_CONFDIR_MACRO,
+ RSPAMD_LOCAL_CONFDIR);
+ ucl_parser_register_variable(parser, RSPAMD_RUNDIR_MACRO,
+ RSPAMD_RUNDIR);
+ ucl_parser_register_variable(parser, RSPAMD_DBDIR_MACRO,
+ RSPAMD_DBDIR);
+ ucl_parser_register_variable(parser, RSPAMD_LOGDIR_MACRO,
+ RSPAMD_LOGDIR);
+ ucl_parser_register_variable(parser,
+ RSPAMD_PLUGINSDIR_MACRO,
+ RSPAMD_PLUGINSDIR);
+ ucl_parser_register_variable(parser,
+ RSPAMD_SHAREDIR_MACRO,
+ RSPAMD_SHAREDIR);
+ ucl_parser_register_variable(parser,
+ RSPAMD_RULESDIR_MACRO,
+ RSPAMD_RULESDIR);
+ ucl_parser_register_variable(parser, RSPAMD_WWWDIR_MACRO,
+ RSPAMD_WWWDIR);
+ ucl_parser_register_variable(parser, RSPAMD_PREFIX_MACRO,
+ RSPAMD_PREFIX);
+ ucl_parser_register_variable(parser, RSPAMD_VERSION_MACRO, RVERSION);
+ ucl_parser_register_variable(parser, RSPAMD_VERSION_MAJOR_MACRO,
+ RSPAMD_VERSION_MAJOR);
+ ucl_parser_register_variable(parser, RSPAMD_VERSION_MINOR_MACRO,
+ RSPAMD_VERSION_MINOR);
+ ucl_parser_register_variable(parser, RSPAMD_BRANCH_VERSION_MACRO,
+ RSPAMD_VERSION_BRANCH);
+
+ auto hostlen = sysconf(_SC_HOST_NAME_MAX);
+
+ if (hostlen <= 0) {
+ hostlen = 256;
+ }
+ else {
+ hostlen++;
+ }
+
+ auto hostbuf = std::string{};
+ hostbuf.resize(hostlen);
+
+ if (gethostname(hostbuf.data(), hostlen) != 0) {
+ hostbuf = "unknown";
+ }
+
+ /* UCL copies variables, so it is safe to pass an ephemeral buffer here */
+ ucl_parser_register_variable(parser, RSPAMD_HOSTNAME_MACRO,
+ hostbuf.c_str());
+
+ if (vars != nullptr) {
+ g_hash_table_iter_init(&it, vars);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ ucl_parser_register_variable(parser, (const char *) k, (const char *) v);
+ }
+ }
+}
+
+void rspamd_ucl_add_conf_macros(struct ucl_parser *parser,
+ struct rspamd_config *cfg)
+{
+ ucl_parser_register_macro(parser,
+ "include_map",
+ rspamd_include_map_handler,
+ cfg);
+}
+
+static void
+symbols_classifiers_callback(gpointer key, gpointer value, gpointer ud)
+{
+ auto *cfg = (struct rspamd_config *) ud;
+
+ /* Actually, statistics should act like any ordinary symbol */
+ rspamd_symcache_add_symbol(cfg->cache, (const char *) key, 0, nullptr, nullptr,
+ SYMBOL_TYPE_CLASSIFIER | SYMBOL_TYPE_NOSTAT, -1);
+}
+
+void rspamd_config_insert_classify_symbols(struct rspamd_config *cfg)
+{
+ g_hash_table_foreach(cfg->classifiers_symbols,
+ symbols_classifiers_callback,
+ cfg);
+}
+
+struct rspamd_classifier_config *
+rspamd_config_find_classifier(struct rspamd_config *cfg, const gchar *name)
+{
+ if (name == nullptr) {
+ return nullptr;
+ }
+
+ auto *cur = cfg->classifiers;
+ while (cur) {
+ auto *cf = (struct rspamd_classifier_config *) cur->data;
+
+ if (g_ascii_strcasecmp(cf->name, name) == 0) {
+ return cf;
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ return nullptr;
+}
+
+gboolean
+rspamd_config_check_statfiles(struct rspamd_classifier_config *cf)
+{
+ gboolean has_other = FALSE, res = FALSE, cur_class = FALSE;
+
+ /* First check classes directly */
+ auto *cur = cf->statfiles;
+ while (cur) {
+ auto *st = (struct rspamd_statfile_config *) cur->data;
+ if (!has_other) {
+ cur_class = st->is_spam;
+ has_other = TRUE;
+ }
+ else {
+ if (cur_class != st->is_spam) {
+ return TRUE;
+ }
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ if (!has_other) {
+ /* We have only one statfile */
+ return FALSE;
+ }
+ /* We have not detected any statfile that has different class, so turn on heuristic based on symbol's name */
+ has_other = FALSE;
+ cur = cf->statfiles;
+ while (cur) {
+ auto *st = (struct rspamd_statfile_config *) cur->data;
+ if (rspamd_substring_search_caseless(st->symbol,
+ strlen(st->symbol), "spam", 4) != -1) {
+ st->is_spam = TRUE;
+ }
+ else if (rspamd_substring_search_caseless(st->symbol,
+ strlen(st->symbol), "ham", 3) != -1) {
+ st->is_spam = FALSE;
+ }
+
+ if (!has_other) {
+ cur_class = st->is_spam;
+ has_other = TRUE;
+ }
+ else {
+ if (cur_class != st->is_spam) {
+ res = TRUE;
+ }
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ return res;
+}
+
+static gchar *
+rspamd_ucl_read_cb(gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ auto *cbdata = (struct rspamd_ucl_map_cbdata *) data->cur_data;
+ auto *prev = (struct rspamd_ucl_map_cbdata *) data->prev_data;
+
+ if (cbdata == nullptr) {
+ cbdata = new rspamd_ucl_map_cbdata{prev->cfg};
+ data->cur_data = cbdata;
+ }
+ cbdata->buf.append(chunk, len);
+
+ /* Say not to copy any part of this buffer */
+ return nullptr;
+}
+
+static void
+rspamd_ucl_fin_cb(struct map_cb_data *data, void **target)
+{
+ auto *cbdata = (struct rspamd_ucl_map_cbdata *) data->cur_data;
+ auto *prev = (struct rspamd_ucl_map_cbdata *) data->prev_data;
+ auto *cfg = data->map->cfg;
+
+ if (cbdata == nullptr) {
+ msg_err_config("map fin error: new data is nullptr");
+ return;
+ }
+
+ /* New data available */
+ auto *parser = ucl_parser_new(0);
+ if (!ucl_parser_add_chunk(parser, (unsigned char *) cbdata->buf.data(),
+ cbdata->buf.size())) {
+ msg_err_config("cannot parse map %s: %s",
+ data->map->name,
+ ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+ }
+ else {
+ auto *obj = ucl_parser_get_object(parser);
+ ucl_object_iter_t it = nullptr;
+
+ for (auto *cur = ucl_object_iterate(obj, &it, true); cur != nullptr; cur = ucl_object_iterate(obj, &it, true)) {
+ ucl_object_replace_key(cbdata->cfg->cfg_ucl_obj, (ucl_object_t *) cur,
+ cur->key, cur->keylen, false);
+ }
+
+ ucl_parser_free(parser);
+ ucl_object_unref(obj);
+ }
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ delete prev;
+}
+
+static void
+rspamd_ucl_dtor_cb(struct map_cb_data *data)
+{
+ auto *cbdata = (struct rspamd_ucl_map_cbdata *) data->cur_data;
+
+ delete cbdata;
+}
+
+gboolean
+rspamd_check_module(struct rspamd_config *cfg, module_t *mod)
+{
+ gboolean ret = TRUE;
+
+ if (mod != nullptr) {
+ if (mod->module_version != RSPAMD_CUR_MODULE_VERSION) {
+ msg_err_config("module %s has incorrect version %xd (%xd expected)",
+ mod->name, (gint) mod->module_version, RSPAMD_CUR_MODULE_VERSION);
+ ret = FALSE;
+ }
+ if (ret && mod->rspamd_version != RSPAMD_VERSION_NUM) {
+ msg_err_config("module %s has incorrect rspamd version %xL (%xL expected)",
+ mod->name, mod->rspamd_version, RSPAMD_VERSION_NUM);
+ ret = FALSE;
+ }
+ if (ret && strcmp(mod->rspamd_features, RSPAMD_FEATURES) != 0) {
+ msg_err_config("module %s has incorrect rspamd features '%s' ('%s' expected)",
+ mod->name, mod->rspamd_features, RSPAMD_FEATURES);
+ ret = FALSE;
+ }
+ }
+ else {
+ ret = FALSE;
+ }
+
+ return ret;
+}
+
+gboolean
+rspamd_check_worker(struct rspamd_config *cfg, worker_t *wrk)
+{
+ gboolean ret = TRUE;
+
+ if (wrk != nullptr) {
+ if (wrk->worker_version != RSPAMD_CUR_WORKER_VERSION) {
+ msg_err_config("worker %s has incorrect version %xd (%xd expected)",
+ wrk->name, wrk->worker_version, RSPAMD_CUR_WORKER_VERSION);
+ ret = FALSE;
+ }
+ if (ret && wrk->rspamd_version != RSPAMD_VERSION_NUM) {
+ msg_err_config("worker %s has incorrect rspamd version %xL (%xL expected)",
+ wrk->name, wrk->rspamd_version, RSPAMD_VERSION_NUM);
+ ret = FALSE;
+ }
+ if (ret && strcmp(wrk->rspamd_features, RSPAMD_FEATURES) != 0) {
+ msg_err_config("worker %s has incorrect rspamd features '%s' ('%s' expected)",
+ wrk->name, wrk->rspamd_features, RSPAMD_FEATURES);
+ ret = FALSE;
+ }
+ }
+ else {
+ ret = FALSE;
+ }
+
+ return ret;
+}
+
+gboolean
+rspamd_init_filters(struct rspamd_config *cfg, bool reconfig, bool strict)
+{
+ GList *cur;
+ module_t *mod, **pmod;
+ guint i = 0;
+ struct module_ctx *mod_ctx, *cur_ctx;
+ gboolean ret = TRUE;
+
+ /* Init all compiled modules */
+
+ for (pmod = cfg->compiled_modules; pmod != nullptr && *pmod != nullptr; pmod++) {
+ mod = *pmod;
+ if (rspamd_check_module(cfg, mod)) {
+ if (mod->module_init_func(cfg, &mod_ctx) == 0) {
+ g_assert(mod_ctx != nullptr);
+ g_ptr_array_add(cfg->c_modules, mod_ctx);
+ mod_ctx->mod = mod;
+ mod->ctx_offset = i++;
+ }
+ }
+ }
+
+ /* Now check what's enabled */
+ cur = g_list_first(cfg->filters);
+
+ while (cur) {
+ /* Perform modules configuring */
+ mod_ctx = nullptr;
+ PTR_ARRAY_FOREACH(cfg->c_modules, i, cur_ctx)
+ {
+ if (g_ascii_strcasecmp(cur_ctx->mod->name,
+ (const gchar *) cur->data) == 0) {
+ mod_ctx = cur_ctx;
+ break;
+ }
+ }
+
+ if (mod_ctx) {
+ mod = mod_ctx->mod;
+ mod_ctx->enabled = rspamd_config_is_module_enabled(cfg, mod->name);
+
+ if (reconfig) {
+ if (!mod->module_reconfig_func(cfg)) {
+ msg_err_config("reconfig of %s failed!", mod->name);
+ }
+ else {
+ msg_info_config("reconfig of %s", mod->name);
+ }
+ }
+ else {
+ if (!mod->module_config_func(cfg, strict)) {
+ msg_err_config("config of %s failed", mod->name);
+ ret = FALSE;
+
+ if (strict) {
+ return FALSE;
+ }
+ }
+ }
+ }
+
+ if (mod_ctx == nullptr) {
+ msg_warn_config("requested unknown module %s", cur->data);
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ ret = rspamd_init_lua_filters(cfg, 0, strict) && ret;
+
+ return ret;
+}
+
+static void
+rspamd_config_new_symbol(struct rspamd_config *cfg, const gchar *symbol,
+ gdouble score, const gchar *description, const gchar *group,
+ guint flags, guint priority, gint nshots)
+{
+ struct rspamd_symbols_group *sym_group;
+ struct rspamd_symbol *sym_def;
+ double *score_ptr;
+
+ sym_def =
+ rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_symbol);
+ score_ptr = rspamd_mempool_alloc_type(cfg->cfg_pool, double);
+
+ if (isnan(score)) {
+ /* In fact, it could be defined later */
+ msg_debug_config("score is not defined for symbol %s, set it to zero",
+ symbol);
+ score = 0.0;
+ /* Also set priority to 0 to allow override by anything */
+ sym_def->priority = 0;
+ flags |= RSPAMD_SYMBOL_FLAG_UNSCORED;
+ }
+ else {
+ sym_def->priority = priority;
+ }
+
+ *score_ptr = score;
+ sym_def->score = score;
+ sym_def->weight_ptr = score_ptr;
+ sym_def->name = rspamd_mempool_strdup(cfg->cfg_pool, symbol);
+ sym_def->flags = flags;
+ sym_def->nshots = nshots != 0 ? nshots : cfg->default_max_shots;
+ sym_def->groups = g_ptr_array_sized_new(1);
+ rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard,
+ sym_def->groups);
+
+ if (description) {
+ sym_def->description = rspamd_mempool_strdup(cfg->cfg_pool, description);
+ }
+
+ msg_debug_config("registered symbol %s with weight %.2f in and group %s",
+ sym_def->name, score, group);
+
+ g_hash_table_insert(cfg->symbols, sym_def->name, sym_def);
+
+ /* Search for symbol group */
+ if (group == nullptr) {
+ group = "ungrouped";
+ sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPED;
+ }
+ else {
+ if (strcmp(group, "ungrouped") == 0) {
+ sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPED;
+ }
+ }
+
+ sym_group = reinterpret_cast<rspamd_symbols_group *>(g_hash_table_lookup(cfg->groups, group));
+ if (sym_group == nullptr) {
+ /* Create new group */
+ sym_group = rspamd_config_new_group(cfg, group);
+ }
+
+ sym_def->gr = sym_group;
+ g_hash_table_insert(sym_group->symbols, sym_def->name, sym_def);
+
+ if (!(sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED)) {
+ g_ptr_array_add(sym_def->groups, sym_group);
+ }
+}
+
+
+gboolean
+rspamd_config_add_symbol(struct rspamd_config *cfg,
+ const gchar *symbol,
+ gdouble score,
+ const gchar *description,
+ const gchar *group,
+ guint flags,
+ guint priority,
+ gint nshots)
+{
+ struct rspamd_symbol *sym_def;
+ struct rspamd_symbols_group *sym_group;
+ guint i;
+
+ g_assert(cfg != nullptr);
+ g_assert(symbol != nullptr);
+
+ sym_def = reinterpret_cast<rspamd_symbol *>(g_hash_table_lookup(cfg->symbols, symbol));
+
+ if (sym_def != nullptr) {
+ if (group != nullptr) {
+ gboolean has_group = FALSE;
+
+ PTR_ARRAY_FOREACH(sym_def->groups, i, sym_group)
+ {
+ if (g_ascii_strcasecmp(sym_group->name, group) == 0) {
+ /* Group is already here */
+ has_group = TRUE;
+ break;
+ }
+ }
+
+ if (!has_group) {
+ /* Non-empty group has a priority over non-grouped one */
+ sym_group = reinterpret_cast<rspamd_symbols_group *>(g_hash_table_lookup(cfg->groups, group));
+
+ if (sym_group == nullptr) {
+ /* Create new group */
+ sym_group = rspamd_config_new_group(cfg, group);
+ }
+
+ if ((!sym_def->gr) || (sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED)) {
+ sym_def->gr = sym_group;
+ sym_def->flags &= ~RSPAMD_SYMBOL_FLAG_UNGROUPED;
+ }
+
+ g_hash_table_insert(sym_group->symbols, sym_def->name, sym_def);
+ sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPED);
+ g_ptr_array_add(sym_def->groups, sym_group);
+ }
+ }
+
+ if (sym_def->priority > priority &&
+ (isnan(score) || !(sym_def->flags & RSPAMD_SYMBOL_FLAG_UNSCORED))) {
+ msg_debug_config("symbol %s has been already registered with "
+ "priority %ud, do not override (new priority: %ud)",
+ symbol,
+ sym_def->priority,
+ priority);
+ /* But we can still add description */
+ if (!sym_def->description && description) {
+ sym_def->description = rspamd_mempool_strdup(cfg->cfg_pool,
+ description);
+ }
+
+ /* Or nshots in case of non-default setting */
+ if (nshots != 0 && sym_def->nshots == cfg->default_max_shots) {
+ sym_def->nshots = nshots;
+ }
+
+ return FALSE;
+ }
+ else {
+
+ if (!isnan(score)) {
+ msg_debug_config("symbol %s has been already registered with "
+ "priority %ud, override it with new priority: %ud, "
+ "old score: %.2f, new score: %.2f",
+ symbol,
+ sym_def->priority,
+ priority,
+ sym_def->score,
+ score);
+
+ *sym_def->weight_ptr = score;
+ sym_def->score = score;
+ sym_def->priority = priority;
+ sym_def->flags &= ~RSPAMD_SYMBOL_FLAG_UNSCORED;
+ }
+
+ sym_def->flags = flags;
+
+ if (nshots != 0) {
+ sym_def->nshots = nshots;
+ }
+ else {
+ /* Do not reset unless we have exactly lower priority */
+ if (sym_def->priority < priority) {
+ sym_def->nshots = cfg->default_max_shots;
+ }
+ }
+
+ if (description) {
+ sym_def->description = rspamd_mempool_strdup(cfg->cfg_pool,
+ description);
+ }
+
+
+ /* We also check group information in this case */
+ if (group != nullptr && sym_def->gr != nullptr &&
+ strcmp(group, sym_def->gr->name) != 0) {
+
+ sym_group = reinterpret_cast<rspamd_symbols_group *>(g_hash_table_lookup(cfg->groups, group));
+
+ if (sym_group == nullptr) {
+ /* Create new group */
+ sym_group = rspamd_config_new_group(cfg, group);
+ }
+
+ if (!(sym_group->flags & RSPAMD_SYMBOL_GROUP_UNGROUPED)) {
+ msg_debug_config("move symbol %s from group %s to %s",
+ sym_def->name, sym_def->gr->name, group);
+ g_hash_table_remove(sym_def->gr->symbols, sym_def->name);
+ sym_def->gr = sym_group;
+ g_hash_table_insert(sym_group->symbols, sym_def->name, sym_def);
+ }
+ }
+
+ return TRUE;
+ }
+ }
+
+ /* This is called merely when we have an undefined symbol */
+ rspamd_config_new_symbol(cfg, symbol, score, description,
+ group, flags, priority, nshots);
+
+ return TRUE;
+}
+
+gboolean
+rspamd_config_add_symbol_group(struct rspamd_config *cfg,
+ const gchar *symbol,
+ const gchar *group)
+{
+ struct rspamd_symbol *sym_def;
+ struct rspamd_symbols_group *sym_group;
+ guint i;
+
+ g_assert(cfg != nullptr);
+ g_assert(symbol != nullptr);
+ g_assert(group != nullptr);
+
+ sym_def = reinterpret_cast<rspamd_symbol *>(g_hash_table_lookup(cfg->symbols, symbol));
+
+ if (sym_def != nullptr) {
+ gboolean has_group = FALSE;
+
+ PTR_ARRAY_FOREACH(sym_def->groups, i, sym_group)
+ {
+ if (g_ascii_strcasecmp(sym_group->name, group) == 0) {
+ /* Group is already here */
+ has_group = TRUE;
+ break;
+ }
+ }
+
+ if (!has_group) {
+ /* Non-empty group has a priority over non-grouped one */
+ sym_group = reinterpret_cast<rspamd_symbols_group *>(g_hash_table_lookup(cfg->groups, group));
+
+ if (sym_group == nullptr) {
+ /* Create new group */
+ sym_group = rspamd_config_new_group(cfg, group);
+ }
+
+ if (!sym_def->gr) {
+ sym_def->gr = sym_group;
+ }
+
+ g_hash_table_insert(sym_group->symbols, sym_def->name, sym_def);
+ sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPED);
+ g_ptr_array_add(sym_def->groups, sym_group);
+
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_config_is_enabled_from_ucl(rspamd_mempool_t *pool,
+ const ucl_object_t *obj)
+{
+
+ const ucl_object_t *enabled;
+
+ enabled = ucl_object_lookup(obj, "enabled");
+
+ if (enabled) {
+ if (ucl_object_type(enabled) == UCL_BOOLEAN) {
+ return ucl_object_toboolean(enabled);
+ }
+ else if (ucl_object_type(enabled) == UCL_STRING) {
+ gint ret = rspamd_config_parse_flag(ucl_object_tostring(enabled), 0);
+
+ if (ret == 0) {
+ return FALSE;
+ }
+ else if (ret == -1) {
+
+ msg_info_pool_check("wrong value for the `enabled` key");
+ return FALSE;
+ }
+ /* Default return is TRUE here */
+ }
+ }
+
+
+ const ucl_object_t *disabled;
+
+ disabled = ucl_object_lookup(obj, "disabled");
+
+ if (disabled) {
+ if (ucl_object_type(disabled) == UCL_BOOLEAN) {
+ return !ucl_object_toboolean(disabled);
+ }
+ else if (ucl_object_type(disabled) == UCL_STRING) {
+ gint ret = rspamd_config_parse_flag(ucl_object_tostring(disabled), 0);
+
+ if (ret == 0) {
+ return TRUE;
+ }
+ else if (ret == -1) {
+
+ msg_info_pool_check("wrong value for the `disabled` key");
+ return FALSE;
+ }
+
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_config_is_module_enabled(struct rspamd_config *cfg,
+ const gchar *module_name)
+{
+ gboolean is_c = FALSE, enabled;
+ const ucl_object_t *conf;
+ GList *cur;
+ struct rspamd_symbols_group *gr;
+ lua_State *L = RSPAMD_LUA_CFG_STATE(cfg);
+ struct module_ctx *cur_ctx;
+ guint i;
+
+ PTR_ARRAY_FOREACH(cfg->c_modules, i, cur_ctx)
+ {
+ if (g_ascii_strcasecmp(cur_ctx->mod->name, module_name) == 0) {
+ is_c = TRUE;
+ break;
+ }
+ }
+
+ if (g_hash_table_lookup(cfg->explicit_modules, module_name) != nullptr) {
+ /* Always load module */
+ rspamd_plugins_table_push_elt(L, "enabled", module_name);
+
+ return TRUE;
+ }
+
+ if (is_c) {
+ gboolean found = FALSE;
+
+ cur = g_list_first(cfg->filters);
+
+ while (cur) {
+ if (strcmp((char *) cur->data, module_name) == 0) {
+ found = TRUE;
+ break;
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ if (!found) {
+ msg_info_config("internal module %s is disable in `filters` line",
+ module_name);
+ rspamd_plugins_table_push_elt(L,
+ "disabled_explicitly", module_name);
+
+ return FALSE;
+ }
+ }
+
+ conf = ucl_object_lookup(cfg->cfg_ucl_obj, module_name);
+
+ if (conf == nullptr) {
+ rspamd_plugins_table_push_elt(L, "disabled_unconfigured", module_name);
+
+ msg_info_config("%s module %s is enabled but has not been configured",
+ is_c ? "internal" : "lua", module_name);
+
+ if (!is_c) {
+ msg_info_config("%s disabling unconfigured lua module", module_name);
+ return FALSE;
+ }
+ }
+ else {
+ enabled = rspamd_config_is_enabled_from_ucl(cfg->cfg_pool, conf);
+
+ if (!enabled) {
+ rspamd_plugins_table_push_elt(L,
+ "disabled_explicitly", module_name);
+
+ msg_info_config(
+ "%s module %s is disabled in the configuration",
+ is_c ? "internal" : "lua", module_name);
+ return FALSE;
+ }
+ }
+
+ /* Now we check symbols group */
+ gr = reinterpret_cast<rspamd_symbols_group *>(g_hash_table_lookup(cfg->groups, module_name));
+
+ if (gr) {
+ if (gr->flags & RSPAMD_SYMBOL_GROUP_DISABLED) {
+ rspamd_plugins_table_push_elt(L,
+ "disabled_explicitly", module_name);
+ msg_info_config("%s module %s is disabled in the configuration as "
+ "its group has been disabled",
+ is_c ? "internal" : "lua", module_name);
+
+ return FALSE;
+ }
+ }
+
+ rspamd_plugins_table_push_elt(L, "enabled", module_name);
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_config_action_from_ucl(struct rspamd_config *cfg,
+ struct rspamd_action *act,
+ const ucl_object_t *obj,
+ guint priority)
+{
+ auto threshold = NAN;
+ int flags = 0;
+
+ auto obj_type = ucl_object_type(obj);
+
+ if (obj_type == UCL_OBJECT) {
+ obj_type = ucl_object_type(obj);
+
+ const auto *elt = ucl_object_lookup_any(obj, "score", "threshold", nullptr);
+
+ if (elt) {
+ threshold = ucl_object_todouble(elt);
+ }
+
+ elt = ucl_object_lookup(obj, "flags");
+
+ if (elt && ucl_object_type(elt) == UCL_ARRAY) {
+ const ucl_object_t *cur;
+ ucl_object_iter_t it = nullptr;
+
+ while ((cur = ucl_object_iterate(elt, &it, true)) != nullptr) {
+ if (ucl_object_type(cur) == UCL_STRING) {
+ const gchar *fl_str = ucl_object_tostring(cur);
+
+ if (g_ascii_strcasecmp(fl_str, "no_threshold") == 0) {
+ flags |= RSPAMD_ACTION_NO_THRESHOLD;
+ }
+ else if (g_ascii_strcasecmp(fl_str, "threshold_only") == 0) {
+ flags |= RSPAMD_ACTION_THRESHOLD_ONLY;
+ }
+ else if (g_ascii_strcasecmp(fl_str, "ham") == 0) {
+ flags |= RSPAMD_ACTION_HAM;
+ }
+ else {
+ msg_warn_config("unknown action flag: %s", fl_str);
+ }
+ }
+ }
+ }
+
+ elt = ucl_object_lookup(obj, "milter");
+
+ if (elt) {
+ const gchar *milter_action = ucl_object_tostring(elt);
+
+ if (strcmp(milter_action, "discard") == 0) {
+ flags |= RSPAMD_ACTION_MILTER;
+ act->action_type = METRIC_ACTION_DISCARD;
+ }
+ else if (strcmp(milter_action, "quarantine") == 0) {
+ flags |= RSPAMD_ACTION_MILTER;
+ act->action_type = METRIC_ACTION_QUARANTINE;
+ }
+ else {
+ msg_warn_config("unknown milter action: %s", milter_action);
+ }
+ }
+ }
+ else if (obj_type == UCL_FLOAT || obj_type == UCL_INT) {
+ threshold = ucl_object_todouble(obj);
+ }
+
+ /* TODO: add lua references support */
+
+ if (isnan(threshold) && !(flags & RSPAMD_ACTION_NO_THRESHOLD)) {
+ msg_err_config("action %s has no threshold being set and it is not"
+ " a no threshold action",
+ act->name);
+
+ return FALSE;
+ }
+
+ act->threshold = threshold;
+ act->flags = flags;
+
+ enum rspamd_action_type std_act;
+
+ if (!(flags & RSPAMD_ACTION_MILTER)) {
+ if (rspamd_action_from_str(act->name, &std_act)) {
+ act->action_type = std_act;
+ }
+ else {
+ act->action_type = METRIC_ACTION_CUSTOM;
+ }
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_config_set_action_score(struct rspamd_config *cfg,
+ const gchar *action_name,
+ const ucl_object_t *obj)
+{
+ enum rspamd_action_type std_act;
+ const ucl_object_t *elt;
+ guint priority = ucl_object_get_priority(obj), obj_type;
+
+ g_assert(cfg != nullptr);
+ g_assert(action_name != nullptr);
+
+ obj_type = ucl_object_type(obj);
+
+ if (obj_type == UCL_OBJECT) {
+ elt = ucl_object_lookup(obj, "priority");
+
+ if (elt) {
+ priority = ucl_object_toint(elt);
+ }
+ }
+
+ /* Here are dragons:
+ * We have `canonical` name for actions, such as `soft reject` and
+ * configuration names for actions (used to be more convenient), such
+ * as `soft_reject`. Unfortunately, we must have heuristic for this
+ * variance of names.
+ */
+
+ if (rspamd_action_from_str(action_name, &std_act)) {
+ action_name = rspamd_action_to_str(std_act);
+ }
+
+ auto actions = RSPAMD_CFG_ACTIONS(cfg);
+ auto existing_act_it = actions->actions_by_name.find(action_name);
+
+ if (existing_act_it != actions->actions_by_name.end()) {
+ auto *act = existing_act_it->second.get();
+ /* Existing element */
+ if (act->priority <= priority) {
+ /* We can replace data */
+ auto old_pri = act->priority;
+ auto old_thr = act->threshold;
+
+ if (rspamd_config_action_from_ucl(cfg, act, obj, priority)) {
+ msg_info_config("action %s has been already registered with "
+ "priority %ud, override it with new priority: %ud, "
+ "old threshold: %.2f, new threshold: %.2f",
+ action_name,
+ old_pri,
+ priority,
+ old_thr,
+ act->threshold);
+ actions->sort();
+ }
+ else {
+ return FALSE;
+ }
+ }
+ else {
+ msg_info_config("action %s has been already registered with "
+ "priority %ud, do not override (new priority: %ud)",
+ action_name,
+ act->priority,
+ priority);
+ }
+ }
+ else {
+ /* Add new element */
+ auto act = std::make_shared<rspamd_action>();
+ act->name = rspamd_mempool_strdup(cfg->cfg_pool, action_name);
+
+ if (rspamd_config_action_from_ucl(cfg, act.get(), obj, priority)) {
+ actions->add_action(std::move(act));
+ }
+ else {
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_config_maybe_disable_action(struct rspamd_config *cfg,
+ const gchar *action_name,
+ guint priority)
+{
+ auto actions = RSPAMD_CFG_ACTIONS(cfg);
+ auto maybe_act = rspamd::find_map(actions->actions_by_name, action_name);
+
+ if (maybe_act) {
+ auto *act = maybe_act.value().get().get();
+ if (priority >= act->priority) {
+ msg_info_config("disable action %s; old priority: %ud, new priority: %ud",
+ action_name,
+ act->priority,
+ priority);
+
+ act->threshold = NAN;
+ act->priority = priority;
+ act->flags |= RSPAMD_ACTION_NO_THRESHOLD;
+
+ return TRUE;
+ }
+ else {
+ msg_info_config("action %s has been already registered with "
+ "priority %ud, cannot disable it with new priority: %ud",
+ action_name,
+ act->priority,
+ priority);
+ }
+ }
+
+ return FALSE;
+}
+
+struct rspamd_action *
+rspamd_config_get_action(struct rspamd_config *cfg, const gchar *name)
+{
+ auto actions = RSPAMD_CFG_ACTIONS(cfg);
+ auto maybe_act = rspamd::find_map(actions->actions_by_name, name);
+
+ if (maybe_act) {
+ return maybe_act.value().get().get();
+ }
+
+ return nullptr;
+}
+
+struct rspamd_action *
+rspamd_config_get_action_by_type(struct rspamd_config *cfg,
+ enum rspamd_action_type type)
+{
+ for (const auto &act: RSPAMD_CFG_ACTIONS(cfg)->actions) {
+ if (act->action_type == type) {
+ return act.get();
+ }
+ }
+
+ return nullptr;
+}
+
+void rspamd_config_actions_foreach(struct rspamd_config *cfg,
+ void (*func)(struct rspamd_action *act, void *d),
+ void *data)
+{
+ for (const auto &act: RSPAMD_CFG_ACTIONS(cfg)->actions) {
+ func(act.get(), data);
+ }
+}
+
+void rspamd_config_actions_foreach_enumerate(struct rspamd_config *cfg,
+ void (*func)(int idx, struct rspamd_action *act, void *d),
+ void *data)
+{
+ for (const auto &[idx, act]: rspamd::enumerate(RSPAMD_CFG_ACTIONS(cfg)->actions)) {
+ func(idx, act.get(), data);
+ }
+}
+
+gsize rspamd_config_actions_size(struct rspamd_config *cfg)
+{
+ return RSPAMD_CFG_ACTIONS(cfg)->actions.size();
+}
+
+gboolean
+rspamd_config_radix_from_ucl(struct rspamd_config *cfg, const ucl_object_t *obj, const gchar *description,
+ struct rspamd_radix_map_helper **target, GError **err,
+ struct rspamd_worker *worker, const gchar *map_name)
+{
+ ucl_type_t type;
+ ucl_object_iter_t it = nullptr;
+ const ucl_object_t *cur, *cur_elt;
+ const gchar *str;
+
+ /* Cleanup */
+ *target = nullptr;
+
+ LL_FOREACH(obj, cur_elt)
+ {
+ type = ucl_object_type(cur_elt);
+
+ switch (type) {
+ case UCL_STRING:
+ /* Either map or a list of IPs */
+ str = ucl_object_tostring(cur_elt);
+
+ if (rspamd_map_is_map(str)) {
+ if (rspamd_map_add_from_ucl(cfg, cur_elt,
+ description,
+ rspamd_radix_read,
+ rspamd_radix_fin,
+ rspamd_radix_dtor,
+ (void **) target,
+ worker, RSPAMD_MAP_DEFAULT) == nullptr) {
+ g_set_error(err,
+ g_quark_from_static_string("rspamd-config"),
+ EINVAL, "bad map definition %s for %s", str,
+ ucl_object_key(obj));
+ return FALSE;
+ }
+
+ return TRUE;
+ }
+ else {
+ /* Just a list */
+ if (!*target) {
+ *target = rspamd_map_helper_new_radix(
+ rspamd_map_add_fake(cfg, description, map_name));
+ }
+
+ rspamd_map_helper_insert_radix_resolve(*target, str, "");
+ }
+ break;
+ case UCL_OBJECT:
+ /* Should be a map description */
+ if (rspamd_map_add_from_ucl(cfg, cur_elt,
+ description,
+ rspamd_radix_read,
+ rspamd_radix_fin,
+ rspamd_radix_dtor,
+ (void **) target,
+ worker, RSPAMD_MAP_DEFAULT) == nullptr) {
+ g_set_error(err,
+ g_quark_from_static_string("rspamd-config"),
+ EINVAL, "bad map object for %s", ucl_object_key(obj));
+ return FALSE;
+ }
+
+ return TRUE;
+ break;
+ case UCL_ARRAY:
+ /* List of IP addresses */
+ it = ucl_object_iterate_new(cur_elt);
+
+ while ((cur = ucl_object_iterate_safe(it, true)) != nullptr) {
+
+
+ if (ucl_object_type(cur) == UCL_STRING) {
+ str = ucl_object_tostring(cur);
+ if (!*target) {
+ *target = rspamd_map_helper_new_radix(
+ rspamd_map_add_fake(cfg, description, map_name));
+ }
+
+ rspamd_map_helper_insert_radix_resolve(*target, str, "");
+ }
+ else {
+ g_set_error(err,
+ g_quark_from_static_string("rspamd-config"),
+ EINVAL, "bad element inside array object for %s: expected string, got: %s",
+ ucl_object_key(obj), ucl_object_type_to_string(ucl_object_type(cur)));
+ ucl_object_iterate_free(it);
+ return FALSE;
+ }
+ }
+
+ ucl_object_iterate_free(it);
+ break;
+ default:
+ g_set_error(err, g_quark_from_static_string("rspamd-config"),
+ EINVAL, "bad map type %s for %s",
+ ucl_object_type_to_string(type),
+ ucl_object_key(obj));
+ return FALSE;
+ }
+ }
+
+ /* Destroy on cfg cleanup */
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_map_helper_destroy_radix,
+ *target);
+
+ return TRUE;
+}
+
+constexpr const auto action_types = frozen::make_unordered_map<frozen::string, enum rspamd_action_type>({
+ {"reject", METRIC_ACTION_REJECT},
+ {"greylist", METRIC_ACTION_GREYLIST},
+ {"add header", METRIC_ACTION_ADD_HEADER},
+ {"add_header", METRIC_ACTION_ADD_HEADER},
+ {"rewrite subject", METRIC_ACTION_REWRITE_SUBJECT},
+ {"rewrite_subject", METRIC_ACTION_REWRITE_SUBJECT},
+ {"soft reject", METRIC_ACTION_SOFT_REJECT},
+ {"soft_reject", METRIC_ACTION_SOFT_REJECT},
+ {"no action", METRIC_ACTION_NOACTION},
+ {"no_action", METRIC_ACTION_NOACTION},
+ {"accept", METRIC_ACTION_NOACTION},
+ {"quarantine", METRIC_ACTION_QUARANTINE},
+ {"discard", METRIC_ACTION_DISCARD},
+
+});
+
+gboolean
+rspamd_action_from_str(const gchar *data, enum rspamd_action_type *result)
+{
+ auto maybe_action = rspamd::find_map(action_types, std::string_view{data});
+
+ if (maybe_action) {
+ *result = maybe_action.value().get();
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
+const gchar *
+rspamd_action_to_str(enum rspamd_action_type action)
+{
+ switch (action) {
+ case METRIC_ACTION_REJECT:
+ return "reject";
+ case METRIC_ACTION_SOFT_REJECT:
+ return "soft reject";
+ case METRIC_ACTION_REWRITE_SUBJECT:
+ return "rewrite subject";
+ case METRIC_ACTION_ADD_HEADER:
+ return "add header";
+ case METRIC_ACTION_GREYLIST:
+ return "greylist";
+ case METRIC_ACTION_NOACTION:
+ return "no action";
+ case METRIC_ACTION_MAX:
+ return "invalid max action";
+ case METRIC_ACTION_CUSTOM:
+ return "custom";
+ case METRIC_ACTION_DISCARD:
+ return "discard";
+ case METRIC_ACTION_QUARANTINE:
+ return "quarantine";
+ }
+
+ return "unknown action";
+}
+
+const gchar *
+rspamd_action_to_str_alt(enum rspamd_action_type action)
+{
+ switch (action) {
+ case METRIC_ACTION_REJECT:
+ return "reject";
+ case METRIC_ACTION_SOFT_REJECT:
+ return "soft_reject";
+ case METRIC_ACTION_REWRITE_SUBJECT:
+ return "rewrite_subject";
+ case METRIC_ACTION_ADD_HEADER:
+ return "add_header";
+ case METRIC_ACTION_GREYLIST:
+ return "greylist";
+ case METRIC_ACTION_NOACTION:
+ return "no action";
+ case METRIC_ACTION_MAX:
+ return "invalid max action";
+ case METRIC_ACTION_CUSTOM:
+ return "custom";
+ case METRIC_ACTION_DISCARD:
+ return "discard";
+ case METRIC_ACTION_QUARANTINE:
+ return "quarantine";
+ }
+
+ return "unknown action";
+}
+
+static void
+rspamd_config_settings_elt_dtor(struct rspamd_config_settings_elt *e)
+{
+ if (e->symbols_enabled) {
+ ucl_object_unref(e->symbols_enabled);
+ }
+ if (e->symbols_disabled) {
+ ucl_object_unref(e->symbols_disabled);
+ }
+}
+
+guint32
+rspamd_config_name_to_id(const gchar *name, gsize namelen)
+{
+ guint64 h;
+
+ h = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64,
+ name, namelen, 0x0);
+ /* Take the lower part of hash as LE number */
+ return ((guint32) GUINT64_TO_LE(h));
+}
+
+struct rspamd_config_settings_elt *
+rspamd_config_find_settings_id_ref(struct rspamd_config *cfg,
+ guint32 id)
+{
+ struct rspamd_config_settings_elt *cur;
+
+ DL_FOREACH(cfg->setting_ids, cur)
+ {
+ if (cur->id == id) {
+ REF_RETAIN(cur);
+ return cur;
+ }
+ }
+
+ return nullptr;
+}
+
+struct rspamd_config_settings_elt *rspamd_config_find_settings_name_ref(
+ struct rspamd_config *cfg,
+ const gchar *name, gsize namelen)
+{
+ guint32 id;
+
+ id = rspamd_config_name_to_id(name, namelen);
+
+ return rspamd_config_find_settings_id_ref(cfg, id);
+}
+
+void rspamd_config_register_settings_id(struct rspamd_config *cfg,
+ const gchar *name,
+ ucl_object_t *symbols_enabled,
+ ucl_object_t *symbols_disabled,
+ enum rspamd_config_settings_policy policy)
+{
+ struct rspamd_config_settings_elt *elt;
+ guint32 id;
+
+ id = rspamd_config_name_to_id(name, strlen(name));
+ elt = rspamd_config_find_settings_id_ref(cfg, id);
+
+ if (elt) {
+ /* Need to replace */
+ struct rspamd_config_settings_elt *nelt;
+
+ DL_DELETE(cfg->setting_ids, elt);
+
+ nelt = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_config_settings_elt);
+
+ nelt->id = id;
+ nelt->name = rspamd_mempool_strdup(cfg->cfg_pool, name);
+
+ if (symbols_enabled) {
+ nelt->symbols_enabled = ucl_object_ref(symbols_enabled);
+ }
+
+ if (symbols_disabled) {
+ nelt->symbols_disabled = ucl_object_ref(symbols_disabled);
+ }
+
+ nelt->policy = policy;
+
+ REF_INIT_RETAIN(nelt, rspamd_config_settings_elt_dtor);
+ msg_warn_config("replace settings id %ud (%s)", id, name);
+ rspamd_symcache_process_settings_elt(cfg->cache, elt);
+ DL_APPEND(cfg->setting_ids, nelt);
+
+ /*
+ * Need to unref old element twice as there are two reference holders:
+ * 1. Config structure as we call REF_INIT_RETAIN
+ * 2. rspamd_config_find_settings_id_ref also increases refcount
+ */
+ REF_RELEASE(elt);
+ REF_RELEASE(elt);
+ }
+ else {
+ elt = rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_config_settings_elt);
+
+ elt->id = id;
+ elt->name = rspamd_mempool_strdup(cfg->cfg_pool, name);
+
+ if (symbols_enabled) {
+ elt->symbols_enabled = ucl_object_ref(symbols_enabled);
+ }
+
+ if (symbols_disabled) {
+ elt->symbols_disabled = ucl_object_ref(symbols_disabled);
+ }
+
+ elt->policy = policy;
+
+ msg_info_config("register new settings id %ud (%s)", id, name);
+ REF_INIT_RETAIN(elt, rspamd_config_settings_elt_dtor);
+ rspamd_symcache_process_settings_elt(cfg->cache, elt);
+ DL_APPEND(cfg->setting_ids, elt);
+ }
+}
+
+int rspamd_config_ev_backend_get(struct rspamd_config *cfg)
+{
+#define AUTO_BACKEND (ev_supported_backends() & ~EVBACKEND_IOURING)
+ if (cfg == nullptr || cfg->events_backend == nullptr) {
+ return AUTO_BACKEND;
+ }
+
+ if (strcmp(cfg->events_backend, "auto") == 0) {
+ return AUTO_BACKEND;
+ }
+ else if (strcmp(cfg->events_backend, "epoll") == 0) {
+ if (ev_supported_backends() & EVBACKEND_EPOLL) {
+ return EVBACKEND_EPOLL;
+ }
+ else {
+ msg_warn_config("unsupported events_backend: %s; defaulting to auto",
+ cfg->events_backend);
+ return AUTO_BACKEND;
+ }
+ }
+ else if (strcmp(cfg->events_backend, "iouring") == 0) {
+ if (ev_supported_backends() & EVBACKEND_IOURING) {
+ return EVBACKEND_IOURING;
+ }
+ else {
+ msg_warn_config("unsupported events_backend: %s; defaulting to auto",
+ cfg->events_backend);
+ return AUTO_BACKEND;
+ }
+ }
+ else if (strcmp(cfg->events_backend, "kqueue") == 0) {
+ if (ev_supported_backends() & EVBACKEND_KQUEUE) {
+ return EVBACKEND_KQUEUE;
+ }
+ else {
+ msg_warn_config("unsupported events_backend: %s; defaulting to auto",
+ cfg->events_backend);
+ return AUTO_BACKEND;
+ }
+ }
+ else if (strcmp(cfg->events_backend, "poll") == 0) {
+ return EVBACKEND_POLL;
+ }
+ else if (strcmp(cfg->events_backend, "select") == 0) {
+ return EVBACKEND_SELECT;
+ }
+ else {
+ msg_warn_config("unknown events_backend: %s; defaulting to auto",
+ cfg->events_backend);
+ }
+
+ return AUTO_BACKEND;
+}
+
+const gchar *
+rspamd_config_ev_backend_to_string(int ev_backend, gboolean *effective)
+{
+#define SET_EFFECTIVE(b) \
+ do { \
+ if ((effective) != nullptr) *(effective) = b; \
+ } while (0)
+
+ if ((ev_backend & EVBACKEND_ALL) == EVBACKEND_ALL) {
+ SET_EFFECTIVE(TRUE);
+ return "auto";
+ }
+
+ if (ev_backend & EVBACKEND_IOURING) {
+ SET_EFFECTIVE(TRUE);
+ return "epoll+io_uring";
+ }
+ if (ev_backend & EVBACKEND_LINUXAIO) {
+ SET_EFFECTIVE(TRUE);
+ return "epoll+aio";
+ }
+ if (ev_backend & EVBACKEND_IOURING) {
+ SET_EFFECTIVE(TRUE);
+ return "epoll+io_uring";
+ }
+ if (ev_backend & EVBACKEND_LINUXAIO) {
+ SET_EFFECTIVE(TRUE);
+ return "epoll+aio";
+ }
+ if (ev_backend & EVBACKEND_EPOLL) {
+ SET_EFFECTIVE(TRUE);
+ return "epoll";
+ }
+ if (ev_backend & EVBACKEND_KQUEUE) {
+ SET_EFFECTIVE(TRUE);
+ return "kqueue";
+ }
+ if (ev_backend & EVBACKEND_POLL) {
+ SET_EFFECTIVE(FALSE);
+ return "poll";
+ }
+ if (ev_backend & EVBACKEND_SELECT) {
+ SET_EFFECTIVE(FALSE);
+ return "select";
+ }
+
+ SET_EFFECTIVE(FALSE);
+ return "unknown";
+#undef SET_EFFECTIVE
+}
+
+struct rspamd_external_libs_ctx *
+rspamd_init_libs(void)
+{
+ struct rlimit rlim;
+ struct ottery_config *ottery_cfg;
+
+ auto *ctx = g_new0(struct rspamd_external_libs_ctx, 1);
+ ctx->crypto_ctx = rspamd_cryptobox_init();
+ ottery_cfg = (struct ottery_config *) g_malloc0(ottery_get_sizeof_config());
+ ottery_config_init(ottery_cfg);
+ ctx->ottery_cfg = ottery_cfg;
+
+ rspamd_openssl_maybe_init();
+
+ /* Check if we have rdrand */
+ if ((ctx->crypto_ctx->cpu_config & CPUID_RDRAND) == 0) {
+ ottery_config_disable_entropy_sources(ottery_cfg,
+ OTTERY_ENTROPY_SRC_RDRAND);
+ }
+
+ g_assert(ottery_init(ottery_cfg) == 0);
+#if OPENSSL_VERSION_NUMBER >= 0x1000104fL && OPENSSL_VERSION_NUMBER < 0x30000000L && !defined(LIBRESSL_VERSION_NUMBER)
+ RAND_set_rand_engine(nullptr);
+#endif
+
+ /* Configure utf8 library */
+ guint utf8_flags = 0;
+
+ if ((ctx->crypto_ctx->cpu_config & CPUID_SSE41)) {
+ utf8_flags |= RSPAMD_FAST_UTF8_FLAG_SSE41;
+ }
+ if ((ctx->crypto_ctx->cpu_config & CPUID_AVX2)) {
+ utf8_flags |= RSPAMD_FAST_UTF8_FLAG_AVX2;
+ }
+
+ rspamd_fast_utf8_library_init(utf8_flags);
+
+#ifdef HAVE_LOCALE_H
+ if (getenv("LANG") == nullptr) {
+ setlocale(LC_ALL, "C");
+ setlocale(LC_CTYPE, "C");
+ setlocale(LC_MESSAGES, "C");
+ setlocale(LC_TIME, "C");
+ }
+ else {
+ /* Just set the default locale */
+ setlocale(LC_ALL, "");
+ /* But for some issues we still want C locale */
+ setlocale(LC_NUMERIC, "C");
+ }
+#endif
+
+ ctx->ssl_ctx = rspamd_init_ssl_ctx();
+ ctx->ssl_ctx_noverify = rspamd_init_ssl_ctx_noverify();
+ rspamd_random_seed_fast();
+
+ /* Set stack size for pcre */
+ getrlimit(RLIMIT_STACK, &rlim);
+ rlim.rlim_cur = 100 * 1024 * 1024;
+ rlim.rlim_max = rlim.rlim_cur;
+ setrlimit(RLIMIT_STACK, &rlim);
+
+ ctx->local_addrs = rspamd_inet_library_init();
+ REF_INIT_RETAIN(ctx, rspamd_deinit_libs);
+
+ return ctx;
+}
+
+static struct zstd_dictionary *
+rspamd_open_zstd_dictionary(const char *path)
+{
+ struct zstd_dictionary *dict;
+
+ dict = g_new0(zstd_dictionary, 1);
+ dict->dict = rspamd_file_xmap(path, PROT_READ, &dict->size, TRUE);
+
+ if (dict->dict == nullptr) {
+ g_free(dict);
+
+ return nullptr;
+ }
+
+ dict->id = -1;
+
+ if (dict->id == 0) {
+ g_free(dict);
+
+ return nullptr;
+ }
+
+ return dict;
+}
+
+static void
+rspamd_free_zstd_dictionary(struct zstd_dictionary *dict)
+{
+ if (dict) {
+ munmap(dict->dict, dict->size);
+ g_free(dict);
+ }
+}
+
+#ifdef HAVE_OPENBLAS_SET_NUM_THREADS
+extern "C" void openblas_set_num_threads(int num_threads);
+#endif
+#ifdef HAVE_BLI_THREAD_SET_NUM_THREADS
+extern "C" void bli_thread_set_num_threads(int num_threads);
+#endif
+
+gboolean
+rspamd_config_libs(struct rspamd_external_libs_ctx *ctx,
+ struct rspamd_config *cfg)
+{
+ size_t r;
+ gboolean ret = TRUE;
+
+ g_assert(cfg != nullptr);
+
+ if (ctx != nullptr) {
+ if (cfg->local_addrs) {
+ GError *err = nullptr;
+ ret = rspamd_config_radix_from_ucl(cfg, cfg->local_addrs,
+ "Local addresses",
+ (struct rspamd_radix_map_helper **) ctx->local_addrs,
+ &err,
+ nullptr, "local addresses");
+
+ if (!ret) {
+ msg_err_config("cannot load local addresses: %e", err);
+ g_error_free(err);
+
+ return ret;
+ }
+ }
+
+ rspamd_free_zstd_dictionary(ctx->in_dict);
+ rspamd_free_zstd_dictionary(ctx->out_dict);
+
+ if (ctx->out_zstream) {
+ ZSTD_freeCStream((ZSTD_CCtx *) ctx->out_zstream);
+ ctx->out_zstream = nullptr;
+ }
+
+ if (ctx->in_zstream) {
+ ZSTD_freeDStream((ZSTD_DCtx *) ctx->in_zstream);
+ ctx->in_zstream = nullptr;
+ }
+
+ if (cfg->zstd_input_dictionary) {
+ ctx->in_dict = rspamd_open_zstd_dictionary(
+ cfg->zstd_input_dictionary);
+
+ if (ctx->in_dict == nullptr) {
+ msg_err_config("cannot open zstd dictionary in %s",
+ cfg->zstd_input_dictionary);
+ }
+ }
+ if (cfg->zstd_output_dictionary) {
+ ctx->out_dict = rspamd_open_zstd_dictionary(
+ cfg->zstd_output_dictionary);
+
+ if (ctx->out_dict == nullptr) {
+ msg_err_config("cannot open zstd dictionary in %s",
+ cfg->zstd_output_dictionary);
+ }
+ }
+
+ if (cfg->fips_mode) {
+#ifdef HAVE_FIPS_MODE
+ int mode = FIPS_mode();
+ unsigned long err = (unsigned long) -1;
+
+ /* Toggle FIPS mode */
+ if (mode == 0) {
+#if defined(OPENSSL_VERSION_MAJOR) && (OPENSSL_VERSION_MAJOR >= 3)
+ if (EVP_set_default_properties(nullptr, "fips=yes") != 1) {
+#else
+ if (FIPS_mode_set(1) != 1) {
+#endif
+ err = ERR_get_error();
+ }
+ }
+ else {
+ msg_info_config("OpenSSL FIPS mode is already enabled");
+ }
+
+ if (err != (unsigned long) -1) {
+#if defined(OPENSSL_VERSION_MAJOR) && (OPENSSL_VERSION_MAJOR >= 3)
+ msg_err_config("EVP_set_default_properties failed: %s",
+#else
+ msg_err_config("FIPS_mode_set failed: %s",
+#endif
+ ERR_error_string(err, nullptr));
+ ret = FALSE;
+ }
+ else {
+ msg_info_config("OpenSSL FIPS mode is enabled");
+ }
+#else
+ msg_warn_config("SSL FIPS mode is enabled but not supported by OpenSSL library!");
+#endif
+ }
+
+ rspamd_ssl_ctx_config(cfg, ctx->ssl_ctx);
+ rspamd_ssl_ctx_config(cfg, ctx->ssl_ctx_noverify);
+
+ /* Init decompression */
+ ctx->in_zstream = ZSTD_createDStream();
+ r = ZSTD_initDStream((ZSTD_DCtx *) ctx->in_zstream);
+
+ if (ZSTD_isError(r)) {
+ msg_err("cannot init decompression stream: %s",
+ ZSTD_getErrorName(r));
+ ZSTD_freeDStream((ZSTD_DCtx *) ctx->in_zstream);
+ ctx->in_zstream = nullptr;
+ }
+
+ /* Init compression */
+ ctx->out_zstream = ZSTD_createCStream();
+ r = ZSTD_initCStream((ZSTD_CCtx *) ctx->out_zstream, 1);
+
+ if (ZSTD_isError(r)) {
+ msg_err("cannot init compression stream: %s",
+ ZSTD_getErrorName(r));
+ ZSTD_freeCStream((ZSTD_CCtx *) ctx->out_zstream);
+ ctx->out_zstream = nullptr;
+ }
+#ifdef HAVE_OPENBLAS_SET_NUM_THREADS
+ openblas_set_num_threads(cfg->max_blas_threads);
+#endif
+#ifdef HAVE_BLI_THREAD_SET_NUM_THREADS
+ bli_thread_set_num_threads(cfg->max_blas_threads);
+#endif
+ }
+
+ return ret;
+}
+
+gboolean
+rspamd_libs_reset_decompression(struct rspamd_external_libs_ctx *ctx)
+{
+ gsize r;
+
+ if (ctx->in_zstream == nullptr) {
+ return FALSE;
+ }
+ else {
+ r = ZSTD_DCtx_reset((ZSTD_DCtx *) ctx->in_zstream, ZSTD_reset_session_only);
+
+ if (ZSTD_isError(r)) {
+ msg_err("cannot init decompression stream: %s",
+ ZSTD_getErrorName(r));
+ ZSTD_freeDStream((ZSTD_DCtx *) ctx->in_zstream);
+ ctx->in_zstream = nullptr;
+
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_libs_reset_compression(struct rspamd_external_libs_ctx *ctx)
+{
+ gsize r;
+
+ if (ctx->out_zstream == nullptr) {
+ return FALSE;
+ }
+ else {
+ /* Dictionary will be reused automatically if specified */
+ r = ZSTD_CCtx_reset((ZSTD_CCtx *) ctx->out_zstream, ZSTD_reset_session_only);
+ if (!ZSTD_isError(r)) {
+ r = ZSTD_CCtx_setPledgedSrcSize((ZSTD_CCtx *) ctx->out_zstream, ZSTD_CONTENTSIZE_UNKNOWN);
+ }
+
+ if (ZSTD_isError(r)) {
+ msg_err("cannot init compression stream: %s",
+ ZSTD_getErrorName(r));
+ ZSTD_freeCStream((ZSTD_CCtx *) ctx->out_zstream);
+ ctx->out_zstream = nullptr;
+
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+void rspamd_deinit_libs(struct rspamd_external_libs_ctx *ctx)
+{
+ if (ctx != nullptr) {
+ g_free(ctx->ottery_cfg);
+
+#ifdef HAVE_OPENSSL
+ EVP_cleanup();
+ ERR_free_strings();
+ rspamd_ssl_ctx_free(ctx->ssl_ctx);
+ rspamd_ssl_ctx_free(ctx->ssl_ctx_noverify);
+#endif
+ rspamd_inet_library_destroy();
+ rspamd_free_zstd_dictionary(ctx->in_dict);
+ rspamd_free_zstd_dictionary(ctx->out_dict);
+
+ if (ctx->out_zstream) {
+ ZSTD_freeCStream((ZSTD_CCtx *) ctx->out_zstream);
+ }
+
+ if (ctx->in_zstream) {
+ ZSTD_freeDStream((ZSTD_DCtx *) ctx->in_zstream);
+ }
+
+ rspamd_cryptobox_deinit(ctx->crypto_ctx);
+
+ g_free(ctx);
+ }
+}
+
+gboolean
+rspamd_ip_is_local_cfg(struct rspamd_config *cfg,
+ const rspamd_inet_addr_t *addr)
+{
+ struct rspamd_radix_map_helper *local_addrs = nullptr;
+
+ if (cfg && cfg->libs_ctx) {
+ local_addrs = *(struct rspamd_radix_map_helper **) cfg->libs_ctx->local_addrs;
+ }
+
+ if (rspamd_inet_address_is_local(addr)) {
+ return TRUE;
+ }
+
+ if (local_addrs) {
+ if (rspamd_match_radix_map_addr(local_addrs, addr) != nullptr) {
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
diff --git a/src/libserver/composites/composites.cxx b/src/libserver/composites/composites.cxx
new file mode 100644
index 0000000..aa231a3
--- /dev/null
+++ b/src/libserver/composites/composites.cxx
@@ -0,0 +1,989 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "logger.h"
+#include "expression.h"
+#include "task.h"
+#include "utlist.h"
+#include "scan_result.h"
+#include "composites.h"
+
+#include <cmath>
+#include <vector>
+#include <variant>
+#include "libutil/cxx/util.hxx"
+#include "contrib/ankerl/unordered_dense.h"
+
+#include "composites_internal.hxx"
+
+#define msg_err_composites(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "composites", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_composites(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "composites", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_composites(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "composites", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+#define msg_debug_composites(...) rspamd_conditional_debug_fast(NULL, task->from_addr, \
+ rspamd_composites_log_id, "composites", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(composites)
+
+
+namespace rspamd::composites {
+static rspamd_expression_atom_t *rspamd_composite_expr_parse(const gchar *line, gsize len,
+ rspamd_mempool_t *pool,
+ gpointer ud, GError **err);
+static gdouble rspamd_composite_expr_process(void *ud, rspamd_expression_atom_t *atom);
+static gint rspamd_composite_expr_priority(rspamd_expression_atom_t *atom);
+static void rspamd_composite_expr_destroy(rspamd_expression_atom_t *atom);
+static void composites_foreach_callback(gpointer key, gpointer value, void *data);
+
+const struct rspamd_atom_subr composite_expr_subr = {
+ .parse = rspamd::composites::rspamd_composite_expr_parse,
+ .process = rspamd::composites::rspamd_composite_expr_process,
+ .priority = rspamd::composites::rspamd_composite_expr_priority,
+ .destroy = rspamd::composites::rspamd_composite_expr_destroy};
+}// namespace rspamd::composites
+
+namespace rspamd::composites {
+
+static constexpr const double epsilon = 0.00001;
+
+struct symbol_remove_data {
+ const char *sym;
+ struct rspamd_composite *comp;
+ GNode *parent;
+ std::uint8_t action;
+};
+
+struct composites_data {
+ struct rspamd_task *task;
+ struct rspamd_composite *composite;
+ struct rspamd_scan_result *metric_res;
+ ankerl::unordered_dense::map<std::string_view,
+ std::vector<symbol_remove_data>>
+ symbols_to_remove;
+ std::vector<bool> checked;
+
+ explicit composites_data(struct rspamd_task *task, struct rspamd_scan_result *mres)
+ : task(task), composite(nullptr), metric_res(mres)
+ {
+ checked.resize(rspamd_composites_manager_nelts(task->cfg->composites_manager) * 2,
+ false);
+ }
+};
+
+struct rspamd_composite_option_match {
+ rspamd_regexp_t *re;
+ std::string match;
+
+ explicit rspamd_composite_option_match(const char *start, std::size_t len) noexcept
+ : re(nullptr), match(start, len)
+ {
+ }
+
+ explicit rspamd_composite_option_match(rspamd_regexp_t *re) noexcept
+ : re(rspamd_regexp_ref(re))
+ {
+ }
+
+ rspamd_composite_option_match(const rspamd_composite_option_match &other) noexcept
+ {
+ if (other.re) {
+ re = rspamd_regexp_ref(other.re);
+ }
+ else {
+ match = other.match;
+ re = nullptr;
+ }
+ }
+ rspamd_composite_option_match &operator=(const rspamd_composite_option_match &other) noexcept
+ {
+ if (other.re) {
+ if (re) {
+ rspamd_regexp_unref(re);
+ }
+ re = rspamd_regexp_ref(other.re);
+ }
+ else {
+ if (re) {
+ rspamd_regexp_unref(re);
+ }
+ re = nullptr;
+ match = other.match;
+ }
+
+ return *this;
+ }
+
+ rspamd_composite_option_match(rspamd_composite_option_match &&other) noexcept
+ {
+ if (other.re) {
+ re = other.re;
+ other.re = nullptr;
+ }
+ else {
+ re = nullptr;
+ match = std::move(other.match);
+ }
+ }
+ rspamd_composite_option_match &operator=(rspamd_composite_option_match &&other) noexcept
+ {
+ if (other.re) {
+ if (re) {
+ rspamd_regexp_unref(re);
+ }
+ re = other.re;
+ other.re = nullptr;
+ }
+ else {
+ if (re) {
+ rspamd_regexp_unref(re);
+ }
+ re = nullptr;
+ match = std::move(other.match);
+ }
+
+ return *this;
+ }
+
+ ~rspamd_composite_option_match()
+ {
+ if (re) {
+ rspamd_regexp_unref(re);
+ }
+ }
+
+ auto match_opt(const std::string_view &data) const -> bool
+ {
+ if (re) {
+ return rspamd_regexp_search(re,
+ data.data(), data.size(),
+ nullptr, nullptr, false, nullptr);
+ }
+ else {
+ return data == match;
+ }
+ }
+
+ auto get_pat() const -> std::string_view
+ {
+ if (re) {
+ return std::string_view(rspamd_regexp_get_pattern(re));
+ }
+ else {
+ return match;
+ }
+ }
+};
+
+enum class rspamd_composite_atom_type {
+ ATOM_UNKNOWN,
+ ATOM_COMPOSITE,
+ ATOM_PLAIN
+};
+
+struct rspamd_composite_atom {
+ std::string symbol;
+ std::string_view norm_symbol;
+ rspamd_composite_atom_type comp_type = rspamd_composite_atom_type::ATOM_UNKNOWN;
+ const struct rspamd_composite *ncomp; /* underlying composite */
+ std::vector<rspamd_composite_option_match> opts;
+};
+
+enum rspamd_composite_action : std::uint8_t {
+ RSPAMD_COMPOSITE_UNTOUCH = 0,
+ RSPAMD_COMPOSITE_REMOVE_SYMBOL = (1u << 0),
+ RSPAMD_COMPOSITE_REMOVE_WEIGHT = (1u << 1),
+ RSPAMD_COMPOSITE_REMOVE_FORCED = (1u << 2)
+};
+
+static GQuark
+rspamd_composites_quark(void)
+{
+ return g_quark_from_static_string("composites");
+}
+
+static auto
+rspamd_composite_atom_dtor(void *ptr)
+{
+ auto *atom = reinterpret_cast<rspamd_composite_atom *>(ptr);
+
+ delete atom;
+}
+
+static rspamd_expression_atom_t *
+rspamd_composite_expr_parse(const gchar *line, gsize len,
+ rspamd_mempool_t *pool,
+ gpointer ud, GError **err)
+{
+ gsize clen = 0;
+ const gchar *p, *end;
+ enum composite_expr_state {
+ comp_state_read_symbol = 0,
+ comp_state_read_obrace,
+ comp_state_read_option,
+ comp_state_read_regexp,
+ comp_state_read_regexp_end,
+ comp_state_read_comma,
+ comp_state_read_ebrace,
+ comp_state_read_end
+ } state = comp_state_read_symbol;
+
+ end = line + len;
+ p = line;
+
+ /* Find length of the atom using a reduced state machine */
+ while (p < end) {
+ if (state == comp_state_read_end) {
+ break;
+ }
+
+ switch (state) {
+ case comp_state_read_symbol:
+ clen = rspamd_memcspn(p, "[; \t()><!|&\n", len);
+ p += clen;
+
+ if (*p == '[') {
+ state = comp_state_read_obrace;
+ }
+ else {
+ state = comp_state_read_end;
+ }
+ break;
+ case comp_state_read_obrace:
+ p++;
+
+ if (*p == '/') {
+ p++;
+ state = comp_state_read_regexp;
+ }
+ else {
+ state = comp_state_read_option;
+ }
+ break;
+ case comp_state_read_regexp:
+ if (*p == '\\' && p + 1 < end) {
+ /* Escaping */
+ p++;
+ }
+ else if (*p == '/') {
+ /* End of regexp, possible flags */
+ state = comp_state_read_regexp_end;
+ }
+ p++;
+ break;
+ case comp_state_read_option:
+ case comp_state_read_regexp_end:
+ if (*p == ',') {
+ p++;
+ state = comp_state_read_comma;
+ }
+ else if (*p == ']') {
+ state = comp_state_read_ebrace;
+ }
+ else {
+ p++;
+ }
+ break;
+ case comp_state_read_comma:
+ if (!g_ascii_isspace(*p)) {
+ if (*p == '/') {
+ state = comp_state_read_regexp;
+ }
+ else if (*p == ']') {
+ state = comp_state_read_ebrace;
+ }
+ else {
+ state = comp_state_read_option;
+ }
+ }
+ else {
+ /* Skip spaces after comma */
+ p++;
+ }
+ break;
+ case comp_state_read_ebrace:
+ p++;
+ state = comp_state_read_end;
+ break;
+ case comp_state_read_end:
+ g_assert_not_reached();
+ }
+ }
+
+ if (state != comp_state_read_end) {
+ g_set_error(err, rspamd_composites_quark(), 100, "invalid composite: %s;"
+ "parser stopped in state %d",
+ line, state);
+ return NULL;
+ }
+
+ clen = p - line;
+ p = line;
+ state = comp_state_read_symbol;
+
+ auto *atom = new rspamd_composite_atom;
+ auto *res = rspamd_mempool_alloc0_type(pool, rspamd_expression_atom_t);
+ res->len = clen;
+ res->str = line;
+
+ /* Full state machine to fill a composite atom */
+ const gchar *opt_start = nullptr;
+
+ while (p < end) {
+ if (state == comp_state_read_end) {
+ break;
+ }
+
+ switch (state) {
+ case comp_state_read_symbol: {
+ clen = rspamd_memcspn(p, "[; \t()><!|&\n", len);
+ p += clen;
+
+ if (*p == '[') {
+ state = comp_state_read_obrace;
+ }
+ else {
+ state = comp_state_read_end;
+ }
+
+ atom->symbol = std::string{line, clen};
+ auto norm_start = std::find_if(atom->symbol.begin(), atom->symbol.end(),
+ [](char c) { return g_ascii_isalnum(c); });
+ if (norm_start == atom->symbol.end()) {
+ msg_err_pool("invalid composite atom: %s", atom->symbol.c_str());
+ }
+ atom->norm_symbol = make_string_view_from_it(norm_start, atom->symbol.end());
+ break;
+ }
+ case comp_state_read_obrace:
+ p++;
+
+ if (*p == '/') {
+ opt_start = p;
+ p++; /* Starting slash */
+ state = comp_state_read_regexp;
+ }
+ else {
+ state = comp_state_read_option;
+ opt_start = p;
+ }
+
+ break;
+ case comp_state_read_regexp:
+ if (*p == '\\' && p + 1 < end) {
+ /* Escaping */
+ p++;
+ }
+ else if (*p == '/') {
+ /* End of regexp, possible flags */
+ state = comp_state_read_regexp_end;
+ }
+ p++;
+ break;
+ case comp_state_read_option:
+ if (*p == ',' || *p == ']') {
+ /* Plain match, copy option to ensure string_view validity */
+ gint opt_len = p - opt_start;
+ auto *opt_buf = rspamd_mempool_alloc_buffer(pool, opt_len + 1);
+ rspamd_strlcpy(opt_buf, opt_start, opt_len + 1);
+ opt_buf = g_strstrip(opt_buf);
+ atom->opts.emplace_back(opt_buf, strlen(opt_buf));
+
+ if (*p == ',') {
+ p++;
+ state = comp_state_read_comma;
+ }
+ else {
+ state = comp_state_read_ebrace;
+ }
+ }
+ else {
+ p++;
+ }
+ break;
+ case comp_state_read_regexp_end:
+ if (*p == ',' || *p == ']') {
+ auto opt_len = p - opt_start;
+ rspamd_regexp_t *re;
+ GError *re_err = nullptr;
+
+ re = rspamd_regexp_new_len(opt_start, opt_len, nullptr, &re_err);
+
+ if (re == nullptr) {
+ msg_err_pool("cannot create regexp from string %*s: %e",
+ opt_len, opt_start, re_err);
+
+ g_error_free(re_err);
+ }
+ else {
+ atom->opts.emplace_back(re);
+ rspamd_regexp_unref(re);
+ }
+
+ if (*p == ',') {
+ p++;
+ state = comp_state_read_comma;
+ }
+ else {
+ state = comp_state_read_ebrace;
+ }
+ }
+ else {
+ p++;
+ }
+ break;
+ case comp_state_read_comma:
+ if (!g_ascii_isspace(*p)) {
+ if (*p == '/') {
+ state = comp_state_read_regexp;
+ opt_start = p;
+ }
+ else if (*p == ']') {
+ state = comp_state_read_ebrace;
+ }
+ else {
+ opt_start = p;
+ state = comp_state_read_option;
+ }
+ }
+ else {
+ /* Skip spaces after comma */
+ p++;
+ }
+ break;
+ case comp_state_read_ebrace:
+ p++;
+ state = comp_state_read_end;
+ break;
+ case comp_state_read_end:
+ g_assert_not_reached();
+ }
+ }
+
+ res->data = atom;
+
+ return res;
+}
+
+static auto
+process_symbol_removal(rspamd_expression_atom_t *atom,
+ struct composites_data *cd,
+ struct rspamd_symbol_result *ms,
+ const std::string &beg) -> void
+{
+ struct rspamd_task *task = cd->task;
+
+ if (ms == nullptr) {
+ return;
+ }
+
+ /*
+ * At this point we know that we need to do something about this symbol,
+ * however, we don't know whether we need to delete it unfortunately,
+ * that depends on the later decisions when the complete expression is
+ * evaluated.
+ */
+ auto rd_it = cd->symbols_to_remove.find(ms->name);
+
+ auto fill_removal_structure = [&](symbol_remove_data &nrd) {
+ nrd.sym = ms->name;
+
+ /* By default remove symbols */
+ switch (cd->composite->policy) {
+ case rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_ALL:
+ default:
+ nrd.action = (RSPAMD_COMPOSITE_REMOVE_SYMBOL | RSPAMD_COMPOSITE_REMOVE_WEIGHT);
+ break;
+ case rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_SYMBOL:
+ nrd.action = RSPAMD_COMPOSITE_REMOVE_SYMBOL;
+ break;
+ case rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_WEIGHT:
+ nrd.action = RSPAMD_COMPOSITE_REMOVE_WEIGHT;
+ break;
+ case rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_LEAVE:
+ nrd.action = 0;
+ break;
+ }
+
+ for (auto t: beg) {
+ if (t == '~') {
+ nrd.action &= ~RSPAMD_COMPOSITE_REMOVE_SYMBOL;
+ }
+ else if (t == '-') {
+ nrd.action &= ~(RSPAMD_COMPOSITE_REMOVE_WEIGHT |
+ RSPAMD_COMPOSITE_REMOVE_SYMBOL);
+ }
+ else if (t == '^') {
+ nrd.action |= RSPAMD_COMPOSITE_REMOVE_FORCED;
+ }
+ else {
+ break;
+ }
+ }
+
+ nrd.comp = cd->composite;
+ nrd.parent = atom->parent;
+ };
+
+ if (rd_it != cd->symbols_to_remove.end()) {
+ fill_removal_structure(rd_it->second.emplace_back());
+ msg_debug_composites("%s: added symbol %s to removal: %d policy, from composite %s",
+ cd->metric_res->name,
+ ms->name, rd_it->second.back().action,
+ cd->composite->sym.c_str());
+ }
+ else {
+ std::vector<symbol_remove_data> nrd;
+ fill_removal_structure(nrd.emplace_back());
+ msg_debug_composites("%s: added symbol %s to removal: %d policy, from composite %s",
+ cd->metric_res->name,
+ ms->name, nrd.front().action,
+ cd->composite->sym.c_str());
+ cd->symbols_to_remove[ms->name] = std::move(nrd);
+ }
+}
+
+static auto
+process_single_symbol(struct composites_data *cd,
+ std::string_view sym,
+ struct rspamd_symbol_result **pms,
+ struct rspamd_composite_atom *atom) -> double
+{
+ struct rspamd_symbol_result *ms = nullptr;
+ gdouble rc = 0;
+ struct rspamd_task *task = cd->task;
+
+ if ((ms = rspamd_task_find_symbol_result(cd->task, sym.data(), cd->metric_res)) == nullptr) {
+ msg_debug_composites("not found symbol %s in composite %s", sym.data(),
+ cd->composite->sym.c_str());
+
+ if (G_UNLIKELY(atom->comp_type == rspamd_composite_atom_type::ATOM_UNKNOWN)) {
+ const struct rspamd_composite *ncomp;
+
+ if ((ncomp = COMPOSITE_MANAGER_FROM_PTR(task->cfg->composites_manager)->find(sym)) != NULL) {
+ atom->comp_type = rspamd_composite_atom_type::ATOM_COMPOSITE;
+ atom->ncomp = ncomp;
+ }
+ else {
+ atom->comp_type = rspamd_composite_atom_type::ATOM_PLAIN;
+ }
+ }
+
+ if (atom->comp_type == rspamd_composite_atom_type::ATOM_COMPOSITE) {
+ msg_debug_composites("symbol %s for composite %s is another composite",
+ sym.data(), cd->composite->sym.c_str());
+
+ if (!cd->checked[atom->ncomp->id * 2]) {
+ msg_debug_composites("composite dependency %s for %s is not checked",
+ sym.data(), cd->composite->sym.c_str());
+ /* Set checked for this symbol to avoid cyclic references */
+ cd->checked[cd->composite->id * 2] = true;
+ auto *saved = cd->composite; /* Save the current composite */
+ composites_foreach_callback((gpointer) atom->ncomp->sym.c_str(),
+ (gpointer) atom->ncomp, (gpointer) cd);
+ /* Restore state */
+ cd->composite = saved;
+ cd->checked[cd->composite->id * 2] = false;
+
+ ms = rspamd_task_find_symbol_result(cd->task, sym.data(),
+ cd->metric_res);
+ }
+ else {
+ /*
+ * XXX: in case of cyclic references this would return 0
+ */
+ if (cd->checked[atom->ncomp->id * 2 + 1]) {
+ ms = rspamd_task_find_symbol_result(cd->task, sym.data(),
+ cd->metric_res);
+ }
+ }
+ }
+ }
+
+ if (ms) {
+ msg_debug_composites("found symbol %s in composite %s, weight: %.3f",
+ sym.data(), cd->composite->sym.c_str(), ms->score);
+
+ /* Now check options */
+ for (const auto &cur_opt: atom->opts) {
+ struct rspamd_symbol_option *opt;
+ auto found = false;
+
+ DL_FOREACH(ms->opts_head, opt)
+ {
+ if (cur_opt.match_opt({opt->option, opt->optlen})) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ auto pat = cur_opt.get_pat();
+ msg_debug_composites("symbol %s in composite %s misses required option %*s",
+ sym.data(),
+ cd->composite->sym.c_str(),
+ (int) pat.size(), pat.data());
+ ms = nullptr;
+
+ break;
+ }
+ }
+
+ if (ms) {
+ if (ms->score == 0) {
+ rc = epsilon * 16.0; /* Distinguish from 0 */
+ }
+ else {
+ rc = ms->score;
+ }
+ }
+ }
+
+ *pms = ms;
+ return rc;
+}
+
+static auto
+rspamd_composite_expr_process(void *ud, rspamd_expression_atom_t *atom) -> double
+{
+ struct composites_data *cd = (struct composites_data *) ud;
+ struct rspamd_composite_atom *comp_atom = (struct rspamd_composite_atom *) atom->data;
+
+ struct rspamd_symbol_result *ms = NULL;
+ struct rspamd_task *task = cd->task;
+ gdouble rc = 0;
+
+ if (cd->checked[cd->composite->id * 2]) {
+ /* We have already checked this composite, so just return its value */
+ if (cd->checked[cd->composite->id * 2 + 1]) {
+ ms = rspamd_task_find_symbol_result(cd->task,
+ comp_atom->norm_symbol.data(),
+ cd->metric_res);
+ }
+
+ if (ms) {
+ if (ms->score == 0) {
+ rc = epsilon; /* Distinguish from 0 */
+ }
+ else {
+ /* Treat negative and positive scores equally... */
+ rc = fabs(ms->score);
+ }
+ }
+
+ msg_debug_composites("composite %s is already checked, result: %.2f",
+ cd->composite->sym.c_str(), rc);
+
+ return rc;
+ }
+
+ /* Note: sym is zero terminated as it is a view on std::string */
+ auto sym = comp_atom->norm_symbol;
+ auto group_process_functor = [&](auto cond, int sub_start) -> double {
+ auto max = 0.;
+ GHashTableIter it;
+ gpointer k, v;
+ struct rspamd_symbols_group *gr;
+
+ gr = (struct rspamd_symbols_group *) g_hash_table_lookup(cd->task->cfg->groups,
+ sym.substr(sub_start).data());
+
+ if (gr != nullptr) {
+ g_hash_table_iter_init(&it, gr->symbols);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ auto *sdef = (rspamd_symbol *) v;
+
+ if (cond(sdef->score)) {
+ rc = process_single_symbol(cd,
+ std::string_view(sdef->name),
+ &ms,
+ comp_atom);
+
+ if (fabs(rc) > epsilon) {
+ process_symbol_removal(atom,
+ cd,
+ ms,
+ comp_atom->symbol);
+
+ if (fabs(rc) > max) {
+ max = fabs(rc);
+ }
+ }
+ }
+ }
+ }
+
+ return max;
+ };
+
+ if (sym.size() > 2) {
+ if (sym.substr(0, 2) == "g:") {
+ rc = group_process_functor([](auto _) { return true; }, 2);
+ }
+ else if (sym.substr(0, 3) == "g+:") {
+ /* Group, positive symbols only */
+ rc = group_process_functor([](auto sc) { return sc > 0.; }, 3);
+ }
+ else if (sym.substr(0, 3) == "g-:") {
+ rc = group_process_functor([](auto sc) { return sc < 0.; }, 3);
+ }
+ else {
+ rc = process_single_symbol(cd, sym, &ms, comp_atom);
+
+ if (fabs(rc) > epsilon) {
+ process_symbol_removal(atom,
+ cd,
+ ms,
+ comp_atom->symbol);
+ }
+ }
+ }
+ else {
+ rc = process_single_symbol(cd, sym, &ms, comp_atom);
+
+ if (fabs(rc) > epsilon) {
+ process_symbol_removal(atom,
+ cd,
+ ms,
+ comp_atom->symbol);
+ }
+ }
+
+ msg_debug_composites("%s: result for atom %s in composite %s is %.4f",
+ cd->metric_res->name,
+ comp_atom->norm_symbol.data(),
+ cd->composite->sym.c_str(), rc);
+
+ return rc;
+}
+
+/*
+ * We don't have preferences for composites
+ */
+static gint
+rspamd_composite_expr_priority(rspamd_expression_atom_t *atom)
+{
+ return 0;
+}
+
+static void
+rspamd_composite_expr_destroy(rspamd_expression_atom_t *atom)
+{
+ rspamd_composite_atom_dtor(atom->data);
+}
+
+static void
+composites_foreach_callback(gpointer key, gpointer value, void *data)
+{
+ auto *cd = (struct composites_data *) data;
+ auto *comp = (struct rspamd_composite *) value;
+ auto *str_key = (const gchar *) key;
+ struct rspamd_task *task;
+ gdouble rc;
+
+ cd->composite = comp;
+ task = cd->task;
+
+ msg_debug_composites("process composite %s", str_key);
+
+ if (!cd->checked[cd->composite->id * 2]) {
+ if (rspamd_symcache_is_checked(cd->task, cd->task->cfg->cache,
+ str_key)) {
+ msg_debug_composites("composite %s is checked in symcache but not "
+ "in composites bitfield",
+ cd->composite->sym.c_str());
+ cd->checked[comp->id * 2] = true;
+ cd->checked[comp->id * 2 + 1] = false;
+ }
+ else {
+ if (rspamd_task_find_symbol_result(cd->task, str_key,
+ cd->metric_res) != nullptr) {
+ /* Already set, no need to check */
+ msg_debug_composites("composite %s is already in metric "
+ "in composites bitfield",
+ cd->composite->sym.c_str());
+ cd->checked[comp->id * 2] = true;
+ cd->checked[comp->id * 2 + 1] = true;
+
+ return;
+ }
+
+ msg_debug_composites("%s: start processing composite %s",
+ cd->metric_res->name,
+ cd->composite->sym.c_str());
+
+ rc = rspamd_process_expression(comp->expr, RSPAMD_EXPRESSION_FLAG_NOOPT,
+ cd);
+
+ /* Checked bit */
+ cd->checked[comp->id * 2] = true;
+
+ msg_debug_composites("%s: final result for composite %s is %.4f",
+ cd->metric_res->name,
+ cd->composite->sym.c_str(), rc);
+
+ /* Result bit */
+ if (fabs(rc) > epsilon) {
+ cd->checked[comp->id * 2 + 1] = true;
+ rspamd_task_insert_result_full(cd->task, str_key, 1.0, NULL,
+ RSPAMD_SYMBOL_INSERT_SINGLE, cd->metric_res);
+ }
+ else {
+ cd->checked[comp->id * 2 + 1] = false;
+ }
+ }
+ }
+}
+
+
+static auto
+remove_symbols(const composites_data &cd, const std::vector<symbol_remove_data> &rd) -> void
+{
+ struct rspamd_task *task = cd.task;
+ gboolean skip = FALSE,
+ has_valid_op = FALSE,
+ want_remove_score = TRUE,
+ want_remove_symbol = TRUE,
+ want_forced = FALSE;
+ const gchar *disable_score_reason = "no policy",
+ *disable_symbol_reason = "no policy";
+
+ task = cd.task;
+
+ for (const auto &cur: rd) {
+ if (!cd.checked[cur.comp->id * 2 + 1]) {
+ continue;
+ }
+ /*
+ * First of all exclude all elements with any parent that is negation:
+ * !A || B -> here we can have both !A and B matched, but we do *NOT*
+ * want to remove symbol in that case
+ */
+ auto *par = cur.parent;
+ skip = FALSE;
+
+ while (par) {
+ if (rspamd_expression_node_is_op(par, OP_NOT)) {
+ skip = TRUE;
+ break;
+ }
+
+ par = par->parent;
+ }
+
+ if (skip) {
+ continue;
+ }
+
+ has_valid_op = TRUE;
+ /*
+ * Now we can try to remove symbols/scores
+ *
+ * We apply the following logic here:
+ * - if no composites would like to save score then we remove score
+ * - if no composites would like to save symbol then we remove symbol
+ */
+ if (!want_forced) {
+ if (!(cur.action & RSPAMD_COMPOSITE_REMOVE_SYMBOL)) {
+ want_remove_symbol = FALSE;
+ disable_symbol_reason = cur.comp->sym.c_str();
+ }
+
+ if (!(cur.action & RSPAMD_COMPOSITE_REMOVE_WEIGHT)) {
+ want_remove_score = FALSE;
+ disable_score_reason = cur.comp->sym.c_str();
+ }
+
+ if (cur.action & RSPAMD_COMPOSITE_REMOVE_FORCED) {
+ want_forced = TRUE;
+ disable_symbol_reason = cur.comp->sym.c_str();
+ disable_score_reason = cur.comp->sym.c_str();
+ }
+ }
+ }
+
+ auto *ms = rspamd_task_find_symbol_result(task, rd.front().sym, cd.metric_res);
+
+ if (has_valid_op && ms && !(ms->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+
+ if (want_remove_score || want_forced) {
+ msg_debug_composites("%s: %s remove symbol weight for %s (was %.2f), "
+ "score removal affected by %s, symbol removal affected by %s",
+ cd.metric_res->name,
+ (want_forced ? "forced" : "normal"), rd.front().sym, ms->score,
+ disable_score_reason, disable_symbol_reason);
+ cd.metric_res->score -= ms->score;
+ ms->score = 0.0;
+ }
+
+ if (want_remove_symbol || want_forced) {
+ ms->flags |= RSPAMD_SYMBOL_RESULT_IGNORED;
+ msg_debug_composites("%s: %s remove symbol %s (score %.2f), "
+ "score removal affected by %s, symbol removal affected by %s",
+ cd.metric_res->name,
+ (want_forced ? "forced" : "normal"), rd.front().sym, ms->score,
+ disable_score_reason, disable_symbol_reason);
+ }
+ }
+}
+
+static void
+composites_metric_callback(struct rspamd_task *task)
+{
+ std::vector<composites_data> comp_data_vec;
+ struct rspamd_scan_result *mres;
+
+ comp_data_vec.reserve(1);
+
+ DL_FOREACH(task->result, mres)
+ {
+ auto &cd = comp_data_vec.emplace_back(task, mres);
+
+ /* Process metric result */
+ rspamd_symcache_composites_foreach(task,
+ task->cfg->cache,
+ composites_foreach_callback,
+ &cd);
+ }
+
+ for (const auto &cd: comp_data_vec) {
+ /* Remove symbols that are in composites */
+ for (const auto &srd_it: cd.symbols_to_remove) {
+ remove_symbols(cd, srd_it.second);
+ }
+ }
+}
+
+}// namespace rspamd::composites
+
+
+void rspamd_composites_process_task(struct rspamd_task *task)
+{
+ if (task->result && !RSPAMD_TASK_IS_SKIPPED(task)) {
+ rspamd::composites::composites_metric_callback(task);
+ }
+}
diff --git a/src/libserver/composites/composites.h b/src/libserver/composites/composites.h
new file mode 100644
index 0000000..5d58029
--- /dev/null
+++ b/src/libserver/composites/composites.h
@@ -0,0 +1,64 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBSERVER_COMPOSITES_H_
+#define SRC_LIBSERVER_COMPOSITES_H_
+
+#include "config.h"
+#include "contrib/libucl/ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_config;
+
+/**
+ * Process all results and form composite metrics from existent metrics as it is defined in config
+ * @param task worker's task that present message from user
+ */
+void rspamd_composites_process_task(struct rspamd_task *task);
+
+/**
+ * Creates a composites manager
+ * @param cfg
+ * @return
+ */
+void *rspamd_composites_manager_create(struct rspamd_config *cfg);
+/**
+ * Returns number of elements in a composite manager
+ * @return
+ */
+gsize rspamd_composites_manager_nelts(void *);
+/**
+ * Adds a composite from config
+ * @return
+ */
+void *rspamd_composites_manager_add_from_ucl(void *, const char *, const ucl_object_t *);
+void *rspamd_composites_manager_add_from_ucl_silent(void *, const char *, const ucl_object_t *);
+
+/**
+ * Adds a composite from config
+ * @return
+ */
+void *rspamd_composites_manager_add_from_string(void *, const char *, const char *);
+void *rspamd_composites_manager_add_from_string_silent(void *, const char *, const char *);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBSERVER_COMPOSITES_H_ */
diff --git a/src/libserver/composites/composites_internal.hxx b/src/libserver/composites/composites_internal.hxx
new file mode 100644
index 0000000..038e217
--- /dev/null
+++ b/src/libserver/composites/composites_internal.hxx
@@ -0,0 +1,112 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_COMPOSITES_INTERNAL_HXX
+#define RSPAMD_COMPOSITES_INTERNAL_HXX
+#pragma once
+
+#include <string>
+#include "libutil/expression.h"
+#include "libutil/cxx/hash_util.hxx"
+#include "libserver/cfg_file.h"
+
+namespace rspamd::composites {
+
+/**
+ * Subr for composite expressions
+ */
+extern const struct rspamd_atom_subr composite_expr_subr;
+
+enum class rspamd_composite_policy {
+ RSPAMD_COMPOSITE_POLICY_REMOVE_ALL = 0,
+ RSPAMD_COMPOSITE_POLICY_REMOVE_SYMBOL,
+ RSPAMD_COMPOSITE_POLICY_REMOVE_WEIGHT,
+ RSPAMD_COMPOSITE_POLICY_LEAVE,
+ RSPAMD_COMPOSITE_POLICY_UNKNOWN
+};
+
+/**
+ * Static composites structure
+ */
+struct rspamd_composite {
+ std::string str_expr;
+ std::string sym;
+ struct rspamd_expression *expr;
+ gint id;
+ rspamd_composite_policy policy;
+};
+
+#define COMPOSITE_MANAGER_FROM_PTR(ptr) (reinterpret_cast<rspamd::composites::composites_manager *>(ptr))
+
+class composites_manager {
+public:
+ composites_manager(struct rspamd_config *_cfg)
+ : cfg(_cfg)
+ {
+ rspamd_mempool_add_destructor(_cfg->cfg_pool, composites_manager_dtor, this);
+ }
+
+ auto size(void) const -> std::size_t
+ {
+ return all_composites.size();
+ }
+
+ auto find(std::string_view name) const -> const rspamd_composite *
+ {
+ auto found = composites.find(std::string(name));
+
+ if (found != composites.end()) {
+ return found->second.get();
+ }
+
+ return nullptr;
+ }
+
+ auto add_composite(std::string_view, const ucl_object_t *, bool silent_duplicate) -> rspamd_composite *;
+ auto add_composite(std::string_view name, std::string_view expression, bool silent_duplicate, double score = NAN) -> rspamd_composite *;
+
+private:
+ ~composites_manager() = default;
+ static void composites_manager_dtor(void *ptr)
+ {
+ delete COMPOSITE_MANAGER_FROM_PTR(ptr);
+ }
+
+ auto new_composite(std::string_view composite_name, rspamd_expression *expr,
+ std::string_view composite_expression) -> auto
+ {
+ auto &composite = all_composites.emplace_back(std::make_shared<rspamd_composite>());
+ composite->expr = expr;
+ composite->id = all_composites.size() - 1;
+ composite->str_expr = composite_expression;
+ composite->sym = composite_name;
+
+ composites[composite->sym] = composite;
+
+ return composite;
+ }
+
+ ankerl::unordered_dense::map<std::string,
+ std::shared_ptr<rspamd_composite>, rspamd::smart_str_hash, rspamd::smart_str_equal>
+ composites;
+ /* Store all composites here, even if we have duplicates */
+ std::vector<std::shared_ptr<rspamd_composite>> all_composites;
+ struct rspamd_config *cfg;
+};
+
+}// namespace rspamd::composites
+
+#endif//RSPAMD_COMPOSITES_INTERNAL_HXX
diff --git a/src/libserver/composites/composites_manager.cxx b/src/libserver/composites/composites_manager.cxx
new file mode 100644
index 0000000..1ee5c40
--- /dev/null
+++ b/src/libserver/composites/composites_manager.cxx
@@ -0,0 +1,330 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+#include <vector>
+#include <cmath>
+#include "contrib/ankerl/unordered_dense.h"
+
+#include "composites.h"
+#include "composites_internal.hxx"
+#include "libserver/cfg_file.h"
+#include "libserver/logger.h"
+#include "libserver/maps/map.h"
+#include "libutil/cxx/util.hxx"
+
+namespace rspamd::composites {
+
+static auto
+composite_policy_from_str(const std::string_view &inp) -> enum rspamd_composite_policy {
+ const static ankerl::unordered_dense::map<std::string_view,
+ enum rspamd_composite_policy>
+ names{
+ {"remove", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_ALL},
+ {"remove_all", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_ALL},
+ {"default", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_ALL},
+ {"remove_symbol", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_SYMBOL},
+ {"remove_weight", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_WEIGHT},
+ {"leave", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_LEAVE},
+ {"remove_none", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_LEAVE},
+ };
+
+ auto found = names.find(inp);
+ if (found != names.end()){
+ return found->second;}
+
+return rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_UNKNOWN;
+}// namespace rspamd::composites
+
+auto composites_manager::add_composite(std::string_view composite_name, const ucl_object_t *obj, bool silent_duplicate) -> rspamd_composite *
+{
+
+ const auto *val = ucl_object_lookup(obj, "enabled");
+ if (val != nullptr && !ucl_object_toboolean(val)) {
+ msg_info_config("composite %s is disabled", composite_name.data());
+ return nullptr;
+ }
+
+ if (composites.contains(composite_name)) {
+ if (silent_duplicate) {
+ msg_debug_config("composite %s is redefined", composite_name.data());
+ return nullptr;
+ }
+ else {
+ msg_warn_config("composite %s is redefined", composite_name.data());
+ }
+ }
+
+ const char *composite_expression = nullptr;
+ val = ucl_object_lookup(obj, "expression");
+
+ if (val == NULL || !ucl_object_tostring_safe(val, &composite_expression)) {
+ msg_err_config("composite must have an expression defined in %s",
+ composite_name.data());
+ return nullptr;
+ }
+
+ GError *err = nullptr;
+ rspamd_expression *expr = nullptr;
+
+ if (!rspamd_parse_expression(composite_expression, 0, &composite_expr_subr,
+ NULL, cfg->cfg_pool, &err, &expr)) {
+ msg_err_config("cannot parse composite expression for %s: %e",
+ composite_name.data(), err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return nullptr;
+ }
+
+ const auto &composite = new_composite(composite_name, expr, composite_expression);
+
+ auto score = std::isnan(cfg->unknown_weight) ? 0.0 : cfg->unknown_weight;
+ val = ucl_object_lookup(obj, "score");
+
+ if (val != nullptr) {
+ ucl_object_todouble_safe(val, &score);
+ }
+
+ /* Also set score in the metric */
+ const auto *group = "composite";
+ val = ucl_object_lookup(obj, "group");
+ if (val != nullptr) {
+ group = ucl_object_tostring(val);
+ }
+
+ const auto *description = composite_expression;
+ val = ucl_object_lookup(obj, "description");
+ if (val != nullptr) {
+ description = ucl_object_tostring(val);
+ }
+
+ rspamd_config_add_symbol(cfg, composite_name.data(), score,
+ description, group,
+ 0,
+ ucl_object_get_priority(obj), /* No +1 as it is default... */
+ 1);
+
+ const auto *elt = ucl_object_lookup(obj, "groups");
+ if (elt && ucl_object_type(elt) == UCL_ARRAY) {
+ const ucl_object_t *cur_gr;
+ auto *gr_it = ucl_object_iterate_new(elt);
+
+ while ((cur_gr = ucl_object_iterate_safe(gr_it, true)) != nullptr) {
+ rspamd_config_add_symbol_group(cfg, composite_name.data(),
+ ucl_object_tostring(cur_gr));
+ }
+
+ ucl_object_iterate_free(gr_it);
+ }
+
+ val = ucl_object_lookup(obj, "policy");
+ if (val) {
+ composite->policy = composite_policy_from_str(ucl_object_tostring(val));
+
+ if (composite->policy == rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_UNKNOWN) {
+ msg_err_config("composite %s has incorrect policy", composite_name.data());
+ return nullptr;
+ }
+ }
+
+ return composite.get();
+}
+
+auto composites_manager::add_composite(std::string_view composite_name,
+ std::string_view composite_expression,
+ bool silent_duplicate, double score) -> rspamd_composite *
+{
+ GError *err = nullptr;
+ rspamd_expression *expr = nullptr;
+
+ if (composites.contains(composite_name)) {
+ /* Duplicate composite - refuse to add */
+ if (silent_duplicate) {
+ msg_debug_config("composite %s is redefined", composite_name.data());
+ return nullptr;
+ }
+ else {
+ msg_warn_config("composite %s is redefined", composite_name.data());
+ }
+ }
+
+ if (!rspamd_parse_expression(composite_expression.data(),
+ composite_expression.size(), &composite_expr_subr,
+ nullptr, cfg->cfg_pool, &err, &expr)) {
+ msg_err_config("cannot parse composite expression for %s: %e",
+ composite_name.data(), err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return nullptr;
+ }
+
+ auto final_score = std::isnan(score) ? (std::isnan(cfg->unknown_weight) ? 0.0 : cfg->unknown_weight) : score;
+ rspamd_config_add_symbol(cfg, composite_name.data(), final_score,
+ composite_name.data(), "composite",
+ 0,
+ 0,
+ 1);
+
+ return new_composite(composite_name, expr, composite_expression).get();
+}
+
+struct map_cbdata {
+ composites_manager *cm;
+ struct rspamd_config *cfg;
+ std::string buf;
+
+ explicit map_cbdata(struct rspamd_config *cfg)
+ : cfg(cfg)
+ {
+ cm = COMPOSITE_MANAGER_FROM_PTR(cfg->composites_manager);
+ }
+
+ static char *map_read(char *chunk, int len,
+ struct map_cb_data *data,
+ gboolean _final)
+ {
+
+ if (data->cur_data == nullptr) {
+ data->cur_data = data->prev_data;
+ reinterpret_cast<map_cbdata *>(data->cur_data)->buf.clear();
+ }
+
+ auto *cbd = reinterpret_cast<map_cbdata *>(data->cur_data);
+
+ cbd->buf.append(chunk, len);
+ return nullptr;
+ }
+
+ static void
+ map_fin(struct map_cb_data *data, void **target)
+ {
+ auto *cbd = reinterpret_cast<map_cbdata *>(data->cur_data);
+
+ if (data->errored) {
+ if (cbd) {
+ cbd->buf.clear();
+ }
+ }
+ else if (cbd != nullptr) {
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ rspamd::string_foreach_line(cbd->buf, [&](std::string_view line) {
+ auto [name_and_score, expr] = rspamd::string_split_on(line, ' ');
+ auto [name, score] = rspamd::string_split_on(name_and_score, ':');
+
+ if (!score.empty()) {
+ /* I wish it was supported properly */
+ //auto conv_res = std::from_chars(value->data(), value->size(), num);
+ char numbuf[128], *endptr = nullptr;
+ rspamd_strlcpy(numbuf, score.data(), MIN(score.size(), sizeof(numbuf)));
+ auto num = g_ascii_strtod(numbuf, &endptr);
+
+ if (fabs(num) >= G_MAXFLOAT || std::isnan(num)) {
+ msg_err("invalid score for %*s", (int) name_and_score.size(), name_and_score.data());
+ return;
+ }
+
+ auto ret = cbd->cm->add_composite(name, expr, true, num);
+
+ if (ret == nullptr) {
+ msg_err("cannot add composite %*s", (int) name_and_score.size(), name_and_score.data());
+ return;
+ }
+ }
+ else {
+ msg_err("missing score for %*s", (int) name_and_score.size(), name_and_score.data());
+ return;
+ }
+ });
+ }
+ else {
+ msg_err("no data read for composites map");
+ }
+ }
+
+ static void
+ map_dtor(struct map_cb_data *data)
+ {
+ auto *cbd = reinterpret_cast<map_cbdata *>(data->cur_data);
+ delete cbd;
+ }
+};
+}
+
+
+void *
+rspamd_composites_manager_create(struct rspamd_config *cfg)
+{
+ auto *cm = new rspamd::composites::composites_manager(cfg);
+
+ return reinterpret_cast<void *>(cm);
+}
+
+
+gsize rspamd_composites_manager_nelts(void *ptr)
+{
+ return COMPOSITE_MANAGER_FROM_PTR(ptr)->size();
+}
+
+void *
+rspamd_composites_manager_add_from_ucl(void *cm, const char *sym, const ucl_object_t *obj)
+{
+ return reinterpret_cast<void *>(COMPOSITE_MANAGER_FROM_PTR(cm)->add_composite(sym, obj, false));
+}
+
+void *
+rspamd_composites_manager_add_from_string(void *cm, const char *sym, const char *expr)
+{
+ return reinterpret_cast<void *>(COMPOSITE_MANAGER_FROM_PTR(cm)->add_composite(sym, expr, false));
+}
+
+void *
+rspamd_composites_manager_add_from_ucl_silent(void *cm, const char *sym, const ucl_object_t *obj)
+{
+ return reinterpret_cast<void *>(COMPOSITE_MANAGER_FROM_PTR(cm)->add_composite(sym, obj, true));
+}
+
+void *
+rspamd_composites_manager_add_from_string_silent(void *cm, const char *sym, const char *expr)
+{
+ return reinterpret_cast<void *>(COMPOSITE_MANAGER_FROM_PTR(cm)->add_composite(sym, expr, true));
+}
+
+
+bool rspamd_composites_add_map_handlers(const ucl_object_t *obj, struct rspamd_config *cfg)
+{
+ auto **pcbdata = rspamd_mempool_alloc_type(cfg->cfg_pool, rspamd::composites::map_cbdata *);
+ auto *cbdata = new rspamd::composites::map_cbdata{cfg};
+ *pcbdata = cbdata;
+
+ if (struct rspamd_map * m; (m = rspamd_map_add_from_ucl(cfg, obj, "composites map",
+ rspamd::composites::map_cbdata::map_read, rspamd::composites::map_cbdata::map_fin,
+ rspamd::composites::map_cbdata::map_dtor, (void **) pcbdata,
+ nullptr, RSPAMD_MAP_DEFAULT)) == nullptr) {
+ msg_err_config("cannot load composites map from %s", ucl_object_key(obj));
+ return false;
+ }
+
+ return true;
+} \ No newline at end of file
diff --git a/src/libserver/css/CMakeLists.txt b/src/libserver/css/CMakeLists.txt
new file mode 100644
index 0000000..c0c9d51
--- /dev/null
+++ b/src/libserver/css/CMakeLists.txt
@@ -0,0 +1,9 @@
+SET(LIBCSSSRC "${CMAKE_CURRENT_SOURCE_DIR}/css.cxx"
+ "${CMAKE_CURRENT_SOURCE_DIR}/css_property.cxx"
+ "${CMAKE_CURRENT_SOURCE_DIR}/css_value.cxx"
+ "${CMAKE_CURRENT_SOURCE_DIR}/css_selector.cxx"
+ "${CMAKE_CURRENT_SOURCE_DIR}/css_tokeniser.cxx"
+ "${CMAKE_CURRENT_SOURCE_DIR}/css_util.cxx"
+ "${CMAKE_CURRENT_SOURCE_DIR}/css_rule.cxx"
+ "${CMAKE_CURRENT_SOURCE_DIR}/css_parser.cxx"
+ PARENT_SCOPE) \ No newline at end of file
diff --git a/src/libserver/css/css.cxx b/src/libserver/css/css.cxx
new file mode 100644
index 0000000..1b369ed
--- /dev/null
+++ b/src/libserver/css/css.cxx
@@ -0,0 +1,227 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "css.hxx"
+#include "contrib/ankerl/unordered_dense.h"
+#include "css_parser.hxx"
+#include "libserver/html/html_tag.hxx"
+#include "libserver/html/html_block.hxx"
+
+/* Keep unit tests implementation here (it'll possibly be moved outside one day) */
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#define DOCTEST_CONFIG_IMPLEMENT
+#include "doctest/doctest.h"
+
+namespace rspamd::css {
+
+INIT_LOG_MODULE_PUBLIC(css);
+
+class css_style_sheet::impl {
+public:
+ using sel_shared_hash = smart_ptr_hash<css_selector>;
+ using sel_shared_eq = smart_ptr_equal<css_selector>;
+ using selector_ptr = std::unique_ptr<css_selector>;
+ using selectors_hash = ankerl::unordered_dense::map<selector_ptr, css_declarations_block_ptr,
+ sel_shared_hash, sel_shared_eq>;
+ using universal_selector_t = std::pair<selector_ptr, css_declarations_block_ptr>;
+ selectors_hash tags_selector;
+ selectors_hash class_selectors;
+ selectors_hash id_selectors;
+ std::optional<universal_selector_t> universal_selector;
+};
+
+css_style_sheet::css_style_sheet(rspamd_mempool_t *pool)
+ : pool(pool), pimpl(new impl)
+{
+}
+css_style_sheet::~css_style_sheet()
+{
+}
+
+auto css_style_sheet::add_selector_rule(std::unique_ptr<css_selector> &&selector,
+ css_declarations_block_ptr decls) -> void
+{
+ impl::selectors_hash *target_hash = nullptr;
+
+ switch (selector->type) {
+ case css_selector::selector_type::SELECTOR_ALL:
+ if (pimpl->universal_selector) {
+ /* Another universal selector */
+ msg_debug_css("redefined universal selector, merging rules");
+ pimpl->universal_selector->second->merge_block(*decls);
+ }
+ else {
+ msg_debug_css("added universal selector");
+ pimpl->universal_selector = std::make_pair(std::move(selector),
+ decls);
+ }
+ break;
+ case css_selector::selector_type::SELECTOR_CLASS:
+ target_hash = &pimpl->class_selectors;
+ break;
+ case css_selector::selector_type::SELECTOR_ID:
+ target_hash = &pimpl->id_selectors;
+ break;
+ case css_selector::selector_type::SELECTOR_TAG:
+ target_hash = &pimpl->tags_selector;
+ break;
+ }
+
+ if (target_hash) {
+ auto found_it = target_hash->find(selector);
+
+ if (found_it == target_hash->end()) {
+ /* Easy case, new element */
+ target_hash->insert({std::move(selector), decls});
+ }
+ else {
+ /* The problem with merging is actually in how to handle selectors chains
+ * For example, we have 2 selectors:
+ * 1. class id tag -> meaning that we first match class, then we ensure that
+ * id is also the same and finally we check the tag
+ * 2. tag class id -> it means that we check first tag, then class and then id
+ * So we have somehow equal path in the xpath terms.
+ * I suppose now, that we merely check parent stuff and handle duplicates
+ * merging when finally resolving paths.
+ */
+ auto sel_str = selector->to_string().value_or("unknown");
+ msg_debug_css("found duplicate selector: %*s", (int) sel_str.size(),
+ sel_str.data());
+ found_it->second->merge_block(*decls);
+ }
+ }
+}
+
+auto css_style_sheet::check_tag_block(const rspamd::html::html_tag *tag) -> rspamd::html::html_block *
+{
+ std::optional<std::string_view> id_comp, class_comp;
+ rspamd::html::html_block *res = nullptr;
+
+ if (!tag) {
+ return nullptr;
+ }
+
+ /* First, find id in a tag and a class */
+ for (const auto &param: tag->components) {
+ if (param.type == html::html_component_type::RSPAMD_HTML_COMPONENT_ID) {
+ id_comp = param.value;
+ }
+ else if (param.type == html::html_component_type::RSPAMD_HTML_COMPONENT_CLASS) {
+ class_comp = param.value;
+ }
+ }
+
+ /* ID part */
+ if (id_comp && !pimpl->id_selectors.empty()) {
+ auto found_id_sel = pimpl->id_selectors.find(css_selector{id_comp.value()});
+
+ if (found_id_sel != pimpl->id_selectors.end()) {
+ const auto &decl = *(found_id_sel->second);
+ res = decl.compile_to_block(pool);
+ }
+ }
+
+ /* Class part */
+ if (class_comp && !pimpl->class_selectors.empty()) {
+ auto sv_split = [](auto strv, std::string_view delims = " ") -> std::vector<std::string_view> {
+ std::vector<decltype(strv)> ret;
+ std::size_t start = 0;
+
+ while (start < strv.size()) {
+ const auto last = strv.find_first_of(delims, start);
+ if (start != last) {
+ ret.emplace_back(strv.substr(start, last - start));
+ }
+
+ if (last == std::string_view::npos) {
+ break;
+ }
+
+ start = last + 1;
+ }
+
+ return ret;
+ };
+
+ auto elts = sv_split(class_comp.value());
+
+ for (const auto &e: elts) {
+ auto found_class_sel = pimpl->class_selectors.find(
+ css_selector{e, css_selector::selector_type::SELECTOR_CLASS});
+
+ if (found_class_sel != pimpl->class_selectors.end()) {
+ const auto &decl = *(found_class_sel->second);
+ auto *tmp = decl.compile_to_block(pool);
+
+ if (res == nullptr) {
+ res = tmp;
+ }
+ else {
+ res->propagate_block(*tmp);
+ }
+ }
+ }
+ }
+
+ /* Tags part */
+ if (!pimpl->tags_selector.empty()) {
+ auto found_tag_sel = pimpl->tags_selector.find(
+ css_selector{static_cast<tag_id_t>(tag->id)});
+
+ if (found_tag_sel != pimpl->tags_selector.end()) {
+ const auto &decl = *(found_tag_sel->second);
+ auto *tmp = decl.compile_to_block(pool);
+
+ if (res == nullptr) {
+ res = tmp;
+ }
+ else {
+ res->propagate_block(*tmp);
+ }
+ }
+ }
+
+ /* Finally, universal selector */
+ if (pimpl->universal_selector) {
+ auto *tmp = pimpl->universal_selector->second->compile_to_block(pool);
+
+ if (res == nullptr) {
+ res = tmp;
+ }
+ else {
+ res->propagate_block(*tmp);
+ }
+ }
+
+ return res;
+}
+
+auto css_parse_style(rspamd_mempool_t *pool,
+ std::string_view input,
+ std::shared_ptr<css_style_sheet> &&existing)
+ -> css_return_pair
+{
+ auto parse_res = rspamd::css::parse_css(pool, input,
+ std::forward<std::shared_ptr<css_style_sheet>>(existing));
+
+ if (parse_res.has_value()) {
+ return std::make_pair(parse_res.value(), css_parse_error());
+ }
+
+ return std::make_pair(nullptr, parse_res.error());
+}
+
+}// namespace rspamd::css \ No newline at end of file
diff --git a/src/libserver/css/css.hxx b/src/libserver/css/css.hxx
new file mode 100644
index 0000000..f0f8120
--- /dev/null
+++ b/src/libserver/css/css.hxx
@@ -0,0 +1,68 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#ifndef RSPAMD_CSS_HXX
+#define RSPAMD_CSS_HXX
+
+#include <string>
+#include <memory>
+#include "logger.h"
+#include "css_rule.hxx"
+#include "css_selector.hxx"
+
+namespace rspamd::html {
+/* Forward declaration */
+struct html_tag;
+struct html_block;
+}// namespace rspamd::html
+
+namespace rspamd::css {
+
+extern int rspamd_css_log_id;
+
+#define msg_debug_css(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_css_log_id, "css", pool->tag.uid, \
+ __FUNCTION__, \
+ __VA_ARGS__)
+#define msg_err_css(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "css", pool->tag.uid, \
+ __FUNCTION__, \
+ __VA_ARGS__)
+
+class css_style_sheet {
+public:
+ css_style_sheet(rspamd_mempool_t *pool);
+ ~css_style_sheet(); /* must be declared separately due to pimpl */
+ auto add_selector_rule(std::unique_ptr<css_selector> &&selector,
+ css_declarations_block_ptr decls) -> void;
+
+ auto check_tag_block(const rspamd::html::html_tag *tag) -> rspamd::html::html_block *;
+
+private:
+ class impl;
+ rspamd_mempool_t *pool;
+ std::unique_ptr<impl> pimpl;
+};
+
+using css_return_pair = std::pair<std::shared_ptr<css_style_sheet>, css_parse_error>;
+auto css_parse_style(rspamd_mempool_t *pool,
+ std::string_view input,
+ std::shared_ptr<css_style_sheet> &&existing) -> css_return_pair;
+
+}// namespace rspamd::css
+
+#endif//RSPAMD_CSS_H \ No newline at end of file
diff --git a/src/libserver/css/css_colors_list.hxx b/src/libserver/css/css_colors_list.hxx
new file mode 100644
index 0000000..6dfe54f
--- /dev/null
+++ b/src/libserver/css/css_colors_list.hxx
@@ -0,0 +1,738 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_CSS_COLORS_LIST_HXX
+#define RSPAMD_CSS_COLORS_LIST_HXX
+
+#pragma once
+
+#include <string_view>
+#include "contrib/ankerl/unordered_dense.h"
+#include "css_value.hxx"
+
+namespace rspamd::css {
+
+/*
+ * List of all colors, intended to use with hashes/sets
+ * TODO: think about frozen structs when we can deal with 700 values without
+ * compiler limits...
+ */
+static const ankerl::unordered_dense::map<std::string_view, css_color> css_colors_map{
+ {"aliceblue", {240, 248, 255}},
+ {"antiquewhite", {250, 235, 215}},
+ {"antiquewhite1", {255, 239, 219}},
+ {"antiquewhite2", {238, 223, 204}},
+ {"antiquewhite3", {205, 192, 176}},
+ {"antiquewhite4", {139, 131, 120}},
+ {"aqua", {0, 255, 255}},
+ {"aquamarine", {127, 255, 212}},
+ {"aquamarine1", {127, 255, 212}},
+ {"aquamarine2", {118, 238, 198}},
+ {"aquamarine3", {102, 205, 170}},
+ {"aquamarine4", {69, 139, 116}},
+ {"azure", {240, 255, 255}},
+ {"azure1", {240, 255, 255}},
+ {"azure2", {224, 238, 238}},
+ {"azure3", {193, 205, 205}},
+ {"azure4", {131, 139, 139}},
+ {"beige", {245, 245, 220}},
+ {"bisque", {255, 228, 196}},
+ {"bisque1", {255, 228, 196}},
+ {"bisque2", {238, 213, 183}},
+ {"bisque3", {205, 183, 158}},
+ {"bisque4", {139, 125, 107}},
+ {"black", {0, 0, 0}},
+ {"blanchedalmond", {255, 235, 205}},
+ {"blue", {0, 0, 255}},
+ {"blue1", {0, 0, 255}},
+ {"blue2", {0, 0, 238}},
+ {"blue3", {0, 0, 205}},
+ {"blue4", {0, 0, 139}},
+ {"blueviolet", {138, 43, 226}},
+ {"brown", {165, 42, 42}},
+ {"brown1", {255, 64, 64}},
+ {"brown2", {238, 59, 59}},
+ {"brown3", {205, 51, 51}},
+ {"brown4", {139, 35, 35}},
+ {"burlywood", {222, 184, 135}},
+ {"burlywood1", {255, 211, 155}},
+ {"burlywood2", {238, 197, 145}},
+ {"burlywood3", {205, 170, 125}},
+ {"burlywood4", {139, 115, 85}},
+ {"cadetblue", {95, 158, 160}},
+ {"cadetblue1", {152, 245, 255}},
+ {"cadetblue2", {142, 229, 238}},
+ {"cadetblue3", {122, 197, 205}},
+ {"cadetblue4", {83, 134, 139}},
+ {"chartreuse", {127, 255, 0}},
+ {"chartreuse1", {127, 255, 0}},
+ {"chartreuse2", {118, 238, 0}},
+ {"chartreuse3", {102, 205, 0}},
+ {"chartreuse4", {69, 139, 0}},
+ {"chocolate", {210, 105, 30}},
+ {"chocolate1", {255, 127, 36}},
+ {"chocolate2", {238, 118, 33}},
+ {"chocolate3", {205, 102, 29}},
+ {"chocolate4", {139, 69, 19}},
+ {"coral", {255, 127, 80}},
+ {"coral1", {255, 114, 86}},
+ {"coral2", {238, 106, 80}},
+ {"coral3", {205, 91, 69}},
+ {"coral4", {139, 62, 47}},
+ {"cornflowerblue", {100, 149, 237}},
+ {"cornsilk", {255, 248, 220}},
+ {"cornsilk1", {255, 248, 220}},
+ {"cornsilk2", {238, 232, 205}},
+ {"cornsilk3", {205, 200, 177}},
+ {"cornsilk4", {139, 136, 120}},
+ {"crimson", {220, 20, 60}},
+ {"cyan", {0, 255, 255}},
+ {"cyan1", {0, 255, 255}},
+ {"cyan2", {0, 238, 238}},
+ {"cyan3", {0, 205, 205}},
+ {"cyan4", {0, 139, 139}},
+ {"darkblue", {0, 0, 139}},
+ {"darkcyan", {0, 139, 139}},
+ {"darkgoldenrod", {184, 134, 11}},
+ {"darkgoldenrod1", {255, 185, 15}},
+ {"darkgoldenrod2", {238, 173, 14}},
+ {"darkgoldenrod3", {205, 149, 12}},
+ {"darkgoldenrod4", {139, 101, 8}},
+ {"darkgray", {169, 169, 169}},
+ {"darkgreen", {0, 100, 0}},
+ {"darkgrey", {169, 169, 169}},
+ {"darkkhaki", {189, 183, 107}},
+ {"darkmagenta", {139, 0, 139}},
+ {"darkolivegreen", {85, 107, 47}},
+ {"darkolivegreen1", {202, 255, 112}},
+ {"darkolivegreen2", {188, 238, 104}},
+ {"darkolivegreen3", {162, 205, 90}},
+ {"darkolivegreen4", {110, 139, 61}},
+ {"darkorange", {255, 140, 0}},
+ {"darkorange1", {255, 127, 0}},
+ {"darkorange2", {238, 118, 0}},
+ {"darkorange3", {205, 102, 0}},
+ {"darkorange4", {139, 69, 0}},
+ {"darkorchid", {153, 50, 204}},
+ {"darkorchid1", {191, 62, 255}},
+ {"darkorchid2", {178, 58, 238}},
+ {"darkorchid3", {154, 50, 205}},
+ {"darkorchid4", {104, 34, 139}},
+ {"darkred", {139, 0, 0}},
+ {"darksalmon", {233, 150, 122}},
+ {"darkseagreen", {143, 188, 143}},
+ {"darkseagreen1", {193, 255, 193}},
+ {"darkseagreen2", {180, 238, 180}},
+ {"darkseagreen3", {155, 205, 155}},
+ {"darkseagreen4", {105, 139, 105}},
+ {"darkslateblue", {72, 61, 139}},
+ {"darkslategray", {47, 79, 79}},
+ {"darkslategray1", {151, 255, 255}},
+ {"darkslategray2", {141, 238, 238}},
+ {"darkslategray3", {121, 205, 205}},
+ {"darkslategray4", {82, 139, 139}},
+ {"darkslategrey", {47, 79, 79}},
+ {"darkturquoise", {0, 206, 209}},
+ {"darkviolet", {148, 0, 211}},
+ {"deeppink", {255, 20, 147}},
+ {"deeppink1", {255, 20, 147}},
+ {"deeppink2", {238, 18, 137}},
+ {"deeppink3", {205, 16, 118}},
+ {"deeppink4", {139, 10, 80}},
+ {"deepskyblue", {0, 191, 255}},
+ {"deepskyblue1", {0, 191, 255}},
+ {"deepskyblue2", {0, 178, 238}},
+ {"deepskyblue3", {0, 154, 205}},
+ {"deepskyblue4", {0, 104, 139}},
+ {"dimgray", {105, 105, 105}},
+ {"dimgrey", {105, 105, 105}},
+ {"dodgerblue", {30, 144, 255}},
+ {"dodgerblue1", {30, 144, 255}},
+ {"dodgerblue2", {28, 134, 238}},
+ {"dodgerblue3", {24, 116, 205}},
+ {"dodgerblue4", {16, 78, 139}},
+ {"firebrick", {178, 34, 34}},
+ {"firebrick1", {255, 48, 48}},
+ {"firebrick2", {238, 44, 44}},
+ {"firebrick3", {205, 38, 38}},
+ {"firebrick4", {139, 26, 26}},
+ {"floralwhite", {255, 250, 240}},
+ {"forestgreen", {34, 139, 34}},
+ {"fuchsia", {255, 0, 255}},
+ {"gainsboro", {220, 220, 220}},
+ {"ghostwhite", {248, 248, 255}},
+ {"gold", {255, 215, 0}},
+ {"gold1", {255, 215, 0}},
+ {"gold2", {238, 201, 0}},
+ {"gold3", {205, 173, 0}},
+ {"gold4", {139, 117, 0}},
+ {"goldenrod", {218, 165, 32}},
+ {"goldenrod1", {255, 193, 37}},
+ {"goldenrod2", {238, 180, 34}},
+ {"goldenrod3", {205, 155, 29}},
+ {"goldenrod4", {139, 105, 20}},
+ {"gray", {190, 190, 190}},
+ {"gray0", {0, 0, 0}},
+ {"gray1", {3, 3, 3}},
+ {"gray10", {26, 26, 26}},
+ {"gray100", {255, 255, 255}},
+ {"gray11", {28, 28, 28}},
+ {"gray12", {31, 31, 31}},
+ {"gray13", {33, 33, 33}},
+ {"gray14", {36, 36, 36}},
+ {"gray15", {38, 38, 38}},
+ {"gray16", {41, 41, 41}},
+ {"gray17", {43, 43, 43}},
+ {"gray18", {46, 46, 46}},
+ {"gray19", {48, 48, 48}},
+ {"gray2", {5, 5, 5}},
+ {"gray20", {51, 51, 51}},
+ {"gray21", {54, 54, 54}},
+ {"gray22", {56, 56, 56}},
+ {"gray23", {59, 59, 59}},
+ {"gray24", {61, 61, 61}},
+ {"gray25", {64, 64, 64}},
+ {"gray26", {66, 66, 66}},
+ {"gray27", {69, 69, 69}},
+ {"gray28", {71, 71, 71}},
+ {"gray29", {74, 74, 74}},
+ {"gray3", {8, 8, 8}},
+ {"gray30", {77, 77, 77}},
+ {"gray31", {79, 79, 79}},
+ {"gray32", {82, 82, 82}},
+ {"gray33", {84, 84, 84}},
+ {"gray34", {87, 87, 87}},
+ {"gray35", {89, 89, 89}},
+ {"gray36", {92, 92, 92}},
+ {"gray37", {94, 94, 94}},
+ {"gray38", {97, 97, 97}},
+ {"gray39", {99, 99, 99}},
+ {"gray4", {10, 10, 10}},
+ {"gray40", {102, 102, 102}},
+ {"gray41", {105, 105, 105}},
+ {"gray42", {107, 107, 107}},
+ {"gray43", {110, 110, 110}},
+ {"gray44", {112, 112, 112}},
+ {"gray45", {115, 115, 115}},
+ {"gray46", {117, 117, 117}},
+ {"gray47", {120, 120, 120}},
+ {"gray48", {122, 122, 122}},
+ {"gray49", {125, 125, 125}},
+ {"gray5", {13, 13, 13}},
+ {"gray50", {127, 127, 127}},
+ {"gray51", {130, 130, 130}},
+ {"gray52", {133, 133, 133}},
+ {"gray53", {135, 135, 135}},
+ {"gray54", {138, 138, 138}},
+ {"gray55", {140, 140, 140}},
+ {"gray56", {143, 143, 143}},
+ {"gray57", {145, 145, 145}},
+ {"gray58", {148, 148, 148}},
+ {"gray59", {150, 150, 150}},
+ {"gray6", {15, 15, 15}},
+ {"gray60", {153, 153, 153}},
+ {"gray61", {156, 156, 156}},
+ {"gray62", {158, 158, 158}},
+ {"gray63", {161, 161, 161}},
+ {"gray64", {163, 163, 163}},
+ {"gray65", {166, 166, 166}},
+ {"gray66", {168, 168, 168}},
+ {"gray67", {171, 171, 171}},
+ {"gray68", {173, 173, 173}},
+ {"gray69", {176, 176, 176}},
+ {"gray7", {18, 18, 18}},
+ {"gray70", {179, 179, 179}},
+ {"gray71", {181, 181, 181}},
+ {"gray72", {184, 184, 184}},
+ {"gray73", {186, 186, 186}},
+ {"gray74", {189, 189, 189}},
+ {"gray75", {191, 191, 191}},
+ {"gray76", {194, 194, 194}},
+ {"gray77", {196, 196, 196}},
+ {"gray78", {199, 199, 199}},
+ {"gray79", {201, 201, 201}},
+ {"gray8", {20, 20, 20}},
+ {"gray80", {204, 204, 204}},
+ {"gray81", {207, 207, 207}},
+ {"gray82", {209, 209, 209}},
+ {"gray83", {212, 212, 212}},
+ {"gray84", {214, 214, 214}},
+ {"gray85", {217, 217, 217}},
+ {"gray86", {219, 219, 219}},
+ {"gray87", {222, 222, 222}},
+ {"gray88", {224, 224, 224}},
+ {"gray89", {227, 227, 227}},
+ {"gray9", {23, 23, 23}},
+ {"gray90", {229, 229, 229}},
+ {"gray91", {232, 232, 232}},
+ {"gray92", {235, 235, 235}},
+ {"gray93", {237, 237, 237}},
+ {"gray94", {240, 240, 240}},
+ {"gray95", {242, 242, 242}},
+ {"gray96", {245, 245, 245}},
+ {"gray97", {247, 247, 247}},
+ {"gray98", {250, 250, 250}},
+ {"gray99", {252, 252, 252}},
+ {"green", {0, 255, 0}},
+ {"green1", {0, 255, 0}},
+ {"green2", {0, 238, 0}},
+ {"green3", {0, 205, 0}},
+ {"green4", {0, 139, 0}},
+ {"greenyellow", {173, 255, 47}},
+ {"grey", {190, 190, 190}},
+ {"grey0", {0, 0, 0}},
+ {"grey1", {3, 3, 3}},
+ {"grey10", {26, 26, 26}},
+ {"grey100", {255, 255, 255}},
+ {"grey11", {28, 28, 28}},
+ {"grey12", {31, 31, 31}},
+ {"grey13", {33, 33, 33}},
+ {"grey14", {36, 36, 36}},
+ {"grey15", {38, 38, 38}},
+ {"grey16", {41, 41, 41}},
+ {"grey17", {43, 43, 43}},
+ {"grey18", {46, 46, 46}},
+ {"grey19", {48, 48, 48}},
+ {"grey2", {5, 5, 5}},
+ {"grey20", {51, 51, 51}},
+ {"grey21", {54, 54, 54}},
+ {"grey22", {56, 56, 56}},
+ {"grey23", {59, 59, 59}},
+ {"grey24", {61, 61, 61}},
+ {"grey25", {64, 64, 64}},
+ {"grey26", {66, 66, 66}},
+ {"grey27", {69, 69, 69}},
+ {"grey28", {71, 71, 71}},
+ {"grey29", {74, 74, 74}},
+ {"grey3", {8, 8, 8}},
+ {"grey30", {77, 77, 77}},
+ {"grey31", {79, 79, 79}},
+ {"grey32", {82, 82, 82}},
+ {"grey33", {84, 84, 84}},
+ {"grey34", {87, 87, 87}},
+ {"grey35", {89, 89, 89}},
+ {"grey36", {92, 92, 92}},
+ {"grey37", {94, 94, 94}},
+ {"grey38", {97, 97, 97}},
+ {"grey39", {99, 99, 99}},
+ {"grey4", {10, 10, 10}},
+ {"grey40", {102, 102, 102}},
+ {"grey41", {105, 105, 105}},
+ {"grey42", {107, 107, 107}},
+ {"grey43", {110, 110, 110}},
+ {"grey44", {112, 112, 112}},
+ {"grey45", {115, 115, 115}},
+ {"grey46", {117, 117, 117}},
+ {"grey47", {120, 120, 120}},
+ {"grey48", {122, 122, 122}},
+ {"grey49", {125, 125, 125}},
+ {"grey5", {13, 13, 13}},
+ {"grey50", {127, 127, 127}},
+ {"grey51", {130, 130, 130}},
+ {"grey52", {133, 133, 133}},
+ {"grey53", {135, 135, 135}},
+ {"grey54", {138, 138, 138}},
+ {"grey55", {140, 140, 140}},
+ {"grey56", {143, 143, 143}},
+ {"grey57", {145, 145, 145}},
+ {"grey58", {148, 148, 148}},
+ {"grey59", {150, 150, 150}},
+ {"grey6", {15, 15, 15}},
+ {"grey60", {153, 153, 153}},
+ {"grey61", {156, 156, 156}},
+ {"grey62", {158, 158, 158}},
+ {"grey63", {161, 161, 161}},
+ {"grey64", {163, 163, 163}},
+ {"grey65", {166, 166, 166}},
+ {"grey66", {168, 168, 168}},
+ {"grey67", {171, 171, 171}},
+ {"grey68", {173, 173, 173}},
+ {"grey69", {176, 176, 176}},
+ {"grey7", {18, 18, 18}},
+ {"grey70", {179, 179, 179}},
+ {"grey71", {181, 181, 181}},
+ {"grey72", {184, 184, 184}},
+ {"grey73", {186, 186, 186}},
+ {"grey74", {189, 189, 189}},
+ {"grey75", {191, 191, 191}},
+ {"grey76", {194, 194, 194}},
+ {"grey77", {196, 196, 196}},
+ {"grey78", {199, 199, 199}},
+ {"grey79", {201, 201, 201}},
+ {"grey8", {20, 20, 20}},
+ {"grey80", {204, 204, 204}},
+ {"grey81", {207, 207, 207}},
+ {"grey82", {209, 209, 209}},
+ {"grey83", {212, 212, 212}},
+ {"grey84", {214, 214, 214}},
+ {"grey85", {217, 217, 217}},
+ {"grey86", {219, 219, 219}},
+ {"grey87", {222, 222, 222}},
+ {"grey88", {224, 224, 224}},
+ {"grey89", {227, 227, 227}},
+ {"grey9", {23, 23, 23}},
+ {"grey90", {229, 229, 229}},
+ {"grey91", {232, 232, 232}},
+ {"grey92", {235, 235, 235}},
+ {"grey93", {237, 237, 237}},
+ {"grey94", {240, 240, 240}},
+ {"grey95", {242, 242, 242}},
+ {"grey96", {245, 245, 245}},
+ {"grey97", {247, 247, 247}},
+ {"grey98", {250, 250, 250}},
+ {"grey99", {252, 252, 252}},
+ {"honeydew", {240, 255, 240}},
+ {"honeydew1", {240, 255, 240}},
+ {"honeydew2", {224, 238, 224}},
+ {"honeydew3", {193, 205, 193}},
+ {"honeydew4", {131, 139, 131}},
+ {"hotpink", {255, 105, 180}},
+ {"hotpink1", {255, 110, 180}},
+ {"hotpink2", {238, 106, 167}},
+ {"hotpink3", {205, 96, 144}},
+ {"hotpink4", {139, 58, 98}},
+ {"indianred", {205, 92, 92}},
+ {"indianred1", {255, 106, 106}},
+ {"indianred2", {238, 99, 99}},
+ {"indianred3", {205, 85, 85}},
+ {"indianred4", {139, 58, 58}},
+ {"indigo", {75, 0, 130}},
+ {"ivory", {255, 255, 240}},
+ {"ivory1", {255, 255, 240}},
+ {"ivory2", {238, 238, 224}},
+ {"ivory3", {205, 205, 193}},
+ {"ivory4", {139, 139, 131}},
+ {"khaki", {240, 230, 140}},
+ {"khaki1", {255, 246, 143}},
+ {"khaki2", {238, 230, 133}},
+ {"khaki3", {205, 198, 115}},
+ {"khaki4", {139, 134, 78}},
+ {"lavender", {230, 230, 250}},
+ {"lavenderblush", {255, 240, 245}},
+ {"lavenderblush1", {255, 240, 245}},
+ {"lavenderblush2", {238, 224, 229}},
+ {"lavenderblush3", {205, 193, 197}},
+ {"lavenderblush4", {139, 131, 134}},
+ {"lawngreen", {124, 252, 0}},
+ {"lemonchiffon", {255, 250, 205}},
+ {"lemonchiffon1", {255, 250, 205}},
+ {"lemonchiffon2", {238, 233, 191}},
+ {"lemonchiffon3", {205, 201, 165}},
+ {"lemonchiffon4", {139, 137, 112}},
+ {"lightblue", {173, 216, 230}},
+ {"lightblue1", {191, 239, 255}},
+ {"lightblue2", {178, 223, 238}},
+ {"lightblue3", {154, 192, 205}},
+ {"lightblue4", {104, 131, 139}},
+ {"lightcoral", {240, 128, 128}},
+ {"lightcyan", {224, 255, 255}},
+ {"lightcyan1", {224, 255, 255}},
+ {"lightcyan2", {209, 238, 238}},
+ {"lightcyan3", {180, 205, 205}},
+ {"lightcyan4", {122, 139, 139}},
+ {"lightgoldenrod", {238, 221, 130}},
+ {"lightgoldenrod1", {255, 236, 139}},
+ {"lightgoldenrod2", {238, 220, 130}},
+ {"lightgoldenrod3", {205, 190, 112}},
+ {"lightgoldenrod4", {139, 129, 76}},
+ {"lightgoldenrodyellow", {250, 250, 210}},
+ {"lightgray", {211, 211, 211}},
+ {"lightgreen", {144, 238, 144}},
+ {"lightgrey", {211, 211, 211}},
+ {"lightpink", {255, 182, 193}},
+ {"lightpink1", {255, 174, 185}},
+ {"lightpink2", {238, 162, 173}},
+ {"lightpink3", {205, 140, 149}},
+ {"lightpink4", {139, 95, 101}},
+ {"lightsalmon", {255, 160, 122}},
+ {"lightsalmon1", {255, 160, 122}},
+ {"lightsalmon2", {238, 149, 114}},
+ {"lightsalmon3", {205, 129, 98}},
+ {"lightsalmon4", {139, 87, 66}},
+ {"lightseagreen", {32, 178, 170}},
+ {"lightskyblue", {135, 206, 250}},
+ {"lightskyblue1", {176, 226, 255}},
+ {"lightskyblue2", {164, 211, 238}},
+ {"lightskyblue3", {141, 182, 205}},
+ {"lightskyblue4", {96, 123, 139}},
+ {"lightslateblue", {132, 112, 255}},
+ {"lightslategray", {119, 136, 153}},
+ {"lightslategrey", {119, 136, 153}},
+ {"lightsteelblue", {176, 196, 222}},
+ {"lightsteelblue1", {202, 225, 255}},
+ {"lightsteelblue2", {188, 210, 238}},
+ {"lightsteelblue3", {162, 181, 205}},
+ {"lightsteelblue4", {110, 123, 139}},
+ {"lightyellow", {255, 255, 224}},
+ {"lightyellow1", {255, 255, 224}},
+ {"lightyellow2", {238, 238, 209}},
+ {"lightyellow3", {205, 205, 180}},
+ {"lightyellow4", {139, 139, 122}},
+ {"lime", {0, 255, 0}},
+ {"limegreen", {50, 205, 50}},
+ {"linen", {250, 240, 230}},
+ {"magenta", {255, 0, 255}},
+ {"magenta1", {255, 0, 255}},
+ {"magenta2", {238, 0, 238}},
+ {"magenta3", {205, 0, 205}},
+ {"magenta4", {139, 0, 139}},
+ {"maroon", {176, 48, 96}},
+ {"maroon1", {255, 52, 179}},
+ {"maroon2", {238, 48, 167}},
+ {"maroon3", {205, 41, 144}},
+ {"maroon4", {139, 28, 98}},
+ {"mediumaquamarine", {102, 205, 170}},
+ {"mediumblue", {0, 0, 205}},
+ {"mediumorchid", {186, 85, 211}},
+ {"mediumorchid1", {224, 102, 255}},
+ {"mediumorchid2", {209, 95, 238}},
+ {"mediumorchid3", {180, 82, 205}},
+ {"mediumorchid4", {122, 55, 139}},
+ {"mediumpurple", {147, 112, 219}},
+ {"mediumpurple1", {171, 130, 255}},
+ {"mediumpurple2", {159, 121, 238}},
+ {"mediumpurple3", {137, 104, 205}},
+ {"mediumpurple4", {93, 71, 139}},
+ {"mediumseagreen", {60, 179, 113}},
+ {"mediumslateblue", {123, 104, 238}},
+ {"mediumspringgreen", {0, 250, 154}},
+ {"mediumturquoise", {72, 209, 204}},
+ {"mediumvioletred", {199, 21, 133}},
+ {"midnightblue", {25, 25, 112}},
+ {"mintcream", {245, 255, 250}},
+ {"mistyrose", {255, 228, 225}},
+ {"mistyrose1", {255, 228, 225}},
+ {"mistyrose2", {238, 213, 210}},
+ {"mistyrose3", {205, 183, 181}},
+ {"mistyrose4", {139, 125, 123}},
+ {"moccasin", {255, 228, 181}},
+ {"navajowhite", {255, 222, 173}},
+ {"navajowhite1", {255, 222, 173}},
+ {"navajowhite2", {238, 207, 161}},
+ {"navajowhite3", {205, 179, 139}},
+ {"navajowhite4", {139, 121, 94}},
+ {"navy", {0, 0, 128}},
+ {"navyblue", {0, 0, 128}},
+ {"oldlace", {253, 245, 230}},
+ {"olive", {128, 128, 0}},
+ {"olivedrab", {107, 142, 35}},
+ {"olivedrab1", {192, 255, 62}},
+ {"olivedrab2", {179, 238, 58}},
+ {"olivedrab3", {154, 205, 50}},
+ {"olivedrab4", {105, 139, 34}},
+ {"orange", {255, 165, 0}},
+ {"orange1", {255, 165, 0}},
+ {"orange2", {238, 154, 0}},
+ {"orange3", {205, 133, 0}},
+ {"orange4", {139, 90, 0}},
+ {"orangered", {255, 69, 0}},
+ {"orangered1", {255, 69, 0}},
+ {"orangered2", {238, 64, 0}},
+ {"orangered3", {205, 55, 0}},
+ {"orangered4", {139, 37, 0}},
+ {"orchid", {218, 112, 214}},
+ {"orchid1", {255, 131, 250}},
+ {"orchid2", {238, 122, 233}},
+ {"orchid3", {205, 105, 201}},
+ {"orchid4", {139, 71, 137}},
+ {"palegoldenrod", {238, 232, 170}},
+ {"palegreen", {152, 251, 152}},
+ {"palegreen1", {154, 255, 154}},
+ {"palegreen2", {144, 238, 144}},
+ {"palegreen3", {124, 205, 124}},
+ {"palegreen4", {84, 139, 84}},
+ {"paleturquoise", {175, 238, 238}},
+ {"paleturquoise1", {187, 255, 255}},
+ {"paleturquoise2", {174, 238, 238}},
+ {"paleturquoise3", {150, 205, 205}},
+ {"paleturquoise4", {102, 139, 139}},
+ {"palevioletred", {219, 112, 147}},
+ {"palevioletred1", {255, 130, 171}},
+ {"palevioletred2", {238, 121, 159}},
+ {"palevioletred3", {205, 104, 137}},
+ {"palevioletred4", {139, 71, 93}},
+ {"papayawhip", {255, 239, 213}},
+ {"peachpuff", {255, 218, 185}},
+ {"peachpuff1", {255, 218, 185}},
+ {"peachpuff2", {238, 203, 173}},
+ {"peachpuff3", {205, 175, 149}},
+ {"peachpuff4", {139, 119, 101}},
+ {"peru", {205, 133, 63}},
+ {"pink", {255, 192, 203}},
+ {"pink1", {255, 181, 197}},
+ {"pink2", {238, 169, 184}},
+ {"pink3", {205, 145, 158}},
+ {"pink4", {139, 99, 108}},
+ {"plum", {221, 160, 221}},
+ {"plum1", {255, 187, 255}},
+ {"plum2", {238, 174, 238}},
+ {"plum3", {205, 150, 205}},
+ {"plum4", {139, 102, 139}},
+ {"powderblue", {176, 224, 230}},
+ {"purple", {160, 32, 240}},
+ {"purple1", {155, 48, 255}},
+ {"purple2", {145, 44, 238}},
+ {"purple3", {125, 38, 205}},
+ {"purple4", {85, 26, 139}},
+ {"rebeccapurple", {102, 51, 153}},
+ {"red", {255, 0, 0}},
+ {"red1", {255, 0, 0}},
+ {"red2", {238, 0, 0}},
+ {"red3", {205, 0, 0}},
+ {"red4", {139, 0, 0}},
+ {"rosybrown", {188, 143, 143}},
+ {"rosybrown1", {255, 193, 193}},
+ {"rosybrown2", {238, 180, 180}},
+ {"rosybrown3", {205, 155, 155}},
+ {"rosybrown4", {139, 105, 105}},
+ {"royalblue", {65, 105, 225}},
+ {"royalblue1", {72, 118, 255}},
+ {"royalblue2", {67, 110, 238}},
+ {"royalblue3", {58, 95, 205}},
+ {"royalblue4", {39, 64, 139}},
+ {"saddlebrown", {139, 69, 19}},
+ {"salmon", {250, 128, 114}},
+ {"salmon1", {255, 140, 105}},
+ {"salmon2", {238, 130, 98}},
+ {"salmon3", {205, 112, 84}},
+ {"salmon4", {139, 76, 57}},
+ {"sandybrown", {244, 164, 96}},
+ {"seagreen", {46, 139, 87}},
+ {"seagreen1", {84, 255, 159}},
+ {"seagreen2", {78, 238, 148}},
+ {"seagreen3", {67, 205, 128}},
+ {"seagreen4", {46, 139, 87}},
+ {"seashell", {255, 245, 238}},
+ {"seashell1", {255, 245, 238}},
+ {"seashell2", {238, 229, 222}},
+ {"seashell3", {205, 197, 191}},
+ {"seashell4", {139, 134, 130}},
+ {"sienna", {160, 82, 45}},
+ {"sienna1", {255, 130, 71}},
+ {"sienna2", {238, 121, 66}},
+ {"sienna3", {205, 104, 57}},
+ {"sienna4", {139, 71, 38}},
+ {"silver", {192, 192, 192}},
+ {"skyblue", {135, 206, 235}},
+ {"skyblue1", {135, 206, 255}},
+ {"skyblue2", {126, 192, 238}},
+ {"skyblue3", {108, 166, 205}},
+ {"skyblue4", {74, 112, 139}},
+ {"slateblue", {106, 90, 205}},
+ {"slateblue1", {131, 111, 255}},
+ {"slateblue2", {122, 103, 238}},
+ {"slateblue3", {105, 89, 205}},
+ {"slateblue4", {71, 60, 139}},
+ {"slategray", {112, 128, 144}},
+ {"slategray1", {198, 226, 255}},
+ {"slategray2", {185, 211, 238}},
+ {"slategray3", {159, 182, 205}},
+ {"slategray4", {108, 123, 139}},
+ {"slategrey", {112, 128, 144}},
+ {"snow", {255, 250, 250}},
+ {"snow1", {255, 250, 250}},
+ {"snow2", {238, 233, 233}},
+ {"snow3", {205, 201, 201}},
+ {"snow4", {139, 137, 137}},
+ {"springgreen", {0, 255, 127}},
+ {"springgreen1", {0, 255, 127}},
+ {"springgreen2", {0, 238, 118}},
+ {"springgreen3", {0, 205, 102}},
+ {"springgreen4", {0, 139, 69}},
+ {"steelblue", {70, 130, 180}},
+ {"steelblue1", {99, 184, 255}},
+ {"steelblue2", {92, 172, 238}},
+ {"steelblue3", {79, 148, 205}},
+ {"steelblue4", {54, 100, 139}},
+ {"tan", {210, 180, 140}},
+ {"tan1", {255, 165, 79}},
+ {"tan2", {238, 154, 73}},
+ {"tan3", {205, 133, 63}},
+ {"tan4", {139, 90, 43}},
+ {"teal", {0, 128, 128}},
+ {"thistle", {216, 191, 216}},
+ {"thistle1", {255, 225, 255}},
+ {"thistle2", {238, 210, 238}},
+ {"thistle3", {205, 181, 205}},
+ {"thistle4", {139, 123, 139}},
+ {"tomato", {255, 99, 71}},
+ {"tomato1", {255, 99, 71}},
+ {"tomato2", {238, 92, 66}},
+ {"tomato3", {205, 79, 57}},
+ {"tomato4", {139, 54, 38}},
+ {"turquoise", {64, 224, 208}},
+ {"turquoise1", {0, 245, 255}},
+ {"turquoise2", {0, 229, 238}},
+ {"turquoise3", {0, 197, 205}},
+ {"turquoise4", {0, 134, 139}},
+ {"violet", {238, 130, 238}},
+ {"violetred", {208, 32, 144}},
+ {"violetred1", {255, 62, 150}},
+ {"violetred2", {238, 58, 140}},
+ {"violetred3", {205, 50, 120}},
+ {"violetred4", {139, 34, 82}},
+ {"webgray", {128, 128, 128}},
+ {"webgreen", {0, 128, 0}},
+ {"webgrey", {128, 128, 128}},
+ {"webmaroon", {128, 0, 0}},
+ {"webpurple", {128, 0, 128}},
+ {"wheat", {245, 222, 179}},
+ {"wheat1", {255, 231, 186}},
+ {"wheat2", {238, 216, 174}},
+ {"wheat3", {205, 186, 150}},
+ {"wheat4", {139, 126, 102}},
+ {"white", {255, 255, 255}},
+ {"whitesmoke", {245, 245, 245}},
+ {"x11gray", {190, 190, 190}},
+ {"x11green", {0, 255, 0}},
+ {"x11grey", {190, 190, 190}},
+ {"x11maroon", {176, 48, 96}},
+ {"x11purple", {160, 32, 240}},
+ {"yellow", {255, 255, 0}},
+ {"yellow1", {255, 255, 0}},
+ {"yellow2", {238, 238, 0}},
+ {"yellow3", {205, 205, 0}},
+ {"yellow4", {139, 139, 0}},
+ {"yellowgreen", {154, 205, 50}},
+ {"activeborder", {180, 180, 180}},
+ {"activecaption", {153, 180, 209}},
+ {"appworkspace", {171, 171, 171}},
+ {"background", {0, 0, 0}},
+ {"buttonhighlight", {255, 255, 255}},
+ {"buttonshadow", {160, 160, 160}},
+ {"captiontext", {0, 0, 0}},
+ {"inactiveborder", {244, 247, 252}},
+ {"inactivecaption", {191, 205, 219}},
+ {"inactivecaptiontext", {0, 0, 0}},
+ {"infobackground", {255, 255, 225}},
+ {"infotext", {0, 0, 0}},
+ {"menu", {240, 240, 240}},
+ {"menutext", {0, 0, 0}},
+ {"scrollbar", {200, 200, 200}},
+ {"threeddarkshadow", {0, 0, 0}},
+ {"threedface", {0, 0, 0}},
+ {"threedhighlight", {0, 0, 0}},
+ {"threedlightshadow", {0, 0, 0}},
+ {"threedshadow", {0, 0, 0}},
+ {"transparent", {0, 0, 0, 0}},
+ {"window", {255, 255, 255}},
+ {"windowframe", {100, 100, 100}},
+ {"windowtext", {0, 0, 0}},
+};
+
+}// namespace rspamd::css
+
+#endif//RSPAMD_CSS_COLORS_LIST_HXX
diff --git a/src/libserver/css/css_parser.cxx b/src/libserver/css/css_parser.cxx
new file mode 100644
index 0000000..aed035a
--- /dev/null
+++ b/src/libserver/css/css_parser.cxx
@@ -0,0 +1,892 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "css_parser.hxx"
+#include "css_tokeniser.hxx"
+#include "css_selector.hxx"
+#include "css_rule.hxx"
+#include "css_util.hxx"
+#include "css.hxx"
+#include "fmt/core.h"
+
+#include <vector>
+#include <unicode/utf8.h>
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+namespace rspamd::css {
+
+const css_consumed_block css_parser_eof_block{};
+
+auto css_consumed_block::attach_block(consumed_block_ptr &&block) -> bool
+{
+ if (std::holds_alternative<std::monostate>(content)) {
+ /* Switch from monostate */
+ content = std::vector<consumed_block_ptr>();
+ }
+ else if (!std::holds_alternative<std::vector<consumed_block_ptr>>(content)) {
+ /* A single component, cannot attach a block ! */
+ return false;
+ }
+
+ auto &value_vec = std::get<std::vector<consumed_block_ptr>>(content);
+ value_vec.push_back(std::move(block));
+
+ return true;
+}
+
+auto css_consumed_block::add_function_argument(consumed_block_ptr &&block) -> bool
+{
+ if (!std::holds_alternative<css_function_block>(content)) {
+ return false;
+ }
+
+ auto &&func_bloc = std::get<css_function_block>(content);
+ func_bloc.args.push_back(std::move(block));
+
+ return true;
+}
+
+auto css_consumed_block::token_type_str(void) const -> const char *
+{
+ const auto *ret = "";
+
+ switch (tag) {
+ case parser_tag_type::css_top_block:
+ ret = "top";
+ break;
+ case parser_tag_type::css_qualified_rule:
+ ret = "qualified rule";
+ break;
+ case parser_tag_type::css_at_rule:
+ ret = "at rule";
+ break;
+ case parser_tag_type::css_simple_block:
+ ret = "simple block";
+ break;
+ case parser_tag_type::css_function:
+ ret = "function";
+ break;
+ case parser_tag_type::css_function_arg:
+ ret = "function arg";
+ break;
+ case parser_tag_type::css_component:
+ ret = "component";
+ break;
+ case parser_tag_type::css_eof_block:
+ ret = "eof";
+ break;
+ }
+
+ return ret;
+}
+
+auto css_consumed_block::debug_str(void) -> std::string
+{
+ std::string ret = fmt::format(R"("type": "{}", "value": )", token_type_str());
+
+ std::visit([&](auto &arg) {
+ using T = std::decay_t<decltype(arg)>;
+
+ if constexpr (std::is_same_v<T, std::vector<consumed_block_ptr>>) {
+ /* Array of blocks */
+ ret += "[";
+ for (const auto &block: arg) {
+ ret += "{";
+ ret += block->debug_str();
+ ret += "}, ";
+ }
+
+ if (*(--ret.end()) == ' ') {
+ ret.pop_back();
+ ret.pop_back(); /* Last ',' */
+ }
+ ret += "]";
+ }
+ else if constexpr (std::is_same_v<T, std::monostate>) {
+ /* Empty block */
+ ret += R"("empty")";
+ }
+ else if constexpr (std::is_same_v<T, css_function_block>) {
+ ret += R"({ "content": {"token": )";
+ ret += "\"" + arg.function.debug_token_str() + "\", ";
+ ret += R"("arguments": [)";
+
+ for (const auto &block: arg.args) {
+ ret += "{";
+ ret += block->debug_str();
+ ret += "}, ";
+ }
+ if (*(--ret.end()) == ' ') {
+ ret.pop_back();
+ ret.pop_back(); /* Last ',' */
+ }
+ ret += "]}}";
+ }
+ else {
+ /* Single element block */
+ ret += "\"" + arg.debug_token_str() + "\"";
+ }
+ },
+ content);
+
+ return ret;
+}
+
+class css_parser {
+public:
+ css_parser(void) = delete; /* Require mempool to be set for logging */
+ explicit css_parser(rspamd_mempool_t *pool)
+ : pool(pool)
+ {
+ style_object.reset();
+ error.type = css_parse_error_type::PARSE_ERROR_NO_ERROR;
+ }
+
+ /*
+ * This constructor captures existing via unique_ptr, but it does not
+ * destruct it on errors (we assume that it is owned somewhere else)
+ */
+ explicit css_parser(std::shared_ptr<css_style_sheet> &&existing, rspamd_mempool_t *pool)
+ : style_object(existing), pool(pool)
+ {
+ error.type = css_parse_error_type::PARSE_ERROR_NO_ERROR;
+ }
+
+ /*
+ * Process input css blocks
+ */
+ std::unique_ptr<css_consumed_block> consume_css_blocks(const std::string_view &sv);
+ /*
+ * Process a single css rule
+ */
+ std::unique_ptr<css_consumed_block> consume_css_rule(const std::string_view &sv);
+ std::optional<css_parse_error> consume_input(const std::string_view &sv);
+
+ auto get_object_maybe(void) -> tl::expected<std::shared_ptr<css_style_sheet>, css_parse_error>
+ {
+ if (style_object) {
+ return style_object;
+ }
+
+ return tl::make_unexpected(error);
+ }
+
+ /* Helper parser methods */
+ static bool need_unescape(const std::string_view &sv);
+
+private:
+ std::shared_ptr<css_style_sheet> style_object;
+ std::unique_ptr<css_tokeniser> tokeniser;
+
+ css_parse_error error;
+ rspamd_mempool_t *pool;
+
+ int rec_level = 0;
+ const int max_rec = 20;
+ bool eof = false;
+
+ /* Consumers */
+ auto component_value_consumer(std::unique_ptr<css_consumed_block> &top) -> bool;
+ auto function_consumer(std::unique_ptr<css_consumed_block> &top) -> bool;
+ auto simple_block_consumer(std::unique_ptr<css_consumed_block> &top,
+ css_parser_token::token_type expected_end,
+ bool consume_current) -> bool;
+ auto qualified_rule_consumer(std::unique_ptr<css_consumed_block> &top) -> bool;
+ auto at_rule_consumer(std::unique_ptr<css_consumed_block> &top) -> bool;
+};
+
+/*
+ * Find if we need to unescape css
+ */
+bool css_parser::need_unescape(const std::string_view &sv)
+{
+ bool in_quote = false;
+ char quote_char, prev_c = 0;
+
+ for (const auto c: sv) {
+ if (!in_quote) {
+ if (c == '"' || c == '\'') {
+ in_quote = true;
+ quote_char = c;
+ }
+ else if (c == '\\') {
+ return true;
+ }
+ }
+ else {
+ if (c == quote_char) {
+ if (prev_c != '\\') {
+ in_quote = false;
+ }
+ }
+ prev_c = c;
+ }
+ }
+
+ return false;
+}
+
+auto css_parser::function_consumer(std::unique_ptr<css_consumed_block> &top) -> bool
+{
+ auto ret = true, want_more = true;
+
+ msg_debug_css("consume function block; top block: %s, recursion level %d",
+ top->token_type_str(), rec_level);
+
+ if (++rec_level > max_rec) {
+ msg_err_css("max nesting reached, ignore style");
+ error = css_parse_error(css_parse_error_type::PARSE_ERROR_BAD_NESTING,
+ "maximum nesting has reached when parsing function value");
+ return false;
+ }
+
+ while (ret && want_more && !eof) {
+ auto next_token = tokeniser->next_token();
+
+ switch (next_token.type) {
+ case css_parser_token::token_type::eof_token:
+ eof = true;
+ break;
+ case css_parser_token::token_type::whitespace_token:
+ /* Ignore whitespaces */
+ break;
+ case css_parser_token::token_type::ebrace_token:
+ ret = true;
+ want_more = false;
+ break;
+ case css_parser_token::token_type::comma_token:
+ case css_parser_token::token_type::delim_token:
+ case css_parser_token::token_type::obrace_token:
+ break;
+ default:
+ /* Attach everything to the function block */
+ top->add_function_argument(std::make_unique<css_consumed_block>(
+ css::css_consumed_block::parser_tag_type::css_function_arg,
+ std::move(next_token)));
+ break;
+ }
+ }
+
+ --rec_level;
+
+ return ret;
+}
+
+auto css_parser::simple_block_consumer(std::unique_ptr<css_consumed_block> &top,
+ css_parser_token::token_type expected_end,
+ bool consume_current) -> bool
+{
+ auto ret = true;
+ std::unique_ptr<css_consumed_block> block;
+
+ msg_debug_css("consume simple block; top block: %s, recursion level %d",
+ top->token_type_str(), rec_level);
+
+ if (!consume_current && ++rec_level > max_rec) {
+ msg_err_css("max nesting reached, ignore style");
+ error = css_parse_error(css_parse_error_type::PARSE_ERROR_BAD_NESTING,
+ "maximum nesting has reached when parsing simple block value");
+ return false;
+ }
+
+ if (!consume_current) {
+ block = std::make_unique<css_consumed_block>(
+ css_consumed_block::parser_tag_type::css_simple_block);
+ }
+
+
+ while (ret && !eof) {
+ auto next_token = tokeniser->next_token();
+
+ if (next_token.type == expected_end) {
+ break;
+ }
+
+ switch (next_token.type) {
+ case css_parser_token::token_type::eof_token:
+ eof = true;
+ break;
+ case css_parser_token::token_type::whitespace_token:
+ /* Ignore whitespaces */
+ break;
+ default:
+ tokeniser->pushback_token(next_token);
+ ret = component_value_consumer(consume_current ? top : block);
+ break;
+ }
+ }
+
+ if (!consume_current && ret) {
+ msg_debug_css("attached node 'simple block' rule %s; length=%d",
+ block->token_type_str(), (int) block->size());
+ top->attach_block(std::move(block));
+ }
+
+ if (!consume_current) {
+ --rec_level;
+ }
+
+ return ret;
+}
+
+auto css_parser::qualified_rule_consumer(std::unique_ptr<css_consumed_block> &top) -> bool
+{
+ msg_debug_css("consume qualified block; top block: %s, recursion level %d",
+ top->token_type_str(), rec_level);
+
+ if (++rec_level > max_rec) {
+ msg_err_css("max nesting reached, ignore style");
+ error = css_parse_error(css_parse_error_type::PARSE_ERROR_BAD_NESTING,
+ "maximum nesting has reached when parsing qualified rule value");
+ return false;
+ }
+
+ auto ret = true, want_more = true;
+ auto block = std::make_unique<css_consumed_block>(
+ css_consumed_block::parser_tag_type::css_qualified_rule);
+
+ while (ret && want_more && !eof) {
+ auto next_token = tokeniser->next_token();
+ switch (next_token.type) {
+ case css_parser_token::token_type::eof_token:
+ eof = true;
+ break;
+ case css_parser_token::token_type::cdo_token:
+ case css_parser_token::token_type::cdc_token:
+ if (top->tag == css_consumed_block::parser_tag_type::css_top_block) {
+ /* Ignore */
+ ret = true;
+ }
+ else {
+ }
+ break;
+ case css_parser_token::token_type::ocurlbrace_token:
+ ret = simple_block_consumer(block,
+ css_parser_token::token_type::ecurlbrace_token, false);
+ want_more = false;
+ break;
+ case css_parser_token::token_type::whitespace_token:
+ /* Ignore whitespaces */
+ break;
+ default:
+ tokeniser->pushback_token(next_token);
+ ret = component_value_consumer(block);
+ break;
+ };
+ }
+
+ if (ret) {
+ if (top->tag == css_consumed_block::parser_tag_type::css_top_block) {
+ msg_debug_css("attached node qualified rule %s; length=%d",
+ block->token_type_str(), (int) block->size());
+ top->attach_block(std::move(block));
+ }
+ }
+
+ --rec_level;
+
+ return ret;
+}
+
+auto css_parser::at_rule_consumer(std::unique_ptr<css_consumed_block> &top) -> bool
+{
+ msg_debug_css("consume at-rule block; top block: %s, recursion level %d",
+ top->token_type_str(), rec_level);
+
+ if (++rec_level > max_rec) {
+ msg_err_css("max nesting reached, ignore style");
+ error = css_parse_error(css_parse_error_type::PARSE_ERROR_BAD_NESTING,
+ "maximum nesting has reached when parsing at keyword");
+ return false;
+ }
+
+ auto ret = true, want_more = true;
+ auto block = std::make_unique<css_consumed_block>(
+ css_consumed_block::parser_tag_type::css_at_rule);
+
+ while (ret && want_more && !eof) {
+ auto next_token = tokeniser->next_token();
+ switch (next_token.type) {
+ case css_parser_token::token_type::eof_token:
+ eof = true;
+ break;
+ case css_parser_token::token_type::cdo_token:
+ case css_parser_token::token_type::cdc_token:
+ if (top->tag == css_consumed_block::parser_tag_type::css_top_block) {
+ /* Ignore */
+ ret = true;
+ }
+ else {
+ }
+ break;
+ case css_parser_token::token_type::ocurlbrace_token:
+ ret = simple_block_consumer(block,
+ css_parser_token::token_type::ecurlbrace_token, false);
+ want_more = false;
+ break;
+ case css_parser_token::token_type::whitespace_token:
+ /* Ignore whitespaces */
+ break;
+ case css_parser_token::token_type::semicolon_token:
+ want_more = false;
+ break;
+ default:
+ tokeniser->pushback_token(next_token);
+ ret = component_value_consumer(block);
+ break;
+ };
+ }
+
+ if (ret) {
+ if (top->tag == css_consumed_block::parser_tag_type::css_top_block) {
+ msg_debug_css("attached node qualified rule %s; length=%d",
+ block->token_type_str(), (int) block->size());
+ top->attach_block(std::move(block));
+ }
+ }
+
+ --rec_level;
+
+ return ret;
+}
+
+auto css_parser::component_value_consumer(std::unique_ptr<css_consumed_block> &top) -> bool
+{
+ auto ret = true, need_more = true;
+ std::unique_ptr<css_consumed_block> block;
+
+ msg_debug_css("consume component block; top block: %s, recursion level %d",
+ top->token_type_str(), rec_level);
+
+ if (++rec_level > max_rec) {
+ error = css_parse_error(css_parse_error_type::PARSE_ERROR_BAD_NESTING,
+ "maximum nesting has reached when parsing component value");
+ return false;
+ }
+
+ while (ret && need_more && !eof) {
+ auto next_token = tokeniser->next_token();
+
+ switch (next_token.type) {
+ case css_parser_token::token_type::eof_token:
+ eof = true;
+ break;
+ case css_parser_token::token_type::ocurlbrace_token:
+ block = std::make_unique<css_consumed_block>(
+ css_consumed_block::parser_tag_type::css_simple_block);
+ ret = simple_block_consumer(block,
+ css_parser_token::token_type::ecurlbrace_token,
+ true);
+ need_more = false;
+ break;
+ case css_parser_token::token_type::obrace_token:
+ block = std::make_unique<css_consumed_block>(
+ css_consumed_block::parser_tag_type::css_simple_block);
+ ret = simple_block_consumer(block,
+ css_parser_token::token_type::ebrace_token,
+ true);
+ need_more = false;
+ break;
+ case css_parser_token::token_type::osqbrace_token:
+ block = std::make_unique<css_consumed_block>(
+ css_consumed_block::parser_tag_type::css_simple_block);
+ ret = simple_block_consumer(block,
+ css_parser_token::token_type::esqbrace_token,
+ true);
+ need_more = false;
+ break;
+ case css_parser_token::token_type::whitespace_token:
+ /* Ignore whitespaces */
+ break;
+ case css_parser_token::token_type::function_token: {
+ need_more = false;
+ block = std::make_unique<css_consumed_block>(
+ css_consumed_block::parser_tag_type::css_function,
+ std::move(next_token));
+
+ /* Consume the rest */
+ ret = function_consumer(block);
+ break;
+ }
+ default:
+ block = std::make_unique<css_consumed_block>(
+ css_consumed_block::parser_tag_type::css_component,
+ std::move(next_token));
+ need_more = false;
+ break;
+ }
+ }
+
+ if (ret && block) {
+ msg_debug_css("attached node component rule %s; length=%d",
+ block->token_type_str(), (int) block->size());
+ top->attach_block(std::move(block));
+ }
+
+ --rec_level;
+
+ return ret;
+}
+
+auto css_parser::consume_css_blocks(const std::string_view &sv) -> std::unique_ptr<css_consumed_block>
+{
+ tokeniser = std::make_unique<css_tokeniser>(pool, sv);
+ auto ret = true;
+
+ auto consumed_blocks =
+ std::make_unique<css_consumed_block>(css_consumed_block::parser_tag_type::css_top_block);
+
+ while (!eof && ret) {
+ auto next_token = tokeniser->next_token();
+
+ switch (next_token.type) {
+ case css_parser_token::token_type::whitespace_token:
+ /* Ignore whitespaces */
+ break;
+ case css_parser_token::token_type::eof_token:
+ eof = true;
+ break;
+ case css_parser_token::token_type::at_keyword_token:
+ tokeniser->pushback_token(next_token);
+ ret = at_rule_consumer(consumed_blocks);
+ break;
+ default:
+ tokeniser->pushback_token(next_token);
+ ret = qualified_rule_consumer(consumed_blocks);
+ break;
+ }
+ }
+
+ tokeniser.reset(nullptr); /* No longer needed */
+
+ return consumed_blocks;
+}
+
+auto css_parser::consume_css_rule(const std::string_view &sv) -> std::unique_ptr<css_consumed_block>
+{
+ tokeniser = std::make_unique<css_tokeniser>(pool, sv);
+ auto ret = true;
+
+ auto rule_block =
+ std::make_unique<css_consumed_block>(css_consumed_block::parser_tag_type::css_simple_block);
+
+ while (!eof && ret) {
+ auto next_token = tokeniser->next_token();
+
+ switch (next_token.type) {
+ case css_parser_token::token_type::eof_token:
+ eof = true;
+ break;
+ case css_parser_token::token_type::whitespace_token:
+ /* Ignore whitespaces */
+ break;
+ default:
+ tokeniser->pushback_token(next_token);
+ ret = component_value_consumer(rule_block);
+ break;
+ }
+ }
+
+ tokeniser.reset(nullptr); /* No longer needed */
+
+ return rule_block;
+}
+
+std::optional<css_parse_error>
+css_parser::consume_input(const std::string_view &sv)
+{
+ auto &&consumed_blocks = consume_css_blocks(sv);
+ const auto &rules = consumed_blocks->get_blocks_or_empty();
+
+ if (rules.empty()) {
+ if (error.type == css_parse_error_type::PARSE_ERROR_NO_ERROR) {
+ return css_parse_error(css_parse_error_type::PARSE_ERROR_EMPTY,
+ "no css rules consumed");
+ }
+ else {
+ return error;
+ }
+ }
+
+ if (!style_object) {
+ style_object = std::make_shared<css_style_sheet>(pool);
+ }
+
+ for (auto &&rule: rules) {
+ /*
+ * For now, we do not need any of the at rules, so we can safely ignore them
+ */
+ auto &&children = rule->get_blocks_or_empty();
+
+ if (children.size() > 1 &&
+ children[0]->tag == css_consumed_block::parser_tag_type::css_component) {
+ auto simple_block = std::find_if(children.begin(), children.end(),
+ [](auto &bl) {
+ return bl->tag == css_consumed_block::parser_tag_type::css_simple_block;
+ });
+
+ if (simple_block != children.end()) {
+ /*
+ * We have a component and a simple block,
+ * so we can parse a selector and then extract
+ * declarations from a simple block
+ */
+
+ /* First, tag all components as preamble */
+ auto selector_it = children.cbegin();
+
+ auto selector_token_functor = [&selector_it, &simple_block](void)
+ -> const css_consumed_block & {
+ for (;;) {
+ if (selector_it == simple_block) {
+ return css_parser_eof_block;
+ }
+
+ const auto &ret = (*selector_it);
+
+ ++selector_it;
+
+ return *ret;
+ }
+ };
+
+ auto selectors_vec = process_selector_tokens(pool, selector_token_functor);
+
+ if (selectors_vec.size() > 0) {
+ msg_debug_css("processed %d selectors", (int) selectors_vec.size());
+ auto decls_it = (*simple_block)->get_blocks_or_empty().cbegin();
+ auto decls_end = (*simple_block)->get_blocks_or_empty().cend();
+ auto declaration_token_functor = [&decls_it, &decls_end](void)
+ -> const css_consumed_block & {
+ for (;;) {
+ if (decls_it == decls_end) {
+ return css_parser_eof_block;
+ }
+
+ const auto &ret = (*decls_it);
+
+ ++decls_it;
+
+ return *ret;
+ }
+ };
+
+ auto declarations_vec = process_declaration_tokens(pool,
+ declaration_token_functor);
+
+ if (declarations_vec && !declarations_vec->get_rules().empty()) {
+ msg_debug_css("processed %d rules",
+ (int) declarations_vec->get_rules().size());
+
+ for (auto &&selector: selectors_vec) {
+ style_object->add_selector_rule(std::move(selector),
+ declarations_vec);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ auto debug_str = consumed_blocks->debug_str();
+ msg_debug_css("consumed css: {%*s}", (int) debug_str.size(), debug_str.data());
+
+ return std::nullopt;
+}
+
+auto get_selectors_parser_functor(rspamd_mempool_t *pool,
+ const std::string_view &st) -> blocks_gen_functor
+{
+ css_parser parser(pool);
+
+ auto &&consumed_blocks = parser.consume_css_blocks(st);
+ const auto &rules = consumed_blocks->get_blocks_or_empty();
+
+ auto rules_it = rules.begin();
+ auto &&children = (*rules_it)->get_blocks_or_empty();
+ auto cur = children.begin();
+ auto last = children.end();
+
+ /*
+ * We use move only wrapper to state the fact that the cosumed blocks
+ * are moved into the closure, not copied.
+ * It prevents us from thinking about copies of the blocks and
+ * functors.
+ * Mutable lambda is required to copy iterators inside of the closure,
+ * as, again, it is C++ where lifetime of the objects must be explicitly
+ * transferred. On the other hand, we could move all stuff inside and remove
+ * mutable.
+ */
+ return [cur, consumed_blocks = std::move(consumed_blocks), last](void) mutable
+ -> const css_consumed_block & {
+ if (cur != last) {
+ const auto &ret = (*cur);
+
+ ++cur;
+
+ return *ret;
+ }
+
+ return css_parser_eof_block;
+ };
+}
+
+auto get_rules_parser_functor(rspamd_mempool_t *pool,
+ const std::string_view &st) -> blocks_gen_functor
+{
+ css_parser parser(pool);
+
+ auto &&consumed_blocks = parser.consume_css_rule(st);
+ const auto &rules = consumed_blocks->get_blocks_or_empty();
+
+ auto cur = rules.begin();
+ auto last = rules.end();
+
+ return [cur, consumed_blocks = std::move(consumed_blocks), last](void) mutable
+ -> const css_consumed_block & {
+ if (cur != last) {
+ const auto &ret = (*cur);
+
+ ++cur;
+
+ return *ret;
+ }
+
+ return css_parser_eof_block;
+ };
+}
+
+
+/*
+ * Wrapper for the parser
+ */
+auto parse_css(rspamd_mempool_t *pool, const std::string_view &st,
+ std::shared_ptr<css_style_sheet> &&other)
+ -> tl::expected<std::shared_ptr<css_style_sheet>, css_parse_error>
+{
+ css_parser parser(std::forward<std::shared_ptr<css_style_sheet>>(other), pool);
+ std::string_view processed_input;
+
+ if (css_parser::need_unescape(st)) {
+ processed_input = rspamd::css::unescape_css(pool, st);
+ }
+ else {
+ /* Lowercase inplace */
+ auto *nspace = rspamd_mempool_alloc_buffer(pool, st.size());
+ rspamd_str_copy_lc(st.data(), nspace, st.size());
+ processed_input = std::string_view{nspace, st.size()};
+ }
+
+ auto maybe_error = parser.consume_input(processed_input);
+ if (!maybe_error) {
+ return parser.get_object_maybe();
+ }
+
+ return tl::make_unexpected(maybe_error.value());
+}
+
+auto parse_css_declaration(rspamd_mempool_t *pool, const std::string_view &st)
+ -> rspamd::html::html_block *
+{
+ std::string_view processed_input;
+
+ if (css_parser::need_unescape(st)) {
+ processed_input = rspamd::css::unescape_css(pool, st);
+ }
+ else {
+ auto *nspace = reinterpret_cast<char *>(rspamd_mempool_alloc(pool, st.size()));
+ auto nlen = rspamd_str_copy_lc(st.data(), nspace, st.size());
+ processed_input = std::string_view{nspace, nlen};
+ }
+ auto &&res = process_declaration_tokens(pool,
+ get_rules_parser_functor(pool, processed_input));
+
+ if (res) {
+ return res->compile_to_block(pool);
+ }
+
+ return nullptr;
+}
+
+TEST_SUITE("css")
+{
+ TEST_CASE("parse colors")
+ {
+ const std::vector<const char *> cases{
+ "P { CoLoR: rgb(100%, 50%, 0%); opacity: -1; width: 1em; display: none; } /* very transparent solid orange теÑÑ‚ */",
+ "p { color: rgb(100%, 50%, 0%); opacity: 2; display: inline; } /* very transparent solid orange */",
+ "p { color: rgb(100%, 50%, 0%); opacity: 0.5; } /* very transparent solid orange */\n",
+ "p { color: rgb(100%, 50%, 0%); opacity: 1; width: 99%; } /* very transparent solid orange */\n",
+ "p { color: rgb(100%, 50%, 0%); opacity: 10%; width: 99%; } /* very transparent solid orange */\n",
+ "p { color: rgb(100%, 50%, 0%); opacity: 10%; width: 100px; } /* very transparent solid orange */\n",
+ "p { color: rgb(100%, 50%, 0%); opacity: 10% } /* very transparent solid orange */\n",
+ "* { color: hsl(0, 100%, 50%) !important } /* red */\n",
+ "* { color: hsl(120, 100%, 50%) important } /* lime */\n",
+ "* { color: hsl(120, 100%, 25%) } /* dark green */\n",
+ "* { color: hsl(120, 100%, 75%) } /* light green */\n",
+ "* { color: hsl(120, 75%, 75%) } /* pastel green, and so on */\n",
+ "em { color: #f00 } /* #rgb */\n",
+ "em { color: #ff0000 } /* #rrggbb */\n",
+ "em { color: rgb(255,0,0) }\n",
+ "em { color: rgb(100%, 0%, 0%) }\n",
+ "body {color: black; background: white }\n",
+ "h1 { color: maroon }\n",
+ "h2 { color: olive }\n",
+ "em { color: rgb(255,0,0) } /* integer range 0 - 255 */\n",
+ "em { color: rgb(300,0,0) } /* clipped to rgb(255,0,0) */\n",
+ "em { color: rgb(255,-10,0) } /* clipped to rgb(255,0,0) */\n",
+ "em { color: rgb(110%, 0%, 0%) } /* clipped to rgb(100%,0%,0%) */\n",
+ "em { color: rgb(255,0,0) } /* integer range 0 - 255 */\n",
+ "em { color: rgba(255,0,0,1) /* the same, with explicit opacity of 1 */\n",
+ "em { color: rgb(100%,0%,0%) } /* float range 0.0% - 100.0% */\n",
+ "em { color: rgba(100%,0%,0%,1) } /* the same, with explicit opacity of 1 */\n",
+ "p { color: rgba(0,0,255,0.5) } /* semi-transparent solid blue */\n",
+ "p { color: rgba(100%, 50%, 0%, 0.1) } /* very transparent solid orange */",
+ ".chat-icon[_ng-cnj-c0]::before{content:url(group-2.63e87cd21fbf8c966dd.svg);width:60px;height:60px;display:block}",
+ "tt{color:#1e3482}",
+ "tt{unicode-range: u+0049-u+004a,u+0020;}",
+ "@import url(https://fonts.googleapis.com/css?family=arial:300,400,7000;",
+ "tt{color:black;\v}",
+ "tt{color:black;\f}",
+ };
+
+ rspamd_mempool_t *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "css", 0);
+ for (const auto &c: cases) {
+ SUBCASE((std::string("parse css: ") + c).c_str())
+ {
+ CHECK(parse_css(pool, c, nullptr).value().get() != nullptr);
+ }
+ }
+
+ /* We now merge all styles together */
+ SUBCASE("merged css parse")
+ {
+ std::shared_ptr<css_style_sheet> merged;
+ for (const auto &c: cases) {
+ auto ret = parse_css(pool, c, std::move(merged));
+ merged.swap(ret.value());
+ }
+
+ CHECK(merged.get() != nullptr);
+ }
+
+ rspamd_mempool_delete(pool);
+ }
+}
+}// namespace rspamd::css
diff --git a/src/libserver/css/css_parser.hxx b/src/libserver/css/css_parser.hxx
new file mode 100644
index 0000000..d5a9671
--- /dev/null
+++ b/src/libserver/css/css_parser.hxx
@@ -0,0 +1,244 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifndef RSPAMD_CSS_PARSER_HXX
+#define RSPAMD_CSS_PARSER_HXX
+
+#include <variant>
+#include <vector>
+#include <memory>
+#include <string>
+
+#include "function2/function2.hpp"
+#include "css_tokeniser.hxx"
+#include "parse_error.hxx"
+#include "contrib/expected/expected.hpp"
+#include "logger.h"
+
+/* Forward declaration */
+namespace rspamd::html {
+struct html_block;
+}
+
+namespace rspamd::css {
+
+/*
+ * Represents a consumed token by a parser
+ */
+class css_consumed_block {
+public:
+ enum class parser_tag_type : std::uint8_t {
+ css_top_block = 0,
+ css_qualified_rule,
+ css_at_rule,
+ css_simple_block,
+ css_function,
+ css_function_arg,
+ css_component,
+ css_eof_block,
+ };
+ using consumed_block_ptr = std::unique_ptr<css_consumed_block>;
+
+ struct css_function_block {
+ css_parser_token function;
+ std::vector<consumed_block_ptr> args;
+
+ css_function_block(css_parser_token &&tok)
+ : function(std::forward<css_parser_token>(tok))
+ {
+ }
+
+ auto as_string() const -> std::string_view
+ {
+ return function.get_string_or_default("");
+ }
+
+ static auto empty_function() -> const css_function_block &
+ {
+ static const css_function_block invalid(
+ css_parser_token(css_parser_token::token_type::eof_token,
+ css_parser_token_placeholder()));
+ return invalid;
+ }
+ };
+
+ css_consumed_block()
+ : tag(parser_tag_type::css_eof_block)
+ {
+ }
+ css_consumed_block(parser_tag_type tag)
+ : tag(tag)
+ {
+ if (tag == parser_tag_type::css_top_block ||
+ tag == parser_tag_type::css_qualified_rule ||
+ tag == parser_tag_type::css_simple_block) {
+ /* Pre-allocate content for known vector blocks */
+ std::vector<consumed_block_ptr> vec;
+ vec.reserve(4);
+ content = std::move(vec);
+ }
+ }
+ /* Construct a block from a single lexer token (for trivial blocks) */
+ explicit css_consumed_block(parser_tag_type tag, css_parser_token &&tok)
+ : tag(tag)
+ {
+ if (tag == parser_tag_type::css_function) {
+ content = css_function_block{std::move(tok)};
+ }
+ else {
+ content = std::move(tok);
+ }
+ }
+
+ /* Attach a new block to the compound block, consuming block inside */
+ auto attach_block(consumed_block_ptr &&block) -> bool;
+ /* Attach a new argument to the compound function block, consuming block inside */
+ auto add_function_argument(consumed_block_ptr &&block) -> bool;
+
+ auto assign_token(css_parser_token &&tok) -> void
+ {
+ content = std::move(tok);
+ }
+
+ /* Empty blocks used to avoid type checks in loops */
+ const inline static std::vector<consumed_block_ptr> empty_block_vec{};
+
+ auto is_blocks_vec() const -> bool
+ {
+ return (std::holds_alternative<std::vector<consumed_block_ptr>>(content));
+ }
+
+ auto get_blocks_or_empty() const -> const std::vector<consumed_block_ptr> &
+ {
+ if (is_blocks_vec()) {
+ return std::get<std::vector<consumed_block_ptr>>(content);
+ }
+
+ return empty_block_vec;
+ }
+
+ auto is_token() const -> bool
+ {
+ return (std::holds_alternative<css_parser_token>(content));
+ }
+
+ auto get_token_or_empty() const -> const css_parser_token &
+ {
+ if (is_token()) {
+ return std::get<css_parser_token>(content);
+ }
+
+ return css_parser_eof_token();
+ }
+
+ auto is_function() const -> bool
+ {
+ return (std::holds_alternative<css_function_block>(content));
+ }
+
+ auto get_function_or_invalid() const -> const css_function_block &
+ {
+ if (is_function()) {
+ return std::get<css_function_block>(content);
+ }
+
+ return css_function_block::empty_function();
+ }
+
+ auto size() const -> std::size_t
+ {
+ auto ret = 0;
+
+ std::visit([&](auto &arg) {
+ using T = std::decay_t<decltype(arg)>;
+
+ if constexpr (std::is_same_v<T, std::vector<consumed_block_ptr>>) {
+ /* Array of blocks */
+ ret = arg.size();
+ }
+ else if constexpr (std::is_same_v<T, std::monostate>) {
+ /* Empty block */
+ ret = 0;
+ }
+ else {
+ /* Single element block */
+ ret = 1;
+ }
+ },
+ content);
+
+ return ret;
+ }
+
+ auto is_eof() -> bool
+ {
+ return tag == parser_tag_type::css_eof_block;
+ }
+
+ /* Debug methods */
+ auto token_type_str(void) const -> const char *;
+ auto debug_str(void) -> std::string;
+
+public:
+ parser_tag_type tag;
+
+private:
+ std::variant<std::monostate,
+ std::vector<consumed_block_ptr>,
+ css_parser_token,
+ css_function_block>
+ content;
+};
+
+extern const css_consumed_block css_parser_eof_block;
+
+using blocks_gen_functor = fu2::unique_function<const css_consumed_block &(void)>;
+
+class css_style_sheet;
+/*
+ * Update the existing stylesheet with another stylesheet
+ */
+auto parse_css(rspamd_mempool_t *pool, const std::string_view &st,
+ std::shared_ptr<css_style_sheet> &&other)
+ -> tl::expected<std::shared_ptr<css_style_sheet>, css_parse_error>;
+
+/*
+ * Creates a functor to consume css selectors sequence
+ */
+auto get_selectors_parser_functor(rspamd_mempool_t *pool,
+ const std::string_view &st) -> blocks_gen_functor;
+
+/*
+ * Creates a functor to process a rule definition (e.g. from embedded style tag for
+ * an element)
+ */
+auto get_rules_parser_functor(rspamd_mempool_t *pool,
+ const std::string_view &st) -> blocks_gen_functor;
+
+/**
+ * Parses a css declaration (e.g. embedded css and returns a completed html block)
+ * @param pool
+ * @param st
+ * @return
+ */
+auto parse_css_declaration(rspamd_mempool_t *pool, const std::string_view &st)
+ -> rspamd::html::html_block *;
+
+}// namespace rspamd::css
+
+#endif//RSPAMD_CSS_PARSER_HXX
diff --git a/src/libserver/css/css_property.cxx b/src/libserver/css/css_property.cxx
new file mode 100644
index 0000000..1557109
--- /dev/null
+++ b/src/libserver/css/css_property.cxx
@@ -0,0 +1,69 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "css_property.hxx"
+#include "frozen/unordered_map.h"
+#include "frozen/string.h"
+#include "libutil/cxx/util.hxx"
+
+namespace rspamd::css {
+
+constexpr const auto prop_names_map = frozen::make_unordered_map<frozen::string, css_property_type>({
+ {"font", css_property_type::PROPERTY_FONT},
+ {"font-color", css_property_type::PROPERTY_FONT_COLOR},
+ {"font-size", css_property_type::PROPERTY_FONT_SIZE},
+ {"color", css_property_type::PROPERTY_COLOR},
+ {"bgcolor", css_property_type::PROPERTY_BGCOLOR},
+ {"background-color", css_property_type::PROPERTY_BGCOLOR},
+ {"background", css_property_type::PROPERTY_BACKGROUND},
+ {"height", css_property_type::PROPERTY_HEIGHT},
+ {"width", css_property_type::PROPERTY_WIDTH},
+ {"display", css_property_type::PROPERTY_DISPLAY},
+ {"visibility", css_property_type::PROPERTY_VISIBILITY},
+ {"opacity", css_property_type::PROPERTY_OPACITY},
+});
+
+/* Ensure that we have all cases listed */
+static_assert(prop_names_map.size() >= static_cast<int>(css_property_type::PROPERTY_NYI));
+
+auto token_string_to_property(const std::string_view &inp)
+ -> css_property_type
+{
+
+ css_property_type ret = css_property_type::PROPERTY_NYI;
+
+ auto known_type = find_map(prop_names_map, inp);
+
+ if (known_type) {
+ ret = known_type.value().get();
+ }
+
+ return ret;
+}
+
+auto css_property::from_token(const css_parser_token &tok)
+ -> tl::expected<css_property, css_parse_error>
+{
+ if (tok.type == css_parser_token::token_type::ident_token) {
+ auto sv = tok.get_string_or_default("");
+
+ return css_property{token_string_to_property(sv), css_property_flag::FLAG_NORMAL};
+ }
+
+ return tl::unexpected{css_parse_error(css_parse_error_type::PARSE_ERROR_NYI)};
+}
+
+}// namespace rspamd::css
diff --git a/src/libserver/css/css_property.hxx b/src/libserver/css/css_property.hxx
new file mode 100644
index 0000000..9661222
--- /dev/null
+++ b/src/libserver/css/css_property.hxx
@@ -0,0 +1,172 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#ifndef RSPAMD_CSS_PROPERTY_HXX
+#define RSPAMD_CSS_PROPERTY_HXX
+
+#include <string>
+#include "css_tokeniser.hxx"
+#include "parse_error.hxx"
+#include "contrib/expected/expected.hpp"
+
+namespace rspamd::css {
+
+/*
+ * To be extended with properties that are interesting from the email
+ * point of view
+ */
+enum class css_property_type : std::uint16_t {
+ PROPERTY_FONT = 0,
+ PROPERTY_FONT_COLOR,
+ PROPERTY_FONT_SIZE,
+ PROPERTY_COLOR,
+ PROPERTY_BGCOLOR,
+ PROPERTY_BACKGROUND,
+ PROPERTY_HEIGHT,
+ PROPERTY_WIDTH,
+ PROPERTY_DISPLAY,
+ PROPERTY_VISIBILITY,
+ PROPERTY_OPACITY,
+ PROPERTY_NYI,
+};
+
+enum class css_property_flag : std::uint16_t {
+ FLAG_NORMAL,
+ FLAG_IMPORTANT,
+ FLAG_NOT_IMPORTANT
+};
+
+struct alignas(int) css_property {
+ css_property_type type;
+ css_property_flag flag;
+
+ css_property(css_property_type t, css_property_flag fl = css_property_flag::FLAG_NORMAL)
+ : type(t), flag(fl)
+ {
+ }
+ static tl::expected<css_property, css_parse_error> from_token(
+ const css_parser_token &tok);
+
+ constexpr auto to_string(void) const -> const char *
+ {
+ const char *ret = "nyi";
+
+ switch (type) {
+ case css_property_type::PROPERTY_FONT:
+ ret = "font";
+ break;
+ case css_property_type::PROPERTY_FONT_COLOR:
+ ret = "font-color";
+ break;
+ case css_property_type::PROPERTY_FONT_SIZE:
+ ret = "font-size";
+ break;
+ case css_property_type::PROPERTY_COLOR:
+ ret = "color";
+ break;
+ case css_property_type::PROPERTY_BGCOLOR:
+ ret = "bgcolor";
+ break;
+ case css_property_type::PROPERTY_BACKGROUND:
+ ret = "background";
+ break;
+ case css_property_type::PROPERTY_HEIGHT:
+ ret = "height";
+ break;
+ case css_property_type::PROPERTY_WIDTH:
+ ret = "width";
+ break;
+ case css_property_type::PROPERTY_DISPLAY:
+ ret = "display";
+ break;
+ case css_property_type::PROPERTY_VISIBILITY:
+ ret = "visibility";
+ break;
+ case css_property_type::PROPERTY_OPACITY:
+ ret = "opacity";
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+ }
+
+ /* Helpers to define which values are valid for which properties */
+ auto is_color(void) const -> bool
+ {
+ return type == css_property_type::PROPERTY_COLOR ||
+ type == css_property_type::PROPERTY_BACKGROUND ||
+ type == css_property_type::PROPERTY_BGCOLOR ||
+ type == css_property_type::PROPERTY_FONT_COLOR ||
+ type == css_property_type::PROPERTY_FONT;
+ }
+ auto is_dimension(void) const -> bool
+ {
+ return type == css_property_type::PROPERTY_HEIGHT ||
+ type == css_property_type::PROPERTY_WIDTH ||
+ type == css_property_type::PROPERTY_FONT_SIZE ||
+ type == css_property_type::PROPERTY_FONT;
+ }
+
+ auto is_normal_number(void) const -> bool
+ {
+ return type == css_property_type::PROPERTY_OPACITY;
+ }
+
+ auto is_display(void) const -> bool
+ {
+ return type == css_property_type::PROPERTY_DISPLAY;
+ }
+
+ auto is_visibility(void) const -> bool
+ {
+ return type == css_property_type::PROPERTY_VISIBILITY;
+ }
+
+ auto operator==(const css_property &other) const
+ {
+ return type == other.type;
+ }
+};
+
+
+}// namespace rspamd::css
+
+/* Make properties hashable */
+namespace std {
+template<>
+class hash<rspamd::css::css_property> {
+public:
+ using is_avalanching = void;
+ /* Mix bits to provide slightly better distribution but being constexpr */
+ constexpr size_t operator()(const rspamd::css::css_property &prop) const
+ {
+ std::size_t key = 0xdeadbeef ^ static_cast<std::size_t>(prop.type);
+ key = (~key) + (key << 21);
+ key = key ^ (key >> 24);
+ key = (key + (key << 3)) + (key << 8);
+ key = key ^ (key >> 14);
+ key = (key + (key << 2)) + (key << 4);
+ key = key ^ (key >> 28);
+ key = key + (key << 31);
+ return key;
+ }
+};
+}// namespace std
+
+#endif//RSPAMD_CSS_PROPERTY_HXX \ No newline at end of file
diff --git a/src/libserver/css/css_rule.cxx b/src/libserver/css/css_rule.cxx
new file mode 100644
index 0000000..4e33ac7
--- /dev/null
+++ b/src/libserver/css/css_rule.cxx
@@ -0,0 +1,531 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "css_rule.hxx"
+#include "css.hxx"
+#include "libserver/html/html_block.hxx"
+#include <limits>
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+namespace rspamd::css {
+
+/* Class methods */
+void css_rule::override_values(const css_rule &other)
+{
+ int bits = 0;
+ /* Ensure that our bitset is large enough */
+ static_assert(1 << std::variant_size_v<decltype(css_value::value)> <
+ std::numeric_limits<int>::max());
+
+ for (const auto &v: values) {
+ bits |= static_cast<int>(1 << v.value.index());
+ }
+
+ for (const auto &ov: other.values) {
+ if (isset(&bits, static_cast<int>(1 << ov.value.index()))) {
+ /* We need to override the existing value */
+ /*
+ * The algorithm is not very efficient,
+ * so we need to sort the values first and have a O(N) algorithm
+ * On the other hand, values vectors are usually limited to the
+ * number of elements about less then 10, so this O(N^2) algorithm
+ * is probably ok here
+ */
+ for (auto &v: values) {
+ if (v.value.index() == ov.value.index()) {
+ v = ov;
+ }
+ }
+ }
+ }
+
+ /* Copy only not set values */
+ std::copy_if(other.values.begin(), other.values.end(), std::back_inserter(values),
+ [&bits](const auto &elt) -> bool {
+ return (bits & (1 << static_cast<int>(elt.value.index()))) == 0;
+ });
+}
+
+void css_rule::merge_values(const css_rule &other)
+{
+ unsigned int bits = 0;
+
+ for (const auto &v: values) {
+ bits |= 1 << v.value.index();
+ }
+
+ /* Copy only not set values */
+ std::copy_if(other.values.begin(), other.values.end(), std::back_inserter(values),
+ [&bits](const auto &elt) -> bool {
+ return (bits & (1 << elt.value.index())) == 0;
+ });
+}
+
+auto css_declarations_block::add_rule(rule_shared_ptr rule) -> bool
+{
+ auto it = rules.find(rule);
+ auto &&remote_prop = rule->get_prop();
+ auto ret = true;
+
+ if (rule->get_values().size() == 0) {
+ /* Ignore rules with no values */
+ return false;
+ }
+
+ if (it != rules.end()) {
+ auto &&local_rule = *it;
+ auto &&local_prop = local_rule->get_prop();
+
+ if (local_prop.flag == css_property_flag::FLAG_IMPORTANT) {
+ if (remote_prop.flag == css_property_flag::FLAG_IMPORTANT) {
+ local_rule->override_values(*rule);
+ }
+ else {
+ /* Override remote not important over local important */
+ local_rule->merge_values(*rule);
+ }
+ }
+ else if (local_prop.flag == css_property_flag::FLAG_NOT_IMPORTANT) {
+ if (remote_prop.flag == css_property_flag::FLAG_NOT_IMPORTANT) {
+ local_rule->override_values(*rule);
+ }
+ else {
+ /* Override local not important over important */
+ local_rule->merge_values(*rule);
+ }
+ }
+ else {
+ if (remote_prop.flag == css_property_flag::FLAG_IMPORTANT) {
+ /* Override with remote */
+ local_rule->override_values(*rule);
+ }
+ else if (remote_prop.flag == css_property_flag::FLAG_NOT_IMPORTANT) {
+ /* Ignore remote not important over local normal */
+ ret = false;
+ }
+ else {
+ /* Merge both */
+ local_rule->merge_values(*rule);
+ }
+ }
+ }
+ else {
+ rules.insert(std::move(rule));
+ }
+
+ return ret;
+}
+
+}// namespace rspamd::css
+
+namespace rspamd::css {
+
+/* Static functions */
+
+static auto
+allowed_property_value(const css_property &prop, const css_consumed_block &parser_block)
+ -> std::optional<css_value>
+{
+ if (prop.is_color()) {
+ if (parser_block.is_token()) {
+ /* A single token */
+ const auto &tok = parser_block.get_token_or_empty();
+
+ if (tok.type == css_parser_token::token_type::hash_token) {
+ return css_value::maybe_color_from_hex(tok.get_string_or_default(""));
+ }
+ else if (tok.type == css_parser_token::token_type::ident_token) {
+ auto &&ret = css_value::maybe_color_from_string(tok.get_string_or_default(""));
+
+ return ret;
+ }
+ }
+ else if (parser_block.is_function()) {
+ const auto &func = parser_block.get_function_or_invalid();
+
+ auto &&ret = css_value::maybe_color_from_function(func);
+ return ret;
+ }
+ }
+ if (prop.is_dimension()) {
+ if (parser_block.is_token()) {
+ /* A single token */
+ const auto &tok = parser_block.get_token_or_empty();
+
+ if (tok.type == css_parser_token::token_type::number_token) {
+ return css_value::maybe_dimension_from_number(tok);
+ }
+ }
+ }
+ if (prop.is_display()) {
+ if (parser_block.is_token()) {
+ /* A single token */
+ const auto &tok = parser_block.get_token_or_empty();
+
+ if (tok.type == css_parser_token::token_type::ident_token) {
+ return css_value::maybe_display_from_string(tok.get_string_or_default(""));
+ }
+ }
+ }
+ if (prop.is_visibility()) {
+ if (parser_block.is_token()) {
+ /* A single token */
+ const auto &tok = parser_block.get_token_or_empty();
+
+ if (tok.type == css_parser_token::token_type::ident_token) {
+ return css_value::maybe_display_from_string(tok.get_string_or_default(""));
+ }
+ }
+ }
+ if (prop.is_normal_number()) {
+ if (parser_block.is_token()) {
+ /* A single token */
+ const auto &tok = parser_block.get_token_or_empty();
+
+ if (tok.type == css_parser_token::token_type::number_token) {
+ return css_value{tok.get_normal_number_or_default(0)};
+ }
+ }
+ }
+
+ return std::nullopt;
+}
+
+auto process_declaration_tokens(rspamd_mempool_t *pool,
+ blocks_gen_functor &&next_block_functor)
+ -> css_declarations_block_ptr
+{
+ css_declarations_block_ptr ret;
+ bool can_continue = true;
+ css_property cur_property{css_property_type::PROPERTY_NYI,
+ css_property_flag::FLAG_NORMAL};
+ static const css_property bad_property{css_property_type::PROPERTY_NYI,
+ css_property_flag::FLAG_NORMAL};
+ std::shared_ptr<css_rule> cur_rule;
+
+ enum {
+ parse_property,
+ parse_value,
+ ignore_value, /* For unknown properties */
+ } state = parse_property;
+
+ auto seen_not = false;
+ ret = std::make_shared<css_declarations_block>();
+
+ while (can_continue) {
+ const auto &next_tok = next_block_functor();
+
+ switch (next_tok.tag) {
+ case css_consumed_block::parser_tag_type::css_component:
+ /* Component can be a property or a compound list of values */
+ if (state == parse_property) {
+ cur_property = css_property::from_token(next_tok.get_token_or_empty())
+ .value_or(bad_property);
+
+ if (cur_property.type == css_property_type::PROPERTY_NYI) {
+ state = ignore_value;
+ /* Ignore everything till ; */
+ continue;
+ }
+
+ msg_debug_css("got css property: %s", cur_property.to_string());
+
+ /* We now expect colon block */
+ const auto &expect_colon_block = next_block_functor();
+
+ if (expect_colon_block.tag != css_consumed_block::parser_tag_type::css_component) {
+ state = ignore_value; /* Ignore up to the next rule */
+ }
+ else {
+ const auto &expect_colon_tok = expect_colon_block.get_token_or_empty();
+
+ if (expect_colon_tok.type != css_parser_token::token_type::colon_token) {
+ msg_debug_css("invalid rule, no colon after property");
+ state = ignore_value; /* Ignore up to the next rule */
+ }
+ else {
+ state = parse_value;
+ cur_rule = std::make_shared<css_rule>(cur_property);
+ }
+ }
+ }
+ else if (state == parse_value) {
+ /* Check semicolon */
+ if (next_tok.is_token()) {
+ const auto &parser_tok = next_tok.get_token_or_empty();
+
+ if (parser_tok.type == css_parser_token::token_type::semicolon_token && cur_rule) {
+ ret->add_rule(std::move(cur_rule));
+ state = parse_property;
+ seen_not = false;
+ continue;
+ }
+ else if (parser_tok.type == css_parser_token::token_type::delim_token) {
+ if (parser_tok.get_string_or_default("") == "!") {
+ /* Probably something like !important */
+ seen_not = true;
+ }
+ }
+ else if (parser_tok.type == css_parser_token::token_type::ident_token) {
+ if (parser_tok.get_string_or_default("") == "important") {
+ if (seen_not) {
+ msg_debug_css("add !important flag to property %s",
+ cur_property.to_string());
+ cur_property.flag = css_property_flag::FLAG_NOT_IMPORTANT;
+ }
+ else {
+ msg_debug_css("add important flag to property %s",
+ cur_property.to_string());
+ cur_property.flag = css_property_flag::FLAG_IMPORTANT;
+ }
+
+ seen_not = false;
+
+ continue;
+ }
+ else {
+ seen_not = false;
+ }
+ }
+ }
+
+ auto maybe_value = allowed_property_value(cur_property, next_tok);
+
+ if (maybe_value) {
+ msg_debug_css("added value %s to the property %s",
+ maybe_value.value().debug_str().c_str(),
+ cur_property.to_string());
+ cur_rule->add_value(maybe_value.value());
+ }
+ }
+ else {
+ /* Ignore all till ; */
+ if (next_tok.is_token()) {
+ const auto &parser_tok = next_tok.get_token_or_empty();
+
+ if (parser_tok.type == css_parser_token::token_type::semicolon_token) {
+ state = parse_property;
+ }
+ }
+ }
+ break;
+ case css_consumed_block::parser_tag_type::css_function:
+ if (state == parse_value) {
+ auto maybe_value = allowed_property_value(cur_property, next_tok);
+
+ if (maybe_value && cur_rule) {
+ msg_debug_css("added value %s to the property %s",
+ maybe_value.value().debug_str().c_str(),
+ cur_property.to_string());
+ cur_rule->add_value(maybe_value.value());
+ }
+ }
+ break;
+ case css_consumed_block::parser_tag_type::css_eof_block:
+ if (state == parse_value) {
+ ret->add_rule(std::move(cur_rule));
+ }
+ can_continue = false;
+ break;
+ default:
+ can_continue = false;
+ break;
+ }
+ }
+
+ return ret; /* copy elision */
+}
+
+auto css_declarations_block::merge_block(const css_declarations_block &other, merge_type how) -> void
+{
+ const auto &other_rules = other.get_rules();
+
+
+ for (auto &rule: other_rules) {
+ auto &&found_it = rules.find(rule);
+
+ if (found_it != rules.end()) {
+ /* Duplicate, need to merge */
+ switch (how) {
+ case merge_type::merge_override:
+ /* Override */
+ (*found_it)->override_values(*rule);
+ break;
+ case merge_type::merge_duplicate:
+ /* Merge values */
+ add_rule(rule);
+ break;
+ case merge_type::merge_parent:
+ /* Do not merge parent rule if more specific local one is presented */
+ break;
+ }
+ }
+ else {
+ /* New property, just insert */
+ rules.insert(rule);
+ }
+ }
+}
+
+auto css_declarations_block::compile_to_block(rspamd_mempool_t *pool) const -> rspamd::html::html_block *
+{
+ auto *block = rspamd_mempool_alloc0_type(pool, rspamd::html::html_block);
+ auto opacity = -1;
+ const css_rule *font_rule = nullptr, *background_rule = nullptr;
+
+ for (const auto &rule: rules) {
+ auto prop = rule->get_prop().type;
+ const auto &vals = rule->get_values();
+
+ if (vals.empty()) {
+ continue;
+ }
+
+ switch (prop) {
+ case css_property_type::PROPERTY_VISIBILITY:
+ case css_property_type::PROPERTY_DISPLAY: {
+ auto disp = vals.back().to_display().value_or(css_display_value::DISPLAY_INLINE);
+ block->set_display(disp);
+ break;
+ }
+ case css_property_type::PROPERTY_FONT_SIZE: {
+ auto fs = vals.back().to_dimension();
+ if (fs) {
+ block->set_font_size(fs.value().dim, fs.value().is_percent);
+ }
+ }
+ case css_property_type::PROPERTY_OPACITY: {
+ opacity = vals.back().to_number().value_or(opacity);
+ break;
+ }
+ case css_property_type::PROPERTY_FONT_COLOR:
+ case css_property_type::PROPERTY_COLOR: {
+ auto color = vals.back().to_color();
+ if (color) {
+ block->set_fgcolor(color.value());
+ }
+ break;
+ }
+ case css_property_type::PROPERTY_BGCOLOR: {
+ auto color = vals.back().to_color();
+ if (color) {
+ block->set_bgcolor(color.value());
+ }
+ break;
+ }
+ case css_property_type::PROPERTY_HEIGHT: {
+ auto w = vals.back().to_dimension();
+ if (w) {
+ block->set_width(w.value().dim, w.value().is_percent);
+ }
+ break;
+ }
+ case css_property_type::PROPERTY_WIDTH: {
+ auto h = vals.back().to_dimension();
+ if (h) {
+ block->set_width(h.value().dim, h.value().is_percent);
+ }
+ break;
+ }
+ /* Optional attributes */
+ case css_property_type::PROPERTY_FONT:
+ font_rule = rule.get();
+ break;
+ case css_property_type::PROPERTY_BACKGROUND:
+ background_rule = rule.get();
+ break;
+ default:
+ /* Do nothing for now */
+ break;
+ }
+ }
+
+ /* Optional properties */
+ if (!(block->fg_color_mask) && font_rule) {
+ auto &vals = font_rule->get_values();
+
+ for (const auto &val: vals) {
+ auto maybe_color = val.to_color();
+
+ if (maybe_color) {
+ block->set_fgcolor(maybe_color.value());
+ }
+ }
+ }
+
+ if (!(block->font_mask) && font_rule) {
+ auto &vals = font_rule->get_values();
+
+ for (const auto &val: vals) {
+ auto maybe_dim = val.to_dimension();
+
+ if (maybe_dim) {
+ block->set_font_size(maybe_dim.value().dim, maybe_dim.value().is_percent);
+ }
+ }
+ }
+
+ if (!(block->bg_color_mask) && background_rule) {
+ auto &vals = background_rule->get_values();
+
+ for (const auto &val: vals) {
+ auto maybe_color = val.to_color();
+
+ if (maybe_color) {
+ block->set_bgcolor(maybe_color.value());
+ }
+ }
+ }
+
+ return block;
+}
+
+void css_rule::add_value(const css_value &value)
+{
+ values.push_back(value);
+}
+
+
+TEST_SUITE("css")
+{
+ TEST_CASE("simple css rules")
+ {
+ const std::vector<std::pair<const char *, std::vector<css_property>>> cases{
+ {"font-size:12.0pt;line-height:115%",
+ {css_property(css_property_type::PROPERTY_FONT_SIZE)}},
+ {"font-size:12.0pt;display:none",
+ {css_property(css_property_type::PROPERTY_FONT_SIZE),
+ css_property(css_property_type::PROPERTY_DISPLAY)}}};
+
+ auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "css", 0);
+
+ for (const auto &c: cases) {
+ auto res = process_declaration_tokens(pool,
+ get_rules_parser_functor(pool, c.first));
+
+ CHECK(res.get() != nullptr);
+
+ for (auto i = 0; i < c.second.size(); i++) {
+ CHECK(res->has_property(c.second[i]));
+ }
+ }
+ }
+}
+
+}// namespace rspamd::css \ No newline at end of file
diff --git a/src/libserver/css/css_rule.hxx b/src/libserver/css/css_rule.hxx
new file mode 100644
index 0000000..114b83e
--- /dev/null
+++ b/src/libserver/css/css_rule.hxx
@@ -0,0 +1,153 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#ifndef RSPAMD_CSS_RULE_HXX
+#define RSPAMD_CSS_RULE_HXX
+
+#include "css_value.hxx"
+#include "css_property.hxx"
+#include "css_parser.hxx"
+#include "contrib/ankerl/unordered_dense.h"
+#include "libutil/cxx/util.hxx"
+#include "libutil/cxx/hash_util.hxx"
+#include <vector>
+#include <memory>
+
+namespace rspamd::html {
+/* Forward declaration */
+struct html_block;
+}// namespace rspamd::html
+
+namespace rspamd::css {
+
+class css_rule {
+ css_property prop;
+ using css_values_vec = std::vector<css_value>;
+ css_values_vec values;
+
+public:
+ /* We must create css rule explicitly from a property and values */
+ css_rule() = delete;
+
+ css_rule(const css_rule &other) = delete;
+
+ /* Constructors */
+ css_rule(css_rule &&other) noexcept = default;
+
+ explicit css_rule(css_property &&prop, css_values_vec &&values) noexcept
+ : prop(prop), values(std::forward<css_values_vec>(values))
+ {
+ }
+
+ explicit css_rule(const css_property &prop) noexcept
+ : prop(prop), values{}
+ {
+ }
+
+ /* Methods */
+ /* Comparison is special, as we care merely about property, not the values */
+ auto operator==(const css_rule &other) const
+ {
+ return prop == other.prop;
+ }
+
+ constexpr const css_values_vec &get_values(void) const
+ {
+ return values;
+ }
+ constexpr const css_property &get_prop(void) const
+ {
+ return prop;
+ }
+
+ /* Import values from another rules according to the importance */
+ void override_values(const css_rule &other);
+ void merge_values(const css_rule &other);
+ void add_value(const css_value &value);
+};
+
+}// namespace rspamd::css
+
+/* Make rules hashable by property */
+namespace std {
+template<>
+class hash<rspamd::css::css_rule> {
+public:
+ using is_avalanching = void;
+ constexpr auto operator()(const rspamd::css::css_rule &rule) const -> auto
+ {
+ return hash<rspamd::css::css_property>()(rule.get_prop());
+ }
+};
+
+}// namespace std
+
+namespace rspamd::css {
+
+/**
+ * Class that is designed to hold css declaration (a set of rules)
+ */
+class css_declarations_block {
+public:
+ using rule_shared_ptr = std::shared_ptr<css_rule>;
+ using rule_shared_hash = smart_ptr_hash<css_rule>;
+ using rule_shared_eq = smart_ptr_equal<css_rule>;
+ enum class merge_type {
+ merge_duplicate,
+ merge_parent,
+ merge_override
+ };
+
+ css_declarations_block() = default;
+ auto add_rule(rule_shared_ptr rule) -> bool;
+ auto merge_block(const css_declarations_block &other,
+ merge_type how = merge_type::merge_duplicate) -> void;
+ auto get_rules(void) const -> const auto &
+ {
+ return rules;
+ }
+
+ /**
+ * Returns if a declaration block has some property
+ * @param prop
+ * @return
+ */
+ auto has_property(const css_property &prop) const -> bool
+ {
+ return (rules.find(css_rule{prop}) != rules.end());
+ }
+
+ /**
+ * Compile CSS declaration to the html block
+ * @param pool used to carry memory required for html_block
+ * @return html block structure
+ */
+ auto compile_to_block(rspamd_mempool_t *pool) const -> rspamd::html::html_block *;
+
+private:
+ ankerl::unordered_dense::set<rule_shared_ptr, rule_shared_hash, rule_shared_eq> rules;
+};
+
+using css_declarations_block_ptr = std::shared_ptr<css_declarations_block>;
+
+auto process_declaration_tokens(rspamd_mempool_t *pool,
+ blocks_gen_functor &&next_token_functor)
+ -> css_declarations_block_ptr;
+
+}// namespace rspamd::css
+
+#endif//RSPAMD_CSS_RULE_HXX \ No newline at end of file
diff --git a/src/libserver/css/css_rule_parser.rl b/src/libserver/css/css_rule_parser.rl
new file mode 100644
index 0000000..e3b1876
--- /dev/null
+++ b/src/libserver/css/css_rule_parser.rl
@@ -0,0 +1,27 @@
+%%{
+ machine css_parser;
+ alphtype unsigned char;
+ include css_syntax "css_syntax.rl";
+
+ main := declaration;
+}%%
+
+%% write data;
+
+#include <cstddef>
+
+namespace rspamd::css {
+
+int
+foo (const unsigned char *data, std::size_t len)
+{
+ const unsigned char *p = data, *pe = data + len, *eof;
+ int cs;
+
+ %% write init;
+ %% write exec;
+
+ return cs;
+}
+
+} \ No newline at end of file
diff --git a/src/libserver/css/css_selector.cxx b/src/libserver/css/css_selector.cxx
new file mode 100644
index 0000000..a62ffff
--- /dev/null
+++ b/src/libserver/css/css_selector.cxx
@@ -0,0 +1,226 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "css_selector.hxx"
+#include "css.hxx"
+#include "libserver/html/html.hxx"
+#include "fmt/core.h"
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+namespace rspamd::css {
+
+auto process_selector_tokens(rspamd_mempool_t *pool,
+ blocks_gen_functor &&next_token_functor)
+ -> selectors_vec
+{
+ selectors_vec ret;
+ bool can_continue = true;
+ enum class selector_process_state {
+ selector_parse_start = 0,
+ selector_expect_ident,
+ selector_ident_consumed,
+ selector_ignore_attribute,
+ selector_ignore_function,
+ selector_ignore_combination
+ } state = selector_process_state::selector_parse_start;
+ std::unique_ptr<css_selector> cur_selector;
+
+
+ while (can_continue) {
+ const auto &next_tok = next_token_functor();
+
+ if (next_tok.tag == css_consumed_block::parser_tag_type::css_component) {
+ const auto &parser_tok = next_tok.get_token_or_empty();
+
+ if (state == selector_process_state::selector_parse_start) {
+ /*
+ * At the beginning of the parsing we can expect either
+ * delim or an ident, everything else is discarded for now
+ */
+ msg_debug_css("start consume selector");
+
+ switch (parser_tok.type) {
+ case css_parser_token::token_type::delim_token: {
+ auto delim_c = parser_tok.get_delim();
+
+ if (delim_c == '.') {
+ cur_selector = std::make_unique<css_selector>(
+ css_selector::selector_type::SELECTOR_CLASS);
+ state = selector_process_state::selector_expect_ident;
+ }
+ else if (delim_c == '#') {
+ cur_selector = std::make_unique<css_selector>(
+ css_selector::selector_type::SELECTOR_ID);
+ state = selector_process_state::selector_expect_ident;
+ }
+ else if (delim_c == '*') {
+ cur_selector = std::make_unique<css_selector>(
+ css_selector::selector_type::SELECTOR_ALL);
+ state = selector_process_state::selector_ident_consumed;
+ }
+ break;
+ }
+ case css_parser_token::token_type::ident_token: {
+ auto tag_id = html::html_tag_by_name(parser_tok.get_string_or_default(""));
+
+ if (tag_id) {
+ cur_selector = std::make_unique<css_selector>(tag_id.value());
+ }
+ state = selector_process_state::selector_ident_consumed;
+ break;
+ }
+ case css_parser_token::token_type::hash_token:
+ cur_selector = std::make_unique<css_selector>(
+ css_selector::selector_type::SELECTOR_ID);
+ cur_selector->value =
+ parser_tok.get_string_or_default("");
+ state = selector_process_state::selector_ident_consumed;
+ break;
+ default:
+ msg_debug_css("cannot consume more of a selector, invalid parser token: %s; expected start",
+ next_tok.token_type_str());
+ can_continue = false;
+ break;
+ }
+ }
+ else if (state == selector_process_state::selector_expect_ident) {
+ /*
+ * We got something like a selector start, so we expect
+ * a plain ident
+ */
+ if (parser_tok.type == css_parser_token::token_type::ident_token && cur_selector) {
+ cur_selector->value = parser_tok.get_string_or_default("");
+ state = selector_process_state::selector_ident_consumed;
+ }
+ else {
+ msg_debug_css("cannot consume more of a selector, invalid parser token: %s; expected ident",
+ next_tok.token_type_str());
+ can_continue = false;
+ }
+ }
+ else if (state == selector_process_state::selector_ident_consumed) {
+ if (parser_tok.type == css_parser_token::token_type::comma_token && cur_selector) {
+ /* Got full selector, attach it to the vector and go further */
+ msg_debug_css("attached selector: %s", cur_selector->debug_str().c_str());
+ ret.push_back(std::move(cur_selector));
+ state = selector_process_state::selector_parse_start;
+ }
+ else if (parser_tok.type == css_parser_token::token_type::semicolon_token) {
+ /* TODO: implement adjustments */
+ state = selector_process_state::selector_ignore_function;
+ }
+ else if (parser_tok.type == css_parser_token::token_type::osqbrace_token) {
+ /* TODO: implement attributes checks */
+ state = selector_process_state::selector_ignore_attribute;
+ }
+ else {
+ /* TODO: implement selectors combinations */
+ state = selector_process_state::selector_ignore_combination;
+ }
+ }
+ else {
+ /* Ignore state; ignore all till ',' token or eof token */
+ if (parser_tok.type == css_parser_token::token_type::comma_token && cur_selector) {
+ /* Got full selector, attach it to the vector and go further */
+ ret.push_back(std::move(cur_selector));
+ state = selector_process_state::selector_parse_start;
+ }
+ else {
+ auto debug_str = parser_tok.get_string_or_default("");
+ msg_debug_css("ignore token %*s", (int) debug_str.size(),
+ debug_str.data());
+ }
+ }
+ }
+ else {
+ /* End of parsing */
+ if (state == selector_process_state::selector_ident_consumed && cur_selector) {
+ msg_debug_css("attached selector: %s", cur_selector->debug_str().c_str());
+ ret.push_back(std::move(cur_selector));
+ }
+ else {
+ msg_debug_css("not attached selector, state: %d", static_cast<int>(state));
+ }
+ can_continue = false;
+ }
+ }
+
+ return ret; /* copy elision */
+}
+
+auto css_selector::debug_str() const -> std::string
+{
+ std::string ret;
+
+ if (type == selector_type::SELECTOR_ID) {
+ ret += "#";
+ }
+ else if (type == selector_type::SELECTOR_CLASS) {
+ ret += ".";
+ }
+ else if (type == selector_type::SELECTOR_ALL) {
+ ret = "*";
+
+ return ret;
+ }
+
+ std::visit([&](auto arg) -> void {
+ using T = std::decay_t<decltype(arg)>;
+
+ if constexpr (std::is_same_v<T, tag_id_t>) {
+ ret += fmt::format("tag: {}", static_cast<int>(arg));
+ }
+ else {
+ ret += arg;
+ }
+ },
+ value);
+
+ return ret;
+}
+
+TEST_SUITE("css")
+{
+ TEST_CASE("simple css selectors")
+ {
+ const std::vector<std::pair<const char *, std::vector<css_selector::selector_type>>> cases{
+ {"em", {css_selector::selector_type::SELECTOR_TAG}},
+ {"*", {css_selector::selector_type::SELECTOR_ALL}},
+ {".class", {css_selector::selector_type::SELECTOR_CLASS}},
+ {"#id", {css_selector::selector_type::SELECTOR_ID}},
+ {"em,.class,#id", {css_selector::selector_type::SELECTOR_TAG, css_selector::selector_type::SELECTOR_CLASS, css_selector::selector_type::SELECTOR_ID}},
+ };
+
+ auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "css", 0);
+
+ for (const auto &c: cases) {
+ auto res = process_selector_tokens(pool,
+ get_selectors_parser_functor(pool, c.first));
+
+ CHECK(c.second.size() == res.size());
+
+ for (auto i = 0; i < c.second.size(); i++) {
+ CHECK(res[i]->type == c.second[i]);
+ }
+ }
+
+ rspamd_mempool_delete(pool);
+ }
+}
+
+}// namespace rspamd::css
diff --git a/src/libserver/css/css_selector.hxx b/src/libserver/css/css_selector.hxx
new file mode 100644
index 0000000..65b185a
--- /dev/null
+++ b/src/libserver/css/css_selector.hxx
@@ -0,0 +1,134 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifndef RSPAMD_CSS_SELECTOR_HXX
+#define RSPAMD_CSS_SELECTOR_HXX
+
+#include <variant>
+#include <string>
+#include <optional>
+#include <vector>
+#include <memory>
+
+#include "function2/function2.hpp"
+#include "parse_error.hxx"
+#include "css_parser.hxx"
+#include "libserver/html/html_tags.h"
+#include "libcryptobox/cryptobox.h"
+
+namespace rspamd::css {
+
+/*
+ * Holds a value for css selector, internal is handled by variant
+ */
+struct css_selector {
+ enum class selector_type {
+ SELECTOR_TAG, /* e.g. tr, for this value we use tag_id_t */
+ SELECTOR_CLASS, /* generic class, e.g. .class */
+ SELECTOR_ID, /* e.g. #id */
+ SELECTOR_ALL /* * selector */
+ };
+
+ selector_type type;
+ std::variant<tag_id_t, std::string_view> value;
+
+ /* Conditions for the css selector */
+ /* Dependency on attributes */
+ struct css_attribute_condition {
+ std::string_view attribute;
+ std::string_view op = "";
+ std::string_view value = "";
+ };
+
+ /* General dependency chain */
+ using css_selector_ptr = std::unique_ptr<css_selector>;
+ using css_selector_dep = std::variant<css_attribute_condition, css_selector_ptr>;
+ std::vector<css_selector_dep> dependencies;
+
+ auto to_tag(void) const -> std::optional<tag_id_t>
+ {
+ if (type == selector_type::SELECTOR_TAG) {
+ return std::get<tag_id_t>(value);
+ }
+ return std::nullopt;
+ }
+
+ auto to_string(void) const -> std::optional<const std::string_view>
+ {
+ if (type != selector_type::SELECTOR_TAG) {
+ return std::string_view(std::get<std::string_view>(value));
+ }
+ return std::nullopt;
+ };
+
+ explicit css_selector(selector_type t)
+ : type(t)
+ {
+ }
+ explicit css_selector(tag_id_t t)
+ : type(selector_type::SELECTOR_TAG)
+ {
+ value = t;
+ }
+ explicit css_selector(const std::string_view &st, selector_type t = selector_type::SELECTOR_ID)
+ : type(t)
+ {
+ value = st;
+ }
+
+ auto operator==(const css_selector &other) const -> bool
+ {
+ return type == other.type && value == other.value;
+ }
+
+ auto debug_str(void) const -> std::string;
+};
+
+
+using selectors_vec = std::vector<std::unique_ptr<css_selector>>;
+
+/*
+ * Consume selectors token and split them to the list of selectors
+ */
+auto process_selector_tokens(rspamd_mempool_t *pool,
+ blocks_gen_functor &&next_token_functor)
+ -> selectors_vec;
+
+}// namespace rspamd::css
+
+/* Selectors hashing */
+namespace std {
+template<>
+class hash<rspamd::css::css_selector> {
+public:
+ using is_avalanching = void;
+ auto operator()(const rspamd::css::css_selector &sel) const -> std::size_t
+ {
+ if (sel.type == rspamd::css::css_selector::selector_type::SELECTOR_TAG) {
+ return static_cast<std::size_t>(std::get<tag_id_t>(sel.value));
+ }
+ else {
+ const auto &sv = std::get<std::string_view>(sel.value);
+
+ return rspamd_cryptobox_fast_hash(sv.data(), sv.size(), 0xdeadbabe);
+ }
+ }
+};
+}// namespace std
+
+#endif//RSPAMD_CSS_SELECTOR_HXX
diff --git a/src/libserver/css/css_selector_parser.rl b/src/libserver/css/css_selector_parser.rl
new file mode 100644
index 0000000..f5ae936
--- /dev/null
+++ b/src/libserver/css/css_selector_parser.rl
@@ -0,0 +1,27 @@
+%%{
+ machine css_parser;
+ alphtype unsigned char;
+ include css_syntax "css_syntax.rl";
+
+ main := selectors_group;
+}%%
+
+%% write data;
+
+#include <cstddef>
+
+namespace rspamd::css {
+
+int
+parse_css_selector (const unsigned char *data, std::size_t len)
+{
+ const unsigned char *p = data, *pe = data + len, *eof;
+ int cs;
+
+ %% write init;
+ %% write exec;
+
+ return cs;
+}
+
+} \ No newline at end of file
diff --git a/src/libserver/css/css_style.hxx b/src/libserver/css/css_style.hxx
new file mode 100644
index 0000000..429e58f
--- /dev/null
+++ b/src/libserver/css/css_style.hxx
@@ -0,0 +1,66 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifndef RSPAMD_CSS_STYLE_HXX
+#define RSPAMD_CSS_STYLE_HXX
+
+#include <memory>
+#include <vector>
+#include "css_rule.hxx"
+#include "css_selector.hxx"
+
+namespace rspamd::css {
+
+/*
+ * Full CSS style representation
+ */
+class css_style {
+public:
+ /* Make class trivial */
+ css_style(const css_style &other) = default;
+
+ css_style(const std::shared_ptr<css_style> &_parent)
+ : parent(_parent)
+ {
+ propagate_from_parent();
+ }
+ css_style(const std::shared_ptr<css_style> &_parent,
+ const std::vector<std::shared_ptr<css_selector>> &_selectors)
+ : parent(_parent)
+ {
+ selectors.reserve(_selectors.size());
+
+ for (const auto &sel_ptr: _selectors) {
+ selectors.emplace_back(sel_ptr);
+ }
+
+ propagate_from_parent();
+ }
+
+private:
+ std::vector<std::weak_ptr<css_selector>> selectors;
+ std::weak_ptr<css_style> parent;
+ std::vector<css_rule> rules;
+
+private:
+ void propagate_from_parent(void); /* Construct full style using parent */
+};
+
+}// namespace rspamd::css
+
+#endif//RSPAMD_CSS_STYLE_HXX
diff --git a/src/libserver/css/css_syntax.rl b/src/libserver/css/css_syntax.rl
new file mode 100644
index 0000000..93da44b
--- /dev/null
+++ b/src/libserver/css/css_syntax.rl
@@ -0,0 +1,110 @@
+%%{
+ # CSS3 EBNF derived
+ machine css_syntax;
+
+ # Primitive Atoms
+ COMMENT = (
+ '/*' ( any )* :>> '*/'
+ );
+ QUOTED_STRING = ('"' ( [^"\\] | /\\./ )* "'");
+ BARE_URL_CHARS = ((0x21
+ | 0x23..0x26
+ | 0x2A..0xFF)+);
+ BARE_URL = BARE_URL_CHARS;
+ URL = 'url(' ( QUOTED_STRING | space* BARE_URL space* ) ')';
+ nonascii = [^0x00-0x7F];
+ nmstart = ([_a-zA-Z] | nonascii);
+ nmchar = ([_a-zA-Z0-9] | 0x2D | nonascii);
+ name = nmchar+;
+ num = ([0-9]+ | ([0-9]* '.' [0-9]+));
+ CRLF = "\r\n" | ("\r" [^\n]) | ([^\r] "\n");
+ IDENT = ([\-]? nmstart nmchar*);
+ ATTR = 'attr(' IDENT ')';
+
+ DIMENSION = '-'? num space? ( 'ch' | 'cm' | 'em' | 'ex' | 'fr' | 'in' | 'mm' | 'pc' | 'pt' | 'px' | 'Q' | 'rem' | 'vh' | 'vmax' | 'vmin' | 'vw' | 'dpi' );
+ NUMBER = '-'? num;
+ HASH = '#' name;
+ HEX = '#' [0-9a-fA-F]{1,6};
+ PERCENTAGE = '-'? num '%';
+ INCLUDES = '~=';
+ DASHMATCH = '|=';
+ PREFIXMATCH = '^=';
+ SUFFIXMATCH = '$=';
+ SUBSTRINGMATCH = '*=';
+ PLUS = '+';
+ GREATER = '>';
+ COMMA = ',';
+ TILDE = '~';
+ S = space;
+
+ # Property name
+ property = ( QUOTED_STRING | IDENT );
+
+ # Values
+ important = space* '!' space* 'important';
+ expression = ( ( '+' | PERCENTAGE | URL | ATTR | HEX | '-' | DIMENSION | NUMBER | QUOTED_STRING | IDENT | ',') S* )+;
+ functional_pseudo = (IDENT - ('attr'|'url')) '(' space* expression? ')';
+ value = ( URL | ATTR | PLUS | HEX | PERCENTAGE | '-' | DIMENSION | NUMBER | QUOTED_STRING | IDENT | functional_pseudo);
+ values = value (space value | '/' value )* ( space* ',' space* value (space value | '/' value )* )* important?;
+
+ # Declaration definition
+ declaration = (property space? ':' (property ':')* space? values);
+
+ # Selectors
+ class = '.' IDENT;
+ element_name = IDENT;
+ namespace_prefix = ( IDENT | '*' )? '|';
+ type_selector = namespace_prefix? element_name;
+ universal = namespace_prefix? '*';
+ attrib = '[' space* namespace_prefix? IDENT space* ( ( PREFIXMATCH | SUFFIXMATCH | SUBSTRINGMATCH | '=' | INCLUDES | DASHMATCH ) space* ( IDENT | QUOTED_STRING ) space* )? ']';
+ pseudo = ':' ':'? ( IDENT | functional_pseudo );
+ atrule = '@' IDENT;
+ mediaquery_selector = '(' declaration ')';
+ negation_arg = type_selector
+ | universal
+ | HASH
+ | class
+ | attrib
+ | pseudo;
+ negation = 'NOT'|'not' space* negation_arg space* ')';
+ # Haha, so simple...
+ # there should be also mediaquery_selector but it makes grammar too large, so rip it off
+ simple_selector_sequence = ( type_selector | universal ) ( HASH | class | attrib | pseudo | negation | atrule )*
+ | ( HASH | class | attrib | pseudo | negation | atrule )+;
+ combinator = space* PLUS space*
+ | space* GREATER space*
+ | space* TILDE space*
+ | space+;
+ # Combine simple stuff and obtain just... an ordinary selector, bingo
+ selector = simple_selector_sequence ( combinator simple_selector_sequence )*;
+ # Multiple beasts
+ selectors_group = selector ( COMMENT? ',' space* selector )*;
+
+ # Rules
+ # This is mostly used stuff
+ rule = selectors_group space? "{" space*
+ (COMMENT? space* declaration ( space? ";" space? declaration?)* ";"? space?)* COMMENT* space* '}';
+ query_declaration = rule;
+
+ # Areas used in css
+ arearule = '@'('bottom-left'|'bottom-right'|'top-left'|'top-right');
+ areaquery = arearule space? '{' space* (COMMENT? space* declaration ( S? ';' S? declaration?)* ';'? space?)* COMMENT* space* '}';
+ # Printed media stuff, useless but we have to parse it :(
+ printcssrule = '@media print';
+ pagearea = ':'('left'|'right');
+ pagerule = '@page' space? pagearea?;
+ pagequery = pagerule space? '{' space* (areaquery| (COMMENT? space* declaration ( space? ';' space? declaration?)* ';'? S?)*) COMMENT* space* '}';
+ printcssquery = printcssrule S? '{' ( S? COMMENT* S? (pagequery| COMMENT|query_declaration) S*)* S? '}';
+ # Something that defines media
+ conditions = ('and'|'screen'|'or'|'only'|'not'|'amzn-mobi'|'amzn-kf8'|'amzn-mobi7'|',');
+ mediarule = '@media' space conditions ( space? conditions| space? mediaquery_selector )*;
+ mediaquery = mediarule space? '{' ( space? COMMENT* query_declaration)* S? '}';
+
+ simple_atrule = ("@charset"|"@namespace") space+ QUOTED_STRING space* ";";
+
+ import_rule = "@import" space+ ( QUOTED_STRING | URL ) space* ";";
+
+ # Final css definition
+ css_style = space* ( ( rule | simple_atrule | import_rule | mediaquery | printcssquery | COMMENT) space* )*;
+
+}%% \ No newline at end of file
diff --git a/src/libserver/css/css_tokeniser.cxx b/src/libserver/css/css_tokeniser.cxx
new file mode 100644
index 0000000..6d3f41e
--- /dev/null
+++ b/src/libserver/css/css_tokeniser.cxx
@@ -0,0 +1,836 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "css_tokeniser.hxx"
+#include "css_util.hxx"
+#include "css.hxx"
+#include "frozen/unordered_map.h"
+#include "frozen/string.h"
+#include <string>
+#include <cmath>
+
+namespace rspamd::css {
+
+/* Helpers to create tokens */
+
+/*
+ * This helper is intended to create tokens either with a tag and value
+ * or with just a tag.
+ */
+template<css_parser_token::token_type T, class Arg>
+auto make_token(const Arg &arg) -> css_parser_token;
+
+template<>
+auto make_token<css_parser_token::token_type::string_token, std::string_view>(const std::string_view &s)
+ -> css_parser_token
+{
+ return css_parser_token{css_parser_token::token_type::string_token, s};
+}
+
+template<>
+auto make_token<css_parser_token::token_type::ident_token, std::string_view>(const std::string_view &s)
+ -> css_parser_token
+{
+ return css_parser_token{css_parser_token::token_type::ident_token, s};
+}
+
+template<>
+auto make_token<css_parser_token::token_type::function_token, std::string_view>(const std::string_view &s)
+ -> css_parser_token
+{
+ return css_parser_token{css_parser_token::token_type::function_token, s};
+}
+
+template<>
+auto make_token<css_parser_token::token_type::url_token, std::string_view>(const std::string_view &s)
+ -> css_parser_token
+{
+ return css_parser_token{css_parser_token::token_type::url_token, s};
+}
+
+template<>
+auto make_token<css_parser_token::token_type::whitespace_token, std::string_view>(const std::string_view &s)
+ -> css_parser_token
+{
+ return css_parser_token{css_parser_token::token_type::whitespace_token, s};
+}
+
+template<>
+auto make_token<css_parser_token::token_type::delim_token, char>(const char &c)
+ -> css_parser_token
+{
+ return css_parser_token{css_parser_token::token_type::delim_token, c};
+}
+
+template<>
+auto make_token<css_parser_token::token_type::number_token, float>(const float &d)
+ -> css_parser_token
+{
+ return css_parser_token{css_parser_token::token_type::number_token, d};
+}
+
+/*
+ * Generic tokens with no value (non-terminals)
+ */
+template<css_parser_token::token_type T>
+auto make_token(void) -> css_parser_token
+{
+ return css_parser_token{T, css_parser_token_placeholder()};
+}
+
+static constexpr inline auto is_plain_ident_start(char c) -> bool
+{
+ if ((c & 0x80) || g_ascii_isalpha(c) || c == '_') {
+ return true;
+ }
+
+ return false;
+};
+
+static constexpr inline auto is_plain_ident(char c) -> bool
+{
+ if (is_plain_ident_start(c) || c == '-' || g_ascii_isdigit(c)) {
+ return true;
+ }
+
+ return false;
+};
+
+struct css_dimension_data {
+ css_parser_token::dim_type dtype;
+ double mult;
+};
+
+/*
+ * Maps from css dimensions to the multipliers that look reasonable in email
+ */
+constexpr const auto max_dims = static_cast<int>(css_parser_token::dim_type::dim_max);
+constexpr frozen::unordered_map<frozen::string, css_dimension_data, max_dims> dimensions_map{
+ {"px", {css_parser_token::dim_type::dim_px, 1.0}},
+ /* EM/REM are 16 px, so multiply and round */
+ {"em", {css_parser_token::dim_type::dim_em, 16.0}},
+ {"rem", {css_parser_token::dim_type::dim_rem, 16.0}},
+ /*
+ * Represents the x-height of the element's font.
+ * On fonts with the "x" letter, this is generally the height
+ * of lowercase letters in the font; 1ex = 0.5em in many fonts.
+ */
+ {"ex", {css_parser_token::dim_type::dim_ex, 8.0}},
+ {"wv", {css_parser_token::dim_type::dim_wv, 8.0}},
+ {"wh", {css_parser_token::dim_type::dim_wh, 6.0}},
+ {"vmax", {css_parser_token::dim_type::dim_vmax, 8.0}},
+ {"vmin", {css_parser_token::dim_type::dim_vmin, 6.0}},
+ /* One point. 1pt = 1/72nd of 1in */
+ {"pt", {css_parser_token::dim_type::dim_pt, 96.0 / 72.0}},
+ /* 96px/2.54 */
+ {"cm", {css_parser_token::dim_type::dim_cm, 96.0 / 2.54}},
+ {"mm", {css_parser_token::dim_type::dim_mm, 9.60 / 2.54}},
+ {"in", {css_parser_token::dim_type::dim_in, 96.0}},
+ /* 1pc = 12pt = 1/6th of 1in. */
+ {"pc", {css_parser_token::dim_type::dim_pc, 96.0 / 6.0}}};
+
+auto css_parser_token::adjust_dim(const css_parser_token &dim_token) -> bool
+{
+ if (!std::holds_alternative<float>(value) ||
+ !std::holds_alternative<std::string_view>(dim_token.value)) {
+ /* Invalid tokens */
+ return false;
+ }
+
+ auto num = std::get<float>(value);
+ auto sv = std::get<std::string_view>(dim_token.value);
+
+ auto dim_found = find_map(dimensions_map, sv);
+
+ if (dim_found) {
+ auto dim_elt = dim_found.value().get();
+ dimension_type = dim_elt.dtype;
+ flags |= css_parser_token::number_dimension;
+ num *= dim_elt.mult;
+ }
+ else {
+ flags |= css_parser_token::flag_bad_dimension;
+
+ return false;
+ }
+
+ value = num;
+
+ return true;
+}
+
+
+/*
+ * Consume functions: return a token and advance lexer offset
+ */
+auto css_tokeniser::consume_ident(bool allow_number) -> struct css_parser_token {
+ auto i = offset;
+ auto need_escape = false;
+ auto allow_middle_minus = false;
+
+ auto maybe_escape_sv = [&](auto cur_pos, auto tok_type) -> auto {
+ if (need_escape) {
+ auto escaped = rspamd::css::unescape_css(pool, {&input[offset],
+ cur_pos - offset});
+ offset = cur_pos;
+
+ return css_parser_token{tok_type, escaped};
+ }
+
+ auto result = std::string_view{&input[offset], cur_pos - offset};
+ offset = cur_pos;
+
+ return css_parser_token{tok_type, result};
+ };
+
+ /* Ident token can start from `-` or `--` */
+ if (input[i] == '-') {
+ i++;
+
+ if (i < input.size() && input[i] == '-') {
+ i++;
+ allow_middle_minus = true;
+ }
+ }
+
+ while (i < input.size()) {
+ auto c = input[i];
+
+ auto is_plain_c = (allow_number || allow_middle_minus) ? is_plain_ident(c) : is_plain_ident_start(c);
+ if (!is_plain_c) {
+ if (c == '\\' && i + 1 < input.size()) {
+ /* Escape token */
+ need_escape = true;
+ auto nhex = 0;
+
+ /* Need to find an escape end */
+ do {
+ c = input[++i];
+ if (g_ascii_isxdigit(c)) {
+ nhex++;
+
+ if (nhex > 6) {
+ /* End of the escape */
+ break;
+ }
+ }
+ else if (nhex > 0 && c == ' ') {
+ /* \[hex]{1,6} */
+ i++; /* Skip one space */
+ break;
+ }
+ else {
+ /* Single \ + char */
+ break;
+ }
+ } while (i < input.size());
+ }
+ else if (c == '(') {
+ /* Function or url token */
+ auto j = i + 1;
+
+ while (j < input.size() && g_ascii_isspace(input[j])) {
+ j++;
+ }
+
+ if (input.size() - offset > 3 && input.substr(offset, 3) == "url") {
+ if (j < input.size() && (input[j] == '"' || input[j] == '\'')) {
+ /* Function token */
+ auto ret = maybe_escape_sv(i,
+ css_parser_token::token_type::function_token);
+ return ret;
+ }
+ else {
+ /* Consume URL token */
+ while (j < input.size() && input[j] != ')') {
+ j++;
+ }
+
+ if (j < input.size() && input[j] == ')') {
+ /* Valid url token */
+ auto ret = maybe_escape_sv(j + 1,
+ css_parser_token::token_type::url_token);
+ return ret;
+ }
+ else {
+ /* Incomplete url token */
+ auto ret = maybe_escape_sv(j,
+ css_parser_token::token_type::url_token);
+
+ ret.flags |= css_parser_token::flag_bad_string;
+ return ret;
+ }
+ }
+ }
+ else {
+ auto ret = maybe_escape_sv(i,
+ css_parser_token::token_type::function_token);
+ return ret;
+ }
+ }
+ else if (c == '-' && allow_middle_minus) {
+ i++;
+ continue;
+ }
+ else {
+ break; /* Not an ident token */
+ }
+ } /* !plain ident */
+ else {
+ allow_middle_minus = true;
+ }
+
+ i++;
+ }
+
+ return maybe_escape_sv(i, css_parser_token::token_type::ident_token);
+}
+
+auto
+css_tokeniser::consume_number() -> struct css_parser_token {
+ auto i = offset;
+ auto seen_dot = false, seen_exp = false;
+
+ if (input[i] == '-' || input[i] == '+') {
+ i++;
+ }
+ if (input[i] == '.' && i < input.size()) {
+ seen_dot = true;
+ i++;
+ }
+
+ while (i < input.size()) {
+ auto c = input[i];
+
+ if (!g_ascii_isdigit(c)) {
+ if (c == '.') {
+ if (!seen_dot) {
+ seen_dot = true;
+ }
+ else {
+ break;
+ }
+ }
+ else if (c == 'e' || c == 'E') {
+ if (!seen_exp) {
+ seen_exp = true;
+ seen_dot = true; /* dots are not allowed after e */
+
+ if (i + 1 < input.size()) {
+ auto next_c = input[i + 1];
+ if (next_c == '+' || next_c == '-') {
+ i++;
+ }
+ else if (!g_ascii_isdigit(next_c)) {
+ /* Not an exponent */
+ break;
+ }
+ }
+ else {
+ /* Not an exponent */
+ break;
+ }
+ }
+ else {
+ break;
+ }
+ }
+ else {
+ break;
+ }
+ }
+
+ i++;
+ }
+
+ if (i > offset) {
+ /* I wish it was supported properly */
+ //auto conv_res = std::from_chars(&input[offset], &input[i], num);
+ char numbuf[128], *endptr = nullptr;
+ rspamd_strlcpy(numbuf, &input[offset], MIN(i - offset + 1, sizeof(numbuf)));
+ auto num = g_ascii_strtod(numbuf, &endptr);
+ offset = i;
+
+ if (fabs(num) >= G_MAXFLOAT || std::isnan(num)) {
+ msg_debug_css("invalid number: %s", numbuf);
+ return make_token<css_parser_token::token_type::delim_token>(input[i - 1]);
+ }
+ else {
+
+ auto ret = make_token<css_parser_token::token_type::number_token>(static_cast<float>(num));
+
+ if (i < input.size()) {
+ if (input[i] == '%') {
+ ret.flags |= css_parser_token::number_percent;
+ i++;
+
+ offset = i;
+ }
+ else if (is_plain_ident_start(input[i])) {
+ auto dim_token = consume_ident();
+
+ if (dim_token.type == css_parser_token::token_type::ident_token) {
+ if (!ret.adjust_dim(dim_token)) {
+ auto sv = std::get<std::string_view>(dim_token.value);
+ msg_debug_css("cannot apply dimension from the token %*s; number value = %.1f",
+ (int) sv.size(), sv.begin(), num);
+ /* Unconsume ident */
+ offset = i;
+ }
+ }
+ else {
+ /* We have no option but to uncosume ident token in this case */
+ msg_debug_css("got invalid ident like token after number, unconsume it");
+ }
+ }
+ else {
+ /* Plain number, nothing to do */
+ }
+ }
+
+ return ret;
+ }
+ }
+ else {
+ msg_err_css("internal error: invalid number, empty token");
+ i++;
+ }
+
+ offset = i;
+ /* Should not happen */
+ return make_token<css_parser_token::token_type::delim_token>(input[i - 1]);
+}
+
+/*
+ * Main routine to produce lexer tokens
+ */
+auto
+css_tokeniser::next_token(void) -> struct css_parser_token {
+ /* Check pushback queue */
+ if (!backlog.empty()) {
+ auto tok = backlog.front();
+ backlog.pop_front();
+
+ return tok;
+ }
+ /* Helpers */
+
+ /*
+ * This lambda eats comment handling nested comments;
+ * offset is set to the next character after a comment (or eof)
+ * Nothing is returned
+ */
+ auto consume_comment = [this]() {
+ auto i = offset;
+ auto nested = 0;
+
+ if (input.empty()) {
+ /* Nothing to consume */
+ return;
+ }
+
+ /* We handle nested comments just because they can exist... */
+ while (i < input.size() - 1) {
+ auto c = input[i];
+ if (c == '*' && input[i + 1] == '/') {
+ if (nested == 0) {
+ offset = i + 2;
+ return;
+ }
+ else {
+ nested--;
+ i += 2;
+ continue;
+ }
+ }
+ else if (c == '/' && input[i + 1] == '*') {
+ nested++;
+ i += 2;
+ continue;
+ }
+
+ i++;
+ }
+
+ offset = i;
+ };
+
+ /*
+ * Consume quoted string, returns a string_view over a string, offset
+ * is set one character after the string. Css unescaping is done automatically
+ * Accepts a quote char to find end of string
+ */
+ auto consume_string = [this](auto quote_char) -> auto {
+ auto i = offset;
+ bool need_unescape = false;
+
+ while (i < input.size()) {
+ auto c = input[i];
+
+ if (c == '\\') {
+ if (i + 1 < input.size()) {
+ need_unescape = true;
+ }
+ else {
+ /* \ at the end -> ignore */
+ }
+ }
+ else if (c == quote_char) {
+ /* End of string */
+ std::string_view res{&input[offset], i - offset};
+
+ if (need_unescape) {
+ res = rspamd::css::unescape_css(pool, res);
+ }
+
+ offset = i + 1;
+
+ return res;
+ }
+ else if (c == '\n') {
+ /* Should be a error, but we ignore it for now */
+ }
+
+ i++;
+ }
+
+ /* EOF with no quote character, consider it fine */
+ std::string_view res{&input[offset], i - offset};
+
+ if (need_unescape) {
+ res = rspamd::css::unescape_css(pool, res);
+ }
+
+ offset = i;
+
+ return res;
+ };
+
+ /* Main tokenisation loop */
+ for (auto i = offset; i < input.size(); ++i) {
+ auto c = input[i];
+
+ switch (c) {
+ case '/':
+ if (i + 1 < input.size() && input[i + 1] == '*') {
+ offset = i + 2;
+ consume_comment(); /* Consume comment and go forward */
+ return next_token(); /* Tail call */
+ }
+ else {
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+ }
+ break;
+ case ' ':
+ case '\t':
+ case '\n':
+ case '\r':
+ case '\f': {
+ /* Consume as much space as we can */
+ while (i < input.size() && g_ascii_isspace(input[i])) {
+ i++;
+ }
+
+ auto ret = make_token<css_parser_token::token_type::whitespace_token>(
+ std::string_view(&input[offset], i - offset));
+ offset = i;
+ return ret;
+ }
+ case '"':
+ case '\'':
+ offset = i + 1;
+ if (offset < input.size()) {
+ return make_token<css_parser_token::token_type::string_token>(consume_string(c));
+ }
+ else {
+ /* Unpaired quote at the end of the rule */
+ return make_token<css_parser_token::token_type::delim_token>(c);
+ }
+ case '(':
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::obrace_token>();
+ case ')':
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::ebrace_token>();
+ case '[':
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::osqbrace_token>();
+ case ']':
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::esqbrace_token>();
+ case '{':
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::ocurlbrace_token>();
+ case '}':
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::ecurlbrace_token>();
+ case ',':
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::comma_token>();
+ case ';':
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::semicolon_token>();
+ case ':':
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::colon_token>();
+ case '<':
+ /* Maybe an xml like comment */
+ if (i + 3 < input.size() && input[i + 1] == '!' && input[i + 2] == '-' && input[i + 3] == '-') {
+ offset += 3;
+
+ return make_token<css_parser_token::token_type::cdo_token>();
+ }
+ else {
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+ }
+ break;
+ case '-':
+ if (i + 1 < input.size()) {
+ auto next_c = input[i + 1];
+
+ if (g_ascii_isdigit(next_c)) {
+ /* negative number */
+ return consume_number();
+ }
+ else if (next_c == '-') {
+ if (i + 2 < input.size() && input[i + 2] == '>') {
+ /* XML like comment */
+ offset += 3;
+
+ return make_token<css_parser_token::token_type::cdc_token>();
+ }
+ }
+ }
+ /* No other options, a delimiter - */
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+
+ break;
+ case '+':
+ case '.':
+ /* Maybe number */
+ if (i + 1 < input.size()) {
+ auto next_c = input[i + 1];
+
+ if (g_ascii_isdigit(next_c)) {
+ /* Numeric token */
+ return consume_number();
+ }
+ else {
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+ }
+ }
+ /* No other options, a delimiter - */
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+
+ break;
+ case '\\':
+ if (i + 1 < input.size()) {
+ if (input[i + 1] == '\n' || input[i + 1] == '\r') {
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+ }
+ else {
+ /* Valid escape, assume ident */
+ return consume_ident();
+ }
+ }
+ else {
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+ }
+ break;
+ case '@':
+ if (i + 3 < input.size()) {
+ if (is_plain_ident_start(input[i + 1]) &&
+ is_plain_ident(input[i + 2]) && is_plain_ident(input[i + 3])) {
+ offset = i + 1;
+ auto ident_token = consume_ident();
+
+ if (ident_token.type == css_parser_token::token_type::ident_token) {
+ /* Update type */
+ ident_token.type = css_parser_token::token_type::at_keyword_token;
+ }
+
+ return ident_token;
+ }
+ else {
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+ }
+ }
+ else {
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+ }
+ break;
+ case '#':
+ /* TODO: make it more conformant */
+ if (i + 2 < input.size()) {
+ auto next_c = input[i + 1], next_next_c = input[i + 2];
+ if ((is_plain_ident(next_c) || next_c == '-') &&
+ (is_plain_ident(next_next_c) || next_next_c == '-')) {
+ offset = i + 1;
+ /* We consume indent, but we allow numbers there */
+ auto ident_token = consume_ident(true);
+
+ if (ident_token.type == css_parser_token::token_type::ident_token) {
+ /* Update type */
+ ident_token.type = css_parser_token::token_type::hash_token;
+ }
+
+ return ident_token;
+ }
+ else {
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+ }
+ }
+ else {
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+ }
+ break;
+ default:
+ /* Generic parsing code */
+
+ if (g_ascii_isdigit(c)) {
+ return consume_number();
+ }
+ else if (is_plain_ident_start(c)) {
+ return consume_ident();
+ }
+ else {
+ offset = i + 1;
+ return make_token<css_parser_token::token_type::delim_token>(c);
+ }
+ break;
+ }
+ }
+
+ return make_token<css_parser_token::token_type::eof_token>();
+}
+
+constexpr auto
+css_parser_token::get_token_type() -> const char *
+{
+ const char *ret = "unknown";
+
+ switch (type) {
+ case token_type::whitespace_token:
+ ret = "whitespace";
+ break;
+ case token_type::ident_token:
+ ret = "ident";
+ break;
+ case token_type::function_token:
+ ret = "function";
+ break;
+ case token_type::at_keyword_token:
+ ret = "atkeyword";
+ break;
+ case token_type::hash_token:
+ ret = "hash";
+ break;
+ case token_type::string_token:
+ ret = "string";
+ break;
+ case token_type::number_token:
+ ret = "number";
+ break;
+ case token_type::url_token:
+ ret = "url";
+ break;
+ case token_type::cdo_token: /* xml open comment */
+ ret = "cdo";
+ break;
+ case token_type::cdc_token: /* xml close comment */
+ ret = "cdc";
+ break;
+ case token_type::delim_token:
+ ret = "delim";
+ break;
+ case token_type::obrace_token: /* ( */
+ ret = "obrace";
+ break;
+ case token_type::ebrace_token: /* ) */
+ ret = "ebrace";
+ break;
+ case token_type::osqbrace_token: /* [ */
+ ret = "osqbrace";
+ break;
+ case token_type::esqbrace_token: /* ] */
+ ret = "esqbrace";
+ break;
+ case token_type::ocurlbrace_token: /* { */
+ ret = "ocurlbrace";
+ break;
+ case token_type::ecurlbrace_token: /* } */
+ ret = "ecurlbrace";
+ break;
+ case token_type::comma_token:
+ ret = "comma";
+ break;
+ case token_type::colon_token:
+ ret = "colon";
+ break;
+ case token_type::semicolon_token:
+ ret = "semicolon";
+ break;
+ case token_type::eof_token:
+ ret = "eof";
+ break;
+ }
+
+ return ret;
+}
+
+
+auto css_parser_token::debug_token_str() -> std::string
+{
+ const auto *token_type_str = get_token_type();
+ std::string ret = token_type_str;
+
+ std::visit([&](auto arg) -> auto {
+ using T = std::decay_t<decltype(arg)>;
+
+ if constexpr (std::is_same_v<T, std::string_view> || std::is_same_v<T, char>) {
+ ret += "; value=";
+ ret += arg;
+ }
+ else if constexpr (std::is_same_v<T, double>) {
+ ret += "; value=";
+ ret += std::to_string(arg);
+ }
+ },
+ value);
+
+ if ((flags & (~number_dimension)) != default_flags) {
+ ret += "; flags=" + std::to_string(flags);
+ }
+
+ if (flags & number_dimension) {
+ ret += "; dim=" + std::to_string(static_cast<int>(dimension_type));
+ }
+
+ return ret; /* Copy elision */
+}
+
+}// namespace rspamd::css \ No newline at end of file
diff --git a/src/libserver/css/css_tokeniser.hxx b/src/libserver/css/css_tokeniser.hxx
new file mode 100644
index 0000000..aa6a1a7
--- /dev/null
+++ b/src/libserver/css/css_tokeniser.hxx
@@ -0,0 +1,215 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifndef RSPAMD_CSS_TOKENISER_HXX
+#define RSPAMD_CSS_TOKENISER_HXX
+
+#include <string_view>
+#include <utility>
+#include <variant>
+#include <list>
+#include <functional>
+#include <cstdint>
+#include "mem_pool.h"
+
+namespace rspamd::css {
+
+struct css_parser_token_placeholder {}; /* For empty tokens */
+
+struct css_parser_token {
+
+ enum class token_type : std::uint8_t {
+ whitespace_token,
+ ident_token,
+ function_token,
+ at_keyword_token,
+ hash_token,
+ string_token,
+ number_token,
+ url_token,
+ cdo_token, /* xml open comment */
+ cdc_token, /* xml close comment */
+ delim_token,
+ obrace_token, /* ( */
+ ebrace_token, /* ) */
+ osqbrace_token, /* [ */
+ esqbrace_token, /* ] */
+ ocurlbrace_token, /* { */
+ ecurlbrace_token, /* } */
+ comma_token,
+ colon_token,
+ semicolon_token,
+ eof_token,
+ };
+
+ enum class dim_type : std::uint8_t {
+ dim_px = 0,
+ dim_em,
+ dim_rem,
+ dim_ex,
+ dim_wv,
+ dim_wh,
+ dim_vmax,
+ dim_vmin,
+ dim_pt,
+ dim_cm,
+ dim_mm,
+ dim_in,
+ dim_pc,
+ dim_max,
+ };
+
+ static const std::uint8_t default_flags = 0;
+ static const std::uint8_t flag_bad_string = (1u << 0u);
+ static const std::uint8_t number_dimension = (1u << 1u);
+ static const std::uint8_t number_percent = (1u << 2u);
+ static const std::uint8_t flag_bad_dimension = (1u << 3u);
+
+ using value_type = std::variant<std::string_view, /* For strings and string like tokens */
+ char, /* For delimiters (might need to move to unicode point) */
+ float, /* For numeric stuff */
+ css_parser_token_placeholder /* For general no token stuff */
+ >;
+
+ /* Typed storage */
+ value_type value;
+
+ int lineno;
+
+ token_type type;
+ std::uint8_t flags = default_flags;
+ dim_type dimension_type;
+
+ css_parser_token() = delete;
+ explicit css_parser_token(token_type type, const value_type &value)
+ : value(value), type(type)
+ {
+ }
+ css_parser_token(css_parser_token &&other) = default;
+ css_parser_token(const css_parser_token &token) = default;
+ auto operator=(css_parser_token &&other) -> css_parser_token & = default;
+ auto adjust_dim(const css_parser_token &dim_token) -> bool;
+
+ auto get_string_or_default(const std::string_view &def) const -> std::string_view
+ {
+ if (std::holds_alternative<std::string_view>(value)) {
+ return std::get<std::string_view>(value);
+ }
+ else if (std::holds_alternative<char>(value)) {
+ return std::string_view(&std::get<char>(value), 1);
+ }
+
+ return def;
+ }
+
+ auto get_delim() const -> char
+ {
+ if (std::holds_alternative<char>(value)) {
+ return std::get<char>(value);
+ }
+
+ return (char) -1;
+ }
+
+ auto get_number_or_default(float def) const -> float
+ {
+ if (std::holds_alternative<float>(value)) {
+ auto dbl = std::get<float>(value);
+
+ if (flags & css_parser_token::number_percent) {
+ dbl /= 100.0;
+ }
+
+ return dbl;
+ }
+
+ return def;
+ }
+
+ auto get_normal_number_or_default(float def) const -> float
+ {
+ if (std::holds_alternative<float>(value)) {
+ auto dbl = std::get<float>(value);
+
+ if (flags & css_parser_token::number_percent) {
+ dbl /= 100.0;
+ }
+
+ if (dbl < 0) {
+ return 0.0;
+ }
+ else if (dbl > 1.0) {
+ return 1.0;
+ }
+
+ return dbl;
+ }
+
+ return def;
+ }
+
+ /* Debugging routines */
+ constexpr auto get_token_type() -> const char *;
+ /* This function might be slow */
+ auto debug_token_str() -> std::string;
+};
+
+static auto css_parser_eof_token(void) -> const css_parser_token &
+{
+ static css_parser_token eof_tok{
+ css_parser_token::token_type::eof_token,
+ css_parser_token_placeholder()};
+
+ return eof_tok;
+}
+
+/* Ensure that parser tokens are simple enough */
+/*
+ * compiler must implement P0602 "variant and optional should propagate copy/move triviality"
+ * This is broken on gcc < 8!
+ */
+static_assert(std::is_trivially_copyable_v<css_parser_token>);
+
+class css_tokeniser {
+public:
+ css_tokeniser() = delete;
+ css_tokeniser(rspamd_mempool_t *pool, const std::string_view &sv)
+ : input(sv), offset(0), pool(pool)
+ {
+ }
+
+ auto next_token(void) -> struct css_parser_token;
+ auto pushback_token(const struct css_parser_token &t) const -> void
+ {
+ backlog.push_back(t);
+ }
+
+private:
+ std::string_view input;
+ std::size_t offset;
+ rspamd_mempool_t *pool;
+ mutable std::list<css_parser_token> backlog;
+
+ auto consume_number() -> struct css_parser_token;
+ auto consume_ident(bool allow_number = false) -> struct css_parser_token;
+};
+
+}// namespace rspamd::css
+
+
+#endif//RSPAMD_CSS_TOKENISER_HXX
diff --git a/src/libserver/css/css_util.cxx b/src/libserver/css/css_util.cxx
new file mode 100644
index 0000000..07f8722
--- /dev/null
+++ b/src/libserver/css/css_util.cxx
@@ -0,0 +1,157 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "css_util.hxx"
+#include "css.hxx"
+#include <unicode/utf8.h>
+
+namespace rspamd::css {
+
+std::string_view unescape_css(rspamd_mempool_t *pool,
+ const std::string_view &sv)
+{
+ auto *nspace = reinterpret_cast<char *>(rspamd_mempool_alloc(pool, sv.length()));
+ auto *d = nspace;
+ auto nleft = sv.length();
+
+ enum {
+ normal = 0,
+ quoted,
+ escape,
+ skip_spaces,
+ } state = normal;
+
+ char quote_char, prev_c = 0;
+ auto escape_offset = 0, i = 0;
+
+#define MAYBE_CONSUME_CHAR(c) \
+ do { \
+ if ((c) == '"' || (c) == '\'') { \
+ state = quoted; \
+ quote_char = (c); \
+ nleft--; \
+ *d++ = (c); \
+ } \
+ else if ((c) == '\\') { \
+ escape_offset = i; \
+ state = escape; \
+ } \
+ else { \
+ state = normal; \
+ nleft--; \
+ *d++ = g_ascii_tolower(c); \
+ } \
+ } while (0)
+
+ for (const auto c: sv) {
+ if (nleft == 0) {
+ msg_err_css("cannot unescape css: truncated buffer of size %d",
+ (int) sv.length());
+ break;
+ }
+ switch (state) {
+ case normal:
+ MAYBE_CONSUME_CHAR(c);
+ break;
+ case quoted:
+ if (c == quote_char) {
+ if (prev_c != '\\') {
+ state = normal;
+ }
+ }
+ prev_c = c;
+ nleft--;
+ *d++ = c;
+ break;
+ case escape:
+ if (!g_ascii_isxdigit(c)) {
+ if (i > escape_offset + 1) {
+ /* Try to decode an escape */
+ const auto *escape_start = &sv[escape_offset + 1];
+ unsigned long val;
+
+ if (!rspamd_xstrtoul(escape_start, i - escape_offset - 1, &val)) {
+ msg_debug_css("invalid broken escape found at pos %d",
+ escape_offset);
+ }
+ else {
+ if (val < 0x80) {
+ /* Trivial case: ascii character */
+ *d++ = (unsigned char) g_ascii_tolower(val);
+ nleft--;
+ }
+ else {
+ UChar32 uc = val;
+ auto off = 0;
+ UTF8_APPEND_CHAR_SAFE((uint8_t *) d, off,
+ sv.length(), u_tolower(uc));
+ d += off;
+ nleft -= off;
+ }
+ }
+ }
+ else {
+ /* Empty escape, ignore it */
+ msg_debug_css("invalid empty escape found at pos %d",
+ escape_offset);
+ }
+
+ if (nleft <= 0) {
+ msg_err_css("cannot unescape css: truncated buffer of size %d",
+ (int) sv.length());
+ }
+ else {
+ /* Escape is done, advance forward */
+ if (g_ascii_isspace(c)) {
+ state = skip_spaces;
+ }
+ else {
+ MAYBE_CONSUME_CHAR(c);
+ }
+ }
+ }
+ break;
+ case skip_spaces:
+ if (!g_ascii_isspace(c)) {
+ MAYBE_CONSUME_CHAR(c);
+ }
+ /* Ignore spaces */
+ break;
+ }
+
+ i++;
+ }
+
+ return std::string_view{nspace, sv.size() - nleft};
+}
+
+}// namespace rspamd::css
+
+/* C API */
+const gchar *rspamd_css_unescape(rspamd_mempool_t *pool,
+ const guchar *begin,
+ gsize len,
+ gsize *outlen)
+{
+ auto sv = rspamd::css::unescape_css(pool, {(const char *) begin, len});
+ const auto *v = sv.begin();
+
+ if (outlen) {
+ *outlen = sv.size();
+ }
+
+ return v;
+} \ No newline at end of file
diff --git a/src/libserver/css/css_util.hxx b/src/libserver/css/css_util.hxx
new file mode 100644
index 0000000..4837a46
--- /dev/null
+++ b/src/libserver/css/css_util.hxx
@@ -0,0 +1,37 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifndef RSPAMD_CSS_UTIL_HXX
+#define RSPAMD_CSS_UTIL_HXX
+
+#include <string_view>
+#include "mem_pool.h"
+
+namespace rspamd::css {
+
+/*
+ * Unescape css escapes
+ * \20AC : must be followed by a space if the next character is one of a-f, A-F, 0-9
+ * \0020AC : must be 6 digits long, no space needed (but can be included)
+ */
+std::string_view unescape_css(rspamd_mempool_t *pool,
+ const std::string_view &sv);
+
+}// namespace rspamd::css
+
+#endif//RSPAMD_CSS_UTIL_HXX
diff --git a/src/libserver/css/css_value.cxx b/src/libserver/css/css_value.cxx
new file mode 100644
index 0000000..2546e01
--- /dev/null
+++ b/src/libserver/css/css_value.cxx
@@ -0,0 +1,449 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "css_value.hxx"
+#include "css_colors_list.hxx"
+#include "frozen/unordered_map.h"
+#include "frozen/string.h"
+#include "libutil/util.h"
+#include "contrib/ankerl/unordered_dense.h"
+#include "fmt/core.h"
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+/* Helper for unit test stringification */
+namespace doctest {
+template<>
+struct StringMaker<rspamd::css::css_color> {
+ static String convert(const rspamd::css::css_color &value)
+ {
+ return fmt::format("r={};g={};b={};alpha={}",
+ value.r, value.g, value.b, value.alpha)
+ .c_str();
+ }
+};
+
+}// namespace doctest
+
+namespace rspamd::css {
+
+auto css_value::maybe_color_from_string(const std::string_view &input)
+ -> std::optional<css_value>
+{
+
+ if (input.size() > 1 && input.front() == '#') {
+ return css_value::maybe_color_from_hex(input.substr(1));
+ }
+ else {
+ auto found_it = css_colors_map.find(input);
+
+ if (found_it != css_colors_map.end()) {
+ return css_value{found_it->second};
+ }
+ }
+
+ return std::nullopt;
+}
+
+constexpr static inline auto hexpair_decode(char c1, char c2) -> std::uint8_t
+{
+ std::uint8_t ret = 0;
+
+ if (c1 >= '0' && c1 <= '9') ret = c1 - '0';
+ else if (c1 >= 'A' && c1 <= 'F')
+ ret = c1 - 'A' + 10;
+ else if (c1 >= 'a' && c1 <= 'f')
+ ret = c1 - 'a' + 10;
+
+ ret *= 16;
+
+ if (c2 >= '0' && c2 <= '9') ret += c2 - '0';
+ else if (c2 >= 'A' && c2 <= 'F')
+ ret += c2 - 'A' + 10;
+ else if (c2 >= 'a' && c2 <= 'f')
+ ret += c2 - 'a' + 10;
+
+ return ret;
+}
+
+auto css_value::maybe_color_from_hex(const std::string_view &input)
+ -> std::optional<css_value>
+{
+ if (input.length() == 6) {
+ /* Plain RGB */
+ css_color col(hexpair_decode(input[0], input[1]),
+ hexpair_decode(input[2], input[3]),
+ hexpair_decode(input[4], input[5]));
+ return css_value(col);
+ }
+ else if (input.length() == 3) {
+ /* Rgb as 3 hex digests */
+ css_color col(hexpair_decode(input[0], input[0]),
+ hexpair_decode(input[1], input[1]),
+ hexpair_decode(input[2], input[2]));
+ return css_value(col);
+ }
+ else if (input.length() == 8) {
+ /* RGBA */
+ css_color col(hexpair_decode(input[0], input[1]),
+ hexpair_decode(input[2], input[3]),
+ hexpair_decode(input[4], input[5]),
+ hexpair_decode(input[6], input[7]));
+ return css_value(col);
+ }
+
+ return std::nullopt;
+}
+
+constexpr static inline auto rgb_color_component_convert(const css_parser_token &tok)
+ -> std::uint8_t
+{
+ std::uint8_t ret = 0;
+
+ if (tok.type == css_parser_token::token_type::number_token) {
+ auto dbl = std::get<float>(tok.value);
+
+ if (tok.flags & css_parser_token::number_percent) {
+ if (dbl > 100) {
+ dbl = 100;
+ }
+ else if (dbl < 0) {
+ dbl = 0;
+ }
+ ret = (std::uint8_t)(dbl / 100.0 * 255.0);
+ }
+ else {
+ if (dbl > 255) {
+ dbl = 255;
+ }
+ else if (dbl < 0) {
+ dbl = 0;
+ }
+
+ ret = (std::uint8_t)(dbl);
+ }
+ }
+
+ return ret;
+}
+
+constexpr static inline auto alpha_component_convert(const css_parser_token &tok)
+ -> std::uint8_t
+{
+ double ret = 1.0;
+
+ if (tok.type == css_parser_token::token_type::number_token) {
+ auto dbl = std::get<float>(tok.value);
+
+ if (tok.flags & css_parser_token::number_percent) {
+ if (dbl > 100) {
+ dbl = 100;
+ }
+ else if (dbl < 0) {
+ dbl = 0;
+ }
+ ret = (dbl / 100.0);
+ }
+ else {
+ if (dbl > 1.0) {
+ dbl = 1.0;
+ }
+ else if (dbl < 0) {
+ dbl = 0;
+ }
+
+ ret = dbl;
+ }
+ }
+
+ return (std::uint8_t)(ret * 255.0);
+}
+
+constexpr static inline auto h_component_convert(const css_parser_token &tok)
+ -> double
+{
+ double ret = 0.0;
+
+ if (tok.type == css_parser_token::token_type::number_token) {
+ auto dbl = std::get<float>(tok.value);
+
+ if (tok.flags & css_parser_token::number_percent) {
+ if (dbl > 100) {
+ dbl = 100;
+ }
+ else if (dbl < 0) {
+ dbl = 0;
+ }
+ ret = (dbl / 100.0);
+ }
+ else {
+ dbl = ((((int) dbl % 360) + 360) % 360); /* Deal with rotations */
+ ret = dbl / 360.0; /* Normalize to 0..1 */
+ }
+ }
+
+ return ret;
+}
+
+constexpr static inline auto sl_component_convert(const css_parser_token &tok)
+ -> double
+{
+ double ret = 0.0;
+
+ if (tok.type == css_parser_token::token_type::number_token) {
+ ret = tok.get_normal_number_or_default(ret);
+ }
+
+ return ret;
+}
+
+static inline auto hsl_to_rgb(double h, double s, double l)
+ -> css_color
+{
+ css_color ret;
+
+ constexpr auto hue2rgb = [](auto p, auto q, auto t) -> auto {
+ if (t < 0.0) {
+ t += 1.0;
+ }
+ if (t > 1.0) {
+ t -= 1.0;
+ }
+ if (t * 6. < 1.0) {
+ return p + (q - p) * 6.0 * t;
+ }
+ if (t * 2. < 1) {
+ return q;
+ }
+ if (t * 3. < 2.) {
+ return p + (q - p) * (2.0 / 3.0 - t) * 6.0;
+ }
+ return p;
+ };
+
+ if (s == 0) {
+ /* Achromatic */
+ ret.r = l;
+ ret.g = l;
+ ret.b = l;
+ }
+ else {
+ auto q = l <= 0.5 ? l * (1.0 + s) : l + s - l * s;
+ auto p = 2.0 * l - q;
+ ret.r = (std::uint8_t)(hue2rgb(p, q, h + 1.0 / 3.0) * 255);
+ ret.g = (std::uint8_t)(hue2rgb(p, q, h) * 255);
+ ret.b = (std::uint8_t)(hue2rgb(p, q, h - 1.0 / 3.0) * 255);
+ }
+
+ ret.alpha = 255;
+
+ return ret;
+}
+
+auto css_value::maybe_color_from_function(const css_consumed_block::css_function_block &func)
+ -> std::optional<css_value>
+{
+
+ if (func.as_string() == "rgb" && func.args.size() == 3) {
+ css_color col{rgb_color_component_convert(func.args[0]->get_token_or_empty()),
+ rgb_color_component_convert(func.args[1]->get_token_or_empty()),
+ rgb_color_component_convert(func.args[2]->get_token_or_empty())};
+
+ return css_value(col);
+ }
+ else if (func.as_string() == "rgba" && func.args.size() == 4) {
+ css_color col{rgb_color_component_convert(func.args[0]->get_token_or_empty()),
+ rgb_color_component_convert(func.args[1]->get_token_or_empty()),
+ rgb_color_component_convert(func.args[2]->get_token_or_empty()),
+ alpha_component_convert(func.args[3]->get_token_or_empty())};
+
+ return css_value(col);
+ }
+ else if (func.as_string() == "hsl" && func.args.size() == 3) {
+ auto h = h_component_convert(func.args[0]->get_token_or_empty());
+ auto s = sl_component_convert(func.args[1]->get_token_or_empty());
+ auto l = sl_component_convert(func.args[2]->get_token_or_empty());
+
+ auto col = hsl_to_rgb(h, s, l);
+
+ return css_value(col);
+ }
+ else if (func.as_string() == "hsla" && func.args.size() == 4) {
+ auto h = h_component_convert(func.args[0]->get_token_or_empty());
+ auto s = sl_component_convert(func.args[1]->get_token_or_empty());
+ auto l = sl_component_convert(func.args[2]->get_token_or_empty());
+
+ auto col = hsl_to_rgb(h, s, l);
+ col.alpha = alpha_component_convert(func.args[3]->get_token_or_empty());
+
+ return css_value(col);
+ }
+
+ return std::nullopt;
+}
+
+auto css_value::maybe_dimension_from_number(const css_parser_token &tok)
+ -> std::optional<css_value>
+{
+ if (std::holds_alternative<float>(tok.value)) {
+ auto dbl = std::get<float>(tok.value);
+ css_dimension dim;
+
+ dim.dim = dbl;
+
+ if (tok.flags & css_parser_token::number_percent) {
+ dim.is_percent = true;
+ }
+ else {
+ dim.is_percent = false;
+ }
+
+ return css_value{dim};
+ }
+
+ return std::nullopt;
+}
+
+constexpr const auto display_names_map = frozen::make_unordered_map<frozen::string, css_display_value>({
+ {"hidden", css_display_value::DISPLAY_HIDDEN},
+ {"none", css_display_value::DISPLAY_HIDDEN},
+ {"inline", css_display_value::DISPLAY_INLINE},
+ {"block", css_display_value::DISPLAY_BLOCK},
+ {"content", css_display_value::DISPLAY_INLINE},
+ {"flex", css_display_value::DISPLAY_BLOCK},
+ {"grid", css_display_value::DISPLAY_BLOCK},
+ {"inline-block", css_display_value::DISPLAY_INLINE},
+ {"inline-flex", css_display_value::DISPLAY_INLINE},
+ {"inline-grid", css_display_value::DISPLAY_INLINE},
+ {"inline-table", css_display_value::DISPLAY_INLINE},
+ {"list-item", css_display_value::DISPLAY_BLOCK},
+ {"run-in", css_display_value::DISPLAY_INLINE},
+ {"table", css_display_value::DISPLAY_BLOCK},
+ {"table-caption", css_display_value::DISPLAY_TABLE_ROW},
+ {"table-column-group", css_display_value::DISPLAY_TABLE_ROW},
+ {"table-header-group", css_display_value::DISPLAY_TABLE_ROW},
+ {"table-footer-group", css_display_value::DISPLAY_TABLE_ROW},
+ {"table-row-group", css_display_value::DISPLAY_TABLE_ROW},
+ {"table-cell", css_display_value::DISPLAY_TABLE_ROW},
+ {"table-column", css_display_value::DISPLAY_TABLE_ROW},
+ {"table-row", css_display_value::DISPLAY_TABLE_ROW},
+ {"initial", css_display_value::DISPLAY_INLINE},
+});
+
+auto css_value::maybe_display_from_string(const std::string_view &input)
+ -> std::optional<css_value>
+{
+ auto f = display_names_map.find(input);
+
+ if (f != display_names_map.end()) {
+ return css_value{f->second};
+ }
+
+ return std::nullopt;
+}
+
+
+auto css_value::debug_str() const -> std::string
+{
+ std::string ret;
+
+ std::visit([&](const auto &arg) {
+ using T = std::decay_t<decltype(arg)>;
+
+ if constexpr (std::is_same_v<T, css_color>) {
+ ret += fmt::format("color: r={};g={};b={};alpha={}",
+ arg.r, arg.g, arg.b, arg.alpha);
+ }
+ else if constexpr (std::is_same_v<T, double>) {
+ ret += "size: " + std::to_string(arg);
+ }
+ else if constexpr (std::is_same_v<T, css_dimension>) {
+ ret += "dimension: " + std::to_string(arg.dim);
+ if (arg.is_percent) {
+ ret += "%";
+ }
+ }
+ else if constexpr (std::is_same_v<T, css_display_value>) {
+ ret += "display: ";
+ switch (arg) {
+ case css_display_value::DISPLAY_HIDDEN:
+ ret += "hidden";
+ break;
+ case css_display_value::DISPLAY_BLOCK:
+ ret += "block";
+ break;
+ case css_display_value::DISPLAY_INLINE:
+ ret += "inline";
+ break;
+ case css_display_value::DISPLAY_TABLE_ROW:
+ ret += "table_row";
+ break;
+ }
+ }
+ else if constexpr (std::is_integral_v<T>) {
+ ret += "integral: " + std::to_string(static_cast<int>(arg));
+ }
+ else {
+ ret += "nyi";
+ }
+ },
+ value);
+
+ return ret;
+}
+
+TEST_SUITE("css"){
+ TEST_CASE("css hex colors"){
+ const std::pair<const char *, css_color> hex_tests[] = {
+ {"000", css_color(0, 0, 0)},
+ {"000000", css_color(0, 0, 0)},
+ {"f00", css_color(255, 0, 0)},
+ {"FEDCBA", css_color(254, 220, 186)},
+ {"234", css_color(34, 51, 68)},
+ };
+
+for (const auto &p: hex_tests) {
+ SUBCASE((std::string("parse hex color: ") + p.first).c_str())
+ {
+ auto col_parsed = css_value::maybe_color_from_hex(p.first);
+ //CHECK_UNARY(col_parsed);
+ //CHECK_UNARY(col_parsed.value().to_color());
+ auto final_col = col_parsed.value().to_color().value();
+ CHECK(final_col == p.second);
+ }
+}
+}// namespace rspamd::css
+TEST_CASE("css colors strings")
+{
+ auto passed = 0;
+ for (const auto &p: css_colors_map) {
+ /* Match some of the colors selected randomly */
+ if (rspamd_random_double_fast() > 0.9) {
+ auto col_parsed = css_value::maybe_color_from_string(p.first);
+ auto final_col = col_parsed.value().to_color().value();
+ CHECK_MESSAGE(final_col == p.second, p.first.data());
+ passed++;
+
+ if (passed > 20) {
+ break;
+ }
+ }
+ }
+}
+}
+;
+}
diff --git a/src/libserver/css/css_value.hxx b/src/libserver/css/css_value.hxx
new file mode 100644
index 0000000..1d57421
--- /dev/null
+++ b/src/libserver/css/css_value.hxx
@@ -0,0 +1,174 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifndef RSPAMD_CSS_VALUE_HXX
+#define RSPAMD_CSS_VALUE_HXX
+
+#include <string>
+#include <variant>
+#include <optional>
+#include <vector>
+#include <iosfwd>
+#include "parse_error.hxx"
+#include "css_parser.hxx"
+#include "contrib/expected/expected.hpp"
+
+namespace rspamd::css {
+
+struct alignas(int) css_color {
+ std::uint8_t r;
+ std::uint8_t g;
+ std::uint8_t b;
+
+ std::uint8_t alpha;
+
+ css_color(std::uint8_t _r, std::uint8_t _g, std::uint8_t _b, std::uint8_t _alpha = 255)
+ : r(_r), g(_g), b(_b), alpha(_alpha)
+ {
+ }
+ css_color() = default;
+ constexpr auto to_number() const -> std::uint32_t
+ {
+ return (std::uint32_t) alpha << 24 |
+ (std::uint32_t) r << 16 |
+ (std::uint32_t) g << 8 |
+ (std::uint32_t) b << 0;
+ }
+
+ constexpr auto to_rgb() const -> std::uint32_t
+ {
+ return (std::uint32_t) r << 16 |
+ (std::uint32_t) g << 8 |
+ (std::uint32_t) b << 0;
+ }
+ friend bool operator==(const css_color &l, const css_color &r)
+ {
+ return (memcmp(&l, &r, sizeof(css_color)) == 0);
+ }
+
+ static auto white() -> css_color
+ {
+ return css_color{255, 255, 255};
+ }
+ static auto black() -> css_color
+ {
+ return css_color{0, 0, 0};
+ }
+};
+
+struct css_dimension {
+ float dim;
+ bool is_percent;
+};
+
+/*
+ * Simple enum class for display stuff
+ */
+enum class css_display_value : std::uint8_t {
+ DISPLAY_INLINE,
+ DISPLAY_BLOCK,
+ DISPLAY_TABLE_ROW,
+ DISPLAY_HIDDEN
+};
+
+/*
+ * Value handler, uses std::variant instead of polymorphic classes for now
+ * for simplicity
+ */
+struct css_value {
+ std::variant<css_color,
+ float,
+ css_display_value,
+ css_dimension,
+ std::monostate>
+ value;
+
+ css_value()
+ {
+ }
+ css_value(const css_color &color)
+ : value(color)
+ {
+ }
+ css_value(float num)
+ : value(num)
+ {
+ }
+ css_value(css_dimension dim)
+ : value(dim)
+ {
+ }
+ css_value(css_display_value d)
+ : value(d)
+ {
+ }
+
+ auto to_color(void) const -> std::optional<css_color>
+ {
+ return extract_value_maybe<css_color>();
+ }
+
+ auto to_number(void) const -> std::optional<float>
+ {
+ return extract_value_maybe<float>();
+ }
+
+ auto to_dimension(void) const -> std::optional<css_dimension>
+ {
+ return extract_value_maybe<css_dimension>();
+ }
+
+ auto to_display(void) const -> std::optional<css_display_value>
+ {
+ return extract_value_maybe<css_display_value>();
+ }
+
+ auto is_valid(void) const -> bool
+ {
+ return !(std::holds_alternative<std::monostate>(value));
+ }
+
+ auto debug_str() const -> std::string;
+
+ static auto maybe_color_from_string(const std::string_view &input)
+ -> std::optional<css_value>;
+ static auto maybe_color_from_hex(const std::string_view &input)
+ -> std::optional<css_value>;
+ static auto maybe_color_from_function(const css_consumed_block::css_function_block &func)
+ -> std::optional<css_value>;
+ static auto maybe_dimension_from_number(const css_parser_token &tok)
+ -> std::optional<css_value>;
+ static auto maybe_display_from_string(const std::string_view &input)
+ -> std::optional<css_value>;
+
+private:
+ template<typename T>
+ auto extract_value_maybe(void) const -> std::optional<T>
+ {
+ if (std::holds_alternative<T>(value)) {
+ return std::get<T>(value);
+ }
+
+ return std::nullopt;
+ }
+};
+
+}// namespace rspamd::css
+
+
+#endif//RSPAMD_CSS_VALUE_HXX
diff --git a/src/libserver/css/parse_error.hxx b/src/libserver/css/parse_error.hxx
new file mode 100644
index 0000000..22b76f0
--- /dev/null
+++ b/src/libserver/css/parse_error.hxx
@@ -0,0 +1,61 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifndef RSPAMD_PARSE_ERROR_HXX
+#define RSPAMD_PARSE_ERROR_HXX
+
+#include <string>
+#include <optional>
+
+namespace rspamd::css {
+
+/*
+ * Generic parser errors
+ */
+enum class css_parse_error_type {
+ PARSE_ERROR_UNKNOWN_OPTION,
+ PARSE_ERROR_INVALID_SYNTAX,
+ PARSE_ERROR_BAD_NESTING,
+ PARSE_ERROR_NYI,
+ PARSE_ERROR_UNKNOWN_ERROR,
+ /* All above is treated as fatal error in parsing */
+ PARSE_ERROR_NO_ERROR,
+ PARSE_ERROR_EMPTY,
+};
+
+struct css_parse_error {
+ css_parse_error_type type = css_parse_error_type::PARSE_ERROR_UNKNOWN_ERROR;
+ std::optional<std::string> description;
+
+ explicit css_parse_error(css_parse_error_type type, const std::string &description)
+ : type(type), description(description)
+ {
+ }
+ explicit css_parse_error(css_parse_error_type type = css_parse_error_type::PARSE_ERROR_NO_ERROR)
+ : type(type)
+ {
+ }
+
+ constexpr auto is_fatal(void) const -> bool
+ {
+ return type < css_parse_error_type::PARSE_ERROR_NO_ERROR;
+ }
+};
+
+}// namespace rspamd::css
+#endif//RSPAMD_PARSE_ERROR_HXX
diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c
new file mode 100644
index 0000000..4318e87
--- /dev/null
+++ b/src/libserver/dkim.c
@@ -0,0 +1,3588 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "message.h"
+#include "dkim.h"
+#include "dns.h"
+#include "utlist.h"
+#include "unix-std.h"
+#include "mempool_vars_internal.h"
+
+#include <openssl/evp.h>
+#include <openssl/rsa.h>
+#include <openssl/engine.h>
+
+/* special DNS tokens */
+#define DKIM_DNSKEYNAME "_domainkey"
+
+/* ed25519 key lengths */
+#define ED25519_B64_BYTES 45
+#define ED25519_BYTES 32
+
+/* Canonization methods */
+#define DKIM_CANON_UNKNOWN (-1) /* unknown method */
+#define DKIM_CANON_SIMPLE 0 /* as specified in DKIM spec */
+#define DKIM_CANON_RELAXED 1 /* as specified in DKIM spec */
+
+#define DKIM_CANON_DEFAULT DKIM_CANON_SIMPLE
+
+#define RSPAMD_SHORT_BH_LEN 8
+
+/* Params */
+enum rspamd_dkim_param_type {
+ DKIM_PARAM_UNKNOWN = -1,
+ DKIM_PARAM_SIGNATURE = 0,
+ DKIM_PARAM_SIGNALG,
+ DKIM_PARAM_DOMAIN,
+ DKIM_PARAM_CANONALG,
+ DKIM_PARAM_QUERYMETHOD,
+ DKIM_PARAM_SELECTOR,
+ DKIM_PARAM_HDRLIST,
+ DKIM_PARAM_VERSION,
+ DKIM_PARAM_IDENTITY,
+ DKIM_PARAM_TIMESTAMP,
+ DKIM_PARAM_EXPIRATION,
+ DKIM_PARAM_COPIEDHDRS,
+ DKIM_PARAM_BODYHASH,
+ DKIM_PARAM_BODYLENGTH,
+ DKIM_PARAM_IDX,
+ DKIM_PARAM_CV,
+ DKIM_PARAM_IGNORE
+};
+
+#define RSPAMD_DKIM_MAX_ARC_IDX 10
+
+#define msg_err_dkim(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "dkim", ctx->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_dkim(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "dkim", ctx->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_dkim(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "dkim", ctx->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_dkim(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_dkim_log_id, "dkim", ctx->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_dkim_taskless(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_dkim_log_id, "dkim", "", \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(dkim)
+
+#define RSPAMD_DKIM_FLAG_OVERSIGN (1u << 0u)
+#define RSPAMD_DKIM_FLAG_OVERSIGN_EXISTING (1u << 1u)
+
+union rspamd_dkim_header_stat {
+ struct _st {
+ guint16 count;
+ guint16 flags;
+ } s;
+ guint32 n;
+};
+
+struct rspamd_dkim_common_ctx {
+ rspamd_mempool_t *pool;
+ guint64 sig_hash;
+ gsize len;
+ GPtrArray *hlist;
+ GHashTable *htable; /* header -> count mapping */
+ EVP_MD_CTX *headers_hash;
+ EVP_MD_CTX *body_hash;
+ enum rspamd_dkim_type type;
+ guint idx;
+ gint header_canon_type;
+ gint body_canon_type;
+ guint body_canonicalised;
+ guint headers_canonicalised;
+ gboolean is_sign;
+};
+
+enum rspamd_arc_seal_cv {
+ RSPAMD_ARC_UNKNOWN = 0,
+ RSPAMD_ARC_NONE,
+ RSPAMD_ARC_INVALID,
+ RSPAMD_ARC_FAIL,
+ RSPAMD_ARC_PASS
+};
+
+
+struct rspamd_dkim_context_s {
+ struct rspamd_dkim_common_ctx common;
+ rspamd_mempool_t *pool;
+ struct rspamd_dns_resolver *resolver;
+ gsize blen;
+ gsize bhlen;
+ gint sig_alg;
+ guint ver;
+ time_t timestamp;
+ time_t expiration;
+ gchar *domain;
+ gchar *selector;
+ gint8 *b;
+ gchar *short_b;
+ gint8 *bh;
+ gchar *dns_key;
+ enum rspamd_arc_seal_cv cv;
+ const gchar *dkim_header;
+};
+
+#define RSPAMD_DKIM_KEY_ID_LEN 16
+
+struct rspamd_dkim_key_s {
+ guint8 *keydata;
+ guint8 *raw_key;
+ gsize keylen;
+ gsize decoded_len;
+ gchar key_id[RSPAMD_DKIM_KEY_ID_LEN];
+ union {
+ RSA *key_rsa;
+ EC_KEY *key_ecdsa;
+ guchar *key_eddsa;
+ } key;
+ BIO *key_bio;
+ EVP_PKEY *key_evp;
+ time_t mtime;
+ guint ttl;
+ enum rspamd_dkim_key_type type;
+ ref_entry_t ref;
+};
+
+struct rspamd_dkim_sign_context_s {
+ struct rspamd_dkim_common_ctx common;
+ rspamd_dkim_sign_key_t *key;
+};
+
+struct rspamd_dkim_header {
+ const gchar *name;
+ gint count;
+};
+
+/* Parser of dkim params */
+typedef gboolean (*dkim_parse_param_f)(rspamd_dkim_context_t *ctx,
+ const gchar *param, gsize len, GError **err);
+
+static gboolean rspamd_dkim_parse_signature(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_signalg(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_domain(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_canonalg(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_ignore(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_selector(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_hdrlist(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_version(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_timestamp(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_expiration(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_bodyhash(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_bodylength(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_idx(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+static gboolean rspamd_dkim_parse_cv(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err);
+
+
+static const dkim_parse_param_f parser_funcs[] = {
+ [DKIM_PARAM_SIGNATURE] = rspamd_dkim_parse_signature,
+ [DKIM_PARAM_SIGNALG] = rspamd_dkim_parse_signalg,
+ [DKIM_PARAM_DOMAIN] = rspamd_dkim_parse_domain,
+ [DKIM_PARAM_CANONALG] = rspamd_dkim_parse_canonalg,
+ [DKIM_PARAM_QUERYMETHOD] = rspamd_dkim_parse_ignore,
+ [DKIM_PARAM_SELECTOR] = rspamd_dkim_parse_selector,
+ [DKIM_PARAM_HDRLIST] = rspamd_dkim_parse_hdrlist,
+ [DKIM_PARAM_VERSION] = rspamd_dkim_parse_version,
+ [DKIM_PARAM_IDENTITY] = rspamd_dkim_parse_ignore,
+ [DKIM_PARAM_TIMESTAMP] = rspamd_dkim_parse_timestamp,
+ [DKIM_PARAM_EXPIRATION] = rspamd_dkim_parse_expiration,
+ [DKIM_PARAM_COPIEDHDRS] = rspamd_dkim_parse_ignore,
+ [DKIM_PARAM_BODYHASH] = rspamd_dkim_parse_bodyhash,
+ [DKIM_PARAM_BODYLENGTH] = rspamd_dkim_parse_bodylength,
+ [DKIM_PARAM_IDX] = rspamd_dkim_parse_idx,
+ [DKIM_PARAM_CV] = rspamd_dkim_parse_cv,
+ [DKIM_PARAM_IGNORE] = rspamd_dkim_parse_ignore,
+};
+
+#define DKIM_ERROR dkim_error_quark()
+GQuark
+dkim_error_quark(void)
+{
+ return g_quark_from_static_string("dkim-error-quark");
+}
+
+/* Parsers implementation */
+static gboolean
+rspamd_dkim_parse_signature(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ ctx->b = rspamd_mempool_alloc0(ctx->pool, len);
+ ctx->short_b = rspamd_mempool_alloc0(ctx->pool, RSPAMD_SHORT_BH_LEN + 1);
+ rspamd_strlcpy(ctx->short_b, param, MIN(len, RSPAMD_SHORT_BH_LEN + 1));
+ (void) rspamd_cryptobox_base64_decode(param, len, ctx->b, &ctx->blen);
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_dkim_parse_signalg(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ /* XXX: ugly size comparison, improve this code style some day */
+ if (len == 8) {
+ if (memcmp(param, "rsa-sha1", len) == 0) {
+ ctx->sig_alg = DKIM_SIGN_RSASHA1;
+ return TRUE;
+ }
+ }
+ else if (len == 10) {
+ if (memcmp(param, "rsa-sha256", len) == 0) {
+ ctx->sig_alg = DKIM_SIGN_RSASHA256;
+ return TRUE;
+ }
+ else if (memcmp(param, "rsa-sha512", len) == 0) {
+ ctx->sig_alg = DKIM_SIGN_RSASHA512;
+ return TRUE;
+ }
+ }
+ else if (len == 15) {
+ if (memcmp(param, "ecdsa256-sha256", len) == 0) {
+ ctx->sig_alg = DKIM_SIGN_ECDSASHA256;
+ return TRUE;
+ }
+ else if (memcmp(param, "ecdsa256-sha512", len) == 0) {
+ ctx->sig_alg = DKIM_SIGN_ECDSASHA512;
+ return TRUE;
+ }
+ }
+ else if (len == 14) {
+ if (memcmp(param, "ed25519-sha256", len) == 0) {
+ ctx->sig_alg = DKIM_SIGN_EDDSASHA256;
+ return TRUE;
+ }
+ }
+
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_INVALID_A,
+ "invalid dkim sign algorithm");
+ return FALSE;
+}
+
+static gboolean
+rspamd_dkim_parse_domain(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ if (!rspamd_str_has_8bit(param, len)) {
+ ctx->domain = rspamd_mempool_alloc(ctx->pool, len + 1);
+ rspamd_strlcpy(ctx->domain, param, len + 1);
+ }
+ else {
+ ctx->domain = rspamd_dns_resolver_idna_convert_utf8(ctx->resolver,
+ ctx->pool, param, len, NULL);
+
+ if (!ctx->domain) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_INVALID_H,
+ "invalid dkim domain tag %.*s: idna failed",
+ (int) len, param);
+
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_dkim_parse_canonalg(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ const gchar *p, *slash = NULL, *end = param + len;
+ gsize sl = 0;
+
+ p = param;
+ while (p != end) {
+ if (*p == '/') {
+ slash = p;
+ break;
+ }
+ p++;
+ sl++;
+ }
+
+ if (slash == NULL) {
+ /* Only check header */
+ if (len == 6 && memcmp(param, "simple", len) == 0) {
+ ctx->common.header_canon_type = DKIM_CANON_SIMPLE;
+ return TRUE;
+ }
+ else if (len == 7 && memcmp(param, "relaxed", len) == 0) {
+ ctx->common.header_canon_type = DKIM_CANON_RELAXED;
+ return TRUE;
+ }
+ }
+ else {
+ /* First check header */
+ if (sl == 6 && memcmp(param, "simple", sl) == 0) {
+ ctx->common.header_canon_type = DKIM_CANON_SIMPLE;
+ }
+ else if (sl == 7 && memcmp(param, "relaxed", sl) == 0) {
+ ctx->common.header_canon_type = DKIM_CANON_RELAXED;
+ }
+ else {
+ goto err;
+ }
+ /* Check body */
+ len -= sl + 1;
+ slash++;
+ if (len == 6 && memcmp(slash, "simple", len) == 0) {
+ ctx->common.body_canon_type = DKIM_CANON_SIMPLE;
+ return TRUE;
+ }
+ else if (len == 7 && memcmp(slash, "relaxed", len) == 0) {
+ ctx->common.body_canon_type = DKIM_CANON_RELAXED;
+ return TRUE;
+ }
+ }
+
+err:
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_INVALID_A,
+ "invalid dkim canonization algorithm");
+ return FALSE;
+}
+
+static gboolean
+rspamd_dkim_parse_ignore(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ /* Just ignore unused params */
+ return TRUE;
+}
+
+static gboolean
+rspamd_dkim_parse_selector(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+
+ if (!rspamd_str_has_8bit(param, len)) {
+ ctx->selector = rspamd_mempool_alloc(ctx->pool, len + 1);
+ rspamd_strlcpy(ctx->selector, param, len + 1);
+ }
+ else {
+ ctx->selector = rspamd_dns_resolver_idna_convert_utf8(ctx->resolver,
+ ctx->pool, param, len, NULL);
+
+ if (!ctx->selector) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_INVALID_H,
+ "invalid dkim selector tag %.*s: idna failed",
+ (int) len, param);
+
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+static void
+rspamd_dkim_hlist_free(void *ud)
+{
+ GPtrArray *a = ud;
+
+ g_ptr_array_free(a, TRUE);
+}
+
+static gboolean
+rspamd_dkim_parse_hdrlist_common(struct rspamd_dkim_common_ctx *ctx,
+ const gchar *param,
+ gsize len,
+ gboolean sign,
+ GError **err)
+{
+ const gchar *c, *p, *end = param + len;
+ gchar *h;
+ gboolean from_found = FALSE, oversign, existing;
+ guint count = 0;
+ struct rspamd_dkim_header *new;
+ gpointer found;
+ union rspamd_dkim_header_stat u;
+
+ p = param;
+ while (p <= end) {
+ if ((p == end || *p == ':')) {
+ count++;
+ }
+ p++;
+ }
+
+ if (count > 0) {
+ ctx->hlist = g_ptr_array_sized_new(count);
+ }
+ else {
+ return FALSE;
+ }
+
+ c = param;
+ p = param;
+ ctx->htable = g_hash_table_new(rspamd_strcase_hash, rspamd_strcase_equal);
+
+ while (p <= end) {
+ if ((p == end || *p == ':') && p - c > 0) {
+ oversign = FALSE;
+ existing = FALSE;
+ h = rspamd_mempool_alloc(ctx->pool, p - c + 1);
+ rspamd_strlcpy(h, c, p - c + 1);
+
+ g_strstrip(h);
+
+ if (sign) {
+ if (rspamd_lc_cmp(h, "(o)", 3) == 0) {
+ oversign = TRUE;
+ h += 3;
+ msg_debug_dkim("oversign header: %s", h);
+ }
+ else if (rspamd_lc_cmp(h, "(x)", 3) == 0) {
+ oversign = TRUE;
+ existing = TRUE;
+ h += 3;
+ msg_debug_dkim("oversign existing header: %s", h);
+ }
+ }
+
+ /* Check mandatory from */
+ if (!from_found && g_ascii_strcasecmp(h, "from") == 0) {
+ from_found = TRUE;
+ }
+
+ new = rspamd_mempool_alloc(ctx->pool,
+ sizeof(struct rspamd_dkim_header));
+ new->name = h;
+ new->count = 0;
+ u.n = 0;
+
+ g_ptr_array_add(ctx->hlist, new);
+ found = g_hash_table_lookup(ctx->htable, h);
+
+ if (oversign) {
+ if (found) {
+ msg_err_dkim("specified oversigned header more than once: %s",
+ h);
+ }
+
+ u.s.flags |= RSPAMD_DKIM_FLAG_OVERSIGN;
+
+ if (existing) {
+ u.s.flags |= RSPAMD_DKIM_FLAG_OVERSIGN_EXISTING;
+ }
+
+ u.s.count = 0;
+ }
+ else {
+ if (found != NULL) {
+ u.n = GPOINTER_TO_UINT(found);
+ new->count = u.s.count;
+ u.s.count++;
+ }
+ else {
+ /* Insert new header order to the list */
+ u.s.count = new->count + 1;
+ }
+ }
+
+ g_hash_table_insert(ctx->htable, h, GUINT_TO_POINTER(u.n));
+
+ c = p + 1;
+ p++;
+ }
+ else {
+ p++;
+ }
+ }
+
+ if (!ctx->hlist) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_INVALID_H,
+ "invalid dkim header list");
+ return FALSE;
+ }
+ else {
+ if (!from_found) {
+ g_ptr_array_free(ctx->hlist, TRUE);
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_INVALID_H,
+ "invalid dkim header list, from header is missing");
+ return FALSE;
+ }
+
+ rspamd_mempool_add_destructor(ctx->pool,
+ (rspamd_mempool_destruct_t) rspamd_dkim_hlist_free,
+ ctx->hlist);
+ rspamd_mempool_add_destructor(ctx->pool,
+ (rspamd_mempool_destruct_t) g_hash_table_unref,
+ ctx->htable);
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_dkim_parse_hdrlist(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ return rspamd_dkim_parse_hdrlist_common(&ctx->common, param, len, FALSE, err);
+}
+
+static gboolean
+rspamd_dkim_parse_version(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ if (len != 1 || *param != '1') {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_VERSION,
+ "invalid dkim version");
+ return FALSE;
+ }
+
+ ctx->ver = 1;
+ return TRUE;
+}
+
+static gboolean
+rspamd_dkim_parse_timestamp(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ gulong val;
+
+ if (!rspamd_strtoul(param, len, &val)) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "invalid dkim timestamp");
+ return FALSE;
+ }
+ ctx->timestamp = val;
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_dkim_parse_expiration(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ gulong val;
+
+ if (!rspamd_strtoul(param, len, &val)) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "invalid dkim expiration");
+ return FALSE;
+ }
+ ctx->expiration = val;
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_dkim_parse_bodyhash(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ ctx->bh = rspamd_mempool_alloc0(ctx->pool, len);
+ (void) rspamd_cryptobox_base64_decode(param, len, ctx->bh, &ctx->bhlen);
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_dkim_parse_bodylength(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ gulong val;
+
+ if (!rspamd_strtoul(param, len, &val)) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_INVALID_L,
+ "invalid dkim body length");
+ return FALSE;
+ }
+ ctx->common.len = val;
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_dkim_parse_idx(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+ gulong val;
+
+ if (!rspamd_strtoul(param, len, &val)) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_INVALID_L,
+ "invalid ARC idx");
+ return FALSE;
+ }
+ ctx->common.idx = val;
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_dkim_parse_cv(rspamd_dkim_context_t *ctx,
+ const gchar *param,
+ gsize len,
+ GError **err)
+{
+
+ /* Only check header */
+ if (len == 4 && memcmp(param, "fail", len) == 0) {
+ ctx->cv = RSPAMD_ARC_FAIL;
+ return TRUE;
+ }
+ else if (len == 4 && memcmp(param, "pass", len) == 0) {
+ ctx->cv = RSPAMD_ARC_PASS;
+ return TRUE;
+ }
+ else if (len == 4 && memcmp(param, "none", len) == 0) {
+ ctx->cv = RSPAMD_ARC_NONE;
+ return TRUE;
+ }
+ else if (len == 7 && memcmp(param, "invalid", len) == 0) {
+ ctx->cv = RSPAMD_ARC_INVALID;
+ return TRUE;
+ }
+
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "invalid arc seal verification result");
+
+ return FALSE;
+}
+
+
+static void
+rspamd_dkim_add_arc_seal_headers(rspamd_mempool_t *pool,
+ struct rspamd_dkim_common_ctx *ctx)
+{
+ struct rspamd_dkim_header *hdr;
+ gint count = ctx->idx, i;
+
+ ctx->hlist = g_ptr_array_sized_new(count * 3 - 1);
+
+ for (i = 0; i < count; i++) {
+ /* Authentication results */
+ hdr = rspamd_mempool_alloc(pool, sizeof(*hdr));
+ hdr->name = RSPAMD_DKIM_ARC_AUTHHEADER;
+ hdr->count = -(i + 1);
+ g_ptr_array_add(ctx->hlist, hdr);
+
+ /* Arc signature */
+ hdr = rspamd_mempool_alloc(pool, sizeof(*hdr));
+ hdr->name = RSPAMD_DKIM_ARC_SIGNHEADER;
+ hdr->count = -(i + 1);
+ g_ptr_array_add(ctx->hlist, hdr);
+
+ /* Arc seal (except last one) */
+ if (i != count - 1) {
+ hdr = rspamd_mempool_alloc(pool, sizeof(*hdr));
+ hdr->name = RSPAMD_DKIM_ARC_SEALHEADER;
+ hdr->count = -(i + 1);
+ g_ptr_array_add(ctx->hlist, hdr);
+ }
+ }
+
+ rspamd_mempool_add_destructor(ctx->pool,
+ (rspamd_mempool_destruct_t) rspamd_dkim_hlist_free,
+ ctx->hlist);
+}
+
+/**
+ * Create new dkim context from signature
+ * @param sig message's signature
+ * @param pool pool to allocate memory from
+ * @param err pointer to error object
+ * @return new context or NULL
+ */
+rspamd_dkim_context_t *
+rspamd_create_dkim_context(const gchar *sig,
+ rspamd_mempool_t *pool,
+ struct rspamd_dns_resolver *resolver,
+ guint time_jitter,
+ enum rspamd_dkim_type type,
+ GError **err)
+{
+ const gchar *p, *c, *tag = NULL, *end;
+ gint taglen;
+ gint param = DKIM_PARAM_UNKNOWN;
+ const EVP_MD *md_alg;
+ time_t now;
+ rspamd_dkim_context_t *ctx;
+ enum {
+ DKIM_STATE_TAG = 0,
+ DKIM_STATE_AFTER_TAG,
+ DKIM_STATE_VALUE,
+ DKIM_STATE_SKIP_SPACES = 99,
+ DKIM_STATE_ERROR = 100
+ } state,
+ next_state;
+
+
+ if (sig == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_EMPTY_B,
+ "empty signature");
+ return NULL;
+ }
+
+ ctx = rspamd_mempool_alloc0(pool, sizeof(rspamd_dkim_context_t));
+ ctx->pool = pool;
+ ctx->resolver = resolver;
+
+ if (type == RSPAMD_DKIM_ARC_SEAL) {
+ ctx->common.header_canon_type = DKIM_CANON_RELAXED;
+ ctx->common.body_canon_type = DKIM_CANON_RELAXED;
+ }
+ else {
+ ctx->common.header_canon_type = DKIM_CANON_DEFAULT;
+ ctx->common.body_canon_type = DKIM_CANON_DEFAULT;
+ }
+
+ ctx->sig_alg = DKIM_SIGN_UNKNOWN;
+ ctx->common.pool = pool;
+ ctx->common.type = type;
+ /* A simple state machine of parsing tags */
+ state = DKIM_STATE_SKIP_SPACES;
+ next_state = DKIM_STATE_TAG;
+ taglen = 0;
+ p = sig;
+ c = sig;
+ end = p + strlen(p);
+ ctx->common.sig_hash = rspamd_cryptobox_fast_hash(sig, end - sig,
+ rspamd_hash_seed());
+
+ msg_debug_dkim("create dkim context sig = %L", ctx->common.sig_hash);
+
+ while (p <= end) {
+ switch (state) {
+ case DKIM_STATE_TAG:
+ if (g_ascii_isspace(*p)) {
+ taglen = (int) (p - c);
+ while (*p && g_ascii_isspace(*p)) {
+ /* Skip spaces before '=' sign */
+ p++;
+ }
+ if (*p != '=') {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "invalid dkim param");
+ state = DKIM_STATE_ERROR;
+ }
+ else {
+ state = DKIM_STATE_SKIP_SPACES;
+ next_state = DKIM_STATE_AFTER_TAG;
+ param = DKIM_PARAM_UNKNOWN;
+ p++;
+ tag = c;
+ }
+ }
+ else if (*p == '=') {
+ state = DKIM_STATE_SKIP_SPACES;
+ next_state = DKIM_STATE_AFTER_TAG;
+ param = DKIM_PARAM_UNKNOWN;
+ p++;
+ tag = c;
+ }
+ else {
+ taglen++;
+
+ if (taglen > G_MAXINT8) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "too long dkim tag");
+ state = DKIM_STATE_ERROR;
+ }
+ else {
+ p++;
+ }
+ }
+ break;
+ case DKIM_STATE_AFTER_TAG:
+ /* We got tag at tag and len at taglen */
+ switch (taglen) {
+ case 0:
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "zero length dkim param");
+ state = DKIM_STATE_ERROR;
+ break;
+ case 1:
+ /* 1 character tags */
+ switch (*tag) {
+ case 'v':
+ if (type == RSPAMD_DKIM_NORMAL) {
+ param = DKIM_PARAM_VERSION;
+ }
+ else {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "invalid ARC v param");
+ state = DKIM_STATE_ERROR;
+ break;
+ }
+ break;
+ case 'a':
+ param = DKIM_PARAM_SIGNALG;
+ break;
+ case 'b':
+ param = DKIM_PARAM_SIGNATURE;
+ break;
+ case 'c':
+ param = DKIM_PARAM_CANONALG;
+ break;
+ case 'd':
+ param = DKIM_PARAM_DOMAIN;
+ break;
+ case 'h':
+ if (type == RSPAMD_DKIM_ARC_SEAL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "ARC seal must NOT have h= tag");
+ state = DKIM_STATE_ERROR;
+ break;
+ }
+ else {
+ param = DKIM_PARAM_HDRLIST;
+ }
+ break;
+ case 'i':
+ if (type == RSPAMD_DKIM_NORMAL) {
+ param = DKIM_PARAM_IDENTITY;
+ }
+ else {
+ param = DKIM_PARAM_IDX;
+ }
+ break;
+ case 'l':
+ param = DKIM_PARAM_BODYLENGTH;
+ break;
+ case 'q':
+ param = DKIM_PARAM_QUERYMETHOD;
+ break;
+ case 's':
+ param = DKIM_PARAM_SELECTOR;
+ break;
+ case 't':
+ param = DKIM_PARAM_TIMESTAMP;
+ break;
+ case 'x':
+ param = DKIM_PARAM_EXPIRATION;
+ break;
+ case 'z':
+ param = DKIM_PARAM_COPIEDHDRS;
+ break;
+ case 'r':
+ param = DKIM_PARAM_IGNORE;
+ break;
+ default:
+ param = DKIM_PARAM_UNKNOWN;
+ msg_debug_dkim("unknown DKIM param %c, ignoring it", *tag);
+ break;
+ }
+ break;
+ case 2:
+ /* Two characters tags, e.g. `bh` */
+ if (tag[0] == 'b' && tag[1] == 'h') {
+ if (type == RSPAMD_DKIM_ARC_SEAL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "ARC seal must NOT have bh= tag");
+ state = DKIM_STATE_ERROR;
+ }
+ else {
+ param = DKIM_PARAM_BODYHASH;
+ }
+ }
+ else if (tag[0] == 'c' && tag[1] == 'v') {
+ if (type != RSPAMD_DKIM_ARC_SEAL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "cv tag is valid for ARC-Seal only");
+ state = DKIM_STATE_ERROR;
+ }
+ else {
+ param = DKIM_PARAM_CV;
+ }
+ }
+ else {
+ param = DKIM_PARAM_UNKNOWN;
+ msg_debug_dkim("unknown DKIM param %*s, ignoring it", taglen, tag);
+ }
+ break;
+ default:
+ /* Long and unknown (yet) DKIM tag */
+ param = DKIM_PARAM_UNKNOWN;
+ msg_debug_dkim("unknown DKIM param %*s, ignoring it", taglen, tag);
+ break;
+ }
+
+ if (state != DKIM_STATE_ERROR) {
+ /* Skip spaces */
+ state = DKIM_STATE_SKIP_SPACES;
+ next_state = DKIM_STATE_VALUE;
+ }
+ break;
+ case DKIM_STATE_VALUE:
+ if (*p == ';') {
+ if (p - c == 0 || c > p) {
+ state = DKIM_STATE_ERROR;
+ }
+ else {
+ /* Cut trailing spaces for value */
+ gint tlen = p - c;
+ const gchar *tmp = p - 1;
+
+ while (tlen > 0) {
+ if (!g_ascii_isspace(*tmp)) {
+ break;
+ }
+ tlen--;
+ tmp--;
+ }
+
+ if (param != DKIM_PARAM_UNKNOWN) {
+ if (!parser_funcs[param](ctx, c, tlen, err)) {
+ state = DKIM_STATE_ERROR;
+ }
+ else {
+ state = DKIM_STATE_SKIP_SPACES;
+ next_state = DKIM_STATE_TAG;
+ p++;
+ taglen = 0;
+ }
+ }
+ else {
+ /* Unknown param has been ignored */
+ msg_debug_dkim("ignored unknown tag parameter value: %*s = %*s",
+ taglen, tag, tlen, c);
+ state = DKIM_STATE_SKIP_SPACES;
+ next_state = DKIM_STATE_TAG;
+ p++;
+ taglen = 0;
+ }
+ }
+ }
+ else if (p == end) {
+ /* Last parameter with no `;` character */
+ gint tlen = p - c;
+ const gchar *tmp = p - 1;
+
+ while (tlen > 0) {
+ if (!g_ascii_isspace(*tmp)) {
+ break;
+ }
+ tlen--;
+ tmp--;
+ }
+
+ if (param != DKIM_PARAM_UNKNOWN) {
+ if (!parser_funcs[param](ctx, c, tlen, err)) {
+ state = DKIM_STATE_ERROR;
+ }
+ }
+ else {
+ msg_debug_dkim("ignored unknown tag parameter value: %*s: %*s",
+ taglen, tag, tlen, c);
+ }
+
+ if (state == DKIM_STATE_ERROR) {
+ /*
+ * We need to return from here as state machine won't
+ * do any more steps after p == end
+ */
+ if (err) {
+ msg_info_dkim("dkim parse failed: %e", *err);
+ }
+
+ return NULL;
+ }
+ /* Finish processing */
+ p++;
+ }
+ else {
+ p++;
+ }
+ break;
+ case DKIM_STATE_SKIP_SPACES:
+ if (g_ascii_isspace(*p)) {
+ p++;
+ }
+ else {
+ c = p;
+ state = next_state;
+ }
+ break;
+ case DKIM_STATE_ERROR:
+ if (err && *err) {
+ msg_info_dkim("dkim parse failed: %s", (*err)->message);
+ return NULL;
+ }
+ else {
+ msg_info_dkim("dkim parse failed: unknown error when parsing %c tag",
+ tag ? *tag : '?');
+ return NULL;
+ }
+ break;
+ }
+ }
+
+ if (type == RSPAMD_DKIM_ARC_SEAL) {
+ rspamd_dkim_add_arc_seal_headers(pool, &ctx->common);
+ }
+
+ /* Now check validity of signature */
+ if (ctx->b == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_EMPTY_B,
+ "b parameter missing");
+ return NULL;
+ }
+ if (ctx->common.type != RSPAMD_DKIM_ARC_SEAL && ctx->bh == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_EMPTY_BH,
+ "bh parameter missing");
+ return NULL;
+ }
+ if (ctx->domain == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_EMPTY_D,
+ "domain parameter missing");
+ return NULL;
+ }
+ if (ctx->selector == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_EMPTY_S,
+ "selector parameter missing");
+ return NULL;
+ }
+ if (ctx->common.type == RSPAMD_DKIM_NORMAL && ctx->ver == 0) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_EMPTY_V,
+ "v parameter missing");
+ return NULL;
+ }
+ if (ctx->common.hlist == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_EMPTY_H,
+ "h parameter missing");
+ return NULL;
+ }
+ if (ctx->sig_alg == DKIM_SIGN_UNKNOWN) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_EMPTY_S,
+ "s parameter missing");
+ return NULL;
+ }
+
+ if (type != RSPAMD_DKIM_ARC_SEAL) {
+ if (ctx->sig_alg == DKIM_SIGN_RSASHA1) {
+ /* Check bh length */
+ if (ctx->bhlen != (guint) EVP_MD_size(EVP_sha1())) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_BADSIG,
+ "signature has incorrect length: %zu",
+ ctx->bhlen);
+ return NULL;
+ }
+ }
+ else if (ctx->sig_alg == DKIM_SIGN_RSASHA256 ||
+ ctx->sig_alg == DKIM_SIGN_ECDSASHA256) {
+ if (ctx->bhlen !=
+ (guint) EVP_MD_size(EVP_sha256())) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_BADSIG,
+ "signature has incorrect length: %zu",
+ ctx->bhlen);
+ return NULL;
+ }
+ }
+ else if (ctx->sig_alg == DKIM_SIGN_RSASHA512 ||
+ ctx->sig_alg == DKIM_SIGN_ECDSASHA512) {
+ if (ctx->bhlen !=
+ (guint) EVP_MD_size(EVP_sha512())) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_BADSIG,
+ "signature has incorrect length: %zu",
+ ctx->bhlen);
+ return NULL;
+ }
+ }
+ }
+
+ /* Check expiration */
+ now = time(NULL);
+ if (ctx->timestamp && now < ctx->timestamp && ctx->timestamp - now > (gint) time_jitter) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_FUTURE,
+ "signature was made in future, ignoring");
+ return NULL;
+ }
+ if (ctx->expiration && ctx->expiration < now) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_EXPIRED,
+ "signature has expired");
+ return NULL;
+ }
+
+ if (ctx->common.type != RSPAMD_DKIM_NORMAL && (ctx->common.idx == 0 ||
+ ctx->common.idx > RSPAMD_DKIM_MAX_ARC_IDX)) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "i parameter missing or invalid for ARC");
+ return NULL;
+ }
+
+ if (ctx->common.type == RSPAMD_DKIM_ARC_SEAL) {
+ if (ctx->cv == RSPAMD_ARC_UNKNOWN) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_UNKNOWN,
+ "cv parameter missing or invalid for ARC");
+ return NULL;
+ }
+ }
+
+ /* Now create dns key to request further */
+ gsize dnslen = strlen(ctx->domain) + strlen(ctx->selector) +
+ sizeof(DKIM_DNSKEYNAME) + 2;
+ ctx->dns_key = rspamd_mempool_alloc(ctx->pool, dnslen);
+ rspamd_snprintf(ctx->dns_key,
+ dnslen,
+ "%s.%s.%s",
+ ctx->selector,
+ DKIM_DNSKEYNAME,
+ ctx->domain);
+
+ /* Create checksums for further operations */
+ if (ctx->sig_alg == DKIM_SIGN_RSASHA1) {
+ md_alg = EVP_sha1();
+ }
+ else if (ctx->sig_alg == DKIM_SIGN_RSASHA256 ||
+ ctx->sig_alg == DKIM_SIGN_ECDSASHA256 ||
+ ctx->sig_alg == DKIM_SIGN_EDDSASHA256) {
+ md_alg = EVP_sha256();
+ }
+ else if (ctx->sig_alg == DKIM_SIGN_RSASHA512 ||
+ ctx->sig_alg == DKIM_SIGN_ECDSASHA512) {
+ md_alg = EVP_sha512();
+ }
+ else {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_BADSIG,
+ "signature has unsupported signature algorithm");
+
+ return NULL;
+ }
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+ ctx->common.body_hash = EVP_MD_CTX_create();
+ EVP_DigestInit_ex(ctx->common.body_hash, md_alg, NULL);
+ ctx->common.headers_hash = EVP_MD_CTX_create();
+ EVP_DigestInit_ex(ctx->common.headers_hash, md_alg, NULL);
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) EVP_MD_CTX_destroy, ctx->common.body_hash);
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) EVP_MD_CTX_destroy, ctx->common.headers_hash);
+#else
+ ctx->common.body_hash = EVP_MD_CTX_new();
+ EVP_DigestInit_ex(ctx->common.body_hash, md_alg, NULL);
+ ctx->common.headers_hash = EVP_MD_CTX_new();
+ EVP_DigestInit_ex(ctx->common.headers_hash, md_alg, NULL);
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) EVP_MD_CTX_free, ctx->common.body_hash);
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) EVP_MD_CTX_free, ctx->common.headers_hash);
+#endif
+ ctx->dkim_header = sig;
+
+ return ctx;
+}
+
+struct rspamd_dkim_key_cbdata {
+ rspamd_dkim_context_t *ctx;
+ dkim_key_handler_f handler;
+ gpointer ud;
+};
+
+rspamd_dkim_key_t *
+rspamd_dkim_make_key(const gchar *keydata,
+ guint keylen, enum rspamd_dkim_key_type type, GError **err)
+{
+ rspamd_dkim_key_t *key = NULL;
+
+ if (keylen < 3) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "DKIM key is too short to be valid");
+ return NULL;
+ }
+
+ key = g_malloc0(sizeof(rspamd_dkim_key_t));
+ REF_INIT_RETAIN(key, rspamd_dkim_key_free);
+ key->keydata = g_malloc0(keylen + 1);
+ key->raw_key = g_malloc(keylen);
+ key->decoded_len = keylen;
+ key->type = type;
+
+ /* Copy key skipping all spaces and newlines */
+ const char *h = keydata;
+ guint8 *t = key->raw_key;
+
+ while (h - keydata < keylen) {
+ if (!g_ascii_isspace(*h)) {
+ *t++ = *h++;
+ }
+ else {
+ h++;
+ }
+ }
+
+ key->keylen = t - key->raw_key;
+
+ if (!rspamd_cryptobox_base64_decode(key->raw_key, key->keylen, key->keydata,
+ &key->decoded_len)) {
+ REF_RELEASE(key);
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "DKIM key is not a valid base64 string");
+
+ return NULL;
+ }
+
+ /* Calculate ID -> md5 */
+ EVP_MD_CTX *mdctx = EVP_MD_CTX_create();
+
+#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
+ EVP_MD_CTX_set_flags(mdctx, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+#endif
+
+ if (EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) == 1) {
+ guint dlen = sizeof(key->key_id);
+
+ EVP_DigestUpdate(mdctx, key->keydata, key->decoded_len);
+ EVP_DigestFinal_ex(mdctx, key->key_id, &dlen);
+ }
+
+ EVP_MD_CTX_destroy(mdctx);
+
+ if (key->type == RSPAMD_DKIM_KEY_EDDSA) {
+ key->key.key_eddsa = key->keydata;
+
+ if (key->decoded_len != rspamd_cryptobox_pk_sig_bytes(
+ RSPAMD_CRYPTOBOX_MODE_25519)) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "DKIM key is has invalid length %d for eddsa; expected %d",
+ (gint) key->decoded_len,
+ rspamd_cryptobox_pk_sig_bytes(RSPAMD_CRYPTOBOX_MODE_25519));
+ REF_RELEASE(key);
+
+ return NULL;
+ }
+ }
+ else {
+ key->key_bio = BIO_new_mem_buf(key->keydata, key->decoded_len);
+
+ if (key->key_bio == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "cannot make ssl bio from key");
+ REF_RELEASE(key);
+
+ return NULL;
+ }
+
+ key->key_evp = d2i_PUBKEY_bio(key->key_bio, NULL);
+
+ if (key->key_evp == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "cannot extract pubkey from bio");
+ REF_RELEASE(key);
+
+ return NULL;
+ }
+
+ if (type == RSPAMD_DKIM_KEY_RSA) {
+ key->key.key_rsa = EVP_PKEY_get1_RSA(key->key_evp);
+
+ if (key->key.key_rsa == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "cannot extract rsa key from evp key");
+ REF_RELEASE(key);
+
+ return NULL;
+ }
+ }
+ else {
+ key->key.key_ecdsa = EVP_PKEY_get1_EC_KEY(key->key_evp);
+
+ if (key->key.key_ecdsa == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "cannot extract ecdsa key from evp key");
+ REF_RELEASE(key);
+
+ return NULL;
+ }
+ }
+ }
+
+ return key;
+}
+
+const guchar *
+rspamd_dkim_key_id(rspamd_dkim_key_t *key)
+{
+ if (key) {
+ return key->key_id;
+ }
+
+ return NULL;
+}
+
+/**
+ * Free DKIM key
+ * @param key
+ */
+void rspamd_dkim_key_free(rspamd_dkim_key_t *key)
+{
+ if (key->key_evp) {
+ EVP_PKEY_free(key->key_evp);
+ }
+
+ if (key->type == RSPAMD_DKIM_KEY_RSA) {
+ if (key->key.key_rsa) {
+ RSA_free(key->key.key_rsa);
+ }
+ }
+ else if (key->type == RSPAMD_DKIM_KEY_ECDSA) {
+ if (key->key.key_ecdsa) {
+ EC_KEY_free(key->key.key_ecdsa);
+ }
+ }
+ /* Nothing in case of eddsa key */
+ if (key->key_bio) {
+ BIO_free(key->key_bio);
+ }
+
+ g_free(key->raw_key);
+ g_free(key->keydata);
+ g_free(key);
+}
+
+void rspamd_dkim_sign_key_free(rspamd_dkim_sign_key_t *key)
+{
+ if (key->key_evp) {
+ EVP_PKEY_free(key->key_evp);
+ }
+ if (key->type == RSPAMD_DKIM_KEY_RSA) {
+ if (key->key.key_rsa) {
+ RSA_free(key->key.key_rsa);
+ }
+ }
+ if (key->key_bio) {
+ BIO_free(key->key_bio);
+ }
+
+ if (key->type == RSPAMD_DKIM_KEY_EDDSA) {
+ rspamd_explicit_memzero(key->key.key_eddsa, key->keylen);
+ g_free(key->keydata);
+ }
+
+ g_free(key);
+}
+
+rspamd_dkim_key_t *
+rspamd_dkim_parse_key(const gchar *txt, gsize *keylen, GError **err)
+{
+ const gchar *c, *p, *end, *key = NULL, *alg = "rsa";
+ enum {
+ read_tag = 0,
+ read_tag_before_eqsign,
+ read_eqsign,
+ read_p_tag,
+ read_k_tag,
+ ignore_value,
+ skip_spaces,
+ } state = read_tag,
+ next_state;
+ gchar tag = '\0';
+ gsize klen = 0, alglen = 0;
+
+ c = txt;
+ p = txt;
+ end = txt + strlen(txt);
+
+ while (p < end) {
+ switch (state) {
+ case read_tag:
+ if (*p == '=') {
+ state = read_eqsign;
+ }
+ else if (g_ascii_isspace(*p)) {
+ state = skip_spaces;
+
+ if (tag != '\0') {
+ /* We had tag letter */
+ next_state = read_tag_before_eqsign;
+ }
+ else {
+ /* We had no tag letter, so we ignore empty tag */
+ next_state = read_tag;
+ }
+ }
+ else {
+ tag = *p;
+ }
+ p++;
+ break;
+ case read_tag_before_eqsign:
+ /* Input: spaces before eqsign
+ * Output: either read a next tag (previous had no value), or read value
+ * p is moved forward
+ */
+ if (*p == '=') {
+ state = read_eqsign;
+ }
+ else {
+ tag = *p;
+ state = read_tag;
+ }
+ p++;
+ break;
+ case read_eqsign:
+ /* Always switch to skip spaces state and do not advance p */
+ state = skip_spaces;
+
+ if (tag == 'p') {
+ next_state = read_p_tag;
+ }
+ else if (tag == 'k') {
+ next_state = read_k_tag;
+ }
+ else {
+ /* Unknown tag, ignore */
+ next_state = ignore_value;
+ tag = '\0';
+ }
+ break;
+ case read_p_tag:
+ if (*p == ';') {
+ klen = p - c;
+ key = c;
+ state = read_tag;
+ tag = '\0';
+ p++;
+ }
+ else {
+ p++;
+ }
+ break;
+ case read_k_tag:
+ if (*p == ';') {
+ alglen = p - c;
+ alg = c;
+ state = read_tag;
+ tag = '\0';
+ p++;
+ }
+ else if (g_ascii_isspace(*p)) {
+ alglen = p - c;
+ alg = c;
+ state = skip_spaces;
+ next_state = read_tag;
+ tag = '\0';
+ }
+ else {
+ p++;
+ }
+ break;
+ case ignore_value:
+ if (*p == ';') {
+ state = read_tag;
+ tag = '\0';
+ p++;
+ }
+ else if (g_ascii_isspace(*p)) {
+ state = skip_spaces;
+ next_state = read_tag;
+ tag = '\0';
+ }
+ else {
+ p++;
+ }
+ break;
+ case skip_spaces:
+ /* Skip spaces and switch to the next state if needed */
+ if (g_ascii_isspace(*p)) {
+ p++;
+ }
+ else {
+ c = p;
+ state = next_state;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ /* Leftover */
+ switch (state) {
+ case read_p_tag:
+ klen = p - c;
+ key = c;
+ break;
+ case read_k_tag:
+ alglen = p - c;
+ alg = c;
+ break;
+ default:
+ break;
+ }
+
+ if (klen == 0 || key == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "key is missing");
+
+ return NULL;
+ }
+
+ if (alglen == 0 || alg == NULL) {
+ alg = "rsa"; /* Implicit */
+ alglen = 3;
+ }
+
+ if (keylen) {
+ *keylen = klen;
+ }
+
+ if (alglen == 8 && rspamd_lc_cmp(alg, "ecdsa256", alglen) == 0) {
+ return rspamd_dkim_make_key(key, klen,
+ RSPAMD_DKIM_KEY_ECDSA, err);
+ }
+ else if (alglen == 7 && rspamd_lc_cmp(alg, "ed25519", alglen) == 0) {
+ return rspamd_dkim_make_key(key, klen,
+ RSPAMD_DKIM_KEY_EDDSA, err);
+ }
+ else {
+ /* We assume RSA default in all cases */
+ return rspamd_dkim_make_key(key, klen,
+ RSPAMD_DKIM_KEY_RSA, err);
+ }
+
+ g_assert_not_reached();
+
+ return NULL;
+}
+
+/* Get TXT request data and parse it */
+static void
+rspamd_dkim_dns_cb(struct rdns_reply *reply, gpointer arg)
+{
+ struct rspamd_dkim_key_cbdata *cbdata = arg;
+ rspamd_dkim_key_t *key = NULL;
+ GError *err = NULL;
+ struct rdns_reply_entry *elt;
+ gsize keylen = 0;
+
+ if (reply->code != RDNS_RC_NOERROR) {
+ gint err_code = DKIM_SIGERROR_NOKEY;
+ if (reply->code == RDNS_RC_NOREC) {
+ err_code = DKIM_SIGERROR_NOREC;
+ }
+ else if (reply->code == RDNS_RC_NXDOMAIN) {
+ err_code = DKIM_SIGERROR_NOREC;
+ }
+ g_set_error(&err,
+ DKIM_ERROR,
+ err_code,
+ "dns request to %s failed: %s",
+ cbdata->ctx->dns_key,
+ rdns_strerror(reply->code));
+ cbdata->handler(NULL, 0, cbdata->ctx, cbdata->ud, err);
+ }
+ else {
+ LL_FOREACH(reply->entries, elt)
+ {
+ if (elt->type == RDNS_REQUEST_TXT) {
+ if (err != NULL) {
+ /* Free error as it is insignificant */
+ g_error_free(err);
+ err = NULL;
+ }
+ key = rspamd_dkim_parse_key(elt->content.txt.data,
+ &keylen,
+ &err);
+ if (key) {
+ key->ttl = elt->ttl;
+ break;
+ }
+ }
+ }
+ cbdata->handler(key, keylen, cbdata->ctx, cbdata->ud, err);
+ }
+}
+
+/**
+ * Make DNS request for specified context and obtain and parse key
+ * @param ctx dkim context from signature
+ * @param resolver dns resolver object
+ * @param s async session to make request
+ * @return
+ */
+gboolean
+rspamd_get_dkim_key(rspamd_dkim_context_t *ctx,
+ struct rspamd_task *task,
+ dkim_key_handler_f handler,
+ gpointer ud)
+{
+ struct rspamd_dkim_key_cbdata *cbdata;
+
+ g_return_val_if_fail(ctx != NULL, FALSE);
+ g_return_val_if_fail(ctx->dns_key != NULL, FALSE);
+
+ cbdata =
+ rspamd_mempool_alloc(ctx->pool,
+ sizeof(struct rspamd_dkim_key_cbdata));
+ cbdata->ctx = ctx;
+ cbdata->handler = handler;
+ cbdata->ud = ud;
+
+ return rspamd_dns_resolver_request_task_forced(task,
+ rspamd_dkim_dns_cb,
+ cbdata,
+ RDNS_REQUEST_TXT,
+ ctx->dns_key);
+}
+
+static gboolean
+rspamd_dkim_relaxed_body_step(struct rspamd_dkim_common_ctx *ctx, EVP_MD_CTX *ck,
+ const gchar **start, guint size,
+ gssize *remain)
+{
+ const gchar *h;
+ gchar *t;
+ guint len, inlen;
+ gssize octets_remain;
+ gboolean got_sp, ret = TRUE;
+ gchar buf[1024];
+
+ len = size;
+ inlen = sizeof(buf) - 1;
+ h = *start;
+ t = buf;
+ got_sp = FALSE;
+ octets_remain = *remain;
+
+ while (len > 0 && inlen > 0 && (octets_remain > 0)) {
+
+ if (*h == '\r' || *h == '\n') {
+ if (got_sp) {
+ /* Ignore spaces at the end of line */
+ t--;
+ }
+ *t++ = '\r';
+ *t++ = '\n';
+
+ if (len > 1 && (*h == '\r' && h[1] == '\n')) {
+ h += 2;
+ len -= 2;
+ octets_remain -= 2;
+ }
+ else {
+ h++;
+ len--;
+ if (octets_remain >= 2) {
+ octets_remain -= 2; /* Input has just \n or \r so we actually add more octets */
+ }
+ else {
+ octets_remain--;
+ break;
+ }
+ }
+ break;
+ }
+ else if (g_ascii_isspace(*h)) {
+ if (got_sp) {
+ /* Ignore multiply spaces */
+ h++;
+ len--;
+ continue;
+ }
+ else {
+ *t++ = ' ';
+ h++;
+ inlen--;
+ len--;
+ octets_remain--;
+ got_sp = TRUE;
+ continue;
+ }
+ }
+ else {
+ got_sp = FALSE;
+ }
+
+ *t++ = *h++;
+ inlen--;
+ len--;
+ octets_remain--;
+ }
+
+ if (octets_remain < 0) {
+ /* Absurdic l tag value, but we still need to rewind the t pointer back */
+ while (t > buf && octets_remain < 0) {
+ t--;
+ octets_remain++;
+ }
+
+ ret = FALSE;
+ }
+
+ *start = h;
+
+ if (t - buf > 0) {
+ gsize cklen = t - buf;
+
+ EVP_DigestUpdate(ck, buf, cklen);
+ ctx->body_canonicalised += cklen;
+ msg_debug_dkim("relaxed update signature with body buffer "
+ "(%z size, %z -> %z remain)",
+ cklen, *remain, octets_remain);
+ *remain = octets_remain;
+ }
+
+ return ret && ((len > 0) && (octets_remain > 0));
+}
+
+static gboolean
+rspamd_dkim_simple_body_step(struct rspamd_dkim_common_ctx *ctx,
+ EVP_MD_CTX *ck, const gchar **start, guint size,
+ gssize *remain)
+{
+ const gchar *h;
+ gchar *t;
+ guint len, inlen;
+ gssize octets_remain;
+ gchar buf[1024];
+
+ len = size;
+ inlen = sizeof(buf) - 1;
+ h = *start;
+ t = &buf[0];
+ octets_remain = *remain;
+
+ while (len > 0 && inlen > 0 && (octets_remain != 0)) {
+ if (*h == '\r' || *h == '\n') {
+ *t++ = '\r';
+ *t++ = '\n';
+
+ if (len > 1 && (*h == '\r' && h[1] == '\n')) {
+ h += 2;
+ len -= 2;
+
+ if (octets_remain >= 2) {
+ octets_remain -= 2; /* Input has just \n or \r so we actually add more octets */
+ }
+ else {
+ octets_remain--;
+ }
+ }
+ else {
+ h++;
+ len--;
+
+ if (octets_remain >= 2) {
+ octets_remain -= 2; /* Input has just \n or \r so we actually add more octets */
+ }
+ else {
+ octets_remain--;
+ }
+ }
+ break;
+ }
+
+ *t++ = *h++;
+ octets_remain--;
+ inlen--;
+ len--;
+ }
+
+ *start = h;
+
+ if (t - buf > 0) {
+ gsize cklen = t - buf;
+
+ EVP_DigestUpdate(ck, buf, cklen);
+ ctx->body_canonicalised += cklen;
+ msg_debug_dkim("simple update signature with body buffer "
+ "(%z size, %z -> %z remain)",
+ cklen, *remain, octets_remain);
+ *remain = octets_remain;
+ }
+
+ return ((len != 0) && (octets_remain != 0));
+}
+
+static const gchar *
+rspamd_dkim_skip_empty_lines(const gchar *start, const gchar *end,
+ guint type, gboolean sign, gboolean *need_crlf)
+{
+ const gchar *p = end - 1, *t;
+ enum {
+ init = 0,
+ init_2,
+ got_cr,
+ got_lf,
+ got_crlf,
+ test_spaces,
+ } state = init;
+ guint skip = 0;
+
+ while (p >= start) {
+ switch (state) {
+ case init:
+ if (*p == '\r') {
+ state = got_cr;
+ }
+ else if (*p == '\n') {
+ state = got_lf;
+ }
+ else if (type == DKIM_CANON_RELAXED && *p == ' ') {
+ skip = 0;
+ state = test_spaces;
+ }
+ else {
+ if (sign || type != DKIM_CANON_RELAXED) {
+ *need_crlf = TRUE;
+ }
+
+ goto end;
+ }
+ break;
+ case init_2:
+ if (*p == '\r') {
+ state = got_cr;
+ }
+ else if (*p == '\n') {
+ state = got_lf;
+ }
+ else if (type == DKIM_CANON_RELAXED && (*p == ' ' || *p == '\t')) {
+ skip = 0;
+ state = test_spaces;
+ }
+ else {
+ goto end;
+ }
+ break;
+ case got_cr:
+ if (p >= start + 1) {
+ if (*(p - 1) == '\r') {
+ p--;
+ state = got_cr;
+ }
+ else if (*(p - 1) == '\n') {
+ if ((*p - 2) == '\r') {
+ /* \r\n\r -> we know about one line */
+ p -= 1;
+ state = got_crlf;
+ }
+ else {
+ /* \n\r -> we know about one line */
+ p -= 1;
+ state = got_lf;
+ }
+ }
+ else if (type == DKIM_CANON_RELAXED && (*(p - 1) == ' ' ||
+ *(p - 1) == '\t')) {
+ skip = 1;
+ state = test_spaces;
+ }
+ else {
+ goto end;
+ }
+ }
+ else {
+ if (g_ascii_isspace(*(p - 1))) {
+ if (type == DKIM_CANON_RELAXED) {
+ p -= 1;
+ }
+ }
+ goto end;
+ }
+ break;
+ case got_lf:
+ if (p >= start + 1) {
+ if (*(p - 1) == '\r') {
+ state = got_crlf;
+ }
+ else if (*(p - 1) == '\n') {
+ /* We know about one line */
+ p--;
+ state = got_lf;
+ }
+ else if (type == DKIM_CANON_RELAXED && (*(p - 1) == ' ' ||
+ *(p - 1) == '\t')) {
+ skip = 1;
+ state = test_spaces;
+ }
+ else {
+ goto end;
+ }
+ }
+ else {
+ if (g_ascii_isspace(*(p - 1))) {
+ if (type == DKIM_CANON_RELAXED) {
+ p -= 1;
+ }
+ }
+ goto end;
+ }
+ break;
+ case got_crlf:
+ if (p >= start + 2) {
+ if (*(p - 2) == '\r') {
+ p -= 2;
+ state = got_cr;
+ }
+ else if (*(p - 2) == '\n') {
+ p -= 2;
+ state = got_lf;
+ }
+ else if (type == DKIM_CANON_RELAXED && (*(p - 2) == ' ' ||
+ *(p - 2) == '\t')) {
+ skip = 2;
+ state = test_spaces;
+ }
+ else {
+ goto end;
+ }
+ }
+ else {
+ if (g_ascii_isspace(*(p - 2))) {
+ if (type == DKIM_CANON_RELAXED) {
+ p -= 2;
+ }
+ }
+ goto end;
+ }
+ break;
+ case test_spaces:
+ t = p - skip;
+
+ while (t >= start + 2 && (*t == ' ' || *t == '\t')) {
+ t--;
+ }
+
+ if (*t == '\r') {
+ p = t;
+ state = got_cr;
+ }
+ else if (*t == '\n') {
+ p = t;
+ state = got_lf;
+ }
+ else {
+ goto end;
+ }
+ break;
+ }
+ }
+
+end:
+ return p;
+}
+
+static gboolean
+rspamd_dkim_canonize_body(struct rspamd_dkim_common_ctx *ctx,
+ const gchar *start,
+ const gchar *end,
+ gboolean sign)
+{
+ const gchar *p;
+ gssize remain = ctx->len ? ctx->len : G_MAXSSIZE;
+ guint total_len = end - start;
+ gboolean need_crlf = FALSE;
+
+ if (start == NULL) {
+ /* Empty body */
+ if (ctx->body_canon_type == DKIM_CANON_SIMPLE) {
+ EVP_DigestUpdate(ctx->body_hash, CRLF, sizeof(CRLF) - 1);
+ ctx->body_canonicalised += sizeof(CRLF) - 1;
+ }
+ else {
+ EVP_DigestUpdate(ctx->body_hash, "", 0);
+ }
+ }
+ else {
+ /* Strip extra ending CRLF */
+ p = rspamd_dkim_skip_empty_lines(start, end, ctx->body_canon_type,
+ sign, &need_crlf);
+ end = p + 1;
+
+ if (end == start) {
+ /* Empty body */
+ if (ctx->body_canon_type == DKIM_CANON_SIMPLE) {
+ EVP_DigestUpdate(ctx->body_hash, CRLF, sizeof(CRLF) - 1);
+ ctx->body_canonicalised += sizeof(CRLF) - 1;
+ }
+ else {
+ EVP_DigestUpdate(ctx->body_hash, "", 0);
+ }
+ }
+ else {
+ if (ctx->body_canon_type == DKIM_CANON_SIMPLE) {
+ /* Simple canonization */
+ while (rspamd_dkim_simple_body_step(ctx, ctx->body_hash,
+ &start, end - start, &remain))
+ ;
+
+ /*
+ * If we have l= tag then we cannot add crlf...
+ */
+ if (need_crlf) {
+ /* l is evil... */
+ if (ctx->len == 0) {
+ remain = 2;
+ }
+ else {
+ if (ctx->len <= total_len) {
+ /* We don't have enough l to add \r\n */
+ remain = 0;
+ }
+ else {
+ if (ctx->len - total_len >= 2) {
+ remain = 2;
+ }
+ else {
+ remain = ctx->len - total_len;
+ }
+ }
+ }
+
+ start = "\r\n";
+ end = start + 2;
+
+ rspamd_dkim_simple_body_step(ctx, ctx->body_hash,
+ &start, end - start, &remain);
+ }
+ }
+ else {
+ while (rspamd_dkim_relaxed_body_step(ctx, ctx->body_hash,
+ &start, end - start, &remain))
+ ;
+ if (need_crlf) {
+ start = "\r\n";
+ end = start + 2;
+ remain = 2;
+ rspamd_dkim_relaxed_body_step(ctx, ctx->body_hash,
+ &start, end - start, &remain);
+ }
+ }
+ }
+ return TRUE;
+ }
+
+ /* TODO: Implement relaxed algorithm */
+ return FALSE;
+}
+
+/* Update hash converting all CR and LF to CRLF */
+static void
+rspamd_dkim_hash_update(EVP_MD_CTX *ck, const gchar *begin, gsize len)
+{
+ const gchar *p, *c, *end;
+
+ end = begin + len;
+ p = begin;
+ c = p;
+
+ while (p < end) {
+ if (*p == '\r') {
+ EVP_DigestUpdate(ck, c, p - c);
+ EVP_DigestUpdate(ck, CRLF, sizeof(CRLF) - 1);
+ p++;
+
+ if (p < end && *p == '\n') {
+ p++;
+ }
+ c = p;
+ }
+ else if (*p == '\n') {
+ EVP_DigestUpdate(ck, c, p - c);
+ EVP_DigestUpdate(ck, CRLF, sizeof(CRLF) - 1);
+ p++;
+ c = p;
+ }
+ else {
+ p++;
+ }
+ }
+
+ if (p > c) {
+ EVP_DigestUpdate(ck, c, p - c);
+ }
+}
+
+/* Update hash by signature value (ignoring b= tag) */
+static void
+rspamd_dkim_signature_update(struct rspamd_dkim_common_ctx *ctx,
+ const gchar *begin,
+ guint len)
+{
+ const gchar *p, *c, *end;
+ gboolean tag, skip;
+
+ end = begin + len;
+ p = begin;
+ c = begin;
+ tag = TRUE;
+ skip = FALSE;
+
+ while (p < end) {
+ if (tag && p[0] == 'b' && p[1] == '=') {
+ /* Add to signature */
+ msg_debug_dkim("initial update hash with signature part: %*s",
+ (gint) (p - c + 2),
+ c);
+ ctx->headers_canonicalised += p - c + 2;
+ rspamd_dkim_hash_update(ctx->headers_hash, c, p - c + 2);
+ skip = TRUE;
+ }
+ else if (skip && (*p == ';' || p == end - 1)) {
+ skip = FALSE;
+ c = p;
+ }
+ else if (!tag && *p == ';') {
+ tag = TRUE;
+ }
+ else if (tag && *p == '=') {
+ tag = FALSE;
+ }
+ p++;
+ }
+
+ p--;
+ /* Skip \r\n at the end */
+ while ((*p == '\r' || *p == '\n') && p >= c) {
+ p--;
+ }
+
+ if (p - c + 1 > 0) {
+ msg_debug_dkim("final update hash with signature part: %*s",
+ (gint) (p - c + 1), c);
+ ctx->headers_canonicalised += p - c + 1;
+ rspamd_dkim_hash_update(ctx->headers_hash, c, p - c + 1);
+ }
+}
+
+goffset
+rspamd_dkim_canonize_header_relaxed_str(const gchar *hname,
+ const gchar *hvalue,
+ gchar *out,
+ gsize outlen)
+{
+ gchar *t;
+ const guchar *h;
+ gboolean got_sp;
+
+ /* Name part */
+ t = out;
+ h = hname;
+
+ while (*h && t - out < outlen) {
+ *t++ = lc_map[*h++];
+ }
+
+ if (t - out >= outlen) {
+ return -1;
+ }
+
+ *t++ = ':';
+
+ /* Value part */
+ h = hvalue;
+ /* Skip spaces at the beginning */
+ while (g_ascii_isspace(*h)) {
+ h++;
+ }
+
+ got_sp = FALSE;
+
+ while (*h && (t - out < outlen)) {
+ if (g_ascii_isspace(*h)) {
+ if (got_sp) {
+ h++;
+ continue;
+ }
+ else {
+ got_sp = TRUE;
+ *t++ = ' ';
+ h++;
+ continue;
+ }
+ }
+ else {
+ got_sp = FALSE;
+ }
+
+ *t++ = *h++;
+ }
+
+ if (g_ascii_isspace(*(t - 1))) {
+ t--;
+ }
+
+ if (t - out >= outlen - 2) {
+ return -1;
+ }
+
+ *t++ = '\r';
+ *t++ = '\n';
+ *t = '\0';
+
+ return t - out;
+}
+
+static gboolean
+rspamd_dkim_canonize_header_relaxed(struct rspamd_dkim_common_ctx *ctx,
+ const gchar *header,
+ const gchar *header_name,
+ gboolean is_sign,
+ guint count,
+ bool is_seal)
+{
+ static gchar st_buf[8192];
+ gchar *buf;
+ guint inlen;
+ goffset r;
+ gboolean allocated = FALSE;
+
+ inlen = strlen(header) + strlen(header_name) + sizeof(":" CRLF);
+
+ if (inlen > sizeof(st_buf)) {
+ buf = g_malloc(inlen);
+ allocated = TRUE;
+ }
+ else {
+ /* Faster */
+ buf = st_buf;
+ }
+
+ r = rspamd_dkim_canonize_header_relaxed_str(header_name, header, buf, inlen);
+
+ g_assert(r != -1);
+
+ if (!is_sign) {
+ msg_debug_dkim("update %s with header (idx=%d): %s",
+ is_seal ? "seal" : "signature", count, buf);
+ EVP_DigestUpdate(ctx->headers_hash, buf, r);
+ }
+ else {
+ rspamd_dkim_signature_update(ctx, buf, r);
+ }
+
+ if (allocated) {
+ g_free(buf);
+ }
+
+ return TRUE;
+}
+
+
+static gboolean
+rspamd_dkim_canonize_header(struct rspamd_dkim_common_ctx *ctx,
+ struct rspamd_task *task,
+ const gchar *header_name,
+ gint count,
+ const gchar *dkim_header,
+ const gchar *dkim_domain)
+{
+ struct rspamd_mime_header *rh, *cur, *sel = NULL;
+ gint hdr_cnt = 0;
+ bool use_idx = false, is_sign = ctx->is_sign;
+
+ /*
+ * TODO:
+ * Temporary hack to prevent linked list being misused until refactored
+ */
+ const guint max_list_iters = 1000;
+
+ if (count < 0) {
+ use_idx = true;
+ count = -(count); /* use i= in header content as it is arc stuff */
+ }
+
+ if (dkim_header == NULL) {
+ rh = rspamd_message_get_header_array(task, header_name,
+ is_sign);
+
+ if (rh) {
+ /* Check uniqueness of the header but we count from the bottom to top */
+ if (!use_idx) {
+ for (cur = rh->prev;; cur = cur->prev) {
+ if (hdr_cnt == count) {
+ sel = cur;
+ }
+
+ hdr_cnt++;
+
+ if (cur == rh || hdr_cnt >= max_list_iters) {
+ /* Cycle */
+ break;
+ }
+ }
+
+ if ((rh->flags & RSPAMD_HEADER_UNIQUE) && hdr_cnt > 1) {
+ guint64 random_cookie = ottery_rand_uint64();
+
+ msg_warn_dkim("header %s is intended to be unique by"
+ " email standards, but we have %d headers of this"
+ " type, artificially break DKIM check",
+ header_name,
+ hdr_cnt);
+ rspamd_dkim_hash_update(ctx->headers_hash,
+ (const gchar *) &random_cookie,
+ sizeof(random_cookie));
+ ctx->headers_canonicalised += sizeof(random_cookie);
+
+ return FALSE;
+ }
+
+ if (hdr_cnt <= count) {
+ /*
+ * If DKIM has less headers requested than there are in a
+ * message, then it's fine, it allows adding extra headers
+ */
+ return TRUE;
+ }
+ }
+ else {
+ /*
+ * This branch is used for ARC headers, and it orders them based on
+ * i=<number> string and not their real order in the list of headers
+ */
+ gchar idx_buf[16];
+ gint id_len, i;
+
+ id_len = rspamd_snprintf(idx_buf, sizeof(idx_buf), "i=%d;",
+ count);
+
+ for (cur = rh->prev, i = 0; i < max_list_iters; cur = cur->prev, i++) {
+ if (cur->decoded &&
+ rspamd_substring_search(cur->decoded, strlen(cur->decoded),
+ idx_buf, id_len) != -1) {
+ sel = cur;
+ break;
+ }
+
+ if (cur == rh) {
+ /* Cycle */
+ break;
+ }
+ }
+
+ if (sel == NULL) {
+ return FALSE;
+ }
+ }
+
+ /* Selected header must be non-null if previous condition is false */
+ g_assert(sel != NULL);
+
+ if (ctx->header_canon_type == DKIM_CANON_SIMPLE) {
+ rspamd_dkim_hash_update(ctx->headers_hash, sel->raw_value,
+ sel->raw_len);
+ ctx->headers_canonicalised += sel->raw_len;
+ msg_debug_dkim("update %s with header (idx=%d): %*s",
+ (use_idx ? "seal" : "signature"),
+ count, (gint) sel->raw_len, sel->raw_value);
+ }
+ else {
+ if (is_sign && (sel->flags & RSPAMD_HEADER_FROM)) {
+ /* Special handling of the From handling when rewrite is done */
+ gboolean has_rewrite = FALSE;
+ guint i;
+ struct rspamd_email_address *addr;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, from_mime), i, addr)
+ {
+ if ((addr->flags & RSPAMD_EMAIL_ADDR_ORIGINAL) && !(addr->flags & RSPAMD_EMAIL_ADDR_ALIASED)) {
+ has_rewrite = TRUE;
+ }
+ }
+
+ if (has_rewrite) {
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, from_mime), i, addr)
+ {
+ if (!(addr->flags & RSPAMD_EMAIL_ADDR_ORIGINAL)) {
+ if (!rspamd_dkim_canonize_header_relaxed(ctx, addr->raw,
+ header_name, FALSE, i, use_idx)) {
+ return FALSE;
+ }
+
+ return TRUE;
+ }
+ }
+ }
+ }
+
+ if (!rspamd_dkim_canonize_header_relaxed(ctx, sel->value,
+ header_name, FALSE, count, use_idx)) {
+ return FALSE;
+ }
+ }
+ }
+ }
+ else {
+ /* For signature check just use the saved dkim header */
+ if (ctx->header_canon_type == DKIM_CANON_SIMPLE) {
+ /* We need to find our own signature and use it */
+ rh = rspamd_message_get_header_array(task, header_name, is_sign);
+
+ if (rh) {
+ /* We need to find our own signature */
+ if (!dkim_domain) {
+ msg_err_dkim("cannot verify dkim as we have no dkim domain!");
+ return FALSE;
+ }
+
+ gboolean found = FALSE;
+
+ DL_FOREACH(rh, cur)
+ {
+ guint64 th = rspamd_cryptobox_fast_hash(cur->decoded,
+ strlen(cur->decoded), rspamd_hash_seed());
+
+ if (th == ctx->sig_hash) {
+ rspamd_dkim_signature_update(ctx, cur->raw_value,
+ cur->raw_len);
+ found = TRUE;
+ break;
+ }
+ }
+ if (!found) {
+ msg_err_dkim("BUGON: cannot verify dkim as we have lost our signature"
+ " during simple canonicalisation, expected hash=%L",
+ ctx->sig_hash);
+ return FALSE;
+ }
+ }
+ else {
+ return FALSE;
+ }
+ }
+ else {
+ if (!rspamd_dkim_canonize_header_relaxed(ctx,
+ dkim_header,
+ header_name,
+ TRUE, 0, use_idx)) {
+ return FALSE;
+ }
+ }
+ }
+
+ return TRUE;
+}
+
+struct rspamd_dkim_cached_hash {
+ guchar *digest_normal;
+ guchar *digest_cr;
+ guchar *digest_crlf;
+ gchar *type;
+};
+
+static struct rspamd_dkim_cached_hash *
+rspamd_dkim_check_bh_cached(struct rspamd_dkim_common_ctx *ctx,
+ struct rspamd_task *task, gsize bhlen, gboolean is_sign)
+{
+ gchar typebuf[64];
+ struct rspamd_dkim_cached_hash *res;
+
+ rspamd_snprintf(typebuf, sizeof(typebuf),
+ RSPAMD_MEMPOOL_DKIM_BH_CACHE "%z_%s_%d_%z",
+ bhlen,
+ ctx->body_canon_type == DKIM_CANON_RELAXED ? "1" : "0",
+ !!is_sign,
+ ctx->len);
+
+ res = rspamd_mempool_get_variable(task->task_pool,
+ typebuf);
+
+ if (!res) {
+ res = rspamd_mempool_alloc0(task->task_pool, sizeof(*res));
+ res->type = rspamd_mempool_strdup(task->task_pool, typebuf);
+ rspamd_mempool_set_variable(task->task_pool,
+ res->type, res, NULL);
+ }
+
+ return res;
+}
+
+static const char *
+rspamd_dkim_type_to_string(enum rspamd_dkim_type t)
+{
+ switch (t) {
+ case RSPAMD_DKIM_NORMAL:
+ return "dkim";
+ case RSPAMD_DKIM_ARC_SIG:
+ return "arc_sig";
+ case RSPAMD_DKIM_ARC_SEAL:
+ default:
+ return "arc_seal";
+ }
+}
+
+/**
+ * Check task for dkim context using dkim key
+ * @param ctx dkim verify context
+ * @param key dkim key (from cache or from dns request)
+ * @param task task to check
+ * @return
+ */
+struct rspamd_dkim_check_result *
+rspamd_dkim_check(rspamd_dkim_context_t *ctx,
+ rspamd_dkim_key_t *key,
+ struct rspamd_task *task)
+{
+ const gchar *body_end, *body_start;
+ guchar raw_digest[EVP_MAX_MD_SIZE];
+ struct rspamd_dkim_cached_hash *cached_bh = NULL;
+ EVP_MD_CTX *cpy_ctx = NULL;
+ gsize dlen = 0;
+ struct rspamd_dkim_check_result *res;
+ guint i;
+ struct rspamd_dkim_header *dh;
+ gint nid;
+
+ g_return_val_if_fail(ctx != NULL, NULL);
+ g_return_val_if_fail(key != NULL, NULL);
+ g_return_val_if_fail(task->msg.len > 0, NULL);
+
+ /* First of all find place of body */
+ body_end = task->msg.begin + task->msg.len;
+
+ body_start = MESSAGE_FIELD(task, raw_headers_content).body_start;
+
+ res = rspamd_mempool_alloc0(task->task_pool, sizeof(*res));
+ res->ctx = ctx;
+ res->selector = ctx->selector;
+ res->domain = ctx->domain;
+ res->fail_reason = NULL;
+ res->short_b = ctx->short_b;
+ res->rcode = DKIM_CONTINUE;
+
+ if (!body_start) {
+ res->rcode = DKIM_ERROR;
+ return res;
+ }
+
+ if (ctx->common.type != RSPAMD_DKIM_ARC_SEAL) {
+ dlen = EVP_MD_CTX_size(ctx->common.body_hash);
+ cached_bh = rspamd_dkim_check_bh_cached(&ctx->common, task,
+ dlen, FALSE);
+
+ if (!cached_bh->digest_normal) {
+ /* Start canonization of body part */
+ if (!rspamd_dkim_canonize_body(&ctx->common, body_start, body_end,
+ FALSE)) {
+ res->rcode = DKIM_RECORD_ERROR;
+ return res;
+ }
+ }
+ }
+
+ /* Now canonize headers */
+ for (i = 0; i < ctx->common.hlist->len; i++) {
+ dh = g_ptr_array_index(ctx->common.hlist, i);
+ rspamd_dkim_canonize_header(&ctx->common, task, dh->name, dh->count,
+ NULL, NULL);
+ }
+
+ /* Canonize dkim signature */
+ switch (ctx->common.type) {
+ case RSPAMD_DKIM_NORMAL:
+ rspamd_dkim_canonize_header(&ctx->common, task, RSPAMD_DKIM_SIGNHEADER, 0,
+ ctx->dkim_header, ctx->domain);
+ break;
+ case RSPAMD_DKIM_ARC_SIG:
+ rspamd_dkim_canonize_header(&ctx->common, task, RSPAMD_DKIM_ARC_SIGNHEADER, 0,
+ ctx->dkim_header, ctx->domain);
+ break;
+ case RSPAMD_DKIM_ARC_SEAL:
+ rspamd_dkim_canonize_header(&ctx->common, task, RSPAMD_DKIM_ARC_SEALHEADER, 0,
+ ctx->dkim_header, ctx->domain);
+ break;
+ }
+
+
+ /* Use cached BH for all but arc seal, if it is not NULL we are not in arc seal mode */
+ if (cached_bh != NULL) {
+ if (!cached_bh->digest_normal) {
+ /* Copy md_ctx to deal with broken CRLF at the end */
+ cpy_ctx = EVP_MD_CTX_create();
+ EVP_MD_CTX_copy(cpy_ctx, ctx->common.body_hash);
+ EVP_DigestFinal_ex(cpy_ctx, raw_digest, NULL);
+
+ cached_bh->digest_normal = rspamd_mempool_alloc(task->task_pool,
+ sizeof(raw_digest));
+ memcpy(cached_bh->digest_normal, raw_digest, sizeof(raw_digest));
+ }
+
+ /* Check bh field */
+ if (memcmp(ctx->bh, cached_bh->digest_normal, ctx->bhlen) != 0) {
+ msg_debug_dkim(
+ "bh value mismatch: %*xs versus %*xs, try add LF; try adding CRLF",
+ (gint) dlen, ctx->bh,
+ (gint) dlen, raw_digest);
+
+ if (cpy_ctx) {
+ /* Try add CRLF */
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+ EVP_MD_CTX_cleanup(cpy_ctx);
+#else
+ EVP_MD_CTX_reset(cpy_ctx);
+#endif
+ EVP_MD_CTX_copy(cpy_ctx, ctx->common.body_hash);
+ EVP_DigestUpdate(cpy_ctx, "\r\n", 2);
+ EVP_DigestFinal_ex(cpy_ctx, raw_digest, NULL);
+ cached_bh->digest_crlf = rspamd_mempool_alloc(task->task_pool,
+ sizeof(raw_digest));
+ memcpy(cached_bh->digest_crlf, raw_digest, sizeof(raw_digest));
+
+ if (memcmp(ctx->bh, raw_digest, ctx->bhlen) != 0) {
+ msg_debug_dkim(
+ "bh value mismatch after added CRLF: %*xs versus %*xs, try add LF",
+ (gint) dlen, ctx->bh,
+ (gint) dlen, raw_digest);
+
+ /* Try add LF */
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+ EVP_MD_CTX_cleanup(cpy_ctx);
+#else
+ EVP_MD_CTX_reset(cpy_ctx);
+#endif
+ EVP_MD_CTX_copy(cpy_ctx, ctx->common.body_hash);
+ EVP_DigestUpdate(cpy_ctx, "\n", 1);
+ EVP_DigestFinal_ex(cpy_ctx, raw_digest, NULL);
+ cached_bh->digest_cr = rspamd_mempool_alloc(task->task_pool,
+ sizeof(raw_digest));
+ memcpy(cached_bh->digest_cr, raw_digest, sizeof(raw_digest));
+
+ if (memcmp(ctx->bh, raw_digest, ctx->bhlen) != 0) {
+ msg_debug_dkim("bh value mismatch after added LF: %*xs versus %*xs",
+ (gint) dlen, ctx->bh,
+ (gint) dlen, raw_digest);
+ res->fail_reason = "body hash did not verify";
+ res->rcode = DKIM_REJECT;
+ }
+ }
+ }
+ else if (cached_bh->digest_crlf) {
+ if (memcmp(ctx->bh, cached_bh->digest_crlf, ctx->bhlen) != 0) {
+ msg_debug_dkim("bh value mismatch after added CRLF: %*xs versus %*xs",
+ (gint) dlen, ctx->bh,
+ (gint) dlen, cached_bh->digest_crlf);
+
+ if (cached_bh->digest_cr) {
+ if (memcmp(ctx->bh, cached_bh->digest_cr, ctx->bhlen) != 0) {
+ msg_debug_dkim(
+ "bh value mismatch after added LF: %*xs versus %*xs",
+ (gint) dlen, ctx->bh,
+ (gint) dlen, cached_bh->digest_cr);
+
+ res->fail_reason = "body hash did not verify";
+ res->rcode = DKIM_REJECT;
+ }
+ }
+ else {
+
+ res->fail_reason = "body hash did not verify";
+ res->rcode = DKIM_REJECT;
+ }
+ }
+ }
+ else {
+ msg_debug_dkim(
+ "bh value mismatch: %*xs versus %*xs",
+ (gint) dlen, ctx->bh,
+ (gint) dlen, cached_bh->digest_normal);
+ res->fail_reason = "body hash did not verify";
+ res->rcode = DKIM_REJECT;
+ }
+ }
+
+ if (cpy_ctx) {
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+ EVP_MD_CTX_cleanup(cpy_ctx);
+#else
+ EVP_MD_CTX_reset(cpy_ctx);
+#endif
+ EVP_MD_CTX_destroy(cpy_ctx);
+ }
+
+ if (res->rcode == DKIM_REJECT) {
+ msg_info_dkim(
+ "%s: bh value mismatch: got %*Bs, expected %*Bs; "
+ "body length %d->%d; d=%s; s=%s",
+ rspamd_dkim_type_to_string(ctx->common.type),
+ (gint) dlen, cached_bh->digest_normal,
+ (gint) dlen, ctx->bh,
+ (gint) (body_end - body_start), ctx->common.body_canonicalised,
+ ctx->domain, ctx->selector);
+
+ return res;
+ }
+ }
+
+ dlen = EVP_MD_CTX_size(ctx->common.headers_hash);
+ EVP_DigestFinal_ex(ctx->common.headers_hash, raw_digest, NULL);
+ /* Check headers signature */
+
+ if (ctx->sig_alg == DKIM_SIGN_RSASHA1) {
+ nid = NID_sha1;
+ }
+ else if (ctx->sig_alg == DKIM_SIGN_RSASHA256 ||
+ ctx->sig_alg == DKIM_SIGN_ECDSASHA256 ||
+ ctx->sig_alg == DKIM_SIGN_EDDSASHA256) {
+ nid = NID_sha256;
+ }
+ else if (ctx->sig_alg == DKIM_SIGN_RSASHA512 ||
+ ctx->sig_alg == DKIM_SIGN_ECDSASHA512) {
+ nid = NID_sha512;
+ }
+ else {
+ /* Not reached */
+ nid = NID_sha1;
+ }
+
+ switch (key->type) {
+ case RSPAMD_DKIM_KEY_RSA:
+ if (RSA_verify(nid, raw_digest, dlen, ctx->b, ctx->blen,
+ key->key.key_rsa) != 1) {
+ msg_debug_dkim("headers rsa verify failed");
+ ERR_clear_error();
+ res->rcode = DKIM_REJECT;
+ res->fail_reason = "headers rsa verify failed";
+
+ msg_info_dkim(
+ "%s: headers RSA verification failure; "
+ "body length %d->%d; headers length %d; d=%s; s=%s; key_md5=%*xs; orig header: %s",
+ rspamd_dkim_type_to_string(ctx->common.type),
+ (gint) (body_end - body_start), ctx->common.body_canonicalised,
+ ctx->common.headers_canonicalised,
+ ctx->domain, ctx->selector,
+ RSPAMD_DKIM_KEY_ID_LEN, rspamd_dkim_key_id(key),
+ ctx->dkim_header);
+ }
+ break;
+ case RSPAMD_DKIM_KEY_ECDSA:
+ if (ECDSA_verify(nid, raw_digest, dlen, ctx->b, ctx->blen,
+ key->key.key_ecdsa) != 1) {
+ msg_info_dkim(
+ "%s: headers ECDSA verification failure; "
+ "body length %d->%d; headers length %d; d=%s; s=%s; key_md5=%*xs; orig header: %s",
+ rspamd_dkim_type_to_string(ctx->common.type),
+ (gint) (body_end - body_start), ctx->common.body_canonicalised,
+ ctx->common.headers_canonicalised,
+ ctx->domain, ctx->selector,
+ RSPAMD_DKIM_KEY_ID_LEN, rspamd_dkim_key_id(key),
+ ctx->dkim_header);
+ msg_debug_dkim("headers ecdsa verify failed");
+ ERR_clear_error();
+ res->rcode = DKIM_REJECT;
+ res->fail_reason = "headers ecdsa verify failed";
+ }
+ break;
+ case RSPAMD_DKIM_KEY_EDDSA:
+ if (!rspamd_cryptobox_verify(ctx->b, ctx->blen, raw_digest, dlen,
+ key->key.key_eddsa, RSPAMD_CRYPTOBOX_MODE_25519)) {
+ msg_info_dkim(
+ "%s: headers EDDSA verification failure; "
+ "body length %d->%d; headers length %d; d=%s; s=%s; key_md5=%*xs; orig header: %s",
+ rspamd_dkim_type_to_string(ctx->common.type),
+ (gint) (body_end - body_start), ctx->common.body_canonicalised,
+ ctx->common.headers_canonicalised,
+ ctx->domain, ctx->selector,
+ RSPAMD_DKIM_KEY_ID_LEN, rspamd_dkim_key_id(key),
+ ctx->dkim_header);
+ msg_debug_dkim("headers eddsa verify failed");
+ res->rcode = DKIM_REJECT;
+ res->fail_reason = "headers eddsa verify failed";
+ }
+ break;
+ }
+
+
+ if (ctx->common.type == RSPAMD_DKIM_ARC_SEAL && res->rcode == DKIM_CONTINUE) {
+ switch (ctx->cv) {
+ case RSPAMD_ARC_INVALID:
+ msg_info_dkim("arc seal is invalid i=%d", ctx->common.idx);
+ res->rcode = DKIM_PERM_ERROR;
+ res->fail_reason = "arc seal is invalid";
+ break;
+ case RSPAMD_ARC_FAIL:
+ msg_info_dkim("arc seal failed i=%d", ctx->common.idx);
+ res->rcode = DKIM_REJECT;
+ res->fail_reason = "arc seal failed";
+ break;
+ default:
+ break;
+ }
+ }
+
+ return res;
+}
+
+struct rspamd_dkim_check_result *
+rspamd_dkim_create_result(rspamd_dkim_context_t *ctx,
+ enum rspamd_dkim_check_rcode rcode,
+ struct rspamd_task *task)
+{
+ struct rspamd_dkim_check_result *res;
+
+ res = rspamd_mempool_alloc0(task->task_pool, sizeof(*res));
+ res->ctx = ctx;
+ res->selector = ctx->selector;
+ res->domain = ctx->domain;
+ res->fail_reason = NULL;
+ res->short_b = ctx->short_b;
+ res->rcode = rcode;
+
+ return res;
+}
+
+rspamd_dkim_key_t *
+rspamd_dkim_key_ref(rspamd_dkim_key_t *k)
+{
+ REF_RETAIN(k);
+
+ return k;
+}
+
+void rspamd_dkim_key_unref(rspamd_dkim_key_t *k)
+{
+ REF_RELEASE(k);
+}
+
+rspamd_dkim_sign_key_t *
+rspamd_dkim_sign_key_ref(rspamd_dkim_sign_key_t *k)
+{
+ REF_RETAIN(k);
+
+ return k;
+}
+
+void rspamd_dkim_sign_key_unref(rspamd_dkim_sign_key_t *k)
+{
+ REF_RELEASE(k);
+}
+
+const gchar *
+rspamd_dkim_get_domain(rspamd_dkim_context_t *ctx)
+{
+ if (ctx) {
+ return ctx->domain;
+ }
+
+ return NULL;
+}
+
+const gchar *
+rspamd_dkim_get_selector(rspamd_dkim_context_t *ctx)
+{
+ if (ctx) {
+ return ctx->selector;
+ }
+
+ return NULL;
+}
+
+guint rspamd_dkim_key_get_ttl(rspamd_dkim_key_t *k)
+{
+ if (k) {
+ return k->ttl;
+ }
+
+ return 0;
+}
+
+const gchar *
+rspamd_dkim_get_dns_key(rspamd_dkim_context_t *ctx)
+{
+ if (ctx) {
+ return ctx->dns_key;
+ }
+
+ return NULL;
+}
+
+#define PEM_SIG "-----BEGIN"
+
+rspamd_dkim_sign_key_t *
+rspamd_dkim_sign_key_load(const gchar *key, gsize len,
+ enum rspamd_dkim_key_format type,
+ GError **err)
+{
+ guchar *map = NULL, *tmp = NULL;
+ gsize maplen;
+ rspamd_dkim_sign_key_t *nkey;
+ time_t mtime = time(NULL);
+
+ if (type < 0 || type > RSPAMD_DKIM_KEY_UNKNOWN || len == 0 || key == NULL) {
+ g_set_error(err, dkim_error_quark(), DKIM_SIGERROR_KEYFAIL,
+ "invalid key type to load: %d", type);
+ return NULL;
+ }
+
+ nkey = g_malloc0(sizeof(*nkey));
+ nkey->mtime = mtime;
+
+ msg_debug_dkim_taskless("got public key with length %z and type %d",
+ len, type);
+
+ /* Load key file if needed */
+ if (type == RSPAMD_DKIM_KEY_FILE) {
+ struct stat st;
+
+ if (stat(key, &st) != 0) {
+ g_set_error(err, dkim_error_quark(), DKIM_SIGERROR_KEYFAIL,
+ "cannot stat key file: '%s' %s", key, strerror(errno));
+ g_free(nkey);
+
+ return NULL;
+ }
+
+ nkey->mtime = st.st_mtime;
+ map = rspamd_file_xmap(key, PROT_READ, &maplen, TRUE);
+
+ if (map == NULL) {
+ g_set_error(err, dkim_error_quark(), DKIM_SIGERROR_KEYFAIL,
+ "cannot map key file: '%s' %s", key, strerror(errno));
+ g_free(nkey);
+
+ return NULL;
+ }
+
+ key = map;
+ len = maplen;
+
+ if (maplen > sizeof(PEM_SIG) &&
+ strncmp(map, PEM_SIG, sizeof(PEM_SIG) - 1) == 0) {
+ type = RSPAMD_DKIM_KEY_PEM;
+ }
+ else if (rspamd_cryptobox_base64_is_valid(map, maplen)) {
+ type = RSPAMD_DKIM_KEY_BASE64;
+ }
+ else {
+ type = RSPAMD_DKIM_KEY_RAW;
+ }
+ }
+
+ if (type == RSPAMD_DKIM_KEY_UNKNOWN) {
+ if (len > sizeof(PEM_SIG) &&
+ memcmp(key, PEM_SIG, sizeof(PEM_SIG) - 1) == 0) {
+ type = RSPAMD_DKIM_KEY_PEM;
+ }
+ else {
+ type = RSPAMD_DKIM_KEY_RAW;
+ }
+ }
+
+ if (type == RSPAMD_DKIM_KEY_BASE64) {
+ type = RSPAMD_DKIM_KEY_RAW;
+ tmp = g_malloc(len);
+ rspamd_cryptobox_base64_decode(key, len, tmp, &len);
+ key = tmp;
+ }
+
+ if (type == RSPAMD_DKIM_KEY_RAW && (len == 32 ||
+ len == rspamd_cryptobox_sk_sig_bytes(RSPAMD_CRYPTOBOX_MODE_25519))) {
+ if (len == 32) {
+ /* Seeded key, need scalarmult */
+ unsigned char pk[32];
+ nkey->type = RSPAMD_DKIM_KEY_EDDSA;
+ nkey->key.key_eddsa = g_malloc(
+ rspamd_cryptobox_sk_sig_bytes(RSPAMD_CRYPTOBOX_MODE_25519));
+ crypto_sign_ed25519_seed_keypair(pk, nkey->key.key_eddsa, key);
+ nkey->keylen = rspamd_cryptobox_sk_sig_bytes(RSPAMD_CRYPTOBOX_MODE_25519);
+ }
+ else {
+ /* Full ed25519 key */
+ unsigned klen = rspamd_cryptobox_sk_sig_bytes(RSPAMD_CRYPTOBOX_MODE_25519);
+ nkey->type = RSPAMD_DKIM_KEY_EDDSA;
+ nkey->key.key_eddsa = g_malloc(klen);
+ memcpy(nkey->key.key_eddsa, key, klen);
+ nkey->keylen = klen;
+ }
+ }
+ else {
+ nkey->key_bio = BIO_new_mem_buf(key, len);
+
+ if (type == RSPAMD_DKIM_KEY_RAW) {
+ if (d2i_PrivateKey_bio(nkey->key_bio, &nkey->key_evp) == NULL) {
+ g_set_error(err, dkim_error_quark(), DKIM_SIGERROR_KEYFAIL,
+ "cannot parse raw private key: %s",
+ ERR_error_string(ERR_get_error(), NULL));
+
+ rspamd_dkim_sign_key_free(nkey);
+ nkey = NULL;
+
+ goto end;
+ }
+ }
+ else {
+ if (!PEM_read_bio_PrivateKey(nkey->key_bio, &nkey->key_evp, NULL, NULL)) {
+ g_set_error(err, dkim_error_quark(), DKIM_SIGERROR_KEYFAIL,
+ "cannot parse pem private key: %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ rspamd_dkim_sign_key_free(nkey);
+ nkey = NULL;
+
+ goto end;
+ }
+ }
+ nkey->key.key_rsa = EVP_PKEY_get1_RSA(nkey->key_evp);
+ if (nkey->key.key_rsa == NULL) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "cannot extract rsa key from evp key");
+ rspamd_dkim_sign_key_free(nkey);
+ nkey = NULL;
+
+ goto end;
+ }
+ nkey->type = RSPAMD_DKIM_KEY_RSA;
+ }
+
+ REF_INIT_RETAIN(nkey, rspamd_dkim_sign_key_free);
+
+end:
+
+ if (map != NULL) {
+ munmap(map, maplen);
+ }
+
+ if (tmp != NULL) {
+ rspamd_explicit_memzero(tmp, len);
+ g_free(tmp);
+ }
+
+ return nkey;
+}
+
+#undef PEM_SIG
+
+gboolean
+rspamd_dkim_sign_key_maybe_invalidate(rspamd_dkim_sign_key_t *key, time_t mtime)
+{
+ if (mtime > key->mtime) {
+ return TRUE;
+ }
+ return FALSE;
+}
+
+rspamd_dkim_sign_context_t *
+rspamd_create_dkim_sign_context(struct rspamd_task *task,
+ rspamd_dkim_sign_key_t *priv_key,
+ gint headers_canon,
+ gint body_canon,
+ const gchar *headers,
+ enum rspamd_dkim_type type,
+ GError **err)
+{
+ rspamd_dkim_sign_context_t *nctx;
+
+ if (headers_canon != DKIM_CANON_SIMPLE && headers_canon != DKIM_CANON_RELAXED) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_INVALID_HC,
+ "bad headers canonicalisation");
+
+ return NULL;
+ }
+ if (body_canon != DKIM_CANON_SIMPLE && body_canon != DKIM_CANON_RELAXED) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_INVALID_BC,
+ "bad body canonicalisation");
+
+ return NULL;
+ }
+
+ if (!priv_key || (!priv_key->key.key_rsa && !priv_key->key.key_eddsa)) {
+ g_set_error(err,
+ DKIM_ERROR,
+ DKIM_SIGERROR_KEYFAIL,
+ "bad key to sign");
+
+ return NULL;
+ }
+
+ nctx = rspamd_mempool_alloc0(task->task_pool, sizeof(*nctx));
+ nctx->common.pool = task->task_pool;
+ nctx->common.header_canon_type = headers_canon;
+ nctx->common.body_canon_type = body_canon;
+ nctx->common.type = type;
+ nctx->common.is_sign = TRUE;
+
+ if (type != RSPAMD_DKIM_ARC_SEAL) {
+ if (!rspamd_dkim_parse_hdrlist_common(&nctx->common, headers,
+ strlen(headers), TRUE,
+ err)) {
+ return NULL;
+ }
+ }
+ else {
+ rspamd_dkim_add_arc_seal_headers(task->task_pool, &nctx->common);
+ }
+
+ nctx->key = rspamd_dkim_sign_key_ref(priv_key);
+
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) rspamd_dkim_sign_key_unref, priv_key);
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+ nctx->common.body_hash = EVP_MD_CTX_create();
+ EVP_DigestInit_ex(nctx->common.body_hash, EVP_sha256(), NULL);
+ nctx->common.headers_hash = EVP_MD_CTX_create();
+ EVP_DigestInit_ex(nctx->common.headers_hash, EVP_sha256(), NULL);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) EVP_MD_CTX_destroy, nctx->common.body_hash);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) EVP_MD_CTX_destroy, nctx->common.headers_hash);
+#else
+ nctx->common.body_hash = EVP_MD_CTX_new();
+ EVP_DigestInit_ex(nctx->common.body_hash, EVP_sha256(), NULL);
+ nctx->common.headers_hash = EVP_MD_CTX_new();
+ EVP_DigestInit_ex(nctx->common.headers_hash, EVP_sha256(), NULL);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) EVP_MD_CTX_free, nctx->common.body_hash);
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) EVP_MD_CTX_free, nctx->common.headers_hash);
+#endif
+
+ return nctx;
+}
+
+
+GString *
+rspamd_dkim_sign(struct rspamd_task *task, const gchar *selector,
+ const gchar *domain, time_t expire, gsize len, guint idx,
+ const gchar *arc_cv, rspamd_dkim_sign_context_t *ctx)
+{
+ GString *hdr;
+ struct rspamd_dkim_header *dh;
+ const gchar *body_end, *body_start, *hname;
+ guchar raw_digest[EVP_MAX_MD_SIZE];
+ struct rspamd_dkim_cached_hash *cached_bh = NULL;
+ gsize dlen = 0;
+ guint i, j;
+ gchar *b64_data;
+ guchar *sig_buf;
+ guint sig_len;
+ guint headers_len = 0, cur_len = 0;
+ union rspamd_dkim_header_stat hstat;
+
+ g_assert(ctx != NULL);
+
+ /* First of all find place of body */
+ body_end = task->msg.begin + task->msg.len;
+ body_start = MESSAGE_FIELD(task, raw_headers_content).body_start;
+
+ if (len > 0) {
+ ctx->common.len = len;
+ }
+
+ if (!body_start) {
+ return NULL;
+ }
+
+ /* Start canonization of body part */
+ if (ctx->common.type != RSPAMD_DKIM_ARC_SEAL) {
+ dlen = EVP_MD_CTX_size(ctx->common.body_hash);
+ cached_bh = rspamd_dkim_check_bh_cached(&ctx->common, task,
+ dlen, TRUE);
+
+ if (!cached_bh->digest_normal) {
+ /* Start canonization of body part */
+ if (!rspamd_dkim_canonize_body(&ctx->common, body_start, body_end,
+ TRUE)) {
+ return NULL;
+ }
+ }
+ }
+
+ hdr = g_string_sized_new(255);
+
+ if (ctx->common.type == RSPAMD_DKIM_NORMAL) {
+ rspamd_printf_gstring(hdr, "v=1; a=%s; c=%s/%s; d=%s; s=%s; ",
+ ctx->key->type == RSPAMD_DKIM_KEY_RSA ? "rsa-sha256" : "ed25519-sha256",
+ ctx->common.header_canon_type == DKIM_CANON_RELAXED ? "relaxed" : "simple",
+ ctx->common.body_canon_type == DKIM_CANON_RELAXED ? "relaxed" : "simple",
+ domain, selector);
+ }
+ else if (ctx->common.type == RSPAMD_DKIM_ARC_SIG) {
+ rspamd_printf_gstring(hdr, "i=%d; a=%s; c=%s/%s; d=%s; s=%s; ",
+ idx,
+ ctx->key->type == RSPAMD_DKIM_KEY_RSA ? "rsa-sha256" : "ed25519-sha256",
+ ctx->common.header_canon_type == DKIM_CANON_RELAXED ? "relaxed" : "simple",
+ ctx->common.body_canon_type == DKIM_CANON_RELAXED ? "relaxed" : "simple",
+ domain, selector);
+ }
+ else {
+ g_assert(arc_cv != NULL);
+ rspamd_printf_gstring(hdr, "i=%d; a=%s; d=%s; s=%s; cv=%s; ",
+ idx,
+ ctx->key->type == RSPAMD_DKIM_KEY_RSA ? "rsa-sha256" : "ed25519-sha256",
+ domain,
+ selector,
+ arc_cv);
+ }
+
+ if (expire > 0) {
+ rspamd_printf_gstring(hdr, "x=%t; ", expire);
+ }
+
+ if (ctx->common.type != RSPAMD_DKIM_ARC_SEAL) {
+ if (len > 0) {
+ rspamd_printf_gstring(hdr, "l=%z; ", len);
+ }
+ }
+
+ rspamd_printf_gstring(hdr, "t=%t; h=", time(NULL));
+
+ /* Now canonize headers */
+ for (i = 0; i < ctx->common.hlist->len; i++) {
+ struct rspamd_mime_header *rh, *cur;
+
+ dh = g_ptr_array_index(ctx->common.hlist, i);
+
+ /* We allow oversigning if dh->count > number of headers with this name */
+ hstat.n = GPOINTER_TO_UINT(g_hash_table_lookup(ctx->common.htable, dh->name));
+
+ if (hstat.s.flags & RSPAMD_DKIM_FLAG_OVERSIGN) {
+ /* Do oversigning */
+ guint count = 0;
+
+ rh = rspamd_message_get_header_array(task, dh->name, FALSE);
+
+ if (rh) {
+ DL_FOREACH(rh, cur)
+ {
+ /* Sign all existing headers */
+ rspamd_dkim_canonize_header(&ctx->common, task, dh->name,
+ count,
+ NULL, NULL);
+ count++;
+ }
+ }
+
+ /* Now add one more entry to oversign */
+ if (count > 0 || !(hstat.s.flags & RSPAMD_DKIM_FLAG_OVERSIGN_EXISTING)) {
+ cur_len = (strlen(dh->name) + 1) * (count + 1);
+ headers_len += cur_len;
+
+ if (headers_len > 70 && i > 0 && i < ctx->common.hlist->len - 1) {
+ rspamd_printf_gstring(hdr, " ");
+ headers_len = cur_len;
+ }
+
+ for (j = 0; j < count + 1; j++) {
+ rspamd_printf_gstring(hdr, "%s:", dh->name);
+ }
+ }
+ }
+ else {
+ rh = rspamd_message_get_header_array(task, dh->name, FALSE);
+
+ if (rh) {
+ if (hstat.s.count > 0) {
+
+ cur_len = (strlen(dh->name) + 1) * (hstat.s.count);
+ headers_len += cur_len;
+ if (headers_len > 70 && i > 0 && i < ctx->common.hlist->len - 1) {
+ rspamd_printf_gstring(hdr, " ");
+ headers_len = cur_len;
+ }
+
+ for (j = 0; j < hstat.s.count; j++) {
+ rspamd_printf_gstring(hdr, "%s:", dh->name);
+ }
+ }
+
+
+ rspamd_dkim_canonize_header(&ctx->common, task,
+ dh->name, dh->count,
+ NULL, NULL);
+ }
+ }
+
+ g_hash_table_remove(ctx->common.htable, dh->name);
+ }
+
+ /* Replace the last ':' with ';' */
+ hdr->str[hdr->len - 1] = ';';
+
+ if (ctx->common.type != RSPAMD_DKIM_ARC_SEAL) {
+ if (!cached_bh->digest_normal) {
+ EVP_DigestFinal_ex(ctx->common.body_hash, raw_digest, NULL);
+ cached_bh->digest_normal = rspamd_mempool_alloc(task->task_pool,
+ sizeof(raw_digest));
+ memcpy(cached_bh->digest_normal, raw_digest, sizeof(raw_digest));
+ }
+
+
+ b64_data = rspamd_encode_base64(cached_bh->digest_normal, dlen, 0, NULL);
+ rspamd_printf_gstring(hdr, " bh=%s; b=", b64_data);
+ g_free(b64_data);
+ }
+ else {
+ rspamd_printf_gstring(hdr, " b=");
+ }
+
+ switch (ctx->common.type) {
+ case RSPAMD_DKIM_NORMAL:
+ default:
+ hname = RSPAMD_DKIM_SIGNHEADER;
+ break;
+ case RSPAMD_DKIM_ARC_SIG:
+ hname = RSPAMD_DKIM_ARC_SIGNHEADER;
+ break;
+ case RSPAMD_DKIM_ARC_SEAL:
+ hname = RSPAMD_DKIM_ARC_SEALHEADER;
+ break;
+ }
+
+ if (ctx->common.header_canon_type == DKIM_CANON_RELAXED) {
+ if (!rspamd_dkim_canonize_header_relaxed(&ctx->common,
+ hdr->str,
+ hname,
+ TRUE,
+ 0,
+ ctx->common.type == RSPAMD_DKIM_ARC_SEAL)) {
+
+ g_string_free(hdr, TRUE);
+ return NULL;
+ }
+ }
+ else {
+ /* Will likely have issues with folding */
+ rspamd_dkim_hash_update(ctx->common.headers_hash, hdr->str,
+ hdr->len);
+ ctx->common.headers_canonicalised += hdr->len;
+ msg_debug_task("update signature with header: %*s",
+ (gint) hdr->len, hdr->str);
+ }
+
+ dlen = EVP_MD_CTX_size(ctx->common.headers_hash);
+ EVP_DigestFinal_ex(ctx->common.headers_hash, raw_digest, NULL);
+ if (ctx->key->type == RSPAMD_DKIM_KEY_RSA) {
+ sig_len = RSA_size(ctx->key->key.key_rsa);
+ sig_buf = g_alloca(sig_len);
+
+ if (RSA_sign(NID_sha256, raw_digest, dlen, sig_buf, &sig_len,
+ ctx->key->key.key_rsa) != 1) {
+ g_string_free(hdr, TRUE);
+ msg_err_task("rsa sign error: %s",
+ ERR_error_string(ERR_get_error(), NULL));
+
+ return NULL;
+ }
+ }
+ else if (ctx->key->type == RSPAMD_DKIM_KEY_EDDSA) {
+ sig_len = rspamd_cryptobox_signature_bytes(RSPAMD_CRYPTOBOX_MODE_25519);
+ sig_buf = g_alloca(sig_len);
+
+ rspamd_cryptobox_sign(sig_buf, NULL, raw_digest, dlen,
+ ctx->key->key.key_eddsa, RSPAMD_CRYPTOBOX_MODE_25519);
+ }
+ else {
+ g_string_free(hdr, TRUE);
+ msg_err_task("unsupported key type for signing");
+
+ return NULL;
+ }
+
+ if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_MILTER) {
+ b64_data = rspamd_encode_base64_fold(sig_buf, sig_len, 70, NULL,
+ RSPAMD_TASK_NEWLINES_LF);
+ }
+ else {
+ b64_data = rspamd_encode_base64_fold(sig_buf, sig_len, 70, NULL,
+ MESSAGE_FIELD(task, nlines_type));
+ }
+
+ rspamd_printf_gstring(hdr, "%s", b64_data);
+ g_free(b64_data);
+
+ return hdr;
+}
+
+gboolean
+rspamd_dkim_match_keys(rspamd_dkim_key_t *pk,
+ rspamd_dkim_sign_key_t *sk,
+ GError **err)
+{
+ if (pk == NULL || sk == NULL) {
+ g_set_error(err, dkim_error_quark(), DKIM_SIGERROR_KEYFAIL,
+ "missing public or private key");
+ return FALSE;
+ }
+ if (pk->type != sk->type) {
+ g_set_error(err, dkim_error_quark(), DKIM_SIGERROR_KEYFAIL,
+ "public and private key types do not match");
+ return FALSE;
+ }
+
+ if (pk->type == RSPAMD_DKIM_KEY_EDDSA) {
+ if (memcmp(sk->key.key_eddsa + 32, pk->key.key_eddsa, 32) != 0) {
+ g_set_error(err, dkim_error_quark(), DKIM_SIGERROR_KEYHASHMISMATCH,
+ "pubkey does not match private key");
+ return FALSE;
+ }
+ }
+ else if (EVP_PKEY_cmp(pk->key_evp, sk->key_evp) != 1) {
+ g_set_error(err, dkim_error_quark(), DKIM_SIGERROR_KEYHASHMISMATCH,
+ "pubkey does not match private key");
+ return FALSE;
+ }
+
+ return TRUE;
+}
diff --git a/src/libserver/dkim.h b/src/libserver/dkim.h
new file mode 100644
index 0000000..50703da
--- /dev/null
+++ b/src/libserver/dkim.h
@@ -0,0 +1,298 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef DKIM_H_
+#define DKIM_H_
+
+#include "config.h"
+#include "contrib/libev/ev.h"
+#include "dns.h"
+#include "ref.h"
+
+
+/* Main types and definitions */
+
+#define RSPAMD_DKIM_SIGNHEADER "DKIM-Signature"
+#define RSPAMD_DKIM_ARC_SIGNHEADER "ARC-Message-Signature"
+#define RSPAMD_DKIM_ARC_AUTHHEADER "ARC-Authentication-Results"
+#define RSPAMD_DKIM_ARC_SEALHEADER "ARC-Seal"
+/* DKIM signature header */
+
+
+/* Errors (from OpenDKIM) */
+
+#define DKIM_SIGERROR_UNKNOWN (-1) /* unknown error */
+#define DKIM_SIGERROR_VERSION 1 /* unsupported version */
+#define DKIM_SIGERROR_EXPIRED 3 /* signature expired */
+#define DKIM_SIGERROR_FUTURE 4 /* signature in the future */
+#define DKIM_SIGERROR_NOREC 6 /* No record */
+#define DKIM_SIGERROR_INVALID_HC 7 /* c= invalid (header) */
+#define DKIM_SIGERROR_INVALID_BC 8 /* c= invalid (body) */
+#define DKIM_SIGERROR_INVALID_A 10 /* a= invalid */
+#define DKIM_SIGERROR_INVALID_L 12 /* l= invalid */
+#define DKIM_SIGERROR_EMPTY_D 16 /* d= empty */
+#define DKIM_SIGERROR_EMPTY_S 18 /* s= empty */
+#define DKIM_SIGERROR_EMPTY_B 20 /* b= empty */
+#define DKIM_SIGERROR_NOKEY 22 /* no key found in DNS */
+#define DKIM_SIGERROR_KEYFAIL 24 /* DNS query failed */
+#define DKIM_SIGERROR_EMPTY_BH 26 /* bh= empty */
+#define DKIM_SIGERROR_BADSIG 28 /* signature mismatch */
+#define DKIM_SIGERROR_EMPTY_H 31 /* h= empty */
+#define DKIM_SIGERROR_INVALID_H 32 /* h= missing req'd entries */
+#define DKIM_SIGERROR_KEYHASHMISMATCH 37 /* sig-key hash mismatch */
+#define DKIM_SIGERROR_EMPTY_V 45 /* v= tag empty */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Check results */
+enum rspamd_dkim_check_rcode {
+ DKIM_CONTINUE = 0,
+ DKIM_REJECT,
+ DKIM_TRYAGAIN,
+ DKIM_NOTFOUND,
+ DKIM_RECORD_ERROR,
+ DKIM_PERM_ERROR,
+};
+
+#define DKIM_CANON_SIMPLE 0 /* as specified in DKIM spec */
+#define DKIM_CANON_RELAXED 1 /* as specified in DKIM spec */
+
+struct rspamd_dkim_context_s;
+typedef struct rspamd_dkim_context_s rspamd_dkim_context_t;
+
+struct rspamd_dkim_sign_context_s;
+typedef struct rspamd_dkim_sign_context_s rspamd_dkim_sign_context_t;
+
+struct rspamd_dkim_key_s;
+typedef struct rspamd_dkim_key_s rspamd_dkim_key_t;
+typedef struct rspamd_dkim_key_s rspamd_dkim_sign_key_t;
+
+struct rspamd_task;
+
+enum rspamd_dkim_key_format {
+ RSPAMD_DKIM_KEY_FILE = 0,
+ RSPAMD_DKIM_KEY_PEM,
+ RSPAMD_DKIM_KEY_BASE64,
+ RSPAMD_DKIM_KEY_RAW,
+ RSPAMD_DKIM_KEY_UNKNOWN
+};
+
+enum rspamd_dkim_type {
+ RSPAMD_DKIM_NORMAL,
+ RSPAMD_DKIM_ARC_SIG,
+ RSPAMD_DKIM_ARC_SEAL
+};
+
+/* Signature methods */
+enum rspamd_sign_type {
+ DKIM_SIGN_UNKNOWN = -2,
+ DKIM_SIGN_RSASHA1 = 0,
+ DKIM_SIGN_RSASHA256,
+ DKIM_SIGN_RSASHA512,
+ DKIM_SIGN_ECDSASHA256,
+ DKIM_SIGN_ECDSASHA512,
+ DKIM_SIGN_EDDSASHA256,
+};
+
+enum rspamd_dkim_key_type {
+ RSPAMD_DKIM_KEY_RSA = 0,
+ RSPAMD_DKIM_KEY_ECDSA,
+ RSPAMD_DKIM_KEY_EDDSA
+};
+
+struct rspamd_dkim_check_result {
+ enum rspamd_dkim_check_rcode rcode;
+ rspamd_dkim_context_t *ctx;
+ /* Processed parts */
+ const gchar *selector;
+ const gchar *domain;
+ const gchar *short_b;
+ const gchar *fail_reason;
+};
+
+
+/* Err MUST be freed if it is not NULL, key is allocated by slice allocator */
+typedef void (*dkim_key_handler_f)(rspamd_dkim_key_t *key, gsize keylen,
+ rspamd_dkim_context_t *ctx, gpointer ud, GError *err);
+
+/**
+ * Create new dkim context from signature
+ * @param sig message's signature
+ * @param pool pool to allocate memory from
+ * @param time_jitter jitter in seconds to allow time diff while checking
+ * @param err pointer to error object
+ * @return new context or NULL
+ */
+rspamd_dkim_context_t *rspamd_create_dkim_context(const gchar *sig,
+ rspamd_mempool_t *pool,
+ struct rspamd_dns_resolver *resolver,
+ guint time_jitter,
+ enum rspamd_dkim_type type,
+ GError **err);
+
+/**
+ * Create new dkim context for making a signature
+ * @param task
+ * @param priv_key
+ * @param err
+ * @return
+ */
+rspamd_dkim_sign_context_t *rspamd_create_dkim_sign_context(struct rspamd_task *task,
+ rspamd_dkim_sign_key_t *priv_key,
+ gint headers_canon,
+ gint body_canon,
+ const gchar *dkim_headers,
+ enum rspamd_dkim_type type,
+ GError **err);
+
+/**
+ * Load dkim key
+ * @param path
+ * @param err
+ * @return
+ */
+rspamd_dkim_sign_key_t *rspamd_dkim_sign_key_load(const gchar *what, gsize len,
+ enum rspamd_dkim_key_format type,
+ GError **err);
+
+/**
+ * Invalidate modified sign key
+ * @param key
+ * @return
+*/
+gboolean rspamd_dkim_sign_key_maybe_invalidate(rspamd_dkim_sign_key_t *key,
+ time_t mtime);
+
+/**
+ * Make DNS request for specified context and obtain and parse key
+ * @param ctx dkim context from signature
+ * @param resolver dns resolver object
+ * @param s async session to make request
+ * @return
+ */
+gboolean rspamd_get_dkim_key(rspamd_dkim_context_t *ctx,
+ struct rspamd_task *task,
+ dkim_key_handler_f handler,
+ gpointer ud);
+
+/**
+ * Check task for dkim context using dkim key
+ * @param ctx dkim verify context
+ * @param key dkim key (from cache or from dns request)
+ * @param task task to check
+ * @return
+ */
+struct rspamd_dkim_check_result *rspamd_dkim_check(rspamd_dkim_context_t *ctx,
+ rspamd_dkim_key_t *key,
+ struct rspamd_task *task);
+
+struct rspamd_dkim_check_result *
+rspamd_dkim_create_result(rspamd_dkim_context_t *ctx,
+ enum rspamd_dkim_check_rcode rcode,
+ struct rspamd_task *task);
+
+GString *rspamd_dkim_sign(struct rspamd_task *task,
+ const gchar *selector,
+ const gchar *domain,
+ time_t expire,
+ gsize len,
+ guint idx,
+ const gchar *arc_cv,
+ rspamd_dkim_sign_context_t *ctx);
+
+rspamd_dkim_key_t *rspamd_dkim_key_ref(rspamd_dkim_key_t *k);
+
+void rspamd_dkim_key_unref(rspamd_dkim_key_t *k);
+
+rspamd_dkim_sign_key_t *rspamd_dkim_sign_key_ref(rspamd_dkim_sign_key_t *k);
+
+void rspamd_dkim_sign_key_unref(rspamd_dkim_sign_key_t *k);
+
+const gchar *rspamd_dkim_get_domain(rspamd_dkim_context_t *ctx);
+
+const gchar *rspamd_dkim_get_selector(rspamd_dkim_context_t *ctx);
+
+const gchar *rspamd_dkim_get_dns_key(rspamd_dkim_context_t *ctx);
+
+guint rspamd_dkim_key_get_ttl(rspamd_dkim_key_t *k);
+
+/**
+ * Create DKIM public key from a raw data
+ * @param keydata
+ * @param keylen
+ * @param type
+ * @param err
+ * @return
+ */
+rspamd_dkim_key_t *rspamd_dkim_make_key(const gchar *keydata, guint keylen,
+ enum rspamd_dkim_key_type type,
+ GError **err);
+
+#define RSPAMD_DKIM_KEY_ID_LEN 16
+/**
+ * Returns key id for dkim key (raw md5 of RSPAMD_DKIM_KEY_ID_LEN)
+ * NOT ZERO TERMINATED, use RSPAMD_DKIM_KEY_ID_LEN for length
+ * @param key
+ * @return
+ */
+const guchar *rspamd_dkim_key_id(rspamd_dkim_key_t *key);
+
+/**
+ * Parse DKIM public key from a TXT record
+ * @param txt
+ * @param keylen
+ * @param err
+ * @return
+ */
+rspamd_dkim_key_t *rspamd_dkim_parse_key(const gchar *txt, gsize *keylen,
+ GError **err);
+
+/**
+ * Canonicalise header using relaxed algorithm
+ * @param hname
+ * @param hvalue
+ * @param out
+ * @param outlen
+ * @return
+ */
+goffset rspamd_dkim_canonize_header_relaxed_str(const gchar *hname,
+ const gchar *hvalue,
+ gchar *out,
+ gsize outlen);
+
+/**
+ * Checks public and private keys for match
+ * @param pk
+ * @param sk
+ * @param err
+ * @return
+ */
+gboolean rspamd_dkim_match_keys(rspamd_dkim_key_t *pk,
+ rspamd_dkim_sign_key_t *sk,
+ GError **err);
+
+/**
+ * Free DKIM key
+ * @param key
+ */
+void rspamd_dkim_key_free(rspamd_dkim_key_t *key);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* DKIM_H_ */
diff --git a/src/libserver/dns.c b/src/libserver/dns.c
new file mode 100644
index 0000000..be2d5a3
--- /dev/null
+++ b/src/libserver/dns.c
@@ -0,0 +1,1124 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include "contrib/librdns/rdns.h"
+#include "config.h"
+#include "dns.h"
+#include "rspamd.h"
+#include "utlist.h"
+#include "contrib/libev/ev.h"
+#include "contrib/librdns/rdns.h"
+#include "contrib/librdns/dns_private.h"
+#include "contrib/librdns/rdns_ev.h"
+#include "unix-std.h"
+
+#include <unicode/uidna.h>
+
+static const gchar *M = "rspamd dns";
+
+static struct rdns_upstream_elt *rspamd_dns_select_upstream(const char *name,
+ size_t len, void *ups_data);
+static struct rdns_upstream_elt *rspamd_dns_select_upstream_retransmit(
+ const char *name,
+ size_t len,
+ struct rdns_upstream_elt *prev_elt,
+ void *ups_data);
+static void rspamd_dns_upstream_ok(struct rdns_upstream_elt *elt,
+ void *ups_data);
+static void rspamd_dns_upstream_fail(struct rdns_upstream_elt *elt,
+ void *ups_data, const gchar *reason);
+static unsigned int rspamd_dns_upstream_count(void *ups_data);
+
+static struct rdns_upstream_context rspamd_ups_ctx = {
+ .select = rspamd_dns_select_upstream,
+ .select_retransmit = rspamd_dns_select_upstream_retransmit,
+ .ok = rspamd_dns_upstream_ok,
+ .fail = rspamd_dns_upstream_fail,
+ .count = rspamd_dns_upstream_count,
+ .data = NULL};
+
+struct rspamd_dns_request_ud {
+ struct rspamd_async_session *session;
+ dns_callback_type cb;
+ gpointer ud;
+ rspamd_mempool_t *pool;
+ struct rspamd_task *task;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rdns_request *req;
+ struct rdns_reply *reply;
+};
+
+struct rspamd_dns_fail_cache_entry {
+ const char *name;
+ gint32 namelen;
+ enum rdns_request_type type;
+};
+
+static const gint8 ascii_dns_table[128] = {
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ /* HYPHEN-MINUS..FULL STOP */
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1,
+ /* 0..9 digits */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1,
+ /* LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z */
+ -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* _ */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, 1,
+ /* LATIN SMALL LETTER A..LATIN SMALL LETTER Z */
+ -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1};
+
+static guint
+rspamd_dns_fail_hash(gconstpointer ptr)
+{
+ struct rspamd_dns_fail_cache_entry *elt =
+ (struct rspamd_dns_fail_cache_entry *) ptr;
+
+ /* We don't care about type when doing hashing */
+ return rspamd_cryptobox_fast_hash(elt->name, elt->namelen,
+ rspamd_hash_seed());
+}
+
+static gboolean
+rspamd_dns_fail_equal(gconstpointer p1, gconstpointer p2)
+{
+ struct rspamd_dns_fail_cache_entry *e1 = (struct rspamd_dns_fail_cache_entry *) p1,
+ *e2 = (struct rspamd_dns_fail_cache_entry *) p2;
+
+ if (e1->type == e2->type && e1->namelen == e2->namelen) {
+ return memcmp(e1->name, e2->name, e1->namelen) == 0;
+ }
+
+ return FALSE;
+}
+
+static void
+rspamd_dns_fin_cb(gpointer arg)
+{
+ struct rspamd_dns_request_ud *reqdata = (struct rspamd_dns_request_ud *) arg;
+
+ if (reqdata->item) {
+ rspamd_symcache_set_cur_item(reqdata->task, reqdata->item);
+ }
+
+ if (reqdata->reply) {
+ reqdata->cb(reqdata->reply, reqdata->ud);
+ }
+ else {
+ struct rdns_reply fake_reply;
+
+ memset(&fake_reply, 0, sizeof(fake_reply));
+ fake_reply.code = RDNS_RC_TIMEOUT;
+ fake_reply.request = reqdata->req;
+ fake_reply.resolver = reqdata->req->resolver;
+ fake_reply.requested_name = reqdata->req->requested_names[0].name;
+
+ reqdata->cb(&fake_reply, reqdata->ud);
+ }
+
+ rdns_request_release(reqdata->req);
+
+ if (reqdata->item) {
+ rspamd_symcache_item_async_dec_check(reqdata->task,
+ reqdata->item, M);
+ }
+
+ if (reqdata->pool == NULL) {
+ g_free(reqdata);
+ }
+}
+
+static void
+rspamd_dns_callback(struct rdns_reply *reply, gpointer ud)
+{
+ struct rspamd_dns_request_ud *reqdata = ud;
+
+ reqdata->reply = reply;
+
+
+ if (reqdata->session) {
+ if (reply->code == RDNS_RC_SERVFAIL &&
+ reqdata->task &&
+ reqdata->task->resolver->fails_cache) {
+
+ /* Add to cache... */
+ const gchar *name = reqdata->req->requested_names[0].name;
+ gchar *target;
+ gsize namelen;
+ struct rspamd_dns_fail_cache_entry *nentry;
+
+ /* Allocate in a single entry to allow further free in a single call */
+ namelen = strlen(name);
+ nentry = g_malloc(sizeof(nentry) + namelen + 1);
+ target = ((gchar *) nentry) + sizeof(nentry);
+ rspamd_strlcpy(target, name, namelen + 1);
+ nentry->type = reqdata->req->requested_names[0].type;
+ nentry->name = target;
+ nentry->namelen = namelen;
+
+ /* Rdns request is retained there */
+ rspamd_lru_hash_insert(reqdata->task->resolver->fails_cache,
+ nentry, rdns_request_retain(reply->request),
+ reqdata->task->task_timestamp,
+ reqdata->task->resolver->fails_cache_time);
+ }
+
+ /*
+ * Ref event to avoid double unref by
+ * event removing
+ */
+ rdns_request_retain(reply->request);
+ rspamd_session_remove_event(reqdata->session,
+ rspamd_dns_fin_cb, reqdata);
+ }
+ else {
+ reqdata->cb(reply, reqdata->ud);
+
+ if (reqdata->pool == NULL) {
+ g_free(reqdata);
+ }
+ }
+}
+
+struct rspamd_dns_request_ud *
+rspamd_dns_resolver_request(struct rspamd_dns_resolver *resolver,
+ struct rspamd_async_session *session,
+ rspamd_mempool_t *pool,
+ dns_callback_type cb,
+ gpointer ud,
+ enum rdns_request_type type,
+ const char *name)
+{
+ struct rdns_request *req;
+ struct rspamd_dns_request_ud *reqdata = NULL;
+ guint nlen = strlen(name);
+ gchar *real_name = NULL;
+
+ g_assert(resolver != NULL);
+
+ if (resolver->r == NULL) {
+ return NULL;
+ }
+
+ if (nlen == 0 || nlen > DNS_D_MAXNAME) {
+ return NULL;
+ }
+
+ if (session && rspamd_session_blocked(session)) {
+ return NULL;
+ }
+
+ if (rspamd_str_has_8bit(name, nlen)) {
+ /* Convert to idna using libicu as it follows all the standards */
+ real_name = rspamd_dns_resolver_idna_convert_utf8(resolver, pool,
+ name, nlen, &nlen);
+
+ if (real_name == NULL) {
+ return NULL;
+ }
+
+ name = real_name;
+ }
+
+ /* Name is now in ASCII only */
+ for (gsize i = 0; i < nlen; i++) {
+ if (ascii_dns_table[((unsigned int) name[i]) & 0x7F] == -1) {
+ /* Invalid DNS name requested */
+
+ if (!pool) {
+ g_free(real_name);
+ }
+
+ return NULL;
+ }
+ }
+
+ if (pool != NULL) {
+ reqdata =
+ rspamd_mempool_alloc0(pool, sizeof(struct rspamd_dns_request_ud));
+ }
+ else {
+ reqdata = g_malloc0(sizeof(struct rspamd_dns_request_ud));
+ }
+
+ reqdata->pool = pool;
+ reqdata->session = session;
+ reqdata->cb = cb;
+ reqdata->ud = ud;
+
+ req = rdns_make_request_full(resolver->r, rspamd_dns_callback, reqdata,
+ resolver->request_timeout, resolver->max_retransmits, 1, name,
+ type);
+ reqdata->req = req;
+
+ if (session) {
+ if (req != NULL) {
+ rspamd_session_add_event(session,
+ (event_finalizer_t) rspamd_dns_fin_cb,
+ reqdata,
+ M);
+ }
+ }
+
+ if (req == NULL) {
+ if (pool == NULL) {
+ g_free(reqdata);
+ g_free(real_name);
+ }
+
+ return NULL;
+ }
+
+ if (real_name && pool == NULL) {
+ g_free(real_name);
+ }
+
+ return reqdata;
+}
+
+struct rspamd_dns_cached_delayed_cbdata {
+ struct rspamd_task *task;
+ dns_callback_type cb;
+ gpointer ud;
+ ev_timer tm;
+ struct rdns_request *req;
+};
+
+static void
+rspamd_fail_cache_cb(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_dns_cached_delayed_cbdata *cbd =
+ (struct rspamd_dns_cached_delayed_cbdata *) w->data;
+ struct rdns_reply fake_reply;
+
+ ev_timer_stop(EV_A_ w);
+ memset(&fake_reply, 0, sizeof(fake_reply));
+ fake_reply.code = RDNS_RC_SERVFAIL;
+ fake_reply.request = cbd->req;
+ fake_reply.resolver = cbd->req->resolver;
+ fake_reply.requested_name = cbd->req->requested_names[0].name;
+ cbd->cb(&fake_reply, cbd->ud);
+ rdns_request_release(cbd->req);
+}
+
+static gboolean
+make_dns_request_task_common(struct rspamd_task *task,
+ dns_callback_type cb,
+ gpointer ud,
+ enum rdns_request_type type,
+ const char *name,
+ gboolean forced)
+{
+ struct rspamd_dns_request_ud *reqdata;
+
+ if (!forced && task->dns_requests >= task->cfg->dns_max_requests) {
+ return FALSE;
+ }
+
+ if (task->resolver->fails_cache) {
+ /* Search in failures cache */
+ struct rspamd_dns_fail_cache_entry search;
+ struct rdns_request *req;
+
+ search.name = name;
+ search.namelen = strlen(name);
+ search.type = type;
+
+ if ((req = rspamd_lru_hash_lookup(task->resolver->fails_cache,
+ &search, task->task_timestamp)) != NULL) {
+ /*
+ * We need to reply with SERVFAIL again to the API, so add a special
+ * timer, uh-oh, and fire it
+ */
+ struct rspamd_dns_cached_delayed_cbdata *cbd =
+ rspamd_mempool_alloc0(task->task_pool, sizeof(*cbd));
+
+ ev_timer_init(&cbd->tm, rspamd_fail_cache_cb, 0.0, 0.0);
+ cbd->task = task;
+ cbd->cb = cb;
+ cbd->ud = ud;
+ cbd->req = rdns_request_retain(req);
+ cbd->tm.data = cbd;
+
+ return TRUE;
+ }
+ }
+
+ reqdata = rspamd_dns_resolver_request(
+ task->resolver, task->s, task->task_pool, cb, ud,
+ type, name);
+
+ if (reqdata) {
+ task->dns_requests++;
+
+ reqdata->task = task;
+ reqdata->item = rspamd_symcache_get_cur_item(task);
+
+ if (reqdata->item) {
+ /* We are inside some session */
+ rspamd_symcache_item_async_inc(task, reqdata->item, M);
+ }
+
+ if (!forced && task->dns_requests >= task->cfg->dns_max_requests) {
+ msg_info_task("stop resolving on reaching %ud requests",
+ task->dns_requests);
+ }
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_dns_resolver_request_task(struct rspamd_task *task,
+ dns_callback_type cb,
+ gpointer ud,
+ enum rdns_request_type type,
+ const char *name)
+{
+ return make_dns_request_task_common(task, cb, ud, type, name, FALSE);
+}
+
+gboolean
+rspamd_dns_resolver_request_task_forced(struct rspamd_task *task,
+ dns_callback_type cb,
+ gpointer ud,
+ enum rdns_request_type type,
+ const char *name)
+{
+ return make_dns_request_task_common(task, cb, ud, type, name, TRUE);
+}
+
+static void rspamd_rnds_log_bridge(
+ void *log_data,
+ enum rdns_log_level level,
+ const char *function,
+ const char *format,
+ va_list args)
+{
+ rspamd_logger_t *logger = log_data;
+
+ rspamd_common_logv(logger, (GLogLevelFlags) level, "rdns", NULL,
+ function, format, args);
+}
+
+static void
+rspamd_dns_server_init(struct upstream *up, guint idx, gpointer ud)
+{
+ struct rspamd_dns_resolver *r = ud;
+ rspamd_inet_addr_t *addr;
+ void *serv;
+ struct rdns_upstream_elt *elt;
+
+ addr = rspamd_upstream_addr_next(up);
+
+ if (r->cfg) {
+ serv = rdns_resolver_add_server(r->r, rspamd_inet_address_to_string(addr),
+ rspamd_inet_address_get_port(addr), 0, r->cfg->dns_io_per_server);
+
+ elt = rspamd_mempool_alloc0(r->cfg->cfg_pool, sizeof(*elt));
+ elt->server = serv;
+ elt->lib_data = up;
+
+ rspamd_upstream_set_data(up, elt);
+ }
+ else {
+ serv = rdns_resolver_add_server(r->r, rspamd_inet_address_to_string(addr),
+ rspamd_inet_address_get_port(addr), 0, 8);
+ }
+
+ g_assert(serv != NULL);
+}
+
+static void
+rspamd_dns_server_reorder(struct upstream *up, guint idx, gpointer ud)
+{
+ struct rspamd_dns_resolver *r = ud;
+
+ rspamd_upstream_set_weight(up, rspamd_upstreams_count(r->ups) - idx + 1);
+}
+
+static bool
+rspamd_dns_resolv_conf_on_server(struct rdns_resolver *resolver,
+ const char *name, unsigned int port,
+ int priority, unsigned int io_cnt, void *ud)
+{
+ struct rspamd_dns_resolver *dns_resolver = ud;
+ struct rspamd_config *cfg;
+ rspamd_inet_addr_t *addr;
+ gint test_fd;
+
+ cfg = dns_resolver->cfg;
+
+ msg_info_config("parsed nameserver %s from resolv.conf", name);
+
+ /* Try to open a connection */
+ if (!rspamd_parse_inet_address(&addr, name, strlen(name),
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ msg_warn_config("cannot parse nameserver address %s", name);
+
+ return FALSE;
+ }
+
+ rspamd_inet_address_set_port(addr, port);
+ test_fd = rspamd_inet_address_connect(addr, SOCK_DGRAM, TRUE);
+
+ if (test_fd == -1 && (errno != EINTR || errno != ECONNREFUSED || errno != ECONNRESET)) {
+ msg_info_config("cannot open connection to nameserver at address %s: %s",
+ name, strerror(errno));
+ rspamd_inet_address_free(addr);
+
+ return FALSE;
+ }
+
+ rspamd_inet_address_free(addr);
+ close(test_fd);
+
+ return rspamd_upstreams_add_upstream(dns_resolver->ups, name, port,
+ RSPAMD_UPSTREAM_PARSE_NAMESERVER,
+ NULL);
+}
+
+static void
+rspamd_process_fake_reply(struct rspamd_config *cfg,
+ struct rspamd_dns_resolver *dns_resolver,
+ const ucl_object_t *cur_arr)
+{
+ const ucl_object_t *cur;
+ ucl_object_iter_t it;
+
+ it = ucl_object_iterate_new(cur_arr);
+
+ while ((cur = ucl_object_iterate_safe(it, true))) {
+ const ucl_object_t *type_obj, *name_obj, *code_obj, *replies_obj;
+ enum rdns_request_type rtype = RDNS_REQUEST_A;
+ enum dns_rcode rcode = RDNS_RC_NOERROR;
+ struct rdns_reply_entry *replies = NULL;
+ const gchar *name = NULL;
+
+ if (ucl_object_type(cur) != UCL_OBJECT) {
+ continue;
+ }
+
+ name_obj = ucl_object_lookup(cur, "name");
+ if (name_obj == NULL ||
+ (name = ucl_object_tostring(name_obj)) == NULL) {
+ msg_err_config("no name for fake dns reply");
+ continue;
+ }
+
+ type_obj = ucl_object_lookup(cur, "type");
+ if (type_obj) {
+ rtype = rdns_type_fromstr(ucl_object_tostring(type_obj));
+
+ if (rtype == RDNS_REQUEST_INVALID) {
+ msg_err_config("invalid type for %s: %s", name,
+ ucl_object_tostring(type_obj));
+ continue;
+ }
+ }
+
+ code_obj = ucl_object_lookup_any(cur, "code", "rcode", NULL);
+ if (code_obj) {
+ rcode = rdns_rcode_fromstr(ucl_object_tostring(code_obj));
+
+ if (rcode == RDNS_RC_INVALID) {
+ msg_err_config("invalid rcode for %s: %s", name,
+ ucl_object_tostring(code_obj));
+ continue;
+ }
+ }
+
+ if (rcode == RDNS_RC_NOERROR) {
+ /* We want replies to be set for this rcode */
+ replies_obj = ucl_object_lookup(cur, "replies");
+
+ if (replies_obj == NULL || ucl_object_type(replies_obj) != UCL_ARRAY) {
+ msg_err_config("invalid replies for fake DNS record %s", name);
+ continue;
+ }
+
+ ucl_object_iter_t rep_it;
+ const ucl_object_t *rep_obj;
+
+ rep_it = ucl_object_iterate_new(replies_obj);
+
+ while ((rep_obj = ucl_object_iterate_safe(rep_it, true))) {
+ const gchar *str_rep = ucl_object_tostring(rep_obj);
+ struct rdns_reply_entry *rep;
+ gchar **svec;
+
+ if (str_rep == NULL) {
+ msg_err_config("invalid reply element for fake DNS record %s",
+ name);
+ continue;
+ }
+
+ rep = calloc(1, sizeof(*rep));
+ g_assert(rep != NULL);
+
+ rep->type = rtype;
+ rep->ttl = 0;
+
+ switch (rtype) {
+ case RDNS_REQUEST_A:
+ if (inet_pton(AF_INET, str_rep, &rep->content.a.addr) != 1) {
+ msg_err_config("invalid A reply element for fake "
+ "DNS record %s: %s",
+ name, str_rep);
+ free(rep);
+ }
+ else {
+ DL_APPEND(replies, rep);
+ }
+ break;
+ case RDNS_REQUEST_NS:
+ rep->content.ns.name = strdup(str_rep);
+ DL_APPEND(replies, rep);
+ break;
+ case RDNS_REQUEST_PTR:
+ rep->content.ptr.name = strdup(str_rep);
+ DL_APPEND(replies, rep);
+ break;
+ case RDNS_REQUEST_MX:
+ svec = g_strsplit_set(str_rep, " :", -1);
+
+ if (svec && svec[0] && svec[1]) {
+ rep->content.mx.priority = strtoul(svec[0], NULL, 10);
+ rep->content.mx.name = strdup(svec[1]);
+ DL_APPEND(replies, rep);
+ }
+ else {
+ msg_err_config("invalid MX reply element for fake "
+ "DNS record %s: %s",
+ name, str_rep);
+ free(rep);
+ }
+
+ g_strfreev(svec);
+ break;
+ case RDNS_REQUEST_TXT:
+ rep->content.txt.data = strdup(str_rep);
+ DL_APPEND(replies, rep);
+ break;
+ case RDNS_REQUEST_SOA:
+ svec = g_strsplit_set(str_rep, " :", -1);
+
+ /* 7 elements */
+ if (svec && svec[0] && svec[1] && svec[2] &&
+ svec[3] && svec[4] && svec[5] && svec[6]) {
+ rep->content.soa.mname = strdup(svec[0]);
+ rep->content.soa.admin = strdup(svec[1]);
+ rep->content.soa.serial = strtoul(svec[2], NULL, 10);
+ rep->content.soa.refresh = strtol(svec[3], NULL, 10);
+ rep->content.soa.retry = strtol(svec[4], NULL, 10);
+ rep->content.soa.expire = strtol(svec[5], NULL, 10);
+ rep->content.soa.minimum = strtoul(svec[6], NULL, 10);
+ DL_APPEND(replies, rep);
+ }
+ else {
+ msg_err_config("invalid MX reply element for fake "
+ "DNS record %s: %s",
+ name, str_rep);
+ free(rep);
+ }
+
+ g_strfreev(svec);
+ break;
+ case RDNS_REQUEST_AAAA:
+ if (inet_pton(AF_INET6, str_rep, &rep->content.aaa.addr) != 1) {
+ msg_err_config("invalid AAAA reply element for fake "
+ "DNS record %s: %s",
+ name, str_rep);
+ free(rep);
+ }
+ else {
+ DL_APPEND(replies, rep);
+ }
+ break;
+ case RDNS_REQUEST_SRV:
+ default:
+ msg_err_config("invalid or unsupported reply element "
+ "for fake DNS record %s(%s): %s",
+ name, rdns_str_from_type(rtype), str_rep);
+ free(rep);
+ break;
+ }
+ }
+
+ ucl_object_iterate_free(rep_it);
+
+ if (replies) {
+ struct rdns_reply_entry *tmp_entry;
+ guint i = 0;
+ DL_COUNT(replies, tmp_entry, i);
+
+ msg_info_config("added fake record: %s(%s); %d replies", name,
+ rdns_str_from_type(rtype), i);
+ rdns_resolver_set_fake_reply(dns_resolver->r,
+ name, rtype, rcode, replies);
+ }
+ else {
+ msg_warn_config("record %s has no replies, not adding",
+ name);
+ }
+ }
+ else {
+ /* This entry returns some non valid code, no replies are possible */
+ replies_obj = ucl_object_lookup(cur, "replies");
+
+ if (replies_obj) {
+ msg_warn_config("replies are set for non-successful return "
+ "code for %s(%s), they will be ignored",
+ name, rdns_str_from_type(rtype));
+ }
+
+ rdns_resolver_set_fake_reply(dns_resolver->r,
+ name, rtype, rcode, NULL);
+ }
+ }
+
+ ucl_object_iterate_free(it);
+}
+
+static bool
+rspamd_dns_read_hosts_file(struct rspamd_config *cfg,
+ struct rspamd_dns_resolver *dns_resolver,
+ const gchar *fname)
+{
+ gchar *linebuf = NULL;
+ gsize buflen = 0;
+ gssize r;
+ FILE *fp;
+ guint nadded = 0;
+
+ fp = fopen(fname, "r");
+
+ if (fp == NULL) {
+ /* Hack to reduce noise */
+ if (strcmp(fname, "/etc/hosts") == 0) {
+ msg_info_config("cannot open hosts file %s: %s", fname,
+ strerror(errno));
+ }
+ else {
+ msg_err_config("cannot open hosts file %s: %s", fname,
+ strerror(errno));
+ }
+
+ return false;
+ }
+
+ while ((r = getline(&linebuf, &buflen, fp)) > 0) {
+ if (linebuf[0] == '#' || g_ascii_isspace(linebuf[0])) {
+ /* Skip comment or empty line */
+ continue;
+ }
+
+ g_strchomp(linebuf);
+
+ gchar **elts = g_strsplit_set(linebuf, " \t\v", -1);
+ rspamd_inet_addr_t *addr;
+
+ if (!rspamd_parse_inet_address(&addr, elts[0], strlen(elts[0]),
+ RSPAMD_INET_ADDRESS_PARSE_REMOTE | RSPAMD_INET_ADDRESS_PARSE_NO_UNIX)) {
+ msg_warn_config("bad hosts file line: %s; cannot parse address", linebuf);
+ }
+ else {
+ /* Add all FQDN + aliases if any */
+ gchar **cur_name = &elts[1];
+
+ while (*cur_name) {
+ if (strlen(*cur_name) == 0) {
+ cur_name++;
+ continue;
+ }
+
+ if (*cur_name[0] == '#') {
+ /* Start of the comment */
+ break;
+ }
+
+ struct rdns_reply_entry *rep;
+ rep = calloc(1, sizeof(*rep));
+ g_assert(rep != NULL);
+
+ rep->ttl = 0;
+
+ if (rspamd_inet_address_get_af(addr) == AF_INET) {
+ socklen_t unused;
+ const struct sockaddr_in *sin = (const struct sockaddr_in *)
+ rspamd_inet_address_get_sa(addr, &unused);
+ rep->type = RDNS_REQUEST_A;
+ memcpy(&rep->content.a.addr, &sin->sin_addr,
+ sizeof(rep->content.a.addr));
+ }
+ else {
+ socklen_t unused;
+ const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)
+ rspamd_inet_address_get_sa(addr, &unused);
+ rep->type = RDNS_REQUEST_AAAA;
+ memcpy(&rep->content.aaa.addr, &sin6->sin6_addr,
+ sizeof(rep->content.aaa.addr));
+ }
+
+ rep->next = NULL;
+ rep->prev = rep;
+ rdns_resolver_set_fake_reply(dns_resolver->r,
+ *cur_name, rep->type, RDNS_RC_NOERROR, rep);
+ msg_debug_config("added fake record %s -> %s from hosts file %s",
+ *cur_name, rspamd_inet_address_to_string(addr), fname);
+ cur_name++;
+ nadded++;
+ }
+
+ rspamd_inet_address_free(addr);
+ }
+
+ g_strfreev(elts);
+ }
+
+ if (linebuf) {
+ free(linebuf);
+ }
+
+ msg_info_config("processed host file %s; %d records added", fname, nadded);
+ fclose(fp);
+
+ return true;
+}
+
+static void
+rspamd_dns_resolver_config_ucl(struct rspamd_config *cfg,
+ struct rspamd_dns_resolver *dns_resolver,
+ const ucl_object_t *dns_section)
+{
+ const ucl_object_t *fake_replies, *fails_cache_size, *fails_cache_time,
+ *hosts;
+ static const ev_tstamp default_fails_cache_time = 10.0;
+
+ /* Process fake replies */
+ fake_replies = ucl_object_lookup_any(dns_section, "fake_records",
+ "fake_replies", NULL);
+
+ if (fake_replies && ucl_object_type(fake_replies) == UCL_ARRAY) {
+ const ucl_object_t *cur_arr;
+
+ DL_FOREACH(fake_replies, cur_arr)
+ {
+ rspamd_process_fake_reply(cfg, dns_resolver, cur_arr);
+ }
+ }
+
+ hosts = ucl_object_lookup(dns_section, "hosts");
+
+ if (hosts == NULL) {
+ /* Read normal `/etc/hosts` file */
+ rspamd_dns_read_hosts_file(cfg, dns_resolver, "/etc/hosts");
+ }
+ else if (ucl_object_type(hosts) == UCL_NULL) {
+ /* Do nothing, hosts are explicitly disabled */
+ }
+ else if (ucl_object_type(hosts) == UCL_STRING) {
+ if (!rspamd_dns_read_hosts_file(cfg, dns_resolver, ucl_object_tostring(hosts))) {
+ msg_err_config("cannot read hosts file %s", ucl_object_tostring(hosts));
+ }
+ }
+ else if (ucl_object_type(hosts) == UCL_ARRAY) {
+ const ucl_object_t *cur;
+ ucl_object_iter_t it = NULL;
+
+ while ((cur = ucl_object_iterate(hosts, &it, true)) != NULL) {
+ if (!rspamd_dns_read_hosts_file(cfg, dns_resolver, ucl_object_tostring(cur))) {
+ msg_err_config("cannot read hosts file %s", ucl_object_tostring(cur));
+ }
+ }
+ }
+ else {
+ msg_err_config("invalid type for hosts parameter: %s",
+ ucl_object_type_to_string(ucl_object_type(hosts)));
+ }
+
+ fails_cache_size = ucl_object_lookup(dns_section, "fails_cache_size");
+ if (fails_cache_size && ucl_object_type(fails_cache_size) == UCL_INT) {
+
+ dns_resolver->fails_cache_time = default_fails_cache_time;
+ fails_cache_time = ucl_object_lookup(dns_section, "fails_cache_time");
+
+ if (fails_cache_time) {
+ dns_resolver->fails_cache_time = ucl_object_todouble(fails_cache_time);
+ }
+
+ dns_resolver->fails_cache = rspamd_lru_hash_new_full(
+ ucl_object_toint(fails_cache_size),
+ g_free, (GDestroyNotify) rdns_request_release,
+ rspamd_dns_fail_hash, rspamd_dns_fail_equal);
+ }
+}
+
+struct rspamd_dns_resolver *
+rspamd_dns_resolver_init(rspamd_logger_t *logger,
+ struct ev_loop *ev_base,
+ struct rspamd_config *cfg)
+{
+ struct rspamd_dns_resolver *dns_resolver;
+
+ dns_resolver = g_malloc0(sizeof(struct rspamd_dns_resolver));
+ dns_resolver->event_loop = ev_base;
+
+ if (cfg != NULL) {
+ dns_resolver->request_timeout = cfg->dns_timeout;
+ dns_resolver->max_retransmits = cfg->dns_retransmits;
+ }
+ else {
+ dns_resolver->request_timeout = 1;
+ dns_resolver->max_retransmits = 2;
+ }
+
+ /* IDN translation is performed in Rspamd now */
+ dns_resolver->r = rdns_resolver_new(RDNS_RESOLVER_NOIDN);
+
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ dns_resolver->uidna = uidna_openUTS46(UIDNA_DEFAULT, &uc_err);
+ g_assert(!U_FAILURE(uc_err));
+ rdns_bind_libev(dns_resolver->r, dns_resolver->event_loop);
+
+ if (cfg != NULL) {
+ rdns_resolver_set_log_level(dns_resolver->r, cfg->log_level);
+ dns_resolver->cfg = cfg;
+ rdns_resolver_set_dnssec(dns_resolver->r, cfg->enable_dnssec);
+
+ if (cfg->nameservers == NULL) {
+ /* Parse resolv.conf */
+ dns_resolver->ups = rspamd_upstreams_create(cfg->ups_ctx);
+ rspamd_upstreams_set_flags(dns_resolver->ups,
+ RSPAMD_UPSTREAM_FLAG_NORESOLVE);
+ rspamd_upstreams_set_rotation(dns_resolver->ups,
+ RSPAMD_UPSTREAM_MASTER_SLAVE);
+
+ if (!rdns_resolver_parse_resolv_conf_cb(dns_resolver->r,
+ "/etc/resolv.conf",
+ rspamd_dns_resolv_conf_on_server,
+ dns_resolver)) {
+ msg_err("cannot parse resolv.conf and no nameservers defined, "
+ "so no ways to resolve addresses");
+ rdns_resolver_release(dns_resolver->r);
+ dns_resolver->r = NULL;
+
+ return dns_resolver;
+ }
+
+ /* Use normal resolv.conf rules */
+ rspamd_upstreams_foreach(dns_resolver->ups, rspamd_dns_server_reorder,
+ dns_resolver);
+ }
+ else {
+ dns_resolver->ups = rspamd_upstreams_create(cfg->ups_ctx);
+ rspamd_upstreams_set_flags(dns_resolver->ups,
+ RSPAMD_UPSTREAM_FLAG_NORESOLVE);
+
+ if (!rspamd_upstreams_from_ucl(dns_resolver->ups, cfg->nameservers,
+ 53, dns_resolver)) {
+ msg_err_config("cannot parse DNS nameservers definitions");
+ rdns_resolver_release(dns_resolver->r);
+ dns_resolver->r = NULL;
+
+ return dns_resolver;
+ }
+ }
+
+ rspamd_upstreams_foreach(dns_resolver->ups, rspamd_dns_server_init,
+ dns_resolver);
+ rdns_resolver_set_upstream_lib(dns_resolver->r, &rspamd_ups_ctx,
+ dns_resolver->ups);
+ cfg->dns_resolver = dns_resolver;
+
+ if (cfg->cfg_ucl_obj) {
+ /* Configure additional options */
+ const ucl_object_t *opts_section, *dns_section, *tmp;
+
+ opts_section = ucl_object_lookup(cfg->cfg_ucl_obj, "options");
+
+ if (opts_section) {
+ /* TODO: implement a more simple merge logic */
+ DL_FOREACH(opts_section, tmp)
+ {
+ dns_section = ucl_object_lookup(opts_section, "dns");
+
+ if (dns_section) {
+ rspamd_dns_resolver_config_ucl(cfg, dns_resolver,
+ dns_section);
+ }
+ }
+ }
+ }
+ }
+
+ rdns_resolver_set_logger(dns_resolver->r, rspamd_rnds_log_bridge, logger);
+ rdns_resolver_init(dns_resolver->r);
+
+ return dns_resolver;
+}
+
+void rspamd_dns_resolver_deinit(struct rspamd_dns_resolver *resolver)
+{
+ if (resolver) {
+ if (resolver->r) {
+ rdns_resolver_release(resolver->r);
+ }
+
+ if (resolver->ups) {
+ rspamd_upstreams_destroy(resolver->ups);
+ }
+
+ if (resolver->fails_cache) {
+ rspamd_lru_hash_destroy(resolver->fails_cache);
+ }
+
+ uidna_close(resolver->uidna);
+
+ g_free(resolver);
+ }
+}
+
+
+static struct rdns_upstream_elt *
+rspamd_dns_select_upstream(const char *name,
+ size_t len, void *ups_data)
+{
+ struct upstream_list *ups = ups_data;
+ struct upstream *up;
+
+ up = rspamd_upstream_get(ups, RSPAMD_UPSTREAM_ROUND_ROBIN, name, len);
+
+ if (up) {
+ msg_debug("select %s", rspamd_upstream_name(up));
+
+ return rspamd_upstream_get_data(up);
+ }
+
+ return NULL;
+}
+
+static struct rdns_upstream_elt *
+rspamd_dns_select_upstream_retransmit(
+ const char *name,
+ size_t len,
+ struct rdns_upstream_elt *prev_elt,
+ void *ups_data)
+{
+ struct upstream_list *ups = ups_data;
+ struct upstream *up;
+
+ if (prev_elt) {
+ up = rspamd_upstream_get_except(ups, (struct upstream *) prev_elt->lib_data,
+ RSPAMD_UPSTREAM_MASTER_SLAVE, name, len);
+ }
+ else {
+ up = rspamd_upstream_get_forced(ups, RSPAMD_UPSTREAM_RANDOM, name, len);
+ }
+
+ if (up) {
+ msg_debug("select forced %s", rspamd_upstream_name(up));
+
+ return rspamd_upstream_get_data(up);
+ }
+
+ return NULL;
+}
+
+static void
+rspamd_dns_upstream_ok(struct rdns_upstream_elt *elt,
+ void *ups_data)
+{
+ struct upstream *up = elt->lib_data;
+
+ rspamd_upstream_ok(up);
+}
+
+static void
+rspamd_dns_upstream_fail(struct rdns_upstream_elt *elt,
+ void *ups_data, const gchar *reason)
+{
+ struct upstream *up = elt->lib_data;
+
+ rspamd_upstream_fail(up, FALSE, reason);
+}
+
+static unsigned int
+rspamd_dns_upstream_count(void *ups_data)
+{
+ struct upstream_list *ups = ups_data;
+
+ return rspamd_upstreams_alive(ups);
+}
+
+gchar *
+rspamd_dns_resolver_idna_convert_utf8(struct rspamd_dns_resolver *resolver,
+ rspamd_mempool_t *pool,
+ const char *name,
+ gint namelen,
+ guint *outlen)
+{
+ if (resolver == NULL || resolver->uidna == NULL || name == NULL || namelen > DNS_D_MAXNAME) {
+ return NULL;
+ }
+
+ guint dest_len;
+ UErrorCode uc_err = U_ZERO_ERROR;
+ UIDNAInfo info = UIDNA_INFO_INITIALIZER;
+ /* Calculate length required */
+ dest_len = uidna_nameToASCII_UTF8(resolver->uidna, name, namelen,
+ NULL, 0, &info, &uc_err);
+
+ if (uc_err == U_BUFFER_OVERFLOW_ERROR) {
+ gchar *dest;
+
+ if (pool) {
+ dest = rspamd_mempool_alloc(pool, dest_len + 1);
+ }
+ else {
+ dest = g_malloc(dest_len + 1);
+ }
+
+ uc_err = U_ZERO_ERROR;
+
+ dest_len = uidna_nameToASCII_UTF8(resolver->uidna, name, namelen,
+ dest, dest_len + 1, &info, &uc_err);
+
+ if (U_FAILURE(uc_err)) {
+
+ if (!pool) {
+ g_free(dest);
+ }
+
+ return NULL;
+ }
+
+ dest[dest_len] = '\0';
+
+ if (outlen) {
+ *outlen = dest_len;
+ }
+
+ return dest;
+ }
+
+ return NULL;
+} \ No newline at end of file
diff --git a/src/libserver/dns.h b/src/libserver/dns.h
new file mode 100644
index 0000000..acf8d09
--- /dev/null
+++ b/src/libserver/dns.h
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_DNS_H
+#define RSPAMD_DNS_H
+
+#include "config.h"
+#include "mem_pool.h"
+#include "async_session.h"
+#include "logger.h"
+#include "rdns.h"
+#include "upstream.h"
+#include "libutil/hash.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_config;
+struct rspamd_task;
+struct event_loop;
+
+struct rspamd_dns_resolver {
+ struct rdns_resolver *r;
+ struct ev_loop *event_loop;
+ rspamd_lru_hash_t *fails_cache;
+ void *uidna;
+ double fails_cache_time;
+ struct upstream_list *ups;
+ struct rspamd_config *cfg;
+ gdouble request_timeout;
+ guint max_retransmits;
+};
+
+/* Rspamd DNS API */
+
+/**
+ * Init DNS resolver, params are obtained from a config file or system file /etc/resolv.conf
+ */
+struct rspamd_dns_resolver *rspamd_dns_resolver_init(rspamd_logger_t *logger,
+ struct ev_loop *ev_base,
+ struct rspamd_config *cfg);
+
+void rspamd_dns_resolver_deinit(struct rspamd_dns_resolver *resolver);
+
+struct rspamd_dns_request_ud;
+
+/**
+ * Make a DNS request
+ * @param resolver resolver object
+ * @param session async session to register event
+ * @param pool memory pool for storage
+ * @param cb callback to call on resolve completing
+ * @param ud user data for callback
+ * @param type request type
+ * @param ... string or ip address based on a request type
+ * @return TRUE if request was sent.
+ */
+struct rspamd_dns_request_ud *rspamd_dns_resolver_request(struct rspamd_dns_resolver *resolver,
+ struct rspamd_async_session *session,
+ rspamd_mempool_t *pool,
+ dns_callback_type cb,
+ gpointer ud,
+ enum rdns_request_type type,
+ const char *name);
+
+gboolean rspamd_dns_resolver_request_task(struct rspamd_task *task,
+ dns_callback_type cb,
+ gpointer ud,
+ enum rdns_request_type type,
+ const char *name);
+
+gboolean rspamd_dns_resolver_request_task_forced(struct rspamd_task *task,
+ dns_callback_type cb,
+ gpointer ud,
+ enum rdns_request_type type,
+ const char *name);
+
+/**
+ * Converts a name into idna from UTF8
+ * @param resolver resolver (must be initialised)
+ * @param pool optional memory pool (can be NULL, then you need to g_free) the result
+ * @param name input name
+ * @param namelen length of input (-1 for zero terminated)
+ * @return encoded string
+ */
+gchar *rspamd_dns_resolver_idna_convert_utf8(struct rspamd_dns_resolver *resolver,
+ rspamd_mempool_t *pool,
+ const char *name,
+ gint namelen,
+ guint *outlen);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/dynamic_cfg.c b/src/libserver/dynamic_cfg.c
new file mode 100644
index 0000000..cd5cc4e
--- /dev/null
+++ b/src/libserver/dynamic_cfg.c
@@ -0,0 +1,743 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "libserver/maps/map.h"
+#include "scan_result.h"
+#include "dynamic_cfg.h"
+#include "unix-std.h"
+#include "lua/lua_common.h"
+
+#include <math.h>
+
+struct config_json_buf {
+ GString *buf;
+ struct rspamd_config *cfg;
+};
+
+/**
+ * Apply configuration to the specified configuration
+ * @param conf_metrics
+ * @param cfg
+ */
+static void
+apply_dynamic_conf(const ucl_object_t *top, struct rspamd_config *cfg)
+{
+ enum rspamd_action_type test_act;
+ const ucl_object_t *cur_elt, *cur_nm, *it_val;
+ ucl_object_iter_t it = NULL;
+ const gchar *name;
+ gdouble nscore;
+ static const guint priority = 3;
+
+ while ((cur_elt = ucl_object_iterate(top, &it, true))) {
+ if (ucl_object_type(cur_elt) != UCL_OBJECT) {
+ msg_err("loaded json array element is not an object");
+ continue;
+ }
+
+ cur_nm = ucl_object_lookup(cur_elt, "metric");
+ if (!cur_nm || ucl_object_type(cur_nm) != UCL_STRING) {
+ msg_err(
+ "loaded json metric object element has no 'metric' attribute");
+ continue;
+ }
+
+ cur_nm = ucl_object_lookup(cur_elt, "symbols");
+ /* Parse symbols */
+ if (cur_nm && ucl_object_type(cur_nm) == UCL_ARRAY) {
+ ucl_object_iter_t nit = NULL;
+
+ while ((it_val = ucl_object_iterate(cur_nm, &nit, true))) {
+ if (ucl_object_lookup(it_val, "name") &&
+ ucl_object_lookup(it_val, "value")) {
+ const ucl_object_t *n =
+ ucl_object_lookup(it_val, "name");
+ const ucl_object_t *v =
+ ucl_object_lookup(it_val, "value");
+
+ nscore = ucl_object_todouble(v);
+
+ /*
+ * We use priority = 3 here
+ */
+ rspamd_config_add_symbol(cfg,
+ ucl_object_tostring(n), nscore, NULL, NULL,
+ 0, priority, cfg->default_max_shots);
+ }
+ else {
+ msg_info(
+ "json symbol object has no mandatory 'name' and 'value' attributes");
+ }
+ }
+ }
+ else {
+ ucl_object_t *arr;
+
+ arr = ucl_object_typed_new(UCL_ARRAY);
+ ucl_object_insert_key((ucl_object_t *) cur_elt, arr, "symbols",
+ sizeof("symbols") - 1, false);
+ }
+ cur_nm = ucl_object_lookup(cur_elt, "actions");
+ /* Parse actions */
+ if (cur_nm && ucl_object_type(cur_nm) == UCL_ARRAY) {
+ ucl_object_iter_t nit = NULL;
+
+ while ((it_val = ucl_object_iterate(cur_nm, &nit, true))) {
+ const ucl_object_t *n = ucl_object_lookup(it_val, "name");
+ const ucl_object_t *v = ucl_object_lookup(it_val, "value");
+
+ if (n != NULL && v != NULL) {
+ name = ucl_object_tostring(n);
+
+ if (!name || !rspamd_action_from_str(name, &test_act)) {
+ msg_err("unknown action: %s",
+ ucl_object_tostring(ucl_object_lookup(it_val,
+ "name")));
+ continue;
+ }
+
+
+ if (ucl_object_type(v) == UCL_NULL) {
+ nscore = NAN;
+ }
+ else {
+ nscore = ucl_object_todouble(v);
+ }
+
+ ucl_object_t *obj_tbl = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj_tbl, ucl_object_fromdouble(nscore),
+ "score", 0, false);
+ ucl_object_insert_key(obj_tbl, ucl_object_fromdouble(priority),
+ "priority", 0, false);
+ rspamd_config_set_action_score(cfg, name, obj_tbl);
+ ucl_object_unref(obj_tbl);
+ }
+ else {
+ msg_info(
+ "json action object has no mandatory 'name' and 'value' attributes");
+ }
+ }
+ }
+ else {
+ ucl_object_t *arr;
+
+ arr = ucl_object_typed_new(UCL_ARRAY);
+ ucl_object_insert_key((ucl_object_t *) cur_elt, arr, "actions",
+ sizeof("actions") - 1, false);
+ }
+ }
+}
+
+/* Callbacks for reading json dynamic rules */
+static gchar *
+json_config_read_cb(gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct config_json_buf *jb, *pd;
+
+ pd = data->prev_data;
+
+ g_assert(pd != NULL);
+
+ if (data->cur_data == NULL) {
+ jb = g_malloc0(sizeof(*jb));
+ jb->cfg = pd->cfg;
+ data->cur_data = jb;
+ }
+ else {
+ jb = data->cur_data;
+ }
+
+ if (jb->buf == NULL) {
+ /* Allocate memory for buffer */
+ jb->buf = g_string_sized_new(MAX(len, BUFSIZ));
+ }
+
+ g_string_append_len(jb->buf, chunk, len);
+
+ return NULL;
+}
+
+static void
+json_config_fin_cb(struct map_cb_data *data, void **target)
+{
+ struct config_json_buf *jb;
+ ucl_object_t *top;
+ struct ucl_parser *parser;
+
+ /* Now parse json */
+ if (data->cur_data) {
+ jb = data->cur_data;
+ }
+ else {
+ return;
+ }
+
+ if (jb->buf == NULL) {
+ msg_err("no data read");
+
+ return;
+ }
+
+ parser = ucl_parser_new(0);
+
+ if (!ucl_parser_add_chunk(parser, jb->buf->str, jb->buf->len)) {
+ msg_err("cannot load json data: parse error %s",
+ ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+ return;
+ }
+
+ top = ucl_parser_get_object(parser);
+ ucl_parser_free(parser);
+
+ if (ucl_object_type(top) != UCL_ARRAY) {
+ ucl_object_unref(top);
+ msg_err("loaded json is not an array");
+ return;
+ }
+
+ ucl_object_unref(jb->cfg->current_dynamic_conf);
+ apply_dynamic_conf(top, jb->cfg);
+ jb->cfg->current_dynamic_conf = top;
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ jb = data->prev_data;
+ /* Clean prev data */
+ if (jb->buf) {
+ g_string_free(jb->buf, TRUE);
+ }
+
+ g_free(jb);
+ }
+}
+
+static void
+json_config_dtor_cb(struct map_cb_data *data)
+{
+ struct config_json_buf *jb;
+
+ if (data->cur_data) {
+ jb = data->cur_data;
+ /* Clean prev data */
+ if (jb->buf) {
+ g_string_free(jb->buf, TRUE);
+ }
+
+ if (jb->cfg && jb->cfg->current_dynamic_conf) {
+ ucl_object_unref(jb->cfg->current_dynamic_conf);
+ }
+
+ g_free(jb);
+ }
+}
+
+/**
+ * Init dynamic configuration using map logic and specific configuration
+ * @param cfg config file
+ */
+void init_dynamic_config(struct rspamd_config *cfg)
+{
+ struct config_json_buf *jb, **pjb;
+
+ if (cfg->dynamic_conf == NULL) {
+ /* No dynamic conf has been specified, so do not try to load it */
+ return;
+ }
+
+ /* Now try to add map with json data */
+ jb = g_malloc(sizeof(struct config_json_buf));
+ pjb = g_malloc(sizeof(struct config_json_buf *));
+ jb->buf = NULL;
+ jb->cfg = cfg;
+ *pjb = jb;
+ cfg->current_dynamic_conf = ucl_object_typed_new(UCL_ARRAY);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) g_free,
+ pjb);
+
+ if (!rspamd_map_add(cfg,
+ cfg->dynamic_conf,
+ "Dynamic configuration map",
+ json_config_read_cb,
+ json_config_fin_cb,
+ json_config_dtor_cb,
+ (void **) pjb, NULL, RSPAMD_MAP_DEFAULT)) {
+ msg_err("cannot add map for configuration %s", cfg->dynamic_conf);
+ }
+}
+
+/**
+ * Dump dynamic configuration to the disk
+ * @param cfg
+ * @return
+ */
+gboolean
+dump_dynamic_config(struct rspamd_config *cfg)
+{
+ struct stat st;
+ gchar *dir, pathbuf[PATH_MAX];
+ gint fd;
+
+ if (cfg->dynamic_conf == NULL || cfg->current_dynamic_conf == NULL) {
+ /* No dynamic conf has been specified, so do not try to dump it */
+ msg_err("cannot save dynamic conf as it is not specified");
+ return FALSE;
+ }
+
+ dir = g_path_get_dirname(cfg->dynamic_conf);
+ if (dir == NULL) {
+ msg_err("invalid path: %s", cfg->dynamic_conf);
+ return FALSE;
+ }
+
+ if (stat(cfg->dynamic_conf, &st) == -1) {
+ msg_debug("%s is unavailable: %s", cfg->dynamic_conf,
+ strerror(errno));
+ st.st_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
+ }
+ if (access(dir, W_OK | R_OK) == -1) {
+ msg_warn("%s is inaccessible: %s", dir, strerror(errno));
+ g_free(dir);
+ return FALSE;
+ }
+ rspamd_snprintf(pathbuf,
+ sizeof(pathbuf),
+ "%s%crconf-XXXXXX",
+ dir,
+ G_DIR_SEPARATOR);
+ g_free(dir);
+#ifdef HAVE_MKSTEMP
+ /* Umask is set before */
+ fd = mkstemp(pathbuf);
+#else
+ fd = g_mkstemp_full(pathbuf, O_RDWR, S_IWUSR | S_IRUSR);
+#endif
+ if (fd == -1) {
+ msg_err("mkstemp error: %s", strerror(errno));
+
+ return FALSE;
+ }
+
+ struct ucl_emitter_functions *emitter_functions;
+ FILE *fp;
+
+ fp = fdopen(fd, "w");
+ emitter_functions = ucl_object_emit_file_funcs(fp);
+
+ if (!ucl_object_emit_full(cfg->current_dynamic_conf, UCL_EMIT_JSON,
+ emitter_functions, NULL)) {
+ msg_err("cannot emit ucl object: %s", strerror(errno));
+ ucl_object_emit_funcs_free(emitter_functions);
+ fclose(fp);
+ return FALSE;
+ }
+
+ (void) unlink(cfg->dynamic_conf);
+
+ /* Rename old config */
+ if (rename(pathbuf, cfg->dynamic_conf) == -1) {
+ msg_err("rename error: %s", strerror(errno));
+ fclose(fp);
+ ucl_object_emit_funcs_free(emitter_functions);
+ unlink(pathbuf);
+
+ return FALSE;
+ }
+ /* Set permissions */
+
+ if (chmod(cfg->dynamic_conf, st.st_mode) == -1) {
+ msg_warn("chmod failed: %s", strerror(errno));
+ }
+
+ fclose(fp);
+ ucl_object_emit_funcs_free(emitter_functions);
+
+ return TRUE;
+}
+
+static ucl_object_t *
+new_dynamic_metric(const gchar *metric_name, ucl_object_t *top)
+{
+ ucl_object_t *metric;
+
+ metric = ucl_object_typed_new(UCL_OBJECT);
+
+ ucl_object_insert_key(metric, ucl_object_fromstring(metric_name),
+ "metric", sizeof("metric") - 1, true);
+ ucl_object_insert_key(metric, ucl_object_typed_new(UCL_ARRAY),
+ "actions", sizeof("actions") - 1, false);
+ ucl_object_insert_key(metric, ucl_object_typed_new(UCL_ARRAY),
+ "symbols", sizeof("symbols") - 1, false);
+
+ ucl_array_append(top, metric);
+
+ return metric;
+}
+
+static ucl_object_t *
+dynamic_metric_find_elt(const ucl_object_t *arr, const gchar *name)
+{
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur, *n;
+
+ it = ucl_object_iterate_new(arr);
+
+ while ((cur = ucl_object_iterate_safe(it, true)) != NULL) {
+ if (cur->type == UCL_OBJECT) {
+ n = ucl_object_lookup(cur, "name");
+ if (n && n->type == UCL_STRING &&
+ strcmp(name, ucl_object_tostring(n)) == 0) {
+ ucl_object_iterate_free(it);
+
+ return (ucl_object_t *) ucl_object_lookup(cur, "value");
+ }
+ }
+ }
+
+ ucl_object_iterate_free(it);
+
+ return NULL;
+}
+
+static ucl_object_t *
+dynamic_metric_find_metric(const ucl_object_t *arr, const gchar *metric)
+{
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur, *n;
+
+ it = ucl_object_iterate_new(arr);
+
+ while ((cur = ucl_object_iterate_safe(it, true)) != NULL) {
+ if (cur->type == UCL_OBJECT) {
+ n = ucl_object_lookup(cur, "metric");
+ if (n && n->type == UCL_STRING &&
+ strcmp(metric, ucl_object_tostring(n)) == 0) {
+ ucl_object_iterate_free(it);
+
+ return (ucl_object_t *) cur;
+ }
+ }
+ }
+
+ ucl_object_iterate_free(it);
+
+ return NULL;
+}
+
+static ucl_object_t *
+new_dynamic_elt(ucl_object_t *arr, const gchar *name, gdouble value)
+{
+ ucl_object_t *n;
+
+ n = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(n, ucl_object_fromstring(name), "name",
+ sizeof("name") - 1, false);
+ ucl_object_insert_key(n, ucl_object_fromdouble(value), "value",
+ sizeof("value") - 1, false);
+
+ ucl_array_append(arr, n);
+
+ return n;
+}
+
+static gint
+rspamd_maybe_add_lua_dynsym(struct rspamd_config *cfg,
+ const gchar *sym,
+ gdouble score)
+{
+ lua_State *L = cfg->lua_state;
+ gint ret = -1;
+ struct rspamd_config **pcfg;
+
+ lua_getglobal(L, "rspamd_plugins");
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_pushstring(L, "dynamic_conf");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_pushstring(L, "add_symbol");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ *pcfg = cfg;
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ lua_pushstring(L, sym);
+ lua_pushnumber(L, score);
+
+ if (lua_pcall(L, 3, 1, 0) != 0) {
+ msg_err_config("cannot execute add_symbol script: %s",
+ lua_tostring(L, -1));
+ }
+ else {
+ ret = lua_toboolean(L, -1);
+ }
+
+ lua_pop(L, 1);
+ }
+ else {
+ lua_pop(L, 1);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+
+ return ret;
+}
+
+static gint
+rspamd_maybe_add_lua_dynact(struct rspamd_config *cfg,
+ const gchar *action,
+ gdouble score)
+{
+ lua_State *L = cfg->lua_state;
+ gint ret = -1;
+ struct rspamd_config **pcfg;
+
+ lua_getglobal(L, "rspamd_plugins");
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_pushstring(L, "dynamic_conf");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_pushstring(L, "add_action");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ *pcfg = cfg;
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ lua_pushstring(L, action);
+ lua_pushnumber(L, score);
+
+ if (lua_pcall(L, 3, 1, 0) != 0) {
+ msg_err_config("cannot execute add_action script: %s",
+ lua_tostring(L, -1));
+ }
+ else {
+ ret = lua_toboolean(L, -1);
+ }
+
+ lua_pop(L, 1);
+ }
+ else {
+ lua_pop(L, 1);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+
+ return ret;
+}
+
+/**
+ * Add symbol for specified metric
+ * @param cfg config file object
+ * @param metric metric's name
+ * @param symbol symbol's name
+ * @param value value of symbol
+ * @return
+ */
+gboolean
+add_dynamic_symbol(struct rspamd_config *cfg,
+ const gchar *metric_name,
+ const gchar *symbol,
+ gdouble value)
+{
+ ucl_object_t *metric, *syms;
+ gint ret;
+
+ if ((ret = rspamd_maybe_add_lua_dynsym(cfg, symbol, value)) != -1) {
+ return ret == 0 ? FALSE : TRUE;
+ }
+
+ if (cfg->dynamic_conf == NULL) {
+ msg_info("dynamic conf is disabled");
+ return FALSE;
+ }
+
+ metric = dynamic_metric_find_metric(cfg->current_dynamic_conf,
+ metric_name);
+ if (metric == NULL) {
+ metric = new_dynamic_metric(metric_name, cfg->current_dynamic_conf);
+ }
+
+ syms = (ucl_object_t *) ucl_object_lookup(metric, "symbols");
+ if (syms != NULL) {
+ ucl_object_t *sym;
+
+ sym = dynamic_metric_find_elt(syms, symbol);
+ if (sym) {
+ sym->value.dv = value;
+ }
+ else {
+ new_dynamic_elt(syms, symbol, value);
+ }
+ }
+
+ apply_dynamic_conf(cfg->current_dynamic_conf, cfg);
+
+ return TRUE;
+}
+
+gboolean
+remove_dynamic_symbol(struct rspamd_config *cfg,
+ const gchar *metric_name,
+ const gchar *symbol)
+{
+ ucl_object_t *metric, *syms;
+ gboolean ret = FALSE;
+
+ if (cfg->dynamic_conf == NULL) {
+ msg_info("dynamic conf is disabled");
+ return FALSE;
+ }
+
+ metric = dynamic_metric_find_metric(cfg->current_dynamic_conf,
+ metric_name);
+ if (metric == NULL) {
+ return FALSE;
+ }
+
+ syms = (ucl_object_t *) ucl_object_lookup(metric, "symbols");
+ if (syms != NULL) {
+ ucl_object_t *sym;
+
+ sym = dynamic_metric_find_elt(syms, symbol);
+
+ if (sym) {
+ ret = ucl_array_delete((ucl_object_t *) syms, sym) != NULL;
+
+ if (ret) {
+ ucl_object_unref(sym);
+ }
+ }
+ }
+
+ if (ret) {
+ apply_dynamic_conf(cfg->current_dynamic_conf, cfg);
+ }
+
+ return ret;
+}
+
+
+/**
+ * Add action for specified metric
+ * @param cfg config file object
+ * @param metric metric's name
+ * @param action action's name
+ * @param value value of symbol
+ * @return
+ */
+gboolean
+add_dynamic_action(struct rspamd_config *cfg,
+ const gchar *metric_name,
+ guint action,
+ gdouble value)
+{
+ ucl_object_t *metric, *acts;
+ const gchar *action_name = rspamd_action_to_str(action);
+ gint ret;
+
+ if ((ret = rspamd_maybe_add_lua_dynact(cfg, action_name, value)) != -1) {
+ return ret == 0 ? FALSE : TRUE;
+ }
+
+ if (cfg->dynamic_conf == NULL) {
+ msg_info("dynamic conf is disabled");
+ return FALSE;
+ }
+
+ metric = dynamic_metric_find_metric(cfg->current_dynamic_conf,
+ metric_name);
+ if (metric == NULL) {
+ metric = new_dynamic_metric(metric_name, cfg->current_dynamic_conf);
+ }
+
+ acts = (ucl_object_t *) ucl_object_lookup(metric, "actions");
+ if (acts != NULL) {
+ ucl_object_t *act;
+
+ act = dynamic_metric_find_elt(acts, action_name);
+ if (act) {
+ act->value.dv = value;
+ }
+ else {
+ new_dynamic_elt(acts, action_name, value);
+ }
+ }
+
+ apply_dynamic_conf(cfg->current_dynamic_conf, cfg);
+
+ return TRUE;
+}
+
+gboolean
+remove_dynamic_action(struct rspamd_config *cfg,
+ const gchar *metric_name,
+ guint action)
+{
+ ucl_object_t *metric, *acts;
+ const gchar *action_name = rspamd_action_to_str(action);
+ gboolean ret = FALSE;
+
+ if (cfg->dynamic_conf == NULL) {
+ msg_info("dynamic conf is disabled");
+ return FALSE;
+ }
+
+ metric = dynamic_metric_find_metric(cfg->current_dynamic_conf,
+ metric_name);
+ if (metric == NULL) {
+ return FALSE;
+ }
+
+ acts = (ucl_object_t *) ucl_object_lookup(metric, "actions");
+
+ if (acts != NULL) {
+ ucl_object_t *act;
+
+ act = dynamic_metric_find_elt(acts, action_name);
+
+ if (act) {
+ ret = ucl_array_delete(acts, act) != NULL;
+ }
+ if (ret) {
+ ucl_object_unref(act);
+ }
+ }
+
+ if (ret) {
+ apply_dynamic_conf(cfg->current_dynamic_conf, cfg);
+ }
+
+ return ret;
+}
diff --git a/src/libserver/dynamic_cfg.h b/src/libserver/dynamic_cfg.h
new file mode 100644
index 0000000..bb386ca
--- /dev/null
+++ b/src/libserver/dynamic_cfg.h
@@ -0,0 +1,81 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef DYNAMIC_CFG_H_
+#define DYNAMIC_CFG_H_
+
+#include "config.h"
+#include "cfg_file.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Init dynamic configuration using map logic and specific configuration
+ * @param cfg config file
+ */
+void init_dynamic_config(struct rspamd_config *cfg);
+
+/**
+ * Dump dynamic configuration to the disk
+ * @param cfg
+ * @return
+ */
+gboolean dump_dynamic_config(struct rspamd_config *cfg);
+
+/**
+ * Add symbol for specified metric
+ * @param cfg config file object
+ * @param metric metric's name
+ * @param symbol symbol's name
+ * @param value value of symbol
+ * @return
+ */
+gboolean add_dynamic_symbol(struct rspamd_config *cfg,
+ const gchar *metric,
+ const gchar *symbol,
+ gdouble value);
+
+gboolean remove_dynamic_symbol(struct rspamd_config *cfg,
+ const gchar *metric,
+ const gchar *symbol);
+
+/**
+ * Add action for specified metric
+ * @param cfg config file object
+ * @param metric metric's name
+ * @param action action's name
+ * @param value value of symbol
+ * @return
+ */
+gboolean add_dynamic_action(struct rspamd_config *cfg,
+ const gchar *metric,
+ guint action,
+ gdouble value);
+
+/**
+ * Removes dynamic action
+ */
+gboolean remove_dynamic_action(struct rspamd_config *cfg,
+ const gchar *metric,
+ guint action);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* DYNAMIC_CFG_H_ */
diff --git a/src/libserver/fuzzy_backend/fuzzy_backend.c b/src/libserver/fuzzy_backend/fuzzy_backend.c
new file mode 100644
index 0000000..9099f38
--- /dev/null
+++ b/src/libserver/fuzzy_backend/fuzzy_backend.c
@@ -0,0 +1,560 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "fuzzy_backend.h"
+#include "fuzzy_backend_sqlite.h"
+#include "fuzzy_backend_redis.h"
+#include "cfg_file.h"
+#include "fuzzy_wire.h"
+
+#define DEFAULT_EXPIRE 172800L
+
+enum rspamd_fuzzy_backend_type {
+ RSPAMD_FUZZY_BACKEND_SQLITE = 0,
+ RSPAMD_FUZZY_BACKEND_REDIS = 1,
+};
+
+static void *rspamd_fuzzy_backend_init_sqlite(struct rspamd_fuzzy_backend *bk,
+ const ucl_object_t *obj, struct rspamd_config *cfg, GError **err);
+static void rspamd_fuzzy_backend_check_sqlite(struct rspamd_fuzzy_backend *bk,
+ const struct rspamd_fuzzy_cmd *cmd,
+ rspamd_fuzzy_check_cb cb, void *ud,
+ void *subr_ud);
+static void rspamd_fuzzy_backend_update_sqlite(struct rspamd_fuzzy_backend *bk,
+ GArray *updates, const gchar *src,
+ rspamd_fuzzy_update_cb cb, void *ud,
+ void *subr_ud);
+static void rspamd_fuzzy_backend_count_sqlite(struct rspamd_fuzzy_backend *bk,
+ rspamd_fuzzy_count_cb cb, void *ud,
+ void *subr_ud);
+static void rspamd_fuzzy_backend_version_sqlite(struct rspamd_fuzzy_backend *bk,
+ const gchar *src,
+ rspamd_fuzzy_version_cb cb, void *ud,
+ void *subr_ud);
+static const gchar *rspamd_fuzzy_backend_id_sqlite(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud);
+static void rspamd_fuzzy_backend_expire_sqlite(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud);
+static void rspamd_fuzzy_backend_close_sqlite(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud);
+
+struct rspamd_fuzzy_backend_subr {
+ void *(*init)(struct rspamd_fuzzy_backend *bk, const ucl_object_t *obj,
+ struct rspamd_config *cfg,
+ GError **err);
+ void (*check)(struct rspamd_fuzzy_backend *bk,
+ const struct rspamd_fuzzy_cmd *cmd,
+ rspamd_fuzzy_check_cb cb, void *ud,
+ void *subr_ud);
+ void (*update)(struct rspamd_fuzzy_backend *bk,
+ GArray *updates, const gchar *src,
+ rspamd_fuzzy_update_cb cb, void *ud,
+ void *subr_ud);
+ void (*count)(struct rspamd_fuzzy_backend *bk,
+ rspamd_fuzzy_count_cb cb, void *ud,
+ void *subr_ud);
+ void (*version)(struct rspamd_fuzzy_backend *bk,
+ const gchar *src,
+ rspamd_fuzzy_version_cb cb, void *ud,
+ void *subr_ud);
+ const gchar *(*id)(struct rspamd_fuzzy_backend *bk, void *subr_ud);
+ void (*periodic)(struct rspamd_fuzzy_backend *bk, void *subr_ud);
+ void (*close)(struct rspamd_fuzzy_backend *bk, void *subr_ud);
+};
+
+static const struct rspamd_fuzzy_backend_subr fuzzy_subrs[] = {
+ [RSPAMD_FUZZY_BACKEND_SQLITE] = {
+ .init = rspamd_fuzzy_backend_init_sqlite,
+ .check = rspamd_fuzzy_backend_check_sqlite,
+ .update = rspamd_fuzzy_backend_update_sqlite,
+ .count = rspamd_fuzzy_backend_count_sqlite,
+ .version = rspamd_fuzzy_backend_version_sqlite,
+ .id = rspamd_fuzzy_backend_id_sqlite,
+ .periodic = rspamd_fuzzy_backend_expire_sqlite,
+ .close = rspamd_fuzzy_backend_close_sqlite,
+ },
+ [RSPAMD_FUZZY_BACKEND_REDIS] = {
+ .init = rspamd_fuzzy_backend_init_redis,
+ .check = rspamd_fuzzy_backend_check_redis,
+ .update = rspamd_fuzzy_backend_update_redis,
+ .count = rspamd_fuzzy_backend_count_redis,
+ .version = rspamd_fuzzy_backend_version_redis,
+ .id = rspamd_fuzzy_backend_id_redis,
+ .periodic = rspamd_fuzzy_backend_expire_redis,
+ .close = rspamd_fuzzy_backend_close_redis,
+ }};
+
+struct rspamd_fuzzy_backend {
+ enum rspamd_fuzzy_backend_type type;
+ gdouble expire;
+ gdouble sync;
+ struct ev_loop *event_loop;
+ rspamd_fuzzy_periodic_cb periodic_cb;
+ void *periodic_ud;
+ const struct rspamd_fuzzy_backend_subr *subr;
+ void *subr_ud;
+ ev_timer periodic_event;
+};
+
+static GQuark
+rspamd_fuzzy_backend_quark(void)
+{
+ return g_quark_from_static_string("fuzzy-backend");
+}
+
+static void *
+rspamd_fuzzy_backend_init_sqlite(struct rspamd_fuzzy_backend *bk,
+ const ucl_object_t *obj, struct rspamd_config *cfg, GError **err)
+{
+ const ucl_object_t *elt;
+
+ elt = ucl_object_lookup_any(obj, "hashfile", "hash_file", "file",
+ "database", NULL);
+
+ if (elt == NULL || ucl_object_type(elt) != UCL_STRING) {
+ g_set_error(err, rspamd_fuzzy_backend_quark(),
+ EINVAL, "missing sqlite3 path");
+ return NULL;
+ }
+
+ return rspamd_fuzzy_backend_sqlite_open(ucl_object_tostring(elt),
+ FALSE, err);
+}
+
+static void
+rspamd_fuzzy_backend_check_sqlite(struct rspamd_fuzzy_backend *bk,
+ const struct rspamd_fuzzy_cmd *cmd,
+ rspamd_fuzzy_check_cb cb, void *ud,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_sqlite *sq = subr_ud;
+ struct rspamd_fuzzy_reply rep;
+
+ rep = rspamd_fuzzy_backend_sqlite_check(sq, cmd, bk->expire);
+
+ if (cb) {
+ cb(&rep, ud);
+ }
+}
+
+static void
+rspamd_fuzzy_backend_update_sqlite(struct rspamd_fuzzy_backend *bk,
+ GArray *updates, const gchar *src,
+ rspamd_fuzzy_update_cb cb, void *ud,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_sqlite *sq = subr_ud;
+ gboolean success = FALSE;
+ guint i;
+ struct fuzzy_peer_cmd *io_cmd;
+ struct rspamd_fuzzy_cmd *cmd;
+ gpointer ptr;
+ guint nupdates = 0, nadded = 0, ndeleted = 0, nextended = 0, nignored = 0;
+
+ if (rspamd_fuzzy_backend_sqlite_prepare_update(sq, src)) {
+ for (i = 0; i < updates->len; i++) {
+ io_cmd = &g_array_index(updates, struct fuzzy_peer_cmd, i);
+
+ if (io_cmd->is_shingle) {
+ cmd = &io_cmd->cmd.shingle.basic;
+ ptr = &io_cmd->cmd.shingle;
+ }
+ else {
+ cmd = &io_cmd->cmd.normal;
+ ptr = &io_cmd->cmd.normal;
+ }
+
+ if (cmd->cmd == FUZZY_WRITE) {
+ rspamd_fuzzy_backend_sqlite_add(sq, ptr);
+ nadded++;
+ nupdates++;
+ }
+ else if (cmd->cmd == FUZZY_DEL) {
+ rspamd_fuzzy_backend_sqlite_del(sq, ptr);
+ ndeleted++;
+ nupdates++;
+ }
+ else {
+ if (cmd->cmd == FUZZY_REFRESH) {
+ nextended++;
+ }
+ else {
+ nignored++;
+ }
+ }
+ }
+
+ if (rspamd_fuzzy_backend_sqlite_finish_update(sq, src,
+ nupdates > 0)) {
+ success = TRUE;
+ }
+ }
+
+ if (cb) {
+ cb(success, nadded, ndeleted, nextended, nignored, ud);
+ }
+}
+
+static void
+rspamd_fuzzy_backend_count_sqlite(struct rspamd_fuzzy_backend *bk,
+ rspamd_fuzzy_count_cb cb, void *ud,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_sqlite *sq = subr_ud;
+ guint64 nhashes;
+
+ nhashes = rspamd_fuzzy_backend_sqlite_count(sq);
+
+ if (cb) {
+ cb(nhashes, ud);
+ }
+}
+
+static void
+rspamd_fuzzy_backend_version_sqlite(struct rspamd_fuzzy_backend *bk,
+ const gchar *src,
+ rspamd_fuzzy_version_cb cb, void *ud,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_sqlite *sq = subr_ud;
+ guint64 rev;
+
+ rev = rspamd_fuzzy_backend_sqlite_version(sq, src);
+
+ if (cb) {
+ cb(rev, ud);
+ }
+}
+
+static const gchar *
+rspamd_fuzzy_backend_id_sqlite(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_sqlite *sq = subr_ud;
+
+ return rspamd_fuzzy_sqlite_backend_id(sq);
+}
+static void
+rspamd_fuzzy_backend_expire_sqlite(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_sqlite *sq = subr_ud;
+
+ rspamd_fuzzy_backend_sqlite_sync(sq, bk->expire, TRUE);
+}
+
+static void
+rspamd_fuzzy_backend_close_sqlite(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_sqlite *sq = subr_ud;
+
+ rspamd_fuzzy_backend_sqlite_close(sq);
+}
+
+
+struct rspamd_fuzzy_backend *
+rspamd_fuzzy_backend_create(struct ev_loop *ev_base,
+ const ucl_object_t *config,
+ struct rspamd_config *cfg,
+ GError **err)
+{
+ struct rspamd_fuzzy_backend *bk;
+ enum rspamd_fuzzy_backend_type type = RSPAMD_FUZZY_BACKEND_SQLITE;
+ const ucl_object_t *elt;
+ gdouble expire = DEFAULT_EXPIRE;
+
+ if (config != NULL) {
+ elt = ucl_object_lookup(config, "backend");
+
+ if (elt != NULL && ucl_object_type(elt) == UCL_STRING) {
+ if (strcmp(ucl_object_tostring(elt), "sqlite") == 0) {
+ type = RSPAMD_FUZZY_BACKEND_SQLITE;
+ }
+ else if (strcmp(ucl_object_tostring(elt), "redis") == 0) {
+ type = RSPAMD_FUZZY_BACKEND_REDIS;
+ }
+ else {
+ g_set_error(err, rspamd_fuzzy_backend_quark(),
+ EINVAL, "invalid backend type: %s",
+ ucl_object_tostring(elt));
+ return NULL;
+ }
+ }
+
+ elt = ucl_object_lookup(config, "expire");
+
+ if (elt != NULL) {
+ expire = ucl_object_todouble(elt);
+ }
+ }
+
+ bk = g_malloc0(sizeof(*bk));
+ bk->event_loop = ev_base;
+ bk->expire = expire;
+ bk->type = type;
+ bk->subr = &fuzzy_subrs[type];
+
+ if ((bk->subr_ud = bk->subr->init(bk, config, cfg, err)) == NULL) {
+ g_free(bk);
+
+ return NULL;
+ }
+
+ return bk;
+}
+
+
+void rspamd_fuzzy_backend_check(struct rspamd_fuzzy_backend *bk,
+ const struct rspamd_fuzzy_cmd *cmd,
+ rspamd_fuzzy_check_cb cb, void *ud)
+{
+ g_assert(bk != NULL);
+
+ bk->subr->check(bk, cmd, cb, ud, bk->subr_ud);
+}
+
+static guint
+rspamd_fuzzy_digest_hash(gconstpointer key)
+{
+ guint ret;
+
+ /* Distributed uniformly already */
+ memcpy(&ret, key, sizeof(ret));
+
+ return ret;
+}
+
+static gboolean
+rspamd_fuzzy_digest_equal(gconstpointer v, gconstpointer v2)
+{
+ return memcmp(v, v2, rspamd_cryptobox_HASHBYTES) == 0;
+}
+
+static void
+rspamd_fuzzy_backend_deduplicate_queue(GArray *updates)
+{
+ GHashTable *seen = g_hash_table_new(rspamd_fuzzy_digest_hash,
+ rspamd_fuzzy_digest_equal);
+ struct fuzzy_peer_cmd *io_cmd, *found;
+ struct rspamd_fuzzy_cmd *cmd;
+ guchar *digest;
+ guint i;
+
+ for (i = 0; i < updates->len; i++) {
+ io_cmd = &g_array_index(updates, struct fuzzy_peer_cmd, i);
+
+ if (io_cmd->is_shingle) {
+ cmd = &io_cmd->cmd.shingle.basic;
+ }
+ else {
+ cmd = &io_cmd->cmd.normal;
+ }
+
+ digest = cmd->digest;
+
+ found = g_hash_table_lookup(seen, digest);
+
+ if (found == NULL) {
+ /* Add to the seen list, if not a duplicate (huh?) */
+ if (cmd->cmd != FUZZY_DUP) {
+ g_hash_table_insert(seen, digest, io_cmd);
+ }
+ }
+ else {
+ if (found->cmd.normal.flag != cmd->flag) {
+ /* TODO: deal with flags better at some point */
+ continue;
+ }
+
+ /* Apply heuristic */
+ switch (cmd->cmd) {
+ case FUZZY_WRITE:
+ if (found->cmd.normal.cmd == FUZZY_WRITE) {
+ /* Already seen */
+ found->cmd.normal.value += cmd->value;
+ cmd->cmd = FUZZY_DUP; /* Ignore this one */
+ }
+ else if (found->cmd.normal.cmd == FUZZY_REFRESH) {
+ /* Seen refresh command, remove it as write has higher priority */
+ g_hash_table_replace(seen, digest, io_cmd);
+ found->cmd.normal.cmd = FUZZY_DUP;
+ }
+ else if (found->cmd.normal.cmd == FUZZY_DEL) {
+ /* Request delete + add, weird, but ignore add */
+ cmd->cmd = FUZZY_DUP; /* Ignore this one */
+ }
+ break;
+ case FUZZY_REFRESH:
+ if (found->cmd.normal.cmd == FUZZY_WRITE) {
+ /* No need to expire, handled by addition */
+ cmd->cmd = FUZZY_DUP; /* Ignore this one */
+ }
+ else if (found->cmd.normal.cmd == FUZZY_DEL) {
+ /* Request delete + expire, ignore expire */
+ cmd->cmd = FUZZY_DUP; /* Ignore this one */
+ }
+ else if (found->cmd.normal.cmd == FUZZY_REFRESH) {
+ /* Already handled */
+ cmd->cmd = FUZZY_DUP; /* Ignore this one */
+ }
+ break;
+ case FUZZY_DEL:
+ /* Delete has priority over all other commands */
+ g_hash_table_replace(seen, digest, io_cmd);
+ found->cmd.normal.cmd = FUZZY_DUP;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ g_hash_table_unref(seen);
+}
+
+void rspamd_fuzzy_backend_process_updates(struct rspamd_fuzzy_backend *bk,
+ GArray *updates, const gchar *src, rspamd_fuzzy_update_cb cb,
+ void *ud)
+{
+ g_assert(bk != NULL);
+ g_assert(updates != NULL);
+
+ if (updates) {
+ rspamd_fuzzy_backend_deduplicate_queue(updates);
+ bk->subr->update(bk, updates, src, cb, ud, bk->subr_ud);
+ }
+ else if (cb) {
+ cb(TRUE, 0, 0, 0, 0, ud);
+ }
+}
+
+
+void rspamd_fuzzy_backend_count(struct rspamd_fuzzy_backend *bk,
+ rspamd_fuzzy_count_cb cb, void *ud)
+{
+ g_assert(bk != NULL);
+
+ bk->subr->count(bk, cb, ud, bk->subr_ud);
+}
+
+
+void rspamd_fuzzy_backend_version(struct rspamd_fuzzy_backend *bk,
+ const gchar *src,
+ rspamd_fuzzy_version_cb cb, void *ud)
+{
+ g_assert(bk != NULL);
+
+ bk->subr->version(bk, src, cb, ud, bk->subr_ud);
+}
+
+const gchar *
+rspamd_fuzzy_backend_id(struct rspamd_fuzzy_backend *bk)
+{
+ g_assert(bk != NULL);
+
+ if (bk->subr->id) {
+ return bk->subr->id(bk, bk->subr_ud);
+ }
+
+ return NULL;
+}
+
+static inline void
+rspamd_fuzzy_backend_periodic_sync(struct rspamd_fuzzy_backend *bk)
+{
+ if (bk->periodic_cb) {
+ if (bk->periodic_cb(bk->periodic_ud)) {
+ if (bk->subr->periodic) {
+ bk->subr->periodic(bk, bk->subr_ud);
+ }
+ }
+ }
+ else {
+ if (bk->subr->periodic) {
+ bk->subr->periodic(bk, bk->subr_ud);
+ }
+ }
+}
+
+static void
+rspamd_fuzzy_backend_periodic_cb(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_fuzzy_backend *bk = (struct rspamd_fuzzy_backend *) w->data;
+ gdouble jittered;
+
+ jittered = rspamd_time_jitter(bk->sync, bk->sync / 2.0);
+ w->repeat = jittered;
+ rspamd_fuzzy_backend_periodic_sync(bk);
+ ev_timer_again(EV_A_ w);
+}
+
+void rspamd_fuzzy_backend_start_update(struct rspamd_fuzzy_backend *bk,
+ gdouble timeout,
+ rspamd_fuzzy_periodic_cb cb,
+ void *ud)
+{
+ gdouble jittered;
+
+ g_assert(bk != NULL);
+
+ if (bk->subr->periodic) {
+ if (bk->sync > 0.0) {
+ ev_timer_stop(bk->event_loop, &bk->periodic_event);
+ }
+
+ if (cb) {
+ bk->periodic_cb = cb;
+ bk->periodic_ud = ud;
+ }
+
+ rspamd_fuzzy_backend_periodic_sync(bk);
+ bk->sync = timeout;
+ jittered = rspamd_time_jitter(timeout, timeout / 2.0);
+
+ bk->periodic_event.data = bk;
+ ev_timer_init(&bk->periodic_event, rspamd_fuzzy_backend_periodic_cb,
+ jittered, 0.0);
+ ev_timer_start(bk->event_loop, &bk->periodic_event);
+ }
+}
+
+void rspamd_fuzzy_backend_close(struct rspamd_fuzzy_backend *bk)
+{
+ g_assert(bk != NULL);
+
+ if (bk->sync > 0.0) {
+ rspamd_fuzzy_backend_periodic_sync(bk);
+ ev_timer_stop(bk->event_loop, &bk->periodic_event);
+ }
+
+ bk->subr->close(bk, bk->subr_ud);
+
+ g_free(bk);
+}
+
+struct ev_loop *
+rspamd_fuzzy_backend_event_base(struct rspamd_fuzzy_backend *backend)
+{
+ return backend->event_loop;
+}
+
+gdouble
+rspamd_fuzzy_backend_get_expire(struct rspamd_fuzzy_backend *backend)
+{
+ return backend->expire;
+}
diff --git a/src/libserver/fuzzy_backend/fuzzy_backend.h b/src/libserver/fuzzy_backend/fuzzy_backend.h
new file mode 100644
index 0000000..a1b74bc
--- /dev/null
+++ b/src/libserver/fuzzy_backend/fuzzy_backend.h
@@ -0,0 +1,131 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBSERVER_FUZZY_BACKEND_H_
+#define SRC_LIBSERVER_FUZZY_BACKEND_H_
+
+#include "config.h"
+#include "contrib/libev/ev.h"
+#include "fuzzy_wire.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_fuzzy_backend;
+struct rspamd_config;
+
+/*
+ * Callbacks for fuzzy methods
+ */
+typedef void (*rspamd_fuzzy_check_cb)(struct rspamd_fuzzy_reply *rep, void *ud);
+
+typedef void (*rspamd_fuzzy_update_cb)(gboolean success,
+ guint nadded,
+ guint ndeleted,
+ guint nextended,
+ guint nignored,
+ void *ud);
+
+typedef void (*rspamd_fuzzy_version_cb)(guint64 rev, void *ud);
+
+typedef void (*rspamd_fuzzy_count_cb)(guint64 count, void *ud);
+
+typedef gboolean (*rspamd_fuzzy_periodic_cb)(void *ud);
+
+/**
+ * Open fuzzy backend
+ * @param ev_base
+ * @param config
+ * @param err
+ * @return
+ */
+struct rspamd_fuzzy_backend *rspamd_fuzzy_backend_create(struct ev_loop *ev_base,
+ const ucl_object_t *config,
+ struct rspamd_config *cfg,
+ GError **err);
+
+
+/**
+ * Check a specific hash in storage
+ * @param cmd
+ * @param cb
+ * @param ud
+ */
+void rspamd_fuzzy_backend_check(struct rspamd_fuzzy_backend *bk,
+ const struct rspamd_fuzzy_cmd *cmd,
+ rspamd_fuzzy_check_cb cb, void *ud);
+
+/**
+ * Process updates for a specific queue
+ * @param bk
+ * @param updates queue of struct fuzzy_peer_cmd
+ * @param src
+ */
+void rspamd_fuzzy_backend_process_updates(struct rspamd_fuzzy_backend *bk,
+ GArray *updates, const gchar *src, rspamd_fuzzy_update_cb cb,
+ void *ud);
+
+/**
+ * Gets number of hashes from the backend
+ * @param bk
+ * @param cb
+ * @param ud
+ */
+void rspamd_fuzzy_backend_count(struct rspamd_fuzzy_backend *bk,
+ rspamd_fuzzy_count_cb cb, void *ud);
+
+/**
+ * Returns number of revision for a specific source
+ * @param bk
+ * @param src
+ * @param cb
+ * @param ud
+ */
+void rspamd_fuzzy_backend_version(struct rspamd_fuzzy_backend *bk,
+ const gchar *src,
+ rspamd_fuzzy_version_cb cb, void *ud);
+
+/**
+ * Returns unique id for backend
+ * @param backend
+ * @return
+ */
+const gchar *rspamd_fuzzy_backend_id(struct rspamd_fuzzy_backend *backend);
+
+/**
+ * Starts expire process for the backend
+ * @param backend
+ */
+void rspamd_fuzzy_backend_start_update(struct rspamd_fuzzy_backend *backend,
+ gdouble timeout,
+ rspamd_fuzzy_periodic_cb cb,
+ void *ud);
+
+struct ev_loop *rspamd_fuzzy_backend_event_base(struct rspamd_fuzzy_backend *backend);
+
+gdouble rspamd_fuzzy_backend_get_expire(struct rspamd_fuzzy_backend *backend);
+
+/**
+ * Closes backend
+ * @param backend
+ */
+void rspamd_fuzzy_backend_close(struct rspamd_fuzzy_backend *backend);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBSERVER_FUZZY_BACKEND_H_ */
diff --git a/src/libserver/fuzzy_backend/fuzzy_backend_redis.c b/src/libserver/fuzzy_backend/fuzzy_backend_redis.c
new file mode 100644
index 0000000..7ab7ca6
--- /dev/null
+++ b/src/libserver/fuzzy_backend/fuzzy_backend_redis.c
@@ -0,0 +1,1666 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "ref.h"
+#include "fuzzy_backend.h"
+#include "fuzzy_backend_redis.h"
+#include "redis_pool.h"
+#include "cryptobox.h"
+#include "str_util.h"
+#include "upstream.h"
+#include "contrib/hiredis/hiredis.h"
+#include "contrib/hiredis/async.h"
+#include "lua/lua_common.h"
+
+#define REDIS_DEFAULT_PORT 6379
+#define REDIS_DEFAULT_OBJECT "fuzzy"
+#define REDIS_DEFAULT_TIMEOUT 2.0
+
+#define msg_err_redis_session(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "fuzzy_redis", session->backend->id, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_warn_redis_session(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "fuzzy_redis", session->backend->id, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_info_redis_session(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "fuzzy_redis", session->backend->id, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_debug_redis_session(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_fuzzy_redis_log_id, "fuzzy_redis", session->backend->id, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(fuzzy_redis)
+
+struct rspamd_fuzzy_backend_redis {
+ lua_State *L;
+ const gchar *redis_object;
+ const gchar *username;
+ const gchar *password;
+ const gchar *dbname;
+ gchar *id;
+ struct rspamd_redis_pool *pool;
+ gdouble timeout;
+ gint conf_ref;
+ bool terminated;
+ ref_entry_t ref;
+};
+
+enum rspamd_fuzzy_redis_command {
+ RSPAMD_FUZZY_REDIS_COMMAND_COUNT,
+ RSPAMD_FUZZY_REDIS_COMMAND_VERSION,
+ RSPAMD_FUZZY_REDIS_COMMAND_UPDATES,
+ RSPAMD_FUZZY_REDIS_COMMAND_CHECK
+};
+
+struct rspamd_fuzzy_redis_session {
+ struct rspamd_fuzzy_backend_redis *backend;
+ redisAsyncContext *ctx;
+ ev_timer timeout;
+ const struct rspamd_fuzzy_cmd *cmd;
+ struct ev_loop *event_loop;
+ float prob;
+ gboolean shingles_checked;
+
+ enum rspamd_fuzzy_redis_command command;
+ guint nargs;
+
+ guint nadded;
+ guint ndeleted;
+ guint nextended;
+ guint nignored;
+
+ union {
+ rspamd_fuzzy_check_cb cb_check;
+ rspamd_fuzzy_update_cb cb_update;
+ rspamd_fuzzy_version_cb cb_version;
+ rspamd_fuzzy_count_cb cb_count;
+ } callback;
+ void *cbdata;
+
+ gchar **argv;
+ gsize *argv_lens;
+ struct upstream *up;
+ guchar found_digest[rspamd_cryptobox_HASHBYTES];
+};
+
+static inline struct upstream_list *
+rspamd_redis_get_servers(struct rspamd_fuzzy_backend_redis *ctx,
+ const gchar *what)
+{
+ lua_State *L = ctx->L;
+ struct upstream_list *res = NULL;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, ctx->conf_ref);
+ lua_pushstring(L, what);
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ res = *((struct upstream_list **) lua_touserdata(L, -1));
+ }
+ else {
+ struct lua_logger_trace tr;
+ gchar outbuf[8192];
+
+ memset(&tr, 0, sizeof(tr));
+ lua_logger_out_type(L, -2, outbuf, sizeof(outbuf) - 1, &tr,
+ LUA_ESCAPE_UNPRINTABLE);
+
+ msg_err("cannot get %s upstreams for Redis fuzzy storage %s; table content: %s",
+ what, ctx->id, outbuf);
+ }
+
+ lua_settop(L, 0);
+
+ return res;
+}
+
+static inline void
+rspamd_fuzzy_redis_session_free_args(struct rspamd_fuzzy_redis_session *session)
+{
+ guint i;
+
+ if (session->argv) {
+ for (i = 0; i < session->nargs; i++) {
+ g_free(session->argv[i]);
+ }
+
+ g_free(session->argv);
+ g_free(session->argv_lens);
+ }
+}
+static void
+rspamd_fuzzy_redis_session_dtor(struct rspamd_fuzzy_redis_session *session,
+ gboolean is_fatal)
+{
+ redisAsyncContext *ac;
+
+
+ if (session->ctx) {
+ ac = session->ctx;
+ session->ctx = NULL;
+ rspamd_redis_pool_release_connection(session->backend->pool,
+ ac,
+ is_fatal ? RSPAMD_REDIS_RELEASE_FATAL : RSPAMD_REDIS_RELEASE_DEFAULT);
+ }
+
+ ev_timer_stop(session->event_loop, &session->timeout);
+ rspamd_fuzzy_redis_session_free_args(session);
+
+ REF_RELEASE(session->backend);
+ rspamd_upstream_unref(session->up);
+ g_free(session);
+}
+
+static void
+rspamd_fuzzy_backend_redis_dtor(struct rspamd_fuzzy_backend_redis *backend)
+{
+ if (!backend->terminated && backend->conf_ref != -1) {
+ luaL_unref(backend->L, LUA_REGISTRYINDEX, backend->conf_ref);
+ }
+
+ if (backend->id) {
+ g_free(backend->id);
+ }
+
+ g_free(backend);
+}
+
+void *
+rspamd_fuzzy_backend_init_redis(struct rspamd_fuzzy_backend *bk,
+ const ucl_object_t *obj, struct rspamd_config *cfg, GError **err)
+{
+ struct rspamd_fuzzy_backend_redis *backend;
+ const ucl_object_t *elt;
+ gboolean ret = FALSE;
+ guchar id_hash[rspamd_cryptobox_HASHBYTES];
+ rspamd_cryptobox_hash_state_t st;
+ lua_State *L = (lua_State *) cfg->lua_state;
+ gint conf_ref = -1;
+
+ backend = g_malloc0(sizeof(*backend));
+
+ backend->timeout = REDIS_DEFAULT_TIMEOUT;
+ backend->redis_object = REDIS_DEFAULT_OBJECT;
+ backend->L = L;
+
+ ret = rspamd_lua_try_load_redis(L, obj, cfg, &conf_ref);
+
+ /* Now try global redis settings */
+ if (!ret) {
+ elt = ucl_object_lookup(cfg->cfg_ucl_obj, "redis");
+
+ if (elt) {
+ const ucl_object_t *specific_obj;
+
+ specific_obj = ucl_object_lookup_any(elt, "fuzzy", "fuzzy_storage",
+ NULL);
+
+ if (specific_obj) {
+ ret = rspamd_lua_try_load_redis(L, specific_obj, cfg, &conf_ref);
+ }
+ else {
+ ret = rspamd_lua_try_load_redis(L, elt, cfg, &conf_ref);
+ }
+ }
+ }
+
+ if (!ret) {
+ msg_err_config("cannot init redis backend for fuzzy storage");
+ g_free(backend);
+
+ return NULL;
+ }
+
+ elt = ucl_object_lookup(obj, "prefix");
+ if (elt == NULL || ucl_object_type(elt) != UCL_STRING) {
+ backend->redis_object = REDIS_DEFAULT_OBJECT;
+ }
+ else {
+ backend->redis_object = ucl_object_tostring(elt);
+ }
+
+ backend->conf_ref = conf_ref;
+
+ /* Check some common table values */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, conf_ref);
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ backend->timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "db");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ backend->dbname = rspamd_mempool_strdup(cfg->cfg_pool,
+ lua_tostring(L, -1));
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "username");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ backend->username = rspamd_mempool_strdup(cfg->cfg_pool,
+ lua_tostring(L, -1));
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "password");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ backend->password = rspamd_mempool_strdup(cfg->cfg_pool,
+ lua_tostring(L, -1));
+ }
+ lua_pop(L, 1);
+
+ lua_settop(L, 0);
+
+ REF_INIT_RETAIN(backend, rspamd_fuzzy_backend_redis_dtor);
+ backend->pool = cfg->redis_pool;
+ rspamd_cryptobox_hash_init(&st, NULL, 0);
+ rspamd_cryptobox_hash_update(&st, backend->redis_object,
+ strlen(backend->redis_object));
+
+ if (backend->dbname) {
+ rspamd_cryptobox_hash_update(&st, backend->dbname,
+ strlen(backend->dbname));
+ }
+
+ if (backend->username) {
+ rspamd_cryptobox_hash_update(&st, backend->username,
+ strlen(backend->username));
+ }
+
+ if (backend->password) {
+ rspamd_cryptobox_hash_update(&st, backend->password,
+ strlen(backend->password));
+ }
+
+ rspamd_cryptobox_hash_final(&st, id_hash);
+ backend->id = rspamd_encode_base32(id_hash, sizeof(id_hash), RSPAMD_BASE32_DEFAULT);
+
+ return backend;
+}
+
+static void
+rspamd_fuzzy_redis_timeout(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_fuzzy_redis_session *session =
+ (struct rspamd_fuzzy_redis_session *) w->data;
+ redisAsyncContext *ac;
+ static char errstr[128];
+
+ if (session->ctx) {
+ ac = session->ctx;
+ session->ctx = NULL;
+ ac->err = REDIS_ERR_IO;
+ /* Should be safe as in hiredis it is char[128] */
+ rspamd_snprintf(errstr, sizeof(errstr), "%s", strerror(ETIMEDOUT));
+ ac->errstr = errstr;
+
+ /* This will cause session closing */
+ rspamd_redis_pool_release_connection(session->backend->pool,
+ ac, RSPAMD_REDIS_RELEASE_FATAL);
+ }
+}
+
+static void rspamd_fuzzy_redis_check_callback(redisAsyncContext *c, gpointer r,
+ gpointer priv);
+
+struct _rspamd_fuzzy_shingles_helper {
+ guchar digest[64];
+ guint found;
+};
+
+static gint
+rspamd_fuzzy_backend_redis_shingles_cmp(const void *a, const void *b)
+{
+ const struct _rspamd_fuzzy_shingles_helper *sha = a,
+ *shb = b;
+
+ return memcmp(sha->digest, shb->digest, sizeof(sha->digest));
+}
+
+static void
+rspamd_fuzzy_redis_shingles_callback(redisAsyncContext *c, gpointer r,
+ gpointer priv)
+{
+ struct rspamd_fuzzy_redis_session *session = priv;
+ redisReply *reply = r, *cur;
+ struct rspamd_fuzzy_reply rep;
+ GString *key;
+ struct _rspamd_fuzzy_shingles_helper *shingles, *prev = NULL, *sel = NULL;
+ guint i, found = 0, max_found = 0, cur_found = 0;
+
+ ev_timer_stop(session->event_loop, &session->timeout);
+ memset(&rep, 0, sizeof(rep));
+
+ if (c->err == 0 && reply != NULL) {
+ rspamd_upstream_ok(session->up);
+
+ if (reply->type == REDIS_REPLY_ARRAY &&
+ reply->elements == RSPAMD_SHINGLE_SIZE) {
+ shingles = g_alloca(sizeof(struct _rspamd_fuzzy_shingles_helper) *
+ RSPAMD_SHINGLE_SIZE);
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ cur = reply->element[i];
+
+ if (cur->type == REDIS_REPLY_STRING) {
+ shingles[i].found = 1;
+ memcpy(shingles[i].digest, cur->str, MIN(64, cur->len));
+ found++;
+ }
+ else {
+ memset(shingles[i].digest, 0, sizeof(shingles[i].digest));
+ shingles[i].found = 0;
+ }
+ }
+
+ if (found > RSPAMD_SHINGLE_SIZE / 2) {
+ /* Now sort to find the most frequent element */
+ qsort(shingles, RSPAMD_SHINGLE_SIZE,
+ sizeof(struct _rspamd_fuzzy_shingles_helper),
+ rspamd_fuzzy_backend_redis_shingles_cmp);
+
+ prev = &shingles[0];
+
+ for (i = 1; i < RSPAMD_SHINGLE_SIZE; i++) {
+ if (!shingles[i].found) {
+ continue;
+ }
+
+ if (memcmp(shingles[i].digest, prev->digest, 64) == 0) {
+ cur_found++;
+
+ if (cur_found > max_found) {
+ max_found = cur_found;
+ sel = &shingles[i];
+ }
+ }
+ else {
+ cur_found = 1;
+ prev = &shingles[i];
+ }
+ }
+
+ if (max_found > RSPAMD_SHINGLE_SIZE / 2) {
+ session->prob = ((float) max_found) / RSPAMD_SHINGLE_SIZE;
+ rep.v1.prob = session->prob;
+
+ g_assert(sel != NULL);
+
+ /* Prepare new check command */
+ rspamd_fuzzy_redis_session_free_args(session);
+ session->nargs = 5;
+ session->argv = g_malloc(sizeof(gchar *) * session->nargs);
+ session->argv_lens = g_malloc(sizeof(gsize) * session->nargs);
+
+ key = g_string_new(session->backend->redis_object);
+ g_string_append_len(key, sel->digest, sizeof(sel->digest));
+ session->argv[0] = g_strdup("HMGET");
+ session->argv_lens[0] = 5;
+ session->argv[1] = key->str;
+ session->argv_lens[1] = key->len;
+ session->argv[2] = g_strdup("V");
+ session->argv_lens[2] = 1;
+ session->argv[3] = g_strdup("F");
+ session->argv_lens[3] = 1;
+ session->argv[4] = g_strdup("C");
+ session->argv_lens[4] = 1;
+ g_string_free(key, FALSE); /* Do not free underlying array */
+ memcpy(session->found_digest, sel->digest,
+ sizeof(session->cmd->digest));
+
+ g_assert(session->ctx != NULL);
+ if (redisAsyncCommandArgv(session->ctx,
+ rspamd_fuzzy_redis_check_callback,
+ session, session->nargs,
+ (const gchar **) session->argv,
+ session->argv_lens) != REDIS_OK) {
+
+ if (session->callback.cb_check) {
+ memset(&rep, 0, sizeof(rep));
+ session->callback.cb_check(&rep, session->cbdata);
+ }
+
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+ }
+ else {
+ /* Add timeout */
+ session->timeout.data = session;
+ ev_now_update_if_cheap((struct ev_loop *) session->event_loop);
+ ev_timer_init(&session->timeout,
+ rspamd_fuzzy_redis_timeout,
+ session->backend->timeout, 0.0);
+ ev_timer_start(session->event_loop, &session->timeout);
+ }
+
+ return;
+ }
+ }
+ }
+ else if (reply->type == REDIS_REPLY_ERROR) {
+ msg_err_redis_session("fuzzy backend redis error: \"%s\"",
+ reply->str);
+ }
+
+ if (session->callback.cb_check) {
+ session->callback.cb_check(&rep, session->cbdata);
+ }
+ }
+ else {
+ if (session->callback.cb_check) {
+ session->callback.cb_check(&rep, session->cbdata);
+ }
+
+ if (c->errstr) {
+ msg_err_redis_session("error getting shingles: %s", c->errstr);
+ rspamd_upstream_fail(session->up, FALSE, c->errstr);
+ }
+ }
+
+ rspamd_fuzzy_redis_session_dtor(session, FALSE);
+}
+
+static void
+rspamd_fuzzy_backend_check_shingles(struct rspamd_fuzzy_redis_session *session)
+{
+ struct rspamd_fuzzy_reply rep;
+ const struct rspamd_fuzzy_shingle_cmd *shcmd;
+ GString *key;
+ guint i, init_len;
+
+ rspamd_fuzzy_redis_session_free_args(session);
+ /* First of all check digest */
+ session->nargs = RSPAMD_SHINGLE_SIZE + 1;
+ session->argv = g_malloc(sizeof(gchar *) * session->nargs);
+ session->argv_lens = g_malloc(sizeof(gsize) * session->nargs);
+ shcmd = (const struct rspamd_fuzzy_shingle_cmd *) session->cmd;
+
+ session->argv[0] = g_strdup("MGET");
+ session->argv_lens[0] = 4;
+ init_len = strlen(session->backend->redis_object);
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+
+ key = g_string_sized_new(init_len + 2 + 2 + sizeof("18446744073709551616"));
+ rspamd_printf_gstring(key, "%s_%d_%uL", session->backend->redis_object,
+ i, shcmd->sgl.hashes[i]);
+ session->argv[i + 1] = key->str;
+ session->argv_lens[i + 1] = key->len;
+ g_string_free(key, FALSE); /* Do not free underlying array */
+ }
+
+ session->shingles_checked = TRUE;
+
+ g_assert(session->ctx != NULL);
+
+ if (redisAsyncCommandArgv(session->ctx, rspamd_fuzzy_redis_shingles_callback,
+ session, session->nargs,
+ (const gchar **) session->argv, session->argv_lens) != REDIS_OK) {
+ msg_err("cannot execute redis command on %s: %s",
+ rspamd_inet_address_to_string_pretty(rspamd_upstream_addr_cur(session->up)),
+ session->ctx->errstr);
+
+ if (session->callback.cb_check) {
+ memset(&rep, 0, sizeof(rep));
+ session->callback.cb_check(&rep, session->cbdata);
+ }
+
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+ }
+ else {
+ /* Add timeout */
+ session->timeout.data = session;
+ ev_now_update_if_cheap((struct ev_loop *) session->event_loop);
+ ev_timer_init(&session->timeout,
+ rspamd_fuzzy_redis_timeout,
+ session->backend->timeout, 0.0);
+ ev_timer_start(session->event_loop, &session->timeout);
+ }
+}
+
+static void
+rspamd_fuzzy_redis_check_callback(redisAsyncContext *c, gpointer r,
+ gpointer priv)
+{
+ struct rspamd_fuzzy_redis_session *session = priv;
+ redisReply *reply = r, *cur;
+ struct rspamd_fuzzy_reply rep;
+ gulong value;
+ guint found_elts = 0;
+
+ ev_timer_stop(session->event_loop, &session->timeout);
+ memset(&rep, 0, sizeof(rep));
+
+ if (c->err == 0 && reply != NULL) {
+ rspamd_upstream_ok(session->up);
+
+ if (reply->type == REDIS_REPLY_ARRAY && reply->elements >= 2) {
+ cur = reply->element[0];
+
+ if (cur->type == REDIS_REPLY_STRING) {
+ value = strtoul(cur->str, NULL, 10);
+ rep.v1.value = value;
+ found_elts++;
+ }
+
+ cur = reply->element[1];
+
+ if (cur->type == REDIS_REPLY_STRING) {
+ value = strtoul(cur->str, NULL, 10);
+ rep.v1.flag = value;
+ found_elts++;
+ }
+
+ if (found_elts >= 2) {
+ rep.v1.prob = session->prob;
+ memcpy(rep.digest, session->found_digest, sizeof(rep.digest));
+ }
+
+ rep.ts = 0;
+
+ if (reply->elements > 2) {
+ cur = reply->element[2];
+
+ if (cur->type == REDIS_REPLY_STRING) {
+ rep.ts = strtoul(cur->str, NULL, 10);
+ }
+ }
+ }
+ else if (reply->type == REDIS_REPLY_ERROR) {
+ msg_err_redis_session("fuzzy backend redis error: \"%s\"",
+ reply->str);
+ }
+
+ if (found_elts < 2) {
+ if (session->cmd->shingles_count > 0 && !session->shingles_checked) {
+ /* We also need to check all shingles here */
+ rspamd_fuzzy_backend_check_shingles(session);
+ /* Do not free session */
+ return;
+ }
+ else {
+ if (session->callback.cb_check) {
+ session->callback.cb_check(&rep, session->cbdata);
+ }
+ }
+ }
+ else {
+ if (session->callback.cb_check) {
+ session->callback.cb_check(&rep, session->cbdata);
+ }
+ }
+ }
+ else {
+ if (session->callback.cb_check) {
+ session->callback.cb_check(&rep, session->cbdata);
+ }
+
+ if (c->errstr) {
+ msg_err_redis_session("error getting hashes on %s: %s",
+ rspamd_inet_address_to_string_pretty(rspamd_upstream_addr_cur(session->up)),
+ c->errstr);
+ rspamd_upstream_fail(session->up, FALSE, c->errstr);
+ }
+ }
+
+ rspamd_fuzzy_redis_session_dtor(session, FALSE);
+}
+
+void rspamd_fuzzy_backend_check_redis(struct rspamd_fuzzy_backend *bk,
+ const struct rspamd_fuzzy_cmd *cmd,
+ rspamd_fuzzy_check_cb cb, void *ud,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_redis *backend = subr_ud;
+ struct rspamd_fuzzy_redis_session *session;
+ struct upstream *up;
+ struct upstream_list *ups;
+ rspamd_inet_addr_t *addr;
+ struct rspamd_fuzzy_reply rep;
+ GString *key;
+
+ g_assert(backend != NULL);
+
+ ups = rspamd_redis_get_servers(backend, "read_servers");
+ if (!ups) {
+ if (cb) {
+ memset(&rep, 0, sizeof(rep));
+ cb(&rep, ud);
+ }
+
+ return;
+ }
+
+ session = g_malloc0(sizeof(*session));
+ session->backend = backend;
+ REF_RETAIN(session->backend);
+
+ session->callback.cb_check = cb;
+ session->cbdata = ud;
+ session->command = RSPAMD_FUZZY_REDIS_COMMAND_CHECK;
+ session->cmd = cmd;
+ session->prob = 1.0;
+ memcpy(rep.digest, session->cmd->digest, sizeof(rep.digest));
+ memcpy(session->found_digest, session->cmd->digest, sizeof(rep.digest));
+ session->event_loop = rspamd_fuzzy_backend_event_base(bk);
+
+ /* First of all check digest */
+ session->nargs = 5;
+ session->argv = g_malloc(sizeof(gchar *) * session->nargs);
+ session->argv_lens = g_malloc(sizeof(gsize) * session->nargs);
+
+ key = g_string_new(backend->redis_object);
+ g_string_append_len(key, cmd->digest, sizeof(cmd->digest));
+ session->argv[0] = g_strdup("HMGET");
+ session->argv_lens[0] = 5;
+ session->argv[1] = key->str;
+ session->argv_lens[1] = key->len;
+ session->argv[2] = g_strdup("V");
+ session->argv_lens[2] = 1;
+ session->argv[3] = g_strdup("F");
+ session->argv_lens[3] = 1;
+ session->argv[4] = g_strdup("C");
+ session->argv_lens[4] = 1;
+ g_string_free(key, FALSE); /* Do not free underlying array */
+
+ up = rspamd_upstream_get(ups,
+ RSPAMD_UPSTREAM_ROUND_ROBIN,
+ NULL,
+ 0);
+
+ session->up = rspamd_upstream_ref(up);
+ addr = rspamd_upstream_addr_next(up);
+ g_assert(addr != NULL);
+ session->ctx = rspamd_redis_pool_connect(backend->pool,
+ backend->dbname,
+ backend->username, backend->password,
+ rspamd_inet_address_to_string(addr),
+ rspamd_inet_address_get_port(addr));
+
+ if (session->ctx == NULL) {
+ rspamd_upstream_fail(up, TRUE, strerror(errno));
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+
+ if (cb) {
+ memset(&rep, 0, sizeof(rep));
+ cb(&rep, ud);
+ }
+ }
+ else {
+ if (redisAsyncCommandArgv(session->ctx, rspamd_fuzzy_redis_check_callback,
+ session, session->nargs,
+ (const gchar **) session->argv, session->argv_lens) != REDIS_OK) {
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+
+ if (cb) {
+ memset(&rep, 0, sizeof(rep));
+ cb(&rep, ud);
+ }
+ }
+ else {
+ /* Add timeout */
+ session->timeout.data = session;
+ ev_now_update_if_cheap((struct ev_loop *) session->event_loop);
+ ev_timer_init(&session->timeout,
+ rspamd_fuzzy_redis_timeout,
+ session->backend->timeout, 0.0);
+ ev_timer_start(session->event_loop, &session->timeout);
+ }
+ }
+}
+
+static void
+rspamd_fuzzy_redis_count_callback(redisAsyncContext *c, gpointer r,
+ gpointer priv)
+{
+ struct rspamd_fuzzy_redis_session *session = priv;
+ redisReply *reply = r;
+ gulong nelts;
+
+ ev_timer_stop(session->event_loop, &session->timeout);
+
+ if (c->err == 0 && reply != NULL) {
+ rspamd_upstream_ok(session->up);
+
+ if (reply->type == REDIS_REPLY_INTEGER) {
+ if (session->callback.cb_count) {
+ session->callback.cb_count(reply->integer, session->cbdata);
+ }
+ }
+ else if (reply->type == REDIS_REPLY_STRING) {
+ nelts = strtoul(reply->str, NULL, 10);
+
+ if (session->callback.cb_count) {
+ session->callback.cb_count(nelts, session->cbdata);
+ }
+ }
+ else {
+ if (reply->type == REDIS_REPLY_ERROR) {
+ msg_err_redis_session("fuzzy backend redis error: \"%s\"",
+ reply->str);
+ }
+ if (session->callback.cb_count) {
+ session->callback.cb_count(0, session->cbdata);
+ }
+ }
+ }
+ else {
+ if (session->callback.cb_count) {
+ session->callback.cb_count(0, session->cbdata);
+ }
+
+ if (c->errstr) {
+ msg_err_redis_session("error getting count on %s: %s",
+ rspamd_inet_address_to_string_pretty(rspamd_upstream_addr_cur(session->up)),
+ c->errstr);
+ rspamd_upstream_fail(session->up, FALSE, c->errstr);
+ }
+ }
+
+ rspamd_fuzzy_redis_session_dtor(session, FALSE);
+}
+
+void rspamd_fuzzy_backend_count_redis(struct rspamd_fuzzy_backend *bk,
+ rspamd_fuzzy_count_cb cb, void *ud,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_redis *backend = subr_ud;
+ struct rspamd_fuzzy_redis_session *session;
+ struct upstream *up;
+ struct upstream_list *ups;
+ rspamd_inet_addr_t *addr;
+ GString *key;
+
+ g_assert(backend != NULL);
+
+ ups = rspamd_redis_get_servers(backend, "read_servers");
+ if (!ups) {
+ if (cb) {
+ cb(0, ud);
+ }
+
+ return;
+ }
+
+ session = g_malloc0(sizeof(*session));
+ session->backend = backend;
+ REF_RETAIN(session->backend);
+
+ session->callback.cb_count = cb;
+ session->cbdata = ud;
+ session->command = RSPAMD_FUZZY_REDIS_COMMAND_COUNT;
+ session->event_loop = rspamd_fuzzy_backend_event_base(bk);
+
+ session->nargs = 2;
+ session->argv = g_malloc(sizeof(gchar *) * 2);
+ session->argv_lens = g_malloc(sizeof(gsize) * 2);
+ key = g_string_new(backend->redis_object);
+ g_string_append(key, "_count");
+ session->argv[0] = g_strdup("GET");
+ session->argv_lens[0] = 3;
+ session->argv[1] = key->str;
+ session->argv_lens[1] = key->len;
+ g_string_free(key, FALSE); /* Do not free underlying array */
+
+ up = rspamd_upstream_get(ups,
+ RSPAMD_UPSTREAM_ROUND_ROBIN,
+ NULL,
+ 0);
+
+ session->up = rspamd_upstream_ref(up);
+ addr = rspamd_upstream_addr_next(up);
+ g_assert(addr != NULL);
+ session->ctx = rspamd_redis_pool_connect(backend->pool,
+ backend->dbname,
+ backend->username, backend->password,
+ rspamd_inet_address_to_string(addr),
+ rspamd_inet_address_get_port(addr));
+
+ if (session->ctx == NULL) {
+ rspamd_upstream_fail(up, TRUE, strerror(errno));
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+
+ if (cb) {
+ cb(0, ud);
+ }
+ }
+ else {
+ if (redisAsyncCommandArgv(session->ctx, rspamd_fuzzy_redis_count_callback,
+ session, session->nargs,
+ (const gchar **) session->argv, session->argv_lens) != REDIS_OK) {
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+
+ if (cb) {
+ cb(0, ud);
+ }
+ }
+ else {
+ /* Add timeout */
+ session->timeout.data = session;
+ ev_now_update_if_cheap((struct ev_loop *) session->event_loop);
+ ev_timer_init(&session->timeout,
+ rspamd_fuzzy_redis_timeout,
+ session->backend->timeout, 0.0);
+ ev_timer_start(session->event_loop, &session->timeout);
+ }
+ }
+}
+
+static void
+rspamd_fuzzy_redis_version_callback(redisAsyncContext *c, gpointer r,
+ gpointer priv)
+{
+ struct rspamd_fuzzy_redis_session *session = priv;
+ redisReply *reply = r;
+ gulong nelts;
+
+ ev_timer_stop(session->event_loop, &session->timeout);
+
+ if (c->err == 0 && reply != NULL) {
+ rspamd_upstream_ok(session->up);
+
+ if (reply->type == REDIS_REPLY_INTEGER) {
+ if (session->callback.cb_version) {
+ session->callback.cb_version(reply->integer, session->cbdata);
+ }
+ }
+ else if (reply->type == REDIS_REPLY_STRING) {
+ nelts = strtoul(reply->str, NULL, 10);
+
+ if (session->callback.cb_version) {
+ session->callback.cb_version(nelts, session->cbdata);
+ }
+ }
+ else {
+ if (reply->type == REDIS_REPLY_ERROR) {
+ msg_err_redis_session("fuzzy backend redis error: \"%s\"",
+ reply->str);
+ }
+ if (session->callback.cb_version) {
+ session->callback.cb_version(0, session->cbdata);
+ }
+ }
+ }
+ else {
+ if (session->callback.cb_version) {
+ session->callback.cb_version(0, session->cbdata);
+ }
+
+ if (c->errstr) {
+ msg_err_redis_session("error getting version on %s: %s",
+ rspamd_inet_address_to_string_pretty(rspamd_upstream_addr_cur(session->up)),
+ c->errstr);
+ rspamd_upstream_fail(session->up, FALSE, c->errstr);
+ }
+ }
+
+ rspamd_fuzzy_redis_session_dtor(session, FALSE);
+}
+
+void rspamd_fuzzy_backend_version_redis(struct rspamd_fuzzy_backend *bk,
+ const gchar *src,
+ rspamd_fuzzy_version_cb cb, void *ud,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_redis *backend = subr_ud;
+ struct rspamd_fuzzy_redis_session *session;
+ struct upstream *up;
+ struct upstream_list *ups;
+ rspamd_inet_addr_t *addr;
+ GString *key;
+
+ g_assert(backend != NULL);
+
+ ups = rspamd_redis_get_servers(backend, "read_servers");
+ if (!ups) {
+ if (cb) {
+ cb(0, ud);
+ }
+
+ return;
+ }
+
+ session = g_malloc0(sizeof(*session));
+ session->backend = backend;
+ REF_RETAIN(session->backend);
+
+ session->callback.cb_version = cb;
+ session->cbdata = ud;
+ session->command = RSPAMD_FUZZY_REDIS_COMMAND_VERSION;
+ session->event_loop = rspamd_fuzzy_backend_event_base(bk);
+
+ session->nargs = 2;
+ session->argv = g_malloc(sizeof(gchar *) * 2);
+ session->argv_lens = g_malloc(sizeof(gsize) * 2);
+ key = g_string_new(backend->redis_object);
+ g_string_append(key, src);
+ session->argv[0] = g_strdup("GET");
+ session->argv_lens[0] = 3;
+ session->argv[1] = key->str;
+ session->argv_lens[1] = key->len;
+ g_string_free(key, FALSE); /* Do not free underlying array */
+
+ up = rspamd_upstream_get(ups,
+ RSPAMD_UPSTREAM_ROUND_ROBIN,
+ NULL,
+ 0);
+
+ session->up = rspamd_upstream_ref(up);
+ addr = rspamd_upstream_addr_next(up);
+ g_assert(addr != NULL);
+ session->ctx = rspamd_redis_pool_connect(backend->pool,
+ backend->dbname,
+ backend->username, backend->password,
+ rspamd_inet_address_to_string(addr),
+ rspamd_inet_address_get_port(addr));
+
+ if (session->ctx == NULL) {
+ rspamd_upstream_fail(up, FALSE, strerror(errno));
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+
+ if (cb) {
+ cb(0, ud);
+ }
+ }
+ else {
+ if (redisAsyncCommandArgv(session->ctx, rspamd_fuzzy_redis_version_callback,
+ session, session->nargs,
+ (const gchar **) session->argv, session->argv_lens) != REDIS_OK) {
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+
+ if (cb) {
+ cb(0, ud);
+ }
+ }
+ else {
+ /* Add timeout */
+ session->timeout.data = session;
+ ev_now_update_if_cheap((struct ev_loop *) session->event_loop);
+ ev_timer_init(&session->timeout,
+ rspamd_fuzzy_redis_timeout,
+ session->backend->timeout, 0.0);
+ ev_timer_start(session->event_loop, &session->timeout);
+ }
+ }
+}
+
+const gchar *
+rspamd_fuzzy_backend_id_redis(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_redis *backend = subr_ud;
+ g_assert(backend != NULL);
+
+ return backend->id;
+}
+
+void rspamd_fuzzy_backend_expire_redis(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_redis *backend = subr_ud;
+
+ g_assert(backend != NULL);
+}
+
+static gboolean
+rspamd_fuzzy_update_append_command(struct rspamd_fuzzy_backend *bk,
+ struct rspamd_fuzzy_redis_session *session,
+ struct fuzzy_peer_cmd *io_cmd, guint *shift)
+{
+ GString *key, *value;
+ guint cur_shift = *shift;
+ guint i, klen;
+ struct rspamd_fuzzy_cmd *cmd;
+
+ if (io_cmd->is_shingle) {
+ cmd = &io_cmd->cmd.shingle.basic;
+ }
+ else {
+ cmd = &io_cmd->cmd.normal;
+ }
+
+ if (cmd->cmd == FUZZY_WRITE) {
+ /*
+ * For each normal hash addition we do 5 redis commands:
+ * HSET <key> F <flag>
+ * HSETNX <key> C <time>
+ * HINCRBY <key> V <weight>
+ * EXPIRE <key> <expire>
+ * Where <key> is <prefix> || <digest>
+ */
+
+ /* HSET */
+ klen = strlen(session->backend->redis_object) +
+ sizeof(cmd->digest) + 1;
+ key = g_string_sized_new(klen);
+ g_string_append(key, session->backend->redis_object);
+ g_string_append_len(key, cmd->digest, sizeof(cmd->digest));
+ value = g_string_sized_new(sizeof("4294967296"));
+ rspamd_printf_gstring(value, "%d", cmd->flag);
+
+ if (cmd->version & RSPAMD_FUZZY_FLAG_WEAK) {
+ session->argv[cur_shift] = g_strdup("HSETNX");
+ session->argv_lens[cur_shift++] = sizeof("HSETNX") - 1;
+ }
+ else {
+ session->argv[cur_shift] = g_strdup("HSET");
+ session->argv_lens[cur_shift++] = sizeof("HSET") - 1;
+ }
+
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ session->argv[cur_shift] = g_strdup("F");
+ session->argv_lens[cur_shift++] = sizeof("F") - 1;
+ session->argv[cur_shift] = value->str;
+ session->argv_lens[cur_shift++] = value->len;
+ g_string_free(key, FALSE);
+ g_string_free(value, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 4,
+ (const gchar **) &session->argv[cur_shift - 4],
+ &session->argv_lens[cur_shift - 4]) != REDIS_OK) {
+
+ return FALSE;
+ }
+
+ /* HSETNX */
+ klen = strlen(session->backend->redis_object) +
+ sizeof(cmd->digest) + 1;
+ key = g_string_sized_new(klen);
+ g_string_append(key, session->backend->redis_object);
+ g_string_append_len(key, cmd->digest, sizeof(cmd->digest));
+ value = g_string_sized_new(sizeof("18446744073709551616"));
+ rspamd_printf_gstring(value, "%L", (gint64) rspamd_get_calendar_ticks());
+ session->argv[cur_shift] = g_strdup("HSETNX");
+ session->argv_lens[cur_shift++] = sizeof("HSETNX") - 1;
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ session->argv[cur_shift] = g_strdup("C");
+ session->argv_lens[cur_shift++] = sizeof("C") - 1;
+ session->argv[cur_shift] = value->str;
+ session->argv_lens[cur_shift++] = value->len;
+ g_string_free(key, FALSE);
+ g_string_free(value, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 4,
+ (const gchar **) &session->argv[cur_shift - 4],
+ &session->argv_lens[cur_shift - 4]) != REDIS_OK) {
+
+ return FALSE;
+ }
+
+ /* HINCRBY */
+ key = g_string_sized_new(klen);
+ g_string_append(key, session->backend->redis_object);
+ g_string_append_len(key, cmd->digest, sizeof(cmd->digest));
+ value = g_string_sized_new(sizeof("4294967296"));
+ rspamd_printf_gstring(value, "%d", cmd->value);
+ session->argv[cur_shift] = g_strdup("HINCRBY");
+ session->argv_lens[cur_shift++] = sizeof("HINCRBY") - 1;
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ session->argv[cur_shift] = g_strdup("V");
+ session->argv_lens[cur_shift++] = sizeof("V") - 1;
+ session->argv[cur_shift] = value->str;
+ session->argv_lens[cur_shift++] = value->len;
+ g_string_free(key, FALSE);
+ g_string_free(value, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 4,
+ (const gchar **) &session->argv[cur_shift - 4],
+ &session->argv_lens[cur_shift - 4]) != REDIS_OK) {
+
+ return FALSE;
+ }
+
+ /* EXPIRE */
+ key = g_string_sized_new(klen);
+ g_string_append(key, session->backend->redis_object);
+ g_string_append_len(key, cmd->digest, sizeof(cmd->digest));
+ value = g_string_sized_new(sizeof("4294967296"));
+ rspamd_printf_gstring(value, "%d",
+ (gint) rspamd_fuzzy_backend_get_expire(bk));
+ session->argv[cur_shift] = g_strdup("EXPIRE");
+ session->argv_lens[cur_shift++] = sizeof("EXPIRE") - 1;
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ session->argv[cur_shift] = value->str;
+ session->argv_lens[cur_shift++] = value->len;
+ g_string_free(key, FALSE);
+ g_string_free(value, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 3,
+ (const gchar **) &session->argv[cur_shift - 3],
+ &session->argv_lens[cur_shift - 3]) != REDIS_OK) {
+
+ return FALSE;
+ }
+
+ /* INCR */
+ key = g_string_sized_new(klen);
+ g_string_append(key, session->backend->redis_object);
+ g_string_append(key, "_count");
+ session->argv[cur_shift] = g_strdup("INCR");
+ session->argv_lens[cur_shift++] = sizeof("INCR") - 1;
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ g_string_free(key, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 2,
+ (const gchar **) &session->argv[cur_shift - 2],
+ &session->argv_lens[cur_shift - 2]) != REDIS_OK) {
+
+ return FALSE;
+ }
+ }
+ else if (cmd->cmd == FUZZY_DEL) {
+ /* DEL */
+ klen = strlen(session->backend->redis_object) +
+ sizeof(cmd->digest) + 1;
+
+ key = g_string_sized_new(klen);
+ g_string_append(key, session->backend->redis_object);
+ g_string_append_len(key, cmd->digest, sizeof(cmd->digest));
+ session->argv[cur_shift] = g_strdup("DEL");
+ session->argv_lens[cur_shift++] = sizeof("DEL") - 1;
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ g_string_free(key, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 2,
+ (const gchar **) &session->argv[cur_shift - 2],
+ &session->argv_lens[cur_shift - 2]) != REDIS_OK) {
+
+ return FALSE;
+ }
+
+ /* DECR */
+ key = g_string_sized_new(klen);
+ g_string_append(key, session->backend->redis_object);
+ g_string_append(key, "_count");
+ session->argv[cur_shift] = g_strdup("DECR");
+ session->argv_lens[cur_shift++] = sizeof("DECR") - 1;
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ g_string_free(key, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 2,
+ (const gchar **) &session->argv[cur_shift - 2],
+ &session->argv_lens[cur_shift - 2]) != REDIS_OK) {
+
+ return FALSE;
+ }
+ }
+ else if (cmd->cmd == FUZZY_REFRESH) {
+ /*
+ * Issue refresh command by just EXPIRE command
+ * EXPIRE <key> <expire>
+ * Where <key> is <prefix> || <digest>
+ */
+
+ klen = strlen(session->backend->redis_object) +
+ sizeof(cmd->digest) + 1;
+
+ /* EXPIRE */
+ key = g_string_sized_new(klen);
+ g_string_append(key, session->backend->redis_object);
+ g_string_append_len(key, cmd->digest, sizeof(cmd->digest));
+ value = g_string_sized_new(sizeof("4294967296"));
+ rspamd_printf_gstring(value, "%d",
+ (gint) rspamd_fuzzy_backend_get_expire(bk));
+ session->argv[cur_shift] = g_strdup("EXPIRE");
+ session->argv_lens[cur_shift++] = sizeof("EXPIRE") - 1;
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ session->argv[cur_shift] = value->str;
+ session->argv_lens[cur_shift++] = value->len;
+ g_string_free(key, FALSE);
+ g_string_free(value, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 3,
+ (const gchar **) &session->argv[cur_shift - 3],
+ &session->argv_lens[cur_shift - 3]) != REDIS_OK) {
+
+ return FALSE;
+ }
+ }
+ else if (cmd->cmd == FUZZY_DUP) {
+ /* Ignore */
+ }
+ else {
+ g_assert_not_reached();
+ }
+
+ if (io_cmd->is_shingle) {
+ if (cmd->cmd == FUZZY_WRITE) {
+ klen = strlen(session->backend->redis_object) +
+ 64 + 1;
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ guchar *hval;
+ /*
+ * For each command with shingles we additionally emit 32 commands:
+ * SETEX <prefix>_<number>_<value> <expire> <digest>
+ */
+
+ /* SETEX */
+ key = g_string_sized_new(klen);
+ rspamd_printf_gstring(key, "%s_%d_%uL",
+ session->backend->redis_object,
+ i,
+ io_cmd->cmd.shingle.sgl.hashes[i]);
+ value = g_string_sized_new(sizeof("4294967296"));
+ rspamd_printf_gstring(value, "%d",
+ (gint) rspamd_fuzzy_backend_get_expire(bk));
+ hval = g_malloc(sizeof(io_cmd->cmd.shingle.basic.digest));
+ memcpy(hval, io_cmd->cmd.shingle.basic.digest,
+ sizeof(io_cmd->cmd.shingle.basic.digest));
+ session->argv[cur_shift] = g_strdup("SETEX");
+ session->argv_lens[cur_shift++] = sizeof("SETEX") - 1;
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ session->argv[cur_shift] = value->str;
+ session->argv_lens[cur_shift++] = value->len;
+ session->argv[cur_shift] = hval;
+ session->argv_lens[cur_shift++] = sizeof(io_cmd->cmd.shingle.basic.digest);
+ g_string_free(key, FALSE);
+ g_string_free(value, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 4,
+ (const gchar **) &session->argv[cur_shift - 4],
+ &session->argv_lens[cur_shift - 4]) != REDIS_OK) {
+
+ return FALSE;
+ }
+ }
+ }
+ else if (cmd->cmd == FUZZY_DEL) {
+ klen = strlen(session->backend->redis_object) +
+ 64 + 1;
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ key = g_string_sized_new(klen);
+ rspamd_printf_gstring(key, "%s_%d_%uL",
+ session->backend->redis_object,
+ i,
+ io_cmd->cmd.shingle.sgl.hashes[i]);
+ session->argv[cur_shift] = g_strdup("DEL");
+ session->argv_lens[cur_shift++] = sizeof("DEL") - 1;
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ g_string_free(key, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 2,
+ (const gchar **) &session->argv[cur_shift - 2],
+ &session->argv_lens[cur_shift - 2]) != REDIS_OK) {
+
+ return FALSE;
+ }
+ }
+ }
+ else if (cmd->cmd == FUZZY_REFRESH) {
+ klen = strlen(session->backend->redis_object) +
+ 64 + 1;
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ /*
+ * For each command with shingles we additionally emit 32 commands:
+ * EXPIRE <prefix>_<number>_<value> <expire>
+ */
+
+ /* Expire */
+ key = g_string_sized_new(klen);
+ rspamd_printf_gstring(key, "%s_%d_%uL",
+ session->backend->redis_object,
+ i,
+ io_cmd->cmd.shingle.sgl.hashes[i]);
+ value = g_string_sized_new(sizeof("18446744073709551616"));
+ rspamd_printf_gstring(value, "%d",
+ (gint) rspamd_fuzzy_backend_get_expire(bk));
+ session->argv[cur_shift] = g_strdup("EXPIRE");
+ session->argv_lens[cur_shift++] = sizeof("EXPIRE") - 1;
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ session->argv[cur_shift] = value->str;
+ session->argv_lens[cur_shift++] = value->len;
+ g_string_free(key, FALSE);
+ g_string_free(value, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 3,
+ (const gchar **) &session->argv[cur_shift - 3],
+ &session->argv_lens[cur_shift - 3]) != REDIS_OK) {
+
+ return FALSE;
+ }
+ }
+ }
+ else if (cmd->cmd == FUZZY_DUP) {
+ /* Ignore */
+ }
+ else {
+ g_assert_not_reached();
+ }
+ }
+
+ *shift = cur_shift;
+
+ return TRUE;
+}
+
+static void
+rspamd_fuzzy_redis_update_callback(redisAsyncContext *c, gpointer r,
+ gpointer priv)
+{
+ struct rspamd_fuzzy_redis_session *session = priv;
+ redisReply *reply = r;
+
+ ev_timer_stop(session->event_loop, &session->timeout);
+
+ if (c->err == 0 && reply != NULL) {
+ rspamd_upstream_ok(session->up);
+
+ if (reply->type == REDIS_REPLY_ARRAY) {
+ /* TODO: check all replies somehow */
+ if (session->callback.cb_update) {
+ session->callback.cb_update(TRUE,
+ session->nadded,
+ session->ndeleted,
+ session->nextended,
+ session->nignored,
+ session->cbdata);
+ }
+ }
+ else {
+ if (reply->type == REDIS_REPLY_ERROR) {
+ msg_err_redis_session("fuzzy backend redis error: \"%s\"",
+ reply->str);
+ }
+ if (session->callback.cb_update) {
+ session->callback.cb_update(FALSE, 0, 0, 0, 0, session->cbdata);
+ }
+ }
+ }
+ else {
+ if (session->callback.cb_update) {
+ session->callback.cb_update(FALSE, 0, 0, 0, 0, session->cbdata);
+ }
+
+ if (c->errstr) {
+ msg_err_redis_session("error sending update to redis %s: %s",
+ rspamd_inet_address_to_string_pretty(rspamd_upstream_addr_cur(session->up)),
+ c->errstr);
+ rspamd_upstream_fail(session->up, FALSE, c->errstr);
+ }
+ }
+
+ rspamd_fuzzy_redis_session_dtor(session, FALSE);
+}
+
+void rspamd_fuzzy_backend_update_redis(struct rspamd_fuzzy_backend *bk,
+ GArray *updates, const gchar *src,
+ rspamd_fuzzy_update_cb cb, void *ud,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_redis *backend = subr_ud;
+ struct rspamd_fuzzy_redis_session *session;
+ struct upstream *up;
+ struct upstream_list *ups;
+ rspamd_inet_addr_t *addr;
+ guint i;
+ GString *key;
+ struct fuzzy_peer_cmd *io_cmd;
+ struct rspamd_fuzzy_cmd *cmd = NULL;
+ guint nargs, cur_shift;
+
+ g_assert(backend != NULL);
+
+ ups = rspamd_redis_get_servers(backend, "write_servers");
+ if (!ups) {
+ if (cb) {
+ cb(FALSE, 0, 0, 0, 0, ud);
+ }
+
+ return;
+ }
+
+ session = g_malloc0(sizeof(*session));
+ session->backend = backend;
+ REF_RETAIN(session->backend);
+
+ /*
+ * For each normal hash addition we do 3 redis commands:
+ * HSET <key> F <flag> **OR** HSETNX <key> F <flag> when flag is weak
+ * HINCRBY <key> V <weight>
+ * EXPIRE <key> <expire>
+ * INCR <prefix||fuzzy_count>
+ *
+ * Where <key> is <prefix> || <digest>
+ *
+ * For each command with shingles we additionally emit 32 commands:
+ * SETEX <prefix>_<number>_<value> <expire> <digest>
+ *
+ * For each delete command we emit:
+ * DEL <key>
+ *
+ * For each delete command with shingles we emit also 32 commands:
+ * DEL <prefix>_<number>_<value>
+ * DECR <prefix||fuzzy_count>
+ */
+
+ nargs = 4;
+
+ for (i = 0; i < updates->len; i++) {
+ io_cmd = &g_array_index(updates, struct fuzzy_peer_cmd, i);
+
+ if (io_cmd->is_shingle) {
+ cmd = &io_cmd->cmd.shingle.basic;
+ }
+ else {
+ cmd = &io_cmd->cmd.normal;
+ }
+
+ if (cmd->cmd == FUZZY_WRITE) {
+ nargs += 17;
+ session->nadded++;
+
+ if (io_cmd->is_shingle) {
+ nargs += RSPAMD_SHINGLE_SIZE * 4;
+ }
+ }
+ else if (cmd->cmd == FUZZY_DEL) {
+ nargs += 4;
+ session->ndeleted++;
+
+ if (io_cmd->is_shingle) {
+ nargs += RSPAMD_SHINGLE_SIZE * 2;
+ }
+ }
+ else if (cmd->cmd == FUZZY_REFRESH) {
+ nargs += 3;
+ session->nextended++;
+
+ if (io_cmd->is_shingle) {
+ nargs += RSPAMD_SHINGLE_SIZE * 3;
+ }
+ }
+ else {
+ session->nignored++;
+ }
+ }
+
+ /* Now we need to create a new request */
+ session->callback.cb_update = cb;
+ session->cbdata = ud;
+ session->command = RSPAMD_FUZZY_REDIS_COMMAND_UPDATES;
+ session->cmd = cmd;
+ session->prob = 1.0f;
+ session->event_loop = rspamd_fuzzy_backend_event_base(bk);
+
+ /* First of all check digest */
+ session->nargs = nargs;
+ session->argv = g_malloc0(sizeof(gchar *) * session->nargs);
+ session->argv_lens = g_malloc0(sizeof(gsize) * session->nargs);
+
+ up = rspamd_upstream_get(ups,
+ RSPAMD_UPSTREAM_MASTER_SLAVE,
+ NULL,
+ 0);
+
+ session->up = rspamd_upstream_ref(up);
+ addr = rspamd_upstream_addr_next(up);
+ g_assert(addr != NULL);
+ session->ctx = rspamd_redis_pool_connect(backend->pool,
+ backend->dbname,
+ backend->username, backend->password,
+ rspamd_inet_address_to_string(addr),
+ rspamd_inet_address_get_port(addr));
+
+ if (session->ctx == NULL) {
+ rspamd_upstream_fail(up, TRUE, strerror(errno));
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+
+ if (cb) {
+ cb(FALSE, 0, 0, 0, 0, ud);
+ }
+ }
+ else {
+ /* Start with MULTI command */
+ session->argv[0] = g_strdup("MULTI");
+ session->argv_lens[0] = 5;
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 1,
+ (const gchar **) session->argv,
+ session->argv_lens) != REDIS_OK) {
+
+ if (cb) {
+ cb(FALSE, 0, 0, 0, 0, ud);
+ }
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+
+ return;
+ }
+
+ /* Now split the rest of commands in packs and emit them command by command */
+ cur_shift = 1;
+
+ for (i = 0; i < updates->len; i++) {
+ io_cmd = &g_array_index(updates, struct fuzzy_peer_cmd, i);
+
+ if (!rspamd_fuzzy_update_append_command(bk, session, io_cmd,
+ &cur_shift)) {
+ if (cb) {
+ cb(FALSE, 0, 0, 0, 0, ud);
+ }
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+
+ return;
+ }
+ }
+
+ /* Now INCR command for the source */
+ key = g_string_new(backend->redis_object);
+ g_string_append(key, src);
+ session->argv[cur_shift] = g_strdup("INCR");
+ session->argv_lens[cur_shift++] = 4;
+ session->argv[cur_shift] = key->str;
+ session->argv_lens[cur_shift++] = key->len;
+ g_string_free(key, FALSE);
+
+ if (redisAsyncCommandArgv(session->ctx, NULL, NULL,
+ 2,
+ (const gchar **) &session->argv[cur_shift - 2],
+ &session->argv_lens[cur_shift - 2]) != REDIS_OK) {
+
+ if (cb) {
+ cb(FALSE, 0, 0, 0, 0, ud);
+ }
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+
+ return;
+ }
+
+ /* Finally we call EXEC with a specific callback */
+ session->argv[cur_shift] = g_strdup("EXEC");
+ session->argv_lens[cur_shift] = 4;
+
+ if (redisAsyncCommandArgv(session->ctx,
+ rspamd_fuzzy_redis_update_callback, session,
+ 1,
+ (const gchar **) &session->argv[cur_shift],
+ &session->argv_lens[cur_shift]) != REDIS_OK) {
+
+ if (cb) {
+ cb(FALSE, 0, 0, 0, 0, ud);
+ }
+ rspamd_fuzzy_redis_session_dtor(session, TRUE);
+
+ return;
+ }
+ else {
+ /* Add timeout */
+ session->timeout.data = session;
+ ev_now_update_if_cheap((struct ev_loop *) session->event_loop);
+ ev_timer_init(&session->timeout,
+ rspamd_fuzzy_redis_timeout,
+ session->backend->timeout, 0.0);
+ ev_timer_start(session->event_loop, &session->timeout);
+ }
+ }
+}
+
+void rspamd_fuzzy_backend_close_redis(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud)
+{
+ struct rspamd_fuzzy_backend_redis *backend = subr_ud;
+
+ g_assert(backend != NULL);
+
+ /*
+ * XXX: we leak lua registry element there to avoid crashing
+ * due to chicken-egg problem between lua state termination and
+ * redis pool termination.
+ * Here, we assume that redis pool is destroyed AFTER lua_state,
+ * so all connections pending will release references but due to
+ * `terminated` hack they will not try to access Lua stuff
+ * This is enabled merely if we have connections pending (e.g. refcount > 1)
+ */
+ if (backend->ref.refcount > 1) {
+ backend->terminated = true;
+ }
+ REF_RELEASE(backend);
+}
diff --git a/src/libserver/fuzzy_backend/fuzzy_backend_redis.h b/src/libserver/fuzzy_backend/fuzzy_backend_redis.h
new file mode 100644
index 0000000..3cfa162
--- /dev/null
+++ b/src/libserver/fuzzy_backend/fuzzy_backend_redis.h
@@ -0,0 +1,67 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBSERVER_FUZZY_BACKEND_REDIS_H_
+#define SRC_LIBSERVER_FUZZY_BACKEND_REDIS_H_
+
+#include "config.h"
+#include "fuzzy_backend.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Subroutines for fuzzy_backend
+ */
+void *rspamd_fuzzy_backend_init_redis(struct rspamd_fuzzy_backend *bk,
+ const ucl_object_t *obj,
+ struct rspamd_config *cfg,
+ GError **err);
+
+void rspamd_fuzzy_backend_check_redis(struct rspamd_fuzzy_backend *bk,
+ const struct rspamd_fuzzy_cmd *cmd,
+ rspamd_fuzzy_check_cb cb, void *ud,
+ void *subr_ud);
+
+void rspamd_fuzzy_backend_update_redis(struct rspamd_fuzzy_backend *bk,
+ GArray *updates, const gchar *src,
+ rspamd_fuzzy_update_cb cb, void *ud,
+ void *subr_ud);
+
+void rspamd_fuzzy_backend_count_redis(struct rspamd_fuzzy_backend *bk,
+ rspamd_fuzzy_count_cb cb, void *ud,
+ void *subr_ud);
+
+void rspamd_fuzzy_backend_version_redis(struct rspamd_fuzzy_backend *bk,
+ const gchar *src,
+ rspamd_fuzzy_version_cb cb, void *ud,
+ void *subr_ud);
+
+const gchar *rspamd_fuzzy_backend_id_redis(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud);
+
+void rspamd_fuzzy_backend_expire_redis(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud);
+
+void rspamd_fuzzy_backend_close_redis(struct rspamd_fuzzy_backend *bk,
+ void *subr_ud);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBSERVER_FUZZY_BACKEND_REDIS_H_ */
diff --git a/src/libserver/fuzzy_backend/fuzzy_backend_sqlite.c b/src/libserver/fuzzy_backend/fuzzy_backend_sqlite.c
new file mode 100644
index 0000000..9ec448e
--- /dev/null
+++ b/src/libserver/fuzzy_backend/fuzzy_backend_sqlite.c
@@ -0,0 +1,1029 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "fuzzy_backend.h"
+#include "fuzzy_backend_sqlite.h"
+#include "unix-std.h"
+
+#include <sqlite3.h>
+#include "libutil/sqlite_utils.h"
+
+struct rspamd_fuzzy_backend_sqlite {
+ sqlite3 *db;
+ char *path;
+ gchar id[MEMPOOL_UID_LEN];
+ gsize count;
+ gsize expired;
+ rspamd_mempool_t *pool;
+};
+
+static const gdouble sql_sleep_time = 0.1;
+static const guint max_retries = 10;
+
+#define msg_err_fuzzy_backend(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ backend->pool->tag.tagname, backend->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_warn_fuzzy_backend(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ backend->pool->tag.tagname, backend->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_info_fuzzy_backend(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ backend->pool->tag.tagname, backend->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_debug_fuzzy_backend(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_fuzzy_sqlite_log_id, backend->pool->tag.tagname, backend->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(fuzzy_sqlite)
+
+static const char *create_tables_sql =
+ "BEGIN;"
+ "CREATE TABLE IF NOT EXISTS digests("
+ " id INTEGER PRIMARY KEY,"
+ " flag INTEGER NOT NULL,"
+ " digest TEXT NOT NULL,"
+ " value INTEGER,"
+ " time INTEGER);"
+ "CREATE TABLE IF NOT EXISTS shingles("
+ " value INTEGER NOT NULL,"
+ " number INTEGER NOT NULL,"
+ " digest_id INTEGER REFERENCES digests(id) ON DELETE CASCADE "
+ " ON UPDATE CASCADE);"
+ "CREATE TABLE IF NOT EXISTS sources("
+ " name TEXT UNIQUE,"
+ " version INTEGER,"
+ " last INTEGER);"
+ "CREATE UNIQUE INDEX IF NOT EXISTS d ON digests(digest);"
+ "CREATE INDEX IF NOT EXISTS t ON digests(time);"
+ "CREATE INDEX IF NOT EXISTS dgst_id ON shingles(digest_id);"
+ "CREATE UNIQUE INDEX IF NOT EXISTS s ON shingles(value, number);"
+ "COMMIT;";
+#if 0
+static const char *create_index_sql =
+ "BEGIN;"
+ "CREATE UNIQUE INDEX IF NOT EXISTS d ON digests(digest);"
+ "CREATE INDEX IF NOT EXISTS t ON digests(time);"
+ "CREATE INDEX IF NOT EXISTS dgst_id ON shingles(digest_id);"
+ "CREATE UNIQUE INDEX IF NOT EXISTS s ON shingles(value, number);"
+ "COMMIT;";
+#endif
+enum rspamd_fuzzy_statement_idx {
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_START = 0,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_ROLLBACK,
+ RSPAMD_FUZZY_BACKEND_INSERT,
+ RSPAMD_FUZZY_BACKEND_UPDATE,
+ RSPAMD_FUZZY_BACKEND_UPDATE_FLAG,
+ RSPAMD_FUZZY_BACKEND_INSERT_SHINGLE,
+ RSPAMD_FUZZY_BACKEND_CHECK,
+ RSPAMD_FUZZY_BACKEND_CHECK_SHINGLE,
+ RSPAMD_FUZZY_BACKEND_GET_DIGEST_BY_ID,
+ RSPAMD_FUZZY_BACKEND_DELETE,
+ RSPAMD_FUZZY_BACKEND_COUNT,
+ RSPAMD_FUZZY_BACKEND_EXPIRE,
+ RSPAMD_FUZZY_BACKEND_VACUUM,
+ RSPAMD_FUZZY_BACKEND_DELETE_ORPHANED,
+ RSPAMD_FUZZY_BACKEND_ADD_SOURCE,
+ RSPAMD_FUZZY_BACKEND_VERSION,
+ RSPAMD_FUZZY_BACKEND_SET_VERSION,
+ RSPAMD_FUZZY_BACKEND_MAX
+};
+static struct rspamd_fuzzy_stmts {
+ enum rspamd_fuzzy_statement_idx idx;
+ const gchar *sql;
+ const gchar *args;
+ sqlite3_stmt *stmt;
+ gint result;
+} prepared_stmts[RSPAMD_FUZZY_BACKEND_MAX] =
+ {
+ {.idx = RSPAMD_FUZZY_BACKEND_TRANSACTION_START,
+ .sql = "BEGIN TRANSACTION;",
+ .args = "",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT,
+ .sql = "COMMIT;",
+ .args = "",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_TRANSACTION_ROLLBACK,
+ .sql = "ROLLBACK;",
+ .args = "",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_INSERT,
+ .sql = "INSERT INTO digests(flag, digest, value, time) VALUES"
+ "(?1, ?2, ?3, strftime('%s','now'));",
+ .args = "SDI",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_UPDATE,
+ .sql = "UPDATE digests SET value = value + ?1, time = strftime('%s','now') WHERE "
+ "digest==?2;",
+ .args = "ID",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_UPDATE_FLAG,
+ .sql = "UPDATE digests SET value = ?1, flag = ?2, time = strftime('%s','now') WHERE "
+ "digest==?3;",
+ .args = "IID",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_INSERT_SHINGLE,
+ .sql = "INSERT OR REPLACE INTO shingles(value, number, digest_id) "
+ "VALUES (?1, ?2, ?3);",
+ .args = "III",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_CHECK,
+ .sql = "SELECT value, time, flag FROM digests WHERE digest==?1;",
+ .args = "D",
+ .stmt = NULL,
+ .result = SQLITE_ROW},
+ {.idx = RSPAMD_FUZZY_BACKEND_CHECK_SHINGLE,
+ .sql = "SELECT digest_id FROM shingles WHERE value=?1 AND number=?2",
+ .args = "IS",
+ .stmt = NULL,
+ .result = SQLITE_ROW},
+ {.idx = RSPAMD_FUZZY_BACKEND_GET_DIGEST_BY_ID,
+ .sql = "SELECT digest, value, time, flag FROM digests WHERE id=?1",
+ .args = "I",
+ .stmt = NULL,
+ .result = SQLITE_ROW},
+ {.idx = RSPAMD_FUZZY_BACKEND_DELETE,
+ .sql = "DELETE FROM digests WHERE digest==?1;",
+ .args = "D",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_COUNT,
+ .sql = "SELECT COUNT(*) FROM digests;",
+ .args = "",
+ .stmt = NULL,
+ .result = SQLITE_ROW},
+ {.idx = RSPAMD_FUZZY_BACKEND_EXPIRE,
+ .sql = "DELETE FROM digests WHERE id IN (SELECT id FROM digests WHERE time < ?1 LIMIT ?2);",
+ .args = "II",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_VACUUM,
+ .sql = "VACUUM;",
+ .args = "",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_DELETE_ORPHANED,
+ .sql = "DELETE FROM shingles WHERE value=?1 AND number=?2;",
+ .args = "II",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_ADD_SOURCE,
+ .sql = "INSERT OR IGNORE INTO sources(name, version, last) VALUES (?1, ?2, ?3);",
+ .args = "TII",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+ {.idx = RSPAMD_FUZZY_BACKEND_VERSION,
+ .sql = "SELECT version FROM sources WHERE name=?1;",
+ .args = "T",
+ .stmt = NULL,
+ .result = SQLITE_ROW},
+ {.idx = RSPAMD_FUZZY_BACKEND_SET_VERSION,
+ .sql = "INSERT OR REPLACE INTO sources (name, version, last) VALUES (?3, ?1, ?2);",
+ .args = "IIT",
+ .stmt = NULL,
+ .result = SQLITE_DONE},
+};
+
+static GQuark
+rspamd_fuzzy_backend_sqlite_quark(void)
+{
+ return g_quark_from_static_string("fuzzy-backend-sqlite");
+}
+
+static gboolean
+rspamd_fuzzy_backend_sqlite_prepare_stmts(struct rspamd_fuzzy_backend_sqlite *bk, GError **err)
+{
+ int i;
+
+ for (i = 0; i < RSPAMD_FUZZY_BACKEND_MAX; i++) {
+ if (prepared_stmts[i].stmt != NULL) {
+ /* Skip already prepared statements */
+ continue;
+ }
+ if (sqlite3_prepare_v2(bk->db, prepared_stmts[i].sql, -1,
+ &prepared_stmts[i].stmt, NULL) != SQLITE_OK) {
+ g_set_error(err, rspamd_fuzzy_backend_sqlite_quark(),
+ -1, "Cannot initialize prepared sql `%s`: %s",
+ prepared_stmts[i].sql, sqlite3_errmsg(bk->db));
+
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+static int
+rspamd_fuzzy_backend_sqlite_cleanup_stmt(struct rspamd_fuzzy_backend_sqlite *backend,
+ int idx)
+{
+ sqlite3_stmt *stmt;
+
+ if (idx < 0 || idx >= RSPAMD_FUZZY_BACKEND_MAX) {
+
+ return -1;
+ }
+
+ msg_debug_fuzzy_backend("resetting `%s`", prepared_stmts[idx].sql);
+ stmt = prepared_stmts[idx].stmt;
+ sqlite3_clear_bindings(stmt);
+ sqlite3_reset(stmt);
+
+ return SQLITE_OK;
+}
+
+static int
+rspamd_fuzzy_backend_sqlite_run_stmt(struct rspamd_fuzzy_backend_sqlite *backend,
+ gboolean auto_cleanup,
+ int idx, ...)
+{
+ int retcode;
+ va_list ap;
+ sqlite3_stmt *stmt;
+ int i;
+ const char *argtypes;
+ guint retries = 0;
+ struct timespec ts;
+
+ if (idx < 0 || idx >= RSPAMD_FUZZY_BACKEND_MAX) {
+
+ return -1;
+ }
+
+ stmt = prepared_stmts[idx].stmt;
+ g_assert((int) prepared_stmts[idx].idx == idx);
+
+ if (stmt == NULL) {
+ if ((retcode = sqlite3_prepare_v2(backend->db, prepared_stmts[idx].sql, -1,
+ &prepared_stmts[idx].stmt, NULL)) != SQLITE_OK) {
+ msg_err_fuzzy_backend("Cannot initialize prepared sql `%s`: %s",
+ prepared_stmts[idx].sql, sqlite3_errmsg(backend->db));
+
+ return retcode;
+ }
+ stmt = prepared_stmts[idx].stmt;
+ }
+
+ msg_debug_fuzzy_backend("executing `%s` %s auto cleanup",
+ prepared_stmts[idx].sql, auto_cleanup ? "with" : "without");
+ argtypes = prepared_stmts[idx].args;
+ sqlite3_clear_bindings(stmt);
+ sqlite3_reset(stmt);
+ va_start(ap, idx);
+
+ for (i = 0; argtypes[i] != '\0'; i++) {
+ switch (argtypes[i]) {
+ case 'T':
+ sqlite3_bind_text(stmt, i + 1, va_arg(ap, const char *), -1,
+ SQLITE_STATIC);
+ break;
+ case 'I':
+ sqlite3_bind_int64(stmt, i + 1, va_arg(ap, gint64));
+ break;
+ case 'S':
+ sqlite3_bind_int(stmt, i + 1, va_arg(ap, gint));
+ break;
+ case 'D':
+ /* Special case for digests variable */
+ sqlite3_bind_text(stmt, i + 1, va_arg(ap, const char *), 64,
+ SQLITE_STATIC);
+ break;
+ }
+ }
+
+ va_end(ap);
+
+retry:
+ retcode = sqlite3_step(stmt);
+
+ if (retcode == prepared_stmts[idx].result) {
+ retcode = SQLITE_OK;
+ }
+ else {
+ if ((retcode == SQLITE_BUSY ||
+ retcode == SQLITE_LOCKED) &&
+ retries++ < max_retries) {
+ double_to_ts(sql_sleep_time, &ts);
+ nanosleep(&ts, NULL);
+ goto retry;
+ }
+
+ msg_debug_fuzzy_backend("failed to execute query %s: %d, %s", prepared_stmts[idx].sql,
+ retcode, sqlite3_errmsg(backend->db));
+ }
+
+ if (auto_cleanup) {
+ sqlite3_clear_bindings(stmt);
+ sqlite3_reset(stmt);
+ }
+
+ return retcode;
+}
+
+static void
+rspamd_fuzzy_backend_sqlite_close_stmts(struct rspamd_fuzzy_backend_sqlite *bk)
+{
+ int i;
+
+ for (i = 0; i < RSPAMD_FUZZY_BACKEND_MAX; i++) {
+ if (prepared_stmts[i].stmt != NULL) {
+ sqlite3_finalize(prepared_stmts[i].stmt);
+ prepared_stmts[i].stmt = NULL;
+ }
+ }
+
+ return;
+}
+
+static gboolean
+rspamd_fuzzy_backend_sqlite_run_sql(const gchar *sql, struct rspamd_fuzzy_backend_sqlite *bk,
+ GError **err)
+{
+ guint retries = 0;
+ struct timespec ts;
+ gint ret;
+
+ do {
+ ret = sqlite3_exec(bk->db, sql, NULL, NULL, NULL);
+ double_to_ts(sql_sleep_time, &ts);
+ } while (ret == SQLITE_BUSY && retries++ < max_retries &&
+ nanosleep(&ts, NULL) == 0);
+
+ if (ret != SQLITE_OK) {
+ g_set_error(err, rspamd_fuzzy_backend_sqlite_quark(),
+ -1, "Cannot execute raw sql `%s`: %s",
+ sql, sqlite3_errmsg(bk->db));
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static struct rspamd_fuzzy_backend_sqlite *
+rspamd_fuzzy_backend_sqlite_open_db(const gchar *path, GError **err)
+{
+ struct rspamd_fuzzy_backend_sqlite *bk;
+ rspamd_cryptobox_hash_state_t st;
+ guchar hash_out[rspamd_cryptobox_HASHBYTES];
+
+ g_assert(path != NULL);
+
+ bk = g_malloc0(sizeof(*bk));
+ bk->path = g_strdup(path);
+ bk->expired = 0;
+ bk->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "fuzzy_backend", 0);
+ bk->db = rspamd_sqlite3_open_or_create(bk->pool, bk->path,
+ create_tables_sql, 1, err);
+
+ if (bk->db == NULL) {
+ rspamd_fuzzy_backend_sqlite_close(bk);
+
+ return NULL;
+ }
+
+ if (!rspamd_fuzzy_backend_sqlite_prepare_stmts(bk, err)) {
+ rspamd_fuzzy_backend_sqlite_close(bk);
+
+ return NULL;
+ }
+
+ /* Set id for the backend */
+ rspamd_cryptobox_hash_init(&st, NULL, 0);
+ rspamd_cryptobox_hash_update(&st, path, strlen(path));
+ rspamd_cryptobox_hash_final(&st, hash_out);
+ rspamd_snprintf(bk->id, sizeof(bk->id), "%xs", hash_out);
+ memcpy(bk->pool->tag.uid, bk->id, sizeof(bk->pool->tag.uid));
+
+ return bk;
+}
+
+struct rspamd_fuzzy_backend_sqlite *
+rspamd_fuzzy_backend_sqlite_open(const gchar *path,
+ gboolean vacuum,
+ GError **err)
+{
+ struct rspamd_fuzzy_backend_sqlite *backend;
+
+ if (path == NULL) {
+ g_set_error(err, rspamd_fuzzy_backend_sqlite_quark(),
+ ENOENT, "Path has not been specified");
+ return NULL;
+ }
+
+ /* Open database */
+ if ((backend = rspamd_fuzzy_backend_sqlite_open_db(path, err)) == NULL) {
+ return NULL;
+ }
+
+ if (rspamd_fuzzy_backend_sqlite_run_stmt(backend, FALSE, RSPAMD_FUZZY_BACKEND_COUNT) == SQLITE_OK) {
+ backend->count = sqlite3_column_int64(
+ prepared_stmts[RSPAMD_FUZZY_BACKEND_COUNT].stmt, 0);
+ }
+
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend, RSPAMD_FUZZY_BACKEND_COUNT);
+
+ return backend;
+}
+
+static gint
+rspamd_fuzzy_backend_sqlite_int64_cmp(const void *a, const void *b)
+{
+ gint64 ia = *(gint64 *) a, ib = *(gint64 *) b;
+
+ return (ia - ib);
+}
+
+struct rspamd_fuzzy_reply
+rspamd_fuzzy_backend_sqlite_check(struct rspamd_fuzzy_backend_sqlite *backend,
+ const struct rspamd_fuzzy_cmd *cmd, gint64 expire)
+{
+ struct rspamd_fuzzy_reply rep;
+ const struct rspamd_fuzzy_shingle_cmd *shcmd;
+ int rc;
+ gint64 timestamp;
+ gint64 shingle_values[RSPAMD_SHINGLE_SIZE], i, sel_id, cur_id,
+ cur_cnt, max_cnt;
+
+ memset(&rep, 0, sizeof(rep));
+ memcpy(rep.digest, cmd->digest, sizeof(rep.digest));
+
+ if (backend == NULL) {
+ return rep;
+ }
+
+ /* Try direct match first of all */
+ rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_START);
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, FALSE,
+ RSPAMD_FUZZY_BACKEND_CHECK,
+ cmd->digest);
+
+ if (rc == SQLITE_OK) {
+ timestamp = sqlite3_column_int64(
+ prepared_stmts[RSPAMD_FUZZY_BACKEND_CHECK].stmt, 1);
+ if (time(NULL) - timestamp > expire) {
+ /* Expire element */
+ msg_debug_fuzzy_backend("requested hash has been expired");
+ }
+ else {
+ rep.v1.value = sqlite3_column_int64(
+ prepared_stmts[RSPAMD_FUZZY_BACKEND_CHECK].stmt, 0);
+ rep.v1.prob = 1.0;
+ rep.v1.flag = sqlite3_column_int(
+ prepared_stmts[RSPAMD_FUZZY_BACKEND_CHECK].stmt, 2);
+ }
+ }
+ else if (cmd->shingles_count > 0) {
+ /* Fuzzy match */
+
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend, RSPAMD_FUZZY_BACKEND_CHECK);
+ shcmd = (const struct rspamd_fuzzy_shingle_cmd *) cmd;
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, FALSE,
+ RSPAMD_FUZZY_BACKEND_CHECK_SHINGLE,
+ shcmd->sgl.hashes[i], i);
+ if (rc == SQLITE_OK) {
+ shingle_values[i] = sqlite3_column_int64(
+ prepared_stmts[RSPAMD_FUZZY_BACKEND_CHECK_SHINGLE].stmt,
+ 0);
+ }
+ else {
+ shingle_values[i] = -1;
+ }
+ msg_debug_fuzzy_backend("looking for shingle %L -> %L: %d", i,
+ shcmd->sgl.hashes[i], rc);
+ }
+
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend,
+ RSPAMD_FUZZY_BACKEND_CHECK_SHINGLE);
+
+ qsort(shingle_values, RSPAMD_SHINGLE_SIZE, sizeof(gint64),
+ rspamd_fuzzy_backend_sqlite_int64_cmp);
+ sel_id = -1;
+ cur_id = -1;
+ cur_cnt = 0;
+ max_cnt = 0;
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ if (shingle_values[i] == -1) {
+ continue;
+ }
+
+ /* We have some value here, so we need to check it */
+ if (shingle_values[i] == cur_id) {
+ cur_cnt++;
+ }
+ else {
+ cur_id = shingle_values[i];
+ if (cur_cnt >= max_cnt) {
+ max_cnt = cur_cnt;
+ sel_id = cur_id;
+ }
+ cur_cnt = 0;
+ }
+ }
+
+ if (cur_cnt > max_cnt) {
+ max_cnt = cur_cnt;
+ }
+
+ if (sel_id != -1) {
+ /* We have some id selected here */
+ rep.v1.prob = (float) max_cnt / (float) RSPAMD_SHINGLE_SIZE;
+
+ if (rep.v1.prob > 0.5) {
+ msg_debug_fuzzy_backend(
+ "found fuzzy hash with probability %.2f",
+ rep.v1.prob);
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, FALSE,
+ RSPAMD_FUZZY_BACKEND_GET_DIGEST_BY_ID, sel_id);
+ if (rc == SQLITE_OK) {
+ timestamp = sqlite3_column_int64(
+ prepared_stmts[RSPAMD_FUZZY_BACKEND_GET_DIGEST_BY_ID].stmt,
+ 2);
+ if (time(NULL) - timestamp > expire) {
+ /* Expire element */
+ msg_debug_fuzzy_backend(
+ "requested hash has been expired");
+ rep.v1.prob = 0.0;
+ }
+ else {
+ rep.ts = timestamp;
+ memcpy(rep.digest, sqlite3_column_blob(prepared_stmts[RSPAMD_FUZZY_BACKEND_GET_DIGEST_BY_ID].stmt, 0), sizeof(rep.digest));
+ rep.v1.value = sqlite3_column_int64(
+ prepared_stmts[RSPAMD_FUZZY_BACKEND_GET_DIGEST_BY_ID].stmt,
+ 1);
+ rep.v1.flag = sqlite3_column_int(
+ prepared_stmts[RSPAMD_FUZZY_BACKEND_GET_DIGEST_BY_ID].stmt,
+ 3);
+ }
+ }
+ }
+ else {
+ /* Otherwise we assume that as error */
+ rep.v1.value = 0;
+ }
+
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend,
+ RSPAMD_FUZZY_BACKEND_GET_DIGEST_BY_ID);
+ }
+ }
+
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend, RSPAMD_FUZZY_BACKEND_CHECK);
+ rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT);
+
+ return rep;
+}
+
+gboolean
+rspamd_fuzzy_backend_sqlite_prepare_update(struct rspamd_fuzzy_backend_sqlite *backend,
+ const gchar *source)
+{
+ gint rc;
+
+ if (backend == NULL) {
+ return FALSE;
+ }
+
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_START);
+
+ if (rc != SQLITE_OK) {
+ msg_warn_fuzzy_backend("cannot start transaction for updates: %s",
+ sqlite3_errmsg(backend->db));
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_fuzzy_backend_sqlite_add(struct rspamd_fuzzy_backend_sqlite *backend,
+ const struct rspamd_fuzzy_cmd *cmd)
+{
+ int rc, i;
+ gint64 id, flag;
+ const struct rspamd_fuzzy_shingle_cmd *shcmd;
+
+ if (backend == NULL) {
+ return FALSE;
+ }
+
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, FALSE,
+ RSPAMD_FUZZY_BACKEND_CHECK,
+ cmd->digest);
+
+ if (rc == SQLITE_OK) {
+ /* Check flag */
+ flag = sqlite3_column_int64(
+ prepared_stmts[RSPAMD_FUZZY_BACKEND_CHECK].stmt,
+ 2);
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend, RSPAMD_FUZZY_BACKEND_CHECK);
+
+ if (flag == cmd->flag) {
+ /* We need to increase weight */
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_UPDATE,
+ (gint64) cmd->value,
+ cmd->digest);
+ if (rc != SQLITE_OK) {
+ msg_warn_fuzzy_backend("cannot update hash to %d -> "
+ "%*xs: %s",
+ (gint) cmd->flag,
+ (gint) sizeof(cmd->digest), cmd->digest,
+ sqlite3_errmsg(backend->db));
+ }
+ }
+ else {
+ /* We need to relearn actually */
+
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_UPDATE_FLAG,
+ (gint64) cmd->value,
+ (gint64) cmd->flag,
+ cmd->digest);
+
+ if (rc != SQLITE_OK) {
+ msg_warn_fuzzy_backend("cannot update hash to %d -> "
+ "%*xs: %s",
+ (gint) cmd->flag,
+ (gint) sizeof(cmd->digest), cmd->digest,
+ sqlite3_errmsg(backend->db));
+ }
+ }
+ }
+ else {
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend, RSPAMD_FUZZY_BACKEND_CHECK);
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, FALSE,
+ RSPAMD_FUZZY_BACKEND_INSERT,
+ (gint) cmd->flag,
+ cmd->digest,
+ (gint64) cmd->value);
+
+ if (rc == SQLITE_OK) {
+ if (cmd->shingles_count > 0) {
+ id = sqlite3_last_insert_rowid(backend->db);
+ shcmd = (const struct rspamd_fuzzy_shingle_cmd *) cmd;
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_INSERT_SHINGLE,
+ shcmd->sgl.hashes[i], (gint64) i, id);
+ msg_debug_fuzzy_backend("add shingle %d -> %L: %L",
+ i,
+ shcmd->sgl.hashes[i],
+ id);
+
+ if (rc != SQLITE_OK) {
+ msg_warn_fuzzy_backend("cannot add shingle %d -> "
+ "%L: %L: %s",
+ i,
+ shcmd->sgl.hashes[i],
+ id, sqlite3_errmsg(backend->db));
+ }
+ }
+ }
+ }
+ else {
+ msg_warn_fuzzy_backend("cannot add hash to %d -> "
+ "%*xs: %s",
+ (gint) cmd->flag,
+ (gint) sizeof(cmd->digest), cmd->digest,
+ sqlite3_errmsg(backend->db));
+ }
+
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend,
+ RSPAMD_FUZZY_BACKEND_INSERT);
+ }
+
+ return (rc == SQLITE_OK);
+}
+
+gboolean
+rspamd_fuzzy_backend_sqlite_finish_update(struct rspamd_fuzzy_backend_sqlite *backend,
+ const gchar *source, gboolean version_bump)
+{
+ gint rc = SQLITE_OK, wal_frames, wal_checkpointed, ver;
+
+ /* Get and update version */
+ if (version_bump) {
+ ver = rspamd_fuzzy_backend_sqlite_version(backend, source);
+ ++ver;
+
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_SET_VERSION,
+ (gint64) ver, (gint64) time(NULL), source);
+ }
+
+ if (rc == SQLITE_OK) {
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT);
+
+ if (rc != SQLITE_OK) {
+ msg_warn_fuzzy_backend("cannot commit updates: %s",
+ sqlite3_errmsg(backend->db));
+ rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_ROLLBACK);
+ return FALSE;
+ }
+ else {
+ if (!rspamd_sqlite3_sync(backend->db, &wal_frames, &wal_checkpointed)) {
+ msg_warn_fuzzy_backend("cannot commit checkpoint: %s",
+ sqlite3_errmsg(backend->db));
+ }
+ else if (wal_checkpointed > 0) {
+ msg_info_fuzzy_backend("total number of frames in the wal file: "
+ "%d, checkpointed: %d",
+ wal_frames, wal_checkpointed);
+ }
+ }
+ }
+ else {
+ msg_warn_fuzzy_backend("cannot update version for %s: %s", source,
+ sqlite3_errmsg(backend->db));
+ rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_ROLLBACK);
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_fuzzy_backend_sqlite_del(struct rspamd_fuzzy_backend_sqlite *backend,
+ const struct rspamd_fuzzy_cmd *cmd)
+{
+ int rc = -1;
+
+ if (backend == NULL) {
+ return FALSE;
+ }
+
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, FALSE,
+ RSPAMD_FUZZY_BACKEND_CHECK,
+ cmd->digest);
+
+ if (rc == SQLITE_OK) {
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend, RSPAMD_FUZZY_BACKEND_CHECK);
+
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_DELETE,
+ cmd->digest);
+ if (rc != SQLITE_OK) {
+ msg_warn_fuzzy_backend("cannot update hash to %d -> "
+ "%*xs: %s",
+ (gint) cmd->flag,
+ (gint) sizeof(cmd->digest), cmd->digest,
+ sqlite3_errmsg(backend->db));
+ }
+ }
+ else {
+ /* Hash is missing */
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend, RSPAMD_FUZZY_BACKEND_CHECK);
+ }
+
+ return (rc == SQLITE_OK);
+}
+
+gboolean
+rspamd_fuzzy_backend_sqlite_sync(struct rspamd_fuzzy_backend_sqlite *backend,
+ gint64 expire,
+ gboolean clean_orphaned)
+{
+ struct orphaned_shingle_elt {
+ gint64 value;
+ gint64 number;
+ };
+
+ /* Do not do more than 5k ops per step */
+ const guint64 max_changes = 5000;
+ gboolean ret = FALSE;
+ gint64 expire_lim, expired;
+ gint rc, i, orphaned_cnt = 0;
+ GError *err = NULL;
+ static const gchar orphaned_shingles[] = "SELECT shingles.value,shingles.number "
+ "FROM shingles "
+ "LEFT JOIN digests ON "
+ "shingles.digest_id=digests.id WHERE "
+ "digests.id IS NULL;";
+ sqlite3_stmt *stmt;
+ GArray *orphaned;
+ struct orphaned_shingle_elt orphaned_elt, *pelt;
+
+
+ if (backend == NULL) {
+ return FALSE;
+ }
+
+ /* Perform expire */
+ if (expire > 0) {
+ expire_lim = time(NULL) - expire;
+
+ if (expire_lim > 0) {
+ ret = rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_START);
+
+ if (ret == SQLITE_OK) {
+
+ rc = rspamd_fuzzy_backend_sqlite_run_stmt(backend, FALSE,
+ RSPAMD_FUZZY_BACKEND_EXPIRE, expire_lim, max_changes);
+
+ if (rc == SQLITE_OK) {
+ expired = sqlite3_changes(backend->db);
+
+ if (expired > 0) {
+ backend->expired += expired;
+ msg_info_fuzzy_backend("expired %L hashes", expired);
+ }
+ }
+ else {
+ msg_warn_fuzzy_backend(
+ "cannot execute expired statement: %s",
+ sqlite3_errmsg(backend->db));
+ }
+
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend,
+ RSPAMD_FUZZY_BACKEND_EXPIRE);
+
+ ret = rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT);
+
+ if (ret != SQLITE_OK) {
+ rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_ROLLBACK);
+ }
+ }
+ if (ret != SQLITE_OK) {
+ msg_warn_fuzzy_backend("cannot expire db: %s",
+ sqlite3_errmsg(backend->db));
+ }
+ }
+ }
+
+ /* Cleanup database */
+ if (clean_orphaned) {
+ ret = rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_START);
+
+ if (ret == SQLITE_OK) {
+ if ((rc = sqlite3_prepare_v2(backend->db,
+ orphaned_shingles,
+ -1,
+ &stmt,
+ NULL)) != SQLITE_OK) {
+ msg_warn_fuzzy_backend("cannot cleanup shingles: %s",
+ sqlite3_errmsg(backend->db));
+ }
+ else {
+ orphaned = g_array_new(FALSE,
+ FALSE,
+ sizeof(struct orphaned_shingle_elt));
+
+ while (sqlite3_step(stmt) == SQLITE_ROW) {
+ orphaned_elt.value = sqlite3_column_int64(stmt, 0);
+ orphaned_elt.number = sqlite3_column_int64(stmt, 1);
+ g_array_append_val(orphaned, orphaned_elt);
+
+ if (orphaned->len > max_changes) {
+ break;
+ }
+ }
+
+ sqlite3_finalize(stmt);
+ orphaned_cnt = orphaned->len;
+
+ if (orphaned_cnt > 0) {
+ msg_info_fuzzy_backend(
+ "going to delete %ud orphaned shingles",
+ orphaned_cnt);
+ /* Need to delete orphaned elements */
+ for (i = 0; i < (gint) orphaned_cnt; i++) {
+ pelt = &g_array_index(orphaned,
+ struct orphaned_shingle_elt,
+ i);
+ rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_DELETE_ORPHANED,
+ pelt->value, pelt->number);
+ }
+ }
+
+
+ g_array_free(orphaned, TRUE);
+ }
+
+ ret = rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT);
+
+ if (ret == SQLITE_OK) {
+ msg_info_fuzzy_backend(
+ "deleted %ud orphaned shingles",
+ orphaned_cnt);
+ }
+ else {
+ msg_warn_fuzzy_backend(
+ "cannot synchronize fuzzy backend: %e",
+ err);
+ rspamd_fuzzy_backend_sqlite_run_stmt(backend, TRUE,
+ RSPAMD_FUZZY_BACKEND_TRANSACTION_ROLLBACK);
+ }
+ }
+ }
+
+ return ret;
+}
+
+
+void rspamd_fuzzy_backend_sqlite_close(struct rspamd_fuzzy_backend_sqlite *backend)
+{
+ if (backend != NULL) {
+ if (backend->db != NULL) {
+ rspamd_fuzzy_backend_sqlite_close_stmts(backend);
+ sqlite3_close(backend->db);
+ }
+
+ if (backend->path != NULL) {
+ g_free(backend->path);
+ }
+
+ if (backend->pool) {
+ rspamd_mempool_delete(backend->pool);
+ }
+
+ g_free(backend);
+ }
+}
+
+
+gsize rspamd_fuzzy_backend_sqlite_count(struct rspamd_fuzzy_backend_sqlite *backend)
+{
+ if (backend) {
+ if (rspamd_fuzzy_backend_sqlite_run_stmt(backend, FALSE,
+ RSPAMD_FUZZY_BACKEND_COUNT) == SQLITE_OK) {
+ backend->count = sqlite3_column_int64(
+ prepared_stmts[RSPAMD_FUZZY_BACKEND_COUNT].stmt, 0);
+ }
+
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend, RSPAMD_FUZZY_BACKEND_COUNT);
+
+ return backend->count;
+ }
+
+ return 0;
+}
+
+gint rspamd_fuzzy_backend_sqlite_version(struct rspamd_fuzzy_backend_sqlite *backend,
+ const gchar *source)
+{
+ gint ret = 0;
+
+ if (backend) {
+ if (rspamd_fuzzy_backend_sqlite_run_stmt(backend, FALSE,
+ RSPAMD_FUZZY_BACKEND_VERSION, source) == SQLITE_OK) {
+ ret = sqlite3_column_int64(
+ prepared_stmts[RSPAMD_FUZZY_BACKEND_VERSION].stmt, 0);
+ }
+
+ rspamd_fuzzy_backend_sqlite_cleanup_stmt(backend, RSPAMD_FUZZY_BACKEND_VERSION);
+ }
+
+ return ret;
+}
+
+gsize rspamd_fuzzy_backend_sqlite_expired(struct rspamd_fuzzy_backend_sqlite *backend)
+{
+ return backend != NULL ? backend->expired : 0;
+}
+
+const gchar *
+rspamd_fuzzy_sqlite_backend_id(struct rspamd_fuzzy_backend_sqlite *backend)
+{
+ return backend != NULL ? backend->id : 0;
+}
diff --git a/src/libserver/fuzzy_backend/fuzzy_backend_sqlite.h b/src/libserver/fuzzy_backend/fuzzy_backend_sqlite.h
new file mode 100644
index 0000000..766f7c9
--- /dev/null
+++ b/src/libserver/fuzzy_backend/fuzzy_backend_sqlite.h
@@ -0,0 +1,107 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FUZZY_BACKEND_H_
+#define FUZZY_BACKEND_H_
+
+#include "config.h"
+#include "fuzzy_wire.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_fuzzy_backend_sqlite;
+
+/**
+ * Open fuzzy backend
+ * @param path file to open (legacy file will be converted automatically)
+ * @param err error pointer
+ * @return backend structure or NULL
+ */
+struct rspamd_fuzzy_backend_sqlite *rspamd_fuzzy_backend_sqlite_open(const gchar *path,
+ gboolean vacuum,
+ GError **err);
+
+/**
+ * Check specified fuzzy in the backend
+ * @param backend
+ * @param cmd
+ * @return reply with probability and weight
+ */
+struct rspamd_fuzzy_reply rspamd_fuzzy_backend_sqlite_check(
+ struct rspamd_fuzzy_backend_sqlite *backend,
+ const struct rspamd_fuzzy_cmd *cmd,
+ gint64 expire);
+
+/**
+ * Prepare storage for updates (by starting transaction)
+ */
+gboolean rspamd_fuzzy_backend_sqlite_prepare_update(struct rspamd_fuzzy_backend_sqlite *backend,
+ const gchar *source);
+
+/**
+ * Add digest to the database
+ * @param backend
+ * @param cmd
+ * @return
+ */
+gboolean rspamd_fuzzy_backend_sqlite_add(struct rspamd_fuzzy_backend_sqlite *backend,
+ const struct rspamd_fuzzy_cmd *cmd);
+
+/**
+ * Delete digest from the database
+ * @param backend
+ * @param cmd
+ * @return
+ */
+gboolean rspamd_fuzzy_backend_sqlite_del(
+ struct rspamd_fuzzy_backend_sqlite *backend,
+ const struct rspamd_fuzzy_cmd *cmd);
+
+/**
+ * Commit updates to storage
+ */
+gboolean rspamd_fuzzy_backend_sqlite_finish_update(struct rspamd_fuzzy_backend_sqlite *backend,
+ const gchar *source, gboolean version_bump);
+
+/**
+ * Sync storage
+ * @param backend
+ * @return
+ */
+gboolean rspamd_fuzzy_backend_sqlite_sync(struct rspamd_fuzzy_backend_sqlite *backend,
+ gint64 expire,
+ gboolean clean_orphaned);
+
+/**
+ * Close storage
+ * @param backend
+ */
+void rspamd_fuzzy_backend_sqlite_close(struct rspamd_fuzzy_backend_sqlite *backend);
+
+gsize rspamd_fuzzy_backend_sqlite_count(struct rspamd_fuzzy_backend_sqlite *backend);
+
+gint rspamd_fuzzy_backend_sqlite_version(struct rspamd_fuzzy_backend_sqlite *backend, const gchar *source);
+
+gsize rspamd_fuzzy_backend_sqlite_expired(struct rspamd_fuzzy_backend_sqlite *backend);
+
+const gchar *rspamd_fuzzy_sqlite_backend_id(struct rspamd_fuzzy_backend_sqlite *backend);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* FUZZY_BACKEND_H_ */
diff --git a/src/libserver/fuzzy_wire.h b/src/libserver/fuzzy_wire.h
new file mode 100644
index 0000000..c2f93b8
--- /dev/null
+++ b/src/libserver/fuzzy_wire.h
@@ -0,0 +1,154 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_FUZZY_STORAGE_H
+#define RSPAMD_FUZZY_STORAGE_H
+
+#include "config.h"
+#include "rspamd.h"
+#include "shingles.h"
+#include "cryptobox.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RSPAMD_FUZZY_VERSION 4
+#define RSPAMD_FUZZY_KEYLEN 8
+
+#define RSPAMD_FUZZY_FLAG_WEAK (1u << 7u)
+/* Use lower 4 bits for the version */
+#define RSPAMD_FUZZY_VERSION_MASK 0x0fu
+/* Commands for fuzzy storage */
+#define FUZZY_CHECK 0
+#define FUZZY_WRITE 1
+#define FUZZY_DEL 2
+#define FUZZY_STAT 3
+#define FUZZY_PING 4
+#define FUZZY_CLIENT_MAX 4
+/* Internal commands */
+#define FUZZY_REFRESH 100 /* Update expire */
+#define FUZZY_DUP 101 /* Skip duplicate in update queue */
+
+/**
+ * The epoch of the fuzzy client
+ */
+enum rspamd_fuzzy_epoch {
+ RSPAMD_FUZZY_EPOCH10, /**< 1.0+ encryption */
+ RSPAMD_FUZZY_EPOCH11, /**< 1.7+ extended reply */
+ RSPAMD_FUZZY_EPOCH_MAX
+};
+
+RSPAMD_PACKED(rspamd_fuzzy_cmd)
+{
+ guint8 version;
+ guint8 cmd;
+ guint8 shingles_count;
+ guint8 flag;
+ gint32 value;
+ guint32 tag;
+ gchar digest[rspamd_cryptobox_HASHBYTES];
+};
+
+RSPAMD_PACKED(rspamd_fuzzy_shingle_cmd)
+{
+ struct rspamd_fuzzy_cmd basic;
+ struct rspamd_shingle sgl;
+};
+
+RSPAMD_PACKED(rspamd_fuzzy_reply_v1)
+{
+ gint32 value;
+ guint32 flag;
+ guint32 tag;
+ float prob;
+};
+
+RSPAMD_PACKED(rspamd_fuzzy_reply)
+{
+ struct rspamd_fuzzy_reply_v1 v1;
+ gchar digest[rspamd_cryptobox_HASHBYTES];
+ guint32 ts;
+ guchar reserved[12];
+};
+
+RSPAMD_PACKED(rspamd_fuzzy_encrypted_req_hdr)
+{
+ guchar magic[4];
+ guchar key_id[RSPAMD_FUZZY_KEYLEN];
+ guchar pubkey[32];
+ guchar nonce[rspamd_cryptobox_MAX_NONCEBYTES];
+ guchar mac[rspamd_cryptobox_MAX_MACBYTES];
+};
+
+RSPAMD_PACKED(rspamd_fuzzy_encrypted_cmd)
+{
+ struct rspamd_fuzzy_encrypted_req_hdr hdr;
+ struct rspamd_fuzzy_cmd cmd;
+};
+
+RSPAMD_PACKED(rspamd_fuzzy_encrypted_shingle_cmd)
+{
+ struct rspamd_fuzzy_encrypted_req_hdr hdr;
+ struct rspamd_fuzzy_shingle_cmd cmd;
+};
+
+RSPAMD_PACKED(rspamd_fuzzy_encrypted_rep_hdr)
+{
+ guchar nonce[rspamd_cryptobox_MAX_NONCEBYTES];
+ guchar mac[rspamd_cryptobox_MAX_MACBYTES];
+};
+
+RSPAMD_PACKED(rspamd_fuzzy_encrypted_reply)
+{
+ struct rspamd_fuzzy_encrypted_rep_hdr hdr;
+ struct rspamd_fuzzy_reply rep;
+};
+
+static const guchar fuzzy_encrypted_magic[4] = {'r', 's', 'f', 'e'};
+
+enum rspamd_fuzzy_extension_type {
+ RSPAMD_FUZZY_EXT_SOURCE_DOMAIN = 'd',
+ RSPAMD_FUZZY_EXT_SOURCE_IP4 = '4',
+ RSPAMD_FUZZY_EXT_SOURCE_IP6 = '6',
+};
+
+struct rspamd_fuzzy_cmd_extension {
+ enum rspamd_fuzzy_extension_type ext;
+ guint length;
+ struct rspamd_fuzzy_cmd_extension *next;
+ guchar *payload;
+};
+
+struct rspamd_fuzzy_stat_entry {
+ const gchar *name;
+ guint64 fuzzy_cnt;
+};
+
+RSPAMD_PACKED(fuzzy_peer_cmd)
+{
+ gint32 is_shingle;
+ union {
+ struct rspamd_fuzzy_cmd normal;
+ struct rspamd_fuzzy_shingle_cmd shingle;
+ } cmd;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/html/html.cxx b/src/libserver/html/html.cxx
new file mode 100644
index 0000000..5861d45
--- /dev/null
+++ b/src/libserver/html/html.cxx
@@ -0,0 +1,2393 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "util.h"
+#include "message.h"
+#include "html.h"
+#include "html_tags.h"
+#include "html_block.hxx"
+#include "html.hxx"
+#include "libserver/css/css_value.hxx"
+#include "libserver/css/css.hxx"
+#include "libserver/task.h"
+#include "libserver/cfg_file.h"
+
+#include "url.h"
+#include "contrib/libucl/khash.h"
+#include "libmime/images.h"
+#include "libutil/cxx/utf8_util.h"
+
+#include "html_tag_defs.hxx"
+#include "html_entities.hxx"
+#include "html_tag.hxx"
+#include "html_url.hxx"
+
+#include <frozen/unordered_map.h>
+#include <frozen/string.h>
+#include <fmt/core.h>
+
+#include <unicode/uversion.h>
+
+namespace rspamd::html {
+
+static const guint max_tags = 8192; /* Ignore tags if this maximum is reached */
+
+static const html_tags_storage html_tags_defs;
+
+auto html_components_map = frozen::make_unordered_map<frozen::string, html_component_type>(
+ {
+ {"name", html_component_type::RSPAMD_HTML_COMPONENT_NAME},
+ {"href", html_component_type::RSPAMD_HTML_COMPONENT_HREF},
+ {"src", html_component_type::RSPAMD_HTML_COMPONENT_HREF},
+ {"action", html_component_type::RSPAMD_HTML_COMPONENT_HREF},
+ {"color", html_component_type::RSPAMD_HTML_COMPONENT_COLOR},
+ {"bgcolor", html_component_type::RSPAMD_HTML_COMPONENT_BGCOLOR},
+ {"style", html_component_type::RSPAMD_HTML_COMPONENT_STYLE},
+ {"class", html_component_type::RSPAMD_HTML_COMPONENT_CLASS},
+ {"width", html_component_type::RSPAMD_HTML_COMPONENT_WIDTH},
+ {"height", html_component_type::RSPAMD_HTML_COMPONENT_HEIGHT},
+ {"size", html_component_type::RSPAMD_HTML_COMPONENT_SIZE},
+ {"rel", html_component_type::RSPAMD_HTML_COMPONENT_REL},
+ {"alt", html_component_type::RSPAMD_HTML_COMPONENT_ALT},
+ {"id", html_component_type::RSPAMD_HTML_COMPONENT_ID},
+ {"hidden", html_component_type::RSPAMD_HTML_COMPONENT_HIDDEN},
+ });
+
+#define msg_debug_html(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_html_log_id, "html", pool->tag.uid, \
+ __FUNCTION__, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(html)
+
+/*
+ * This function is expected to be called on a closing tag to fill up all tags
+ * and return the current parent (meaning unclosed) tag
+ */
+static auto
+html_check_balance(struct html_content *hc,
+ struct html_tag *tag,
+ goffset tag_start_offset,
+ goffset tag_end_offset) -> html_tag *
+{
+ /* As agreed, the closing tag has the last opening at the parent ptr */
+ auto *opening_tag = tag->parent;
+
+ auto calculate_content_length = [tag_start_offset, tag_end_offset](html_tag *t) {
+ auto opening_content_offset = t->content_offset;
+
+ if (t->flags & (CM_EMPTY)) {
+ /* Attach closing tag just at the opening tag */
+ t->closing.start = t->tag_start;
+ t->closing.end = t->content_offset;
+ }
+ else {
+
+ if (opening_content_offset <= tag_start_offset) {
+ t->closing.start = tag_start_offset;
+ t->closing.end = tag_end_offset;
+ }
+ else {
+
+ t->closing.start = t->content_offset;
+ t->closing.end = tag_end_offset;
+ }
+ }
+ };
+
+ auto balance_tag = [&]() -> html_tag * {
+ auto it = tag->parent;
+ auto found_pair = false;
+
+ for (; it != nullptr; it = it->parent) {
+ if (it->id == tag->id && !(it->flags & FL_CLOSED)) {
+ found_pair = true;
+ break;
+ }
+ }
+
+ /*
+ * If we have found a closing pair, then we need to close all tags and
+ * return the top-most tag
+ */
+ if (found_pair) {
+ for (it = tag->parent; it != nullptr; it = it->parent) {
+ it->flags |= FL_CLOSED;
+ /* Insert a virtual closing tag for all tags that are not closed */
+ calculate_content_length(it);
+ if (it->id == tag->id && !(it->flags & FL_CLOSED)) {
+ break;
+ }
+ }
+
+ return it;
+ }
+ else {
+ /*
+ * We have not found a pair, so this closing tag is bogus and should
+ * be ignored completely.
+ * Unfortunately, it also means that we need to insert another tag,
+ * as the current closing tag is unusable for that purposes.
+ *
+ * We assume that callee will recognise that and reconstruct the
+ * tag at the tag_end_closing state, so we return nullptr...
+ */
+ }
+
+ /* Tag must be ignored and reconstructed */
+ return nullptr;
+ };
+
+ if (opening_tag) {
+
+ if (opening_tag->id == tag->id) {
+ opening_tag->flags |= FL_CLOSED;
+
+ calculate_content_length(opening_tag);
+ /* All good */
+ return opening_tag->parent;
+ }
+ else {
+ return balance_tag();
+ }
+ }
+ else {
+ /*
+ * We have no opening tag
+ * There are two possibilities:
+ *
+ * 1) We have some block tag in hc->all_tags;
+ * 2) We have no tags
+ */
+
+ if (hc->all_tags.empty()) {
+ hc->all_tags.push_back(std::make_unique<html_tag>());
+ auto *vtag = hc->all_tags.back().get();
+ vtag->id = Tag_HTML;
+ vtag->flags = FL_VIRTUAL;
+ vtag->tag_start = 0;
+ vtag->content_offset = 0;
+ calculate_content_length(vtag);
+
+ if (!hc->root_tag) {
+ hc->root_tag = vtag;
+ }
+ else {
+ vtag->parent = hc->root_tag;
+ }
+
+ tag->parent = vtag;
+
+ /* Recursively call with a virtual <html> tag inserted */
+ return html_check_balance(hc, tag, tag_start_offset, tag_end_offset);
+ }
+ }
+
+ return nullptr;
+}
+
+auto html_component_from_string(const std::string_view &st) -> std::optional<html_component_type>
+{
+ auto known_component_it = html_components_map.find(st);
+
+ if (known_component_it != html_components_map.end()) {
+ return known_component_it->second;
+ }
+ else {
+ return std::nullopt;
+ }
+}
+
+enum tag_parser_state {
+ parse_start = 0,
+ parse_name,
+ parse_attr_name,
+ parse_equal,
+ parse_start_dquote,
+ parse_dqvalue,
+ parse_end_dquote,
+ parse_start_squote,
+ parse_sqvalue,
+ parse_end_squote,
+ parse_value,
+ spaces_before_eq,
+ spaces_after_eq,
+ spaces_after_param,
+ ignore_bad_tag,
+ tag_end,
+ slash_after_value,
+ slash_in_unquoted_value,
+};
+struct tag_content_parser_state {
+ tag_parser_state cur_state = parse_start;
+ std::string buf;
+ std::optional<html_component_type> cur_component;
+
+ void reset()
+ {
+ cur_state = parse_start;
+ buf.clear();
+ cur_component = std::nullopt;
+ }
+};
+
+static inline void
+html_parse_tag_content(rspamd_mempool_t *pool,
+ struct html_content *hc,
+ struct html_tag *tag,
+ const char *in,
+ struct tag_content_parser_state &parser_env)
+{
+ auto state = parser_env.cur_state;
+
+ /*
+ * Stores tag component if it doesn't exist, performing copy of the
+ * value + decoding of the entities
+ * Parser env is set to clear the current html attribute fields (saved_p and
+ * cur_component)
+ */
+ auto store_component_value = [&]() -> void {
+ if (parser_env.cur_component) {
+
+ if (parser_env.buf.empty()) {
+ tag->components.emplace_back(parser_env.cur_component.value(),
+ std::string_view{});
+ }
+ else {
+ /* We need to copy buf to a persistent storage */
+ auto *s = rspamd_mempool_alloc_buffer(pool, parser_env.buf.size());
+
+ if (parser_env.cur_component.value() == html_component_type::RSPAMD_HTML_COMPONENT_ID ||
+ parser_env.cur_component.value() == html_component_type::RSPAMD_HTML_COMPONENT_CLASS) {
+ /* Lowercase */
+ rspamd_str_copy_lc(parser_env.buf.data(), s, parser_env.buf.size());
+ }
+ else {
+ memcpy(s, parser_env.buf.data(), parser_env.buf.size());
+ }
+
+ auto sz = rspamd_html_decode_entitles_inplace(s, parser_env.buf.size());
+ tag->components.emplace_back(parser_env.cur_component.value(),
+ std::string_view{s, sz});
+ }
+ }
+
+ parser_env.buf.clear();
+ parser_env.cur_component = std::nullopt;
+ };
+
+ auto store_component_name = [&]() -> bool {
+ decode_html_entitles_inplace(parser_env.buf);
+ auto known_component_it = html_components_map.find(std::string_view{parser_env.buf});
+ parser_env.buf.clear();
+
+ if (known_component_it != html_components_map.end()) {
+ parser_env.cur_component = known_component_it->second;
+
+ return true;
+ }
+ else {
+ parser_env.cur_component = std::nullopt;
+ }
+
+ return false;
+ };
+
+ auto store_value_character = [&](bool lc) -> void {
+ auto c = lc ? g_ascii_tolower(*in) : *in;
+
+ if (c == '\0') {
+ /* Replace with u0FFD */
+ parser_env.buf.append((const char *) u8"\uFFFD");
+ }
+ else {
+ parser_env.buf.push_back(c);
+ }
+ };
+
+ switch (state) {
+ case parse_start:
+ if (!g_ascii_isalpha(*in) && !g_ascii_isspace(*in)) {
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ state = ignore_bad_tag;
+ tag->id = N_TAGS;
+ tag->flags |= FL_BROKEN;
+ }
+ else if (g_ascii_isalpha(*in)) {
+ state = parse_name;
+ store_value_character(true);
+ }
+ break;
+
+ case parse_name:
+ if ((g_ascii_isspace(*in) || *in == '>' || *in == '/')) {
+ if (*in == '/') {
+ tag->flags |= FL_CLOSED;
+ }
+
+ if (parser_env.buf.empty()) {
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ tag->id = N_TAGS;
+ tag->flags |= FL_BROKEN;
+ state = ignore_bad_tag;
+ }
+ else {
+ decode_html_entitles_inplace(parser_env.buf);
+ const auto *tag_def = rspamd::html::html_tags_defs.by_name(parser_env.buf);
+
+ if (tag_def == nullptr) {
+ hc->flags |= RSPAMD_HTML_FLAG_UNKNOWN_ELEMENTS;
+ /* Assign -hash to match closing tag if needed */
+ auto nhash = static_cast<std::int32_t>(std::hash<std::string>{}(parser_env.buf));
+ /* Always negative */
+ tag->id = static_cast<tag_id_t>(nhash | G_MININT32);
+ }
+ else {
+ tag->id = tag_def->id;
+ tag->flags = tag_def->flags;
+ }
+
+ parser_env.buf.clear();
+
+ state = spaces_after_param;
+ }
+ }
+ else {
+ store_value_character(true);
+ }
+ break;
+
+ case parse_attr_name:
+ if (*in == '=') {
+ if (!parser_env.buf.empty()) {
+ store_component_name();
+ }
+ state = parse_equal;
+ }
+ else if (g_ascii_isspace(*in)) {
+ store_component_name();
+ state = spaces_before_eq;
+ }
+ else if (*in == '/') {
+ store_component_name();
+ store_component_value();
+ state = slash_after_value;
+ }
+ else if (*in == '>') {
+ store_component_name();
+ store_component_value();
+ state = tag_end;
+ }
+ else {
+ if (*in == '"' || *in == '\'' || *in == '<') {
+ /* Should never be in attribute names but ignored */
+ tag->flags |= FL_BROKEN;
+ }
+
+ store_value_character(true);
+ }
+
+ break;
+
+ case spaces_before_eq:
+ if (*in == '=') {
+ state = parse_equal;
+ }
+ else if (!g_ascii_isspace(*in)) {
+ /*
+ * HTML defines that crap could still be restored and
+ * calculated somehow... So we have to follow this stupid behaviour
+ */
+ /*
+ * TODO: estimate what insane things do email clients in each case
+ */
+ if (*in == '>') {
+ /*
+ * Attribute name followed by end of tag
+ * Should be okay (empty attribute). The rest is handled outside
+ * this automata.
+ */
+ store_component_value();
+ state = tag_end;
+ }
+ else if (*in == '"' || *in == '\'' || *in == '<') {
+ /* Attribute followed by quote... Missing '=' ? Dunno, need to test */
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ tag->flags |= FL_BROKEN;
+ store_component_value();
+ store_value_character(true);
+ state = spaces_after_param;
+ }
+ else {
+ /* Empty attribute */
+ store_component_value();
+ store_value_character(true);
+ state = spaces_after_param;
+ }
+ }
+ break;
+
+ case spaces_after_eq:
+ if (*in == '"') {
+ state = parse_start_dquote;
+ }
+ else if (*in == '\'') {
+ state = parse_start_squote;
+ }
+ else if (!g_ascii_isspace(*in)) {
+ store_value_character(true);
+ state = parse_value;
+ }
+ break;
+
+ case parse_equal:
+ if (g_ascii_isspace(*in)) {
+ state = spaces_after_eq;
+ }
+ else if (*in == '"') {
+ state = parse_start_dquote;
+ }
+ else if (*in == '\'') {
+ state = parse_start_squote;
+ }
+ else {
+ store_value_character(true);
+ state = parse_value;
+ }
+ break;
+
+ case parse_start_dquote:
+ if (*in == '"') {
+ state = spaces_after_param;
+ }
+ else {
+ store_value_character(false);
+ state = parse_dqvalue;
+ }
+ break;
+
+ case parse_start_squote:
+ if (*in == '\'') {
+ state = spaces_after_param;
+ }
+ else {
+ store_value_character(false);
+ state = parse_sqvalue;
+ }
+ break;
+
+ case parse_dqvalue:
+ if (*in == '"') {
+ store_component_value();
+ state = parse_end_dquote;
+ }
+ else {
+ store_value_character(false);
+ }
+ break;
+
+ case parse_sqvalue:
+ if (*in == '\'') {
+ store_component_value();
+ state = parse_end_squote;
+ }
+ else {
+ store_value_character(false);
+ }
+
+ break;
+
+ case parse_value:
+ if (*in == '/') {
+ state = slash_in_unquoted_value;
+ }
+ else if (g_ascii_isspace(*in) || *in == '>' || *in == '"') {
+ store_component_value();
+ state = spaces_after_param;
+ }
+ else {
+ store_value_character(false);
+ }
+ break;
+
+ case parse_end_dquote:
+ case parse_end_squote:
+ if (g_ascii_isspace(*in)) {
+ state = spaces_after_param;
+ }
+ else if (*in == '/') {
+ store_component_value();
+ store_value_character(true);
+ state = slash_after_value;
+ }
+ else {
+ /* No space, proceed immediately to the attribute name */
+ state = parse_attr_name;
+ store_component_value();
+ store_value_character(true);
+ }
+ break;
+
+ case spaces_after_param:
+ if (!g_ascii_isspace(*in)) {
+ if (*in == '/') {
+ state = slash_after_value;
+ }
+ else if (*in == '=') {
+ /* Attributes cannot start with '=' */
+ tag->flags |= FL_BROKEN;
+ store_value_character(true);
+ state = parse_attr_name;
+ }
+ else {
+ store_value_character(true);
+ state = parse_attr_name;
+ }
+ }
+ break;
+ case slash_after_value:
+ if (*in == '>') {
+ tag->flags |= FL_CLOSED;
+ state = tag_end;
+ }
+ else if (!g_ascii_isspace(*in)) {
+ tag->flags |= FL_BROKEN;
+ state = parse_attr_name;
+ }
+ break;
+ case slash_in_unquoted_value:
+ if (*in == '>') {
+ /* That slash was in fact closing tag slash, woohoo */
+ tag->flags |= FL_CLOSED;
+ state = tag_end;
+ store_component_value();
+ }
+ else {
+ /* Welcome to the world of html, revert state and save missing / */
+ parser_env.buf.push_back('/');
+ store_value_character(false);
+ state = parse_value;
+ }
+ break;
+ case ignore_bad_tag:
+ case tag_end:
+ break;
+ }
+
+ parser_env.cur_state = state;
+}
+
+static inline auto
+html_is_absolute_url(std::string_view st) -> bool
+{
+ auto alnum_pos = std::find_if(std::begin(st), std::end(st),
+ [](auto c) { return !g_ascii_isalnum(c); });
+
+ if (alnum_pos != std::end(st) && alnum_pos != std::begin(st)) {
+ if (*alnum_pos == ':') {
+ if (st.substr(0, std::distance(std::begin(st), alnum_pos)) == "mailto") {
+ return true;
+ }
+
+ std::advance(alnum_pos, 1);
+ if (alnum_pos != std::end(st)) {
+ /* Include even malformed urls */
+ if (*alnum_pos == '/' || *alnum_pos == '\\') {
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+}
+
+static auto
+html_process_url_tag(rspamd_mempool_t *pool,
+ struct html_tag *tag,
+ struct html_content *hc) -> std::optional<struct rspamd_url *>
+{
+ auto found_href_maybe = tag->find_component(html_component_type::RSPAMD_HTML_COMPONENT_HREF);
+
+ if (found_href_maybe) {
+ /* Check base url */
+ auto &href_value = found_href_maybe.value();
+
+ if (hc && hc->base_url) {
+ /*
+ * Relative url cannot start from the following:
+ * schema://
+ * data:
+ * slash
+ */
+
+ if (!html_is_absolute_url(href_value)) {
+
+ if (href_value.size() >= sizeof("data:") &&
+ g_ascii_strncasecmp(href_value.data(), "data:", sizeof("data:") - 1) == 0) {
+ /* Image data url, never insert as url */
+ return std::nullopt;
+ }
+
+ /* Assume relative url */
+ auto need_slash = false;
+
+ auto orig_len = href_value.size();
+ auto len = orig_len + hc->base_url->urllen;
+
+ if (hc->base_url->datalen == 0) {
+ need_slash = true;
+ len++;
+ }
+
+ auto *buf = rspamd_mempool_alloc_buffer(pool, len + 1);
+ auto nlen = (std::size_t) rspamd_snprintf(buf, len + 1,
+ "%*s%s%*s",
+ (int) hc->base_url->urllen, hc->base_url->string,
+ need_slash ? "/" : "",
+ (gint) orig_len, href_value.data());
+ href_value = {buf, nlen};
+ }
+ else if (href_value.size() > 2 && href_value[0] == '/' && href_value[1] != '/') {
+ /* Relative to the hostname */
+ auto orig_len = href_value.size();
+ auto len = orig_len + hc->base_url->hostlen + hc->base_url->protocollen +
+ 3 /* for :// */;
+ auto *buf = rspamd_mempool_alloc_buffer(pool, len + 1);
+ auto nlen = (std::size_t) rspamd_snprintf(buf, len + 1, "%*s://%*s/%*s",
+ (int) hc->base_url->protocollen, hc->base_url->string,
+ (int) hc->base_url->hostlen, rspamd_url_host_unsafe(hc->base_url),
+ (gint) orig_len, href_value.data());
+ href_value = {buf, nlen};
+ }
+ }
+
+ auto url = html_process_url(pool, href_value).value_or(nullptr);
+
+ if (url) {
+ if (tag->id != Tag_A) {
+ /* Mark special tags special */
+ url->flags |= RSPAMD_URL_FLAG_SPECIAL;
+ }
+
+ if (std::holds_alternative<std::monostate>(tag->extra)) {
+ tag->extra = url;
+ }
+
+ return url;
+ }
+
+ return std::nullopt;
+ }
+
+ return std::nullopt;
+}
+
+struct rspamd_html_url_query_cbd {
+ rspamd_mempool_t *pool;
+ khash_t(rspamd_url_hash) * url_set;
+ struct rspamd_url *url;
+ GPtrArray *part_urls;
+};
+
+static gboolean
+html_url_query_callback(struct rspamd_url *url, gsize start_offset,
+ gsize end_offset, gpointer ud)
+{
+ struct rspamd_html_url_query_cbd *cbd =
+ (struct rspamd_html_url_query_cbd *) ud;
+ rspamd_mempool_t *pool;
+
+ pool = cbd->pool;
+
+ if (url->protocol == PROTOCOL_MAILTO) {
+ if (url->userlen == 0) {
+ return FALSE;
+ }
+ }
+
+ msg_debug_html("found url %s in query of url"
+ " %*s",
+ url->string,
+ cbd->url->querylen, rspamd_url_query_unsafe(cbd->url));
+
+ url->flags |= RSPAMD_URL_FLAG_QUERY;
+
+ if (rspamd_url_set_add_or_increase(cbd->url_set, url, false) && cbd->part_urls) {
+ g_ptr_array_add(cbd->part_urls, url);
+ }
+
+ return TRUE;
+}
+
+static void
+html_process_query_url(rspamd_mempool_t *pool, struct rspamd_url *url,
+ khash_t(rspamd_url_hash) * url_set,
+ GPtrArray *part_urls)
+{
+ if (url->querylen > 0) {
+ struct rspamd_html_url_query_cbd qcbd;
+
+ qcbd.pool = pool;
+ qcbd.url_set = url_set;
+ qcbd.url = url;
+ qcbd.part_urls = part_urls;
+
+ rspamd_url_find_multiple(pool,
+ rspamd_url_query_unsafe(url), url->querylen,
+ RSPAMD_URL_FIND_ALL, NULL,
+ html_url_query_callback, &qcbd);
+ }
+
+ if (part_urls) {
+ g_ptr_array_add(part_urls, url);
+ }
+}
+
+static auto
+html_process_data_image(rspamd_mempool_t *pool,
+ struct html_image *img,
+ std::string_view input) -> void
+{
+ /*
+ * Here, we do very basic processing of the data:
+ * detect if we have something like: `data:image/xxx;base64,yyyzzz==`
+ * We only parse base64 encoded data.
+ * We ignore content type so far
+ */
+ struct rspamd_image *parsed_image;
+ const gchar *semicolon_pos = input.data(),
+ *end = input.data() + input.size();
+
+ if ((semicolon_pos = (const gchar *) memchr(semicolon_pos, ';', end - semicolon_pos)) != NULL) {
+ if (end - semicolon_pos > sizeof("base64,")) {
+ if (memcmp(semicolon_pos + 1, "base64,", sizeof("base64,") - 1) == 0) {
+ const gchar *data_pos = semicolon_pos + sizeof("base64,");
+ gchar *decoded;
+ gsize encoded_len = end - data_pos, decoded_len;
+ rspamd_ftok_t inp;
+
+ decoded_len = (encoded_len / 4 * 3) + 12;
+ decoded = rspamd_mempool_alloc_buffer(pool, decoded_len);
+ rspamd_cryptobox_base64_decode(data_pos, encoded_len,
+ reinterpret_cast<guchar *>(decoded), &decoded_len);
+ inp.begin = decoded;
+ inp.len = decoded_len;
+
+ parsed_image = rspamd_maybe_process_image(pool, &inp);
+
+ if (parsed_image) {
+ msg_debug_html("detected %s image of size %ud x %ud in data url",
+ rspamd_image_type_str(parsed_image->type),
+ parsed_image->width, parsed_image->height);
+ img->embedded_image = parsed_image;
+ }
+ }
+ }
+ else {
+ /* Nothing useful */
+ return;
+ }
+ }
+}
+
+static void
+html_process_img_tag(rspamd_mempool_t *pool,
+ struct html_tag *tag,
+ struct html_content *hc,
+ khash_t(rspamd_url_hash) * url_set,
+ GPtrArray *part_urls)
+{
+ struct html_image *img;
+
+ img = rspamd_mempool_alloc0_type(pool, struct html_image);
+ img->tag = tag;
+
+ for (const auto &param: tag->components) {
+
+ if (param.type == html_component_type::RSPAMD_HTML_COMPONENT_HREF) {
+ /* Check base url */
+ const auto &href_value = param.value;
+
+ if (href_value.size() > 0) {
+ rspamd_ftok_t fstr;
+ fstr.begin = href_value.data();
+ fstr.len = href_value.size();
+ img->src = rspamd_mempool_ftokdup(pool, &fstr);
+
+ if (href_value.size() > sizeof("cid:") - 1 && memcmp(href_value.data(),
+ "cid:", sizeof("cid:") - 1) == 0) {
+ /* We have an embedded image */
+ img->src += sizeof("cid:") - 1;
+ img->flags |= RSPAMD_HTML_FLAG_IMAGE_EMBEDDED;
+ }
+ else {
+ if (href_value.size() > sizeof("data:") - 1 && memcmp(href_value.data(),
+ "data:", sizeof("data:") - 1) == 0) {
+ /* We have an embedded image in HTML tag */
+ img->flags |=
+ (RSPAMD_HTML_FLAG_IMAGE_EMBEDDED | RSPAMD_HTML_FLAG_IMAGE_DATA);
+ html_process_data_image(pool, img, href_value);
+ hc->flags |= RSPAMD_HTML_FLAG_HAS_DATA_URLS;
+ }
+ else {
+ img->flags |= RSPAMD_HTML_FLAG_IMAGE_EXTERNAL;
+ if (img->src) {
+
+ std::string_view cpy{href_value};
+ auto maybe_url = html_process_url(pool, cpy);
+
+ if (maybe_url) {
+ img->url = maybe_url.value();
+ struct rspamd_url *existing;
+
+ img->url->flags |= RSPAMD_URL_FLAG_IMAGE;
+ existing = rspamd_url_set_add_or_return(url_set,
+ img->url);
+
+ if (existing && existing != img->url) {
+ /*
+ * We have some other URL that could be
+ * found, e.g. from another part. However,
+ * we still want to set an image flag on it
+ */
+ existing->flags |= img->url->flags;
+ existing->count++;
+ }
+ else if (part_urls) {
+ /* New url */
+ g_ptr_array_add(part_urls, img->url);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+ if (param.type == html_component_type::RSPAMD_HTML_COMPONENT_HEIGHT) {
+ unsigned long val;
+
+ rspamd_strtoul(param.value.data(), param.value.size(), &val);
+ img->height = val;
+ }
+
+ if (param.type == html_component_type::RSPAMD_HTML_COMPONENT_WIDTH) {
+ unsigned long val;
+
+ rspamd_strtoul(param.value.data(), param.value.size(), &val);
+ img->width = val;
+ }
+
+ /* TODO: rework to css at some time */
+ if (param.type == html_component_type::RSPAMD_HTML_COMPONENT_STYLE) {
+ if (img->height == 0) {
+ auto style_st = param.value;
+ auto pos = rspamd_substring_search_caseless(style_st.data(),
+ style_st.size(),
+ "height", sizeof("height") - 1);
+ if (pos != -1) {
+ auto substr = style_st.substr(pos + sizeof("height") - 1);
+
+ for (auto i = 0; i < substr.size(); i++) {
+ auto t = substr[i];
+ if (g_ascii_isdigit(t)) {
+ unsigned long val;
+ rspamd_strtoul(substr.data(),
+ substr.size(), &val);
+ img->height = val;
+ break;
+ }
+ else if (!g_ascii_isspace(t) && t != '=' && t != ':') {
+ /* Fallback */
+ break;
+ }
+ }
+ }
+ }
+ if (img->width == 0) {
+ auto style_st = param.value;
+ auto pos = rspamd_substring_search_caseless(style_st.data(),
+ style_st.size(),
+ "width", sizeof("width") - 1);
+ if (pos != -1) {
+ auto substr = style_st.substr(pos + sizeof("width") - 1);
+
+ for (auto i = 0; i < substr.size(); i++) {
+ auto t = substr[i];
+ if (g_ascii_isdigit(t)) {
+ unsigned long val;
+ rspamd_strtoul(substr.data(),
+ substr.size(), &val);
+ img->width = val;
+ break;
+ }
+ else if (!g_ascii_isspace(t) && t != '=' && t != ':') {
+ /* Fallback */
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (img->embedded_image) {
+ if (img->height == 0) {
+ img->height = img->embedded_image->height;
+ }
+ if (img->width == 0) {
+ img->width = img->embedded_image->width;
+ }
+ }
+
+ hc->images.push_back(img);
+
+ if (std::holds_alternative<std::monostate>(tag->extra)) {
+ tag->extra = img;
+ }
+}
+
+static auto
+html_process_link_tag(rspamd_mempool_t *pool, struct html_tag *tag,
+ struct html_content *hc,
+ khash_t(rspamd_url_hash) * url_set,
+ GPtrArray *part_urls) -> void
+{
+ auto found_rel_maybe = tag->find_component(html_component_type::RSPAMD_HTML_COMPONENT_REL);
+
+ if (found_rel_maybe) {
+ if (found_rel_maybe.value() == "icon") {
+ html_process_img_tag(pool, tag, hc, url_set, part_urls);
+ }
+ }
+}
+
+static auto
+html_process_block_tag(rspamd_mempool_t *pool, struct html_tag *tag,
+ struct html_content *hc) -> void
+{
+ std::optional<css::css_value> maybe_fgcolor, maybe_bgcolor;
+ bool hidden = false;
+
+ for (const auto &param: tag->components) {
+ if (param.type == html_component_type::RSPAMD_HTML_COMPONENT_COLOR) {
+ maybe_fgcolor = css::css_value::maybe_color_from_string(param.value);
+ }
+
+ if (param.type == html_component_type::RSPAMD_HTML_COMPONENT_BGCOLOR) {
+ maybe_bgcolor = css::css_value::maybe_color_from_string(param.value);
+ }
+
+ if (param.type == html_component_type::RSPAMD_HTML_COMPONENT_STYLE) {
+ tag->block = rspamd::css::parse_css_declaration(pool, param.value);
+ }
+
+ if (param.type == html_component_type::RSPAMD_HTML_COMPONENT_HIDDEN) {
+ hidden = true;
+ }
+ }
+
+ if (!tag->block) {
+ tag->block = html_block::undefined_html_block_pool(pool);
+ }
+
+ if (hidden) {
+ tag->block->set_display(false);
+ }
+
+ if (maybe_fgcolor) {
+ tag->block->set_fgcolor(maybe_fgcolor->to_color().value());
+ }
+
+ if (maybe_bgcolor) {
+ tag->block->set_bgcolor(maybe_bgcolor->to_color().value());
+ }
+}
+
+static inline auto
+html_append_parsed(struct html_content *hc,
+ std::string_view data,
+ bool transparent,
+ std::size_t input_len,
+ std::string &dest) -> std::size_t
+{
+ auto cur_offset = dest.size();
+
+ if (dest.size() > input_len) {
+ /* Impossible case, refuse to append */
+ return 0;
+ }
+
+ if (data.size() > 0) {
+ /* Handle multiple spaces at the begin */
+
+ if (cur_offset > 0) {
+ auto last = dest.back();
+ if (!g_ascii_isspace(last) && g_ascii_isspace(data.front())) {
+ dest.append(" ");
+ data = {data.data() + 1, data.size() - 1};
+ cur_offset++;
+ }
+ }
+
+ if (data.find('\0') != std::string_view::npos) {
+ auto replace_zero_func = [](const auto &input, auto &output) {
+ const auto last = input.cend();
+ for (auto it = input.cbegin(); it != last; ++it) {
+ if (*it == '\0') {
+ output.append((const char *) u8"\uFFFD");
+ }
+ else {
+ output.push_back(*it);
+ }
+ }
+ };
+
+ dest.reserve(dest.size() + data.size() + sizeof(u8"\uFFFD"));
+ replace_zero_func(data, dest);
+ hc->flags |= RSPAMD_HTML_FLAG_HAS_ZEROS;
+ }
+ else {
+ dest.append(data);
+ }
+ }
+
+ auto nlen = decode_html_entitles_inplace(dest.data() + cur_offset,
+ dest.size() - cur_offset, true);
+
+ dest.resize(nlen + cur_offset);
+
+ if (transparent) {
+ /* Replace all visible characters with spaces */
+ auto start = std::next(dest.begin(), cur_offset);
+ std::replace_if(
+ start, std::end(dest), [](const auto c) {
+ return !g_ascii_isspace(c);
+ },
+ ' ');
+ }
+
+ return nlen;
+}
+
+static auto
+html_process_displayed_href_tag(rspamd_mempool_t *pool,
+ struct html_content *hc,
+ std::string_view data,
+ const struct html_tag *cur_tag,
+ GList **exceptions,
+ khash_t(rspamd_url_hash) * url_set,
+ goffset dest_offset) -> void
+{
+
+ if (std::holds_alternative<rspamd_url *>(cur_tag->extra)) {
+ auto *url = std::get<rspamd_url *>(cur_tag->extra);
+
+ html_check_displayed_url(pool,
+ exceptions, url_set,
+ data,
+ dest_offset,
+ url);
+ }
+}
+
+static auto
+html_append_tag_content(rspamd_mempool_t *pool,
+ const gchar *start, gsize len,
+ struct html_content *hc,
+ html_tag *tag,
+ GList **exceptions,
+ khash_t(rspamd_url_hash) * url_set) -> goffset
+{
+ auto is_visible = true, is_block = false, is_spaces = false, is_transparent = false;
+ goffset next_tag_offset = tag->closing.end,
+ initial_parsed_offset = hc->parsed.size(),
+ initial_invisible_offset = hc->invisible.size();
+
+ auto calculate_final_tag_offsets = [&]() -> void {
+ if (is_visible) {
+ tag->content_offset = initial_parsed_offset;
+ tag->closing.start = hc->parsed.size();
+ }
+ else {
+ tag->content_offset = initial_invisible_offset;
+ tag->closing.start = hc->invisible.size();
+ }
+ };
+
+ if (tag->closing.end == -1) {
+ if (tag->closing.start != -1) {
+ next_tag_offset = tag->closing.start;
+ tag->closing.end = tag->closing.start;
+ }
+ else {
+ next_tag_offset = tag->content_offset;
+ tag->closing.end = tag->content_offset;
+ }
+ }
+ if (tag->closing.start == -1) {
+ tag->closing.start = tag->closing.end;
+ }
+
+ auto append_margin = [&](char c) -> void {
+ /* We do care about visible margins only */
+ if (is_visible) {
+ if (!hc->parsed.empty() && hc->parsed.back() != c && hc->parsed.back() != '\n') {
+ if (hc->parsed.back() == ' ') {
+ /* We also strip extra spaces at the end, but limiting the start */
+ auto last = std::make_reverse_iterator(hc->parsed.begin() + initial_parsed_offset);
+ auto first = std::find_if(hc->parsed.rbegin(), last,
+ [](auto ch) -> auto {
+ return ch != ' ';
+ });
+ hc->parsed.erase(first.base(), hc->parsed.end());
+ g_assert(hc->parsed.size() >= initial_parsed_offset);
+ }
+ hc->parsed.push_back(c);
+ }
+ }
+ };
+
+ if (tag->id == Tag_BR || tag->id == Tag_HR) {
+
+ if (!(tag->flags & FL_IGNORE)) {
+ hc->parsed.append("\n");
+ }
+
+ auto ret = tag->content_offset;
+ calculate_final_tag_offsets();
+
+ return ret;
+ }
+ else if ((tag->id == Tag_HEAD && (tag->flags & FL_IGNORE)) || (tag->flags & CM_HEAD)) {
+ auto ret = tag->closing.end;
+ calculate_final_tag_offsets();
+
+ return ret;
+ }
+
+ if ((tag->flags & (FL_COMMENT | FL_XML | FL_IGNORE | CM_HEAD))) {
+ is_visible = false;
+ }
+ else {
+ if (!tag->block) {
+ is_visible = true;
+ }
+ else if (!tag->block->is_visible()) {
+ if (!tag->block->is_transparent()) {
+ is_visible = false;
+ }
+ else {
+ if (tag->block->has_display() &&
+ tag->block->display == css::css_display_value::DISPLAY_HIDDEN) {
+ is_visible = false;
+ }
+ else {
+ is_transparent = true;
+ }
+ }
+ }
+ else {
+ if (tag->block->display == css::css_display_value::DISPLAY_BLOCK) {
+ is_block = true;
+ }
+ else if (tag->block->display == css::css_display_value::DISPLAY_TABLE_ROW) {
+ is_spaces = true;
+ }
+ }
+ }
+
+ if (is_block) {
+ append_margin('\n');
+ }
+ else if (is_spaces) {
+ append_margin(' ');
+ }
+
+ goffset cur_offset = tag->content_offset;
+
+ for (auto *cld: tag->children) {
+ auto enclosed_start = cld->tag_start;
+ goffset initial_part_len = enclosed_start - cur_offset;
+
+ if (initial_part_len > 0) {
+ if (is_visible) {
+ html_append_parsed(hc,
+ {start + cur_offset, std::size_t(initial_part_len)},
+ is_transparent, len, hc->parsed);
+ }
+ else {
+ html_append_parsed(hc,
+ {start + cur_offset, std::size_t(initial_part_len)},
+ is_transparent, len, hc->invisible);
+ }
+ }
+
+ auto next_offset = html_append_tag_content(pool, start, len,
+ hc, cld, exceptions, url_set);
+
+ /* Do not allow shifting back */
+ if (next_offset > cur_offset) {
+ cur_offset = next_offset;
+ }
+ }
+
+ if (cur_offset < tag->closing.start) {
+ goffset final_part_len = tag->closing.start - cur_offset;
+
+ if (final_part_len > 0) {
+ if (is_visible) {
+ html_append_parsed(hc,
+ {start + cur_offset, std::size_t(final_part_len)},
+ is_transparent,
+ len,
+ hc->parsed);
+ }
+ else {
+ html_append_parsed(hc,
+ {start + cur_offset, std::size_t(final_part_len)},
+ is_transparent,
+ len,
+ hc->invisible);
+ }
+ }
+ }
+ if (is_block) {
+ append_margin('\n');
+ }
+ else if (is_spaces) {
+ append_margin(' ');
+ }
+
+ if (is_visible) {
+ if (tag->id == Tag_A) {
+ auto written_len = hc->parsed.size() - initial_parsed_offset;
+ html_process_displayed_href_tag(pool, hc,
+ {hc->parsed.data() + initial_parsed_offset, std::size_t(written_len)},
+ tag, exceptions,
+ url_set, initial_parsed_offset);
+ }
+ else if (tag->id == Tag_IMG) {
+ /* Process ALT if presented */
+ auto maybe_alt = tag->find_component(html_component_type::RSPAMD_HTML_COMPONENT_ALT);
+
+ if (maybe_alt) {
+ if (!hc->parsed.empty() && !g_ascii_isspace(hc->parsed.back())) {
+ /* Add a space */
+ hc->parsed += ' ';
+ }
+
+ hc->parsed.append(maybe_alt.value());
+
+ if (!hc->parsed.empty() && !g_ascii_isspace(hc->parsed.back())) {
+ /* Add a space */
+ hc->parsed += ' ';
+ }
+ }
+ }
+ }
+ else {
+ /* Invisible stuff */
+ if (std::holds_alternative<rspamd_url *>(tag->extra)) {
+ auto *url_enclosed = std::get<rspamd_url *>(tag->extra);
+
+ /*
+ * TODO: when hash is fixed to include flags we need to remove and add
+ * url to the hash set
+ */
+ if (url_enclosed) {
+ url_enclosed->flags |= RSPAMD_URL_FLAG_INVISIBLE;
+ }
+ }
+ }
+
+ calculate_final_tag_offsets();
+
+ return next_tag_offset;
+}
+
+auto html_process_input(struct rspamd_task *task,
+ GByteArray *in,
+ GList **exceptions,
+ khash_t(rspamd_url_hash) * url_set,
+ GPtrArray *part_urls,
+ bool allow_css,
+ std::uint16_t *cur_url_order) -> html_content *
+{
+ const gchar *p, *c, *end, *start;
+ guchar t;
+ auto closing = false;
+ guint obrace = 0, ebrace = 0;
+ struct rspamd_url *url = nullptr;
+ gint href_offset = -1;
+ auto overflow_input = false;
+ struct html_tag *cur_tag = nullptr, *parent_tag = nullptr, cur_closing_tag;
+ struct tag_content_parser_state content_parser_env;
+ auto process_size = in->len;
+
+
+ enum {
+ parse_start = 0,
+ content_before_start,
+ tag_begin,
+ sgml_tag,
+ xml_tag,
+ compound_tag,
+ comment_tag,
+ comment_content,
+ sgml_content,
+ tag_content,
+ tag_end_opening,
+ tag_end_closing,
+ html_text_content,
+ xml_tag_end,
+ tag_raw_text,
+ tag_raw_text_less_than,
+ tags_limit_overflow,
+ } state = parse_start;
+
+ enum class html_document_state {
+ doctype,
+ head,
+ body
+ } html_document_state = html_document_state::doctype;
+
+ g_assert(in != NULL);
+ g_assert(task != NULL);
+
+ auto *pool = task->task_pool;
+ auto cur_url_part_order = 0u;
+
+ auto *hc = new html_content;
+ rspamd_mempool_add_destructor(task->task_pool, html_content::html_content_dtor, hc);
+
+ if (task->cfg && in->len > task->cfg->max_html_len) {
+ msg_notice_task("html input is too big: %z, limit is %z",
+ in->len,
+ task->cfg->max_html_len);
+ process_size = task->cfg->max_html_len;
+ overflow_input = true;
+ }
+
+ auto new_tag = [&](int flags = 0) -> struct html_tag *
+ {
+
+ if (hc->all_tags.size() > rspamd::html::max_tags) {
+ hc->flags |= RSPAMD_HTML_FLAG_TOO_MANY_TAGS;
+
+ return nullptr;
+ }
+
+ hc->all_tags.emplace_back(std::make_unique<html_tag>());
+ auto *ntag = hc->all_tags.back().get();
+ ntag->tag_start = c - start;
+ ntag->flags = flags;
+
+ if (cur_tag && !(cur_tag->flags & (CM_EMPTY | FL_CLOSED)) && cur_tag != &cur_closing_tag) {
+ parent_tag = cur_tag;
+ }
+
+ if (flags & FL_XML) {
+ return ntag;
+ }
+
+ return ntag;
+ };
+
+ auto process_opening_tag = [&]() {
+ if (cur_tag->id > Tag_UNKNOWN) {
+ if (cur_tag->flags & CM_UNIQUE) {
+ if (!hc->tags_seen[cur_tag->id]) {
+ /* Duplicate tag has been found */
+ hc->flags |= RSPAMD_HTML_FLAG_DUPLICATE_ELEMENTS;
+ }
+ }
+ hc->tags_seen[cur_tag->id] = true;
+ }
+
+ /* Shift to the first unclosed tag */
+ auto *pt = parent_tag;
+ while (pt && (pt->flags & FL_CLOSED)) {
+ pt = pt->parent;
+ }
+
+ if (pt) {
+ g_assert(cur_tag != pt);
+ cur_tag->parent = pt;
+ g_assert(cur_tag->parent != &cur_closing_tag);
+ parent_tag = pt;
+ parent_tag->children.push_back(cur_tag);
+ }
+ else {
+ if (hc->root_tag) {
+ if (cur_tag != hc->root_tag) {
+ cur_tag->parent = hc->root_tag;
+ g_assert(cur_tag->parent != cur_tag);
+ hc->root_tag->children.push_back(cur_tag);
+ parent_tag = hc->root_tag;
+ }
+ }
+ else {
+ if (cur_tag->id == Tag_HTML) {
+ hc->root_tag = cur_tag;
+ }
+ else {
+ /* Insert a fake html tag */
+ hc->all_tags.emplace_back(std::make_unique<html_tag>());
+ auto *top_tag = hc->all_tags.back().get();
+ top_tag->tag_start = 0;
+ top_tag->flags = FL_VIRTUAL;
+ top_tag->id = Tag_HTML;
+ top_tag->content_offset = 0;
+ top_tag->children.push_back(cur_tag);
+ cur_tag->parent = top_tag;
+ g_assert(cur_tag->parent != cur_tag);
+ hc->root_tag = top_tag;
+ parent_tag = top_tag;
+ }
+ }
+ }
+
+ if (cur_tag->flags & FL_HREF && html_document_state == html_document_state::body) {
+ auto maybe_url = html_process_url_tag(pool, cur_tag, hc);
+
+ if (maybe_url.has_value()) {
+ url = maybe_url.value();
+
+ if (url_set != NULL) {
+ struct rspamd_url *maybe_existing =
+ rspamd_url_set_add_or_return(url_set, maybe_url.value());
+ if (maybe_existing == maybe_url.value()) {
+ if (cur_url_order) {
+ url->order = (*cur_url_order)++;
+ }
+ url->part_order = cur_url_part_order++;
+ html_process_query_url(pool, url, url_set,
+ part_urls);
+ }
+ else {
+ url = maybe_existing;
+ /* Replace extra as well */
+ cur_tag->extra = maybe_existing;
+ /* Increase count to avoid odd checks failure */
+ url->count++;
+ }
+ }
+ if (part_urls) {
+ g_ptr_array_add(part_urls, url);
+ }
+
+ href_offset = hc->parsed.size();
+ }
+ }
+ else if (cur_tag->id == Tag_BASE) {
+ /*
+ * Base is allowed only within head tag but HTML is retarded
+ */
+ auto maybe_url = html_process_url_tag(pool, cur_tag, hc);
+
+ if (maybe_url) {
+ msg_debug_html("got valid base tag");
+ cur_tag->extra = maybe_url.value();
+ cur_tag->flags |= FL_HREF;
+
+ if (hc->base_url == nullptr) {
+ hc->base_url = maybe_url.value();
+ }
+ else {
+ msg_debug_html("ignore redundant base tag");
+ }
+ }
+ else {
+ msg_debug_html("got invalid base tag!");
+ }
+ }
+
+ if (cur_tag->id == Tag_IMG) {
+ html_process_img_tag(pool, cur_tag, hc, url_set,
+ part_urls);
+ }
+ else if (cur_tag->id == Tag_LINK) {
+ html_process_link_tag(pool, cur_tag, hc, url_set,
+ part_urls);
+ }
+
+ if (!(cur_tag->flags & CM_EMPTY)) {
+ html_process_block_tag(pool, cur_tag, hc);
+ }
+ else {
+ /* Implicitly close */
+ cur_tag->flags |= FL_CLOSED;
+ }
+
+ if (cur_tag->flags & FL_CLOSED) {
+ cur_tag->closing.end = cur_tag->content_offset;
+ cur_tag->closing.start = cur_tag->tag_start;
+
+ cur_tag = parent_tag;
+ }
+ };
+
+ p = (const char *) in->data;
+ c = p;
+ end = p + process_size;
+ start = c;
+
+ while (p < end) {
+ t = *p;
+
+ switch (state) {
+ case parse_start:
+ if (t == '<') {
+ state = tag_begin;
+ }
+ else {
+ /* We have no starting tag, so assume that it's content */
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_START;
+ cur_tag = new_tag();
+ html_document_state = html_document_state::body;
+
+ if (cur_tag) {
+ cur_tag->id = Tag_HTML;
+ hc->root_tag = cur_tag;
+ state = content_before_start;
+ }
+ else {
+ state = tags_limit_overflow;
+ }
+ }
+ break;
+ case content_before_start:
+ if (t == '<') {
+ state = tag_begin;
+ }
+ else {
+ p++;
+ }
+ break;
+ case tag_begin:
+ switch (t) {
+ case '<':
+ c = p;
+ p++;
+ closing = FALSE;
+ break;
+ case '!':
+ cur_tag = new_tag(FL_XML | FL_CLOSED);
+ if (cur_tag) {
+ state = sgml_tag;
+ }
+ else {
+ state = tags_limit_overflow;
+ }
+ p++;
+ break;
+ case '?':
+ cur_tag = new_tag(FL_XML | FL_CLOSED);
+ if (cur_tag) {
+ state = xml_tag;
+ }
+ else {
+ state = tags_limit_overflow;
+ }
+ hc->flags |= RSPAMD_HTML_FLAG_XML;
+ p++;
+ break;
+ case '/':
+ closing = TRUE;
+ /* We fill fake closing tag to fill it with the content parser */
+ cur_closing_tag.clear();
+ /*
+ * For closing tags, we need to find some corresponding opening tag.
+ * However, at this point we have not even parsed a name, so we
+ * can not assume anything about balancing, etc.
+ *
+ * So we need to ensure that:
+ * 1) We have some opening tag in the chain cur_tag->parent...
+ * 2) cur_tag is nullptr - okay, html is just brain damaged
+ * 3) cur_tag must NOT be equal to cur_closing tag. It means that
+ * we had some poor closing tag but we still need to find an opening
+ * tag... Somewhere...
+ */
+
+ if (cur_tag == &cur_closing_tag) {
+ if (parent_tag != &cur_closing_tag) {
+ cur_closing_tag.parent = parent_tag;
+ }
+ else {
+ cur_closing_tag.parent = nullptr;
+ }
+ }
+ else if (cur_tag && cur_tag->flags & FL_CLOSED) {
+ /* Cur tag is already closed, we should find something else */
+ auto *tmp = cur_tag;
+ while (tmp) {
+ tmp = tmp->parent;
+
+ if (tmp == nullptr || !(tmp->flags & FL_CLOSED)) {
+ break;
+ }
+ }
+
+ cur_closing_tag.parent = tmp;
+ }
+ else {
+ cur_closing_tag.parent = cur_tag;
+ }
+
+ cur_tag = &cur_closing_tag;
+ p++;
+ break;
+ case '>':
+ /* Empty tag */
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ state = html_text_content;
+ continue;
+ default:
+ if (g_ascii_isalpha(t)) {
+ state = tag_content;
+ content_parser_env.reset();
+
+ if (!closing) {
+ cur_tag = new_tag();
+ }
+
+ if (cur_tag) {
+ state = tag_content;
+ }
+ else {
+ state = tags_limit_overflow;
+ }
+ }
+ else {
+ /* Wrong bad tag */
+ state = html_text_content;
+ }
+ break;
+ }
+
+ break;
+
+ case sgml_tag:
+ switch (t) {
+ case '[':
+ state = compound_tag;
+ obrace = 1;
+ ebrace = 0;
+ p++;
+ break;
+ case '-':
+ cur_tag->flags |= FL_COMMENT;
+ state = comment_tag;
+ p++;
+ break;
+ default:
+ state = sgml_content;
+ break;
+ }
+
+ break;
+
+ case xml_tag:
+ if (t == '?') {
+ state = xml_tag_end;
+ }
+ else if (t == '>') {
+ /* Misformed xml tag */
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ state = tag_end_opening;
+ continue;
+ }
+ /* We efficiently ignore xml tags */
+ p++;
+ break;
+
+ case xml_tag_end:
+ if (t == '>') {
+ state = tag_end_opening;
+ cur_tag->content_offset = p - start + 1;
+ continue;
+ }
+ else {
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ }
+ p++;
+ break;
+
+ case compound_tag:
+ if (t == '[') {
+ obrace++;
+ }
+ else if (t == ']') {
+ ebrace++;
+ }
+ else if (t == '>' && obrace == ebrace) {
+ state = tag_end_opening;
+ cur_tag->content_offset = p - start + 1;
+ continue;
+ }
+ p++;
+ break;
+
+ case comment_tag:
+ if (t != '-') {
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ state = tag_end_opening;
+ }
+ else {
+ p++;
+ ebrace = 0;
+ /*
+ * https://www.w3.org/TR/2012/WD-html5-20120329/syntax.html#syntax-comments
+ * ... the text must not start with a single
+ * U+003E GREATER-THAN SIGN character (>),
+ * nor start with a "-" (U+002D) character followed by
+ * a U+003E GREATER-THAN SIGN (>) character,
+ * nor contain two consecutive U+002D HYPHEN-MINUS
+ * characters (--), nor end with a "-" (U+002D) character.
+ */
+ if (p[0] == '-' && p + 1 < end && p[1] == '>') {
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ p++;
+ state = tag_end_opening;
+ }
+ else if (*p == '>') {
+ hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
+ state = tag_end_opening;
+ }
+ else {
+ state = comment_content;
+ }
+ }
+ break;
+
+ case comment_content:
+ if (t == '-') {
+ ebrace++;
+ }
+ else if (t == '>' && ebrace >= 2) {
+ cur_tag->content_offset = p - start + 1;
+ state = tag_end_opening;
+ continue;
+ }
+ else {
+ ebrace = 0;
+ }
+
+ p++;
+ break;
+
+ case html_text_content:
+ if (t != '<') {
+ p++;
+ }
+ else {
+ state = tag_begin;
+ }
+ break;
+
+ case tag_raw_text:
+ if (t == '<') {
+ c = p;
+ state = tag_raw_text_less_than;
+ }
+ p++;
+ break;
+ case tag_raw_text_less_than:
+ if (t == '/') {
+ /* Here are special things: we look for obrace and then ensure
+ * that if there is any closing brace nearby
+ * (we look maximum at 30 characters). We also need to ensure
+ * that we have no special characters, such as punctuation marks and
+ * so on.
+ * Basically, we validate the input to be sane.
+ * Since closing tags must not have attributes, these assumptions
+ * seems to be reasonable enough for our toy parser.
+ */
+ gint cur_lookahead = 1;
+ gint max_lookahead = MIN(end - p, 30);
+ bool valid_closing_tag = true;
+
+ if (p + 1 < end && !g_ascii_isalpha(p[1])) {
+ valid_closing_tag = false;
+ }
+ else {
+ while (cur_lookahead < max_lookahead) {
+ gchar tt = p[cur_lookahead];
+ if (tt == '>') {
+ break;
+ }
+ else if (tt < '\n' || tt == ',') {
+ valid_closing_tag = false;
+ break;
+ }
+ cur_lookahead++;
+ }
+
+ if (cur_lookahead == max_lookahead) {
+ valid_closing_tag = false;
+ }
+ }
+
+ if (valid_closing_tag) {
+ /* Shift back */
+ p = c;
+ state = tag_begin;
+ }
+ else {
+ p++;
+ state = tag_raw_text;
+ }
+ }
+ else {
+ p++;
+ state = tag_raw_text;
+ }
+ break;
+ case sgml_content:
+ /* TODO: parse DOCTYPE here */
+ if (t == '>') {
+ cur_tag->content_offset = p - start + 1;
+ state = tag_end_opening;
+ }
+ else {
+ p++;
+ }
+ break;
+
+ case tag_content:
+ html_parse_tag_content(pool, hc, cur_tag, p, content_parser_env);
+
+ if (t == '>') {
+ if (content_parser_env.cur_state != parse_dqvalue && content_parser_env.cur_state != parse_sqvalue) {
+ /* We have a closing element */
+ if (closing) {
+ cur_tag->closing.start = c - start;
+ cur_tag->closing.end = p - start + 1;
+
+ closing = FALSE;
+ state = tag_end_closing;
+ }
+ else {
+ cur_tag->content_offset = p - start + 1;
+ state = tag_end_opening;
+ }
+ }
+ else {
+ /*
+ * We are in the parse_quoted value state but got
+ * an unescaped `>` character.
+ * HTML is written for monkeys, so there are two possibilities:
+ * 1) We have missing ending quote
+ * 2) We have unescaped `>` character
+ * How to distinguish between those possibilities?
+ * Well, the idea is to do some lookahead and try to find a
+ * quote. If we can find a quote, we just pretend as we have
+ * not seen `>` character. Otherwise, we pretend that it is an
+ * unquoted stuff. This logic is quite fragile but I really
+ * don't know any better options...
+ */
+ auto end_quote = content_parser_env.cur_state == parse_sqvalue ? '\'' : '"';
+ if (memchr(p, end_quote, end - p) != nullptr) {
+ /* Unencoded `>` */
+ p++;
+ continue;
+ }
+ else {
+ if (closing) {
+ cur_tag->closing.start = c - start;
+ cur_tag->closing.end = p - start + 1;
+
+ closing = FALSE;
+ state = tag_end_closing;
+ }
+ else {
+ cur_tag->content_offset = p - start + 1;
+ state = tag_end_opening;
+ }
+ }
+ }
+ continue;
+ }
+ p++;
+ break;
+
+ case tag_end_opening:
+ content_parser_env.reset();
+ state = html_text_content;
+
+ if (cur_tag) {
+ if (cur_tag->id == Tag_STYLE || cur_tag->id == Tag_NOSCRIPT || cur_tag->id == Tag_SCRIPT) {
+ state = tag_raw_text;
+ }
+ if (html_document_state == html_document_state::doctype) {
+ if (cur_tag->id == Tag_HEAD || (cur_tag->flags & CM_HEAD)) {
+ html_document_state = html_document_state::head;
+ cur_tag->flags |= FL_IGNORE;
+ }
+ else if (cur_tag->id != Tag_HTML) {
+ html_document_state = html_document_state::body;
+ }
+ }
+ else if (html_document_state == html_document_state::head) {
+ if (!(cur_tag->flags & (CM_EMPTY | CM_HEAD))) {
+ if (parent_tag && (parent_tag->id == Tag_HEAD || !(parent_tag->flags & CM_HEAD))) {
+ /*
+ * As by standard, we have to close the HEAD tag
+ * and switch to the body state
+ */
+ parent_tag->flags |= FL_CLOSED;
+ parent_tag->closing.start = cur_tag->tag_start;
+ parent_tag->closing.end = cur_tag->content_offset;
+
+ html_document_state = html_document_state::body;
+ }
+ else if (cur_tag->id == Tag_BODY) {
+ html_document_state = html_document_state::body;
+ }
+ else {
+ /*
+ * For propagation in something like
+ * <title><p><a>ololo</a></p></title> - should be unprocessed
+ */
+ cur_tag->flags |= CM_HEAD;
+ }
+ }
+ }
+
+ process_opening_tag();
+ }
+
+ p++;
+ c = p;
+ break;
+ case tag_end_closing: {
+ if (cur_tag) {
+
+ if (cur_tag->flags & CM_EMPTY) {
+ /* Ignore closing empty tags */
+ cur_tag->flags |= FL_IGNORE;
+ }
+ if (html_document_state == html_document_state::doctype) {
+ }
+ else if (html_document_state == html_document_state::head) {
+ if (cur_tag->id == Tag_HEAD) {
+ html_document_state = html_document_state::body;
+ }
+ }
+
+ /* cur_tag here is a closing tag */
+ auto *next_cur_tag = html_check_balance(hc, cur_tag,
+ c - start, p - start + 1);
+
+ if (cur_tag->id == Tag_STYLE && allow_css) {
+ auto *opening_tag = cur_tag->parent;
+
+ if (opening_tag && opening_tag->id == Tag_STYLE &&
+ (int) opening_tag->content_offset < opening_tag->closing.start) {
+ auto ret_maybe = rspamd::css::parse_css(pool,
+ {start + opening_tag->content_offset,
+ opening_tag->closing.start - opening_tag->content_offset},
+ std::move(hc->css_style));
+
+ if (!ret_maybe.has_value()) {
+ if (ret_maybe.error().is_fatal()) {
+ auto err_str = fmt::format(
+ "cannot parse css (error code: {}): {}",
+ static_cast<int>(ret_maybe.error().type),
+ ret_maybe.error().description.value_or("unknown error"));
+ msg_info_pool("%*s", (int) err_str.size(), err_str.data());
+ }
+ }
+ else {
+ hc->css_style = ret_maybe.value();
+ }
+ }
+ }
+
+ if (next_cur_tag != nullptr) {
+ cur_tag = next_cur_tag;
+ }
+ else {
+ /*
+ * Here, we handle cases like <p>lala</b>...
+ * So the tag </b> is bogus and unpaired
+ * However, we need to exclude it from the output of <p> tag
+ * To do that, we create a fake opening tag and insert that to
+ * the current opening tag
+ */
+ auto *cur_opening_tag = cur_tag->parent;
+
+ while (cur_opening_tag && (cur_opening_tag->flags & FL_CLOSED)) {
+ cur_opening_tag = cur_opening_tag->parent;
+ }
+
+ if (!cur_opening_tag) {
+ cur_opening_tag = hc->root_tag;
+ }
+
+ auto &&vtag = std::make_unique<html_tag>();
+ vtag->id = cur_tag->id;
+ vtag->flags = FL_VIRTUAL | FL_CLOSED | cur_tag->flags;
+ vtag->tag_start = cur_tag->closing.start;
+ vtag->content_offset = p - start + 1;
+ vtag->closing = cur_tag->closing;
+ vtag->parent = cur_opening_tag;
+ g_assert(vtag->parent != &cur_closing_tag);
+ cur_opening_tag->children.push_back(vtag.get());
+ hc->all_tags.emplace_back(std::move(vtag));
+ cur_tag = cur_opening_tag;
+ parent_tag = cur_tag->parent;
+ g_assert(cur_tag->parent != &cur_closing_tag);
+ }
+ } /* if cur_tag != nullptr */
+ state = html_text_content;
+ p++;
+ c = p;
+ break;
+ }
+ case tags_limit_overflow:
+ msg_warn_pool("tags limit of %d tags is reached at the position %d;"
+ " ignoring the rest of the HTML content",
+ (int) hc->all_tags.size(), (int) (p - start));
+ c = p;
+ p = end;
+ break;
+ }
+ }
+
+ if (cur_tag && !(cur_tag->flags & FL_CLOSED) && cur_tag != &cur_closing_tag) {
+ cur_closing_tag.parent = cur_tag;
+ cur_closing_tag.id = cur_tag->id;
+ cur_tag = &cur_closing_tag;
+ html_check_balance(hc, cur_tag,
+ end - start, end - start);
+ }
+
+ /* Propagate styles */
+ hc->traverse_block_tags([&hc, &pool](const html_tag *tag) -> bool {
+ if (hc->css_style && tag->id > Tag_UNKNOWN && tag->id < Tag_MAX) {
+ auto *css_block = hc->css_style->check_tag_block(tag);
+
+ if (css_block) {
+ if (tag->block) {
+ tag->block->set_block(*css_block);
+ }
+ else {
+ tag->block = css_block;
+ }
+ }
+ }
+ if (tag->block) {
+ if (!tag->block->has_display()) {
+ /* If we have no display field, we can check it by tag */
+ if (tag->flags & CM_HEAD) {
+ tag->block->set_display(css::css_display_value::DISPLAY_HIDDEN,
+ html_block::set);
+ }
+ else if (tag->flags & (CM_BLOCK | CM_TABLE)) {
+ tag->block->set_display(css::css_display_value::DISPLAY_BLOCK,
+ html_block::implicit);
+ }
+ else if (tag->flags & CM_ROW) {
+ tag->block->set_display(css::css_display_value::DISPLAY_TABLE_ROW,
+ html_block::implicit);
+ }
+ else {
+ tag->block->set_display(css::css_display_value::DISPLAY_INLINE,
+ html_block::implicit);
+ }
+ }
+
+ tag->block->compute_visibility();
+
+ for (const auto *cld_tag: tag->children) {
+
+ if (cld_tag->block) {
+ cld_tag->block->propagate_block(*tag->block);
+ }
+ else {
+ cld_tag->block = rspamd_mempool_alloc0_type(pool, html_block);
+ *cld_tag->block = *tag->block;
+ }
+ }
+ }
+ return true;
+ },
+ html_content::traverse_type::PRE_ORDER);
+
+ /* Leftover before content */
+ switch (state) {
+ case tag_end_opening:
+ if (cur_tag != nullptr) {
+ process_opening_tag();
+ }
+ break;
+ default:
+ /* Do nothing */
+ break;
+ }
+
+ if (!hc->all_tags.empty() && hc->root_tag) {
+ html_append_tag_content(pool, start, end - start, hc, hc->root_tag,
+ exceptions, url_set);
+ }
+
+ /* Leftover after content */
+ switch (state) {
+ case tags_limit_overflow:
+ html_append_parsed(hc, {c, (std::size_t)(end - c)},
+ false, end - start, hc->parsed);
+ break;
+ default:
+ /* Do nothing */
+ break;
+ }
+
+ if (overflow_input) {
+ /*
+ * Append the rest of the input as raw html, this might work as
+ * further algorithms can skip words when auto *pool = task->task_pool;there are too many.
+ * It is still unclear about urls though...
+ */
+ html_append_parsed(hc, {end, in->len - process_size}, false,
+ end - start, hc->parsed);
+ }
+
+ if (!hc->parsed.empty()) {
+ /* Trim extra spaces at the end if needed */
+ if (g_ascii_isspace(hc->parsed.back())) {
+ auto last_it = std::end(hc->parsed);
+
+ /* Allow last newline */
+ if (hc->parsed.back() == '\n') {
+ --last_it;
+ }
+
+ hc->parsed.erase(std::find_if(hc->parsed.rbegin(), hc->parsed.rend(),
+ [](auto ch) -> auto {
+ return !g_ascii_isspace(ch);
+ })
+ .base(),
+ last_it);
+ }
+ }
+
+ return hc;
+}
+
+static auto
+html_find_image_by_cid(const html_content &hc, std::string_view cid)
+ -> std::optional<const html_image *>
+{
+ for (const auto *html_image: hc.images) {
+ /* Filter embedded images */
+ if (html_image->flags & RSPAMD_HTML_FLAG_IMAGE_EMBEDDED &&
+ html_image->src != nullptr) {
+ if (cid == html_image->src) {
+ return html_image;
+ }
+ }
+ }
+
+ return std::nullopt;
+}
+
+auto html_debug_structure(const html_content &hc) -> std::string
+{
+ std::string output;
+
+ if (hc.root_tag) {
+ auto rec_functor = [&](const html_tag *t, int level, auto rec_functor) -> void {
+ std::string pluses(level, '+');
+
+ if (!(t->flags & (FL_VIRTUAL | FL_IGNORE))) {
+ if (t->flags & FL_XML) {
+ output += fmt::format("{}xml;", pluses);
+ }
+ else {
+ output += fmt::format("{}{};", pluses,
+ html_tags_defs.name_by_id_safe(t->id));
+ }
+ level++;
+ }
+ for (const auto *cld: t->children) {
+ rec_functor(cld, level, rec_functor);
+ }
+ };
+
+ rec_functor(hc.root_tag, 1, rec_functor);
+ }
+
+ return output;
+}
+
+auto html_tag_by_name(const std::string_view &name)
+ -> std::optional<tag_id_t>
+{
+ const auto *td = rspamd::html::html_tags_defs.by_name(name);
+
+ if (td != nullptr) {
+ return td->id;
+ }
+
+ return std::nullopt;
+}
+
+auto html_tag::get_content(const struct html_content *hc) const -> std::string_view
+{
+ const std::string *dest = &hc->parsed;
+
+ if (block && !block->is_visible()) {
+ dest = &hc->invisible;
+ }
+ const auto clen = get_content_length();
+ if (content_offset < dest->size()) {
+ if (dest->size() - content_offset >= clen) {
+ return std::string_view{*dest}.substr(content_offset, clen);
+ }
+ else {
+ return std::string_view{*dest}.substr(content_offset, dest->size() - content_offset);
+ }
+ }
+
+ return std::string_view{};
+}
+
+}// namespace rspamd::html
+
+void *
+rspamd_html_process_part_full(struct rspamd_task *task,
+ GByteArray *in, GList **exceptions,
+ khash_t(rspamd_url_hash) * url_set,
+ GPtrArray *part_urls,
+ bool allow_css,
+ uint16_t *cur_url_order)
+{
+ return rspamd::html::html_process_input(task, in, exceptions, url_set,
+ part_urls, allow_css, cur_url_order);
+}
+
+void *
+rspamd_html_process_part(rspamd_mempool_t *pool,
+ GByteArray *in)
+{
+ struct rspamd_task fake_task;
+ memset(&fake_task, 0, sizeof(fake_task));
+ fake_task.task_pool = pool;
+ uint16_t order = 0;
+
+ return rspamd_html_process_part_full(&fake_task, in, NULL,
+ NULL, NULL, FALSE, &order);
+}
+
+guint rspamd_html_decode_entitles_inplace(gchar *s, gsize len)
+{
+ return rspamd::html::decode_html_entitles_inplace(s, len);
+}
+
+gint rspamd_html_tag_by_name(const gchar *name)
+{
+ const auto *td = rspamd::html::html_tags_defs.by_name(name);
+
+ if (td != nullptr) {
+ return td->id;
+ }
+
+ return -1;
+}
+
+gboolean
+rspamd_html_tag_seen(void *ptr, const gchar *tagname)
+{
+ gint id;
+ auto *hc = rspamd::html::html_content::from_ptr(ptr);
+
+ g_assert(hc != NULL);
+
+ id = rspamd_html_tag_by_name(tagname);
+
+ if (id != -1) {
+ return hc->tags_seen[id];
+ }
+
+ return FALSE;
+}
+
+const gchar *
+rspamd_html_tag_by_id(gint id)
+{
+ if (id > Tag_UNKNOWN && id < Tag_MAX) {
+ const auto *td = rspamd::html::html_tags_defs.by_id(id);
+
+ if (td != nullptr) {
+ return td->name.c_str();
+ }
+ }
+
+ return nullptr;
+}
+
+const gchar *
+rspamd_html_tag_name(void *p, gsize *len)
+{
+ auto *tag = reinterpret_cast<rspamd::html::html_tag *>(p);
+ auto tname = rspamd::html::html_tags_defs.name_by_id_safe(tag->id);
+
+ if (len) {
+ *len = tname.size();
+ }
+
+ return tname.data();
+}
+
+struct html_image *
+rspamd_html_find_embedded_image(void *html_content,
+ const char *cid, gsize cid_len)
+{
+ auto *hc = rspamd::html::html_content::from_ptr(html_content);
+
+ auto maybe_img = rspamd::html::html_find_image_by_cid(*hc, {cid, cid_len});
+
+ if (maybe_img) {
+ return (html_image *) maybe_img.value();
+ }
+
+ return nullptr;
+}
+
+bool rspamd_html_get_parsed_content(void *html_content, rspamd_ftok_t *dest)
+{
+ auto *hc = rspamd::html::html_content::from_ptr(html_content);
+
+ dest->begin = hc->parsed.data();
+ dest->len = hc->parsed.size();
+
+ return true;
+}
+
+gsize rspamd_html_get_tags_count(void *html_content)
+{
+ auto *hc = rspamd::html::html_content::from_ptr(html_content);
+
+ if (!hc) {
+ return 0;
+ }
+
+ return hc->all_tags.size();
+} \ No newline at end of file
diff --git a/src/libserver/html/html.h b/src/libserver/html/html.h
new file mode 100644
index 0000000..2d34f2a
--- /dev/null
+++ b/src/libserver/html/html.h
@@ -0,0 +1,137 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_HTML_H
+#define RSPAMD_HTML_H
+
+#include "config.h"
+#include "libutil/mem_pool.h"
+#include "libserver/url.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * HTML content flags
+ */
+#define RSPAMD_HTML_FLAG_BAD_START (1 << 0)
+#define RSPAMD_HTML_FLAG_BAD_ELEMENTS (1 << 1)
+#define RSPAMD_HTML_FLAG_XML (1 << 2)
+#define RSPAMD_HTML_FLAG_UNBALANCED (1 << 3)
+#define RSPAMD_HTML_FLAG_UNKNOWN_ELEMENTS (1 << 4)
+#define RSPAMD_HTML_FLAG_DUPLICATE_ELEMENTS (1 << 5)
+#define RSPAMD_HTML_FLAG_TOO_MANY_TAGS (1 << 6)
+#define RSPAMD_HTML_FLAG_HAS_DATA_URLS (1 << 7)
+#define RSPAMD_HTML_FLAG_HAS_ZEROS (1 << 8)
+
+/*
+ * Image flags
+ */
+#define RSPAMD_HTML_FLAG_IMAGE_EMBEDDED (1 << 0)
+#define RSPAMD_HTML_FLAG_IMAGE_EXTERNAL (1 << 1)
+#define RSPAMD_HTML_FLAG_IMAGE_DATA (1 << 2)
+
+
+struct rspamd_image;
+
+struct html_image {
+ guint height;
+ guint width;
+ guint flags;
+ gchar *src;
+ struct rspamd_url *url;
+ struct rspamd_image *embedded_image;
+ void *tag;
+};
+
+
+/* Forwarded declaration */
+struct rspamd_task;
+
+/*
+ * Decode HTML entitles in text. Text is modified in place.
+ */
+guint rspamd_html_decode_entitles_inplace(gchar *s, gsize len);
+
+void *rspamd_html_process_part(rspamd_mempool_t *pool,
+ GByteArray *in);
+
+void *rspamd_html_process_part_full(struct rspamd_task *task,
+ GByteArray *in, GList **exceptions,
+ khash_t(rspamd_url_hash) * url_set,
+ GPtrArray *part_urls,
+ bool allow_css,
+ uint16_t *cur_url_order);
+
+/*
+ * Returns true if a specified tag has been seen in a part
+ */
+gboolean rspamd_html_tag_seen(void *ptr, const gchar *tagname);
+
+/**
+ * Returns name for the specified tag id
+ * @param id
+ * @return
+ */
+const gchar *rspamd_html_tag_by_id(gint id);
+
+/**
+ * Returns HTML tag id by name
+ * @param name
+ * @return
+ */
+gint rspamd_html_tag_by_name(const gchar *name);
+
+/**
+ * Gets a name for a tag
+ * @param tag
+ * @param len
+ * @return
+ */
+const gchar *rspamd_html_tag_name(void *tag, gsize *len);
+
+/**
+ * Find HTML image by content id
+ * @param html_content
+ * @param cid
+ * @param cid_len
+ * @return
+ */
+struct html_image *rspamd_html_find_embedded_image(void *html_content,
+ const char *cid, gsize cid_len);
+
+/**
+ * Stores parsed content in ftok_t structure
+ * @param html_content
+ * @param dest
+ * @return
+ */
+bool rspamd_html_get_parsed_content(void *html_content, rspamd_ftok_t *dest);
+
+/**
+ * Returns number of tags in the html content
+ * @param html_content
+ * @return
+ */
+gsize rspamd_html_get_tags_count(void *html_content);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/html/html.hxx b/src/libserver/html/html.hxx
new file mode 100644
index 0000000..3320fd6
--- /dev/null
+++ b/src/libserver/html/html.hxx
@@ -0,0 +1,146 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_HTML_HXX
+#define RSPAMD_HTML_HXX
+#pragma once
+
+#include "config.h"
+#include "libserver/url.h"
+#include "libserver/html/html_tag.hxx"
+#include "libserver/html/html.h"
+#include "libserver/html/html_tags.h"
+
+
+#include <vector>
+#include <memory>
+#include <string>
+#include "function2/function2.hpp"
+
+namespace rspamd::css {
+/* Forward declaration */
+class css_style_sheet;
+}// namespace rspamd::css
+
+namespace rspamd::html {
+
+struct html_block;
+
+struct html_content {
+ struct rspamd_url *base_url = nullptr;
+ struct html_tag *root_tag = nullptr;
+ gint flags = 0;
+ std::vector<bool> tags_seen;
+ std::vector<html_image *> images;
+ std::vector<std::unique_ptr<struct html_tag>> all_tags;
+ std::string parsed;
+ std::string invisible;
+ std::shared_ptr<css::css_style_sheet> css_style;
+
+ /* Preallocate and reserve all internal structures */
+ html_content()
+ {
+ tags_seen.resize(Tag_MAX, false);
+ all_tags.reserve(128);
+ parsed.reserve(256);
+ }
+
+ static void html_content_dtor(void *ptr)
+ {
+ delete html_content::from_ptr(ptr);
+ }
+
+ static auto from_ptr(void *ptr) -> html_content *
+ {
+ return static_cast<html_content *>(ptr);
+ }
+
+ enum class traverse_type {
+ PRE_ORDER,
+ POST_ORDER
+ };
+ auto traverse_block_tags(fu2::function<bool(const html_tag *)> &&func,
+ traverse_type how = traverse_type::PRE_ORDER) const -> bool
+ {
+
+ if (root_tag == nullptr) {
+ return false;
+ }
+
+ auto rec_functor_pre_order = [&](const html_tag *root, auto &&rec) -> bool {
+ if (func(root)) {
+
+ for (const auto *c: root->children) {
+ if (!rec(c, rec)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+ return false;
+ };
+ auto rec_functor_post_order = [&](const html_tag *root, auto &&rec) -> bool {
+ for (const auto *c: root->children) {
+ if (!rec(c, rec)) {
+ return false;
+ }
+ }
+
+ return func(root);
+ };
+
+ switch (how) {
+ case traverse_type::PRE_ORDER:
+ return rec_functor_pre_order(root_tag, rec_functor_pre_order);
+ case traverse_type::POST_ORDER:
+ return rec_functor_post_order(root_tag, rec_functor_post_order);
+ default:
+ RSPAMD_UNREACHABLE;
+ }
+ }
+
+ auto traverse_all_tags(fu2::function<bool(const html_tag *)> &&func) const -> bool
+ {
+ for (const auto &tag: all_tags) {
+ if (!(tag->flags & (FL_XML | FL_VIRTUAL))) {
+ if (!func(tag.get())) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+
+private:
+ ~html_content() = default;
+};
+
+
+auto html_tag_by_name(const std::string_view &name) -> std::optional<tag_id_t>;
+auto html_process_input(struct rspamd_task *task,
+ GByteArray *in,
+ GList **exceptions,
+ khash_t(rspamd_url_hash) * url_set,
+ GPtrArray *part_urls,
+ bool allow_css,
+ std::uint16_t *cur_url_order) -> html_content *;
+auto html_debug_structure(const html_content &hc) -> std::string;
+
+}// namespace rspamd::html
+
+#endif//RSPAMD_HTML_HXX
diff --git a/src/libserver/html/html_block.hxx b/src/libserver/html/html_block.hxx
new file mode 100644
index 0000000..f9b5184
--- /dev/null
+++ b/src/libserver/html/html_block.hxx
@@ -0,0 +1,358 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_HTML_BLOCK_HXX
+#define RSPAMD_HTML_BLOCK_HXX
+#pragma once
+
+#include "libserver/css/css_value.hxx"
+#include <cmath>
+
+namespace rspamd::html {
+
+/*
+ * Block tag definition
+ */
+struct html_block {
+ rspamd::css::css_color fg_color;
+ rspamd::css::css_color bg_color;
+ std::int16_t height;
+ std::int16_t width;
+ rspamd::css::css_display_value display;
+ std::int8_t font_size;
+
+ unsigned fg_color_mask : 2;
+ unsigned bg_color_mask : 2;
+ unsigned height_mask : 2;
+ unsigned width_mask : 2;
+ unsigned font_mask : 2;
+ unsigned display_mask : 2;
+ unsigned visibility_mask : 2;
+
+ constexpr static const auto unset = 0;
+ constexpr static const auto inherited = 1;
+ constexpr static const auto implicit = 1;
+ constexpr static const auto set = 3;
+ constexpr static const auto invisible_flag = 1;
+ constexpr static const auto transparent_flag = 2;
+
+ /* Helpers to set mask when setting the elements */
+ auto set_fgcolor(const rspamd::css::css_color &c, int how = html_block::set) -> void
+ {
+ fg_color = c;
+ fg_color_mask = how;
+ }
+ auto set_bgcolor(const rspamd::css::css_color &c, int how = html_block::set) -> void
+ {
+ bg_color = c;
+ bg_color_mask = how;
+ }
+ auto set_height(float h, bool is_percent = false, int how = html_block::set) -> void
+ {
+ h = is_percent ? (-h) : h;
+ if (h < INT16_MIN) {
+ /* Negative numbers encode percents... */
+ height = -100;
+ }
+ else if (h > INT16_MAX) {
+ height = INT16_MAX;
+ }
+ else {
+ height = h;
+ }
+ height_mask = how;
+ }
+
+ auto set_width(float w, bool is_percent = false, int how = html_block::set) -> void
+ {
+ w = is_percent ? (-w) : w;
+ if (w < INT16_MIN) {
+ width = INT16_MIN;
+ }
+ else if (w > INT16_MAX) {
+ width = INT16_MAX;
+ }
+ else {
+ width = w;
+ }
+ width_mask = how;
+ }
+
+ auto set_display(bool v, int how = html_block::set) -> void
+ {
+ if (v) {
+ display = rspamd::css::css_display_value::DISPLAY_INLINE;
+ }
+ else {
+ display = rspamd::css::css_display_value::DISPLAY_HIDDEN;
+ }
+ display_mask = how;
+ }
+
+ auto set_display(rspamd::css::css_display_value v, int how = html_block::set) -> void
+ {
+ display = v;
+ display_mask = how;
+ }
+
+ auto set_font_size(float fs, bool is_percent = false, int how = html_block::set) -> void
+ {
+ fs = is_percent ? (-fs) : fs;
+ if (fs < INT8_MIN) {
+ font_size = -100;
+ }
+ else if (fs > INT8_MAX) {
+ font_size = INT8_MAX;
+ }
+ else {
+ font_size = fs;
+ }
+ font_mask = how;
+ }
+
+private:
+ template<typename T, typename MT>
+ static constexpr auto simple_prop(MT mask_val, MT other_mask, T &our_val,
+ T other_val) -> MT
+ {
+ if (other_mask && other_mask > mask_val) {
+ our_val = other_val;
+ mask_val = html_block::inherited;
+ }
+
+ return mask_val;
+ }
+
+ /* Sizes propagation logic
+ * We can have multiple cases:
+ * 1) Our size is > 0 and we can use it as is
+ * 2) Parent size is > 0 and our size is undefined, so propagate parent
+ * 3) Parent size is < 0 and our size is undefined - propagate parent
+ * 4) Parent size is > 0 and our size is < 0 - multiply parent by abs(ours)
+ * 5) Parent size is undefined and our size is < 0 - tricky stuff, assume some defaults
+ */
+ template<typename T, typename MT>
+ static constexpr auto size_prop(MT mask_val, MT other_mask, T &our_val,
+ T other_val, T default_val) -> MT
+ {
+ if (mask_val) {
+ /* We have our value */
+ if (our_val < 0) {
+ if (other_mask > 0) {
+ if (other_val >= 0) {
+ our_val = other_val * (-our_val / 100.0);
+ }
+ else {
+ our_val *= (-other_val / 100.0);
+ }
+ }
+ else {
+ /* Parent value is not defined and our value is relative */
+ our_val = default_val * (-our_val / 100.0);
+ }
+ }
+ else if (other_mask && other_mask > mask_val) {
+ our_val = other_val;
+ mask_val = html_block::inherited;
+ }
+ }
+ else {
+ /* We propagate parent if defined */
+ if (other_mask && other_mask > mask_val) {
+ our_val = other_val;
+ mask_val = html_block::inherited;
+ }
+ /* Otherwise do nothing */
+ }
+
+ return mask_val;
+ }
+
+public:
+ /**
+ * Propagate values from the block if they are not defined by the current block
+ * @param other
+ * @return
+ */
+ auto propagate_block(const html_block &other) -> void
+ {
+ fg_color_mask = html_block::simple_prop(fg_color_mask, other.fg_color_mask,
+ fg_color, other.fg_color);
+ bg_color_mask = html_block::simple_prop(bg_color_mask, other.bg_color_mask,
+ bg_color, other.bg_color);
+ display_mask = html_block::simple_prop(display_mask, other.display_mask,
+ display, other.display);
+
+ height_mask = html_block::size_prop(height_mask, other.height_mask,
+ height, other.height, static_cast<std::int16_t>(800));
+ width_mask = html_block::size_prop(width_mask, other.width_mask,
+ width, other.width, static_cast<std::int16_t>(1024));
+ font_mask = html_block::size_prop(font_mask, other.font_mask,
+ font_size, other.font_size, static_cast<std::int8_t>(10));
+ }
+
+ /*
+ * Set block overriding all inherited values
+ */
+ auto set_block(const html_block &other) -> void
+ {
+ constexpr auto set_value = [](auto mask_val, auto other_mask, auto &our_val,
+ auto other_val) constexpr -> int {
+ if (other_mask && mask_val != html_block::set) {
+ our_val = other_val;
+ mask_val = other_mask;
+ }
+
+ return mask_val;
+ };
+
+ fg_color_mask = set_value(fg_color_mask, other.fg_color_mask, fg_color, other.fg_color);
+ bg_color_mask = set_value(bg_color_mask, other.bg_color_mask, bg_color, other.bg_color);
+ display_mask = set_value(display_mask, other.display_mask, display, other.display);
+ height_mask = set_value(height_mask, other.height_mask, height, other.height);
+ width_mask = set_value(width_mask, other.width_mask, width, other.width);
+ font_mask = set_value(font_mask, other.font_mask, font_size, other.font_size);
+ }
+
+ auto compute_visibility(void) -> void
+ {
+ if (display_mask) {
+ if (display == css::css_display_value::DISPLAY_HIDDEN) {
+ visibility_mask = html_block::invisible_flag;
+
+ return;
+ }
+ }
+
+ if (font_mask) {
+ if (font_size == 0) {
+ visibility_mask = html_block::invisible_flag;
+
+ return;
+ }
+ }
+
+ auto is_similar_colors = [](const rspamd::css::css_color &fg, const rspamd::css::css_color &bg) -> bool {
+ constexpr const auto min_visible_diff = 0.1f;
+ auto diff_r = ((float) fg.r - bg.r);
+ auto diff_g = ((float) fg.g - bg.g);
+ auto diff_b = ((float) fg.b - bg.b);
+ auto ravg = ((float) fg.r + bg.r) / 2.0f;
+
+ /* Square diffs */
+ diff_r *= diff_r;
+ diff_g *= diff_g;
+ diff_b *= diff_b;
+
+ auto diff = std::sqrt(2.0f * diff_r + 4.0f * diff_g + 3.0f * diff_b +
+ (ravg * (diff_r - diff_b) / 256.0f)) /
+ 256.0f;
+
+ return diff < min_visible_diff;
+ };
+ /* Check if we have both bg/fg colors */
+ if (fg_color_mask && bg_color_mask) {
+ if (fg_color.alpha < 10) {
+ /* Too transparent */
+ visibility_mask = html_block::transparent_flag;
+
+ return;
+ }
+
+ if (bg_color.alpha > 10) {
+ if (is_similar_colors(fg_color, bg_color)) {
+ visibility_mask = html_block::transparent_flag;
+ return;
+ }
+ }
+ }
+ else if (fg_color_mask) {
+ /* Merely fg color */
+ if (fg_color.alpha < 10) {
+ /* Too transparent */
+ visibility_mask = html_block::transparent_flag;
+
+ return;
+ }
+
+ /* Implicit fg color */
+ if (is_similar_colors(fg_color, rspamd::css::css_color::white())) {
+ visibility_mask = html_block::transparent_flag;
+ return;
+ }
+ }
+ else if (bg_color_mask) {
+ if (bg_color.alpha > 10) {
+ if (is_similar_colors(rspamd::css::css_color::black(), bg_color)) {
+ visibility_mask = html_block::transparent_flag;
+ return;
+ }
+ }
+ }
+
+ visibility_mask = html_block::unset;
+ }
+
+ constexpr auto is_visible(void) const -> bool
+ {
+ return visibility_mask == html_block::unset;
+ }
+
+ constexpr auto is_transparent(void) const -> bool
+ {
+ return visibility_mask == html_block::transparent_flag;
+ }
+
+ constexpr auto has_display(int how = html_block::set) const -> bool
+ {
+ return display_mask >= how;
+ }
+
+ /**
+ * Returns a default html block for root HTML element
+ * @return
+ */
+ static auto default_html_block(void) -> html_block
+ {
+ return html_block{.fg_color = rspamd::css::css_color::black(),
+ .bg_color = rspamd::css::css_color::white(),
+ .height = 0,
+ .width = 0,
+ .display = rspamd::css::css_display_value::DISPLAY_INLINE,
+ .font_size = 12,
+ .fg_color_mask = html_block::inherited,
+ .bg_color_mask = html_block::inherited,
+ .height_mask = html_block::unset,
+ .width_mask = html_block::unset,
+ .font_mask = html_block::unset,
+ .display_mask = html_block::inherited,
+ .visibility_mask = html_block::unset};
+ }
+ /**
+ * Produces html block with no defined values allocated from the pool
+ * @param pool
+ * @return
+ */
+ static auto undefined_html_block_pool(rspamd_mempool_t *pool) -> html_block *
+ {
+ auto *bl = rspamd_mempool_alloc0_type(pool, html_block);
+
+ return bl;
+ }
+};
+
+}// namespace rspamd::html
+
+#endif//RSPAMD_HTML_BLOCK_HXX
diff --git a/src/libserver/html/html_entities.cxx b/src/libserver/html/html_entities.cxx
new file mode 100644
index 0000000..c642536
--- /dev/null
+++ b/src/libserver/html/html_entities.cxx
@@ -0,0 +1,2644 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "html_entities.hxx"
+
+#include <string>
+#include <utility>
+#include <vector>
+#include "contrib/ankerl/unordered_dense.h"
+#include <unicode/utf8.h>
+#include <unicode/uchar.h>
+#include "libutil/cxx/util.hxx"
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+namespace rspamd::html {
+
+struct html_entity_def {
+ const char *name;
+ const char *replacement;
+ unsigned code;
+ bool allow_heuristic;
+};
+
+#define ENTITY_DEF(name, code, replacement) \
+ html_entity_def \
+ { \
+ (name), (replacement), (code), false \
+ }
+#define ENTITY_DEF_HEUR(name, code, replacement) \
+ html_entity_def \
+ { \
+ (name), (replacement), (code), true \
+ }
+
+static const html_entity_def html_entities_array[] = {
+ ENTITY_DEF_HEUR("szlig", 223, "\xc3\x9f"),
+ ENTITY_DEF("prime", 8242, "\xe2\x80\xb2"),
+ ENTITY_DEF("lnsim", 8934, "\xe2\x8b\xa6"),
+ ENTITY_DEF("nvDash", 8877, "\xe2\x8a\xad"),
+ ENTITY_DEF("isinsv", 8947, "\xe2\x8b\xb3"),
+ ENTITY_DEF("notin", 8713, "\xe2\x88\x89"),
+ ENTITY_DEF("becaus", 8757, "\xe2\x88\xb5"),
+ ENTITY_DEF("Leftrightarrow", 8660, "\xe2\x87\x94"),
+ ENTITY_DEF("EmptySmallSquare", 9723, "\xe2\x97\xbb"),
+ ENTITY_DEF("SquareUnion", 8852, "\xe2\x8a\x94"),
+ ENTITY_DEF("subdot", 10941, "\xe2\xaa\xbd"),
+ ENTITY_DEF("Dstrok", 272, "\xc4\x90"),
+ ENTITY_DEF("rrarr", 8649, "\xe2\x87\x89"),
+ ENTITY_DEF("rArr", 8658, "\xe2\x87\x92"),
+ ENTITY_DEF_HEUR("Aacute", 193, "\xc3\x81"),
+ ENTITY_DEF("kappa", 954, "\xce\xba"),
+ ENTITY_DEF("Iopf", 120128, "\xf0\x9d\x95\x80"),
+ ENTITY_DEF("hyphen", 8208, "\xe2\x80\x90"),
+ ENTITY_DEF("rarrbfs", 10528, "\xe2\xa4\xa0"),
+ ENTITY_DEF("supsetneqq", 10956, "\xe2\xab\x8c"),
+ ENTITY_DEF("gacute", 501, "\xc7\xb5"),
+ ENTITY_DEF("VeryThinSpace", 8202, "\xe2\x80\x8a"),
+ ENTITY_DEF("tint", 8749, "\xe2\x88\xad"),
+ ENTITY_DEF("ffr", 120099, "\xf0\x9d\x94\xa3"),
+ ENTITY_DEF("kgreen", 312, "\xc4\xb8"),
+ ENTITY_DEF("nis", 8956, "\xe2\x8b\xbc"),
+ ENTITY_DEF("NotRightTriangleBar", 10704, "\xe2\xa7\x90\xcc\xb8"),
+ ENTITY_DEF("Eogon", 280, "\xc4\x98"),
+ ENTITY_DEF("lbrke", 10635, "\xe2\xa6\x8b"),
+ ENTITY_DEF("phi", 966, "\xcf\x86"),
+ ENTITY_DEF("notnivc", 8957, "\xe2\x8b\xbd"),
+ ENTITY_DEF("utilde", 361, "\xc5\xa9"),
+ ENTITY_DEF("Fopf", 120125, "\xf0\x9d\x94\xbd"),
+ ENTITY_DEF("Vcy", 1042, "\xd0\x92"),
+ ENTITY_DEF("erDot", 8787, "\xe2\x89\x93"),
+ ENTITY_DEF("nsubE", 10949, "\xe2\xab\x85\xcc\xb8"),
+ ENTITY_DEF_HEUR("egrave", 232, "\xc3\xa8"),
+ ENTITY_DEF("Lcedil", 315, "\xc4\xbb"),
+ ENTITY_DEF("lharul", 10602, "\xe2\xa5\xaa"),
+ ENTITY_DEF_HEUR("middot", 183, "\xc2\xb7"),
+ ENTITY_DEF("ggg", 8921, "\xe2\x8b\x99"),
+ ENTITY_DEF("NestedLessLess", 8810, "\xe2\x89\xaa"),
+ ENTITY_DEF("tau", 964, "\xcf\x84"),
+ ENTITY_DEF("setmn", 8726, "\xe2\x88\x96"),
+ ENTITY_DEF("frac78", 8542, "\xe2\x85\x9e"),
+ ENTITY_DEF_HEUR("para", 182, "\xc2\xb6"),
+ ENTITY_DEF("Rcedil", 342, "\xc5\x96"),
+ ENTITY_DEF("propto", 8733, "\xe2\x88\x9d"),
+ ENTITY_DEF("sqsubset", 8847, "\xe2\x8a\x8f"),
+ ENTITY_DEF("ensp", 8194, "\xe2\x80\x82"),
+ ENTITY_DEF("boxvH", 9578, "\xe2\x95\xaa"),
+ ENTITY_DEF("NotGreaterTilde", 8821, "\xe2\x89\xb5"),
+ ENTITY_DEF("ffllig", 64260, "\xef\xac\x84"),
+ ENTITY_DEF("kcedil", 311, "\xc4\xb7"),
+ ENTITY_DEF("omega", 969, "\xcf\x89"),
+ ENTITY_DEF("sime", 8771, "\xe2\x89\x83"),
+ ENTITY_DEF("LeftTriangleEqual", 8884, "\xe2\x8a\xb4"),
+ ENTITY_DEF("bsemi", 8271, "\xe2\x81\x8f"),
+ ENTITY_DEF("rdquor", 8221, "\xe2\x80\x9d"),
+ ENTITY_DEF("Utilde", 360, "\xc5\xa8"),
+ ENTITY_DEF("bsol", 92, "\x5c"),
+ ENTITY_DEF("risingdotseq", 8787, "\xe2\x89\x93"),
+ ENTITY_DEF("ultri", 9720, "\xe2\x97\xb8"),
+ ENTITY_DEF("rhov", 1009, "\xcf\xb1"),
+ ENTITY_DEF("TildeEqual", 8771, "\xe2\x89\x83"),
+ ENTITY_DEF("jukcy", 1108, "\xd1\x94"),
+ ENTITY_DEF("perp", 8869, "\xe2\x8a\xa5"),
+ ENTITY_DEF("capbrcup", 10825, "\xe2\xa9\x89"),
+ ENTITY_DEF("ltrie", 8884, "\xe2\x8a\xb4"),
+ ENTITY_DEF("LessTilde", 8818, "\xe2\x89\xb2"),
+ ENTITY_DEF("popf", 120161, "\xf0\x9d\x95\xa1"),
+ ENTITY_DEF("dbkarow", 10511, "\xe2\xa4\x8f"),
+ ENTITY_DEF("roang", 10221, "\xe2\x9f\xad"),
+ ENTITY_DEF_HEUR("brvbar", 166, "\xc2\xa6"),
+ ENTITY_DEF("CenterDot", 183, "\xc2\xb7"),
+ ENTITY_DEF("notindot", 8949, "\xe2\x8b\xb5\xcc\xb8"),
+ ENTITY_DEF("supmult", 10946, "\xe2\xab\x82"),
+ ENTITY_DEF("multimap", 8888, "\xe2\x8a\xb8"),
+ ENTITY_DEF_HEUR("frac34", 190, "\xc2\xbe"),
+ ENTITY_DEF("mapsto", 8614, "\xe2\x86\xa6"),
+ ENTITY_DEF("flat", 9837, "\xe2\x99\xad"),
+ ENTITY_DEF("updownarrow", 8597, "\xe2\x86\x95"),
+ ENTITY_DEF("gne", 10888, "\xe2\xaa\x88"),
+ ENTITY_DEF("nrarrc", 10547, "\xe2\xa4\xb3\xcc\xb8"),
+ ENTITY_DEF("suphsol", 10185, "\xe2\x9f\x89"),
+ ENTITY_DEF("nGtv", 8811, "\xe2\x89\xab\xcc\xb8"),
+ ENTITY_DEF("hopf", 120153, "\xf0\x9d\x95\x99"),
+ ENTITY_DEF("pointint", 10773, "\xe2\xa8\x95"),
+ ENTITY_DEF("glj", 10916, "\xe2\xaa\xa4"),
+ ENTITY_DEF("LeftDoubleBracket", 10214, "\xe2\x9f\xa6"),
+ ENTITY_DEF("NotSupersetEqual", 8841, "\xe2\x8a\x89"),
+ ENTITY_DEF("dot", 729, "\xcb\x99"),
+ ENTITY_DEF("tbrk", 9140, "\xe2\x8e\xb4"),
+ ENTITY_DEF("LeftUpDownVector", 10577, "\xe2\xa5\x91"),
+ ENTITY_DEF_HEUR("uml", 168, "\xc2\xa8"),
+ ENTITY_DEF("bbrk", 9141, "\xe2\x8e\xb5"),
+ ENTITY_DEF("nearrow", 8599, "\xe2\x86\x97"),
+ ENTITY_DEF("backsimeq", 8909, "\xe2\x8b\x8d"),
+ ENTITY_DEF("dblac", 733, "\xcb\x9d"),
+ ENTITY_DEF("circleddash", 8861, "\xe2\x8a\x9d"),
+ ENTITY_DEF("ldsh", 8626, "\xe2\x86\xb2"),
+ ENTITY_DEF("sce", 10928, "\xe2\xaa\xb0"),
+ ENTITY_DEF("angst", 197, "\xc3\x85"),
+ ENTITY_DEF_HEUR("yen", 165, "\xc2\xa5"),
+ ENTITY_DEF("nsupE", 10950, "\xe2\xab\x86\xcc\xb8"),
+ ENTITY_DEF("Uscr", 119984, "\xf0\x9d\x92\xb0"),
+ ENTITY_DEF("subplus", 10943, "\xe2\xaa\xbf"),
+ ENTITY_DEF("nleqq", 8806, "\xe2\x89\xa6\xcc\xb8"),
+ ENTITY_DEF("nprcue", 8928, "\xe2\x8b\xa0"),
+ ENTITY_DEF("Ocirc", 212, "\xc3\x94"),
+ ENTITY_DEF("disin", 8946, "\xe2\x8b\xb2"),
+ ENTITY_DEF("EqualTilde", 8770, "\xe2\x89\x82"),
+ ENTITY_DEF("YUcy", 1070, "\xd0\xae"),
+ ENTITY_DEF("Kscr", 119974, "\xf0\x9d\x92\xa6"),
+ ENTITY_DEF("lg", 8822, "\xe2\x89\xb6"),
+ ENTITY_DEF("nLeftrightarrow", 8654, "\xe2\x87\x8e"),
+ ENTITY_DEF("eplus", 10865, "\xe2\xa9\xb1"),
+ ENTITY_DEF("les", 10877, "\xe2\xa9\xbd"),
+ ENTITY_DEF("sfr", 120112, "\xf0\x9d\x94\xb0"),
+ ENTITY_DEF("HumpDownHump", 8782, "\xe2\x89\x8e"),
+ ENTITY_DEF("Fouriertrf", 8497, "\xe2\x84\xb1"),
+ ENTITY_DEF("Updownarrow", 8661, "\xe2\x87\x95"),
+ ENTITY_DEF("nrarr", 8603, "\xe2\x86\x9b"),
+ ENTITY_DEF("radic", 8730, "\xe2\x88\x9a"),
+ ENTITY_DEF("gnap", 10890, "\xe2\xaa\x8a"),
+ ENTITY_DEF("zeta", 950, "\xce\xb6"),
+ ENTITY_DEF("Qscr", 119980, "\xf0\x9d\x92\xac"),
+ ENTITY_DEF("NotRightTriangleEqual", 8941, "\xe2\x8b\xad"),
+ ENTITY_DEF("nshortmid", 8740, "\xe2\x88\xa4"),
+ ENTITY_DEF("SHCHcy", 1065, "\xd0\xa9"),
+ ENTITY_DEF("piv", 982, "\xcf\x96"),
+ ENTITY_DEF("angmsdaa", 10664, "\xe2\xa6\xa8"),
+ ENTITY_DEF("curlywedge", 8911, "\xe2\x8b\x8f"),
+ ENTITY_DEF("sqcaps", 8851, "\xe2\x8a\x93\xef\xb8\x80"),
+ ENTITY_DEF("sum", 8721, "\xe2\x88\x91"),
+ ENTITY_DEF("rarrtl", 8611, "\xe2\x86\xa3"),
+ ENTITY_DEF("gescc", 10921, "\xe2\xaa\xa9"),
+ ENTITY_DEF("sup", 8835, "\xe2\x8a\x83"),
+ ENTITY_DEF("smid", 8739, "\xe2\x88\xa3"),
+ ENTITY_DEF("cularr", 8630, "\xe2\x86\xb6"),
+ ENTITY_DEF("olcross", 10683, "\xe2\xa6\xbb"),
+ ENTITY_DEF_HEUR("GT", 62, "\x3e"),
+ ENTITY_DEF("scap", 10936, "\xe2\xaa\xb8"),
+ ENTITY_DEF("capcup", 10823, "\xe2\xa9\x87"),
+ ENTITY_DEF("NotSquareSubsetEqual", 8930, "\xe2\x8b\xa2"),
+ ENTITY_DEF("uhblk", 9600, "\xe2\x96\x80"),
+ ENTITY_DEF("latail", 10521, "\xe2\xa4\x99"),
+ ENTITY_DEF("smtes", 10924, "\xe2\xaa\xac\xef\xb8\x80"),
+ ENTITY_DEF("RoundImplies", 10608, "\xe2\xa5\xb0"),
+ ENTITY_DEF("wreath", 8768, "\xe2\x89\x80"),
+ ENTITY_DEF("curlyvee", 8910, "\xe2\x8b\x8e"),
+ ENTITY_DEF("uscr", 120010, "\xf0\x9d\x93\x8a"),
+ ENTITY_DEF("nleftrightarrow", 8622, "\xe2\x86\xae"),
+ ENTITY_DEF("ucy", 1091, "\xd1\x83"),
+ ENTITY_DEF("nvge", 8805, "\xe2\x89\xa5\xe2\x83\x92"),
+ ENTITY_DEF("bnot", 8976, "\xe2\x8c\x90"),
+ ENTITY_DEF("alefsym", 8501, "\xe2\x84\xb5"),
+ ENTITY_DEF("star", 9734, "\xe2\x98\x86"),
+ ENTITY_DEF("boxHd", 9572, "\xe2\x95\xa4"),
+ ENTITY_DEF("vsubnE", 10955, "\xe2\xab\x8b\xef\xb8\x80"),
+ ENTITY_DEF("Popf", 8473, "\xe2\x84\x99"),
+ ENTITY_DEF("simgE", 10912, "\xe2\xaa\xa0"),
+ ENTITY_DEF("upsilon", 965, "\xcf\x85"),
+ ENTITY_DEF("NoBreak", 8288, "\xe2\x81\xa0"),
+ ENTITY_DEF("realine", 8475, "\xe2\x84\x9b"),
+ ENTITY_DEF("frac38", 8540, "\xe2\x85\x9c"),
+ ENTITY_DEF("YAcy", 1071, "\xd0\xaf"),
+ ENTITY_DEF("bnequiv", 8801, "\xe2\x89\xa1\xe2\x83\xa5"),
+ ENTITY_DEF("cudarrr", 10549, "\xe2\xa4\xb5"),
+ ENTITY_DEF("lsime", 10893, "\xe2\xaa\x8d"),
+ ENTITY_DEF("lowbar", 95, "\x5f"),
+ ENTITY_DEF("utdot", 8944, "\xe2\x8b\xb0"),
+ ENTITY_DEF("ReverseElement", 8715, "\xe2\x88\x8b"),
+ ENTITY_DEF("nshortparallel", 8742, "\xe2\x88\xa6"),
+ ENTITY_DEF("DJcy", 1026, "\xd0\x82"),
+ ENTITY_DEF("nsube", 8840, "\xe2\x8a\x88"),
+ ENTITY_DEF("VDash", 8875, "\xe2\x8a\xab"),
+ ENTITY_DEF("Ncaron", 327, "\xc5\x87"),
+ ENTITY_DEF("LeftUpVector", 8639, "\xe2\x86\xbf"),
+ ENTITY_DEF("Kcy", 1050, "\xd0\x9a"),
+ ENTITY_DEF("NotLeftTriangleEqual", 8940, "\xe2\x8b\xac"),
+ ENTITY_DEF("nvHarr", 10500, "\xe2\xa4\x84"),
+ ENTITY_DEF("lotimes", 10804, "\xe2\xa8\xb4"),
+ ENTITY_DEF("RightFloor", 8971, "\xe2\x8c\x8b"),
+ ENTITY_DEF("succ", 8827, "\xe2\x89\xbb"),
+ ENTITY_DEF("Ucy", 1059, "\xd0\xa3"),
+ ENTITY_DEF("darr", 8595, "\xe2\x86\x93"),
+ ENTITY_DEF("lbarr", 10508, "\xe2\xa4\x8c"),
+ ENTITY_DEF("xfr", 120117, "\xf0\x9d\x94\xb5"),
+ ENTITY_DEF("zopf", 120171, "\xf0\x9d\x95\xab"),
+ ENTITY_DEF("Phi", 934, "\xce\xa6"),
+ ENTITY_DEF("ord", 10845, "\xe2\xa9\x9d"),
+ ENTITY_DEF("iinfin", 10716, "\xe2\xa7\x9c"),
+ ENTITY_DEF("Xfr", 120091, "\xf0\x9d\x94\x9b"),
+ ENTITY_DEF("qint", 10764, "\xe2\xa8\x8c"),
+ ENTITY_DEF("Upsilon", 933, "\xce\xa5"),
+ ENTITY_DEF("NotSubset", 8834, "\xe2\x8a\x82\xe2\x83\x92"),
+ ENTITY_DEF("gfr", 120100, "\xf0\x9d\x94\xa4"),
+ ENTITY_DEF("notnivb", 8958, "\xe2\x8b\xbe"),
+ ENTITY_DEF("Afr", 120068, "\xf0\x9d\x94\x84"),
+ ENTITY_DEF_HEUR("ge", 8805, "\xe2\x89\xa5"),
+ ENTITY_DEF_HEUR("iexcl", 161, "\xc2\xa1"),
+ ENTITY_DEF("dfr", 120097, "\xf0\x9d\x94\xa1"),
+ ENTITY_DEF("rsaquo", 8250, "\xe2\x80\xba"),
+ ENTITY_DEF("xcap", 8898, "\xe2\x8b\x82"),
+ ENTITY_DEF("Jopf", 120129, "\xf0\x9d\x95\x81"),
+ ENTITY_DEF("Hstrok", 294, "\xc4\xa6"),
+ ENTITY_DEF("ldca", 10550, "\xe2\xa4\xb6"),
+ ENTITY_DEF("lmoust", 9136, "\xe2\x8e\xb0"),
+ ENTITY_DEF("wcirc", 373, "\xc5\xb5"),
+ ENTITY_DEF("DownRightVector", 8641, "\xe2\x87\x81"),
+ ENTITY_DEF("LessFullEqual", 8806, "\xe2\x89\xa6"),
+ ENTITY_DEF("dotsquare", 8865, "\xe2\x8a\xa1"),
+ ENTITY_DEF("zhcy", 1078, "\xd0\xb6"),
+ ENTITY_DEF("mDDot", 8762, "\xe2\x88\xba"),
+ ENTITY_DEF("Prime", 8243, "\xe2\x80\xb3"),
+ ENTITY_DEF("prec", 8826, "\xe2\x89\xba"),
+ ENTITY_DEF("swnwar", 10538, "\xe2\xa4\xaa"),
+ ENTITY_DEF_HEUR("COPY", 169, "\xc2\xa9"),
+ ENTITY_DEF("cong", 8773, "\xe2\x89\x85"),
+ ENTITY_DEF("sacute", 347, "\xc5\x9b"),
+ ENTITY_DEF("Nopf", 8469, "\xe2\x84\x95"),
+ ENTITY_DEF("it", 8290, "\xe2\x81\xa2"),
+ ENTITY_DEF("SOFTcy", 1068, "\xd0\xac"),
+ ENTITY_DEF("uuarr", 8648, "\xe2\x87\x88"),
+ ENTITY_DEF("iota", 953, "\xce\xb9"),
+ ENTITY_DEF("notinE", 8953, "\xe2\x8b\xb9\xcc\xb8"),
+ ENTITY_DEF("jfr", 120103, "\xf0\x9d\x94\xa7"),
+ ENTITY_DEF_HEUR("QUOT", 34, "\x22"),
+ ENTITY_DEF("vsupnE", 10956, "\xe2\xab\x8c\xef\xb8\x80"),
+ ENTITY_DEF_HEUR("igrave", 236, "\xc3\xac"),
+ ENTITY_DEF("bsim", 8765, "\xe2\x88\xbd"),
+ ENTITY_DEF("npreceq", 10927, "\xe2\xaa\xaf\xcc\xb8"),
+ ENTITY_DEF("zcaron", 382, "\xc5\xbe"),
+ ENTITY_DEF("DD", 8517, "\xe2\x85\x85"),
+ ENTITY_DEF("gamma", 947, "\xce\xb3"),
+ ENTITY_DEF("homtht", 8763, "\xe2\x88\xbb"),
+ ENTITY_DEF("NonBreakingSpace", 160, "\xc2\xa0"),
+ ENTITY_DEF("Proportion", 8759, "\xe2\x88\xb7"),
+ ENTITY_DEF("nedot", 8784, "\xe2\x89\x90\xcc\xb8"),
+ ENTITY_DEF("nabla", 8711, "\xe2\x88\x87"),
+ ENTITY_DEF("ac", 8766, "\xe2\x88\xbe"),
+ ENTITY_DEF("nsupe", 8841, "\xe2\x8a\x89"),
+ ENTITY_DEF("ell", 8467, "\xe2\x84\x93"),
+ ENTITY_DEF("boxvR", 9566, "\xe2\x95\x9e"),
+ ENTITY_DEF("LowerRightArrow", 8600, "\xe2\x86\x98"),
+ ENTITY_DEF("boxHu", 9575, "\xe2\x95\xa7"),
+ ENTITY_DEF("lE", 8806, "\xe2\x89\xa6"),
+ ENTITY_DEF("dzigrarr", 10239, "\xe2\x9f\xbf"),
+ ENTITY_DEF("rfloor", 8971, "\xe2\x8c\x8b"),
+ ENTITY_DEF("gneq", 10888, "\xe2\xaa\x88"),
+ ENTITY_DEF("rightleftharpoons", 8652, "\xe2\x87\x8c"),
+ ENTITY_DEF("gtquest", 10876, "\xe2\xa9\xbc"),
+ ENTITY_DEF("searhk", 10533, "\xe2\xa4\xa5"),
+ ENTITY_DEF("gesdoto", 10882, "\xe2\xaa\x82"),
+ ENTITY_DEF("cross", 10007, "\xe2\x9c\x97"),
+ ENTITY_DEF("rdquo", 8221, "\xe2\x80\x9d"),
+ ENTITY_DEF("sqsupset", 8848, "\xe2\x8a\x90"),
+ ENTITY_DEF("divonx", 8903, "\xe2\x8b\x87"),
+ ENTITY_DEF("lat", 10923, "\xe2\xaa\xab"),
+ ENTITY_DEF("rmoustache", 9137, "\xe2\x8e\xb1"),
+ ENTITY_DEF("succapprox", 10936, "\xe2\xaa\xb8"),
+ ENTITY_DEF("nhpar", 10994, "\xe2\xab\xb2"),
+ ENTITY_DEF("sharp", 9839, "\xe2\x99\xaf"),
+ ENTITY_DEF("lrcorner", 8991, "\xe2\x8c\x9f"),
+ ENTITY_DEF("Vscr", 119985, "\xf0\x9d\x92\xb1"),
+ ENTITY_DEF("varsigma", 962, "\xcf\x82"),
+ ENTITY_DEF("bsolb", 10693, "\xe2\xa7\x85"),
+ ENTITY_DEF("cupcap", 10822, "\xe2\xa9\x86"),
+ ENTITY_DEF("leftrightarrow", 8596, "\xe2\x86\x94"),
+ ENTITY_DEF("LeftTee", 8867, "\xe2\x8a\xa3"),
+ ENTITY_DEF("Sqrt", 8730, "\xe2\x88\x9a"),
+ ENTITY_DEF("Odblac", 336, "\xc5\x90"),
+ ENTITY_DEF("ocir", 8858, "\xe2\x8a\x9a"),
+ ENTITY_DEF("eqslantless", 10901, "\xe2\xaa\x95"),
+ ENTITY_DEF("supedot", 10948, "\xe2\xab\x84"),
+ ENTITY_DEF("intercal", 8890, "\xe2\x8a\xba"),
+ ENTITY_DEF("Gbreve", 286, "\xc4\x9e"),
+ ENTITY_DEF("xrArr", 10233, "\xe2\x9f\xb9"),
+ ENTITY_DEF("NotTildeEqual", 8772, "\xe2\x89\x84"),
+ ENTITY_DEF("Bfr", 120069, "\xf0\x9d\x94\x85"),
+ ENTITY_DEF_HEUR("Iuml", 207, "\xc3\x8f"),
+ ENTITY_DEF("leg", 8922, "\xe2\x8b\x9a"),
+ ENTITY_DEF("boxhU", 9576, "\xe2\x95\xa8"),
+ ENTITY_DEF("Gopf", 120126, "\xf0\x9d\x94\xbe"),
+ ENTITY_DEF("af", 8289, "\xe2\x81\xa1"),
+ ENTITY_DEF("xwedge", 8896, "\xe2\x8b\x80"),
+ ENTITY_DEF("precapprox", 10935, "\xe2\xaa\xb7"),
+ ENTITY_DEF("lcedil", 316, "\xc4\xbc"),
+ ENTITY_DEF("between", 8812, "\xe2\x89\xac"),
+ ENTITY_DEF_HEUR("Oslash", 216, "\xc3\x98"),
+ ENTITY_DEF("breve", 728, "\xcb\x98"),
+ ENTITY_DEF("caps", 8745, "\xe2\x88\xa9\xef\xb8\x80"),
+ ENTITY_DEF("vangrt", 10652, "\xe2\xa6\x9c"),
+ ENTITY_DEF("lagran", 8466, "\xe2\x84\x92"),
+ ENTITY_DEF("kopf", 120156, "\xf0\x9d\x95\x9c"),
+ ENTITY_DEF("ReverseUpEquilibrium", 10607, "\xe2\xa5\xaf"),
+ ENTITY_DEF("nlsim", 8820, "\xe2\x89\xb4"),
+ ENTITY_DEF("Cap", 8914, "\xe2\x8b\x92"),
+ ENTITY_DEF("angmsdac", 10666, "\xe2\xa6\xaa"),
+ ENTITY_DEF("iocy", 1105, "\xd1\x91"),
+ ENTITY_DEF("seswar", 10537, "\xe2\xa4\xa9"),
+ ENTITY_DEF("dzcy", 1119, "\xd1\x9f"),
+ ENTITY_DEF("nsubset", 8834, "\xe2\x8a\x82\xe2\x83\x92"),
+ ENTITY_DEF("cup", 8746, "\xe2\x88\xaa"),
+ ENTITY_DEF("npar", 8742, "\xe2\x88\xa6"),
+ ENTITY_DEF("late", 10925, "\xe2\xaa\xad"),
+ ENTITY_DEF("plussim", 10790, "\xe2\xa8\xa6"),
+ ENTITY_DEF("Darr", 8609, "\xe2\x86\xa1"),
+ ENTITY_DEF("nexist", 8708, "\xe2\x88\x84"),
+ ENTITY_DEF_HEUR("cent", 162, "\xc2\xa2"),
+ ENTITY_DEF("khcy", 1093, "\xd1\x85"),
+ ENTITY_DEF("smallsetminus", 8726, "\xe2\x88\x96"),
+ ENTITY_DEF("ycirc", 375, "\xc5\xb7"),
+ ENTITY_DEF("lharu", 8636, "\xe2\x86\xbc"),
+ ENTITY_DEF("upuparrows", 8648, "\xe2\x87\x88"),
+ ENTITY_DEF("sigmaf", 962, "\xcf\x82"),
+ ENTITY_DEF("nltri", 8938, "\xe2\x8b\xaa"),
+ ENTITY_DEF("mstpos", 8766, "\xe2\x88\xbe"),
+ ENTITY_DEF("Zopf", 8484, "\xe2\x84\xa4"),
+ ENTITY_DEF("dwangle", 10662, "\xe2\xa6\xa6"),
+ ENTITY_DEF("bowtie", 8904, "\xe2\x8b\x88"),
+ ENTITY_DEF("Dfr", 120071, "\xf0\x9d\x94\x87"),
+ ENTITY_DEF_HEUR("iacute", 237, "\xc3\xad"),
+ ENTITY_DEF("njcy", 1114, "\xd1\x9a"),
+ ENTITY_DEF("cfr", 120096, "\xf0\x9d\x94\xa0"),
+ ENTITY_DEF("TripleDot", 8411, "\xe2\x83\x9b"),
+ ENTITY_DEF("Or", 10836, "\xe2\xa9\x94"),
+ ENTITY_DEF("blk34", 9619, "\xe2\x96\x93"),
+ ENTITY_DEF("equiv", 8801, "\xe2\x89\xa1"),
+ ENTITY_DEF("fflig", 64256, "\xef\xac\x80"),
+ ENTITY_DEF("Rang", 10219, "\xe2\x9f\xab"),
+ ENTITY_DEF("Wopf", 120142, "\xf0\x9d\x95\x8e"),
+ ENTITY_DEF("boxUl", 9564, "\xe2\x95\x9c"),
+ ENTITY_DEF_HEUR("frac12", 189, "\xc2\xbd"),
+ ENTITY_DEF("clubs", 9827, "\xe2\x99\xa3"),
+ ENTITY_DEF("amalg", 10815, "\xe2\xa8\xbf"),
+ ENTITY_DEF("Lang", 10218, "\xe2\x9f\xaa"),
+ ENTITY_DEF("asymp", 8776, "\xe2\x89\x88"),
+ ENTITY_DEF("models", 8871, "\xe2\x8a\xa7"),
+ ENTITY_DEF("emptyset", 8709, "\xe2\x88\x85"),
+ ENTITY_DEF("Tscr", 119983, "\xf0\x9d\x92\xaf"),
+ ENTITY_DEF("nleftarrow", 8602, "\xe2\x86\x9a"),
+ ENTITY_DEF("Omacr", 332, "\xc5\x8c"),
+ ENTITY_DEF("gtrarr", 10616, "\xe2\xa5\xb8"),
+ ENTITY_DEF("excl", 33, "\x21"),
+ ENTITY_DEF("rarrw", 8605, "\xe2\x86\x9d"),
+ ENTITY_DEF("abreve", 259, "\xc4\x83"),
+ ENTITY_DEF("CircleTimes", 8855, "\xe2\x8a\x97"),
+ ENTITY_DEF("aopf", 120146, "\xf0\x9d\x95\x92"),
+ ENTITY_DEF("eqvparsl", 10725, "\xe2\xa7\xa5"),
+ ENTITY_DEF("boxv", 9474, "\xe2\x94\x82"),
+ ENTITY_DEF("SuchThat", 8715, "\xe2\x88\x8b"),
+ ENTITY_DEF("varphi", 981, "\xcf\x95"),
+ ENTITY_DEF("Ropf", 8477, "\xe2\x84\x9d"),
+ ENTITY_DEF("rscr", 120007, "\xf0\x9d\x93\x87"),
+ ENTITY_DEF("Rrightarrow", 8667, "\xe2\x87\x9b"),
+ ENTITY_DEF("equest", 8799, "\xe2\x89\x9f"),
+ ENTITY_DEF_HEUR("ntilde", 241, "\xc3\xb1"),
+ ENTITY_DEF("Escr", 8496, "\xe2\x84\xb0"),
+ ENTITY_DEF("Lopf", 120131, "\xf0\x9d\x95\x83"),
+ ENTITY_DEF("GreaterGreater", 10914, "\xe2\xaa\xa2"),
+ ENTITY_DEF("pluscir", 10786, "\xe2\xa8\xa2"),
+ ENTITY_DEF("nsupset", 8835, "\xe2\x8a\x83\xe2\x83\x92"),
+ ENTITY_DEF("uArr", 8657, "\xe2\x87\x91"),
+ ENTITY_DEF("nwarhk", 10531, "\xe2\xa4\xa3"),
+ ENTITY_DEF("Ycirc", 374, "\xc5\xb6"),
+ ENTITY_DEF("tdot", 8411, "\xe2\x83\x9b"),
+ ENTITY_DEF("circledS", 9416, "\xe2\x93\x88"),
+ ENTITY_DEF("lhard", 8637, "\xe2\x86\xbd"),
+ ENTITY_DEF("iukcy", 1110, "\xd1\x96"),
+ ENTITY_DEF("PrecedesSlantEqual", 8828, "\xe2\x89\xbc"),
+ ENTITY_DEF("Sfr", 120086, "\xf0\x9d\x94\x96"),
+ ENTITY_DEF("egs", 10902, "\xe2\xaa\x96"),
+ ENTITY_DEF("oelig", 339, "\xc5\x93"),
+ ENTITY_DEF("bigtriangledown", 9661, "\xe2\x96\xbd"),
+ ENTITY_DEF("EmptyVerySmallSquare", 9643, "\xe2\x96\xab"),
+ ENTITY_DEF("Backslash", 8726, "\xe2\x88\x96"),
+ ENTITY_DEF("nscr", 120003, "\xf0\x9d\x93\x83"),
+ ENTITY_DEF("uogon", 371, "\xc5\xb3"),
+ ENTITY_DEF("circeq", 8791, "\xe2\x89\x97"),
+ ENTITY_DEF("check", 10003, "\xe2\x9c\x93"),
+ ENTITY_DEF("Sup", 8913, "\xe2\x8b\x91"),
+ ENTITY_DEF("Rcaron", 344, "\xc5\x98"),
+ ENTITY_DEF("lneqq", 8808, "\xe2\x89\xa8"),
+ ENTITY_DEF("lrhar", 8651, "\xe2\x87\x8b"),
+ ENTITY_DEF("ulcorn", 8988, "\xe2\x8c\x9c"),
+ ENTITY_DEF("timesd", 10800, "\xe2\xa8\xb0"),
+ ENTITY_DEF("Sum", 8721, "\xe2\x88\x91"),
+ ENTITY_DEF("varpropto", 8733, "\xe2\x88\x9d"),
+ ENTITY_DEF("Lcaron", 317, "\xc4\xbd"),
+ ENTITY_DEF("lbrkslu", 10637, "\xe2\xa6\x8d"),
+ ENTITY_DEF_HEUR("AElig", 198, "\xc3\x86"),
+ ENTITY_DEF("varr", 8597, "\xe2\x86\x95"),
+ ENTITY_DEF("nvinfin", 10718, "\xe2\xa7\x9e"),
+ ENTITY_DEF("leq", 8804, "\xe2\x89\xa4"),
+ ENTITY_DEF("biguplus", 10756, "\xe2\xa8\x84"),
+ ENTITY_DEF("rpar", 41, "\x29"),
+ ENTITY_DEF("eng", 331, "\xc5\x8b"),
+ ENTITY_DEF("NegativeThinSpace", 8203, "\xe2\x80\x8b"),
+ ENTITY_DEF("lesssim", 8818, "\xe2\x89\xb2"),
+ ENTITY_DEF("lBarr", 10510, "\xe2\xa4\x8e"),
+ ENTITY_DEF("LeftUpTeeVector", 10592, "\xe2\xa5\xa0"),
+ ENTITY_DEF("gnE", 8809, "\xe2\x89\xa9"),
+ ENTITY_DEF("efr", 120098, "\xf0\x9d\x94\xa2"),
+ ENTITY_DEF("barvee", 8893, "\xe2\x8a\xbd"),
+ ENTITY_DEF("ee", 8519, "\xe2\x85\x87"),
+ ENTITY_DEF("Uogon", 370, "\xc5\xb2"),
+ ENTITY_DEF("gnapprox", 10890, "\xe2\xaa\x8a"),
+ ENTITY_DEF("olcir", 10686, "\xe2\xa6\xbe"),
+ ENTITY_DEF("boxUL", 9565, "\xe2\x95\x9d"),
+ ENTITY_DEF("Gg", 8921, "\xe2\x8b\x99"),
+ ENTITY_DEF("CloseCurlyQuote", 8217, "\xe2\x80\x99"),
+ ENTITY_DEF("leftharpoondown", 8637, "\xe2\x86\xbd"),
+ ENTITY_DEF("vfr", 120115, "\xf0\x9d\x94\xb3"),
+ ENTITY_DEF("gvertneqq", 8809, "\xe2\x89\xa9\xef\xb8\x80"),
+ ENTITY_DEF_HEUR("ouml", 246, "\xc3\xb6"),
+ ENTITY_DEF("raemptyv", 10675, "\xe2\xa6\xb3"),
+ ENTITY_DEF("Zcaron", 381, "\xc5\xbd"),
+ ENTITY_DEF("scE", 10932, "\xe2\xaa\xb4"),
+ ENTITY_DEF("boxvh", 9532, "\xe2\x94\xbc"),
+ ENTITY_DEF("ominus", 8854, "\xe2\x8a\x96"),
+ ENTITY_DEF("oopf", 120160, "\xf0\x9d\x95\xa0"),
+ ENTITY_DEF("nsucceq", 10928, "\xe2\xaa\xb0\xcc\xb8"),
+ ENTITY_DEF("RBarr", 10512, "\xe2\xa4\x90"),
+ ENTITY_DEF("iprod", 10812, "\xe2\xa8\xbc"),
+ ENTITY_DEF("lvnE", 8808, "\xe2\x89\xa8\xef\xb8\x80"),
+ ENTITY_DEF("andand", 10837, "\xe2\xa9\x95"),
+ ENTITY_DEF("upharpoonright", 8638, "\xe2\x86\xbe"),
+ ENTITY_DEF("ncongdot", 10861, "\xe2\xa9\xad\xcc\xb8"),
+ ENTITY_DEF("drcrop", 8972, "\xe2\x8c\x8c"),
+ ENTITY_DEF("nsimeq", 8772, "\xe2\x89\x84"),
+ ENTITY_DEF("subsub", 10965, "\xe2\xab\x95"),
+ ENTITY_DEF("hardcy", 1098, "\xd1\x8a"),
+ ENTITY_DEF("leqslant", 10877, "\xe2\xa9\xbd"),
+ ENTITY_DEF("uharl", 8639, "\xe2\x86\xbf"),
+ ENTITY_DEF("expectation", 8496, "\xe2\x84\xb0"),
+ ENTITY_DEF("mdash", 8212, "\xe2\x80\x94"),
+ ENTITY_DEF("VerticalTilde", 8768, "\xe2\x89\x80"),
+ ENTITY_DEF("rdldhar", 10601, "\xe2\xa5\xa9"),
+ ENTITY_DEF("leftharpoonup", 8636, "\xe2\x86\xbc"),
+ ENTITY_DEF("mu", 956, "\xce\xbc"),
+ ENTITY_DEF("curarrm", 10556, "\xe2\xa4\xbc"),
+ ENTITY_DEF("Cdot", 266, "\xc4\x8a"),
+ ENTITY_DEF("NotTildeTilde", 8777, "\xe2\x89\x89"),
+ ENTITY_DEF("boxul", 9496, "\xe2\x94\x98"),
+ ENTITY_DEF("planckh", 8462, "\xe2\x84\x8e"),
+ ENTITY_DEF("CapitalDifferentialD", 8517, "\xe2\x85\x85"),
+ ENTITY_DEF("boxDL", 9559, "\xe2\x95\x97"),
+ ENTITY_DEF("cupbrcap", 10824, "\xe2\xa9\x88"),
+ ENTITY_DEF("boxdL", 9557, "\xe2\x95\x95"),
+ ENTITY_DEF("supe", 8839, "\xe2\x8a\x87"),
+ ENTITY_DEF("nvlt", 60, "\x3c\xe2\x83\x92"),
+ ENTITY_DEF("par", 8741, "\xe2\x88\xa5"),
+ ENTITY_DEF("InvisibleComma", 8291, "\xe2\x81\xa3"),
+ ENTITY_DEF("ring", 730, "\xcb\x9a"),
+ ENTITY_DEF("nvap", 8781, "\xe2\x89\x8d\xe2\x83\x92"),
+ ENTITY_DEF("veeeq", 8794, "\xe2\x89\x9a"),
+ ENTITY_DEF("Hfr", 8460, "\xe2\x84\x8c"),
+ ENTITY_DEF("dstrok", 273, "\xc4\x91"),
+ ENTITY_DEF("gesles", 10900, "\xe2\xaa\x94"),
+ ENTITY_DEF("dash", 8208, "\xe2\x80\x90"),
+ ENTITY_DEF("SHcy", 1064, "\xd0\xa8"),
+ ENTITY_DEF("congdot", 10861, "\xe2\xa9\xad"),
+ ENTITY_DEF("imagline", 8464, "\xe2\x84\x90"),
+ ENTITY_DEF("ncy", 1085, "\xd0\xbd"),
+ ENTITY_DEF("bigstar", 9733, "\xe2\x98\x85"),
+ ENTITY_DEF_HEUR("REG", 174, "\xc2\xae"),
+ ENTITY_DEF("triangleq", 8796, "\xe2\x89\x9c"),
+ ENTITY_DEF("rsqb", 93, "\x5d"),
+ ENTITY_DEF("ddarr", 8650, "\xe2\x87\x8a"),
+ ENTITY_DEF("csub", 10959, "\xe2\xab\x8f"),
+ ENTITY_DEF("quest", 63, "\x3f"),
+ ENTITY_DEF("Star", 8902, "\xe2\x8b\x86"),
+ ENTITY_DEF_HEUR("LT", 60, "\x3c"),
+ ENTITY_DEF("ncong", 8775, "\xe2\x89\x87"),
+ ENTITY_DEF("prnE", 10933, "\xe2\xaa\xb5"),
+ ENTITY_DEF("bigtriangleup", 9651, "\xe2\x96\xb3"),
+ ENTITY_DEF("Tilde", 8764, "\xe2\x88\xbc"),
+ ENTITY_DEF("ltrif", 9666, "\xe2\x97\x82"),
+ ENTITY_DEF("ldrdhar", 10599, "\xe2\xa5\xa7"),
+ ENTITY_DEF("lcaron", 318, "\xc4\xbe"),
+ ENTITY_DEF("equivDD", 10872, "\xe2\xa9\xb8"),
+ ENTITY_DEF("lHar", 10594, "\xe2\xa5\xa2"),
+ ENTITY_DEF("vBar", 10984, "\xe2\xab\xa8"),
+ ENTITY_DEF("Mopf", 120132, "\xf0\x9d\x95\x84"),
+ ENTITY_DEF("LeftArrow", 8592, "\xe2\x86\x90"),
+ ENTITY_DEF("Rho", 929, "\xce\xa1"),
+ ENTITY_DEF("Ccirc", 264, "\xc4\x88"),
+ ENTITY_DEF("ifr", 120102, "\xf0\x9d\x94\xa6"),
+ ENTITY_DEF("cacute", 263, "\xc4\x87"),
+ ENTITY_DEF("centerdot", 183, "\xc2\xb7"),
+ ENTITY_DEF("dollar", 36, "\x24"),
+ ENTITY_DEF("lang", 10216, "\xe2\x9f\xa8"),
+ ENTITY_DEF("curvearrowright", 8631, "\xe2\x86\xb7"),
+ ENTITY_DEF("half", 189, "\xc2\xbd"),
+ ENTITY_DEF("Ecy", 1069, "\xd0\xad"),
+ ENTITY_DEF("rcub", 125, "\x7d"),
+ ENTITY_DEF("rcy", 1088, "\xd1\x80"),
+ ENTITY_DEF("isins", 8948, "\xe2\x8b\xb4"),
+ ENTITY_DEF("bsolhsub", 10184, "\xe2\x9f\x88"),
+ ENTITY_DEF("boxuL", 9563, "\xe2\x95\x9b"),
+ ENTITY_DEF("shchcy", 1097, "\xd1\x89"),
+ ENTITY_DEF("cwconint", 8754, "\xe2\x88\xb2"),
+ ENTITY_DEF("euro", 8364, "\xe2\x82\xac"),
+ ENTITY_DEF("lesseqqgtr", 10891, "\xe2\xaa\x8b"),
+ ENTITY_DEF("sim", 8764, "\xe2\x88\xbc"),
+ ENTITY_DEF("rarrc", 10547, "\xe2\xa4\xb3"),
+ ENTITY_DEF("boxdl", 9488, "\xe2\x94\x90"),
+ ENTITY_DEF("Epsilon", 917, "\xce\x95"),
+ ENTITY_DEF("iiiint", 10764, "\xe2\xa8\x8c"),
+ ENTITY_DEF("Rightarrow", 8658, "\xe2\x87\x92"),
+ ENTITY_DEF("conint", 8750, "\xe2\x88\xae"),
+ ENTITY_DEF("boxDl", 9558, "\xe2\x95\x96"),
+ ENTITY_DEF("kappav", 1008, "\xcf\xb0"),
+ ENTITY_DEF("profsurf", 8979, "\xe2\x8c\x93"),
+ ENTITY_DEF_HEUR("auml", 228, "\xc3\xa4"),
+ ENTITY_DEF("heartsuit", 9829, "\xe2\x99\xa5"),
+ ENTITY_DEF_HEUR("eacute", 233, "\xc3\xa9"),
+ ENTITY_DEF_HEUR("gt", 62, "\x3e"),
+ ENTITY_DEF("Gcedil", 290, "\xc4\xa2"),
+ ENTITY_DEF("easter", 10862, "\xe2\xa9\xae"),
+ ENTITY_DEF("Tcy", 1058, "\xd0\xa2"),
+ ENTITY_DEF("swarrow", 8601, "\xe2\x86\x99"),
+ ENTITY_DEF("lopf", 120157, "\xf0\x9d\x95\x9d"),
+ ENTITY_DEF("Agrave", 192, "\xc3\x80"),
+ ENTITY_DEF("Aring", 197, "\xc3\x85"),
+ ENTITY_DEF("fpartint", 10765, "\xe2\xa8\x8d"),
+ ENTITY_DEF("xoplus", 10753, "\xe2\xa8\x81"),
+ ENTITY_DEF("LeftDownTeeVector", 10593, "\xe2\xa5\xa1"),
+ ENTITY_DEF("int", 8747, "\xe2\x88\xab"),
+ ENTITY_DEF("Zeta", 918, "\xce\x96"),
+ ENTITY_DEF("loz", 9674, "\xe2\x97\x8a"),
+ ENTITY_DEF("ncup", 10818, "\xe2\xa9\x82"),
+ ENTITY_DEF("napE", 10864, "\xe2\xa9\xb0\xcc\xb8"),
+ ENTITY_DEF("csup", 10960, "\xe2\xab\x90"),
+ ENTITY_DEF("Ncedil", 325, "\xc5\x85"),
+ ENTITY_DEF("cuwed", 8911, "\xe2\x8b\x8f"),
+ ENTITY_DEF("Dot", 168, "\xc2\xa8"),
+ ENTITY_DEF("SquareIntersection", 8851, "\xe2\x8a\x93"),
+ ENTITY_DEF("map", 8614, "\xe2\x86\xa6"),
+ ENTITY_DEF_HEUR("aelig", 230, "\xc3\xa6"),
+ ENTITY_DEF("RightArrow", 8594, "\xe2\x86\x92"),
+ ENTITY_DEF("rightharpoondown", 8641, "\xe2\x87\x81"),
+ ENTITY_DEF("bNot", 10989, "\xe2\xab\xad"),
+ ENTITY_DEF("nsccue", 8929, "\xe2\x8b\xa1"),
+ ENTITY_DEF("zigrarr", 8669, "\xe2\x87\x9d"),
+ ENTITY_DEF("Sacute", 346, "\xc5\x9a"),
+ ENTITY_DEF("orv", 10843, "\xe2\xa9\x9b"),
+ ENTITY_DEF("RightVectorBar", 10579, "\xe2\xa5\x93"),
+ ENTITY_DEF("nrarrw", 8605, "\xe2\x86\x9d\xcc\xb8"),
+ ENTITY_DEF("nbump", 8782, "\xe2\x89\x8e\xcc\xb8"),
+ ENTITY_DEF_HEUR("iquest", 191, "\xc2\xbf"),
+ ENTITY_DEF("wr", 8768, "\xe2\x89\x80"),
+ ENTITY_DEF("UpArrow", 8593, "\xe2\x86\x91"),
+ ENTITY_DEF("notinva", 8713, "\xe2\x88\x89"),
+ ENTITY_DEF("ddagger", 8225, "\xe2\x80\xa1"),
+ ENTITY_DEF("nLeftarrow", 8653, "\xe2\x87\x8d"),
+ ENTITY_DEF("rbbrk", 10099, "\xe2\x9d\xb3"),
+ ENTITY_DEF("RightTriangle", 8883, "\xe2\x8a\xb3"),
+ ENTITY_DEF("leqq", 8806, "\xe2\x89\xa6"),
+ ENTITY_DEF("Vert", 8214, "\xe2\x80\x96"),
+ ENTITY_DEF("gesl", 8923, "\xe2\x8b\x9b\xef\xb8\x80"),
+ ENTITY_DEF("LeftTeeVector", 10586, "\xe2\xa5\x9a"),
+ ENTITY_DEF("Union", 8899, "\xe2\x8b\x83"),
+ ENTITY_DEF("sc", 8827, "\xe2\x89\xbb"),
+ ENTITY_DEF("ofr", 120108, "\xf0\x9d\x94\xac"),
+ ENTITY_DEF("quatint", 10774, "\xe2\xa8\x96"),
+ ENTITY_DEF("apacir", 10863, "\xe2\xa9\xaf"),
+ ENTITY_DEF("profalar", 9006, "\xe2\x8c\xae"),
+ ENTITY_DEF("subsetneq", 8842, "\xe2\x8a\x8a"),
+ ENTITY_DEF("Vvdash", 8874, "\xe2\x8a\xaa"),
+ ENTITY_DEF("ohbar", 10677, "\xe2\xa6\xb5"),
+ ENTITY_DEF("Gt", 8811, "\xe2\x89\xab"),
+ ENTITY_DEF("exist", 8707, "\xe2\x88\x83"),
+ ENTITY_DEF("gtrapprox", 10886, "\xe2\xaa\x86"),
+ ENTITY_DEF_HEUR("euml", 235, "\xc3\xab"),
+ ENTITY_DEF("Equilibrium", 8652, "\xe2\x87\x8c"),
+ ENTITY_DEF_HEUR("aacute", 225, "\xc3\xa1"),
+ ENTITY_DEF("omid", 10678, "\xe2\xa6\xb6"),
+ ENTITY_DEF("loarr", 8701, "\xe2\x87\xbd"),
+ ENTITY_DEF("SucceedsSlantEqual", 8829, "\xe2\x89\xbd"),
+ ENTITY_DEF("angsph", 8738, "\xe2\x88\xa2"),
+ ENTITY_DEF("nsmid", 8740, "\xe2\x88\xa4"),
+ ENTITY_DEF("lsquor", 8218, "\xe2\x80\x9a"),
+ ENTITY_DEF("cemptyv", 10674, "\xe2\xa6\xb2"),
+ ENTITY_DEF("rAarr", 8667, "\xe2\x87\x9b"),
+ ENTITY_DEF("searr", 8600, "\xe2\x86\x98"),
+ ENTITY_DEF("complexes", 8450, "\xe2\x84\x82"),
+ ENTITY_DEF("UnderParenthesis", 9181, "\xe2\x8f\x9d"),
+ ENTITY_DEF("nparsl", 11005, "\xe2\xab\xbd\xe2\x83\xa5"),
+ ENTITY_DEF("Lacute", 313, "\xc4\xb9"),
+ ENTITY_DEF_HEUR("deg", 176, "\xc2\xb0"),
+ ENTITY_DEF("Racute", 340, "\xc5\x94"),
+ ENTITY_DEF("Verbar", 8214, "\xe2\x80\x96"),
+ ENTITY_DEF("sqcups", 8852, "\xe2\x8a\x94\xef\xb8\x80"),
+ ENTITY_DEF("Hopf", 8461, "\xe2\x84\x8d"),
+ ENTITY_DEF("naturals", 8469, "\xe2\x84\x95"),
+ ENTITY_DEF("Cedilla", 184, "\xc2\xb8"),
+ ENTITY_DEF("exponentiale", 8519, "\xe2\x85\x87"),
+ ENTITY_DEF("vnsup", 8835, "\xe2\x8a\x83\xe2\x83\x92"),
+ ENTITY_DEF("leftrightarrows", 8646, "\xe2\x87\x86"),
+ ENTITY_DEF("Laplacetrf", 8466, "\xe2\x84\x92"),
+ ENTITY_DEF("vartriangleright", 8883, "\xe2\x8a\xb3"),
+ ENTITY_DEF("rtri", 9657, "\xe2\x96\xb9"),
+ ENTITY_DEF("gE", 8807, "\xe2\x89\xa7"),
+ ENTITY_DEF("SmallCircle", 8728, "\xe2\x88\x98"),
+ ENTITY_DEF("diamondsuit", 9830, "\xe2\x99\xa6"),
+ ENTITY_DEF_HEUR("Otilde", 213, "\xc3\x95"),
+ ENTITY_DEF("lneq", 10887, "\xe2\xaa\x87"),
+ ENTITY_DEF("lesdoto", 10881, "\xe2\xaa\x81"),
+ ENTITY_DEF("ltquest", 10875, "\xe2\xa9\xbb"),
+ ENTITY_DEF("thinsp", 8201, "\xe2\x80\x89"),
+ ENTITY_DEF("barwed", 8965, "\xe2\x8c\x85"),
+ ENTITY_DEF("elsdot", 10903, "\xe2\xaa\x97"),
+ ENTITY_DEF("circ", 710, "\xcb\x86"),
+ ENTITY_DEF("ni", 8715, "\xe2\x88\x8b"),
+ ENTITY_DEF("mlcp", 10971, "\xe2\xab\x9b"),
+ ENTITY_DEF("Vdash", 8873, "\xe2\x8a\xa9"),
+ ENTITY_DEF("ShortRightArrow", 8594, "\xe2\x86\x92"),
+ ENTITY_DEF("upharpoonleft", 8639, "\xe2\x86\xbf"),
+ ENTITY_DEF("UnderBracket", 9141, "\xe2\x8e\xb5"),
+ ENTITY_DEF("rAtail", 10524, "\xe2\xa4\x9c"),
+ ENTITY_DEF("iopf", 120154, "\xf0\x9d\x95\x9a"),
+ ENTITY_DEF("longleftarrow", 10229, "\xe2\x9f\xb5"),
+ ENTITY_DEF("Zacute", 377, "\xc5\xb9"),
+ ENTITY_DEF("duhar", 10607, "\xe2\xa5\xaf"),
+ ENTITY_DEF("Mfr", 120080, "\xf0\x9d\x94\x90"),
+ ENTITY_DEF("prnap", 10937, "\xe2\xaa\xb9"),
+ ENTITY_DEF("eqcirc", 8790, "\xe2\x89\x96"),
+ ENTITY_DEF("rarrlp", 8620, "\xe2\x86\xac"),
+ ENTITY_DEF("le", 8804, "\xe2\x89\xa4"),
+ ENTITY_DEF("Oscr", 119978, "\xf0\x9d\x92\xaa"),
+ ENTITY_DEF("langd", 10641, "\xe2\xa6\x91"),
+ ENTITY_DEF("Ucirc", 219, "\xc3\x9b"),
+ ENTITY_DEF("precnapprox", 10937, "\xe2\xaa\xb9"),
+ ENTITY_DEF("succcurlyeq", 8829, "\xe2\x89\xbd"),
+ ENTITY_DEF("Tau", 932, "\xce\xa4"),
+ ENTITY_DEF("larr", 8592, "\xe2\x86\x90"),
+ ENTITY_DEF("neArr", 8663, "\xe2\x87\x97"),
+ ENTITY_DEF("subsim", 10951, "\xe2\xab\x87"),
+ ENTITY_DEF("DScy", 1029, "\xd0\x85"),
+ ENTITY_DEF("preccurlyeq", 8828, "\xe2\x89\xbc"),
+ ENTITY_DEF("NotLessLess", 8810, "\xe2\x89\xaa\xcc\xb8"),
+ ENTITY_DEF("succnapprox", 10938, "\xe2\xaa\xba"),
+ ENTITY_DEF("prcue", 8828, "\xe2\x89\xbc"),
+ ENTITY_DEF("Downarrow", 8659, "\xe2\x87\x93"),
+ ENTITY_DEF("angmsdah", 10671, "\xe2\xa6\xaf"),
+ ENTITY_DEF("Emacr", 274, "\xc4\x92"),
+ ENTITY_DEF("lsh", 8624, "\xe2\x86\xb0"),
+ ENTITY_DEF("simne", 8774, "\xe2\x89\x86"),
+ ENTITY_DEF("Bumpeq", 8782, "\xe2\x89\x8e"),
+ ENTITY_DEF("RightUpTeeVector", 10588, "\xe2\xa5\x9c"),
+ ENTITY_DEF("Sigma", 931, "\xce\xa3"),
+ ENTITY_DEF("nvltrie", 8884, "\xe2\x8a\xb4\xe2\x83\x92"),
+ ENTITY_DEF("lfr", 120105, "\xf0\x9d\x94\xa9"),
+ ENTITY_DEF("emsp13", 8196, "\xe2\x80\x84"),
+ ENTITY_DEF("parsl", 11005, "\xe2\xab\xbd"),
+ ENTITY_DEF_HEUR("ucirc", 251, "\xc3\xbb"),
+ ENTITY_DEF("gsiml", 10896, "\xe2\xaa\x90"),
+ ENTITY_DEF("xsqcup", 10758, "\xe2\xa8\x86"),
+ ENTITY_DEF("Omicron", 927, "\xce\x9f"),
+ ENTITY_DEF("gsime", 10894, "\xe2\xaa\x8e"),
+ ENTITY_DEF("circlearrowleft", 8634, "\xe2\x86\xba"),
+ ENTITY_DEF("sqsupe", 8850, "\xe2\x8a\x92"),
+ ENTITY_DEF("supE", 10950, "\xe2\xab\x86"),
+ ENTITY_DEF("dlcrop", 8973, "\xe2\x8c\x8d"),
+ ENTITY_DEF("RightDownTeeVector", 10589, "\xe2\xa5\x9d"),
+ ENTITY_DEF("Colone", 10868, "\xe2\xa9\xb4"),
+ ENTITY_DEF("awconint", 8755, "\xe2\x88\xb3"),
+ ENTITY_DEF("smte", 10924, "\xe2\xaa\xac"),
+ ENTITY_DEF("lEg", 10891, "\xe2\xaa\x8b"),
+ ENTITY_DEF("circledast", 8859, "\xe2\x8a\x9b"),
+ ENTITY_DEF("ecolon", 8789, "\xe2\x89\x95"),
+ ENTITY_DEF("rect", 9645, "\xe2\x96\xad"),
+ ENTITY_DEF("Equal", 10869, "\xe2\xa9\xb5"),
+ ENTITY_DEF("nwnear", 10535, "\xe2\xa4\xa7"),
+ ENTITY_DEF("capdot", 10816, "\xe2\xa9\x80"),
+ ENTITY_DEF("straightphi", 981, "\xcf\x95"),
+ ENTITY_DEF("forkv", 10969, "\xe2\xab\x99"),
+ ENTITY_DEF("ZHcy", 1046, "\xd0\x96"),
+ ENTITY_DEF("Element", 8712, "\xe2\x88\x88"),
+ ENTITY_DEF("rthree", 8908, "\xe2\x8b\x8c"),
+ ENTITY_DEF("vzigzag", 10650, "\xe2\xa6\x9a"),
+ ENTITY_DEF("hybull", 8259, "\xe2\x81\x83"),
+ ENTITY_DEF("intprod", 10812, "\xe2\xa8\xbc"),
+ ENTITY_DEF("HumpEqual", 8783, "\xe2\x89\x8f"),
+ ENTITY_DEF("bigsqcup", 10758, "\xe2\xa8\x86"),
+ ENTITY_DEF("mp", 8723, "\xe2\x88\x93"),
+ ENTITY_DEF("lescc", 10920, "\xe2\xaa\xa8"),
+ ENTITY_DEF("NotPrecedes", 8832, "\xe2\x8a\x80"),
+ ENTITY_DEF("wedge", 8743, "\xe2\x88\xa7"),
+ ENTITY_DEF("Supset", 8913, "\xe2\x8b\x91"),
+ ENTITY_DEF("pm", 177, "\xc2\xb1"),
+ ENTITY_DEF("kfr", 120104, "\xf0\x9d\x94\xa8"),
+ ENTITY_DEF("ufisht", 10622, "\xe2\xa5\xbe"),
+ ENTITY_DEF("ecaron", 283, "\xc4\x9b"),
+ ENTITY_DEF("chcy", 1095, "\xd1\x87"),
+ ENTITY_DEF("Esim", 10867, "\xe2\xa9\xb3"),
+ ENTITY_DEF("fltns", 9649, "\xe2\x96\xb1"),
+ ENTITY_DEF("nsce", 10928, "\xe2\xaa\xb0\xcc\xb8"),
+ ENTITY_DEF("hookrightarrow", 8618, "\xe2\x86\xaa"),
+ ENTITY_DEF("semi", 59, "\x3b"),
+ ENTITY_DEF("ges", 10878, "\xe2\xa9\xbe"),
+ ENTITY_DEF("approxeq", 8778, "\xe2\x89\x8a"),
+ ENTITY_DEF("rarrsim", 10612, "\xe2\xa5\xb4"),
+ ENTITY_DEF("boxhD", 9573, "\xe2\x95\xa5"),
+ ENTITY_DEF("varpi", 982, "\xcf\x96"),
+ ENTITY_DEF("larrb", 8676, "\xe2\x87\xa4"),
+ ENTITY_DEF("copf", 120148, "\xf0\x9d\x95\x94"),
+ ENTITY_DEF("Dopf", 120123, "\xf0\x9d\x94\xbb"),
+ ENTITY_DEF("LeftVector", 8636, "\xe2\x86\xbc"),
+ ENTITY_DEF("iff", 8660, "\xe2\x87\x94"),
+ ENTITY_DEF("lnap", 10889, "\xe2\xaa\x89"),
+ ENTITY_DEF("NotGreaterFullEqual", 8807, "\xe2\x89\xa7\xcc\xb8"),
+ ENTITY_DEF("varrho", 1009, "\xcf\xb1"),
+ ENTITY_DEF("NotSucceeds", 8833, "\xe2\x8a\x81"),
+ ENTITY_DEF("ltrPar", 10646, "\xe2\xa6\x96"),
+ ENTITY_DEF("nlE", 8806, "\xe2\x89\xa6\xcc\xb8"),
+ ENTITY_DEF("Zfr", 8488, "\xe2\x84\xa8"),
+ ENTITY_DEF("LeftArrowBar", 8676, "\xe2\x87\xa4"),
+ ENTITY_DEF("boxplus", 8862, "\xe2\x8a\x9e"),
+ ENTITY_DEF("sqsube", 8849, "\xe2\x8a\x91"),
+ ENTITY_DEF("Re", 8476, "\xe2\x84\x9c"),
+ ENTITY_DEF("Wfr", 120090, "\xf0\x9d\x94\x9a"),
+ ENTITY_DEF("epsi", 949, "\xce\xb5"),
+ ENTITY_DEF("oacute", 243, "\xc3\xb3"),
+ ENTITY_DEF("bdquo", 8222, "\xe2\x80\x9e"),
+ ENTITY_DEF("wscr", 120012, "\xf0\x9d\x93\x8c"),
+ ENTITY_DEF("bullet", 8226, "\xe2\x80\xa2"),
+ ENTITY_DEF("frown", 8994, "\xe2\x8c\xa2"),
+ ENTITY_DEF("siml", 10909, "\xe2\xaa\x9d"),
+ ENTITY_DEF("Rarr", 8608, "\xe2\x86\xa0"),
+ ENTITY_DEF("Scaron", 352, "\xc5\xa0"),
+ ENTITY_DEF("gtreqqless", 10892, "\xe2\xaa\x8c"),
+ ENTITY_DEF("Larr", 8606, "\xe2\x86\x9e"),
+ ENTITY_DEF("notniva", 8716, "\xe2\x88\x8c"),
+ ENTITY_DEF("gg", 8811, "\xe2\x89\xab"),
+ ENTITY_DEF("phmmat", 8499, "\xe2\x84\xb3"),
+ ENTITY_DEF("boxVL", 9571, "\xe2\x95\xa3"),
+ ENTITY_DEF("sigmav", 962, "\xcf\x82"),
+ ENTITY_DEF("order", 8500, "\xe2\x84\xb4"),
+ ENTITY_DEF("subsup", 10963, "\xe2\xab\x93"),
+ ENTITY_DEF("afr", 120094, "\xf0\x9d\x94\x9e"),
+ ENTITY_DEF("lbrace", 123, "\x7b"),
+ ENTITY_DEF("urcorn", 8989, "\xe2\x8c\x9d"),
+ ENTITY_DEF("Im", 8465, "\xe2\x84\x91"),
+ ENTITY_DEF("CounterClockwiseContourIntegral", 8755, "\xe2\x88\xb3"),
+ ENTITY_DEF("lne", 10887, "\xe2\xaa\x87"),
+ ENTITY_DEF("chi", 967, "\xcf\x87"),
+ ENTITY_DEF("cudarrl", 10552, "\xe2\xa4\xb8"),
+ ENTITY_DEF("ang", 8736, "\xe2\x88\xa0"),
+ ENTITY_DEF("isindot", 8949, "\xe2\x8b\xb5"),
+ ENTITY_DEF("Lfr", 120079, "\xf0\x9d\x94\x8f"),
+ ENTITY_DEF("Rsh", 8625, "\xe2\x86\xb1"),
+ ENTITY_DEF("Ocy", 1054, "\xd0\x9e"),
+ ENTITY_DEF("nvrArr", 10499, "\xe2\xa4\x83"),
+ ENTITY_DEF("otimes", 8855, "\xe2\x8a\x97"),
+ ENTITY_DEF("eqslantgtr", 10902, "\xe2\xaa\x96"),
+ ENTITY_DEF("Rfr", 8476, "\xe2\x84\x9c"),
+ ENTITY_DEF("blacktriangleleft", 9666, "\xe2\x97\x82"),
+ ENTITY_DEF("Lsh", 8624, "\xe2\x86\xb0"),
+ ENTITY_DEF("boxvr", 9500, "\xe2\x94\x9c"),
+ ENTITY_DEF("scedil", 351, "\xc5\x9f"),
+ ENTITY_DEF_HEUR("iuml", 239, "\xc3\xaf"),
+ ENTITY_DEF("NJcy", 1034, "\xd0\x8a"),
+ ENTITY_DEF("Dagger", 8225, "\xe2\x80\xa1"),
+ ENTITY_DEF("rarrap", 10613, "\xe2\xa5\xb5"),
+ ENTITY_DEF("udblac", 369, "\xc5\xb1"),
+ ENTITY_DEF("Sopf", 120138, "\xf0\x9d\x95\x8a"),
+ ENTITY_DEF("scnsim", 8937, "\xe2\x8b\xa9"),
+ ENTITY_DEF("hbar", 8463, "\xe2\x84\x8f"),
+ ENTITY_DEF("frac15", 8533, "\xe2\x85\x95"),
+ ENTITY_DEF_HEUR("sup3", 179, "\xc2\xb3"),
+ ENTITY_DEF("NegativeThickSpace", 8203, "\xe2\x80\x8b"),
+ ENTITY_DEF("npr", 8832, "\xe2\x8a\x80"),
+ ENTITY_DEF("doteq", 8784, "\xe2\x89\x90"),
+ ENTITY_DEF("subrarr", 10617, "\xe2\xa5\xb9"),
+ ENTITY_DEF("SquareSubset", 8847, "\xe2\x8a\x8f"),
+ ENTITY_DEF("vprop", 8733, "\xe2\x88\x9d"),
+ ENTITY_DEF("OpenCurlyQuote", 8216, "\xe2\x80\x98"),
+ ENTITY_DEF("supseteq", 8839, "\xe2\x8a\x87"),
+ ENTITY_DEF("nRightarrow", 8655, "\xe2\x87\x8f"),
+ ENTITY_DEF("Longleftarrow", 10232, "\xe2\x9f\xb8"),
+ ENTITY_DEF("lsquo", 8216, "\xe2\x80\x98"),
+ ENTITY_DEF("hstrok", 295, "\xc4\xa7"),
+ ENTITY_DEF("NotTilde", 8769, "\xe2\x89\x81"),
+ ENTITY_DEF("ogt", 10689, "\xe2\xa7\x81"),
+ ENTITY_DEF("block", 9608, "\xe2\x96\x88"),
+ ENTITY_DEF("minusd", 8760, "\xe2\x88\xb8"),
+ ENTITY_DEF("esdot", 8784, "\xe2\x89\x90"),
+ ENTITY_DEF("nsim", 8769, "\xe2\x89\x81"),
+ ENTITY_DEF("scsim", 8831, "\xe2\x89\xbf"),
+ ENTITY_DEF("boxVl", 9570, "\xe2\x95\xa2"),
+ ENTITY_DEF("ltimes", 8905, "\xe2\x8b\x89"),
+ ENTITY_DEF("thkap", 8776, "\xe2\x89\x88"),
+ ENTITY_DEF("vnsub", 8834, "\xe2\x8a\x82\xe2\x83\x92"),
+ ENTITY_DEF("thetasym", 977, "\xcf\x91"),
+ ENTITY_DEF("eopf", 120150, "\xf0\x9d\x95\x96"),
+ ENTITY_DEF("image", 8465, "\xe2\x84\x91"),
+ ENTITY_DEF("doteqdot", 8785, "\xe2\x89\x91"),
+ ENTITY_DEF("Udblac", 368, "\xc5\xb0"),
+ ENTITY_DEF("gnsim", 8935, "\xe2\x8b\xa7"),
+ ENTITY_DEF("yicy", 1111, "\xd1\x97"),
+ ENTITY_DEF("vopf", 120167, "\xf0\x9d\x95\xa7"),
+ ENTITY_DEF("DDotrahd", 10513, "\xe2\xa4\x91"),
+ ENTITY_DEF("Iota", 921, "\xce\x99"),
+ ENTITY_DEF("GJcy", 1027, "\xd0\x83"),
+ ENTITY_DEF("rightthreetimes", 8908, "\xe2\x8b\x8c"),
+ ENTITY_DEF("nrtri", 8939, "\xe2\x8b\xab"),
+ ENTITY_DEF("TildeFullEqual", 8773, "\xe2\x89\x85"),
+ ENTITY_DEF("Dcaron", 270, "\xc4\x8e"),
+ ENTITY_DEF("ccaron", 269, "\xc4\x8d"),
+ ENTITY_DEF("lacute", 314, "\xc4\xba"),
+ ENTITY_DEF("VerticalBar", 8739, "\xe2\x88\xa3"),
+ ENTITY_DEF("Igrave", 204, "\xc3\x8c"),
+ ENTITY_DEF("boxH", 9552, "\xe2\x95\x90"),
+ ENTITY_DEF("Pfr", 120083, "\xf0\x9d\x94\x93"),
+ ENTITY_DEF("equals", 61, "\x3d"),
+ ENTITY_DEF("rbrack", 93, "\x5d"),
+ ENTITY_DEF("OverParenthesis", 9180, "\xe2\x8f\x9c"),
+ ENTITY_DEF("in", 8712, "\xe2\x88\x88"),
+ ENTITY_DEF("llcorner", 8990, "\xe2\x8c\x9e"),
+ ENTITY_DEF("mcomma", 10793, "\xe2\xa8\xa9"),
+ ENTITY_DEF("NotGreater", 8815, "\xe2\x89\xaf"),
+ ENTITY_DEF("midcir", 10992, "\xe2\xab\xb0"),
+ ENTITY_DEF("Edot", 278, "\xc4\x96"),
+ ENTITY_DEF("oplus", 8853, "\xe2\x8a\x95"),
+ ENTITY_DEF("geqq", 8807, "\xe2\x89\xa7"),
+ ENTITY_DEF("curvearrowleft", 8630, "\xe2\x86\xb6"),
+ ENTITY_DEF("Poincareplane", 8460, "\xe2\x84\x8c"),
+ ENTITY_DEF("yscr", 120014, "\xf0\x9d\x93\x8e"),
+ ENTITY_DEF("ccaps", 10829, "\xe2\xa9\x8d"),
+ ENTITY_DEF("rpargt", 10644, "\xe2\xa6\x94"),
+ ENTITY_DEF("topfork", 10970, "\xe2\xab\x9a"),
+ ENTITY_DEF("Gamma", 915, "\xce\x93"),
+ ENTITY_DEF("umacr", 363, "\xc5\xab"),
+ ENTITY_DEF("frac13", 8531, "\xe2\x85\x93"),
+ ENTITY_DEF("cirfnint", 10768, "\xe2\xa8\x90"),
+ ENTITY_DEF("xlArr", 10232, "\xe2\x9f\xb8"),
+ ENTITY_DEF("digamma", 989, "\xcf\x9d"),
+ ENTITY_DEF("Hat", 94, "\x5e"),
+ ENTITY_DEF("lates", 10925, "\xe2\xaa\xad\xef\xb8\x80"),
+ ENTITY_DEF("lgE", 10897, "\xe2\xaa\x91"),
+ ENTITY_DEF("commat", 64, "\x40"),
+ ENTITY_DEF("NotPrecedesSlantEqual", 8928, "\xe2\x8b\xa0"),
+ ENTITY_DEF("phone", 9742, "\xe2\x98\x8e"),
+ ENTITY_DEF("Ecirc", 202, "\xc3\x8a"),
+ ENTITY_DEF_HEUR("lt", 60, "\x3c"),
+ ENTITY_DEF("intcal", 8890, "\xe2\x8a\xba"),
+ ENTITY_DEF("xdtri", 9661, "\xe2\x96\xbd"),
+ ENTITY_DEF("Abreve", 258, "\xc4\x82"),
+ ENTITY_DEF("gopf", 120152, "\xf0\x9d\x95\x98"),
+ ENTITY_DEF("Xopf", 120143, "\xf0\x9d\x95\x8f"),
+ ENTITY_DEF("Iacute", 205, "\xc3\x8d"),
+ ENTITY_DEF("Aopf", 120120, "\xf0\x9d\x94\xb8"),
+ ENTITY_DEF("gbreve", 287, "\xc4\x9f"),
+ ENTITY_DEF("nleq", 8816, "\xe2\x89\xb0"),
+ ENTITY_DEF("xopf", 120169, "\xf0\x9d\x95\xa9"),
+ ENTITY_DEF("SquareSupersetEqual", 8850, "\xe2\x8a\x92"),
+ ENTITY_DEF("NotLessTilde", 8820, "\xe2\x89\xb4"),
+ ENTITY_DEF("SubsetEqual", 8838, "\xe2\x8a\x86"),
+ ENTITY_DEF("Sc", 10940, "\xe2\xaa\xbc"),
+ ENTITY_DEF("sdote", 10854, "\xe2\xa9\xa6"),
+ ENTITY_DEF("loplus", 10797, "\xe2\xa8\xad"),
+ ENTITY_DEF("zfr", 120119, "\xf0\x9d\x94\xb7"),
+ ENTITY_DEF("subseteqq", 10949, "\xe2\xab\x85"),
+ ENTITY_DEF("Vdashl", 10982, "\xe2\xab\xa6"),
+ ENTITY_DEF("integers", 8484, "\xe2\x84\xa4"),
+ ENTITY_DEF("Umacr", 362, "\xc5\xaa"),
+ ENTITY_DEF("dopf", 120149, "\xf0\x9d\x95\x95"),
+ ENTITY_DEF("RightDownVectorBar", 10581, "\xe2\xa5\x95"),
+ ENTITY_DEF("angmsdaf", 10669, "\xe2\xa6\xad"),
+ ENTITY_DEF("Jfr", 120077, "\xf0\x9d\x94\x8d"),
+ ENTITY_DEF("bernou", 8492, "\xe2\x84\xac"),
+ ENTITY_DEF("lceil", 8968, "\xe2\x8c\x88"),
+ ENTITY_DEF("nvsim", 8764, "\xe2\x88\xbc\xe2\x83\x92"),
+ ENTITY_DEF("NotSucceedsSlantEqual", 8929, "\xe2\x8b\xa1"),
+ ENTITY_DEF("hearts", 9829, "\xe2\x99\xa5"),
+ ENTITY_DEF("vee", 8744, "\xe2\x88\xa8"),
+ ENTITY_DEF("LJcy", 1033, "\xd0\x89"),
+ ENTITY_DEF("nlt", 8814, "\xe2\x89\xae"),
+ ENTITY_DEF("because", 8757, "\xe2\x88\xb5"),
+ ENTITY_DEF("hairsp", 8202, "\xe2\x80\x8a"),
+ ENTITY_DEF("comma", 44, "\x2c"),
+ ENTITY_DEF("iecy", 1077, "\xd0\xb5"),
+ ENTITY_DEF("npre", 10927, "\xe2\xaa\xaf\xcc\xb8"),
+ ENTITY_DEF("NotSquareSubset", 8847, "\xe2\x8a\x8f\xcc\xb8"),
+ ENTITY_DEF("mscr", 120002, "\xf0\x9d\x93\x82"),
+ ENTITY_DEF("jopf", 120155, "\xf0\x9d\x95\x9b"),
+ ENTITY_DEF("bumpE", 10926, "\xe2\xaa\xae"),
+ ENTITY_DEF("thicksim", 8764, "\xe2\x88\xbc"),
+ ENTITY_DEF("Nfr", 120081, "\xf0\x9d\x94\x91"),
+ ENTITY_DEF("yucy", 1102, "\xd1\x8e"),
+ ENTITY_DEF("notinvc", 8950, "\xe2\x8b\xb6"),
+ ENTITY_DEF("lstrok", 322, "\xc5\x82"),
+ ENTITY_DEF("robrk", 10215, "\xe2\x9f\xa7"),
+ ENTITY_DEF("LeftTriangleBar", 10703, "\xe2\xa7\x8f"),
+ ENTITY_DEF("hksearow", 10533, "\xe2\xa4\xa5"),
+ ENTITY_DEF("bigcap", 8898, "\xe2\x8b\x82"),
+ ENTITY_DEF("udhar", 10606, "\xe2\xa5\xae"),
+ ENTITY_DEF("Yscr", 119988, "\xf0\x9d\x92\xb4"),
+ ENTITY_DEF("smeparsl", 10724, "\xe2\xa7\xa4"),
+ ENTITY_DEF("NotLess", 8814, "\xe2\x89\xae"),
+ ENTITY_DEF("dcaron", 271, "\xc4\x8f"),
+ ENTITY_DEF("ange", 10660, "\xe2\xa6\xa4"),
+ ENTITY_DEF("dHar", 10597, "\xe2\xa5\xa5"),
+ ENTITY_DEF("UpperRightArrow", 8599, "\xe2\x86\x97"),
+ ENTITY_DEF("trpezium", 9186, "\xe2\x8f\xa2"),
+ ENTITY_DEF("boxminus", 8863, "\xe2\x8a\x9f"),
+ ENTITY_DEF("notni", 8716, "\xe2\x88\x8c"),
+ ENTITY_DEF("dtrif", 9662, "\xe2\x96\xbe"),
+ ENTITY_DEF("nhArr", 8654, "\xe2\x87\x8e"),
+ ENTITY_DEF("larrpl", 10553, "\xe2\xa4\xb9"),
+ ENTITY_DEF("simeq", 8771, "\xe2\x89\x83"),
+ ENTITY_DEF("geqslant", 10878, "\xe2\xa9\xbe"),
+ ENTITY_DEF("RightUpVectorBar", 10580, "\xe2\xa5\x94"),
+ ENTITY_DEF("nsc", 8833, "\xe2\x8a\x81"),
+ ENTITY_DEF("div", 247, "\xc3\xb7"),
+ ENTITY_DEF("orslope", 10839, "\xe2\xa9\x97"),
+ ENTITY_DEF("lparlt", 10643, "\xe2\xa6\x93"),
+ ENTITY_DEF("trie", 8796, "\xe2\x89\x9c"),
+ ENTITY_DEF("cirmid", 10991, "\xe2\xab\xaf"),
+ ENTITY_DEF("wp", 8472, "\xe2\x84\x98"),
+ ENTITY_DEF("dagger", 8224, "\xe2\x80\xa0"),
+ ENTITY_DEF("utri", 9653, "\xe2\x96\xb5"),
+ ENTITY_DEF("supnE", 10956, "\xe2\xab\x8c"),
+ ENTITY_DEF("eg", 10906, "\xe2\xaa\x9a"),
+ ENTITY_DEF("LeftDownVector", 8643, "\xe2\x87\x83"),
+ ENTITY_DEF("NotLessEqual", 8816, "\xe2\x89\xb0"),
+ ENTITY_DEF("Bopf", 120121, "\xf0\x9d\x94\xb9"),
+ ENTITY_DEF("LongLeftRightArrow", 10231, "\xe2\x9f\xb7"),
+ ENTITY_DEF("Gfr", 120074, "\xf0\x9d\x94\x8a"),
+ ENTITY_DEF("sqsubseteq", 8849, "\xe2\x8a\x91"),
+ ENTITY_DEF_HEUR("ograve", 242, "\xc3\xb2"),
+ ENTITY_DEF("larrhk", 8617, "\xe2\x86\xa9"),
+ ENTITY_DEF("sigma", 963, "\xcf\x83"),
+ ENTITY_DEF("NotSquareSupersetEqual", 8931, "\xe2\x8b\xa3"),
+ ENTITY_DEF("gvnE", 8809, "\xe2\x89\xa9\xef\xb8\x80"),
+ ENTITY_DEF("timesbar", 10801, "\xe2\xa8\xb1"),
+ ENTITY_DEF("Iukcy", 1030, "\xd0\x86"),
+ ENTITY_DEF("bscr", 119991, "\xf0\x9d\x92\xb7"),
+ ENTITY_DEF("Exists", 8707, "\xe2\x88\x83"),
+ ENTITY_DEF("tscr", 120009, "\xf0\x9d\x93\x89"),
+ ENTITY_DEF("tcy", 1090, "\xd1\x82"),
+ ENTITY_DEF("nwarr", 8598, "\xe2\x86\x96"),
+ ENTITY_DEF("hoarr", 8703, "\xe2\x87\xbf"),
+ ENTITY_DEF("lnapprox", 10889, "\xe2\xaa\x89"),
+ ENTITY_DEF("nu", 957, "\xce\xbd"),
+ ENTITY_DEF("bcy", 1073, "\xd0\xb1"),
+ ENTITY_DEF("ndash", 8211, "\xe2\x80\x93"),
+ ENTITY_DEF("smt", 10922, "\xe2\xaa\xaa"),
+ ENTITY_DEF("scaron", 353, "\xc5\xa1"),
+ ENTITY_DEF("IOcy", 1025, "\xd0\x81"),
+ ENTITY_DEF("Ifr", 8465, "\xe2\x84\x91"),
+ ENTITY_DEF("cularrp", 10557, "\xe2\xa4\xbd"),
+ ENTITY_DEF("lvertneqq", 8808, "\xe2\x89\xa8\xef\xb8\x80"),
+ ENTITY_DEF("nlarr", 8602, "\xe2\x86\x9a"),
+ ENTITY_DEF("colon", 58, "\x3a"),
+ ENTITY_DEF("ddotseq", 10871, "\xe2\xa9\xb7"),
+ ENTITY_DEF("zacute", 378, "\xc5\xba"),
+ ENTITY_DEF("DoubleVerticalBar", 8741, "\xe2\x88\xa5"),
+ ENTITY_DEF("larrfs", 10525, "\xe2\xa4\x9d"),
+ ENTITY_DEF("NotExists", 8708, "\xe2\x88\x84"),
+ ENTITY_DEF("geq", 8805, "\xe2\x89\xa5"),
+ ENTITY_DEF("Ffr", 120073, "\xf0\x9d\x94\x89"),
+ ENTITY_DEF_HEUR("divide", 247, "\xc3\xb7"),
+ ENTITY_DEF("blank", 9251, "\xe2\x90\xa3"),
+ ENTITY_DEF("IEcy", 1045, "\xd0\x95"),
+ ENTITY_DEF_HEUR("ordm", 186, "\xc2\xba"),
+ ENTITY_DEF("fopf", 120151, "\xf0\x9d\x95\x97"),
+ ENTITY_DEF("ecir", 8790, "\xe2\x89\x96"),
+ ENTITY_DEF("complement", 8705, "\xe2\x88\x81"),
+ ENTITY_DEF("top", 8868, "\xe2\x8a\xa4"),
+ ENTITY_DEF("DoubleContourIntegral", 8751, "\xe2\x88\xaf"),
+ ENTITY_DEF("nisd", 8954, "\xe2\x8b\xba"),
+ ENTITY_DEF("bcong", 8780, "\xe2\x89\x8c"),
+ ENTITY_DEF("plusdu", 10789, "\xe2\xa8\xa5"),
+ ENTITY_DEF("TildeTilde", 8776, "\xe2\x89\x88"),
+ ENTITY_DEF("lnE", 8808, "\xe2\x89\xa8"),
+ ENTITY_DEF("DoubleLongRightArrow", 10233, "\xe2\x9f\xb9"),
+ ENTITY_DEF("nsubseteqq", 10949, "\xe2\xab\x85\xcc\xb8"),
+ ENTITY_DEF("DownTeeArrow", 8615, "\xe2\x86\xa7"),
+ ENTITY_DEF("Cscr", 119966, "\xf0\x9d\x92\x9e"),
+ ENTITY_DEF("NegativeVeryThinSpace", 8203, "\xe2\x80\x8b"),
+ ENTITY_DEF("emsp", 8195, "\xe2\x80\x83"),
+ ENTITY_DEF("vartriangleleft", 8882, "\xe2\x8a\xb2"),
+ ENTITY_DEF("ropar", 10630, "\xe2\xa6\x86"),
+ ENTITY_DEF("checkmark", 10003, "\xe2\x9c\x93"),
+ ENTITY_DEF("Ycy", 1067, "\xd0\xab"),
+ ENTITY_DEF("supset", 8835, "\xe2\x8a\x83"),
+ ENTITY_DEF("gneqq", 8809, "\xe2\x89\xa9"),
+ ENTITY_DEF("Lstrok", 321, "\xc5\x81"),
+ ENTITY_DEF_HEUR("AMP", 38, "\x26"),
+ ENTITY_DEF("acE", 8766, "\xe2\x88\xbe\xcc\xb3"),
+ ENTITY_DEF("sqsupseteq", 8850, "\xe2\x8a\x92"),
+ ENTITY_DEF("nle", 8816, "\xe2\x89\xb0"),
+ ENTITY_DEF("nesear", 10536, "\xe2\xa4\xa8"),
+ ENTITY_DEF("LeftDownVectorBar", 10585, "\xe2\xa5\x99"),
+ ENTITY_DEF("Integral", 8747, "\xe2\x88\xab"),
+ ENTITY_DEF("Beta", 914, "\xce\x92"),
+ ENTITY_DEF("nvdash", 8876, "\xe2\x8a\xac"),
+ ENTITY_DEF("nges", 10878, "\xe2\xa9\xbe\xcc\xb8"),
+ ENTITY_DEF("demptyv", 10673, "\xe2\xa6\xb1"),
+ ENTITY_DEF("eta", 951, "\xce\xb7"),
+ ENTITY_DEF("GreaterSlantEqual", 10878, "\xe2\xa9\xbe"),
+ ENTITY_DEF_HEUR("ccedil", 231, "\xc3\xa7"),
+ ENTITY_DEF("pfr", 120109, "\xf0\x9d\x94\xad"),
+ ENTITY_DEF("bbrktbrk", 9142, "\xe2\x8e\xb6"),
+ ENTITY_DEF("mcy", 1084, "\xd0\xbc"),
+ ENTITY_DEF("Not", 10988, "\xe2\xab\xac"),
+ ENTITY_DEF("qscr", 120006, "\xf0\x9d\x93\x86"),
+ ENTITY_DEF("zwj", 8205, "\xe2\x80\x8d"),
+ ENTITY_DEF("ntrianglerighteq", 8941, "\xe2\x8b\xad"),
+ ENTITY_DEF("permil", 8240, "\xe2\x80\xb0"),
+ ENTITY_DEF("squarf", 9642, "\xe2\x96\xaa"),
+ ENTITY_DEF("apos", 39, "\x27"),
+ ENTITY_DEF("lrm", 8206, "\xe2\x80\x8e"),
+ ENTITY_DEF("male", 9794, "\xe2\x99\x82"),
+ ENTITY_DEF_HEUR("agrave", 224, "\xc3\xa0"),
+ ENTITY_DEF("Lt", 8810, "\xe2\x89\xaa"),
+ ENTITY_DEF("capand", 10820, "\xe2\xa9\x84"),
+ ENTITY_DEF_HEUR("aring", 229, "\xc3\xa5"),
+ ENTITY_DEF("Jukcy", 1028, "\xd0\x84"),
+ ENTITY_DEF("bumpe", 8783, "\xe2\x89\x8f"),
+ ENTITY_DEF("dd", 8518, "\xe2\x85\x86"),
+ ENTITY_DEF("tscy", 1094, "\xd1\x86"),
+ ENTITY_DEF("oS", 9416, "\xe2\x93\x88"),
+ ENTITY_DEF("succeq", 10928, "\xe2\xaa\xb0"),
+ ENTITY_DEF("xharr", 10231, "\xe2\x9f\xb7"),
+ ENTITY_DEF("pluse", 10866, "\xe2\xa9\xb2"),
+ ENTITY_DEF("rfisht", 10621, "\xe2\xa5\xbd"),
+ ENTITY_DEF("HorizontalLine", 9472, "\xe2\x94\x80"),
+ ENTITY_DEF("DiacriticalAcute", 180, "\xc2\xb4"),
+ ENTITY_DEF("hfr", 120101, "\xf0\x9d\x94\xa5"),
+ ENTITY_DEF("preceq", 10927, "\xe2\xaa\xaf"),
+ ENTITY_DEF("rationals", 8474, "\xe2\x84\x9a"),
+ ENTITY_DEF_HEUR("Auml", 196, "\xc3\x84"),
+ ENTITY_DEF("LeftRightArrow", 8596, "\xe2\x86\x94"),
+ ENTITY_DEF("blacktriangleright", 9656, "\xe2\x96\xb8"),
+ ENTITY_DEF("dharr", 8642, "\xe2\x87\x82"),
+ ENTITY_DEF("isin", 8712, "\xe2\x88\x88"),
+ ENTITY_DEF("ldrushar", 10571, "\xe2\xa5\x8b"),
+ ENTITY_DEF("squ", 9633, "\xe2\x96\xa1"),
+ ENTITY_DEF("rbrksld", 10638, "\xe2\xa6\x8e"),
+ ENTITY_DEF("bigwedge", 8896, "\xe2\x8b\x80"),
+ ENTITY_DEF("swArr", 8665, "\xe2\x87\x99"),
+ ENTITY_DEF("IJlig", 306, "\xc4\xb2"),
+ ENTITY_DEF("harr", 8596, "\xe2\x86\x94"),
+ ENTITY_DEF("range", 10661, "\xe2\xa6\xa5"),
+ ENTITY_DEF("urtri", 9721, "\xe2\x97\xb9"),
+ ENTITY_DEF("NotVerticalBar", 8740, "\xe2\x88\xa4"),
+ ENTITY_DEF("ic", 8291, "\xe2\x81\xa3"),
+ ENTITY_DEF("solbar", 9023, "\xe2\x8c\xbf"),
+ ENTITY_DEF("approx", 8776, "\xe2\x89\x88"),
+ ENTITY_DEF("SquareSuperset", 8848, "\xe2\x8a\x90"),
+ ENTITY_DEF("numsp", 8199, "\xe2\x80\x87"),
+ ENTITY_DEF("nLt", 8810, "\xe2\x89\xaa\xe2\x83\x92"),
+ ENTITY_DEF("tilde", 732, "\xcb\x9c"),
+ ENTITY_DEF("rlarr", 8644, "\xe2\x87\x84"),
+ ENTITY_DEF("langle", 10216, "\xe2\x9f\xa8"),
+ ENTITY_DEF("nleqslant", 10877, "\xe2\xa9\xbd\xcc\xb8"),
+ ENTITY_DEF("Nacute", 323, "\xc5\x83"),
+ ENTITY_DEF("NotLeftTriangle", 8938, "\xe2\x8b\xaa"),
+ ENTITY_DEF("sopf", 120164, "\xf0\x9d\x95\xa4"),
+ ENTITY_DEF("xmap", 10236, "\xe2\x9f\xbc"),
+ ENTITY_DEF("supne", 8843, "\xe2\x8a\x8b"),
+ ENTITY_DEF("Int", 8748, "\xe2\x88\xac"),
+ ENTITY_DEF("nsupseteqq", 10950, "\xe2\xab\x86\xcc\xb8"),
+ ENTITY_DEF("circlearrowright", 8635, "\xe2\x86\xbb"),
+ ENTITY_DEF("NotCongruent", 8802, "\xe2\x89\xa2"),
+ ENTITY_DEF("Scedil", 350, "\xc5\x9e"),
+ ENTITY_DEF_HEUR("raquo", 187, "\xc2\xbb"),
+ ENTITY_DEF("ycy", 1099, "\xd1\x8b"),
+ ENTITY_DEF("notinvb", 8951, "\xe2\x8b\xb7"),
+ ENTITY_DEF("andv", 10842, "\xe2\xa9\x9a"),
+ ENTITY_DEF("nap", 8777, "\xe2\x89\x89"),
+ ENTITY_DEF("shcy", 1096, "\xd1\x88"),
+ ENTITY_DEF("ssetmn", 8726, "\xe2\x88\x96"),
+ ENTITY_DEF("downarrow", 8595, "\xe2\x86\x93"),
+ ENTITY_DEF("gesdotol", 10884, "\xe2\xaa\x84"),
+ ENTITY_DEF("Congruent", 8801, "\xe2\x89\xa1"),
+ ENTITY_DEF_HEUR("pound", 163, "\xc2\xa3"),
+ ENTITY_DEF("ZeroWidthSpace", 8203, "\xe2\x80\x8b"),
+ ENTITY_DEF("rdca", 10551, "\xe2\xa4\xb7"),
+ ENTITY_DEF("rmoust", 9137, "\xe2\x8e\xb1"),
+ ENTITY_DEF("zcy", 1079, "\xd0\xb7"),
+ ENTITY_DEF("Square", 9633, "\xe2\x96\xa1"),
+ ENTITY_DEF("subE", 10949, "\xe2\xab\x85"),
+ ENTITY_DEF("infintie", 10717, "\xe2\xa7\x9d"),
+ ENTITY_DEF("Cayleys", 8493, "\xe2\x84\xad"),
+ ENTITY_DEF("lsaquo", 8249, "\xe2\x80\xb9"),
+ ENTITY_DEF("realpart", 8476, "\xe2\x84\x9c"),
+ ENTITY_DEF("nprec", 8832, "\xe2\x8a\x80"),
+ ENTITY_DEF("RightTriangleBar", 10704, "\xe2\xa7\x90"),
+ ENTITY_DEF("Kopf", 120130, "\xf0\x9d\x95\x82"),
+ ENTITY_DEF("Ubreve", 364, "\xc5\xac"),
+ ENTITY_DEF("Uopf", 120140, "\xf0\x9d\x95\x8c"),
+ ENTITY_DEF("trianglelefteq", 8884, "\xe2\x8a\xb4"),
+ ENTITY_DEF("rotimes", 10805, "\xe2\xa8\xb5"),
+ ENTITY_DEF("qfr", 120110, "\xf0\x9d\x94\xae"),
+ ENTITY_DEF("gtcc", 10919, "\xe2\xaa\xa7"),
+ ENTITY_DEF("fnof", 402, "\xc6\x92"),
+ ENTITY_DEF("tritime", 10811, "\xe2\xa8\xbb"),
+ ENTITY_DEF("andslope", 10840, "\xe2\xa9\x98"),
+ ENTITY_DEF("harrw", 8621, "\xe2\x86\xad"),
+ ENTITY_DEF("NotSquareSuperset", 8848, "\xe2\x8a\x90\xcc\xb8"),
+ ENTITY_DEF("Amacr", 256, "\xc4\x80"),
+ ENTITY_DEF("OpenCurlyDoubleQuote", 8220, "\xe2\x80\x9c"),
+ ENTITY_DEF_HEUR("thorn", 254, "\xc3\xbe"),
+ ENTITY_DEF_HEUR("ordf", 170, "\xc2\xaa"),
+ ENTITY_DEF("natur", 9838, "\xe2\x99\xae"),
+ ENTITY_DEF("xi", 958, "\xce\xbe"),
+ ENTITY_DEF("infin", 8734, "\xe2\x88\x9e"),
+ ENTITY_DEF("nspar", 8742, "\xe2\x88\xa6"),
+ ENTITY_DEF("Jcy", 1049, "\xd0\x99"),
+ ENTITY_DEF("DownLeftTeeVector", 10590, "\xe2\xa5\x9e"),
+ ENTITY_DEF("rbarr", 10509, "\xe2\xa4\x8d"),
+ ENTITY_DEF("Xi", 926, "\xce\x9e"),
+ ENTITY_DEF("bull", 8226, "\xe2\x80\xa2"),
+ ENTITY_DEF("cuesc", 8927, "\xe2\x8b\x9f"),
+ ENTITY_DEF("backcong", 8780, "\xe2\x89\x8c"),
+ ENTITY_DEF("frac35", 8535, "\xe2\x85\x97"),
+ ENTITY_DEF("hscr", 119997, "\xf0\x9d\x92\xbd"),
+ ENTITY_DEF("LessEqualGreater", 8922, "\xe2\x8b\x9a"),
+ ENTITY_DEF("Implies", 8658, "\xe2\x87\x92"),
+ ENTITY_DEF("ETH", 208, "\xc3\x90"),
+ ENTITY_DEF_HEUR("Yacute", 221, "\xc3\x9d"),
+ ENTITY_DEF_HEUR("shy", 173, "\xc2\xad"),
+ ENTITY_DEF("Rarrtl", 10518, "\xe2\xa4\x96"),
+ ENTITY_DEF_HEUR("sup1", 185, "\xc2\xb9"),
+ ENTITY_DEF("reals", 8477, "\xe2\x84\x9d"),
+ ENTITY_DEF("blacklozenge", 10731, "\xe2\xa7\xab"),
+ ENTITY_DEF("ncedil", 326, "\xc5\x86"),
+ ENTITY_DEF("Lambda", 923, "\xce\x9b"),
+ ENTITY_DEF("uopf", 120166, "\xf0\x9d\x95\xa6"),
+ ENTITY_DEF("bigodot", 10752, "\xe2\xa8\x80"),
+ ENTITY_DEF("ubreve", 365, "\xc5\xad"),
+ ENTITY_DEF("drbkarow", 10512, "\xe2\xa4\x90"),
+ ENTITY_DEF("els", 10901, "\xe2\xaa\x95"),
+ ENTITY_DEF("shortparallel", 8741, "\xe2\x88\xa5"),
+ ENTITY_DEF("Pcy", 1055, "\xd0\x9f"),
+ ENTITY_DEF("dsol", 10742, "\xe2\xa7\xb6"),
+ ENTITY_DEF("supsim", 10952, "\xe2\xab\x88"),
+ ENTITY_DEF("Longrightarrow", 10233, "\xe2\x9f\xb9"),
+ ENTITY_DEF("ThickSpace", 8287, "\xe2\x81\x9f\xe2\x80\x8a"),
+ ENTITY_DEF("Itilde", 296, "\xc4\xa8"),
+ ENTITY_DEF("nparallel", 8742, "\xe2\x88\xa6"),
+ ENTITY_DEF("And", 10835, "\xe2\xa9\x93"),
+ ENTITY_DEF("boxhd", 9516, "\xe2\x94\xac"),
+ ENTITY_DEF("Dashv", 10980, "\xe2\xab\xa4"),
+ ENTITY_DEF("NotSuperset", 8835, "\xe2\x8a\x83\xe2\x83\x92"),
+ ENTITY_DEF("Eta", 919, "\xce\x97"),
+ ENTITY_DEF("Qopf", 8474, "\xe2\x84\x9a"),
+ ENTITY_DEF("period", 46, "\x2e"),
+ ENTITY_DEF("angmsd", 8737, "\xe2\x88\xa1"),
+ ENTITY_DEF("fllig", 64258, "\xef\xac\x82"),
+ ENTITY_DEF("cuvee", 8910, "\xe2\x8b\x8e"),
+ ENTITY_DEF("wedbar", 10847, "\xe2\xa9\x9f"),
+ ENTITY_DEF("Fscr", 8497, "\xe2\x84\xb1"),
+ ENTITY_DEF("veebar", 8891, "\xe2\x8a\xbb"),
+ ENTITY_DEF("Longleftrightarrow", 10234, "\xe2\x9f\xba"),
+ ENTITY_DEF_HEUR("reg", 174, "\xc2\xae"),
+ ENTITY_DEF("NegativeMediumSpace", 8203, "\xe2\x80\x8b"),
+ ENTITY_DEF("Upsi", 978, "\xcf\x92"),
+ ENTITY_DEF("Mellintrf", 8499, "\xe2\x84\xb3"),
+ ENTITY_DEF("boxHU", 9577, "\xe2\x95\xa9"),
+ ENTITY_DEF("frac56", 8538, "\xe2\x85\x9a"),
+ ENTITY_DEF("utrif", 9652, "\xe2\x96\xb4"),
+ ENTITY_DEF("LeftTriangle", 8882, "\xe2\x8a\xb2"),
+ ENTITY_DEF("nsime", 8772, "\xe2\x89\x84"),
+ ENTITY_DEF("rcedil", 343, "\xc5\x97"),
+ ENTITY_DEF("aogon", 261, "\xc4\x85"),
+ ENTITY_DEF("uHar", 10595, "\xe2\xa5\xa3"),
+ ENTITY_DEF("ForAll", 8704, "\xe2\x88\x80"),
+ ENTITY_DEF("prE", 10931, "\xe2\xaa\xb3"),
+ ENTITY_DEF("boxV", 9553, "\xe2\x95\x91"),
+ ENTITY_DEF("softcy", 1100, "\xd1\x8c"),
+ ENTITY_DEF("hercon", 8889, "\xe2\x8a\xb9"),
+ ENTITY_DEF("lmoustache", 9136, "\xe2\x8e\xb0"),
+ ENTITY_DEF("Product", 8719, "\xe2\x88\x8f"),
+ ENTITY_DEF("lsimg", 10895, "\xe2\xaa\x8f"),
+ ENTITY_DEF("verbar", 124, "\x7c"),
+ ENTITY_DEF("ofcir", 10687, "\xe2\xa6\xbf"),
+ ENTITY_DEF("curlyeqprec", 8926, "\xe2\x8b\x9e"),
+ ENTITY_DEF("ldquo", 8220, "\xe2\x80\x9c"),
+ ENTITY_DEF("bot", 8869, "\xe2\x8a\xa5"),
+ ENTITY_DEF("Psi", 936, "\xce\xa8"),
+ ENTITY_DEF("OElig", 338, "\xc5\x92"),
+ ENTITY_DEF("DownRightVectorBar", 10583, "\xe2\xa5\x97"),
+ ENTITY_DEF("minusb", 8863, "\xe2\x8a\x9f"),
+ ENTITY_DEF("Iscr", 8464, "\xe2\x84\x90"),
+ ENTITY_DEF("Tcedil", 354, "\xc5\xa2"),
+ ENTITY_DEF("ffilig", 64259, "\xef\xac\x83"),
+ ENTITY_DEF("Gcy", 1043, "\xd0\x93"),
+ ENTITY_DEF("oline", 8254, "\xe2\x80\xbe"),
+ ENTITY_DEF("bottom", 8869, "\xe2\x8a\xa5"),
+ ENTITY_DEF("nVDash", 8879, "\xe2\x8a\xaf"),
+ ENTITY_DEF("lessdot", 8918, "\xe2\x8b\x96"),
+ ENTITY_DEF("cups", 8746, "\xe2\x88\xaa\xef\xb8\x80"),
+ ENTITY_DEF("gla", 10917, "\xe2\xaa\xa5"),
+ ENTITY_DEF("hellip", 8230, "\xe2\x80\xa6"),
+ ENTITY_DEF("hookleftarrow", 8617, "\xe2\x86\xa9"),
+ ENTITY_DEF("Cup", 8915, "\xe2\x8b\x93"),
+ ENTITY_DEF("upsi", 965, "\xcf\x85"),
+ ENTITY_DEF("DownArrowBar", 10515, "\xe2\xa4\x93"),
+ ENTITY_DEF("lowast", 8727, "\xe2\x88\x97"),
+ ENTITY_DEF("profline", 8978, "\xe2\x8c\x92"),
+ ENTITY_DEF("ngsim", 8821, "\xe2\x89\xb5"),
+ ENTITY_DEF("boxhu", 9524, "\xe2\x94\xb4"),
+ ENTITY_DEF("operp", 10681, "\xe2\xa6\xb9"),
+ ENTITY_DEF("cap", 8745, "\xe2\x88\xa9"),
+ ENTITY_DEF("Hcirc", 292, "\xc4\xa4"),
+ ENTITY_DEF("Ncy", 1053, "\xd0\x9d"),
+ ENTITY_DEF("zeetrf", 8488, "\xe2\x84\xa8"),
+ ENTITY_DEF("cuepr", 8926, "\xe2\x8b\x9e"),
+ ENTITY_DEF("supsetneq", 8843, "\xe2\x8a\x8b"),
+ ENTITY_DEF("lfloor", 8970, "\xe2\x8c\x8a"),
+ ENTITY_DEF("ngtr", 8815, "\xe2\x89\xaf"),
+ ENTITY_DEF("ccups", 10828, "\xe2\xa9\x8c"),
+ ENTITY_DEF("pscr", 120005, "\xf0\x9d\x93\x85"),
+ ENTITY_DEF("Cfr", 8493, "\xe2\x84\xad"),
+ ENTITY_DEF("dtri", 9663, "\xe2\x96\xbf"),
+ ENTITY_DEF("icirc", 238, "\xc3\xae"),
+ ENTITY_DEF("leftarrow", 8592, "\xe2\x86\x90"),
+ ENTITY_DEF("vdash", 8866, "\xe2\x8a\xa2"),
+ ENTITY_DEF("leftrightharpoons", 8651, "\xe2\x87\x8b"),
+ ENTITY_DEF("rightrightarrows", 8649, "\xe2\x87\x89"),
+ ENTITY_DEF("strns", 175, "\xc2\xaf"),
+ ENTITY_DEF("intlarhk", 10775, "\xe2\xa8\x97"),
+ ENTITY_DEF("downharpoonright", 8642, "\xe2\x87\x82"),
+ ENTITY_DEF_HEUR("yacute", 253, "\xc3\xbd"),
+ ENTITY_DEF("boxUr", 9561, "\xe2\x95\x99"),
+ ENTITY_DEF("triangleleft", 9667, "\xe2\x97\x83"),
+ ENTITY_DEF("DiacriticalDot", 729, "\xcb\x99"),
+ ENTITY_DEF("thetav", 977, "\xcf\x91"),
+ ENTITY_DEF("OverBracket", 9140, "\xe2\x8e\xb4"),
+ ENTITY_DEF("PrecedesTilde", 8830, "\xe2\x89\xbe"),
+ ENTITY_DEF("rtrie", 8885, "\xe2\x8a\xb5"),
+ ENTITY_DEF("Scirc", 348, "\xc5\x9c"),
+ ENTITY_DEF("vsupne", 8843, "\xe2\x8a\x8b\xef\xb8\x80"),
+ ENTITY_DEF("OverBrace", 9182, "\xe2\x8f\x9e"),
+ ENTITY_DEF("Yfr", 120092, "\xf0\x9d\x94\x9c"),
+ ENTITY_DEF("scnE", 10934, "\xe2\xaa\xb6"),
+ ENTITY_DEF("simlE", 10911, "\xe2\xaa\x9f"),
+ ENTITY_DEF("Proportional", 8733, "\xe2\x88\x9d"),
+ ENTITY_DEF("edot", 279, "\xc4\x97"),
+ ENTITY_DEF("loang", 10220, "\xe2\x9f\xac"),
+ ENTITY_DEF("gesdot", 10880, "\xe2\xaa\x80"),
+ ENTITY_DEF("DownBreve", 785, "\xcc\x91"),
+ ENTITY_DEF("pcy", 1087, "\xd0\xbf"),
+ ENTITY_DEF("Succeeds", 8827, "\xe2\x89\xbb"),
+ ENTITY_DEF("mfr", 120106, "\xf0\x9d\x94\xaa"),
+ ENTITY_DEF("Leftarrow", 8656, "\xe2\x87\x90"),
+ ENTITY_DEF("boxDr", 9555, "\xe2\x95\x93"),
+ ENTITY_DEF("Nscr", 119977, "\xf0\x9d\x92\xa9"),
+ ENTITY_DEF("diam", 8900, "\xe2\x8b\x84"),
+ ENTITY_DEF("CHcy", 1063, "\xd0\xa7"),
+ ENTITY_DEF("boxdr", 9484, "\xe2\x94\x8c"),
+ ENTITY_DEF("rlm", 8207, "\xe2\x80\x8f"),
+ ENTITY_DEF("Coproduct", 8720, "\xe2\x88\x90"),
+ ENTITY_DEF("RightTeeArrow", 8614, "\xe2\x86\xa6"),
+ ENTITY_DEF("tridot", 9708, "\xe2\x97\xac"),
+ ENTITY_DEF("ldquor", 8222, "\xe2\x80\x9e"),
+ ENTITY_DEF("sol", 47, "\x2f"),
+ ENTITY_DEF_HEUR("ecirc", 234, "\xc3\xaa"),
+ ENTITY_DEF("DoubleLeftArrow", 8656, "\xe2\x87\x90"),
+ ENTITY_DEF("Gscr", 119970, "\xf0\x9d\x92\xa2"),
+ ENTITY_DEF("ap", 8776, "\xe2\x89\x88"),
+ ENTITY_DEF("rbrke", 10636, "\xe2\xa6\x8c"),
+ ENTITY_DEF("LeftFloor", 8970, "\xe2\x8c\x8a"),
+ ENTITY_DEF("blk12", 9618, "\xe2\x96\x92"),
+ ENTITY_DEF("Conint", 8751, "\xe2\x88\xaf"),
+ ENTITY_DEF("triangledown", 9663, "\xe2\x96\xbf"),
+ ENTITY_DEF("Icy", 1048, "\xd0\x98"),
+ ENTITY_DEF("backprime", 8245, "\xe2\x80\xb5"),
+ ENTITY_DEF("longleftrightarrow", 10231, "\xe2\x9f\xb7"),
+ ENTITY_DEF("ntriangleleft", 8938, "\xe2\x8b\xaa"),
+ ENTITY_DEF_HEUR("copy", 169, "\xc2\xa9"),
+ ENTITY_DEF("mapstodown", 8615, "\xe2\x86\xa7"),
+ ENTITY_DEF("seArr", 8664, "\xe2\x87\x98"),
+ ENTITY_DEF("ENG", 330, "\xc5\x8a"),
+ ENTITY_DEF("DoubleRightArrow", 8658, "\xe2\x87\x92"),
+ ENTITY_DEF("tfr", 120113, "\xf0\x9d\x94\xb1"),
+ ENTITY_DEF("rharul", 10604, "\xe2\xa5\xac"),
+ ENTITY_DEF("bfr", 120095, "\xf0\x9d\x94\x9f"),
+ ENTITY_DEF("origof", 8886, "\xe2\x8a\xb6"),
+ ENTITY_DEF("Therefore", 8756, "\xe2\x88\xb4"),
+ ENTITY_DEF("glE", 10898, "\xe2\xaa\x92"),
+ ENTITY_DEF("leftarrowtail", 8610, "\xe2\x86\xa2"),
+ ENTITY_DEF("NotEqual", 8800, "\xe2\x89\xa0"),
+ ENTITY_DEF("LeftCeiling", 8968, "\xe2\x8c\x88"),
+ ENTITY_DEF("lArr", 8656, "\xe2\x87\x90"),
+ ENTITY_DEF("subseteq", 8838, "\xe2\x8a\x86"),
+ ENTITY_DEF("larrbfs", 10527, "\xe2\xa4\x9f"),
+ ENTITY_DEF("Gammad", 988, "\xcf\x9c"),
+ ENTITY_DEF("rtriltri", 10702, "\xe2\xa7\x8e"),
+ ENTITY_DEF("Fcy", 1060, "\xd0\xa4"),
+ ENTITY_DEF("Vopf", 120141, "\xf0\x9d\x95\x8d"),
+ ENTITY_DEF("lrarr", 8646, "\xe2\x87\x86"),
+ ENTITY_DEF("delta", 948, "\xce\xb4"),
+ ENTITY_DEF("xodot", 10752, "\xe2\xa8\x80"),
+ ENTITY_DEF("larrtl", 8610, "\xe2\x86\xa2"),
+ ENTITY_DEF("gsim", 8819, "\xe2\x89\xb3"),
+ ENTITY_DEF("ratail", 10522, "\xe2\xa4\x9a"),
+ ENTITY_DEF("vsubne", 8842, "\xe2\x8a\x8a\xef\xb8\x80"),
+ ENTITY_DEF("boxur", 9492, "\xe2\x94\x94"),
+ ENTITY_DEF("succsim", 8831, "\xe2\x89\xbf"),
+ ENTITY_DEF("triplus", 10809, "\xe2\xa8\xb9"),
+ ENTITY_DEF("nless", 8814, "\xe2\x89\xae"),
+ ENTITY_DEF("uharr", 8638, "\xe2\x86\xbe"),
+ ENTITY_DEF("lambda", 955, "\xce\xbb"),
+ ENTITY_DEF_HEUR("uuml", 252, "\xc3\xbc"),
+ ENTITY_DEF("horbar", 8213, "\xe2\x80\x95"),
+ ENTITY_DEF("ccirc", 265, "\xc4\x89"),
+ ENTITY_DEF("sqcup", 8852, "\xe2\x8a\x94"),
+ ENTITY_DEF("Pscr", 119979, "\xf0\x9d\x92\xab"),
+ ENTITY_DEF("supsup", 10966, "\xe2\xab\x96"),
+ ENTITY_DEF("Cacute", 262, "\xc4\x86"),
+ ENTITY_DEF("upsih", 978, "\xcf\x92"),
+ ENTITY_DEF("precsim", 8830, "\xe2\x89\xbe"),
+ ENTITY_DEF("longrightarrow", 10230, "\xe2\x9f\xb6"),
+ ENTITY_DEF("circledR", 174, "\xc2\xae"),
+ ENTITY_DEF("UpTeeArrow", 8613, "\xe2\x86\xa5"),
+ ENTITY_DEF("bepsi", 1014, "\xcf\xb6"),
+ ENTITY_DEF("oast", 8859, "\xe2\x8a\x9b"),
+ ENTITY_DEF("yfr", 120118, "\xf0\x9d\x94\xb6"),
+ ENTITY_DEF("rdsh", 8627, "\xe2\x86\xb3"),
+ ENTITY_DEF("Ograve", 210, "\xc3\x92"),
+ ENTITY_DEF("LeftVectorBar", 10578, "\xe2\xa5\x92"),
+ ENTITY_DEF("NotNestedLessLess", 10913, "\xe2\xaa\xa1\xcc\xb8"),
+ ENTITY_DEF("Jscr", 119973, "\xf0\x9d\x92\xa5"),
+ ENTITY_DEF("psi", 968, "\xcf\x88"),
+ ENTITY_DEF("orarr", 8635, "\xe2\x86\xbb"),
+ ENTITY_DEF("Subset", 8912, "\xe2\x8b\x90"),
+ ENTITY_DEF("curarr", 8631, "\xe2\x86\xb7"),
+ ENTITY_DEF("CirclePlus", 8853, "\xe2\x8a\x95"),
+ ENTITY_DEF("gtrless", 8823, "\xe2\x89\xb7"),
+ ENTITY_DEF("nvle", 8804, "\xe2\x89\xa4\xe2\x83\x92"),
+ ENTITY_DEF("prop", 8733, "\xe2\x88\x9d"),
+ ENTITY_DEF("gEl", 10892, "\xe2\xaa\x8c"),
+ ENTITY_DEF("gtlPar", 10645, "\xe2\xa6\x95"),
+ ENTITY_DEF("frasl", 8260, "\xe2\x81\x84"),
+ ENTITY_DEF("nearr", 8599, "\xe2\x86\x97"),
+ ENTITY_DEF("NotSubsetEqual", 8840, "\xe2\x8a\x88"),
+ ENTITY_DEF("planck", 8463, "\xe2\x84\x8f"),
+ ENTITY_DEF_HEUR("Uuml", 220, "\xc3\x9c"),
+ ENTITY_DEF("spadesuit", 9824, "\xe2\x99\xa0"),
+ ENTITY_DEF_HEUR("sect", 167, "\xc2\xa7"),
+ ENTITY_DEF("cdot", 267, "\xc4\x8b"),
+ ENTITY_DEF("boxVh", 9579, "\xe2\x95\xab"),
+ ENTITY_DEF("zscr", 120015, "\xf0\x9d\x93\x8f"),
+ ENTITY_DEF("nsqsube", 8930, "\xe2\x8b\xa2"),
+ ENTITY_DEF("grave", 96, "\x60"),
+ ENTITY_DEF("angrtvb", 8894, "\xe2\x8a\xbe"),
+ ENTITY_DEF("MediumSpace", 8287, "\xe2\x81\x9f"),
+ ENTITY_DEF("Ntilde", 209, "\xc3\x91"),
+ ENTITY_DEF("solb", 10692, "\xe2\xa7\x84"),
+ ENTITY_DEF("angzarr", 9084, "\xe2\x8d\xbc"),
+ ENTITY_DEF("nopf", 120159, "\xf0\x9d\x95\x9f"),
+ ENTITY_DEF("rtrif", 9656, "\xe2\x96\xb8"),
+ ENTITY_DEF("nrightarrow", 8603, "\xe2\x86\x9b"),
+ ENTITY_DEF("Kappa", 922, "\xce\x9a"),
+ ENTITY_DEF("simrarr", 10610, "\xe2\xa5\xb2"),
+ ENTITY_DEF("imacr", 299, "\xc4\xab"),
+ ENTITY_DEF("vrtri", 8883, "\xe2\x8a\xb3"),
+ ENTITY_DEF("part", 8706, "\xe2\x88\x82"),
+ ENTITY_DEF("esim", 8770, "\xe2\x89\x82"),
+ ENTITY_DEF_HEUR("atilde", 227, "\xc3\xa3"),
+ ENTITY_DEF("DownRightTeeVector", 10591, "\xe2\xa5\x9f"),
+ ENTITY_DEF("jcirc", 309, "\xc4\xb5"),
+ ENTITY_DEF("Ecaron", 282, "\xc4\x9a"),
+ ENTITY_DEF("VerticalSeparator", 10072, "\xe2\x9d\x98"),
+ ENTITY_DEF("rHar", 10596, "\xe2\xa5\xa4"),
+ ENTITY_DEF("rcaron", 345, "\xc5\x99"),
+ ENTITY_DEF("subnE", 10955, "\xe2\xab\x8b"),
+ ENTITY_DEF("ii", 8520, "\xe2\x85\x88"),
+ ENTITY_DEF("Cconint", 8752, "\xe2\x88\xb0"),
+ ENTITY_DEF("Mcy", 1052, "\xd0\x9c"),
+ ENTITY_DEF("eqcolon", 8789, "\xe2\x89\x95"),
+ ENTITY_DEF("cupor", 10821, "\xe2\xa9\x85"),
+ ENTITY_DEF("DoubleUpArrow", 8657, "\xe2\x87\x91"),
+ ENTITY_DEF("boxbox", 10697, "\xe2\xa7\x89"),
+ ENTITY_DEF("setminus", 8726, "\xe2\x88\x96"),
+ ENTITY_DEF("Lleftarrow", 8666, "\xe2\x87\x9a"),
+ ENTITY_DEF("nang", 8736, "\xe2\x88\xa0\xe2\x83\x92"),
+ ENTITY_DEF("TRADE", 8482, "\xe2\x84\xa2"),
+ ENTITY_DEF("urcorner", 8989, "\xe2\x8c\x9d"),
+ ENTITY_DEF("lsqb", 91, "\x5b"),
+ ENTITY_DEF("cupcup", 10826, "\xe2\xa9\x8a"),
+ ENTITY_DEF("kjcy", 1116, "\xd1\x9c"),
+ ENTITY_DEF("llhard", 10603, "\xe2\xa5\xab"),
+ ENTITY_DEF("mumap", 8888, "\xe2\x8a\xb8"),
+ ENTITY_DEF("iiint", 8749, "\xe2\x88\xad"),
+ ENTITY_DEF("RightTee", 8866, "\xe2\x8a\xa2"),
+ ENTITY_DEF("Tcaron", 356, "\xc5\xa4"),
+ ENTITY_DEF("bigcirc", 9711, "\xe2\x97\xaf"),
+ ENTITY_DEF("trianglerighteq", 8885, "\xe2\x8a\xb5"),
+ ENTITY_DEF("NotLessGreater", 8824, "\xe2\x89\xb8"),
+ ENTITY_DEF("hArr", 8660, "\xe2\x87\x94"),
+ ENTITY_DEF("ocy", 1086, "\xd0\xbe"),
+ ENTITY_DEF("tosa", 10537, "\xe2\xa4\xa9"),
+ ENTITY_DEF("twixt", 8812, "\xe2\x89\xac"),
+ ENTITY_DEF("square", 9633, "\xe2\x96\xa1"),
+ ENTITY_DEF("Otimes", 10807, "\xe2\xa8\xb7"),
+ ENTITY_DEF("Kcedil", 310, "\xc4\xb6"),
+ ENTITY_DEF("beth", 8502, "\xe2\x84\xb6"),
+ ENTITY_DEF("triminus", 10810, "\xe2\xa8\xba"),
+ ENTITY_DEF("nlArr", 8653, "\xe2\x87\x8d"),
+ ENTITY_DEF("Oacute", 211, "\xc3\x93"),
+ ENTITY_DEF("zwnj", 8204, "\xe2\x80\x8c"),
+ ENTITY_DEF("ll", 8810, "\xe2\x89\xaa"),
+ ENTITY_DEF("smashp", 10803, "\xe2\xa8\xb3"),
+ ENTITY_DEF("ngeqq", 8807, "\xe2\x89\xa7\xcc\xb8"),
+ ENTITY_DEF("rnmid", 10990, "\xe2\xab\xae"),
+ ENTITY_DEF("nwArr", 8662, "\xe2\x87\x96"),
+ ENTITY_DEF("RightUpDownVector", 10575, "\xe2\xa5\x8f"),
+ ENTITY_DEF("lbbrk", 10098, "\xe2\x9d\xb2"),
+ ENTITY_DEF("compfn", 8728, "\xe2\x88\x98"),
+ ENTITY_DEF("eDDot", 10871, "\xe2\xa9\xb7"),
+ ENTITY_DEF("Jsercy", 1032, "\xd0\x88"),
+ ENTITY_DEF("HARDcy", 1066, "\xd0\xaa"),
+ ENTITY_DEF("nexists", 8708, "\xe2\x88\x84"),
+ ENTITY_DEF("theta", 952, "\xce\xb8"),
+ ENTITY_DEF("plankv", 8463, "\xe2\x84\x8f"),
+ ENTITY_DEF_HEUR("sup2", 178, "\xc2\xb2"),
+ ENTITY_DEF("lessapprox", 10885, "\xe2\xaa\x85"),
+ ENTITY_DEF("gdot", 289, "\xc4\xa1"),
+ ENTITY_DEF("angmsdae", 10668, "\xe2\xa6\xac"),
+ ENTITY_DEF("Superset", 8835, "\xe2\x8a\x83"),
+ ENTITY_DEF("prap", 10935, "\xe2\xaa\xb7"),
+ ENTITY_DEF("Zscr", 119989, "\xf0\x9d\x92\xb5"),
+ ENTITY_DEF("nsucc", 8833, "\xe2\x8a\x81"),
+ ENTITY_DEF("supseteqq", 10950, "\xe2\xab\x86"),
+ ENTITY_DEF("UpTee", 8869, "\xe2\x8a\xa5"),
+ ENTITY_DEF("LowerLeftArrow", 8601, "\xe2\x86\x99"),
+ ENTITY_DEF("ssmile", 8995, "\xe2\x8c\xa3"),
+ ENTITY_DEF("niv", 8715, "\xe2\x88\x8b"),
+ ENTITY_DEF("bigvee", 8897, "\xe2\x8b\x81"),
+ ENTITY_DEF("kscr", 120000, "\xf0\x9d\x93\x80"),
+ ENTITY_DEF("xutri", 9651, "\xe2\x96\xb3"),
+ ENTITY_DEF("caret", 8257, "\xe2\x81\x81"),
+ ENTITY_DEF("caron", 711, "\xcb\x87"),
+ ENTITY_DEF("Wedge", 8896, "\xe2\x8b\x80"),
+ ENTITY_DEF("sdotb", 8865, "\xe2\x8a\xa1"),
+ ENTITY_DEF("bigoplus", 10753, "\xe2\xa8\x81"),
+ ENTITY_DEF("Breve", 728, "\xcb\x98"),
+ ENTITY_DEF("ImaginaryI", 8520, "\xe2\x85\x88"),
+ ENTITY_DEF("longmapsto", 10236, "\xe2\x9f\xbc"),
+ ENTITY_DEF("boxVH", 9580, "\xe2\x95\xac"),
+ ENTITY_DEF("lozenge", 9674, "\xe2\x97\x8a"),
+ ENTITY_DEF("toea", 10536, "\xe2\xa4\xa8"),
+ ENTITY_DEF("nbumpe", 8783, "\xe2\x89\x8f\xcc\xb8"),
+ ENTITY_DEF("gcirc", 285, "\xc4\x9d"),
+ ENTITY_DEF("NotHumpEqual", 8783, "\xe2\x89\x8f\xcc\xb8"),
+ ENTITY_DEF("pre", 10927, "\xe2\xaa\xaf"),
+ ENTITY_DEF("ascr", 119990, "\xf0\x9d\x92\xb6"),
+ ENTITY_DEF("Acirc", 194, "\xc3\x82"),
+ ENTITY_DEF("questeq", 8799, "\xe2\x89\x9f"),
+ ENTITY_DEF("ncaron", 328, "\xc5\x88"),
+ ENTITY_DEF("LeftTeeArrow", 8612, "\xe2\x86\xa4"),
+ ENTITY_DEF("xcirc", 9711, "\xe2\x97\xaf"),
+ ENTITY_DEF("swarr", 8601, "\xe2\x86\x99"),
+ ENTITY_DEF("MinusPlus", 8723, "\xe2\x88\x93"),
+ ENTITY_DEF("plus", 43, "\x2b"),
+ ENTITY_DEF("NotDoubleVerticalBar", 8742, "\xe2\x88\xa6"),
+ ENTITY_DEF("rppolint", 10770, "\xe2\xa8\x92"),
+ ENTITY_DEF("NotTildeFullEqual", 8775, "\xe2\x89\x87"),
+ ENTITY_DEF("ltdot", 8918, "\xe2\x8b\x96"),
+ ENTITY_DEF("NotNestedGreaterGreater", 10914, "\xe2\xaa\xa2\xcc\xb8"),
+ ENTITY_DEF("Lscr", 8466, "\xe2\x84\x92"),
+ ENTITY_DEF("pitchfork", 8916, "\xe2\x8b\x94"),
+ ENTITY_DEF("Eopf", 120124, "\xf0\x9d\x94\xbc"),
+ ENTITY_DEF("ropf", 120163, "\xf0\x9d\x95\xa3"),
+ ENTITY_DEF("Delta", 916, "\xce\x94"),
+ ENTITY_DEF("lozf", 10731, "\xe2\xa7\xab"),
+ ENTITY_DEF("RightTeeVector", 10587, "\xe2\xa5\x9b"),
+ ENTITY_DEF("UpDownArrow", 8597, "\xe2\x86\x95"),
+ ENTITY_DEF("bump", 8782, "\xe2\x89\x8e"),
+ ENTITY_DEF("Rscr", 8475, "\xe2\x84\x9b"),
+ ENTITY_DEF("slarr", 8592, "\xe2\x86\x90"),
+ ENTITY_DEF("lcy", 1083, "\xd0\xbb"),
+ ENTITY_DEF("Vee", 8897, "\xe2\x8b\x81"),
+ ENTITY_DEF("Iogon", 302, "\xc4\xae"),
+ ENTITY_DEF("minus", 8722, "\xe2\x88\x92"),
+ ENTITY_DEF("GreaterFullEqual", 8807, "\xe2\x89\xa7"),
+ ENTITY_DEF("xhArr", 10234, "\xe2\x9f\xba"),
+ ENTITY_DEF("shortmid", 8739, "\xe2\x88\xa3"),
+ ENTITY_DEF("DoubleDownArrow", 8659, "\xe2\x87\x93"),
+ ENTITY_DEF("Wscr", 119986, "\xf0\x9d\x92\xb2"),
+ ENTITY_DEF("rang", 10217, "\xe2\x9f\xa9"),
+ ENTITY_DEF("lcub", 123, "\x7b"),
+ ENTITY_DEF("mnplus", 8723, "\xe2\x88\x93"),
+ ENTITY_DEF("ulcrop", 8975, "\xe2\x8c\x8f"),
+ ENTITY_DEF("wfr", 120116, "\xf0\x9d\x94\xb4"),
+ ENTITY_DEF("DifferentialD", 8518, "\xe2\x85\x86"),
+ ENTITY_DEF("ThinSpace", 8201, "\xe2\x80\x89"),
+ ENTITY_DEF("NotGreaterGreater", 8811, "\xe2\x89\xab\xcc\xb8"),
+ ENTITY_DEF("Topf", 120139, "\xf0\x9d\x95\x8b"),
+ ENTITY_DEF("sbquo", 8218, "\xe2\x80\x9a"),
+ ENTITY_DEF("sdot", 8901, "\xe2\x8b\x85"),
+ ENTITY_DEF("DoubleLeftTee", 10980, "\xe2\xab\xa4"),
+ ENTITY_DEF("vBarv", 10985, "\xe2\xab\xa9"),
+ ENTITY_DEF("subne", 8842, "\xe2\x8a\x8a"),
+ ENTITY_DEF("gtrdot", 8919, "\xe2\x8b\x97"),
+ ENTITY_DEF("opar", 10679, "\xe2\xa6\xb7"),
+ ENTITY_DEF("apid", 8779, "\xe2\x89\x8b"),
+ ENTITY_DEF("Cross", 10799, "\xe2\xa8\xaf"),
+ ENTITY_DEF("lhblk", 9604, "\xe2\x96\x84"),
+ ENTITY_DEF("capcap", 10827, "\xe2\xa9\x8b"),
+ ENTITY_DEF("midast", 42, "\x2a"),
+ ENTITY_DEF("lscr", 120001, "\xf0\x9d\x93\x81"),
+ ENTITY_DEF("nGt", 8811, "\xe2\x89\xab\xe2\x83\x92"),
+ ENTITY_DEF_HEUR("Euml", 203, "\xc3\x8b"),
+ ENTITY_DEF("blacktriangledown", 9662, "\xe2\x96\xbe"),
+ ENTITY_DEF("Rcy", 1056, "\xd0\xa0"),
+ ENTITY_DEF("dfisht", 10623, "\xe2\xa5\xbf"),
+ ENTITY_DEF("dashv", 8867, "\xe2\x8a\xa3"),
+ ENTITY_DEF("ast", 42, "\x2a"),
+ ENTITY_DEF("ContourIntegral", 8750, "\xe2\x88\xae"),
+ ENTITY_DEF("Ofr", 120082, "\xf0\x9d\x94\x92"),
+ ENTITY_DEF("Lcy", 1051, "\xd0\x9b"),
+ ENTITY_DEF("nltrie", 8940, "\xe2\x8b\xac"),
+ ENTITY_DEF("ShortUpArrow", 8593, "\xe2\x86\x91"),
+ ENTITY_DEF("acy", 1072, "\xd0\xb0"),
+ ENTITY_DEF("rightarrow", 8594, "\xe2\x86\x92"),
+ ENTITY_DEF("UnderBar", 95, "\x5f"),
+ ENTITY_DEF("LongLeftArrow", 10229, "\xe2\x9f\xb5"),
+ ENTITY_DEF("andd", 10844, "\xe2\xa9\x9c"),
+ ENTITY_DEF("xlarr", 10229, "\xe2\x9f\xb5"),
+ ENTITY_DEF("percnt", 37, "\x25"),
+ ENTITY_DEF("rharu", 8640, "\xe2\x87\x80"),
+ ENTITY_DEF("plusdo", 8724, "\xe2\x88\x94"),
+ ENTITY_DEF("TScy", 1062, "\xd0\xa6"),
+ ENTITY_DEF("kcy", 1082, "\xd0\xba"),
+ ENTITY_DEF("boxVR", 9568, "\xe2\x95\xa0"),
+ ENTITY_DEF("looparrowleft", 8619, "\xe2\x86\xab"),
+ ENTITY_DEF("scirc", 349, "\xc5\x9d"),
+ ENTITY_DEF("drcorn", 8991, "\xe2\x8c\x9f"),
+ ENTITY_DEF("iiota", 8489, "\xe2\x84\xa9"),
+ ENTITY_DEF("Zcy", 1047, "\xd0\x97"),
+ ENTITY_DEF("frac58", 8541, "\xe2\x85\x9d"),
+ ENTITY_DEF("alpha", 945, "\xce\xb1"),
+ ENTITY_DEF("daleth", 8504, "\xe2\x84\xb8"),
+ ENTITY_DEF("gtreqless", 8923, "\xe2\x8b\x9b"),
+ ENTITY_DEF("tstrok", 359, "\xc5\xa7"),
+ ENTITY_DEF("plusb", 8862, "\xe2\x8a\x9e"),
+ ENTITY_DEF("odsold", 10684, "\xe2\xa6\xbc"),
+ ENTITY_DEF("varsupsetneqq", 10956, "\xe2\xab\x8c\xef\xb8\x80"),
+ ENTITY_DEF_HEUR("otilde", 245, "\xc3\xb5"),
+ ENTITY_DEF("gtcir", 10874, "\xe2\xa9\xba"),
+ ENTITY_DEF("lltri", 9722, "\xe2\x97\xba"),
+ ENTITY_DEF("rx", 8478, "\xe2\x84\x9e"),
+ ENTITY_DEF("ljcy", 1113, "\xd1\x99"),
+ ENTITY_DEF("parsim", 10995, "\xe2\xab\xb3"),
+ ENTITY_DEF("NotElement", 8713, "\xe2\x88\x89"),
+ ENTITY_DEF_HEUR("plusmn", 177, "\xc2\xb1"),
+ ENTITY_DEF("varsubsetneq", 8842, "\xe2\x8a\x8a\xef\xb8\x80"),
+ ENTITY_DEF("subset", 8834, "\xe2\x8a\x82"),
+ ENTITY_DEF("awint", 10769, "\xe2\xa8\x91"),
+ ENTITY_DEF("laemptyv", 10676, "\xe2\xa6\xb4"),
+ ENTITY_DEF("phiv", 981, "\xcf\x95"),
+ ENTITY_DEF("sfrown", 8994, "\xe2\x8c\xa2"),
+ ENTITY_DEF("DoubleUpDownArrow", 8661, "\xe2\x87\x95"),
+ ENTITY_DEF("lpar", 40, "\x28"),
+ ENTITY_DEF("frac45", 8536, "\xe2\x85\x98"),
+ ENTITY_DEF("rBarr", 10511, "\xe2\xa4\x8f"),
+ ENTITY_DEF("npolint", 10772, "\xe2\xa8\x94"),
+ ENTITY_DEF("emacr", 275, "\xc4\x93"),
+ ENTITY_DEF("maltese", 10016, "\xe2\x9c\xa0"),
+ ENTITY_DEF("PlusMinus", 177, "\xc2\xb1"),
+ ENTITY_DEF("ReverseEquilibrium", 8651, "\xe2\x87\x8b"),
+ ENTITY_DEF("oscr", 8500, "\xe2\x84\xb4"),
+ ENTITY_DEF("blacksquare", 9642, "\xe2\x96\xaa"),
+ ENTITY_DEF("TSHcy", 1035, "\xd0\x8b"),
+ ENTITY_DEF("gap", 10886, "\xe2\xaa\x86"),
+ ENTITY_DEF("xnis", 8955, "\xe2\x8b\xbb"),
+ ENTITY_DEF("Ll", 8920, "\xe2\x8b\x98"),
+ ENTITY_DEF("PrecedesEqual", 10927, "\xe2\xaa\xaf"),
+ ENTITY_DEF("incare", 8453, "\xe2\x84\x85"),
+ ENTITY_DEF("nharr", 8622, "\xe2\x86\xae"),
+ ENTITY_DEF("varnothing", 8709, "\xe2\x88\x85"),
+ ENTITY_DEF("ShortDownArrow", 8595, "\xe2\x86\x93"),
+ ENTITY_DEF_HEUR("nbsp", 160, " "),
+ ENTITY_DEF("asympeq", 8781, "\xe2\x89\x8d"),
+ ENTITY_DEF("rbrkslu", 10640, "\xe2\xa6\x90"),
+ ENTITY_DEF("rho", 961, "\xcf\x81"),
+ ENTITY_DEF("Mscr", 8499, "\xe2\x84\xb3"),
+ ENTITY_DEF_HEUR("eth", 240, "\xc3\xb0"),
+ ENTITY_DEF("suplarr", 10619, "\xe2\xa5\xbb"),
+ ENTITY_DEF("Tab", 9, "\x09"),
+ ENTITY_DEF("omicron", 959, "\xce\xbf"),
+ ENTITY_DEF("blacktriangle", 9652, "\xe2\x96\xb4"),
+ ENTITY_DEF("nldr", 8229, "\xe2\x80\xa5"),
+ ENTITY_DEF("downharpoonleft", 8643, "\xe2\x87\x83"),
+ ENTITY_DEF("circledcirc", 8858, "\xe2\x8a\x9a"),
+ ENTITY_DEF("leftleftarrows", 8647, "\xe2\x87\x87"),
+ ENTITY_DEF("NotHumpDownHump", 8782, "\xe2\x89\x8e\xcc\xb8"),
+ ENTITY_DEF("nvgt", 62, "\x3e\xe2\x83\x92"),
+ ENTITY_DEF("rhard", 8641, "\xe2\x87\x81"),
+ ENTITY_DEF("nGg", 8921, "\xe2\x8b\x99\xcc\xb8"),
+ ENTITY_DEF("lurdshar", 10570, "\xe2\xa5\x8a"),
+ ENTITY_DEF("cirE", 10691, "\xe2\xa7\x83"),
+ ENTITY_DEF("isinE", 8953, "\xe2\x8b\xb9"),
+ ENTITY_DEF("eparsl", 10723, "\xe2\xa7\xa3"),
+ ENTITY_DEF("RightAngleBracket", 10217, "\xe2\x9f\xa9"),
+ ENTITY_DEF("hcirc", 293, "\xc4\xa5"),
+ ENTITY_DEF("bumpeq", 8783, "\xe2\x89\x8f"),
+ ENTITY_DEF("cire", 8791, "\xe2\x89\x97"),
+ ENTITY_DEF("dotplus", 8724, "\xe2\x88\x94"),
+ ENTITY_DEF("itilde", 297, "\xc4\xa9"),
+ ENTITY_DEF("uwangle", 10663, "\xe2\xa6\xa7"),
+ ENTITY_DEF("rlhar", 8652, "\xe2\x87\x8c"),
+ ENTITY_DEF("rbrace", 125, "\x7d"),
+ ENTITY_DEF("mid", 8739, "\xe2\x88\xa3"),
+ ENTITY_DEF("el", 10905, "\xe2\xaa\x99"),
+ ENTITY_DEF("KJcy", 1036, "\xd0\x8c"),
+ ENTITY_DEF("odiv", 10808, "\xe2\xa8\xb8"),
+ ENTITY_DEF("amacr", 257, "\xc4\x81"),
+ ENTITY_DEF("qprime", 8279, "\xe2\x81\x97"),
+ ENTITY_DEF("tcedil", 355, "\xc5\xa3"),
+ ENTITY_DEF("UpArrowDownArrow", 8645, "\xe2\x87\x85"),
+ ENTITY_DEF("spades", 9824, "\xe2\x99\xa0"),
+ ENTITY_DEF("napos", 329, "\xc5\x89"),
+ ENTITY_DEF("straightepsilon", 1013, "\xcf\xb5"),
+ ENTITY_DEF("CupCap", 8781, "\xe2\x89\x8d"),
+ ENTITY_DEF("Oopf", 120134, "\xf0\x9d\x95\x86"),
+ ENTITY_DEF("sub", 8834, "\xe2\x8a\x82"),
+ ENTITY_DEF("ohm", 937, "\xce\xa9"),
+ ENTITY_DEF("UnderBrace", 9183, "\xe2\x8f\x9f"),
+ ENTITY_DEF("looparrowright", 8620, "\xe2\x86\xac"),
+ ENTITY_DEF("xotime", 10754, "\xe2\xa8\x82"),
+ ENTITY_DEF("ntgl", 8825, "\xe2\x89\xb9"),
+ ENTITY_DEF("minusdu", 10794, "\xe2\xa8\xaa"),
+ ENTITY_DEF("rarrb", 8677, "\xe2\x87\xa5"),
+ ENTITY_DEF("nvlArr", 10498, "\xe2\xa4\x82"),
+ ENTITY_DEF("triangle", 9653, "\xe2\x96\xb5"),
+ ENTITY_DEF("nacute", 324, "\xc5\x84"),
+ ENTITY_DEF("boxHD", 9574, "\xe2\x95\xa6"),
+ ENTITY_DEF("ratio", 8758, "\xe2\x88\xb6"),
+ ENTITY_DEF("larrsim", 10611, "\xe2\xa5\xb3"),
+ ENTITY_DEF("LessLess", 10913, "\xe2\xaa\xa1"),
+ ENTITY_DEF("yacy", 1103, "\xd1\x8f"),
+ ENTITY_DEF("ctdot", 8943, "\xe2\x8b\xaf"),
+ ENTITY_DEF("and", 8743, "\xe2\x88\xa7"),
+ ENTITY_DEF("lrtri", 8895, "\xe2\x8a\xbf"),
+ ENTITY_DEF("eDot", 8785, "\xe2\x89\x91"),
+ ENTITY_DEF("sqsub", 8847, "\xe2\x8a\x8f"),
+ ENTITY_DEF("real", 8476, "\xe2\x84\x9c"),
+ ENTITY_DEF("Dcy", 1044, "\xd0\x94"),
+ ENTITY_DEF("vartheta", 977, "\xcf\x91"),
+ ENTITY_DEF("nsub", 8836, "\xe2\x8a\x84"),
+ ENTITY_DEF("DownTee", 8868, "\xe2\x8a\xa4"),
+ ENTITY_DEF_HEUR("acute", 180, "\xc2\xb4"),
+ ENTITY_DEF("GreaterLess", 8823, "\xe2\x89\xb7"),
+ ENTITY_DEF("supplus", 10944, "\xe2\xab\x80"),
+ ENTITY_DEF("Vbar", 10987, "\xe2\xab\xab"),
+ ENTITY_DEF("divideontimes", 8903, "\xe2\x8b\x87"),
+ ENTITY_DEF("lsim", 8818, "\xe2\x89\xb2"),
+ ENTITY_DEF("nearhk", 10532, "\xe2\xa4\xa4"),
+ ENTITY_DEF("nLtv", 8810, "\xe2\x89\xaa\xcc\xb8"),
+ ENTITY_DEF("RuleDelayed", 10740, "\xe2\xa7\xb4"),
+ ENTITY_DEF("smile", 8995, "\xe2\x8c\xa3"),
+ ENTITY_DEF("coprod", 8720, "\xe2\x88\x90"),
+ ENTITY_DEF("imof", 8887, "\xe2\x8a\xb7"),
+ ENTITY_DEF("ecy", 1101, "\xd1\x8d"),
+ ENTITY_DEF("RightCeiling", 8969, "\xe2\x8c\x89"),
+ ENTITY_DEF("dlcorn", 8990, "\xe2\x8c\x9e"),
+ ENTITY_DEF("Nu", 925, "\xce\x9d"),
+ ENTITY_DEF("frac18", 8539, "\xe2\x85\x9b"),
+ ENTITY_DEF("diamond", 8900, "\xe2\x8b\x84"),
+ ENTITY_DEF("Icirc", 206, "\xc3\x8e"),
+ ENTITY_DEF("ngeq", 8817, "\xe2\x89\xb1"),
+ ENTITY_DEF("epsilon", 949, "\xce\xb5"),
+ ENTITY_DEF("fork", 8916, "\xe2\x8b\x94"),
+ ENTITY_DEF("xrarr", 10230, "\xe2\x9f\xb6"),
+ ENTITY_DEF("racute", 341, "\xc5\x95"),
+ ENTITY_DEF("ntlg", 8824, "\xe2\x89\xb8"),
+ ENTITY_DEF("xvee", 8897, "\xe2\x8b\x81"),
+ ENTITY_DEF("LeftArrowRightArrow", 8646, "\xe2\x87\x86"),
+ ENTITY_DEF("DownLeftRightVector", 10576, "\xe2\xa5\x90"),
+ ENTITY_DEF("Eacute", 201, "\xc3\x89"),
+ ENTITY_DEF("gimel", 8503, "\xe2\x84\xb7"),
+ ENTITY_DEF("rtimes", 8906, "\xe2\x8b\x8a"),
+ ENTITY_DEF("forall", 8704, "\xe2\x88\x80"),
+ ENTITY_DEF("DiacriticalDoubleAcute", 733, "\xcb\x9d"),
+ ENTITY_DEF("dArr", 8659, "\xe2\x87\x93"),
+ ENTITY_DEF("fallingdotseq", 8786, "\xe2\x89\x92"),
+ ENTITY_DEF("Aogon", 260, "\xc4\x84"),
+ ENTITY_DEF("PartialD", 8706, "\xe2\x88\x82"),
+ ENTITY_DEF("mapstoup", 8613, "\xe2\x86\xa5"),
+ ENTITY_DEF("die", 168, "\xc2\xa8"),
+ ENTITY_DEF("ngt", 8815, "\xe2\x89\xaf"),
+ ENTITY_DEF("vcy", 1074, "\xd0\xb2"),
+ ENTITY_DEF("fjlig", (unsigned) -1, "\x66\x6a"),
+ ENTITY_DEF("submult", 10945, "\xe2\xab\x81"),
+ ENTITY_DEF("ubrcy", 1118, "\xd1\x9e"),
+ ENTITY_DEF("ovbar", 9021, "\xe2\x8c\xbd"),
+ ENTITY_DEF("bsime", 8909, "\xe2\x8b\x8d"),
+ ENTITY_DEF("precnsim", 8936, "\xe2\x8b\xa8"),
+ ENTITY_DEF("DiacriticalTilde", 732, "\xcb\x9c"),
+ ENTITY_DEF("cwint", 8753, "\xe2\x88\xb1"),
+ ENTITY_DEF("Scy", 1057, "\xd0\xa1"),
+ ENTITY_DEF("NotGreaterEqual", 8817, "\xe2\x89\xb1"),
+ ENTITY_DEF("boxUR", 9562, "\xe2\x95\x9a"),
+ ENTITY_DEF("LessSlantEqual", 10877, "\xe2\xa9\xbd"),
+ ENTITY_DEF("Barwed", 8966, "\xe2\x8c\x86"),
+ ENTITY_DEF("supdot", 10942, "\xe2\xaa\xbe"),
+ ENTITY_DEF("gel", 8923, "\xe2\x8b\x9b"),
+ ENTITY_DEF("iscr", 119998, "\xf0\x9d\x92\xbe"),
+ ENTITY_DEF("doublebarwedge", 8966, "\xe2\x8c\x86"),
+ ENTITY_DEF("Idot", 304, "\xc4\xb0"),
+ ENTITY_DEF("DoubleDot", 168, "\xc2\xa8"),
+ ENTITY_DEF("rsquo", 8217, "\xe2\x80\x99"),
+ ENTITY_DEF("subsetneqq", 10955, "\xe2\xab\x8b"),
+ ENTITY_DEF("UpEquilibrium", 10606, "\xe2\xa5\xae"),
+ ENTITY_DEF("copysr", 8471, "\xe2\x84\x97"),
+ ENTITY_DEF("RightDoubleBracket", 10215, "\xe2\x9f\xa7"),
+ ENTITY_DEF("LeftRightVector", 10574, "\xe2\xa5\x8e"),
+ ENTITY_DEF("DownLeftVectorBar", 10582, "\xe2\xa5\x96"),
+ ENTITY_DEF("suphsub", 10967, "\xe2\xab\x97"),
+ ENTITY_DEF_HEUR("cedil", 184, "\xc2\xb8"),
+ ENTITY_DEF("prurel", 8880, "\xe2\x8a\xb0"),
+ ENTITY_DEF("imagpart", 8465, "\xe2\x84\x91"),
+ ENTITY_DEF("Hscr", 8459, "\xe2\x84\x8b"),
+ ENTITY_DEF("jmath", 567, "\xc8\xb7"),
+ ENTITY_DEF("nrtrie", 8941, "\xe2\x8b\xad"),
+ ENTITY_DEF("nsup", 8837, "\xe2\x8a\x85"),
+ ENTITY_DEF("Ubrcy", 1038, "\xd0\x8e"),
+ ENTITY_DEF("succnsim", 8937, "\xe2\x8b\xa9"),
+ ENTITY_DEF("nesim", 8770, "\xe2\x89\x82\xcc\xb8"),
+ ENTITY_DEF("varepsilon", 1013, "\xcf\xb5"),
+ ENTITY_DEF("DoubleRightTee", 8872, "\xe2\x8a\xa8"),
+ ENTITY_DEF_HEUR("not", 172, "\xc2\xac"),
+ ENTITY_DEF("lesdot", 10879, "\xe2\xa9\xbf"),
+ ENTITY_DEF("backepsilon", 1014, "\xcf\xb6"),
+ ENTITY_DEF("srarr", 8594, "\xe2\x86\x92"),
+ ENTITY_DEF("varsubsetneqq", 10955, "\xe2\xab\x8b\xef\xb8\x80"),
+ ENTITY_DEF("sqcap", 8851, "\xe2\x8a\x93"),
+ ENTITY_DEF("rightleftarrows", 8644, "\xe2\x87\x84"),
+ ENTITY_DEF("diams", 9830, "\xe2\x99\xa6"),
+ ENTITY_DEF("boxdR", 9554, "\xe2\x95\x92"),
+ ENTITY_DEF("ngeqslant", 10878, "\xe2\xa9\xbe\xcc\xb8"),
+ ENTITY_DEF("boxDR", 9556, "\xe2\x95\x94"),
+ ENTITY_DEF("sext", 10038, "\xe2\x9c\xb6"),
+ ENTITY_DEF("backsim", 8765, "\xe2\x88\xbd"),
+ ENTITY_DEF("nfr", 120107, "\xf0\x9d\x94\xab"),
+ ENTITY_DEF("CloseCurlyDoubleQuote", 8221, "\xe2\x80\x9d"),
+ ENTITY_DEF("npart", 8706, "\xe2\x88\x82\xcc\xb8"),
+ ENTITY_DEF("dharl", 8643, "\xe2\x87\x83"),
+ ENTITY_DEF("NewLine", 10, "\x0a"),
+ ENTITY_DEF("bigotimes", 10754, "\xe2\xa8\x82"),
+ ENTITY_DEF("lAtail", 10523, "\xe2\xa4\x9b"),
+ ENTITY_DEF_HEUR("frac14", 188, "\xc2\xbc"),
+ ENTITY_DEF("or", 8744, "\xe2\x88\xa8"),
+ ENTITY_DEF("subedot", 10947, "\xe2\xab\x83"),
+ ENTITY_DEF("nmid", 8740, "\xe2\x88\xa4"),
+ ENTITY_DEF("DownArrowUpArrow", 8693, "\xe2\x87\xb5"),
+ ENTITY_DEF("icy", 1080, "\xd0\xb8"),
+ ENTITY_DEF("num", 35, "\x23"),
+ ENTITY_DEF("Gdot", 288, "\xc4\xa0"),
+ ENTITY_DEF("urcrop", 8974, "\xe2\x8c\x8e"),
+ ENTITY_DEF("epsiv", 1013, "\xcf\xb5"),
+ ENTITY_DEF("topcir", 10993, "\xe2\xab\xb1"),
+ ENTITY_DEF("ne", 8800, "\xe2\x89\xa0"),
+ ENTITY_DEF("osol", 8856, "\xe2\x8a\x98"),
+ ENTITY_DEF_HEUR("amp", 38, "\x26"),
+ ENTITY_DEF("ncap", 10819, "\xe2\xa9\x83"),
+ ENTITY_DEF("Sscr", 119982, "\xf0\x9d\x92\xae"),
+ ENTITY_DEF("sung", 9834, "\xe2\x99\xaa"),
+ ENTITY_DEF("ltri", 9667, "\xe2\x97\x83"),
+ ENTITY_DEF("frac25", 8534, "\xe2\x85\x96"),
+ ENTITY_DEF("DZcy", 1039, "\xd0\x8f"),
+ ENTITY_DEF("RightUpVector", 8638, "\xe2\x86\xbe"),
+ ENTITY_DEF("rsquor", 8217, "\xe2\x80\x99"),
+ ENTITY_DEF("uplus", 8846, "\xe2\x8a\x8e"),
+ ENTITY_DEF("triangleright", 9657, "\xe2\x96\xb9"),
+ ENTITY_DEF("lAarr", 8666, "\xe2\x87\x9a"),
+ ENTITY_DEF("HilbertSpace", 8459, "\xe2\x84\x8b"),
+ ENTITY_DEF("there4", 8756, "\xe2\x88\xb4"),
+ ENTITY_DEF("vscr", 120011, "\xf0\x9d\x93\x8b"),
+ ENTITY_DEF("cirscir", 10690, "\xe2\xa7\x82"),
+ ENTITY_DEF("roarr", 8702, "\xe2\x87\xbe"),
+ ENTITY_DEF("hslash", 8463, "\xe2\x84\x8f"),
+ ENTITY_DEF("supdsub", 10968, "\xe2\xab\x98"),
+ ENTITY_DEF("simg", 10910, "\xe2\xaa\x9e"),
+ ENTITY_DEF("trade", 8482, "\xe2\x84\xa2"),
+ ENTITY_DEF("searrow", 8600, "\xe2\x86\x98"),
+ ENTITY_DEF("DownLeftVector", 8637, "\xe2\x86\xbd"),
+ ENTITY_DEF("FilledSmallSquare", 9724, "\xe2\x97\xbc"),
+ ENTITY_DEF("prod", 8719, "\xe2\x88\x8f"),
+ ENTITY_DEF("oror", 10838, "\xe2\xa9\x96"),
+ ENTITY_DEF("udarr", 8645, "\xe2\x87\x85"),
+ ENTITY_DEF("jsercy", 1112, "\xd1\x98"),
+ ENTITY_DEF("tprime", 8244, "\xe2\x80\xb4"),
+ ENTITY_DEF("bprime", 8245, "\xe2\x80\xb5"),
+ ENTITY_DEF("malt", 10016, "\xe2\x9c\xa0"),
+ ENTITY_DEF("bigcup", 8899, "\xe2\x8b\x83"),
+ ENTITY_DEF("oint", 8750, "\xe2\x88\xae"),
+ ENTITY_DEF("female", 9792, "\xe2\x99\x80"),
+ ENTITY_DEF("omacr", 333, "\xc5\x8d"),
+ ENTITY_DEF("SquareSubsetEqual", 8849, "\xe2\x8a\x91"),
+ ENTITY_DEF("SucceedsEqual", 10928, "\xe2\xaa\xb0"),
+ ENTITY_DEF("plusacir", 10787, "\xe2\xa8\xa3"),
+ ENTITY_DEF("Gcirc", 284, "\xc4\x9c"),
+ ENTITY_DEF("lesdotor", 10883, "\xe2\xaa\x83"),
+ ENTITY_DEF("escr", 8495, "\xe2\x84\xaf"),
+ ENTITY_DEF_HEUR("THORN", 222, "\xc3\x9e"),
+ ENTITY_DEF("UpArrowBar", 10514, "\xe2\xa4\x92"),
+ ENTITY_DEF("nvrtrie", 8885, "\xe2\x8a\xb5\xe2\x83\x92"),
+ ENTITY_DEF("varkappa", 1008, "\xcf\xb0"),
+ ENTITY_DEF("NotReverseElement", 8716, "\xe2\x88\x8c"),
+ ENTITY_DEF("zdot", 380, "\xc5\xbc"),
+ ENTITY_DEF("ExponentialE", 8519, "\xe2\x85\x87"),
+ ENTITY_DEF("lesseqgtr", 8922, "\xe2\x8b\x9a"),
+ ENTITY_DEF("cscr", 119992, "\xf0\x9d\x92\xb8"),
+ ENTITY_DEF("Dscr", 119967, "\xf0\x9d\x92\x9f"),
+ ENTITY_DEF("lthree", 8907, "\xe2\x8b\x8b"),
+ ENTITY_DEF("Ccedil", 199, "\xc3\x87"),
+ ENTITY_DEF("nge", 8817, "\xe2\x89\xb1"),
+ ENTITY_DEF("UpperLeftArrow", 8598, "\xe2\x86\x96"),
+ ENTITY_DEF("vDash", 8872, "\xe2\x8a\xa8"),
+ ENTITY_DEF("efDot", 8786, "\xe2\x89\x92"),
+ ENTITY_DEF("telrec", 8981, "\xe2\x8c\x95"),
+ ENTITY_DEF("vellip", 8942, "\xe2\x8b\xae"),
+ ENTITY_DEF("nrArr", 8655, "\xe2\x87\x8f"),
+ ENTITY_DEF_HEUR("ugrave", 249, "\xc3\xb9"),
+ ENTITY_DEF("uring", 367, "\xc5\xaf"),
+ ENTITY_DEF("Bernoullis", 8492, "\xe2\x84\xac"),
+ ENTITY_DEF("nles", 10877, "\xe2\xa9\xbd\xcc\xb8"),
+ ENTITY_DEF_HEUR("macr", 175, "\xc2\xaf"),
+ ENTITY_DEF("boxuR", 9560, "\xe2\x95\x98"),
+ ENTITY_DEF("clubsuit", 9827, "\xe2\x99\xa3"),
+ ENTITY_DEF("rightarrowtail", 8611, "\xe2\x86\xa3"),
+ ENTITY_DEF("epar", 8917, "\xe2\x8b\x95"),
+ ENTITY_DEF("ltcc", 10918, "\xe2\xaa\xa6"),
+ ENTITY_DEF("twoheadleftarrow", 8606, "\xe2\x86\x9e"),
+ ENTITY_DEF("aleph", 8501, "\xe2\x84\xb5"),
+ ENTITY_DEF("Colon", 8759, "\xe2\x88\xb7"),
+ ENTITY_DEF("vltri", 8882, "\xe2\x8a\xb2"),
+ ENTITY_DEF("quaternions", 8461, "\xe2\x84\x8d"),
+ ENTITY_DEF("rfr", 120111, "\xf0\x9d\x94\xaf"),
+ ENTITY_DEF_HEUR("Ouml", 214, "\xc3\x96"),
+ ENTITY_DEF("rsh", 8625, "\xe2\x86\xb1"),
+ ENTITY_DEF("emptyv", 8709, "\xe2\x88\x85"),
+ ENTITY_DEF("sqsup", 8848, "\xe2\x8a\x90"),
+ ENTITY_DEF("marker", 9646, "\xe2\x96\xae"),
+ ENTITY_DEF("Efr", 120072, "\xf0\x9d\x94\x88"),
+ ENTITY_DEF("DotEqual", 8784, "\xe2\x89\x90"),
+ ENTITY_DEF("eqsim", 8770, "\xe2\x89\x82"),
+ ENTITY_DEF("NotSucceedsEqual", 10928, "\xe2\xaa\xb0\xcc\xb8"),
+ ENTITY_DEF("primes", 8473, "\xe2\x84\x99"),
+ ENTITY_DEF_HEUR("times", 215, "\xc3\x97"),
+ ENTITY_DEF("rangd", 10642, "\xe2\xa6\x92"),
+ ENTITY_DEF("rightharpoonup", 8640, "\xe2\x87\x80"),
+ ENTITY_DEF("lrhard", 10605, "\xe2\xa5\xad"),
+ ENTITY_DEF("ape", 8778, "\xe2\x89\x8a"),
+ ENTITY_DEF("varsupsetneq", 8843, "\xe2\x8a\x8b\xef\xb8\x80"),
+ ENTITY_DEF("larrlp", 8619, "\xe2\x86\xab"),
+ ENTITY_DEF("NotPrecedesEqual", 10927, "\xe2\xaa\xaf\xcc\xb8"),
+ ENTITY_DEF("ulcorner", 8988, "\xe2\x8c\x9c"),
+ ENTITY_DEF("acd", 8767, "\xe2\x88\xbf"),
+ ENTITY_DEF("Hacek", 711, "\xcb\x87"),
+ ENTITY_DEF("xuplus", 10756, "\xe2\xa8\x84"),
+ ENTITY_DEF("therefore", 8756, "\xe2\x88\xb4"),
+ ENTITY_DEF("YIcy", 1031, "\xd0\x87"),
+ ENTITY_DEF("Tfr", 120087, "\xf0\x9d\x94\x97"),
+ ENTITY_DEF("Jcirc", 308, "\xc4\xb4"),
+ ENTITY_DEF("LessGreater", 8822, "\xe2\x89\xb6"),
+ ENTITY_DEF("Uring", 366, "\xc5\xae"),
+ ENTITY_DEF("Ugrave", 217, "\xc3\x99"),
+ ENTITY_DEF("rarr", 8594, "\xe2\x86\x92"),
+ ENTITY_DEF("wopf", 120168, "\xf0\x9d\x95\xa8"),
+ ENTITY_DEF("imath", 305, "\xc4\xb1"),
+ ENTITY_DEF("Yopf", 120144, "\xf0\x9d\x95\x90"),
+ ENTITY_DEF("colone", 8788, "\xe2\x89\x94"),
+ ENTITY_DEF("csube", 10961, "\xe2\xab\x91"),
+ ENTITY_DEF("odash", 8861, "\xe2\x8a\x9d"),
+ ENTITY_DEF("olarr", 8634, "\xe2\x86\xba"),
+ ENTITY_DEF("angrt", 8735, "\xe2\x88\x9f"),
+ ENTITY_DEF("NotLeftTriangleBar", 10703, "\xe2\xa7\x8f\xcc\xb8"),
+ ENTITY_DEF("GreaterEqual", 8805, "\xe2\x89\xa5"),
+ ENTITY_DEF("scnap", 10938, "\xe2\xaa\xba"),
+ ENTITY_DEF("pi", 960, "\xcf\x80"),
+ ENTITY_DEF("lesg", 8922, "\xe2\x8b\x9a\xef\xb8\x80"),
+ ENTITY_DEF("orderof", 8500, "\xe2\x84\xb4"),
+ ENTITY_DEF_HEUR("uacute", 250, "\xc3\xba"),
+ ENTITY_DEF("Barv", 10983, "\xe2\xab\xa7"),
+ ENTITY_DEF("Theta", 920, "\xce\x98"),
+ ENTITY_DEF("leftrightsquigarrow", 8621, "\xe2\x86\xad"),
+ ENTITY_DEF("Atilde", 195, "\xc3\x83"),
+ ENTITY_DEF("cupdot", 8845, "\xe2\x8a\x8d"),
+ ENTITY_DEF("ntriangleright", 8939, "\xe2\x8b\xab"),
+ ENTITY_DEF("measuredangle", 8737, "\xe2\x88\xa1"),
+ ENTITY_DEF("jscr", 119999, "\xf0\x9d\x92\xbf"),
+ ENTITY_DEF("inodot", 305, "\xc4\xb1"),
+ ENTITY_DEF("mopf", 120158, "\xf0\x9d\x95\x9e"),
+ ENTITY_DEF("hkswarow", 10534, "\xe2\xa4\xa6"),
+ ENTITY_DEF("lopar", 10629, "\xe2\xa6\x85"),
+ ENTITY_DEF("thksim", 8764, "\xe2\x88\xbc"),
+ ENTITY_DEF("bkarow", 10509, "\xe2\xa4\x8d"),
+ ENTITY_DEF("rarrfs", 10526, "\xe2\xa4\x9e"),
+ ENTITY_DEF("ntrianglelefteq", 8940, "\xe2\x8b\xac"),
+ ENTITY_DEF("Bscr", 8492, "\xe2\x84\xac"),
+ ENTITY_DEF("topf", 120165, "\xf0\x9d\x95\xa5"),
+ ENTITY_DEF("Uacute", 218, "\xc3\x9a"),
+ ENTITY_DEF("lap", 10885, "\xe2\xaa\x85"),
+ ENTITY_DEF("djcy", 1106, "\xd1\x92"),
+ ENTITY_DEF("bopf", 120147, "\xf0\x9d\x95\x93"),
+ ENTITY_DEF("empty", 8709, "\xe2\x88\x85"),
+ ENTITY_DEF("LeftAngleBracket", 10216, "\xe2\x9f\xa8"),
+ ENTITY_DEF("Imacr", 298, "\xc4\xaa"),
+ ENTITY_DEF("ltcir", 10873, "\xe2\xa9\xb9"),
+ ENTITY_DEF("trisb", 10701, "\xe2\xa7\x8d"),
+ ENTITY_DEF("gjcy", 1107, "\xd1\x93"),
+ ENTITY_DEF("pr", 8826, "\xe2\x89\xba"),
+ ENTITY_DEF("Mu", 924, "\xce\x9c"),
+ ENTITY_DEF("ogon", 731, "\xcb\x9b"),
+ ENTITY_DEF("pertenk", 8241, "\xe2\x80\xb1"),
+ ENTITY_DEF("plustwo", 10791, "\xe2\xa8\xa7"),
+ ENTITY_DEF("Vfr", 120089, "\xf0\x9d\x94\x99"),
+ ENTITY_DEF("ApplyFunction", 8289, "\xe2\x81\xa1"),
+ ENTITY_DEF("Sub", 8912, "\xe2\x8b\x90"),
+ ENTITY_DEF("DoubleLeftRightArrow", 8660, "\xe2\x87\x94"),
+ ENTITY_DEF("Lmidot", 319, "\xc4\xbf"),
+ ENTITY_DEF("nwarrow", 8598, "\xe2\x86\x96"),
+ ENTITY_DEF("angrtvbd", 10653, "\xe2\xa6\x9d"),
+ ENTITY_DEF("fcy", 1092, "\xd1\x84"),
+ ENTITY_DEF("ltlarr", 10614, "\xe2\xa5\xb6"),
+ ENTITY_DEF("CircleMinus", 8854, "\xe2\x8a\x96"),
+ ENTITY_DEF("angmsdab", 10665, "\xe2\xa6\xa9"),
+ ENTITY_DEF("wedgeq", 8793, "\xe2\x89\x99"),
+ ENTITY_DEF("iogon", 303, "\xc4\xaf"),
+ ENTITY_DEF_HEUR("laquo", 171, "\xc2\xab"),
+ ENTITY_DEF("NestedGreaterGreater", 8811, "\xe2\x89\xab"),
+ ENTITY_DEF("UnionPlus", 8846, "\xe2\x8a\x8e"),
+ ENTITY_DEF("CircleDot", 8857, "\xe2\x8a\x99"),
+ ENTITY_DEF("coloneq", 8788, "\xe2\x89\x94"),
+ ENTITY_DEF("csupe", 10962, "\xe2\xab\x92"),
+ ENTITY_DEF("tcaron", 357, "\xc5\xa5"),
+ ENTITY_DEF("GreaterTilde", 8819, "\xe2\x89\xb3"),
+ ENTITY_DEF("Map", 10501, "\xe2\xa4\x85"),
+ ENTITY_DEF("DoubleLongLeftArrow", 10232, "\xe2\x9f\xb8"),
+ ENTITY_DEF("Uparrow", 8657, "\xe2\x87\x91"),
+ ENTITY_DEF("scy", 1089, "\xd1\x81"),
+ ENTITY_DEF("llarr", 8647, "\xe2\x87\x87"),
+ ENTITY_DEF("rangle", 10217, "\xe2\x9f\xa9"),
+ ENTITY_DEF("sstarf", 8902, "\xe2\x8b\x86"),
+ ENTITY_DEF("InvisibleTimes", 8290, "\xe2\x81\xa2"),
+ ENTITY_DEF("egsdot", 10904, "\xe2\xaa\x98"),
+ ENTITY_DEF("target", 8982, "\xe2\x8c\x96"),
+ ENTITY_DEF("lesges", 10899, "\xe2\xaa\x93"),
+ ENTITY_DEF_HEUR("curren", 164, "\xc2\xa4"),
+ ENTITY_DEF("yopf", 120170, "\xf0\x9d\x95\xaa"),
+ ENTITY_DEF("frac23", 8532, "\xe2\x85\x94"),
+ ENTITY_DEF("NotSucceedsTilde", 8831, "\xe2\x89\xbf\xcc\xb8"),
+ ENTITY_DEF("napprox", 8777, "\xe2\x89\x89"),
+ ENTITY_DEF("odblac", 337, "\xc5\x91"),
+ ENTITY_DEF("gammad", 989, "\xcf\x9d"),
+ ENTITY_DEF("dscr", 119993, "\xf0\x9d\x92\xb9"),
+ ENTITY_DEF("SupersetEqual", 8839, "\xe2\x8a\x87"),
+ ENTITY_DEF("squf", 9642, "\xe2\x96\xaa"),
+ ENTITY_DEF("Because", 8757, "\xe2\x88\xb5"),
+ ENTITY_DEF("sccue", 8829, "\xe2\x89\xbd"),
+ ENTITY_DEF("KHcy", 1061, "\xd0\xa5"),
+ ENTITY_DEF("Wcirc", 372, "\xc5\xb4"),
+ ENTITY_DEF("uparrow", 8593, "\xe2\x86\x91"),
+ ENTITY_DEF("lessgtr", 8822, "\xe2\x89\xb6"),
+ ENTITY_DEF("thickapprox", 8776, "\xe2\x89\x88"),
+ ENTITY_DEF("lbrksld", 10639, "\xe2\xa6\x8f"),
+ ENTITY_DEF_HEUR("oslash", 248, "\xc3\xb8"),
+ ENTITY_DEF("NotCupCap", 8813, "\xe2\x89\xad"),
+ ENTITY_DEF("elinters", 9191, "\xe2\x8f\xa7"),
+ ENTITY_DEF("Assign", 8788, "\xe2\x89\x94"),
+ ENTITY_DEF("ClockwiseContourIntegral", 8754, "\xe2\x88\xb2"),
+ ENTITY_DEF("lfisht", 10620, "\xe2\xa5\xbc"),
+ ENTITY_DEF("DownArrow", 8595, "\xe2\x86\x93"),
+ ENTITY_DEF("Zdot", 379, "\xc5\xbb"),
+ ENTITY_DEF("xscr", 120013, "\xf0\x9d\x93\x8d"),
+ ENTITY_DEF("DiacriticalGrave", 96, "\x60"),
+ ENTITY_DEF("DoubleLongLeftRightArrow", 10234, "\xe2\x9f\xba"),
+ ENTITY_DEF("angle", 8736, "\xe2\x88\xa0"),
+ ENTITY_DEF("race", 8765, "\xe2\x88\xbd\xcc\xb1"),
+ ENTITY_DEF("Ascr", 119964, "\xf0\x9d\x92\x9c"),
+ ENTITY_DEF("Xscr", 119987, "\xf0\x9d\x92\xb3"),
+ ENTITY_DEF_HEUR("acirc", 226, "\xc3\xa2"),
+ ENTITY_DEF("otimesas", 10806, "\xe2\xa8\xb6"),
+ ENTITY_DEF("gscr", 8458, "\xe2\x84\x8a"),
+ ENTITY_DEF("gcy", 1075, "\xd0\xb3"),
+ ENTITY_DEF("angmsdag", 10670, "\xe2\xa6\xae"),
+ ENTITY_DEF("tshcy", 1115, "\xd1\x9b"),
+ ENTITY_DEF("Acy", 1040, "\xd0\x90"),
+ ENTITY_DEF("NotGreaterLess", 8825, "\xe2\x89\xb9"),
+ ENTITY_DEF("dtdot", 8945, "\xe2\x8b\xb1"),
+ ENTITY_DEF_HEUR("quot", 34, "\x22"),
+ ENTITY_DEF_HEUR("micro", 181, "\xc2\xb5"),
+ ENTITY_DEF("simplus", 10788, "\xe2\xa8\xa4"),
+ ENTITY_DEF("nsupseteq", 8841, "\xe2\x8a\x89"),
+ ENTITY_DEF("Ufr", 120088, "\xf0\x9d\x94\x98"),
+ ENTITY_DEF("Pr", 10939, "\xe2\xaa\xbb"),
+ ENTITY_DEF("napid", 8779, "\xe2\x89\x8b\xcc\xb8"),
+ ENTITY_DEF("rceil", 8969, "\xe2\x8c\x89"),
+ ENTITY_DEF("boxtimes", 8864, "\xe2\x8a\xa0"),
+ ENTITY_DEF("erarr", 10609, "\xe2\xa5\xb1"),
+ ENTITY_DEF("downdownarrows", 8650, "\xe2\x87\x8a"),
+ ENTITY_DEF("Kfr", 120078, "\xf0\x9d\x94\x8e"),
+ ENTITY_DEF("mho", 8487, "\xe2\x84\xa7"),
+ ENTITY_DEF("scpolint", 10771, "\xe2\xa8\x93"),
+ ENTITY_DEF("vArr", 8661, "\xe2\x87\x95"),
+ ENTITY_DEF("Ccaron", 268, "\xc4\x8c"),
+ ENTITY_DEF("NotRightTriangle", 8939, "\xe2\x8b\xab"),
+ ENTITY_DEF("topbot", 9014, "\xe2\x8c\xb6"),
+ ENTITY_DEF("qopf", 120162, "\xf0\x9d\x95\xa2"),
+ ENTITY_DEF("eogon", 281, "\xc4\x99"),
+ ENTITY_DEF("luruhar", 10598, "\xe2\xa5\xa6"),
+ ENTITY_DEF("gtdot", 8919, "\xe2\x8b\x97"),
+ ENTITY_DEF("Egrave", 200, "\xc3\x88"),
+ ENTITY_DEF("roplus", 10798, "\xe2\xa8\xae"),
+ ENTITY_DEF("Intersection", 8898, "\xe2\x8b\x82"),
+ ENTITY_DEF("Uarr", 8607, "\xe2\x86\x9f"),
+ ENTITY_DEF("dcy", 1076, "\xd0\xb4"),
+ ENTITY_DEF("boxvl", 9508, "\xe2\x94\xa4"),
+ ENTITY_DEF("RightArrowBar", 8677, "\xe2\x87\xa5"),
+ ENTITY_DEF_HEUR("yuml", 255, "\xc3\xbf"),
+ ENTITY_DEF("parallel", 8741, "\xe2\x88\xa5"),
+ ENTITY_DEF("succneqq", 10934, "\xe2\xaa\xb6"),
+ ENTITY_DEF("bemptyv", 10672, "\xe2\xa6\xb0"),
+ ENTITY_DEF("starf", 9733, "\xe2\x98\x85"),
+ ENTITY_DEF("OverBar", 8254, "\xe2\x80\xbe"),
+ ENTITY_DEF("Alpha", 913, "\xce\x91"),
+ ENTITY_DEF("LeftUpVectorBar", 10584, "\xe2\xa5\x98"),
+ ENTITY_DEF("ufr", 120114, "\xf0\x9d\x94\xb2"),
+ ENTITY_DEF("swarhk", 10534, "\xe2\xa4\xa6"),
+ ENTITY_DEF("GreaterEqualLess", 8923, "\xe2\x8b\x9b"),
+ ENTITY_DEF("sscr", 120008, "\xf0\x9d\x93\x88"),
+ ENTITY_DEF("Pi", 928, "\xce\xa0"),
+ ENTITY_DEF("boxh", 9472, "\xe2\x94\x80"),
+ ENTITY_DEF("frac16", 8537, "\xe2\x85\x99"),
+ ENTITY_DEF("lbrack", 91, "\x5b"),
+ ENTITY_DEF("vert", 124, "\x7c"),
+ ENTITY_DEF("precneqq", 10933, "\xe2\xaa\xb5"),
+ ENTITY_DEF("NotGreaterSlantEqual", 10878, "\xe2\xa9\xbe\xcc\xb8"),
+ ENTITY_DEF("Omega", 937, "\xce\xa9"),
+ ENTITY_DEF("uarr", 8593, "\xe2\x86\x91"),
+ ENTITY_DEF("boxVr", 9567, "\xe2\x95\x9f"),
+ ENTITY_DEF("ruluhar", 10600, "\xe2\xa5\xa8"),
+ ENTITY_DEF("ShortLeftArrow", 8592, "\xe2\x86\x90"),
+ ENTITY_DEF("Qfr", 120084, "\xf0\x9d\x94\x94"),
+ ENTITY_DEF("olt", 10688, "\xe2\xa7\x80"),
+ ENTITY_DEF("nequiv", 8802, "\xe2\x89\xa2"),
+ ENTITY_DEF("fscr", 119995, "\xf0\x9d\x92\xbb"),
+ ENTITY_DEF("rarrhk", 8618, "\xe2\x86\xaa"),
+ ENTITY_DEF("nsqsupe", 8931, "\xe2\x8b\xa3"),
+ ENTITY_DEF("nsubseteq", 8840, "\xe2\x8a\x88"),
+ ENTITY_DEF("numero", 8470, "\xe2\x84\x96"),
+ ENTITY_DEF("emsp14", 8197, "\xe2\x80\x85"),
+ ENTITY_DEF("gl", 8823, "\xe2\x89\xb7"),
+ ENTITY_DEF("ocirc", 244, "\xc3\xb4"),
+ ENTITY_DEF("weierp", 8472, "\xe2\x84\x98"),
+ ENTITY_DEF("boxvL", 9569, "\xe2\x95\xa1"),
+ ENTITY_DEF("RightArrowLeftArrow", 8644, "\xe2\x87\x84"),
+ ENTITY_DEF("Precedes", 8826, "\xe2\x89\xba"),
+ ENTITY_DEF("RightVector", 8640, "\xe2\x87\x80"),
+ ENTITY_DEF("xcup", 8899, "\xe2\x8b\x83"),
+ ENTITY_DEF("angmsdad", 10667, "\xe2\xa6\xab"),
+ ENTITY_DEF("gtrsim", 8819, "\xe2\x89\xb3"),
+ ENTITY_DEF("natural", 9838, "\xe2\x99\xae"),
+ ENTITY_DEF("nVdash", 8878, "\xe2\x8a\xae"),
+ ENTITY_DEF("RightTriangleEqual", 8885, "\xe2\x8a\xb5"),
+ ENTITY_DEF("dscy", 1109, "\xd1\x95"),
+ ENTITY_DEF("leftthreetimes", 8907, "\xe2\x8b\x8b"),
+ ENTITY_DEF("prsim", 8830, "\xe2\x89\xbe"),
+ ENTITY_DEF("Bcy", 1041, "\xd0\x91"),
+ ENTITY_DEF("Chi", 935, "\xce\xa7"),
+ ENTITY_DEF("timesb", 8864, "\xe2\x8a\xa0"),
+ ENTITY_DEF("Del", 8711, "\xe2\x88\x87"),
+ ENTITY_DEF("lmidot", 320, "\xc5\x80"),
+ ENTITY_DEF("RightDownVector", 8642, "\xe2\x87\x82"),
+ ENTITY_DEF("simdot", 10858, "\xe2\xa9\xaa"),
+ ENTITY_DEF("FilledVerySmallSquare", 9642, "\xe2\x96\xaa"),
+ ENTITY_DEF("NotLessSlantEqual", 10877, "\xe2\xa9\xbd\xcc\xb8"),
+ ENTITY_DEF("SucceedsTilde", 8831, "\xe2\x89\xbf"),
+ ENTITY_DEF("duarr", 8693, "\xe2\x87\xb5"),
+ ENTITY_DEF("apE", 10864, "\xe2\xa9\xb0"),
+ ENTITY_DEF("odot", 8857, "\xe2\x8a\x99"),
+ ENTITY_DEF("mldr", 8230, "\xe2\x80\xa6"),
+ ENTITY_DEF("Uarrocir", 10569, "\xe2\xa5\x89"),
+ ENTITY_DEF("nLl", 8920, "\xe2\x8b\x98\xcc\xb8"),
+ ENTITY_DEF("rarrpl", 10565, "\xe2\xa5\x85"),
+ ENTITY_DEF("cir", 9675, "\xe2\x97\x8b"),
+ ENTITY_DEF("blk14", 9617, "\xe2\x96\x91"),
+ ENTITY_DEF("VerticalLine", 124, "\x7c"),
+ ENTITY_DEF("jcy", 1081, "\xd0\xb9"),
+ ENTITY_DEF("filig", 64257, "\xef\xac\x81"),
+ ENTITY_DEF("LongRightArrow", 10230, "\xe2\x9f\xb6"),
+ ENTITY_DEF("beta", 946, "\xce\xb2"),
+ ENTITY_DEF("ccupssm", 10832, "\xe2\xa9\x90"),
+ ENTITY_DEF("supsub", 10964, "\xe2\xab\x94"),
+ ENTITY_DEF("spar", 8741, "\xe2\x88\xa5"),
+ ENTITY_DEF("Tstrok", 358, "\xc5\xa6"),
+ ENTITY_DEF("isinv", 8712, "\xe2\x88\x88"),
+ ENTITY_DEF("rightsquigarrow", 8605, "\xe2\x86\x9d"),
+ ENTITY_DEF("Diamond", 8900, "\xe2\x8b\x84"),
+ ENTITY_DEF("curlyeqsucc", 8927, "\xe2\x8b\x9f"),
+ ENTITY_DEF("ijlig", 307, "\xc4\xb3"),
+ ENTITY_DEF("puncsp", 8200, "\xe2\x80\x88"),
+ ENTITY_DEF("hamilt", 8459, "\xe2\x84\x8b"),
+ ENTITY_DEF("mapstoleft", 8612, "\xe2\x86\xa4"),
+ ENTITY_DEF("Copf", 8450, "\xe2\x84\x82"),
+ ENTITY_DEF("prnsim", 8936, "\xe2\x8b\xa8"),
+ ENTITY_DEF("DotDot", 8412, "\xe2\x83\x9c"),
+ ENTITY_DEF("lobrk", 10214, "\xe2\x9f\xa6"),
+ ENTITY_DEF("twoheadrightarrow", 8608, "\xe2\x86\xa0"),
+ ENTITY_DEF("ngE", 8807, "\xe2\x89\xa7\xcc\xb8"),
+ ENTITY_DEF("cylcty", 9005, "\xe2\x8c\xad"),
+ ENTITY_DEF("sube", 8838, "\xe2\x8a\x86"),
+ ENTITY_DEF("NotEqualTilde", 8770, "\xe2\x89\x82\xcc\xb8"),
+ ENTITY_DEF_HEUR("Yuml", 376, "\xc5\xb8"),
+ ENTITY_DEF("comp", 8705, "\xe2\x88\x81"),
+ ENTITY_DEF("dotminus", 8760, "\xe2\x88\xb8"),
+ ENTITY_DEF("crarr", 8629, "\xe2\x86\xb5"),
+ ENTITY_DEF("imped", 437, "\xc6\xb5"),
+ ENTITY_DEF("barwedge", 8965, "\xe2\x8c\x85"),
+ ENTITY_DEF("harrcir", 10568, "\xe2\xa5\x88")};
+
+class html_entities_storage {
+ ankerl::unordered_dense::map<std::string_view, html_entity_def> entity_by_name;
+ ankerl::unordered_dense::map<std::string_view, html_entity_def> entity_by_name_heur;
+ ankerl::unordered_dense::map<unsigned, html_entity_def> entity_by_id;
+
+public:
+ html_entities_storage()
+ {
+ auto nelts = G_N_ELEMENTS(html_entities_array);
+ entity_by_name.reserve(nelts);
+ entity_by_id.reserve(nelts);
+
+ for (const auto &e: html_entities_array) {
+ entity_by_name[e.name] = e;
+ entity_by_id[e.code] = e;
+
+ if (e.allow_heuristic) {
+ entity_by_name_heur[e.name] = e;
+ }
+ }
+ }
+
+ auto by_name(std::string_view name, bool use_heuristic = false) const -> const html_entity_def *
+ {
+ const decltype(entity_by_name) *htb;
+
+ if (use_heuristic) {
+ htb = &entity_by_name_heur;
+ }
+ else {
+ htb = &entity_by_name;
+ }
+ auto it = htb->find(name);
+
+ if (it != htb->end()) {
+ return &(it->second);
+ }
+
+ return nullptr;
+ }
+
+ auto by_id(int id) const -> const html_entity_def *
+ {
+ auto it = entity_by_id.find(id);
+ if (it != entity_by_id.end()) {
+ return &(it->second);
+ }
+
+ return nullptr;
+ }
+};
+
+static const html_entities_storage html_entities_defs;
+
+std::size_t
+decode_html_entitles_inplace(char *s, std::size_t len, bool norm_spaces)
+{
+ /*
+ * t - tortoise (destination ptr)
+ * h - hare (source ptr)
+ * e - begin of entity
+ */
+ char *t = s, *h = s, *e = s;
+ const gchar *end;
+ bool seen_hash = false, seen_hex = false;
+ enum {
+ do_undefined,
+ do_digits_only,
+ do_mixed,
+ } seen_digit_only;
+ enum class parser_state {
+ normal_content,
+ ampersand,
+ skip_multi_spaces,
+ skip_start_spaces,
+ } state = parser_state::normal_content;
+
+ end = s + len;
+
+ auto replace_named_entity = [&](const char *entity, std::size_t len) -> bool {
+ const auto *entity_def = html_entities_defs.by_name({entity,
+ (std::size_t)(h - entity)},
+ false);
+
+ auto replace_entity = [&]() -> void {
+ auto l = strlen(entity_def->replacement);
+ memcpy(t, entity_def->replacement, l);
+ t += l;
+ };
+
+ if (entity_def) {
+ replace_entity();
+ return true;
+ }
+ else {
+ /* Try heuristic */
+ auto heuristic_lookup_func = [&](std::size_t lookup_len) -> bool {
+ if (!entity_def && h - e > lookup_len) {
+ entity_def = html_entities_defs.by_name({entity, lookup_len}, true);
+
+ if (entity_def) {
+ replace_entity();
+ /* Adjust h back */
+ h = e + lookup_len;
+
+ return true;
+ }
+
+ entity_def = nullptr;
+ }
+
+ return false;
+ };
+
+ heuristic_lookup_func(5);
+ heuristic_lookup_func(4);
+ heuristic_lookup_func(3);
+ heuristic_lookup_func(2);
+
+ /* Leave undecoded */
+ if (!entity_def && (end - t > h - e)) {
+ memmove(t, e, h - e);
+ t += h - e;
+ }
+ else if (entity_def) {
+ return true;
+ }
+ }
+
+ return false;
+ };
+
+ /* Strtoul works merely for 0 terminated strings, so leave it alone... */
+ auto dec_to_int = [](const char *str, std::size_t len) -> std::optional<int> {
+ int n = 0;
+
+ /* Avoid INT_MIN overflow by moving to negative numbers */
+ while (len > 0 && g_ascii_isdigit(*str)) {
+ n = 10 * n - (*str++ - '0');
+ len--;
+ }
+
+ if (len == 0) {
+ return -(n);
+ }
+ else {
+ return std::nullopt;
+ }
+ };
+ auto hex_to_int = [](const char *str, std::size_t len) -> std::optional<int> {
+ int n = 0;
+
+ /* Avoid INT_MIN overflow by moving to negative numbers */
+ while (len > 0 && g_ascii_isxdigit(*str)) {
+ if (*str <= 0x39) {
+ n = 16 * n - (*str++ - '0');
+ }
+ else {
+ n = 16 * n - (((*str++) | ' ') - 'a' + 10);
+ }
+ len--;
+ }
+
+ if (len == 0) {
+ return -(n);
+ }
+ else {
+ return std::nullopt;
+ }
+ };
+ auto oct_to_int = [](const char *str, std::size_t len) -> std::optional<int> {
+ int n = 0;
+
+ /* Avoid INT_MIN overflow by moving to negative numbers */
+ while (len > 0 && g_ascii_isdigit(*str)) {
+ if (*str > '7') {
+ break;
+ }
+ else {
+ n = 8 * n - (*str++ - '0');
+ }
+ len--;
+ }
+
+ if (len == 0) {
+ return -(n);
+ }
+ else {
+ return std::nullopt;
+ }
+ };
+
+ auto replace_numeric_entity = [&](const char *entity) -> bool {
+ UChar32 uc;
+ std::optional<int> maybe_num;
+
+ if (*entity == 'x' || *entity == 'X') {
+ maybe_num = hex_to_int(entity + 1, h - (entity + 1));
+ }
+ else if (*entity == 'o' || *entity == 'O') {
+ maybe_num = oct_to_int(entity + 1, h - (entity + 1));
+ }
+ else {
+ maybe_num = dec_to_int(entity, h - entity);
+ }
+
+ if (!maybe_num) {
+ /* Skip undecoded */
+ if (end - t >= h - e) {
+ memmove(t, e, h - e);
+ t += h - e;
+ }
+
+ return false;
+ }
+ else {
+ uc = maybe_num.value();
+ /* Search for a replacement */
+ const auto *entity_def = html_entities_defs.by_id(uc);
+
+ if (entity_def) {
+ auto rep_len = strlen(entity_def->replacement);
+
+ if (end - t >= rep_len) {
+ memcpy(t, entity_def->replacement,
+ rep_len);
+ t += rep_len;
+ }
+
+ return true;
+ }
+ else {
+ /* Unicode point */
+ goffset off = t - s;
+ UBool is_error = 0;
+
+ if (uc > 0) {
+ U8_APPEND((std::uint8_t *) s, off, len, uc, is_error);
+
+ if (!is_error) {
+ t = s + off;
+ }
+ else if (end - t > 3) {
+ /* Not printable code point replace with 0xFFFD */
+ *t++ = '\357';
+ *t++ = '\277';
+ *t++ = '\275';
+
+ return true;
+ }
+ }
+ else if (end - t > 3) {
+ /* Not printable code point replace with 0xFFFD */
+ *t++ = '\357';
+ *t++ = '\277';
+ *t++ = '\275';
+ }
+ }
+
+ return true;
+ }
+
+ return false;
+ };
+
+ auto replace_entity = [&]() -> bool {
+ if (e + 1 < end) {
+ const auto *entity_start = e + 1;
+
+ if (*entity_start != '#') {
+ return replace_named_entity(entity_start, (h - entity_start));
+ }
+ else if (entity_start + 1 < h) {
+ return replace_numeric_entity(entity_start + 1);
+ }
+ }
+
+ return false;
+ };
+
+ if (norm_spaces && g_ascii_isspace(*h)) {
+ state = parser_state::skip_start_spaces;
+ }
+
+ while (h - s < len && t <= h) {
+ switch (state) {
+ case parser_state::normal_content:
+ if (*h == '&') {
+ state = parser_state::ampersand;
+ seen_hash = false;
+ seen_hex = false;
+ seen_digit_only = do_undefined;
+ e = h;
+ h++;
+ continue;
+ }
+ else {
+ if (norm_spaces && g_ascii_isspace(*h)) {
+ *t++ = ' ';
+ state = parser_state::skip_multi_spaces;
+ h++;
+ }
+ else {
+ *t++ = *h++;
+ }
+ }
+ break;
+ case parser_state::ampersand:
+ if ((*h == ';' || g_ascii_isspace(*h)) && h > e) {
+ replace_entity();
+ state = parser_state::normal_content;
+
+ if (g_ascii_isspace(*h)) {
+ /* Avoid increase of h */
+ continue;
+ }
+ }
+ else if (*h == '&') {
+ /* Previous `&` was bogus */
+ state = parser_state::ampersand;
+
+ if (end - t > h - e) {
+ memmove(t, e, h - e);
+ t += h - e;
+ }
+
+ e = h;
+ }
+ else if (*h == '#') {
+ seen_hash = true;
+
+ if (h + 1 < end && h[1] == 'x') {
+ seen_hex = true;
+ /* Skip one more character */
+ h++;
+ }
+ }
+ else if (seen_digit_only != do_mixed &&
+ (g_ascii_isdigit(*h) || (seen_hex && g_ascii_isxdigit(*h)))) {
+ seen_digit_only = do_digits_only;
+ }
+ else {
+ if (seen_digit_only == do_digits_only && seen_hash && h > e) {
+ /* We have seen some digits, so we can try to decode, eh */
+ /* Fuck retarded email clients... */
+ replace_entity();
+ state = parser_state::normal_content;
+ continue;
+ }
+
+ seen_digit_only = do_mixed;
+ }
+
+ h++;
+
+ break;
+ case parser_state::skip_multi_spaces:
+ if (g_ascii_isspace(*h)) {
+ h++;
+ }
+ else {
+ state = parser_state::normal_content;
+ }
+ break;
+ case parser_state::skip_start_spaces:
+ if (g_ascii_isspace(*h)) {
+ h++;
+ }
+ else {
+ state = parser_state::normal_content;
+ }
+ break;
+ }
+ }
+
+ /* Leftover */
+ if (state == parser_state::ampersand && h > e) {
+ /* Unfinished entity, copy as is */
+ if (replace_entity()) {
+ /* To follow FSM semantics */
+ h++;
+ }
+ else {
+ h = e; /* Include the last & */
+ }
+
+ /* Leftover after replacement */
+ if (h < end && t + (end - h) <= end) {
+ memmove(t, h, end - h);
+ t += end - h;
+ }
+ }
+
+ if (norm_spaces) {
+ bool seen_spaces = false;
+
+ while (t > s && g_ascii_isspace(*(t - 1))) {
+ seen_spaces = true;
+ t--;
+ }
+
+ if (seen_spaces) {
+ *t++ = ' ';
+ }
+ }
+
+ return (t - s);
+}
+
+auto decode_html_entitles_inplace(std::string &st) -> void
+{
+ auto nlen = decode_html_entitles_inplace(st.data(), st.size());
+ st.resize(nlen);
+}
+
+TEST_SUITE("html entities")
+{
+
+ TEST_CASE("html entities decode")
+ {
+ std::vector<std::pair<std::string, std::string>> cases{
+ {"", ""},
+ {"abc", "abc"},
+ {"abc def", "abc def"},
+ {"abc def", "abc def"},
+ {"abc\ndef", "abc def"},
+ {"abc\n \tdef", "abc def"},
+ {" abc def ", "abc def "},
+ {"FOO&gt;BAR", "FOO>BAR"},
+ {"FOO&gtBAR", "FOO>BAR"},
+ {"FOO&gt BAR", "FOO> BAR"},
+ {"FOO&gt;;;BAR", "FOO>;;BAR"},
+ {"I'm &notit;", "I'm ¬it;"},
+ {"I'm &notin;", "I'm ∉"},
+ {"FOO& BAR", "FOO& BAR"},
+ {"FOO&&&&gt;BAR", "FOO&&&>BAR"},
+ {"FOO&#41;BAR", "FOO)BAR"},
+ {"FOO&#x41;BAR", "FOOABAR"},
+ {"FOO&#X41;BAR", "FOOABAR"},
+ {"FOO&#BAR", "FOO&#BAR"},
+ {"FOO&#ZOO", "FOO&#ZOO"},
+ {"FOO&#xBAR", "FOOºR"},
+ {"FOO&#x41BAR", "FOO䆺R"},
+ {"FOO&#x0000;ZOO", "FOO\uFFFDZOO"},
+ {"FOO&#x0081;ZOO", "FOO\u0081ZOO"},
+ {"FOO&#xD800;ZOO", "FOO\uFFFDZOO"},
+ {"FOO&#xFFFFFF;ZOO", "FOO\uFFFDZOO"},
+ {"ZZ&pound_id=23", "ZZ£_id=23"},
+ {"ZZ&prod_id=23", "ZZ&prod_id=23"},
+ {"ZZ&gt", "ZZ>"},
+ {"ZZ&", "ZZ&"},
+ {"ZZ&AElig=", "ZZÆ="},
+ };
+
+ for (const auto &c: cases) {
+ SUBCASE(("decode entities: " + c.first).c_str())
+ {
+ auto *cpy = new char[c.first.size()];
+ memcpy(cpy, c.first.data(), c.first.size());
+ auto nlen = decode_html_entitles_inplace(cpy, c.first.size(), true);
+ CHECK(std::string{cpy, nlen} == c.second);
+ delete[] cpy;
+ }
+ }
+ }
+}
+
+}// namespace rspamd::html \ No newline at end of file
diff --git a/src/libserver/html/html_entities.hxx b/src/libserver/html/html_entities.hxx
new file mode 100644
index 0000000..fc1f7cc
--- /dev/null
+++ b/src/libserver/html/html_entities.hxx
@@ -0,0 +1,31 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_HTML_ENTITIES_H
+#define RSPAMD_HTML_ENTITIES_H
+#pragma once
+
+#include <utility>
+#include <string>
+
+namespace rspamd::html {
+
+auto decode_html_entitles_inplace(char *s, std::size_t len, bool norm_spaces = false) -> std::size_t;
+auto decode_html_entitles_inplace(std::string &st) -> void;
+
+}// namespace rspamd::html
+
+#endif
diff --git a/src/libserver/html/html_tag.hxx b/src/libserver/html/html_tag.hxx
new file mode 100644
index 0000000..309d761
--- /dev/null
+++ b/src/libserver/html/html_tag.hxx
@@ -0,0 +1,159 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_HTML_TAG_HXX
+#define RSPAMD_HTML_TAG_HXX
+#pragma once
+
+#include <utility>
+#include <string_view>
+#include <variant>
+#include <vector>
+#include <optional>
+#include <cstdint>
+
+#include "html_tags.h"
+
+struct rspamd_url;
+struct html_image;
+
+namespace rspamd::html {
+
+struct html_content; /* Forward declaration */
+
+enum class html_component_type : std::uint8_t {
+ RSPAMD_HTML_COMPONENT_NAME = 0,
+ RSPAMD_HTML_COMPONENT_HREF,
+ RSPAMD_HTML_COMPONENT_COLOR,
+ RSPAMD_HTML_COMPONENT_BGCOLOR,
+ RSPAMD_HTML_COMPONENT_STYLE,
+ RSPAMD_HTML_COMPONENT_CLASS,
+ RSPAMD_HTML_COMPONENT_WIDTH,
+ RSPAMD_HTML_COMPONENT_HEIGHT,
+ RSPAMD_HTML_COMPONENT_SIZE,
+ RSPAMD_HTML_COMPONENT_REL,
+ RSPAMD_HTML_COMPONENT_ALT,
+ RSPAMD_HTML_COMPONENT_ID,
+ RSPAMD_HTML_COMPONENT_HIDDEN,
+};
+
+/* Public tags flags */
+/* XML tag */
+#define FL_XML (1u << CM_USER_SHIFT)
+/* Fully closed tag (e.g. <a attrs />) */
+#define FL_CLOSED (1 << (CM_USER_SHIFT + 1))
+#define FL_BROKEN (1 << (CM_USER_SHIFT + 2))
+#define FL_IGNORE (1 << (CM_USER_SHIFT + 3))
+#define FL_BLOCK (1 << (CM_USER_SHIFT + 4))
+#define FL_HREF (1 << (CM_USER_SHIFT + 5))
+#define FL_COMMENT (1 << (CM_USER_SHIFT + 6))
+#define FL_VIRTUAL (1 << (CM_USER_SHIFT + 7))
+
+/**
+ * Returns component type from a string
+ * @param st
+ * @return
+ */
+auto html_component_from_string(const std::string_view &st) -> std::optional<html_component_type>;
+
+using html_tag_extra_t = std::variant<std::monostate, struct rspamd_url *, struct html_image *>;
+struct html_tag_component {
+ html_component_type type;
+ std::string_view value;
+
+ html_tag_component(html_component_type type, std::string_view value)
+ : type(type), value(value)
+ {
+ }
+};
+
+/* Pairing closing tag representation */
+struct html_closing_tag {
+ int start = -1;
+ int end = -1;
+
+ auto clear() -> void
+ {
+ start = end = -1;
+ }
+};
+
+struct html_tag {
+ unsigned int tag_start = 0;
+ unsigned int content_offset = 0;
+ std::uint32_t flags = 0;
+ std::int32_t id = Tag_UNKNOWN;
+ html_closing_tag closing;
+
+ std::vector<html_tag_component> components;
+
+ html_tag_extra_t extra;
+ mutable struct html_block *block = nullptr;
+ std::vector<struct html_tag *> children;
+ struct html_tag *parent;
+
+ auto find_component(html_component_type what) const -> std::optional<std::string_view>
+ {
+ for (const auto &comp: components) {
+ if (comp.type == what) {
+ return comp.value;
+ }
+ }
+
+ return std::nullopt;
+ }
+
+ auto find_component(std::optional<html_component_type> what) const -> std::optional<std::string_view>
+ {
+ if (what) {
+ return find_component(what.value());
+ }
+
+ return std::nullopt;
+ }
+
+ auto clear(void) -> void
+ {
+ id = Tag_UNKNOWN;
+ tag_start = content_offset = 0;
+ extra = std::monostate{};
+ components.clear();
+ flags = 0;
+ block = nullptr;
+ children.clear();
+ closing.clear();
+ }
+
+ constexpr auto get_content_length() const -> std::size_t
+ {
+ if (flags & (FL_IGNORE | CM_HEAD)) {
+ return 0;
+ }
+ if (closing.start > content_offset) {
+ return closing.start - content_offset;
+ }
+
+ return 0;
+ }
+
+ auto get_content(const struct html_content *hc) const -> std::string_view;
+};
+
+static_assert(CM_USER_SHIFT + 7 < sizeof(html_tag::flags) * NBBY);
+
+}// namespace rspamd::html
+
+#endif//RSPAMD_HTML_TAG_HXX
diff --git a/src/libserver/html/html_tag_defs.hxx b/src/libserver/html/html_tag_defs.hxx
new file mode 100644
index 0000000..647f7c3
--- /dev/null
+++ b/src/libserver/html/html_tag_defs.hxx
@@ -0,0 +1,194 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_HTML_TAG_DEFS_HXX
+#define RSPAMD_HTML_TAG_DEFS_HXX
+
+#include "config.h"
+#include "html_tags.h"
+#include "libutil/cxx/util.hxx"
+
+#include <string>
+#include "contrib/ankerl/unordered_dense.h"
+
+namespace rspamd::html {
+
+struct html_tag_def {
+ std::string name;
+ tag_id_t id;
+ guint flags;
+};
+
+#define TAG_DEF(id, name, flags) \
+ html_tag_def \
+ { \
+ (name), (id), (flags) \
+ }
+
+static const auto html_tag_defs_array = rspamd::array_of(
+ /* W3C defined elements */
+ TAG_DEF(Tag_A, "a", FL_HREF),
+ TAG_DEF(Tag_ABBR, "abbr", (CM_INLINE)),
+ TAG_DEF(Tag_ACRONYM, "acronym", (CM_INLINE)),
+ TAG_DEF(Tag_ADDRESS, "address", (CM_BLOCK)),
+ TAG_DEF(Tag_APPLET, "applet", (CM_IMG | CM_INLINE | CM_PARAM)),
+ TAG_DEF(Tag_AREA, "area", (CM_BLOCK | CM_EMPTY | FL_HREF)),
+ TAG_DEF(Tag_B, "b", (CM_INLINE | FL_BLOCK)),
+ TAG_DEF(Tag_BASE, "base", (CM_HEAD | CM_EMPTY)),
+ TAG_DEF(Tag_BASEFONT, "basefont", (CM_INLINE | CM_EMPTY)),
+ TAG_DEF(Tag_BDO, "bdo", (CM_INLINE)),
+ TAG_DEF(Tag_BIG, "big", (CM_INLINE)),
+ TAG_DEF(Tag_BLOCKQUOTE, "blockquote", (CM_BLOCK)),
+ TAG_DEF(Tag_BODY, "body", (CM_HTML | CM_OPT | CM_OMITST | CM_UNIQUE | FL_BLOCK)),
+ TAG_DEF(Tag_BR, "br", (CM_INLINE | CM_EMPTY)),
+ TAG_DEF(Tag_BUTTON, "button", (CM_INLINE | FL_BLOCK)),
+ TAG_DEF(Tag_CAPTION, "caption", (CM_TABLE)),
+ TAG_DEF(Tag_CENTER, "center", (CM_BLOCK)),
+ TAG_DEF(Tag_CITE, "cite", (CM_INLINE)),
+ TAG_DEF(Tag_CODE, "code", (CM_INLINE)),
+ TAG_DEF(Tag_COL, "col", (CM_TABLE | CM_EMPTY)),
+ TAG_DEF(Tag_COLGROUP, "colgroup", (CM_TABLE | CM_OPT)),
+ TAG_DEF(Tag_DD, "dd", (CM_DEFLIST | CM_OPT | CM_NO_INDENT)),
+ TAG_DEF(Tag_DEL, "del", (CM_INLINE | CM_BLOCK)),
+ TAG_DEF(Tag_DFN, "dfn", (CM_INLINE)),
+ TAG_DEF(Tag_DIR, "dir", (CM_BLOCK)),
+ TAG_DEF(Tag_DIV, "div", (CM_BLOCK | FL_BLOCK)),
+ TAG_DEF(Tag_DL, "dl", (CM_BLOCK | FL_BLOCK)),
+ TAG_DEF(Tag_DT, "dt", (CM_DEFLIST | CM_OPT | CM_NO_INDENT)),
+ TAG_DEF(Tag_EM, "em", (CM_INLINE)),
+ TAG_DEF(Tag_FIELDSET, "fieldset", (CM_BLOCK)),
+ TAG_DEF(Tag_FONT, "font", (FL_BLOCK)),
+ TAG_DEF(Tag_FORM, "form", (CM_BLOCK | FL_HREF)),
+ TAG_DEF(Tag_FRAME, "frame", (CM_EMPTY | FL_HREF)),
+ TAG_DEF(Tag_FRAMESET, "frameset", (CM_HTML)),
+ TAG_DEF(Tag_H1, "h1", (CM_BLOCK)),
+ TAG_DEF(Tag_H2, "h2", (CM_BLOCK)),
+ TAG_DEF(Tag_H3, "h3", (CM_BLOCK)),
+ TAG_DEF(Tag_H4, "h4", (CM_BLOCK)),
+ TAG_DEF(Tag_H5, "h5", (CM_BLOCK)),
+ TAG_DEF(Tag_H6, "h6", (CM_BLOCK)),
+ TAG_DEF(Tag_HEAD, "head", (CM_HTML | CM_OPT | CM_OMITST | CM_UNIQUE)),
+ TAG_DEF(Tag_HR, "hr", (CM_BLOCK | CM_EMPTY)),
+ TAG_DEF(Tag_HTML, "html", (CM_HTML | CM_OPT | CM_OMITST | CM_UNIQUE)),
+ TAG_DEF(Tag_I, "i", (CM_INLINE)),
+ TAG_DEF(Tag_IFRAME, "iframe", (FL_HREF)),
+ TAG_DEF(Tag_IMG, "img", (CM_INLINE | CM_IMG | CM_EMPTY)),
+ TAG_DEF(Tag_INPUT, "input", (CM_INLINE | CM_IMG | CM_EMPTY)),
+ TAG_DEF(Tag_INS, "ins", (CM_INLINE | CM_BLOCK)),
+ TAG_DEF(Tag_ISINDEX, "isindex", (CM_BLOCK | CM_EMPTY)),
+ TAG_DEF(Tag_KBD, "kbd", (CM_INLINE)),
+ TAG_DEF(Tag_LABEL, "label", (CM_INLINE)),
+ TAG_DEF(Tag_LEGEND, "legend", (CM_INLINE)),
+ TAG_DEF(Tag_LI, "li", (CM_LIST | CM_OPT | CM_NO_INDENT | FL_BLOCK)),
+ TAG_DEF(Tag_LINK, "link", (CM_EMPTY | FL_HREF)),
+ TAG_DEF(Tag_LISTING, "listing", (CM_BLOCK)),
+ TAG_DEF(Tag_MAP, "map", (CM_INLINE | FL_HREF)),
+ TAG_DEF(Tag_MENU, "menu", (CM_BLOCK)),
+ TAG_DEF(Tag_META, "meta", (CM_HEAD | CM_INLINE | CM_EMPTY)),
+ TAG_DEF(Tag_NOFRAMES, "noframes", (CM_BLOCK)),
+ TAG_DEF(Tag_NOSCRIPT, "noscript", (CM_BLOCK | CM_INLINE | CM_RAW)),
+ TAG_DEF(Tag_OBJECT, "object", (CM_HEAD | CM_IMG | CM_INLINE | CM_PARAM)),
+ TAG_DEF(Tag_OL, "ol", (CM_BLOCK | FL_BLOCK)),
+ TAG_DEF(Tag_OPTGROUP, "optgroup", (CM_FIELD | CM_OPT)),
+ TAG_DEF(Tag_OPTION, "option", (CM_FIELD | CM_OPT)),
+ TAG_DEF(Tag_P, "p", (CM_BLOCK | CM_OPT | FL_BLOCK)),
+ TAG_DEF(Tag_PARAM, "param", (CM_INLINE | CM_EMPTY)),
+ TAG_DEF(Tag_PLAINTEXT, "plaintext", (CM_BLOCK)),
+ TAG_DEF(Tag_PRE, "pre", (CM_BLOCK)),
+ TAG_DEF(Tag_Q, "q", (CM_INLINE)),
+ TAG_DEF(Tag_RB, "rb", (CM_INLINE)),
+ TAG_DEF(Tag_RBC, "rbc", (CM_INLINE)),
+ TAG_DEF(Tag_RP, "rp", (CM_INLINE)),
+ TAG_DEF(Tag_RT, "rt", (CM_INLINE)),
+ TAG_DEF(Tag_RTC, "rtc", (CM_INLINE)),
+ TAG_DEF(Tag_RUBY, "ruby", (CM_INLINE)),
+ TAG_DEF(Tag_S, "s", (CM_INLINE)),
+ TAG_DEF(Tag_SAMP, "samp", (CM_INLINE)),
+ TAG_DEF(Tag_SCRIPT, "script", (CM_HEAD | CM_RAW)),
+ TAG_DEF(Tag_SELECT, "select", (CM_INLINE | CM_FIELD)),
+ TAG_DEF(Tag_SMALL, "small", (CM_INLINE)),
+ TAG_DEF(Tag_SPAN, "span", (CM_NO_INDENT | FL_BLOCK)),
+ TAG_DEF(Tag_STRIKE, "strike", (CM_INLINE)),
+ TAG_DEF(Tag_STRONG, "strong", (CM_INLINE)),
+ TAG_DEF(Tag_STYLE, "style", (CM_HEAD | CM_RAW)),
+ TAG_DEF(Tag_SUB, "sub", (CM_INLINE)),
+ TAG_DEF(Tag_SUP, "sup", (CM_INLINE)),
+ TAG_DEF(Tag_TABLE, "table", (CM_BLOCK | FL_BLOCK)),
+ TAG_DEF(Tag_TBODY, "tbody", (CM_TABLE | CM_ROWGRP | CM_OPT | FL_BLOCK)),
+ TAG_DEF(Tag_TD, "td", (CM_ROW | CM_OPT | CM_NO_INDENT | FL_BLOCK)),
+ TAG_DEF(Tag_TEXTAREA, "textarea", (CM_INLINE | CM_FIELD)),
+ TAG_DEF(Tag_TFOOT, "tfoot", (CM_TABLE | CM_ROWGRP | CM_OPT)),
+ TAG_DEF(Tag_TH, "th", (CM_ROW | CM_OPT | CM_NO_INDENT | FL_BLOCK)),
+ TAG_DEF(Tag_THEAD, "thead", (CM_TABLE | CM_ROWGRP | CM_OPT)),
+ TAG_DEF(Tag_TITLE, "title", (CM_HEAD | CM_UNIQUE)),
+ TAG_DEF(Tag_TR, "tr", (CM_TABLE | CM_OPT | FL_BLOCK)),
+ TAG_DEF(Tag_TT, "tt", (CM_INLINE)),
+ TAG_DEF(Tag_U, "u", (CM_INLINE)),
+ TAG_DEF(Tag_UL, "ul", (CM_BLOCK | FL_BLOCK)),
+ TAG_DEF(Tag_VAR, "var", (CM_INLINE)),
+ TAG_DEF(Tag_XMP, "xmp", (CM_BLOCK)),
+ TAG_DEF(Tag_NEXTID, "nextid", (CM_HEAD | CM_EMPTY)));
+
+class html_tags_storage {
+ ankerl::unordered_dense::map<std::string_view, html_tag_def> tag_by_name;
+ ankerl::unordered_dense::map<tag_id_t, html_tag_def> tag_by_id;
+
+public:
+ html_tags_storage()
+ {
+ tag_by_name.reserve(html_tag_defs_array.size());
+ tag_by_id.reserve(html_tag_defs_array.size());
+
+ for (const auto &t: html_tag_defs_array) {
+ tag_by_name[t.name] = t;
+ tag_by_id[t.id] = t;
+ }
+ }
+
+ auto by_name(std::string_view name) const -> const html_tag_def *
+ {
+ auto it = tag_by_name.find(name);
+
+ if (it != tag_by_name.end()) {
+ return &(it->second);
+ }
+
+ return nullptr;
+ }
+
+ auto by_id(int id) const -> const html_tag_def *
+ {
+ auto it = tag_by_id.find(static_cast<tag_id_t>(id));
+ if (it != tag_by_id.end()) {
+ return &(it->second);
+ }
+
+ return nullptr;
+ }
+
+ auto name_by_id_safe(int id) const -> std::string_view
+ {
+ auto it = tag_by_id.find(static_cast<tag_id_t>(id));
+ if (it != tag_by_id.end()) {
+ return it->second.name;
+ }
+
+ return "unknown";
+ }
+};
+
+}// namespace rspamd::html
+
+#endif//RSPAMD_HTML_TAG_DEFS_HXX
diff --git a/src/libserver/html/html_tags.h b/src/libserver/html/html_tags.h
new file mode 100644
index 0000000..c186314
--- /dev/null
+++ b/src/libserver/html/html_tags.h
@@ -0,0 +1,176 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBSERVER_HTML_TAGS_H_
+#define SRC_LIBSERVER_HTML_TAGS_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Known HTML tags */
+typedef enum {
+ Tag_UNKNOWN = 0, /**< Unknown tag! */
+ Tag_A, /**< A */
+ Tag_ABBR, /**< ABBR */
+ Tag_ACRONYM, /**< ACRONYM */
+ Tag_ADDRESS, /**< ADDRESS */
+ Tag_APPLET, /**< APPLET */
+ Tag_AREA, /**< AREA */
+ Tag_B, /**< B */
+ Tag_BASE, /**< BASE */
+ Tag_BASEFONT, /**< BASEFONT */
+ Tag_BDO, /**< BDO */
+ Tag_BIG, /**< BIG */
+ Tag_BLOCKQUOTE, /**< BLOCKQUOTE */
+ Tag_BODY, /**< BODY */
+ Tag_BR, /**< BR */
+ Tag_BUTTON, /**< BUTTON */
+ Tag_CAPTION, /**< CAPTION */
+ Tag_CENTER, /**< CENTER */
+ Tag_CITE, /**< CITE */
+ Tag_CODE, /**< CODE */
+ Tag_COL, /**< COL */
+ Tag_COLGROUP, /**< COLGROUP */
+ Tag_DD, /**< DD */
+ Tag_DEL, /**< DEL */
+ Tag_DFN, /**< DFN */
+ Tag_DIR, /**< DIR */
+ Tag_DIV, /**< DIF */
+ Tag_DL, /**< DL */
+ Tag_DT, /**< DT */
+ Tag_EM, /**< EM */
+ Tag_FIELDSET, /**< FIELDSET */
+ Tag_FONT, /**< FONT */
+ Tag_FORM, /**< FORM */
+ Tag_FRAME, /**< FRAME */
+ Tag_FRAMESET, /**< FRAMESET */
+ Tag_H1, /**< H1 */
+ Tag_H2, /**< H2 */
+ Tag_H3, /**< H3 */
+ Tag_H4, /**< H4 */
+ Tag_H5, /**< H5 */
+ Tag_H6, /**< H6 */
+ Tag_HEAD, /**< HEAD */
+ Tag_HR, /**< HR */
+ Tag_HTML, /**< HTML */
+ Tag_I, /**< I */
+ Tag_IFRAME, /**< IFRAME */
+ Tag_IMG, /**< IMG */
+ Tag_INPUT, /**< INPUT */
+ Tag_INS, /**< INS */
+ Tag_ISINDEX, /**< ISINDEX */
+ Tag_KBD, /**< KBD */
+ Tag_KEYGEN, /**< KEYGEN */
+ Tag_LABEL, /**< LABEL */
+ Tag_LEGEND, /**< LEGEND */
+ Tag_LI, /**< LI */
+ Tag_LINK, /**< LINK */
+ Tag_LISTING, /**< LISTING */
+ Tag_MAP, /**< MAP */
+ Tag_MENU, /**< MENU */
+ Tag_META, /**< META */
+ Tag_NOFRAMES, /**< NOFRAMES */
+ Tag_NOSCRIPT, /**< NOSCRIPT */
+ Tag_OBJECT, /**< OBJECT */
+ Tag_OL, /**< OL */
+ Tag_OPTGROUP, /**< OPTGROUP */
+ Tag_OPTION, /**< OPTION */
+ Tag_P, /**< P */
+ Tag_PARAM, /**< PARAM */
+ Tag_PLAINTEXT, /**< PLAINTEXT */
+ Tag_PRE, /**< PRE */
+ Tag_Q, /**< Q */
+ Tag_RB, /**< RB */
+ Tag_RBC, /**< RBC */
+ Tag_RP, /**< RP */
+ Tag_RT, /**< RT */
+ Tag_RTC, /**< RTC */
+ Tag_RUBY, /**< RUBY */
+ Tag_S, /**< S */
+ Tag_SAMP, /**< SAMP */
+ Tag_SCRIPT, /**< SCRIPT */
+ Tag_SELECT, /**< SELECT */
+ Tag_SMALL, /**< SMALL */
+ Tag_SPAN, /**< SPAN */
+ Tag_STRIKE, /**< STRIKE */
+ Tag_STRONG, /**< STRONG */
+ Tag_STYLE, /**< STYLE */
+ Tag_SUB, /**< SUB */
+ Tag_SUP, /**< SUP */
+ Tag_TABLE, /**< TABLE */
+ Tag_TBODY, /**< TBODY */
+ Tag_TD, /**< TD */
+ Tag_TEXTAREA, /**< TEXTAREA */
+ Tag_TFOOT, /**< TFOOT */
+ Tag_TH, /**< TH */
+ Tag_THEAD, /**< THEAD */
+ Tag_TITLE, /**< TITLE */
+ Tag_TR, /**< TR */
+ Tag_TT, /**< TT */
+ Tag_U, /**< U */
+ Tag_UL, /**< UL */
+ Tag_VAR, /**< VAR */
+ Tag_XMP, /**< XMP */
+ Tag_NEXTID, /**< NEXTID */
+ Tag_MAX,
+
+ N_TAGS = -1 /**< Must be -1 */
+} tag_id_t;
+
+#define CM_UNKNOWN 0
+/* Elements with no content. Map to HTML specification. */
+#define CM_EMPTY (1 << 0)
+/* Elements that appear outside of "BODY". */
+#define CM_HTML (1 << 1)
+/* Elements that can appear within HEAD. */
+#define CM_HEAD (1 << 2)
+/* HTML "block" elements. */
+#define CM_BLOCK (1 << 3)
+/* HTML "inline" elements. */
+#define CM_INLINE (1 << 4)
+/* Elements that mark list item ("LI"). */
+#define CM_LIST (1 << 5)
+/* Elements that mark definition list item ("DL", "DT"). */
+#define CM_DEFLIST (1 << 6)
+/* Elements that can appear inside TABLE. */
+#define CM_TABLE (1 << 7)
+/* Used for "THEAD", "TFOOT" or "TBODY". */
+#define CM_ROWGRP (1 << 8)
+/* Used for "TD", "TH" */
+#define CM_ROW (1 << 9)
+/* Elements whose content must be protected against white space movement.
+ Includes some elements that can found in forms. */
+#define CM_FIELD (1 << 10)
+#define CM_RAW (1 << 11)
+/* Elements that allows "PARAM". */
+#define CM_PARAM (1 << 12)
+/* Elements with an optional end tag. */
+#define CM_OPT (1 << 13)
+/* Elements that use "align" attribute for vertical position. */
+#define CM_IMG (1 << 14)
+#define CM_NO_INDENT (1 << 15)
+/* Elements that cannot be omitted. */
+#define CM_OMITST (1 << 16)
+/* Unique elements */
+#define CM_UNIQUE (1 << 17)
+
+#define CM_USER_SHIFT (18)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBSERVER_HTML_TAGS_H_ */
diff --git a/src/libserver/html/html_tests.cxx b/src/libserver/html/html_tests.cxx
new file mode 100644
index 0000000..2fe6702
--- /dev/null
+++ b/src/libserver/html/html_tests.cxx
@@ -0,0 +1,304 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "html.hxx"
+#include "libserver/task.h"
+
+#include <vector>
+#include <fmt/core.h>
+
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+namespace rspamd::html {
+
+/*
+ * Tests part
+ */
+
+TEST_SUITE("html")
+{
+ TEST_CASE("html parsing")
+ {
+
+ const std::vector<std::pair<std::string, std::string>> cases{
+ {"<html><!DOCTYPE html><body>", "+html;++xml;++body;"},
+ {"<html><div><div></div></div></html>", "+html;++div;+++div;"},
+ {"<html><div><div></div></html>", "+html;++div;+++div;"},
+ {"<html><div><div></div></html></div>", "+html;++div;+++div;"},
+ {"<p><p><a></p></a></a>", "+p;++p;+++a;"},
+ {"<div><a href=\"http://example.com\"></div></a>", "+div;++a;"},
+ /* Broken, as I don't know how the hell this should be really parsed */
+ //{"<html><!DOCTYPE html><body><head><body></body></html></body></html>",
+ // "+html;++xml;++body;+++head;+++body;"}
+ };
+
+ rspamd_url_init(NULL);
+ auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "html", 0);
+ struct rspamd_task fake_task;
+ memset(&fake_task, 0, sizeof(fake_task));
+ fake_task.task_pool = pool;
+
+ for (const auto &c: cases) {
+ SUBCASE((std::string("extract tags from: ") + c.first).c_str())
+ {
+ GByteArray *tmp = g_byte_array_sized_new(c.first.size());
+ g_byte_array_append(tmp, (const guint8 *) c.first.data(), c.first.size());
+ auto *hc = html_process_input(&fake_task, tmp, nullptr, nullptr, nullptr, true, nullptr);
+ CHECK(hc != nullptr);
+ auto dump = html_debug_structure(*hc);
+ CHECK(c.second == dump);
+ g_byte_array_free(tmp, TRUE);
+ }
+ }
+
+ rspamd_mempool_delete(pool);
+ }
+
+ TEST_CASE("html text extraction")
+ {
+ using namespace std::string_literals;
+ const std::vector<std::pair<std::string, std::string>> cases{
+ {"test", "test"},
+ {"test\0"s, "test\uFFFD"s},
+ {"test\0test"s, "test\uFFFDtest"s},
+ {"test\0\0test"s, "test\uFFFD\uFFFDtest"s},
+ {"test ", "test"},
+ {"test foo, bar", "test foo, bar"},
+ {"<p>text</p>", "text\n"},
+ {"olo<p>text</p>lolo", "olo\ntext\nlolo"},
+ {"<div>foo</div><div>bar</div>", "foo\nbar\n"},
+ {"<b>foo<i>bar</b>baz</i>", "foobarbaz"},
+ {"<b>foo<i>bar</i>baz</b>", "foobarbaz"},
+ {"foo<br>baz", "foo\nbaz"},
+ {"<a href=https://example.com>test</a>", "test"},
+ {"<img alt=test>", "test"},
+ {" <body>\n"
+ " <!-- escape content -->\n"
+ " a&nbsp;b a &gt; b a &lt; b a &amp; b &apos;a &quot;a&quot;\n"
+ " </body>",
+ R"|(a b a > b a < b a & b 'a "a")|"},
+ /* XML tags */
+ {"<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n"
+ " <!DOCTYPE html\n"
+ " PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n"
+ " \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n"
+ "<body>test</body>",
+ "test"},
+ {"<html><head><meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\"></head>"
+ " <body>\n"
+ " <p><br>\n"
+ " </p>\n"
+ " <div class=\"moz-forward-container\"><br>\n"
+ " <br>\n"
+ " test</div>"
+ "</body>",
+ "\n\n\ntest\n"},
+ {"<div>fi<span style=\"FONT-SIZE: 0px\">le </span>"
+ "sh<span style=\"FONT-SIZE: 0px\">aring </span></div>",
+ "fish\n"},
+ /* FIXME: broken until rework of css parser */
+ //{"<div>fi<span style=\"FONT-SIZE: 0px\">le </span>"
+ // "sh<span style=\"FONT-SIZE: 0px\">aring </div>foo</span>", "fish\nfoo"},
+ /* Complex html with bad tags */
+ {"<!DOCTYPE html>\n"
+ "<html lang=\"en\">\n"
+ " <head>\n"
+ " <meta charset=\"utf-8\">\n"
+ " <title>title</title>\n"
+ " <link rel=\"stylesheet\" href=\"style.css\">\n"
+ " <script src=\"script.js\"></script>\n"
+ " </head>\n"
+ " <body>\n"
+ " <!-- page content -->\n"
+ " Hello, world! <b>test</b>\n"
+ " <p>data<>\n"
+ " </P>\n"
+ " <b>stuff</p>?\n"
+ " </body>\n"
+ "</html>",
+ "Hello, world! test \ndata<>\nstuff\n?"},
+ {"<p><!--comment-->test</br></hr><br>", "test\n"},
+ /* Tables */
+ {"<table>\n"
+ " <tr>\n"
+ " <th>heada</th>\n"
+ " <th>headb</th>\n"
+ " </tr>\n"
+ " <tr>\n"
+ " <td>data1</td>\n"
+ " <td>data2</td>\n"
+ " </tr>\n"
+ " </table>",
+ "heada headb\ndata1 data2\n"},
+ /* Invalid closing br and hr + comment */
+ {" <body>\n"
+ " <!-- page content -->\n"
+ " Hello, world!<br>test</br><br>content</hr>more content<br>\n"
+ " <div>\n"
+ " content inside div\n"
+ " </div>\n"
+ " </body>",
+ "Hello, world!\ntest\ncontentmore content\ncontent inside div\n"},
+ /* First closing tag */
+ {"</head>\n"
+ "<body>\n"
+ "<p> Hello. I have some bad news.\n"
+ "<br /> <br /> <br /> <strong> <br /> <br /> <br /> <br /> <br /> <br /> <br /> <br /> </strong><span> <br /> </span>test</p>\n"
+ "</body>\n"
+ "</html>",
+ "Hello. I have some bad news. \n\n\n\n\n\n\n\n\n\n\n\ntest\n"},
+ /* Invalid tags */
+ {"lol <sht> omg </sht> oh my!\n"
+ "<name>words words</name> goodbye",
+ "lol omg oh my! words words goodbye"},
+ /* Invisible stuff */
+ {"<div style=\"color:#555555;font-family:Arial, 'Helvetica Neue', Helvetica, sans-serif;line-height:1.2;padding-top:10px;padding-right:10px;padding-bottom:10px;padding-left:10px;font-style: italic;\">\n"
+ "<p style=\"font-size: 11px; line-height: 1.2; color: #555555; font-family: Arial, 'Helvetica Neue', Helvetica, sans-serif; mso-line-height-alt: 14px; margin: 0;\">\n"
+ "<span style=\"color:#FFFFFF; \">F</span>Sincerely,</p>\n"
+ "<p style=\"font-size: 11px; line-height: 1.2; color: #555555; font-family: Arial, 'Helvetica Neue', Helvetica, sans-serif; mso-line-height-alt: 14px; margin: 0;\">\n"
+ "<span style=\"color:#FFFFFF; \">8</span>Sky<span style=\"opacity:1;\"></span>pe<span style=\"color:#FFFFFF; \">F</span>Web<span style=\"color:#FFFFFF; \">F</span></p>\n"
+ "<span style=\"color:#FFFFFF; \">kreyes</span>\n"
+ "<p style=\"font-size: 11px; line-height: 1.2; color: #555555; font-family: Arial, 'Helvetica Neue', Helvetica, sans-serif; mso-line-height-alt: 14px; margin: 0;\">\n"
+ "&nbsp;</p>",
+ " Sincerely,\n Skype Web\n"},
+ {"lala<p hidden>fafa</p>", "lala"},
+ {"<table style=\"FONT-SIZE: 0px;\"><tbody><tr><td>\n"
+ "DONKEY\n"
+ "</td></tr></tbody></table>",
+ ""},
+ /* bgcolor propagation */
+ {"<a style=\"display: inline-block; color: #ffffff; background-color: #00aff0;\">\n"
+ "<span style=\"color: #00aff0;\">F</span>Rev<span style=\"opacity: 1;\"></span></span>ie<span style=\"opacity: 1;\"></span>"
+ "</span>w<span style=\"color: #00aff0;\">F<span style=\"opacity: 1;\">̹</span></span>",
+ " Review"},
+ {"<td style=\"color:#ffffff\" bgcolor=\"#005595\">\n"
+ "hello world\n"
+ "</td>",
+ "hello world"},
+ /* Colors */
+ {"goodbye <span style=\"COLOR: rgb(64,64,64)\">cruel</span>"
+ "<span>world</span>",
+ "goodbye cruelworld"},
+ /* Font-size propagation */
+ {"<p style=\"font-size: 11pt;line-height:22px\">goodbye <span style=\"font-size:0px\">cruel</span>world</p>",
+ "goodbye world\n"},
+ /* Newline before tag -> must be space */
+ {"goodbye <span style=\"COLOR: rgb(64,64,64)\">cruel</span>\n"
+ "<span>world</span>",
+ "goodbye cruel world"},
+ /* Head tag with some stuff */
+ {"<html><head><p>oh my god</head><body></body></html>", "oh my god\n"},
+ {"<html><head><title>oh my god</head><body></body></html>", ""},
+ {"<html><body><html><head>displayed</body></html></body></html>", "displayed"},
+
+ };
+
+ rspamd_url_init(NULL);
+ auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "html", 0);
+ struct rspamd_task fake_task;
+ memset(&fake_task, 0, sizeof(fake_task));
+ fake_task.task_pool = pool;
+
+ auto replace_newlines = [](std::string &str) {
+ auto start_pos = 0;
+ while ((start_pos = str.find("\n", start_pos, 1)) != std::string::npos) {
+ str.replace(start_pos, 1, "\\n", 2);
+ start_pos += 2;
+ }
+ };
+
+ auto i = 1;
+ for (const auto &c: cases) {
+ SUBCASE((fmt::format("html extraction case {}", i)).c_str())
+ {
+ GByteArray *tmp = g_byte_array_sized_new(c.first.size());
+ g_byte_array_append(tmp, (const guint8 *) c.first.data(), c.first.size());
+ auto *hc = html_process_input(&fake_task, tmp, nullptr, nullptr, nullptr, true, nullptr);
+ CHECK(hc != nullptr);
+ replace_newlines(hc->parsed);
+ auto expected = c.second;
+ replace_newlines(expected);
+ CHECK(hc->parsed == expected);
+ g_byte_array_free(tmp, TRUE);
+ }
+ i++;
+ }
+
+ rspamd_mempool_delete(pool);
+ }
+
+ TEST_CASE("html urls extraction")
+ {
+ using namespace std::string_literals;
+ const std::vector<std::tuple<std::string, std::vector<std::string>, std::optional<std::string>>> cases{
+ {"<style></style><a href=\"https://www.example.com\">yolo</a>",
+ {"https://www.example.com"},
+ "yolo"},
+ {"<a href=\"https://example.com\">test</a>", {"https://example.com"}, "test"},
+ {"<a <poo href=\"http://example.com\">hello</a>", {"http://example.com"}, "hello"},
+ {"<html>\n"
+ "<META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html; charset=utf-8\">\n"
+ "<body>\n"
+ "<a href=\"https://www.example.com\">hello</a>\n"
+ "</body>\n"
+ "</html>",
+ {"https://www.example.com"},
+ "hello"},
+ };
+
+ rspamd_url_init(NULL);
+ auto *pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "html", 0);
+ struct rspamd_task fake_task;
+ memset(&fake_task, 0, sizeof(fake_task));
+ fake_task.task_pool = pool;
+
+ auto i = 1;
+ for (const auto &c: cases) {
+ SUBCASE((fmt::format("html url extraction case {}", i)).c_str())
+ {
+ GPtrArray *purls = g_ptr_array_new();
+ auto input = std::get<0>(c);
+ GByteArray *tmp = g_byte_array_sized_new(input.size());
+ g_byte_array_append(tmp, (const guint8 *) input.data(), input.size());
+ auto *hc = html_process_input(&fake_task, tmp, nullptr, nullptr, purls, true, nullptr);
+ CHECK(hc != nullptr);
+ auto &expected_text = std::get<2>(c);
+ if (expected_text.has_value()) {
+ CHECK(hc->parsed == expected_text.value());
+ }
+ const auto &expected_urls = std::get<1>(c);
+ CHECK(expected_urls.size() == purls->len);
+ for (auto j = 0; j < expected_urls.size(); ++j) {
+ auto *url = (rspamd_url *) g_ptr_array_index(purls, j);
+ CHECK(expected_urls[j] == std::string{url->string, url->urllen});
+ }
+ g_byte_array_free(tmp, TRUE);
+ g_ptr_array_free(purls, TRUE);
+ }
+ ++i;
+ }
+
+ rspamd_mempool_delete(pool);
+ }
+}
+
+} /* namespace rspamd::html */
diff --git a/src/libserver/html/html_url.cxx b/src/libserver/html/html_url.cxx
new file mode 100644
index 0000000..8f29f2c
--- /dev/null
+++ b/src/libserver/html/html_url.cxx
@@ -0,0 +1,496 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "html_url.hxx"
+#include "libutil/str_util.h"
+#include "libserver/url.h"
+#include "libserver/logger.h"
+#include "rspamd.h"
+
+#include <unicode/idna.h>
+
+namespace rspamd::html {
+
+static auto
+rspamd_url_is_subdomain(std::string_view t1, std::string_view t2) -> bool
+{
+ const auto *p1 = t1.data() + t1.size() - 1;
+ const auto *p2 = t2.data() + t2.size() - 1;
+
+ /* Skip trailing dots */
+ while (p1 > t1.data()) {
+ if (*p1 != '.') {
+ break;
+ }
+
+ p1--;
+ }
+
+ while (p2 > t2.data()) {
+ if (*p2 != '.') {
+ break;
+ }
+
+ p2--;
+ }
+
+ while (p1 > t1.data() && p2 > t2.data()) {
+ if (*p1 != *p2) {
+ break;
+ }
+
+ p1--;
+ p2--;
+ }
+
+ if (p2 == t2.data()) {
+ /* p2 can be subdomain of p1 if *p1 is '.' */
+ if (p1 != t1.data() && *(p1 - 1) == '.') {
+ return true;
+ }
+ }
+ else if (p1 == t1.data()) {
+ if (p2 != t2.data() && *(p2 - 1) == '.') {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+
+static auto
+get_icu_idna_instance(void) -> auto
+{
+ auto uc_err = U_ZERO_ERROR;
+ static auto *udn = icu::IDNA::createUTS46Instance(UIDNA_DEFAULT, uc_err);
+
+ return udn;
+}
+
+static auto
+convert_idna_hostname_maybe(rspamd_mempool_t *pool, struct rspamd_url *url, bool use_tld)
+ -> std::string_view
+{
+ std::string_view ret = use_tld ? std::string_view{rspamd_url_tld_unsafe(url), url->tldlen} : std::string_view{rspamd_url_host_unsafe(url), url->hostlen};
+
+ /* Handle IDN url's */
+ if (ret.size() > 4 &&
+ rspamd_substring_search_caseless(ret.data(), ret.size(), "xn--", 4) != -1) {
+
+ const auto buf_capacity = ret.size() * 2 + 1;
+ auto *idn_hbuf = (char *) rspamd_mempool_alloc(pool, buf_capacity);
+ icu::CheckedArrayByteSink byte_sink{idn_hbuf, (int) buf_capacity};
+
+ /* We need to convert it to the normal value first */
+ icu::IDNAInfo info;
+ auto uc_err = U_ZERO_ERROR;
+ auto *udn = get_icu_idna_instance();
+ udn->nameToUnicodeUTF8(icu::StringPiece(ret.data(), ret.size()),
+ byte_sink, info, uc_err);
+
+ if (uc_err == U_ZERO_ERROR && !info.hasErrors()) {
+ /* idn_hbuf is allocated in mempool, so it is safe to use */
+ ret = std::string_view{idn_hbuf, (std::size_t) byte_sink.NumberOfBytesWritten()};
+ }
+ else {
+ msg_err_pool("cannot convert to IDN: %s (0x%xd)",
+ u_errorName(uc_err), info.getErrors());
+ }
+ }
+
+ return ret;
+};
+
+constexpr auto sv_equals(std::string_view s1, std::string_view s2) -> auto
+{
+ return (s1.size() == s2.size()) &&
+ std::equal(s1.begin(), s1.end(), s2.begin(), s2.end(),
+ [](const auto c1, const auto c2) {
+ return g_ascii_tolower(c1) == g_ascii_tolower(c2);
+ });
+}
+
+constexpr auto
+is_transfer_proto(struct rspamd_url *u) -> bool
+{
+ return (u->protocol & (PROTOCOL_HTTP | PROTOCOL_HTTPS | PROTOCOL_FTP)) != 0;
+}
+
+auto html_url_is_phished(rspamd_mempool_t *pool,
+ struct rspamd_url *href_url,
+ std::string_view text_data) -> std::optional<rspamd_url *>
+{
+ struct rspamd_url *text_url;
+ std::string_view disp_tok, href_tok;
+ goffset url_pos;
+ gchar *url_str = NULL;
+
+ auto sz = text_data.size();
+ const auto *trimmed = rspamd_string_unicode_trim_inplace(text_data.data(), &sz);
+ text_data = std::string_view(trimmed, sz);
+
+ if (text_data.size() > 4 &&
+ rspamd_url_find(pool, text_data.data(), text_data.size(), &url_str,
+ RSPAMD_URL_FIND_ALL,
+ &url_pos, NULL) &&
+ url_str != nullptr) {
+
+ if (url_pos > 0) {
+ /*
+ * We have some url at some offset, so we need to check what is
+ * at the start of the text
+ */
+ return std::nullopt;
+ }
+
+ text_url = rspamd_mempool_alloc0_type(pool, struct rspamd_url);
+ auto rc = rspamd_url_parse(text_url, url_str, strlen(url_str), pool,
+ RSPAMD_URL_PARSE_TEXT);
+
+ if (rc == URI_ERRNO_OK) {
+ text_url->flags |= RSPAMD_URL_FLAG_HTML_DISPLAYED;
+ href_url->flags |= RSPAMD_URL_FLAG_DISPLAY_URL;
+
+ /* Check for phishing */
+ if (is_transfer_proto(text_url) == is_transfer_proto(href_url)) {
+ disp_tok = convert_idna_hostname_maybe(pool, text_url, false);
+ href_tok = convert_idna_hostname_maybe(pool, href_url, false);
+
+ if (!sv_equals(disp_tok, href_tok) &&
+ text_url->tldlen > 0 && href_url->tldlen > 0) {
+
+ /* Apply the same logic for TLD */
+ disp_tok = convert_idna_hostname_maybe(pool, text_url, true);
+ href_tok = convert_idna_hostname_maybe(pool, href_url, true);
+
+ if (!sv_equals(disp_tok, href_tok)) {
+ /* Check if one url is a subdomain for another */
+
+ if (!rspamd_url_is_subdomain(disp_tok, href_tok)) {
+ href_url->flags |= RSPAMD_URL_FLAG_PHISHED;
+ text_url->flags |= RSPAMD_URL_FLAG_HTML_DISPLAYED;
+
+ if (href_url->ext == nullptr) {
+ href_url->ext = rspamd_mempool_alloc0_type(pool, rspamd_url_ext);
+ }
+ href_url->ext->linked_url = text_url;
+ }
+ }
+ }
+ }
+
+ return text_url;
+ }
+ else {
+ /*
+ * We have found something that looks like an url but it was
+ * not parsed correctly.
+ * Sometimes it means an obfuscation attempt, so we have to check
+ * what's inside of the text
+ */
+ gboolean obfuscation_found = FALSE;
+
+ if (text_data.size() > 4 && g_ascii_strncasecmp(text_data.begin(), "http", 4) == 0 &&
+ rspamd_substring_search(text_data.begin(), text_data.size(), "://", 3) != -1) {
+ /* Clearly an obfuscation attempt */
+ obfuscation_found = TRUE;
+ }
+
+ msg_info_pool("extract of url '%s' failed: %s; obfuscation detected: %s",
+ url_str,
+ rspamd_url_strerror(rc),
+ obfuscation_found ? "yes" : "no");
+
+ if (obfuscation_found) {
+ href_url->flags |= RSPAMD_URL_FLAG_PHISHED | RSPAMD_URL_FLAG_OBSCURED;
+ }
+ }
+ }
+
+ return std::nullopt;
+}
+
+void html_check_displayed_url(rspamd_mempool_t *pool,
+ GList **exceptions,
+ void *url_set,
+ std::string_view visible_part,
+ goffset href_offset,
+ struct rspamd_url *url)
+{
+ struct rspamd_url *displayed_url = nullptr;
+ struct rspamd_url *turl;
+ struct rspamd_process_exception *ex;
+ guint saved_flags = 0;
+ gsize dlen;
+
+ if (visible_part.empty()) {
+ /* No displayed url, just some text within <a> tag */
+ return;
+ }
+
+ if (url->ext == nullptr) {
+ url->ext = rspamd_mempool_alloc0_type(pool, rspamd_url_ext);
+ }
+ url->ext->visible_part = rspamd_mempool_alloc_buffer(pool, visible_part.size() + 1);
+ rspamd_strlcpy(url->ext->visible_part,
+ visible_part.data(),
+ visible_part.size() + 1);
+ dlen = visible_part.size();
+
+ /* Strip unicode spaces from the start and the end */
+ url->ext->visible_part = const_cast<char *>(
+ rspamd_string_unicode_trim_inplace(url->ext->visible_part,
+ &dlen));
+ auto maybe_url = html_url_is_phished(pool, url,
+ {url->ext->visible_part, dlen});
+
+ if (maybe_url) {
+ url->flags |= saved_flags;
+ displayed_url = maybe_url.value();
+ }
+
+ if (exceptions && displayed_url != nullptr) {
+ ex = rspamd_mempool_alloc_type(pool, struct rspamd_process_exception);
+ ex->pos = href_offset;
+ ex->len = dlen;
+ ex->type = RSPAMD_EXCEPTION_URL;
+ ex->ptr = url;
+
+ *exceptions = g_list_prepend(*exceptions, ex);
+ }
+
+ if (displayed_url && url_set) {
+ turl = rspamd_url_set_add_or_return((khash_t(rspamd_url_hash) *) url_set, displayed_url);
+
+ if (turl != nullptr) {
+ /* Here, we assume the following:
+ * if we have a URL in the text part which
+ * is the same as displayed URL in the
+ * HTML part, we assume that it is also
+ * hint only.
+ */
+ if (turl->flags & RSPAMD_URL_FLAG_FROM_TEXT) {
+
+ /*
+ * We have the same URL for href and displayed url, so we
+ * know that this url cannot be both target and display (as
+ * it breaks logic in many places), so we do not
+ * propagate html flags
+ */
+ if (!(turl->flags & RSPAMD_URL_FLAG_DISPLAY_URL)) {
+ turl->flags |= displayed_url->flags;
+ }
+ turl->flags &= ~RSPAMD_URL_FLAG_FROM_TEXT;
+ }
+
+ turl->count++;
+ }
+ else {
+ /* Already inserted by `rspamd_url_set_add_or_return` */
+ }
+ }
+
+ rspamd_normalise_unicode_inplace(url->ext->visible_part, &dlen);
+}
+
+auto html_process_url(rspamd_mempool_t *pool, std::string_view &input)
+ -> std::optional<struct rspamd_url *>
+{
+ struct rspamd_url *url;
+ guint saved_flags = 0;
+ gint rc;
+ const gchar *s, *prefix = "http://";
+ gchar *d;
+ gsize dlen;
+ gboolean has_bad_chars = FALSE, no_prefix = FALSE;
+ static const gchar hexdigests[] = "0123456789abcdef";
+
+ auto sz = input.length();
+ const auto *trimmed = rspamd_string_unicode_trim_inplace(input.data(), &sz);
+ input = {trimmed, sz};
+
+ const auto *start = input.data();
+ s = start;
+ dlen = 0;
+
+ for (auto i = 0; i < sz; i++) {
+ if (G_UNLIKELY(((guint) s[i]) < 0x80 && !g_ascii_isgraph(s[i]))) {
+ dlen += 3;
+ }
+ else {
+ dlen++;
+ }
+ }
+
+ if (rspamd_substring_search(start, sz, "://", 3) == -1) {
+ if (sz >= sizeof("mailto:") &&
+ (memcmp(start, "mailto:", sizeof("mailto:") - 1) == 0 ||
+ memcmp(start, "tel:", sizeof("tel:") - 1) == 0 ||
+ memcmp(start, "callto:", sizeof("callto:") - 1) == 0)) {
+ /* Exclusion, has valid but 'strange' prefix */
+ }
+ else {
+ for (auto i = 0; i < sz; i++) {
+ if (!((s[i] & 0x80) || g_ascii_isalnum(s[i]))) {
+ if (i == 0 && sz > 2 && s[i] == '/' && s[i + 1] == '/') {
+ prefix = "http:";
+ dlen += sizeof("http:") - 1;
+ no_prefix = TRUE;
+ }
+ else if (s[i] == '@') {
+ /* Likely email prefix */
+ prefix = "mailto://";
+ dlen += sizeof("mailto://") - 1;
+ no_prefix = TRUE;
+ }
+ else if (s[i] == ':' && i != 0) {
+ /* Special case */
+ no_prefix = FALSE;
+ }
+ else {
+ if (i == 0) {
+ /* No valid data */
+ return std::nullopt;
+ }
+ else {
+ no_prefix = TRUE;
+ dlen += strlen(prefix);
+ }
+ }
+
+ break;
+ }
+ }
+ }
+ }
+
+ auto *decoded = rspamd_mempool_alloc_buffer(pool, dlen + 1);
+ d = decoded;
+
+ if (no_prefix) {
+ gsize plen = strlen(prefix);
+ memcpy(d, prefix, plen);
+ d += plen;
+ }
+
+ /*
+ * We also need to remove all internal newlines, spaces
+ * and encode unsafe characters
+ * Another obfuscation find in the wild was encoding of the SAFE url characters,
+ * including essential ones
+ */
+ for (auto i = 0; i < sz; i++) {
+ if (G_UNLIKELY(g_ascii_isspace(s[i]))) {
+ continue;
+ }
+ else if (G_UNLIKELY(((guint) s[i]) < 0x80 && !g_ascii_isgraph(s[i]))) {
+ /* URL encode */
+ *d++ = '%';
+ *d++ = hexdigests[(s[i] >> 4) & 0xf];
+ *d++ = hexdigests[s[i] & 0xf];
+ has_bad_chars = TRUE;
+ }
+ else if (G_UNLIKELY(s[i] == '%')) {
+ if (i + 2 < sz) {
+ auto c1 = s[i + 1];
+ auto c2 = s[i + 2];
+
+ if (g_ascii_isxdigit(c1) && g_ascii_isxdigit(c2)) {
+ auto codepoint = 0;
+
+ if (c1 >= '0' && c1 <= '9') codepoint = c1 - '0';
+ else if (c1 >= 'A' && c1 <= 'F')
+ codepoint = c1 - 'A' + 10;
+ else if (c1 >= 'a' && c1 <= 'f')
+ codepoint = c1 - 'a' + 10;
+
+ codepoint <<= 4;
+
+ if (c2 >= '0' && c2 <= '9') codepoint += c2 - '0';
+ else if (c2 >= 'A' && c2 <= 'F')
+ codepoint += c2 - 'A' + 10;
+ else if (c2 >= 'a' && c2 <= 'f')
+ codepoint += c2 - 'a' + 10;
+
+ /* Now check for 'interesting' codepoints */
+ if (codepoint == '@' || codepoint == ':' || codepoint == '|' ||
+ codepoint == '?' || codepoint == '\\' || codepoint == '/') {
+ /* Replace it back */
+ *d++ = (char) (codepoint & 0xff);
+ i += 2;
+ }
+ else {
+ *d++ = s[i];
+ }
+ }
+ else {
+ *d++ = s[i];
+ }
+ }
+ else {
+ *d++ = s[i];
+ }
+ }
+ else {
+ *d++ = s[i];
+ }
+ }
+
+ *d = '\0';
+ dlen = d - decoded;
+
+ url = rspamd_mempool_alloc0_type(pool, struct rspamd_url);
+ rspamd_url_normalise_propagate_flags(pool, decoded, &dlen, saved_flags);
+ rc = rspamd_url_parse(url, decoded, dlen, pool, RSPAMD_URL_PARSE_HREF);
+
+ /* Filter some completely damaged urls */
+ if (rc == URI_ERRNO_OK && url->hostlen > 0 &&
+ !((url->protocol & PROTOCOL_UNKNOWN))) {
+ url->flags |= saved_flags;
+
+ if (has_bad_chars) {
+ url->flags |= RSPAMD_URL_FLAG_OBSCURED;
+ }
+
+ if (no_prefix) {
+ url->flags |= RSPAMD_URL_FLAG_SCHEMALESS;
+
+ if (url->tldlen == 0 || (url->flags & RSPAMD_URL_FLAG_NO_TLD)) {
+ /* Ignore urls with both no schema and no tld */
+ return std::nullopt;
+ }
+ }
+
+ decoded = url->string;
+
+ input = {decoded, url->urllen};
+
+ /* Spaces in href usually mean an attempt to obfuscate URL */
+ /* See https://github.com/vstakhov/rspamd/issues/593 */
+#if 0
+ if (has_spaces) {
+ url->flags |= RSPAMD_URL_FLAG_OBSCURED;
+ }
+#endif
+
+ return url;
+ }
+
+ return std::nullopt;
+}
+
+}// namespace rspamd::html \ No newline at end of file
diff --git a/src/libserver/html/html_url.hxx b/src/libserver/html/html_url.hxx
new file mode 100644
index 0000000..46dde6d
--- /dev/null
+++ b/src/libserver/html/html_url.hxx
@@ -0,0 +1,68 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_HTML_URL_HXX
+#define RSPAMD_HTML_URL_HXX
+#pragma once
+
+#include "libutil/mem_pool.h"
+
+#include <string_view>
+#include <optional>
+
+struct rspamd_url; /* Forward declaration */
+
+namespace rspamd::html {
+
+
+/**
+ * Checks if an html url is likely phished by some displayed url
+ * @param pool
+ * @param href_url
+ * @param text_data
+ * @return
+ */
+auto html_url_is_phished(rspamd_mempool_t *pool,
+ struct rspamd_url *href_url,
+ std::string_view text_data) -> std::optional<rspamd_url *>;
+
+/**
+ * Check displayed part of the url at specified offset
+ * @param pool
+ * @param exceptions
+ * @param url_set
+ * @param visible_part
+ * @param href_offset
+ * @param url
+ */
+auto html_check_displayed_url(rspamd_mempool_t *pool,
+ GList **exceptions,
+ void *url_set,
+ std::string_view visible_part,
+ goffset href_offset,
+ struct rspamd_url *url) -> void;
+
+/**
+ * Process HTML url (e.g. for href component)
+ * @param pool
+ * @param input may be modified during the process
+ * @return
+ */
+auto html_process_url(rspamd_mempool_t *pool, std::string_view &input)
+ -> std::optional<struct rspamd_url *>;
+}// namespace rspamd::html
+
+#endif//RSPAMD_HTML_URL_HXX \ No newline at end of file
diff --git a/src/libserver/http/http_connection.c b/src/libserver/http/http_connection.c
new file mode 100644
index 0000000..5557fbf
--- /dev/null
+++ b/src/libserver/http/http_connection.c
@@ -0,0 +1,2649 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "http_connection.h"
+#include "http_private.h"
+#include "http_message.h"
+#include "utlist.h"
+#include "util.h"
+#include "printf.h"
+#include "logger.h"
+#include "ref.h"
+#include "ottery.h"
+#include "keypair_private.h"
+#include "cryptobox.h"
+#include "libutil/libev_helper.h"
+#include "libserver/ssl_util.h"
+#include "libserver/url.h"
+
+#include "contrib/mumhash/mum.h"
+#include "contrib/http-parser/http_parser.h"
+#include "unix-std.h"
+
+#include <openssl/err.h>
+
+#define ENCRYPTED_VERSION " HTTP/1.0"
+
+struct _rspamd_http_privbuf {
+ rspamd_fstring_t *data;
+ const gchar *zc_buf;
+ gsize zc_remain;
+ ref_entry_t ref;
+};
+
+enum rspamd_http_priv_flags {
+ RSPAMD_HTTP_CONN_FLAG_ENCRYPTED = 1u << 0u,
+ RSPAMD_HTTP_CONN_FLAG_NEW_HEADER = 1u << 1u,
+ RSPAMD_HTTP_CONN_FLAG_RESETED = 1u << 2u,
+ RSPAMD_HTTP_CONN_FLAG_TOO_LARGE = 1u << 3u,
+ RSPAMD_HTTP_CONN_FLAG_ENCRYPTION_NEEDED = 1u << 4u,
+ RSPAMD_HTTP_CONN_FLAG_PROXY = 1u << 5u,
+ RSPAMD_HTTP_CONN_FLAG_PROXY_REQUEST = 1u << 6u,
+ RSPAMD_HTTP_CONN_OWN_SOCKET = 1u << 7u,
+};
+
+#define IS_CONN_ENCRYPTED(c) ((c)->flags & RSPAMD_HTTP_CONN_FLAG_ENCRYPTED)
+#define IS_CONN_RESETED(c) ((c)->flags & RSPAMD_HTTP_CONN_FLAG_RESETED)
+
+struct rspamd_http_connection_private {
+ struct rspamd_http_context *ctx;
+ struct rspamd_ssl_connection *ssl;
+ struct _rspamd_http_privbuf *buf;
+ struct rspamd_keypair_cache *cache;
+ struct rspamd_cryptobox_pubkey *peer_key;
+ struct rspamd_cryptobox_keypair *local_key;
+ struct rspamd_http_header *header;
+ struct http_parser parser;
+ struct http_parser_settings parser_cb;
+ struct rspamd_io_ev ev;
+ ev_tstamp timeout;
+ struct rspamd_http_message *msg;
+ struct iovec *out;
+ guint outlen;
+ enum rspamd_http_priv_flags flags;
+ gsize wr_pos;
+ gsize wr_total;
+};
+
+static const rspamd_ftok_t key_header = {
+ .begin = "Key",
+ .len = 3};
+static const rspamd_ftok_t date_header = {
+ .begin = "Date",
+ .len = 4};
+static const rspamd_ftok_t last_modified_header = {
+ .begin = "Last-Modified",
+ .len = 13};
+
+static void rspamd_http_event_handler(int fd, short what, gpointer ud);
+static void rspamd_http_ssl_err_handler(gpointer ud, GError *err);
+
+
+#define HTTP_ERROR http_error_quark()
+GQuark
+http_error_quark(void)
+{
+ return g_quark_from_static_string("http-error-quark");
+}
+
+static void
+rspamd_http_privbuf_dtor(gpointer ud)
+{
+ struct _rspamd_http_privbuf *p = (struct _rspamd_http_privbuf *) ud;
+
+ if (p->data) {
+ rspamd_fstring_free(p->data);
+ }
+
+ g_free(p);
+}
+
+static const gchar *
+rspamd_http_code_to_str(gint code)
+{
+ if (code == 200) {
+ return "OK";
+ }
+ else if (code == 404) {
+ return "Not found";
+ }
+ else if (code == 403 || code == 401) {
+ return "Not authorized";
+ }
+ else if (code >= 400 && code < 500) {
+ return "Bad request";
+ }
+ else if (code >= 300 && code < 400) {
+ return "See Other";
+ }
+ else if (code >= 500 && code < 600) {
+ return "Internal server error";
+ }
+
+ return "Unknown error";
+}
+
+static void
+rspamd_http_parse_key(rspamd_ftok_t *data, struct rspamd_http_connection *conn,
+ struct rspamd_http_connection_private *priv)
+{
+ guchar *decoded_id;
+ const gchar *eq_pos;
+ gsize id_len;
+ struct rspamd_cryptobox_pubkey *pk;
+
+ if (priv->local_key == NULL) {
+ /* In this case we cannot do anything, e.g. we cannot decrypt payload */
+ priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_ENCRYPTED;
+ }
+ else {
+ /* Check sanity of what we have */
+ eq_pos = memchr(data->begin, '=', data->len);
+ if (eq_pos != NULL) {
+ decoded_id = rspamd_decode_base32(data->begin, eq_pos - data->begin,
+ &id_len, RSPAMD_BASE32_DEFAULT);
+
+ if (decoded_id != NULL && id_len >= RSPAMD_KEYPAIR_SHORT_ID_LEN) {
+ pk = rspamd_pubkey_from_base32(eq_pos + 1,
+ data->begin + data->len - eq_pos - 1,
+ RSPAMD_KEYPAIR_KEX,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+ if (pk != NULL) {
+ if (memcmp(rspamd_keypair_get_id(priv->local_key),
+ decoded_id,
+ RSPAMD_KEYPAIR_SHORT_ID_LEN) == 0) {
+ priv->msg->peer_key = pk;
+
+ if (priv->cache && priv->msg->peer_key) {
+ rspamd_keypair_cache_process(priv->cache,
+ priv->local_key,
+ priv->msg->peer_key);
+ }
+ }
+ else {
+ rspamd_pubkey_unref(pk);
+ }
+ }
+ }
+
+ priv->flags |= RSPAMD_HTTP_CONN_FLAG_ENCRYPTED;
+ g_free(decoded_id);
+ }
+ }
+}
+
+static inline void
+rspamd_http_check_special_header(struct rspamd_http_connection *conn,
+ struct rspamd_http_connection_private *priv)
+{
+ if (rspamd_ftok_casecmp(&priv->header->name, &date_header) == 0) {
+ priv->msg->date = rspamd_http_parse_date(priv->header->value.begin,
+ priv->header->value.len);
+ }
+ else if (rspamd_ftok_casecmp(&priv->header->name, &key_header) == 0) {
+ rspamd_http_parse_key(&priv->header->value, conn, priv);
+ }
+ else if (rspamd_ftok_casecmp(&priv->header->name, &last_modified_header) == 0) {
+ priv->msg->last_modified = rspamd_http_parse_date(
+ priv->header->value.begin,
+ priv->header->value.len);
+ }
+}
+
+static gint
+rspamd_http_on_url(http_parser *parser, const gchar *at, size_t length)
+{
+ struct rspamd_http_connection *conn =
+ (struct rspamd_http_connection *) parser->data;
+ struct rspamd_http_connection_private *priv;
+
+ priv = conn->priv;
+
+ priv->msg->url = rspamd_fstring_append(priv->msg->url, at, length);
+
+ return 0;
+}
+
+static gint
+rspamd_http_on_status(http_parser *parser, const gchar *at, size_t length)
+{
+ struct rspamd_http_connection *conn =
+ (struct rspamd_http_connection *) parser->data;
+ struct rspamd_http_connection_private *priv;
+
+ priv = conn->priv;
+
+ if (parser->status_code != 200) {
+ if (priv->msg->status == NULL) {
+ priv->msg->status = rspamd_fstring_new();
+ }
+
+ priv->msg->status = rspamd_fstring_append(priv->msg->status, at, length);
+ }
+
+ return 0;
+}
+
+static void
+rspamd_http_finish_header(struct rspamd_http_connection *conn,
+ struct rspamd_http_connection_private *priv)
+{
+ struct rspamd_http_header *hdr;
+ khiter_t k;
+ gint r;
+
+ priv->header->combined = rspamd_fstring_append(priv->header->combined,
+ "\r\n", 2);
+ priv->header->value.len = priv->header->combined->len -
+ priv->header->name.len - 4;
+ priv->header->value.begin = priv->header->combined->str +
+ priv->header->name.len + 2;
+ priv->header->name.begin = priv->header->combined->str;
+
+ k = kh_put(rspamd_http_headers_hash, priv->msg->headers, &priv->header->name,
+ &r);
+
+ if (r != 0) {
+ kh_value(priv->msg->headers, k) = priv->header;
+ hdr = NULL;
+ }
+ else {
+ hdr = kh_value(priv->msg->headers, k);
+ }
+
+ DL_APPEND(hdr, priv->header);
+
+ rspamd_http_check_special_header(conn, priv);
+}
+
+static void
+rspamd_http_init_header(struct rspamd_http_connection_private *priv)
+{
+ priv->header = g_malloc0(sizeof(struct rspamd_http_header));
+ priv->header->combined = rspamd_fstring_new();
+}
+
+static gint
+rspamd_http_on_header_field(http_parser *parser,
+ const gchar *at,
+ size_t length)
+{
+ struct rspamd_http_connection *conn =
+ (struct rspamd_http_connection *) parser->data;
+ struct rspamd_http_connection_private *priv;
+
+ priv = conn->priv;
+
+ if (priv->header == NULL) {
+ rspamd_http_init_header(priv);
+ }
+ else if (priv->flags & RSPAMD_HTTP_CONN_FLAG_NEW_HEADER) {
+ rspamd_http_finish_header(conn, priv);
+ rspamd_http_init_header(priv);
+ }
+
+ priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_NEW_HEADER;
+ priv->header->combined = rspamd_fstring_append(priv->header->combined,
+ at, length);
+
+ return 0;
+}
+
+static gint
+rspamd_http_on_header_value(http_parser *parser,
+ const gchar *at,
+ size_t length)
+{
+ struct rspamd_http_connection *conn =
+ (struct rspamd_http_connection *) parser->data;
+ struct rspamd_http_connection_private *priv;
+
+ priv = conn->priv;
+
+ if (priv->header == NULL) {
+ /* Should not happen */
+ return -1;
+ }
+
+ if (!(priv->flags & RSPAMD_HTTP_CONN_FLAG_NEW_HEADER)) {
+ priv->flags |= RSPAMD_HTTP_CONN_FLAG_NEW_HEADER;
+ priv->header->combined = rspamd_fstring_append(priv->header->combined,
+ ": ", 2);
+ priv->header->name.len = priv->header->combined->len - 2;
+ }
+
+ priv->header->combined = rspamd_fstring_append(priv->header->combined,
+ at, length);
+
+ return 0;
+}
+
+static int
+rspamd_http_on_headers_complete(http_parser *parser)
+{
+ struct rspamd_http_connection *conn =
+ (struct rspamd_http_connection *) parser->data;
+ struct rspamd_http_connection_private *priv;
+ struct rspamd_http_message *msg;
+ int ret;
+
+ priv = conn->priv;
+ msg = priv->msg;
+
+ if (priv->header != NULL) {
+ rspamd_http_finish_header(conn, priv);
+
+ priv->header = NULL;
+ priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_NEW_HEADER;
+ }
+
+ if (msg->method == HTTP_HEAD) {
+ /* We don't care about the rest */
+ rspamd_ev_watcher_stop(priv->ctx->event_loop, &priv->ev);
+
+ msg->code = parser->status_code;
+ rspamd_http_connection_ref(conn);
+ ret = conn->finish_handler(conn, msg);
+
+ if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) {
+ rspamd_http_context_push_keepalive(conn->priv->ctx, conn,
+ msg, conn->priv->ctx->event_loop);
+ rspamd_http_connection_reset(conn);
+ }
+ else {
+ conn->finished = TRUE;
+ }
+
+ rspamd_http_connection_unref(conn);
+
+ return ret;
+ }
+
+ /*
+ * HTTP parser sets content length to (-1) when it doesn't know the real
+ * length, for example, in case of chunked encoding.
+ *
+ * Hence, we skip body setup here
+ */
+ if (parser->content_length != ULLONG_MAX && parser->content_length != 0 &&
+ msg->method != HTTP_HEAD) {
+ if (conn->max_size > 0 &&
+ parser->content_length > conn->max_size) {
+ /* Too large message */
+ priv->flags |= RSPAMD_HTTP_CONN_FLAG_TOO_LARGE;
+ return -1;
+ }
+
+ if (!rspamd_http_message_set_body(msg, NULL, parser->content_length)) {
+ return -1;
+ }
+ }
+
+ if (parser->flags & F_SPAMC) {
+ msg->flags |= RSPAMD_HTTP_FLAG_SPAMC;
+ }
+
+
+ msg->method = parser->method;
+ msg->code = parser->status_code;
+
+ return 0;
+}
+
+static void
+rspamd_http_switch_zc(struct _rspamd_http_privbuf *pbuf,
+ struct rspamd_http_message *msg)
+{
+ pbuf->zc_buf = msg->body_buf.begin + msg->body_buf.len;
+ pbuf->zc_remain = msg->body_buf.allocated_len - msg->body_buf.len;
+}
+
+static int
+rspamd_http_on_body(http_parser *parser, const gchar *at, size_t length)
+{
+ struct rspamd_http_connection *conn =
+ (struct rspamd_http_connection *) parser->data;
+ struct rspamd_http_connection_private *priv;
+ struct rspamd_http_message *msg;
+ struct _rspamd_http_privbuf *pbuf;
+ const gchar *p;
+
+ priv = conn->priv;
+ msg = priv->msg;
+ pbuf = priv->buf;
+ p = at;
+
+ if (!(msg->flags & RSPAMD_HTTP_FLAG_HAS_BODY)) {
+ if (!rspamd_http_message_set_body(msg, NULL, parser->content_length)) {
+ return -1;
+ }
+ }
+
+ if (conn->finished) {
+ return 0;
+ }
+
+ if (conn->max_size > 0 &&
+ msg->body_buf.len + length > conn->max_size) {
+ /* Body length overflow */
+ priv->flags |= RSPAMD_HTTP_CONN_FLAG_TOO_LARGE;
+ return -1;
+ }
+
+ if (!pbuf->zc_buf) {
+ if (!rspamd_http_message_append_body(msg, at, length)) {
+ return -1;
+ }
+
+ /* We might have some leftover in our private buffer */
+ if (pbuf->data->len == length) {
+ /* Switch to zero-copy mode */
+ rspamd_http_switch_zc(pbuf, msg);
+ }
+ }
+ else {
+ if (msg->body_buf.begin + msg->body_buf.len != at) {
+ /* Likely chunked encoding */
+ memmove((gchar *) msg->body_buf.begin + msg->body_buf.len, at, length);
+ p = msg->body_buf.begin + msg->body_buf.len;
+ }
+
+ /* Adjust zero-copy buf */
+ msg->body_buf.len += length;
+
+ if (!(msg->flags & RSPAMD_HTTP_FLAG_SHMEM)) {
+ msg->body_buf.c.normal->len += length;
+ }
+
+ pbuf->zc_buf = msg->body_buf.begin + msg->body_buf.len;
+ pbuf->zc_remain = msg->body_buf.allocated_len - msg->body_buf.len;
+ }
+
+ if ((conn->opts & RSPAMD_HTTP_BODY_PARTIAL) && !IS_CONN_ENCRYPTED(priv)) {
+ /* Incremental update is impossible for encrypted requests so far */
+ return (conn->body_handler(conn, msg, p, length));
+ }
+
+ return 0;
+}
+
+static int
+rspamd_http_on_body_decrypted(http_parser *parser, const gchar *at, size_t length)
+{
+ struct rspamd_http_connection *conn =
+ (struct rspamd_http_connection *) parser->data;
+ struct rspamd_http_connection_private *priv;
+
+ priv = conn->priv;
+
+ if (priv->header != NULL) {
+ rspamd_http_finish_header(conn, priv);
+ priv->header = NULL;
+ }
+
+ if (conn->finished) {
+ return 0;
+ }
+
+ if (priv->msg->body_buf.len == 0) {
+
+ priv->msg->body_buf.begin = at;
+ priv->msg->method = parser->method;
+ priv->msg->code = parser->status_code;
+ }
+
+ priv->msg->body_buf.len += length;
+
+ return 0;
+}
+
+static int
+rspamd_http_on_headers_complete_decrypted(http_parser *parser)
+{
+ struct rspamd_http_connection *conn =
+ (struct rspamd_http_connection *) parser->data;
+ struct rspamd_http_connection_private *priv;
+ struct rspamd_http_message *msg;
+ int ret;
+
+ priv = conn->priv;
+ msg = priv->msg;
+
+ if (priv->header != NULL) {
+ rspamd_http_finish_header(conn, priv);
+
+ priv->header = NULL;
+ priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_NEW_HEADER;
+ }
+
+ if (parser->flags & F_SPAMC) {
+ priv->msg->flags |= RSPAMD_HTTP_FLAG_SPAMC;
+ }
+
+ if (msg->method == HTTP_HEAD) {
+ /* We don't care about the rest */
+ rspamd_ev_watcher_stop(priv->ctx->event_loop, &priv->ev);
+ msg->code = parser->status_code;
+ rspamd_http_connection_ref(conn);
+ ret = conn->finish_handler(conn, msg);
+
+ if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) {
+ rspamd_http_context_push_keepalive(conn->priv->ctx, conn,
+ msg, conn->priv->ctx->event_loop);
+ rspamd_http_connection_reset(conn);
+ }
+ else {
+ conn->finished = TRUE;
+ }
+
+ rspamd_http_connection_unref(conn);
+
+ return ret;
+ }
+
+ priv->msg->method = parser->method;
+ priv->msg->code = parser->status_code;
+
+ return 0;
+}
+
+static int
+rspamd_http_decrypt_message(struct rspamd_http_connection *conn,
+ struct rspamd_http_connection_private *priv,
+ struct rspamd_cryptobox_pubkey *peer_key)
+{
+ guchar *nonce, *m;
+ const guchar *nm;
+ gsize dec_len;
+ struct rspamd_http_message *msg = priv->msg;
+ struct rspamd_http_header *hdr, *hcur, *hcurtmp;
+ struct http_parser decrypted_parser;
+ struct http_parser_settings decrypted_cb;
+ enum rspamd_cryptobox_mode mode;
+
+ mode = rspamd_keypair_alg(priv->local_key);
+ nonce = msg->body_buf.str;
+ m = msg->body_buf.str + rspamd_cryptobox_nonce_bytes(mode) +
+ rspamd_cryptobox_mac_bytes(mode);
+ dec_len = msg->body_buf.len - rspamd_cryptobox_nonce_bytes(mode) -
+ rspamd_cryptobox_mac_bytes(mode);
+
+ if ((nm = rspamd_pubkey_get_nm(peer_key, priv->local_key)) == NULL) {
+ nm = rspamd_pubkey_calculate_nm(peer_key, priv->local_key);
+ }
+
+ if (!rspamd_cryptobox_decrypt_nm_inplace(m, dec_len, nonce,
+ nm, m - rspamd_cryptobox_mac_bytes(mode), mode)) {
+ msg_err("cannot verify encrypted message, first bytes of the input: %*xs",
+ (gint) MIN(msg->body_buf.len, 64), msg->body_buf.begin);
+ return -1;
+ }
+
+ /* Cleanup message */
+ kh_foreach_value (msg->headers, hdr, {
+ DL_FOREACH_SAFE (hdr, hcur, hcurtmp) {
+ rspamd_fstring_free (hcur->combined);
+ g_free (hcur);
+}
+});
+
+kh_destroy(rspamd_http_headers_hash, msg->headers);
+msg->headers = kh_init(rspamd_http_headers_hash);
+
+if (msg->url != NULL) {
+ msg->url = rspamd_fstring_assign(msg->url, "", 0);
+}
+
+msg->body_buf.len = 0;
+
+memset(&decrypted_parser, 0, sizeof(decrypted_parser));
+http_parser_init(&decrypted_parser,
+ conn->type == RSPAMD_HTTP_SERVER ? HTTP_REQUEST : HTTP_RESPONSE);
+
+memset(&decrypted_cb, 0, sizeof(decrypted_cb));
+decrypted_cb.on_url = rspamd_http_on_url;
+decrypted_cb.on_status = rspamd_http_on_status;
+decrypted_cb.on_header_field = rspamd_http_on_header_field;
+decrypted_cb.on_header_value = rspamd_http_on_header_value;
+decrypted_cb.on_headers_complete = rspamd_http_on_headers_complete_decrypted;
+decrypted_cb.on_body = rspamd_http_on_body_decrypted;
+decrypted_parser.data = conn;
+decrypted_parser.content_length = dec_len;
+
+if (http_parser_execute(&decrypted_parser, &decrypted_cb, m,
+ dec_len) != (size_t) dec_len) {
+ msg_err("HTTP parser error: %s when parsing encrypted request",
+ http_errno_description(decrypted_parser.http_errno));
+ return -1;
+}
+
+return 0;
+}
+
+static int
+rspamd_http_on_message_complete(http_parser *parser)
+{
+ struct rspamd_http_connection *conn =
+ (struct rspamd_http_connection *) parser->data;
+ struct rspamd_http_connection_private *priv;
+ int ret = 0;
+ enum rspamd_cryptobox_mode mode;
+
+ if (conn->finished) {
+ return 0;
+ }
+
+ priv = conn->priv;
+
+ if ((conn->opts & RSPAMD_HTTP_REQUIRE_ENCRYPTION) && !IS_CONN_ENCRYPTED(priv)) {
+ priv->flags |= RSPAMD_HTTP_CONN_FLAG_ENCRYPTION_NEEDED;
+ msg_err("unencrypted connection when encryption has been requested");
+ return -1;
+ }
+
+ if ((conn->opts & RSPAMD_HTTP_BODY_PARTIAL) == 0 && IS_CONN_ENCRYPTED(priv)) {
+ mode = rspamd_keypair_alg(priv->local_key);
+
+ if (priv->local_key == NULL || priv->msg->peer_key == NULL ||
+ priv->msg->body_buf.len < rspamd_cryptobox_nonce_bytes(mode) +
+ rspamd_cryptobox_mac_bytes(mode)) {
+ msg_err("cannot decrypt message");
+ return -1;
+ }
+
+ /* We have keys, so we can decrypt message */
+ ret = rspamd_http_decrypt_message(conn, priv, priv->msg->peer_key);
+
+ if (ret != 0) {
+ return ret;
+ }
+
+ if (conn->body_handler != NULL) {
+ rspamd_http_connection_ref(conn);
+ ret = conn->body_handler(conn,
+ priv->msg,
+ priv->msg->body_buf.begin,
+ priv->msg->body_buf.len);
+ rspamd_http_connection_unref(conn);
+ }
+ }
+ else if ((conn->opts & RSPAMD_HTTP_BODY_PARTIAL) == 0 && conn->body_handler) {
+ g_assert(conn->body_handler != NULL);
+ rspamd_http_connection_ref(conn);
+ ret = conn->body_handler(conn,
+ priv->msg,
+ priv->msg->body_buf.begin,
+ priv->msg->body_buf.len);
+ rspamd_http_connection_unref(conn);
+ }
+
+ if (ret == 0) {
+ rspamd_ev_watcher_stop(priv->ctx->event_loop, &priv->ev);
+ rspamd_http_connection_ref(conn);
+ ret = conn->finish_handler(conn, priv->msg);
+
+ if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) {
+ rspamd_http_context_push_keepalive(conn->priv->ctx, conn,
+ priv->msg, conn->priv->ctx->event_loop);
+ rspamd_http_connection_reset(conn);
+ }
+ else {
+ conn->finished = TRUE;
+ }
+
+ rspamd_http_connection_unref(conn);
+ }
+
+ return ret;
+}
+
+static void
+rspamd_http_simple_client_helper(struct rspamd_http_connection *conn)
+{
+ struct rspamd_http_connection_private *priv;
+ gpointer ssl;
+ gint request_method;
+ GString *prev_host = NULL;
+
+ priv = conn->priv;
+ ssl = priv->ssl;
+ priv->ssl = NULL;
+
+ /* Preserve data */
+ if (priv->msg) {
+ request_method = priv->msg->method;
+ /* Preserve host for keepalive */
+ prev_host = priv->msg->host;
+ priv->msg->host = NULL;
+ }
+
+ rspamd_http_connection_reset(conn);
+ priv->ssl = ssl;
+
+ /* Plan read message */
+
+ if (conn->opts & RSPAMD_HTTP_CLIENT_SHARED) {
+ rspamd_http_connection_read_message_shared(conn, conn->ud,
+ conn->priv->timeout);
+ }
+ else {
+ rspamd_http_connection_read_message(conn, conn->ud,
+ conn->priv->timeout);
+ }
+
+ if (priv->msg) {
+ priv->msg->method = request_method;
+ priv->msg->host = prev_host;
+ }
+ else {
+ if (prev_host) {
+ g_string_free(prev_host, TRUE);
+ }
+ }
+}
+
+static void
+rspamd_http_write_helper(struct rspamd_http_connection *conn)
+{
+ struct rspamd_http_connection_private *priv;
+ struct iovec *start;
+ guint niov, i;
+ gint flags = 0;
+ gsize remain;
+ gssize r;
+ GError *err;
+ struct iovec *cur_iov;
+ struct msghdr msg;
+
+ priv = conn->priv;
+
+ if (priv->wr_pos == priv->wr_total) {
+ goto call_finish_handler;
+ }
+
+ start = &priv->out[0];
+ niov = priv->outlen;
+ remain = priv->wr_pos;
+ /* We know that niov is small enough for that */
+ if (priv->ssl) {
+ /* Might be recursive! */
+ cur_iov = g_malloc(niov * sizeof(struct iovec));
+ }
+ else {
+ cur_iov = alloca(niov * sizeof(struct iovec));
+ }
+ memcpy(cur_iov, priv->out, niov * sizeof(struct iovec));
+ for (i = 0; i < priv->outlen && remain > 0; i++) {
+ /* Find out the first iov required */
+ start = &cur_iov[i];
+ if (start->iov_len <= remain) {
+ remain -= start->iov_len;
+ start = &cur_iov[i + 1];
+ niov--;
+ }
+ else {
+ start->iov_base = (void *) ((char *) start->iov_base + remain);
+ start->iov_len -= remain;
+ remain = 0;
+ }
+ }
+
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = start;
+ msg.msg_iovlen = MIN(IOV_MAX, niov);
+ g_assert(niov > 0);
+#ifdef MSG_NOSIGNAL
+ flags = MSG_NOSIGNAL;
+#endif
+
+ if (priv->ssl) {
+ r = rspamd_ssl_writev(priv->ssl, msg.msg_iov, msg.msg_iovlen);
+ g_free(cur_iov);
+ }
+ else {
+ r = sendmsg(conn->fd, &msg, flags);
+ }
+
+ if (r == -1) {
+ if (!priv->ssl) {
+ err = g_error_new(HTTP_ERROR, 500, "IO write error: %s", strerror(errno));
+ rspamd_http_connection_ref(conn);
+ conn->error_handler(conn, err);
+ rspamd_http_connection_unref(conn);
+ g_error_free(err);
+ }
+
+ return;
+ }
+ else {
+ priv->wr_pos += r;
+ }
+
+ if (priv->wr_pos >= priv->wr_total) {
+ goto call_finish_handler;
+ }
+ else {
+ /* Want to write more */
+ priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_RESETED;
+
+ if (priv->ssl && r > 0) {
+ /* We can write more data... */
+ rspamd_http_write_helper(conn);
+ return;
+ }
+ }
+
+ return;
+
+call_finish_handler:
+ rspamd_ev_watcher_stop(priv->ctx->event_loop, &priv->ev);
+
+ if ((conn->opts & RSPAMD_HTTP_CLIENT_SIMPLE) == 0) {
+ rspamd_http_connection_ref(conn);
+ conn->finished = TRUE;
+ conn->finish_handler(conn, priv->msg);
+ rspamd_http_connection_unref(conn);
+ }
+ else {
+ /* Plan read message */
+ rspamd_http_simple_client_helper(conn);
+ }
+}
+
+static gssize
+rspamd_http_try_read(gint fd,
+ struct rspamd_http_connection *conn,
+ struct rspamd_http_connection_private *priv,
+ struct _rspamd_http_privbuf *pbuf,
+ const gchar **buf_ptr)
+{
+ gssize r;
+ gchar *data;
+ gsize len;
+ struct rspamd_http_message *msg;
+
+ msg = priv->msg;
+
+ if (pbuf->zc_buf == NULL) {
+ data = priv->buf->data->str;
+ len = priv->buf->data->allocated;
+ }
+ else {
+ data = (gchar *) pbuf->zc_buf;
+ len = pbuf->zc_remain;
+
+ if (len == 0) {
+ rspamd_http_message_grow_body(priv->msg, priv->buf->data->allocated);
+ rspamd_http_switch_zc(pbuf, msg);
+ data = (gchar *) pbuf->zc_buf;
+ len = pbuf->zc_remain;
+ }
+ }
+
+ if (priv->ssl) {
+ r = rspamd_ssl_read(priv->ssl, data, len);
+ }
+ else {
+ r = read(fd, data, len);
+ }
+
+ if (r <= 0) {
+ return r;
+ }
+ else {
+ if (pbuf->zc_buf == NULL) {
+ priv->buf->data->len = r;
+ }
+ else {
+ pbuf->zc_remain -= r;
+ pbuf->zc_buf += r;
+ }
+ }
+
+ if (buf_ptr) {
+ *buf_ptr = data;
+ }
+
+ return r;
+}
+
+static void
+rspamd_http_ssl_err_handler(gpointer ud, GError *err)
+{
+ struct rspamd_http_connection *conn = (struct rspamd_http_connection *) ud;
+
+ rspamd_http_connection_ref(conn);
+ conn->error_handler(conn, err);
+ rspamd_http_connection_unref(conn);
+}
+
+static void
+rspamd_http_event_handler(int fd, short what, gpointer ud)
+{
+ struct rspamd_http_connection *conn = (struct rspamd_http_connection *) ud;
+ struct rspamd_http_connection_private *priv;
+ struct _rspamd_http_privbuf *pbuf;
+ const gchar *d;
+ gssize r;
+ GError *err;
+
+ priv = conn->priv;
+ pbuf = priv->buf;
+ REF_RETAIN(pbuf);
+ rspamd_http_connection_ref(conn);
+
+ if (what == EV_READ) {
+ r = rspamd_http_try_read(fd, conn, priv, pbuf, &d);
+
+ if (r > 0) {
+ if (http_parser_execute(&priv->parser, &priv->parser_cb,
+ d, r) != (size_t) r ||
+ priv->parser.http_errno != 0) {
+ if (priv->flags & RSPAMD_HTTP_CONN_FLAG_TOO_LARGE) {
+ err = g_error_new(HTTP_ERROR, 413,
+ "Request entity too large: %zu",
+ (size_t) priv->parser.content_length);
+ }
+ else if (priv->flags & RSPAMD_HTTP_CONN_FLAG_ENCRYPTION_NEEDED) {
+ err = g_error_new(HTTP_ERROR, 400,
+ "Encryption required");
+ }
+ else if (priv->parser.http_errno == HPE_CLOSED_CONNECTION) {
+ msg_err("got garbage after end of the message, ignore it");
+
+ REF_RELEASE(pbuf);
+ rspamd_http_connection_unref(conn);
+
+ return;
+ }
+ else {
+ if (priv->parser.http_errno > HPE_CB_status) {
+ err = g_error_new(HTTP_ERROR, 400,
+ "HTTP parser error: %s",
+ http_errno_description(priv->parser.http_errno));
+ }
+ else {
+ err = g_error_new(HTTP_ERROR, 500,
+ "HTTP parser internal error: %s",
+ http_errno_description(priv->parser.http_errno));
+ }
+ }
+
+ if (!conn->finished) {
+ conn->error_handler(conn, err);
+ }
+ else {
+ msg_err("got error after HTTP request is finished: %e", err);
+ }
+
+ g_error_free(err);
+
+ REF_RELEASE(pbuf);
+ rspamd_http_connection_unref(conn);
+
+ return;
+ }
+ }
+ else if (r == 0) {
+ /* We can still call http parser */
+ http_parser_execute(&priv->parser, &priv->parser_cb, d, r);
+
+ if (!conn->finished) {
+ err = g_error_new(HTTP_ERROR,
+ 400,
+ "IO read error: unexpected EOF");
+ conn->error_handler(conn, err);
+ g_error_free(err);
+ }
+ REF_RELEASE(pbuf);
+ rspamd_http_connection_unref(conn);
+
+ return;
+ }
+ else {
+ if (!priv->ssl) {
+ err = g_error_new(HTTP_ERROR,
+ 500,
+ "HTTP IO read error: %s",
+ strerror(errno));
+ conn->error_handler(conn, err);
+ g_error_free(err);
+ }
+
+ REF_RELEASE(pbuf);
+ rspamd_http_connection_unref(conn);
+
+ return;
+ }
+ }
+ else if (what == EV_TIMEOUT) {
+ if (!priv->ssl) {
+ /* Let's try to read from the socket first */
+ r = rspamd_http_try_read(fd, conn, priv, pbuf, &d);
+
+ if (r > 0) {
+ if (http_parser_execute(&priv->parser, &priv->parser_cb,
+ d, r) != (size_t) r ||
+ priv->parser.http_errno != 0) {
+ err = g_error_new(HTTP_ERROR, 400,
+ "HTTP parser error: %s",
+ http_errno_description(priv->parser.http_errno));
+
+ if (!conn->finished) {
+ conn->error_handler(conn, err);
+ }
+ else {
+ msg_err("got error after HTTP request is finished: %e", err);
+ }
+
+ g_error_free(err);
+
+ REF_RELEASE(pbuf);
+ rspamd_http_connection_unref(conn);
+
+ return;
+ }
+ }
+ else {
+ err = g_error_new(HTTP_ERROR, 408,
+ "IO timeout");
+ conn->error_handler(conn, err);
+ g_error_free(err);
+
+ REF_RELEASE(pbuf);
+ rspamd_http_connection_unref(conn);
+
+ return;
+ }
+ }
+ else {
+ /* In case of SSL we disable this logic as we already came from SSL handler */
+ REF_RELEASE(pbuf);
+ rspamd_http_connection_unref(conn);
+
+ return;
+ }
+ }
+ else if (what == EV_WRITE) {
+ rspamd_http_write_helper(conn);
+ }
+
+ REF_RELEASE(pbuf);
+ rspamd_http_connection_unref(conn);
+}
+
+static void
+rspamd_http_parser_reset(struct rspamd_http_connection *conn)
+{
+ struct rspamd_http_connection_private *priv = conn->priv;
+
+ http_parser_init(&priv->parser,
+ conn->type == RSPAMD_HTTP_SERVER ? HTTP_REQUEST : HTTP_RESPONSE);
+
+ priv->parser_cb.on_url = rspamd_http_on_url;
+ priv->parser_cb.on_status = rspamd_http_on_status;
+ priv->parser_cb.on_header_field = rspamd_http_on_header_field;
+ priv->parser_cb.on_header_value = rspamd_http_on_header_value;
+ priv->parser_cb.on_headers_complete = rspamd_http_on_headers_complete;
+ priv->parser_cb.on_body = rspamd_http_on_body;
+ priv->parser_cb.on_message_complete = rspamd_http_on_message_complete;
+}
+
+static struct rspamd_http_connection *
+rspamd_http_connection_new_common(struct rspamd_http_context *ctx,
+ gint fd,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ unsigned opts,
+ enum rspamd_http_connection_type type,
+ enum rspamd_http_priv_flags priv_flags,
+ struct upstream *proxy_upstream)
+{
+ struct rspamd_http_connection *conn;
+ struct rspamd_http_connection_private *priv;
+
+ g_assert(error_handler != NULL && finish_handler != NULL);
+
+ if (ctx == NULL) {
+ ctx = rspamd_http_context_default();
+ }
+
+ conn = g_malloc0(sizeof(struct rspamd_http_connection));
+ conn->opts = opts;
+ conn->type = type;
+ conn->body_handler = body_handler;
+ conn->error_handler = error_handler;
+ conn->finish_handler = finish_handler;
+ conn->fd = fd;
+ conn->ref = 1;
+ conn->finished = FALSE;
+
+ /* Init priv */
+ priv = g_malloc0(sizeof(struct rspamd_http_connection_private));
+ conn->priv = priv;
+ priv->ctx = ctx;
+ priv->flags = priv_flags;
+
+ if (type == RSPAMD_HTTP_SERVER) {
+ priv->cache = ctx->server_kp_cache;
+ }
+ else {
+ priv->cache = ctx->client_kp_cache;
+ if (ctx->client_kp) {
+ priv->local_key = rspamd_keypair_ref(ctx->client_kp);
+ }
+ }
+
+ rspamd_http_parser_reset(conn);
+ priv->parser.data = conn;
+
+ return conn;
+}
+
+struct rspamd_http_connection *
+rspamd_http_connection_new_server(struct rspamd_http_context *ctx,
+ gint fd,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ unsigned opts)
+{
+ return rspamd_http_connection_new_common(ctx, fd, body_handler,
+ error_handler, finish_handler, opts, RSPAMD_HTTP_SERVER, 0, NULL);
+}
+
+struct rspamd_http_connection *
+rspamd_http_connection_new_client_socket(struct rspamd_http_context *ctx,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ unsigned opts,
+ gint fd)
+{
+ return rspamd_http_connection_new_common(ctx, fd, body_handler,
+ error_handler, finish_handler, opts, RSPAMD_HTTP_CLIENT, 0, NULL);
+}
+
+struct rspamd_http_connection *
+rspamd_http_connection_new_client(struct rspamd_http_context *ctx,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ unsigned opts,
+ rspamd_inet_addr_t *addr)
+{
+ gint fd;
+
+ if (ctx == NULL) {
+ ctx = rspamd_http_context_default();
+ }
+
+ if (ctx->http_proxies) {
+ struct upstream *up = rspamd_upstream_get(ctx->http_proxies,
+ RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0);
+
+ if (up) {
+ rspamd_inet_addr_t *proxy_addr = rspamd_upstream_addr_next(up);
+
+ fd = rspamd_inet_address_connect(proxy_addr, SOCK_STREAM, TRUE);
+
+ if (fd == -1) {
+ msg_info("cannot connect to http proxy %s: %s",
+ rspamd_inet_address_to_string_pretty(proxy_addr),
+ strerror(errno));
+ rspamd_upstream_fail(up, TRUE, strerror(errno));
+
+ return NULL;
+ }
+
+ return rspamd_http_connection_new_common(ctx, fd, body_handler,
+ error_handler, finish_handler, opts,
+ RSPAMD_HTTP_CLIENT,
+ RSPAMD_HTTP_CONN_OWN_SOCKET | RSPAMD_HTTP_CONN_FLAG_PROXY,
+ up);
+ }
+ }
+
+ /* Unproxied version */
+ fd = rspamd_inet_address_connect(addr, SOCK_STREAM, TRUE);
+
+ if (fd == -1) {
+ msg_info("cannot connect make http connection to %s: %s",
+ rspamd_inet_address_to_string_pretty(addr),
+ strerror(errno));
+
+ return NULL;
+ }
+
+ return rspamd_http_connection_new_common(ctx, fd, body_handler,
+ error_handler, finish_handler, opts,
+ RSPAMD_HTTP_CLIENT,
+ RSPAMD_HTTP_CONN_OWN_SOCKET,
+ NULL);
+}
+
+struct rspamd_http_connection *
+rspamd_http_connection_new_client_keepalive(struct rspamd_http_context *ctx,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ unsigned opts,
+ rspamd_inet_addr_t *addr,
+ const gchar *host)
+{
+ struct rspamd_http_connection *conn;
+
+ if (ctx == NULL) {
+ ctx = rspamd_http_context_default();
+ }
+
+ conn = rspamd_http_context_check_keepalive(ctx, addr, host,
+ opts & RSPAMD_HTTP_CLIENT_SSL);
+
+ if (conn) {
+ return conn;
+ }
+
+ conn = rspamd_http_connection_new_client(ctx,
+ body_handler, error_handler, finish_handler,
+ opts | RSPAMD_HTTP_CLIENT_SIMPLE | RSPAMD_HTTP_CLIENT_KEEP_ALIVE,
+ addr);
+
+ if (conn) {
+ rspamd_http_context_prepare_keepalive(ctx, conn, addr, host,
+ opts & RSPAMD_HTTP_CLIENT_SSL);
+ }
+
+ return conn;
+}
+
+void rspamd_http_connection_reset(struct rspamd_http_connection *conn)
+{
+ struct rspamd_http_connection_private *priv;
+ struct rspamd_http_message *msg;
+
+ priv = conn->priv;
+ msg = priv->msg;
+
+ /* Clear request */
+ if (msg != NULL) {
+ if (msg->peer_key) {
+ priv->peer_key = msg->peer_key;
+ msg->peer_key = NULL;
+ }
+ rspamd_http_message_unref(msg);
+ priv->msg = NULL;
+ }
+
+ conn->finished = FALSE;
+ /* Clear priv */
+ rspamd_ev_watcher_stop(priv->ctx->event_loop, &priv->ev);
+
+ if (!(priv->flags & RSPAMD_HTTP_CONN_FLAG_RESETED)) {
+ rspamd_http_parser_reset(conn);
+ }
+
+ if (priv->buf != NULL) {
+ REF_RELEASE(priv->buf);
+ priv->buf = NULL;
+ }
+
+ if (priv->out != NULL) {
+ g_free(priv->out);
+ priv->out = NULL;
+ }
+
+ priv->flags |= RSPAMD_HTTP_CONN_FLAG_RESETED;
+}
+
+struct rspamd_http_message *
+rspamd_http_connection_steal_msg(struct rspamd_http_connection *conn)
+{
+ struct rspamd_http_connection_private *priv;
+ struct rspamd_http_message *msg;
+
+ priv = conn->priv;
+ msg = priv->msg;
+
+ /* Clear request */
+ if (msg != NULL) {
+ if (msg->peer_key) {
+ priv->peer_key = msg->peer_key;
+ msg->peer_key = NULL;
+ }
+ priv->msg = NULL;
+ }
+
+ return msg;
+}
+
+struct rspamd_http_message *
+rspamd_http_connection_copy_msg(struct rspamd_http_message *msg, GError **err)
+{
+ struct rspamd_http_message *new_msg;
+ struct rspamd_http_header *hdr, *nhdr, *nhdrs, *hcur;
+ const gchar *old_body;
+ gsize old_len;
+ struct stat st;
+ union _rspamd_storage_u *storage;
+
+ new_msg = rspamd_http_new_message(msg->type);
+ new_msg->flags = msg->flags;
+
+ if (msg->body_buf.len > 0) {
+
+ if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
+ /* Avoid copying by just mapping a shared segment */
+ new_msg->flags |= RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE;
+
+ storage = &new_msg->body_buf.c;
+ storage->shared.shm_fd = dup(msg->body_buf.c.shared.shm_fd);
+
+ if (storage->shared.shm_fd == -1) {
+ rspamd_http_message_unref(new_msg);
+ g_set_error(err, http_error_quark(), errno,
+ "cannot dup shmem fd: %d: %s",
+ msg->body_buf.c.shared.shm_fd, strerror(errno));
+
+ return NULL;
+ }
+
+ if (fstat(storage->shared.shm_fd, &st) == -1) {
+ g_set_error(err, http_error_quark(), errno,
+ "cannot stat shmem fd: %d: %s",
+ storage->shared.shm_fd, strerror(errno));
+ rspamd_http_message_unref(new_msg);
+
+ return NULL;
+ }
+
+ /* We don't own segment, so do not try to touch it */
+
+ if (msg->body_buf.c.shared.name) {
+ storage->shared.name = msg->body_buf.c.shared.name;
+ REF_RETAIN(storage->shared.name);
+ }
+
+ new_msg->body_buf.str = mmap(NULL, st.st_size,
+ PROT_READ, MAP_SHARED,
+ storage->shared.shm_fd, 0);
+
+ if (new_msg->body_buf.str == MAP_FAILED) {
+ g_set_error(err, http_error_quark(), errno,
+ "cannot mmap shmem fd: %d: %s",
+ storage->shared.shm_fd, strerror(errno));
+ rspamd_http_message_unref(new_msg);
+
+ return NULL;
+ }
+
+ new_msg->body_buf.begin = new_msg->body_buf.str;
+ new_msg->body_buf.len = msg->body_buf.len;
+ new_msg->body_buf.begin = new_msg->body_buf.str +
+ (msg->body_buf.begin - msg->body_buf.str);
+ }
+ else {
+ old_body = rspamd_http_message_get_body(msg, &old_len);
+
+ if (!rspamd_http_message_set_body(new_msg, old_body, old_len)) {
+ g_set_error(err, http_error_quark(), errno,
+ "cannot set body for message, length: %zd",
+ old_len);
+ rspamd_http_message_unref(new_msg);
+
+ return NULL;
+ }
+ }
+ }
+
+ if (msg->url) {
+ if (new_msg->url) {
+ new_msg->url = rspamd_fstring_append(new_msg->url, msg->url->str,
+ msg->url->len);
+ }
+ else {
+ new_msg->url = rspamd_fstring_new_init(msg->url->str,
+ msg->url->len);
+ }
+ }
+
+ if (msg->host) {
+ new_msg->host = g_string_new_len(msg->host->str, msg->host->len);
+ }
+
+ new_msg->method = msg->method;
+ new_msg->port = msg->port;
+ new_msg->date = msg->date;
+ new_msg->last_modified = msg->last_modified;
+
+ kh_foreach_value(msg->headers, hdr, {
+ nhdrs = NULL;
+
+ DL_FOREACH(hdr, hcur)
+ {
+ nhdr = g_malloc(sizeof(struct rspamd_http_header));
+
+ nhdr->combined = rspamd_fstring_new_init(hcur->combined->str,
+ hcur->combined->len);
+ nhdr->name.begin = nhdr->combined->str +
+ (hcur->name.begin - hcur->combined->str);
+ nhdr->name.len = hcur->name.len;
+ nhdr->value.begin = nhdr->combined->str +
+ (hcur->value.begin - hcur->combined->str);
+ nhdr->value.len = hcur->value.len;
+ DL_APPEND(nhdrs, nhdr);
+ }
+
+ gint r;
+ khiter_t k = kh_put(rspamd_http_headers_hash, new_msg->headers,
+ &nhdrs->name, &r);
+
+ if (r != 0) {
+ kh_value(new_msg->headers, k) = nhdrs;
+ }
+ else {
+ DL_CONCAT(kh_value(new_msg->headers, k), nhdrs);
+ }
+ });
+
+ return new_msg;
+}
+
+void rspamd_http_connection_free(struct rspamd_http_connection *conn)
+{
+ struct rspamd_http_connection_private *priv;
+
+ priv = conn->priv;
+
+ if (priv != NULL) {
+ rspamd_http_connection_reset(conn);
+
+ if (priv->ssl) {
+ rspamd_ssl_connection_free(priv->ssl);
+ priv->ssl = NULL;
+ }
+
+ if (priv->local_key) {
+ rspamd_keypair_unref(priv->local_key);
+ }
+ if (priv->peer_key) {
+ rspamd_pubkey_unref(priv->peer_key);
+ }
+
+ if (priv->flags & RSPAMD_HTTP_CONN_OWN_SOCKET) {
+ /* Fd is owned by a connection */
+ close(conn->fd);
+ }
+
+ g_free(priv);
+ }
+
+ g_free(conn);
+}
+
+static void
+rspamd_http_connection_read_message_common(struct rspamd_http_connection *conn,
+ gpointer ud, ev_tstamp timeout,
+ gint flags)
+{
+ struct rspamd_http_connection_private *priv = conn->priv;
+ struct rspamd_http_message *req;
+
+ conn->ud = ud;
+ req = rspamd_http_new_message(
+ conn->type == RSPAMD_HTTP_SERVER ? HTTP_REQUEST : HTTP_RESPONSE);
+ priv->msg = req;
+ req->flags = flags;
+
+ if (flags & RSPAMD_HTTP_FLAG_SHMEM) {
+ req->body_buf.c.shared.shm_fd = -1;
+ }
+
+ if (priv->peer_key) {
+ priv->msg->peer_key = priv->peer_key;
+ priv->peer_key = NULL;
+ priv->flags |= RSPAMD_HTTP_CONN_FLAG_ENCRYPTED;
+ }
+
+ priv->timeout = timeout;
+ priv->header = NULL;
+ priv->buf = g_malloc0(sizeof(*priv->buf));
+ REF_INIT_RETAIN(priv->buf, rspamd_http_privbuf_dtor);
+ priv->buf->data = rspamd_fstring_sized_new(8192);
+ priv->flags |= RSPAMD_HTTP_CONN_FLAG_NEW_HEADER;
+
+ if (!priv->ssl) {
+ rspamd_ev_watcher_init(&priv->ev, conn->fd, EV_READ,
+ rspamd_http_event_handler, conn);
+ rspamd_ev_watcher_start(priv->ctx->event_loop, &priv->ev, priv->timeout);
+ }
+ else {
+ rspamd_ssl_connection_restore_handlers(priv->ssl,
+ rspamd_http_event_handler,
+ rspamd_http_ssl_err_handler,
+ conn,
+ EV_READ);
+ }
+
+ priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_RESETED;
+}
+
+void rspamd_http_connection_read_message(struct rspamd_http_connection *conn,
+ gpointer ud, ev_tstamp timeout)
+{
+ rspamd_http_connection_read_message_common(conn, ud, timeout, 0);
+}
+
+void rspamd_http_connection_read_message_shared(struct rspamd_http_connection *conn,
+ gpointer ud, ev_tstamp timeout)
+{
+ rspamd_http_connection_read_message_common(conn, ud, timeout,
+ RSPAMD_HTTP_FLAG_SHMEM);
+}
+
+static void
+rspamd_http_connection_encrypt_message(
+ struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ struct rspamd_http_connection_private *priv,
+ guchar *pbody,
+ guint bodylen,
+ guchar *pmethod,
+ guint methodlen,
+ guint preludelen,
+ gint hdrcount,
+ guchar *np,
+ guchar *mp,
+ struct rspamd_cryptobox_pubkey *peer_key)
+{
+ struct rspamd_cryptobox_segment *segments;
+ guchar *crlfp;
+ const guchar *nm;
+ gint i, cnt;
+ guint outlen;
+ struct rspamd_http_header *hdr, *hcur;
+ enum rspamd_cryptobox_mode mode;
+
+ mode = rspamd_keypair_alg(priv->local_key);
+ crlfp = mp + rspamd_cryptobox_mac_bytes(mode);
+
+ outlen = priv->out[0].iov_len + priv->out[1].iov_len;
+ /*
+ * Create segments from the following:
+ * Method, [URL], CRLF, nheaders, CRLF, body
+ */
+ segments = g_new(struct rspamd_cryptobox_segment, hdrcount + 5);
+
+ segments[0].data = pmethod;
+ segments[0].len = methodlen;
+
+ if (conn->type != RSPAMD_HTTP_SERVER) {
+ segments[1].data = msg->url->str;
+ segments[1].len = msg->url->len;
+ /* space + HTTP version + crlf */
+ segments[2].data = crlfp;
+ segments[2].len = preludelen - 2;
+ crlfp += segments[2].len;
+ i = 3;
+ }
+ else {
+ /* Here we send just CRLF */
+ segments[1].data = crlfp;
+ segments[1].len = 2;
+ crlfp += segments[1].len;
+
+ i = 2;
+ }
+
+
+ kh_foreach_value (msg->headers, hdr, {
+ DL_FOREACH (hdr, hcur) {
+ segments[i].data = hcur->combined->str;
+ segments[i++].len = hcur->combined->len;
+}
+});
+
+/* crlfp should point now at the second crlf */
+segments[i].data = crlfp;
+segments[i++].len = 2;
+
+if (pbody) {
+ segments[i].data = pbody;
+ segments[i++].len = bodylen;
+}
+
+cnt = i;
+
+if ((nm = rspamd_pubkey_get_nm(peer_key, priv->local_key)) == NULL) {
+ nm = rspamd_pubkey_calculate_nm(peer_key, priv->local_key);
+}
+
+rspamd_cryptobox_encryptv_nm_inplace(segments, cnt, np, nm, mp, mode);
+
+/*
+ * iov[0] = base HTTP request
+ * iov[1] = CRLF
+ * iov[2] = nonce
+ * iov[3] = mac
+ * iov[4..i] = encrypted HTTP request/reply
+ */
+priv->out[2].iov_base = np;
+priv->out[2].iov_len = rspamd_cryptobox_nonce_bytes(mode);
+priv->out[3].iov_base = mp;
+priv->out[3].iov_len = rspamd_cryptobox_mac_bytes(mode);
+
+outlen += rspamd_cryptobox_nonce_bytes(mode) +
+ rspamd_cryptobox_mac_bytes(mode);
+
+for (i = 0; i < cnt; i++) {
+ priv->out[i + 4].iov_base = segments[i].data;
+ priv->out[i + 4].iov_len = segments[i].len;
+ outlen += segments[i].len;
+}
+
+priv->wr_total = outlen;
+
+g_free(segments);
+}
+
+static void
+rspamd_http_detach_shared(struct rspamd_http_message *msg)
+{
+ rspamd_fstring_t *cpy_str;
+
+ cpy_str = rspamd_fstring_new_init(msg->body_buf.begin, msg->body_buf.len);
+ rspamd_http_message_set_body_from_fstring_steal(msg, cpy_str);
+}
+
+gint rspamd_http_message_write_header(const gchar *mime_type, gboolean encrypted,
+ gchar *repbuf, gsize replen, gsize bodylen, gsize enclen, const gchar *host,
+ struct rspamd_http_connection *conn, struct rspamd_http_message *msg,
+ rspamd_fstring_t **buf,
+ struct rspamd_http_connection_private *priv,
+ struct rspamd_cryptobox_pubkey *peer_key)
+{
+ gchar datebuf[64];
+ gint meth_len = 0;
+ const gchar *conn_type = "close";
+
+ if (conn->type == RSPAMD_HTTP_SERVER) {
+ /* Format reply */
+ if (msg->method < HTTP_SYMBOLS) {
+ rspamd_ftok_t status;
+
+ rspamd_http_date_format(datebuf, sizeof(datebuf), msg->date);
+
+ if (mime_type == NULL) {
+ mime_type =
+ encrypted ? "application/octet-stream" : "text/plain";
+ }
+
+ if (msg->status == NULL || msg->status->len == 0) {
+ if (msg->code == 200) {
+ RSPAMD_FTOK_ASSIGN(&status, "OK");
+ }
+ else if (msg->code == 404) {
+ RSPAMD_FTOK_ASSIGN(&status, "Not Found");
+ }
+ else if (msg->code == 403) {
+ RSPAMD_FTOK_ASSIGN(&status, "Forbidden");
+ }
+ else if (msg->code >= 500 && msg->code < 600) {
+ RSPAMD_FTOK_ASSIGN(&status, "Internal Server Error");
+ }
+ else {
+ RSPAMD_FTOK_ASSIGN(&status, "Undefined Error");
+ }
+ }
+ else {
+ status.begin = msg->status->str;
+ status.len = msg->status->len;
+ }
+
+ if (encrypted) {
+ /* Internal reply (encrypted) */
+ if (mime_type) {
+ meth_len =
+ rspamd_snprintf(repbuf, replen,
+ "HTTP/1.1 %d %T\r\n"
+ "Connection: close\r\n"
+ "Server: %s\r\n"
+ "Date: %s\r\n"
+ "Content-Length: %z\r\n"
+ "Content-Type: %s", /* NO \r\n at the end ! */
+ msg->code, &status, priv->ctx->config.server_hdr,
+ datebuf,
+ bodylen, mime_type);
+ }
+ else {
+ meth_len =
+ rspamd_snprintf(repbuf, replen,
+ "HTTP/1.1 %d %T\r\n"
+ "Connection: close\r\n"
+ "Server: %s\r\n"
+ "Date: %s\r\n"
+ "Content-Length: %z", /* NO \r\n at the end ! */
+ msg->code, &status, priv->ctx->config.server_hdr,
+ datebuf,
+ bodylen);
+ }
+ enclen += meth_len;
+ /* External reply */
+ rspamd_printf_fstring(buf,
+ "HTTP/1.1 200 OK\r\n"
+ "Connection: close\r\n"
+ "Server: %s\r\n"
+ "Date: %s\r\n"
+ "Content-Length: %z\r\n"
+ "Content-Type: application/octet-stream\r\n",
+ priv->ctx->config.server_hdr,
+ datebuf, enclen);
+ }
+ else {
+ if (mime_type) {
+ meth_len =
+ rspamd_printf_fstring(buf,
+ "HTTP/1.1 %d %T\r\n"
+ "Connection: close\r\n"
+ "Server: %s\r\n"
+ "Date: %s\r\n"
+ "Content-Length: %z\r\n"
+ "Content-Type: %s\r\n",
+ msg->code, &status, priv->ctx->config.server_hdr,
+ datebuf,
+ bodylen, mime_type);
+ }
+ else {
+ meth_len =
+ rspamd_printf_fstring(buf,
+ "HTTP/1.1 %d %T\r\n"
+ "Connection: close\r\n"
+ "Server: %s\r\n"
+ "Date: %s\r\n"
+ "Content-Length: %z\r\n",
+ msg->code, &status, priv->ctx->config.server_hdr,
+ datebuf,
+ bodylen);
+ }
+ }
+ }
+ else {
+ /* Legacy spamd reply */
+ if (msg->flags & RSPAMD_HTTP_FLAG_SPAMC) {
+ gsize real_bodylen;
+ goffset eoh_pos;
+ GString tmp;
+
+ /* Unfortunately, spamc protocol is deadly brain damaged */
+ tmp.str = (gchar *) msg->body_buf.begin;
+ tmp.len = msg->body_buf.len;
+
+ if (rspamd_string_find_eoh(&tmp, &eoh_pos) != -1 &&
+ bodylen > eoh_pos) {
+ real_bodylen = bodylen - eoh_pos;
+ }
+ else {
+ real_bodylen = bodylen;
+ }
+
+ rspamd_printf_fstring(buf, "SPAMD/1.1 0 EX_OK\r\n"
+ "Content-length: %z\r\n",
+ real_bodylen);
+ }
+ else {
+ rspamd_printf_fstring(buf, "RSPAMD/1.3 0 EX_OK\r\n");
+ }
+ }
+ }
+ else {
+
+ /* Client request */
+ if (conn->opts & RSPAMD_HTTP_CLIENT_KEEP_ALIVE) {
+ conn_type = "keep-alive";
+ }
+
+ /* Format request */
+ enclen += RSPAMD_FSTRING_LEN(msg->url) +
+ strlen(http_method_str(msg->method)) + 1;
+
+ if (host == NULL && msg->host == NULL) {
+ /* Fallback to HTTP/1.0 */
+ if (encrypted) {
+ rspamd_printf_fstring(buf,
+ "%s %s HTTP/1.0\r\n"
+ "Content-Length: %z\r\n"
+ "Content-Type: application/octet-stream\r\n"
+ "Connection: %s\r\n",
+ "POST",
+ "/post",
+ enclen,
+ conn_type);
+ }
+ else {
+ rspamd_printf_fstring(buf,
+ "%s %V HTTP/1.0\r\n"
+ "Content-Length: %z\r\n"
+ "Connection: %s\r\n",
+ http_method_str(msg->method),
+ msg->url,
+ bodylen,
+ conn_type);
+
+ if (bodylen > 0) {
+ if (mime_type == NULL) {
+ mime_type = "text/plain";
+ }
+
+ rspamd_printf_fstring(buf,
+ "Content-Type: %s\r\n",
+ mime_type);
+ }
+ }
+ }
+ else {
+ /* Normal HTTP/1.1 with Host */
+ if (host == NULL) {
+ host = msg->host->str;
+ }
+
+ if (encrypted) {
+ /* TODO: Add proxy support to HTTPCrypt */
+ if (rspamd_http_message_is_standard_port(msg)) {
+ rspamd_printf_fstring(buf,
+ "%s %s HTTP/1.1\r\n"
+ "Connection: %s\r\n"
+ "Host: %s\r\n"
+ "Content-Length: %z\r\n"
+ "Content-Type: application/octet-stream\r\n",
+ "POST",
+ "/post",
+ conn_type,
+ host,
+ enclen);
+ }
+ else {
+ rspamd_printf_fstring(buf,
+ "%s %s HTTP/1.1\r\n"
+ "Connection: %s\r\n"
+ "Host: %s:%d\r\n"
+ "Content-Length: %z\r\n"
+ "Content-Type: application/octet-stream\r\n",
+ "POST",
+ "/post",
+ conn_type,
+ host,
+ msg->port,
+ enclen);
+ }
+ }
+ else {
+ if (conn->priv->flags & RSPAMD_HTTP_CONN_FLAG_PROXY) {
+ /* Write proxied request */
+ if ((msg->flags & RSPAMD_HTTP_FLAG_HAS_HOST_HEADER)) {
+ rspamd_printf_fstring(buf,
+ "%s %s://%s:%d/%V HTTP/1.1\r\n"
+ "Connection: %s\r\n"
+ "Content-Length: %z\r\n",
+ http_method_str(msg->method),
+ (conn->opts & RSPAMD_HTTP_CLIENT_SSL) ? "https" : "http",
+ host,
+ msg->port,
+ msg->url,
+ conn_type,
+ bodylen);
+ }
+ else {
+ if (rspamd_http_message_is_standard_port(msg)) {
+ rspamd_printf_fstring(buf,
+ "%s %s://%s:%d/%V HTTP/1.1\r\n"
+ "Connection: %s\r\n"
+ "Host: %s\r\n"
+ "Content-Length: %z\r\n",
+ http_method_str(msg->method),
+ (conn->opts & RSPAMD_HTTP_CLIENT_SSL) ? "https" : "http",
+ host,
+ msg->port,
+ msg->url,
+ conn_type,
+ host,
+ bodylen);
+ }
+ else {
+ rspamd_printf_fstring(buf,
+ "%s %s://%s:%d/%V HTTP/1.1\r\n"
+ "Connection: %s\r\n"
+ "Host: %s:%d\r\n"
+ "Content-Length: %z\r\n",
+ http_method_str(msg->method),
+ (conn->opts & RSPAMD_HTTP_CLIENT_SSL) ? "https" : "http",
+ host,
+ msg->port,
+ msg->url,
+ conn_type,
+ host,
+ msg->port,
+ bodylen);
+ }
+ }
+ }
+ else {
+ /* Unproxied version */
+ if ((msg->flags & RSPAMD_HTTP_FLAG_HAS_HOST_HEADER)) {
+ rspamd_printf_fstring(buf,
+ "%s %V HTTP/1.1\r\n"
+ "Connection: %s\r\n"
+ "Content-Length: %z\r\n",
+ http_method_str(msg->method),
+ msg->url,
+ conn_type,
+ bodylen);
+ }
+ else {
+ if (rspamd_http_message_is_standard_port(msg)) {
+ rspamd_printf_fstring(buf,
+ "%s %V HTTP/1.1\r\n"
+ "Connection: %s\r\n"
+ "Host: %s\r\n"
+ "Content-Length: %z\r\n",
+ http_method_str(msg->method),
+ msg->url,
+ conn_type,
+ host,
+ bodylen);
+ }
+ else {
+ rspamd_printf_fstring(buf,
+ "%s %V HTTP/1.1\r\n"
+ "Connection: %s\r\n"
+ "Host: %s:%d\r\n"
+ "Content-Length: %z\r\n",
+ http_method_str(msg->method),
+ msg->url,
+ conn_type,
+ host,
+ msg->port,
+ bodylen);
+ }
+ }
+ }
+
+ if (bodylen > 0) {
+ if (mime_type != NULL) {
+ rspamd_printf_fstring(buf,
+ "Content-Type: %s\r\n",
+ mime_type);
+ }
+ }
+ }
+ }
+
+ if (encrypted) {
+ GString *b32_key, *b32_id;
+
+ b32_key = rspamd_keypair_print(priv->local_key,
+ RSPAMD_KEYPAIR_PUBKEY | RSPAMD_KEYPAIR_BASE32);
+ b32_id = rspamd_pubkey_print(peer_key,
+ RSPAMD_KEYPAIR_ID_SHORT | RSPAMD_KEYPAIR_BASE32);
+ /* XXX: add some fuzz here */
+ rspamd_printf_fstring(&*buf, "Key: %v=%v\r\n", b32_id, b32_key);
+ g_string_free(b32_key, TRUE);
+ g_string_free(b32_id, TRUE);
+ }
+ }
+
+ return meth_len;
+}
+
+static gboolean
+rspamd_http_connection_write_message_common(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *host,
+ const gchar *mime_type,
+ gpointer ud,
+ ev_tstamp timeout,
+ gboolean allow_shared)
+{
+ struct rspamd_http_connection_private *priv = conn->priv;
+ struct rspamd_http_header *hdr, *hcur;
+ gchar repbuf[512], *pbody;
+ gint i, hdrcount, meth_len = 0, preludelen = 0;
+ gsize bodylen, enclen = 0;
+ rspamd_fstring_t *buf;
+ gboolean encrypted = FALSE;
+ guchar nonce[rspamd_cryptobox_MAX_NONCEBYTES], mac[rspamd_cryptobox_MAX_MACBYTES];
+ guchar *np = NULL, *mp = NULL, *meth_pos = NULL;
+ struct rspamd_cryptobox_pubkey *peer_key = NULL;
+ enum rspamd_cryptobox_mode mode;
+ GError *err;
+
+ conn->ud = ud;
+ priv->msg = msg;
+ priv->timeout = timeout;
+
+ priv->header = NULL;
+ priv->buf = g_malloc0(sizeof(*priv->buf));
+ REF_INIT_RETAIN(priv->buf, rspamd_http_privbuf_dtor);
+ priv->buf->data = rspamd_fstring_sized_new(512);
+ buf = priv->buf->data;
+
+ if ((msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL) && !(conn->opts & RSPAMD_HTTP_CLIENT_SSL)) {
+ err = g_error_new(HTTP_ERROR, 400,
+ "SSL connection requested but not created properly, internal error");
+ rspamd_http_connection_ref(conn);
+ conn->error_handler(conn, err);
+ rspamd_http_connection_unref(conn);
+ g_error_free(err);
+ return FALSE;
+ }
+
+ if (priv->peer_key && priv->local_key) {
+ priv->msg->peer_key = priv->peer_key;
+ priv->peer_key = NULL;
+ priv->flags |= RSPAMD_HTTP_CONN_FLAG_ENCRYPTED;
+ }
+
+ if (msg->peer_key != NULL) {
+ if (priv->local_key == NULL) {
+ /* Automatically generate a temporary keypair */
+ priv->local_key = rspamd_keypair_new(RSPAMD_KEYPAIR_KEX,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+ }
+
+ encrypted = TRUE;
+
+ if (priv->cache) {
+ rspamd_keypair_cache_process(priv->cache,
+ priv->local_key, priv->msg->peer_key);
+ }
+ }
+
+ if (encrypted && (msg->flags &
+ (RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE | RSPAMD_HTTP_FLAG_SHMEM))) {
+ /* We cannot use immutable body to encrypt message in place */
+ allow_shared = FALSE;
+ rspamd_http_detach_shared(msg);
+ }
+
+ if (allow_shared) {
+ gchar tmpbuf[64];
+
+ if (!(msg->flags & RSPAMD_HTTP_FLAG_SHMEM) ||
+ msg->body_buf.c.shared.name == NULL) {
+ allow_shared = FALSE;
+ }
+ else {
+ /* Insert new headers */
+ rspamd_http_message_add_header(msg, "Shm",
+ msg->body_buf.c.shared.name->shm_name);
+ rspamd_snprintf(tmpbuf, sizeof(tmpbuf), "%d",
+ (int) (msg->body_buf.begin - msg->body_buf.str));
+ rspamd_http_message_add_header(msg, "Shm-Offset",
+ tmpbuf);
+ rspamd_snprintf(tmpbuf, sizeof(tmpbuf), "%z",
+ msg->body_buf.len);
+ rspamd_http_message_add_header(msg, "Shm-Length",
+ tmpbuf);
+ }
+ }
+
+ if (priv->ctx->config.user_agent && conn->type == RSPAMD_HTTP_CLIENT) {
+ rspamd_ftok_t srch;
+ khiter_t k;
+ gint r;
+
+ RSPAMD_FTOK_ASSIGN(&srch, "User-Agent");
+
+ k = kh_put(rspamd_http_headers_hash, msg->headers, &srch, &r);
+
+ if (r != 0) {
+ hdr = g_malloc0(sizeof(struct rspamd_http_header));
+ guint vlen = strlen(priv->ctx->config.user_agent);
+ hdr->combined = rspamd_fstring_sized_new(srch.len + vlen + 4);
+ rspamd_printf_fstring(&hdr->combined, "%T: %*s\r\n", &srch, vlen,
+ priv->ctx->config.user_agent);
+ hdr->name.begin = hdr->combined->str;
+ hdr->name.len = srch.len;
+ hdr->value.begin = hdr->combined->str + srch.len + 2;
+ hdr->value.len = vlen;
+ hdr->prev = hdr; /* for utlists */
+
+ kh_value(msg->headers, k) = hdr;
+ /* as we searched using static buffer */
+ kh_key(msg->headers, k) = &hdr->name;
+ }
+ }
+
+ if (encrypted) {
+ mode = rspamd_keypair_alg(priv->local_key);
+
+ if (msg->body_buf.len == 0) {
+ pbody = NULL;
+ bodylen = 0;
+ msg->method = HTTP_GET;
+ }
+ else {
+ pbody = (gchar *) msg->body_buf.begin;
+ bodylen = msg->body_buf.len;
+ msg->method = HTTP_POST;
+ }
+
+ if (conn->type == RSPAMD_HTTP_SERVER) {
+ /*
+ * iov[0] = base reply
+ * iov[1] = CRLF
+ * iov[2] = nonce
+ * iov[3] = mac
+ * iov[4] = encrypted reply
+ * iov[6] = encrypted crlf
+ * iov[7..n] = encrypted headers
+ * iov[n + 1] = encrypted crlf
+ * [iov[n + 2] = encrypted body]
+ */
+ priv->outlen = 7;
+ enclen = rspamd_cryptobox_nonce_bytes(mode) +
+ rspamd_cryptobox_mac_bytes(mode) +
+ 4 + /* 2 * CRLF */
+ bodylen;
+ }
+ else {
+ /*
+ * iov[0] = base request
+ * iov[1] = CRLF
+ * iov[2] = nonce
+ * iov[3] = mac
+ * iov[4] = encrypted method + space
+ * iov[5] = encrypted url
+ * iov[7] = encrypted prelude
+ * iov[8..n] = encrypted headers
+ * iov[n + 1] = encrypted crlf
+ * [iov[n + 2] = encrypted body]
+ */
+ priv->outlen = 8;
+
+ if (bodylen > 0) {
+ if (mime_type != NULL) {
+ preludelen = rspamd_snprintf(repbuf, sizeof(repbuf), "%s\r\n"
+ "Content-Length: %z\r\n"
+ "Content-Type: %s\r\n"
+ "\r\n",
+ ENCRYPTED_VERSION, bodylen,
+ mime_type);
+ }
+ else {
+ preludelen = rspamd_snprintf(repbuf, sizeof(repbuf), "%s\r\n"
+ "Content-Length: %z\r\n"
+ ""
+ "\r\n",
+ ENCRYPTED_VERSION, bodylen);
+ }
+ }
+ else {
+ preludelen = rspamd_snprintf(repbuf, sizeof(repbuf),
+ "%s\r\n\r\n",
+ ENCRYPTED_VERSION);
+ }
+
+ enclen = rspamd_cryptobox_nonce_bytes(mode) +
+ rspamd_cryptobox_mac_bytes(mode) +
+ preludelen + /* version [content-length] + 2 * CRLF */
+ bodylen;
+ }
+
+ if (bodylen > 0) {
+ priv->outlen++;
+ }
+ }
+ else {
+ if (msg->method < HTTP_SYMBOLS) {
+ if (msg->body_buf.len == 0 || allow_shared) {
+ pbody = NULL;
+ bodylen = 0;
+ priv->outlen = 2;
+
+ if (msg->method == HTTP_INVALID) {
+ msg->method = HTTP_GET;
+ }
+ }
+ else {
+ pbody = (gchar *) msg->body_buf.begin;
+ bodylen = msg->body_buf.len;
+ priv->outlen = 3;
+
+ if (msg->method == HTTP_INVALID) {
+ msg->method = HTTP_POST;
+ }
+ }
+ }
+ else if (msg->body_buf.len > 0) {
+ allow_shared = FALSE;
+ pbody = (gchar *) msg->body_buf.begin;
+ bodylen = msg->body_buf.len;
+ priv->outlen = 2;
+ }
+ else {
+ /* Invalid body for spamc method */
+ abort();
+ }
+ }
+
+ peer_key = msg->peer_key;
+
+ priv->wr_total = bodylen + 2;
+
+ hdrcount = 0;
+
+ if (msg->method < HTTP_SYMBOLS) {
+ kh_foreach_value (msg->headers, hdr, {
+ DL_FOREACH (hdr, hcur) {
+ /* <name: value\r\n> */
+ priv->wr_total += hcur->combined->len;
+ enclen += hcur->combined->len;
+ priv->outlen ++;
+ hdrcount ++;
+ }
+});
+}
+
+/* Allocate iov */
+priv->out = g_malloc0(sizeof(struct iovec) * priv->outlen);
+priv->wr_pos = 0;
+
+meth_len = rspamd_http_message_write_header(mime_type, encrypted,
+ repbuf, sizeof(repbuf), bodylen, enclen,
+ host, conn, msg,
+ &buf, priv, peer_key);
+priv->wr_total += buf->len;
+
+/* Setup external request body */
+priv->out[0].iov_base = buf->str;
+priv->out[0].iov_len = buf->len;
+
+/* Buf will be used eventually for encryption */
+if (encrypted) {
+ gint meth_offset, nonce_offset, mac_offset;
+ mode = rspamd_keypair_alg(priv->local_key);
+
+ ottery_rand_bytes(nonce, rspamd_cryptobox_nonce_bytes(mode));
+ memset(mac, 0, rspamd_cryptobox_mac_bytes(mode));
+ meth_offset = buf->len;
+
+ if (conn->type == RSPAMD_HTTP_SERVER) {
+ buf = rspamd_fstring_append(buf, repbuf, meth_len);
+ }
+ else {
+ meth_len = strlen(http_method_str(msg->method)) + 1; /* + space */
+ buf = rspamd_fstring_append(buf, http_method_str(msg->method),
+ meth_len - 1);
+ buf = rspamd_fstring_append(buf, " ", 1);
+ }
+
+ nonce_offset = buf->len;
+ buf = rspamd_fstring_append(buf, nonce,
+ rspamd_cryptobox_nonce_bytes(mode));
+ mac_offset = buf->len;
+ buf = rspamd_fstring_append(buf, mac,
+ rspamd_cryptobox_mac_bytes(mode));
+
+ /* Need to be encrypted */
+ if (conn->type == RSPAMD_HTTP_SERVER) {
+ buf = rspamd_fstring_append(buf, "\r\n\r\n", 4);
+ }
+ else {
+ buf = rspamd_fstring_append(buf, repbuf, preludelen);
+ }
+
+ meth_pos = buf->str + meth_offset;
+ np = buf->str + nonce_offset;
+ mp = buf->str + mac_offset;
+}
+
+/* During previous writes, buf might be reallocated and changed */
+priv->buf->data = buf;
+
+if (encrypted) {
+ /* Finish external HTTP request */
+ priv->out[1].iov_base = "\r\n";
+ priv->out[1].iov_len = 2;
+ /* Encrypt the real request */
+ rspamd_http_connection_encrypt_message(conn, msg, priv, pbody, bodylen,
+ meth_pos, meth_len, preludelen, hdrcount, np, mp, peer_key);
+}
+else {
+ i = 1;
+ if (msg->method < HTTP_SYMBOLS) {
+ kh_foreach_value (msg->headers, hdr, {
+ DL_FOREACH (hdr, hcur) {
+ priv->out[i].iov_base = hcur->combined->str;
+ priv->out[i++].iov_len = hcur->combined->len;
+ }
+});
+
+priv->out[i].iov_base = "\r\n";
+priv->out[i++].iov_len = 2;
+}
+else
+{
+ /* No CRLF for compatibility reply */
+ priv->wr_total -= 2;
+}
+
+if (pbody != NULL) {
+ priv->out[i].iov_base = pbody;
+ priv->out[i++].iov_len = bodylen;
+}
+}
+
+priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_RESETED;
+
+if ((priv->flags & RSPAMD_HTTP_CONN_FLAG_PROXY) && (conn->opts & RSPAMD_HTTP_CLIENT_SSL)) {
+ /* We need to disable SSL flag! */
+ err = g_error_new(HTTP_ERROR, 400, "cannot use proxy for SSL connections");
+ rspamd_http_connection_ref(conn);
+ conn->error_handler(conn, err);
+ rspamd_http_connection_unref(conn);
+ g_error_free(err);
+ return FALSE;
+}
+
+rspamd_ev_watcher_stop(priv->ctx->event_loop, &priv->ev);
+
+if (conn->opts & RSPAMD_HTTP_CLIENT_SSL) {
+ gpointer ssl_ctx = (msg->flags & RSPAMD_HTTP_FLAG_SSL_NOVERIFY) ? priv->ctx->ssl_ctx_noverify : priv->ctx->ssl_ctx;
+
+ if (!ssl_ctx) {
+ err = g_error_new(HTTP_ERROR, 400, "ssl message requested "
+ "with no ssl ctx");
+ rspamd_http_connection_ref(conn);
+ conn->error_handler(conn, err);
+ rspamd_http_connection_unref(conn);
+ g_error_free(err);
+ return FALSE;
+ }
+ else {
+ if (!priv->ssl) {
+ priv->ssl = rspamd_ssl_connection_new(ssl_ctx, priv->ctx->event_loop,
+ !(msg->flags & RSPAMD_HTTP_FLAG_SSL_NOVERIFY),
+ conn->log_tag);
+ g_assert(priv->ssl != NULL);
+
+ if (!rspamd_ssl_connect_fd(priv->ssl, conn->fd, host, &priv->ev,
+ priv->timeout, rspamd_http_event_handler,
+ rspamd_http_ssl_err_handler, conn)) {
+
+ err = g_error_new(HTTP_ERROR, 400,
+ "ssl connection error: ssl error=%s, errno=%s",
+ ERR_error_string(ERR_get_error(), NULL),
+ strerror(errno));
+ rspamd_http_connection_ref(conn);
+ conn->error_handler(conn, err);
+ rspamd_http_connection_unref(conn);
+ g_error_free(err);
+ return FALSE;
+ }
+ }
+ else {
+ /* Just restore SSL handlers */
+ rspamd_ssl_connection_restore_handlers(priv->ssl,
+ rspamd_http_event_handler,
+ rspamd_http_ssl_err_handler,
+ conn,
+ EV_WRITE);
+ }
+ }
+}
+else {
+ rspamd_ev_watcher_init(&priv->ev, conn->fd, EV_WRITE,
+ rspamd_http_event_handler, conn);
+ rspamd_ev_watcher_start(priv->ctx->event_loop, &priv->ev, priv->timeout);
+}
+
+return TRUE;
+}
+
+gboolean
+rspamd_http_connection_write_message(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *host,
+ const gchar *mime_type,
+ gpointer ud,
+ ev_tstamp timeout)
+{
+ return rspamd_http_connection_write_message_common(conn, msg, host, mime_type,
+ ud, timeout, FALSE);
+}
+
+gboolean
+rspamd_http_connection_write_message_shared(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *host,
+ const gchar *mime_type,
+ gpointer ud,
+ ev_tstamp timeout)
+{
+ return rspamd_http_connection_write_message_common(conn, msg, host, mime_type,
+ ud, timeout, TRUE);
+}
+
+
+void rspamd_http_connection_set_max_size(struct rspamd_http_connection *conn,
+ gsize sz)
+{
+ conn->max_size = sz;
+}
+
+void rspamd_http_connection_set_key(struct rspamd_http_connection *conn,
+ struct rspamd_cryptobox_keypair *key)
+{
+ struct rspamd_http_connection_private *priv = conn->priv;
+
+ g_assert(key != NULL);
+ priv->local_key = rspamd_keypair_ref(key);
+}
+
+void rspamd_http_connection_own_socket(struct rspamd_http_connection *conn)
+{
+ struct rspamd_http_connection_private *priv = conn->priv;
+
+ priv->flags |= RSPAMD_HTTP_CONN_OWN_SOCKET;
+}
+
+const struct rspamd_cryptobox_pubkey *
+rspamd_http_connection_get_peer_key(struct rspamd_http_connection *conn)
+{
+ struct rspamd_http_connection_private *priv = conn->priv;
+
+ if (priv->peer_key) {
+ return priv->peer_key;
+ }
+ else if (priv->msg) {
+ return priv->msg->peer_key;
+ }
+
+ return NULL;
+}
+
+gboolean
+rspamd_http_connection_is_encrypted(struct rspamd_http_connection *conn)
+{
+ struct rspamd_http_connection_private *priv = conn->priv;
+
+ if (priv->peer_key != NULL) {
+ return TRUE;
+ }
+ else if (priv->msg) {
+ return priv->msg->peer_key != NULL;
+ }
+
+ return FALSE;
+}
+
+GHashTable *
+rspamd_http_message_parse_query(struct rspamd_http_message *msg)
+{
+ GHashTable *res;
+ rspamd_fstring_t *key = NULL, *value = NULL;
+ rspamd_ftok_t *key_tok = NULL, *value_tok = NULL;
+ const gchar *p, *c, *end;
+ struct http_parser_url u;
+ enum {
+ parse_key,
+ parse_eqsign,
+ parse_value,
+ parse_ampersand
+ } state = parse_key;
+
+ res = g_hash_table_new_full(rspamd_ftok_icase_hash,
+ rspamd_ftok_icase_equal,
+ rspamd_fstring_mapped_ftok_free,
+ rspamd_fstring_mapped_ftok_free);
+
+ if (msg->url && msg->url->len > 0) {
+ http_parser_parse_url(msg->url->str, msg->url->len, TRUE, &u);
+
+ if (u.field_set & (1 << UF_QUERY)) {
+ p = msg->url->str + u.field_data[UF_QUERY].off;
+ c = p;
+ end = p + u.field_data[UF_QUERY].len;
+
+ while (p <= end) {
+ switch (state) {
+ case parse_key:
+ if ((p == end || *p == '&') && p > c) {
+ /* We have a single parameter without a value */
+ key = rspamd_fstring_new_init(c, p - c);
+ key_tok = rspamd_ftok_map(key);
+ key_tok->len = rspamd_url_decode(key->str, key->str,
+ key->len);
+
+ value = rspamd_fstring_new_init("", 0);
+ value_tok = rspamd_ftok_map(value);
+
+ g_hash_table_replace(res, key_tok, value_tok);
+ state = parse_ampersand;
+ }
+ else if (*p == '=' && p > c) {
+ /* We have something like key=value */
+ key = rspamd_fstring_new_init(c, p - c);
+ key_tok = rspamd_ftok_map(key);
+ key_tok->len = rspamd_url_decode(key->str, key->str,
+ key->len);
+
+ state = parse_eqsign;
+ }
+ else {
+ p++;
+ }
+ break;
+
+ case parse_eqsign:
+ if (*p != '=') {
+ c = p;
+ state = parse_value;
+ }
+ else {
+ p++;
+ }
+ break;
+
+ case parse_value:
+ if ((p == end || *p == '&') && p >= c) {
+ g_assert(key != NULL);
+ if (p > c) {
+ value = rspamd_fstring_new_init(c, p - c);
+ value_tok = rspamd_ftok_map(value);
+ value_tok->len = rspamd_url_decode(value->str,
+ value->str,
+ value->len);
+ /* Detect quotes for value */
+ if (value_tok->begin[0] == '"') {
+ memmove(value->str, value->str + 1,
+ value_tok->len - 1);
+ value_tok->len--;
+ }
+ if (value_tok->begin[value_tok->len - 1] == '"') {
+ value_tok->len--;
+ }
+ }
+ else {
+ value = rspamd_fstring_new_init("", 0);
+ value_tok = rspamd_ftok_map(value);
+ }
+
+ g_hash_table_replace(res, key_tok, value_tok);
+ key = value = NULL;
+ key_tok = value_tok = NULL;
+ state = parse_ampersand;
+ }
+ else {
+ p++;
+ }
+ break;
+
+ case parse_ampersand:
+ if (p != end && *p != '&') {
+ c = p;
+ state = parse_key;
+ }
+ else {
+ p++;
+ }
+ break;
+ }
+ }
+ }
+
+ if (state != parse_ampersand && key != NULL) {
+ rspamd_fstring_free(key);
+ }
+ }
+
+ return res;
+}
+
+
+struct rspamd_http_message *
+rspamd_http_message_ref(struct rspamd_http_message *msg)
+{
+ REF_RETAIN(msg);
+
+ return msg;
+}
+
+void rspamd_http_message_unref(struct rspamd_http_message *msg)
+{
+ REF_RELEASE(msg);
+}
+
+void rspamd_http_connection_disable_encryption(struct rspamd_http_connection *conn)
+{
+ struct rspamd_http_connection_private *priv;
+
+ priv = conn->priv;
+
+ if (priv) {
+ if (priv->local_key) {
+ rspamd_keypair_unref(priv->local_key);
+ }
+ if (priv->peer_key) {
+ rspamd_pubkey_unref(priv->peer_key);
+ }
+
+ priv->local_key = NULL;
+ priv->peer_key = NULL;
+ priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_ENCRYPTED;
+ }
+} \ No newline at end of file
diff --git a/src/libserver/http/http_connection.h b/src/libserver/http/http_connection.h
new file mode 100644
index 0000000..e98d164
--- /dev/null
+++ b/src/libserver/http/http_connection.h
@@ -0,0 +1,320 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef HTTP_H_
+#define HTTP_H_
+
+/**
+ * @file http.h
+ *
+ * This is an interface for HTTP client and conn.
+ * This code uses HTTP parser written by Joyent Inc based on nginx code.
+ */
+
+#include "config.h"
+#include "http_context.h"
+#include "fstring.h"
+#include "ref.h"
+#include "http_message.h"
+#include "http_util.h"
+#include "addr.h"
+
+#include "contrib/libev/ev.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum rspamd_http_connection_type {
+ RSPAMD_HTTP_SERVER,
+ RSPAMD_HTTP_CLIENT
+};
+
+struct rspamd_http_header;
+struct rspamd_http_message;
+struct rspamd_http_connection_private;
+struct rspamd_http_connection;
+struct rspamd_http_connection_router;
+struct rspamd_http_connection_entry;
+struct rspamd_keepalive_hash_key;
+
+struct rspamd_storage_shmem {
+ gchar *shm_name;
+ ref_entry_t ref;
+};
+
+/**
+ * Legacy spamc protocol
+ */
+#define RSPAMD_HTTP_FLAG_SPAMC (1 << 0)
+/**
+ * Store body of the message in a shared memory segment
+ */
+#define RSPAMD_HTTP_FLAG_SHMEM (1 << 2)
+/**
+ * Store body of the message in an immutable shared memory segment
+ */
+#define RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE (1 << 3)
+/**
+ * Body has been set for a message
+ */
+#define RSPAMD_HTTP_FLAG_HAS_BODY (1 << 5)
+/**
+ * Do not verify server's certificate
+ */
+#define RSPAMD_HTTP_FLAG_SSL_NOVERIFY (1 << 6)
+/**
+ * Body has been set for a message
+ */
+#define RSPAMD_HTTP_FLAG_HAS_HOST_HEADER (1 << 7)
+/**
+ * Message is intended for SSL connection
+ */
+#define RSPAMD_HTTP_FLAG_WANT_SSL (1 << 8)
+/**
+ * Options for HTTP connection
+ */
+enum rspamd_http_options {
+ RSPAMD_HTTP_BODY_PARTIAL = 1, /**< Call body handler on all body data portions */
+ RSPAMD_HTTP_CLIENT_SIMPLE = 1u << 1, /**< Read HTTP client reply automatically */
+ RSPAMD_HTTP_CLIENT_ENCRYPTED = 1u << 2, /**< Encrypt data for client */
+ RSPAMD_HTTP_CLIENT_SHARED = 1u << 3, /**< Store reply in shared memory */
+ RSPAMD_HTTP_REQUIRE_ENCRYPTION = 1u << 4,
+ RSPAMD_HTTP_CLIENT_KEEP_ALIVE = 1u << 5,
+ RSPAMD_HTTP_CLIENT_SSL = 1u << 6u,
+};
+
+typedef int (*rspamd_http_body_handler_t)(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *chunk,
+ gsize len);
+
+typedef void (*rspamd_http_error_handler_t)(struct rspamd_http_connection *conn,
+ GError *err);
+
+typedef int (*rspamd_http_finish_handler_t)(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg);
+
+/**
+ * HTTP connection structure
+ */
+struct rspamd_http_connection {
+ struct rspamd_http_connection_private *priv;
+ rspamd_http_body_handler_t body_handler;
+ rspamd_http_error_handler_t error_handler;
+ rspamd_http_finish_handler_t finish_handler;
+ gpointer ud;
+ const gchar *log_tag;
+ /* Used for keepalive */
+ struct rspamd_keepalive_hash_key *keepalive_hash_key;
+ gsize max_size;
+ unsigned opts;
+ enum rspamd_http_connection_type type;
+ gboolean finished;
+ gint fd;
+ gint ref;
+};
+
+/**
+ * Creates a new HTTP server connection from an opened FD returned by accept function
+ * @param ctx
+ * @param fd
+ * @param body_handler
+ * @param error_handler
+ * @param finish_handler
+ * @param opts
+ * @return
+ */
+struct rspamd_http_connection *rspamd_http_connection_new_server(
+ struct rspamd_http_context *ctx,
+ gint fd,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ unsigned opts);
+
+/**
+ * Creates or reuses a new keepalive client connection identified by hostname and inet_addr
+ * @param ctx
+ * @param body_handler
+ * @param error_handler
+ * @param finish_handler
+ * @param addr
+ * @param host
+ * @return
+ */
+struct rspamd_http_connection *rspamd_http_connection_new_client_keepalive(
+ struct rspamd_http_context *ctx,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ unsigned opts,
+ rspamd_inet_addr_t *addr,
+ const gchar *host);
+
+/**
+ * Creates an ordinary connection using the address specified (if proxy is not set)
+ * @param ctx
+ * @param body_handler
+ * @param error_handler
+ * @param finish_handler
+ * @param opts
+ * @param addr
+ * @return
+ */
+struct rspamd_http_connection *rspamd_http_connection_new_client(
+ struct rspamd_http_context *ctx,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ unsigned opts,
+ rspamd_inet_addr_t *addr);
+
+/**
+ * Creates an ordinary client connection using ready file descriptor (ignores proxy)
+ * @param ctx
+ * @param body_handler
+ * @param error_handler
+ * @param finish_handler
+ * @param opts
+ * @param addr
+ * @return
+ */
+struct rspamd_http_connection *rspamd_http_connection_new_client_socket(
+ struct rspamd_http_context *ctx,
+ rspamd_http_body_handler_t body_handler,
+ rspamd_http_error_handler_t error_handler,
+ rspamd_http_finish_handler_t finish_handler,
+ unsigned opts,
+ gint fd);
+
+/**
+ * Set key pointed by an opaque pointer
+ * @param conn connection structure
+ * @param key opaque key structure
+ */
+void rspamd_http_connection_set_key(struct rspamd_http_connection *conn,
+ struct rspamd_cryptobox_keypair *key);
+
+/**
+ * Transfer ownership on socket to an HTTP connection
+ * @param conn
+ */
+void rspamd_http_connection_own_socket(struct rspamd_http_connection *conn);
+
+/**
+ * Get peer's public key
+ * @param conn connection structure
+ * @return pubkey structure or NULL
+ */
+const struct rspamd_cryptobox_pubkey *rspamd_http_connection_get_peer_key(
+ struct rspamd_http_connection *conn);
+
+/**
+ * Returns TRUE if a connection is encrypted
+ * @param conn
+ * @return
+ */
+gboolean rspamd_http_connection_is_encrypted(struct rspamd_http_connection *conn);
+
+/**
+ * Handle a request using socket fd and user data ud
+ * @param conn connection structure
+ * @param ud opaque user data
+ * @param fd fd to read/write
+ */
+void rspamd_http_connection_read_message(
+ struct rspamd_http_connection *conn,
+ gpointer ud,
+ ev_tstamp timeout);
+
+void rspamd_http_connection_read_message_shared(
+ struct rspamd_http_connection *conn,
+ gpointer ud,
+ ev_tstamp timeout);
+
+/**
+ * Send reply using initialised connection
+ * @param conn connection structure
+ * @param msg HTTP message
+ * @param ud opaque user data
+ * @param fd fd to read/write
+ */
+gboolean rspamd_http_connection_write_message(
+ struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *host,
+ const gchar *mime_type,
+ gpointer ud,
+ ev_tstamp timeout);
+
+gboolean rspamd_http_connection_write_message_shared(
+ struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *host,
+ const gchar *mime_type,
+ gpointer ud,
+ ev_tstamp timeout);
+
+/**
+ * Free connection structure
+ * @param conn
+ */
+void rspamd_http_connection_free(struct rspamd_http_connection *conn);
+
+/**
+ * Increase refcount for a connection
+ * @param conn
+ * @return
+ */
+static inline struct rspamd_http_connection *
+rspamd_http_connection_ref(struct rspamd_http_connection *conn)
+{
+ conn->ref++;
+ return conn;
+}
+
+/**
+ * Decrease a refcount for a connection and free it if refcount is equal to zero
+ * @param conn
+ */
+static void
+rspamd_http_connection_unref(struct rspamd_http_connection *conn)
+{
+ if (--conn->ref <= 0) {
+ rspamd_http_connection_free(conn);
+ }
+}
+
+/**
+ * Reset connection for a new request
+ * @param conn
+ */
+void rspamd_http_connection_reset(struct rspamd_http_connection *conn);
+
+/**
+ * Sets global maximum size for HTTP message being processed
+ * @param sz
+ */
+void rspamd_http_connection_set_max_size(struct rspamd_http_connection *conn,
+ gsize sz);
+
+void rspamd_http_connection_disable_encryption(struct rspamd_http_connection *conn);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* HTTP_H_ */
diff --git a/src/libserver/http/http_context.c b/src/libserver/http/http_context.c
new file mode 100644
index 0000000..f08e33b
--- /dev/null
+++ b/src/libserver/http/http_context.c
@@ -0,0 +1,670 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "http_context.h"
+#include "http_private.h"
+#include "keypair.h"
+#include "keypairs_cache.h"
+#include "cfg_file.h"
+#include "contrib/libottery/ottery.h"
+#include "contrib/http-parser/http_parser.h"
+#include "ssl_util.h"
+#include "rspamd.h"
+#include "libev_helper.h"
+
+INIT_LOG_MODULE(http_context)
+
+#define msg_debug_http_context(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_http_context_log_id, "http_context", NULL, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+static struct rspamd_http_context *default_ctx = NULL;
+
+struct rspamd_http_keepalive_cbdata {
+ struct rspamd_http_connection *conn;
+ struct rspamd_http_context *ctx;
+ GQueue *queue;
+ GList *link;
+ struct rspamd_io_ev ev;
+};
+
+static void
+rspamd_http_keepalive_queue_cleanup(GQueue *conns)
+{
+ GList *cur;
+
+ cur = conns->head;
+
+ while (cur) {
+ struct rspamd_http_keepalive_cbdata *cbd;
+
+ cbd = (struct rspamd_http_keepalive_cbdata *) cur->data;
+ /* unref call closes fd, so we need to remove ev watcher first! */
+ rspamd_ev_watcher_stop(cbd->ctx->event_loop, &cbd->ev);
+ rspamd_http_connection_unref(cbd->conn);
+ g_free(cbd);
+
+ cur = cur->next;
+ }
+
+ g_queue_clear(conns);
+}
+
+static void
+rspamd_http_context_client_rotate_ev(struct ev_loop *loop, ev_timer *w, int revents)
+{
+ struct rspamd_http_context *ctx = (struct rspamd_http_context *) w->data;
+ gpointer kp;
+
+ w->repeat = rspamd_time_jitter(ctx->config.client_key_rotate_time, 0);
+ msg_debug_http_context("rotate local keypair, next rotate in %.0f seconds",
+ w->repeat);
+
+ ev_timer_again(loop, w);
+
+ kp = ctx->client_kp;
+ ctx->client_kp = rspamd_keypair_new(RSPAMD_KEYPAIR_KEX,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+ rspamd_keypair_unref(kp);
+}
+
+static struct rspamd_http_context *
+rspamd_http_context_new_default(struct rspamd_config *cfg,
+ struct ev_loop *ev_base,
+ struct upstream_ctx *ups_ctx)
+{
+ struct rspamd_http_context *ctx;
+
+ static const int default_kp_size = 1024;
+ static const gdouble default_rotate_time = 120;
+ static const gdouble default_keepalive_interval = 65;
+ static const gchar *default_user_agent = "rspamd-" RSPAMD_VERSION_FULL;
+ static const gchar *default_server_hdr = "rspamd/" RSPAMD_VERSION_FULL;
+
+ ctx = g_malloc0(sizeof(*ctx));
+ ctx->config.kp_cache_size_client = default_kp_size;
+ ctx->config.kp_cache_size_server = default_kp_size;
+ ctx->config.client_key_rotate_time = default_rotate_time;
+ ctx->config.user_agent = default_user_agent;
+ ctx->config.keepalive_interval = default_keepalive_interval;
+ ctx->config.server_hdr = default_server_hdr;
+ ctx->ups_ctx = ups_ctx;
+
+ if (cfg) {
+ ctx->ssl_ctx = cfg->libs_ctx->ssl_ctx;
+ ctx->ssl_ctx_noverify = cfg->libs_ctx->ssl_ctx_noverify;
+ }
+ else {
+ ctx->ssl_ctx = rspamd_init_ssl_ctx();
+ ctx->ssl_ctx_noverify = rspamd_init_ssl_ctx_noverify();
+ }
+
+ ctx->event_loop = ev_base;
+
+ ctx->keep_alive_hash = kh_init(rspamd_keep_alive_hash);
+
+ return ctx;
+}
+
+static void
+rspamd_http_context_parse_proxy(struct rspamd_http_context *ctx,
+ const gchar *name,
+ struct upstream_list **pls)
+{
+ struct http_parser_url u;
+ struct upstream_list *uls;
+
+ if (!ctx->ups_ctx) {
+ msg_err("cannot parse http_proxy %s - upstreams context is undefined", name);
+ return;
+ }
+
+ memset(&u, 0, sizeof(u));
+
+ if (http_parser_parse_url(name, strlen(name), 1, &u) == 0) {
+ if (!(u.field_set & (1u << UF_HOST)) || u.port == 0) {
+ msg_err("cannot parse http(s) proxy %s - invalid host or port", name);
+
+ return;
+ }
+
+ uls = rspamd_upstreams_create(ctx->ups_ctx);
+
+ if (!rspamd_upstreams_parse_line_len(uls,
+ name + u.field_data[UF_HOST].off,
+ u.field_data[UF_HOST].len, u.port, NULL)) {
+ msg_err("cannot parse http(s) proxy %s - invalid data", name);
+
+ rspamd_upstreams_destroy(uls);
+ }
+ else {
+ *pls = uls;
+ msg_info("set http(s) proxy to %s", name);
+ }
+ }
+ else {
+ uls = rspamd_upstreams_create(ctx->ups_ctx);
+
+ if (!rspamd_upstreams_parse_line(uls,
+ name, 3128, NULL)) {
+ msg_err("cannot parse http(s) proxy %s - invalid data", name);
+
+ rspamd_upstreams_destroy(uls);
+ }
+ else {
+ *pls = uls;
+ msg_info("set http(s) proxy to %s", name);
+ }
+ }
+}
+
+static void
+rspamd_http_context_init(struct rspamd_http_context *ctx)
+{
+ if (ctx->config.kp_cache_size_client > 0) {
+ ctx->client_kp_cache = rspamd_keypair_cache_new(ctx->config.kp_cache_size_client);
+ }
+
+ if (ctx->config.kp_cache_size_server > 0) {
+ ctx->server_kp_cache = rspamd_keypair_cache_new(ctx->config.kp_cache_size_server);
+ }
+
+ if (ctx->config.client_key_rotate_time > 0 && ctx->event_loop) {
+ double jittered = rspamd_time_jitter(ctx->config.client_key_rotate_time,
+ 0);
+
+ ev_timer_init(&ctx->client_rotate_ev,
+ rspamd_http_context_client_rotate_ev, jittered, 0);
+ ev_timer_start(ctx->event_loop, &ctx->client_rotate_ev);
+ ctx->client_rotate_ev.data = ctx;
+ }
+
+ if (ctx->config.http_proxy) {
+ rspamd_http_context_parse_proxy(ctx, ctx->config.http_proxy,
+ &ctx->http_proxies);
+ }
+
+ default_ctx = ctx;
+}
+
+struct rspamd_http_context *
+rspamd_http_context_create(struct rspamd_config *cfg,
+ struct ev_loop *ev_base,
+ struct upstream_ctx *ups_ctx)
+{
+ struct rspamd_http_context *ctx;
+ const ucl_object_t *http_obj;
+
+ ctx = rspamd_http_context_new_default(cfg, ev_base, ups_ctx);
+ http_obj = ucl_object_lookup(cfg->cfg_ucl_obj, "http");
+
+ if (http_obj) {
+ const ucl_object_t *server_obj, *client_obj;
+
+ client_obj = ucl_object_lookup(http_obj, "client");
+
+ if (client_obj) {
+ const ucl_object_t *kp_size;
+
+ kp_size = ucl_object_lookup(client_obj, "cache_size");
+
+ if (kp_size) {
+ ctx->config.kp_cache_size_client = ucl_object_toint(kp_size);
+ }
+
+ const ucl_object_t *rotate_time;
+
+ rotate_time = ucl_object_lookup(client_obj, "rotate_time");
+
+ if (rotate_time) {
+ ctx->config.client_key_rotate_time = ucl_object_todouble(rotate_time);
+ }
+
+ const ucl_object_t *user_agent;
+
+ user_agent = ucl_object_lookup(client_obj, "user_agent");
+
+ if (user_agent) {
+ ctx->config.user_agent = ucl_object_tostring(user_agent);
+
+ if (ctx->config.user_agent && strlen(ctx->config.user_agent) == 0) {
+ ctx->config.user_agent = NULL;
+ }
+ }
+
+ const ucl_object_t *server_hdr;
+ server_hdr = ucl_object_lookup(client_obj, "server_hdr");
+
+ if (server_hdr) {
+ ctx->config.server_hdr = ucl_object_tostring(server_hdr);
+
+ if (ctx->config.server_hdr && strlen(ctx->config.server_hdr) == 0) {
+ ctx->config.server_hdr = "";
+ }
+ }
+
+ const ucl_object_t *keepalive_interval;
+
+ keepalive_interval = ucl_object_lookup(client_obj, "keepalive_interval");
+
+ if (keepalive_interval) {
+ ctx->config.keepalive_interval = ucl_object_todouble(keepalive_interval);
+ }
+
+ const ucl_object_t *http_proxy;
+ http_proxy = ucl_object_lookup(client_obj, "http_proxy");
+
+ if (http_proxy) {
+ ctx->config.http_proxy = ucl_object_tostring(http_proxy);
+ }
+ }
+
+ server_obj = ucl_object_lookup(http_obj, "server");
+
+ if (server_obj) {
+ const ucl_object_t *kp_size;
+
+ kp_size = ucl_object_lookup(server_obj, "cache_size");
+
+ if (kp_size) {
+ ctx->config.kp_cache_size_server = ucl_object_toint(kp_size);
+ }
+ }
+ }
+
+ rspamd_http_context_init(ctx);
+
+ return ctx;
+}
+
+
+void rspamd_http_context_free(struct rspamd_http_context *ctx)
+{
+ if (ctx == default_ctx) {
+ default_ctx = NULL;
+ }
+
+ if (ctx->client_kp_cache) {
+ rspamd_keypair_cache_destroy(ctx->client_kp_cache);
+ }
+
+ if (ctx->server_kp_cache) {
+ rspamd_keypair_cache_destroy(ctx->server_kp_cache);
+ }
+
+ if (ctx->config.client_key_rotate_time > 0) {
+ ev_timer_stop(ctx->event_loop, &ctx->client_rotate_ev);
+
+ if (ctx->client_kp) {
+ rspamd_keypair_unref(ctx->client_kp);
+ }
+ }
+
+ struct rspamd_keepalive_hash_key *hk;
+
+ kh_foreach_key(ctx->keep_alive_hash, hk, {
+ msg_debug_http_context("cleanup keepalive elt %s (%s)",
+ rspamd_inet_address_to_string_pretty(hk->addr),
+ hk->host);
+
+ if (hk->host) {
+ g_free(hk->host);
+ }
+
+ rspamd_inet_address_free(hk->addr);
+ rspamd_http_keepalive_queue_cleanup(&hk->conns);
+ g_free(hk);
+ });
+
+ kh_destroy(rspamd_keep_alive_hash, ctx->keep_alive_hash);
+
+ if (ctx->http_proxies) {
+ rspamd_upstreams_destroy(ctx->http_proxies);
+ }
+
+ g_free(ctx);
+}
+
+struct rspamd_http_context *
+rspamd_http_context_create_config(struct rspamd_http_context_cfg *cfg,
+ struct ev_loop *ev_base,
+ struct upstream_ctx *ups_ctx)
+{
+ struct rspamd_http_context *ctx;
+
+ ctx = rspamd_http_context_new_default(NULL, ev_base, ups_ctx);
+ memcpy(&ctx->config, cfg, sizeof(*cfg));
+ rspamd_http_context_init(ctx);
+
+ return ctx;
+}
+
+struct rspamd_http_context *
+rspamd_http_context_default(void)
+{
+ g_assert(default_ctx != NULL);
+
+ return default_ctx;
+}
+
+gint32
+rspamd_keep_alive_key_hash(struct rspamd_keepalive_hash_key *k)
+{
+ rspamd_cryptobox_fast_hash_state_t hst;
+
+ rspamd_cryptobox_fast_hash_init(&hst, 0);
+
+ if (k->host) {
+ rspamd_cryptobox_fast_hash_update(&hst, k->host, strlen(k->host));
+ }
+
+ rspamd_cryptobox_fast_hash_update(&hst, &k->port, sizeof(k->port));
+ rspamd_cryptobox_fast_hash_update(&hst, &k->is_ssl, sizeof(k->is_ssl));
+
+ return rspamd_cryptobox_fast_hash_final(&hst);
+}
+
+bool rspamd_keep_alive_key_equal(struct rspamd_keepalive_hash_key *k1,
+ struct rspamd_keepalive_hash_key *k2)
+{
+ if (k1->is_ssl != k2->is_ssl) {
+ return false;
+ }
+
+ if (k1->host && k2->host) {
+ if (k1->port == k2->port) {
+ return strcmp(k1->host, k2->host) == 0;
+ }
+ }
+ else if (!k1->host && !k2->host) {
+ return (k1->port == k2->port);
+ }
+
+ /* One has host and another has no host */
+ return false;
+}
+
+struct rspamd_http_connection *
+rspamd_http_context_check_keepalive(struct rspamd_http_context *ctx,
+ const rspamd_inet_addr_t *addr,
+ const gchar *host,
+ bool is_ssl)
+{
+ struct rspamd_keepalive_hash_key hk, *phk;
+ khiter_t k;
+
+ if (ctx == NULL) {
+ ctx = rspamd_http_context_default();
+ }
+
+ hk.addr = (rspamd_inet_addr_t *) addr;
+ hk.host = (gchar *) host;
+ hk.port = rspamd_inet_address_get_port(addr);
+ hk.is_ssl = is_ssl;
+
+ k = kh_get(rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk);
+
+ if (k != kh_end(ctx->keep_alive_hash)) {
+ phk = kh_key(ctx->keep_alive_hash, k);
+ GQueue *conns = &phk->conns;
+
+ /* Use stack based approach */
+
+ if (g_queue_get_length(conns) > 0) {
+ struct rspamd_http_keepalive_cbdata *cbd;
+ struct rspamd_http_connection *conn;
+ gint err;
+ socklen_t len = sizeof(gint);
+
+ cbd = g_queue_pop_head(conns);
+ rspamd_ev_watcher_stop(ctx->event_loop, &cbd->ev);
+ conn = cbd->conn;
+ g_free(cbd);
+
+ if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void *) &err, &len) == -1) {
+ err = errno;
+ }
+
+ if (err != 0) {
+ rspamd_http_connection_unref(conn);
+
+ msg_debug_http_context("invalid reused keepalive element %s (%s, ssl=%d); "
+ "%s error; "
+ "%d connections queued",
+ rspamd_inet_address_to_string_pretty(phk->addr),
+ phk->host,
+ (int) phk->is_ssl,
+ g_strerror(err),
+ conns->length);
+
+ return NULL;
+ }
+
+ msg_debug_http_context("reused keepalive element %s (%s, ssl=%d), %d connections queued",
+ rspamd_inet_address_to_string_pretty(phk->addr),
+ phk->host,
+ (int) phk->is_ssl,
+ conns->length);
+
+ /* We transfer refcount here! */
+ return conn;
+ }
+ else {
+ msg_debug_http_context("found empty keepalive element %s (%s), cannot reuse",
+ rspamd_inet_address_to_string_pretty(phk->addr),
+ phk->host);
+ }
+ }
+
+ return NULL;
+}
+
+const rspamd_inet_addr_t *
+rspamd_http_context_has_keepalive(struct rspamd_http_context *ctx,
+ const gchar *host,
+ unsigned port,
+ bool is_ssl)
+{
+ struct rspamd_keepalive_hash_key hk, *phk;
+ khiter_t k;
+
+ if (ctx == NULL) {
+ ctx = rspamd_http_context_default();
+ }
+
+ hk.host = (gchar *) host;
+ hk.port = port;
+ hk.is_ssl = is_ssl;
+
+ k = kh_get(rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk);
+
+ if (k != kh_end(ctx->keep_alive_hash)) {
+ phk = kh_key(ctx->keep_alive_hash, k);
+ GQueue *conns = &phk->conns;
+
+ if (g_queue_get_length(conns) > 0) {
+ return phk->addr;
+ }
+ }
+
+ return NULL;
+}
+
+void rspamd_http_context_prepare_keepalive(struct rspamd_http_context *ctx,
+ struct rspamd_http_connection *conn,
+ const rspamd_inet_addr_t *addr,
+ const gchar *host,
+ bool is_ssl)
+{
+ struct rspamd_keepalive_hash_key hk, *phk;
+ khiter_t k;
+
+ hk.addr = (rspamd_inet_addr_t *) addr;
+ hk.host = (gchar *) host;
+ hk.is_ssl = is_ssl;
+ hk.port = rspamd_inet_address_get_port(addr);
+
+ k = kh_get(rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk);
+
+ if (k != kh_end(ctx->keep_alive_hash)) {
+ /* Reuse existing */
+ conn->keepalive_hash_key = kh_key(ctx->keep_alive_hash, k);
+ msg_debug_http_context("use existing keepalive element %s (%s)",
+ rspamd_inet_address_to_string_pretty(conn->keepalive_hash_key->addr),
+ conn->keepalive_hash_key->host);
+ }
+ else {
+ /* Create new one */
+ GQueue empty_init = G_QUEUE_INIT;
+ gint r;
+
+ phk = g_malloc(sizeof(*phk));
+ phk->conns = empty_init;
+ phk->host = g_strdup(host);
+ phk->is_ssl = is_ssl;
+ phk->addr = rspamd_inet_address_copy(addr, NULL);
+ phk->port = hk.port;
+
+
+ kh_put(rspamd_keep_alive_hash, ctx->keep_alive_hash, phk, &r);
+ conn->keepalive_hash_key = phk;
+
+ msg_debug_http_context("create new keepalive element %s (%s)",
+ rspamd_inet_address_to_string_pretty(conn->keepalive_hash_key->addr),
+ conn->keepalive_hash_key->host);
+ }
+}
+
+static void
+rspamd_http_keepalive_handler(gint fd, short what, gpointer ud)
+{
+ struct rspamd_http_keepalive_cbdata *cbdata =
+ (struct rspamd_http_keepalive_cbdata *) ud; /*
+ * We can get here if a remote side reported something or it has
+ * timed out. In both cases we just terminate keepalive connection.
+ */
+
+ g_queue_delete_link(cbdata->queue, cbdata->link);
+ msg_debug_http_context("remove keepalive element %s (%s), %d connections left",
+ rspamd_inet_address_to_string_pretty(cbdata->conn->keepalive_hash_key->addr),
+ cbdata->conn->keepalive_hash_key->host,
+ cbdata->queue->length);
+ /* unref call closes fd, so we need to remove ev watcher first! */
+ rspamd_ev_watcher_stop(cbdata->ctx->event_loop, &cbdata->ev);
+ rspamd_http_connection_unref(cbdata->conn);
+ g_free(cbdata);
+}
+
+/* Non-static for unit testing */
+long rspamd_http_parse_keepalive_timeout(const rspamd_ftok_t *tok)
+{
+ long timeout = -1;
+ goffset pos = rspamd_substring_search(tok->begin,
+ tok->len, "timeout", sizeof("timeout") - 1);
+
+ if (pos != -1) {
+ pos += sizeof("timeout") - 1;
+
+ /* Skip spaces and equal sign */
+ while (pos < tok->len) {
+ if (tok->begin[pos] != '=' && !g_ascii_isspace(tok->begin[pos])) {
+ break;
+ }
+ pos++;
+ }
+
+ gsize ndigits = rspamd_memspn(tok->begin + pos, "0123456789", tok->len - pos);
+ glong real_timeout;
+
+ if (ndigits > 0) {
+ if (rspamd_strtoul(tok->begin + pos, ndigits, &real_timeout)) {
+ timeout = real_timeout;
+ msg_debug_http_context("got timeout attr %l", timeout);
+ }
+ }
+ }
+
+ return timeout;
+}
+
+void rspamd_http_context_push_keepalive(struct rspamd_http_context *ctx,
+ struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ struct ev_loop *event_loop)
+{
+ struct rspamd_http_keepalive_cbdata *cbdata;
+ gdouble timeout = ctx->config.keepalive_interval;
+
+ g_assert(conn->keepalive_hash_key != NULL);
+
+ if (msg) {
+ const rspamd_ftok_t *tok;
+ rspamd_ftok_t cmp;
+
+ tok = rspamd_http_message_find_header(msg, "Connection");
+
+ if (!tok) {
+ /* Server has not stated that it can do keep alive */
+ conn->finished = TRUE;
+ msg_debug_http_context("no Connection header");
+ return;
+ }
+
+ RSPAMD_FTOK_ASSIGN(&cmp, "keep-alive");
+
+ if (rspamd_ftok_casecmp(&cmp, tok) != 0) {
+ conn->finished = TRUE;
+ msg_debug_http_context("connection header is not `keep-alive`");
+ return;
+ }
+
+ /* We can proceed, check timeout */
+
+ tok = rspamd_http_message_find_header(msg, "Keep-Alive");
+
+ if (tok) {
+ long maybe_timeout = rspamd_http_parse_keepalive_timeout(tok);
+
+ if (maybe_timeout > 0) {
+ timeout = maybe_timeout;
+ }
+ }
+ }
+
+ /* Move connection to the keepalive pool */
+ cbdata = g_malloc0(sizeof(*cbdata));
+
+ cbdata->conn = rspamd_http_connection_ref(conn);
+ /* Use stack like approach to that would easy reading */
+ g_queue_push_head(&conn->keepalive_hash_key->conns, cbdata);
+ cbdata->link = conn->keepalive_hash_key->conns.head;
+
+ cbdata->queue = &conn->keepalive_hash_key->conns;
+ cbdata->ctx = ctx;
+ conn->finished = FALSE;
+
+ rspamd_ev_watcher_init(&cbdata->ev, conn->fd, EV_READ,
+ rspamd_http_keepalive_handler,
+ cbdata);
+ rspamd_ev_watcher_start(event_loop, &cbdata->ev, timeout);
+
+ msg_debug_http_context("push keepalive element %s (%s), %d connections queued, %.1f timeout",
+ rspamd_inet_address_to_string_pretty(cbdata->conn->keepalive_hash_key->addr),
+ cbdata->conn->keepalive_hash_key->host,
+ cbdata->queue->length,
+ timeout);
+} \ No newline at end of file
diff --git a/src/libserver/http/http_context.h b/src/libserver/http/http_context.h
new file mode 100644
index 0000000..f3622ae
--- /dev/null
+++ b/src/libserver/http/http_context.h
@@ -0,0 +1,122 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_HTTP_CONTEXT_H
+#define RSPAMD_HTTP_CONTEXT_H
+
+#include "config.h"
+#include "ucl.h"
+#include "addr.h"
+
+#include "contrib/libev/ev.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_http_context;
+struct rspamd_config;
+struct rspamd_http_message;
+struct upstream_ctx;
+
+struct rspamd_http_context_cfg {
+ guint kp_cache_size_client;
+ guint kp_cache_size_server;
+ guint ssl_cache_size;
+ gdouble keepalive_interval;
+ gdouble client_key_rotate_time;
+ const gchar *user_agent;
+ const gchar *http_proxy;
+ const gchar *server_hdr;
+};
+
+/**
+ * Creates and configures new HTTP context
+ * @param root_conf configuration object
+ * @param ev_base event base
+ * @return new context used for both client and server HTTP connections
+ */
+struct rspamd_http_context *rspamd_http_context_create(struct rspamd_config *cfg,
+ struct ev_loop *ev_base,
+ struct upstream_ctx *ctx);
+
+struct rspamd_http_context *rspamd_http_context_create_config(
+ struct rspamd_http_context_cfg *cfg,
+ struct ev_loop *ev_base,
+ struct upstream_ctx *ctx);
+
+/**
+ * Destroys context
+ * @param ctx
+ */
+void rspamd_http_context_free(struct rspamd_http_context *ctx);
+
+struct rspamd_http_context *rspamd_http_context_default(void);
+
+/**
+ * Returns preserved keepalive connection if it's available.
+ * Refcount is transferred to caller!
+ * @param ctx
+ * @param addr
+ * @param host
+ * @return
+ */
+struct rspamd_http_connection *rspamd_http_context_check_keepalive(struct rspamd_http_context *ctx,
+ const rspamd_inet_addr_t *addr,
+ const gchar *host,
+ bool is_ssl);
+
+/**
+ * Checks if there is a valid keepalive connection
+ * @param ctx
+ * @param addr
+ * @param host
+ * @param is_ssl
+ * @return
+ */
+const rspamd_inet_addr_t *rspamd_http_context_has_keepalive(struct rspamd_http_context *ctx,
+ const gchar *host,
+ unsigned port,
+ bool is_ssl);
+
+/**
+ * Prepares keepalive key for a connection by creating a new entry or by reusing existent
+ * Bear in mind, that keepalive pool has currently no cleanup methods!
+ * @param ctx
+ * @param conn
+ * @param addr
+ * @param host
+ */
+void rspamd_http_context_prepare_keepalive(struct rspamd_http_context *ctx, struct rspamd_http_connection *conn,
+ const rspamd_inet_addr_t *addr, const gchar *host, bool is_ssl);
+
+/**
+ * Pushes a connection to keepalive pool after client request is finished,
+ * keepalive key *must* be prepared before using of this function
+ * @param ctx
+ * @param conn
+ * @param msg
+ */
+void rspamd_http_context_push_keepalive(struct rspamd_http_context *ctx,
+ struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ struct ev_loop *ev_base);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/http/http_message.c b/src/libserver/http/http_message.c
new file mode 100644
index 0000000..670122d
--- /dev/null
+++ b/src/libserver/http/http_message.c
@@ -0,0 +1,725 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "http_message.h"
+#include "http_connection.h"
+#include "http_private.h"
+#include "libutil/printf.h"
+#include "libserver/logger.h"
+#include "utlist.h"
+#include "unix-std.h"
+
+struct rspamd_http_message *
+rspamd_http_new_message(enum rspamd_http_message_type type)
+{
+ struct rspamd_http_message *new;
+
+ new = g_malloc0(sizeof(struct rspamd_http_message));
+
+ if (type == HTTP_REQUEST) {
+ new->url = rspamd_fstring_new();
+ }
+ else {
+ new->url = NULL;
+ new->code = 200;
+ }
+
+ new->port = 80;
+ new->type = type;
+ new->method = HTTP_INVALID;
+ new->headers = kh_init(rspamd_http_headers_hash);
+
+ REF_INIT_RETAIN(new, rspamd_http_message_free);
+
+ return new;
+}
+
+struct rspamd_http_message *
+rspamd_http_message_from_url(const gchar *url)
+{
+ struct http_parser_url pu;
+ struct rspamd_http_message *msg;
+ const gchar *host, *path;
+ size_t pathlen, urllen;
+ guint flags = 0;
+
+ if (url == NULL) {
+ return NULL;
+ }
+
+ urllen = strlen(url);
+ memset(&pu, 0, sizeof(pu));
+
+ if (http_parser_parse_url(url, urllen, FALSE, &pu) != 0) {
+ msg_warn("cannot parse URL: %s", url);
+ return NULL;
+ }
+
+ if ((pu.field_set & (1 << UF_HOST)) == 0) {
+ msg_warn("no host argument in URL: %s", url);
+ return NULL;
+ }
+
+ if ((pu.field_set & (1 << UF_SCHEMA))) {
+ if (pu.field_data[UF_SCHEMA].len == sizeof("https") - 1 &&
+ memcmp(url + pu.field_data[UF_SCHEMA].off, "https", 5) == 0) {
+ flags |= RSPAMD_HTTP_FLAG_WANT_SSL;
+ }
+ }
+
+ if ((pu.field_set & (1 << UF_PATH)) == 0) {
+ path = "/";
+ pathlen = 1;
+ }
+ else {
+ path = url + pu.field_data[UF_PATH].off;
+ pathlen = urllen - pu.field_data[UF_PATH].off;
+ }
+
+ msg = rspamd_http_new_message(HTTP_REQUEST);
+ host = url + pu.field_data[UF_HOST].off;
+ msg->flags = flags;
+
+ if ((pu.field_set & (1 << UF_PORT)) != 0) {
+ msg->port = pu.port;
+ }
+ else {
+ /* XXX: magic constant */
+ if (flags & RSPAMD_HTTP_FLAG_WANT_SSL) {
+ msg->port = 443;
+ }
+ else {
+ msg->port = 80;
+ }
+ }
+
+ msg->host = g_string_new_len(host, pu.field_data[UF_HOST].len);
+ msg->url = rspamd_fstring_append(msg->url, path, pathlen);
+
+ REF_INIT_RETAIN(msg, rspamd_http_message_free);
+
+ return msg;
+}
+
+const gchar *
+rspamd_http_message_get_body(struct rspamd_http_message *msg,
+ gsize *blen)
+{
+ const gchar *ret = NULL;
+
+ if (msg->body_buf.len > 0) {
+ ret = msg->body_buf.begin;
+ }
+
+ if (blen) {
+ *blen = msg->body_buf.len;
+ }
+
+ return ret;
+}
+
+static void
+rspamd_http_shname_dtor(void *p)
+{
+ struct rspamd_storage_shmem *n = p;
+
+#ifdef HAVE_SANE_SHMEM
+ shm_unlink(n->shm_name);
+#else
+ unlink(n->shm_name);
+#endif
+ g_free(n->shm_name);
+ g_free(n);
+}
+
+struct rspamd_storage_shmem *
+rspamd_http_message_shmem_ref(struct rspamd_http_message *msg)
+{
+ if ((msg->flags & RSPAMD_HTTP_FLAG_SHMEM) && msg->body_buf.c.shared.name) {
+ REF_RETAIN(msg->body_buf.c.shared.name);
+ return msg->body_buf.c.shared.name;
+ }
+
+ return NULL;
+}
+
+guint rspamd_http_message_get_flags(struct rspamd_http_message *msg)
+{
+ return msg->flags;
+}
+
+void rspamd_http_message_shmem_unref(struct rspamd_storage_shmem *p)
+{
+ REF_RELEASE(p);
+}
+
+gboolean
+rspamd_http_message_set_body(struct rspamd_http_message *msg,
+ const gchar *data, gsize len)
+{
+ union _rspamd_storage_u *storage;
+ storage = &msg->body_buf.c;
+
+ rspamd_http_message_storage_cleanup(msg);
+
+ if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
+ storage->shared.name = g_malloc(sizeof(*storage->shared.name));
+ REF_INIT_RETAIN(storage->shared.name, rspamd_http_shname_dtor);
+#ifdef HAVE_SANE_SHMEM
+#if defined(__DragonFly__)
+ // DragonFly uses regular files for shm. User rspamd is not allowed to create
+ // files in the root.
+ storage->shared.name->shm_name = g_strdup("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX");
+#else
+ storage->shared.name->shm_name = g_strdup("/rhm.XXXXXXXXXXXXXXXXXXXX");
+#endif
+ storage->shared.shm_fd = rspamd_shmem_mkstemp(storage->shared.name->shm_name);
+#else
+ /* XXX: assume that tempdir is /tmp */
+ storage->shared.name->shm_name = g_strdup("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX");
+ storage->shared.shm_fd = mkstemp(storage->shared.name->shm_name);
+#endif
+
+ if (storage->shared.shm_fd == -1) {
+ return FALSE;
+ }
+
+ if (len != 0 && len != G_MAXSIZE) {
+ if (ftruncate(storage->shared.shm_fd, len) == -1) {
+ return FALSE;
+ }
+
+ msg->body_buf.str = mmap(NULL, len,
+ PROT_WRITE | PROT_READ, MAP_SHARED,
+ storage->shared.shm_fd, 0);
+
+ if (msg->body_buf.str == MAP_FAILED) {
+ return FALSE;
+ }
+
+ msg->body_buf.begin = msg->body_buf.str;
+ msg->body_buf.allocated_len = len;
+
+ if (data != NULL) {
+ memcpy(msg->body_buf.str, data, len);
+ msg->body_buf.len = len;
+ }
+ }
+ else {
+ msg->body_buf.len = 0;
+ msg->body_buf.begin = NULL;
+ msg->body_buf.str = NULL;
+ msg->body_buf.allocated_len = 0;
+ }
+ }
+ else {
+ if (len != 0 && len != G_MAXSIZE) {
+ if (data == NULL) {
+ storage->normal = rspamd_fstring_sized_new(len);
+ msg->body_buf.len = 0;
+ }
+ else {
+ storage->normal = rspamd_fstring_new_init(data, len);
+ msg->body_buf.len = len;
+ }
+ }
+ else {
+ storage->normal = rspamd_fstring_new();
+ }
+
+ msg->body_buf.begin = storage->normal->str;
+ msg->body_buf.str = storage->normal->str;
+ msg->body_buf.allocated_len = storage->normal->allocated;
+ }
+
+ msg->flags |= RSPAMD_HTTP_FLAG_HAS_BODY;
+
+ return TRUE;
+}
+
+void rspamd_http_message_set_method(struct rspamd_http_message *msg,
+ const gchar *method)
+{
+ gint i;
+
+ /* Linear search: not very efficient method */
+ for (i = 0; i < HTTP_METHOD_MAX; i++) {
+ if (g_ascii_strcasecmp(method, http_method_str(i)) == 0) {
+ msg->method = i;
+ }
+ }
+}
+
+gboolean
+rspamd_http_message_set_body_from_fd(struct rspamd_http_message *msg,
+ gint fd)
+{
+ union _rspamd_storage_u *storage;
+ struct stat st;
+
+ rspamd_http_message_storage_cleanup(msg);
+
+ storage = &msg->body_buf.c;
+ msg->flags |= RSPAMD_HTTP_FLAG_SHMEM | RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE;
+
+ storage->shared.shm_fd = dup(fd);
+ msg->body_buf.str = MAP_FAILED;
+
+ if (storage->shared.shm_fd == -1) {
+ return FALSE;
+ }
+
+ if (fstat(storage->shared.shm_fd, &st) == -1) {
+ return FALSE;
+ }
+
+ msg->body_buf.str = mmap(NULL, st.st_size,
+ PROT_READ, MAP_SHARED,
+ storage->shared.shm_fd, 0);
+
+ if (msg->body_buf.str == MAP_FAILED) {
+ return FALSE;
+ }
+
+ msg->body_buf.begin = msg->body_buf.str;
+ msg->body_buf.len = st.st_size;
+ msg->body_buf.allocated_len = st.st_size;
+
+ return TRUE;
+}
+
+gboolean
+rspamd_http_message_set_body_from_fstring_steal(struct rspamd_http_message *msg,
+ rspamd_fstring_t *fstr)
+{
+ union _rspamd_storage_u *storage;
+
+ rspamd_http_message_storage_cleanup(msg);
+
+ storage = &msg->body_buf.c;
+ msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM | RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE);
+
+ storage->normal = fstr;
+ msg->body_buf.str = fstr->str;
+ msg->body_buf.begin = msg->body_buf.str;
+ msg->body_buf.len = fstr->len;
+ msg->body_buf.allocated_len = fstr->allocated;
+
+ return TRUE;
+}
+
+gboolean
+rspamd_http_message_set_body_from_fstring_copy(struct rspamd_http_message *msg,
+ const rspamd_fstring_t *fstr)
+{
+ union _rspamd_storage_u *storage;
+
+ rspamd_http_message_storage_cleanup(msg);
+
+ storage = &msg->body_buf.c;
+ msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM | RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE);
+
+ storage->normal = rspamd_fstring_new_init(fstr->str, fstr->len);
+ msg->body_buf.str = storage->normal->str;
+ msg->body_buf.begin = msg->body_buf.str;
+ msg->body_buf.len = storage->normal->len;
+ msg->body_buf.allocated_len = storage->normal->allocated;
+
+ return TRUE;
+}
+
+
+gboolean
+rspamd_http_message_grow_body(struct rspamd_http_message *msg, gsize len)
+{
+ struct stat st;
+ union _rspamd_storage_u *storage;
+ gsize newlen;
+
+ storage = &msg->body_buf.c;
+
+ if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
+ if (storage->shared.shm_fd == -1) {
+ return FALSE;
+ }
+
+ if (fstat(storage->shared.shm_fd, &st) == -1) {
+ return FALSE;
+ }
+
+ /* Check if we need to grow */
+ if ((gsize) st.st_size < msg->body_buf.len + len) {
+ /* Need to grow */
+ newlen = rspamd_fstring_suggest_size(msg->body_buf.len, st.st_size,
+ len);
+ /* Unmap as we need another size of segment */
+ if (msg->body_buf.str != MAP_FAILED) {
+ munmap(msg->body_buf.str, st.st_size);
+ }
+
+ if (ftruncate(storage->shared.shm_fd, newlen) == -1) {
+ return FALSE;
+ }
+
+ msg->body_buf.str = mmap(NULL, newlen,
+ PROT_WRITE | PROT_READ, MAP_SHARED,
+ storage->shared.shm_fd, 0);
+ if (msg->body_buf.str == MAP_FAILED) {
+ return FALSE;
+ }
+
+ msg->body_buf.begin = msg->body_buf.str;
+ msg->body_buf.allocated_len = newlen;
+ }
+ }
+ else {
+ storage->normal = rspamd_fstring_grow(storage->normal, len);
+
+ /* Append might cause realloc */
+ msg->body_buf.begin = storage->normal->str;
+ msg->body_buf.len = storage->normal->len;
+ msg->body_buf.str = storage->normal->str;
+ msg->body_buf.allocated_len = storage->normal->allocated;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_http_message_append_body(struct rspamd_http_message *msg,
+ const gchar *data, gsize len)
+{
+ union _rspamd_storage_u *storage;
+
+ storage = &msg->body_buf.c;
+
+ if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
+ if (!rspamd_http_message_grow_body(msg, len)) {
+ return FALSE;
+ }
+
+ memcpy(msg->body_buf.str + msg->body_buf.len, data, len);
+ msg->body_buf.len += len;
+ }
+ else {
+ storage->normal = rspamd_fstring_append(storage->normal, data, len);
+
+ /* Append might cause realloc */
+ msg->body_buf.begin = storage->normal->str;
+ msg->body_buf.len = storage->normal->len;
+ msg->body_buf.str = storage->normal->str;
+ msg->body_buf.allocated_len = storage->normal->allocated;
+ }
+
+ return TRUE;
+}
+
+void rspamd_http_message_storage_cleanup(struct rspamd_http_message *msg)
+{
+ union _rspamd_storage_u *storage;
+ struct stat st;
+
+ if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
+ storage = &msg->body_buf.c;
+
+ if (storage->shared.shm_fd > 0) {
+ g_assert(fstat(storage->shared.shm_fd, &st) != -1);
+
+ if (msg->body_buf.str != MAP_FAILED) {
+ munmap(msg->body_buf.str, st.st_size);
+ }
+
+ close(storage->shared.shm_fd);
+ }
+
+ if (storage->shared.name != NULL) {
+ REF_RELEASE(storage->shared.name);
+ }
+
+ storage->shared.shm_fd = -1;
+ msg->body_buf.str = MAP_FAILED;
+ }
+ else {
+ if (msg->body_buf.c.normal) {
+ rspamd_fstring_free(msg->body_buf.c.normal);
+ }
+
+ msg->body_buf.c.normal = NULL;
+ }
+
+ msg->body_buf.len = 0;
+}
+
+void rspamd_http_message_free(struct rspamd_http_message *msg)
+{
+ struct rspamd_http_header *hdr, *hcur, *hcurtmp;
+
+ kh_foreach_value (msg->headers, hdr, {
+ DL_FOREACH_SAFE (hdr, hcur, hcurtmp) {
+ rspamd_fstring_free (hcur->combined);
+ g_free (hcur);
+}
+});
+
+kh_destroy(rspamd_http_headers_hash, msg->headers);
+rspamd_http_message_storage_cleanup(msg);
+
+if (msg->url != NULL) {
+ rspamd_fstring_free(msg->url);
+}
+if (msg->status != NULL) {
+ rspamd_fstring_free(msg->status);
+}
+if (msg->host != NULL) {
+ g_string_free(msg->host, TRUE);
+}
+if (msg->peer_key != NULL) {
+ rspamd_pubkey_unref(msg->peer_key);
+}
+
+g_free(msg);
+}
+
+void rspamd_http_message_set_peer_key(struct rspamd_http_message *msg,
+ struct rspamd_cryptobox_pubkey *pk)
+{
+ if (msg->peer_key != NULL) {
+ rspamd_pubkey_unref(msg->peer_key);
+ }
+
+ if (pk) {
+ msg->peer_key = rspamd_pubkey_ref(pk);
+ }
+ else {
+ msg->peer_key = NULL;
+ }
+}
+
+void rspamd_http_message_add_header_len(struct rspamd_http_message *msg,
+ const gchar *name,
+ const gchar *value,
+ gsize len)
+{
+ struct rspamd_http_header *hdr, *found;
+ guint nlen, vlen;
+ khiter_t k;
+ gint r;
+
+ if (msg != NULL && name != NULL && value != NULL) {
+ hdr = g_malloc0(sizeof(struct rspamd_http_header));
+ nlen = strlen(name);
+ vlen = len;
+
+ if (g_ascii_strcasecmp(name, "host") == 0) {
+ msg->flags |= RSPAMD_HTTP_FLAG_HAS_HOST_HEADER;
+ }
+
+ hdr->combined = rspamd_fstring_sized_new(nlen + vlen + 4);
+ rspamd_printf_fstring(&hdr->combined, "%s: %*s\r\n", name, (gint) vlen,
+ value);
+ hdr->name.begin = hdr->combined->str;
+ hdr->name.len = nlen;
+ hdr->value.begin = hdr->combined->str + nlen + 2;
+ hdr->value.len = vlen;
+
+ k = kh_put(rspamd_http_headers_hash, msg->headers, &hdr->name,
+ &r);
+
+ if (r != 0) {
+ kh_value(msg->headers, k) = hdr;
+ found = NULL;
+ }
+ else {
+ found = kh_value(msg->headers, k);
+ }
+
+ DL_APPEND(found, hdr);
+ }
+}
+
+void rspamd_http_message_add_header(struct rspamd_http_message *msg,
+ const gchar *name,
+ const gchar *value)
+{
+ if (value) {
+ rspamd_http_message_add_header_len(msg, name, value, strlen(value));
+ }
+}
+
+void rspamd_http_message_add_header_fstr(struct rspamd_http_message *msg,
+ const gchar *name,
+ rspamd_fstring_t *value)
+{
+ struct rspamd_http_header *hdr, *found = NULL;
+ guint nlen, vlen;
+ khiter_t k;
+ gint r;
+
+ if (msg != NULL && name != NULL && value != NULL) {
+ hdr = g_malloc0(sizeof(struct rspamd_http_header));
+ nlen = strlen(name);
+ vlen = value->len;
+ hdr->combined = rspamd_fstring_sized_new(nlen + vlen + 4);
+ rspamd_printf_fstring(&hdr->combined, "%s: %V\r\n", name, value);
+ hdr->name.begin = hdr->combined->str;
+ hdr->name.len = nlen;
+ hdr->value.begin = hdr->combined->str + nlen + 2;
+ hdr->value.len = vlen;
+
+ k = kh_put(rspamd_http_headers_hash, msg->headers, &hdr->name,
+ &r);
+
+ if (r != 0) {
+ kh_value(msg->headers, k) = hdr;
+ found = NULL;
+ }
+ else {
+ found = kh_value(msg->headers, k);
+ }
+
+ DL_APPEND(found, hdr);
+ }
+}
+
+const rspamd_ftok_t *
+rspamd_http_message_find_header(struct rspamd_http_message *msg,
+ const gchar *name)
+{
+ const rspamd_ftok_t *res = NULL;
+ rspamd_ftok_t srch;
+ guint slen = strlen(name);
+ khiter_t k;
+
+ if (msg != NULL) {
+ srch.begin = name;
+ srch.len = slen;
+
+ k = kh_get(rspamd_http_headers_hash, msg->headers, &srch);
+
+ if (k != kh_end(msg->headers)) {
+ res = &(kh_value(msg->headers, k)->value);
+ }
+ }
+
+ return res;
+}
+
+GPtrArray *
+rspamd_http_message_find_header_multiple(
+ struct rspamd_http_message *msg,
+ const gchar *name)
+{
+ GPtrArray *res = NULL;
+ struct rspamd_http_header *hdr, *cur;
+ rspamd_ftok_t srch;
+ khiter_t k;
+ guint cnt = 0;
+
+ guint slen = strlen(name);
+
+ if (msg != NULL) {
+ srch.begin = name;
+ srch.len = slen;
+
+ k = kh_get(rspamd_http_headers_hash, msg->headers, &srch);
+
+ if (k != kh_end(msg->headers)) {
+ hdr = kh_value(msg->headers, k);
+
+ LL_COUNT(hdr, cur, cnt);
+ res = g_ptr_array_sized_new(cnt);
+
+ LL_FOREACH(hdr, cur)
+ {
+ g_ptr_array_add(res, &cur->value);
+ }
+ }
+ }
+
+
+ return res;
+}
+
+
+gboolean
+rspamd_http_message_remove_header(struct rspamd_http_message *msg,
+ const gchar *name)
+{
+ struct rspamd_http_header *hdr, *hcur, *hcurtmp;
+ gboolean res = FALSE;
+ guint slen = strlen(name);
+ rspamd_ftok_t srch;
+ khiter_t k;
+
+ if (msg != NULL) {
+ srch.begin = name;
+ srch.len = slen;
+
+ k = kh_get(rspamd_http_headers_hash, msg->headers, &srch);
+
+ if (k != kh_end(msg->headers)) {
+ hdr = kh_value(msg->headers, k);
+ kh_del(rspamd_http_headers_hash, msg->headers, k);
+ res = TRUE;
+
+ DL_FOREACH_SAFE(hdr, hcur, hcurtmp)
+ {
+ rspamd_fstring_free(hcur->combined);
+ g_free(hcur);
+ }
+ }
+ }
+
+ return res;
+}
+
+const gchar *
+rspamd_http_message_get_http_host(struct rspamd_http_message *msg,
+ gsize *hostlen)
+{
+ if (msg->flags & RSPAMD_HTTP_FLAG_HAS_HOST_HEADER) {
+ rspamd_ftok_t srch;
+
+ RSPAMD_FTOK_ASSIGN(&srch, "Host");
+
+ khiter_t k = kh_get(rspamd_http_headers_hash, msg->headers, &srch);
+
+ if (k != kh_end(msg->headers)) {
+ *hostlen = (kh_value(msg->headers, k)->value).len;
+ return (kh_value(msg->headers, k)->value).begin;
+ }
+ else if (msg->host) {
+ *hostlen = msg->host->len;
+ return msg->host->str;
+ }
+ }
+ else {
+ if (msg->host) {
+ *hostlen = msg->host->len;
+ return msg->host->str;
+ }
+ }
+
+ return NULL;
+}
+
+bool rspamd_http_message_is_standard_port(struct rspamd_http_message *msg)
+{
+ if (msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL) {
+ return msg->port == 443;
+ }
+
+ return msg->port == 80;
+} \ No newline at end of file
diff --git a/src/libserver/http/http_message.h b/src/libserver/http/http_message.h
new file mode 100644
index 0000000..fa8ed04
--- /dev/null
+++ b/src/libserver/http/http_message.h
@@ -0,0 +1,254 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_HTTP_MESSAGE_H
+#define RSPAMD_HTTP_MESSAGE_H
+
+#include "config.h"
+#include "keypair.h"
+#include "keypairs_cache.h"
+#include "fstring.h"
+#include "ref.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_http_connection;
+
+enum rspamd_http_message_type {
+ HTTP_REQUEST = 0,
+ HTTP_RESPONSE
+};
+
+/**
+ * Extract the current message from a connection to deal with separately
+ * @param conn
+ * @return
+ */
+struct rspamd_http_message *rspamd_http_connection_steal_msg(
+ struct rspamd_http_connection *conn);
+
+/**
+ * Copy the current message from a connection to deal with separately
+ * @param conn
+ * @return
+ */
+struct rspamd_http_message *rspamd_http_connection_copy_msg(
+ struct rspamd_http_message *msg, GError **err);
+
+/**
+ * Create new HTTP message
+ * @param type request or response
+ * @return new http message
+ */
+struct rspamd_http_message *rspamd_http_new_message(enum rspamd_http_message_type type);
+
+/**
+ * Increase refcount number for an HTTP message
+ * @param msg message to use
+ * @return
+ */
+struct rspamd_http_message *rspamd_http_message_ref(struct rspamd_http_message *msg);
+
+/**
+ * Decrease number of refcounts for http message
+ * @param msg
+ */
+void rspamd_http_message_unref(struct rspamd_http_message *msg);
+
+/**
+ * Sets a key for peer
+ * @param msg
+ * @param pk
+ */
+void rspamd_http_message_set_peer_key(struct rspamd_http_message *msg,
+ struct rspamd_cryptobox_pubkey *pk);
+
+/**
+ * Create HTTP message from URL
+ * @param url
+ * @return new message or NULL
+ */
+struct rspamd_http_message *rspamd_http_message_from_url(const gchar *url);
+
+/**
+ * Returns body for a message
+ * @param msg
+ * @param blen pointer where to save body length
+ * @return pointer to body start
+ */
+const gchar *rspamd_http_message_get_body(struct rspamd_http_message *msg,
+ gsize *blen);
+
+/**
+ * Set message's body from the string
+ * @param msg
+ * @param data
+ * @param len
+ * @return TRUE if a message's body has been set
+ */
+gboolean rspamd_http_message_set_body(struct rspamd_http_message *msg,
+ const gchar *data, gsize len);
+
+/**
+ * Set message's method by name
+ * @param msg
+ * @param method
+ */
+void rspamd_http_message_set_method(struct rspamd_http_message *msg,
+ const gchar *method);
+
+/**
+ * Maps fd as message's body
+ * @param msg
+ * @param fd
+ * @return TRUE if a message's body has been set
+ */
+gboolean rspamd_http_message_set_body_from_fd(struct rspamd_http_message *msg,
+ gint fd);
+
+/**
+ * Uses rspamd_fstring_t as message's body, string is consumed by this operation
+ * @param msg
+ * @param fstr
+ * @return TRUE if a message's body has been set
+ */
+gboolean rspamd_http_message_set_body_from_fstring_steal(struct rspamd_http_message *msg,
+ rspamd_fstring_t *fstr);
+
+/**
+ * Uses rspamd_fstring_t as message's body, string is copied by this operation
+ * @param msg
+ * @param fstr
+ * @return TRUE if a message's body has been set
+ */
+gboolean rspamd_http_message_set_body_from_fstring_copy(struct rspamd_http_message *msg,
+ const rspamd_fstring_t *fstr);
+
+/**
+ * Appends data to message's body
+ * @param msg
+ * @param data
+ * @param len
+ * @return TRUE if a message's body has been set
+ */
+gboolean rspamd_http_message_append_body(struct rspamd_http_message *msg,
+ const gchar *data, gsize len);
+
+/**
+ * Append a header to http message
+ * @param rep
+ * @param name
+ * @param value
+ */
+void rspamd_http_message_add_header(struct rspamd_http_message *msg,
+ const gchar *name,
+ const gchar *value);
+
+void rspamd_http_message_add_header_len(struct rspamd_http_message *msg,
+ const gchar *name,
+ const gchar *value,
+ gsize len);
+
+void rspamd_http_message_add_header_fstr(struct rspamd_http_message *msg,
+ const gchar *name,
+ rspamd_fstring_t *value);
+
+/**
+ * Search for a specified header in message
+ * @param msg message
+ * @param name name of header
+ */
+const rspamd_ftok_t *rspamd_http_message_find_header(
+ struct rspamd_http_message *msg,
+ const gchar *name);
+
+/**
+ * Search for a header that has multiple values
+ * @param msg
+ * @param name
+ * @return list of rspamd_ftok_t * with values
+ */
+GPtrArray *rspamd_http_message_find_header_multiple(
+ struct rspamd_http_message *msg,
+ const gchar *name);
+
+/**
+ * Remove specific header from a message
+ * @param msg
+ * @param name
+ * @return
+ */
+gboolean rspamd_http_message_remove_header(struct rspamd_http_message *msg,
+ const gchar *name);
+
+/**
+ * Free HTTP message
+ * @param msg
+ */
+void rspamd_http_message_free(struct rspamd_http_message *msg);
+
+/**
+ * Extract arguments from a message's URI contained inside query string decoding
+ * them if needed
+ * @param msg HTTP request message
+ * @return new GHashTable which maps rspamd_ftok_t* to rspamd_ftok_t*
+ * (table must be freed by a caller)
+ */
+GHashTable *rspamd_http_message_parse_query(struct rspamd_http_message *msg);
+
+/**
+ * Increase refcount for shared file (if any) to prevent early memory unlinking
+ * @param msg
+ */
+struct rspamd_storage_shmem *rspamd_http_message_shmem_ref(struct rspamd_http_message *msg);
+
+/**
+ * Decrease external ref for shmem segment associated with a message
+ * @param msg
+ */
+void rspamd_http_message_shmem_unref(struct rspamd_storage_shmem *p);
+
+/**
+ * Returns message's flags
+ * @param msg
+ * @return
+ */
+guint rspamd_http_message_get_flags(struct rspamd_http_message *msg);
+
+/**
+ * Returns an HTTP hostname for a message, derived from a header if it has it
+ * or from a url if it doesn't
+ * @param msg
+ * @param hostlen output of the host length
+ * @return
+ */
+const gchar *rspamd_http_message_get_http_host(struct rspamd_http_message *msg,
+ gsize *hostlen);
+
+/**
+ * Returns true if a message has standard port (80 or 443 for https)
+ * @param msg
+ * @return
+ */
+bool rspamd_http_message_is_standard_port(struct rspamd_http_message *msg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/http/http_private.h b/src/libserver/http/http_private.h
new file mode 100644
index 0000000..096545e
--- /dev/null
+++ b/src/libserver/http/http_private.h
@@ -0,0 +1,129 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBUTIL_HTTP_PRIVATE_H_
+#define SRC_LIBUTIL_HTTP_PRIVATE_H_
+
+#include "http_connection.h"
+#include "http_parser.h"
+#include "str_util.h"
+#include "keypair.h"
+#include "keypairs_cache.h"
+#include "ref.h"
+#include "upstream.h"
+#include "khash.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * HTTP header structure
+ */
+struct rspamd_http_header {
+ rspamd_fstring_t *combined;
+ rspamd_ftok_t name;
+ rspamd_ftok_t value;
+ struct rspamd_http_header *prev, *next;
+};
+
+KHASH_INIT(rspamd_http_headers_hash, rspamd_ftok_t *,
+ struct rspamd_http_header *, 1,
+ rspamd_ftok_icase_hash, rspamd_ftok_icase_equal);
+
+/**
+ * HTTP message structure, used for requests and replies
+ */
+struct rspamd_http_message {
+ rspamd_fstring_t *url;
+ GString *host;
+ rspamd_fstring_t *status;
+ khash_t(rspamd_http_headers_hash) * headers;
+
+ struct _rspamd_body_buf_s {
+ /* Data start */
+ const gchar *begin;
+ /* Data len */
+ gsize len;
+ /* Allocated len */
+ gsize allocated_len;
+ /* Data buffer (used to write data inside) */
+ gchar *str;
+
+ /* Internal storage */
+ union _rspamd_storage_u {
+ rspamd_fstring_t *normal;
+ struct _rspamd_storage_shared_s {
+ struct rspamd_storage_shmem *name;
+ gint shm_fd;
+ } shared;
+ } c;
+ } body_buf;
+
+ struct rspamd_cryptobox_pubkey *peer_key;
+ time_t date;
+ time_t last_modified;
+ unsigned port;
+ int type;
+ gint code;
+ enum http_method method;
+ gint flags;
+ ref_entry_t ref;
+};
+
+struct rspamd_keepalive_hash_key {
+ rspamd_inet_addr_t *addr;
+ gchar *host;
+ gboolean is_ssl;
+ unsigned port;
+ GQueue conns;
+};
+
+gint32 rspamd_keep_alive_key_hash(struct rspamd_keepalive_hash_key *k);
+
+bool rspamd_keep_alive_key_equal(struct rspamd_keepalive_hash_key *k1,
+ struct rspamd_keepalive_hash_key *k2);
+
+KHASH_INIT(rspamd_keep_alive_hash, struct rspamd_keepalive_hash_key *,
+ char, 0, rspamd_keep_alive_key_hash, rspamd_keep_alive_key_equal);
+
+struct rspamd_http_context {
+ struct rspamd_http_context_cfg config;
+ struct rspamd_keypair_cache *client_kp_cache;
+ struct rspamd_cryptobox_keypair *client_kp;
+ struct rspamd_keypair_cache *server_kp_cache;
+ struct upstream_ctx *ups_ctx;
+ struct upstream_list *http_proxies;
+ gpointer ssl_ctx;
+ gpointer ssl_ctx_noverify;
+ struct ev_loop *event_loop;
+ ev_timer client_rotate_ev;
+ khash_t(rspamd_keep_alive_hash) * keep_alive_hash;
+};
+
+#define HTTP_ERROR http_error_quark()
+
+GQuark http_error_quark(void);
+
+void rspamd_http_message_storage_cleanup(struct rspamd_http_message *msg);
+
+gboolean rspamd_http_message_grow_body(struct rspamd_http_message *msg,
+ gsize len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBUTIL_HTTP_PRIVATE_H_ */
diff --git a/src/libserver/http/http_router.c b/src/libserver/http/http_router.c
new file mode 100644
index 0000000..2fdfe48
--- /dev/null
+++ b/src/libserver/http/http_router.c
@@ -0,0 +1,559 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "http_router.h"
+#include "http_connection.h"
+#include "http_private.h"
+#include "libutil/regexp.h"
+#include "libutil/printf.h"
+#include "libserver/logger.h"
+#include "utlist.h"
+#include "unix-std.h"
+
+enum http_magic_type {
+ HTTP_MAGIC_PLAIN = 0,
+ HTTP_MAGIC_HTML,
+ HTTP_MAGIC_CSS,
+ HTTP_MAGIC_JS,
+ HTTP_MAGIC_ICO,
+ HTTP_MAGIC_PNG,
+ HTTP_MAGIC_JPG
+};
+
+static const struct _rspamd_http_magic {
+ const gchar *ext;
+ const gchar *ct;
+} http_file_types[] = {
+ [HTTP_MAGIC_PLAIN] = {"txt", "text/plain"},
+ [HTTP_MAGIC_HTML] = {"html", "text/html"},
+ [HTTP_MAGIC_CSS] = {"css", "text/css"},
+ [HTTP_MAGIC_JS] = {"js", "application/javascript"},
+ [HTTP_MAGIC_ICO] = {"ico", "image/x-icon"},
+ [HTTP_MAGIC_PNG] = {"png", "image/png"},
+ [HTTP_MAGIC_JPG] = {"jpg", "image/jpeg"},
+};
+
+/*
+ * HTTP router functions
+ */
+
+static void
+rspamd_http_entry_free(struct rspamd_http_connection_entry *entry)
+{
+ if (entry != NULL) {
+ close(entry->conn->fd);
+ rspamd_http_connection_unref(entry->conn);
+ if (entry->rt->finish_handler) {
+ entry->rt->finish_handler(entry);
+ }
+
+ DL_DELETE(entry->rt->conns, entry);
+ g_free(entry);
+ }
+}
+
+static void
+rspamd_http_router_error_handler(struct rspamd_http_connection *conn,
+ GError *err)
+{
+ struct rspamd_http_connection_entry *entry = conn->ud;
+ struct rspamd_http_message *msg;
+
+ if (entry->is_reply) {
+ /* At this point we need to finish this session and close owned socket */
+ if (entry->rt->error_handler != NULL) {
+ entry->rt->error_handler(entry, err);
+ }
+ rspamd_http_entry_free(entry);
+ }
+ else {
+ /* Here we can write a reply to a client */
+ if (entry->rt->error_handler != NULL) {
+ entry->rt->error_handler(entry, err);
+ }
+ msg = rspamd_http_new_message(HTTP_RESPONSE);
+ msg->date = time(NULL);
+ msg->code = err->code;
+ rspamd_http_message_set_body(msg, err->message, strlen(err->message));
+ rspamd_http_connection_reset(entry->conn);
+ rspamd_http_connection_write_message(entry->conn,
+ msg,
+ NULL,
+ "text/plain",
+ entry,
+ entry->rt->timeout);
+ entry->is_reply = TRUE;
+ }
+}
+
+static const gchar *
+rspamd_http_router_detect_ct(const gchar *path)
+{
+ const gchar *dot;
+ guint i;
+
+ dot = strrchr(path, '.');
+ if (dot == NULL) {
+ return http_file_types[HTTP_MAGIC_PLAIN].ct;
+ }
+ dot++;
+
+ for (i = 0; i < G_N_ELEMENTS(http_file_types); i++) {
+ if (strcmp(http_file_types[i].ext, dot) == 0) {
+ return http_file_types[i].ct;
+ }
+ }
+
+ return http_file_types[HTTP_MAGIC_PLAIN].ct;
+}
+
+static gboolean
+rspamd_http_router_is_subdir(const gchar *parent, const gchar *sub)
+{
+ if (parent == NULL || sub == NULL || *parent == '\0') {
+ return FALSE;
+ }
+
+ while (*parent != '\0') {
+ if (*sub != *parent) {
+ return FALSE;
+ }
+ parent++;
+ sub++;
+ }
+
+ parent--;
+ if (*parent == G_DIR_SEPARATOR) {
+ return TRUE;
+ }
+
+ return (*sub == G_DIR_SEPARATOR || *sub == '\0');
+}
+
+static gboolean
+rspamd_http_router_try_file(struct rspamd_http_connection_entry *entry,
+ rspamd_ftok_t *lookup, gboolean expand_path)
+{
+ struct stat st;
+ gint fd;
+ gchar filebuf[PATH_MAX], realbuf[PATH_MAX], *dir;
+ struct rspamd_http_message *reply_msg;
+
+ rspamd_snprintf(filebuf, sizeof(filebuf), "%s%c%T",
+ entry->rt->default_fs_path, G_DIR_SEPARATOR, lookup);
+
+ if (realpath(filebuf, realbuf) == NULL ||
+ lstat(realbuf, &st) == -1) {
+ return FALSE;
+ }
+
+ if (S_ISDIR(st.st_mode) && expand_path) {
+ /* Try to append 'index.html' to the url */
+ rspamd_fstring_t *nlookup;
+ rspamd_ftok_t tok;
+ gboolean ret;
+
+ nlookup = rspamd_fstring_sized_new(lookup->len + sizeof("index.html"));
+ rspamd_printf_fstring(&nlookup, "%T%c%s", lookup, G_DIR_SEPARATOR,
+ "index.html");
+ tok.begin = nlookup->str;
+ tok.len = nlookup->len;
+ ret = rspamd_http_router_try_file(entry, &tok, FALSE);
+ rspamd_fstring_free(nlookup);
+
+ return ret;
+ }
+ else if (!S_ISREG(st.st_mode)) {
+ return FALSE;
+ }
+
+ /* We also need to ensure that file is inside the defined dir */
+ rspamd_strlcpy(filebuf, realbuf, sizeof(filebuf));
+ dir = dirname(filebuf);
+
+ if (dir == NULL ||
+ !rspamd_http_router_is_subdir(entry->rt->default_fs_path,
+ dir)) {
+ return FALSE;
+ }
+
+ fd = open(realbuf, O_RDONLY);
+ if (fd == -1) {
+ return FALSE;
+ }
+
+ reply_msg = rspamd_http_new_message(HTTP_RESPONSE);
+ reply_msg->date = time(NULL);
+ reply_msg->code = 200;
+ rspamd_http_router_insert_headers(entry->rt, reply_msg);
+
+ if (!rspamd_http_message_set_body_from_fd(reply_msg, fd)) {
+ rspamd_http_message_free(reply_msg);
+ close(fd);
+ return FALSE;
+ }
+
+ close(fd);
+
+ rspamd_http_connection_reset(entry->conn);
+
+ msg_debug("requested file %s", realbuf);
+ rspamd_http_connection_write_message(entry->conn, reply_msg, NULL,
+ rspamd_http_router_detect_ct(realbuf), entry,
+ entry->rt->timeout);
+
+ return TRUE;
+}
+
+static void
+rspamd_http_router_send_error(GError *err,
+ struct rspamd_http_connection_entry *entry)
+{
+ struct rspamd_http_message *err_msg;
+
+ err_msg = rspamd_http_new_message(HTTP_RESPONSE);
+ err_msg->date = time(NULL);
+ err_msg->code = err->code;
+ rspamd_http_message_set_body(err_msg, err->message,
+ strlen(err->message));
+ entry->is_reply = TRUE;
+ err_msg->status = rspamd_fstring_new_init(err->message, strlen(err->message));
+ rspamd_http_router_insert_headers(entry->rt, err_msg);
+ rspamd_http_connection_reset(entry->conn);
+ rspamd_http_connection_write_message(entry->conn,
+ err_msg,
+ NULL,
+ "text/plain",
+ entry,
+ entry->rt->timeout);
+}
+
+
+static int
+rspamd_http_router_finish_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_http_connection_entry *entry = conn->ud;
+ rspamd_http_router_handler_t handler = NULL;
+ gpointer found;
+
+ GError *err;
+ rspamd_ftok_t lookup;
+ const rspamd_ftok_t *encoding;
+ struct http_parser_url u;
+ guint i;
+ rspamd_regexp_t *re;
+ struct rspamd_http_connection_router *router;
+ gchar *pathbuf = NULL;
+
+ G_STATIC_ASSERT(sizeof(rspamd_http_router_handler_t) ==
+ sizeof(gpointer));
+
+ memset(&lookup, 0, sizeof(lookup));
+ router = entry->rt;
+
+ if (entry->is_reply) {
+ /* Request is finished, it is safe to free a connection */
+ rspamd_http_entry_free(entry);
+ }
+ else {
+ if (G_UNLIKELY(msg->method != HTTP_GET && msg->method != HTTP_POST)) {
+ if (router->unknown_method_handler) {
+ return router->unknown_method_handler(entry, msg);
+ }
+ else {
+ err = g_error_new(HTTP_ERROR, 500,
+ "Invalid method");
+ if (entry->rt->error_handler != NULL) {
+ entry->rt->error_handler(entry, err);
+ }
+
+ rspamd_http_router_send_error(err, entry);
+ g_error_free(err);
+
+ return 0;
+ }
+ }
+
+ /* Search for path */
+ if (msg->url != NULL && msg->url->len != 0) {
+
+ http_parser_parse_url(msg->url->str, msg->url->len, TRUE, &u);
+
+ if (u.field_set & (1 << UF_PATH)) {
+ gsize unnorm_len;
+
+ pathbuf = g_malloc(u.field_data[UF_PATH].len);
+ memcpy(pathbuf, msg->url->str + u.field_data[UF_PATH].off,
+ u.field_data[UF_PATH].len);
+ lookup.begin = pathbuf;
+ lookup.len = u.field_data[UF_PATH].len;
+
+ rspamd_normalize_path_inplace(pathbuf,
+ lookup.len,
+ &unnorm_len);
+ lookup.len = unnorm_len;
+ }
+ else {
+ lookup.begin = msg->url->str;
+ lookup.len = msg->url->len;
+ }
+
+ found = g_hash_table_lookup(entry->rt->paths, &lookup);
+ memcpy(&handler, &found, sizeof(found));
+ msg_debug("requested known path: %T", &lookup);
+ }
+ else {
+ err = g_error_new(HTTP_ERROR, 404,
+ "Empty path requested");
+ if (entry->rt->error_handler != NULL) {
+ entry->rt->error_handler(entry, err);
+ }
+
+ rspamd_http_router_send_error(err, entry);
+ g_error_free(err);
+
+ return 0;
+ }
+
+ entry->is_reply = TRUE;
+
+ encoding = rspamd_http_message_find_header(msg, "Accept-Encoding");
+
+ if (encoding && rspamd_substring_search(encoding->begin, encoding->len,
+ "gzip", 4) != -1) {
+ entry->support_gzip = TRUE;
+ }
+
+ if (handler != NULL) {
+ if (pathbuf) {
+ g_free(pathbuf);
+ }
+
+ return handler(entry, msg);
+ }
+ else {
+ /* Try regexps */
+ for (i = 0; i < router->regexps->len; i++) {
+ re = g_ptr_array_index(router->regexps, i);
+ if (rspamd_regexp_match(re, lookup.begin, lookup.len,
+ TRUE)) {
+ found = rspamd_regexp_get_ud(re);
+ memcpy(&handler, &found, sizeof(found));
+
+ if (pathbuf) {
+ g_free(pathbuf);
+ }
+
+ return handler(entry, msg);
+ }
+ }
+
+ /* Now try plain file */
+ if (entry->rt->default_fs_path == NULL || lookup.len == 0 ||
+ !rspamd_http_router_try_file(entry, &lookup, TRUE)) {
+
+ err = g_error_new(HTTP_ERROR, 404,
+ "Not found");
+ if (entry->rt->error_handler != NULL) {
+ entry->rt->error_handler(entry, err);
+ }
+
+ msg_info("path: %T not found", &lookup);
+ rspamd_http_router_send_error(err, entry);
+ g_error_free(err);
+ }
+ }
+ }
+
+ if (pathbuf) {
+ g_free(pathbuf);
+ }
+
+ return 0;
+}
+
+struct rspamd_http_connection_router *
+rspamd_http_router_new(rspamd_http_router_error_handler_t eh,
+ rspamd_http_router_finish_handler_t fh,
+ ev_tstamp timeout,
+ const char *default_fs_path,
+ struct rspamd_http_context *ctx)
+{
+ struct rspamd_http_connection_router *nrouter;
+ struct stat st;
+
+ nrouter = g_malloc0(sizeof(struct rspamd_http_connection_router));
+ nrouter->paths = g_hash_table_new_full(rspamd_ftok_icase_hash,
+ rspamd_ftok_icase_equal, rspamd_fstring_mapped_ftok_free, NULL);
+ nrouter->regexps = g_ptr_array_new();
+ nrouter->conns = NULL;
+ nrouter->error_handler = eh;
+ nrouter->finish_handler = fh;
+ nrouter->response_headers = g_hash_table_new_full(rspamd_strcase_hash,
+ rspamd_strcase_equal, g_free, g_free);
+ nrouter->event_loop = ctx->event_loop;
+ nrouter->timeout = timeout;
+ nrouter->default_fs_path = NULL;
+
+ if (default_fs_path != NULL) {
+ if (stat(default_fs_path, &st) == -1) {
+ msg_err("cannot stat %s", default_fs_path);
+ }
+ else {
+ if (!S_ISDIR(st.st_mode)) {
+ msg_err("path %s is not a directory", default_fs_path);
+ }
+ else {
+ nrouter->default_fs_path = realpath(default_fs_path, NULL);
+ }
+ }
+ }
+
+ nrouter->ctx = ctx;
+
+ return nrouter;
+}
+
+void rspamd_http_router_set_key(struct rspamd_http_connection_router *router,
+ struct rspamd_cryptobox_keypair *key)
+{
+ g_assert(key != NULL);
+
+ router->key = rspamd_keypair_ref(key);
+}
+
+void rspamd_http_router_add_path(struct rspamd_http_connection_router *router,
+ const gchar *path, rspamd_http_router_handler_t handler)
+{
+ gpointer ptr;
+ rspamd_ftok_t *key;
+ rspamd_fstring_t *storage;
+ G_STATIC_ASSERT(sizeof(rspamd_http_router_handler_t) ==
+ sizeof(gpointer));
+
+ if (path != NULL && handler != NULL && router != NULL) {
+ memcpy(&ptr, &handler, sizeof(ptr));
+ storage = rspamd_fstring_new_init(path, strlen(path));
+ key = g_malloc0(sizeof(*key));
+ key->begin = storage->str;
+ key->len = storage->len;
+ g_hash_table_insert(router->paths, key, ptr);
+ }
+}
+
+void rspamd_http_router_set_unknown_handler(struct rspamd_http_connection_router *router,
+ rspamd_http_router_handler_t handler)
+{
+ if (router != NULL) {
+ router->unknown_method_handler = handler;
+ }
+}
+
+void rspamd_http_router_add_header(struct rspamd_http_connection_router *router,
+ const gchar *name, const gchar *value)
+{
+ if (name != NULL && value != NULL && router != NULL) {
+ g_hash_table_replace(router->response_headers, g_strdup(name),
+ g_strdup(value));
+ }
+}
+
+void rspamd_http_router_insert_headers(struct rspamd_http_connection_router *router,
+ struct rspamd_http_message *msg)
+{
+ GHashTableIter it;
+ gpointer k, v;
+
+ if (router && msg) {
+ g_hash_table_iter_init(&it, router->response_headers);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ rspamd_http_message_add_header(msg, k, v);
+ }
+ }
+}
+
+void rspamd_http_router_add_regexp(struct rspamd_http_connection_router *router,
+ struct rspamd_regexp_s *re, rspamd_http_router_handler_t handler)
+{
+ gpointer ptr;
+ G_STATIC_ASSERT(sizeof(rspamd_http_router_handler_t) ==
+ sizeof(gpointer));
+
+ if (re != NULL && handler != NULL && router != NULL) {
+ memcpy(&ptr, &handler, sizeof(ptr));
+ rspamd_regexp_set_ud(re, ptr);
+ g_ptr_array_add(router->regexps, rspamd_regexp_ref(re));
+ }
+}
+
+void rspamd_http_router_handle_socket(struct rspamd_http_connection_router *router,
+ gint fd, gpointer ud)
+{
+ struct rspamd_http_connection_entry *conn;
+
+ conn = g_malloc0(sizeof(struct rspamd_http_connection_entry));
+ conn->rt = router;
+ conn->ud = ud;
+ conn->is_reply = FALSE;
+
+ conn->conn = rspamd_http_connection_new_server(router->ctx,
+ fd,
+ NULL,
+ rspamd_http_router_error_handler,
+ rspamd_http_router_finish_handler,
+ 0);
+
+ if (router->key) {
+ rspamd_http_connection_set_key(conn->conn, router->key);
+ }
+
+ rspamd_http_connection_read_message(conn->conn, conn, router->timeout);
+ DL_PREPEND(router->conns, conn);
+}
+
+void rspamd_http_router_free(struct rspamd_http_connection_router *router)
+{
+ struct rspamd_http_connection_entry *conn, *tmp;
+ rspamd_regexp_t *re;
+ guint i;
+
+ if (router) {
+ DL_FOREACH_SAFE(router->conns, conn, tmp)
+ {
+ rspamd_http_entry_free(conn);
+ }
+
+ if (router->key) {
+ rspamd_keypair_unref(router->key);
+ }
+
+ if (router->default_fs_path != NULL) {
+ g_free(router->default_fs_path);
+ }
+
+ for (i = 0; i < router->regexps->len; i++) {
+ re = g_ptr_array_index(router->regexps, i);
+ rspamd_regexp_unref(re);
+ }
+
+ g_ptr_array_free(router->regexps, TRUE);
+ g_hash_table_unref(router->paths);
+ g_hash_table_unref(router->response_headers);
+ g_free(router);
+ }
+}
diff --git a/src/libserver/http/http_router.h b/src/libserver/http/http_router.h
new file mode 100644
index 0000000..1bf70ed
--- /dev/null
+++ b/src/libserver/http/http_router.h
@@ -0,0 +1,149 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_HTTP_ROUTER_H
+#define RSPAMD_HTTP_ROUTER_H
+
+#include "config.h"
+#include "http_connection.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_http_connection_router;
+struct rspamd_http_connection_entry;
+
+typedef int (*rspamd_http_router_handler_t)(struct rspamd_http_connection_entry
+ *conn_ent,
+ struct rspamd_http_message *msg);
+
+typedef void (*rspamd_http_router_error_handler_t)(struct rspamd_http_connection_entry *conn_ent,
+ GError *err);
+
+typedef void (*rspamd_http_router_finish_handler_t)(struct rspamd_http_connection_entry *conn_ent);
+
+
+struct rspamd_http_connection_entry {
+ struct rspamd_http_connection_router *rt;
+ struct rspamd_http_connection *conn;
+ gpointer ud;
+ gboolean is_reply;
+ gboolean support_gzip;
+ struct rspamd_http_connection_entry *prev, *next;
+};
+
+struct rspamd_http_connection_router {
+ struct rspamd_http_connection_entry *conns;
+ GHashTable *paths;
+ GHashTable *response_headers;
+ GPtrArray *regexps;
+ ev_tstamp timeout;
+ struct ev_loop *event_loop;
+ struct rspamd_http_context *ctx;
+ gchar *default_fs_path;
+ rspamd_http_router_handler_t unknown_method_handler;
+ struct rspamd_cryptobox_keypair *key;
+ rspamd_http_router_error_handler_t error_handler;
+ rspamd_http_router_finish_handler_t finish_handler;
+};
+
+/**
+ * Create new http connection router and the associated HTTP connection
+ * @param eh error handler callback
+ * @param fh finish handler callback
+ * @param default_fs_path if not NULL try to serve static files from
+ * the specified directory
+ * @return
+ */
+struct rspamd_http_connection_router *rspamd_http_router_new(
+ rspamd_http_router_error_handler_t eh,
+ rspamd_http_router_finish_handler_t fh,
+ ev_tstamp timeout,
+ const char *default_fs_path,
+ struct rspamd_http_context *ctx);
+
+/**
+ * Set encryption key for the HTTP router
+ * @param router router structure
+ * @param key opaque key structure
+ */
+void rspamd_http_router_set_key(struct rspamd_http_connection_router *router,
+ struct rspamd_cryptobox_keypair *key);
+
+/**
+ * Add new path to the router
+ */
+void rspamd_http_router_add_path(struct rspamd_http_connection_router *router,
+ const gchar *path, rspamd_http_router_handler_t handler);
+
+/**
+ * Add custom header to append to router replies
+ * @param router
+ * @param name
+ * @param value
+ */
+void rspamd_http_router_add_header(struct rspamd_http_connection_router *router,
+ const gchar *name, const gchar *value);
+
+/**
+ * Sets method to handle unknown request methods
+ * @param router
+ * @param handler
+ */
+void rspamd_http_router_set_unknown_handler(struct rspamd_http_connection_router *router,
+ rspamd_http_router_handler_t handler);
+
+/**
+ * Inserts router headers to the outbound message
+ * @param router
+ * @param msg
+ */
+void rspamd_http_router_insert_headers(struct rspamd_http_connection_router *router,
+ struct rspamd_http_message *msg);
+
+struct rspamd_regexp_s;
+
+/**
+ * Adds new pattern to router, regexp object is refcounted by this function
+ * @param router
+ * @param re
+ * @param handler
+ */
+void rspamd_http_router_add_regexp(struct rspamd_http_connection_router *router,
+ struct rspamd_regexp_s *re, rspamd_http_router_handler_t handler);
+
+/**
+ * Handle new accepted socket
+ * @param router router object
+ * @param fd server socket
+ * @param ud opaque userdata
+ */
+void rspamd_http_router_handle_socket(
+ struct rspamd_http_connection_router *router,
+ gint fd,
+ gpointer ud);
+
+/**
+ * Free router and all connections associated
+ * @param router
+ */
+void rspamd_http_router_free(struct rspamd_http_connection_router *router);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/http/http_util.c b/src/libserver/http/http_util.c
new file mode 100644
index 0000000..d5c4a57
--- /dev/null
+++ b/src/libserver/http/http_util.c
@@ -0,0 +1,295 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "libserver/http/http_util.h"
+#include "libutil/printf.h"
+#include "libutil/util.h"
+
+static const gchar *http_week[] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"};
+static const gchar *http_month[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun",
+ "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"};
+
+/*
+ * Obtained from nginx
+ * Copyright (C) Igor Sysoev
+ */
+static guint mday[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
+
+time_t
+rspamd_http_parse_date(const gchar *header, gsize len)
+{
+ const gchar *p, *end;
+ gint month;
+ guint day, year, hour, min, sec;
+ guint64 time;
+ enum {
+ no = 0,
+ rfc822, /* Tue, 10 Nov 2002 23:50:13 */
+ rfc850, /* Tuesday, 10-Dec-02 23:50:13 */
+ isoc /* Tue Dec 10 23:50:13 2002 */
+ } fmt;
+
+ fmt = 0;
+ if (len > 0) {
+ end = header + len;
+ }
+ else {
+ end = header + strlen(header);
+ }
+
+ day = 32;
+ year = 2038;
+
+ for (p = header; p < end; p++) {
+ if (*p == ',') {
+ break;
+ }
+
+ if (*p == ' ') {
+ fmt = isoc;
+ break;
+ }
+ }
+
+ for (p++; p < end; p++)
+ if (*p != ' ') {
+ break;
+ }
+
+ if (end - p < 18) {
+ return (time_t) -1;
+ }
+
+ if (fmt != isoc) {
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
+ return (time_t) -1;
+ }
+
+ day = (*p - '0') * 10 + *(p + 1) - '0';
+ p += 2;
+
+ if (*p == ' ') {
+ if (end - p < 18) {
+ return (time_t) -1;
+ }
+ fmt = rfc822;
+ }
+ else if (*p == '-') {
+ fmt = rfc850;
+ }
+ else {
+ return (time_t) -1;
+ }
+
+ p++;
+ }
+
+ switch (*p) {
+
+ case 'J':
+ month = *(p + 1) == 'a' ? 0 : *(p + 2) == 'n' ? 5
+ : 6;
+ break;
+
+ case 'F':
+ month = 1;
+ break;
+
+ case 'M':
+ month = *(p + 2) == 'r' ? 2 : 4;
+ break;
+
+ case 'A':
+ month = *(p + 1) == 'p' ? 3 : 7;
+ break;
+
+ case 'S':
+ month = 8;
+ break;
+
+ case 'O':
+ month = 9;
+ break;
+
+ case 'N':
+ month = 10;
+ break;
+
+ case 'D':
+ month = 11;
+ break;
+
+ default:
+ return (time_t) -1;
+ }
+
+ p += 3;
+
+ if ((fmt == rfc822 && *p != ' ') || (fmt == rfc850 && *p != '-')) {
+ return (time_t) -1;
+ }
+
+ p++;
+
+ if (fmt == rfc822) {
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9' || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0' || *(p + 3) > '9') {
+ return (time_t) -1;
+ }
+
+ year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100 + (*(p + 2) - '0') * 10 + *(p + 3) - '0';
+ p += 4;
+ }
+ else if (fmt == rfc850) {
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
+ return (time_t) -1;
+ }
+
+ year = (*p - '0') * 10 + *(p + 1) - '0';
+ year += (year < 70) ? 2000 : 1900;
+ p += 2;
+ }
+
+ if (fmt == isoc) {
+ if (*p == ' ') {
+ p++;
+ }
+
+ if (*p < '0' || *p > '9') {
+ return (time_t) -1;
+ }
+
+ day = *p++ - '0';
+
+ if (*p != ' ') {
+ if (*p < '0' || *p > '9') {
+ return (time_t) -1;
+ }
+
+ day = day * 10 + *p++ - '0';
+ }
+
+ if (end - p < 14) {
+ return (time_t) -1;
+ }
+ }
+
+ if (*p++ != ' ') {
+ return (time_t) -1;
+ }
+
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
+ return (time_t) -1;
+ }
+
+ hour = (*p - '0') * 10 + *(p + 1) - '0';
+ p += 2;
+
+ if (*p++ != ':') {
+ return (time_t) -1;
+ }
+
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
+ return (time_t) -1;
+ }
+
+ min = (*p - '0') * 10 + *(p + 1) - '0';
+ p += 2;
+
+ if (*p++ != ':') {
+ return (time_t) -1;
+ }
+
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
+ return (time_t) -1;
+ }
+
+ sec = (*p - '0') * 10 + *(p + 1) - '0';
+
+ if (fmt == isoc) {
+ p += 2;
+
+ if (*p++ != ' ') {
+ return (time_t) -1;
+ }
+
+ if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9' || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0' || *(p + 3) > '9') {
+ return (time_t) -1;
+ }
+
+ year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100 + (*(p + 2) - '0') * 10 + *(p + 3) - '0';
+ }
+
+ if (hour > 23 || min > 59 || sec > 59) {
+ return (time_t) -1;
+ }
+
+ if (day == 29 && month == 1) {
+ if ((year & 3) || ((year % 100 == 0) && (year % 400) != 0)) {
+ return (time_t) -1;
+ }
+ }
+ else if (day > mday[month]) {
+ return (time_t) -1;
+ }
+
+ /*
+ * shift new year to March 1 and start months from 1 (not 0),
+ * it is needed for Gauss' formula
+ */
+
+ if (--month <= 0) {
+ month += 12;
+ year -= 1;
+ }
+
+ /* Gauss' formula for Gregorian days since March 1, 1 BC */
+
+ time = (guint64) (
+ /* days in years including leap years since March 1, 1 BC */
+
+ 365 * year + year / 4 - year / 100 + year / 400
+
+ /* days before the month */
+
+ + 367 * month / 12 - 30
+
+ /* days before the day */
+
+ + day - 1
+
+ /*
+ * 719527 days were between March 1, 1 BC and March 1, 1970,
+ * 31 and 28 days were in January and February 1970
+ */
+
+ - 719527 + 31 + 28) *
+ 86400 +
+ hour * 3600 + min * 60 + sec;
+
+ return (time_t) time;
+}
+
+glong rspamd_http_date_format(gchar *buf, gsize len, time_t time)
+{
+ struct tm tms;
+
+ rspamd_gmtime(time, &tms);
+
+ return rspamd_snprintf(buf, len, "%s, %02d %s %4d %02d:%02d:%02d GMT",
+ http_week[tms.tm_wday], tms.tm_mday,
+ http_month[tms.tm_mon], tms.tm_year + 1900,
+ tms.tm_hour, tms.tm_min, tms.tm_sec);
+} \ No newline at end of file
diff --git a/src/libserver/http/http_util.h b/src/libserver/http/http_util.h
new file mode 100644
index 0000000..ec57508
--- /dev/null
+++ b/src/libserver/http/http_util.h
@@ -0,0 +1,47 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_HTTP_UTIL_H
+#define RSPAMD_HTTP_UTIL_H
+
+#include "config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Parse HTTP date header and return it as time_t
+ * @param header HTTP date header
+ * @param len length of header
+ * @return time_t or (time_t)-1 in case of error
+ */
+time_t rspamd_http_parse_date(const gchar *header, gsize len);
+
+/**
+ * Prints HTTP date from `time` to `buf` using standard HTTP date format
+ * @param buf date buffer
+ * @param len length of buffer
+ * @param time time in unix seconds
+ * @return number of bytes written
+ */
+glong rspamd_http_date_format(gchar *buf, gsize len, time_t time);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/hyperscan_tools.cxx b/src/libserver/hyperscan_tools.cxx
new file mode 100644
index 0000000..7d1ecf3
--- /dev/null
+++ b/src/libserver/hyperscan_tools.cxx
@@ -0,0 +1,627 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+
+#ifdef WITH_HYPERSCAN
+#include <string>
+#include <filesystem>
+#include "contrib/ankerl/unordered_dense.h"
+#include "contrib/ankerl/svector.h"
+#include "fmt/core.h"
+#include "libutil/cxx/file_util.hxx"
+#include "libutil/cxx/error.hxx"
+#include "hs.h"
+#include "logger.h"
+#include "worker_util.h"
+#include "hyperscan_tools.h"
+
+#include <glob.h> /* for glob */
+#include <unistd.h> /* for unlink */
+#include <optional>
+#include <cstdlib> /* for std::getenv */
+#include "unix-std.h"
+#include "rspamd_control.h"
+
+#define HYPERSCAN_LOG_TAG "hsxxxx"
+
+// Hyperscan does not provide any API to check validity of it's databases
+// However, it is required for us to perform migrations properly without
+// failing at `hs_alloc_scratch` phase or even `hs_scan` which is **way too late**
+// Hence, we have to check hyperscan internal guts to prevent that situation...
+
+#ifdef HS_MAJOR
+#ifndef HS_VERSION_32BIT
+#define HS_VERSION_32BIT ((HS_MAJOR << 24) | (HS_MINOR << 16) | (HS_PATCH << 8) | 0)
+#endif
+#endif// defined(HS_MAJOR)
+
+#if !defined(HS_DB_VERSION) && defined(HS_VERSION_32BIT)
+#define HS_DB_VERSION HS_VERSION_32BIT
+#endif
+
+#ifndef HS_DB_MAGIC
+#define HS_DB_MAGIC (0xdbdbdbdbU)
+#endif
+
+#define msg_info_hyperscan(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "hyperscan", HYPERSCAN_LOG_TAG, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_hyperscan_lambda(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "hyperscan", HYPERSCAN_LOG_TAG, \
+ log_func, \
+ __VA_ARGS__)
+#define msg_err_hyperscan(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "hyperscan", HYPERSCAN_LOG_TAG, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_hyperscan(...) rspamd_conditional_debug_fast(nullptr, nullptr, \
+ rspamd_hyperscan_log_id, "hyperscan", HYPERSCAN_LOG_TAG, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_hyperscan_lambda(...) rspamd_conditional_debug_fast(nullptr, nullptr, \
+ rspamd_hyperscan_log_id, "hyperscan", HYPERSCAN_LOG_TAG, \
+ log_func, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE_PUBLIC(hyperscan)
+
+namespace rspamd::util {
+
+/*
+ * A singleton class that is responsible for deletion of the outdated hyperscan files
+ * One issue is that it must know about HS files in all workers, which is a problem
+ * TODO: we need to export hyperscan caches from all workers to a single place where
+ * we can clean them up (probably, to the main process)
+ */
+class hs_known_files_cache {
+private:
+ // These fields are filled when we add new known cache files
+ ankerl::svector<std::string, 4> cache_dirs;
+ ankerl::svector<std::string, 8> cache_extensions;
+ ankerl::unordered_dense::set<std::string> known_cached_files;
+ bool loaded = false;
+
+private:
+ hs_known_files_cache() = default;
+
+ virtual ~hs_known_files_cache()
+ {
+ // Cleanup cache dir
+ cleanup_maybe();
+ }
+
+public:
+ hs_known_files_cache(const hs_known_files_cache &) = delete;
+ hs_known_files_cache(hs_known_files_cache &&) = delete;
+
+ static auto get() -> hs_known_files_cache &
+ {
+ static hs_known_files_cache *singleton = nullptr;
+
+ if (singleton == nullptr) {
+ singleton = new hs_known_files_cache;
+ }
+
+ return *singleton;
+ }
+
+ void add_cached_file(const raii_file &file)
+ {
+ auto fpath = std::filesystem::path{file.get_name()};
+ std::error_code ec;
+
+ fpath = std::filesystem::canonical(fpath, ec);
+
+ if (ec && ec.value() != 0) {
+ msg_err_hyperscan("invalid path: \"%s\", error message: %s", fpath.c_str(), ec.message().c_str());
+ return;
+ }
+
+ auto dir = fpath.parent_path();
+ auto ext = fpath.extension();
+
+ if (std::find_if(cache_dirs.begin(), cache_dirs.end(),
+ [&](const auto &item) { return item == dir; }) == std::end(cache_dirs)) {
+ cache_dirs.emplace_back(std::string{dir});
+ }
+ if (std::find_if(cache_extensions.begin(), cache_extensions.end(),
+ [&](const auto &item) { return item == ext; }) == std::end(cache_extensions)) {
+ cache_extensions.emplace_back(std::string{ext});
+ }
+
+ auto is_known = known_cached_files.insert(fpath.string());
+ msg_debug_hyperscan("added %s hyperscan file: %s",
+ is_known.second ? "new" : "already known",
+ fpath.c_str());
+ }
+
+ void add_cached_file(const char *fname)
+ {
+ auto fpath = std::filesystem::path{fname};
+ std::error_code ec;
+
+ fpath = std::filesystem::canonical(fpath, ec);
+
+ if (ec && ec.value() != 0) {
+ msg_err_hyperscan("invalid path: \"%s\", error message: %s", fname, ec.message().c_str());
+ return;
+ }
+
+ auto dir = fpath.parent_path();
+ auto ext = fpath.extension();
+
+ if (std::find_if(cache_dirs.begin(), cache_dirs.end(),
+ [&](const auto &item) { return item == dir; }) == std::end(cache_dirs)) {
+ cache_dirs.emplace_back(dir.string());
+ }
+ if (std::find_if(cache_extensions.begin(), cache_extensions.end(),
+ [&](const auto &item) { return item == ext; }) == std::end(cache_extensions)) {
+ cache_extensions.emplace_back(ext.string());
+ }
+
+ auto is_known = known_cached_files.insert(fpath.string());
+ msg_debug_hyperscan("added %s hyperscan file: %s",
+ is_known.second ? "new" : "already known",
+ fpath.c_str());
+ }
+
+ void delete_cached_file(const char *fname)
+ {
+ auto fpath = std::filesystem::path{fname};
+ std::error_code ec;
+
+ fpath = std::filesystem::canonical(fpath, ec);
+
+ if (ec && ec.value() != 0) {
+ msg_err_hyperscan("invalid path to remove: \"%s\", error message: %s",
+ fname, ec.message().c_str());
+ return;
+ }
+
+ if (fpath.empty()) {
+ msg_err_hyperscan("attempt to remove an empty hyperscan file!");
+ return;
+ }
+
+ if (unlink(fpath.c_str()) == -1) {
+ msg_err_hyperscan("cannot remove hyperscan file %s: %s",
+ fpath.c_str(), strerror(errno));
+ }
+ else {
+ msg_debug_hyperscan("removed hyperscan file %s", fpath.c_str());
+ }
+
+ known_cached_files.erase(fpath.string());
+ }
+
+ auto cleanup_maybe() -> void
+ {
+ auto env_cleanup_disable = std::getenv("RSPAMD_NO_CLEANUP");
+ /* We clean dir merely if we are running from the main process */
+ if (rspamd_current_worker == nullptr && env_cleanup_disable == nullptr && loaded) {
+ const auto *log_func = RSPAMD_LOG_FUNC;
+ auto cleanup_dir = [&](std::string_view dir) -> void {
+ for (const auto &ext: cache_extensions) {
+ glob_t globbuf;
+
+ auto glob_pattern = fmt::format("{}{}*{}",
+ dir, G_DIR_SEPARATOR_S, ext);
+ msg_debug_hyperscan_lambda("perform glob for pattern: %s",
+ glob_pattern.c_str());
+ memset(&globbuf, 0, sizeof(globbuf));
+
+ if (glob(glob_pattern.c_str(), 0, nullptr, &globbuf) == 0) {
+ for (auto i = 0; i < globbuf.gl_pathc; i++) {
+ auto path = std::string{globbuf.gl_pathv[i]};
+ std::size_t nsz;
+ struct stat st;
+
+ rspamd_normalize_path_inplace(path.data(), path.size(), &nsz);
+ path.resize(nsz);
+
+ if (stat(path.c_str(), &st) == -1) {
+ msg_debug_hyperscan_lambda("cannot stat file %s: %s",
+ path.c_str(), strerror(errno));
+ continue;
+ }
+
+ if (S_ISREG(st.st_mode)) {
+ if (!known_cached_files.contains(path)) {
+ msg_info_hyperscan_lambda("remove stale hyperscan file %s", path.c_str());
+ unlink(path.c_str());
+ }
+ else {
+ msg_debug_hyperscan_lambda("found known hyperscan file %s, size: %Hz",
+ path.c_str(), st.st_size);
+ }
+ }
+ }
+ }
+
+ globfree(&globbuf);
+ }
+ };
+
+ for (const auto &dir: cache_dirs) {
+ msg_info_hyperscan("cleaning up directory %s", dir.c_str());
+ cleanup_dir(dir);
+ }
+
+ cache_dirs.clear();
+ cache_extensions.clear();
+ known_cached_files.clear();
+ }
+ else if (rspamd_current_worker == nullptr && env_cleanup_disable != nullptr) {
+ msg_info_hyperscan("disable hyperscan cleanup: env variable RSPAMD_NO_CLEANUP is set");
+ }
+ else if (!loaded) {
+ msg_info_hyperscan("disable hyperscan cleanup: not loaded");
+ }
+ }
+
+ auto notice_loaded() -> void
+ {
+ loaded = true;
+ }
+};
+
+
+/**
+ * This is a higher level representation of the cached hyperscan file
+ */
+struct hs_shared_database {
+ hs_database_t *db = nullptr; /**< internal database (might be in a shared memory) */
+ std::optional<raii_mmaped_file> maybe_map;
+ std::string cached_path;
+
+ ~hs_shared_database()
+ {
+ if (!maybe_map) {
+ hs_free_database(db);
+ }
+ // Otherwise, handled by maybe_map dtor
+ }
+
+ explicit hs_shared_database(raii_mmaped_file &&map, hs_database_t *db)
+ : db(db), maybe_map(std::move(map))
+ {
+ cached_path = maybe_map.value().get_file().get_name();
+ }
+ explicit hs_shared_database(hs_database_t *db, const char *fname)
+ : db(db), maybe_map(std::nullopt)
+ {
+ if (fname) {
+ cached_path = fname;
+ }
+ else {
+ /* Likely a test case */
+ cached_path = "";
+ }
+ }
+ hs_shared_database(const hs_shared_database &other) = delete;
+ hs_shared_database() = default;
+ hs_shared_database(hs_shared_database &&other) noexcept
+ {
+ *this = std::move(other);
+ }
+ hs_shared_database &operator=(hs_shared_database &&other) noexcept
+ {
+ std::swap(db, other.db);
+ std::swap(maybe_map, other.maybe_map);
+ return *this;
+ }
+};
+
+struct real_hs_db {
+ std::uint32_t magic;
+ std::uint32_t version;
+ std::uint32_t length;
+ std::uint64_t platform;
+ std::uint32_t crc32;
+};
+static auto
+hs_is_valid_database(void *raw, std::size_t len, std::string_view fname) -> tl::expected<bool, std::string>
+{
+ if (len < sizeof(real_hs_db)) {
+ return tl::make_unexpected(fmt::format("cannot load hyperscan database from {}: too short", fname));
+ }
+
+ static real_hs_db test;
+
+ memcpy(&test, raw, sizeof(test));
+
+ if (test.magic != HS_DB_MAGIC) {
+ return tl::make_unexpected(fmt::format("cannot load hyperscan database from {}: invalid magic: {} ({} expected)",
+ fname, test.magic, HS_DB_MAGIC));
+ }
+
+#ifdef HS_DB_VERSION
+ if (test.version != HS_DB_VERSION) {
+ return tl::make_unexpected(fmt::format("cannot load hyperscan database from {}: invalid version: {} ({} expected)",
+ fname, test.version, HS_DB_VERSION));
+ }
+#endif
+
+ return true;
+}
+
+static auto
+hs_shared_from_unserialized(hs_known_files_cache &hs_cache, raii_mmaped_file &&map) -> tl::expected<hs_shared_database, error>
+{
+ auto ptr = map.get_map();
+ auto db = (hs_database_t *) ptr;
+
+ auto is_valid = hs_is_valid_database(map.get_map(), map.get_size(), map.get_file().get_name());
+ if (!is_valid) {
+ return tl::make_unexpected(error{is_valid.error(), -1, error_category::IMPORTANT});
+ }
+
+ hs_cache.add_cached_file(map.get_file());
+ return tl::expected<hs_shared_database, error>{tl::in_place, std::move(map), db};
+}
+
+static auto
+hs_shared_from_serialized(hs_known_files_cache &hs_cache, raii_mmaped_file &&map, std::int64_t offset) -> tl::expected<hs_shared_database, error>
+{
+ hs_database_t *target = nullptr;
+
+ if (auto ret = hs_deserialize_database((const char *) map.get_map() + offset,
+ map.get_size() - offset, &target);
+ ret != HS_SUCCESS) {
+ return tl::make_unexpected(error{"cannot deserialize database", ret});
+ }
+
+ hs_cache.add_cached_file(map.get_file());
+ return tl::expected<hs_shared_database, error>{tl::in_place, target, map.get_file().get_name().data()};
+}
+
+auto load_cached_hs_file(const char *fname, std::int64_t offset = 0) -> tl::expected<hs_shared_database, error>
+{
+ auto &hs_cache = hs_known_files_cache::get();
+ const auto *log_func = RSPAMD_LOG_FUNC;
+
+ return raii_mmaped_file::mmap_shared(fname, O_RDONLY, PROT_READ, 0)
+ .and_then([&]<class T>(T &&cached_serialized) -> tl::expected<hs_shared_database, error> {
+ if (cached_serialized.get_size() <= offset) {
+ return tl::make_unexpected(error{"Invalid offset", EINVAL, error_category::CRITICAL});
+ }
+#if defined(HS_MAJOR) && defined(HS_MINOR) && HS_MAJOR >= 5 && HS_MINOR >= 4
+ auto unserialized_fname = fmt::format("{}.unser", fname);
+ auto unserialized_file = raii_locked_file::create(unserialized_fname.c_str(), O_CREAT | O_RDWR | O_EXCL,
+ 00644)
+ .and_then([&](auto &&new_file_locked) -> tl::expected<raii_file, error> {
+ auto tmpfile_pattern = fmt::format("{}{}hsmp-XXXXXXXXXXXXXXXXXX",
+ cached_serialized.get_file().get_dir(), G_DIR_SEPARATOR);
+ auto tmpfile = raii_locked_file::mkstemp(tmpfile_pattern.data(), O_CREAT | O_RDWR | O_EXCL,
+ 00644);
+
+ if (!tmpfile) {
+ return tl::make_unexpected(tmpfile.error());
+ }
+ else {
+ auto &tmpfile_checked = tmpfile.value();
+ // Store owned string
+ auto tmpfile_name = std::string{tmpfile_checked.get_name()};
+ std::size_t unserialized_size;
+
+ if (auto ret = hs_serialized_database_size(((const char *) cached_serialized.get_map()) + offset,
+ cached_serialized.get_size() - offset, &unserialized_size);
+ ret != HS_SUCCESS) {
+ return tl::make_unexpected(error{
+ fmt::format("cannot get unserialized database size: {}", ret),
+ EINVAL,
+ error_category::IMPORTANT});
+ }
+
+ msg_debug_hyperscan_lambda("multipattern: create new database in %s; %Hz size",
+ tmpfile_name.c_str(), unserialized_size);
+ void *buf;
+#ifdef HAVE_GETPAGESIZE
+ auto page_size = getpagesize();
+#else
+ auto page_size = sysconf(_SC_PAGESIZE);
+#endif
+ if (page_size == -1) {
+ page_size = 4096;
+ }
+ auto errcode = posix_memalign(&buf, page_size, unserialized_size);
+ if (errcode != 0 || buf == nullptr) {
+ return tl::make_unexpected(error{"Cannot allocate memory",
+ errno, error_category::CRITICAL});
+ }
+
+ if (auto ret = hs_deserialize_database_at(((const char *) cached_serialized.get_map()) + offset,
+ cached_serialized.get_size() - offset, (hs_database_t *) buf);
+ ret != HS_SUCCESS) {
+ return tl::make_unexpected(error{
+ fmt::format("cannot deserialize hyperscan database: {}", ret), ret});
+ }
+ else {
+ if (write(tmpfile_checked.get_fd(), buf, unserialized_size) == -1) {
+ free(buf);
+ return tl::make_unexpected(error{fmt::format("cannot write to {}: {}",
+ tmpfile_name, ::strerror(errno)),
+ errno, error_category::CRITICAL});
+ }
+ else {
+ free(buf);
+ /*
+ * Unlink target file before renaming to avoid
+ * race condition.
+ * So what we have is that `new_file_locked`
+ * will have flock on that file, so it will be
+ * replaced after unlink safely, and also unlocked.
+ */
+ (void) unlink(unserialized_fname.c_str());
+ if (rename(tmpfile_name.c_str(),
+ unserialized_fname.c_str()) == -1) {
+ if (errno != EEXIST) {
+ msg_info_hyperscan_lambda("cannot rename %s -> %s: %s",
+ tmpfile_name.c_str(),
+ unserialized_fname.c_str(),
+ strerror(errno));
+ }
+ }
+ else {
+ /* Unlock file but mark it as immortal first to avoid deletion */
+ tmpfile_checked.make_immortal();
+ (void) tmpfile_checked.unlock();
+ }
+ }
+ }
+ /* Reopen in RO mode */
+ return raii_file::open(unserialized_fname.c_str(), O_RDONLY);
+ };
+ })
+ .or_else([&](auto unused) -> tl::expected<raii_file, error> {
+ // Cannot create file, so try to open it in RO mode
+ return raii_file::open(unserialized_fname.c_str(), O_RDONLY);
+ });
+
+ tl::expected<hs_shared_database, error> ret;
+
+ if (unserialized_file.has_value()) {
+
+ auto &unserialized_checked = unserialized_file.value();
+
+ if (unserialized_checked.get_size() == 0) {
+ /*
+ * This is a case when we have a file that is currently
+ * being created by another process.
+ * We cannot use it!
+ */
+ ret = hs_shared_from_serialized(hs_cache, std::forward<T>(cached_serialized), offset);
+ }
+ else {
+ ret = raii_mmaped_file::mmap_shared(std::move(unserialized_checked), PROT_READ)
+ .and_then([&]<class U>(U &&mmapped_unserialized) -> auto {
+ return hs_shared_from_unserialized(hs_cache, std::forward<U>(mmapped_unserialized));
+ });
+ }
+ }
+ else {
+ ret = hs_shared_from_serialized(hs_cache, std::forward<T>(cached_serialized), offset);
+ }
+#else // defined(HS_MAJOR) && defined(HS_MINOR) && HS_MAJOR >= 5 && HS_MINOR >= 4
+ auto ret = hs_shared_from_serialized(hs_cache, std::forward<T>(cached_serialized), offset);
+#endif// defined(HS_MAJOR) && defined(HS_MINOR) && HS_MAJOR >= 5 && HS_MINOR >= 4 \
+ // Add serialized file to cache merely if we have successfully loaded the actual db
+ if (ret.has_value()) {
+ hs_cache.add_cached_file(cached_serialized.get_file());
+ }
+ return ret;
+ });
+}
+}// namespace rspamd::util
+
+/* C API */
+
+#define CXX_DB_FROM_C(obj) (reinterpret_cast<rspamd::util::hs_shared_database *>(obj))
+#define C_DB_FROM_CXX(obj) (reinterpret_cast<rspamd_hyperscan_t *>(obj))
+
+rspamd_hyperscan_t *
+rspamd_hyperscan_maybe_load(const char *filename, goffset offset)
+{
+ auto maybe_db = rspamd::util::load_cached_hs_file(filename, offset);
+
+ if (maybe_db.has_value()) {
+ auto *ndb = new rspamd::util::hs_shared_database;
+ *ndb = std::move(maybe_db.value());
+ return C_DB_FROM_CXX(ndb);
+ }
+ else {
+ auto error = maybe_db.error();
+
+ switch (error.category) {
+ case rspamd::util::error_category::CRITICAL:
+ msg_err_hyperscan("critical error when trying to load cached hyperscan: %s",
+ error.error_message.data());
+ break;
+ case rspamd::util::error_category::IMPORTANT:
+ msg_info_hyperscan("error when trying to load cached hyperscan: %s",
+ error.error_message.data());
+ break;
+ default:
+ msg_debug_hyperscan("error when trying to load cached hyperscan: %s",
+ error.error_message.data());
+ break;
+ }
+ }
+
+ return nullptr;
+}
+
+hs_database_t *
+rspamd_hyperscan_get_database(rspamd_hyperscan_t *db)
+{
+ auto *real_db = CXX_DB_FROM_C(db);
+ return real_db->db;
+}
+
+rspamd_hyperscan_t *
+rspamd_hyperscan_from_raw_db(hs_database_t *db, const char *fname)
+{
+ auto *ndb = new rspamd::util::hs_shared_database{db, fname};
+
+ return C_DB_FROM_CXX(ndb);
+}
+
+void rspamd_hyperscan_free(rspamd_hyperscan_t *db, bool invalid)
+{
+ auto *real_db = CXX_DB_FROM_C(db);
+
+ if (invalid && !real_db->cached_path.empty()) {
+ rspamd::util::hs_known_files_cache::get().delete_cached_file(real_db->cached_path.c_str());
+ }
+ delete real_db;
+}
+
+void rspamd_hyperscan_notice_known(const char *fname)
+{
+ rspamd::util::hs_known_files_cache::get().add_cached_file(fname);
+
+ if (rspamd_current_worker != nullptr) {
+ /* Also notify main process */
+ struct rspamd_srv_command notice_cmd;
+
+ if (strlen(fname) >= sizeof(notice_cmd.cmd.hyperscan_cache_file.path)) {
+ msg_err("internal error: length of the filename %d ('%s') is larger than control buffer path: %d",
+ (int) strlen(fname), fname, (int) sizeof(notice_cmd.cmd.hyperscan_cache_file.path));
+ }
+ else {
+ notice_cmd.type = RSPAMD_SRV_NOTICE_HYPERSCAN_CACHE;
+ rspamd_strlcpy(notice_cmd.cmd.hyperscan_cache_file.path, fname, sizeof(notice_cmd.cmd.hyperscan_cache_file.path));
+ rspamd_srv_send_command(rspamd_current_worker,
+ rspamd_current_worker->srv->event_loop, &notice_cmd, -1,
+ nullptr,
+ nullptr);
+ }
+ }
+}
+
+void rspamd_hyperscan_cleanup_maybe(void)
+{
+ rspamd::util::hs_known_files_cache::get().cleanup_maybe();
+}
+
+void rspamd_hyperscan_notice_loaded(void)
+{
+ rspamd::util::hs_known_files_cache::get().notice_loaded();
+}
+
+#endif// WITH_HYPERSCAN \ No newline at end of file
diff --git a/src/libserver/hyperscan_tools.h b/src/libserver/hyperscan_tools.h
new file mode 100644
index 0000000..624b7b0
--- /dev/null
+++ b/src/libserver/hyperscan_tools.h
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+
+#ifndef RSPAMD_HYPERSCAN_TOOLS_H
+#define RSPAMD_HYPERSCAN_TOOLS_H
+
+#ifdef WITH_HYPERSCAN
+
+#include "hs.h"
+
+G_BEGIN_DECLS
+
+/**
+ * Opaque structure that represents hyperscan (maybe shared/cached database)
+ */
+typedef struct rspamd_hyperscan_s rspamd_hyperscan_t;
+
+/**
+ * Maybe load or mmap shared a hyperscan from a file
+ * @param filename
+ * @return cached database if available
+ */
+rspamd_hyperscan_t *rspamd_hyperscan_maybe_load(const char *filename, goffset offset);
+
+/**
+ * Creates a wrapper for a raw hs db. Ownership is transferred to the enclosing object returned
+ * @param filename
+ * @return
+ */
+rspamd_hyperscan_t *rspamd_hyperscan_from_raw_db(hs_database_t *db, const char *fname);
+/**
+ * Get the internal database
+ * @param db
+ * @return
+ */
+hs_database_t *rspamd_hyperscan_get_database(rspamd_hyperscan_t *db);
+/**
+ * Free the database
+ * @param db
+ */
+void rspamd_hyperscan_free(rspamd_hyperscan_t *db, bool invalid);
+
+/**
+ * Notice a known hyperscan file (e.g. externally serialized)
+ * @param fname
+ */
+void rspamd_hyperscan_notice_known(const char *fname);
+
+/**
+ * Notice that hyperscan files are all loaded (e.g. in the main process), so we can cleanup old files on termination
+ */
+void rspamd_hyperscan_notice_loaded(void);
+
+/**
+ * Cleans up old files. This method should be called on config free (in the main process)
+ */
+void rspamd_hyperscan_cleanup_maybe(void);
+
+G_END_DECLS
+
+#endif
+
+#endif
diff --git a/src/libserver/logger.h b/src/libserver/logger.h
new file mode 100644
index 0000000..8d4e313
--- /dev/null
+++ b/src/libserver/logger.h
@@ -0,0 +1,403 @@
+#ifndef RSPAMD_LOGGER_H
+#define RSPAMD_LOGGER_H
+
+#include "config.h"
+#include "radix.h"
+#include "util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef G_LOG_LEVEL_USER_SHIFT
+#define G_LOG_LEVEL_USER_SHIFT 8
+#endif
+
+#define RSPAMD_LOG_ID_LEN 6
+
+struct rspamd_config;
+
+enum rspamd_log_flags {
+ RSPAMD_LOG_FORCED = (1 << G_LOG_LEVEL_USER_SHIFT),
+ RSPAMD_LOG_ENCRYPTED = (1 << (G_LOG_LEVEL_USER_SHIFT + 1)),
+ RSPAMD_LOG_LEVEL_MASK = ~(RSPAMD_LOG_FORCED | RSPAMD_LOG_ENCRYPTED)
+};
+
+typedef struct rspamd_logger_s rspamd_logger_t;
+typedef bool (*rspamd_log_func_t)(const gchar *module, const gchar *id,
+ const gchar *function,
+ gint level_flags,
+ const gchar *message,
+ gsize mlen,
+ rspamd_logger_t *logger,
+ gpointer arg);
+typedef void *(*rspamd_log_init_func)(rspamd_logger_t *logger,
+ struct rspamd_config *cfg,
+ uid_t uid, gid_t gid,
+ GError **err);
+typedef bool (*rspamd_log_on_fork_func)(rspamd_logger_t *logger,
+ struct rspamd_config *cfg,
+ gpointer arg,
+ GError **err);
+typedef void *(*rspamd_log_reload_func)(rspamd_logger_t *logger,
+ struct rspamd_config *cfg,
+ gpointer arg,
+ uid_t uid, gid_t gid,
+ GError **err);
+typedef void (*rspamd_log_dtor_func)(rspamd_logger_t *logger,
+ gpointer arg);
+
+struct rspamd_logger_funcs {
+ rspamd_log_init_func init;
+ rspamd_log_reload_func reload;
+ rspamd_log_dtor_func dtor;
+ rspamd_log_func_t log;
+ rspamd_log_on_fork_func on_fork;
+ gpointer specific;
+};
+
+#if defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64)
+#define RSPAMD_LOGBUF_SIZE 8192
+#else
+/* Use a smaller buffer */
+#define RSPAMD_LOGBUF_SIZE 2048
+#endif
+
+/**
+ * Opens a new (initial) logger with console type
+ * This logger is also used as an emergency logger
+ * @return new rspamd logger object
+ */
+rspamd_logger_t *rspamd_log_open_emergency(rspamd_mempool_t *pool, gint flags);
+
+/**
+ * Open specific (configured logging)
+ * @param pool
+ * @param config
+ * @param uid
+ * @param gid
+ * @return
+ */
+rspamd_logger_t *rspamd_log_open_specific(rspamd_mempool_t *pool,
+ struct rspamd_config *config,
+ const gchar *ptype,
+ uid_t uid, gid_t gid);
+
+/**
+ * Set log level (from GLogLevelFlags)
+ * @param logger
+ * @param level
+ */
+void rspamd_log_set_log_level(rspamd_logger_t *logger, gint level);
+gint rspamd_log_get_log_level(rspamd_logger_t *logger);
+const gchar *rspamd_get_log_severity_string(gint level_flags);
+/**
+ * Set log flags (from enum rspamd_log_flags)
+ * @param logger
+ * @param flags
+ */
+void rspamd_log_set_log_flags(rspamd_logger_t *logger, gint flags);
+
+/**
+ * Close log file or destroy other structures
+ */
+void rspamd_log_close(rspamd_logger_t *logger);
+
+
+rspamd_logger_t *rspamd_log_default_logger(void);
+rspamd_logger_t *rspamd_log_emergency_logger(void);
+
+/**
+ * Close and open log again for privileged processes
+ */
+bool rspamd_log_reopen(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ uid_t uid, gid_t gid);
+
+/**
+ * Set log pid
+ */
+void rspamd_log_on_fork(GQuark ptype, struct rspamd_config *cfg,
+ rspamd_logger_t *logger);
+
+/**
+ * Log function that is compatible for glib messages
+ */
+void rspamd_glib_log_function(const gchar *log_domain,
+ GLogLevelFlags log_level,
+ const gchar *message,
+ gpointer arg);
+
+/**
+ * Log function for printing glib assertions
+ */
+void rspamd_glib_printerr_function(const gchar *message);
+
+/**
+ * Function with variable number of arguments support
+ */
+bool rspamd_common_log_function(rspamd_logger_t *logger,
+ gint level_flags,
+ const gchar *module, const gchar *id,
+ const gchar *function, const gchar *fmt, ...);
+
+bool rspamd_common_logv(rspamd_logger_t *logger, gint level_flags,
+ const gchar *module, const gchar *id, const gchar *function,
+ const gchar *fmt, va_list args);
+
+/**
+ * Add new logging module, returns module ID
+ * @param mod
+ * @return
+ */
+gint rspamd_logger_add_debug_module(const gchar *mod);
+
+/*
+ * Macro to use for faster debug modules
+ */
+#define INIT_LOG_MODULE(mname) \
+ static gint rspamd_##mname##_log_id = -1; \
+ RSPAMD_CONSTRUCTOR(rspamd_##mname##_log_init) \
+ { \
+ rspamd_##mname##_log_id = rspamd_logger_add_debug_module(#mname); \
+ }
+
+
+#define INIT_LOG_MODULE_PUBLIC(mname) \
+ gint rspamd_##mname##_log_id = -1; \
+ RSPAMD_CONSTRUCTOR(rspamd_##mname##_log_init) \
+ { \
+ rspamd_##mname##_log_id = rspamd_logger_add_debug_module(#mname); \
+ }
+
+#define EXTERN_LOG_MODULE_DEF(mname) \
+ extern gint rspamd_##mname##_log_id
+
+void rspamd_logger_configure_modules(GHashTable *mods_enabled);
+
+/**
+ * Conditional debug function
+ */
+bool rspamd_conditional_debug(rspamd_logger_t *logger,
+ rspamd_inet_addr_t *addr, const gchar *module, const gchar *id,
+ const gchar *function, const gchar *fmt, ...);
+
+bool rspamd_conditional_debug_fast(rspamd_logger_t *logger,
+ rspamd_inet_addr_t *addr,
+ gint mod_id,
+ const gchar *module, const gchar *id,
+ const gchar *function, const gchar *fmt, ...);
+bool rspamd_conditional_debug_fast_num_id(rspamd_logger_t *logger,
+ rspamd_inet_addr_t *addr,
+ gint mod_id,
+ const gchar *module, guint64 id,
+ const gchar *function, const gchar *fmt, ...);
+gboolean rspamd_logger_need_log(rspamd_logger_t *rspamd_log,
+ GLogLevelFlags log_level,
+ gint module_id);
+
+/**
+ * Function with variable number of arguments support that uses static default logger
+ */
+bool rspamd_default_log_function(gint level_flags,
+ const gchar *module, const gchar *id,
+ const gchar *function,
+ const gchar *fmt,
+ ...);
+
+/**
+ * Varargs version of default log function
+ * @param log_level
+ * @param function
+ * @param fmt
+ * @param args
+ */
+bool rspamd_default_logv(gint level_flags,
+ const gchar *module, const gchar *id,
+ const gchar *function,
+ const gchar *fmt,
+ va_list args);
+
+/**
+ * Temporary turn on debug
+ */
+void rspamd_log_debug(rspamd_logger_t *logger);
+
+/**
+ * Turn off debug
+ */
+void rspamd_log_nodebug(rspamd_logger_t *logger);
+
+/**
+ * Return array of counters (4 numbers):
+ * 0 - errors
+ * 1 - warnings
+ * 2 - info messages
+ * 3 - debug messages
+ */
+const guint64 *rspamd_log_counters(rspamd_logger_t *logger);
+
+/**
+ * Returns errors ring buffer as ucl array
+ * @param logger
+ * @return
+ */
+ucl_object_t *rspamd_log_errorbuf_export(const rspamd_logger_t *logger);
+
+/**
+ * Sets new logger functions and initialise logging if needed
+ * @param logger
+ * @param nfuncs
+ * @return static pointer to the old functions (so this function is not reentrant)
+ */
+struct rspamd_logger_funcs *rspamd_logger_set_log_function(rspamd_logger_t *logger,
+ struct rspamd_logger_funcs *nfuncs);
+
+/* Typical functions */
+
+extern guint rspamd_task_log_id;
+#ifdef __cplusplus
+#define RSPAMD_LOG_FUNC __func__
+#else
+#define RSPAMD_LOG_FUNC G_STRFUNC
+#endif
+
+/* Logging in postfix style */
+#define msg_err(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ NULL, NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ NULL, NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ NULL, NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_notice(...) rspamd_default_log_function(G_LOG_LEVEL_MESSAGE, \
+ NULL, NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug(...) rspamd_default_log_function(G_LOG_LEVEL_DEBUG, \
+ NULL, NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+#define debug_task(...) rspamd_conditional_debug_fast(NULL, \
+ task->from_addr, \
+ rspamd_task_log_id, "task", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+/* Use the following macros if you have `task` in the function */
+#define msg_err_task(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ task->task_pool->tag.tagname, task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_err_task_lambda(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ task->task_pool->tag.tagname, task->task_pool->tag.uid, \
+ log_func, \
+ __VA_ARGS__)
+#define msg_warn_task(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ task->task_pool->tag.tagname, task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_notice_task(...) rspamd_default_log_function(G_LOG_LEVEL_MESSAGE, \
+ task->task_pool->tag.tagname, task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_task(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ task->task_pool->tag.tagname, task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_task_lambda(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ task->task_pool->tag.tagname, task->task_pool->tag.uid, \
+ log_func, \
+ __VA_ARGS__)
+#define msg_debug_task(...) rspamd_conditional_debug_fast(NULL, task->from_addr, \
+ rspamd_task_log_id, "task", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_task_lambda(...) rspamd_conditional_debug_fast(NULL, task->from_addr, \
+ rspamd_task_log_id, "task", task->task_pool->tag.uid, \
+ log_func, \
+ __VA_ARGS__)
+#define msg_err_task_encrypted(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL | RSPAMD_LOG_ENCRYPTED, \
+ task->task_pool->tag.tagname, task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_task_encrypted(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING | RSPAMD_LOG_ENCRYPTED, \
+ task->task_pool->tag.tagname, task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_notice_task_encrypted(...) rspamd_default_log_function(G_LOG_LEVEL_MESSAGE | RSPAMD_LOG_ENCRYPTED, \
+ task->task_pool->tag.tagname, task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_task_encrypted(...) rspamd_default_log_function(G_LOG_LEVEL_INFO | RSPAMD_LOG_ENCRYPTED, \
+ task->task_pool->tag.tagname, task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+/* Check for NULL pointer first */
+#define msg_err_task_check(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ task ? task->task_pool->tag.tagname : NULL, task ? task->task_pool->tag.uid : NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_task_check(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ task ? task->task_pool->tag.tagname : NULL, task ? task->task_pool->tag.uid : NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_task_check(...) rspamd_default_log_function(G_LOG_LEVEL_MESSAGE, \
+ task ? task->task_pool->tag.tagname : NULL, task ? task->task_pool->tag.uid : NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_notice_task_check(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ task ? task->task_pool->tag.tagname : NULL, task ? task->task_pool->tag.uid : NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_task_check(...) rspamd_conditional_debug_fast(NULL, \
+ task ? task->from_addr : NULL, \
+ rspamd_task_log_id, "task", task ? task->task_pool->tag.uid : NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+/* Use the following macros if you have `pool` in the function */
+#define msg_err_pool(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ pool->tag.tagname, pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_pool(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ pool->tag.tagname, pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_pool(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ pool->tag.tagname, pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_pool(...) rspamd_conditional_debug(NULL, NULL, \
+ pool->tag.tagname, pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+/* Check for NULL pointer first */
+#define msg_err_pool_check(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ pool ? pool->tag.tagname : NULL, pool ? pool->tag.uid : NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_pool_check(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ pool ? pool->tag.tagname : NULL, pool ? pool->tag.uid : NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_pool_check(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ pool ? pool->tag.tagname : NULL, pool ? pool->tag.uid : NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_pool_check(...) rspamd_conditional_debug(NULL, NULL, \
+ pool ? pool->tag.tagname : NULL, pool ? pool->tag.uid : NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/logger/logger.c b/src/libserver/logger/logger.c
new file mode 100644
index 0000000..2dae632
--- /dev/null
+++ b/src/libserver/logger/logger.c
@@ -0,0 +1,1319 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "logger.h"
+#include "rspamd.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
+#include "ottery.h"
+#include "unix-std.h"
+#include "logger_private.h"
+
+
+static rspamd_logger_t *default_logger = NULL;
+static rspamd_logger_t *emergency_logger = NULL;
+static struct rspamd_log_modules *log_modules = NULL;
+
+static const gchar lf_chr = '\n';
+
+guint rspamd_task_log_id = (guint) -1;
+RSPAMD_CONSTRUCTOR(rspamd_task_log_init)
+{
+ rspamd_task_log_id = rspamd_logger_add_debug_module("task");
+}
+
+rspamd_logger_t *
+rspamd_log_default_logger(void)
+{
+ return default_logger;
+}
+
+rspamd_logger_t *
+rspamd_log_emergency_logger(void)
+{
+ return emergency_logger;
+}
+
+void rspamd_log_set_log_level(rspamd_logger_t *logger, gint level)
+{
+ if (logger == NULL) {
+ logger = default_logger;
+ }
+
+ logger->log_level = level;
+}
+
+gint rspamd_log_get_log_level(rspamd_logger_t *logger)
+{
+ if (logger == NULL) {
+ logger = default_logger;
+ }
+
+ return logger->log_level;
+}
+
+void rspamd_log_set_log_flags(rspamd_logger_t *logger, gint flags)
+{
+ g_assert(logger != NULL);
+
+ logger->flags = flags;
+}
+
+void rspamd_log_close(rspamd_logger_t *logger)
+{
+ g_assert(logger != NULL);
+
+ if (logger->closed) {
+ return;
+ }
+
+ logger->closed = TRUE;
+
+ if (logger->debug_ip) {
+ rspamd_map_helper_destroy_radix(logger->debug_ip);
+ }
+
+ if (logger->pk) {
+ rspamd_pubkey_unref(logger->pk);
+ }
+
+ if (logger->keypair) {
+ rspamd_keypair_unref(logger->keypair);
+ }
+
+ logger->ops.dtor(logger, logger->ops.specific);
+
+ /* TODO: Do we really need that ? */
+ if (logger == default_logger) {
+ default_logger = NULL;
+ }
+
+ if (logger == emergency_logger) {
+ emergency_logger = NULL;
+ }
+
+ if (!logger->pool) {
+ g_free(logger);
+ }
+}
+
+bool rspamd_log_reopen(rspamd_logger_t *rspamd_log, struct rspamd_config *cfg,
+ uid_t uid, gid_t gid)
+{
+ void *nspec;
+ GError *err = NULL;
+
+ g_assert(rspamd_log != NULL);
+
+ nspec = rspamd_log->ops.reload(rspamd_log, cfg, rspamd_log->ops.specific,
+ uid, gid, &err);
+
+ if (nspec != NULL) {
+ rspamd_log->ops.specific = nspec;
+ }
+ else {
+ }
+
+ return nspec != NULL;
+}
+
+static void
+rspamd_emergency_logger_dtor(gpointer d)
+{
+ rspamd_logger_t *logger = (rspamd_logger_t *) d;
+
+ rspamd_log_close(logger);
+}
+
+rspamd_logger_t *
+rspamd_log_open_emergency(rspamd_mempool_t *pool, gint flags)
+{
+ rspamd_logger_t *logger;
+ GError *err = NULL;
+
+ g_assert(default_logger == NULL);
+ g_assert(emergency_logger == NULL);
+
+ if (pool) {
+ logger = rspamd_mempool_alloc0(pool, sizeof(rspamd_logger_t));
+ logger->mtx = rspamd_mempool_get_mutex(pool);
+ }
+ else {
+ logger = g_malloc0(sizeof(rspamd_logger_t));
+ }
+
+ logger->flags = flags;
+ logger->pool = pool;
+ logger->process_type = "main";
+ logger->pid = getpid();
+
+ const struct rspamd_logger_funcs *funcs = &console_log_funcs;
+ memcpy(&logger->ops, funcs, sizeof(*funcs));
+
+ logger->ops.specific = logger->ops.init(logger, NULL, -1, -1, &err);
+
+ if (logger->ops.specific == NULL) {
+ rspamd_fprintf(stderr, "fatal error: cannot init console logging: %e\n",
+ err);
+ g_error_free(err);
+
+ exit(EXIT_FAILURE);
+ }
+
+ default_logger = logger;
+ emergency_logger = logger;
+
+ rspamd_mempool_add_destructor(pool, rspamd_emergency_logger_dtor,
+ emergency_logger);
+
+ return logger;
+}
+
+rspamd_logger_t *
+rspamd_log_open_specific(rspamd_mempool_t *pool,
+ struct rspamd_config *cfg,
+ const gchar *ptype,
+ uid_t uid, gid_t gid)
+{
+ rspamd_logger_t *logger;
+ GError *err = NULL;
+
+ if (pool) {
+ logger = rspamd_mempool_alloc0(pool, sizeof(rspamd_logger_t));
+ logger->mtx = rspamd_mempool_get_mutex(pool);
+ }
+ else {
+ logger = g_malloc0(sizeof(rspamd_logger_t));
+ }
+
+ logger->pool = pool;
+
+ if (cfg) {
+ if (cfg->log_error_elts > 0 && pool) {
+ logger->errlog = rspamd_mempool_alloc0_shared(pool,
+ sizeof(*logger->errlog));
+ logger->errlog->pool = pool;
+ logger->errlog->max_elts = cfg->log_error_elts;
+ logger->errlog->elt_len = cfg->log_error_elt_maxlen;
+ logger->errlog->elts = rspamd_mempool_alloc0_shared(pool,
+ sizeof(struct rspamd_logger_error_elt) * cfg->log_error_elts +
+ cfg->log_error_elt_maxlen * cfg->log_error_elts);
+ }
+
+ logger->log_level = cfg->log_level;
+ logger->flags = cfg->log_flags;
+
+ if (!(logger->flags & RSPAMD_LOG_FLAG_ENFORCED)) {
+ logger->log_level = cfg->log_level;
+ }
+ }
+
+ const struct rspamd_logger_funcs *funcs = NULL;
+
+ if (cfg) {
+ switch (cfg->log_type) {
+ case RSPAMD_LOG_CONSOLE:
+ funcs = &console_log_funcs;
+ break;
+ case RSPAMD_LOG_SYSLOG:
+ funcs = &syslog_log_funcs;
+ break;
+ case RSPAMD_LOG_FILE:
+ funcs = &file_log_funcs;
+ break;
+ }
+ }
+ else {
+ funcs = &console_log_funcs;
+ }
+
+ g_assert(funcs != NULL);
+ memcpy(&logger->ops, funcs, sizeof(*funcs));
+
+ logger->ops.specific = logger->ops.init(logger, cfg, uid, gid, &err);
+
+ if (emergency_logger && logger->ops.specific == NULL) {
+ rspamd_common_log_function(emergency_logger, G_LOG_LEVEL_CRITICAL,
+ "logger", NULL, G_STRFUNC,
+ "cannot open specific logger: %e", err);
+ g_error_free(err);
+
+ return NULL;
+ }
+
+ logger->pid = getpid();
+ logger->process_type = ptype;
+ logger->enabled = TRUE;
+
+ /* Set up conditional logging */
+ if (cfg) {
+ if (cfg->debug_ip_map != NULL) {
+ /* Try to add it as map first of all */
+ if (logger->debug_ip) {
+ rspamd_map_helper_destroy_radix(logger->debug_ip);
+ }
+
+ logger->debug_ip = NULL;
+ rspamd_config_radix_from_ucl(cfg,
+ cfg->debug_ip_map,
+ "IP addresses for which debug logs are enabled",
+ &logger->debug_ip,
+ NULL,
+ NULL, "debug ip");
+ }
+
+ if (cfg->log_encryption_key) {
+ logger->pk = rspamd_pubkey_ref(cfg->log_encryption_key);
+ logger->keypair = rspamd_keypair_new(RSPAMD_KEYPAIR_KEX,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+ rspamd_pubkey_calculate_nm(logger->pk, logger->keypair);
+ }
+ }
+
+ default_logger = logger;
+
+ return logger;
+}
+
+
+/**
+ * Used after fork() for updating structure params
+ */
+void rspamd_log_on_fork(GQuark ptype, struct rspamd_config *cfg,
+ rspamd_logger_t *logger)
+{
+ logger->pid = getpid();
+ logger->process_type = g_quark_to_string(ptype);
+
+ if (logger->ops.on_fork) {
+ GError *err = NULL;
+
+ bool ret = logger->ops.on_fork(logger, cfg, logger->ops.specific, &err);
+
+ if (!ret && emergency_logger) {
+ rspamd_common_log_function(emergency_logger, G_LOG_LEVEL_CRITICAL,
+ "logger", NULL, G_STRFUNC,
+ "cannot update logging on fork: %e", err);
+ g_error_free(err);
+ }
+ }
+}
+
+inline gboolean
+rspamd_logger_need_log(rspamd_logger_t *rspamd_log, GLogLevelFlags log_level,
+ gint module_id)
+{
+ g_assert(rspamd_log != NULL);
+
+ if ((log_level & RSPAMD_LOG_FORCED) ||
+ (log_level & (RSPAMD_LOG_LEVEL_MASK & G_LOG_LEVEL_MASK)) <= rspamd_log->log_level) {
+ return TRUE;
+ }
+
+ if (module_id != -1 && isset(log_modules->bitset, module_id)) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static gchar *
+rspamd_log_encrypt_message(const gchar *begin, const gchar *end, gsize *enc_len,
+ rspamd_logger_t *rspamd_log)
+{
+ guchar *out;
+ gchar *b64;
+ guchar *p, *nonce, *mac;
+ const guchar *comp;
+ guint len, inlen;
+
+ g_assert(end > begin);
+ /* base64 (pubkey | nonce | message) */
+ inlen = rspamd_cryptobox_nonce_bytes(RSPAMD_CRYPTOBOX_MODE_25519) +
+ rspamd_cryptobox_pk_bytes(RSPAMD_CRYPTOBOX_MODE_25519) +
+ rspamd_cryptobox_mac_bytes(RSPAMD_CRYPTOBOX_MODE_25519) +
+ (end - begin);
+ out = g_malloc(inlen);
+
+ p = out;
+ comp = rspamd_pubkey_get_pk(rspamd_log->pk, &len);
+ memcpy(p, comp, len);
+ p += len;
+ ottery_rand_bytes(p, rspamd_cryptobox_nonce_bytes(RSPAMD_CRYPTOBOX_MODE_25519));
+ nonce = p;
+ p += rspamd_cryptobox_nonce_bytes(RSPAMD_CRYPTOBOX_MODE_25519);
+ mac = p;
+ p += rspamd_cryptobox_mac_bytes(RSPAMD_CRYPTOBOX_MODE_25519);
+ memcpy(p, begin, end - begin);
+ comp = rspamd_pubkey_get_nm(rspamd_log->pk, rspamd_log->keypair);
+ g_assert(comp != NULL);
+ rspamd_cryptobox_encrypt_nm_inplace(p, end - begin, nonce, comp, mac,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+ b64 = rspamd_encode_base64(out, inlen, 0, enc_len);
+ g_free(out);
+
+ return b64;
+}
+
+static void
+rspamd_log_write_ringbuffer(rspamd_logger_t *rspamd_log,
+ const gchar *module, const gchar *id,
+ const gchar *data, glong len)
+{
+ guint32 row_num;
+ struct rspamd_logger_error_log *elog;
+ struct rspamd_logger_error_elt *elt;
+
+ if (!rspamd_log->errlog) {
+ return;
+ }
+
+ elog = rspamd_log->errlog;
+
+ g_atomic_int_compare_and_exchange(&elog->cur_row, elog->max_elts, 0);
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30))
+ row_num = g_atomic_int_add(&elog->cur_row, 1);
+#else
+ row_num = g_atomic_int_exchange_and_add(&elog->cur_row, 1);
+#endif
+
+ if (row_num < elog->max_elts) {
+ elt = (struct rspamd_logger_error_elt *) (((guchar *) elog->elts) +
+ (sizeof(*elt) + elog->elt_len) * row_num);
+ g_atomic_int_set(&elt->completed, 0);
+ }
+ else {
+ /* Race condition */
+ elog->cur_row = 0;
+ return;
+ }
+
+ elt->pid = rspamd_log->pid;
+ elt->ptype = g_quark_from_string(rspamd_log->process_type);
+ elt->ts = rspamd_get_calendar_ticks();
+
+ if (id) {
+ rspamd_strlcpy(elt->id, id, sizeof(elt->id));
+ }
+ else {
+ rspamd_strlcpy(elt->id, "", sizeof(elt->id));
+ }
+
+ if (module) {
+ rspamd_strlcpy(elt->module, module, sizeof(elt->module));
+ }
+ else {
+ rspamd_strlcpy(elt->module, "", sizeof(elt->module));
+ }
+
+ rspamd_strlcpy(elt->message, data, MIN(len + 1, elog->elt_len));
+ g_atomic_int_set(&elt->completed, 1);
+}
+
+bool rspamd_common_logv(rspamd_logger_t *rspamd_log, gint level_flags,
+ const gchar *module, const gchar *id, const gchar *function,
+ const gchar *fmt, va_list args)
+{
+ gchar *end;
+ gint level = level_flags & (RSPAMD_LOG_LEVEL_MASK & G_LOG_LEVEL_MASK), mod_id;
+ bool ret = false;
+ gchar logbuf[RSPAMD_LOGBUF_SIZE], *log_line;
+ gsize nescaped;
+
+ if (G_UNLIKELY(rspamd_log == NULL)) {
+ rspamd_log = default_logger;
+ }
+
+ log_line = logbuf;
+
+ if (G_UNLIKELY(rspamd_log == NULL)) {
+ /* Just fprintf message to stderr */
+ if (level >= G_LOG_LEVEL_INFO) {
+ end = rspamd_vsnprintf(logbuf, sizeof(logbuf), fmt, args);
+ rspamd_fprintf(stderr, "%*s\n", (gint) (end - log_line),
+ log_line);
+ }
+ }
+ else {
+ if (level == G_LOG_LEVEL_DEBUG) {
+ mod_id = rspamd_logger_add_debug_module(module);
+ }
+ else {
+ mod_id = -1;
+ }
+
+ if (rspamd_logger_need_log(rspamd_log, level_flags, mod_id)) {
+ end = rspamd_vsnprintf(logbuf, sizeof(logbuf), fmt, args);
+
+ if (!(rspamd_log->flags & RSPAMD_LOG_FLAG_RSPAMADM)) {
+ if ((nescaped = rspamd_log_line_need_escape(logbuf, end - logbuf)) != 0) {
+ gsize unescaped_len = end - logbuf;
+ gchar *logbuf_escaped = g_alloca(unescaped_len + nescaped * 4);
+ log_line = logbuf_escaped;
+
+ end = rspamd_log_line_hex_escape(logbuf, unescaped_len,
+ logbuf_escaped, unescaped_len + nescaped * 4);
+ }
+ }
+
+ if ((level_flags & RSPAMD_LOG_ENCRYPTED) && rspamd_log->pk) {
+ gchar *encrypted;
+ gsize enc_len;
+
+ encrypted = rspamd_log_encrypt_message(log_line, end, &enc_len,
+ rspamd_log);
+ ret = rspamd_log->ops.log(module, id,
+ function,
+ level_flags,
+ encrypted,
+ enc_len,
+ rspamd_log,
+ rspamd_log->ops.specific);
+ g_free(encrypted);
+ }
+ else {
+ ret = rspamd_log->ops.log(module, id,
+ function,
+ level_flags,
+ log_line,
+ end - log_line,
+ rspamd_log,
+ rspamd_log->ops.specific);
+ }
+
+ switch (level) {
+ case G_LOG_LEVEL_CRITICAL:
+ rspamd_log->log_cnt[0]++;
+ rspamd_log_write_ringbuffer(rspamd_log, module, id, log_line,
+ end - log_line);
+ break;
+ case G_LOG_LEVEL_WARNING:
+ rspamd_log->log_cnt[1]++;
+ break;
+ case G_LOG_LEVEL_INFO:
+ rspamd_log->log_cnt[2]++;
+ break;
+ case G_LOG_LEVEL_DEBUG:
+ rspamd_log->log_cnt[3]++;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ return ret;
+}
+
+/**
+ * This log functions select real logger and write message if level is less or equal to configured log level
+ */
+bool rspamd_common_log_function(rspamd_logger_t *rspamd_log,
+ gint level_flags,
+ const gchar *module, const gchar *id,
+ const gchar *function,
+ const gchar *fmt,
+ ...)
+{
+ va_list vp;
+
+ va_start(vp, fmt);
+ bool ret = rspamd_common_logv(rspamd_log, level_flags, module, id, function, fmt, vp);
+ va_end(vp);
+
+ return ret;
+}
+
+bool rspamd_default_logv(gint level_flags, const gchar *module, const gchar *id,
+ const gchar *function,
+ const gchar *fmt, va_list args)
+{
+ return rspamd_common_logv(NULL, level_flags, module, id, function, fmt, args);
+}
+
+bool rspamd_default_log_function(gint level_flags,
+ const gchar *module, const gchar *id,
+ const gchar *function, const gchar *fmt, ...)
+{
+
+ va_list vp;
+
+ va_start(vp, fmt);
+ bool ret = rspamd_default_logv(level_flags, module, id, function, fmt, vp);
+ va_end(vp);
+
+ return ret;
+}
+
+
+/**
+ * Main file interface for logging
+ */
+/**
+ * Write log line depending on ip
+ */
+bool rspamd_conditional_debug(rspamd_logger_t *rspamd_log,
+ rspamd_inet_addr_t *addr, const gchar *module, const gchar *id,
+ const gchar *function, const gchar *fmt, ...)
+{
+ static gchar logbuf[LOGBUF_LEN];
+ va_list vp;
+ gchar *end;
+ gint mod_id;
+
+ if (rspamd_log == NULL) {
+ rspamd_log = default_logger;
+ }
+
+ mod_id = rspamd_logger_add_debug_module(module);
+
+ if (rspamd_logger_need_log(rspamd_log, G_LOG_LEVEL_DEBUG, mod_id) ||
+ rspamd_log->is_debug) {
+ if (rspamd_log->debug_ip && addr != NULL) {
+ if (rspamd_match_radix_map_addr(rspamd_log->debug_ip,
+ addr) == NULL) {
+ return false;
+ }
+ }
+
+ va_start(vp, fmt);
+ end = rspamd_vsnprintf(logbuf, sizeof(logbuf), fmt, vp);
+ *end = '\0';
+ va_end(vp);
+ return rspamd_log->ops.log(module, id,
+ function,
+ G_LOG_LEVEL_DEBUG | RSPAMD_LOG_FORCED,
+ logbuf,
+ end - logbuf,
+ rspamd_log,
+ rspamd_log->ops.specific);
+ }
+
+ return false;
+}
+
+bool rspamd_conditional_debug_fast(rspamd_logger_t *rspamd_log,
+ rspamd_inet_addr_t *addr,
+ gint mod_id, const gchar *module, const gchar *id,
+ const gchar *function, const gchar *fmt, ...)
+{
+ static gchar logbuf[LOGBUF_LEN];
+ va_list vp;
+ gchar *end;
+
+ if (rspamd_log == NULL) {
+ rspamd_log = default_logger;
+ }
+
+ if (rspamd_logger_need_log(rspamd_log, G_LOG_LEVEL_DEBUG, mod_id) ||
+ rspamd_log->is_debug) {
+ if (rspamd_log->debug_ip && addr != NULL) {
+ if (rspamd_match_radix_map_addr(rspamd_log->debug_ip, addr) == NULL) {
+ return false;
+ }
+ }
+
+ va_start(vp, fmt);
+ end = rspamd_vsnprintf(logbuf, sizeof(logbuf), fmt, vp);
+ *end = '\0';
+ va_end(vp);
+ return rspamd_log->ops.log(module, id,
+ function,
+ G_LOG_LEVEL_DEBUG | RSPAMD_LOG_FORCED,
+ logbuf,
+ end - logbuf,
+ rspamd_log,
+ rspamd_log->ops.specific);
+ }
+
+ return false;
+}
+
+bool rspamd_conditional_debug_fast_num_id(rspamd_logger_t *rspamd_log,
+ rspamd_inet_addr_t *addr,
+ gint mod_id, const gchar *module, guint64 id,
+ const gchar *function, const gchar *fmt, ...)
+{
+ static gchar logbuf[LOGBUF_LEN], idbuf[64];
+ va_list vp;
+ gchar *end;
+
+ if (rspamd_log == NULL) {
+ rspamd_log = default_logger;
+ }
+
+ if (rspamd_logger_need_log(rspamd_log, G_LOG_LEVEL_DEBUG, mod_id) ||
+ rspamd_log->is_debug) {
+ if (rspamd_log->debug_ip && addr != NULL) {
+ if (rspamd_match_radix_map_addr(rspamd_log->debug_ip, addr) == NULL) {
+ return false;
+ }
+ }
+
+ rspamd_snprintf(idbuf, sizeof(idbuf), "%XuL", id);
+ va_start(vp, fmt);
+ end = rspamd_vsnprintf(logbuf, sizeof(logbuf), fmt, vp);
+ *end = '\0';
+ va_end(vp);
+ return rspamd_log->ops.log(module, idbuf,
+ function,
+ G_LOG_LEVEL_DEBUG | RSPAMD_LOG_FORCED,
+ logbuf,
+ end - logbuf,
+ rspamd_log,
+ rspamd_log->ops.specific);
+ }
+
+ return false;
+}
+
+/**
+ * Wrapper for glib logger
+ */
+void rspamd_glib_log_function(const gchar *log_domain,
+ GLogLevelFlags log_level,
+ const gchar *message,
+ gpointer arg)
+{
+ rspamd_logger_t *rspamd_log = (rspamd_logger_t *) arg;
+
+ if (rspamd_log->enabled &&
+ rspamd_logger_need_log(rspamd_log, log_level, -1)) {
+ rspamd_log->ops.log("glib", NULL,
+ NULL,
+ log_level,
+ message,
+ strlen(message),
+ rspamd_log,
+ rspamd_log->ops.specific);
+ }
+}
+
+void rspamd_glib_printerr_function(const gchar *message)
+{
+ rspamd_common_log_function(NULL, G_LOG_LEVEL_CRITICAL, "glib",
+ NULL, G_STRFUNC,
+ "%s", message);
+}
+
+/**
+ * Temporary turn on debugging
+ */
+void rspamd_log_debug(rspamd_logger_t *rspamd_log)
+{
+ rspamd_log->is_debug = TRUE;
+}
+
+/**
+ * Turn off temporary debugging
+ */
+void rspamd_log_nodebug(rspamd_logger_t *rspamd_log)
+{
+ rspamd_log->is_debug = FALSE;
+}
+
+const guint64 *
+rspamd_log_counters(rspamd_logger_t *logger)
+{
+ if (logger) {
+ return logger->log_cnt;
+ }
+
+ return NULL;
+}
+
+static gint
+rspamd_log_errlog_cmp(const ucl_object_t **o1, const ucl_object_t **o2)
+{
+ const ucl_object_t *ts1, *ts2;
+
+ ts1 = ucl_object_lookup(*o1, "ts");
+ ts2 = ucl_object_lookup(*o2, "ts");
+
+ if (ts1 && ts2) {
+ gdouble t1 = ucl_object_todouble(ts1), t2 = ucl_object_todouble(ts2);
+
+ if (t1 > t2) {
+ return -1;
+ }
+ else if (t2 > t1) {
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+ucl_object_t *
+rspamd_log_errorbuf_export(const rspamd_logger_t *logger)
+{
+ struct rspamd_logger_error_elt *cpy, *cur;
+ ucl_object_t *top = ucl_object_typed_new(UCL_ARRAY);
+ guint i;
+
+ if (logger->errlog == NULL) {
+ return top;
+ }
+
+ cpy = g_malloc0_n(logger->errlog->max_elts,
+ sizeof(*cpy) + logger->errlog->elt_len);
+ memcpy(cpy, logger->errlog->elts, logger->errlog->max_elts * (sizeof(*cpy) + logger->errlog->elt_len));
+
+ for (i = 0; i < logger->errlog->max_elts; i++) {
+ cur = (struct rspamd_logger_error_elt *) ((guchar *) cpy +
+ i * ((sizeof(*cpy) + logger->errlog->elt_len)));
+ if (cur->completed) {
+ ucl_object_t *obj = ucl_object_typed_new(UCL_OBJECT);
+
+ ucl_object_insert_key(obj, ucl_object_fromdouble(cur->ts),
+ "ts", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromint(cur->pid),
+ "pid", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromstring(g_quark_to_string(cur->ptype)),
+ "type", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromstring(cur->id),
+ "id", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromstring(cur->module),
+ "module", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromstring(cur->message),
+ "message", 0, false);
+
+ ucl_array_append(top, obj);
+ }
+ }
+
+ ucl_object_array_sort(top, rspamd_log_errlog_cmp);
+ g_free(cpy);
+
+ return top;
+}
+
+static guint
+rspamd_logger_allocate_mod_bit(void)
+{
+ if (log_modules->bitset_allocated * NBBY > log_modules->bitset_len + 1) {
+ log_modules->bitset_len++;
+ return log_modules->bitset_len - 1;
+ }
+ else {
+ /* Need to expand */
+ log_modules->bitset_allocated *= 2;
+ log_modules->bitset = g_realloc(log_modules->bitset,
+ log_modules->bitset_allocated);
+
+ return rspamd_logger_allocate_mod_bit();
+ }
+}
+
+RSPAMD_DESTRUCTOR(rspamd_debug_modules_dtor)
+{
+ if (log_modules) {
+ g_hash_table_unref(log_modules->modules);
+ g_free(log_modules->bitset);
+ g_free(log_modules);
+ }
+}
+
+gint rspamd_logger_add_debug_module(const gchar *mname)
+{
+ struct rspamd_log_module *m;
+
+ if (mname == NULL) {
+ return -1;
+ }
+
+ if (log_modules == NULL) {
+ /*
+ * This is usually called from constructors, so we call init check
+ * each time to avoid dependency issues between ctors calls
+ */
+ log_modules = g_malloc0(sizeof(*log_modules));
+ log_modules->modules = g_hash_table_new_full(rspamd_strcase_hash,
+ rspamd_strcase_equal, g_free, g_free);
+ log_modules->bitset_allocated = 16;
+ log_modules->bitset_len = 0;
+ log_modules->bitset = g_malloc0(log_modules->bitset_allocated);
+ }
+
+ if ((m = g_hash_table_lookup(log_modules->modules, mname)) == NULL) {
+ m = g_malloc0(sizeof(*m));
+ m->mname = g_strdup(mname);
+ m->id = rspamd_logger_allocate_mod_bit();
+ clrbit(log_modules->bitset, m->id);
+ g_hash_table_insert(log_modules->modules, m->mname, m);
+ }
+
+ return m->id;
+}
+
+void rspamd_logger_configure_modules(GHashTable *mods_enabled)
+{
+ GHashTableIter it;
+ gpointer k, v;
+ guint id;
+
+ /* Clear all in bitset_allocated -> this are bytes not bits */
+ memset(log_modules->bitset, 0, log_modules->bitset_allocated);
+ /* On first iteration, we go through all modules enabled and add missing ones */
+ g_hash_table_iter_init(&it, mods_enabled);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ rspamd_logger_add_debug_module((const gchar *) k);
+ }
+
+ g_hash_table_iter_init(&it, mods_enabled);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ id = rspamd_logger_add_debug_module((const gchar *) k);
+
+ if (isclr(log_modules->bitset, id)) {
+ msg_info("enable debugging for module %s (%d)", (const gchar *) k,
+ id);
+ setbit(log_modules->bitset, id);
+ }
+ }
+}
+
+struct rspamd_logger_funcs *
+rspamd_logger_set_log_function(rspamd_logger_t *logger,
+ struct rspamd_logger_funcs *nfuncs)
+{
+ /* TODO: write this */
+
+ return NULL;
+}
+
+
+gchar *
+rspamd_log_line_hex_escape(const guchar *src, gsize srclen,
+ gchar *dst, gsize dstlen)
+{
+ static const gchar hexdigests[16] = "0123456789ABCDEF";
+ gchar *d = dst;
+
+ static guint32 escape[] = {
+ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */
+
+ /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */
+ 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0100 */
+
+ /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */
+ 0x00000000, /* 0001 0000 0000 0000 0000 0000 0000 0000 */
+
+ /* ~}| {zyx wvut srqp onml kjih gfed cba` */
+ 0x80000000, /* 1000 0000 0000 0000 0000 0000 0000 0000 */
+
+ /* Allow all 8bit characters (assuming they are valid utf8) */
+ 0x00000000,
+ 0x00000000,
+ 0x00000000,
+ 0x00000000,
+ };
+
+ while (srclen && dstlen) {
+ if (escape[*src >> 5] & (1U << (*src & 0x1f))) {
+ if (dstlen >= 4) {
+ *d++ = '\\';
+ *d++ = 'x';
+ *d++ = hexdigests[*src >> 4];
+ *d++ = hexdigests[*src & 0xf];
+ src++;
+ dstlen -= 4;
+ }
+ else {
+ /* Overflow */
+ break;
+ }
+ }
+ else {
+ *d++ = *src++;
+ dstlen--;
+ }
+
+ srclen--;
+ }
+
+ return d;
+}
+
+gsize rspamd_log_line_need_escape(const guchar *src, gsize srclen)
+{
+ static guint32 escape[] = {
+ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */
+
+ /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */
+ 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0100 */
+
+ /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */
+ 0x00000000, /* 0001 0000 0000 0000 0000 0000 0000 0000 */
+
+ /* ~}| {zyx wvut srqp onml kjih gfed cba` */
+ 0x80000000, /* 1000 0000 0000 0000 0000 0000 0000 0000 */
+
+ /* Allow all 8bit characters (assuming they are valid utf8) */
+ 0x00000000,
+ 0x00000000,
+ 0x00000000,
+ 0x00000000,
+ };
+ gsize n = 0;
+
+ while (srclen) {
+ if (escape[*src >> 5] & (1U << (*src & 0x1f))) {
+ n++;
+ }
+
+ src++;
+ srclen--;
+ }
+
+ return n;
+}
+
+const gchar *
+rspamd_get_log_severity_string(gint level_flags)
+{
+ unsigned int bitnum;
+ static const char *level_strs[G_LOG_LEVEL_USER_SHIFT] = {
+ "", /* G_LOG_FLAG_RECURSION */
+ "", /* G_LOG_FLAG_FATAL */
+ "crit",
+ "error",
+ "warn",
+ "notice",
+ "info",
+ "debug"};
+ level_flags &= ((1u << G_LOG_LEVEL_USER_SHIFT) - 1u) & ~(G_LOG_FLAG_RECURSION | G_LOG_FLAG_FATAL);
+#ifdef __GNUC__
+ /* We assume gcc >= 3 and clang >= 5 anyway */
+ bitnum = __builtin_ffs(level_flags) - 1;
+#else
+ bitnum = ffs(level_flags) - 1;
+#endif
+ return level_strs[bitnum];
+}
+
+static inline void
+log_time(gdouble now, rspamd_logger_t *rspamd_log, gchar *timebuf,
+ size_t len)
+{
+ time_t sec = (time_t) now;
+ gsize r;
+ struct tm tms;
+
+ rspamd_localtime(sec, &tms);
+ r = strftime(timebuf, len, "%F %H:%M:%S", &tms);
+
+ if (rspamd_log->flags & RSPAMD_LOG_FLAG_USEC) {
+ gchar usec_buf[16];
+
+ rspamd_snprintf(usec_buf, sizeof(usec_buf), "%.5f",
+ now - (gdouble) sec);
+ rspamd_snprintf(timebuf + r, len - r,
+ "%s", usec_buf + 1);
+ }
+}
+
+void rspamd_log_fill_iov(struct rspamd_logger_iov_ctx *iov_ctx,
+ double ts,
+ const gchar *module,
+ const gchar *id,
+ const gchar *function,
+ gint level_flags,
+ const gchar *message,
+ gsize mlen,
+ rspamd_logger_t *logger)
+{
+ bool log_color = (logger->flags & RSPAMD_LOG_FLAG_COLOR);
+ bool log_severity = (logger->flags & RSPAMD_LOG_FLAG_SEVERITY);
+ bool log_rspamadm = (logger->flags & RSPAMD_LOG_FLAG_RSPAMADM);
+ bool log_systemd = (logger->flags & RSPAMD_LOG_FLAG_SYSTEMD);
+ bool log_json = (logger->flags & RSPAMD_LOG_FLAG_JSON);
+
+ if (log_json) {
+ /* Some sanity to avoid too many branches */
+ log_color = false;
+ log_severity = true;
+ log_systemd = false;
+ }
+
+ glong r;
+ static gchar timebuf[64], modulebuf[64];
+ static gchar tmpbuf[256];
+
+ if (!log_json && !log_systemd) {
+ log_time(ts, logger, timebuf, sizeof(timebuf));
+ }
+
+ if (G_UNLIKELY(log_json)) {
+ /* Perform JSON logging */
+ guint slen = id ? strlen(id) : strlen("(NULL)");
+ slen = MIN(RSPAMD_LOG_ID_LEN, slen);
+ r = rspamd_snprintf(tmpbuf, sizeof(tmpbuf), "{\"ts\": %f, "
+ "\"pid\": %P, "
+ "\"severity\": \"%s\", "
+ "\"worker_type\": \"%s\", "
+ "\"id\": \"%*.s\", "
+ "\"module\": \"%s\", "
+ "\"function\": \"%s\", "
+ "\"message\": \"",
+ ts,
+ logger->pid,
+ rspamd_get_log_severity_string(level_flags),
+ logger->process_type,
+ slen, id,
+ module,
+ function);
+ iov_ctx->iov[0].iov_base = tmpbuf;
+ iov_ctx->iov[0].iov_len = r;
+ /* TODO: is it possible to have other 'bad' symbols here? */
+ if (rspamd_memcspn(message, "\"\\\r\n\b\t\v", mlen) == mlen) {
+ iov_ctx->iov[1].iov_base = (void *) message;
+ iov_ctx->iov[1].iov_len = mlen;
+ }
+ else {
+ /* We need to do JSON escaping of the quotes */
+ const char *p, *end = message + mlen;
+ long escaped_len;
+
+ for (p = message, escaped_len = 0; p < end; p++, escaped_len++) {
+ switch (*p) {
+ case '\v':
+ case '\0':
+ escaped_len += 5;
+ break;
+ case '\\':
+ case '"':
+ case '\n':
+ case '\r':
+ case '\b':
+ case '\t':
+ escaped_len++;
+ break;
+ default:
+ break;
+ }
+ }
+
+
+ struct rspamd_logger_iov_thrash_stack *thrash_stack_elt = g_malloc(
+ sizeof(struct rspamd_logger_iov_thrash_stack) +
+ escaped_len);
+
+ char *dst = ((char *) thrash_stack_elt) + sizeof(struct rspamd_logger_iov_thrash_stack);
+ char *d;
+
+ thrash_stack_elt->prev = iov_ctx->thrash_stack;
+ iov_ctx->thrash_stack = thrash_stack_elt;
+
+ for (p = message, d = dst; p < end; p++, d++) {
+ switch (*p) {
+ case '\n':
+ *d++ = '\\';
+ *d = 'n';
+ break;
+ case '\r':
+ *d++ = '\\';
+ *d = 'r';
+ break;
+ case '\b':
+ *d++ = '\\';
+ *d = 'b';
+ break;
+ case '\t':
+ *d++ = '\\';
+ *d = 't';
+ break;
+ case '\f':
+ *d++ = '\\';
+ *d = 'f';
+ break;
+ case '\0':
+ *d++ = '\\';
+ *d++ = 'u';
+ *d++ = '0';
+ *d++ = '0';
+ *d++ = '0';
+ *d = '0';
+ break;
+ case '\v':
+ *d++ = '\\';
+ *d++ = 'u';
+ *d++ = '0';
+ *d++ = '0';
+ *d++ = '0';
+ *d = 'B';
+ break;
+ case '\\':
+ *d++ = '\\';
+ *d = '\\';
+ break;
+ case '"':
+ *d++ = '\\';
+ *d = '"';
+ break;
+ default:
+ *d = *p;
+ break;
+ }
+ }
+
+ iov_ctx->iov[1].iov_base = dst;
+ iov_ctx->iov[1].iov_len = d - dst;
+ }
+ iov_ctx->iov[2].iov_base = (void *) "\"}\n";
+ iov_ctx->iov[2].iov_len = sizeof("\"}\n") - 1;
+
+ iov_ctx->niov = 3;
+ }
+ else if (G_LIKELY(!log_rspamadm)) {
+ if (!log_systemd) {
+ if (log_color) {
+ if (level_flags & (G_LOG_LEVEL_INFO | G_LOG_LEVEL_MESSAGE)) {
+ /* White */
+ r = rspamd_snprintf(tmpbuf, sizeof(tmpbuf), "\033[0;37m");
+ }
+ else if (level_flags & G_LOG_LEVEL_WARNING) {
+ /* Magenta */
+ r = rspamd_snprintf(tmpbuf, sizeof(tmpbuf), "\033[0;32m");
+ }
+ else if (level_flags & G_LOG_LEVEL_CRITICAL) {
+ /* Red */
+ r = rspamd_snprintf(tmpbuf, sizeof(tmpbuf), "\033[1;31m");
+ }
+ else {
+ r = 0;
+ }
+ }
+ else {
+ r = 0;
+ }
+
+ if (log_severity) {
+ r += rspamd_snprintf(tmpbuf + r,
+ sizeof(tmpbuf) - r,
+ "%s [%s] #%P(%s) ",
+ timebuf,
+ rspamd_get_log_severity_string(level_flags),
+ logger->pid,
+ logger->process_type);
+ }
+ else {
+ r += rspamd_snprintf(tmpbuf + r,
+ sizeof(tmpbuf) - r,
+ "%s #%P(%s) ",
+ timebuf,
+ logger->pid,
+ logger->process_type);
+ }
+ }
+ else {
+ r = 0;
+ r += rspamd_snprintf(tmpbuf + r,
+ sizeof(tmpbuf) - r,
+ "(%s) ",
+ logger->process_type);
+ }
+
+ glong mremain, mr;
+ char *m;
+
+ modulebuf[0] = '\0';
+ mremain = sizeof(modulebuf);
+ m = modulebuf;
+
+ if (id != NULL) {
+ guint slen = strlen(id);
+ slen = MIN(RSPAMD_LOG_ID_LEN, slen);
+ mr = rspamd_snprintf(m, mremain, "<%*.s>; ", slen,
+ id);
+ m += mr;
+ mremain -= mr;
+ }
+ if (module != NULL) {
+ mr = rspamd_snprintf(m, mremain, "%s; ", module);
+ m += mr;
+ mremain -= mr;
+ }
+ if (function != NULL) {
+ mr = rspamd_snprintf(m, mremain, "%s: ", function);
+ m += mr;
+ mremain -= mr;
+ }
+ else {
+ mr = rspamd_snprintf(m, mremain, ": ");
+ m += mr;
+ mremain -= mr;
+ }
+
+ /* Ensure that we have a space at the end */
+ if (m > modulebuf && *(m - 1) != ' ') {
+ *(m - 1) = ' ';
+ }
+
+ /* Construct IOV for log line */
+ iov_ctx->iov[0].iov_base = tmpbuf;
+ iov_ctx->iov[0].iov_len = r;
+ iov_ctx->iov[1].iov_base = modulebuf;
+ iov_ctx->iov[1].iov_len = m - modulebuf;
+ iov_ctx->iov[2].iov_base = (void *) message;
+ iov_ctx->iov[2].iov_len = mlen;
+ iov_ctx->iov[3].iov_base = (void *) &lf_chr;
+ iov_ctx->iov[3].iov_len = 1;
+
+ iov_ctx->niov = 4;
+
+ if (log_color) {
+ iov_ctx->iov[4].iov_base = "\033[0m";
+ iov_ctx->iov[4].iov_len = sizeof("\033[0m") - 1;
+
+ iov_ctx->niov = 5;
+ }
+ }
+ else {
+ /* Rspamadm case */
+ int niov = 0;
+
+ if (logger->log_level == G_LOG_LEVEL_DEBUG) {
+ iov_ctx->iov[niov].iov_base = (void *) timebuf;
+ iov_ctx->iov[niov++].iov_len = strlen(timebuf);
+ iov_ctx->iov[niov].iov_base = (void *) " ";
+ iov_ctx->iov[niov++].iov_len = 1;
+ }
+
+ iov_ctx->iov[niov].iov_base = (void *) message;
+ iov_ctx->iov[niov++].iov_len = mlen;
+ iov_ctx->iov[niov].iov_base = (void *) &lf_chr;
+ iov_ctx->iov[niov++].iov_len = 1;
+
+ iov_ctx->niov = niov;
+ }
+
+ // this is kind of "after-the-fact" check, but it's mostly for debugging-only
+ g_assert(iov_ctx->niov <= G_N_ELEMENTS(iov_ctx->iov));
+}
+
+void rspamd_log_iov_free(struct rspamd_logger_iov_ctx *iov_ctx)
+{
+ struct rspamd_logger_iov_thrash_stack *st = iov_ctx->thrash_stack;
+
+ while (st) {
+ struct rspamd_logger_iov_thrash_stack *nst = st->prev;
+ g_free(st);
+ st = nst;
+ }
+} \ No newline at end of file
diff --git a/src/libserver/logger/logger_console.c b/src/libserver/logger/logger_console.c
new file mode 100644
index 0000000..7f3c770
--- /dev/null
+++ b/src/libserver/logger/logger_console.c
@@ -0,0 +1,211 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "logger.h"
+#include "libserver/cfg_file.h"
+#include "libcryptobox/cryptobox.h"
+#include "unix-std.h"
+
+#include "logger_private.h"
+
+#define CONSOLE_LOG_QUARK g_quark_from_static_string("console_logger")
+
+static const gchar lf_chr = '\n';
+struct rspamd_console_logger_priv {
+ gint fd;
+ gint crit_fd;
+};
+
+/* Copy & paste :( */
+static inline void
+log_time(gdouble now, rspamd_logger_t *rspamd_log, gchar *timebuf,
+ size_t len)
+{
+ time_t sec = (time_t) now;
+ gsize r;
+ struct tm tms;
+
+ rspamd_localtime(sec, &tms);
+ r = strftime(timebuf, len, "%F %H:%M:%S", &tms);
+
+ if (rspamd_log->flags & RSPAMD_LOG_FLAG_USEC) {
+ gchar usec_buf[16];
+
+ rspamd_snprintf(usec_buf, sizeof(usec_buf), "%.5f",
+ now - (gdouble) sec);
+ rspamd_snprintf(timebuf + r, len - r,
+ "%s", usec_buf + 1);
+ }
+}
+
+void *
+rspamd_log_console_init(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ uid_t uid, gid_t gid, GError **err)
+{
+ struct rspamd_console_logger_priv *priv;
+
+ priv = g_malloc0(sizeof(*priv));
+
+ if (logger->flags & RSPAMD_LOG_FLAG_RSPAMADM) {
+ priv->fd = dup(STDOUT_FILENO);
+ priv->crit_fd = dup(STDERR_FILENO);
+ }
+ else {
+ priv->fd = dup(STDERR_FILENO);
+ priv->crit_fd = priv->fd;
+ }
+
+ if (priv->fd == -1) {
+ g_set_error(err, CONSOLE_LOG_QUARK, errno,
+ "open_log: cannot dup console fd: %s\n",
+ strerror(errno));
+ rspamd_log_console_dtor(logger, priv);
+
+ return NULL;
+ }
+
+ if (!isatty(priv->fd)) {
+ if (logger->flags & RSPAMD_LOG_FLAG_COLOR) {
+ /* Disable colors for not a tty */
+ logger->flags &= ~RSPAMD_LOG_FLAG_COLOR;
+ }
+ }
+
+ return priv;
+}
+
+void *
+rspamd_log_console_reload(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ gpointer arg, uid_t uid, gid_t gid, GError **err)
+{
+ struct rspamd_console_logger_priv *npriv;
+
+ npriv = rspamd_log_console_init(logger, cfg, uid, gid, err);
+
+ if (npriv) {
+ /* Close old */
+ rspamd_log_console_dtor(logger, arg);
+ }
+
+ return npriv;
+}
+
+void rspamd_log_console_dtor(rspamd_logger_t *logger, gpointer arg)
+{
+ struct rspamd_console_logger_priv *priv = (struct rspamd_console_logger_priv *) arg;
+
+ if (priv->fd != -1) {
+ if (priv->fd != priv->crit_fd) {
+ /* Two different FD case */
+ if (close(priv->crit_fd) == -1) {
+ rspamd_fprintf(stderr, "cannot close log crit_fd %d: %s\n",
+ priv->crit_fd, strerror(errno));
+ }
+ }
+
+ if (close(priv->fd) == -1) {
+ rspamd_fprintf(stderr, "cannot close log fd %d: %s\n",
+ priv->fd, strerror(errno));
+ }
+
+ /* Avoid the next if to be executed as crit_fd is equal to fd */
+ priv->crit_fd = -1;
+ }
+
+ if (priv->crit_fd != -1) {
+ if (close(priv->crit_fd) == -1) {
+ rspamd_fprintf(stderr, "cannot close log crit_fd %d: %s\n",
+ priv->crit_fd, strerror(errno));
+ }
+ }
+
+ g_free(priv);
+}
+
+bool rspamd_log_console_log(const gchar *module, const gchar *id,
+ const gchar *function,
+ gint level_flags,
+ const gchar *message,
+ gsize mlen,
+ rspamd_logger_t *rspamd_log,
+ gpointer arg)
+{
+ struct rspamd_console_logger_priv *priv = (struct rspamd_console_logger_priv *) arg;
+ gint fd, r;
+ double now;
+
+ if (level_flags & G_LOG_LEVEL_CRITICAL) {
+ fd = priv->crit_fd;
+ }
+ else {
+ /* Use stderr if we are in rspamadm mode and severity is more than WARNING */
+ if ((rspamd_log->flags & RSPAMD_LOG_FLAG_RSPAMADM) && (level_flags & G_LOG_LEVEL_WARNING)) {
+ fd = priv->crit_fd;
+ }
+ else {
+ fd = priv->fd;
+ }
+ }
+
+#ifndef DISABLE_PTHREAD_MUTEX
+ if (rspamd_log->mtx) {
+ rspamd_mempool_lock_mutex(rspamd_log->mtx);
+ }
+ else {
+ rspamd_file_lock(fd, FALSE);
+ }
+#else
+ rspamd_file_lock(fd, FALSE);
+#endif
+
+ now = rspamd_get_calendar_ticks();
+
+ struct rspamd_logger_iov_ctx iov_ctx;
+ memset(&iov_ctx, 0, sizeof(iov_ctx));
+ rspamd_log_fill_iov(&iov_ctx, now, module, id,
+ function, level_flags, message,
+ mlen, rspamd_log);
+
+again:
+ r = writev(fd, iov_ctx.iov, iov_ctx.niov);
+
+ if (r == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ goto again;
+ }
+
+ if (rspamd_log->mtx) {
+ rspamd_mempool_unlock_mutex(rspamd_log->mtx);
+ }
+ else {
+ rspamd_file_unlock(fd, FALSE);
+ }
+
+ rspamd_log_iov_free(&iov_ctx);
+ return false;
+ }
+
+ if (rspamd_log->mtx) {
+ rspamd_mempool_unlock_mutex(rspamd_log->mtx);
+ }
+ else {
+ rspamd_file_unlock(fd, FALSE);
+ }
+
+ rspamd_log_iov_free(&iov_ctx);
+ return true;
+} \ No newline at end of file
diff --git a/src/libserver/logger/logger_file.c b/src/libserver/logger/logger_file.c
new file mode 100644
index 0000000..20b04b8
--- /dev/null
+++ b/src/libserver/logger/logger_file.c
@@ -0,0 +1,510 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "logger.h"
+#include "libserver/cfg_file.h"
+#include "libcryptobox/cryptobox.h"
+#include "unix-std.h"
+
+#include "logger_private.h"
+
+#define FILE_LOG_QUARK g_quark_from_static_string("file_logger")
+
+struct rspamd_file_logger_priv {
+ gint fd;
+ struct {
+ guint32 size;
+ guint32 used;
+ u_char *buf;
+ } io_buf;
+ gboolean throttling;
+ gchar *log_file;
+ gboolean is_buffered;
+ gboolean log_severity;
+ time_t throttling_time;
+ guint32 repeats;
+ guint64 last_line_cksum;
+ gchar *saved_message;
+ gsize saved_mlen;
+ gchar *saved_function;
+ gchar *saved_module;
+ gchar *saved_id;
+ guint saved_loglevel;
+};
+
+/**
+ * Calculate checksum for log line (used for repeating logic)
+ */
+static inline guint64
+rspamd_log_calculate_cksum(const gchar *message, size_t mlen)
+{
+ return rspamd_cryptobox_fast_hash(message, mlen, rspamd_hash_seed());
+}
+
+/*
+ * Write a line to log file (unbuffered)
+ */
+static bool
+direct_write_log_line(rspamd_logger_t *rspamd_log,
+ struct rspamd_file_logger_priv *priv,
+ void *data,
+ gsize count,
+ gboolean is_iov,
+ gint level_flags)
+{
+ struct iovec *iov;
+ const gchar *line;
+ glong r;
+ gint fd;
+ gboolean locked = FALSE;
+
+ iov = (struct iovec *) data;
+ fd = priv->fd;
+
+ if (!rspamd_log->no_lock) {
+ gsize tlen;
+
+ if (is_iov) {
+ tlen = 0;
+
+ for (guint i = 0; i < count; i++) {
+ tlen += iov[i].iov_len;
+ }
+ }
+ else {
+ tlen = count;
+ }
+
+ if (tlen > PIPE_BUF) {
+ locked = TRUE;
+
+#ifndef DISABLE_PTHREAD_MUTEX
+ if (rspamd_log->mtx) {
+ rspamd_mempool_lock_mutex(rspamd_log->mtx);
+ }
+ else {
+ rspamd_file_lock(fd, FALSE);
+ }
+#else
+ rspamd_file_lock(fd, FALSE);
+#endif
+ }
+ }
+
+ if (is_iov) {
+ r = writev(fd, iov, count);
+ }
+ else {
+ line = (const gchar *) data;
+ r = write(fd, line, count);
+ }
+
+ if (locked) {
+#ifndef DISABLE_PTHREAD_MUTEX
+ if (rspamd_log->mtx) {
+ rspamd_mempool_unlock_mutex(rspamd_log->mtx);
+ }
+ else {
+ rspamd_file_unlock(fd, FALSE);
+ }
+#else
+ rspamd_file_unlock(fd, FALSE);
+#endif
+ }
+
+ if (r == -1) {
+ /* We cannot write message to file, so we need to detect error and make decision */
+ if (errno == EINTR) {
+ /* Try again */
+ return direct_write_log_line(rspamd_log, priv, data, count, is_iov, level_flags);
+ }
+
+ if (errno == EFAULT || errno == EINVAL || errno == EFBIG ||
+ errno == ENOSPC) {
+ /* Rare case */
+ priv->throttling = TRUE;
+ priv->throttling_time = time(NULL);
+ }
+ else if (errno == EPIPE || errno == EBADF) {
+ /* We write to some pipe and it disappears, disable logging or we has opened bad file descriptor */
+ rspamd_log->enabled = FALSE;
+ }
+
+ return false;
+ }
+ else if (priv->throttling) {
+ priv->throttling = FALSE;
+ }
+
+ return true;
+}
+
+/**
+ * Fill buffer with message (limits must be checked BEFORE this call)
+ */
+static void
+fill_buffer(rspamd_logger_t *rspamd_log,
+ struct rspamd_file_logger_priv *priv,
+ const struct iovec *iov, gint iovcnt)
+{
+ gint i;
+
+ for (i = 0; i < iovcnt; i++) {
+ memcpy(priv->io_buf.buf + priv->io_buf.used,
+ iov[i].iov_base,
+ iov[i].iov_len);
+ priv->io_buf.used += iov[i].iov_len;
+ }
+}
+
+static void
+rspamd_log_flush(rspamd_logger_t *rspamd_log, struct rspamd_file_logger_priv *priv)
+{
+ if (priv->is_buffered) {
+ direct_write_log_line(rspamd_log,
+ priv,
+ priv->io_buf.buf,
+ priv->io_buf.used,
+ FALSE,
+ rspamd_log->log_level);
+ priv->io_buf.used = 0;
+ }
+}
+
+/*
+ * Write message to buffer or to file (using direct_write_log_line function)
+ */
+static bool
+file_log_helper(rspamd_logger_t *rspamd_log,
+ struct rspamd_file_logger_priv *priv,
+ const struct iovec *iov,
+ guint iovcnt,
+ gint level_flags)
+{
+ size_t len = 0;
+ guint i;
+
+ if (!priv->is_buffered) {
+ /* Write string directly */
+ return direct_write_log_line(rspamd_log, priv, (void *) iov, iovcnt,
+ TRUE, level_flags);
+ }
+ else {
+ /* Calculate total length */
+ for (i = 0; i < iovcnt; i++) {
+ len += iov[i].iov_len;
+ }
+ /* Fill buffer */
+ if (priv->io_buf.size < len) {
+ /* Buffer is too small to hold this string, so write it directly */
+ rspamd_log_flush(rspamd_log, priv);
+ return direct_write_log_line(rspamd_log, priv, (void *) iov, iovcnt,
+ TRUE, level_flags);
+ }
+ else if (priv->io_buf.used + len >= priv->io_buf.size) {
+ /* Buffer is full, try to write it directly */
+ rspamd_log_flush(rspamd_log, priv);
+ fill_buffer(rspamd_log, priv, iov, iovcnt);
+ }
+ else {
+ /* Copy incoming string to buffer */
+ fill_buffer(rspamd_log, priv, iov, iovcnt);
+ }
+ }
+
+ return true;
+}
+
+static void
+rspamd_log_reset_repeated(rspamd_logger_t *rspamd_log,
+ struct rspamd_file_logger_priv *priv)
+{
+ gchar tmpbuf[256];
+ gssize r;
+
+ if (priv->repeats > REPEATS_MIN) {
+ r = rspamd_snprintf(tmpbuf,
+ sizeof(tmpbuf),
+ "Last message repeated %ud times",
+ priv->repeats - REPEATS_MIN);
+ priv->repeats = 0;
+
+ if (priv->saved_message) {
+ rspamd_log_file_log(priv->saved_module,
+ priv->saved_id,
+ priv->saved_function,
+ priv->saved_loglevel | RSPAMD_LOG_FORCED,
+ priv->saved_message,
+ priv->saved_mlen,
+ rspamd_log,
+ priv);
+
+ g_free(priv->saved_message);
+ g_free(priv->saved_function);
+ g_free(priv->saved_module);
+ g_free(priv->saved_id);
+ priv->saved_message = NULL;
+ priv->saved_function = NULL;
+ priv->saved_module = NULL;
+ priv->saved_id = NULL;
+ }
+
+ /* It is safe to use temporary buffer here as it is not static */
+ rspamd_log_file_log(NULL, NULL,
+ G_STRFUNC,
+ priv->saved_loglevel | RSPAMD_LOG_FORCED,
+ tmpbuf,
+ r,
+ rspamd_log,
+ priv);
+ rspamd_log_flush(rspamd_log, priv);
+ }
+}
+
+static gint
+rspamd_try_open_log_fd(rspamd_logger_t *rspamd_log,
+ struct rspamd_file_logger_priv *priv,
+ uid_t uid, gid_t gid,
+ GError **err)
+{
+ gint fd;
+
+ fd = open(priv->log_file,
+ O_CREAT | O_WRONLY | O_APPEND,
+ S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH);
+ if (fd == -1) {
+ g_set_error(err, FILE_LOG_QUARK, errno,
+ "open_log: cannot open desired log file: %s, %s\n",
+ priv->log_file, strerror(errno));
+ return -1;
+ }
+
+ if (uid != -1 || gid != -1) {
+ if (fchown(fd, uid, gid) == -1) {
+ g_set_error(err, FILE_LOG_QUARK, errno,
+ "open_log: cannot chown desired log file: %s, %s\n",
+ priv->log_file, strerror(errno));
+ close(fd);
+
+ return -1;
+ }
+ }
+
+ return fd;
+}
+
+void *
+rspamd_log_file_init(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ uid_t uid, gid_t gid, GError **err)
+{
+ struct rspamd_file_logger_priv *priv;
+
+ if (!cfg || !cfg->cfg_name) {
+ g_set_error(err, FILE_LOG_QUARK, EINVAL,
+ "no log file specified");
+ return NULL;
+ }
+
+ priv = g_malloc0(sizeof(*priv));
+
+ if (cfg->log_buffered) {
+ if (cfg->log_buf_size != 0) {
+ priv->io_buf.size = cfg->log_buf_size;
+ }
+ else {
+ priv->io_buf.size = LOGBUF_LEN;
+ }
+ priv->is_buffered = TRUE;
+ priv->io_buf.buf = g_malloc(priv->io_buf.size);
+ }
+
+ if (cfg->log_file) {
+ priv->log_file = g_strdup(cfg->log_file);
+ }
+
+ priv->log_severity = (logger->flags & RSPAMD_LOG_FLAG_SEVERITY);
+ priv->fd = rspamd_try_open_log_fd(logger, priv, uid, gid, err);
+
+ if (priv->fd == -1) {
+ rspamd_log_file_dtor(logger, priv);
+
+ return NULL;
+ }
+
+ return priv;
+}
+
+void rspamd_log_file_dtor(rspamd_logger_t *logger, gpointer arg)
+{
+ struct rspamd_file_logger_priv *priv = (struct rspamd_file_logger_priv *) arg;
+
+ rspamd_log_reset_repeated(logger, priv);
+ rspamd_log_flush(logger, priv);
+
+ if (priv->fd != -1) {
+ if (close(priv->fd) == -1) {
+ rspamd_fprintf(stderr, "cannot close log fd %d: %s; log file = %s\n",
+ priv->fd, strerror(errno), priv->log_file);
+ }
+ }
+
+ g_free(priv->log_file);
+ g_free(priv);
+}
+
+bool rspamd_log_file_log(const gchar *module, const gchar *id,
+ const gchar *function,
+ gint level_flags,
+ const gchar *message,
+ gsize mlen,
+ rspamd_logger_t *rspamd_log,
+ gpointer arg)
+{
+ struct rspamd_file_logger_priv *priv = (struct rspamd_file_logger_priv *) arg;
+ gdouble now;
+ guint64 cksum;
+ gboolean got_time = FALSE;
+
+
+ if (!(level_flags & RSPAMD_LOG_FORCED) && !rspamd_log->enabled) {
+ return false;
+ }
+
+ /* Check throttling due to write errors */
+ if (!(level_flags & RSPAMD_LOG_FORCED) && priv->throttling) {
+ now = rspamd_get_calendar_ticks();
+
+ if (priv->throttling_time != now) {
+ priv->throttling_time = now;
+ got_time = TRUE;
+ }
+ else {
+ /* Do not try to write to file too often while throttling */
+ return false;
+ }
+ }
+
+ /* Check repeats */
+ cksum = rspamd_log_calculate_cksum(message, mlen);
+
+ if (cksum == priv->last_line_cksum) {
+ priv->repeats++;
+
+ if (priv->repeats > REPEATS_MIN && priv->repeats <
+ REPEATS_MAX) {
+ /* Do not log anything but save message for future */
+ if (priv->saved_message == NULL) {
+ priv->saved_function = g_strdup(function);
+ priv->saved_mlen = mlen;
+ priv->saved_message = g_malloc(mlen);
+ memcpy(priv->saved_message, message, mlen);
+
+ if (module) {
+ priv->saved_module = g_strdup(module);
+ }
+
+ if (id) {
+ priv->saved_id = g_strdup(id);
+ }
+
+ priv->saved_loglevel = level_flags;
+ }
+
+ return true;
+ }
+ else if (priv->repeats > REPEATS_MAX) {
+ rspamd_log_reset_repeated(rspamd_log, priv);
+
+ bool ret = rspamd_log_file_log(module, id,
+ function,
+ level_flags,
+ message,
+ mlen,
+ rspamd_log,
+ priv);
+
+ /* Probably we have more repeats in future */
+ priv->repeats = REPEATS_MIN + 1;
+
+ return ret;
+ }
+ }
+ else {
+ /* Reset counter if new message differs from saved message */
+ priv->last_line_cksum = cksum;
+
+ if (priv->repeats > REPEATS_MIN) {
+ rspamd_log_reset_repeated(rspamd_log, priv);
+ return rspamd_log_file_log(module, id,
+ function,
+ level_flags,
+ message,
+ mlen,
+ rspamd_log,
+ arg);
+ }
+ else {
+ priv->repeats = 0;
+ }
+ }
+ if (!got_time) {
+ now = rspamd_get_calendar_ticks();
+ }
+
+ struct rspamd_logger_iov_ctx iov_ctx;
+ memset(&iov_ctx, 0, sizeof(iov_ctx));
+ rspamd_log_fill_iov(&iov_ctx, now, module, id, function, level_flags, message,
+ mlen, rspamd_log);
+
+ bool ret = file_log_helper(rspamd_log, priv, iov_ctx.iov, iov_ctx.niov, level_flags);
+ rspamd_log_iov_free(&iov_ctx);
+
+ return ret;
+}
+
+void *
+rspamd_log_file_reload(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ gpointer arg, uid_t uid, gid_t gid, GError **err)
+{
+ struct rspamd_file_logger_priv *npriv;
+
+ if (!cfg->cfg_name) {
+ g_set_error(err, FILE_LOG_QUARK, EINVAL,
+ "no log file specified");
+ return NULL;
+ }
+
+ npriv = rspamd_log_file_init(logger, cfg, uid, gid, err);
+
+ if (npriv) {
+ /* Close old */
+ rspamd_log_file_dtor(logger, arg);
+ }
+
+ return npriv;
+}
+
+bool rspamd_log_file_on_fork(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ gpointer arg, GError **err)
+{
+ struct rspamd_file_logger_priv *priv = (struct rspamd_file_logger_priv *) arg;
+
+ rspamd_log_reset_repeated(logger, priv);
+ rspamd_log_flush(logger, priv);
+
+ return true;
+} \ No newline at end of file
diff --git a/src/libserver/logger/logger_private.h b/src/libserver/logger/logger_private.h
new file mode 100644
index 0000000..234a207
--- /dev/null
+++ b/src/libserver/logger/logger_private.h
@@ -0,0 +1,218 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_LOGGER_PRIVATE_H
+#define RSPAMD_LOGGER_PRIVATE_H
+
+#include "logger.h"
+
+/* How much message should be repeated before it is count to be repeated one */
+#define REPEATS_MIN 3
+#define REPEATS_MAX 300
+#define LOGBUF_LEN 8192
+
+struct rspamd_log_module {
+ gchar *mname;
+ guint id;
+};
+
+struct rspamd_log_modules {
+ guchar *bitset;
+ guint bitset_len; /* Number of BITS used in bitset */
+ guint bitset_allocated; /* Size of bitset allocated in BYTES */
+ GHashTable *modules;
+};
+
+struct rspamd_logger_error_elt {
+ gint completed;
+ GQuark ptype;
+ pid_t pid;
+ gdouble ts;
+ gchar id[RSPAMD_LOG_ID_LEN + 1];
+ gchar module[9];
+ gchar message[];
+};
+
+struct rspamd_logger_error_log {
+ struct rspamd_logger_error_elt *elts;
+ rspamd_mempool_t *pool;
+ guint32 max_elts;
+ guint32 elt_len;
+ /* Avoid false cache sharing */
+ guchar __padding[64 - sizeof(gpointer) * 2 - sizeof(guint64)];
+ guint cur_row;
+};
+
+/**
+ * Static structure that store logging parameters
+ * It is NOT shared between processes and is created by main process
+ */
+struct rspamd_logger_s {
+ struct rspamd_logger_funcs ops;
+ gint log_level;
+
+ struct rspamd_logger_error_log *errlog;
+ struct rspamd_cryptobox_pubkey *pk;
+ struct rspamd_cryptobox_keypair *keypair;
+
+ guint flags;
+ gboolean closed;
+ gboolean enabled;
+ gboolean is_debug;
+ gboolean no_lock;
+
+ pid_t pid;
+ const gchar *process_type;
+ struct rspamd_radix_map_helper *debug_ip;
+ rspamd_mempool_mutex_t *mtx;
+ rspamd_mempool_t *pool;
+ guint64 log_cnt[4];
+};
+
+/*
+ * Common logging prototypes
+ */
+
+/*
+ * File logging
+ */
+void *rspamd_log_file_init(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ uid_t uid, gid_t gid, GError **err);
+void *rspamd_log_file_reload(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ gpointer arg, uid_t uid, gid_t gid, GError **err);
+void rspamd_log_file_dtor(rspamd_logger_t *logger, gpointer arg);
+bool rspamd_log_file_log(const gchar *module, const gchar *id,
+ const gchar *function,
+ gint level_flags,
+ const gchar *message,
+ gsize mlen,
+ rspamd_logger_t *rspamd_log,
+ gpointer arg);
+bool rspamd_log_file_on_fork(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ gpointer arg, GError **err);
+
+struct rspamd_logger_iov_thrash_stack {
+ struct rspamd_logger_iov_thrash_stack *prev;
+ char data[0];
+};
+#define RSPAMD_LOGGER_MAX_IOV 8
+struct rspamd_logger_iov_ctx {
+ struct iovec iov[RSPAMD_LOGGER_MAX_IOV];
+ int niov;
+ struct rspamd_logger_iov_thrash_stack *thrash_stack;
+};
+/**
+ * Fills IOV of logger (usable for file/console logging)
+ * Warning: this function is NOT reentrant, do not call it twice from a single moment of execution
+ * @param iov filled by this function
+ * @param module
+ * @param id
+ * @param function
+ * @param level_flags
+ * @param message
+ * @param mlen
+ * @param rspamd_log
+ * @return number of iov elements being filled
+ */
+void rspamd_log_fill_iov(struct rspamd_logger_iov_ctx *iov_ctx,
+ double ts,
+ const gchar *module, const gchar *id,
+ const gchar *function,
+ gint level_flags,
+ const gchar *message,
+ gsize mlen,
+ rspamd_logger_t *rspamd_log);
+
+/**
+ * Frees IOV context
+ * @param iov_ctx
+ */
+void rspamd_log_iov_free(struct rspamd_logger_iov_ctx *iov_ctx);
+/**
+ * Escape log line by replacing unprintable characters to hex escapes like \xNN
+ * @param src
+ * @param srclen
+ * @param dst
+ * @param dstlen
+ * @return end of the escaped buffer
+ */
+gchar *rspamd_log_line_hex_escape(const guchar *src, gsize srclen,
+ gchar *dst, gsize dstlen);
+/**
+ * Returns number of characters to be escaped, e.g. a caller can allocate a new buffer
+ * the desired number of characters
+ * @param src
+ * @param srclen
+ * @return number of characters to be escaped
+ */
+gsize rspamd_log_line_need_escape(const guchar *src, gsize srclen);
+
+static const struct rspamd_logger_funcs file_log_funcs = {
+ .init = rspamd_log_file_init,
+ .dtor = rspamd_log_file_dtor,
+ .reload = rspamd_log_file_reload,
+ .log = rspamd_log_file_log,
+ .on_fork = rspamd_log_file_on_fork,
+};
+
+/*
+ * Syslog logging
+ */
+void *rspamd_log_syslog_init(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ uid_t uid, gid_t gid, GError **err);
+void *rspamd_log_syslog_reload(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ gpointer arg, uid_t uid, gid_t gid, GError **err);
+void rspamd_log_syslog_dtor(rspamd_logger_t *logger, gpointer arg);
+bool rspamd_log_syslog_log(const gchar *module, const gchar *id,
+ const gchar *function,
+ gint level_flags,
+ const gchar *message,
+ gsize mlen,
+ rspamd_logger_t *rspamd_log,
+ gpointer arg);
+
+static const struct rspamd_logger_funcs syslog_log_funcs = {
+ .init = rspamd_log_syslog_init,
+ .dtor = rspamd_log_syslog_dtor,
+ .reload = rspamd_log_syslog_reload,
+ .log = rspamd_log_syslog_log,
+ .on_fork = NULL,
+};
+
+/*
+ * Console logging
+ */
+void *rspamd_log_console_init(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ uid_t uid, gid_t gid, GError **err);
+void *rspamd_log_console_reload(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ gpointer arg, uid_t uid, gid_t gid, GError **err);
+void rspamd_log_console_dtor(rspamd_logger_t *logger, gpointer arg);
+bool rspamd_log_console_log(const gchar *module, const gchar *id,
+ const gchar *function,
+ gint level_flags,
+ const gchar *message,
+ gsize mlen,
+ rspamd_logger_t *rspamd_log,
+ gpointer arg);
+
+static const struct rspamd_logger_funcs console_log_funcs = {
+ .init = rspamd_log_console_init,
+ .dtor = rspamd_log_console_dtor,
+ .reload = rspamd_log_console_reload,
+ .log = rspamd_log_console_log,
+ .on_fork = NULL,
+};
+
+#endif
diff --git a/src/libserver/logger/logger_syslog.c b/src/libserver/logger/logger_syslog.c
new file mode 100644
index 0000000..3c4f7f7
--- /dev/null
+++ b/src/libserver/logger/logger_syslog.c
@@ -0,0 +1,143 @@
+/*-
+ * Copyright 2020 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "logger.h"
+#include "libserver/cfg_file.h"
+#include "logger_private.h"
+
+#define SYSLOG_LOG_QUARK g_quark_from_static_string("syslog_logger")
+
+struct rspamd_syslog_logger_priv {
+ gint log_facility;
+};
+
+#ifdef HAVE_SYSLOG_H
+#include <syslog.h>
+
+void *
+rspamd_log_syslog_init(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ uid_t uid, gid_t gid, GError **err)
+{
+ struct rspamd_syslog_logger_priv *priv;
+
+ if (!cfg) {
+ g_set_error(err, SYSLOG_LOG_QUARK, EINVAL,
+ "no log config specified");
+ return NULL;
+ }
+
+ priv = g_malloc0(sizeof(*priv));
+
+ priv->log_facility = cfg->log_facility;
+ openlog("rspamd", LOG_NDELAY | LOG_PID, priv->log_facility);
+
+ return priv;
+}
+
+void rspamd_log_syslog_dtor(rspamd_logger_t *logger, gpointer arg)
+{
+ struct rspamd_syslog_logger_priv *priv = (struct rspamd_syslog_logger_priv *) arg;
+
+ closelog();
+ g_free(priv);
+}
+bool rspamd_log_syslog_log(const gchar *module, const gchar *id,
+ const gchar *function,
+ gint level_flags,
+ const gchar *message,
+ gsize mlen,
+ rspamd_logger_t *rspamd_log,
+ gpointer arg)
+{
+ static const struct {
+ GLogLevelFlags glib_level;
+ gint syslog_level;
+ } levels_match[] = {
+ {G_LOG_LEVEL_DEBUG, LOG_DEBUG},
+ {G_LOG_LEVEL_INFO, LOG_INFO},
+ {G_LOG_LEVEL_WARNING, LOG_WARNING},
+ {G_LOG_LEVEL_CRITICAL, LOG_ERR}};
+ unsigned i;
+ gint syslog_level;
+
+ if (!(level_flags & RSPAMD_LOG_FORCED) && !rspamd_log->enabled) {
+ return false;
+ }
+
+ /* Detect level */
+ syslog_level = LOG_DEBUG;
+
+ for (i = 0; i < G_N_ELEMENTS(levels_match); i++) {
+ if (level_flags & levels_match[i].glib_level) {
+ syslog_level = levels_match[i].syslog_level;
+ break;
+ }
+ }
+
+ syslog(syslog_level, "<%.*s>; %s; %s: %.*s",
+ RSPAMD_LOG_ID_LEN, id != NULL ? id : "",
+ module != NULL ? module : "",
+ function != NULL ? function : "",
+ (gint) mlen, message);
+
+ return true;
+}
+
+#else
+
+void *
+rspamd_log_syslog_init(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ uid_t uid, gid_t gid, GError **err)
+{
+ g_set_error(err, SYSLOG_LOG_QUARK, EINVAL, "syslog support is not compiled in");
+
+ return NULL;
+}
+
+bool rspamd_log_syslog_log(const gchar *module, const gchar *id,
+ const gchar *function,
+ gint level_flags,
+ const gchar *message,
+ gsize mlen,
+ rspamd_logger_t *rspamd_log,
+ gpointer arg)
+{
+ return false;
+}
+
+void rspamd_log_syslog_dtor(rspamd_logger_t *logger, gpointer arg)
+{
+ /* Left blank intentionally */
+}
+
+#endif
+
+void *
+rspamd_log_syslog_reload(rspamd_logger_t *logger, struct rspamd_config *cfg,
+ gpointer arg, uid_t uid, gid_t gid, GError **err)
+{
+ struct rspamd_syslog_logger_priv *npriv;
+
+ npriv = rspamd_log_syslog_init(logger, cfg, uid, gid, err);
+
+ if (npriv) {
+ /* Close old */
+ rspamd_log_syslog_dtor(logger, arg);
+ }
+
+ return npriv;
+}
diff --git a/src/libserver/maps/map.c b/src/libserver/maps/map.c
new file mode 100644
index 0000000..7f6a48f
--- /dev/null
+++ b/src/libserver/maps/map.c
@@ -0,0 +1,3195 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Implementation of map files handling
+ */
+
+#include "config.h"
+#include "map.h"
+#include "map_private.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
+#include "rspamd.h"
+#include "contrib/libev/ev.h"
+#include "contrib/uthash/utlist.h"
+
+#ifdef SYS_ZSTD
+#include "zstd.h"
+#else
+#include "contrib/zstd/zstd.h"
+#endif
+
+#undef MAP_DEBUG_REFS
+#ifdef MAP_DEBUG_REFS
+#define MAP_RETAIN(x, t) \
+ do { \
+ msg_err(G_GNUC_PRETTY_FUNCTION ": " t ": retain ref %p, refcount: %d -> %d", (x), (x)->ref.refcount, (x)->ref.refcount + 1); \
+ REF_RETAIN(x); \
+ } while (0)
+
+#define MAP_RELEASE(x, t) \
+ do { \
+ msg_err(G_GNUC_PRETTY_FUNCTION ": " t ": release ref %p, refcount: %d -> %d", (x), (x)->ref.refcount, (x)->ref.refcount - 1); \
+ REF_RELEASE(x); \
+ } while (0)
+#else
+#define MAP_RETAIN(x, t) REF_RETAIN(x)
+#define MAP_RELEASE(x, t) REF_RELEASE(x)
+#endif
+
+enum rspamd_map_periodic_opts {
+ RSPAMD_MAP_SCHEDULE_NORMAL = 0,
+ RSPAMD_MAP_SCHEDULE_ERROR = (1u << 0u),
+ RSPAMD_MAP_SCHEDULE_LOCKED = (1u << 1u),
+ RSPAMD_MAP_SCHEDULE_INIT = (1u << 2u),
+};
+
+static void free_http_cbdata_common(struct http_callback_data *cbd,
+ gboolean plan_new);
+static void free_http_cbdata_dtor(gpointer p);
+static void free_http_cbdata(struct http_callback_data *cbd);
+static void rspamd_map_process_periodic(struct map_periodic_cbdata *cbd);
+static void rspamd_map_schedule_periodic(struct rspamd_map *map, int how);
+static gboolean read_map_file_chunks(struct rspamd_map *map,
+ struct map_cb_data *cbdata,
+ const gchar *fname,
+ gsize len,
+ goffset off);
+static gboolean rspamd_map_save_http_cached_file(struct rspamd_map *map,
+ struct rspamd_map_backend *bk,
+ struct http_map_data *htdata,
+ const guchar *data,
+ gsize len);
+static gboolean rspamd_map_update_http_cached_file(struct rspamd_map *map,
+ struct rspamd_map_backend *bk,
+ struct http_map_data *htdata);
+
+guint rspamd_map_log_id = (guint) -1;
+RSPAMD_CONSTRUCTOR(rspamd_map_log_init)
+{
+ rspamd_map_log_id = rspamd_logger_add_debug_module("map");
+}
+
+/**
+ * Write HTTP request
+ */
+static void
+write_http_request(struct http_callback_data *cbd)
+{
+ gchar datebuf[128];
+ struct rspamd_http_message *msg;
+
+ msg = rspamd_http_new_message(HTTP_REQUEST);
+ if (cbd->check) {
+ msg->method = HTTP_HEAD;
+ }
+
+ msg->url = rspamd_fstring_append(msg->url,
+ cbd->data->path, strlen(cbd->data->path));
+
+ if (cbd->check) {
+ if (cbd->data->last_modified != 0) {
+ rspamd_http_date_format(datebuf, sizeof(datebuf),
+ cbd->data->last_modified);
+ rspamd_http_message_add_header(msg, "If-Modified-Since",
+ datebuf);
+ }
+ if (cbd->data->etag) {
+ rspamd_http_message_add_header_len(msg, "If-None-Match",
+ cbd->data->etag->str, cbd->data->etag->len);
+ }
+ }
+
+ msg->url = rspamd_fstring_append(msg->url, cbd->data->rest,
+ strlen(cbd->data->rest));
+
+ if (cbd->data->userinfo) {
+ rspamd_http_message_add_header(msg, "Authorization",
+ cbd->data->userinfo);
+ }
+
+ MAP_RETAIN(cbd, "http_callback_data");
+ rspamd_http_connection_write_message(cbd->conn,
+ msg,
+ cbd->data->host,
+ NULL,
+ cbd,
+ cbd->timeout);
+}
+
+/**
+ * Callback for destroying HTTP callback data
+ */
+static void
+free_http_cbdata_common(struct http_callback_data *cbd, gboolean plan_new)
+{
+ struct map_periodic_cbdata *periodic = cbd->periodic;
+
+ if (cbd->shmem_data) {
+ rspamd_http_message_shmem_unref(cbd->shmem_data);
+ }
+
+ if (cbd->pk) {
+ rspamd_pubkey_unref(cbd->pk);
+ }
+
+ if (cbd->conn) {
+ rspamd_http_connection_unref(cbd->conn);
+ cbd->conn = NULL;
+ }
+
+ if (cbd->addrs) {
+ rspamd_inet_addr_t *addr;
+ guint i;
+
+ PTR_ARRAY_FOREACH(cbd->addrs, i, addr)
+ {
+ rspamd_inet_address_free(addr);
+ }
+
+ g_ptr_array_free(cbd->addrs, TRUE);
+ }
+
+
+ MAP_RELEASE(cbd->bk, "rspamd_map_backend");
+
+ if (periodic) {
+ /* Detached in case of HTTP error */
+ MAP_RELEASE(periodic, "periodic");
+ }
+
+ g_free(cbd);
+}
+
+static void
+free_http_cbdata(struct http_callback_data *cbd)
+{
+ cbd->map->tmp_dtor = NULL;
+ cbd->map->tmp_dtor_data = NULL;
+
+ free_http_cbdata_common(cbd, TRUE);
+}
+
+static void
+free_http_cbdata_dtor(gpointer p)
+{
+ struct http_callback_data *cbd = p;
+ struct rspamd_map *map;
+
+ map = cbd->map;
+ if (cbd->stage == http_map_http_conn) {
+ REF_RELEASE(cbd);
+ }
+ else {
+ /* We cannot terminate DNS requests sent */
+ cbd->stage = http_map_terminated;
+ }
+
+ msg_warn_map("%s: "
+ "connection with http server is terminated: worker is stopping",
+ map->name);
+}
+
+/*
+ * HTTP callbacks
+ */
+static void
+http_map_error(struct rspamd_http_connection *conn,
+ GError *err)
+{
+ struct http_callback_data *cbd = conn->ud;
+ struct rspamd_map *map;
+
+ map = cbd->map;
+
+ if (cbd->periodic) {
+ cbd->periodic->errored = TRUE;
+ msg_err_map("error reading %s(%s): "
+ "connection with http server terminated incorrectly: %e",
+ cbd->bk->uri,
+ cbd->addr ? rspamd_inet_address_to_string_pretty(cbd->addr) : "",
+ err);
+
+ rspamd_map_process_periodic(cbd->periodic);
+ }
+
+ MAP_RELEASE(cbd, "http_callback_data");
+}
+
+static void
+rspamd_map_cache_cb(struct ev_loop *loop, ev_timer *w, int revents)
+{
+ struct rspamd_http_map_cached_cbdata *cache_cbd = (struct rspamd_http_map_cached_cbdata *)
+ w->data;
+ struct rspamd_map *map;
+ struct http_map_data *data;
+
+ map = cache_cbd->map;
+ data = cache_cbd->data;
+
+ if (cache_cbd->gen != cache_cbd->data->gen) {
+ /* We have another update, so this cache element is obviously expired */
+ /*
+ * Important!: we do not set cache availability to zero here, as there
+ * might be fresh cache
+ */
+ msg_info_map("cached data is now expired (gen mismatch %L != %L) for %s; shm name=%s; refcount=%d",
+ cache_cbd->gen, cache_cbd->data->gen, map->name, cache_cbd->shm->shm_name,
+ cache_cbd->shm->ref.refcount);
+ MAP_RELEASE(cache_cbd->shm, "rspamd_http_map_cached_cbdata");
+ ev_timer_stop(loop, &cache_cbd->timeout);
+ g_free(cache_cbd);
+ }
+ else if (cache_cbd->data->last_checked >= cache_cbd->last_checked) {
+ /*
+ * We checked map but we have not found anything more recent,
+ * reschedule cache check
+ */
+ if (cache_cbd->map->poll_timeout >
+ rspamd_get_calendar_ticks() - cache_cbd->data->last_checked) {
+ w->repeat = cache_cbd->map->poll_timeout -
+ (rspamd_get_calendar_ticks() - cache_cbd->data->last_checked);
+ }
+ else {
+ w->repeat = cache_cbd->map->poll_timeout;
+ }
+
+ if (w->repeat < 0) {
+ msg_info_map("cached data for %s has skewed check time: %d last checked, "
+ "%d poll timeout, %.2f diff; shm name=%s; refcount=%d",
+ map->name, (int) cache_cbd->data->last_checked,
+ (int) cache_cbd->map->poll_timeout,
+ (rspamd_get_calendar_ticks() - cache_cbd->data->last_checked),
+ cache_cbd->shm->shm_name,
+ cache_cbd->shm->ref.refcount);
+ w->repeat = 0.0;
+ }
+
+ cache_cbd->last_checked = cache_cbd->data->last_checked;
+ msg_debug_map("cached data is up to date for %s", map->name);
+ ev_timer_again(loop, &cache_cbd->timeout);
+ }
+ else {
+ data->cur_cache_cbd = NULL;
+ g_atomic_int_set(&data->cache->available, 0);
+ msg_info_map("cached data is now expired for %s; shm name=%s; refcount=%d",
+ map->name,
+ cache_cbd->shm->shm_name,
+ cache_cbd->shm->ref.refcount);
+ MAP_RELEASE(cache_cbd->shm, "rspamd_http_map_cached_cbdata");
+ ev_timer_stop(loop, &cache_cbd->timeout);
+ g_free(cache_cbd);
+ }
+}
+
+static int
+http_map_finish(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct http_callback_data *cbd = conn->ud;
+ struct rspamd_map *map;
+ struct rspamd_map_backend *bk;
+ struct http_map_data *data;
+ struct rspamd_http_map_cached_cbdata *cache_cbd;
+ const rspamd_ftok_t *expires_hdr, *etag_hdr;
+ char next_check_date[128];
+ guchar *in = NULL;
+ gsize dlen = 0;
+
+ map = cbd->map;
+ bk = cbd->bk;
+ data = bk->data.hd;
+
+ if (msg->code == 200) {
+
+ if (cbd->check) {
+ msg_info_map("need to reread map from %s", cbd->bk->uri);
+ cbd->periodic->need_modify = TRUE;
+ /* Reset the whole chain */
+ cbd->periodic->cur_backend = 0;
+ /* Reset cache, old cached data will be cleaned on timeout */
+ g_atomic_int_set(&data->cache->available, 0);
+ data->cur_cache_cbd = NULL;
+
+ rspamd_map_process_periodic(cbd->periodic);
+ MAP_RELEASE(cbd, "http_callback_data");
+
+ return 0;
+ }
+
+ cbd->data->last_checked = msg->date;
+
+ if (msg->last_modified) {
+ cbd->data->last_modified = msg->last_modified;
+ }
+ else {
+ cbd->data->last_modified = msg->date;
+ }
+
+
+ /* Unsigned version - just open file */
+ cbd->shmem_data = rspamd_http_message_shmem_ref(msg);
+ cbd->data_len = msg->body_buf.len;
+
+ if (cbd->data_len == 0) {
+ msg_err_map("cannot read empty map");
+ goto err;
+ }
+
+ g_assert(cbd->shmem_data != NULL);
+
+ in = rspamd_shmem_xmap(cbd->shmem_data->shm_name, PROT_READ, &dlen);
+
+ if (in == NULL) {
+ msg_err_map("cannot read tempfile %s: %s",
+ cbd->shmem_data->shm_name,
+ strerror(errno));
+ goto err;
+ }
+
+ /* Check for expires */
+ double cached_timeout = map->poll_timeout * 2;
+
+ expires_hdr = rspamd_http_message_find_header(msg, "Expires");
+
+ if (expires_hdr) {
+ time_t hdate;
+
+ hdate = rspamd_http_parse_date(expires_hdr->begin, expires_hdr->len);
+
+ if (hdate != (time_t) -1 && hdate > msg->date) {
+ cached_timeout = map->next_check - msg->date +
+ map->poll_timeout * 2;
+
+ map->next_check = hdate;
+ }
+ else {
+ msg_info_map("invalid expires header: %T, ignore it", expires_hdr);
+ map->next_check = 0;
+ }
+ }
+
+ /* Check for etag */
+ etag_hdr = rspamd_http_message_find_header(msg, "ETag");
+
+ if (etag_hdr) {
+ if (cbd->data->etag) {
+ /* Remove old etag */
+ rspamd_fstring_free(cbd->data->etag);
+ }
+
+ cbd->data->etag = rspamd_fstring_new_init(etag_hdr->begin,
+ etag_hdr->len);
+ }
+ else {
+ if (cbd->data->etag) {
+ /* Remove and clear old etag */
+ rspamd_fstring_free(cbd->data->etag);
+ cbd->data->etag = NULL;
+ }
+ }
+
+ MAP_RETAIN(cbd->shmem_data, "shmem_data");
+ cbd->data->gen++;
+ /*
+ * We know that a map is in the locked state
+ */
+ g_atomic_int_set(&data->cache->available, 1);
+ /* Store cached data */
+ rspamd_strlcpy(data->cache->shmem_name, cbd->shmem_data->shm_name,
+ sizeof(data->cache->shmem_name));
+ data->cache->len = cbd->data_len;
+ data->cache->last_modified = cbd->data->last_modified;
+ cache_cbd = g_malloc0(sizeof(*cache_cbd));
+ cache_cbd->shm = cbd->shmem_data;
+ cache_cbd->event_loop = cbd->event_loop;
+ cache_cbd->map = map;
+ cache_cbd->data = cbd->data;
+ cache_cbd->last_checked = cbd->data->last_checked;
+ cache_cbd->gen = cbd->data->gen;
+ MAP_RETAIN(cache_cbd->shm, "shmem_data");
+ msg_info_map("stored map data in a shared memory cache: %s",
+ cache_cbd->shm->shm_name);
+
+ ev_timer_init(&cache_cbd->timeout, rspamd_map_cache_cb, cached_timeout,
+ 0.0);
+ ev_timer_start(cbd->event_loop, &cache_cbd->timeout);
+ cache_cbd->timeout.data = cache_cbd;
+ data->cur_cache_cbd = cache_cbd;
+
+ if (map->next_check) {
+ rspamd_http_date_format(next_check_date, sizeof(next_check_date),
+ map->next_check);
+ }
+ else {
+ rspamd_http_date_format(next_check_date, sizeof(next_check_date),
+ rspamd_get_calendar_ticks() + map->poll_timeout);
+ }
+
+
+ if (cbd->bk->is_compressed) {
+ ZSTD_DStream *zstream;
+ ZSTD_inBuffer zin;
+ ZSTD_outBuffer zout;
+ guchar *out;
+ gsize outlen, r;
+
+ zstream = ZSTD_createDStream();
+ ZSTD_initDStream(zstream);
+
+ zin.pos = 0;
+ zin.src = in;
+ zin.size = dlen;
+
+ if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
+ outlen = ZSTD_DStreamOutSize();
+ }
+
+ out = g_malloc(outlen);
+
+ zout.dst = out;
+ zout.pos = 0;
+ zout.size = outlen;
+
+ while (zin.pos < zin.size) {
+ r = ZSTD_decompressStream(zstream, &zout, &zin);
+
+ if (ZSTD_isError(r)) {
+ msg_err_map("%s(%s): cannot decompress data: %s",
+ cbd->bk->uri,
+ rspamd_inet_address_to_string_pretty(cbd->addr),
+ ZSTD_getErrorName(r));
+ ZSTD_freeDStream(zstream);
+ g_free(out);
+ MAP_RELEASE(cbd->shmem_data, "shmem_data");
+ goto err;
+ }
+
+ if (zout.pos == zout.size) {
+ /* We need to extend output buffer */
+ zout.size = zout.size * 2 + 1.0;
+ out = g_realloc(zout.dst, zout.size);
+ zout.dst = out;
+ }
+ }
+
+ ZSTD_freeDStream(zstream);
+ msg_info_map("%s(%s): read map data %z bytes compressed, "
+ "%z uncompressed, next check at %s",
+ cbd->bk->uri,
+ rspamd_inet_address_to_string_pretty(cbd->addr),
+ dlen, zout.pos, next_check_date);
+ map->read_callback(out, zout.pos, &cbd->periodic->cbdata, TRUE);
+ rspamd_map_save_http_cached_file(map, bk, cbd->data, out, zout.pos);
+ g_free(out);
+ }
+ else {
+ msg_info_map("%s(%s): read map data %z bytes, next check at %s",
+ cbd->bk->uri,
+ rspamd_inet_address_to_string_pretty(cbd->addr),
+ dlen, next_check_date);
+ rspamd_map_save_http_cached_file(map, bk, cbd->data, in, cbd->data_len);
+ map->read_callback(in, cbd->data_len, &cbd->periodic->cbdata, TRUE);
+ }
+
+ MAP_RELEASE(cbd->shmem_data, "shmem_data");
+
+ cbd->periodic->cur_backend++;
+ munmap(in, dlen);
+ rspamd_map_process_periodic(cbd->periodic);
+ }
+ else if (msg->code == 304 && cbd->check) {
+ cbd->data->last_checked = msg->date;
+
+ if (msg->last_modified) {
+ cbd->data->last_modified = msg->last_modified;
+ }
+ else {
+ cbd->data->last_modified = msg->date;
+ }
+
+ expires_hdr = rspamd_http_message_find_header(msg, "Expires");
+
+ if (expires_hdr) {
+ time_t hdate;
+
+ hdate = rspamd_http_parse_date(expires_hdr->begin, expires_hdr->len);
+ if (hdate != (time_t) -1 && hdate > msg->date) {
+ map->next_check = hdate;
+ }
+ else {
+ msg_info_map("invalid expires header: %T, ignore it", expires_hdr);
+ map->next_check = 0;
+ }
+ }
+
+ etag_hdr = rspamd_http_message_find_header(msg, "ETag");
+
+ if (etag_hdr) {
+ if (cbd->data->etag) {
+ /* Remove old etag */
+ rspamd_fstring_free(cbd->data->etag);
+ cbd->data->etag = rspamd_fstring_new_init(etag_hdr->begin,
+ etag_hdr->len);
+ }
+ }
+
+ if (map->next_check) {
+ rspamd_http_date_format(next_check_date, sizeof(next_check_date),
+ map->next_check);
+ msg_info_map("data is not modified for server %s, next check at %s "
+ "(http cache based: %T)",
+ cbd->data->host, next_check_date, expires_hdr);
+ }
+ else {
+ rspamd_http_date_format(next_check_date, sizeof(next_check_date),
+ rspamd_get_calendar_ticks() + map->poll_timeout);
+ msg_info_map("data is not modified for server %s, next check at %s "
+ "(timer based)",
+ cbd->data->host, next_check_date);
+ }
+
+ rspamd_map_update_http_cached_file(map, bk, cbd->data);
+ cbd->periodic->cur_backend++;
+ rspamd_map_process_periodic(cbd->periodic);
+ }
+ else {
+ msg_info_map("cannot load map %s from %s: HTTP error %d",
+ bk->uri, cbd->data->host, msg->code);
+ goto err;
+ }
+
+ MAP_RELEASE(cbd, "http_callback_data");
+ return 0;
+
+err:
+ cbd->periodic->errored = 1;
+ rspamd_map_process_periodic(cbd->periodic);
+ MAP_RELEASE(cbd, "http_callback_data");
+
+ return 0;
+}
+
+static gboolean
+read_map_file_chunks(struct rspamd_map *map, struct map_cb_data *cbdata,
+ const gchar *fname, gsize len, goffset off)
+{
+ gint fd;
+ gssize r, avail;
+ gsize buflen = 1024 * 1024;
+ gchar *pos, *bytes;
+
+ fd = rspamd_file_xopen(fname, O_RDONLY, 0, TRUE);
+
+ if (fd == -1) {
+ msg_err_map("can't open map for buffered reading %s: %s",
+ fname, strerror(errno));
+ return FALSE;
+ }
+
+ if (lseek(fd, off, SEEK_SET) == -1) {
+ msg_err_map("can't seek in map to pos %d for buffered reading %s: %s",
+ (gint) off, fname, strerror(errno));
+ close(fd);
+
+ return FALSE;
+ }
+
+ buflen = MIN(len, buflen);
+ bytes = g_malloc(buflen);
+ avail = buflen;
+ pos = bytes;
+
+ while ((r = read(fd, pos, avail)) > 0) {
+ gchar *end = bytes + (pos - bytes) + r;
+ msg_debug_map("%s: read map chunk, %z bytes", fname,
+ r);
+ pos = map->read_callback(bytes, end - bytes, cbdata, r == len);
+
+ if (pos && pos > bytes && pos < end) {
+ guint remain = end - pos;
+
+ memmove(bytes, pos, remain);
+ pos = bytes + remain;
+ /* Need to preserve the remain */
+ avail = ((gssize) buflen) - remain;
+
+ if (avail <= 0) {
+ /* Try realloc, too large element */
+ g_assert(buflen >= remain);
+ bytes = g_realloc(bytes, buflen * 2);
+
+ pos = bytes + remain; /* Adjust */
+ avail += buflen;
+ buflen *= 2;
+ }
+ }
+ else {
+ avail = buflen;
+ pos = bytes;
+ }
+
+ len -= r;
+ }
+
+ if (r == -1) {
+ msg_err_map("can't read from map %s: %s", fname, strerror(errno));
+ close(fd);
+ g_free(bytes);
+
+ return FALSE;
+ }
+
+ close(fd);
+ g_free(bytes);
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_map_check_sig_pk_mem(const guchar *sig,
+ gsize siglen,
+ struct rspamd_map *map,
+ const guchar *input,
+ gsize inlen,
+ struct rspamd_cryptobox_pubkey *pk)
+{
+ GString *b32_key;
+ gboolean ret = TRUE;
+
+ if (siglen != rspamd_cryptobox_signature_bytes(RSPAMD_CRYPTOBOX_MODE_25519)) {
+ msg_err_map("can't open signature for %s: invalid size: %z", map->name, siglen);
+
+ ret = FALSE;
+ }
+
+ if (ret && !rspamd_cryptobox_verify(sig, siglen, input, inlen,
+ rspamd_pubkey_get_pk(pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519)) {
+ msg_err_map("can't verify signature for %s: incorrect signature", map->name);
+
+ ret = FALSE;
+ }
+
+ if (ret) {
+ b32_key = rspamd_pubkey_print(pk,
+ RSPAMD_KEYPAIR_BASE32 | RSPAMD_KEYPAIR_PUBKEY);
+ msg_info_map("verified signature for %s using trusted key %v",
+ map->name, b32_key);
+ g_string_free(b32_key, TRUE);
+ }
+
+ return ret;
+}
+
+static gboolean
+rspamd_map_check_file_sig(const char *fname,
+ struct rspamd_map *map,
+ struct rspamd_map_backend *bk,
+ const guchar *input,
+ gsize inlen)
+{
+ guchar *data;
+ struct rspamd_cryptobox_pubkey *pk = NULL;
+ GString *b32_key;
+ gboolean ret = TRUE;
+ gsize len = 0;
+ gchar fpath[PATH_MAX];
+
+ if (bk->trusted_pubkey == NULL) {
+ /* Try to load and check pubkey */
+ rspamd_snprintf(fpath, sizeof(fpath), "%s.pub", fname);
+ data = rspamd_file_xmap(fpath, PROT_READ, &len, TRUE);
+
+ if (data == NULL) {
+ msg_err_map("can't open pubkey %s: %s", fpath, strerror(errno));
+ return FALSE;
+ }
+
+ pk = rspamd_pubkey_from_base32(data, len, RSPAMD_KEYPAIR_SIGN,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+ munmap(data, len);
+
+ if (pk == NULL) {
+ msg_err_map("can't load pubkey %s", fpath);
+ return FALSE;
+ }
+
+ /* We just check pk against the trusted db of keys */
+ b32_key = rspamd_pubkey_print(pk,
+ RSPAMD_KEYPAIR_BASE32 | RSPAMD_KEYPAIR_PUBKEY);
+ g_assert(b32_key != NULL);
+
+ if (g_hash_table_lookup(map->cfg->trusted_keys, b32_key->str) == NULL) {
+ msg_err_map("pubkey loaded from %s is untrusted: %v", fpath,
+ b32_key);
+ g_string_free(b32_key, TRUE);
+ rspamd_pubkey_unref(pk);
+
+ return FALSE;
+ }
+
+ g_string_free(b32_key, TRUE);
+ }
+ else {
+ pk = rspamd_pubkey_ref(bk->trusted_pubkey);
+ }
+
+ rspamd_snprintf(fpath, sizeof(fpath), "%s.sig", fname);
+ data = rspamd_shmem_xmap(fpath, PROT_READ, &len);
+
+ if (data == NULL) {
+ msg_err_map("can't open signature %s: %s", fpath, strerror(errno));
+ ret = FALSE;
+ }
+
+ if (ret) {
+ ret = rspamd_map_check_sig_pk_mem(data, len, map, input, inlen, pk);
+ munmap(data, len);
+ }
+
+ rspamd_pubkey_unref(pk);
+
+ return ret;
+}
+
+/**
+ * Callback for reading data from file
+ */
+static gboolean
+read_map_file(struct rspamd_map *map, struct file_map_data *data,
+ struct rspamd_map_backend *bk, struct map_periodic_cbdata *periodic)
+{
+ gchar *bytes;
+ gsize len;
+ struct stat st;
+
+ if (map->read_callback == NULL || map->fin_callback == NULL) {
+ msg_err_map("%s: bad callback for reading map file",
+ data->filename);
+ return FALSE;
+ }
+
+ if (stat(data->filename, &st) == -1) {
+ /* File does not exist, skipping */
+ if (errno != ENOENT) {
+ msg_err_map("%s: map file is unavailable for reading: %s",
+ data->filename, strerror(errno));
+
+ return FALSE;
+ }
+ else {
+ msg_info_map("%s: map file is not found; "
+ "it will be read automatically if created",
+ data->filename);
+ return TRUE;
+ }
+ }
+
+ ev_stat_stat(map->event_loop, &data->st_ev);
+ len = st.st_size;
+
+ if (bk->is_signed) {
+ bytes = rspamd_file_xmap(data->filename, PROT_READ, &len, TRUE);
+
+ if (bytes == NULL) {
+ msg_err_map("can't open map %s: %s", data->filename, strerror(errno));
+ return FALSE;
+ }
+
+ if (!rspamd_map_check_file_sig(data->filename, map, bk, bytes, len)) {
+ munmap(bytes, len);
+
+ return FALSE;
+ }
+
+ munmap(bytes, len);
+ }
+
+ if (len > 0) {
+ if (map->no_file_read) {
+ /* We just call read callback with backend name */
+ map->read_callback(data->filename, strlen(data->filename),
+ &periodic->cbdata, TRUE);
+ }
+ else {
+ if (bk->is_compressed) {
+ bytes = rspamd_file_xmap(data->filename, PROT_READ, &len, TRUE);
+
+ if (bytes == NULL) {
+ msg_err_map("can't open map %s: %s", data->filename, strerror(errno));
+ return FALSE;
+ }
+
+ ZSTD_DStream *zstream;
+ ZSTD_inBuffer zin;
+ ZSTD_outBuffer zout;
+ guchar *out;
+ gsize outlen, r;
+
+ zstream = ZSTD_createDStream();
+ ZSTD_initDStream(zstream);
+
+ zin.pos = 0;
+ zin.src = bytes;
+ zin.size = len;
+
+ if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
+ outlen = ZSTD_DStreamOutSize();
+ }
+
+ out = g_malloc(outlen);
+
+ zout.dst = out;
+ zout.pos = 0;
+ zout.size = outlen;
+
+ while (zin.pos < zin.size) {
+ r = ZSTD_decompressStream(zstream, &zout, &zin);
+
+ if (ZSTD_isError(r)) {
+ msg_err_map("%s: cannot decompress data: %s",
+ data->filename,
+ ZSTD_getErrorName(r));
+ ZSTD_freeDStream(zstream);
+ g_free(out);
+ munmap(bytes, len);
+ return FALSE;
+ }
+
+ if (zout.pos == zout.size) {
+ /* We need to extend output buffer */
+ zout.size = zout.size * 2 + 1;
+ out = g_realloc(zout.dst, zout.size);
+ zout.dst = out;
+ }
+ }
+
+ ZSTD_freeDStream(zstream);
+ msg_info_map("%s: read map data, %z bytes compressed, "
+ "%z uncompressed)",
+ data->filename,
+ len, zout.pos);
+ map->read_callback(out, zout.pos, &periodic->cbdata, TRUE);
+ g_free(out);
+
+ munmap(bytes, len);
+ }
+ else {
+ /* Perform buffered read: fail-safe */
+ if (!read_map_file_chunks(map, &periodic->cbdata, data->filename,
+ len, 0)) {
+ return FALSE;
+ }
+ }
+ }
+ }
+ else {
+ /* Empty map */
+ map->read_callback(NULL, 0, &periodic->cbdata, TRUE);
+ }
+
+ return TRUE;
+}
+
+static gboolean
+read_map_static(struct rspamd_map *map, struct static_map_data *data,
+ struct rspamd_map_backend *bk, struct map_periodic_cbdata *periodic)
+{
+ guchar *bytes;
+ gsize len;
+
+ if (map->read_callback == NULL || map->fin_callback == NULL) {
+ msg_err_map("%s: bad callback for reading map file", map->name);
+ data->processed = TRUE;
+ return FALSE;
+ }
+
+ bytes = data->data;
+ len = data->len;
+
+ if (len > 0) {
+ if (bk->is_compressed) {
+ ZSTD_DStream *zstream;
+ ZSTD_inBuffer zin;
+ ZSTD_outBuffer zout;
+ guchar *out;
+ gsize outlen, r;
+
+ zstream = ZSTD_createDStream();
+ ZSTD_initDStream(zstream);
+
+ zin.pos = 0;
+ zin.src = bytes;
+ zin.size = len;
+
+ if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
+ outlen = ZSTD_DStreamOutSize();
+ }
+
+ out = g_malloc(outlen);
+
+ zout.dst = out;
+ zout.pos = 0;
+ zout.size = outlen;
+
+ while (zin.pos < zin.size) {
+ r = ZSTD_decompressStream(zstream, &zout, &zin);
+
+ if (ZSTD_isError(r)) {
+ msg_err_map("%s: cannot decompress data: %s",
+ map->name,
+ ZSTD_getErrorName(r));
+ ZSTD_freeDStream(zstream);
+ g_free(out);
+
+ return FALSE;
+ }
+
+ if (zout.pos == zout.size) {
+ /* We need to extend output buffer */
+ zout.size = zout.size * 2 + 1;
+ out = g_realloc(zout.dst, zout.size);
+ zout.dst = out;
+ }
+ }
+
+ ZSTD_freeDStream(zstream);
+ msg_info_map("%s: read map data, %z bytes compressed, "
+ "%z uncompressed)",
+ map->name,
+ len, zout.pos);
+ map->read_callback(out, zout.pos, &periodic->cbdata, TRUE);
+ g_free(out);
+ }
+ else {
+ msg_info_map("%s: read map data, %z bytes",
+ map->name, len);
+ map->read_callback(bytes, len, &periodic->cbdata, TRUE);
+ }
+ }
+ else {
+ map->read_callback(NULL, 0, &periodic->cbdata, TRUE);
+ }
+
+ data->processed = TRUE;
+
+ return TRUE;
+}
+
+static void
+rspamd_map_periodic_dtor(struct map_periodic_cbdata *periodic)
+{
+ struct rspamd_map *map;
+
+ map = periodic->map;
+ msg_debug_map("periodic dtor %p", periodic);
+
+ if (periodic->need_modify || periodic->cbdata.errored) {
+ /* Need to notify the real data structure */
+ periodic->map->fin_callback(&periodic->cbdata, periodic->map->user_data);
+
+ if (map->on_load_function) {
+ map->on_load_function(map, map->on_load_ud);
+ }
+ }
+ else {
+ /* Not modified */
+ }
+
+ if (periodic->locked) {
+ g_atomic_int_set(periodic->map->locked, 0);
+ msg_debug_map("unlocked map %s", periodic->map->name);
+
+ if (periodic->map->wrk->state == rspamd_worker_state_running) {
+ rspamd_map_schedule_periodic(periodic->map,
+ RSPAMD_SYMBOL_RESULT_NORMAL);
+ }
+ else {
+ msg_debug_map("stop scheduling periodics for %s; terminating state",
+ periodic->map->name);
+ }
+ }
+
+ g_free(periodic);
+}
+
+/* Called on timer execution */
+static void
+rspamd_map_periodic_callback(struct ev_loop *loop, ev_timer *w, int revents)
+{
+ struct map_periodic_cbdata *cbd = (struct map_periodic_cbdata *) w->data;
+
+ MAP_RETAIN(cbd, "periodic");
+ ev_timer_stop(loop, w);
+ rspamd_map_process_periodic(cbd);
+ MAP_RELEASE(cbd, "periodic");
+}
+
+static void
+rspamd_map_schedule_periodic(struct rspamd_map *map, int how)
+{
+ const gdouble error_mult = 20.0, lock_mult = 0.1;
+ static const gdouble min_timer_interval = 2.0;
+ const gchar *reason = "unknown reason";
+ gdouble jittered_sec;
+ gdouble timeout;
+ struct map_periodic_cbdata *cbd;
+
+ if (map->scheduled_check || (map->wrk &&
+ map->wrk->state != rspamd_worker_state_running)) {
+ /*
+ * Do not schedule check if some check is already scheduled or
+ * if worker is going to die
+ */
+ return;
+ }
+
+ if (!(how & RSPAMD_MAP_SCHEDULE_INIT) && map->static_only) {
+ /* No need to schedule anything for static maps */
+ return;
+ }
+
+ if (map->non_trivial && map->next_check != 0) {
+ timeout = map->next_check - rspamd_get_calendar_ticks();
+ map->next_check = 0;
+
+ if (timeout > 0 && timeout < map->poll_timeout) {
+ /* Early check case, jitter */
+ gdouble poll_timeout = map->poll_timeout;
+
+ if (how & RSPAMD_MAP_SCHEDULE_ERROR) {
+ poll_timeout = map->poll_timeout * error_mult;
+ reason = "early active non-trivial check (after error)";
+ }
+ else if (how & RSPAMD_MAP_SCHEDULE_LOCKED) {
+ poll_timeout = map->poll_timeout * lock_mult;
+ reason = "early active non-trivial check (after being locked)";
+ }
+ else {
+ reason = "early active non-trivial check";
+ }
+
+ jittered_sec = MIN(timeout, poll_timeout);
+ }
+ else if (timeout <= 0) {
+ /* Data is already expired, need to check */
+ if (how & RSPAMD_MAP_SCHEDULE_ERROR) {
+ /* In case of error we still need to increase delay */
+ jittered_sec = map->poll_timeout * error_mult;
+ reason = "expired non-trivial data (after error)";
+ }
+ else {
+ jittered_sec = 0.0;
+ reason = "expired non-trivial data";
+ }
+ }
+ else {
+ /* No need to check now, wait till next_check */
+ jittered_sec = timeout;
+ reason = "valid non-trivial data";
+ }
+ }
+ else {
+ /* No valid information when to check a map, plan a timer based check */
+ timeout = map->poll_timeout;
+
+ if (how & RSPAMD_MAP_SCHEDULE_INIT) {
+ if (map->active_http) {
+ /* Spill maps load to get better chances to hit ssl cache */
+ timeout = rspamd_time_jitter(0.0, 2.0);
+ }
+ else {
+ timeout = 0.0;
+ }
+
+ reason = "init scheduled check";
+ }
+ else {
+ if (how & RSPAMD_MAP_SCHEDULE_ERROR) {
+ timeout = map->poll_timeout * error_mult;
+ reason = "errored scheduled check";
+ }
+ else if (how & RSPAMD_MAP_SCHEDULE_LOCKED) {
+ timeout = map->poll_timeout * lock_mult;
+ reason = "locked scheduled check";
+ }
+ else {
+ reason = "normal scheduled check";
+ }
+ }
+
+ jittered_sec = rspamd_time_jitter(timeout, 0);
+ }
+
+ /* Now, we do some sanity checks for jittered seconds */
+ if (!(how & RSPAMD_MAP_SCHEDULE_INIT)) {
+ /* Never allow too low interval between timer checks, it is expensive */
+ if (jittered_sec < min_timer_interval) {
+ jittered_sec = rspamd_time_jitter(min_timer_interval, 0);
+ }
+
+ if (map->non_trivial) {
+ /*
+ * Even if we are reported that we need to reload cache often, we
+ * still want to be sane in terms of events...
+ */
+ if (jittered_sec < min_timer_interval * 2.0) {
+ if (map->nelts > 0) {
+ jittered_sec = min_timer_interval * 3.0;
+ }
+ }
+ }
+ }
+
+ cbd = g_malloc0(sizeof(*cbd));
+ cbd->cbdata.prev_data = *map->user_data;
+ cbd->cbdata.cur_data = NULL;
+ cbd->cbdata.map = map;
+ cbd->map = map;
+ map->scheduled_check = cbd;
+ REF_INIT_RETAIN(cbd, rspamd_map_periodic_dtor);
+
+ cbd->ev.data = cbd;
+ ev_timer_init(&cbd->ev, rspamd_map_periodic_callback, jittered_sec, 0.0);
+ ev_timer_start(map->event_loop, &cbd->ev);
+
+ msg_debug_map("schedule new periodic event %p in %.3f seconds for %s; reason: %s",
+ cbd, jittered_sec, map->name, reason);
+}
+
+static gint
+rspamd_map_af_to_weight(const rspamd_inet_addr_t *addr)
+{
+ int ret;
+
+ switch (rspamd_inet_address_get_af(addr)) {
+ case AF_UNIX:
+ ret = 2;
+ break;
+ case AF_INET:
+ ret = 1;
+ break;
+ default:
+ ret = 0;
+ break;
+ }
+
+ return ret;
+}
+
+static gint
+rspamd_map_dns_address_sort_func(gconstpointer a, gconstpointer b)
+{
+ const rspamd_inet_addr_t *ip1 = *(const rspamd_inet_addr_t **) a,
+ *ip2 = *(const rspamd_inet_addr_t **) b;
+ gint w1, w2;
+
+ w1 = rspamd_map_af_to_weight(ip1);
+ w2 = rspamd_map_af_to_weight(ip2);
+
+ /* Inverse order */
+ return w2 - w1;
+}
+
+static void
+rspamd_map_dns_callback(struct rdns_reply *reply, void *arg)
+{
+ struct http_callback_data *cbd = arg;
+ struct rdns_reply_entry *cur_rep;
+ struct rspamd_map *map;
+ guint flags = RSPAMD_HTTP_CLIENT_SIMPLE | RSPAMD_HTTP_CLIENT_SHARED;
+
+ map = cbd->map;
+
+ msg_debug_map("got dns reply with code %s on stage %d",
+ rdns_strerror(reply->code), cbd->stage);
+
+ if (cbd->stage == http_map_terminated) {
+ MAP_RELEASE(cbd, "http_callback_data");
+ return;
+ }
+
+ if (reply->code == RDNS_RC_NOERROR) {
+ DL_FOREACH(reply->entries, cur_rep)
+ {
+ rspamd_inet_addr_t *addr;
+ addr = rspamd_inet_address_from_rnds(cur_rep);
+
+ if (addr != NULL) {
+ rspamd_inet_address_set_port(addr, cbd->data->port);
+ g_ptr_array_add(cbd->addrs, (void *) addr);
+ }
+ }
+
+ if (cbd->stage == http_map_resolve_host2) {
+ /* We have still one request pending */
+ cbd->stage = http_map_resolve_host1;
+ }
+ else if (cbd->stage == http_map_resolve_host1) {
+ cbd->stage = http_map_http_conn;
+ }
+ }
+ else if (cbd->stage < http_map_http_conn) {
+ if (cbd->stage == http_map_resolve_host2) {
+ /* We have still one request pending */
+ cbd->stage = http_map_resolve_host1;
+ }
+ else if (cbd->addrs->len == 0) {
+ /* We could not resolve host, so cowardly fail here */
+ msg_err_map("cannot resolve %s: %s", cbd->data->host,
+ rdns_strerror(reply->code));
+ cbd->periodic->errored = 1;
+ rspamd_map_process_periodic(cbd->periodic);
+ }
+ else {
+ /* We have at least one address, so we can continue... */
+ cbd->stage = http_map_http_conn;
+ }
+ }
+
+ if (cbd->stage == http_map_http_conn && cbd->addrs->len > 0) {
+ rspamd_ptr_array_shuffle(cbd->addrs);
+ gint idx = 0;
+ /*
+ * For the existing addr we can just select any address as we have
+ * data available
+ */
+ if (cbd->map->nelts > 0 && rspamd_random_double_fast() > 0.5) {
+ /* Already shuffled, use whatever is the first */
+ cbd->addr = (rspamd_inet_addr_t *) g_ptr_array_index(cbd->addrs, idx);
+ }
+ else {
+ /* Always prefer IPv4 as IPv6 is almost all the time broken */
+ g_ptr_array_sort(cbd->addrs, rspamd_map_dns_address_sort_func);
+ cbd->addr = (rspamd_inet_addr_t *) g_ptr_array_index(cbd->addrs, idx);
+ }
+
+ retry:
+ msg_debug_map("try open http connection to %s",
+ rspamd_inet_address_to_string_pretty(cbd->addr));
+ if (cbd->bk->protocol == MAP_PROTO_HTTPS) {
+ flags |= RSPAMD_HTTP_CLIENT_SSL;
+ }
+ cbd->conn = rspamd_http_connection_new_client(NULL,
+ NULL,
+ http_map_error,
+ http_map_finish,
+ flags,
+ cbd->addr);
+
+ if (cbd->conn != NULL) {
+ write_http_request(cbd);
+ }
+ else {
+ if (idx < cbd->addrs->len - 1) {
+ /* We can retry */
+ idx++;
+ rspamd_inet_addr_t *prev_addr = cbd->addr;
+ cbd->addr = (rspamd_inet_addr_t *) g_ptr_array_index(cbd->addrs, idx);
+ msg_info_map("cannot connect to %s to get data for %s: %s, retry with %s (%d of %d)",
+ rspamd_inet_address_to_string_pretty(prev_addr),
+ cbd->bk->uri,
+ strerror(errno),
+ rspamd_inet_address_to_string_pretty(cbd->addr),
+ idx + 1, cbd->addrs->len);
+ goto retry;
+ }
+ else {
+ /* Nothing else left */
+ cbd->periodic->errored = TRUE;
+ msg_err_map("error reading %s(%s): "
+ "connection with http server terminated incorrectly: %s",
+ cbd->bk->uri,
+ cbd->addr ? rspamd_inet_address_to_string_pretty(cbd->addr) : "",
+ strerror(errno));
+
+ rspamd_map_process_periodic(cbd->periodic);
+ }
+ }
+ }
+
+ MAP_RELEASE(cbd, "http_callback_data");
+}
+
+static gboolean
+rspamd_map_read_cached(struct rspamd_map *map, struct rspamd_map_backend *bk,
+ struct map_periodic_cbdata *periodic, const gchar *host)
+{
+ gsize mmap_len, len;
+ gpointer in;
+ struct http_map_data *data;
+
+ data = bk->data.hd;
+
+ in = rspamd_shmem_xmap(data->cache->shmem_name, PROT_READ, &mmap_len);
+
+ if (in == NULL) {
+ msg_err("cannot map cache from %s: %s", data->cache->shmem_name,
+ strerror(errno));
+ return FALSE;
+ }
+
+ if (mmap_len < data->cache->len) {
+ msg_err("cannot map cache from %s: truncated length %z, %z expected",
+ data->cache->shmem_name,
+ mmap_len, data->cache->len);
+ munmap(in, mmap_len);
+
+ return FALSE;
+ }
+
+ /*
+ * Len is taken from the shmem file size that can be larger than the
+ * actual data length, as we use shared memory as a growing buffer for the
+ * HTTP input.
+ * Hence, we need to use len from the saved cache data, counting that it is
+ * at least not more than the cached file length (this is checked above).
+ */
+ len = data->cache->len;
+
+ if (bk->is_compressed) {
+ ZSTD_DStream *zstream;
+ ZSTD_inBuffer zin;
+ ZSTD_outBuffer zout;
+ guchar *out;
+ gsize outlen, r;
+
+ zstream = ZSTD_createDStream();
+ ZSTD_initDStream(zstream);
+
+ zin.pos = 0;
+ zin.src = in;
+ zin.size = len;
+
+ if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
+ outlen = ZSTD_DStreamOutSize();
+ }
+
+ out = g_malloc(outlen);
+
+ zout.dst = out;
+ zout.pos = 0;
+ zout.size = outlen;
+
+ while (zin.pos < zin.size) {
+ r = ZSTD_decompressStream(zstream, &zout, &zin);
+
+ if (ZSTD_isError(r)) {
+ msg_err_map("%s: cannot decompress data: %s",
+ bk->uri,
+ ZSTD_getErrorName(r));
+ ZSTD_freeDStream(zstream);
+ g_free(out);
+ munmap(in, mmap_len);
+ return FALSE;
+ }
+
+ if (zout.pos == zout.size) {
+ /* We need to extend output buffer */
+ zout.size = zout.size * 2 + 1;
+ out = g_realloc(zout.dst, zout.size);
+ zout.dst = out;
+ }
+ }
+
+ ZSTD_freeDStream(zstream);
+ msg_info_map("%s: read map data cached %z bytes compressed, "
+ "%z uncompressed",
+ bk->uri,
+ len, zout.pos);
+ map->read_callback(out, zout.pos, &periodic->cbdata, TRUE);
+ g_free(out);
+ }
+ else {
+ msg_info_map("%s: read map data cached %z bytes", bk->uri, len);
+ map->read_callback(in, len, &periodic->cbdata, TRUE);
+ }
+
+ munmap(in, mmap_len);
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_map_has_http_cached_file(struct rspamd_map *map,
+ struct rspamd_map_backend *bk)
+{
+ gchar path[PATH_MAX];
+ guchar digest[rspamd_cryptobox_HASHBYTES];
+ struct rspamd_config *cfg = map->cfg;
+ struct stat st;
+
+ if (cfg->maps_cache_dir == NULL || cfg->maps_cache_dir[0] == '\0') {
+ return FALSE;
+ }
+
+ rspamd_cryptobox_hash(digest, bk->uri, strlen(bk->uri), NULL, 0);
+ rspamd_snprintf(path, sizeof(path), "%s%c%*xs.map", cfg->maps_cache_dir,
+ G_DIR_SEPARATOR, 20, digest);
+
+ if (stat(path, &st) != -1 && st.st_size >
+ sizeof(struct rspamd_http_file_data)) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_map_save_http_cached_file(struct rspamd_map *map,
+ struct rspamd_map_backend *bk,
+ struct http_map_data *htdata,
+ const guchar *data,
+ gsize len)
+{
+ gchar path[PATH_MAX];
+ guchar digest[rspamd_cryptobox_HASHBYTES];
+ struct rspamd_config *cfg = map->cfg;
+ gint fd;
+ struct rspamd_http_file_data header;
+
+ if (cfg->maps_cache_dir == NULL || cfg->maps_cache_dir[0] == '\0') {
+ return FALSE;
+ }
+
+ rspamd_cryptobox_hash(digest, bk->uri, strlen(bk->uri), NULL, 0);
+ rspamd_snprintf(path, sizeof(path), "%s%c%*xs.map", cfg->maps_cache_dir,
+ G_DIR_SEPARATOR, 20, digest);
+
+ fd = rspamd_file_xopen(path, O_WRONLY | O_TRUNC | O_CREAT,
+ 00600, FALSE);
+
+ if (fd == -1) {
+ return FALSE;
+ }
+
+ if (!rspamd_file_lock(fd, FALSE)) {
+ msg_err_map("cannot lock file %s: %s", path, strerror(errno));
+ close(fd);
+
+ return FALSE;
+ }
+
+ memcpy(header.magic, rspamd_http_file_magic, sizeof(rspamd_http_file_magic));
+ header.mtime = htdata->last_modified;
+ header.next_check = map->next_check;
+ header.data_off = sizeof(header);
+
+ if (htdata->etag) {
+ header.data_off += RSPAMD_FSTRING_LEN(htdata->etag);
+ header.etag_len = RSPAMD_FSTRING_LEN(htdata->etag);
+ }
+ else {
+ header.etag_len = 0;
+ }
+
+ if (write(fd, &header, sizeof(header)) != sizeof(header)) {
+ msg_err_map("cannot write file %s (header stage): %s", path, strerror(errno));
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+
+ return FALSE;
+ }
+
+ if (header.etag_len > 0) {
+ if (write(fd, RSPAMD_FSTRING_DATA(htdata->etag), header.etag_len) !=
+ header.etag_len) {
+ msg_err_map("cannot write file %s (etag stage): %s", path, strerror(errno));
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+
+ return FALSE;
+ }
+ }
+
+ /* Now write the rest */
+ if (write(fd, data, len) != len) {
+ msg_err_map("cannot write file %s (data stage): %s", path, strerror(errno));
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+
+ return FALSE;
+ }
+
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+
+ msg_info_map("saved data from %s in %s, %uz bytes", bk->uri, path, len + sizeof(header) + header.etag_len);
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_map_update_http_cached_file(struct rspamd_map *map,
+ struct rspamd_map_backend *bk,
+ struct http_map_data *htdata)
+{
+ gchar path[PATH_MAX];
+ guchar digest[rspamd_cryptobox_HASHBYTES];
+ struct rspamd_config *cfg = map->cfg;
+ gint fd;
+ struct rspamd_http_file_data header;
+
+ if (!rspamd_map_has_http_cached_file(map, bk)) {
+ return FALSE;
+ }
+
+ rspamd_cryptobox_hash(digest, bk->uri, strlen(bk->uri), NULL, 0);
+ rspamd_snprintf(path, sizeof(path), "%s%c%*xs.map", cfg->maps_cache_dir,
+ G_DIR_SEPARATOR, 20, digest);
+
+ fd = rspamd_file_xopen(path, O_WRONLY,
+ 00600, FALSE);
+
+ if (fd == -1) {
+ return FALSE;
+ }
+
+ if (!rspamd_file_lock(fd, FALSE)) {
+ msg_err_map("cannot lock file %s: %s", path, strerror(errno));
+ close(fd);
+
+ return FALSE;
+ }
+
+ memcpy(header.magic, rspamd_http_file_magic, sizeof(rspamd_http_file_magic));
+ header.mtime = htdata->last_modified;
+ header.next_check = map->next_check;
+ header.data_off = sizeof(header);
+
+ if (htdata->etag) {
+ header.data_off += RSPAMD_FSTRING_LEN(htdata->etag);
+ header.etag_len = RSPAMD_FSTRING_LEN(htdata->etag);
+ }
+ else {
+ header.etag_len = 0;
+ }
+
+ if (write(fd, &header, sizeof(header)) != sizeof(header)) {
+ msg_err_map("cannot update file %s (header stage): %s", path, strerror(errno));
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+
+ return FALSE;
+ }
+
+ if (header.etag_len > 0) {
+ if (write(fd, RSPAMD_FSTRING_DATA(htdata->etag), header.etag_len) !=
+ header.etag_len) {
+ msg_err_map("cannot update file %s (etag stage): %s", path, strerror(errno));
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+
+ return FALSE;
+ }
+ }
+
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+
+ return TRUE;
+}
+
+
+static gboolean
+rspamd_map_read_http_cached_file(struct rspamd_map *map,
+ struct rspamd_map_backend *bk,
+ struct http_map_data *htdata,
+ struct map_cb_data *cbdata)
+{
+ gchar path[PATH_MAX];
+ guchar digest[rspamd_cryptobox_HASHBYTES];
+ struct rspamd_config *cfg = map->cfg;
+ gint fd;
+ struct stat st;
+ struct rspamd_http_file_data header;
+
+ if (cfg->maps_cache_dir == NULL || cfg->maps_cache_dir[0] == '\0') {
+ return FALSE;
+ }
+
+ rspamd_cryptobox_hash(digest, bk->uri, strlen(bk->uri), NULL, 0);
+ rspamd_snprintf(path, sizeof(path), "%s%c%*xs.map", cfg->maps_cache_dir,
+ G_DIR_SEPARATOR, 20, digest);
+
+ fd = rspamd_file_xopen(path, O_RDONLY, 00600, FALSE);
+
+ if (fd == -1) {
+ return FALSE;
+ }
+
+ if (!rspamd_file_lock(fd, FALSE)) {
+ msg_err_map("cannot lock file %s: %s", path, strerror(errno));
+ close(fd);
+
+ return FALSE;
+ }
+
+ (void) fstat(fd, &st);
+
+ if (read(fd, &header, sizeof(header)) != sizeof(header)) {
+ msg_err_map("cannot read file %s (header stage): %s", path, strerror(errno));
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+
+ return FALSE;
+ }
+
+ if (memcmp(header.magic, rspamd_http_file_magic,
+ sizeof(rspamd_http_file_magic)) != 0) {
+ msg_warn_map("invalid or old version magic in file %s; ignore it", path);
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+
+ return FALSE;
+ }
+
+ double now = rspamd_get_calendar_ticks();
+
+ if (header.next_check > now) {
+ map->next_check = header.next_check;
+ }
+ else {
+ map->next_check = now;
+ }
+
+ htdata->last_modified = header.mtime;
+
+ if (header.etag_len > 0) {
+ rspamd_fstring_t *etag = rspamd_fstring_sized_new(header.etag_len);
+
+ if (read(fd, RSPAMD_FSTRING_DATA(etag), header.etag_len) != header.etag_len) {
+ msg_err_map("cannot read file %s (etag stage): %s", path,
+ strerror(errno));
+ rspamd_file_unlock(fd, FALSE);
+ rspamd_fstring_free(etag);
+ close(fd);
+
+ return FALSE;
+ }
+
+ etag->len = header.etag_len;
+
+ if (htdata->etag) {
+ /* FIXME: should be dealt somehow better */
+ msg_warn_map("etag is already defined as %V; cached is %V; ignore cached",
+ htdata->etag, etag);
+ rspamd_fstring_free(etag);
+ }
+ else {
+ htdata->etag = etag;
+ }
+ }
+
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+
+ /* Now read file data */
+ /* Perform buffered read: fail-safe */
+ if (!read_map_file_chunks(map, cbdata, path,
+ st.st_size - header.data_off, header.data_off)) {
+ return FALSE;
+ }
+
+ struct tm tm;
+ gchar ncheck_buf[32], lm_buf[32];
+
+ rspamd_localtime(map->next_check, &tm);
+ strftime(ncheck_buf, sizeof(ncheck_buf) - 1, "%Y-%m-%d %H:%M:%S", &tm);
+ rspamd_localtime(htdata->last_modified, &tm);
+ strftime(lm_buf, sizeof(lm_buf) - 1, "%Y-%m-%d %H:%M:%S", &tm);
+
+ msg_info_map("read cached data for %s from %s, %uz bytes; next check at: %s;"
+ " last modified on: %s; etag: %V",
+ bk->uri,
+ path,
+ (size_t) (st.st_size - header.data_off),
+ ncheck_buf,
+ lm_buf,
+ htdata->etag);
+
+ return TRUE;
+}
+
+/**
+ * Async HTTP callback
+ */
+static void
+rspamd_map_common_http_callback(struct rspamd_map *map,
+ struct rspamd_map_backend *bk,
+ struct map_periodic_cbdata *periodic,
+ gboolean check)
+{
+ struct http_map_data *data;
+ struct http_callback_data *cbd;
+ guint flags = RSPAMD_HTTP_CLIENT_SIMPLE | RSPAMD_HTTP_CLIENT_SHARED;
+
+ data = bk->data.hd;
+
+ if (g_atomic_int_get(&data->cache->available) == 1) {
+ /* Read cached data */
+ if (check) {
+ if (data->last_modified < data->cache->last_modified) {
+ msg_info_map("need to reread cached map triggered by %s "
+ "(%d our modify time, %d cached modify time)",
+ bk->uri,
+ (int) data->last_modified,
+ (int) data->cache->last_modified);
+ periodic->need_modify = TRUE;
+ /* Reset the whole chain */
+ periodic->cur_backend = 0;
+ rspamd_map_process_periodic(periodic);
+ }
+ else {
+ if (map->active_http) {
+ /* Check even if there is a cached version */
+ goto check;
+ }
+ else {
+ /* Switch to the next backend */
+ periodic->cur_backend++;
+ rspamd_map_process_periodic(periodic);
+ }
+ }
+
+ return;
+ }
+ else {
+ if (map->active_http &&
+ data->last_modified > data->cache->last_modified) {
+ goto check;
+ }
+ else if (rspamd_map_read_cached(map, bk, periodic, data->host)) {
+ /* Switch to the next backend */
+ periodic->cur_backend++;
+ data->last_modified = data->cache->last_modified;
+ rspamd_map_process_periodic(periodic);
+
+ return;
+ }
+ }
+ }
+ else if (!map->active_http) {
+ /* Switch to the next backend */
+ periodic->cur_backend++;
+ rspamd_map_process_periodic(periodic);
+
+ return;
+ }
+
+check:
+ cbd = g_malloc0(sizeof(struct http_callback_data));
+
+ cbd->event_loop = map->event_loop;
+ cbd->addrs = g_ptr_array_sized_new(4);
+ cbd->map = map;
+ cbd->data = data;
+ cbd->check = check;
+ cbd->periodic = periodic;
+ MAP_RETAIN(periodic, "periodic");
+ cbd->bk = bk;
+ MAP_RETAIN(bk, "rspamd_map_backend");
+ cbd->stage = http_map_terminated;
+ REF_INIT_RETAIN(cbd, free_http_cbdata);
+
+ msg_debug_map("%s map data from %s", check ? "checking" : "reading",
+ data->host);
+
+ /* Try address */
+ rspamd_inet_addr_t *addr = NULL;
+
+ if (rspamd_parse_inet_address(&addr, data->host,
+ strlen(data->host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ rspamd_inet_address_set_port(addr, cbd->data->port);
+ g_ptr_array_add(cbd->addrs, (void *) addr);
+
+ if (bk->protocol == MAP_PROTO_HTTPS) {
+ flags |= RSPAMD_HTTP_CLIENT_SSL;
+ }
+
+ cbd->conn = rspamd_http_connection_new_client(
+ NULL,
+ NULL,
+ http_map_error,
+ http_map_finish,
+ flags,
+ addr);
+
+ if (cbd->conn != NULL) {
+ cbd->stage = http_map_http_conn;
+ write_http_request(cbd);
+ cbd->addr = addr;
+ MAP_RELEASE(cbd, "http_callback_data");
+ }
+ else {
+ msg_warn_map("cannot load map: cannot connect to %s: %s",
+ data->host, strerror(errno));
+ MAP_RELEASE(cbd, "http_callback_data");
+ }
+
+ return;
+ }
+ else if (map->r->r) {
+ /* Send both A and AAAA requests */
+ guint nreq = 0;
+
+ if (rdns_make_request_full(map->r->r, rspamd_map_dns_callback, cbd,
+ map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
+ data->host, RDNS_REQUEST_A)) {
+ MAP_RETAIN(cbd, "http_callback_data");
+ nreq++;
+ }
+ if (rdns_make_request_full(map->r->r, rspamd_map_dns_callback, cbd,
+ map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
+ data->host, RDNS_REQUEST_AAAA)) {
+ MAP_RETAIN(cbd, "http_callback_data");
+ nreq++;
+ }
+
+ if (nreq == 2) {
+ cbd->stage = http_map_resolve_host2;
+ }
+ else if (nreq == 1) {
+ cbd->stage = http_map_resolve_host1;
+ }
+
+ map->tmp_dtor = free_http_cbdata_dtor;
+ map->tmp_dtor_data = cbd;
+ }
+ else {
+ msg_warn_map("cannot load map: DNS resolver is not initialized");
+ cbd->periodic->errored = TRUE;
+ }
+
+ MAP_RELEASE(cbd, "http_callback_data");
+}
+
+static void
+rspamd_map_http_check_callback(struct map_periodic_cbdata *cbd)
+{
+ struct rspamd_map *map;
+ struct rspamd_map_backend *bk;
+
+ map = cbd->map;
+ bk = g_ptr_array_index(cbd->map->backends, cbd->cur_backend);
+
+ rspamd_map_common_http_callback(map, bk, cbd, TRUE);
+}
+
+static void
+rspamd_map_http_read_callback(struct map_periodic_cbdata *cbd)
+{
+ struct rspamd_map *map;
+ struct rspamd_map_backend *bk;
+
+ map = cbd->map;
+ bk = g_ptr_array_index(cbd->map->backends, cbd->cur_backend);
+ rspamd_map_common_http_callback(map, bk, cbd, FALSE);
+}
+
+static void
+rspamd_map_file_check_callback(struct map_periodic_cbdata *periodic)
+{
+ struct rspamd_map *map;
+ struct file_map_data *data;
+ struct rspamd_map_backend *bk;
+
+ map = periodic->map;
+ bk = g_ptr_array_index(map->backends, periodic->cur_backend);
+ data = bk->data.fd;
+
+ if (data->need_modify) {
+ periodic->need_modify = TRUE;
+ periodic->cur_backend = 0;
+ data->need_modify = FALSE;
+
+ rspamd_map_process_periodic(periodic);
+
+ return;
+ }
+
+ map = periodic->map;
+ /* Switch to the next backend as the rest is handled by ev_stat */
+ periodic->cur_backend++;
+ rspamd_map_process_periodic(periodic);
+}
+
+static void
+rspamd_map_static_check_callback(struct map_periodic_cbdata *periodic)
+{
+ struct rspamd_map *map;
+ struct static_map_data *data;
+ struct rspamd_map_backend *bk;
+
+ map = periodic->map;
+ bk = g_ptr_array_index(map->backends, periodic->cur_backend);
+ data = bk->data.sd;
+
+ if (!data->processed) {
+ periodic->need_modify = TRUE;
+ periodic->cur_backend = 0;
+
+ rspamd_map_process_periodic(periodic);
+
+ return;
+ }
+
+ /* Switch to the next backend */
+ periodic->cur_backend++;
+ rspamd_map_process_periodic(periodic);
+}
+
+static void
+rspamd_map_file_read_callback(struct map_periodic_cbdata *periodic)
+{
+ struct rspamd_map *map;
+ struct file_map_data *data;
+ struct rspamd_map_backend *bk;
+
+ map = periodic->map;
+
+ bk = g_ptr_array_index(map->backends, periodic->cur_backend);
+ data = bk->data.fd;
+
+ msg_info_map("rereading map file %s", data->filename);
+
+ if (!read_map_file(map, data, bk, periodic)) {
+ periodic->errored = TRUE;
+ }
+
+ /* Switch to the next backend */
+ periodic->cur_backend++;
+ rspamd_map_process_periodic(periodic);
+}
+
+static void
+rspamd_map_static_read_callback(struct map_periodic_cbdata *periodic)
+{
+ struct rspamd_map *map;
+ struct static_map_data *data;
+ struct rspamd_map_backend *bk;
+
+ map = periodic->map;
+
+ bk = g_ptr_array_index(map->backends, periodic->cur_backend);
+ data = bk->data.sd;
+
+ msg_info_map("rereading static map");
+
+ if (!read_map_static(map, data, bk, periodic)) {
+ periodic->errored = TRUE;
+ }
+
+ /* Switch to the next backend */
+ periodic->cur_backend++;
+ rspamd_map_process_periodic(periodic);
+}
+
+static void
+rspamd_map_process_periodic(struct map_periodic_cbdata *cbd)
+{
+ struct rspamd_map_backend *bk;
+ struct rspamd_map *map;
+
+ map = cbd->map;
+ map->scheduled_check = NULL;
+
+ if (!map->file_only && !cbd->locked) {
+ if (!g_atomic_int_compare_and_exchange(cbd->map->locked,
+ 0, 1)) {
+ msg_debug_map(
+ "don't try to reread map %s as it is locked by other process, "
+ "will reread it later",
+ cbd->map->name);
+ rspamd_map_schedule_periodic(map, RSPAMD_MAP_SCHEDULE_LOCKED);
+ MAP_RELEASE(cbd, "periodic");
+
+ return;
+ }
+ else {
+ msg_debug_map("locked map %s", cbd->map->name);
+ cbd->locked = TRUE;
+ }
+ }
+
+ if (cbd->errored) {
+ /* We should not check other backends if some backend has failed*/
+ rspamd_map_schedule_periodic(cbd->map, RSPAMD_MAP_SCHEDULE_ERROR);
+
+ if (cbd->locked) {
+ g_atomic_int_set(cbd->map->locked, 0);
+ cbd->locked = FALSE;
+ }
+
+ /* Also set error flag for the map consumer */
+ cbd->cbdata.errored = true;
+
+ msg_debug_map("unlocked map %s, refcount=%d", cbd->map->name,
+ cbd->ref.refcount);
+ MAP_RELEASE(cbd, "periodic");
+
+ return;
+ }
+
+ /* For each backend we need to check for modifications */
+ if (cbd->cur_backend >= cbd->map->backends->len) {
+ /* Last backend */
+ msg_debug_map("finished map: %d of %d", cbd->cur_backend,
+ cbd->map->backends->len);
+ MAP_RELEASE(cbd, "periodic");
+
+ return;
+ }
+
+ if (cbd->map->wrk && cbd->map->wrk->state == rspamd_worker_state_running) {
+ bk = g_ptr_array_index(cbd->map->backends, cbd->cur_backend);
+ g_assert(bk != NULL);
+
+ if (cbd->need_modify) {
+ /* Load data from the next backend */
+ switch (bk->protocol) {
+ case MAP_PROTO_HTTP:
+ case MAP_PROTO_HTTPS:
+ rspamd_map_http_read_callback(cbd);
+ break;
+ case MAP_PROTO_FILE:
+ rspamd_map_file_read_callback(cbd);
+ break;
+ case MAP_PROTO_STATIC:
+ rspamd_map_static_read_callback(cbd);
+ break;
+ }
+ }
+ else {
+ /* Check the next backend */
+ switch (bk->protocol) {
+ case MAP_PROTO_HTTP:
+ case MAP_PROTO_HTTPS:
+ rspamd_map_http_check_callback(cbd);
+ break;
+ case MAP_PROTO_FILE:
+ rspamd_map_file_check_callback(cbd);
+ break;
+ case MAP_PROTO_STATIC:
+ rspamd_map_static_check_callback(cbd);
+ break;
+ }
+ }
+ }
+}
+
+static void
+rspamd_map_on_stat(struct ev_loop *loop, ev_stat *w, int revents)
+{
+ struct rspamd_map *map = (struct rspamd_map *) w->data;
+
+ if (w->attr.st_nlink > 0) {
+ msg_info_map("old mtime is %t (size = %Hz), "
+ "new mtime is %t (size = %Hz) for map file %s",
+ w->prev.st_mtime, (gsize) w->prev.st_size,
+ w->attr.st_mtime, (gsize) w->attr.st_size,
+ w->path);
+
+ /* Fire need modify flag */
+ struct rspamd_map_backend *bk;
+ guint i;
+
+ PTR_ARRAY_FOREACH(map->backends, i, bk)
+ {
+ if (bk->protocol == MAP_PROTO_FILE) {
+ bk->data.fd->need_modify = TRUE;
+ }
+ }
+
+ map->next_check = 0;
+
+ if (map->scheduled_check) {
+ ev_timer_stop(map->event_loop, &map->scheduled_check->ev);
+ MAP_RELEASE(map->scheduled_check, "rspamd_map_on_stat");
+ map->scheduled_check = NULL;
+ }
+
+ rspamd_map_schedule_periodic(map, RSPAMD_MAP_SCHEDULE_INIT);
+ }
+}
+
+/* Start watching event for all maps */
+void rspamd_map_watch(struct rspamd_config *cfg,
+ struct ev_loop *event_loop,
+ struct rspamd_dns_resolver *resolver,
+ struct rspamd_worker *worker,
+ enum rspamd_map_watch_type how)
+{
+ GList *cur = cfg->maps;
+ struct rspamd_map *map;
+ struct rspamd_map_backend *bk;
+ guint i;
+
+ g_assert(how > RSPAMD_MAP_WATCH_MIN && how < RSPAMD_MAP_WATCH_MAX);
+
+ /* First of all do synced read of data */
+ while (cur) {
+ map = cur->data;
+ map->event_loop = event_loop;
+ map->r = resolver;
+
+ if (map->wrk == NULL && how != RSPAMD_MAP_WATCH_WORKER) {
+ /* Generic scanner map */
+ map->wrk = worker;
+
+ if (how == RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER) {
+ map->active_http = TRUE;
+ }
+ else {
+ map->active_http = FALSE;
+ }
+ }
+ else if (map->wrk != NULL && map->wrk == worker) {
+ /* Map is bound to a specific worker */
+ map->active_http = TRUE;
+ }
+ else {
+ /* Skip map for this worker as irrelevant */
+ cur = g_list_next(cur);
+ continue;
+ }
+
+ if (!map->active_http) {
+ /* Check cached version more frequently as it is cheap */
+
+ if (map->poll_timeout >= cfg->map_timeout &&
+ cfg->map_file_watch_multiplier < 1.0) {
+ map->poll_timeout =
+ map->poll_timeout * cfg->map_file_watch_multiplier;
+ }
+ }
+
+ map->file_only = TRUE;
+ map->static_only = TRUE;
+
+ PTR_ARRAY_FOREACH(map->backends, i, bk)
+ {
+ bk->event_loop = event_loop;
+
+ if (bk->protocol == MAP_PROTO_FILE) {
+ struct file_map_data *data;
+
+ data = bk->data.fd;
+
+ if (map->user_data == NULL || *map->user_data == NULL) {
+ /* Map has not been read, init it's reading if possible */
+ struct stat st;
+
+ if (stat(data->filename, &st) != -1) {
+ data->need_modify = TRUE;
+ }
+ }
+
+ ev_stat_init(&data->st_ev, rspamd_map_on_stat,
+ data->filename, map->poll_timeout * cfg->map_file_watch_multiplier);
+ data->st_ev.data = map;
+ ev_stat_start(event_loop, &data->st_ev);
+ map->static_only = FALSE;
+ }
+ else if ((bk->protocol == MAP_PROTO_HTTP ||
+ bk->protocol == MAP_PROTO_HTTPS)) {
+ if (map->active_http) {
+ map->non_trivial = TRUE;
+ }
+
+ map->static_only = FALSE;
+ map->file_only = FALSE;
+ }
+ }
+
+ rspamd_map_schedule_periodic(map, RSPAMD_MAP_SCHEDULE_INIT);
+
+ cur = g_list_next(cur);
+ }
+}
+
+void rspamd_map_preload(struct rspamd_config *cfg)
+{
+ GList *cur = cfg->maps;
+ struct rspamd_map *map;
+ struct rspamd_map_backend *bk;
+ guint i;
+ gboolean map_ok;
+
+ /* First of all do synced read of data */
+ while (cur) {
+ map = cur->data;
+ map_ok = TRUE;
+
+ PTR_ARRAY_FOREACH(map->backends, i, bk)
+ {
+ if (!(bk->protocol == MAP_PROTO_FILE ||
+ bk->protocol == MAP_PROTO_STATIC)) {
+
+ if (bk->protocol == MAP_PROTO_HTTP ||
+ bk->protocol == MAP_PROTO_HTTPS) {
+ if (!rspamd_map_has_http_cached_file(map, bk)) {
+
+ if (!map->fallback_backend) {
+ map_ok = FALSE;
+ }
+ break;
+ }
+ else {
+ continue; /* We are yet fine */
+ }
+ }
+ map_ok = FALSE;
+ break;
+ }
+ }
+
+ if (map_ok) {
+ struct map_periodic_cbdata fake_cbd;
+ gboolean succeed = TRUE;
+
+ memset(&fake_cbd, 0, sizeof(fake_cbd));
+ fake_cbd.cbdata.state = 0;
+ fake_cbd.cbdata.prev_data = *map->user_data;
+ fake_cbd.cbdata.cur_data = NULL;
+ fake_cbd.cbdata.map = map;
+ fake_cbd.map = map;
+
+ PTR_ARRAY_FOREACH(map->backends, i, bk)
+ {
+ fake_cbd.cur_backend = i;
+
+ if (bk->protocol == MAP_PROTO_FILE) {
+ if (!read_map_file(map, bk->data.fd, bk, &fake_cbd)) {
+ succeed = FALSE;
+ break;
+ }
+ }
+ else if (bk->protocol == MAP_PROTO_STATIC) {
+ if (!read_map_static(map, bk->data.sd, bk, &fake_cbd)) {
+ succeed = FALSE;
+ break;
+ }
+ }
+ else if (bk->protocol == MAP_PROTO_HTTP ||
+ bk->protocol == MAP_PROTO_HTTPS) {
+ if (!rspamd_map_read_http_cached_file(map, bk, bk->data.hd,
+ &fake_cbd.cbdata)) {
+
+ if (map->fallback_backend) {
+ /* Try fallback */
+ g_assert(map->fallback_backend->protocol ==
+ MAP_PROTO_FILE);
+ if (!read_map_file(map,
+ map->fallback_backend->data.fd,
+ map->fallback_backend, &fake_cbd)) {
+ succeed = FALSE;
+ break;
+ }
+ }
+ else {
+ succeed = FALSE;
+ break;
+ }
+ }
+ }
+ else {
+ g_assert_not_reached();
+ }
+ }
+
+ if (succeed) {
+ map->fin_callback(&fake_cbd.cbdata, map->user_data);
+
+ if (map->on_load_function) {
+ map->on_load_function(map, map->on_load_ud);
+ }
+ }
+ else {
+ msg_info_map("preload of %s failed", map->name);
+ }
+ }
+
+ cur = g_list_next(cur);
+ }
+}
+
+void rspamd_map_remove_all(struct rspamd_config *cfg)
+{
+ struct rspamd_map *map;
+ GList *cur;
+ struct rspamd_map_backend *bk;
+ struct map_cb_data cbdata;
+ guint i;
+
+ for (cur = cfg->maps; cur != NULL; cur = g_list_next(cur)) {
+ map = cur->data;
+
+ if (map->tmp_dtor) {
+ map->tmp_dtor(map->tmp_dtor_data);
+ }
+
+ if (map->dtor) {
+ cbdata.prev_data = NULL;
+ cbdata.map = map;
+ cbdata.cur_data = *map->user_data;
+
+ map->dtor(&cbdata);
+ *map->user_data = NULL;
+ }
+
+ if (map->on_load_ud_dtor) {
+ map->on_load_ud_dtor(map->on_load_ud);
+ }
+
+ for (i = 0; i < map->backends->len; i++) {
+ bk = g_ptr_array_index(map->backends, i);
+
+ MAP_RELEASE(bk, "rspamd_map_backend");
+ }
+
+ if (map->fallback_backend) {
+ MAP_RELEASE(map->fallback_backend, "rspamd_map_backend");
+ }
+ }
+
+ g_list_free(cfg->maps);
+ cfg->maps = NULL;
+}
+
+static const gchar *
+rspamd_map_check_proto(struct rspamd_config *cfg,
+ const gchar *map_line, struct rspamd_map_backend *bk)
+{
+ const gchar *pos = map_line, *end, *end_key;
+
+ g_assert(bk != NULL);
+ g_assert(pos != NULL);
+
+ end = pos + strlen(pos);
+
+ /* Static check */
+ if (g_ascii_strcasecmp(pos, "static") == 0) {
+ bk->protocol = MAP_PROTO_STATIC;
+ bk->uri = g_strdup(pos);
+
+ return pos;
+ }
+ else if (g_ascii_strcasecmp(pos, "zst+static") == 0) {
+ bk->protocol = MAP_PROTO_STATIC;
+ bk->uri = g_strdup(pos + 4);
+ bk->is_compressed = TRUE;
+
+ return pos + 4;
+ }
+
+ for (;;) {
+ if (g_ascii_strncasecmp(pos, "sign+", sizeof("sign+") - 1) == 0) {
+ bk->is_signed = TRUE;
+ pos += sizeof("sign+") - 1;
+ }
+ else if (g_ascii_strncasecmp(pos, "fallback+", sizeof("fallback+") - 1) == 0) {
+ bk->is_fallback = TRUE;
+ pos += sizeof("fallback+") - 1;
+ }
+ else if (g_ascii_strncasecmp(pos, "key=", sizeof("key=") - 1) == 0) {
+ pos += sizeof("key=") - 1;
+ end_key = memchr(pos, '+', end - pos);
+
+ if (end_key != NULL) {
+ bk->trusted_pubkey = rspamd_pubkey_from_base32(pos, end_key - pos,
+ RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519);
+
+ if (bk->trusted_pubkey == NULL) {
+ msg_err_config("cannot read pubkey from map: %s",
+ map_line);
+ return NULL;
+ }
+ pos = end_key + 1;
+ }
+ else if (end - pos > 64) {
+ /* Try hex encoding */
+ bk->trusted_pubkey = rspamd_pubkey_from_hex(pos, 64,
+ RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519);
+
+ if (bk->trusted_pubkey == NULL) {
+ msg_err_config("cannot read pubkey from map: %s",
+ map_line);
+ return NULL;
+ }
+ pos += 64;
+ }
+ else {
+ msg_err_config("cannot read pubkey from map: %s",
+ map_line);
+ return NULL;
+ }
+
+ if (*pos == '+' || *pos == ':') {
+ pos++;
+ }
+ }
+ else {
+ /* No known flags */
+ break;
+ }
+ }
+
+ bk->protocol = MAP_PROTO_FILE;
+
+ if (g_ascii_strncasecmp(pos, "http://", sizeof("http://") - 1) == 0) {
+ bk->protocol = MAP_PROTO_HTTP;
+ /* Include http:// */
+ bk->uri = g_strdup(pos);
+ pos += sizeof("http://") - 1;
+ }
+ else if (g_ascii_strncasecmp(pos, "https://", sizeof("https://") - 1) == 0) {
+ bk->protocol = MAP_PROTO_HTTPS;
+ /* Include https:// */
+ bk->uri = g_strdup(pos);
+ pos += sizeof("https://") - 1;
+ }
+ else if (g_ascii_strncasecmp(pos, "file://", sizeof("file://") - 1) == 0) {
+ pos += sizeof("file://") - 1;
+ /* Exclude file:// */
+ bk->uri = g_strdup(pos);
+ }
+ else if (*pos == '/') {
+ /* Trivial file case */
+ bk->uri = g_strdup(pos);
+ }
+ else {
+ msg_err_config("invalid map fetching protocol: %s", map_line);
+
+ return NULL;
+ }
+
+ if (bk->protocol != MAP_PROTO_FILE && bk->is_signed) {
+ msg_err_config("signed maps are no longer supported for HTTP(s): %s", map_line);
+ }
+
+ return pos;
+}
+
+gboolean
+rspamd_map_is_map(const gchar *map_line)
+{
+ gboolean ret = FALSE;
+
+ g_assert(map_line != NULL);
+
+ if (map_line[0] == '/') {
+ ret = TRUE;
+ }
+ else if (g_ascii_strncasecmp(map_line, "sign+", sizeof("sign+") - 1) == 0) {
+ ret = TRUE;
+ }
+ else if (g_ascii_strncasecmp(map_line, "fallback+", sizeof("fallback+") - 1) == 0) {
+ ret = TRUE;
+ }
+ else if (g_ascii_strncasecmp(map_line, "file://", sizeof("file://") - 1) == 0) {
+ ret = TRUE;
+ }
+ else if (g_ascii_strncasecmp(map_line, "http://", sizeof("http://") - 1) == 0) {
+ ret = TRUE;
+ }
+ else if (g_ascii_strncasecmp(map_line, "https://", sizeof("https://") - 1) == 0) {
+ ret = TRUE;
+ }
+
+ return ret;
+}
+
+static void
+rspamd_map_backend_dtor(struct rspamd_map_backend *bk)
+{
+ switch (bk->protocol) {
+ case MAP_PROTO_FILE:
+ if (bk->data.fd) {
+ ev_stat_stop(bk->event_loop, &bk->data.fd->st_ev);
+ g_free(bk->data.fd->filename);
+ g_free(bk->data.fd);
+ }
+ break;
+ case MAP_PROTO_STATIC:
+ if (bk->data.sd) {
+ if (bk->data.sd->data) {
+ g_free(bk->data.sd->data);
+ }
+
+ g_free(bk->data.sd);
+ }
+ break;
+ case MAP_PROTO_HTTP:
+ case MAP_PROTO_HTTPS:
+ if (bk->data.hd) {
+ struct http_map_data *data = bk->data.hd;
+
+ g_free(data->host);
+ g_free(data->path);
+ g_free(data->rest);
+
+ if (data->userinfo) {
+ g_free(data->userinfo);
+ }
+
+ if (data->etag) {
+ rspamd_fstring_free(data->etag);
+ }
+
+ /*
+ * Clear cached file, but check if a worker is an active http worker
+ * as cur_cache_cbd is meaningful merely for active worker, who actually
+ * owns the cache
+ */
+ if (bk->map && bk->map->active_http) {
+ if (g_atomic_int_compare_and_exchange(&data->cache->available, 1, 0)) {
+ if (data->cur_cache_cbd) {
+ msg_info("clear shared memory cache for a map in %s as backend \"%s\" is closing",
+ data->cur_cache_cbd->shm->shm_name,
+ bk->uri);
+ MAP_RELEASE(data->cur_cache_cbd->shm,
+ "rspamd_http_map_cached_cbdata");
+ ev_timer_stop(data->cur_cache_cbd->event_loop,
+ &data->cur_cache_cbd->timeout);
+ g_free(data->cur_cache_cbd);
+ data->cur_cache_cbd = NULL;
+ }
+ }
+ }
+
+ g_free(bk->data.hd);
+ }
+ break;
+ }
+
+ if (bk->trusted_pubkey) {
+ rspamd_pubkey_unref(bk->trusted_pubkey);
+ }
+
+ g_free(bk->uri);
+ g_free(bk);
+}
+
+static struct rspamd_map_backend *
+rspamd_map_parse_backend(struct rspamd_config *cfg, const gchar *map_line)
+{
+ struct rspamd_map_backend *bk;
+ struct file_map_data *fdata = NULL;
+ struct http_map_data *hdata = NULL;
+ struct static_map_data *sdata = NULL;
+ struct http_parser_url up;
+ const gchar *end, *p;
+ rspamd_ftok_t tok;
+
+ bk = g_malloc0(sizeof(*bk));
+ REF_INIT_RETAIN(bk, rspamd_map_backend_dtor);
+
+ if (!rspamd_map_check_proto(cfg, map_line, bk)) {
+ goto err;
+ }
+
+ if (bk->is_fallback && bk->protocol != MAP_PROTO_FILE) {
+ msg_err_config("fallback backend must be file for %s", bk->uri);
+
+ goto err;
+ }
+
+ end = map_line + strlen(map_line);
+ if (end - map_line > 5) {
+ p = end - 5;
+ if (g_ascii_strcasecmp(p, ".zstd") == 0) {
+ bk->is_compressed = TRUE;
+ }
+ p = end - 4;
+ if (g_ascii_strcasecmp(p, ".zst") == 0) {
+ bk->is_compressed = TRUE;
+ }
+ }
+
+ /* Now check for each proto separately */
+ if (bk->protocol == MAP_PROTO_FILE) {
+ fdata = g_malloc0(sizeof(struct file_map_data));
+
+ if (access(bk->uri, R_OK) == -1) {
+ if (errno != ENOENT) {
+ msg_err_config("cannot open file '%s': %s", bk->uri, strerror(errno));
+ goto err;
+ }
+
+ msg_info_config(
+ "map '%s' is not found, but it can be loaded automatically later",
+ bk->uri);
+ }
+
+ fdata->filename = g_strdup(bk->uri);
+ bk->data.fd = fdata;
+ }
+ else if (bk->protocol == MAP_PROTO_HTTP || bk->protocol == MAP_PROTO_HTTPS) {
+ hdata = g_malloc0(sizeof(struct http_map_data));
+
+ memset(&up, 0, sizeof(up));
+ if (http_parser_parse_url(bk->uri, strlen(bk->uri), FALSE,
+ &up) != 0) {
+ msg_err_config("cannot parse HTTP url: %s", bk->uri);
+ goto err;
+ }
+ else {
+ if (!(up.field_set & 1u << UF_HOST)) {
+ msg_err_config("cannot parse HTTP url: %s: no host", bk->uri);
+ goto err;
+ }
+
+ tok.begin = bk->uri + up.field_data[UF_HOST].off;
+ tok.len = up.field_data[UF_HOST].len;
+ hdata->host = rspamd_ftokdup(&tok);
+
+ if (up.field_set & (1u << UF_PORT)) {
+ hdata->port = up.port;
+ }
+ else {
+ if (bk->protocol == MAP_PROTO_HTTP) {
+ hdata->port = 80;
+ }
+ else {
+ hdata->port = 443;
+ }
+ }
+
+ if (up.field_set & (1u << UF_PATH)) {
+ tok.begin = bk->uri + up.field_data[UF_PATH].off;
+ tok.len = up.field_data[UF_PATH].len;
+
+ hdata->path = rspamd_ftokdup(&tok);
+
+ /* We also need to check query + fragment */
+ if (up.field_set & ((1u << UF_QUERY) | (1u << UF_FRAGMENT))) {
+ tok.begin = bk->uri + up.field_data[UF_PATH].off +
+ up.field_data[UF_PATH].len;
+ tok.len = strlen(tok.begin);
+ hdata->rest = rspamd_ftokdup(&tok);
+ }
+ else {
+ hdata->rest = g_strdup("");
+ }
+ }
+
+ if (up.field_set & (1u << UF_USERINFO)) {
+ /* Create authorisation header for basic auth */
+ guint len = sizeof("Basic ") +
+ up.field_data[UF_USERINFO].len * 8 / 5 + 4;
+ hdata->userinfo = g_malloc(len);
+ rspamd_snprintf(hdata->userinfo, len, "Basic %*Bs",
+ (int) up.field_data[UF_USERINFO].len,
+ bk->uri + up.field_data[UF_USERINFO].off);
+
+ msg_debug("added userinfo for the map from the URL: %s", hdata->host);
+ }
+ else {
+ /* Try to obtain authentication data from options in the configuration */
+ const ucl_object_t *auth_obj, *opts_obj;
+
+ opts_obj = ucl_object_lookup(cfg->cfg_ucl_obj, "options");
+ if (opts_obj != NULL) {
+ auth_obj = ucl_object_lookup(opts_obj, "http_auth");
+ if (auth_obj != NULL && ucl_object_type(auth_obj) == UCL_OBJECT) {
+ const ucl_object_t *host_obj;
+
+ /*
+ * Search first by the full URL and then by the host part
+ */
+ host_obj = ucl_object_lookup(auth_obj, map_line);
+
+ if (host_obj == NULL) {
+ host_obj = ucl_object_lookup(auth_obj, hdata->host);
+ }
+
+ if (host_obj != NULL && ucl_object_type(host_obj) == UCL_OBJECT) {
+ const ucl_object_t *user_obj, *password_obj;
+
+ user_obj = ucl_object_lookup(host_obj, "user");
+ password_obj = ucl_object_lookup(host_obj, "password");
+
+ if (user_obj != NULL && password_obj != NULL &&
+ ucl_object_type(user_obj) == UCL_STRING &&
+ ucl_object_type(password_obj) == UCL_STRING) {
+
+ gchar *tmpbuf;
+ unsigned tlen;
+
+ /* User + password + ':' */
+ tlen = strlen(ucl_object_tostring(user_obj)) +
+ strlen(ucl_object_tostring(password_obj)) + 1;
+ tmpbuf = g_malloc(tlen + 1);
+ rspamd_snprintf(tmpbuf, tlen + 1, "%s:%s",
+ ucl_object_tostring(user_obj),
+ ucl_object_tostring(password_obj));
+ /* Base64 encoding is not so greedy, but we add some space for simplicity */
+ tlen *= 2;
+ tlen += sizeof("Basic ") - 1;
+ hdata->userinfo = g_malloc(tlen + 1);
+ rspamd_snprintf(hdata->userinfo, tlen + 1, "Basic %Bs", tmpbuf);
+ g_free(tmpbuf);
+ msg_debug("added userinfo for the map from the configuration: %s", map_line);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ hdata->cache = rspamd_mempool_alloc0_shared(cfg->cfg_pool,
+ sizeof(*hdata->cache));
+
+ bk->data.hd = hdata;
+ }
+ else if (bk->protocol == MAP_PROTO_STATIC) {
+ sdata = g_malloc0(sizeof(*sdata));
+ bk->data.sd = sdata;
+ }
+
+ bk->id = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_T1HA,
+ bk->uri, strlen(bk->uri),
+ 0xdeadbabe);
+
+ return bk;
+
+err:
+ MAP_RELEASE(bk, "rspamd_map_backend");
+
+ if (hdata) {
+ g_free(hdata);
+ }
+
+ if (fdata) {
+ g_free(fdata);
+ }
+
+ if (sdata) {
+ g_free(sdata);
+ }
+
+ return NULL;
+}
+
+static void
+rspamd_map_calculate_hash(struct rspamd_map *map)
+{
+ struct rspamd_map_backend *bk;
+ guint i;
+ rspamd_cryptobox_hash_state_t st;
+ gchar *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES];
+
+ rspamd_cryptobox_hash_init(&st, NULL, 0);
+
+ for (i = 0; i < map->backends->len; i++) {
+ bk = g_ptr_array_index(map->backends, i);
+ rspamd_cryptobox_hash_update(&st, bk->uri, strlen(bk->uri));
+ }
+
+ rspamd_cryptobox_hash_final(&st, cksum);
+ cksum_encoded = rspamd_encode_base32(cksum, sizeof(cksum), RSPAMD_BASE32_DEFAULT);
+ rspamd_strlcpy(map->tag, cksum_encoded, sizeof(map->tag));
+ g_free(cksum_encoded);
+}
+
+static gboolean
+rspamd_map_add_static_string(struct rspamd_config *cfg,
+ const ucl_object_t *elt,
+ GString *target)
+{
+ gsize sz;
+ const gchar *dline;
+
+ if (ucl_object_type(elt) != UCL_STRING) {
+ msg_err_config("map has static backend but `data` is "
+ "not string like: %s",
+ ucl_object_type_to_string(elt->type));
+ return FALSE;
+ }
+
+ /* Otherwise, we copy data to the backend */
+ dline = ucl_object_tolstring(elt, &sz);
+
+ if (sz == 0) {
+ msg_err_config("map has static backend but empty no data");
+ return FALSE;
+ }
+
+ g_string_append_len(target, dline, sz);
+ g_string_append_c(target, '\n');
+
+ return TRUE;
+}
+
+struct rspamd_map *
+rspamd_map_add(struct rspamd_config *cfg,
+ const gchar *map_line,
+ const gchar *description,
+ map_cb_t read_callback,
+ map_fin_cb_t fin_callback,
+ map_dtor_t dtor,
+ void **user_data,
+ struct rspamd_worker *worker,
+ int flags)
+{
+ struct rspamd_map *map;
+ struct rspamd_map_backend *bk;
+
+ bk = rspamd_map_parse_backend(cfg, map_line);
+ if (bk == NULL) {
+ return NULL;
+ }
+
+ if (bk->is_fallback) {
+ msg_err_config("cannot add map with fallback only backend: %s", bk->uri);
+ REF_RELEASE(bk);
+
+ return NULL;
+ }
+
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(struct rspamd_map));
+ map->read_callback = read_callback;
+ map->fin_callback = fin_callback;
+ map->dtor = dtor;
+ map->user_data = user_data;
+ map->cfg = cfg;
+ map->id = rspamd_random_uint64_fast();
+ map->locked =
+ rspamd_mempool_alloc0_shared(cfg->cfg_pool, sizeof(gint));
+ map->backends = g_ptr_array_sized_new(1);
+ map->wrk = worker;
+ rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard,
+ map->backends);
+ g_ptr_array_add(map->backends, bk);
+ map->name = rspamd_mempool_strdup(cfg->cfg_pool, map_line);
+ map->no_file_read = (flags & RSPAMD_MAP_FILE_NO_READ);
+
+ if (bk->protocol == MAP_PROTO_FILE) {
+ map->poll_timeout = (cfg->map_timeout * cfg->map_file_watch_multiplier);
+ }
+ else {
+ map->poll_timeout = cfg->map_timeout;
+ }
+
+ if (description != NULL) {
+ map->description = rspamd_mempool_strdup(cfg->cfg_pool, description);
+ }
+
+ rspamd_map_calculate_hash(map);
+ msg_info_map("added map %s", bk->uri);
+ bk->map = map;
+
+ cfg->maps = g_list_prepend(cfg->maps, map);
+
+ return map;
+}
+
+struct rspamd_map *
+rspamd_map_add_fake(struct rspamd_config *cfg,
+ const gchar *description,
+ const gchar *name)
+{
+ struct rspamd_map *map;
+
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(struct rspamd_map));
+ map->cfg = cfg;
+ map->id = rspamd_random_uint64_fast();
+ map->name = rspamd_mempool_strdup(cfg->cfg_pool, name);
+ map->user_data = (void **) &map; /* to prevent null pointer dereferencing */
+
+ if (description != NULL) {
+ map->description = rspamd_mempool_strdup(cfg->cfg_pool, description);
+ }
+
+ return map;
+}
+
+static inline void
+rspamd_map_add_backend(struct rspamd_map *map, struct rspamd_map_backend *bk)
+{
+ if (bk->is_fallback) {
+ if (map->fallback_backend) {
+ msg_warn_map("redefining fallback backend from %s to %s",
+ map->fallback_backend->uri, bk->uri);
+ }
+
+ map->fallback_backend = bk;
+ }
+ else {
+ g_ptr_array_add(map->backends, bk);
+ }
+
+ bk->map = map;
+}
+
+struct rspamd_map *
+rspamd_map_add_from_ucl(struct rspamd_config *cfg,
+ const ucl_object_t *obj,
+ const gchar *description,
+ map_cb_t read_callback,
+ map_fin_cb_t fin_callback,
+ map_dtor_t dtor,
+ void **user_data,
+ struct rspamd_worker *worker,
+ gint flags)
+{
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur, *elt;
+ struct rspamd_map *map;
+ struct rspamd_map_backend *bk;
+ guint i;
+
+ g_assert(obj != NULL);
+
+ if (ucl_object_type(obj) == UCL_STRING) {
+ /* Just a plain string */
+ return rspamd_map_add(cfg, ucl_object_tostring(obj), description,
+ read_callback, fin_callback, dtor, user_data, worker, flags);
+ }
+
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(struct rspamd_map));
+ map->read_callback = read_callback;
+ map->fin_callback = fin_callback;
+ map->dtor = dtor;
+ map->user_data = user_data;
+ map->cfg = cfg;
+ map->id = rspamd_random_uint64_fast();
+ map->locked =
+ rspamd_mempool_alloc0_shared(cfg->cfg_pool, sizeof(gint));
+ map->backends = g_ptr_array_new();
+ map->wrk = worker;
+ map->no_file_read = (flags & RSPAMD_MAP_FILE_NO_READ);
+ rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard,
+ map->backends);
+ map->poll_timeout = cfg->map_timeout;
+
+ if (description) {
+ map->description = rspamd_mempool_strdup(cfg->cfg_pool, description);
+ }
+
+ if (ucl_object_type(obj) == UCL_ARRAY) {
+ /* Add array of maps as multiple backends */
+ while ((cur = ucl_object_iterate(obj, &it, true)) != NULL) {
+ if (ucl_object_type(cur) == UCL_STRING) {
+ bk = rspamd_map_parse_backend(cfg, ucl_object_tostring(cur));
+
+ if (bk != NULL) {
+ rspamd_map_add_backend(map, bk);
+
+ if (!map->name) {
+ map->name = rspamd_mempool_strdup(cfg->cfg_pool,
+ ucl_object_tostring(cur));
+ }
+ }
+ }
+ else {
+ msg_err_config("bad map element type: %s",
+ ucl_object_type_to_string(ucl_object_type(cur)));
+ }
+ }
+
+ if (map->backends->len == 0) {
+ msg_err_config("map has no urls to be loaded: empty list");
+ goto err;
+ }
+ }
+ else if (ucl_object_type(obj) == UCL_OBJECT) {
+ elt = ucl_object_lookup(obj, "name");
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ map->name = rspamd_mempool_strdup(cfg->cfg_pool,
+ ucl_object_tostring(elt));
+ }
+
+ elt = ucl_object_lookup(obj, "description");
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ map->description = rspamd_mempool_strdup(cfg->cfg_pool,
+ ucl_object_tostring(elt));
+ }
+
+ elt = ucl_object_lookup_any(obj, "timeout", "poll", "poll_time",
+ "watch_interval", NULL);
+ if (elt) {
+ map->poll_timeout = ucl_object_todouble(elt);
+ }
+
+ elt = ucl_object_lookup_any(obj, "upstreams", "url", "urls", NULL);
+ if (elt == NULL) {
+ msg_err_config("map has no urls to be loaded: no elt");
+ goto err;
+ }
+
+ if (ucl_object_type(elt) == UCL_ARRAY) {
+ /* Add array of maps as multiple backends */
+ it = ucl_object_iterate_new(elt);
+
+ while ((cur = ucl_object_iterate_safe(it, true)) != NULL) {
+ if (ucl_object_type(cur) == UCL_STRING) {
+ bk = rspamd_map_parse_backend(cfg, ucl_object_tostring(cur));
+
+ if (bk != NULL) {
+ rspamd_map_add_backend(map, bk);
+
+ if (!map->name) {
+ map->name = rspamd_mempool_strdup(cfg->cfg_pool,
+ ucl_object_tostring(cur));
+ }
+ }
+ }
+ else {
+ msg_err_config("bad map element type: %s",
+ ucl_object_type_to_string(ucl_object_type(cur)));
+ ucl_object_iterate_free(it);
+ goto err;
+ }
+ }
+
+ ucl_object_iterate_free(it);
+
+ if (map->backends->len == 0) {
+ msg_err_config("map has no urls to be loaded: empty object list");
+ goto err;
+ }
+ }
+ else if (ucl_object_type(elt) == UCL_STRING) {
+ bk = rspamd_map_parse_backend(cfg, ucl_object_tostring(elt));
+
+ if (bk != NULL) {
+ rspamd_map_add_backend(map, bk);
+
+ if (!map->name) {
+ map->name = rspamd_mempool_strdup(cfg->cfg_pool,
+ ucl_object_tostring(elt));
+ }
+ }
+ }
+
+ if (!map->backends || map->backends->len == 0) {
+ msg_err_config("map has no urls to be loaded: no valid backends");
+ goto err;
+ }
+ }
+ else {
+ msg_err_config("map has invalid type for value: %s",
+ ucl_object_type_to_string(ucl_object_type(obj)));
+ goto err;
+ }
+
+ gboolean all_local = TRUE;
+
+ PTR_ARRAY_FOREACH(map->backends, i, bk)
+ {
+ if (bk->protocol == MAP_PROTO_STATIC) {
+ GString *map_data;
+ /* We need data field in ucl */
+ elt = ucl_object_lookup(obj, "data");
+
+ if (elt == NULL) {
+ msg_err_config("map has static backend but no `data` field");
+ goto err;
+ }
+
+
+ if (ucl_object_type(elt) == UCL_STRING) {
+ map_data = g_string_sized_new(32);
+
+ if (rspamd_map_add_static_string(cfg, elt, map_data)) {
+ bk->data.sd->data = map_data->str;
+ bk->data.sd->len = map_data->len;
+ g_string_free(map_data, FALSE);
+ }
+ else {
+ g_string_free(map_data, TRUE);
+ msg_err_config("map has static backend with invalid `data` field");
+ goto err;
+ }
+ }
+ else if (ucl_object_type(elt) == UCL_ARRAY) {
+ map_data = g_string_sized_new(32);
+ it = ucl_object_iterate_new(elt);
+
+ while ((cur = ucl_object_iterate_safe(it, true))) {
+ if (!rspamd_map_add_static_string(cfg, cur, map_data)) {
+ g_string_free(map_data, TRUE);
+ msg_err_config("map has static backend with invalid "
+ "`data` field");
+ ucl_object_iterate_free(it);
+ goto err;
+ }
+ }
+
+ ucl_object_iterate_free(it);
+ bk->data.sd->data = map_data->str;
+ bk->data.sd->len = map_data->len;
+ g_string_free(map_data, FALSE);
+ }
+ }
+ else if (bk->protocol != MAP_PROTO_FILE) {
+ all_local = FALSE;
+ }
+ }
+
+ if (all_local) {
+ map->poll_timeout = (map->poll_timeout *
+ cfg->map_file_watch_multiplier);
+ }
+
+ rspamd_map_calculate_hash(map);
+ msg_debug_map("added map from ucl");
+
+ cfg->maps = g_list_prepend(cfg->maps, map);
+
+ return map;
+
+err:
+
+ if (map) {
+ PTR_ARRAY_FOREACH(map->backends, i, bk)
+ {
+ MAP_RELEASE(bk, "rspamd_map_backend");
+ }
+ }
+
+ return NULL;
+}
+
+rspamd_map_traverse_function
+rspamd_map_get_traverse_function(struct rspamd_map *map)
+{
+ if (map) {
+ return map->traverse_function;
+ }
+
+ return NULL;
+}
+
+void rspamd_map_traverse(struct rspamd_map *map, rspamd_map_traverse_cb cb,
+ gpointer cbdata, gboolean reset_hits)
+{
+ if (*map->user_data && map->traverse_function) {
+ map->traverse_function(*map->user_data, cb, cbdata, reset_hits);
+ }
+}
+
+void rspamd_map_set_on_load_function(struct rspamd_map *map, rspamd_map_on_load_function cb,
+ gpointer cbdata, GDestroyNotify dtor)
+{
+ if (map) {
+ map->on_load_function = cb;
+ map->on_load_ud = cbdata;
+ map->on_load_ud_dtor = dtor;
+ }
+}
diff --git a/src/libserver/maps/map.h b/src/libserver/maps/map.h
new file mode 100644
index 0000000..04df16e
--- /dev/null
+++ b/src/libserver/maps/map.h
@@ -0,0 +1,168 @@
+#ifndef RSPAMD_MAP_H
+#define RSPAMD_MAP_H
+
+#include "config.h"
+#include "contrib/libev/ev.h"
+
+#include "ucl.h"
+#include "mem_pool.h"
+#include "radix.h"
+#include "dns.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Maps API is designed to load lists data from different dynamic sources.
+ * It monitor files and HTTP locations for modifications and reload them if they are
+ * modified.
+ */
+struct map_cb_data;
+struct rspamd_worker;
+
+/**
+ * Common map object
+ */
+struct rspamd_config;
+struct rspamd_map;
+
+/**
+ * Callback types
+ */
+typedef gchar *(*map_cb_t)(gchar *chunk, gint len,
+ struct map_cb_data *data, gboolean final);
+
+typedef void (*map_fin_cb_t)(struct map_cb_data *data, void **target);
+
+typedef void (*map_dtor_t)(struct map_cb_data *data);
+
+typedef gboolean (*rspamd_map_traverse_cb)(gconstpointer key,
+ gconstpointer value, gsize hits, gpointer ud);
+
+typedef void (*rspamd_map_traverse_function)(void *data,
+ rspamd_map_traverse_cb cb,
+ gpointer cbdata, gboolean reset_hits);
+typedef void (*rspamd_map_on_load_function)(struct rspamd_map *map, gpointer ud);
+
+/**
+ * Callback data for async load
+ */
+struct map_cb_data {
+ struct rspamd_map *map;
+ gint state;
+ bool errored;
+ void *prev_data;
+ void *cur_data;
+};
+
+/**
+ * Returns TRUE if line looks like a map definition
+ * @param map_line
+ * @return
+ */
+gboolean rspamd_map_is_map(const gchar *map_line);
+
+enum rspamd_map_flags {
+ RSPAMD_MAP_DEFAULT = 0,
+ RSPAMD_MAP_FILE_ONLY = 1u << 0u,
+ RSPAMD_MAP_FILE_NO_READ = 1u << 1u,
+};
+
+/**
+ * Add map from line
+ */
+struct rspamd_map *rspamd_map_add(struct rspamd_config *cfg,
+ const gchar *map_line,
+ const gchar *description,
+ map_cb_t read_callback,
+ map_fin_cb_t fin_callback,
+ map_dtor_t dtor,
+ void **user_data,
+ struct rspamd_worker *worker,
+ int flags);
+
+/**
+ * Add map from ucl
+ */
+struct rspamd_map *rspamd_map_add_from_ucl(struct rspamd_config *cfg,
+ const ucl_object_t *obj,
+ const gchar *description,
+ map_cb_t read_callback,
+ map_fin_cb_t fin_callback,
+ map_dtor_t dtor,
+ void **user_data,
+ struct rspamd_worker *worker,
+ int flags);
+
+/**
+ * Adds a fake map structure (for logging purposes mainly)
+ * @param cfg
+ * @param description
+ * @return
+ */
+struct rspamd_map *rspamd_map_add_fake(struct rspamd_config *cfg,
+ const gchar *description,
+ const gchar *name);
+
+
+enum rspamd_map_watch_type {
+ RSPAMD_MAP_WATCH_MIN = 9,
+ RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER,
+ RSPAMD_MAP_WATCH_SCANNER,
+ RSPAMD_MAP_WATCH_WORKER,
+ RSPAMD_MAP_WATCH_MAX
+};
+
+/**
+ * Start watching of maps by adding events to libevent event loop
+ */
+void rspamd_map_watch(struct rspamd_config *cfg,
+ struct ev_loop *event_loop,
+ struct rspamd_dns_resolver *resolver,
+ struct rspamd_worker *worker,
+ enum rspamd_map_watch_type how);
+
+/**
+ * Preloads maps where all backends are file
+ * @param cfg
+ */
+void rspamd_map_preload(struct rspamd_config *cfg);
+
+/**
+ * Remove all maps watched (remove events)
+ */
+void rspamd_map_remove_all(struct rspamd_config *cfg);
+
+/**
+ * Get traverse function for specific map
+ * @param map
+ * @return
+ */
+rspamd_map_traverse_function rspamd_map_get_traverse_function(struct rspamd_map *map);
+
+/**
+ * Perform map traverse
+ * @param map
+ * @param cb
+ * @param cbdata
+ * @param reset_hits
+ * @return
+ */
+void rspamd_map_traverse(struct rspamd_map *map, rspamd_map_traverse_cb cb,
+ gpointer cbdata, gboolean reset_hits);
+
+/**
+ * Set map on load callback
+ * @param map
+ * @param cb
+ * @param cbdata
+ */
+void rspamd_map_set_on_load_function(struct rspamd_map *map, rspamd_map_on_load_function cb,
+ gpointer cbdata, GDestroyNotify dtor);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/maps/map_helpers.c b/src/libserver/maps/map_helpers.c
new file mode 100644
index 0000000..65478c5
--- /dev/null
+++ b/src/libserver/maps/map_helpers.c
@@ -0,0 +1,1845 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "map_helpers.h"
+#include "map_private.h"
+#include "khash.h"
+#include "radix.h"
+#include "rspamd.h"
+#include "cryptobox.h"
+#include "mempool_vars_internal.h"
+#include "contrib/fastutf8/fastutf8.h"
+#include "contrib/cdb/cdb.h"
+
+#ifdef WITH_HYPERSCAN
+#include "hs.h"
+#include "hyperscan_tools.h"
+#endif
+#ifndef WITH_PCRE2
+#include <pcre.h>
+#else
+#include <pcre2.h>
+#endif
+
+
+static const guint64 map_hash_seed = 0xdeadbabeULL;
+static const gchar *const hash_fill = "1";
+
+struct rspamd_map_helper_value {
+ gsize hits;
+ gconstpointer key;
+ gchar value[]; /* Null terminated */
+};
+
+#define rspamd_map_ftok_hash(t) (rspamd_icase_hash((t).begin, (t).len, rspamd_hash_seed()))
+#define rspamd_map_ftok_equal(a, b) ((a).len == (b).len && rspamd_lc_cmp((a).begin, (b).begin, (a).len) == 0)
+
+KHASH_INIT(rspamd_map_hash, rspamd_ftok_t,
+ struct rspamd_map_helper_value *, true,
+ rspamd_map_ftok_hash, rspamd_map_ftok_equal);
+
+struct rspamd_radix_map_helper {
+ rspamd_mempool_t *pool;
+ khash_t(rspamd_map_hash) * htb;
+ radix_compressed_t *trie;
+ struct rspamd_map *map;
+ rspamd_cryptobox_fast_hash_state_t hst;
+};
+
+struct rspamd_hash_map_helper {
+ rspamd_mempool_t *pool;
+ khash_t(rspamd_map_hash) * htb;
+ struct rspamd_map *map;
+ rspamd_cryptobox_fast_hash_state_t hst;
+};
+
+struct rspamd_cdb_map_helper {
+ GQueue cdbs;
+ struct rspamd_map *map;
+ rspamd_cryptobox_fast_hash_state_t hst;
+ gsize total_size;
+};
+
+struct rspamd_regexp_map_helper {
+ rspamd_cryptobox_hash_state_t hst;
+ guchar re_digest[rspamd_cryptobox_HASHBYTES];
+ rspamd_mempool_t *pool;
+ struct rspamd_map *map;
+ GPtrArray *regexps;
+ GPtrArray *values;
+ khash_t(rspamd_map_hash) * htb;
+ enum rspamd_regexp_map_flags map_flags;
+#ifdef WITH_HYPERSCAN
+ rspamd_hyperscan_t *hs_db;
+ hs_scratch_t *hs_scratch;
+ gchar **patterns;
+ gint *flags;
+ gint *ids;
+#endif
+};
+
+/**
+ * FSM for parsing lists
+ */
+
+#define MAP_STORE_KEY \
+ do { \
+ while (g_ascii_isspace(*c) && p > c) { c++; } \
+ key = g_malloc(p - c + 1); \
+ rspamd_strlcpy(key, c, p - c + 1); \
+ stripped_key = g_strstrip(key); \
+ } while (0)
+
+#define MAP_STORE_VALUE \
+ do { \
+ while (g_ascii_isspace(*c) && p > c) { c++; } \
+ value = g_malloc(p - c + 1); \
+ rspamd_strlcpy(value, c, p - c + 1); \
+ stripped_value = g_strstrip(value); \
+ } while (0)
+
+gchar *
+rspamd_parse_kv_list(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ rspamd_map_insert_func func,
+ const gchar *default_value,
+ gboolean final)
+{
+ enum {
+ map_skip_spaces_before_key = 0,
+ map_read_key,
+ map_read_key_quoted,
+ map_read_key_slashed,
+ map_skip_spaces_after_key,
+ map_backslash_quoted,
+ map_backslash_slashed,
+ map_read_key_after_slash,
+ map_read_value,
+ map_read_comment_start,
+ map_skip_comment,
+ map_read_eol,
+ };
+
+ gchar *c, *p, *key = NULL, *value = NULL, *stripped_key, *stripped_value, *end;
+ struct rspamd_map *map = data->map;
+ guint line_number = 0;
+
+ p = chunk;
+ c = p;
+ end = p + len;
+
+ while (p < end) {
+ switch (data->state) {
+ case map_skip_spaces_before_key:
+ if (g_ascii_isspace(*p)) {
+ p++;
+ }
+ else {
+ if (*p == '"') {
+ p++;
+ c = p;
+ data->state = map_read_key_quoted;
+ }
+ else if (*p == '/') {
+ /* Note that c is on '/' here as '/' is a part of key */
+ c = p;
+ p++;
+ data->state = map_read_key_slashed;
+ }
+ else {
+ c = p;
+ data->state = map_read_key;
+ }
+ }
+ break;
+ case map_read_key:
+ /* read key */
+ /* Check here comments, eol and end of buffer */
+ if (*p == '#' && (p == c || *(p - 1) != '\\')) {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_KEY;
+ func(data->cur_data, stripped_key, default_value);
+ msg_debug_map("insert key only pair: %s -> %s; line: %d",
+ stripped_key, default_value, line_number);
+ g_free(key);
+ }
+
+ key = NULL;
+ data->state = map_read_comment_start;
+ }
+ else if (*p == '\r' || *p == '\n') {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_KEY;
+ func(data->cur_data, stripped_key, default_value);
+ msg_debug_map("insert key only pair: %s -> %s; line: %d",
+ stripped_key, default_value, line_number);
+ g_free(key);
+ }
+
+ data->state = map_read_eol;
+ key = NULL;
+ }
+ else if (g_ascii_isspace(*p)) {
+ if (p - c > 0) {
+ MAP_STORE_KEY;
+ data->state = map_skip_spaces_after_key;
+ }
+ else {
+ msg_err_map("empty or invalid key found on line %d", line_number);
+ data->state = map_skip_comment;
+ }
+ }
+ else {
+ p++;
+ }
+ break;
+ case map_read_key_quoted:
+ if (*p == '\\') {
+ data->state = map_backslash_quoted;
+ p++;
+ }
+ else if (*p == '"') {
+ /* Allow empty keys in this case */
+ if (p - c >= 0) {
+ MAP_STORE_KEY;
+ data->state = map_skip_spaces_after_key;
+ }
+ else {
+ g_assert_not_reached();
+ }
+ p++;
+ }
+ else {
+ p++;
+ }
+ break;
+ case map_read_key_slashed:
+ if (*p == '\\') {
+ data->state = map_backslash_slashed;
+ p++;
+ }
+ else if (*p == '/') {
+ /* Allow empty keys in this case */
+ if (p - c >= 0) {
+ data->state = map_read_key_after_slash;
+ }
+ else {
+ g_assert_not_reached();
+ }
+ }
+ else {
+ p++;
+ }
+ break;
+ case map_read_key_after_slash:
+ /*
+ * This state is equal to reading of key but '/' is not
+ * treated specially
+ */
+ if (*p == '#') {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_KEY;
+ func(data->cur_data, stripped_key, default_value);
+ msg_debug_map("insert key only pair: %s -> %s; line: %d",
+ stripped_key, default_value, line_number);
+ g_free(key);
+ key = NULL;
+ }
+
+ data->state = map_read_comment_start;
+ }
+ else if (*p == '\r' || *p == '\n') {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_KEY;
+ func(data->cur_data, stripped_key, default_value);
+
+ msg_debug_map("insert key only pair: %s -> %s; line: %d",
+ stripped_key, default_value, line_number);
+ g_free(key);
+ key = NULL;
+ }
+
+ data->state = map_read_eol;
+ key = NULL;
+ }
+ else if (g_ascii_isspace(*p)) {
+ if (p - c > 0) {
+ MAP_STORE_KEY;
+ data->state = map_skip_spaces_after_key;
+ }
+ else {
+ msg_err_map("empty or invalid key found on line %d", line_number);
+ data->state = map_skip_comment;
+ }
+ }
+ else {
+ p++;
+ }
+ break;
+ case map_backslash_quoted:
+ p++;
+ data->state = map_read_key_quoted;
+ break;
+ case map_backslash_slashed:
+ p++;
+ data->state = map_read_key_slashed;
+ break;
+ case map_skip_spaces_after_key:
+ if (*p == ' ' || *p == '\t') {
+ p++;
+ }
+ else {
+ c = p;
+ data->state = map_read_value;
+ }
+ break;
+ case map_read_value:
+ if (key == NULL) {
+ /* Ignore line */
+ msg_err_map("empty or invalid key found on line %d", line_number);
+ data->state = map_skip_comment;
+ }
+ else {
+ if (*p == '#') {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_VALUE;
+ func(data->cur_data, stripped_key, stripped_value);
+ msg_debug_map("insert key value pair: %s -> %s; line: %d",
+ stripped_key, stripped_value, line_number);
+ g_free(key);
+ g_free(value);
+ key = NULL;
+ value = NULL;
+ }
+ else {
+ func(data->cur_data, stripped_key, default_value);
+ msg_debug_map("insert key only pair: %s -> %s; line: %d",
+ stripped_key, default_value, line_number);
+ g_free(key);
+ key = NULL;
+ }
+
+ data->state = map_read_comment_start;
+ }
+ else if (*p == '\r' || *p == '\n') {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_VALUE;
+ func(data->cur_data, stripped_key, stripped_value);
+ msg_debug_map("insert key value pair: %s -> %s",
+ stripped_key, stripped_value);
+ g_free(key);
+ g_free(value);
+ key = NULL;
+ value = NULL;
+ }
+ else {
+ func(data->cur_data, stripped_key, default_value);
+ msg_debug_map("insert key only pair: %s -> %s",
+ stripped_key, default_value);
+ g_free(key);
+ key = NULL;
+ }
+
+ data->state = map_read_eol;
+ key = NULL;
+ }
+ else {
+ p++;
+ }
+ }
+ break;
+ case map_read_comment_start:
+ if (*p == '#') {
+ data->state = map_skip_comment;
+ p++;
+ key = NULL;
+ value = NULL;
+ }
+ else {
+ g_assert_not_reached();
+ }
+ break;
+ case map_skip_comment:
+ if (*p == '\r' || *p == '\n') {
+ data->state = map_read_eol;
+ }
+ else {
+ p++;
+ }
+ break;
+ case map_read_eol:
+ /* Skip \r\n and whitespaces */
+ if (*p == '\r' || *p == '\n') {
+ if (*p == '\n') {
+ /* We don't care about \r only line separators, they are too rare */
+ line_number++;
+ }
+ p++;
+ }
+ else {
+ data->state = map_skip_spaces_before_key;
+ }
+ break;
+ default:
+ g_assert_not_reached();
+ break;
+ }
+ }
+
+ if (final) {
+ /* Examine the state */
+ switch (data->state) {
+ case map_read_key:
+ case map_read_key_slashed:
+ case map_read_key_quoted:
+ case map_read_key_after_slash:
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_KEY;
+ func(data->cur_data, stripped_key, default_value);
+ msg_debug_map("insert key only pair: %s -> %s",
+ stripped_key, default_value);
+ g_free(key);
+ key = NULL;
+ }
+ break;
+ case map_read_value:
+ if (key == NULL) {
+ /* Ignore line */
+ msg_err_map("empty or invalid key found on line %d", line_number);
+ data->state = map_skip_comment;
+ }
+ else {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_VALUE;
+ func(data->cur_data, stripped_key, stripped_value);
+ msg_debug_map("insert key value pair: %s -> %s",
+ stripped_key, stripped_value);
+ g_free(key);
+ g_free(value);
+ key = NULL;
+ value = NULL;
+ }
+ else {
+ func(data->cur_data, stripped_key, default_value);
+ msg_debug_map("insert key only pair: %s -> %s",
+ stripped_key, default_value);
+ g_free(key);
+ key = NULL;
+ }
+ }
+ break;
+ }
+
+ data->state = map_skip_spaces_before_key;
+ }
+
+ return c;
+}
+
+/**
+ * Radix tree helper function
+ */
+void rspamd_map_helper_insert_radix(gpointer st, gconstpointer key, gconstpointer value)
+{
+ struct rspamd_radix_map_helper *r = (struct rspamd_radix_map_helper *) st;
+ struct rspamd_map_helper_value *val;
+ gsize vlen;
+ khiter_t k;
+ gconstpointer nk;
+ rspamd_ftok_t tok;
+ gint res;
+ struct rspamd_map *map;
+
+ map = r->map;
+ tok.begin = key;
+ tok.len = strlen(key);
+
+ k = kh_get(rspamd_map_hash, r->htb, tok);
+
+ if (k == kh_end(r->htb)) {
+ nk = rspamd_mempool_strdup(r->pool, key);
+ tok.begin = nk;
+ k = kh_put(rspamd_map_hash, r->htb, tok, &res);
+ }
+ else {
+ val = kh_value(r->htb, k);
+
+ if (strcmp(value, val->value) == 0) {
+ /* Same element, skip */
+ return;
+ }
+ else {
+ msg_warn_map("duplicate radix entry found for map %s: %s (old value: '%s', new: '%s')",
+ map->name, key, val->value, value);
+ }
+
+ nk = kh_key(r->htb, k).begin;
+ val->key = nk;
+ kh_value(r->htb, k) = val;
+
+ return; /* do not touch radix in case of exact duplicate */
+ }
+
+ vlen = strlen(value);
+ val = rspamd_mempool_alloc0(r->pool, sizeof(*val) +
+ vlen + 1);
+ memcpy(val->value, value, vlen);
+
+ nk = kh_key(r->htb, k).begin;
+ val->key = nk;
+ kh_value(r->htb, k) = val;
+ rspamd_radix_add_iplist(key, ",", r->trie, val, FALSE,
+ r->map->name);
+ rspamd_cryptobox_fast_hash_update(&r->hst, nk, tok.len);
+}
+
+void rspamd_map_helper_insert_radix_resolve(gpointer st, gconstpointer key, gconstpointer value)
+{
+ struct rspamd_radix_map_helper *r = (struct rspamd_radix_map_helper *) st;
+ struct rspamd_map_helper_value *val;
+ gsize vlen;
+ khiter_t k;
+ gconstpointer nk;
+ rspamd_ftok_t tok;
+ gint res;
+ struct rspamd_map *map;
+
+ map = r->map;
+
+ if (!key) {
+ msg_warn_map("cannot insert NULL value in the map: %s",
+ map->name);
+ return;
+ }
+
+ tok.begin = key;
+ tok.len = strlen(key);
+
+ k = kh_get(rspamd_map_hash, r->htb, tok);
+
+ if (k == kh_end(r->htb)) {
+ nk = rspamd_mempool_strdup(r->pool, key);
+ tok.begin = nk;
+ k = kh_put(rspamd_map_hash, r->htb, tok, &res);
+ }
+ else {
+ val = kh_value(r->htb, k);
+
+ if (strcmp(value, val->value) == 0) {
+ /* Same element, skip */
+ return;
+ }
+ else {
+ msg_warn_map("duplicate radix entry found for map %s: %s (old value: '%s', new: '%s')",
+ map->name, key, val->value, value);
+ }
+
+ nk = kh_key(r->htb, k).begin;
+ val->key = nk;
+ kh_value(r->htb, k) = val;
+
+ return; /* do not touch radix in case of exact duplicate */
+ }
+
+ vlen = strlen(value);
+ val = rspamd_mempool_alloc0(r->pool, sizeof(*val) +
+ vlen + 1);
+ memcpy(val->value, value, vlen);
+ nk = kh_key(r->htb, k).begin;
+ val->key = nk;
+ kh_value(r->htb, k) = val;
+ rspamd_radix_add_iplist(key, ",", r->trie, val, TRUE,
+ r->map->name);
+ rspamd_cryptobox_fast_hash_update(&r->hst, nk, tok.len);
+}
+
+void rspamd_map_helper_insert_hash(gpointer st, gconstpointer key, gconstpointer value)
+{
+ struct rspamd_hash_map_helper *ht = st;
+ struct rspamd_map_helper_value *val;
+ khiter_t k;
+ gconstpointer nk;
+ gsize vlen;
+ gint r;
+ rspamd_ftok_t tok;
+ struct rspamd_map *map;
+
+ tok.begin = key;
+ tok.len = strlen(key);
+ map = ht->map;
+
+ k = kh_get(rspamd_map_hash, ht->htb, tok);
+
+ if (k == kh_end(ht->htb)) {
+ nk = rspamd_mempool_strdup(ht->pool, key);
+ tok.begin = nk;
+ k = kh_put(rspamd_map_hash, ht->htb, tok, &r);
+ }
+ else {
+ val = kh_value(ht->htb, k);
+
+ if (strcmp(value, val->value) == 0) {
+ /* Same element, skip */
+ return;
+ }
+ else {
+ msg_warn_map("duplicate hash entry found for map %s: %s (old value: '%s', new: '%s')",
+ map->name, key, val->value, value);
+ }
+ }
+
+ /* Null termination due to alloc0 */
+ vlen = strlen(value);
+ val = rspamd_mempool_alloc0(ht->pool, sizeof(*val) + vlen + 1);
+ memcpy(val->value, value, vlen);
+
+ tok = kh_key(ht->htb, k);
+ nk = tok.begin;
+ val->key = nk;
+ kh_value(ht->htb, k) = val;
+
+ rspamd_cryptobox_fast_hash_update(&ht->hst, nk, tok.len);
+}
+
+void rspamd_map_helper_insert_re(gpointer st, gconstpointer key, gconstpointer value)
+{
+ struct rspamd_regexp_map_helper *re_map = st;
+ struct rspamd_map *map;
+ rspamd_regexp_t *re;
+ gchar *escaped;
+ GError *err = NULL;
+ gint pcre_flags;
+ gsize escaped_len;
+ struct rspamd_map_helper_value *val;
+ khiter_t k;
+ rspamd_ftok_t tok;
+ gconstpointer nk;
+ gsize vlen;
+ gint r;
+
+ map = re_map->map;
+
+ tok.begin = key;
+ tok.len = strlen(key);
+
+ k = kh_get(rspamd_map_hash, re_map->htb, tok);
+
+ if (k == kh_end(re_map->htb)) {
+ nk = rspamd_mempool_strdup(re_map->pool, key);
+ tok.begin = nk;
+ k = kh_put(rspamd_map_hash, re_map->htb, tok, &r);
+ }
+ else {
+ val = kh_value(re_map->htb, k);
+
+ /* Always warn about regexp duplicate as it's likely a bad mistake */
+ msg_warn_map("duplicate re entry found for map %s: %s (old value: '%s', new: '%s')",
+ map->name, key, val->value, value);
+
+ if (strcmp(val->value, value) == 0) {
+ /* Same value, skip */
+ return;
+ }
+
+ /* Replace value but do not touch regexp */
+ nk = kh_key(re_map->htb, k).begin;
+ val->key = nk;
+ kh_value(re_map->htb, k) = val;
+
+ return;
+ }
+
+ /* Check regexp stuff */
+ if (re_map->map_flags & RSPAMD_REGEXP_MAP_FLAG_GLOB) {
+ escaped = rspamd_str_regexp_escape(key, strlen(key), &escaped_len,
+ RSPAMD_REGEXP_ESCAPE_GLOB | RSPAMD_REGEXP_ESCAPE_UTF);
+ re = rspamd_regexp_new(escaped, NULL, &err);
+ g_free(escaped);
+ }
+ else {
+ re = rspamd_regexp_new(key, NULL, &err);
+ }
+
+ if (re == NULL) {
+ msg_err_map("cannot parse regexp %s: %e", key, err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return;
+ }
+
+ vlen = strlen(value);
+ val = rspamd_mempool_alloc0(re_map->pool, sizeof(*val) +
+ vlen + 1);
+ memcpy(val->value, value, vlen); /* Null terminated due to alloc0 previously */
+ nk = kh_key(re_map->htb, k).begin;
+ val->key = nk;
+ kh_value(re_map->htb, k) = val;
+ rspamd_cryptobox_hash_update(&re_map->hst, nk, tok.len);
+
+ pcre_flags = rspamd_regexp_get_pcre_flags(re);
+
+#ifndef WITH_PCRE2
+ if (pcre_flags & PCRE_FLAG(UTF8)) {
+ re_map->map_flags |= RSPAMD_REGEXP_MAP_FLAG_UTF;
+ }
+#else
+ if (pcre_flags & PCRE_FLAG(UTF)) {
+ re_map->map_flags |= RSPAMD_REGEXP_MAP_FLAG_UTF;
+ }
+#endif
+
+ g_ptr_array_add(re_map->regexps, re);
+ g_ptr_array_add(re_map->values, val);
+}
+
+static void
+rspamd_map_helper_traverse_regexp(void *data,
+ rspamd_map_traverse_cb cb,
+ gpointer cbdata,
+ gboolean reset_hits)
+{
+ rspamd_ftok_t tok;
+ struct rspamd_map_helper_value *val;
+ struct rspamd_regexp_map_helper *re_map = data;
+
+ kh_foreach(re_map->htb, tok, val, {
+ if (!cb(tok.begin, val->value, val->hits, cbdata)) {
+ break;
+ }
+
+ if (reset_hits) {
+ val->hits = 0;
+ }
+ });
+}
+
+struct rspamd_hash_map_helper *
+rspamd_map_helper_new_hash(struct rspamd_map *map)
+{
+ struct rspamd_hash_map_helper *htb;
+ rspamd_mempool_t *pool;
+
+ if (map) {
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ map->tag, 0);
+ }
+ else {
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ NULL, 0);
+ }
+
+ htb = rspamd_mempool_alloc0_type(pool, struct rspamd_hash_map_helper);
+ htb->htb = kh_init(rspamd_map_hash);
+ htb->pool = pool;
+ htb->map = map;
+ rspamd_cryptobox_fast_hash_init(&htb->hst, map_hash_seed);
+
+ return htb;
+}
+
+void rspamd_map_helper_destroy_hash(struct rspamd_hash_map_helper *r)
+{
+ if (r == NULL || r->pool == NULL) {
+ return;
+ }
+
+ rspamd_mempool_t *pool = r->pool;
+ kh_destroy(rspamd_map_hash, r->htb);
+ memset(r, 0, sizeof(*r));
+ rspamd_mempool_delete(pool);
+}
+
+static void
+rspamd_map_helper_traverse_hash(void *data,
+ rspamd_map_traverse_cb cb,
+ gpointer cbdata,
+ gboolean reset_hits)
+{
+ rspamd_ftok_t tok;
+ struct rspamd_map_helper_value *val;
+ struct rspamd_hash_map_helper *ht = data;
+
+ kh_foreach(ht->htb, tok, val, {
+ if (!cb(tok.begin, val->value, val->hits, cbdata)) {
+ break;
+ }
+
+ if (reset_hits) {
+ val->hits = 0;
+ }
+ });
+}
+
+struct rspamd_radix_map_helper *
+rspamd_map_helper_new_radix(struct rspamd_map *map)
+{
+ struct rspamd_radix_map_helper *r;
+ rspamd_mempool_t *pool;
+ const gchar *name = "unnamed";
+
+ if (map) {
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ map->tag, 0);
+ name = map->name;
+ }
+ else {
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ NULL, 0);
+ }
+
+ r = rspamd_mempool_alloc0_type(pool, struct rspamd_radix_map_helper);
+ r->trie = radix_create_compressed_with_pool(pool, name);
+ r->htb = kh_init(rspamd_map_hash);
+ r->pool = pool;
+ r->map = map;
+ rspamd_cryptobox_fast_hash_init(&r->hst, map_hash_seed);
+
+ return r;
+}
+
+void rspamd_map_helper_destroy_radix(struct rspamd_radix_map_helper *r)
+{
+ if (r == NULL || !r->pool) {
+ return;
+ }
+
+ kh_destroy(rspamd_map_hash, r->htb);
+ rspamd_mempool_t *pool = r->pool;
+ memset(r, 0, sizeof(*r));
+ rspamd_mempool_delete(pool);
+}
+
+static void
+rspamd_map_helper_traverse_radix(void *data,
+ rspamd_map_traverse_cb cb,
+ gpointer cbdata,
+ gboolean reset_hits)
+{
+ rspamd_ftok_t tok;
+ struct rspamd_map_helper_value *val;
+ struct rspamd_radix_map_helper *r = data;
+
+ kh_foreach(r->htb, tok, val, {
+ if (!cb(tok.begin, val->value, val->hits, cbdata)) {
+ break;
+ }
+
+ if (reset_hits) {
+ val->hits = 0;
+ }
+ });
+}
+
+struct rspamd_regexp_map_helper *
+rspamd_map_helper_new_regexp(struct rspamd_map *map,
+ enum rspamd_regexp_map_flags flags)
+{
+ struct rspamd_regexp_map_helper *re_map;
+ rspamd_mempool_t *pool;
+
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ map->tag, 0);
+
+ re_map = rspamd_mempool_alloc0_type(pool, struct rspamd_regexp_map_helper);
+ re_map->pool = pool;
+ re_map->values = g_ptr_array_new();
+ re_map->regexps = g_ptr_array_new();
+ re_map->map = map;
+ re_map->map_flags = flags;
+ re_map->htb = kh_init(rspamd_map_hash);
+ rspamd_cryptobox_hash_init(&re_map->hst, NULL, 0);
+
+ return re_map;
+}
+
+
+void rspamd_map_helper_destroy_regexp(struct rspamd_regexp_map_helper *re_map)
+{
+ rspamd_regexp_t *re;
+ guint i;
+
+ if (!re_map || !re_map->regexps) {
+ return;
+ }
+
+#ifdef WITH_HYPERSCAN
+ if (re_map->hs_scratch) {
+ hs_free_scratch(re_map->hs_scratch);
+ }
+ if (re_map->hs_db) {
+ rspamd_hyperscan_free(re_map->hs_db, false);
+ }
+ if (re_map->patterns) {
+ for (i = 0; i < re_map->regexps->len; i++) {
+ g_free(re_map->patterns[i]);
+ }
+
+ g_free(re_map->patterns);
+ }
+ if (re_map->flags) {
+ g_free(re_map->flags);
+ }
+ if (re_map->ids) {
+ g_free(re_map->ids);
+ }
+#endif
+
+ for (i = 0; i < re_map->regexps->len; i++) {
+ re = g_ptr_array_index(re_map->regexps, i);
+ rspamd_regexp_unref(re);
+ }
+
+ g_ptr_array_free(re_map->regexps, TRUE);
+ g_ptr_array_free(re_map->values, TRUE);
+ kh_destroy(rspamd_map_hash, re_map->htb);
+
+ rspamd_mempool_t *pool = re_map->pool;
+ memset(re_map, 0, sizeof(*re_map));
+ rspamd_mempool_delete(pool);
+}
+
+gchar *
+rspamd_kv_list_read(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ if (data->cur_data == NULL) {
+ data->cur_data = rspamd_map_helper_new_hash(data->map);
+ }
+
+ return rspamd_parse_kv_list(
+ chunk,
+ len,
+ data,
+ rspamd_map_helper_insert_hash,
+ "",
+ final);
+}
+
+void rspamd_kv_list_fin(struct map_cb_data *data, void **target)
+{
+ struct rspamd_map *map = data->map;
+ struct rspamd_hash_map_helper *htb;
+
+ if (data->errored) {
+ /* Clean up the current data and do not touch prev data */
+ if (data->cur_data) {
+ msg_info_map("cleanup unfinished new data as error occurred for %s",
+ map->name);
+ htb = (struct rspamd_hash_map_helper *) data->cur_data;
+ rspamd_map_helper_destroy_hash(htb);
+ data->cur_data = NULL;
+ }
+ }
+ else {
+ if (data->cur_data) {
+ htb = (struct rspamd_hash_map_helper *) data->cur_data;
+ msg_info_map("read hash of %d elements from %s", kh_size(htb->htb),
+ map->name);
+ data->map->traverse_function = rspamd_map_helper_traverse_hash;
+ data->map->nelts = kh_size(htb->htb);
+ data->map->digest = rspamd_cryptobox_fast_hash_final(&htb->hst);
+ }
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ htb = (struct rspamd_hash_map_helper *) data->prev_data;
+ rspamd_map_helper_destroy_hash(htb);
+ }
+ }
+}
+
+void rspamd_kv_list_dtor(struct map_cb_data *data)
+{
+ struct rspamd_hash_map_helper *htb;
+
+ if (data->cur_data) {
+ htb = (struct rspamd_hash_map_helper *) data->cur_data;
+ rspamd_map_helper_destroy_hash(htb);
+ }
+}
+
+gchar *
+rspamd_radix_read(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct rspamd_radix_map_helper *r;
+ struct rspamd_map *map = data->map;
+
+ if (data->cur_data == NULL) {
+ r = rspamd_map_helper_new_radix(map);
+ data->cur_data = r;
+ }
+
+ return rspamd_parse_kv_list(
+ chunk,
+ len,
+ data,
+ rspamd_map_helper_insert_radix,
+ hash_fill,
+ final);
+}
+
+void rspamd_radix_fin(struct map_cb_data *data, void **target)
+{
+ struct rspamd_map *map = data->map;
+ struct rspamd_radix_map_helper *r;
+
+ if (data->errored) {
+ /* Clean up the current data and do not touch prev data */
+ if (data->cur_data) {
+ msg_info_map("cleanup unfinished new data as error occurred for %s",
+ map->name);
+ r = (struct rspamd_radix_map_helper *) data->cur_data;
+ rspamd_map_helper_destroy_radix(r);
+ data->cur_data = NULL;
+ }
+ }
+ else {
+ if (data->cur_data) {
+ r = (struct rspamd_radix_map_helper *) data->cur_data;
+ msg_info_map("read radix trie of %z elements: %s",
+ radix_get_size(r->trie), radix_get_info(r->trie));
+ data->map->traverse_function = rspamd_map_helper_traverse_radix;
+ data->map->nelts = kh_size(r->htb);
+ data->map->digest = rspamd_cryptobox_fast_hash_final(&r->hst);
+ }
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ r = (struct rspamd_radix_map_helper *) data->prev_data;
+ rspamd_map_helper_destroy_radix(r);
+ }
+ }
+}
+
+void rspamd_radix_dtor(struct map_cb_data *data)
+{
+ struct rspamd_radix_map_helper *r;
+
+ if (data->cur_data) {
+ r = (struct rspamd_radix_map_helper *) data->cur_data;
+ rspamd_map_helper_destroy_radix(r);
+ }
+}
+
+#ifdef WITH_HYPERSCAN
+
+static gboolean
+rspamd_try_load_re_map_cache(struct rspamd_regexp_map_helper *re_map)
+{
+ gchar fp[PATH_MAX];
+ struct rspamd_map *map;
+
+ map = re_map->map;
+
+ if (!map->cfg->hs_cache_dir) {
+ return FALSE;
+ }
+
+ rspamd_snprintf(fp, sizeof(fp), "%s/%*xs.hsmc",
+ map->cfg->hs_cache_dir,
+ (gint) rspamd_cryptobox_HASHBYTES / 2, re_map->re_digest);
+
+ re_map->hs_db = rspamd_hyperscan_maybe_load(fp, 0);
+
+ return re_map->hs_db != NULL;
+}
+
+static gboolean
+rspamd_try_save_re_map_cache(struct rspamd_regexp_map_helper *re_map)
+{
+ gchar fp[PATH_MAX], np[PATH_MAX];
+ gsize len;
+ gint fd;
+ char *bytes = NULL;
+ struct rspamd_map *map;
+
+ map = re_map->map;
+
+ if (!map->cfg->hs_cache_dir) {
+ return FALSE;
+ }
+
+ rspamd_snprintf(fp, sizeof(fp), "%s/hsmc-XXXXXXXXXXXXX",
+ re_map->map->cfg->hs_cache_dir);
+
+ if ((fd = g_mkstemp_full(fp, O_WRONLY | O_CREAT | O_EXCL, 00644)) != -1) {
+ if (hs_serialize_database(rspamd_hyperscan_get_database(re_map->hs_db), &bytes, &len) == HS_SUCCESS) {
+ if (write(fd, bytes, len) == -1) {
+ msg_warn_map("cannot write hyperscan cache to %s: %s",
+ fp, strerror(errno));
+ unlink(fp);
+ free(bytes);
+ }
+ else {
+ free(bytes);
+ fsync(fd);
+
+ rspamd_snprintf(np, sizeof(np), "%s/%*xs.hsmc",
+ re_map->map->cfg->hs_cache_dir,
+ (gint) rspamd_cryptobox_HASHBYTES / 2, re_map->re_digest);
+
+ if (rename(fp, np) == -1) {
+ msg_warn_map("cannot rename hyperscan cache from %s to %s: %s",
+ fp, np, strerror(errno));
+ unlink(fp);
+ }
+ else {
+ msg_info_map("written cached hyperscan data for %s to %s (%Hz length)",
+ map->name, np, len);
+ rspamd_hyperscan_notice_known(np);
+ }
+ }
+ }
+ else {
+ msg_warn_map("cannot serialize hyperscan cache to %s: %s",
+ fp, strerror(errno));
+ unlink(fp);
+ }
+
+
+ close(fd);
+ }
+
+ return FALSE;
+}
+
+#endif
+
+static void
+rspamd_re_map_finalize(struct rspamd_regexp_map_helper *re_map)
+{
+#ifdef WITH_HYPERSCAN
+ guint i;
+ hs_platform_info_t plt;
+ hs_compile_error_t *err;
+ struct rspamd_map *map;
+ rspamd_regexp_t *re;
+ gint pcre_flags;
+
+ map = re_map->map;
+
+#if !defined(__aarch64__) && !defined(__powerpc64__)
+ if (!(map->cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) {
+ msg_info_map("disable hyperscan for map %s, ssse3 instructions are not supported by CPU",
+ map->name);
+ return;
+ }
+#endif
+
+ if (hs_populate_platform(&plt) != HS_SUCCESS) {
+ msg_err_map("cannot populate hyperscan platform");
+ return;
+ }
+
+ re_map->patterns = g_new(gchar *, re_map->regexps->len);
+ re_map->flags = g_new(gint, re_map->regexps->len);
+ re_map->ids = g_new(gint, re_map->regexps->len);
+
+ for (i = 0; i < re_map->regexps->len; i++) {
+ const gchar *pat;
+ gchar *escaped;
+ gint pat_flags;
+
+ re = g_ptr_array_index(re_map->regexps, i);
+ pcre_flags = rspamd_regexp_get_pcre_flags(re);
+ pat = rspamd_regexp_get_pattern(re);
+ pat_flags = rspamd_regexp_get_flags(re);
+
+ if (pat_flags & RSPAMD_REGEXP_FLAG_UTF) {
+ escaped = rspamd_str_regexp_escape(pat, strlen(pat), NULL,
+ RSPAMD_REGEXP_ESCAPE_RE | RSPAMD_REGEXP_ESCAPE_UTF);
+ re_map->flags[i] |= HS_FLAG_UTF8;
+ }
+ else {
+ escaped = rspamd_str_regexp_escape(pat, strlen(pat), NULL,
+ RSPAMD_REGEXP_ESCAPE_RE);
+ }
+
+ re_map->patterns[i] = escaped;
+ re_map->flags[i] = HS_FLAG_SINGLEMATCH;
+
+#ifndef WITH_PCRE2
+ if (pcre_flags & PCRE_FLAG(UTF8)) {
+ re_map->flags[i] |= HS_FLAG_UTF8;
+ }
+#else
+ if (pcre_flags & PCRE_FLAG(UTF)) {
+ re_map->flags[i] |= HS_FLAG_UTF8;
+ }
+#endif
+ if (pcre_flags & PCRE_FLAG(CASELESS)) {
+ re_map->flags[i] |= HS_FLAG_CASELESS;
+ }
+ if (pcre_flags & PCRE_FLAG(MULTILINE)) {
+ re_map->flags[i] |= HS_FLAG_MULTILINE;
+ }
+ if (pcre_flags & PCRE_FLAG(DOTALL)) {
+ re_map->flags[i] |= HS_FLAG_DOTALL;
+ }
+ if (rspamd_regexp_get_maxhits(re) == 1) {
+ re_map->flags[i] |= HS_FLAG_SINGLEMATCH;
+ }
+
+ re_map->ids[i] = i;
+ }
+
+ if (re_map->regexps->len > 0 && re_map->patterns) {
+
+ if (!rspamd_try_load_re_map_cache(re_map)) {
+ gdouble ts1 = rspamd_get_ticks(FALSE);
+ hs_database_t *hs_db = NULL;
+
+ if (hs_compile_multi((const gchar **) re_map->patterns,
+ re_map->flags,
+ re_map->ids,
+ re_map->regexps->len,
+ HS_MODE_BLOCK,
+ &plt,
+ &hs_db,
+ &err) != HS_SUCCESS) {
+
+ msg_err_map("cannot create tree of regexp when processing '%s': %s",
+ err->expression >= 0 ? re_map->patterns[err->expression] : "unknown regexp", err->message);
+ re_map->hs_db = NULL;
+ hs_free_compile_error(err);
+
+ return;
+ }
+
+ if (re_map->map->cfg->hs_cache_dir) {
+ char fpath[PATH_MAX];
+ rspamd_snprintf(fpath, sizeof(fpath), "%s/%*xs.hsmc",
+ re_map->map->cfg->hs_cache_dir,
+ (gint) rspamd_cryptobox_HASHBYTES / 2, re_map->re_digest);
+ re_map->hs_db = rspamd_hyperscan_from_raw_db(hs_db, fpath);
+ }
+ else {
+ re_map->hs_db = rspamd_hyperscan_from_raw_db(hs_db, NULL);
+ }
+
+ ts1 = (rspamd_get_ticks(FALSE) - ts1) * 1000.0;
+ msg_info_map("hyperscan compiled %d regular expressions from %s in %.1f ms",
+ re_map->regexps->len, re_map->map->name, ts1);
+ rspamd_try_save_re_map_cache(re_map);
+ }
+ else {
+ msg_info_map("hyperscan read %d cached regular expressions from %s",
+ re_map->regexps->len, re_map->map->name);
+ }
+
+ if (hs_alloc_scratch(rspamd_hyperscan_get_database(re_map->hs_db), &re_map->hs_scratch) != HS_SUCCESS) {
+ msg_err_map("cannot allocate scratch space for hyperscan");
+ rspamd_hyperscan_free(re_map->hs_db, true);
+ re_map->hs_db = NULL;
+ }
+ }
+ else {
+ msg_err_map("regexp map is empty");
+ }
+#endif
+}
+
+gchar *
+rspamd_regexp_list_read_single(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct rspamd_regexp_map_helper *re_map;
+
+ if (data->cur_data == NULL) {
+ re_map = rspamd_map_helper_new_regexp(data->map, 0);
+ data->cur_data = re_map;
+ }
+
+ return rspamd_parse_kv_list(
+ chunk,
+ len,
+ data,
+ rspamd_map_helper_insert_re,
+ hash_fill,
+ final);
+}
+
+gchar *
+rspamd_glob_list_read_single(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct rspamd_regexp_map_helper *re_map;
+
+ if (data->cur_data == NULL) {
+ re_map = rspamd_map_helper_new_regexp(data->map, RSPAMD_REGEXP_MAP_FLAG_GLOB);
+ data->cur_data = re_map;
+ }
+
+ return rspamd_parse_kv_list(
+ chunk,
+ len,
+ data,
+ rspamd_map_helper_insert_re,
+ hash_fill,
+ final);
+}
+
+gchar *
+rspamd_regexp_list_read_multiple(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct rspamd_regexp_map_helper *re_map;
+
+ if (data->cur_data == NULL) {
+ re_map = rspamd_map_helper_new_regexp(data->map,
+ RSPAMD_REGEXP_MAP_FLAG_MULTIPLE);
+ data->cur_data = re_map;
+ }
+
+ return rspamd_parse_kv_list(
+ chunk,
+ len,
+ data,
+ rspamd_map_helper_insert_re,
+ hash_fill,
+ final);
+}
+
+gchar *
+rspamd_glob_list_read_multiple(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct rspamd_regexp_map_helper *re_map;
+
+ if (data->cur_data == NULL) {
+ re_map = rspamd_map_helper_new_regexp(data->map,
+ RSPAMD_REGEXP_MAP_FLAG_GLOB | RSPAMD_REGEXP_MAP_FLAG_MULTIPLE);
+ data->cur_data = re_map;
+ }
+
+ return rspamd_parse_kv_list(
+ chunk,
+ len,
+ data,
+ rspamd_map_helper_insert_re,
+ hash_fill,
+ final);
+}
+
+
+void rspamd_regexp_list_fin(struct map_cb_data *data, void **target)
+{
+ struct rspamd_regexp_map_helper *re_map = NULL, *old_re_map;
+ struct rspamd_map *map = data->map;
+
+ if (data->errored) {
+ /* Clean up the current data and do not touch prev data */
+ if (data->cur_data) {
+ msg_info_map("cleanup unfinished new data as error occurred for %s",
+ map->name);
+ re_map = (struct rspamd_regexp_map_helper *) data->cur_data;
+ rspamd_map_helper_destroy_regexp(re_map);
+ data->cur_data = NULL;
+ }
+ }
+ else {
+ if (data->cur_data) {
+ re_map = data->cur_data;
+ rspamd_cryptobox_hash_final(&re_map->hst, re_map->re_digest);
+ memcpy(&data->map->digest, re_map->re_digest, sizeof(data->map->digest));
+ rspamd_re_map_finalize(re_map);
+ msg_info_map("read regexp list of %ud elements",
+ re_map->regexps->len);
+ data->map->traverse_function = rspamd_map_helper_traverse_regexp;
+ data->map->nelts = kh_size(re_map->htb);
+ }
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ old_re_map = data->prev_data;
+ rspamd_map_helper_destroy_regexp(old_re_map);
+ }
+ }
+}
+void rspamd_regexp_list_dtor(struct map_cb_data *data)
+{
+ if (data->cur_data) {
+ rspamd_map_helper_destroy_regexp(data->cur_data);
+ }
+}
+
+#ifdef WITH_HYPERSCAN
+static int
+rspamd_match_hs_single_handler(unsigned int id, unsigned long long from,
+ unsigned long long to,
+ unsigned int flags, void *context)
+{
+ guint *i = context;
+ /* Always return non-zero as we need a single match here */
+
+ *i = id;
+
+ return 1;
+}
+#endif
+
+gconstpointer
+rspamd_match_regexp_map_single(struct rspamd_regexp_map_helper *map,
+ const gchar *in, gsize len)
+{
+ guint i;
+ rspamd_regexp_t *re;
+ gint res = 0;
+ gpointer ret = NULL;
+ struct rspamd_map_helper_value *val;
+ gboolean validated = FALSE;
+
+ g_assert(in != NULL);
+
+ if (map == NULL || len == 0 || map->regexps == NULL) {
+ return NULL;
+ }
+
+ if (map->map_flags & RSPAMD_REGEXP_MAP_FLAG_UTF) {
+ if (rspamd_fast_utf8_validate(in, len) == 0) {
+ validated = TRUE;
+ }
+ }
+ else {
+ validated = TRUE;
+ }
+
+#ifdef WITH_HYPERSCAN
+ if (map->hs_db && map->hs_scratch) {
+
+ if (validated) {
+
+ res = hs_scan(rspamd_hyperscan_get_database(map->hs_db), in, len, 0,
+ map->hs_scratch,
+ rspamd_match_hs_single_handler, (void *) &i);
+
+ if (res == HS_SCAN_TERMINATED) {
+ res = 1;
+ val = g_ptr_array_index(map->values, i);
+
+ ret = val->value;
+ val->hits++;
+ }
+
+ return ret;
+ }
+ }
+#endif
+
+ if (!res) {
+ /* PCRE version */
+ for (i = 0; i < map->regexps->len; i++) {
+ re = g_ptr_array_index(map->regexps, i);
+
+ if (rspamd_regexp_search(re, in, len, NULL, NULL, !validated, NULL)) {
+ val = g_ptr_array_index(map->values, i);
+
+ ret = val->value;
+ val->hits++;
+ break;
+ }
+ }
+ }
+
+ return ret;
+}
+
+#ifdef WITH_HYPERSCAN
+struct rspamd_multiple_cbdata {
+ GPtrArray *ar;
+ struct rspamd_regexp_map_helper *map;
+};
+
+static int
+rspamd_match_hs_multiple_handler(unsigned int id, unsigned long long from,
+ unsigned long long to,
+ unsigned int flags, void *context)
+{
+ struct rspamd_multiple_cbdata *cbd = context;
+ struct rspamd_map_helper_value *val;
+
+
+ if (id < cbd->map->values->len) {
+ val = g_ptr_array_index(cbd->map->values, id);
+ val->hits++;
+ g_ptr_array_add(cbd->ar, val->value);
+ }
+
+ /* Always return zero as we need all matches here */
+ return 0;
+}
+#endif
+
+GPtrArray *
+rspamd_match_regexp_map_all(struct rspamd_regexp_map_helper *map,
+ const gchar *in, gsize len)
+{
+ guint i;
+ rspamd_regexp_t *re;
+ GPtrArray *ret;
+ gint res = 0;
+ gboolean validated = FALSE;
+ struct rspamd_map_helper_value *val;
+
+ if (map == NULL || map->regexps == NULL || len == 0) {
+ return NULL;
+ }
+
+ g_assert(in != NULL);
+
+ if (map->map_flags & RSPAMD_REGEXP_MAP_FLAG_UTF) {
+ if (rspamd_fast_utf8_validate(in, len) == 0) {
+ validated = TRUE;
+ }
+ }
+ else {
+ validated = TRUE;
+ }
+
+ ret = g_ptr_array_new();
+
+#ifdef WITH_HYPERSCAN
+ if (map->hs_db && map->hs_scratch) {
+
+ if (validated) {
+ struct rspamd_multiple_cbdata cbd;
+
+ cbd.ar = ret;
+ cbd.map = map;
+
+ if (hs_scan(rspamd_hyperscan_get_database(map->hs_db), in, len,
+ 0, map->hs_scratch,
+ rspamd_match_hs_multiple_handler, &cbd) == HS_SUCCESS) {
+ res = 1;
+ }
+ }
+ }
+#endif
+
+ if (!res) {
+ /* PCRE version */
+ for (i = 0; i < map->regexps->len; i++) {
+ re = g_ptr_array_index(map->regexps, i);
+
+ if (rspamd_regexp_search(re, in, len, NULL, NULL,
+ !validated, NULL)) {
+ val = g_ptr_array_index(map->values, i);
+ val->hits++;
+ g_ptr_array_add(ret, val->value);
+ }
+ }
+ }
+
+ if (ret->len > 0) {
+ return ret;
+ }
+
+ g_ptr_array_free(ret, TRUE);
+
+ return NULL;
+}
+
+gconstpointer
+rspamd_match_hash_map(struct rspamd_hash_map_helper *map, const gchar *in,
+ gsize len)
+{
+ khiter_t k;
+ struct rspamd_map_helper_value *val;
+ rspamd_ftok_t tok;
+
+ if (map == NULL || map->htb == NULL) {
+ return NULL;
+ }
+
+ tok.begin = in;
+ tok.len = len;
+
+ k = kh_get(rspamd_map_hash, map->htb, tok);
+
+ if (k != kh_end(map->htb)) {
+ val = kh_value(map->htb, k);
+ val->hits++;
+
+ return val->value;
+ }
+
+ return NULL;
+}
+
+gconstpointer
+rspamd_match_radix_map(struct rspamd_radix_map_helper *map,
+ const guchar *in, gsize inlen)
+{
+ struct rspamd_map_helper_value *val;
+
+ if (map == NULL || map->trie == NULL) {
+ return NULL;
+ }
+
+ val = (struct rspamd_map_helper_value *) radix_find_compressed(map->trie,
+ in, inlen);
+
+ if (val != (gconstpointer) RADIX_NO_VALUE) {
+ val->hits++;
+
+ return val->value;
+ }
+
+ return NULL;
+}
+
+gconstpointer
+rspamd_match_radix_map_addr(struct rspamd_radix_map_helper *map,
+ const rspamd_inet_addr_t *addr)
+{
+ struct rspamd_map_helper_value *val;
+
+ if (map == NULL || map->trie == NULL) {
+ return NULL;
+ }
+
+ val = (struct rspamd_map_helper_value *) radix_find_compressed_addr(map->trie, addr);
+
+ if (val != (gconstpointer) RADIX_NO_VALUE) {
+ val->hits++;
+
+ return val->value;
+ }
+
+ return NULL;
+}
+
+
+/*
+ * CBD stuff
+ */
+
+struct rspamd_cdb_map_helper *
+rspamd_map_helper_new_cdb(struct rspamd_map *map)
+{
+ struct rspamd_cdb_map_helper *n;
+
+ n = g_malloc0(sizeof(*n));
+ n->cdbs = (GQueue) G_QUEUE_INIT;
+ n->map = map;
+
+ rspamd_cryptobox_fast_hash_init(&n->hst, map_hash_seed);
+
+ return n;
+}
+
+void rspamd_map_helper_destroy_cdb(struct rspamd_cdb_map_helper *c)
+{
+ if (c == NULL) {
+ return;
+ }
+
+ GList *cur = c->cdbs.head;
+
+ while (cur) {
+ struct cdb *cdb = (struct cdb *) cur->data;
+
+ cdb_free(cdb);
+ g_free(cdb->filename);
+ close(cdb->cdb_fd);
+ g_free(cdb);
+
+ cur = g_list_next(cur);
+ }
+
+ g_queue_clear(&c->cdbs);
+
+ g_free(c);
+}
+
+gchar *
+rspamd_cdb_list_read(gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct rspamd_cdb_map_helper *cdb_data;
+ struct cdb *found = NULL;
+ struct rspamd_map *map = data->map;
+
+ g_assert(map->no_file_read);
+
+ if (data->cur_data == NULL) {
+ cdb_data = rspamd_map_helper_new_cdb(data->map);
+ data->cur_data = cdb_data;
+ }
+ else {
+ cdb_data = (struct rspamd_cdb_map_helper *) data->cur_data;
+ }
+
+ GList *cur = cdb_data->cdbs.head;
+
+ while (cur) {
+ struct cdb *elt = (struct cdb *) cur->data;
+
+ if (strcmp(elt->filename, chunk) == 0) {
+ found = elt;
+ break;
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ if (found == NULL) {
+ /* New cdb */
+ gint fd;
+ struct cdb *cdb;
+
+ fd = rspamd_file_xopen(chunk, O_RDONLY, 0, TRUE);
+
+ if (fd == -1) {
+ msg_err_map("cannot open cdb map from %s: %s", chunk, strerror(errno));
+
+ return NULL;
+ }
+
+ cdb = g_malloc0(sizeof(struct cdb));
+
+ if (cdb_init(cdb, fd) == -1) {
+ g_free(cdb);
+ msg_err_map("cannot init cdb map from %s: %s", chunk, strerror(errno));
+
+ return NULL;
+ }
+
+ cdb->filename = g_strdup(chunk);
+ g_queue_push_tail(&cdb_data->cdbs, cdb);
+ cdb_data->total_size += cdb->cdb_fsize;
+ rspamd_cryptobox_fast_hash_update(&cdb_data->hst, chunk, len);
+ }
+
+ return chunk + len;
+}
+
+void rspamd_cdb_list_fin(struct map_cb_data *data, void **target)
+{
+ struct rspamd_map *map = data->map;
+ struct rspamd_cdb_map_helper *cdb_data;
+
+ if (data->errored) {
+ /* Clean up the current data and do not touch prev data */
+ if (data->cur_data) {
+ msg_info_map("cleanup unfinished new data as error occurred for %s",
+ map->name);
+ cdb_data = (struct rspamd_cdb_map_helper *) data->cur_data;
+ rspamd_map_helper_destroy_cdb(cdb_data);
+ data->cur_data = NULL;
+ }
+ }
+ else {
+ if (data->cur_data) {
+ cdb_data = (struct rspamd_cdb_map_helper *) data->cur_data;
+ msg_info_map("read cdb of %Hz size", cdb_data->total_size);
+ data->map->traverse_function = NULL;
+ data->map->nelts = 0;
+ data->map->digest = rspamd_cryptobox_fast_hash_final(&cdb_data->hst);
+ }
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ cdb_data = (struct rspamd_cdb_map_helper *) data->prev_data;
+ rspamd_map_helper_destroy_cdb(cdb_data);
+ }
+ }
+}
+void rspamd_cdb_list_dtor(struct map_cb_data *data)
+{
+ if (data->cur_data) {
+ rspamd_map_helper_destroy_cdb(data->cur_data);
+ }
+}
+
+gconstpointer
+rspamd_match_cdb_map(struct rspamd_cdb_map_helper *map,
+ const gchar *in, gsize inlen)
+{
+ if (map == NULL || map->cdbs.head == NULL) {
+ return NULL;
+ }
+
+ GList *cur = map->cdbs.head;
+ static rspamd_ftok_t found;
+
+ while (cur) {
+ struct cdb *cdb = (struct cdb *) cur->data;
+
+ if (cdb_find(cdb, in, inlen) > 0) {
+ /* Extract and push value to lua as string */
+ unsigned vlen;
+ gconstpointer vpos;
+
+ vpos = cdb->cdb_mem + cdb_datapos(cdb);
+ vlen = cdb_datalen(cdb);
+ found.len = vlen;
+ found.begin = vpos;
+
+ return &found; /* Do not reuse! */
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ return NULL;
+}
diff --git a/src/libserver/maps/map_helpers.h b/src/libserver/maps/map_helpers.h
new file mode 100644
index 0000000..82c62b6
--- /dev/null
+++ b/src/libserver/maps/map_helpers.h
@@ -0,0 +1,269 @@
+/*-
+ * Copyright 2018 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_MAP_HELPERS_H
+#define RSPAMD_MAP_HELPERS_H
+
+#include "config.h"
+#include "map.h"
+#include "addr.h"
+
+/**
+ * @file map_helpers.h
+ *
+ * Defines helper structures to deal with different map types
+ */
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Common structures, abstract for simplicity
+ */
+struct rspamd_radix_map_helper;
+struct rspamd_hash_map_helper;
+struct rspamd_regexp_map_helper;
+struct rspamd_cdb_map_helper;
+struct rspamd_map_helper_value;
+
+enum rspamd_regexp_map_flags {
+ RSPAMD_REGEXP_MAP_FLAG_UTF = (1u << 0),
+ RSPAMD_REGEXP_MAP_FLAG_MULTIPLE = (1u << 1),
+ RSPAMD_REGEXP_MAP_FLAG_GLOB = (1u << 2),
+};
+
+typedef void (*rspamd_map_insert_func)(gpointer st, gconstpointer key,
+ gconstpointer value);
+
+/**
+ * Radix list is a list like ip/mask
+ */
+gchar *rspamd_radix_read(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+
+void rspamd_radix_fin(struct map_cb_data *data, void **target);
+
+void rspamd_radix_dtor(struct map_cb_data *data);
+
+/**
+ * Kv list is an ordinal list of keys and values separated by whitespace
+ */
+gchar *rspamd_kv_list_read(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+
+void rspamd_kv_list_fin(struct map_cb_data *data, void **target);
+
+void rspamd_kv_list_dtor(struct map_cb_data *data);
+
+/**
+ * Cdb is a cdb mapped file with shared data
+ * chunk must be filename!
+ */
+gchar *rspamd_cdb_list_read(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+void rspamd_cdb_list_fin(struct map_cb_data *data, void **target);
+void rspamd_cdb_list_dtor(struct map_cb_data *data);
+
+/**
+ * Regexp list is a list of regular expressions
+ */
+
+gchar *rspamd_regexp_list_read_single(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+
+gchar *rspamd_regexp_list_read_multiple(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+
+gchar *rspamd_glob_list_read_single(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+
+gchar *rspamd_glob_list_read_multiple(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+
+void rspamd_regexp_list_fin(struct map_cb_data *data, void **target);
+
+void rspamd_regexp_list_dtor(struct map_cb_data *data);
+
+/**
+ * FSM for lists parsing (support comments, blank lines and partial replies)
+ */
+gchar *
+rspamd_parse_kv_list(
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ rspamd_map_insert_func func,
+ const gchar *default_value,
+ gboolean final);
+
+/**
+ * Find a single (any) matching regexp for the specified text or NULL if
+ * no matches found
+ * @param map
+ * @param in
+ * @param len
+ * @return
+ */
+gconstpointer rspamd_match_regexp_map_single(struct rspamd_regexp_map_helper *map,
+ const gchar *in, gsize len);
+
+/**
+ * Find a multiple (all) matching regexp for the specified text or NULL if
+ * no matches found. Returns GPtrArray that *must* be freed by a caller if not NULL
+ * @param map
+ * @param in
+ * @param len
+ * @return
+ */
+GPtrArray *rspamd_match_regexp_map_all(struct rspamd_regexp_map_helper *map,
+ const gchar *in, gsize len);
+
+/**
+ * Find value matching specific key in a hash map
+ * @param map
+ * @param in
+ * @param len
+ * @return
+ */
+gconstpointer rspamd_match_hash_map(struct rspamd_hash_map_helper *map,
+ const gchar *in, gsize len);
+
+/**
+ * Find value matching specific key in a cdb map
+ * @param map
+ * @param in
+ * @param len
+ * @return rspamd_ftok_t pointer (allocated in a static buffer!)
+ */
+gconstpointer rspamd_match_cdb_map(struct rspamd_cdb_map_helper *map,
+ const gchar *in, gsize len);
+
+/**
+ * Find value matching specific key in a hash map
+ * @param map
+ * @param in raw ip address
+ * @param inlen ip address length (4 for IPv4 and 16 for IPv6)
+ * @return
+ */
+gconstpointer rspamd_match_radix_map(struct rspamd_radix_map_helper *map,
+ const guchar *in, gsize inlen);
+
+gconstpointer rspamd_match_radix_map_addr(struct rspamd_radix_map_helper *map,
+ const rspamd_inet_addr_t *addr);
+
+/**
+ * Creates radix map helper
+ * @param map
+ * @return
+ */
+struct rspamd_radix_map_helper *rspamd_map_helper_new_radix(struct rspamd_map *map);
+
+/**
+ * Inserts new value into radix map
+ * @param st
+ * @param key
+ * @param value
+ */
+void rspamd_map_helper_insert_radix(gpointer st, gconstpointer key, gconstpointer value);
+
+/**
+ * Inserts new value into radix map performing synchronous resolving
+ * @param st
+ * @param key
+ * @param value
+ */
+void rspamd_map_helper_insert_radix_resolve(gpointer st, gconstpointer key,
+ gconstpointer value);
+
+/**
+ * Destroys radix map helper
+ * @param r
+ */
+void rspamd_map_helper_destroy_radix(struct rspamd_radix_map_helper *r);
+
+
+/**
+ * Creates hash map helper
+ * @param map
+ * @return
+ */
+struct rspamd_hash_map_helper *rspamd_map_helper_new_hash(struct rspamd_map *map);
+
+/**
+ * Inserts a new value into a hash map
+ * @param st
+ * @param key
+ * @param value
+ */
+void rspamd_map_helper_insert_hash(gpointer st, gconstpointer key, gconstpointer value);
+
+/**
+ * Destroys hash map helper
+ * @param r
+ */
+void rspamd_map_helper_destroy_hash(struct rspamd_hash_map_helper *r);
+
+/**
+ * Create new regexp map
+ * @param map
+ * @param flags
+ * @return
+ */
+struct rspamd_regexp_map_helper *rspamd_map_helper_new_regexp(struct rspamd_map *map,
+ enum rspamd_regexp_map_flags flags);
+
+/**
+ * Inserts a new regexp into regexp map
+ * @param st
+ * @param key
+ * @param value
+ */
+void rspamd_map_helper_insert_re(gpointer st, gconstpointer key, gconstpointer value);
+
+/**
+ * Destroy regexp map
+ * @param re_map
+ */
+void rspamd_map_helper_destroy_regexp(struct rspamd_regexp_map_helper *re_map);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/maps/map_private.h b/src/libserver/maps/map_private.h
new file mode 100644
index 0000000..60751c0
--- /dev/null
+++ b/src/libserver/maps/map_private.h
@@ -0,0 +1,226 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBUTIL_MAP_PRIVATE_H_
+#define SRC_LIBUTIL_MAP_PRIVATE_H_
+
+#include "config.h"
+#include "mem_pool.h"
+#include "keypair.h"
+#include "unix-std.h"
+#include "map.h"
+#include "ref.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void (*rspamd_map_tmp_dtor)(gpointer p);
+
+extern guint rspamd_map_log_id;
+#define msg_err_map(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "map", map->tag, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_map(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "map", map->tag, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_map(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "map", map->tag, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_map(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_map_log_id, "map", map->tag, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+enum fetch_proto {
+ MAP_PROTO_FILE,
+ MAP_PROTO_HTTP,
+ MAP_PROTO_HTTPS,
+ MAP_PROTO_STATIC
+};
+
+/**
+ * Data specific to file maps
+ */
+struct file_map_data {
+ gchar *filename;
+ gboolean need_modify;
+ ev_stat st_ev;
+};
+
+
+struct http_map_data;
+
+struct rspamd_http_map_cached_cbdata {
+ ev_timer timeout;
+ struct ev_loop *event_loop;
+ struct rspamd_storage_shmem *shm;
+ struct rspamd_map *map;
+ struct http_map_data *data;
+ guint64 gen;
+ time_t last_checked;
+};
+
+struct rspamd_map_cachepoint {
+ gint available;
+ gsize len;
+ time_t last_modified;
+ gchar shmem_name[256];
+};
+
+/**
+ * Data specific to HTTP maps
+ */
+struct http_map_data {
+ /* Shared cache data */
+ struct rspamd_map_cachepoint *cache;
+ /* Non-shared for cache owner, used to cleanup cache */
+ struct rspamd_http_map_cached_cbdata *cur_cache_cbd;
+ gchar *userinfo;
+ gchar *path;
+ gchar *host;
+ gchar *rest;
+ rspamd_fstring_t *etag;
+ time_t last_modified;
+ time_t last_checked;
+ gboolean request_sent;
+ guint64 gen;
+ guint16 port;
+};
+
+struct static_map_data {
+ guchar *data;
+ gsize len;
+ gboolean processed;
+};
+
+union rspamd_map_backend_data {
+ struct file_map_data *fd;
+ struct http_map_data *hd;
+ struct static_map_data *sd;
+};
+
+
+struct rspamd_map;
+struct rspamd_map_backend {
+ enum fetch_proto protocol;
+ gboolean is_signed;
+ gboolean is_compressed;
+ gboolean is_fallback;
+ struct rspamd_map *map;
+ struct ev_loop *event_loop;
+ guint32 id;
+ struct rspamd_cryptobox_pubkey *trusted_pubkey;
+ union rspamd_map_backend_data data;
+ gchar *uri;
+ ref_entry_t ref;
+};
+
+struct map_periodic_cbdata;
+
+struct rspamd_map {
+ struct rspamd_dns_resolver *r;
+ struct rspamd_config *cfg;
+ GPtrArray *backends;
+ struct rspamd_map_backend *fallback_backend;
+ map_cb_t read_callback;
+ map_fin_cb_t fin_callback;
+ map_dtor_t dtor;
+ void **user_data;
+ struct ev_loop *event_loop;
+ struct rspamd_worker *wrk;
+ gchar *description;
+ gchar *name;
+ guint32 id;
+ struct map_periodic_cbdata *scheduled_check;
+ rspamd_map_tmp_dtor tmp_dtor;
+ gpointer tmp_dtor_data;
+ rspamd_map_traverse_function traverse_function;
+ rspamd_map_on_load_function on_load_function;
+ gpointer on_load_ud;
+ GDestroyNotify on_load_ud_dtor;
+ gpointer lua_map;
+ gsize nelts;
+ guint64 digest;
+ /* Should we check HTTP or just load cached data */
+ ev_tstamp timeout;
+ gdouble poll_timeout;
+ time_t next_check;
+ bool active_http;
+ bool non_trivial; /* E.g. has http backends in active mode */
+ bool file_only; /* No HTTP backends found */
+ bool static_only; /* No need to check */
+ bool no_file_read; /* Do not read files */
+ /* Shared lock for temporary disabling of map reading (e.g. when this map is written by UI) */
+ gint *locked;
+ gchar tag[MEMPOOL_UID_LEN];
+};
+
+enum rspamd_map_http_stage {
+ http_map_resolve_host2 = 0, /* 2 requests sent */
+ http_map_resolve_host1, /* 1 requests sent */
+ http_map_http_conn, /* http connection */
+ http_map_terminated /* terminated when doing resolving */
+};
+
+struct map_periodic_cbdata {
+ struct rspamd_map *map;
+ struct map_cb_data cbdata;
+ ev_timer ev;
+ gboolean need_modify;
+ gboolean errored;
+ gboolean locked;
+ guint cur_backend;
+ ref_entry_t ref;
+};
+
+static const gchar rspamd_http_file_magic[] =
+ {'r', 'm', 'c', 'd', '2', '0', '0', '0'};
+
+struct rspamd_http_file_data {
+ guchar magic[sizeof(rspamd_http_file_magic)];
+ goffset data_off;
+ gulong mtime;
+ gulong next_check;
+ gulong etag_len;
+};
+
+struct http_callback_data {
+ struct ev_loop *event_loop;
+ struct rspamd_http_connection *conn;
+ GPtrArray *addrs;
+ rspamd_inet_addr_t *addr;
+ struct rspamd_map *map;
+ struct rspamd_map_backend *bk;
+ struct http_map_data *data;
+ struct map_periodic_cbdata *periodic;
+ struct rspamd_cryptobox_pubkey *pk;
+ struct rspamd_storage_shmem *shmem_data;
+ gsize data_len;
+ gboolean check;
+ enum rspamd_map_http_stage stage;
+ ev_tstamp timeout;
+
+ ref_entry_t ref;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBUTIL_MAP_PRIVATE_H_ */
diff --git a/src/libserver/mempool_vars_internal.h b/src/libserver/mempool_vars_internal.h
new file mode 100644
index 0000000..6c95538
--- /dev/null
+++ b/src/libserver/mempool_vars_internal.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_MEMPOOL_VARS_INTERNAL_H
+#define RSPAMD_MEMPOOL_VARS_INTERNAL_H
+
+/* Basic rspamd mempool variables names */
+#define RSPAMD_MEMPOOL_AVG_WORDS_LEN "avg_words_len"
+#define RSPAMD_MEMPOOL_SHORT_WORDS_CNT "short_words_cnt"
+#define RSPAMD_MEMPOOL_HEADERS_HASH "headers_hash"
+#define RSPAMD_MEMPOOL_MTA_TAG "MTA-Tag"
+#define RSPAMD_MEMPOOL_MTA_NAME "MTA-Name"
+#define RSPAMD_MEMPOOL_SPF_DOMAIN "spf_domain"
+#define RSPAMD_MEMPOOL_SPF_RECORD "spf_record"
+#define RSPAMD_MEMPOOL_PRINCIPAL_RECIPIENT "principal_recipient"
+#define RSPAMD_MEMPOOL_PROFILE "profile"
+#define RSPAMD_MEMPOOL_MILTER_REPLY "milter_reply"
+#define RSPAMD_MEMPOOL_DKIM_SIGNATURE "dkim-signature"
+#define RSPAMD_MEMPOOL_DMARC_CHECKS "dmarc_checks"
+#define RSPAMD_MEMPOOL_DKIM_BH_CACHE "dkim_bh_cache"
+#define RSPAMD_MEMPOOL_DKIM_CHECK_RESULTS "dkim_results"
+#define RSPAMD_MEMPOOL_DKIM_SIGN_KEY "dkim_key"
+#define RSPAMD_MEMPOOL_DKIM_SIGN_SELECTOR "dkim_selector"
+#define RSPAMD_MEMPOOL_ARC_SIGN_KEY "arc_key"
+#define RSPAMD_MEMPOOL_ARC_SIGN_SELECTOR "arc_selector"
+#define RSPAMD_MEMPOOL_STAT_SIGNATURE "stat_signature"
+#define RSPAMD_MEMPOOL_FUZZY_RESULT "fuzzy_hashes"
+#define RSPAMD_MEMPOOL_SPAM_LEARNS "spam_learns"
+#define RSPAMD_MEMPOOL_HAM_LEARNS "ham_learns"
+#define RSPAMD_MEMPOOL_RE_MAPS_CACHE "re_maps_cache"
+#define RSPAMD_MEMPOOL_HTTP_STAT_BACKEND_RUNTIME "stat_http_runtime"
+#define RSPAMD_MEMPOOL_FUZZY_STAT "fuzzy_stat"
+
+#endif
diff --git a/src/libserver/milter.c b/src/libserver/milter.c
new file mode 100644
index 0000000..cfb7d3c
--- /dev/null
+++ b/src/libserver/milter.c
@@ -0,0 +1,2232 @@
+/*-
+ * Copyright 2017 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "milter.h"
+#include "milter_internal.h"
+#include "email_addr.h"
+#include "addr.h"
+#include "unix-std.h"
+#include "logger.h"
+#include "ottery.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
+#include "libserver/protocol_internal.h"
+#include "libserver/cfg_file_private.h"
+#include "libmime/scan_result.h"
+#include "libserver/worker_util.h"
+#include "utlist.h"
+
+#define msg_err_milter(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "milter", priv->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_milter(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "milter", priv->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_milter(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "milter", priv->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_milter(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_milter_log_id, "milter", priv->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(milter)
+
+static const struct rspamd_milter_context *milter_ctx = NULL;
+
+static gboolean rspamd_milter_handle_session(
+ struct rspamd_milter_session *session,
+ struct rspamd_milter_private *priv);
+static inline void rspamd_milter_plan_io(struct rspamd_milter_session *session,
+ struct rspamd_milter_private *priv, gshort what);
+
+static GQuark
+rspamd_milter_quark(void)
+{
+ return g_quark_from_static_string("milter");
+}
+
+static void
+rspamd_milter_obuf_free(struct rspamd_milter_outbuf *obuf)
+{
+ if (obuf) {
+ if (obuf->buf) {
+ rspamd_fstring_free(obuf->buf);
+ }
+
+ g_free(obuf);
+ }
+}
+
+#define RSPAMD_MILTER_RESET_COMMON (1 << 0)
+#define RSPAMD_MILTER_RESET_IO (1 << 1)
+#define RSPAMD_MILTER_RESET_ADDR (1 << 2)
+#define RSPAMD_MILTER_RESET_MACRO (1 << 3)
+#define RSPAMD_MILTER_RESET_ALL (RSPAMD_MILTER_RESET_COMMON | \
+ RSPAMD_MILTER_RESET_IO | \
+ RSPAMD_MILTER_RESET_ADDR | \
+ RSPAMD_MILTER_RESET_MACRO)
+#define RSPAMD_MILTER_RESET_QUIT_NC (RSPAMD_MILTER_RESET_COMMON | \
+ RSPAMD_MILTER_RESET_ADDR | \
+ RSPAMD_MILTER_RESET_MACRO)
+#define RSPAMD_MILTER_RESET_ABORT (RSPAMD_MILTER_RESET_COMMON)
+
+static void
+rspamd_milter_session_reset(struct rspamd_milter_session *session,
+ guint how)
+{
+ struct rspamd_milter_outbuf *obuf, *obuf_tmp;
+ struct rspamd_milter_private *priv = session->priv;
+ struct rspamd_email_address *cur;
+ guint i;
+
+ if (how & RSPAMD_MILTER_RESET_IO) {
+ msg_debug_milter("cleanup IO on abort");
+
+ DL_FOREACH_SAFE(priv->out_chain, obuf, obuf_tmp)
+ {
+ rspamd_milter_obuf_free(obuf);
+ }
+
+ priv->out_chain = NULL;
+
+ if (priv->parser.buf) {
+ priv->parser.buf->len = 0;
+ }
+ }
+
+ if (how & RSPAMD_MILTER_RESET_COMMON) {
+ msg_debug_milter("cleanup common data on abort");
+
+ if (session->message) {
+ session->message->len = 0;
+ msg_debug_milter("cleanup message on abort");
+ }
+
+ if (session->rcpts) {
+ PTR_ARRAY_FOREACH(session->rcpts, i, cur)
+ {
+ rspamd_email_address_free(cur);
+ }
+
+ msg_debug_milter("cleanup %d recipients on abort",
+ (gint) session->rcpts->len);
+
+ g_ptr_array_free(session->rcpts, TRUE);
+ session->rcpts = NULL;
+ }
+
+ if (session->from) {
+ msg_debug_milter("cleanup from");
+ rspamd_email_address_free(session->from);
+ session->from = NULL;
+ }
+
+ if (priv->headers) {
+ msg_debug_milter("cleanup headers");
+ gchar *k;
+ GArray *ar;
+
+ kh_foreach(priv->headers, k, ar, {
+ g_free(k);
+ g_array_free(ar, TRUE);
+ });
+
+ kh_clear(milter_headers_hash_t, priv->headers);
+ }
+
+ priv->cur_hdr = 0;
+ }
+
+ if (how & RSPAMD_MILTER_RESET_ADDR) {
+ if (session->addr) {
+ msg_debug_milter("cleanup addr");
+ rspamd_inet_address_free(session->addr);
+ session->addr = NULL;
+ }
+ if (session->hostname) {
+ msg_debug_milter("cleanup hostname");
+ session->hostname->len = 0;
+ }
+ }
+
+ if (how & RSPAMD_MILTER_RESET_MACRO) {
+ if (session->macros) {
+ msg_debug_milter("cleanup macros");
+ g_hash_table_unref(session->macros);
+ session->macros = NULL;
+ }
+ }
+}
+
+static void
+rspamd_milter_session_dtor(struct rspamd_milter_session *session)
+{
+ struct rspamd_milter_private *priv;
+
+ if (session) {
+ priv = session->priv;
+ msg_debug_milter("destroying milter session");
+
+ rspamd_ev_watcher_stop(priv->event_loop, &priv->ev);
+ rspamd_milter_session_reset(session, RSPAMD_MILTER_RESET_ALL);
+ close(priv->fd);
+
+ if (priv->parser.buf) {
+ rspamd_fstring_free(priv->parser.buf);
+ }
+
+ if (session->message) {
+ rspamd_fstring_free(session->message);
+ }
+
+ if (session->helo) {
+ rspamd_fstring_free(session->helo);
+ }
+
+ if (session->hostname) {
+ rspamd_fstring_free(session->hostname);
+ }
+
+ if (priv->headers) {
+ gchar *k;
+ GArray *ar;
+
+ kh_foreach(priv->headers, k, ar, {
+ g_free(k);
+ g_array_free(ar, TRUE);
+ });
+
+ kh_destroy(milter_headers_hash_t, priv->headers);
+ }
+
+ if (milter_ctx->sessions_cache) {
+ rspamd_worker_session_cache_remove(milter_ctx->sessions_cache,
+ session);
+ }
+
+ rspamd_mempool_delete(priv->pool);
+ g_free(priv);
+ g_free(session);
+ }
+}
+
+static void
+rspamd_milter_on_protocol_error(struct rspamd_milter_session *session,
+ struct rspamd_milter_private *priv, GError *err)
+{
+ msg_debug_milter("protocol error: %e", err);
+ priv->state = RSPAMD_MILTER_WANNA_DIE;
+ REF_RETAIN(session);
+ priv->err_cb(priv->fd, session, priv->ud, err);
+ REF_RELEASE(session);
+ g_error_free(err);
+
+ rspamd_milter_plan_io(session, priv, EV_WRITE);
+}
+
+static void
+rspamd_milter_on_protocol_ping(struct rspamd_milter_session *session,
+ struct rspamd_milter_private *priv)
+{
+ GError *err = NULL;
+ static const gchar reply[] = "HTTP/1.1 200 OK\r\n"
+ "Connection: close\r\n"
+ "Server: rspamd/2.7 (milter mode)\r\n"
+ "Content-Length: 6\r\n"
+ "Content-Type: text/plain\r\n"
+ "\r\n"
+ "pong\r\n";
+
+ if (write(priv->fd, reply, sizeof(reply)) == -1) {
+ gint serrno = errno;
+ msg_err_milter("cannot write pong reply: %s", strerror(serrno));
+ g_set_error(&err, rspamd_milter_quark(), serrno, "ping command IO error: %s",
+ strerror(serrno));
+ priv->state = RSPAMD_MILTER_WANNA_DIE;
+ REF_RETAIN(session);
+ priv->err_cb(priv->fd, session, priv->ud, err);
+ REF_RELEASE(session);
+ g_error_free(err);
+ }
+ else {
+ priv->state = RSPAMD_MILTER_PONG_AND_DIE;
+ rspamd_milter_plan_io(session, priv, EV_WRITE);
+ }
+}
+
+static gint
+rspamd_milter_http_on_url(http_parser *parser, const gchar *at, size_t length)
+{
+ GString *url = (GString *) parser->data;
+
+ g_string_append_len(url, at, length);
+
+ return 0;
+}
+
+static void
+rspamd_milter_io_handler(gint fd, gshort what, void *ud)
+{
+ struct rspamd_milter_session *session = ud;
+ struct rspamd_milter_private *priv;
+ GError *err;
+
+ priv = session->priv;
+
+ if (what == EV_TIMEOUT) {
+ msg_debug_milter("connection timed out");
+ err = g_error_new(rspamd_milter_quark(), ETIMEDOUT, "connection "
+ "timed out");
+ rspamd_milter_on_protocol_error(session, priv, err);
+ }
+ else {
+ rspamd_milter_handle_session(session, priv);
+ }
+}
+
+static inline void
+rspamd_milter_plan_io(struct rspamd_milter_session *session,
+ struct rspamd_milter_private *priv, gshort what)
+{
+ rspamd_ev_watcher_reschedule(priv->event_loop, &priv->ev, what);
+}
+
+
+#define READ_INT_32(pos, var) \
+ do { \
+ memcpy(&(var), (pos), sizeof(var)); \
+ (pos) += sizeof(var); \
+ (var) = ntohl(var); \
+ } while (0)
+#define READ_INT_16(pos, var) \
+ do { \
+ memcpy(&(var), (pos), sizeof(var)); \
+ (pos) += sizeof(var); \
+ (var) = ntohs(var); \
+ } while (0)
+
+static gboolean
+rspamd_milter_process_command(struct rspamd_milter_session *session,
+ struct rspamd_milter_private *priv)
+{
+ GError *err;
+ rspamd_fstring_t *buf;
+ const guchar *pos, *end, *zero;
+ guint cmdlen;
+ guint32 version, actions, protocol;
+
+ buf = priv->parser.buf;
+ pos = buf->str + priv->parser.cmd_start;
+ cmdlen = priv->parser.datalen;
+ end = pos + cmdlen;
+
+ switch (priv->parser.cur_cmd) {
+ case RSPAMD_MILTER_CMD_ABORT:
+ msg_debug_milter("got abort command");
+ rspamd_milter_session_reset(session, RSPAMD_MILTER_RESET_ABORT);
+ break;
+ case RSPAMD_MILTER_CMD_BODY:
+ if (!session->message) {
+ session->message = rspamd_fstring_sized_new(
+ RSPAMD_MILTER_MESSAGE_CHUNK);
+ }
+
+ msg_debug_milter("got body chunk: %d bytes", (int) cmdlen);
+ session->message = rspamd_fstring_append(session->message,
+ pos, cmdlen);
+ break;
+ case RSPAMD_MILTER_CMD_CONNECT:
+ msg_debug_milter("got connect command");
+
+ /*
+ * char hostname[]: Hostname, NUL terminated
+ * char family: Protocol family
+ * uint16 port: Port number (SMFIA_INET or SMFIA_INET6 only)
+ * char address[]: IP address (ASCII) or unix socket path, NUL terminated
+ */
+ zero = memchr(pos, '\0', cmdlen);
+
+ if (zero == NULL || zero > (end - sizeof(guint16) + 1)) {
+ err = g_error_new(rspamd_milter_quark(), EINVAL, "invalid "
+ "connect command (no name)");
+ rspamd_milter_on_protocol_error(session, priv, err);
+
+ return FALSE;
+ }
+ else {
+ guchar proto;
+ guint16 port;
+ gchar ip6_str[INET6_ADDRSTRLEN + 3];
+ gsize r;
+
+ /*
+ * Important notice: Postfix do NOT use this command to pass
+ * client's info (e.g. hostname is not really here)
+ * Sendmail will pass it here
+ */
+ if (session->hostname == NULL) {
+ session->hostname = rspamd_fstring_new_init(pos, zero - pos);
+ msg_debug_milter("got hostname on connect phase: %V",
+ session->hostname);
+ }
+ else {
+ session->hostname = rspamd_fstring_assign(session->hostname,
+ pos, zero - pos);
+ msg_debug_milter("rewrote hostname on connect phase: %V",
+ session->hostname);
+ }
+
+ pos = zero + 1;
+ proto = *pos++;
+
+ if (proto == RSPAMD_MILTER_CONN_UNKNOWN) {
+ /* We have no information about host */
+ msg_debug_milter("unknown connect address");
+ }
+ else {
+ READ_INT_16(pos, port);
+
+ if (pos >= end) {
+ /* No IP somehow */
+ msg_debug_milter("unknown connect IP/socket");
+ }
+ else {
+ zero = memchr(pos, '\0', end - pos);
+
+ if (zero == NULL) {
+ err = g_error_new(rspamd_milter_quark(), EINVAL, "invalid "
+ "connect command (no zero terminated IP)");
+ rspamd_milter_on_protocol_error(session, priv, err);
+
+ return FALSE;
+ }
+
+ switch (proto) {
+ case RSPAMD_MILTER_CONN_UNIX:
+ session->addr = rspamd_inet_address_new(AF_UNIX,
+ pos);
+ break;
+
+ case RSPAMD_MILTER_CONN_INET:
+ session->addr = rspamd_inet_address_new(AF_INET, NULL);
+
+ if (!rspamd_parse_inet_address_ip(pos, zero - pos,
+ session->addr)) {
+ err = g_error_new(rspamd_milter_quark(), EINVAL,
+ "invalid connect command (bad IPv4)");
+ rspamd_milter_on_protocol_error(session, priv,
+ err);
+
+ return FALSE;
+ }
+
+ rspamd_inet_address_set_port(session->addr, port);
+ break;
+
+ case RSPAMD_MILTER_CONN_INET6:
+ session->addr = rspamd_inet_address_new(AF_INET6, NULL);
+
+ if (zero - pos > sizeof("IPv6:") &&
+ rspamd_lc_cmp(pos, "IPv6:",
+ sizeof("IPv6:") - 1) == 0) {
+ /* Kill sendmail please */
+ pos += sizeof("IPv6:") - 1;
+
+ if (*pos != '[') {
+ /* Add explicit braces */
+ r = rspamd_snprintf(ip6_str, sizeof(ip6_str),
+ "[%*s]", (int) (zero - pos), pos);
+ }
+ else {
+ r = rspamd_strlcpy(ip6_str, pos, sizeof(ip6_str));
+ }
+ }
+ else {
+ r = rspamd_strlcpy(ip6_str, pos, sizeof(ip6_str));
+ }
+
+ if (!rspamd_parse_inet_address_ip(ip6_str, r,
+ session->addr)) {
+ err = g_error_new(rspamd_milter_quark(), EINVAL,
+ "invalid connect command (bad IPv6)");
+ rspamd_milter_on_protocol_error(session, priv,
+ err);
+
+ return FALSE;
+ }
+
+ rspamd_inet_address_set_port(session->addr, port);
+ break;
+
+ default:
+ err = g_error_new(rspamd_milter_quark(), EINVAL,
+ "invalid connect command (bad protocol: %c)",
+ proto);
+ rspamd_milter_on_protocol_error(session, priv,
+ err);
+
+ return FALSE;
+ }
+ }
+ }
+
+ msg_info_milter("got connection from %s",
+ rspamd_inet_address_to_string_pretty(session->addr));
+ }
+ break;
+ case RSPAMD_MILTER_CMD_MACRO:
+ msg_debug_milter("got macro command");
+ /*
+ * Format is
+ * 1 byte - command associated (we don't care about it)
+ * 0-terminated name
+ * 0-terminated value
+ * ...
+ */
+ if (session->macros == NULL) {
+ session->macros = g_hash_table_new_full(rspamd_ftok_icase_hash,
+ rspamd_ftok_icase_equal,
+ rspamd_fstring_mapped_ftok_free,
+ rspamd_fstring_mapped_ftok_free);
+ }
+
+ /* Ignore one byte */
+ pos++;
+
+ while (pos < end) {
+ zero = memchr(pos, '\0', cmdlen);
+
+ if (zero == NULL || zero >= end) {
+ err = g_error_new(rspamd_milter_quark(), EINVAL, "invalid "
+ "macro command (no name)");
+ rspamd_milter_on_protocol_error(session, priv, err);
+
+ return FALSE;
+ }
+ else {
+ rspamd_fstring_t *name, *value;
+ rspamd_ftok_t *name_tok, *value_tok;
+ const guchar *zero_val;
+
+ zero_val = memchr(zero + 1, '\0', end - zero - 1);
+
+ if (zero_val != NULL && end > zero_val) {
+ name = rspamd_fstring_new_init(pos, zero - pos);
+ value = rspamd_fstring_new_init(zero + 1,
+ zero_val - zero - 1);
+ name_tok = rspamd_ftok_map(name);
+ value_tok = rspamd_ftok_map(value);
+
+ g_hash_table_replace(session->macros, name_tok, value_tok);
+ msg_debug_milter("got macro: %T -> %T",
+ name_tok, value_tok);
+
+ cmdlen -= zero_val - pos;
+ pos = zero_val + 1;
+ }
+ else {
+ err = g_error_new(rspamd_milter_quark(), EINVAL,
+ "invalid macro command (bad value)");
+ rspamd_milter_on_protocol_error(session, priv, err);
+
+ return FALSE;
+ }
+ }
+ }
+ break;
+ case RSPAMD_MILTER_CMD_BODYEOB:
+ msg_debug_milter("got eob command");
+ REF_RETAIN(session);
+ priv->fin_cb(priv->fd, session, priv->ud);
+ REF_RELEASE(session);
+ break;
+ case RSPAMD_MILTER_CMD_HELO:
+ msg_debug_milter("got helo command");
+
+ if (end > pos && *(end - 1) == '\0') {
+ if (session->helo == NULL) {
+ session->helo = rspamd_fstring_new_init(pos, cmdlen - 1);
+ }
+ else {
+ session->helo = rspamd_fstring_assign(session->helo,
+ pos, cmdlen - 1);
+ }
+ }
+ else if (end > pos) {
+ /* Should not happen */
+ if (session->helo == NULL) {
+ session->helo = rspamd_fstring_new_init(pos, cmdlen);
+ }
+ else {
+ session->helo = rspamd_fstring_assign(session->helo,
+ pos, cmdlen);
+ }
+ }
+
+ msg_debug_milter("got helo value: %V", session->helo);
+
+ break;
+ case RSPAMD_MILTER_CMD_QUIT_NC:
+ /* We need to reset session and start over */
+ msg_debug_milter("got quit_nc command");
+ rspamd_milter_session_reset(session, RSPAMD_MILTER_RESET_QUIT_NC);
+ break;
+ case RSPAMD_MILTER_CMD_HEADER:
+ msg_debug_milter("got header command");
+ if (!session->message) {
+ session->message = rspamd_fstring_sized_new(
+ RSPAMD_MILTER_MESSAGE_CHUNK);
+ }
+ zero = memchr(pos, '\0', cmdlen);
+
+ if (zero == NULL) {
+ err = g_error_new(rspamd_milter_quark(), EINVAL, "invalid "
+ "header command (no name)");
+ rspamd_milter_on_protocol_error(session, priv, err);
+
+ return FALSE;
+ }
+ else {
+ if (end > zero && *(end - 1) == '\0') {
+ khiter_t k;
+ gint res;
+
+ k = kh_get(milter_headers_hash_t, priv->headers, (gchar *) pos);
+
+ if (k == kh_end(priv->headers)) {
+ GArray *ar;
+
+ k = kh_put(milter_headers_hash_t, priv->headers,
+ g_strdup(pos), &res);
+ ar = g_array_new(FALSE, FALSE, sizeof(gint));
+ g_array_append_val(ar, priv->cur_hdr);
+ kh_value(priv->headers, k) = ar;
+ }
+ else {
+ g_array_append_val(kh_value(priv->headers, k),
+ priv->cur_hdr);
+ }
+
+ rspamd_printf_fstring(&session->message, "%*s: %*s\r\n",
+ (int) (zero - pos), pos,
+ (int) (end - zero - 2), zero + 1);
+ priv->cur_hdr++;
+ }
+ else {
+ err = g_error_new(rspamd_milter_quark(), EINVAL, "invalid "
+ "header command (bad value)");
+ rspamd_milter_on_protocol_error(session, priv, err);
+
+ return FALSE;
+ }
+ }
+ break;
+ case RSPAMD_MILTER_CMD_MAIL:
+ msg_debug_milter("mail command");
+
+ while (pos < end) {
+ struct rspamd_email_address *addr;
+ gchar *cpy;
+
+ zero = memchr(pos, '\0', end - pos);
+
+ if (zero && zero > pos) {
+ cpy = rspamd_mempool_alloc(priv->pool, zero - pos);
+ memcpy(cpy, pos, zero - pos);
+ msg_debug_milter("got mail: %*s", (int) (zero - pos), cpy);
+ addr = rspamd_email_address_from_smtp(cpy, zero - pos);
+
+ if (addr) {
+ session->from = addr;
+ }
+
+ /* TODO: parse esmtp arguments */
+ break;
+ }
+ else {
+ msg_debug_milter("got weird from: %*s", (int) (end - pos),
+ pos);
+ /* That actually should not happen */
+ cpy = rspamd_mempool_alloc(priv->pool, end - pos);
+ memcpy(cpy, pos, end - pos);
+ addr = rspamd_email_address_from_smtp(cpy, end - pos);
+
+ if (addr) {
+ session->from = addr;
+ }
+
+ break;
+ }
+ }
+ break;
+ case RSPAMD_MILTER_CMD_EOH:
+ msg_debug_milter("got eoh command");
+
+ if (!session->message) {
+ session->message = rspamd_fstring_sized_new(
+ RSPAMD_MILTER_MESSAGE_CHUNK);
+ }
+
+ session->message = rspamd_fstring_append(session->message,
+ "\r\n", 2);
+ break;
+ case RSPAMD_MILTER_CMD_OPTNEG:
+ if (cmdlen != sizeof(guint32) * 3) {
+ err = g_error_new(rspamd_milter_quark(), EINVAL, "invalid "
+ "optneg command");
+ rspamd_milter_on_protocol_error(session, priv, err);
+
+ return FALSE;
+ }
+
+ READ_INT_32(pos, version);
+ READ_INT_32(pos, actions);
+ READ_INT_32(pos, protocol);
+
+ msg_debug_milter("optneg: version: %d, actions: %d, protocol: %d",
+ version, actions, protocol);
+
+ if (version < RSPAMD_MILTER_PROTO_VER) {
+ msg_warn_milter("MTA specifies too old protocol: %d, "
+ "aborting connection",
+ version);
+
+ err = g_error_new(rspamd_milter_quark(), EINVAL, "invalid "
+ "protocol version: %d",
+ version);
+ rspamd_milter_on_protocol_error(session, priv, err);
+
+ return FALSE;
+ }
+
+ version = RSPAMD_MILTER_PROTO_VER;
+ actions |= RSPAMD_MILTER_ACTIONS_MASK;
+ protocol = RSPAMD_MILTER_FLAG_NOREPLY_MASK;
+
+ return rspamd_milter_send_action(session, RSPAMD_MILTER_OPTNEG,
+ version, actions, protocol);
+ break;
+ case RSPAMD_MILTER_CMD_QUIT:
+ if (priv->out_chain) {
+ msg_debug_milter("quit command, refcount: %d, "
+ "some output buffers left - draining",
+ session->ref.refcount);
+
+ priv->state = RSPAMD_MILTER_WRITE_AND_DIE;
+ }
+ else {
+ msg_debug_milter("quit command, refcount: %d",
+ session->ref.refcount);
+
+ priv->state = RSPAMD_MILTER_WANNA_DIE;
+ REF_RETAIN(session);
+ priv->fin_cb(priv->fd, session, priv->ud);
+ REF_RELEASE(session);
+ return FALSE;
+ }
+ break;
+ case RSPAMD_MILTER_CMD_RCPT:
+ msg_debug_milter("rcpt command");
+
+ while (pos < end) {
+ struct rspamd_email_address *addr;
+ gchar *cpy;
+
+ zero = memchr(pos, '\0', end - pos);
+
+ if (zero && zero > pos) {
+ cpy = rspamd_mempool_alloc(priv->pool, end - pos);
+ memcpy(cpy, pos, end - pos);
+
+ msg_debug_milter("got rcpt: %*s", (int) (zero - pos), cpy);
+ addr = rspamd_email_address_from_smtp(cpy, zero - pos);
+
+ if (addr) {
+ if (!session->rcpts) {
+ session->rcpts = g_ptr_array_sized_new(1);
+ }
+
+ g_ptr_array_add(session->rcpts, addr);
+ }
+
+ pos = zero + 1;
+ }
+ else {
+ cpy = rspamd_mempool_alloc(priv->pool, end - pos);
+ memcpy(cpy, pos, end - pos);
+
+ msg_debug_milter("got weird rcpt: %*s", (int) (end - pos),
+ pos);
+ /* That actually should not happen */
+ addr = rspamd_email_address_from_smtp(cpy, end - pos);
+
+ if (addr) {
+ if (!session->rcpts) {
+ session->rcpts = g_ptr_array_sized_new(1);
+ }
+
+ g_ptr_array_add(session->rcpts, addr);
+ }
+
+ break;
+ }
+ }
+ break;
+ case RSPAMD_MILTER_CMD_DATA:
+ if (!session->message) {
+ session->message = rspamd_fstring_sized_new(
+ RSPAMD_MILTER_MESSAGE_CHUNK);
+ }
+ msg_debug_milter("got data command");
+ /* We do not need reply as specified */
+ break;
+ default:
+ msg_debug_milter("got bad command: %c", priv->parser.cur_cmd);
+ break;
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_milter_is_valid_cmd(guchar c)
+{
+ switch (c) {
+ case RSPAMD_MILTER_CMD_ABORT:
+ case RSPAMD_MILTER_CMD_BODY:
+ case RSPAMD_MILTER_CMD_CONNECT:
+ case RSPAMD_MILTER_CMD_MACRO:
+ case RSPAMD_MILTER_CMD_BODYEOB:
+ case RSPAMD_MILTER_CMD_HELO:
+ case RSPAMD_MILTER_CMD_QUIT_NC:
+ case RSPAMD_MILTER_CMD_HEADER:
+ case RSPAMD_MILTER_CMD_MAIL:
+ case RSPAMD_MILTER_CMD_EOH:
+ case RSPAMD_MILTER_CMD_OPTNEG:
+ case RSPAMD_MILTER_CMD_QUIT:
+ case RSPAMD_MILTER_CMD_RCPT:
+ case RSPAMD_MILTER_CMD_DATA:
+ case RSPAMD_MILTER_CMD_UNKNOWN:
+ return TRUE;
+ default:
+ break;
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_milter_consume_input(struct rspamd_milter_session *session,
+ struct rspamd_milter_private *priv)
+{
+ const guchar *p, *end;
+ GError *err;
+
+ p = priv->parser.buf->str + priv->parser.pos;
+ end = priv->parser.buf->str + priv->parser.buf->len;
+
+ while (p < end) {
+ msg_debug_milter("offset: %d, state: %d",
+ (gint) (p - (const guchar *) priv->parser.buf->str),
+ priv->parser.state);
+
+ switch (priv->parser.state) {
+ case st_len_1:
+ /* The first length byte in big endian order */
+ priv->parser.datalen = 0;
+ priv->parser.datalen |= ((gsize) *p) << 24;
+ priv->parser.state = st_len_2;
+ p++;
+ break;
+ case st_len_2:
+ /* The second length byte in big endian order */
+ priv->parser.datalen |= ((gsize) *p) << 16;
+ priv->parser.state = st_len_3;
+ p++;
+ break;
+ case st_len_3:
+ /* The third length byte in big endian order */
+ priv->parser.datalen |= ((gsize) *p) << 8;
+ priv->parser.state = st_len_4;
+ p++;
+ break;
+ case st_len_4:
+ /* The fourth length byte in big endian order */
+ priv->parser.datalen |= ((gsize) *p);
+ priv->parser.state = st_read_cmd;
+ p++;
+ break;
+ case st_read_cmd:
+ priv->parser.cur_cmd = *p;
+ priv->parser.state = st_read_data;
+
+ if (priv->parser.datalen < 1) {
+ err = g_error_new(rspamd_milter_quark(), EINVAL,
+ "Command length is too short");
+ rspamd_milter_on_protocol_error(session, priv, err);
+
+ return FALSE;
+ }
+ else {
+ /* Eat command itself */
+ priv->parser.datalen--;
+ }
+
+ p++;
+ priv->parser.cmd_start = p - (const guchar *) priv->parser.buf->str;
+ break;
+ case st_read_data:
+ /* We might need some more data in buffer for further steps */
+ if (priv->parser.datalen >
+ RSPAMD_MILTER_MESSAGE_CHUNK * 2) {
+ /* Check if we have HTTP input instead of milter */
+ if (priv->parser.buf->len > sizeof("GET") &&
+ memcmp(priv->parser.buf->str, "GET", 3) == 0) {
+ struct http_parser http_parser;
+ struct http_parser_settings http_callbacks;
+ GString *url = g_string_new(NULL);
+
+ /* Hack, hack, hack */
+ /*
+ * This code is assumed to read `/ping` command and
+ * handle it to monitor port's availability since
+ * milter protocol is stupid and does not allow to do that
+ * This code also assumes that HTTP request can be read
+ * as as single data chunk which is not true in some cases
+ * In general, don't use it for anything but ping checks
+ */
+ memset(&http_callbacks, 0, sizeof(http_callbacks));
+ http_parser.data = url;
+ http_parser_init(&http_parser, HTTP_REQUEST);
+ http_callbacks.on_url = rspamd_milter_http_on_url;
+ http_parser_execute(&http_parser, &http_callbacks,
+ priv->parser.buf->str, priv->parser.buf->len);
+
+ if (url->len == sizeof("/ping") - 1 &&
+ rspamd_lc_cmp(url->str, "/ping", url->len) == 0) {
+ rspamd_milter_on_protocol_ping(session, priv);
+ g_string_free(url, TRUE);
+
+ return TRUE;
+ }
+ else {
+ err = g_error_new(rspamd_milter_quark(), EINVAL,
+ "HTTP GET request is not supported in milter mode, url: %s",
+ url->str);
+ }
+
+ g_string_free(url, TRUE);
+ }
+ else if (priv->parser.buf->len > sizeof("POST") &&
+ memcmp(priv->parser.buf->str, "POST", 4) == 0) {
+ err = g_error_new(rspamd_milter_quark(), EINVAL,
+ "HTTP POST request is not supported in milter mode");
+ }
+ else {
+ err = g_error_new(rspamd_milter_quark(), E2BIG,
+ "Command length is too big: %zd",
+ priv->parser.datalen);
+ }
+
+ rspamd_milter_on_protocol_error(session, priv, err);
+
+ return FALSE;
+ }
+ if (!rspamd_milter_is_valid_cmd(priv->parser.cur_cmd)) {
+ err = g_error_new(rspamd_milter_quark(), E2BIG,
+ "Unvalid command: %c",
+ priv->parser.cur_cmd);
+ rspamd_milter_on_protocol_error(session, priv, err);
+
+ return FALSE;
+ }
+ if (priv->parser.buf->allocated < priv->parser.datalen) {
+ priv->parser.pos = p - (const guchar *) priv->parser.buf->str;
+ priv->parser.buf = rspamd_fstring_grow(priv->parser.buf,
+ priv->parser.buf->len + priv->parser.datalen);
+ /* This can realloc buffer */
+ rspamd_milter_plan_io(session, priv, EV_READ);
+ goto end;
+ }
+ else {
+ /* We may have the full command available */
+ if (p + priv->parser.datalen <= end) {
+ /* We can process command */
+ if (!rspamd_milter_process_command(session, priv)) {
+ return FALSE;
+ }
+
+ p += priv->parser.datalen;
+ priv->parser.state = st_len_1;
+ priv->parser.cur_cmd = '\0';
+ priv->parser.cmd_start = 0;
+ }
+ else {
+ /* Need to read more */
+ priv->parser.pos = p - (const guchar *) priv->parser.buf->str;
+ rspamd_milter_plan_io(session, priv, EV_READ);
+ goto end;
+ }
+ }
+ break;
+ }
+ }
+
+ /* Leftover */
+ switch (priv->parser.state) {
+ case st_read_data:
+ if (p + priv->parser.datalen <= end) {
+ if (!rspamd_milter_process_command(session, priv)) {
+ return FALSE;
+ }
+
+ priv->parser.state = st_len_1;
+ priv->parser.cur_cmd = '\0';
+ priv->parser.cmd_start = 0;
+ }
+ break;
+ default:
+ /* No need to do anything */
+ break;
+ }
+
+ if (p == end) {
+ priv->parser.buf->len = 0;
+ priv->parser.pos = 0;
+ priv->parser.cmd_start = 0;
+ }
+
+ if (priv->out_chain) {
+ rspamd_milter_plan_io(session, priv, EV_READ | EV_WRITE);
+ }
+ else {
+ rspamd_milter_plan_io(session, priv, EV_READ);
+ }
+end:
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_milter_handle_session(struct rspamd_milter_session *session,
+ struct rspamd_milter_private *priv)
+{
+ struct rspamd_milter_outbuf *obuf, *obuf_tmp;
+ gssize r, to_write;
+ GError *err;
+
+ g_assert(session != NULL);
+
+ switch (priv->state) {
+ case RSPAMD_MILTER_READ_MORE:
+ if (priv->parser.buf->len >= priv->parser.buf->allocated) {
+ priv->parser.buf = rspamd_fstring_grow(priv->parser.buf,
+ priv->parser.buf->len * 2);
+ }
+
+ r = read(priv->fd, priv->parser.buf->str + priv->parser.buf->len,
+ priv->parser.buf->allocated - priv->parser.buf->len);
+
+ msg_debug_milter("read %z bytes, %z remain, %z allocated",
+ r, priv->parser.buf->len, priv->parser.buf->allocated);
+
+ if (r == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ rspamd_milter_plan_io(session, priv, EV_READ);
+
+ return TRUE;
+ }
+ else {
+ /* Fatal IO error */
+ err = g_error_new(rspamd_milter_quark(), errno,
+ "IO read error: %s", strerror(errno));
+ REF_RETAIN(session);
+ priv->err_cb(priv->fd, session, priv->ud, err);
+ REF_RELEASE(session);
+ g_error_free(err);
+
+ REF_RELEASE(session);
+
+ return FALSE;
+ }
+ }
+ else if (r == 0) {
+ err = g_error_new(rspamd_milter_quark(), ECONNRESET,
+ "Unexpected EOF");
+ REF_RETAIN(session);
+ priv->err_cb(priv->fd, session, priv->ud, err);
+ REF_RELEASE(session);
+ g_error_free(err);
+
+ REF_RELEASE(session);
+
+ return FALSE;
+ }
+ else {
+ priv->parser.buf->len += r;
+
+ return rspamd_milter_consume_input(session, priv);
+ }
+
+ break;
+ case RSPAMD_MILTER_WRITE_REPLY:
+ case RSPAMD_MILTER_WRITE_AND_DIE:
+ if (priv->out_chain == NULL) {
+ if (priv->state == RSPAMD_MILTER_WRITE_AND_DIE) {
+ /* Finished writing, let's die finally */
+ msg_debug_milter("output drained, terminating, refcount: %d",
+ session->ref.refcount);
+
+ /* Session should be destroyed by fin_cb... */
+ REF_RETAIN(session);
+ priv->fin_cb(priv->fd, session, priv->ud);
+ REF_RELEASE(session);
+
+ return FALSE;
+ }
+ else {
+ /* We have written everything, so we can read something */
+ priv->state = RSPAMD_MILTER_READ_MORE;
+ rspamd_milter_plan_io(session, priv, EV_READ);
+ }
+ }
+ else {
+ DL_FOREACH_SAFE(priv->out_chain, obuf, obuf_tmp)
+ {
+ to_write = obuf->buf->len - obuf->pos;
+
+ g_assert(to_write > 0);
+
+ r = write(priv->fd, obuf->buf->str + obuf->pos, to_write);
+
+ if (r == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ rspamd_milter_plan_io(session, priv, EV_WRITE);
+ }
+ else {
+ /* Fatal IO error */
+ err = g_error_new(rspamd_milter_quark(), errno,
+ "IO write error: %s", strerror(errno));
+ REF_RETAIN(session);
+ priv->err_cb(priv->fd, session, priv->ud, err);
+ REF_RELEASE(session);
+ g_error_free(err);
+
+ REF_RELEASE(session);
+
+ return FALSE;
+ }
+ }
+ else if (r == 0) {
+ err = g_error_new(rspamd_milter_quark(), ECONNRESET,
+ "Unexpected EOF");
+ REF_RETAIN(session);
+ priv->err_cb(priv->fd, session, priv->ud, err);
+ REF_RELEASE(session);
+ g_error_free(err);
+
+ REF_RELEASE(session);
+
+ return FALSE;
+ }
+ else {
+ if (r == to_write) {
+ /* We have done with this buf */
+ DL_DELETE(priv->out_chain, obuf);
+ rspamd_milter_obuf_free(obuf);
+ }
+ else {
+ /* We need to plan another write */
+ obuf->pos += r;
+ rspamd_milter_plan_io(session, priv, EV_WRITE);
+
+ return TRUE;
+ }
+ }
+ }
+
+ /* Here we have written everything, so we can plan reading */
+ priv->state = RSPAMD_MILTER_READ_MORE;
+ rspamd_milter_plan_io(session, priv, EV_READ);
+ }
+ break;
+ case RSPAMD_MILTER_WANNA_DIE:
+ /* We are here after processing everything, so release session */
+ REF_RELEASE(session);
+ return FALSE;
+ break;
+ case RSPAMD_MILTER_PONG_AND_DIE:
+ err = g_error_new(rspamd_milter_quark(), 0,
+ "ping command");
+ REF_RETAIN(session);
+ priv->err_cb(priv->fd, session, priv->ud, err);
+ REF_RELEASE(session);
+ g_error_free(err);
+ REF_RELEASE(session);
+ return FALSE;
+ break;
+ }
+
+ return TRUE;
+}
+
+
+gboolean
+rspamd_milter_handle_socket(gint fd, ev_tstamp timeout,
+ rspamd_mempool_t *pool,
+ struct ev_loop *ev_base, rspamd_milter_finish finish_cb,
+ rspamd_milter_error error_cb, void *ud)
+{
+ struct rspamd_milter_session *session;
+ struct rspamd_milter_private *priv;
+ gint nfd = dup(fd);
+
+ if (nfd == -1) {
+ GError *err = g_error_new(rspamd_milter_quark(), errno,
+ "dup failed: %s", strerror(errno));
+ error_cb(fd, NULL, ud, err);
+
+ return FALSE;
+ }
+
+ g_assert(finish_cb != NULL);
+ g_assert(error_cb != NULL);
+ g_assert(milter_ctx != NULL);
+
+ session = g_malloc0(sizeof(*session));
+ priv = g_malloc0(sizeof(*priv));
+ priv->fd = nfd;
+ priv->ud = ud;
+ priv->fin_cb = finish_cb;
+ priv->err_cb = error_cb;
+ priv->parser.state = st_len_1;
+ priv->parser.buf = rspamd_fstring_sized_new(RSPAMD_MILTER_MESSAGE_CHUNK + 5);
+ priv->event_loop = ev_base;
+ priv->state = RSPAMD_MILTER_READ_MORE;
+ priv->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), "milter", 0);
+ priv->discard_on_reject = milter_ctx->discard_on_reject;
+ priv->quarantine_on_reject = milter_ctx->quarantine_on_reject;
+ priv->ev.timeout = timeout;
+
+ rspamd_ev_watcher_init(&priv->ev, priv->fd, EV_READ | EV_WRITE,
+ rspamd_milter_io_handler, session);
+
+ if (pool) {
+ /* Copy tag */
+ memcpy(priv->pool->tag.uid, pool->tag.uid, sizeof(pool->tag.uid));
+ }
+
+ priv->headers = kh_init(milter_headers_hash_t);
+ kh_resize(milter_headers_hash_t, priv->headers, 32);
+
+ session->priv = priv;
+ REF_INIT_RETAIN(session, rspamd_milter_session_dtor);
+
+ if (milter_ctx->sessions_cache) {
+ rspamd_worker_session_cache_add(milter_ctx->sessions_cache,
+ priv->pool->tag.uid, &session->ref.refcount, session);
+ }
+
+ return rspamd_milter_handle_session(session, priv);
+}
+
+gboolean
+rspamd_milter_set_reply(struct rspamd_milter_session *session,
+ rspamd_fstring_t *rcode,
+ rspamd_fstring_t *xcode,
+ rspamd_fstring_t *reply)
+{
+ GString *buf;
+ gboolean ret;
+
+ buf = g_string_sized_new(xcode->len + rcode->len + reply->len + 2);
+ rspamd_printf_gstring(buf, "%V %V %V", rcode, xcode, reply);
+ ret = rspamd_milter_send_action(session, RSPAMD_MILTER_REPLYCODE,
+ buf);
+ g_string_free(buf, TRUE);
+
+ return ret;
+}
+
+#define SET_COMMAND(cmd, sz, reply, pos) \
+ do { \
+ guint32 _len; \
+ _len = (sz) + 1; \
+ (reply) = rspamd_fstring_sized_new(sizeof(_len) + _len); \
+ (reply)->len = sizeof(_len) + _len; \
+ _len = htonl(_len); \
+ memcpy((reply)->str, &_len, sizeof(_len)); \
+ (reply)->str[sizeof(_len)] = (cmd); \
+ (pos) = (guchar *) (reply)->str + sizeof(_len) + 1; \
+ } while (0)
+
+gboolean
+rspamd_milter_send_action(struct rspamd_milter_session *session,
+ enum rspamd_milter_reply act, ...)
+{
+ guint32 ver, actions, protocol, idx;
+ va_list ap;
+ guchar cmd, *pos;
+ rspamd_fstring_t *reply = NULL;
+ gsize len;
+ GString *name, *value;
+ const char *reason, *body_str;
+ struct rspamd_milter_outbuf *obuf;
+ struct rspamd_milter_private *priv = session->priv;
+
+ va_start(ap, act);
+ cmd = act;
+
+ switch (act) {
+ case RSPAMD_MILTER_ACCEPT:
+ case RSPAMD_MILTER_CONTINUE:
+ case RSPAMD_MILTER_DISCARD:
+ case RSPAMD_MILTER_PROGRESS:
+ case RSPAMD_MILTER_REJECT:
+ case RSPAMD_MILTER_TEMPFAIL:
+ /* No additional arguments */
+ msg_debug_milter("send %c command", cmd);
+ SET_COMMAND(cmd, 0, reply, pos);
+ break;
+ case RSPAMD_MILTER_QUARANTINE:
+ reason = va_arg(ap, const char *);
+
+ if (reason == NULL) {
+ reason = "";
+ }
+
+ len = strlen(reason);
+ msg_debug_milter("send quarantine action %s", reason);
+ SET_COMMAND(cmd, len + 1, reply, pos);
+ memcpy(pos, reason, len + 1);
+ break;
+ case RSPAMD_MILTER_ADDHEADER:
+ name = va_arg(ap, GString *);
+ value = va_arg(ap, GString *);
+
+ /* Name and value must be zero terminated */
+ msg_debug_milter("add header command - \"%v\"=\"%v\"", name, value);
+ SET_COMMAND(cmd, name->len + value->len + 2, reply, pos);
+ memcpy(pos, name->str, name->len + 1);
+ pos += name->len + 1;
+ memcpy(pos, value->str, value->len + 1);
+ break;
+ case RSPAMD_MILTER_CHGHEADER:
+ case RSPAMD_MILTER_INSHEADER:
+ idx = va_arg(ap, guint32);
+ name = va_arg(ap, GString *);
+ value = va_arg(ap, GString *);
+
+ msg_debug_milter("change/insert header command pos = %d- \"%v\"=\"%v\"",
+ idx, name, value);
+ /* Name and value must be zero terminated */
+ SET_COMMAND(cmd, name->len + value->len + 2 + sizeof(guint32),
+ reply, pos);
+ idx = htonl(idx);
+ memcpy(pos, &idx, sizeof(idx));
+ pos += sizeof(idx);
+ memcpy(pos, name->str, name->len + 1);
+ pos += name->len + 1;
+ memcpy(pos, value->str, value->len + 1);
+ break;
+ case RSPAMD_MILTER_REPLBODY:
+ len = va_arg(ap, gsize);
+ body_str = va_arg(ap, const char *);
+ msg_debug_milter("want to change body; size = %uz",
+ len);
+ SET_COMMAND(cmd, len, reply, pos);
+ memcpy(pos, body_str, len);
+ break;
+ case RSPAMD_MILTER_REPLYCODE:
+ case RSPAMD_MILTER_ADDRCPT:
+ case RSPAMD_MILTER_DELRCPT:
+ case RSPAMD_MILTER_CHGFROM:
+ /* Single GString * argument */
+ value = va_arg(ap, GString *);
+ msg_debug_milter("command %c; value=%v", cmd, value);
+ SET_COMMAND(cmd, value->len + 1, reply, pos);
+ memcpy(pos, value->str, value->len + 1);
+ break;
+ case RSPAMD_MILTER_OPTNEG:
+ ver = va_arg(ap, guint32);
+ actions = va_arg(ap, guint32);
+ protocol = va_arg(ap, guint32);
+
+ msg_debug_milter("optneg reply: ver=%d, actions=%d, protocol=%d",
+ ver, actions, protocol);
+ ver = htonl(ver);
+ actions = htonl(actions);
+ protocol = htonl(protocol);
+ SET_COMMAND(cmd, sizeof(guint32) * 3, reply, pos);
+ memcpy(pos, &ver, sizeof(ver));
+ pos += sizeof(ver);
+ memcpy(pos, &actions, sizeof(actions));
+ pos += sizeof(actions);
+ memcpy(pos, &protocol, sizeof(protocol));
+ break;
+ default:
+ msg_err_milter("invalid command: %c", cmd);
+ break;
+ }
+
+ va_end(ap);
+
+ if (reply) {
+ obuf = g_malloc(sizeof(*obuf));
+ obuf->buf = reply;
+ obuf->pos = 0;
+ DL_APPEND(priv->out_chain, obuf);
+ priv->state = RSPAMD_MILTER_WRITE_REPLY;
+ rspamd_milter_plan_io(session, priv, EV_WRITE);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_milter_add_header(struct rspamd_milter_session *session,
+ GString *name, GString *value)
+{
+ return rspamd_milter_send_action(session, RSPAMD_MILTER_ADDHEADER,
+ name, value);
+}
+
+gboolean
+rspamd_milter_del_header(struct rspamd_milter_session *session,
+ GString *name)
+{
+ GString value;
+ guint32 idx = 1;
+
+ value.str = (gchar *) "";
+ value.len = 0;
+
+ return rspamd_milter_send_action(session, RSPAMD_MILTER_CHGHEADER,
+ idx, name, &value);
+}
+
+void rspamd_milter_session_unref(struct rspamd_milter_session *session)
+{
+ REF_RELEASE(session);
+}
+
+struct rspamd_milter_session *
+rspamd_milter_session_ref(struct rspamd_milter_session *session)
+{
+ REF_RETAIN(session);
+
+ return session;
+}
+
+#define IF_MACRO(lit) \
+ RSPAMD_FTOK_ASSIGN(&srch, (lit)); \
+ found = g_hash_table_lookup(session->macros, &srch); \
+ if (found)
+
+static void
+rspamd_milter_macro_http(struct rspamd_milter_session *session,
+ struct rspamd_http_message *msg)
+{
+ rspamd_ftok_t *found, srch;
+ struct rspamd_milter_private *priv = session->priv;
+
+ /*
+ * We assume postfix macros here, sendmail ones might be slightly
+ * different
+ */
+
+ if (!session->macros) {
+ return;
+ }
+
+ IF_MACRO("{i}")
+ {
+ rspamd_http_message_add_header_len(msg, QUEUE_ID_HEADER,
+ found->begin, found->len);
+ }
+ else
+ {
+ IF_MACRO("i")
+ {
+ rspamd_http_message_add_header_len(msg, QUEUE_ID_HEADER,
+ found->begin, found->len);
+ }
+ }
+
+ IF_MACRO("{v}")
+ {
+ rspamd_http_message_add_header_len(msg, USER_AGENT_HEADER,
+ found->begin, found->len);
+ }
+ else
+ {
+ IF_MACRO("v")
+ {
+ rspamd_http_message_add_header_len(msg, USER_AGENT_HEADER,
+ found->begin, found->len);
+ }
+ }
+
+ IF_MACRO("{cipher}")
+ {
+ rspamd_http_message_add_header_len(msg, TLS_CIPHER_HEADER,
+ found->begin, found->len);
+ }
+
+ IF_MACRO("{tls_version}")
+ {
+ rspamd_http_message_add_header_len(msg, TLS_VERSION_HEADER,
+ found->begin, found->len);
+ }
+
+ IF_MACRO("{auth_authen}")
+ {
+ rspamd_http_message_add_header_len(msg, USER_HEADER,
+ found->begin, found->len);
+ }
+
+ IF_MACRO("{rcpt_mailer}")
+ {
+ rspamd_http_message_add_header_len(msg, MAILER_HEADER,
+ found->begin, found->len);
+ }
+
+ if (milter_ctx->client_ca_name) {
+ IF_MACRO("{cert_issuer}")
+ {
+ rspamd_http_message_add_header_len(msg, CERT_ISSUER_HEADER,
+ found->begin, found->len);
+
+ if (found->len == strlen(milter_ctx->client_ca_name) &&
+ rspamd_cryptobox_memcmp(found->begin,
+ milter_ctx->client_ca_name, found->len) == 0) {
+ msg_debug_milter("process certificate issued by %T", found);
+ IF_MACRO("{cert_subject}")
+ {
+ rspamd_http_message_add_header_len(msg, USER_HEADER,
+ found->begin, found->len);
+ }
+ }
+ else {
+ msg_debug_milter("skip certificate issued by %T", found);
+ }
+ }
+ }
+ else {
+ IF_MACRO("{cert_issuer}")
+ {
+ rspamd_http_message_add_header_len(msg, CERT_ISSUER_HEADER,
+ found->begin, found->len);
+ }
+ }
+
+ if (!session->hostname || session->hostname->len == 0) {
+ IF_MACRO("{client_name}")
+ {
+ if (!(found->len == sizeof("unknown") - 1 &&
+ memcmp(found->begin, "unknown",
+ sizeof("unknown") - 1) == 0)) {
+ rspamd_http_message_add_header_len(msg, HOSTNAME_HEADER,
+ found->begin, found->len);
+ }
+ else {
+ msg_debug_milter("skip unknown hostname from being added");
+ }
+ }
+ }
+
+ IF_MACRO("{daemon_name}")
+ {
+ /* Postfix style */
+ rspamd_http_message_add_header_len(msg, MTA_NAME_HEADER,
+ found->begin, found->len);
+ }
+ else
+ {
+ /* Sendmail style */
+ IF_MACRO("{j}")
+ {
+ rspamd_http_message_add_header_len(msg, MTA_NAME_HEADER,
+ found->begin, found->len);
+ }
+ else
+ {
+ IF_MACRO("j")
+ {
+ rspamd_http_message_add_header_len(msg, MTA_NAME_HEADER,
+ found->begin, found->len);
+ }
+ }
+ }
+}
+
+struct rspamd_http_message *
+rspamd_milter_to_http(struct rspamd_milter_session *session)
+{
+ struct rspamd_http_message *msg;
+ guint i;
+ struct rspamd_email_address *rcpt;
+ struct rspamd_milter_private *priv = session->priv;
+
+ g_assert(session != NULL);
+
+ msg = rspamd_http_new_message(HTTP_REQUEST);
+
+ msg->url = rspamd_fstring_assign(msg->url, "/" MSG_CMD_CHECK_V2,
+ sizeof("/" MSG_CMD_CHECK_V2) - 1);
+
+ if (session->message) {
+ rspamd_http_message_set_body_from_fstring_steal(msg, session->message);
+ session->message = NULL;
+ }
+
+ if (session->hostname && RSPAMD_FSTRING_LEN(session->hostname) > 0) {
+ if (!(session->hostname->len == sizeof("unknown") - 1 &&
+ memcmp(RSPAMD_FSTRING_DATA(session->hostname), "unknown",
+ sizeof("unknown") - 1) == 0)) {
+ rspamd_http_message_add_header_fstr(msg, HOSTNAME_HEADER,
+ session->hostname);
+ }
+ else {
+ msg_debug_milter("skip unknown hostname from being added");
+ }
+ }
+
+ if (session->helo && session->helo->len > 0) {
+ rspamd_http_message_add_header_fstr(msg, HELO_HEADER,
+ session->helo);
+ }
+
+ if (session->from) {
+ rspamd_http_message_add_header_len(msg, FROM_HEADER,
+ session->from->raw, session->from->raw_len);
+ }
+
+ if (session->rcpts) {
+ PTR_ARRAY_FOREACH(session->rcpts, i, rcpt)
+ {
+ rspamd_http_message_add_header_len(msg, RCPT_HEADER,
+ rcpt->raw, rcpt->raw_len);
+ }
+ }
+
+ if (session->addr) {
+ if (rspamd_inet_address_get_af(session->addr) != AF_UNIX) {
+ rspamd_http_message_add_header(msg, IP_ADDR_HEADER,
+ rspamd_inet_address_to_string_pretty(session->addr));
+ }
+ else {
+ rspamd_http_message_add_header(msg, IP_ADDR_HEADER,
+ rspamd_inet_address_to_string(session->addr));
+ }
+ }
+
+ rspamd_milter_macro_http(session, msg);
+ rspamd_http_message_add_header(msg, FLAGS_HEADER, "milter,body_block");
+
+ return msg;
+}
+
+void *
+rspamd_milter_update_userdata(struct rspamd_milter_session *session,
+ void *ud)
+{
+ struct rspamd_milter_private *priv = session->priv;
+ void *prev_ud;
+
+ prev_ud = priv->ud;
+ priv->ud = ud;
+
+ return prev_ud;
+}
+
+static void
+rspamd_milter_remove_header_safe(struct rspamd_milter_session *session,
+ const gchar *key, gint nhdr)
+{
+ gint i;
+ GString *hname, *hvalue;
+ struct rspamd_milter_private *priv = session->priv;
+ khiter_t k;
+ GArray *ar;
+
+ k = kh_get(milter_headers_hash_t, priv->headers, (char *) key);
+
+ if (k != kh_end(priv->headers)) {
+ ar = kh_val(priv->headers, k);
+
+ hname = g_string_new(key);
+ hvalue = g_string_new("");
+
+ if (nhdr > 0) {
+ if (ar->len >= nhdr) {
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_CHGHEADER,
+ nhdr, hname, hvalue);
+ priv->cur_hdr--;
+ }
+ }
+ else if (nhdr == 0) {
+ /* We need to clear all headers */
+ for (i = ar->len; i > 0; i--) {
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_CHGHEADER,
+ i, hname, hvalue);
+ priv->cur_hdr--;
+ }
+ }
+ else {
+ /* Remove from the end */
+ if (nhdr >= -(ar->len)) {
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_CHGHEADER,
+ ar->len + nhdr + 1, hname, hvalue);
+ priv->cur_hdr--;
+ }
+ }
+
+ g_string_free(hname, TRUE);
+ g_string_free(hvalue, TRUE);
+
+ if (priv->cur_hdr < 0) {
+ msg_err_milter("negative header count after removing %s", key);
+ priv->cur_hdr = 0;
+ }
+ }
+}
+
+static void
+rspamd_milter_extract_single_header(struct rspamd_milter_session *session,
+ const gchar *hdr, const ucl_object_t *obj)
+{
+ GString *hname, *hvalue;
+ struct rspamd_milter_private *priv = session->priv;
+ gint idx = -1;
+ const ucl_object_t *val;
+
+ val = ucl_object_lookup(obj, "value");
+
+ if (val && ucl_object_type(val) == UCL_STRING) {
+ const ucl_object_t *idx_obj;
+ gboolean has_idx = FALSE;
+
+ idx_obj = ucl_object_lookup_any(obj, "order",
+ "index", NULL);
+
+ if (idx_obj && (ucl_object_type(idx_obj) == UCL_INT || ucl_object_type(idx_obj) == UCL_FLOAT)) {
+ idx = ucl_object_toint(idx_obj);
+ has_idx = TRUE;
+ }
+
+ hname = g_string_new(hdr);
+ hvalue = g_string_new(ucl_object_tostring(val));
+
+ if (has_idx) {
+ if (idx >= 0) {
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_INSHEADER,
+ idx,
+ hname, hvalue);
+ }
+ else {
+ /* Calculate negative offset */
+
+ if (idx == -1) {
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_ADDHEADER,
+ hname, hvalue);
+ }
+ else if (-idx <= priv->cur_hdr) {
+ /*
+ * Note: We should account MTA's own "Received:" field
+ * which wasn't passed by Milter's header command.
+ */
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_INSHEADER,
+ priv->cur_hdr + idx + 2,
+ hname, hvalue);
+ }
+ else {
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_INSHEADER,
+ 0,
+ hname, hvalue);
+ }
+ }
+ }
+ else {
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_ADDHEADER,
+ hname, hvalue);
+ }
+
+ priv->cur_hdr++;
+
+ g_string_free(hname, TRUE);
+ g_string_free(hvalue, TRUE);
+ }
+}
+
+/*
+ * Returns `TRUE` if action has been processed internally by this function
+ */
+static gboolean
+rspamd_milter_process_milter_block(struct rspamd_milter_session *session,
+ const ucl_object_t *obj, struct rspamd_action *action)
+{
+ const ucl_object_t *elt, *cur;
+ ucl_object_iter_t it;
+ struct rspamd_milter_private *priv = session->priv;
+ GString *hname, *hvalue;
+
+ if (obj && ucl_object_type(obj) == UCL_OBJECT) {
+ elt = ucl_object_lookup(obj, "remove_headers");
+ /*
+ * remove_headers: {"name": 1, ... }
+ * where number is the header's position starting from '1'
+ */
+ if (elt && ucl_object_type(elt) == UCL_OBJECT) {
+ it = NULL;
+
+ while ((cur = ucl_object_iterate(elt, &it, true)) != NULL) {
+ if (ucl_object_type(cur) == UCL_INT) {
+ rspamd_milter_remove_header_safe(session,
+ ucl_object_key(cur),
+ ucl_object_toint(cur));
+ }
+ }
+ }
+
+ elt = ucl_object_lookup(obj, "add_headers");
+ /*
+ * add_headers: {"name": "value", ... }
+ * name could have multiple values
+ * -or- (since 1.7)
+ * {"name": {"value": "val", "order": 0}, ... }
+ */
+ if (elt && ucl_object_type(elt) == UCL_OBJECT) {
+ it = NULL;
+
+ while ((cur = ucl_object_iterate(elt, &it, true)) != NULL) {
+
+ const char *key_name = ucl_object_key(cur);
+
+ if (ucl_object_type(cur) == UCL_STRING) {
+ /*
+ * Legacy support of {"name": "value", ... } with
+ * multiple names under the same name
+ */
+ ucl_object_iter_t *elt_it;
+ const ucl_object_t *cur_elt;
+
+ elt_it = ucl_object_iterate_new(cur);
+ while ((cur_elt = ucl_object_iterate_safe(elt_it, false)) != NULL) {
+ if (ucl_object_type(cur_elt) == UCL_STRING) {
+ hname = g_string_new(key_name);
+ hvalue = g_string_new(ucl_object_tostring(cur_elt));
+
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_ADDHEADER,
+ hname, hvalue);
+ g_string_free(hname, TRUE);
+ g_string_free(hvalue, TRUE);
+ }
+ else {
+ msg_warn_milter("legacy header with name %s, that has not a string value: %s",
+ key_name, ucl_object_type_to_string(cur_elt->type));
+ }
+ }
+ ucl_object_iterate_free(elt_it);
+ }
+ else {
+ if (ucl_object_type(cur) == UCL_OBJECT) {
+ rspamd_milter_extract_single_header(session,
+ key_name, cur);
+ }
+ else if (ucl_object_type(cur) == UCL_ARRAY) {
+ /* Multiple values for the same key */
+ ucl_object_iter_t *array_it;
+ const ucl_object_t *array_elt;
+
+ array_it = ucl_object_iterate_new(cur);
+
+ while ((array_elt = ucl_object_iterate_safe(array_it,
+ true)) != NULL) {
+ rspamd_milter_extract_single_header(session,
+ key_name, array_elt);
+ }
+
+ ucl_object_iterate_free(array_it);
+ }
+ else {
+ msg_warn_milter("non-legacy header with name %s, that has unsupported value type: %s",
+ key_name, ucl_object_type_to_string(cur->type));
+ }
+ }
+ }
+ }
+
+ elt = ucl_object_lookup(obj, "change_from");
+
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ hvalue = g_string_new(ucl_object_tostring(elt));
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_CHGFROM,
+ hvalue);
+ g_string_free(hvalue, TRUE);
+ }
+
+ elt = ucl_object_lookup(obj, "add_rcpt");
+
+ if (elt && ucl_object_type(elt) == UCL_ARRAY) {
+ it = NULL;
+
+ while ((cur = ucl_object_iterate(elt, &it, true)) != NULL) {
+ hvalue = g_string_new(ucl_object_tostring(cur));
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_ADDRCPT,
+ hvalue);
+ g_string_free(hvalue, TRUE);
+ }
+ }
+
+ elt = ucl_object_lookup(obj, "del_rcpt");
+
+ if (elt && ucl_object_type(elt) == UCL_ARRAY) {
+ it = NULL;
+
+ while ((cur = ucl_object_iterate(elt, &it, true)) != NULL) {
+ hvalue = g_string_new(ucl_object_tostring(cur));
+ rspamd_milter_send_action(session,
+ RSPAMD_MILTER_DELRCPT,
+ hvalue);
+ g_string_free(hvalue, TRUE);
+ }
+ }
+
+ elt = ucl_object_lookup(obj, "reject");
+
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ if (strcmp(ucl_object_tostring(elt), "discard") == 0) {
+ priv->discard_on_reject = TRUE;
+ msg_info_milter("discard message instead of rejection");
+ }
+ else if (strcmp(ucl_object_tostring(elt), "quarantine") == 0) {
+ priv->quarantine_on_reject = TRUE;
+ msg_info_milter("quarantine message instead of rejection");
+ }
+ else {
+ priv->discard_on_reject = FALSE;
+ priv->quarantine_on_reject = FALSE;
+ }
+ }
+
+ elt = ucl_object_lookup(obj, "no_action");
+
+ if (elt && ucl_object_type(elt) == UCL_BOOLEAN) {
+ priv->no_action = ucl_object_toboolean(elt);
+ }
+ }
+
+ if (action->action_type == METRIC_ACTION_ADD_HEADER) {
+ elt = ucl_object_lookup(obj, "spam_header");
+
+ if (elt) {
+ if (ucl_object_type(elt) == UCL_STRING) {
+ rspamd_milter_remove_header_safe(session,
+ milter_ctx->spam_header,
+ 0);
+
+ hname = g_string_new(milter_ctx->spam_header);
+ hvalue = g_string_new(ucl_object_tostring(elt));
+ rspamd_milter_send_action(session, RSPAMD_MILTER_CHGHEADER,
+ (guint32) 1, hname, hvalue);
+ g_string_free(hname, TRUE);
+ g_string_free(hvalue, TRUE);
+ rspamd_milter_send_action(session, RSPAMD_MILTER_ACCEPT);
+
+ return TRUE;
+ }
+ else if (ucl_object_type(elt) == UCL_OBJECT) {
+ it = NULL;
+
+ while ((cur = ucl_object_iterate(elt, &it, true)) != NULL) {
+ rspamd_milter_remove_header_safe(session,
+ ucl_object_key(cur),
+ 0);
+
+ hname = g_string_new(ucl_object_key(cur));
+ hvalue = g_string_new(ucl_object_tostring(cur));
+ rspamd_milter_send_action(session, RSPAMD_MILTER_CHGHEADER,
+ (guint32) 1, hname, hvalue);
+ g_string_free(hname, TRUE);
+ g_string_free(hvalue, TRUE);
+ }
+
+ rspamd_milter_send_action(session, RSPAMD_MILTER_ACCEPT);
+
+ return TRUE;
+ }
+ }
+ }
+
+ return FALSE;
+}
+
+void rspamd_milter_send_task_results(struct rspamd_milter_session *session,
+ const ucl_object_t *results,
+ const gchar *new_body,
+ gsize bodylen)
+{
+ const ucl_object_t *elt;
+ struct rspamd_milter_private *priv = session->priv;
+ const gchar *str_action;
+ struct rspamd_action *action;
+ rspamd_fstring_t *xcode = NULL, *rcode = NULL, *reply = NULL;
+ GString *hname, *hvalue;
+ gboolean processed = FALSE;
+
+ if (results == NULL) {
+ msg_err_milter("cannot find scan results, tempfail");
+ rspamd_milter_send_action(session, RSPAMD_MILTER_TEMPFAIL);
+
+ goto cleanup;
+ }
+
+ elt = ucl_object_lookup(results, "action");
+
+ if (!elt) {
+ msg_err_milter("cannot find action in results, tempfail");
+ rspamd_milter_send_action(session, RSPAMD_MILTER_TEMPFAIL);
+
+ goto cleanup;
+ }
+
+ str_action = ucl_object_tostring(elt);
+ action = rspamd_config_get_action(milter_ctx->cfg, str_action);
+
+ if (action == NULL) {
+ msg_err_milter("action %s has not been registered", str_action);
+ rspamd_milter_send_action(session, RSPAMD_MILTER_TEMPFAIL);
+
+ goto cleanup;
+ }
+
+ elt = ucl_object_lookup(results, "messages");
+ if (elt) {
+ const ucl_object_t *smtp_res;
+ const gchar *msg;
+ gsize len = 0;
+
+ smtp_res = ucl_object_lookup(elt, "smtp_message");
+
+ if (smtp_res) {
+ msg = ucl_object_tolstring(smtp_res, &len);
+ reply = rspamd_fstring_new_init(msg, len);
+ }
+ }
+
+ /* Deal with milter headers */
+ elt = ucl_object_lookup(results, "milter");
+
+ if (elt) {
+ processed = rspamd_milter_process_milter_block(session, elt, action);
+ }
+
+ /* DKIM-Signature */
+ elt = ucl_object_lookup(results, "dkim-signature");
+
+ if (elt) {
+ hname = g_string_new(RSPAMD_MILTER_DKIM_HEADER);
+
+ if (ucl_object_type(elt) == UCL_STRING) {
+ hvalue = g_string_new(ucl_object_tostring(elt));
+
+ rspamd_milter_send_action(session, RSPAMD_MILTER_INSHEADER,
+ 1, hname, hvalue);
+
+ g_string_free(hvalue, TRUE);
+ }
+ else {
+ ucl_object_iter_t it;
+ const ucl_object_t *cur;
+ int i = 1;
+
+ it = ucl_object_iterate_new(elt);
+
+ while ((cur = ucl_object_iterate_safe(it, true)) != NULL) {
+ hvalue = g_string_new(ucl_object_tostring(cur));
+
+ rspamd_milter_send_action(session, RSPAMD_MILTER_INSHEADER,
+ i++, hname, hvalue);
+
+ g_string_free(hvalue, TRUE);
+ }
+
+ ucl_object_iterate_free(it);
+ }
+
+ g_string_free(hname, TRUE);
+ }
+
+ if (processed) {
+ goto cleanup;
+ }
+
+ if (new_body) {
+ rspamd_milter_send_action(session, RSPAMD_MILTER_REPLBODY,
+ bodylen, new_body);
+ }
+
+ if (priv->no_action) {
+ msg_info_milter("do not apply action %s, no_action is set",
+ str_action);
+ hname = g_string_new(RSPAMD_MILTER_ACTION_HEADER);
+ hvalue = g_string_new(str_action);
+
+ rspamd_milter_send_action(session, RSPAMD_MILTER_ADDHEADER,
+ hname, hvalue);
+ g_string_free(hname, TRUE);
+ g_string_free(hvalue, TRUE);
+ rspamd_milter_send_action(session, RSPAMD_MILTER_ACCEPT);
+
+ goto cleanup;
+ }
+
+ switch (action->action_type) {
+ case METRIC_ACTION_REJECT:
+ if (priv->discard_on_reject) {
+ rspamd_milter_send_action(session, RSPAMD_MILTER_DISCARD);
+ }
+ else if (priv->quarantine_on_reject) {
+ /* TODO: be more flexible about SMTP messages */
+ rspamd_milter_send_action(session, RSPAMD_MILTER_QUARANTINE,
+ RSPAMD_MILTER_QUARANTINE_MESSAGE);
+
+ /* Quarantine also requires accept action, all hail Sendmail */
+ rspamd_milter_send_action(session, RSPAMD_MILTER_ACCEPT);
+ }
+ else {
+ rcode = rspamd_fstring_new_init(RSPAMD_MILTER_RCODE_REJECT,
+ sizeof(RSPAMD_MILTER_RCODE_REJECT) - 1);
+ xcode = rspamd_fstring_new_init(RSPAMD_MILTER_XCODE_REJECT,
+ sizeof(RSPAMD_MILTER_XCODE_REJECT) - 1);
+
+ if (!reply) {
+ if (milter_ctx->reject_message == NULL) {
+ reply = rspamd_fstring_new_init(
+ RSPAMD_MILTER_REJECT_MESSAGE,
+ sizeof(RSPAMD_MILTER_REJECT_MESSAGE) - 1);
+ }
+ else {
+ reply = rspamd_fstring_new_init(milter_ctx->reject_message,
+ strlen(milter_ctx->reject_message));
+ }
+ }
+
+ rspamd_milter_set_reply(session, rcode, xcode, reply);
+ }
+ break;
+ case METRIC_ACTION_SOFT_REJECT:
+ rcode = rspamd_fstring_new_init(RSPAMD_MILTER_RCODE_TEMPFAIL,
+ sizeof(RSPAMD_MILTER_RCODE_TEMPFAIL) - 1);
+ xcode = rspamd_fstring_new_init(RSPAMD_MILTER_XCODE_TEMPFAIL,
+ sizeof(RSPAMD_MILTER_XCODE_TEMPFAIL) - 1);
+
+ if (!reply) {
+ reply = rspamd_fstring_new_init(RSPAMD_MILTER_TEMPFAIL_MESSAGE,
+ sizeof(RSPAMD_MILTER_TEMPFAIL_MESSAGE) - 1);
+ }
+
+ rspamd_milter_set_reply(session, rcode, xcode, reply);
+ break;
+
+ case METRIC_ACTION_REWRITE_SUBJECT:
+ elt = ucl_object_lookup(results, "subject");
+
+ if (elt) {
+ hname = g_string_new("Subject");
+ hvalue = g_string_new(ucl_object_tostring(elt));
+
+ rspamd_milter_send_action(session, RSPAMD_MILTER_CHGHEADER,
+ (guint32) 1, hname, hvalue);
+ g_string_free(hname, TRUE);
+ g_string_free(hvalue, TRUE);
+ }
+
+ rspamd_milter_send_action(session, RSPAMD_MILTER_ACCEPT);
+ break;
+
+ case METRIC_ACTION_ADD_HEADER:
+ /* Remove existing headers */
+ rspamd_milter_remove_header_safe(session,
+ milter_ctx->spam_header,
+ 0);
+
+ hname = g_string_new(milter_ctx->spam_header);
+ hvalue = g_string_new("Yes");
+ rspamd_milter_send_action(session, RSPAMD_MILTER_CHGHEADER,
+ (guint32) 1, hname, hvalue);
+ g_string_free(hname, TRUE);
+ g_string_free(hvalue, TRUE);
+ rspamd_milter_send_action(session, RSPAMD_MILTER_ACCEPT);
+ break;
+
+ case METRIC_ACTION_QUARANTINE:
+ /* TODO: be more flexible about SMTP messages */
+ rspamd_milter_send_action(session, RSPAMD_MILTER_QUARANTINE,
+ RSPAMD_MILTER_QUARANTINE_MESSAGE);
+
+ /* Quarantine also requires accept action, all hail Sendmail */
+ rspamd_milter_send_action(session, RSPAMD_MILTER_ACCEPT);
+ break;
+ case METRIC_ACTION_DISCARD:
+ rspamd_milter_send_action(session, RSPAMD_MILTER_DISCARD);
+ break;
+ case METRIC_ACTION_GREYLIST:
+ case METRIC_ACTION_NOACTION:
+ default:
+ rspamd_milter_send_action(session, RSPAMD_MILTER_ACCEPT);
+ break;
+ }
+
+cleanup:
+ rspamd_fstring_free(rcode);
+ rspamd_fstring_free(xcode);
+ rspamd_fstring_free(reply);
+
+ rspamd_milter_session_reset(session, RSPAMD_MILTER_RESET_ABORT);
+}
+
+void rspamd_milter_init_library(const struct rspamd_milter_context *ctx)
+{
+ milter_ctx = ctx;
+}
+
+rspamd_mempool_t *
+rspamd_milter_get_session_pool(struct rspamd_milter_session *session)
+{
+ struct rspamd_milter_private *priv = session->priv;
+
+ return priv->pool;
+}
diff --git a/src/libserver/milter.h b/src/libserver/milter.h
new file mode 100644
index 0000000..096cda8
--- /dev/null
+++ b/src/libserver/milter.h
@@ -0,0 +1,188 @@
+/*-
+ * Copyright 2017 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_MILTER_H
+#define RSPAMD_MILTER_H
+
+#include "config.h"
+#include "fstring.h"
+#include "addr.h"
+#include "contrib/libucl/ucl.h"
+#include "contrib/libev/ev.h"
+#include "ref.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum rspamd_milter_reply {
+ RSPAMD_MILTER_ADDRCPT = '+',
+ RSPAMD_MILTER_DELRCPT = '-',
+ RSPAMD_MILTER_ACCEPT = 'a',
+ RSPAMD_MILTER_CONTINUE = 'c',
+ RSPAMD_MILTER_DISCARD = 'd',
+ RSPAMD_MILTER_CHGFROM = 'e',
+ RSPAMD_MILTER_ADDHEADER = 'h',
+ RSPAMD_MILTER_CHGHEADER = 'm',
+ RSPAMD_MILTER_INSHEADER = 'i',
+ RSPAMD_MILTER_REPLBODY = 'b',
+ RSPAMD_MILTER_REJECT = 'r',
+ RSPAMD_MILTER_TEMPFAIL = 't',
+ RSPAMD_MILTER_REPLYCODE = 'y',
+ RSPAMD_MILTER_OPTNEG = 'O',
+ RSPAMD_MILTER_PROGRESS = 'p',
+ RSPAMD_MILTER_QUARANTINE = 'q',
+};
+
+struct rspamd_email_address;
+struct ev_loop;
+struct rspamd_http_message;
+struct rspamd_config;
+
+struct rspamd_milter_context {
+ const gchar *spam_header;
+ const gchar *client_ca_name;
+ const gchar *reject_message;
+ void *sessions_cache;
+ struct rspamd_config *cfg;
+ gboolean discard_on_reject;
+ gboolean quarantine_on_reject;
+};
+
+struct rspamd_milter_session {
+ GHashTable *macros;
+ rspamd_inet_addr_t *addr;
+ struct rspamd_email_address *from;
+ GPtrArray *rcpts;
+ rspamd_fstring_t *helo;
+ rspamd_fstring_t *hostname;
+ rspamd_fstring_t *message;
+ void *priv;
+ ref_entry_t ref;
+};
+
+typedef void (*rspamd_milter_finish)(gint fd,
+ struct rspamd_milter_session *session, void *ud);
+
+typedef void (*rspamd_milter_error)(gint fd,
+ struct rspamd_milter_session *session,
+ void *ud, GError *err);
+
+/**
+ * Handles socket with milter protocol
+ * @param fd
+ * @param finish_cb
+ * @param error_cb
+ * @param ud
+ * @return
+ */
+gboolean rspamd_milter_handle_socket(gint fd, ev_tstamp timeout,
+ rspamd_mempool_t *pool,
+ struct ev_loop *ev_base, rspamd_milter_finish finish_cb,
+ rspamd_milter_error error_cb, void *ud);
+
+/**
+ * Updates userdata for a session, returns previous userdata
+ * @param session
+ * @param ud
+ * @return
+ */
+void *rspamd_milter_update_userdata(struct rspamd_milter_session *session,
+ void *ud);
+
+/**
+ * Sets SMTP reply string
+ * @param session
+ * @param rcode
+ * @param xcode
+ * @param reply
+ * @return
+ */
+gboolean rspamd_milter_set_reply(struct rspamd_milter_session *session,
+ rspamd_fstring_t *rcode,
+ rspamd_fstring_t *xcode,
+ rspamd_fstring_t *reply);
+
+/**
+ * Send some action to the MTA
+ * @param fd
+ * @param session
+ * @param act
+ * @return
+ */
+gboolean rspamd_milter_send_action(struct rspamd_milter_session *session,
+ enum rspamd_milter_reply act, ...);
+
+/**
+ * Adds some header
+ * @param session
+ * @param name
+ * @param value
+ * @return
+ */
+gboolean rspamd_milter_add_header(struct rspamd_milter_session *session,
+ GString *name, GString *value);
+
+/**
+ * Removes some header
+ * @param session
+ * @param name
+ * @return
+ */
+gboolean rspamd_milter_del_header(struct rspamd_milter_session *session,
+ GString *name);
+
+void rspamd_milter_session_unref(struct rspamd_milter_session *session);
+
+struct rspamd_milter_session *rspamd_milter_session_ref(
+ struct rspamd_milter_session *session);
+
+/**
+ * Converts milter session to HTTP session that is suitable for Rspamd
+ * @param session
+ * @return
+ */
+struct rspamd_http_message *rspamd_milter_to_http(
+ struct rspamd_milter_session *session);
+
+/**
+ * Sends task results to the
+ * @param session
+ * @param results
+ */
+void rspamd_milter_send_task_results(struct rspamd_milter_session *session,
+ const ucl_object_t *results,
+ const gchar *new_body,
+ gsize bodylen);
+
+/**
+ * Init internal milter context
+ * @param spam_header spam header name (must NOT be NULL)
+ */
+void rspamd_milter_init_library(const struct rspamd_milter_context *ctx);
+
+/**
+ * Returns pool for a session
+ * @param session
+ * @return
+ */
+rspamd_mempool_t *rspamd_milter_get_session_pool(
+ struct rspamd_milter_session *session);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/milter_internal.h b/src/libserver/milter_internal.h
new file mode 100644
index 0000000..bc292d3
--- /dev/null
+++ b/src/libserver/milter_internal.h
@@ -0,0 +1,176 @@
+/*-
+ * Copyright 2017 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_MILTER_INTERNAL_H
+#define RSPAMD_MILTER_INTERNAL_H
+
+#include "config.h"
+#include "libutil/mem_pool.h"
+#include "contrib/libev/ev.h"
+#include "khash.h"
+#include "libutil/str_util.h"
+#include "libutil/libev_helper.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum rspamd_milter_state {
+ st_len_1 = 0,
+ st_len_2,
+ st_len_3,
+ st_len_4,
+ st_read_cmd,
+ st_read_data
+};
+
+struct rspamd_milter_parser {
+ rspamd_fstring_t *buf;
+ goffset pos;
+ goffset cmd_start;
+ gsize datalen;
+ enum rspamd_milter_state state;
+ gchar cur_cmd;
+};
+
+struct rspamd_milter_outbuf {
+ rspamd_fstring_t *buf;
+ goffset pos;
+ struct rspamd_milter_outbuf *next, *prev;
+};
+
+enum rspamd_milter_io_state {
+ RSPAMD_MILTER_READ_MORE,
+ RSPAMD_MILTER_WRITE_REPLY,
+ RSPAMD_MILTER_WANNA_DIE,
+ RSPAMD_MILTER_WRITE_AND_DIE,
+ RSPAMD_MILTER_PONG_AND_DIE,
+};
+
+KHASH_INIT(milter_headers_hash_t, char *, GArray *, true,
+ rspamd_strcase_hash, rspamd_strcase_equal);
+
+struct rspamd_milter_private {
+ struct rspamd_milter_parser parser;
+ struct rspamd_io_ev ev;
+ struct rspamd_milter_outbuf *out_chain;
+ struct ev_loop *event_loop;
+ rspamd_mempool_t *pool;
+ khash_t(milter_headers_hash_t) * headers;
+ gint cur_hdr;
+ rspamd_milter_finish fin_cb;
+ rspamd_milter_error err_cb;
+ void *ud;
+ enum rspamd_milter_io_state state;
+ int fd;
+ gboolean discard_on_reject;
+ gboolean quarantine_on_reject;
+ gboolean no_action;
+};
+
+enum rspamd_milter_io_cmd {
+ RSPAMD_MILTER_CMD_ABORT = 'A', /* Abort */
+ RSPAMD_MILTER_CMD_BODY = 'B', /* Body chunk */
+ RSPAMD_MILTER_CMD_CONNECT = 'C', /* Connection information */
+ RSPAMD_MILTER_CMD_MACRO = 'D', /* Define macro */
+ RSPAMD_MILTER_CMD_BODYEOB = 'E', /* final body chunk (end of message) */
+ RSPAMD_MILTER_CMD_HELO = 'H', /* HELO/EHLO */
+ RSPAMD_MILTER_CMD_QUIT_NC = 'K', /* QUIT but new connection follows */
+ RSPAMD_MILTER_CMD_HEADER = 'L', /* Header */
+ RSPAMD_MILTER_CMD_MAIL = 'M', /* MAIL from */
+ RSPAMD_MILTER_CMD_EOH = 'N', /* EOH */
+ RSPAMD_MILTER_CMD_OPTNEG = 'O', /* Option negotiation */
+ RSPAMD_MILTER_CMD_QUIT = 'Q', /* QUIT */
+ RSPAMD_MILTER_CMD_RCPT = 'R', /* RCPT to */
+ RSPAMD_MILTER_CMD_DATA = 'T', /* DATA */
+ RSPAMD_MILTER_CMD_UNKNOWN = 'U' /* Any unknown command */
+};
+
+/*
+ * Protocol flags
+ */
+#define RSPAMD_MILTER_FLAG_NOUNKNOWN (1L << 8) /* filter does not want unknown cmd */
+#define RSPAMD_MILTER_FLAG_NODATA (1L << 9) /* filter does not want DATA */
+#define RSPAMD_MILTER_FLAG_NR_HDR (1L << 7) /* filter won't reply for header */
+#define RSPAMD_MILTER_FLAG_SKIP (1L << 10) /* MTA supports SMFIR_SKIP */
+#define RSPAMD_MILTER_FLAG_RCPT_REJ (1L << 11) /* filter wants rejected RCPTs */
+#define RSPAMD_MILTER_FLAG_NR_CONN (1L << 12) /* filter won't reply for connect */
+#define RSPAMD_MILTER_FLAG_NR_HELO (1L << 13) /* filter won't reply for HELO */
+#define RSPAMD_MILTER_FLAG_NR_MAIL (1L << 14) /* filter won't reply for MAIL */
+#define RSPAMD_MILTER_FLAG_NR_RCPT (1L << 15) /* filter won't reply for RCPT */
+#define RSPAMD_MILTER_FLAG_NR_DATA (1L << 16) /* filter won't reply for DATA */
+#define RSPAMD_MILTER_FLAG_NR_UNKN (1L << 17) /* filter won't reply for UNKNOWN */
+#define RSPAMD_MILTER_FLAG_NR_EOH (1L << 18) /* filter won't reply for eoh */
+#define RSPAMD_MILTER_FLAG_NR_BODY (1L << 19) /* filter won't reply for body chunk */
+
+/*
+ * For now, we specify that we want to reply just after EOM
+ */
+#define RSPAMD_MILTER_FLAG_NOREPLY_MASK \
+ (RSPAMD_MILTER_FLAG_NR_CONN | RSPAMD_MILTER_FLAG_NR_HELO | \
+ RSPAMD_MILTER_FLAG_NR_MAIL | RSPAMD_MILTER_FLAG_NR_RCPT | \
+ RSPAMD_MILTER_FLAG_NR_DATA | RSPAMD_MILTER_FLAG_NR_UNKN | \
+ RSPAMD_MILTER_FLAG_NR_HDR | RSPAMD_MILTER_FLAG_NR_EOH | \
+ RSPAMD_MILTER_FLAG_NR_BODY)
+
+/*
+ * Options that the filter may send at initial handshake time, and message
+ * modifications that the filter may request at the end of the message body.
+ */
+#define RSPAMD_MILTER_FLAG_ADDHDRS (1L << 0) /* filter may add headers */
+#define RSPAMD_MILTER_FLAG_CHGBODY (1L << 1) /* filter may replace body */
+#define RSPAMD_MILTER_FLAG_ADDRCPT (1L << 2) /* filter may add recipients */
+#define RSPAMD_MILTER_FLAG_DELRCPT (1L << 3) /* filter may delete recipients */
+#define RSPAMD_MILTER_FLAG_CHGHDRS (1L << 4) /* filter may change/delete headers */
+#define RSPAMD_MILTER_FLAG_QUARANTINE (1L << 5) /* filter may request quarantine */
+
+#define RSPAMD_MILTER_ACTIONS_MASK \
+ (RSPAMD_MILTER_FLAG_ADDHDRS | RSPAMD_MILTER_FLAG_ADDRCPT | \
+ RSPAMD_MILTER_FLAG_DELRCPT | RSPAMD_MILTER_FLAG_CHGHDRS | \
+ RSPAMD_MILTER_FLAG_CHGBODY | RSPAMD_MILTER_FLAG_QUARANTINE)
+
+enum rspamd_milter_connect_proto {
+ RSPAMD_MILTER_CONN_UNKNOWN = 'U',
+ RSPAMD_MILTER_CONN_UNIX = 'L',
+ RSPAMD_MILTER_CONN_INET = '4',
+ RSPAMD_MILTER_CONN_INET6 = '6',
+};
+
+/*
+ * Rspamd supports just version 6 of the protocol, failing all versions below
+ * this one
+ */
+#define RSPAMD_MILTER_PROTO_VER 6
+
+#define RSPAMD_MILTER_MESSAGE_CHUNK 65536
+
+#define RSPAMD_MILTER_RCODE_REJECT "554"
+#define RSPAMD_MILTER_RCODE_TEMPFAIL "451"
+#define RSPAMD_MILTER_RCODE_LATER "452"
+#define RSPAMD_MILTER_XCODE_REJECT "5.7.1"
+#define RSPAMD_MILTER_XCODE_TEMPFAIL "4.7.1"
+#define RSPAMD_MILTER_REJECT_MESSAGE "Spam message rejected"
+#define RSPAMD_MILTER_QUARANTINE_MESSAGE "Spam message quarantined"
+#define RSPAMD_MILTER_TEMPFAIL_MESSAGE "Try again later"
+#define RSPAMD_MILTER_SPAM_HEADER "X-Spam"
+#define RSPAMD_MILTER_DKIM_HEADER "DKIM-Signature"
+#define RSPAMD_MILTER_ACTION_HEADER "X-Rspamd-Action"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif \ No newline at end of file
diff --git a/src/libserver/monitored.c b/src/libserver/monitored.c
new file mode 100644
index 0000000..3aebaf6
--- /dev/null
+++ b/src/libserver/monitored.c
@@ -0,0 +1,735 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <contrib/librdns/rdns.h>
+#include "rdns.h"
+#include "mem_pool.h"
+#include "cfg_file.h"
+#include "cryptobox.h"
+#include "logger.h"
+#include "contrib/uthash/utlist.h"
+
+static const gdouble default_monitoring_interval = 60.0;
+static const guint default_max_errors = 2;
+static const gdouble default_max_monitored_mult = 32;
+static const gdouble default_min_monitored_mult = 0.1;
+static const gdouble default_initial_monitored_mult = default_min_monitored_mult;
+static const gdouble default_offline_monitored_mult = 8.0;
+
+struct rspamd_monitored_methods {
+ void *(*monitored_config)(struct rspamd_monitored *m,
+ struct rspamd_monitored_ctx *ctx,
+ const ucl_object_t *opts);
+ gboolean (*monitored_update)(struct rspamd_monitored *m,
+ struct rspamd_monitored_ctx *ctx, gpointer ud);
+ void (*monitored_dtor)(struct rspamd_monitored *m,
+ struct rspamd_monitored_ctx *ctx, gpointer ud);
+ gpointer ud;
+};
+
+struct rspamd_monitored_ctx {
+ struct rspamd_config *cfg;
+ struct rdns_resolver *resolver;
+ struct ev_loop *event_loop;
+ GPtrArray *elts;
+ GHashTable *helts;
+ mon_change_cb change_cb;
+ gpointer ud;
+ gdouble monitoring_interval;
+ gdouble max_monitored_mult;
+ gdouble min_monitored_mult;
+ gdouble initial_monitored_mult;
+ gdouble offline_monitored_mult;
+ guint max_errors;
+ gboolean initialized;
+};
+
+struct rspamd_monitored {
+ gchar *url;
+ gdouble monitoring_mult;
+ gdouble offline_time;
+ gdouble total_offline_time;
+ gdouble latency;
+ guint nchecks;
+ guint max_errors;
+ guint cur_errors;
+ gboolean alive;
+ enum rspamd_monitored_type type;
+ enum rspamd_monitored_flags flags;
+ struct rspamd_monitored_ctx *ctx;
+ struct rspamd_monitored_methods proc;
+ ev_timer periodic;
+ gchar tag[RSPAMD_MONITORED_TAG_LEN];
+};
+
+#define msg_err_mon(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "monitored", m->tag, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_mon(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "monitored", m->tag, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_mon(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "monitored", m->tag, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_notice_mon(...) rspamd_default_log_function(G_LOG_LEVEL_MESSAGE, \
+ "monitored", m->tag, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_mon(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_monitored_log_id, "monitored", m->tag, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(monitored)
+
+static inline void
+rspamd_monitored_propagate_error(struct rspamd_monitored *m,
+ const gchar *error)
+{
+ if (m->alive) {
+ if (m->cur_errors < m->max_errors) {
+
+ m->cur_errors++;
+ /* Reduce timeout */
+ rspamd_monitored_stop(m);
+
+ if (m->monitoring_mult > m->ctx->min_monitored_mult) {
+ if (m->monitoring_mult < 1.0) {
+ m->monitoring_mult = 1.0;
+ }
+ else {
+ m->monitoring_mult /= 2.0;
+ }
+ }
+
+ msg_debug_mon("%s on resolving %s, %d retries left; next check in %.2f",
+ error, m->url, m->max_errors - m->cur_errors,
+ m->ctx->monitoring_interval * m->monitoring_mult);
+
+ rspamd_monitored_start(m);
+ }
+ else {
+ msg_notice_mon("%s on resolving %s, disable object",
+ error, m->url);
+ m->alive = FALSE;
+ m->offline_time = rspamd_get_calendar_ticks();
+ rspamd_monitored_stop(m);
+ m->monitoring_mult = 2.0;
+ rspamd_monitored_start(m);
+
+ if (m->ctx->change_cb) {
+ m->ctx->change_cb(m->ctx, m, FALSE, m->ctx->ud);
+ }
+ }
+ }
+ else {
+ if (m->monitoring_mult < m->ctx->offline_monitored_mult) {
+ /* Increase timeout */
+ rspamd_monitored_stop(m);
+ m->monitoring_mult *= 2.0;
+ rspamd_monitored_start(m);
+ }
+ else {
+ rspamd_monitored_stop(m);
+ m->monitoring_mult = m->ctx->offline_monitored_mult;
+ rspamd_monitored_start(m);
+ }
+ }
+}
+
+static inline void
+rspamd_monitored_propagate_success(struct rspamd_monitored *m, gdouble lat)
+{
+ gdouble t;
+
+ m->cur_errors = 0;
+
+ if (!m->alive) {
+ m->monitoring_mult = 1.0;
+ t = rspamd_get_calendar_ticks();
+ m->total_offline_time += t - m->offline_time;
+ m->alive = TRUE;
+ msg_notice_mon("restoring %s after %.1f seconds of downtime, "
+ "total downtime: %.1f",
+ m->url, t - m->offline_time, m->total_offline_time);
+ m->offline_time = 0;
+ m->nchecks = 1;
+ m->latency = lat;
+ rspamd_monitored_stop(m);
+ rspamd_monitored_start(m);
+
+ if (m->ctx->change_cb) {
+ m->ctx->change_cb(m->ctx, m, TRUE, m->ctx->ud);
+ }
+ }
+ else {
+ /* Increase monitored interval */
+ if (m->monitoring_mult < m->ctx->max_monitored_mult) {
+ if (m->monitoring_mult < 1.0) {
+ /* Upgrade fast from the initial mult */
+ m->monitoring_mult = 1.0;
+ }
+ else {
+ m->monitoring_mult *= 2.0;
+ }
+ }
+ else {
+ m->monitoring_mult = m->ctx->max_monitored_mult;
+ }
+ m->latency = (lat + m->latency * m->nchecks) / (m->nchecks + 1);
+ m->nchecks++;
+ }
+}
+
+static void
+rspamd_monitored_periodic(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_monitored *m = (struct rspamd_monitored *) w->data;
+ gdouble jittered;
+ gboolean ret = FALSE;
+
+ if (m->proc.monitored_update) {
+ ret = m->proc.monitored_update(m, m->ctx, m->proc.ud);
+ }
+
+ jittered = rspamd_time_jitter(m->ctx->monitoring_interval * m->monitoring_mult,
+ 0.0);
+
+ if (ret) {
+ m->periodic.repeat = jittered;
+ ev_timer_again(EV_A_ & m->periodic);
+ }
+}
+
+struct rspamd_dns_monitored_conf {
+ enum rdns_request_type rt;
+ GString *request;
+ radix_compressed_t *expected;
+ struct rspamd_monitored *m;
+ gint expected_code;
+ gdouble check_tm;
+};
+
+static void
+rspamd_monitored_dns_random(struct rspamd_monitored *m,
+ struct rspamd_dns_monitored_conf *conf)
+{
+ gchar random_prefix[32];
+ const gchar dns_chars[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_";
+ gint len;
+
+ len = rspamd_random_uint64_fast() % sizeof(random_prefix);
+
+ if (len < 8) {
+ len = 8;
+ }
+
+ for (guint i = 0; i < len; i++) {
+ guint idx = rspamd_random_uint64_fast() % (sizeof(dns_chars) - 1);
+ random_prefix[i] = dns_chars[idx];
+ }
+
+ conf->request->len = 0;
+ rspamd_printf_gstring(conf->request, "%*.s.%s", len, random_prefix,
+ m->url);
+}
+
+static void *
+rspamd_monitored_dns_conf(struct rspamd_monitored *m,
+ struct rspamd_monitored_ctx *ctx,
+ const ucl_object_t *opts)
+{
+ struct rspamd_dns_monitored_conf *conf;
+ const ucl_object_t *elt;
+ gint rt;
+ GString *req = g_string_sized_new(127);
+
+ conf = g_malloc0(sizeof(*conf));
+ conf->rt = RDNS_REQUEST_A;
+ conf->m = m;
+ conf->expected_code = -1;
+
+ if (opts) {
+ elt = ucl_object_lookup(opts, "type");
+
+ if (elt) {
+ rt = rdns_type_fromstr(ucl_object_tostring(elt));
+
+ if (rt != -1) {
+ conf->rt = rt;
+ }
+ else {
+ msg_err_mon("invalid resolve type: %s",
+ ucl_object_tostring(elt));
+ }
+ }
+
+ if (!(m->flags & RSPAMD_MONITORED_RANDOM)) {
+ /* Prefix is useless for random monitored */
+ elt = ucl_object_lookup(opts, "prefix");
+
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ rspamd_printf_gstring(req, "%s.", ucl_object_tostring(elt));
+ }
+ }
+
+ elt = ucl_object_lookup(opts, "ipnet");
+
+ if (elt) {
+ if (ucl_object_type(elt) == UCL_STRING) {
+ radix_add_generic_iplist(ucl_object_tostring(elt),
+ &conf->expected, FALSE, NULL);
+ }
+ else if (ucl_object_type(elt) == UCL_ARRAY) {
+ const ucl_object_t *cur;
+ ucl_object_iter_t it = NULL;
+
+ while ((cur = ucl_object_iterate(elt, &it, true)) != NULL) {
+ radix_add_generic_iplist(ucl_object_tostring(elt),
+ &conf->expected, FALSE, NULL);
+ }
+ }
+ }
+
+ elt = ucl_object_lookup(opts, "rcode");
+ if (elt) {
+ rt = rdns_rcode_fromstr(ucl_object_tostring(elt));
+
+ if (rt != -1) {
+ conf->expected_code = rt;
+ }
+ else {
+ msg_err_mon("invalid resolve rcode: %s",
+ ucl_object_tostring(elt));
+ }
+ }
+ }
+
+ if (!(m->flags & RSPAMD_MONITORED_RANDOM)) {
+ rspamd_printf_gstring(req, "%s", m->url);
+ }
+
+ conf->request = req;
+
+ return conf;
+}
+
+static void
+rspamd_monitored_dns_cb(struct rdns_reply *reply, void *arg)
+{
+ struct rspamd_dns_monitored_conf *conf = arg;
+ struct rspamd_monitored *m;
+ struct rdns_reply_entry *cur;
+ gboolean is_special_reply = FALSE;
+ gdouble lat;
+
+ m = conf->m;
+ lat = rspamd_get_calendar_ticks() - conf->check_tm;
+ conf->check_tm = 0;
+ msg_debug_mon("dns callback for %s in %.2f: %s", m->url, lat,
+ rdns_strerror(reply->code));
+
+ if (reply->code == RDNS_RC_TIMEOUT) {
+ rspamd_monitored_propagate_error(m, "timeout");
+ }
+ else if (reply->code == RDNS_RC_SERVFAIL) {
+ rspamd_monitored_propagate_error(m, "servfail");
+ }
+ else if (reply->code == RDNS_RC_REFUSED) {
+ rspamd_monitored_propagate_error(m, "refused");
+ }
+ else {
+ if (conf->expected_code != -1) {
+ if (reply->code != conf->expected_code) {
+ if (reply->code == RDNS_RC_NOREC &&
+ conf->expected_code == RDNS_RC_NXDOMAIN) {
+ rspamd_monitored_propagate_success(m, lat);
+ }
+ else {
+ LL_FOREACH(reply->entries, cur)
+ {
+ if (cur->type == RDNS_REQUEST_A) {
+ if ((guint32) cur->content.a.addr.s_addr ==
+ htonl(INADDR_LOOPBACK)) {
+ is_special_reply = TRUE;
+ }
+ }
+ }
+
+ if (is_special_reply) {
+ msg_notice_mon("DNS query blocked on %s "
+ "(127.0.0.1 returned), "
+ "possibly due to high volume",
+ m->url);
+ }
+ else {
+ msg_notice_mon("DNS reply returned '%s' for %s while '%s' "
+ "was expected when querying for '%s'"
+ "(likely DNS spoofing or BL internal issues)",
+ rdns_strerror(reply->code),
+ m->url,
+ rdns_strerror(conf->expected_code),
+ conf->request->str);
+ }
+
+ rspamd_monitored_propagate_error(m, "invalid return");
+ }
+ }
+ else {
+ rspamd_monitored_propagate_success(m, lat);
+ }
+ }
+ else if (conf->expected) {
+ /* We also need to check IP */
+ if (reply->code != RDNS_RC_NOERROR) {
+ rspamd_monitored_propagate_error(m, "no record");
+ }
+ else {
+ rspamd_inet_addr_t *addr;
+
+ addr = rspamd_inet_address_from_rnds(reply->entries);
+
+ if (!addr) {
+ rspamd_monitored_propagate_error(m,
+ "unreadable address");
+ }
+ else if (radix_find_compressed_addr(conf->expected, addr)) {
+ msg_notice_mon("bad address %s is returned when monitoring %s",
+ rspamd_inet_address_to_string(addr),
+ conf->request->str);
+ rspamd_monitored_propagate_error(m,
+ "invalid address");
+
+ rspamd_inet_address_free(addr);
+ }
+ else {
+ rspamd_monitored_propagate_success(m, lat);
+ rspamd_inet_address_free(addr);
+ }
+ }
+ }
+ else {
+ rspamd_monitored_propagate_success(m, lat);
+ }
+ }
+}
+
+static gboolean
+rspamd_monitored_dns_mon(struct rspamd_monitored *m,
+ struct rspamd_monitored_ctx *ctx, gpointer ud)
+{
+ struct rspamd_dns_monitored_conf *conf = ud;
+
+ if (m->flags & RSPAMD_MONITORED_RANDOM) {
+ rspamd_monitored_dns_random(m, conf);
+ }
+
+ if (!rdns_make_request_full(ctx->resolver, rspamd_monitored_dns_cb,
+ conf, ctx->cfg->dns_timeout, ctx->cfg->dns_retransmits,
+ 1, conf->request->str, conf->rt)) {
+ msg_notice_mon("cannot make request to resolve %s (%s monitored url)",
+ conf->request->str, conf->m->url);
+
+ m->cur_errors++;
+ rspamd_monitored_propagate_error(m, "failed to make DNS request");
+
+ return FALSE;
+ }
+ else {
+ conf->check_tm = rspamd_get_calendar_ticks();
+ }
+
+ return TRUE;
+}
+
+void rspamd_monitored_dns_dtor(struct rspamd_monitored *m,
+ struct rspamd_monitored_ctx *ctx, gpointer ud)
+{
+ struct rspamd_dns_monitored_conf *conf = ud;
+
+ g_string_free(conf->request, TRUE);
+
+ if (conf->expected) {
+ radix_destroy_compressed(conf->expected);
+ }
+
+ g_free(conf);
+}
+
+struct rspamd_monitored_ctx *
+rspamd_monitored_ctx_init(void)
+{
+ struct rspamd_monitored_ctx *ctx;
+
+ ctx = g_malloc0(sizeof(*ctx));
+ ctx->monitoring_interval = default_monitoring_interval;
+ ctx->max_errors = default_max_errors;
+ ctx->offline_monitored_mult = default_offline_monitored_mult;
+ ctx->initial_monitored_mult = default_initial_monitored_mult;
+ ctx->max_monitored_mult = default_max_monitored_mult;
+ ctx->min_monitored_mult = default_min_monitored_mult;
+ ctx->elts = g_ptr_array_new();
+ ctx->helts = g_hash_table_new(g_str_hash, g_str_equal);
+
+ return ctx;
+}
+
+
+void rspamd_monitored_ctx_config(struct rspamd_monitored_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct ev_loop *ev_base,
+ struct rdns_resolver *resolver,
+ mon_change_cb change_cb,
+ gpointer ud)
+{
+ struct rspamd_monitored *m;
+ guint i;
+
+ g_assert(ctx != NULL);
+ ctx->event_loop = ev_base;
+ ctx->resolver = resolver;
+ ctx->cfg = cfg;
+ ctx->initialized = TRUE;
+ ctx->change_cb = change_cb;
+ ctx->ud = ud;
+
+ if (cfg->monitored_interval != 0) {
+ ctx->monitoring_interval = cfg->monitored_interval;
+ }
+
+ /* Start all events */
+ for (i = 0; i < ctx->elts->len; i++) {
+ m = g_ptr_array_index(ctx->elts, i);
+ m->monitoring_mult = ctx->initial_monitored_mult;
+ rspamd_monitored_start(m);
+ m->monitoring_mult = 1.0;
+ }
+}
+
+
+struct ev_loop *
+rspamd_monitored_ctx_get_ev_base(struct rspamd_monitored_ctx *ctx)
+{
+ return ctx->event_loop;
+}
+
+
+struct rspamd_monitored *
+rspamd_monitored_create_(struct rspamd_monitored_ctx *ctx,
+ const gchar *line,
+ enum rspamd_monitored_type type,
+ enum rspamd_monitored_flags flags,
+ const ucl_object_t *opts,
+ const gchar *loc)
+{
+ struct rspamd_monitored *m;
+ rspamd_cryptobox_hash_state_t st;
+ gchar *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES];
+
+ g_assert(ctx != NULL);
+
+ m = g_malloc0(sizeof(*m));
+ m->type = type;
+ m->flags = flags;
+
+ m->url = g_strdup(line);
+ m->ctx = ctx;
+ m->monitoring_mult = ctx->initial_monitored_mult;
+ m->max_errors = ctx->max_errors;
+ m->alive = TRUE;
+
+ if (type == RSPAMD_MONITORED_DNS) {
+ m->proc.monitored_update = rspamd_monitored_dns_mon;
+ m->proc.monitored_config = rspamd_monitored_dns_conf;
+ m->proc.monitored_dtor = rspamd_monitored_dns_dtor;
+ }
+ else {
+ g_free(m);
+
+ return NULL;
+ }
+
+ if (opts) {
+ const ucl_object_t *rnd_obj;
+
+ rnd_obj = ucl_object_lookup(opts, "random");
+
+ if (rnd_obj && ucl_object_type(rnd_obj) == UCL_BOOLEAN) {
+ if (ucl_object_toboolean(rnd_obj)) {
+ m->flags |= RSPAMD_MONITORED_RANDOM;
+ }
+ }
+ }
+
+ m->proc.ud = m->proc.monitored_config(m, ctx, opts);
+
+ if (m->proc.ud == NULL) {
+ g_free(m);
+
+ return NULL;
+ }
+
+ /* Create a persistent tag */
+ rspamd_cryptobox_hash_init(&st, NULL, 0);
+ rspamd_cryptobox_hash_update(&st, m->url, strlen(m->url));
+ rspamd_cryptobox_hash_update(&st, loc, strlen(loc));
+ rspamd_cryptobox_hash_final(&st, cksum);
+ cksum_encoded = rspamd_encode_base32(cksum, sizeof(cksum), RSPAMD_BASE32_DEFAULT);
+ rspamd_strlcpy(m->tag, cksum_encoded, sizeof(m->tag));
+
+ if (g_hash_table_lookup(ctx->helts, m->tag) != NULL) {
+ msg_err("monitored error: tag collision detected for %s; "
+ "url: %s",
+ m->tag, m->url);
+ }
+ else {
+ g_hash_table_insert(ctx->helts, m->tag, m);
+ }
+
+ g_free(cksum_encoded);
+
+ g_ptr_array_add(ctx->elts, m);
+
+ if (ctx->event_loop) {
+ rspamd_monitored_start(m);
+ }
+
+ return m;
+}
+
+gboolean
+rspamd_monitored_alive(struct rspamd_monitored *m)
+{
+ g_assert(m != NULL);
+
+ return m->alive;
+}
+
+gboolean
+rspamd_monitored_set_alive(struct rspamd_monitored *m, gboolean alive)
+{
+ gboolean st;
+
+ g_assert(m != NULL);
+ st = m->alive;
+ m->alive = alive;
+
+ return st;
+}
+
+gdouble
+rspamd_monitored_offline_time(struct rspamd_monitored *m)
+{
+ g_assert(m != NULL);
+
+ if (m->offline_time > 0) {
+ return rspamd_get_calendar_ticks() - m->offline_time;
+ }
+
+ return 0;
+}
+
+gdouble
+rspamd_monitored_total_offline_time(struct rspamd_monitored *m)
+{
+ g_assert(m != NULL);
+
+ if (m->offline_time > 0) {
+ return rspamd_get_calendar_ticks() - m->offline_time + m->total_offline_time;
+ }
+
+
+ return m->total_offline_time;
+}
+
+gdouble
+rspamd_monitored_latency(struct rspamd_monitored *m)
+{
+ g_assert(m != NULL);
+
+ return m->latency;
+}
+
+void rspamd_monitored_stop(struct rspamd_monitored *m)
+{
+ g_assert(m != NULL);
+
+ ev_timer_stop(m->ctx->event_loop, &m->periodic);
+}
+
+void rspamd_monitored_start(struct rspamd_monitored *m)
+{
+ gdouble jittered;
+
+ g_assert(m != NULL);
+ jittered = rspamd_time_jitter(m->ctx->monitoring_interval * m->monitoring_mult,
+ 0.0);
+
+ msg_debug_mon("started monitored object %s in %.2f seconds", m->url, jittered);
+
+ if (ev_can_stop(&m->periodic)) {
+ ev_timer_stop(m->ctx->event_loop, &m->periodic);
+ }
+
+ m->periodic.data = m;
+ ev_timer_init(&m->periodic, rspamd_monitored_periodic, jittered, 0.0);
+ ev_timer_start(m->ctx->event_loop, &m->periodic);
+}
+
+void rspamd_monitored_ctx_destroy(struct rspamd_monitored_ctx *ctx)
+{
+ struct rspamd_monitored *m;
+ guint i;
+
+ g_assert(ctx != NULL);
+
+ for (i = 0; i < ctx->elts->len; i++) {
+ m = g_ptr_array_index(ctx->elts, i);
+ rspamd_monitored_stop(m);
+ m->proc.monitored_dtor(m, m->ctx, m->proc.ud);
+ g_free(m->url);
+ g_free(m);
+ }
+
+ g_ptr_array_free(ctx->elts, TRUE);
+ g_hash_table_unref(ctx->helts);
+ g_free(ctx);
+}
+
+struct rspamd_monitored *
+rspamd_monitored_by_tag(struct rspamd_monitored_ctx *ctx,
+ guchar tag[RSPAMD_MONITORED_TAG_LEN])
+{
+ struct rspamd_monitored *res;
+ gchar rtag[RSPAMD_MONITORED_TAG_LEN];
+
+ rspamd_strlcpy(rtag, tag, sizeof(rtag));
+ res = g_hash_table_lookup(ctx->helts, rtag);
+
+ return res;
+}
+
+
+void rspamd_monitored_get_tag(struct rspamd_monitored *m,
+ guchar tag_out[RSPAMD_MONITORED_TAG_LEN])
+{
+ g_assert(m != NULL);
+
+ rspamd_strlcpy(tag_out, m->tag, RSPAMD_MONITORED_TAG_LEN);
+} \ No newline at end of file
diff --git a/src/libserver/monitored.h b/src/libserver/monitored.h
new file mode 100644
index 0000000..01f050a
--- /dev/null
+++ b/src/libserver/monitored.h
@@ -0,0 +1,161 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBSERVER_MONITORED_H_
+#define SRC_LIBSERVER_MONITORED_H_
+
+#include "config.h"
+#include "rdns.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_monitored;
+struct rspamd_monitored_ctx;
+struct rspamd_config;
+
+#define RSPAMD_MONITORED_TAG_LEN 32
+
+enum rspamd_monitored_type {
+ RSPAMD_MONITORED_DNS = 0,
+};
+
+enum rspamd_monitored_flags {
+ RSPAMD_MONITORED_DEFAULT = 0u,
+ RSPAMD_MONITORED_RBL = (1u << 0u),
+ RSPAMD_MONITORED_RANDOM = (1u << 1u)
+};
+
+/**
+ * Initialize new monitored context
+ * @return opaque context pointer (should be configured)
+ */
+struct rspamd_monitored_ctx *rspamd_monitored_ctx_init(void);
+
+typedef void (*mon_change_cb)(struct rspamd_monitored_ctx *ctx,
+ struct rspamd_monitored *m, gboolean alive,
+ void *ud);
+
+/**
+ * Configure context for monitored objects
+ * @param ctx context
+ * @param cfg configuration
+ * @param ev_base events base
+ * @param resolver resolver object
+ */
+void rspamd_monitored_ctx_config(struct rspamd_monitored_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct ev_loop *ev_base,
+ struct rdns_resolver *resolver,
+ mon_change_cb change_cb,
+ gpointer ud);
+
+struct ev_loop *rspamd_monitored_ctx_get_ev_base(struct rspamd_monitored_ctx *ctx);
+
+/**
+ * Create monitored object
+ * @param ctx context
+ * @param line string definition (e.g. hostname)
+ * @param type type of monitoring
+ * @param flags specific flags for monitoring
+ * @return new monitored object
+ */
+struct rspamd_monitored *rspamd_monitored_create_(
+ struct rspamd_monitored_ctx *ctx,
+ const gchar *line,
+ enum rspamd_monitored_type type,
+ enum rspamd_monitored_flags flags,
+ const ucl_object_t *opts,
+ const gchar *loc);
+
+#define rspamd_monitored_create(ctx, line, type, flags, opts) \
+ rspamd_monitored_create_(ctx, line, type, flags, opts, G_STRFUNC)
+
+/**
+ * Return monitored by its tag
+ * @param ctx
+ * @param tag
+ * @return
+ */
+struct rspamd_monitored *rspamd_monitored_by_tag(struct rspamd_monitored_ctx *ctx,
+ guchar tag[RSPAMD_MONITORED_TAG_LEN]);
+
+/**
+ * Sets `tag_out` to the monitored tag
+ * @param m
+ * @param tag_out
+ */
+void rspamd_monitored_get_tag(struct rspamd_monitored *m,
+ guchar tag_out[RSPAMD_MONITORED_TAG_LEN]);
+
+/**
+ * Return TRUE if monitored object is alive
+ * @param m monitored object
+ * @return TRUE or FALSE
+ */
+gboolean rspamd_monitored_alive(struct rspamd_monitored *m);
+
+/**
+ * Force alive flag for a monitored object
+ * @param m monitored object
+ * @return TRUE or FALSE
+ */
+gboolean rspamd_monitored_set_alive(struct rspamd_monitored *m, gboolean alive);
+
+/**
+ * Returns the current offline time for a monitored object
+ * @param m
+ * @return
+ */
+gdouble rspamd_monitored_offline_time(struct rspamd_monitored *m);
+
+/**
+ * Returns the total offline time for a monitored object
+ * @param m
+ * @return
+ */
+gdouble rspamd_monitored_total_offline_time(struct rspamd_monitored *m);
+
+/**
+ * Returns the latency for monitored object (in seconds)
+ * @param m
+ * @return
+ */
+gdouble rspamd_monitored_latency(struct rspamd_monitored *m);
+
+/**
+ * Explicitly disable monitored object
+ * @param m
+ */
+void rspamd_monitored_stop(struct rspamd_monitored *m);
+
+/**
+ * Explicitly enable monitored object
+ * @param m
+ */
+void rspamd_monitored_start(struct rspamd_monitored *m);
+
+/**
+ * Destroy monitored context and all monitored objects inside
+ * @param ctx
+ */
+void rspamd_monitored_ctx_destroy(struct rspamd_monitored_ctx *ctx);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBSERVER_MONITORED_H_ */
diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c
new file mode 100644
index 0000000..8674557
--- /dev/null
+++ b/src/libserver/protocol.c
@@ -0,0 +1,2185 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "message.h"
+#include "utlist.h"
+#include "libserver/http/http_private.h"
+#include "worker_private.h"
+#include "libserver/cfg_file_private.h"
+#include "libmime/scan_result_private.h"
+#include "lua/lua_common.h"
+#include "unix-std.h"
+#include "protocol_internal.h"
+#include "libserver/mempool_vars_internal.h"
+#include "contrib/fastutf8/fastutf8.h"
+#include "task.h"
+#include <math.h>
+
+#ifdef SYS_ZSTD
+#include "zstd.h"
+#else
+#include "contrib/zstd/zstd.h"
+#endif
+
+INIT_LOG_MODULE(protocol)
+
+#define msg_err_protocol(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "protocol", task->task_pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_warn_protocol(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "protocol", task->task_pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_info_protocol(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "protocol", task->task_pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_debug_protocol(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_protocol_log_id, "protocol", task->task_pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+static GQuark
+rspamd_protocol_quark(void)
+{
+ return g_quark_from_static_string("protocol-error");
+}
+
+/*
+ * Remove <> from the fixed string and copy it to the pool
+ */
+static gchar *
+rspamd_protocol_escape_braces(struct rspamd_task *task, rspamd_ftok_t *in)
+{
+ guint nchars = 0;
+ const gchar *p;
+ rspamd_ftok_t tok;
+ gboolean has_obrace = FALSE;
+
+ g_assert(in != NULL);
+ g_assert(in->len > 0);
+
+ p = in->begin;
+
+ while ((g_ascii_isspace(*p) || *p == '<') && nchars < in->len) {
+ if (*p == '<') {
+ has_obrace = TRUE;
+ }
+
+ p++;
+ nchars++;
+ }
+
+ tok.begin = p;
+
+ p = in->begin + in->len - 1;
+ tok.len = in->len - nchars;
+
+ while (g_ascii_isspace(*p) && tok.len > 0) {
+ p--;
+ tok.len--;
+ }
+
+ if (has_obrace && *p == '>') {
+ tok.len--;
+ }
+
+ return rspamd_mempool_ftokdup(task->task_pool, &tok);
+}
+
+#define COMPARE_CMD(str, cmd, len) (sizeof(cmd) - 1 == (len) && rspamd_lc_cmp((str), (cmd), (len)) == 0)
+
+static gboolean
+rspamd_protocol_handle_url(struct rspamd_task *task,
+ struct rspamd_http_message *msg)
+{
+ GHashTable *query_args;
+ GHashTableIter it;
+ struct http_parser_url u;
+ const gchar *p;
+ gsize pathlen;
+ rspamd_ftok_t *key, *value;
+ gpointer k, v;
+
+ if (msg->url == NULL || msg->url->len == 0) {
+ g_set_error(&task->err, rspamd_protocol_quark(), 400, "missing command");
+ return FALSE;
+ }
+
+ if (http_parser_parse_url(msg->url->str, msg->url->len, 0, &u) != 0) {
+ g_set_error(&task->err, rspamd_protocol_quark(), 400, "bad request URL");
+
+ return FALSE;
+ }
+
+ if (!(u.field_set & (1 << UF_PATH))) {
+ g_set_error(&task->err, rspamd_protocol_quark(), 400,
+ "bad request URL: missing path");
+
+ return FALSE;
+ }
+
+ p = msg->url->str + u.field_data[UF_PATH].off;
+ pathlen = u.field_data[UF_PATH].len;
+
+ if (*p == '/') {
+ p++;
+ pathlen--;
+ }
+
+ switch (*p) {
+ case 'c':
+ case 'C':
+ /* check */
+ if (COMPARE_CMD(p, MSG_CMD_CHECK_V2, pathlen)) {
+ task->cmd = CMD_CHECK_V2;
+ msg_debug_protocol("got checkv2 command");
+ }
+ else if (COMPARE_CMD(p, MSG_CMD_CHECK, pathlen)) {
+ task->cmd = CMD_CHECK;
+ msg_debug_protocol("got check command");
+ }
+ else {
+ goto err;
+ }
+ break;
+ case 's':
+ case 'S':
+ /* symbols, skip */
+ if (COMPARE_CMD(p, MSG_CMD_SYMBOLS, pathlen)) {
+ task->cmd = CMD_CHECK;
+ msg_debug_protocol("got symbols -> old check command");
+ }
+ else if (COMPARE_CMD(p, MSG_CMD_SCAN, pathlen)) {
+ task->cmd = CMD_CHECK;
+ msg_debug_protocol("got scan -> old check command");
+ }
+ else if (COMPARE_CMD(p, MSG_CMD_SKIP, pathlen)) {
+ msg_debug_protocol("got skip command");
+ task->cmd = CMD_SKIP;
+ }
+ else {
+ goto err;
+ }
+ break;
+ case 'p':
+ case 'P':
+ /* ping, process */
+ if (COMPARE_CMD(p, MSG_CMD_PING, pathlen)) {
+ msg_debug_protocol("got ping command");
+ task->cmd = CMD_PING;
+ task->flags |= RSPAMD_TASK_FLAG_SKIP;
+ task->processed_stages |= RSPAMD_TASK_STAGE_DONE; /* Skip all */
+ }
+ else if (COMPARE_CMD(p, MSG_CMD_PROCESS, pathlen)) {
+ msg_debug_protocol("got process -> old check command");
+ task->cmd = CMD_CHECK;
+ }
+ else {
+ goto err;
+ }
+ break;
+ case 'r':
+ case 'R':
+ /* report, report_ifspam */
+ if (COMPARE_CMD(p, MSG_CMD_REPORT, pathlen)) {
+ msg_debug_protocol("got report -> old check command");
+ task->cmd = CMD_CHECK;
+ }
+ else if (COMPARE_CMD(p, MSG_CMD_REPORT_IFSPAM, pathlen)) {
+ msg_debug_protocol("got reportifspam -> old check command");
+ task->cmd = CMD_CHECK;
+ }
+ else {
+ goto err;
+ }
+ break;
+ default:
+ goto err;
+ }
+
+ if (u.field_set & (1u << UF_QUERY)) {
+ /* In case if we have a query, we need to store it somewhere */
+ query_args = rspamd_http_message_parse_query(msg);
+
+ /* Insert the rest of query params as HTTP headers */
+ g_hash_table_iter_init(&it, query_args);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ gchar *key_cpy;
+ key = k;
+ value = v;
+
+ key_cpy = rspamd_mempool_ftokdup(task->task_pool, key);
+
+ rspamd_http_message_add_header_len(msg, key_cpy,
+ value->begin, value->len);
+ msg_debug_protocol("added header \"%T\" -> \"%T\" from HTTP query",
+ key, value);
+ }
+
+ g_hash_table_unref(query_args);
+ }
+
+ return TRUE;
+
+err:
+ g_set_error(&task->err, rspamd_protocol_quark(), 400, "invalid command");
+
+ return FALSE;
+}
+
+static void
+rspamd_protocol_process_recipients(struct rspamd_task *task,
+ const rspamd_ftok_t *hdr)
+{
+ enum {
+ skip_spaces,
+ quoted_string,
+ normal_string,
+ } state = skip_spaces;
+ const gchar *p, *end, *start_addr;
+ struct rspamd_email_address *addr;
+
+ p = hdr->begin;
+ end = hdr->begin + hdr->len;
+ start_addr = NULL;
+
+ while (p < end) {
+ switch (state) {
+ case skip_spaces:
+ if (g_ascii_isspace(*p)) {
+ p++;
+ }
+ else if (*p == '"') {
+ start_addr = p;
+ p++;
+ state = quoted_string;
+ }
+ else {
+ state = normal_string;
+ start_addr = p;
+ }
+ break;
+ case quoted_string:
+ if (*p == '"') {
+ state = normal_string;
+ p++;
+ }
+ else if (*p == '\\') {
+ /* Quoted pair */
+ p += 2;
+ }
+ else {
+ p++;
+ }
+ break;
+ case normal_string:
+ if (*p == '"') {
+ state = quoted_string;
+ p++;
+ }
+ else if (*p == ',' && start_addr != NULL && p > start_addr) {
+ /* We have finished address, check what we have */
+ addr = rspamd_email_address_from_smtp(start_addr,
+ p - start_addr);
+
+ if (addr) {
+ if (task->rcpt_envelope == NULL) {
+ task->rcpt_envelope = g_ptr_array_sized_new(
+ 2);
+ }
+
+ g_ptr_array_add(task->rcpt_envelope, addr);
+ }
+ else {
+ msg_err_protocol("bad rcpt address: '%*s'",
+ (int) (p - start_addr), start_addr);
+ task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
+ }
+ start_addr = NULL;
+ p++;
+ state = skip_spaces;
+ }
+ else {
+ p++;
+ }
+ break;
+ }
+ }
+
+ /* Check remainder */
+ if (start_addr && p > start_addr) {
+ switch (state) {
+ case normal_string:
+ addr = rspamd_email_address_from_smtp(start_addr, end - start_addr);
+
+ if (addr) {
+ if (task->rcpt_envelope == NULL) {
+ task->rcpt_envelope = g_ptr_array_sized_new(
+ 2);
+ }
+
+ g_ptr_array_add(task->rcpt_envelope, addr);
+ }
+ else {
+ msg_err_protocol("bad rcpt address: '%*s'",
+ (int) (end - start_addr), start_addr);
+ task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
+ }
+ break;
+ case skip_spaces:
+ /* Do nothing */
+ break;
+ case quoted_string:
+ default:
+ msg_err_protocol("bad state when parsing rcpt address: '%*s'",
+ (int) (end - start_addr), start_addr);
+ task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
+ }
+ }
+}
+
+#define COMPARE_FLAG_LIT(lit) (len == sizeof(lit) - 1 && memcmp((lit), str, len) == 0)
+#define CHECK_PROTOCOL_FLAG(lit, fl) \
+ do { \
+ if (!known && COMPARE_FLAG_LIT(lit)) { \
+ task->protocol_flags |= (fl); \
+ known = TRUE; \
+ msg_debug_protocol("add protocol flag %s", lit); \
+ } \
+ } while (0)
+#define CHECK_TASK_FLAG(lit, fl) \
+ do { \
+ if (!known && COMPARE_FLAG_LIT(lit)) { \
+ task->flags |= (fl); \
+ known = TRUE; \
+ msg_debug_protocol("add task flag %s", lit); \
+ } \
+ } while (0)
+
+static void
+rspamd_protocol_handle_flag(struct rspamd_task *task, const gchar *str,
+ gsize len)
+{
+ gboolean known = FALSE;
+
+ CHECK_TASK_FLAG("pass_all", RSPAMD_TASK_FLAG_PASS_ALL);
+ CHECK_TASK_FLAG("no_log", RSPAMD_TASK_FLAG_NO_LOG);
+ CHECK_TASK_FLAG("skip", RSPAMD_TASK_FLAG_SKIP);
+ CHECK_TASK_FLAG("skip_process", RSPAMD_TASK_FLAG_SKIP_PROCESS);
+ CHECK_TASK_FLAG("no_stat", RSPAMD_TASK_FLAG_NO_STAT);
+ CHECK_TASK_FLAG("ssl", RSPAMD_TASK_FLAG_SSL);
+ CHECK_TASK_FLAG("profile", RSPAMD_TASK_FLAG_PROFILE);
+
+ CHECK_PROTOCOL_FLAG("milter", RSPAMD_TASK_PROTOCOL_FLAG_MILTER);
+ CHECK_PROTOCOL_FLAG("zstd", RSPAMD_TASK_PROTOCOL_FLAG_COMPRESSED);
+ CHECK_PROTOCOL_FLAG("ext_urls", RSPAMD_TASK_PROTOCOL_FLAG_EXT_URLS);
+ CHECK_PROTOCOL_FLAG("body_block", RSPAMD_TASK_PROTOCOL_FLAG_BODY_BLOCK);
+ CHECK_PROTOCOL_FLAG("groups", RSPAMD_TASK_PROTOCOL_FLAG_GROUPS);
+
+ if (!known) {
+ msg_warn_protocol("unknown flag: %*s", (gint) len, str);
+ }
+}
+
+#undef COMPARE_FLAG
+#undef CHECK_PROTOCOL_FLAG
+
+static void
+rspamd_protocol_process_flags(struct rspamd_task *task, const rspamd_ftok_t *hdr)
+{
+ enum {
+ skip_spaces,
+ read_flag,
+ } state = skip_spaces;
+ const gchar *p, *end, *start;
+
+ p = hdr->begin;
+ end = hdr->begin + hdr->len;
+ start = NULL;
+
+ while (p < end) {
+ switch (state) {
+ case skip_spaces:
+ if (g_ascii_isspace(*p)) {
+ p++;
+ }
+ else {
+ state = read_flag;
+ start = p;
+ }
+ break;
+ case read_flag:
+ if (*p == ',') {
+ if (p > start) {
+ rspamd_protocol_handle_flag(task, start, p - start);
+ }
+ start = NULL;
+ state = skip_spaces;
+ p++;
+ }
+ else {
+ p++;
+ }
+ break;
+ }
+ }
+
+ /* Check remainder */
+ if (start && end > start && state == read_flag) {
+ rspamd_protocol_handle_flag(task, start, end - start);
+ }
+}
+
+#define IF_HEADER(name) \
+ srch.begin = (name); \
+ srch.len = sizeof(name) - 1; \
+ if (rspamd_ftok_casecmp(hn_tok, &srch) == 0)
+
+gboolean
+rspamd_protocol_handle_headers(struct rspamd_task *task,
+ struct rspamd_http_message *msg)
+{
+ rspamd_ftok_t *hn_tok, *hv_tok, srch;
+ gboolean has_ip = FALSE, seen_settings_header = FALSE;
+ struct rspamd_http_header *header, *h;
+ gchar *ntok;
+
+ kh_foreach_value (msg->headers, header, {
+ DL_FOREACH (header, h) {
+ ntok = rspamd_mempool_ftokdup (task->task_pool, &h->name);
+ hn_tok = rspamd_mempool_alloc (task->task_pool, sizeof (*hn_tok));
+ hn_tok->begin = ntok;
+ hn_tok->len = h->name.len;
+
+
+ ntok = rspamd_mempool_ftokdup (task->task_pool, &h->value);
+ hv_tok = rspamd_mempool_alloc (task->task_pool, sizeof (*hv_tok));
+ hv_tok->begin = ntok;
+ hv_tok->len = h->value.len;
+
+ switch (*hn_tok->begin) {
+ case 'd':
+ case 'D':
+ IF_HEADER(DELIVER_TO_HEADER)
+ {
+ task->deliver_to = rspamd_protocol_escape_braces(task, hv_tok);
+ msg_debug_protocol("read deliver-to header, value: %s",
+ task->deliver_to);
+ }
+ else
+ {
+ msg_debug_protocol("wrong header: %T", hn_tok);
+ }
+ break;
+ case 'h':
+ case 'H':
+ IF_HEADER(HELO_HEADER)
+ {
+ task->helo = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
+ msg_debug_protocol("read helo header, value: %s", task->helo);
+ }
+ IF_HEADER(HOSTNAME_HEADER)
+ {
+ task->hostname = rspamd_mempool_ftokdup(task->task_pool,
+ hv_tok);
+ msg_debug_protocol("read hostname header, value: %s", task->hostname);
+ }
+ break;
+ case 'f':
+ case 'F':
+ IF_HEADER(FROM_HEADER)
+ {
+ if (hv_tok->len == 0) {
+ /* Replace '' with '<>' to fix parsing issue */
+ RSPAMD_FTOK_ASSIGN(hv_tok, "<>");
+ }
+ task->from_envelope = rspamd_email_address_from_smtp(
+ hv_tok->begin,
+ hv_tok->len);
+ msg_debug_protocol("read from header, value: %T", hv_tok);
+
+ if (!task->from_envelope) {
+ msg_err_protocol("bad from header: '%T'", hv_tok);
+ task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
+ }
+ }
+ IF_HEADER(FILENAME_HEADER)
+ {
+ task->msg.fpath = rspamd_mempool_ftokdup(task->task_pool,
+ hv_tok);
+ msg_debug_protocol("read filename header, value: %s", task->msg.fpath);
+ }
+ IF_HEADER(FLAGS_HEADER)
+ {
+ msg_debug_protocol("read flags header, value: %T", hv_tok);
+ rspamd_protocol_process_flags(task, hv_tok);
+ }
+ break;
+ case 'q':
+ case 'Q':
+ IF_HEADER(QUEUE_ID_HEADER)
+ {
+ task->queue_id = rspamd_mempool_ftokdup(task->task_pool,
+ hv_tok);
+ msg_debug_protocol("read queue_id header, value: %s", task->queue_id);
+ }
+ else
+ {
+ msg_debug_protocol("wrong header: %T", hn_tok);
+ }
+ break;
+ case 'r':
+ case 'R':
+ IF_HEADER(RCPT_HEADER)
+ {
+ rspamd_protocol_process_recipients(task, hv_tok);
+ msg_debug_protocol("read rcpt header, value: %T", hv_tok);
+ }
+ IF_HEADER(RAW_DATA_HEADER)
+ {
+ srch.begin = "yes";
+ srch.len = 3;
+
+ msg_debug_protocol("read raw data header, value: %T", hv_tok);
+
+ if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
+ task->flags &= ~RSPAMD_TASK_FLAG_MIME;
+ msg_debug_protocol("disable mime parsing");
+ }
+ }
+ break;
+ case 'i':
+ case 'I':
+ IF_HEADER(IP_ADDR_HEADER)
+ {
+ if (!rspamd_parse_inet_address(&task->from_addr,
+ hv_tok->begin, hv_tok->len,
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ msg_err_protocol("bad ip header: '%T'", hv_tok);
+ }
+ else {
+ msg_debug_protocol("read IP header, value: %T", hv_tok);
+ has_ip = TRUE;
+ }
+ }
+ else
+ {
+ msg_debug_protocol("wrong header: %T", hn_tok);
+ }
+ break;
+ case 'p':
+ case 'P':
+ IF_HEADER(PASS_HEADER)
+ {
+ srch.begin = "all";
+ srch.len = 3;
+
+ msg_debug_protocol("read pass header, value: %T", hv_tok);
+
+ if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
+ task->flags |= RSPAMD_TASK_FLAG_PASS_ALL;
+ msg_debug_protocol("pass all filters");
+ }
+ }
+ IF_HEADER(PROFILE_HEADER)
+ {
+ msg_debug_protocol("read profile header, value: %T", hv_tok);
+ task->flags |= RSPAMD_TASK_FLAG_PROFILE;
+ }
+ break;
+ case 's':
+ case 'S':
+ IF_HEADER(SETTINGS_ID_HEADER)
+ {
+ msg_debug_protocol("read settings-id header, value: %T", hv_tok);
+ task->settings_elt = rspamd_config_find_settings_name_ref(
+ task->cfg, hv_tok->begin, hv_tok->len);
+
+ if (task->settings_elt == NULL) {
+ GString *known_ids = g_string_new(NULL);
+ struct rspamd_config_settings_elt *cur;
+
+ DL_FOREACH(task->cfg->setting_ids, cur)
+ {
+ rspamd_printf_gstring(known_ids, "%s(%ud);",
+ cur->name, cur->id);
+ }
+
+ msg_warn_protocol("unknown settings id: %T(%d); known_ids: %v",
+ hv_tok,
+ rspamd_config_name_to_id(hv_tok->begin, hv_tok->len),
+ known_ids);
+
+ g_string_free(known_ids, TRUE);
+ }
+ else {
+ msg_debug_protocol("applied settings id %T -> %ud", hv_tok,
+ task->settings_elt->id);
+ }
+ }
+ IF_HEADER(SETTINGS_HEADER)
+ {
+ msg_debug_protocol("read settings header, value: %T", hv_tok);
+ seen_settings_header = TRUE;
+ }
+ break;
+ case 'u':
+ case 'U':
+ IF_HEADER(USER_HEADER)
+ {
+ /*
+ * We must ignore User header in case of spamc, as SA has
+ * different meaning of this header
+ */
+ msg_debug_protocol("read user header, value: %T", hv_tok);
+ if (!RSPAMD_TASK_IS_SPAMC(task)) {
+ task->auth_user = rspamd_mempool_ftokdup(task->task_pool,
+ hv_tok);
+ }
+ else {
+ msg_info_protocol("ignore user header: legacy SA protocol");
+ }
+ }
+ IF_HEADER(URLS_HEADER)
+ {
+ msg_debug_protocol("read urls header, value: %T", hv_tok);
+
+ srch.begin = "extended";
+ srch.len = 8;
+
+ if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
+ task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_EXT_URLS;
+ msg_debug_protocol("extended urls information");
+ }
+
+ /* TODO: add more formats there */
+ }
+ IF_HEADER(USER_AGENT_HEADER)
+ {
+ msg_debug_protocol("read user-agent header, value: %T", hv_tok);
+
+ if (hv_tok->len == 6 &&
+ rspamd_lc_cmp(hv_tok->begin, "rspamc", 6) == 0) {
+ task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_LOCAL_CLIENT;
+ }
+ }
+ break;
+ case 'l':
+ case 'L':
+ IF_HEADER(NO_LOG_HEADER)
+ {
+ msg_debug_protocol("read log header, value: %T", hv_tok);
+ srch.begin = "no";
+ srch.len = 2;
+
+ if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
+ task->flags |= RSPAMD_TASK_FLAG_NO_LOG;
+ }
+ }
+ break;
+ case 'm':
+ case 'M':
+ IF_HEADER(MLEN_HEADER)
+ {
+ msg_debug_protocol("read message length header, value: %T",
+ hv_tok);
+ task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_HAS_CONTROL;
+ }
+ IF_HEADER(MTA_TAG_HEADER)
+ {
+ gchar *mta_tag;
+ mta_tag = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_MTA_TAG,
+ mta_tag, NULL);
+ msg_debug_protocol("read MTA-Tag header, value: %s", mta_tag);
+ }
+ IF_HEADER(MTA_NAME_HEADER)
+ {
+ gchar *mta_name;
+ mta_name = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_MTA_NAME,
+ mta_name, NULL);
+ msg_debug_protocol("read MTA-Name header, value: %s", mta_name);
+ }
+ IF_HEADER(MILTER_HEADER)
+ {
+ task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_MILTER;
+ msg_debug_protocol("read Milter header, value: %T", hv_tok);
+ }
+ break;
+ case 't':
+ case 'T':
+ IF_HEADER(TLS_CIPHER_HEADER)
+ {
+ task->flags |= RSPAMD_TASK_FLAG_SSL;
+ msg_debug_protocol("read TLS cipher header, value: %T", hv_tok);
+ }
+ break;
+ default:
+ msg_debug_protocol("generic header: %T", hn_tok);
+ break;
+ }
+
+ rspamd_task_add_request_header (task, hn_tok, hv_tok);
+}
+}); /* End of kh_foreach_value */
+
+if (seen_settings_header && task->settings_elt) {
+ msg_warn_task("ignore settings id %s as settings header is also presented",
+ task->settings_elt->name);
+ REF_RELEASE(task->settings_elt);
+
+ task->settings_elt = NULL;
+}
+
+if (!has_ip) {
+ task->flags |= RSPAMD_TASK_FLAG_NO_IP;
+}
+
+return TRUE;
+}
+
+#define BOOL_TO_FLAG(val, flags, flag) \
+ do { \
+ if ((val)) (flags) |= (flag); \
+ else \
+ (flags) &= ~(flag); \
+ } while (0)
+
+gboolean
+rspamd_protocol_parse_task_flags(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ struct rspamd_rcl_struct_parser *pd = ud;
+ gint *target;
+ const gchar *key;
+ gboolean value;
+
+ target = (gint *) (((gchar *) pd->user_struct) + pd->offset);
+ key = ucl_object_key(obj);
+ value = ucl_object_toboolean(obj);
+
+ if (key != NULL) {
+ if (g_ascii_strcasecmp(key, "pass_all") == 0) {
+ BOOL_TO_FLAG(value, *target, RSPAMD_TASK_FLAG_PASS_ALL);
+ }
+ else if (g_ascii_strcasecmp(key, "no_log") == 0) {
+ BOOL_TO_FLAG(value, *target, RSPAMD_TASK_FLAG_NO_LOG);
+ }
+ }
+
+ return TRUE;
+}
+
+static struct rspamd_rcl_sections_map *control_parser = NULL;
+
+RSPAMD_CONSTRUCTOR(rspamd_protocol_control_parser_ctor)
+{
+
+ struct rspamd_rcl_section *sub = rspamd_rcl_add_section(&control_parser, NULL,
+ "*",
+ NULL,
+ NULL,
+ UCL_OBJECT,
+ FALSE,
+ TRUE);
+ /* Default handlers */
+ rspamd_rcl_add_default_handler(sub,
+ "ip",
+ rspamd_rcl_parse_struct_addr,
+ G_STRUCT_OFFSET(struct rspamd_task, from_addr),
+ 0,
+ NULL);
+ rspamd_rcl_add_default_handler(sub,
+ "from",
+ rspamd_rcl_parse_struct_mime_addr,
+ G_STRUCT_OFFSET(struct rspamd_task, from_envelope),
+ 0,
+ NULL);
+ rspamd_rcl_add_default_handler(sub,
+ "rcpt",
+ rspamd_rcl_parse_struct_mime_addr,
+ G_STRUCT_OFFSET(struct rspamd_task, rcpt_envelope),
+ 0,
+ NULL);
+ rspamd_rcl_add_default_handler(sub,
+ "helo",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_task, helo),
+ 0,
+ NULL);
+ rspamd_rcl_add_default_handler(sub,
+ "user",
+ rspamd_rcl_parse_struct_string,
+ G_STRUCT_OFFSET(struct rspamd_task, auth_user),
+ 0,
+ NULL);
+ rspamd_rcl_add_default_handler(sub,
+ "pass_all",
+ rspamd_protocol_parse_task_flags,
+ G_STRUCT_OFFSET(struct rspamd_task, flags),
+ 0,
+ NULL);
+ rspamd_rcl_add_default_handler(sub,
+ "json",
+ rspamd_protocol_parse_task_flags,
+ G_STRUCT_OFFSET(struct rspamd_task, flags),
+ 0,
+ NULL);
+}
+
+RSPAMD_DESTRUCTOR(rspamd_protocol_control_parser_dtor)
+{
+ rspamd_rcl_sections_free(control_parser);
+}
+
+gboolean
+rspamd_protocol_handle_control(struct rspamd_task *task,
+ const ucl_object_t *control)
+{
+ GError *err = NULL;
+
+ if (!rspamd_rcl_parse(control_parser, task->cfg, task, task->task_pool,
+ control, &err)) {
+ msg_warn_protocol("cannot parse control block: %e", err);
+ g_error_free(err);
+
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_protocol_handle_request(struct rspamd_task *task,
+ struct rspamd_http_message *msg)
+{
+ gboolean ret = TRUE;
+
+ if (msg->method == HTTP_SYMBOLS) {
+ msg_debug_protocol("got legacy SYMBOLS method, enable rspamc protocol workaround");
+ task->cmd = CMD_CHECK_RSPAMC;
+ }
+ else if (msg->method == HTTP_CHECK) {
+ msg_debug_protocol("got legacy CHECK method, enable rspamc protocol workaround");
+ task->cmd = CMD_CHECK_RSPAMC;
+ }
+ else {
+ ret = rspamd_protocol_handle_url(task, msg);
+ }
+
+ if (msg->flags & RSPAMD_HTTP_FLAG_SPAMC) {
+ msg_debug_protocol("got legacy SA input, enable spamc protocol workaround");
+ task->cmd = CMD_CHECK_SPAMC;
+ }
+
+ return ret;
+}
+
+/* Structure for writing tree data */
+struct tree_cb_data {
+ ucl_object_t *top;
+ khash_t(rspamd_url_host_hash) * seen;
+ struct rspamd_task *task;
+};
+
+static ucl_object_t *
+rspamd_protocol_extended_url(struct rspamd_task *task,
+ struct rspamd_url *url,
+ const gchar *encoded, gsize enclen)
+{
+ ucl_object_t *obj, *elt;
+
+ obj = ucl_object_typed_new(UCL_OBJECT);
+
+ elt = ucl_object_fromstring_common(encoded, enclen, 0);
+ ucl_object_insert_key(obj, elt, "url", 0, false);
+
+ if (url->tldlen > 0) {
+ elt = ucl_object_fromstring_common(rspamd_url_tld_unsafe(url),
+ url->tldlen, 0);
+ ucl_object_insert_key(obj, elt, "tld", 0, false);
+ }
+ if (url->hostlen > 0) {
+ elt = ucl_object_fromstring_common(rspamd_url_host_unsafe(url),
+ url->hostlen, 0);
+ ucl_object_insert_key(obj, elt, "host", 0, false);
+ }
+
+ ucl_object_t *flags = ucl_object_typed_new(UCL_ARRAY);
+
+ for (unsigned int i = 0; i < RSPAMD_URL_MAX_FLAG_SHIFT; i++) {
+ if (url->flags & (1u << i)) {
+ ucl_object_t *fl = ucl_object_fromstring(rspamd_url_flag_to_string(1u << i));
+ ucl_array_append(flags, fl);
+ }
+ }
+
+ ucl_object_insert_key(obj, flags, "flags", 0, false);
+
+ if (url->ext && url->ext->linked_url) {
+ encoded = rspamd_url_encode(url->ext->linked_url, &enclen, task->task_pool);
+ elt = rspamd_protocol_extended_url(task, url->ext->linked_url, encoded,
+ enclen);
+ ucl_object_insert_key(obj, elt, "linked_url", 0, false);
+ }
+
+ return obj;
+}
+
+/*
+ * Callback for writing urls
+ */
+static void
+urls_protocol_cb(struct rspamd_url *url, struct tree_cb_data *cb)
+{
+ ucl_object_t *obj;
+ struct rspamd_task *task = cb->task;
+ const gchar *user_field = "unknown", *encoded = NULL;
+ gboolean has_user = FALSE;
+ guint len = 0;
+ gsize enclen = 0;
+
+ if (!(task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_EXT_URLS)) {
+ if (url->hostlen > 0) {
+ if (rspamd_url_host_set_has(cb->seen, url)) {
+ return;
+ }
+
+ goffset err_offset;
+
+ if ((err_offset = rspamd_fast_utf8_validate(rspamd_url_host_unsafe(url),
+ url->hostlen)) == 0) {
+ obj = ucl_object_fromstring_common(rspamd_url_host_unsafe(url),
+ url->hostlen, 0);
+ }
+ else {
+ obj = ucl_object_fromstring_common(rspamd_url_host_unsafe(url),
+ err_offset - 1, 0);
+ }
+ }
+ else {
+ return;
+ }
+
+ rspamd_url_host_set_add(cb->seen, url);
+ }
+ else {
+ encoded = rspamd_url_encode(url, &enclen, task->task_pool);
+ obj = rspamd_protocol_extended_url(task, url, encoded, enclen);
+ }
+
+ ucl_array_append(cb->top, obj);
+
+ if (cb->task->cfg->log_urls) {
+ if (task->auth_user) {
+ user_field = task->auth_user;
+ len = strlen(task->auth_user);
+ has_user = TRUE;
+ }
+ else if (task->from_envelope) {
+ user_field = task->from_envelope->addr;
+ len = task->from_envelope->addr_len;
+ }
+
+ if (!encoded) {
+ encoded = rspamd_url_encode(url, &enclen, task->task_pool);
+ }
+
+ msg_notice_task_encrypted("<%s> %s: %*s; ip: %s; URL: %*s",
+ MESSAGE_FIELD_CHECK(task, message_id),
+ has_user ? "user" : "from",
+ len, user_field,
+ rspamd_inet_address_to_string(task->from_addr),
+ (gint) enclen, encoded);
+ }
+}
+
+static ucl_object_t *
+rspamd_urls_tree_ucl(khash_t(rspamd_url_hash) * set,
+ struct rspamd_task *task)
+{
+ struct tree_cb_data cb;
+ ucl_object_t *obj;
+ struct rspamd_url *u;
+
+ obj = ucl_object_typed_new(UCL_ARRAY);
+ cb.top = obj;
+ cb.task = task;
+ cb.seen = kh_init(rspamd_url_host_hash);
+
+ kh_foreach_key(set, u, {
+ if (!(u->protocol & PROTOCOL_MAILTO)) {
+ urls_protocol_cb(u, &cb);
+ }
+ });
+
+ kh_destroy(rspamd_url_host_hash, cb.seen);
+
+ return obj;
+}
+
+static void
+emails_protocol_cb(struct rspamd_url *url, struct tree_cb_data *cb)
+{
+ ucl_object_t *obj;
+
+ if (url->userlen > 0 && url->hostlen > 0) {
+ obj = ucl_object_fromlstring(rspamd_url_user_unsafe(url),
+ url->userlen + url->hostlen + 1);
+ ucl_array_append(cb->top, obj);
+ }
+}
+
+static ucl_object_t *
+rspamd_emails_tree_ucl(khash_t(rspamd_url_hash) * set,
+ struct rspamd_task *task)
+{
+ struct tree_cb_data cb;
+ ucl_object_t *obj;
+ struct rspamd_url *u;
+
+ obj = ucl_object_typed_new(UCL_ARRAY);
+ cb.top = obj;
+ cb.task = task;
+
+ kh_foreach_key(set, u, {
+ if ((u->protocol & PROTOCOL_MAILTO)) {
+ emails_protocol_cb(u, &cb);
+ }
+ });
+
+
+ return obj;
+}
+
+
+/* Write new subject */
+static const gchar *
+rspamd_protocol_rewrite_subject(struct rspamd_task *task)
+{
+ GString *subj_buf;
+ gchar *res;
+ const gchar *s, *c, *p;
+ gsize slen = 0;
+
+ c = rspamd_mempool_get_variable(task->task_pool, "metric_subject");
+
+ if (c == NULL) {
+ c = task->cfg->subject;
+ }
+
+ if (c == NULL) {
+ c = SPAM_SUBJECT;
+ }
+
+ p = c;
+ s = MESSAGE_FIELD_CHECK(task, subject);
+
+ if (s) {
+ slen = strlen(s);
+ }
+
+ subj_buf = g_string_sized_new(strlen(c) + slen);
+
+ while (*p) {
+ if (*p == '%') {
+ switch (p[1]) {
+ case 's':
+ g_string_append_len(subj_buf, c, p - c);
+
+ if (s) {
+ g_string_append_len(subj_buf, s, slen);
+ }
+ c = p + 2;
+ p += 2;
+ break;
+ case 'd':
+ g_string_append_len(subj_buf, c, p - c);
+ rspamd_printf_gstring(subj_buf, "%.2f", task->result->score);
+ c = p + 2;
+ p += 2;
+ break;
+ case '%':
+ g_string_append_len(subj_buf, c, p - c);
+ g_string_append_c(subj_buf, '%');
+ c = p + 2;
+ p += 2;
+ break;
+ default:
+ p++; /* Just % something unknown */
+ break;
+ }
+ }
+ else {
+ p++;
+ }
+ }
+
+ if (p > c) {
+ g_string_append_len(subj_buf, c, p - c);
+ }
+
+ res = rspamd_mime_header_encode(subj_buf->str, subj_buf->len);
+
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) g_free,
+ res);
+ g_string_free(subj_buf, TRUE);
+
+ return res;
+}
+
+static ucl_object_t *
+rspamd_metric_symbol_ucl(struct rspamd_task *task, struct rspamd_symbol_result *sym)
+{
+ ucl_object_t *obj = NULL, *ar;
+ const gchar *description = NULL;
+ struct rspamd_symbol_option *opt;
+
+ if (sym->sym != NULL) {
+ description = sym->sym->description;
+ }
+
+ obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj, ucl_object_fromstring(sym->name), "name", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromdouble(sym->score), "score", 0, false);
+
+ if (task->cmd == CMD_CHECK_V2) {
+ if (sym->sym) {
+ ucl_object_insert_key(obj, ucl_object_fromdouble(sym->sym->score), "metric_score", 0, false);
+ }
+ else {
+ ucl_object_insert_key(obj, ucl_object_fromdouble(0.0),
+ "metric_score", 0, false);
+ }
+ }
+
+ if (description) {
+ ucl_object_insert_key(obj, ucl_object_fromstring(description),
+ "description", 0, false);
+ }
+
+ if (sym->options != NULL) {
+ ar = ucl_object_typed_new(UCL_ARRAY);
+
+ DL_FOREACH(sym->opts_head, opt)
+ {
+ ucl_array_append(ar, ucl_object_fromstring_common(opt->option,
+ opt->optlen, 0));
+ }
+
+ ucl_object_insert_key(obj, ar, "options", 0, false);
+ }
+
+ return obj;
+}
+
+static ucl_object_t *
+rspamd_metric_group_ucl(struct rspamd_task *task,
+ struct rspamd_symbols_group *gr, gdouble score)
+{
+ ucl_object_t *obj = NULL;
+
+ obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj, ucl_object_fromdouble(score),
+ "score", 0, false);
+
+ if (gr->description) {
+ ucl_object_insert_key(obj, ucl_object_fromstring(gr->description),
+ "description", 0, false);
+ }
+
+ return obj;
+}
+
+static ucl_object_t *
+rspamd_scan_result_ucl(struct rspamd_task *task,
+ struct rspamd_scan_result *mres, ucl_object_t *top)
+{
+ struct rspamd_symbol_result *sym;
+ gboolean is_spam;
+ struct rspamd_action *action;
+ ucl_object_t *obj = NULL, *sobj;
+ const gchar *subject;
+ struct rspamd_passthrough_result *pr = NULL;
+
+ action = rspamd_check_action_metric(task, &pr, NULL);
+ is_spam = !(action->flags & RSPAMD_ACTION_HAM);
+
+ if (task->cmd == CMD_CHECK) {
+ obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj,
+ ucl_object_frombool(is_spam),
+ "is_spam", 0, false);
+ }
+ else {
+ obj = top;
+ }
+
+ if (pr) {
+ if (pr->message && !(pr->flags & RSPAMD_PASSTHROUGH_NO_SMTP_MESSAGE)) {
+ /* Add smtp message if it does not exist: see #3269 for details */
+ if (ucl_object_lookup(task->messages, "smtp_message") == NULL) {
+ ucl_object_insert_key(task->messages,
+ ucl_object_fromstring_common(pr->message, 0, UCL_STRING_RAW),
+ "smtp_message", 0,
+ false);
+ }
+ }
+
+ ucl_object_insert_key(obj,
+ ucl_object_fromstring(pr->module),
+ "passthrough_module", 0, false);
+ }
+
+ ucl_object_insert_key(obj,
+ ucl_object_frombool(RSPAMD_TASK_IS_SKIPPED(task)),
+ "is_skipped", 0, false);
+
+ if (!isnan(mres->score)) {
+ ucl_object_insert_key(obj, ucl_object_fromdouble(mres->score),
+ "score", 0, false);
+ }
+ else {
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(0.0), "score", 0, false);
+ }
+
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(rspamd_task_get_required_score(task, mres)),
+ "required_score", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromstring(action->name),
+ "action", 0, false);
+
+ if (action->action_type == METRIC_ACTION_REWRITE_SUBJECT) {
+ subject = rspamd_protocol_rewrite_subject(task);
+
+ if (subject) {
+ ucl_object_insert_key(obj, ucl_object_fromstring(subject),
+ "subject", 0, false);
+ }
+ }
+ if (action->flags & RSPAMD_ACTION_MILTER) {
+ /* Treat milter action specially */
+ if (action->action_type == METRIC_ACTION_DISCARD) {
+ ucl_object_insert_key(obj, ucl_object_fromstring("discard"),
+ "reject", 0, false);
+ }
+ else if (action->action_type == METRIC_ACTION_QUARANTINE) {
+ ucl_object_insert_key(obj, ucl_object_fromstring("quarantine"),
+ "reject", 0, false);
+ }
+ }
+
+ /* Now handle symbols */
+ if (task->cmd != CMD_CHECK) {
+ /* Insert actions thresholds */
+ ucl_object_t *actions_obj = ucl_object_typed_new(UCL_OBJECT);
+
+ for (int i = task->result->nactions - 1; i >= 0; i--) {
+ struct rspamd_action_config *action_lim = &task->result->actions_config[i];
+
+ if (!isnan(action_lim->cur_limit) &&
+ !(action_lim->action->flags & (RSPAMD_ACTION_NO_THRESHOLD | RSPAMD_ACTION_HAM))) {
+ ucl_object_insert_key(actions_obj, ucl_object_fromdouble(action_lim->cur_limit),
+ action_lim->action->name, 0, true);
+ }
+ }
+
+ ucl_object_insert_key(obj, actions_obj, "thresholds", 0, false);
+
+ /* For checkv2 we insert symbols as a separate object */
+ obj = ucl_object_typed_new(UCL_OBJECT);
+ }
+
+ kh_foreach_value(mres->symbols, sym, {
+ if (!(sym->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ sobj = rspamd_metric_symbol_ucl(task, sym);
+ ucl_object_insert_key(obj, sobj, sym->name, 0, false);
+ }
+ })
+
+ if (task->cmd != CMD_CHECK)
+ {
+ /* For checkv2 we insert symbols as a separate object */
+ ucl_object_insert_key(top, obj, "symbols", 0, false);
+ }
+ else
+ {
+ /* For legacy check we just insert it as "default" all together */
+ ucl_object_insert_key(top, obj, DEFAULT_METRIC, 0, false);
+ }
+
+ /* Handle groups if needed */
+ if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_GROUPS) {
+ struct rspamd_symbols_group *gr;
+ gdouble gr_score;
+
+ obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_reserve(obj, kh_size(mres->sym_groups));
+
+ kh_foreach(mres->sym_groups, gr, gr_score, {
+ if (task->cfg->public_groups_only &&
+ !(gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC)) {
+ continue;
+ }
+ sobj = rspamd_metric_group_ucl(task, gr, gr_score);
+ ucl_object_insert_key(obj, sobj, gr->name, 0, false);
+ });
+
+ ucl_object_insert_key(top, obj, "groups", 0, false);
+ }
+
+ return obj;
+}
+
+void rspamd_ucl_torspamc_output(const ucl_object_t *top,
+ rspamd_fstring_t **out)
+{
+ const ucl_object_t *symbols, *score,
+ *required_score, *is_spam, *elt, *cur;
+ ucl_object_iter_t iter = NULL;
+
+ score = ucl_object_lookup(top, "score");
+ required_score = ucl_object_lookup(top, "required_score");
+ is_spam = ucl_object_lookup(top, "is_spam");
+ rspamd_printf_fstring(out,
+ "Metric: default; %s; %.2f / %.2f / 0.0\r\n",
+ ucl_object_toboolean(is_spam) ? "True" : "False",
+ ucl_object_todouble(score),
+ ucl_object_todouble(required_score));
+ elt = ucl_object_lookup(top, "action");
+ if (elt != NULL) {
+ rspamd_printf_fstring(out, "Action: %s\r\n",
+ ucl_object_tostring(elt));
+ }
+
+ elt = ucl_object_lookup(top, "subject");
+ if (elt != NULL) {
+ rspamd_printf_fstring(out, "Subject: %s\r\n",
+ ucl_object_tostring(elt));
+ }
+
+ symbols = ucl_object_lookup(top, "symbols");
+
+ if (symbols != NULL) {
+ iter = NULL;
+ while ((elt = ucl_object_iterate(symbols, &iter, true)) != NULL) {
+ if (elt->type == UCL_OBJECT) {
+ const ucl_object_t *sym_score;
+ sym_score = ucl_object_lookup(elt, "score");
+ rspamd_printf_fstring(out, "Symbol: %s(%.2f)\r\n",
+ ucl_object_key(elt),
+ ucl_object_todouble(sym_score));
+ }
+ }
+ }
+
+ elt = ucl_object_lookup(top, "messages");
+ if (elt != NULL) {
+ iter = NULL;
+ while ((cur = ucl_object_iterate(elt, &iter, true)) != NULL) {
+ if (cur->type == UCL_STRING) {
+ rspamd_printf_fstring(out, "Message: %s\r\n",
+ ucl_object_tostring(cur));
+ }
+ }
+ }
+
+ elt = ucl_object_lookup(top, "message-id");
+ if (elt != NULL) {
+ rspamd_printf_fstring(out, "Message-ID: %s\r\n",
+ ucl_object_tostring(elt));
+ }
+}
+
+void rspamd_ucl_tospamc_output(const ucl_object_t *top,
+ rspamd_fstring_t **out)
+{
+ const ucl_object_t *symbols, *score,
+ *required_score, *is_spam, *elt;
+ ucl_object_iter_t iter = NULL;
+ rspamd_fstring_t *f;
+
+ score = ucl_object_lookup(top, "score");
+ required_score = ucl_object_lookup(top, "required_score");
+ is_spam = ucl_object_lookup(top, "is_spam");
+ rspamd_printf_fstring(out,
+ "Spam: %s ; %.2f / %.2f\r\n\r\n",
+ ucl_object_toboolean(is_spam) ? "True" : "False",
+ ucl_object_todouble(score),
+ ucl_object_todouble(required_score));
+
+ symbols = ucl_object_lookup(top, "symbols");
+
+ if (symbols != NULL) {
+ while ((elt = ucl_object_iterate(symbols, &iter, true)) != NULL) {
+ if (elt->type == UCL_OBJECT) {
+ rspamd_printf_fstring(out, "%s,",
+ ucl_object_key(elt));
+ }
+ }
+ /* Ugly hack, but the whole spamc is ugly */
+ f = *out;
+ if (f->str[f->len - 1] == ',') {
+ f->len--;
+
+ *out = rspamd_fstring_append(*out, CRLF, 2);
+ }
+ }
+}
+
+static void
+rspamd_protocol_output_profiling(struct rspamd_task *task,
+ ucl_object_t *top)
+{
+ GHashTable *tbl;
+ GHashTableIter it;
+ gpointer k, v;
+ ucl_object_t *prof;
+ gdouble val;
+
+ prof = ucl_object_typed_new(UCL_OBJECT);
+ tbl = rspamd_mempool_get_variable(task->task_pool, "profile");
+
+ if (tbl) {
+ g_hash_table_iter_init(&it, tbl);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ val = *(gdouble *) v;
+ ucl_object_insert_key(prof, ucl_object_fromdouble(val),
+ (const char *) k, 0, false);
+ }
+ }
+
+ ucl_object_insert_key(top, prof, "profile", 0, false);
+}
+
+ucl_object_t *
+rspamd_protocol_write_ucl(struct rspamd_task *task,
+ enum rspamd_protocol_flags flags)
+{
+ ucl_object_t *top = NULL;
+ GString *dkim_sig;
+ GList *dkim_sigs;
+ const ucl_object_t *milter_reply;
+
+ rspamd_task_set_finish_time(task);
+ top = ucl_object_typed_new(UCL_OBJECT);
+
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) ucl_object_unref, top);
+
+ if (flags & RSPAMD_PROTOCOL_METRICS) {
+ rspamd_scan_result_ucl(task, task->result, top);
+ }
+
+ if (flags & RSPAMD_PROTOCOL_MESSAGES) {
+ if (G_UNLIKELY(task->cfg->compat_messages)) {
+ const ucl_object_t *cur;
+ ucl_object_t *msg_object;
+ ucl_object_iter_t iter = NULL;
+
+ msg_object = ucl_object_typed_new(UCL_ARRAY);
+
+ while ((cur = ucl_object_iterate(task->messages, &iter, true)) != NULL) {
+ if (cur->type == UCL_STRING) {
+ ucl_array_append(msg_object, ucl_object_ref(cur));
+ }
+ }
+
+ ucl_object_insert_key(top, msg_object, "messages", 0, false);
+ }
+ else {
+ ucl_object_insert_key(top, ucl_object_ref(task->messages),
+ "messages", 0, false);
+ }
+ }
+
+ if (flags & RSPAMD_PROTOCOL_URLS && task->message) {
+ if (kh_size(MESSAGE_FIELD(task, urls)) > 0) {
+ ucl_object_insert_key(top,
+ rspamd_urls_tree_ucl(MESSAGE_FIELD(task, urls), task),
+ "urls", 0, false);
+ ucl_object_insert_key(top,
+ rspamd_emails_tree_ucl(MESSAGE_FIELD(task, urls), task),
+ "emails", 0, false);
+ }
+ }
+
+ if (flags & RSPAMD_PROTOCOL_EXTRA) {
+ if (G_UNLIKELY(RSPAMD_TASK_IS_PROFILING(task))) {
+ rspamd_protocol_output_profiling(task, top);
+ }
+ }
+
+ if (flags & RSPAMD_PROTOCOL_BASIC) {
+ ucl_object_insert_key(top,
+ ucl_object_fromstring(MESSAGE_FIELD_CHECK(task, message_id)),
+ "message-id", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromdouble(task->time_real_finish - task->task_timestamp),
+ "time_real", 0, false);
+ }
+
+ if (flags & RSPAMD_PROTOCOL_DKIM) {
+ dkim_sigs = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_DKIM_SIGNATURE);
+
+ if (dkim_sigs) {
+ if (dkim_sigs->next) {
+ /* Multiple DKIM signatures */
+ ucl_object_t *ar = ucl_object_typed_new(UCL_ARRAY);
+
+ for (; dkim_sigs != NULL; dkim_sigs = dkim_sigs->next) {
+ GString *folded_header;
+ dkim_sig = (GString *) dkim_sigs->data;
+
+ if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_MILTER ||
+ !task->message) {
+
+ folded_header = rspamd_header_value_fold(
+ "DKIM-Signature", strlen("DKIM-Signature"),
+ dkim_sig->str, dkim_sig->len,
+ 80, RSPAMD_TASK_NEWLINES_LF, NULL);
+ }
+ else {
+ folded_header = rspamd_header_value_fold(
+ "DKIM-Signature", strlen("DKIM-Signature"),
+ dkim_sig->str, dkim_sig->len,
+ 80,
+ MESSAGE_FIELD(task, nlines_type),
+ NULL);
+ }
+
+ ucl_array_append(ar,
+ ucl_object_fromstring_common(folded_header->str,
+ folded_header->len, UCL_STRING_RAW));
+ g_string_free(folded_header, TRUE);
+ }
+
+ ucl_object_insert_key(top,
+ ar,
+ "dkim-signature", 0,
+ false);
+ }
+ else {
+ /* Single DKIM signature */
+ GString *folded_header;
+ dkim_sig = (GString *) dkim_sigs->data;
+
+ if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_MILTER) {
+ folded_header = rspamd_header_value_fold(
+ "DKIM-Signature", strlen("DKIM-Signature"),
+ dkim_sig->str, dkim_sig->len,
+ 80, RSPAMD_TASK_NEWLINES_LF, NULL);
+ }
+ else {
+ folded_header = rspamd_header_value_fold(
+ "DKIM-Signature", strlen("DKIM-Signature"),
+ dkim_sig->str, dkim_sig->len,
+ 80, MESSAGE_FIELD(task, nlines_type),
+ NULL);
+ }
+
+ ucl_object_insert_key(top,
+ ucl_object_fromstring_common(folded_header->str,
+ folded_header->len, UCL_STRING_RAW),
+ "dkim-signature", 0, false);
+ g_string_free(folded_header, TRUE);
+ }
+ }
+ }
+
+ if (flags & RSPAMD_PROTOCOL_RMILTER) {
+ milter_reply = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_MILTER_REPLY);
+
+ if (milter_reply) {
+ if (task->cmd != CMD_CHECK) {
+ ucl_object_insert_key(top, ucl_object_ref(milter_reply),
+ "milter", 0, false);
+ }
+ else {
+ ucl_object_insert_key(top, ucl_object_ref(milter_reply),
+ "rmilter", 0, false);
+ }
+ }
+ }
+
+ return top;
+}
+
+void rspamd_protocol_http_reply(struct rspamd_http_message *msg,
+ struct rspamd_task *task, ucl_object_t **pobj)
+{
+ struct rspamd_scan_result *metric_res;
+ const struct rspamd_re_cache_stat *restat;
+
+ ucl_object_t *top = NULL;
+ rspamd_fstring_t *reply;
+ gint flags = RSPAMD_PROTOCOL_DEFAULT;
+ struct rspamd_action *action;
+
+ /* Removed in 2.0 */
+#if 0
+ GHashTableIter hiter;
+ gpointer h, v;
+ /* Write custom headers */
+ g_hash_table_iter_init (&hiter, task->reply_headers);
+ while (g_hash_table_iter_next (&hiter, &h, &v)) {
+ rspamd_ftok_t *hn = h, *hv = v;
+
+ rspamd_http_message_add_header (msg, hn->begin, hv->begin);
+ }
+#endif
+
+ flags |= RSPAMD_PROTOCOL_URLS;
+
+ top = rspamd_protocol_write_ucl(task, flags);
+
+ if (pobj) {
+ *pobj = top;
+ }
+
+ if (!(task->flags & RSPAMD_TASK_FLAG_NO_LOG)) {
+ rspamd_roll_history_update(task->worker->srv->history, task);
+ }
+ else {
+ msg_debug_protocol("skip history update due to no log flag");
+ }
+
+ rspamd_task_write_log(task);
+
+ if (task->cfg->log_flags & RSPAMD_LOG_FLAG_RE_CACHE) {
+ restat = rspamd_re_cache_get_stat(task->re_rt);
+ g_assert(restat != NULL);
+ msg_notice_task(
+ "regexp statistics: %ud pcre regexps scanned, %ud regexps matched,"
+ " %ud regexps total, %ud regexps cached,"
+ " %HL scanned using pcre, %HL scanned total",
+ restat->regexp_checked,
+ restat->regexp_matched,
+ restat->regexp_total,
+ restat->regexp_fast_cached,
+ restat->bytes_scanned_pcre,
+ restat->bytes_scanned);
+ }
+
+ reply = rspamd_fstring_sized_new(1000);
+
+ if (msg->method < HTTP_SYMBOLS && !RSPAMD_TASK_IS_SPAMC(task)) {
+ msg_debug_protocol("writing json reply");
+ rspamd_ucl_emit_fstring(top, UCL_EMIT_JSON_COMPACT, &reply);
+ }
+ else {
+ if (RSPAMD_TASK_IS_SPAMC(task)) {
+ msg_debug_protocol("writing spamc legacy reply to client");
+ rspamd_ucl_tospamc_output(top, &reply);
+ }
+ else {
+ msg_debug_protocol("writing rspamc legacy reply to client");
+ rspamd_ucl_torspamc_output(top, &reply);
+ }
+ }
+
+ if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_BODY_BLOCK) {
+ /* Check if we need to insert a body block */
+ if (task->flags & RSPAMD_TASK_FLAG_MESSAGE_REWRITE) {
+ GString *hdr_offset = g_string_sized_new(30);
+
+ rspamd_printf_gstring(hdr_offset, "%z", RSPAMD_FSTRING_LEN(reply));
+ rspamd_http_message_add_header(msg, MESSAGE_OFFSET_HEADER,
+ hdr_offset->str);
+ msg_debug_protocol("write body block at position %s",
+ hdr_offset->str);
+ g_string_free(hdr_offset, TRUE);
+
+ /* In case of milter, we append just body, otherwise - full message */
+ if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_MILTER) {
+ const gchar *start;
+ goffset len, hdr_off;
+
+ start = task->msg.begin;
+ len = task->msg.len;
+
+ hdr_off = MESSAGE_FIELD(task, raw_headers_content).len;
+
+ if (hdr_off < len) {
+ start += hdr_off;
+ len -= hdr_off;
+
+ /* The problem here is that we need not end of headers, we need
+ * start of body.
+ *
+ * Hence, we need to skip one \r\n till there is anything else in
+ * a line.
+ */
+
+ if (*start == '\r' && len > 0) {
+ start++;
+ len--;
+ }
+
+ if (*start == '\n' && len > 0) {
+ start++;
+ len--;
+ }
+
+ msg_debug_protocol("milter version of body block size %d",
+ (int) len);
+ reply = rspamd_fstring_append(reply, start, len);
+ }
+ }
+ else {
+ msg_debug_protocol("general version of body block size %d",
+ (int) task->msg.len);
+ reply = rspamd_fstring_append(reply,
+ task->msg.begin, task->msg.len);
+ }
+ }
+ }
+
+ if ((task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_COMPRESSED) &&
+ rspamd_libs_reset_compression(task->cfg->libs_ctx)) {
+ /* We can compress output */
+ ZSTD_inBuffer zin;
+ ZSTD_outBuffer zout;
+ ZSTD_CStream *zstream;
+ rspamd_fstring_t *compressed_reply;
+ gsize r;
+
+ zstream = task->cfg->libs_ctx->out_zstream;
+ compressed_reply = rspamd_fstring_sized_new(ZSTD_compressBound(reply->len));
+ zin.pos = 0;
+ zin.src = reply->str;
+ zin.size = reply->len;
+ zout.pos = 0;
+ zout.dst = compressed_reply->str;
+ zout.size = compressed_reply->allocated;
+
+ while (zin.pos < zin.size) {
+ r = ZSTD_compressStream(zstream, &zout, &zin);
+
+ if (ZSTD_isError(r)) {
+ msg_err_protocol("cannot compress: %s", ZSTD_getErrorName(r));
+ rspamd_fstring_free(compressed_reply);
+ rspamd_http_message_set_body_from_fstring_steal(msg, reply);
+
+ goto end;
+ }
+ }
+
+ ZSTD_flushStream(zstream, &zout);
+ r = ZSTD_endStream(zstream, &zout);
+
+ if (ZSTD_isError(r)) {
+ msg_err_protocol("cannot finalize compress: %s", ZSTD_getErrorName(r));
+ rspamd_fstring_free(compressed_reply);
+ rspamd_http_message_set_body_from_fstring_steal(msg, reply);
+
+ goto end;
+ }
+
+ msg_info_protocol("writing compressed results: %z bytes before "
+ "%z bytes after",
+ zin.pos, zout.pos);
+ compressed_reply->len = zout.pos;
+ rspamd_fstring_free(reply);
+ rspamd_http_message_set_body_from_fstring_steal(msg, compressed_reply);
+ rspamd_http_message_add_header(msg, COMPRESSION_HEADER, "zstd");
+
+ if (task->cfg->libs_ctx->out_dict &&
+ task->cfg->libs_ctx->out_dict->id != 0) {
+ gchar dict_str[32];
+
+ rspamd_snprintf(dict_str, sizeof(dict_str), "%ud",
+ task->cfg->libs_ctx->out_dict->id);
+ rspamd_http_message_add_header(msg, "Dictionary", dict_str);
+ }
+ }
+ else {
+ rspamd_http_message_set_body_from_fstring_steal(msg, reply);
+ }
+
+end:
+ if (!(task->flags & RSPAMD_TASK_FLAG_NO_STAT)) {
+ /* Update stat for default metric */
+
+ msg_debug_protocol("skip stats update due to no_stat flag");
+ metric_res = task->result;
+
+ if (metric_res != NULL) {
+
+ action = rspamd_check_action_metric(task, NULL, NULL);
+ /* TODO: handle custom actions in stats */
+ if (action->action_type == METRIC_ACTION_SOFT_REJECT &&
+ (task->flags & RSPAMD_TASK_FLAG_GREYLISTED)) {
+ /* Set stat action to greylist to display greylisted messages */
+#ifndef HAVE_ATOMIC_BUILTINS
+ task->worker->srv->stat->actions_stat[METRIC_ACTION_GREYLIST]++;
+#else
+ __atomic_add_fetch(&task->worker->srv->stat->actions_stat[METRIC_ACTION_GREYLIST],
+ 1, __ATOMIC_RELEASE);
+#endif
+ }
+ else if (action->action_type < METRIC_ACTION_MAX) {
+#ifndef HAVE_ATOMIC_BUILTINS
+ task->worker->srv->stat->actions_stat[action->action_type]++;
+#else
+ __atomic_add_fetch(&task->worker->srv->stat->actions_stat[action->action_type],
+ 1, __ATOMIC_RELEASE);
+#endif
+ }
+ }
+
+ /* Increase counters */
+#ifndef HAVE_ATOMIC_BUILTINS
+ task->worker->srv->stat->messages_scanned++;
+#else
+ __atomic_add_fetch(&task->worker->srv->stat->messages_scanned,
+ 1, __ATOMIC_RELEASE);
+#endif
+
+ /* Set average processing time */
+ guint32 slot;
+ float processing_time = task->time_real_finish - task->task_timestamp;
+
+#ifndef HAVE_ATOMIC_BUILTINS
+ slot = task->worker->srv->stat->avg_time.cur_slot++;
+#else
+ slot = __atomic_fetch_add(&task->worker->srv->stat->avg_time.cur_slot,
+ 1, __ATOMIC_RELEASE);
+#endif
+ slot = slot % MAX_AVG_TIME_SLOTS;
+ /* TODO: this should be atomic but it is not supported in C */
+ task->worker->srv->stat->avg_time.avg_time[slot] = processing_time;
+ }
+}
+
+void rspamd_protocol_write_log_pipe(struct rspamd_task *task)
+{
+ struct rspamd_worker_log_pipe *lp;
+ struct rspamd_protocol_log_message_sum *ls;
+ lua_State *L = task->cfg->lua_state;
+ struct rspamd_scan_result *mres;
+ struct rspamd_symbol_result *sym;
+ gint id, i;
+ guint32 n = 0, nextra = 0;
+ gsize sz;
+ GArray *extra;
+ struct rspamd_protocol_log_symbol_result er;
+ struct rspamd_task **ptask;
+
+ /* Get extra results from lua plugins */
+ extra = g_array_new(FALSE, FALSE, sizeof(er));
+
+ lua_getglobal(L, "rspamd_plugins");
+ if (lua_istable(L, -1)) {
+ lua_pushnil(L);
+
+ while (lua_next(L, -2)) {
+ if (lua_istable(L, -1)) {
+ lua_pushvalue(L, -2);
+ /* stack:
+ * -1: copy of key
+ * -2: value (module table)
+ * -3: key (module name)
+ * -4: global
+ */
+ lua_pushstring(L, "log_callback");
+ lua_gettable(L, -3);
+ /* stack:
+ * -1: func
+ * -2: copy of key
+ * -3: value (module table)
+ * -3: key (module name)
+ * -4: global
+ */
+ if (lua_isfunction(L, -1)) {
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ /* stack:
+ * -1: task
+ * -2: func
+ * -3: key copy
+ * -4: value (module table)
+ * -5: key (module name)
+ * -6: global
+ */
+ msg_debug_protocol("calling for %s", lua_tostring(L, -3));
+ if (lua_pcall(L, 1, 1, 0) != 0) {
+ msg_info_protocol("call to log callback %s failed: %s",
+ lua_tostring(L, -2), lua_tostring(L, -1));
+ lua_pop(L, 1);
+ /* stack:
+ * -1: key copy
+ * -2: value
+ * -3: key
+ */
+ }
+ else {
+ /* stack:
+ * -1: result
+ * -2: key copy
+ * -3: value
+ * -4: key
+ */
+ if (lua_istable(L, -1)) {
+ /* Another iteration */
+ lua_pushnil(L);
+
+ while (lua_next(L, -2)) {
+ /* stack:
+ * -1: value
+ * -2: key
+ * -3: result table (pcall)
+ * -4: key copy (parent)
+ * -5: value (parent)
+ * -6: key (parent)
+ */
+ if (lua_istable(L, -1)) {
+ er.id = 0;
+ er.score = 0.0;
+
+ lua_rawgeti(L, -1, 1);
+ if (lua_isnumber(L, -1)) {
+ er.id = lua_tonumber(L, -1);
+ }
+ lua_rawgeti(L, -2, 2);
+ if (lua_isnumber(L, -1)) {
+ er.score = lua_tonumber(L, -1);
+ }
+ /* stack:
+ * -1: value[2]
+ * -2: value[1]
+ * -3: values
+ * -4: key
+ * -5: result table (pcall)
+ * -6: key copy (parent)
+ * -7: value (parent)
+ * -8: key (parent)
+ */
+ lua_pop(L, 2); /* Values */
+ g_array_append_val(extra, er);
+ }
+
+ lua_pop(L, 1); /* Value for lua_next */
+ }
+
+ lua_pop(L, 1); /* Table result of pcall */
+ }
+ else {
+ msg_info_protocol("call to log callback %s returned "
+ "wrong type: %s",
+ lua_tostring(L, -2),
+ lua_typename(L, lua_type(L, -1)));
+ lua_pop(L, 1); /* Returned error */
+ }
+ }
+ }
+ else {
+ lua_pop(L, 1);
+ /* stack:
+ * -1: key copy
+ * -2: value
+ * -3: key
+ */
+ }
+ }
+
+ lua_pop(L, 2); /* Top table + key copy */
+ }
+
+ lua_pop(L, 1); /* rspamd_plugins global */
+ }
+ else {
+ lua_pop(L, 1);
+ }
+
+ nextra = extra->len;
+
+ LL_FOREACH(task->cfg->log_pipes, lp)
+ {
+ if (lp->fd != -1) {
+ switch (lp->type) {
+ case RSPAMD_LOG_PIPE_SYMBOLS:
+ mres = task->result;
+
+ if (mres) {
+ n = kh_size(mres->symbols);
+ sz = sizeof(*ls) +
+ sizeof(struct rspamd_protocol_log_symbol_result) *
+ (n + nextra);
+ ls = g_malloc0(sz);
+
+ /* Handle settings id */
+
+ if (task->settings_elt) {
+ ls->settings_id = task->settings_elt->id;
+ }
+ else {
+ ls->settings_id = 0;
+ }
+
+ ls->score = mres->score;
+ ls->required_score = rspamd_task_get_required_score(task,
+ mres);
+ ls->nresults = n;
+ ls->nextra = nextra;
+
+ i = 0;
+
+ kh_foreach_value(mres->symbols, sym, {
+ id = rspamd_symcache_find_symbol(task->cfg->cache,
+ sym->name);
+
+ if (id >= 0) {
+ ls->results[i].id = id;
+ ls->results[i].score = sym->score;
+ }
+ else {
+ ls->results[i].id = -1;
+ ls->results[i].score = 0.0;
+ }
+
+ i++;
+ });
+
+ memcpy(&ls->results[n], extra->data, nextra * sizeof(er));
+ }
+ else {
+ sz = sizeof(*ls);
+ ls = g_malloc0(sz);
+ ls->nresults = 0;
+ }
+
+ /* We don't really care about return value here */
+ if (write(lp->fd, ls, sz) == -1) {
+ msg_info_protocol("cannot write to log pipe: %s",
+ strerror(errno));
+ }
+
+ g_free(ls);
+ break;
+ default:
+ msg_err_protocol("unknown log format %d", lp->type);
+ break;
+ }
+ }
+ }
+
+ g_array_free(extra, TRUE);
+}
+
+void rspamd_protocol_write_reply(struct rspamd_task *task, ev_tstamp timeout)
+{
+ struct rspamd_http_message *msg;
+ const gchar *ctype = "application/json";
+ rspamd_fstring_t *reply;
+
+ msg = rspamd_http_new_message(HTTP_RESPONSE);
+
+ if (rspamd_http_connection_is_encrypted(task->http_conn)) {
+ msg_info_protocol("<%s> writing encrypted reply",
+ MESSAGE_FIELD_CHECK(task, message_id));
+ }
+
+ /* Compatibility */
+ if (task->cmd == CMD_CHECK_RSPAMC) {
+ msg->method = HTTP_SYMBOLS;
+ }
+ else if (task->cmd == CMD_CHECK_SPAMC) {
+ msg->method = HTTP_SYMBOLS;
+ msg->flags |= RSPAMD_HTTP_FLAG_SPAMC;
+ }
+
+ if (task->err != NULL) {
+ msg_debug_protocol("writing error reply to client");
+ ucl_object_t *top = NULL;
+
+ top = ucl_object_typed_new(UCL_OBJECT);
+ msg->code = 500 + task->err->code % 100;
+ msg->status = rspamd_fstring_new_init(task->err->message,
+ strlen(task->err->message));
+ ucl_object_insert_key(top, ucl_object_fromstring(task->err->message),
+ "error", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromstring(g_quark_to_string(task->err->domain)),
+ "error_domain", 0, false);
+ reply = rspamd_fstring_sized_new(256);
+ rspamd_ucl_emit_fstring(top, UCL_EMIT_JSON_COMPACT, &reply);
+ ucl_object_unref(top);
+
+ /* We also need to validate utf8 */
+ if (rspamd_fast_utf8_validate(reply->str, reply->len) != 0) {
+ gsize valid_len;
+ gchar *validated;
+
+ /* We copy reply several times here but it should be a rare case */
+ validated = rspamd_str_make_utf_valid(reply->str, reply->len,
+ &valid_len, task->task_pool);
+ rspamd_http_message_set_body(msg, validated, valid_len);
+ rspamd_fstring_free(reply);
+ }
+ else {
+ rspamd_http_message_set_body_from_fstring_steal(msg, reply);
+ }
+ }
+ else {
+ msg->status = rspamd_fstring_new_init("OK", 2);
+
+ switch (task->cmd) {
+ case CMD_CHECK:
+ case CMD_CHECK_RSPAMC:
+ case CMD_CHECK_SPAMC:
+ case CMD_SKIP:
+ case CMD_CHECK_V2:
+ rspamd_protocol_http_reply(msg, task, NULL);
+ rspamd_protocol_write_log_pipe(task);
+ break;
+ case CMD_PING:
+ msg_debug_protocol("writing pong to client");
+ rspamd_http_message_set_body(msg, "pong" CRLF, 6);
+ ctype = "text/plain";
+ break;
+ default:
+ msg_err_protocol("BROKEN");
+ break;
+ }
+ }
+
+ ev_now_update(task->event_loop);
+ msg->date = ev_time();
+
+ rspamd_http_connection_reset(task->http_conn);
+ rspamd_http_connection_write_message(task->http_conn, msg, NULL,
+ ctype, task, timeout);
+
+ task->processed_stages |= RSPAMD_TASK_STAGE_REPLIED;
+}
diff --git a/src/libserver/protocol.h b/src/libserver/protocol.h
new file mode 100644
index 0000000..0e3c187
--- /dev/null
+++ b/src/libserver/protocol.h
@@ -0,0 +1,130 @@
+/**
+ * @file protocol.h
+ * Rspamd protocol definition
+ */
+
+#ifndef RSPAMD_PROTOCOL_H
+#define RSPAMD_PROTOCOL_H
+
+#include "config.h"
+#include "scan_result.h"
+#include "libserver/http/http_connection.h"
+#include "task.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RSPAMD_BASE_ERROR 500
+#define RSPAMD_FILTER_ERROR RSPAMD_BASE_ERROR + 1
+#define RSPAMD_NETWORK_ERROR RSPAMD_BASE_ERROR + 2
+#define RSPAMD_PROTOCOL_ERROR RSPAMD_BASE_ERROR + 3
+#define RSPAMD_LENGTH_ERROR RSPAMD_BASE_ERROR + 4
+#define RSPAMD_STATFILE_ERROR RSPAMD_BASE_ERROR + 5
+
+struct rspamd_protocol_log_symbol_result {
+ guint32 id;
+ float score;
+};
+struct rspamd_protocol_log_message_sum {
+ guint32 nresults;
+ guint32 nextra;
+ guint32 settings_id;
+ gdouble score;
+ gdouble required_score;
+ struct rspamd_protocol_log_symbol_result results[];
+};
+
+struct rspamd_metric;
+
+/**
+ * Process headers into HTTP message and set appropriate task fields
+ * @param task
+ * @param msg
+ * @return
+ */
+gboolean rspamd_protocol_handle_headers(struct rspamd_task *task,
+ struct rspamd_http_message *msg);
+
+/**
+ * Process control chunk and update task structure accordingly
+ * @param task
+ * @param control
+ * @return
+ */
+gboolean rspamd_protocol_handle_control(struct rspamd_task *task,
+ const ucl_object_t *control);
+
+/**
+ * Process HTTP request to the task structure
+ * @param task
+ * @param msg
+ * @return
+ */
+gboolean rspamd_protocol_handle_request(struct rspamd_task *task,
+ struct rspamd_http_message *msg);
+
+/**
+ * Write task results to http message
+ * @param msg
+ * @param task
+ */
+void rspamd_protocol_http_reply(struct rspamd_http_message *msg,
+ struct rspamd_task *task, ucl_object_t **pobj);
+
+/**
+ * Write data to log pipes
+ * @param task
+ */
+void rspamd_protocol_write_log_pipe(struct rspamd_task *task);
+
+enum rspamd_protocol_flags {
+ RSPAMD_PROTOCOL_BASIC = 1 << 0,
+ RSPAMD_PROTOCOL_METRICS = 1 << 1,
+ RSPAMD_PROTOCOL_MESSAGES = 1 << 2,
+ RSPAMD_PROTOCOL_RMILTER = 1 << 3,
+ RSPAMD_PROTOCOL_DKIM = 1 << 4,
+ RSPAMD_PROTOCOL_URLS = 1 << 5,
+ RSPAMD_PROTOCOL_EXTRA = 1 << 6,
+};
+
+#define RSPAMD_PROTOCOL_DEFAULT (RSPAMD_PROTOCOL_BASIC | \
+ RSPAMD_PROTOCOL_METRICS | \
+ RSPAMD_PROTOCOL_MESSAGES | \
+ RSPAMD_PROTOCOL_RMILTER | \
+ RSPAMD_PROTOCOL_DKIM | \
+ RSPAMD_PROTOCOL_EXTRA)
+
+/**
+ * Write reply to ucl object filling log buffer
+ * @param task
+ * @param logbuf
+ * @return
+ */
+ucl_object_t *rspamd_protocol_write_ucl(struct rspamd_task *task,
+ enum rspamd_protocol_flags flags);
+
+/**
+ * Write reply for specified task command
+ * @param task task object
+ * @return 0 if we wrote reply and -1 if there was some error
+ */
+void rspamd_protocol_write_reply(struct rspamd_task *task, ev_tstamp timeout);
+
+/**
+ * Convert rspamd output to legacy protocol reply
+ * @param task
+ * @param top
+ * @param out
+ */
+void rspamd_ucl_torspamc_output(const ucl_object_t *top,
+ rspamd_fstring_t **out);
+
+void rspamd_ucl_tospamc_output(const ucl_object_t *top,
+ rspamd_fstring_t **out);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/protocol_internal.h b/src/libserver/protocol_internal.h
new file mode 100644
index 0000000..c604e96
--- /dev/null
+++ b/src/libserver/protocol_internal.h
@@ -0,0 +1,99 @@
+/*-
+ * Copyright 2017 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_PROTOCOL_INTERNAL_H
+#define RSPAMD_PROTOCOL_INTERNAL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Just check if the passed message is spam or not and reply as
+ * described below
+ */
+#define MSG_CMD_CHECK "check"
+
+/*
+ * Modern check version
+ */
+#define MSG_CMD_CHECK_V2 "checkv2"
+#define MSG_CMD_SCAN "scan"
+
+/*
+ * Check if message is spam or not, and return score plus list
+ * of symbols hit
+ */
+#define MSG_CMD_SYMBOLS "symbols"
+/*
+ * Check if message is spam or not, and return score plus report
+ */
+#define MSG_CMD_REPORT "report"
+/*
+ * Check if message is spam or not, and return score plus report
+ * if the message is spam
+ */
+#define MSG_CMD_REPORT_IFSPAM "report_ifspam"
+/*
+ * Ignore this message -- client opened connection then changed
+ */
+#define MSG_CMD_SKIP "skip"
+/*
+ * Return a confirmation that spamd is alive
+ */
+#define MSG_CMD_PING "ping"
+/*
+ * Process this message as described above and return modified message
+ */
+#define MSG_CMD_PROCESS "process"
+/*
+ * Headers
+ */
+#define HELO_HEADER "Helo"
+#define FROM_HEADER "From"
+#define IP_ADDR_HEADER "IP"
+#define RCPT_HEADER "Rcpt"
+#define SUBJECT_HEADER "Subject"
+#define SETTINGS_ID_HEADER "Settings-ID"
+#define SETTINGS_HEADER "Settings"
+#define QUEUE_ID_HEADER "Queue-ID"
+#define USER_HEADER "User"
+#define URLS_HEADER "URL-Format"
+#define PASS_HEADER "Pass"
+#define HOSTNAME_HEADER "Hostname"
+#define DELIVER_TO_HEADER "Deliver-To"
+#define NO_LOG_HEADER "Log"
+#define MLEN_HEADER "Message-Length"
+#define USER_AGENT_HEADER "User-Agent"
+#define MTA_TAG_HEADER "MTA-Tag"
+#define PROFILE_HEADER "Profile"
+#define TLS_CIPHER_HEADER "TLS-Cipher"
+#define TLS_VERSION_HEADER "TLS-Version"
+#define MTA_NAME_HEADER "MTA-Name"
+#define MILTER_HEADER "Milter"
+#define FILENAME_HEADER "Filename"
+#define FLAGS_HEADER "Flags"
+#define CERT_ISSUER_HEADER "TLS-Cert-Issuer"
+#define MAILER_HEADER "Mailer"
+#define RAW_DATA_HEADER "Raw"
+#define COMPRESSION_HEADER "Compression"
+#define MESSAGE_OFFSET_HEADER "Message-Offset"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif//RSPAMD_PROTOCOL_INTERNAL_H
diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c
new file mode 100644
index 0000000..d51dba6
--- /dev/null
+++ b/src/libserver/re_cache.c
@@ -0,0 +1,2712 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "libmime/message.h"
+#include "re_cache.h"
+#include "cryptobox.h"
+#include "ref.h"
+#include "libserver/url.h"
+#include "libserver/task.h"
+#include "libserver/cfg_file.h"
+#include "libutil/util.h"
+#include "libutil/regexp.h"
+#include "lua/lua_common.h"
+#include "libstat/stat_api.h"
+#include "contrib/uthash/utlist.h"
+
+#include "khash.h"
+
+#ifdef WITH_HYPERSCAN
+#include "hs.h"
+#include "hyperscan_tools.h"
+#endif
+
+#include "unix-std.h"
+#include <signal.h>
+#include <stdalign.h>
+#include <math.h>
+#include "contrib/libev/ev.h"
+
+#ifndef WITH_PCRE2
+#include <pcre.h>
+#else
+#include <pcre2.h>
+#endif
+
+#include "contrib/fastutf8/fastutf8.h"
+
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+
+#define msg_err_re_cache(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "re_cache", cache->hash, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_re_cache(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "re_cache", cache->hash, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_re_cache(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "re_cache", cache->hash, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+#define msg_debug_re_task(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_re_cache_log_id, "re_cache", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_re_cache(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_re_cache_log_id, "re_cache", cache->hash, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(re_cache)
+
+#ifdef WITH_HYPERSCAN
+#define RSPAMD_HS_MAGIC_LEN (sizeof(rspamd_hs_magic))
+static const guchar rspamd_hs_magic[] = {'r', 's', 'h', 's', 'r', 'e', '1', '1'},
+ rspamd_hs_magic_vector[] = {'r', 's', 'h', 's', 'r', 'v', '1', '1'};
+#endif
+
+
+struct rspamd_re_class {
+ guint64 id;
+ enum rspamd_re_type type;
+ gboolean has_utf8; /* if there are any utf8 regexps */
+ gpointer type_data;
+ gsize type_len;
+ GHashTable *re;
+ rspamd_cryptobox_hash_state_t *st;
+
+ gchar hash[rspamd_cryptobox_HASHBYTES + 1];
+
+#ifdef WITH_HYPERSCAN
+ rspamd_hyperscan_t *hs_db;
+ hs_scratch_t *hs_scratch;
+ gint *hs_ids;
+ guint nhs;
+#endif
+};
+
+enum rspamd_re_cache_elt_match_type {
+ RSPAMD_RE_CACHE_PCRE = 0,
+ RSPAMD_RE_CACHE_HYPERSCAN,
+ RSPAMD_RE_CACHE_HYPERSCAN_PRE
+};
+
+struct rspamd_re_cache_elt {
+ rspamd_regexp_t *re;
+ gint lua_cbref;
+ enum rspamd_re_cache_elt_match_type match_type;
+};
+
+KHASH_INIT(lua_selectors_hash, gchar *, int, 1, kh_str_hash_func, kh_str_hash_equal);
+
+struct rspamd_re_cache {
+ GHashTable *re_classes;
+
+ GPtrArray *re;
+ khash_t(lua_selectors_hash) * selectors;
+ ref_entry_t ref;
+ guint nre;
+ guint max_re_data;
+ gchar hash[rspamd_cryptobox_HASHBYTES + 1];
+ lua_State *L;
+#ifdef WITH_HYPERSCAN
+ enum rspamd_hyperscan_status hyperscan_loaded;
+ gboolean disable_hyperscan;
+ hs_platform_info_t plt;
+#endif
+};
+
+struct rspamd_re_selector_result {
+ guchar **scvec;
+ guint *lenvec;
+ guint cnt;
+};
+
+KHASH_INIT(selectors_results_hash, int, struct rspamd_re_selector_result, 1,
+ kh_int_hash_func, kh_int_hash_equal);
+
+struct rspamd_re_runtime {
+ guchar *checked;
+ guchar *results;
+ khash_t(selectors_results_hash) * sel_cache;
+ struct rspamd_re_cache *cache;
+ struct rspamd_re_cache_stat stat;
+ gboolean has_hs;
+};
+
+static GQuark
+rspamd_re_cache_quark(void)
+{
+ return g_quark_from_static_string("re_cache");
+}
+
+static guint64
+rspamd_re_cache_class_id(enum rspamd_re_type type,
+ gconstpointer type_data,
+ gsize datalen)
+{
+ rspamd_cryptobox_fast_hash_state_t st;
+
+ rspamd_cryptobox_fast_hash_init(&st, 0xdeadbabe);
+ rspamd_cryptobox_fast_hash_update(&st, &type, sizeof(type));
+
+ if (datalen > 0) {
+ rspamd_cryptobox_fast_hash_update(&st, type_data, datalen);
+ }
+
+ return rspamd_cryptobox_fast_hash_final(&st);
+}
+
+static void
+rspamd_re_cache_destroy(struct rspamd_re_cache *cache)
+{
+ GHashTableIter it;
+ gpointer k, v;
+ struct rspamd_re_class *re_class;
+ gchar *skey;
+ gint sref;
+
+ g_assert(cache != NULL);
+ g_hash_table_iter_init(&it, cache->re_classes);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ re_class = v;
+ g_hash_table_iter_steal(&it);
+ g_hash_table_unref(re_class->re);
+
+ if (re_class->type_data) {
+ g_free(re_class->type_data);
+ }
+
+#ifdef WITH_HYPERSCAN
+ if (re_class->hs_db) {
+ rspamd_hyperscan_free(re_class->hs_db, false);
+ }
+ if (re_class->hs_scratch) {
+ hs_free_scratch(re_class->hs_scratch);
+ }
+ if (re_class->hs_ids) {
+ g_free(re_class->hs_ids);
+ }
+#endif
+ g_free(re_class);
+ }
+
+ if (cache->L) {
+ kh_foreach(cache->selectors, skey, sref, {
+ luaL_unref(cache->L, LUA_REGISTRYINDEX, sref);
+ g_free(skey);
+ });
+
+ struct rspamd_re_cache_elt *elt;
+ guint i;
+
+ PTR_ARRAY_FOREACH(cache->re, i, elt)
+ {
+ if (elt->lua_cbref != -1) {
+ luaL_unref(cache->L, LUA_REGISTRYINDEX, elt->lua_cbref);
+ }
+ }
+ }
+
+ kh_destroy(lua_selectors_hash, cache->selectors);
+
+ g_hash_table_unref(cache->re_classes);
+ g_ptr_array_free(cache->re, TRUE);
+ g_free(cache);
+}
+
+static void
+rspamd_re_cache_elt_dtor(gpointer e)
+{
+ struct rspamd_re_cache_elt *elt = e;
+
+ rspamd_regexp_unref(elt->re);
+ g_free(elt);
+}
+
+struct rspamd_re_cache *
+rspamd_re_cache_new(void)
+{
+ struct rspamd_re_cache *cache;
+
+ cache = g_malloc0(sizeof(*cache));
+ cache->re_classes = g_hash_table_new(g_int64_hash, g_int64_equal);
+ cache->nre = 0;
+ cache->re = g_ptr_array_new_full(256, rspamd_re_cache_elt_dtor);
+ cache->selectors = kh_init(lua_selectors_hash);
+#ifdef WITH_HYPERSCAN
+ cache->hyperscan_loaded = RSPAMD_HYPERSCAN_UNKNOWN;
+#endif
+ REF_INIT_RETAIN(cache, rspamd_re_cache_destroy);
+
+ return cache;
+}
+
+enum rspamd_hyperscan_status
+rspamd_re_cache_is_hs_loaded(struct rspamd_re_cache *cache)
+{
+ g_assert(cache != NULL);
+
+#ifdef WITH_HYPERSCAN
+ return cache->hyperscan_loaded;
+#else
+ return RSPAMD_HYPERSCAN_UNSUPPORTED;
+#endif
+}
+
+rspamd_regexp_t *
+rspamd_re_cache_add(struct rspamd_re_cache *cache,
+ rspamd_regexp_t *re,
+ enum rspamd_re_type type,
+ gconstpointer type_data, gsize datalen,
+ gint lua_cbref)
+{
+ guint64 class_id;
+ struct rspamd_re_class *re_class;
+ rspamd_regexp_t *nre;
+ struct rspamd_re_cache_elt *elt;
+
+ g_assert(cache != NULL);
+ g_assert(re != NULL);
+
+ class_id = rspamd_re_cache_class_id(type, type_data, datalen);
+ re_class = g_hash_table_lookup(cache->re_classes, &class_id);
+
+ if (re_class == NULL) {
+ re_class = g_malloc0(sizeof(*re_class));
+ re_class->id = class_id;
+ re_class->type_len = datalen;
+ re_class->type = type;
+ re_class->re = g_hash_table_new_full(rspamd_regexp_hash,
+ rspamd_regexp_equal, NULL, (GDestroyNotify) rspamd_regexp_unref);
+
+ if (datalen > 0) {
+ re_class->type_data = g_malloc0(datalen);
+ memcpy(re_class->type_data, type_data, datalen);
+ }
+
+ g_hash_table_insert(cache->re_classes, &re_class->id, re_class);
+ }
+
+ if ((nre = g_hash_table_lookup(re_class->re, rspamd_regexp_get_id(re))) == NULL) {
+ /*
+ * We set re id based on the global position in the cache
+ */
+ elt = g_malloc0(sizeof(*elt));
+ /* One ref for re_class */
+ nre = rspamd_regexp_ref(re);
+ rspamd_regexp_set_cache_id(re, cache->nre++);
+ /* One ref for cache */
+ elt->re = rspamd_regexp_ref(re);
+ g_ptr_array_add(cache->re, elt);
+ rspamd_regexp_set_class(re, re_class);
+ elt->lua_cbref = lua_cbref;
+
+ g_hash_table_insert(re_class->re, rspamd_regexp_get_id(nre), nre);
+ }
+
+ if (rspamd_regexp_get_flags(re) & RSPAMD_REGEXP_FLAG_UTF) {
+ re_class->has_utf8 = TRUE;
+ }
+
+ return nre;
+}
+
+void rspamd_re_cache_replace(struct rspamd_re_cache *cache,
+ rspamd_regexp_t *what,
+ rspamd_regexp_t *with)
+{
+ guint64 re_id;
+ struct rspamd_re_class *re_class;
+ rspamd_regexp_t *src;
+ struct rspamd_re_cache_elt *elt;
+
+ g_assert(cache != NULL);
+ g_assert(what != NULL);
+ g_assert(with != NULL);
+
+ re_class = rspamd_regexp_get_class(what);
+
+ if (re_class != NULL) {
+ re_id = rspamd_regexp_get_cache_id(what);
+
+ g_assert(re_id != RSPAMD_INVALID_ID);
+ src = g_hash_table_lookup(re_class->re, rspamd_regexp_get_id(what));
+ elt = g_ptr_array_index(cache->re, re_id);
+ g_assert(elt != NULL);
+ g_assert(src != NULL);
+
+ rspamd_regexp_set_cache_id(what, RSPAMD_INVALID_ID);
+ rspamd_regexp_set_class(what, NULL);
+ rspamd_regexp_set_cache_id(with, re_id);
+ rspamd_regexp_set_class(with, re_class);
+ /*
+ * On calling of this function, we actually unref old re (what)
+ */
+ g_hash_table_insert(re_class->re,
+ rspamd_regexp_get_id(what),
+ rspamd_regexp_ref(with));
+
+ rspamd_regexp_unref(elt->re);
+ elt->re = rspamd_regexp_ref(with);
+ /* XXX: do not touch match type here */
+ }
+}
+
+static gint
+rspamd_re_cache_sort_func(gconstpointer a, gconstpointer b)
+{
+ struct rspamd_re_cache_elt *const *re1 = a, *const *re2 = b;
+
+ return rspamd_regexp_cmp(rspamd_regexp_get_id((*re1)->re),
+ rspamd_regexp_get_id((*re2)->re));
+}
+
+void rspamd_re_cache_init(struct rspamd_re_cache *cache, struct rspamd_config *cfg)
+{
+ guint i, fl;
+ GHashTableIter it;
+ gpointer k, v;
+ struct rspamd_re_class *re_class;
+ rspamd_cryptobox_hash_state_t st_global;
+ rspamd_regexp_t *re;
+ struct rspamd_re_cache_elt *elt;
+ guchar hash_out[rspamd_cryptobox_HASHBYTES];
+
+ g_assert(cache != NULL);
+
+ rspamd_cryptobox_hash_init(&st_global, NULL, 0);
+ /* Resort all regexps */
+ g_ptr_array_sort(cache->re, rspamd_re_cache_sort_func);
+
+ for (i = 0; i < cache->re->len; i++) {
+ elt = g_ptr_array_index(cache->re, i);
+ re = elt->re;
+ re_class = rspamd_regexp_get_class(re);
+ g_assert(re_class != NULL);
+ rspamd_regexp_set_cache_id(re, i);
+
+ if (re_class->st == NULL) {
+ (void) !posix_memalign((void **) &re_class->st, RSPAMD_ALIGNOF(rspamd_cryptobox_hash_state_t),
+ sizeof(*re_class->st));
+ g_assert(re_class->st != NULL);
+ rspamd_cryptobox_hash_init(re_class->st, NULL, 0);
+ }
+
+ /* Update hashes */
+ /* Id of re class */
+ rspamd_cryptobox_hash_update(re_class->st, (gpointer) &re_class->id,
+ sizeof(re_class->id));
+ rspamd_cryptobox_hash_update(&st_global, (gpointer) &re_class->id,
+ sizeof(re_class->id));
+ /* Id of re expression */
+ rspamd_cryptobox_hash_update(re_class->st, rspamd_regexp_get_id(re),
+ rspamd_cryptobox_HASHBYTES);
+ rspamd_cryptobox_hash_update(&st_global, rspamd_regexp_get_id(re),
+ rspamd_cryptobox_HASHBYTES);
+ /* PCRE flags */
+ fl = rspamd_regexp_get_pcre_flags(re);
+ rspamd_cryptobox_hash_update(re_class->st, (const guchar *) &fl,
+ sizeof(fl));
+ rspamd_cryptobox_hash_update(&st_global, (const guchar *) &fl,
+ sizeof(fl));
+ /* Rspamd flags */
+ fl = rspamd_regexp_get_flags(re);
+ rspamd_cryptobox_hash_update(re_class->st, (const guchar *) &fl,
+ sizeof(fl));
+ rspamd_cryptobox_hash_update(&st_global, (const guchar *) &fl,
+ sizeof(fl));
+ /* Limit of hits */
+ fl = rspamd_regexp_get_maxhits(re);
+ rspamd_cryptobox_hash_update(re_class->st, (const guchar *) &fl,
+ sizeof(fl));
+ rspamd_cryptobox_hash_update(&st_global, (const guchar *) &fl,
+ sizeof(fl));
+ /* Numeric order */
+ rspamd_cryptobox_hash_update(re_class->st, (const guchar *) &i,
+ sizeof(i));
+ rspamd_cryptobox_hash_update(&st_global, (const guchar *) &i,
+ sizeof(i));
+ }
+
+ rspamd_cryptobox_hash_final(&st_global, hash_out);
+ rspamd_snprintf(cache->hash, sizeof(cache->hash), "%*xs",
+ (gint) rspamd_cryptobox_HASHBYTES, hash_out);
+
+ /* Now finalize all classes */
+ g_hash_table_iter_init(&it, cache->re_classes);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ re_class = v;
+
+ if (re_class->st) {
+ /*
+ * We finally update all classes with the number of expressions
+ * in the cache to ensure that if even a single re has been changed
+ * we won't be broken due to id mismatch
+ */
+ rspamd_cryptobox_hash_update(re_class->st,
+ (gpointer) &cache->re->len,
+ sizeof(cache->re->len));
+ rspamd_cryptobox_hash_final(re_class->st, hash_out);
+ rspamd_snprintf(re_class->hash, sizeof(re_class->hash), "%*xs",
+ (gint) rspamd_cryptobox_HASHBYTES, hash_out);
+ free(re_class->st); /* Due to posix_memalign */
+ re_class->st = NULL;
+ }
+ }
+
+ cache->L = cfg->lua_state;
+
+#ifdef WITH_HYPERSCAN
+ const gchar *platform = "generic";
+ rspamd_fstring_t *features = rspamd_fstring_new();
+
+ cache->disable_hyperscan = cfg->disable_hyperscan;
+
+ g_assert(hs_populate_platform(&cache->plt) == HS_SUCCESS);
+
+ /* Now decode what we do have */
+ switch (cache->plt.tune) {
+ case HS_TUNE_FAMILY_HSW:
+ platform = "haswell";
+ break;
+ case HS_TUNE_FAMILY_SNB:
+ platform = "sandy";
+ break;
+ case HS_TUNE_FAMILY_BDW:
+ platform = "broadwell";
+ break;
+ case HS_TUNE_FAMILY_IVB:
+ platform = "ivy";
+ break;
+ default:
+ break;
+ }
+
+ if (cache->plt.cpu_features & HS_CPU_FEATURES_AVX2) {
+ features = rspamd_fstring_append(features, "AVX2", 4);
+ }
+
+ hs_set_allocator(g_malloc, g_free);
+
+ msg_info_re_cache("loaded hyperscan engine with cpu tune '%s' and features '%V'",
+ platform, features);
+
+ rspamd_fstring_free(features);
+#endif
+}
+
+struct rspamd_re_runtime *
+rspamd_re_cache_runtime_new(struct rspamd_re_cache *cache)
+{
+ struct rspamd_re_runtime *rt;
+ g_assert(cache != NULL);
+
+ rt = g_malloc0(sizeof(*rt) + NBYTES(cache->nre) + cache->nre);
+ rt->cache = cache;
+ REF_RETAIN(cache);
+ rt->checked = ((guchar *) rt) + sizeof(*rt);
+ rt->results = rt->checked + NBYTES(cache->nre);
+ rt->stat.regexp_total = cache->nre;
+#ifdef WITH_HYPERSCAN
+ rt->has_hs = cache->hyperscan_loaded;
+#endif
+
+ return rt;
+}
+
+const struct rspamd_re_cache_stat *
+rspamd_re_cache_get_stat(struct rspamd_re_runtime *rt)
+{
+ g_assert(rt != NULL);
+
+ return &rt->stat;
+}
+
+static gboolean
+rspamd_re_cache_check_lua_condition(struct rspamd_task *task,
+ rspamd_regexp_t *re,
+ const guchar *in, gsize len,
+ goffset start, goffset end,
+ gint lua_cbref)
+{
+ lua_State *L = (lua_State *) task->cfg->lua_state;
+ GError *err = NULL;
+ struct rspamd_lua_text __attribute__((unused)) * t;
+ gint text_pos;
+
+ if (G_LIKELY(lua_cbref == -1)) {
+ return TRUE;
+ }
+
+ t = lua_new_text(L, in, len, FALSE);
+ text_pos = lua_gettop(L);
+
+ if (!rspamd_lua_universal_pcall(L, lua_cbref,
+ G_STRLOC, 1, "utii", &err,
+ "rspamd{task}", task,
+ text_pos, start, end)) {
+ msg_warn_task("cannot call for re_cache_check_lua_condition for re %s: %e",
+ rspamd_regexp_get_pattern(re), err);
+ g_error_free(err);
+ lua_settop(L, text_pos - 1);
+
+ return TRUE;
+ }
+
+ gboolean res = lua_toboolean(L, -1);
+
+ lua_settop(L, text_pos - 1);
+
+ return res;
+}
+
+static guint
+rspamd_re_cache_process_pcre(struct rspamd_re_runtime *rt,
+ rspamd_regexp_t *re, struct rspamd_task *task,
+ const guchar *in, gsize len,
+ gboolean is_raw,
+ gint lua_cbref)
+{
+ guint r = 0;
+ const gchar *start = NULL, *end = NULL;
+ guint max_hits = rspamd_regexp_get_maxhits(re);
+ guint64 id = rspamd_regexp_get_cache_id(re);
+ gdouble t1 = NAN, t2, pr;
+ const gdouble slow_time = 1e8;
+
+ if (in == NULL) {
+ return rt->results[id];
+ }
+
+ if (len == 0) {
+ return rt->results[id];
+ }
+
+ if (rt->cache->max_re_data > 0 && len > rt->cache->max_re_data) {
+ len = rt->cache->max_re_data;
+ }
+
+ r = rt->results[id];
+
+ if (max_hits == 0 || r < max_hits) {
+ pr = rspamd_random_double_fast();
+
+ if (pr > 0.9) {
+ t1 = rspamd_get_ticks(TRUE);
+ }
+
+ while (rspamd_regexp_search(re,
+ in,
+ len,
+ &start,
+ &end,
+ is_raw,
+ NULL)) {
+ if (rspamd_re_cache_check_lua_condition(task, re, in, len,
+ start - (const gchar *) in, end - (const gchar *) in, lua_cbref)) {
+ r++;
+ msg_debug_re_task("found regexp /%s/, total hits: %d",
+ rspamd_regexp_get_pattern(re), r);
+ }
+
+ if (max_hits > 0 && r >= max_hits) {
+ break;
+ }
+ }
+
+ rt->results[id] += r;
+ rt->stat.regexp_checked++;
+ rt->stat.bytes_scanned_pcre += len;
+ rt->stat.bytes_scanned += len;
+
+ if (r > 0) {
+ rt->stat.regexp_matched += r;
+ }
+
+ if (!isnan(t1)) {
+ t2 = rspamd_get_ticks(TRUE);
+
+ if (t2 - t1 > slow_time) {
+ rspamd_symcache_enable_profile(task);
+ msg_info_task("regexp '%16s' took %.0f ticks to execute",
+ rspamd_regexp_get_pattern(re), t2 - t1);
+ }
+ }
+ }
+
+ return r;
+}
+
+#ifdef WITH_HYPERSCAN
+struct rspamd_re_hyperscan_cbdata {
+ struct rspamd_re_runtime *rt;
+ const guchar **ins;
+ const guint *lens;
+ guint count;
+ rspamd_regexp_t *re;
+ struct rspamd_task *task;
+};
+
+static gint
+rspamd_re_cache_hyperscan_cb(unsigned int id,
+ unsigned long long from,
+ unsigned long long to,
+ unsigned int flags,
+ void *ud)
+{
+ struct rspamd_re_hyperscan_cbdata *cbdata = ud;
+ struct rspamd_re_runtime *rt;
+ struct rspamd_re_cache_elt *cache_elt;
+ guint ret, maxhits, i, processed;
+ struct rspamd_task *task;
+
+ rt = cbdata->rt;
+ task = cbdata->task;
+ cache_elt = g_ptr_array_index(rt->cache->re, id);
+ maxhits = rspamd_regexp_get_maxhits(cache_elt->re);
+
+ if (cache_elt->match_type == RSPAMD_RE_CACHE_HYPERSCAN) {
+ if (rspamd_re_cache_check_lua_condition(task, cache_elt->re,
+ cbdata->ins[0], cbdata->lens[0], from, to, cache_elt->lua_cbref)) {
+ ret = 1;
+ setbit(rt->checked, id);
+
+ if (maxhits == 0 || rt->results[id] < maxhits) {
+ rt->results[id] += ret;
+ rt->stat.regexp_matched++;
+ }
+ msg_debug_re_task("found regexp /%s/ using hyperscan only, total hits: %d",
+ rspamd_regexp_get_pattern(cache_elt->re), rt->results[id]);
+ }
+ }
+ else {
+ if (!isset(rt->checked, id)) {
+
+ processed = 0;
+
+ for (i = 0; i < cbdata->count; i++) {
+ rspamd_re_cache_process_pcre(rt,
+ cache_elt->re,
+ cbdata->task,
+ cbdata->ins[i],
+ cbdata->lens[i],
+ FALSE,
+ cache_elt->lua_cbref);
+ setbit(rt->checked, id);
+
+ processed += cbdata->lens[i];
+
+ if (processed >= to) {
+ break;
+ }
+ }
+ }
+ }
+
+ return 0;
+}
+#endif
+
+static guint
+rspamd_re_cache_process_regexp_data(struct rspamd_re_runtime *rt,
+ rspamd_regexp_t *re, struct rspamd_task *task,
+ const guchar **in, guint *lens,
+ guint count,
+ gboolean is_raw,
+ gboolean *processed_hyperscan)
+{
+
+ guint64 re_id;
+ guint ret = 0;
+ guint i;
+ struct rspamd_re_cache_elt *cache_elt;
+
+ re_id = rspamd_regexp_get_cache_id(re);
+
+ if (count == 0 || in == NULL) {
+ /* We assume this as absence of the specified data */
+ setbit(rt->checked, re_id);
+ rt->results[re_id] = ret;
+ return ret;
+ }
+
+ cache_elt = (struct rspamd_re_cache_elt *) g_ptr_array_index(rt->cache->re, re_id);
+
+#ifndef WITH_HYPERSCAN
+ for (i = 0; i < count; i++) {
+ ret = rspamd_re_cache_process_pcre(rt,
+ re,
+ task,
+ in[i],
+ lens[i],
+ is_raw,
+ cache_elt->lua_cbref);
+ rt->results[re_id] = ret;
+ }
+
+ setbit(rt->checked, re_id);
+#else
+ struct rspamd_re_class *re_class;
+ struct rspamd_re_hyperscan_cbdata cbdata;
+
+ cache_elt = g_ptr_array_index(rt->cache->re, re_id);
+ re_class = rspamd_regexp_get_class(re);
+
+ if (rt->cache->disable_hyperscan || cache_elt->match_type == RSPAMD_RE_CACHE_PCRE ||
+ !rt->has_hs || (is_raw && re_class->has_utf8)) {
+ for (i = 0; i < count; i++) {
+ ret = rspamd_re_cache_process_pcre(rt,
+ re,
+ task,
+ in[i],
+ lens[i],
+ is_raw,
+ cache_elt->lua_cbref);
+ }
+
+ setbit(rt->checked, re_id);
+ }
+ else {
+ for (i = 0; i < count; i++) {
+ /* For Hyperscan we can probably safely disable all those limits */
+#if 0
+ if (rt->cache->max_re_data > 0 && lens[i] > rt->cache->max_re_data) {
+ lens[i] = rt->cache->max_re_data;
+ }
+#endif
+ rt->stat.bytes_scanned += lens[i];
+ }
+
+ g_assert(re_class->hs_scratch != NULL);
+ g_assert(re_class->hs_db != NULL);
+
+ /* Go through hyperscan API */
+ for (i = 0; i < count; i++) {
+ cbdata.ins = &in[i];
+ cbdata.re = re;
+ cbdata.rt = rt;
+ cbdata.lens = &lens[i];
+ cbdata.count = 1;
+ cbdata.task = task;
+
+ if ((hs_scan(rspamd_hyperscan_get_database(re_class->hs_db),
+ in[i], lens[i], 0,
+ re_class->hs_scratch,
+ rspamd_re_cache_hyperscan_cb, &cbdata)) != HS_SUCCESS) {
+ ret = 0;
+ }
+ else {
+ ret = rt->results[re_id];
+ *processed_hyperscan = TRUE;
+ }
+ }
+ }
+#endif
+
+ return ret;
+}
+
+static void
+rspamd_re_cache_finish_class(struct rspamd_task *task,
+ struct rspamd_re_runtime *rt,
+ struct rspamd_re_class *re_class,
+ const gchar *class_name)
+{
+#ifdef WITH_HYPERSCAN
+ guint i;
+ guint64 re_id;
+ guint found = 0;
+
+ /* Set all bits that are not checked and included in hyperscan to 1 */
+ for (i = 0; i < re_class->nhs; i++) {
+ re_id = re_class->hs_ids[i];
+
+ if (!isset(rt->checked, re_id)) {
+ g_assert(rt->results[re_id] == 0);
+ rt->results[re_id] = 0;
+ setbit(rt->checked, re_id);
+ }
+ else {
+ found++;
+ }
+ }
+
+ msg_debug_re_task("finished hyperscan for class %s; %d "
+ "matches found; %d hyperscan supported regexps; %d total regexps",
+ class_name, found, re_class->nhs, (gint) g_hash_table_size(re_class->re));
+#endif
+}
+
+static gboolean
+rspamd_re_cache_process_selector(struct rspamd_task *task,
+ struct rspamd_re_runtime *rt,
+ const gchar *name,
+ guchar ***svec,
+ guint **lenvec,
+ guint *n)
+{
+ gint ref;
+ khiter_t k;
+ lua_State *L;
+ gint err_idx, ret;
+ struct rspamd_task **ptask;
+ gboolean result = FALSE;
+ struct rspamd_re_cache *cache = rt->cache;
+ struct rspamd_re_selector_result *sr;
+
+ L = cache->L;
+ k = kh_get(lua_selectors_hash, cache->selectors, (gchar *) name);
+
+ if (k == kh_end(cache->selectors)) {
+ msg_err_task("cannot find selector %s, not registered", name);
+
+ return FALSE;
+ }
+
+ ref = kh_value(cache->selectors, k);
+
+ /* First, search for the cached result */
+ if (rt->sel_cache) {
+ k = kh_get(selectors_results_hash, rt->sel_cache, ref);
+
+ if (k != kh_end(rt->sel_cache)) {
+ sr = &kh_value(rt->sel_cache, k);
+
+ *svec = sr->scvec;
+ *lenvec = sr->lenvec;
+ *n = sr->cnt;
+
+ return TRUE;
+ }
+ }
+ else {
+ rt->sel_cache = kh_init(selectors_results_hash);
+ }
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, ref);
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ if ((ret = lua_pcall(L, 1, 1, err_idx)) != 0) {
+ msg_err_task("call to selector %s "
+ "failed (%d): %s",
+ name, ret,
+ lua_tostring(L, -1));
+ }
+ else {
+ struct rspamd_lua_text *txt;
+ gsize slen;
+ const gchar *sel_data;
+
+ if (lua_type(L, -1) != LUA_TTABLE) {
+ txt = lua_check_text_or_string(L, -1);
+
+
+ if (txt) {
+ msg_debug_re_cache("re selector %s returned 1 element", name);
+ sel_data = txt->start;
+ slen = txt->len;
+ *n = 1;
+ *svec = g_malloc(sizeof(guchar *));
+ *lenvec = g_malloc(sizeof(guint));
+ (*svec)[0] = g_malloc(slen);
+ memcpy((*svec)[0], sel_data, slen);
+ (*lenvec)[0] = slen;
+ result = TRUE;
+ }
+ else {
+ msg_debug_re_cache("re selector %s returned NULL", name);
+ }
+ }
+ else {
+ *n = rspamd_lua_table_size(L, -1);
+
+ msg_debug_re_cache("re selector %s returned %d elements", name, *n);
+
+ if (*n > 0) {
+ *svec = g_malloc(sizeof(guchar *) * (*n));
+ *lenvec = g_malloc(sizeof(guint) * (*n));
+
+ for (int i = 0; i < *n; i++) {
+ lua_rawgeti(L, -1, i + 1);
+
+ txt = lua_check_text_or_string(L, -1);
+ if (txt && txt->len > 0) {
+ sel_data = txt->start;
+ slen = txt->len;
+ (*svec)[i] = g_malloc(slen);
+ memcpy((*svec)[i], sel_data, slen);
+ }
+ else {
+ /* A hack to avoid malloc(0) */
+ sel_data = "";
+ slen = 0;
+ (*svec)[i] = g_malloc(1);
+ memcpy((*svec)[i], sel_data, 1);
+ }
+
+ (*lenvec)[i] = slen;
+ lua_pop(L, 1);
+ }
+ }
+
+ /* Empty table is also a valid result */
+ result = TRUE;
+ }
+ }
+
+ lua_settop(L, err_idx - 1);
+
+ if (result) {
+ k = kh_put(selectors_results_hash, rt->sel_cache, ref, &ret);
+ sr = &kh_value(rt->sel_cache, k);
+
+ sr->cnt = *n;
+ sr->scvec = *svec;
+ sr->lenvec = *lenvec;
+ }
+
+ return result;
+}
+
+static inline guint
+rspamd_process_words_vector(GArray *words,
+ const guchar **scvec,
+ guint *lenvec,
+ struct rspamd_re_class *re_class,
+ guint cnt,
+ gboolean *raw)
+{
+ guint j;
+ rspamd_stat_token_t *tok;
+
+ if (words) {
+ for (j = 0; j < words->len; j++) {
+ tok = &g_array_index(words, rspamd_stat_token_t, j);
+
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) {
+ if (!(tok->flags & RSPAMD_STAT_TOKEN_FLAG_UTF)) {
+ if (!re_class->has_utf8) {
+ *raw = TRUE;
+ }
+ else {
+ continue; /* Skip */
+ }
+ }
+ }
+ else {
+ continue; /* Skip non text */
+ }
+
+ if (re_class->type == RSPAMD_RE_RAWWORDS) {
+ if (tok->original.len > 0) {
+ scvec[cnt] = tok->original.begin;
+ lenvec[cnt++] = tok->original.len;
+ }
+ }
+ else if (re_class->type == RSPAMD_RE_WORDS) {
+ if (tok->normalized.len > 0) {
+ scvec[cnt] = tok->normalized.begin;
+ lenvec[cnt++] = tok->normalized.len;
+ }
+ }
+ else {
+ /* Stemmed words */
+ if (tok->stemmed.len > 0) {
+ scvec[cnt] = tok->stemmed.begin;
+ lenvec[cnt++] = tok->stemmed.len;
+ }
+ }
+ }
+ }
+
+ return cnt;
+}
+
+static guint
+rspamd_re_cache_process_headers_list(struct rspamd_task *task,
+ struct rspamd_re_runtime *rt,
+ rspamd_regexp_t *re,
+ struct rspamd_re_class *re_class,
+ struct rspamd_mime_header *rh,
+ gboolean is_strong,
+ gboolean *processed_hyperscan)
+{
+ const guchar **scvec, *in;
+ gboolean raw = FALSE;
+ guint *lenvec;
+ struct rspamd_mime_header *cur;
+ guint cnt = 0, i = 0, ret = 0;
+
+ DL_COUNT(rh, cur, cnt);
+
+ scvec = g_malloc(sizeof(*scvec) * cnt);
+ lenvec = g_malloc(sizeof(*lenvec) * cnt);
+
+ DL_FOREACH(rh, cur)
+ {
+
+ if (is_strong && strcmp(cur->name, re_class->type_data) != 0) {
+ /* Skip a different case */
+ continue;
+ }
+
+ if (re_class->type == RSPAMD_RE_RAWHEADER) {
+ in = (const guchar *) cur->value;
+ lenvec[i] = strlen(cur->value);
+
+ if (rspamd_fast_utf8_validate(in, lenvec[i]) != 0) {
+ raw = TRUE;
+ }
+ }
+ else {
+ in = (const guchar *) cur->decoded;
+ /* Validate input^W^WNo need to validate as it is already valid */
+ if (!in) {
+ lenvec[i] = 0;
+ scvec[i] = (guchar *) "";
+ continue;
+ }
+
+ lenvec[i] = strlen(in);
+ }
+
+ scvec[i] = in;
+
+ i++;
+ }
+
+ if (i > 0) {
+ ret = rspamd_re_cache_process_regexp_data(rt, re,
+ task, scvec, lenvec, i, raw, processed_hyperscan);
+ msg_debug_re_task("checking header %s regexp: %s=%*s -> %d",
+ re_class->type_data,
+ rspamd_regexp_get_pattern(re),
+ (int) lenvec[0], scvec[0], ret);
+ }
+
+ g_free(scvec);
+ g_free(lenvec);
+
+ return ret;
+}
+
+/*
+ * Calculates the specified regexp for the specified class if it's not calculated
+ */
+static guint
+rspamd_re_cache_exec_re(struct rspamd_task *task,
+ struct rspamd_re_runtime *rt,
+ rspamd_regexp_t *re,
+ struct rspamd_re_class *re_class,
+ gboolean is_strong)
+{
+ guint ret = 0, i, re_id;
+ struct rspamd_mime_header *rh;
+ const gchar *in;
+ const guchar **scvec = NULL;
+ guint *lenvec = NULL;
+ gboolean raw = FALSE, processed_hyperscan = FALSE;
+ struct rspamd_mime_text_part *text_part;
+ struct rspamd_mime_part *mime_part;
+ struct rspamd_url *url;
+ guint len = 0, cnt = 0;
+ const gchar *class_name;
+
+ class_name = rspamd_re_cache_type_to_string(re_class->type);
+ msg_debug_re_task("start check re type: %s: /%s/",
+ class_name,
+ rspamd_regexp_get_pattern(re));
+ re_id = rspamd_regexp_get_cache_id(re);
+
+ switch (re_class->type) {
+ case RSPAMD_RE_HEADER:
+ case RSPAMD_RE_RAWHEADER:
+ /* Get list of specified headers */
+ rh = rspamd_message_get_header_array(task,
+ re_class->type_data, FALSE);
+
+ if (rh) {
+ ret = rspamd_re_cache_process_headers_list(task, rt, re,
+ re_class, rh, is_strong, &processed_hyperscan);
+ msg_debug_re_task("checked header(%s) regexp: %s -> %d",
+ (const char *) re_class->type_data,
+ rspamd_regexp_get_pattern(re),
+ ret);
+ }
+ break;
+ case RSPAMD_RE_ALLHEADER:
+ raw = TRUE;
+ in = MESSAGE_FIELD(task, raw_headers_content).begin;
+ len = MESSAGE_FIELD(task, raw_headers_content).len;
+ ret = rspamd_re_cache_process_regexp_data(rt, re,
+ task, (const guchar **) &in, &len, 1, raw, &processed_hyperscan);
+ msg_debug_re_task("checked allheader regexp: %s -> %d",
+ rspamd_regexp_get_pattern(re), ret);
+ break;
+ case RSPAMD_RE_MIMEHEADER:
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, mime_part)
+ {
+ if (mime_part->parent_part == NULL ||
+ !IS_PART_MULTIPART(mime_part->parent_part) ||
+ IS_PART_MESSAGE(mime_part)) {
+ /* We filter parts that have no multipart parent or are a messages here */
+ continue;
+ }
+ rh = rspamd_message_get_header_from_hash(mime_part->raw_headers,
+ re_class->type_data, FALSE);
+
+ if (rh) {
+ ret += rspamd_re_cache_process_headers_list(task, rt, re,
+ re_class, rh, is_strong, &processed_hyperscan);
+ }
+ msg_debug_re_task("checked mime header(%s) regexp: %s -> %d",
+ (const char *) re_class->type_data,
+ rspamd_regexp_get_pattern(re),
+ ret);
+ }
+ break;
+ case RSPAMD_RE_MIME:
+ case RSPAMD_RE_RAWMIME:
+ /* Iterate through text parts */
+ if (MESSAGE_FIELD(task, text_parts)->len > 0) {
+ cnt = MESSAGE_FIELD(task, text_parts)->len;
+ scvec = g_malloc(sizeof(*scvec) * cnt);
+ lenvec = g_malloc(sizeof(*lenvec) * cnt);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, text_part)
+ {
+ /* Select data for regexp */
+ if (re_class->type == RSPAMD_RE_RAWMIME) {
+ if (text_part->raw.len == 0) {
+ len = 0;
+ in = "";
+ }
+ else {
+ in = text_part->raw.begin;
+ len = text_part->raw.len;
+ }
+
+ raw = TRUE;
+ }
+ else {
+ /* Skip empty parts */
+ if (IS_TEXT_PART_EMPTY(text_part)) {
+ len = 0;
+ in = "";
+ }
+ else {
+ /* Check raw flags */
+ if (!IS_TEXT_PART_UTF(text_part)) {
+ raw = TRUE;
+ }
+
+ in = text_part->utf_content.begin;
+ len = text_part->utf_content.len;
+ }
+ }
+
+ scvec[i] = (guchar *) in;
+ lenvec[i] = len;
+ }
+
+ ret = rspamd_re_cache_process_regexp_data(rt, re,
+ task, scvec, lenvec, cnt, raw, &processed_hyperscan);
+ msg_debug_re_task("checked mime regexp: %s -> %d",
+ rspamd_regexp_get_pattern(re), ret);
+ g_free(scvec);
+ g_free(lenvec);
+ }
+ break;
+ case RSPAMD_RE_URL:
+ cnt = kh_size(MESSAGE_FIELD(task, urls));
+
+ if (cnt > 0) {
+ scvec = g_malloc(sizeof(*scvec) * cnt);
+ lenvec = g_malloc(sizeof(*lenvec) * cnt);
+ i = 0;
+ raw = FALSE;
+
+ kh_foreach_key(MESSAGE_FIELD(task, urls), url, {
+ if ((url->protocol & PROTOCOL_MAILTO)) {
+ continue;
+ }
+ in = url->string;
+ len = url->urllen;
+
+ if (len > 0 && !(url->flags & RSPAMD_URL_FLAG_IMAGE)) {
+ scvec[i] = (guchar *) in;
+ lenvec[i++] = len;
+ }
+ });
+
+ /* URL regexps do not include emails, that's why the code below is commented */
+#if 0
+ g_hash_table_iter_init (&it, MESSAGE_FIELD (task, emails));
+
+ while (g_hash_table_iter_next (&it, &k, &v)) {
+ url = v;
+ in = url->string;
+ len = url->urllen;
+
+ if (len > 0 && !(url->flags & RSPAMD_URL_FLAG_IMAGE)) {
+ scvec[i] = (guchar *) in;
+ lenvec[i++] = len;
+ }
+ }
+#endif
+ ret = rspamd_re_cache_process_regexp_data(rt, re,
+ task, scvec, lenvec, i, raw, &processed_hyperscan);
+ msg_debug_re_task("checked url regexp: %s -> %d",
+ rspamd_regexp_get_pattern(re), ret);
+ g_free(scvec);
+ g_free(lenvec);
+ }
+ break;
+ case RSPAMD_RE_EMAIL:
+ cnt = kh_size(MESSAGE_FIELD(task, urls));
+
+ if (cnt > 0) {
+ scvec = g_malloc(sizeof(*scvec) * cnt);
+ lenvec = g_malloc(sizeof(*lenvec) * cnt);
+ i = 0;
+ raw = FALSE;
+
+ kh_foreach_key(MESSAGE_FIELD(task, urls), url, {
+ if (!(url->protocol & PROTOCOL_MAILTO)) {
+ continue;
+ }
+ if (url->userlen == 0 || url->hostlen == 0) {
+ continue;
+ }
+
+ in = rspamd_url_user_unsafe(url);
+ len = url->userlen + 1 + url->hostlen;
+ scvec[i] = (guchar *) in;
+ lenvec[i++] = len;
+ });
+
+ ret = rspamd_re_cache_process_regexp_data(rt, re,
+ task, scvec, lenvec, i, raw, &processed_hyperscan);
+ msg_debug_re_task("checked email regexp: %s -> %d",
+ rspamd_regexp_get_pattern(re), ret);
+ g_free(scvec);
+ g_free(lenvec);
+ }
+ break;
+ case RSPAMD_RE_BODY:
+ raw = TRUE;
+ in = task->msg.begin;
+ len = task->msg.len;
+
+ ret = rspamd_re_cache_process_regexp_data(rt, re, task,
+ (const guchar **) &in, &len, 1, raw, &processed_hyperscan);
+ msg_debug_re_task("checked rawbody regexp: %s -> %d",
+ rspamd_regexp_get_pattern(re), ret);
+ break;
+ case RSPAMD_RE_SABODY:
+ /* According to SA docs:
+ * The 'body' in this case is the textual parts of the message body;
+ * any non-text MIME parts are stripped, and the message decoded from
+ * Quoted-Printable or Base-64-encoded format if necessary. The message
+ * Subject header is considered part of the body and becomes the first
+ * paragraph when running the rules. All HTML tags and line breaks will
+ * be removed before matching.
+ */
+ cnt = MESSAGE_FIELD(task, text_parts)->len + 1;
+ scvec = g_malloc(sizeof(*scvec) * cnt);
+ lenvec = g_malloc(sizeof(*lenvec) * cnt);
+
+ /*
+ * Body rules also include the Subject as the first line
+ * of the body content.
+ */
+
+ rh = rspamd_message_get_header_array(task, "Subject", FALSE);
+
+ if (rh) {
+ scvec[0] = (guchar *) rh->decoded;
+ lenvec[0] = strlen(rh->decoded);
+ }
+ else {
+ scvec[0] = (guchar *) "";
+ lenvec[0] = 0;
+ }
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, text_part)
+ {
+ if (text_part->utf_stripped_content) {
+ scvec[i + 1] = (guchar *) text_part->utf_stripped_content->data;
+ lenvec[i + 1] = text_part->utf_stripped_content->len;
+
+ if (!IS_TEXT_PART_UTF(text_part)) {
+ raw = TRUE;
+ }
+ }
+ else {
+ scvec[i + 1] = (guchar *) "";
+ lenvec[i + 1] = 0;
+ }
+ }
+
+ ret = rspamd_re_cache_process_regexp_data(rt, re,
+ task, scvec, lenvec, cnt, raw, &processed_hyperscan);
+ msg_debug_re_task("checked sa body regexp: %s -> %d",
+ rspamd_regexp_get_pattern(re), ret);
+ g_free(scvec);
+ g_free(lenvec);
+ break;
+ case RSPAMD_RE_SARAWBODY:
+ /* According to SA docs:
+ * The 'raw body' of a message is the raw data inside all textual
+ * parts. The text will be decoded from base64 or quoted-printable
+ * encoding, but HTML tags and line breaks will still be present.
+ * Multiline expressions will need to be used to match strings that are
+ * broken by line breaks.
+ */
+ if (MESSAGE_FIELD(task, text_parts)->len > 0) {
+ cnt = MESSAGE_FIELD(task, text_parts)->len;
+ scvec = g_malloc(sizeof(*scvec) * cnt);
+ lenvec = g_malloc(sizeof(*lenvec) * cnt);
+
+ for (i = 0; i < cnt; i++) {
+ text_part = g_ptr_array_index(MESSAGE_FIELD(task, text_parts), i);
+
+ if (text_part->parsed.len > 0) {
+ scvec[i] = (guchar *) text_part->parsed.begin;
+ lenvec[i] = text_part->parsed.len;
+
+ if (!IS_TEXT_PART_UTF(text_part)) {
+ raw = TRUE;
+ }
+ }
+ else {
+ scvec[i] = (guchar *) "";
+ lenvec[i] = 0;
+ }
+ }
+
+ ret = rspamd_re_cache_process_regexp_data(rt, re,
+ task, scvec, lenvec, cnt, raw, &processed_hyperscan);
+ msg_debug_re_task("checked sa rawbody regexp: %s -> %d",
+ rspamd_regexp_get_pattern(re), ret);
+ g_free(scvec);
+ g_free(lenvec);
+ }
+ break;
+ case RSPAMD_RE_WORDS:
+ case RSPAMD_RE_STEMWORDS:
+ case RSPAMD_RE_RAWWORDS:
+ if (MESSAGE_FIELD(task, text_parts)->len > 0) {
+ cnt = 0;
+ raw = FALSE;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, text_part)
+ {
+ if (text_part->utf_words) {
+ cnt += text_part->utf_words->len;
+ }
+ }
+
+ if (task->meta_words && task->meta_words->len > 0) {
+ cnt += task->meta_words->len;
+ }
+
+ if (cnt > 0) {
+ scvec = g_malloc(sizeof(*scvec) * cnt);
+ lenvec = g_malloc(sizeof(*lenvec) * cnt);
+
+ cnt = 0;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, text_part)
+ {
+ if (text_part->utf_words) {
+ cnt = rspamd_process_words_vector(text_part->utf_words,
+ scvec, lenvec, re_class, cnt, &raw);
+ }
+ }
+
+ if (task->meta_words) {
+ cnt = rspamd_process_words_vector(task->meta_words,
+ scvec, lenvec, re_class, cnt, &raw);
+ }
+
+ ret = rspamd_re_cache_process_regexp_data(rt, re,
+ task, scvec, lenvec, cnt, raw, &processed_hyperscan);
+
+ msg_debug_re_task("checked sa words regexp: %s -> %d",
+ rspamd_regexp_get_pattern(re), ret);
+ g_free(scvec);
+ g_free(lenvec);
+ }
+ }
+ break;
+ case RSPAMD_RE_SELECTOR:
+ if (rspamd_re_cache_process_selector(task, rt,
+ re_class->type_data,
+ (guchar ***) &scvec,
+ &lenvec, &cnt)) {
+
+ ret = rspamd_re_cache_process_regexp_data(rt, re,
+ task, scvec, lenvec, cnt, raw, &processed_hyperscan);
+ msg_debug_re_task("checked selector(%s) regexp: %s -> %d",
+ re_class->type_data,
+ rspamd_regexp_get_pattern(re), ret);
+
+ /* Do not free vectors as they are managed by rt->sel_cache */
+ }
+ break;
+ case RSPAMD_RE_MAX:
+ msg_err_task("regexp of class invalid has been called: %s",
+ rspamd_regexp_get_pattern(re));
+ break;
+ }
+
+#if WITH_HYPERSCAN
+ if (processed_hyperscan) {
+ rspamd_re_cache_finish_class(task, rt, re_class, class_name);
+ }
+#endif
+
+ setbit(rt->checked, re_id);
+
+ return rt->results[re_id];
+}
+
+gint rspamd_re_cache_process(struct rspamd_task *task,
+ rspamd_regexp_t *re,
+ enum rspamd_re_type type,
+ gconstpointer type_data,
+ gsize datalen,
+ gboolean is_strong)
+{
+ guint64 re_id;
+ struct rspamd_re_class *re_class;
+ struct rspamd_re_cache *cache;
+ struct rspamd_re_runtime *rt;
+
+ g_assert(task != NULL);
+ rt = task->re_rt;
+ g_assert(rt != NULL);
+ g_assert(re != NULL);
+
+ cache = rt->cache;
+ re_id = rspamd_regexp_get_cache_id(re);
+
+ if (re_id == RSPAMD_INVALID_ID || re_id > cache->nre) {
+ msg_err_task("re '%s' has no valid id for the cache",
+ rspamd_regexp_get_pattern(re));
+ return 0;
+ }
+
+ if (isset(rt->checked, re_id)) {
+ /* Fast path */
+ rt->stat.regexp_fast_cached++;
+ return rt->results[re_id];
+ }
+ else {
+ /* Slow path */
+ re_class = rspamd_regexp_get_class(re);
+
+ if (re_class == NULL) {
+ msg_err_task("cannot find re class for regexp '%s'",
+ rspamd_regexp_get_pattern(re));
+ return 0;
+ }
+
+ return rspamd_re_cache_exec_re(task, rt, re, re_class,
+ is_strong);
+ }
+
+ return 0;
+}
+
+int rspamd_re_cache_process_ffi(void *ptask,
+ void *pre,
+ int type,
+ void *type_data,
+ int is_strong)
+{
+ struct rspamd_lua_regexp **lua_re = pre;
+ struct rspamd_task **real_task = ptask;
+ gsize typelen = 0;
+
+ if (type_data) {
+ typelen = strlen(type_data);
+ }
+
+ return rspamd_re_cache_process(*real_task, (*lua_re)->re,
+ type, type_data, typelen, is_strong);
+}
+
+void rspamd_re_cache_runtime_destroy(struct rspamd_re_runtime *rt)
+{
+ g_assert(rt != NULL);
+
+ if (rt->sel_cache) {
+ struct rspamd_re_selector_result sr;
+
+ kh_foreach_value(rt->sel_cache, sr, {
+ for (guint i = 0; i < sr.cnt; i++) {
+ g_free((gpointer) sr.scvec[i]);
+ }
+
+ g_free(sr.scvec);
+ g_free(sr.lenvec);
+ });
+ kh_destroy(selectors_results_hash, rt->sel_cache);
+ }
+
+ REF_RELEASE(rt->cache);
+ g_free(rt);
+}
+
+void rspamd_re_cache_unref(struct rspamd_re_cache *cache)
+{
+ if (cache) {
+ REF_RELEASE(cache);
+ }
+}
+
+struct rspamd_re_cache *
+rspamd_re_cache_ref(struct rspamd_re_cache *cache)
+{
+ if (cache) {
+ REF_RETAIN(cache);
+ }
+
+ return cache;
+}
+
+guint rspamd_re_cache_set_limit(struct rspamd_re_cache *cache, guint limit)
+{
+ guint old;
+
+ g_assert(cache != NULL);
+
+ old = cache->max_re_data;
+ cache->max_re_data = limit;
+
+ return old;
+}
+
+const gchar *
+rspamd_re_cache_type_to_string(enum rspamd_re_type type)
+{
+ const gchar *ret = "unknown";
+
+ switch (type) {
+ case RSPAMD_RE_HEADER:
+ ret = "header";
+ break;
+ case RSPAMD_RE_RAWHEADER:
+ ret = "raw header";
+ break;
+ case RSPAMD_RE_MIMEHEADER:
+ ret = "mime header";
+ break;
+ case RSPAMD_RE_ALLHEADER:
+ ret = "all headers";
+ break;
+ case RSPAMD_RE_MIME:
+ ret = "part";
+ break;
+ case RSPAMD_RE_RAWMIME:
+ ret = "raw part";
+ break;
+ case RSPAMD_RE_BODY:
+ ret = "rawbody";
+ break;
+ case RSPAMD_RE_URL:
+ ret = "url";
+ break;
+ case RSPAMD_RE_EMAIL:
+ ret = "email";
+ break;
+ case RSPAMD_RE_SABODY:
+ ret = "sa body";
+ break;
+ case RSPAMD_RE_SARAWBODY:
+ ret = "sa raw body";
+ break;
+ case RSPAMD_RE_SELECTOR:
+ ret = "selector";
+ break;
+ case RSPAMD_RE_WORDS:
+ ret = "words";
+ break;
+ case RSPAMD_RE_RAWWORDS:
+ ret = "raw_words";
+ break;
+ case RSPAMD_RE_STEMWORDS:
+ ret = "stem_words";
+ break;
+ case RSPAMD_RE_MAX:
+ default:
+ ret = "invalid class";
+ break;
+ }
+
+ return ret;
+}
+
+enum rspamd_re_type
+rspamd_re_cache_type_from_string(const char *str)
+{
+ enum rspamd_re_type ret;
+ guint64 h;
+
+ /*
+ * To optimize this function, we apply hash to input string and
+ * pre-select it from the values
+ */
+
+ if (str != NULL) {
+ h = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64,
+ str, strlen(str), 0xdeadbabe);
+
+ switch (h) {
+ case G_GUINT64_CONSTANT(0x298b9c8a58887d44): /* header */
+ ret = RSPAMD_RE_HEADER;
+ break;
+ case G_GUINT64_CONSTANT(0x467bfb5cd7ddf890): /* rawheader */
+ ret = RSPAMD_RE_RAWHEADER;
+ break;
+ case G_GUINT64_CONSTANT(0xda081341fb600389): /* mime */
+ ret = RSPAMD_RE_MIME;
+ break;
+ case G_GUINT64_CONSTANT(0xc35831e067a8221d): /* rawmime */
+ ret = RSPAMD_RE_RAWMIME;
+ break;
+ case G_GUINT64_CONSTANT(0xc625e13dbe636de2): /* body */
+ case G_GUINT64_CONSTANT(0xCCDEBA43518F721C): /* message */
+ ret = RSPAMD_RE_BODY;
+ break;
+ case G_GUINT64_CONSTANT(0x286edbe164c791d2): /* url */
+ case G_GUINT64_CONSTANT(0x7D9ACDF6685661A1): /* uri */
+ ret = RSPAMD_RE_URL;
+ break;
+ case G_GUINT64_CONSTANT(0x7e232b0f60b571be): /* email */
+ ret = RSPAMD_RE_EMAIL;
+ break;
+ case G_GUINT64_CONSTANT(0x796d62205a8778c7): /* allheader */
+ ret = RSPAMD_RE_ALLHEADER;
+ break;
+ case G_GUINT64_CONSTANT(0xa3c6c153b3b00a5e): /* mimeheader */
+ ret = RSPAMD_RE_MIMEHEADER;
+ break;
+ case G_GUINT64_CONSTANT(0x7794501506e604e9): /* sabody */
+ ret = RSPAMD_RE_SABODY;
+ break;
+ case G_GUINT64_CONSTANT(0x28828962E7D2A05F): /* sarawbody */
+ ret = RSPAMD_RE_SARAWBODY;
+ break;
+ default:
+ ret = RSPAMD_RE_MAX;
+ break;
+ }
+ }
+ else {
+ ret = RSPAMD_RE_MAX;
+ }
+
+ return ret;
+}
+
+#ifdef WITH_HYPERSCAN
+static gchar *
+rspamd_re_cache_hs_pattern_from_pcre(rspamd_regexp_t *re)
+{
+ /*
+ * Workaround for bug in ragel 7.0.0.11
+ * https://github.com/intel/hyperscan/issues/133
+ */
+ const gchar *pat = rspamd_regexp_get_pattern(re);
+ guint flags = rspamd_regexp_get_flags(re), esc_flags = RSPAMD_REGEXP_ESCAPE_RE;
+ gchar *escaped;
+ gsize esc_len;
+
+ if (flags & RSPAMD_REGEXP_FLAG_UTF) {
+ esc_flags |= RSPAMD_REGEXP_ESCAPE_UTF;
+ }
+
+ escaped = rspamd_str_regexp_escape(pat, strlen(pat), &esc_len, esc_flags);
+
+ return escaped;
+}
+
+static gboolean
+rspamd_re_cache_is_finite(struct rspamd_re_cache *cache,
+ rspamd_regexp_t *re, gint flags, gdouble max_time)
+{
+ pid_t cld;
+ gint status;
+ struct timespec ts;
+ hs_compile_error_t *hs_errors;
+ hs_database_t *test_db;
+ gdouble wait_time;
+ const gint max_tries = 10;
+ gint tries = 0, rc;
+ void (*old_hdl)(int);
+
+ wait_time = max_time / max_tries;
+ /* We need to restore SIGCHLD processing */
+ old_hdl = signal(SIGCHLD, SIG_DFL);
+ cld = fork();
+
+ if (cld == 0) {
+ /* Try to compile pattern */
+
+ gchar *pat = rspamd_re_cache_hs_pattern_from_pcre(re);
+
+ if (hs_compile(pat,
+ flags | HS_FLAG_PREFILTER,
+ HS_MODE_BLOCK,
+ &cache->plt,
+ &test_db,
+ &hs_errors) != HS_SUCCESS) {
+
+ msg_info_re_cache("cannot compile (prefilter mode) '%s' to hyperscan: '%s'",
+ pat,
+ hs_errors != NULL ? hs_errors->message : "unknown error");
+
+ hs_free_compile_error(hs_errors);
+ g_free(pat);
+
+ exit(EXIT_FAILURE);
+ }
+
+ g_free(pat);
+ exit(EXIT_SUCCESS);
+ }
+ else if (cld > 0) {
+ double_to_ts(wait_time, &ts);
+
+ while ((rc = waitpid(cld, &status, WNOHANG)) == 0 && tries++ < max_tries) {
+ (void) nanosleep(&ts, NULL);
+ }
+
+ /* Child has been terminated */
+ if (rc > 0) {
+ /* Forget about SIGCHLD after this point */
+ signal(SIGCHLD, old_hdl);
+
+ if (WIFEXITED(status) && WEXITSTATUS(status) == EXIT_SUCCESS) {
+ return TRUE;
+ }
+ else {
+ msg_err_re_cache(
+ "cannot approximate %s to hyperscan",
+ rspamd_regexp_get_pattern(re));
+
+ return FALSE;
+ }
+ }
+ else {
+ /* We consider that as timeout */
+ kill(cld, SIGKILL);
+ g_assert(waitpid(cld, &status, 0) != -1);
+ msg_err_re_cache(
+ "cannot approximate %s to hyperscan: timeout waiting",
+ rspamd_regexp_get_pattern(re));
+ signal(SIGCHLD, old_hdl);
+ }
+ }
+ else {
+ msg_err_re_cache(
+ "cannot approximate %s to hyperscan: fork failed: %s",
+ rspamd_regexp_get_pattern(re), strerror(errno));
+ signal(SIGCHLD, old_hdl);
+ }
+
+ return FALSE;
+}
+#endif
+
+#ifdef WITH_HYPERSCAN
+struct rspamd_re_cache_hs_compile_cbdata {
+ GHashTableIter it;
+ struct rspamd_re_cache *cache;
+ const char *cache_dir;
+ gdouble max_time;
+ gboolean silent;
+ guint total;
+ void (*cb)(guint ncompiled, GError *err, void *cbd);
+ void *cbd;
+};
+
+static void
+rspamd_re_cache_compile_err(EV_P_ ev_timer *w, GError *err,
+ struct rspamd_re_cache_hs_compile_cbdata *cbdata, bool is_fatal)
+{
+ cbdata->cb(cbdata->total, err, cbdata->cbd);
+
+ if (is_fatal) {
+ ev_timer_stop(EV_A_ w);
+ g_free(w);
+ g_free(cbdata);
+ }
+ else {
+ /* Continue compilation */
+ ev_timer_again(EV_A_ w);
+ }
+ g_error_free(err);
+}
+
+static void
+rspamd_re_cache_compile_timer_cb(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_re_cache_hs_compile_cbdata *cbdata =
+ (struct rspamd_re_cache_hs_compile_cbdata *) w->data;
+ GHashTableIter cit;
+ gpointer k, v;
+ struct rspamd_re_class *re_class;
+ gchar path[PATH_MAX], npath[PATH_MAX];
+ hs_database_t *test_db;
+ gint fd, i, n, *hs_ids = NULL, pcre_flags, re_flags;
+ rspamd_cryptobox_fast_hash_state_t crc_st;
+ guint64 crc;
+ rspamd_regexp_t *re;
+ hs_compile_error_t *hs_errors = NULL;
+ guint *hs_flags = NULL;
+ const hs_expr_ext_t **hs_exts = NULL;
+ gchar **hs_pats = NULL;
+ gchar *hs_serialized = NULL;
+ gsize serialized_len;
+ struct iovec iov[7];
+ struct rspamd_re_cache *cache;
+ GError *err;
+ pid_t our_pid = getpid();
+
+ cache = cbdata->cache;
+
+ if (!g_hash_table_iter_next(&cbdata->it, &k, &v)) {
+ /* All done */
+ ev_timer_stop(EV_A_ w);
+ cbdata->cb(cbdata->total, NULL, cbdata->cbd);
+ g_free(w);
+ g_free(cbdata);
+
+ return;
+ }
+
+ re_class = v;
+ rspamd_snprintf(path, sizeof(path), "%s%c%s.hs", cbdata->cache_dir,
+ G_DIR_SEPARATOR, re_class->hash);
+
+ if (rspamd_re_cache_is_valid_hyperscan_file(cache, path, TRUE, TRUE, NULL)) {
+
+ fd = open(path, O_RDONLY, 00600);
+
+ /* Read number of regexps */
+ g_assert(fd != -1);
+ g_assert(lseek(fd, RSPAMD_HS_MAGIC_LEN + sizeof(cache->plt), SEEK_SET) != -1);
+ g_assert(read(fd, &n, sizeof(n)) == sizeof(n));
+ close(fd);
+
+ if (re_class->type_len > 0) {
+ if (!cbdata->silent) {
+ msg_info_re_cache(
+ "skip already valid class %s(%*s) to cache %6s, %d regexps",
+ rspamd_re_cache_type_to_string(re_class->type),
+ (gint) re_class->type_len - 1,
+ re_class->type_data,
+ re_class->hash,
+ n);
+ }
+ }
+ else {
+ if (!cbdata->silent) {
+ msg_info_re_cache(
+ "skip already valid class %s to cache %6s, %d regexps",
+ rspamd_re_cache_type_to_string(re_class->type),
+ re_class->hash,
+ n);
+ }
+ }
+
+ ev_timer_again(EV_A_ w);
+ return;
+ }
+
+ rspamd_snprintf(path, sizeof(path), "%s%c%s%P-XXXXXXXXXX", cbdata->cache_dir,
+ G_DIR_SEPARATOR, re_class->hash, our_pid);
+ fd = g_mkstemp_full(path, O_CREAT | O_TRUNC | O_EXCL | O_WRONLY, 00600);
+
+ if (fd == -1) {
+ err = g_error_new(rspamd_re_cache_quark(), errno,
+ "cannot open file %s: %s", path, strerror(errno));
+ rspamd_re_cache_compile_err(EV_A_ w, err, cbdata, false);
+ return;
+ }
+
+ g_hash_table_iter_init(&cit, re_class->re);
+ n = g_hash_table_size(re_class->re);
+ hs_flags = g_new0(guint, n);
+ hs_ids = g_new0(guint, n);
+ hs_pats = g_new0(char *, n);
+ hs_exts = g_new0(const hs_expr_ext_t *, n);
+ i = 0;
+
+ while (g_hash_table_iter_next(&cit, &k, &v)) {
+ re = v;
+
+ pcre_flags = rspamd_regexp_get_pcre_flags(re);
+ re_flags = rspamd_regexp_get_flags(re);
+
+ if (re_flags & RSPAMD_REGEXP_FLAG_PCRE_ONLY) {
+ /* Do not try to compile bad regexp */
+ msg_info_re_cache(
+ "do not try compile %s to hyperscan as it is PCRE only",
+ rspamd_regexp_get_pattern(re));
+ continue;
+ }
+
+ hs_flags[i] = 0;
+ hs_exts[i] = NULL;
+#ifndef WITH_PCRE2
+ if (pcre_flags & PCRE_FLAG(UTF8)) {
+ hs_flags[i] |= HS_FLAG_UTF8;
+ }
+#else
+ if (pcre_flags & PCRE_FLAG(UTF)) {
+ hs_flags[i] |= HS_FLAG_UTF8;
+ }
+#endif
+ if (pcre_flags & PCRE_FLAG(CASELESS)) {
+ hs_flags[i] |= HS_FLAG_CASELESS;
+ }
+ if (pcre_flags & PCRE_FLAG(MULTILINE)) {
+ hs_flags[i] |= HS_FLAG_MULTILINE;
+ }
+ if (pcre_flags & PCRE_FLAG(DOTALL)) {
+ hs_flags[i] |= HS_FLAG_DOTALL;
+ }
+
+
+ if (re_flags & RSPAMD_REGEXP_FLAG_LEFTMOST) {
+ hs_flags[i] |= HS_FLAG_SOM_LEFTMOST;
+ }
+ else if (rspamd_regexp_get_maxhits(re) == 1) {
+ hs_flags[i] |= HS_FLAG_SINGLEMATCH;
+ }
+
+ gchar *pat = rspamd_re_cache_hs_pattern_from_pcre(re);
+
+ if (hs_compile(pat,
+ hs_flags[i],
+ HS_MODE_BLOCK,
+ &cache->plt,
+ &test_db,
+ &hs_errors) != HS_SUCCESS) {
+ msg_info_re_cache("cannot compile '%s' to hyperscan: '%s', try prefilter match",
+ pat,
+ hs_errors != NULL ? hs_errors->message : "unknown error");
+ hs_free_compile_error(hs_errors);
+
+ /* The approximation operation might take a significant
+ * amount of time, so we need to check if it's finite
+ */
+ if (rspamd_re_cache_is_finite(cache, re, hs_flags[i], cbdata->max_time)) {
+ hs_flags[i] |= HS_FLAG_PREFILTER;
+ hs_ids[i] = rspamd_regexp_get_cache_id(re);
+ hs_pats[i] = pat;
+ i++;
+ }
+ else {
+ g_free(pat); /* Avoid leak */
+ }
+ }
+ else {
+ hs_ids[i] = rspamd_regexp_get_cache_id(re);
+ hs_pats[i] = pat;
+ i++;
+ hs_free_database(test_db);
+ }
+ }
+ /* Adjust real re number */
+ n = i;
+
+#define CLEANUP_ALLOCATED(is_err) \
+ do { \
+ g_free(hs_flags); \
+ g_free(hs_ids); \
+ for (guint j = 0; j < i; j++) { \
+ g_free(hs_pats[j]); \
+ } \
+ g_free(hs_pats); \
+ g_free(hs_exts); \
+ if (is_err) { \
+ close(fd); \
+ unlink(path); \
+ if (hs_errors) hs_free_compile_error(hs_errors); \
+ } \
+ } while (0)
+
+ if (n > 0) {
+ /* Create the hs tree */
+ hs_errors = NULL;
+ if (hs_compile_ext_multi((const char **) hs_pats,
+ hs_flags,
+ hs_ids,
+ hs_exts,
+ n,
+ HS_MODE_BLOCK,
+ &cache->plt,
+ &test_db,
+ &hs_errors) != HS_SUCCESS) {
+
+ err = g_error_new(rspamd_re_cache_quark(), EINVAL,
+ "cannot create tree of regexp when processing '%s': %s",
+ hs_pats[hs_errors->expression], hs_errors->message);
+ CLEANUP_ALLOCATED(true);
+ rspamd_re_cache_compile_err(EV_A_ w, err, cbdata, false);
+
+ return;
+ }
+
+ if (hs_serialize_database(test_db, &hs_serialized,
+ &serialized_len) != HS_SUCCESS) {
+ err = g_error_new(rspamd_re_cache_quark(),
+ errno,
+ "cannot serialize tree of regexp for %s",
+ re_class->hash);
+
+ CLEANUP_ALLOCATED(true);
+ hs_free_database(test_db);
+ rspamd_re_cache_compile_err(EV_A_ w, err, cbdata, false);
+ return;
+ }
+
+ hs_free_database(test_db);
+
+ /*
+ * Magic - 8 bytes
+ * Platform - sizeof (platform)
+ * n - number of regexps
+ * n * <regexp ids>
+ * n * <regexp flags>
+ * crc - 8 bytes checksum
+ * <hyperscan blob>
+ */
+ rspamd_cryptobox_fast_hash_init(&crc_st, 0xdeadbabe);
+ /* IDs -> Flags -> Hs blob */
+ rspamd_cryptobox_fast_hash_update(&crc_st,
+ hs_ids, sizeof(*hs_ids) * n);
+ rspamd_cryptobox_fast_hash_update(&crc_st,
+ hs_flags, sizeof(*hs_flags) * n);
+ rspamd_cryptobox_fast_hash_update(&crc_st,
+ hs_serialized, serialized_len);
+ crc = rspamd_cryptobox_fast_hash_final(&crc_st);
+
+
+ iov[0].iov_base = (void *) rspamd_hs_magic;
+ iov[0].iov_len = RSPAMD_HS_MAGIC_LEN;
+ iov[1].iov_base = &cache->plt;
+ iov[1].iov_len = sizeof(cache->plt);
+ iov[2].iov_base = &n;
+ iov[2].iov_len = sizeof(n);
+ iov[3].iov_base = hs_ids;
+ iov[3].iov_len = sizeof(*hs_ids) * n;
+ iov[4].iov_base = hs_flags;
+ iov[4].iov_len = sizeof(*hs_flags) * n;
+ iov[5].iov_base = &crc;
+ iov[5].iov_len = sizeof(crc);
+ iov[6].iov_base = hs_serialized;
+ iov[6].iov_len = serialized_len;
+
+ if (writev(fd, iov, G_N_ELEMENTS(iov)) == -1) {
+ err = g_error_new(rspamd_re_cache_quark(),
+ errno,
+ "cannot serialize tree of regexp to %s: %s",
+ path, strerror(errno));
+
+ CLEANUP_ALLOCATED(true);
+ g_free(hs_serialized);
+
+ rspamd_re_cache_compile_err(EV_A_ w, err, cbdata, false);
+ return;
+ }
+
+ if (re_class->type_len > 0) {
+ msg_info_re_cache(
+ "compiled class %s(%*s) to cache %6s, %d/%d regexps",
+ rspamd_re_cache_type_to_string(re_class->type),
+ (gint) re_class->type_len - 1,
+ re_class->type_data,
+ re_class->hash,
+ n,
+ (gint) g_hash_table_size(re_class->re));
+ }
+ else {
+ msg_info_re_cache(
+ "compiled class %s to cache %6s, %d/%d regexps",
+ rspamd_re_cache_type_to_string(re_class->type),
+ re_class->hash,
+ n,
+ (gint) g_hash_table_size(re_class->re));
+ }
+
+ cbdata->total += n;
+ CLEANUP_ALLOCATED(false);
+
+ /* Now rename temporary file to the new .hs file */
+ rspamd_snprintf(npath, sizeof(npath), "%s%c%s.hs", cbdata->cache_dir,
+ G_DIR_SEPARATOR, re_class->hash);
+
+ if (rename(path, npath) == -1) {
+ err = g_error_new(rspamd_re_cache_quark(),
+ errno,
+ "cannot rename %s to %s: %s",
+ path, npath, strerror(errno));
+ unlink(path);
+ close(fd);
+
+ rspamd_re_cache_compile_err(EV_A_ w, err, cbdata, false);
+ return;
+ }
+
+ close(fd);
+ }
+ else {
+ err = g_error_new(rspamd_re_cache_quark(),
+ errno,
+ "no suitable regular expressions %s (%d original): "
+ "remove temporary file %s",
+ rspamd_re_cache_type_to_string(re_class->type),
+ (gint) g_hash_table_size(re_class->re),
+ path);
+
+ CLEANUP_ALLOCATED(true);
+ rspamd_re_cache_compile_err(EV_A_ w, err, cbdata, false);
+
+ return;
+ }
+
+ /* Continue process */
+ ev_timer_again(EV_A_ w);
+}
+
+#endif
+
+gint rspamd_re_cache_compile_hyperscan(struct rspamd_re_cache *cache,
+ const char *cache_dir,
+ gdouble max_time,
+ gboolean silent,
+ struct ev_loop *event_loop,
+ void (*cb)(guint ncompiled, GError *err, void *cbd),
+ void *cbd)
+{
+ g_assert(cache != NULL);
+ g_assert(cache_dir != NULL);
+
+#ifndef WITH_HYPERSCAN
+ return -1;
+#else
+ static ev_timer *timer;
+ static const ev_tstamp timer_interval = 0.1;
+ struct rspamd_re_cache_hs_compile_cbdata *cbdata;
+
+ cbdata = g_malloc0(sizeof(*cbdata));
+ g_hash_table_iter_init(&cbdata->it, cache->re_classes);
+ cbdata->cache = cache;
+ cbdata->cache_dir = cache_dir;
+ cbdata->cb = cb;
+ cbdata->cbd = cbd;
+ cbdata->max_time = max_time;
+ cbdata->silent = silent;
+ cbdata->total = 0;
+ timer = g_malloc0(sizeof(*timer));
+ timer->data = (void *) cbdata; /* static */
+
+ ev_timer_init(timer, rspamd_re_cache_compile_timer_cb,
+ timer_interval, timer_interval);
+ ev_timer_start(event_loop, timer);
+
+ return 0;
+#endif
+}
+
+gboolean
+rspamd_re_cache_is_valid_hyperscan_file(struct rspamd_re_cache *cache,
+ const char *path, gboolean silent, gboolean try_load, GError **err)
+{
+ g_assert(cache != NULL);
+ g_assert(path != NULL);
+
+#ifndef WITH_HYPERSCAN
+ return FALSE;
+#else
+ gint fd, n, ret;
+ guchar magicbuf[RSPAMD_HS_MAGIC_LEN];
+ const guchar *mb;
+ GHashTableIter it;
+ gpointer k, v;
+ struct rspamd_re_class *re_class;
+ gsize len;
+ const gchar *hash_pos;
+ hs_platform_info_t test_plt;
+ hs_database_t *test_db = NULL;
+ guchar *map, *p, *end;
+ rspamd_cryptobox_fast_hash_state_t crc_st;
+ guint64 crc, valid_crc;
+
+ len = strlen(path);
+
+ if (len < sizeof(rspamd_cryptobox_HASHBYTES + 3)) {
+ if (!silent) {
+ msg_err_re_cache("cannot open hyperscan cache file %s: too short filename",
+ path);
+ }
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "too short filename");
+
+ return FALSE;
+ }
+
+ if (memcmp(path + len - 3, ".hs", 3) != 0) {
+ if (!silent) {
+ msg_err_re_cache("cannot open hyperscan cache file %s: not ending with .hs",
+ path);
+ }
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "not ending with .hs");
+ return FALSE;
+ }
+
+ hash_pos = path + len - 3 - (sizeof(re_class->hash) - 1);
+ g_hash_table_iter_init(&it, cache->re_classes);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ re_class = v;
+
+ if (memcmp(hash_pos, re_class->hash, sizeof(re_class->hash) - 1) == 0) {
+ /* Open file and check magic */
+ gssize r;
+
+ fd = open(path, O_RDONLY);
+
+ if (fd == -1) {
+ if (errno != ENOENT || !silent) {
+ msg_err_re_cache("cannot open hyperscan cache file %s: %s",
+ path, strerror(errno));
+ }
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "%s",
+ strerror(errno));
+ return FALSE;
+ }
+
+ if ((r = read(fd, magicbuf, sizeof(magicbuf))) != sizeof(magicbuf)) {
+ if (r == -1) {
+ msg_err_re_cache("cannot read magic from hyperscan "
+ "cache file %s: %s",
+ path, strerror(errno));
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "cannot read magic: %s",
+ strerror(errno));
+ }
+ else {
+ msg_err_re_cache("truncated read magic from hyperscan "
+ "cache file %s: %z, %z wanted",
+ path, r, (gsize) sizeof(magicbuf));
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "truncated read magic %zd, %zd wanted",
+ r, (gsize) sizeof(magicbuf));
+ }
+
+ close(fd);
+ return FALSE;
+ }
+
+ mb = rspamd_hs_magic;
+
+ if (memcmp(magicbuf, mb, sizeof(magicbuf)) != 0) {
+ msg_err_re_cache("cannot open hyperscan cache file %s: "
+ "bad magic ('%*xs', '%*xs' expected)",
+ path, (int) RSPAMD_HS_MAGIC_LEN, magicbuf,
+ (int) RSPAMD_HS_MAGIC_LEN, mb);
+
+ close(fd);
+ g_set_error(err, rspamd_re_cache_quark(), 0, "invalid magic");
+ return FALSE;
+ }
+
+ if ((r = read(fd, &test_plt, sizeof(test_plt))) != sizeof(test_plt)) {
+ if (r == -1) {
+ msg_err_re_cache("cannot read platform data from hyperscan "
+ "cache file %s: %s",
+ path, strerror(errno));
+ }
+ else {
+ msg_err_re_cache("truncated read platform data from hyperscan "
+ "cache file %s: %z, %z wanted",
+ path, r, (gsize) sizeof(magicbuf));
+ }
+
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "cannot read platform data: %s", strerror(errno));
+
+ close(fd);
+ return FALSE;
+ }
+
+ if (test_plt.cpu_features != cache->plt.cpu_features) {
+ msg_err_re_cache("cannot open hyperscan cache file %s: "
+ "compiled for a different platform",
+ path);
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "compiled for a different platform");
+
+ close(fd);
+ return FALSE;
+ }
+
+ close(fd);
+
+ if (try_load) {
+ map = rspamd_file_xmap(path, PROT_READ, &len, TRUE);
+
+ if (map == NULL) {
+ msg_err_re_cache("cannot mmap hyperscan cache file %s: "
+ "%s",
+ path, strerror(errno));
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "mmap error: %s", strerror(errno));
+ return FALSE;
+ }
+
+ p = map + RSPAMD_HS_MAGIC_LEN + sizeof(test_plt);
+ end = map + len;
+ memcpy(&n, p, sizeof(n));
+ p += sizeof(gint);
+
+ if (n <= 0 || 2 * n * sizeof(gint) + /* IDs + flags */
+ sizeof(guint64) + /* crc */
+ RSPAMD_HS_MAGIC_LEN + /* header */
+ sizeof(cache->plt) >
+ len) {
+ /* Some wrong amount of regexps */
+ msg_err_re_cache("bad number of expressions in %s: %d",
+ path, n);
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "bad number of expressions: %d", n);
+ munmap(map, len);
+ return FALSE;
+ }
+
+ /*
+ * Magic - 8 bytes
+ * Platform - sizeof (platform)
+ * n - number of regexps
+ * n * <regexp ids>
+ * n * <regexp flags>
+ * crc - 8 bytes checksum
+ * <hyperscan blob>
+ */
+
+ memcpy(&crc, p + n * 2 * sizeof(gint), sizeof(crc));
+ rspamd_cryptobox_fast_hash_init(&crc_st, 0xdeadbabe);
+ /* IDs */
+ rspamd_cryptobox_fast_hash_update(&crc_st, p, n * sizeof(gint));
+ /* Flags */
+ rspamd_cryptobox_fast_hash_update(&crc_st, p + n * sizeof(gint),
+ n * sizeof(gint));
+ /* HS database */
+ p += n * sizeof(gint) * 2 + sizeof(guint64);
+ rspamd_cryptobox_fast_hash_update(&crc_st, p, end - p);
+ valid_crc = rspamd_cryptobox_fast_hash_final(&crc_st);
+
+ if (crc != valid_crc) {
+ msg_warn_re_cache("outdated or invalid hs database in %s: "
+ "crc read %xL, crc expected %xL",
+ path, crc, valid_crc);
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "outdated or invalid hs database, crc check failure");
+ munmap(map, len);
+
+ return FALSE;
+ }
+
+ if ((ret = hs_deserialize_database(p, end - p, &test_db)) != HS_SUCCESS) {
+ msg_err_re_cache("bad hs database in %s: %d", path, ret);
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "deserialize error: %d", ret);
+ munmap(map, len);
+
+ return FALSE;
+ }
+
+ hs_free_database(test_db);
+ munmap(map, len);
+ }
+ /* XXX: add crc check */
+
+ return TRUE;
+ }
+ }
+
+ if (!silent) {
+ msg_warn_re_cache("unknown hyperscan cache file %s", path);
+ }
+
+ g_set_error(err, rspamd_re_cache_quark(), 0,
+ "unknown hyperscan file");
+
+ return FALSE;
+#endif
+}
+
+
+enum rspamd_hyperscan_status
+rspamd_re_cache_load_hyperscan(struct rspamd_re_cache *cache,
+ const char *cache_dir, bool try_load)
+{
+ g_assert(cache != NULL);
+ g_assert(cache_dir != NULL);
+
+#ifndef WITH_HYPERSCAN
+ return RSPAMD_HYPERSCAN_UNSUPPORTED;
+#else
+ gchar path[PATH_MAX];
+ gint fd, i, n, *hs_ids = NULL, *hs_flags = NULL, total = 0, ret;
+ GHashTableIter it;
+ gpointer k, v;
+ guint8 *map, *p;
+ struct rspamd_re_class *re_class;
+ struct rspamd_re_cache_elt *elt;
+ struct stat st;
+ gboolean has_valid = FALSE, all_valid = FALSE;
+
+ g_hash_table_iter_init(&it, cache->re_classes);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ re_class = v;
+ rspamd_snprintf(path, sizeof(path), "%s%c%s.hs", cache_dir,
+ G_DIR_SEPARATOR, re_class->hash);
+
+ if (rspamd_re_cache_is_valid_hyperscan_file(cache, path, try_load, FALSE, NULL)) {
+ msg_debug_re_cache("load hyperscan database from '%s'",
+ re_class->hash);
+
+ fd = open(path, O_RDONLY);
+
+ /* Read number of regexps */
+ g_assert(fd != -1);
+ fstat(fd, &st);
+
+ map = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
+
+ if (map == MAP_FAILED) {
+ if (!try_load) {
+ msg_err_re_cache("cannot mmap %s: %s", path, strerror(errno));
+ }
+ else {
+ msg_debug_re_cache("cannot mmap %s: %s", path, strerror(errno));
+ }
+
+ close(fd);
+ all_valid = FALSE;
+ continue;
+ }
+
+ close(fd);
+ p = map + RSPAMD_HS_MAGIC_LEN + sizeof(cache->plt);
+ n = *(gint *) p;
+
+ if (n <= 0 || 2 * n * sizeof(gint) + /* IDs + flags */
+ sizeof(guint64) + /* crc */
+ RSPAMD_HS_MAGIC_LEN + /* header */
+ sizeof(cache->plt) >
+ (gsize) st.st_size) {
+ /* Some wrong amount of regexps */
+ if (!try_load) {
+ msg_err_re_cache("bad number of expressions in %s: %d",
+ path, n);
+ }
+ else {
+ msg_debug_re_cache("bad number of expressions in %s: %d",
+ path, n);
+ }
+
+ munmap(map, st.st_size);
+ all_valid = FALSE;
+ continue;
+ }
+
+ total += n;
+ p += sizeof(n);
+ hs_ids = g_malloc(n * sizeof(*hs_ids));
+ memcpy(hs_ids, p, n * sizeof(*hs_ids));
+ p += n * sizeof(*hs_ids);
+ hs_flags = g_malloc(n * sizeof(*hs_flags));
+ memcpy(hs_flags, p, n * sizeof(*hs_flags));
+
+ /* Skip crc */
+ p += n * sizeof(*hs_ids) + sizeof(guint64);
+
+ /* Cleanup */
+ if (re_class->hs_scratch != NULL) {
+ hs_free_scratch(re_class->hs_scratch);
+ }
+
+ if (re_class->hs_db != NULL) {
+ rspamd_hyperscan_free(re_class->hs_db, false);
+ }
+
+ if (re_class->hs_ids) {
+ g_free(re_class->hs_ids);
+ }
+
+ re_class->hs_ids = NULL;
+ re_class->hs_scratch = NULL;
+ re_class->hs_db = NULL;
+ munmap(map, st.st_size);
+
+ re_class->hs_db = rspamd_hyperscan_maybe_load(path, p - map);
+ if (re_class->hs_db == NULL) {
+ if (!try_load) {
+ msg_err_re_cache("bad hs database in %s", path);
+ }
+ else {
+ msg_debug_re_cache("bad hs database in %s", path);
+ }
+ g_free(hs_ids);
+ g_free(hs_flags);
+
+ re_class->hs_ids = NULL;
+ re_class->hs_scratch = NULL;
+ re_class->hs_db = NULL;
+ all_valid = FALSE;
+
+ continue;
+ }
+
+ if ((ret = hs_alloc_scratch(rspamd_hyperscan_get_database(re_class->hs_db),
+ &re_class->hs_scratch)) != HS_SUCCESS) {
+ if (!try_load) {
+ msg_err_re_cache("bad hs database in %s; error code: %d", path, ret);
+ }
+ else {
+ msg_debug_re_cache("bad hs database in %s; error code: %d", path, ret);
+ }
+ g_free(hs_ids);
+ g_free(hs_flags);
+
+ rspamd_hyperscan_free(re_class->hs_db, true);
+ re_class->hs_ids = NULL;
+ re_class->hs_scratch = NULL;
+ re_class->hs_db = NULL;
+ all_valid = FALSE;
+
+ continue;
+ }
+
+ /*
+ * Now find hyperscan elts that are successfully compiled and
+ * specify that they should be matched using hyperscan
+ */
+ for (i = 0; i < n; i++) {
+ g_assert((gint) cache->re->len > hs_ids[i] && hs_ids[i] >= 0);
+ elt = g_ptr_array_index(cache->re, hs_ids[i]);
+
+ if (hs_flags[i] & HS_FLAG_PREFILTER) {
+ elt->match_type = RSPAMD_RE_CACHE_HYPERSCAN_PRE;
+ }
+ else {
+ elt->match_type = RSPAMD_RE_CACHE_HYPERSCAN;
+ }
+ }
+
+ re_class->hs_ids = hs_ids;
+ g_free(hs_flags);
+ re_class->nhs = n;
+
+ if (!has_valid) {
+ has_valid = TRUE;
+ all_valid = TRUE;
+ }
+ }
+ else {
+ if (!try_load) {
+ msg_err_re_cache("invalid hyperscan hash file '%s'",
+ path);
+ }
+ else {
+ msg_debug_re_cache("invalid hyperscan hash file '%s'",
+ path);
+ }
+ all_valid = FALSE;
+ continue;
+ }
+ }
+
+ if (has_valid) {
+ if (all_valid) {
+ msg_info_re_cache("full hyperscan database of %d regexps has been loaded", total);
+ cache->hyperscan_loaded = RSPAMD_HYPERSCAN_LOADED_FULL;
+ }
+ else {
+ msg_info_re_cache("partial hyperscan database of %d regexps has been loaded", total);
+ cache->hyperscan_loaded = RSPAMD_HYPERSCAN_LOADED_PARTIAL;
+ }
+ }
+ else {
+ msg_info_re_cache("hyperscan database has NOT been loaded; no valid expressions");
+ cache->hyperscan_loaded = RSPAMD_HYPERSCAN_LOAD_ERROR;
+ }
+
+
+ return cache->hyperscan_loaded;
+#endif
+}
+
+void rspamd_re_cache_add_selector(struct rspamd_re_cache *cache,
+ const gchar *sname,
+ gint ref)
+{
+ khiter_t k;
+
+ k = kh_get(lua_selectors_hash, cache->selectors, (gchar *) sname);
+
+ if (k == kh_end(cache->selectors)) {
+ gchar *cpy = g_strdup(sname);
+ gint res;
+
+ k = kh_put(lua_selectors_hash, cache->selectors, cpy, &res);
+
+ kh_value(cache->selectors, k) = ref;
+ }
+ else {
+ msg_warn_re_cache("replacing selector with name %s", sname);
+
+ if (cache->L) {
+ luaL_unref(cache->L, LUA_REGISTRYINDEX, kh_value(cache->selectors, k));
+ }
+
+ kh_value(cache->selectors, k) = ref;
+ }
+}
diff --git a/src/libserver/re_cache.h b/src/libserver/re_cache.h
new file mode 100644
index 0000000..d6449a9
--- /dev/null
+++ b/src/libserver/re_cache.h
@@ -0,0 +1,212 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_RE_CACHE_H
+#define RSPAMD_RE_CACHE_H
+
+#include "config.h"
+#include "libutil/regexp.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_re_cache;
+struct rspamd_re_runtime;
+struct rspamd_task;
+struct rspamd_config;
+
+enum rspamd_re_type {
+ RSPAMD_RE_HEADER,
+ RSPAMD_RE_RAWHEADER,
+ RSPAMD_RE_ALLHEADER,
+ RSPAMD_RE_MIMEHEADER,
+ RSPAMD_RE_MIME,
+ RSPAMD_RE_RAWMIME,
+ RSPAMD_RE_URL,
+ RSPAMD_RE_EMAIL,
+ RSPAMD_RE_BODY, /* full in SA */
+ RSPAMD_RE_SABODY, /* body in SA */
+ RSPAMD_RE_SARAWBODY, /* rawbody in SA */
+ RSPAMD_RE_WORDS, /* normalized words */
+ RSPAMD_RE_RAWWORDS, /* raw words */
+ RSPAMD_RE_STEMWORDS, /* stemmed words */
+ RSPAMD_RE_SELECTOR, /* use lua selector to process regexp */
+ RSPAMD_RE_MAX
+};
+
+struct rspamd_re_cache_stat {
+ guint64 bytes_scanned;
+ guint64 bytes_scanned_pcre;
+ guint regexp_checked;
+ guint regexp_matched;
+ guint regexp_total;
+ guint regexp_fast_cached;
+};
+
+/**
+ * Initialize re_cache persistent structure
+ */
+struct rspamd_re_cache *rspamd_re_cache_new(void);
+
+/**
+ * Add the existing regexp to the cache
+ * @param cache cache object
+ * @param re regexp object
+ * @param type type of object
+ * @param type_data associated data with the type (e.g. header name)
+ * @param datalen associated data length
+ * @param lua_cbref optional lua callback reference for matching purposes
+ */
+rspamd_regexp_t *
+rspamd_re_cache_add(struct rspamd_re_cache *cache, rspamd_regexp_t *re,
+ enum rspamd_re_type type,
+ gconstpointer type_data, gsize datalen,
+ gint lua_cbref);
+
+/**
+ * Replace regexp in the cache with another regexp
+ * @param cache cache object
+ * @param what re to replace
+ * @param with regexp object to replace the origin
+ */
+void rspamd_re_cache_replace(struct rspamd_re_cache *cache,
+ rspamd_regexp_t *what,
+ rspamd_regexp_t *with);
+
+/**
+ * Initialize and optimize re cache structure
+ */
+void rspamd_re_cache_init(struct rspamd_re_cache *cache,
+ struct rspamd_config *cfg);
+
+enum rspamd_hyperscan_status {
+ RSPAMD_HYPERSCAN_UNKNOWN = 0,
+ RSPAMD_HYPERSCAN_UNSUPPORTED,
+ RSPAMD_HYPERSCAN_LOADED_PARTIAL,
+ RSPAMD_HYPERSCAN_LOADED_FULL,
+ RSPAMD_HYPERSCAN_LOAD_ERROR,
+};
+
+/**
+ * Returns true when hyperscan is loaded
+ * @param cache
+ * @return
+ */
+enum rspamd_hyperscan_status rspamd_re_cache_is_hs_loaded(struct rspamd_re_cache *cache);
+
+/**
+ * Get runtime data for a cache
+ */
+struct rspamd_re_runtime *rspamd_re_cache_runtime_new(struct rspamd_re_cache *cache);
+
+/**
+ * Get runtime statistics
+ */
+const struct rspamd_re_cache_stat *
+rspamd_re_cache_get_stat(struct rspamd_re_runtime *rt);
+
+/**
+ * Process regexp runtime and return the result for a specific regexp
+ * @param task task object
+ * @param rt cache runtime object
+ * @param re regexp object
+ * @param type type of object
+ * @param type_data associated data with the type (e.g. header name)
+ * @param datalen associated data length
+ * @param is_strong use case sensitive match when looking for headers
+ */
+gint rspamd_re_cache_process(struct rspamd_task *task,
+ rspamd_regexp_t *re,
+ enum rspamd_re_type type,
+ gconstpointer type_data,
+ gsize datalen,
+ gboolean is_strong);
+
+int rspamd_re_cache_process_ffi(void *ptask,
+ void *pre,
+ int type,
+ void *type_data,
+ int is_strong);
+
+/**
+ * Destroy runtime data
+ */
+void rspamd_re_cache_runtime_destroy(struct rspamd_re_runtime *rt);
+
+/**
+ * Unref re cache
+ */
+void rspamd_re_cache_unref(struct rspamd_re_cache *cache);
+
+/**
+ * Retain reference to re cache
+ */
+struct rspamd_re_cache *rspamd_re_cache_ref(struct rspamd_re_cache *cache);
+
+/**
+ * Set limit for all regular expressions in the cache, returns previous limit
+ */
+guint rspamd_re_cache_set_limit(struct rspamd_re_cache *cache, guint limit);
+
+/**
+ * Convert re type to a human readable string (constant one)
+ */
+const gchar *rspamd_re_cache_type_to_string(enum rspamd_re_type type);
+
+/**
+ * Convert re type string to the type enum
+ */
+enum rspamd_re_type rspamd_re_cache_type_from_string(const char *str);
+
+struct ev_loop;
+/**
+ * Compile expressions to the hyperscan tree and store in the `cache_dir`
+ */
+gint rspamd_re_cache_compile_hyperscan(struct rspamd_re_cache *cache,
+ const char *cache_dir,
+ gdouble max_time,
+ gboolean silent,
+ struct ev_loop *event_loop,
+ void (*cb)(guint ncompiled, GError *err, void *cbd),
+ void *cbd);
+
+/**
+ * Returns TRUE if the specified file is valid hyperscan cache
+ */
+gboolean rspamd_re_cache_is_valid_hyperscan_file(struct rspamd_re_cache *cache,
+ const char *path,
+ gboolean silent,
+ gboolean try_load,
+ GError **err);
+
+/**
+ * Loads all hyperscan regexps precompiled
+ */
+enum rspamd_hyperscan_status rspamd_re_cache_load_hyperscan(
+ struct rspamd_re_cache *cache,
+ const char *cache_dir, bool try_load);
+
+/**
+ * Registers lua selector in the cache
+ */
+void rspamd_re_cache_add_selector(struct rspamd_re_cache *cache,
+ const gchar *sname, gint ref);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/redis_pool.cxx b/src/libserver/redis_pool.cxx
new file mode 100644
index 0000000..9c2d6cf
--- /dev/null
+++ b/src/libserver/redis_pool.cxx
@@ -0,0 +1,663 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "contrib/libev/ev.h"
+#include "redis_pool.h"
+#include "cfg_file.h"
+#include "contrib/hiredis/hiredis.h"
+#include "contrib/hiredis/async.h"
+#include "contrib/hiredis/adapters/libev.h"
+#include "cryptobox.h"
+#include "logger.h"
+#include "contrib/ankerl/unordered_dense.h"
+
+#include <list>
+#include <unordered_map>
+
+namespace rspamd {
+class redis_pool_elt;
+class redis_pool;
+
+#define msg_debug_rpool(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_redis_pool_log_id, "redis_pool", conn->tag, \
+ __FUNCTION__, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(redis_pool)
+
+enum class rspamd_redis_pool_connection_state : std::uint8_t {
+ RSPAMD_REDIS_POOL_CONN_INACTIVE = 0,
+ RSPAMD_REDIS_POOL_CONN_ACTIVE,
+ RSPAMD_REDIS_POOL_CONN_FINALISING
+};
+
+struct redis_pool_connection {
+ using redis_pool_connection_ptr = std::unique_ptr<redis_pool_connection>;
+ using conn_iter_t = std::list<redis_pool_connection_ptr>::iterator;
+ struct redisAsyncContext *ctx;
+ redis_pool_elt *elt;
+ redis_pool *pool;
+ conn_iter_t elt_pos;
+ ev_timer timeout;
+ gchar tag[MEMPOOL_UID_LEN];
+ rspamd_redis_pool_connection_state state;
+
+ auto schedule_timeout() -> void;
+ ~redis_pool_connection();
+
+ explicit redis_pool_connection(redis_pool *_pool,
+ redis_pool_elt *_elt,
+ const std::string &db,
+ const std::string &username,
+ const std::string &password,
+ struct redisAsyncContext *_ctx);
+
+private:
+ static auto redis_conn_timeout_cb(EV_P_ ev_timer *w, int revents) -> void;
+ static auto redis_quit_cb(redisAsyncContext *c, void *r, void *priv) -> void;
+ static auto redis_on_disconnect(const struct redisAsyncContext *ac, int status) -> auto;
+};
+
+
+using redis_pool_key_t = std::uint64_t;
+class redis_pool;
+
+class redis_pool_elt {
+ using redis_pool_connection_ptr = std::unique_ptr<redis_pool_connection>;
+ redis_pool *pool;
+ /*
+ * These lists owns connections, so if an element is removed from both
+ * lists, it is destructed
+ */
+ std::list<redis_pool_connection_ptr> active;
+ std::list<redis_pool_connection_ptr> inactive;
+ std::list<redis_pool_connection_ptr> terminating;
+ std::string ip;
+ std::string db;
+ std::string username;
+ std::string password;
+ int port;
+ redis_pool_key_t key;
+ bool is_unix;
+
+public:
+ /* Disable copy */
+ redis_pool_elt() = delete;
+ redis_pool_elt(const redis_pool_elt &) = delete;
+ /* Enable move */
+ redis_pool_elt(redis_pool_elt &&other) = default;
+
+ explicit redis_pool_elt(redis_pool *_pool,
+ const gchar *_db, const gchar *_username,
+ const gchar *_password,
+ const char *_ip, int _port)
+ : pool(_pool), ip(_ip), port(_port),
+ key(redis_pool_elt::make_key(_db, _username, _password, _ip, _port))
+ {
+ is_unix = ip[0] == '.' || ip[0] == '/';
+
+ if (_db) {
+ db = _db;
+ }
+ if (_username) {
+ username = _username;
+ }
+ if (_password) {
+ password = _password;
+ }
+ }
+
+ auto new_connection() -> redisAsyncContext *;
+
+ auto release_connection(const redis_pool_connection *conn) -> void
+ {
+ switch (conn->state) {
+ case rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_ACTIVE:
+ active.erase(conn->elt_pos);
+ break;
+ case rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_INACTIVE:
+ inactive.erase(conn->elt_pos);
+ break;
+ case rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_FINALISING:
+ terminating.erase(conn->elt_pos);
+ break;
+ }
+ }
+
+ auto move_to_inactive(redis_pool_connection *conn) -> void
+ {
+ inactive.splice(std::end(inactive), active, conn->elt_pos);
+ conn->elt_pos = std::prev(std::end(inactive));
+ }
+
+ auto move_to_terminating(redis_pool_connection *conn) -> void
+ {
+ terminating.splice(std::end(terminating), inactive, conn->elt_pos);
+ conn->elt_pos = std::prev(std::end(terminating));
+ }
+
+ inline static auto make_key(const gchar *db, const gchar *username,
+ const gchar *password, const char *ip, int port) -> redis_pool_key_t
+ {
+ rspamd_cryptobox_fast_hash_state_t st;
+
+ rspamd_cryptobox_fast_hash_init(&st, rspamd_hash_seed());
+
+ if (db) {
+ rspamd_cryptobox_fast_hash_update(&st, db, strlen(db));
+ }
+ if (username) {
+ rspamd_cryptobox_fast_hash_update(&st, username, strlen(username));
+ }
+ if (password) {
+ rspamd_cryptobox_fast_hash_update(&st, password, strlen(password));
+ }
+
+ rspamd_cryptobox_fast_hash_update(&st, ip, strlen(ip));
+ rspamd_cryptobox_fast_hash_update(&st, &port, sizeof(port));
+
+ return rspamd_cryptobox_fast_hash_final(&st);
+ }
+
+ auto num_active() const -> auto
+ {
+ return active.size();
+ }
+
+ ~redis_pool_elt()
+ {
+ rspamd_explicit_memzero(password.data(), password.size());
+ }
+
+private:
+ auto redis_async_new() -> redisAsyncContext *
+ {
+ struct redisAsyncContext *ctx;
+
+ if (is_unix) {
+ ctx = redisAsyncConnectUnix(ip.c_str());
+ }
+ else {
+ ctx = redisAsyncConnect(ip.c_str(), port);
+ }
+
+ if (ctx && ctx->err != REDIS_OK) {
+ msg_err("cannot connect to redis %s (port %d): %s", ip.c_str(), port,
+ ctx->errstr);
+ redisAsyncFree(ctx);
+
+ return nullptr;
+ }
+
+ return ctx;
+ }
+};
+
+class redis_pool final {
+ static constexpr const double default_timeout = 10.0;
+ static constexpr const unsigned default_max_conns = 100;
+
+ /* We want to have references integrity */
+ ankerl::unordered_dense::map<redisAsyncContext *,
+ redis_pool_connection *>
+ conns_by_ctx;
+ /*
+ * We store a pointer to the element in each connection, so this has to be
+ * a buckets map with pointers/references stability guarantees.
+ */
+ std::unordered_map<redis_pool_key_t, redis_pool_elt> elts_by_key;
+ bool wanna_die = false; /* Hiredis is 'clever' so we can call ourselves from destructor */
+public:
+ double timeout = default_timeout;
+ unsigned max_conns = default_max_conns;
+ struct ev_loop *event_loop;
+ struct rspamd_config *cfg;
+
+public:
+ explicit redis_pool()
+ : event_loop(nullptr), cfg(nullptr)
+ {
+ conns_by_ctx.reserve(max_conns);
+ }
+
+ /* Legacy stuff */
+ auto do_config(struct ev_loop *_loop, struct rspamd_config *_cfg) -> void
+ {
+ event_loop = _loop;
+ cfg = _cfg;
+ }
+
+ auto new_connection(const gchar *db, const gchar *username,
+ const gchar *password, const char *ip, int port) -> redisAsyncContext *;
+
+ auto release_connection(redisAsyncContext *ctx,
+ enum rspamd_redis_pool_release_type how) -> void;
+
+ auto unregister_context(redisAsyncContext *ctx) -> void
+ {
+ conns_by_ctx.erase(ctx);
+ }
+
+ auto register_context(redisAsyncContext *ctx, redis_pool_connection *conn)
+ {
+ conns_by_ctx.emplace(ctx, conn);
+ }
+
+ /* Hack to prevent Redis callbacks to be executed */
+ auto prepare_to_die() -> void
+ {
+ wanna_die = true;
+ }
+
+ ~redis_pool()
+ {
+ }
+};
+
+
+redis_pool_connection::~redis_pool_connection()
+{
+ const auto *conn = this; /* For debug */
+
+ if (state == rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_ACTIVE) {
+ msg_debug_rpool("active connection destructed: %p", ctx);
+
+ if (ctx) {
+ pool->unregister_context(ctx);
+
+ if (!(ctx->c.flags & REDIS_FREEING)) {
+ auto *ac = ctx;
+ ctx = nullptr;
+ ac->onDisconnect = nullptr;
+ redisAsyncFree(ac);
+ }
+ }
+ }
+ else {
+ msg_debug_rpool("inactive connection destructed: %p", ctx);
+
+ ev_timer_stop(pool->event_loop, &timeout);
+ if (ctx) {
+ pool->unregister_context(ctx);
+
+ if (!(ctx->c.flags & REDIS_FREEING)) {
+ auto *ac = ctx;
+ /* To prevent on_disconnect here */
+ ctx = nullptr;
+ ac->onDisconnect = nullptr;
+ redisAsyncFree(ac);
+ }
+ }
+ }
+}
+
+auto redis_pool_connection::redis_quit_cb(redisAsyncContext *c, void *r, void *priv) -> void
+{
+ struct redis_pool_connection *conn =
+ (struct redis_pool_connection *) priv;
+
+ msg_debug_rpool("quit command reply for the connection %p",
+ conn->ctx);
+ /*
+ * The connection will be freed by hiredis itself as we are here merely after
+ * quit command has succeeded and we have timer being set already.
+ * The problem is that when this callback is called, our connection is likely
+ * dead, so probably even on_disconnect callback has been already called...
+ *
+ * Hence, the connection might already be freed, so even (conn) pointer may be
+ * inaccessible.
+ *
+ * TODO: Use refcounts to prevent this stuff to happen, the problem is how
+ * to handle Redis timeout on `quit` command in fact... The good thing is that
+ * it will not likely happen.
+ */
+}
+
+/*
+ * Called for inactive connections that due to be removed
+ */
+auto redis_pool_connection::redis_conn_timeout_cb(EV_P_ ev_timer *w, int revents) -> void
+{
+ auto *conn = (struct redis_pool_connection *) w->data;
+
+ g_assert(conn->state != rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_ACTIVE);
+
+ if (conn->state == rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_INACTIVE) {
+ msg_debug_rpool("scheduled soft removal of connection %p",
+ conn->ctx);
+ conn->state = rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_FINALISING;
+ ev_timer_again(EV_A_ w);
+ redisAsyncCommand(conn->ctx, redis_pool_connection::redis_quit_cb, conn, "QUIT");
+ conn->elt->move_to_terminating(conn);
+ }
+ else {
+ /* Finalising by timeout */
+ ev_timer_stop(EV_A_ w);
+ msg_debug_rpool("final removal of connection %p, refcount: %d",
+ conn->ctx);
+
+ /* Erasure of shared pointer will cause it to be removed */
+ conn->elt->release_connection(conn);
+ }
+}
+
+auto redis_pool_connection::redis_on_disconnect(const struct redisAsyncContext *ac, int status) -> auto
+{
+ auto *conn = (struct redis_pool_connection *) ac->data;
+
+ /*
+ * Here, we know that redis itself will free this connection
+ * so, we need to do something very clever about it
+ */
+ if (conn->state != rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_ACTIVE) {
+ /* Do nothing for active connections as it is already handled somewhere */
+ if (conn->ctx) {
+ msg_debug_rpool("inactive connection terminated: %s",
+ conn->ctx->errstr);
+ }
+
+ /* Erasure of shared pointer will cause it to be removed */
+ conn->elt->release_connection(conn);
+ }
+}
+
+auto redis_pool_connection::schedule_timeout() -> void
+{
+ const auto *conn = this; /* For debug */
+ double real_timeout;
+ auto active_elts = elt->num_active();
+
+ if (active_elts > pool->max_conns) {
+ real_timeout = pool->timeout / 2.0;
+ real_timeout = rspamd_time_jitter(real_timeout, real_timeout / 4.0);
+ }
+ else {
+ real_timeout = pool->timeout;
+ real_timeout = rspamd_time_jitter(real_timeout, real_timeout / 2.0);
+ }
+
+ msg_debug_rpool("scheduled connection %p cleanup in %.1f seconds",
+ ctx, real_timeout);
+
+ timeout.data = this;
+ /* Restore in case if these fields have been modified externally */
+ ctx->data = this;
+ redisAsyncSetDisconnectCallback(ctx, redis_pool_connection::redis_on_disconnect);
+ ev_timer_init(&timeout,
+ redis_pool_connection::redis_conn_timeout_cb,
+ real_timeout, real_timeout / 2.0);
+ ev_timer_start(pool->event_loop, &timeout);
+}
+
+
+redis_pool_connection::redis_pool_connection(redis_pool *_pool,
+ redis_pool_elt *_elt,
+ const std::string &db,
+ const std::string &username,
+ const std::string &password,
+ struct redisAsyncContext *_ctx)
+ : ctx(_ctx), elt(_elt), pool(_pool)
+{
+
+ state = rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_ACTIVE;
+
+ pool->register_context(ctx, this);
+ ctx->data = this;
+ memset(tag, 0, sizeof(tag));
+ rspamd_random_hex(tag, sizeof(tag) - 1);
+
+ redisLibevAttach(pool->event_loop, ctx);
+ redisAsyncSetDisconnectCallback(ctx, redis_pool_connection::redis_on_disconnect);
+
+ if (!username.empty()) {
+ if (!password.empty()) {
+ redisAsyncCommand(ctx, nullptr, nullptr,
+ "AUTH %s %s", username.c_str(), password.c_str());
+ }
+ else {
+ msg_warn("Redis requires a password when username is supplied");
+ }
+ }
+ else if (!password.empty()) {
+ redisAsyncCommand(ctx, nullptr, nullptr,
+ "AUTH %s", password.c_str());
+ }
+ if (!db.empty()) {
+ redisAsyncCommand(ctx, nullptr, nullptr,
+ "SELECT %s", db.c_str());
+ }
+}
+
+auto redis_pool_elt::new_connection() -> redisAsyncContext *
+{
+ if (!inactive.empty()) {
+ decltype(inactive)::value_type conn;
+ conn.swap(inactive.back());
+ inactive.pop_back();
+
+ g_assert(conn->state != rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_ACTIVE);
+ if (conn->ctx->err == REDIS_OK) {
+ /* Also check SO_ERROR */
+ gint err;
+ socklen_t len = sizeof(gint);
+
+ if (getsockopt(conn->ctx->c.fd, SOL_SOCKET, SO_ERROR,
+ (void *) &err, &len) == -1) {
+ err = errno;
+ }
+
+ if (err != 0) {
+ /*
+ * We cannot reuse connection, so we just recursively call
+ * this function one more time
+ */
+ return new_connection();
+ }
+ else {
+ /* Reuse connection */
+ ev_timer_stop(pool->event_loop, &conn->timeout);
+ conn->state = rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_ACTIVE;
+ msg_debug_rpool("reused existing connection to %s:%d: %p",
+ ip.c_str(), port, conn->ctx);
+ active.emplace_front(std::move(conn));
+ active.front()->elt_pos = active.begin();
+
+ return active.front()->ctx;
+ }
+ }
+ else {
+ auto *nctx = redis_async_new();
+ if (nctx) {
+ active.emplace_front(std::make_unique<redis_pool_connection>(pool, this,
+ db.c_str(), username.c_str(), password.c_str(), nctx));
+ active.front()->elt_pos = active.begin();
+ }
+
+ return nctx;
+ }
+ }
+ else {
+ auto *nctx = redis_async_new();
+ if (nctx) {
+ active.emplace_front(std::make_unique<redis_pool_connection>(pool, this,
+ db.c_str(), username.c_str(), password.c_str(), nctx));
+ active.front()->elt_pos = active.begin();
+ }
+
+ return nctx;
+ }
+
+ RSPAMD_UNREACHABLE;
+}
+
+auto redis_pool::new_connection(const gchar *db, const gchar *username,
+ const gchar *password, const char *ip, int port) -> redisAsyncContext *
+{
+
+ if (!wanna_die) {
+ auto key = redis_pool_elt::make_key(db, username, password, ip, port);
+ auto found_elt = elts_by_key.find(key);
+
+ if (found_elt != elts_by_key.end()) {
+ auto &elt = found_elt->second;
+
+ return elt.new_connection();
+ }
+ else {
+ /* Need to create a pool */
+ auto nelt = elts_by_key.try_emplace(key,
+ this, db, username, password, ip, port);
+
+ return nelt.first->second.new_connection();
+ }
+ }
+
+ return nullptr;
+}
+
+auto redis_pool::release_connection(redisAsyncContext *ctx,
+ enum rspamd_redis_pool_release_type how) -> void
+{
+ if (!wanna_die) {
+ auto conn_it = conns_by_ctx.find(ctx);
+ if (conn_it != conns_by_ctx.end()) {
+ auto *conn = conn_it->second;
+ g_assert(conn->state == rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_ACTIVE);
+
+ if (ctx->err != REDIS_OK) {
+ /* We need to terminate connection forcefully */
+ msg_debug_rpool("closed connection %p due to an error", conn->ctx);
+ }
+ else {
+ if (how == RSPAMD_REDIS_RELEASE_DEFAULT) {
+ /* Ensure that there are no callbacks attached to this conn */
+ if (ctx->replies.head == nullptr && (ctx->c.flags & REDIS_CONNECTED)) {
+ /* Just move it to the inactive queue */
+ conn->state = rspamd_redis_pool_connection_state::RSPAMD_REDIS_POOL_CONN_INACTIVE;
+ conn->elt->move_to_inactive(conn);
+ conn->schedule_timeout();
+ msg_debug_rpool("mark connection %p inactive", conn->ctx);
+
+ return;
+ }
+ else {
+ msg_debug_rpool("closed connection %p due to callbacks left",
+ conn->ctx);
+ }
+ }
+ else {
+ if (how == RSPAMD_REDIS_RELEASE_FATAL) {
+ msg_debug_rpool("closed connection %p due to an fatal termination",
+ conn->ctx);
+ }
+ else {
+ msg_debug_rpool("closed connection %p due to explicit termination",
+ conn->ctx);
+ }
+ }
+ }
+
+ conn->elt->release_connection(conn);
+ }
+ else {
+ msg_err("fatal internal error, connection with ctx %p is not found in the Redis pool",
+ ctx);
+ RSPAMD_UNREACHABLE;
+ }
+ }
+}
+
+}// namespace rspamd
+
+void *
+rspamd_redis_pool_init(void)
+{
+ return new rspamd::redis_pool{};
+}
+
+void rspamd_redis_pool_config(void *p,
+ struct rspamd_config *cfg,
+ struct ev_loop *ev_base)
+{
+ g_assert(p != NULL);
+ auto *pool = reinterpret_cast<class rspamd::redis_pool *>(p);
+
+ pool->do_config(ev_base, cfg);
+}
+
+
+struct redisAsyncContext *
+rspamd_redis_pool_connect(void *p,
+ const gchar *db, const gchar *username,
+ const gchar *password, const char *ip, int port)
+{
+ g_assert(p != NULL);
+ auto *pool = reinterpret_cast<class rspamd::redis_pool *>(p);
+
+ return pool->new_connection(db, username, password, ip, port);
+}
+
+
+void rspamd_redis_pool_release_connection(void *p,
+ struct redisAsyncContext *ctx, enum rspamd_redis_pool_release_type how)
+{
+ g_assert(p != NULL);
+ g_assert(ctx != NULL);
+ auto *pool = reinterpret_cast<class rspamd::redis_pool *>(p);
+
+ pool->release_connection(ctx, how);
+}
+
+
+void rspamd_redis_pool_destroy(void *p)
+{
+ auto *pool = reinterpret_cast<class rspamd::redis_pool *>(p);
+
+ pool->prepare_to_die();
+ delete pool;
+}
+
+const gchar *
+rspamd_redis_type_to_string(int type)
+{
+ const gchar *ret = "unknown";
+
+ switch (type) {
+ case REDIS_REPLY_STRING:
+ ret = "string";
+ break;
+ case REDIS_REPLY_ARRAY:
+ ret = "array";
+ break;
+ case REDIS_REPLY_INTEGER:
+ ret = "int";
+ break;
+ case REDIS_REPLY_STATUS:
+ ret = "status";
+ break;
+ case REDIS_REPLY_NIL:
+ ret = "nil";
+ break;
+ case REDIS_REPLY_ERROR:
+ ret = "error";
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
diff --git a/src/libserver/redis_pool.h b/src/libserver/redis_pool.h
new file mode 100644
index 0000000..ecdaa0f
--- /dev/null
+++ b/src/libserver/redis_pool.h
@@ -0,0 +1,91 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBSERVER_REDIS_POOL_H_
+#define SRC_LIBSERVER_REDIS_POOL_H_
+
+#include "config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+struct rspamd_config;
+struct redisAsyncContext;
+struct ev_loop;
+
+/**
+ * Creates new redis pool
+ * @return
+ */
+void *rspamd_redis_pool_init(void);
+
+/**
+ * Configure redis pool and binds it to a specific event base
+ * @param cfg
+ * @param ev_base
+ */
+void rspamd_redis_pool_config(void *pool,
+ struct rspamd_config *cfg,
+ struct ev_loop *ev_base);
+
+
+/**
+ * Create or reuse the specific redis connection
+ * @param pool
+ * @param db
+ * @param username
+ * @param password
+ * @param ip
+ * @param port
+ * @return
+ */
+struct redisAsyncContext *rspamd_redis_pool_connect(
+ void *pool,
+ const gchar *db, const gchar *username, const gchar *password,
+ const char *ip, int port);
+
+enum rspamd_redis_pool_release_type {
+ RSPAMD_REDIS_RELEASE_DEFAULT = 0,
+ RSPAMD_REDIS_RELEASE_FATAL = 1,
+ RSPAMD_REDIS_RELEASE_ENFORCE
+};
+
+/**
+ * Release a connection to the pool
+ * @param pool
+ * @param ctx
+ */
+void rspamd_redis_pool_release_connection(void *pool,
+ struct redisAsyncContext *ctx,
+ enum rspamd_redis_pool_release_type how);
+
+/**
+ * Stops redis pool and destroys it
+ * @param pool
+ */
+void rspamd_redis_pool_destroy(void *pool);
+
+/**
+ * Missing in hiredis
+ * @param type
+ * @return
+ */
+const gchar *rspamd_redis_type_to_string(int type);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBSERVER_REDIS_POOL_H_ */
diff --git a/src/libserver/roll_history.c b/src/libserver/roll_history.c
new file mode 100644
index 0000000..f567b0b
--- /dev/null
+++ b/src/libserver/roll_history.c
@@ -0,0 +1,432 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "libmime/message.h"
+#include "lua/lua_common.h"
+#include "unix-std.h"
+#include "cfg_file_private.h"
+
+static const gchar rspamd_history_magic_old[] = {'r', 's', 'h', '1'};
+
+/**
+ * Returns new roll history
+ * @param pool pool for shared memory
+ * @return new structure
+ */
+struct roll_history *
+rspamd_roll_history_new(rspamd_mempool_t *pool, guint max_rows,
+ struct rspamd_config *cfg)
+{
+ struct roll_history *history;
+ lua_State *L = cfg->lua_state;
+
+ if (pool == NULL || max_rows == 0) {
+ return NULL;
+ }
+
+ history = rspamd_mempool_alloc0_shared(pool, sizeof(struct roll_history));
+
+ /*
+ * Here, we check if there is any plugin that handles history,
+ * in this case, we disable this code completely
+ */
+ lua_getglobal(L, "rspamd_plugins");
+ if (lua_istable(L, -1)) {
+ lua_pushstring(L, "history");
+ lua_gettable(L, -2);
+
+ if (lua_istable(L, -1)) {
+ history->disabled = TRUE;
+ }
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+
+ if (!history->disabled) {
+ history->rows = rspamd_mempool_alloc0_shared(pool,
+ sizeof(struct roll_history_row) * max_rows);
+ history->nrows = max_rows;
+ }
+
+ return history;
+}
+
+struct history_metric_callback_data {
+ gchar *pos;
+ gint remain;
+};
+
+static void
+roll_history_symbols_callback(gpointer key, gpointer value, void *user_data)
+{
+ struct history_metric_callback_data *cb = user_data;
+ struct rspamd_symbol_result *s = value;
+ guint wr;
+
+ if (s->flags & RSPAMD_SYMBOL_RESULT_IGNORED) {
+ return;
+ }
+
+ if (cb->remain > 0) {
+ wr = rspamd_snprintf(cb->pos, cb->remain, "%s, ", s->name);
+ cb->pos += wr;
+ cb->remain -= wr;
+ }
+}
+
+/**
+ * Update roll history with data from task
+ * @param history roll history object
+ * @param task task object
+ */
+void rspamd_roll_history_update(struct roll_history *history,
+ struct rspamd_task *task)
+{
+ guint row_num;
+ struct roll_history_row *row;
+ struct rspamd_scan_result *metric_res;
+ struct history_metric_callback_data cbdata;
+ struct rspamd_action *action;
+
+ if (history->disabled) {
+ return;
+ }
+
+ /* First of all obtain check and obtain row number */
+ g_atomic_int_compare_and_exchange(&history->cur_row, history->nrows, 0);
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30))
+ row_num = g_atomic_int_add(&history->cur_row, 1);
+#else
+ row_num = g_atomic_int_exchange_and_add(&history->cur_row, 1);
+#endif
+
+ if (row_num < history->nrows) {
+ row = &history->rows[row_num];
+ g_atomic_int_set(&row->completed, FALSE);
+ }
+ else {
+ /* Race condition */
+ history->cur_row = 0;
+ return;
+ }
+
+ /* Add information from task to roll history */
+ if (task->from_addr) {
+ rspamd_strlcpy(row->from_addr,
+ rspamd_inet_address_to_string(task->from_addr),
+ sizeof(row->from_addr));
+ }
+ else {
+ rspamd_strlcpy(row->from_addr, "unknown", sizeof(row->from_addr));
+ }
+
+ row->timestamp = task->task_timestamp;
+
+ /* Strings */
+ if (task->message) {
+ rspamd_strlcpy(row->message_id, MESSAGE_FIELD(task, message_id),
+ sizeof(row->message_id));
+ }
+ if (task->auth_user) {
+ rspamd_strlcpy(row->user, task->auth_user, sizeof(row->user));
+ }
+ else {
+ row->user[0] = '\0';
+ }
+
+ /* Get default metric */
+ metric_res = task->result;
+
+ if (metric_res == NULL) {
+ row->symbols[0] = '\0';
+ row->action = METRIC_ACTION_NOACTION;
+ }
+ else {
+ row->score = metric_res->score;
+ action = rspamd_check_action_metric(task, NULL, NULL);
+ row->action = action->action_type;
+ row->required_score = rspamd_task_get_required_score(task, metric_res);
+ cbdata.pos = row->symbols;
+ cbdata.remain = sizeof(row->symbols);
+ rspamd_task_symbol_result_foreach(task, NULL,
+ roll_history_symbols_callback,
+ &cbdata);
+ if (cbdata.remain > 0) {
+ /* Remove last whitespace and comma */
+ *cbdata.pos-- = '\0';
+ *cbdata.pos-- = '\0';
+ *cbdata.pos = '\0';
+ }
+ }
+
+ row->scan_time = task->time_real_finish - task->task_timestamp;
+ row->len = task->msg.len;
+ g_atomic_int_set(&row->completed, TRUE);
+}
+
+/**
+ * Load previously saved history from file
+ * @param history roll history object
+ * @param filename filename to load from
+ * @return TRUE if history has been loaded
+ */
+gboolean
+rspamd_roll_history_load(struct roll_history *history, const gchar *filename)
+{
+ gint fd;
+ struct stat st;
+ gchar magic[sizeof(rspamd_history_magic_old)];
+ ucl_object_t *top;
+ const ucl_object_t *cur, *elt;
+ struct ucl_parser *parser;
+ struct roll_history_row *row;
+ guint n, i;
+
+ g_assert(history != NULL);
+ if (history->disabled) {
+ return TRUE;
+ }
+
+ if (stat(filename, &st) == -1) {
+ msg_info("cannot load history from %s: %s", filename,
+ strerror(errno));
+ return FALSE;
+ }
+
+ if ((fd = open(filename, O_RDONLY)) == -1) {
+ msg_info("cannot load history from %s: %s", filename,
+ strerror(errno));
+ return FALSE;
+ }
+
+ /* Check for old format */
+ if (read(fd, magic, sizeof(magic)) == -1) {
+ close(fd);
+ msg_info("cannot read history from %s: %s", filename,
+ strerror(errno));
+ return FALSE;
+ }
+
+ if (memcmp(magic, rspamd_history_magic_old, sizeof(magic)) == 0) {
+ close(fd);
+ msg_warn("cannot read history from old format %s, "
+ "it will be replaced after restart",
+ filename);
+ return FALSE;
+ }
+
+ parser = ucl_parser_new(0);
+
+ if (!ucl_parser_add_fd(parser, fd)) {
+ msg_warn("cannot parse history file %s: %s", filename,
+ ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+ close(fd);
+
+ return FALSE;
+ }
+
+ top = ucl_parser_get_object(parser);
+ ucl_parser_free(parser);
+ close(fd);
+
+ if (top == NULL) {
+ msg_warn("cannot parse history file %s: no object", filename);
+
+ return FALSE;
+ }
+
+ if (ucl_object_type(top) != UCL_ARRAY) {
+ msg_warn("invalid object type read from: %s", filename);
+ ucl_object_unref(top);
+
+ return FALSE;
+ }
+
+ if (top->len > history->nrows) {
+ msg_warn("stored history is larger than the current one: %ud (file) vs "
+ "%ud (history)",
+ top->len, history->nrows);
+ n = history->nrows;
+ }
+ else if (top->len < history->nrows) {
+ msg_warn(
+ "stored history is smaller than the current one: %ud (file) vs "
+ "%ud (history)",
+ top->len, history->nrows);
+ n = top->len;
+ }
+ else {
+ n = top->len;
+ }
+
+ for (i = 0; i < n; i++) {
+ cur = ucl_array_find_index(top, i);
+
+ if (cur != NULL && ucl_object_type(cur) == UCL_OBJECT) {
+ row = &history->rows[i];
+ memset(row, 0, sizeof(*row));
+
+ elt = ucl_object_lookup(cur, "time");
+
+ if (elt && ucl_object_type(elt) == UCL_FLOAT) {
+ row->timestamp = ucl_object_todouble(elt);
+ }
+
+ elt = ucl_object_lookup(cur, "id");
+
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ rspamd_strlcpy(row->message_id, ucl_object_tostring(elt),
+ sizeof(row->message_id));
+ }
+
+ elt = ucl_object_lookup(cur, "symbols");
+
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ rspamd_strlcpy(row->symbols, ucl_object_tostring(elt),
+ sizeof(row->symbols));
+ }
+
+ elt = ucl_object_lookup(cur, "user");
+
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ rspamd_strlcpy(row->user, ucl_object_tostring(elt),
+ sizeof(row->user));
+ }
+
+ elt = ucl_object_lookup(cur, "from");
+
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ rspamd_strlcpy(row->from_addr, ucl_object_tostring(elt),
+ sizeof(row->from_addr));
+ }
+
+ elt = ucl_object_lookup(cur, "len");
+
+ if (elt && ucl_object_type(elt) == UCL_INT) {
+ row->len = ucl_object_toint(elt);
+ }
+
+ elt = ucl_object_lookup(cur, "scan_time");
+
+ if (elt && ucl_object_type(elt) == UCL_FLOAT) {
+ row->scan_time = ucl_object_todouble(elt);
+ }
+
+ elt = ucl_object_lookup(cur, "score");
+
+ if (elt && ucl_object_type(elt) == UCL_FLOAT) {
+ row->score = ucl_object_todouble(elt);
+ }
+
+ elt = ucl_object_lookup(cur, "required_score");
+
+ if (elt && ucl_object_type(elt) == UCL_FLOAT) {
+ row->required_score = ucl_object_todouble(elt);
+ }
+
+ elt = ucl_object_lookup(cur, "action");
+
+ if (elt && ucl_object_type(elt) == UCL_INT) {
+ row->action = ucl_object_toint(elt);
+ }
+
+ row->completed = TRUE;
+ }
+ }
+
+ ucl_object_unref(top);
+
+ history->cur_row = n;
+
+ return TRUE;
+}
+
+/**
+ * Save history to file
+ * @param history roll history object
+ * @param filename filename to load from
+ * @return TRUE if history has been saved
+ */
+gboolean
+rspamd_roll_history_save(struct roll_history *history, const gchar *filename)
+{
+ gint fd;
+ FILE *fp;
+ ucl_object_t *obj, *elt;
+ guint i;
+ struct roll_history_row *row;
+ struct ucl_emitter_functions *emitter_func;
+
+ g_assert(history != NULL);
+
+ if (history->disabled) {
+ return TRUE;
+ }
+
+ if ((fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 00600)) == -1) {
+ msg_info("cannot save history to %s: %s", filename, strerror(errno));
+ return FALSE;
+ }
+
+ fp = fdopen(fd, "w");
+ obj = ucl_object_typed_new(UCL_ARRAY);
+
+ for (i = 0; i < history->nrows; i++) {
+ row = &history->rows[i];
+
+ if (!row->completed) {
+ continue;
+ }
+
+ elt = ucl_object_typed_new(UCL_OBJECT);
+
+ ucl_object_insert_key(elt, ucl_object_fromdouble(row->timestamp),
+ "time", 0, false);
+ ucl_object_insert_key(elt, ucl_object_fromstring(row->message_id),
+ "id", 0, false);
+ ucl_object_insert_key(elt, ucl_object_fromstring(row->symbols),
+ "symbols", 0, false);
+ ucl_object_insert_key(elt, ucl_object_fromstring(row->user),
+ "user", 0, false);
+ ucl_object_insert_key(elt, ucl_object_fromstring(row->from_addr),
+ "from", 0, false);
+ ucl_object_insert_key(elt, ucl_object_fromint(row->len),
+ "len", 0, false);
+ ucl_object_insert_key(elt, ucl_object_fromdouble(row->scan_time),
+ "scan_time", 0, false);
+ ucl_object_insert_key(elt, ucl_object_fromdouble(row->score),
+ "score", 0, false);
+ ucl_object_insert_key(elt, ucl_object_fromdouble(row->required_score),
+ "required_score", 0, false);
+ ucl_object_insert_key(elt, ucl_object_fromint(row->action),
+ "action", 0, false);
+
+ ucl_array_append(obj, elt);
+ }
+
+ emitter_func = ucl_object_emit_file_funcs(fp);
+ ucl_object_emit_full(obj, UCL_EMIT_JSON_COMPACT, emitter_func, NULL);
+ ucl_object_emit_funcs_free(emitter_func);
+ ucl_object_unref(obj);
+
+ fclose(fp);
+
+ return TRUE;
+}
diff --git a/src/libserver/roll_history.h b/src/libserver/roll_history.h
new file mode 100644
index 0000000..62bce7f
--- /dev/null
+++ b/src/libserver/roll_history.h
@@ -0,0 +1,98 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef ROLL_HISTORY_H_
+#define ROLL_HISTORY_H_
+
+#include "config.h"
+#include "mem_pool.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Roll history is a special cycled buffer for checked messages, it is designed for writing history messages
+ * and displaying them in webui
+ */
+
+#define HISTORY_MAX_ID 256
+#define HISTORY_MAX_SYMBOLS 256
+#define HISTORY_MAX_USER 32
+#define HISTORY_MAX_ADDR 32
+
+struct rspamd_task;
+struct rspamd_config;
+
+struct roll_history_row {
+ ev_tstamp timestamp;
+ gchar message_id[HISTORY_MAX_ID];
+ gchar symbols[HISTORY_MAX_SYMBOLS];
+ gchar user[HISTORY_MAX_USER];
+ gchar from_addr[HISTORY_MAX_ADDR];
+ gsize len;
+ gdouble scan_time;
+ gdouble score;
+ gdouble required_score;
+ gint action;
+ guint completed;
+};
+
+struct roll_history {
+ struct roll_history_row *rows;
+ gboolean disabled;
+ guint nrows;
+ guint cur_row;
+};
+
+/**
+ * Returns new roll history
+ * @param pool pool for shared memory
+ * @return new structure
+ */
+struct roll_history *rspamd_roll_history_new(rspamd_mempool_t *pool,
+ guint max_rows, struct rspamd_config *cfg);
+
+/**
+ * Update roll history with data from task
+ * @param history roll history object
+ * @param task task object
+ */
+void rspamd_roll_history_update(struct roll_history *history,
+ struct rspamd_task *task);
+
+/**
+ * Load previously saved history from file
+ * @param history roll history object
+ * @param filename filename to load from
+ * @return TRUE if history has been loaded
+ */
+gboolean rspamd_roll_history_load(struct roll_history *history,
+ const gchar *filename);
+
+/**
+ * Save history to file
+ * @param history roll history object
+ * @param filename filename to load from
+ * @return TRUE if history has been saved
+ */
+gboolean rspamd_roll_history_save(struct roll_history *history,
+ const gchar *filename);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ROLL_HISTORY_H_ */
diff --git a/src/libserver/rspamd_control.c b/src/libserver/rspamd_control.c
new file mode 100644
index 0000000..69af059
--- /dev/null
+++ b/src/libserver/rspamd_control.c
@@ -0,0 +1,1334 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "rspamd_control.h"
+#include "worker_util.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
+#include "libutil/libev_helper.h"
+#include "unix-std.h"
+#include "utlist.h"
+
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
+
+#ifdef WITH_HYPERSCAN
+#include "hyperscan_tools.h"
+#endif
+
+static ev_tstamp io_timeout = 30.0;
+static ev_tstamp worker_io_timeout = 0.5;
+
+struct rspamd_control_session;
+
+struct rspamd_control_reply_elt {
+ struct rspamd_control_reply reply;
+ struct rspamd_io_ev ev;
+ struct ev_loop *event_loop;
+ GQuark wrk_type;
+ pid_t wrk_pid;
+ gpointer ud;
+ gint attached_fd;
+ GHashTable *pending_elts;
+ struct rspamd_control_reply_elt *prev, *next;
+};
+
+struct rspamd_control_session {
+ gint fd;
+ struct ev_loop *event_loop;
+ struct rspamd_main *rspamd_main;
+ struct rspamd_http_connection *conn;
+ struct rspamd_control_command cmd;
+ struct rspamd_control_reply_elt *replies;
+ rspamd_inet_addr_t *addr;
+ guint replies_remain;
+ gboolean is_reply;
+};
+
+static const struct rspamd_control_cmd_match {
+ rspamd_ftok_t name;
+ enum rspamd_control_type type;
+} cmd_matches[] = {
+ {.name = {
+ .begin = "/stat",
+ .len = sizeof("/stat") - 1},
+ .type = RSPAMD_CONTROL_STAT},
+ {.name = {.begin = "/reload", .len = sizeof("/reload") - 1}, .type = RSPAMD_CONTROL_RELOAD},
+ {.name = {.begin = "/reresolve", .len = sizeof("/reresolve") - 1}, .type = RSPAMD_CONTROL_RERESOLVE},
+ {.name = {.begin = "/recompile", .len = sizeof("/recompile") - 1}, .type = RSPAMD_CONTROL_RECOMPILE},
+ {.name = {.begin = "/fuzzystat", .len = sizeof("/fuzzystat") - 1}, .type = RSPAMD_CONTROL_FUZZY_STAT},
+ {.name = {.begin = "/fuzzysync", .len = sizeof("/fuzzysync") - 1}, .type = RSPAMD_CONTROL_FUZZY_SYNC},
+};
+
+static void rspamd_control_ignore_io_handler(int fd, short what, void *ud);
+
+static void
+rspamd_control_stop_pending(struct rspamd_control_reply_elt *elt)
+{
+ GHashTable *htb;
+ /* It stops event and frees hash */
+ htb = elt->pending_elts;
+ g_hash_table_remove(elt->pending_elts, elt);
+ /* Release hash reference */
+ g_hash_table_unref(htb);
+}
+
+void rspamd_control_send_error(struct rspamd_control_session *session,
+ gint code, const gchar *error_msg, ...)
+{
+ struct rspamd_http_message *msg;
+ rspamd_fstring_t *reply;
+ va_list args;
+
+ msg = rspamd_http_new_message(HTTP_RESPONSE);
+
+ va_start(args, error_msg);
+ msg->status = rspamd_fstring_new();
+ rspamd_vprintf_fstring(&msg->status, error_msg, args);
+ va_end(args);
+
+ msg->date = time(NULL);
+ msg->code = code;
+ reply = rspamd_fstring_sized_new(msg->status->len + 16);
+ rspamd_printf_fstring(&reply, "{\"error\":\"%V\"}", msg->status);
+ rspamd_http_message_set_body_from_fstring_steal(msg, reply);
+ rspamd_http_connection_reset(session->conn);
+ rspamd_http_connection_write_message(session->conn,
+ msg,
+ NULL,
+ "application/json",
+ session,
+ io_timeout);
+}
+
+static void
+rspamd_control_send_ucl(struct rspamd_control_session *session,
+ ucl_object_t *obj)
+{
+ struct rspamd_http_message *msg;
+ rspamd_fstring_t *reply;
+
+ msg = rspamd_http_new_message(HTTP_RESPONSE);
+ msg->date = time(NULL);
+ msg->code = 200;
+ msg->status = rspamd_fstring_new_init("OK", 2);
+ reply = rspamd_fstring_sized_new(BUFSIZ);
+ rspamd_ucl_emit_fstring(obj, UCL_EMIT_JSON_COMPACT, &reply);
+ rspamd_http_message_set_body_from_fstring_steal(msg, reply);
+ rspamd_http_connection_reset(session->conn);
+ rspamd_http_connection_write_message(session->conn,
+ msg,
+ NULL,
+ "application/json",
+ session,
+ io_timeout);
+}
+
+static void
+rspamd_control_connection_close(struct rspamd_control_session *session)
+{
+ struct rspamd_control_reply_elt *elt, *telt;
+ struct rspamd_main *rspamd_main;
+
+ rspamd_main = session->rspamd_main;
+ msg_info_main("finished connection from %s",
+ rspamd_inet_address_to_string(session->addr));
+
+ DL_FOREACH_SAFE(session->replies, elt, telt)
+ {
+ rspamd_control_stop_pending(elt);
+ }
+
+ rspamd_inet_address_free(session->addr);
+ rspamd_http_connection_unref(session->conn);
+ close(session->fd);
+ g_free(session);
+}
+
+static void
+rspamd_control_write_reply(struct rspamd_control_session *session)
+{
+ ucl_object_t *rep, *cur, *workers;
+ struct rspamd_control_reply_elt *elt;
+ gchar tmpbuf[64];
+ gdouble total_utime = 0, total_systime = 0;
+ struct ucl_parser *parser;
+ guint total_conns = 0;
+
+ rep = ucl_object_typed_new(UCL_OBJECT);
+ workers = ucl_object_typed_new(UCL_OBJECT);
+
+ DL_FOREACH(session->replies, elt)
+ {
+ /* Skip incompatible worker for fuzzy_stat */
+ if ((session->cmd.type == RSPAMD_CONTROL_FUZZY_STAT ||
+ session->cmd.type == RSPAMD_CONTROL_FUZZY_SYNC) &&
+ elt->wrk_type != g_quark_from_static_string("fuzzy")) {
+ continue;
+ }
+
+ rspamd_snprintf(tmpbuf, sizeof(tmpbuf), "%P", elt->wrk_pid);
+ cur = ucl_object_typed_new(UCL_OBJECT);
+
+ ucl_object_insert_key(cur, ucl_object_fromstring(g_quark_to_string(elt->wrk_type)), "type", 0, false);
+
+ switch (session->cmd.type) {
+ case RSPAMD_CONTROL_STAT:
+ ucl_object_insert_key(cur, ucl_object_fromint(elt->reply.reply.stat.conns), "conns", 0, false);
+ ucl_object_insert_key(cur, ucl_object_fromdouble(elt->reply.reply.stat.utime), "utime", 0, false);
+ ucl_object_insert_key(cur, ucl_object_fromdouble(elt->reply.reply.stat.systime), "systime", 0, false);
+ ucl_object_insert_key(cur, ucl_object_fromdouble(elt->reply.reply.stat.uptime), "uptime", 0, false);
+ ucl_object_insert_key(cur, ucl_object_fromint(elt->reply.reply.stat.maxrss), "maxrss", 0, false);
+
+ total_utime += elt->reply.reply.stat.utime;
+ total_systime += elt->reply.reply.stat.systime;
+ total_conns += elt->reply.reply.stat.conns;
+
+ break;
+
+ case RSPAMD_CONTROL_RELOAD:
+ ucl_object_insert_key(cur, ucl_object_fromint(elt->reply.reply.reload.status), "status", 0, false);
+ break;
+ case RSPAMD_CONTROL_RECOMPILE:
+ ucl_object_insert_key(cur, ucl_object_fromint(elt->reply.reply.recompile.status), "status", 0, false);
+ break;
+ case RSPAMD_CONTROL_RERESOLVE:
+ ucl_object_insert_key(cur, ucl_object_fromint(elt->reply.reply.reresolve.status), "status", 0, false);
+ break;
+ case RSPAMD_CONTROL_FUZZY_STAT:
+ if (elt->attached_fd != -1) {
+ /* We have some data to parse */
+ parser = ucl_parser_new(0);
+ ucl_object_insert_key(cur,
+ ucl_object_fromint(
+ elt->reply.reply.fuzzy_stat.status),
+ "status",
+ 0,
+ false);
+
+ if (ucl_parser_add_fd(parser, elt->attached_fd)) {
+ ucl_object_insert_key(cur, ucl_parser_get_object(parser),
+ "data", 0, false);
+ ucl_parser_free(parser);
+ }
+ else {
+
+ ucl_object_insert_key(cur, ucl_object_fromstring(ucl_parser_get_error(parser)), "error", 0, false);
+
+ ucl_parser_free(parser);
+ }
+
+ ucl_object_insert_key(cur,
+ ucl_object_fromlstring(
+ elt->reply.reply.fuzzy_stat.storage_id,
+ MEMPOOL_UID_LEN - 1),
+ "id",
+ 0,
+ false);
+ }
+ else {
+ ucl_object_insert_key(cur,
+ ucl_object_fromstring("missing file"),
+ "error",
+ 0,
+ false);
+ ucl_object_insert_key(cur,
+ ucl_object_fromint(
+ elt->reply.reply.fuzzy_stat.status),
+ "status",
+ 0,
+ false);
+ }
+ break;
+ case RSPAMD_CONTROL_FUZZY_SYNC:
+ ucl_object_insert_key(cur, ucl_object_fromint(elt->reply.reply.fuzzy_sync.status), "status", 0, false);
+ break;
+ default:
+ break;
+ }
+
+ if (elt->attached_fd != -1) {
+ close(elt->attached_fd);
+ elt->attached_fd = -1;
+ }
+
+ ucl_object_insert_key(workers, cur, tmpbuf, 0, true);
+ }
+
+ ucl_object_insert_key(rep, workers, "workers", 0, false);
+
+ if (session->cmd.type == RSPAMD_CONTROL_STAT) {
+ /* Total stats */
+ cur = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(cur, ucl_object_fromint(total_conns), "conns", 0, false);
+ ucl_object_insert_key(cur, ucl_object_fromdouble(total_utime), "utime", 0, false);
+ ucl_object_insert_key(cur, ucl_object_fromdouble(total_systime), "systime", 0, false);
+
+ ucl_object_insert_key(rep, cur, "total", 0, false);
+ }
+
+ rspamd_control_send_ucl(session, rep);
+ ucl_object_unref(rep);
+}
+
+static void
+rspamd_control_wrk_io(gint fd, short what, gpointer ud)
+{
+ struct rspamd_control_reply_elt *elt = ud;
+ struct rspamd_control_session *session;
+ guchar fdspace[CMSG_SPACE(sizeof(int))];
+ struct iovec iov;
+ struct msghdr msg;
+ gssize r;
+
+ session = elt->ud;
+ elt->attached_fd = -1;
+
+ if (what == EV_READ) {
+ iov.iov_base = &elt->reply;
+ iov.iov_len = sizeof(elt->reply);
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_control = fdspace;
+ msg.msg_controllen = sizeof(fdspace);
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+
+ r = recvmsg(fd, &msg, 0);
+ if (r == -1) {
+ msg_err("cannot read reply from the worker %P (%s): %s",
+ elt->wrk_pid, g_quark_to_string(elt->wrk_type),
+ strerror(errno));
+ }
+ else if (r >= (gssize) sizeof(elt->reply)) {
+ if (msg.msg_controllen >= CMSG_LEN(sizeof(int))) {
+ elt->attached_fd = *(int *) CMSG_DATA(CMSG_FIRSTHDR(&msg));
+ }
+ }
+ }
+ else {
+ /* Timeout waiting */
+ msg_warn("timeout waiting reply from %P (%s)",
+ elt->wrk_pid, g_quark_to_string(elt->wrk_type));
+ }
+
+ session->replies_remain--;
+ rspamd_ev_watcher_stop(session->event_loop,
+ &elt->ev);
+
+ if (session->replies_remain == 0) {
+ rspamd_control_write_reply(session);
+ }
+}
+
+static void
+rspamd_control_error_handler(struct rspamd_http_connection *conn, GError *err)
+{
+ struct rspamd_control_session *session = conn->ud;
+ struct rspamd_main *rspamd_main;
+
+ rspamd_main = session->rspamd_main;
+
+ if (!session->is_reply) {
+ msg_info_main("abnormally closing control connection: %e", err);
+ session->is_reply = TRUE;
+ rspamd_control_send_error(session, err->code, "%s", err->message);
+ }
+ else {
+ rspamd_control_connection_close(session);
+ }
+}
+
+void rspamd_pending_control_free(gpointer p)
+{
+ struct rspamd_control_reply_elt *rep_elt = (struct rspamd_control_reply_elt *) p;
+
+ rspamd_ev_watcher_stop(rep_elt->event_loop, &rep_elt->ev);
+ g_free(rep_elt);
+}
+
+static struct rspamd_control_reply_elt *
+rspamd_control_broadcast_cmd(struct rspamd_main *rspamd_main,
+ struct rspamd_control_command *cmd,
+ gint attached_fd,
+ rspamd_ev_cb handler,
+ gpointer ud,
+ pid_t except_pid)
+{
+ GHashTableIter it;
+ struct rspamd_worker *wrk;
+ struct rspamd_control_reply_elt *rep_elt, *res = NULL;
+ gpointer k, v;
+ struct msghdr msg;
+ struct cmsghdr *cmsg;
+ struct iovec iov;
+ guchar fdspace[CMSG_SPACE(sizeof(int))];
+ gssize r;
+
+ g_hash_table_iter_init(&it, rspamd_main->workers);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ wrk = v;
+
+ /* No control pipe */
+ if (wrk->control_pipe[0] == -1) {
+ continue;
+ }
+
+ if (except_pid != 0 && wrk->pid == except_pid) {
+ continue;
+ }
+
+ /* Worker is terminating, do not bother sending stuff */
+ if (wrk->state == rspamd_worker_state_terminating) {
+ continue;
+ }
+
+ memset(&msg, 0, sizeof(msg));
+
+ /* Attach fd to the message */
+ if (attached_fd != -1) {
+ memset(fdspace, 0, sizeof(fdspace));
+ msg.msg_control = fdspace;
+ msg.msg_controllen = sizeof(fdspace);
+ cmsg = CMSG_FIRSTHDR(&msg);
+ cmsg->cmsg_level = SOL_SOCKET;
+ cmsg->cmsg_type = SCM_RIGHTS;
+ cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+ memcpy(CMSG_DATA(cmsg), &attached_fd, sizeof(int));
+ }
+
+ iov.iov_base = cmd;
+ iov.iov_len = sizeof(*cmd);
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+
+ r = sendmsg(wrk->control_pipe[0], &msg, 0);
+
+ if (r == sizeof(*cmd)) {
+ rep_elt = g_malloc0(sizeof(*rep_elt));
+ rep_elt->wrk_pid = wrk->pid;
+ rep_elt->wrk_type = wrk->type;
+ rep_elt->event_loop = rspamd_main->event_loop;
+ rep_elt->ud = ud;
+ rep_elt->pending_elts = g_hash_table_ref(wrk->control_events_pending);
+ rspamd_ev_watcher_init(&rep_elt->ev,
+ wrk->control_pipe[0],
+ EV_READ, handler,
+ rep_elt);
+ rspamd_ev_watcher_start(rspamd_main->event_loop,
+ &rep_elt->ev, worker_io_timeout);
+ g_hash_table_insert(wrk->control_events_pending, rep_elt, rep_elt);
+
+ DL_APPEND(res, rep_elt);
+ }
+ else {
+ msg_err_main("cannot write command %d(%z) to the worker %P(%s), fd: %d: %s",
+ (int) cmd->type, iov.iov_len,
+ wrk->pid,
+ g_quark_to_string(wrk->type),
+ wrk->control_pipe[0],
+ strerror(errno));
+ }
+ }
+
+ return res;
+}
+
+void rspamd_control_broadcast_srv_cmd(struct rspamd_main *rspamd_main,
+ struct rspamd_control_command *cmd,
+ pid_t except_pid)
+{
+ rspamd_control_broadcast_cmd(rspamd_main, cmd, -1,
+ rspamd_control_ignore_io_handler, NULL, except_pid);
+}
+
+static gint
+rspamd_control_finish_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_control_session *session = conn->ud;
+ rspamd_ftok_t srch;
+ guint i;
+ gboolean found = FALSE;
+ struct rspamd_control_reply_elt *cur;
+
+
+ if (!session->is_reply) {
+ if (msg->url == NULL) {
+ rspamd_control_connection_close(session);
+
+ return 0;
+ }
+
+ srch.begin = msg->url->str;
+ srch.len = msg->url->len;
+
+ session->is_reply = TRUE;
+
+ for (i = 0; i < G_N_ELEMENTS(cmd_matches); i++) {
+ if (rspamd_ftok_casecmp(&srch, &cmd_matches[i].name) == 0) {
+ session->cmd.type = cmd_matches[i].type;
+ found = TRUE;
+ break;
+ }
+ }
+
+ if (!found) {
+ rspamd_control_send_error(session, 404, "Command not defined");
+ }
+ else {
+ /* Send command to all workers */
+ session->replies = rspamd_control_broadcast_cmd(
+ session->rspamd_main, &session->cmd, -1,
+ rspamd_control_wrk_io, session, 0);
+
+ DL_FOREACH(session->replies, cur)
+ {
+ session->replies_remain++;
+ }
+ }
+ }
+ else {
+ rspamd_control_connection_close(session);
+ }
+
+
+ return 0;
+}
+
+void rspamd_control_process_client_socket(struct rspamd_main *rspamd_main,
+ gint fd, rspamd_inet_addr_t *addr)
+{
+ struct rspamd_control_session *session;
+
+ session = g_malloc0(sizeof(*session));
+
+ session->fd = fd;
+ session->conn = rspamd_http_connection_new_server(rspamd_main->http_ctx,
+ fd,
+ NULL,
+ rspamd_control_error_handler,
+ rspamd_control_finish_handler,
+ 0);
+ session->rspamd_main = rspamd_main;
+ session->addr = addr;
+ session->event_loop = rspamd_main->event_loop;
+ rspamd_http_connection_read_message(session->conn, session,
+ io_timeout);
+}
+
+struct rspamd_worker_control_data {
+ ev_io io_ev;
+ struct rspamd_worker *worker;
+ struct ev_loop *ev_base;
+ struct {
+ rspamd_worker_control_handler handler;
+ gpointer ud;
+ } handlers[RSPAMD_CONTROL_MAX];
+};
+
+static void
+rspamd_control_default_cmd_handler(gint fd,
+ gint attached_fd,
+ struct rspamd_worker_control_data *cd,
+ struct rspamd_control_command *cmd)
+{
+ struct rspamd_control_reply rep;
+ gssize r;
+ struct rusage rusg;
+ struct rspamd_config *cfg;
+ struct rspamd_main *rspamd_main;
+
+ memset(&rep, 0, sizeof(rep));
+ rep.type = cmd->type;
+ rspamd_main = cd->worker->srv;
+
+ switch (cmd->type) {
+ case RSPAMD_CONTROL_STAT:
+ if (getrusage(RUSAGE_SELF, &rusg) == -1) {
+ msg_err_main("cannot get rusage stats: %s",
+ strerror(errno));
+ }
+ else {
+ rep.reply.stat.utime = tv_to_double(&rusg.ru_utime);
+ rep.reply.stat.systime = tv_to_double(&rusg.ru_stime);
+ rep.reply.stat.maxrss = rusg.ru_maxrss;
+ }
+
+ rep.reply.stat.conns = cd->worker->nconns;
+ rep.reply.stat.uptime = rspamd_get_calendar_ticks() - cd->worker->start_time;
+ break;
+ case RSPAMD_CONTROL_RELOAD:
+ case RSPAMD_CONTROL_RECOMPILE:
+ case RSPAMD_CONTROL_HYPERSCAN_LOADED:
+ case RSPAMD_CONTROL_MONITORED_CHANGE:
+ case RSPAMD_CONTROL_FUZZY_STAT:
+ case RSPAMD_CONTROL_FUZZY_SYNC:
+ case RSPAMD_CONTROL_LOG_PIPE:
+ case RSPAMD_CONTROL_CHILD_CHANGE:
+ case RSPAMD_CONTROL_FUZZY_BLOCKED:
+ break;
+ case RSPAMD_CONTROL_RERESOLVE:
+ if (cd->worker->srv->cfg) {
+ REF_RETAIN(cd->worker->srv->cfg);
+ cfg = cd->worker->srv->cfg;
+
+ if (cfg->ups_ctx) {
+ msg_info_config("reresolving upstreams");
+ rspamd_upstream_reresolve(cfg->ups_ctx);
+ }
+
+ rep.reply.reresolve.status = 0;
+ REF_RELEASE(cfg);
+ }
+ else {
+ rep.reply.reresolve.status = EINVAL;
+ }
+ break;
+ default:
+ break;
+ }
+
+ r = write(fd, &rep, sizeof(rep));
+
+ if (r != sizeof(rep)) {
+ msg_err_main("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ if (attached_fd != -1) {
+ close(attached_fd);
+ }
+}
+
+static void
+rspamd_control_default_worker_handler(EV_P_ ev_io *w, int revents)
+{
+ struct rspamd_worker_control_data *cd =
+ (struct rspamd_worker_control_data *) w->data;
+ static struct rspamd_control_command cmd;
+ static struct msghdr msg;
+ static struct iovec iov;
+ static guchar fdspace[CMSG_SPACE(sizeof(int))];
+ gint rfd = -1;
+ gssize r;
+
+ iov.iov_base = &cmd;
+ iov.iov_len = sizeof(cmd);
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_control = fdspace;
+ msg.msg_controllen = sizeof(fdspace);
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+
+ r = recvmsg(w->fd, &msg, 0);
+
+ if (r == -1) {
+ if (errno != EAGAIN && errno != EINTR) {
+ if (errno != ECONNRESET) {
+ /*
+ * In case of connection reset it means that main process
+ * has died, so do not pollute logs
+ */
+ msg_err("cannot read request from the control socket: %s",
+ strerror(errno));
+ }
+ ev_io_stop(cd->ev_base, &cd->io_ev);
+ close(w->fd);
+ }
+ }
+ else if (r < (gint) sizeof(cmd)) {
+ msg_err("short read of control command: %d of %d", (gint) r,
+ (gint) sizeof(cmd));
+
+ if (r == 0) {
+ ev_io_stop(cd->ev_base, &cd->io_ev);
+ close(w->fd);
+ }
+ }
+ else if ((gint) cmd.type >= 0 && cmd.type < RSPAMD_CONTROL_MAX) {
+
+ if (msg.msg_controllen >= CMSG_LEN(sizeof(int))) {
+ rfd = *(int *) CMSG_DATA(CMSG_FIRSTHDR(&msg));
+ }
+
+ if (cd->handlers[cmd.type].handler) {
+ cd->handlers[cmd.type].handler(cd->worker->srv,
+ cd->worker,
+ w->fd,
+ rfd,
+ &cmd,
+ cd->handlers[cmd.type].ud);
+ }
+ else {
+ rspamd_control_default_cmd_handler(w->fd, rfd, cd, &cmd);
+ }
+ }
+ else {
+ msg_err("unknown command: %d", (gint) cmd.type);
+ }
+}
+
+void rspamd_control_worker_add_default_cmd_handlers(struct rspamd_worker *worker,
+ struct ev_loop *ev_base)
+{
+ struct rspamd_worker_control_data *cd;
+
+ cd = g_malloc0(sizeof(*cd));
+ cd->worker = worker;
+ cd->ev_base = ev_base;
+
+ cd->io_ev.data = cd;
+ ev_io_init(&cd->io_ev, rspamd_control_default_worker_handler,
+ worker->control_pipe[1], EV_READ);
+ ev_io_start(ev_base, &cd->io_ev);
+
+ worker->control_data = cd;
+}
+
+/**
+ * Register custom handler for a specific control command for this worker
+ */
+void rspamd_control_worker_add_cmd_handler(struct rspamd_worker *worker,
+ enum rspamd_control_type type,
+ rspamd_worker_control_handler handler,
+ gpointer ud)
+{
+ struct rspamd_worker_control_data *cd;
+
+ g_assert(type >= 0 && type < RSPAMD_CONTROL_MAX);
+ g_assert(handler != NULL);
+ g_assert(worker->control_data != NULL);
+
+ cd = worker->control_data;
+ cd->handlers[type].handler = handler;
+ cd->handlers[type].ud = ud;
+}
+
+struct rspamd_srv_reply_data {
+ struct rspamd_worker *worker;
+ struct rspamd_main *srv;
+ gint fd;
+ struct rspamd_srv_reply rep;
+};
+
+static void
+rspamd_control_ignore_io_handler(int fd, short what, void *ud)
+{
+ struct rspamd_control_reply_elt *elt =
+ (struct rspamd_control_reply_elt *) ud;
+
+ struct rspamd_control_reply rep;
+
+ /* At this point we just ignore replies from the workers */
+ if (read(fd, &rep, sizeof(rep)) == -1) {
+ msg_debug("cannot read %d bytes: %s", (int) sizeof(rep), strerror(errno));
+ }
+ rspamd_control_stop_pending(elt);
+}
+
+static void
+rspamd_control_log_pipe_io_handler(int fd, short what, void *ud)
+{
+ struct rspamd_control_reply_elt *elt =
+ (struct rspamd_control_reply_elt *) ud;
+ struct rspamd_control_reply rep;
+
+ /* At this point we just ignore replies from the workers */
+ (void) !read(fd, &rep, sizeof(rep));
+ rspamd_control_stop_pending(elt);
+}
+
+static void
+rspamd_control_handle_on_fork(struct rspamd_srv_command *cmd,
+ struct rspamd_main *srv)
+{
+ struct rspamd_worker *parent, *child;
+
+ parent = g_hash_table_lookup(srv->workers,
+ GSIZE_TO_POINTER(cmd->cmd.on_fork.ppid));
+
+ if (parent == NULL) {
+ msg_err("cannot find parent for a forked process %P (%P child)",
+ cmd->cmd.on_fork.ppid, cmd->cmd.on_fork.cpid);
+
+ return;
+ }
+
+ if (cmd->cmd.on_fork.state == child_dead) {
+ /* We need to remove stale worker */
+ child = g_hash_table_lookup(srv->workers,
+ GSIZE_TO_POINTER(cmd->cmd.on_fork.cpid));
+
+ if (child == NULL) {
+ msg_err("cannot find child for a forked process %P (%P parent)",
+ cmd->cmd.on_fork.cpid, cmd->cmd.on_fork.ppid);
+
+ return;
+ }
+
+ REF_RELEASE(child->cf);
+ g_hash_table_remove(srv->workers,
+ GSIZE_TO_POINTER(cmd->cmd.on_fork.cpid));
+ g_hash_table_unref(child->control_events_pending);
+ g_free(child);
+ }
+ else {
+ child = g_malloc0(sizeof(struct rspamd_worker));
+ child->srv = srv;
+ child->type = parent->type;
+ child->pid = cmd->cmd.on_fork.cpid;
+ child->srv_pipe[0] = -1;
+ child->srv_pipe[1] = -1;
+ child->control_pipe[0] = -1;
+ child->control_pipe[1] = -1;
+ child->cf = parent->cf;
+ child->ppid = parent->pid;
+ REF_RETAIN(child->cf);
+ child->control_events_pending = g_hash_table_new_full(g_direct_hash, g_direct_equal,
+ NULL, rspamd_pending_control_free);
+ g_hash_table_insert(srv->workers,
+ GSIZE_TO_POINTER(cmd->cmd.on_fork.cpid), child);
+ }
+}
+
+static void
+rspamd_fill_health_reply(struct rspamd_main *srv, struct rspamd_srv_reply *rep)
+{
+ GHashTableIter it;
+ gpointer k, v;
+
+ memset(&rep->reply.health, 0, sizeof(rep->reply));
+ g_hash_table_iter_init(&it, srv->workers);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ struct rspamd_worker *wrk = (struct rspamd_worker *) v;
+
+ if (wrk->hb.nbeats < 0) {
+ rep->reply.health.workers_hb_lost++;
+ }
+ else if (rspamd_worker_is_scanner(wrk)) {
+ rep->reply.health.scanners_count++;
+ }
+
+ rep->reply.health.workers_count++;
+ }
+
+ rep->reply.status = (g_hash_table_size(srv->workers) > 0);
+}
+
+
+static void
+rspamd_srv_handler(EV_P_ ev_io *w, int revents)
+{
+ struct rspamd_worker *worker;
+ static struct rspamd_srv_command cmd;
+ struct rspamd_main *rspamd_main;
+ struct rspamd_srv_reply_data *rdata;
+ struct msghdr msg;
+ struct cmsghdr *cmsg;
+ static struct iovec iov;
+ static guchar fdspace[CMSG_SPACE(sizeof(int))];
+ gint *spair, rfd = -1;
+ gchar *nid;
+ struct rspamd_control_command wcmd;
+ gssize r;
+
+ if (revents == EV_READ) {
+ worker = (struct rspamd_worker *) w->data;
+ rspamd_main = worker->srv;
+ iov.iov_base = &cmd;
+ iov.iov_len = sizeof(cmd);
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_control = fdspace;
+ msg.msg_controllen = sizeof(fdspace);
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+
+ r = recvmsg(w->fd, &msg, 0);
+
+ if (r == -1) {
+ if (errno != EAGAIN) {
+ msg_err_main("cannot read from worker's srv pipe: %s",
+ strerror(errno));
+ }
+ else {
+ return;
+ }
+ }
+ else if (r == 0) {
+ /*
+ * Usually this means that a worker is dead, so do not try to read
+ * anything
+ */
+ msg_err_main("cannot read from worker's srv pipe connection closed; command = %s",
+ rspamd_srv_command_to_string(cmd.type));
+ ev_io_stop(EV_A_ w);
+ }
+ else if (r != sizeof(cmd)) {
+ msg_err_main("cannot read from worker's srv pipe incomplete command: %d != %d; command = %s",
+ (gint) r, (gint) sizeof(cmd), rspamd_srv_command_to_string(cmd.type));
+ }
+ else {
+ rdata = g_malloc0(sizeof(*rdata));
+ rdata->worker = worker;
+ rdata->srv = rspamd_main;
+ rdata->rep.id = cmd.id;
+ rdata->rep.type = cmd.type;
+ rdata->fd = -1;
+ worker->tmp_data = rdata;
+
+ if (msg.msg_controllen >= CMSG_LEN(sizeof(int))) {
+ rfd = *(int *) CMSG_DATA(CMSG_FIRSTHDR(&msg));
+ }
+
+ switch (cmd.type) {
+ case RSPAMD_SRV_SOCKETPAIR:
+ spair = g_hash_table_lookup(rspamd_main->spairs, cmd.cmd.spair.pair_id);
+ if (spair == NULL) {
+ spair = g_malloc(sizeof(gint) * 2);
+
+ if (rspamd_socketpair(spair, cmd.cmd.spair.af) == -1) {
+ rdata->rep.reply.spair.code = errno;
+ msg_err_main("cannot create socket pair: %s", strerror(errno));
+ }
+ else {
+ nid = g_malloc(sizeof(cmd.cmd.spair.pair_id));
+ memcpy(nid, cmd.cmd.spair.pair_id,
+ sizeof(cmd.cmd.spair.pair_id));
+ g_hash_table_insert(rspamd_main->spairs, nid, spair);
+ rdata->rep.reply.spair.code = 0;
+ rdata->fd = cmd.cmd.spair.pair_num ? spair[1] : spair[0];
+ }
+ }
+ else {
+ rdata->rep.reply.spair.code = 0;
+ rdata->fd = cmd.cmd.spair.pair_num ? spair[1] : spair[0];
+ }
+ break;
+ case RSPAMD_SRV_HYPERSCAN_LOADED:
+#ifdef WITH_HYPERSCAN
+ /* Load RE cache to provide it for new forks */
+ if (rspamd_re_cache_is_hs_loaded(rspamd_main->cfg->re_cache) != RSPAMD_HYPERSCAN_LOADED_FULL ||
+ cmd.cmd.hs_loaded.forced) {
+ rspamd_re_cache_load_hyperscan(
+ rspamd_main->cfg->re_cache,
+ cmd.cmd.hs_loaded.cache_dir,
+ false);
+ }
+
+ /* After getting this notice, we can clean up old hyperscan files */
+
+ rspamd_hyperscan_notice_loaded();
+
+ msg_info_main("received hyperscan cache loaded from %s",
+ cmd.cmd.hs_loaded.cache_dir);
+
+ /* Broadcast command to all workers */
+ memset(&wcmd, 0, sizeof(wcmd));
+ wcmd.type = RSPAMD_CONTROL_HYPERSCAN_LOADED;
+ rspamd_strlcpy(wcmd.cmd.hs_loaded.cache_dir,
+ cmd.cmd.hs_loaded.cache_dir,
+ sizeof(wcmd.cmd.hs_loaded.cache_dir));
+ wcmd.cmd.hs_loaded.forced = cmd.cmd.hs_loaded.forced;
+ rspamd_control_broadcast_cmd(rspamd_main, &wcmd, rfd,
+ rspamd_control_ignore_io_handler, NULL, worker->pid);
+#endif
+ break;
+ case RSPAMD_SRV_MONITORED_CHANGE:
+ /* Broadcast command to all workers */
+ memset(&wcmd, 0, sizeof(wcmd));
+ wcmd.type = RSPAMD_CONTROL_MONITORED_CHANGE;
+ rspamd_strlcpy(wcmd.cmd.monitored_change.tag,
+ cmd.cmd.monitored_change.tag,
+ sizeof(wcmd.cmd.monitored_change.tag));
+ wcmd.cmd.monitored_change.alive = cmd.cmd.monitored_change.alive;
+ wcmd.cmd.monitored_change.sender = cmd.cmd.monitored_change.sender;
+ rspamd_control_broadcast_cmd(rspamd_main, &wcmd, rfd,
+ rspamd_control_ignore_io_handler, NULL, 0);
+ break;
+ case RSPAMD_SRV_LOG_PIPE:
+ memset(&wcmd, 0, sizeof(wcmd));
+ wcmd.type = RSPAMD_CONTROL_LOG_PIPE;
+ wcmd.cmd.log_pipe.type = cmd.cmd.log_pipe.type;
+ rspamd_control_broadcast_cmd(rspamd_main, &wcmd, rfd,
+ rspamd_control_log_pipe_io_handler, NULL, 0);
+ break;
+ case RSPAMD_SRV_ON_FORK:
+ rdata->rep.reply.on_fork.status = 0;
+ rspamd_control_handle_on_fork(&cmd, rspamd_main);
+ break;
+ case RSPAMD_SRV_HEARTBEAT:
+ worker->hb.last_event = ev_time();
+ rdata->rep.reply.heartbeat.status = 0;
+ break;
+ case RSPAMD_SRV_HEALTH:
+ rspamd_fill_health_reply(rspamd_main, &rdata->rep);
+ break;
+ case RSPAMD_SRV_NOTICE_HYPERSCAN_CACHE:
+#ifdef WITH_HYPERSCAN
+ rspamd_hyperscan_notice_known(cmd.cmd.hyperscan_cache_file.path);
+#endif
+ rdata->rep.reply.hyperscan_cache_file.unused = 0;
+ break;
+ case RSPAMD_SRV_FUZZY_BLOCKED:
+ /* Broadcast command to all workers */
+ memset(&wcmd, 0, sizeof(wcmd));
+ wcmd.type = RSPAMD_CONTROL_FUZZY_BLOCKED;
+ /* Ensure that memcpy is safe */
+ G_STATIC_ASSERT(sizeof(wcmd.cmd.fuzzy_blocked) == sizeof(cmd.cmd.fuzzy_blocked));
+ memcpy(&wcmd.cmd.fuzzy_blocked, &cmd.cmd.fuzzy_blocked, sizeof(wcmd.cmd.fuzzy_blocked));
+ rspamd_control_broadcast_cmd(rspamd_main, &wcmd, rfd,
+ rspamd_control_ignore_io_handler, NULL, worker->pid);
+ break;
+ default:
+ msg_err_main("unknown command type: %d", cmd.type);
+ break;
+ }
+
+ if (rfd != -1) {
+ /* Close our copy to avoid descriptors leak */
+ close(rfd);
+ }
+
+ /* Now plan write event and send data back */
+ w->data = rdata;
+ ev_io_stop(EV_A_ w);
+ ev_io_set(w, worker->srv_pipe[0], EV_WRITE);
+ ev_io_start(EV_A_ w);
+ }
+ }
+ else if (revents == EV_WRITE) {
+ rdata = (struct rspamd_srv_reply_data *) w->data;
+ worker = rdata->worker;
+ worker->tmp_data = NULL; /* Avoid race */
+ rspamd_main = rdata->srv;
+
+ memset(&msg, 0, sizeof(msg));
+
+ /* Attach fd to the message */
+ if (rdata->fd != -1) {
+ memset(fdspace, 0, sizeof(fdspace));
+ msg.msg_control = fdspace;
+ msg.msg_controllen = sizeof(fdspace);
+ cmsg = CMSG_FIRSTHDR(&msg);
+ cmsg->cmsg_level = SOL_SOCKET;
+ cmsg->cmsg_type = SCM_RIGHTS;
+ cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+ memcpy(CMSG_DATA(cmsg), &rdata->fd, sizeof(int));
+ }
+
+ iov.iov_base = &rdata->rep;
+ iov.iov_len = sizeof(rdata->rep);
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+
+ r = sendmsg(w->fd, &msg, 0);
+
+ if (r == -1) {
+ msg_err_main("cannot write to worker's srv pipe when writing reply: %s; command = %s",
+ strerror(errno), rspamd_srv_command_to_string(rdata->rep.type));
+ }
+ else if (r != sizeof(rdata->rep)) {
+ msg_err_main("cannot write to worker's srv pipe: %d != %d; command = %s",
+ (int) r, (int) sizeof(rdata->rep),
+ rspamd_srv_command_to_string(rdata->rep.type));
+ }
+
+ g_free(rdata);
+ w->data = worker;
+ ev_io_stop(EV_A_ w);
+ ev_io_set(w, worker->srv_pipe[0], EV_READ);
+ ev_io_start(EV_A_ w);
+ }
+}
+
+void rspamd_srv_start_watching(struct rspamd_main *srv,
+ struct rspamd_worker *worker,
+ struct ev_loop *ev_base)
+{
+ g_assert(worker != NULL);
+
+ worker->tmp_data = NULL;
+ worker->srv_ev.data = worker;
+ ev_io_init(&worker->srv_ev, rspamd_srv_handler, worker->srv_pipe[0], EV_READ);
+ ev_io_start(ev_base, &worker->srv_ev);
+}
+
+struct rspamd_srv_request_data {
+ struct rspamd_worker *worker;
+ struct rspamd_srv_command cmd;
+ gint attached_fd;
+ struct rspamd_srv_reply rep;
+ rspamd_srv_reply_handler handler;
+ ev_io io_ev;
+ gpointer ud;
+};
+
+static void
+rspamd_srv_request_handler(EV_P_ ev_io *w, int revents)
+{
+ struct rspamd_srv_request_data *rd = (struct rspamd_srv_request_data *) w->data;
+ struct msghdr msg;
+ struct iovec iov;
+ guchar fdspace[CMSG_SPACE(sizeof(int))];
+ struct cmsghdr *cmsg;
+ gssize r;
+ gint rfd = -1;
+
+ if (revents == EV_WRITE) {
+ /* Send request to server */
+ memset(&msg, 0, sizeof(msg));
+
+ /* Attach fd to the message */
+ if (rd->attached_fd != -1) {
+ memset(fdspace, 0, sizeof(fdspace));
+ msg.msg_control = fdspace;
+ msg.msg_controllen = sizeof(fdspace);
+ cmsg = CMSG_FIRSTHDR(&msg);
+ cmsg->cmsg_level = SOL_SOCKET;
+ cmsg->cmsg_type = SCM_RIGHTS;
+ cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+ memcpy(CMSG_DATA(cmsg), &rd->attached_fd, sizeof(int));
+ }
+
+ iov.iov_base = &rd->cmd;
+ iov.iov_len = sizeof(rd->cmd);
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+
+ r = sendmsg(w->fd, &msg, 0);
+
+ if (r == -1) {
+ if (r == ENOBUFS) {
+ /* On BSD derived systems we can have this error when trying to send
+ * requests too fast.
+ * It might be good to retry...
+ */
+ msg_info("cannot write to server pipe: %s; command = %s; retrying sending",
+ strerror(errno),
+ rspamd_srv_command_to_string(rd->cmd.type));
+ return;
+ }
+ msg_err("cannot write to server pipe: %s; command = %s", strerror(errno),
+ rspamd_srv_command_to_string(rd->cmd.type));
+ goto cleanup;
+ }
+ else if (r != sizeof(rd->cmd)) {
+ msg_err("incomplete write to the server pipe: %d != %d, command = %s",
+ (int) r, (int) sizeof(rd->cmd), rspamd_srv_command_to_string(rd->cmd.type));
+ goto cleanup;
+ }
+
+ ev_io_stop(EV_A_ w);
+ ev_io_set(w, rd->worker->srv_pipe[1], EV_READ);
+ ev_io_start(EV_A_ w);
+ }
+ else {
+ iov.iov_base = &rd->rep;
+ iov.iov_len = sizeof(rd->rep);
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_control = fdspace;
+ msg.msg_controllen = sizeof(fdspace);
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+
+ r = recvmsg(w->fd, &msg, 0);
+
+ if (r == -1) {
+ msg_err("cannot read from server pipe: %s; command = %s", strerror(errno),
+ rspamd_srv_command_to_string(rd->cmd.type));
+ goto cleanup;
+ }
+
+ if (r != (gint) sizeof(rd->rep)) {
+ msg_err("cannot read from server pipe, invalid length: %d != %d; command = %s",
+ (gint) r, (int) sizeof(rd->rep), rspamd_srv_command_to_string(rd->cmd.type));
+ goto cleanup;
+ }
+
+ if (msg.msg_controllen >= CMSG_LEN(sizeof(int))) {
+ rfd = *(int *) CMSG_DATA(CMSG_FIRSTHDR(&msg));
+ }
+
+ /* Reply has been received */
+ if (rd->handler) {
+ rd->handler(rd->worker, &rd->rep, rfd, rd->ud);
+ }
+
+ goto cleanup;
+ }
+
+ return;
+
+
+cleanup:
+ ev_io_stop(EV_A_ w);
+ g_free(rd);
+}
+
+void rspamd_srv_send_command(struct rspamd_worker *worker,
+ struct ev_loop *ev_base,
+ struct rspamd_srv_command *cmd,
+ gint attached_fd,
+ rspamd_srv_reply_handler handler,
+ gpointer ud)
+{
+ struct rspamd_srv_request_data *rd;
+
+ g_assert(cmd != NULL);
+ g_assert(worker != NULL);
+
+ rd = g_malloc0(sizeof(*rd));
+ cmd->id = ottery_rand_uint64();
+ memcpy(&rd->cmd, cmd, sizeof(rd->cmd));
+ rd->handler = handler;
+ rd->ud = ud;
+ rd->worker = worker;
+ rd->rep.id = cmd->id;
+ rd->rep.type = cmd->type;
+ rd->attached_fd = attached_fd;
+
+ rd->io_ev.data = rd;
+ ev_io_init(&rd->io_ev, rspamd_srv_request_handler,
+ rd->worker->srv_pipe[1], EV_WRITE);
+ ev_io_start(ev_base, &rd->io_ev);
+}
+
+enum rspamd_control_type
+rspamd_control_command_from_string(const gchar *str)
+{
+ enum rspamd_control_type ret = RSPAMD_CONTROL_MAX;
+
+ if (!str) {
+ return ret;
+ }
+
+ if (g_ascii_strcasecmp(str, "hyperscan_loaded") == 0) {
+ ret = RSPAMD_CONTROL_HYPERSCAN_LOADED;
+ }
+ else if (g_ascii_strcasecmp(str, "stat") == 0) {
+ ret = RSPAMD_CONTROL_STAT;
+ }
+ else if (g_ascii_strcasecmp(str, "reload") == 0) {
+ ret = RSPAMD_CONTROL_RELOAD;
+ }
+ else if (g_ascii_strcasecmp(str, "reresolve") == 0) {
+ ret = RSPAMD_CONTROL_RERESOLVE;
+ }
+ else if (g_ascii_strcasecmp(str, "recompile") == 0) {
+ ret = RSPAMD_CONTROL_RECOMPILE;
+ }
+ else if (g_ascii_strcasecmp(str, "log_pipe") == 0) {
+ ret = RSPAMD_CONTROL_LOG_PIPE;
+ }
+ else if (g_ascii_strcasecmp(str, "fuzzy_stat") == 0) {
+ ret = RSPAMD_CONTROL_FUZZY_STAT;
+ }
+ else if (g_ascii_strcasecmp(str, "fuzzy_sync") == 0) {
+ ret = RSPAMD_CONTROL_FUZZY_SYNC;
+ }
+ else if (g_ascii_strcasecmp(str, "monitored_change") == 0) {
+ ret = RSPAMD_CONTROL_MONITORED_CHANGE;
+ }
+ else if (g_ascii_strcasecmp(str, "child_change") == 0) {
+ ret = RSPAMD_CONTROL_CHILD_CHANGE;
+ }
+
+ return ret;
+}
+
+const gchar *
+rspamd_control_command_to_string(enum rspamd_control_type cmd)
+{
+ const gchar *reply = "unknown";
+
+ switch (cmd) {
+ case RSPAMD_CONTROL_STAT:
+ reply = "stat";
+ break;
+ case RSPAMD_CONTROL_RELOAD:
+ reply = "reload";
+ break;
+ case RSPAMD_CONTROL_RERESOLVE:
+ reply = "reresolve";
+ break;
+ case RSPAMD_CONTROL_RECOMPILE:
+ reply = "recompile";
+ break;
+ case RSPAMD_CONTROL_HYPERSCAN_LOADED:
+ reply = "hyperscan_loaded";
+ break;
+ case RSPAMD_CONTROL_LOG_PIPE:
+ reply = "log_pipe";
+ break;
+ case RSPAMD_CONTROL_FUZZY_STAT:
+ reply = "fuzzy_stat";
+ break;
+ case RSPAMD_CONTROL_FUZZY_SYNC:
+ reply = "fuzzy_sync";
+ break;
+ case RSPAMD_CONTROL_MONITORED_CHANGE:
+ reply = "monitored_change";
+ break;
+ case RSPAMD_CONTROL_CHILD_CHANGE:
+ reply = "child_change";
+ break;
+ default:
+ break;
+ }
+
+ return reply;
+}
+
+const gchar *rspamd_srv_command_to_string(enum rspamd_srv_type cmd)
+{
+ const gchar *reply = "unknown";
+
+ switch (cmd) {
+ case RSPAMD_SRV_SOCKETPAIR:
+ reply = "socketpair";
+ break;
+ case RSPAMD_SRV_HYPERSCAN_LOADED:
+ reply = "hyperscan_loaded";
+ break;
+ case RSPAMD_SRV_MONITORED_CHANGE:
+ reply = "monitored_change";
+ break;
+ case RSPAMD_SRV_LOG_PIPE:
+ reply = "log_pipe";
+ break;
+ case RSPAMD_SRV_ON_FORK:
+ reply = "on_fork";
+ break;
+ case RSPAMD_SRV_HEARTBEAT:
+ reply = "heartbeat";
+ break;
+ case RSPAMD_SRV_HEALTH:
+ reply = "health";
+ break;
+ case RSPAMD_SRV_NOTICE_HYPERSCAN_CACHE:
+ reply = "notice_hyperscan_cache";
+ break;
+ case RSPAMD_SRV_FUZZY_BLOCKED:
+ reply = "fuzzy_blocked";
+ break;
+ }
+
+ return reply;
+}
diff --git a/src/libserver/rspamd_control.h b/src/libserver/rspamd_control.h
new file mode 100644
index 0000000..c3c861f
--- /dev/null
+++ b/src/libserver/rspamd_control.h
@@ -0,0 +1,328 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_RSPAMD_CONTROL_H
+#define RSPAMD_RSPAMD_CONTROL_H
+
+#include "config.h"
+#include "mem_pool.h"
+#include "contrib/libev/ev.h"
+
+G_BEGIN_DECLS
+
+struct rspamd_main;
+struct rspamd_worker;
+
+enum rspamd_control_type {
+ RSPAMD_CONTROL_STAT = 0,
+ RSPAMD_CONTROL_RELOAD,
+ RSPAMD_CONTROL_RERESOLVE,
+ RSPAMD_CONTROL_RECOMPILE,
+ RSPAMD_CONTROL_HYPERSCAN_LOADED,
+ RSPAMD_CONTROL_LOG_PIPE,
+ RSPAMD_CONTROL_FUZZY_STAT,
+ RSPAMD_CONTROL_FUZZY_SYNC,
+ RSPAMD_CONTROL_MONITORED_CHANGE,
+ RSPAMD_CONTROL_CHILD_CHANGE,
+ RSPAMD_CONTROL_FUZZY_BLOCKED,
+ RSPAMD_CONTROL_MAX
+};
+
+enum rspamd_srv_type {
+ RSPAMD_SRV_SOCKETPAIR = 0,
+ RSPAMD_SRV_HYPERSCAN_LOADED,
+ RSPAMD_SRV_MONITORED_CHANGE,
+ RSPAMD_SRV_LOG_PIPE,
+ RSPAMD_SRV_ON_FORK,
+ RSPAMD_SRV_HEARTBEAT,
+ RSPAMD_SRV_HEALTH,
+ RSPAMD_SRV_NOTICE_HYPERSCAN_CACHE,
+ RSPAMD_SRV_FUZZY_BLOCKED, /* Used to notify main process about a blocked ip */
+};
+
+enum rspamd_log_pipe_type {
+ RSPAMD_LOG_PIPE_SYMBOLS = 0,
+};
+#define CONTROL_PATHLEN MIN(PATH_MAX, PIPE_BUF - sizeof(int) * 2 - sizeof(gint64) * 2)
+struct rspamd_control_command {
+ enum rspamd_control_type type;
+ union {
+ struct {
+ guint unused;
+ } stat;
+ struct {
+ guint unused;
+ } reload;
+ struct {
+ guint unused;
+ } reresolve;
+ struct {
+ guint unused;
+ } recompile;
+ struct {
+ gboolean forced;
+ gchar cache_dir[CONTROL_PATHLEN];
+ } hs_loaded;
+ struct {
+ gchar tag[32];
+ gboolean alive;
+ pid_t sender;
+ } monitored_change;
+ struct {
+ enum rspamd_log_pipe_type type;
+ } log_pipe;
+ struct {
+ guint unused;
+ } fuzzy_stat;
+ struct {
+ guint unused;
+ } fuzzy_sync;
+ struct {
+ enum {
+ rspamd_child_offline,
+ rspamd_child_online,
+ rspamd_child_terminated,
+ } what;
+ pid_t pid;
+ guint additional;
+ } child_change;
+ struct {
+ union {
+ struct sockaddr sa;
+ struct sockaddr_in s4;
+ struct sockaddr_in6 s6;
+ } addr;
+ sa_family_t af;
+ } fuzzy_blocked;
+ } cmd;
+};
+
+struct rspamd_control_reply {
+ enum rspamd_control_type type;
+ union {
+ struct {
+ guint conns;
+ gdouble uptime;
+ gdouble utime;
+ gdouble systime;
+ gulong maxrss;
+ } stat;
+ struct {
+ guint status;
+ } reload;
+ struct {
+ guint status;
+ } reresolve;
+ struct {
+ guint status;
+ } recompile;
+ struct {
+ guint status;
+ } hs_loaded;
+ struct {
+ guint status;
+ } monitored_change;
+ struct {
+ guint status;
+ } log_pipe;
+ struct {
+ guint status;
+ gchar storage_id[MEMPOOL_UID_LEN];
+ } fuzzy_stat;
+ struct {
+ guint status;
+ } fuzzy_sync;
+ struct {
+ guint status;
+ } fuzzy_blocked;
+ } reply;
+};
+
+#define PAIR_ID_LEN 16
+
+struct rspamd_srv_command {
+ enum rspamd_srv_type type;
+ guint64 id;
+ union {
+ struct {
+ gint af;
+ gchar pair_id[PAIR_ID_LEN];
+ guint pair_num;
+ } spair;
+ struct {
+ gboolean forced;
+ gchar cache_dir[CONTROL_PATHLEN];
+ } hs_loaded;
+ struct {
+ gchar tag[32];
+ gboolean alive;
+ pid_t sender;
+ } monitored_change;
+ struct {
+ enum rspamd_log_pipe_type type;
+ } log_pipe;
+ struct {
+ pid_t ppid;
+ pid_t cpid;
+ enum {
+ child_create = 0,
+ child_dead,
+ } state;
+ } on_fork;
+ struct {
+ guint status;
+ /* TODO: add more fields */
+ } heartbeat;
+ struct {
+ guint status;
+ } health;
+ /* Used when a worker loads a valid hyperscan file */
+ struct {
+ char path[CONTROL_PATHLEN];
+ } hyperscan_cache_file;
+ /* Send when one worker has blocked some IP address */
+ struct {
+ union {
+ struct sockaddr sa;
+ struct sockaddr_in s4;
+ struct sockaddr_in6 s6;
+ } addr;
+ sa_family_t af;
+ } fuzzy_blocked;
+ } cmd;
+};
+
+struct rspamd_srv_reply {
+ enum rspamd_srv_type type;
+ guint64 id;
+ union {
+ struct {
+ gint code;
+ } spair;
+ struct {
+ gint forced;
+ } hs_loaded;
+ struct {
+ gint status;
+ };
+ struct {
+ enum rspamd_log_pipe_type type;
+ } log_pipe;
+ struct {
+ gint status;
+ } on_fork;
+ struct {
+ gint status;
+ } heartbeat;
+ struct {
+ guint status;
+ guint workers_count;
+ guint scanners_count;
+ guint workers_hb_lost;
+ } health;
+ struct {
+ int unused;
+ } hyperscan_cache_file;
+ struct {
+ int unused;
+ } fuzzy_blocked;
+ } reply;
+};
+
+typedef gboolean (*rspamd_worker_control_handler)(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker,
+ gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud);
+
+typedef void (*rspamd_srv_reply_handler)(struct rspamd_worker *worker,
+ struct rspamd_srv_reply *rep, gint rep_fd,
+ gpointer ud);
+
+/**
+ * Process client socket connection
+ */
+void rspamd_control_process_client_socket(struct rspamd_main *rspamd_main,
+ gint fd, rspamd_inet_addr_t *addr);
+
+/**
+ * Register default handlers for a worker
+ */
+void rspamd_control_worker_add_default_cmd_handlers(struct rspamd_worker *worker,
+ struct ev_loop *ev_base);
+
+/**
+ * Register custom handler for a specific control command for this worker
+ */
+void rspamd_control_worker_add_cmd_handler(struct rspamd_worker *worker,
+ enum rspamd_control_type type,
+ rspamd_worker_control_handler handler,
+ gpointer ud);
+
+/**
+ * Start watching on srv pipe
+ */
+void rspamd_srv_start_watching(struct rspamd_main *srv,
+ struct rspamd_worker *worker,
+ struct ev_loop *ev_base);
+
+
+/**
+ * Send command to srv pipe and read reply calling the specified callback at the
+ * end
+ */
+void rspamd_srv_send_command(struct rspamd_worker *worker,
+ struct ev_loop *ev_base,
+ struct rspamd_srv_command *cmd,
+ gint attached_fd,
+ rspamd_srv_reply_handler handler,
+ gpointer ud);
+
+/**
+ * Broadcast srv cmd from rspamd_main to workers
+ * @param rspamd_main
+ * @param cmd
+ * @param except_pid
+ */
+void rspamd_control_broadcast_srv_cmd(struct rspamd_main *rspamd_main,
+ struct rspamd_control_command *cmd,
+ pid_t except_pid);
+
+/**
+ * Returns command from a specified string (case insensitive)
+ * @param str
+ * @return
+ */
+enum rspamd_control_type rspamd_control_command_from_string(const gchar *str);
+
+/**
+ * Returns command name from it's type
+ * @param cmd
+ * @return
+ */
+const gchar *rspamd_control_command_to_string(enum rspamd_control_type cmd);
+
+const gchar *rspamd_srv_command_to_string(enum rspamd_srv_type cmd);
+
+/**
+ * Used to cleanup pending events
+ * @param p
+ */
+void rspamd_pending_control_free(gpointer p);
+
+G_END_DECLS
+
+#endif
diff --git a/src/libserver/rspamd_symcache.h b/src/libserver/rspamd_symcache.h
new file mode 100644
index 0000000..2c67cba
--- /dev/null
+++ b/src/libserver/rspamd_symcache.h
@@ -0,0 +1,578 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_SYMBOLS_CACHE_H
+#define RSPAMD_SYMBOLS_CACHE_H
+
+#include "config.h"
+#include "ucl.h"
+#include "cfg_file.h"
+#include "contrib/libev/ev.h"
+
+#include <lua.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_config;
+struct rspamd_symcache;
+struct rspamd_worker;
+struct rspamd_symcache_dynamic_item;
+struct rspamd_symcache_item;
+struct rspamd_config_settings_elt;
+
+typedef void (*symbol_func_t)(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ gpointer user_data);
+
+enum rspamd_symbol_type {
+ SYMBOL_TYPE_NORMAL = (1u << 0u),
+ SYMBOL_TYPE_VIRTUAL = (1u << 1u),
+ SYMBOL_TYPE_CALLBACK = (1u << 2u),
+ SYMBOL_TYPE_GHOST = (1u << 3u),
+ SYMBOL_TYPE_SKIPPED = (1u << 4u),
+ SYMBOL_TYPE_COMPOSITE = (1u << 5u),
+ SYMBOL_TYPE_CLASSIFIER = (1u << 6u),
+ SYMBOL_TYPE_FINE = (1u << 7u),
+ SYMBOL_TYPE_EMPTY = (1u << 8u), /* Allow execution on empty tasks */
+ SYMBOL_TYPE_CONNFILTER = (1u << 9u), /* Connection stage filter */
+ SYMBOL_TYPE_PREFILTER = (1u << 10u),
+ SYMBOL_TYPE_POSTFILTER = (1u << 11u),
+ SYMBOL_TYPE_NOSTAT = (1u << 12u), /* Skip as statistical symbol */
+ SYMBOL_TYPE_IDEMPOTENT = (1u << 13u), /* Symbol cannot change metric */
+ SYMBOL_TYPE_TRIVIAL = (1u << 14u), /* Symbol is trivial */
+ SYMBOL_TYPE_MIME_ONLY = (1u << 15u), /* Symbol is mime only */
+ SYMBOL_TYPE_EXPLICIT_DISABLE = (1u << 16u), /* Symbol should be disabled explicitly only */
+ SYMBOL_TYPE_IGNORE_PASSTHROUGH = (1u << 17u), /* Symbol ignores passthrough result */
+ SYMBOL_TYPE_EXPLICIT_ENABLE = (1u << 18u), /* Symbol should be enabled explicitly only */
+ SYMBOL_TYPE_USE_CORO = (1u << 19u), /* Symbol uses lua coroutines */
+};
+
+/**
+ * Abstract structure for saving callback data for symbols
+ */
+struct rspamd_abstract_callback_data {
+ guint64 magic;
+ char data[];
+};
+
+/**
+ * Shared memory block specific for each symbol
+ */
+struct rspamd_symcache_item_stat {
+ struct rspamd_counter_data time_counter;
+ gdouble avg_time;
+ gdouble weight;
+ guint hits;
+ guint64 total_hits;
+ struct rspamd_counter_data frequency_counter;
+ gdouble avg_frequency;
+ gdouble stddev_frequency;
+};
+
+/**
+ * Creates new cache structure
+ * @return
+ */
+struct rspamd_symcache *rspamd_symcache_new(struct rspamd_config *cfg);
+
+/**
+ * Remove the cache structure syncing data if needed
+ * @param cache
+ */
+void rspamd_symcache_destroy(struct rspamd_symcache *cache);
+
+/**
+ * Saves symbols cache to disk if possible
+ * @param cache
+ */
+void rspamd_symcache_save(struct rspamd_symcache *cache);
+
+/**
+ * Load symbols cache from file, must be called _after_ init_symbols_cache
+ */
+gboolean rspamd_symcache_init(struct rspamd_symcache *cache);
+
+/**
+ * Generic function to register a symbol
+ * @param cache
+ * @param name
+ * @param weight
+ * @param priority
+ * @param func
+ * @param user_data
+ * @param type
+ * @param parent
+ */
+gint rspamd_symcache_add_symbol(struct rspamd_symcache *cache,
+ const gchar *name,
+ gint priority,
+ symbol_func_t func,
+ gpointer user_data,
+ int type,
+ gint parent);
+
+/**
+ * Adds augmentation to the symbol
+ * @param cache
+ * @param sym_id
+ * @param augmentation
+ * @return
+ */
+bool rspamd_symcache_add_symbol_augmentation(struct rspamd_symcache *cache,
+ int sym_id,
+ const char *augmentation,
+ const char *value);
+
+/**
+ * Add callback to be executed whenever symbol has peak value
+ * @param cache
+ * @param cbref
+ */
+void rspamd_symcache_set_peak_callback(struct rspamd_symcache *cache,
+ gint cbref);
+
+/**
+ * Add delayed condition to the specific symbol in cache. So symbol can be absent
+ * to the moment of addition
+ * @param cache
+ * @param id id of symbol
+ * @param L lua state pointer
+ * @param cbref callback reference (returned by luaL_ref)
+ * @return TRUE if condition has been added
+ */
+gboolean rspamd_symcache_add_condition_delayed(struct rspamd_symcache *cache,
+ const gchar *sym,
+ lua_State *L, gint cbref);
+
+/**
+ * Find symbol in cache by id and returns its id resolving virtual symbols if
+ * applicable
+ * @param cache
+ * @param name
+ * @return id of symbol or (-1) if a symbol has not been found
+ */
+gint rspamd_symcache_find_symbol(struct rspamd_symcache *cache,
+ const gchar *name);
+
+/**
+ * Get statistics for a specific symbol
+ * @param cache
+ * @param name
+ * @param frequency
+ * @param tm
+ * @return
+ */
+gboolean rspamd_symcache_stat_symbol(struct rspamd_symcache *cache,
+ const gchar *name,
+ gdouble *frequency,
+ gdouble *freq_stddev,
+ gdouble *tm,
+ guint *nhits);
+
+/**
+ * Returns number of symbols registered in symbols cache
+ * @param cache
+ * @return number of symbols in the cache
+ */
+guint rspamd_symcache_stats_symbols_count(struct rspamd_symcache *cache);
+
+/**
+ * Validate cache items against theirs weights defined in metrics
+ * @param cache symbols cache
+ * @param cfg configuration
+ * @param strict do strict checks - symbols MUST be described in metrics
+ */
+gboolean rspamd_symcache_validate(struct rspamd_symcache *cache,
+ struct rspamd_config *cfg,
+ gboolean strict);
+
+/**
+ * Call function for cached symbol using saved callback
+ * @param task task object
+ * @param cache symbols cache
+ * @param saved_item pointer to currently saved item
+ */
+gboolean rspamd_symcache_process_symbols(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ guint stage);
+
+/**
+ * Return statistics about the cache as ucl object (array of objects one per item)
+ * @param cache
+ * @return
+ */
+ucl_object_t *rspamd_symcache_counters(struct rspamd_symcache *cache);
+
+/**
+ * Start cache reloading
+ * @param cache
+ * @param ev_base
+ */
+void *rspamd_symcache_start_refresh(struct rspamd_symcache *cache,
+ struct ev_loop *ev_base,
+ struct rspamd_worker *w);
+
+/**
+ * Increases counter for a specific symbol
+ * @param cache
+ * @param symbol
+ */
+void rspamd_symcache_inc_frequency(struct rspamd_symcache *_cache,
+ struct rspamd_symcache_item *item,
+ const gchar *sym_name);
+
+/**
+ * Add delayed dependency that is resolved on cache post-load routine
+ * @param cache
+ * @param from
+ * @param to
+ */
+void rspamd_symcache_add_delayed_dependency(struct rspamd_symcache *cache,
+ const gchar *from, const gchar *to);
+
+/**
+ * Get abstract callback data for a symbol (or its parent symbol)
+ * @param cache cache object
+ * @param symbol symbol name
+ * @return abstract callback data or NULL if symbol is absent or has no data attached
+ */
+struct rspamd_abstract_callback_data *rspamd_symcache_get_cbdata(
+ struct rspamd_symcache *cache, const gchar *symbol);
+
+/**
+ * Returns symbol's parent name (or symbol name itself)
+ * @param cache
+ * @param symbol
+ * @return
+ */
+const gchar *rspamd_symcache_get_parent(struct rspamd_symcache *cache,
+ const gchar *symbol);
+
+guint rspamd_symcache_get_symbol_flags(struct rspamd_symcache *cache,
+ const gchar *symbol);
+
+void rspamd_symcache_get_symbol_details(struct rspamd_symcache *cache,
+ const gchar *symbol,
+ ucl_object_t *this_sym_ucl);
+
+
+/**
+ * Process settings for task
+ * @param task
+ * @param cache
+ * @return
+ */
+gboolean rspamd_symcache_process_settings(struct rspamd_task *task,
+ struct rspamd_symcache *cache);
+
+
+/**
+ * Checks if a symbol specified has been checked (or disabled)
+ * @param task
+ * @param cache
+ * @param symbol
+ * @return
+ */
+gboolean rspamd_symcache_is_checked(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ const gchar *symbol);
+
+/**
+ * Returns checksum for all cache items
+ * @param cache
+ * @return
+ */
+guint64 rspamd_symcache_get_cksum(struct rspamd_symcache *cache);
+
+/**
+ * Checks if a symbols is enabled (not checked and conditions return true if present)
+ * @param task
+ * @param cache
+ * @param symbol
+ * @return
+ */
+gboolean rspamd_symcache_is_symbol_enabled(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ const gchar *symbol);
+
+/**
+ * Enable this symbol for task
+ * @param task
+ * @param cache
+ * @param symbol
+ * @return TRUE if a symbol has been enabled (not executed before)
+ */
+gboolean rspamd_symcache_enable_symbol(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ const gchar *symbol);
+
+/**
+ * Enable this symbol for task
+ * @param task
+ * @param cache
+ * @param symbol
+ * @return TRUE if a symbol has been disabled (not executed before)
+ */
+gboolean rspamd_symcache_disable_symbol(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ const gchar *symbol);
+
+/**
+ * Disable execution of a symbol or a pattern (a string enclosed in `//`) permanently
+ * @param task
+ * @param cache
+ * @param symbol
+ * @return
+ */
+void rspamd_symcache_disable_symbol_static(struct rspamd_symcache *cache,
+ const gchar *symbol);
+/**
+ * Add a symbol or a pattern to the list of explicitly and statically enabled symbols
+ * @param cache
+ * @param symbol
+ * @return
+ */
+void rspamd_symcache_enable_symbol_static(struct rspamd_symcache *cache,
+ const gchar *symbol);
+
+/**
+ * Process specific function for each cache element (in order they are added)
+ * @param cache
+ * @param func
+ * @param ud
+ */
+void rspamd_symcache_foreach(struct rspamd_symcache *cache,
+ void (*func)(struct rspamd_symcache_item *item, gpointer /* userdata */),
+ gpointer ud);
+
+/**
+ * Returns the current item being processed (if any)
+ * @param task
+ * @return
+ */
+struct rspamd_symcache_dynamic_item *rspamd_symcache_get_cur_item(struct rspamd_task *task);
+
+/**
+ * Replaces the current item being processed.
+ * Returns the current item being processed (if any)
+ * @param task
+ * @param item
+ * @return
+ */
+struct rspamd_symcache_dynamic_item *rspamd_symcache_set_cur_item(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item);
+
+
+/**
+ * Finalize the current async element potentially calling its deps
+ */
+void rspamd_symcache_finalize_item(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item);
+
+/*
+ * Increase number of async events pending for an item
+ */
+guint rspamd_symcache_item_async_inc_full(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ const gchar *subsystem,
+ const gchar *loc);
+
+#define rspamd_symcache_item_async_inc(task, item, subsystem) \
+ rspamd_symcache_item_async_inc_full(task, item, subsystem, G_STRLOC)
+
+/*
+ * Decrease number of async events pending for an item, asserts if no events pending
+ */
+guint rspamd_symcache_item_async_dec_full(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ const gchar *subsystem,
+ const gchar *loc);
+
+#define rspamd_symcache_item_async_dec(task, item, subsystem) \
+ rspamd_symcache_item_async_dec_full(task, item, subsystem, G_STRLOC)
+
+/**
+ * Decrease number of async events pending for an item, asserts if no events pending
+ * If no events are left, this function calls `rspamd_symbols_cache_finalize_item` and returns TRUE
+ * @param task
+ * @param item
+ * @return
+ */
+gboolean rspamd_symcache_item_async_dec_check_full(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ const gchar *subsystem,
+ const gchar *loc);
+
+#define rspamd_symcache_item_async_dec_check(task, item, subsystem) \
+ rspamd_symcache_item_async_dec_check_full(task, item, subsystem, G_STRLOC)
+
+/**
+ * Disables execution of all symbols, excluding those specified in `skip_mask`
+ * @param task
+ * @param cache
+ * @param skip_mask
+ */
+void rspamd_symcache_disable_all_symbols(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ guint skip_mask);
+
+/**
+ * Iterates over the list of the enabled composites calling specified function
+ * @param task
+ * @param cache
+ * @param func
+ * @param fd
+ */
+void rspamd_symcache_composites_foreach(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ GHFunc func,
+ gpointer fd);
+
+/**
+ * Sets allowed settings ids for a symbol
+ * @param cache
+ * @param symbol
+ * @param ids
+ * @param nids
+ */
+bool rspamd_symcache_set_allowed_settings_ids(struct rspamd_symcache *cache,
+ const gchar *symbol,
+ const guint32 *ids,
+ guint nids);
+/**
+ * Sets denied settings ids for a symbol
+ * @param cache
+ * @param symbol
+ * @param ids
+ * @param nids
+ */
+bool rspamd_symcache_set_forbidden_settings_ids(struct rspamd_symcache *cache,
+ const gchar *symbol,
+ const guint32 *ids,
+ guint nids);
+
+/**
+ * Returns allowed ids for a symbol as a constant array
+ * @param cache
+ * @param symbol
+ * @param nids
+ * @return
+ */
+const guint32 *rspamd_symcache_get_allowed_settings_ids(struct rspamd_symcache *cache,
+ const gchar *symbol,
+ guint *nids);
+
+/**
+ * Returns denied ids for a symbol as a constant array
+ * @param cache
+ * @param symbol
+ * @param nids
+ * @return
+ */
+const guint32 *rspamd_symcache_get_forbidden_settings_ids(struct rspamd_symcache *cache,
+ const gchar *symbol,
+ guint *nids);
+
+
+/**
+ * Processes settings_elt in cache and converts it to a set of
+ * adjustments for forbidden/allowed settings_ids for each symbol
+ * @param cache
+ * @param elt
+ */
+void rspamd_symcache_process_settings_elt(struct rspamd_symcache *cache,
+ struct rspamd_config_settings_elt *elt);
+
+/**
+ * Check if a symbol is allowed for execution/insertion, this does not involve
+ * condition scripts to be checked (so it is intended to be fast).
+ * @param task
+ * @param item
+ * @param exec_only
+ * @return
+ */
+gboolean rspamd_symcache_is_item_allowed(struct rspamd_task *task,
+ struct rspamd_symcache_item *item,
+ gboolean exec_only);
+
+/**
+ * Returns symcache item flags
+ * @param item
+ * @return
+ */
+gint rspamd_symcache_dyn_item_flags(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *dyn_item);
+gint rspamd_symcache_item_flags(struct rspamd_symcache_item *item);
+
+/**
+ * Returns cache item name
+ * @param item
+ * @return
+ */
+const gchar *rspamd_symcache_dyn_item_name(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *dyn_item);
+const gchar *rspamd_symcache_item_name(struct rspamd_symcache_item *item);
+
+/**
+ * Returns the current item stat
+ * @param item
+ * @return
+ */
+const struct rspamd_symcache_item_stat *
+rspamd_symcache_item_stat(struct rspamd_symcache_item *item);
+
+/**
+ * Enable profiling for task (e.g. when a slow rule has been found)
+ * @param task
+ */
+void rspamd_symcache_enable_profile(struct rspamd_task *task);
+
+struct rspamd_symcache_timeout_item {
+ double timeout;
+ const struct rspamd_symcache_item *item;
+};
+
+struct rspamd_symcache_timeout_result {
+ double max_timeout;
+ struct rspamd_symcache_timeout_item *items;
+ size_t nitems;
+};
+/**
+ * Gets maximum timeout announced by symbols cache
+ * @param cache
+ * @return new symcache timeout_result structure, that should be freed by call
+ * `rspamd_symcache_timeout_result_free`
+ */
+struct rspamd_symcache_timeout_result *rspamd_symcache_get_max_timeout(struct rspamd_symcache *cache);
+
+/**
+ * Frees results obtained from the previous function
+ * @param res
+ */
+void rspamd_symcache_timeout_result_free(struct rspamd_symcache_timeout_result *res);
+
+/**
+ * Destroy internal state of the symcache runtime
+ * @param task
+ */
+void rspamd_symcache_runtime_destroy(struct rspamd_task *task);
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/spf.c b/src/libserver/spf.c
new file mode 100644
index 0000000..72d8b99
--- /dev/null
+++ b/src/libserver/spf.c
@@ -0,0 +1,2799 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "dns.h"
+#include "spf.h"
+#include "rspamd.h"
+#include "message.h"
+#include "utlist.h"
+#include "libserver/mempool_vars_internal.h"
+#include "contrib/librdns/rdns.h"
+#include "contrib/mumhash/mum.h"
+
+#define SPF_VER1_STR "v=spf1"
+#define SPF_VER2_STR "spf2."
+#define SPF_SCOPE_PRA "pra"
+#define SPF_SCOPE_MFROM "mfrom"
+#define SPF_ALL "all"
+#define SPF_A "a"
+#define SPF_IP4 "ip4"
+#define SPF_IP4_ALT "ipv4"
+#define SPF_IP6 "ip6"
+#define SPF_IP6_ALT "ipv6"
+#define SPF_PTR "ptr"
+#define SPF_MX "mx"
+#define SPF_EXISTS "exists"
+#define SPF_INCLUDE "include"
+#define SPF_REDIRECT "redirect"
+#define SPF_EXP "exp"
+
+struct spf_resolved_element {
+ GPtrArray *elts;
+ gchar *cur_domain;
+ gboolean redirected; /* Ignore level, it's redirected */
+};
+
+struct spf_record {
+ gint nested;
+ gint dns_requests;
+ gint requests_inflight;
+
+ guint ttl;
+ GPtrArray *resolved;
+ /* Array of struct spf_resolved_element */
+ const gchar *sender;
+ const gchar *sender_domain;
+ const gchar *top_record;
+ gchar *local_part;
+ struct rspamd_task *task;
+ spf_cb_t callback;
+ gpointer cbdata;
+ gboolean done;
+};
+
+struct rspamd_spf_library_ctx {
+ guint max_dns_nesting;
+ guint max_dns_requests;
+ guint min_cache_ttl;
+ gboolean disable_ipv6;
+ rspamd_lru_hash_t *spf_hash;
+};
+
+struct rspamd_spf_library_ctx *spf_lib_ctx = NULL;
+
+/**
+ * BNF for SPF record:
+ *
+ * spf_mech ::= +|-|~|?
+ *
+ * spf_body ::= spf=v1 <spf_command> [<spf_command>]
+ * spf_command ::= [spf_mech]all|a|<ip4>|<ip6>|ptr|mx|<exists>|<include>|<redirect>
+ *
+ * spf_domain ::= [:domain][/mask]
+ * spf_ip4 ::= ip[/mask]
+ * ip4 ::= ip4:<spf_ip4>
+ * mx ::= mx<spf_domain>
+ * a ::= a<spf_domain>
+ * ptr ::= ptr[:domain]
+ * exists ::= exists:domain
+ * include ::= include:domain
+ * redirect ::= redirect:domain
+ * exp ::= exp:domain
+ *
+ */
+
+#undef SPF_DEBUG
+
+#define msg_err_spf(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "spf", rec->task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_spf(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "spf", rec->task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_spf(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "spf", rec->task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_notice_spf(...) rspamd_default_log_function(G_LOG_LEVEL_MESSAGE, \
+ "spf", rec->task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_spf(...) rspamd_conditional_debug_fast(NULL, rec->task->from_addr, \
+ rspamd_spf_log_id, "spf", rec->task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_spf_flatten(...) rspamd_conditional_debug_fast_num_id(NULL, NULL, \
+ rspamd_spf_log_id, "spf", (flat)->digest, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(spf)
+
+struct spf_dns_cb {
+ struct spf_record *rec;
+ struct spf_addr *addr;
+ struct spf_resolved_element *resolved;
+ const gchar *ptr_host;
+ spf_action_t cur_action;
+ gboolean in_include;
+};
+
+#define CHECK_REC(rec) \
+ do { \
+ if (spf_lib_ctx->max_dns_nesting > 0 && \
+ (rec)->nested > spf_lib_ctx->max_dns_nesting) { \
+ msg_warn_spf("spf nesting limit: %d > %d is reached, domain: %s", \
+ (rec)->nested, spf_lib_ctx->max_dns_nesting, \
+ (rec)->sender_domain); \
+ return FALSE; \
+ } \
+ if (spf_lib_ctx->max_dns_requests > 0 && \
+ (rec)->dns_requests > spf_lib_ctx->max_dns_requests) { \
+ msg_warn_spf("spf dns requests limit: %d > %d is reached, domain: %s", \
+ (rec)->dns_requests, spf_lib_ctx->max_dns_requests, \
+ (rec)->sender_domain); \
+ return FALSE; \
+ } \
+ } while (0)
+
+RSPAMD_CONSTRUCTOR(rspamd_spf_lib_ctx_ctor)
+{
+ spf_lib_ctx = g_malloc0(sizeof(*spf_lib_ctx));
+ spf_lib_ctx->max_dns_nesting = SPF_MAX_NESTING;
+ spf_lib_ctx->max_dns_requests = SPF_MAX_DNS_REQUESTS;
+ spf_lib_ctx->min_cache_ttl = SPF_MIN_CACHE_TTL;
+ spf_lib_ctx->disable_ipv6 = FALSE;
+}
+
+RSPAMD_DESTRUCTOR(rspamd_spf_lib_ctx_dtor)
+{
+ if (spf_lib_ctx->spf_hash) {
+ rspamd_lru_hash_destroy(spf_lib_ctx->spf_hash);
+ }
+ g_free(spf_lib_ctx);
+ spf_lib_ctx = NULL;
+}
+
+static void
+spf_record_cached_unref_dtor(gpointer p)
+{
+ struct spf_resolved *flat = (struct spf_resolved *) p;
+
+ _spf_record_unref(flat, "LRU cache");
+}
+
+void spf_library_config(const ucl_object_t *obj)
+{
+ const ucl_object_t *value;
+ gint64 ival;
+ bool bval;
+
+ if (obj == NULL) {
+ /* No specific config */
+ return;
+ }
+
+ if ((value = ucl_object_find_key(obj, "min_cache_ttl")) != NULL) {
+ if (ucl_object_toint_safe(value, &ival) && ival >= 0) {
+ spf_lib_ctx->min_cache_ttl = ival;
+ }
+ }
+
+ if ((value = ucl_object_find_key(obj, "max_dns_nesting")) != NULL) {
+ if (ucl_object_toint_safe(value, &ival) && ival >= 0) {
+ spf_lib_ctx->max_dns_nesting = ival;
+ }
+ }
+
+ if ((value = ucl_object_find_key(obj, "max_dns_requests")) != NULL) {
+ if (ucl_object_toint_safe(value, &ival) && ival >= 0) {
+ spf_lib_ctx->max_dns_requests = ival;
+ }
+ }
+ if ((value = ucl_object_find_key(obj, "disable_ipv6")) != NULL) {
+ if (ucl_object_toboolean_safe(value, &bval)) {
+ spf_lib_ctx->disable_ipv6 = bval;
+ }
+ }
+
+ if (spf_lib_ctx->spf_hash) {
+ rspamd_lru_hash_destroy(spf_lib_ctx->spf_hash);
+ spf_lib_ctx->spf_hash = NULL;
+ }
+
+ if ((value = ucl_object_find_key(obj, "spf_cache_size")) != NULL) {
+ if (ucl_object_toint_safe(value, &ival) && ival > 0) {
+ spf_lib_ctx->spf_hash = rspamd_lru_hash_new(
+ ival,
+ g_free,
+ spf_record_cached_unref_dtor);
+ }
+ }
+ else {
+ /* Preserve compatibility */
+ spf_lib_ctx->spf_hash = rspamd_lru_hash_new(
+ 2048,
+ g_free,
+ spf_record_cached_unref_dtor);
+ }
+}
+
+static gboolean start_spf_parse(struct spf_record *rec,
+ struct spf_resolved_element *resolved, gchar *begin);
+
+/* Determine spf mech */
+static spf_mech_t
+check_spf_mech(const gchar *elt, gboolean *need_shift)
+{
+ g_assert(elt != NULL);
+
+ *need_shift = TRUE;
+
+ switch (*elt) {
+ case '-':
+ return SPF_FAIL;
+ case '~':
+ return SPF_SOFT_FAIL;
+ case '+':
+ return SPF_PASS;
+ case '?':
+ return SPF_NEUTRAL;
+ default:
+ *need_shift = FALSE;
+ return SPF_PASS;
+ }
+}
+
+static const gchar *
+rspamd_spf_dns_action_to_str(spf_action_t act)
+{
+ const char *ret = "unknown";
+
+ switch (act) {
+ case SPF_RESOLVE_MX:
+ ret = "MX";
+ break;
+ case SPF_RESOLVE_A:
+ ret = "A";
+ break;
+ case SPF_RESOLVE_PTR:
+ ret = "PTR";
+ break;
+ case SPF_RESOLVE_AAA:
+ ret = "AAAA";
+ break;
+ case SPF_RESOLVE_REDIRECT:
+ ret = "REDIRECT";
+ break;
+ case SPF_RESOLVE_INCLUDE:
+ ret = "INCLUDE";
+ break;
+ case SPF_RESOLVE_EXISTS:
+ ret = "EXISTS";
+ break;
+ case SPF_RESOLVE_EXP:
+ ret = "EXP";
+ break;
+ }
+
+ return ret;
+}
+
+static struct spf_addr *
+rspamd_spf_new_addr(struct spf_record *rec,
+ struct spf_resolved_element *resolved, const gchar *elt)
+{
+ gboolean need_shift = FALSE;
+ struct spf_addr *naddr;
+
+ naddr = g_malloc0(sizeof(*naddr));
+ naddr->mech = check_spf_mech(elt, &need_shift);
+
+ if (need_shift) {
+ naddr->spf_string = g_strdup(elt + 1);
+ }
+ else {
+ naddr->spf_string = g_strdup(elt);
+ }
+
+ g_ptr_array_add(resolved->elts, naddr);
+ naddr->prev = naddr;
+ naddr->next = NULL;
+
+ return naddr;
+}
+
+static void
+rspamd_spf_free_addr(gpointer a)
+{
+ struct spf_addr *addr = a, *tmp, *cur;
+
+ if (addr) {
+ g_free(addr->spf_string);
+ DL_FOREACH_SAFE(addr, cur, tmp)
+ {
+ g_free(cur);
+ }
+ }
+}
+
+static struct spf_resolved_element *
+rspamd_spf_new_addr_list(struct spf_record *rec, const gchar *domain)
+{
+ struct spf_resolved_element *resolved;
+
+ resolved = g_malloc0(sizeof(*resolved));
+ resolved->redirected = FALSE;
+ resolved->cur_domain = g_strdup(domain);
+ resolved->elts = g_ptr_array_new_full(8, rspamd_spf_free_addr);
+
+ g_ptr_array_add(rec->resolved, resolved);
+
+ return g_ptr_array_index(rec->resolved, rec->resolved->len - 1);
+}
+
+/*
+ * Destructor for spf record
+ */
+static void
+spf_record_destructor(gpointer r)
+{
+ struct spf_record *rec = r;
+ struct spf_resolved_element *elt;
+ guint i;
+
+ if (rec) {
+ for (i = 0; i < rec->resolved->len; i++) {
+ elt = g_ptr_array_index(rec->resolved, i);
+ g_ptr_array_free(elt->elts, TRUE);
+ g_free(elt->cur_domain);
+ g_free(elt);
+ }
+
+ g_ptr_array_free(rec->resolved, TRUE);
+ }
+}
+
+static void
+rspamd_flatten_record_dtor(struct spf_resolved *r)
+{
+ struct spf_addr *addr;
+ guint i;
+
+ for (i = 0; i < r->elts->len; i++) {
+ addr = &g_array_index(r->elts, struct spf_addr, i);
+ g_free(addr->spf_string);
+ }
+
+ g_free(r->top_record);
+ g_free(r->domain);
+ g_array_free(r->elts, TRUE);
+ g_free(r);
+}
+
+static void
+rspamd_spf_process_reference(struct spf_resolved *target,
+ struct spf_addr *addr, struct spf_record *rec, gboolean top)
+{
+ struct spf_resolved_element *elt, *relt;
+ struct spf_addr *cur = NULL, taddr, *cur_addr;
+ guint i;
+
+ if (addr) {
+ g_assert(addr->m.idx < rec->resolved->len);
+
+ elt = g_ptr_array_index(rec->resolved, addr->m.idx);
+ }
+ else {
+ elt = g_ptr_array_index(rec->resolved, 0);
+ }
+
+ if (rec->ttl < target->ttl) {
+ msg_debug_spf("reducing ttl from %d to %d after subrecord processing %s",
+ target->ttl, rec->ttl, rec->sender_domain);
+ target->ttl = rec->ttl;
+ }
+
+ if (elt->redirected) {
+ g_assert(elt->elts->len > 0);
+
+ for (i = 0; i < elt->elts->len; i++) {
+ cur = g_ptr_array_index(elt->elts, i);
+ if (cur->flags & RSPAMD_SPF_FLAG_REDIRECT) {
+ break;
+ }
+ }
+
+ g_assert(cur != NULL);
+ if (!(cur->flags & (RSPAMD_SPF_FLAG_PARSED | RSPAMD_SPF_FLAG_RESOLVED))) {
+ /* Unresolved redirect */
+ msg_info_spf("redirect to %s cannot be resolved for domain %s", cur->spf_string, rec->sender_domain);
+ }
+ else {
+ g_assert(cur->flags & RSPAMD_SPF_FLAG_REFERENCE);
+ g_assert(cur->m.idx < rec->resolved->len);
+ relt = g_ptr_array_index(rec->resolved, cur->m.idx);
+ msg_debug_spf("domain %s is redirected to %s", elt->cur_domain,
+ relt->cur_domain);
+ }
+ }
+
+ for (i = 0; i < elt->elts->len; i++) {
+ cur = g_ptr_array_index(elt->elts, i);
+
+ if (cur->flags & RSPAMD_SPF_FLAG_TEMPFAIL) {
+ target->flags |= RSPAMD_SPF_RESOLVED_TEMP_FAILED;
+ continue;
+ }
+ if (cur->flags & RSPAMD_SPF_FLAG_PERMFAIL) {
+ if (cur->flags & RSPAMD_SPF_FLAG_REDIRECT) {
+ target->flags |= RSPAMD_SPF_RESOLVED_PERM_FAILED;
+ }
+ continue;
+ }
+ if (cur->flags & RSPAMD_SPF_FLAG_NA) {
+ target->flags |= RSPAMD_SPF_RESOLVED_NA;
+ continue;
+ }
+ if (cur->flags & RSPAMD_SPF_FLAG_INVALID) {
+ /* Ignore invalid elements */
+ continue;
+ }
+ if ((cur->flags & (RSPAMD_SPF_FLAG_PARSED | RSPAMD_SPF_FLAG_RESOLVED)) !=
+ (RSPAMD_SPF_FLAG_RESOLVED | RSPAMD_SPF_FLAG_PARSED)) {
+ /* Ignore unparsed addrs */
+ continue;
+ }
+ if (cur->flags & RSPAMD_SPF_FLAG_REFERENCE) {
+ /* Process reference */
+ if (cur->flags & RSPAMD_SPF_FLAG_REDIRECT) {
+ /* Stop on redirected domain */
+ rspamd_spf_process_reference(target, cur, rec, top);
+ break;
+ }
+ else {
+ rspamd_spf_process_reference(target, cur, rec, FALSE);
+ }
+ }
+ else {
+ if ((cur->flags & RSPAMD_SPF_FLAG_ANY) && !top) {
+ /* Ignore wide policies in includes */
+ continue;
+ }
+
+ DL_FOREACH(cur, cur_addr)
+ {
+ memcpy(&taddr, cur_addr, sizeof(taddr));
+ taddr.spf_string = g_strdup(cur_addr->spf_string);
+ g_array_append_val(target->elts, taddr);
+ }
+ }
+ }
+}
+
+/*
+ * Parse record and flatten it to a simple structure
+ */
+static struct spf_resolved *
+rspamd_spf_record_flatten(struct spf_record *rec)
+{
+ struct spf_resolved *res;
+
+ g_assert(rec != NULL);
+
+ res = g_malloc0(sizeof(*res));
+ res->domain = g_strdup(rec->sender_domain);
+ res->ttl = rec->ttl;
+ /* Not precise but okay */
+ res->timestamp = rec->task->task_timestamp;
+ res->digest = mum_hash_init(0xa4aa40bbeec59e2bULL);
+ res->top_record = g_strdup(rec->top_record);
+ REF_INIT_RETAIN(res, rspamd_flatten_record_dtor);
+
+ if (rec->resolved) {
+ res->elts = g_array_sized_new(FALSE, FALSE, sizeof(struct spf_addr),
+ rec->resolved->len);
+
+ if (rec->resolved->len > 0) {
+ rspamd_spf_process_reference(res, NULL, rec, TRUE);
+ }
+ }
+ else {
+ res->elts = g_array_new(FALSE, FALSE, sizeof(struct spf_addr));
+ }
+
+ return res;
+}
+
+static gint
+rspamd_spf_elts_cmp(gconstpointer a, gconstpointer b)
+{
+ struct spf_addr *addr_a, *addr_b;
+
+ addr_a = (struct spf_addr *) a;
+ addr_b = (struct spf_addr *) b;
+
+ if (addr_a->flags == addr_b->flags) {
+ if (addr_a->flags & RSPAMD_SPF_FLAG_ANY) {
+ return 0;
+ }
+ else if (addr_a->flags & RSPAMD_SPF_FLAG_IPV4) {
+ return (addr_a->m.dual.mask_v4 - addr_b->m.dual.mask_v4) ||
+ memcmp(addr_a->addr4, addr_b->addr4, sizeof(addr_a->addr4));
+ }
+ else if (addr_a->flags & RSPAMD_SPF_FLAG_IPV6) {
+ return (addr_a->m.dual.mask_v6 - addr_b->m.dual.mask_v6) ||
+ memcmp(addr_a->addr6, addr_b->addr6, sizeof(addr_a->addr6));
+ }
+ else {
+ return 0;
+ }
+ }
+ else {
+ if (addr_a->flags & RSPAMD_SPF_FLAG_ANY) {
+ return 1;
+ }
+ else if (addr_b->flags & RSPAMD_SPF_FLAG_ANY) {
+ return -1;
+ }
+ else if (addr_a->flags & RSPAMD_SPF_FLAG_IPV4) {
+ return -1;
+ }
+
+ return 1;
+ }
+}
+
+static void
+rspamd_spf_record_postprocess(struct spf_resolved *rec, struct rspamd_task *task)
+{
+ g_array_sort(rec->elts, rspamd_spf_elts_cmp);
+
+ for (guint i = 0; i < rec->elts->len; i++) {
+ struct spf_addr *cur_addr = &g_array_index(rec->elts, struct spf_addr, i);
+
+ if (cur_addr->flags & RSPAMD_SPF_FLAG_IPV6) {
+ guint64 t[3];
+
+ /*
+ * Fill hash entry for ipv6 addr with 2 int64 from ipv6 address,
+ * the remaining int64 has mech + mask
+ */
+ memcpy(t, cur_addr->addr6, sizeof(guint64) * 2);
+ t[2] = ((guint64) (cur_addr->mech)) << 48u;
+ t[2] |= cur_addr->m.dual.mask_v6;
+
+ for (guint j = 0; j < G_N_ELEMENTS(t); j++) {
+ rec->digest = mum_hash_step(rec->digest, t[j]);
+ }
+ }
+ else if (cur_addr->flags & RSPAMD_SPF_FLAG_IPV4) {
+ guint64 t = 0;
+
+ memcpy(&t, cur_addr->addr4, sizeof(guint32));
+ t |= ((guint64) (cur_addr->mech)) << 48u;
+ t |= ((guint64) cur_addr->m.dual.mask_v4) << 32u;
+
+ rec->digest = mum_hash_step(rec->digest, t);
+ }
+ }
+
+ if (spf_lib_ctx->min_cache_ttl > 0) {
+ if (rec->ttl != 0 && rec->ttl < spf_lib_ctx->min_cache_ttl) {
+ msg_info_task("increasing ttl from %d to %d as it lower than a limit",
+ rec->ttl, spf_lib_ctx->min_cache_ttl);
+ rec->ttl = spf_lib_ctx->min_cache_ttl;
+ }
+ }
+}
+
+static void
+rspamd_spf_maybe_return(struct spf_record *rec)
+{
+ struct spf_resolved *flat;
+ struct rspamd_task *task = rec->task;
+ bool cached = false;
+
+ if (rec->requests_inflight == 0 && !rec->done) {
+ flat = rspamd_spf_record_flatten(rec);
+ rspamd_spf_record_postprocess(flat, rec->task);
+
+ if (flat->ttl > 0 && flat->flags == 0) {
+
+ if (spf_lib_ctx->spf_hash) {
+ rspamd_lru_hash_insert(spf_lib_ctx->spf_hash,
+ g_strdup(flat->domain),
+ spf_record_ref(flat),
+ flat->timestamp, flat->ttl);
+
+ msg_info_task("stored SPF record for %s (0x%xuL) in LRU cache for %d seconds, "
+ "%d/%d elements in the cache",
+ flat->domain,
+ flat->digest,
+ flat->ttl,
+ rspamd_lru_hash_size(spf_lib_ctx->spf_hash),
+ rspamd_lru_hash_capacity(spf_lib_ctx->spf_hash));
+ cached = true;
+ }
+ }
+
+ if (!cached) {
+ /* Still write a log line */
+ msg_info_task("not stored SPF record for %s (0x%xuL) in LRU cache; flags=%d; ttl=%d",
+ flat->domain,
+ flat->digest,
+ flat->flags,
+ flat->ttl);
+ }
+
+ rec->callback(flat, rec->task, rec->cbdata);
+ spf_record_unref(flat);
+ rec->done = TRUE;
+ }
+}
+
+static gboolean
+spf_check_ptr_host(struct spf_dns_cb *cb, const char *name)
+{
+ const char *dend, *nend, *dstart, *nstart;
+ struct spf_record *rec = cb->rec;
+
+ if (cb->ptr_host != NULL) {
+ dstart = cb->ptr_host;
+ }
+ else {
+ dstart = cb->resolved->cur_domain;
+ }
+
+ if (name == NULL || dstart == NULL) {
+ return FALSE;
+ }
+
+ msg_debug_spf("check ptr %s vs %s", name, dstart);
+
+ /* We need to check whether `cur_domain` is a subdomain for `name` */
+ dend = dstart + strlen(dstart) - 1;
+ nstart = name;
+ nend = nstart + strlen(nstart) - 1;
+
+ if (nend <= nstart || dend <= dstart) {
+ return FALSE;
+ }
+ /* Strip last '.' from names */
+ if (*nend == '.') {
+ nend--;
+ }
+ if (*dend == '.') {
+ dend--;
+ }
+ if (nend <= nstart || dend <= dstart) {
+ return FALSE;
+ }
+
+ /* Now compare from end to start */
+ for (;;) {
+ if (g_ascii_tolower(*dend) != g_ascii_tolower(*nend)) {
+ msg_debug_spf("ptr records mismatch: %s and %s", dend, nend);
+ return FALSE;
+ }
+
+ if (dend == dstart) {
+ break;
+ }
+ if (nend == nstart) {
+ /* Name is shorter than cur_domain */
+ return FALSE;
+ }
+ nend--;
+ dend--;
+ }
+
+ if (nend > nstart && *(nend - 1) != '.') {
+ /* Not a subdomain */
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static void
+spf_record_process_addr(struct spf_record *rec, struct spf_addr *addr, struct rdns_reply_entry *reply)
+{
+ struct spf_addr *naddr;
+
+ if (!(addr->flags & RSPAMD_SPF_FLAG_PROCESSED)) {
+ /* That's the first address */
+ if (reply->type == RDNS_REQUEST_AAAA) {
+ memcpy(addr->addr6,
+ &reply->content.aaa.addr,
+ sizeof(addr->addr6));
+ addr->flags |= RSPAMD_SPF_FLAG_IPV6;
+ }
+ else if (reply->type == RDNS_REQUEST_A) {
+ memcpy(addr->addr4, &reply->content.a.addr, sizeof(addr->addr4));
+ addr->flags |= RSPAMD_SPF_FLAG_IPV4;
+ }
+ else {
+ msg_err_spf(
+ "internal error, bad DNS reply is treated as address: %s; domain: %s",
+ rdns_strtype(reply->type),
+ rec->sender_domain);
+ }
+
+ addr->flags |= RSPAMD_SPF_FLAG_PROCESSED;
+ }
+ else {
+ /* We need to create a new address */
+ naddr = g_malloc0(sizeof(*naddr));
+ memcpy(naddr, addr, sizeof(*naddr));
+ naddr->next = NULL;
+ naddr->prev = NULL;
+
+ if (reply->type == RDNS_REQUEST_AAAA) {
+ memcpy(naddr->addr6,
+ &reply->content.aaa.addr,
+ sizeof(addr->addr6));
+ naddr->flags |= RSPAMD_SPF_FLAG_IPV6;
+ }
+ else if (reply->type == RDNS_REQUEST_A) {
+ memcpy(naddr->addr4, &reply->content.a.addr, sizeof(addr->addr4));
+ naddr->flags |= RSPAMD_SPF_FLAG_IPV4;
+ }
+ else {
+ msg_err_spf(
+ "internal error, bad DNS reply is treated as address: %s; domain: %s",
+ rdns_strtype(reply->type),
+ rec->sender_domain);
+ }
+
+ DL_APPEND(addr, naddr);
+ }
+}
+
+static void
+spf_record_addr_set(struct spf_addr *addr, gboolean allow_any)
+{
+ guchar fill;
+
+ if (!(addr->flags & RSPAMD_SPF_FLAG_PROCESSED)) {
+ if (allow_any) {
+ fill = 0;
+ addr->m.dual.mask_v4 = 0;
+ addr->m.dual.mask_v6 = 0;
+ }
+ else {
+ fill = 0xff;
+ }
+
+ memset(addr->addr4, fill, sizeof(addr->addr4));
+ memset(addr->addr6, fill, sizeof(addr->addr6));
+
+
+ addr->flags |= RSPAMD_SPF_FLAG_IPV4;
+ addr->flags |= RSPAMD_SPF_FLAG_IPV6;
+ }
+}
+
+static gboolean
+spf_process_txt_record(struct spf_record *rec, struct spf_resolved_element *resolved,
+ struct rdns_reply *reply, struct rdns_reply_entry **pselected)
+{
+ struct rdns_reply_entry *elt, *selected = NULL;
+ gboolean ret = FALSE;
+
+ /*
+ * We prefer spf version 1 as other records are mostly likely garbage
+ * or incorrect records (e.g. spf2 records)
+ */
+ LL_FOREACH(reply->entries, elt)
+ {
+ if (elt->type == RDNS_REQUEST_TXT) {
+ if (strncmp(elt->content.txt.data, "v=spf1", sizeof("v=spf1") - 1) == 0) {
+ selected = elt;
+
+ if (pselected != NULL) {
+ *pselected = selected;
+ }
+
+ break;
+ }
+ }
+ }
+
+ if (!selected) {
+ LL_FOREACH(reply->entries, elt)
+ {
+ /*
+ * Rubbish spf record? Let's still try to process it, but merely for
+ * TXT RRs
+ */
+ if (elt->type == RDNS_REQUEST_TXT) {
+ if (start_spf_parse(rec, resolved, elt->content.txt.data)) {
+ ret = TRUE;
+ if (pselected != NULL) {
+ *pselected = elt;
+ }
+ break;
+ }
+ }
+ }
+ }
+ else {
+ ret = start_spf_parse(rec, resolved, selected->content.txt.data);
+ }
+
+ return ret;
+}
+
+static void
+spf_record_dns_callback(struct rdns_reply *reply, gpointer arg)
+{
+ struct spf_dns_cb *cb = arg;
+ struct rdns_reply_entry *elt_data;
+ struct rspamd_task *task;
+ struct spf_addr *addr;
+ struct spf_record *rec;
+ const struct rdns_request_name *req_name;
+ bool truncated = false;
+
+ rec = cb->rec;
+ task = rec->task;
+
+ cb->rec->requests_inflight--;
+ addr = cb->addr;
+ req_name = rdns_request_get_name(reply->request, NULL);
+
+ if (reply->flags & RDNS_TRUNCATED) {
+ /* Do not process truncated DNS replies */
+ truncated = true;
+
+ if (req_name) {
+ msg_notice_spf("got a truncated record when trying to resolve %s (%s type) for SPF domain %s",
+ req_name->name, rdns_str_from_type(req_name->type),
+ rec->sender_domain);
+ }
+ else {
+ msg_notice_spf("got a truncated record when trying to resolve ??? "
+ "(internal error) for SPF domain %s",
+ rec->sender_domain);
+ }
+ }
+
+ if (reply->code == RDNS_RC_NOERROR && !truncated) {
+
+ LL_FOREACH(reply->entries, elt_data)
+ {
+ /* Adjust ttl if a resolved record has lower ttl than spf record itself */
+ if ((guint) elt_data->ttl < rec->ttl) {
+ msg_debug_spf("reducing ttl from %d to %d after DNS resolving",
+ rec->ttl, elt_data->ttl);
+ rec->ttl = elt_data->ttl;
+ }
+
+ if (elt_data->type == RDNS_REQUEST_CNAME) {
+ /* Skip cname aliases - it must be handled by a recursor */
+ continue;
+ }
+
+ switch (cb->cur_action) {
+ case SPF_RESOLVE_MX:
+ if (elt_data->type == RDNS_REQUEST_MX) {
+ /* Now resolve A record for this MX */
+ msg_debug_spf("resolve %s after resolving of MX",
+ elt_data->content.mx.name);
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_record_dns_callback, (void *) cb,
+ RDNS_REQUEST_A,
+ elt_data->content.mx.name)) {
+ cb->rec->requests_inflight++;
+ }
+
+ if (!spf_lib_ctx->disable_ipv6) {
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_record_dns_callback, (void *) cb,
+ RDNS_REQUEST_AAAA,
+ elt_data->content.mx.name)) {
+ cb->rec->requests_inflight++;
+ }
+ }
+ else {
+ msg_debug_spf("skip AAAA request for MX resolution");
+ }
+ }
+ else {
+ cb->addr->flags |= RSPAMD_SPF_FLAG_RESOLVED;
+ cb->addr->flags &= ~RSPAMD_SPF_FLAG_PERMFAIL;
+ msg_debug_spf("resolved MX addr");
+ spf_record_process_addr(rec, addr, elt_data);
+ }
+ break;
+ case SPF_RESOLVE_A:
+ case SPF_RESOLVE_AAA:
+ cb->addr->flags |= RSPAMD_SPF_FLAG_RESOLVED;
+ cb->addr->flags &= ~RSPAMD_SPF_FLAG_PERMFAIL;
+ spf_record_process_addr(rec, addr, elt_data);
+ break;
+ case SPF_RESOLVE_PTR:
+ if (elt_data->type == RDNS_REQUEST_PTR) {
+ /* Validate returned records prior to making A requests */
+ if (spf_check_ptr_host(cb,
+ elt_data->content.ptr.name)) {
+ msg_debug_spf("resolve PTR %s after resolving of PTR",
+ elt_data->content.ptr.name);
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_record_dns_callback, (void *) cb,
+ RDNS_REQUEST_A,
+ elt_data->content.ptr.name)) {
+ cb->rec->requests_inflight++;
+ }
+
+ if (!spf_lib_ctx->disable_ipv6) {
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_record_dns_callback, (void *) cb,
+ RDNS_REQUEST_AAAA,
+ elt_data->content.ptr.name)) {
+ cb->rec->requests_inflight++;
+ }
+ }
+ else {
+ msg_debug_spf("skip AAAA request for PTR resolution");
+ }
+ }
+ else {
+ cb->addr->flags |= RSPAMD_SPF_FLAG_RESOLVED;
+ cb->addr->flags &= ~RSPAMD_SPF_FLAG_PERMFAIL;
+ }
+ }
+ else {
+ cb->addr->flags |= RSPAMD_SPF_FLAG_RESOLVED;
+ cb->addr->flags &= ~RSPAMD_SPF_FLAG_PERMFAIL;
+ spf_record_process_addr(rec, addr, elt_data);
+ }
+ break;
+ case SPF_RESOLVE_REDIRECT:
+ if (elt_data->type == RDNS_REQUEST_TXT) {
+ cb->addr->flags |= RSPAMD_SPF_FLAG_RESOLVED;
+ if (reply->entries) {
+ msg_debug_spf("got redirection record for %s: '%s'",
+ req_name->name,
+ reply->entries[0].content.txt.data);
+ }
+
+ if (!spf_process_txt_record(rec, cb->resolved, reply, NULL)) {
+ cb->addr->flags |= RSPAMD_SPF_FLAG_PERMFAIL;
+ }
+ }
+
+ goto end;
+ break;
+ case SPF_RESOLVE_INCLUDE:
+ if (elt_data->type == RDNS_REQUEST_TXT) {
+ struct rdns_reply_entry *selected = NULL;
+
+ cb->addr->flags |= RSPAMD_SPF_FLAG_RESOLVED;
+ spf_process_txt_record(rec, cb->resolved, reply, &selected);
+ if (selected) {
+ msg_debug_spf("got include record for %s: '%s'",
+ req_name->name,
+ selected->content.txt.data);
+ }
+ else {
+ msg_debug_spf("no include record for %s",
+ req_name->name);
+ }
+ }
+ goto end;
+
+ break;
+ case SPF_RESOLVE_EXP:
+ break;
+ case SPF_RESOLVE_EXISTS:
+ if (elt_data->type == RDNS_REQUEST_A ||
+ elt_data->type == RDNS_REQUEST_AAAA) {
+ /*
+ * If specified address resolves, we can accept
+ * connection from every IP
+ */
+ addr->flags |= RSPAMD_SPF_FLAG_RESOLVED;
+ spf_record_addr_set(addr, TRUE);
+ }
+ break;
+ }
+ }
+ }
+ else if (reply->code == RDNS_RC_NXDOMAIN || reply->code == RDNS_RC_NOREC) {
+ switch (cb->cur_action) {
+ case SPF_RESOLVE_MX:
+ if (!(cb->addr->flags & RSPAMD_SPF_FLAG_RESOLVED)) {
+ cb->addr->flags |= RSPAMD_SPF_FLAG_PERMFAIL;
+ msg_info_spf(
+ "spf error for domain %s: cannot find MX"
+ " record for %s: %s",
+ cb->rec->sender_domain,
+ cb->resolved->cur_domain,
+ rdns_strerror(reply->code));
+ spf_record_addr_set(addr, FALSE);
+ }
+ break;
+ case SPF_RESOLVE_A:
+ if (!(cb->addr->flags & RSPAMD_SPF_FLAG_RESOLVED)) {
+ cb->addr->flags |= RSPAMD_SPF_FLAG_PERMFAIL;
+ msg_info_spf(
+ "spf error for domain %s: cannot resolve A"
+ " record for %s: %s",
+ cb->rec->sender_domain,
+ cb->resolved->cur_domain,
+ rdns_strerror(reply->code));
+
+ if (rdns_request_has_type(reply->request, RDNS_REQUEST_A)) {
+ spf_record_addr_set(addr, FALSE);
+ }
+ }
+ break;
+ case SPF_RESOLVE_AAA:
+ if (!(cb->addr->flags & RSPAMD_SPF_FLAG_RESOLVED)) {
+ cb->addr->flags |= RSPAMD_SPF_FLAG_PERMFAIL;
+ msg_info_spf(
+ "spf error for domain %s: cannot resolve AAAA"
+ " record for %s: %s",
+ cb->rec->sender_domain,
+ cb->resolved->cur_domain,
+ rdns_strerror(reply->code));
+ if (rdns_request_has_type(reply->request, RDNS_REQUEST_AAAA)) {
+ spf_record_addr_set(addr, FALSE);
+ }
+ }
+ break;
+ case SPF_RESOLVE_PTR:
+ if (!(cb->addr->flags & RSPAMD_SPF_FLAG_RESOLVED)) {
+ msg_info_spf(
+ "spf error for domain %s: cannot resolve PTR"
+ " record for %s: %s",
+ cb->rec->sender_domain,
+ cb->resolved->cur_domain,
+ rdns_strerror(reply->code));
+ cb->addr->flags |= RSPAMD_SPF_FLAG_PERMFAIL;
+
+ spf_record_addr_set(addr, FALSE);
+ }
+ break;
+ case SPF_RESOLVE_REDIRECT:
+ if (!(cb->addr->flags & RSPAMD_SPF_FLAG_RESOLVED)) {
+ cb->addr->flags |= RSPAMD_SPF_FLAG_PERMFAIL;
+ msg_info_spf(
+ "spf error for domain %s: cannot resolve REDIRECT"
+ " record for %s: %s",
+ cb->rec->sender_domain,
+ cb->resolved->cur_domain,
+ rdns_strerror(reply->code));
+ }
+
+ break;
+ case SPF_RESOLVE_INCLUDE:
+ if (!(cb->addr->flags & RSPAMD_SPF_FLAG_RESOLVED)) {
+ msg_info_spf(
+ "spf error for domain %s: cannot resolve INCLUDE"
+ " record for %s: %s",
+ cb->rec->sender_domain,
+ cb->resolved->cur_domain,
+ rdns_strerror(reply->code));
+
+ cb->addr->flags |= RSPAMD_SPF_FLAG_PERMFAIL;
+ }
+ break;
+ case SPF_RESOLVE_EXP:
+ break;
+ case SPF_RESOLVE_EXISTS:
+ if (!(cb->addr->flags & RSPAMD_SPF_FLAG_RESOLVED)) {
+ msg_debug_spf(
+ "spf macro resolution for domain %s: cannot resolve EXISTS"
+ " macro for %s: %s",
+ cb->rec->sender_domain,
+ cb->resolved->cur_domain,
+ rdns_strerror(reply->code));
+ spf_record_addr_set(addr, FALSE);
+ }
+ break;
+ }
+ }
+ else {
+ cb->addr->flags |= RSPAMD_SPF_FLAG_TEMPFAIL;
+ msg_info_spf(
+ "spf error for domain %s: cannot resolve %s DNS record for"
+ " %s: %s",
+ cb->rec->sender_domain,
+ rspamd_spf_dns_action_to_str(cb->cur_action),
+ cb->ptr_host,
+ rdns_strerror(reply->code));
+ }
+
+end:
+ rspamd_spf_maybe_return(cb->rec);
+}
+
+/*
+ * The syntax defined by the following BNF:
+ * [ ":" domain-spec ] [ dual-cidr-length ]
+ * ip4-cidr-length = "/" 1*DIGIT
+ * ip6-cidr-length = "/" 1*DIGIT
+ * dual-cidr-length = [ ip4-cidr-length ] [ "/" ip6-cidr-length ]
+ */
+static const gchar *
+parse_spf_domain_mask(struct spf_record *rec, struct spf_addr *addr,
+ struct spf_resolved_element *resolved,
+ gboolean allow_mask)
+{
+ struct rspamd_task *task = rec->task;
+ enum {
+ parse_spf_elt = 0,
+ parse_semicolon,
+ parse_domain,
+ parse_slash,
+ parse_ipv4_mask,
+ parse_second_slash,
+ parse_ipv6_mask,
+ skip_garbage
+ } state = 0;
+ const gchar *p = addr->spf_string, *host, *c;
+ gchar *hostbuf;
+ gchar t;
+ guint16 cur_mask = 0;
+
+ host = resolved->cur_domain;
+ c = p;
+
+ while (*p) {
+ t = *p;
+
+ switch (state) {
+ case parse_spf_elt:
+ if (t == ':' || t == '=') {
+ state = parse_semicolon;
+ }
+ else if (t == '/') {
+ /* No domain but mask */
+ state = parse_slash;
+ }
+ p++;
+ break;
+ case parse_semicolon:
+ if (t == '/') {
+ /* Empty domain, technically an error */
+ state = parse_slash;
+ }
+ else {
+ c = p;
+ state = parse_domain;
+ }
+ break;
+ case parse_domain:
+ if (t == '/') {
+ hostbuf = rspamd_mempool_alloc(task->task_pool, p - c + 1);
+ rspamd_strlcpy(hostbuf, c, p - c + 1);
+ host = hostbuf;
+ state = parse_slash;
+ }
+ p++;
+ break;
+ case parse_slash:
+ c = p;
+ if (allow_mask) {
+ state = parse_ipv4_mask;
+ }
+ else {
+ state = skip_garbage;
+ }
+ cur_mask = 0;
+ break;
+ case parse_ipv4_mask:
+ if (g_ascii_isdigit(t)) {
+ /* Ignore errors here */
+ cur_mask = cur_mask * 10 + (t - '0');
+ }
+ else if (t == '/') {
+ if (cur_mask <= 32) {
+ addr->m.dual.mask_v4 = cur_mask;
+ }
+ else {
+ msg_notice_spf("bad ipv4 mask for %s: %d",
+ rec->sender_domain, cur_mask);
+ }
+ state = parse_second_slash;
+ }
+ p++;
+ break;
+ case parse_second_slash:
+ c = p;
+ state = parse_ipv6_mask;
+ cur_mask = 0;
+ break;
+ case parse_ipv6_mask:
+ if (g_ascii_isdigit(t)) {
+ /* Ignore errors here */
+ cur_mask = cur_mask * 10 + (t - '0');
+ }
+ p++;
+ break;
+ case skip_garbage:
+ p++;
+ break;
+ }
+ }
+
+ /* Process end states */
+ if (state == parse_ipv4_mask) {
+ if (cur_mask <= 32) {
+ addr->m.dual.mask_v4 = cur_mask;
+ }
+ else {
+ msg_notice_spf("bad ipv4 mask for %s: %d", rec->sender_domain, cur_mask);
+ }
+ }
+ else if (state == parse_ipv6_mask) {
+ if (cur_mask <= 128) {
+ addr->m.dual.mask_v6 = cur_mask;
+ }
+ else {
+ msg_notice_spf("bad ipv6 mask: %d", cur_mask);
+ }
+ }
+ else if (state == parse_domain && p - c > 0) {
+ hostbuf = rspamd_mempool_alloc(task->task_pool, p - c + 1);
+ rspamd_strlcpy(hostbuf, c, p - c + 1);
+ host = hostbuf;
+ }
+
+ if (cur_mask == 0) {
+ addr->m.dual.mask_v4 = 32;
+ addr->m.dual.mask_v6 = 64;
+ }
+
+ return host;
+}
+
+static gboolean
+parse_spf_a(struct spf_record *rec,
+ struct spf_resolved_element *resolved, struct spf_addr *addr)
+{
+ struct spf_dns_cb *cb;
+ const gchar *host = NULL;
+ struct rspamd_task *task = rec->task;
+
+ CHECK_REC(rec);
+
+ host = parse_spf_domain_mask(rec, addr, resolved, TRUE);
+
+ if (host == NULL) {
+ return FALSE;
+ }
+
+ rec->dns_requests++;
+ cb = rspamd_mempool_alloc(task->task_pool, sizeof(struct spf_dns_cb));
+ cb->rec = rec;
+ cb->ptr_host = host;
+ cb->addr = addr;
+ cb->cur_action = SPF_RESOLVE_A;
+ cb->resolved = resolved;
+ msg_debug_spf("resolve a %s", host);
+
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_record_dns_callback, (void *) cb, RDNS_REQUEST_A, host)) {
+ rec->requests_inflight++;
+
+ cb = rspamd_mempool_alloc(task->task_pool, sizeof(struct spf_dns_cb));
+ cb->rec = rec;
+ cb->ptr_host = host;
+ cb->addr = addr;
+ cb->cur_action = SPF_RESOLVE_AAA;
+ cb->resolved = resolved;
+
+ if (!spf_lib_ctx->disable_ipv6) {
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_record_dns_callback, (void *) cb, RDNS_REQUEST_AAAA, host)) {
+ rec->requests_inflight++;
+ }
+ }
+ else {
+ msg_debug_spf("skip AAAA request for a record resolution");
+ }
+
+ return TRUE;
+ }
+ else {
+ msg_notice_spf("unresolvable A element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ }
+
+ return FALSE;
+}
+
+static gboolean
+parse_spf_ptr(struct spf_record *rec,
+ struct spf_resolved_element *resolved, struct spf_addr *addr)
+{
+ struct spf_dns_cb *cb;
+ const gchar *host;
+ gchar *ptr;
+ struct rspamd_task *task = rec->task;
+
+ CHECK_REC(rec);
+
+ host = parse_spf_domain_mask(rec, addr, resolved, FALSE);
+
+ rec->dns_requests++;
+ cb = rspamd_mempool_alloc(task->task_pool, sizeof(struct spf_dns_cb));
+ cb->rec = rec;
+ cb->addr = addr;
+ cb->cur_action = SPF_RESOLVE_PTR;
+ cb->resolved = resolved;
+ cb->ptr_host = rspamd_mempool_strdup(task->task_pool, host);
+ ptr =
+ rdns_generate_ptr_from_str(rspamd_inet_address_to_string(
+ task->from_addr));
+
+ if (ptr == NULL) {
+ return FALSE;
+ }
+
+ rspamd_mempool_add_destructor(task->task_pool, free, ptr);
+ msg_debug_spf("resolve ptr %s for %s", ptr, host);
+
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_record_dns_callback, (void *) cb, RDNS_REQUEST_PTR, ptr)) {
+ rec->requests_inflight++;
+ rec->ttl = 0;
+ msg_debug_spf("disable SPF caching as there is PTR expansion");
+
+ return TRUE;
+ }
+ else {
+ msg_notice_spf("unresolvable PTR element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ }
+
+ return FALSE;
+}
+
+static gboolean
+parse_spf_mx(struct spf_record *rec,
+ struct spf_resolved_element *resolved, struct spf_addr *addr)
+{
+ struct spf_dns_cb *cb;
+ const gchar *host;
+ struct rspamd_task *task = rec->task;
+
+ CHECK_REC(rec);
+
+ host = parse_spf_domain_mask(rec, addr, resolved, TRUE);
+
+ if (host == NULL) {
+ return FALSE;
+ }
+
+ rec->dns_requests++;
+ cb = rspamd_mempool_alloc(task->task_pool, sizeof(struct spf_dns_cb));
+ cb->rec = rec;
+ cb->addr = addr;
+ cb->cur_action = SPF_RESOLVE_MX;
+ cb->ptr_host = host;
+ cb->resolved = resolved;
+
+ msg_debug_spf("resolve mx for %s", host);
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_record_dns_callback, (void *) cb, RDNS_REQUEST_MX, host)) {
+ rec->requests_inflight++;
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static gboolean
+parse_spf_all(struct spf_record *rec, struct spf_addr *addr)
+{
+ /* All is 0/0 */
+ memset(&addr->addr4, 0, sizeof(addr->addr4));
+ memset(&addr->addr6, 0, sizeof(addr->addr6));
+ /* Here we set all masks to 0 */
+ addr->m.idx = 0;
+ addr->flags |= RSPAMD_SPF_FLAG_ANY | RSPAMD_SPF_FLAG_RESOLVED;
+ msg_debug_spf("parsed all elt");
+
+ /* Disallow +all */
+ if (addr->mech == SPF_PASS) {
+ addr->flags |= RSPAMD_SPF_FLAG_INVALID;
+ msg_notice_spf("domain %s allows any SPF (+all), ignore SPF record completely",
+ rec->sender_domain);
+ }
+
+ return TRUE;
+}
+
+static gboolean
+parse_spf_ip4(struct spf_record *rec, struct spf_addr *addr)
+{
+ /* ip4:addr[/mask] */
+ const gchar *semicolon, *slash;
+ gsize len;
+ gchar ipbuf[INET_ADDRSTRLEN + 1];
+ guint32 mask;
+ static const guint32 min_valid_mask = 8;
+
+ semicolon = strchr(addr->spf_string, ':');
+
+ if (semicolon == NULL) {
+ semicolon = strchr(addr->spf_string, '=');
+
+ if (semicolon == NULL) {
+ msg_notice_spf("invalid ip4 element for %s: %s, no '=' or ':'", addr->spf_string,
+ rec->sender_domain);
+ return FALSE;
+ }
+ }
+
+ semicolon++;
+ slash = strchr(semicolon, '/');
+
+ if (slash) {
+ len = slash - semicolon;
+ }
+ else {
+ len = strlen(semicolon);
+ }
+
+ rspamd_strlcpy(ipbuf, semicolon, MIN(len + 1, sizeof(ipbuf)));
+
+ if (inet_pton(AF_INET, ipbuf, addr->addr4) != 1) {
+ msg_notice_spf("invalid ip4 element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ return FALSE;
+ }
+
+ if (slash) {
+ gchar *end = NULL;
+
+ mask = strtoul(slash + 1, &end, 10);
+ if (mask > 32) {
+ msg_notice_spf("invalid mask for ip4 element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ return FALSE;
+ }
+
+ if (end != NULL && !g_ascii_isspace(*end) && *end != '\0') {
+ /* Invalid mask definition */
+ msg_notice_spf("invalid mask for ip4 element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ return FALSE;
+ }
+
+ addr->m.dual.mask_v4 = mask;
+
+ if (mask < min_valid_mask) {
+ addr->flags |= RSPAMD_SPF_FLAG_INVALID;
+ msg_notice_spf("too wide SPF record for %s: %s/%d",
+ rec->sender_domain,
+ ipbuf, addr->m.dual.mask_v4);
+ }
+ }
+ else {
+ addr->m.dual.mask_v4 = 32;
+ }
+
+ addr->flags |= RSPAMD_SPF_FLAG_IPV4 | RSPAMD_SPF_FLAG_RESOLVED;
+ msg_debug_spf("parsed ipv4 record %s/%d", ipbuf, addr->m.dual.mask_v4);
+
+ return TRUE;
+}
+
+static gboolean
+parse_spf_ip6(struct spf_record *rec, struct spf_addr *addr)
+{
+ /* ip6:addr[/mask] */
+ const gchar *semicolon, *slash;
+ gsize len;
+ gchar ipbuf[INET6_ADDRSTRLEN + 1];
+ guint32 mask;
+ static const guint32 min_valid_mask = 8;
+
+ semicolon = strchr(addr->spf_string, ':');
+
+ if (semicolon == NULL) {
+ semicolon = strchr(addr->spf_string, '=');
+
+ if (semicolon == NULL) {
+ msg_notice_spf("invalid ip6 element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ return FALSE;
+ }
+ }
+
+ semicolon++;
+ slash = strchr(semicolon, '/');
+
+ if (slash) {
+ len = slash - semicolon;
+ }
+ else {
+ len = strlen(semicolon);
+ }
+
+ rspamd_strlcpy(ipbuf, semicolon, MIN(len + 1, sizeof(ipbuf)));
+
+ if (inet_pton(AF_INET6, ipbuf, addr->addr6) != 1) {
+ msg_notice_spf("invalid ip6 element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ return FALSE;
+ }
+
+ if (slash) {
+ gchar *end = NULL;
+ mask = strtoul(slash + 1, &end, 10);
+ if (mask > 128) {
+ msg_notice_spf("invalid mask for ip6 element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ return FALSE;
+ }
+
+ if (end != NULL && !g_ascii_isspace(*end) && *end != '\0') {
+ /* Invalid mask definition */
+ msg_notice_spf("invalid mask for ip4 element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ return FALSE;
+ }
+
+ addr->m.dual.mask_v6 = mask;
+
+ if (mask < min_valid_mask) {
+ addr->flags |= RSPAMD_SPF_FLAG_INVALID;
+ msg_notice_spf("too wide SPF record for %s: %s/%d",
+ rec->sender_domain,
+ ipbuf, addr->m.dual.mask_v6);
+ }
+ }
+ else {
+ addr->m.dual.mask_v6 = 128;
+ }
+
+ addr->flags |= RSPAMD_SPF_FLAG_IPV6 | RSPAMD_SPF_FLAG_RESOLVED;
+ msg_debug_spf("parsed ipv6 record %s/%d", ipbuf, addr->m.dual.mask_v6);
+
+ return TRUE;
+}
+
+
+static gboolean
+parse_spf_include(struct spf_record *rec, struct spf_addr *addr)
+{
+ struct spf_dns_cb *cb;
+ const gchar *domain;
+ struct rspamd_task *task = rec->task;
+
+ CHECK_REC(rec);
+ domain = strchr(addr->spf_string, ':');
+
+ if (domain == NULL) {
+ /* Common mistake */
+ domain = strchr(addr->spf_string, '=');
+
+ if (domain == NULL) {
+ msg_notice_spf("invalid include element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ return FALSE;
+ }
+ }
+
+ domain++;
+
+ rec->dns_requests++;
+
+ cb = rspamd_mempool_alloc(task->task_pool, sizeof(struct spf_dns_cb));
+ cb->rec = rec;
+ cb->addr = addr;
+ cb->cur_action = SPF_RESOLVE_INCLUDE;
+ addr->m.idx = rec->resolved->len;
+ cb->resolved = rspamd_spf_new_addr_list(rec, domain);
+ cb->ptr_host = domain;
+ /* Set reference */
+ addr->flags |= RSPAMD_SPF_FLAG_REFERENCE;
+ msg_debug_spf("resolve include %s", domain);
+
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_record_dns_callback, (void *) cb, RDNS_REQUEST_TXT, domain)) {
+ rec->requests_inflight++;
+
+ return TRUE;
+ }
+ else {
+ msg_notice_spf("unresolvable include element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ }
+
+
+ return FALSE;
+}
+
+static gboolean
+parse_spf_exp(struct spf_record *rec, struct spf_addr *addr)
+{
+ msg_info_spf("exp record is ignored");
+ return TRUE;
+}
+
+static gboolean
+parse_spf_redirect(struct spf_record *rec,
+ struct spf_resolved_element *resolved, struct spf_addr *addr)
+{
+ struct spf_dns_cb *cb;
+ const gchar *domain;
+ struct rspamd_task *task = rec->task;
+
+ CHECK_REC(rec);
+
+ domain = strchr(addr->spf_string, '=');
+
+ if (domain == NULL) {
+ /* Common mistake */
+ domain = strchr(addr->spf_string, ':');
+
+ if (domain == NULL) {
+ msg_notice_spf("invalid redirect element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ return FALSE;
+ }
+ }
+
+ domain++;
+
+ rec->dns_requests++;
+ resolved->redirected = TRUE;
+
+ cb = rspamd_mempool_alloc(task->task_pool, sizeof(struct spf_dns_cb));
+ /* Set reference */
+ addr->flags |= RSPAMD_SPF_FLAG_REFERENCE | RSPAMD_SPF_FLAG_REDIRECT;
+ addr->m.idx = rec->resolved->len;
+
+ cb->rec = rec;
+ cb->addr = addr;
+ cb->cur_action = SPF_RESOLVE_REDIRECT;
+ cb->resolved = rspamd_spf_new_addr_list(rec, domain);
+ cb->ptr_host = domain;
+ msg_debug_spf("resolve redirect %s", domain);
+
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_record_dns_callback, (void *) cb, RDNS_REQUEST_TXT, domain)) {
+ rec->requests_inflight++;
+
+ return TRUE;
+ }
+ else {
+ msg_notice_spf("unresolvable redirect element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ }
+
+ return FALSE;
+}
+
+static gboolean
+parse_spf_exists(struct spf_record *rec, struct spf_addr *addr)
+{
+ struct spf_dns_cb *cb;
+ const gchar *host;
+ struct rspamd_task *task = rec->task;
+ struct spf_resolved_element *resolved;
+
+ resolved = g_ptr_array_index(rec->resolved, rec->resolved->len - 1);
+ CHECK_REC(rec);
+
+ host = strchr(addr->spf_string, ':');
+ if (host == NULL) {
+ host = strchr(addr->spf_string, '=');
+
+ if (host == NULL) {
+ msg_notice_spf("invalid exists element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ return FALSE;
+ }
+ }
+
+ host++;
+ rec->dns_requests++;
+
+ cb = rspamd_mempool_alloc(task->task_pool, sizeof(struct spf_dns_cb));
+ cb->rec = rec;
+ cb->addr = addr;
+ cb->cur_action = SPF_RESOLVE_EXISTS;
+ cb->resolved = resolved;
+ cb->ptr_host = host;
+
+ msg_debug_spf("resolve exists %s", host);
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_record_dns_callback, (void *) cb, RDNS_REQUEST_A, host)) {
+ rec->requests_inflight++;
+
+ return TRUE;
+ }
+ else {
+ msg_notice_spf("unresolvable exists element for %s: %s", addr->spf_string,
+ rec->sender_domain);
+ }
+
+ return FALSE;
+}
+
+static gsize
+rspamd_spf_split_elt(const gchar *val, gsize len, gint *pos,
+ gsize poslen, gchar delim)
+{
+ const gchar *p, *end;
+ guint cur_pos = 0, cur_st = 0, nsub = 0;
+
+ p = val;
+ end = val + len;
+
+ while (p < end && cur_pos + 2 < poslen) {
+ if (*p == delim) {
+ if (p - val > cur_st) {
+ pos[cur_pos] = cur_st;
+ pos[cur_pos + 1] = p - val;
+ cur_st = p - val + 1;
+ cur_pos += 2;
+ nsub++;
+ }
+
+ p++;
+ }
+ else {
+ p++;
+ }
+ }
+
+ if (cur_pos + 2 < poslen) {
+ if (end - val > cur_st) {
+ pos[cur_pos] = cur_st;
+ pos[cur_pos + 1] = end - val;
+ nsub++;
+ }
+ }
+ else {
+ pos[cur_pos] = p - val;
+ pos[cur_pos + 1] = end - val;
+ nsub++;
+ }
+
+ return nsub;
+}
+
+static gsize
+rspamd_spf_process_substitution(const gchar *macro_value,
+ gsize macro_len, guint ndelim, gchar delim, gboolean reversed,
+ gchar *dest)
+{
+ gchar *d = dest;
+ const gchar canon_delim = '.';
+ guint vlen, i;
+ gint pos[49 * 2], tlen;
+
+ if (!reversed && ndelim == 0 && delim == canon_delim) {
+ /* Trivial case */
+ memcpy(dest, macro_value, macro_len);
+
+ return macro_len;
+ }
+
+ vlen = rspamd_spf_split_elt(macro_value, macro_len,
+ pos, G_N_ELEMENTS(pos), delim);
+
+ if (vlen > 0) {
+ if (reversed) {
+ for (i = vlen - 1;; i--) {
+ tlen = pos[i * 2 + 1] - pos[i * 2];
+
+ if (i != 0) {
+ memcpy(d, &macro_value[pos[i * 2]], tlen);
+ d += tlen;
+ *d++ = canon_delim;
+ }
+ else {
+ memcpy(d, &macro_value[pos[i * 2]], tlen);
+ d += tlen;
+ break;
+ }
+ }
+ }
+ else {
+ for (i = 0; i < vlen; i++) {
+ tlen = pos[i * 2 + 1] - pos[i * 2];
+
+ if (i != vlen - 1) {
+ memcpy(d, &macro_value[pos[i * 2]], tlen);
+ d += tlen;
+ *d++ = canon_delim;
+ }
+ else {
+ memcpy(d, &macro_value[pos[i * 2]], tlen);
+ d += tlen;
+ }
+ }
+ }
+ }
+ else {
+ /* Trivial case */
+ memcpy(dest, macro_value, macro_len);
+
+ return macro_len;
+ }
+
+ return (d - dest);
+}
+
+static const gchar *
+expand_spf_macro(struct spf_record *rec, struct spf_resolved_element *resolved,
+ const gchar *begin)
+{
+ const gchar *p, *macro_value = NULL;
+ gchar *c, *new, *tmp, delim = '.';
+ gsize len = 0, macro_len = 0;
+ gint state = 0, ndelim = 0;
+ gchar ip_buf[64 + 1]; /* cannot use INET6_ADDRSTRLEN as we use ptr lookup */
+ gboolean need_expand = FALSE, reversed;
+ struct rspamd_task *task;
+
+ g_assert(rec != NULL);
+ g_assert(begin != NULL);
+
+ task = rec->task;
+ p = begin;
+ /* Calculate length */
+ while (*p) {
+ switch (state) {
+ case 0:
+ /* Skip any character and wait for % in input */
+ if (*p == '%') {
+ state = 1;
+ }
+ else {
+ len++;
+ }
+
+ p++;
+ break;
+ case 1:
+ /* We got % sign, so we should whether wait for { or for - or for _ or for % */
+ if (*p == '%' || *p == '_') {
+ /* Just a single % sign or space */
+ len++;
+ state = 0;
+ }
+ else if (*p == '-') {
+ /* %20 */
+ len += sizeof("%20") - 1;
+ state = 0;
+ }
+ else if (*p == '{') {
+ state = 2;
+ }
+ else {
+ /* Something unknown */
+ msg_notice_spf(
+ "spf error for domain %s: unknown spf element",
+ rec->sender_domain);
+ return begin;
+ }
+ p++;
+
+ break;
+ case 2:
+ /* Read macro name */
+ switch (g_ascii_tolower(*p)) {
+ case 'i':
+ len += sizeof(ip_buf) - 1;
+ break;
+ case 's':
+ if (rec->sender) {
+ len += strlen(rec->sender);
+ }
+ else {
+ len += sizeof("unknown") - 1;
+ }
+ break;
+ case 'l':
+ if (rec->local_part) {
+ len += strlen(rec->local_part);
+ }
+ else {
+ len += sizeof("unknown") - 1;
+ }
+ break;
+ case 'o':
+ if (rec->sender_domain) {
+ len += strlen(rec->sender_domain);
+ }
+ else {
+ len += sizeof("unknown") - 1;
+ }
+ break;
+ case 'd':
+ if (resolved->cur_domain) {
+ len += strlen(resolved->cur_domain);
+ }
+ else {
+ len += sizeof("unknown") - 1;
+ }
+ break;
+ case 'v':
+ len += sizeof("in-addr") - 1;
+ break;
+ case 'h':
+ if (task->helo) {
+ len += strlen(task->helo);
+ }
+ else {
+ len += sizeof("unknown") - 1;
+ }
+ break;
+ default:
+ msg_notice_spf(
+ "spf error for domain %s: unknown or "
+ "unsupported spf macro %c in %s",
+ rec->sender_domain,
+ *p,
+ begin);
+ return begin;
+ }
+ p++;
+ state = 3;
+ break;
+ case 3:
+ /* Read modifier */
+ if (*p == '}') {
+ state = 0;
+ need_expand = TRUE;
+ }
+ p++;
+ break;
+
+ default:
+ g_assert_not_reached();
+ }
+ }
+
+ if (!need_expand) {
+ /* No expansion needed */
+ return begin;
+ }
+
+ new = rspamd_mempool_alloc(task->task_pool, len + 1);
+
+ /* Reduce TTL to avoid caching of records with macros */
+ if (rec->ttl != 0) {
+ rec->ttl = 0;
+ msg_debug_spf("disable SPF caching as there is macro expansion");
+ }
+
+ c = new;
+ p = begin;
+ state = 0;
+ /* Begin macro expansion */
+
+ while (*p) {
+ switch (state) {
+ case 0:
+ /* Skip any character and wait for % in input */
+ if (*p == '%') {
+ state = 1;
+ }
+ else {
+ *c = *p;
+ c++;
+ }
+
+ p++;
+ break;
+ case 1:
+ /* We got % sign, so we should whether wait for { or for - or for _ or for % */
+ if (*p == '%') {
+ /* Just a single % sign or space */
+ *c++ = '%';
+ state = 0;
+ }
+ else if (*p == '_') {
+ *c++ = ' ';
+ state = 0;
+ }
+ else if (*p == '-') {
+ /* %20 */
+ *c++ = '%';
+ *c++ = '2';
+ *c++ = '0';
+ state = 0;
+ }
+ else if (*p == '{') {
+ state = 2;
+ }
+ else {
+ /* Something unknown */
+ msg_info_spf(
+ "spf error for domain %s: unknown spf element",
+ rec->sender_domain);
+ return begin;
+ }
+ p++;
+ break;
+ case 2:
+ /* Read macro name */
+ switch (g_ascii_tolower(*p)) {
+ case 'i':
+ if (task->from_addr) {
+ if (rspamd_inet_address_get_af(task->from_addr) == AF_INET) {
+ macro_len = rspamd_strlcpy(ip_buf,
+ rspamd_inet_address_to_string(task->from_addr),
+ sizeof(ip_buf));
+ macro_value = ip_buf;
+ }
+ else if (rspamd_inet_address_get_af(task->from_addr) == AF_INET6) {
+ /* See #3625 for details */
+ socklen_t slen;
+ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)
+ rspamd_inet_address_get_sa(task->from_addr, &slen);
+
+ /* Expand IPv6 address */
+#define IPV6_OCTET(x) bytes[(x)] >> 4, bytes[(x)] & 0xF
+ unsigned char *bytes = (unsigned char *) &sin6->sin6_addr;
+ macro_len = rspamd_snprintf(ip_buf, sizeof(ip_buf),
+ "%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd."
+ "%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd.%xd",
+ IPV6_OCTET(0), IPV6_OCTET(1),
+ IPV6_OCTET(2), IPV6_OCTET(3),
+ IPV6_OCTET(4), IPV6_OCTET(5),
+ IPV6_OCTET(6), IPV6_OCTET(7),
+ IPV6_OCTET(8), IPV6_OCTET(9),
+ IPV6_OCTET(10), IPV6_OCTET(11),
+ IPV6_OCTET(12), IPV6_OCTET(13),
+ IPV6_OCTET(14), IPV6_OCTET(15));
+ macro_value = ip_buf;
+#undef IPV6_OCTET
+ }
+ else {
+ macro_len = rspamd_snprintf(ip_buf, sizeof(ip_buf),
+ "127.0.0.1");
+ macro_value = ip_buf;
+ }
+ }
+ else {
+ macro_len = rspamd_snprintf(ip_buf, sizeof(ip_buf),
+ "127.0.0.1");
+ macro_value = ip_buf;
+ }
+ break;
+ case 's':
+ if (rec->sender) {
+ macro_len = strlen(rec->sender);
+ macro_value = rec->sender;
+ }
+ else {
+ macro_len = sizeof("unknown") - 1;
+ macro_value = "unknown";
+ }
+ break;
+ case 'l':
+ if (rec->local_part) {
+ macro_len = strlen(rec->local_part);
+ macro_value = rec->local_part;
+ }
+ else {
+ macro_len = sizeof("unknown") - 1;
+ macro_value = "unknown";
+ }
+ break;
+ case 'o':
+ if (rec->sender_domain) {
+ macro_len = strlen(rec->sender_domain);
+ macro_value = rec->sender_domain;
+ }
+ else {
+ macro_len = sizeof("unknown") - 1;
+ macro_value = "unknown";
+ }
+ break;
+ case 'd':
+ if (resolved && resolved->cur_domain) {
+ macro_len = strlen(resolved->cur_domain);
+ macro_value = resolved->cur_domain;
+ }
+ else {
+ macro_len = sizeof("unknown") - 1;
+ macro_value = "unknown";
+ }
+ break;
+ case 'v':
+ if (task->from_addr) {
+ if (rspamd_inet_address_get_af(task->from_addr) == AF_INET) {
+ macro_len = sizeof("in-addr") - 1;
+ macro_value = "in-addr";
+ }
+ else {
+ macro_len = sizeof("ip6") - 1;
+ macro_value = "ip6";
+ }
+ }
+ else {
+ macro_len = sizeof("in-addr") - 1;
+ macro_value = "in-addr";
+ }
+ break;
+ case 'h':
+ if (task->helo) {
+ tmp = strchr(task->helo, '@');
+ if (tmp) {
+ macro_len = strlen(tmp + 1);
+ macro_value = tmp + 1;
+ }
+ else {
+ macro_len = strlen(task->helo);
+ macro_value = task->helo;
+ }
+ }
+ else {
+ macro_len = sizeof("unknown") - 1;
+ macro_value = "unknown";
+ }
+ break;
+ default:
+ msg_info_spf(
+ "spf error for domain %s: unknown or "
+ "unsupported spf macro %c in %s",
+ rec->sender_domain,
+ *p,
+ begin);
+ return begin;
+ }
+
+ p++;
+ state = 3;
+ ndelim = 0;
+ delim = '.';
+ reversed = FALSE;
+ break;
+
+ case 3:
+ /* Read modifier */
+ if (*p == '}') {
+ state = 0;
+ len = rspamd_spf_process_substitution(macro_value,
+ macro_len, ndelim, delim, reversed, c);
+ c += len;
+ }
+ else if (*p == 'r' && len != 0) {
+ reversed = TRUE;
+ }
+ else if (g_ascii_isdigit(*p)) {
+ ndelim = strtoul(p, &tmp, 10);
+
+ if (tmp == NULL || tmp == p) {
+ p++;
+ }
+ else {
+ p = tmp;
+
+ continue;
+ }
+ }
+ else if (*p == '+' || *p == '-' ||
+ *p == '.' || *p == ',' || *p == '/' || *p == '_' ||
+ *p == '=') {
+ delim = *p;
+ }
+ else {
+ msg_info_spf("spf error for domain %s: unknown or "
+ "unsupported spf macro %c in %s",
+ rec->sender_domain,
+ *p,
+ begin);
+ return begin;
+ }
+ p++;
+ break;
+ }
+ }
+ /* Null terminate */
+ *c = '\0';
+
+ return new;
+}
+
+/* Read current element and try to parse record */
+static gboolean
+spf_process_element(struct spf_record *rec,
+ struct spf_resolved_element *resolved,
+ const gchar *elt,
+ const gchar **elts)
+{
+ struct spf_addr *addr = NULL;
+ gboolean res = FALSE;
+ const gchar *begin;
+ gchar t;
+
+ g_assert(elt != NULL);
+ g_assert(rec != NULL);
+
+ if (*elt == '\0' || resolved->redirected) {
+ return TRUE;
+ }
+
+ begin = expand_spf_macro(rec, resolved, elt);
+ addr = rspamd_spf_new_addr(rec, resolved, begin);
+ g_assert(addr != NULL);
+ t = g_ascii_tolower(addr->spf_string[0]);
+ begin = addr->spf_string;
+
+ /* Now check what we have */
+ switch (t) {
+ case 'a':
+ /* all or a */
+ if (g_ascii_strncasecmp(begin, SPF_ALL,
+ sizeof(SPF_ALL) - 1) == 0) {
+ res = parse_spf_all(rec, addr);
+ }
+ else if (g_ascii_strncasecmp(begin, SPF_A,
+ sizeof(SPF_A) - 1) == 0) {
+ res = parse_spf_a(rec, resolved, addr);
+ }
+ else {
+ msg_notice_spf("spf error for domain %s: bad spf command %s",
+ rec->sender_domain, begin);
+ }
+ break;
+ case 'i':
+ /* include or ip4 */
+ if (g_ascii_strncasecmp(begin, SPF_IP4, sizeof(SPF_IP4) - 1) == 0) {
+ res = parse_spf_ip4(rec, addr);
+ }
+ else if (g_ascii_strncasecmp(begin, SPF_INCLUDE, sizeof(SPF_INCLUDE) - 1) == 0) {
+ res = parse_spf_include(rec, addr);
+ }
+ else if (g_ascii_strncasecmp(begin, SPF_IP6, sizeof(SPF_IP6) - 1) == 0) {
+ res = parse_spf_ip6(rec, addr);
+ }
+ else if (g_ascii_strncasecmp(begin, SPF_IP4_ALT, sizeof(SPF_IP4_ALT) - 1) == 0) {
+ res = parse_spf_ip4(rec, addr);
+ }
+ else if (g_ascii_strncasecmp(begin, SPF_IP6_ALT, sizeof(SPF_IP6_ALT) - 1) == 0) {
+ res = parse_spf_ip6(rec, addr);
+ }
+ else {
+ msg_notice_spf("spf error for domain %s: bad spf command %s",
+ rec->sender_domain, begin);
+ }
+ break;
+ case 'm':
+ /* mx */
+ if (g_ascii_strncasecmp(begin, SPF_MX, sizeof(SPF_MX) - 1) == 0) {
+ res = parse_spf_mx(rec, resolved, addr);
+ }
+ else {
+ msg_notice_spf("spf error for domain %s: bad spf command %s",
+ rec->sender_domain, begin);
+ }
+ break;
+ case 'p':
+ /* ptr */
+ if (g_ascii_strncasecmp(begin, SPF_PTR,
+ sizeof(SPF_PTR) - 1) == 0) {
+ res = parse_spf_ptr(rec, resolved, addr);
+ }
+ else {
+ msg_notice_spf("spf error for domain %s: bad spf command %s",
+ rec->sender_domain, begin);
+ }
+ break;
+ case 'e':
+ /* exp or exists */
+ if (g_ascii_strncasecmp(begin, SPF_EXP,
+ sizeof(SPF_EXP) - 1) == 0) {
+ res = parse_spf_exp(rec, addr);
+ }
+ else if (g_ascii_strncasecmp(begin, SPF_EXISTS,
+ sizeof(SPF_EXISTS) - 1) == 0) {
+ res = parse_spf_exists(rec, addr);
+ }
+ else {
+ msg_notice_spf("spf error for domain %s: bad spf command %s",
+ rec->sender_domain, begin);
+ }
+ break;
+ case 'r':
+ /* redirect */
+ if (g_ascii_strncasecmp(begin, SPF_REDIRECT,
+ sizeof(SPF_REDIRECT) - 1) == 0) {
+ /*
+ * According to https://tools.ietf.org/html/rfc7208#section-6.1
+ * There must be no ALL element anywhere in the record,
+ * redirect must be ignored
+ */
+ gboolean ignore_redirect = FALSE;
+
+ for (const gchar **tmp = elts; *tmp != NULL; tmp++) {
+ if (g_ascii_strcasecmp((*tmp) + 1, "all") == 0) {
+ ignore_redirect = TRUE;
+ break;
+ }
+ }
+
+ if (!ignore_redirect) {
+ res = parse_spf_redirect(rec, resolved, addr);
+ }
+ else {
+ msg_notice_spf("ignore SPF redirect (%s) for domain %s as there is also all element",
+ begin, rec->sender_domain);
+
+ /* Pop the current addr as it is ignored */
+ g_ptr_array_remove_index_fast(resolved->elts,
+ resolved->elts->len - 1);
+
+ return TRUE;
+ }
+ }
+ else {
+ msg_notice_spf("spf error for domain %s: bad spf command %s",
+ rec->sender_domain, begin);
+ }
+ break;
+ case 'v':
+ if (g_ascii_strncasecmp(begin, "v=spf",
+ sizeof("v=spf") - 1) == 0) {
+ /* Skip this element till the end of record */
+ while (*begin && !g_ascii_isspace(*begin)) {
+ begin++;
+ }
+ }
+ break;
+ default:
+ msg_notice_spf("spf error for domain %s: bad spf command %s",
+ rec->sender_domain, begin);
+ break;
+ }
+
+ if (res) {
+ addr->flags |= RSPAMD_SPF_FLAG_PARSED;
+ }
+
+ return res;
+}
+
+static void
+parse_spf_scopes(struct spf_record *rec, gchar **begin)
+{
+ for (;;) {
+ if (g_ascii_strncasecmp(*begin, SPF_SCOPE_PRA, sizeof(SPF_SCOPE_PRA) - 1) == 0) {
+ *begin += sizeof(SPF_SCOPE_PRA) - 1;
+ /* XXX: Implement actual PRA check */
+ /* extract_pra_info (rec); */
+ continue;
+ }
+ else if (g_ascii_strncasecmp(*begin, SPF_SCOPE_MFROM,
+ sizeof(SPF_SCOPE_MFROM) - 1) == 0) {
+ /* mfrom is standard spf1 check */
+ *begin += sizeof(SPF_SCOPE_MFROM) - 1;
+ continue;
+ }
+ else if (**begin != ',') {
+ break;
+ }
+ (*begin)++;
+ }
+}
+
+static gboolean
+start_spf_parse(struct spf_record *rec, struct spf_resolved_element *resolved,
+ gchar *begin)
+{
+ gchar **elts, **cur_elt;
+ gsize len;
+
+ /* Skip spaces */
+ while (g_ascii_isspace(*begin)) {
+ begin++;
+ }
+
+ len = strlen(begin);
+
+ if (g_ascii_strncasecmp(begin, SPF_VER1_STR, sizeof(SPF_VER1_STR) - 1) ==
+ 0) {
+ begin += sizeof(SPF_VER1_STR) - 1;
+
+ while (g_ascii_isspace(*begin) && *begin) {
+ begin++;
+ }
+ }
+ else if (g_ascii_strncasecmp(begin, SPF_VER2_STR, sizeof(SPF_VER2_STR) - 1) == 0) {
+ /* Skip one number of record, so no we are here spf2.0/ */
+ begin += sizeof(SPF_VER2_STR);
+ if (*begin != '/') {
+ msg_notice_spf("spf error for domain %s: sender id is invalid",
+ rec->sender_domain);
+ }
+ else {
+ begin++;
+ parse_spf_scopes(rec, &begin);
+ }
+ /* Now common spf record */
+ }
+ else {
+ msg_debug_spf(
+ "spf error for domain %s: bad spf record start: %*s",
+ rec->sender_domain,
+ (gint) len,
+ begin);
+
+ return FALSE;
+ }
+
+ while (g_ascii_isspace(*begin) && *begin) {
+ begin++;
+ }
+
+ elts = g_strsplit_set(begin, " ", 0);
+
+ if (elts) {
+ cur_elt = elts;
+
+ while (*cur_elt) {
+ spf_process_element(rec, resolved, *cur_elt, (const gchar **) elts);
+ cur_elt++;
+ }
+
+ g_strfreev(elts);
+ }
+
+ rspamd_spf_maybe_return(rec);
+
+ return TRUE;
+}
+
+static void
+spf_dns_callback(struct rdns_reply *reply, gpointer arg)
+{
+ struct spf_record *rec = arg;
+ struct spf_resolved_element *resolved = NULL;
+ struct spf_addr *addr;
+
+ rec->requests_inflight--;
+
+ if (reply->flags & RDNS_TRUNCATED) {
+ msg_warn_spf("got a truncated record when trying to resolve TXT record for %s",
+ rec->sender_domain);
+ resolved = rspamd_spf_new_addr_list(rec, rec->sender_domain);
+ addr = g_malloc0(sizeof(*addr));
+ addr->flags |= RSPAMD_SPF_FLAG_TEMPFAIL;
+ g_ptr_array_insert(resolved->elts, 0, addr);
+
+ rspamd_spf_maybe_return(rec);
+
+ return;
+ }
+ else {
+ if (reply->code == RDNS_RC_NOERROR) {
+ resolved = rspamd_spf_new_addr_list(rec, rec->sender_domain);
+ if (rec->resolved->len == 1) {
+ /* Top level resolved element */
+ rec->ttl = reply->entries->ttl;
+ }
+ }
+ else if ((reply->code == RDNS_RC_NOREC || reply->code == RDNS_RC_NXDOMAIN) && rec->dns_requests == 0) {
+ resolved = rspamd_spf_new_addr_list(rec, rec->sender_domain);
+ addr = g_malloc0(sizeof(*addr));
+ addr->flags |= RSPAMD_SPF_FLAG_NA;
+ g_ptr_array_insert(resolved->elts, 0, addr);
+ }
+ else if (reply->code != RDNS_RC_NOREC && reply->code != RDNS_RC_NXDOMAIN && rec->dns_requests == 0) {
+ resolved = rspamd_spf_new_addr_list(rec, rec->sender_domain);
+ addr = g_malloc0(sizeof(*addr));
+ addr->flags |= RSPAMD_SPF_FLAG_TEMPFAIL;
+ g_ptr_array_insert(resolved->elts, 0, addr);
+ }
+ }
+
+ if (resolved) {
+ struct rdns_reply_entry *selected = NULL;
+
+ if (!spf_process_txt_record(rec, resolved, reply, &selected)) {
+ resolved = g_ptr_array_index(rec->resolved, 0);
+
+ if (rec->resolved->len > 1) {
+ addr = g_ptr_array_index(resolved->elts, 0);
+ if ((reply->code == RDNS_RC_NOREC || reply->code == RDNS_RC_NXDOMAIN) && (addr->flags & RSPAMD_SPF_FLAG_REDIRECT)) {
+ addr->flags |= RSPAMD_SPF_FLAG_PERMFAIL;
+ }
+ else {
+ addr->flags |= RSPAMD_SPF_FLAG_TEMPFAIL;
+ }
+ }
+ else {
+ addr = g_malloc0(sizeof(*addr));
+
+ if (reply->code == RDNS_RC_NOREC || reply->code == RDNS_RC_NXDOMAIN || reply->code == RDNS_RC_NOERROR) {
+ addr->flags |= RSPAMD_SPF_FLAG_NA;
+ }
+ else {
+ addr->flags |= RSPAMD_SPF_FLAG_TEMPFAIL;
+ }
+ g_ptr_array_insert(resolved->elts, 0, addr);
+ }
+ }
+ else {
+ rec->top_record = rspamd_mempool_strdup(rec->task->task_pool,
+ selected->content.txt.data);
+ rspamd_mempool_set_variable(rec->task->task_pool,
+ RSPAMD_MEMPOOL_SPF_RECORD,
+ (gpointer) rec->top_record, NULL);
+ }
+ }
+
+ rspamd_spf_maybe_return(rec);
+}
+
+static struct rspamd_spf_cred *
+rspamd_spf_cache_domain(struct rspamd_task *task)
+{
+ struct rspamd_email_address *addr;
+ struct rspamd_spf_cred *cred = NULL;
+
+ addr = rspamd_task_get_sender(task);
+ if (!addr || (addr->flags & RSPAMD_EMAIL_ADDR_EMPTY)) {
+ /* Get domain from helo */
+
+ if (task->helo) {
+ GString *fs = g_string_new("");
+
+ cred = rspamd_mempool_alloc(task->task_pool, sizeof(*cred));
+ cred->domain = task->helo;
+ cred->local_part = "postmaster";
+ rspamd_printf_gstring(fs, "postmaster@%s", cred->domain);
+ cred->sender = fs->str;
+ rspamd_mempool_add_destructor(task->task_pool,
+ rspamd_gstring_free_hard, fs);
+ }
+ }
+ else {
+ rspamd_ftok_t tok;
+
+ cred = rspamd_mempool_alloc(task->task_pool, sizeof(*cred));
+ tok.begin = addr->domain;
+ tok.len = addr->domain_len;
+ cred->domain = rspamd_mempool_ftokdup(task->task_pool, &tok);
+ tok.begin = addr->user;
+ tok.len = addr->user_len;
+ cred->local_part = rspamd_mempool_ftokdup(task->task_pool, &tok);
+ tok.begin = addr->addr;
+ tok.len = addr->addr_len;
+ cred->sender = rspamd_mempool_ftokdup(task->task_pool, &tok);
+ }
+
+ if (cred) {
+ rspamd_mempool_set_variable(task->task_pool, RSPAMD_MEMPOOL_SPF_DOMAIN,
+ cred, NULL);
+ }
+
+ return cred;
+}
+
+struct rspamd_spf_cred *
+rspamd_spf_get_cred(struct rspamd_task *task)
+{
+ struct rspamd_spf_cred *cred;
+
+ cred = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_SPF_DOMAIN);
+
+ if (!cred) {
+ cred = rspamd_spf_cache_domain(task);
+ }
+
+ return cred;
+}
+
+const gchar *
+rspamd_spf_get_domain(struct rspamd_task *task)
+{
+ gchar *domain = NULL;
+ struct rspamd_spf_cred *cred;
+
+ cred = rspamd_spf_get_cred(task);
+
+ if (cred) {
+ domain = cred->domain;
+ }
+
+ return domain;
+}
+
+gboolean
+rspamd_spf_resolve(struct rspamd_task *task, spf_cb_t callback,
+ gpointer cbdata, struct rspamd_spf_cred *cred)
+{
+ struct spf_record *rec;
+
+ if (!cred || !cred->domain) {
+ return FALSE;
+ }
+
+ /* First lookup in the hash */
+ if (spf_lib_ctx->spf_hash) {
+ struct spf_resolved *cached;
+
+ cached = rspamd_lru_hash_lookup(spf_lib_ctx->spf_hash, cred->domain,
+ task->task_timestamp);
+
+ if (cached) {
+ cached->flags |= RSPAMD_SPF_FLAG_CACHED;
+
+ if (cached->top_record) {
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_SPF_RECORD,
+ rspamd_mempool_strdup(task->task_pool,
+ cached->top_record),
+ NULL);
+ }
+ callback(cached, task, cbdata);
+
+ return TRUE;
+ }
+ }
+
+
+ rec = rspamd_mempool_alloc0(task->task_pool, sizeof(struct spf_record));
+ rec->task = task;
+ rec->callback = callback;
+ rec->cbdata = cbdata;
+
+ rec->resolved = g_ptr_array_sized_new(8);
+
+ /* Add destructor */
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) spf_record_destructor,
+ rec);
+
+ /* Extract from data */
+ rec->sender = cred->sender;
+ rec->local_part = cred->local_part;
+ rec->sender_domain = cred->domain;
+
+ if (rspamd_dns_resolver_request_task_forced(task,
+ spf_dns_callback,
+ (void *) rec, RDNS_REQUEST_TXT, rec->sender_domain)) {
+ rec->requests_inflight++;
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+struct spf_resolved *
+_spf_record_ref(struct spf_resolved *flat, const gchar *loc)
+{
+ REF_RETAIN(flat);
+ return flat;
+}
+
+void _spf_record_unref(struct spf_resolved *flat, const gchar *loc)
+{
+ REF_RELEASE(flat);
+}
+
+gchar *
+spf_addr_mask_to_string(struct spf_addr *addr)
+{
+ GString *res;
+ gchar *s, ipstr[INET6_ADDRSTRLEN + 1];
+
+ if (addr->flags & RSPAMD_SPF_FLAG_ANY) {
+ res = g_string_new("any");
+ }
+ else if (addr->flags & RSPAMD_SPF_FLAG_IPV4) {
+ (void) inet_ntop(AF_INET, addr->addr4, ipstr, sizeof(ipstr));
+ res = g_string_sized_new(sizeof(ipstr));
+ rspamd_printf_gstring(res, "%s/%d", ipstr, addr->m.dual.mask_v4);
+ }
+ else if (addr->flags & RSPAMD_SPF_FLAG_IPV6) {
+ (void) inet_ntop(AF_INET6, addr->addr6, ipstr, sizeof(ipstr));
+ res = g_string_sized_new(sizeof(ipstr));
+ rspamd_printf_gstring(res, "%s/%d", ipstr, addr->m.dual.mask_v6);
+ }
+ else {
+ res = g_string_new(NULL);
+ rspamd_printf_gstring(res, "unknown, flags = %d", addr->flags);
+ }
+
+ s = res->str;
+ g_string_free(res, FALSE);
+
+
+ return s;
+}
+
+struct spf_addr *
+spf_addr_match_task(struct rspamd_task *task, struct spf_resolved *rec)
+{
+ const guint8 *s, *d;
+ guint af, mask, bmask, addrlen;
+ struct spf_addr *selected = NULL, *addr, *any_addr = NULL;
+ guint i;
+
+ if (task->from_addr == NULL) {
+ return FALSE;
+ }
+
+ for (i = 0; i < rec->elts->len; i++) {
+ addr = &g_array_index(rec->elts, struct spf_addr, i);
+ if (addr->flags & RSPAMD_SPF_FLAG_TEMPFAIL) {
+ continue;
+ }
+
+ af = rspamd_inet_address_get_af(task->from_addr);
+ /* Basic comparing algorithm */
+ if (((addr->flags & RSPAMD_SPF_FLAG_IPV6) && af == AF_INET6) ||
+ ((addr->flags & RSPAMD_SPF_FLAG_IPV4) && af == AF_INET)) {
+ d = rspamd_inet_address_get_hash_key(task->from_addr, &addrlen);
+
+ if (af == AF_INET6) {
+ s = (const guint8 *) addr->addr6;
+ mask = addr->m.dual.mask_v6;
+ }
+ else {
+ s = (const guint8 *) addr->addr4;
+ mask = addr->m.dual.mask_v4;
+ }
+
+ /* Compare the first bytes */
+ bmask = mask / CHAR_BIT;
+ if (mask > addrlen * CHAR_BIT) {
+ msg_info_task("bad mask length: %d", mask);
+ }
+ else if (memcmp(s, d, bmask) == 0) {
+ if (bmask * CHAR_BIT < mask) {
+ /* Compare the remaining bits */
+ s += bmask;
+ d += bmask;
+ mask = (0xffu << (CHAR_BIT - (mask - bmask * 8u))) & 0xffu;
+
+ if ((*s & mask) == (*d & mask)) {
+ selected = addr;
+ break;
+ }
+ }
+ else {
+ selected = addr;
+ break;
+ }
+ }
+ }
+ else {
+ if (addr->flags & RSPAMD_SPF_FLAG_ANY) {
+ any_addr = addr;
+ }
+ }
+ }
+
+ if (selected) {
+ return selected;
+ }
+
+ return any_addr;
+} \ No newline at end of file
diff --git a/src/libserver/spf.h b/src/libserver/spf.h
new file mode 100644
index 0000000..871ed29
--- /dev/null
+++ b/src/libserver/spf.h
@@ -0,0 +1,159 @@
+#ifndef RSPAMD_SPF_H
+#define RSPAMD_SPF_H
+
+#include "config.h"
+#include "ref.h"
+#include "addr.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct spf_resolved;
+
+typedef void (*spf_cb_t)(struct spf_resolved *record,
+ struct rspamd_task *task, gpointer cbdata);
+
+typedef enum spf_mech_e {
+ SPF_FAIL,
+ SPF_SOFT_FAIL,
+ SPF_PASS,
+ SPF_NEUTRAL
+} spf_mech_t;
+
+static inline gchar spf_mech_char(spf_mech_t mech)
+{
+ switch (mech) {
+ case SPF_FAIL:
+ return '-';
+ case SPF_SOFT_FAIL:
+ return '~';
+ case SPF_PASS:
+ return '+';
+ case SPF_NEUTRAL:
+ default:
+ return '?';
+ }
+}
+
+typedef enum spf_action_e {
+ SPF_RESOLVE_MX,
+ SPF_RESOLVE_A,
+ SPF_RESOLVE_PTR,
+ SPF_RESOLVE_AAA,
+ SPF_RESOLVE_REDIRECT,
+ SPF_RESOLVE_INCLUDE,
+ SPF_RESOLVE_EXISTS,
+ SPF_RESOLVE_EXP
+} spf_action_t;
+
+#define RSPAMD_SPF_FLAG_IPV6 (1u << 0u)
+#define RSPAMD_SPF_FLAG_IPV4 (1u << 1u)
+#define RSPAMD_SPF_FLAG_PROCESSED (1u << 2u)
+#define RSPAMD_SPF_FLAG_ANY (1u << 3u)
+#define RSPAMD_SPF_FLAG_PARSED (1u << 4u)
+#define RSPAMD_SPF_FLAG_INVALID (1u << 5u)
+#define RSPAMD_SPF_FLAG_REFERENCE (1u << 6u)
+#define RSPAMD_SPF_FLAG_REDIRECT (1u << 7u)
+#define RSPAMD_SPF_FLAG_TEMPFAIL (1u << 8u)
+#define RSPAMD_SPF_FLAG_NA (1u << 9u)
+#define RSPAMD_SPF_FLAG_PERMFAIL (1u << 10u)
+#define RSPAMD_SPF_FLAG_RESOLVED (1u << 11u)
+#define RSPAMD_SPF_FLAG_CACHED (1u << 12u)
+
+/** Default SPF limits for avoiding abuse **/
+#define SPF_MAX_NESTING 10
+#define SPF_MAX_DNS_REQUESTS 30
+#define SPF_MIN_CACHE_TTL (60 * 5) /* 5 minutes */
+
+struct spf_addr {
+ guchar addr6[sizeof(struct in6_addr)];
+ guchar addr4[sizeof(struct in_addr)];
+ union {
+ struct {
+ guint16 mask_v4;
+ guint16 mask_v6;
+ } dual;
+ guint32 idx;
+ } m;
+ guint flags;
+ spf_mech_t mech;
+ gchar *spf_string;
+ struct spf_addr *prev, *next;
+};
+
+enum rspamd_spf_resolved_flags {
+ RSPAMD_SPF_RESOLVED_NORMAL = 0,
+ RSPAMD_SPF_RESOLVED_TEMP_FAILED = (1u << 0u),
+ RSPAMD_SPF_RESOLVED_PERM_FAILED = (1u << 1u),
+ RSPAMD_SPF_RESOLVED_NA = (1u << 2u),
+};
+
+struct spf_resolved {
+ gchar *domain;
+ gchar *top_record;
+ guint ttl;
+ gint flags;
+ gdouble timestamp;
+ guint64 digest;
+ GArray *elts; /* Flat list of struct spf_addr */
+ ref_entry_t ref; /* Refcounting */
+};
+
+struct rspamd_spf_cred {
+ gchar *local_part;
+ gchar *domain;
+ gchar *sender;
+};
+
+/*
+ * Resolve spf record for specified task and call a callback after resolution fails/succeed
+ */
+gboolean rspamd_spf_resolve(struct rspamd_task *task,
+ spf_cb_t callback,
+ gpointer cbdata,
+ struct rspamd_spf_cred *cred);
+
+/*
+ * Get a domain for spf for specified task
+ */
+const gchar *rspamd_spf_get_domain(struct rspamd_task *task);
+
+struct rspamd_spf_cred *rspamd_spf_get_cred(struct rspamd_task *task);
+/*
+ * Increase refcount
+ */
+struct spf_resolved *_spf_record_ref(struct spf_resolved *rec, const gchar *loc);
+#define spf_record_ref(rec) \
+ _spf_record_ref((rec), G_STRLOC)
+/*
+ * Decrease refcount
+ */
+void _spf_record_unref(struct spf_resolved *rec, const gchar *loc);
+#define spf_record_unref(rec) \
+ _spf_record_unref((rec), G_STRLOC)
+
+/**
+ * Prints address + mask in a freshly allocated string (must be freed)
+ * @param addr
+ * @return
+ */
+gchar *spf_addr_mask_to_string(struct spf_addr *addr);
+
+/**
+ * Returns spf address that matches the specific task (or nil if not matched)
+ * @param task
+ * @param rec
+ * @return
+ */
+struct spf_addr *spf_addr_match_task(struct rspamd_task *task,
+ struct spf_resolved *rec);
+
+void spf_library_config(const ucl_object_t *obj);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/ssl_util.c b/src/libserver/ssl_util.c
new file mode 100644
index 0000000..8ee53b0
--- /dev/null
+++ b/src/libserver/ssl_util.c
@@ -0,0 +1,1133 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "libutil/util.h"
+#include "libutil/hash.h"
+#include "libserver/logger.h"
+#include "libserver/cfg_file.h"
+#include "ssl_util.h"
+#include "unix-std.h"
+#include "cryptobox.h"
+#include "contrib/libottery/ottery.h"
+
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#include <openssl/rand.h>
+#include <openssl/conf.h>
+#include <openssl/evp.h>
+#include <openssl/engine.h>
+#include <openssl/x509v3.h>
+
+enum rspamd_ssl_state {
+ ssl_conn_reset = 0,
+ ssl_conn_init,
+ ssl_conn_connected,
+ ssl_next_read,
+ ssl_next_write,
+ ssl_next_shutdown,
+};
+
+enum rspamd_ssl_shutdown {
+ ssl_shut_default = 0,
+ ssl_shut_unclean,
+};
+
+struct rspamd_ssl_ctx {
+ SSL_CTX *s;
+ rspamd_lru_hash_t *sessions;
+};
+
+struct rspamd_ssl_connection {
+ gint fd;
+ enum rspamd_ssl_state state;
+ enum rspamd_ssl_shutdown shut;
+ gboolean verify_peer;
+ SSL *ssl;
+ struct rspamd_ssl_ctx *ssl_ctx;
+ gchar *hostname;
+ struct rspamd_io_ev *ev;
+ struct rspamd_io_ev *shut_ev;
+ struct ev_loop *event_loop;
+ rspamd_ssl_handler_t handler;
+ rspamd_ssl_error_handler_t err_handler;
+ gpointer handler_data;
+ gchar log_tag[8];
+};
+
+#define msg_debug_ssl(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_ssl_log_id, "ssl", conn->log_tag, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+static void rspamd_ssl_event_handler(gint fd, short what, gpointer ud);
+
+INIT_LOG_MODULE(ssl)
+
+static GQuark
+rspamd_ssl_quark(void)
+{
+ return g_quark_from_static_string("rspamd-ssl");
+}
+
+#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(LIBRESSL_VERSION_NUMBER)
+#ifndef X509_get_notBefore
+#define X509_get_notBefore(x) X509_get0_notBefore(x)
+#endif
+#ifndef X509_get_notAfter
+#define X509_get_notAfter(x) X509_get0_notAfter(x)
+#endif
+#ifndef ASN1_STRING_data
+#define ASN1_STRING_data(x) ASN1_STRING_get0_data(x)
+#endif
+#endif
+
+/* $OpenBSD: tls_verify.c,v 1.14 2015/09/29 10:17:04 deraadt Exp $ */
+/*
+ * Copyright (c) 2014 Jeremie Courreges-Anglas <jca@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+static gboolean
+rspamd_tls_match_name(const char *cert_name, const char *name)
+{
+ const char *cert_domain, *domain, *next_dot;
+
+ if (g_ascii_strcasecmp(cert_name, name) == 0) {
+ return TRUE;
+ }
+
+ /* Wildcard match? */
+ if (cert_name[0] == '*') {
+ /*
+ * Valid wildcards:
+ * - "*.domain.tld"
+ * - "*.sub.domain.tld"
+ * - etc.
+ * Reject "*.tld".
+ * No attempt to prevent the use of eg. "*.co.uk".
+ */
+ cert_domain = &cert_name[1];
+ /* Disallow "*" */
+ if (cert_domain[0] == '\0') {
+ return FALSE;
+ }
+
+ /* Disallow "*foo" */
+ if (cert_domain[0] != '.') {
+ return FALSE;
+ }
+ /* Disallow "*.." */
+ if (cert_domain[1] == '.') {
+ return FALSE;
+ }
+ next_dot = strchr(&cert_domain[1], '.');
+ /* Disallow "*.bar" */
+ if (next_dot == NULL) {
+ return FALSE;
+ }
+ /* Disallow "*.bar.." */
+ if (next_dot[1] == '.') {
+ return FALSE;
+ }
+
+ domain = strchr(name, '.');
+
+ /* No wildcard match against a name with no host part. */
+ if (name[0] == '.') {
+ return FALSE;
+ }
+ /* No wildcard match against a name with no domain part. */
+ if (domain == NULL || strlen(domain) == 1) {
+ return FALSE;
+ }
+
+ if (g_ascii_strcasecmp(cert_domain, domain) == 0) {
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+/* See RFC 5280 section 4.2.1.6 for SubjectAltName details. */
+static gboolean
+rspamd_tls_check_subject_altname(X509 *cert, const char *name)
+{
+ STACK_OF(GENERAL_NAME) *altname_stack = NULL;
+ int addrlen, type;
+ int count, i;
+ union {
+ struct in_addr ip4;
+ struct in6_addr ip6;
+ } addrbuf;
+ gboolean ret = FALSE;
+
+ altname_stack = X509_get_ext_d2i(cert, NID_subject_alt_name, NULL, NULL);
+
+ if (altname_stack == NULL) {
+ return FALSE;
+ }
+
+ if (inet_pton(AF_INET, name, &addrbuf) == 1) {
+ type = GEN_IPADD;
+ addrlen = 4;
+ }
+ else if (inet_pton(AF_INET6, name, &addrbuf) == 1) {
+ type = GEN_IPADD;
+ addrlen = 16;
+ }
+ else {
+ type = GEN_DNS;
+ addrlen = 0;
+ }
+
+ count = sk_GENERAL_NAME_num(altname_stack);
+
+ for (i = 0; i < count; i++) {
+ GENERAL_NAME *altname;
+
+ altname = sk_GENERAL_NAME_value(altname_stack, i);
+
+ if (altname->type != type) {
+ continue;
+ }
+
+ if (type == GEN_DNS) {
+ const char *data;
+ int format, len;
+
+ format = ASN1_STRING_type(altname->d.dNSName);
+
+ if (format == V_ASN1_IA5STRING) {
+ data = (const char *) ASN1_STRING_data(altname->d.dNSName);
+ len = ASN1_STRING_length(altname->d.dNSName);
+
+ if (len < 0 || len != (gint) strlen(data)) {
+ ret = FALSE;
+ break;
+ }
+
+ /*
+ * Per RFC 5280 section 4.2.1.6:
+ * " " is a legal domain name, but that
+ * dNSName must be rejected.
+ */
+ if (strcmp(data, " ") == 0) {
+ ret = FALSE;
+ break;
+ }
+
+ if (rspamd_tls_match_name(data, name)) {
+ ret = TRUE;
+ break;
+ }
+ }
+ }
+ else if (type == GEN_IPADD) {
+ const char *data;
+ int datalen;
+
+ datalen = ASN1_STRING_length(altname->d.iPAddress);
+ data = (const char *) ASN1_STRING_data(altname->d.iPAddress);
+
+ if (datalen < 0) {
+ ret = FALSE;
+ break;
+ }
+
+ /*
+ * Per RFC 5280 section 4.2.1.6:
+ * IPv4 must use 4 octets and IPv6 must use 16 octets.
+ */
+ if (datalen == addrlen && memcmp(data, &addrbuf, addrlen) == 0) {
+ ret = TRUE;
+ break;
+ }
+ }
+ }
+
+ sk_GENERAL_NAME_pop_free(altname_stack, GENERAL_NAME_free);
+ return ret;
+}
+
+static gboolean
+rspamd_tls_check_common_name(X509 *cert, const char *name)
+{
+ X509_NAME *subject_name;
+ char *common_name = NULL;
+ union {
+ struct in_addr ip4;
+ struct in6_addr ip6;
+ } addrbuf;
+ int common_name_len;
+ gboolean ret = FALSE;
+
+ subject_name = X509_get_subject_name(cert);
+ if (subject_name == NULL) {
+ goto out;
+ }
+
+ common_name_len = X509_NAME_get_text_by_NID(subject_name, NID_commonName, NULL, 0);
+
+ if (common_name_len < 0) {
+ goto out;
+ }
+
+ common_name = g_malloc0(common_name_len + 1);
+ X509_NAME_get_text_by_NID(subject_name, NID_commonName, common_name,
+ common_name_len + 1);
+
+ /* NUL bytes in CN? */
+ if (common_name_len != (gint) strlen(common_name)) {
+ goto out;
+ }
+
+ if (inet_pton(AF_INET, name, &addrbuf) == 1 || inet_pton(AF_INET6, name, &addrbuf) == 1) {
+ /*
+ * We don't want to attempt wildcard matching against IP
+ * addresses, so perform a simple comparison here.
+ */
+ if (strcmp(common_name, name) == 0) {
+ ret = TRUE;
+ }
+ else {
+ ret = FALSE;
+ }
+
+ goto out;
+ }
+
+ if (rspamd_tls_match_name(common_name, name)) {
+ ret = TRUE;
+ }
+
+out:
+ g_free(common_name);
+
+ return ret;
+}
+
+static gboolean
+rspamd_tls_check_name(X509 *cert, const char *name)
+{
+ gboolean ret;
+
+ ret = rspamd_tls_check_subject_altname(cert, name);
+ if (ret) {
+ return ret;
+ }
+
+ return rspamd_tls_check_common_name(cert, name);
+}
+
+static gboolean
+rspamd_ssl_peer_verify(struct rspamd_ssl_connection *c)
+{
+ X509 *server_cert;
+ glong ver_err;
+ GError *err = NULL;
+
+ ver_err = SSL_get_verify_result(c->ssl);
+
+ if (ver_err != X509_V_OK) {
+ g_set_error(&err, rspamd_ssl_quark(), 400, "certificate validation "
+ "failed: %s",
+ X509_verify_cert_error_string(ver_err));
+ c->err_handler(c->handler_data, err);
+ g_error_free(err);
+
+ return FALSE;
+ }
+
+ /* Get server's certificate */
+ server_cert = SSL_get_peer_certificate(c->ssl);
+ if (server_cert == NULL) {
+ g_set_error(&err, rspamd_ssl_quark(), 401, "peer certificate is absent");
+ c->err_handler(c->handler_data, err);
+ g_error_free(err);
+
+ return FALSE;
+ }
+
+ if (c->hostname) {
+ if (!rspamd_tls_check_name(server_cert, c->hostname)) {
+ X509_free(server_cert);
+ g_set_error(&err, rspamd_ssl_quark(), 403, "peer certificate fails "
+ "hostname verification for %s",
+ c->hostname);
+ c->err_handler(c->handler_data, err);
+ g_error_free(err);
+
+ return FALSE;
+ }
+ }
+
+ X509_free(server_cert);
+
+ return TRUE;
+}
+
+static void
+rspamd_tls_set_error(gint retcode, const gchar *stage, GError **err)
+{
+ GString *reason;
+ gchar buf[120];
+ gint err_code = 0;
+
+ reason = g_string_sized_new(sizeof(buf));
+
+ if (retcode == SSL_ERROR_SYSCALL) {
+ rspamd_printf_gstring(reason, "syscall fail: %s", strerror(errno));
+ err_code = 500;
+ }
+ else {
+ while ((err_code = ERR_get_error()) != 0) {
+ ERR_error_string(err_code, buf);
+ rspamd_printf_gstring(reason, "ssl error: %s,", buf);
+ }
+
+ err_code = 400;
+
+ if (reason->len > 0 && reason->str[reason->len - 1] == ',') {
+ reason->str[reason->len - 1] = '\0';
+ reason->len--;
+ }
+ }
+
+ g_set_error(err, rspamd_ssl_quark(), err_code,
+ "ssl %s error: %s", stage, reason->str);
+ g_string_free(reason, TRUE);
+}
+
+static void
+rspamd_ssl_connection_dtor(struct rspamd_ssl_connection *conn)
+{
+ msg_debug_ssl("closing SSL connection %p; %d sessions in the cache",
+ conn->ssl, rspamd_lru_hash_size(conn->ssl_ctx->sessions));
+ SSL_free(conn->ssl);
+
+ if (conn->hostname) {
+ g_free(conn->hostname);
+ }
+
+ /*
+ * Try to workaround for the race between timeout and ssl error
+ */
+ if (conn->shut_ev != conn->ev && ev_can_stop(&conn->ev->tm)) {
+ rspamd_ev_watcher_stop(conn->event_loop, conn->ev);
+ }
+
+ if (conn->shut_ev) {
+ rspamd_ev_watcher_stop(conn->event_loop, conn->shut_ev);
+ g_free(conn->shut_ev);
+ }
+
+ close(conn->fd);
+ g_free(conn);
+}
+
+static void
+rspamd_ssl_shutdown(struct rspamd_ssl_connection *conn)
+{
+ gint ret = 0, nret, retries;
+ static const gint max_retries = 5;
+
+ /*
+ * Fucking openssl...
+ * From the manual, 0 means: "The shutdown is not yet finished.
+ * Call SSL_shutdown() for a second time,
+ * if a bidirectional shutdown shall be performed.
+ * The output of SSL_get_error(3) may be misleading,
+ * as an erroneous SSL_ERROR_SYSCALL may be flagged
+ * even though no error occurred."
+ *
+ * What is `second`, what if `second` also returns 0?
+ * What a retarded behaviour!
+ */
+ for (retries = 0; retries < max_retries; retries++) {
+ ret = SSL_shutdown(conn->ssl);
+
+ if (ret != 0) {
+ break;
+ }
+ }
+
+ if (ret == 1) {
+ /* All done */
+ msg_debug_ssl("ssl shutdown: all done");
+ rspamd_ssl_connection_dtor(conn);
+ }
+ else if (ret < 0) {
+ short what;
+
+ nret = SSL_get_error(conn->ssl, ret);
+ conn->state = ssl_next_shutdown;
+
+ if (nret == SSL_ERROR_WANT_READ) {
+ msg_debug_ssl("ssl shutdown: need read");
+ what = EV_READ;
+ }
+ else if (nret == SSL_ERROR_WANT_WRITE) {
+ msg_debug_ssl("ssl shutdown: need write");
+ what = EV_WRITE;
+ }
+ else {
+ /* Cannot do anything else, fatal error */
+ GError *err = NULL;
+
+ rspamd_tls_set_error(nret, "final shutdown", &err);
+ msg_debug_ssl("ssl shutdown: fatal error: %e; retries=%d; ret=%d",
+ err, retries, ret);
+ g_error_free(err);
+ rspamd_ssl_connection_dtor(conn);
+
+ return;
+ }
+
+ /* As we own fd, we can try to perform shutdown one more time */
+ /* BUGON: but we DO NOT own conn->ev, and it's a big issue */
+ static const ev_tstamp shutdown_time = 5.0;
+
+ if (conn->shut_ev == NULL) {
+ rspamd_ev_watcher_stop(conn->event_loop, conn->ev);
+ conn->shut_ev = g_malloc0(sizeof(*conn->shut_ev));
+ rspamd_ev_watcher_init(conn->shut_ev, conn->fd, what,
+ rspamd_ssl_event_handler, conn);
+ rspamd_ev_watcher_start(conn->event_loop, conn->shut_ev, shutdown_time);
+ /* XXX: can it be done safely ? */
+ conn->ev = conn->shut_ev;
+ }
+ else {
+ rspamd_ev_watcher_reschedule(conn->event_loop, conn->shut_ev, what);
+ }
+
+ conn->state = ssl_next_shutdown;
+ }
+ else if (ret == 0) {
+ /* What can we do here?? */
+ msg_debug_ssl("ssl shutdown: openssl failed to initiate shutdown after "
+ "%d attempts!",
+ max_retries);
+ rspamd_ssl_connection_dtor(conn);
+ }
+}
+
+static void
+rspamd_ssl_event_handler(gint fd, short what, gpointer ud)
+{
+ struct rspamd_ssl_connection *conn = ud;
+ gint ret;
+ GError *err = NULL;
+
+ if (what == EV_TIMER) {
+ if (conn->state == ssl_next_shutdown) {
+ /* No way to restore, just terminate */
+ rspamd_ssl_connection_dtor(conn);
+ }
+ else {
+ conn->shut = ssl_shut_unclean;
+ rspamd_ev_watcher_stop(conn->event_loop, conn->ev);
+ g_set_error(&err, rspamd_ssl_quark(), 408,
+ "ssl connection timed out");
+ conn->err_handler(conn->handler_data, err);
+ g_error_free(err);
+ }
+
+ return;
+ }
+
+ msg_debug_ssl("ssl event; what=%d; c->state=%d", (int) what,
+ (int) conn->state);
+
+ switch (conn->state) {
+ case ssl_conn_init:
+ /* Continue connection */
+ ret = SSL_connect(conn->ssl);
+
+ if (ret == 1) {
+ rspamd_ev_watcher_stop(conn->event_loop, conn->ev);
+ /* Verify certificate */
+ if ((!conn->verify_peer) || rspamd_ssl_peer_verify(conn)) {
+ msg_debug_ssl("ssl connect: connected");
+ conn->state = ssl_conn_connected;
+ conn->handler(fd, EV_WRITE, conn->handler_data);
+ }
+ else {
+ return;
+ }
+ }
+ else {
+ ret = SSL_get_error(conn->ssl, ret);
+
+ if (ret == SSL_ERROR_WANT_READ) {
+ msg_debug_ssl("ssl connect: need read");
+ what = EV_READ;
+ }
+ else if (ret == SSL_ERROR_WANT_WRITE) {
+ msg_debug_ssl("ssl connect: need write");
+ what = EV_WRITE;
+ }
+ else {
+ rspamd_ev_watcher_stop(conn->event_loop, conn->ev);
+ rspamd_tls_set_error(ret, "connect", &err);
+ conn->err_handler(conn->handler_data, err);
+ g_error_free(err);
+ return;
+ }
+
+ rspamd_ev_watcher_reschedule(conn->event_loop, conn->ev, what);
+ }
+ break;
+ case ssl_next_read:
+ rspamd_ev_watcher_reschedule(conn->event_loop, conn->ev, EV_READ);
+ conn->state = ssl_conn_connected;
+ conn->handler(fd, EV_READ, conn->handler_data);
+ break;
+ case ssl_next_write:
+ rspamd_ev_watcher_reschedule(conn->event_loop, conn->ev, EV_WRITE);
+ conn->state = ssl_conn_connected;
+ conn->handler(fd, EV_WRITE, conn->handler_data);
+ break;
+ case ssl_conn_connected:
+ rspamd_ev_watcher_reschedule(conn->event_loop, conn->ev, what);
+ conn->state = ssl_conn_connected;
+ conn->handler(fd, what, conn->handler_data);
+ break;
+ case ssl_next_shutdown:
+ rspamd_ssl_shutdown(conn);
+ break;
+ default:
+ rspamd_ev_watcher_stop(conn->event_loop, conn->ev);
+ g_set_error(&err, rspamd_ssl_quark(), 500,
+ "ssl bad state error: %d", conn->state);
+ conn->err_handler(conn->handler_data, err);
+ g_error_free(err);
+ break;
+ }
+}
+
+struct rspamd_ssl_connection *
+rspamd_ssl_connection_new(gpointer ssl_ctx, struct ev_loop *ev_base,
+ gboolean verify_peer, const gchar *log_tag)
+{
+ struct rspamd_ssl_connection *conn;
+ struct rspamd_ssl_ctx *ctx = (struct rspamd_ssl_ctx *) ssl_ctx;
+
+ g_assert(ssl_ctx != NULL);
+ conn = g_malloc0(sizeof(*conn));
+ conn->ssl_ctx = ctx;
+ conn->event_loop = ev_base;
+ conn->verify_peer = verify_peer;
+
+ if (log_tag) {
+ rspamd_strlcpy(conn->log_tag, log_tag, sizeof(conn->log_tag));
+ }
+ else {
+ rspamd_random_hex(conn->log_tag, sizeof(log_tag) - 1);
+ conn->log_tag[sizeof(log_tag) - 1] = '\0';
+ }
+
+ return conn;
+}
+
+
+gboolean
+rspamd_ssl_connect_fd(struct rspamd_ssl_connection *conn, gint fd,
+ const gchar *hostname, struct rspamd_io_ev *ev, ev_tstamp timeout,
+ rspamd_ssl_handler_t handler, rspamd_ssl_error_handler_t err_handler,
+ gpointer handler_data)
+{
+ gint ret;
+ SSL_SESSION *session = NULL;
+
+ g_assert(conn != NULL);
+
+ /* Ensure that we start from the empty SSL errors stack */
+ ERR_clear_error();
+ conn->ssl = SSL_new(conn->ssl_ctx->s);
+
+ if (hostname) {
+ session = rspamd_lru_hash_lookup(conn->ssl_ctx->sessions, hostname,
+ ev_now(conn->event_loop));
+ }
+
+ if (session) {
+ SSL_set_session(conn->ssl, session);
+ }
+
+ SSL_set_app_data(conn->ssl, conn);
+ msg_debug_ssl("new ssl connection %p; session reused=%s",
+ conn->ssl, SSL_session_reused(conn->ssl) ? "true" : "false");
+
+ if (conn->state != ssl_conn_reset) {
+ return FALSE;
+ }
+
+ /* We dup fd to allow graceful closing */
+ gint nfd = dup(fd);
+
+ if (nfd == -1) {
+ return FALSE;
+ }
+
+ conn->fd = nfd;
+ conn->ev = ev;
+ conn->handler = handler;
+ conn->err_handler = err_handler;
+ conn->handler_data = handler_data;
+
+ if (SSL_set_fd(conn->ssl, conn->fd) != 1) {
+ close(conn->fd);
+
+ return FALSE;
+ }
+
+ if (hostname) {
+ conn->hostname = g_strdup(hostname);
+#ifdef HAVE_SSL_TLSEXT_HOSTNAME
+ SSL_set_tlsext_host_name(conn->ssl, conn->hostname);
+#endif
+ }
+
+ conn->state = ssl_conn_init;
+
+ ret = SSL_connect(conn->ssl);
+
+ if (ret == 1) {
+ conn->state = ssl_conn_connected;
+
+ msg_debug_ssl("connected, start write event");
+ rspamd_ev_watcher_stop(conn->event_loop, ev);
+ rspamd_ev_watcher_init(ev, nfd, EV_WRITE, rspamd_ssl_event_handler, conn);
+ rspamd_ev_watcher_start(conn->event_loop, ev, timeout);
+ }
+ else {
+ ret = SSL_get_error(conn->ssl, ret);
+
+ if (ret == SSL_ERROR_WANT_READ) {
+ msg_debug_ssl("not connected, want read");
+ }
+ else if (ret == SSL_ERROR_WANT_WRITE) {
+ msg_debug_ssl("not connected, want write");
+ }
+ else {
+ GError *err = NULL;
+
+ conn->shut = ssl_shut_unclean;
+ rspamd_tls_set_error(ret, "initial connect", &err);
+ msg_debug_ssl("not connected, fatal error %e", err);
+ g_error_free(err);
+
+
+ return FALSE;
+ }
+
+ rspamd_ev_watcher_stop(conn->event_loop, ev);
+ rspamd_ev_watcher_init(ev, nfd, EV_WRITE | EV_READ,
+ rspamd_ssl_event_handler, conn);
+ rspamd_ev_watcher_start(conn->event_loop, ev, timeout);
+ }
+
+ return TRUE;
+}
+
+void rspamd_ssl_connection_restore_handlers(struct rspamd_ssl_connection *conn,
+ rspamd_ssl_handler_t handler,
+ rspamd_ssl_error_handler_t err_handler,
+ gpointer handler_data,
+ short ev_what)
+{
+ conn->handler = handler;
+ conn->err_handler = err_handler;
+ conn->handler_data = handler_data;
+
+ rspamd_ev_watcher_stop(conn->event_loop, conn->ev);
+ rspamd_ev_watcher_init(conn->ev, conn->fd, ev_what, rspamd_ssl_event_handler, conn);
+ rspamd_ev_watcher_start(conn->event_loop, conn->ev, conn->ev->timeout);
+}
+
+gssize
+rspamd_ssl_read(struct rspamd_ssl_connection *conn, gpointer buf,
+ gsize buflen)
+{
+ gint ret;
+ short what;
+ GError *err = NULL;
+
+ g_assert(conn != NULL);
+
+ if (conn->state != ssl_conn_connected && conn->state != ssl_next_read) {
+ errno = EINVAL;
+ g_set_error(&err, rspamd_ssl_quark(), 400,
+ "ssl state error: cannot read data");
+ conn->shut = ssl_shut_unclean;
+ conn->err_handler(conn->handler_data, err);
+ g_error_free(err);
+
+ return -1;
+ }
+
+ ret = SSL_read(conn->ssl, buf, buflen);
+ msg_debug_ssl("ssl read: %d", ret);
+
+ if (ret > 0) {
+ conn->state = ssl_conn_connected;
+ return ret;
+ }
+ else if (ret == 0) {
+ ret = SSL_get_error(conn->ssl, ret);
+
+ if (ret == SSL_ERROR_ZERO_RETURN || ret == SSL_ERROR_SYSCALL) {
+ conn->state = ssl_conn_reset;
+ return 0;
+ }
+ else {
+ conn->shut = ssl_shut_unclean;
+ rspamd_tls_set_error(ret, "read", &err);
+ conn->err_handler(conn->handler_data, err);
+ g_error_free(err);
+ errno = EINVAL;
+
+ return -1;
+ }
+ }
+ else {
+ ret = SSL_get_error(conn->ssl, ret);
+ conn->state = ssl_next_read;
+ what = 0;
+
+ if (ret == SSL_ERROR_WANT_READ) {
+ msg_debug_ssl("ssl read: need read");
+ what |= EV_READ;
+ }
+ else if (ret == SSL_ERROR_WANT_WRITE) {
+ msg_debug_ssl("ssl read: need write");
+ what |= EV_WRITE;
+ }
+ else {
+ conn->shut = ssl_shut_unclean;
+ rspamd_tls_set_error(ret, "read", &err);
+ conn->err_handler(conn->handler_data, err);
+ g_error_free(err);
+ errno = EINVAL;
+
+ return -1;
+ }
+
+ rspamd_ev_watcher_reschedule(conn->event_loop, conn->ev, what);
+ errno = EAGAIN;
+ }
+
+ return -1;
+}
+
+gssize
+rspamd_ssl_write(struct rspamd_ssl_connection *conn, gconstpointer buf,
+ gsize buflen)
+{
+ gint ret;
+ short what;
+ GError *err = NULL;
+
+ g_assert(conn != NULL);
+
+ if (conn->state != ssl_conn_connected && conn->state != ssl_next_write) {
+ errno = EINVAL;
+ return -1;
+ }
+
+ ret = SSL_write(conn->ssl, buf, buflen);
+ msg_debug_ssl("ssl write: ret=%d, buflen=%z", ret, buflen);
+
+ if (ret > 0) {
+ conn->state = ssl_conn_connected;
+ return ret;
+ }
+ else if (ret == 0) {
+ ret = SSL_get_error(conn->ssl, ret);
+
+ if (ret == SSL_ERROR_ZERO_RETURN) {
+ rspamd_tls_set_error(ret, "write", &err);
+ conn->err_handler(conn->handler_data, err);
+ g_error_free(err);
+ errno = ECONNRESET;
+ conn->state = ssl_conn_reset;
+
+ return -1;
+ }
+ else {
+ conn->shut = ssl_shut_unclean;
+ rspamd_tls_set_error(ret, "write", &err);
+ conn->err_handler(conn->handler_data, err);
+ g_error_free(err);
+ errno = EINVAL;
+
+ return -1;
+ }
+ }
+ else {
+ ret = SSL_get_error(conn->ssl, ret);
+ conn->state = ssl_next_write;
+
+ if (ret == SSL_ERROR_WANT_READ) {
+ msg_debug_ssl("ssl write: need read");
+ what = EV_READ;
+ }
+ else if (ret == SSL_ERROR_WANT_WRITE) {
+ msg_debug_ssl("ssl write: need write");
+ what = EV_WRITE;
+ }
+ else {
+ conn->shut = ssl_shut_unclean;
+ rspamd_tls_set_error(ret, "write", &err);
+ conn->err_handler(conn->handler_data, err);
+ g_error_free(err);
+ errno = EINVAL;
+
+ return -1;
+ }
+
+ rspamd_ev_watcher_reschedule(conn->event_loop, conn->ev, what);
+ errno = EAGAIN;
+ }
+
+ return -1;
+}
+
+gssize
+rspamd_ssl_writev(struct rspamd_ssl_connection *conn, struct iovec *iov,
+ gsize iovlen)
+{
+ /*
+ * Static is needed to avoid issue:
+ * https://github.com/openssl/openssl/issues/6865
+ */
+ static guchar ssl_buf[16384];
+ guchar *p;
+ struct iovec *cur;
+ gsize i, remain;
+
+ remain = sizeof(ssl_buf);
+ p = ssl_buf;
+
+ for (i = 0; i < iovlen; i++) {
+ cur = &iov[i];
+
+ if (cur->iov_len > 0) {
+ if (remain >= cur->iov_len) {
+ memcpy(p, cur->iov_base, cur->iov_len);
+ p += cur->iov_len;
+ remain -= cur->iov_len;
+ }
+ else {
+ memcpy(p, cur->iov_base, remain);
+ p += remain;
+ remain = 0;
+ break;
+ }
+ }
+ }
+
+ return rspamd_ssl_write(conn, ssl_buf, p - ssl_buf);
+}
+
+/**
+ * Removes connection data
+ * @param conn
+ */
+void rspamd_ssl_connection_free(struct rspamd_ssl_connection *conn)
+{
+ if (conn) {
+ if (conn->shut == ssl_shut_unclean) {
+ /* Ignore return result and close socket */
+ msg_debug_ssl("unclean shutdown");
+ SSL_set_quiet_shutdown(conn->ssl, 1);
+ (void) SSL_shutdown(conn->ssl);
+ rspamd_ssl_connection_dtor(conn);
+ }
+ else {
+ msg_debug_ssl("normal shutdown");
+ rspamd_ssl_shutdown(conn);
+ }
+ }
+}
+
+static int
+rspamd_ssl_new_client_session(SSL *ssl, SSL_SESSION *sess)
+{
+ struct rspamd_ssl_connection *conn;
+
+ conn = SSL_get_app_data(ssl);
+
+ if (conn->hostname) {
+ rspamd_lru_hash_insert(conn->ssl_ctx->sessions,
+ g_strdup(conn->hostname), SSL_get1_session(ssl),
+ ev_now(conn->event_loop), SSL_CTX_get_timeout(conn->ssl_ctx->s));
+ msg_debug_ssl("saved new session for %s: %p", conn->hostname, conn);
+ }
+
+ return 0;
+}
+
+static struct rspamd_ssl_ctx *
+rspamd_init_ssl_ctx_common(void)
+{
+ struct rspamd_ssl_ctx *ret;
+ SSL_CTX *ssl_ctx;
+ gint ssl_options;
+ static const guint client_cache_size = 1024;
+
+ rspamd_openssl_maybe_init();
+
+ ret = g_malloc0(sizeof(*ret));
+ ssl_options = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
+ ssl_ctx = SSL_CTX_new(SSLv23_method());
+
+#ifdef SSL_OP_NO_COMPRESSION
+ ssl_options |= SSL_OP_NO_COMPRESSION;
+#elif OPENSSL_VERSION_NUMBER >= 0x00908000L
+ sk_SSL_COMP_zero(SSL_COMP_get_compression_methods());
+#endif
+
+ SSL_CTX_set_options(ssl_ctx, ssl_options);
+
+#ifdef TLS1_3_VERSION
+ SSL_CTX_set_min_proto_version(ssl_ctx, 0);
+ SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION);
+#endif
+
+#ifdef SSL_SESS_CACHE_CLIENT
+ SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_CLIENT | SSL_SESS_CACHE_NO_INTERNAL_STORE);
+#endif
+
+ ret->s = ssl_ctx;
+ ret->sessions = rspamd_lru_hash_new_full(client_cache_size,
+ g_free, (GDestroyNotify) SSL_SESSION_free, rspamd_str_hash,
+ rspamd_str_equal);
+ SSL_CTX_set_app_data(ssl_ctx, ret);
+ SSL_CTX_sess_set_new_cb(ssl_ctx, rspamd_ssl_new_client_session);
+
+ return ret;
+}
+
+gpointer
+rspamd_init_ssl_ctx(void)
+{
+ struct rspamd_ssl_ctx *ssl_ctx = rspamd_init_ssl_ctx_common();
+
+ SSL_CTX_set_verify(ssl_ctx->s, SSL_VERIFY_PEER, NULL);
+ SSL_CTX_set_verify_depth(ssl_ctx->s, 4);
+
+ return ssl_ctx;
+}
+
+gpointer rspamd_init_ssl_ctx_noverify(void)
+{
+ struct rspamd_ssl_ctx *ssl_ctx_noverify = rspamd_init_ssl_ctx_common();
+
+ SSL_CTX_set_verify(ssl_ctx_noverify->s, SSL_VERIFY_NONE, NULL);
+
+ return ssl_ctx_noverify;
+}
+
+void rspamd_openssl_maybe_init(void)
+{
+ static gboolean openssl_initialized = FALSE;
+
+ if (!openssl_initialized) {
+ ERR_load_crypto_strings();
+ SSL_load_error_strings();
+
+ OpenSSL_add_all_algorithms();
+ OpenSSL_add_all_digests();
+ OpenSSL_add_all_ciphers();
+
+#if OPENSSL_VERSION_NUMBER >= 0x1000104fL && OPENSSL_VERSION_NUMBER < 0x30000000L && !defined(LIBRESSL_VERSION_NUMBER)
+ ENGINE_load_builtin_engines();
+#endif
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+ SSL_library_init();
+#else
+ OPENSSL_init_ssl(0, NULL);
+#endif
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+ OPENSSL_config(NULL);
+#endif
+ if (RAND_status() == 0) {
+ guchar seed[128];
+
+ /* Try to use ottery to seed rand */
+ ottery_rand_bytes(seed, sizeof(seed));
+ RAND_seed(seed, sizeof(seed));
+ rspamd_explicit_memzero(seed, sizeof(seed));
+ }
+
+ openssl_initialized = TRUE;
+ }
+}
+
+void rspamd_ssl_ctx_config(struct rspamd_config *cfg, gpointer ssl_ctx)
+{
+ struct rspamd_ssl_ctx *ctx = (struct rspamd_ssl_ctx *) ssl_ctx;
+ static const char default_secure_ciphers[] = "HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4";
+
+ if (cfg->ssl_ca_path) {
+ if (SSL_CTX_load_verify_locations(ctx->s, cfg->ssl_ca_path,
+ NULL) != 1) {
+ msg_err_config("cannot load CA certs from %s: %s",
+ cfg->ssl_ca_path,
+ ERR_error_string(ERR_get_error(), NULL));
+ }
+ }
+ else {
+ msg_debug_config("ssl_ca_path is not set, using default CA path");
+ SSL_CTX_set_default_verify_paths(ctx->s);
+ }
+
+ if (cfg->ssl_ciphers) {
+ if (SSL_CTX_set_cipher_list(ctx->s, cfg->ssl_ciphers) != 1) {
+ msg_err_config(
+ "cannot set ciphers set to %s: %s; fallback to %s",
+ cfg->ssl_ciphers,
+ ERR_error_string(ERR_get_error(), NULL),
+ default_secure_ciphers);
+ /* Default settings */
+ SSL_CTX_set_cipher_list(ctx->s, default_secure_ciphers);
+ }
+ }
+}
+
+void rspamd_ssl_ctx_free(gpointer ssl_ctx)
+{
+ struct rspamd_ssl_ctx *ctx = (struct rspamd_ssl_ctx *) ssl_ctx;
+
+ rspamd_lru_hash_destroy(ctx->sessions);
+ SSL_CTX_free(ctx->s);
+ g_free(ssl_ctx);
+} \ No newline at end of file
diff --git a/src/libserver/ssl_util.h b/src/libserver/ssl_util.h
new file mode 100644
index 0000000..cde7d47
--- /dev/null
+++ b/src/libserver/ssl_util.h
@@ -0,0 +1,120 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBUTIL_SSL_UTIL_H_
+#define SRC_LIBUTIL_SSL_UTIL_H_
+
+#include "config.h"
+#include "libutil/addr.h"
+#include "libutil/libev_helper.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_ssl_connection;
+
+typedef void (*rspamd_ssl_handler_t)(gint fd, short what, gpointer d);
+
+typedef void (*rspamd_ssl_error_handler_t)(gpointer d, GError *err);
+
+/**
+ * Creates a new ssl connection data structure
+ * @param ssl_ctx initialized SSL_CTX structure
+ * @return opaque connection data
+ */
+struct rspamd_ssl_connection *rspamd_ssl_connection_new(gpointer ssl_ctx,
+ struct ev_loop *ev_base,
+ gboolean verify_peer,
+ const gchar *log_tag);
+
+/**
+ * Connects SSL session using the specified (connected) FD
+ * @param conn connection
+ * @param fd fd to use
+ * @param hostname hostname for SNI
+ * @param ev event to use
+ * @param tv timeout for connection
+ * @param handler connected session handler
+ * @param handler_data opaque data
+ * @return TRUE if a session has been connected
+ */
+gboolean rspamd_ssl_connect_fd(struct rspamd_ssl_connection *conn, gint fd,
+ const gchar *hostname, struct rspamd_io_ev *ev, ev_tstamp timeout,
+ rspamd_ssl_handler_t handler, rspamd_ssl_error_handler_t err_handler,
+ gpointer handler_data);
+
+/**
+ * Restores SSL handlers for the existing ssl connection (e.g. after keepalive)
+ * @param conn
+ * @param handler
+ * @param err_handler
+ * @param handler_data
+ */
+void rspamd_ssl_connection_restore_handlers(struct rspamd_ssl_connection *conn,
+ rspamd_ssl_handler_t handler,
+ rspamd_ssl_error_handler_t err_handler,
+ gpointer handler_data,
+ short ev_what);
+
+/**
+ * Perform async read from SSL socket
+ * @param conn
+ * @param buf
+ * @param buflen
+ * @return
+ */
+gssize rspamd_ssl_read(struct rspamd_ssl_connection *conn, gpointer buf,
+ gsize buflen);
+
+/**
+ * Perform async write to ssl buffer
+ * @param conn
+ * @param buf
+ * @param buflen
+ * @param ev
+ * @param tv
+ * @return
+ */
+gssize rspamd_ssl_write(struct rspamd_ssl_connection *conn, gconstpointer buf,
+ gsize buflen);
+
+/**
+ * Emulate writev by copying iovec to a temporary buffer
+ * @param conn
+ * @param buf
+ * @param buflen
+ * @return
+ */
+gssize rspamd_ssl_writev(struct rspamd_ssl_connection *conn, struct iovec *iov,
+ gsize iovlen);
+
+/**
+ * Removes connection data
+ * @param conn
+ */
+void rspamd_ssl_connection_free(struct rspamd_ssl_connection *conn);
+
+gpointer rspamd_init_ssl_ctx(void);
+gpointer rspamd_init_ssl_ctx_noverify(void);
+void rspamd_ssl_ctx_config(struct rspamd_config *cfg, gpointer ssl_ctx);
+void rspamd_ssl_ctx_free(gpointer ssl_ctx);
+void rspamd_openssl_maybe_init(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBUTIL_SSL_UTIL_H_ */
diff --git a/src/libserver/symcache/symcache_c.cxx b/src/libserver/symcache/symcache_c.cxx
new file mode 100644
index 0000000..6a7e41c
--- /dev/null
+++ b/src/libserver/symcache/symcache_c.cxx
@@ -0,0 +1,715 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "symcache_internal.hxx"
+#include "symcache_periodic.hxx"
+#include "symcache_item.hxx"
+#include "symcache_runtime.hxx"
+
+/**
+ * C API for symcache
+ */
+
+#define C_API_SYMCACHE(ptr) (reinterpret_cast<rspamd::symcache::symcache *>(ptr))
+#define C_API_SYMCACHE_RUNTIME(ptr) (reinterpret_cast<rspamd::symcache::symcache_runtime *>(ptr))
+#define C_API_SYMCACHE_ITEM(ptr) (reinterpret_cast<rspamd::symcache::cache_item *>(ptr))
+#define C_API_SYMCACHE_DYN_ITEM(ptr) (reinterpret_cast<rspamd::symcache::cache_dynamic_item *>(ptr))
+
+void rspamd_symcache_destroy(struct rspamd_symcache *cache)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ delete real_cache;
+}
+
+struct rspamd_symcache *
+rspamd_symcache_new(struct rspamd_config *cfg)
+{
+ auto *ncache = new rspamd::symcache::symcache(cfg);
+
+ return (struct rspamd_symcache *) ncache;
+}
+
+gboolean
+rspamd_symcache_init(struct rspamd_symcache *cache)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ return real_cache->init();
+}
+
+void rspamd_symcache_save(struct rspamd_symcache *cache)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ real_cache->save_items();
+}
+
+gint rspamd_symcache_add_symbol(struct rspamd_symcache *cache,
+ const gchar *name,
+ gint priority,
+ symbol_func_t func,
+ gpointer user_data,
+ int type,
+ gint parent)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ /* Legacy stuff */
+ if (name == nullptr) {
+ name = "";
+ }
+
+ if (parent == -1) {
+ return real_cache->add_symbol_with_callback(name, priority, func, user_data, type);
+ }
+ else {
+ return real_cache->add_virtual_symbol(name, parent, type);
+ }
+}
+
+bool rspamd_symcache_add_symbol_augmentation(struct rspamd_symcache *cache,
+ int sym_id,
+ const char *augmentation,
+ const char *value)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+ auto log_tag = [&]() { return real_cache->log_tag(); };
+
+ if (augmentation == nullptr) {
+ msg_err_cache("null augmentation is not allowed for item %d", sym_id);
+ return false;
+ }
+
+
+ auto *item = real_cache->get_item_by_id_mut(sym_id, false);
+
+ if (item == nullptr) {
+ msg_err_cache("item %d is not found", sym_id);
+ return false;
+ }
+
+ /* Handle empty or absent strings equally */
+ if (value == nullptr || value[0] == '\0') {
+ return item->add_augmentation(*real_cache, augmentation, std::nullopt);
+ }
+
+ return item->add_augmentation(*real_cache, augmentation, value);
+}
+
+void rspamd_symcache_set_peak_callback(struct rspamd_symcache *cache, gint cbref)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ real_cache->set_peak_cb(cbref);
+}
+
+gboolean
+rspamd_symcache_add_condition_delayed(struct rspamd_symcache *cache,
+ const gchar *sym, lua_State *L, gint cbref)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ real_cache->add_delayed_condition(sym, cbref);
+
+ return TRUE;
+}
+
+gint rspamd_symcache_find_symbol(struct rspamd_symcache *cache,
+ const gchar *name)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ /* Legacy stuff but used */
+ if (name == nullptr) {
+ return -1;
+ }
+
+ auto sym_maybe = real_cache->get_item_by_name(name, false);
+
+ if (sym_maybe != nullptr) {
+ return sym_maybe->id;
+ }
+
+ return -1;
+}
+
+gboolean
+rspamd_symcache_stat_symbol(struct rspamd_symcache *cache,
+ const gchar *name,
+ gdouble *frequency,
+ gdouble *freq_stddev,
+ gdouble *tm,
+ guint *nhits)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ auto sym_maybe = real_cache->get_item_by_name(name, false);
+
+ if (sym_maybe != nullptr) {
+ *frequency = sym_maybe->st->avg_frequency;
+ *freq_stddev = sqrt(sym_maybe->st->stddev_frequency);
+ *tm = sym_maybe->st->time_counter.mean;
+
+ if (nhits) {
+ *nhits = sym_maybe->st->hits;
+ }
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+
+guint rspamd_symcache_stats_symbols_count(struct rspamd_symcache *cache)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+ return real_cache->get_stats_symbols_count();
+}
+
+guint64
+rspamd_symcache_get_cksum(struct rspamd_symcache *cache)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+ return real_cache->get_cksum();
+}
+
+gboolean
+rspamd_symcache_validate(struct rspamd_symcache *cache,
+ struct rspamd_config *cfg,
+ gboolean strict)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ return real_cache->validate(strict);
+}
+
+ucl_object_t *
+rspamd_symcache_counters(struct rspamd_symcache *cache)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+ return real_cache->counters();
+}
+
+void *
+rspamd_symcache_start_refresh(struct rspamd_symcache *cache,
+ struct ev_loop *ev_base, struct rspamd_worker *w)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+ return new rspamd::symcache::cache_refresh_cbdata{real_cache, ev_base, w};
+}
+
+void rspamd_symcache_inc_frequency(struct rspamd_symcache *cache, struct rspamd_symcache_item *item,
+ const char *sym_name)
+{
+ auto *real_item = C_API_SYMCACHE_ITEM(item);
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ if (real_item) {
+ real_item->inc_frequency(sym_name, *real_cache);
+ }
+}
+
+void rspamd_symcache_add_delayed_dependency(struct rspamd_symcache *cache,
+ const gchar *from, const gchar *to)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+ real_cache->add_delayed_dependency(from, to);
+}
+
+const gchar *
+rspamd_symcache_get_parent(struct rspamd_symcache *cache,
+ const gchar *symbol)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ auto *sym = real_cache->get_item_by_name(symbol, false);
+
+ if (sym && sym->is_virtual()) {
+ auto *parent = sym->get_parent(*real_cache);
+
+ if (parent) {
+ return parent->get_name().c_str();
+ }
+ }
+
+ return nullptr;
+}
+
+const gchar *
+rspamd_symcache_item_name(struct rspamd_symcache_item *item)
+{
+ auto *real_item = C_API_SYMCACHE_ITEM(item);
+
+ if (real_item == nullptr) {
+ return nullptr;
+ }
+
+ return real_item->get_name().c_str();
+}
+
+gint rspamd_symcache_item_flags(struct rspamd_symcache_item *item)
+{
+ auto *real_item = C_API_SYMCACHE_ITEM(item);
+
+ if (real_item == nullptr) {
+ return 0;
+ }
+
+ return real_item->get_flags();
+}
+
+
+const gchar *
+rspamd_symcache_dyn_item_name(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *dyn_item)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ auto *real_dyn_item = C_API_SYMCACHE_DYN_ITEM(dyn_item);
+
+ if (cache_runtime == nullptr || real_dyn_item == nullptr) {
+ return nullptr;
+ }
+
+ auto static_item = cache_runtime->get_item_by_dynamic_item(real_dyn_item);
+
+ return static_item->get_name().c_str();
+}
+
+gint rspamd_symcache_item_flags(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *dyn_item)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ auto *real_dyn_item = C_API_SYMCACHE_DYN_ITEM(dyn_item);
+
+ if (cache_runtime == nullptr || real_dyn_item == nullptr) {
+ return 0;
+ }
+
+ auto static_item = cache_runtime->get_item_by_dynamic_item(real_dyn_item);
+
+ return static_item->get_flags();
+}
+
+guint rspamd_symcache_get_symbol_flags(struct rspamd_symcache *cache,
+ const gchar *symbol)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ auto *sym = real_cache->get_item_by_name(symbol, false);
+
+ if (sym) {
+ return sym->get_flags();
+ }
+
+ return 0;
+}
+
+const struct rspamd_symcache_item_stat *
+rspamd_symcache_item_stat(struct rspamd_symcache_item *item)
+{
+ auto *real_item = C_API_SYMCACHE_ITEM(item);
+ return real_item->st;
+}
+
+void rspamd_symcache_get_symbol_details(struct rspamd_symcache *cache,
+ const gchar *symbol,
+ ucl_object_t *this_sym_ucl)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ auto *sym = real_cache->get_item_by_name(symbol, false);
+
+ if (sym) {
+ ucl_object_insert_key(this_sym_ucl,
+ ucl_object_fromstring(sym->get_type_str()),
+ "type", strlen("type"), false);
+ }
+}
+
+void rspamd_symcache_foreach(struct rspamd_symcache *cache,
+ void (*func)(struct rspamd_symcache_item *item, gpointer /* userdata */),
+ gpointer ud)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ real_cache->symbols_foreach([&](const rspamd::symcache::cache_item *item) {
+ func((struct rspamd_symcache_item *) item, ud);
+ });
+}
+
+void rspamd_symcache_process_settings_elt(struct rspamd_symcache *cache,
+ struct rspamd_config_settings_elt *elt)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ real_cache->process_settings_elt(elt);
+}
+
+bool rspamd_symcache_set_allowed_settings_ids(struct rspamd_symcache *cache,
+ const gchar *symbol,
+ const guint32 *ids,
+ guint nids)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ auto *item = real_cache->get_item_by_name_mut(symbol, false);
+
+ if (item == nullptr) {
+ return false;
+ }
+
+ item->allowed_ids.set_ids(ids, nids);
+ return true;
+}
+
+bool rspamd_symcache_set_forbidden_settings_ids(struct rspamd_symcache *cache,
+ const gchar *symbol,
+ const guint32 *ids,
+ guint nids)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ auto *item = real_cache->get_item_by_name_mut(symbol, false);
+
+ if (item == nullptr) {
+ return false;
+ }
+
+ item->forbidden_ids.set_ids(ids, nids);
+ return true;
+}
+
+const guint32 *
+rspamd_symcache_get_allowed_settings_ids(struct rspamd_symcache *cache,
+ const gchar *symbol,
+ guint *nids)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ const auto *item = real_cache->get_item_by_name(symbol, false);
+ return item->allowed_ids.get_ids(*nids);
+}
+
+const guint32 *
+rspamd_symcache_get_forbidden_settings_ids(struct rspamd_symcache *cache,
+ const gchar *symbol,
+ guint *nids)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ const auto *item = real_cache->get_item_by_name(symbol, false);
+ return item->forbidden_ids.get_ids(*nids);
+}
+
+void rspamd_symcache_disable_all_symbols(struct rspamd_task *task,
+ struct rspamd_symcache *_cache,
+ guint skip_mask)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+
+ cache_runtime->disable_all_symbols(skip_mask);
+}
+
+gboolean
+rspamd_symcache_disable_symbol(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ const gchar *symbol)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ if (cache_runtime == nullptr) {
+ return FALSE;
+ }
+
+ return cache_runtime->disable_symbol(task, *real_cache, symbol);
+}
+
+gboolean
+rspamd_symcache_enable_symbol(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ const gchar *symbol)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ if (cache_runtime == nullptr) {
+ return FALSE;
+ }
+
+ return cache_runtime->enable_symbol(task, *real_cache, symbol);
+}
+
+void rspamd_symcache_disable_symbol_static(struct rspamd_symcache *cache,
+ const gchar *symbol)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ real_cache->disable_symbol_delayed(symbol);
+}
+
+void rspamd_symcache_enable_symbol_static(struct rspamd_symcache *cache,
+ const gchar *symbol)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ real_cache->enable_symbol_delayed(symbol);
+}
+
+/* A real structure to match C results without extra copying */
+struct rspamd_symcache_real_timeout_result {
+ struct rspamd_symcache_timeout_result c_api_result;
+ std::vector<std::pair<double, const rspamd::symcache::cache_item *>> elts;
+};
+
+struct rspamd_symcache_timeout_result *
+rspamd_symcache_get_max_timeout(struct rspamd_symcache *cache)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+ auto *res = new rspamd_symcache_real_timeout_result;
+
+ res->c_api_result.max_timeout = real_cache->get_max_timeout(res->elts);
+ res->c_api_result.items = reinterpret_cast<struct rspamd_symcache_timeout_item *>(res->elts.data());
+ res->c_api_result.nitems = res->elts.size();
+
+ return &res->c_api_result;
+}
+
+void rspamd_symcache_timeout_result_free(struct rspamd_symcache_timeout_result *res)
+{
+ auto *real_result = reinterpret_cast<rspamd_symcache_real_timeout_result *>(res);
+ delete real_result;
+}
+
+gboolean
+rspamd_symcache_is_checked(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ const gchar *symbol)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ if (cache_runtime == nullptr) {
+ return FALSE;
+ }
+
+ return cache_runtime->is_symbol_checked(*real_cache, symbol);
+}
+
+gboolean
+rspamd_symcache_process_settings(struct rspamd_task *task,
+ struct rspamd_symcache *cache)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ if (cache_runtime == nullptr) {
+ return FALSE;
+ }
+
+ return cache_runtime->process_settings(task, *real_cache);
+}
+
+gboolean
+rspamd_symcache_is_item_allowed(struct rspamd_task *task,
+ struct rspamd_symcache_item *item,
+ gboolean exec_only)
+{
+ auto *real_item = C_API_SYMCACHE_ITEM(item);
+
+ if (real_item == nullptr) {
+ return TRUE;
+ }
+
+ return real_item->is_allowed(task, exec_only);
+}
+
+gboolean
+rspamd_symcache_is_symbol_enabled(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ const gchar *symbol)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ if (!cache_runtime) {
+ return TRUE;
+ }
+
+ return cache_runtime->is_symbol_enabled(task, *real_cache, symbol);
+}
+
+struct rspamd_symcache_dynamic_item *
+rspamd_symcache_get_cur_item(struct rspamd_task *task)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+
+ if (!cache_runtime) {
+ return nullptr;
+ }
+
+ return (struct rspamd_symcache_dynamic_item *) cache_runtime->get_cur_item();
+}
+
+struct rspamd_symcache_dynamic_item *
+rspamd_symcache_set_cur_item(struct rspamd_task *task, struct rspamd_symcache_dynamic_item *item)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ auto *real_dyn_item = C_API_SYMCACHE_DYN_ITEM(item);
+
+ if (!cache_runtime || !real_dyn_item) {
+ return nullptr;
+ }
+
+ return (struct rspamd_symcache_dynamic_item *) cache_runtime->set_cur_item(real_dyn_item);
+}
+
+void rspamd_symcache_enable_profile(struct rspamd_task *task)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ if (!cache_runtime) {
+ return;
+ }
+
+ cache_runtime->set_profile_mode(true);
+}
+
+guint rspamd_symcache_item_async_inc_full(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ const gchar *subsystem,
+ const gchar *loc)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ auto *real_dyn_item = C_API_SYMCACHE_DYN_ITEM(item);
+
+ auto *static_item = cache_runtime->get_item_by_dynamic_item(real_dyn_item);
+ msg_debug_cache_task("increase async events counter for %s(%d) = %d + 1; "
+ "subsystem %s (%s)",
+ static_item->symbol.c_str(), static_item->id,
+ real_dyn_item->async_events, subsystem, loc);
+
+ return ++real_dyn_item->async_events;
+}
+
+guint rspamd_symcache_item_async_dec_full(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ const gchar *subsystem,
+ const gchar *loc)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ auto *real_dyn_item = C_API_SYMCACHE_DYN_ITEM(item);
+
+ auto *static_item = cache_runtime->get_item_by_dynamic_item(real_dyn_item);
+ msg_debug_cache_task("decrease async events counter for %s(%d) = %d - 1; "
+ "subsystem %s (%s)",
+ static_item->symbol.c_str(), static_item->id,
+ real_dyn_item->async_events, subsystem, loc);
+
+ if (G_UNLIKELY(real_dyn_item->async_events == 0)) {
+ msg_err_cache_task("INTERNAL ERROR: trying decrease async events counter for %s(%d) that is already zero; "
+ "subsystem %s (%s)",
+ static_item->symbol.c_str(), static_item->id,
+ real_dyn_item->async_events, subsystem, loc);
+ g_abort();
+ g_assert_not_reached();
+ }
+
+ return --real_dyn_item->async_events;
+}
+
+gboolean
+rspamd_symcache_item_async_dec_check_full(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ const gchar *subsystem,
+ const gchar *loc)
+{
+ if (rspamd_symcache_item_async_dec_full(task, item, subsystem, loc) == 0) {
+ rspamd_symcache_finalize_item(task, item);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+struct rspamd_abstract_callback_data *
+rspamd_symcache_get_cbdata(struct rspamd_symcache *cache,
+ const gchar *symbol)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ auto *item = real_cache->get_item_by_name(symbol, true);
+
+ if (item) {
+ return (struct rspamd_abstract_callback_data *) item->get_cbdata();
+ }
+
+ return nullptr;
+}
+
+void rspamd_symcache_composites_foreach(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ GHFunc func,
+ gpointer fd)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+
+ real_cache->composites_foreach([&](const auto *item) {
+ auto *dyn_item = cache_runtime->get_dynamic_item(item->id);
+
+ if (dyn_item && !dyn_item->started) {
+ auto *old_item = cache_runtime->set_cur_item(dyn_item);
+ func((void *) item->get_name().c_str(), item->get_cbdata(), fd);
+ dyn_item->finished = true;
+ cache_runtime->set_cur_item(old_item);
+ }
+ });
+
+ cache_runtime->set_cur_item(nullptr);
+}
+
+gboolean
+rspamd_symcache_process_symbols(struct rspamd_task *task,
+ struct rspamd_symcache *cache,
+ guint stage)
+{
+ auto *real_cache = C_API_SYMCACHE(cache);
+
+ if (task->symcache_runtime == nullptr) {
+ task->symcache_runtime = rspamd::symcache::symcache_runtime::create(task, *real_cache);
+ }
+
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ return cache_runtime->process_symbols(task, *real_cache, stage);
+}
+
+void rspamd_symcache_finalize_item(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ auto *real_dyn_item = C_API_SYMCACHE_DYN_ITEM(item);
+
+ cache_runtime->finalize_item(task, real_dyn_item);
+}
+
+void rspamd_symcache_runtime_destroy(struct rspamd_task *task)
+{
+ auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+ cache_runtime->savepoint_dtor();
+} \ No newline at end of file
diff --git a/src/libserver/symcache/symcache_id_list.hxx b/src/libserver/symcache/symcache_id_list.hxx
new file mode 100644
index 0000000..bef4fa9
--- /dev/null
+++ b/src/libserver/symcache/symcache_id_list.hxx
@@ -0,0 +1,95 @@
+/*-
+ * Copyright 2022 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_SYMCACHE_ID_LIST_HXX
+#define RSPAMD_SYMCACHE_ID_LIST_HXX
+#pragma once
+
+#include <cstdint>
+#include <cstring> // for memset
+#include <algorithm>// for sort/bsearch
+
+#include "config.h"
+#include "libutil/mem_pool.h"
+#include "contrib/ankerl/svector.h"
+
+namespace rspamd::symcache {
+/*
+ * This structure is optimised to store ids list:
+ * - If the first element is -1 then use dynamic part, else use static part
+ * There is no std::variant to save space
+ */
+
+constexpr const auto id_capacity = 4;
+constexpr const auto id_sort_threshold = 32;
+
+struct id_list {
+ ankerl::svector<std::uint32_t, id_capacity> data;
+
+ id_list() = default;
+
+ auto reset()
+ {
+ data.clear();
+ }
+
+ /**
+ * Returns ids from a compressed list, accepting a mutable reference for number of elements
+ * @param nids output of the number of elements
+ * @return
+ */
+ auto get_ids(unsigned &nids) const -> const std::uint32_t *
+ {
+ nids = data.size();
+
+ return data.data();
+ }
+
+ auto add_id(std::uint32_t id) -> void
+ {
+ data.push_back(id);
+
+ /* Check sort threshold */
+ if (data.size() > id_sort_threshold) {
+ std::sort(data.begin(), data.end());
+ }
+ }
+
+ auto set_ids(const std::uint32_t *ids, std::size_t nids) -> void
+ {
+ data.resize(nids);
+
+ for (auto &id: data) {
+ id = *ids++;
+ }
+
+ if (data.size() > id_sort_threshold) {
+ std::sort(data.begin(), data.end());
+ }
+ }
+
+ auto check_id(unsigned int id) const -> bool
+ {
+ if (data.size() > id_sort_threshold) {
+ return std::binary_search(data.begin(), data.end(), id);
+ }
+ return std::find(data.begin(), data.end(), id) != data.end();
+ }
+};
+
+}// namespace rspamd::symcache
+
+#endif//RSPAMD_SYMCACHE_ID_LIST_HXX
diff --git a/src/libserver/symcache/symcache_impl.cxx b/src/libserver/symcache/symcache_impl.cxx
new file mode 100644
index 0000000..93675ac
--- /dev/null
+++ b/src/libserver/symcache/symcache_impl.cxx
@@ -0,0 +1,1316 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua/lua_common.h"
+#include "symcache_internal.hxx"
+#include "symcache_item.hxx"
+#include "symcache_runtime.hxx"
+#include "unix-std.h"
+#include "libutil/cxx/file_util.hxx"
+#include "libutil/cxx/util.hxx"
+#include "fmt/core.h"
+#include "contrib/t1ha/t1ha.h"
+
+#ifdef __has_include
+#if __has_include(<version>)
+#include <version>
+#endif
+#endif
+#include <cmath>
+
+namespace rspamd::symcache {
+
+INIT_LOG_MODULE_PUBLIC(symcache)
+
+auto symcache::init() -> bool
+{
+ auto res = true;
+ reload_time = cfg->cache_reload_time;
+
+ if (cfg->cache_filename != nullptr) {
+ msg_debug_cache("loading symcache saved data from %s", cfg->cache_filename);
+ load_items();
+ }
+
+ ankerl::unordered_dense::set<int> disabled_ids;
+ /* Process enabled/disabled symbols */
+ for (const auto &[id, it]: items_by_id) {
+ if (disabled_symbols) {
+ /*
+ * Due to the ability to add patterns, this is now O(N^2), but it is done
+ * once on configuration and the amount of static patterns is usually low
+ * The possible optimization is to store non patterns in a different set to check it
+ * quickly. However, it is unlikely that this would be used to something really heavy.
+ */
+ for (const auto &disable_pat: *disabled_symbols) {
+ if (disable_pat.matches(it->get_name())) {
+ msg_debug_cache("symbol %s matches %*s disable pattern", it->get_name().c_str(),
+ (int) disable_pat.to_string_view().size(), disable_pat.to_string_view().data());
+ auto need_disable = true;
+
+ if (enabled_symbols) {
+ for (const auto &enable_pat: *enabled_symbols) {
+ if (enable_pat.matches(it->get_name())) {
+ msg_debug_cache("symbol %s matches %*s enable pattern; skip disabling", it->get_name().c_str(),
+ (int) enable_pat.to_string_view().size(), enable_pat.to_string_view().data());
+ need_disable = false;
+ break;
+ }
+ }
+ }
+
+ if (need_disable) {
+ disabled_ids.insert(it->id);
+
+ if (it->is_virtual()) {
+ auto real_elt = it->get_parent(*this);
+
+ if (real_elt) {
+ disabled_ids.insert(real_elt->id);
+
+ const auto *children = real_elt->get_children();
+ if (children != nullptr) {
+ for (const auto &cld: *children) {
+ msg_debug_cache("symbol %s is a virtual sibling of the disabled symbol %s",
+ cld->get_name().c_str(), it->get_name().c_str());
+ disabled_ids.insert(cld->id);
+ }
+ }
+ }
+ }
+ else {
+ /* Also disable all virtual children of this element */
+ const auto *children = it->get_children();
+
+ if (children != nullptr) {
+ for (const auto &cld: *children) {
+ msg_debug_cache("symbol %s is a virtual child of the disabled symbol %s",
+ cld->get_name().c_str(), it->get_name().c_str());
+ disabled_ids.insert(cld->id);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /* Deal with the delayed dependencies */
+ msg_debug_cache("resolving delayed dependencies: %d in list", (int) delayed_deps->size());
+ for (const auto &delayed_dep: *delayed_deps) {
+ auto virt_item = get_item_by_name(delayed_dep.from, false);
+ auto real_item = get_item_by_name(delayed_dep.from, true);
+
+ if (virt_item == nullptr || real_item == nullptr) {
+ msg_err_cache("cannot register delayed dependency between %s and %s: "
+ "%s is missing",
+ delayed_dep.from.data(),
+ delayed_dep.to.data(), delayed_dep.from.data());
+ }
+ else {
+
+ if (!disabled_ids.contains(real_item->id)) {
+ msg_debug_cache("delayed between %s(%d:%d) -> %s",
+ delayed_dep.from.data(),
+ real_item->id, virt_item->id,
+ delayed_dep.to.data());
+ add_dependency(real_item->id, delayed_dep.to,
+ virt_item != real_item ? virt_item->id : -1);
+ }
+ else {
+ msg_debug_cache("no delayed between %s(%d:%d) -> %s; %s is disabled",
+ delayed_dep.from.data(),
+ real_item->id, virt_item->id,
+ delayed_dep.to.data(),
+ delayed_dep.from.data());
+ }
+ }
+ }
+
+ /* Remove delayed dependencies, as they are no longer needed at this point */
+ delayed_deps.reset();
+
+ /* Physically remove ids that are disabled statically */
+ for (auto id_to_disable: disabled_ids) {
+ /*
+ * This erasure is inefficient, we can swap the last element with the removed id
+ * But in this way, our ids are still sorted by addition
+ */
+
+ /* Preserve refcount here */
+ auto deleted_element_refcount = items_by_id[id_to_disable];
+ items_by_id.erase(id_to_disable);
+ items_by_symbol.erase(deleted_element_refcount->get_name());
+
+ auto &additional_vec = get_item_specific_vector(*deleted_element_refcount);
+#if defined(__cpp_lib_erase_if)
+ std::erase_if(additional_vec, [id_to_disable](cache_item *elt) {
+ return elt->id == id_to_disable;
+ });
+#else
+ auto it = std::remove_if(additional_vec.begin(),
+ additional_vec.end(), [id_to_disable](cache_item *elt) {
+ return elt->id == id_to_disable;
+ });
+ additional_vec.erase(it, additional_vec.end());
+#endif
+
+ /* Refcount is dropped, so the symbol should be freed, ensure that nothing else owns this symbol */
+ g_assert(deleted_element_refcount.use_count() == 1);
+ }
+
+ /* Remove no longer used stuff */
+ enabled_symbols.reset();
+ disabled_symbols.reset();
+
+ /* Deal with the delayed conditions */
+ msg_debug_cache("resolving delayed conditions: %d in list", (int) delayed_conditions->size());
+ for (const auto &delayed_cond: *delayed_conditions) {
+ auto it = get_item_by_name_mut(delayed_cond.sym, true);
+
+ if (it == nullptr) {
+ msg_err_cache(
+ "cannot register delayed condition for %s",
+ delayed_cond.sym.c_str());
+ luaL_unref(delayed_cond.L, LUA_REGISTRYINDEX, delayed_cond.cbref);
+ }
+ else {
+ if (!it->add_condition(delayed_cond.L, delayed_cond.cbref)) {
+ msg_err_cache(
+ "cannot register delayed condition for %s: virtual parent; qed",
+ delayed_cond.sym.c_str());
+ g_abort();
+ }
+
+ msg_debug_cache("added a condition to the symbol %s", it->symbol.c_str());
+ }
+ }
+ delayed_conditions.reset();
+
+ msg_debug_cache("process dependencies");
+ for (const auto &[_id, it]: items_by_id) {
+ it->process_deps(*this);
+ }
+
+ /* Sorting stuff */
+ constexpr auto postfilters_cmp = [](const auto &it1, const auto &it2) -> bool {
+ return it1->priority < it2->priority;
+ };
+ constexpr auto prefilters_cmp = [](const auto &it1, const auto &it2) -> bool {
+ return it1->priority > it2->priority;
+ };
+
+ msg_debug_cache("sorting stuff");
+ std::stable_sort(std::begin(connfilters), std::end(connfilters), prefilters_cmp);
+ std::stable_sort(std::begin(prefilters), std::end(prefilters), prefilters_cmp);
+ std::stable_sort(std::begin(postfilters), std::end(postfilters), postfilters_cmp);
+ std::stable_sort(std::begin(idempotent), std::end(idempotent), postfilters_cmp);
+
+ resort();
+
+ /* Connect metric symbols with symcache symbols */
+ if (cfg->symbols) {
+ msg_debug_cache("connect metrics");
+ g_hash_table_foreach(cfg->symbols,
+ symcache::metric_connect_cb,
+ (void *) this);
+ }
+
+ return res;
+}
+
+auto symcache::load_items() -> bool
+{
+ auto cached_map = util::raii_mmaped_file::mmap_shared(cfg->cache_filename,
+ O_RDONLY, PROT_READ);
+
+ if (!cached_map.has_value()) {
+ if (cached_map.error().category == util::error_category::CRITICAL) {
+ msg_err_cache("%s", cached_map.error().error_message.data());
+ }
+ else {
+ msg_info_cache("%s", cached_map.error().error_message.data());
+ }
+ return false;
+ }
+
+
+ if (cached_map->get_size() < (gint) sizeof(symcache_header)) {
+ msg_info_cache("cannot use file %s, truncated: %z", cfg->cache_filename,
+ errno, strerror(errno));
+ return false;
+ }
+
+ const auto *hdr = (struct symcache_header *) cached_map->get_map();
+
+ if (memcmp(hdr->magic, symcache_magic,
+ sizeof(symcache_magic)) != 0) {
+ msg_info_cache("cannot use file %s, bad magic", cfg->cache_filename);
+
+ return false;
+ }
+
+ auto *parser = ucl_parser_new(0);
+ const auto *p = (const std::uint8_t *) (hdr + 1);
+
+ if (!ucl_parser_add_chunk(parser, p, cached_map->get_size() - sizeof(*hdr))) {
+ msg_info_cache("cannot use file %s, cannot parse: %s", cfg->cache_filename,
+ ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+
+ return false;
+ }
+
+ auto *top = ucl_parser_get_object(parser);
+ ucl_parser_free(parser);
+
+ if (top == nullptr || ucl_object_type(top) != UCL_OBJECT) {
+ msg_info_cache("cannot use file %s, bad object", cfg->cache_filename);
+ ucl_object_unref(top);
+
+ return false;
+ }
+
+ auto it = ucl_object_iterate_new(top);
+ const ucl_object_t *cur;
+ while ((cur = ucl_object_iterate_safe(it, true)) != nullptr) {
+ auto item_it = items_by_symbol.find(ucl_object_key(cur));
+
+ if (item_it != items_by_symbol.end()) {
+ auto item = item_it->second;
+ /* Copy saved info */
+ /*
+ * XXX: don't save or load weight, it should be obtained from the
+ * metric
+ */
+#if 0
+ elt = ucl_object_lookup (cur, "weight");
+
+ if (elt) {
+ w = ucl_object_todouble (elt);
+ if (w != 0) {
+ item->weight = w;
+ }
+ }
+#endif
+ const auto *elt = ucl_object_lookup(cur, "time");
+ if (elt) {
+ item->st->avg_time = ucl_object_todouble(elt);
+ }
+
+ elt = ucl_object_lookup(cur, "count");
+ if (elt) {
+ item->st->total_hits = ucl_object_toint(elt);
+ item->last_count = item->st->total_hits;
+ }
+
+ elt = ucl_object_lookup(cur, "frequency");
+ if (elt && ucl_object_type(elt) == UCL_OBJECT) {
+ const ucl_object_t *freq_elt;
+
+ freq_elt = ucl_object_lookup(elt, "avg");
+
+ if (freq_elt) {
+ item->st->avg_frequency = ucl_object_todouble(freq_elt);
+ }
+ freq_elt = ucl_object_lookup(elt, "stddev");
+
+ if (freq_elt) {
+ item->st->stddev_frequency = ucl_object_todouble(freq_elt);
+ }
+ }
+
+ if (item->is_virtual() && !item->is_ghost()) {
+ const auto &parent = item->get_parent(*this);
+
+ if (parent) {
+ if (parent->st->weight < item->st->weight) {
+ parent->st->weight = item->st->weight;
+ }
+ }
+ /*
+ * We maintain avg_time for virtual symbols equal to the
+ * parent item avg_time
+ */
+ item->st->avg_time = parent->st->avg_time;
+ }
+
+ total_weight += fabs(item->st->weight);
+ total_hits += item->st->total_hits;
+ }
+ }
+
+ ucl_object_iterate_free(it);
+ ucl_object_unref(top);
+
+ return true;
+}
+
+template<typename T>
+static constexpr auto round_to_hundreds(T x)
+{
+ return (::floor(x) * 100.0) / 100.0;
+}
+
+bool symcache::save_items() const
+{
+ if (cfg->cache_filename == nullptr) {
+ return false;
+ }
+
+ auto file_sink = util::raii_file_sink::create(cfg->cache_filename,
+ O_WRONLY | O_TRUNC, 00644);
+
+ if (!file_sink.has_value()) {
+ if (errno == EEXIST) {
+ /* Some other process is already writing data, give up silently */
+ return false;
+ }
+
+ msg_err_cache("%s", file_sink.error().error_message.data());
+
+ return false;
+ }
+
+ struct symcache_header hdr;
+ memset(&hdr, 0, sizeof(hdr));
+ memcpy(hdr.magic, symcache_magic, sizeof(symcache_magic));
+
+ if (write(file_sink->get_fd(), &hdr, sizeof(hdr)) == -1) {
+ msg_err_cache("cannot write to file %s, error %d, %s", cfg->cache_filename,
+ errno, strerror(errno));
+
+ return false;
+ }
+
+ auto *top = ucl_object_typed_new(UCL_OBJECT);
+
+ for (const auto &it: items_by_symbol) {
+ auto item = it.second;
+ auto elt = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(elt,
+ ucl_object_fromdouble(round_to_hundreds(item->st->weight)),
+ "weight", 0, false);
+ ucl_object_insert_key(elt,
+ ucl_object_fromdouble(round_to_hundreds(item->st->time_counter.mean)),
+ "time", 0, false);
+ ucl_object_insert_key(elt, ucl_object_fromint(item->st->total_hits),
+ "count", 0, false);
+
+ auto *freq = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(freq,
+ ucl_object_fromdouble(round_to_hundreds(item->st->frequency_counter.mean)),
+ "avg", 0, false);
+ ucl_object_insert_key(freq,
+ ucl_object_fromdouble(round_to_hundreds(item->st->frequency_counter.stddev)),
+ "stddev", 0, false);
+ ucl_object_insert_key(elt, freq, "frequency", 0, false);
+
+ ucl_object_insert_key(top, elt, it.first.data(), 0, true);
+ }
+
+ auto fp = fdopen(file_sink->get_fd(), "a");
+ auto *efunc = ucl_object_emit_file_funcs(fp);
+ auto ret = ucl_object_emit_full(top, UCL_EMIT_JSON_COMPACT, efunc, nullptr);
+ ucl_object_emit_funcs_free(efunc);
+ ucl_object_unref(top);
+ fclose(fp);
+
+ return ret;
+}
+
+auto symcache::metric_connect_cb(void *k, void *v, void *ud) -> void
+{
+ auto *cache = (symcache *) ud;
+ const auto *sym = (const char *) k;
+ auto *s = (struct rspamd_symbol *) v;
+ auto weight = *s->weight_ptr;
+ auto *item = cache->get_item_by_name_mut(sym, false);
+
+ if (item) {
+ item->st->weight = weight;
+ s->cache_item = (void *) item;
+ }
+}
+
+
+auto symcache::get_item_by_id(int id, bool resolve_parent) const -> const cache_item *
+{
+ if (id < 0 || id >= items_by_id.size()) {
+ msg_err_cache("internal error: requested item with id %d, when we have just %d items in the cache",
+ id, (int) items_by_id.size());
+ return nullptr;
+ }
+
+ const auto &maybe_item = rspamd::find_map(items_by_id, id);
+
+ if (!maybe_item.has_value()) {
+ msg_err_cache("internal error: requested item with id %d but it is empty; qed",
+ id);
+ return nullptr;
+ }
+
+ const auto &item = maybe_item.value().get();
+
+ if (resolve_parent && item->is_virtual()) {
+ return item->get_parent(*this);
+ }
+
+ return item.get();
+}
+
+auto symcache::get_item_by_id_mut(int id, bool resolve_parent) const -> cache_item *
+{
+ if (id < 0 || id >= items_by_id.size()) {
+ msg_err_cache("internal error: requested item with id %d, when we have just %d items in the cache",
+ id, (int) items_by_id.size());
+ return nullptr;
+ }
+
+ const auto &maybe_item = rspamd::find_map(items_by_id, id);
+
+ if (!maybe_item.has_value()) {
+ msg_err_cache("internal error: requested item with id %d but it is empty; qed",
+ id);
+ return nullptr;
+ }
+
+ const auto &item = maybe_item.value().get();
+
+ if (resolve_parent && item->is_virtual()) {
+ return const_cast<cache_item *>(item->get_parent(*this));
+ }
+
+ return item.get();
+}
+
+auto symcache::get_item_by_name(std::string_view name, bool resolve_parent) const -> const cache_item *
+{
+ auto it = items_by_symbol.find(name);
+
+ if (it == items_by_symbol.end()) {
+ return nullptr;
+ }
+
+ if (resolve_parent && it->second->is_virtual()) {
+ it->second->resolve_parent(*this);
+ return it->second->get_parent(*this);
+ }
+
+ return it->second;
+}
+
+auto symcache::get_item_by_name_mut(std::string_view name, bool resolve_parent) const -> cache_item *
+{
+ auto it = items_by_symbol.find(name);
+
+ if (it == items_by_symbol.end()) {
+ return nullptr;
+ }
+
+ if (resolve_parent && it->second->is_virtual()) {
+ return (cache_item *) it->second->get_parent(*this);
+ }
+
+ return it->second;
+}
+
+auto symcache::add_dependency(int id_from, std::string_view to, int virtual_id_from) -> void
+{
+ g_assert(id_from >= 0 && id_from < (gint) items_by_id.size());
+ const auto &source = items_by_id[id_from];
+ g_assert(source.get() != nullptr);
+
+ source->deps.emplace_back(nullptr,
+ std::string(to),
+ id_from,
+ -1);
+
+
+ if (virtual_id_from >= 0) {
+ g_assert(virtual_id_from < (gint) items_by_id.size());
+ /* We need that for settings id propagation */
+ const auto &vsource = items_by_id[virtual_id_from];
+ g_assert(vsource.get() != nullptr);
+ vsource->deps.emplace_back(nullptr,
+ std::string(to),
+ -1,
+ virtual_id_from);
+ }
+}
+
+auto symcache::resort() -> void
+{
+ auto log_func = RSPAMD_LOG_FUNC;
+ auto ord = std::make_shared<order_generation>(filters.size() +
+ prefilters.size() +
+ composites.size() +
+ postfilters.size() +
+ idempotent.size() +
+ connfilters.size() +
+ classifiers.size(),
+ cur_order_gen);
+
+ for (auto &it: filters) {
+ if (it) {
+ total_hits += it->st->total_hits;
+ /* Unmask topological order */
+ it->order = 0;
+ ord->d.emplace_back(it->getptr());
+ }
+ }
+
+ enum class tsort_mask {
+ PERM,
+ TEMP
+ };
+
+ constexpr auto tsort_unmask = [](cache_item *it) -> auto {
+ return (it->order & ~((1u << 31) | (1u << 30)));
+ };
+
+ /* Recursive topological sort helper */
+ const auto tsort_visit = [&](cache_item *it, unsigned cur_order, auto &&rec) {
+ constexpr auto tsort_mark = [](cache_item *it, tsort_mask how) {
+ switch (how) {
+ case tsort_mask::PERM:
+ it->order |= (1u << 31);
+ break;
+ case tsort_mask::TEMP:
+ it->order |= (1u << 30);
+ break;
+ }
+ };
+ constexpr auto tsort_is_marked = [](cache_item *it, tsort_mask how) {
+ switch (how) {
+ case tsort_mask::PERM:
+ return (it->order & (1u << 31));
+ case tsort_mask::TEMP:
+ return (it->order & (1u << 30));
+ }
+
+ return 100500u; /* Because fuck compilers, that's why */
+ };
+
+ if (tsort_is_marked(it, tsort_mask::PERM)) {
+ if (cur_order > tsort_unmask(it)) {
+ /* Need to recalculate the whole chain */
+ it->order = cur_order; /* That also removes all masking */
+ }
+ else {
+ /* We are fine, stop DFS */
+ return;
+ }
+ }
+ else if (tsort_is_marked(it, tsort_mask::TEMP)) {
+ msg_err_cache_lambda("cyclic dependencies found when checking '%s'!",
+ it->symbol.c_str());
+ return;
+ }
+
+ tsort_mark(it, tsort_mask::TEMP);
+ msg_debug_cache_lambda("visiting node: %s (%d)", it->symbol.c_str(), cur_order);
+
+ for (const auto &dep: it->deps) {
+ msg_debug_cache_lambda("visiting dep: %s (%d)", dep.item->symbol.c_str(), cur_order + 1);
+ rec(dep.item, cur_order + 1, rec);
+ }
+
+ it->order = cur_order;
+ tsort_mark(it, tsort_mask::PERM);
+ };
+ /*
+ * Topological sort
+ */
+ total_hits = 0;
+ auto used_items = ord->d.size();
+
+ for (const auto &it: ord->d) {
+ if (it->order == 0) {
+ tsort_visit(it.get(), 0, tsort_visit);
+ }
+ }
+
+
+ /* Main sorting comparator */
+ constexpr auto score_functor = [](auto w, auto f, auto t) -> auto {
+ auto time_alpha = 1.0, weight_alpha = 0.1, freq_alpha = 0.01;
+
+ return ((w > 0.0 ? w : weight_alpha) * (f > 0.0 ? f : freq_alpha) /
+ (t > time_alpha ? t : time_alpha));
+ };
+
+ auto cache_order_cmp = [&](const auto &it1, const auto &it2) -> auto {
+ constexpr const auto topology_mult = 1e7,
+ priority_mult = 1e6,
+ augmentations1_mult = 1e5;
+ auto w1 = tsort_unmask(it1.get()) * topology_mult,
+ w2 = tsort_unmask(it2.get()) * topology_mult;
+
+ w1 += it1->priority * priority_mult;
+ w2 += it2->priority * priority_mult;
+ w1 += it1->get_augmentation_weight() * augmentations1_mult;
+ w2 += it2->get_augmentation_weight() * augmentations1_mult;
+
+ auto avg_freq = ((double) total_hits / used_items);
+ auto avg_weight = (total_weight / used_items);
+ auto f1 = (double) it1->st->total_hits / avg_freq;
+ auto f2 = (double) it2->st->total_hits / avg_freq;
+ auto weight1 = std::fabs(it1->st->weight) / avg_weight;
+ auto weight2 = std::fabs(it2->st->weight) / avg_weight;
+ auto t1 = it1->st->avg_time;
+ auto t2 = it2->st->avg_time;
+ w1 += score_functor(weight1, f1, t1);
+ w2 += score_functor(weight2, f2, t2);
+
+ return w1 > w2;
+ };
+
+ std::stable_sort(std::begin(ord->d), std::end(ord->d), cache_order_cmp);
+ /*
+ * Here lives some ugly legacy!
+ * We have several filters classes, connfilters, prefilters, filters... etc
+ *
+ * Our order is meaningful merely for filters, but we have to add other classes
+ * to understand if those symbols are checked or disabled.
+ * We can disable symbols for almost everything but not for virtual symbols.
+ * The rule of thumb is that if a symbol has explicit parent, then it is a
+ * virtual symbol that follows it's special rules
+ */
+
+ /*
+ * We enrich ord with all other symbol types without any sorting,
+ * as it is done in another place
+ */
+ constexpr auto append_items_vec = [](const auto &vec, auto &out) {
+ for (const auto &it: vec) {
+ if (it) {
+ out.emplace_back(it->getptr());
+ }
+ }
+ };
+
+ append_items_vec(connfilters, ord->d);
+ append_items_vec(prefilters, ord->d);
+ append_items_vec(postfilters, ord->d);
+ append_items_vec(idempotent, ord->d);
+ append_items_vec(composites, ord->d);
+ append_items_vec(classifiers, ord->d);
+
+ /* After sorting is done, we can assign all elements in the by_symbol hash */
+ for (const auto [i, it]: rspamd::enumerate(ord->d)) {
+ ord->by_symbol.emplace(it->get_name(), i);
+ ord->by_cache_id[it->id] = i;
+ }
+ /* Finally set the current order */
+ std::swap(ord, items_by_order);
+}
+
+auto symcache::add_symbol_with_callback(std::string_view name,
+ int priority,
+ symbol_func_t func,
+ void *user_data,
+ int flags_and_type) -> int
+{
+ auto real_type_pair_maybe = item_type_from_c(flags_and_type);
+
+ if (!real_type_pair_maybe.has_value()) {
+ msg_err_cache("incompatible flags when adding %s: %s", name.data(),
+ real_type_pair_maybe.error().c_str());
+ return -1;
+ }
+
+ auto real_type_pair = real_type_pair_maybe.value();
+
+ if (real_type_pair.first != symcache_item_type::FILTER) {
+ real_type_pair.second |= SYMBOL_TYPE_NOSTAT;
+ }
+ if (real_type_pair.second & (SYMBOL_TYPE_GHOST | SYMBOL_TYPE_CALLBACK)) {
+ real_type_pair.second |= SYMBOL_TYPE_NOSTAT;
+ }
+
+ if (real_type_pair.first == symcache_item_type::VIRTUAL) {
+ msg_err_cache("trying to add virtual symbol %s as real (no parent)", name.data());
+ return -1;
+ }
+
+ std::string static_string_name;
+
+ if (name.empty()) {
+ static_string_name = fmt::format("AUTO_{}_{}", (void *) func, user_data);
+ msg_warn_cache("trying to add an empty symbol name, convert it to %s",
+ static_string_name.c_str());
+ }
+ else {
+ static_string_name = name;
+ }
+
+ if (real_type_pair.first == symcache_item_type::IDEMPOTENT && priority != 0) {
+ msg_warn_cache("priority has been set for idempotent symbol %s: %d",
+ static_string_name.c_str(), priority);
+ }
+
+ if ((real_type_pair.second & SYMBOL_TYPE_FINE) && priority == 0) {
+ /* Adjust priority for negative weighted symbols */
+ priority = 1;
+ }
+
+ if (items_by_symbol.contains(static_string_name)) {
+ msg_err_cache("duplicate symbol name: %s", static_string_name.data());
+ return -1;
+ }
+
+ auto id = items_by_id.size();
+
+ auto item = cache_item::create_with_function(static_pool, id,
+ std::move(static_string_name),
+ priority, func, user_data,
+ real_type_pair.first, real_type_pair.second);
+
+ items_by_symbol.emplace(item->get_name(), item.get());
+ get_item_specific_vector(*item).push_back(item.get());
+ items_by_id.emplace(id, std::move(item));// Takes ownership
+
+ if (!(real_type_pair.second & SYMBOL_TYPE_NOSTAT)) {
+ cksum = t1ha(name.data(), name.size(), cksum);
+ stats_symbols_count++;
+ }
+
+ return id;
+}
+
+auto symcache::add_virtual_symbol(std::string_view name, int parent_id, int flags_and_type) -> int
+{
+ if (name.empty()) {
+ msg_err_cache("cannot register a virtual symbol with no name; qed");
+ return -1;
+ }
+
+ auto real_type_pair_maybe = item_type_from_c(flags_and_type);
+
+ if (!real_type_pair_maybe.has_value()) {
+ msg_err_cache("incompatible flags when adding %s: %s", name.data(),
+ real_type_pair_maybe.error().c_str());
+ return -1;
+ }
+
+ auto real_type_pair = real_type_pair_maybe.value();
+
+ if (items_by_symbol.contains(name)) {
+ msg_err_cache("duplicate symbol name: %s", name.data());
+ return -1;
+ }
+
+ if (items_by_id.size() < parent_id) {
+ msg_err_cache("parent id %d is out of bounds for virtual symbol %s", parent_id, name.data());
+ return -1;
+ }
+
+ auto id = items_by_id.size();
+
+ auto item = cache_item::create_with_virtual(static_pool,
+ id,
+ std::string{name},
+ parent_id, real_type_pair.first, real_type_pair.second);
+ const auto &parent = items_by_id[parent_id].get();
+ parent->add_child(item.get());
+ items_by_symbol.emplace(item->get_name(), item.get());
+ get_item_specific_vector(*item).push_back(item.get());
+ items_by_id.emplace(id, std::move(item));// Takes ownership
+
+ return id;
+}
+
+auto symcache::set_peak_cb(int cbref) -> void
+{
+ if (peak_cb != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, peak_cb);
+ }
+
+ peak_cb = cbref;
+ msg_info_cache("registered peak callback");
+}
+
+auto symcache::add_delayed_condition(std::string_view sym, int cbref) -> void
+{
+ delayed_conditions->emplace_back(sym, cbref, (lua_State *) cfg->lua_state);
+}
+
+auto symcache::validate(bool strict) -> bool
+{
+ total_weight = 1.0;
+
+ for (auto &pair: items_by_symbol) {
+ auto &item = pair.second;
+ auto ghost = item->st->weight == 0 ? true : false;
+ auto skipped = !ghost;
+
+ if (item->is_scoreable() && g_hash_table_lookup(cfg->symbols, item->symbol.c_str()) == nullptr) {
+ if (!std::isnan(cfg->unknown_weight)) {
+ item->st->weight = cfg->unknown_weight;
+ auto *s = rspamd_mempool_alloc0_type(static_pool,
+ struct rspamd_symbol);
+ /* Legit as we actually never modify this data */
+ s->name = (char *) item->symbol.c_str();
+ s->weight_ptr = &item->st->weight;
+ g_hash_table_insert(cfg->symbols, (void *) s->name, (void *) s);
+
+ msg_info_cache("adding unknown symbol %s with weight: %.2f",
+ item->symbol.c_str(), cfg->unknown_weight);
+ ghost = false;
+ skipped = false;
+ }
+ else {
+ skipped = true;
+ }
+ }
+ else {
+ skipped = false;
+ }
+
+ if (!ghost && skipped) {
+ if (!(item->flags & SYMBOL_TYPE_SKIPPED)) {
+ item->flags |= SYMBOL_TYPE_SKIPPED;
+ msg_warn_cache("symbol %s has no score registered, skip its check",
+ item->symbol.c_str());
+ }
+ }
+
+ if (ghost) {
+ msg_debug_cache("symbol %s is registered as ghost symbol, it won't be inserted "
+ "to any metric",
+ item->symbol.c_str());
+ }
+
+ if (item->st->weight < 0 && item->priority == 0) {
+ item->priority++;
+ }
+
+ if (item->is_virtual()) {
+ if (!(item->flags & SYMBOL_TYPE_GHOST)) {
+ auto *parent = const_cast<cache_item *>(item->get_parent(*this));
+
+ if (parent == nullptr) {
+ item->resolve_parent(*this);
+ parent = const_cast<cache_item *>(item->get_parent(*this));
+ }
+
+ if (::fabs(parent->st->weight) < ::fabs(item->st->weight)) {
+ parent->st->weight = item->st->weight;
+ }
+
+ auto p1 = ::abs(item->priority);
+ auto p2 = ::abs(parent->priority);
+
+ if (p1 != p2) {
+ parent->priority = MAX(p1, p2);
+ item->priority = parent->priority;
+ }
+ }
+ }
+
+ total_weight += fabs(item->st->weight);
+ }
+
+ /* Now check each metric item and find corresponding symbol in a cache */
+ auto ret = true;
+ GHashTableIter it;
+ void *k, *v;
+ g_hash_table_iter_init(&it, cfg->symbols);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ auto ignore_symbol = false;
+ auto sym_def = (struct rspamd_symbol *) v;
+
+ if (sym_def && (sym_def->flags &
+ (RSPAMD_SYMBOL_FLAG_IGNORE_METRIC | RSPAMD_SYMBOL_FLAG_DISABLED))) {
+ ignore_symbol = true;
+ }
+
+ if (!ignore_symbol) {
+ if (!items_by_symbol.contains((const char *) k)) {
+ msg_debug_cache(
+ "symbol '%s' has its score defined but there is no "
+ "corresponding rule registered",
+ k);
+ }
+ }
+ else if (sym_def->flags & RSPAMD_SYMBOL_FLAG_DISABLED) {
+ auto item = get_item_by_name_mut((const char *) k, false);
+
+ if (item) {
+ item->enabled = FALSE;
+ }
+ }
+ }
+
+ return ret;
+}
+
+auto symcache::counters() const -> ucl_object_t *
+{
+ auto *top = ucl_object_typed_new(UCL_ARRAY);
+ constexpr const auto round_float = [](const auto x, const int digits) -> auto {
+ const auto power10 = ::pow(10, digits);
+ return (::floor(x * power10) / power10);
+ };
+
+ for (auto &pair: items_by_symbol) {
+ auto &item = pair.second;
+ auto symbol = pair.first;
+
+ auto *obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj, ucl_object_fromlstring(symbol.data(), symbol.size()),
+ "symbol", 0, false);
+
+ if (item->is_virtual()) {
+ if (!(item->flags & SYMBOL_TYPE_GHOST)) {
+ const auto *parent = item->get_parent(*this);
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(round_float(item->st->weight, 3)),
+ "weight", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(round_float(parent->st->avg_frequency, 3)),
+ "frequency", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromint(parent->st->total_hits),
+ "hits", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(round_float(parent->st->avg_time, 3)),
+ "time", 0, false);
+ }
+ else {
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(round_float(item->st->weight, 3)),
+ "weight", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(0.0),
+ "frequency", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(0.0),
+ "hits", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(0.0),
+ "time", 0, false);
+ }
+ }
+ else {
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(round_float(item->st->weight, 3)),
+ "weight", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(round_float(item->st->avg_frequency, 3)),
+ "frequency", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromint(item->st->total_hits),
+ "hits", 0, false);
+ ucl_object_insert_key(obj,
+ ucl_object_fromdouble(round_float(item->st->avg_time, 3)),
+ "time", 0, false);
+ }
+
+ ucl_array_append(top, obj);
+ }
+
+ return top;
+}
+
+auto symcache::periodic_resort(struct ev_loop *ev_loop, double cur_time, double last_resort) -> void
+{
+ for (const auto &item: filters) {
+
+ if (item->update_counters_check_peak(L, ev_loop, cur_time, last_resort)) {
+ auto cur_value = (item->st->total_hits - item->last_count) /
+ (cur_time - last_resort);
+ auto cur_err = (item->st->avg_frequency - cur_value);
+ cur_err *= cur_err;
+ msg_debug_cache("peak found for %s is %.2f, avg: %.2f, "
+ "stddev: %.2f, error: %.2f, peaks: %d",
+ item->symbol.c_str(), cur_value,
+ item->st->avg_frequency,
+ item->st->stddev_frequency,
+ cur_err,
+ item->frequency_peaks);
+
+ if (peak_cb != -1) {
+ struct ev_loop **pbase;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, peak_cb);
+ pbase = (struct ev_loop **) lua_newuserdata(L, sizeof(*pbase));
+ *pbase = ev_loop;
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+ lua_pushlstring(L, item->symbol.c_str(), item->symbol.size());
+ lua_pushnumber(L, item->st->avg_frequency);
+ lua_pushnumber(L, ::sqrt(item->st->stddev_frequency));
+ lua_pushnumber(L, cur_value);
+ lua_pushnumber(L, cur_err);
+
+ if (lua_pcall(L, 6, 0, 0) != 0) {
+ msg_info_cache("call to peak function for %s failed: %s",
+ item->symbol.c_str(), lua_tostring(L, -1));
+ lua_pop(L, 1);
+ }
+ }
+ }
+ }
+}
+
+symcache::~symcache()
+{
+ if (peak_cb != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, peak_cb);
+ }
+}
+
+auto symcache::maybe_resort() -> bool
+{
+ if (items_by_order->generation_id != cur_order_gen) {
+ /*
+ * Cache has been modified, need to resort it
+ */
+ msg_info_cache("symbols cache has been modified since last check:"
+ " old id: %ud, new id: %ud",
+ items_by_order->generation_id, cur_order_gen);
+ resort();
+
+ return true;
+ }
+
+ return false;
+}
+
+auto symcache::get_item_specific_vector(const cache_item &it) -> symcache::items_ptr_vec &
+{
+ switch (it.get_type()) {
+ case symcache_item_type::CONNFILTER:
+ return connfilters;
+ case symcache_item_type::FILTER:
+ return filters;
+ case symcache_item_type::IDEMPOTENT:
+ return idempotent;
+ case symcache_item_type::PREFILTER:
+ return prefilters;
+ case symcache_item_type::POSTFILTER:
+ return postfilters;
+ case symcache_item_type::COMPOSITE:
+ return composites;
+ case symcache_item_type::CLASSIFIER:
+ return classifiers;
+ case symcache_item_type::VIRTUAL:
+ return virtual_symbols;
+ }
+
+ RSPAMD_UNREACHABLE;
+}
+
+auto symcache::process_settings_elt(struct rspamd_config_settings_elt *elt) -> void
+{
+
+ auto id = elt->id;
+
+ if (elt->symbols_disabled) {
+ /* Process denied symbols */
+ ucl_object_iter_t iter = nullptr;
+ const ucl_object_t *cur;
+
+ while ((cur = ucl_object_iterate(elt->symbols_disabled, &iter, true)) != NULL) {
+ const auto *sym = ucl_object_key(cur);
+ auto *item = get_item_by_name_mut(sym, false);
+
+ if (item != nullptr) {
+ if (item->is_virtual()) {
+ /*
+ * Virtual symbols are special:
+ * we ignore them in symcache but prevent them from being
+ * inserted.
+ */
+ item->forbidden_ids.add_id(id);
+ msg_debug_cache("deny virtual symbol %s for settings %ud (%s); "
+ "parent can still be executed",
+ sym, id, elt->name);
+ }
+ else {
+ /* Normal symbol, disable it */
+ item->forbidden_ids.add_id(id);
+ msg_debug_cache("deny symbol %s for settings %ud (%s)",
+ sym, id, elt->name);
+ }
+ }
+ else {
+ msg_warn_cache("cannot find a symbol to disable %s "
+ "when processing settings %ud (%s)",
+ sym, id, elt->name);
+ }
+ }
+ }
+
+ if (elt->symbols_enabled) {
+ ucl_object_iter_t iter = nullptr;
+ const ucl_object_t *cur;
+
+ while ((cur = ucl_object_iterate(elt->symbols_enabled, &iter, true)) != nullptr) {
+ /* Here, we resolve parent and explicitly allow it */
+ const auto *sym = ucl_object_key(cur);
+
+ auto *item = get_item_by_name_mut(sym, false);
+
+ if (item != nullptr) {
+ if (item->is_virtual()) {
+ auto *parent = get_item_by_name_mut(sym, true);
+
+ if (parent) {
+ if (elt->symbols_disabled &&
+ ucl_object_lookup(elt->symbols_disabled, parent->symbol.data())) {
+ msg_err_cache("conflict in %s: cannot enable disabled symbol %s, "
+ "wanted to enable symbol %s",
+ elt->name, parent->symbol.data(), sym);
+ continue;
+ }
+
+ parent->exec_only_ids.add_id(id);
+ msg_debug_cache("allow just execution of symbol %s for settings %ud (%s)",
+ parent->symbol.data(), id, elt->name);
+ }
+ }
+
+ item->allowed_ids.add_id(id);
+ msg_debug_cache("allow execution of symbol %s for settings %ud (%s)",
+ sym, id, elt->name);
+ }
+ else {
+ msg_warn_cache("cannot find a symbol to enable %s "
+ "when processing settings %ud (%s)",
+ sym, id, elt->name);
+ }
+ }
+ }
+}
+
+auto symcache::get_max_timeout(std::vector<std::pair<double, const cache_item *>> &elts) const -> double
+{
+ auto accumulated_timeout = 0.0;
+ auto log_func = RSPAMD_LOG_FUNC;
+ ankerl::unordered_dense::set<const cache_item *> seen_items;
+
+ auto get_item_timeout = [](cache_item *it) {
+ return it->get_numeric_augmentation("timeout").value_or(0.0);
+ };
+
+ /* This function returns the timeout for an item and all it's dependencies */
+ auto get_filter_timeout = [&](cache_item *it, auto self) -> double {
+ auto own_timeout = get_item_timeout(it);
+ auto max_child_timeout = 0.0;
+
+ for (const auto &dep: it->deps) {
+ auto cld_timeout = self(dep.item, self);
+
+ if (cld_timeout > max_child_timeout) {
+ max_child_timeout = cld_timeout;
+ }
+ }
+
+ return own_timeout + max_child_timeout;
+ };
+
+ /* For prefilters and postfilters, we just care about priorities */
+ auto pre_postfilter_iter = [&](const items_ptr_vec &vec) -> double {
+ auto saved_priority = -1;
+ auto max_timeout = 0.0, added_timeout = 0.0;
+ const cache_item *max_elt = nullptr;
+ for (const auto &it: vec) {
+ if (it->priority != saved_priority && max_elt != nullptr && max_timeout > 0) {
+ if (!seen_items.contains(max_elt)) {
+ accumulated_timeout += max_timeout;
+ added_timeout += max_timeout;
+
+ msg_debug_cache_lambda("added %.2f to the timeout (%.2f) as the priority has changed (%d -> %d); "
+ "symbol: %s",
+ max_timeout, accumulated_timeout, saved_priority, it->priority,
+ max_elt->symbol.c_str());
+ elts.emplace_back(max_timeout, max_elt);
+ seen_items.insert(max_elt);
+ }
+ max_timeout = 0;
+ saved_priority = it->priority;
+ max_elt = nullptr;
+ }
+
+ auto timeout = get_item_timeout(it);
+
+ if (timeout > max_timeout) {
+ max_timeout = timeout;
+ max_elt = it;
+ }
+ }
+
+ if (max_elt != nullptr && max_timeout > 0) {
+ if (!seen_items.contains(max_elt)) {
+ accumulated_timeout += max_timeout;
+ added_timeout += max_timeout;
+
+ msg_debug_cache_lambda("added %.2f to the timeout (%.2f) end of processing; "
+ "symbol: %s",
+ max_timeout, accumulated_timeout,
+ max_elt->symbol.c_str());
+ elts.emplace_back(max_timeout, max_elt);
+ seen_items.insert(max_elt);
+ }
+ }
+
+ return added_timeout;
+ };
+
+ auto prefilters_timeout = pre_postfilter_iter(this->prefilters);
+
+ /* For normal filters, we check the maximum chain of the dependencies
+ * This function might have O(N^2) complexity if all symbols are in a single
+ * dependencies chain. But it is not the case in practice
+ */
+ double max_filters_timeout = 0;
+ for (const auto &it: this->filters) {
+ auto timeout = get_filter_timeout(it, get_filter_timeout);
+
+ if (timeout > max_filters_timeout) {
+ max_filters_timeout = timeout;
+ if (!seen_items.contains(it)) {
+ elts.emplace_back(timeout, it);
+ seen_items.insert(it);
+ }
+ }
+ }
+
+ accumulated_timeout += max_filters_timeout;
+
+ auto postfilters_timeout = pre_postfilter_iter(this->postfilters);
+ auto idempotent_timeout = pre_postfilter_iter(this->idempotent);
+
+ /* Sort in decreasing order by timeout */
+ std::stable_sort(std::begin(elts), std::end(elts),
+ [](const auto &p1, const auto &p2) {
+ return p1.first > p2.first;
+ });
+
+ msg_debug_cache("overall cache timeout: %.2f, %.2f from prefilters,"
+ " %.2f from postfilters, %.2f from idempotent filters,"
+ " %.2f from normal filters",
+ accumulated_timeout, prefilters_timeout, postfilters_timeout,
+ idempotent_timeout, max_filters_timeout);
+
+ return accumulated_timeout;
+}
+
+}// namespace rspamd::symcache \ No newline at end of file
diff --git a/src/libserver/symcache/symcache_internal.hxx b/src/libserver/symcache/symcache_internal.hxx
new file mode 100644
index 0000000..255a4b1
--- /dev/null
+++ b/src/libserver/symcache/symcache_internal.hxx
@@ -0,0 +1,652 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Internal C++ structures and classes for symcache
+ */
+
+#ifndef RSPAMD_SYMCACHE_INTERNAL_HXX
+#define RSPAMD_SYMCACHE_INTERNAL_HXX
+#pragma once
+
+#include <cmath>
+#include <cstdlib>
+#include <cstdint>
+#include <utility>
+#include <vector>
+#include <string>
+#include <string_view>
+#include <memory>
+#include <variant>
+
+#include "rspamd_symcache.h"
+#include "contrib/libev/ev.h"
+#include "contrib/ankerl/unordered_dense.h"
+#include "contrib/expected/expected.hpp"
+#include "cfg_file.h"
+
+#include "symcache_id_list.hxx"
+
+#define msg_err_cache(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "symcache", log_tag(), \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_err_cache_lambda(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "symcache", log_tag(), \
+ log_func, \
+ __VA_ARGS__)
+#define msg_err_cache_task(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "symcache", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_cache(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "symcache", log_tag(), \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_cache(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "symcache", log_tag(), \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_cache(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ ::rspamd::symcache::rspamd_symcache_log_id, "symcache", log_tag(), \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_cache_lambda(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ ::rspamd::symcache::rspamd_symcache_log_id, "symcache", log_tag(), \
+ log_func, \
+ __VA_ARGS__)
+#define msg_debug_cache_task(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ ::rspamd::symcache::rspamd_symcache_log_id, "symcache", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_cache_task_lambda(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ ::rspamd::symcache::rspamd_symcache_log_id, "symcache", task->task_pool->tag.uid, \
+ log_func, \
+ __VA_ARGS__)
+
+struct lua_State;
+
+namespace rspamd::symcache {
+
+/* Defined in symcache_impl.cxx */
+extern int rspamd_symcache_log_id;
+
+static const std::uint8_t symcache_magic[8] = {'r', 's', 'c', 2, 0, 0, 0, 0};
+
+struct symcache_header {
+ std::uint8_t magic[8];
+ unsigned int nitems;
+ std::uint8_t checksum[64];
+ std::uint8_t unused[128];
+};
+
+struct cache_item;
+using cache_item_ptr = std::shared_ptr<cache_item>;
+
+/**
+ * This structure is intended to keep the current ordering for all symbols
+ * It is designed to be shared among all tasks and keep references to the real
+ * symbols.
+ * If some symbol has been added or removed to the symbol cache, it will not affect
+ * the current order, and it will only be regenerated for the subsequent tasks.
+ * This allows safe and no copy sharing and keeping track of all symbols in the
+ * cache runtime.
+ */
+struct order_generation {
+ /* All items ordered */
+ std::vector<cache_item_ptr> d;
+ /* Mapping from symbol name to the position in the order array */
+ ankerl::unordered_dense::map<std::string_view, unsigned int> by_symbol;
+ /* Mapping from symbol id to the position in the order array */
+ ankerl::unordered_dense::map<unsigned int, unsigned int> by_cache_id;
+ /* It matches cache->generation_id; if not, a fresh ordering is required */
+ unsigned int generation_id;
+
+ explicit order_generation(std::size_t nelts, unsigned id)
+ : generation_id(id)
+ {
+ d.reserve(nelts);
+ by_symbol.reserve(nelts);
+ by_cache_id.reserve(nelts);
+ }
+
+ auto size() const -> auto
+ {
+ return d.size();
+ }
+};
+
+using order_generation_ptr = std::shared_ptr<order_generation>;
+
+
+struct delayed_cache_dependency {
+ std::string from;
+ std::string to;
+
+ delayed_cache_dependency(std::string_view _from, std::string_view _to)
+ : from(_from), to(_to)
+ {
+ }
+};
+
+struct delayed_cache_condition {
+ std::string sym;
+ int cbref;
+ lua_State *L;
+
+public:
+ delayed_cache_condition(std::string_view sym, int cbref, lua_State *L)
+ : sym(sym), cbref(cbref), L(L)
+ {
+ }
+};
+
+class delayed_symbol_elt {
+private:
+ std::variant<std::string, rspamd_regexp_t *> content;
+
+public:
+ /* Disable copy */
+ delayed_symbol_elt() = delete;
+ delayed_symbol_elt(const delayed_symbol_elt &) = delete;
+ delayed_symbol_elt &operator=(const delayed_symbol_elt &) = delete;
+ /* Enable move */
+ delayed_symbol_elt(delayed_symbol_elt &&other) noexcept = default;
+ delayed_symbol_elt &operator=(delayed_symbol_elt &&other) noexcept = default;
+
+ explicit delayed_symbol_elt(std::string_view elt) noexcept
+ {
+ if (!elt.empty() && elt[0] == '/') {
+ /* Possibly regexp */
+ auto *re = rspamd_regexp_new_len(elt.data(), elt.size(), nullptr, nullptr);
+
+ if (re != nullptr) {
+ std::get<rspamd_regexp_t *>(content) = re;
+ }
+ else {
+ std::get<std::string>(content) = elt;
+ }
+ }
+ else {
+ std::get<std::string>(content) = elt;
+ }
+ }
+
+ ~delayed_symbol_elt()
+ {
+ if (std::holds_alternative<rspamd_regexp_t *>(content)) {
+ rspamd_regexp_unref(std::get<rspamd_regexp_t *>(content));
+ }
+ }
+
+ auto matches(std::string_view what) const -> bool
+ {
+ return std::visit([&](auto &elt) {
+ using T = typeof(elt);
+ if constexpr (std::is_same_v<T, rspamd_regexp_t *>) {
+ if (rspamd_regexp_match(elt, what.data(), what.size(), false)) {
+ return true;
+ }
+ }
+ else if constexpr (std::is_same_v<T, std::string>) {
+ return elt == what;
+ }
+
+ return false;
+ },
+ content);
+ }
+
+ auto to_string_view() const -> std::string_view
+ {
+ return std::visit([&](auto &elt) {
+ using T = typeof(elt);
+ if constexpr (std::is_same_v<T, rspamd_regexp_t *>) {
+ return std::string_view{rspamd_regexp_get_pattern(elt)};
+ }
+ else if constexpr (std::is_same_v<T, std::string>) {
+ return std::string_view{elt};
+ }
+
+ return std::string_view{};
+ },
+ content);
+ }
+};
+
+struct delayed_symbol_elt_equal {
+ using is_transparent = void;
+ auto operator()(const delayed_symbol_elt &a, const delayed_symbol_elt &b) const
+ {
+ return a.to_string_view() == b.to_string_view();
+ }
+ auto operator()(const delayed_symbol_elt &a, const std::string_view &b) const
+ {
+ return a.to_string_view() == b;
+ }
+ auto operator()(const std::string_view &a, const delayed_symbol_elt &b) const
+ {
+ return a == b.to_string_view();
+ }
+};
+
+struct delayed_symbol_elt_hash {
+ using is_transparent = void;
+ auto operator()(const delayed_symbol_elt &a) const
+ {
+ return ankerl::unordered_dense::hash<std::string_view>()(a.to_string_view());
+ }
+ auto operator()(const std::string_view &a) const
+ {
+ return ankerl::unordered_dense::hash<std::string_view>()(a);
+ }
+};
+
+class symcache {
+private:
+ using items_ptr_vec = std::vector<cache_item *>;
+ /* Map indexed by symbol name: all symbols must have unique names, so this map holds ownership */
+ ankerl::unordered_dense::map<std::string_view, cache_item *> items_by_symbol;
+ ankerl::unordered_dense::map<int, cache_item_ptr> items_by_id;
+
+ /* Items sorted into some order */
+ order_generation_ptr items_by_order;
+ unsigned int cur_order_gen;
+
+ /* Specific vectors for execution/iteration */
+ items_ptr_vec connfilters;
+ items_ptr_vec prefilters;
+ items_ptr_vec filters;
+ items_ptr_vec postfilters;
+ items_ptr_vec composites;
+ items_ptr_vec idempotent;
+ items_ptr_vec classifiers;
+ items_ptr_vec virtual_symbols;
+
+ /* These are stored within pointer to clean up after init */
+ std::unique_ptr<std::vector<delayed_cache_dependency>> delayed_deps;
+ std::unique_ptr<std::vector<delayed_cache_condition>> delayed_conditions;
+ /* Delayed statically enabled or disabled symbols */
+ using delayed_symbol_names = ankerl::unordered_dense::set<delayed_symbol_elt,
+ delayed_symbol_elt_hash, delayed_symbol_elt_equal>;
+ std::unique_ptr<delayed_symbol_names> disabled_symbols;
+ std::unique_ptr<delayed_symbol_names> enabled_symbols;
+
+ rspamd_mempool_t *static_pool;
+ std::uint64_t cksum;
+ double total_weight;
+ std::size_t stats_symbols_count;
+
+private:
+ std::uint64_t total_hits;
+
+ struct rspamd_config *cfg;
+ lua_State *L;
+ double reload_time;
+ double last_profile;
+
+private:
+ int peak_cb;
+ int cache_id;
+
+private:
+ /* Internal methods */
+ auto load_items() -> bool;
+ auto resort() -> void;
+ auto get_item_specific_vector(const cache_item &) -> items_ptr_vec &;
+ /* Helper for g_hash_table_foreach */
+ static auto metric_connect_cb(void *k, void *v, void *ud) -> void;
+
+public:
+ explicit symcache(struct rspamd_config *cfg)
+ : cfg(cfg)
+ {
+ /* XXX: do we need a special pool for symcache? I don't think so */
+ static_pool = cfg->cfg_pool;
+ reload_time = cfg->cache_reload_time;
+ total_hits = 1;
+ total_weight = 1.0;
+ cksum = 0xdeadbabe;
+ peak_cb = -1;
+ cache_id = rspamd_random_uint64_fast();
+ L = (lua_State *) cfg->lua_state;
+ delayed_conditions = std::make_unique<std::vector<delayed_cache_condition>>();
+ delayed_deps = std::make_unique<std::vector<delayed_cache_dependency>>();
+ }
+
+ virtual ~symcache();
+
+ /**
+ * Saves items on disk (if possible)
+ * @return
+ */
+ auto save_items() const -> bool;
+
+ /**
+ * Get an item by ID
+ * @param id
+ * @param resolve_parent
+ * @return
+ */
+ auto get_item_by_id(int id, bool resolve_parent) const -> const cache_item *;
+ auto get_item_by_id_mut(int id, bool resolve_parent) const -> cache_item *;
+ /**
+ * Get an item by it's name
+ * @param name
+ * @param resolve_parent
+ * @return
+ */
+ auto get_item_by_name(std::string_view name, bool resolve_parent) const -> const cache_item *;
+ /**
+ * Get an item by it's name, mutable pointer
+ * @param name
+ * @param resolve_parent
+ * @return
+ */
+ auto get_item_by_name_mut(std::string_view name, bool resolve_parent) const -> cache_item *;
+
+ /**
+ * Add a direct dependency
+ * @param id_from
+ * @param to
+ * @param virtual_id_from
+ * @return
+ */
+ auto add_dependency(int id_from, std::string_view to, int virtual_id_from) -> void;
+
+ /**
+ * Add a delayed dependency between symbols that will be resolved on the init stage
+ * @param from
+ * @param to
+ */
+ auto add_delayed_dependency(std::string_view from, std::string_view to) -> void
+ {
+ if (!delayed_deps) {
+ delayed_deps = std::make_unique<std::vector<delayed_cache_dependency>>();
+ }
+
+ delayed_deps->emplace_back(from, to);
+ }
+
+ /**
+ * Adds a symbol to the list of the disabled symbols
+ * @param sym
+ * @return
+ */
+ auto disable_symbol_delayed(std::string_view sym) -> bool
+ {
+ if (!disabled_symbols) {
+ disabled_symbols = std::make_unique<delayed_symbol_names>();
+ }
+
+ if (!disabled_symbols->contains(sym)) {
+ disabled_symbols->emplace(sym);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * Adds a symbol to the list of the enabled symbols
+ * @param sym
+ * @return
+ */
+ auto enable_symbol_delayed(std::string_view sym) -> bool
+ {
+ if (!enabled_symbols) {
+ enabled_symbols = std::make_unique<delayed_symbol_names>();
+ }
+
+ if (!enabled_symbols->contains(sym)) {
+ enabled_symbols->emplace(sym);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * Initialises the symbols cache, must be called after all symbols are added
+ * and the config file is loaded
+ */
+ auto init() -> bool;
+
+ /**
+ * Log helper that returns cfg checksum
+ * @return
+ */
+ auto log_tag() const -> const char *
+ {
+ return cfg->checksum;
+ }
+
+ /**
+ * Helper to return a memory pool associated with the cache
+ * @return
+ */
+ auto get_pool() const
+ {
+ return static_pool;
+ }
+
+ /**
+ * A method to add a generic symbol with a callback to couple with C API
+ * @param name name of the symbol, unlike C API it must be "" for callback only (compat) symbols, in this case an automatic name is generated
+ * @param priority
+ * @param func
+ * @param user_data
+ * @param flags_and_type mix of flags and type in a messy C enum
+ * @return id of a new symbol or -1 in case of failure
+ */
+ auto add_symbol_with_callback(std::string_view name,
+ int priority,
+ symbol_func_t func,
+ void *user_data,
+ int flags_and_type) -> int;
+ /**
+ * A method to add a generic virtual symbol with no function associated
+ * @param name must have some value, or a fatal error will strike you
+ * @param parent_id if this param is -1 then this symbol is associated with nothing
+ * @param flags_and_type mix of flags and type in a messy C enum
+ * @return id of a new symbol or -1 in case of failure
+ */
+ auto add_virtual_symbol(std::string_view name, int parent_id,
+ int flags_and_type) -> int;
+
+ /**
+ * Sets a lua callback to be called on peaks in execution time
+ * @param cbref
+ */
+ auto set_peak_cb(int cbref) -> void;
+
+ /**
+ * Add a delayed condition for a symbol that might not be registered yet
+ * @param sym
+ * @param cbref
+ */
+ auto add_delayed_condition(std::string_view sym, int cbref) -> void;
+
+ /**
+ * Returns number of symbols that needs to be checked in statistical algorithm
+ * @return
+ */
+ auto get_stats_symbols_count() const
+ {
+ return stats_symbols_count;
+ }
+
+ /**
+ * Returns a checksum for the cache
+ * @return
+ */
+ auto get_cksum() const
+ {
+ return cksum;
+ }
+
+ /**
+ * Validate symbols in the cache
+ * @param strict
+ * @return
+ */
+ auto validate(bool strict) -> bool;
+
+ /**
+ * Returns counters for the cache
+ * @return
+ */
+ auto counters() const -> ucl_object_t *;
+
+ /**
+ * Adjusts stats of the cache for the periodic counter
+ */
+ auto periodic_resort(struct ev_loop *ev_loop, double cur_time, double last_resort) -> void;
+
+ /**
+ * A simple helper to get the reload time
+ * @return
+ */
+ auto get_reload_time() const
+ {
+ return reload_time;
+ };
+
+ /**
+ * Iterate over all symbols using a specific functor
+ * @tparam Functor
+ * @param f
+ */
+ template<typename Functor>
+ auto symbols_foreach(Functor f) -> void
+ {
+ for (const auto &sym_it: items_by_symbol) {
+ f(sym_it.second);
+ }
+ }
+
+ /**
+ * Iterate over all composites using a specific functor
+ * @tparam Functor
+ * @param f
+ */
+ template<typename Functor>
+ auto composites_foreach(Functor f) -> void
+ {
+ for (const auto &sym_it: composites) {
+ f(sym_it);
+ }
+ }
+
+ /**
+ * Iterate over all composites using a specific functor
+ * @tparam Functor
+ * @param f
+ */
+ template<typename Functor>
+ auto connfilters_foreach(Functor f) -> bool
+ {
+ return std::all_of(std::begin(connfilters), std::end(connfilters),
+ [&](const auto &sym_it) {
+ return f(sym_it);
+ });
+ }
+ template<typename Functor>
+ auto prefilters_foreach(Functor f) -> bool
+ {
+ return std::all_of(std::begin(prefilters), std::end(prefilters),
+ [&](const auto &sym_it) {
+ return f(sym_it);
+ });
+ }
+ template<typename Functor>
+ auto postfilters_foreach(Functor f) -> bool
+ {
+ return std::all_of(std::begin(postfilters), std::end(postfilters),
+ [&](const auto &sym_it) {
+ return f(sym_it);
+ });
+ }
+ template<typename Functor>
+ auto idempotent_foreach(Functor f) -> bool
+ {
+ return std::all_of(std::begin(idempotent), std::end(idempotent),
+ [&](const auto &sym_it) {
+ return f(sym_it);
+ });
+ }
+ template<typename Functor>
+ auto filters_foreach(Functor f) -> bool
+ {
+ return std::all_of(std::begin(filters), std::end(filters),
+ [&](const auto &sym_it) {
+ return f(sym_it);
+ });
+ }
+
+ /**
+ * Resort cache if anything has been changed since last time
+ * @return
+ */
+ auto maybe_resort() -> bool;
+
+ /**
+ * Returns current set of items ordered for sharing ownership
+ * @return
+ */
+ auto get_cache_order() const -> auto
+ {
+ return items_by_order;
+ }
+
+ /**
+ * Get last profile timestamp
+ * @return
+ */
+ auto get_last_profile() const -> auto
+ {
+ return last_profile;
+ }
+
+ /**
+ * Sets last profile timestamp
+ * @param last_profile
+ * @return
+ */
+ auto set_last_profile(double last_profile)
+ {
+ symcache::last_profile = last_profile;
+ }
+
+ /**
+ * Process settings elt identified by id
+ * @param elt
+ */
+ auto process_settings_elt(struct rspamd_config_settings_elt *elt) -> void;
+
+ /**
+ * Returns maximum timeout that is requested by all rules
+ * @return
+ */
+ auto get_max_timeout(std::vector<std::pair<double, const cache_item *>> &elts) const -> double;
+};
+
+
+}// namespace rspamd::symcache
+
+#endif//RSPAMD_SYMCACHE_INTERNAL_HXX
diff --git a/src/libserver/symcache/symcache_item.cxx b/src/libserver/symcache/symcache_item.cxx
new file mode 100644
index 0000000..ac901f5
--- /dev/null
+++ b/src/libserver/symcache/symcache_item.cxx
@@ -0,0 +1,652 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua/lua_common.h"
+#include "symcache_internal.hxx"
+#include "symcache_item.hxx"
+#include "fmt/core.h"
+#include "libserver/task.h"
+#include "libutil/cxx/util.hxx"
+#include <numeric>
+#include <functional>
+
+namespace rspamd::symcache {
+
+enum class augmentation_value_type {
+ NO_VALUE,
+ STRING_VALUE,
+ NUMBER_VALUE,
+};
+
+struct augmentation_info {
+ int weight = 0;
+ int implied_flags = 0;
+ augmentation_value_type value_type = augmentation_value_type::NO_VALUE;
+};
+
+/* A list of internal augmentations that are known to Rspamd with their weight */
+static const auto known_augmentations =
+ ankerl::unordered_dense::map<std::string, augmentation_info, rspamd::smart_str_hash, rspamd::smart_str_equal>{
+ {"passthrough", {.weight = 10, .implied_flags = SYMBOL_TYPE_IGNORE_PASSTHROUGH}},
+ {"single_network", {.weight = 1, .implied_flags = 0}},
+ {"no_network", {.weight = 0, .implied_flags = 0}},
+ {"many_network", {.weight = 1, .implied_flags = 0}},
+ {"important", {.weight = 5, .implied_flags = SYMBOL_TYPE_FINE}},
+ {"timeout", {
+ .weight = 0,
+ .implied_flags = 0,
+ .value_type = augmentation_value_type::NUMBER_VALUE,
+ }}};
+
+auto cache_item::get_parent(const symcache &cache) const -> const cache_item *
+{
+ if (is_virtual()) {
+ const auto &virtual_sp = std::get<virtual_item>(specific);
+
+ return virtual_sp.get_parent(cache);
+ }
+
+ return nullptr;
+}
+
+auto cache_item::get_parent_mut(const symcache &cache) -> cache_item *
+{
+ if (is_virtual()) {
+ auto &virtual_sp = std::get<virtual_item>(specific);
+
+ return virtual_sp.get_parent_mut(cache);
+ }
+
+ return nullptr;
+}
+
+auto cache_item::process_deps(const symcache &cache) -> void
+{
+ /* Allow logging macros to work */
+ auto log_tag = [&]() { return cache.log_tag(); };
+
+ for (auto &dep: deps) {
+ msg_debug_cache("process real dependency %s on %s", symbol.c_str(), dep.sym.c_str());
+ auto *dit = cache.get_item_by_name_mut(dep.sym, true);
+
+ if (dep.vid >= 0) {
+ /* Case of the virtual symbol that depends on another (maybe virtual) symbol */
+ const auto *vdit = cache.get_item_by_name(dep.sym, false);
+
+ if (!vdit) {
+ if (dit) {
+ msg_err_cache("cannot add dependency from %s on %s: no dependency symbol registered",
+ dep.sym.c_str(), dit->symbol.c_str());
+ }
+ }
+ else {
+ msg_debug_cache("process virtual dependency %s(%d) on %s(%d)", symbol.c_str(),
+ dep.vid, vdit->symbol.c_str(), vdit->id);
+
+ unsigned nids = 0;
+
+ /* Propagate ids */
+ msg_debug_cache("check id propagation for dependency %s from %s",
+ symbol.c_str(), dit->symbol.c_str());
+
+ const auto *ids = dit->allowed_ids.get_ids(nids);
+
+ if (nids > 0) {
+ msg_debug_cache("propagate allowed ids from %s to %s",
+ dit->symbol.c_str(), symbol.c_str());
+
+ allowed_ids.set_ids(ids, nids);
+ }
+
+ ids = dit->forbidden_ids.get_ids(nids);
+
+ if (nids > 0) {
+ msg_debug_cache("propagate forbidden ids from %s to %s",
+ dit->symbol.c_str(), symbol.c_str());
+
+ forbidden_ids.set_ids(ids, nids);
+ }
+ }
+ }
+
+ if (dit != nullptr) {
+ if (!dit->is_filter()) {
+ /*
+ * Check sanity:
+ * - filters -> prefilter dependency is OK and always satisfied
+ * - postfilter -> (filter, prefilter) dep is ok
+ * - idempotent -> (any) dep is OK
+ *
+ * Otherwise, emit error
+ * However, even if everything is fine this dep is useless ¯\_(ツ)_/¯
+ */
+ auto ok_dep = false;
+
+ if (dit->get_type() == type) {
+ ok_dep = true;
+ }
+ else if (type < dit->get_type()) {
+ ok_dep = true;
+ }
+
+ if (!ok_dep) {
+ msg_err_cache("cannot add dependency from %s on %s: invalid symbol types",
+ dep.sym.c_str(), symbol.c_str());
+
+ continue;
+ }
+ }
+ else {
+ if (dit->id == id) {
+ msg_err_cache("cannot add dependency on self: %s -> %s "
+ "(resolved to %s)",
+ symbol.c_str(), dep.sym.c_str(), dit->symbol.c_str());
+ }
+ else {
+ /* Create a reverse dep */
+ if (is_virtual()) {
+ auto *parent = get_parent_mut(cache);
+
+ if (parent) {
+ dit->rdeps.emplace_back(parent, parent->symbol, parent->id, -1);
+ dep.item = dit;
+ dep.id = dit->id;
+
+ msg_debug_cache("added reverse dependency from %d on %d", parent->id,
+ dit->id);
+ }
+ }
+ else {
+ dep.item = dit;
+ dep.id = dit->id;
+ dit->rdeps.emplace_back(this, symbol, id, -1);
+ msg_debug_cache("added reverse dependency from %d on %d", id,
+ dit->id);
+ }
+ }
+ }
+ }
+ else if (dep.id >= 0) {
+ msg_err_cache("cannot find dependency on symbol %s for symbol %s",
+ dep.sym.c_str(), symbol.c_str());
+
+ continue;
+ }
+ }
+
+ // Remove empty deps
+ deps.erase(std::remove_if(std::begin(deps), std::end(deps),
+ [](const auto &dep) { return !dep.item; }),
+ std::end(deps));
+}
+
+auto cache_item::resolve_parent(const symcache &cache) -> bool
+{
+ auto log_tag = [&]() { return cache.log_tag(); };
+
+ if (is_virtual()) {
+ auto &virt = std::get<virtual_item>(specific);
+
+ if (virt.get_parent(cache)) {
+ msg_debug_cache("trying to resolve parent twice for %s", symbol.c_str());
+
+ return false;
+ }
+
+ return virt.resolve_parent(cache);
+ }
+ else {
+ msg_warn_cache("trying to resolve a parent for non-virtual symbol %s", symbol.c_str());
+ }
+
+ return false;
+}
+
+auto cache_item::update_counters_check_peak(lua_State *L,
+ struct ev_loop *ev_loop,
+ double cur_time,
+ double last_resort) -> bool
+{
+ auto ret = false;
+ static const double decay_rate = 0.25;
+
+ st->total_hits += st->hits;
+ g_atomic_int_set(&st->hits, 0);
+
+ if (last_count > 0) {
+ auto cur_value = (st->total_hits - last_count) /
+ (cur_time - last_resort);
+ rspamd_set_counter_ema(&st->frequency_counter,
+ cur_value, decay_rate);
+ st->avg_frequency = st->frequency_counter.mean;
+ st->stddev_frequency = st->frequency_counter.stddev;
+
+ auto cur_err = (st->avg_frequency - cur_value);
+ cur_err *= cur_err;
+
+ if (st->frequency_counter.number > 10 &&
+ cur_err > ::sqrt(st->stddev_frequency) * 3) {
+ frequency_peaks++;
+ ret = true;
+ }
+ }
+
+ last_count = st->total_hits;
+
+ if (cd->number > 0) {
+ if (!is_virtual()) {
+ st->avg_time = cd->mean;
+ rspamd_set_counter_ema(&st->time_counter,
+ st->avg_time, decay_rate);
+ st->avg_time = st->time_counter.mean;
+ memset(cd, 0, sizeof(*cd));
+ }
+ }
+
+ return ret;
+}
+
+auto cache_item::inc_frequency(const char *sym_name, symcache &cache) -> void
+{
+ if (sym_name && symbol != sym_name) {
+ if (is_filter()) {
+ const auto *children = get_children();
+ if (children) {
+ /* Likely a callback symbol with some virtual symbol that needs to be adjusted */
+ for (const auto &cld: *children) {
+ if (cld->get_name() == sym_name) {
+ cld->inc_frequency(sym_name, cache);
+ }
+ }
+ }
+ }
+ else {
+ /* Name not equal to symbol name, so we need to find the proper name */
+ auto *another_item = cache.get_item_by_name_mut(sym_name, false);
+ if (another_item != nullptr) {
+ another_item->inc_frequency(sym_name, cache);
+ }
+ }
+ }
+ else {
+ /* Symbol and sym name are the same */
+ g_atomic_int_inc(&st->hits);
+ }
+}
+
+auto cache_item::get_type_str() const -> const char *
+{
+ switch (type) {
+ case symcache_item_type::CONNFILTER:
+ return "connfilter";
+ case symcache_item_type::FILTER:
+ return "filter";
+ case symcache_item_type::IDEMPOTENT:
+ return "idempotent";
+ case symcache_item_type::PREFILTER:
+ return "prefilter";
+ case symcache_item_type::POSTFILTER:
+ return "postfilter";
+ case symcache_item_type::COMPOSITE:
+ return "composite";
+ case symcache_item_type::CLASSIFIER:
+ return "classifier";
+ case symcache_item_type::VIRTUAL:
+ return "virtual";
+ }
+
+ RSPAMD_UNREACHABLE;
+}
+
+auto cache_item::is_allowed(struct rspamd_task *task, bool exec_only) const -> bool
+{
+ const auto *what = "execution";
+
+ if (!exec_only) {
+ what = "symbol insertion";
+ }
+
+ /* Static checks */
+ if (!enabled ||
+ (RSPAMD_TASK_IS_EMPTY(task) && !(flags & SYMBOL_TYPE_EMPTY)) ||
+ (flags & SYMBOL_TYPE_MIME_ONLY && !RSPAMD_TASK_IS_MIME(task))) {
+
+ if (!enabled) {
+ msg_debug_cache_task("skipping %s of %s as it is permanently disabled",
+ what, symbol.c_str());
+
+ return false;
+ }
+ else {
+ /*
+ * If we check merely execution (not insertion), then we disallow
+ * mime symbols for non mime tasks and vice versa
+ */
+ if (exec_only) {
+ msg_debug_cache_task("skipping check of %s as it cannot be "
+ "executed for this task type",
+ symbol.c_str());
+
+ return FALSE;
+ }
+ }
+ }
+
+ /* Settings checks */
+ if (task->settings_elt != nullptr) {
+ if (forbidden_ids.check_id(task->settings_elt->id)) {
+ msg_debug_cache_task("deny %s of %s as it is forbidden for "
+ "settings id %ud",
+ what,
+ symbol.c_str(),
+ task->settings_elt->id);
+
+ return false;
+ }
+
+ if (!(flags & SYMBOL_TYPE_EXPLICIT_DISABLE)) {
+ if (!allowed_ids.check_id(task->settings_elt->id)) {
+
+ if (task->settings_elt->policy == RSPAMD_SETTINGS_POLICY_IMPLICIT_ALLOW) {
+ msg_debug_cache_task("allow execution of %s settings id %ud "
+ "allows implicit execution of the symbols;",
+ symbol.c_str(),
+ id);
+
+ return true;
+ }
+
+ if (exec_only) {
+ /*
+ * Special case if any of our virtual children are enabled
+ */
+ if (exec_only_ids.check_id(task->settings_elt->id)) {
+ return true;
+ }
+ }
+
+ msg_debug_cache_task("deny %s of %s as it is not listed "
+ "as allowed for settings id %ud",
+ what,
+ symbol.c_str(),
+ task->settings_elt->id);
+ return false;
+ }
+ }
+ else {
+ msg_debug_cache_task("allow %s of %s for "
+ "settings id %ud as it can be only disabled explicitly",
+ what,
+ symbol.c_str(),
+ task->settings_elt->id);
+ }
+ }
+ else if (flags & SYMBOL_TYPE_EXPLICIT_ENABLE) {
+ msg_debug_cache_task("deny %s of %s as it must be explicitly enabled",
+ what,
+ symbol.c_str());
+ return false;
+ }
+
+ /* Allow all symbols with no settings id */
+ return true;
+}
+
+auto cache_item::add_augmentation(const symcache &cache, std::string_view augmentation,
+ std::optional<std::string_view> value) -> bool
+{
+ auto log_tag = [&]() { return cache.log_tag(); };
+
+ if (augmentations.contains(augmentation)) {
+ msg_warn_cache("duplicate augmentation: %s", augmentation.data());
+
+ return false;
+ }
+
+ auto maybe_known = rspamd::find_map(known_augmentations, augmentation);
+
+ if (maybe_known.has_value()) {
+ auto &known_info = maybe_known.value().get();
+
+ if (known_info.implied_flags) {
+ if ((known_info.implied_flags & flags) == 0) {
+ msg_info_cache("added implied flags (%bd) for symbol %s as it has %s augmentation",
+ known_info.implied_flags, symbol.data(), augmentation.data());
+ flags |= known_info.implied_flags;
+ }
+ }
+
+ if (known_info.value_type == augmentation_value_type::NO_VALUE) {
+ if (value.has_value()) {
+ msg_err_cache("value specified for augmentation %s, that has no value",
+ augmentation.data());
+
+ return false;
+ }
+ return augmentations.try_emplace(augmentation, known_info.weight).second;
+ }
+ else {
+ if (!value.has_value()) {
+ msg_err_cache("value is not specified for augmentation %s, that requires explicit value",
+ augmentation.data());
+
+ return false;
+ }
+
+ if (known_info.value_type == augmentation_value_type::STRING_VALUE) {
+ return augmentations.try_emplace(augmentation, std::string{value.value()},
+ known_info.weight)
+ .second;
+ }
+ else if (known_info.value_type == augmentation_value_type::NUMBER_VALUE) {
+ /* I wish it was supported properly */
+ //auto conv_res = std::from_chars(value->data(), value->size(), num);
+ char numbuf[128], *endptr = nullptr;
+ rspamd_strlcpy(numbuf, value->data(), MIN(value->size(), sizeof(numbuf)));
+ auto num = g_ascii_strtod(numbuf, &endptr);
+
+ if (fabs(num) >= G_MAXFLOAT || std::isnan(num)) {
+ msg_err_cache("value for augmentation %s is not numeric: %*s",
+ augmentation.data(),
+ (int) value->size(), value->data());
+ return false;
+ }
+
+ return augmentations.try_emplace(augmentation, num,
+ known_info.weight)
+ .second;
+ }
+ }
+ }
+ else {
+ msg_debug_cache("added unknown augmentation %s for symbol %s",
+ "unknown", augmentation.data(), symbol.data());
+ return augmentations.try_emplace(augmentation, 0).second;
+ }
+
+ // Should not be reached
+ return false;
+}
+
+auto cache_item::get_augmentation_weight() const -> int
+{
+ return std::accumulate(std::begin(augmentations), std::end(augmentations),
+ 0, [](int acc, const auto &map_pair) {
+ return acc + map_pair.second.weight;
+ });
+}
+
+auto cache_item::get_numeric_augmentation(std::string_view name) const -> std::optional<double>
+{
+ const auto augmentation_value_maybe = rspamd::find_map(this->augmentations, name);
+
+ if (augmentation_value_maybe.has_value()) {
+ const auto &augmentation = augmentation_value_maybe.value().get();
+
+ if (std::holds_alternative<double>(augmentation.value)) {
+ return std::get<double>(augmentation.value);
+ }
+ }
+
+ return std::nullopt;
+}
+
+
+auto virtual_item::get_parent(const symcache &cache) const -> const cache_item *
+{
+ if (parent) {
+ return parent;
+ }
+
+ return cache.get_item_by_id(parent_id, false);
+}
+
+auto virtual_item::get_parent_mut(const symcache &cache) -> cache_item *
+{
+ if (parent) {
+ return parent;
+ }
+
+ return const_cast<cache_item *>(cache.get_item_by_id(parent_id, false));
+}
+
+auto virtual_item::resolve_parent(const symcache &cache) -> bool
+{
+ if (parent) {
+ return false;
+ }
+
+ auto item_ptr = cache.get_item_by_id(parent_id, true);
+
+ if (item_ptr) {
+ parent = const_cast<cache_item *>(item_ptr);
+
+ return true;
+ }
+
+ return false;
+}
+
+auto item_type_from_c(int type) -> tl::expected<std::pair<symcache_item_type, int>, std::string>
+{
+ constexpr const auto trivial_types = SYMBOL_TYPE_CONNFILTER | SYMBOL_TYPE_PREFILTER | SYMBOL_TYPE_POSTFILTER | SYMBOL_TYPE_IDEMPOTENT | SYMBOL_TYPE_COMPOSITE | SYMBOL_TYPE_CLASSIFIER | SYMBOL_TYPE_VIRTUAL;
+
+ constexpr auto all_but_one_ty = [&](int type, int exclude_bit) -> auto {
+ return (type & trivial_types) & (trivial_types & ~exclude_bit);
+ };
+
+ if (type & trivial_types) {
+ auto check_trivial = [&](auto flag,
+ symcache_item_type ty) -> tl::expected<std::pair<symcache_item_type, int>, std::string> {
+ if (all_but_one_ty(type, flag)) {
+ return tl::make_unexpected(fmt::format("invalid flags for a symbol: {}", (int) type));
+ }
+
+ return std::make_pair(ty, type & ~flag);
+ };
+ if (type & SYMBOL_TYPE_CONNFILTER) {
+ return check_trivial(SYMBOL_TYPE_CONNFILTER, symcache_item_type::CONNFILTER);
+ }
+ else if (type & SYMBOL_TYPE_PREFILTER) {
+ return check_trivial(SYMBOL_TYPE_PREFILTER, symcache_item_type::PREFILTER);
+ }
+ else if (type & SYMBOL_TYPE_POSTFILTER) {
+ return check_trivial(SYMBOL_TYPE_POSTFILTER, symcache_item_type::POSTFILTER);
+ }
+ else if (type & SYMBOL_TYPE_IDEMPOTENT) {
+ return check_trivial(SYMBOL_TYPE_IDEMPOTENT, symcache_item_type::IDEMPOTENT);
+ }
+ else if (type & SYMBOL_TYPE_COMPOSITE) {
+ return check_trivial(SYMBOL_TYPE_COMPOSITE, symcache_item_type::COMPOSITE);
+ }
+ else if (type & SYMBOL_TYPE_CLASSIFIER) {
+ return check_trivial(SYMBOL_TYPE_CLASSIFIER, symcache_item_type::CLASSIFIER);
+ }
+ else if (type & SYMBOL_TYPE_VIRTUAL) {
+ return check_trivial(SYMBOL_TYPE_VIRTUAL, symcache_item_type::VIRTUAL);
+ }
+
+ return tl::make_unexpected(fmt::format("internal error: impossible flags combination: {}", (int) type));
+ }
+
+ /* Maybe check other flags combination here? */
+ return std::make_pair(symcache_item_type::FILTER, type);
+}
+
+bool operator<(symcache_item_type lhs, symcache_item_type rhs)
+{
+ auto ret = false;
+ switch (lhs) {
+ case symcache_item_type::CONNFILTER:
+ break;
+ case symcache_item_type::PREFILTER:
+ if (rhs == symcache_item_type::CONNFILTER) {
+ ret = true;
+ }
+ break;
+ case symcache_item_type::FILTER:
+ if (rhs == symcache_item_type::CONNFILTER || rhs == symcache_item_type::PREFILTER) {
+ ret = true;
+ }
+ break;
+ case symcache_item_type::POSTFILTER:
+ if (rhs != symcache_item_type::IDEMPOTENT) {
+ ret = true;
+ }
+ break;
+ case symcache_item_type::IDEMPOTENT:
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+item_condition::~item_condition()
+{
+ if (cb != -1 && L != nullptr) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cb);
+ }
+}
+
+auto item_condition::check(std::string_view sym_name, struct rspamd_task *task) const -> bool
+{
+ if (cb != -1 && L != nullptr) {
+ auto ret = false;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ auto err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cb);
+ rspamd_lua_task_push(L, task);
+
+ if (lua_pcall(L, 1, 1, err_idx) != 0) {
+ msg_info_task("call to condition for %s failed: %s",
+ sym_name.data(), lua_tostring(L, -1));
+ }
+ else {
+ ret = lua_toboolean(L, -1);
+ }
+
+ lua_settop(L, err_idx - 1);
+
+ return ret;
+ }
+
+ return true;
+}
+
+}// namespace rspamd::symcache
diff --git a/src/libserver/symcache/symcache_item.hxx b/src/libserver/symcache/symcache_item.hxx
new file mode 100644
index 0000000..a60213a
--- /dev/null
+++ b/src/libserver/symcache/symcache_item.hxx
@@ -0,0 +1,561 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_SYMCACHE_ITEM_HXX
+#define RSPAMD_SYMCACHE_ITEM_HXX
+
+#pragma once
+
+#include <utility>
+#include <vector>
+#include <string>
+#include <string_view>
+#include <memory>
+#include <variant>
+#include <algorithm>
+#include <optional>
+
+#include "rspamd_symcache.h"
+#include "symcache_id_list.hxx"
+#include "contrib/expected/expected.hpp"
+#include "contrib/libev/ev.h"
+#include "symcache_runtime.hxx"
+#include "libutil/cxx/hash_util.hxx"
+
+namespace rspamd::symcache {
+
+class symcache;
+struct cache_item;
+using cache_item_ptr = std::shared_ptr<cache_item>;
+
+enum class symcache_item_type {
+ CONNFILTER, /* Executed on connection stage */
+ PREFILTER, /* Executed before all filters */
+ FILTER, /* Normal symbol with a callback */
+ POSTFILTER, /* Executed after all filters */
+ IDEMPOTENT, /* Executed after postfilters, cannot change results */
+ CLASSIFIER, /* A virtual classifier symbol */
+ COMPOSITE, /* A virtual composite symbol */
+ VIRTUAL, /* A virtual symbol... */
+};
+
+/*
+ * Compare item types: earlier stages symbols are > than later stages symbols
+ * Order for virtual stuff is not defined.
+ */
+bool operator<(symcache_item_type lhs, symcache_item_type rhs);
+
+constexpr static auto item_type_to_str(symcache_item_type t) -> const char *
+{
+ switch (t) {
+ case symcache_item_type::CONNFILTER:
+ return "connfilter";
+ case symcache_item_type::PREFILTER:
+ return "prefilter";
+ case symcache_item_type::FILTER:
+ return "filter";
+ case symcache_item_type::POSTFILTER:
+ return "postfilter";
+ case symcache_item_type::IDEMPOTENT:
+ return "idempotent";
+ case symcache_item_type::CLASSIFIER:
+ return "classifier";
+ case symcache_item_type::COMPOSITE:
+ return "composite";
+ case symcache_item_type::VIRTUAL:
+ return "virtual";
+ }
+}
+
+/**
+ * This is a public helper to convert a legacy C type to a more static type
+ * @param type input type as a C enum
+ * @return pair of type safe symcache_item_type + the remaining flags or an error
+ */
+auto item_type_from_c(int type) -> tl::expected<std::pair<symcache_item_type, int>, std::string>;
+
+struct item_condition {
+private:
+ lua_State *L = nullptr;
+ int cb = -1;
+
+public:
+ explicit item_condition(lua_State *L_, int cb_) noexcept
+ : L(L_), cb(cb_)
+ {
+ }
+ item_condition(item_condition &&other) noexcept
+ {
+ *this = std::move(other);
+ }
+ /* Make it move only */
+ item_condition(const item_condition &) = delete;
+ item_condition &operator=(item_condition &&other) noexcept
+ {
+ std::swap(other.L, L);
+ std::swap(other.cb, cb);
+ return *this;
+ }
+ ~item_condition();
+
+ auto check(std::string_view sym_name, struct rspamd_task *task) const -> bool;
+};
+
+class normal_item {
+private:
+ symbol_func_t func = nullptr;
+ void *user_data = nullptr;
+ std::vector<cache_item *> virtual_children;
+ std::vector<item_condition> conditions;
+
+public:
+ explicit normal_item(symbol_func_t _func, void *_user_data)
+ : func(_func), user_data(_user_data)
+ {
+ }
+
+ auto add_condition(lua_State *L, int cbref) -> void
+ {
+ conditions.emplace_back(L, cbref);
+ }
+
+ auto call(struct rspamd_task *task, struct rspamd_symcache_dynamic_item *item) const -> void
+ {
+ func(task, item, user_data);
+ }
+
+ auto check_conditions(std::string_view sym_name, struct rspamd_task *task) const -> bool
+ {
+ return std::all_of(std::begin(conditions), std::end(conditions),
+ [&](const auto &cond) { return cond.check(sym_name, task); });
+ }
+
+ auto get_cbdata() const -> auto
+ {
+ return user_data;
+ }
+
+ auto add_child(cache_item *ptr) -> void
+ {
+ virtual_children.push_back(ptr);
+ }
+
+ auto get_childen() const -> const std::vector<cache_item *> &
+ {
+ return virtual_children;
+ }
+};
+
+class virtual_item {
+private:
+ int parent_id = -1;
+ cache_item *parent = nullptr;
+
+public:
+ explicit virtual_item(int _parent_id)
+ : parent_id(_parent_id)
+ {
+ }
+
+ auto get_parent(const symcache &cache) const -> const cache_item *;
+ auto get_parent_mut(const symcache &cache) -> cache_item *;
+
+ auto resolve_parent(const symcache &cache) -> bool;
+};
+
+struct cache_dependency {
+ cache_item *item; /* Real dependency */
+ std::string sym; /* Symbolic dep name */
+ int id; /* Real from */
+ int vid; /* Virtual from */
+public:
+ /* Default piecewise constructor */
+ explicit cache_dependency(cache_item *_item, std::string _sym, int _id, int _vid)
+ : item(_item), sym(std::move(_sym)), id(_id), vid(_vid)
+ {
+ }
+};
+
+/*
+ * Used to store augmentation values
+ */
+struct item_augmentation {
+ std::variant<std::monostate, std::string, double> value;
+ int weight;
+
+ explicit item_augmentation(int weight)
+ : value(std::monostate{}), weight(weight)
+ {
+ }
+ explicit item_augmentation(std::string str_value, int weight)
+ : value(str_value), weight(weight)
+ {
+ }
+ explicit item_augmentation(double double_value, int weight)
+ : value(double_value), weight(weight)
+ {
+ }
+};
+
+struct cache_item : std::enable_shared_from_this<cache_item> {
+ /* The following fields will live in shared memory */
+ struct rspamd_symcache_item_stat *st = nullptr;
+ struct rspamd_counter_data *cd = nullptr;
+
+ /* Unique id - counter */
+ int id;
+ std::uint64_t last_count = 0;
+ std::string symbol;
+ symcache_item_type type;
+ int flags;
+
+ /* Condition of execution */
+ bool enabled = true;
+
+ /* Priority */
+ int priority = 0;
+ /* Topological order */
+ unsigned int order = 0;
+ int frequency_peaks = 0;
+
+ /* Specific data for virtual and callback symbols */
+ std::variant<normal_item, virtual_item> specific;
+
+ /* Settings ids */
+ id_list allowed_ids;
+ /* Allows execution but not symbols insertion */
+ id_list exec_only_ids;
+ id_list forbidden_ids;
+
+ /* Set of augmentations */
+ ankerl::unordered_dense::map<std::string, item_augmentation,
+ rspamd::smart_str_hash, rspamd::smart_str_equal>
+ augmentations;
+
+ /* Dependencies */
+ std::vector<cache_dependency> deps;
+ /* Reverse dependencies */
+ std::vector<cache_dependency> rdeps;
+
+public:
+ /**
+ * Create a normal item with a callback
+ * @param name
+ * @param priority
+ * @param func
+ * @param user_data
+ * @param type
+ * @param flags
+ * @return
+ */
+ template<typename T>
+ static auto create_with_function(rspamd_mempool_t *pool,
+ int id,
+ T &&name,
+ int priority,
+ symbol_func_t func,
+ void *user_data,
+ symcache_item_type type,
+ int flags) -> cache_item_ptr
+ {
+ return std::shared_ptr<cache_item>(new cache_item(pool,
+ id, std::forward<T>(name), priority,
+ func, user_data,
+ type, flags));
+ }
+
+ /**
+ * Create a virtual item
+ * @param name
+ * @param priority
+ * @param parent
+ * @param type
+ * @param flags
+ * @return
+ */
+ template<typename T>
+ static auto create_with_virtual(rspamd_mempool_t *pool,
+ int id,
+ T &&name,
+ int parent,
+ symcache_item_type type,
+ int flags) -> cache_item_ptr
+ {
+ return std::shared_ptr<cache_item>(new cache_item(pool, id, std::forward<T>(name),
+ parent, type, flags));
+ }
+
+ /**
+ * Share ownership on the item
+ * @return
+ */
+ auto getptr() -> cache_item_ptr
+ {
+ return shared_from_this();
+ }
+
+ /**
+ * Process and resolve dependencies for the item
+ * @param cache
+ */
+ auto process_deps(const symcache &cache) -> void;
+
+ auto is_virtual() const -> bool
+ {
+ return std::holds_alternative<virtual_item>(specific);
+ }
+
+ auto is_filter() const -> bool
+ {
+ return std::holds_alternative<normal_item>(specific) &&
+ (type == symcache_item_type::FILTER);
+ }
+
+ /**
+ * Returns true if a symbol should have some score defined
+ * @return
+ */
+ auto is_scoreable() const -> bool
+ {
+ return !(flags & SYMBOL_TYPE_CALLBACK) &&
+ ((type == symcache_item_type::FILTER) ||
+ is_virtual() ||
+ (type == symcache_item_type::COMPOSITE) ||
+ (type == symcache_item_type::CLASSIFIER));
+ }
+
+ auto is_ghost() const -> bool
+ {
+ return flags & SYMBOL_TYPE_GHOST;
+ }
+
+ auto get_parent(const symcache &cache) const -> const cache_item *;
+ auto get_parent_mut(const symcache &cache) -> cache_item *;
+
+ auto resolve_parent(const symcache &cache) -> bool;
+
+ auto get_type() const -> auto
+ {
+ return type;
+ }
+
+ auto get_type_str() const -> const char *;
+
+ auto get_name() const -> const std::string &
+ {
+ return symbol;
+ }
+
+ auto get_flags() const -> auto
+ {
+ return flags;
+ };
+
+ auto add_condition(lua_State *L, int cbref) -> bool
+ {
+ if (!is_virtual()) {
+ auto &normal = std::get<normal_item>(specific);
+ normal.add_condition(L, cbref);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ auto update_counters_check_peak(lua_State *L,
+ struct ev_loop *ev_loop,
+ double cur_time,
+ double last_resort) -> bool;
+
+ /**
+ * Increase frequency for a symbol
+ */
+ auto inc_frequency(const char *sym_name, symcache &cache) -> void;
+
+ /**
+ * Check if an item is allowed to be executed not checking item conditions
+ * @param task
+ * @param exec_only
+ * @return
+ */
+ auto is_allowed(struct rspamd_task *task, bool exec_only) const -> bool;
+
+ /**
+ * Returns callback data
+ * @return
+ */
+ auto get_cbdata() const -> void *
+ {
+ if (std::holds_alternative<normal_item>(specific)) {
+ const auto &filter_data = std::get<normal_item>(specific);
+
+ return filter_data.get_cbdata();
+ }
+
+ return nullptr;
+ }
+
+ /**
+ * Check all conditions for an item
+ * @param task
+ * @return
+ */
+ auto check_conditions(struct rspamd_task *task) const -> auto
+ {
+ if (std::holds_alternative<normal_item>(specific)) {
+ const auto &filter_data = std::get<normal_item>(specific);
+
+ return filter_data.check_conditions(symbol, task);
+ }
+
+ return false;
+ }
+
+ auto call(struct rspamd_task *task, cache_dynamic_item *dyn_item) const -> void
+ {
+ if (std::holds_alternative<normal_item>(specific)) {
+ const auto &filter_data = std::get<normal_item>(specific);
+
+ filter_data.call(task, (struct rspamd_symcache_dynamic_item *) dyn_item);
+ }
+ }
+
+ /**
+ * Add an augmentation to the item, returns `true` if augmentation is known and unique, false otherwise
+ * @param augmentation
+ * @return
+ */
+ auto add_augmentation(const symcache &cache, std::string_view augmentation,
+ std::optional<std::string_view> value) -> bool;
+
+ /**
+ * Return sum weight of all known augmentations
+ * @return
+ */
+ auto get_augmentation_weight() const -> int;
+
+ /**
+ * Returns numeric augmentation value
+ * @param name
+ * @return
+ */
+ auto get_numeric_augmentation(std::string_view name) const -> std::optional<double>;
+
+ /**
+ * Returns string augmentation value
+ * @param name
+ * @return
+ */
+ auto get_string_augmentation(std::string_view name) const -> std::optional<std::string_view>;
+
+ /**
+ * Add a virtual symbol as a child of some normal symbol
+ * @param ptr
+ */
+ auto add_child(cache_item *ptr) -> void
+ {
+ if (std::holds_alternative<normal_item>(specific)) {
+ auto &filter_data = std::get<normal_item>(specific);
+
+ filter_data.add_child(ptr);
+ }
+ else {
+ g_assert("add child is called for a virtual symbol!");
+ }
+ }
+
+ /**
+ * Returns virtual children for a normal item
+ * @param ptr
+ * @return
+ */
+ auto get_children() const -> const std::vector<cache_item *> *
+ {
+ if (std::holds_alternative<normal_item>(specific)) {
+ const auto &filter_data = std::get<normal_item>(specific);
+
+ return &filter_data.get_childen();
+ }
+
+ return nullptr;
+ }
+
+private:
+ /**
+ * Constructor for a normal symbols with callback
+ * @param name
+ * @param _priority
+ * @param func
+ * @param user_data
+ * @param _type
+ * @param _flags
+ */
+ cache_item(rspamd_mempool_t *pool,
+ int _id,
+ std::string &&name,
+ int _priority,
+ symbol_func_t func,
+ void *user_data,
+ symcache_item_type _type,
+ int _flags)
+ : id(_id),
+ symbol(std::move(name)),
+ type(_type),
+ flags(_flags),
+ priority(_priority),
+ specific(normal_item{func, user_data})
+ {
+ /* These structures are kept trivial, so they need to be explicitly reset */
+ forbidden_ids.reset();
+ allowed_ids.reset();
+ exec_only_ids.reset();
+ st = rspamd_mempool_alloc0_shared_type(pool, std::remove_pointer_t<decltype(st)>);
+ cd = rspamd_mempool_alloc0_shared_type(pool, std::remove_pointer_t<decltype(cd)>);
+ }
+
+ /**
+ * Constructor for a virtual symbol
+ * @param name
+ * @param _priority
+ * @param parent
+ * @param _type
+ * @param _flags
+ */
+ cache_item(rspamd_mempool_t *pool,
+ int _id,
+ std::string &&name,
+ int parent,
+ symcache_item_type _type,
+ int _flags)
+ : id(_id),
+ symbol(std::move(name)),
+ type(_type),
+ flags(_flags),
+ specific(virtual_item{parent})
+ {
+ /* These structures are kept trivial, so they need to be explicitly reset */
+ forbidden_ids.reset();
+ allowed_ids.reset();
+ exec_only_ids.reset();
+ st = rspamd_mempool_alloc0_shared_type(pool, std::remove_pointer_t<decltype(st)>);
+ cd = rspamd_mempool_alloc0_shared_type(pool, std::remove_pointer_t<decltype(cd)>);
+ }
+};
+
+}// namespace rspamd::symcache
+
+#endif//RSPAMD_SYMCACHE_ITEM_HXX
diff --git a/src/libserver/symcache/symcache_periodic.hxx b/src/libserver/symcache/symcache_periodic.hxx
new file mode 100644
index 0000000..535956b
--- /dev/null
+++ b/src/libserver/symcache/symcache_periodic.hxx
@@ -0,0 +1,89 @@
+/*-
+ * Copyright 2022 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#ifndef RSPAMD_SYMCACHE_PERIODIC_HXX
+#define RSPAMD_SYMCACHE_PERIODIC_HXX
+
+#pragma once
+
+#include "config.h"
+#include "contrib/libev/ev.h"
+#include "symcache_internal.hxx"
+#include "worker_util.h"
+
+namespace rspamd::symcache {
+struct cache_refresh_cbdata {
+private:
+ symcache *cache;
+ struct ev_loop *event_loop;
+ struct rspamd_worker *w;
+ double reload_time;
+ double last_resort;
+ ev_timer resort_ev;
+
+public:
+ explicit cache_refresh_cbdata(symcache *_cache,
+ struct ev_loop *_ev_base,
+ struct rspamd_worker *_w)
+ : cache(_cache), event_loop(_ev_base), w(_w)
+ {
+ auto log_tag = [&]() { return cache->log_tag(); };
+ last_resort = rspamd_get_ticks(TRUE);
+ reload_time = cache->get_reload_time();
+ auto tm = rspamd_time_jitter(reload_time, 0);
+ msg_debug_cache("next reload in %.2f seconds", tm);
+ ev_timer_init(&resort_ev, cache_refresh_cbdata::resort_cb,
+ tm, tm);
+ resort_ev.data = (void *) this;
+ ev_timer_start(event_loop, &resort_ev);
+ rspamd_mempool_add_destructor(cache->get_pool(),
+ cache_refresh_cbdata::refresh_dtor, (void *) this);
+ }
+
+ static void refresh_dtor(void *d)
+ {
+ auto *cbdata = (struct cache_refresh_cbdata *) d;
+ delete cbdata;
+ }
+
+ static void resort_cb(EV_P_ ev_timer *w, int _revents)
+ {
+ auto *cbdata = (struct cache_refresh_cbdata *) w->data;
+
+ auto log_tag = [&]() { return cbdata->cache->log_tag(); };
+
+ if (rspamd_worker_is_primary_controller(cbdata->w)) {
+ /* Plan new event */
+ auto tm = rspamd_time_jitter(cbdata->reload_time, 0);
+ msg_debug_cache("resort symbols cache, next reload in %.2f seconds", tm);
+ cbdata->resort_ev.repeat = tm;
+ ev_timer_again(EV_A_ w);
+ auto cur_time = rspamd_get_ticks(FALSE);
+ cbdata->cache->periodic_resort(cbdata->event_loop, cur_time, cbdata->last_resort);
+ cbdata->last_resort = cur_time;
+ }
+ }
+
+private:
+ ~cache_refresh_cbdata()
+ {
+ ev_timer_stop(event_loop, &resort_ev);
+ }
+};
+}// namespace rspamd::symcache
+
+#endif//RSPAMD_SYMCACHE_PERIODIC_HXX
diff --git a/src/libserver/symcache/symcache_runtime.cxx b/src/libserver/symcache/symcache_runtime.cxx
new file mode 100644
index 0000000..d9622d8
--- /dev/null
+++ b/src/libserver/symcache/symcache_runtime.cxx
@@ -0,0 +1,823 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "symcache_internal.hxx"
+#include "symcache_item.hxx"
+#include "symcache_runtime.hxx"
+#include "libutil/cxx/util.hxx"
+#include "libserver/task.h"
+#include "libmime/scan_result.h"
+#include "utlist.h"
+#include "libserver/worker_util.h"
+#include <limits>
+#include <cmath>
+
+namespace rspamd::symcache {
+
+/* At least once per minute */
+constexpr static const auto PROFILE_MAX_TIME = 60.0;
+/* For messages larger than 2Mb enable profiling */
+constexpr static const auto PROFILE_MESSAGE_SIZE_THRESHOLD = 1024ul * 1024 * 2;
+/* Enable profile at least once per this amount of messages processed */
+constexpr static const auto PROFILE_PROBABILITY = 0.01;
+
+auto symcache_runtime::create(struct rspamd_task *task, symcache &cache) -> symcache_runtime *
+{
+ cache.maybe_resort();
+
+ auto &&cur_order = cache.get_cache_order();
+ auto *checkpoint = (symcache_runtime *) rspamd_mempool_alloc0(task->task_pool,
+ sizeof(symcache_runtime) +
+ sizeof(struct cache_dynamic_item) * cur_order->size());
+
+ checkpoint->order = cache.get_cache_order();
+
+ /* Calculate profile probability */
+ ev_now_update_if_cheap(task->event_loop);
+ ev_tstamp now = ev_now(task->event_loop);
+ checkpoint->profile_start = now;
+ checkpoint->lim = rspamd_task_get_required_score(task, task->result);
+
+ if ((cache.get_last_profile() == 0.0 || now > cache.get_last_profile() + PROFILE_MAX_TIME) ||
+ (task->msg.len >= PROFILE_MESSAGE_SIZE_THRESHOLD) ||
+ (rspamd_random_double_fast() >= (1 - PROFILE_PROBABILITY))) {
+ msg_debug_cache_task("enable profiling of symbols for task");
+ checkpoint->profile = true;
+ cache.set_last_profile(now);
+ }
+
+ task->symcache_runtime = (void *) checkpoint;
+
+ return checkpoint;
+}
+
+auto symcache_runtime::process_settings(struct rspamd_task *task, const symcache &cache) -> bool
+{
+ if (!task->settings) {
+ msg_err_task("`process_settings` is called with no settings");
+ return false;
+ }
+
+ const auto *wl = ucl_object_lookup(task->settings, "whitelist");
+
+ if (wl != nullptr) {
+ msg_info_task("task is whitelisted");
+ task->flags |= RSPAMD_TASK_FLAG_SKIP;
+ return true;
+ }
+
+ auto already_disabled = false;
+
+ auto process_group = [&](const ucl_object_t *gr_obj, auto functor) -> void {
+ ucl_object_iter_t it = nullptr;
+ const ucl_object_t *cur;
+
+ if (gr_obj) {
+ while ((cur = ucl_iterate_object(gr_obj, &it, true)) != nullptr) {
+ if (ucl_object_type(cur) == UCL_STRING) {
+ auto *gr = (struct rspamd_symbols_group *)
+ g_hash_table_lookup(task->cfg->groups,
+ ucl_object_tostring(cur));
+
+ if (gr) {
+ GHashTableIter gr_it;
+ void *k, *v;
+ g_hash_table_iter_init(&gr_it, gr->symbols);
+
+ while (g_hash_table_iter_next(&gr_it, &k, &v)) {
+ functor((const char *) k);
+ }
+ }
+ }
+ }
+ }
+ };
+
+ ucl_object_iter_t it = nullptr;
+ const ucl_object_t *cur;
+
+ const auto *enabled = ucl_object_lookup(task->settings, "symbols_enabled");
+
+ if (enabled) {
+ msg_debug_cache_task("disable all symbols as `symbols_enabled` is found");
+ /* Disable all symbols but selected */
+ disable_all_symbols(SYMBOL_TYPE_EXPLICIT_DISABLE);
+ already_disabled = true;
+ it = nullptr;
+
+ while ((cur = ucl_iterate_object(enabled, &it, true)) != nullptr) {
+ enable_symbol(task, cache, ucl_object_tostring(cur));
+ }
+ }
+
+ /* Enable groups of symbols */
+ enabled = ucl_object_lookup(task->settings, "groups_enabled");
+ if (enabled && !already_disabled) {
+ disable_all_symbols(SYMBOL_TYPE_EXPLICIT_DISABLE);
+ }
+ process_group(enabled, [&](const char *sym) {
+ enable_symbol(task, cache, sym);
+ });
+
+ const auto *disabled = ucl_object_lookup(task->settings, "symbols_disabled");
+
+ if (disabled) {
+ it = nullptr;
+
+ while ((cur = ucl_iterate_object(disabled, &it, true)) != nullptr) {
+ disable_symbol(task, cache, ucl_object_tostring(cur));
+ }
+ }
+
+ /* Disable groups of symbols */
+ disabled = ucl_object_lookup(task->settings, "groups_disabled");
+ process_group(disabled, [&](const char *sym) {
+ disable_symbol(task, cache, sym);
+ });
+
+ /* Update required limit */
+ lim = rspamd_task_get_required_score(task, task->result);
+
+ return false;
+}
+
+auto symcache_runtime::disable_all_symbols(int skip_mask) -> void
+{
+ for (auto [i, item]: rspamd::enumerate(order->d)) {
+ auto *dyn_item = &dynamic_items[i];
+
+ if (!(item->get_flags() & skip_mask)) {
+ dyn_item->finished = true;
+ dyn_item->started = true;
+ }
+ }
+}
+
+auto symcache_runtime::disable_symbol(struct rspamd_task *task, const symcache &cache, std::string_view name) -> bool
+{
+ const auto *item = cache.get_item_by_name(name, true);
+
+ if (item != nullptr) {
+
+ auto *dyn_item = get_dynamic_item(item->id);
+
+ if (dyn_item) {
+ dyn_item->finished = true;
+ dyn_item->started = true;
+ msg_debug_cache_task("disable execution of %s", name.data());
+
+ return true;
+ }
+ else {
+ msg_debug_cache_task("cannot disable %s: id not found %d", name.data(), item->id);
+ }
+ }
+ else {
+ msg_debug_cache_task("cannot disable %s: symbol not found", name.data());
+ }
+
+ return false;
+}
+
+auto symcache_runtime::enable_symbol(struct rspamd_task *task, const symcache &cache, std::string_view name) -> bool
+{
+ const auto *item = cache.get_item_by_name(name, true);
+
+ if (item != nullptr) {
+
+ auto *dyn_item = get_dynamic_item(item->id);
+
+ if (dyn_item) {
+ dyn_item->finished = false;
+ dyn_item->started = false;
+ msg_debug_cache_task("enable execution of %s", name.data());
+
+ return true;
+ }
+ else {
+ msg_debug_cache_task("cannot enable %s: id not found %d", name.data(), item->id);
+ }
+ }
+ else {
+ msg_debug_cache_task("cannot enable %s: symbol not found", name.data());
+ }
+
+ return false;
+}
+
+auto symcache_runtime::is_symbol_checked(const symcache &cache, std::string_view name) -> bool
+{
+ const auto *item = cache.get_item_by_name(name, true);
+
+ if (item != nullptr) {
+
+ auto *dyn_item = get_dynamic_item(item->id);
+
+ if (dyn_item) {
+ return dyn_item->started;
+ }
+ }
+
+ return false;
+}
+
+auto symcache_runtime::is_symbol_enabled(struct rspamd_task *task, const symcache &cache, std::string_view name) -> bool
+{
+
+ const auto *item = cache.get_item_by_name(name, true);
+ if (item) {
+
+ if (!item->is_allowed(task, true)) {
+ return false;
+ }
+ else {
+ auto *dyn_item = get_dynamic_item(item->id);
+
+ if (dyn_item) {
+ if (dyn_item->started) {
+ /* Already started */
+ return false;
+ }
+
+ if (!item->is_virtual()) {
+ return std::get<normal_item>(item->specific).check_conditions(item->symbol, task);
+ }
+ }
+ else {
+ /* Unknown item */
+ msg_debug_cache_task("cannot enable %s: symbol not found", name.data());
+ }
+ }
+ }
+
+ return true;
+}
+
+auto symcache_runtime::get_dynamic_item(int id) const -> cache_dynamic_item *
+{
+
+ /* Not found in the cache, do a hash lookup */
+ auto our_id_maybe = rspamd::find_map(order->by_cache_id, id);
+
+ if (our_id_maybe) {
+ return &dynamic_items[our_id_maybe.value()];
+ }
+
+ return nullptr;
+}
+
+auto symcache_runtime::process_symbols(struct rspamd_task *task, symcache &cache, unsigned int stage) -> bool
+{
+ msg_debug_cache_task("symbols processing stage at pass: %d", stage);
+
+ if (RSPAMD_TASK_IS_SKIPPED(task)) {
+ return true;
+ }
+
+ switch (stage) {
+ case RSPAMD_TASK_STAGE_CONNFILTERS:
+ case RSPAMD_TASK_STAGE_PRE_FILTERS:
+ case RSPAMD_TASK_STAGE_POST_FILTERS:
+ case RSPAMD_TASK_STAGE_IDEMPOTENT:
+ return process_pre_postfilters(task, cache,
+ rspamd_session_events_pending(task->s), stage);
+ break;
+
+ case RSPAMD_TASK_STAGE_FILTERS:
+ return process_filters(task, cache, rspamd_session_events_pending(task->s));
+ break;
+
+ default:
+ g_assert_not_reached();
+ }
+}
+
+auto symcache_runtime::process_pre_postfilters(struct rspamd_task *task,
+ symcache &cache,
+ int start_events,
+ unsigned int stage) -> bool
+{
+ auto saved_priority = std::numeric_limits<int>::min();
+ auto all_done = true;
+ auto log_func = RSPAMD_LOG_FUNC;
+ auto compare_functor = +[](int a, int b) { return a < b; };
+
+ auto proc_func = [&](cache_item *item) {
+ /*
+ * We can safely ignore all pre/postfilters except idempotent ones and
+ * those that are marked as ignore passthrough result
+ */
+ if (stage != RSPAMD_TASK_STAGE_IDEMPOTENT &&
+ !(item->flags & SYMBOL_TYPE_IGNORE_PASSTHROUGH)) {
+ if (check_metric_limit(task)) {
+ msg_debug_cache_task_lambda("task has already the result being set, ignore further checks");
+
+ return true;
+ }
+ }
+
+ auto dyn_item = get_dynamic_item(item->id);
+
+ if (!dyn_item->started && !dyn_item->finished) {
+ if (has_slow) {
+ /* Delay */
+ has_slow = false;
+
+ return false;
+ }
+
+ if (saved_priority == std::numeric_limits<int>::min()) {
+ saved_priority = item->priority;
+ }
+ else {
+ if (compare_functor(item->priority, saved_priority) &&
+ rspamd_session_events_pending(task->s) > start_events) {
+ /*
+ * Delay further checks as we have higher
+ * priority filters to be processed
+ */
+ return false;
+ }
+ }
+
+ return process_symbol(task, cache, item, dyn_item);
+ }
+
+ /* Continue processing */
+ return true;
+ };
+
+ switch (stage) {
+ case RSPAMD_TASK_STAGE_CONNFILTERS:
+ all_done = cache.connfilters_foreach(proc_func);
+ break;
+ case RSPAMD_TASK_STAGE_PRE_FILTERS:
+ all_done = cache.prefilters_foreach(proc_func);
+ break;
+ case RSPAMD_TASK_STAGE_POST_FILTERS:
+ compare_functor = +[](int a, int b) { return a > b; };
+ all_done = cache.postfilters_foreach(proc_func);
+ break;
+ case RSPAMD_TASK_STAGE_IDEMPOTENT:
+ compare_functor = +[](int a, int b) { return a > b; };
+ all_done = cache.idempotent_foreach(proc_func);
+ break;
+ default:
+ g_error("invalid invocation");
+ break;
+ }
+
+ return all_done;
+}
+
+auto symcache_runtime::process_filters(struct rspamd_task *task, symcache &cache, int start_events) -> bool
+{
+ auto all_done = true;
+ auto log_func = RSPAMD_LOG_FUNC;
+ auto has_passtrough = false;
+
+ for (const auto [idx, item]: rspamd::enumerate(order->d)) {
+ /* Exclude all non filters */
+ if (item->type != symcache_item_type::FILTER) {
+ /*
+ * We use breaking the loop as we append non-filters to the end of the list
+ * so, it is safe to stop processing immediately
+ */
+ break;
+ }
+
+ if (!(item->flags & (SYMBOL_TYPE_FINE | SYMBOL_TYPE_IGNORE_PASSTHROUGH))) {
+ if (has_passtrough || check_metric_limit(task)) {
+ msg_debug_cache_task_lambda("task has already the result being set, ignore further checks");
+ has_passtrough = true;
+ /* Skip this item */
+ continue;
+ }
+ }
+
+ auto dyn_item = &dynamic_items[idx];
+
+ if (!dyn_item->started) {
+ all_done = false;
+
+ if (!check_item_deps(task, cache, item.get(),
+ dyn_item, false)) {
+ msg_debug_cache_task("blocked execution of %d(%s) unless deps are "
+ "resolved",
+ item->id, item->symbol.c_str());
+
+ continue;
+ }
+
+ process_symbol(task, cache, item.get(), dyn_item);
+
+ if (has_slow) {
+ /* Delay */
+ has_slow = false;
+
+ return false;
+ }
+ }
+ }
+
+ return all_done;
+}
+
+auto symcache_runtime::process_symbol(struct rspamd_task *task, symcache &cache, cache_item *item,
+ cache_dynamic_item *dyn_item) -> bool
+{
+ if (item->type == symcache_item_type::CLASSIFIER || item->type == symcache_item_type::COMPOSITE) {
+ /* Classifiers are special :( */
+ return true;
+ }
+
+ if (rspamd_session_blocked(task->s)) {
+ /*
+ * We cannot add new events as session is either destroyed or
+ * being cleaned up.
+ */
+ return true;
+ }
+
+ g_assert(!item->is_virtual());
+ if (dyn_item->started) {
+ /*
+ * This can actually happen when deps span over different layers
+ */
+ return dyn_item->finished;
+ }
+
+ /* Check has been started */
+ dyn_item->started = true;
+ auto check = true;
+
+ if (!item->is_allowed(task, true) || !item->check_conditions(task)) {
+ check = false;
+ }
+
+ if (check) {
+ msg_debug_cache_task("execute %s, %d; symbol type = %s", item->symbol.data(),
+ item->id, item_type_to_str(item->type));
+
+ if (profile) {
+ ev_now_update_if_cheap(task->event_loop);
+ dyn_item->start_msec = (ev_now(task->event_loop) -
+ profile_start) *
+ 1e3;
+ }
+ dyn_item->async_events = 0;
+ cur_item = dyn_item;
+ items_inflight++;
+ /* Callback now must finalize itself */
+ item->call(task, dyn_item);
+ cur_item = nullptr;
+
+ if (items_inflight == 0) {
+ return true;
+ }
+
+ if (dyn_item->async_events == 0 && !dyn_item->finished) {
+ msg_err_cache_task("critical error: item %s has no async events pending, "
+ "but it is not finalised",
+ item->symbol.data());
+ g_assert_not_reached();
+ }
+
+ return false;
+ }
+ else {
+ dyn_item->finished = true;
+ }
+
+ return true;
+}
+
+auto symcache_runtime::check_metric_limit(struct rspamd_task *task) -> bool
+{
+ if (task->flags & RSPAMD_TASK_FLAG_PASS_ALL) {
+ return false;
+ }
+
+ /* Check score limit */
+ if (!std::isnan(lim)) {
+ if (task->result->score > lim) {
+ return true;
+ }
+ }
+
+ if (task->result->passthrough_result != nullptr) {
+ /* We also need to check passthrough results */
+ auto *pr = task->result->passthrough_result;
+ DL_FOREACH(task->result->passthrough_result, pr)
+ {
+ struct rspamd_action_config *act_config =
+ rspamd_find_action_config_for_action(task->result, pr->action);
+
+ /* Skip least results */
+ if (pr->flags & RSPAMD_PASSTHROUGH_LEAST) {
+ continue;
+ }
+
+ /* Skip disabled actions */
+ if (act_config && (act_config->flags & RSPAMD_ACTION_RESULT_DISABLED)) {
+ continue;
+ }
+
+ /* Immediately stop on non least passthrough action */
+ return true;
+ }
+ }
+
+ return false;
+}
+
+auto symcache_runtime::check_item_deps(struct rspamd_task *task, symcache &cache, cache_item *item,
+ cache_dynamic_item *dyn_item, bool check_only) -> bool
+{
+ constexpr const auto max_recursion = 20;
+ auto log_func = RSPAMD_LOG_FUNC;
+
+ auto inner_functor = [&](int recursion, cache_item *item, cache_dynamic_item *dyn_item, auto rec_functor) -> bool {
+ if (recursion > max_recursion) {
+ msg_err_task_lambda("cyclic dependencies: maximum check level %ud exceed when "
+ "checking dependencies for %s",
+ max_recursion, item->symbol.c_str());
+
+ return true;
+ }
+
+ auto ret = true;
+
+ for (const auto &dep: item->deps) {
+ if (!dep.item) {
+ /* Assume invalid deps as done */
+ msg_debug_cache_task_lambda("symbol %d(%s) has invalid dependencies on %d(%s)",
+ item->id, item->symbol.c_str(), dep.id, dep.sym.c_str());
+ continue;
+ }
+
+ auto *dep_dyn_item = get_dynamic_item(dep.item->id);
+
+ if (!dep_dyn_item->finished) {
+ if (!dep_dyn_item->started) {
+ /* Not started */
+ if (!check_only) {
+ if (!rec_functor(recursion + 1,
+ dep.item,
+ dep_dyn_item,
+ rec_functor)) {
+
+ ret = false;
+ msg_debug_cache_task_lambda("delayed dependency %d(%s) for "
+ "symbol %d(%s)",
+ dep.id, dep.sym.c_str(), item->id, item->symbol.c_str());
+ }
+ else if (!process_symbol(task, cache, dep.item, dep_dyn_item)) {
+ /* Now started, but has events pending */
+ ret = false;
+ msg_debug_cache_task_lambda("started check of %d(%s) symbol "
+ "as dep for "
+ "%d(%s)",
+ dep.id, dep.sym.c_str(), item->id, item->symbol.c_str());
+ }
+ else {
+ msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) is "
+ "already processed",
+ dep.id, dep.sym.c_str(), item->id, item->symbol.c_str());
+ }
+ }
+ else {
+ msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) "
+ "cannot be started now",
+ dep.id, dep.sym.c_str(), item->id, item->symbol.c_str());
+ ret = false;
+ }
+ }
+ else {
+ /* Started but not finished */
+ msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) is "
+ "still executing",
+ dep.id, dep.sym.c_str(), item->id, item->symbol.c_str());
+ ret = false;
+ }
+ }
+ else {
+ msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) is already "
+ "checked",
+ dep.id, dep.sym.c_str(), item->id, item->symbol.c_str());
+ }
+ }
+
+ return ret;
+ };
+
+ return inner_functor(0, item, dyn_item, inner_functor);
+}
+
+
+struct rspamd_symcache_delayed_cbdata {
+ cache_item *item;
+ struct rspamd_task *task;
+ symcache_runtime *runtime;
+ struct rspamd_async_event *event;
+ struct ev_timer tm;
+};
+
+static void
+rspamd_symcache_delayed_item_fin(gpointer ud)
+{
+ auto *cbd = (struct rspamd_symcache_delayed_cbdata *) ud;
+
+ cbd->event = nullptr;
+ cbd->runtime->unset_slow();
+ ev_timer_stop(cbd->task->event_loop, &cbd->tm);
+}
+
+static void
+rspamd_symcache_delayed_item_cb(EV_P_ ev_timer *w, int what)
+{
+ auto *cbd = (struct rspamd_symcache_delayed_cbdata *) w->data;
+
+ if (cbd->event) {
+ cbd->event = nullptr;
+
+ /* Timer will be stopped here */
+ rspamd_session_remove_event(cbd->task->s,
+ rspamd_symcache_delayed_item_fin, cbd);
+
+ cbd->runtime->process_item_rdeps(cbd->task, cbd->item);
+ }
+}
+
+static void
+rspamd_delayed_timer_dtor(gpointer d)
+{
+ auto *cbd = (struct rspamd_symcache_delayed_cbdata *) d;
+
+ if (cbd->event) {
+ /* Event has not been executed, this will also stop a timer */
+ rspamd_session_remove_event(cbd->task->s,
+ rspamd_symcache_delayed_item_fin, cbd);
+ cbd->event = nullptr;
+ }
+}
+
+auto symcache_runtime::finalize_item(struct rspamd_task *task, cache_dynamic_item *dyn_item) -> void
+{
+ /* Limit to consider a rule as slow (in milliseconds) */
+ constexpr const gdouble slow_diff_limit = 300;
+ auto *item = get_item_by_dynamic_item(dyn_item);
+ /* Sanity checks */
+ g_assert(items_inflight > 0);
+ g_assert(item != nullptr);
+
+ if (dyn_item->async_events > 0) {
+ /*
+ * XXX: Race condition
+ *
+ * It is possible that some async event is still in flight, but we
+ * already know its result, however, it is the responsibility of that
+ * event to decrease async events count and call this function
+ * one more time
+ */
+ msg_debug_cache_task("postpone finalisation of %s(%d) as there are %d "
+ "async events pending",
+ item->symbol.c_str(), item->id, dyn_item->async_events);
+
+ return;
+ }
+
+ msg_debug_cache_task("process finalize for item %s(%d)", item->symbol.c_str(), item->id);
+ dyn_item->finished = true;
+ items_inflight--;
+ cur_item = nullptr;
+
+ auto enable_slow_timer = [&]() -> bool {
+ auto *cbd = rspamd_mempool_alloc0_type(task->task_pool, rspamd_symcache_delayed_cbdata);
+ /* Add timer to allow something else to be executed */
+ ev_timer *tm = &cbd->tm;
+
+ cbd->event = rspamd_session_add_event(task->s,
+ rspamd_symcache_delayed_item_fin, cbd,
+ "symcache");
+ cbd->runtime = this;
+
+ /*
+ * If no event could be added, then we are already in the destruction
+ * phase. So the main issue is to deal with has slow here
+ */
+ if (cbd->event) {
+ ev_timer_init(tm, rspamd_symcache_delayed_item_cb, 0.1, 0.0);
+ ev_set_priority(tm, EV_MINPRI);
+ rspamd_mempool_add_destructor(task->task_pool,
+ rspamd_delayed_timer_dtor, cbd);
+
+ cbd->task = task;
+ cbd->item = item;
+ tm->data = cbd;
+ ev_timer_start(task->event_loop, tm);
+ }
+ else {
+ /* Just reset as no timer is added */
+ has_slow = FALSE;
+ return false;
+ }
+
+ return true;
+ };
+
+ if (profile) {
+ ev_now_update_if_cheap(task->event_loop);
+ auto diff = ((ev_now(task->event_loop) - profile_start) * 1e3 -
+ dyn_item->start_msec);
+
+ if (diff > slow_diff_limit) {
+
+ if (!has_slow) {
+ has_slow = true;
+
+ msg_info_task("slow rule: %s(%d): %.2f ms; enable slow timer delay",
+ item->symbol.c_str(), item->id,
+ diff);
+
+ if (enable_slow_timer()) {
+ /* Allow network execution */
+ return;
+ }
+ }
+ else {
+ msg_info_task("slow rule: %s(%d): %.2f ms",
+ item->symbol.c_str(), item->id,
+ diff);
+ }
+ }
+
+ if (G_UNLIKELY(RSPAMD_TASK_IS_PROFILING(task))) {
+ rspamd_task_profile_set(task, item->symbol.c_str(), diff);
+ }
+
+ if (rspamd_worker_is_scanner(task->worker)) {
+ rspamd_set_counter(item->cd, diff);
+ }
+ }
+
+ process_item_rdeps(task, item);
+}
+
+auto symcache_runtime::process_item_rdeps(struct rspamd_task *task, cache_item *item) -> void
+{
+ auto *cache_ptr = reinterpret_cast<symcache *>(task->cfg->cache);
+
+ // Avoid race condition with the runtime destruction and the delay timer
+ if (!order) {
+ return;
+ }
+
+ for (const auto &rdep: item->rdeps) {
+ if (rdep.item) {
+ auto *dyn_item = get_dynamic_item(rdep.item->id);
+ if (!dyn_item->started) {
+ msg_debug_cache_task("check item %d(%s) rdep of %s ",
+ rdep.item->id, rdep.item->symbol.c_str(), item->symbol.c_str());
+
+ if (!check_item_deps(task, *cache_ptr, rdep.item, dyn_item, false)) {
+ msg_debug_cache_task("blocked execution of %d(%s) rdep of %s "
+ "unless deps are resolved",
+ rdep.item->id, rdep.item->symbol.c_str(), item->symbol.c_str());
+ }
+ else {
+ process_symbol(task, *cache_ptr, rdep.item,
+ dyn_item);
+ }
+ }
+ }
+ }
+}
+
+auto symcache_runtime::get_item_by_dynamic_item(cache_dynamic_item *dyn_item) const -> cache_item *
+{
+ auto idx = dyn_item - dynamic_items;
+
+ if (idx >= 0 && idx < order->size()) {
+ return order->d[idx].get();
+ }
+
+ msg_err("internal error: invalid index to get: %d", (int) idx);
+
+ return nullptr;
+}
+
+}// namespace rspamd::symcache
diff --git a/src/libserver/symcache/symcache_runtime.hxx b/src/libserver/symcache/symcache_runtime.hxx
new file mode 100644
index 0000000..aa8f66c
--- /dev/null
+++ b/src/libserver/symcache/symcache_runtime.hxx
@@ -0,0 +1,209 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * Symcache runtime is produced for each task and it consists of symbols
+ * being executed, being dynamically disabled/enabled and it also captures
+ * the current order of the symbols (produced by resort periodic)
+ */
+
+#ifndef RSPAMD_SYMCACHE_RUNTIME_HXX
+#define RSPAMD_SYMCACHE_RUNTIME_HXX
+#pragma once
+
+#include "symcache_internal.hxx"
+
+struct rspamd_scan_result;
+
+namespace rspamd::symcache {
+/**
+ * These items are saved within task structure and are used to track
+ * symbols execution.
+ * Each symcache item occupies a single dynamic item, that currently has 8 bytes
+ * length
+ */
+struct cache_dynamic_item {
+ std::uint16_t start_msec; /* Relative to task time */
+ bool started;
+ bool finished;
+ std::uint32_t async_events;
+};
+
+static_assert(sizeof(cache_dynamic_item) == sizeof(std::uint64_t));
+static_assert(std::is_trivial_v<cache_dynamic_item>);
+
+class symcache_runtime {
+ unsigned items_inflight;
+ bool profile;
+ bool has_slow;
+
+ double profile_start;
+ double lim;
+
+ struct cache_dynamic_item *cur_item;
+ order_generation_ptr order;
+ /* Dynamically expanded as needed */
+ mutable struct cache_dynamic_item dynamic_items[];
+ /* We allocate this structure merely in memory pool, so destructor is absent */
+ ~symcache_runtime() = delete;
+
+ auto process_symbol(struct rspamd_task *task, symcache &cache, cache_item *item,
+ cache_dynamic_item *dyn_item) -> bool;
+ /* Specific stages of the processing */
+ auto process_pre_postfilters(struct rspamd_task *task, symcache &cache, int start_events, unsigned int stage) -> bool;
+ auto process_filters(struct rspamd_task *task, symcache &cache, int start_events) -> bool;
+ auto check_metric_limit(struct rspamd_task *task) -> bool;
+ auto check_item_deps(struct rspamd_task *task, symcache &cache, cache_item *item,
+ cache_dynamic_item *dyn_item, bool check_only) -> bool;
+
+public:
+ /* Dropper for a shared ownership */
+ auto savepoint_dtor() -> void
+ {
+
+ /* Drop shared ownership */
+ order.reset();
+ }
+ /**
+ * Creates a cache runtime using task mempool
+ * @param task
+ * @param cache
+ * @return
+ */
+ static auto create(struct rspamd_task *task, symcache &cache) -> symcache_runtime *;
+ /**
+ * Process task settings
+ * @param task
+ * @return
+ */
+ auto process_settings(struct rspamd_task *task, const symcache &cache) -> bool;
+
+ /**
+ * Disable all symbols but not touching ones that are in the specific mask
+ * @param skip_mask
+ */
+ auto disable_all_symbols(int skip_mask) -> void;
+
+ /**
+ * Disable a symbol (or it's parent)
+ * @param name
+ * @return
+ */
+ auto disable_symbol(struct rspamd_task *task, const symcache &cache, std::string_view name) -> bool;
+
+ /**
+ * Enable a symbol (or it's parent)
+ * @param name
+ * @return
+ */
+ auto enable_symbol(struct rspamd_task *task, const symcache &cache, std::string_view name) -> bool;
+
+ /**
+ * Checks if an item has been checked/disabled
+ * @param cache
+ * @param name
+ * @return
+ */
+ auto is_symbol_checked(const symcache &cache, std::string_view name) -> bool;
+
+ /**
+ * Checks if a symbol is enabled for execution, checking all pending conditions
+ * @param task
+ * @param cache
+ * @param name
+ * @return
+ */
+ auto is_symbol_enabled(struct rspamd_task *task, const symcache &cache, std::string_view name) -> bool;
+
+ /**
+ * Get the current processed item
+ * @return
+ */
+ auto get_cur_item() const -> auto
+ {
+ return cur_item;
+ }
+
+ /**
+ * Set the current processed item
+ * @param item
+ * @return
+ */
+ auto set_cur_item(cache_dynamic_item *item) -> auto
+ {
+ std::swap(item, cur_item);
+ return item;
+ }
+
+ /**
+ * Set profile mode for the runtime
+ * @param enable
+ * @return
+ */
+ auto set_profile_mode(bool enable) -> auto
+ {
+ std::swap(profile, enable);
+ return enable;
+ }
+
+ /**
+ * Returns the dynamic item by static item id
+ * @param id
+ * @return
+ */
+ auto get_dynamic_item(int id) const -> cache_dynamic_item *;
+
+ /**
+ * Returns static cache item by dynamic cache item
+ * @return
+ */
+ auto get_item_by_dynamic_item(cache_dynamic_item *) const -> cache_item *;
+
+ /**
+ * Process symbols in the cache
+ * @param task
+ * @param cache
+ * @param stage
+ * @return
+ */
+ auto process_symbols(struct rspamd_task *task, symcache &cache, unsigned int stage) -> bool;
+
+ /**
+ * Finalize execution of some item in the cache
+ * @param task
+ * @param item
+ */
+ auto finalize_item(struct rspamd_task *task, cache_dynamic_item *item) -> void;
+
+ /**
+ * Process unblocked reverse dependencies of the specific item
+ * @param task
+ * @param item
+ */
+ auto process_item_rdeps(struct rspamd_task *task, cache_item *item) -> void;
+
+ /* XXX: a helper to allow hiding internal implementation of the slow timer structure */
+ auto unset_slow() -> void
+ {
+ has_slow = false;
+ }
+};
+
+
+}// namespace rspamd::symcache
+
+#endif//RSPAMD_SYMCACHE_RUNTIME_HXX
diff --git a/src/libserver/task.c b/src/libserver/task.c
new file mode 100644
index 0000000..9763d1e
--- /dev/null
+++ b/src/libserver/task.c
@@ -0,0 +1,1975 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "task.h"
+#include "rspamd.h"
+#include "scan_result.h"
+#include "libserver/protocol.h"
+#include "libserver/protocol_internal.h"
+#include "message.h"
+#include "lua/lua_common.h"
+#include "email_addr.h"
+#include "src/libserver/composites/composites.h"
+#include "stat_api.h"
+#include "unix-std.h"
+#include "utlist.h"
+#include "libserver/mempool_vars_internal.h"
+#include "libserver/cfg_file_private.h"
+#include "libmime/lang_detection.h"
+#include "libmime/scan_result_private.h"
+
+#ifdef WITH_JEMALLOC
+#include <jemalloc/jemalloc.h>
+#else
+#if defined(__GLIBC__) && defined(_GNU_SOURCE)
+#include <malloc.h>
+#endif
+#endif
+
+#include <math.h>
+
+#ifdef SYS_ZSTD
+#include "zstd.h"
+#else
+#include "contrib/zstd/zstd.h"
+#endif
+
+__KHASH_IMPL(rspamd_req_headers_hash, static inline,
+ rspamd_ftok_t *, struct rspamd_request_header_chain *, 1,
+ rspamd_ftok_icase_hash, rspamd_ftok_icase_equal)
+
+static GQuark
+rspamd_task_quark(void)
+{
+ return g_quark_from_static_string("task-error");
+}
+
+/*
+ * Create new task
+ */
+struct rspamd_task *
+rspamd_task_new(struct rspamd_worker *worker,
+ struct rspamd_config *cfg,
+ rspamd_mempool_t *pool,
+ struct rspamd_lang_detector *lang_det,
+ struct ev_loop *event_loop,
+ gboolean debug_mem)
+{
+ struct rspamd_task *new_task;
+ rspamd_mempool_t *task_pool;
+ guint flags = 0;
+
+ if (pool == NULL) {
+ task_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "task", debug_mem ? RSPAMD_MEMPOOL_DEBUG : 0);
+ flags |= RSPAMD_TASK_FLAG_OWN_POOL;
+ }
+ else {
+ task_pool = pool;
+ }
+
+ new_task = rspamd_mempool_alloc0(task_pool, sizeof(struct rspamd_task));
+ new_task->task_pool = task_pool;
+ new_task->flags = flags;
+ new_task->worker = worker;
+ new_task->lang_det = lang_det;
+
+ if (cfg) {
+ new_task->cfg = cfg;
+ REF_RETAIN(cfg);
+
+ if (cfg->check_all_filters) {
+ new_task->flags |= RSPAMD_TASK_FLAG_PASS_ALL;
+ }
+
+
+ if (cfg->re_cache) {
+ new_task->re_rt = rspamd_re_cache_runtime_new(cfg->re_cache);
+ }
+
+ if (new_task->lang_det == NULL && cfg->lang_det != NULL) {
+ new_task->lang_det = cfg->lang_det;
+ }
+ }
+
+ new_task->event_loop = event_loop;
+ new_task->task_timestamp = ev_time();
+ new_task->time_real_finish = NAN;
+
+ new_task->request_headers = kh_init(rspamd_req_headers_hash);
+ new_task->sock = -1;
+ new_task->flags |= (RSPAMD_TASK_FLAG_MIME);
+ /* Default results chain */
+ rspamd_create_metric_result(new_task, NULL, -1);
+
+ new_task->queue_id = "undef";
+ new_task->messages = ucl_object_typed_new(UCL_OBJECT);
+ kh_static_init(rspamd_task_lua_cache, &new_task->lua_cache);
+
+ return new_task;
+}
+
+
+static void
+rspamd_task_reply(struct rspamd_task *task)
+{
+ const ev_tstamp write_timeout = 5.0;
+
+ if (task->fin_callback) {
+ task->fin_callback(task, task->fin_arg);
+ }
+ else {
+ if (!(task->processed_stages & RSPAMD_TASK_STAGE_REPLIED)) {
+ rspamd_protocol_write_reply(task, write_timeout);
+ }
+ }
+}
+
+/*
+ * Called if all filters are processed
+ * @return TRUE if session should be terminated
+ */
+gboolean
+rspamd_task_fin(void *arg)
+{
+ struct rspamd_task *task = (struct rspamd_task *) arg;
+
+ /* Task is already finished or skipped */
+ if (RSPAMD_TASK_IS_PROCESSED(task)) {
+ rspamd_task_reply(task);
+ return TRUE;
+ }
+
+ if (!rspamd_task_process(task, RSPAMD_TASK_PROCESS_ALL)) {
+ rspamd_task_reply(task);
+ return TRUE;
+ }
+
+ if (RSPAMD_TASK_IS_PROCESSED(task)) {
+ rspamd_task_reply(task);
+ return TRUE;
+ }
+
+ /* One more iteration */
+ return FALSE;
+}
+
+/*
+ * Free all structures of worker_task
+ */
+void rspamd_task_free(struct rspamd_task *task)
+{
+ struct rspamd_email_address *addr;
+ static guint free_iters = 0;
+ guint i;
+
+ if (task) {
+ debug_task("free pointer %p", task);
+
+ if (task->rcpt_envelope) {
+ for (i = 0; i < task->rcpt_envelope->len; i++) {
+ addr = g_ptr_array_index(task->rcpt_envelope, i);
+ rspamd_email_address_free(addr);
+ }
+
+ g_ptr_array_free(task->rcpt_envelope, TRUE);
+ }
+
+ if (task->from_envelope) {
+ rspamd_email_address_free(task->from_envelope);
+ }
+
+ if (task->from_envelope_orig) {
+ rspamd_email_address_free(task->from_envelope_orig);
+ }
+
+ if (task->meta_words) {
+ g_array_free(task->meta_words, TRUE);
+ }
+
+ ucl_object_unref(task->messages);
+
+ if (task->re_rt) {
+ rspamd_re_cache_runtime_destroy(task->re_rt);
+ }
+
+ if (task->http_conn != NULL) {
+ rspamd_http_connection_reset(task->http_conn);
+ rspamd_http_connection_unref(task->http_conn);
+ }
+
+ if (task->settings != NULL) {
+ ucl_object_unref(task->settings);
+ }
+
+ if (task->settings_elt != NULL) {
+ REF_RELEASE(task->settings_elt);
+ }
+
+ if (task->client_addr) {
+ rspamd_inet_address_free(task->client_addr);
+ }
+
+ if (task->from_addr) {
+ rspamd_inet_address_free(task->from_addr);
+ }
+
+ if (task->err) {
+ g_error_free(task->err);
+ }
+
+ ev_timer_stop(task->event_loop, &task->timeout_ev);
+ ev_io_stop(task->event_loop, &task->guard_ev);
+
+ if (task->sock != -1) {
+ close(task->sock);
+ }
+
+ if (task->cfg) {
+
+
+ struct rspamd_lua_cached_entry entry;
+
+ kh_foreach_value(&task->lua_cache, entry, {
+ luaL_unref(task->cfg->lua_state,
+ LUA_REGISTRYINDEX, entry.ref);
+ });
+ kh_static_destroy(rspamd_task_lua_cache, &task->lua_cache);
+
+ if (task->cfg->full_gc_iters && (++free_iters > task->cfg->full_gc_iters)) {
+ /* Perform more expensive cleanup cycle */
+ gsize allocated = 0, active = 0, metadata = 0,
+ resident = 0, mapped = 0, old_lua_mem = 0;
+ gdouble t1, t2;
+
+ old_lua_mem = lua_gc(task->cfg->lua_state, LUA_GCCOUNT, 0);
+ t1 = rspamd_get_ticks(FALSE);
+
+#ifdef WITH_JEMALLOC
+ gsize sz = sizeof(gsize);
+ mallctl("stats.allocated", &allocated, &sz, NULL, 0);
+ mallctl("stats.active", &active, &sz, NULL, 0);
+ mallctl("stats.metadata", &metadata, &sz, NULL, 0);
+ mallctl("stats.resident", &resident, &sz, NULL, 0);
+ mallctl("stats.mapped", &mapped, &sz, NULL, 0);
+#else
+#if defined(__GLIBC__) && defined(_GNU_SOURCE)
+ malloc_trim(0);
+#endif
+#endif
+ lua_gc(task->cfg->lua_state, LUA_GCCOLLECT, 0);
+ t2 = rspamd_get_ticks(FALSE);
+
+ msg_notice_task("perform full gc cycle; memory stats: "
+ "%Hz allocated, %Hz active, %Hz metadata, %Hz resident, %Hz mapped;"
+ " lua memory: %z kb -> %d kb; %f ms for gc iter",
+ allocated, active, metadata, resident, mapped,
+ old_lua_mem, lua_gc(task->cfg->lua_state, LUA_GCCOUNT, 0),
+ (t2 - t1) * 1000.0);
+ free_iters = rspamd_time_jitter(0,
+ (gdouble) task->cfg->full_gc_iters / 2);
+ }
+
+ REF_RELEASE(task->cfg);
+ }
+
+ kh_destroy(rspamd_req_headers_hash, task->request_headers);
+ rspamd_message_unref(task->message);
+
+ if (task->flags & RSPAMD_TASK_FLAG_OWN_POOL) {
+ rspamd_mempool_destructors_enforce(task->task_pool);
+
+ if (task->symcache_runtime) {
+ rspamd_symcache_runtime_destroy(task);
+ }
+
+ rspamd_mempool_delete(task->task_pool);
+ }
+ else if (task->symcache_runtime) {
+ rspamd_symcache_runtime_destroy(task);
+ }
+ }
+}
+
+struct rspamd_task_map {
+ gpointer begin;
+ gulong len;
+ gint fd;
+};
+
+static void
+rspamd_task_unmapper(gpointer ud)
+{
+ struct rspamd_task_map *m = ud;
+
+ munmap(m->begin, m->len);
+ close(m->fd);
+}
+
+gboolean
+rspamd_task_load_message(struct rspamd_task *task,
+ struct rspamd_http_message *msg, const gchar *start, gsize len)
+{
+ guint control_len, r;
+ struct ucl_parser *parser;
+ ucl_object_t *control_obj;
+ gchar filepath[PATH_MAX], *fp;
+ gint fd, flen;
+ gulong offset = 0, shmem_size = 0;
+ rspamd_ftok_t *tok;
+ gpointer map;
+ struct stat st;
+ struct rspamd_task_map *m;
+ const gchar *ft;
+
+#ifdef HAVE_SANE_SHMEM
+ ft = "shm";
+#else
+ ft = "file";
+#endif
+
+ if (msg) {
+ rspamd_protocol_handle_headers(task, msg);
+ }
+
+ tok = rspamd_task_get_request_header(task, "shm");
+
+ if (tok) {
+ /* Shared memory part */
+ r = rspamd_strlcpy(filepath, tok->begin,
+ MIN(sizeof(filepath), tok->len + 1));
+
+ rspamd_url_decode(filepath, filepath, r + 1);
+ flen = strlen(filepath);
+
+ if (filepath[0] == '"' && flen > 2) {
+ /* We need to unquote filepath */
+ fp = &filepath[1];
+ fp[flen - 2] = '\0';
+ }
+ else {
+ fp = &filepath[0];
+ }
+#ifdef HAVE_SANE_SHMEM
+ fd = shm_open(fp, O_RDONLY, 00600);
+#else
+ fd = open(fp, O_RDONLY, 00600);
+#endif
+ if (fd == -1) {
+ g_set_error(&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
+ "Cannot open %s segment (%s): %s", ft, fp, strerror(errno));
+ return FALSE;
+ }
+
+ if (fstat(fd, &st) == -1) {
+ g_set_error(&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
+ "Cannot stat %s segment (%s): %s", ft, fp, strerror(errno));
+ close(fd);
+
+ return FALSE;
+ }
+
+ map = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
+
+ if (map == MAP_FAILED) {
+ close(fd);
+ g_set_error(&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
+ "Cannot mmap %s (%s): %s", ft, fp, strerror(errno));
+ return FALSE;
+ }
+
+ tok = rspamd_task_get_request_header(task, "shm-offset");
+
+ if (tok) {
+ rspamd_strtoul(tok->begin, tok->len, &offset);
+
+ if (offset > (gulong) st.st_size) {
+ msg_err_task("invalid offset %ul (%ul available) for shm "
+ "segment %s",
+ offset, (gulong) st.st_size, fp);
+ munmap(map, st.st_size);
+ close(fd);
+
+ return FALSE;
+ }
+ }
+
+ tok = rspamd_task_get_request_header(task, "shm-length");
+ shmem_size = st.st_size;
+
+
+ if (tok) {
+ rspamd_strtoul(tok->begin, tok->len, &shmem_size);
+
+ if (shmem_size > (gulong) st.st_size) {
+ msg_err_task("invalid length %ul (%ul available) for %s "
+ "segment %s",
+ shmem_size, (gulong) st.st_size, ft, fp);
+ munmap(map, st.st_size);
+ close(fd);
+
+ return FALSE;
+ }
+ }
+
+ task->msg.begin = ((guchar *) map) + offset;
+ task->msg.len = shmem_size;
+ m = rspamd_mempool_alloc(task->task_pool, sizeof(*m));
+ m->begin = map;
+ m->len = st.st_size;
+ m->fd = fd;
+
+ msg_info_task("loaded message from shared memory %s (%ul size, %ul offset), fd=%d",
+ fp, shmem_size, offset, fd);
+
+ rspamd_mempool_add_destructor(task->task_pool, rspamd_task_unmapper, m);
+
+ return TRUE;
+ }
+
+ tok = rspamd_task_get_request_header(task, "file");
+
+ if (tok == NULL) {
+ tok = rspamd_task_get_request_header(task, "path");
+ }
+
+ if (tok) {
+ debug_task("want to scan file %T", tok);
+
+ r = rspamd_strlcpy(filepath, tok->begin,
+ MIN(sizeof(filepath), tok->len + 1));
+
+ rspamd_url_decode(filepath, filepath, r + 1);
+ flen = strlen(filepath);
+
+ if (filepath[0] == '"' && flen > 2) {
+ /* We need to unquote filepath */
+ fp = &filepath[1];
+ fp[flen - 2] = '\0';
+ }
+ else {
+ fp = &filepath[0];
+ }
+
+ if (stat(fp, &st) == -1) {
+ g_set_error(&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
+ "Invalid file (%s): %s", fp, strerror(errno));
+ return FALSE;
+ }
+
+ if (G_UNLIKELY(st.st_size == 0)) {
+ /* Empty file */
+ task->flags |= RSPAMD_TASK_FLAG_EMPTY;
+ task->msg.begin = rspamd_mempool_strdup(task->task_pool, "");
+ task->msg.len = 0;
+ }
+ else {
+ fd = open(fp, O_RDONLY);
+
+ if (fd == -1) {
+ g_set_error(&task->err, rspamd_task_quark(),
+ RSPAMD_PROTOCOL_ERROR,
+ "Cannot open file (%s): %s", fp, strerror(errno));
+ return FALSE;
+ }
+
+ map = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
+
+
+ if (map == MAP_FAILED) {
+ close(fd);
+ g_set_error(&task->err, rspamd_task_quark(),
+ RSPAMD_PROTOCOL_ERROR,
+ "Cannot mmap file (%s): %s", fp, strerror(errno));
+ return FALSE;
+ }
+
+ task->msg.begin = map;
+ task->msg.len = st.st_size;
+ m = rspamd_mempool_alloc(task->task_pool, sizeof(*m));
+ m->begin = map;
+ m->len = st.st_size;
+ m->fd = fd;
+
+ rspamd_mempool_add_destructor(task->task_pool, rspamd_task_unmapper, m);
+ }
+
+ task->msg.fpath = rspamd_mempool_strdup(task->task_pool, fp);
+ task->flags |= RSPAMD_TASK_FLAG_FILE;
+
+ msg_info_task("loaded message from file %s", fp);
+
+ return TRUE;
+ }
+
+ /* Plain data */
+ debug_task("got input of length %z", task->msg.len);
+
+ /* Check compression */
+ tok = rspamd_task_get_request_header(task, "compression");
+
+ if (tok) {
+ /* Need to uncompress */
+ rspamd_ftok_t t;
+
+ t.begin = "zstd";
+ t.len = 4;
+
+ if (rspamd_ftok_casecmp(tok, &t) == 0) {
+ ZSTD_DStream *zstream;
+ ZSTD_inBuffer zin;
+ ZSTD_outBuffer zout;
+ guchar *out;
+ gsize outlen, r;
+ gulong dict_id;
+
+ if (!rspamd_libs_reset_decompression(task->cfg->libs_ctx)) {
+ g_set_error(&task->err, rspamd_task_quark(),
+ RSPAMD_PROTOCOL_ERROR,
+ "Cannot decompress, decompressor init failed");
+
+ return FALSE;
+ }
+
+ tok = rspamd_task_get_request_header(task, "dictionary");
+
+ if (tok != NULL) {
+ /* We need to use custom dictionary */
+ if (!rspamd_strtoul(tok->begin, tok->len, &dict_id)) {
+ g_set_error(&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
+ "Non numeric dictionary");
+
+ return FALSE;
+ }
+
+ if (!task->cfg->libs_ctx->in_dict) {
+ g_set_error(&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
+ "Unknown dictionary, undefined locally");
+
+ return FALSE;
+ }
+
+ if (task->cfg->libs_ctx->in_dict->id != dict_id) {
+ g_set_error(&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
+ "Unknown dictionary, invalid dictionary id");
+
+ return FALSE;
+ }
+ }
+
+ zstream = task->cfg->libs_ctx->in_zstream;
+
+ zin.pos = 0;
+ zin.src = start;
+ zin.size = len;
+
+ if ((outlen = ZSTD_getDecompressedSize(start, len)) == 0) {
+ outlen = ZSTD_DStreamOutSize();
+ }
+
+ out = g_malloc(outlen);
+ zout.dst = out;
+ zout.pos = 0;
+ zout.size = outlen;
+
+ while (zin.pos < zin.size) {
+ r = ZSTD_decompressStream(zstream, &zout, &zin);
+
+ if (ZSTD_isError(r)) {
+ g_set_error(&task->err, rspamd_task_quark(),
+ RSPAMD_PROTOCOL_ERROR,
+ "Decompression error: %s", ZSTD_getErrorName(r));
+
+ return FALSE;
+ }
+
+ if (zout.pos == zout.size) {
+ /* We need to extend output buffer */
+ zout.size = zout.size * 2 + 1;
+ zout.dst = g_realloc(zout.dst, zout.size);
+ }
+ }
+
+ rspamd_mempool_add_destructor(task->task_pool, g_free, zout.dst);
+ task->msg.begin = zout.dst;
+ task->msg.len = zout.pos;
+ task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_COMPRESSED;
+
+ msg_info_task("loaded message from zstd compressed stream; "
+ "compressed: %ul; uncompressed: %ul",
+ (gulong) zin.size, (gulong) zout.pos);
+ }
+ else {
+ g_set_error(&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
+ "Invalid compression method");
+ return FALSE;
+ }
+ }
+ else {
+ task->msg.begin = start;
+ task->msg.len = len;
+ }
+
+ if (task->msg.len == 0) {
+ task->flags |= RSPAMD_TASK_FLAG_EMPTY;
+ }
+
+ if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_HAS_CONTROL) {
+ rspamd_ftok_t *hv = rspamd_task_get_request_header(task, MLEN_HEADER);
+ gulong message_len = 0;
+
+ if (!hv || !rspamd_strtoul(hv->begin, hv->len, &message_len) ||
+ task->msg.len < message_len) {
+ msg_warn_task("message has invalid message length: %ul and total len: %ul",
+ message_len, task->msg.len);
+ g_set_error(&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
+ "Invalid length");
+ return FALSE;
+ }
+
+ control_len = task->msg.len - message_len;
+
+ if (control_len > 0) {
+ parser = ucl_parser_new(UCL_PARSER_KEY_LOWERCASE);
+
+ if (!ucl_parser_add_chunk(parser, task->msg.begin, control_len)) {
+ msg_warn_task("processing of control chunk failed: %s",
+ ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+ }
+ else {
+ control_obj = ucl_parser_get_object(parser);
+ ucl_parser_free(parser);
+ rspamd_protocol_handle_control(task, control_obj);
+ ucl_object_unref(control_obj);
+ }
+
+ task->msg.begin += control_len;
+ task->msg.len -= control_len;
+ }
+ }
+
+ return TRUE;
+}
+
+static guint
+rspamd_task_select_processing_stage(struct rspamd_task *task, guint stages)
+{
+ guint st, mask;
+
+ mask = task->processed_stages;
+
+ if (mask == 0) {
+ st = 0;
+ }
+ else {
+ for (st = 1; mask != 1; st++) {
+ mask = mask >> 1u;
+ }
+ }
+
+ st = 1 << st;
+
+ if (stages & st) {
+ return st;
+ }
+ else if (st < RSPAMD_TASK_STAGE_DONE) {
+ /* We assume that the stage that was not requested is done */
+ task->processed_stages |= st;
+ return rspamd_task_select_processing_stage(task, stages);
+ }
+
+ /* We are done */
+ return RSPAMD_TASK_STAGE_DONE;
+}
+
+gboolean
+rspamd_task_process(struct rspamd_task *task, guint stages)
+{
+ guint st;
+ gboolean ret = TRUE, all_done = TRUE;
+ GError *stat_error = NULL;
+
+ /* Avoid nested calls */
+ if (task->flags & RSPAMD_TASK_FLAG_PROCESSING) {
+ return TRUE;
+ }
+
+ if (RSPAMD_TASK_IS_PROCESSED(task)) {
+ return TRUE;
+ }
+
+ task->flags |= RSPAMD_TASK_FLAG_PROCESSING;
+
+ st = rspamd_task_select_processing_stage(task, stages);
+
+ switch (st) {
+ case RSPAMD_TASK_STAGE_CONNFILTERS:
+ all_done = rspamd_symcache_process_symbols(task, task->cfg->cache, st);
+ break;
+
+ case RSPAMD_TASK_STAGE_READ_MESSAGE:
+ if (!rspamd_message_parse(task)) {
+ ret = FALSE;
+ }
+ break;
+
+ case RSPAMD_TASK_STAGE_PROCESS_MESSAGE:
+ if (!(task->flags & RSPAMD_TASK_FLAG_SKIP_PROCESS)) {
+ rspamd_message_process(task);
+ }
+ break;
+
+ case RSPAMD_TASK_STAGE_PRE_FILTERS:
+ case RSPAMD_TASK_STAGE_FILTERS:
+ all_done = rspamd_symcache_process_symbols(task, task->cfg->cache, st);
+ break;
+
+ case RSPAMD_TASK_STAGE_CLASSIFIERS:
+ case RSPAMD_TASK_STAGE_CLASSIFIERS_PRE:
+ case RSPAMD_TASK_STAGE_CLASSIFIERS_POST:
+ if (!RSPAMD_TASK_IS_EMPTY(task)) {
+ if (rspamd_stat_classify(task, task->cfg->lua_state, st, &stat_error) ==
+ RSPAMD_STAT_PROCESS_ERROR) {
+ msg_err_task("classify error: %e", stat_error);
+ g_error_free(stat_error);
+ }
+ }
+ break;
+
+ case RSPAMD_TASK_STAGE_COMPOSITES:
+ rspamd_composites_process_task(task);
+ task->result->nresults_postfilters = task->result->nresults;
+ break;
+
+ case RSPAMD_TASK_STAGE_POST_FILTERS:
+ all_done = rspamd_symcache_process_symbols(task, task->cfg->cache,
+ st);
+
+ if (all_done && (task->flags & RSPAMD_TASK_FLAG_LEARN_AUTO) &&
+ !RSPAMD_TASK_IS_EMPTY(task) &&
+ !(task->flags & (RSPAMD_TASK_FLAG_LEARN_SPAM | RSPAMD_TASK_FLAG_LEARN_HAM))) {
+ rspamd_stat_check_autolearn(task);
+ }
+ break;
+
+ case RSPAMD_TASK_STAGE_LEARN:
+ case RSPAMD_TASK_STAGE_LEARN_PRE:
+ case RSPAMD_TASK_STAGE_LEARN_POST:
+ if (task->flags & (RSPAMD_TASK_FLAG_LEARN_SPAM | RSPAMD_TASK_FLAG_LEARN_HAM)) {
+ if (task->err == NULL) {
+ if (!rspamd_stat_learn(task,
+ task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM,
+ task->cfg->lua_state, task->classifier,
+ st, &stat_error)) {
+
+ if (stat_error == NULL) {
+ g_set_error(&stat_error,
+ g_quark_from_static_string("stat"), 500,
+ "Unknown statistics error, found on stage %s;"
+ " classifier: %s",
+ rspamd_task_stage_name(st), task->classifier);
+ }
+
+ if (stat_error->code >= 400) {
+ msg_err_task("learn error: %e", stat_error);
+ }
+ else {
+ msg_notice_task("skip learning: %e", stat_error);
+ }
+
+ if (!(task->flags & RSPAMD_TASK_FLAG_LEARN_AUTO)) {
+ task->err = stat_error;
+ task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
+ }
+ else {
+ /* Do not skip idempotent in case of learn error */
+ if (stat_error) {
+ g_error_free(stat_error);
+ }
+
+ task->processed_stages |= RSPAMD_TASK_STAGE_LEARN |
+ RSPAMD_TASK_STAGE_LEARN_PRE |
+ RSPAMD_TASK_STAGE_LEARN_POST;
+ }
+ }
+ }
+ }
+ break;
+ case RSPAMD_TASK_STAGE_COMPOSITES_POST:
+ /* Second run of composites processing before idempotent filters (if needed) */
+ if (task->result->nresults_postfilters != task->result->nresults) {
+ rspamd_composites_process_task(task);
+ }
+ else {
+ msg_debug_task("skip second run of composites as the result has not been changed");
+ }
+ break;
+
+ case RSPAMD_TASK_STAGE_IDEMPOTENT:
+ /* Stop task timeout */
+ if (ev_can_stop(&task->timeout_ev)) {
+ ev_timer_stop(task->event_loop, &task->timeout_ev);
+ }
+
+ all_done = rspamd_symcache_process_symbols(task, task->cfg->cache, st);
+ break;
+
+ case RSPAMD_TASK_STAGE_DONE:
+ task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
+ break;
+
+ default:
+ /* TODO: not implemented stage */
+ break;
+ }
+
+ if (RSPAMD_TASK_IS_SKIPPED(task)) {
+ /* Set all bits except idempotent filters */
+ task->processed_stages |= 0x7FFF;
+ }
+
+ task->flags &= ~RSPAMD_TASK_FLAG_PROCESSING;
+
+ if (!ret || RSPAMD_TASK_IS_PROCESSED(task)) {
+ if (!ret) {
+ /* Set processed flags */
+ task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
+ }
+
+ msg_debug_task("task is processed");
+
+ return ret;
+ }
+
+ if (ret) {
+ if (rspamd_session_events_pending(task->s) != 0) {
+ /* We have events pending, so we consider this stage as incomplete */
+ msg_debug_task("need more work on stage %d", st);
+ }
+ else {
+ if (all_done) {
+ /* Mark the current stage as done and go to the next stage */
+ msg_debug_task("completed stage %d", st);
+ task->processed_stages |= st;
+ }
+ else {
+ msg_debug_task("need more processing on stage %d", st);
+ }
+
+ /* Tail recursion */
+ return rspamd_task_process(task, stages);
+ }
+ }
+
+ return ret;
+}
+
+struct rspamd_email_address *
+rspamd_task_get_sender(struct rspamd_task *task)
+{
+ return task->from_envelope;
+}
+
+static const gchar *
+rspamd_task_cache_principal_recipient(struct rspamd_task *task,
+ const gchar *rcpt, gsize len)
+{
+ gchar *rcpt_lc;
+
+ if (rcpt == NULL) {
+ return NULL;
+ }
+
+ rcpt_lc = rspamd_mempool_alloc(task->task_pool, len + 1);
+ rspamd_strlcpy(rcpt_lc, rcpt, len + 1);
+ rspamd_str_lc(rcpt_lc, len);
+
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_PRINCIPAL_RECIPIENT, rcpt_lc, NULL);
+
+ return rcpt_lc;
+}
+
+const gchar *
+rspamd_task_get_principal_recipient(struct rspamd_task *task)
+{
+ const gchar *val;
+ struct rspamd_email_address *addr;
+ guint i;
+
+ val = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_PRINCIPAL_RECIPIENT);
+
+ if (val) {
+ return val;
+ }
+
+ if (task->deliver_to) {
+ return rspamd_task_cache_principal_recipient(task, task->deliver_to,
+ strlen(task->deliver_to));
+ }
+ if (task->rcpt_envelope != NULL) {
+
+ PTR_ARRAY_FOREACH(task->rcpt_envelope, i, addr)
+ {
+ if (addr->addr && !(addr->flags & RSPAMD_EMAIL_ADDR_ORIGINAL)) {
+ return rspamd_task_cache_principal_recipient(task, addr->addr,
+ addr->addr_len);
+ }
+ }
+ }
+
+ GPtrArray *rcpt_mime = MESSAGE_FIELD_CHECK(task, rcpt_mime);
+ if (rcpt_mime != NULL && rcpt_mime->len > 0) {
+ PTR_ARRAY_FOREACH(rcpt_mime, i, addr)
+ {
+ if (addr->addr && !(addr->flags & RSPAMD_EMAIL_ADDR_ORIGINAL)) {
+ return rspamd_task_cache_principal_recipient(task, addr->addr,
+ addr->addr_len);
+ }
+ }
+ }
+
+ return NULL;
+}
+
+gboolean
+rspamd_learn_task_spam(struct rspamd_task *task,
+ gboolean is_spam,
+ const gchar *classifier,
+ GError **err)
+{
+ if (is_spam) {
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_SPAM;
+ }
+ else {
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_HAM;
+ }
+
+ task->classifier = classifier;
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_task_log_check_condition(struct rspamd_task *task,
+ struct rspamd_log_format *lf)
+{
+ gboolean ret = FALSE;
+
+ switch (lf->type) {
+ case RSPAMD_LOG_MID:
+ if (MESSAGE_FIELD_CHECK(task, message_id) &&
+ strcmp(MESSAGE_FIELD(task, message_id), "undef") != 0) {
+ ret = TRUE;
+ }
+ break;
+ case RSPAMD_LOG_QID:
+ if (task->queue_id && strcmp(task->queue_id, "undef") != 0) {
+ ret = TRUE;
+ }
+ break;
+ case RSPAMD_LOG_USER:
+ if (task->auth_user) {
+ ret = TRUE;
+ }
+ break;
+ case RSPAMD_LOG_IP:
+ if (task->from_addr && rspamd_ip_is_valid(task->from_addr)) {
+ ret = TRUE;
+ }
+ break;
+ case RSPAMD_LOG_SMTP_RCPT:
+ case RSPAMD_LOG_SMTP_RCPTS:
+ if (task->rcpt_envelope && task->rcpt_envelope->len > 0) {
+ ret = TRUE;
+ }
+ break;
+ case RSPAMD_LOG_MIME_RCPT:
+ case RSPAMD_LOG_MIME_RCPTS:
+ if (MESSAGE_FIELD_CHECK(task, rcpt_mime) &&
+ MESSAGE_FIELD(task, rcpt_mime)->len > 0) {
+ ret = TRUE;
+ }
+ break;
+ case RSPAMD_LOG_SMTP_FROM:
+ if (task->from_envelope) {
+ ret = TRUE;
+ }
+ break;
+ case RSPAMD_LOG_MIME_FROM:
+ if (MESSAGE_FIELD_CHECK(task, from_mime) &&
+ MESSAGE_FIELD(task, from_mime)->len > 0) {
+ ret = TRUE;
+ }
+ break;
+ case RSPAMD_LOG_FILENAME:
+ if (task->msg.fpath) {
+ ret = TRUE;
+ }
+ break;
+ case RSPAMD_LOG_FORCED_ACTION:
+ if (task->result->passthrough_result) {
+ ret = TRUE;
+ }
+ break;
+ case RSPAMD_LOG_SETTINGS_ID:
+ if (task->settings_elt) {
+ ret = TRUE;
+ }
+ break;
+ default:
+ ret = TRUE;
+ break;
+ }
+
+ return ret;
+}
+
+/*
+ * Sort by symbol's score -> name
+ */
+static gint
+rspamd_task_compare_log_sym(gconstpointer a, gconstpointer b)
+{
+ const struct rspamd_symbol_result *s1 = *(const struct rspamd_symbol_result **) a,
+ *s2 = *(const struct rspamd_symbol_result **) b;
+ gdouble w1, w2;
+
+
+ w1 = fabs(s1->score);
+ w2 = fabs(s2->score);
+
+ if (w1 == w2 && s1->name && s2->name) {
+ return strcmp(s1->name, s2->name);
+ }
+
+ return (w2 - w1) * 1000.0;
+}
+
+static gint
+rspamd_task_compare_log_group(gconstpointer a, gconstpointer b)
+{
+ const struct rspamd_symbols_group *s1 = *(const struct rspamd_symbols_group **) a,
+ *s2 = *(const struct rspamd_symbols_group **) b;
+
+ return strcmp(s1->name, s2->name);
+}
+
+
+static rspamd_ftok_t
+rspamd_task_log_metric_res(struct rspamd_task *task,
+ struct rspamd_log_format *lf)
+{
+ static gchar scorebuf[32];
+ rspamd_ftok_t res = {.begin = NULL, .len = 0};
+ struct rspamd_scan_result *mres;
+ gboolean first = TRUE;
+ rspamd_fstring_t *symbuf;
+ struct rspamd_symbol_result *sym;
+ GPtrArray *sorted_symbols;
+ struct rspamd_action *act;
+ struct rspamd_symbols_group *gr;
+ guint i, j;
+ khiter_t k;
+ guint max_log_elts = task->cfg->log_task_max_elts;
+
+ mres = task->result;
+ act = rspamd_check_action_metric(task, NULL, NULL);
+
+ if (mres != NULL) {
+ switch (lf->type) {
+ case RSPAMD_LOG_ISSPAM:
+ if (RSPAMD_TASK_IS_SKIPPED(task)) {
+ res.begin = "S";
+ }
+ else if (!(act->flags & RSPAMD_ACTION_HAM)) {
+ res.begin = "T";
+ }
+ else {
+ res.begin = "F";
+ }
+
+ res.len = 1;
+ break;
+ case RSPAMD_LOG_ACTION:
+ res.begin = act->name;
+ res.len = strlen(res.begin);
+ break;
+ case RSPAMD_LOG_SCORES:
+ res.len = rspamd_snprintf(scorebuf, sizeof(scorebuf), "%.2f/%.2f",
+ mres->score, rspamd_task_get_required_score(task, mres));
+ res.begin = scorebuf;
+ break;
+ case RSPAMD_LOG_SYMBOLS:
+ symbuf = rspamd_fstring_sized_new(128);
+ sorted_symbols = g_ptr_array_sized_new(kh_size(mres->symbols));
+
+ kh_foreach_value(mres->symbols, sym, {
+ if (!(sym->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ g_ptr_array_add(sorted_symbols, (gpointer) sym);
+ }
+ });
+
+ g_ptr_array_sort(sorted_symbols, rspamd_task_compare_log_sym);
+
+ for (i = 0; i < sorted_symbols->len; i++) {
+ sym = g_ptr_array_index(sorted_symbols, i);
+
+ if (first) {
+ rspamd_printf_fstring(&symbuf, "%s", sym->name);
+ }
+ else {
+ rspamd_printf_fstring(&symbuf, ",%s", sym->name);
+ }
+
+ if (lf->flags & RSPAMD_LOG_FMT_FLAG_SYMBOLS_SCORES) {
+ rspamd_printf_fstring(&symbuf, "(%.2f)", sym->score);
+ }
+
+ if (lf->flags & RSPAMD_LOG_FMT_FLAG_SYMBOLS_PARAMS) {
+ rspamd_printf_fstring(&symbuf, "{");
+
+ if (sym->options) {
+ struct rspamd_symbol_option *opt;
+
+ j = 0;
+
+ DL_FOREACH(sym->opts_head, opt)
+ {
+ rspamd_printf_fstring(&symbuf, "%*s;",
+ (gint) opt->optlen, opt->option);
+
+ if (j >= max_log_elts && opt->next) {
+ rspamd_printf_fstring(&symbuf, "...;");
+ break;
+ }
+
+ j++;
+ }
+ }
+
+ rspamd_printf_fstring(&symbuf, "}");
+ }
+
+ first = FALSE;
+ }
+
+ g_ptr_array_free(sorted_symbols, TRUE);
+
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) rspamd_fstring_free,
+ symbuf);
+ rspamd_mempool_notify_alloc(task->task_pool, symbuf->len);
+ res.begin = symbuf->str;
+ res.len = symbuf->len;
+ break;
+
+ case RSPAMD_LOG_GROUPS:
+ case RSPAMD_LOG_PUBLIC_GROUPS:
+
+ symbuf = rspamd_fstring_sized_new(128);
+ sorted_symbols = g_ptr_array_sized_new(kh_size(mres->sym_groups));
+
+ kh_foreach_key(mres->sym_groups, gr, {
+ if (!(gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC)) {
+ if (lf->type == RSPAMD_LOG_PUBLIC_GROUPS) {
+ continue;
+ }
+ }
+
+ g_ptr_array_add(sorted_symbols, gr);
+ });
+
+ g_ptr_array_sort(sorted_symbols, rspamd_task_compare_log_group);
+
+ for (i = 0; i < sorted_symbols->len; i++) {
+ gr = g_ptr_array_index(sorted_symbols, i);
+
+ if (first) {
+ rspamd_printf_fstring(&symbuf, "%s", gr->name);
+ }
+ else {
+ rspamd_printf_fstring(&symbuf, ",%s", gr->name);
+ }
+
+ k = kh_get(rspamd_symbols_group_hash, mres->sym_groups, gr);
+
+ rspamd_printf_fstring(&symbuf, "(%.2f)",
+ kh_value(mres->sym_groups, k));
+
+ first = FALSE;
+ }
+
+ g_ptr_array_free(sorted_symbols, TRUE);
+
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) rspamd_fstring_free,
+ symbuf);
+ rspamd_mempool_notify_alloc(task->task_pool, symbuf->len);
+ res.begin = symbuf->str;
+ res.len = symbuf->len;
+ break;
+ default:
+ break;
+ }
+ }
+
+ return res;
+}
+
+static rspamd_fstring_t *
+rspamd_task_log_write_var(struct rspamd_task *task, rspamd_fstring_t *logbuf,
+ const rspamd_ftok_t *var, const rspamd_ftok_t *content)
+{
+ rspamd_fstring_t *res = logbuf;
+ const gchar *p, *c, *end;
+
+ if (content == NULL) {
+ /* Just output variable */
+ res = rspamd_fstring_append(res, var->begin, var->len);
+ }
+ else {
+ /* Replace $ with variable value */
+ p = content->begin;
+ c = p;
+ end = p + content->len;
+
+ while (p < end) {
+ if (*p == '$') {
+ if (p > c) {
+ res = rspamd_fstring_append(res, c, p - c);
+ }
+
+ res = rspamd_fstring_append(res, var->begin, var->len);
+ p++;
+ c = p;
+ }
+ else {
+ p++;
+ }
+ }
+
+ if (p > c) {
+ res = rspamd_fstring_append(res, c, p - c);
+ }
+ }
+
+ return res;
+}
+
+static rspamd_fstring_t *
+rspamd_task_write_ialist(struct rspamd_task *task,
+ GPtrArray *addrs, gint lim,
+ struct rspamd_log_format *lf,
+ rspamd_fstring_t *logbuf)
+{
+ rspamd_fstring_t *res = logbuf, *varbuf;
+ rspamd_ftok_t var = {.begin = NULL, .len = 0};
+ struct rspamd_email_address *addr;
+ gint i, nchars = 0, wr = 0, cur_chars;
+ gboolean has_orig = FALSE;
+ guint max_log_elts = task->cfg->log_task_max_elts;
+
+ if (addrs && lim <= 0) {
+ lim = addrs->len;
+ }
+
+ PTR_ARRAY_FOREACH(addrs, i, addr)
+ {
+ if (addr->flags & RSPAMD_EMAIL_ADDR_ORIGINAL) {
+ has_orig = TRUE;
+ break;
+ }
+ }
+
+ varbuf = rspamd_fstring_new();
+
+ PTR_ARRAY_FOREACH(addrs, i, addr)
+ {
+ if (wr >= lim) {
+ break;
+ }
+
+ if (has_orig) {
+ /* Report merely original addresses */
+ if (!(addr->flags & RSPAMD_EMAIL_ADDR_ORIGINAL)) {
+ continue;
+ }
+ }
+
+ bool last = i == lim - 1;
+
+ cur_chars = addr->addr_len;
+ varbuf = rspamd_fstring_append(varbuf, addr->addr,
+ cur_chars);
+ nchars += cur_chars;
+ wr++;
+
+ if (varbuf->len > 0 && !last) {
+ varbuf = rspamd_fstring_append(varbuf, ",", 1);
+ }
+
+ if (!last && (wr >= max_log_elts || nchars >= max_log_elts * 16)) {
+ varbuf = rspamd_fstring_append(varbuf, "...", 3);
+ break;
+ }
+ }
+
+ if (varbuf->len > 0) {
+ var.begin = varbuf->str;
+ var.len = varbuf->len;
+ res = rspamd_task_log_write_var(task, logbuf,
+ &var, (const rspamd_ftok_t *) lf->data);
+ }
+
+ rspamd_fstring_free(varbuf);
+
+ return res;
+}
+
+static rspamd_fstring_t *
+rspamd_task_write_addr_list(struct rspamd_task *task,
+ GPtrArray *addrs, gint lim,
+ struct rspamd_log_format *lf,
+ rspamd_fstring_t *logbuf)
+{
+ rspamd_fstring_t *res = logbuf, *varbuf;
+ rspamd_ftok_t var = {.begin = NULL, .len = 0};
+ struct rspamd_email_address *addr;
+ guint max_log_elts = task->cfg->log_task_max_elts;
+ guint i;
+
+ if (lim <= 0) {
+ lim = addrs->len;
+ }
+
+ varbuf = rspamd_fstring_new();
+
+ for (i = 0; i < lim; i++) {
+ addr = g_ptr_array_index(addrs, i);
+ bool last = i == lim - 1;
+
+ if (addr->addr) {
+ varbuf = rspamd_fstring_append(varbuf, addr->addr, addr->addr_len);
+ }
+
+ if (varbuf->len > 0 && !last) {
+ varbuf = rspamd_fstring_append(varbuf, ",", 1);
+ }
+
+ if (!last && i >= max_log_elts) {
+ varbuf = rspamd_fstring_append(varbuf, "...", 3);
+ break;
+ }
+ }
+
+ if (varbuf->len > 0) {
+ var.begin = varbuf->str;
+ var.len = varbuf->len;
+ res = rspamd_task_log_write_var(task, logbuf,
+ &var, (const rspamd_ftok_t *) lf->data);
+ }
+
+ rspamd_fstring_free(varbuf);
+
+ return res;
+}
+
+static rspamd_fstring_t *
+rspamd_task_log_variable(struct rspamd_task *task,
+ struct rspamd_log_format *lf, rspamd_fstring_t *logbuf)
+{
+ rspamd_fstring_t *res = logbuf;
+ rspamd_ftok_t var = {.begin = NULL, .len = 0};
+ static gchar numbuf[128];
+ static const gchar undef[] = "undef";
+
+ switch (lf->type) {
+ /* String vars */
+ case RSPAMD_LOG_MID:
+ if (MESSAGE_FIELD_CHECK(task, message_id)) {
+ var.begin = MESSAGE_FIELD(task, message_id);
+ var.len = strlen(var.begin);
+ }
+ else {
+ var.begin = undef;
+ var.len = sizeof(undef) - 1;
+ }
+ break;
+ case RSPAMD_LOG_QID:
+ if (task->queue_id) {
+ var.begin = task->queue_id;
+ var.len = strlen(var.begin);
+ }
+ else {
+ var.begin = undef;
+ var.len = sizeof(undef) - 1;
+ }
+ break;
+ case RSPAMD_LOG_USER:
+ if (task->auth_user) {
+ var.begin = task->auth_user;
+ var.len = strlen(var.begin);
+ }
+ else {
+ var.begin = undef;
+ var.len = sizeof(undef) - 1;
+ }
+ break;
+ case RSPAMD_LOG_IP:
+ if (task->from_addr && rspamd_ip_is_valid(task->from_addr)) {
+ var.begin = rspamd_inet_address_to_string(task->from_addr);
+ var.len = strlen(var.begin);
+ }
+ else {
+ var.begin = undef;
+ var.len = sizeof(undef) - 1;
+ }
+ break;
+ /* Numeric vars */
+ case RSPAMD_LOG_LEN:
+ var.len = rspamd_snprintf(numbuf, sizeof(numbuf), "%uz",
+ task->msg.len);
+ var.begin = numbuf;
+ break;
+ case RSPAMD_LOG_DNS_REQ:
+ var.len = rspamd_snprintf(numbuf, sizeof(numbuf), "%uD",
+ task->dns_requests);
+ var.begin = numbuf;
+ break;
+ case RSPAMD_LOG_TIME_REAL:
+ case RSPAMD_LOG_TIME_VIRTUAL:
+ var.begin = rspamd_log_check_time(task->task_timestamp,
+ task->time_real_finish,
+ task->cfg->clock_res);
+ var.len = strlen(var.begin);
+ break;
+ /* InternetAddress vars */
+ case RSPAMD_LOG_SMTP_FROM:
+ if (task->from_envelope) {
+ var.begin = task->from_envelope->addr;
+ var.len = task->from_envelope->addr_len;
+ }
+ break;
+ case RSPAMD_LOG_MIME_FROM:
+ if (MESSAGE_FIELD_CHECK(task, from_mime)) {
+ return rspamd_task_write_ialist(task,
+ MESSAGE_FIELD(task, from_mime),
+ 1,
+ lf,
+ logbuf);
+ }
+ break;
+ case RSPAMD_LOG_SMTP_RCPT:
+ if (task->rcpt_envelope) {
+ return rspamd_task_write_addr_list(task, task->rcpt_envelope, 1, lf,
+ logbuf);
+ }
+ break;
+ case RSPAMD_LOG_MIME_RCPT:
+ if (MESSAGE_FIELD_CHECK(task, rcpt_mime)) {
+ return rspamd_task_write_ialist(task,
+ MESSAGE_FIELD(task, rcpt_mime),
+ 1,
+ lf,
+ logbuf);
+ }
+ break;
+ case RSPAMD_LOG_SMTP_RCPTS:
+ if (task->rcpt_envelope) {
+ return rspamd_task_write_addr_list(task, task->rcpt_envelope, -1, lf,
+ logbuf);
+ }
+ break;
+ case RSPAMD_LOG_MIME_RCPTS:
+ if (MESSAGE_FIELD_CHECK(task, rcpt_mime)) {
+ return rspamd_task_write_ialist(task,
+ MESSAGE_FIELD(task, rcpt_mime),
+ -1, /* All addresses */
+ lf,
+ logbuf);
+ }
+ break;
+ case RSPAMD_LOG_DIGEST:
+ if (task->message) {
+ var.len = rspamd_snprintf(numbuf, sizeof(numbuf), "%*xs",
+ (gint) sizeof(MESSAGE_FIELD(task, digest)),
+ MESSAGE_FIELD(task, digest));
+ var.begin = numbuf;
+ }
+ else {
+ var.begin = undef;
+ var.len = sizeof(undef) - 1;
+ }
+ break;
+ case RSPAMD_LOG_FILENAME:
+ if (task->msg.fpath) {
+ var.len = strlen(task->msg.fpath);
+ var.begin = task->msg.fpath;
+ }
+ else {
+ var.begin = undef;
+ var.len = sizeof(undef) - 1;
+ }
+ break;
+ case RSPAMD_LOG_FORCED_ACTION:
+ if (task->result->passthrough_result) {
+ struct rspamd_passthrough_result *pr = task->result->passthrough_result;
+
+ if (!isnan(pr->target_score)) {
+ var.len = rspamd_snprintf(numbuf, sizeof(numbuf),
+ "%s \"%s\"; score=%.2f (set by %s)",
+ pr->action->name,
+ pr->message,
+ pr->target_score,
+ pr->module);
+ }
+ else {
+ var.len = rspamd_snprintf(numbuf, sizeof(numbuf),
+ "%s \"%s\"; score=nan (set by %s)",
+ pr->action->name,
+ pr->message,
+ pr->module);
+ }
+ var.begin = numbuf;
+ }
+ else {
+ var.begin = undef;
+ var.len = sizeof(undef) - 1;
+ }
+ break;
+ case RSPAMD_LOG_SETTINGS_ID:
+ if (task->settings_elt) {
+ var.begin = task->settings_elt->name;
+ var.len = strlen(task->settings_elt->name);
+ }
+ else {
+ var.begin = undef;
+ var.len = sizeof(undef) - 1;
+ }
+ break;
+ case RSPAMD_LOG_MEMPOOL_SIZE:
+ var.len = rspamd_snprintf(numbuf, sizeof(numbuf),
+ "%Hz",
+ rspamd_mempool_get_used_size(task->task_pool));
+ var.begin = numbuf;
+ break;
+ case RSPAMD_LOG_MEMPOOL_WASTE:
+ var.len = rspamd_snprintf(numbuf, sizeof(numbuf),
+ "%Hz",
+ rspamd_mempool_get_wasted_size(task->task_pool));
+ var.begin = numbuf;
+ break;
+ default:
+ var = rspamd_task_log_metric_res(task, lf);
+ break;
+ }
+
+ if (var.len > 0) {
+ res = rspamd_task_log_write_var(task, logbuf,
+ &var, (const rspamd_ftok_t *) lf->data);
+ }
+
+ return res;
+}
+
+void rspamd_task_write_log(struct rspamd_task *task)
+{
+ rspamd_fstring_t *logbuf;
+ struct rspamd_log_format *lf;
+ struct rspamd_task **ptask;
+ const gchar *lua_str;
+ gsize lua_str_len;
+ lua_State *L;
+
+ g_assert(task != NULL);
+
+ if (task->cfg->log_format == NULL ||
+ (task->flags & RSPAMD_TASK_FLAG_NO_LOG)) {
+ msg_debug_task("skip logging due to no log flag");
+ return;
+ }
+
+ logbuf = rspamd_fstring_sized_new(1000);
+
+ DL_FOREACH(task->cfg->log_format, lf)
+ {
+ switch (lf->type) {
+ case RSPAMD_LOG_STRING:
+ logbuf = rspamd_fstring_append(logbuf, lf->data, lf->len);
+ break;
+ case RSPAMD_LOG_LUA:
+ L = task->cfg->lua_state;
+ lua_rawgeti(L, LUA_REGISTRYINDEX, GPOINTER_TO_INT(lf->data));
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+
+ if (lua_pcall(L, 1, 1, 0) != 0) {
+ msg_err_task("call to log function failed: %s",
+ lua_tostring(L, -1));
+ lua_pop(L, 1);
+ }
+ else {
+ lua_str = lua_tolstring(L, -1, &lua_str_len);
+
+ if (lua_str != NULL) {
+ logbuf = rspamd_fstring_append(logbuf, lua_str, lua_str_len);
+ }
+ lua_pop(L, 1);
+ }
+ break;
+ default:
+ /* We have a variable in log format */
+ if (lf->flags & RSPAMD_LOG_FMT_FLAG_CONDITION) {
+ if (!rspamd_task_log_check_condition(task, lf)) {
+ continue;
+ }
+ }
+
+ logbuf = rspamd_task_log_variable(task, lf, logbuf);
+ break;
+ }
+ }
+
+ msg_notice_task("%V", logbuf);
+
+ rspamd_fstring_free(logbuf);
+}
+
+gdouble
+rspamd_task_get_required_score(struct rspamd_task *task, struct rspamd_scan_result *m)
+{
+ if (m == NULL) {
+ m = task->result;
+
+ if (m == NULL) {
+ return NAN;
+ }
+ }
+
+ for (guint i = m->nactions; i-- > 0;) {
+ struct rspamd_action_config *action_lim = &m->actions_config[i];
+
+
+ if (!isnan(action_lim->cur_limit) &&
+ !(action_lim->action->flags & (RSPAMD_ACTION_NO_THRESHOLD | RSPAMD_ACTION_HAM))) {
+ return m->actions_config[i].cur_limit;
+ }
+ }
+
+ return NAN;
+}
+
+rspamd_ftok_t *
+rspamd_task_get_request_header(struct rspamd_task *task,
+ const gchar *name)
+{
+ struct rspamd_request_header_chain *ret =
+ rspamd_task_get_request_header_multiple(task, name);
+
+ if (ret) {
+ return ret->hdr;
+ }
+
+ return NULL;
+}
+
+struct rspamd_request_header_chain *
+rspamd_task_get_request_header_multiple(struct rspamd_task *task,
+ const gchar *name)
+{
+ struct rspamd_request_header_chain *ret = NULL;
+ rspamd_ftok_t srch;
+ khiter_t k;
+
+ srch.begin = (gchar *) name;
+ srch.len = strlen(name);
+
+ k = kh_get(rspamd_req_headers_hash, task->request_headers,
+ &srch);
+
+ if (k != kh_end(task->request_headers)) {
+ ret = kh_value(task->request_headers, k);
+ }
+
+ return ret;
+}
+
+
+void rspamd_task_add_request_header(struct rspamd_task *task,
+ rspamd_ftok_t *name, rspamd_ftok_t *value)
+{
+
+ khiter_t k;
+ gint res;
+ struct rspamd_request_header_chain *chain, *nchain;
+
+ k = kh_put(rspamd_req_headers_hash, task->request_headers,
+ name, &res);
+
+ if (res == 0) {
+ /* Existing name */
+ nchain = rspamd_mempool_alloc(task->task_pool, sizeof(*nchain));
+ nchain->hdr = value;
+ nchain->next = NULL;
+ chain = kh_value(task->request_headers, k);
+
+ /* Slow but OK here */
+ LL_APPEND(chain, nchain);
+ }
+ else {
+ nchain = rspamd_mempool_alloc(task->task_pool, sizeof(*nchain));
+ nchain->hdr = value;
+ nchain->next = NULL;
+
+ kh_value(task->request_headers, k) = nchain;
+ }
+}
+
+
+void rspamd_task_profile_set(struct rspamd_task *task, const gchar *key,
+ gdouble value)
+{
+ GHashTable *tbl;
+ gdouble *pval;
+
+ if (key == NULL) {
+ return;
+ }
+
+ tbl = rspamd_mempool_get_variable(task->task_pool, RSPAMD_MEMPOOL_PROFILE);
+
+ if (tbl == NULL) {
+ tbl = g_hash_table_new(rspamd_str_hash, rspamd_str_equal);
+ rspamd_mempool_set_variable(task->task_pool, RSPAMD_MEMPOOL_PROFILE,
+ tbl, (rspamd_mempool_destruct_t) g_hash_table_unref);
+ }
+
+ pval = g_hash_table_lookup(tbl, key);
+
+ if (pval == NULL) {
+ pval = rspamd_mempool_alloc(task->task_pool, sizeof(*pval));
+ *pval = value;
+ g_hash_table_insert(tbl, (void *) key, pval);
+ }
+ else {
+ *pval = value;
+ }
+}
+
+gdouble *
+rspamd_task_profile_get(struct rspamd_task *task, const gchar *key)
+{
+ GHashTable *tbl;
+ gdouble *pval = NULL;
+
+ tbl = rspamd_mempool_get_variable(task->task_pool, RSPAMD_MEMPOOL_PROFILE);
+
+ if (tbl != NULL) {
+ pval = g_hash_table_lookup(tbl, key);
+ }
+
+ return pval;
+}
+
+
+gboolean
+rspamd_task_set_finish_time(struct rspamd_task *task)
+{
+ if (isnan(task->time_real_finish)) {
+ task->time_real_finish = ev_time();
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+const gchar *
+rspamd_task_stage_name(enum rspamd_task_stage stg)
+{
+ const gchar *ret = "unknown stage";
+
+ switch (stg) {
+ case RSPAMD_TASK_STAGE_CONNECT:
+ ret = "connect";
+ break;
+ case RSPAMD_TASK_STAGE_CONNFILTERS:
+ ret = "connection_filter";
+ break;
+ case RSPAMD_TASK_STAGE_READ_MESSAGE:
+ ret = "read_message";
+ break;
+ case RSPAMD_TASK_STAGE_PRE_FILTERS:
+ ret = "prefilters";
+ break;
+ case RSPAMD_TASK_STAGE_PROCESS_MESSAGE:
+ ret = "process_message";
+ break;
+ case RSPAMD_TASK_STAGE_FILTERS:
+ ret = "filters";
+ break;
+ case RSPAMD_TASK_STAGE_CLASSIFIERS_PRE:
+ ret = "classifiers_pre";
+ break;
+ case RSPAMD_TASK_STAGE_CLASSIFIERS:
+ ret = "classifiers";
+ break;
+ case RSPAMD_TASK_STAGE_CLASSIFIERS_POST:
+ ret = "classifiers_post";
+ break;
+ case RSPAMD_TASK_STAGE_COMPOSITES:
+ ret = "composites";
+ break;
+ case RSPAMD_TASK_STAGE_POST_FILTERS:
+ ret = "postfilters";
+ break;
+ case RSPAMD_TASK_STAGE_LEARN_PRE:
+ ret = "learn_pre";
+ break;
+ case RSPAMD_TASK_STAGE_LEARN:
+ ret = "learn";
+ break;
+ case RSPAMD_TASK_STAGE_LEARN_POST:
+ ret = "learn_post";
+ break;
+ case RSPAMD_TASK_STAGE_COMPOSITES_POST:
+ ret = "composites_post";
+ break;
+ case RSPAMD_TASK_STAGE_IDEMPOTENT:
+ ret = "idempotent";
+ break;
+ case RSPAMD_TASK_STAGE_DONE:
+ ret = "done";
+ break;
+ case RSPAMD_TASK_STAGE_REPLIED:
+ ret = "replied";
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+void rspamd_task_timeout(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_task *task = (struct rspamd_task *) w->data;
+
+ if (!(task->processed_stages & RSPAMD_TASK_STAGE_FILTERS)) {
+ ev_now_update_if_cheap(task->event_loop);
+ msg_info_task("processing of task time out: %.1fs spent; %.1fs limit; "
+ "forced processing",
+ ev_now(task->event_loop) - task->task_timestamp,
+ w->repeat);
+
+ if (task->cfg->soft_reject_on_timeout) {
+ struct rspamd_action *action, *soft_reject;
+
+ action = rspamd_check_action_metric(task, NULL, NULL);
+
+ if (action->action_type != METRIC_ACTION_REJECT) {
+ soft_reject = rspamd_config_get_action_by_type(task->cfg,
+ METRIC_ACTION_SOFT_REJECT);
+ rspamd_add_passthrough_result(task,
+ soft_reject,
+ 0,
+ NAN,
+ "timeout processing message",
+ "task timeout",
+ 0, NULL);
+ }
+ }
+
+ ev_timer_again(EV_A_ w);
+ task->processed_stages |= RSPAMD_TASK_STAGE_FILTERS;
+ rspamd_session_cleanup(task->s, true);
+ rspamd_task_process(task, RSPAMD_TASK_PROCESS_ALL);
+ rspamd_session_pending(task->s);
+ }
+ else {
+ /* Postprocessing timeout */
+ msg_info_task("post-processing of task time out: %.1f second spent; forced processing",
+ ev_now(task->event_loop) - task->task_timestamp);
+
+ if (task->cfg->soft_reject_on_timeout) {
+ struct rspamd_action *action, *soft_reject;
+
+ action = rspamd_check_action_metric(task, NULL, NULL);
+
+ if (action->action_type != METRIC_ACTION_REJECT) {
+ soft_reject = rspamd_config_get_action_by_type(task->cfg,
+ METRIC_ACTION_SOFT_REJECT);
+ rspamd_add_passthrough_result(task,
+ soft_reject,
+ 0,
+ NAN,
+ "timeout post-processing message",
+ "task timeout",
+ 0, NULL);
+ }
+ }
+
+ ev_timer_stop(EV_A_ w);
+ task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
+ rspamd_session_cleanup(task->s, true);
+ rspamd_task_process(task, RSPAMD_TASK_PROCESS_ALL);
+ rspamd_session_pending(task->s);
+ }
+}
+
+void rspamd_worker_guard_handler(EV_P_ ev_io *w, int revents)
+{
+ struct rspamd_task *task = (struct rspamd_task *) w->data;
+ gchar fake_buf[1024];
+ gssize r;
+
+ r = read(w->fd, fake_buf, sizeof(fake_buf));
+
+ if (r > 0) {
+ msg_warn_task("received extra data after task is loaded, ignoring");
+ }
+ else {
+ if (r == 0) {
+ /*
+ * Poor man approach, that might break things in case of
+ * shutdown (SHUT_WR) but sockets are so bad that there's no
+ * reliable way to distinguish between shutdown(SHUT_WR) and
+ * close.
+ */
+ if (task->cmd != CMD_CHECK_V2 && task->cfg->enable_shutdown_workaround) {
+ msg_info_task("workaround for shutdown enabled, please update "
+ "your client, this support might be removed in future");
+ shutdown(w->fd, SHUT_RD);
+ ev_io_stop(task->event_loop, &task->guard_ev);
+ }
+ else {
+ msg_err_task("the peer has closed connection unexpectedly");
+ rspamd_session_destroy(task->s);
+ }
+ }
+ else if (errno != EAGAIN) {
+ msg_err_task("the peer has closed connection unexpectedly: %s",
+ strerror(errno));
+ rspamd_session_destroy(task->s);
+ }
+ else {
+ return;
+ }
+ }
+}
diff --git a/src/libserver/task.h b/src/libserver/task.h
new file mode 100644
index 0000000..5404a11
--- /dev/null
+++ b/src/libserver/task.h
@@ -0,0 +1,392 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef TASK_H_
+#define TASK_H_
+
+#include "config.h"
+#include "libserver/http/http_connection.h"
+#include "async_session.h"
+#include "util.h"
+#include "mem_pool.h"
+#include "dns.h"
+#include "re_cache.h"
+#include "khash.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum rspamd_command {
+ CMD_SKIP = 0,
+ CMD_PING,
+ CMD_CHECK_SPAMC, /* Legacy spamassassin format */
+ CMD_CHECK_RSPAMC, /* Legacy rspamc format (like SA one) */
+ CMD_CHECK, /* Legacy check - metric json reply */
+ CMD_CHECK_V2, /* Modern check - symbols in json reply */
+};
+
+enum rspamd_task_stage {
+ RSPAMD_TASK_STAGE_CONNECT = (1u << 0u),
+ RSPAMD_TASK_STAGE_CONNFILTERS = (1u << 1u),
+ RSPAMD_TASK_STAGE_READ_MESSAGE = (1u << 2u),
+ RSPAMD_TASK_STAGE_PROCESS_MESSAGE = (1u << 3u),
+ RSPAMD_TASK_STAGE_PRE_FILTERS = (1u << 4u),
+ RSPAMD_TASK_STAGE_FILTERS = (1u << 5u),
+ RSPAMD_TASK_STAGE_CLASSIFIERS_PRE = (1u << 6u),
+ RSPAMD_TASK_STAGE_CLASSIFIERS = (1u << 7u),
+ RSPAMD_TASK_STAGE_CLASSIFIERS_POST = (1u << 8u),
+ RSPAMD_TASK_STAGE_COMPOSITES = (1u << 9u),
+ RSPAMD_TASK_STAGE_POST_FILTERS = (1u << 10u),
+ RSPAMD_TASK_STAGE_LEARN_PRE = (1u << 11u),
+ RSPAMD_TASK_STAGE_LEARN = (1u << 12u),
+ RSPAMD_TASK_STAGE_LEARN_POST = (1u << 13u),
+ RSPAMD_TASK_STAGE_COMPOSITES_POST = (1u << 14u),
+ RSPAMD_TASK_STAGE_IDEMPOTENT = (1u << 15u),
+ RSPAMD_TASK_STAGE_DONE = (1u << 16u),
+ RSPAMD_TASK_STAGE_REPLIED = (1u << 17u)
+};
+
+#define RSPAMD_TASK_PROCESS_ALL (RSPAMD_TASK_STAGE_CONNECT | \
+ RSPAMD_TASK_STAGE_CONNFILTERS | \
+ RSPAMD_TASK_STAGE_READ_MESSAGE | \
+ RSPAMD_TASK_STAGE_PRE_FILTERS | \
+ RSPAMD_TASK_STAGE_PROCESS_MESSAGE | \
+ RSPAMD_TASK_STAGE_FILTERS | \
+ RSPAMD_TASK_STAGE_CLASSIFIERS_PRE | \
+ RSPAMD_TASK_STAGE_CLASSIFIERS | \
+ RSPAMD_TASK_STAGE_CLASSIFIERS_POST | \
+ RSPAMD_TASK_STAGE_COMPOSITES | \
+ RSPAMD_TASK_STAGE_POST_FILTERS | \
+ RSPAMD_TASK_STAGE_LEARN_PRE | \
+ RSPAMD_TASK_STAGE_LEARN | \
+ RSPAMD_TASK_STAGE_LEARN_POST | \
+ RSPAMD_TASK_STAGE_COMPOSITES_POST | \
+ RSPAMD_TASK_STAGE_IDEMPOTENT | \
+ RSPAMD_TASK_STAGE_DONE)
+#define RSPAMD_TASK_PROCESS_LEARN (RSPAMD_TASK_STAGE_CONNECT | \
+ RSPAMD_TASK_STAGE_READ_MESSAGE | \
+ RSPAMD_TASK_STAGE_PROCESS_MESSAGE | \
+ RSPAMD_TASK_STAGE_CLASSIFIERS_PRE | \
+ RSPAMD_TASK_STAGE_CLASSIFIERS | \
+ RSPAMD_TASK_STAGE_CLASSIFIERS_POST | \
+ RSPAMD_TASK_STAGE_LEARN_PRE | \
+ RSPAMD_TASK_STAGE_LEARN | \
+ RSPAMD_TASK_STAGE_LEARN_POST | \
+ RSPAMD_TASK_STAGE_DONE)
+
+#define RSPAMD_TASK_FLAG_MIME (1u << 0u)
+#define RSPAMD_TASK_FLAG_SKIP_PROCESS (1u << 1u)
+#define RSPAMD_TASK_FLAG_SKIP (1u << 2u)
+#define RSPAMD_TASK_FLAG_PASS_ALL (1u << 3u)
+#define RSPAMD_TASK_FLAG_NO_LOG (1u << 4u)
+#define RSPAMD_TASK_FLAG_NO_IP (1u << 5u)
+#define RSPAMD_TASK_FLAG_PROCESSING (1u << 6u)
+#define RSPAMD_TASK_FLAG_GTUBE (1u << 7u)
+#define RSPAMD_TASK_FLAG_FILE (1u << 8u)
+#define RSPAMD_TASK_FLAG_NO_STAT (1u << 9u)
+#define RSPAMD_TASK_FLAG_UNLEARN (1u << 10u)
+#define RSPAMD_TASK_FLAG_ALREADY_LEARNED (1u << 11u)
+#define RSPAMD_TASK_FLAG_LEARN_SPAM (1u << 12u)
+#define RSPAMD_TASK_FLAG_LEARN_HAM (1u << 13u)
+#define RSPAMD_TASK_FLAG_LEARN_AUTO (1u << 14u)
+#define RSPAMD_TASK_FLAG_BROKEN_HEADERS (1u << 15u)
+#define RSPAMD_TASK_FLAG_HAS_SPAM_TOKENS (1u << 16u)
+#define RSPAMD_TASK_FLAG_HAS_HAM_TOKENS (1u << 17u)
+#define RSPAMD_TASK_FLAG_EMPTY (1u << 18u)
+#define RSPAMD_TASK_FLAG_PROFILE (1u << 19u)
+#define RSPAMD_TASK_FLAG_GREYLISTED (1u << 20u)
+#define RSPAMD_TASK_FLAG_OWN_POOL (1u << 21u)
+#define RSPAMD_TASK_FLAG_SSL (1u << 22u)
+#define RSPAMD_TASK_FLAG_BAD_UNICODE (1u << 23u)
+#define RSPAMD_TASK_FLAG_MESSAGE_REWRITE (1u << 24u)
+#define RSPAMD_TASK_FLAG_MAX_SHIFT (24u)
+
+
+/* Request has a JSON control block */
+#define RSPAMD_TASK_PROTOCOL_FLAG_HAS_CONTROL (1u << 0u)
+/* Request has been done by a local client */
+#define RSPAMD_TASK_PROTOCOL_FLAG_LOCAL_CLIENT (1u << 1u)
+/* Request has been sent via milter */
+#define RSPAMD_TASK_PROTOCOL_FLAG_MILTER (1u << 2u)
+/* Compress protocol reply */
+#define RSPAMD_TASK_PROTOCOL_FLAG_COMPRESSED (1u << 3u)
+/* Include all URLs */
+#define RSPAMD_TASK_PROTOCOL_FLAG_EXT_URLS (1u << 4u)
+/* Client allows body block (including headers in no FLAG_MILTER) */
+#define RSPAMD_TASK_PROTOCOL_FLAG_BODY_BLOCK (1u << 5u)
+/* Emit groups information */
+#define RSPAMD_TASK_PROTOCOL_FLAG_GROUPS (1u << 6u)
+#define RSPAMD_TASK_PROTOCOL_FLAG_MAX_SHIFT (6u)
+
+#define RSPAMD_TASK_IS_SKIPPED(task) (G_UNLIKELY((task)->flags & RSPAMD_TASK_FLAG_SKIP))
+#define RSPAMD_TASK_IS_SPAMC(task) (G_UNLIKELY((task)->cmd == CMD_CHECK_SPAMC))
+#define RSPAMD_TASK_IS_PROCESSED(task) (G_UNLIKELY((task)->processed_stages & RSPAMD_TASK_STAGE_DONE))
+#define RSPAMD_TASK_IS_CLASSIFIED(task) (((task)->processed_stages & RSPAMD_TASK_STAGE_CLASSIFIERS))
+#define RSPAMD_TASK_IS_EMPTY(task) (G_UNLIKELY((task)->flags & RSPAMD_TASK_FLAG_EMPTY))
+#define RSPAMD_TASK_IS_PROFILING(task) (G_UNLIKELY((task)->flags & RSPAMD_TASK_FLAG_PROFILE))
+#define RSPAMD_TASK_IS_MIME(task) (G_LIKELY((task)->flags & RSPAMD_TASK_FLAG_MIME))
+
+struct rspamd_email_address;
+struct rspamd_lang_detector;
+enum rspamd_newlines_type;
+struct rspamd_message;
+
+struct rspamd_task_data_storage {
+ const gchar *begin;
+ gsize len;
+ gchar *fpath;
+};
+
+struct rspamd_request_header_chain {
+ rspamd_ftok_t *hdr;
+ struct rspamd_request_header_chain *next;
+};
+
+__KHASH_TYPE(rspamd_req_headers_hash, rspamd_ftok_t *, struct rspamd_request_header_chain *);
+
+struct rspamd_lua_cached_entry {
+ gint ref;
+ guint id;
+};
+
+KHASH_INIT(rspamd_task_lua_cache, char *, struct rspamd_lua_cached_entry, 1, kh_str_hash_func, kh_str_hash_equal);
+
+/**
+ * Worker task structure
+ */
+struct rspamd_task {
+ struct rspamd_worker *worker; /**< pointer to worker object */
+ enum rspamd_command cmd; /**< command */
+ gint sock; /**< socket descriptor */
+ guint32 dns_requests; /**< number of DNS requests per this task */
+ guint32 flags; /**< Bit flags */
+ guint32 protocol_flags;
+ guint32 processed_stages; /**< bits of stages that are processed */
+ gchar *helo; /**< helo header value */
+ gchar *queue_id; /**< queue id if specified */
+ rspamd_inet_addr_t *from_addr; /**< from addr for a task */
+ rspamd_inet_addr_t *client_addr; /**< address of connected socket */
+ gchar *deliver_to; /**< address to deliver */
+ gchar *auth_user; /**< SMTP authenticated user */
+ const gchar *hostname; /**< hostname reported by MTA */
+ khash_t(rspamd_req_headers_hash) * request_headers; /**< HTTP headers in a request */
+ struct rspamd_task_data_storage msg; /**< message buffer */
+ struct rspamd_http_connection *http_conn; /**< HTTP server connection */
+ struct rspamd_async_session *s; /**< async session object */
+ struct rspamd_scan_result *result; /**< Metric result */
+ khash_t(rspamd_task_lua_cache) lua_cache; /**< cache of lua objects */
+ GPtrArray *tokens; /**< statistics tokens */
+ GArray *meta_words; /**< rspamd_stat_token_t produced from meta headers
+ (e.g. Subject) */
+
+ GPtrArray *rcpt_envelope; /**< array of rspamd_email_address */
+ struct rspamd_email_address *from_envelope;
+ struct rspamd_email_address *from_envelope_orig;
+
+ ucl_object_t *messages; /**< list of messages that would be reported */
+ struct rspamd_re_runtime *re_rt; /**< regexp runtime */
+ GPtrArray *stat_runtimes; /**< backend runtime */
+ struct rspamd_config *cfg; /**< pointer to config object */
+ GError *err;
+ rspamd_mempool_t *task_pool; /**< memory pool for task */
+ double time_real_finish;
+ ev_tstamp task_timestamp;
+
+ gboolean (*fin_callback)(struct rspamd_task *task, void *arg);
+ /**< callback for filters finalizing */
+ void *fin_arg; /**< argument for fin callback */
+
+ struct rspamd_dns_resolver *resolver; /**< DNS resolver */
+ struct ev_loop *event_loop; /**< Event base */
+ struct ev_timer timeout_ev; /**< Global task timeout */
+ struct ev_io guard_ev; /**< Event for input sanity guard */
+
+ gpointer symcache_runtime; /**< Opaque checkpoint data */
+ ucl_object_t *settings; /**< Settings applied to task */
+ struct rspamd_config_settings_elt *settings_elt; /**< preprocessed settings id elt */
+
+ const gchar *classifier; /**< Classifier to learn (if needed) */
+ struct rspamd_lang_detector *lang_det; /**< Languages detector */
+ struct rspamd_message *message;
+};
+
+/**
+ * Construct new task for worker
+ */
+struct rspamd_task *rspamd_task_new(struct rspamd_worker *worker,
+ struct rspamd_config *cfg,
+ rspamd_mempool_t *pool,
+ struct rspamd_lang_detector *lang_det,
+ struct ev_loop *event_loop,
+ gboolean debug_mem);
+
+/**
+ * Destroy task object and remove its IO dispatcher if it exists
+ */
+void rspamd_task_free(struct rspamd_task *task);
+
+/**
+ * Called if all filters are processed
+ * @return TRUE if session should be terminated
+ */
+gboolean rspamd_task_fin(void *arg);
+
+/**
+ * Load HTTP message with body in `msg` to an rspamd_task
+ * @param task
+ * @param msg
+ * @param start
+ * @param len
+ * @return
+ */
+gboolean rspamd_task_load_message(struct rspamd_task *task,
+ struct rspamd_http_message *msg,
+ const gchar *start, gsize len);
+
+/**
+ * Process task
+ * @param task task to process
+ * @return task has been successfully parsed and processed
+ */
+gboolean rspamd_task_process(struct rspamd_task *task, guint stages);
+
+/**
+ * Return address of sender or NULL
+ * @param task
+ * @return
+ */
+struct rspamd_email_address *rspamd_task_get_sender(struct rspamd_task *task);
+
+/**
+ * Return addresses in the following precedence:
+ * - deliver to
+ * - the first smtp recipient
+ * - the first mime recipient
+ * @param task
+ * @return
+ */
+const gchar *rspamd_task_get_principal_recipient(struct rspamd_task *task);
+
+/**
+ * Add a recipient for a task
+ * @param task task object
+ * @param rcpt string representation of recipient address
+ * @return TRUE if an address has been parsed and added
+ */
+gboolean rspamd_task_add_recipient(struct rspamd_task *task, const gchar *rcpt);
+
+/**
+ * Learn specified statfile with message in a task
+ * @param task worker's task object
+ * @param classifier classifier to learn (or NULL to learn all)
+ * @param err pointer to GError
+ * @return true if learn succeed
+ */
+gboolean rspamd_learn_task_spam(struct rspamd_task *task,
+ gboolean is_spam,
+ const gchar *classifier,
+ GError **err);
+
+/**
+ * Returns required score for a message (usually reject score)
+ * @param task
+ * @param m
+ * @return
+ */
+struct rspamd_scan_result;
+
+gdouble rspamd_task_get_required_score(struct rspamd_task *task,
+ struct rspamd_scan_result *m);
+
+/**
+ * Returns the first header as value for a header
+ * @param task
+ * @param name
+ * @return
+ */
+rspamd_ftok_t *rspamd_task_get_request_header(struct rspamd_task *task,
+ const gchar *name);
+
+/**
+ * Returns all headers with the specific name
+ * @param task
+ * @param name
+ * @return
+ */
+struct rspamd_request_header_chain *rspamd_task_get_request_header_multiple(
+ struct rspamd_task *task,
+ const gchar *name);
+
+/**
+ * Adds a new request header to task (name and value should be mapped to fstring)
+ * @param task
+ * @param name
+ * @param value
+ */
+void rspamd_task_add_request_header(struct rspamd_task *task,
+ rspamd_ftok_t *name, rspamd_ftok_t *value);
+
+/**
+ * Write log line about the specified task if needed
+ */
+void rspamd_task_write_log(struct rspamd_task *task);
+
+/**
+ * Set profiling value for a specific key
+ * @param task
+ * @param key
+ * @param value
+ */
+void rspamd_task_profile_set(struct rspamd_task *task, const gchar *key,
+ gdouble value);
+
+/**
+ * Get value for a specific profiling key
+ * @param task
+ * @param key
+ * @return
+ */
+gdouble *rspamd_task_profile_get(struct rspamd_task *task, const gchar *key);
+
+/**
+ * Sets finishing time for a task if not yet set
+ * @param task
+ * @return
+ */
+gboolean rspamd_task_set_finish_time(struct rspamd_task *task);
+
+/**
+ * Returns task processing stage name
+ * @param stg
+ * @return
+ */
+const gchar *rspamd_task_stage_name(enum rspamd_task_stage stg);
+
+/*
+ * Called on forced timeout
+ */
+void rspamd_task_timeout(EV_P_ ev_timer *w, int revents);
+
+/*
+ * Called on unexpected IO error (e.g. ECONNRESET)
+ */
+void rspamd_worker_guard_handler(EV_P_ ev_io *w, int revents);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* TASK_H_ */
diff --git a/src/libserver/url.c b/src/libserver/url.c
new file mode 100644
index 0000000..0842a1e
--- /dev/null
+++ b/src/libserver/url.c
@@ -0,0 +1,4365 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "url.h"
+#include "util.h"
+#include "rspamd.h"
+#include "message.h"
+#include "multipattern.h"
+#include "contrib/uthash/utlist.h"
+#include "contrib/http-parser/http_parser.h"
+#include <unicode/utf8.h>
+#include <unicode/uchar.h>
+#include <unicode/usprep.h>
+#include <unicode/ucnv.h>
+
+typedef struct url_match_s {
+ const gchar *m_begin;
+ gsize m_len;
+ const gchar *pattern;
+ const gchar *prefix;
+ const gchar *newline_pos;
+ const gchar *prev_newline_pos;
+ gboolean add_prefix;
+ gchar st;
+} url_match_t;
+
+#define URL_MATCHER_FLAG_NOHTML (1u << 0u)
+#define URL_MATCHER_FLAG_TLD_MATCH (1u << 1u)
+#define URL_MATCHER_FLAG_STAR_MATCH (1u << 2u)
+#define URL_MATCHER_FLAG_REGEXP (1u << 3u)
+
+struct url_callback_data;
+
+static const struct {
+ enum rspamd_url_protocol proto;
+ const gchar *name;
+ gsize len;
+} rspamd_url_protocols[] = {
+ {.proto = PROTOCOL_FILE,
+ .name = "file",
+ .len = 4},
+ {.proto = PROTOCOL_FTP,
+ .name = "ftp",
+ .len = 3},
+ {.proto = PROTOCOL_HTTP,
+ .name = "http",
+ .len = 4},
+ {.proto = PROTOCOL_HTTPS,
+ .name = "https",
+ .len = 5},
+ {.proto = PROTOCOL_MAILTO,
+ .name = "mailto",
+ .len = 6},
+ {.proto = PROTOCOL_TELEPHONE,
+ .name = "tel",
+ .len = 3},
+ {.proto = PROTOCOL_TELEPHONE,
+ .name = "callto",
+ .len = 3},
+ {.proto = PROTOCOL_UNKNOWN,
+ .name = NULL,
+ .len = 0}};
+struct url_matcher {
+ const gchar *pattern;
+ const gchar *prefix;
+
+ gboolean (*start)(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+ gboolean (*end)(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+ gint flags;
+};
+
+static gboolean url_file_start(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+static gboolean url_file_end(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+static gboolean url_web_start(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+static gboolean url_web_end(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+static gboolean url_tld_start(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+static gboolean url_tld_end(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+static gboolean url_email_start(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+static gboolean url_email_end(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+static gboolean url_tel_start(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+static gboolean url_tel_end(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match);
+
+struct url_matcher static_matchers[] = {
+ /* Common prefixes */
+ {"file://", "", url_file_start, url_file_end,
+ 0},
+ {"file:\\\\", "", url_file_start, url_file_end,
+ 0},
+ {"ftp://", "", url_web_start, url_web_end,
+ 0},
+ {"ftp:\\\\", "", url_web_start, url_web_end,
+ 0},
+ {"sftp://", "", url_web_start, url_web_end,
+ 0},
+ {"http:", "", url_web_start, url_web_end,
+ 0},
+ {"https:", "", url_web_start, url_web_end,
+ 0},
+ {"news://", "", url_web_start, url_web_end,
+ 0},
+ {"nntp://", "", url_web_start, url_web_end,
+ 0},
+ {"telnet://", "", url_web_start, url_web_end,
+ 0},
+ {"tel:", "", url_tel_start, url_tel_end,
+ 0},
+ {"webcal://", "", url_web_start, url_web_end,
+ 0},
+ {"mailto:", "", url_email_start, url_email_end,
+ 0},
+ {"callto:", "", url_tel_start, url_tel_end,
+ 0},
+ {"h323:", "", url_web_start, url_web_end,
+ 0},
+ {"sip:", "", url_web_start, url_web_end,
+ 0},
+ {"www\\.[0-9a-z]", "http://", url_web_start, url_web_end,
+ URL_MATCHER_FLAG_REGEXP},
+ {"ftp.", "ftp://", url_web_start, url_web_end,
+ 0},
+ /* Likely emails */
+ {
+ "@", "mailto://", url_email_start, url_email_end,
+ 0}};
+
+struct rspamd_url_flag_name {
+ const gchar *name;
+ gint flag;
+ gint hash;
+} url_flag_names[] = {
+ {"phished", RSPAMD_URL_FLAG_PHISHED, -1},
+ {"numeric", RSPAMD_URL_FLAG_NUMERIC, -1},
+ {"obscured", RSPAMD_URL_FLAG_OBSCURED, -1},
+ {"redirected", RSPAMD_URL_FLAG_REDIRECTED, -1},
+ {"html_displayed", RSPAMD_URL_FLAG_HTML_DISPLAYED, -1},
+ {"text", RSPAMD_URL_FLAG_FROM_TEXT, -1},
+ {"subject", RSPAMD_URL_FLAG_SUBJECT, -1},
+ {"host_encoded", RSPAMD_URL_FLAG_HOSTENCODED, -1},
+ {"schema_encoded", RSPAMD_URL_FLAG_SCHEMAENCODED, -1},
+ {"path_encoded", RSPAMD_URL_FLAG_PATHENCODED, -1},
+ {"query_encoded", RSPAMD_URL_FLAG_QUERYENCODED, -1},
+ {"missing_slashes", RSPAMD_URL_FLAG_MISSINGSLASHES, -1},
+ {"idn", RSPAMD_URL_FLAG_IDN, -1},
+ {"has_port", RSPAMD_URL_FLAG_HAS_PORT, -1},
+ {"has_user", RSPAMD_URL_FLAG_HAS_USER, -1},
+ {"schemaless", RSPAMD_URL_FLAG_SCHEMALESS, -1},
+ {"unnormalised", RSPAMD_URL_FLAG_UNNORMALISED, -1},
+ {"zw_spaces", RSPAMD_URL_FLAG_ZW_SPACES, -1},
+ {"url_displayed", RSPAMD_URL_FLAG_DISPLAY_URL, -1},
+ {"image", RSPAMD_URL_FLAG_IMAGE, -1},
+ {"query", RSPAMD_URL_FLAG_QUERY, -1},
+ {"content", RSPAMD_URL_FLAG_CONTENT, -1},
+ {"no_tld", RSPAMD_URL_FLAG_NO_TLD, -1},
+ {"truncated", RSPAMD_URL_FLAG_TRUNCATED, -1},
+ {"redirect_target", RSPAMD_URL_FLAG_REDIRECT_TARGET, -1},
+ {"invisible", RSPAMD_URL_FLAG_INVISIBLE, -1},
+ {"special", RSPAMD_URL_FLAG_SPECIAL, -1},
+};
+
+
+static inline khint_t rspamd_url_hash(struct rspamd_url *u);
+
+static inline khint_t rspamd_url_host_hash(struct rspamd_url *u);
+static inline bool rspamd_urls_cmp(struct rspamd_url *a, struct rspamd_url *b);
+static inline bool rspamd_urls_host_cmp(struct rspamd_url *a, struct rspamd_url *b);
+
+/* Hash table implementation */
+__KHASH_IMPL(rspamd_url_hash, kh_inline, struct rspamd_url *, char, false,
+ rspamd_url_hash, rspamd_urls_cmp);
+__KHASH_IMPL(rspamd_url_host_hash, kh_inline, struct rspamd_url *, char, false,
+ rspamd_url_host_hash, rspamd_urls_host_cmp);
+
+struct url_callback_data {
+ const gchar *begin;
+ gchar *url_str;
+ rspamd_mempool_t *pool;
+ gint len;
+ enum rspamd_url_find_type how;
+ gboolean prefix_added;
+ guint newline_idx;
+ GArray *matchers;
+ GPtrArray *newlines;
+ const gchar *start;
+ const gchar *fin;
+ const gchar *end;
+ const gchar *last_at;
+ url_insert_function func;
+ void *funcd;
+};
+
+struct url_match_scanner {
+ GArray *matchers_full;
+ GArray *matchers_strict;
+ struct rspamd_multipattern *search_trie_full;
+ struct rspamd_multipattern *search_trie_strict;
+ bool has_tld_file;
+};
+
+struct url_match_scanner *url_scanner = NULL;
+
+enum {
+ IS_LWSP = (1 << 0),
+ IS_DOMAIN = (1 << 1),
+ IS_URLSAFE = (1 << 2),
+ IS_MAILSAFE = (1 << 3),
+ IS_DOMAIN_END = (1 << 4)
+};
+
+static const unsigned int url_scanner_table[256] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, IS_LWSP, IS_LWSP, IS_LWSP, IS_LWSP, IS_LWSP, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, IS_LWSP /* */,
+ IS_MAILSAFE /* ! */, IS_URLSAFE | IS_DOMAIN_END | IS_MAILSAFE /* " */,
+ IS_MAILSAFE /* # */, IS_MAILSAFE /* $ */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* % */, 0 /* & */, IS_MAILSAFE /* ' */,
+ 0 /* ( */, 0 /* ) */, IS_MAILSAFE /* * */,
+ IS_MAILSAFE /* + */, IS_MAILSAFE /* , */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* - */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* . */, IS_DOMAIN_END | IS_MAILSAFE /* / */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 0 */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 1 */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 2 */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 3 */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 4 */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 5 */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 6 */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 7 */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 8 */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 9 */, IS_DOMAIN_END /* : */,
+ 0 /* ; */, IS_URLSAFE | IS_DOMAIN_END /* < */, 0 /* = */,
+ IS_URLSAFE | IS_DOMAIN_END /* > */, IS_DOMAIN_END /* ? */, 0 /* @ */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* A */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* B */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* C */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* D */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* E */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* F */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* G */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* H */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* I */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* J */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* K */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* L */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* M */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* N */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* O */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* P */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* Q */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* R */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* S */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* T */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* U */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* V */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* W */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* X */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* Y */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* Z */, 0 /* [ */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* \ */, 0 /* ] */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* ^ */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* _ */,
+ IS_URLSAFE | IS_DOMAIN_END /* ` */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* a */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* b */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* c */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* d */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* e */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* f */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* g */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* h */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* i */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* j */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* k */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* l */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* m */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* n */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* o */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* p */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* q */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* r */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* s */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* t */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* u */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* v */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* w */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* x */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* y */,
+ IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* z */,
+ IS_URLSAFE | IS_DOMAIN_END | IS_MAILSAFE /* { */,
+ IS_URLSAFE | IS_DOMAIN_END | IS_MAILSAFE /* | */,
+ IS_URLSAFE | IS_DOMAIN_END | IS_MAILSAFE /* } */,
+ IS_URLSAFE | IS_DOMAIN_END | IS_MAILSAFE /* ~ */, 0, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+ IS_URLSAFE | IS_DOMAIN};
+
+#define is_lwsp(x) ((url_scanner_table[(guchar) (x)] & IS_LWSP) != 0)
+#define is_mailsafe(x) ((url_scanner_table[(guchar) (x)] & (IS_MAILSAFE)) != 0)
+#define is_domain(x) ((url_scanner_table[(guchar) (x)] & IS_DOMAIN) != 0)
+#define is_urlsafe(x) ((url_scanner_table[(guchar) (x)] & (IS_URLSAFE)) != 0)
+
+const gchar *
+rspamd_url_strerror(int err)
+{
+ switch (err) {
+ case URI_ERRNO_OK:
+ return "Parsing went well";
+ case URI_ERRNO_EMPTY:
+ return "The URI string was empty";
+ case URI_ERRNO_INVALID_PROTOCOL:
+ return "No protocol was found";
+ case URI_ERRNO_BAD_FORMAT:
+ return "Bad URL format";
+ case URI_ERRNO_BAD_ENCODING:
+ return "Invalid symbols encoded";
+ case URI_ERRNO_INVALID_PORT:
+ return "Port number is bad";
+ case URI_ERRNO_TLD_MISSING:
+ return "TLD part is not detected";
+ case URI_ERRNO_HOST_MISSING:
+ return "Host part is missing";
+ case URI_ERRNO_TOO_LONG:
+ return "URL is too long";
+ }
+
+ return NULL;
+}
+
+static gboolean
+rspamd_url_parse_tld_file(const gchar *fname,
+ struct url_match_scanner *scanner)
+{
+ FILE *f;
+ struct url_matcher m;
+ gchar *linebuf = NULL, *p;
+ gsize buflen = 0;
+ gssize r;
+ gint flags;
+
+ f = fopen(fname, "r");
+
+ if (f == NULL) {
+ msg_err("cannot open TLD file %s: %s", fname, strerror(errno));
+ return FALSE;
+ }
+
+ m.end = url_tld_end;
+ m.start = url_tld_start;
+ m.prefix = "http://";
+
+ while ((r = getline(&linebuf, &buflen, f)) > 0) {
+ if (linebuf[0] == '/' || g_ascii_isspace(linebuf[0])) {
+ /* Skip comment or empty line */
+ continue;
+ }
+
+ g_strchomp(linebuf);
+
+ /* TODO: add support for ! patterns */
+ if (linebuf[0] == '!') {
+ msg_debug("skip '!' patterns from parsing for now: %s", linebuf);
+ continue;
+ }
+
+ flags = URL_MATCHER_FLAG_NOHTML | URL_MATCHER_FLAG_TLD_MATCH;
+
+ if (linebuf[0] == '*') {
+ flags |= URL_MATCHER_FLAG_STAR_MATCH;
+ p = strchr(linebuf, '.');
+
+ if (p == NULL) {
+ msg_err("got bad star line, skip it: %s", linebuf);
+ continue;
+ }
+ p++;
+ }
+ else {
+ p = linebuf;
+ }
+
+ m.flags = flags;
+ rspamd_multipattern_add_pattern(url_scanner->search_trie_full, p,
+ RSPAMD_MULTIPATTERN_TLD | RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+ m.pattern = rspamd_multipattern_get_pattern(url_scanner->search_trie_full,
+ rspamd_multipattern_get_npatterns(url_scanner->search_trie_full) - 1);
+
+ g_array_append_val(url_scanner->matchers_full, m);
+ }
+
+ free(linebuf);
+ fclose(f);
+
+ return TRUE;
+}
+
+static void
+rspamd_url_add_static_matchers(struct url_match_scanner *sc)
+{
+ gint n = G_N_ELEMENTS(static_matchers), i;
+
+ for (i = 0; i < n; i++) {
+ if (static_matchers[i].flags & URL_MATCHER_FLAG_REGEXP) {
+ rspamd_multipattern_add_pattern(url_scanner->search_trie_strict,
+ static_matchers[i].pattern,
+ RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8 |
+ RSPAMD_MULTIPATTERN_RE);
+ }
+ else {
+ rspamd_multipattern_add_pattern(url_scanner->search_trie_strict,
+ static_matchers[i].pattern,
+ RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+ }
+ }
+
+ g_array_append_vals(sc->matchers_strict, static_matchers, n);
+
+ if (sc->matchers_full) {
+ for (i = 0; i < n; i++) {
+ if (static_matchers[i].flags & URL_MATCHER_FLAG_REGEXP) {
+ rspamd_multipattern_add_pattern(url_scanner->search_trie_full,
+ static_matchers[i].pattern,
+ RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8 |
+ RSPAMD_MULTIPATTERN_RE);
+ }
+ else {
+ rspamd_multipattern_add_pattern(url_scanner->search_trie_full,
+ static_matchers[i].pattern,
+ RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+ }
+ }
+ g_array_append_vals(sc->matchers_full, static_matchers, n);
+ }
+}
+
+void rspamd_url_deinit(void)
+{
+ if (url_scanner != NULL) {
+ if (url_scanner->search_trie_full) {
+ rspamd_multipattern_destroy(url_scanner->search_trie_full);
+ g_array_free(url_scanner->matchers_full, TRUE);
+ }
+
+ rspamd_multipattern_destroy(url_scanner->search_trie_strict);
+ g_array_free(url_scanner->matchers_strict, TRUE);
+ g_free(url_scanner);
+
+ url_scanner = NULL;
+ }
+}
+
+void rspamd_url_init(const gchar *tld_file)
+{
+ GError *err = NULL;
+ gboolean ret = TRUE;
+
+ if (url_scanner != NULL) {
+ rspamd_url_deinit();
+ }
+
+ url_scanner = g_malloc(sizeof(struct url_match_scanner));
+
+ url_scanner->matchers_strict = g_array_sized_new(FALSE, TRUE,
+ sizeof(struct url_matcher), G_N_ELEMENTS(static_matchers));
+ url_scanner->search_trie_strict = rspamd_multipattern_create_sized(
+ G_N_ELEMENTS(static_matchers),
+ RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+
+ if (tld_file) {
+ /* Reserve larger multipattern */
+ url_scanner->matchers_full = g_array_sized_new(FALSE, TRUE,
+ sizeof(struct url_matcher), 13000);
+ url_scanner->search_trie_full = rspamd_multipattern_create_sized(13000,
+ RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+ url_scanner->has_tld_file = true;
+ }
+ else {
+ url_scanner->matchers_full = NULL;
+ url_scanner->search_trie_full = NULL;
+ url_scanner->has_tld_file = false;
+ }
+
+ rspamd_url_add_static_matchers(url_scanner);
+
+ if (tld_file != NULL) {
+ ret = rspamd_url_parse_tld_file(tld_file, url_scanner);
+ }
+
+ if (url_scanner->matchers_full && url_scanner->matchers_full->len > 1000) {
+ msg_info("start compiling of %d TLD suffixes; it might take a long time",
+ url_scanner->matchers_full->len);
+ }
+
+ if (!rspamd_multipattern_compile(url_scanner->search_trie_strict, &err)) {
+ msg_err("cannot compile url matcher static patterns, fatal error: %e", err);
+ abort();
+ }
+
+ if (url_scanner->search_trie_full) {
+ if (!rspamd_multipattern_compile(url_scanner->search_trie_full, &err)) {
+ msg_err("cannot compile tld patterns, url matching will be "
+ "incomplete: %e",
+ err);
+ g_error_free(err);
+ ret = FALSE;
+ }
+ }
+
+ if (tld_file != NULL) {
+ if (ret) {
+ msg_info("initialized %ud url match suffixes from '%s'",
+ url_scanner->matchers_full->len - url_scanner->matchers_strict->len,
+ tld_file);
+ }
+ else {
+ msg_err("failed to initialize url tld suffixes from '%s', "
+ "use %ud internal match suffixes",
+ tld_file,
+ url_scanner->matchers_strict->len);
+ }
+ }
+
+ /* Generate hashes for flags */
+ for (gint i = 0; i < G_N_ELEMENTS(url_flag_names); i++) {
+ url_flag_names[i].hash =
+ rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT,
+ url_flag_names[i].name,
+ strlen(url_flag_names[i].name), 0);
+ }
+ /* Ensure that we have no hashes collisions O(N^2) but this array is small */
+ for (gint i = 0; i < G_N_ELEMENTS(url_flag_names) - 1; i++) {
+ for (gint j = i + 1; j < G_N_ELEMENTS(url_flag_names); j++) {
+ if (url_flag_names[i].hash == url_flag_names[j].hash) {
+ msg_err("collision: both %s and %s map to %d",
+ url_flag_names[i].name, url_flag_names[j].name,
+ url_flag_names[i].hash);
+ abort();
+ }
+ }
+ }
+}
+
+#define SET_U(u, field) \
+ do { \
+ if ((u) != NULL) { \
+ (u)->field_set |= 1 << (field); \
+ (u)->field_data[(field)].len = p - c; \
+ (u)->field_data[(field)].off = c - str; \
+ } \
+ } while (0)
+
+static bool
+is_url_start(gchar c)
+{
+ if (c == '(' ||
+ c == '{' ||
+ c == '[' ||
+ c == '<' ||
+ c == '\'') {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static bool
+is_url_end(gchar c)
+{
+ if (c == ')' ||
+ c == '}' ||
+ c == ']' ||
+ c == '>' ||
+ c == '\'') {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static bool
+is_domain_start(int p)
+{
+ if (g_ascii_isalnum(p) ||
+ p == '[' ||
+ p == '%' ||
+ p == '_' ||
+ (p & 0x80)) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static const guint max_domain_length = 253;
+static const guint max_dns_label = 63;
+static const guint max_email_user = 64;
+
+static gint
+rspamd_mailto_parse(struct http_parser_url *u,
+ const gchar *str, gsize len,
+ gchar const **end,
+ enum rspamd_url_parse_flags parse_flags, guint *flags)
+{
+ const gchar *p = str, *c = str, *last = str + len;
+ gchar t;
+ gint ret = 1;
+ enum {
+ parse_mailto,
+ parse_slash,
+ parse_slash_slash,
+ parse_semicolon,
+ parse_prefix_question,
+ parse_destination,
+ parse_equal,
+ parse_user,
+ parse_at,
+ parse_domain,
+ parse_suffix_question,
+ parse_query
+ } st = parse_mailto;
+
+ if (u != NULL) {
+ memset(u, 0, sizeof(*u));
+ }
+
+ while (p < last) {
+ t = *p;
+
+ if (p - str > max_email_user + max_domain_length + 1) {
+ goto out;
+ }
+
+ switch (st) {
+ case parse_mailto:
+ if (t == ':') {
+ st = parse_semicolon;
+ SET_U(u, UF_SCHEMA);
+ }
+ p++;
+ break;
+ case parse_semicolon:
+ if (t == '/' || t == '\\') {
+ st = parse_slash;
+ p++;
+ }
+ else {
+ *flags |= RSPAMD_URL_FLAG_MISSINGSLASHES;
+ st = parse_slash_slash;
+ }
+ break;
+ case parse_slash:
+ if (t == '/' || t == '\\') {
+ st = parse_slash_slash;
+ }
+ else {
+ goto out;
+ }
+ p++;
+ break;
+ case parse_slash_slash:
+ if (t == '?') {
+ st = parse_prefix_question;
+ p++;
+ }
+ else if (t != '/' && t != '\\') {
+ c = p;
+ st = parse_user;
+ }
+ else {
+ /* Skip multiple slashes */
+ p++;
+ }
+ break;
+ case parse_prefix_question:
+ if (t == 't') {
+ /* XXX: accept only to= */
+ st = parse_destination;
+ }
+ else {
+ goto out;
+ }
+ break;
+ case parse_destination:
+ if (t == '=') {
+ st = parse_equal;
+ }
+ p++;
+ break;
+ case parse_equal:
+ c = p;
+ st = parse_user;
+ break;
+ case parse_user:
+ if (t == '@') {
+ if (p - c == 0) {
+ goto out;
+ }
+ SET_U(u, UF_USERINFO);
+ st = parse_at;
+ }
+ else if (!is_mailsafe(t)) {
+ goto out;
+ }
+ else if (p - c > max_email_user) {
+ goto out;
+ }
+ p++;
+ break;
+ case parse_at:
+ c = p;
+ st = parse_domain;
+ break;
+ case parse_domain:
+ if (t == '?') {
+ SET_U(u, UF_HOST);
+ st = parse_suffix_question;
+ }
+ else if (!is_domain(t) && t != '.' && t != '_') {
+ goto out;
+ }
+ else if (p - c > max_domain_length) {
+ goto out;
+ }
+ p++;
+ break;
+ case parse_suffix_question:
+ c = p;
+ st = parse_query;
+ break;
+ case parse_query:
+ if (t == '#') {
+ if (p - c != 0) {
+ SET_U(u, UF_QUERY);
+ }
+ c = p + 1;
+ ret = 0;
+
+ goto out;
+ }
+ else if (!(parse_flags & RSPAMD_URL_PARSE_HREF) && is_url_end(t)) {
+ ret = 0;
+ goto out;
+ }
+ else if (is_lwsp(t)) {
+ if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+ if (g_ascii_isspace(t)) {
+ ret = 0;
+ }
+ goto out;
+ }
+ else {
+ goto out;
+ }
+ }
+ p++;
+ break;
+ }
+ }
+
+ if (st == parse_domain) {
+ if (p - c != 0) {
+ SET_U(u, UF_HOST);
+ ret = 0;
+ }
+ }
+ else if (st == parse_query) {
+ if (p - c > 0) {
+ SET_U(u, UF_QUERY);
+ }
+
+ ret = 0;
+ }
+
+out:
+ if (end != NULL) {
+ *end = p;
+ }
+
+ if ((parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+ return 0;
+ }
+
+ return ret;
+}
+
+static gint
+rspamd_telephone_parse(struct http_parser_url *u,
+ const gchar *str, gsize len,
+ gchar const **end,
+ enum rspamd_url_parse_flags parse_flags,
+ guint *flags)
+{
+ enum {
+ parse_protocol,
+ parse_semicolon,
+ parse_slash,
+ parse_slash_slash,
+ parse_spaces,
+ parse_plus,
+ parse_phone_start,
+ parse_phone,
+ } st = parse_protocol;
+
+ const gchar *p = str, *c = str, *last = str + len;
+ gchar t;
+ gint ret = 1, i;
+ UChar32 uc;
+
+ if (u != NULL) {
+ memset(u, 0, sizeof(*u));
+ }
+
+ while (p < last) {
+ t = *p;
+
+ if (p - str > max_email_user) {
+ goto out;
+ }
+
+ switch (st) {
+ case parse_protocol:
+ if (t == ':') {
+ st = parse_semicolon;
+ SET_U(u, UF_SCHEMA);
+ }
+ p++;
+ break;
+ case parse_semicolon:
+ if (t == '/' || t == '\\') {
+ st = parse_slash;
+ p++;
+ }
+ else {
+ st = parse_slash_slash;
+ }
+ break;
+ case parse_slash:
+ if (t == '/' || t == '\\') {
+ st = parse_slash_slash;
+ }
+ else {
+ goto out;
+ }
+ p++;
+ break;
+ case parse_slash_slash:
+ if (g_ascii_isspace(t)) {
+ st = parse_spaces;
+ p++;
+ }
+ else if (t == '+') {
+ c = p;
+ st = parse_plus;
+ }
+ else if (t == '/') {
+ /* Skip multiple slashes */
+ p++;
+ }
+ else {
+ st = parse_phone_start;
+ c = p;
+ }
+ break;
+ case parse_spaces:
+ if (t == '+') {
+ c = p;
+ st = parse_plus;
+ }
+ else if (!g_ascii_isspace(t)) {
+ st = parse_phone_start;
+ c = p;
+ }
+ else {
+ p++;
+ }
+ break;
+ case parse_plus:
+ c = p;
+ p++;
+ st = parse_phone_start;
+ break;
+ case parse_phone_start:
+ if (*p == '%' || *p == '(' || g_ascii_isdigit(*p)) {
+ st = parse_phone;
+ p++;
+ }
+ else {
+ goto out;
+ }
+ break;
+ case parse_phone:
+ i = p - str;
+ U8_NEXT(str, i, len, uc);
+ p = str + i;
+
+ if (u_isdigit(uc) || uc == '(' || uc == ')' || uc == '[' || uc == ']' || u_isspace(uc) || uc == '%') {
+ /* p is already incremented by U8_NEXT! */
+ }
+ else if (uc <= 0 || is_url_end(uc)) {
+ ret = 0;
+ goto set;
+ }
+ break;
+ }
+ }
+
+set:
+ if (st == parse_phone) {
+ if (p - c != 0) {
+ SET_U(u, UF_HOST);
+ ret = 0;
+ }
+ }
+
+out:
+ if (end != NULL) {
+ *end = p;
+ }
+
+ if ((parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+ return 0;
+ }
+
+ return ret;
+}
+
+static gint
+rspamd_web_parse(struct http_parser_url *u, const gchar *str, gsize len,
+ gchar const **end,
+ enum rspamd_url_parse_flags parse_flags,
+ guint *flags)
+{
+ const gchar *p = str, *c = str, *last = str + len, *slash = NULL,
+ *password_start = NULL, *user_start = NULL;
+ gchar t = 0;
+ UChar32 uc;
+ glong pt;
+ gint ret = 1;
+ gboolean user_seen = FALSE;
+ enum {
+ parse_protocol,
+ parse_slash,
+ parse_slash_slash,
+ parse_semicolon,
+ parse_user,
+ parse_at,
+ parse_multiple_at,
+ parse_password_start,
+ parse_password,
+ parse_domain_start,
+ parse_domain,
+ parse_ipv6,
+ parse_port_password,
+ parse_port,
+ parse_suffix_slash,
+ parse_path,
+ parse_query,
+ parse_part
+ } st = parse_protocol;
+
+ if (u != NULL) {
+ memset(u, 0, sizeof(*u));
+ }
+
+ while (p < last) {
+ t = *p;
+
+ switch (st) {
+ case parse_protocol:
+ if (t == ':') {
+ st = parse_semicolon;
+ SET_U(u, UF_SCHEMA);
+ }
+ else if (!g_ascii_isalnum(t) && t != '+' && t != '-') {
+ if ((parse_flags & RSPAMD_URL_PARSE_CHECK) && p > c) {
+ /* We might have some domain, but no protocol */
+ st = parse_domain_start;
+ p = c;
+ slash = c;
+ break;
+ }
+ else {
+ goto out;
+ }
+ }
+ p++;
+ break;
+ case parse_semicolon:
+ if (t == '/' || t == '\\') {
+ st = parse_slash;
+ p++;
+ }
+ else {
+ st = parse_slash_slash;
+ *(flags) |= RSPAMD_URL_FLAG_MISSINGSLASHES;
+ }
+ break;
+ case parse_slash:
+ if (t == '/' || t == '\\') {
+ st = parse_slash_slash;
+ }
+ else {
+ goto out;
+ }
+ p++;
+ break;
+ case parse_slash_slash:
+
+ if (t != '/' && t != '\\') {
+ c = p;
+ slash = p;
+ st = parse_domain_start;
+
+ /*
+ * Unfortunately, due to brain damage of the RFC 3986 authors,
+ * we have to distinguish two possibilities here:
+ * authority = [ userinfo "@" ] host [ ":" port ]
+ * So if we have @ somewhere before hostname then we must process
+ * with the username state. Otherwise, we have to process via
+ * the hostname state. Unfortunately, there is no way to distinguish
+ * them aside of running NFA or two DFA or performing lookahead.
+ * Lookahead approach looks easier to implement.
+ */
+
+ const char *tp = p;
+ while (tp < last) {
+ if (*tp == '@') {
+ user_seen = TRUE;
+ st = parse_user;
+ break;
+ }
+ else if (*tp == '/' || *tp == '#' || *tp == '?') {
+ st = parse_domain_start;
+ break;
+ }
+
+ tp++;
+ }
+
+ if (st == parse_domain_start && *p == '[') {
+ st = parse_ipv6;
+ p++;
+ c = p;
+ }
+ }
+ else {
+ /* Skip multiple slashes */
+ p++;
+ }
+ break;
+ case parse_ipv6:
+ if (t == ']') {
+ if (p - c == 0) {
+ goto out;
+ }
+ SET_U(u, UF_HOST);
+ p++;
+
+ if (*p == ':') {
+ st = parse_port;
+ c = p + 1;
+ }
+ else if (*p == '/' || *p == '\\') {
+ st = parse_path;
+ c = p + 1;
+ }
+ else if (*p == '?') {
+ st = parse_query;
+ c = p + 1;
+ }
+ else if (*p == '#') {
+ st = parse_part;
+ c = p + 1;
+ }
+ else if (p != last) {
+ goto out;
+ }
+ }
+ else if (!g_ascii_isxdigit(t) && t != ':' && t != '.') {
+ goto out;
+ }
+ p++;
+ break;
+ case parse_user:
+ if (t == ':') {
+ if (p - c == 0) {
+ goto out;
+ }
+ user_start = c;
+ st = parse_password_start;
+ }
+ else if (t == '@') {
+ /* No password */
+ if (p - c == 0) {
+ /* We have multiple at in fact */
+ st = parse_multiple_at;
+ user_seen = TRUE;
+ *flags |= RSPAMD_URL_FLAG_OBSCURED;
+
+ continue;
+ }
+
+ SET_U(u, UF_USERINFO);
+ *flags |= RSPAMD_URL_FLAG_HAS_USER;
+ st = parse_at;
+ }
+ else if (!g_ascii_isgraph(t)) {
+ goto out;
+ }
+ else if (p - c > max_email_user) {
+ goto out;
+ }
+
+ p++;
+ break;
+ case parse_multiple_at:
+ if (t != '@') {
+ if (p - c == 0) {
+ goto out;
+ }
+
+ /* For now, we ignore all that stuff as it is bogus */
+ /* Off by one */
+ p--;
+ SET_U(u, UF_USERINFO);
+ p++;
+ *flags |= RSPAMD_URL_FLAG_HAS_USER;
+ st = parse_at;
+ }
+ else {
+ p++;
+ }
+ break;
+ case parse_password_start:
+ if (t == '@') {
+ /* Empty password */
+ SET_U(u, UF_USERINFO);
+ if (u != NULL && u->field_data[UF_USERINFO].len > 0) {
+ /* Eat semicolon */
+ u->field_data[UF_USERINFO].len--;
+ }
+ *flags |= RSPAMD_URL_FLAG_HAS_USER;
+ st = parse_at;
+ }
+ else {
+ c = p;
+ password_start = p;
+ st = parse_password;
+ }
+ p++;
+ break;
+ case parse_password:
+ if (t == '@') {
+ /* XXX: password is not stored */
+ if (u != NULL) {
+ if (u->field_data[UF_USERINFO].len == 0 && password_start && user_start && password_start > user_start + 1) {
+ *flags |= RSPAMD_URL_FLAG_HAS_USER;
+ u->field_set |= 1u << (UF_USERINFO);
+ u->field_data[UF_USERINFO].len =
+ password_start - user_start - 1;
+ u->field_data[UF_USERINFO].off =
+ user_start - str;
+ }
+ }
+ st = parse_at;
+ }
+ else if (!g_ascii_isgraph(t)) {
+ goto out;
+ }
+ else if (p - c > max_domain_length) {
+ goto out;
+ }
+ p++;
+ break;
+ case parse_at:
+ c = p;
+
+ if (t == '@') {
+ *flags |= RSPAMD_URL_FLAG_OBSCURED;
+ p++;
+ }
+ else if (t == '[') {
+ st = parse_ipv6;
+ p++;
+ c = p;
+ }
+ else {
+ st = parse_domain_start;
+ }
+ break;
+ case parse_domain_start:
+ if (is_domain_start(t)) {
+ st = parse_domain;
+ }
+ else {
+ goto out;
+ }
+ break;
+ case parse_domain:
+ if (p - c > max_domain_length) {
+ /* Too large domain */
+ goto out;
+ }
+ if (t == '/' || t == '\\' || t == ':' || t == '?' || t == '#') {
+ if (p - c == 0) {
+ goto out;
+ }
+ if (t == '/' || t == '\\') {
+ SET_U(u, UF_HOST);
+ st = parse_suffix_slash;
+ }
+ else if (t == '?') {
+ SET_U(u, UF_HOST);
+ st = parse_query;
+ c = p + 1;
+ }
+ else if (t == '#') {
+ SET_U(u, UF_HOST);
+ st = parse_part;
+ c = p + 1;
+ }
+ else if (t == ':' && !user_seen) {
+ /*
+ * Here we can have both port and password, hence we need
+ * to apply some heuristic here
+ */
+ st = parse_port_password;
+ }
+ else {
+ /*
+ * We can go only for parsing port here
+ */
+ SET_U(u, UF_HOST);
+ st = parse_port;
+ c = p + 1;
+ }
+ p++;
+ }
+ else {
+ if (is_url_end(t) || is_url_start(t)) {
+ goto set;
+ }
+ else if (*p == '@' && !user_seen) {
+ /* We need to fallback and test user */
+ p = slash;
+ user_seen = TRUE;
+ st = parse_user;
+ }
+ else if (*p != '.' && *p != '-' && *p != '_' && *p != '%') {
+ if (*p & 0x80) {
+ guint i = 0;
+
+ U8_NEXT(((const guchar *) p), i, last - p, uc);
+
+ if (uc < 0) {
+ /* Bad utf8 */
+ goto out;
+ }
+
+ if (!u_isalnum(uc)) {
+ /* Bad symbol */
+ if (IS_ZERO_WIDTH_SPACE(uc)) {
+ (*flags) |= RSPAMD_URL_FLAG_ZW_SPACES;
+ }
+ else {
+ if (!u_isgraph(uc)) {
+ if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+ goto out;
+ }
+ else {
+ goto set;
+ }
+ }
+ }
+ }
+ else {
+ (*flags) |= RSPAMD_URL_FLAG_IDN;
+ }
+
+ p = p + i;
+ }
+ else if (is_urlsafe(*p)) {
+ p++;
+ }
+ else {
+ if (parse_flags & RSPAMD_URL_PARSE_HREF) {
+ /* We have to use all shit we are given here */
+ p++;
+ (*flags) |= RSPAMD_URL_FLAG_OBSCURED;
+ }
+ else {
+ if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+ goto out;
+ }
+ else {
+ goto set;
+ }
+ }
+ }
+ }
+ else {
+ p++;
+ }
+ }
+ break;
+ case parse_port_password:
+ if (g_ascii_isdigit(t)) {
+ const gchar *tmp = p;
+
+ while (tmp < last) {
+ if (!g_ascii_isdigit(*tmp)) {
+ if (*tmp == '/' || *tmp == '#' || *tmp == '?' ||
+ is_url_end(*tmp) || g_ascii_isspace(*tmp)) {
+ /* Port + something */
+ st = parse_port;
+ c = slash;
+ p--;
+ SET_U(u, UF_HOST);
+ p++;
+ c = p;
+ break;
+ }
+ else {
+ /* Not a port, bad character at the end */
+ break;
+ }
+ }
+ tmp++;
+ }
+
+ if (tmp == last) {
+ /* Host + port only */
+ st = parse_port;
+ c = slash;
+ p--;
+ SET_U(u, UF_HOST);
+ p++;
+ c = p;
+ }
+
+ if (st != parse_port) {
+ /* Fallback to user:password */
+ p = slash;
+ c = slash;
+ user_seen = TRUE;
+ st = parse_user;
+ }
+ }
+ else {
+ /* Rewind back */
+ p = slash;
+ c = slash;
+ user_seen = TRUE;
+ st = parse_user;
+ }
+ break;
+ case parse_port:
+ if (t == '/' || t == '\\') {
+ pt = strtoul(c, NULL, 10);
+ if (pt == 0 || pt > 65535) {
+ goto out;
+ }
+ if (u != NULL) {
+ u->port = pt;
+ *flags |= RSPAMD_URL_FLAG_HAS_PORT;
+ }
+ st = parse_suffix_slash;
+ }
+ else if (t == '?') {
+ pt = strtoul(c, NULL, 10);
+ if (pt == 0 || pt > 65535) {
+ goto out;
+ }
+ if (u != NULL) {
+ u->port = pt;
+ *flags |= RSPAMD_URL_FLAG_HAS_PORT;
+ }
+
+ c = p + 1;
+ st = parse_query;
+ }
+ else if (t == '#') {
+ pt = strtoul(c, NULL, 10);
+ if (pt == 0 || pt > 65535) {
+ goto out;
+ }
+ if (u != NULL) {
+ u->port = pt;
+ *flags |= RSPAMD_URL_FLAG_HAS_PORT;
+ }
+
+ c = p + 1;
+ st = parse_part;
+ }
+ else if (is_url_end(t)) {
+ goto set;
+ }
+ else if (!g_ascii_isdigit(t)) {
+ if (!(parse_flags & RSPAMD_URL_PARSE_CHECK) ||
+ !g_ascii_isspace(t)) {
+ goto out;
+ }
+ else {
+ goto set;
+ }
+ }
+ p++;
+ break;
+ case parse_suffix_slash:
+ if (t != '/' && t != '\\') {
+ c = p;
+ st = parse_path;
+ }
+ else {
+ /* Skip extra slashes */
+ p++;
+ }
+ break;
+ case parse_path:
+ if (t == '?') {
+ if (p - c != 0) {
+ SET_U(u, UF_PATH);
+ }
+ c = p + 1;
+ st = parse_query;
+ }
+ else if (t == '#') {
+ /* No query, just fragment */
+ if (p - c != 0) {
+ SET_U(u, UF_PATH);
+ }
+ c = p + 1;
+ st = parse_part;
+ }
+ else if (!(parse_flags & RSPAMD_URL_PARSE_HREF) && is_url_end(t)) {
+ goto set;
+ }
+ else if (is_lwsp(t)) {
+ if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+ if (g_ascii_isspace(t)) {
+ goto set;
+ }
+ goto out;
+ }
+ else {
+ goto set;
+ }
+ }
+ p++;
+ break;
+ case parse_query:
+ if (t == '#') {
+ if (p - c != 0) {
+ SET_U(u, UF_QUERY);
+ }
+ c = p + 1;
+ st = parse_part;
+ }
+ else if (!(parse_flags & RSPAMD_URL_PARSE_HREF) && is_url_end(t)) {
+ goto set;
+ }
+ else if (is_lwsp(t)) {
+ if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+ if (g_ascii_isspace(t)) {
+ goto set;
+ }
+ goto out;
+ }
+ else {
+ goto set;
+ }
+ }
+ p++;
+ break;
+ case parse_part:
+ if (!(parse_flags & RSPAMD_URL_PARSE_HREF) && is_url_end(t)) {
+ goto set;
+ }
+ else if (is_lwsp(t)) {
+ if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+ if (g_ascii_isspace(t)) {
+ goto set;
+ }
+ goto out;
+ }
+ else {
+ goto set;
+ }
+ }
+ p++;
+ break;
+ }
+ }
+
+set:
+ /* Parse remaining */
+ switch (st) {
+ case parse_domain:
+ if (p - c == 0 || !is_domain(*(p - 1)) || !is_domain(*c)) {
+ goto out;
+ }
+ SET_U(u, UF_HOST);
+ ret = 0;
+
+ break;
+ case parse_port:
+ pt = strtoul(c, NULL, 10);
+ if (pt == 0 || pt > 65535) {
+ goto out;
+ }
+ if (u != NULL) {
+ u->port = pt;
+ }
+
+ ret = 0;
+ break;
+ case parse_suffix_slash:
+ /* Url ends with '/' */
+ ret = 0;
+ break;
+ case parse_path:
+ if (p - c > 0) {
+ SET_U(u, UF_PATH);
+ }
+ ret = 0;
+ break;
+ case parse_query:
+ if (p - c > 0) {
+ SET_U(u, UF_QUERY);
+ }
+ ret = 0;
+ break;
+ case parse_part:
+ if (p - c > 0) {
+ SET_U(u, UF_FRAGMENT);
+ }
+ ret = 0;
+ break;
+ case parse_ipv6:
+ if (t != ']') {
+ ret = 1;
+ }
+ else {
+ /* e.g. http://[::] */
+ ret = 0;
+ }
+ break;
+ default:
+ /* Error state */
+ ret = 1;
+ break;
+ }
+out:
+ if (end != NULL) {
+ *end = p;
+ }
+
+ return ret;
+}
+
+#undef SET_U
+
+static gint
+rspamd_tld_trie_callback(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint match_pos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ struct url_matcher *matcher;
+ const gchar *start, *pos, *p;
+ struct rspamd_url *url = context;
+ gint ndots;
+
+ matcher = &g_array_index(url_scanner->matchers_full, struct url_matcher,
+ strnum);
+ ndots = 1;
+
+ if (matcher->flags & URL_MATCHER_FLAG_STAR_MATCH) {
+ /* Skip one more tld component */
+ ndots++;
+ }
+
+ pos = text + match_start;
+ p = pos - 1;
+ start = rspamd_url_host_unsafe(url);
+
+ if (*pos != '.' || match_pos != (gint) url->hostlen) {
+ /* Something weird has been found */
+ if (match_pos == (gint) url->hostlen - 1) {
+ pos = rspamd_url_host_unsafe(url) + match_pos;
+ if (*pos == '.') {
+ /* This is dot at the end of domain */
+ url->hostlen--;
+ }
+ else {
+ return 0;
+ }
+ }
+ else {
+ return 0;
+ }
+ }
+
+ /* Now we need to find top level domain */
+ pos = start;
+ while (p >= start && ndots > 0) {
+ if (*p == '.') {
+ ndots--;
+ pos = p + 1;
+ }
+ else {
+ pos = p;
+ }
+
+ p--;
+ }
+
+ if ((ndots == 0 || p == start - 1) &&
+ url->tldlen < rspamd_url_host_unsafe(url) + url->hostlen - pos) {
+ url->tldshift = (pos - url->string);
+ url->tldlen = rspamd_url_host_unsafe(url) + url->hostlen - pos;
+ }
+
+ return 0;
+}
+
+static void
+rspamd_url_regen_from_inet_addr(struct rspamd_url *uri, const void *addr, int af,
+ rspamd_mempool_t *pool)
+{
+ gchar *strbuf, *p;
+ const gchar *start_offset;
+ gsize slen = uri->urllen - uri->hostlen;
+ goffset r = 0;
+
+ if (af == AF_INET) {
+ slen += INET_ADDRSTRLEN;
+ }
+ else {
+ slen += INET6_ADDRSTRLEN;
+ }
+
+ if (uri->flags & RSPAMD_URL_FLAG_HAS_PORT) {
+ slen += sizeof("65535") - 1;
+ }
+
+ /* Allocate new string to build it from IP */
+ strbuf = rspamd_mempool_alloc(pool, slen + 1);
+ r += rspamd_snprintf(strbuf + r, slen - r, "%*s",
+ (gint) (uri->hostshift),
+ uri->string);
+
+ uri->hostshift = r;
+ uri->tldshift = r;
+ start_offset = strbuf + r;
+ inet_ntop(af, addr, strbuf + r, slen - r + 1);
+ uri->hostlen = strlen(start_offset);
+ r += uri->hostlen;
+ uri->tldlen = uri->hostlen;
+ uri->flags |= RSPAMD_URL_FLAG_NUMERIC;
+
+ /* Reconstruct URL */
+ if (uri->flags & RSPAMD_URL_FLAG_HAS_PORT && uri->ext) {
+ p = strbuf + r;
+ start_offset = p + 1;
+ r += rspamd_snprintf(strbuf + r, slen - r, ":%ud",
+ (unsigned int) uri->ext->port);
+ }
+ if (uri->datalen > 0) {
+ p = strbuf + r;
+ start_offset = p + 1;
+ r += rspamd_snprintf(strbuf + r, slen - r, "/%*s",
+ (gint) uri->datalen,
+ rspamd_url_data_unsafe(uri));
+ uri->datashift = start_offset - strbuf;
+ }
+ else {
+ /* Add trailing slash if needed */
+ if (uri->hostlen + uri->hostshift < uri->urllen &&
+ *(rspamd_url_host_unsafe(uri) + uri->hostlen) == '/') {
+ r += rspamd_snprintf(strbuf + r, slen - r, "/");
+ }
+ }
+
+ if (uri->querylen > 0) {
+ p = strbuf + r;
+ start_offset = p + 1;
+ r += rspamd_snprintf(strbuf + r, slen - r, "?%*s",
+ (gint) uri->querylen,
+ rspamd_url_query_unsafe(uri));
+ uri->queryshift = start_offset - strbuf;
+ }
+ if (uri->fragmentlen > 0) {
+ p = strbuf + r;
+ start_offset = p + 1;
+ r += rspamd_snprintf(strbuf + r, slen - r, "#%*s",
+ (gint) uri->fragmentlen,
+ rspamd_url_fragment_unsafe(uri));
+ uri->fragmentshift = start_offset - strbuf;
+ }
+
+ uri->string = strbuf;
+ uri->urllen = r;
+}
+
+static gboolean
+rspamd_url_maybe_regenerate_from_ip(struct rspamd_url *uri, rspamd_mempool_t *pool)
+{
+ const gchar *p, *end, *c;
+ gchar *errstr;
+ struct in_addr in4;
+ struct in6_addr in6;
+ gboolean ret = FALSE, check_num = TRUE;
+ guint32 n, dots, t = 0, i = 0, shift, nshift;
+
+ p = rspamd_url_host_unsafe(uri);
+ end = p + uri->hostlen;
+
+ if (*p == '[' && *(end - 1) == ']') {
+ p++;
+ end--;
+ }
+
+ while (*(end - 1) == '.' && end > p) {
+ end--;
+ }
+
+ if (end - p == 0 || end - p > INET6_ADDRSTRLEN) {
+ return FALSE;
+ }
+
+ if (rspamd_str_has_8bit(p, end - p)) {
+ return FALSE;
+ }
+
+ if (rspamd_parse_inet_address_ip4(p, end - p, &in4)) {
+ rspamd_url_regen_from_inet_addr(uri, &in4, AF_INET, pool);
+ ret = TRUE;
+ }
+ else if (rspamd_parse_inet_address_ip6(p, end - p, &in6)) {
+ rspamd_url_regen_from_inet_addr(uri, &in6, AF_INET6, pool);
+ ret = TRUE;
+ }
+ else {
+ /* Heuristics for broken urls */
+ gchar buf[INET6_ADDRSTRLEN + 1];
+ /* Try also numeric notation */
+ c = p;
+ n = 0;
+ dots = 0;
+ shift = 0;
+
+ while (p <= end && check_num) {
+ if (shift < 32 &&
+ ((*p == '.' && dots < 3) || (p == end && dots <= 3))) {
+ if (p - c + 1 >= (gint) sizeof(buf)) {
+ msg_debug_pool("invalid numeric url %*.s...: too long",
+ INET6_ADDRSTRLEN, c);
+ return FALSE;
+ }
+
+ rspamd_strlcpy(buf, c, p - c + 1);
+ c = p + 1;
+
+ if (p < end && *p == '.') {
+ dots++;
+ }
+
+ glong long_n = strtol(buf, &errstr, 0);
+
+ if ((errstr == NULL || *errstr == '\0') && long_n >= 0) {
+
+ t = long_n; /* Truncate as windows does */
+ /*
+ * Even if we have zero, we need to shift by 1 octet
+ */
+ nshift = (t == 0 ? shift + 8 : shift);
+
+ /*
+ * Here we count number of octets encoded in this element
+ */
+ for (i = 0; i < 4; i++) {
+ if ((t >> (8 * i)) > 0) {
+ nshift += 8;
+ }
+ else {
+ break;
+ }
+ }
+ /*
+ * Here we need to find the proper shift of the previous
+ * components, so we check possible cases:
+ * 1) 1 octet - just use it applying shift
+ * 2) 2 octets - convert to big endian 16 bit number
+ * 3) 3 octets - convert to big endian 24 bit number
+ * 4) 4 octets - convert to big endian 32 bit number
+ */
+ switch (i) {
+ case 4:
+ t = GUINT32_TO_BE(t);
+ break;
+ case 3:
+ t = (GUINT32_TO_BE(t & 0xFFFFFFU)) >> 8;
+ break;
+ case 2:
+ t = GUINT16_TO_BE(t & 0xFFFFU);
+ break;
+ default:
+ t = t & 0xFF;
+ break;
+ }
+
+ if (p != end) {
+ n |= t << shift;
+
+ shift = nshift;
+ }
+ }
+ else {
+ check_num = FALSE;
+ }
+ }
+
+ p++;
+ }
+
+ /* The last component should be last according to url normalization:
+ * 192.168.1 -> 192.168.0.1
+ * 192 -> 0.0.0.192
+ * 192.168 -> 192.0.0.168
+ */
+ shift = 8 * (4 - i);
+
+ if (shift < 32) {
+ n |= t << shift;
+ }
+
+ if (check_num) {
+ if (dots <= 4) {
+ memcpy(&in4, &n, sizeof(in4));
+ rspamd_url_regen_from_inet_addr(uri, &in4, AF_INET, pool);
+ uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
+ ret = TRUE;
+ }
+ else if (end - c > (gint) sizeof(buf) - 1) {
+ rspamd_strlcpy(buf, c, end - c + 1);
+
+ if (inet_pton(AF_INET6, buf, &in6) == 1) {
+ rspamd_url_regen_from_inet_addr(uri, &in6, AF_INET6, pool);
+ uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
+ ret = TRUE;
+ }
+ }
+ }
+ }
+
+ return ret;
+}
+
+static void
+rspamd_url_shift(struct rspamd_url *uri, gsize nlen,
+ enum http_parser_url_fields field)
+{
+ guint old_shift, shift = 0;
+ gint remain;
+
+ /* Shift remaining data */
+ switch (field) {
+ case UF_SCHEMA:
+ if (nlen >= uri->protocollen) {
+ return;
+ }
+ else {
+ shift = uri->protocollen - nlen;
+ }
+
+ old_shift = uri->protocollen;
+ uri->protocollen -= shift;
+ remain = uri->urllen - uri->protocollen;
+ g_assert(remain >= 0);
+ memmove(uri->string + uri->protocollen, uri->string + old_shift,
+ remain);
+ uri->urllen -= shift;
+ uri->flags |= RSPAMD_URL_FLAG_SCHEMAENCODED;
+ break;
+ case UF_HOST:
+ if (nlen >= uri->hostlen) {
+ return;
+ }
+ else {
+ shift = uri->hostlen - nlen;
+ }
+
+ old_shift = uri->hostlen;
+ uri->hostlen -= shift;
+ remain = (uri->urllen - (uri->hostshift)) - old_shift;
+ g_assert(remain >= 0);
+ memmove(rspamd_url_host_unsafe(uri) + uri->hostlen,
+ rspamd_url_host_unsafe(uri) + old_shift,
+ remain);
+ uri->urllen -= shift;
+ uri->flags |= RSPAMD_URL_FLAG_HOSTENCODED;
+ break;
+ case UF_PATH:
+ if (nlen >= uri->datalen) {
+ return;
+ }
+ else {
+ shift = uri->datalen - nlen;
+ }
+
+ old_shift = uri->datalen;
+ uri->datalen -= shift;
+ remain = (uri->urllen - (uri->datashift)) - old_shift;
+ g_assert(remain >= 0);
+ memmove(rspamd_url_data_unsafe(uri) + uri->datalen,
+ rspamd_url_data_unsafe(uri) + old_shift,
+ remain);
+ uri->urllen -= shift;
+ uri->flags |= RSPAMD_URL_FLAG_PATHENCODED;
+ break;
+ case UF_QUERY:
+ if (nlen >= uri->querylen) {
+ return;
+ }
+ else {
+ shift = uri->querylen - nlen;
+ }
+
+ old_shift = uri->querylen;
+ uri->querylen -= shift;
+ remain = (uri->urllen - (uri->queryshift)) - old_shift;
+ g_assert(remain >= 0);
+ memmove(rspamd_url_query_unsafe(uri) + uri->querylen,
+ rspamd_url_query_unsafe(uri) + old_shift,
+ remain);
+ uri->urllen -= shift;
+ uri->flags |= RSPAMD_URL_FLAG_QUERYENCODED;
+ break;
+ case UF_FRAGMENT:
+ if (nlen >= uri->fragmentlen) {
+ return;
+ }
+ else {
+ shift = uri->fragmentlen - nlen;
+ }
+
+ uri->fragmentlen -= shift;
+ uri->urllen -= shift;
+ break;
+ default:
+ break;
+ }
+
+ /* Now adjust lengths and offsets */
+ switch (field) {
+ case UF_SCHEMA:
+ if (uri->userlen > 0) {
+ uri->usershift -= shift;
+ }
+ if (uri->hostlen > 0) {
+ uri->hostshift -= shift;
+ }
+ /* Go forward */
+ /* FALLTHRU */
+ case UF_HOST:
+ if (uri->datalen > 0) {
+ uri->datashift -= shift;
+ }
+ /* Go forward */
+ /* FALLTHRU */
+ case UF_PATH:
+ if (uri->querylen > 0) {
+ uri->queryshift -= shift;
+ }
+ /* Go forward */
+ /* FALLTHRU */
+ case UF_QUERY:
+ if (uri->fragmentlen > 0) {
+ uri->fragmentshift -= shift;
+ }
+ /* Go forward */
+ /* FALLTHRU */
+ case UF_FRAGMENT:
+ default:
+ break;
+ }
+}
+
+static void
+rspamd_telephone_normalise_inplace(struct rspamd_url *uri)
+{
+ gchar *t, *h, *end;
+ gint i = 0, w, orig_len;
+ UChar32 uc;
+
+ t = rspamd_url_host_unsafe(uri);
+ h = t;
+ end = t + uri->hostlen;
+ orig_len = uri->hostlen;
+
+ if (*h == '+') {
+ h++;
+ t++;
+ }
+
+ while (h < end) {
+ i = 0;
+ U8_NEXT(h, i, end - h, uc);
+
+ if (u_isdigit(uc)) {
+ w = 0;
+ U8_APPEND_UNSAFE(t, w, uc);
+ t += w;
+ }
+
+ h += i;
+ }
+
+ uri->hostlen = t - rspamd_url_host_unsafe(uri);
+ uri->urllen -= (orig_len - uri->hostlen);
+}
+
+static inline bool
+is_idna_label_dot(UChar ch)
+{
+ switch (ch) {
+ case 0x3002:
+ case 0xFF0E:
+ case 0xFF61:
+ return true;
+ default:
+ return false;
+ }
+}
+
+/*
+ * All credits for this investigation should go to
+ * Dr. Hajime Shimada and Mr. Shirakura as they have revealed this case in their
+ * research.
+ */
+
+/*
+ * This function replaces unsafe IDNA dots in host labels. Unfortunately,
+ * IDNA extends dot definition from '.' to multiple other characters that
+ * should be treated equally.
+ * This function replaces such dots and returns `true` if these dots are found.
+ * In this case, it should be treated as obfuscation attempt.
+ */
+static bool
+rspamd_url_remove_dots(struct rspamd_url *uri)
+{
+ const gchar *hstart = rspamd_url_host_unsafe(uri);
+ gchar *t;
+ UChar32 uc;
+ gint i = 0, hlen;
+ bool ret = false;
+
+ if (uri->hostlen == 0) {
+ return false;
+ }
+
+ hlen = uri->hostlen;
+ t = rspamd_url_host_unsafe(uri);
+
+ while (i < hlen) {
+ gint prev_i = i;
+ U8_NEXT(hstart, i, hlen, uc);
+
+ if (is_idna_label_dot(uc)) {
+ *t++ = '.';
+ ret = true;
+ }
+ else {
+ if (ret) {
+ /* We have to shift the remaining stuff */
+ while (prev_i < i) {
+ *t++ = *(hstart + prev_i);
+ prev_i++;
+ }
+ }
+ else {
+ t += (i - prev_i);
+ }
+ }
+ }
+
+ if (ret) {
+ rspamd_url_shift(uri, t - hstart, UF_HOST);
+ }
+
+ return ret;
+}
+
+enum uri_errno
+rspamd_url_parse(struct rspamd_url *uri,
+ gchar *uristring, gsize len,
+ rspamd_mempool_t *pool,
+ enum rspamd_url_parse_flags parse_flags)
+{
+ struct http_parser_url u;
+ gchar *p;
+ const gchar *end;
+ guint complen, ret, flags = 0;
+ gsize unquoted_len = 0;
+
+ memset(uri, 0, sizeof(*uri));
+ memset(&u, 0, sizeof(u));
+ uri->count = 1;
+ /* Undefine order */
+ uri->order = -1;
+ uri->part_order = -1;
+
+ if (*uristring == '\0') {
+ return URI_ERRNO_EMPTY;
+ }
+
+ if (len >= G_MAXUINT16 / 2) {
+ flags |= RSPAMD_URL_FLAG_TRUNCATED;
+ len = G_MAXUINT16 / 2;
+ }
+
+ p = uristring;
+ uri->protocol = PROTOCOL_UNKNOWN;
+
+ if (len > sizeof("mailto:") - 1) {
+ /* For mailto: urls we also need to add slashes to make it a valid URL */
+ if (g_ascii_strncasecmp(p, "mailto:", sizeof("mailto:") - 1) == 0) {
+ ret = rspamd_mailto_parse(&u, uristring, len, &end, parse_flags,
+ &flags);
+ }
+ else if (g_ascii_strncasecmp(p, "tel:", sizeof("tel:") - 1) == 0 ||
+ g_ascii_strncasecmp(p, "callto:", sizeof("callto:") - 1) == 0) {
+ ret = rspamd_telephone_parse(&u, uristring, len, &end, parse_flags,
+ &flags);
+ uri->protocol = PROTOCOL_TELEPHONE;
+ }
+ else {
+ ret = rspamd_web_parse(&u, uristring, len, &end, parse_flags,
+ &flags);
+ }
+ }
+ else {
+ ret = rspamd_web_parse(&u, uristring, len, &end, parse_flags, &flags);
+ }
+
+ if (ret != 0) {
+ return URI_ERRNO_BAD_FORMAT;
+ }
+
+ if (end > uristring && (guint) (end - uristring) != len) {
+ len = end - uristring;
+ }
+
+ uri->raw = p;
+ uri->rawlen = len;
+
+ if (flags & RSPAMD_URL_FLAG_MISSINGSLASHES) {
+ len += 2;
+ uri->string = rspamd_mempool_alloc(pool, len + 1);
+ memcpy(uri->string, p, u.field_data[UF_SCHEMA].len);
+ memcpy(uri->string + u.field_data[UF_SCHEMA].len, "://", 3);
+ rspamd_strlcpy(uri->string + u.field_data[UF_SCHEMA].len + 3,
+ p + u.field_data[UF_SCHEMA].len + 1,
+ len - 2 - u.field_data[UF_SCHEMA].len);
+ /* Compensate slashes added */
+ for (int i = UF_SCHEMA + 1; i < UF_MAX; i++) {
+ if (u.field_set & (1 << i)) {
+ u.field_data[i].off += 2;
+ }
+ }
+ }
+ else {
+ uri->string = rspamd_mempool_alloc(pool, len + 1);
+ rspamd_strlcpy(uri->string, p, len + 1);
+ }
+
+ uri->urllen = len;
+ uri->flags = flags;
+
+ for (guint i = 0; i < UF_MAX; i++) {
+ if (u.field_set & (1 << i)) {
+ guint shift = u.field_data[i].off;
+ complen = u.field_data[i].len;
+
+ if (complen >= G_MAXUINT16) {
+ /* Too large component length */
+ return URI_ERRNO_BAD_FORMAT;
+ }
+
+ switch (i) {
+ case UF_SCHEMA:
+ uri->protocollen = u.field_data[i].len;
+ break;
+ case UF_HOST:
+ uri->hostshift = shift;
+ uri->hostlen = complen;
+ break;
+ case UF_PATH:
+ uri->datashift = shift;
+ uri->datalen = complen;
+ break;
+ case UF_QUERY:
+ uri->queryshift = shift;
+ uri->querylen = complen;
+ break;
+ case UF_FRAGMENT:
+ uri->fragmentshift = shift;
+ uri->fragmentlen = complen;
+ break;
+ case UF_USERINFO:
+ uri->usershift = shift;
+ uri->userlen = complen;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ /* Port is 'special' in case of url_parser as it is not a part of UF_* macro logic */
+ if (u.port != 0) {
+ if (!uri->ext) {
+ uri->ext = rspamd_mempool_alloc0_type(pool, struct rspamd_url_ext);
+ }
+ uri->flags |= RSPAMD_URL_FLAG_HAS_PORT;
+ uri->ext->port = u.port;
+ }
+
+ if (!uri->hostlen) {
+ return URI_ERRNO_HOST_MISSING;
+ }
+
+ /* Now decode url symbols */
+ unquoted_len = rspamd_url_decode(uri->string,
+ uri->string,
+ uri->protocollen);
+ rspamd_url_shift(uri, unquoted_len, UF_SCHEMA);
+ unquoted_len = rspamd_url_decode(rspamd_url_host_unsafe(uri),
+ rspamd_url_host_unsafe(uri), uri->hostlen);
+
+ rspamd_url_normalise_propagate_flags(pool, rspamd_url_host_unsafe(uri),
+ &unquoted_len, uri->flags);
+
+ rspamd_url_shift(uri, unquoted_len, UF_HOST);
+
+ if (rspamd_url_remove_dots(uri)) {
+ uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
+ }
+
+ if (uri->protocol & (PROTOCOL_HTTP | PROTOCOL_HTTPS | PROTOCOL_MAILTO | PROTOCOL_FTP | PROTOCOL_FILE)) {
+ /* Ensure that hostname starts with something sane (exclude numeric urls) */
+ const gchar *host = rspamd_url_host_unsafe(uri);
+
+ if (!(is_domain_start(host[0]) || host[0] == ':')) {
+ return URI_ERRNO_BAD_FORMAT;
+ }
+ }
+
+ /* Apply nameprep algorithm */
+ static UStringPrepProfile *nameprep = NULL;
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ if (nameprep == NULL) {
+ /* Open and cache profile */
+ nameprep = usprep_openByType(USPREP_RFC3491_NAMEPREP, &uc_err);
+
+ g_assert(U_SUCCESS(uc_err));
+ }
+
+ UChar *utf16_hostname, *norm_utf16;
+ gint32 utf16_len, norm_utf16_len, norm_utf8_len;
+ UParseError parse_error;
+
+ utf16_hostname = rspamd_mempool_alloc(pool, uri->hostlen * sizeof(UChar));
+ struct UConverter *utf8_conv = rspamd_get_utf8_converter();
+
+ utf16_len = ucnv_toUChars(utf8_conv, utf16_hostname, uri->hostlen,
+ rspamd_url_host_unsafe(uri), uri->hostlen, &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+
+ return URI_ERRNO_BAD_FORMAT;
+ }
+
+ norm_utf16 = rspamd_mempool_alloc(pool, utf16_len * sizeof(UChar));
+ norm_utf16_len = usprep_prepare(nameprep, utf16_hostname, utf16_len,
+ norm_utf16, utf16_len, USPREP_DEFAULT, &parse_error, &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+
+ return URI_ERRNO_BAD_FORMAT;
+ }
+
+ /* Convert back to utf8, sigh... */
+ norm_utf8_len = ucnv_fromUChars(utf8_conv,
+ rspamd_url_host_unsafe(uri), uri->hostlen,
+ norm_utf16, norm_utf16_len, &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+
+ return URI_ERRNO_BAD_FORMAT;
+ }
+
+ /* Final shift of lengths */
+ rspamd_url_shift(uri, norm_utf8_len, UF_HOST);
+
+ /* Process data part */
+ if (uri->datalen) {
+ unquoted_len = rspamd_url_decode(rspamd_url_data_unsafe(uri),
+ rspamd_url_data_unsafe(uri), uri->datalen);
+
+ rspamd_url_normalise_propagate_flags(pool, rspamd_url_data_unsafe(uri),
+ &unquoted_len, uri->flags);
+
+ rspamd_url_shift(uri, unquoted_len, UF_PATH);
+ /* We now normalize path */
+ rspamd_normalize_path_inplace(rspamd_url_data_unsafe(uri),
+ uri->datalen, &unquoted_len);
+ rspamd_url_shift(uri, unquoted_len, UF_PATH);
+ }
+
+ if (uri->querylen) {
+ unquoted_len = rspamd_url_decode(rspamd_url_query_unsafe(uri),
+ rspamd_url_query_unsafe(uri),
+ uri->querylen);
+
+ rspamd_url_normalise_propagate_flags(pool, rspamd_url_query_unsafe(uri),
+ &unquoted_len, uri->flags);
+ rspamd_url_shift(uri, unquoted_len, UF_QUERY);
+ }
+
+ if (uri->fragmentlen) {
+ unquoted_len = rspamd_url_decode(rspamd_url_fragment_unsafe(uri),
+ rspamd_url_fragment_unsafe(uri),
+ uri->fragmentlen);
+
+ rspamd_url_normalise_propagate_flags(pool, rspamd_url_fragment_unsafe(uri),
+ &unquoted_len, uri->flags);
+ rspamd_url_shift(uri, unquoted_len, UF_FRAGMENT);
+ }
+
+ rspamd_str_lc(uri->string, uri->protocollen);
+ unquoted_len = rspamd_str_lc_utf8(rspamd_url_host_unsafe(uri), uri->hostlen);
+ rspamd_url_shift(uri, unquoted_len, UF_HOST);
+
+ if (uri->protocol == PROTOCOL_UNKNOWN) {
+ for (int i = 0; i < G_N_ELEMENTS(rspamd_url_protocols); i++) {
+ if (uri->protocollen == rspamd_url_protocols[i].len) {
+ if (memcmp(uri->string,
+ rspamd_url_protocols[i].name, uri->protocollen) == 0) {
+ uri->protocol = rspamd_url_protocols[i].proto;
+ break;
+ }
+ }
+ }
+ }
+
+ if (uri->protocol & (PROTOCOL_HTTP | PROTOCOL_HTTPS | PROTOCOL_MAILTO | PROTOCOL_FTP | PROTOCOL_FILE)) {
+ /* Find TLD part */
+ if (url_scanner->search_trie_full) {
+ rspamd_multipattern_lookup(url_scanner->search_trie_full,
+ rspamd_url_host_unsafe(uri), uri->hostlen,
+ rspamd_tld_trie_callback, uri, NULL);
+ }
+
+ if (uri->tldlen == 0) {
+ /*
+ * If we have not detected eSLD, but there are no dots in the hostname,
+ * then we should treat the whole hostname as eSLD - a rule of thumb
+ *
+ * We also check that a hostname ends with a permitted character, and all characters are forming
+ * DNS label. We also need to check for a numeric IP within this check.
+ */
+ const char *dot_pos = memchr(rspamd_url_host_unsafe(uri), '.', uri->hostlen);
+ bool is_whole_hostname_tld = false;
+
+ if (uri->hostlen > 0 && (dot_pos == NULL || dot_pos == rspamd_url_host_unsafe(uri) + uri->hostlen - 1)) {
+ bool all_chars_domain = true;
+
+ for (int i = 0; i < uri->hostlen; i++) {
+ if (!is_domain(rspamd_url_host_unsafe(uri)[i])) {
+ all_chars_domain = false;
+ break;
+ }
+ }
+
+ char last_c = rspamd_url_host_unsafe(uri)[uri->hostlen - 1];
+
+ if (all_chars_domain) {
+ /* Also check the last character to be either a dot or alphanumeric character */
+ if (last_c != '.' && !g_ascii_isalnum(last_c)) {
+ all_chars_domain = false;
+ }
+ }
+
+ if (all_chars_domain) {
+ /* Additionally check for a numeric IP as we can have some number here... */
+ rspamd_url_maybe_regenerate_from_ip(uri, pool);
+
+ if (last_c == '.' && uri->hostlen > 1) {
+ /* Skip the last dot */
+ uri->tldlen = uri->hostlen - 1;
+ }
+ else {
+ uri->tldlen = uri->hostlen;
+ }
+
+ uri->tldshift = uri->hostshift;
+ is_whole_hostname_tld = true;
+ }
+ }
+
+ if (!is_whole_hostname_tld) {
+ if (uri->protocol != PROTOCOL_MAILTO) {
+ if (url_scanner->has_tld_file && !(parse_flags & RSPAMD_URL_PARSE_HREF)) {
+ /* Ignore URL's without TLD if it is not a numeric URL */
+ if (!rspamd_url_maybe_regenerate_from_ip(uri, pool)) {
+ return URI_ERRNO_TLD_MISSING;
+ }
+ }
+ else {
+ if (!rspamd_url_maybe_regenerate_from_ip(uri, pool)) {
+ /* Assume tld equal to host */
+ uri->tldshift = uri->hostshift;
+ uri->tldlen = uri->hostlen;
+ }
+ else if (uri->flags & RSPAMD_URL_FLAG_SCHEMALESS) {
+ /* Ignore urls with both no schema and no tld */
+ return URI_ERRNO_TLD_MISSING;
+ }
+
+ uri->flags |= RSPAMD_URL_FLAG_NO_TLD;
+ }
+ }
+ else {
+ /* Ignore IP like domains for mailto, as it is really never supported */
+ return URI_ERRNO_TLD_MISSING;
+ }
+ }
+ }
+
+ /* Replace stupid '\' with '/' after schema */
+ if (uri->protocol & (PROTOCOL_HTTP | PROTOCOL_HTTPS | PROTOCOL_FTP) &&
+ uri->protocollen > 0 && uri->urllen > uri->protocollen + 2) {
+
+ gchar *pos = &uri->string[uri->protocollen],
+ *host_start = rspamd_url_host_unsafe(uri);
+
+ while (pos < host_start) {
+ if (*pos == '\\') {
+ *pos = '/';
+ uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
+ }
+ pos++;
+ }
+ }
+ }
+ else if (uri->protocol & PROTOCOL_TELEPHONE) {
+ /* We need to normalise phone number: remove all spaces and braces */
+ rspamd_telephone_normalise_inplace(uri);
+
+ if (rspamd_url_host_unsafe(uri)[0] == '+') {
+ uri->tldshift = uri->hostshift + 1;
+ uri->tldlen = uri->hostlen - 1;
+ }
+ else {
+ uri->tldshift = uri->hostshift;
+ uri->tldlen = uri->hostlen;
+ }
+ }
+
+ if (uri->protocol == PROTOCOL_UNKNOWN) {
+ if (!(parse_flags & RSPAMD_URL_PARSE_HREF)) {
+ return URI_ERRNO_INVALID_PROTOCOL;
+ }
+ else {
+ /* Hack, hack, hack */
+ uri->protocol = PROTOCOL_UNKNOWN;
+ }
+ }
+
+ return URI_ERRNO_OK;
+}
+
+struct tld_trie_cbdata {
+ const gchar *begin;
+ gsize len;
+ rspamd_ftok_t *out;
+};
+
+static gint
+rspamd_tld_trie_find_callback(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint match_pos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ struct url_matcher *matcher;
+ const gchar *start, *pos, *p;
+ struct tld_trie_cbdata *cbdata = context;
+ gint ndots = 1;
+
+ matcher = &g_array_index(url_scanner->matchers_full, struct url_matcher,
+ strnum);
+
+ if (matcher->flags & URL_MATCHER_FLAG_STAR_MATCH) {
+ /* Skip one more tld component */
+ ndots = 2;
+ }
+
+ pos = text + match_start;
+ p = pos - 1;
+ start = text;
+
+ if (*pos != '.' || match_pos != (gint) cbdata->len) {
+ /* Something weird has been found */
+ if (match_pos != (gint) cbdata->len - 1) {
+ /* Search more */
+ return 0;
+ }
+ }
+
+ /* Now we need to find top level domain */
+ pos = start;
+
+ while (p >= start && ndots > 0) {
+ if (*p == '.') {
+ ndots--;
+ pos = p + 1;
+ }
+ else {
+ pos = p;
+ }
+
+ p--;
+ }
+
+ if (ndots == 0 || p == start - 1) {
+ if (cbdata->begin + cbdata->len - pos > cbdata->out->len) {
+ cbdata->out->begin = pos;
+ cbdata->out->len = cbdata->begin + cbdata->len - pos;
+ }
+ }
+
+ return 0;
+}
+
+gboolean
+rspamd_url_find_tld(const gchar *in, gsize inlen, rspamd_ftok_t *out)
+{
+ struct tld_trie_cbdata cbdata;
+
+ g_assert(in != NULL);
+ g_assert(out != NULL);
+ g_assert(url_scanner != NULL);
+
+ cbdata.begin = in;
+ cbdata.len = inlen;
+ cbdata.out = out;
+ out->len = 0;
+
+ if (url_scanner->search_trie_full) {
+ rspamd_multipattern_lookup(url_scanner->search_trie_full, in, inlen,
+ rspamd_tld_trie_find_callback, &cbdata, NULL);
+ }
+
+ if (out->len > 0) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static const gchar url_braces[] = {
+ '(', ')',
+ '{', '}',
+ '[', ']',
+ '<', '>',
+ '|', '|',
+ '\'', '\''};
+
+
+static gboolean
+url_file_start(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match)
+{
+ match->m_begin = pos;
+
+ if (pos > cb->begin) {
+ match->st = *(pos - 1);
+ }
+ else {
+ match->st = '\0';
+ }
+
+ return TRUE;
+}
+
+static gboolean
+url_file_end(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match)
+{
+ const gchar *p;
+ gchar stop;
+ guint i;
+
+ p = pos + strlen(match->pattern);
+ stop = *p;
+ if (*p == '/') {
+ p++;
+ }
+
+ for (i = 0; i < G_N_ELEMENTS(url_braces) / 2; i += 2) {
+ if (*p == url_braces[i]) {
+ stop = url_braces[i + 1];
+ break;
+ }
+ }
+
+ while (p < cb->end && *p != stop && is_urlsafe(*p)) {
+ p++;
+ }
+
+ if (p == cb->begin) {
+ return FALSE;
+ }
+ match->m_len = p - match->m_begin;
+
+ return TRUE;
+}
+
+static gboolean
+url_tld_start(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match)
+{
+ const gchar *p = pos;
+ guint processed = 0;
+ static const guint max_shift = 253 + sizeof("https://");
+
+ /* Try to find the start of the url by finding any non-urlsafe character or whitespace/punctuation */
+ while (p >= cb->begin) {
+ if (!is_domain(*p) || g_ascii_isspace(*p) || is_url_start(*p) ||
+ p == match->prev_newline_pos) {
+ if (!is_url_start(*p) && !g_ascii_isspace(*p) &&
+ p != match->prev_newline_pos) {
+ return FALSE;
+ }
+
+ if (p != match->prev_newline_pos) {
+ match->st = *p;
+
+ p++;
+ }
+ else {
+ match->st = '\n';
+ }
+
+ if (!g_ascii_isalnum(*p)) {
+ /* Urls cannot start with strange symbols */
+ return FALSE;
+ }
+
+ match->m_begin = p;
+ return TRUE;
+ }
+ else if (p == cb->begin && p != pos) {
+ match->st = '\0';
+ match->m_begin = p;
+
+ return TRUE;
+ }
+ else if (*p == '.') {
+ if (p == cb->begin) {
+ /* Urls cannot start with a dot */
+ return FALSE;
+ }
+ if (!g_ascii_isalnum(p[1])) {
+ /* Wrong we have an invalid character after dot */
+ return FALSE;
+ }
+ }
+ else if (*p == '/') {
+ /* Urls cannot contain '/' in their body */
+ return FALSE;
+ }
+
+ p--;
+ processed++;
+
+ if (processed > max_shift) {
+ /* Too long */
+ return FALSE;
+ }
+ }
+
+ return FALSE;
+}
+
+static gboolean
+url_tld_end(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match)
+{
+ const gchar *p;
+ gboolean ret = FALSE;
+
+ p = pos + match->m_len;
+
+ if (p == cb->end) {
+ match->m_len = p - match->m_begin;
+ return TRUE;
+ }
+ else if (*p == '/' || *p == ':' || is_url_end(*p) || is_lwsp(*p) ||
+ (match->st != '<' && p == match->newline_pos)) {
+ /* Parse arguments, ports by normal way by url default function */
+ p = match->m_begin;
+ /* Check common prefix */
+ if (g_ascii_strncasecmp(p, "http://", sizeof("http://") - 1) == 0) {
+ ret = url_web_end(cb,
+ match->m_begin + sizeof("http://") - 1,
+ match);
+ }
+ else {
+ ret = url_web_end(cb, match->m_begin, match);
+ }
+ }
+ else if (*p == '.') {
+ p++;
+ if (p < cb->end) {
+ if (g_ascii_isspace(*p) || *p == '/' ||
+ *p == '?' || *p == ':') {
+ ret = url_web_end(cb, match->m_begin, match);
+ }
+ }
+ }
+
+ if (ret) {
+ /* Check sanity of match found */
+ if (match->m_begin + match->m_len <= pos) {
+ return FALSE;
+ }
+ }
+
+ return ret;
+}
+
+static gboolean
+url_web_start(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match)
+{
+ /* Check what we have found */
+ if (pos > cb->begin) {
+ if (g_ascii_strncasecmp(pos, "www", 3) == 0) {
+
+ if (!(is_url_start(*(pos - 1)) ||
+ g_ascii_isspace(*(pos - 1)) ||
+ pos - 1 == match->prev_newline_pos ||
+ (*(pos - 1) & 0x80))) { /* Chinese trick */
+ return FALSE;
+ }
+ }
+ else {
+ guchar prev = *(pos - 1);
+
+ if (g_ascii_isalnum(prev)) {
+ /* Part of another url */
+ return FALSE;
+ }
+ }
+ }
+
+ if (*pos == '.') {
+ /* Urls cannot start with . */
+ return FALSE;
+ }
+
+ if (pos > cb->begin) {
+ match->st = *(pos - 1);
+ }
+ else {
+ match->st = '\0';
+ }
+
+ match->m_begin = pos;
+
+ return TRUE;
+}
+
+static gboolean
+url_web_end(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match)
+{
+ const gchar *last = NULL;
+ gint len = cb->end - pos;
+ guint flags = 0;
+
+ if (match->newline_pos && match->st != '<') {
+ /* We should also limit our match end to the newline */
+ len = MIN(len, match->newline_pos - pos);
+ }
+
+ if (rspamd_web_parse(NULL, pos, len, &last,
+ RSPAMD_URL_PARSE_CHECK, &flags) != 0) {
+ return FALSE;
+ }
+
+ if (last < cb->end && (*last == '>' && last != match->newline_pos)) {
+ /* We need to ensure that url also starts with '>' */
+ if (match->st != '<') {
+ if (last + 1 < cb->end) {
+ if (g_ascii_isspace(last[1])) {
+ return FALSE;
+ }
+ }
+ else {
+ return FALSE;
+ }
+ }
+ }
+
+ match->m_len = (last - pos);
+ cb->fin = last + 1;
+
+ return TRUE;
+}
+
+
+static gboolean
+url_email_start(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match)
+{
+ if (!match->prefix || match->prefix[0] == '\0') {
+ /* We have mailto:// at the beginning */
+ match->m_begin = pos;
+
+ if (pos >= cb->begin + 1) {
+ match->st = *(pos - 1);
+ }
+ else {
+ match->st = '\0';
+ }
+ }
+ else {
+ /* Just '@' */
+
+ /* Check if this match is a part of the previous mailto: email */
+ if (cb->last_at != NULL && cb->last_at == pos) {
+ cb->last_at = NULL;
+ return FALSE;
+ }
+ else if (pos == cb->begin) {
+ /* Just @ at the start of input */
+ return FALSE;
+ }
+
+ match->st = '\0';
+ }
+
+ return TRUE;
+}
+
+static gboolean
+url_email_end(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match)
+{
+ const gchar *last = NULL;
+ struct http_parser_url u;
+ gint len = cb->end - pos;
+ guint flags = 0;
+
+ if (match->newline_pos && match->st != '<') {
+ /* We should also limit our match end to the newline */
+ len = MIN(len, match->newline_pos - pos);
+ }
+
+ if (!match->prefix || match->prefix[0] == '\0') {
+ /* We have mailto:// at the beginning */
+ if (rspamd_mailto_parse(&u, pos, len, &last,
+ RSPAMD_URL_PARSE_CHECK, &flags) != 0) {
+ return FALSE;
+ }
+
+ if (!(u.field_set & (1 << UF_USERINFO))) {
+ return FALSE;
+ }
+
+ cb->last_at = match->m_begin + u.field_data[UF_USERINFO].off +
+ u.field_data[UF_USERINFO].len;
+
+ g_assert(*cb->last_at == '@');
+ match->m_len = (last - pos);
+
+ return TRUE;
+ }
+ else {
+ const gchar *c, *p;
+ /*
+ * Here we have just '@', so we need to find both start and end of the
+ * pattern
+ */
+ g_assert(*pos == '@');
+
+ if (pos >= cb->end - 2 || pos < cb->begin + 1) {
+ /* Boundary violation */
+ return FALSE;
+ }
+
+ /* Check the next character after `@` */
+ if (!g_ascii_isalnum(pos[1]) || !g_ascii_isalnum(*(pos - 1))) {
+ return FALSE;
+ }
+
+
+ c = pos - 1;
+ while (c > cb->begin) {
+ if (!is_mailsafe(*c)) {
+ break;
+ }
+ if (c == match->prev_newline_pos) {
+ break;
+ }
+
+ c--;
+ }
+ /* Rewind to the first alphanumeric character */
+ while (c < pos && !g_ascii_isalnum(*c)) {
+ c++;
+ }
+
+ /* Find the end of email */
+ p = pos + 1;
+ while (p < cb->end && is_domain(*p)) {
+ if (p == match->newline_pos) {
+ break;
+ }
+
+ p++;
+ }
+
+ /* Rewind it again to avoid bad emails to be detected */
+ while (p > pos && p < cb->end && !g_ascii_isalnum(*p)) {
+ p--;
+ }
+
+ if (p < cb->end && g_ascii_isalnum(*p) &&
+ (match->newline_pos == NULL || p < match->newline_pos)) {
+ p++;
+ }
+
+ if (p > c) {
+ match->m_begin = c;
+ match->m_len = p - c;
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+static gboolean
+url_tel_start(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match)
+{
+ match->m_begin = pos;
+
+ if (pos >= cb->begin + 1) {
+ match->st = *(pos - 1);
+ }
+ else {
+ match->st = '\0';
+ }
+
+ return TRUE;
+}
+
+static gboolean
+url_tel_end(struct url_callback_data *cb,
+ const gchar *pos,
+ url_match_t *match)
+{
+ const gchar *last = NULL;
+ struct http_parser_url u;
+ gint len = cb->end - pos;
+ guint flags = 0;
+
+ if (match->newline_pos && match->st != '<') {
+ /* We should also limit our match end to the newline */
+ len = MIN(len, match->newline_pos - pos);
+ }
+
+ if (rspamd_telephone_parse(&u, pos, len, &last,
+ RSPAMD_URL_PARSE_CHECK, &flags) != 0) {
+ return FALSE;
+ }
+
+ if (!(u.field_set & (1 << UF_HOST))) {
+ return FALSE;
+ }
+
+ match->m_len = (last - pos);
+
+ return TRUE;
+}
+
+
+static gboolean
+rspamd_url_trie_is_match(struct url_matcher *matcher, const gchar *pos,
+ const gchar *end, const gchar *newline_pos)
+{
+ if (matcher->flags & URL_MATCHER_FLAG_TLD_MATCH) {
+ /* Immediately check pos for valid chars */
+ if (pos < end) {
+ if (pos != newline_pos && !g_ascii_isspace(*pos) && *pos != '/' && *pos != '?' &&
+ *pos != ':' && !is_url_end(*pos)) {
+ if (*pos == '.') {
+ /* We allow . at the end of the domain however */
+ pos++;
+ if (pos < end) {
+ if (!g_ascii_isspace(*pos) && *pos != '/' &&
+ *pos != '?' && *pos != ':' && !is_url_end(*pos)) {
+ return FALSE;
+ }
+ }
+ }
+ else {
+ return FALSE;
+ }
+ }
+ }
+ }
+
+ return TRUE;
+}
+
+static gint
+rspamd_url_trie_callback(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint match_pos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ struct url_matcher *matcher;
+ url_match_t m;
+ const gchar *pos, *newline_pos = NULL;
+ struct url_callback_data *cb = context;
+
+ pos = text + match_pos;
+
+ if (cb->fin > pos) {
+ /* Already seen */
+ return 0;
+ }
+
+ matcher = &g_array_index(cb->matchers, struct url_matcher,
+ strnum);
+
+ if ((matcher->flags & URL_MATCHER_FLAG_NOHTML) && cb->how == RSPAMD_URL_FIND_STRICT) {
+ /* Do not try to match non-html like urls in html texts */
+ return 0;
+ }
+
+ memset(&m, 0, sizeof(m));
+ m.m_begin = text + match_start;
+ m.m_len = match_pos - match_start;
+
+ if (cb->newlines && cb->newlines->len > 0) {
+ newline_pos = g_ptr_array_index(cb->newlines, cb->newline_idx);
+
+ while (pos > newline_pos && cb->newline_idx < cb->newlines->len) {
+ cb->newline_idx++;
+ newline_pos = g_ptr_array_index(cb->newlines, cb->newline_idx);
+ }
+
+ if (pos > newline_pos) {
+ newline_pos = NULL;
+ }
+
+ if (cb->newline_idx > 0) {
+ m.prev_newline_pos = g_ptr_array_index(cb->newlines,
+ cb->newline_idx - 1);
+ }
+ }
+
+ if (!rspamd_url_trie_is_match(matcher, pos, cb->end, newline_pos)) {
+ return 0;
+ }
+
+ m.pattern = matcher->pattern;
+ m.prefix = matcher->prefix;
+ m.add_prefix = FALSE;
+ m.newline_pos = newline_pos;
+ pos = cb->begin + match_start;
+
+ if (matcher->start(cb, pos, &m) &&
+ matcher->end(cb, pos, &m)) {
+ if (m.add_prefix || matcher->prefix[0] != '\0') {
+ cb->len = m.m_len + strlen(matcher->prefix);
+ cb->url_str = rspamd_mempool_alloc(cb->pool, cb->len + 1);
+ cb->len = rspamd_snprintf(cb->url_str,
+ cb->len + 1,
+ "%s%*s",
+ m.prefix,
+ (gint) m.m_len,
+ m.m_begin);
+ cb->prefix_added = TRUE;
+ }
+ else {
+ cb->url_str = rspamd_mempool_alloc(cb->pool, m.m_len + 1);
+ rspamd_strlcpy(cb->url_str, m.m_begin, m.m_len + 1);
+ }
+
+ cb->start = m.m_begin;
+
+ if (pos > cb->fin) {
+ cb->fin = pos;
+ }
+
+ return 1;
+ }
+ else {
+ cb->url_str = NULL;
+ }
+
+ /* Continue search */
+ return 0;
+}
+
+gboolean
+rspamd_url_find(rspamd_mempool_t *pool,
+ const gchar *begin, gsize len,
+ gchar **url_str,
+ enum rspamd_url_find_type how,
+ goffset *url_pos,
+ gboolean *prefix_added)
+{
+ struct url_callback_data cb;
+ gint ret;
+
+ memset(&cb, 0, sizeof(cb));
+ cb.begin = begin;
+ cb.end = begin + len;
+ cb.how = how;
+ cb.pool = pool;
+
+ if (how == RSPAMD_URL_FIND_ALL) {
+ if (url_scanner->search_trie_full) {
+ cb.matchers = url_scanner->matchers_full;
+ ret = rspamd_multipattern_lookup(url_scanner->search_trie_full,
+ begin, len,
+ rspamd_url_trie_callback, &cb, NULL);
+ }
+ else {
+ cb.matchers = url_scanner->matchers_strict;
+ ret = rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+ begin, len,
+ rspamd_url_trie_callback, &cb, NULL);
+ }
+ }
+ else {
+ cb.matchers = url_scanner->matchers_strict;
+ ret = rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+ begin, len,
+ rspamd_url_trie_callback, &cb, NULL);
+ }
+
+ if (ret) {
+ if (url_str) {
+ *url_str = cb.url_str;
+ }
+
+ if (url_pos) {
+ *url_pos = cb.start - begin;
+ }
+
+ if (prefix_added) {
+ *prefix_added = cb.prefix_added;
+ }
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static gint
+rspamd_url_trie_generic_callback_common(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint match_pos,
+ const gchar *text,
+ gsize len,
+ void *context,
+ gboolean multiple)
+{
+ struct rspamd_url *url;
+ struct url_matcher *matcher;
+ url_match_t m;
+ const gchar *pos, *newline_pos = NULL;
+ struct url_callback_data *cb = context;
+ gint rc;
+ rspamd_mempool_t *pool;
+
+ pos = text + match_pos;
+
+ if (cb->fin > pos) {
+ /* Already seen */
+ return 0;
+ }
+
+ matcher = &g_array_index(cb->matchers, struct url_matcher,
+ strnum);
+ pool = cb->pool;
+
+ if ((matcher->flags & URL_MATCHER_FLAG_NOHTML) && cb->how == RSPAMD_URL_FIND_STRICT) {
+ /* Do not try to match non-html like urls in html texts, continue matching */
+ return 0;
+ }
+
+ memset(&m, 0, sizeof(m));
+
+
+ /* Find the next newline after our pos */
+ if (cb->newlines && cb->newlines->len > 0) {
+ newline_pos = g_ptr_array_index(cb->newlines, cb->newline_idx);
+
+ while (pos > newline_pos && cb->newline_idx < cb->newlines->len - 1) {
+ cb->newline_idx++;
+ newline_pos = g_ptr_array_index(cb->newlines, cb->newline_idx);
+ }
+
+ if (pos > newline_pos) {
+ newline_pos = NULL;
+ }
+ if (cb->newline_idx > 0) {
+ m.prev_newline_pos = g_ptr_array_index(cb->newlines,
+ cb->newline_idx - 1);
+ }
+ }
+
+ if (!rspamd_url_trie_is_match(matcher, pos, text + len, newline_pos)) {
+ /* Mismatch, continue */
+ return 0;
+ }
+
+ pos = cb->begin + match_start;
+ m.pattern = matcher->pattern;
+ m.prefix = matcher->prefix;
+ m.add_prefix = FALSE;
+ m.m_begin = text + match_start;
+ m.m_len = match_pos - match_start;
+ m.newline_pos = newline_pos;
+
+ if (matcher->start(cb, pos, &m) &&
+ matcher->end(cb, pos, &m)) {
+ if (m.add_prefix || matcher->prefix[0] != '\0') {
+ cb->len = m.m_len + strlen(matcher->prefix);
+ cb->url_str = rspamd_mempool_alloc(cb->pool, cb->len + 1);
+ cb->len = rspamd_snprintf(cb->url_str,
+ cb->len + 1,
+ "%s%*s",
+ m.prefix,
+ (gint) m.m_len,
+ m.m_begin);
+ cb->prefix_added = TRUE;
+ }
+ else {
+ cb->url_str = rspamd_mempool_alloc(cb->pool, m.m_len + 1);
+ cb->len = rspamd_strlcpy(cb->url_str, m.m_begin, m.m_len + 1);
+ }
+
+ cb->start = m.m_begin;
+
+ if (pos > cb->fin) {
+ cb->fin = pos;
+ }
+
+ url = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_url));
+ g_strstrip(cb->url_str);
+ rc = rspamd_url_parse(url, cb->url_str,
+ strlen(cb->url_str), pool,
+ RSPAMD_URL_PARSE_TEXT);
+
+ if (rc == URI_ERRNO_OK && url->hostlen > 0) {
+ if (cb->prefix_added) {
+ url->flags |= RSPAMD_URL_FLAG_SCHEMALESS;
+ cb->prefix_added = FALSE;
+ }
+
+ if (cb->func) {
+ if (!cb->func(url, cb->start - text, (m.m_begin + m.m_len) - text,
+ cb->funcd)) {
+ /* We need to stop here in any case! */
+ return -1;
+ }
+ }
+ }
+ else if (rc != URI_ERRNO_OK) {
+ msg_debug_pool_check("extract of url '%s' failed: %s",
+ cb->url_str,
+ rspamd_url_strerror(rc));
+ }
+ }
+ else {
+ cb->url_str = NULL;
+ /* Continue search if no pattern has been found */
+ return 0;
+ }
+
+ /* Continue search if required (return 0 means continue) */
+ return !multiple;
+}
+
+static gint
+rspamd_url_trie_generic_callback_multiple(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint match_pos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ return rspamd_url_trie_generic_callback_common(mp, strnum, match_start,
+ match_pos, text, len, context, TRUE);
+}
+
+static gint
+rspamd_url_trie_generic_callback_single(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint match_pos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ return rspamd_url_trie_generic_callback_common(mp, strnum, match_start,
+ match_pos, text, len, context, FALSE);
+}
+
+struct rspamd_url_mimepart_cbdata {
+ struct rspamd_task *task;
+ struct rspamd_mime_text_part *part;
+ gsize url_len;
+ uint16_t *cur_url_order; /* Global ordering */
+ uint16_t cur_part_order; /* Per part ordering */
+};
+
+static gboolean
+rspamd_url_query_callback(struct rspamd_url *url, gsize start_offset,
+ gsize end_offset, gpointer ud)
+{
+ struct rspamd_url_mimepart_cbdata *cbd =
+ (struct rspamd_url_mimepart_cbdata *) ud;
+ struct rspamd_task *task;
+
+ task = cbd->task;
+
+ if (url->protocol == PROTOCOL_MAILTO) {
+ if (url->userlen == 0) {
+ return FALSE;
+ }
+ }
+ /* Also check max urls */
+ if (cbd->task->cfg && cbd->task->cfg->max_urls > 0) {
+ if (kh_size(MESSAGE_FIELD(task, urls)) > cbd->task->cfg->max_urls) {
+ msg_err_task("part has too many URLs, we cannot process more: "
+ "%d urls extracted ",
+ (guint) kh_size(MESSAGE_FIELD(task, urls)));
+
+ return FALSE;
+ }
+ }
+
+ url->flags |= RSPAMD_URL_FLAG_QUERY;
+
+
+ if (rspamd_url_set_add_or_increase(MESSAGE_FIELD(task, urls), url, false)) {
+ if (cbd->part && cbd->part->mime_part->urls) {
+ g_ptr_array_add(cbd->part->mime_part->urls, url);
+ }
+
+ url->part_order = cbd->cur_part_order++;
+
+ if (cbd->cur_url_order) {
+ url->order = (*cbd->cur_url_order)++;
+ }
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_url_text_part_callback(struct rspamd_url *url, gsize start_offset,
+ gsize end_offset, gpointer ud)
+{
+ struct rspamd_url_mimepart_cbdata *cbd =
+ (struct rspamd_url_mimepart_cbdata *) ud;
+ struct rspamd_process_exception *ex;
+ struct rspamd_task *task;
+
+ task = cbd->task;
+ ex = rspamd_mempool_alloc0(task->task_pool, sizeof(struct rspamd_process_exception));
+
+ ex->pos = start_offset;
+ ex->len = end_offset - start_offset;
+ ex->type = RSPAMD_EXCEPTION_URL;
+ ex->ptr = url;
+
+ cbd->url_len += ex->len;
+
+ if (cbd->part->utf_stripped_content &&
+ cbd->url_len > cbd->part->utf_stripped_content->len * 10) {
+ /* Absurd case, stop here now */
+ msg_err_task("part has too many URLs, we cannot process more: %z url len; "
+ "%d stripped content length",
+ cbd->url_len, cbd->part->utf_stripped_content->len);
+
+ return FALSE;
+ }
+
+ if (url->protocol == PROTOCOL_MAILTO) {
+ if (url->userlen == 0) {
+ return FALSE;
+ }
+ }
+ /* Also check max urls */
+ if (cbd->task->cfg && cbd->task->cfg->max_urls > 0) {
+ if (kh_size(MESSAGE_FIELD(task, urls)) > cbd->task->cfg->max_urls) {
+ msg_err_task("part has too many URLs, we cannot process more: "
+ "%d urls extracted ",
+ (guint) kh_size(MESSAGE_FIELD(task, urls)));
+
+ return FALSE;
+ }
+ }
+
+ url->flags |= RSPAMD_URL_FLAG_FROM_TEXT;
+
+ if (rspamd_url_set_add_or_increase(MESSAGE_FIELD(task, urls), url, false) &&
+ cbd->part->mime_part->urls) {
+ url->part_order = cbd->cur_part_order++;
+
+ if (cbd->cur_url_order) {
+ url->order = (*cbd->cur_url_order)++;
+ }
+ g_ptr_array_add(cbd->part->mime_part->urls, url);
+ }
+
+ cbd->part->exceptions = g_list_prepend(
+ cbd->part->exceptions,
+ ex);
+
+ /* We also search the query for additional url inside */
+ if (url->querylen > 0) {
+ rspamd_url_find_multiple(task->task_pool,
+ rspamd_url_query_unsafe(url), url->querylen,
+ RSPAMD_URL_FIND_ALL, NULL,
+ rspamd_url_query_callback, cbd);
+ }
+
+ return TRUE;
+}
+
+void rspamd_url_text_extract(rspamd_mempool_t *pool,
+ struct rspamd_task *task,
+ struct rspamd_mime_text_part *part,
+ uint16_t *cur_url_order,
+ enum rspamd_url_find_type how)
+{
+ struct rspamd_url_mimepart_cbdata mcbd;
+
+ if (part->utf_stripped_content == NULL || part->utf_stripped_content->len == 0) {
+ msg_warn_task("got empty text part");
+ return;
+ }
+
+ mcbd.task = task;
+ mcbd.part = part;
+ mcbd.url_len = 0;
+ mcbd.cur_url_order = cur_url_order;
+ mcbd.cur_part_order = 0;
+
+ rspamd_url_find_multiple(task->task_pool, part->utf_stripped_content->data,
+ part->utf_stripped_content->len, how, part->newlines,
+ rspamd_url_text_part_callback, &mcbd);
+}
+
+void rspamd_url_find_multiple(rspamd_mempool_t *pool,
+ const gchar *in,
+ gsize inlen,
+ enum rspamd_url_find_type how,
+ GPtrArray *nlines,
+ url_insert_function func,
+ gpointer ud)
+{
+ struct url_callback_data cb;
+
+ g_assert(in != NULL);
+
+ if (inlen == 0) {
+ inlen = strlen(in);
+ }
+
+ memset(&cb, 0, sizeof(cb));
+ cb.begin = in;
+ cb.end = in + inlen;
+ cb.how = how;
+ cb.pool = pool;
+
+ cb.funcd = ud;
+ cb.func = func;
+ cb.newlines = nlines;
+
+ if (how == RSPAMD_URL_FIND_ALL) {
+ if (url_scanner->search_trie_full) {
+ cb.matchers = url_scanner->matchers_full;
+ rspamd_multipattern_lookup(url_scanner->search_trie_full,
+ in, inlen,
+ rspamd_url_trie_generic_callback_multiple, &cb, NULL);
+ }
+ else {
+ cb.matchers = url_scanner->matchers_strict;
+ rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+ in, inlen,
+ rspamd_url_trie_generic_callback_multiple, &cb, NULL);
+ }
+ }
+ else {
+ cb.matchers = url_scanner->matchers_strict;
+ rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+ in, inlen,
+ rspamd_url_trie_generic_callback_multiple, &cb, NULL);
+ }
+}
+
+void rspamd_url_find_single(rspamd_mempool_t *pool,
+ const gchar *in,
+ gsize inlen,
+ enum rspamd_url_find_type how,
+ url_insert_function func,
+ gpointer ud)
+{
+ struct url_callback_data cb;
+
+ g_assert(in != NULL);
+
+ if (inlen == 0) {
+ inlen = strlen(in);
+ }
+
+ /*
+ * We might have a situation when we need to parse URLs on config file
+ * parsing, but there is no valid url_scanner loaded. Hence, we just load
+ * some defaults and it should be fine...
+ */
+ if (url_scanner == NULL) {
+ rspamd_url_init(NULL);
+ }
+
+ memset(&cb, 0, sizeof(cb));
+ cb.begin = in;
+ cb.end = in + inlen;
+ cb.how = how;
+ cb.pool = pool;
+
+ cb.funcd = ud;
+ cb.func = func;
+
+ if (how == RSPAMD_URL_FIND_ALL) {
+ if (url_scanner->search_trie_full) {
+ cb.matchers = url_scanner->matchers_full;
+ rspamd_multipattern_lookup(url_scanner->search_trie_full,
+ in, inlen,
+ rspamd_url_trie_generic_callback_single, &cb, NULL);
+ }
+ else {
+ cb.matchers = url_scanner->matchers_strict;
+ rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+ in, inlen,
+ rspamd_url_trie_generic_callback_single, &cb, NULL);
+ }
+ }
+ else {
+ cb.matchers = url_scanner->matchers_strict;
+ rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+ in, inlen,
+ rspamd_url_trie_generic_callback_single, &cb, NULL);
+ }
+}
+
+
+gboolean
+rspamd_url_task_subject_callback(struct rspamd_url *url, gsize start_offset,
+ gsize end_offset, gpointer ud)
+{
+ struct rspamd_task *task = ud;
+ gchar *url_str = NULL;
+ struct rspamd_url *query_url;
+ gint rc;
+ gboolean prefix_added;
+
+ /* It is just a displayed URL, we should not check it for certain things */
+ url->flags |= RSPAMD_URL_FLAG_HTML_DISPLAYED | RSPAMD_URL_FLAG_SUBJECT;
+
+ if (url->protocol == PROTOCOL_MAILTO) {
+ if (url->userlen == 0) {
+ return FALSE;
+ }
+ }
+
+ rspamd_url_set_add_or_increase(MESSAGE_FIELD(task, urls), url, false);
+
+ /* We also search the query for additional url inside */
+ if (url->querylen > 0) {
+ if (rspamd_url_find(task->task_pool, rspamd_url_query_unsafe(url), url->querylen,
+ &url_str, RSPAMD_URL_FIND_ALL, NULL, &prefix_added)) {
+
+ query_url = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct rspamd_url));
+ rc = rspamd_url_parse(query_url,
+ url_str,
+ strlen(url_str),
+ task->task_pool,
+ RSPAMD_URL_PARSE_TEXT);
+
+ if (rc == URI_ERRNO_OK &&
+ url->hostlen > 0) {
+ msg_debug_task("found url %s in query of url"
+ " %*s",
+ url_str, url->querylen, rspamd_url_query_unsafe(url));
+
+ if (prefix_added) {
+ query_url->flags |= RSPAMD_URL_FLAG_SCHEMALESS;
+ }
+
+ if (query_url->protocol == PROTOCOL_MAILTO) {
+ if (query_url->userlen == 0) {
+ return TRUE;
+ }
+ }
+
+ rspamd_url_set_add_or_increase(MESSAGE_FIELD(task, urls),
+ query_url, false);
+ }
+ }
+ }
+
+ return TRUE;
+}
+
+static inline khint_t
+rspamd_url_hash(struct rspamd_url *url)
+{
+ if (url->urllen > 0) {
+ return (khint_t) rspamd_cryptobox_fast_hash(url->string, url->urllen,
+ rspamd_hash_seed());
+ }
+
+ return 0;
+}
+
+static inline khint_t
+rspamd_url_host_hash(struct rspamd_url *url)
+{
+ if (url->hostlen > 0) {
+ return (khint_t) rspamd_cryptobox_fast_hash(rspamd_url_host_unsafe(url),
+ url->hostlen,
+ rspamd_hash_seed());
+ }
+
+ return 0;
+}
+
+/* Compare two emails for building emails tree */
+static inline bool
+rspamd_emails_cmp(struct rspamd_url *u1, struct rspamd_url *u2)
+{
+ gint r;
+
+ if (u1->hostlen != u2->hostlen || u1->hostlen == 0) {
+ return FALSE;
+ }
+ else {
+ if ((r = rspamd_lc_cmp(rspamd_url_host_unsafe(u1),
+ rspamd_url_host_unsafe(u2), u1->hostlen)) == 0) {
+ if (u1->userlen != u2->userlen || u1->userlen == 0) {
+ return FALSE;
+ }
+ else {
+ return (rspamd_lc_cmp(rspamd_url_user_unsafe(u1),
+ rspamd_url_user_unsafe(u2),
+ u1->userlen) == 0);
+ }
+ }
+ else {
+ return r == 0;
+ }
+ }
+
+ return FALSE;
+}
+
+static inline bool
+rspamd_urls_cmp(struct rspamd_url *u1, struct rspamd_url *u2)
+{
+ int r = 0;
+
+ if (u1->protocol != u2->protocol || u1->urllen != u2->urllen) {
+ return false;
+ }
+ else {
+ if (u1->protocol & PROTOCOL_MAILTO) {
+ return rspamd_emails_cmp(u1, u2);
+ }
+
+ r = memcmp(u1->string, u2->string, u1->urllen);
+ }
+
+ return r == 0;
+}
+
+static inline bool
+rspamd_urls_host_cmp(struct rspamd_url *u1, struct rspamd_url *u2)
+{
+ int r = 0;
+
+ if (u1->hostlen != u2->hostlen) {
+ return false;
+ }
+ else {
+ r = memcmp(rspamd_url_host_unsafe(u1), rspamd_url_host_unsafe(u2),
+ u1->hostlen);
+ }
+
+ return r == 0;
+}
+
+gsize rspamd_url_decode(gchar *dst, const gchar *src, gsize size)
+{
+ gchar *d, ch, c, decoded;
+ const gchar *s;
+ enum {
+ sw_usual = 0,
+ sw_quoted,
+ sw_quoted_second
+ } state;
+
+ d = dst;
+ s = src;
+
+ state = 0;
+ decoded = 0;
+
+ while (size--) {
+
+ ch = *s++;
+
+ switch (state) {
+ case sw_usual:
+
+ if (ch == '%') {
+ state = sw_quoted;
+ break;
+ }
+ else if (ch == '+') {
+ *d++ = ' ';
+ }
+ else {
+ *d++ = ch;
+ }
+ break;
+
+ case sw_quoted:
+
+ if (ch >= '0' && ch <= '9') {
+ decoded = (ch - '0');
+ state = sw_quoted_second;
+ break;
+ }
+
+ c = (ch | 0x20);
+ if (c >= 'a' && c <= 'f') {
+ decoded = (c - 'a' + 10);
+ state = sw_quoted_second;
+ break;
+ }
+
+ /* the invalid quoted character */
+
+ state = sw_usual;
+
+ *d++ = ch;
+
+ break;
+
+ case sw_quoted_second:
+
+ state = sw_usual;
+
+ if (ch >= '0' && ch <= '9') {
+ ch = ((decoded << 4) + ch - '0');
+ *d++ = ch;
+
+ break;
+ }
+
+ c = (u_char) (ch | 0x20);
+ if (c >= 'a' && c <= 'f') {
+ ch = ((decoded << 4) + c - 'a' + 10);
+
+ *d++ = ch;
+ break;
+ }
+
+ /* the invalid quoted character */
+ break;
+ }
+ }
+
+ return (d - dst);
+}
+
+enum rspamd_url_char_class {
+ RSPAMD_URL_UNRESERVED = (1 << 0),
+ RSPAMD_URL_SUBDELIM = (1 << 1),
+ RSPAMD_URL_PATHSAFE = (1 << 2),
+ RSPAMD_URL_QUERYSAFE = (1 << 3),
+ RSPAMD_URL_FRAGMENTSAFE = (1 << 4),
+ RSPAMD_URL_HOSTSAFE = (1 << 5),
+ RSPAMD_URL_USERSAFE = (1 << 6),
+};
+
+#define RSPAMD_URL_FLAGS_HOSTSAFE (RSPAMD_URL_UNRESERVED | RSPAMD_URL_HOSTSAFE | RSPAMD_URL_SUBDELIM)
+#define RSPAMD_URL_FLAGS_USERSAFE (RSPAMD_URL_UNRESERVED | RSPAMD_URL_USERSAFE | RSPAMD_URL_SUBDELIM)
+#define RSPAMD_URL_FLAGS_PATHSAFE (RSPAMD_URL_UNRESERVED | RSPAMD_URL_PATHSAFE | RSPAMD_URL_SUBDELIM)
+#define RSPAMD_URL_FLAGS_QUERYSAFE (RSPAMD_URL_UNRESERVED | RSPAMD_URL_QUERYSAFE | RSPAMD_URL_SUBDELIM)
+#define RSPAMD_URL_FLAGS_FRAGMENTSAFE (RSPAMD_URL_UNRESERVED | RSPAMD_URL_FRAGMENTSAFE | RSPAMD_URL_SUBDELIM)
+
+static const unsigned char rspamd_url_encoding_classes[256] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0 /* */, RSPAMD_URL_SUBDELIM /* ! */, 0 /* " */, 0 /* # */,
+ RSPAMD_URL_SUBDELIM /* $ */, 0 /* % */, RSPAMD_URL_SUBDELIM /* & */,
+ RSPAMD_URL_SUBDELIM /* ' */, RSPAMD_URL_SUBDELIM /* ( */,
+ RSPAMD_URL_SUBDELIM /* ) */, RSPAMD_URL_SUBDELIM /* * */,
+ RSPAMD_URL_SUBDELIM /* + */, RSPAMD_URL_SUBDELIM /* , */,
+ RSPAMD_URL_UNRESERVED /* - */, RSPAMD_URL_UNRESERVED /* . */,
+ RSPAMD_URL_PATHSAFE | RSPAMD_URL_QUERYSAFE | RSPAMD_URL_FRAGMENTSAFE /* / */,
+ RSPAMD_URL_UNRESERVED /* 0 */, RSPAMD_URL_UNRESERVED /* 1 */,
+ RSPAMD_URL_UNRESERVED /* 2 */, RSPAMD_URL_UNRESERVED /* 3 */,
+ RSPAMD_URL_UNRESERVED /* 4 */, RSPAMD_URL_UNRESERVED /* 5 */,
+ RSPAMD_URL_UNRESERVED /* 6 */, RSPAMD_URL_UNRESERVED /* 7 */,
+ RSPAMD_URL_UNRESERVED /* 8 */, RSPAMD_URL_UNRESERVED /* 9 */,
+ RSPAMD_URL_USERSAFE | RSPAMD_URL_HOSTSAFE | RSPAMD_URL_PATHSAFE | RSPAMD_URL_QUERYSAFE | RSPAMD_URL_FRAGMENTSAFE /* : */,
+ RSPAMD_URL_SUBDELIM /* ; */, 0 /* < */, RSPAMD_URL_SUBDELIM /* = */, 0 /* > */,
+ RSPAMD_URL_QUERYSAFE | RSPAMD_URL_FRAGMENTSAFE /* ? */,
+ RSPAMD_URL_PATHSAFE | RSPAMD_URL_QUERYSAFE | RSPAMD_URL_FRAGMENTSAFE /* @ */,
+ RSPAMD_URL_UNRESERVED /* A */, RSPAMD_URL_UNRESERVED /* B */,
+ RSPAMD_URL_UNRESERVED /* C */, RSPAMD_URL_UNRESERVED /* D */,
+ RSPAMD_URL_UNRESERVED /* E */, RSPAMD_URL_UNRESERVED /* F */,
+ RSPAMD_URL_UNRESERVED /* G */, RSPAMD_URL_UNRESERVED /* H */,
+ RSPAMD_URL_UNRESERVED /* I */, RSPAMD_URL_UNRESERVED /* J */,
+ RSPAMD_URL_UNRESERVED /* K */, RSPAMD_URL_UNRESERVED /* L */,
+ RSPAMD_URL_UNRESERVED /* M */, RSPAMD_URL_UNRESERVED /* N */,
+ RSPAMD_URL_UNRESERVED /* O */, RSPAMD_URL_UNRESERVED /* P */,
+ RSPAMD_URL_UNRESERVED /* Q */, RSPAMD_URL_UNRESERVED /* R */,
+ RSPAMD_URL_UNRESERVED /* S */, RSPAMD_URL_UNRESERVED /* T */,
+ RSPAMD_URL_UNRESERVED /* U */, RSPAMD_URL_UNRESERVED /* V */,
+ RSPAMD_URL_UNRESERVED /* W */, RSPAMD_URL_UNRESERVED /* X */,
+ RSPAMD_URL_UNRESERVED /* Y */, RSPAMD_URL_UNRESERVED /* Z */,
+ RSPAMD_URL_HOSTSAFE /* [ */, 0 /* \ */, RSPAMD_URL_HOSTSAFE /* ] */, 0 /* ^ */,
+ RSPAMD_URL_UNRESERVED /* _ */, 0 /* ` */, RSPAMD_URL_UNRESERVED /* a */,
+ RSPAMD_URL_UNRESERVED /* b */, RSPAMD_URL_UNRESERVED /* c */,
+ RSPAMD_URL_UNRESERVED /* d */, RSPAMD_URL_UNRESERVED /* e */,
+ RSPAMD_URL_UNRESERVED /* f */, RSPAMD_URL_UNRESERVED /* g */,
+ RSPAMD_URL_UNRESERVED /* h */, RSPAMD_URL_UNRESERVED /* i */,
+ RSPAMD_URL_UNRESERVED /* j */, RSPAMD_URL_UNRESERVED /* k */,
+ RSPAMD_URL_UNRESERVED /* l */, RSPAMD_URL_UNRESERVED /* m */,
+ RSPAMD_URL_UNRESERVED /* n */, RSPAMD_URL_UNRESERVED /* o */,
+ RSPAMD_URL_UNRESERVED /* p */, RSPAMD_URL_UNRESERVED /* q */,
+ RSPAMD_URL_UNRESERVED /* r */, RSPAMD_URL_UNRESERVED /* s */,
+ RSPAMD_URL_UNRESERVED /* t */, RSPAMD_URL_UNRESERVED /* u */,
+ RSPAMD_URL_UNRESERVED /* v */, RSPAMD_URL_UNRESERVED /* w */,
+ RSPAMD_URL_UNRESERVED /* x */, RSPAMD_URL_UNRESERVED /* y */,
+ RSPAMD_URL_UNRESERVED /* z */, 0 /* { */, 0 /* | */, 0 /* } */,
+ RSPAMD_URL_UNRESERVED /* ~ */, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+#define CHECK_URL_COMPONENT(beg, len, flags) \
+ do { \
+ for (i = 0; i < (len); i++) { \
+ if ((rspamd_url_encoding_classes[(guchar) (beg)[i]] & (flags)) == 0) { \
+ dlen += 2; \
+ } \
+ } \
+ } while (0)
+
+#define ENCODE_URL_COMPONENT(beg, len, flags) \
+ do { \
+ for (i = 0; i < (len) && dend > d; i++) { \
+ if ((rspamd_url_encoding_classes[(guchar) (beg)[i]] & (flags)) == 0) { \
+ *d++ = '%'; \
+ *d++ = hexdigests[(guchar) ((beg)[i] >> 4) & 0xf]; \
+ *d++ = hexdigests[(guchar) (beg)[i] & 0xf]; \
+ } \
+ else { \
+ *d++ = (beg)[i]; \
+ } \
+ } \
+ } while (0)
+
+const gchar *
+rspamd_url_encode(struct rspamd_url *url, gsize *pdlen,
+ rspamd_mempool_t *pool)
+{
+ guchar *dest, *d, *dend;
+ static const gchar hexdigests[16] = "0123456789ABCDEF";
+ guint i;
+ gsize dlen = 0;
+
+ g_assert(pdlen != NULL && url != NULL && pool != NULL);
+
+ CHECK_URL_COMPONENT(rspamd_url_host_unsafe(url), url->hostlen,
+ RSPAMD_URL_FLAGS_HOSTSAFE);
+ CHECK_URL_COMPONENT(rspamd_url_user_unsafe(url), url->userlen,
+ RSPAMD_URL_FLAGS_USERSAFE);
+ CHECK_URL_COMPONENT(rspamd_url_data_unsafe(url), url->datalen,
+ RSPAMD_URL_FLAGS_PATHSAFE);
+ CHECK_URL_COMPONENT(rspamd_url_query_unsafe(url), url->querylen,
+ RSPAMD_URL_FLAGS_QUERYSAFE);
+ CHECK_URL_COMPONENT(rspamd_url_fragment_unsafe(url), url->fragmentlen,
+ RSPAMD_URL_FLAGS_FRAGMENTSAFE);
+
+ if (dlen == 0) {
+ *pdlen = url->urllen;
+
+ return url->string;
+ }
+
+ /* Need to encode */
+ dlen += url->urllen + sizeof("telephone://"); /* Protocol hack */
+ dest = rspamd_mempool_alloc(pool, dlen + 1);
+ d = dest;
+ dend = d + dlen;
+
+ if (url->protocollen > 0) {
+ if (!(url->protocol & PROTOCOL_UNKNOWN)) {
+ const gchar *known_proto = rspamd_url_protocol_name(url->protocol);
+ d += rspamd_snprintf((gchar *) d, dend - d,
+ "%s://",
+ known_proto);
+ }
+ else {
+ d += rspamd_snprintf((gchar *) d, dend - d,
+ "%*s://",
+ (gint) url->protocollen, url->string);
+ }
+ }
+ else {
+ d += rspamd_snprintf((gchar *) d, dend - d, "http://");
+ }
+
+ if (url->userlen > 0) {
+ ENCODE_URL_COMPONENT(rspamd_url_user_unsafe(url), url->userlen,
+ RSPAMD_URL_FLAGS_USERSAFE);
+ *d++ = '@';
+ }
+
+ ENCODE_URL_COMPONENT(rspamd_url_host_unsafe(url), url->hostlen,
+ RSPAMD_URL_FLAGS_HOSTSAFE);
+
+ if (url->datalen > 0) {
+ *d++ = '/';
+ ENCODE_URL_COMPONENT(rspamd_url_data_unsafe(url), url->datalen,
+ RSPAMD_URL_FLAGS_PATHSAFE);
+ }
+
+ if (url->querylen > 0) {
+ *d++ = '?';
+ ENCODE_URL_COMPONENT(rspamd_url_query_unsafe(url), url->querylen,
+ RSPAMD_URL_FLAGS_QUERYSAFE);
+ }
+
+ if (url->fragmentlen > 0) {
+ *d++ = '#';
+ ENCODE_URL_COMPONENT(rspamd_url_fragment_unsafe(url), url->fragmentlen,
+ RSPAMD_URL_FLAGS_FRAGMENTSAFE);
+ }
+
+ *pdlen = (d - dest);
+
+ return (const gchar *) dest;
+}
+
+gboolean
+rspamd_url_is_domain(int c)
+{
+ return is_domain((guchar) c);
+}
+
+const gchar *
+rspamd_url_protocol_name(enum rspamd_url_protocol proto)
+{
+ const gchar *ret = "unknown";
+
+ switch (proto) {
+ case PROTOCOL_HTTP:
+ ret = "http";
+ break;
+ case PROTOCOL_HTTPS:
+ ret = "https";
+ break;
+ case PROTOCOL_FTP:
+ ret = "ftp";
+ break;
+ case PROTOCOL_FILE:
+ ret = "file";
+ break;
+ case PROTOCOL_MAILTO:
+ ret = "mailto";
+ break;
+ case PROTOCOL_TELEPHONE:
+ ret = "telephone";
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+enum rspamd_url_protocol
+rspamd_url_protocol_from_string(const gchar *str)
+{
+ enum rspamd_url_protocol ret = PROTOCOL_UNKNOWN;
+
+ if (strcmp(str, "http") == 0) {
+ ret = PROTOCOL_HTTP;
+ }
+ else if (strcmp(str, "https") == 0) {
+ ret = PROTOCOL_HTTPS;
+ }
+ else if (strcmp(str, "mailto") == 0) {
+ ret = PROTOCOL_MAILTO;
+ }
+ else if (strcmp(str, "ftp") == 0) {
+ ret = PROTOCOL_FTP;
+ }
+ else if (strcmp(str, "file") == 0) {
+ ret = PROTOCOL_FILE;
+ }
+ else if (strcmp(str, "telephone") == 0) {
+ ret = PROTOCOL_TELEPHONE;
+ }
+
+ return ret;
+}
+
+
+bool rspamd_url_set_add_or_increase(khash_t(rspamd_url_hash) * set,
+ struct rspamd_url *u,
+ bool enforce_replace)
+{
+ khiter_t k;
+ gint r;
+
+ k = kh_get(rspamd_url_hash, set, u);
+
+ if (k != kh_end(set)) {
+ /* Existing url */
+ struct rspamd_url *ex = kh_key(set, k);
+#define SUSPICIOUS_URL_FLAGS (RSPAMD_URL_FLAG_PHISHED | RSPAMD_URL_FLAG_OBSCURED | RSPAMD_URL_FLAG_ZW_SPACES)
+ if (enforce_replace) {
+ kh_key(set, k) = u;
+ u->count++;
+ }
+ else {
+ if (u->flags & SUSPICIOUS_URL_FLAGS) {
+ if (!(ex->flags & SUSPICIOUS_URL_FLAGS)) {
+ /* Propagate new url to an old one */
+ kh_key(set, k) = u;
+ u->count++;
+ }
+ else {
+ ex->count++;
+ }
+ }
+ else {
+ ex->count++;
+ }
+ }
+
+ return false;
+ }
+ else {
+ k = kh_put(rspamd_url_hash, set, u, &r);
+ }
+
+ return true;
+}
+
+struct rspamd_url *
+rspamd_url_set_add_or_return(khash_t(rspamd_url_hash) * set,
+ struct rspamd_url *u)
+{
+ khiter_t k;
+ gint r;
+
+ if (set) {
+ k = kh_get(rspamd_url_hash, set, u);
+
+ if (k != kh_end(set)) {
+ return kh_key(set, k);
+ }
+ else {
+ k = kh_put(rspamd_url_hash, set, u, &r);
+
+ return kh_key(set, k);
+ }
+ }
+
+ return NULL;
+}
+
+bool rspamd_url_host_set_add(khash_t(rspamd_url_host_hash) * set,
+ struct rspamd_url *u)
+{
+ gint r;
+
+ if (set) {
+ kh_put(rspamd_url_host_hash, set, u, &r);
+
+ if (r == 0) {
+ return false;
+ }
+
+ return true;
+ }
+
+ return false;
+}
+
+bool rspamd_url_set_has(khash_t(rspamd_url_hash) * set, struct rspamd_url *u)
+{
+ khiter_t k;
+
+ if (set) {
+ k = kh_get(rspamd_url_hash, set, u);
+
+ if (k == kh_end(set)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ return false;
+}
+
+bool rspamd_url_host_set_has(khash_t(rspamd_url_host_hash) * set, struct rspamd_url *u)
+{
+ khiter_t k;
+
+ if (set) {
+ k = kh_get(rspamd_url_host_hash, set, u);
+
+ if (k == kh_end(set)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ return false;
+}
+
+bool rspamd_url_flag_from_string(const gchar *str, gint *flag)
+{
+ gint h = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT,
+ str, strlen(str), 0);
+
+ for (int i = 0; i < G_N_ELEMENTS(url_flag_names); i++) {
+ if (url_flag_names[i].hash == h) {
+ *flag |= url_flag_names[i].flag;
+
+ return true;
+ }
+ }
+
+ return false;
+}
+
+
+const gchar *
+rspamd_url_flag_to_string(int flag)
+{
+ for (int i = 0; i < G_N_ELEMENTS(url_flag_names); i++) {
+ if (url_flag_names[i].flag & flag) {
+ return url_flag_names[i].name;
+ }
+ }
+
+ return NULL;
+}
+
+inline int
+rspamd_url_cmp(const struct rspamd_url *u1, const struct rspamd_url *u2)
+{
+ int min_len = MIN(u1->urllen, u2->urllen);
+ int r;
+
+ if (u1->protocol != u2->protocol) {
+ return u1->protocol - u2->protocol;
+ }
+
+ if (u1->protocol & PROTOCOL_MAILTO) {
+ /* Emails specialisation (hosts must be compared in a case insensitive matter */
+ min_len = MIN(u1->hostlen, u2->hostlen);
+
+ if ((r = rspamd_lc_cmp(rspamd_url_host_unsafe(u1),
+ rspamd_url_host_unsafe(u2), min_len)) == 0) {
+ if (u1->hostlen == u2->hostlen) {
+ if (u1->userlen != u2->userlen || u1->userlen == 0) {
+ r = (int) u1->userlen - (int) u2->userlen;
+ }
+ else {
+ r = memcmp(rspamd_url_user_unsafe(u1),
+ rspamd_url_user_unsafe(u2),
+ u1->userlen);
+ }
+ }
+ else {
+ r = u1->hostlen - u2->hostlen;
+ }
+ }
+ }
+ else {
+ if (u1->urllen != u2->urllen) {
+ /* Different length, compare common part and then compare length */
+ r = memcmp(u1->string, u2->string, min_len);
+
+ if (r == 0) {
+ r = u1->urllen - u2->urllen;
+ }
+ }
+ else {
+ /* Equal length */
+ r = memcmp(u1->string, u2->string, u1->urllen);
+ }
+ }
+
+ return r;
+}
+
+int rspamd_url_cmp_qsort(const void *_u1, const void *_u2)
+{
+ const struct rspamd_url *u1 = *(struct rspamd_url **) _u1,
+ *u2 = *(struct rspamd_url **) _u2;
+
+ return rspamd_url_cmp(u1, u2);
+}
diff --git a/src/libserver/url.h b/src/libserver/url.h
new file mode 100644
index 0000000..d1fb8c9
--- /dev/null
+++ b/src/libserver/url.h
@@ -0,0 +1,430 @@
+/* URL check functions */
+#ifndef URL_H
+#define URL_H
+
+#include "config.h"
+#include "mem_pool.h"
+#include "khash.h"
+#include "fstring.h"
+#include "libutil/cxx/utf8_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_mime_text_part;
+
+enum rspamd_url_flags {
+ RSPAMD_URL_FLAG_PHISHED = 1u << 0u,
+ RSPAMD_URL_FLAG_NUMERIC = 1u << 1u,
+ RSPAMD_URL_FLAG_OBSCURED = 1u << 2u,
+ RSPAMD_URL_FLAG_REDIRECTED = 1u << 3u,
+ RSPAMD_URL_FLAG_HTML_DISPLAYED = 1u << 4u,
+ RSPAMD_URL_FLAG_FROM_TEXT = 1u << 5u,
+ RSPAMD_URL_FLAG_SUBJECT = 1u << 6u,
+ RSPAMD_URL_FLAG_HOSTENCODED = 1u << 7u,
+ RSPAMD_URL_FLAG_SCHEMAENCODED = 1u << 8u,
+ RSPAMD_URL_FLAG_PATHENCODED = 1u << 9u,
+ RSPAMD_URL_FLAG_QUERYENCODED = 1u << 10u,
+ RSPAMD_URL_FLAG_MISSINGSLASHES = 1u << 11u,
+ RSPAMD_URL_FLAG_IDN = 1u << 12u,
+ RSPAMD_URL_FLAG_HAS_PORT = 1u << 13u,
+ RSPAMD_URL_FLAG_HAS_USER = 1u << 14u,
+ RSPAMD_URL_FLAG_SCHEMALESS = 1u << 15u,
+ RSPAMD_URL_FLAG_UNNORMALISED = 1u << 16u,
+ RSPAMD_URL_FLAG_ZW_SPACES = 1u << 17u,
+ RSPAMD_URL_FLAG_DISPLAY_URL = 1u << 18u,
+ RSPAMD_URL_FLAG_IMAGE = 1u << 19u,
+ RSPAMD_URL_FLAG_QUERY = 1u << 20u,
+ RSPAMD_URL_FLAG_CONTENT = 1u << 21u,
+ RSPAMD_URL_FLAG_NO_TLD = 1u << 22u,
+ RSPAMD_URL_FLAG_TRUNCATED = 1u << 23u,
+ RSPAMD_URL_FLAG_REDIRECT_TARGET = 1u << 24u,
+ RSPAMD_URL_FLAG_INVISIBLE = 1u << 25u,
+ RSPAMD_URL_FLAG_SPECIAL = 1u << 26u,
+
+};
+#define RSPAMD_URL_MAX_FLAG_SHIFT (26u)
+
+struct rspamd_url_tag {
+ const gchar *data;
+ struct rspamd_url_tag *prev, *next;
+};
+
+struct rspamd_url_ext;
+/**
+ * URL structure
+ */
+struct rspamd_url {
+ char *string;
+ char *raw;
+ struct rspamd_url_ext *ext;
+
+ uint32_t flags;
+
+ uint8_t protocol;
+ uint8_t protocollen;
+
+ uint16_t hostshift;
+ uint16_t datashift;
+ uint16_t queryshift;
+ uint16_t fragmentshift;
+ uint16_t tldshift;
+ guint16 usershift;
+ guint16 userlen;
+
+ uint16_t hostlen;
+ uint16_t datalen;
+ uint16_t querylen;
+ uint16_t fragmentlen;
+ uint16_t tldlen;
+ uint16_t count;
+ uint16_t urllen;
+ uint16_t rawlen;
+
+ /* Absolute order of the URL in a message */
+ uint16_t order;
+ /* Order of the URL in a specific part of message */
+ uint16_t part_order;
+};
+
+/**
+ * Rarely used url fields
+ */
+struct rspamd_url_ext {
+ gchar *visible_part;
+ struct rspamd_url *linked_url;
+
+ guint16 port;
+};
+
+#define rspamd_url_user(u) ((u)->userlen > 0 ? (u)->string + (u)->usershift : NULL)
+#define rspamd_url_user_unsafe(u) ((u)->string + (u)->usershift)
+
+#define rspamd_url_host(u) ((u)->hostlen > 0 ? (u)->string + (u)->hostshift : NULL)
+#define rspamd_url_host_unsafe(u) ((u)->string + (u)->hostshift)
+#define rspamd_url_tld_unsafe(u) ((u)->string + (u)->tldshift)
+
+#define rspamd_url_data_unsafe(u) ((u)->string + (u)->datashift)
+#define rspamd_url_query_unsafe(u) ((u)->string + (u)->queryshift)
+#define rspamd_url_fragment_unsafe(u) ((u)->string + (u)->fragmentshift)
+
+enum uri_errno {
+ URI_ERRNO_OK = 0, /* Parsing went well */
+ URI_ERRNO_EMPTY, /* The URI string was empty */
+ URI_ERRNO_INVALID_PROTOCOL, /* No protocol was found */
+ URI_ERRNO_INVALID_PORT, /* Port number is bad */
+ URI_ERRNO_BAD_ENCODING, /* Bad characters encoding */
+ URI_ERRNO_BAD_FORMAT,
+ URI_ERRNO_TLD_MISSING,
+ URI_ERRNO_HOST_MISSING,
+ URI_ERRNO_TOO_LONG,
+};
+
+enum rspamd_url_protocol {
+ PROTOCOL_FILE = 1u << 0u,
+ PROTOCOL_FTP = 1u << 1u,
+ PROTOCOL_HTTP = 1u << 2u,
+ PROTOCOL_HTTPS = 1u << 3u,
+ PROTOCOL_MAILTO = 1u << 4u,
+ PROTOCOL_TELEPHONE = 1u << 5u,
+ PROTOCOL_UNKNOWN = 1u << 7u,
+};
+
+enum rspamd_url_parse_flags {
+ RSPAMD_URL_PARSE_TEXT = 0u,
+ RSPAMD_URL_PARSE_HREF = (1u << 0u),
+ RSPAMD_URL_PARSE_CHECK = (1u << 1u),
+};
+
+enum rspamd_url_find_type {
+ RSPAMD_URL_FIND_ALL = 0,
+ RSPAMD_URL_FIND_STRICT,
+};
+
+/**
+ * Initialize url library
+ * @param cfg
+ */
+void rspamd_url_init(const gchar *tld_file);
+
+void rspamd_url_deinit(void);
+
+/*
+ * Parse urls inside text
+ * @param pool memory pool
+ * @param task task object
+ * @param part current text part
+ * @param is_html turn on html heuristic
+ */
+void rspamd_url_text_extract(rspamd_mempool_t *pool,
+ struct rspamd_task *task,
+ struct rspamd_mime_text_part *part,
+ uint16_t *cur_order,
+ enum rspamd_url_find_type how);
+
+/*
+ * Parse a single url into an uri structure
+ * @param pool memory pool
+ * @param uristring text form of url
+ * @param uri url object, must be pre allocated
+ */
+enum uri_errno rspamd_url_parse(struct rspamd_url *uri,
+ gchar *uristring,
+ gsize len,
+ rspamd_mempool_t *pool,
+ enum rspamd_url_parse_flags flags);
+
+/*
+ * Try to extract url from a text
+ * @param pool memory pool
+ * @param begin begin of text
+ * @param len length of text
+ * @param start storage for start position of url found (or NULL)
+ * @param end storage for end position of url found (or NULL)
+ * @param url_str storage for url string(or NULL)
+ * @return TRUE if url is found in specified text
+ */
+gboolean rspamd_url_find(rspamd_mempool_t *pool,
+ const gchar *begin, gsize len,
+ gchar **url_str,
+ enum rspamd_url_find_type how,
+ goffset *url_pos,
+ gboolean *prefix_added);
+
+/*
+ * Return text representation of url parsing error
+ */
+const gchar *rspamd_url_strerror(int err);
+
+
+/**
+ * Find TLD for a specified host string
+ * @param in input host
+ * @param inlen length of input
+ * @param out output rspamd_ftok_t with tld position
+ * @return TRUE if tld has been found
+ */
+gboolean rspamd_url_find_tld(const gchar *in, gsize inlen, rspamd_ftok_t *out);
+
+typedef gboolean (*url_insert_function)(struct rspamd_url *url,
+ gsize start_offset, gsize end_offset, void *ud);
+
+/**
+ * Search for multiple urls in text and call `func` for each url found
+ * @param pool
+ * @param in
+ * @param inlen
+ * @param is_html
+ * @param func
+ * @param ud
+ */
+void rspamd_url_find_multiple(rspamd_mempool_t *pool,
+ const gchar *in, gsize inlen,
+ enum rspamd_url_find_type how,
+ GPtrArray *nlines,
+ url_insert_function func,
+ gpointer ud);
+
+/**
+ * Search for a single url in text and call `func` for each url found
+ * @param pool
+ * @param in
+ * @param inlen
+ * @param is_html
+ * @param func
+ * @param ud
+ */
+void rspamd_url_find_single(rspamd_mempool_t *pool,
+ const gchar *in, gsize inlen,
+ enum rspamd_url_find_type how,
+ url_insert_function func,
+ gpointer ud);
+
+/**
+ * Generic callback to insert URLs into rspamd_task
+ * @param url
+ * @param start_offset
+ * @param end_offset
+ * @param ud
+ */
+gboolean rspamd_url_task_subject_callback(struct rspamd_url *url,
+ gsize start_offset,
+ gsize end_offset, gpointer ud);
+
+/**
+ * Decode URL encoded string in-place and return new length of a string, src and dst are NULL terminated
+ * @param dst
+ * @param src
+ * @param size
+ * @return
+ */
+gsize rspamd_url_decode(gchar *dst, const gchar *src, gsize size);
+
+/**
+ * Encode url if needed. In this case, memory is allocated from the specific pool.
+ * Returns pointer to begin and encoded length in `dlen`
+ * @param url
+ * @param pool
+ * @return
+ */
+const gchar *rspamd_url_encode(struct rspamd_url *url, gsize *dlen,
+ rspamd_mempool_t *pool);
+
+
+/**
+ * Returns if a character is domain character
+ * @param c
+ * @return
+ */
+gboolean rspamd_url_is_domain(int c);
+
+/**
+ * Returns symbolic name for protocol
+ * @param proto
+ * @return
+ */
+const gchar *rspamd_url_protocol_name(enum rspamd_url_protocol proto);
+
+
+/**
+ * Converts string to a numeric protocol
+ * @param str
+ * @return
+ */
+enum rspamd_url_protocol rspamd_url_protocol_from_string(const gchar *str);
+
+/**
+ * Converts string to a url flag
+ * @param str
+ * @param flag
+ * @return
+ */
+bool rspamd_url_flag_from_string(const gchar *str, gint *flag);
+
+/**
+ * Converts url flag to a string
+ * @param flag
+ * @return
+ */
+const gchar *rspamd_url_flag_to_string(int flag);
+
+/* Defines sets of urls indexed by url as is */
+KHASH_DECLARE(rspamd_url_hash, struct rspamd_url *, char);
+KHASH_DECLARE(rspamd_url_host_hash, struct rspamd_url *, char);
+
+/* Convenience functions for url sets */
+/**
+ * Add an url to set or increase the existing url count
+ * @param set
+ * @param u
+ * @return true if a new url has been added
+ */
+bool rspamd_url_set_add_or_increase(khash_t(rspamd_url_hash) * set,
+ struct rspamd_url *u,
+ bool enforce_replace);
+
+/**
+ * Same as rspamd_url_set_add_or_increase but returns the existing url if found
+ * @param set
+ * @param u
+ * @return
+ */
+struct rspamd_url *rspamd_url_set_add_or_return(khash_t(rspamd_url_hash) * set,
+ struct rspamd_url *u);
+/**
+ * Helper for url host set
+ * @param set
+ * @param u
+ * @return
+ */
+bool rspamd_url_host_set_add(khash_t(rspamd_url_host_hash) * set,
+ struct rspamd_url *u);
+/**
+ * Checks if a url is in set
+ * @param set
+ * @param u
+ * @return
+ */
+bool rspamd_url_set_has(khash_t(rspamd_url_hash) * set, struct rspamd_url *u);
+
+bool rspamd_url_host_set_has(khash_t(rspamd_url_host_hash) * set, struct rspamd_url *u);
+
+/**
+ * Compares two urls (similar to C comparison functions) lexicographically
+ * @param u1
+ * @param u2
+ * @return
+ */
+int rspamd_url_cmp(const struct rspamd_url *u1, const struct rspamd_url *u2);
+
+/**
+ * Same but used for qsort to sort `struct rspamd_url *[]` array
+ * @param u1
+ * @param u2
+ * @return
+ */
+int rspamd_url_cmp_qsort(const void *u1, const void *u2);
+
+/**
+ * Returns a port for some url
+ * @param u
+ * @return
+ */
+static RSPAMD_PURE_FUNCTION inline uint16_t rspamd_url_get_port(struct rspamd_url *u)
+{
+ if ((u->flags & RSPAMD_URL_FLAG_HAS_PORT) && u->ext) {
+ return u->ext->port;
+ }
+ else {
+ /* Assume standard port */
+ if (u->protocol == PROTOCOL_HTTPS) {
+ return 443;
+ }
+ else {
+ return 80;
+ }
+ }
+}
+
+/**
+ * Returns a port for some url if it is set
+ * @param u
+ * @return
+ */
+static RSPAMD_PURE_FUNCTION inline uint16_t rspamd_url_get_port_if_special(struct rspamd_url *u)
+{
+ if ((u->flags & RSPAMD_URL_FLAG_HAS_PORT) && u->ext) {
+ return u->ext->port;
+ }
+
+ return 0;
+}
+
+/**
+ * Normalize unicode input and set out url flags as appropriate
+ * @param pool
+ * @param input
+ * @param len_out (must be &var)
+ * @param url_flags_out (must be just a var with no dereference)
+ */
+#define rspamd_url_normalise_propagate_flags(pool, input, len_out, url_flags_out) \
+ do { \
+ enum rspamd_utf8_normalise_result norm_res; \
+ norm_res = rspamd_normalise_unicode_inplace((input), (len_out)); \
+ if (norm_res & RSPAMD_UNICODE_NORM_UNNORMAL) { \
+ url_flags_out |= RSPAMD_URL_FLAG_UNNORMALISED; \
+ } \
+ if (norm_res & RSPAMD_UNICODE_NORM_ZERO_SPACES) { \
+ url_flags_out |= RSPAMD_URL_FLAG_ZW_SPACES; \
+ } \
+ if (norm_res & (RSPAMD_UNICODE_NORM_ERROR)) { \
+ url_flags_out |= RSPAMD_URL_FLAG_OBSCURED; \
+ } \
+ } while (0)
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c
new file mode 100644
index 0000000..74a3cf8
--- /dev/null
+++ b/src/libserver/worker_util.c
@@ -0,0 +1,2313 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "lua/lua_common.h"
+#include "worker_util.h"
+#include "unix-std.h"
+#include "utlist.h"
+#include "ottery.h"
+#include "rspamd_control.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_private.h"
+#include "libserver/http/http_private.h"
+#include "libserver/http/http_router.h"
+#include "libutil/rrd.h"
+
+/* sys/resource.h */
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
+/* pwd and grp */
+#ifdef HAVE_PWD_H
+#include <pwd.h>
+#endif
+#ifdef HAVE_GRP_H
+#include <grp.h>
+#endif
+#ifdef HAVE_LIBUTIL_H
+#include <libutil.h>
+#endif
+#include "zlib.h"
+
+#ifdef HAVE_UCONTEXT_H
+#include <ucontext.h>
+#elif defined(HAVE_SYS_UCONTEXT_H)
+#include <sys/ucontext.h>
+#endif
+
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#include <math.h>
+
+#endif
+
+#include "contrib/libev/ev.h"
+#include "libstat/stat_api.h"
+
+struct rspamd_worker *rspamd_current_worker = NULL;
+
+/* Forward declaration */
+static void rspamd_worker_heartbeat_start(struct rspamd_worker *,
+ struct ev_loop *);
+
+static void rspamd_worker_ignore_signal(struct rspamd_worker_signal_handler *);
+/**
+ * Return worker's control structure by its type
+ * @param type
+ * @return worker's control structure or NULL
+ */
+worker_t *
+rspamd_get_worker_by_type(struct rspamd_config *cfg, GQuark type)
+{
+ worker_t **pwrk;
+
+ pwrk = cfg->compiled_workers;
+ while (pwrk && *pwrk) {
+ if (rspamd_check_worker(cfg, *pwrk)) {
+ if (g_quark_from_string((*pwrk)->name) == type) {
+ return *pwrk;
+ }
+ }
+
+ pwrk++;
+ }
+
+ return NULL;
+}
+
+static void
+rspamd_worker_check_finished(EV_P_ ev_timer *w, int revents)
+{
+ int *pnchecks = (int *) w->data;
+
+ if (*pnchecks > SOFT_SHUTDOWN_TIME * 10) {
+ msg_warn("terminating worker before finishing of terminate handlers");
+ ev_break(EV_A_ EVBREAK_ONE);
+ }
+ else {
+ int refcount = ev_active_cnt(EV_A);
+
+ if (refcount == 1) {
+ ev_break(EV_A_ EVBREAK_ONE);
+ }
+ else {
+ ev_timer_again(EV_A_ w);
+ }
+ }
+}
+
+static gboolean
+rspamd_worker_finalize(gpointer user_data)
+{
+ struct rspamd_task *task = user_data;
+
+ if (!(task->flags & RSPAMD_TASK_FLAG_PROCESSING)) {
+ msg_info_task("finishing actions has been processed, terminating");
+ /* ev_break (task->event_loop, EVBREAK_ALL); */
+ task->worker->state = rspamd_worker_wanna_die;
+ rspamd_session_destroy(task->s);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_worker_call_finish_handlers(struct rspamd_worker *worker)
+{
+ struct rspamd_task *task;
+ struct rspamd_config *cfg = worker->srv->cfg;
+ struct rspamd_abstract_worker_ctx *ctx;
+ struct rspamd_config_cfg_lua_script *sc;
+
+ if (cfg->on_term_scripts) {
+ ctx = (struct rspamd_abstract_worker_ctx *) worker->ctx;
+ /* Create a fake task object for async events */
+ task = rspamd_task_new(worker, cfg, NULL, NULL, ctx->event_loop, FALSE);
+ task->resolver = ctx->resolver;
+ task->flags |= RSPAMD_TASK_FLAG_PROCESSING;
+ task->s = rspamd_session_create(task->task_pool,
+ rspamd_worker_finalize,
+ NULL,
+ (event_finalizer_t) rspamd_task_free,
+ task);
+
+ DL_FOREACH(cfg->on_term_scripts, sc)
+ {
+ lua_call_finish_script(sc, task);
+ }
+
+ task->flags &= ~RSPAMD_TASK_FLAG_PROCESSING;
+
+ if (rspamd_session_pending(task->s)) {
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+static void
+rspamd_worker_terminate_handlers(struct rspamd_worker *w)
+{
+ if (w->nconns == 0 &&
+ (!(w->flags & RSPAMD_WORKER_SCANNER) || w->srv->cfg->on_term_scripts == NULL)) {
+ /*
+ * We are here either:
+ * - No active connections are represented
+ * - No term scripts are registered
+ * - Worker is not a scanner, so it can die safely
+ */
+ w->state = rspamd_worker_wanna_die;
+ }
+ else {
+ if (w->nconns > 0) {
+ /*
+ * Wait until all connections are terminated
+ */
+ w->state = rspamd_worker_wait_connections;
+ }
+ else {
+ /*
+ * Start finish scripts
+ */
+ if (w->state != rspamd_worker_wait_final_scripts) {
+ w->state = rspamd_worker_wait_final_scripts;
+
+ if ((w->flags & RSPAMD_WORKER_SCANNER) &&
+ rspamd_worker_call_finish_handlers(w)) {
+ msg_info("performing async finishing actions");
+ w->state = rspamd_worker_wait_final_scripts;
+ }
+ else {
+ /*
+ * We are done now
+ */
+ msg_info("no async finishing actions, terminating");
+ w->state = rspamd_worker_wanna_die;
+ }
+ }
+ }
+ }
+}
+
+static void
+rspamd_worker_on_delayed_shutdown(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_worker *worker = (struct rspamd_worker *) w->data;
+
+ worker->state = rspamd_worker_wanna_die;
+ ev_timer_stop(EV_A_ w);
+ ev_break(loop, EVBREAK_ALL);
+}
+
+static void
+rspamd_worker_shutdown_check(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_worker *worker = (struct rspamd_worker *) w->data;
+
+ if (worker->state != rspamd_worker_wanna_die) {
+ rspamd_worker_terminate_handlers(worker);
+
+ if (worker->state == rspamd_worker_wanna_die) {
+ /* We are done, kill event loop */
+ ev_timer_stop(EV_A_ w);
+ ev_break(EV_A_ EVBREAK_ALL);
+ }
+ else {
+ /* Try again later */
+ ev_timer_again(EV_A_ w);
+ }
+ }
+ else {
+ ev_timer_stop(EV_A_ w);
+ ev_break(EV_A_ EVBREAK_ALL);
+ }
+}
+
+/*
+ * Config reload is designed by sending sigusr2 to active workers and pending shutdown of them
+ */
+static gboolean
+rspamd_worker_usr2_handler(struct rspamd_worker_signal_handler *sigh, void *arg)
+{
+ /* Do not accept new connections, preparing to end worker's process */
+ if (sigh->worker->state == rspamd_worker_state_running) {
+ static ev_timer shutdown_ev, shutdown_check_ev;
+ ev_tstamp shutdown_ts;
+
+ if (sigh->worker->flags & RSPAMD_WORKER_NO_TERMINATE_DELAY) {
+ shutdown_ts = 0.0;
+ }
+ else {
+ shutdown_ts = MAX(SOFT_SHUTDOWN_TIME,
+ sigh->worker->srv->cfg->task_timeout * 2.0);
+ }
+
+ rspamd_worker_ignore_signal(sigh);
+ sigh->worker->state = rspamd_worker_state_terminating;
+
+ rspamd_default_log_function(G_LOG_LEVEL_INFO,
+ sigh->worker->srv->server_pool->tag.tagname,
+ sigh->worker->srv->server_pool->tag.uid,
+ G_STRFUNC,
+ "worker's shutdown is pending in %.2f sec",
+ shutdown_ts);
+
+ /* Soft shutdown timer */
+ shutdown_ev.data = sigh->worker;
+ ev_timer_init(&shutdown_ev, rspamd_worker_on_delayed_shutdown,
+ shutdown_ts, 0.0);
+ ev_timer_start(sigh->event_loop, &shutdown_ev);
+
+ if (!(sigh->worker->flags & RSPAMD_WORKER_NO_TERMINATE_DELAY)) {
+ /* This timer checks if we are ready to die and is called frequently */
+ shutdown_check_ev.data = sigh->worker;
+ ev_timer_init(&shutdown_check_ev, rspamd_worker_shutdown_check,
+ 0.5, 0.5);
+ ev_timer_start(sigh->event_loop, &shutdown_check_ev);
+ }
+
+ rspamd_worker_stop_accept(sigh->worker);
+ }
+
+ /* No more signals */
+ return FALSE;
+}
+
+/*
+ * Reopen log is designed by sending sigusr1 to active workers and pending shutdown of them
+ */
+static gboolean
+rspamd_worker_usr1_handler(struct rspamd_worker_signal_handler *sigh, void *arg)
+{
+ struct rspamd_main *rspamd_main = sigh->worker->srv;
+
+ rspamd_log_reopen(sigh->worker->srv->logger, rspamd_main->cfg, -1, -1);
+ msg_info_main("logging reinitialised");
+
+ /* Get more signals */
+ return TRUE;
+}
+
+static gboolean
+rspamd_worker_term_handler(struct rspamd_worker_signal_handler *sigh, void *arg)
+{
+ if (sigh->worker->state == rspamd_worker_state_running) {
+ static ev_timer shutdown_ev, shutdown_check_ev;
+ ev_tstamp shutdown_ts;
+
+ if (sigh->worker->flags & RSPAMD_WORKER_NO_TERMINATE_DELAY) {
+ shutdown_ts = 0.0;
+ }
+ else {
+ shutdown_ts = MAX(SOFT_SHUTDOWN_TIME,
+ sigh->worker->srv->cfg->task_timeout * 2.0);
+ }
+
+ rspamd_worker_ignore_signal(sigh);
+ sigh->worker->state = rspamd_worker_state_terminating;
+ rspamd_default_log_function(G_LOG_LEVEL_INFO,
+ sigh->worker->srv->server_pool->tag.tagname,
+ sigh->worker->srv->server_pool->tag.uid,
+ G_STRFUNC,
+ "terminating after receiving signal %s",
+ g_strsignal(sigh->signo));
+
+ rspamd_worker_stop_accept(sigh->worker);
+ rspamd_worker_terminate_handlers(sigh->worker);
+
+ /* Check if we are ready to die */
+ if (sigh->worker->state != rspamd_worker_wanna_die) {
+ /* This timer is called when we have no choices but to die */
+ shutdown_ev.data = sigh->worker;
+ ev_timer_init(&shutdown_ev, rspamd_worker_on_delayed_shutdown,
+ shutdown_ts, 0.0);
+ ev_timer_start(sigh->event_loop, &shutdown_ev);
+
+ if (!(sigh->worker->flags & RSPAMD_WORKER_NO_TERMINATE_DELAY)) {
+ /* This timer checks if we are ready to die and is called frequently */
+ shutdown_check_ev.data = sigh->worker;
+ ev_timer_init(&shutdown_check_ev, rspamd_worker_shutdown_check,
+ 0.5, 0.5);
+ ev_timer_start(sigh->event_loop, &shutdown_check_ev);
+ }
+ }
+ else {
+ /* Flag to die has been already set */
+ ev_break(sigh->event_loop, EVBREAK_ALL);
+ }
+ }
+
+ /* Stop reacting on signals */
+ return FALSE;
+}
+
+static void
+rspamd_worker_signal_handle(EV_P_ ev_signal *w, int revents)
+{
+ struct rspamd_worker_signal_handler *sigh =
+ (struct rspamd_worker_signal_handler *) w->data;
+ struct rspamd_worker_signal_handler_elt *cb, *cbtmp;
+
+ /* Call all signal handlers registered */
+ DL_FOREACH_SAFE(sigh->cb, cb, cbtmp)
+ {
+ if (!cb->handler(sigh, cb->handler_data)) {
+ DL_DELETE(sigh->cb, cb);
+ g_free(cb);
+ }
+ }
+}
+
+static void
+rspamd_worker_ignore_signal(struct rspamd_worker_signal_handler *sigh)
+{
+ sigset_t set;
+
+ ev_signal_stop(sigh->event_loop, &sigh->ev_sig);
+ sigemptyset(&set);
+ sigaddset(&set, sigh->signo);
+ sigprocmask(SIG_BLOCK, &set, NULL);
+}
+
+static void
+rspamd_worker_default_signal(int signo)
+{
+ struct sigaction sig;
+
+ sigemptyset(&sig.sa_mask);
+ sigaddset(&sig.sa_mask, signo);
+ sig.sa_handler = SIG_DFL;
+ sig.sa_flags = 0;
+ sigaction(signo, &sig, NULL);
+}
+
+static void
+rspamd_sigh_free(void *p)
+{
+ struct rspamd_worker_signal_handler *sigh = p;
+ struct rspamd_worker_signal_handler_elt *cb, *tmp;
+
+ DL_FOREACH_SAFE(sigh->cb, cb, tmp)
+ {
+ DL_DELETE(sigh->cb, cb);
+ g_free(cb);
+ }
+
+ ev_signal_stop(sigh->event_loop, &sigh->ev_sig);
+ rspamd_worker_default_signal(sigh->signo);
+ g_free(sigh);
+}
+
+void rspamd_worker_set_signal_handler(int signo, struct rspamd_worker *worker,
+ struct ev_loop *event_loop,
+ rspamd_worker_signal_cb_t handler,
+ void *handler_data)
+{
+ struct rspamd_worker_signal_handler *sigh;
+ struct rspamd_worker_signal_handler_elt *cb;
+
+ sigh = g_hash_table_lookup(worker->signal_events, GINT_TO_POINTER(signo));
+
+ if (sigh == NULL) {
+ sigh = g_malloc0(sizeof(*sigh));
+ sigh->signo = signo;
+ sigh->worker = worker;
+ sigh->event_loop = event_loop;
+ sigh->enabled = TRUE;
+
+ sigh->ev_sig.data = sigh;
+ ev_signal_init(&sigh->ev_sig, rspamd_worker_signal_handle, signo);
+ ev_signal_start(event_loop, &sigh->ev_sig);
+
+ g_hash_table_insert(worker->signal_events,
+ GINT_TO_POINTER(signo),
+ sigh);
+ }
+
+ cb = g_malloc0(sizeof(*cb));
+ cb->handler = handler;
+ cb->handler_data = handler_data;
+ DL_APPEND(sigh->cb, cb);
+}
+
+void rspamd_worker_init_signals(struct rspamd_worker *worker,
+ struct ev_loop *event_loop)
+{
+ /* A set of terminating signals */
+ rspamd_worker_set_signal_handler(SIGTERM, worker, event_loop,
+ rspamd_worker_term_handler, NULL);
+ rspamd_worker_set_signal_handler(SIGINT, worker, event_loop,
+ rspamd_worker_term_handler, NULL);
+ rspamd_worker_set_signal_handler(SIGHUP, worker, event_loop,
+ rspamd_worker_term_handler, NULL);
+
+ /* Special purpose signals */
+ rspamd_worker_set_signal_handler(SIGUSR1, worker, event_loop,
+ rspamd_worker_usr1_handler, NULL);
+ rspamd_worker_set_signal_handler(SIGUSR2, worker, event_loop,
+ rspamd_worker_usr2_handler, NULL);
+}
+
+
+struct ev_loop *
+rspamd_prepare_worker(struct rspamd_worker *worker, const char *name,
+ rspamd_accept_handler hdl)
+{
+ struct ev_loop *event_loop;
+ GList *cur;
+ struct rspamd_worker_listen_socket *ls;
+ struct rspamd_worker_accept_event *accept_ev;
+
+ worker->signal_events = g_hash_table_new_full(g_direct_hash, g_direct_equal,
+ NULL, rspamd_sigh_free);
+
+ event_loop = ev_loop_new(rspamd_config_ev_backend_get(worker->srv->cfg));
+
+ worker->srv->event_loop = event_loop;
+
+ rspamd_worker_init_signals(worker, event_loop);
+ rspamd_control_worker_add_default_cmd_handlers(worker, event_loop);
+ rspamd_worker_heartbeat_start(worker, event_loop);
+ rspamd_redis_pool_config(worker->srv->cfg->redis_pool,
+ worker->srv->cfg, event_loop);
+
+ /* Accept all sockets */
+ if (hdl) {
+ cur = worker->cf->listen_socks;
+
+ while (cur) {
+ ls = cur->data;
+
+ if (ls->fd != -1) {
+ accept_ev = g_malloc0(sizeof(*accept_ev));
+ accept_ev->event_loop = event_loop;
+ accept_ev->accept_ev.data = worker;
+ ev_io_init(&accept_ev->accept_ev, hdl, ls->fd, EV_READ);
+ ev_io_start(event_loop, &accept_ev->accept_ev);
+
+ DL_APPEND(worker->accept_events, accept_ev);
+ }
+
+ cur = g_list_next(cur);
+ }
+ }
+
+ return event_loop;
+}
+
+void rspamd_worker_stop_accept(struct rspamd_worker *worker)
+{
+ struct rspamd_worker_accept_event *cur, *tmp;
+
+ /* Remove all events */
+ DL_FOREACH_SAFE(worker->accept_events, cur, tmp)
+ {
+
+ if (ev_can_stop(&cur->accept_ev)) {
+ ev_io_stop(cur->event_loop, &cur->accept_ev);
+ }
+
+
+ if (ev_can_stop(&cur->throttling_ev)) {
+ ev_timer_stop(cur->event_loop, &cur->throttling_ev);
+ }
+
+ g_free(cur);
+ }
+
+ /* XXX: we need to do it much later */
+#if 0
+ g_hash_table_iter_init (&it, worker->signal_events);
+
+ while (g_hash_table_iter_next (&it, &k, &v)) {
+ sigh = (struct rspamd_worker_signal_handler *)v;
+ g_hash_table_iter_steal (&it);
+
+ if (sigh->enabled) {
+ event_del (&sigh->ev);
+ }
+
+ g_free (sigh);
+ }
+
+ g_hash_table_unref (worker->signal_events);
+#endif
+}
+
+static rspamd_fstring_t *
+rspamd_controller_maybe_compress(struct rspamd_http_connection_entry *entry,
+ rspamd_fstring_t *buf, struct rspamd_http_message *msg)
+{
+ if (entry->support_gzip) {
+ if (rspamd_fstring_gzip(&buf)) {
+ rspamd_http_message_add_header(msg, "Content-Encoding", "gzip");
+ }
+ }
+
+ return buf;
+}
+
+void rspamd_controller_send_error(struct rspamd_http_connection_entry *entry,
+ gint code, const gchar *error_msg, ...)
+{
+ struct rspamd_http_message *msg;
+ va_list args;
+ rspamd_fstring_t *reply;
+
+ msg = rspamd_http_new_message(HTTP_RESPONSE);
+
+ va_start(args, error_msg);
+ msg->status = rspamd_fstring_new();
+ rspamd_vprintf_fstring(&msg->status, error_msg, args);
+ va_end(args);
+
+ msg->date = time(NULL);
+ msg->code = code;
+ reply = rspamd_fstring_sized_new(msg->status->len + 16);
+ rspamd_printf_fstring(&reply, "{\"error\":\"%V\"}", msg->status);
+ rspamd_http_message_set_body_from_fstring_steal(msg,
+ rspamd_controller_maybe_compress(entry, reply, msg));
+ rspamd_http_connection_reset(entry->conn);
+ rspamd_http_router_insert_headers(entry->rt, msg);
+ rspamd_http_connection_write_message(entry->conn,
+ msg,
+ NULL,
+ "application/json",
+ entry,
+ entry->rt->timeout);
+ entry->is_reply = TRUE;
+}
+
+void rspamd_controller_send_openmetrics(struct rspamd_http_connection_entry *entry,
+ rspamd_fstring_t *str)
+{
+ struct rspamd_http_message *msg;
+
+ msg = rspamd_http_new_message(HTTP_RESPONSE);
+ msg->date = time(NULL);
+ msg->code = 200;
+ msg->status = rspamd_fstring_new_init("OK", 2);
+
+ rspamd_http_message_set_body_from_fstring_steal(msg,
+ rspamd_controller_maybe_compress(entry, str, msg));
+ rspamd_http_connection_reset(entry->conn);
+ rspamd_http_router_insert_headers(entry->rt, msg);
+ rspamd_http_connection_write_message(entry->conn,
+ msg,
+ NULL,
+ "application/openmetrics-text; version=1.0.0; charset=utf-8",
+ entry,
+ entry->rt->timeout);
+ entry->is_reply = TRUE;
+}
+
+void rspamd_controller_send_string(struct rspamd_http_connection_entry *entry,
+ const gchar *str)
+{
+ struct rspamd_http_message *msg;
+ rspamd_fstring_t *reply;
+
+ msg = rspamd_http_new_message(HTTP_RESPONSE);
+ msg->date = time(NULL);
+ msg->code = 200;
+ msg->status = rspamd_fstring_new_init("OK", 2);
+
+ if (str) {
+ reply = rspamd_fstring_new_init(str, strlen(str));
+ }
+ else {
+ reply = rspamd_fstring_new_init("null", 4);
+ }
+
+ rspamd_http_message_set_body_from_fstring_steal(msg,
+ rspamd_controller_maybe_compress(entry, reply, msg));
+ rspamd_http_connection_reset(entry->conn);
+ rspamd_http_router_insert_headers(entry->rt, msg);
+ rspamd_http_connection_write_message(entry->conn,
+ msg,
+ NULL,
+ "application/json",
+ entry,
+ entry->rt->timeout);
+ entry->is_reply = TRUE;
+}
+
+void rspamd_controller_send_ucl(struct rspamd_http_connection_entry *entry,
+ ucl_object_t *obj)
+{
+ struct rspamd_http_message *msg;
+ rspamd_fstring_t *reply;
+
+ msg = rspamd_http_new_message(HTTP_RESPONSE);
+ msg->date = time(NULL);
+ msg->code = 200;
+ msg->status = rspamd_fstring_new_init("OK", 2);
+ reply = rspamd_fstring_sized_new(BUFSIZ);
+ rspamd_ucl_emit_fstring(obj, UCL_EMIT_JSON_COMPACT, &reply);
+ rspamd_http_message_set_body_from_fstring_steal(msg,
+ rspamd_controller_maybe_compress(entry, reply, msg));
+ rspamd_http_connection_reset(entry->conn);
+ rspamd_http_router_insert_headers(entry->rt, msg);
+ rspamd_http_connection_write_message(entry->conn,
+ msg,
+ NULL,
+ "application/json",
+ entry,
+ entry->rt->timeout);
+ entry->is_reply = TRUE;
+}
+
+static void
+rspamd_worker_drop_priv(struct rspamd_main *rspamd_main)
+{
+ if (rspamd_main->is_privileged) {
+ if (setgid(rspamd_main->workers_gid) == -1) {
+ msg_err_main("cannot setgid to %d (%s), aborting",
+ (gint) rspamd_main->workers_gid,
+ strerror(errno));
+ exit(-errno);
+ }
+
+ if (rspamd_main->cfg->rspamd_user &&
+ initgroups(rspamd_main->cfg->rspamd_user,
+ rspamd_main->workers_gid) == -1) {
+ msg_err_main("initgroups failed (%s), aborting", strerror(errno));
+ exit(-errno);
+ }
+
+ if (setuid(rspamd_main->workers_uid) == -1) {
+ msg_err_main("cannot setuid to %d (%s), aborting",
+ (gint) rspamd_main->workers_uid,
+ strerror(errno));
+ exit(-errno);
+ }
+ }
+}
+
+static void
+rspamd_worker_set_limits(struct rspamd_main *rspamd_main,
+ struct rspamd_worker_conf *cf)
+{
+ struct rlimit rlmt;
+
+ if (cf->rlimit_nofile != 0) {
+ rlmt.rlim_cur = (rlim_t) cf->rlimit_nofile;
+ rlmt.rlim_max = (rlim_t) cf->rlimit_nofile;
+
+ if (setrlimit(RLIMIT_NOFILE, &rlmt) == -1) {
+ msg_warn_main("cannot set files rlimit: %L, %s",
+ cf->rlimit_nofile,
+ strerror(errno));
+ }
+
+ memset(&rlmt, 0, sizeof(rlmt));
+
+ if (getrlimit(RLIMIT_NOFILE, &rlmt) == -1) {
+ msg_warn_main("cannot get max files rlimit: %HL, %s",
+ cf->rlimit_maxcore,
+ strerror(errno));
+ }
+ else {
+ msg_info_main("set max file descriptors limit: %HL cur and %HL max",
+ (guint64) rlmt.rlim_cur,
+ (guint64) rlmt.rlim_max);
+ }
+ }
+ else {
+ /* Just report */
+ if (getrlimit(RLIMIT_NOFILE, &rlmt) == -1) {
+ msg_warn_main("cannot get max files rlimit: %HL, %s",
+ cf->rlimit_maxcore,
+ strerror(errno));
+ }
+ else {
+ msg_info_main("use system max file descriptors limit: %HL cur and %HL max",
+ (guint64) rlmt.rlim_cur,
+ (guint64) rlmt.rlim_max);
+ }
+ }
+
+ if (rspamd_main->cores_throttling) {
+ msg_info_main("disable core files for the new worker as limits are reached");
+ rlmt.rlim_cur = 0;
+ rlmt.rlim_max = 0;
+
+ if (setrlimit(RLIMIT_CORE, &rlmt) == -1) {
+ msg_warn_main("cannot disable core dumps: error when setting limits: %s",
+ strerror(errno));
+ }
+ }
+ else {
+ if (cf->rlimit_maxcore != 0) {
+ rlmt.rlim_cur = (rlim_t) cf->rlimit_maxcore;
+ rlmt.rlim_max = (rlim_t) cf->rlimit_maxcore;
+
+ if (setrlimit(RLIMIT_CORE, &rlmt) == -1) {
+ msg_warn_main("cannot set max core size limit: %HL, %s",
+ cf->rlimit_maxcore,
+ strerror(errno));
+ }
+
+ /* Ensure that we did it */
+ memset(&rlmt, 0, sizeof(rlmt));
+
+ if (getrlimit(RLIMIT_CORE, &rlmt) == -1) {
+ msg_warn_main("cannot get max core size rlimit: %HL, %s",
+ cf->rlimit_maxcore,
+ strerror(errno));
+ }
+ else {
+ if (rlmt.rlim_cur != cf->rlimit_maxcore ||
+ rlmt.rlim_max != cf->rlimit_maxcore) {
+ msg_warn_main("setting of core file limits was unsuccessful: "
+ "%HL was wanted, "
+ "but we have %HL cur and %HL max",
+ cf->rlimit_maxcore,
+ (guint64) rlmt.rlim_cur,
+ (guint64) rlmt.rlim_max);
+ }
+ else {
+ msg_info_main("set max core size limit: %HL cur and %HL max",
+ (guint64) rlmt.rlim_cur,
+ (guint64) rlmt.rlim_max);
+ }
+ }
+ }
+ else {
+ /* Just report */
+ if (getrlimit(RLIMIT_CORE, &rlmt) == -1) {
+ msg_warn_main("cannot get max core size limit: %HL, %s",
+ cf->rlimit_maxcore,
+ strerror(errno));
+ }
+ else {
+ msg_info_main("use system max core size limit: %HL cur and %HL max",
+ (guint64) rlmt.rlim_cur,
+ (guint64) rlmt.rlim_max);
+ }
+ }
+ }
+}
+
+static void
+rspamd_worker_on_term(EV_P_ ev_child *w, int revents)
+{
+ struct rspamd_worker *wrk = (struct rspamd_worker *) w->data;
+
+ if (wrk->ppid == getpid()) {
+ if (wrk->term_handler) {
+ wrk->term_handler(EV_A_ w, wrk->srv, wrk);
+ }
+ else {
+ rspamd_check_termination_clause(wrk->srv, wrk, w->rstatus);
+ }
+ }
+ else {
+ /* Ignore SIGCHLD for not our children... */
+ }
+}
+
+static void
+rspamd_worker_heartbeat_cb(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_worker *wrk = (struct rspamd_worker *) w->data;
+ struct rspamd_srv_command cmd;
+
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.type = RSPAMD_SRV_HEARTBEAT;
+ rspamd_srv_send_command(wrk, EV_A, &cmd, -1, NULL, NULL);
+}
+
+static void
+rspamd_worker_heartbeat_start(struct rspamd_worker *wrk, struct ev_loop *event_loop)
+{
+ wrk->hb.heartbeat_ev.data = (void *) wrk;
+ ev_timer_init(&wrk->hb.heartbeat_ev, rspamd_worker_heartbeat_cb,
+ 0.0, wrk->srv->cfg->heartbeat_interval);
+ ev_timer_start(event_loop, &wrk->hb.heartbeat_ev);
+}
+
+static void
+rspamd_main_heartbeat_cb(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_worker *wrk = (struct rspamd_worker *) w->data;
+ gdouble time_from_last = ev_time();
+ struct rspamd_main *rspamd_main;
+ static struct rspamd_control_command cmd;
+ struct tm tm;
+ gchar timebuf[64];
+ gchar usec_buf[16];
+ gint r;
+
+ time_from_last -= wrk->hb.last_event;
+ rspamd_main = wrk->srv;
+
+ if (wrk->hb.last_event > 0 &&
+ time_from_last > 0 &&
+ time_from_last >= rspamd_main->cfg->heartbeat_interval * 2) {
+
+ rspamd_localtime(wrk->hb.last_event, &tm);
+ r = strftime(timebuf, sizeof(timebuf), "%F %H:%M:%S", &tm);
+ rspamd_snprintf(usec_buf, sizeof(usec_buf), "%.5f",
+ wrk->hb.last_event - (gdouble) (time_t) wrk->hb.last_event);
+ rspamd_snprintf(timebuf + r, sizeof(timebuf) - r,
+ "%s", usec_buf + 1);
+
+ if (wrk->hb.nbeats > 0) {
+ /* First time lost event */
+ cmd.type = RSPAMD_CONTROL_CHILD_CHANGE;
+ cmd.cmd.child_change.what = rspamd_child_offline;
+ cmd.cmd.child_change.pid = wrk->pid;
+ rspamd_control_broadcast_srv_cmd(rspamd_main, &cmd, wrk->pid);
+ msg_warn_main("lost heartbeat from worker type %s with pid %P, "
+ "last beat on: %s (%L beats received previously)",
+ g_quark_to_string(wrk->type), wrk->pid,
+ timebuf,
+ wrk->hb.nbeats);
+ wrk->hb.nbeats = -1;
+ /* TODO: send notify about worker problem */
+ }
+ else {
+ wrk->hb.nbeats--;
+ msg_warn_main("lost %L heartbeat from worker type %s with pid %P, "
+ "last beat on: %s",
+ -(wrk->hb.nbeats),
+ g_quark_to_string(wrk->type),
+ wrk->pid,
+ timebuf);
+
+ if (rspamd_main->cfg->heartbeats_loss_max > 0 &&
+ -(wrk->hb.nbeats) >= rspamd_main->cfg->heartbeats_loss_max) {
+
+
+ if (-(wrk->hb.nbeats) > rspamd_main->cfg->heartbeats_loss_max + 1) {
+ msg_err_main("force kill worker type %s with pid %P, "
+ "last beat on: %s; %L heartbeat lost",
+ g_quark_to_string(wrk->type),
+ wrk->pid,
+ timebuf,
+ -(wrk->hb.nbeats));
+ kill(wrk->pid, SIGKILL);
+ }
+ else {
+ msg_err_main("terminate worker type %s with pid %P, "
+ "last beat on: %s; %L heartbeat lost",
+ g_quark_to_string(wrk->type),
+ wrk->pid,
+ timebuf,
+ -(wrk->hb.nbeats));
+ kill(wrk->pid, SIGTERM);
+ }
+ }
+ }
+ }
+ else if (wrk->hb.nbeats < 0) {
+ rspamd_localtime(wrk->hb.last_event, &tm);
+ r = strftime(timebuf, sizeof(timebuf), "%F %H:%M:%S", &tm);
+ rspamd_snprintf(usec_buf, sizeof(usec_buf), "%.5f",
+ wrk->hb.last_event - (gdouble) (time_t) wrk->hb.last_event);
+ rspamd_snprintf(timebuf + r, sizeof(timebuf) - r,
+ "%s", usec_buf + 1);
+
+ cmd.type = RSPAMD_CONTROL_CHILD_CHANGE;
+ cmd.cmd.child_change.what = rspamd_child_online;
+ cmd.cmd.child_change.pid = wrk->pid;
+ rspamd_control_broadcast_srv_cmd(rspamd_main, &cmd, wrk->pid);
+ msg_info_main("received heartbeat from worker type %s with pid %P, "
+ "last beat on: %s (%L beats lost previously)",
+ g_quark_to_string(wrk->type), wrk->pid,
+ timebuf,
+ -(wrk->hb.nbeats));
+ wrk->hb.nbeats = 1;
+ /* TODO: send notify about worker restoration */
+ }
+}
+
+static void
+rspamd_main_heartbeat_start(struct rspamd_worker *wrk, struct ev_loop *event_loop)
+{
+ wrk->hb.heartbeat_ev.data = (void *) wrk;
+ ev_timer_init(&wrk->hb.heartbeat_ev, rspamd_main_heartbeat_cb,
+ 0.0, wrk->srv->cfg->heartbeat_interval * 2);
+ ev_timer_start(event_loop, &wrk->hb.heartbeat_ev);
+}
+
+static bool
+rspamd_maybe_reuseport_socket(struct rspamd_worker_listen_socket *ls)
+{
+ if (ls->is_systemd) {
+ /* No need to reuseport */
+ return true;
+ }
+
+ if (ls->fd != -1 && rspamd_inet_address_get_af(ls->addr) == AF_UNIX) {
+ /* Just try listen */
+
+ if (listen(ls->fd, -1) == -1) {
+ return false;
+ }
+
+ return true;
+ }
+
+#if defined(SO_REUSEPORT) && defined(SO_REUSEADDR) && defined(LINUX)
+ gint nfd = -1;
+
+ if (ls->type == RSPAMD_WORKER_SOCKET_UDP) {
+ nfd = rspamd_inet_address_listen(ls->addr,
+ (ls->type == RSPAMD_WORKER_SOCKET_UDP ? SOCK_DGRAM : SOCK_STREAM),
+ RSPAMD_INET_ADDRESS_LISTEN_ASYNC | RSPAMD_INET_ADDRESS_LISTEN_REUSEPORT,
+ -1);
+
+ if (nfd == -1) {
+ msg_warn("cannot create reuseport listen socket for %d: %s",
+ ls->fd, strerror(errno));
+ nfd = ls->fd;
+ }
+ else {
+ if (ls->fd != -1) {
+ close(ls->fd);
+ }
+ ls->fd = nfd;
+ nfd = -1;
+ }
+ }
+ else {
+ /*
+ * Reuseport is broken with the current architecture, so it is easier not
+ * to use it at all
+ */
+ nfd = ls->fd;
+ }
+#endif
+
+#if 0
+ /* This needed merely if we have reuseport for tcp, but for now it is disabled */
+ /* This means that we have an fd with no listening enabled */
+ if (nfd != -1) {
+ if (ls->type == RSPAMD_WORKER_SOCKET_TCP) {
+ if (listen (nfd, -1) == -1) {
+ return false;
+ }
+ }
+ }
+#endif
+
+ return true;
+}
+
+/**
+ * Handles worker after fork returned zero
+ * @param wrk
+ * @param rspamd_main
+ * @param cf
+ * @param listen_sockets
+ */
+static void __attribute__((noreturn))
+rspamd_handle_child_fork(struct rspamd_worker *wrk,
+ struct rspamd_main *rspamd_main,
+ struct rspamd_worker_conf *cf,
+ GHashTable *listen_sockets)
+{
+ gint rc;
+ struct rlimit rlim;
+
+ /* Update pid for logging */
+ rspamd_log_on_fork(cf->type, rspamd_main->cfg, rspamd_main->logger);
+ wrk->pid = getpid();
+
+ /* Init PRNG after fork */
+ rc = ottery_init(rspamd_main->cfg->libs_ctx->ottery_cfg);
+ if (rc != OTTERY_ERR_NONE) {
+ msg_err_main("cannot initialize PRNG: %d", rc);
+ abort();
+ }
+
+ rspamd_random_seed_fast();
+#ifdef HAVE_EVUTIL_RNG_INIT
+ evutil_secure_rng_init();
+#endif
+
+ /*
+ * Libev stores all signals in a global table, so
+ * previous handlers must be explicitly detached and forgotten
+ * before starting a new loop
+ */
+ ev_signal_stop(rspamd_main->event_loop, &rspamd_main->int_ev);
+ ev_signal_stop(rspamd_main->event_loop, &rspamd_main->term_ev);
+ ev_signal_stop(rspamd_main->event_loop, &rspamd_main->hup_ev);
+ ev_signal_stop(rspamd_main->event_loop, &rspamd_main->usr1_ev);
+ /* Remove the inherited event base */
+ ev_loop_destroy(rspamd_main->event_loop);
+ rspamd_main->event_loop = NULL;
+
+ /* Close unused sockets */
+ GHashTableIter it;
+ gpointer k, v;
+
+
+ g_hash_table_iter_init(&it, listen_sockets);
+
+ /*
+ * Close listen sockets of not our process (inherited from other forks)
+ */
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ GList *elt = (GList *) v;
+ GList *our = cf->listen_socks;
+
+ if (g_list_position(our, elt) == -1) {
+ GList *cur = elt;
+
+ while (cur) {
+ struct rspamd_worker_listen_socket *ls =
+ (struct rspamd_worker_listen_socket *) cur->data;
+
+ if (ls->fd != -1 && close(ls->fd) == -1) {
+ msg_err("cannot close fd %d (addr = %s): %s",
+ ls->fd,
+ rspamd_inet_address_to_string_pretty(ls->addr),
+ strerror(errno));
+ }
+
+ ls->fd = -1;
+
+ cur = g_list_next(cur);
+ }
+ }
+ }
+
+ /* Reuseport before dropping privs */
+ GList *cur = cf->listen_socks;
+
+ while (cur) {
+ struct rspamd_worker_listen_socket *ls =
+ (struct rspamd_worker_listen_socket *) cur->data;
+
+ if (!rspamd_maybe_reuseport_socket(ls)) {
+ msg_err("cannot listen on socket %s: %s",
+ rspamd_inet_address_to_string_pretty(ls->addr),
+ strerror(errno));
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ /* Drop privileges */
+ rspamd_worker_drop_priv(rspamd_main);
+ /* Set limits */
+ rspamd_worker_set_limits(rspamd_main, cf);
+ /* Re-set stack limit */
+ getrlimit(RLIMIT_STACK, &rlim);
+ rlim.rlim_cur = 100 * 1024 * 1024;
+ rlim.rlim_max = rlim.rlim_cur;
+ setrlimit(RLIMIT_STACK, &rlim);
+
+ if (cf->bind_conf) {
+ rspamd_setproctitle("%s process (%s)", cf->worker->name,
+ cf->bind_conf->bind_line);
+ }
+ else {
+ rspamd_setproctitle("%s process", cf->worker->name);
+ }
+
+ if (rspamd_main->pfh) {
+ rspamd_pidfile_close(rspamd_main->pfh);
+ }
+
+ if (rspamd_main->cfg->log_silent_workers) {
+ rspamd_log_set_log_level(rspamd_main->logger, G_LOG_LEVEL_MESSAGE);
+ }
+
+ wrk->start_time = rspamd_get_calendar_ticks();
+
+ if (cf->bind_conf) {
+ GString *listen_conf_stringified = g_string_new(NULL);
+ struct rspamd_worker_bind_conf *cur_conf;
+
+ LL_FOREACH(cf->bind_conf, cur_conf)
+ {
+ if (cur_conf->next) {
+ rspamd_printf_gstring(listen_conf_stringified, "%s, ",
+ cur_conf->bind_line);
+ }
+ else {
+ rspamd_printf_gstring(listen_conf_stringified, "%s",
+ cur_conf->bind_line);
+ }
+ }
+
+ msg_info_main("starting %s process %P (%d); listen on: %v",
+ cf->worker->name,
+ getpid(), wrk->index, listen_conf_stringified);
+ g_string_free(listen_conf_stringified, TRUE);
+ }
+ else {
+ msg_info_main("starting %s process %P (%d); no listen",
+ cf->worker->name,
+ getpid(), wrk->index);
+ }
+ /* Close parent part of socketpair */
+ close(wrk->control_pipe[0]);
+ close(wrk->srv_pipe[0]);
+ /*
+ * Read comments in `rspamd_handle_main_fork` for details why these channel
+ * is blocking.
+ */
+ rspamd_socket_nonblocking(wrk->control_pipe[1]);
+#if 0
+ rspamd_socket_nonblocking (wrk->srv_pipe[1]);
+#endif
+ rspamd_main->cfg->cur_worker = wrk;
+ /* Execute worker (this function should not return normally!) */
+ cf->worker->worker_start_func(wrk);
+ /* To distinguish from normal termination */
+ exit(EXIT_FAILURE);
+}
+
+static void
+rspamd_handle_main_fork(struct rspamd_worker *wrk,
+ struct rspamd_main *rspamd_main,
+ struct rspamd_worker_conf *cf,
+ struct ev_loop *ev_base)
+{
+ /* Close worker part of socketpair */
+ close(wrk->control_pipe[1]);
+ close(wrk->srv_pipe[1]);
+
+ /*
+ * There are no reasons why control pipes are blocking: the messages
+ * there are rare and are strictly bounded by command sizes, so if we block
+ * on some pipe, it is ok, as we still poll that for all operations.
+ * It is also impossible to block on writing in normal conditions.
+ * And if the conditions are not normal, e.g. a worker is unresponsive, then
+ * we can safely think that the non-blocking behaviour as it is implemented
+ * currently will not make things better, as it would lead to incomplete
+ * reads/writes that are not handled anyhow and are totally broken from the
+ * beginning.
+ */
+#if 0
+ rspamd_socket_nonblocking (wrk->srv_pipe[0]);
+#endif
+ rspamd_socket_nonblocking(wrk->control_pipe[0]);
+
+ rspamd_srv_start_watching(rspamd_main, wrk, ev_base);
+ /* Child event */
+ wrk->cld_ev.data = wrk;
+ ev_child_init(&wrk->cld_ev, rspamd_worker_on_term, wrk->pid, 0);
+ ev_child_start(rspamd_main->event_loop, &wrk->cld_ev);
+ /* Heartbeats */
+ rspamd_main_heartbeat_start(wrk, rspamd_main->event_loop);
+ /* Insert worker into worker's table, pid is index */
+ g_hash_table_insert(rspamd_main->workers,
+ GSIZE_TO_POINTER(wrk->pid), wrk);
+
+#if defined(SO_REUSEPORT) && defined(SO_REUSEADDR) && defined(LINUX)
+ /*
+ * Close listen sockets in the main process once a child is handling them,
+ * if we have reuseport
+ */
+ GList *cur = cf->listen_socks;
+
+ while (cur) {
+ struct rspamd_worker_listen_socket *ls =
+ (struct rspamd_worker_listen_socket *) cur->data;
+
+ if (ls->fd != -1 && ls->type == RSPAMD_WORKER_SOCKET_UDP) {
+ close(ls->fd);
+ ls->fd = -1;
+ }
+
+ cur = g_list_next(cur);
+ }
+#endif
+}
+
+#ifndef SOCK_SEQPACKET
+#define SOCK_SEQPACKET SOCK_DGRAM
+#endif
+struct rspamd_worker *
+rspamd_fork_worker(struct rspamd_main *rspamd_main,
+ struct rspamd_worker_conf *cf,
+ guint index,
+ struct ev_loop *ev_base,
+ rspamd_worker_term_cb term_handler,
+ GHashTable *listen_sockets)
+{
+ struct rspamd_worker *wrk;
+
+ /* Starting worker process */
+ wrk = (struct rspamd_worker *) g_malloc0(sizeof(struct rspamd_worker));
+
+ if (!rspamd_socketpair(wrk->control_pipe, SOCK_SEQPACKET)) {
+ msg_err("socketpair failure: %s", strerror(errno));
+ rspamd_hard_terminate(rspamd_main);
+ }
+
+ if (!rspamd_socketpair(wrk->srv_pipe, SOCK_SEQPACKET)) {
+ msg_err("socketpair failure: %s", strerror(errno));
+ rspamd_hard_terminate(rspamd_main);
+ }
+
+ if (cf->bind_conf) {
+ msg_info_main("prepare to fork process %s (%d); listen on: %s",
+ cf->worker->name,
+ index, cf->bind_conf->name);
+ }
+ else {
+ msg_info_main("prepare to fork process %s (%d), no bind socket",
+ cf->worker->name,
+ index);
+ }
+
+ wrk->srv = rspamd_main;
+ wrk->type = cf->type;
+ wrk->cf = cf;
+ wrk->flags = cf->worker->flags;
+ REF_RETAIN(cf);
+ wrk->index = index;
+ wrk->ctx = cf->ctx;
+ wrk->ppid = getpid();
+ wrk->pid = fork();
+ wrk->cores_throttled = rspamd_main->cores_throttling;
+ wrk->term_handler = term_handler;
+ wrk->control_events_pending = g_hash_table_new_full(g_direct_hash, g_direct_equal,
+ NULL, rspamd_pending_control_free);
+
+ switch (wrk->pid) {
+ case 0:
+ rspamd_current_worker = wrk;
+ rspamd_handle_child_fork(wrk, rspamd_main, cf, listen_sockets);
+ break;
+ case -1:
+ msg_err_main("cannot fork main process: %s", strerror(errno));
+
+ if (rspamd_main->pfh) {
+ rspamd_pidfile_remove(rspamd_main->pfh);
+ }
+
+ rspamd_hard_terminate(rspamd_main);
+ break;
+ default:
+ rspamd_handle_main_fork(wrk, rspamd_main, cf, ev_base);
+ break;
+ }
+
+ return wrk;
+}
+
+void rspamd_worker_block_signals(void)
+{
+ sigset_t set;
+
+ sigemptyset(&set);
+ sigaddset(&set, SIGTERM);
+ sigaddset(&set, SIGINT);
+ sigaddset(&set, SIGHUP);
+ sigaddset(&set, SIGUSR1);
+ sigaddset(&set, SIGUSR2);
+ sigprocmask(SIG_BLOCK, &set, NULL);
+}
+
+void rspamd_worker_unblock_signals(void)
+{
+ sigset_t set;
+
+ sigemptyset(&set);
+ sigaddset(&set, SIGTERM);
+ sigaddset(&set, SIGINT);
+ sigaddset(&set, SIGHUP);
+ sigaddset(&set, SIGUSR1);
+ sigaddset(&set, SIGUSR2);
+ sigprocmask(SIG_UNBLOCK, &set, NULL);
+}
+
+void rspamd_hard_terminate(struct rspamd_main *rspamd_main)
+{
+ GHashTableIter it;
+ gpointer k, v;
+ struct rspamd_worker *w;
+ sigset_t set;
+
+ /* Block all signals */
+ sigemptyset(&set);
+ sigaddset(&set, SIGTERM);
+ sigaddset(&set, SIGINT);
+ sigaddset(&set, SIGHUP);
+ sigaddset(&set, SIGUSR1);
+ sigaddset(&set, SIGUSR2);
+ sigaddset(&set, SIGCHLD);
+ sigprocmask(SIG_BLOCK, &set, NULL);
+
+ /* We need to terminate all workers that might be already spawned */
+ rspamd_worker_block_signals();
+ g_hash_table_iter_init(&it, rspamd_main->workers);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ w = v;
+ msg_err_main("kill worker %P as Rspamd is terminating due to "
+ "an unrecoverable error",
+ w->pid);
+ kill(w->pid, SIGKILL);
+ }
+
+ msg_err_main("shutting down Rspamd due to fatal error");
+
+ rspamd_log_close(rspamd_main->logger);
+ exit(EXIT_FAILURE);
+}
+
+gboolean
+rspamd_worker_is_scanner(struct rspamd_worker *w)
+{
+
+ if (w) {
+ return !!(w->flags & RSPAMD_WORKER_SCANNER);
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_worker_is_primary_controller(struct rspamd_worker *w)
+{
+
+ if (w) {
+ return !!(w->flags & RSPAMD_WORKER_CONTROLLER) && w->index == 0;
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_worker_check_controller_presence(struct rspamd_worker *w)
+{
+ if (w->index == 0) {
+ GQuark our_type = w->type;
+ gboolean controller_seen = FALSE;
+ GList *cur;
+
+ enum {
+ low_priority_worker,
+ high_priority_worker
+ } our_priority;
+
+ if (our_type == g_quark_from_static_string("rspamd_proxy")) {
+ our_priority = low_priority_worker;
+ }
+ else if (our_type == g_quark_from_static_string("normal")) {
+ our_priority = high_priority_worker;
+ }
+ else {
+ msg_err("function is called for a wrong worker type: %s", g_quark_to_string(our_type));
+ return FALSE;
+ }
+
+ cur = w->srv->cfg->workers;
+
+ while (cur) {
+ struct rspamd_worker_conf *cf;
+
+ cf = (struct rspamd_worker_conf *) cur->data;
+
+ if (our_priority == low_priority_worker) {
+ if ((cf->type == g_quark_from_static_string("controller")) ||
+ (cf->type == g_quark_from_static_string("normal"))) {
+
+ if (cf->enabled && cf->count >= 0) {
+ controller_seen = TRUE;
+ break;
+ }
+ }
+ }
+ else {
+ if (cf->type == g_quark_from_static_string("controller")) {
+ if (cf->enabled && cf->count >= 0) {
+ controller_seen = TRUE;
+ break;
+ }
+ }
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ if (!controller_seen) {
+ msg_info("no controller or normal workers defined, execute "
+ "controller periodics in this worker");
+ w->flags |= RSPAMD_WORKER_CONTROLLER;
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+struct rspamd_worker_session_elt {
+ void *ptr;
+ guint *pref;
+ const gchar *tag;
+ time_t when;
+};
+
+struct rspamd_worker_session_cache {
+ struct ev_loop *ev_base;
+ GHashTable *cache;
+ struct rspamd_config *cfg;
+ struct ev_timer periodic;
+};
+
+static gint
+rspamd_session_cache_sort_cmp(gconstpointer pa, gconstpointer pb)
+{
+ const struct rspamd_worker_session_elt
+ *e1 = *(const struct rspamd_worker_session_elt **) pa,
+ *e2 = *(const struct rspamd_worker_session_elt **) pb;
+
+ return e2->when < e1->when;
+}
+
+static void
+rspamd_sessions_cache_periodic(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_worker_session_cache *c =
+ (struct rspamd_worker_session_cache *) w->data;
+ GHashTableIter it;
+ gchar timebuf[32];
+ gpointer k, v;
+ struct rspamd_worker_session_elt *elt;
+ struct tm tms;
+ GPtrArray *res;
+ guint i;
+
+ if (g_hash_table_size(c->cache) > c->cfg->max_sessions_cache) {
+ res = g_ptr_array_sized_new(g_hash_table_size(c->cache));
+ g_hash_table_iter_init(&it, c->cache);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ g_ptr_array_add(res, v);
+ }
+
+ msg_err("sessions cache is overflowed %d elements where %d is limit",
+ (gint) res->len, (gint) c->cfg->max_sessions_cache);
+ g_ptr_array_sort(res, rspamd_session_cache_sort_cmp);
+
+ PTR_ARRAY_FOREACH(res, i, elt)
+ {
+ rspamd_localtime(elt->when, &tms);
+ strftime(timebuf, sizeof(timebuf), "%F %H:%M:%S", &tms);
+
+ msg_warn("redundant session; ptr: %p, "
+ "tag: %s, refcount: %d, time: %s",
+ elt->ptr, elt->tag ? elt->tag : "unknown",
+ elt->pref ? *elt->pref : 0,
+ timebuf);
+ }
+ }
+
+ ev_timer_again(EV_A_ w);
+}
+
+void *
+rspamd_worker_session_cache_new(struct rspamd_worker *w,
+ struct ev_loop *ev_base)
+{
+ struct rspamd_worker_session_cache *c;
+ static const gdouble periodic_interval = 60.0;
+
+ c = g_malloc0(sizeof(*c));
+ c->ev_base = ev_base;
+ c->cache = g_hash_table_new_full(g_direct_hash, g_direct_equal,
+ NULL, g_free);
+ c->cfg = w->srv->cfg;
+ c->periodic.data = c;
+ ev_timer_init(&c->periodic, rspamd_sessions_cache_periodic, periodic_interval,
+ periodic_interval);
+ ev_timer_start(ev_base, &c->periodic);
+
+ return c;
+}
+
+
+void rspamd_worker_session_cache_add(void *cache, const gchar *tag,
+ guint *pref, void *ptr)
+{
+ struct rspamd_worker_session_cache *c = cache;
+ struct rspamd_worker_session_elt *elt;
+
+ elt = g_malloc0(sizeof(*elt));
+ elt->pref = pref;
+ elt->ptr = ptr;
+ elt->tag = tag;
+ elt->when = time(NULL);
+
+ g_hash_table_insert(c->cache, elt->ptr, elt);
+}
+
+
+void rspamd_worker_session_cache_remove(void *cache, void *ptr)
+{
+ struct rspamd_worker_session_cache *c = cache;
+
+ g_hash_table_remove(c->cache, ptr);
+}
+
+static void
+rspamd_worker_monitored_on_change(struct rspamd_monitored_ctx *ctx,
+ struct rspamd_monitored *m, gboolean alive,
+ void *ud)
+{
+ struct rspamd_worker *worker = ud;
+ struct rspamd_config *cfg = worker->srv->cfg;
+ struct ev_loop *ev_base;
+ guchar tag[RSPAMD_MONITORED_TAG_LEN];
+ static struct rspamd_srv_command srv_cmd;
+
+ rspamd_monitored_get_tag(m, tag);
+ ev_base = rspamd_monitored_ctx_get_ev_base(ctx);
+ memset(&srv_cmd, 0, sizeof(srv_cmd));
+ srv_cmd.type = RSPAMD_SRV_MONITORED_CHANGE;
+ rspamd_strlcpy(srv_cmd.cmd.monitored_change.tag, tag,
+ sizeof(srv_cmd.cmd.monitored_change.tag));
+ srv_cmd.cmd.monitored_change.alive = alive;
+ srv_cmd.cmd.monitored_change.sender = getpid();
+ msg_info_config("broadcast monitored update for %s: %s",
+ srv_cmd.cmd.monitored_change.tag, alive ? "alive" : "dead");
+
+ rspamd_srv_send_command(worker, ev_base, &srv_cmd, -1, NULL, NULL);
+}
+
+void rspamd_worker_init_monitored(struct rspamd_worker *worker,
+ struct ev_loop *ev_base,
+ struct rspamd_dns_resolver *resolver)
+{
+ rspamd_monitored_ctx_config(worker->srv->cfg->monitored_ctx,
+ worker->srv->cfg, ev_base, resolver->r,
+ rspamd_worker_monitored_on_change, worker);
+}
+
+#ifdef HAVE_SA_SIGINFO
+
+static struct rspamd_main *saved_main = NULL;
+static gboolean
+rspamd_crash_propagate(gpointer key, gpointer value, gpointer unused)
+{
+ struct rspamd_worker *w = value;
+
+ /* Kill children softly */
+ kill(w->pid, SIGTERM);
+
+ return TRUE;
+}
+
+#ifdef BACKWARD_ENABLE
+/* See backtrace.cxx */
+extern void rspamd_print_crash(void);
+#endif
+
+static void
+rspamd_crash_sig_handler(int sig, siginfo_t *info, void *ctx)
+{
+ struct sigaction sa;
+ ucontext_t *uap = ctx;
+ pid_t pid;
+
+ pid = getpid();
+ msg_err("caught fatal signal %d(%s), "
+ "pid: %P, trace: ",
+ sig, strsignal(sig), pid);
+ (void) uap;
+#ifdef BACKWARD_ENABLE
+ rspamd_print_crash();
+#endif
+ msg_err("please see Rspamd FAQ to learn how to dump core files and how to "
+ "fill a bug report");
+
+ if (saved_main) {
+ if (pid == saved_main->pid) {
+ /*
+ * Main process has crashed, propagate crash further to trigger
+ * monitoring alerts and mass panic
+ */
+ g_hash_table_foreach_remove(saved_main->workers,
+ rspamd_crash_propagate, NULL);
+ }
+ }
+
+ /*
+ * Invoke signal with the default handler
+ */
+ sigemptyset(&sa.sa_mask);
+ sa.sa_handler = SIG_DFL;
+ sa.sa_flags = 0;
+ sigaction(sig, &sa, NULL);
+ kill(pid, sig);
+}
+#endif
+
+RSPAMD_NO_SANITIZE void
+rspamd_set_crash_handler(struct rspamd_main *rspamd_main)
+{
+#ifdef HAVE_SA_SIGINFO
+ struct sigaction sa;
+
+#ifdef HAVE_SIGALTSTACK
+ void *stack_mem;
+ stack_t ss;
+ memset(&ss, 0, sizeof ss);
+
+ ss.ss_size = MAX(SIGSTKSZ, 8192 * 4);
+ stack_mem = g_malloc0(ss.ss_size);
+ ss.ss_sp = stack_mem;
+ sigaltstack(&ss, NULL);
+#endif
+ saved_main = rspamd_main;
+ sigemptyset(&sa.sa_mask);
+ sa.sa_sigaction = &rspamd_crash_sig_handler;
+ sa.sa_flags = SA_RESTART | SA_SIGINFO | SA_ONSTACK;
+ sigaction(SIGSEGV, &sa, NULL);
+ sigaction(SIGBUS, &sa, NULL);
+ sigaction(SIGABRT, &sa, NULL);
+ sigaction(SIGFPE, &sa, NULL);
+ sigaction(SIGSYS, &sa, NULL);
+#endif
+}
+
+RSPAMD_NO_SANITIZE void rspamd_unset_crash_handler(struct rspamd_main *unused_)
+{
+#ifdef HAVE_SIGALTSTACK
+ int ret;
+ stack_t ss;
+ ret = sigaltstack(NULL, &ss);
+
+ if (ret != -1) {
+ if (ss.ss_size > 0 && ss.ss_sp) {
+ g_free(ss.ss_sp);
+ }
+
+ ss.ss_size = 0;
+ ss.ss_sp = NULL;
+#ifdef SS_DISABLE
+ ss.ss_flags |= SS_DISABLE;
+#endif
+ sigaltstack(&ss, NULL);
+ }
+#endif
+}
+
+static void
+rspamd_enable_accept_event(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_worker_accept_event *ac_ev =
+ (struct rspamd_worker_accept_event *) w->data;
+
+ ev_timer_stop(EV_A_ w);
+ ev_io_start(EV_A_ & ac_ev->accept_ev);
+}
+
+void rspamd_worker_throttle_accept_events(gint sock, void *data)
+{
+ struct rspamd_worker_accept_event *head, *cur;
+ const gdouble throttling = 0.5;
+
+ head = (struct rspamd_worker_accept_event *) data;
+
+ DL_FOREACH(head, cur)
+ {
+
+ ev_io_stop(cur->event_loop, &cur->accept_ev);
+ cur->throttling_ev.data = cur;
+ ev_timer_init(&cur->throttling_ev, rspamd_enable_accept_event,
+ throttling, 0.0);
+ ev_timer_start(cur->event_loop, &cur->throttling_ev);
+ }
+}
+
+gboolean
+rspamd_check_termination_clause(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *wrk,
+ int res)
+{
+ gboolean need_refork = TRUE;
+
+ if (wrk->state != rspamd_worker_state_running || rspamd_main->wanna_die ||
+ (wrk->flags & RSPAMD_WORKER_OLD_CONFIG)) {
+ /* Do not refork workers that are intended to be terminated */
+ need_refork = FALSE;
+ }
+
+ if (WIFEXITED(res) && WEXITSTATUS(res) == 0) {
+ /* Normal worker termination, do not fork one more */
+
+ if (wrk->flags & RSPAMD_WORKER_OLD_CONFIG) {
+ /* Never re-fork old workers */
+ msg_info_main("%s process %P terminated normally",
+ g_quark_to_string(wrk->type),
+ wrk->pid);
+ need_refork = FALSE;
+ }
+ else {
+ if (wrk->hb.nbeats < 0 && rspamd_main->cfg->heartbeats_loss_max > 0 &&
+ -(wrk->hb.nbeats) >= rspamd_main->cfg->heartbeats_loss_max) {
+ msg_info_main("%s process %P terminated normally, but lost %L "
+ "heartbeats, refork it",
+ g_quark_to_string(wrk->type),
+ wrk->pid,
+ -(wrk->hb.nbeats));
+ need_refork = TRUE;
+ }
+ else {
+ msg_info_main("%s process %P terminated normally",
+ g_quark_to_string(wrk->type),
+ wrk->pid);
+ need_refork = FALSE;
+ }
+ }
+ }
+ else {
+ if (WIFSIGNALED(res)) {
+#ifdef WCOREDUMP
+ if (WCOREDUMP(res)) {
+ msg_warn_main(
+ "%s process %P terminated abnormally by signal: %s"
+ " and created core file; please see Rspamd FAQ "
+ "to learn how to extract data from core file and "
+ "fill a bug report",
+ g_quark_to_string(wrk->type),
+ wrk->pid,
+ g_strsignal(WTERMSIG(res)));
+ }
+ else {
+#ifdef HAVE_SYS_RESOURCE_H
+ struct rlimit rlmt;
+ (void) getrlimit(RLIMIT_CORE, &rlmt);
+
+ msg_warn_main(
+ "%s process %P terminated abnormally with exit code %d by "
+ "signal: %s"
+ " but NOT created core file (throttled=%s); "
+ "core file limits: %L current, %L max",
+ g_quark_to_string(wrk->type),
+ wrk->pid,
+ WEXITSTATUS(res),
+ g_strsignal(WTERMSIG(res)),
+ wrk->cores_throttled ? "yes" : "no",
+ (gint64) rlmt.rlim_cur,
+ (gint64) rlmt.rlim_max);
+#else
+ msg_warn_main(
+ "%s process %P terminated abnormally with exit code %d by signal: %s"
+ " but NOT created core file (throttled=%s); ",
+ g_quark_to_string(wrk->type),
+ wrk->pid, WEXITSTATUS(res),
+ g_strsignal(WTERMSIG(res)),
+ wrk->cores_throttled ? "yes" : "no");
+#endif
+ }
+#else
+ msg_warn_main(
+ "%s process %P terminated abnormally with exit code %d by signal: %s",
+ g_quark_to_string(wrk->type),
+ wrk->pid, WEXITSTATUS(res),
+ g_strsignal(WTERMSIG(res)));
+#endif
+ if (WTERMSIG(res) == SIGUSR2) {
+ /*
+ * It is actually race condition when not started process
+ * has been requested to be reloaded.
+ *
+ * We shouldn't refork on this
+ */
+ need_refork = FALSE;
+ }
+ }
+ else {
+ msg_warn_main("%s process %P terminated abnormally "
+ "(but it was not killed by a signal) "
+ "with exit code %d",
+ g_quark_to_string(wrk->type),
+ wrk->pid,
+ WEXITSTATUS(res));
+ }
+ }
+
+ return need_refork;
+}
+
+#ifdef WITH_HYPERSCAN
+gboolean
+rspamd_worker_hyperscan_ready(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker, gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud)
+{
+ struct rspamd_control_reply rep;
+ struct rspamd_re_cache *cache = worker->srv->cfg->re_cache;
+
+ memset(&rep, 0, sizeof(rep));
+ rep.type = RSPAMD_CONTROL_HYPERSCAN_LOADED;
+
+ if (rspamd_re_cache_is_hs_loaded(cache) != RSPAMD_HYPERSCAN_LOADED_FULL ||
+ cmd->cmd.hs_loaded.forced) {
+
+ msg_info("loading hyperscan expressions after receiving compilation "
+ "notice: %s",
+ (rspamd_re_cache_is_hs_loaded(cache) != RSPAMD_HYPERSCAN_LOADED_FULL) ? "new db" : "forced update");
+ rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan(
+ worker->srv->cfg->re_cache, cmd->cmd.hs_loaded.cache_dir, false);
+ }
+
+ if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
+ msg_err("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ return TRUE;
+}
+#endif /* With Hyperscan */
+
+gboolean
+rspamd_worker_check_context(gpointer ctx, guint64 magic)
+{
+ struct rspamd_abstract_worker_ctx *actx = (struct rspamd_abstract_worker_ctx *) ctx;
+
+ return actx->magic == magic;
+}
+
+static gboolean
+rspamd_worker_log_pipe_handler(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker, gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud)
+{
+ struct rspamd_config *cfg = ud;
+ struct rspamd_worker_log_pipe *lp;
+ struct rspamd_control_reply rep;
+
+ memset(&rep, 0, sizeof(rep));
+ rep.type = RSPAMD_CONTROL_LOG_PIPE;
+
+ if (attached_fd != -1) {
+ lp = g_malloc0(sizeof(*lp));
+ lp->fd = attached_fd;
+ lp->type = cmd->cmd.log_pipe.type;
+
+ DL_APPEND(cfg->log_pipes, lp);
+ msg_info("added new log pipe");
+ }
+ else {
+ rep.reply.log_pipe.status = ENOENT;
+ msg_err("cannot attach log pipe: invalid fd");
+ }
+
+ if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
+ msg_err("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_worker_monitored_handler(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker, gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud)
+{
+ struct rspamd_control_reply rep;
+ struct rspamd_monitored *m;
+ struct rspamd_monitored_ctx *mctx = worker->srv->cfg->monitored_ctx;
+ struct rspamd_config *cfg = ud;
+
+ memset(&rep, 0, sizeof(rep));
+ rep.type = RSPAMD_CONTROL_MONITORED_CHANGE;
+
+ if (cmd->cmd.monitored_change.sender != getpid()) {
+ m = rspamd_monitored_by_tag(mctx, cmd->cmd.monitored_change.tag);
+
+ if (m != NULL) {
+ rspamd_monitored_set_alive(m, cmd->cmd.monitored_change.alive);
+ rep.reply.monitored_change.status = 1;
+ msg_info_config("updated monitored status for %s: %s",
+ cmd->cmd.monitored_change.tag,
+ cmd->cmd.monitored_change.alive ? "alive" : "dead");
+ }
+ else {
+ msg_err("cannot find monitored by tag: %*s", 32,
+ cmd->cmd.monitored_change.tag);
+ rep.reply.monitored_change.status = 0;
+ }
+ }
+
+ if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
+ msg_err("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ return TRUE;
+}
+
+void rspamd_worker_init_scanner(struct rspamd_worker *worker,
+ struct ev_loop *ev_base,
+ struct rspamd_dns_resolver *resolver,
+ struct rspamd_lang_detector **plang_det)
+{
+ rspamd_stat_init(worker->srv->cfg, ev_base);
+#ifdef WITH_HYPERSCAN
+ rspamd_control_worker_add_cmd_handler(worker,
+ RSPAMD_CONTROL_HYPERSCAN_LOADED,
+ rspamd_worker_hyperscan_ready,
+ NULL);
+#endif
+ rspamd_control_worker_add_cmd_handler(worker,
+ RSPAMD_CONTROL_LOG_PIPE,
+ rspamd_worker_log_pipe_handler,
+ worker->srv->cfg);
+ rspamd_control_worker_add_cmd_handler(worker,
+ RSPAMD_CONTROL_MONITORED_CHANGE,
+ rspamd_worker_monitored_handler,
+ worker->srv->cfg);
+
+ *plang_det = worker->srv->cfg->lang_det;
+}
+
+void rspamd_controller_store_saved_stats(struct rspamd_main *rspamd_main,
+ struct rspamd_config *cfg)
+{
+ struct rspamd_stat *stat;
+ ucl_object_t *top, *sub;
+ struct ucl_emitter_functions *efuncs;
+ gint i, fd;
+ FILE *fp;
+ gchar fpath[PATH_MAX];
+
+ if (cfg->stats_file == NULL) {
+ return;
+ }
+
+ rspamd_snprintf(fpath, sizeof(fpath), "%s.XXXXXXXX", cfg->stats_file);
+ fd = g_mkstemp_full(fpath, O_WRONLY | O_TRUNC, 00644);
+
+ if (fd == -1) {
+ msg_err_config("cannot open for writing controller stats from %s: %s",
+ fpath, strerror(errno));
+ return;
+ }
+
+ fp = fdopen(fd, "w");
+ stat = rspamd_main->stat;
+
+ top = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(top, ucl_object_fromint(stat->messages_scanned), "scanned", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(stat->messages_learned), "learned", 0, false);
+
+ if (stat->messages_scanned > 0) {
+ sub = ucl_object_typed_new(UCL_OBJECT);
+ for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
+ ucl_object_insert_key(sub,
+ ucl_object_fromint(stat->actions_stat[i]),
+ rspamd_action_to_str(i), 0, false);
+ }
+ ucl_object_insert_key(top, sub, "actions", 0, false);
+ }
+
+ ucl_object_insert_key(top,
+ ucl_object_fromint(stat->connections_count),
+ "connections", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(stat->control_connections_count),
+ "control_connections", 0, false);
+
+ efuncs = ucl_object_emit_file_funcs(fp);
+ if (!ucl_object_emit_full(top, UCL_EMIT_JSON_COMPACT,
+ efuncs, NULL)) {
+ msg_err_config("cannot write stats to %s: %s",
+ fpath, strerror(errno));
+
+ unlink(fpath);
+ }
+ else {
+ if (rename(fpath, cfg->stats_file) == -1) {
+ msg_err_config("cannot rename stats from %s to %s: %s",
+ fpath, cfg->stats_file, strerror(errno));
+ }
+ }
+
+ ucl_object_unref(top);
+ fclose(fp);
+ ucl_object_emit_funcs_free(efuncs);
+}
+
+static ev_timer rrd_timer;
+
+void rspamd_controller_on_terminate(struct rspamd_worker *worker,
+ struct rspamd_rrd_file *rrd)
+{
+ struct rspamd_abstract_worker_ctx *ctx;
+
+ ctx = (struct rspamd_abstract_worker_ctx *) worker->ctx;
+ rspamd_controller_store_saved_stats(worker->srv, worker->srv->cfg);
+
+ if (rrd) {
+ ev_timer_stop(ctx->event_loop, &rrd_timer);
+ msg_info("closing rrd file: %s", rrd->filename);
+ rspamd_rrd_close(rrd);
+ }
+}
+
+static void
+rspamd_controller_load_saved_stats(struct rspamd_main *rspamd_main,
+ struct rspamd_config *cfg)
+{
+ struct ucl_parser *parser;
+ ucl_object_t *obj;
+ const ucl_object_t *elt, *subelt;
+ struct rspamd_stat *stat, stat_copy;
+ gint i;
+
+ if (cfg->stats_file == NULL) {
+ return;
+ }
+
+ if (access(cfg->stats_file, R_OK) == -1) {
+ msg_err_config("cannot load controller stats from %s: %s",
+ cfg->stats_file, strerror(errno));
+ return;
+ }
+
+ parser = ucl_parser_new(0);
+
+ if (!ucl_parser_add_file(parser, cfg->stats_file)) {
+ msg_err_config("cannot parse controller stats from %s: %s",
+ cfg->stats_file, ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+
+ return;
+ }
+
+ obj = ucl_parser_get_object(parser);
+ ucl_parser_free(parser);
+
+ stat = rspamd_main->stat;
+ memcpy(&stat_copy, stat, sizeof(stat_copy));
+
+ elt = ucl_object_lookup(obj, "scanned");
+
+ if (elt != NULL && ucl_object_type(elt) == UCL_INT) {
+ stat_copy.messages_scanned = ucl_object_toint(elt);
+ }
+
+ elt = ucl_object_lookup(obj, "learned");
+
+ if (elt != NULL && ucl_object_type(elt) == UCL_INT) {
+ stat_copy.messages_learned = ucl_object_toint(elt);
+ }
+
+ elt = ucl_object_lookup(obj, "actions");
+
+ if (elt != NULL) {
+ for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
+ subelt = ucl_object_lookup(elt, rspamd_action_to_str(i));
+
+ if (subelt && ucl_object_type(subelt) == UCL_INT) {
+ stat_copy.actions_stat[i] = ucl_object_toint(subelt);
+ }
+ }
+ }
+
+ elt = ucl_object_lookup(obj, "connections_count");
+
+ if (elt != NULL && ucl_object_type(elt) == UCL_INT) {
+ stat_copy.connections_count = ucl_object_toint(elt);
+ }
+
+ elt = ucl_object_lookup(obj, "control_connections_count");
+
+ if (elt != NULL && ucl_object_type(elt) == UCL_INT) {
+ stat_copy.control_connections_count = ucl_object_toint(elt);
+ }
+
+ ucl_object_unref(obj);
+ memcpy(stat, &stat_copy, sizeof(stat_copy));
+}
+
+struct rspamd_controller_periodics_cbdata {
+ struct rspamd_worker *worker;
+ struct rspamd_rrd_file *rrd;
+ struct rspamd_stat *stat;
+ ev_timer save_stats_event;
+};
+
+static void
+rspamd_controller_rrd_update(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_controller_periodics_cbdata *cbd =
+ (struct rspamd_controller_periodics_cbdata *) w->data;
+ struct rspamd_stat *stat;
+ GArray ar;
+ gdouble points[METRIC_ACTION_MAX];
+ GError *err = NULL;
+ guint i;
+
+ g_assert(cbd->rrd != NULL);
+ stat = cbd->stat;
+
+ for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) {
+ points[i] = stat->actions_stat[i];
+ }
+
+ ar.data = (gchar *) points;
+ ar.len = sizeof(points);
+
+ if (!rspamd_rrd_add_record(cbd->rrd, &ar, rspamd_get_calendar_ticks(),
+ &err)) {
+ msg_err("cannot update rrd file: %e", err);
+ g_error_free(err);
+ }
+
+ /* Plan new event */
+ ev_timer_again(EV_A_ w);
+}
+
+static void
+rspamd_controller_stats_save_periodic(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_controller_periodics_cbdata *cbd =
+ (struct rspamd_controller_periodics_cbdata *) w->data;
+
+ rspamd_controller_store_saved_stats(cbd->worker->srv, cbd->worker->srv->cfg);
+ ev_timer_again(EV_A_ w);
+}
+
+void rspamd_worker_init_controller(struct rspamd_worker *worker,
+ struct rspamd_rrd_file **prrd)
+{
+ struct rspamd_abstract_worker_ctx *ctx;
+ static const ev_tstamp rrd_update_time = 1.0;
+
+ ctx = (struct rspamd_abstract_worker_ctx *) worker->ctx;
+ rspamd_controller_load_saved_stats(worker->srv, worker->srv->cfg);
+
+ if (worker->index == 0) {
+ /* Enable periodics and other stuff */
+ static struct rspamd_controller_periodics_cbdata cbd;
+ const ev_tstamp save_stats_interval = 60; /* 1 minute */
+
+ memset(&cbd, 0, sizeof(cbd));
+ cbd.save_stats_event.data = &cbd;
+ cbd.worker = worker;
+ cbd.stat = worker->srv->stat;
+
+ ev_timer_init(&cbd.save_stats_event,
+ rspamd_controller_stats_save_periodic,
+ save_stats_interval, save_stats_interval);
+ ev_timer_start(ctx->event_loop, &cbd.save_stats_event);
+
+ rspamd_map_watch(worker->srv->cfg, ctx->event_loop,
+ ctx->resolver, worker,
+ RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER);
+
+ if (prrd != NULL) {
+ if (ctx->cfg->rrd_file && worker->index == 0) {
+ GError *rrd_err = NULL;
+
+ *prrd = rspamd_rrd_file_default(ctx->cfg->rrd_file, &rrd_err);
+
+ if (*prrd) {
+ cbd.rrd = *prrd;
+ rrd_timer.data = &cbd;
+ ev_timer_init(&rrd_timer, rspamd_controller_rrd_update,
+ rrd_update_time, rrd_update_time);
+ ev_timer_start(ctx->event_loop, &rrd_timer);
+ }
+ else if (rrd_err) {
+ msg_err("cannot load rrd from %s: %e", ctx->cfg->rrd_file,
+ rrd_err);
+ g_error_free(rrd_err);
+ }
+ else {
+ msg_err("cannot load rrd from %s: unknown error",
+ ctx->cfg->rrd_file);
+ }
+ }
+ else {
+ *prrd = NULL;
+ }
+ }
+
+ if (!ctx->cfg->disable_monitored) {
+ rspamd_worker_init_monitored(worker,
+ ctx->event_loop, ctx->resolver);
+ }
+ }
+ else {
+ rspamd_map_watch(worker->srv->cfg, ctx->event_loop,
+ ctx->resolver, worker, RSPAMD_MAP_WATCH_SCANNER);
+ }
+}
+
+gdouble
+rspamd_worker_check_and_adjust_timeout(struct rspamd_config *cfg, gdouble timeout)
+{
+ if (isnan(timeout)) {
+ /* Use implicit timeout from cfg->task_timeout */
+ timeout = cfg->task_timeout;
+ }
+
+ if (isnan(timeout)) {
+ return timeout;
+ }
+
+ struct rspamd_symcache_timeout_result *tres = rspamd_symcache_get_max_timeout(cfg->cache);
+ g_assert(tres != 0);
+
+ if (tres->max_timeout > timeout) {
+ msg_info_config("configured task_timeout %.2f is less than maximum symbols cache timeout %.2f; "
+ "some symbols can be terminated before checks",
+ timeout, tres->max_timeout);
+ GString *buf = g_string_sized_new(512);
+ static const int max_displayed_items = 12;
+
+ for (int i = 0; i < MIN(tres->nitems, max_displayed_items); i++) {
+ if (i == 0) {
+ rspamd_printf_gstring(buf, "%s(%.2f)",
+ rspamd_symcache_item_name((struct rspamd_symcache_item *) tres->items[i].item),
+ tres->items[i].timeout);
+ }
+ else {
+ rspamd_printf_gstring(buf, "; %s(%.2f)",
+ rspamd_symcache_item_name((struct rspamd_symcache_item *) tres->items[i].item),
+ tres->items[i].timeout);
+ }
+ }
+ msg_info_config("list of top %d symbols by execution time: %v",
+ (int) MIN(tres->nitems, max_displayed_items),
+ buf);
+
+ g_string_free(buf, TRUE);
+ }
+
+ rspamd_symcache_timeout_result_free(tres);
+
+ /* TODO: maybe adjust timeout */
+ return timeout;
+}
diff --git a/src/libserver/worker_util.h b/src/libserver/worker_util.h
new file mode 100644
index 0000000..ef48188
--- /dev/null
+++ b/src/libserver/worker_util.h
@@ -0,0 +1,334 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef WORKER_UTIL_H_
+#define WORKER_UTIL_H_
+
+#include "config.h"
+#include "util.h"
+#include "libserver/http/http_connection.h"
+#include "rspamd.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef HAVE_SA_SIGINFO
+typedef void (*rspamd_sig_handler_t)(gint);
+#else
+
+typedef void (*rspamd_sig_handler_t)(gint, siginfo_t *, void *);
+
+#endif
+
+struct rspamd_worker;
+struct rspamd_worker_signal_handler;
+
+extern struct rspamd_worker *rspamd_current_worker;
+
+/**
+ * Init basic signals for a worker
+ * @param worker
+ * @param event_loop
+ */
+void rspamd_worker_init_signals(struct rspamd_worker *worker, struct ev_loop *event_loop);
+
+typedef void (*rspamd_accept_handler)(struct ev_loop *loop, ev_io *w, int revents);
+
+/**
+ * Prepare worker's startup
+ * @param worker worker structure
+ * @param name name of the worker
+ * @param sig_handler handler of main signals
+ * @param accept_handler handler of accept event for listen sockets
+ * @return event base suitable for a worker
+ */
+struct ev_loop *
+rspamd_prepare_worker(struct rspamd_worker *worker, const char *name,
+ rspamd_accept_handler hdl);
+
+/**
+ * Should be used to validate context for a worker as in assert like invocation
+ * @param ctx
+ * @param magic
+ * @return
+ */
+gboolean rspamd_worker_check_context(gpointer ctx, guint64 magic);
+
+/**
+ * Set special signal handler for a worker
+ */
+void rspamd_worker_set_signal_handler(int signo,
+ struct rspamd_worker *worker,
+ struct ev_loop *event_loop,
+ rspamd_worker_signal_cb_t handler,
+ void *handler_data);
+
+/**
+ * Stop accepting new connections for a worker
+ * @param worker
+ */
+void rspamd_worker_stop_accept(struct rspamd_worker *worker);
+
+typedef gint (*rspamd_controller_func_t)(
+ struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg,
+ struct module_ctx *ctx);
+
+struct rspamd_custom_controller_command {
+ const gchar *command;
+ struct module_ctx *ctx;
+ gboolean privileged;
+ gboolean require_message;
+ rspamd_controller_func_t handler;
+};
+
+struct rspamd_controller_worker_ctx;
+struct rspamd_lang_detector;
+
+struct rspamd_controller_session {
+ struct rspamd_controller_worker_ctx *ctx;
+ struct rspamd_worker *wrk;
+ rspamd_mempool_t *pool;
+ struct rspamd_task *task;
+ gchar *classifier;
+ rspamd_inet_addr_t *from_addr;
+ struct rspamd_config *cfg;
+ struct rspamd_lang_detector *lang_det;
+ gboolean is_spam;
+ gboolean is_read_only;
+};
+
+/**
+ * Send error using HTTP and JSON output
+ * @param entry router entry
+ * @param code error code
+ * @param error_msg error message
+ */
+void rspamd_controller_send_error(struct rspamd_http_connection_entry *entry,
+ gint code, const gchar *error_msg, ...);
+
+/**
+ * Send openmetrics-formatted strings using HTTP
+ * @param entry router entry
+ * @param str rspamd fstring buffer, ownership is transferred
+ */
+void rspamd_controller_send_openmetrics(struct rspamd_http_connection_entry *entry,
+ rspamd_fstring_t *str);
+
+/**
+ * Send a custom string using HTTP
+ * @param entry router entry
+ * @param str string to send
+ */
+void rspamd_controller_send_string(struct rspamd_http_connection_entry *entry,
+ const gchar *str);
+
+/**
+ * Send UCL using HTTP and JSON serialization
+ * @param entry router entry
+ * @param obj object to send
+ */
+void rspamd_controller_send_ucl(struct rspamd_http_connection_entry *entry,
+ ucl_object_t *obj);
+
+/**
+ * Return worker's control structure by its type
+ * @param type
+ * @return worker's control structure or NULL
+ */
+worker_t *rspamd_get_worker_by_type(struct rspamd_config *cfg, GQuark type);
+
+/**
+ * Block signals before terminations
+ */
+void rspamd_worker_block_signals(void);
+
+/**
+ * Unblock signals
+ */
+void rspamd_worker_unblock_signals(void);
+
+/**
+ * Kill rspamd main and all workers
+ * @param rspamd_main
+ */
+void rspamd_hard_terminate(struct rspamd_main *rspamd_main) G_GNUC_NORETURN;
+
+/**
+ * Returns TRUE if a specific worker is a scanner worker
+ * @param w
+ * @return
+ */
+gboolean rspamd_worker_is_scanner(struct rspamd_worker *w);
+
+/**
+ * Checks
+ * @param cfg
+ * @param timeout
+ * @return
+ */
+gdouble rspamd_worker_check_and_adjust_timeout(struct rspamd_config *cfg,
+ gdouble timeout);
+
+/**
+ * Returns TRUE if a specific worker is a primary controller
+ * @param w
+ * @return
+ */
+gboolean rspamd_worker_is_primary_controller(struct rspamd_worker *w);
+
+/**
+ * Returns TRUE if a specific worker should take a role of a controller
+ */
+gboolean rspamd_worker_check_controller_presence(struct rspamd_worker *w);
+
+/**
+ * Creates new session cache
+ * @param w
+ * @return
+ */
+void *rspamd_worker_session_cache_new(struct rspamd_worker *w,
+ struct ev_loop *ev_base);
+
+/**
+ * Adds a new session identified by pointer
+ * @param cache
+ * @param tag
+ * @param pref
+ * @param ptr
+ */
+void rspamd_worker_session_cache_add(void *cache, const gchar *tag,
+ guint *pref, void *ptr);
+
+/**
+ * Removes session from cache
+ * @param cache
+ * @param ptr
+ */
+void rspamd_worker_session_cache_remove(void *cache, void *ptr);
+
+/**
+ * Fork new worker with the specified configuration
+ */
+struct rspamd_worker *rspamd_fork_worker(struct rspamd_main *,
+ struct rspamd_worker_conf *, guint idx,
+ struct ev_loop *ev_base,
+ rspamd_worker_term_cb term_handler,
+ GHashTable *listen_sockets);
+
+/**
+ * Sets crash signals handlers if compiled with libunwind
+ */
+RSPAMD_NO_SANITIZE void rspamd_set_crash_handler(struct rspamd_main *);
+
+/**
+ * Restore memory for crash signals
+ */
+RSPAMD_NO_SANITIZE void rspamd_unset_crash_handler(struct rspamd_main *);
+
+/**
+ * Initialise the main monitoring worker
+ * @param worker
+ * @param ev_base
+ * @param resolver
+ */
+void rspamd_worker_init_monitored(struct rspamd_worker *worker,
+ struct ev_loop *ev_base,
+ struct rspamd_dns_resolver *resolver);
+
+/**
+ * Performs throttling for accept events
+ * @param sock
+ * @param data struct rspamd_worker_accept_event * list
+ */
+void rspamd_worker_throttle_accept_events(gint sock, void *data);
+
+/**
+ * Checks (and logs) the worker's termination status. Returns TRUE if a worker
+ * should be restarted.
+ * @param rspamd_main
+ * @param wrk
+ * @param status waitpid res
+ * @return TRUE if refork is desired
+ */
+gboolean rspamd_check_termination_clause(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *wrk, int status);
+
+/**
+ * Call for final scripts for a worker
+ * @param worker
+ * @return
+ */
+gboolean rspamd_worker_call_finish_handlers(struct rspamd_worker *worker);
+
+struct rspamd_rrd_file;
+/**
+ * Terminate controller worker
+ * @param worker
+ */
+void rspamd_controller_on_terminate(struct rspamd_worker *worker,
+ struct rspamd_rrd_file *rrd);
+
+/**
+ * Inits controller worker
+ * @param worker
+ * @param ev_base
+ * @param prrd
+ */
+void rspamd_worker_init_controller(struct rspamd_worker *worker,
+ struct rspamd_rrd_file **prrd);
+
+/**
+ * Saves stats
+ * @param rspamd_main
+ * @param cfg
+ */
+void rspamd_controller_store_saved_stats(struct rspamd_main *rspamd_main,
+ struct rspamd_config *cfg);
+
+#ifdef WITH_HYPERSCAN
+struct rspamd_control_command;
+
+gboolean rspamd_worker_hyperscan_ready(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker, gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud);
+
+#endif
+
+#define msg_err_main(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ rspamd_main->server_pool->tag.tagname, rspamd_main->server_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_main(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ rspamd_main->server_pool->tag.tagname, rspamd_main->server_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_notice_main(...) rspamd_default_log_function(G_LOG_LEVEL_MESSAGE, \
+ rspamd_main->server_pool->tag.tagname, rspamd_main->server_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_main(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ rspamd_main->server_pool->tag.tagname, rspamd_main->server_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* WORKER_UTIL_H_ */
diff --git a/src/libstat/CMakeLists.txt b/src/libstat/CMakeLists.txt
new file mode 100644
index 0000000..64d572a
--- /dev/null
+++ b/src/libstat/CMakeLists.txt
@@ -0,0 +1,25 @@
+# Librspamdserver
+SET(LIBSTATSRC ${CMAKE_CURRENT_SOURCE_DIR}/stat_config.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/stat_process.c)
+
+SET(TOKENIZERSSRC ${CMAKE_CURRENT_SOURCE_DIR}/tokenizers/tokenizers.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/tokenizers/osb.c)
+
+SET(CLASSIFIERSSRC ${CMAKE_CURRENT_SOURCE_DIR}/classifiers/bayes.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/classifiers/lua_classifier.c)
+
+SET(BACKENDSSRC ${CMAKE_CURRENT_SOURCE_DIR}/backends/mmaped_file.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/backends/sqlite3_backend.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/backends/cdb_backend.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/backends/http_backend.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/backends/redis_backend.cxx)
+
+SET(CACHESSRC ${CMAKE_CURRENT_SOURCE_DIR}/learn_cache/sqlite3_cache.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/learn_cache/redis_cache.cxx)
+
+SET(RSPAMD_STAT ${LIBSTATSRC}
+ ${TOKENIZERSSRC}
+ ${CLASSIFIERSSRC}
+ ${BACKENDSSRC}
+ ${CACHESSRC} PARENT_SCOPE)
+
diff --git a/src/libstat/backends/backends.h b/src/libstat/backends/backends.h
new file mode 100644
index 0000000..4b16950
--- /dev/null
+++ b/src/libstat/backends/backends.h
@@ -0,0 +1,127 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef BACKENDS_H_
+#define BACKENDS_H_
+
+#include "config.h"
+#include "ucl.h"
+
+#define RSPAMD_DEFAULT_BACKEND "mmap"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Forwarded declarations */
+struct rspamd_classifier_config;
+struct rspamd_statfile_config;
+struct rspamd_config;
+struct rspamd_stat_ctx;
+struct rspamd_token_result;
+struct rspamd_statfile;
+struct rspamd_task;
+
+struct rspamd_stat_backend {
+ const char *name;
+ bool read_only;
+
+ gpointer (*init)(struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg,
+ struct rspamd_statfile *st);
+
+ gpointer (*runtime)(struct rspamd_task *task,
+ struct rspamd_statfile_config *stcf,
+ gboolean learn, gpointer ctx,
+ gint id);
+
+ gboolean (*process_tokens)(struct rspamd_task *task, GPtrArray *tokens,
+ gint id,
+ gpointer ctx);
+
+ gboolean (*finalize_process)(struct rspamd_task *task,
+ gpointer runtime, gpointer ctx);
+
+ gboolean (*learn_tokens)(struct rspamd_task *task, GPtrArray *tokens,
+ gint id,
+ gpointer ctx);
+
+ gulong (*total_learns)(struct rspamd_task *task,
+ gpointer runtime, gpointer ctx);
+
+ gboolean (*finalize_learn)(struct rspamd_task *task,
+ gpointer runtime, gpointer ctx, GError **err);
+
+ gulong (*inc_learns)(struct rspamd_task *task,
+ gpointer runtime, gpointer ctx);
+
+ gulong (*dec_learns)(struct rspamd_task *task,
+ gpointer runtime, gpointer ctx);
+
+ ucl_object_t *(*get_stat)(gpointer runtime, gpointer ctx);
+
+ void (*close)(gpointer ctx);
+
+ gpointer (*load_tokenizer_config)(gpointer runtime, gsize *sz);
+
+ gpointer ctx;
+};
+
+#define RSPAMD_STAT_BACKEND_DEF(name) \
+ gpointer rspamd_##name##_init(struct rspamd_stat_ctx *ctx, \
+ struct rspamd_config *cfg, struct rspamd_statfile *st); \
+ gpointer rspamd_##name##_runtime(struct rspamd_task *task, \
+ struct rspamd_statfile_config *stcf, \
+ gboolean learn, gpointer ctx, gint id); \
+ gboolean rspamd_##name##_process_tokens(struct rspamd_task *task, \
+ GPtrArray *tokens, gint id, \
+ gpointer runtime); \
+ gboolean rspamd_##name##_finalize_process(struct rspamd_task *task, \
+ gpointer runtime, \
+ gpointer ctx); \
+ gboolean rspamd_##name##_learn_tokens(struct rspamd_task *task, \
+ GPtrArray *tokens, gint id, \
+ gpointer runtime); \
+ gboolean rspamd_##name##_finalize_learn(struct rspamd_task *task, \
+ gpointer runtime, \
+ gpointer ctx, GError **err); \
+ gulong rspamd_##name##_total_learns(struct rspamd_task *task, \
+ gpointer runtime, \
+ gpointer ctx); \
+ gulong rspamd_##name##_inc_learns(struct rspamd_task *task, \
+ gpointer runtime, \
+ gpointer ctx); \
+ gulong rspamd_##name##_dec_learns(struct rspamd_task *task, \
+ gpointer runtime, \
+ gpointer ctx); \
+ gulong rspamd_##name##_learns(struct rspamd_task *task, \
+ gpointer runtime, \
+ gpointer ctx); \
+ ucl_object_t *rspamd_##name##_get_stat(gpointer runtime, \
+ gpointer ctx); \
+ gpointer rspamd_##name##_load_tokenizer_config(gpointer runtime, \
+ gsize *len); \
+ void rspamd_##name##_close(gpointer ctx)
+
+RSPAMD_STAT_BACKEND_DEF(mmaped_file);
+RSPAMD_STAT_BACKEND_DEF(sqlite3);
+RSPAMD_STAT_BACKEND_DEF(cdb);
+RSPAMD_STAT_BACKEND_DEF(redis);
+RSPAMD_STAT_BACKEND_DEF(http);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* BACKENDS_H_ */
diff --git a/src/libstat/backends/cdb_backend.cxx b/src/libstat/backends/cdb_backend.cxx
new file mode 100644
index 0000000..81d87f3
--- /dev/null
+++ b/src/libstat/backends/cdb_backend.cxx
@@ -0,0 +1,491 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * CDB read only statistics backend
+ */
+
+#include "config.h"
+#include "stat_internal.h"
+#include "contrib/cdb/cdb.h"
+
+#include <utility>
+#include <memory>
+#include <string>
+#include <optional>
+#include "contrib/expected/expected.hpp"
+#include "contrib/ankerl/unordered_dense.h"
+#include "fmt/core.h"
+
+namespace rspamd::stat::cdb {
+
+/*
+ * Utility class to share cdb instances over statfiles instances, as each
+ * cdb has tokens for both ham and spam classes
+ */
+class cdb_shared_storage {
+public:
+ using cdb_element_t = std::shared_ptr<struct cdb>;
+ cdb_shared_storage() = default;
+
+ auto get_cdb(const char *path) const -> std::optional<cdb_element_t>
+ {
+ auto found = elts.find(path);
+
+ if (found != elts.end()) {
+ if (!found->second.expired()) {
+ return found->second.lock();
+ }
+ }
+
+ return std::nullopt;
+ }
+ /* Create a new smart pointer over POD cdb structure */
+ static auto new_cdb() -> cdb_element_t
+ {
+ auto ret = cdb_element_t(new struct cdb, cdb_deleter());
+ memset(ret.get(), 0, sizeof(struct cdb));
+ return ret;
+ }
+ /* Enclose cdb into storage */
+ auto push_cdb(const char *path, cdb_element_t cdbp) -> cdb_element_t
+ {
+ auto found = elts.find(path);
+
+ if (found != elts.end()) {
+ if (found->second.expired()) {
+ /* OK, move in lieu of the expired weak pointer */
+
+ found->second = cdbp;
+ return cdbp;
+ }
+ else {
+ /*
+ * Existing and not expired, return the existing one
+ */
+ return found->second.lock();
+ }
+ }
+ else {
+ /* Not existing, make a weak ptr and return the original */
+ elts.emplace(path, std::weak_ptr<struct cdb>(cdbp));
+ return cdbp;
+ }
+ }
+
+private:
+ /*
+ * We store weak pointers here to allow owning cdb statfiles to free
+ * expensive cdb before this cache is terminated (e.g. on dynamic cdb reload)
+ */
+ ankerl::unordered_dense::map<std::string, std::weak_ptr<struct cdb>> elts;
+
+ struct cdb_deleter {
+ void operator()(struct cdb *c) const
+ {
+ cdb_free(c);
+ delete c;
+ }
+ };
+};
+
+static cdb_shared_storage cdb_shared_storage;
+
+class ro_backend final {
+public:
+ explicit ro_backend(struct rspamd_statfile *_st, cdb_shared_storage::cdb_element_t _db)
+ : st(_st), db(std::move(_db))
+ {
+ }
+ ro_backend() = delete;
+ ro_backend(const ro_backend &) = delete;
+ ro_backend(ro_backend &&other) noexcept
+ {
+ *this = std::move(other);
+ }
+ ro_backend &operator=(ro_backend &&other) noexcept
+ {
+ std::swap(st, other.st);
+ std::swap(db, other.db);
+ std::swap(loaded, other.loaded);
+ std::swap(learns_spam, other.learns_spam);
+ std::swap(learns_ham, other.learns_ham);
+
+ return *this;
+ }
+ ~ro_backend()
+ {
+ }
+
+ auto load_cdb() -> tl::expected<bool, std::string>;
+ auto process_token(const rspamd_token_t *tok) const -> std::optional<float>;
+ constexpr auto is_spam() const -> bool
+ {
+ return st->stcf->is_spam;
+ }
+ auto get_learns() const -> std::uint64_t
+ {
+ if (is_spam()) {
+ return learns_spam;
+ }
+ else {
+ return learns_ham;
+ }
+ }
+ auto get_total_learns() const -> std::uint64_t
+ {
+ return learns_spam + learns_ham;
+ }
+
+private:
+ struct rspamd_statfile *st;
+ cdb_shared_storage::cdb_element_t db;
+ bool loaded = false;
+ std::uint64_t learns_spam = 0;
+ std::uint64_t learns_ham = 0;
+};
+
+template<typename T>
+static inline auto
+cdb_get_key_as_int64(struct cdb *cdb, T key) -> std::optional<std::int64_t>
+{
+ auto pos = cdb_find(cdb, (void *) &key, sizeof(key));
+
+ if (pos > 0) {
+ auto vpos = cdb_datapos(cdb);
+ auto vlen = cdb_datalen(cdb);
+
+ if (vlen == sizeof(std::int64_t)) {
+ std::int64_t ret;
+ cdb_read(cdb, (void *) &ret, vlen, vpos);
+
+ return ret;
+ }
+ }
+
+ return std::nullopt;
+}
+
+template<typename T>
+static inline auto
+cdb_get_key_as_float_pair(struct cdb *cdb, T key) -> std::optional<std::pair<float, float>>
+{
+ auto pos = cdb_find(cdb, (void *) &key, sizeof(key));
+
+ if (pos > 0) {
+ auto vpos = cdb_datapos(cdb);
+ auto vlen = cdb_datalen(cdb);
+
+ if (vlen == sizeof(float) * 2) {
+ union {
+ struct {
+ float v1;
+ float v2;
+ } d;
+ char c[sizeof(float) * 2];
+ } u;
+ cdb_read(cdb, (void *) u.c, vlen, vpos);
+
+ return std::make_pair(u.d.v1, u.d.v2);
+ }
+ }
+
+ return std::nullopt;
+}
+
+
+auto ro_backend::load_cdb() -> tl::expected<bool, std::string>
+{
+ if (!db) {
+ return tl::make_unexpected("no database loaded");
+ }
+
+ /* Now get number of learns */
+ std::int64_t cdb_key;
+ static const char learn_spam_key[9] = "_lrnspam", learn_ham_key[9] = "_lrnham_";
+
+ auto check_key = [&](const char *key, std::uint64_t &target) -> tl::expected<bool, std::string> {
+ memcpy((void *) &cdb_key, key, sizeof(cdb_key));
+
+ auto maybe_value = cdb_get_key_as_int64(db.get(), cdb_key);
+
+ if (!maybe_value) {
+ return tl::make_unexpected(fmt::format("missing {} key", key));
+ }
+
+ target = (std::uint64_t) maybe_value.value();
+
+ return true;
+ };
+
+ auto res = check_key(learn_spam_key, learns_spam);
+
+ if (!res) {
+ return res;
+ }
+
+ res = check_key(learn_ham_key, learns_ham);
+
+ if (!res) {
+ return res;
+ }
+
+ loaded = true;
+
+ return true;// expected
+}
+
+auto ro_backend::process_token(const rspamd_token_t *tok) const -> std::optional<float>
+{
+ if (!loaded) {
+ return std::nullopt;
+ }
+
+ auto maybe_value = cdb_get_key_as_float_pair(db.get(), tok->data);
+
+ if (maybe_value) {
+ auto [spam_count, ham_count] = maybe_value.value();
+
+ if (is_spam()) {
+ return spam_count;
+ }
+ else {
+ return ham_count;
+ }
+ }
+
+ return std::nullopt;
+}
+
+auto open_cdb(struct rspamd_statfile *st) -> tl::expected<ro_backend, std::string>
+{
+ const char *path = nullptr;
+ const auto *stf = st->stcf;
+
+ auto get_filename = [](const ucl_object_t *obj) -> const char * {
+ const auto *filename = ucl_object_lookup_any(obj,
+ "filename", "path", "cdb", nullptr);
+
+ if (filename && ucl_object_type(filename) == UCL_STRING) {
+ return ucl_object_tostring(filename);
+ }
+
+ return nullptr;
+ };
+
+ /* First search in backend configuration */
+ const auto *obj = ucl_object_lookup(st->classifier->cfg->opts, "backend");
+ if (obj != NULL && ucl_object_type(obj) == UCL_OBJECT) {
+ path = get_filename(obj);
+ }
+
+ /* Now try statfiles config */
+ if (!path && stf->opts) {
+ path = get_filename(stf->opts);
+ }
+
+ /* Now try classifier config */
+ if (!path && st->classifier->cfg->opts) {
+ path = get_filename(st->classifier->cfg->opts);
+ }
+
+ if (!path) {
+ return tl::make_unexpected("missing/malformed filename attribute");
+ }
+
+ auto cached_cdb_maybe = cdb_shared_storage.get_cdb(path);
+ cdb_shared_storage::cdb_element_t cdbp;
+
+ if (!cached_cdb_maybe) {
+
+ auto fd = rspamd_file_xopen(path, O_RDONLY, 0, true);
+
+ if (fd == -1) {
+ return tl::make_unexpected(fmt::format("cannot open {}: {}",
+ path, strerror(errno)));
+ }
+
+ cdbp = cdb_shared_storage::new_cdb();
+
+ if (cdb_init(cdbp.get(), fd) == -1) {
+ close(fd);
+
+ return tl::make_unexpected(fmt::format("cannot init cdb in {}: {}",
+ path, strerror(errno)));
+ }
+
+ cdbp = cdb_shared_storage.push_cdb(path, cdbp);
+
+ close(fd);
+ }
+ else {
+ cdbp = cached_cdb_maybe.value();
+ }
+
+ if (!cdbp) {
+ return tl::make_unexpected(fmt::format("cannot init cdb in {}: internal error",
+ path));
+ }
+
+ ro_backend bk{st, std::move(cdbp)};
+
+ auto res = bk.load_cdb();
+
+ if (!res) {
+ return tl::make_unexpected(res.error());
+ }
+
+ return bk;
+}
+
+}// namespace rspamd::stat::cdb
+
+#define CDB_FROM_RAW(p) (reinterpret_cast<rspamd::stat::cdb::ro_backend *>(p))
+
+/* C exports */
+gpointer
+rspamd_cdb_init(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct rspamd_statfile *st)
+{
+ auto maybe_backend = rspamd::stat::cdb::open_cdb(st);
+
+ if (maybe_backend) {
+ /* Move into a new pointer */
+ auto *result = new rspamd::stat::cdb::ro_backend(std::move(maybe_backend.value()));
+
+ return result;
+ }
+ else {
+ msg_err_config("cannot load cdb backend: %s", maybe_backend.error().c_str());
+ }
+
+ return nullptr;
+}
+gpointer
+rspamd_cdb_runtime(struct rspamd_task *task,
+ struct rspamd_statfile_config *stcf,
+ gboolean learn,
+ gpointer ctx,
+ gint _id)
+{
+ /* In CDB we don't have any dynamic stuff */
+ return ctx;
+}
+
+gboolean
+rspamd_cdb_process_tokens(struct rspamd_task *task,
+ GPtrArray *tokens,
+ gint id,
+ gpointer runtime)
+{
+ auto *cdbp = CDB_FROM_RAW(runtime);
+ bool seen_values = false;
+
+ for (auto i = 0u; i < tokens->len; i++) {
+ rspamd_token_t *tok;
+ tok = reinterpret_cast<rspamd_token_t *>(g_ptr_array_index(tokens, i));
+
+ auto res = cdbp->process_token(tok);
+
+ if (res) {
+ tok->values[id] = res.value();
+ seen_values = true;
+ }
+ else {
+ tok->values[id] = 0;
+ }
+ }
+
+ if (seen_values) {
+ if (cdbp->is_spam()) {
+ task->flags |= RSPAMD_TASK_FLAG_HAS_SPAM_TOKENS;
+ }
+ else {
+ task->flags |= RSPAMD_TASK_FLAG_HAS_HAM_TOKENS;
+ }
+ }
+
+ return true;
+}
+gboolean
+rspamd_cdb_finalize_process(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx)
+{
+ return true;
+}
+gboolean
+rspamd_cdb_learn_tokens(struct rspamd_task *task,
+ GPtrArray *tokens,
+ gint id,
+ gpointer ctx)
+{
+ return false;
+}
+gboolean
+rspamd_cdb_finalize_learn(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx,
+ GError **err)
+{
+ return false;
+}
+
+gulong rspamd_cdb_total_learns(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx)
+{
+ auto *cdbp = CDB_FROM_RAW(ctx);
+ return cdbp->get_total_learns();
+}
+gulong
+rspamd_cdb_inc_learns(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx)
+{
+ return (gulong) -1;
+}
+gulong
+rspamd_cdb_dec_learns(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx)
+{
+ return (gulong) -1;
+}
+gulong
+rspamd_cdb_learns(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx)
+{
+ auto *cdbp = CDB_FROM_RAW(ctx);
+ return cdbp->get_learns();
+}
+ucl_object_t *
+rspamd_cdb_get_stat(gpointer runtime, gpointer ctx)
+{
+ return nullptr;
+}
+gpointer
+rspamd_cdb_load_tokenizer_config(gpointer runtime, gsize *len)
+{
+ return nullptr;
+}
+void rspamd_cdb_close(gpointer ctx)
+{
+ auto *cdbp = CDB_FROM_RAW(ctx);
+ delete cdbp;
+} \ No newline at end of file
diff --git a/src/libstat/backends/http_backend.cxx b/src/libstat/backends/http_backend.cxx
new file mode 100644
index 0000000..075e508
--- /dev/null
+++ b/src/libstat/backends/http_backend.cxx
@@ -0,0 +1,440 @@
+/*-
+ * Copyright 2022 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "stat_internal.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/mempool_vars_internal.h"
+#include "upstream.h"
+#include "contrib/ankerl/unordered_dense.h"
+#include <algorithm>
+#include <vector>
+
+namespace rspamd::stat::http {
+
+#define msg_debug_stat_http(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_stat_http_log_id, "stat_http", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(stat_http)
+
+/* Represents all http backends defined in some configuration */
+class http_backends_collection {
+ std::vector<struct rspamd_statfile *> backends;
+ double timeout = 1.0; /* Default timeout */
+ struct upstream_list *read_servers = nullptr;
+ struct upstream_list *write_servers = nullptr;
+
+public:
+ static auto get() -> http_backends_collection &
+ {
+ static http_backends_collection *singleton = nullptr;
+
+ if (singleton == nullptr) {
+ singleton = new http_backends_collection;
+ }
+
+ return *singleton;
+ }
+
+ /**
+ * Add a new backend and (optionally initialize the basic backend parameters
+ * @param ctx
+ * @param cfg
+ * @param st
+ * @return
+ */
+ auto add_backend(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct rspamd_statfile *st) -> bool;
+ /**
+ * Remove a statfile cleaning things up if the last statfile is removed
+ * @param st
+ * @return
+ */
+ auto remove_backend(struct rspamd_statfile *st) -> bool;
+
+ upstream *get_upstream(bool is_learn);
+
+private:
+ http_backends_collection() = default;
+ auto first_init(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct rspamd_statfile *st) -> bool;
+};
+
+/*
+ * Created one per each task
+ */
+class http_backend_runtime final {
+public:
+ static auto create(struct rspamd_task *task, bool is_learn) -> http_backend_runtime *;
+ /* Add a new statfile with a specific id to the list of statfiles */
+ auto notice_statfile(int id, const struct rspamd_statfile_config *st) -> void
+ {
+ seen_statfiles[id] = st;
+ }
+
+ auto process_tokens(struct rspamd_task *task,
+ GPtrArray *tokens,
+ gint id,
+ bool learn) -> bool;
+
+private:
+ http_backends_collection *all_backends;
+ ankerl::unordered_dense::map<int, const struct rspamd_statfile_config *> seen_statfiles;
+ struct upstream *selected;
+
+private:
+ http_backend_runtime(struct rspamd_task *task, bool is_learn)
+ : all_backends(&http_backends_collection::get())
+ {
+ selected = all_backends->get_upstream(is_learn);
+ }
+ ~http_backend_runtime() = default;
+ static auto dtor(void *p) -> void
+ {
+ ((http_backend_runtime *) p)->~http_backend_runtime();
+ }
+};
+
+/*
+ * Efficient way to make a messagepack payload from stat tokens,
+ * avoiding any intermediate libraries, as we would send many tokens
+ * all together
+ */
+static auto
+stat_tokens_to_msgpack(GPtrArray *tokens) -> std::vector<std::uint8_t>
+{
+ std::vector<std::uint8_t> ret;
+ rspamd_token_t *cur;
+ int i;
+
+ /*
+ * We define array, it's size and N elements each is uint64_t
+ * Layout:
+ * 0xdd - array marker
+ * [4 bytes be] - size of the array
+ * [ 0xcf + <8 bytes BE integer>] * N - array elements
+ */
+ ret.resize(tokens->len * (sizeof(std::uint64_t) + 1) + 5);
+ ret.push_back('\xdd');
+ std::uint32_t ulen = GUINT32_TO_BE(tokens->len);
+ std::copy((const std::uint8_t *) &ulen,
+ ((const std::uint8_t *) &ulen) + sizeof(ulen), std::back_inserter(ret));
+
+ PTR_ARRAY_FOREACH(tokens, i, cur)
+ {
+ ret.push_back('\xcf');
+ std::uint64_t val = GUINT64_TO_BE(cur->data);
+ std::copy((const std::uint8_t *) &val,
+ ((const std::uint8_t *) &val) + sizeof(val), std::back_inserter(ret));
+ }
+
+ return ret;
+}
+
+auto http_backend_runtime::create(struct rspamd_task *task, bool is_learn) -> http_backend_runtime *
+{
+ /* Alloc type provide proper size and alignment */
+ auto *allocated_runtime = rspamd_mempool_alloc_type(task->task_pool, http_backend_runtime);
+
+ rspamd_mempool_add_destructor(task->task_pool, http_backend_runtime::dtor, allocated_runtime);
+
+ return new (allocated_runtime) http_backend_runtime{task, is_learn};
+}
+
+auto http_backend_runtime::process_tokens(struct rspamd_task *task, GPtrArray *tokens, gint id, bool learn) -> bool
+{
+ if (!learn) {
+ if (id == seen_statfiles.size() - 1) {
+ /* Emit http request on the last statfile */
+ }
+ }
+ else {
+ /* On learn we need to learn all statfiles that we were requested to learn */
+ if (seen_statfiles.empty()) {
+ /* Request has been already set, or nothing to learn */
+ return true;
+ }
+ else {
+ seen_statfiles.clear();
+ }
+ }
+
+ return true;
+}
+
+auto http_backends_collection::add_backend(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct rspamd_statfile *st) -> bool
+{
+ /* On empty list of backends we know that we need to load backend data actually */
+ if (backends.empty()) {
+ if (!first_init(ctx, cfg, st)) {
+ return false;
+ }
+ }
+
+ backends.push_back(st);
+
+ return true;
+}
+
+auto http_backends_collection::first_init(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct rspamd_statfile *st) -> bool
+{
+ auto try_load_backend_config = [&](const ucl_object_t *obj) -> bool {
+ if (!obj || ucl_object_type(obj) != UCL_OBJECT) {
+ return false;
+ }
+
+ /* First try to load read servers */
+ auto *rs = ucl_object_lookup_any(obj, "read_servers", "servers", nullptr);
+ if (rs) {
+ read_servers = rspamd_upstreams_create(cfg->ups_ctx);
+
+ if (read_servers == nullptr) {
+ return false;
+ }
+
+ if (!rspamd_upstreams_from_ucl(read_servers, rs, 80, this)) {
+ rspamd_upstreams_destroy(read_servers);
+ return false;
+ }
+ }
+ auto *ws = ucl_object_lookup_any(obj, "write_servers", "servers", nullptr);
+ if (ws) {
+ write_servers = rspamd_upstreams_create(cfg->ups_ctx);
+
+ if (write_servers == nullptr) {
+ return false;
+ }
+
+ if (!rspamd_upstreams_from_ucl(write_servers, rs, 80, this)) {
+ rspamd_upstreams_destroy(write_servers);
+ return false;
+ }
+ }
+
+ auto *tim = ucl_object_lookup(obj, "timeout");
+
+ if (tim) {
+ timeout = ucl_object_todouble(tim);
+ }
+
+ return true;
+ };
+
+ auto ret = false;
+ auto obj = ucl_object_lookup(st->classifier->cfg->opts, "backend");
+ if (obj != nullptr) {
+ ret = try_load_backend_config(obj);
+ }
+
+ /* Now try statfiles config */
+ if (!ret && st->stcf->opts) {
+ ret = try_load_backend_config(st->stcf->opts);
+ }
+
+ /* Now try classifier config */
+ if (!ret && st->classifier->cfg->opts) {
+ ret = try_load_backend_config(st->classifier->cfg->opts);
+ }
+
+ return ret;
+}
+
+auto http_backends_collection::remove_backend(struct rspamd_statfile *st) -> bool
+{
+ auto backend_it = std::remove(std::begin(backends), std::end(backends), st);
+
+ if (backend_it != std::end(backends)) {
+ /* Fast erasure with no order preservation */
+ std::swap(*backend_it, backends.back());
+ backends.pop_back();
+
+ if (backends.empty()) {
+ /* De-init collection - likely config reload */
+ if (read_servers) {
+ rspamd_upstreams_destroy(read_servers);
+ read_servers = nullptr;
+ }
+
+ if (write_servers) {
+ rspamd_upstreams_destroy(write_servers);
+ write_servers = nullptr;
+ }
+ }
+
+ return true;
+ }
+
+ return false;
+}
+
+upstream *http_backends_collection::get_upstream(bool is_learn)
+{
+ auto *ups_list = read_servers;
+ if (is_learn) {
+ ups_list = write_servers;
+ }
+
+ return rspamd_upstream_get(ups_list, RSPAMD_UPSTREAM_ROUND_ROBIN, nullptr, 0);
+}
+
+}// namespace rspamd::stat::http
+
+/* C API */
+
+gpointer
+rspamd_http_init(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct rspamd_statfile *st)
+{
+ auto &collections = rspamd::stat::http::http_backends_collection::get();
+
+ if (!collections.add_backend(ctx, cfg, st)) {
+ msg_err_config("cannot load http backend");
+
+ return nullptr;
+ }
+
+ return (void *) &collections;
+}
+gpointer
+rspamd_http_runtime(struct rspamd_task *task,
+ struct rspamd_statfile_config *stcf,
+ gboolean learn,
+ gpointer ctx,
+ gint id)
+{
+ auto maybe_existing = rspamd_mempool_get_variable(task->task_pool, RSPAMD_MEMPOOL_HTTP_STAT_BACKEND_RUNTIME);
+
+ if (maybe_existing != nullptr) {
+ auto real_runtime = (rspamd::stat::http::http_backend_runtime *) maybe_existing;
+ real_runtime->notice_statfile(id, stcf);
+
+ return maybe_existing;
+ }
+
+ auto runtime = rspamd::stat::http::http_backend_runtime::create(task, learn);
+
+ if (runtime) {
+ runtime->notice_statfile(id, stcf);
+ rspamd_mempool_set_variable(task->task_pool, RSPAMD_MEMPOOL_HTTP_STAT_BACKEND_RUNTIME,
+ (void *) runtime, nullptr);
+ }
+
+ return (void *) runtime;
+}
+
+gboolean
+rspamd_http_process_tokens(struct rspamd_task *task,
+ GPtrArray *tokens,
+ gint id,
+ gpointer runtime)
+{
+ auto real_runtime = (rspamd::stat::http::http_backend_runtime *) runtime;
+
+ if (real_runtime) {
+ return real_runtime->process_tokens(task, tokens, id, false);
+ }
+
+
+ return false;
+}
+gboolean
+rspamd_http_finalize_process(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx)
+{
+ /* Not needed */
+ return true;
+}
+
+gboolean
+rspamd_http_learn_tokens(struct rspamd_task *task,
+ GPtrArray *tokens,
+ gint id,
+ gpointer runtime)
+{
+ auto real_runtime = (rspamd::stat::http::http_backend_runtime *) runtime;
+
+ if (real_runtime) {
+ return real_runtime->process_tokens(task, tokens, id, true);
+ }
+
+
+ return false;
+}
+gboolean
+rspamd_http_finalize_learn(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx,
+ GError **err)
+{
+ return false;
+}
+
+gulong rspamd_http_total_learns(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx)
+{
+ /* TODO */
+ return 0;
+}
+gulong
+rspamd_http_inc_learns(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx)
+{
+ /* TODO */
+ return 0;
+}
+gulong
+rspamd_http_dec_learns(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx)
+{
+ /* TODO */
+ return (gulong) -1;
+}
+gulong
+rspamd_http_learns(struct rspamd_task *task,
+ gpointer runtime,
+ gpointer ctx)
+{
+ /* TODO */
+ return 0;
+}
+ucl_object_t *
+rspamd_http_get_stat(gpointer runtime, gpointer ctx)
+{
+ /* TODO */
+ return nullptr;
+}
+gpointer
+rspamd_http_load_tokenizer_config(gpointer runtime, gsize *len)
+{
+ return nullptr;
+}
+void rspamd_http_close(gpointer ctx)
+{
+ /* TODO */
+} \ No newline at end of file
diff --git a/src/libstat/backends/mmaped_file.c b/src/libstat/backends/mmaped_file.c
new file mode 100644
index 0000000..5c20207
--- /dev/null
+++ b/src/libstat/backends/mmaped_file.c
@@ -0,0 +1,1113 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "stat_internal.h"
+#include "unix-std.h"
+
+#define CHAIN_LENGTH 128
+
+/* Section types */
+#define STATFILE_SECTION_COMMON 1
+
+/**
+ * Common statfile header
+ */
+struct stat_file_header {
+ u_char magic[3]; /**< magic signature ('r' 's' 'd') */
+ u_char version[2]; /**< version of statfile */
+ u_char padding[3]; /**< padding */
+ guint64 create_time; /**< create time (time_t->guint64) */
+ guint64 revision; /**< revision number */
+ guint64 rev_time; /**< revision time */
+ guint64 used_blocks; /**< used blocks number */
+ guint64 total_blocks; /**< total number of blocks */
+ guint64 tokenizer_conf_len; /**< length of tokenizer configuration */
+ u_char unused[231]; /**< some bytes that can be used in future */
+};
+
+/**
+ * Section header
+ */
+struct stat_file_section {
+ guint64 code; /**< section's code */
+ guint64 length; /**< section's length in blocks */
+};
+
+/**
+ * Block of data in statfile
+ */
+struct stat_file_block {
+ guint32 hash1; /**< hash1 (also acts as index) */
+ guint32 hash2; /**< hash2 */
+ double value; /**< double value */
+};
+
+/**
+ * Statistic file
+ */
+struct stat_file {
+ struct stat_file_header header; /**< header */
+ struct stat_file_section section; /**< first section */
+ struct stat_file_block blocks[1]; /**< first block of data */
+};
+
+/**
+ * Common view of statfile object
+ */
+typedef struct {
+#ifdef HAVE_PATH_MAX
+ gchar filename[PATH_MAX]; /**< name of file */
+#else
+ gchar filename[MAXPATHLEN]; /**< name of file */
+#endif
+ rspamd_mempool_t *pool;
+ gint fd; /**< descriptor */
+ void *map; /**< mmaped area */
+ off_t seek_pos; /**< current seek position */
+ struct stat_file_section cur_section; /**< current section */
+ size_t len; /**< length of file(in bytes) */
+ struct rspamd_statfile_config *cf;
+} rspamd_mmaped_file_t;
+
+
+#define RSPAMD_STATFILE_VERSION \
+ { \
+ '1', '2' \
+ }
+#define BACKUP_SUFFIX ".old"
+
+static void rspamd_mmaped_file_set_block_common(rspamd_mempool_t *pool,
+ rspamd_mmaped_file_t *file,
+ guint32 h1, guint32 h2, double value);
+
+rspamd_mmaped_file_t *rspamd_mmaped_file_open(rspamd_mempool_t *pool,
+ const gchar *filename, size_t size,
+ struct rspamd_statfile_config *stcf);
+gint rspamd_mmaped_file_create(const gchar *filename, size_t size,
+ struct rspamd_statfile_config *stcf,
+ rspamd_mempool_t *pool);
+gint rspamd_mmaped_file_close_file(rspamd_mempool_t *pool,
+ rspamd_mmaped_file_t *file);
+
+double
+rspamd_mmaped_file_get_block(rspamd_mmaped_file_t *file,
+ guint32 h1,
+ guint32 h2)
+{
+ struct stat_file_block *block;
+ guint i, blocknum;
+ u_char *c;
+
+ if (!file->map) {
+ return 0;
+ }
+
+ blocknum = h1 % file->cur_section.length;
+ c = (u_char *) file->map + file->seek_pos + blocknum * sizeof(struct stat_file_block);
+ block = (struct stat_file_block *) c;
+
+ for (i = 0; i < CHAIN_LENGTH; i++) {
+ if (i + blocknum >= file->cur_section.length) {
+ break;
+ }
+ if (block->hash1 == h1 && block->hash2 == h2) {
+ return block->value;
+ }
+ c += sizeof(struct stat_file_block);
+ block = (struct stat_file_block *) c;
+ }
+
+
+ return 0;
+}
+
+static void
+rspamd_mmaped_file_set_block_common(rspamd_mempool_t *pool,
+ rspamd_mmaped_file_t *file,
+ guint32 h1, guint32 h2, double value)
+{
+ struct stat_file_block *block, *to_expire = NULL;
+ struct stat_file_header *header;
+ guint i, blocknum;
+ u_char *c;
+ double min = G_MAXDOUBLE;
+
+ if (!file->map) {
+ return;
+ }
+
+ blocknum = h1 % file->cur_section.length;
+ header = (struct stat_file_header *) file->map;
+ c = (u_char *) file->map + file->seek_pos + blocknum * sizeof(struct stat_file_block);
+ block = (struct stat_file_block *) c;
+
+ for (i = 0; i < CHAIN_LENGTH; i++) {
+ if (i + blocknum >= file->cur_section.length) {
+ /* Need to expire some block in chain */
+ msg_info_pool("chain %ud is full in statfile %s, starting expire",
+ blocknum,
+ file->filename);
+ break;
+ }
+ /* First try to find block in chain */
+ if (block->hash1 == h1 && block->hash2 == h2) {
+ msg_debug_pool("%s found existing block %ud in chain %ud, value %.2f",
+ file->filename,
+ i,
+ blocknum,
+ value);
+ block->value = value;
+ return;
+ }
+ /* Check whether we have a free block in chain */
+ if (block->hash1 == 0 && block->hash2 == 0) {
+ /* Write new block here */
+ msg_debug_pool("%s found free block %ud in chain %ud, set h1=%ud, h2=%ud",
+ file->filename,
+ i,
+ blocknum,
+ h1,
+ h2);
+ block->hash1 = h1;
+ block->hash2 = h2;
+ block->value = value;
+ header->used_blocks++;
+
+ return;
+ }
+
+ /* Expire block with minimum value otherwise */
+ if (block->value < min) {
+ to_expire = block;
+ min = block->value;
+ }
+ c += sizeof(struct stat_file_block);
+ block = (struct stat_file_block *) c;
+ }
+
+ /* Try expire some block */
+ if (to_expire) {
+ block = to_expire;
+ }
+ else {
+ /* Expire first block in chain */
+ c = (u_char *) file->map + file->seek_pos + blocknum * sizeof(struct stat_file_block);
+ block = (struct stat_file_block *) c;
+ }
+
+ block->hash1 = h1;
+ block->hash2 = h2;
+ block->value = value;
+}
+
+void rspamd_mmaped_file_set_block(rspamd_mempool_t *pool,
+ rspamd_mmaped_file_t *file,
+ guint32 h1,
+ guint32 h2,
+ double value)
+{
+ rspamd_mmaped_file_set_block_common(pool, file, h1, h2, value);
+}
+
+gboolean
+rspamd_mmaped_file_set_revision(rspamd_mmaped_file_t *file, guint64 rev, time_t time)
+{
+ struct stat_file_header *header;
+
+ if (file == NULL || file->map == NULL) {
+ return FALSE;
+ }
+
+ header = (struct stat_file_header *) file->map;
+
+ header->revision = rev;
+ header->rev_time = time;
+
+ return TRUE;
+}
+
+gboolean
+rspamd_mmaped_file_inc_revision(rspamd_mmaped_file_t *file)
+{
+ struct stat_file_header *header;
+
+ if (file == NULL || file->map == NULL) {
+ return FALSE;
+ }
+
+ header = (struct stat_file_header *) file->map;
+
+ header->revision++;
+
+ return TRUE;
+}
+
+gboolean
+rspamd_mmaped_file_dec_revision(rspamd_mmaped_file_t *file)
+{
+ struct stat_file_header *header;
+
+ if (file == NULL || file->map == NULL) {
+ return FALSE;
+ }
+
+ header = (struct stat_file_header *) file->map;
+
+ header->revision--;
+
+ return TRUE;
+}
+
+
+gboolean
+rspamd_mmaped_file_get_revision(rspamd_mmaped_file_t *file, guint64 *rev, time_t *time)
+{
+ struct stat_file_header *header;
+
+ if (file == NULL || file->map == NULL) {
+ return FALSE;
+ }
+
+ header = (struct stat_file_header *) file->map;
+
+ if (rev != NULL) {
+ *rev = header->revision;
+ }
+ if (time != NULL) {
+ *time = header->rev_time;
+ }
+
+ return TRUE;
+}
+
+guint64
+rspamd_mmaped_file_get_used(rspamd_mmaped_file_t *file)
+{
+ struct stat_file_header *header;
+
+ if (file == NULL || file->map == NULL) {
+ return (guint64) -1;
+ }
+
+ header = (struct stat_file_header *) file->map;
+
+ return header->used_blocks;
+}
+
+guint64
+rspamd_mmaped_file_get_total(rspamd_mmaped_file_t *file)
+{
+ struct stat_file_header *header;
+
+ if (file == NULL || file->map == NULL) {
+ return (guint64) -1;
+ }
+
+ header = (struct stat_file_header *) file->map;
+
+ /* If total blocks is 0 we have old version of header, so set total blocks correctly */
+ if (header->total_blocks == 0) {
+ header->total_blocks = file->cur_section.length;
+ }
+
+ return header->total_blocks;
+}
+
+/* Check whether specified file is statistic file and calculate its len in blocks */
+static gint
+rspamd_mmaped_file_check(rspamd_mempool_t *pool, rspamd_mmaped_file_t *file)
+{
+ struct stat_file *f;
+ gchar *c;
+ static gchar valid_version[] = RSPAMD_STATFILE_VERSION;
+
+
+ if (!file || !file->map) {
+ return -1;
+ }
+
+ if (file->len < sizeof(struct stat_file)) {
+ msg_info_pool("file %s is too short to be stat file: %z",
+ file->filename,
+ file->len);
+ return -1;
+ }
+
+ f = (struct stat_file *) file->map;
+ c = &f->header.magic[0];
+ /* Check magic and version */
+ if (*c++ != 'r' || *c++ != 's' || *c++ != 'd') {
+ msg_info_pool("file %s is invalid stat file", file->filename);
+ return -1;
+ }
+
+ c = &f->header.version[0];
+ /* Now check version and convert old version to new one (that can be used for sync */
+ if (*c == 1 && *(c + 1) == 0) {
+ return -1;
+ }
+ else if (memcmp(c, valid_version, sizeof(valid_version)) != 0) {
+ /* Unknown version */
+ msg_info_pool("file %s has invalid version %c.%c",
+ file->filename,
+ '0' + *c,
+ '0' + *(c + 1));
+ return -1;
+ }
+
+ /* Check first section and set new offset */
+ file->cur_section.code = f->section.code;
+ file->cur_section.length = f->section.length;
+ if (file->cur_section.length * sizeof(struct stat_file_block) >
+ file->len) {
+ msg_info_pool("file %s is truncated: %z, must be %z",
+ file->filename,
+ file->len,
+ file->cur_section.length * sizeof(struct stat_file_block));
+ return -1;
+ }
+ file->seek_pos = sizeof(struct stat_file) -
+ sizeof(struct stat_file_block);
+
+ return 0;
+}
+
+
+static rspamd_mmaped_file_t *
+rspamd_mmaped_file_reindex(rspamd_mempool_t *pool,
+ const gchar *filename,
+ size_t old_size,
+ size_t size,
+ struct rspamd_statfile_config *stcf)
+{
+ gchar *backup, *lock;
+ gint fd, lock_fd;
+ rspamd_mmaped_file_t *new, *old = NULL;
+ u_char *map, *pos;
+ struct stat_file_block *block;
+ struct stat_file_header *header, *nh;
+ struct timespec sleep_ts = {
+ .tv_sec = 0,
+ .tv_nsec = 1000000};
+
+ if (size <
+ sizeof(struct stat_file_header) + sizeof(struct stat_file_section) +
+ sizeof(block)) {
+ msg_err_pool("file %s is too small to carry any statistic: %z",
+ filename,
+ size);
+ return NULL;
+ }
+
+ lock = g_strconcat(filename, ".lock", NULL);
+ lock_fd = open(lock, O_WRONLY | O_CREAT | O_EXCL, 00600);
+
+ while (lock_fd == -1) {
+ /* Wait for lock */
+ lock_fd = open(lock, O_WRONLY | O_CREAT | O_EXCL, 00600);
+ if (lock_fd != -1) {
+ unlink(lock);
+ close(lock_fd);
+ g_free(lock);
+
+ return rspamd_mmaped_file_open(pool, filename, size, stcf);
+ }
+ else {
+ nanosleep(&sleep_ts, NULL);
+ }
+ }
+
+ backup = g_strconcat(filename, ".old", NULL);
+ if (rename(filename, backup) == -1) {
+ msg_err_pool("cannot rename %s to %s: %s", filename, backup, strerror(errno));
+ g_free(backup);
+ unlink(lock);
+ g_free(lock);
+ close(lock_fd);
+
+ return NULL;
+ }
+
+ old = rspamd_mmaped_file_open(pool, backup, old_size, stcf);
+
+ if (old == NULL) {
+ msg_warn_pool("old file %s is invalid mmapped file, just move it",
+ backup);
+ }
+
+ /* We need to release our lock here */
+ unlink(lock);
+ close(lock_fd);
+ g_free(lock);
+
+ /* Now create new file with required size */
+ if (rspamd_mmaped_file_create(filename, size, stcf, pool) != 0) {
+ msg_err_pool("cannot create new file");
+ rspamd_mmaped_file_close(old);
+ g_free(backup);
+
+ return NULL;
+ }
+
+ new = rspamd_mmaped_file_open(pool, filename, size, stcf);
+
+ if (old) {
+ /* Now open new file and start copying */
+ fd = open(backup, O_RDONLY);
+ if (fd == -1 || new == NULL) {
+ if (fd != -1) {
+ close(fd);
+ }
+
+ msg_err_pool("cannot open file: %s", strerror(errno));
+ rspamd_mmaped_file_close(old);
+ g_free(backup);
+ return NULL;
+ }
+
+
+ /* Now start reading blocks from old statfile */
+ if ((map =
+ mmap(NULL, old_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
+ msg_err_pool("cannot mmap file: %s", strerror(errno));
+ close(fd);
+ rspamd_mmaped_file_close(old);
+ g_free(backup);
+ return NULL;
+ }
+
+ pos = map + (sizeof(struct stat_file) - sizeof(struct stat_file_block));
+
+ if (pos - map < (gssize) old_size) {
+ while ((gssize) old_size - (pos - map) >= (gssize) sizeof(struct stat_file_block)) {
+ block = (struct stat_file_block *) pos;
+ if (block->hash1 != 0 && block->value != 0) {
+ rspamd_mmaped_file_set_block_common(pool,
+ new, block->hash1,
+ block->hash2, block->value);
+ }
+ pos += sizeof(block);
+ }
+ }
+
+ header = (struct stat_file_header *) map;
+ rspamd_mmaped_file_set_revision(new, header->revision, header->rev_time);
+ nh = new->map;
+ /* Copy tokenizer configuration */
+ memcpy(nh->unused, header->unused, sizeof(header->unused));
+ nh->tokenizer_conf_len = header->tokenizer_conf_len;
+
+ munmap(map, old_size);
+ close(fd);
+ rspamd_mmaped_file_close_file(pool, old);
+ }
+
+ unlink(backup);
+ g_free(backup);
+
+ return new;
+}
+
+/*
+ * Pre-load mmaped file into memory
+ */
+static void
+rspamd_mmaped_file_preload(rspamd_mmaped_file_t *file)
+{
+ guint8 *pos, *end;
+ volatile guint8 t;
+ gsize size;
+
+ pos = (guint8 *) file->map;
+ end = (guint8 *) file->map + file->len;
+
+ if (madvise(pos, end - pos, MADV_SEQUENTIAL) == -1) {
+ msg_info("madvise failed: %s", strerror(errno));
+ }
+ else {
+ /* Load pages of file */
+#ifdef HAVE_GETPAGESIZE
+ size = getpagesize();
+#else
+ size = sysconf(_SC_PAGESIZE);
+#endif
+ while (pos < end) {
+ t = *pos;
+ (void) t;
+ pos += size;
+ }
+ }
+}
+
+rspamd_mmaped_file_t *
+rspamd_mmaped_file_open(rspamd_mempool_t *pool,
+ const gchar *filename, size_t size,
+ struct rspamd_statfile_config *stcf)
+{
+ struct stat st;
+ rspamd_mmaped_file_t *new_file;
+ gchar *lock;
+ gint lock_fd;
+
+ lock = g_strconcat(filename, ".lock", NULL);
+ lock_fd = open(lock, O_WRONLY | O_CREAT | O_EXCL, 00600);
+
+ if (lock_fd == -1) {
+ g_free(lock);
+ msg_info_pool("cannot open file %s, it is locked by another process",
+ filename);
+ return NULL;
+ }
+
+ close(lock_fd);
+ unlink(lock);
+ g_free(lock);
+
+ if (stat(filename, &st) == -1) {
+ msg_info_pool("cannot stat file %s, error %s, %d", filename, strerror(errno), errno);
+ return NULL;
+ }
+
+ if (labs((glong) size - st.st_size) > (long) sizeof(struct stat_file) * 2 && size > sizeof(struct stat_file)) {
+ msg_warn_pool("need to reindex statfile old size: %Hz, new size: %Hz",
+ (size_t) st.st_size, size);
+ return rspamd_mmaped_file_reindex(pool, filename, st.st_size, size, stcf);
+ }
+ else if (size < sizeof(struct stat_file)) {
+ msg_err_pool("requested to shrink statfile to %Hz but it is too small",
+ size);
+ }
+
+ new_file = g_malloc0(sizeof(rspamd_mmaped_file_t));
+ if ((new_file->fd = open(filename, O_RDWR)) == -1) {
+ msg_info_pool("cannot open file %s, error %d, %s",
+ filename,
+ errno,
+ strerror(errno));
+ g_free(new_file);
+ return NULL;
+ }
+
+ if ((new_file->map =
+ mmap(NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED,
+ new_file->fd, 0)) == MAP_FAILED) {
+ close(new_file->fd);
+ msg_info_pool("cannot mmap file %s, error %d, %s",
+ filename,
+ errno,
+ strerror(errno));
+ g_free(new_file);
+ return NULL;
+ }
+
+ rspamd_strlcpy(new_file->filename, filename, sizeof(new_file->filename));
+ new_file->len = st.st_size;
+ /* Try to lock pages in RAM */
+
+ /* Acquire lock for this operation */
+ if (!rspamd_file_lock(new_file->fd, FALSE)) {
+ close(new_file->fd);
+ munmap(new_file->map, st.st_size);
+ msg_info_pool("cannot lock file %s, error %d, %s",
+ filename,
+ errno,
+ strerror(errno));
+ g_free(new_file);
+ return NULL;
+ }
+
+ if (rspamd_mmaped_file_check(pool, new_file) == -1) {
+ close(new_file->fd);
+ rspamd_file_unlock(new_file->fd, FALSE);
+ munmap(new_file->map, st.st_size);
+ g_free(new_file);
+ return NULL;
+ }
+
+ rspamd_file_unlock(new_file->fd, FALSE);
+ new_file->cf = stcf;
+ new_file->pool = pool;
+ rspamd_mmaped_file_preload(new_file);
+
+ g_assert(stcf->clcf != NULL);
+
+ msg_debug_pool("opened statfile %s of size %l", filename, (long) size);
+
+ return new_file;
+}
+
+gint rspamd_mmaped_file_close_file(rspamd_mempool_t *pool,
+ rspamd_mmaped_file_t *file)
+{
+ if (file->map) {
+ msg_info_pool("syncing statfile %s", file->filename);
+ msync(file->map, file->len, MS_ASYNC);
+ munmap(file->map, file->len);
+ }
+ if (file->fd != -1) {
+ close(file->fd);
+ }
+
+ g_free(file);
+
+ return 0;
+}
+
+gint rspamd_mmaped_file_create(const gchar *filename,
+ size_t size,
+ struct rspamd_statfile_config *stcf,
+ rspamd_mempool_t *pool)
+{
+ struct stat_file_header header = {
+ .magic = {'r', 's', 'd'},
+ .version = RSPAMD_STATFILE_VERSION,
+ .padding = {0, 0, 0},
+ .revision = 0,
+ .rev_time = 0,
+ .used_blocks = 0};
+ struct stat_file_section section = {
+ .code = STATFILE_SECTION_COMMON,
+ };
+ struct stat_file_block block = {0, 0, 0};
+ struct rspamd_stat_tokenizer *tokenizer;
+ gint fd, lock_fd;
+ guint buflen = 0, nblocks;
+ gchar *buf = NULL, *lock;
+ struct stat sb;
+ gpointer tok_conf;
+ gsize tok_conf_len;
+ struct timespec sleep_ts = {
+ .tv_sec = 0,
+ .tv_nsec = 1000000};
+
+ if (size <
+ sizeof(struct stat_file_header) + sizeof(struct stat_file_section) +
+ sizeof(block)) {
+ msg_err_pool("file %s is too small to carry any statistic: %z",
+ filename,
+ size);
+ return -1;
+ }
+
+ lock = g_strconcat(filename, ".lock", NULL);
+ lock_fd = open(lock, O_WRONLY | O_CREAT | O_EXCL, 00600);
+
+ while (lock_fd == -1) {
+ /* Wait for lock */
+ lock_fd = open(lock, O_WRONLY | O_CREAT | O_EXCL, 00600);
+ if (lock_fd != -1) {
+ if (stat(filename, &sb) != -1) {
+ /* File has been created by some other process */
+ unlink(lock);
+ close(lock_fd);
+ g_free(lock);
+
+ return 0;
+ }
+
+ /* We still need to create it */
+ goto create;
+ }
+ else {
+ nanosleep(&sleep_ts, NULL);
+ }
+ }
+
+create:
+
+ msg_debug_pool("create statfile %s of size %l", filename, (long) size);
+ nblocks =
+ (size - sizeof(struct stat_file_header) -
+ sizeof(struct stat_file_section)) /
+ sizeof(struct stat_file_block);
+ header.total_blocks = nblocks;
+
+ if ((fd =
+ open(filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) {
+ msg_info_pool("cannot create file %s, error %d, %s",
+ filename,
+ errno,
+ strerror(errno));
+ unlink(lock);
+ close(lock_fd);
+ g_free(lock);
+
+ return -1;
+ }
+
+ rspamd_fallocate(fd,
+ 0,
+ sizeof(header) + sizeof(section) + sizeof(block) * nblocks);
+
+ header.create_time = (guint64) time(NULL);
+ g_assert(stcf->clcf != NULL);
+ g_assert(stcf->clcf->tokenizer != NULL);
+ tokenizer = rspamd_stat_get_tokenizer(stcf->clcf->tokenizer->name);
+ g_assert(tokenizer != NULL);
+ tok_conf = tokenizer->get_config(pool, stcf->clcf->tokenizer, &tok_conf_len);
+ header.tokenizer_conf_len = tok_conf_len;
+ g_assert(tok_conf_len < sizeof(header.unused) - sizeof(guint64));
+ memcpy(header.unused, tok_conf, tok_conf_len);
+
+ if (write(fd, &header, sizeof(header)) == -1) {
+ msg_info_pool("cannot write header to file %s, error %d, %s",
+ filename,
+ errno,
+ strerror(errno));
+ close(fd);
+ unlink(lock);
+ close(lock_fd);
+ g_free(lock);
+
+ return -1;
+ }
+
+ section.length = (guint64) nblocks;
+ if (write(fd, &section, sizeof(section)) == -1) {
+ msg_info_pool("cannot write section header to file %s, error %d, %s",
+ filename,
+ errno,
+ strerror(errno));
+ close(fd);
+ unlink(lock);
+ close(lock_fd);
+ g_free(lock);
+
+ return -1;
+ }
+
+ /* Buffer for write 256 blocks at once */
+ if (nblocks > 256) {
+ buflen = sizeof(block) * 256;
+ buf = g_malloc0(buflen);
+ }
+
+ while (nblocks) {
+ if (nblocks > 256) {
+ /* Just write buffer */
+ if (write(fd, buf, buflen) == -1) {
+ msg_info_pool("cannot write blocks buffer to file %s, error %d, %s",
+ filename,
+ errno,
+ strerror(errno));
+ close(fd);
+ g_free(buf);
+ unlink(lock);
+ close(lock_fd);
+ g_free(lock);
+
+ return -1;
+ }
+ nblocks -= 256;
+ }
+ else {
+ if (write(fd, &block, sizeof(block)) == -1) {
+ msg_info_pool("cannot write block to file %s, error %d, %s",
+ filename,
+ errno,
+ strerror(errno));
+ close(fd);
+ if (buf) {
+ g_free(buf);
+ }
+
+ unlink(lock);
+ close(lock_fd);
+ g_free(lock);
+
+ return -1;
+ }
+ nblocks--;
+ }
+ }
+
+ close(fd);
+
+ if (buf) {
+ g_free(buf);
+ }
+
+ unlink(lock);
+ close(lock_fd);
+ g_free(lock);
+ msg_debug_pool("created statfile %s of size %l", filename, (long) size);
+
+ return 0;
+}
+
+gpointer
+rspamd_mmaped_file_init(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg, struct rspamd_statfile *st)
+{
+ struct rspamd_statfile_config *stf = st->stcf;
+ rspamd_mmaped_file_t *mf;
+ const ucl_object_t *filenameo, *sizeo;
+ const gchar *filename;
+ gsize size;
+
+ filenameo = ucl_object_lookup(stf->opts, "filename");
+
+ if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) {
+ filenameo = ucl_object_lookup(stf->opts, "path");
+
+ if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) {
+ msg_err_config("statfile %s has no filename defined", stf->symbol);
+ return NULL;
+ }
+ }
+
+ filename = ucl_object_tostring(filenameo);
+
+ sizeo = ucl_object_lookup(stf->opts, "size");
+
+ if (sizeo == NULL || ucl_object_type(sizeo) != UCL_INT) {
+ msg_err_config("statfile %s has no size defined", stf->symbol);
+ return NULL;
+ }
+
+ size = ucl_object_toint(sizeo);
+ mf = rspamd_mmaped_file_open(cfg->cfg_pool, filename, size, stf);
+
+ if (mf != NULL) {
+ mf->pool = cfg->cfg_pool;
+ }
+ else {
+ /* Create file here */
+
+ filenameo = ucl_object_find_key(stf->opts, "filename");
+ if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) {
+ filenameo = ucl_object_find_key(stf->opts, "path");
+ if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) {
+ msg_err_config("statfile %s has no filename defined", stf->symbol);
+ return NULL;
+ }
+ }
+
+ filename = ucl_object_tostring(filenameo);
+
+ sizeo = ucl_object_find_key(stf->opts, "size");
+ if (sizeo == NULL || ucl_object_type(sizeo) != UCL_INT) {
+ msg_err_config("statfile %s has no size defined", stf->symbol);
+ return NULL;
+ }
+
+ size = ucl_object_toint(sizeo);
+
+ if (rspamd_mmaped_file_create(filename, size, stf, cfg->cfg_pool) != 0) {
+ msg_err_config("cannot create new file");
+ }
+
+ mf = rspamd_mmaped_file_open(cfg->cfg_pool, filename, size, stf);
+ }
+
+ return (gpointer) mf;
+}
+
+void rspamd_mmaped_file_close(gpointer p)
+{
+ rspamd_mmaped_file_t *mf = p;
+
+
+ if (mf) {
+ rspamd_mmaped_file_close_file(mf->pool, mf);
+ }
+}
+
+gpointer
+rspamd_mmaped_file_runtime(struct rspamd_task *task,
+ struct rspamd_statfile_config *stcf,
+ gboolean learn,
+ gpointer p,
+ gint _id)
+{
+ rspamd_mmaped_file_t *mf = p;
+
+ return (gpointer) mf;
+}
+
+gboolean
+rspamd_mmaped_file_process_tokens(struct rspamd_task *task, GPtrArray *tokens,
+ gint id,
+ gpointer p)
+{
+ rspamd_mmaped_file_t *mf = p;
+ guint32 h1, h2;
+ rspamd_token_t *tok;
+ guint i;
+
+ g_assert(tokens != NULL);
+ g_assert(p != NULL);
+
+ for (i = 0; i < tokens->len; i++) {
+ tok = g_ptr_array_index(tokens, i);
+ memcpy(&h1, (guchar *) &tok->data, sizeof(h1));
+ memcpy(&h2, ((guchar *) &tok->data) + sizeof(h1), sizeof(h2));
+ tok->values[id] = rspamd_mmaped_file_get_block(mf, h1, h2);
+ }
+
+ if (mf->cf->is_spam) {
+ task->flags |= RSPAMD_TASK_FLAG_HAS_SPAM_TOKENS;
+ }
+ else {
+ task->flags |= RSPAMD_TASK_FLAG_HAS_HAM_TOKENS;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_mmaped_file_learn_tokens(struct rspamd_task *task, GPtrArray *tokens,
+ gint id,
+ gpointer p)
+{
+ rspamd_mmaped_file_t *mf = p;
+ guint32 h1, h2;
+ rspamd_token_t *tok;
+ guint i;
+
+ g_assert(tokens != NULL);
+ g_assert(p != NULL);
+
+ for (i = 0; i < tokens->len; i++) {
+ tok = g_ptr_array_index(tokens, i);
+ memcpy(&h1, (guchar *) &tok->data, sizeof(h1));
+ memcpy(&h2, ((guchar *) &tok->data) + sizeof(h1), sizeof(h2));
+ rspamd_mmaped_file_set_block(task->task_pool, mf, h1, h2,
+ tok->values[id]);
+ }
+
+ return TRUE;
+}
+
+gulong
+rspamd_mmaped_file_total_learns(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *) runtime;
+ guint64 rev = 0;
+ time_t t;
+
+ if (mf != NULL) {
+ rspamd_mmaped_file_get_revision(mf, &rev, &t);
+ }
+
+ return rev;
+}
+
+gulong
+rspamd_mmaped_file_inc_learns(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *) runtime;
+ guint64 rev = 0;
+ time_t t;
+
+ if (mf != NULL) {
+ rspamd_mmaped_file_inc_revision(mf);
+ rspamd_mmaped_file_get_revision(mf, &rev, &t);
+ }
+
+ return rev;
+}
+
+gulong
+rspamd_mmaped_file_dec_learns(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *) runtime;
+ guint64 rev = 0;
+ time_t t;
+
+ if (mf != NULL) {
+ rspamd_mmaped_file_dec_revision(mf);
+ rspamd_mmaped_file_get_revision(mf, &rev, &t);
+ }
+
+ return rev;
+}
+
+
+ucl_object_t *
+rspamd_mmaped_file_get_stat(gpointer runtime,
+ gpointer ctx)
+{
+ ucl_object_t *res = NULL;
+ guint64 rev;
+ rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *) runtime;
+
+ if (mf != NULL) {
+ res = ucl_object_typed_new(UCL_OBJECT);
+ rspamd_mmaped_file_get_revision(mf, &rev, NULL);
+ ucl_object_insert_key(res, ucl_object_fromint(rev), "revision",
+ 0, false);
+ ucl_object_insert_key(res, ucl_object_fromint(mf->len), "size",
+ 0, false);
+ ucl_object_insert_key(res, ucl_object_fromint(rspamd_mmaped_file_get_total(mf)), "total", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromint(rspamd_mmaped_file_get_used(mf)), "used", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromstring(mf->cf->symbol),
+ "symbol", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromstring("mmap"),
+ "type", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromint(0),
+ "languages", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromint(0),
+ "users", 0, false);
+
+ if (mf->cf->label) {
+ ucl_object_insert_key(res, ucl_object_fromstring(mf->cf->label),
+ "label", 0, false);
+ }
+ }
+
+ return res;
+}
+
+gboolean
+rspamd_mmaped_file_finalize_learn(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx, GError **err)
+{
+ rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *) runtime;
+
+ if (mf != NULL) {
+ msync(mf->map, mf->len, MS_INVALIDATE | MS_ASYNC);
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_mmaped_file_finalize_process(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ return TRUE;
+}
+
+gpointer
+rspamd_mmaped_file_load_tokenizer_config(gpointer runtime,
+ gsize *len)
+{
+ rspamd_mmaped_file_t *mf = runtime;
+ struct stat_file_header *header;
+
+ g_assert(mf != NULL);
+ header = mf->map;
+
+ if (len) {
+ *len = header->tokenizer_conf_len;
+ }
+
+ return header->unused;
+}
diff --git a/src/libstat/backends/redis_backend.cxx b/src/libstat/backends/redis_backend.cxx
new file mode 100644
index 0000000..cd0c379
--- /dev/null
+++ b/src/libstat/backends/redis_backend.cxx
@@ -0,0 +1,1132 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "lua/lua_common.h"
+#include "rspamd.h"
+#include "stat_internal.h"
+#include "upstream.h"
+#include "libserver/mempool_vars_internal.h"
+#include "fmt/core.h"
+
+#include "libutil/cxx/error.hxx"
+
+#include <string>
+#include <cstdint>
+#include <vector>
+#include <optional>
+
+#define msg_debug_stat_redis(...) rspamd_conditional_debug_fast(nullptr, nullptr, \
+ rspamd_stat_redis_log_id, "stat_redis", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(stat_redis)
+
+#define REDIS_CTX(p) (reinterpret_cast<struct redis_stat_ctx *>(p))
+#define REDIS_RUNTIME(p) (reinterpret_cast<struct redis_stat_runtime<float> *>(p))
+#define REDIS_DEFAULT_OBJECT "%s%l"
+#define REDIS_DEFAULT_USERS_OBJECT "%s%l%r"
+#define REDIS_DEFAULT_TIMEOUT 0.5
+#define REDIS_STAT_TIMEOUT 30
+#define REDIS_MAX_USERS 1000
+
+struct redis_stat_ctx {
+ lua_State *L;
+ struct rspamd_statfile_config *stcf;
+ const char *redis_object = REDIS_DEFAULT_OBJECT;
+ bool enable_users = false;
+ bool store_tokens = false;
+ bool enable_signatures = false;
+ int cbref_user = -1;
+
+ int cbref_classify = -1;
+ int cbref_learn = -1;
+
+ ucl_object_t *cur_stat = nullptr;
+
+ explicit redis_stat_ctx(lua_State *_L)
+ : L(_L)
+ {
+ }
+
+ ~redis_stat_ctx()
+ {
+ if (cbref_user != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbref_user);
+ }
+
+ if (cbref_classify != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbref_classify);
+ }
+
+ if (cbref_learn != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbref_learn);
+ }
+ }
+};
+
+
+template<class T, std::enable_if_t<std::is_convertible_v<T, float>, bool> = true>
+struct redis_stat_runtime {
+ struct redis_stat_ctx *ctx;
+ struct rspamd_task *task;
+ struct rspamd_statfile_config *stcf;
+ GPtrArray *tokens = nullptr;
+ const char *redis_object_expanded;
+ std::uint64_t learned = 0;
+ int id;
+ std::vector<std::pair<int, T>> *results = nullptr;
+ bool need_redis_call = true;
+ std::optional<rspamd::util::error> err;
+
+ using result_type = std::vector<std::pair<int, T>>;
+
+private:
+ /* Called on connection termination */
+ static void rt_dtor(gpointer data)
+ {
+ auto *rt = REDIS_RUNTIME(data);
+
+ delete rt;
+ }
+
+ /* Avoid occasional deletion */
+ ~redis_stat_runtime()
+ {
+ if (tokens) {
+ g_ptr_array_unref(tokens);
+ }
+
+ delete results;
+ }
+
+public:
+ explicit redis_stat_runtime(struct redis_stat_ctx *_ctx, struct rspamd_task *_task, const char *_redis_object_expanded)
+ : ctx(_ctx), task(_task), stcf(_ctx->stcf), redis_object_expanded(_redis_object_expanded)
+ {
+ rspamd_mempool_add_destructor(task->task_pool, redis_stat_runtime<T>::rt_dtor, this);
+ }
+
+ static auto maybe_recover_from_mempool(struct rspamd_task *task, const char *redis_object_expanded,
+ bool is_spam) -> std::optional<redis_stat_runtime<T> *>
+ {
+ auto var_name = fmt::format("{}_{}", redis_object_expanded, is_spam ? "S" : "H");
+ auto *res = rspamd_mempool_get_variable(task->task_pool, var_name.c_str());
+
+ if (res) {
+ msg_debug_bayes("recovered runtime from mempool at %s", var_name.c_str());
+ return reinterpret_cast<redis_stat_runtime<T> *>(res);
+ }
+ else {
+ msg_debug_bayes("no runtime at %s", var_name.c_str());
+ return std::nullopt;
+ }
+ }
+
+ void set_results(std::vector<std::pair<int, T>> *results)
+ {
+ this->results = results;
+ }
+
+ /* Propagate results from internal representation to the tokens array */
+ auto process_tokens(GPtrArray *tokens) const -> bool
+ {
+ rspamd_token_t *tok;
+
+ if (!results) {
+ return false;
+ }
+
+ for (auto [idx, val]: *results) {
+ tok = (rspamd_token_t *) g_ptr_array_index(tokens, idx - 1);
+ tok->values[id] = val;
+ }
+
+ return true;
+ }
+
+ auto save_in_mempool(bool is_spam) const
+ {
+ auto var_name = fmt::format("{}_{}", redis_object_expanded, is_spam ? "S" : "H");
+ /* We do not set destructor for the variable, as it should be already added on creation */
+ rspamd_mempool_set_variable(task->task_pool, var_name.c_str(), (gpointer) this, nullptr);
+ msg_debug_bayes("saved runtime in mempool at %s", var_name.c_str());
+ }
+};
+
+#define GET_TASK_ELT(task, elt) (task == nullptr ? nullptr : (task)->elt)
+
+static const gchar *M = "redis statistics";
+
+static GQuark
+rspamd_redis_stat_quark(void)
+{
+ return g_quark_from_static_string(M);
+}
+
+/*
+ * Non-static for lua unit testing
+ */
+gsize rspamd_redis_expand_object(const gchar *pattern,
+ struct redis_stat_ctx *ctx,
+ struct rspamd_task *task,
+ gchar **target)
+{
+ gsize tlen = 0;
+ const gchar *p = pattern, *elt;
+ gchar *d, *end;
+ enum {
+ just_char,
+ percent_char,
+ mod_char
+ } state = just_char;
+ struct rspamd_statfile_config *stcf;
+ lua_State *L = nullptr;
+ struct rspamd_task **ptask;
+ const gchar *rcpt = nullptr;
+ gint err_idx;
+
+ g_assert(ctx != nullptr);
+ g_assert(task != nullptr);
+ stcf = ctx->stcf;
+
+ L = RSPAMD_LUA_CFG_STATE(task->cfg);
+ g_assert(L != nullptr);
+
+ if (ctx->enable_users) {
+ if (ctx->cbref_user == -1) {
+ rcpt = rspamd_task_get_principal_recipient(task);
+ }
+ else {
+ /* Execute lua function to get userdata */
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, ctx->cbref_user);
+ ptask = (struct rspamd_task **) lua_newuserdata(L, sizeof(struct rspamd_task *));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ if (lua_pcall(L, 1, 1, err_idx) != 0) {
+ msg_err_task("call to user extraction script failed: %s",
+ lua_tostring(L, -1));
+ }
+ else {
+ rcpt = rspamd_mempool_strdup(task->task_pool, lua_tostring(L, -1));
+ }
+
+ /* Result + error function */
+ lua_settop(L, err_idx - 1);
+ }
+
+ if (rcpt) {
+ rspamd_mempool_set_variable(task->task_pool, "stat_user",
+ (gpointer) rcpt, nullptr);
+ }
+ }
+
+ /* Length calculation */
+ while (*p) {
+ switch (state) {
+ case just_char:
+ if (*p == '%') {
+ state = percent_char;
+ }
+ else {
+ tlen++;
+ }
+ p++;
+ break;
+ case percent_char:
+ switch (*p) {
+ case '%':
+ tlen++;
+ state = just_char;
+ break;
+ case 'u':
+ elt = GET_TASK_ELT(task, auth_user);
+ if (elt) {
+ tlen += strlen(elt);
+ }
+ break;
+ case 'r':
+
+ if (rcpt == nullptr) {
+ elt = rspamd_task_get_principal_recipient(task);
+ }
+ else {
+ elt = rcpt;
+ }
+
+ if (elt) {
+ tlen += strlen(elt);
+ }
+ break;
+ case 'l':
+ if (stcf->label) {
+ tlen += strlen(stcf->label);
+ }
+ /* Label miss is OK */
+ break;
+ case 's':
+ tlen += sizeof("RS") - 1;
+ break;
+ default:
+ state = just_char;
+ tlen++;
+ break;
+ }
+
+ if (state == percent_char) {
+ state = mod_char;
+ }
+ p++;
+ break;
+
+ case mod_char:
+ switch (*p) {
+ case 'd':
+ p++;
+ state = just_char;
+ break;
+ default:
+ state = just_char;
+ break;
+ }
+ break;
+ }
+ }
+
+
+ if (target == nullptr) {
+ return -1;
+ }
+
+ *target = (gchar *) rspamd_mempool_alloc(task->task_pool, tlen + 1);
+ d = *target;
+ end = d + tlen + 1;
+ d[tlen] = '\0';
+ p = pattern;
+ state = just_char;
+
+ /* Expand string */
+ while (*p && d < end) {
+ switch (state) {
+ case just_char:
+ if (*p == '%') {
+ state = percent_char;
+ }
+ else {
+ *d++ = *p;
+ }
+ p++;
+ break;
+ case percent_char:
+ switch (*p) {
+ case '%':
+ *d++ = *p;
+ state = just_char;
+ break;
+ case 'u':
+ elt = GET_TASK_ELT(task, auth_user);
+ if (elt) {
+ d += rspamd_strlcpy(d, elt, end - d);
+ }
+ break;
+ case 'r':
+ if (rcpt == nullptr) {
+ elt = rspamd_task_get_principal_recipient(task);
+ }
+ else {
+ elt = rcpt;
+ }
+
+ if (elt) {
+ d += rspamd_strlcpy(d, elt, end - d);
+ }
+ break;
+ case 'l':
+ if (stcf->label) {
+ d += rspamd_strlcpy(d, stcf->label, end - d);
+ }
+ break;
+ case 's':
+ d += rspamd_strlcpy(d, "RS", end - d);
+ break;
+ default:
+ state = just_char;
+ *d++ = *p;
+ break;
+ }
+
+ if (state == percent_char) {
+ state = mod_char;
+ }
+ p++;
+ break;
+
+ case mod_char:
+ switch (*p) {
+ case 'd':
+ /* TODO: not supported yet */
+ p++;
+ state = just_char;
+ break;
+ default:
+ state = just_char;
+ break;
+ }
+ break;
+ }
+ }
+
+ return tlen;
+}
+
+static int
+rspamd_redis_stat_cb(lua_State *L)
+{
+ const auto *cookie = lua_tostring(L, lua_upvalueindex(1));
+ auto *cfg = lua_check_config(L, 1);
+ auto *backend = REDIS_CTX(rspamd_mempool_get_variable(cfg->cfg_pool, cookie));
+
+ if (backend == nullptr) {
+ msg_err("internal error: cookie %s is not found", cookie);
+
+ return 0;
+ }
+
+ auto *cur_obj = ucl_object_lua_import(L, 2);
+ msg_debug_bayes_cfg("got stat object for %s", backend->stcf->symbol);
+ /* Enrich with some default values that are meaningless for redis */
+ ucl_object_insert_key(cur_obj,
+ ucl_object_typed_new(UCL_INT), "used", 0, false);
+ ucl_object_insert_key(cur_obj,
+ ucl_object_typed_new(UCL_INT), "total", 0, false);
+ ucl_object_insert_key(cur_obj,
+ ucl_object_typed_new(UCL_INT), "size", 0, false);
+ ucl_object_insert_key(cur_obj,
+ ucl_object_fromstring(backend->stcf->symbol),
+ "symbol", 0, false);
+ ucl_object_insert_key(cur_obj, ucl_object_fromstring("redis"),
+ "type", 0, false);
+ ucl_object_insert_key(cur_obj, ucl_object_fromint(0),
+ "languages", 0, false);
+
+ if (backend->cur_stat) {
+ ucl_object_unref(backend->cur_stat);
+ }
+
+ backend->cur_stat = cur_obj;
+
+ return 0;
+}
+
+static void
+rspamd_redis_parse_classifier_opts(struct redis_stat_ctx *backend,
+ const ucl_object_t *statfile_obj,
+ const ucl_object_t *classifier_obj,
+ struct rspamd_config *cfg)
+{
+ const gchar *lua_script;
+ const ucl_object_t *elt, *users_enabled;
+ auto *L = RSPAMD_LUA_CFG_STATE(cfg);
+
+ users_enabled = ucl_object_lookup_any(classifier_obj, "per_user",
+ "users_enabled", nullptr);
+
+ if (users_enabled != nullptr) {
+ if (ucl_object_type(users_enabled) == UCL_BOOLEAN) {
+ backend->enable_users = ucl_object_toboolean(users_enabled);
+ backend->cbref_user = -1;
+ }
+ else if (ucl_object_type(users_enabled) == UCL_STRING) {
+ lua_script = ucl_object_tostring(users_enabled);
+
+ if (luaL_dostring(L, lua_script) != 0) {
+ msg_err_config("cannot execute lua script for users "
+ "extraction: %s",
+ lua_tostring(L, -1));
+ }
+ else {
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ backend->enable_users = TRUE;
+ backend->cbref_user = luaL_ref(L,
+ LUA_REGISTRYINDEX);
+ }
+ else {
+ msg_err_config("lua script must return "
+ "function(task) and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ }
+ }
+ }
+ else {
+ backend->enable_users = FALSE;
+ backend->cbref_user = -1;
+ }
+
+ elt = ucl_object_lookup(classifier_obj, "prefix");
+ if (elt == nullptr || ucl_object_type(elt) != UCL_STRING) {
+ /* Default non-users statistics */
+ if (backend->enable_users || backend->cbref_user != -1) {
+ backend->redis_object = REDIS_DEFAULT_USERS_OBJECT;
+ }
+ else {
+ backend->redis_object = REDIS_DEFAULT_OBJECT;
+ }
+ }
+ else {
+ /* XXX: sanity check */
+ backend->redis_object = ucl_object_tostring(elt);
+ }
+
+ elt = ucl_object_lookup(classifier_obj, "store_tokens");
+ if (elt) {
+ backend->store_tokens = ucl_object_toboolean(elt);
+ }
+ else {
+ backend->store_tokens = FALSE;
+ }
+
+ elt = ucl_object_lookup(classifier_obj, "signatures");
+ if (elt) {
+ backend->enable_signatures = ucl_object_toboolean(elt);
+ }
+ else {
+ backend->enable_signatures = FALSE;
+ }
+}
+
+gpointer
+rspamd_redis_init(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg, struct rspamd_statfile *st)
+{
+ auto *L = RSPAMD_LUA_CFG_STATE(cfg);
+
+ auto backend = std::make_unique<struct redis_stat_ctx>(L);
+ lua_settop(L, 0);
+
+ rspamd_redis_parse_classifier_opts(backend.get(), st->stcf->opts, st->classifier->cfg->opts, cfg);
+
+ st->stcf->clcf->flags |= RSPAMD_FLAG_CLASSIFIER_INCREMENTING_BACKEND;
+ backend->stcf = st->stcf;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ auto err_idx = lua_gettop(L);
+
+ /* Obtain function */
+ if (!rspamd_lua_require_function(L, "lua_bayes_redis", "lua_bayes_init_statfile")) {
+ msg_err_config("cannot require lua_bayes_redis.lua_bayes_init_statfile");
+ lua_settop(L, err_idx - 1);
+
+ return nullptr;
+ }
+
+ /* Push arguments */
+ ucl_object_push_lua(L, st->classifier->cfg->opts, false);
+ ucl_object_push_lua(L, st->stcf->opts, false);
+ lua_pushstring(L, backend->stcf->symbol);
+ lua_pushboolean(L, backend->stcf->is_spam);
+ auto **pev_base = (struct ev_loop **) lua_newuserdata(L, sizeof(struct ev_loop *));
+ *pev_base = ctx->event_loop;
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+
+ /* Store backend in random cookie */
+ char *cookie = (char *) rspamd_mempool_alloc(cfg->cfg_pool, 16);
+ rspamd_random_hex(cookie, 16);
+ cookie[15] = '\0';
+ rspamd_mempool_set_variable(cfg->cfg_pool, cookie, backend.get(), nullptr);
+ /* Callback + 1 upvalue */
+ lua_pushstring(L, cookie);
+ lua_pushcclosure(L, &rspamd_redis_stat_cb, 1);
+
+ if (lua_pcall(L, 6, 2, err_idx) != 0) {
+ msg_err("call to lua_bayes_init_classifier "
+ "script failed: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+
+ return nullptr;
+ }
+
+ /* Results are in the stack:
+ * top - 1 - classifier function (idx = -2)
+ * top - learn function (idx = -1)
+ */
+
+ lua_pushvalue(L, -2);
+ backend->cbref_classify = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ lua_pushvalue(L, -1);
+ backend->cbref_learn = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ lua_settop(L, err_idx - 1);
+
+ return backend.release();
+}
+
+gpointer
+rspamd_redis_runtime(struct rspamd_task *task,
+ struct rspamd_statfile_config *stcf,
+ gboolean learn, gpointer c, gint _id)
+{
+ struct redis_stat_ctx *ctx = REDIS_CTX(c);
+ char *object_expanded = nullptr;
+
+ g_assert(ctx != nullptr);
+ g_assert(stcf != nullptr);
+
+ if (rspamd_redis_expand_object(ctx->redis_object, ctx, task,
+ &object_expanded) == 0) {
+ msg_err_task("expansion for %s failed for symbol %s "
+ "(maybe learning per user classifier with no user or recipient)",
+ learn ? "learning" : "classifying",
+ stcf->symbol);
+ return nullptr;
+ }
+
+ /* Look for the cached results */
+ if (!learn) {
+ auto maybe_existing = redis_stat_runtime<float>::maybe_recover_from_mempool(task,
+ object_expanded, stcf->is_spam);
+
+ if (maybe_existing) {
+ auto *rt = maybe_existing.value();
+ /* Update stcf and ctx to correspond to what we have been asked */
+ rt->stcf = stcf;
+ rt->ctx = ctx;
+ return rt;
+ }
+ }
+
+ /* No cached result (or learn), create new one */
+ auto *rt = new redis_stat_runtime<float>(ctx, task, object_expanded);
+
+ if (!learn) {
+ /*
+ * For check, we also need to create the opposite class runtime to avoid
+ * double call for Redis scripts.
+ * This runtime will be filled later.
+ */
+ auto maybe_opposite_rt = redis_stat_runtime<float>::maybe_recover_from_mempool(task,
+ object_expanded,
+ !stcf->is_spam);
+
+ if (!maybe_opposite_rt) {
+ auto *opposite_rt = new redis_stat_runtime<float>(ctx, task, object_expanded);
+ opposite_rt->save_in_mempool(!stcf->is_spam);
+ opposite_rt->need_redis_call = false;
+ }
+ }
+
+ rt->save_in_mempool(stcf->is_spam);
+
+ return rt;
+}
+
+void rspamd_redis_close(gpointer p)
+{
+ struct redis_stat_ctx *ctx = REDIS_CTX(p);
+ delete ctx;
+}
+
+static constexpr auto
+msgpack_emit_str(const std::string_view st, char *out) -> std::size_t
+{
+ auto len = st.size();
+ constexpr const unsigned char fix_mask = 0xA0, l8_ch = 0xd9, l16_ch = 0xda, l32_ch = 0xdb;
+ auto blen = 0;
+ if (len <= 0x1F) {
+ blen = 1;
+ out[0] = (len | fix_mask) & 0xff;
+ }
+ else if (len <= 0xff) {
+ blen = 2;
+ out[0] = l8_ch;
+ out[1] = len & 0xff;
+ }
+ else if (len <= 0xffff) {
+ uint16_t bl = GUINT16_TO_BE(len);
+
+ blen = 3;
+ out[0] = l16_ch;
+ memcpy(&out[1], &bl, sizeof(bl));
+ }
+ else {
+ uint32_t bl = GUINT32_TO_BE(len);
+
+ blen = 5;
+ out[0] = l32_ch;
+ memcpy(&out[1], &bl, sizeof(bl));
+ }
+
+ memcpy(&out[blen], st.data(), st.size());
+
+ return blen + len;
+}
+
+static constexpr auto
+msgpack_str_len(std::size_t len) -> std::size_t
+{
+ if (len <= 0x1F) {
+ return 1 + len;
+ }
+ else if (len <= 0xff) {
+ return 2 + len;
+ }
+ else if (len <= 0xffff) {
+ return 3 + len;
+ }
+ else {
+ return 4 + len;
+ }
+}
+
+/*
+ * Serialise stat tokens to message pack
+ */
+static char *
+rspamd_redis_serialize_tokens(struct rspamd_task *task, const gchar *prefix, GPtrArray *tokens, gsize *ser_len)
+{
+ /* Each token is int64_t that requires 10 bytes (2 int32_t) + 4 bytes array len + 1 byte array magic */
+ char max_int64_str[] = "18446744073709551615";
+ auto prefix_len = strlen(prefix);
+ std::size_t req_len = 5;
+ rspamd_token_t *tok;
+
+ /* Calculate required length */
+ req_len += tokens->len * (msgpack_str_len(sizeof(max_int64_str) + prefix_len) + 1);
+
+ auto *buf = (gchar *) rspamd_mempool_alloc(task->task_pool, req_len);
+ auto *p = buf;
+
+ /* Array */
+ *p++ = (gchar) 0xdd;
+ /* Length in big-endian (4 bytes) */
+ *p++ = (gchar) ((tokens->len >> 24) & 0xff);
+ *p++ = (gchar) ((tokens->len >> 16) & 0xff);
+ *p++ = (gchar) ((tokens->len >> 8) & 0xff);
+ *p++ = (gchar) (tokens->len & 0xff);
+
+
+ int i;
+ auto numbuf_len = sizeof(max_int64_str) + prefix_len + 1;
+ auto *numbuf = (char *) g_alloca(numbuf_len);
+
+ PTR_ARRAY_FOREACH(tokens, i, tok)
+ {
+ std::size_t r = rspamd_snprintf(numbuf, numbuf_len, "%s_%uL", prefix, tok->data);
+ auto shift = msgpack_emit_str({numbuf, r}, p);
+ p += shift;
+ }
+
+ *ser_len = p - buf;
+
+ return buf;
+}
+
+static char *
+rspamd_redis_serialize_text_tokens(struct rspamd_task *task, GPtrArray *tokens, gsize *ser_len)
+{
+ rspamd_token_t *tok;
+ auto req_len = 5; /* Messagepack array prefix */
+ int i;
+
+ /*
+ * First we need to determine the requested length
+ */
+ PTR_ARRAY_FOREACH(tokens, i, tok)
+ {
+ if (tok->t1 && tok->t2) {
+ /* Two tokens */
+ req_len += msgpack_str_len(tok->t1->stemmed.len) + msgpack_str_len(tok->t2->stemmed.len);
+ }
+ else if (tok->t1) {
+ req_len += msgpack_str_len(tok->t1->stemmed.len);
+ req_len += 1; /* null */
+ }
+ else {
+ req_len += 2; /* 2 nulls */
+ }
+ }
+
+ auto *buf = (gchar *) rspamd_mempool_alloc(task->task_pool, req_len);
+ auto *p = buf;
+
+ /* Array */
+ std::uint32_t nlen = tokens->len * 2;
+ nlen = GUINT32_TO_BE(nlen);
+ *p++ = (gchar) 0xdd;
+ /* Length in big-endian (4 bytes) */
+ memcpy(p, &nlen, sizeof(nlen));
+ p += sizeof(nlen);
+
+ PTR_ARRAY_FOREACH(tokens, i, tok)
+ {
+ if (tok->t1 && tok->t2) {
+ auto step = msgpack_emit_str({tok->t1->stemmed.begin, tok->t1->stemmed.len}, p);
+ p += step;
+ step = msgpack_emit_str({tok->t2->stemmed.begin, tok->t2->stemmed.len}, p);
+ p += step;
+ }
+ else if (tok->t1) {
+ auto step = msgpack_emit_str({tok->t1->stemmed.begin, tok->t1->stemmed.len}, p);
+ p += step;
+ *p++ = 0xc0;
+ }
+ else {
+ *p++ = 0xc0;
+ *p++ = 0xc0;
+ }
+ }
+
+ *ser_len = p - buf;
+
+ return buf;
+}
+
+static gint
+rspamd_redis_classified(lua_State *L)
+{
+ const auto *cookie = lua_tostring(L, lua_upvalueindex(1));
+ auto *task = lua_check_task(L, 1);
+ auto *rt = REDIS_RUNTIME(rspamd_mempool_get_variable(task->task_pool, cookie));
+
+ if (rt == nullptr) {
+ msg_err_task("internal error: cannot find runtime for cookie %s", cookie);
+
+ return 0;
+ }
+
+ bool result = lua_toboolean(L, 2);
+
+ if (result) {
+ /* Indexes:
+ * 3 - learned_ham (int)
+ * 4 - learned_spam (int)
+ * 5 - ham_tokens (pair<int, int>)
+ * 6 - spam_tokens (pair<int, int>)
+ */
+
+ /*
+ * We need to fill our runtime AND the opposite runtime
+ */
+ auto filler_func = [](redis_stat_runtime<float> *rt, lua_State *L, unsigned learned, int tokens_pos) {
+ rt->learned = learned;
+ redis_stat_runtime<float>::result_type *res;
+
+ res = new redis_stat_runtime<float>::result_type();
+
+ for (lua_pushnil(L); lua_next(L, tokens_pos); lua_pop(L, 1)) {
+ lua_rawgeti(L, -1, 1);
+ auto idx = lua_tointeger(L, -1);
+ lua_pop(L, 1);
+
+ lua_rawgeti(L, -1, 2);
+ auto value = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+
+ res->emplace_back(idx, value);
+ }
+
+ rt->set_results(res);
+ };
+
+ auto opposite_rt_maybe = redis_stat_runtime<float>::maybe_recover_from_mempool(task,
+ rt->redis_object_expanded,
+ !rt->stcf->is_spam);
+
+ if (!opposite_rt_maybe) {
+ msg_err_task("internal error: cannot find opposite runtime for cookie %s", cookie);
+
+ return 0;
+ }
+
+ if (rt->stcf->is_spam) {
+ filler_func(rt, L, lua_tointeger(L, 4), 6);
+ filler_func(opposite_rt_maybe.value(), L, lua_tointeger(L, 3), 5);
+ }
+ else {
+ filler_func(rt, L, lua_tointeger(L, 3), 5);
+ filler_func(opposite_rt_maybe.value(), L, lua_tointeger(L, 4), 6);
+ }
+
+ /* Mark task as being processed */
+ task->flags |= RSPAMD_TASK_FLAG_HAS_SPAM_TOKENS | RSPAMD_TASK_FLAG_HAS_HAM_TOKENS;
+
+ /* Process all tokens */
+ g_assert(rt->tokens != nullptr);
+ rt->process_tokens(rt->tokens);
+ opposite_rt_maybe.value()->process_tokens(rt->tokens);
+ }
+ else {
+ /* Error message is on index 3 */
+ const auto *err_msg = lua_tostring(L, 3);
+ rt->err = rspamd::util::error(err_msg, 500);
+ msg_err_task("cannot classify task: %s",
+ err_msg);
+ }
+
+ return 0;
+}
+
+gboolean
+rspamd_redis_process_tokens(struct rspamd_task *task,
+ GPtrArray *tokens,
+ gint id, gpointer p)
+{
+ auto *rt = REDIS_RUNTIME(p);
+ auto *L = rt->ctx->L;
+
+ if (rspamd_session_blocked(task->s)) {
+ return FALSE;
+ }
+
+ if (tokens == nullptr || tokens->len == 0) {
+ return FALSE;
+ }
+
+ if (!rt->need_redis_call) {
+ /* No need to do anything, as it is already done in the opposite class processing */
+ /* However, we need to store id as it is needed for further tokens processing */
+ rt->id = id;
+ rt->tokens = g_ptr_array_ref(tokens);
+
+ return TRUE;
+ }
+
+ gsize tokens_len;
+ gchar *tokens_buf = rspamd_redis_serialize_tokens(task, rt->redis_object_expanded, tokens, &tokens_len);
+ rt->id = id;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(L);
+
+ /* Function arguments */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, rt->ctx->cbref_classify);
+ rspamd_lua_task_push(L, task);
+ lua_pushstring(L, rt->redis_object_expanded);
+ lua_pushinteger(L, id);
+ lua_pushboolean(L, rt->stcf->is_spam);
+ lua_new_text(L, tokens_buf, tokens_len, false);
+
+ /* Store rt in random cookie */
+ char *cookie = (char *) rspamd_mempool_alloc(task->task_pool, 16);
+ rspamd_random_hex(cookie, 16);
+ cookie[15] = '\0';
+ rspamd_mempool_set_variable(task->task_pool, cookie, rt, nullptr);
+ /* Callback */
+ lua_pushstring(L, cookie);
+ lua_pushcclosure(L, &rspamd_redis_classified, 1);
+
+ if (lua_pcall(L, 6, 0, err_idx) != 0) {
+ msg_err_task("call to redis failed: %s", lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+ return FALSE;
+ }
+
+ rt->tokens = g_ptr_array_ref(tokens);
+
+ lua_settop(L, err_idx - 1);
+ return TRUE;
+}
+
+gboolean
+rspamd_redis_finalize_process(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ auto *rt = REDIS_RUNTIME(runtime);
+
+ return !rt->err.has_value();
+}
+
+
+static gint
+rspamd_redis_learned(lua_State *L)
+{
+ const auto *cookie = lua_tostring(L, lua_upvalueindex(1));
+ auto *task = lua_check_task(L, 1);
+ auto *rt = REDIS_RUNTIME(rspamd_mempool_get_variable(task->task_pool, cookie));
+
+ if (rt == nullptr) {
+ msg_err_task("internal error: cannot find runtime for cookie %s", cookie);
+
+ return 0;
+ }
+
+ bool result = lua_toboolean(L, 2);
+
+ if (result) {
+ /* TODO: write it */
+ }
+ else {
+ /* Error message is on index 3 */
+ const auto *err_msg = lua_tostring(L, 3);
+ rt->err = rspamd::util::error(err_msg, 500);
+ msg_err_task("cannot learn task: %s", err_msg);
+ }
+
+ return 0;
+}
+
+gboolean
+rspamd_redis_learn_tokens(struct rspamd_task *task,
+ GPtrArray *tokens,
+ gint id, gpointer p)
+{
+ auto *rt = REDIS_RUNTIME(p);
+ auto *L = rt->ctx->L;
+
+ if (rspamd_session_blocked(task->s)) {
+ return FALSE;
+ }
+
+ if (tokens == nullptr || tokens->len == 0) {
+ return FALSE;
+ }
+
+ gsize tokens_len;
+ gchar *tokens_buf = rspamd_redis_serialize_tokens(task, rt->redis_object_expanded, tokens, &tokens_len);
+
+ rt->id = id;
+
+ gsize text_tokens_len = 0;
+ gchar *text_tokens_buf = nullptr;
+
+ if (rt->ctx->store_tokens) {
+ text_tokens_buf = rspamd_redis_serialize_text_tokens(task, tokens, &text_tokens_len);
+ }
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(L);
+ auto nargs = 8;
+
+ /* Function arguments */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, rt->ctx->cbref_learn);
+ rspamd_lua_task_push(L, task);
+ lua_pushstring(L, rt->redis_object_expanded);
+ lua_pushinteger(L, id);
+ lua_pushboolean(L, rt->stcf->is_spam);
+ lua_pushstring(L, rt->stcf->symbol);
+
+ /* Detect unlearn */
+ auto *tok = (rspamd_token_t *) g_ptr_array_index(task->tokens, 0);
+
+ if (tok->values[id] > 0) {
+ lua_pushboolean(L, FALSE);// Learn
+ }
+ else {
+ lua_pushboolean(L, TRUE);// Unlearn
+ }
+ lua_new_text(L, tokens_buf, tokens_len, false);
+
+ /* Store rt in random cookie */
+ char *cookie = (char *) rspamd_mempool_alloc(task->task_pool, 16);
+ rspamd_random_hex(cookie, 16);
+ cookie[15] = '\0';
+ rspamd_mempool_set_variable(task->task_pool, cookie, rt, nullptr);
+ /* Callback */
+ lua_pushstring(L, cookie);
+ lua_pushcclosure(L, &rspamd_redis_learned, 1);
+
+ if (text_tokens_len) {
+ nargs = 9;
+ lua_new_text(L, text_tokens_buf, text_tokens_len, false);
+ }
+
+ if (lua_pcall(L, nargs, 0, err_idx) != 0) {
+ msg_err_task("call to script failed: %s", lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+ return FALSE;
+ }
+
+ rt->tokens = g_ptr_array_ref(tokens);
+
+ lua_settop(L, err_idx - 1);
+ return TRUE;
+}
+
+
+gboolean
+rspamd_redis_finalize_learn(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx, GError **err)
+{
+ auto *rt = REDIS_RUNTIME(runtime);
+
+ if (rt->err.has_value()) {
+ rt->err->into_g_error_set(rspamd_redis_stat_quark(), err);
+
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gulong
+rspamd_redis_total_learns(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ auto *rt = REDIS_RUNTIME(runtime);
+
+ return rt->learned;
+}
+
+gulong
+rspamd_redis_inc_learns(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ auto *rt = REDIS_RUNTIME(runtime);
+
+ /* XXX: may cause races */
+ return rt->learned + 1;
+}
+
+gulong
+rspamd_redis_dec_learns(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ auto *rt = REDIS_RUNTIME(runtime);
+
+ /* XXX: may cause races */
+ return rt->learned + 1;
+}
+
+gulong
+rspamd_redis_learns(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ auto *rt = REDIS_RUNTIME(runtime);
+
+ return rt->learned;
+}
+
+ucl_object_t *
+rspamd_redis_get_stat(gpointer runtime,
+ gpointer ctx)
+{
+ auto *rt = REDIS_RUNTIME(runtime);
+
+ return ucl_object_ref(rt->ctx->cur_stat);
+}
+
+gpointer
+rspamd_redis_load_tokenizer_config(gpointer runtime,
+ gsize *len)
+{
+ return nullptr;
+}
diff --git a/src/libstat/backends/sqlite3_backend.c b/src/libstat/backends/sqlite3_backend.c
new file mode 100644
index 0000000..2fd34d8
--- /dev/null
+++ b/src/libstat/backends/sqlite3_backend.c
@@ -0,0 +1,907 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "sqlite3.h"
+#include "libutil/sqlite_utils.h"
+#include "libstat/stat_internal.h"
+#include "libmime/message.h"
+#include "lua/lua_common.h"
+#include "unix-std.h"
+
+#define SQLITE3_BACKEND_TYPE "sqlite3"
+#define SQLITE3_SCHEMA_VERSION "1"
+#define SQLITE3_DEFAULT "default"
+
+struct rspamd_stat_sqlite3_db {
+ sqlite3 *sqlite;
+ gchar *fname;
+ GArray *prstmt;
+ lua_State *L;
+ rspamd_mempool_t *pool;
+ gboolean in_transaction;
+ gboolean enable_users;
+ gboolean enable_languages;
+ gint cbref_user;
+ gint cbref_language;
+};
+
+struct rspamd_stat_sqlite3_rt {
+ struct rspamd_task *task;
+ struct rspamd_stat_sqlite3_db *db;
+ struct rspamd_statfile_config *cf;
+ gint64 user_id;
+ gint64 lang_id;
+};
+
+static const char *create_tables_sql =
+ "BEGIN IMMEDIATE;"
+ "CREATE TABLE tokenizer(data BLOB);"
+ "CREATE TABLE users("
+ "id INTEGER PRIMARY KEY,"
+ "name TEXT,"
+ "learns INTEGER"
+ ");"
+ "CREATE TABLE languages("
+ "id INTEGER PRIMARY KEY,"
+ "name TEXT,"
+ "learns INTEGER"
+ ");"
+ "CREATE TABLE tokens("
+ "token INTEGER NOT NULL,"
+ "user INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,"
+ "language INTEGER NOT NULL REFERENCES languages(id) ON DELETE CASCADE,"
+ "value INTEGER,"
+ "modified INTEGER,"
+ "CONSTRAINT tid UNIQUE (token, user, language) ON CONFLICT REPLACE"
+ ");"
+ "CREATE UNIQUE INDEX IF NOT EXISTS un ON users(name);"
+ "CREATE INDEX IF NOT EXISTS tok ON tokens(token);"
+ "CREATE UNIQUE INDEX IF NOT EXISTS ln ON languages(name);"
+ "PRAGMA user_version=" SQLITE3_SCHEMA_VERSION ";"
+ "INSERT INTO users(id, name, learns) VALUES(0, '" SQLITE3_DEFAULT "',0);"
+ "INSERT INTO languages(id, name, learns) VALUES(0, '" SQLITE3_DEFAULT "',0);"
+ "COMMIT;";
+
+enum rspamd_stat_sqlite3_stmt_idx {
+ RSPAMD_STAT_BACKEND_TRANSACTION_START_IM = 0,
+ RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF,
+ RSPAMD_STAT_BACKEND_TRANSACTION_START_EXCL,
+ RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT,
+ RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK,
+ RSPAMD_STAT_BACKEND_GET_TOKEN_FULL,
+ RSPAMD_STAT_BACKEND_GET_TOKEN_SIMPLE,
+ RSPAMD_STAT_BACKEND_SET_TOKEN,
+ RSPAMD_STAT_BACKEND_INC_LEARNS_LANG,
+ RSPAMD_STAT_BACKEND_INC_LEARNS_USER,
+ RSPAMD_STAT_BACKEND_DEC_LEARNS_LANG,
+ RSPAMD_STAT_BACKEND_DEC_LEARNS_USER,
+ RSPAMD_STAT_BACKEND_GET_LEARNS,
+ RSPAMD_STAT_BACKEND_GET_LANGUAGE,
+ RSPAMD_STAT_BACKEND_GET_USER,
+ RSPAMD_STAT_BACKEND_INSERT_USER,
+ RSPAMD_STAT_BACKEND_INSERT_LANGUAGE,
+ RSPAMD_STAT_BACKEND_SAVE_TOKENIZER,
+ RSPAMD_STAT_BACKEND_LOAD_TOKENIZER,
+ RSPAMD_STAT_BACKEND_NTOKENS,
+ RSPAMD_STAT_BACKEND_NLANGUAGES,
+ RSPAMD_STAT_BACKEND_NUSERS,
+ RSPAMD_STAT_BACKEND_MAX
+};
+
+static struct rspamd_sqlite3_prstmt prepared_stmts[RSPAMD_STAT_BACKEND_MAX] =
+ {
+ [RSPAMD_STAT_BACKEND_TRANSACTION_START_IM] = {
+ .idx = RSPAMD_STAT_BACKEND_TRANSACTION_START_IM,
+ .sql = "BEGIN IMMEDIATE TRANSACTION;",
+ .args = "",
+ .stmt = NULL,
+ .result = SQLITE_DONE,
+ .flags = 0,
+ .ret = "",
+ },
+ [RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF] = {.idx = RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF, .sql = "BEGIN DEFERRED TRANSACTION;", .args = "", .stmt = NULL, .result = SQLITE_DONE, .flags = 0, .ret = ""},
+ [RSPAMD_STAT_BACKEND_TRANSACTION_START_EXCL] = {.idx = RSPAMD_STAT_BACKEND_TRANSACTION_START_EXCL, .sql = "BEGIN EXCLUSIVE TRANSACTION;", .args = "", .stmt = NULL, .result = SQLITE_DONE, .flags = 0, .ret = ""},
+ [RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT] = {.idx = RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT, .sql = "COMMIT;", .args = "", .stmt = NULL, .result = SQLITE_DONE, .flags = 0, .ret = ""},
+ [RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK] = {.idx = RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK, .sql = "ROLLBACK;", .args = "", .stmt = NULL, .result = SQLITE_DONE, .flags = 0, .ret = ""},
+ [RSPAMD_STAT_BACKEND_GET_TOKEN_FULL] = {.idx = RSPAMD_STAT_BACKEND_GET_TOKEN_FULL, .sql = "SELECT value FROM tokens "
+ "LEFT JOIN languages ON tokens.language=languages.id "
+ "LEFT JOIN users ON tokens.user=users.id "
+ "WHERE token=?1 AND (users.id=?2) "
+ "AND (languages.id=?3 OR languages.id=0);",
+ .stmt = NULL,
+ .args = "III",
+ .result = SQLITE_ROW,
+ .flags = 0,
+ .ret = "I"},
+ [RSPAMD_STAT_BACKEND_GET_TOKEN_SIMPLE] = {.idx = RSPAMD_STAT_BACKEND_GET_TOKEN_SIMPLE, .sql = "SELECT value FROM tokens WHERE token=?1", .stmt = NULL, .args = "I", .result = SQLITE_ROW, .flags = 0, .ret = "I"},
+ [RSPAMD_STAT_BACKEND_SET_TOKEN] = {.idx = RSPAMD_STAT_BACKEND_SET_TOKEN, .sql = "INSERT OR REPLACE INTO tokens (token, user, language, value, modified) "
+ "VALUES (?1, ?2, ?3, ?4, strftime('%s','now'))",
+ .stmt = NULL,
+ .args = "IIII",
+ .result = SQLITE_DONE,
+ .flags = 0,
+ .ret = ""},
+ [RSPAMD_STAT_BACKEND_INC_LEARNS_LANG] = {.idx = RSPAMD_STAT_BACKEND_INC_LEARNS_LANG, .sql = "UPDATE languages SET learns=learns + 1 WHERE id=?1", .stmt = NULL, .args = "I", .result = SQLITE_DONE, .flags = 0, .ret = ""},
+ [RSPAMD_STAT_BACKEND_INC_LEARNS_USER] = {.idx = RSPAMD_STAT_BACKEND_INC_LEARNS_USER, .sql = "UPDATE users SET learns=learns + 1 WHERE id=?1", .stmt = NULL, .args = "I", .result = SQLITE_DONE, .flags = 0, .ret = ""},
+ [RSPAMD_STAT_BACKEND_DEC_LEARNS_LANG] = {.idx = RSPAMD_STAT_BACKEND_DEC_LEARNS_LANG, .sql = "UPDATE languages SET learns=MAX(0, learns - 1) WHERE id=?1", .stmt = NULL, .args = "I", .result = SQLITE_DONE, .flags = 0, .ret = ""},
+ [RSPAMD_STAT_BACKEND_DEC_LEARNS_USER] = {.idx = RSPAMD_STAT_BACKEND_DEC_LEARNS_USER, .sql = "UPDATE users SET learns=MAX(0, learns - 1) WHERE id=?1", .stmt = NULL, .args = "I", .result = SQLITE_DONE, .flags = 0, .ret = ""},
+ [RSPAMD_STAT_BACKEND_GET_LEARNS] = {.idx = RSPAMD_STAT_BACKEND_GET_LEARNS, .sql = "SELECT SUM(MAX(0, learns)) FROM languages", .stmt = NULL, .args = "", .result = SQLITE_ROW, .flags = 0, .ret = "I"},
+ [RSPAMD_STAT_BACKEND_GET_LANGUAGE] = {.idx = RSPAMD_STAT_BACKEND_GET_LANGUAGE, .sql = "SELECT id FROM languages WHERE name=?1", .stmt = NULL, .args = "T", .result = SQLITE_ROW, .flags = 0, .ret = "I"},
+ [RSPAMD_STAT_BACKEND_GET_USER] = {.idx = RSPAMD_STAT_BACKEND_GET_USER, .sql = "SELECT id FROM users WHERE name=?1", .stmt = NULL, .args = "T", .result = SQLITE_ROW, .flags = 0, .ret = "I"},
+ [RSPAMD_STAT_BACKEND_INSERT_USER] = {.idx = RSPAMD_STAT_BACKEND_INSERT_USER, .sql = "INSERT INTO users (name, learns) VALUES (?1, 0)", .stmt = NULL, .args = "T", .result = SQLITE_DONE, .flags = 0, .ret = "L"},
+ [RSPAMD_STAT_BACKEND_INSERT_LANGUAGE] = {.idx = RSPAMD_STAT_BACKEND_INSERT_LANGUAGE, .sql = "INSERT INTO languages (name, learns) VALUES (?1, 0)", .stmt = NULL, .args = "T", .result = SQLITE_DONE, .flags = 0, .ret = "L"},
+ [RSPAMD_STAT_BACKEND_SAVE_TOKENIZER] = {.idx = RSPAMD_STAT_BACKEND_SAVE_TOKENIZER, .sql = "INSERT INTO tokenizer(data) VALUES (?1)", .stmt = NULL, .args = "B", .result = SQLITE_DONE, .flags = 0, .ret = ""},
+ [RSPAMD_STAT_BACKEND_LOAD_TOKENIZER] = {.idx = RSPAMD_STAT_BACKEND_LOAD_TOKENIZER, .sql = "SELECT data FROM tokenizer", .stmt = NULL, .args = "", .result = SQLITE_ROW, .flags = 0, .ret = "B"},
+ [RSPAMD_STAT_BACKEND_NTOKENS] = {.idx = RSPAMD_STAT_BACKEND_NTOKENS, .sql = "SELECT COUNT(*) FROM tokens", .stmt = NULL, .args = "", .result = SQLITE_ROW, .flags = 0, .ret = "I"},
+ [RSPAMD_STAT_BACKEND_NLANGUAGES] = {.idx = RSPAMD_STAT_BACKEND_NLANGUAGES, .sql = "SELECT COUNT(*) FROM languages", .stmt = NULL, .args = "", .result = SQLITE_ROW, .flags = 0, .ret = "I"},
+ [RSPAMD_STAT_BACKEND_NUSERS] = {.idx = RSPAMD_STAT_BACKEND_NUSERS, .sql = "SELECT COUNT(*) FROM users", .stmt = NULL, .args = "", .result = SQLITE_ROW, .flags = 0, .ret = "I"}};
+
+static GQuark
+rspamd_sqlite3_backend_quark(void)
+{
+ return g_quark_from_static_string("sqlite3-stat-backend");
+}
+
+static gint64
+rspamd_sqlite3_get_user(struct rspamd_stat_sqlite3_db *db,
+ struct rspamd_task *task, gboolean learn)
+{
+ gint64 id = 0; /* Default user is 0 */
+ gint rc, err_idx;
+ const gchar *user = NULL;
+ struct rspamd_task **ptask;
+ lua_State *L = db->L;
+
+ if (db->cbref_user == -1) {
+ user = rspamd_task_get_principal_recipient(task);
+ }
+ else {
+ /* Execute lua function to get userdata */
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, db->cbref_user);
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ if (lua_pcall(L, 1, 1, err_idx) != 0) {
+ msg_err_task("call to user extraction script failed: %s",
+ lua_tostring(L, -1));
+ }
+ else {
+ user = rspamd_mempool_strdup(task->task_pool, lua_tostring(L, -1));
+ }
+
+ /* Result + error function */
+ lua_settop(L, err_idx - 1);
+ }
+
+
+ if (user != NULL) {
+ rspamd_mempool_set_variable(task->task_pool, "stat_user",
+ (gpointer) user, NULL);
+
+ rc = rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt,
+ RSPAMD_STAT_BACKEND_GET_USER, user, &id);
+
+ if (rc != SQLITE_OK && learn) {
+ /* We need to insert a new user */
+ if (!db->in_transaction) {
+ rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_START_IM);
+ db->in_transaction = TRUE;
+ }
+
+ rc = rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt,
+ RSPAMD_STAT_BACKEND_INSERT_USER, user, &id);
+ }
+ }
+
+ return id;
+}
+
+static gint64
+rspamd_sqlite3_get_language(struct rspamd_stat_sqlite3_db *db,
+ struct rspamd_task *task, gboolean learn)
+{
+ gint64 id = 0; /* Default language is 0 */
+ gint rc, err_idx;
+ guint i;
+ const gchar *language = NULL;
+ struct rspamd_mime_text_part *tp;
+ struct rspamd_task **ptask;
+ lua_State *L = db->L;
+
+ if (db->cbref_language == -1) {
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, tp)
+ {
+
+ if (tp->language != NULL && tp->language[0] != '\0' &&
+ strcmp(tp->language, "en") != 0) {
+ language = tp->language;
+ break;
+ }
+ }
+ }
+ else {
+ /* Execute lua function to get userdata */
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, db->cbref_language);
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ if (lua_pcall(L, 1, 1, err_idx) != 0) {
+ msg_err_task("call to language extraction script failed: %s",
+ lua_tostring(L, -1));
+ }
+ else {
+ language = rspamd_mempool_strdup(task->task_pool,
+ lua_tostring(L, -1));
+ }
+
+ /* Result + error function */
+ lua_settop(L, err_idx - 1);
+ }
+
+
+ /* XXX: We ignore multiple languages but default + extra */
+ if (language != NULL) {
+ rc = rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt,
+ RSPAMD_STAT_BACKEND_GET_LANGUAGE, language, &id);
+
+ if (rc != SQLITE_OK && learn) {
+ /* We need to insert a new language */
+ if (!db->in_transaction) {
+ rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_START_IM);
+ db->in_transaction = TRUE;
+ }
+
+ rc = rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt,
+ RSPAMD_STAT_BACKEND_INSERT_LANGUAGE, language, &id);
+ }
+ }
+
+ return id;
+}
+
+static struct rspamd_stat_sqlite3_db *
+rspamd_sqlite3_opendb(rspamd_mempool_t *pool,
+ struct rspamd_statfile_config *stcf,
+ const gchar *path, const ucl_object_t *opts,
+ gboolean create, GError **err)
+{
+ struct rspamd_stat_sqlite3_db *bk;
+ struct rspamd_stat_tokenizer *tokenizer;
+ gpointer tk_conf;
+ gsize sz = 0;
+ gint64 sz64 = 0;
+ gchar *tok_conf_encoded;
+ gint ret, ntries = 0;
+ const gint max_tries = 100;
+ struct timespec sleep_ts = {
+ .tv_sec = 0,
+ .tv_nsec = 1000000};
+
+ bk = g_malloc0(sizeof(*bk));
+ bk->sqlite = rspamd_sqlite3_open_or_create(pool, path, create_tables_sql,
+ 0, err);
+ bk->pool = pool;
+
+ if (bk->sqlite == NULL) {
+ g_free(bk);
+
+ return NULL;
+ }
+
+ bk->fname = g_strdup(path);
+
+ bk->prstmt = rspamd_sqlite3_init_prstmt(bk->sqlite, prepared_stmts,
+ RSPAMD_STAT_BACKEND_MAX, err);
+
+ if (bk->prstmt == NULL) {
+ sqlite3_close(bk->sqlite);
+ g_free(bk);
+
+ return NULL;
+ }
+
+ /* Check tokenizer configuration */
+ if (rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_LOAD_TOKENIZER, &sz64, &tk_conf) != SQLITE_OK ||
+ sz64 == 0) {
+
+ while ((ret = rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_START_EXCL)) == SQLITE_BUSY &&
+ ++ntries <= max_tries) {
+ nanosleep(&sleep_ts, NULL);
+ }
+
+ msg_info_pool("absent tokenizer conf in %s, creating a new one",
+ bk->fname);
+ g_assert(stcf->clcf->tokenizer != NULL);
+ tokenizer = rspamd_stat_get_tokenizer(stcf->clcf->tokenizer->name);
+ g_assert(tokenizer != NULL);
+ tk_conf = tokenizer->get_config(pool, stcf->clcf->tokenizer, &sz);
+
+ /* Encode to base32 */
+ tok_conf_encoded = rspamd_encode_base32(tk_conf, sz, RSPAMD_BASE32_DEFAULT);
+
+ if (rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_SAVE_TOKENIZER,
+ (gint64) strlen(tok_conf_encoded),
+ tok_conf_encoded) != SQLITE_OK) {
+ sqlite3_close(bk->sqlite);
+ g_free(bk);
+ g_free(tok_conf_encoded);
+
+ return NULL;
+ }
+
+ rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT);
+ g_free(tok_conf_encoded);
+ }
+ else {
+ g_free(tk_conf);
+ }
+
+ return bk;
+}
+
+gpointer
+rspamd_sqlite3_init(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct rspamd_statfile *st)
+{
+ struct rspamd_classifier_config *clf = st->classifier->cfg;
+ struct rspamd_statfile_config *stf = st->stcf;
+ const ucl_object_t *filenameo, *lang_enabled, *users_enabled;
+ const gchar *filename, *lua_script;
+ struct rspamd_stat_sqlite3_db *bk;
+ GError *err = NULL;
+
+ filenameo = ucl_object_lookup(stf->opts, "filename");
+ if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) {
+ filenameo = ucl_object_lookup(stf->opts, "path");
+ if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) {
+ msg_err_config("statfile %s has no filename defined", stf->symbol);
+ return NULL;
+ }
+ }
+
+ filename = ucl_object_tostring(filenameo);
+
+ if ((bk = rspamd_sqlite3_opendb(cfg->cfg_pool, stf, filename,
+ stf->opts, TRUE, &err)) == NULL) {
+ msg_err_config("cannot open sqlite3 db %s: %e", filename, err);
+ g_error_free(err);
+ return NULL;
+ }
+
+ bk->L = cfg->lua_state;
+
+ users_enabled = ucl_object_lookup_any(clf->opts, "per_user",
+ "users_enabled", NULL);
+ if (users_enabled != NULL) {
+ if (ucl_object_type(users_enabled) == UCL_BOOLEAN) {
+ bk->enable_users = ucl_object_toboolean(users_enabled);
+ bk->cbref_user = -1;
+ }
+ else if (ucl_object_type(users_enabled) == UCL_STRING) {
+ lua_script = ucl_object_tostring(users_enabled);
+
+ if (luaL_dostring(cfg->lua_state, lua_script) != 0) {
+ msg_err_config("cannot execute lua script for users "
+ "extraction: %s",
+ lua_tostring(cfg->lua_state, -1));
+ }
+ else {
+ if (lua_type(cfg->lua_state, -1) == LUA_TFUNCTION) {
+ bk->enable_users = TRUE;
+ bk->cbref_user = luaL_ref(cfg->lua_state,
+ LUA_REGISTRYINDEX);
+ }
+ else {
+ msg_err_config("lua script must return "
+ "function(task) and not %s",
+ lua_typename(cfg->lua_state, lua_type(
+ cfg->lua_state, -1)));
+ }
+ }
+ }
+ }
+ else {
+ bk->enable_users = FALSE;
+ }
+
+ lang_enabled = ucl_object_lookup_any(clf->opts,
+ "per_language", "languages_enabled", NULL);
+
+ if (lang_enabled != NULL) {
+ if (ucl_object_type(lang_enabled) == UCL_BOOLEAN) {
+ bk->enable_languages = ucl_object_toboolean(lang_enabled);
+ bk->cbref_language = -1;
+ }
+ else if (ucl_object_type(lang_enabled) == UCL_STRING) {
+ lua_script = ucl_object_tostring(lang_enabled);
+
+ if (luaL_dostring(cfg->lua_state, lua_script) != 0) {
+ msg_err_config(
+ "cannot execute lua script for languages "
+ "extraction: %s",
+ lua_tostring(cfg->lua_state, -1));
+ }
+ else {
+ if (lua_type(cfg->lua_state, -1) == LUA_TFUNCTION) {
+ bk->enable_languages = TRUE;
+ bk->cbref_language = luaL_ref(cfg->lua_state,
+ LUA_REGISTRYINDEX);
+ }
+ else {
+ msg_err_config("lua script must return "
+ "function(task) and not %s",
+ lua_typename(cfg->lua_state,
+ lua_type(cfg->lua_state, -1)));
+ }
+ }
+ }
+ }
+ else {
+ bk->enable_languages = FALSE;
+ }
+
+ if (bk->enable_languages) {
+ msg_info_config("enable per language statistics for %s",
+ stf->symbol);
+ }
+
+ if (bk->enable_users) {
+ msg_info_config("enable per users statistics for %s",
+ stf->symbol);
+ }
+
+
+ return (gpointer) bk;
+}
+
+void rspamd_sqlite3_close(gpointer p)
+{
+ struct rspamd_stat_sqlite3_db *bk = p;
+
+ if (bk->sqlite) {
+ if (bk->in_transaction) {
+ rspamd_sqlite3_run_prstmt(bk->pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT);
+ }
+
+ rspamd_sqlite3_close_prstmt(bk->sqlite, bk->prstmt);
+ sqlite3_close(bk->sqlite);
+ g_free(bk->fname);
+ g_free(bk);
+ }
+}
+
+gpointer
+rspamd_sqlite3_runtime(struct rspamd_task *task,
+ struct rspamd_statfile_config *stcf, gboolean learn, gpointer p, gint _id)
+{
+ struct rspamd_stat_sqlite3_rt *rt = NULL;
+ struct rspamd_stat_sqlite3_db *bk = p;
+
+ if (bk) {
+ rt = rspamd_mempool_alloc(task->task_pool, sizeof(*rt));
+ rt->db = bk;
+ rt->task = task;
+ rt->user_id = -1;
+ rt->lang_id = -1;
+ rt->cf = stcf;
+ }
+
+ return rt;
+}
+
+gboolean
+rspamd_sqlite3_process_tokens(struct rspamd_task *task,
+ GPtrArray *tokens,
+ gint id, gpointer p)
+{
+ struct rspamd_stat_sqlite3_db *bk;
+ struct rspamd_stat_sqlite3_rt *rt = p;
+ gint64 iv = 0;
+ guint i;
+ rspamd_token_t *tok;
+
+ g_assert(p != NULL);
+ g_assert(tokens != NULL);
+
+ bk = rt->db;
+
+ for (i = 0; i < tokens->len; i++) {
+ tok = g_ptr_array_index(tokens, i);
+
+ if (bk == NULL) {
+ /* Statfile is does not exist, so all values are zero */
+ tok->values[id] = 0.0f;
+ continue;
+ }
+
+ if (!bk->in_transaction) {
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF);
+ bk->in_transaction = TRUE;
+ }
+
+ if (rt->user_id == -1) {
+ if (bk->enable_users) {
+ rt->user_id = rspamd_sqlite3_get_user(bk, task, FALSE);
+ }
+ else {
+ rt->user_id = 0;
+ }
+ }
+
+ if (rt->lang_id == -1) {
+ if (bk->enable_languages) {
+ rt->lang_id = rspamd_sqlite3_get_language(bk, task, FALSE);
+ }
+ else {
+ rt->lang_id = 0;
+ }
+ }
+
+ if (bk->enable_languages || bk->enable_users) {
+ if (rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_GET_TOKEN_FULL,
+ tok->data, rt->user_id, rt->lang_id, &iv) == SQLITE_OK) {
+ tok->values[id] = iv;
+ }
+ else {
+ tok->values[id] = 0.0f;
+ }
+ }
+ else {
+ if (rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_GET_TOKEN_SIMPLE,
+ tok->data, &iv) == SQLITE_OK) {
+ tok->values[id] = iv;
+ }
+ else {
+ tok->values[id] = 0.0f;
+ }
+ }
+
+ if (rt->cf->is_spam) {
+ task->flags |= RSPAMD_TASK_FLAG_HAS_SPAM_TOKENS;
+ }
+ else {
+ task->flags |= RSPAMD_TASK_FLAG_HAS_HAM_TOKENS;
+ }
+ }
+
+
+ return TRUE;
+}
+
+gboolean
+rspamd_sqlite3_finalize_process(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ struct rspamd_stat_sqlite3_rt *rt = runtime;
+ struct rspamd_stat_sqlite3_db *bk;
+
+ g_assert(rt != NULL);
+ bk = rt->db;
+
+ if (bk->in_transaction) {
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT);
+ bk->in_transaction = FALSE;
+ }
+
+ rt->lang_id = -1;
+ rt->user_id = -1;
+
+ return TRUE;
+}
+
+gboolean
+rspamd_sqlite3_learn_tokens(struct rspamd_task *task, GPtrArray *tokens,
+ gint id, gpointer p)
+{
+ struct rspamd_stat_sqlite3_db *bk;
+ struct rspamd_stat_sqlite3_rt *rt = p;
+ gint64 iv = 0;
+ guint i;
+ rspamd_token_t *tok;
+
+ g_assert(tokens != NULL);
+ g_assert(p != NULL);
+
+ bk = rt->db;
+
+ for (i = 0; i < tokens->len; i++) {
+ tok = g_ptr_array_index(tokens, i);
+ if (bk == NULL) {
+ /* Statfile is does not exist, so all values are zero */
+ return FALSE;
+ }
+
+ if (!bk->in_transaction) {
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_START_IM);
+ bk->in_transaction = TRUE;
+ }
+
+ if (rt->user_id == -1) {
+ if (bk->enable_users) {
+ rt->user_id = rspamd_sqlite3_get_user(bk, task, TRUE);
+ }
+ else {
+ rt->user_id = 0;
+ }
+ }
+
+ if (rt->lang_id == -1) {
+ if (bk->enable_languages) {
+ rt->lang_id = rspamd_sqlite3_get_language(bk, task, TRUE);
+ }
+ else {
+ rt->lang_id = 0;
+ }
+ }
+
+ iv = tok->values[id];
+
+ if (rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_SET_TOKEN,
+ tok->data, rt->user_id, rt->lang_id, iv) != SQLITE_OK) {
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK);
+ bk->in_transaction = FALSE;
+
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_sqlite3_finalize_learn(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx, GError **err)
+{
+ struct rspamd_stat_sqlite3_rt *rt = runtime;
+ struct rspamd_stat_sqlite3_db *bk;
+ gint wal_frames, wal_checkpointed, mode;
+
+ g_assert(rt != NULL);
+ bk = rt->db;
+
+ if (bk->in_transaction) {
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT);
+ bk->in_transaction = FALSE;
+ }
+
+#ifdef SQLITE_OPEN_WAL
+#ifdef SQLITE_CHECKPOINT_TRUNCATE
+ mode = SQLITE_CHECKPOINT_TRUNCATE;
+#elif defined(SQLITE_CHECKPOINT_RESTART)
+ mode = SQLITE_CHECKPOINT_RESTART;
+#elif defined(SQLITE_CHECKPOINT_FULL)
+ mode = SQLITE_CHECKPOINT_FULL;
+#endif
+ /* Perform wal checkpoint (might be long) */
+ if (sqlite3_wal_checkpoint_v2(bk->sqlite,
+ NULL,
+ mode,
+ &wal_frames,
+ &wal_checkpointed) != SQLITE_OK) {
+ msg_warn_task("cannot commit checkpoint: %s",
+ sqlite3_errmsg(bk->sqlite));
+
+ g_set_error(err, rspamd_sqlite3_backend_quark(), 500,
+ "cannot commit checkpoint: %s",
+ sqlite3_errmsg(bk->sqlite));
+ return FALSE;
+ }
+#endif
+
+ return TRUE;
+}
+
+gulong
+rspamd_sqlite3_total_learns(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ struct rspamd_stat_sqlite3_rt *rt = runtime;
+ struct rspamd_stat_sqlite3_db *bk;
+ guint64 res;
+
+ g_assert(rt != NULL);
+ bk = rt->db;
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_GET_LEARNS, &res);
+
+ return res;
+}
+
+gulong
+rspamd_sqlite3_inc_learns(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ struct rspamd_stat_sqlite3_rt *rt = runtime;
+ struct rspamd_stat_sqlite3_db *bk;
+ guint64 res;
+
+ g_assert(rt != NULL);
+ bk = rt->db;
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_INC_LEARNS_LANG,
+ rt->lang_id);
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_INC_LEARNS_USER,
+ rt->user_id);
+
+ if (bk->in_transaction) {
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT);
+ bk->in_transaction = FALSE;
+ }
+
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_GET_LEARNS, &res);
+
+ return res;
+}
+
+gulong
+rspamd_sqlite3_dec_learns(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ struct rspamd_stat_sqlite3_rt *rt = runtime;
+ struct rspamd_stat_sqlite3_db *bk;
+ guint64 res;
+
+ g_assert(rt != NULL);
+ bk = rt->db;
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_DEC_LEARNS_LANG,
+ rt->lang_id);
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_DEC_LEARNS_USER,
+ rt->user_id);
+
+ if (bk->in_transaction) {
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT);
+ bk->in_transaction = FALSE;
+ }
+
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_GET_LEARNS, &res);
+
+ return res;
+}
+
+gulong
+rspamd_sqlite3_learns(struct rspamd_task *task, gpointer runtime,
+ gpointer ctx)
+{
+ struct rspamd_stat_sqlite3_rt *rt = runtime;
+ struct rspamd_stat_sqlite3_db *bk;
+ guint64 res;
+
+ g_assert(rt != NULL);
+ bk = rt->db;
+ rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_GET_LEARNS, &res);
+
+ return res;
+}
+
+ucl_object_t *
+rspamd_sqlite3_get_stat(gpointer runtime,
+ gpointer ctx)
+{
+ ucl_object_t *res = NULL;
+ struct rspamd_stat_sqlite3_rt *rt = runtime;
+ struct rspamd_stat_sqlite3_db *bk;
+ rspamd_mempool_t *pool;
+ struct stat st;
+ gint64 rev;
+
+ g_assert(rt != NULL);
+ bk = rt->db;
+ pool = bk->pool;
+
+ (void) stat(bk->fname, &st);
+ rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_GET_LEARNS, &rev);
+
+ res = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(res, ucl_object_fromint(rev), "revision",
+ 0, false);
+ ucl_object_insert_key(res, ucl_object_fromint(st.st_size), "size",
+ 0, false);
+ rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_NTOKENS, &rev);
+ ucl_object_insert_key(res, ucl_object_fromint(rev), "total", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromint(rev), "used", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromstring(rt->cf->symbol),
+ "symbol", 0, false);
+ ucl_object_insert_key(res, ucl_object_fromstring("sqlite3"),
+ "type", 0, false);
+ rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_NLANGUAGES, &rev);
+ ucl_object_insert_key(res, ucl_object_fromint(rev),
+ "languages", 0, false);
+ rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_NUSERS, &rev);
+ ucl_object_insert_key(res, ucl_object_fromint(rev),
+ "users", 0, false);
+
+ if (rt->cf->label) {
+ ucl_object_insert_key(res, ucl_object_fromstring(rt->cf->label),
+ "label", 0, false);
+ }
+
+ return res;
+}
+
+gpointer
+rspamd_sqlite3_load_tokenizer_config(gpointer runtime,
+ gsize *len)
+{
+ gpointer tk_conf, copied_conf;
+ guint64 sz;
+ struct rspamd_stat_sqlite3_rt *rt = runtime;
+ struct rspamd_stat_sqlite3_db *bk;
+
+ g_assert(rt != NULL);
+ bk = rt->db;
+
+ g_assert(rspamd_sqlite3_run_prstmt(rt->db->pool, bk->sqlite, bk->prstmt,
+ RSPAMD_STAT_BACKEND_LOAD_TOKENIZER, &sz, &tk_conf) == SQLITE_OK);
+ g_assert(sz > 0);
+ /*
+ * Here we can have either decoded or undecoded version of tokenizer config
+ * XXX: dirty hack to check if we have osb magic here
+ */
+ if (sz > 7 && memcmp(tk_conf, "osbtokv", 7) == 0) {
+ copied_conf = rspamd_mempool_alloc(rt->task->task_pool, sz);
+ memcpy(copied_conf, tk_conf, sz);
+ g_free(tk_conf);
+ }
+ else {
+ /* Need to decode */
+ copied_conf = rspamd_decode_base32(tk_conf, sz, len, RSPAMD_BASE32_DEFAULT);
+ g_free(tk_conf);
+ rspamd_mempool_add_destructor(rt->task->task_pool, g_free, copied_conf);
+ }
+
+ if (len) {
+ *len = sz;
+ }
+
+ return copied_conf;
+}
diff --git a/src/libstat/classifiers/bayes.c b/src/libstat/classifiers/bayes.c
new file mode 100644
index 0000000..513db9a
--- /dev/null
+++ b/src/libstat/classifiers/bayes.c
@@ -0,0 +1,551 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Bayesian classifier
+ */
+#include "classifiers.h"
+#include "rspamd.h"
+#include "stat_internal.h"
+#include "math.h"
+
+#define msg_err_bayes(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "bayes", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_bayes(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "bayes", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_bayes(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "bayes", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE_PUBLIC(bayes)
+
+static inline GQuark
+bayes_error_quark(void)
+{
+ return g_quark_from_static_string("bayes-error");
+}
+
+/**
+ * Returns probability of chisquare > value with specified number of freedom
+ * degrees
+ * @param value value to test
+ * @param freedom_deg number of degrees of freedom
+ * @return
+ */
+static gdouble
+inv_chi_square(struct rspamd_task *task, gdouble value, gint freedom_deg)
+{
+ double prob, sum, m;
+ gint i;
+
+ errno = 0;
+ m = -value;
+ prob = exp(value);
+
+ if (errno == ERANGE) {
+ /*
+ * e^x where x is large *NEGATIVE* number is OK, so we have a very strong
+ * confidence that inv-chi-square is close to zero
+ */
+ msg_debug_bayes("exp overflow");
+
+ if (value < 0) {
+ return 0;
+ }
+ else {
+ return 1.0;
+ }
+ }
+
+ sum = prob;
+
+ msg_debug_bayes("m: %f, probability: %g", m, prob);
+
+ /*
+ * m is our confidence in class
+ * prob is e ^ x (small value since x is normally less than zero
+ * So we integrate over degrees of freedom and produce the total result
+ * from 1.0 (no confidence) to 0.0 (full confidence)
+ */
+ for (i = 1; i < freedom_deg; i++) {
+ prob *= m / (gdouble) i;
+ sum += prob;
+ msg_debug_bayes("i=%d, probability: %g, sum: %g", i, prob, sum);
+ }
+
+ return MIN(1.0, sum);
+}
+
+struct bayes_task_closure {
+ double ham_prob;
+ double spam_prob;
+ gdouble meta_skip_prob;
+ guint64 processed_tokens;
+ guint64 total_hits;
+ guint64 text_tokens;
+ struct rspamd_task *task;
+};
+
+/*
+ * Mathematically we use pow(complexity, complexity), where complexity is the
+ * window index
+ */
+static const double feature_weight[] = {0, 3125, 256, 27, 1, 0, 0, 0};
+
+#define PROB_COMBINE(prob, cnt, weight, assumed) (((weight) * (assumed) + (cnt) * (prob)) / ((weight) + (cnt)))
+/*
+ * In this callback we calculate local probabilities for tokens
+ */
+static void
+bayes_classify_token(struct rspamd_classifier *ctx,
+ rspamd_token_t *tok, struct bayes_task_closure *cl)
+{
+ guint i;
+ gint id;
+ guint spam_count = 0, ham_count = 0, total_count = 0;
+ struct rspamd_statfile *st;
+ struct rspamd_task *task;
+ const gchar *token_type = "txt";
+ double spam_prob, spam_freq, ham_freq, bayes_spam_prob, bayes_ham_prob,
+ ham_prob, fw, w, val;
+
+ task = cl->task;
+
+#if 0
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_LUA_META) {
+ /* Ignore lua metatokens for now */
+ return;
+ }
+#endif
+
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_META && cl->meta_skip_prob > 0) {
+ val = rspamd_random_double_fast();
+
+ if (val <= cl->meta_skip_prob) {
+ if (tok->t1 && tok->t2) {
+ msg_debug_bayes(
+ "token(meta) %uL <%*s:%*s> probabilistically skipped",
+ tok->data,
+ (int) tok->t1->original.len, tok->t1->original.begin,
+ (int) tok->t2->original.len, tok->t2->original.begin);
+ }
+
+ return;
+ }
+ }
+
+ for (i = 0; i < ctx->statfiles_ids->len; i++) {
+ id = g_array_index(ctx->statfiles_ids, gint, i);
+ st = g_ptr_array_index(ctx->ctx->statfiles, id);
+ g_assert(st != NULL);
+ val = tok->values[id];
+
+ if (val > 0) {
+ if (st->stcf->is_spam) {
+ spam_count += val;
+ }
+ else {
+ ham_count += val;
+ }
+
+ total_count += val;
+ cl->total_hits += val;
+ }
+ }
+
+ /* Probability for this token */
+ if (total_count >= ctx->cfg->min_token_hits) {
+ spam_freq = ((double) spam_count / MAX(1., (double) ctx->spam_learns));
+ ham_freq = ((double) ham_count / MAX(1., (double) ctx->ham_learns));
+ spam_prob = spam_freq / (spam_freq + ham_freq);
+ ham_prob = ham_freq / (spam_freq + ham_freq);
+
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_UNIGRAM) {
+ fw = 1.0;
+ }
+ else {
+ fw = feature_weight[tok->window_idx %
+ G_N_ELEMENTS(feature_weight)];
+ }
+
+
+ w = (fw * total_count) / (1.0 + fw * total_count);
+
+ bayes_spam_prob = PROB_COMBINE(spam_prob, total_count, w, 0.5);
+
+ if ((bayes_spam_prob > 0.5 && bayes_spam_prob < 0.5 + ctx->cfg->min_prob_strength) ||
+ (bayes_spam_prob < 0.5 && bayes_spam_prob > 0.5 - ctx->cfg->min_prob_strength)) {
+ msg_debug_bayes(
+ "token %uL <%*s:%*s> skipped, probability not in range: %f",
+ tok->data,
+ (int) tok->t1->stemmed.len, tok->t1->stemmed.begin,
+ (int) tok->t2->stemmed.len, tok->t2->stemmed.begin,
+ bayes_spam_prob);
+
+ return;
+ }
+
+ bayes_ham_prob = PROB_COMBINE(ham_prob, total_count, w, 0.5);
+
+ cl->spam_prob += log(bayes_spam_prob);
+ cl->ham_prob += log(bayes_ham_prob);
+ cl->processed_tokens++;
+
+ if (!(tok->flags & RSPAMD_STAT_TOKEN_FLAG_META)) {
+ cl->text_tokens++;
+ }
+ else {
+ token_type = "meta";
+ }
+
+ if (tok->t1 && tok->t2) {
+ msg_debug_bayes("token(%s) %uL <%*s:%*s>: weight: %f, cf: %f, "
+ "total_count: %ud, "
+ "spam_count: %ud, ham_count: %ud,"
+ "spam_prob: %.3f, ham_prob: %.3f, "
+ "bayes_spam_prob: %.3f, bayes_ham_prob: %.3f, "
+ "current spam probability: %.3f, current ham probability: %.3f",
+ token_type,
+ tok->data,
+ (int) tok->t1->stemmed.len, tok->t1->stemmed.begin,
+ (int) tok->t2->stemmed.len, tok->t2->stemmed.begin,
+ fw, w, total_count, spam_count, ham_count,
+ spam_prob, ham_prob,
+ bayes_spam_prob, bayes_ham_prob,
+ cl->spam_prob, cl->ham_prob);
+ }
+ else {
+ msg_debug_bayes("token(%s) %uL <?:?>: weight: %f, cf: %f, "
+ "total_count: %ud, "
+ "spam_count: %ud, ham_count: %ud,"
+ "spam_prob: %.3f, ham_prob: %.3f, "
+ "bayes_spam_prob: %.3f, bayes_ham_prob: %.3f, "
+ "current spam probability: %.3f, current ham probability: %.3f",
+ token_type,
+ tok->data,
+ fw, w, total_count, spam_count, ham_count,
+ spam_prob, ham_prob,
+ bayes_spam_prob, bayes_ham_prob,
+ cl->spam_prob, cl->ham_prob);
+ }
+ }
+}
+
+
+gboolean
+bayes_init(struct rspamd_config *cfg,
+ struct ev_loop *ev_base,
+ struct rspamd_classifier *cl)
+{
+ cl->cfg->flags |= RSPAMD_FLAG_CLASSIFIER_INTEGER;
+
+ return TRUE;
+}
+
+void bayes_fin(struct rspamd_classifier *cl)
+{
+}
+
+gboolean
+bayes_classify(struct rspamd_classifier *ctx,
+ GPtrArray *tokens,
+ struct rspamd_task *task)
+{
+ double final_prob, h, s, *pprob;
+ gchar sumbuf[32];
+ struct rspamd_statfile *st = NULL;
+ struct bayes_task_closure cl;
+ rspamd_token_t *tok;
+ guint i, text_tokens = 0;
+ gint id;
+
+ g_assert(ctx != NULL);
+ g_assert(tokens != NULL);
+
+ memset(&cl, 0, sizeof(cl));
+ cl.task = task;
+
+ /* Check min learns */
+ if (ctx->cfg->min_learns > 0) {
+ if (ctx->ham_learns < ctx->cfg->min_learns) {
+ msg_info_task("not classified as ham. The ham class needs more "
+ "training samples. Currently: %ul; minimum %ud required",
+ ctx->ham_learns, ctx->cfg->min_learns);
+
+ return TRUE;
+ }
+ if (ctx->spam_learns < ctx->cfg->min_learns) {
+ msg_info_task("not classified as spam. The spam class needs more "
+ "training samples. Currently: %ul; minimum %ud required",
+ ctx->spam_learns, ctx->cfg->min_learns);
+
+ return TRUE;
+ }
+ }
+
+ for (i = 0; i < tokens->len; i++) {
+ tok = g_ptr_array_index(tokens, i);
+ if (!(tok->flags & RSPAMD_STAT_TOKEN_FLAG_META)) {
+ text_tokens++;
+ }
+ }
+
+ if (text_tokens == 0) {
+ msg_info_task("skipped classification as there are no text tokens. "
+ "Total tokens: %ud",
+ tokens->len);
+
+ return TRUE;
+ }
+
+ /*
+ * Skip some metatokens if we don't have enough text tokens
+ */
+ if (text_tokens > tokens->len - text_tokens) {
+ cl.meta_skip_prob = 0.0;
+ }
+ else {
+ cl.meta_skip_prob = 1.0 - text_tokens / tokens->len;
+ }
+
+ for (i = 0; i < tokens->len; i++) {
+ tok = g_ptr_array_index(tokens, i);
+
+ bayes_classify_token(ctx, tok, &cl);
+ }
+
+ if (cl.processed_tokens == 0) {
+ msg_info_bayes("no tokens found in bayes database "
+ "(%ud total tokens, %ud text tokens), ignore stats",
+ tokens->len, text_tokens);
+
+ return TRUE;
+ }
+
+ if (ctx->cfg->min_tokens > 0 &&
+ cl.text_tokens < (gint) (ctx->cfg->min_tokens * 0.1)) {
+ msg_info_bayes("ignore bayes probability since we have "
+ "found too few text tokens: %uL (of %ud checked), "
+ "at least %d required",
+ cl.text_tokens,
+ text_tokens,
+ (gint) (ctx->cfg->min_tokens * 0.1));
+
+ return TRUE;
+ }
+
+ if (cl.spam_prob > -300 && cl.ham_prob > -300) {
+ /* Fisher value is low enough to apply inv_chi_square */
+ h = 1 - inv_chi_square(task, cl.spam_prob, cl.processed_tokens);
+ s = 1 - inv_chi_square(task, cl.ham_prob, cl.processed_tokens);
+ }
+ else {
+ /* Use naive method */
+ if (cl.spam_prob < cl.ham_prob) {
+ h = (1.0 - exp(cl.spam_prob - cl.ham_prob)) /
+ (1.0 + exp(cl.spam_prob - cl.ham_prob));
+ s = 1.0 - h;
+ }
+ else {
+ s = (1.0 - exp(cl.ham_prob - cl.spam_prob)) /
+ (1.0 + exp(cl.ham_prob - cl.spam_prob));
+ h = 1.0 - s;
+ }
+ }
+
+ if (isfinite(s) && isfinite(h)) {
+ final_prob = (s + 1.0 - h) / 2.;
+ msg_debug_bayes(
+ "got ham probability %.2f -> %.2f and spam probability %.2f -> %.2f,"
+ " %L tokens processed of %ud total tokens;"
+ " %uL text tokens found of %ud text tokens)",
+ cl.ham_prob,
+ h,
+ cl.spam_prob,
+ s,
+ cl.processed_tokens,
+ tokens->len,
+ cl.text_tokens,
+ text_tokens);
+ }
+ else {
+ /*
+ * We have some overflow, hence we need to check which class
+ * is NaN
+ */
+ if (isfinite(h)) {
+ final_prob = 1.0;
+ msg_debug_bayes("spam class is full: no"
+ " ham samples");
+ }
+ else if (isfinite(s)) {
+ final_prob = 0.0;
+ msg_debug_bayes("ham class is full: no"
+ " spam samples");
+ }
+ else {
+ final_prob = 0.5;
+ msg_warn_bayes("spam and ham classes are both full");
+ }
+ }
+
+ pprob = rspamd_mempool_alloc(task->task_pool, sizeof(*pprob));
+ *pprob = final_prob;
+ rspamd_mempool_set_variable(task->task_pool, "bayes_prob", pprob, NULL);
+
+ if (cl.processed_tokens > 0 && fabs(final_prob - 0.5) > 0.05) {
+ /* Now we can have exactly one HAM and exactly one SPAM statfiles per classifier */
+ for (i = 0; i < ctx->statfiles_ids->len; i++) {
+ id = g_array_index(ctx->statfiles_ids, gint, i);
+ st = g_ptr_array_index(ctx->ctx->statfiles, id);
+
+ if (final_prob > 0.5 && st->stcf->is_spam) {
+ break;
+ }
+ else if (final_prob < 0.5 && !st->stcf->is_spam) {
+ break;
+ }
+ }
+
+ /* Correctly scale HAM */
+ if (final_prob < 0.5) {
+ final_prob = 1.0 - final_prob;
+ }
+
+ /*
+ * Bayes p is from 0.5 to 1.0, but confidence is from 0 to 1, so
+ * we need to rescale it to display correctly
+ */
+ rspamd_snprintf(sumbuf, sizeof(sumbuf), "%.2f%%",
+ (final_prob - 0.5) * 200.);
+ final_prob = rspamd_normalize_probability(final_prob, 0.5);
+ g_assert(st != NULL);
+
+ if (final_prob > 1 || final_prob < 0) {
+ msg_err_bayes("internal error: probability %f is outside of the "
+ "allowed range [0..1]",
+ final_prob);
+
+ if (final_prob > 1) {
+ final_prob = 1.0;
+ }
+ else {
+ final_prob = 0.0;
+ }
+ }
+
+ rspamd_task_insert_result(task,
+ st->stcf->symbol,
+ final_prob,
+ sumbuf);
+ }
+
+ return TRUE;
+}
+
+gboolean
+bayes_learn_spam(struct rspamd_classifier *ctx,
+ GPtrArray *tokens,
+ struct rspamd_task *task,
+ gboolean is_spam,
+ gboolean unlearn,
+ GError **err)
+{
+ guint i, j, total_cnt, spam_cnt, ham_cnt;
+ gint id;
+ struct rspamd_statfile *st;
+ rspamd_token_t *tok;
+ gboolean incrementing;
+
+ g_assert(ctx != NULL);
+ g_assert(tokens != NULL);
+
+ incrementing = ctx->cfg->flags & RSPAMD_FLAG_CLASSIFIER_INCREMENTING_BACKEND;
+
+ for (i = 0; i < tokens->len; i++) {
+ total_cnt = 0;
+ spam_cnt = 0;
+ ham_cnt = 0;
+ tok = g_ptr_array_index(tokens, i);
+
+ for (j = 0; j < ctx->statfiles_ids->len; j++) {
+ id = g_array_index(ctx->statfiles_ids, gint, j);
+ st = g_ptr_array_index(ctx->ctx->statfiles, id);
+ g_assert(st != NULL);
+
+ if (!!st->stcf->is_spam == !!is_spam) {
+ if (incrementing) {
+ tok->values[id] = 1;
+ }
+ else {
+ tok->values[id]++;
+ }
+
+ total_cnt += tok->values[id];
+
+ if (st->stcf->is_spam) {
+ spam_cnt += tok->values[id];
+ }
+ else {
+ ham_cnt += tok->values[id];
+ }
+ }
+ else {
+ if (tok->values[id] > 0 && unlearn) {
+ /* Unlearning */
+ if (incrementing) {
+ tok->values[id] = -1;
+ }
+ else {
+ tok->values[id]--;
+ }
+
+ if (st->stcf->is_spam) {
+ spam_cnt += tok->values[id];
+ }
+ else {
+ ham_cnt += tok->values[id];
+ }
+ total_cnt += tok->values[id];
+ }
+ else if (incrementing) {
+ tok->values[id] = 0;
+ }
+ }
+ }
+
+ if (tok->t1 && tok->t2) {
+ msg_debug_bayes("token %uL <%*s:%*s>: window: %d, total_count: %d, "
+ "spam_count: %d, ham_count: %d",
+ tok->data,
+ (int) tok->t1->stemmed.len, tok->t1->stemmed.begin,
+ (int) tok->t2->stemmed.len, tok->t2->stemmed.begin,
+ tok->window_idx, total_cnt, spam_cnt, ham_cnt);
+ }
+ else {
+ msg_debug_bayes("token %uL <?:?>: window: %d, total_count: %d, "
+ "spam_count: %d, ham_count: %d",
+ tok->data,
+ tok->window_idx, total_cnt, spam_cnt, ham_cnt);
+ }
+ }
+
+ return TRUE;
+}
diff --git a/src/libstat/classifiers/classifiers.h b/src/libstat/classifiers/classifiers.h
new file mode 100644
index 0000000..949408c
--- /dev/null
+++ b/src/libstat/classifiers/classifiers.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CLASSIFIERS_H
+#define CLASSIFIERS_H
+
+#include "config.h"
+#include "mem_pool.h"
+#include "contrib/libev/ev.h"
+
+#define RSPAMD_DEFAULT_CLASSIFIER "bayes"
+/* Consider this value as 0 */
+#define ALPHA 0.0001
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_classifier_config;
+struct rspamd_task;
+struct rspamd_config;
+struct rspamd_classifier;
+
+struct token_node_s;
+
+struct rspamd_stat_classifier {
+ char *name;
+
+ gboolean (*init_func)(struct rspamd_config *cfg,
+ struct ev_loop *ev_base,
+ struct rspamd_classifier *cl);
+
+ gboolean (*classify_func)(struct rspamd_classifier *ctx,
+ GPtrArray *tokens,
+ struct rspamd_task *task);
+
+ gboolean (*learn_spam_func)(struct rspamd_classifier *ctx,
+ GPtrArray *input,
+ struct rspamd_task *task,
+ gboolean is_spam,
+ gboolean unlearn,
+ GError **err);
+
+ void (*fin_func)(struct rspamd_classifier *cl);
+};
+
+/* Bayes algorithm */
+gboolean bayes_init(struct rspamd_config *cfg,
+ struct ev_loop *ev_base,
+ struct rspamd_classifier *);
+
+gboolean bayes_classify(struct rspamd_classifier *ctx,
+ GPtrArray *tokens,
+ struct rspamd_task *task);
+
+gboolean bayes_learn_spam(struct rspamd_classifier *ctx,
+ GPtrArray *tokens,
+ struct rspamd_task *task,
+ gboolean is_spam,
+ gboolean unlearn,
+ GError **err);
+
+void bayes_fin(struct rspamd_classifier *);
+
+/* Generic lua classifier */
+gboolean lua_classifier_init(struct rspamd_config *cfg,
+ struct ev_loop *ev_base,
+ struct rspamd_classifier *);
+
+gboolean lua_classifier_classify(struct rspamd_classifier *ctx,
+ GPtrArray *tokens,
+ struct rspamd_task *task);
+
+gboolean lua_classifier_learn_spam(struct rspamd_classifier *ctx,
+ GPtrArray *tokens,
+ struct rspamd_task *task,
+ gboolean is_spam,
+ gboolean unlearn,
+ GError **err);
+
+extern gint rspamd_bayes_log_id;
+#define msg_debug_bayes(...) rspamd_conditional_debug_fast(NULL, task->from_addr, \
+ rspamd_bayes_log_id, "bayes", task->task_pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_debug_bayes_cfg(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_bayes_log_id, "bayes", cfg->cfg_pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libstat/classifiers/lua_classifier.c b/src/libstat/classifiers/lua_classifier.c
new file mode 100644
index 0000000..b74330d
--- /dev/null
+++ b/src/libstat/classifiers/lua_classifier.c
@@ -0,0 +1,237 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "classifiers.h"
+#include "cfg_file.h"
+#include "stat_internal.h"
+#include "lua/lua_common.h"
+
+struct rspamd_lua_classifier_ctx {
+ gchar *name;
+ gint classify_ref;
+ gint learn_ref;
+};
+
+static GHashTable *lua_classifiers = NULL;
+
+#define msg_err_luacl(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "luacl", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_luacl(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "luacl", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_luacl(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "luacl", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_debug_luacl(...) rspamd_conditional_debug_fast(NULL, task->from_addr, \
+ rspamd_luacl_log_id, "luacl", task->task_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(luacl)
+
+gboolean
+lua_classifier_init(struct rspamd_config *cfg,
+ struct ev_loop *ev_base,
+ struct rspamd_classifier *cl)
+{
+ struct rspamd_lua_classifier_ctx *ctx;
+ lua_State *L = cl->ctx->cfg->lua_state;
+ gint cb_classify = -1, cb_learn = -1;
+
+ if (lua_classifiers == NULL) {
+ lua_classifiers = g_hash_table_new_full(rspamd_strcase_hash,
+ rspamd_strcase_equal, g_free, g_free);
+ }
+
+ ctx = g_hash_table_lookup(lua_classifiers, cl->subrs->name);
+
+ if (ctx != NULL) {
+ msg_err_config("duplicate lua classifier definition: %s",
+ cl->subrs->name);
+
+ return FALSE;
+ }
+
+ lua_getglobal(L, "rspamd_classifiers");
+ if (lua_type(L, -1) != LUA_TTABLE) {
+ msg_err_config("cannot register classifier %s: no rspamd_classifier global",
+ cl->subrs->name);
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+
+ lua_pushstring(L, cl->subrs->name);
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TTABLE) {
+ msg_err_config("cannot register classifier %s: bad lua type: %s",
+ cl->subrs->name, lua_typename(L, lua_type(L, -1)));
+ lua_pop(L, 2);
+
+ return FALSE;
+ }
+
+ lua_pushstring(L, "classify");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_err_config("cannot register classifier %s: bad lua type for classify: %s",
+ cl->subrs->name, lua_typename(L, lua_type(L, -1)));
+ lua_pop(L, 3);
+
+ return FALSE;
+ }
+
+ cb_classify = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ lua_pushstring(L, "learn");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_err_config("cannot register classifier %s: bad lua type for learn: %s",
+ cl->subrs->name, lua_typename(L, lua_type(L, -1)));
+ lua_pop(L, 3);
+
+ return FALSE;
+ }
+
+ cb_learn = luaL_ref(L, LUA_REGISTRYINDEX);
+ lua_pop(L, 2); /* Table + global */
+
+ ctx = g_malloc0(sizeof(*ctx));
+ ctx->name = g_strdup(cl->subrs->name);
+ ctx->classify_ref = cb_classify;
+ ctx->learn_ref = cb_learn;
+ cl->cfg->flags |= RSPAMD_FLAG_CLASSIFIER_NO_BACKEND;
+ g_hash_table_insert(lua_classifiers, ctx->name, ctx);
+
+ return TRUE;
+}
+gboolean
+lua_classifier_classify(struct rspamd_classifier *cl,
+ GPtrArray *tokens,
+ struct rspamd_task *task)
+{
+ struct rspamd_lua_classifier_ctx *ctx;
+ struct rspamd_task **ptask;
+ struct rspamd_classifier_config **pcfg;
+ lua_State *L;
+ rspamd_token_t *tok;
+ guint i;
+ guint64 v;
+
+ ctx = g_hash_table_lookup(lua_classifiers, cl->subrs->name);
+ g_assert(ctx != NULL);
+ L = task->cfg->lua_state;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, ctx->classify_ref);
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ *pcfg = cl->cfg;
+ rspamd_lua_setclass(L, "rspamd{classifier}", -1);
+
+ lua_createtable(L, tokens->len, 0);
+
+ for (i = 0; i < tokens->len; i++) {
+ tok = g_ptr_array_index(tokens, i);
+ v = tok->data;
+ lua_createtable(L, 3, 0);
+ /* High word, low word, order */
+ lua_pushinteger(L, (guint32) (v >> 32));
+ lua_rawseti(L, -2, 1);
+ lua_pushinteger(L, (guint32) (v));
+ lua_rawseti(L, -2, 2);
+ lua_pushinteger(L, tok->window_idx);
+ lua_rawseti(L, -2, 3);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ if (lua_pcall(L, 3, 0, 0) != 0) {
+ msg_err_luacl("error running classify function for %s: %s", ctx->name,
+ lua_tostring(L, -1));
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
+lua_classifier_learn_spam(struct rspamd_classifier *cl,
+ GPtrArray *tokens,
+ struct rspamd_task *task,
+ gboolean is_spam,
+ gboolean unlearn,
+ GError **err)
+{
+ struct rspamd_lua_classifier_ctx *ctx;
+ struct rspamd_task **ptask;
+ struct rspamd_classifier_config **pcfg;
+ lua_State *L;
+ rspamd_token_t *tok;
+ guint i;
+ guint64 v;
+
+ ctx = g_hash_table_lookup(lua_classifiers, cl->subrs->name);
+ g_assert(ctx != NULL);
+ L = task->cfg->lua_state;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, ctx->learn_ref);
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ *pcfg = cl->cfg;
+ rspamd_lua_setclass(L, "rspamd{classifier}", -1);
+
+ lua_createtable(L, tokens->len, 0);
+
+ for (i = 0; i < tokens->len; i++) {
+ tok = g_ptr_array_index(tokens, i);
+ v = 0;
+ v = tok->data;
+ lua_createtable(L, 3, 0);
+ /* High word, low word, order */
+ lua_pushinteger(L, (guint32) (v >> 32));
+ lua_rawseti(L, -2, 1);
+ lua_pushinteger(L, (guint32) (v));
+ lua_rawseti(L, -2, 2);
+ lua_pushinteger(L, tok->window_idx);
+ lua_rawseti(L, -2, 3);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ lua_pushboolean(L, is_spam);
+ lua_pushboolean(L, unlearn);
+
+ if (lua_pcall(L, 5, 0, 0) != 0) {
+ msg_err_luacl("error running learn function for %s: %s", ctx->name,
+ lua_tostring(L, -1));
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+
+ return TRUE;
+}
diff --git a/src/libstat/learn_cache/learn_cache.h b/src/libstat/learn_cache/learn_cache.h
new file mode 100644
index 0000000..11a66fc
--- /dev/null
+++ b/src/libstat/learn_cache/learn_cache.h
@@ -0,0 +1,79 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef LEARN_CACHE_H_
+#define LEARN_CACHE_H_
+
+#include "config.h"
+#include "ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RSPAMD_DEFAULT_CACHE "sqlite3"
+
+struct rspamd_task;
+struct rspamd_stat_ctx;
+struct rspamd_config;
+struct rspamd_statfile;
+
+struct rspamd_stat_cache {
+ const char *name;
+
+ gpointer (*init)(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct rspamd_statfile *st,
+ const ucl_object_t *cf);
+
+ gpointer (*runtime)(struct rspamd_task *task,
+ gpointer ctx, gboolean learn);
+
+ gint (*check)(struct rspamd_task *task,
+ gboolean is_spam,
+ gpointer runtime);
+
+ gint (*learn)(struct rspamd_task *task,
+ gboolean is_spam,
+ gpointer runtime);
+
+ void (*close)(gpointer ctx);
+
+ gpointer ctx;
+};
+
+#define RSPAMD_STAT_CACHE_DEF(name) \
+ gpointer rspamd_stat_cache_##name##_init(struct rspamd_stat_ctx *ctx, \
+ struct rspamd_config *cfg, \
+ struct rspamd_statfile *st, \
+ const ucl_object_t *cf); \
+ gpointer rspamd_stat_cache_##name##_runtime(struct rspamd_task *task, \
+ gpointer ctx, gboolean learn); \
+ gint rspamd_stat_cache_##name##_check(struct rspamd_task *task, \
+ gboolean is_spam, \
+ gpointer runtime); \
+ gint rspamd_stat_cache_##name##_learn(struct rspamd_task *task, \
+ gboolean is_spam, \
+ gpointer runtime); \
+ void rspamd_stat_cache_##name##_close(gpointer ctx)
+
+RSPAMD_STAT_CACHE_DEF(sqlite3);
+RSPAMD_STAT_CACHE_DEF(redis);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* LEARN_CACHE_H_ */
diff --git a/src/libstat/learn_cache/redis_cache.cxx b/src/libstat/learn_cache/redis_cache.cxx
new file mode 100644
index 0000000..0be56bc
--- /dev/null
+++ b/src/libstat/learn_cache/redis_cache.cxx
@@ -0,0 +1,254 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+// Include early to avoid `extern "C"` issues
+#include "lua/lua_common.h"
+#include "learn_cache.h"
+#include "rspamd.h"
+#include "stat_api.h"
+#include "stat_internal.h"
+#include "cryptobox.h"
+#include "ucl.h"
+#include "libmime/message.h"
+
+#include <memory>
+
+struct rspamd_redis_cache_ctx {
+ lua_State *L;
+ struct rspamd_statfile_config *stcf;
+ int check_ref = -1;
+ int learn_ref = -1;
+
+ rspamd_redis_cache_ctx() = delete;
+ explicit rspamd_redis_cache_ctx(lua_State *L)
+ : L(L)
+ {
+ }
+
+ ~rspamd_redis_cache_ctx()
+ {
+ if (check_ref != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, check_ref);
+ }
+
+ if (learn_ref != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, learn_ref);
+ }
+ }
+};
+
+static void
+rspamd_stat_cache_redis_generate_id(struct rspamd_task *task)
+{
+ rspamd_cryptobox_hash_state_t st;
+ rspamd_cryptobox_hash_init(&st, nullptr, 0);
+
+ const auto *user = (const char *) rspamd_mempool_get_variable(task->task_pool, "stat_user");
+ /* Use dedicated hash space for per users cache */
+ if (user != nullptr) {
+ rspamd_cryptobox_hash_update(&st, (const unsigned char *) user, strlen(user));
+ }
+
+ for (auto i = 0; i < task->tokens->len; i++) {
+ const auto *tok = (rspamd_token_t *) g_ptr_array_index(task->tokens, i);
+ rspamd_cryptobox_hash_update(&st, (const unsigned char *) &tok->data,
+ sizeof(tok->data));
+ }
+
+ guchar out[rspamd_cryptobox_HASHBYTES];
+ rspamd_cryptobox_hash_final(&st, out);
+
+ auto *b32out = rspamd_mempool_alloc_array_type(task->task_pool,
+ sizeof(out) * 8 / 5 + 3, char);
+ auto out_sz = rspamd_encode_base32_buf(out, sizeof(out), b32out,
+ sizeof(out) * 8 / 5 + 2, RSPAMD_BASE32_DEFAULT);
+
+ if (out_sz > 0) {
+ /* Zero terminate */
+ b32out[out_sz] = '\0';
+ rspamd_mempool_set_variable(task->task_pool, "words_hash", b32out, nullptr);
+ }
+}
+
+gpointer
+rspamd_stat_cache_redis_init(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct rspamd_statfile *st,
+ const ucl_object_t *cf)
+{
+ std::unique_ptr<rspamd_redis_cache_ctx> cache_ctx = std::make_unique<rspamd_redis_cache_ctx>(RSPAMD_LUA_CFG_STATE(cfg));
+
+ auto *L = RSPAMD_LUA_CFG_STATE(cfg);
+ lua_settop(L, 0);
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ auto err_idx = lua_gettop(L);
+
+ /* Obtain function */
+ if (!rspamd_lua_require_function(L, "lua_bayes_redis", "lua_bayes_init_cache")) {
+ msg_err_config("cannot require lua_bayes_redis.lua_bayes_init_cache");
+ lua_settop(L, err_idx - 1);
+
+ return nullptr;
+ }
+
+ /* Push arguments */
+ ucl_object_push_lua(L, st->classifier->cfg->opts, false);
+ ucl_object_push_lua(L, st->stcf->opts, false);
+
+ if (lua_pcall(L, 2, 2, err_idx) != 0) {
+ msg_err("call to lua_bayes_init_cache "
+ "script failed: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+
+ return nullptr;
+ }
+
+ /*
+ * Results are in the stack:
+ * top - 1 - check function (idx = -2)
+ * top - learn function (idx = -1)
+ */
+ lua_pushvalue(L, -2);
+ cache_ctx->check_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ lua_pushvalue(L, -1);
+ cache_ctx->learn_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ lua_settop(L, err_idx - 1);
+
+ return (gpointer) cache_ctx.release();
+}
+
+gpointer
+rspamd_stat_cache_redis_runtime(struct rspamd_task *task,
+ gpointer c, gboolean learn)
+{
+ auto *ctx = (struct rspamd_redis_cache_ctx *) c;
+
+ if (task->tokens == nullptr || task->tokens->len == 0) {
+ return nullptr;
+ }
+
+ if (!learn) {
+ /* On check, we produce words_hash variable, on learn it is guaranteed to be set */
+ rspamd_stat_cache_redis_generate_id(task);
+ }
+
+ return (void *) ctx;
+}
+
+static gint
+rspamd_stat_cache_checked(lua_State *L)
+{
+ auto *task = lua_check_task(L, 1);
+ auto res = lua_toboolean(L, 2);
+
+ if (res) {
+ auto val = lua_tointeger(L, 3);
+
+ if ((val > 0 && (task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM)) ||
+ (val <= 0 && (task->flags & RSPAMD_TASK_FLAG_LEARN_HAM))) {
+ /* Already learned */
+ msg_info_task("<%s> has been already "
+ "learned as %s, ignore it",
+ MESSAGE_FIELD(task, message_id),
+ (task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM) ? "spam" : "ham");
+ task->flags |= RSPAMD_TASK_FLAG_ALREADY_LEARNED;
+ }
+ else if (val != 0) {
+ /* Unlearn flag */
+ task->flags |= RSPAMD_TASK_FLAG_UNLEARN;
+ }
+ }
+
+ /* Ignore errors for now, as we can do nothing about them at the moment */
+
+ return 0;
+}
+
+gint rspamd_stat_cache_redis_check(struct rspamd_task *task,
+ gboolean is_spam,
+ gpointer runtime)
+{
+ auto *ctx = (struct rspamd_redis_cache_ctx *) runtime;
+ auto *h = (char *) rspamd_mempool_get_variable(task->task_pool, "words_hash");
+
+ if (h == nullptr) {
+ return RSPAMD_LEARN_IGNORE;
+ }
+
+ auto *L = ctx->L;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(L);
+
+ /* Function arguments */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, ctx->check_ref);
+ rspamd_lua_task_push(L, task);
+ lua_pushstring(L, h);
+
+ lua_pushcclosure(L, &rspamd_stat_cache_checked, 0);
+
+ if (lua_pcall(L, 3, 0, err_idx) != 0) {
+ msg_err_task("call to redis failed: %s", lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+ return RSPAMD_LEARN_IGNORE;
+ }
+
+ /* We need to return OK every time */
+ return RSPAMD_LEARN_OK;
+}
+
+gint rspamd_stat_cache_redis_learn(struct rspamd_task *task,
+ gboolean is_spam,
+ gpointer runtime)
+{
+ auto *ctx = (struct rspamd_redis_cache_ctx *) runtime;
+
+ if (rspamd_session_blocked(task->s)) {
+ return RSPAMD_LEARN_IGNORE;
+ }
+
+ auto *h = (char *) rspamd_mempool_get_variable(task->task_pool, "words_hash");
+ g_assert(h != nullptr);
+ auto *L = ctx->L;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(L);
+
+ /* Function arguments */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, ctx->learn_ref);
+ rspamd_lua_task_push(L, task);
+ lua_pushstring(L, h);
+ lua_pushboolean(L, is_spam);
+
+ if (lua_pcall(L, 3, 0, err_idx) != 0) {
+ msg_err_task("call to redis failed: %s", lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+ return RSPAMD_LEARN_IGNORE;
+ }
+
+ /* We need to return OK every time */
+ return RSPAMD_LEARN_OK;
+}
+
+void rspamd_stat_cache_redis_close(gpointer c)
+{
+ auto *ctx = (struct rspamd_redis_cache_ctx *) c;
+ delete ctx;
+}
diff --git a/src/libstat/learn_cache/sqlite3_cache.c b/src/libstat/learn_cache/sqlite3_cache.c
new file mode 100644
index 0000000..d8ad20a
--- /dev/null
+++ b/src/libstat/learn_cache/sqlite3_cache.c
@@ -0,0 +1,274 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "learn_cache.h"
+#include "rspamd.h"
+#include "stat_api.h"
+#include "stat_internal.h"
+#include "cryptobox.h"
+#include "ucl.h"
+#include "fstring.h"
+#include "message.h"
+#include "libutil/sqlite_utils.h"
+
+static const char *create_tables_sql =
+ ""
+ "CREATE TABLE IF NOT EXISTS learns("
+ "id INTEGER PRIMARY KEY,"
+ "flag INTEGER NOT NULL,"
+ "digest TEXT NOT NULL);"
+ "CREATE UNIQUE INDEX IF NOT EXISTS d ON learns(digest);"
+ "";
+
+#define SQLITE_CACHE_PATH RSPAMD_DBDIR "/learn_cache.sqlite"
+
+enum rspamd_stat_sqlite3_stmt_idx {
+ RSPAMD_STAT_CACHE_TRANSACTION_START_IM = 0,
+ RSPAMD_STAT_CACHE_TRANSACTION_START_DEF,
+ RSPAMD_STAT_CACHE_TRANSACTION_COMMIT,
+ RSPAMD_STAT_CACHE_TRANSACTION_ROLLBACK,
+ RSPAMD_STAT_CACHE_GET_LEARN,
+ RSPAMD_STAT_CACHE_ADD_LEARN,
+ RSPAMD_STAT_CACHE_UPDATE_LEARN,
+ RSPAMD_STAT_CACHE_MAX
+};
+
+static struct rspamd_sqlite3_prstmt prepared_stmts[RSPAMD_STAT_CACHE_MAX] =
+ {
+ {.idx = RSPAMD_STAT_CACHE_TRANSACTION_START_IM,
+ .sql = "BEGIN IMMEDIATE TRANSACTION;",
+ .args = "",
+ .stmt = NULL,
+ .result = SQLITE_DONE,
+ .ret = ""},
+ {.idx = RSPAMD_STAT_CACHE_TRANSACTION_START_DEF,
+ .sql = "BEGIN DEFERRED TRANSACTION;",
+ .args = "",
+ .stmt = NULL,
+ .result = SQLITE_DONE,
+ .ret = ""},
+ {.idx = RSPAMD_STAT_CACHE_TRANSACTION_COMMIT,
+ .sql = "COMMIT;",
+ .args = "",
+ .stmt = NULL,
+ .result = SQLITE_DONE,
+ .ret = ""},
+ {.idx = RSPAMD_STAT_CACHE_TRANSACTION_ROLLBACK,
+ .sql = "ROLLBACK;",
+ .args = "",
+ .stmt = NULL,
+ .result = SQLITE_DONE,
+ .ret = ""},
+ {.idx = RSPAMD_STAT_CACHE_GET_LEARN,
+ .sql = "SELECT flag FROM learns WHERE digest=?1",
+ .args = "V",
+ .stmt = NULL,
+ .result = SQLITE_ROW,
+ .ret = "I"},
+ {.idx = RSPAMD_STAT_CACHE_ADD_LEARN,
+ .sql = "INSERT INTO learns(digest, flag) VALUES (?1, ?2);",
+ .args = "VI",
+ .stmt = NULL,
+ .result = SQLITE_DONE,
+ .ret = ""},
+ {.idx = RSPAMD_STAT_CACHE_UPDATE_LEARN,
+ .sql = "UPDATE learns SET flag=?1 WHERE digest=?2;",
+ .args = "IV",
+ .stmt = NULL,
+ .result = SQLITE_DONE,
+ .ret = ""}};
+
+struct rspamd_stat_sqlite3_ctx {
+ sqlite3 *db;
+ GArray *prstmt;
+};
+
+gpointer
+rspamd_stat_cache_sqlite3_init(struct rspamd_stat_ctx *ctx,
+ struct rspamd_config *cfg,
+ struct rspamd_statfile *st,
+ const ucl_object_t *cf)
+{
+ struct rspamd_stat_sqlite3_ctx *new = NULL;
+ const ucl_object_t *elt;
+ gchar dbpath[PATH_MAX];
+ const gchar *path = SQLITE_CACHE_PATH;
+ sqlite3 *sqlite;
+ GError *err = NULL;
+
+ if (cf) {
+ elt = ucl_object_lookup_any(cf, "path", "file", NULL);
+
+ if (elt != NULL) {
+ path = ucl_object_tostring(elt);
+ }
+ }
+
+ rspamd_snprintf(dbpath, sizeof(dbpath), "%s", path);
+
+ sqlite = rspamd_sqlite3_open_or_create(cfg->cfg_pool,
+ dbpath, create_tables_sql, 0, &err);
+
+ if (sqlite == NULL) {
+ msg_err("cannot open sqlite3 cache: %e", err);
+ g_error_free(err);
+ err = NULL;
+ }
+ else {
+ new = g_malloc0(sizeof(*new));
+ new->db = sqlite;
+ new->prstmt = rspamd_sqlite3_init_prstmt(sqlite, prepared_stmts,
+ RSPAMD_STAT_CACHE_MAX, &err);
+
+ if (new->prstmt == NULL) {
+ msg_err("cannot open sqlite3 cache: %e", err);
+ g_error_free(err);
+ err = NULL;
+ sqlite3_close(sqlite);
+ g_free(new);
+ new = NULL;
+ }
+ }
+
+ return new;
+}
+
+gpointer
+rspamd_stat_cache_sqlite3_runtime(struct rspamd_task *task,
+ gpointer ctx, gboolean learn)
+{
+ /* No need of runtime for this type of classifier */
+ return ctx;
+}
+
+gint rspamd_stat_cache_sqlite3_check(struct rspamd_task *task,
+ gboolean is_spam,
+ gpointer runtime)
+{
+ struct rspamd_stat_sqlite3_ctx *ctx = runtime;
+ rspamd_cryptobox_hash_state_t st;
+ rspamd_token_t *tok;
+ guchar *out;
+ gchar *user = NULL;
+ guint i;
+ gint rc;
+ gint64 flag;
+
+ if (task->tokens == NULL || task->tokens->len == 0) {
+ return RSPAMD_LEARN_IGNORE;
+ }
+
+ if (ctx != NULL && ctx->db != NULL) {
+ out = rspamd_mempool_alloc(task->task_pool, rspamd_cryptobox_HASHBYTES);
+
+ rspamd_cryptobox_hash_init(&st, NULL, 0);
+
+ user = rspamd_mempool_get_variable(task->task_pool, "stat_user");
+ /* Use dedicated hash space for per users cache */
+ if (user != NULL) {
+ rspamd_cryptobox_hash_update(&st, user, strlen(user));
+ }
+
+ for (i = 0; i < task->tokens->len; i++) {
+ tok = g_ptr_array_index(task->tokens, i);
+ rspamd_cryptobox_hash_update(&st, (guchar *) &tok->data,
+ sizeof(tok->data));
+ }
+
+ rspamd_cryptobox_hash_final(&st, out);
+
+ rspamd_sqlite3_run_prstmt(task->task_pool, ctx->db, ctx->prstmt,
+ RSPAMD_STAT_CACHE_TRANSACTION_START_DEF);
+ rc = rspamd_sqlite3_run_prstmt(task->task_pool, ctx->db, ctx->prstmt,
+ RSPAMD_STAT_CACHE_GET_LEARN, (gint64) rspamd_cryptobox_HASHBYTES,
+ out, &flag);
+ rspamd_sqlite3_run_prstmt(task->task_pool, ctx->db, ctx->prstmt,
+ RSPAMD_STAT_CACHE_TRANSACTION_COMMIT);
+
+ /* Save hash into variables */
+ rspamd_mempool_set_variable(task->task_pool, "words_hash", out, NULL);
+
+ if (rc == SQLITE_OK) {
+ /* We have some existing record in the table */
+ if (!!flag == !!is_spam) {
+ /* Already learned */
+ msg_warn_task("already seen stat hash: %*bs",
+ rspamd_cryptobox_HASHBYTES, out);
+ return RSPAMD_LEARN_IGNORE;
+ }
+ else {
+ /* Need to relearn */
+ return RSPAMD_LEARN_UNLEARN;
+ }
+ }
+ }
+
+ return RSPAMD_LEARN_OK;
+}
+
+gint rspamd_stat_cache_sqlite3_learn(struct rspamd_task *task,
+ gboolean is_spam,
+ gpointer runtime)
+{
+ struct rspamd_stat_sqlite3_ctx *ctx = runtime;
+ gboolean unlearn = !!(task->flags & RSPAMD_TASK_FLAG_UNLEARN);
+ guchar *h;
+ gint64 flag;
+
+ h = rspamd_mempool_get_variable(task->task_pool, "words_hash");
+
+ if (h == NULL) {
+ return RSPAMD_LEARN_IGNORE;
+ }
+
+ flag = !!is_spam ? 1 : 0;
+
+ if (!unlearn) {
+ /* Insert result new id */
+ rspamd_sqlite3_run_prstmt(task->task_pool, ctx->db, ctx->prstmt,
+ RSPAMD_STAT_CACHE_TRANSACTION_START_IM);
+ rspamd_sqlite3_run_prstmt(task->task_pool, ctx->db, ctx->prstmt,
+ RSPAMD_STAT_CACHE_ADD_LEARN,
+ (gint64) rspamd_cryptobox_HASHBYTES, h, flag);
+ rspamd_sqlite3_run_prstmt(task->task_pool, ctx->db, ctx->prstmt,
+ RSPAMD_STAT_CACHE_TRANSACTION_COMMIT);
+ }
+ else {
+ rspamd_sqlite3_run_prstmt(task->task_pool, ctx->db, ctx->prstmt,
+ RSPAMD_STAT_CACHE_TRANSACTION_START_IM);
+ rspamd_sqlite3_run_prstmt(task->task_pool, ctx->db, ctx->prstmt,
+ RSPAMD_STAT_CACHE_UPDATE_LEARN,
+ flag,
+ (gint64) rspamd_cryptobox_HASHBYTES, h);
+ rspamd_sqlite3_run_prstmt(task->task_pool, ctx->db, ctx->prstmt,
+ RSPAMD_STAT_CACHE_TRANSACTION_COMMIT);
+ }
+
+ rspamd_sqlite3_sync(ctx->db, NULL, NULL);
+
+ return RSPAMD_LEARN_OK;
+}
+
+void rspamd_stat_cache_sqlite3_close(gpointer c)
+{
+ struct rspamd_stat_sqlite3_ctx *ctx = (struct rspamd_stat_sqlite3_ctx *) c;
+
+ if (ctx != NULL) {
+ rspamd_sqlite3_close_prstmt(ctx->db, ctx->prstmt);
+ sqlite3_close(ctx->db);
+ g_free(ctx);
+ }
+}
diff --git a/src/libstat/stat_api.h b/src/libstat/stat_api.h
new file mode 100644
index 0000000..1badb20
--- /dev/null
+++ b/src/libstat/stat_api.h
@@ -0,0 +1,147 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef STAT_API_H_
+#define STAT_API_H_
+
+#include "config.h"
+#include "task.h"
+#include "lua/lua_common.h"
+#include "contrib/libev/ev.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @file stat_api.h
+ * High level statistics API
+ */
+
+#define RSPAMD_STAT_TOKEN_FLAG_TEXT (1u << 0)
+#define RSPAMD_STAT_TOKEN_FLAG_META (1u << 1)
+#define RSPAMD_STAT_TOKEN_FLAG_LUA_META (1u << 2)
+#define RSPAMD_STAT_TOKEN_FLAG_EXCEPTION (1u << 3)
+#define RSPAMD_STAT_TOKEN_FLAG_HEADER (1u << 4)
+#define RSPAMD_STAT_TOKEN_FLAG_UNIGRAM (1u << 5)
+#define RSPAMD_STAT_TOKEN_FLAG_UTF (1u << 6)
+#define RSPAMD_STAT_TOKEN_FLAG_NORMALISED (1u << 7)
+#define RSPAMD_STAT_TOKEN_FLAG_STEMMED (1u << 8)
+#define RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE (1u << 9)
+#define RSPAMD_STAT_TOKEN_FLAG_STOP_WORD (1u << 10)
+#define RSPAMD_STAT_TOKEN_FLAG_SKIPPED (1u << 11)
+#define RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES (1u << 12)
+#define RSPAMD_STAT_TOKEN_FLAG_EMOJI (1u << 13)
+
+typedef struct rspamd_stat_token_s {
+ rspamd_ftok_t original; /* utf8 raw */
+ rspamd_ftok_unicode_t unicode; /* array of unicode characters, normalized, lowercased */
+ rspamd_ftok_t normalized; /* normalized and lowercased utf8 */
+ rspamd_ftok_t stemmed; /* stemmed utf8 */
+ guint flags;
+} rspamd_stat_token_t;
+
+typedef struct token_node_s {
+ guint64 data;
+ guint window_idx;
+ guint flags;
+ rspamd_stat_token_t *t1;
+ rspamd_stat_token_t *t2;
+ float values[];
+} rspamd_token_t;
+
+struct rspamd_stat_ctx;
+
+/**
+ * The results of statistics processing:
+ * - error
+ * - need to do additional job for processing
+ * - all processed
+ */
+typedef enum rspamd_stat_result_e {
+ RSPAMD_STAT_PROCESS_ERROR = 0,
+ RSPAMD_STAT_PROCESS_DELAYED = 1,
+ RSPAMD_STAT_PROCESS_OK
+} rspamd_stat_result_t;
+
+/**
+ * Initialise statistics modules
+ * @param cfg
+ */
+void rspamd_stat_init(struct rspamd_config *cfg, struct ev_loop *ev_base);
+
+/**
+ * Finalize statistics
+ */
+void rspamd_stat_close(void);
+
+/**
+ * Tokenize task
+ * @param st_ctx
+ * @param task
+ */
+void rspamd_stat_process_tokenize(struct rspamd_stat_ctx *st_ctx,
+ struct rspamd_task *task);
+
+/**
+ * Classify the task specified and insert symbols if needed
+ * @param task
+ * @param L lua state
+ * @param err error returned
+ * @return TRUE if task has been classified
+ */
+rspamd_stat_result_t rspamd_stat_classify(struct rspamd_task *task,
+ lua_State *L, guint stage, GError **err);
+
+
+/**
+ * Check if a task should be learned and set the appropriate flags for it
+ * @param task
+ * @return
+ */
+gboolean rspamd_stat_check_autolearn(struct rspamd_task *task);
+
+/**
+ * Learn task as spam or ham, task must be processed prior to this call
+ * @param task task to learn
+ * @param spam if TRUE learn spam, otherwise learn ham
+ * @param L lua state
+ * @param classifier NULL to learn all classifiers, name to learn a specific one
+ * @param err error returned
+ * @return TRUE if task has been learned
+ */
+rspamd_stat_result_t rspamd_stat_learn(struct rspamd_task *task,
+ gboolean spam, lua_State *L, const gchar *classifier,
+ guint stage,
+ GError **err);
+
+/**
+ * Get the overall statistics for all statfile backends
+ * @param cfg configuration
+ * @param total_learns the total number of learns is stored here
+ * @return array of statistical information
+ */
+rspamd_stat_result_t rspamd_stat_statistics(struct rspamd_task *task,
+ struct rspamd_config *cfg,
+ guint64 *total_learns,
+ ucl_object_t **res);
+
+void rspamd_stat_unload(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* STAT_API_H_ */
diff --git a/src/libstat/stat_config.c b/src/libstat/stat_config.c
new file mode 100644
index 0000000..2748044
--- /dev/null
+++ b/src/libstat/stat_config.c
@@ -0,0 +1,603 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "stat_api.h"
+#include "rspamd.h"
+#include "cfg_rcl.h"
+#include "stat_internal.h"
+#include "lua/lua_common.h"
+
+static struct rspamd_stat_ctx *stat_ctx = NULL;
+
+static struct rspamd_stat_classifier lua_classifier = {
+ .name = "lua",
+ .init_func = lua_classifier_init,
+ .classify_func = lua_classifier_classify,
+ .learn_spam_func = lua_classifier_learn_spam,
+ .fin_func = NULL,
+};
+
+static struct rspamd_stat_classifier stat_classifiers[] = {
+ {
+ .name = "bayes",
+ .init_func = bayes_init,
+ .classify_func = bayes_classify,
+ .learn_spam_func = bayes_learn_spam,
+ .fin_func = bayes_fin,
+ }};
+
+static struct rspamd_stat_tokenizer stat_tokenizers[] = {
+ {
+ .name = "osb-text",
+ .get_config = rspamd_tokenizer_osb_get_config,
+ .tokenize_func = rspamd_tokenizer_osb,
+ },
+ {
+ .name = "osb",
+ .get_config = rspamd_tokenizer_osb_get_config,
+ .tokenize_func = rspamd_tokenizer_osb,
+ },
+};
+
+#define RSPAMD_STAT_BACKEND_ELT(nam, eltn) \
+ { \
+ .name = #nam, \
+ .read_only = false, \
+ .init = rspamd_##eltn##_init, \
+ .runtime = rspamd_##eltn##_runtime, \
+ .process_tokens = rspamd_##eltn##_process_tokens, \
+ .finalize_process = rspamd_##eltn##_finalize_process, \
+ .learn_tokens = rspamd_##eltn##_learn_tokens, \
+ .finalize_learn = rspamd_##eltn##_finalize_learn, \
+ .total_learns = rspamd_##eltn##_total_learns, \
+ .inc_learns = rspamd_##eltn##_inc_learns, \
+ .dec_learns = rspamd_##eltn##_dec_learns, \
+ .get_stat = rspamd_##eltn##_get_stat, \
+ .load_tokenizer_config = rspamd_##eltn##_load_tokenizer_config, \
+ .close = rspamd_##eltn##_close \
+ }
+#define RSPAMD_STAT_BACKEND_ELT_READONLY(nam, eltn) \
+ { \
+ .name = #nam, \
+ .read_only = true, \
+ .init = rspamd_##eltn##_init, \
+ .runtime = rspamd_##eltn##_runtime, \
+ .process_tokens = rspamd_##eltn##_process_tokens, \
+ .finalize_process = rspamd_##eltn##_finalize_process, \
+ .learn_tokens = NULL, \
+ .finalize_learn = NULL, \
+ .total_learns = rspamd_##eltn##_total_learns, \
+ .inc_learns = NULL, \
+ .dec_learns = NULL, \
+ .get_stat = rspamd_##eltn##_get_stat, \
+ .load_tokenizer_config = rspamd_##eltn##_load_tokenizer_config, \
+ .close = rspamd_##eltn##_close \
+ }
+
+static struct rspamd_stat_backend stat_backends[] = {
+ RSPAMD_STAT_BACKEND_ELT(mmap, mmaped_file),
+ RSPAMD_STAT_BACKEND_ELT(sqlite3, sqlite3),
+ RSPAMD_STAT_BACKEND_ELT_READONLY(cdb, cdb),
+ RSPAMD_STAT_BACKEND_ELT(redis, redis)};
+
+#define RSPAMD_STAT_CACHE_ELT(nam, eltn) \
+ { \
+ .name = #nam, \
+ .init = rspamd_stat_cache_##eltn##_init, \
+ .runtime = rspamd_stat_cache_##eltn##_runtime, \
+ .check = rspamd_stat_cache_##eltn##_check, \
+ .learn = rspamd_stat_cache_##eltn##_learn, \
+ .close = rspamd_stat_cache_##eltn##_close \
+ }
+
+static struct rspamd_stat_cache stat_caches[] = {
+ RSPAMD_STAT_CACHE_ELT(sqlite3, sqlite3),
+ RSPAMD_STAT_CACHE_ELT(redis, redis),
+};
+
+void rspamd_stat_init(struct rspamd_config *cfg, struct ev_loop *ev_base)
+{
+ GList *cur, *curst;
+ struct rspamd_classifier_config *clf;
+ struct rspamd_statfile_config *stf;
+ struct rspamd_stat_backend *bk;
+ struct rspamd_statfile *st;
+ struct rspamd_classifier *cl;
+ const ucl_object_t *cache_obj = NULL, *cache_name_obj;
+ const gchar *cache_name = NULL;
+ lua_State *L = cfg->lua_state;
+ guint lua_classifiers_cnt = 0, i;
+ gboolean skip_cache = FALSE;
+
+ if (stat_ctx == NULL) {
+ stat_ctx = g_malloc0(sizeof(*stat_ctx));
+ }
+
+ lua_getglobal(L, "rspamd_classifiers");
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ lua_classifiers_cnt++;
+ lua_pop(L, 1);
+ }
+ }
+
+ lua_pop(L, 1);
+
+ stat_ctx->classifiers_count = G_N_ELEMENTS(stat_classifiers) +
+ lua_classifiers_cnt;
+ stat_ctx->classifiers_subrs = g_new0(struct rspamd_stat_classifier,
+ stat_ctx->classifiers_count);
+
+ for (i = 0; i < G_N_ELEMENTS(stat_classifiers); i++) {
+ memcpy(&stat_ctx->classifiers_subrs[i], &stat_classifiers[i],
+ sizeof(struct rspamd_stat_classifier));
+ }
+
+ lua_getglobal(L, "rspamd_classifiers");
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ lua_pushvalue(L, -2);
+ memcpy(&stat_ctx->classifiers_subrs[i], &lua_classifier,
+ sizeof(struct rspamd_stat_classifier));
+ stat_ctx->classifiers_subrs[i].name = g_strdup(lua_tostring(L, -1));
+ i++;
+ lua_pop(L, 2);
+ }
+ }
+
+ lua_pop(L, 1);
+ stat_ctx->backends_subrs = stat_backends;
+ stat_ctx->backends_count = G_N_ELEMENTS(stat_backends);
+
+ stat_ctx->tokenizers_subrs = stat_tokenizers;
+ stat_ctx->tokenizers_count = G_N_ELEMENTS(stat_tokenizers);
+ stat_ctx->caches_subrs = stat_caches;
+ stat_ctx->caches_count = G_N_ELEMENTS(stat_caches);
+ stat_ctx->cfg = cfg;
+ stat_ctx->statfiles = g_ptr_array_new();
+ stat_ctx->classifiers = g_ptr_array_new();
+ stat_ctx->async_elts = g_queue_new();
+ stat_ctx->event_loop = ev_base;
+ stat_ctx->lua_stat_tokens_ref = -1;
+
+ /* Interact with lua_stat */
+ if (luaL_dostring(L, "return require \"lua_stat\"") != 0) {
+ msg_err_config("cannot require lua_stat: %s",
+ lua_tostring(L, -1));
+ }
+ else {
+#if LUA_VERSION_NUM >= 504
+ lua_settop(L, -2);
+#endif
+ if (lua_type(L, -1) != LUA_TTABLE) {
+ msg_err_config("lua stat must return "
+ "table and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ else {
+ lua_pushstring(L, "gen_stat_tokens");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_err_config("gen_stat_tokens must return "
+ "function and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ else {
+ /* Call this function to obtain closure */
+ gint err_idx, ret;
+ struct rspamd_config **pcfg;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_pushvalue(L, err_idx - 1);
+
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ *pcfg = cfg;
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+
+ if ((ret = lua_pcall(L, 1, 1, err_idx)) != 0) {
+ msg_err_config("call to gen_stat_tokens lua "
+ "script failed (%d): %s",
+ ret,
+ lua_tostring(L, -1));
+ }
+ else {
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_err_config("gen_stat_tokens invocation must return "
+ "function and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ else {
+ stat_ctx->lua_stat_tokens_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ }
+ }
+ }
+ }
+
+ /* Cleanup mess */
+ lua_settop(L, 0);
+
+ /* Create statfiles from the classifiers */
+ cur = cfg->classifiers;
+
+ while (cur) {
+ bk = NULL;
+ clf = cur->data;
+ cl = g_malloc0(sizeof(*cl));
+ cl->cfg = clf;
+ cl->ctx = stat_ctx;
+ cl->statfiles_ids = g_array_new(FALSE, FALSE, sizeof(gint));
+ cl->subrs = rspamd_stat_get_classifier(clf->classifier);
+
+ if (cl->subrs == NULL) {
+ g_free(cl);
+ msg_err_config("cannot init classifier type %s", clf->name);
+ cur = g_list_next(cur);
+ continue;
+ }
+
+ if (!cl->subrs->init_func(cfg, ev_base, cl)) {
+ g_free(cl);
+ msg_err_config("cannot init classifier type %s", clf->name);
+ cur = g_list_next(cur);
+ continue;
+ }
+
+ if (!(clf->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND)) {
+ bk = rspamd_stat_get_backend(clf->backend);
+
+ if (bk == NULL) {
+ msg_err_config("cannot get backend of type %s, so disable classifier"
+ " %s completely",
+ clf->backend, clf->name);
+ cur = g_list_next(cur);
+ continue;
+ }
+ }
+ else {
+ /* This actually is not implemented so it should never happen */
+ g_free(cl);
+ cur = g_list_next(cur);
+ continue;
+ }
+
+ /* XXX:
+ * Here we get the first classifier tokenizer config as the only one
+ * We NO LONGER support multiple tokenizers per rspamd instance
+ */
+ if (stat_ctx->tkcf == NULL) {
+ stat_ctx->tokenizer = rspamd_stat_get_tokenizer(clf->tokenizer->name);
+ g_assert(stat_ctx->tokenizer != NULL);
+ stat_ctx->tkcf = stat_ctx->tokenizer->get_config(cfg->cfg_pool,
+ clf->tokenizer, NULL);
+ }
+
+ /* Init classifier cache */
+ cache_name = NULL;
+
+ if (!bk->read_only) {
+ if (clf->opts) {
+ cache_obj = ucl_object_lookup(clf->opts, "cache");
+ cache_name_obj = NULL;
+
+ if (cache_obj && ucl_object_type(cache_obj) == UCL_NULL) {
+ skip_cache = TRUE;
+ }
+ else {
+ if (cache_obj) {
+ cache_name_obj = ucl_object_lookup_any(cache_obj,
+ "name", "type", NULL);
+ }
+
+ if (cache_name_obj) {
+ cache_name = ucl_object_tostring(cache_name_obj);
+ }
+ }
+ }
+ }
+ else {
+ skip_cache = true;
+ }
+
+ if (cache_name == NULL && !skip_cache) {
+ /* We assume that learn cache is the same as backend */
+ cache_name = clf->backend;
+ }
+
+ curst = clf->statfiles;
+
+ while (curst) {
+ stf = curst->data;
+ st = g_malloc0(sizeof(*st));
+ st->classifier = cl;
+ st->stcf = stf;
+
+ if (!(cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND)) {
+ st->backend = bk;
+ st->bkcf = bk->init(stat_ctx, cfg, st);
+ msg_info_config("added backend %s for symbol %s",
+ bk->name, stf->symbol);
+ }
+ else {
+ msg_debug_config("added backend-less statfile for symbol %s",
+ stf->symbol);
+ }
+
+ /* XXX: bad hack to pass statfiles configuration to cache */
+ if (cl->cache == NULL && !skip_cache) {
+ cl->cache = rspamd_stat_get_cache(cache_name);
+ g_assert(cl->cache != NULL);
+ cl->cachecf = cl->cache->init(stat_ctx, cfg, st, cache_obj);
+
+ if (cl->cachecf == NULL) {
+ msg_err_config("error adding cache %s for symbol %s",
+ cl->cache->name, stf->symbol);
+ cl->cache = NULL;
+ }
+ else {
+ msg_debug_config("added cache %s for symbol %s",
+ cl->cache->name, stf->symbol);
+ }
+ }
+
+ if (st->bkcf == NULL &&
+ !(cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND)) {
+ msg_err_config("cannot init backend %s for statfile %s",
+ clf->backend, stf->symbol);
+
+ g_free(st);
+ }
+ else {
+ st->id = stat_ctx->statfiles->len;
+ g_ptr_array_add(stat_ctx->statfiles, st);
+ g_array_append_val(cl->statfiles_ids, st->id);
+ }
+
+ curst = curst->next;
+ }
+
+ g_ptr_array_add(stat_ctx->classifiers, cl);
+
+ cur = cur->next;
+ }
+}
+
+void rspamd_stat_close(void)
+{
+ struct rspamd_classifier *cl;
+ struct rspamd_statfile *st;
+ struct rspamd_stat_ctx *st_ctx;
+ struct rspamd_stat_async_elt *aelt;
+ GList *cur;
+ guint i, j;
+ gint id;
+
+ st_ctx = rspamd_stat_get_ctx();
+ g_assert(st_ctx != NULL);
+
+ for (i = 0; i < st_ctx->classifiers->len; i++) {
+ cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+ for (j = 0; j < cl->statfiles_ids->len; j++) {
+ id = g_array_index(cl->statfiles_ids, gint, j);
+ st = g_ptr_array_index(st_ctx->statfiles, id);
+ if (!(st->classifier->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND)) {
+ st->backend->close(st->bkcf);
+ }
+
+ g_free(st);
+ }
+
+ if (cl->cache && cl->cachecf) {
+ cl->cache->close(cl->cachecf);
+ }
+
+ g_array_free(cl->statfiles_ids, TRUE);
+
+ if (cl->subrs->fin_func) {
+ cl->subrs->fin_func(cl);
+ }
+
+ g_free(cl);
+ }
+
+ cur = st_ctx->async_elts->head;
+
+ while (cur) {
+ aelt = cur->data;
+ REF_RELEASE(aelt);
+ cur = g_list_next(cur);
+ }
+
+ g_queue_free(stat_ctx->async_elts);
+ g_ptr_array_free(st_ctx->statfiles, TRUE);
+ g_ptr_array_free(st_ctx->classifiers, TRUE);
+
+ if (st_ctx->lua_stat_tokens_ref != -1) {
+ luaL_unref(st_ctx->cfg->lua_state, LUA_REGISTRYINDEX,
+ st_ctx->lua_stat_tokens_ref);
+ }
+
+ g_free(st_ctx->classifiers_subrs);
+ g_free(st_ctx);
+
+ /* Set global var to NULL */
+ stat_ctx = NULL;
+}
+
+struct rspamd_stat_ctx *
+rspamd_stat_get_ctx(void)
+{
+ return stat_ctx;
+}
+
+struct rspamd_stat_classifier *
+rspamd_stat_get_classifier(const gchar *name)
+{
+ guint i;
+
+ if (name == NULL || name[0] == '\0') {
+ name = RSPAMD_DEFAULT_CLASSIFIER;
+ }
+
+ for (i = 0; i < stat_ctx->classifiers_count; i++) {
+ if (strcmp(name, stat_ctx->classifiers_subrs[i].name) == 0) {
+ return &stat_ctx->classifiers_subrs[i];
+ }
+ }
+
+ msg_err("cannot find classifier named %s", name);
+
+ return NULL;
+}
+
+struct rspamd_stat_backend *
+rspamd_stat_get_backend(const gchar *name)
+{
+ guint i;
+
+ if (name == NULL || name[0] == '\0') {
+ name = RSPAMD_DEFAULT_BACKEND;
+ }
+
+ for (i = 0; i < stat_ctx->backends_count; i++) {
+ if (strcmp(name, stat_ctx->backends_subrs[i].name) == 0) {
+ return &stat_ctx->backends_subrs[i];
+ }
+ }
+
+ msg_err("cannot find backend named %s", name);
+
+ return NULL;
+}
+
+struct rspamd_stat_tokenizer *
+rspamd_stat_get_tokenizer(const gchar *name)
+{
+ guint i;
+
+ if (name == NULL || name[0] == '\0') {
+ name = RSPAMD_DEFAULT_TOKENIZER;
+ }
+
+ for (i = 0; i < stat_ctx->tokenizers_count; i++) {
+ if (strcmp(name, stat_ctx->tokenizers_subrs[i].name) == 0) {
+ return &stat_ctx->tokenizers_subrs[i];
+ }
+ }
+
+ msg_err("cannot find tokenizer named %s", name);
+
+ return NULL;
+}
+
+struct rspamd_stat_cache *
+rspamd_stat_get_cache(const gchar *name)
+{
+ guint i;
+
+ if (name == NULL || name[0] == '\0') {
+ name = RSPAMD_DEFAULT_CACHE;
+ }
+
+ for (i = 0; i < stat_ctx->caches_count; i++) {
+ if (strcmp(name, stat_ctx->caches_subrs[i].name) == 0) {
+ return &stat_ctx->caches_subrs[i];
+ }
+ }
+
+ msg_err("cannot find cache named %s", name);
+
+ return NULL;
+}
+
+static void
+rspamd_async_elt_dtor(struct rspamd_stat_async_elt *elt)
+{
+ if (elt->cleanup) {
+ elt->cleanup(elt, elt->ud);
+ }
+
+ ev_timer_stop(elt->event_loop, &elt->timer_ev);
+ g_free(elt);
+}
+
+static void
+rspamd_async_elt_on_timer(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_stat_async_elt *elt = (struct rspamd_stat_async_elt *) w->data;
+ gdouble jittered_time;
+
+
+ if (elt->enabled) {
+ elt->handler(elt, elt->ud);
+ }
+
+ jittered_time = rspamd_time_jitter(elt->timeout, 0);
+ elt->timer_ev.repeat = jittered_time;
+ ev_timer_again(EV_A_ w);
+}
+
+struct rspamd_stat_async_elt *
+rspamd_stat_ctx_register_async(rspamd_stat_async_handler handler,
+ rspamd_stat_async_cleanup cleanup,
+ gpointer d,
+ gdouble timeout)
+{
+ struct rspamd_stat_async_elt *elt;
+ struct rspamd_stat_ctx *st_ctx;
+
+ st_ctx = rspamd_stat_get_ctx();
+ g_assert(st_ctx != NULL);
+
+ elt = g_malloc0(sizeof(*elt));
+ elt->handler = handler;
+ elt->cleanup = cleanup;
+ elt->ud = d;
+ elt->timeout = timeout;
+ elt->event_loop = st_ctx->event_loop;
+ REF_INIT_RETAIN(elt, rspamd_async_elt_dtor);
+ /* Enabled by default */
+
+
+ if (st_ctx->event_loop) {
+ elt->enabled = TRUE;
+ /*
+ * First we set timeval to zero as we want cb to be executed as
+ * fast as possible
+ */
+ elt->timer_ev.data = elt;
+ ev_timer_init(&elt->timer_ev, rspamd_async_elt_on_timer,
+ 0.1, 0.0);
+ ev_timer_start(st_ctx->event_loop, &elt->timer_ev);
+ }
+ else {
+ elt->enabled = FALSE;
+ }
+
+ g_queue_push_tail(st_ctx->async_elts, elt);
+
+ return elt;
+}
diff --git a/src/libstat/stat_internal.h b/src/libstat/stat_internal.h
new file mode 100644
index 0000000..8d0ebd4
--- /dev/null
+++ b/src/libstat/stat_internal.h
@@ -0,0 +1,134 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef STAT_INTERNAL_H_
+#define STAT_INTERNAL_H_
+
+#include "config.h"
+#include "task.h"
+#include "ref.h"
+#include "classifiers/classifiers.h"
+#include "tokenizers/tokenizers.h"
+#include "backends/backends.h"
+#include "learn_cache/learn_cache.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_statfile_runtime {
+ struct rspamd_statfile_config *st;
+ gpointer backend_runtime;
+ guint64 hits;
+ guint64 total_hits;
+};
+
+/* Common classifier structure */
+struct rspamd_classifier {
+ struct rspamd_stat_ctx *ctx;
+ GArray *statfiles_ids; /* int */
+ struct rspamd_stat_cache *cache;
+ gpointer cachecf;
+ gulong spam_learns;
+ gulong ham_learns;
+ gint autolearn_cbref;
+ struct rspamd_classifier_config *cfg;
+ struct rspamd_stat_classifier *subrs;
+ gpointer specific;
+};
+
+struct rspamd_statfile {
+ gint id;
+ struct rspamd_statfile_config *stcf;
+ struct rspamd_classifier *classifier;
+ struct rspamd_stat_backend *backend;
+ gpointer bkcf;
+};
+
+struct rspamd_stat_async_elt;
+
+typedef void (*rspamd_stat_async_handler)(struct rspamd_stat_async_elt *elt,
+ gpointer ud);
+
+typedef void (*rspamd_stat_async_cleanup)(struct rspamd_stat_async_elt *elt,
+ gpointer ud);
+
+struct rspamd_stat_async_elt {
+ rspamd_stat_async_handler handler;
+ rspamd_stat_async_cleanup cleanup;
+ struct ev_loop *event_loop;
+ ev_timer timer_ev;
+ gdouble timeout;
+ gboolean enabled;
+ gpointer ud;
+ ref_entry_t ref;
+};
+
+struct rspamd_stat_ctx {
+ /* Subroutines for all objects */
+ struct rspamd_stat_classifier *classifiers_subrs;
+ guint classifiers_count;
+ struct rspamd_stat_tokenizer *tokenizers_subrs;
+ guint tokenizers_count;
+ struct rspamd_stat_backend *backends_subrs;
+ guint backends_count;
+ struct rspamd_stat_cache *caches_subrs;
+ guint caches_count;
+
+ /* Runtime configuration */
+ GPtrArray *statfiles; /* struct rspamd_statfile */
+ GPtrArray *classifiers; /* struct rspamd_classifier */
+ GQueue *async_elts; /* struct rspamd_stat_async_elt */
+ struct rspamd_config *cfg;
+
+ gint lua_stat_tokens_ref;
+
+ /* Global tokenizer */
+ struct rspamd_stat_tokenizer *tokenizer;
+ gpointer tkcf;
+
+ struct ev_loop *event_loop;
+};
+
+typedef enum rspamd_learn_cache_result {
+ RSPAMD_LEARN_OK = 0,
+ RSPAMD_LEARN_UNLEARN,
+ RSPAMD_LEARN_IGNORE
+} rspamd_learn_t;
+
+struct rspamd_stat_ctx *rspamd_stat_get_ctx(void);
+
+struct rspamd_stat_classifier *rspamd_stat_get_classifier(const gchar *name);
+
+struct rspamd_stat_backend *rspamd_stat_get_backend(const gchar *name);
+
+struct rspamd_stat_tokenizer *rspamd_stat_get_tokenizer(const gchar *name);
+
+struct rspamd_stat_cache *rspamd_stat_get_cache(const gchar *name);
+
+struct rspamd_stat_async_elt *rspamd_stat_ctx_register_async(
+ rspamd_stat_async_handler handler, rspamd_stat_async_cleanup cleanup,
+ gpointer d, gdouble timeout);
+
+static GQuark rspamd_stat_quark(void)
+{
+ return g_quark_from_static_string("rspamd-statistics");
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* STAT_INTERNAL_H_ */
diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c
new file mode 100644
index 0000000..8c1d8ff
--- /dev/null
+++ b/src/libstat/stat_process.c
@@ -0,0 +1,1250 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "stat_api.h"
+#include "rspamd.h"
+#include "stat_internal.h"
+#include "libmime/message.h"
+#include "libmime/images.h"
+#include "libserver/html/html.h"
+#include "lua/lua_common.h"
+#include "libserver/mempool_vars_internal.h"
+#include "utlist.h"
+#include <math.h>
+
+#define RSPAMD_CLASSIFY_OP 0
+#define RSPAMD_LEARN_OP 1
+#define RSPAMD_UNLEARN_OP 2
+
+static const gdouble similarity_threshold = 80.0;
+
+static void
+rspamd_stat_tokenize_parts_metadata(struct rspamd_stat_ctx *st_ctx,
+ struct rspamd_task *task)
+{
+ GArray *ar;
+ rspamd_stat_token_t elt;
+ guint i;
+ lua_State *L = task->cfg->lua_state;
+
+ ar = g_array_sized_new(FALSE, FALSE, sizeof(elt), 16);
+ memset(&elt, 0, sizeof(elt));
+ elt.flags = RSPAMD_STAT_TOKEN_FLAG_META;
+
+ if (st_ctx->lua_stat_tokens_ref != -1) {
+ gint err_idx, ret;
+ struct rspamd_task **ptask;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, st_ctx->lua_stat_tokens_ref);
+
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ if ((ret = lua_pcall(L, 1, 1, err_idx)) != 0) {
+ msg_err_task("call to stat_tokens lua "
+ "script failed (%d): %s",
+ ret, lua_tostring(L, -1));
+ }
+ else {
+ if (lua_type(L, -1) != LUA_TTABLE) {
+ msg_err_task("stat_tokens invocation must return "
+ "table and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ else {
+ guint vlen;
+ rspamd_ftok_t tok;
+
+ vlen = rspamd_lua_table_size(L, -1);
+
+ for (i = 0; i < vlen; i++) {
+ lua_rawgeti(L, -1, i + 1);
+ tok.begin = lua_tolstring(L, -1, &tok.len);
+
+ if (tok.begin && tok.len > 0) {
+ elt.original.begin =
+ rspamd_mempool_ftokdup(task->task_pool, &tok);
+ elt.original.len = tok.len;
+ elt.stemmed.begin = elt.original.begin;
+ elt.stemmed.len = elt.original.len;
+ elt.normalized.begin = elt.original.begin;
+ elt.normalized.len = elt.original.len;
+
+ g_array_append_val(ar, elt);
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ }
+
+ lua_settop(L, 0);
+ }
+
+
+ if (ar->len > 0) {
+ st_ctx->tokenizer->tokenize_func(st_ctx,
+ task,
+ ar,
+ TRUE,
+ "M",
+ task->tokens);
+ }
+
+ rspamd_mempool_add_destructor(task->task_pool,
+ rspamd_array_free_hard, ar);
+}
+
+/*
+ * Tokenize task using the tokenizer specified
+ */
+void rspamd_stat_process_tokenize(struct rspamd_stat_ctx *st_ctx,
+ struct rspamd_task *task)
+{
+ struct rspamd_mime_text_part *part;
+ rspamd_cryptobox_hash_state_t hst;
+ rspamd_token_t *st_tok;
+ guint i, reserved_len = 0;
+ gdouble *pdiff;
+ guchar hout[rspamd_cryptobox_HASHBYTES];
+ gchar *b32_hout;
+
+ if (st_ctx == NULL) {
+ st_ctx = rspamd_stat_get_ctx();
+ }
+
+ g_assert(st_ctx != NULL);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, part)
+ {
+ if (!IS_TEXT_PART_EMPTY(part) && part->utf_words != NULL) {
+ reserved_len += part->utf_words->len;
+ }
+ /* XXX: normal window size */
+ reserved_len += 5;
+ }
+
+ task->tokens = g_ptr_array_sized_new(reserved_len);
+ rspamd_mempool_add_destructor(task->task_pool,
+ rspamd_ptr_array_free_hard, task->tokens);
+ rspamd_mempool_notify_alloc(task->task_pool, reserved_len * sizeof(gpointer));
+ pdiff = rspamd_mempool_get_variable(task->task_pool, "parts_distance");
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, part)
+ {
+ if (!IS_TEXT_PART_EMPTY(part) && part->utf_words != NULL) {
+ st_ctx->tokenizer->tokenize_func(st_ctx, task,
+ part->utf_words, IS_TEXT_PART_UTF(part),
+ NULL, task->tokens);
+ }
+
+
+ if (pdiff != NULL && (1.0 - *pdiff) * 100.0 > similarity_threshold) {
+ msg_debug_bayes("message has two common parts (%.2f), so skip the last one",
+ *pdiff);
+ break;
+ }
+ }
+
+ if (task->meta_words != NULL) {
+ st_ctx->tokenizer->tokenize_func(st_ctx,
+ task,
+ task->meta_words,
+ TRUE,
+ "SUBJECT",
+ task->tokens);
+ }
+
+ rspamd_stat_tokenize_parts_metadata(st_ctx, task);
+
+ /* Produce signature */
+ rspamd_cryptobox_hash_init(&hst, NULL, 0);
+
+ PTR_ARRAY_FOREACH(task->tokens, i, st_tok)
+ {
+ rspamd_cryptobox_hash_update(&hst, (guchar *) &st_tok->data,
+ sizeof(st_tok->data));
+ }
+
+ rspamd_cryptobox_hash_final(&hst, hout);
+ b32_hout = rspamd_encode_base32(hout, sizeof(hout), RSPAMD_BASE32_DEFAULT);
+ /*
+ * We need to strip it to 32 characters providing ~160 bits of
+ * hash distribution
+ */
+ b32_hout[32] = '\0';
+ rspamd_mempool_set_variable(task->task_pool, RSPAMD_MEMPOOL_STAT_SIGNATURE,
+ b32_hout, g_free);
+}
+
+static gboolean
+rspamd_stat_classifier_is_skipped(struct rspamd_task *task,
+ struct rspamd_classifier *cl, gboolean is_learn, gboolean is_spam)
+{
+ GList *cur = is_learn ? cl->cfg->learn_conditions : cl->cfg->classify_conditions;
+ lua_State *L = task->cfg->lua_state;
+ gboolean ret = FALSE;
+
+ while (cur) {
+ gint cb_ref = GPOINTER_TO_INT(cur->data);
+ gint old_top = lua_gettop(L);
+ gint nargs;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cb_ref);
+ /* Push task and two booleans: is_spam and is_unlearn */
+ struct rspamd_task **ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ if (is_learn) {
+ lua_pushboolean(L, is_spam);
+ lua_pushboolean(L,
+ task->flags & RSPAMD_TASK_FLAG_UNLEARN ? true : false);
+ nargs = 3;
+ }
+ else {
+ nargs = 1;
+ }
+
+ if (lua_pcall(L, nargs, LUA_MULTRET, 0) != 0) {
+ msg_err_task("call to %s failed: %s",
+ "condition callback",
+ lua_tostring(L, -1));
+ }
+ else {
+ if (lua_isboolean(L, 1)) {
+ if (!lua_toboolean(L, 1)) {
+ ret = TRUE;
+ }
+ }
+
+ if (lua_isstring(L, 2)) {
+ if (ret) {
+ msg_notice_task("%s condition for classifier %s returned: %s; skip classifier",
+ is_learn ? "learn" : "classify", cl->cfg->name,
+ lua_tostring(L, 2));
+ }
+ else {
+ msg_info_task("%s condition for classifier %s returned: %s",
+ is_learn ? "learn" : "classify", cl->cfg->name,
+ lua_tostring(L, 2));
+ }
+ }
+ else if (ret) {
+ msg_notice_task("%s condition for classifier %s returned false; skip classifier",
+ is_learn ? "learn" : "classify", cl->cfg->name);
+ }
+
+ if (ret) {
+ lua_settop(L, old_top);
+ break;
+ }
+ }
+
+ lua_settop(L, old_top);
+ cur = g_list_next(cur);
+ }
+
+ return ret;
+}
+
+static void
+rspamd_stat_preprocess(struct rspamd_stat_ctx *st_ctx,
+ struct rspamd_task *task, gboolean is_learn, gboolean is_spam)
+{
+ guint i;
+ struct rspamd_statfile *st;
+ gpointer bk_run;
+
+ if (task->tokens == NULL) {
+ rspamd_stat_process_tokenize(st_ctx, task);
+ }
+
+ task->stat_runtimes = g_ptr_array_sized_new(st_ctx->statfiles->len);
+ g_ptr_array_set_size(task->stat_runtimes, st_ctx->statfiles->len);
+ rspamd_mempool_add_destructor(task->task_pool,
+ rspamd_ptr_array_free_hard, task->stat_runtimes);
+
+ /* Temporary set all stat_runtimes to some max size to distinguish from NULL */
+ for (i = 0; i < st_ctx->statfiles->len; i++) {
+ g_ptr_array_index(task->stat_runtimes, i) = GSIZE_TO_POINTER(G_MAXSIZE);
+ }
+
+ for (i = 0; i < st_ctx->classifiers->len; i++) {
+ struct rspamd_classifier *cl = g_ptr_array_index(st_ctx->classifiers, i);
+ gboolean skip_classifier = FALSE;
+
+ if (cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND) {
+ skip_classifier = TRUE;
+ }
+ else {
+ if (rspamd_stat_classifier_is_skipped(task, cl, is_learn, is_spam)) {
+ skip_classifier = TRUE;
+ }
+ }
+
+ if (skip_classifier) {
+ /* Set NULL for all statfiles indexed by id */
+ for (int j = 0; j < cl->statfiles_ids->len; j++) {
+ int id = g_array_index(cl->statfiles_ids, gint, j);
+ g_ptr_array_index(task->stat_runtimes, id) = NULL;
+ }
+ }
+ }
+
+ for (i = 0; i < st_ctx->statfiles->len; i++) {
+ st = g_ptr_array_index(st_ctx->statfiles, i);
+ g_assert(st != NULL);
+
+ if (g_ptr_array_index(task->stat_runtimes, i) == NULL) {
+ /* The whole classifier is skipped */
+ continue;
+ }
+
+ if (is_learn && st->backend->read_only) {
+ /* Read only backend, skip it */
+ g_ptr_array_index(task->stat_runtimes, i) = NULL;
+ continue;
+ }
+
+ if (!is_learn && !rspamd_symcache_is_symbol_enabled(task, task->cfg->cache,
+ st->stcf->symbol)) {
+ g_ptr_array_index(task->stat_runtimes, i) = NULL;
+ msg_debug_bayes("symbol %s is disabled, skip classification",
+ st->stcf->symbol);
+ continue;
+ }
+
+ bk_run = st->backend->runtime(task, st->stcf, is_learn, st->bkcf, i);
+
+ if (bk_run == NULL) {
+ msg_err_task("cannot init backend %s for statfile %s",
+ st->backend->name, st->stcf->symbol);
+ }
+
+ g_ptr_array_index(task->stat_runtimes, i) = bk_run;
+ }
+}
+
+static void
+rspamd_stat_backends_process(struct rspamd_stat_ctx *st_ctx,
+ struct rspamd_task *task)
+{
+ guint i;
+ struct rspamd_statfile *st;
+ gpointer bk_run;
+
+ g_assert(task->stat_runtimes != NULL);
+
+ for (i = 0; i < st_ctx->statfiles->len; i++) {
+ st = g_ptr_array_index(st_ctx->statfiles, i);
+ bk_run = g_ptr_array_index(task->stat_runtimes, i);
+
+ if (bk_run != NULL) {
+ st->backend->process_tokens(task, task->tokens, i, bk_run);
+ }
+ }
+}
+
+static void
+rspamd_stat_classifiers_process(struct rspamd_stat_ctx *st_ctx,
+ struct rspamd_task *task)
+{
+ guint i, j, id;
+ struct rspamd_classifier *cl;
+ struct rspamd_statfile *st;
+ gpointer bk_run;
+ gboolean skip;
+
+ if (st_ctx->classifiers->len == 0) {
+ return;
+ }
+
+ /*
+ * Do not classify a message if some class is missing
+ */
+ if (!(task->flags & RSPAMD_TASK_FLAG_HAS_SPAM_TOKENS)) {
+ msg_info_task("skip statistics as SPAM class is missing");
+
+ return;
+ }
+ if (!(task->flags & RSPAMD_TASK_FLAG_HAS_HAM_TOKENS)) {
+ msg_info_task("skip statistics as HAM class is missing");
+
+ return;
+ }
+
+ for (i = 0; i < st_ctx->classifiers->len; i++) {
+ cl = g_ptr_array_index(st_ctx->classifiers, i);
+ cl->spam_learns = 0;
+ cl->ham_learns = 0;
+ }
+
+ g_assert(task->stat_runtimes != NULL);
+
+ for (i = 0; i < st_ctx->statfiles->len; i++) {
+ st = g_ptr_array_index(st_ctx->statfiles, i);
+ cl = st->classifier;
+
+ bk_run = g_ptr_array_index(task->stat_runtimes, i);
+ g_assert(st != NULL);
+
+ if (bk_run != NULL) {
+ if (st->stcf->is_spam) {
+ cl->spam_learns += st->backend->total_learns(task,
+ bk_run,
+ st_ctx);
+ }
+ else {
+ cl->ham_learns += st->backend->total_learns(task,
+ bk_run,
+ st_ctx);
+ }
+ }
+ }
+
+ for (i = 0; i < st_ctx->classifiers->len; i++) {
+ cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+ g_assert(cl != NULL);
+
+ skip = FALSE;
+
+ /* Do not process classifiers on backend failures */
+ for (j = 0; j < cl->statfiles_ids->len; j++) {
+ id = g_array_index(cl->statfiles_ids, gint, j);
+ bk_run = g_ptr_array_index(task->stat_runtimes, id);
+ st = g_ptr_array_index(st_ctx->statfiles, id);
+
+ if (bk_run != NULL) {
+ if (!st->backend->finalize_process(task, bk_run, st_ctx)) {
+ skip = TRUE;
+ break;
+ }
+ }
+ }
+
+ /* Ensure that all symbols enabled */
+ if (!skip && !(cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND)) {
+ for (j = 0; j < cl->statfiles_ids->len; j++) {
+ id = g_array_index(cl->statfiles_ids, gint, j);
+ bk_run = g_ptr_array_index(task->stat_runtimes, id);
+ st = g_ptr_array_index(st_ctx->statfiles, id);
+
+ if (bk_run == NULL) {
+ skip = TRUE;
+ msg_debug_bayes("disable classifier %s as statfile symbol %s is disabled",
+ cl->cfg->name, st->stcf->symbol);
+ break;
+ }
+ }
+ }
+
+ if (!skip) {
+ if (cl->cfg->min_tokens > 0 && task->tokens->len < cl->cfg->min_tokens) {
+ msg_debug_bayes(
+ "contains less tokens than required for %s classifier: "
+ "%ud < %ud",
+ cl->cfg->name,
+ task->tokens->len,
+ cl->cfg->min_tokens);
+ continue;
+ }
+ else if (cl->cfg->max_tokens > 0 && task->tokens->len > cl->cfg->max_tokens) {
+ msg_debug_bayes(
+ "contains more tokens than allowed for %s classifier: "
+ "%ud > %ud",
+ cl->cfg->name,
+ task->tokens->len,
+ cl->cfg->max_tokens);
+ continue;
+ }
+
+ cl->subrs->classify_func(cl, task->tokens, task);
+ }
+ }
+}
+
+rspamd_stat_result_t
+rspamd_stat_classify(struct rspamd_task *task, lua_State *L, guint stage,
+ GError **err)
+{
+ struct rspamd_stat_ctx *st_ctx;
+ rspamd_stat_result_t ret = RSPAMD_STAT_PROCESS_OK;
+
+ st_ctx = rspamd_stat_get_ctx();
+ g_assert(st_ctx != NULL);
+
+ if (st_ctx->classifiers->len == 0) {
+ task->processed_stages |= stage;
+ return ret;
+ }
+
+ if (stage == RSPAMD_TASK_STAGE_CLASSIFIERS_PRE) {
+ /* Preprocess tokens */
+ rspamd_stat_preprocess(st_ctx, task, FALSE, FALSE);
+ }
+ else if (stage == RSPAMD_TASK_STAGE_CLASSIFIERS) {
+ /* Process backends */
+ rspamd_stat_backends_process(st_ctx, task);
+ }
+ else if (stage == RSPAMD_TASK_STAGE_CLASSIFIERS_POST) {
+ /* Process classifiers */
+ rspamd_stat_classifiers_process(st_ctx, task);
+ }
+
+ task->processed_stages |= stage;
+
+ return ret;
+}
+
+static gboolean
+rspamd_stat_cache_check(struct rspamd_stat_ctx *st_ctx,
+ struct rspamd_task *task,
+ const gchar *classifier,
+ gboolean spam,
+ GError **err)
+{
+ rspamd_learn_t learn_res = RSPAMD_LEARN_OK;
+ struct rspamd_classifier *cl, *sel = NULL;
+ gpointer rt;
+ guint i;
+
+ /* Check whether we have learned that file */
+ for (i = 0; i < st_ctx->classifiers->len; i++) {
+ cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+ /* Skip other classifiers if they are not needed */
+ if (classifier != NULL && (cl->cfg->name == NULL ||
+ g_ascii_strcasecmp(classifier, cl->cfg->name) != 0)) {
+ continue;
+ }
+
+ sel = cl;
+
+ if (sel->cache && sel->cachecf) {
+ rt = cl->cache->runtime(task, sel->cachecf, FALSE);
+ learn_res = cl->cache->check(task, spam, rt);
+ }
+
+ if (learn_res == RSPAMD_LEARN_IGNORE) {
+ /* Do not learn twice */
+ g_set_error(err, rspamd_stat_quark(), 404, "<%s> has been already "
+ "learned as %s, ignore it",
+ MESSAGE_FIELD(task, message_id),
+ spam ? "spam" : "ham");
+ task->flags |= RSPAMD_TASK_FLAG_ALREADY_LEARNED;
+
+ return FALSE;
+ }
+ else if (learn_res == RSPAMD_LEARN_UNLEARN) {
+ task->flags |= RSPAMD_TASK_FLAG_UNLEARN;
+ break;
+ }
+ }
+
+ if (sel == NULL) {
+ if (classifier) {
+ g_set_error(err, rspamd_stat_quark(), 404, "cannot find classifier "
+ "with name %s",
+ classifier);
+ }
+ else {
+ g_set_error(err, rspamd_stat_quark(), 404, "no classifiers defined");
+ }
+
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_stat_classifiers_learn(struct rspamd_stat_ctx *st_ctx,
+ struct rspamd_task *task,
+ const gchar *classifier,
+ gboolean spam,
+ GError **err)
+{
+ struct rspamd_classifier *cl, *sel = NULL;
+ guint i;
+ gboolean learned = FALSE, too_small = FALSE, too_large = FALSE;
+
+ if ((task->flags & RSPAMD_TASK_FLAG_ALREADY_LEARNED) && err != NULL &&
+ *err == NULL) {
+ /* Do not learn twice */
+ g_set_error(err, rspamd_stat_quark(), 208, "<%s> has been already "
+ "learned as %s, ignore it",
+ MESSAGE_FIELD(task, message_id),
+ spam ? "spam" : "ham");
+
+ return FALSE;
+ }
+
+ /* Check whether we have learned that file */
+ for (i = 0; i < st_ctx->classifiers->len; i++) {
+ cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+ /* Skip other classifiers if they are not needed */
+ if (classifier != NULL && (cl->cfg->name == NULL ||
+ g_ascii_strcasecmp(classifier, cl->cfg->name) != 0)) {
+ continue;
+ }
+
+ sel = cl;
+
+ /* Now check max and min tokens */
+ if (cl->cfg->min_tokens > 0 && task->tokens->len < cl->cfg->min_tokens) {
+ msg_info_task(
+ "<%s> contains less tokens than required for %s classifier: "
+ "%ud < %ud",
+ MESSAGE_FIELD(task, message_id),
+ cl->cfg->name,
+ task->tokens->len,
+ cl->cfg->min_tokens);
+ too_small = TRUE;
+ continue;
+ }
+ else if (cl->cfg->max_tokens > 0 && task->tokens->len > cl->cfg->max_tokens) {
+ msg_info_task(
+ "<%s> contains more tokens than allowed for %s classifier: "
+ "%ud > %ud",
+ MESSAGE_FIELD(task, message_id),
+ cl->cfg->name,
+ task->tokens->len,
+ cl->cfg->max_tokens);
+ too_large = TRUE;
+ continue;
+ }
+
+ if (cl->subrs->learn_spam_func(cl, task->tokens, task, spam,
+ task->flags & RSPAMD_TASK_FLAG_UNLEARN, err)) {
+ learned = TRUE;
+ }
+ }
+
+ if (sel == NULL) {
+ if (classifier) {
+ g_set_error(err, rspamd_stat_quark(), 404, "cannot find classifier "
+ "with name %s",
+ classifier);
+ }
+ else {
+ g_set_error(err, rspamd_stat_quark(), 404, "no classifiers defined");
+ }
+
+ return FALSE;
+ }
+
+ if (!learned && err && *err == NULL) {
+ if (too_large) {
+ g_set_error(err, rspamd_stat_quark(), 204,
+ "<%s> contains more tokens than allowed for %s classifier: "
+ "%d > %d",
+ MESSAGE_FIELD(task, message_id),
+ sel->cfg->name,
+ task->tokens->len,
+ sel->cfg->max_tokens);
+ }
+ else if (too_small) {
+ g_set_error(err, rspamd_stat_quark(), 204,
+ "<%s> contains less tokens than required for %s classifier: "
+ "%d < %d",
+ MESSAGE_FIELD(task, message_id),
+ sel->cfg->name,
+ task->tokens->len,
+ sel->cfg->min_tokens);
+ }
+ }
+
+ return learned;
+}
+
+static gboolean
+rspamd_stat_backends_learn(struct rspamd_stat_ctx *st_ctx,
+ struct rspamd_task *task,
+ const gchar *classifier,
+ gboolean spam,
+ GError **err)
+{
+ struct rspamd_classifier *cl, *sel = NULL;
+ struct rspamd_statfile *st;
+ gpointer bk_run;
+ guint i, j;
+ gint id;
+ gboolean res = FALSE, backend_found = FALSE;
+
+ for (i = 0; i < st_ctx->classifiers->len; i++) {
+ cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+ /* Skip other classifiers if they are not needed */
+ if (classifier != NULL && (cl->cfg->name == NULL ||
+ g_ascii_strcasecmp(classifier, cl->cfg->name) != 0)) {
+ continue;
+ }
+
+ if (cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND) {
+ res = TRUE;
+ continue;
+ }
+
+ sel = cl;
+
+ for (j = 0; j < cl->statfiles_ids->len; j++) {
+ id = g_array_index(cl->statfiles_ids, gint, j);
+ st = g_ptr_array_index(st_ctx->statfiles, id);
+ bk_run = g_ptr_array_index(task->stat_runtimes, id);
+
+ g_assert(st != NULL);
+
+ if (bk_run == NULL) {
+ /* XXX: must be error */
+ if (task->result->passthrough_result) {
+ /* Passthrough email, cannot learn */
+ g_set_error(err, rspamd_stat_quark(), 204,
+ "Cannot learn statistics when passthrough "
+ "result has been set; not classified");
+
+ res = FALSE;
+ goto end;
+ }
+
+ msg_debug_task("no runtime for backend %s; classifier %s; symbol %s",
+ st->backend->name, cl->cfg->name, st->stcf->symbol);
+ continue;
+ }
+
+ /* We set sel merely when we have runtime */
+ backend_found = TRUE;
+
+ if (!(task->flags & RSPAMD_TASK_FLAG_UNLEARN)) {
+ if (!!spam != !!st->stcf->is_spam) {
+ /* If we are not unlearning, then do not touch another class */
+ continue;
+ }
+ }
+
+ if (!st->backend->learn_tokens(task, task->tokens, id, bk_run)) {
+ g_set_error(err, rspamd_stat_quark(), 500,
+ "Cannot push "
+ "learned results to the backend");
+
+ res = FALSE;
+ goto end;
+ }
+ else {
+ if (!!spam == !!st->stcf->is_spam) {
+ st->backend->inc_learns(task, bk_run, st_ctx);
+ }
+ else if (task->flags & RSPAMD_TASK_FLAG_UNLEARN) {
+ st->backend->dec_learns(task, bk_run, st_ctx);
+ }
+
+ res = TRUE;
+ }
+ }
+ }
+
+end:
+
+ if (!res) {
+ if (err && *err) {
+ /* Error has been set already */
+ return res;
+ }
+
+ if (sel == NULL) {
+ if (classifier) {
+ g_set_error(err, rspamd_stat_quark(), 404, "cannot find classifier "
+ "with name %s",
+ classifier);
+ }
+ else {
+ g_set_error(err, rspamd_stat_quark(), 404, "no classifiers defined");
+ }
+
+ return FALSE;
+ }
+ else if (!backend_found) {
+ g_set_error(err, rspamd_stat_quark(), 204, "all learn conditions "
+ "denied learning %s in %s",
+ spam ? "spam" : "ham",
+ classifier ? classifier : "default classifier");
+ }
+ else {
+ g_set_error(err, rspamd_stat_quark(), 404, "cannot find statfile "
+ "backend to learn %s in %s",
+ spam ? "spam" : "ham",
+ classifier ? classifier : "default classifier");
+ }
+ }
+
+ return res;
+}
+
+static gboolean
+rspamd_stat_backends_post_learn(struct rspamd_stat_ctx *st_ctx,
+ struct rspamd_task *task,
+ const gchar *classifier,
+ gboolean spam,
+ GError **err)
+{
+ struct rspamd_classifier *cl;
+ struct rspamd_statfile *st;
+ gpointer bk_run, cache_run;
+ guint i, j;
+ gint id;
+ gboolean res = TRUE;
+
+ for (i = 0; i < st_ctx->classifiers->len; i++) {
+ cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+ /* Skip other classifiers if they are not needed */
+ if (classifier != NULL && (cl->cfg->name == NULL ||
+ g_ascii_strcasecmp(classifier, cl->cfg->name) != 0)) {
+ continue;
+ }
+
+ if (cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND) {
+ res = TRUE;
+ continue;
+ }
+
+ for (j = 0; j < cl->statfiles_ids->len; j++) {
+ id = g_array_index(cl->statfiles_ids, gint, j);
+ st = g_ptr_array_index(st_ctx->statfiles, id);
+ bk_run = g_ptr_array_index(task->stat_runtimes, id);
+
+ g_assert(st != NULL);
+
+ if (bk_run == NULL) {
+ /* XXX: must be error */
+ continue;
+ }
+
+ if (!st->backend->finalize_learn(task, bk_run, st_ctx, err)) {
+ return RSPAMD_STAT_PROCESS_ERROR;
+ }
+ }
+
+ if (cl->cache) {
+ cache_run = cl->cache->runtime(task, cl->cachecf, TRUE);
+ cl->cache->learn(task, spam, cache_run);
+ }
+ }
+
+ g_atomic_int_add(&task->worker->srv->stat->messages_learned, 1);
+
+ return res;
+}
+
+rspamd_stat_result_t
+rspamd_stat_learn(struct rspamd_task *task,
+ gboolean spam, lua_State *L, const gchar *classifier, guint stage,
+ GError **err)
+{
+ struct rspamd_stat_ctx *st_ctx;
+ rspamd_stat_result_t ret = RSPAMD_STAT_PROCESS_OK;
+
+ /*
+ * We assume now that a task has been already classified before
+ * coming to learn
+ */
+ g_assert(RSPAMD_TASK_IS_CLASSIFIED(task));
+
+ st_ctx = rspamd_stat_get_ctx();
+ g_assert(st_ctx != NULL);
+
+ if (st_ctx->classifiers->len == 0) {
+ task->processed_stages |= stage;
+ return ret;
+ }
+
+ if (stage == RSPAMD_TASK_STAGE_LEARN_PRE) {
+ /* Process classifiers */
+ rspamd_stat_preprocess(st_ctx, task, TRUE, spam);
+
+ if (!rspamd_stat_cache_check(st_ctx, task, classifier, spam, err)) {
+ return RSPAMD_STAT_PROCESS_ERROR;
+ }
+ }
+ else if (stage == RSPAMD_TASK_STAGE_LEARN) {
+ /* Process classifiers */
+ if (!rspamd_stat_classifiers_learn(st_ctx, task, classifier,
+ spam, err)) {
+ if (err && *err == NULL) {
+ g_set_error(err, rspamd_stat_quark(), 500,
+ "Unknown statistics error, found when learning classifiers;"
+ " classifier: %s",
+ task->classifier);
+ }
+ return RSPAMD_STAT_PROCESS_ERROR;
+ }
+
+ /* Process backends */
+ if (!rspamd_stat_backends_learn(st_ctx, task, classifier, spam, err)) {
+ if (err && *err == NULL) {
+ g_set_error(err, rspamd_stat_quark(), 500,
+ "Unknown statistics error, found when storing data on backend;"
+ " classifier: %s",
+ task->classifier);
+ }
+ return RSPAMD_STAT_PROCESS_ERROR;
+ }
+ }
+ else if (stage == RSPAMD_TASK_STAGE_LEARN_POST) {
+ if (!rspamd_stat_backends_post_learn(st_ctx, task, classifier, spam, err)) {
+ return RSPAMD_STAT_PROCESS_ERROR;
+ }
+ }
+
+ task->processed_stages |= stage;
+
+ return ret;
+}
+
+static gboolean
+rspamd_stat_has_classifier_symbols(struct rspamd_task *task,
+ struct rspamd_scan_result *mres,
+ struct rspamd_classifier *cl)
+{
+ guint i;
+ gint id;
+ struct rspamd_statfile *st;
+ struct rspamd_stat_ctx *st_ctx;
+ gboolean is_spam;
+
+ if (mres == NULL) {
+ return FALSE;
+ }
+
+ st_ctx = rspamd_stat_get_ctx();
+ is_spam = !!(task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM);
+
+ for (i = 0; i < cl->statfiles_ids->len; i++) {
+ id = g_array_index(cl->statfiles_ids, gint, i);
+ st = g_ptr_array_index(st_ctx->statfiles, id);
+
+ if (rspamd_task_find_symbol_result(task, st->stcf->symbol, NULL)) {
+ if (is_spam == !!st->stcf->is_spam) {
+ msg_debug_bayes("do not autolearn %s as symbol %s is already "
+ "added",
+ is_spam ? "spam" : "ham", st->stcf->symbol);
+
+ return TRUE;
+ }
+ }
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_stat_check_autolearn(struct rspamd_task *task)
+{
+ struct rspamd_stat_ctx *st_ctx;
+ struct rspamd_classifier *cl;
+ const ucl_object_t *obj, *elt1, *elt2;
+ struct rspamd_scan_result *mres = NULL;
+ struct rspamd_task **ptask;
+ lua_State *L;
+ guint i;
+ gint err_idx;
+ gboolean ret = FALSE;
+ gdouble ham_score, spam_score;
+ const gchar *lua_script, *lua_ret;
+
+ g_assert(RSPAMD_TASK_IS_CLASSIFIED(task));
+ st_ctx = rspamd_stat_get_ctx();
+ g_assert(st_ctx != NULL);
+
+ L = task->cfg->lua_state;
+
+ for (i = 0; i < st_ctx->classifiers->len; i++) {
+ cl = g_ptr_array_index(st_ctx->classifiers, i);
+ ret = FALSE;
+
+ if (cl->cfg->opts) {
+ obj = ucl_object_lookup(cl->cfg->opts, "autolearn");
+
+ if (ucl_object_type(obj) == UCL_BOOLEAN) {
+ /* Legacy true/false */
+ if (ucl_object_toboolean(obj)) {
+ /*
+ * Default learning algorithm:
+ *
+ * - We learn spam if action is ACTION_REJECT
+ * - We learn ham if score is less than zero
+ */
+ mres = task->result;
+
+ if (mres) {
+ if (mres->score > rspamd_task_get_required_score(task, mres)) {
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_SPAM;
+
+ ret = TRUE;
+ }
+ else if (mres->score < 0) {
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_HAM;
+ ret = TRUE;
+ }
+ }
+ }
+ }
+ else if (ucl_object_type(obj) == UCL_ARRAY && obj->len == 2) {
+ /* Legacy thresholds */
+ /*
+ * We have an array of 2 elements, treat it as a
+ * ham_score, spam_score
+ */
+ elt1 = ucl_array_find_index(obj, 0);
+ elt2 = ucl_array_find_index(obj, 1);
+
+ if ((ucl_object_type(elt1) == UCL_FLOAT ||
+ ucl_object_type(elt1) == UCL_INT) &&
+ (ucl_object_type(elt2) == UCL_FLOAT ||
+ ucl_object_type(elt2) == UCL_INT)) {
+ ham_score = ucl_object_todouble(elt1);
+ spam_score = ucl_object_todouble(elt2);
+
+ if (ham_score > spam_score) {
+ gdouble t;
+
+ t = ham_score;
+ ham_score = spam_score;
+ spam_score = t;
+ }
+
+ mres = task->result;
+
+ if (mres) {
+ if (mres->score >= spam_score) {
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_SPAM;
+
+ ret = TRUE;
+ }
+ else if (mres->score <= ham_score) {
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_HAM;
+ ret = TRUE;
+ }
+ }
+ }
+ }
+ else if (ucl_object_type(obj) == UCL_STRING) {
+ /* Legacy script */
+ lua_script = ucl_object_tostring(obj);
+
+ if (luaL_dostring(L, lua_script) != 0) {
+ msg_err_task("cannot execute lua script for autolearn "
+ "extraction: %s",
+ lua_tostring(L, -1));
+ }
+ else {
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_pushvalue(L, -2); /* Function itself */
+
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ if (lua_pcall(L, 1, 1, err_idx) != 0) {
+ msg_err_task("call to autolearn script failed: "
+ "%s",
+ lua_tostring(L, -1));
+ }
+ else {
+ lua_ret = lua_tostring(L, -1);
+
+ /* We can have immediate results */
+ if (lua_ret) {
+ if (strcmp(lua_ret, "ham") == 0) {
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_HAM;
+ ret = TRUE;
+ }
+ else if (strcmp(lua_ret, "spam") == 0) {
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_SPAM;
+ ret = TRUE;
+ }
+ }
+ }
+
+ /* Result + error function + original function */
+ lua_pop(L, 3);
+ }
+ else {
+ msg_err_task("lua script must return "
+ "function(task) and not %s",
+ lua_typename(L, lua_type(
+ L, -1)));
+ }
+ }
+ }
+ else if (ucl_object_type(obj) == UCL_OBJECT) {
+ /* Try to find autolearn callback */
+ if (cl->autolearn_cbref == 0) {
+ /* We don't have preprocessed cb id, so try to get it */
+ if (!rspamd_lua_require_function(L, "lua_bayes_learn",
+ "autolearn")) {
+ msg_err_task("cannot get autolearn library from "
+ "`lua_bayes_learn`");
+ }
+ else {
+ cl->autolearn_cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ }
+
+ if (cl->autolearn_cbref != -1) {
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cl->autolearn_cbref);
+
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ /* Push the whole object as well */
+ ucl_object_push_lua(L, obj, true);
+
+ if (lua_pcall(L, 2, 1, err_idx) != 0) {
+ msg_err_task("call to autolearn script failed: "
+ "%s",
+ lua_tostring(L, -1));
+ }
+ else {
+ lua_ret = lua_tostring(L, -1);
+
+ if (lua_ret) {
+ if (strcmp(lua_ret, "ham") == 0) {
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_HAM;
+ ret = TRUE;
+ }
+ else if (strcmp(lua_ret, "spam") == 0) {
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_SPAM;
+ ret = TRUE;
+ }
+ }
+ }
+
+ lua_settop(L, err_idx - 1);
+ }
+ }
+
+ if (ret) {
+ /* Do not autolearn if we have this symbol already */
+ if (rspamd_stat_has_classifier_symbols(task, mres, cl)) {
+ ret = FALSE;
+ task->flags &= ~(RSPAMD_TASK_FLAG_LEARN_HAM |
+ RSPAMD_TASK_FLAG_LEARN_SPAM);
+ }
+ else if (mres != NULL) {
+ if (task->flags & RSPAMD_TASK_FLAG_LEARN_HAM) {
+ msg_info_task("<%s>: autolearn ham for classifier "
+ "'%s' as message's "
+ "score is negative: %.2f",
+ MESSAGE_FIELD(task, message_id), cl->cfg->name,
+ mres->score);
+ }
+ else {
+ msg_info_task("<%s>: autolearn spam for classifier "
+ "'%s' as message's "
+ "action is reject, score: %.2f",
+ MESSAGE_FIELD(task, message_id), cl->cfg->name,
+ mres->score);
+ }
+
+ task->classifier = cl->cfg->name;
+ break;
+ }
+ }
+ }
+ }
+
+ return ret;
+}
+
+/**
+ * Get the overall statistics for all statfile backends
+ * @param cfg configuration
+ * @param total_learns the total number of learns is stored here
+ * @return array of statistical information
+ */
+rspamd_stat_result_t
+rspamd_stat_statistics(struct rspamd_task *task,
+ struct rspamd_config *cfg,
+ guint64 *total_learns,
+ ucl_object_t **target)
+{
+ struct rspamd_stat_ctx *st_ctx;
+ struct rspamd_classifier *cl;
+ struct rspamd_statfile *st;
+ gpointer backend_runtime;
+ ucl_object_t *res = NULL, *elt;
+ guint64 learns = 0;
+ guint i, j;
+ gint id;
+
+ st_ctx = rspamd_stat_get_ctx();
+ g_assert(st_ctx != NULL);
+
+ res = ucl_object_typed_new(UCL_ARRAY);
+
+ for (i = 0; i < st_ctx->classifiers->len; i++) {
+ cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+ if (cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND) {
+ continue;
+ }
+
+ for (j = 0; j < cl->statfiles_ids->len; j++) {
+ id = g_array_index(cl->statfiles_ids, gint, j);
+ st = g_ptr_array_index(st_ctx->statfiles, id);
+ backend_runtime = st->backend->runtime(task, st->stcf, FALSE,
+ st->bkcf, id);
+ elt = st->backend->get_stat(backend_runtime, st->bkcf);
+
+ if (elt && ucl_object_type(elt) == UCL_OBJECT) {
+ const ucl_object_t *rev = ucl_object_lookup(elt, "revision");
+
+ learns += ucl_object_toint(rev);
+ }
+ else {
+ learns += st->backend->total_learns(task, backend_runtime,
+ st->bkcf);
+ }
+
+ if (elt != NULL) {
+ ucl_array_append(res, elt);
+ }
+ }
+ }
+
+ if (total_learns != NULL) {
+ *total_learns = learns;
+ }
+
+ if (target) {
+ *target = res;
+ }
+ else {
+ ucl_object_unref(res);
+ }
+
+ return RSPAMD_STAT_PROCESS_OK;
+}
diff --git a/src/libstat/tokenizers/osb.c b/src/libstat/tokenizers/osb.c
new file mode 100644
index 0000000..d871c7a
--- /dev/null
+++ b/src/libstat/tokenizers/osb.c
@@ -0,0 +1,424 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * OSB tokenizer
+ */
+
+
+#include "tokenizers.h"
+#include "stat_internal.h"
+#include "libmime/lang_detection.h"
+
+/* Size for features pipe */
+#define DEFAULT_FEATURE_WINDOW_SIZE 5
+#define DEFAULT_OSB_VERSION 2
+
+static const int primes[] = {
+ 1,
+ 7,
+ 3,
+ 13,
+ 5,
+ 29,
+ 11,
+ 51,
+ 23,
+ 101,
+ 47,
+ 203,
+ 97,
+ 407,
+ 197,
+ 817,
+ 397,
+ 1637,
+ 797,
+ 3277,
+};
+
+static const guchar osb_tokenizer_magic[] = {'o', 's', 'b', 't', 'o', 'k', 'v', '2'};
+
+enum rspamd_osb_hash_type {
+ RSPAMD_OSB_HASH_COMPAT = 0,
+ RSPAMD_OSB_HASH_XXHASH,
+ RSPAMD_OSB_HASH_SIPHASH
+};
+
+struct rspamd_osb_tokenizer_config {
+ guchar magic[8];
+ gshort version;
+ gshort window_size;
+ enum rspamd_osb_hash_type ht;
+ guint64 seed;
+ rspamd_sipkey_t sk;
+};
+
+/*
+ * Return default config
+ */
+static struct rspamd_osb_tokenizer_config *
+rspamd_tokenizer_osb_default_config(void)
+{
+ static struct rspamd_osb_tokenizer_config def;
+
+ if (memcmp(def.magic, osb_tokenizer_magic, sizeof(osb_tokenizer_magic)) != 0) {
+ memset(&def, 0, sizeof(def));
+ memcpy(def.magic, osb_tokenizer_magic, sizeof(osb_tokenizer_magic));
+ def.version = DEFAULT_OSB_VERSION;
+ def.window_size = DEFAULT_FEATURE_WINDOW_SIZE;
+ def.ht = RSPAMD_OSB_HASH_XXHASH;
+ def.seed = 0xdeadbabe;
+ }
+
+ return &def;
+}
+
+static struct rspamd_osb_tokenizer_config *
+rspamd_tokenizer_osb_config_from_ucl(rspamd_mempool_t *pool,
+ const ucl_object_t *obj)
+{
+ const ucl_object_t *elt;
+ struct rspamd_osb_tokenizer_config *cf, *def;
+ guchar *key = NULL;
+ gsize keylen;
+
+
+ if (pool != NULL) {
+ cf = rspamd_mempool_alloc0(pool, sizeof(*cf));
+ }
+ else {
+ cf = g_malloc0(sizeof(*cf));
+ }
+
+ /* Use default config */
+ def = rspamd_tokenizer_osb_default_config();
+ memcpy(cf, def, sizeof(*cf));
+
+ elt = ucl_object_lookup(obj, "hash");
+ if (elt != NULL && ucl_object_type(elt) == UCL_STRING) {
+ if (g_ascii_strncasecmp(ucl_object_tostring(elt), "xxh", 3) == 0) {
+ cf->ht = RSPAMD_OSB_HASH_XXHASH;
+ elt = ucl_object_lookup(obj, "seed");
+ if (elt != NULL && ucl_object_type(elt) == UCL_INT) {
+ cf->seed = ucl_object_toint(elt);
+ }
+ }
+ else if (g_ascii_strncasecmp(ucl_object_tostring(elt), "sip", 3) == 0) {
+ cf->ht = RSPAMD_OSB_HASH_SIPHASH;
+ elt = ucl_object_lookup(obj, "key");
+
+ if (elt != NULL && ucl_object_type(elt) == UCL_STRING) {
+ key = rspamd_decode_base32(ucl_object_tostring(elt),
+ 0, &keylen, RSPAMD_BASE32_DEFAULT);
+ if (keylen < sizeof(rspamd_sipkey_t)) {
+ msg_warn("siphash key is too short: %z", keylen);
+ g_free(key);
+ }
+ else {
+ memcpy(cf->sk, key, sizeof(cf->sk));
+ g_free(key);
+ }
+ }
+ else {
+ msg_warn_pool("siphash cannot be used without key");
+ }
+ }
+ }
+ else {
+ elt = ucl_object_lookup(obj, "compat");
+ if (elt != NULL && ucl_object_toboolean(elt)) {
+ cf->ht = RSPAMD_OSB_HASH_COMPAT;
+ }
+ }
+
+ elt = ucl_object_lookup(obj, "window");
+ if (elt != NULL && ucl_object_type(elt) == UCL_INT) {
+ cf->window_size = ucl_object_toint(elt);
+ if (cf->window_size > DEFAULT_FEATURE_WINDOW_SIZE * 4) {
+ msg_err_pool("too large window size: %d", cf->window_size);
+ cf->window_size = DEFAULT_FEATURE_WINDOW_SIZE;
+ }
+ }
+
+ return cf;
+}
+
+gpointer
+rspamd_tokenizer_osb_get_config(rspamd_mempool_t *pool,
+ struct rspamd_tokenizer_config *cf,
+ gsize *len)
+{
+ struct rspamd_osb_tokenizer_config *osb_cf, *def;
+
+ if (cf != NULL && cf->opts != NULL) {
+ osb_cf = rspamd_tokenizer_osb_config_from_ucl(pool, cf->opts);
+ }
+ else {
+ def = rspamd_tokenizer_osb_default_config();
+ osb_cf = rspamd_mempool_alloc(pool, sizeof(*osb_cf));
+ memcpy(osb_cf, def, sizeof(*osb_cf));
+ /* Do not write sipkey to statfile */
+ }
+
+ if (osb_cf->ht == RSPAMD_OSB_HASH_SIPHASH) {
+ msg_info_pool("siphash key is not stored into statfiles, so you'd "
+ "need to keep it inside the configuration");
+ }
+
+ memset(osb_cf->sk, 0, sizeof(osb_cf->sk));
+
+ if (len != NULL) {
+ *len = sizeof(*osb_cf);
+ }
+
+ return osb_cf;
+}
+
+#if 0
+gboolean
+rspamd_tokenizer_osb_compatible_config (struct rspamd_tokenizer_runtime *rt,
+ gpointer ptr, gsize len)
+{
+ struct rspamd_osb_tokenizer_config *osb_cf, *test_cf;
+ gboolean ret = FALSE;
+
+ test_cf = rt->config;
+ g_assert (test_cf != NULL);
+
+ if (len == sizeof (*osb_cf)) {
+ osb_cf = ptr;
+
+ if (memcmp (osb_cf, osb_tokenizer_magic, sizeof (osb_tokenizer_magic)) != 0) {
+ ret = test_cf->ht == RSPAMD_OSB_HASH_COMPAT;
+ }
+ else {
+ if (osb_cf->version == DEFAULT_OSB_VERSION) {
+ /* We can compare them directly now */
+ ret = (memcmp (osb_cf, test_cf, sizeof (*osb_cf)
+ - sizeof (osb_cf->sk))) == 0;
+ }
+ }
+ }
+ else {
+ /* We are compatible now merely with fallback config */
+ if (test_cf->ht == RSPAMD_OSB_HASH_COMPAT) {
+ ret = TRUE;
+ }
+ }
+
+ return ret;
+}
+
+gboolean
+rspamd_tokenizer_osb_load_config (rspamd_mempool_t *pool,
+ struct rspamd_tokenizer_runtime *rt,
+ gpointer ptr, gsize len)
+{
+ struct rspamd_osb_tokenizer_config *osb_cf;
+
+ if (ptr == NULL || len == 0) {
+ osb_cf = rspamd_tokenizer_osb_config_from_ucl (pool, rt->tkcf->opts);
+
+ if (osb_cf->ht != RSPAMD_OSB_HASH_COMPAT) {
+ /* Trying to load incompatible configuration */
+ msg_err_pool ("cannot load tokenizer configuration from a legacy "
+ "statfile; maybe you have forgotten to set 'compat' option"
+ " in the tokenizer configuration");
+
+ return FALSE;
+ }
+ }
+ else {
+ g_assert (len == sizeof (*osb_cf));
+ osb_cf = ptr;
+ }
+
+ rt->config = osb_cf;
+ rt->conf_len = sizeof (*osb_cf);
+
+ return TRUE;
+}
+
+gboolean
+rspamd_tokenizer_osb_is_compat (struct rspamd_tokenizer_runtime *rt)
+{
+ struct rspamd_osb_tokenizer_config *osb_cf = rt->config;
+
+ return (osb_cf->ht == RSPAMD_OSB_HASH_COMPAT);
+}
+#endif
+
+struct token_pipe_entry {
+ guint64 h;
+ rspamd_stat_token_t *t;
+};
+
+gint rspamd_tokenizer_osb(struct rspamd_stat_ctx *ctx,
+ struct rspamd_task *task,
+ GArray *words,
+ gboolean is_utf,
+ const gchar *prefix,
+ GPtrArray *result)
+{
+ rspamd_token_t *new_tok = NULL;
+ rspamd_stat_token_t *token;
+ struct rspamd_osb_tokenizer_config *osb_cf;
+ guint64 cur, seed;
+ struct token_pipe_entry *hashpipe;
+ guint32 h1, h2;
+ gsize token_size;
+ guint processed = 0, i, w, window_size, token_flags = 0;
+
+ if (words == NULL) {
+ return FALSE;
+ }
+
+ osb_cf = ctx->tkcf;
+ window_size = osb_cf->window_size;
+
+ if (prefix) {
+ seed = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64,
+ prefix, strlen(prefix), osb_cf->seed);
+ }
+ else {
+ seed = osb_cf->seed;
+ }
+
+ hashpipe = g_alloca(window_size * sizeof(hashpipe[0]));
+ for (i = 0; i < window_size; i++) {
+ hashpipe[i].h = 0xfe;
+ hashpipe[i].t = NULL;
+ }
+
+ token_size = sizeof(rspamd_token_t) +
+ sizeof(gdouble) * ctx->statfiles->len;
+ g_assert(token_size > 0);
+
+ for (w = 0; w < words->len; w++) {
+ token = &g_array_index(words, rspamd_stat_token_t, w);
+ token_flags = token->flags;
+ const gchar *begin;
+ gsize len;
+
+ if (token->flags &
+ (RSPAMD_STAT_TOKEN_FLAG_STOP_WORD | RSPAMD_STAT_TOKEN_FLAG_SKIPPED)) {
+ /* Skip stop/skipped words */
+ continue;
+ }
+
+ if (token->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) {
+ begin = token->stemmed.begin;
+ len = token->stemmed.len;
+ }
+ else {
+ begin = token->original.begin;
+ len = token->original.len;
+ }
+
+ if (osb_cf->ht == RSPAMD_OSB_HASH_COMPAT) {
+ rspamd_ftok_t ftok;
+
+ ftok.begin = begin;
+ ftok.len = len;
+ cur = rspamd_fstrhash_lc(&ftok, is_utf);
+ }
+ else {
+ /* We know that the words are normalized */
+ if (osb_cf->ht == RSPAMD_OSB_HASH_XXHASH) {
+ cur = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64,
+ begin, len, osb_cf->seed);
+ }
+ else {
+ rspamd_cryptobox_siphash((guchar *) &cur, begin,
+ len, osb_cf->sk);
+
+ if (prefix) {
+ cur ^= seed;
+ }
+ }
+ }
+
+ if (token_flags & RSPAMD_STAT_TOKEN_FLAG_UNIGRAM) {
+ new_tok = rspamd_mempool_alloc0(task->task_pool, token_size);
+ new_tok->flags = token_flags;
+ new_tok->t1 = token;
+ new_tok->t2 = token;
+ new_tok->data = cur;
+ new_tok->window_idx = 0;
+ g_ptr_array_add(result, new_tok);
+
+ continue;
+ }
+
+#define ADD_TOKEN \
+ do { \
+ new_tok = rspamd_mempool_alloc0(task->task_pool, token_size); \
+ new_tok->flags = token_flags; \
+ new_tok->t1 = hashpipe[0].t; \
+ new_tok->t2 = hashpipe[i].t; \
+ if (osb_cf->ht == RSPAMD_OSB_HASH_COMPAT) { \
+ h1 = ((guint32) hashpipe[0].h) * primes[0] + \
+ ((guint32) hashpipe[i].h) * primes[i << 1]; \
+ h2 = ((guint32) hashpipe[0].h) * primes[1] + \
+ ((guint32) hashpipe[i].h) * primes[(i << 1) - 1]; \
+ memcpy((guchar *) &new_tok->data, &h1, sizeof(h1)); \
+ memcpy(((guchar *) &new_tok->data) + sizeof(h1), &h2, sizeof(h2)); \
+ } \
+ else { \
+ new_tok->data = hashpipe[0].h * primes[0] + hashpipe[i].h * primes[i << 1]; \
+ } \
+ new_tok->window_idx = i; \
+ g_ptr_array_add(result, new_tok); \
+ } while (0)
+
+ if (processed < window_size) {
+ /* Just fill a hashpipe */
+ ++processed;
+ hashpipe[window_size - processed].h = cur;
+ hashpipe[window_size - processed].t = token;
+ }
+ else {
+ /* Shift hashpipe */
+ for (i = window_size - 1; i > 0; i--) {
+ hashpipe[i] = hashpipe[i - 1];
+ }
+ hashpipe[0].h = cur;
+ hashpipe[0].t = token;
+
+ processed++;
+
+ for (i = 1; i < window_size; i++) {
+ if (!(hashpipe[i].t->flags & RSPAMD_STAT_TOKEN_FLAG_EXCEPTION)) {
+ ADD_TOKEN;
+ }
+ }
+ }
+ }
+
+ if (processed > 1 && processed <= window_size) {
+ processed--;
+ memmove(hashpipe, &hashpipe[window_size - processed],
+ processed * sizeof(hashpipe[0]));
+
+ for (i = 1; i < processed; i++) {
+ ADD_TOKEN;
+ }
+ }
+
+#undef ADD_TOKEN
+
+ return TRUE;
+}
diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
new file mode 100644
index 0000000..ee7234d
--- /dev/null
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -0,0 +1,955 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Common tokenization functions
+ */
+
+#include "rspamd.h"
+#include "tokenizers.h"
+#include "stat_internal.h"
+#include "contrib/mumhash/mum.h"
+#include "libmime/lang_detection.h"
+#include "libstemmer.h"
+
+#include <unicode/utf8.h>
+#include <unicode/uchar.h>
+#include <unicode/uiter.h>
+#include <unicode/ubrk.h>
+#include <unicode/ucnv.h>
+#if U_ICU_VERSION_MAJOR_NUM >= 44
+#include <unicode/unorm2.h>
+#endif
+
+#include <math.h>
+
+typedef gboolean (*token_get_function)(rspamd_stat_token_t *buf, gchar const **pos,
+ rspamd_stat_token_t *token,
+ GList **exceptions, gsize *rl, gboolean check_signature);
+
+const gchar t_delimiters[256] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
+ 1, 0, 0, 1, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 1, 0, 1, 1, 1, 1, 1, 0,
+ 1, 1, 1, 1, 1, 1, 1, 1, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1,
+ 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 1, 1, 1, 1, 1, 1, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 1, 1, 1, 1, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0};
+
+/* Get next word from specified f_str_t buf */
+static gboolean
+rspamd_tokenizer_get_word_raw(rspamd_stat_token_t *buf,
+ gchar const **cur, rspamd_stat_token_t *token,
+ GList **exceptions, gsize *rl, gboolean unused)
+{
+ gsize remain, pos;
+ const gchar *p;
+ struct rspamd_process_exception *ex = NULL;
+
+ if (buf == NULL) {
+ return FALSE;
+ }
+
+ g_assert(cur != NULL);
+
+ if (exceptions != NULL && *exceptions != NULL) {
+ ex = (*exceptions)->data;
+ }
+
+ if (token->original.begin == NULL || *cur == NULL) {
+ if (ex != NULL) {
+ if (ex->pos == 0) {
+ token->original.begin = buf->original.begin + ex->len;
+ token->original.len = ex->len;
+ token->flags = RSPAMD_STAT_TOKEN_FLAG_EXCEPTION;
+ }
+ else {
+ token->original.begin = buf->original.begin;
+ token->original.len = 0;
+ }
+ }
+ else {
+ token->original.begin = buf->original.begin;
+ token->original.len = 0;
+ }
+ *cur = token->original.begin;
+ }
+
+ token->original.len = 0;
+
+ pos = *cur - buf->original.begin;
+ if (pos >= buf->original.len) {
+ return FALSE;
+ }
+
+ remain = buf->original.len - pos;
+ p = *cur;
+
+ /* Skip non delimiters symbols */
+ do {
+ if (ex != NULL && ex->pos == pos) {
+ /* Go to the next exception */
+ *exceptions = g_list_next(*exceptions);
+ *cur = p + ex->len;
+ return TRUE;
+ }
+ pos++;
+ p++;
+ remain--;
+ } while (remain > 0 && t_delimiters[(guchar) *p]);
+
+ token->original.begin = p;
+
+ while (remain > 0 && !t_delimiters[(guchar) *p]) {
+ if (ex != NULL && ex->pos == pos) {
+ *exceptions = g_list_next(*exceptions);
+ *cur = p + ex->len;
+ return TRUE;
+ }
+ token->original.len++;
+ pos++;
+ remain--;
+ p++;
+ }
+
+ if (remain == 0) {
+ return FALSE;
+ }
+
+ if (rl) {
+ *rl = token->original.len;
+ }
+
+ token->flags = RSPAMD_STAT_TOKEN_FLAG_TEXT;
+
+ *cur = p;
+
+ return TRUE;
+}
+
+static inline gboolean
+rspamd_tokenize_check_limit(gboolean decay,
+ guint word_decay,
+ guint nwords,
+ guint64 *hv,
+ guint64 *prob,
+ const rspamd_stat_token_t *token,
+ gssize remain,
+ gssize total)
+{
+ static const gdouble avg_word_len = 6.0;
+
+ if (!decay) {
+ if (token->original.len >= sizeof(guint64)) {
+ guint64 tmp;
+ memcpy(&tmp, token->original.begin, sizeof(tmp));
+ *hv = mum_hash_step(*hv, tmp);
+ }
+
+ /* Check for decay */
+ if (word_decay > 0 && nwords > word_decay && remain < (gssize) total) {
+ /* Start decay */
+ gdouble decay_prob;
+
+ *hv = mum_hash_finish(*hv);
+
+ /* We assume that word is 6 symbols length in average */
+ decay_prob = (gdouble) word_decay / ((total - (remain)) / avg_word_len) * 10;
+ decay_prob = floor(decay_prob) / 10.0;
+
+ if (decay_prob >= 1.0) {
+ *prob = G_MAXUINT64;
+ }
+ else {
+ *prob = (guint64) (decay_prob * (double) G_MAXUINT64);
+ }
+
+ return TRUE;
+ }
+ }
+ else {
+ /* Decaying probability */
+ /* LCG64 x[n] = a x[n - 1] + b mod 2^64 */
+ *hv = (*hv) * 2862933555777941757ULL + 3037000493ULL;
+
+ if (*hv > *prob) {
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+static inline gboolean
+rspamd_utf_word_valid(const guchar *text, const guchar *end,
+ gint32 start, gint32 finish)
+{
+ const guchar *st = text + start, *fin = text + finish;
+ UChar32 c;
+
+ if (st >= end || fin > end || st >= fin) {
+ return FALSE;
+ }
+
+ U8_NEXT(text, start, finish, c);
+
+ if (u_isJavaIDPart(c)) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+#define SHIFT_EX \
+ do { \
+ cur = g_list_next(cur); \
+ if (cur) { \
+ ex = (struct rspamd_process_exception *) cur->data; \
+ } \
+ else { \
+ ex = NULL; \
+ } \
+ } while (0)
+
+static inline void
+rspamd_tokenize_exception(struct rspamd_process_exception *ex, GArray *res)
+{
+ rspamd_stat_token_t token;
+
+ memset(&token, 0, sizeof(token));
+
+ if (ex->type == RSPAMD_EXCEPTION_GENERIC) {
+ token.original.begin = "!!EX!!";
+ token.original.len = sizeof("!!EX!!") - 1;
+ token.flags = RSPAMD_STAT_TOKEN_FLAG_EXCEPTION;
+
+ g_array_append_val(res, token);
+ token.flags = 0;
+ }
+ else if (ex->type == RSPAMD_EXCEPTION_URL) {
+ struct rspamd_url *uri;
+
+ uri = ex->ptr;
+
+ if (uri && uri->tldlen > 0) {
+ token.original.begin = rspamd_url_tld_unsafe(uri);
+ token.original.len = uri->tldlen;
+ }
+ else {
+ token.original.begin = "!!EX!!";
+ token.original.len = sizeof("!!EX!!") - 1;
+ }
+
+ token.flags = RSPAMD_STAT_TOKEN_FLAG_EXCEPTION;
+ g_array_append_val(res, token);
+ token.flags = 0;
+ }
+}
+
+
+GArray *
+rspamd_tokenize_text(const gchar *text, gsize len,
+ const UText *utxt,
+ enum rspamd_tokenize_type how,
+ struct rspamd_config *cfg,
+ GList *exceptions,
+ guint64 *hash,
+ GArray *cur_words,
+ rspamd_mempool_t *pool)
+{
+ rspamd_stat_token_t token, buf;
+ const gchar *pos = NULL;
+ gsize l = 0;
+ GArray *res;
+ GList *cur = exceptions;
+ guint min_len = 0, max_len = 0, word_decay = 0, initial_size = 128;
+ guint64 hv = 0;
+ gboolean decay = FALSE, long_text_mode = FALSE;
+ guint64 prob = 0;
+ static UBreakIterator *bi = NULL;
+ static const gsize long_text_limit = 1 * 1024 * 1024;
+ static const ev_tstamp max_exec_time = 0.2; /* 200 ms */
+ ev_tstamp start;
+
+ if (text == NULL) {
+ return cur_words;
+ }
+
+ if (len > long_text_limit) {
+ /*
+ * In this mode we do additional checks to avoid performance issues
+ */
+ long_text_mode = TRUE;
+ start = ev_time();
+ }
+
+ buf.original.begin = text;
+ buf.original.len = len;
+ buf.flags = 0;
+
+ memset(&token, 0, sizeof(token));
+
+ if (cfg != NULL) {
+ min_len = cfg->min_word_len;
+ max_len = cfg->max_word_len;
+ word_decay = cfg->words_decay;
+ initial_size = word_decay * 2;
+ }
+
+ if (!cur_words) {
+ res = g_array_sized_new(FALSE, FALSE, sizeof(rspamd_stat_token_t),
+ initial_size);
+ }
+ else {
+ res = cur_words;
+ }
+
+ if (G_UNLIKELY(how == RSPAMD_TOKENIZE_RAW || utxt == NULL)) {
+ while (rspamd_tokenizer_get_word_raw(&buf, &pos, &token, &cur, &l, FALSE)) {
+ if (l == 0 || (min_len > 0 && l < min_len) ||
+ (max_len > 0 && l > max_len)) {
+ token.original.begin = pos;
+ continue;
+ }
+
+ if (token.original.len > 0 &&
+ rspamd_tokenize_check_limit(decay, word_decay, res->len,
+ &hv, &prob, &token, pos - text, len)) {
+ if (!decay) {
+ decay = TRUE;
+ }
+ else {
+ token.original.begin = pos;
+ continue;
+ }
+ }
+
+ if (long_text_mode) {
+ if ((res->len + 1) % 16 == 0) {
+ ev_tstamp now = ev_time();
+
+ if (now - start > max_exec_time) {
+ msg_warn_pool_check(
+ "too long time has been spent on tokenization:"
+ " %.1f ms, limit is %.1f ms; %d words added so far",
+ (now - start) * 1e3, max_exec_time * 1e3,
+ res->len);
+
+ goto end;
+ }
+ }
+ }
+
+ g_array_append_val(res, token);
+
+ if (((gsize) res->len) * sizeof(token) > (0x1ull << 30u)) {
+ /* Due to bug in glib ! */
+ msg_err_pool_check(
+ "too many words found: %d, stop tokenization to avoid DoS",
+ res->len);
+
+ goto end;
+ }
+
+ token.original.begin = pos;
+ }
+ }
+ else {
+ /* UTF8 boundaries */
+ UErrorCode uc_err = U_ZERO_ERROR;
+ int32_t last, p;
+ struct rspamd_process_exception *ex = NULL;
+
+ if (bi == NULL) {
+ bi = ubrk_open(UBRK_WORD, NULL, NULL, 0, &uc_err);
+
+ g_assert(U_SUCCESS(uc_err));
+ }
+
+ ubrk_setUText(bi, (UText *) utxt, &uc_err);
+ last = ubrk_first(bi);
+ p = last;
+
+ if (cur) {
+ ex = (struct rspamd_process_exception *) cur->data;
+ }
+
+ while (p != UBRK_DONE) {
+ start_over:
+ token.original.len = 0;
+
+ if (p > last) {
+ if (ex && cur) {
+ /* Check exception */
+ if (ex->pos >= last && ex->pos <= p) {
+ /* We have an exception within boundary */
+ /* First, start to drain exceptions from the start */
+ while (cur && ex->pos <= last) {
+ /* We have an exception at the beginning, skip those */
+ last += ex->len;
+ rspamd_tokenize_exception(ex, res);
+
+ if (last > p) {
+ /* Exception spread over the boundaries */
+ while (last > p && p != UBRK_DONE) {
+ gint32 old_p = p;
+ p = ubrk_next(bi);
+
+ if (p != UBRK_DONE && p <= old_p) {
+ msg_warn_pool_check(
+ "tokenization reversed back on position %d,"
+ "%d new position (%d backward), likely libicu bug!",
+ (gint) (p), (gint) (old_p), old_p - p);
+
+ goto end;
+ }
+ }
+
+ /* We need to reset our scan with new p and last */
+ SHIFT_EX;
+ goto start_over;
+ }
+
+ SHIFT_EX;
+ }
+
+ /* Now, we can have an exception within boundary again */
+ if (cur && ex->pos >= last && ex->pos <= p) {
+ /* Append the first part */
+ if (rspamd_utf_word_valid(text, text + len, last,
+ ex->pos)) {
+ token.original.begin = text + last;
+ token.original.len = ex->pos - last;
+ token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT |
+ RSPAMD_STAT_TOKEN_FLAG_UTF;
+ }
+
+ /* Process the current exception */
+ last += ex->len + (ex->pos - last);
+
+ rspamd_tokenize_exception(ex, res);
+
+ if (last > p) {
+ /* Exception spread over the boundaries */
+ while (last > p && p != UBRK_DONE) {
+ gint32 old_p = p;
+ p = ubrk_next(bi);
+ if (p != UBRK_DONE && p <= old_p) {
+ msg_warn_pool_check(
+ "tokenization reversed back on position %d,"
+ "%d new position (%d backward), likely libicu bug!",
+ (gint) (p), (gint) (old_p), old_p - p);
+
+ goto end;
+ }
+ }
+ /* We need to reset our scan with new p and last */
+ SHIFT_EX;
+ goto start_over;
+ }
+
+ SHIFT_EX;
+ }
+ else if (p > last) {
+ if (rspamd_utf_word_valid(text, text + len, last, p)) {
+ token.original.begin = text + last;
+ token.original.len = p - last;
+ token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT |
+ RSPAMD_STAT_TOKEN_FLAG_UTF;
+ }
+ }
+ }
+ else if (ex->pos < last) {
+ /* Forward exceptions list */
+ while (cur && ex->pos <= last) {
+ /* We have an exception at the beginning, skip those */
+ SHIFT_EX;
+ }
+
+ if (rspamd_utf_word_valid(text, text + len, last, p)) {
+ token.original.begin = text + last;
+ token.original.len = p - last;
+ token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT |
+ RSPAMD_STAT_TOKEN_FLAG_UTF;
+ }
+ }
+ else {
+ /* No exceptions within boundary */
+ if (rspamd_utf_word_valid(text, text + len, last, p)) {
+ token.original.begin = text + last;
+ token.original.len = p - last;
+ token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT |
+ RSPAMD_STAT_TOKEN_FLAG_UTF;
+ }
+ }
+ }
+ else {
+ if (rspamd_utf_word_valid(text, text + len, last, p)) {
+ token.original.begin = text + last;
+ token.original.len = p - last;
+ token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT |
+ RSPAMD_STAT_TOKEN_FLAG_UTF;
+ }
+ }
+
+ if (token.original.len > 0 &&
+ rspamd_tokenize_check_limit(decay, word_decay, res->len,
+ &hv, &prob, &token, p, len)) {
+ if (!decay) {
+ decay = TRUE;
+ }
+ else {
+ token.flags |= RSPAMD_STAT_TOKEN_FLAG_SKIPPED;
+ }
+ }
+ }
+
+ if (token.original.len > 0) {
+ /* Additional check for number of words */
+ if (((gsize) res->len) * sizeof(token) > (0x1ull << 30u)) {
+ /* Due to bug in glib ! */
+ msg_err("too many words found: %d, stop tokenization to avoid DoS",
+ res->len);
+
+ goto end;
+ }
+
+ g_array_append_val(res, token);
+ }
+
+ /* Also check for long text mode */
+ if (long_text_mode) {
+ /* Check time each 128 words added */
+ const int words_check_mask = 0x7F;
+
+ if ((res->len & words_check_mask) == words_check_mask) {
+ ev_tstamp now = ev_time();
+
+ if (now - start > max_exec_time) {
+ msg_warn_pool_check(
+ "too long time has been spent on tokenization:"
+ " %.1f ms, limit is %.1f ms; %d words added so far",
+ (now - start) * 1e3, max_exec_time * 1e3,
+ res->len);
+
+ goto end;
+ }
+ }
+ }
+
+ last = p;
+ p = ubrk_next(bi);
+
+ if (p != UBRK_DONE && p <= last) {
+ msg_warn_pool_check("tokenization reversed back on position %d,"
+ "%d new position (%d backward), likely libicu bug!",
+ (gint) (p), (gint) (last), last - p);
+
+ goto end;
+ }
+ }
+ }
+
+end:
+ if (!decay) {
+ hv = mum_hash_finish(hv);
+ }
+
+ if (hash) {
+ *hash = hv;
+ }
+
+ return res;
+}
+
+#undef SHIFT_EX
+
+static void
+rspamd_add_metawords_from_str(const gchar *beg, gsize len,
+ struct rspamd_task *task)
+{
+ UText utxt = UTEXT_INITIALIZER;
+ UErrorCode uc_err = U_ZERO_ERROR;
+ guint i = 0;
+ UChar32 uc;
+ gboolean valid_utf = TRUE;
+
+ while (i < len) {
+ U8_NEXT(beg, i, len, uc);
+
+ if (((gint32) uc) < 0) {
+ valid_utf = FALSE;
+ break;
+ }
+
+#if U_ICU_VERSION_MAJOR_NUM < 50
+ if (u_isalpha(uc)) {
+ gint32 sc = ublock_getCode(uc);
+
+ if (sc == UBLOCK_THAI) {
+ valid_utf = FALSE;
+ msg_info_task("enable workaround for Thai characters for old libicu");
+ break;
+ }
+ }
+#endif
+ }
+
+ if (valid_utf) {
+ utext_openUTF8(&utxt,
+ beg,
+ len,
+ &uc_err);
+
+ task->meta_words = rspamd_tokenize_text(beg, len,
+ &utxt, RSPAMD_TOKENIZE_UTF,
+ task->cfg, NULL, NULL,
+ task->meta_words,
+ task->task_pool);
+
+ utext_close(&utxt);
+ }
+ else {
+ task->meta_words = rspamd_tokenize_text(beg, len,
+ NULL, RSPAMD_TOKENIZE_RAW,
+ task->cfg, NULL, NULL, task->meta_words,
+ task->task_pool);
+ }
+}
+
+void rspamd_tokenize_meta_words(struct rspamd_task *task)
+{
+ guint i = 0;
+ rspamd_stat_token_t *tok;
+
+ if (MESSAGE_FIELD(task, subject)) {
+ rspamd_add_metawords_from_str(MESSAGE_FIELD(task, subject),
+ strlen(MESSAGE_FIELD(task, subject)), task);
+ }
+
+ if (MESSAGE_FIELD(task, from_mime) && MESSAGE_FIELD(task, from_mime)->len > 0) {
+ struct rspamd_email_address *addr;
+
+ addr = g_ptr_array_index(MESSAGE_FIELD(task, from_mime), 0);
+
+ if (addr->name) {
+ rspamd_add_metawords_from_str(addr->name, strlen(addr->name), task);
+ }
+ }
+
+ if (task->meta_words != NULL) {
+ const gchar *language = NULL;
+
+ if (MESSAGE_FIELD(task, text_parts) &&
+ MESSAGE_FIELD(task, text_parts)->len > 0) {
+ struct rspamd_mime_text_part *tp = g_ptr_array_index(
+ MESSAGE_FIELD(task, text_parts), 0);
+
+ if (tp->language) {
+ language = tp->language;
+ }
+ }
+
+ rspamd_normalize_words(task->meta_words, task->task_pool);
+ rspamd_stem_words(task->meta_words, task->task_pool, language,
+ task->lang_det);
+
+ for (i = 0; i < task->meta_words->len; i++) {
+ tok = &g_array_index(task->meta_words, rspamd_stat_token_t, i);
+ tok->flags |= RSPAMD_STAT_TOKEN_FLAG_HEADER;
+ }
+ }
+}
+
+static inline void
+rspamd_uchars_to_ucs32(const UChar *src, gsize srclen,
+ rspamd_stat_token_t *tok,
+ rspamd_mempool_t *pool)
+{
+ UChar32 *dest, t, *d;
+ gint32 i = 0;
+
+ dest = rspamd_mempool_alloc(pool, srclen * sizeof(UChar32));
+ d = dest;
+
+ while (i < srclen) {
+ U16_NEXT_UNSAFE(src, i, t);
+
+ if (u_isgraph(t)) {
+ UCharCategory cat;
+
+ cat = u_charType(t);
+#if U_ICU_VERSION_MAJOR_NUM >= 57
+ if (u_hasBinaryProperty(t, UCHAR_EMOJI)) {
+ tok->flags |= RSPAMD_STAT_TOKEN_FLAG_EMOJI;
+ }
+#endif
+
+ if ((cat >= U_UPPERCASE_LETTER && cat <= U_OTHER_NUMBER) ||
+ cat == U_CONNECTOR_PUNCTUATION ||
+ cat == U_MATH_SYMBOL ||
+ cat == U_CURRENCY_SYMBOL) {
+ *d++ = u_tolower(t);
+ }
+ }
+ else {
+ /* Invisible spaces ! */
+ tok->flags |= RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES;
+ }
+ }
+
+ tok->unicode.begin = dest;
+ tok->unicode.len = d - dest;
+}
+
+static inline void
+rspamd_ucs32_to_normalised(rspamd_stat_token_t *tok,
+ rspamd_mempool_t *pool)
+{
+ guint i, doff = 0;
+ gsize utflen = 0;
+ gchar *dest;
+ UChar32 t;
+
+ for (i = 0; i < tok->unicode.len; i++) {
+ utflen += U8_LENGTH(tok->unicode.begin[i]);
+ }
+
+ dest = rspamd_mempool_alloc(pool, utflen + 1);
+
+ for (i = 0; i < tok->unicode.len; i++) {
+ t = tok->unicode.begin[i];
+ U8_APPEND_UNSAFE(dest, doff, t);
+ }
+
+ g_assert(doff <= utflen);
+ dest[doff] = '\0';
+
+ tok->normalized.len = doff;
+ tok->normalized.begin = dest;
+}
+
+void rspamd_normalize_single_word(rspamd_stat_token_t *tok, rspamd_mempool_t *pool)
+{
+ UErrorCode uc_err = U_ZERO_ERROR;
+ UConverter *utf8_converter;
+ UChar tmpbuf[1024]; /* Assume that we have no longer words... */
+ gsize ulen;
+
+ utf8_converter = rspamd_get_utf8_converter();
+
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) {
+ ulen = ucnv_toUChars(utf8_converter,
+ tmpbuf,
+ G_N_ELEMENTS(tmpbuf),
+ tok->original.begin,
+ tok->original.len,
+ &uc_err);
+
+ /* Now, we need to understand if we need to normalise the word */
+ if (!U_SUCCESS(uc_err)) {
+ tok->flags |= RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE;
+ tok->unicode.begin = NULL;
+ tok->unicode.len = 0;
+ tok->normalized.begin = NULL;
+ tok->normalized.len = 0;
+ }
+ else {
+#if U_ICU_VERSION_MAJOR_NUM >= 44
+ const UNormalizer2 *norm = rspamd_get_unicode_normalizer();
+ gint32 end;
+
+ /* We can now check if we need to decompose */
+ end = unorm2_spanQuickCheckYes(norm, tmpbuf, ulen, &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ rspamd_uchars_to_ucs32(tmpbuf, ulen, tok, pool);
+ tok->normalized.begin = NULL;
+ tok->normalized.len = 0;
+ tok->flags |= RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE;
+ }
+ else {
+ if (end == ulen) {
+ /* Already normalised, just lowercase */
+ rspamd_uchars_to_ucs32(tmpbuf, ulen, tok, pool);
+ rspamd_ucs32_to_normalised(tok, pool);
+ }
+ else {
+ /* Perform normalization */
+ UChar normbuf[1024];
+
+ g_assert(end < G_N_ELEMENTS(normbuf));
+ /* First part */
+ memcpy(normbuf, tmpbuf, end * sizeof(UChar));
+ /* Second part */
+ ulen = unorm2_normalizeSecondAndAppend(norm,
+ normbuf, end,
+ G_N_ELEMENTS(normbuf),
+ tmpbuf + end,
+ ulen - end,
+ &uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ if (uc_err != U_BUFFER_OVERFLOW_ERROR) {
+ msg_warn_pool_check("cannot normalise text '%*s': %s",
+ (gint) tok->original.len, tok->original.begin,
+ u_errorName(uc_err));
+ rspamd_uchars_to_ucs32(tmpbuf, ulen, tok, pool);
+ rspamd_ucs32_to_normalised(tok, pool);
+ tok->flags |= RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE;
+ }
+ }
+ else {
+ /* Copy normalised back */
+ rspamd_uchars_to_ucs32(normbuf, ulen, tok, pool);
+ tok->flags |= RSPAMD_STAT_TOKEN_FLAG_NORMALISED;
+ rspamd_ucs32_to_normalised(tok, pool);
+ }
+ }
+ }
+#else
+ /* Legacy version with no unorm2 interface */
+ rspamd_uchars_to_ucs32(tmpbuf, ulen, tok, pool);
+ rspamd_ucs32_to_normalised(tok, pool);
+#endif
+ }
+ }
+ else {
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) {
+ /* Simple lowercase */
+ gchar *dest;
+
+ dest = rspamd_mempool_alloc(pool, tok->original.len + 1);
+ rspamd_strlcpy(dest, tok->original.begin, tok->original.len + 1);
+ rspamd_str_lc(dest, tok->original.len);
+ tok->normalized.len = tok->original.len;
+ tok->normalized.begin = dest;
+ }
+ }
+}
+
+void rspamd_normalize_words(GArray *words, rspamd_mempool_t *pool)
+{
+ rspamd_stat_token_t *tok;
+ guint i;
+
+ for (i = 0; i < words->len; i++) {
+ tok = &g_array_index(words, rspamd_stat_token_t, i);
+ rspamd_normalize_single_word(tok, pool);
+ }
+}
+
+void rspamd_stem_words(GArray *words, rspamd_mempool_t *pool,
+ const gchar *language,
+ struct rspamd_lang_detector *lang_detector)
+{
+ static GHashTable *stemmers = NULL;
+ struct sb_stemmer *stem = NULL;
+ guint i;
+ rspamd_stat_token_t *tok;
+ gchar *dest;
+ gsize dlen;
+
+ if (!stemmers) {
+ stemmers = g_hash_table_new(rspamd_strcase_hash,
+ rspamd_strcase_equal);
+ }
+
+ if (language && language[0] != '\0') {
+ stem = g_hash_table_lookup(stemmers, language);
+
+ if (stem == NULL) {
+
+ stem = sb_stemmer_new(language, "UTF_8");
+
+ if (stem == NULL) {
+ msg_debug_pool(
+ "cannot create lemmatizer for %s language",
+ language);
+ g_hash_table_insert(stemmers, g_strdup(language),
+ GINT_TO_POINTER(-1));
+ }
+ else {
+ g_hash_table_insert(stemmers, g_strdup(language),
+ stem);
+ }
+ }
+ else if (stem == GINT_TO_POINTER(-1)) {
+ /* Negative cache */
+ stem = NULL;
+ }
+ }
+ for (i = 0; i < words->len; i++) {
+ tok = &g_array_index(words, rspamd_stat_token_t, i);
+
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) {
+ if (stem) {
+ const gchar *stemmed = NULL;
+
+ stemmed = sb_stemmer_stem(stem,
+ tok->normalized.begin, tok->normalized.len);
+
+ dlen = sb_stemmer_length(stem);
+
+ if (stemmed != NULL && dlen > 0) {
+ dest = rspamd_mempool_alloc(pool, dlen);
+ memcpy(dest, stemmed, dlen);
+ tok->stemmed.len = dlen;
+ tok->stemmed.begin = dest;
+ tok->flags |= RSPAMD_STAT_TOKEN_FLAG_STEMMED;
+ }
+ else {
+ /* Fallback */
+ tok->stemmed.len = tok->normalized.len;
+ tok->stemmed.begin = tok->normalized.begin;
+ }
+ }
+ else {
+ tok->stemmed.len = tok->normalized.len;
+ tok->stemmed.begin = tok->normalized.begin;
+ }
+
+ if (tok->stemmed.len > 0 && lang_detector != NULL &&
+ rspamd_language_detector_is_stop_word(lang_detector, tok->stemmed.begin, tok->stemmed.len)) {
+ tok->flags |= RSPAMD_STAT_TOKEN_FLAG_STOP_WORD;
+ }
+ }
+ else {
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) {
+ /* Raw text, lowercase */
+ tok->stemmed.len = tok->normalized.len;
+ tok->stemmed.begin = tok->normalized.begin;
+ }
+ }
+ }
+} \ No newline at end of file
diff --git a/src/libstat/tokenizers/tokenizers.h b/src/libstat/tokenizers/tokenizers.h
new file mode 100644
index 0000000..d696364
--- /dev/null
+++ b/src/libstat/tokenizers/tokenizers.h
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TOKENIZERS_H
+#define TOKENIZERS_H
+
+#include "config.h"
+#include "mem_pool.h"
+#include "fstring.h"
+#include "rspamd.h"
+#include "stat_api.h"
+
+#include <unicode/utext.h>
+
+#define RSPAMD_DEFAULT_TOKENIZER "osb"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_tokenizer_runtime;
+struct rspamd_stat_ctx;
+
+/* Common tokenizer structure */
+struct rspamd_stat_tokenizer {
+ gchar *name;
+
+ gpointer (*get_config)(rspamd_mempool_t *pool,
+ struct rspamd_tokenizer_config *cf, gsize *len);
+
+ gint (*tokenize_func)(struct rspamd_stat_ctx *ctx,
+ struct rspamd_task *task,
+ GArray *words,
+ gboolean is_utf,
+ const gchar *prefix,
+ GPtrArray *result);
+};
+
+enum rspamd_tokenize_type {
+ RSPAMD_TOKENIZE_UTF = 0,
+ RSPAMD_TOKENIZE_RAW,
+ RSPAMD_TOKENIZE_UNICODE
+};
+
+/* Compare two token nodes */
+gint token_node_compare_func(gconstpointer a, gconstpointer b);
+
+
+/* Tokenize text into array of words (rspamd_stat_token_t type) */
+GArray *rspamd_tokenize_text(const gchar *text, gsize len,
+ const UText *utxt,
+ enum rspamd_tokenize_type how,
+ struct rspamd_config *cfg,
+ GList *exceptions,
+ guint64 *hash,
+ GArray *cur_words,
+ rspamd_mempool_t *pool);
+
+/* OSB tokenize function */
+gint rspamd_tokenizer_osb(struct rspamd_stat_ctx *ctx,
+ struct rspamd_task *task,
+ GArray *words,
+ gboolean is_utf,
+ const gchar *prefix,
+ GPtrArray *result);
+
+gpointer rspamd_tokenizer_osb_get_config(rspamd_mempool_t *pool,
+ struct rspamd_tokenizer_config *cf,
+ gsize *len);
+
+struct rspamd_lang_detector;
+
+void rspamd_normalize_single_word(rspamd_stat_token_t *tok, rspamd_mempool_t *pool);
+
+void rspamd_normalize_words(GArray *words, rspamd_mempool_t *pool);
+
+void rspamd_stem_words(GArray *words, rspamd_mempool_t *pool,
+ const gchar *language,
+ struct rspamd_lang_detector *lang_detector);
+
+void rspamd_tokenize_meta_words(struct rspamd_task *task);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libutil/CMakeLists.txt b/src/libutil/CMakeLists.txt
new file mode 100644
index 0000000..67b7e94
--- /dev/null
+++ b/src/libutil/CMakeLists.txt
@@ -0,0 +1,24 @@
+# Librspamd-util
+SET(LIBRSPAMDUTILSRC
+ ${CMAKE_CURRENT_SOURCE_DIR}/addr.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/libev_helper.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/expression.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/fstring.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/hash.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/mem_pool.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/printf.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/radix.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/regexp.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/rrd.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/shingles.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/sqlite_utils.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/str_util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/upstream.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/heap.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/multipattern.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/cxx/utf8_util.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/cxx/util_tests.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/cxx/file_util.cxx)
+# Rspamdutil
+SET(RSPAMD_UTIL ${LIBRSPAMDUTILSRC} PARENT_SCOPE) \ No newline at end of file
diff --git a/src/libutil/addr.c b/src/libutil/addr.c
new file mode 100644
index 0000000..e011c99
--- /dev/null
+++ b/src/libutil/addr.c
@@ -0,0 +1,2049 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "addr.h"
+#include "util.h"
+#include "logger.h"
+#include "cryptobox.h"
+#include "unix-std.h"
+/* pwd and grp */
+#ifdef HAVE_PWD_H
+#include <pwd.h>
+#endif
+
+#ifdef HAVE_GRP_H
+#include <grp.h>
+#endif
+
+static void *local_addrs;
+
+enum {
+ RSPAMD_IPV6_UNDEFINED = 0,
+ RSPAMD_IPV6_SUPPORTED,
+ RSPAMD_IPV6_UNSUPPORTED
+} ipv6_status = RSPAMD_IPV6_UNDEFINED;
+
+/**
+ * Union that is used for storing sockaddrs
+ */
+union sa_union {
+ struct sockaddr sa;
+ struct sockaddr_in s4;
+ struct sockaddr_in6 s6;
+ struct sockaddr_un su;
+ struct sockaddr_storage ss;
+};
+
+union sa_inet {
+ struct sockaddr sa;
+ struct sockaddr_in s4;
+ struct sockaddr_in6 s6;
+};
+
+struct rspamd_addr_unix {
+ struct sockaddr_un addr;
+ gint mode;
+ uid_t owner;
+ gid_t group;
+};
+
+struct rspamd_addr_inet {
+ union sa_inet addr;
+};
+
+struct rspamd_inet_addr_s {
+ union {
+ struct rspamd_addr_inet in;
+ struct rspamd_addr_unix *un;
+ } u;
+ gint af;
+ socklen_t slen;
+};
+
+static void
+rspamd_ip_validate_af(rspamd_inet_addr_t *addr)
+{
+ if (addr->af != AF_UNIX) {
+ if (addr->u.in.addr.sa.sa_family != addr->af) {
+ addr->u.in.addr.sa.sa_family = addr->af;
+ }
+ }
+ else {
+ addr->u.un->addr.sun_family = AF_UNIX;
+ }
+
+ if (addr->af == AF_INET) {
+ addr->slen = sizeof(struct sockaddr_in);
+ }
+ else if (addr->af == AF_INET6) {
+ addr->slen = sizeof(struct sockaddr_in6);
+ }
+ else if (addr->af == AF_UNIX) {
+#ifdef SUN_LEN
+ addr->slen = SUN_LEN(&addr->u.un->addr);
+#else
+ addr->slen = sizeof(addr->u.un->addr);
+#endif
+#if defined(FREEBSD) || defined(__APPLE__)
+ addr->u.un->addr.sun_len = addr->slen;
+#endif
+ }
+}
+
+#define RSPAMD_MAYBE_ALLOC_POOL(pool, sz) \
+ (pool != NULL) ? rspamd_mempool_alloc((pool), (sz)) : g_malloc(sz)
+#define RSPAMD_MAYBE_ALLOC0_POOL(pool, sz) \
+ (pool != NULL) ? rspamd_mempool_alloc0((pool), (sz)) : g_malloc0(sz)
+
+static rspamd_inet_addr_t *
+rspamd_inet_addr_create(gint af, rspamd_mempool_t *pool)
+{
+ rspamd_inet_addr_t *addr;
+
+ addr = RSPAMD_MAYBE_ALLOC0_POOL(pool, sizeof(*addr));
+
+ addr->af = af;
+
+ if (af == AF_UNIX) {
+ addr->u.un = RSPAMD_MAYBE_ALLOC0_POOL(pool, sizeof(*addr->u.un));
+ addr->slen = sizeof(addr->u.un->addr);
+ }
+ else {
+ rspamd_ip_validate_af(addr);
+ }
+
+ return addr;
+}
+
+void rspamd_inet_address_free(rspamd_inet_addr_t *addr)
+{
+ if (addr) {
+ if (addr->af == AF_UNIX) {
+ if (addr->u.un) {
+ g_free(addr->u.un);
+ }
+ }
+ g_free(addr);
+ }
+}
+
+static void
+rspamd_ip_check_ipv6(void)
+{
+ if (ipv6_status == RSPAMD_IPV6_UNDEFINED) {
+ gint s;
+
+ s = socket(AF_INET6, SOCK_STREAM, 0);
+
+ if (s == -1) {
+ ipv6_status = RSPAMD_IPV6_UNSUPPORTED;
+ }
+ else {
+ /*
+ * Try to check /proc if we are on Linux (the common case)
+ */
+ struct stat st;
+
+ close(s);
+
+ if (stat("/proc/net/dev", &st) != -1) {
+ if (stat("/proc/net/if_inet6", &st) != -1) {
+ ipv6_status = RSPAMD_IPV6_SUPPORTED;
+ }
+ else {
+ ipv6_status = RSPAMD_IPV6_UNSUPPORTED;
+ }
+ }
+ else {
+ /* Not a Linux, so we assume it supports ipv6 somehow... */
+ ipv6_status = RSPAMD_IPV6_SUPPORTED;
+ }
+ }
+ }
+}
+
+gboolean
+rspamd_ip_is_valid(const rspamd_inet_addr_t *addr)
+{
+ const struct in_addr ip4_any = {INADDR_ANY}, ip4_none = {INADDR_NONE};
+ const struct in6_addr ip6_any = IN6ADDR_ANY_INIT;
+ gboolean ret = FALSE;
+
+ if (G_LIKELY(addr->af == AF_INET)) {
+ if (memcmp(&addr->u.in.addr.s4.sin_addr, &ip4_any,
+ sizeof(struct in_addr)) != 0 &&
+ memcmp(&addr->u.in.addr.s4.sin_addr, &ip4_none,
+ sizeof(struct in_addr)) != 0) {
+ ret = TRUE;
+ }
+ }
+ else if (G_UNLIKELY(addr->af == AF_INET6)) {
+ if (memcmp(&addr->u.in.addr.s6.sin6_addr, &ip6_any,
+ sizeof(struct in6_addr)) != 0) {
+ ret = TRUE;
+ }
+ }
+
+ return ret;
+}
+
+gint rspamd_accept_from_socket(gint sock, rspamd_inet_addr_t **target,
+ rspamd_accept_throttling_handler hdl,
+ void *hdl_data)
+{
+ gint nfd, serrno;
+ union sa_union su;
+ socklen_t len = sizeof(su);
+ rspamd_inet_addr_t *addr = NULL;
+
+ if ((nfd = accept(sock, &su.sa, &len)) == -1) {
+ if (target) {
+ *target = NULL;
+ }
+
+ if (errno == EAGAIN || errno == EINTR || errno == EWOULDBLOCK) {
+ return 0;
+ }
+ else if (errno == EMFILE || errno == ENFILE) {
+ /* Temporary disable accept event */
+ if (hdl) {
+ hdl(sock, hdl_data);
+ }
+
+ return 0;
+ }
+
+ return -1;
+ }
+
+ if (su.sa.sa_family == AF_INET6) {
+ /* Deal with bloody v4 mapped to v6 addresses */
+
+ static const guint8 mask[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+ const guint8 *p;
+
+ if (memcmp((const guint8 *) &su.s6.sin6_addr, mask, sizeof(mask)) == 0) {
+ p = (const guint8 *) &su.s6.sin6_addr;
+
+ if ((p[10] == 0xff && p[11] == 0xff)) {
+ addr = rspamd_inet_addr_create(AF_INET, NULL);
+ memcpy(&addr->u.in.addr.s4.sin_addr, &p[12],
+ sizeof(struct in_addr));
+ addr->u.in.addr.s4.sin_port = su.s6.sin6_port;
+ }
+ else {
+ /* Something strange but not mapped v4 address */
+ addr = rspamd_inet_addr_create(AF_INET6, NULL);
+ memcpy(&addr->u.in.addr.s6, &su.s6,
+ sizeof(struct sockaddr_in6));
+ }
+ }
+ else {
+ addr = rspamd_inet_addr_create(AF_INET6, NULL);
+ memcpy(&addr->u.in.addr.s6, &su.s6,
+ sizeof(struct sockaddr_in6));
+ }
+ }
+ else {
+ addr = rspamd_inet_addr_create(su.sa.sa_family, NULL);
+ addr->slen = len;
+
+ if (addr->af == AF_UNIX) {
+ len = sizeof(su);
+
+ if (getsockname(sock, &su.sa, &len) != -1) {
+ memcpy(&addr->u.un->addr, &su.su, MIN(len, sizeof(struct sockaddr_un)));
+ }
+ else {
+ /* Just copy socket address */
+ memcpy(&addr->u.un->addr, &su.sa, sizeof(struct sockaddr));
+ }
+ }
+ else {
+ memcpy(&addr->u.in.addr, &su, MIN(len, sizeof(addr->u.in.addr)));
+ }
+ }
+
+ if (rspamd_socket_nonblocking(nfd) < 0) {
+ goto out;
+ }
+
+ /* Set close on exec */
+ if (fcntl(nfd, F_SETFD, FD_CLOEXEC) == -1) {
+ msg_warn("fcntl failed: %d, '%s'", errno, strerror(errno));
+ goto out;
+ }
+
+ if (target) {
+ *target = addr;
+ }
+ else {
+ /* Avoid leak */
+ rspamd_inet_address_free(addr);
+ }
+
+ return (nfd);
+
+out:
+ serrno = errno;
+ close(nfd);
+ errno = serrno;
+ rspamd_inet_address_free(addr);
+
+ return (-1);
+}
+
+static gboolean
+rspamd_parse_unix_path(rspamd_inet_addr_t **target,
+ const char *src, gsize len,
+ rspamd_mempool_t *pool,
+ enum rspamd_inet_address_parse_flags how)
+{
+ gchar **tokens, **cur_tok, *p, *pwbuf;
+ glong pwlen;
+ struct passwd pw, *ppw;
+ struct group gr, *pgr;
+ rspamd_inet_addr_t *addr;
+ bool has_group = false;
+
+ addr = rspamd_inet_addr_create(AF_UNIX, pool);
+
+ addr->u.un->mode = 00644;
+ addr->u.un->owner = (uid_t) -1;
+ addr->u.un->group = (gid_t) -1;
+
+ if (!(how & RSPAMD_INET_ADDRESS_PARSE_REMOTE)) {
+ tokens = rspamd_string_len_split(src, len, " ,", -1, pool);
+
+ if (tokens[0] == NULL) {
+
+ if (!pool) {
+ rspamd_inet_address_free(addr);
+ g_strfreev(tokens);
+ }
+
+ return FALSE;
+ }
+
+ rspamd_strlcpy(addr->u.un->addr.sun_path, tokens[0],
+ sizeof(addr->u.un->addr.sun_path));
+#if defined(FREEBSD) || defined(__APPLE__)
+ addr->u.un->addr.sun_len = SUN_LEN(&addr->u.un->addr);
+#endif
+ }
+ else {
+ rspamd_strlcpy(addr->u.un->addr.sun_path, src,
+ MIN(len + 1, sizeof(addr->u.un->addr.sun_path)));
+#if defined(FREEBSD) || defined(__APPLE__)
+ addr->u.un->addr.sun_len = SUN_LEN(&addr->u.un->addr);
+#endif
+
+ if (target) {
+ rspamd_ip_validate_af(addr);
+ *target = addr;
+ }
+ else {
+ if (!pool) {
+ rspamd_inet_address_free(addr);
+ }
+ }
+
+ return TRUE;
+ }
+
+ /* Skip for remote */
+ cur_tok = &tokens[1];
+#ifdef _SC_GETPW_R_SIZE_MAX
+ pwlen = sysconf(_SC_GETPW_R_SIZE_MAX);
+ if (pwlen <= 0) {
+ pwlen = 8192;
+ }
+#else
+ pwlen = 8192;
+#endif
+
+ pwbuf = g_malloc0(pwlen);
+
+ while (*cur_tok) {
+ if (g_ascii_strncasecmp(*cur_tok, "mode=", sizeof("mode=") - 1) == 0) {
+ p = strchr(*cur_tok, '=');
+ /* XXX: add error check */
+ addr->u.un->mode = strtoul(p + 1, NULL, 0);
+
+ if (addr->u.un->mode == 0) {
+ msg_err("bad mode: %s", p + 1);
+ errno = EINVAL;
+ goto err;
+ }
+ }
+ else if (g_ascii_strncasecmp(*cur_tok, "owner=",
+ sizeof("owner=") - 1) == 0) {
+ p = strchr(*cur_tok, '=');
+
+ if (getpwnam_r(p + 1, &pw, pwbuf, pwlen, &ppw) != 0 || ppw == NULL) {
+ msg_err("bad user: %s", p + 1);
+ if (ppw == NULL) {
+ errno = ENOENT;
+ }
+ goto err;
+ }
+ addr->u.un->owner = pw.pw_uid;
+
+ if (!has_group) {
+ addr->u.un->group = pw.pw_gid;
+ }
+ }
+ else if (g_ascii_strncasecmp(*cur_tok, "group=",
+ sizeof("group=") - 1) == 0) {
+ p = strchr(*cur_tok, '=');
+
+ if (getgrnam_r(p + 1, &gr, pwbuf, pwlen, &pgr) != 0 || pgr == NULL) {
+ msg_err("bad group: %s", p + 1);
+ if (pgr == NULL) {
+ errno = ENOENT;
+ }
+ goto err;
+ }
+
+ has_group = true;
+ addr->u.un->group = gr.gr_gid;
+ }
+ cur_tok++;
+ }
+
+ g_free(pwbuf);
+
+ if (!pool) {
+ g_strfreev(tokens);
+ }
+
+ if (target) {
+ rspamd_ip_validate_af(addr);
+ *target = addr;
+ }
+ else {
+ if (!pool) {
+ rspamd_inet_address_free(addr);
+ }
+ }
+
+ return TRUE;
+
+err:
+
+ g_free(pwbuf);
+
+ if (!pool) {
+ g_strfreev(tokens);
+ rspamd_inet_address_free(addr);
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_parse_inet_address_ip4(const guchar *text, gsize len, gpointer target)
+{
+ const guchar *p;
+ guchar c;
+ guint32 addr = 0, *addrptr = target;
+ guint octet = 0, n = 0;
+
+ g_assert(text != NULL);
+ g_assert(target != NULL);
+
+ if (len == 0) {
+ len = strlen(text);
+ }
+
+ for (p = text; p < text + len; p++) {
+ c = *p;
+
+ if (c >= '0' && c <= '9') {
+ octet = octet * 10 + (c - '0');
+
+ if (octet > 255) {
+ return FALSE;
+ }
+
+ continue;
+ }
+
+ if (c == '.') {
+ addr = (addr << 8) + octet;
+ octet = 0;
+ n++;
+ continue;
+ }
+
+ return FALSE;
+ }
+
+ if (n == 3) {
+ addr = (addr << 8) + octet;
+ *addrptr = ntohl(addr);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+gboolean
+rspamd_parse_inet_address_ip6(const guchar *text, gsize len, gpointer target)
+{
+ guchar t, *zero = NULL, *s, *d, *addr = target;
+ const guchar *p, *digit = NULL, *percent;
+ gsize len4 = 0;
+ guint n = 8, nibbles = 0, word = 0;
+
+ g_assert(text != NULL);
+ g_assert(target != NULL);
+
+ p = text;
+ if (len == 0) {
+ len = strlen(text);
+ }
+
+ /* Check IPv6 scope */
+ if ((percent = memchr(p, '%', len)) != NULL && percent > p) {
+ len = percent - p; /* Ignore scope */
+ }
+
+ if (len > sizeof("IPv6:") - 1 &&
+ g_ascii_strncasecmp(p, "IPv6:", sizeof("IPv6:") - 1) == 0) {
+ /* Special case, SMTP conformant IPv6 address */
+ p += sizeof("IPv6:") - 1;
+ len -= sizeof("IPv6:") - 1;
+ }
+
+ if (*p == '[' && len > 1 && p[len - 1] == ']') {
+ /* Strip [] as well */
+ p++;
+ len -= 2;
+ }
+
+ /* Ignore leading colon */
+ if (len > 0 && *p == ':') {
+ p++;
+ len--;
+ }
+
+ for (/* void */; len; len--) {
+ t = *p++;
+
+ if (t == ':') {
+ if (nibbles) {
+ digit = p;
+ len4 = len;
+ *addr++ = (u_char) (word >> 8);
+ *addr++ = (u_char) (word & 0xff);
+
+ if (--n) {
+ nibbles = 0;
+ word = 0;
+ continue;
+ }
+ }
+ else {
+ if (zero == NULL) {
+ digit = p;
+ len4 = len;
+ zero = addr;
+ continue;
+ }
+ }
+
+ return FALSE;
+ }
+
+ if (t == '.' && nibbles) {
+ if (n < 2 || digit == NULL) {
+ return FALSE;
+ }
+
+ /* IPv4 encoded in IPv6 */
+ if (!rspamd_parse_inet_address_ip4(digit, len4 - 1, &word)) {
+ return FALSE;
+ }
+
+ word = ntohl(word);
+ *addr++ = (guchar) ((word >> 24) & 0xff);
+ *addr++ = (guchar) ((word >> 16) & 0xff);
+ n--;
+ break;
+ }
+
+ if (++nibbles > 4) {
+ /* Too many digits */
+ return FALSE;
+ }
+
+ /* Restore from hex */
+ if (t >= '0' && t <= '9') {
+ word = word * 16 + (t - '0');
+ continue;
+ }
+
+ t |= 0x20;
+
+ if (t >= 'a' && t <= 'f') {
+ word = word * 16 + (t - 'a') + 10;
+ continue;
+ }
+
+ return FALSE;
+ }
+
+ if (nibbles == 0 && zero == NULL) {
+ return FALSE;
+ }
+
+ *addr++ = (guchar) (word >> 8);
+ *addr++ = (guchar) (word & 0xff);
+
+ if (--n) {
+ if (zero) {
+ n *= 2;
+ s = addr - 1;
+ d = s + n;
+ while (s >= zero) {
+ *d-- = *s--;
+ }
+ memset(zero, 0, n);
+
+ return TRUE;
+ }
+ }
+ else {
+ if (zero == NULL) {
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+/* Checks for ipv6 mapped address */
+static rspamd_inet_addr_t *
+rspamd_inet_address_v6_maybe_map(const struct sockaddr_in6 *sin6,
+ rspamd_mempool_t *pool)
+{
+ rspamd_inet_addr_t *addr = NULL;
+ /* 10 zero bytes or 80 bits */
+ static const guint8 mask[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+ const guint8 *p;
+
+ if (memcmp((const guint8 *) &sin6->sin6_addr, mask, sizeof(mask)) == 0) {
+ p = (const guint8 *) &sin6->sin6_addr;
+
+ if ((p[10] == 0xff && p[11] == 0xff)) {
+ addr = rspamd_inet_addr_create(AF_INET, pool);
+ memcpy(&addr->u.in.addr.s4.sin_addr, &p[12],
+ sizeof(struct in_addr));
+ }
+ else {
+ /* Something strange but not mapped v4 address */
+ addr = rspamd_inet_addr_create(AF_INET6, pool);
+ memcpy(&addr->u.in.addr.s6.sin6_addr, &sin6->sin6_addr,
+ sizeof(struct in6_addr));
+ }
+ }
+ else {
+ addr = rspamd_inet_addr_create(AF_INET6, pool);
+ memcpy(&addr->u.in.addr.s6.sin6_addr, &sin6->sin6_addr,
+ sizeof(struct in6_addr));
+ }
+
+ return addr;
+}
+
+static void
+rspamd_inet_address_v6_maybe_map_static(const struct sockaddr_in6 *sin6,
+ rspamd_inet_addr_t *addr)
+{
+ /* 10 zero bytes or 80 bits */
+ static const guint8 mask[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+ const guint8 *p;
+
+ if (memcmp((const guint8 *) &sin6->sin6_addr, mask, sizeof(mask)) == 0) {
+ p = (const guint8 *) &sin6->sin6_addr;
+
+ if ((p[10] == 0xff && p[11] == 0xff)) {
+ memcpy(&addr->u.in.addr.s4.sin_addr, &p[12],
+ sizeof(struct in_addr));
+ addr->af = AF_INET;
+ addr->slen = sizeof(addr->u.in.addr.s4);
+ }
+ else {
+ /* Something strange but not mapped v4 address */
+ memcpy(&addr->u.in.addr.s6.sin6_addr, &sin6->sin6_addr,
+ sizeof(struct in6_addr));
+ addr->af = AF_INET6;
+ addr->slen = sizeof(addr->u.in.addr.s6);
+ }
+ }
+ else {
+ memcpy(&addr->u.in.addr.s6.sin6_addr, &sin6->sin6_addr,
+ sizeof(struct in6_addr));
+ addr->af = AF_INET6;
+ addr->slen = sizeof(addr->u.in.addr.s6);
+ }
+}
+
+static gboolean
+rspamd_parse_inet_address_common(rspamd_inet_addr_t **target,
+ const char *src,
+ gsize srclen,
+ rspamd_mempool_t *pool,
+ enum rspamd_inet_address_parse_flags how)
+{
+ gboolean ret = FALSE;
+ rspamd_inet_addr_t *addr = NULL;
+ union sa_inet su;
+ const char *end = NULL;
+ char ipbuf[INET6_ADDRSTRLEN + 1];
+ guint iplen;
+ gulong portnum;
+
+ if (srclen == 0) {
+ return FALSE;
+ }
+
+ g_assert(src != NULL);
+ g_assert(target != NULL);
+
+ rspamd_ip_check_ipv6();
+
+ if (!(how & RSPAMD_INET_ADDRESS_PARSE_NO_UNIX) &&
+ (src[0] == '/' || src[0] == '.')) {
+ return rspamd_parse_unix_path(target, src, srclen, pool, how);
+ }
+
+ if (src[0] == '[') {
+ const gchar *ip_start;
+ /* Ipv6 address in format [::1]:port or just [::1] */
+ end = memchr(src + 1, ']', srclen - 1);
+
+ if (end == NULL) {
+ return FALSE;
+ }
+
+ iplen = end - src - 1;
+
+ if (iplen == 0 || iplen >= sizeof(ipbuf)) {
+ return FALSE;
+ }
+
+ ip_start = src + 1;
+ rspamd_strlcpy(ipbuf, ip_start, iplen + 1);
+
+ if (rspamd_parse_inet_address_ip6(ipbuf, iplen,
+ &su.s6.sin6_addr)) {
+ addr = rspamd_inet_address_v6_maybe_map(&su.s6, pool);
+ ret = TRUE;
+ }
+
+ if (!(how & RSPAMD_INET_ADDRESS_PARSE_NO_PORT) && ret && end[1] == ':') {
+ /* Port part */
+ rspamd_strtoul(end + 1, srclen - iplen - 3, &portnum);
+ rspamd_inet_address_set_port(addr, portnum);
+ }
+ }
+ else {
+
+ if (!(how & RSPAMD_INET_ADDRESS_PARSE_NO_PORT) &&
+ (end = memchr(src, ':', srclen)) != NULL) {
+ /* This is either port number and ipv4 addr or ipv6 addr */
+ /* Search for another semicolon */
+ if (memchr(end + 1, ':', srclen - (end - src + 1)) &&
+ rspamd_parse_inet_address_ip6(src, srclen,
+ &su.s6.sin6_addr)) {
+ addr = rspamd_inet_address_v6_maybe_map(&su.s6, pool);
+ ret = TRUE;
+ }
+ else {
+ /* Not ipv6, so try ip:port */
+ iplen = end - src;
+
+ if (iplen >= sizeof(ipbuf) || iplen <= 1) {
+ return FALSE;
+ }
+ else {
+ rspamd_strlcpy(ipbuf, src, iplen + 1);
+ }
+
+ if (rspamd_parse_inet_address_ip4(ipbuf, iplen,
+ &su.s4.sin_addr)) {
+ addr = rspamd_inet_addr_create(AF_INET, pool);
+ memcpy(&addr->u.in.addr.s4.sin_addr, &su.s4.sin_addr,
+ sizeof(struct in_addr));
+ rspamd_strtoul(end + 1, srclen - iplen - 1, &portnum);
+ rspamd_inet_address_set_port(addr, portnum);
+ ret = TRUE;
+ }
+ }
+ }
+ else {
+ if (rspamd_parse_inet_address_ip4(src, srclen, &su.s4.sin_addr)) {
+ addr = rspamd_inet_addr_create(AF_INET, pool);
+ memcpy(&addr->u.in.addr.s4.sin_addr, &su.s4.sin_addr,
+ sizeof(struct in_addr));
+ ret = TRUE;
+ }
+ else if (rspamd_parse_inet_address_ip6(src, srclen, &su.s6.sin6_addr)) {
+ addr = rspamd_inet_address_v6_maybe_map(&su.s6, pool);
+ ret = TRUE;
+ }
+ }
+ }
+
+ if (ret && target) {
+ *target = addr;
+ }
+
+ return ret;
+}
+
+gboolean
+rspamd_parse_inet_address(rspamd_inet_addr_t **target,
+ const char *src,
+ gsize srclen,
+ enum rspamd_inet_address_parse_flags how)
+{
+ return rspamd_parse_inet_address_common(target, src, srclen, NULL, how);
+}
+
+rspamd_inet_addr_t *
+rspamd_parse_inet_address_pool(const char *src,
+ gsize srclen,
+ rspamd_mempool_t *pool,
+ enum rspamd_inet_address_parse_flags how)
+{
+ rspamd_inet_addr_t *ret = NULL;
+
+ if (!rspamd_parse_inet_address_common(&ret, src, srclen, pool, how)) {
+ return NULL;
+ }
+
+ return ret;
+}
+
+gboolean
+rspamd_parse_inet_address_ip(const char *src, gsize srclen,
+ rspamd_inet_addr_t *target)
+{
+ const char *end;
+ char ipbuf[INET6_ADDRSTRLEN + 1];
+ guint iplen;
+ gulong portnum;
+ gboolean ret = FALSE;
+ union sa_inet su;
+
+ g_assert(target != NULL);
+ g_assert(src != NULL);
+
+ if (src[0] == '[') {
+ /* Ipv6 address in format [::1]:port or just [::1] */
+ end = memchr(src + 1, ']', srclen - 1);
+
+ if (end == NULL) {
+ return FALSE;
+ }
+
+ iplen = end - src - 1;
+
+ if (iplen == 0 || iplen >= sizeof(ipbuf)) {
+ return FALSE;
+ }
+
+ rspamd_strlcpy(ipbuf, src + 1, iplen + 1);
+
+ if (rspamd_parse_inet_address_ip6(ipbuf, iplen,
+ &su.s6.sin6_addr)) {
+ rspamd_inet_address_v6_maybe_map_static(&su.s6, target);
+ ret = TRUE;
+ }
+
+ if (ret && end[1] == ':') {
+ /* Port part */
+ rspamd_strtoul(end + 1, srclen - iplen - 3, &portnum);
+ rspamd_inet_address_set_port(target, portnum);
+ }
+ }
+ else {
+
+ if ((end = memchr(src, ':', srclen)) != NULL) {
+ /* This is either port number and ipv4 addr or ipv6 addr */
+ /* Search for another semicolon */
+ if (memchr(end + 1, ':', srclen - (end - src + 1)) &&
+ rspamd_parse_inet_address_ip6(src, srclen, &su.s6.sin6_addr)) {
+ rspamd_inet_address_v6_maybe_map_static(&su.s6, target);
+ ret = TRUE;
+ }
+ else {
+ /* Not ipv6, so try ip:port */
+ iplen = end - src;
+
+ if (iplen >= sizeof(ipbuf) || iplen <= 1) {
+ return FALSE;
+ }
+ else {
+ rspamd_strlcpy(ipbuf, src, iplen + 1);
+ }
+
+ if (rspamd_parse_inet_address_ip4(ipbuf, iplen,
+ &su.s4.sin_addr)) {
+ memcpy(&target->u.in.addr.s4.sin_addr, &su.s4.sin_addr,
+ sizeof(struct in_addr));
+ target->af = AF_INET;
+ target->slen = sizeof(target->u.in.addr.s4);
+ rspamd_strtoul(end + 1, srclen - iplen - 1, &portnum);
+ rspamd_inet_address_set_port(target, portnum);
+ ret = TRUE;
+ }
+ }
+ }
+ else {
+ if (rspamd_parse_inet_address_ip4(src, srclen, &su.s4.sin_addr)) {
+ memcpy(&target->u.in.addr.s4.sin_addr, &su.s4.sin_addr,
+ sizeof(struct in_addr));
+ target->af = AF_INET;
+ target->slen = sizeof(target->u.in.addr.s4);
+ ret = TRUE;
+ }
+ else if (rspamd_parse_inet_address_ip6(src, srclen,
+ &su.s6.sin6_addr)) {
+ rspamd_inet_address_v6_maybe_map_static(&su.s6, target);
+ ret = TRUE;
+ }
+ }
+ }
+
+ return ret;
+}
+
+/*
+ * This is used to allow rspamd_inet_address_to_string to be used several times
+ * at the same function invocation, like printf("%s -> %s", f(ip1), f(ip2));
+ * Yes, it is bad but it helps to utilise this function without temporary buffers
+ * for up to 5 simultaneous invocations.
+ */
+#define NADDR_BUFS 5
+
+const char *
+rspamd_inet_address_to_string(const rspamd_inet_addr_t *addr)
+{
+ static char addr_str[NADDR_BUFS][INET6_ADDRSTRLEN + 1];
+ static guint cur_addr = 0;
+ char *addr_buf;
+
+ if (addr == NULL) {
+ return "<empty inet address>";
+ }
+
+ addr_buf = addr_str[cur_addr++ % NADDR_BUFS];
+
+ switch (addr->af) {
+ case AF_INET:
+ return inet_ntop(addr->af, &addr->u.in.addr.s4.sin_addr, addr_buf,
+ INET6_ADDRSTRLEN + 1);
+ case AF_INET6:
+ return inet_ntop(addr->af, &addr->u.in.addr.s6.sin6_addr, addr_buf,
+ INET6_ADDRSTRLEN + 1);
+ case AF_UNIX:
+ return addr->u.un->addr.sun_path;
+ }
+
+ return "undefined";
+}
+
+#define PRETTY_IP_BUFSIZE 128
+
+const char *
+rspamd_inet_address_to_string_pretty(const rspamd_inet_addr_t *addr)
+{
+ static char addr_str[NADDR_BUFS][PRETTY_IP_BUFSIZE];
+ static guint cur_addr = 0;
+ char *addr_buf;
+
+ if (addr == NULL) {
+ return "<empty inet address>";
+ }
+
+ addr_buf = addr_str[cur_addr++ % NADDR_BUFS];
+
+ switch (addr->af) {
+ case AF_INET:
+ rspamd_snprintf(addr_buf, PRETTY_IP_BUFSIZE, "%s:%d",
+ rspamd_inet_address_to_string(addr),
+ rspamd_inet_address_get_port(addr));
+ break;
+ case AF_INET6:
+ rspamd_snprintf(addr_buf, PRETTY_IP_BUFSIZE, "[%s]:%d",
+ rspamd_inet_address_to_string(addr),
+ rspamd_inet_address_get_port(addr));
+ break;
+ case AF_UNIX:
+ rspamd_snprintf(addr_buf, PRETTY_IP_BUFSIZE, "unix:%s",
+ rspamd_inet_address_to_string(addr));
+ break;
+ }
+
+ return addr_buf;
+}
+
+uint16_t
+rspamd_inet_address_get_port(const rspamd_inet_addr_t *addr)
+{
+ switch (addr->af) {
+ case AF_INET:
+ return ntohs(addr->u.in.addr.s4.sin_port);
+ case AF_INET6:
+ return ntohs(addr->u.in.addr.s6.sin6_port);
+ }
+
+ return 0;
+}
+
+void rspamd_inet_address_set_port(rspamd_inet_addr_t *addr, uint16_t port)
+{
+ switch (addr->af) {
+ case AF_INET:
+ addr->u.in.addr.s4.sin_port = htons(port);
+ break;
+ case AF_INET6:
+ addr->u.in.addr.s6.sin6_port = htons(port);
+ break;
+ }
+}
+
+int rspamd_inet_address_connect(const rspamd_inet_addr_t *addr, gint type,
+ gboolean async)
+{
+ int fd, r;
+ const struct sockaddr *sa;
+
+ if (addr == NULL) {
+ return -1;
+ }
+
+ fd = rspamd_socket_create(addr->af, type, 0, async);
+ if (fd == -1) {
+ return -1;
+ }
+
+ if (addr->af == AF_UNIX) {
+ sa = (const struct sockaddr *) &addr->u.un->addr;
+
+ if (type == (int) SOCK_DGRAM) {
+ struct sockaddr ca;
+
+ memset(&ca, 0, sizeof(ca));
+ ca.sa_family = AF_UNIX;
+
+ r = bind(fd, &ca, sizeof(sa_family_t));
+ if (r == -1) {
+ msg_info("unix socket client autobind failed: %s, '%s'",
+ addr->u.un->addr.sun_path, strerror(errno));
+ }
+ }
+ }
+ else {
+ sa = &addr->u.in.addr.sa;
+ }
+
+ r = connect(fd, sa, addr->slen);
+
+ if (r == -1) {
+ if (!async || errno != EINPROGRESS) {
+ close(fd);
+ msg_info("connect %s failed: %d, '%s'",
+ rspamd_inet_address_to_string_pretty(addr),
+ errno, strerror(errno));
+ return -1;
+ }
+ }
+
+ return fd;
+}
+
+int rspamd_inet_address_listen(const rspamd_inet_addr_t *addr, gint type,
+ enum rspamd_inet_address_listen_opts opts,
+ gint listen_queue)
+{
+ gint fd, r;
+ gint on = 1, serrno;
+ const struct sockaddr *sa;
+ const char *path;
+
+ if (addr == NULL) {
+ return -1;
+ }
+
+ fd = rspamd_socket_create(addr->af, type, 0,
+ (opts & RSPAMD_INET_ADDRESS_LISTEN_ASYNC));
+ if (fd == -1) {
+ return -1;
+ }
+
+ if (addr->af == AF_UNIX && access(addr->u.un->addr.sun_path, W_OK) != -1) {
+ /* Unlink old socket */
+ (void) unlink(addr->u.un->addr.sun_path);
+ }
+
+ if (addr->af == AF_UNIX) {
+ sa = (const struct sockaddr *) &addr->u.un->addr;
+ }
+ else {
+ sa = &addr->u.in.addr.sa;
+ }
+
+#if defined(SO_REUSEADDR)
+ if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const void *) &on, sizeof(gint)) == -1) {
+ msg_err("cannot set SO_REUSEADDR on %s (fd=%d): %s",
+ rspamd_inet_address_to_string_pretty(addr),
+ fd, strerror(errno));
+ goto err;
+ }
+#endif
+
+#if defined(SO_REUSEPORT) && defined(LINUX)
+ if (opts & RSPAMD_INET_ADDRESS_LISTEN_REUSEPORT) {
+ on = 1;
+
+ if (setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, (const void *) &on, sizeof(gint)) == -1) {
+ msg_err("cannot set SO_REUSEPORT on %s (fd=%d): %s",
+ rspamd_inet_address_to_string_pretty(addr),
+ fd, strerror(errno));
+ goto err;
+ }
+ }
+#endif
+
+#ifdef HAVE_IPV6_V6ONLY
+ if (addr->af == AF_INET6) {
+ /* We need to set this flag to avoid errors */
+ on = 1;
+#ifdef SOL_IPV6
+ (void) setsockopt(fd, SOL_IPV6, IPV6_V6ONLY, (const void *) &on, sizeof(gint));
+#elif defined(IPPROTO_IPV6)
+ (void) setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, (const void *) &on, sizeof(gint));
+#endif
+ }
+#endif
+
+ r = bind(fd, sa, addr->slen);
+ if (r == -1) {
+ if (!(opts & RSPAMD_INET_ADDRESS_LISTEN_ASYNC) || errno != EINPROGRESS) {
+ msg_warn("bind %s failed: %d, '%s'",
+ rspamd_inet_address_to_string_pretty(addr),
+ errno,
+ strerror(errno));
+
+ goto err;
+ }
+ }
+
+ if (addr->af == AF_UNIX) {
+ path = addr->u.un->addr.sun_path;
+ /* Try to set mode and owner */
+
+ if (addr->u.un->owner != (uid_t) -1 || addr->u.un->group != (gid_t) -1) {
+ if (chown(path, addr->u.un->owner, addr->u.un->group) == -1) {
+ msg_info("cannot change owner for %s to %d:%d: %s",
+ path, addr->u.un->owner, addr->u.un->group,
+ strerror(errno));
+ }
+ }
+
+ if (chmod(path, addr->u.un->mode) == -1) {
+ msg_info("cannot change mode for %s to %od %s",
+ path, addr->u.un->mode, strerror(errno));
+ }
+ }
+
+ if (type != (int) SOCK_DGRAM) {
+
+ if (!(opts & RSPAMD_INET_ADDRESS_LISTEN_NOLISTEN)) {
+ r = listen(fd, listen_queue);
+
+ if (r == -1) {
+ msg_warn("listen %s failed: %d, '%s'",
+ rspamd_inet_address_to_string_pretty(addr),
+ errno, strerror(errno));
+
+ goto err;
+ }
+ }
+ }
+
+ return fd;
+
+err:
+ /* Error path */
+ serrno = errno;
+
+ if (fd != -1) {
+ close(fd);
+ }
+
+ errno = serrno;
+
+ return -1;
+}
+
+gssize
+rspamd_inet_address_recvfrom(gint fd, void *buf, gsize len, gint fl,
+ rspamd_inet_addr_t **target)
+{
+ gssize ret;
+ union sa_union su;
+ socklen_t slen = sizeof(su);
+ rspamd_inet_addr_t *addr = NULL;
+
+ if ((ret = recvfrom(fd, buf, len, fl, &su.sa, &slen)) == -1) {
+ if (target) {
+ *target = NULL;
+ }
+
+ return -1;
+ }
+
+ if (target) {
+ addr = rspamd_inet_addr_create(su.sa.sa_family, NULL);
+ addr->slen = slen;
+
+ if (addr->af == AF_UNIX) {
+ addr->u.un = g_malloc(sizeof(*addr->u.un));
+ memcpy(&addr->u.un->addr, &su.su, sizeof(struct sockaddr_un));
+ }
+ else {
+ memcpy(&addr->u.in.addr, &su.sa, MIN(slen, sizeof(addr->u.in.addr)));
+ }
+
+ *target = addr;
+ }
+
+ return (ret);
+}
+
+gssize
+rspamd_inet_address_sendto(gint fd, const void *buf, gsize len, gint fl,
+ const rspamd_inet_addr_t *addr)
+{
+ gssize r;
+ const struct sockaddr *sa;
+
+ if (addr == NULL) {
+#ifdef EADDRNOTAVAIL
+ errno = EADDRNOTAVAIL;
+#endif
+ return -1;
+ }
+
+ if (addr->af == AF_UNIX) {
+ sa = (struct sockaddr *) &addr->u.un->addr;
+ }
+ else {
+ sa = &addr->u.in.addr.sa;
+ }
+
+ r = sendto(fd, buf, len, fl, sa, addr->slen);
+
+ return r;
+}
+
+static gboolean
+rspamd_check_port_priority(const char *line, guint default_port,
+ guint *priority, gchar *out,
+ gsize outlen, rspamd_mempool_t *pool)
+{
+ guint real_port = default_port, real_priority = 0;
+ gchar *err_str, *err_str_prio;
+
+ if (line && line[0] == ':') {
+ errno = 0;
+ real_port = strtoul(line + 1, &err_str, 10);
+
+ if (err_str && *err_str == ':') {
+ /* We have priority */
+ real_priority = strtoul(err_str + 1, &err_str_prio, 10);
+
+ if (err_str_prio && *err_str_prio != '\0') {
+ msg_err_pool_check(
+ "cannot parse priority: %s, at symbol %c, error: %s",
+ line,
+ *err_str_prio,
+ strerror(errno));
+
+ return FALSE;
+ }
+ }
+ else if (err_str && *err_str != '\0') {
+ msg_err_pool_check(
+ "cannot parse port: %s, at symbol %c, error: %s",
+ line,
+ *err_str,
+ strerror(errno));
+
+ return FALSE;
+ }
+ }
+
+ if (priority) {
+ *priority = real_priority;
+ }
+
+ rspamd_snprintf(out, outlen, "%ud", real_port);
+
+ return TRUE;
+}
+
+static enum rspamd_parse_host_port_result
+rspamd_resolve_addrs(const char *begin, size_t len, GPtrArray **addrs,
+ const gchar *portbuf, gint flags,
+ rspamd_mempool_t *pool)
+{
+ struct addrinfo hints, *res, *cur;
+ rspamd_inet_addr_t *cur_addr = NULL;
+ gint r, addr_cnt;
+ gchar *addr_cpy = NULL;
+ enum rspamd_parse_host_port_result ret = RSPAMD_PARSE_ADDR_FAIL;
+
+ rspamd_ip_check_ipv6();
+
+ if (rspamd_parse_inet_address(&cur_addr,
+ begin, len, RSPAMD_INET_ADDRESS_PARSE_DEFAULT) &&
+ cur_addr != NULL) {
+ if (*addrs == NULL) {
+ *addrs = g_ptr_array_new_full(1,
+ (GDestroyNotify) rspamd_inet_address_free);
+
+ if (pool != NULL) {
+ rspamd_mempool_add_destructor(pool,
+ rspamd_ptr_array_free_hard, *addrs);
+ }
+ }
+
+ rspamd_inet_address_set_port(cur_addr, strtoul(portbuf, NULL, 10));
+ g_ptr_array_add(*addrs, cur_addr);
+ ret = RSPAMD_PARSE_ADDR_NUMERIC;
+ }
+ else {
+ memset(&hints, 0, sizeof(hints));
+ hints.ai_socktype = SOCK_STREAM; /* Type of the socket */
+ hints.ai_flags = AI_NUMERICSERV | flags;
+
+ if (len > 0) {
+ if (pool) {
+ addr_cpy = rspamd_mempool_alloc(pool, len + 1);
+ }
+ else {
+ addr_cpy = g_malloc(len + 1);
+ }
+
+ rspamd_strlcpy(addr_cpy, begin, len + 1);
+ }
+ /* Otherwise it will be NULL */
+
+ if (ipv6_status == RSPAMD_IPV6_SUPPORTED) {
+ hints.ai_family = AF_UNSPEC;
+ }
+ else {
+ hints.ai_family = AF_INET;
+ }
+
+ if ((r = getaddrinfo(addr_cpy, portbuf, &hints, &res)) == 0) {
+ /* Now copy up to max_addrs of addresses */
+ addr_cnt = 0;
+ cur = res;
+ while (cur) {
+ cur = cur->ai_next;
+ addr_cnt++;
+ }
+
+ if (*addrs == NULL) {
+ *addrs = g_ptr_array_new_full(addr_cnt,
+ (GDestroyNotify) rspamd_inet_address_free);
+
+ if (pool != NULL) {
+ rspamd_mempool_add_destructor(pool,
+ rspamd_ptr_array_free_hard, *addrs);
+ }
+ }
+
+ cur = res;
+ while (cur) {
+ cur_addr = rspamd_inet_address_from_sa(cur->ai_addr,
+ cur->ai_addrlen);
+
+ if (cur_addr != NULL) {
+ g_ptr_array_add(*addrs, cur_addr);
+ }
+ cur = cur->ai_next;
+ }
+
+ freeaddrinfo(res);
+ ret = RSPAMD_PARSE_ADDR_RESOLVED;
+ }
+ else if (addr_cpy) {
+ msg_err_pool_check("address resolution for %s failed: %s",
+ addr_cpy,
+ gai_strerror(r));
+
+ if (pool == NULL) {
+ g_free(addr_cpy);
+ }
+
+ return RSPAMD_PARSE_ADDR_FAIL;
+ }
+ else {
+ /* Should never ever happen */
+ g_assert(0);
+ }
+ }
+
+ if (pool == NULL) {
+ g_free(addr_cpy);
+ }
+
+ return ret;
+}
+
+enum rspamd_parse_host_port_result
+rspamd_parse_host_port_priority(const gchar *str,
+ GPtrArray **addrs,
+ guint *priority,
+ gchar **name_ptr,
+ guint default_port,
+ gboolean allow_listen,
+ rspamd_mempool_t *pool)
+{
+ gchar portbuf[8];
+ const gchar *p, *name = NULL;
+ gsize namelen;
+ rspamd_inet_addr_t *cur_addr = NULL;
+ enum rspamd_parse_host_port_result ret = RSPAMD_PARSE_ADDR_FAIL;
+ union sa_union su;
+
+ /*
+ * In this function, we can have several possibilities:
+ * 1) Unix socket: check for '.' or '/' at the begin of string
+ * 2) \[ipv6\]: check for '[' at the beginning
+ * 3) '*': means listening on any address
+ * 4) ip|host[:port[:priority]]
+ */
+
+ if (allow_listen && str[0] == '*') {
+ bool v4_any = true, v6_any = true;
+
+ p = &str[1];
+
+ if (g_ascii_strncasecmp(p, "v4", 2) == 0) {
+ p += 2;
+ name = "*v4";
+ v6_any = false;
+ }
+ else if (g_ascii_strncasecmp(p, "v6", 2) == 0) {
+ p += 2;
+ name = "*v6";
+ v4_any = false;
+ }
+ else {
+ name = "*";
+ }
+
+ if (!rspamd_check_port_priority(p, default_port, priority,
+ portbuf, sizeof(portbuf), pool)) {
+ return ret;
+ }
+
+ if (*addrs == NULL) {
+ *addrs = g_ptr_array_new_full(1,
+ pool == NULL ? NULL : (GDestroyNotify) rspamd_inet_address_free);
+
+ if (pool != NULL) {
+ rspamd_mempool_add_destructor(pool,
+ rspamd_ptr_array_free_hard, *addrs);
+ }
+ }
+
+ if (v4_any) {
+ cur_addr = rspamd_inet_addr_create(AF_INET, NULL);
+ rspamd_parse_inet_address_ip4("0.0.0.0",
+ sizeof("0.0.0.0") - 1, &su.s4.sin_addr);
+ memcpy(&cur_addr->u.in.addr.s4.sin_addr, &su.s4.sin_addr,
+ sizeof(struct in_addr));
+ rspamd_inet_address_set_port(cur_addr,
+ strtoul(portbuf, NULL, 10));
+ g_ptr_array_add(*addrs, cur_addr);
+ }
+ if (v6_any) {
+ cur_addr = rspamd_inet_addr_create(AF_INET6, NULL);
+ rspamd_parse_inet_address_ip6("::",
+ sizeof("::") - 1, &su.s6.sin6_addr);
+ memcpy(&cur_addr->u.in.addr.s6.sin6_addr, &su.s6.sin6_addr,
+ sizeof(struct in6_addr));
+ rspamd_inet_address_set_port(cur_addr,
+ strtoul(portbuf, NULL, 10));
+ g_ptr_array_add(*addrs, cur_addr);
+ }
+
+ namelen = strlen(name);
+ ret = RSPAMD_PARSE_ADDR_NUMERIC; /* No resolution here */
+ }
+ else if (str[0] == '[') {
+ /* This is braced IPv6 address */
+ p = strchr(str, ']');
+
+ if (p == NULL) {
+ msg_err_pool_check("cannot parse address definition %s: %s",
+ str,
+ strerror(EINVAL));
+
+ return ret;
+ }
+
+ name = str + 1;
+ namelen = p - str - 1;
+
+ if (!rspamd_check_port_priority(p + 1, default_port, priority, portbuf,
+ sizeof(portbuf), pool)) {
+ return ret;
+ }
+
+ ret = rspamd_resolve_addrs(name, namelen, addrs, portbuf, 0, pool);
+ }
+ else if (str[0] == '/' || str[0] == '.') {
+ /* Special case of unix socket, as getaddrinfo cannot deal with them */
+ if (*addrs == NULL) {
+ *addrs = g_ptr_array_new_full(1,
+ (GDestroyNotify) rspamd_inet_address_free);
+
+ if (pool != NULL) {
+ rspamd_mempool_add_destructor(pool,
+ rspamd_ptr_array_free_hard, *addrs);
+ }
+ }
+
+ if (!rspamd_parse_inet_address(&cur_addr,
+ str, strlen(str), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ msg_err_pool_check("cannot parse unix socket definition %s: %s",
+ str,
+ strerror(errno));
+
+ return ret;
+ }
+
+ g_ptr_array_add(*addrs, cur_addr);
+ name = str;
+ namelen = strlen(str);
+ ret = RSPAMD_PARSE_ADDR_NUMERIC; /* No resolution here: unix socket */
+ }
+ else {
+ p = strchr(str, ':');
+
+ if (p == NULL) {
+ /* Just address or IP */
+ name = str;
+ namelen = strlen(str);
+ rspamd_check_port_priority("", default_port, priority, portbuf,
+ sizeof(portbuf), pool);
+
+ ret = rspamd_resolve_addrs(name, namelen, addrs,
+ portbuf, 0, pool);
+ }
+ else {
+ const gchar *second_semicolon = strchr(p + 1, ':');
+
+ name = str;
+
+ if (second_semicolon) {
+ /* name + port part excluding priority */
+ namelen = second_semicolon - str;
+ }
+ else {
+ /* Full ip/name + port */
+ namelen = strlen(str);
+ }
+
+ if (!rspamd_check_port_priority(p, default_port, priority, portbuf,
+ sizeof(portbuf), pool)) {
+ return ret;
+ }
+
+ ret = rspamd_resolve_addrs(str, p - str, addrs,
+ portbuf, 0, pool);
+ }
+ }
+
+ if (name_ptr != NULL) {
+ if (pool) {
+ *name_ptr = rspamd_mempool_alloc(pool, namelen + 1);
+ }
+ else {
+ *name_ptr = g_malloc(namelen + 1);
+ }
+
+ rspamd_strlcpy(*name_ptr, name, namelen + 1);
+ }
+
+ return ret;
+}
+
+guchar *
+rspamd_inet_address_get_hash_key(const rspamd_inet_addr_t *addr, guint *klen)
+{
+ guchar *res = NULL;
+ static struct in_addr local = {INADDR_LOOPBACK};
+
+ g_assert(addr != NULL);
+ g_assert(klen != NULL);
+
+ if (addr->af == AF_INET) {
+ *klen = sizeof(struct in_addr);
+ res = (guchar *) &addr->u.in.addr.s4.sin_addr;
+ }
+ else if (addr->af == AF_INET6) {
+ *klen = sizeof(struct in6_addr);
+ res = (guchar *) &addr->u.in.addr.s6.sin6_addr;
+ }
+ else if (addr->af == AF_UNIX) {
+ *klen = sizeof(struct in_addr);
+ res = (guchar *) &local;
+ }
+ else {
+ *klen = 0;
+ res = NULL;
+ }
+
+ return res;
+}
+
+
+rspamd_inet_addr_t *
+rspamd_inet_address_new(int af, const void *init)
+{
+ rspamd_inet_addr_t *addr;
+
+ addr = rspamd_inet_addr_create(af, NULL);
+
+ if (init != NULL) {
+ if (af == AF_UNIX) {
+ /* Init is a path */
+ rspamd_strlcpy(addr->u.un->addr.sun_path, init,
+ sizeof(addr->u.un->addr.sun_path));
+#if defined(FREEBSD) || defined(__APPLE__)
+ addr->u.un->addr.sun_len = SUN_LEN(&addr->u.un->addr);
+#endif
+ }
+ else if (af == AF_INET) {
+ memcpy(&addr->u.in.addr.s4.sin_addr, init, sizeof(struct in_addr));
+ }
+ else if (af == AF_INET6) {
+ memcpy(&addr->u.in.addr.s6.sin6_addr, init, sizeof(struct in6_addr));
+ }
+ }
+
+ return addr;
+}
+
+rspamd_inet_addr_t *
+rspamd_inet_address_from_sa(const struct sockaddr *sa, socklen_t slen)
+{
+ rspamd_inet_addr_t *addr;
+
+ g_assert(sa != NULL);
+ /* Address of an AF_UNIX socket can be tiny */
+ g_assert(slen >= sizeof(sa_family_t) + 1);
+
+ addr = rspamd_inet_addr_create(sa->sa_family, NULL);
+
+ if (sa->sa_family == AF_UNIX) {
+ /* Init is a path */
+ const struct sockaddr_un *un = (const struct sockaddr_un *) sa;
+
+ g_assert(slen >= SUN_LEN(un));
+ g_assert(slen <= sizeof(addr->u.un->addr));
+
+ /* sun_path can legally contain intermittent NULL bytes */
+ memcpy(&addr->u.un->addr, un, slen);
+
+ /* length of AF_UNIX addresses is variable */
+ addr->slen = slen;
+ }
+ else if (sa->sa_family == AF_INET) {
+ g_assert(slen >= sizeof(struct sockaddr_in));
+ memcpy(&addr->u.in.addr.s4, sa, sizeof(struct sockaddr_in));
+ }
+ else if (sa->sa_family == AF_INET6) {
+ g_assert(slen >= sizeof(struct sockaddr_in6));
+ memcpy(&addr->u.in.addr.s6, sa, sizeof(struct sockaddr_in6));
+ }
+ else {
+ /* XXX: currently we cannot deal with other AF */
+ g_assert(0);
+ }
+
+ return addr;
+}
+
+rspamd_inet_addr_t *
+rspamd_inet_address_from_rnds(const struct rdns_reply_entry *rep)
+{
+ rspamd_inet_addr_t *addr = NULL;
+
+ g_assert(rep != NULL);
+
+ if (rep->type == RDNS_REQUEST_A) {
+ addr = rspamd_inet_addr_create(AF_INET, NULL);
+ memcpy(&addr->u.in.addr.s4.sin_addr, &rep->content.a.addr,
+ sizeof(struct in_addr));
+ }
+ else if (rep->type == RDNS_REQUEST_AAAA) {
+ addr = rspamd_inet_addr_create(AF_INET6, NULL);
+ memcpy(&addr->u.in.addr.s6.sin6_addr, &rep->content.aaa.addr,
+ sizeof(struct in6_addr));
+ }
+
+ return addr;
+}
+
+void rspamd_inet_address_apply_mask(rspamd_inet_addr_t *addr, guint mask)
+{
+ guint32 umsk, *p;
+
+ if (mask > 0 && addr != NULL) {
+ if (addr->af == AF_INET && mask <= 32) {
+ umsk = htonl(G_MAXUINT32 << (32 - mask));
+ addr->u.in.addr.s4.sin_addr.s_addr &= umsk;
+ }
+ else if (addr->af == AF_INET6 && mask <= 128) {
+ p = (uint32_t *) &addr->u.in.addr.s6.sin6_addr;
+ mask = 128 - mask;
+ p += 3;
+
+ for (;;) {
+ if (mask >= 32) {
+ mask -= 32;
+ *p = 0;
+ }
+ else {
+ umsk = htonl(G_MAXUINT32 << mask);
+ *p &= umsk;
+ break;
+ }
+
+ p--;
+ }
+ }
+ }
+}
+
+static gint
+rspamd_inet_address_af_order(const rspamd_inet_addr_t *addr)
+{
+ int ret;
+
+ switch (addr->af) {
+ case AF_UNIX:
+ ret = 2;
+ break;
+ case AF_INET:
+ ret = 1;
+ break;
+ default:
+ ret = 0;
+ break;
+ }
+
+ return ret;
+}
+
+gint rspamd_inet_address_compare(const rspamd_inet_addr_t *a1,
+ const rspamd_inet_addr_t *a2, gboolean compare_ports)
+{
+ g_assert(a1 != NULL);
+ g_assert(a2 != NULL);
+
+ if (a1->af != a2->af) {
+ return (rspamd_inet_address_af_order(a2) -
+ rspamd_inet_address_af_order(a1));
+ }
+ else {
+ switch (a1->af) {
+ case AF_INET:
+ if (!compare_ports) {
+ return memcmp(&a1->u.in.addr.s4.sin_addr,
+ &a2->u.in.addr.s4.sin_addr, sizeof(struct in_addr));
+ }
+ else {
+ if (a1->u.in.addr.s4.sin_port == a2->u.in.addr.s4.sin_port) {
+ return memcmp(&a1->u.in.addr.s4.sin_addr,
+ &a2->u.in.addr.s4.sin_addr, sizeof(struct in_addr));
+ }
+ else {
+ return a1->u.in.addr.s4.sin_port - a2->u.in.addr.s4.sin_port;
+ }
+ }
+ case AF_INET6:
+ if (!compare_ports) {
+ return memcmp(&a1->u.in.addr.s6.sin6_addr,
+ &a2->u.in.addr.s6.sin6_addr, sizeof(struct in6_addr));
+ }
+ else {
+ if (a1->u.in.addr.s6.sin6_port == a2->u.in.addr.s6.sin6_port) {
+ return memcmp(&a1->u.in.addr.s6.sin6_addr,
+ &a2->u.in.addr.s6.sin6_addr, sizeof(struct in6_addr));
+ }
+ else {
+ return a1->u.in.addr.s6.sin6_port - a2->u.in.addr.s6.sin6_port;
+ }
+ }
+ case AF_UNIX:
+ return strncmp(a1->u.un->addr.sun_path,
+ a2->u.un->addr.sun_path, sizeof(a1->u.un->addr.sun_path));
+ default:
+ return memcmp(&a1->u.in, &a2->u.in, sizeof(a1->u.in));
+ }
+ }
+
+ return 0;
+}
+
+gint rspamd_inet_address_compare_ptr(gconstpointer a1,
+ gconstpointer a2)
+{
+ const rspamd_inet_addr_t **i1 = (const rspamd_inet_addr_t **) a1,
+ **i2 = (const rspamd_inet_addr_t **) a2;
+
+ return rspamd_inet_address_compare(*i1, *i2, FALSE);
+}
+
+rspamd_inet_addr_t *
+rspamd_inet_address_copy(const rspamd_inet_addr_t *addr, rspamd_mempool_t *pool)
+{
+ rspamd_inet_addr_t *n;
+
+ if (addr == NULL) {
+ return NULL;
+ }
+
+ n = rspamd_inet_addr_create(addr->af, pool);
+
+ if (n->af == AF_UNIX) {
+ memcpy(n->u.un, addr->u.un, sizeof(*addr->u.un));
+ }
+ else {
+ memcpy(&n->u.in, &addr->u.in, sizeof(addr->u.in));
+ }
+
+ return n;
+}
+
+gint rspamd_inet_address_get_af(const rspamd_inet_addr_t *addr)
+{
+ g_assert(addr != NULL);
+
+ return addr->af;
+}
+
+struct sockaddr *
+rspamd_inet_address_get_sa(const rspamd_inet_addr_t *addr,
+ socklen_t *sz)
+{
+ g_assert(addr != NULL);
+
+ if (addr->af == AF_UNIX) {
+ *sz = addr->slen;
+ return (struct sockaddr *) &addr->u.un->addr;
+ }
+ else {
+ *sz = addr->slen;
+ return (struct sockaddr *) &addr->u.in.addr.sa;
+ }
+}
+
+
+guint rspamd_inet_address_hash(gconstpointer a)
+{
+ const rspamd_inet_addr_t *addr = a;
+ struct {
+ gchar buf[sizeof(struct in6_addr)]; /* 16 bytes */
+ int af;
+ } layout;
+
+ gint32 k;
+
+ if (addr->af == AF_UNIX && addr->u.un) {
+ rspamd_cryptobox_fast_hash_state_t st;
+
+ rspamd_cryptobox_fast_hash_init(&st, rspamd_hash_seed());
+ rspamd_cryptobox_fast_hash_update(&st, &addr->af, sizeof(addr->af));
+ rspamd_cryptobox_fast_hash_update(&st, addr->u.un, sizeof(*addr->u.un));
+
+ return rspamd_cryptobox_fast_hash_final(&st);
+ }
+ else {
+ memset(&layout, 0, sizeof(layout));
+ layout.af = addr->af;
+
+ /* We ignore port part here */
+ if (addr->af == AF_INET) {
+ memcpy(layout.buf, &addr->u.in.addr.s4.sin_addr,
+ sizeof(addr->u.in.addr.s4.sin_addr));
+ }
+ else {
+ memcpy(layout.buf, &addr->u.in.addr.s6.sin6_addr,
+ sizeof(addr->u.in.addr.s6.sin6_addr));
+ }
+
+ k = rspamd_cryptobox_fast_hash(&layout, sizeof(layout),
+ rspamd_hash_seed());
+ }
+
+ return k;
+}
+
+guint rspamd_inet_address_port_hash(gconstpointer a)
+{
+ const rspamd_inet_addr_t *addr = a;
+ struct {
+ gchar buf[sizeof(struct in6_addr)]; /* 16 bytes */
+ int port;
+ int af;
+ } layout;
+
+ gint32 k;
+
+ if (addr->af == AF_UNIX && addr->u.un) {
+ rspamd_cryptobox_fast_hash_state_t st;
+
+ rspamd_cryptobox_fast_hash_init(&st, rspamd_hash_seed());
+ rspamd_cryptobox_fast_hash_update(&st, &addr->af, sizeof(addr->af));
+ rspamd_cryptobox_fast_hash_update(&st, addr->u.un, sizeof(*addr->u.un));
+
+ return rspamd_cryptobox_fast_hash_final(&st);
+ }
+ else {
+ memset(&layout, 0, sizeof(layout));
+ layout.af = addr->af;
+
+ /* We consider port part here */
+ if (addr->af == AF_INET) {
+ memcpy(layout.buf, &addr->u.in.addr.s4.sin_addr,
+ sizeof(addr->u.in.addr.s4.sin_addr));
+ layout.port = addr->u.in.addr.s4.sin_port;
+ }
+ else {
+ memcpy(layout.buf, &addr->u.in.addr.s6.sin6_addr,
+ sizeof(addr->u.in.addr.s6.sin6_addr));
+ layout.port = addr->u.in.addr.s6.sin6_port;
+ }
+
+ k = rspamd_cryptobox_fast_hash(&layout, sizeof(layout),
+ rspamd_hash_seed());
+ }
+
+ return k;
+}
+
+gboolean
+rspamd_inet_address_equal(gconstpointer a, gconstpointer b)
+{
+ const rspamd_inet_addr_t *a1 = a, *a2 = b;
+
+ return rspamd_inet_address_compare(a1, a2, FALSE) == 0;
+}
+
+gboolean
+rspamd_inet_address_port_equal(gconstpointer a, gconstpointer b)
+{
+ const rspamd_inet_addr_t *a1 = a, *a2 = b;
+
+ return rspamd_inet_address_compare(a1, a2, TRUE) == 0;
+}
+
+#ifndef IN6_IS_ADDR_LOOPBACK
+#define IN6_IS_ADDR_LOOPBACK(a) \
+ ((*(const __uint32_t *) (const void *) (&(a)->s6_addr[0]) == 0) && \
+ (*(const __uint32_t *) (const void *) (&(a)->s6_addr[4]) == 0) && \
+ (*(const __uint32_t *) (const void *) (&(a)->s6_addr[8]) == 0) && \
+ (*(const __uint32_t *) (const void *) (&(a)->s6_addr[12]) == ntohl(1)))
+#endif
+#ifndef IN6_IS_ADDR_LINKLOCAL
+#define IN6_IS_ADDR_LINKLOCAL(a) \
+ (((a)->s6_addr[0] == 0xfe) && (((a)->s6_addr[1] & 0xc0) == 0x80))
+#endif
+#ifndef IN6_IS_ADDR_SITELOCAL
+#define IN6_IS_ADDR_SITELOCAL(a) \
+ (((a)->s6_addr[0] == 0xfe) && (((a)->s6_addr[1] & 0xc0) == 0xc0))
+#endif
+
+gboolean
+rspamd_inet_address_is_local(const rspamd_inet_addr_t *addr)
+{
+ if (addr == NULL) {
+ return FALSE;
+ }
+
+ if (addr->af == AF_UNIX) {
+ /* Always true for unix sockets */
+ return TRUE;
+ }
+ else {
+ if (addr->af == AF_INET) {
+ if ((ntohl(addr->u.in.addr.s4.sin_addr.s_addr) & 0xff000000) == 0x7f000000) {
+ return TRUE;
+ }
+ }
+ else if (addr->af == AF_INET6) {
+ if (IN6_IS_ADDR_LOOPBACK(&addr->u.in.addr.s6.sin6_addr) ||
+ IN6_IS_ADDR_LINKLOCAL(&addr->u.in.addr.s6.sin6_addr) ||
+ IN6_IS_ADDR_SITELOCAL(&addr->u.in.addr.s6.sin6_addr)) {
+ return TRUE;
+ }
+ }
+ }
+
+ return FALSE;
+}
+
+void **
+rspamd_inet_library_init(void)
+{
+ return &local_addrs;
+}
+
+void *
+rspamd_inet_library_get_lib_ctx(void)
+{
+ return local_addrs;
+}
+
+void rspamd_inet_library_destroy(void)
+{
+ /* Ugly: local_addrs will actually be freed by config object */
+}
+
+gsize rspamd_inet_address_storage_size(void)
+{
+ return sizeof(rspamd_inet_addr_t);
+}
diff --git a/src/libutil/addr.h b/src/libutil/addr.h
new file mode 100644
index 0000000..25a3641
--- /dev/null
+++ b/src/libutil/addr.h
@@ -0,0 +1,356 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef ADDR_H_
+#define ADDR_H_
+
+#include "config.h"
+#include "rdns.h"
+
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+/* unix sockets */
+#ifdef HAVE_SYS_UN_H
+#include <sys/un.h>
+#endif
+
+#include "mem_pool.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Opaque structure
+ */
+typedef struct rspamd_inet_addr_s rspamd_inet_addr_t;
+
+/**
+ * Returns pointer storage for global singleton (map for local addresses)
+ * @return
+ */
+void **rspamd_inet_library_init(void);
+/**
+ * Returns local addresses singleton
+ * @return
+ */
+void *rspamd_inet_library_get_lib_ctx(void);
+/**
+ * Cleanup library (currently it does nothing)
+ */
+void rspamd_inet_library_destroy(void);
+
+/**
+ * Create new inet address structure based on the address family and opaque init pointer
+ * @param af
+ * @param init
+ * @return new inet addr
+ */
+rspamd_inet_addr_t *rspamd_inet_address_new(int af, const void *init);
+
+/**
+ * Create new inet address structure from struct sockaddr
+ * @param sa
+ * @param slen
+ * @return
+ */
+rspamd_inet_addr_t *rspamd_inet_address_from_sa(const struct sockaddr *sa,
+ socklen_t slen);
+
+/**
+ * Create new inet address from rdns reply
+ * @param rep reply element
+ * @return new ipv4 or ipv6 addr (port is NOT set)
+ */
+rspamd_inet_addr_t *rspamd_inet_address_from_rnds(
+ const struct rdns_reply_entry *rep);
+
+/**
+ * Parse string with ipv6 address of length `len` to `target` which should be
+ * at least sizeof (struct in6_addr)
+ * @param text input string
+ * @param len length of `text` (if 0, then `text` must be zero terminated)
+ * @param target target structure
+ * @return TRUE if the address has been parsed, otherwise `target` content is undefined
+ */
+gboolean rspamd_parse_inet_address_ip6(const guchar *text, gsize len,
+ gpointer target);
+
+enum rspamd_inet_address_parse_flags {
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT = 0,
+ RSPAMD_INET_ADDRESS_PARSE_REMOTE = 1u << 0u,
+ RSPAMD_INET_ADDRESS_PARSE_NO_UNIX = 1u << 1u,
+ RSPAMD_INET_ADDRESS_PARSE_NO_PORT = 1u << 2u,
+};
+
+/**
+ * Parse string with ipv4 address of length `len` to `target` which should be
+ * at least sizeof (in4_addr_t)
+ * @param text input string
+ * @param len length of `text` (if 0, then `text` must be zero terminated)
+ * @param target target structure
+ * @return TRUE if the address has been parsed, otherwise `target` content is undefined
+ */
+gboolean rspamd_parse_inet_address_ip4(const guchar *text, gsize len,
+ gpointer target);
+
+/**
+ * Parse ipv4 or ipv6 address to a static buffer `target`. Does not support Unix sockets
+ * @param src
+ * @param srclen
+ * @param target
+ * @return
+ */
+gboolean rspamd_parse_inet_address_ip(const char *src,
+ gsize srclen,
+ rspamd_inet_addr_t *target);
+
+/**
+ * Try to parse address from string
+ * @param target target to fill
+ * @param src IP string representation
+ * @return TRUE if addr has been parsed
+ */
+gboolean rspamd_parse_inet_address(rspamd_inet_addr_t **target,
+ const char *src,
+ gsize srclen,
+ enum rspamd_inet_address_parse_flags how);
+
+/**
+ * Use memory pool allocated inet address
+ * @param src
+ * @param srclen
+ * @param pool
+ * @return
+ */
+rspamd_inet_addr_t *rspamd_parse_inet_address_pool(const char *src,
+ gsize srclen,
+ rspamd_mempool_t *pool,
+ enum rspamd_inet_address_parse_flags how);
+
+/**
+ * Returns string representation of inet address
+ * @param addr
+ * @return statically allocated string pointer (not thread safe)
+ */
+const char *rspamd_inet_address_to_string(const rspamd_inet_addr_t *addr);
+
+/**
+ * Returns pretty string representation of inet address
+ * @param addr
+ * @return statically allocated string pointer (not thread safe)
+ */
+const char *rspamd_inet_address_to_string_pretty(const rspamd_inet_addr_t *addr);
+
+/**
+ * Returns port number for the specified inet address in host byte order
+ * @param addr
+ * @return
+ */
+uint16_t rspamd_inet_address_get_port(const rspamd_inet_addr_t *addr);
+
+/**
+ * Returns address family of inet address
+ * @param addr
+ * @return
+ */
+gint rspamd_inet_address_get_af(const rspamd_inet_addr_t *addr);
+
+/**
+ * Returns sockaddr and size for this address
+ * @param addr
+ * @param sz
+ * @return
+ */
+struct sockaddr *rspamd_inet_address_get_sa(const rspamd_inet_addr_t *addr,
+ socklen_t *sz);
+
+/**
+ * Makes a radix key from inet address
+ * @param addr
+ * @param klen
+ * @return
+ */
+guchar *rspamd_inet_address_get_hash_key(const rspamd_inet_addr_t *addr, guint *klen);
+
+/**
+ * Receive data from an unconnected socket and fill the inet_addr structure if needed
+ * @param fd
+ * @param buf
+ * @param len
+ * @param target
+ * @return same as recvfrom(2)
+ */
+gssize rspamd_inet_address_recvfrom(gint fd, void *buf, gsize len, gint fl,
+ rspamd_inet_addr_t **target);
+
+/**
+ * Send data via unconnected socket using the specified inet_addr structure
+ * @param fd
+ * @param buf
+ * @param len
+ * @param target
+ * @return
+ */
+gssize rspamd_inet_address_sendto(gint fd, const void *buf, gsize len, gint fl,
+ const rspamd_inet_addr_t *addr);
+
+/**
+ * Set port for inet address
+ */
+void rspamd_inet_address_set_port(rspamd_inet_addr_t *addr, uint16_t port);
+
+/**
+ * Connect to inet_addr address
+ * @param addr
+ * @param async perform operations asynchronously
+ * @return newly created and connected socket
+ */
+int rspamd_inet_address_connect(const rspamd_inet_addr_t *addr, gint type,
+ gboolean async);
+
+enum rspamd_inet_address_listen_opts {
+ RSPAMD_INET_ADDRESS_LISTEN_DEFAULT = 0,
+ RSPAMD_INET_ADDRESS_LISTEN_ASYNC = (1u << 0u),
+ RSPAMD_INET_ADDRESS_LISTEN_REUSEPORT = (1u << 1u),
+ RSPAMD_INET_ADDRESS_LISTEN_NOLISTEN = (1u << 2u),
+};
+/**
+ * Listen on a specified inet address
+ * @param addr
+ * @param type
+ * @param opts
+ * @return
+ */
+int rspamd_inet_address_listen(const rspamd_inet_addr_t *addr, gint type,
+ enum rspamd_inet_address_listen_opts opts,
+ gint listen_queue);
+
+/**
+ * Check whether specified ip is valid (not INADDR_ANY or INADDR_NONE) for ipv4 or ipv6
+ * @param ptr pointer to struct in_addr or struct in6_addr
+ * @param af address family (AF_INET or AF_INET6)
+ * @return TRUE if the address is valid
+ */
+gboolean rspamd_ip_is_valid(const rspamd_inet_addr_t *addr);
+
+typedef void (*rspamd_accept_throttling_handler)(gint, void *);
+
+/**
+ * Accept from listening socket filling addr structure
+ * @param sock listening socket
+ * @param target allocated inet addr structure
+ * @return
+ */
+gint rspamd_accept_from_socket(gint sock,
+ rspamd_inet_addr_t **target,
+ rspamd_accept_throttling_handler hdl,
+ void *hdl_data);
+
+enum rspamd_parse_host_port_result {
+ RSPAMD_PARSE_ADDR_FAIL = 0,
+ RSPAMD_PARSE_ADDR_RESOLVED = 1,
+ RSPAMD_PARSE_ADDR_NUMERIC = 2,
+};
+/**
+ * Parse host[:port[:priority]] line
+ * @param ina host address
+ * @param port port
+ * @param priority priority
+ * @return RSPAMD_PARSE_ADDR_FAIL in case of error, RSPAMD_PARSE_ADDR_NUMERIC in case of pure ip/unix socket
+ */
+enum rspamd_parse_host_port_result
+rspamd_parse_host_port_priority(const gchar *str,
+ GPtrArray **addrs,
+ guint *priority, gchar **name,
+ guint default_port,
+ gboolean allow_listen,
+ rspamd_mempool_t *pool);
+
+/**
+ * Destroy the specified IP address
+ * @param addr
+ */
+void rspamd_inet_address_free(rspamd_inet_addr_t *addr);
+
+/**
+ * Apply the specified mask to an address (ignored for AF_UNIX)
+ * @param addr
+ * @param mask
+ */
+void rspamd_inet_address_apply_mask(rspamd_inet_addr_t *addr, guint mask);
+
+/**
+ * Compare a1 and a2 and return value >0, ==0 and <0 if a1 is more, equal or less than a2 correspondingly
+ * @param a1
+ * @param a2
+ * @return
+ */
+gint rspamd_inet_address_compare(const rspamd_inet_addr_t *a1,
+ const rspamd_inet_addr_t *a2, gboolean compare_ports);
+
+/**
+ * Utility function to compare addresses by in g_ptr_array
+ * @param a1
+ * @param a2
+ * @return
+ */
+gint rspamd_inet_address_compare_ptr(gconstpointer a1,
+ gconstpointer a2);
+
+/**
+ * Performs deep copy of rspamd inet addr
+ * @param addr
+ * @return
+ */
+rspamd_inet_addr_t *rspamd_inet_address_copy(const rspamd_inet_addr_t *addr, rspamd_mempool_t *pool);
+
+/**
+ * Returns hash for inet address (ignoring port)
+ */
+guint rspamd_inet_address_hash(gconstpointer a);
+
+guint rspamd_inet_address_port_hash(gconstpointer a);
+
+/**
+ * Returns true if two address are equal
+ */
+gboolean rspamd_inet_address_equal(gconstpointer a, gconstpointer b);
+
+gboolean rspamd_inet_address_port_equal(gconstpointer a, gconstpointer b);
+
+/**
+ * Returns TRUE if an address belongs to some local address
+ */
+gboolean rspamd_inet_address_is_local(const rspamd_inet_addr_t *addr);
+
+/**
+ * Returns size of storage required to store a complete IP address
+ * @return
+ */
+gsize rspamd_inet_address_storage_size(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ADDR_H_ */
diff --git a/src/libutil/cxx/error.hxx b/src/libutil/cxx/error.hxx
new file mode 100644
index 0000000..4689d42
--- /dev/null
+++ b/src/libutil/cxx/error.hxx
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_ERROR_HXX
+#define RSPAMD_ERROR_HXX
+#pragma once
+
+#include "config.h"
+#include <string>
+#include <string_view>
+#include <cstdint>
+#include <optional>
+
+/***
+ * This unit is used to represent Rspamd C++ errors in a way to interoperate
+ * with C code if needed and avoid allocations for static strings
+ */
+namespace rspamd::util {
+
+enum class error_category : std::uint8_t {
+ INFORMAL,
+ IMPORTANT,
+ CRITICAL
+};
+
+struct error {
+public:
+ /**
+ * Construct from a static string, this string must live long enough to outlive this object
+ * @param msg
+ * @param code
+ * @param category
+ */
+ error(const char *msg, int code, error_category category = error_category::INFORMAL)
+ : error_message(msg), error_code(code), category(category)
+ {
+ }
+ /**
+ * Construct error from a temporary string taking membership
+ * @param msg
+ * @param code
+ * @param category
+ */
+ error(std::string &&msg, int code, error_category category = error_category::INFORMAL)
+ : error_code(code), category(category)
+ {
+ static_storage = std::move(msg);
+ error_message = static_storage.value();
+ }
+ /**
+ * Construct error from another string copying it into own storage
+ * @param msg
+ * @param code
+ * @param category
+ */
+ error(const std::string &msg, int code, error_category category = error_category::INFORMAL)
+ : error_code(code), category(category)
+ {
+ static_storage = msg;
+ error_message = static_storage.value();
+ }
+
+ error(const error &other)
+ : error_code(other.error_code), category(other.category)
+ {
+ if (other.static_storage) {
+ static_storage = other.static_storage;
+ error_message = static_storage.value();
+ }
+ else {
+ error_message = other.error_message;
+ }
+ }
+
+ error(error &&other) noexcept
+ {
+ *this = std::move(other);
+ }
+
+ error &operator=(error &&other) noexcept
+ {
+ if (other.static_storage.has_value()) {
+ std::swap(static_storage, other.static_storage);
+ error_message = static_storage.value();
+ }
+ else {
+ std::swap(error_message, other.error_message);
+ }
+ std::swap(other.error_code, error_code);
+ std::swap(other.category, category);
+
+ return *this;
+ }
+
+ /**
+ * Convert into GError
+ * @return
+ */
+ auto into_g_error() const -> GError *
+ {
+ return g_error_new(g_quark_from_static_string("rspamd"), error_code, "%s",
+ error_message.data());
+ }
+
+ /**
+ * Convenience alias for the `into_g_error`
+ * @param err
+ */
+ auto into_g_error_set(GError **err) const -> void
+ {
+ if (err && *err == nullptr) {
+ *err = into_g_error();
+ }
+ }
+
+ /**
+ * Convert into GError
+ * @return
+ */
+ auto into_g_error(GQuark quark) const -> GError *
+ {
+ return g_error_new(quark, error_code, "%s",
+ error_message.data());
+ }
+
+ /**
+ * Convenience alias for the `into_g_error`
+ * @param err
+ */
+ auto into_g_error_set(GQuark quark, GError **err) const -> void
+ {
+ if (err && *err == nullptr) {
+ *err = into_g_error(quark);
+ }
+ }
+
+public:
+ std::string_view error_message;
+ int error_code;
+ error_category category;
+
+private:
+ std::optional<std::string> static_storage;
+};
+
+}// namespace rspamd::util
+
+#endif//RSPAMD_ERROR_HXX
diff --git a/src/libutil/cxx/file_util.cxx b/src/libutil/cxx/file_util.cxx
new file mode 100644
index 0000000..2f031f0
--- /dev/null
+++ b/src/libutil/cxx/file_util.cxx
@@ -0,0 +1,457 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "file_util.hxx"
+#include <fmt/core.h>
+#include "libutil/util.h"
+#include "libutil/unix-std.h"
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+
+#include "doctest/doctest.h"
+
+namespace rspamd::util {
+
+auto raii_file::open(const char *fname, int flags) -> tl::expected<raii_file, error>
+{
+ int oflags = flags;
+#ifdef O_CLOEXEC
+ oflags |= O_CLOEXEC;
+#endif
+
+ if (fname == nullptr) {
+ return tl::make_unexpected(error{"cannot open file; filename is nullptr", EINVAL, error_category::CRITICAL});
+ }
+
+ auto fd = ::open(fname, oflags);
+
+ if (fd == -1) {
+ return tl::make_unexpected(error{fmt::format("cannot open file {}: {}", fname, ::strerror(errno)), errno});
+ }
+
+ auto ret = raii_file{fname, fd, false};
+
+ if (fstat(ret.fd, &ret.st) == -1) {
+ return tl::make_unexpected(error{fmt::format("cannot stat file {}: {}", fname, ::strerror(errno)), errno});
+ }
+
+ return ret;
+}
+
+auto raii_file::create(const char *fname, int flags, int perms) -> tl::expected<raii_file, error>
+{
+ int oflags = flags | O_CREAT;
+#ifdef O_CLOEXEC
+ oflags |= O_CLOEXEC;
+#endif
+
+ if (fname == nullptr) {
+ return tl::make_unexpected(error{"cannot create file; filename is nullptr", EINVAL, error_category::CRITICAL});
+ }
+
+ auto fd = ::open(fname, oflags, perms);
+
+ if (fd == -1) {
+ return tl::make_unexpected(error{fmt::format("cannot create file {}: {}", fname, ::strerror(errno)), errno});
+ }
+
+ auto ret = raii_file{fname, fd, false};
+
+ if (fstat(ret.fd, &ret.st) == -1) {
+ return tl::make_unexpected(error{fmt::format("cannot stat file {}: {}", fname, ::strerror(errno)), errno});
+ }
+
+ return ret;
+}
+
+auto raii_file::create_temp(const char *fname, int flags, int perms) -> tl::expected<raii_file, error>
+{
+ int oflags = flags;
+#ifdef O_CLOEXEC
+ oflags |= O_CLOEXEC | O_CREAT | O_EXCL;
+#endif
+ if (fname == nullptr) {
+ return tl::make_unexpected(error{"cannot open file; filename is nullptr", EINVAL, error_category::CRITICAL});
+ }
+
+ auto fd = ::open(fname, oflags, perms);
+
+ if (fd == -1) {
+ return tl::make_unexpected(error{fmt::format("cannot create file {}: {}", fname, ::strerror(errno)), errno});
+ }
+
+ auto ret = raii_file{fname, fd, true};
+
+ if (fstat(ret.fd, &ret.st) == -1) {
+ return tl::make_unexpected(error{fmt::format("cannot stat file {}: {}", fname, ::strerror(errno)), errno});
+ }
+
+ return ret;
+}
+
+auto raii_file::mkstemp(const char *pattern, int flags, int perms) -> tl::expected<raii_file, error>
+{
+ int oflags = flags;
+#ifdef O_CLOEXEC
+ oflags |= O_CLOEXEC | O_CREAT | O_EXCL;
+#endif
+ if (pattern == nullptr) {
+ return tl::make_unexpected(error{"cannot open file; pattern is nullptr", EINVAL, error_category::CRITICAL});
+ }
+
+ std::string mutable_pattern = pattern;
+
+ auto fd = g_mkstemp_full(mutable_pattern.data(), oflags, perms);
+
+ if (fd == -1) {
+ return tl::make_unexpected(error{fmt::format("cannot create file {}: {}", pattern, ::strerror(errno)), errno});
+ }
+
+ auto ret = raii_file{mutable_pattern.c_str(), fd, true};
+
+ if (fstat(ret.fd, &ret.st) == -1) {
+ return tl::make_unexpected(error{fmt::format("cannot stat file {}: {}",
+ mutable_pattern, ::strerror(errno)),
+ errno});
+ }
+
+ return ret;
+}
+
+raii_file::~raii_file() noexcept
+{
+ if (fd != -1) {
+ if (temp) {
+ (void) unlink(fname.c_str());
+ }
+ close(fd);
+ }
+}
+
+auto raii_file::update_stat() noexcept -> bool
+{
+ return fstat(fd, &st) != -1;
+}
+
+raii_file::raii_file(const char *fname, int fd, bool temp)
+ : fd(fd), temp(temp)
+{
+ std::size_t nsz;
+
+ /* Normalize path */
+ this->fname = fname;
+ rspamd_normalize_path_inplace(this->fname.data(), this->fname.size(), &nsz);
+ this->fname.resize(nsz);
+}
+
+
+raii_locked_file::~raii_locked_file() noexcept
+{
+ if (fd != -1) {
+ (void) rspamd_file_unlock(fd, FALSE);
+ }
+}
+
+auto raii_locked_file::lock_raii_file(raii_file &&unlocked) -> tl::expected<raii_locked_file, error>
+{
+ if (!rspamd_file_lock(unlocked.get_fd(), TRUE)) {
+ return tl::make_unexpected(
+ error{fmt::format("cannot lock file {}: {}", unlocked.get_name(), ::strerror(errno)), errno});
+ }
+
+ return raii_locked_file{std::move(unlocked)};
+}
+
+auto raii_locked_file::unlock() -> raii_file
+{
+ if (fd != -1) {
+ (void) rspamd_file_unlock(fd, FALSE);
+ }
+
+ return raii_file{static_cast<raii_file &&>(std::move(*this))};
+}
+
+raii_mmaped_file::raii_mmaped_file(raii_file &&file, void *map, std::size_t sz)
+ : file(std::move(file)), map(map), map_size(sz)
+{
+}
+
+auto raii_mmaped_file::mmap_shared(raii_file &&file,
+ int flags, std::int64_t offset) -> tl::expected<raii_mmaped_file, error>
+{
+ void *map;
+
+ if (file.get_stat().st_size < offset || offset < 0) {
+ return tl::make_unexpected(error{
+ fmt::format("cannot mmap file {} due to incorrect offset; offset={}, size={}",
+ file.get_name(), offset, file.get_size()),
+ EINVAL});
+ }
+ /* Update stat on file to ensure it is up-to-date */
+ file.update_stat();
+ map = mmap(nullptr, (std::size_t)(file.get_size() - offset), flags, MAP_SHARED, file.get_fd(), offset);
+
+ if (map == MAP_FAILED) {
+ return tl::make_unexpected(error{fmt::format("cannot mmap file {}: {}",
+ file.get_name(), ::strerror(errno)),
+ errno});
+ }
+
+ return raii_mmaped_file{std::move(file), map, (std::size_t)(file.get_size() - offset)};
+}
+
+auto raii_mmaped_file::mmap_shared(const char *fname, int open_flags,
+ int mmap_flags, std::int64_t offset) -> tl::expected<raii_mmaped_file, error>
+{
+ auto file = raii_file::open(fname, open_flags);
+
+ if (!file.has_value()) {
+ return tl::make_unexpected(file.error());
+ }
+
+ return raii_mmaped_file::mmap_shared(std::move(file.value()), mmap_flags, offset);
+}
+
+raii_mmaped_file::~raii_mmaped_file()
+{
+ if (map != nullptr) {
+ munmap(map, map_size);
+ }
+}
+
+raii_mmaped_file::raii_mmaped_file(raii_mmaped_file &&other) noexcept
+ : file(std::move(other.file))
+{
+ std::swap(map, other.map);
+ std::swap(map_size, other.map_size);
+}
+
+auto raii_file_sink::create(const char *fname, int flags, int perms,
+ const char *suffix) -> tl::expected<raii_file_sink, error>
+{
+ if (!fname || !suffix) {
+ return tl::make_unexpected(error{"cannot create file; filename is nullptr", EINVAL, error_category::CRITICAL});
+ }
+
+ auto tmp_fname = fmt::format("{}.{}", fname, suffix);
+ auto file = raii_locked_file::create(tmp_fname.c_str(), flags, perms);
+
+ if (!file.has_value()) {
+ return tl::make_unexpected(file.error());
+ }
+
+ return raii_file_sink{std::move(file.value()), fname, std::move(tmp_fname)};
+}
+
+auto raii_file_sink::write_output() -> bool
+{
+ if (success) {
+ /* We cannot write output twice */
+ return false;
+ }
+
+ if (rename(tmp_fname.c_str(), output_fname.c_str()) == -1) {
+ return false;
+ }
+
+ success = true;
+
+ return true;
+}
+
+raii_file_sink::~raii_file_sink()
+{
+ if (!success) {
+ /* Unlink sink */
+ unlink(tmp_fname.c_str());
+ }
+}
+
+raii_file_sink::raii_file_sink(raii_locked_file &&_file, const char *_output, std::string &&_tmp_fname)
+ : file(std::move(_file)), output_fname(_output), tmp_fname(std::move(_tmp_fname)), success(false)
+{
+}
+
+raii_file_sink::raii_file_sink(raii_file_sink &&other) noexcept
+ : file(std::move(other.file)),
+ output_fname(std::move(other.output_fname)),
+ tmp_fname(std::move(other.tmp_fname)),
+ success(other.success)
+{
+}
+
+namespace tests {
+template<class T>
+static auto test_read_file(const T &f)
+{
+ auto fd = f.get_fd();
+ (void) ::lseek(fd, 0, SEEK_SET);
+ std::string buf('\0', (std::size_t) f.get_size());
+ ::read(fd, buf.data(), buf.size());
+ return buf;
+}
+template<class T>
+static auto test_write_file(const T &f, const std::string_view &buf)
+{
+ auto fd = f.get_fd();
+ (void) ::lseek(fd, 0, SEEK_SET);
+ return ::write(fd, buf.data(), buf.size());
+}
+auto random_fname(std::string_view extension)
+{
+ const auto *tmpdir = getenv("TMPDIR");
+ if (tmpdir == nullptr) {
+ tmpdir = G_DIR_SEPARATOR_S "tmp";
+ }
+
+ std::string out_fname{tmpdir};
+ out_fname += G_DIR_SEPARATOR_S;
+
+ char hexbuf[32];
+ rspamd_random_hex(hexbuf, sizeof(hexbuf));
+ out_fname.append((const char *) hexbuf, sizeof(hexbuf));
+ if (!extension.empty()) {
+ out_fname.append(".");
+ out_fname.append(extension);
+ }
+
+ return out_fname;
+}
+TEST_SUITE("loked files utils")
+{
+
+ TEST_CASE("create and delete file")
+ {
+ auto fname = random_fname("tmp");
+ {
+ auto raii_locked_file = raii_locked_file::create_temp(fname.c_str(), O_RDONLY, 00600);
+ CHECK(raii_locked_file.has_value());
+ CHECK(raii_locked_file.value().get_extension() == "tmp");
+ CHECK(::access(fname.c_str(), R_OK) == 0);
+ }
+ // File must be deleted after this call
+ auto ret = ::access(fname.c_str(), R_OK);
+ auto serrno = errno;
+ CHECK(ret == -1);
+ CHECK(serrno == ENOENT);
+ // Create one more time
+ {
+ auto raii_locked_file = raii_locked_file::create_temp(fname.c_str(), O_RDONLY, 00600);
+ CHECK(raii_locked_file.has_value());
+ CHECK(::access(fname.c_str(), R_OK) == 0);
+ }
+ ret = ::access(fname.c_str(), R_OK);
+ serrno = errno;
+ CHECK(ret == -1);
+ CHECK(serrno == ENOENT);
+ }
+
+ TEST_CASE("check lock")
+ {
+ auto fname = random_fname("");
+ {
+ auto raii_locked_file = raii_locked_file::create_temp(fname.c_str(), O_RDONLY, 00600);
+ CHECK(raii_locked_file.has_value());
+ CHECK(raii_locked_file.value().get_extension() == "");
+ CHECK(::access(fname.c_str(), R_OK) == 0);
+ auto raii_locked_file2 = raii_locked_file::open(fname.c_str(), O_RDONLY);
+ CHECK(!raii_locked_file2.has_value());
+ CHECK(::access(fname.c_str(), R_OK) == 0);
+ }
+ // File must be deleted after this call
+ auto ret = ::access(fname.c_str(), R_OK);
+ auto serrno = errno;
+ CHECK(ret == -1);
+ CHECK(serrno == ENOENT);
+ }
+
+ auto get_tmpdir()->std::string
+ {
+ const auto *tmpdir = getenv("TMPDIR");
+ if (tmpdir == nullptr) {
+ tmpdir = G_DIR_SEPARATOR_S "tmp";
+ }
+
+ std::size_t sz;
+ std::string mut_fname = tmpdir;
+ rspamd_normalize_path_inplace(mut_fname.data(), mut_fname.size(), &sz);
+ mut_fname.resize(sz);
+
+ if (!mut_fname.ends_with(G_DIR_SEPARATOR)) {
+ mut_fname += G_DIR_SEPARATOR;
+ }
+
+ return mut_fname;
+ }
+
+ TEST_CASE("tempfile")
+ {
+ std::string tmpname;
+ const std::string tmpdir{get_tmpdir()};
+ {
+ auto raii_locked_file = raii_locked_file::mkstemp(std::string(tmpdir + G_DIR_SEPARATOR_S + "doctest-XXXXXXXX").c_str(),
+ O_RDONLY, 00600);
+ CHECK(raii_locked_file.has_value());
+ CHECK(raii_locked_file.value().get_dir() == tmpdir);
+ CHECK(access(raii_locked_file.value().get_name().data(), R_OK) == 0);
+ auto raii_locked_file2 = raii_locked_file::open(raii_locked_file.value().get_name().data(), O_RDONLY);
+ CHECK(!raii_locked_file2.has_value());
+ CHECK(access(raii_locked_file.value().get_name().data(), R_OK) == 0);
+ tmpname = raii_locked_file.value().get_name();
+ }
+ // File must be deleted after this call
+ auto ret = ::access(tmpname.c_str(), R_OK);
+ auto serrno = errno;
+ CHECK(ret == -1);
+ CHECK(serrno == ENOENT);
+ }
+
+ TEST_CASE("mmap")
+ {
+ std::string tmpname;
+ const std::string tmpdir{get_tmpdir()};
+ {
+ auto raii_file = raii_file::mkstemp(std::string(tmpdir + G_DIR_SEPARATOR_S + "doctest-XXXXXXXX").c_str(),
+ O_RDWR | O_CREAT | O_EXCL, 00600);
+ CHECK(raii_file.has_value());
+ CHECK(raii_file->get_dir() == tmpdir);
+ CHECK(access(raii_file->get_name().data(), R_OK) == 0);
+ tmpname = std::string{raii_file->get_name()};
+ char payload[] = {'1', '2', '3'};
+ CHECK(write(raii_file->get_fd(), payload, sizeof(payload)) == sizeof(payload));
+ auto mmapped_file1 = raii_mmaped_file::mmap_shared(std::move(raii_file.value()), PROT_READ | PROT_WRITE);
+ CHECK(mmapped_file1.has_value());
+ CHECK(!raii_file->is_valid());
+ CHECK(mmapped_file1->get_size() == sizeof(payload));
+ CHECK(memcmp(mmapped_file1->get_map(), payload, sizeof(payload)) == 0);
+ *(char *) mmapped_file1->get_map() = '2';
+ auto mmapped_file2 = raii_mmaped_file::mmap_shared(tmpname.c_str(), O_RDONLY, PROT_READ);
+ CHECK(mmapped_file2.has_value());
+ CHECK(mmapped_file2->get_size() == sizeof(payload));
+ CHECK(memcmp(mmapped_file2->get_map(), payload, sizeof(payload)) != 0);
+ CHECK(memcmp(mmapped_file2->get_map(), mmapped_file1->get_map(), sizeof(payload)) == 0);
+ }
+ // File must be deleted after this call
+ auto ret = ::access(tmpname.c_str(), R_OK);
+ auto serrno = errno;
+ CHECK(ret == -1);
+ CHECK(serrno == ENOENT);
+ }
+
+}// TEST_SUITE
+
+}// namespace tests
+
+}// namespace rspamd::util
diff --git a/src/libutil/cxx/file_util.hxx b/src/libutil/cxx/file_util.hxx
new file mode 100644
index 0000000..4528905
--- /dev/null
+++ b/src/libutil/cxx/file_util.hxx
@@ -0,0 +1,312 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_FILE_UTIL_HXX
+#define RSPAMD_FILE_UTIL_HXX
+#pragma once
+
+#include "config.h"
+#include "contrib/expected/expected.hpp"
+#include "libutil/cxx/error.hxx"
+#include <string>
+#include <sys/stat.h>
+
+namespace rspamd::util {
+/**
+ * A simple RAII object to contain a move only file descriptor
+ * A file is unlocked and closed when not needed
+ */
+struct raii_file {
+public:
+ virtual ~raii_file() noexcept;
+
+ static auto open(const char *fname, int flags) -> tl::expected<raii_file, error>;
+ static auto open(const std::string &fname, int flags) -> tl::expected<raii_file, error>
+ {
+ return open(fname.c_str(), flags);
+ };
+ static auto create(const char *fname, int flags, int perms) -> tl::expected<raii_file, error>;
+ static auto create(const std::string &fname, int flags, int perms) -> tl::expected<raii_file, error>
+ {
+ return create(fname.c_str(), flags, perms);
+ };
+
+ static auto create_temp(const char *fname, int flags, int perms) -> tl::expected<raii_file, error>;
+ static auto mkstemp(const char *pattern, int flags, int perms) -> tl::expected<raii_file, error>;
+
+ auto get_fd() const -> int
+ {
+ return fd;
+ }
+
+ auto get_stat() const -> const struct stat &
+ {
+ return st;
+ };
+
+ auto get_size() const -> std::size_t
+ {
+ return st.st_size;
+ };
+
+ auto get_name() const -> std::string_view
+ {
+ return std::string_view{fname};
+ }
+
+ auto get_dir() const -> std::string_view
+ {
+ auto sep_pos = fname.rfind(G_DIR_SEPARATOR);
+
+ if (sep_pos == std::string::npos) {
+ return std::string_view{fname};
+ }
+
+ while (sep_pos >= 1 && fname[sep_pos - 1] == G_DIR_SEPARATOR) {
+ sep_pos--;
+ }
+
+ return std::string_view{fname.c_str(), sep_pos + 1};
+ }
+
+ auto get_extension() const -> std::string_view
+ {
+ auto sep_pos = fname.rfind(G_DIR_SEPARATOR);
+
+ if (sep_pos == std::string::npos) {
+ sep_pos = 0;
+ }
+
+ auto filename = std::string_view{fname.c_str() + sep_pos};
+ auto dot_pos = filename.find('.');
+
+ if (dot_pos == std::string::npos) {
+ return std::string_view{};
+ }
+ else {
+ return std::string_view{filename.data() + dot_pos + 1, filename.size() - dot_pos - 1};
+ }
+ }
+
+ raii_file &operator=(raii_file &&other) noexcept
+ {
+ std::swap(fd, other.fd);
+ std::swap(temp, other.temp);
+ std::swap(fname, other.fname);
+ std::swap(st, other.st);
+
+ return *this;
+ }
+
+ raii_file(raii_file &&other) noexcept
+ {
+ *this = std::move(other);
+ }
+
+ /**
+ * Prevent file from being deleted
+ * @return
+ */
+ auto make_immortal() noexcept
+ {
+ temp = false;
+ }
+
+ /**
+ * Performs fstat on an opened file to refresh internal stat
+ * @return
+ */
+ auto update_stat() noexcept -> bool;
+
+ auto is_valid() noexcept -> bool
+ {
+ return fd != -1;
+ }
+
+ /* Do not allow copy/default ctor */
+ const raii_file &operator=(const raii_file &other) = delete;
+ raii_file() = delete;
+ raii_file(const raii_file &other) = delete;
+
+protected:
+ int fd = -1;
+ bool temp;
+ std::string fname;
+ struct stat st;
+
+ explicit raii_file(const char *fname, int fd, bool temp);
+};
+/**
+ * A simple RAII object to contain a file descriptor with an flock wrap
+ * A file is unlocked and closed when not needed
+ */
+struct raii_locked_file final : public raii_file {
+public:
+ ~raii_locked_file() noexcept override;
+
+ static auto open(const char *fname, int flags) -> tl::expected<raii_locked_file, error>
+ {
+ auto locked = raii_file::open(fname, flags).and_then([]<class T>(T &&file) {
+ return lock_raii_file(std::forward<T>(file));
+ });
+
+ return locked;
+ }
+ static auto create(const char *fname, int flags, int perms) -> tl::expected<raii_locked_file, error>
+ {
+ auto locked = raii_file::create(fname, flags, perms).and_then([]<class T>(T &&file) {
+ return lock_raii_file(std::forward<T>(file));
+ });
+
+ return locked;
+ }
+ static auto create_temp(const char *fname, int flags, int perms) -> tl::expected<raii_locked_file, error>
+ {
+ auto locked = raii_file::create_temp(fname, flags, perms).and_then([]<class T>(T &&file) {
+ return lock_raii_file(std::forward<T>(file));
+ });
+
+ return locked;
+ }
+ static auto mkstemp(const char *pattern, int flags, int perms) -> tl::expected<raii_locked_file, error>
+ {
+ auto locked = raii_file::mkstemp(pattern, flags, perms).and_then([]<class T>(T &&file) {
+ return lock_raii_file(std::forward<T>(file));
+ });
+
+ return locked;
+ }
+
+ raii_locked_file &operator=(raii_locked_file &&other) noexcept
+ {
+ std::swap(fd, other.fd);
+ std::swap(temp, other.temp);
+ std::swap(fname, other.fname);
+ std::swap(st, other.st);
+
+ return *this;
+ }
+
+ /**
+ * Unlock a locked file and return back unlocked file transferring ownership.
+ * A locked file cannot be used after this method.
+ */
+ auto unlock() -> raii_file;
+
+ raii_locked_file(raii_locked_file &&other) noexcept
+ : raii_file(static_cast<raii_file &&>(std::move(other)))
+ {
+ }
+ /* Do not allow copy/default ctor */
+ const raii_locked_file &operator=(const raii_locked_file &other) = delete;
+ raii_locked_file() = delete;
+ raii_locked_file(const raii_locked_file &other) = delete;
+
+private:
+ static auto lock_raii_file(raii_file &&unlocked) -> tl::expected<raii_locked_file, error>;
+ raii_locked_file(raii_file &&other) noexcept
+ : raii_file(std::move(other))
+ {
+ }
+ explicit raii_locked_file(const char *fname, int fd, bool temp)
+ : raii_file(fname, fd, temp)
+ {
+ }
+};
+
+/**
+ * A mmap wrapper on top of a locked file
+ */
+struct raii_mmaped_file final {
+ ~raii_mmaped_file();
+ static auto mmap_shared(raii_file &&file, int flags, std::int64_t offset = 0) -> tl::expected<raii_mmaped_file, error>;
+ static auto mmap_shared(const char *fname, int open_flags, int mmap_flags, std::int64_t offset = 0) -> tl::expected<raii_mmaped_file, error>;
+ // Returns a constant pointer to the underlying map
+ auto get_map() const -> void *
+ {
+ return map;
+ }
+ auto get_file() const -> const raii_file &
+ {
+ return file;
+ }
+ // Passes the ownership of the mmaped memory to the callee
+ auto steal_map() -> std::tuple<void *, std::size_t>
+ {
+ auto ret = std::make_tuple(this->map, map_size);
+ this->map = nullptr;
+ return ret;
+ }
+
+ auto get_size() const -> std::size_t
+ {
+ return file.get_stat().st_size;
+ }
+
+ raii_mmaped_file &operator=(raii_mmaped_file &&other) noexcept
+ {
+ std::swap(map, other.map);
+ std::swap(map_size, other.map_size);
+ file = std::move(other.file);
+
+ return *this;
+ }
+
+ raii_mmaped_file(raii_mmaped_file &&other) noexcept;
+
+ /* Do not allow copy/default ctor */
+ const raii_mmaped_file &operator=(const raii_mmaped_file &other) = delete;
+ raii_mmaped_file() = delete;
+ raii_mmaped_file(const raii_mmaped_file &other) = delete;
+
+private:
+ /* Is intended to be used with map_shared */
+ explicit raii_mmaped_file(raii_file &&_file, void *_map, std::size_t sz);
+ raii_file file;
+ void *map = nullptr;
+ std::size_t map_size;
+};
+
+/**
+ * A helper to have a file to write that will be renamed to the
+ * target file if successful or deleted in the case of failure
+ */
+struct raii_file_sink final {
+ static auto create(const char *fname, int flags, int perms, const char *suffix = "new")
+ -> tl::expected<raii_file_sink, error>;
+ auto write_output() -> bool;
+ ~raii_file_sink();
+ auto get_fd() const -> int
+ {
+ return file.get_fd();
+ }
+
+ raii_file_sink(raii_file_sink &&other) noexcept;
+ /* Do not allow copy/default ctor */
+ const raii_file_sink &operator=(const raii_file_sink &other) = delete;
+ raii_file_sink() = delete;
+ raii_file_sink(const raii_file_sink &other) = delete;
+
+private:
+ explicit raii_file_sink(raii_locked_file &&_file, const char *_output, std::string &&_tmp_fname);
+ raii_locked_file file;
+ std::string output_fname;
+ std::string tmp_fname;
+ bool success;
+};
+
+}// namespace rspamd::util
+
+#endif//RSPAMD_FILE_UTIL_HXX
diff --git a/src/libutil/cxx/hash_util.hxx b/src/libutil/cxx/hash_util.hxx
new file mode 100644
index 0000000..05f3d97
--- /dev/null
+++ b/src/libutil/cxx/hash_util.hxx
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_HASH_UTIL_HXX
+#define RSPAMD_HASH_UTIL_HXX
+
+#pragma once
+
+#include <string_view>
+#include <string>
+#include "contrib/ankerl/unordered_dense.h"
+
+
+namespace rspamd {
+/*
+ * Transparent smart pointers hashing
+ */
+template<typename T>
+struct smart_ptr_equal {
+ using is_transparent = void; /* We want to find values in a set of shared_ptr by reference */
+ auto operator()(const std::shared_ptr<T> &a, const std::shared_ptr<T> &b) const
+ {
+ return (*a) == (*b);
+ }
+ auto operator()(const std::shared_ptr<T> &a, const T &b) const
+ {
+ return (*a) == b;
+ }
+ auto operator()(const T &a, const std::shared_ptr<T> &b) const
+ {
+ return a == (*b);
+ }
+ auto operator()(const std::unique_ptr<T> &a, const std::unique_ptr<T> &b) const
+ {
+ return (*a) == (*b);
+ }
+ auto operator()(const std::unique_ptr<T> &a, const T &b) const
+ {
+ return (*a) == b;
+ }
+ auto operator()(const T &a, const std::unique_ptr<T> &b) const
+ {
+ return a == (*b);
+ }
+};
+
+template<typename T>
+struct smart_ptr_hash {
+ using is_transparent = void; /* We want to find values in a set of shared_ptr by reference */
+ using is_avalanching = void;
+ auto operator()(const std::shared_ptr<T> &a) const
+ {
+ return std::hash<T>()(*a);
+ }
+ auto operator()(const std::unique_ptr<T> &a) const
+ {
+ return std::hash<T>()(*a);
+ }
+ auto operator()(const T &a) const
+ {
+ return std::hash<T>()(a);
+ }
+};
+
+/* Enable lookup by string view */
+struct smart_str_equal {
+ using is_transparent = void;
+ auto operator()(const std::string &a, const std::string &b) const
+ {
+ return a == b;
+ }
+ auto operator()(const std::string_view &a, const std::string &b) const
+ {
+ return a == b;
+ }
+ auto operator()(const std::string &a, const std::string_view &b) const
+ {
+ return a == b;
+ }
+};
+
+struct smart_str_hash {
+ using is_transparent = void;
+ using is_avalanching = void;
+ auto operator()(const std::string &a) const
+ {
+ return ankerl::unordered_dense::hash<std::string>()(a);
+ }
+ auto operator()(const std::string_view &a) const
+ {
+ return ankerl::unordered_dense::hash<std::string_view>()(a);
+ }
+};
+
+}// namespace rspamd
+
+#endif//RSPAMD_HASH_UTIL_HXX
diff --git a/src/libutil/cxx/local_shared_ptr.hxx b/src/libutil/cxx/local_shared_ptr.hxx
new file mode 100644
index 0000000..78ed5ba
--- /dev/null
+++ b/src/libutil/cxx/local_shared_ptr.hxx
@@ -0,0 +1,440 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_LOCAL_SHARED_PTR_HXX
+#define RSPAMD_LOCAL_SHARED_PTR_HXX
+
+#pragma once
+
+#include <memory>
+#include <algorithm> // for std::swap
+#include <cstddef> // for std::size_t
+#include <functional>// for std::less
+
+/*
+ * Smart pointers with no atomic refcounts to speed up Rspamd which is
+ * apparently single threaded
+ */
+namespace rspamd {
+
+namespace detail {
+
+class ref_cnt {
+public:
+ using refcount_t = int;
+
+ constexpr auto add_shared() -> refcount_t
+ {
+ return ++ref_shared;
+ }
+ constexpr auto add_weak() -> refcount_t
+ {
+ return ++ref_weak;
+ }
+ constexpr auto release_shared() -> refcount_t
+ {
+ return --ref_shared;
+ }
+ constexpr auto release_weak() -> refcount_t
+ {
+ return --ref_weak;
+ }
+ constexpr auto shared_count() const -> refcount_t
+ {
+ return ref_shared;
+ }
+ constexpr auto weak_count() const -> refcount_t
+ {
+ return ref_weak;
+ }
+ virtual ~ref_cnt()
+ {
+ }
+ virtual void dispose() = 0;
+
+private:
+ refcount_t ref_weak = 0;
+ refcount_t ref_shared = 1;
+};
+
+template<class T>
+class obj_and_refcnt : public ref_cnt {
+private:
+ typedef typename std::aligned_storage<sizeof(T), std::alignment_of<T>::value>::type storage_type;
+ storage_type storage;
+ bool initialized;
+ virtual void dispose() override
+ {
+ if (initialized) {
+ T *p = reinterpret_cast<T *>(&storage);
+ p->~T();
+ initialized = false;
+ }
+ }
+
+public:
+ template<typename... Args>
+ explicit obj_and_refcnt(Args &&...args)
+ : initialized(true)
+ {
+ new (&storage) T(std::forward<Args>(args)...);
+ }
+ auto get(void) -> T *
+ {
+ if (initialized) {
+ return reinterpret_cast<T *>(&storage);
+ }
+
+ return nullptr;
+ }
+ virtual ~obj_and_refcnt() = default;
+};
+
+template<class T, class D = typename std::default_delete<T>>
+class ptr_and_refcnt : public ref_cnt {
+private:
+ T *ptr;
+ D deleter;
+ virtual void dispose() override
+ {
+ deleter(ptr);
+ ptr = nullptr;
+ }
+
+public:
+ explicit ptr_and_refcnt(T *_ptr, D &&d = std::default_delete<T>())
+ : ptr(_ptr),
+ deleter(std::move(d))
+ {
+ }
+ virtual ~ptr_and_refcnt() = default;
+};
+
+}// namespace detail
+
+template<class T>
+class local_weak_ptr;
+
+template<class T>
+class local_shared_ptr {
+public:
+ typedef T element_type;
+ typedef local_weak_ptr<T> weak_type;
+
+ // Simplified comparing to libc++, no custom deleter and no rebind here
+ // constructors:
+ constexpr local_shared_ptr() noexcept
+ : px(nullptr), cnt(nullptr)
+ {
+ }
+
+ template<class Y, typename std::enable_if<
+ std::is_convertible<Y *, element_type *>::value, bool>::type = true>
+ explicit local_shared_ptr(Y *p)
+ : px(p), cnt(new detail::ptr_and_refcnt(p))
+ {
+ }
+
+ // custom deleter
+ template<class Y, class D, typename std::enable_if<std::is_convertible<Y *, element_type *>::value, bool>::type = true>
+ explicit local_shared_ptr(Y *p, D &&d)
+ : px(p), cnt(new detail::ptr_and_refcnt<Y, D>(p, std::forward<D>(d)))
+ {
+ }
+
+ local_shared_ptr(const local_shared_ptr &r) noexcept
+ : px(r.px), cnt(r.cnt)
+ {
+ if (cnt) {
+ cnt->add_shared();
+ }
+ }
+ local_shared_ptr(local_shared_ptr &&r) noexcept
+ : px(r.px), cnt(r.cnt)
+ {
+ r.px = nullptr;
+ r.cnt = nullptr;
+ }
+ template<class Y>
+ explicit local_shared_ptr(const local_weak_ptr<Y> &r)
+ : px(r.px), cnt(r.cnt)
+ {
+ if (cnt) {
+ cnt->add_shared();
+ }
+ }
+ local_shared_ptr(std::nullptr_t)
+ : local_shared_ptr()
+ {
+ }
+
+ ~local_shared_ptr()
+ {
+ if (cnt) {
+ if (cnt->release_shared() <= 0) {
+ cnt->dispose();
+
+ if (cnt->weak_count() == 0) {
+ delete cnt;
+ }
+ }
+ }
+ }
+
+ // assignment:
+ local_shared_ptr &operator=(const local_shared_ptr &r) noexcept
+ {
+ local_shared_ptr(r).swap(*this);
+ return *this;
+ }
+ local_shared_ptr &operator=(local_shared_ptr &&r) noexcept
+ {
+ local_shared_ptr(std::move(r)).swap(*this);
+ return *this;
+ }
+
+ // Mutators
+ void swap(local_shared_ptr &r) noexcept
+ {
+ std::swap(this->cnt, r.cnt);
+ std::swap(this->px, r.px);
+ }
+ void reset() noexcept
+ {
+ local_shared_ptr().swap(*this);
+ }
+
+ // Observers:
+ T *get() const noexcept
+ {
+ return px;
+ }
+
+ T &operator*() const noexcept
+ {
+ return *px;
+ }
+ T *operator->() const noexcept
+ {
+ return px;
+ }
+ long use_count() const noexcept
+ {
+ if (cnt) {
+ return cnt->shared_count();
+ }
+
+ return 0;
+ }
+ bool unique() const noexcept
+ {
+ return use_count() == 1;
+ }
+
+ explicit operator bool() const noexcept
+ {
+ return px != nullptr;
+ }
+
+ template<class Y, typename std::enable_if<
+ std::is_convertible<Y *, element_type *>::value, bool>::type = true>
+ auto operator==(const local_shared_ptr<Y> &other) const -> bool
+ {
+ return px == other.px;
+ }
+
+ template<class Y, typename std::enable_if<
+ std::is_convertible<Y *, element_type *>::value, bool>::type = true>
+ auto operator<(const local_shared_ptr<Y> &other) const -> auto
+ {
+ return *px < *other.px;
+ }
+
+private:
+ T *px;// contained pointer
+ detail::ref_cnt *cnt;
+
+ template<class _T, class... Args>
+ friend local_shared_ptr<_T> local_make_shared(Args &&...args);
+ friend class local_weak_ptr<T>;
+};
+
+template<class T, class... Args>
+local_shared_ptr<T> local_make_shared(Args &&...args)
+{
+ local_shared_ptr<T> ptr;
+ auto tmp_object = new detail::obj_and_refcnt<T>(std::forward<Args>(args)...);
+ ptr.px = tmp_object->get();
+ ptr.cnt = tmp_object;
+
+ return ptr;
+}
+
+template<class T>
+class local_weak_ptr {
+public:
+ typedef T element_type;
+
+ // constructors
+ constexpr local_weak_ptr() noexcept
+ : px(nullptr), cnt(nullptr)
+ {
+ }
+ template<class Y, typename std::enable_if<
+ std::is_convertible<Y *, element_type *>::value, bool>::type = true>
+ local_weak_ptr(local_shared_ptr<Y> const &r) noexcept
+ : px(r.px), cnt(r.cnt)
+ {
+ if (cnt) {
+ cnt->add_weak();
+ }
+ }
+
+ local_weak_ptr(local_weak_ptr const &r) noexcept
+ : px(r.px), cnt(r.cnt)
+ {
+ if (cnt) {
+ cnt->add_weak();
+ }
+ }
+ local_weak_ptr(local_weak_ptr &&r) noexcept
+ : px(r.px), cnt(r.cnt)
+ {
+ r.px = nullptr;
+ r.cnt = nullptr;
+ }
+
+ ~local_weak_ptr()
+ {
+ if (cnt) {
+ if (cnt->release_weak() <= 0 && cnt->shared_count() == 0) {
+ delete cnt;
+ }
+ }
+ }
+
+ // assignment
+ local_weak_ptr &operator=(local_weak_ptr const &r) noexcept
+ {
+ local_weak_ptr(r).swap(*this);
+ return *this;
+ }
+ local_weak_ptr &operator=(local_shared_ptr<T> const &r) noexcept
+ {
+ local_weak_ptr(r).swap(*this);
+ return *this;
+ }
+
+ template<class Y, typename std::enable_if<
+ std::is_convertible<Y *, element_type *>::value, bool>::type = true>
+ local_weak_ptr &operator=(local_weak_ptr<Y> const &r) noexcept
+ {
+ local_weak_ptr(r).swap(*this);
+ return *this;
+ }
+ local_weak_ptr &operator=(local_weak_ptr &&r) noexcept
+ {
+ local_weak_ptr(std::move(r)).swap(*this);
+ return *this;
+ }
+
+ // modifiers
+ void swap(local_weak_ptr &r) noexcept
+ {
+ std::swap(this->cnt, r.cnt);
+ std::swap(this->px, r.px);
+ }
+ void reset() noexcept
+ {
+ local_weak_ptr().swap(*this);
+ }
+
+ // observers
+ long use_count() const noexcept
+ {
+ if (cnt) {
+ return cnt->shared_count();
+ }
+ return 0;
+ }
+ bool expired() const noexcept
+ {
+ if (cnt) {
+ return cnt->shared_count() == 0;
+ }
+
+ return true;
+ }
+
+ local_shared_ptr<T> lock() const noexcept
+ {
+ local_shared_ptr<T> tmp;
+ tmp.cnt = cnt;
+
+ if (cnt) {
+ cnt->add_shared();
+ tmp.px = px;
+ }
+
+ return tmp;
+ }
+
+private:
+ element_type *px;
+ detail::ref_cnt *cnt;
+};
+
+
+}// namespace rspamd
+
+/* Hashing stuff */
+namespace std {
+template<class T>
+struct hash<rspamd::local_shared_ptr<T>> {
+ inline auto operator()(const rspamd::local_shared_ptr<T> &p) const -> auto
+ {
+ if (!p) {
+ throw std::logic_error("no hash for dangling pointer");
+ }
+ return hash<T>()(*p.get());
+ }
+};
+template<class T>
+struct hash<rspamd::local_weak_ptr<T>> {
+ inline auto operator()(const rspamd::local_weak_ptr<T> &p) const -> auto
+ {
+ if (!p) {
+ throw std::logic_error("no hash for dangling pointer");
+ }
+ return hash<T>()(*p.get());
+ }
+};
+
+template<class T>
+inline void swap(rspamd::local_shared_ptr<T> &x, rspamd::local_shared_ptr<T> &y) noexcept
+{
+ x.swap(y);
+}
+
+template<class T>
+inline void swap(rspamd::local_weak_ptr<T> &x, rspamd::local_weak_ptr<T> &y) noexcept
+{
+ x.swap(y);
+}
+
+}// namespace std
+
+#endif//RSPAMD_LOCAL_SHARED_PTR_HXX
diff --git a/src/libutil/cxx/utf8_util.cxx b/src/libutil/cxx/utf8_util.cxx
new file mode 100644
index 0000000..5fc83ca
--- /dev/null
+++ b/src/libutil/cxx/utf8_util.cxx
@@ -0,0 +1,421 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define U_CHARSET_IS_UTF8 1
+#include <unicode/utypes.h>
+#include <unicode/utf8.h>
+#include <unicode/uchar.h>
+#include <unicode/normalizer2.h>
+#include <unicode/schriter.h>
+#include <unicode/coll.h>
+#include <unicode/translit.h>
+#include <utility>
+#include <tuple>
+#include <string>
+#include <limits>
+#include <memory>
+
+#include "utf8_util.h"
+#include "str_util.h"
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+const char *
+rspamd_string_unicode_trim_inplace(const char *str, size_t *len)
+{
+ const auto *p = str, *end = str + *len;
+ auto i = 0;
+
+ while (i < *len) {
+ UChar32 uc;
+ auto prev_i = i;
+
+ U8_NEXT(p, i, *len, uc);
+
+ if (!u_isUWhiteSpace(uc) && !IS_ZERO_WIDTH_SPACE(uc)) {
+ i = prev_i;
+ break;
+ }
+ }
+
+ p += i;
+ (*len) -= i;
+ i = end - p;
+ auto *ret = p;
+
+ if (i > 0) {
+
+ while (i > 0) {
+ UChar32 uc;
+ auto prev_i = i;
+
+ U8_PREV(p, 0, i, uc);
+
+ if (!u_isUWhiteSpace(uc) && !IS_ZERO_WIDTH_SPACE(uc)) {
+ i = prev_i;
+ break;
+ }
+ }
+
+ *len = i;
+ }
+
+ return ret;
+}
+
+enum rspamd_utf8_normalise_result
+rspamd_normalise_unicode_inplace(char *start, size_t *len)
+{
+ UErrorCode uc_err = U_ZERO_ERROR;
+ const auto *nfkc_norm = icu::Normalizer2::getNFKCInstance(uc_err);
+ static icu::UnicodeSet zw_spaces{};
+
+ if (!zw_spaces.isFrozen()) {
+ /* Add zw spaces to the set */
+ zw_spaces.add(0x200B);
+ /* TODO: ZW non joiner, it might be used for ligatures, so it should possibly be excluded as well */
+ zw_spaces.add(0x200C);
+ /* See github issue #4290 for explanation. It seems that the ZWJ has many legit use cases */
+ //zw_spaces.add(0x200D);
+ zw_spaces.add(0xFEF);
+ zw_spaces.add(0x00AD);
+ zw_spaces.freeze();
+ }
+
+ int ret = RSPAMD_UNICODE_NORM_NORMAL;
+
+ g_assert(U_SUCCESS(uc_err));
+
+ auto uc_string = icu::UnicodeString::fromUTF8(icu::StringPiece(start, *len));
+ auto is_normal = nfkc_norm->quickCheck(uc_string, uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ return RSPAMD_UNICODE_NORM_ERROR;
+ }
+
+ /* Filter zero width spaces and push resulting string back */
+ const auto filter_zw_spaces_and_push_back = [&](const icu::UnicodeString &input) -> size_t {
+ icu::StringCharacterIterator it{input};
+ size_t i = 0;
+
+ while (it.hasNext()) {
+ /* libicu is very 'special' if it comes to 'safe' macro */
+ if (i >= *len) {
+ ret |= RSPAMD_UNICODE_NORM_ERROR;
+ break;
+ }
+
+ auto uc = it.next32PostInc();
+
+ if (zw_spaces.contains(uc)) {
+ ret |= RSPAMD_UNICODE_NORM_ZERO_SPACES;
+ }
+ else {
+ UBool err = 0;
+
+ if (uc == 0xFFFD) {
+ ret |= RSPAMD_UNICODE_NORM_UNNORMAL;
+ }
+ U8_APPEND((uint8_t *) start, i, *len, uc, err);
+
+ if (err) {
+ ret |= RSPAMD_UNICODE_NORM_ERROR;
+ break;
+ }
+ }
+ }
+
+ return i;
+ };
+
+ if (is_normal != UNORM_YES) {
+ /* Need to normalise */
+ ret |= RSPAMD_UNICODE_NORM_UNNORMAL;
+
+ auto normalised = nfkc_norm->normalize(uc_string, uc_err);
+
+ if (!U_SUCCESS(uc_err)) {
+ return RSPAMD_UNICODE_NORM_ERROR;
+ }
+
+ *len = filter_zw_spaces_and_push_back(normalised);
+ }
+ else {
+ *len = filter_zw_spaces_and_push_back(uc_string);
+ }
+
+ return static_cast<enum rspamd_utf8_normalise_result>(ret);
+}
+
+gchar *
+rspamd_utf8_transliterate(const gchar *start, gsize len, gsize *target_len)
+{
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ static std::unique_ptr<icu::Transliterator> transliterator;
+
+ if (!transliterator) {
+ UParseError parse_err;
+ static const auto rules = icu::UnicodeString{":: Any-Latin;"
+ ":: [:Nonspacing Mark:] Remove;"
+ ":: [:Punctuation:] Remove;"
+ ":: [:Symbol:] Remove;"
+ ":: [:Format:] Remove;"
+ ":: Latin-ASCII;"
+ ":: Lower();"
+ ":: NULL;"
+ "[:Space Separator:] > ' '"};
+ transliterator = std::unique_ptr<icu::Transliterator>(
+ icu::Transliterator::createFromRules("RspamdTranslit", rules, UTRANS_FORWARD, parse_err, uc_err));
+
+ if (U_FAILURE(uc_err) || !transliterator) {
+ auto context = icu::UnicodeString(parse_err.postContext, sizeof(parse_err.preContext) / sizeof(UChar));
+ g_error("fatal error: cannot init libicu transliteration engine: %s, line: %d, offset: %d",
+ u_errorName(uc_err), parse_err.line, parse_err.offset);
+ abort();
+ }
+ }
+
+ auto uc_string = icu::UnicodeString::fromUTF8(icu::StringPiece(start, len));
+ transliterator->transliterate(uc_string);
+
+ // We assume that all characters are now ascii
+ auto dest_len = uc_string.length();
+ gchar *dest = (gchar *) g_malloc(dest_len + 1);
+ auto sink = icu::CheckedArrayByteSink(dest, dest_len);
+ uc_string.toUTF8(sink);
+
+ *target_len = sink.NumberOfBytesWritten();
+ dest[*target_len] = '\0';
+
+ return dest;
+}
+
+struct rspamd_icu_collate_storage {
+ icu::Collator *collator = nullptr;
+ rspamd_icu_collate_storage()
+ {
+ UErrorCode uc_err = U_ZERO_ERROR;
+ collator = icu::Collator::createInstance(icu::Locale::getEnglish(), uc_err);
+
+ if (U_FAILURE(uc_err) || collator == nullptr) {
+ g_error("fatal error: cannot init libicu collation engine: %s",
+ u_errorName(uc_err));
+ abort();
+ }
+ /* Ignore all difference except functional */
+ collator->setStrength(icu::Collator::PRIMARY);
+ }
+
+ ~rspamd_icu_collate_storage()
+ {
+ if (collator) {
+ delete collator;
+ }
+ }
+};
+
+static rspamd_icu_collate_storage collate_storage;
+
+int rspamd_utf8_strcmp_sizes(const char *s1, gsize n1, const char *s2, gsize n2)
+{
+ if (n1 >= std::numeric_limits<int>::max() || n2 >= std::numeric_limits<int>::max()) {
+ /*
+ * It's hard to say what to do here... But libicu wants int, so we fall
+ * back to g_ascii_strcasecmp which can deal with size_t
+ */
+ if (n1 == n2) {
+ return g_ascii_strncasecmp(s1, s2, n1);
+ }
+ else {
+ return n1 - n2;
+ }
+ }
+
+ UErrorCode success = U_ZERO_ERROR;
+ auto res = collate_storage.collator->compareUTF8({s1, (int) n1}, {s2, (int) n2},
+ success);
+
+ switch (res) {
+ case UCOL_EQUAL:
+ return 0;
+ case UCOL_GREATER:
+ return 1;
+ case UCOL_LESS:
+ default:
+ return -1;
+ }
+}
+
+int rspamd_utf8_strcmp(const char *s1, const char *s2, gsize n)
+{
+ return rspamd_utf8_strcmp_sizes(s1, n, s2, n);
+}
+
+TEST_SUITE("utf8 utils")
+{
+ TEST_CASE("utf8 normalise")
+ {
+ std::tuple<const char *, const char *, int> cases[] = {
+ {"abc", "abc", RSPAMD_UNICODE_NORM_NORMAL},
+ {"теÑÑ‚", "теÑÑ‚", RSPAMD_UNICODE_NORM_NORMAL},
+ /* Zero width spaces */
+ {"\xE2\x80\x8B"
+ "те"
+ "\xE2\x80\x8B"
+ "ÑÑ‚",
+ "теÑÑ‚", RSPAMD_UNICODE_NORM_ZERO_SPACES},
+ /* Special case of diacritic */
+ {"13_\u0020\u0308\u0301\u038e\u03ab", "13_ ̈ÌΎΫ", RSPAMD_UNICODE_NORM_UNNORMAL},
+ // String containing a non-joiner character
+ {"س\u200Cت", "ست", RSPAMD_UNICODE_NORM_ZERO_SPACES},
+ // String containing a soft hyphen
+ {"in\u00ADter\u00ADest\u00ADing", "interesting", RSPAMD_UNICODE_NORM_ZERO_SPACES},
+ // String with ligature
+ {"ï¬sh", "fish", RSPAMD_UNICODE_NORM_UNNORMAL},
+ // String with accented characters and zero-width spaces
+ {"café\u200Blatté\u200C", "cafélatté", RSPAMD_UNICODE_NORM_ZERO_SPACES},
+ /* Same with zw spaces */
+ {"13\u200C_\u0020\u0308\u0301\u038e\u03ab", "13_ ̈ÌΎΫ",
+ RSPAMD_UNICODE_NORM_UNNORMAL | RSPAMD_UNICODE_NORM_ZERO_SPACES},
+ /* Buffer overflow case */
+ {"u\xC2\xC2\xC2\xC2\xC2\xC2"
+ "abcdef"
+ "abcdef",
+ "u\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD",
+ RSPAMD_UNICODE_NORM_UNNORMAL | RSPAMD_UNICODE_NORM_ERROR},
+ // String with a mix of special characters, ligatures, and zero-width spaces
+ {"ï¬sh\u200Bcafé\u200C\u200Dlatté\u200D\u00AD", "fishcafé\u200Dlatté\u200D", RSPAMD_UNICODE_NORM_UNNORMAL | RSPAMD_UNICODE_NORM_ZERO_SPACES},
+ // Empty string
+ {"", "", RSPAMD_UNICODE_NORM_NORMAL},
+ };
+
+ for (const auto &c: cases) {
+ std::string cpy{std::get<0>(c)};
+ auto ns = cpy.size();
+ auto res = rspamd_normalise_unicode_inplace(cpy.data(), &ns);
+ cpy.resize(ns);
+ CHECK(cpy == std::string(std::get<1>(c)));
+ CHECK(res == std::get<2>(c));
+ }
+ }
+
+ TEST_CASE("utf8 trim")
+ {
+ std::pair<const char *, const char *> cases[] = {
+ {" \u200B"
+ "abc ",
+ "abc"},
+ {" ", ""},
+ {" a", "a"},
+ {"a ", "a"},
+ {"a a", "a a"},
+ {"abc", "abc"},
+ {"a ", "a"},
+ {" abc ", "abc"},
+ {" abc ", "abc"},
+ {" \xE2\x80\x8B"
+ "a\xE2\x80\x8B"
+ "bc ",
+ "a\xE2\x80\x8B"
+ "bc"},
+ {" \xE2\x80\x8B"
+ "abc\xE2\x80\x8B ",
+ "abc"},
+ {" \xE2\x80\x8B"
+ "abc \xE2\x80\x8B ",
+ "abc"},
+ };
+
+ for (const auto &c: cases) {
+ std::string cpy{c.first};
+ auto ns = cpy.size();
+ auto *nstart = rspamd_string_unicode_trim_inplace(cpy.data(), &ns);
+ std::string res{nstart, ns};
+ CHECK(res == std::string{c.second});
+ }
+ }
+
+
+ TEST_CASE("utf8 strcmp")
+ {
+ std::tuple<const char *, const char *, int, int> cases[] = {
+ {"abc", "abc", -1, 0},
+ {"", "", -1, 0},
+ {"aBc", "AbC", -1, 0},
+ {"abc", "ab", 2, 0},
+ {"теСт", "ТеÑТ", -1, 0},
+ {"теСт", "Тезт", 4, 0},
+ {"теСт", "Тезт", -1, 1},
+ {"abc", "ABD", -1, -1},
+ {"\0a\0", "\0a\1", 2, 0},
+ {"\0a\0", "\0b\1", 3, -1},
+ };
+
+ for (const auto &c: cases) {
+ auto [s1, s2, n, expected] = c;
+ if (n == -1) {
+ n = MIN(strlen(s1), strlen(s2));
+ }
+ SUBCASE((std::string("test case: ") + s1 + " <=> " + s2).c_str())
+ {
+ auto ret = rspamd_utf8_strcmp(s1, s2, n);
+ CHECK(ret == expected);
+ }
+ }
+ }
+
+ TEST_CASE("transliterate")
+ {
+ using namespace std::literals;
+ std::tuple<std::string_view, const char *> cases[] = {
+ {"abc"sv, "abc"},
+ {""sv, ""},
+ {"теÑÑ‚"sv, "test"},
+ // Diacritic to ascii
+ {"ÎŽ"sv, "y"},
+ // Chinese to pinyin
+ {"你好"sv, "ni hao"},
+ // Japanese to romaji
+ {"ã“ã‚“ã«ã¡ã¯"sv, "konnichiha"},
+ // Devanagari to latin
+ {"नमसà¥à¤¤à¥‡"sv, "namaste"},
+ // Arabic to latin
+ {"مرحبا"sv, "mrhba"},
+ // Remove of punctuation
+ {"a.b.c"sv, "abc"},
+ // Lowercase
+ {"ABC"sv, "abc"},
+ // Remove zero-width spaces
+ {"\xE2\x80\x8B"
+ "abc\xE2\x80\x8B"
+ "def"sv,
+ "abcdef"},
+ };
+
+ for (const auto &c: cases) {
+ auto [s1, s2] = c;
+ SUBCASE((std::string("test case: ") + std::string(s1) + " => " + s2).c_str())
+ {
+ gsize tlen;
+ auto *ret = rspamd_utf8_transliterate(s1.data(), s1.length(), &tlen);
+ CHECK(tlen == strlen(s2));
+ CHECK(strcmp(s2, ret) == 0);
+ }
+ }
+ }
+} \ No newline at end of file
diff --git a/src/libutil/cxx/utf8_util.h b/src/libutil/cxx/utf8_util.h
new file mode 100644
index 0000000..044beae
--- /dev/null
+++ b/src/libutil/cxx/utf8_util.h
@@ -0,0 +1,85 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifndef RSPAMD_UTF8_UTIL_H
+#define RSPAMD_UTF8_UTIL_H
+
+#include "config.h"
+#include "mem_pool.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Removes all unicode spaces from a string
+ * @param str start of the string
+ * @param len length
+ * @return new length of the string trimmed
+ */
+const char *rspamd_string_unicode_trim_inplace(const char *str, size_t *len);
+
+enum rspamd_utf8_normalise_result {
+ RSPAMD_UNICODE_NORM_NORMAL = 0,
+ RSPAMD_UNICODE_NORM_UNNORMAL = (1 << 0),
+ RSPAMD_UNICODE_NORM_ZERO_SPACES = (1 << 1),
+ RSPAMD_UNICODE_NORM_ERROR = (1 << 2),
+ RSPAMD_UNICODE_NORM_OVERFLOW = (1 << 3)
+};
+
+/**
+ * Gets a string in UTF8 and normalises it to NFKC_Casefold form
+ * @param pool optional memory pool used for logging purposes
+ * @param start
+ * @param len
+ * @return TRUE if a string has been normalised
+ */
+enum rspamd_utf8_normalise_result rspamd_normalise_unicode_inplace(gchar *start, gsize *len);
+
+/**
+ * Transliterate a string to ASCII
+ * @param start
+ * @param len
+ * @param target_len
+ * @return a new string that should be freed with g_free
+ */
+gchar *rspamd_utf8_transliterate(const gchar *start, gsize len, gsize *target_len);
+
+/**
+ * Compare two strings using libicu collator
+ * @param s1
+ * @param s2
+ * @param n
+ * @return an integer greater than, equal to, or less than 0, according as the string s1 is greater than, equal to, or less than the string s2.
+ */
+int rspamd_utf8_strcmp(const char *s1, const char *s2, gsize n);
+/**
+ * Similar to rspamd_utf8_strcmp but accepts two sizes
+ * @param s1
+ * @param n1
+ * @param s2
+ * @param n2
+ * @return
+ */
+int rspamd_utf8_strcmp_sizes(const char *s1, gsize n1, const char *s2, gsize n2);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif//RSPAMD_UTF8_UTIL_H
diff --git a/src/libutil/cxx/util.hxx b/src/libutil/cxx/util.hxx
new file mode 100644
index 0000000..32ec0b5
--- /dev/null
+++ b/src/libutil/cxx/util.hxx
@@ -0,0 +1,238 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_UTIL_HXX
+#define RSPAMD_UTIL_HXX
+
+#pragma once
+
+#include <memory>
+#include <array>
+#include <string_view>
+#include <optional>
+#include <tuple>
+#include <algorithm>
+
+/*
+ * Common C++ utilities
+ */
+
+namespace rspamd {
+/*
+ * Creates std::array from a standard C style array with automatic size calculation
+ */
+template<typename... Ts>
+constexpr auto array_of(Ts &&...t) -> std::array<typename std::decay_t<typename std::common_type_t<Ts...>>, sizeof...(Ts)>
+{
+ using T = typename std::decay_t<typename std::common_type_t<Ts...>>;
+ return {{std::forward<T>(t)...}};
+}
+
+/**
+ * Find a value in a map
+ * @tparam C Map type
+ * @tparam K Key type
+ * @tparam V Value type
+ * @param c Map to search
+ * @param k Key to search
+ * @return Value if found or std::nullopt otherwise
+ */
+template<class C, class K, class V = typename C::mapped_type, typename std::enable_if_t<std::is_constructible_v<typename C::key_type, K> && std::is_constructible_v<typename C::mapped_type, V>, bool> = false>
+constexpr auto find_map(const C &c, const K &k) -> std::optional<std::reference_wrapper<const V>>
+{
+ auto f = c.find(k);
+
+ if (f != c.end()) {
+ return std::cref<V>(f->second);
+ }
+
+ return std::nullopt;
+}
+
+
+template<typename It>
+inline constexpr auto make_string_view_from_it(It begin, It end)
+{
+ using result_type = std::string_view;
+
+ return result_type{((begin != end) ? &*begin : nullptr),
+ (typename result_type::size_type) std::max(std::distance(begin, end),
+ (typename result_type::difference_type) 0)};
+}
+
+/**
+ * Iterate over lines in a string, newline characters are dropped
+ * @tparam S
+ * @tparam F
+ * @param input
+ * @param functor
+ * @return
+ */
+template<class S, class F, typename std::enable_if_t<std::is_invocable_v<F, std::string_view> && std::is_constructible_v<std::string_view, S>, bool> = true>
+inline auto string_foreach_line(const S &input, const F &functor)
+{
+ auto it = input.begin();
+ auto end = input.end();
+
+ while (it != end) {
+ auto next = std::find(it, end, '\n');
+ while (next >= it && (*next == '\n' || *next == '\r')) {
+ --next;
+ }
+ functor(make_string_view_from_it(it, next));
+ it = next;
+
+ if (it != end) {
+ ++it;
+ }
+ }
+}
+
+/**
+ * Iterate over elements in a string
+ * @tparam S string type
+ * @tparam D delimiter type
+ * @tparam F functor type
+ * @param input string to iterate
+ * @param delim delimiter to use
+ * @param functor functor to call
+ * @param ignore_empty ignore empty elements
+ * @return nothing
+ */
+template<class S, class D, class F,
+ typename std::enable_if_t<std::is_invocable_v<F, std::string_view> && std::is_constructible_v<std::string_view, S> && std::is_constructible_v<std::string_view, D>, bool> = true>
+inline auto string_foreach_delim(const S &input, const D &delim, const F &functor, const bool ignore_empty = true) -> void
+{
+ size_t first = 0;
+ auto sv_input = std::string_view{input};
+ auto sv_delim = std::string_view{delim};
+
+ while (first < sv_input.size()) {
+ const auto second = sv_input.find_first_of(sv_delim, first);
+
+ if (first != second || !ignore_empty) {
+ functor(sv_input.substr(first, second - first));
+ }
+
+ if (second == std::string_view::npos) {
+ break;
+ }
+
+ first = second + 1;
+ }
+}
+
+/**
+ * Split string on a character
+ * @tparam S string type
+ * @param input string to split
+ * @param chr character to split on
+ * @return pair of strings
+ */
+template<class S, typename std::enable_if_t<std::is_constructible_v<std::string_view, S>, bool> = true>
+inline auto string_split_on(const S &input, std::string_view::value_type chr) -> std::pair<std::string_view, std::string_view>
+{
+ auto pos = std::find(std::begin(input), std::end(input), chr);
+
+ if (pos != input.end()) {
+ auto first = std::string_view{std::begin(input), static_cast<std::size_t>(std::distance(std::begin(input), pos))};
+ while (*pos == chr && pos != input.end()) {
+ ++pos;
+ }
+ auto last = std::string_view{pos, static_cast<std::size_t>(std::distance(pos, std::end(input)))};
+
+ return {first, last};
+ }
+
+ return {std::string_view{input}, std::string_view{}};
+}
+
+/**
+ * Enumerate for range loop
+ * @tparam T iterable type
+ * @tparam TIter iterator type
+ * @param iterable iterable object
+ * @return iterator object
+ */
+template<typename T,
+ typename TIter = decltype(std::begin(std::declval<T>())),
+ typename = decltype(std::end(std::declval<T>()))>
+constexpr auto enumerate(T &&iterable)
+{
+ struct iterator {
+ size_t i;
+ TIter iter;
+ bool operator!=(const iterator &other) const
+ {
+ return iter != other.iter;
+ }
+ void operator++()
+ {
+ ++i;
+ ++iter;
+ }
+ auto operator*() const
+ {
+ return std::tie(i, *iter);
+ }
+ };
+ struct iterable_wrapper {
+ T iterable;
+ auto begin()
+ {
+ return iterator{0, std::begin(iterable)};
+ }
+ auto end()
+ {
+ return iterator{0, std::end(iterable)};
+ }
+ };
+ return iterable_wrapper{std::forward<T>(iterable)};
+}
+
+/**
+ * Allocator that cleans up memory in a secure way on destruction
+ * @tparam T
+ */
+template<class T>
+class secure_mem_allocator : public std::allocator<T> {
+public:
+ using value_type = typename std::allocator<T>::value_type;
+ using size_type = typename std::allocator<T>::size_type;
+ template<class U>
+ struct rebind {
+ typedef secure_mem_allocator<U> other;
+ };
+ secure_mem_allocator() noexcept = default;
+ secure_mem_allocator(const secure_mem_allocator &_) noexcept
+ : std::allocator<T>(_)
+ {
+ }
+ template<class U>
+ explicit secure_mem_allocator(const secure_mem_allocator<U> &) noexcept
+ {
+ }
+
+ void deallocate(value_type *p, size_type num) noexcept
+ {
+ rspamd_explicit_memzero((void *) p, num);
+ std::allocator<T>::deallocate(p, num);
+ }
+};
+
+
+}// namespace rspamd
+
+#endif//RSPAMD_UTIL_HXX
diff --git a/src/libutil/cxx/util_tests.cxx b/src/libutil/cxx/util_tests.cxx
new file mode 100644
index 0000000..6c3c177
--- /dev/null
+++ b/src/libutil/cxx/util_tests.cxx
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "util.hxx"
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+#include <vector>
+
+using namespace rspamd;
+using namespace std::literals::string_view_literals;
+
+TEST_SUITE("cxx utils")
+{
+ TEST_CASE("string_split_on")
+ {
+ std::tuple<std::string_view, char, std::pair<std::string_view, std::string_view>> cases[] = {
+ {"test test"sv, ' ', std::pair{"test"sv, "test"sv}},
+ {"test test"sv, ' ', std::pair{"test"sv, "test"sv}},
+ {"test test "sv, ' ', std::pair{"test"sv, "test "sv}},
+ {"testtest "sv, ' ', std::pair{"testtest"sv, ""sv}},
+ {" testtest "sv, ' ', std::pair{""sv, "testtest "sv}},
+ {"testtest"sv, ' ', std::pair{"testtest"sv, ""sv}},
+ {""sv, ' ', std::pair{""sv, ""sv}},
+ };
+
+ for (const auto &c: cases) {
+ auto res = string_split_on(std::get<0>(c), std::get<1>(c));
+ auto expected = std::get<2>(c);
+ CHECK(res.first == expected.first);
+ CHECK(res.second == expected.second);
+ }
+ }
+
+ TEST_CASE("string_foreach_delim")
+ {
+ std::tuple<std::string_view, std::string_view, std::pair<std::vector<std::string_view>, std::vector<std::string_view>>> cases[] = {
+ {"test"sv, ","sv, {{"test"}, {"test"}}},
+ {"test,test"sv, ","sv, {{"test", "test"}, {"test", "test"}}},
+ {"test, test"sv, ", "sv, {{"test", "test"}, {"test", "", "test"}}},
+ {"test, test,,"sv, ", "sv, {{"test", "test"}, {"test", "", "test", ""}}},
+ };
+
+ for (const auto &c: cases) {
+ auto res = std::vector<std::string_view>();
+ string_foreach_delim(std::get<0>(c), std::get<1>(c), [&](const auto &v) {
+ res.push_back(v);
+ });
+
+ auto compare_vec = []<class T>(const std::vector<T> &v1, const std::vector<T> &v2) {
+ CHECK(v1.size() == v2.size());
+ for (size_t i = 0; i < v1.size(); ++i) {
+ CHECK(v1[i] == v2[i]);
+ }
+ };
+
+ compare_vec(res, std::get<2>(c).first);
+
+ res.clear();
+ // Perform the same test but with no skip empty
+ string_foreach_delim(
+ std::get<0>(c), std::get<1>(c), [&](const auto &v) {
+ res.push_back(v);
+ },
+ false);
+ compare_vec(res, std::get<2>(c).second);
+ }
+ }
+} \ No newline at end of file
diff --git a/src/libutil/expression.c b/src/libutil/expression.c
new file mode 100644
index 0000000..957c47f
--- /dev/null
+++ b/src/libutil/expression.c
@@ -0,0 +1,1635 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "expression.h"
+#include "printf.h"
+#include "regexp.h"
+#include "util.h"
+#include "utlist.h"
+#include "ottery.h"
+#include "libserver/logger.h"
+#include "libcryptobox/cryptobox.h"
+#include <math.h>
+
+#define RSPAMD_EXPR_FLAG_NEGATE (1 << 0)
+#define RSPAMD_EXPR_FLAG_PROCESSED (1 << 1)
+
+#define MIN_RESORT_EVALS 50
+#define MAX_RESORT_EVALS 150
+
+enum rspamd_expression_elt_type {
+ ELT_OP = 0,
+ ELT_ATOM,
+ ELT_LIMIT
+};
+
+enum rspamd_expression_op_flag {
+ RSPAMD_EXPRESSION_UNARY = 1u << 0u,
+ RSPAMD_EXPRESSION_BINARY = 1u << 1u,
+ RSPAMD_EXPRESSION_NARY = 1u << 2u,
+ RSPAMD_EXPRESSION_ARITHMETIC = 1u << 3u,
+ RSPAMD_EXPRESSION_LOGICAL = 1u << 4u,
+ RSPAMD_EXPRESSION_COMPARISON = 1u << 5u,
+};
+
+struct rspamd_expression_operation {
+ enum rspamd_expression_op op;
+ guint logical_priority;
+ guint op_flags;
+};
+
+struct rspamd_expression_elt {
+ enum rspamd_expression_elt_type type;
+ union {
+ rspamd_expression_atom_t *atom;
+ struct rspamd_expression_operation op;
+ gdouble lim;
+ } p;
+
+ gint flags;
+ gint priority;
+ gdouble value;
+};
+
+struct rspamd_expression {
+ const struct rspamd_atom_subr *subr;
+ GArray *expressions;
+ GPtrArray *expression_stack;
+ GNode *ast;
+ gchar *log_id;
+ guint next_resort;
+ guint evals;
+};
+
+struct rspamd_expr_process_data {
+ gpointer *ud;
+ gint flags;
+ /* != NULL if trace is collected */
+ GPtrArray *trace;
+ rspamd_expression_process_cb process_closure;
+};
+
+#define msg_debug_expression(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_expression_log_id, "expression", e->log_id, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+#ifdef DEBUG_EXPRESSIONS
+#define msg_debug_expression_verbose(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_expression_log_id, "expression", e->log_id, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#else
+#define msg_debug_expression_verbose(...) \
+ do { \
+ } while (0)
+#endif
+
+INIT_LOG_MODULE(expression)
+
+static GQuark
+rspamd_expr_quark(void)
+{
+ return g_quark_from_static_string("rspamd-expression");
+}
+
+static const gchar *RSPAMD_CONST_FUNCTION
+rspamd_expr_op_to_str(enum rspamd_expression_op op);
+static const gchar *
+rspamd_expr_op_to_str(enum rspamd_expression_op op)
+{
+ const gchar *op_str = NULL;
+
+ switch (op) {
+ case OP_AND:
+ op_str = "&";
+ break;
+ case OP_OR:
+ op_str = "|";
+ break;
+ case OP_MULT:
+ op_str = "*";
+ break;
+ case OP_PLUS:
+ op_str = "+";
+ break;
+ case OP_MINUS:
+ op_str = "-";
+ break;
+ case OP_DIVIDE:
+ op_str = "/";
+ break;
+ case OP_NOT:
+ op_str = "!";
+ break;
+ case OP_GE:
+ op_str = ">=";
+ break;
+ case OP_GT:
+ op_str = ">";
+ break;
+ case OP_LE:
+ op_str = "<=";
+ break;
+ case OP_LT:
+ op_str = "<";
+ break;
+ case OP_EQ:
+ op_str = "==";
+ break;
+ case OP_NE:
+ op_str = "!=";
+ break;
+ case OP_OBRACE:
+ op_str = "(";
+ break;
+ case OP_CBRACE:
+ op_str = ")";
+ break;
+ default:
+ op_str = "???";
+ break;
+ }
+
+ return op_str;
+}
+
+#define G_ARRAY_LAST(ar, type) (&g_array_index((ar), type, (ar)->len - 1))
+
+static void
+rspamd_expr_stack_elt_push(GPtrArray *stack,
+ gpointer elt)
+{
+ g_ptr_array_add(stack, elt);
+}
+
+
+static gpointer
+rspamd_expr_stack_elt_pop(GPtrArray *stack)
+{
+ gpointer e;
+ gint idx;
+
+ if (stack->len == 0) {
+ return NULL;
+ }
+
+ idx = stack->len - 1;
+ e = g_ptr_array_index(stack, idx);
+ g_ptr_array_remove_index_fast(stack, idx);
+
+ return e;
+}
+
+
+static void
+rspamd_expr_stack_push(struct rspamd_expression *expr,
+ gpointer elt)
+{
+ rspamd_expr_stack_elt_push(expr->expression_stack, elt);
+}
+
+static gpointer
+rspamd_expr_stack_pop(struct rspamd_expression *expr)
+{
+ return rspamd_expr_stack_elt_pop(expr->expression_stack);
+}
+
+static gpointer
+rspamd_expr_stack_peek(struct rspamd_expression *expr)
+{
+ gpointer e;
+ gint idx;
+ GPtrArray *stack = expr->expression_stack;
+
+ if (stack->len == 0) {
+ return NULL;
+ }
+
+ idx = stack->len - 1;
+ e = g_ptr_array_index(stack, idx);
+
+ return e;
+}
+
+/*
+ * Return operation priority
+ */
+static gint RSPAMD_CONST_FUNCTION
+rspamd_expr_logic_priority(enum rspamd_expression_op op);
+static gint
+rspamd_expr_logic_priority(enum rspamd_expression_op op)
+{
+ gint ret = 0;
+
+ switch (op) {
+ case OP_NOT:
+ ret = 7;
+ break;
+ case OP_MULT:
+ case OP_DIVIDE:
+ ret = 6;
+ break;
+ case OP_PLUS:
+ case OP_MINUS:
+ ret = 5;
+ break;
+ case OP_GE:
+ case OP_GT:
+ case OP_LE:
+ case OP_LT:
+ case OP_EQ:
+ case OP_NE:
+ ret = 4;
+ break;
+ case OP_AND:
+ ret = 3;
+ break;
+ case OP_OR:
+ ret = 2;
+ break;
+ case OP_OBRACE:
+ case OP_CBRACE:
+ ret = 1;
+ break;
+ case OP_INVALID:
+ ret = -1;
+ break;
+ }
+
+ return ret;
+}
+
+static guint RSPAMD_CONST_FUNCTION
+rspamd_expr_op_flags(enum rspamd_expression_op op);
+
+static guint
+rspamd_expr_op_flags(enum rspamd_expression_op op)
+{
+ guint ret = 0;
+
+ switch (op) {
+ case OP_NOT:
+ ret |= RSPAMD_EXPRESSION_UNARY | RSPAMD_EXPRESSION_LOGICAL;
+ break;
+ case OP_MULT:
+ ret |= RSPAMD_EXPRESSION_NARY | RSPAMD_EXPRESSION_ARITHMETIC;
+ break;
+ case OP_DIVIDE:
+ ret |= RSPAMD_EXPRESSION_BINARY | RSPAMD_EXPRESSION_ARITHMETIC;
+ break;
+ case OP_PLUS:
+ ret |= RSPAMD_EXPRESSION_NARY | RSPAMD_EXPRESSION_ARITHMETIC;
+ break;
+ case OP_MINUS:
+ ret |= RSPAMD_EXPRESSION_BINARY | RSPAMD_EXPRESSION_ARITHMETIC;
+ break;
+ case OP_GE:
+ case OP_GT:
+ case OP_LE:
+ case OP_LT:
+ case OP_EQ:
+ case OP_NE:
+ ret |= RSPAMD_EXPRESSION_BINARY | RSPAMD_EXPRESSION_COMPARISON;
+ break;
+ case OP_AND:
+ case OP_OR:
+ ret |= RSPAMD_EXPRESSION_NARY | RSPAMD_EXPRESSION_LOGICAL;
+ break;
+ case OP_OBRACE:
+ case OP_CBRACE:
+ case OP_INVALID:
+ break;
+ }
+
+ return ret;
+}
+
+/*
+ * Return FALSE if symbol is not operation symbol (operand)
+ * Return TRUE if symbol is operation symbol
+ */
+static gboolean RSPAMD_CONST_FUNCTION
+rspamd_expr_is_operation_symbol(gchar a);
+static gboolean
+rspamd_expr_is_operation_symbol(gchar a)
+{
+ switch (a) {
+ case '!':
+ case '&':
+ case '|':
+ case '(':
+ case ')':
+ case '>':
+ case '<':
+ case '+':
+ case '*':
+ case '-':
+ case '/':
+ case '=':
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_expr_is_operation(struct rspamd_expression *e,
+ const gchar *p, const gchar *end, rspamd_regexp_t *num_re)
+{
+ if (rspamd_expr_is_operation_symbol(*p)) {
+ if (p + 1 < end) {
+ gchar t = *(p + 1);
+
+ if (t == ':') {
+ /* Special case, treat it as an atom */
+ }
+ else if (*p == '/') {
+ /* Lookahead for division operation to distinguish from regexp */
+ const gchar *track = p + 1;
+
+ /* Skip spaces */
+ while (track < end && g_ascii_isspace(*track)) {
+ track++;
+ }
+
+ /* Check for a number */
+ if (rspamd_regexp_search(num_re,
+ track,
+ end - track,
+ NULL,
+ NULL,
+ FALSE,
+ NULL)) {
+ msg_debug_expression_verbose("found divide operation");
+ return TRUE;
+ }
+
+ msg_debug_expression_verbose("false divide operation");
+ /* Fallback to PARSE_ATOM state */
+ }
+ else if (*p == '-') {
+ /* - is used in composites, so we need to distinguish - from
+ * 1) unary minus of a limit!
+ * 2) -BLAH in composites
+ * Decision is simple: require a space after binary `-` op
+ */
+ if (g_ascii_isspace(t)) {
+ return TRUE;
+ }
+ /* Fallback to PARSE_ATOM state */
+ msg_debug_expression_verbose("false minus operation");
+ }
+ else {
+ /* Generic operation */
+ return TRUE;
+ }
+ }
+ else {
+ /* Last op */
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+/* Return character representation of operation */
+static enum rspamd_expression_op
+rspamd_expr_str_to_op(const gchar *a, const gchar *end, const gchar **next)
+{
+ enum rspamd_expression_op op = OP_INVALID;
+
+ g_assert(a < end);
+
+ switch (*a) {
+ case '!':
+ case '&':
+ case '|':
+ case '+':
+ case '*':
+ case '/':
+ case '-':
+ case '(':
+ case ')':
+ case '=': {
+ if (a < end - 1) {
+ if ((a[0] == '&' && a[1] == '&') ||
+ (a[0] == '|' && a[1] == '|') ||
+ (a[0] == '!' && a[1] == '=') ||
+ (a[0] == '=' && a[1] == '=')) {
+ *next = a + 2;
+ }
+ else {
+ *next = a + 1;
+ }
+ }
+ else {
+ *next = end;
+ }
+ /* XXX: not especially effective */
+ switch (*a) {
+ case '!':
+ if (a < end - 1 && a[1] == '=') {
+ op = OP_NE;
+ }
+ else {
+ op = OP_NOT;
+ }
+ break;
+ case '&':
+ op = OP_AND;
+ break;
+ case '*':
+ op = OP_MULT;
+ break;
+ case '|':
+ op = OP_OR;
+ break;
+ case '+':
+ op = OP_PLUS;
+ break;
+ case '/':
+ op = OP_DIVIDE;
+ break;
+ case '-':
+ op = OP_MINUS;
+ break;
+ case '=':
+ op = OP_EQ;
+ break;
+ case ')':
+ op = OP_CBRACE;
+ break;
+ case '(':
+ op = OP_OBRACE;
+ break;
+ default:
+ op = OP_INVALID;
+ break;
+ }
+ break;
+ }
+ case 'O':
+ case 'o':
+ if ((gulong) (end - a) >= sizeof("or") &&
+ g_ascii_strncasecmp(a, "or", sizeof("or") - 1) == 0) {
+ *next = a + sizeof("or") - 1;
+ op = OP_OR;
+ }
+ break;
+ case 'A':
+ case 'a':
+ if ((gulong) (end - a) >= sizeof("and") &&
+ g_ascii_strncasecmp(a, "and", sizeof("and") - 1) == 0) {
+ *next = a + sizeof("and") - 1;
+ op = OP_AND;
+ }
+ break;
+ case 'N':
+ case 'n':
+ if ((gulong) (end - a) >= sizeof("not") &&
+ g_ascii_strncasecmp(a, "not", sizeof("not") - 1) == 0) {
+ *next = a + sizeof("not") - 1;
+ op = OP_NOT;
+ }
+ break;
+ case '>':
+ if (a < end - 1 && a[1] == '=') {
+ *next = a + 2;
+ op = OP_GE;
+ }
+ else {
+ *next = a + 1;
+ op = OP_GT;
+ }
+ break;
+ case '<':
+ if (a < end - 1 && a[1] == '=') {
+ *next = a + 2;
+ op = OP_LE;
+ }
+ else {
+ *next = a + 1;
+ op = OP_LT;
+ }
+ break;
+ default:
+ op = OP_INVALID;
+ break;
+ }
+
+ return op;
+}
+
+static void
+rspamd_expression_destroy(struct rspamd_expression *expr)
+{
+ guint i;
+ struct rspamd_expression_elt *elt;
+
+ if (expr != NULL) {
+
+ if (expr->subr->destroy) {
+ /* Free atoms */
+ for (i = 0; i < expr->expressions->len; i++) {
+ elt = &g_array_index(expr->expressions,
+ struct rspamd_expression_elt, i);
+
+ if (elt->type == ELT_ATOM) {
+ expr->subr->destroy(elt->p.atom);
+ }
+ }
+ }
+
+ if (expr->expressions) {
+ g_array_free(expr->expressions, TRUE);
+ }
+ if (expr->expression_stack) {
+ g_ptr_array_free(expr->expression_stack, TRUE);
+ }
+ if (expr->ast) {
+ g_node_destroy(expr->ast);
+ }
+ if (expr->log_id) {
+ g_free(expr->log_id);
+ }
+
+ g_free(expr);
+ }
+}
+
+static gboolean
+rspamd_ast_add_node(struct rspamd_expression *e,
+ GPtrArray *operands,
+ struct rspamd_expression_elt *op,
+ GError **err)
+{
+
+ GNode *res, *a1, *a2, *test;
+
+ g_assert(op->type == ELT_OP);
+
+ if (op->p.op.op_flags & RSPAMD_EXPRESSION_UNARY) {
+ /* Unary operator */
+ struct rspamd_expression_elt *test_elt;
+
+ res = g_node_new(op);
+ a1 = rspamd_expr_stack_elt_pop(operands);
+
+ if (a1 == NULL) {
+ g_set_error(err, rspamd_expr_quark(), EINVAL, "no operand to "
+ "unary '%s' operation",
+ rspamd_expr_op_to_str(op->p.op.op));
+ g_node_destroy(res);
+
+ return FALSE;
+ }
+
+ g_node_append(res, a1);
+ test_elt = a1->data;
+
+ if (test_elt->type == ELT_ATOM) {
+ test_elt->p.atom->parent = res;
+ msg_debug_expression("added unary op %s to AST; operand: %*s",
+ rspamd_expr_op_to_str(op->p.op.op),
+ (int) test_elt->p.atom->len, test_elt->p.atom->str);
+ }
+ else {
+ msg_debug_expression("added unary op %s to AST; operand type: %d",
+ rspamd_expr_op_to_str(op->p.op.op),
+ test_elt->type);
+ }
+ }
+ else {
+ struct rspamd_expression_elt *e1, *e2;
+ /* For binary/nary operators we might want to examine chains */
+ a2 = rspamd_expr_stack_elt_pop(operands);
+ a1 = rspamd_expr_stack_elt_pop(operands);
+
+ if (a2 == NULL) {
+ g_set_error(err, rspamd_expr_quark(), EINVAL, "no left operand to "
+ "'%s' operation",
+ rspamd_expr_op_to_str(op->p.op.op));
+ return FALSE;
+ }
+
+ if (a1 == NULL) {
+ g_set_error(err, rspamd_expr_quark(), EINVAL, "no right operand to "
+ "'%s' operation",
+ rspamd_expr_op_to_str(op->p.op.op));
+ return FALSE;
+ }
+
+ /* Nary stuff */
+ if (op->p.op.op_flags & RSPAMD_EXPRESSION_NARY) {
+ /*
+ * We convert a set of ops like X + Y + Z to a nary tree like
+ * X Y Z +
+ * for the longest possible prefix of atoms/limits
+ */
+
+ /* First try with a1 */
+ test = a1;
+ e1 = test->data;
+
+ if (e1->type == ELT_OP && e1->p.op.op == op->p.op.op) {
+ /* Add children */
+ g_node_append(test, a2);
+ rspamd_expr_stack_elt_push(operands, a1);
+
+ msg_debug_expression("added nary op %s to AST merged with the first operand",
+ rspamd_expr_op_to_str(op->p.op.op));
+
+ return TRUE;
+ }
+
+ /* Now test a2 */
+ test = a2;
+ e2 = test->data;
+
+ if (e2->type == ELT_OP && e2->p.op.op == op->p.op.op) {
+ /* Add children */
+ g_node_prepend(test, a1);
+ rspamd_expr_stack_elt_push(operands, a2);
+
+ msg_debug_expression("added nary op %s to AST merged with the second operand",
+ rspamd_expr_op_to_str(op->p.op.op));
+
+ return TRUE;
+ }
+ }
+
+ /* No optimizations possible, so create a new level */
+ res = g_node_new(op);
+ g_node_append(res, a1);
+ g_node_append(res, a2);
+
+ e1 = a1->data;
+ e2 = a2->data;
+
+ if (e1->type == ELT_ATOM) {
+ e1->p.atom->parent = res;
+ }
+
+ if (e2->type == ELT_ATOM) {
+ e2->p.atom->parent = res;
+ }
+
+ if (e1->type == ELT_ATOM && e2->type == ELT_ATOM) {
+ msg_debug_expression("added binary op %s to AST; operands: (%*s; %*s)",
+ rspamd_expr_op_to_str(op->p.op.op),
+ (int) e1->p.atom->len, e1->p.atom->str,
+ (int) e2->p.atom->len, e2->p.atom->str);
+ }
+ else {
+ msg_debug_expression("added binary op %s to AST; operands (types): (%d; %d)",
+ rspamd_expr_op_to_str(op->p.op.op),
+ e1->type,
+ e2->type);
+ }
+ }
+
+ /* Push back resulting node to the stack */
+ rspamd_expr_stack_elt_push(operands, res);
+
+ return TRUE;
+}
+
+static gboolean
+rspamd_ast_priority_traverse(GNode *node, gpointer d)
+{
+ struct rspamd_expression_elt *elt = node->data, *cur_elt;
+ struct rspamd_expression *expr = d;
+ gint cnt = 0;
+ GNode *cur;
+
+ if (node->children) {
+ cur = node->children;
+ while (cur) {
+ cur_elt = cur->data;
+ cnt += cur_elt->priority;
+ cur = cur->next;
+ }
+ elt->priority = cnt;
+ }
+ else {
+ /* It is atom or limit */
+ g_assert(elt->type != ELT_OP);
+
+ if (elt->type == ELT_LIMIT) {
+ /* Always push limit first */
+ elt->priority = 0;
+ }
+ else {
+ elt->priority = RSPAMD_EXPRESSION_MAX_PRIORITY;
+
+ if (expr->subr->priority != NULL) {
+ elt->priority = RSPAMD_EXPRESSION_MAX_PRIORITY -
+ expr->subr->priority(elt->p.atom);
+ }
+ elt->p.atom->hits = 0;
+ }
+ }
+
+ return FALSE;
+}
+
+#define ATOM_PRIORITY(a) ((a)->p.atom->hits / ((a)->p.atom->exec_time.mean > 0 ? (a)->p.atom->exec_time.mean * 10000000 : 1.0))
+
+static gint
+rspamd_ast_priority_cmp(GNode *a, GNode *b)
+{
+ struct rspamd_expression_elt *ea = a->data, *eb = b->data;
+ gdouble w1, w2;
+
+ if (ea->type == ELT_LIMIT) {
+ return 1;
+ }
+ else if (eb->type == ELT_LIMIT) {
+ return -1;
+ }
+
+ /* Special logic for atoms */
+ if (ea->type == ELT_ATOM && eb->type == ELT_ATOM &&
+ ea->priority == eb->priority) {
+ w1 = ATOM_PRIORITY(ea);
+ w2 = ATOM_PRIORITY(eb);
+
+ ea->p.atom->hits = 0;
+
+ return w1 - w2;
+ }
+ else {
+ return ea->priority - eb->priority;
+ }
+}
+
+static gboolean
+rspamd_ast_resort_traverse(GNode *node, gpointer unused)
+{
+ GNode *children, *last;
+ struct rspamd_expression_elt *elt;
+
+ elt = (struct rspamd_expression_elt *) node->data;
+
+ /*
+ * We sort merely logical operations, everything else is dangerous
+ */
+ if (elt->type == ELT_OP && elt->p.op.op_flags & RSPAMD_EXPRESSION_LOGICAL) {
+
+ if (node->children) {
+
+ children = node->children;
+ last = g_node_last_sibling(children);
+ /* Needed for utlist compatibility */
+ children->prev = last;
+ DL_SORT(node->children, rspamd_ast_priority_cmp);
+ /* Restore GLIB compatibility */
+ children = node->children;
+ children->prev = NULL;
+ }
+ }
+
+ return FALSE;
+}
+
+static struct rspamd_expression_elt *
+rspamd_expr_dup_elt(rspamd_mempool_t *pool, struct rspamd_expression_elt *elt)
+{
+ struct rspamd_expression_elt *n;
+
+ n = rspamd_mempool_alloc(pool, sizeof(*n));
+ memcpy(n, elt, sizeof(*n));
+
+ return n;
+}
+
+gboolean
+rspamd_parse_expression(const gchar *line, gsize len,
+ const struct rspamd_atom_subr *subr, gpointer subr_data,
+ rspamd_mempool_t *pool, GError **err,
+ struct rspamd_expression **target)
+{
+ struct rspamd_expression *e;
+ struct rspamd_expression_elt elt;
+ rspamd_expression_atom_t *atom;
+ rspamd_regexp_t *num_re;
+ enum rspamd_expression_op op, op_stack;
+ const gchar *p, *c, *end;
+ GPtrArray *operand_stack;
+ GNode *tmp;
+
+ enum {
+ PARSE_ATOM = 0,
+ PARSE_OP,
+ PARSE_LIM,
+ SKIP_SPACES
+ } state = PARSE_ATOM;
+
+ g_assert(line != NULL);
+ g_assert(subr != NULL && subr->parse != NULL);
+
+ if (len == 0) {
+ len = strlen(line);
+ }
+
+ memset(&elt, 0, sizeof(elt));
+ num_re = rspamd_regexp_cache_create(NULL,
+ "/^(?:[+-]?([0-9]*[.])?[0-9]+)(?:\\s+|[)]|$)/", NULL, NULL);
+
+ p = line;
+ c = line;
+ end = line + len;
+ e = g_malloc0(sizeof(*e));
+ e->expressions = g_array_new(FALSE, FALSE,
+ sizeof(struct rspamd_expression_elt));
+ operand_stack = g_ptr_array_sized_new(32);
+ e->ast = NULL;
+ e->expression_stack = g_ptr_array_sized_new(32);
+ e->subr = subr;
+ e->evals = 0;
+ e->next_resort = ottery_rand_range(MAX_RESORT_EVALS) + MIN_RESORT_EVALS;
+ e->log_id = g_malloc0(RSPAMD_LOG_ID_LEN + 1);
+ guint64 h = rspamd_cryptobox_fast_hash(line, len, 0xdeadbabe);
+ rspamd_snprintf(e->log_id, RSPAMD_LOG_ID_LEN + 1, "%xL", h);
+ msg_debug_expression("start to parse expression '%*s'", (int) len, line);
+
+ /* Shunting-yard algorithm */
+ while (p < end) {
+ switch (state) {
+ case PARSE_ATOM:
+ if (g_ascii_isspace(*p)) {
+ state = SKIP_SPACES;
+ continue;
+ }
+ else if (rspamd_expr_is_operation(e, p, end, num_re)) {
+ /* Lookahead */
+ state = PARSE_OP;
+ continue;
+ }
+
+ /*
+ * First of all, we check some pre-conditions:
+ * 1) if we have 'and ' or 'or ' or 'not ' strings, they are op
+ * 2) if we have full numeric string, then we check for
+ * the following expression:
+ * ^\d+\s*[><]$
+ * and check the operation on stack
+ */
+ if ((gulong) (end - p) > sizeof("and ") &&
+ (g_ascii_strncasecmp(p, "and ", sizeof("and ") - 1) == 0 ||
+ g_ascii_strncasecmp(p, "not ", sizeof("not ") - 1) == 0)) {
+ state = PARSE_OP;
+ }
+ else if ((gulong) (end - p) > sizeof("or ") &&
+ g_ascii_strncasecmp(p, "or ", sizeof("or ") - 1) == 0) {
+ state = PARSE_OP;
+ }
+ else {
+ /*
+ * If we have any comparison or arithmetic operator in the stack, then try
+ * to parse limit
+ */
+ op = GPOINTER_TO_INT(rspamd_expr_stack_peek(e));
+
+ if (op == OP_MULT || op == OP_MINUS || op == OP_DIVIDE ||
+ op == OP_PLUS || (op >= OP_LT && op <= OP_NE)) {
+ if (rspamd_regexp_search(num_re,
+ p,
+ end - p,
+ NULL,
+ NULL,
+ FALSE,
+ NULL)) {
+ c = p;
+ state = PARSE_LIM;
+ continue;
+ }
+ /* Fallback to atom parsing */
+ }
+
+ /* Try to parse atom */
+ atom = subr->parse(p, end - p, pool, subr_data, err);
+ if (atom == NULL || atom->len == 0) {
+ /* We couldn't parse the atom, so go out */
+ if (err != NULL && *err == NULL) {
+ g_set_error(err,
+ rspamd_expr_quark(),
+ 500,
+ "Cannot parse atom: callback function failed"
+ " to parse '%.*s'",
+ (int) (end - p),
+ p);
+ }
+ goto error_label;
+ }
+
+ if (atom->str == NULL) {
+ atom->str = p;
+ }
+
+ p = p + atom->len;
+
+ /* Push to output */
+ elt.type = ELT_ATOM;
+ elt.p.atom = atom;
+ g_array_append_val(e->expressions, elt);
+ rspamd_expr_stack_elt_push(operand_stack,
+ g_node_new(rspamd_expr_dup_elt(pool, &elt)));
+ msg_debug_expression("found atom: %*s; pushed onto operand stack (%d size)",
+ (int) atom->len, atom->str, operand_stack->len);
+ }
+ break;
+ case PARSE_LIM:
+ if ((g_ascii_isdigit(*p) || *p == '-' || *p == '.') && p < end - 1) {
+ p++;
+ }
+ else {
+ if (p == end - 1 && g_ascii_isdigit(*p)) {
+ p++;
+ }
+
+ if (p - c > 0) {
+ elt.type = ELT_LIMIT;
+ elt.p.lim = strtod(c, NULL);
+ g_array_append_val(e->expressions, elt);
+ rspamd_expr_stack_elt_push(operand_stack,
+ g_node_new(rspamd_expr_dup_elt(pool, &elt)));
+ msg_debug_expression("found limit: %.1f; pushed onto operand stack (%d size)",
+ elt.p.lim, operand_stack->len);
+ c = p;
+ state = SKIP_SPACES;
+ }
+ else {
+ g_set_error(err, rspamd_expr_quark(), 400, "Empty number");
+ goto error_label;
+ }
+ }
+ break;
+ case PARSE_OP:
+ op = rspamd_expr_str_to_op(p, end, &p);
+ if (op == OP_INVALID) {
+ g_set_error(err, rspamd_expr_quark(), 500, "Bad operator %c",
+ *p);
+ goto error_label;
+ }
+ else if (op == OP_OBRACE) {
+ /*
+ * If the token is a left parenthesis, then push it onto
+ * the stack.
+ */
+ rspamd_expr_stack_push(e, GINT_TO_POINTER(op));
+ msg_debug_expression("found obrace, pushed to operators stack (%d size)",
+ e->expression_stack->len);
+ }
+ else if (op == OP_CBRACE) {
+ /*
+ * Until the token at the top of the stack is a left
+ * parenthesis, pop operators off the stack onto the
+ * output queue.
+ *
+ * Pop the left parenthesis from the stack,
+ * but not onto the output queue.
+ *
+ * If the stack runs out without finding a left parenthesis,
+ * then there are mismatched parentheses.
+ */
+ msg_debug_expression("found cbrace, rewind operators stack (%d size)",
+ e->expression_stack->len);
+
+ do {
+ op = GPOINTER_TO_INT(rspamd_expr_stack_pop(e));
+
+ if (op == OP_INVALID) {
+ g_set_error(err, rspamd_expr_quark(), 600,
+ "Braces mismatch");
+ goto error_label;
+ }
+
+ guint op_priority = rspamd_expr_logic_priority(op);
+ msg_debug_expression("found op: %s; priority = %d",
+ rspamd_expr_op_to_str(op), op_priority);
+
+ if (op != OP_OBRACE) {
+ elt.type = ELT_OP;
+ elt.p.op.op = op;
+ elt.p.op.op_flags = rspamd_expr_op_flags(op);
+ elt.p.op.logical_priority = op_priority;
+ g_array_append_val(e->expressions, elt);
+
+ if (!rspamd_ast_add_node(e, operand_stack,
+ rspamd_expr_dup_elt(pool, &elt), err)) {
+ goto error_label;
+ }
+ }
+
+ } while (op != OP_OBRACE);
+ }
+ else {
+ /*
+ * While there is an operator token, o2, at the top of
+ * the operator stack, and either:
+ *
+ * - o1 is left-associative and its precedence is less than
+ * or equal to that of o2, or
+ * - o1 is right associative, and has precedence less than
+ * that of o2,
+ *
+ * then pop o2 off the operator stack, onto the output queue;
+ *
+ * push o1 onto the operator stack.
+ */
+
+ for (;;) {
+ op_stack = GPOINTER_TO_INT(rspamd_expr_stack_pop(e));
+
+ if (op_stack == OP_INVALID) {
+ /* Stack is empty */
+ msg_debug_expression("no operations in operators stack");
+ break;
+ }
+
+ /* We ignore associativity for now */
+ guint op_priority = rspamd_expr_logic_priority(op),
+ stack_op_priority = rspamd_expr_logic_priority(op_stack);
+
+ msg_debug_expression("operators stack %d; operands stack: %d; "
+ "process operation '%s'(%d); pop operation '%s'(%d)",
+ e->expression_stack->len,
+ operand_stack->len,
+ rspamd_expr_op_to_str(op), op_priority,
+ rspamd_expr_op_to_str(op_stack), stack_op_priority);
+
+ if (op_stack != OP_OBRACE &&
+ op_priority < stack_op_priority) {
+ elt.type = ELT_OP;
+ elt.p.op.op = op_stack;
+ elt.p.op.op_flags = rspamd_expr_op_flags(op_stack);
+ elt.p.op.logical_priority = op_priority;
+
+ g_array_append_val(e->expressions, elt);
+
+ if (!rspamd_ast_add_node(e, operand_stack,
+ rspamd_expr_dup_elt(pool, &elt), err)) {
+ goto error_label;
+ }
+ }
+ else {
+ /* Push op_stack back */
+ msg_debug_expression("operators stack %d; operands stack: %d; "
+ "process operation '%s'(%d); push back to stack '%s'(%d)",
+ e->expression_stack->len,
+ operand_stack->len,
+ rspamd_expr_op_to_str(op), op_priority,
+ rspamd_expr_op_to_str(op_stack), stack_op_priority);
+ rspamd_expr_stack_push(e, GINT_TO_POINTER(op_stack));
+ break;
+ }
+ }
+
+ /* Push new operator itself */
+ msg_debug_expression("operators stack %d; operands stack: %d; "
+ "process operation '%s'; push to stack",
+ e->expression_stack->len,
+ operand_stack->len,
+ rspamd_expr_op_to_str(op));
+ rspamd_expr_stack_push(e, GINT_TO_POINTER(op));
+ }
+
+ state = SKIP_SPACES;
+ break;
+ case SKIP_SPACES:
+ if (g_ascii_isspace(*p)) {
+ p++;
+ }
+ if (rspamd_expr_is_operation(e, p, end, num_re)) {
+ /* Lookahead */
+ state = PARSE_OP;
+ }
+ else {
+ state = PARSE_ATOM;
+ }
+ break;
+ }
+ }
+
+ /* Now we process the stack and push operators to the output */
+ while ((op_stack = GPOINTER_TO_INT(rspamd_expr_stack_pop(e))) != OP_INVALID) {
+ msg_debug_expression("operators stack %d; operands stack: %d; "
+ "rewind stack; op: %s",
+ e->expression_stack->len,
+ operand_stack->len,
+ rspamd_expr_op_to_str(op_stack));
+
+ if (op_stack != OP_OBRACE) {
+ elt.type = ELT_OP;
+ elt.p.op.op = op_stack;
+ elt.p.op.op_flags = rspamd_expr_op_flags(op_stack);
+ elt.p.op.logical_priority = rspamd_expr_logic_priority(op_stack);
+
+ g_array_append_val(e->expressions, elt);
+ if (!rspamd_ast_add_node(e, operand_stack,
+ rspamd_expr_dup_elt(pool, &elt), err)) {
+ goto error_label;
+ }
+ }
+ else {
+ g_set_error(err, rspamd_expr_quark(), 600,
+ "Braces mismatch");
+ goto error_label;
+ }
+ }
+
+ if (operand_stack->len != 1) {
+ g_set_error(err, rspamd_expr_quark(), 601,
+ "Operators mismatch: %d elts in stack", operand_stack->len);
+ goto error_label;
+ }
+
+ e->ast = rspamd_expr_stack_elt_pop(operand_stack);
+ g_ptr_array_free(operand_stack, TRUE);
+
+ /* Set priorities for branches */
+ g_node_traverse(e->ast, G_POST_ORDER, G_TRAVERSE_ALL, -1,
+ rspamd_ast_priority_traverse, e);
+
+ /* Now set less expensive branches to be evaluated first */
+ g_node_traverse(e->ast, G_POST_ORDER, G_TRAVERSE_NON_LEAVES, -1,
+ rspamd_ast_resort_traverse, NULL);
+
+ if (target) {
+ *target = e;
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) rspamd_expression_destroy, e);
+ }
+ else {
+ rspamd_expression_destroy(e);
+ }
+
+ return TRUE;
+
+error_label:
+ if (err && *err) {
+ msg_debug_expression("fatal expression parse error: %e", *err);
+ }
+
+ while ((tmp = rspamd_expr_stack_elt_pop(operand_stack)) != NULL) {
+ g_node_destroy(tmp);
+ }
+
+ g_ptr_array_free(operand_stack, TRUE);
+ rspamd_expression_destroy(e);
+
+ return FALSE;
+}
+
+/*
+ * Node optimizer function: skip nodes that are not relevant
+ */
+static gboolean
+rspamd_ast_node_done(struct rspamd_expression_elt *elt, gdouble acc)
+{
+ gboolean ret = FALSE;
+
+ g_assert(elt->type == ELT_OP);
+
+ switch (elt->p.op.op) {
+ case OP_NOT:
+ ret = TRUE;
+ break;
+ case OP_AND:
+ ret = acc == 0;
+ break;
+ case OP_OR:
+ ret = acc != 0;
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+
+static gdouble
+rspamd_ast_do_unary_op(struct rspamd_expression_elt *elt, gdouble operand)
+{
+ gdouble ret;
+ g_assert(elt->type == ELT_OP);
+
+ switch (elt->p.op.op) {
+ case OP_NOT:
+ ret = fabs(operand) > DBL_EPSILON ? 0.0 : 1.0;
+ break;
+ default:
+ g_assert_not_reached();
+ }
+
+ return ret;
+}
+
+static gdouble
+rspamd_ast_do_binary_op(struct rspamd_expression_elt *elt, gdouble op1, gdouble op2)
+{
+ gdouble ret;
+
+ g_assert(elt->type == ELT_OP);
+
+ switch (elt->p.op.op) {
+ case OP_MINUS:
+ ret = op1 - op2;
+ break;
+ case OP_DIVIDE:
+ ret = op1 / op2;
+ break;
+ case OP_GE:
+ ret = op1 >= op2;
+ break;
+ case OP_GT:
+ ret = op1 > op2;
+ break;
+ case OP_LE:
+ ret = op1 <= op2;
+ break;
+ case OP_LT:
+ ret = op1 < op2;
+ break;
+ case OP_EQ:
+ ret = op1 == op2;
+ break;
+ case OP_NE:
+ ret = op1 != op2;
+ break;
+
+ case OP_NOT:
+ case OP_PLUS:
+ case OP_MULT:
+ case OP_AND:
+ case OP_OR:
+ default:
+ g_assert_not_reached();
+ break;
+ }
+
+ return ret;
+}
+
+static gdouble
+rspamd_ast_do_nary_op(struct rspamd_expression_elt *elt, gdouble val, gdouble acc)
+{
+ gdouble ret;
+
+ g_assert(elt->type == ELT_OP);
+
+ if (isnan(acc)) {
+ return val;
+ }
+
+ switch (elt->p.op.op) {
+ case OP_PLUS:
+ ret = acc + val;
+ break;
+ case OP_MULT:
+ ret = acc * val;
+ break;
+ case OP_AND:
+ ret = (fabs(acc) > DBL_EPSILON) && (fabs(val) > DBL_EPSILON);
+ break;
+ case OP_OR:
+ ret = (fabs(acc) > DBL_EPSILON) || (fabs(val) > DBL_EPSILON);
+ break;
+ default:
+ case OP_NOT:
+ case OP_MINUS:
+ case OP_DIVIDE:
+ case OP_GE:
+ case OP_GT:
+ case OP_LE:
+ case OP_LT:
+ case OP_EQ:
+ case OP_NE:
+ g_assert_not_reached();
+ break;
+ }
+
+ return ret;
+}
+
+static gdouble
+rspamd_ast_process_node(struct rspamd_expression *e, GNode *node,
+ struct rspamd_expr_process_data *process_data)
+{
+ struct rspamd_expression_elt *elt;
+ GNode *cld;
+ gdouble acc = NAN;
+ float t1, t2;
+ gdouble val;
+ gboolean calc_ticks = FALSE;
+ __attribute__((unused)) const gchar *op_name = NULL;
+
+ elt = node->data;
+
+ switch (elt->type) {
+ case ELT_ATOM:
+ if (!(elt->flags & RSPAMD_EXPR_FLAG_PROCESSED)) {
+ /*
+ * Check once per 256 evaluations approx
+ */
+ calc_ticks = (rspamd_random_uint64_fast() & 0xff) == 0xff;
+ if (calc_ticks) {
+ t1 = rspamd_get_ticks(TRUE);
+ }
+
+ elt->value = process_data->process_closure(process_data->ud, elt->p.atom);
+
+ if (fabs(elt->value) > DBL_EPSILON) {
+ elt->p.atom->hits++;
+
+ if (process_data->trace) {
+ g_ptr_array_add(process_data->trace, elt->p.atom);
+ }
+ }
+
+ if (calc_ticks) {
+ t2 = rspamd_get_ticks(TRUE);
+ rspamd_set_counter_ema(&elt->p.atom->exec_time, (t2 - t1), 0.5f);
+ }
+
+ elt->flags |= RSPAMD_EXPR_FLAG_PROCESSED;
+ }
+
+ acc = elt->value;
+ msg_debug_expression_verbose("atom: elt=%s; acc=%.1f", elt->p.atom->str, acc);
+ break;
+ case ELT_LIMIT:
+
+ acc = elt->p.lim;
+ msg_debug_expression_verbose("limit: lim=%.1f; acc=%.1f;", elt->p.lim, acc);
+ break;
+ case ELT_OP:
+ g_assert(node->children != NULL);
+#ifdef DEBUG_EXPRESSIONS
+ op_name = rspamd_expr_op_to_str(elt->p.op.op);
+#endif
+
+ if (elt->p.op.op_flags & RSPAMD_EXPRESSION_NARY) {
+ msg_debug_expression_verbose("proceed nary operation %s", op_name);
+ /* Proceed all ops in chain */
+ DL_FOREACH(node->children, cld)
+ {
+ val = rspamd_ast_process_node(e, cld, process_data);
+ msg_debug_expression_verbose("before op: op=%s; acc=%.1f; val = %.2f", op_name,
+ acc, val);
+ acc = rspamd_ast_do_nary_op(elt, val, acc);
+ msg_debug_expression_verbose("after op: op=%s; acc=%.1f; val = %.2f", op_name,
+ acc, val);
+
+ /* Check if we need to process further */
+ if (!(process_data->flags & RSPAMD_EXPRESSION_FLAG_NOOPT)) {
+ if (rspamd_ast_node_done(elt, acc)) {
+ msg_debug_expression_verbose("optimizer: done");
+ return acc;
+ }
+ }
+ }
+ }
+ else if (elt->p.op.op_flags & RSPAMD_EXPRESSION_BINARY) {
+ GNode *c1 = node->children, *c2;
+
+ c2 = c1->next;
+ g_assert(c2->next == NULL);
+ gdouble val1, val2;
+
+ msg_debug_expression_verbose("proceed binary operation %s",
+ op_name);
+ val1 = rspamd_ast_process_node(e, c1, process_data);
+ val2 = rspamd_ast_process_node(e, c2, process_data);
+
+ msg_debug_expression_verbose("before op: op=%s; op1 = %.1f, op2 = %.1f",
+ op_name, val1, val2);
+ acc = rspamd_ast_do_binary_op(elt, val1, val2);
+ msg_debug_expression_verbose("after op: op=%s; res=%.1f",
+ op_name, acc);
+ }
+ else if (elt->p.op.op_flags & RSPAMD_EXPRESSION_UNARY) {
+ GNode *c1 = node->children;
+
+ g_assert(c1->next == NULL);
+
+ msg_debug_expression_verbose("proceed unary operation %s",
+ op_name);
+ val = rspamd_ast_process_node(e, c1, process_data);
+
+ msg_debug_expression_verbose("before op: op=%s; op1 = %.1f",
+ op_name, val);
+ acc = rspamd_ast_do_unary_op(elt, val);
+ msg_debug_expression_verbose("after op: op=%s; res=%.1f",
+ op_name, acc);
+ }
+ break;
+ }
+
+ return acc;
+}
+
+static gboolean
+rspamd_ast_cleanup_traverse(GNode *n, gpointer d)
+{
+ struct rspamd_expression_elt *elt = n->data;
+
+ elt->value = 0;
+ elt->flags = 0;
+
+ return FALSE;
+}
+
+gdouble
+rspamd_process_expression_closure(struct rspamd_expression *expr,
+ rspamd_expression_process_cb cb,
+ gint flags,
+ gpointer runtime_ud,
+ GPtrArray **track)
+{
+ struct rspamd_expr_process_data pd;
+ gdouble ret = 0;
+
+ g_assert(expr != NULL);
+ /* Ensure that stack is empty at this point */
+ g_assert(expr->expression_stack->len == 0);
+
+ expr->evals++;
+
+ memset(&pd, 0, sizeof(pd));
+ pd.process_closure = cb;
+ pd.flags = flags;
+ pd.ud = runtime_ud;
+
+ if (track) {
+ pd.trace = g_ptr_array_sized_new(32);
+ *track = pd.trace;
+ }
+
+ ret = rspamd_ast_process_node(expr, expr->ast, &pd);
+
+ /* Cleanup */
+ g_node_traverse(expr->ast, G_IN_ORDER, G_TRAVERSE_ALL, -1,
+ rspamd_ast_cleanup_traverse, NULL);
+
+ /* Check if we need to resort */
+ if (expr->evals % expr->next_resort == 0) {
+ expr->next_resort = ottery_rand_range(MAX_RESORT_EVALS) +
+ MIN_RESORT_EVALS;
+ /* Set priorities for branches */
+ g_node_traverse(expr->ast, G_POST_ORDER, G_TRAVERSE_ALL, -1,
+ rspamd_ast_priority_traverse, expr);
+
+ /* Now set less expensive branches to be evaluated first */
+ g_node_traverse(expr->ast, G_POST_ORDER, G_TRAVERSE_NON_LEAVES, -1,
+ rspamd_ast_resort_traverse, NULL);
+ }
+
+ return ret;
+}
+
+gdouble
+rspamd_process_expression_track(struct rspamd_expression *expr,
+ gint flags,
+ gpointer runtime_ud,
+ GPtrArray **track)
+{
+ return rspamd_process_expression_closure(expr,
+ expr->subr->process, flags, runtime_ud, track);
+}
+
+gdouble
+rspamd_process_expression(struct rspamd_expression *expr,
+ gint flags,
+ gpointer runtime_ud)
+{
+ return rspamd_process_expression_closure(expr,
+ expr->subr->process, flags, runtime_ud, NULL);
+}
+
+static gboolean
+rspamd_ast_string_traverse(GNode *n, gpointer d)
+{
+ GString *res = d;
+ gint cnt;
+ GNode *cur;
+ struct rspamd_expression_elt *elt = n->data;
+ const char *op_str = NULL;
+
+ if (elt->type == ELT_ATOM) {
+ rspamd_printf_gstring(res, "(%*s)",
+ (int) elt->p.atom->len, elt->p.atom->str);
+ }
+ else if (elt->type == ELT_LIMIT) {
+ if (elt->p.lim == (double) (gint64) elt->p.lim) {
+ rspamd_printf_gstring(res, "%L", (gint64) elt->p.lim);
+ }
+ else {
+ rspamd_printf_gstring(res, "%f", elt->p.lim);
+ }
+ }
+ else {
+ op_str = rspamd_expr_op_to_str(elt->p.op.op);
+ g_string_append(res, op_str);
+
+ if (n->children) {
+ LL_COUNT(n->children, cur, cnt);
+
+ if (cnt > 2) {
+ /* Print n-ary of the operator */
+ g_string_append_printf(res, "(%d)", cnt);
+ }
+ }
+ }
+
+ g_string_append_c(res, ' ');
+
+ return FALSE;
+}
+
+GString *
+rspamd_expression_tostring(struct rspamd_expression *expr)
+{
+ GString *res;
+
+ g_assert(expr != NULL);
+
+ res = g_string_new(NULL);
+ g_node_traverse(expr->ast, G_POST_ORDER, G_TRAVERSE_ALL, -1,
+ rspamd_ast_string_traverse, res);
+
+ /* Last space */
+ if (res->len > 0) {
+ g_string_erase(res, res->len - 1, 1);
+ }
+
+ return res;
+}
+
+struct atom_foreach_cbdata {
+ rspamd_expression_atom_foreach_cb cb;
+ gpointer cbdata;
+};
+
+static gboolean
+rspamd_ast_atom_traverse(GNode *n, gpointer d)
+{
+ struct atom_foreach_cbdata *data = d;
+ struct rspamd_expression_elt *elt = n->data;
+ rspamd_ftok_t tok;
+
+ if (elt->type == ELT_ATOM) {
+ tok.begin = elt->p.atom->str;
+ tok.len = elt->p.atom->len;
+
+ data->cb(&tok, data->cbdata);
+ }
+
+ return FALSE;
+}
+
+void rspamd_expression_atom_foreach(struct rspamd_expression *expr,
+ rspamd_expression_atom_foreach_cb cb, gpointer cbdata)
+{
+ struct atom_foreach_cbdata data;
+
+ g_assert(expr != NULL);
+
+ data.cb = cb;
+ data.cbdata = cbdata;
+ g_node_traverse(expr->ast, G_POST_ORDER, G_TRAVERSE_ALL, -1,
+ rspamd_ast_atom_traverse, &data);
+}
+
+gboolean
+rspamd_expression_node_is_op(GNode *node, enum rspamd_expression_op op)
+{
+ struct rspamd_expression_elt *elt;
+
+ g_assert(node != NULL);
+
+ elt = node->data;
+
+ if (elt->type == ELT_OP && elt->p.op.op == op) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
diff --git a/src/libutil/expression.h b/src/libutil/expression.h
new file mode 100644
index 0000000..ea4e102
--- /dev/null
+++ b/src/libutil/expression.h
@@ -0,0 +1,173 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBUTIL_EXPRESSION_H_
+#define SRC_LIBUTIL_EXPRESSION_H_
+
+#include "config.h"
+#include "mem_pool.h"
+#include "fstring.h"
+#include "util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RSPAMD_EXPRESSION_MAX_PRIORITY 1024
+
+#define RSPAMD_EXPRESSION_FLAG_NOOPT (1 << 0)
+
+enum rspamd_expression_op {
+ OP_INVALID = 0,
+ OP_PLUS, /* + */
+ OP_MULT, /* * */
+ OP_MINUS, /* - */
+ OP_DIVIDE, /* / */
+ OP_OR, /* || or | */
+ OP_AND, /* && or & */
+ OP_NOT, /* ! */
+ OP_LT, /* < */
+ OP_GT, /* > */
+ OP_LE, /* <= */
+ OP_GE, /* >= */
+ OP_EQ, /* == */
+ OP_NE, /* != */
+ OP_OBRACE, /* ( */
+ OP_CBRACE /* ) */
+};
+
+typedef struct rspamd_expression_atom_s {
+ /* Parent node */
+ GNode *parent;
+ /* Opaque userdata */
+ gpointer data;
+ /* String representation of atom */
+ const gchar *str;
+ /* Length of the string representation of atom */
+ guint len;
+ /* Relative priority */
+ gint priority;
+ guint hits;
+ struct rspamd_counter_data exec_time;
+} rspamd_expression_atom_t;
+
+typedef gdouble (*rspamd_expression_process_cb)(gpointer runtime_data,
+ rspamd_expression_atom_t *atom);
+
+struct rspamd_atom_subr {
+ /* Parses atom from string and returns atom structure */
+ rspamd_expression_atom_t *(*parse)(const gchar *line, gsize len,
+ rspamd_mempool_t *pool, gpointer ud, GError **err);
+
+ /* Process atom via the opaque pointer (e.g. struct rspamd_task *) */
+ rspamd_expression_process_cb process;
+
+ /* Calculates the relative priority of the expression */
+ gint (*priority)(rspamd_expression_atom_t *atom);
+
+ void (*destroy)(rspamd_expression_atom_t *atom);
+};
+
+/* Opaque structure */
+struct rspamd_expression;
+
+/**
+ * Parse symbolic expression and create the expression using the specified subroutines for atoms processing
+ * @param line line to parse
+ * @param len length of the line (if 0 then line should be NULL terminated)
+ * @param subr subroutines for atoms parsing
+ * @param subr_data opaque dat pointer
+ * @param pool pool to use for memory allocations
+ * @param err error pointer
+ * @param target the target expression
+ * @return TRUE if an expression have been parsed
+ */
+gboolean rspamd_parse_expression(const gchar *line, gsize len,
+ const struct rspamd_atom_subr *subr, gpointer subr_data,
+ rspamd_mempool_t *pool, GError **err,
+ struct rspamd_expression **target);
+
+/**
+ * Process the expression and return its value using atom 'process' functions with the specified data pointer
+ * @param expr expression to process
+ * @param data opaque data pointer for all the atoms
+ * @return the value of expression
+ */
+gdouble rspamd_process_expression(struct rspamd_expression *expr,
+ gint flags,
+ gpointer runtime_ud);
+
+/**
+ * Process the expression and return its value using atom 'process' functions with the specified data pointer.
+ * This function also accepts `track` argument where it writes matched atoms (those whose value is more than 0)
+ * @param expr expression to process
+ * @param data opaque data pointer for all the atoms
+ * @param track pointer array to atoms tracking
+ * @return the value of expression
+ */
+gdouble rspamd_process_expression_track(struct rspamd_expression *expr,
+ gint flags,
+ gpointer runtime_ud,
+ GPtrArray **track);
+
+/**
+ * Process the expression with the custom processor
+ * @param expr
+ * @param cb
+ * @param process_data
+ * @return
+ */
+gdouble rspamd_process_expression_closure(struct rspamd_expression *expr,
+ rspamd_expression_process_cb cb,
+ gint flags,
+ gpointer runtime_ud,
+ GPtrArray **track);
+
+/**
+ * Shows string representation of an expression
+ * @param expr expression to show
+ * @return freshly allocated string with expression
+ */
+GString *rspamd_expression_tostring(struct rspamd_expression *expr);
+
+/**
+ * Callback that is called on @see rspamd_expression_atom_foreach, atom is ephemeral
+ * and should not be modified within callback
+ */
+typedef void (*rspamd_expression_atom_foreach_cb)(const rspamd_ftok_t *atom,
+ gpointer ud);
+
+/**
+ * Traverse over all atoms in the expression
+ * @param expr expression
+ * @param cb callback to be called
+ * @param ud opaque data passed to `cb`
+ */
+void rspamd_expression_atom_foreach(struct rspamd_expression *expr,
+ rspamd_expression_atom_foreach_cb cb, gpointer cbdata);
+
+/**
+ * Checks if a specified node in AST is the specified operation
+ * @param node AST node packed in GNode container
+ * @param op operation to check
+ * @return TRUE if node is operation node and is exactly the specified option
+ */
+gboolean rspamd_expression_node_is_op(GNode *node, enum rspamd_expression_op op);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBUTIL_EXPRESSION_H_ */
diff --git a/src/libutil/fstring.c b/src/libutil/fstring.c
new file mode 100644
index 0000000..a921f32
--- /dev/null
+++ b/src/libutil/fstring.c
@@ -0,0 +1,482 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "fstring.h"
+#include "str_util.h"
+#include "contrib/fastutf8/fastutf8.h"
+#include "contrib/mumhash/mum.h"
+
+
+#ifdef WITH_JEMALLOC
+#include <jemalloc/jemalloc.h>
+#if (JEMALLOC_VERSION_MAJOR == 3 && JEMALLOC_VERSION_MINOR >= 6) || (JEMALLOC_VERSION_MAJOR > 3)
+#define HAVE_MALLOC_SIZE 1
+#define sys_alloc_size(sz) nallocx(sz, 0)
+#endif
+#elif defined(__APPLE__)
+#include <malloc/malloc.h>
+#define HAVE_MALLOC_SIZE 1
+#define sys_alloc_size(sz) malloc_good_size(sz)
+#endif
+
+static const gsize default_initial_size = 16;
+
+#define fstravail(s) ((s)->allocated - (s)->len)
+
+rspamd_fstring_t *
+rspamd_fstring_new(void)
+{
+ rspamd_fstring_t *s;
+
+ if ((s = malloc(default_initial_size + sizeof(*s))) == NULL) {
+ g_error("%s: failed to allocate %" G_GSIZE_FORMAT " bytes",
+ G_STRLOC, default_initial_size + sizeof(*s));
+
+ return NULL;
+ }
+
+ s->len = 0;
+ s->allocated = default_initial_size;
+
+ return s;
+}
+
+rspamd_fstring_t *
+rspamd_fstring_sized_new(gsize initial_size)
+{
+ rspamd_fstring_t *s;
+ gsize real_size = MAX(default_initial_size, initial_size);
+
+ if ((s = malloc(real_size + sizeof(*s))) == NULL) {
+ g_error("%s: failed to allocate %" G_GSIZE_FORMAT " bytes",
+ G_STRLOC, real_size + sizeof(*s));
+
+ return NULL;
+ }
+ s->len = 0;
+ s->allocated = real_size;
+
+ return s;
+}
+
+rspamd_fstring_t *
+rspamd_fstring_new_init(const gchar *init, gsize len)
+{
+ rspamd_fstring_t *s;
+ gsize real_size = MAX(default_initial_size, len);
+
+ if ((s = malloc(real_size + sizeof(*s))) == NULL) {
+ g_error("%s: failed to allocate %" G_GSIZE_FORMAT " bytes",
+ G_STRLOC, real_size + sizeof(*s));
+
+ abort();
+ }
+
+ s->len = len;
+ s->allocated = real_size;
+ memcpy(s->str, init, len);
+
+ return s;
+}
+
+rspamd_fstring_t *
+rspamd_fstring_assign(rspamd_fstring_t *str, const gchar *init, gsize len)
+{
+ gsize avail;
+
+ if (str == NULL) {
+ return rspamd_fstring_new_init(init, len);
+ }
+
+ avail = fstravail(str);
+
+ if (avail < len) {
+ str = rspamd_fstring_grow(str, len);
+ }
+
+ if (len > 0) {
+ memcpy(str->str, init, len);
+ }
+
+ str->len = len;
+
+ return str;
+}
+
+void rspamd_fstring_free(rspamd_fstring_t *str)
+{
+ free(str);
+}
+
+inline gsize
+rspamd_fstring_suggest_size(gsize len, gsize allocated, gsize needed_len)
+{
+ gsize newlen, optlen = 0;
+
+ if (allocated < 4096) {
+ newlen = MAX(len + needed_len, allocated * 2);
+ }
+ else {
+ newlen = MAX(len + needed_len, 1 + allocated * 3 / 2);
+ }
+
+#ifdef HAVE_MALLOC_SIZE
+ optlen = sys_alloc_size(newlen + sizeof(rspamd_fstring_t));
+#endif
+
+ return MAX(newlen, optlen);
+}
+
+rspamd_fstring_t *
+rspamd_fstring_grow(rspamd_fstring_t *str, gsize needed_len)
+{
+ gsize newlen;
+ gpointer nptr;
+
+ newlen = rspamd_fstring_suggest_size(str->len, str->allocated, needed_len);
+
+ nptr = realloc(str, newlen + sizeof(*str));
+
+ if (nptr == NULL) {
+ /* Avoid memory leak */
+ free(str);
+ g_error("%s: failed to re-allocate %" G_GSIZE_FORMAT " bytes",
+ G_STRLOC, newlen + sizeof(*str));
+ abort();
+ }
+
+ str = nptr;
+ str->allocated = newlen;
+
+ return str;
+}
+
+rspamd_fstring_t *
+rspamd_fstring_append(rspamd_fstring_t *str, const char *in, gsize len)
+{
+ if (str == NULL) {
+ str = rspamd_fstring_new_init(in, len);
+ }
+ else {
+ gsize avail = fstravail(str);
+
+ if (avail < len) {
+ str = rspamd_fstring_grow(str, len);
+ }
+
+ memcpy(str->str + str->len, in, len);
+ str->len += len;
+ }
+
+ return str;
+}
+
+rspamd_fstring_t *
+rspamd_fstring_append_chars(rspamd_fstring_t *str,
+ char c, gsize len)
+{
+ if (str == NULL) {
+ str = rspamd_fstring_sized_new(len);
+
+ memset(str->str + str->len, c, len);
+ str->len += len;
+ }
+ else {
+ gsize avail = fstravail(str);
+
+ if (avail < len) {
+ str = rspamd_fstring_grow(str, len);
+ }
+
+ memset(str->str + str->len, c, len);
+ str->len += len;
+ }
+
+ return str;
+}
+
+void rspamd_fstring_erase(rspamd_fstring_t *str, gsize pos, gsize len)
+{
+ if (pos < str->len) {
+ if (pos + len > str->len) {
+ len = str->len - pos;
+ }
+
+ if (len == str->len - pos) {
+ /* Fast path */
+ str->len = pos;
+ }
+ else {
+ memmove(str->str + pos, str->str + pos + len, str->len - pos);
+ str->len -= pos;
+ }
+ }
+ else {
+ /* Do nothing */
+ }
+}
+
+/* Compat code */
+static guint64
+fstrhash_c(guint64 c, guint64 hval)
+{
+ return mum_hash_step(hval, c);
+}
+
+
+/*
+ * Return hash value for a string
+ */
+guint32
+rspamd_fstrhash_lc(const rspamd_ftok_t *str, gboolean is_utf)
+{
+ gsize i;
+ guint64 hval;
+ const gchar *p, *end = NULL;
+ gunichar uc;
+
+ if (str == NULL) {
+ return 0;
+ }
+
+ p = str->begin;
+ hval = str->len;
+ end = p + str->len;
+
+ if (is_utf) {
+ if (rspamd_fast_utf8_validate(p, str->len) != 0) {
+ return rspamd_fstrhash_lc(str, FALSE);
+ }
+ while (p < end) {
+ uc = g_unichar_tolower(g_utf8_get_char(p));
+ hval = fstrhash_c(uc, hval);
+ p = g_utf8_next_char(p);
+ }
+ }
+ else {
+ gsize large_steps = str->len / sizeof(guint64);
+ for (i = 0; i < large_steps; i++, p += sizeof(guint64)) {
+ /* Copy to the uint64 lowercasing each byte */
+ union {
+ char c[sizeof(guint64)];
+ guint64 iu64;
+ } t;
+ for (int j = 0; j < sizeof(guint64); j++) {
+ t.c[j] = g_ascii_tolower(p[j]);
+ }
+ hval = fstrhash_c(t.iu64, hval);
+ }
+
+ gsize remain = str->len % sizeof(guint64);
+ for (i = 0; i < remain; i++, p++) {
+ hval = fstrhash_c(g_ascii_tolower(*p), hval);
+ }
+ }
+
+ return hval;
+}
+
+gboolean
+rspamd_fstring_equal(const rspamd_fstring_t *s1,
+ const rspamd_fstring_t *s2)
+{
+ g_assert(s1 != NULL && s2 != NULL);
+
+ if (s1->len == s2->len) {
+ return (memcmp(s1->str, s2->str, s1->len) == 0);
+ }
+
+ return FALSE;
+}
+
+gint rspamd_fstring_casecmp(const rspamd_fstring_t *s1,
+ const rspamd_fstring_t *s2)
+{
+ gint ret = 0;
+
+ g_assert(s1 != NULL && s2 != NULL);
+
+ if (s1->len == s2->len) {
+ ret = rspamd_lc_cmp(s1->str, s2->str, s1->len);
+ }
+ else {
+ ret = s1->len - s2->len;
+ }
+
+ return ret;
+}
+
+gint rspamd_fstring_cmp(const rspamd_fstring_t *s1,
+ const rspamd_fstring_t *s2)
+{
+ g_assert(s1 != NULL && s2 != NULL);
+
+ if (s1->len == s2->len) {
+ return memcmp(s1->str, s2->str, s1->len);
+ }
+
+ return s1->len - s2->len;
+}
+
+gint rspamd_ftok_casecmp(const rspamd_ftok_t *s1,
+ const rspamd_ftok_t *s2)
+{
+ gint ret = 0;
+
+ g_assert(s1 != NULL && s2 != NULL);
+
+ if (s1->len == s2->len) {
+ ret = rspamd_lc_cmp(s1->begin, s2->begin, s1->len);
+ }
+ else {
+ ret = s1->len - s2->len;
+ }
+
+ return ret;
+}
+
+gint rspamd_ftok_cmp(const rspamd_ftok_t *s1,
+ const rspamd_ftok_t *s2)
+{
+ g_assert(s1 != NULL && s2 != NULL);
+
+ if (s1->len == s2->len) {
+ return memcmp(s1->begin, s2->begin, s1->len);
+ }
+
+ return s1->len - s2->len;
+}
+
+gboolean
+rspamd_ftok_starts_with(const rspamd_ftok_t *s1,
+ const rspamd_ftok_t *s2)
+{
+ g_assert(s1 != NULL && s2 != NULL);
+
+ if (s1->len >= s2->len) {
+ return !!(memcmp(s1->begin, s2->begin, s2->len) == 0);
+ }
+
+ return FALSE;
+}
+
+void rspamd_fstring_mapped_ftok_free(gpointer p)
+{
+ rspamd_ftok_t *tok = p;
+ rspamd_fstring_t *storage;
+
+ storage = (rspamd_fstring_t *) (tok->begin - 2 * sizeof(gsize));
+ rspamd_fstring_free(storage);
+ g_free(tok);
+}
+
+rspamd_ftok_t *
+rspamd_ftok_map(const rspamd_fstring_t *s)
+{
+ rspamd_ftok_t *tok;
+
+ g_assert(s != NULL);
+
+ tok = g_malloc(sizeof(*tok));
+ tok->begin = s->str;
+ tok->len = s->len;
+
+ return tok;
+}
+
+char *
+rspamd_fstring_cstr(const rspamd_fstring_t *s)
+{
+ char *result;
+
+ if (s == NULL) {
+ return NULL;
+ }
+
+ result = g_malloc(s->len + 1);
+ memcpy(result, s->str, s->len);
+ result[s->len] = '\0';
+
+ return result;
+}
+
+char *
+rspamd_ftok_cstr(const rspamd_ftok_t *s)
+{
+ char *result;
+
+ if (s == NULL) {
+ return NULL;
+ }
+
+ result = g_malloc(s->len + 1);
+ memcpy(result, s->begin, s->len);
+ result[s->len] = '\0';
+
+ return result;
+}
+
+gboolean
+rspamd_ftok_cstr_equal(const rspamd_ftok_t *s, const gchar *pat,
+ gboolean icase)
+{
+ gsize slen;
+ rspamd_ftok_t srch;
+
+ g_assert(s != NULL);
+ g_assert(pat != NULL);
+
+ slen = strlen(pat);
+ srch.begin = pat;
+ srch.len = slen;
+
+ if (icase) {
+ return (rspamd_ftok_casecmp(s, &srch) == 0);
+ }
+
+ return (rspamd_ftok_cmp(s, &srch) == 0);
+}
+
+gchar *
+rspamd_ftokdup(const rspamd_ftok_t *src)
+{
+ gchar *newstr;
+
+ if (src == NULL) {
+ return NULL;
+ }
+
+ newstr = g_malloc(src->len + 1);
+ memcpy(newstr, src->begin, src->len);
+ newstr[src->len] = '\0';
+
+ return newstr;
+}
+
+gchar *
+rspamd_fstringdup(const rspamd_fstring_t *src)
+{
+ gchar *newstr;
+
+ if (src == NULL) {
+ return NULL;
+ }
+
+ newstr = g_malloc(src->len + 1);
+ memcpy(newstr, src->str, src->len);
+ newstr[src->len] = '\0';
+
+ return newstr;
+}
diff --git a/src/libutil/fstring.h b/src/libutil/fstring.h
new file mode 100644
index 0000000..9eacf21
--- /dev/null
+++ b/src/libutil/fstring.h
@@ -0,0 +1,231 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FSTRING_H
+#define FSTRING_H
+
+#include "config.h"
+#include "mem_pool.h"
+#include <unicode/uchar.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Fixed strings library
+ * These strings are NOT null-terminated for speed
+ */
+
+typedef struct f_str_s {
+ gsize len;
+ gsize allocated;
+ gchar str[];
+} rspamd_fstring_t;
+
+#define RSPAMD_FSTRING_DATA(s) ((s)->str)
+#define RSPAMD_FSTRING_LEN(s) ((s)->len)
+#define RSPAMD_FSTRING_LIT(lit) rspamd_fstring_new_init((lit), sizeof(lit) - 1)
+
+typedef struct f_str_tok {
+ gsize len;
+ const gchar *begin;
+} rspamd_ftok_t;
+
+typedef struct f_str_unicode_tok {
+ gsize len; /* in UChar32 */
+ const UChar32 *begin;
+} rspamd_ftok_unicode_t;
+
+/**
+ * Create new fixed length string
+ */
+rspamd_fstring_t *rspamd_fstring_new(void)
+ G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Create new fixed length string with preallocated size
+ */
+rspamd_fstring_t *rspamd_fstring_sized_new(gsize initial_size)
+ G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Create new fixed length string and initialize it with the initial data
+ */
+rspamd_fstring_t *rspamd_fstring_new_init(const gchar *init, gsize len)
+ G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Assign new value to fixed string
+ */
+rspamd_fstring_t *rspamd_fstring_assign(rspamd_fstring_t *str,
+ const gchar *init, gsize len) G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Free fixed length string
+ */
+void rspamd_fstring_free(rspamd_fstring_t *str);
+
+/**
+ * Append data to a fixed length string
+ */
+rspamd_fstring_t *rspamd_fstring_append(rspamd_fstring_t *str,
+ const char *in, gsize len) G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Append `len` repeated chars `c` to string `str`
+ */
+rspamd_fstring_t *rspamd_fstring_append_chars(rspamd_fstring_t *str,
+ char c, gsize len) G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Erase `len` characters at position `pos`
+ */
+void rspamd_fstring_erase(rspamd_fstring_t *str, gsize pos, gsize len);
+
+#define rspamd_fstring_clear(s) rspamd_fstring_erase(s, 0, s->len)
+
+/**
+ * Convert fixed string to a zero terminated string. This string must be
+ * freed by a caller
+ */
+char *rspamd_fstring_cstr(const rspamd_fstring_t *str)
+ G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Convert fixed string usign ftok_t to a zero terminated string. This string must be
+ * freed by a caller
+ */
+char *rspamd_ftok_cstr(const rspamd_ftok_t *str)
+ G_GNUC_WARN_UNUSED_RESULT;
+
+/*
+ * Return fast hash value for fixed string converted to lowercase
+ */
+guint32 rspamd_fstrhash_lc(const rspamd_ftok_t *str, gboolean is_utf);
+
+/**
+ * Return true if two strings are equal
+ */
+gboolean rspamd_fstring_equal(const rspamd_fstring_t *s1,
+ const rspamd_fstring_t *s2);
+
+/**
+ * Compare two fixed strings ignoring case
+ */
+gint rspamd_fstring_casecmp(const rspamd_fstring_t *s1,
+ const rspamd_fstring_t *s2);
+
+/**
+ * Compare two fixed strings
+ */
+gint rspamd_fstring_cmp(const rspamd_fstring_t *s1,
+ const rspamd_fstring_t *s2);
+
+/**
+ * Compare two fixed tokens ignoring case
+ */
+gint rspamd_ftok_casecmp(const rspamd_ftok_t *s1,
+ const rspamd_ftok_t *s2);
+
+/**
+ * Compare two fixed tokens
+ */
+gint rspamd_ftok_cmp(const rspamd_ftok_t *s1,
+ const rspamd_ftok_t *s2);
+
+/**
+ * Returns true if `s1` starts with `s2`
+ * @param s1
+ * @param s2
+ * @return
+ */
+gboolean rspamd_ftok_starts_with(const rspamd_ftok_t *s1,
+ const rspamd_ftok_t *s2);
+
+/**
+ * Return TRUE if ftok is equal to specified C string
+ */
+gboolean rspamd_ftok_cstr_equal(const rspamd_ftok_t *s,
+ const gchar *pat, gboolean icase);
+
+/**
+ * Free fstring_t that is mapped to ftok_t
+ *
+ * | len | allocated | <data> -- fstring_t
+ * <begin> -- tok
+ *
+ * tok is expected to be allocated with g_malloc
+ */
+void rspamd_fstring_mapped_ftok_free(gpointer p);
+
+/**
+ * Map token to a specified string. Token must be freed using g_free
+ */
+rspamd_ftok_t *rspamd_ftok_map(const rspamd_fstring_t *s);
+
+/**
+ * Suggest suitable size to grow fstring
+ * @param len
+ * @param allocated
+ * @param needed_len
+ * @return
+ */
+gsize rspamd_fstring_suggest_size(gsize len, gsize allocated, gsize needed_len);
+
+/**
+ * Grow the specified fixed string
+ * @param str
+ * @param needed_len
+ * @return
+ */
+rspamd_fstring_t *rspamd_fstring_grow(rspamd_fstring_t *str,
+ gsize needed_len) G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Copies ftok to zero terminated string (must be freed using g_free)
+ * @param src
+ * @return
+ */
+gchar *rspamd_ftokdup(const rspamd_ftok_t *src) G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Copies fstring to zero terminated string (must be freed using g_free)
+ * @param src
+ * @return
+ */
+gchar *rspamd_fstringdup(const rspamd_fstring_t *src) G_GNUC_WARN_UNUSED_RESULT;
+
+#define RSPAMD_FTOK_ASSIGN(t, lit) \
+ do { \
+ (t)->begin = (lit); \
+ (t)->len = sizeof(lit) - 1; \
+ } while (0)
+#define RSPAMD_FTOK_FROM_STR(t, str) \
+ do { \
+ if (G_LIKELY(str)) { \
+ (t)->begin = (const char *) (str); \
+ (t)->len = strlen(str); \
+ } \
+ else { \
+ (t)->begin = NULL; \
+ (t)->len = 0; \
+ } \
+ } while (0)
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/src/libutil/hash.c b/src/libutil/hash.c
new file mode 100644
index 0000000..d2af88c
--- /dev/null
+++ b/src/libutil/hash.c
@@ -0,0 +1,716 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "hash.h"
+#include "util.h"
+#include "khash.h"
+
+/**
+ * LRU hashing
+ */
+
+static const guint log_base = 10;
+static const guint eviction_candidates = 16;
+static const gdouble lfu_base_value = 5.0;
+
+struct rspamd_lru_volatile_element_s;
+
+struct rspamd_lru_hash_s {
+ guint maxsize;
+ guint eviction_min_prio;
+ guint eviction_used;
+ struct rspamd_lru_element_s **eviction_pool;
+
+ GDestroyNotify value_destroy;
+ GDestroyNotify key_destroy;
+ GHashFunc hfunc;
+ GEqualFunc eqfunc;
+
+ khint_t n_buckets, size, n_occupied, upper_bound;
+ khint32_t *flags;
+ gpointer *keys;
+ struct rspamd_lru_volatile_element_s *vals;
+};
+
+enum rspamd_lru_element_flags {
+ RSPAMD_LRU_ELEMENT_NORMAL = 0,
+ RSPAMD_LRU_ELEMENT_VOLATILE = (1 << 0),
+ RSPAMD_LRU_ELEMENT_IMMORTAL = (1 << 1),
+};
+
+struct rspamd_lru_element_s {
+ guint16 last;
+ guint8 lg_usages;
+ guint8 eviction_pos;
+ guint8 flags;
+ gpointer data;
+};
+
+struct rspamd_lru_volatile_element_s {
+ struct rspamd_lru_element_s e;
+ time_t creation_time;
+ time_t ttl;
+};
+typedef struct rspamd_lru_volatile_element_s rspamd_lru_vol_element_t;
+
+#define TIME_TO_TS(t) ((guint16) (((t) / 60) & 0xFFFFU))
+
+static rspamd_lru_vol_element_t *
+rspamd_lru_hash_get(const rspamd_lru_hash_t *h, gconstpointer key)
+{
+ if (h->n_buckets) {
+ khint_t k, i, last, mask, step = 0;
+ mask = h->n_buckets - 1;
+ k = h->hfunc(key);
+ i = k & mask;
+ last = i;
+
+ while (!__ac_isempty(h->flags, i) &&
+ (__ac_isdel(h->flags, i) || !h->eqfunc(h->keys[i], key))) {
+ i = (i + (++step)) & mask;
+ if (i == last) {
+ return NULL;
+ }
+ }
+
+ return __ac_iseither(h->flags, i) ? NULL : &h->vals[i];
+ }
+
+ return NULL;
+}
+
+static int
+rspamd_lru_hash_resize(rspamd_lru_hash_t *h,
+ khint_t new_n_buckets)
+{
+ /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */
+ khint32_t *new_flags = 0;
+ khint_t j = 1;
+
+ kroundup32(new_n_buckets);
+ if (new_n_buckets < 4) {
+ new_n_buckets = 4;
+ }
+
+ if (h->size >= (khint_t) (new_n_buckets * __ac_HASH_UPPER + 0.5)) {
+ j = 0;
+ /* requested size is too small */
+ }
+ else {
+ /* hash table size to be changed (shrink or expand); rehash */
+ new_flags = (khint32_t *) g_malloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t));
+
+ if (!new_flags) {
+ return -1;
+ }
+
+ memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t));
+ if (h->n_buckets < new_n_buckets) {
+ /* expand */
+ gpointer *new_keys = (gpointer *) g_realloc((void *) h->keys,
+ new_n_buckets * sizeof(gpointer));
+
+ if (!new_keys) {
+ g_free(new_flags);
+ return -1;
+ }
+
+ h->keys = new_keys;
+ rspamd_lru_vol_element_t *new_vals =
+ (rspamd_lru_vol_element_t *) g_realloc((void *) h->vals,
+ new_n_buckets * sizeof(rspamd_lru_vol_element_t));
+ if (!new_vals) {
+ g_free(new_flags);
+ return -1;
+ }
+
+ h->vals = new_vals;
+ }
+ /* Shrink */
+ }
+
+ if (j) {
+ /* rehashing is needed */
+ h->eviction_used = 0;
+
+ for (j = 0; j != h->n_buckets; ++j) {
+ if (__ac_iseither(h->flags, j) == 0) {
+ gpointer key = h->keys[j];
+ rspamd_lru_vol_element_t val;
+ khint_t new_mask;
+ new_mask = new_n_buckets - 1;
+ val = h->vals[j];
+ val.e.eviction_pos = (guint8) -1;
+ __ac_set_isdel_true(h->flags, j);
+
+ while (1) { /* kick-out process; sort of like in Cuckoo hashing */
+ khint_t k, i, step = 0;
+ k = h->hfunc(key);
+ i = k & new_mask;
+
+ while (!__ac_isempty(new_flags, i)) {
+ i = (i + (++step)) & new_mask;
+ }
+
+ __ac_set_isempty_false(new_flags, i);
+
+ if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) {
+ /* kick out the existing element */
+ {
+ gpointer tmp = h->keys[i];
+ h->keys[i] = key;
+ key = tmp;
+ }
+ {
+ rspamd_lru_vol_element_t tmp = h->vals[i];
+ h->vals[i] = val;
+ val = tmp;
+ val.e.eviction_pos = (guint8) -1;
+ }
+ __ac_set_isdel_true(h->flags, i);
+ /* mark it as deleted in the old hash table */
+ }
+ else { /* write the element and jump out of the loop */
+ h->keys[i] = key;
+ h->vals[i] = val;
+ break;
+ }
+ }
+ }
+ }
+
+ if (h->n_buckets > new_n_buckets) {
+ /* shrink the hash table */
+ h->keys = (gpointer *) g_realloc((void *) h->keys,
+ new_n_buckets * sizeof(gpointer));
+ h->vals = (rspamd_lru_vol_element_t *) g_realloc((void *) h->vals,
+ new_n_buckets * sizeof(rspamd_lru_vol_element_t));
+ }
+
+ g_free(h->flags); /* free the working space */
+ h->flags = new_flags;
+ h->n_buckets = new_n_buckets;
+ h->n_occupied = h->size;
+ h->upper_bound = (khint_t) (h->n_buckets * __ac_HASH_UPPER + 0.5);
+ }
+
+ return 0;
+}
+
+static rspamd_lru_vol_element_t *
+rspamd_lru_hash_put(rspamd_lru_hash_t *h, gpointer key, int *ret)
+{
+ khint_t x;
+
+ if (h->n_occupied >= h->upper_bound) {
+ /* update the hash table */
+ if (h->n_buckets > (h->size << 1)) {
+ if (rspamd_lru_hash_resize(h, h->n_buckets - 1) < 0) {
+ /* clear "deleted" elements */
+ *ret = -1;
+ return NULL;
+ }
+ }
+ else if (rspamd_lru_hash_resize(h, h->n_buckets + 1) < 0) {
+ /* expand the hash table */
+ *ret = -1;
+ return NULL;
+ }
+ }
+
+ khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0;
+ x = site = h->n_buckets;
+ k = h->hfunc(key);
+ i = k & mask;
+
+ if (__ac_isempty(h->flags, i)) {
+ x = i; /* for speed up */
+ }
+ else {
+ last = i;
+ while (!__ac_isempty(h->flags, i) &&
+ (__ac_isdel(h->flags, i) ||
+ !h->eqfunc(h->keys[i], key))) {
+ if (__ac_isdel(h->flags, i)) {
+ site = i;
+ }
+
+ i = (i + (++step)) & mask;
+
+ if (i == last) {
+ x = site;
+ break;
+ }
+ }
+
+ if (x == h->n_buckets) {
+ if (__ac_isempty(h->flags, i) && site != h->n_buckets) {
+ x = site;
+ }
+ else {
+ x = i;
+ }
+ }
+ }
+
+ if (__ac_isempty(h->flags, x)) { /* not present at all */
+ h->keys[x] = key;
+ __ac_set_isboth_false(h->flags, x);
+ ++h->size;
+ ++h->n_occupied;
+ *ret = 1;
+ }
+ else if (__ac_isdel(h->flags, x)) { /* deleted */
+ h->keys[x] = key;
+ __ac_set_isboth_false(h->flags, x);
+ ++h->size;
+ *ret = 2;
+ }
+ else {
+ /* Don't touch h->keys[x] if present and not deleted */
+ *ret = 0;
+ }
+
+ return &h->vals[x];
+}
+
+static void
+rspamd_lru_hash_del(rspamd_lru_hash_t *h, rspamd_lru_vol_element_t *elt)
+{
+ khint_t x = elt - h->vals;
+
+ if (x != h->n_buckets && !__ac_iseither(h->flags, x)) {
+ __ac_set_isdel_true(h->flags, x);
+ --h->size;
+
+ if (h->key_destroy) {
+ h->key_destroy(h->keys[x]);
+ }
+
+ if (h->value_destroy) {
+ h->value_destroy(elt->e.data);
+ }
+ }
+}
+
+static void
+rspamd_lru_hash_remove_evicted(rspamd_lru_hash_t *hash,
+ rspamd_lru_element_t *elt)
+{
+ guint i;
+ rspamd_lru_element_t *cur;
+
+ g_assert(hash->eviction_used > 0);
+ g_assert(elt->eviction_pos < hash->eviction_used);
+
+ memmove(&hash->eviction_pool[elt->eviction_pos],
+ &hash->eviction_pool[elt->eviction_pos + 1],
+ sizeof(rspamd_lru_element_t *) *
+ (eviction_candidates - elt->eviction_pos - 1));
+
+ hash->eviction_used--;
+
+ if (hash->eviction_used > 0) {
+ /* We also need to update min_prio and renumber eviction list */
+ hash->eviction_min_prio = G_MAXUINT;
+
+ for (i = 0; i < hash->eviction_used; i++) {
+ cur = hash->eviction_pool[i];
+
+ if (hash->eviction_min_prio > cur->lg_usages) {
+ hash->eviction_min_prio = cur->lg_usages;
+ }
+
+ cur->eviction_pos = i;
+ }
+ }
+ else {
+ hash->eviction_min_prio = G_MAXUINT;
+ }
+}
+
+static void
+rspamd_lru_hash_update_counter(rspamd_lru_element_t *elt)
+{
+ guint8 counter = elt->lg_usages;
+
+ if (counter != 255) {
+ double r, baseval, p;
+
+ r = rspamd_random_double_fast();
+ baseval = counter - lfu_base_value;
+
+ if (baseval < 0) {
+ baseval = 0;
+ }
+
+ p = 1.0 / (baseval * log_base + 1);
+
+ if (r < p) {
+ elt->lg_usages++;
+ }
+ }
+}
+
+static inline void
+rspamd_lru_hash_decrease_counter(rspamd_lru_element_t *elt, time_t now)
+{
+ if (now - elt->last > lfu_base_value) {
+ /* Penalise counters for outdated records */
+ elt->lg_usages /= 2;
+ }
+}
+
+static gboolean
+rspamd_lru_hash_maybe_evict(rspamd_lru_hash_t *hash,
+ rspamd_lru_element_t *elt)
+{
+ guint i;
+ rspamd_lru_element_t *cur;
+
+ if (elt->eviction_pos == (guint8) -1) {
+ if (hash->eviction_used < eviction_candidates) {
+ /* There are free places in eviction pool */
+ hash->eviction_pool[hash->eviction_used] = elt;
+ elt->eviction_pos = hash->eviction_used;
+ hash->eviction_used++;
+
+ if (hash->eviction_min_prio > elt->lg_usages) {
+ hash->eviction_min_prio = elt->lg_usages;
+ }
+
+ return TRUE;
+ }
+ else {
+ /* Find any candidate that has higher usage count */
+ for (i = 0; i < hash->eviction_used; i++) {
+ cur = hash->eviction_pool[i];
+
+ if (cur->lg_usages > elt->lg_usages) {
+ cur->eviction_pos = -1;
+ elt->eviction_pos = i;
+ hash->eviction_pool[i] = elt;
+
+ if (hash->eviction_min_prio > elt->lg_usages) {
+ hash->eviction_min_prio = elt->lg_usages;
+ }
+
+ return TRUE;
+ }
+ }
+ }
+ }
+ else {
+ /* Already in the eviction list */
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static void
+rspamd_lru_hash_remove_node(rspamd_lru_hash_t *hash, rspamd_lru_element_t *elt)
+{
+ if (elt->eviction_pos != (guint8) -1) {
+ rspamd_lru_hash_remove_evicted(hash, elt);
+ }
+
+ rspamd_lru_hash_del(hash, (rspamd_lru_vol_element_t *) elt);
+}
+
+static void
+rspamd_lru_hash_evict(rspamd_lru_hash_t *hash, time_t now)
+{
+ double r;
+ guint i;
+ rspamd_lru_element_t *elt = NULL;
+ guint nexpired = 0;
+
+ /*
+ * We either evict one node from the eviction list
+ * or, at some probability scan all table and update eviction
+ * list first
+ */
+ r = rspamd_random_double_fast();
+
+ if (r < ((double) eviction_candidates) / hash->maxsize) {
+ /* Full hash scan */
+ rspamd_lru_vol_element_t *cur;
+ rspamd_lru_element_t *selected = NULL;
+
+ kh_foreach_value_ptr(hash, cur, {
+ rspamd_lru_element_t *node = &cur->e;
+
+ if (node->flags & RSPAMD_LRU_ELEMENT_IMMORTAL) {
+ continue;
+ }
+
+ if (node->flags & RSPAMD_LRU_ELEMENT_VOLATILE) {
+ /* If element is expired, just remove it */
+ if (now - cur->creation_time > cur->ttl) {
+ rspamd_lru_hash_remove_node(hash, node);
+
+ nexpired++;
+ continue;
+ }
+ }
+ else {
+ rspamd_lru_hash_decrease_counter(node, now);
+
+ if (rspamd_lru_hash_maybe_evict(hash, node)) {
+ if (selected && node->lg_usages < selected->lg_usages) {
+ selected = node;
+ }
+ else if (selected == NULL) {
+ selected = node;
+ }
+ }
+ }
+ });
+
+ if (selected) {
+ elt = selected;
+ }
+ }
+ else {
+ /* Fast random eviction */
+ for (i = 0; i < hash->eviction_used; i++) {
+ elt = hash->eviction_pool[i];
+
+ if (elt->lg_usages <= hash->eviction_min_prio) {
+ break;
+ }
+ }
+ }
+
+ /* Evict if nothing else has been cleaned */
+ if (elt && nexpired == 0) {
+ rspamd_lru_hash_remove_node(hash, elt);
+ }
+}
+
+rspamd_lru_hash_t *
+rspamd_lru_hash_new_full(gint maxsize,
+ GDestroyNotify key_destroy,
+ GDestroyNotify value_destroy,
+ GHashFunc hf,
+ GEqualFunc cmpf)
+{
+ rspamd_lru_hash_t *h;
+
+ if (maxsize < eviction_candidates * 2) {
+ maxsize = eviction_candidates * 2;
+ }
+
+ h = g_malloc0(sizeof(rspamd_lru_hash_t));
+ h->hfunc = hf;
+ h->eqfunc = cmpf;
+ h->eviction_pool = g_malloc0(sizeof(rspamd_lru_element_t *) *
+ eviction_candidates);
+ h->maxsize = maxsize;
+ h->value_destroy = value_destroy;
+ h->key_destroy = key_destroy;
+ h->eviction_min_prio = G_MAXUINT;
+
+ /* Preallocate some elements */
+ rspamd_lru_hash_resize(h, MIN(h->maxsize, 128));
+
+ return h;
+}
+
+rspamd_lru_hash_t *
+rspamd_lru_hash_new(gint maxsize,
+ GDestroyNotify key_destroy,
+ GDestroyNotify value_destroy)
+{
+ return rspamd_lru_hash_new_full(maxsize,
+ key_destroy, value_destroy,
+ rspamd_strcase_hash, rspamd_strcase_equal);
+}
+
+gpointer
+rspamd_lru_hash_lookup(rspamd_lru_hash_t *hash, gconstpointer key, time_t now)
+{
+ rspamd_lru_element_t *res;
+ rspamd_lru_vol_element_t *vnode;
+
+ vnode = rspamd_lru_hash_get(hash, (gpointer) key);
+ if (vnode != NULL) {
+ res = &vnode->e;
+
+ if (res->flags & RSPAMD_LRU_ELEMENT_VOLATILE) {
+ /* Check ttl */
+
+ if (now - vnode->creation_time > vnode->ttl) {
+ rspamd_lru_hash_remove_node(hash, res);
+
+ return NULL;
+ }
+ }
+
+ now = TIME_TO_TS(now);
+ res->last = MAX(res->last, now);
+ rspamd_lru_hash_update_counter(res);
+ rspamd_lru_hash_maybe_evict(hash, res);
+
+ return res->data;
+ }
+
+ return NULL;
+}
+
+gboolean
+rspamd_lru_hash_remove(rspamd_lru_hash_t *hash,
+ gconstpointer key)
+{
+ rspamd_lru_vol_element_t *res;
+
+ res = rspamd_lru_hash_get(hash, key);
+
+ if (res != NULL) {
+ rspamd_lru_hash_remove_node(hash, &res->e);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+void rspamd_lru_hash_insert(rspamd_lru_hash_t *hash,
+ gpointer key,
+ gpointer value,
+ time_t now,
+ guint ttl)
+{
+ rspamd_lru_element_t *node;
+ rspamd_lru_vol_element_t *vnode;
+ gint ret;
+
+ vnode = rspamd_lru_hash_put(hash, key, &ret);
+ node = &vnode->e;
+
+ if (ret == 0) {
+ /* Existing element, be careful about destructors */
+ if (hash->value_destroy) {
+ /* Remove old data */
+ hash->value_destroy(vnode->e.data);
+ }
+
+ if (hash->key_destroy) {
+ /* Here are dragons! */
+ goffset off = vnode - hash->vals;
+
+ hash->key_destroy(hash->keys[off]);
+ hash->keys[off] = key;
+ }
+ }
+
+
+ if (ttl == 0) {
+ node->flags = RSPAMD_LRU_ELEMENT_NORMAL;
+ }
+ else {
+ vnode->creation_time = now;
+ vnode->ttl = ttl;
+ node->flags = RSPAMD_LRU_ELEMENT_VOLATILE;
+ }
+
+ node->data = value;
+ node->lg_usages = (guint8) lfu_base_value;
+ node->last = TIME_TO_TS(now);
+ node->eviction_pos = (guint8) -1;
+
+ if (ret != 0) {
+ /* Also need to check maxsize */
+ if (kh_size(hash) >= hash->maxsize) {
+ node->flags |= RSPAMD_LRU_ELEMENT_IMMORTAL;
+ rspamd_lru_hash_evict(hash, now);
+ node->flags &= ~RSPAMD_LRU_ELEMENT_IMMORTAL;
+ }
+ }
+
+ rspamd_lru_hash_maybe_evict(hash, node);
+}
+
+void rspamd_lru_hash_destroy(rspamd_lru_hash_t *hash)
+{
+ if (hash) {
+ if (hash->key_destroy || hash->value_destroy) {
+ gpointer k;
+ rspamd_lru_vol_element_t cur;
+
+ kh_foreach(hash, k, cur, {
+ if (hash->key_destroy) {
+ hash->key_destroy(k);
+ }
+ if (hash->value_destroy) {
+ hash->value_destroy(cur.e.data);
+ }
+ });
+ }
+
+ g_free(hash->keys);
+ g_free(hash->vals);
+ g_free(hash->flags);
+ g_free(hash->eviction_pool);
+ g_free(hash);
+ }
+}
+
+gpointer
+rspamd_lru_hash_element_data(rspamd_lru_element_t *elt)
+{
+ return elt->data;
+}
+
+int rspamd_lru_hash_foreach(rspamd_lru_hash_t *h, int it, gpointer *k,
+ gpointer *v)
+{
+ gint i;
+ g_assert(it >= 0);
+
+ for (i = it; i != kh_end(h); ++i) {
+ if (!kh_exist(h, i)) {
+ continue;
+ }
+
+ *k = h->keys[i];
+ *v = h->vals[i].e.data;
+
+ break;
+ }
+
+ if (i == kh_end(h)) {
+ return -1;
+ }
+
+ return i + 1;
+}
+
+
+guint rspamd_lru_hash_size(rspamd_lru_hash_t *hash)
+{
+ return kh_size(hash);
+}
+
+/**
+ * Returns hash capacity
+ * @param hash hash object
+ */
+guint rspamd_lru_hash_capacity(rspamd_lru_hash_t *hash)
+{
+ return hash->maxsize;
+} \ No newline at end of file
diff --git a/src/libutil/hash.h b/src/libutil/hash.h
new file mode 100644
index 0000000..3882ce5
--- /dev/null
+++ b/src/libutil/hash.h
@@ -0,0 +1,114 @@
+/**
+ * @file hash.h
+ * Hash table implementation that allows using memory pools for storage as well as using
+ * shared memory for this purpose
+ */
+
+#ifndef RSPAMD_HASH_H
+#define RSPAMD_HASH_H
+
+#include "config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_lru_hash_s;
+typedef struct rspamd_lru_hash_s rspamd_lru_hash_t;
+struct rspamd_lru_element_s;
+typedef struct rspamd_lru_element_s rspamd_lru_element_t;
+
+
+/**
+ * Create new lru hash
+ * @param maxsize maximum elements in a hash
+ * @param maxage maximum age of element
+ * @param hash_func pointer to hash function
+ * @param key_equal_func pointer to function for comparing keys
+ * @return new rspamd_hash object
+ */
+rspamd_lru_hash_t *rspamd_lru_hash_new(gint maxsize,
+ GDestroyNotify key_destroy,
+ GDestroyNotify value_destroy);
+
+
+/**
+ * Create new lru hash
+ * @param maxsize maximum elements in a hash
+ * @param maxage maximum age of element
+ * @param hash_func pointer to hash function
+ * @param key_equal_func pointer to function for comparing keys
+ * @return new rspamd_hash object
+ */
+rspamd_lru_hash_t *rspamd_lru_hash_new_full(gint maxsize,
+ GDestroyNotify key_destroy,
+ GDestroyNotify value_destroy,
+ GHashFunc hfunc,
+ GEqualFunc eqfunc);
+
+/**
+ * Lookup item from hash
+ * @param hash hash object
+ * @param key key to find
+ * @return value of key or NULL if key is not found
+ */
+gpointer rspamd_lru_hash_lookup(rspamd_lru_hash_t *hash,
+ gconstpointer key,
+ time_t now);
+
+/**
+ * Removes key from LRU cache
+ * @param hash
+ * @param key
+ * @return TRUE if key has been found and removed
+ */
+gboolean rspamd_lru_hash_remove(rspamd_lru_hash_t *hash,
+ gconstpointer key);
+
+/**
+ * Insert item in hash
+ * @param hash hash object
+ * @param key key to insert
+ * @param value value of key
+ */
+void rspamd_lru_hash_insert(rspamd_lru_hash_t *hash,
+ gpointer key,
+ gpointer value,
+ time_t now,
+ guint ttl);
+
+/**
+ * Remove lru hash
+ * @param hash hash object
+ */
+
+void rspamd_lru_hash_destroy(rspamd_lru_hash_t *hash);
+
+/**
+ * Iterate over lru hash. Iterations must start from it=0 and are done when it==-1
+ * @param hash
+ * @param it
+ * @param k
+ * @param v
+ * @return new it or -1 if iteration has been reached over
+ */
+int rspamd_lru_hash_foreach(rspamd_lru_hash_t *hash, int it, gpointer *k,
+ gpointer *v);
+
+/**
+ * Returns number of elements in a hash
+ * @param hash hash object
+ */
+guint rspamd_lru_hash_size(rspamd_lru_hash_t *hash);
+
+/**
+ * Returns hash capacity
+ * @param hash hash object
+ */
+guint rspamd_lru_hash_capacity(rspamd_lru_hash_t *hash);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libutil/heap.c b/src/libutil/heap.c
new file mode 100644
index 0000000..8ce70cf
--- /dev/null
+++ b/src/libutil/heap.c
@@ -0,0 +1,197 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "libutil/heap.h"
+
+struct rspamd_min_heap {
+ GPtrArray *ar;
+};
+
+#define __SWAP(a, b) \
+ do { \
+ __typeof__(a) _a = (a); \
+ __typeof__(b) _b = (b); \
+ a = _b; \
+ b = _a; \
+ } while (0)
+#define heap_swap(h, e1, e2) \
+ do { \
+ __SWAP((h)->ar->pdata[(e1)->idx - 1], (h)->ar->pdata[(e2)->idx - 1]); \
+ __SWAP((e1)->idx, (e2)->idx); \
+ } while (0)
+
+#define min_elt(e1, e2) ((e1)->pri <= (e2)->pri ? (e1) : (e2))
+
+/*
+ * Swims element added (or changed) to preserve heap's invariant
+ */
+static void
+rspamd_min_heap_swim(struct rspamd_min_heap *heap,
+ struct rspamd_min_heap_elt *elt)
+{
+ struct rspamd_min_heap_elt *parent;
+
+ while (elt->idx > 1) {
+ parent = g_ptr_array_index(heap->ar, elt->idx / 2 - 1);
+
+ if (parent->pri > elt->pri) {
+ heap_swap(heap, elt, parent);
+ }
+ else {
+ break;
+ }
+ }
+}
+
+/*
+ * Sinks the element popped (or changed) to preserve heap's invariant
+ */
+static void
+rspamd_min_heap_sink(struct rspamd_min_heap *heap,
+ struct rspamd_min_heap_elt *elt)
+{
+ struct rspamd_min_heap_elt *c1, *c2, *m;
+
+ while (elt->idx * 2 < heap->ar->len) {
+ c1 = g_ptr_array_index(heap->ar, elt->idx * 2 - 1);
+ c2 = g_ptr_array_index(heap->ar, elt->idx * 2);
+ m = min_elt(c1, c2);
+
+ if (elt->pri > m->pri) {
+ heap_swap(heap, elt, m);
+ }
+ else {
+ break;
+ }
+ }
+
+ if (elt->idx * 2 - 1 < heap->ar->len) {
+ m = g_ptr_array_index(heap->ar, elt->idx * 2 - 1);
+ if (elt->pri > m->pri) {
+ heap_swap(heap, elt, m);
+ }
+ }
+}
+
+struct rspamd_min_heap *
+rspamd_min_heap_create(gsize reserved_size)
+{
+ struct rspamd_min_heap *heap;
+
+ heap = g_malloc(sizeof(*heap));
+ heap->ar = g_ptr_array_sized_new(reserved_size);
+
+ return heap;
+}
+
+void rspamd_min_heap_push(struct rspamd_min_heap *heap,
+ struct rspamd_min_heap_elt *elt)
+{
+ g_assert(heap != NULL);
+ g_assert(elt != NULL);
+
+ /* Add to the end */
+ elt->idx = heap->ar->len + 1;
+ g_ptr_array_add(heap->ar, elt);
+ /* Now swim it up */
+ rspamd_min_heap_swim(heap, elt);
+}
+
+struct rspamd_min_heap_elt *
+rspamd_min_heap_pop(struct rspamd_min_heap *heap)
+{
+ struct rspamd_min_heap_elt *elt, *last;
+
+ g_assert(heap != NULL);
+
+ if (heap->ar->len == 0) {
+ return NULL;
+ }
+
+ elt = g_ptr_array_index(heap->ar, 0);
+ last = g_ptr_array_index(heap->ar, heap->ar->len - 1);
+
+ if (elt != last) {
+ /* Now replace elt with the last element and sink it if needed */
+ heap_swap(heap, elt, last);
+ g_ptr_array_remove_index_fast(heap->ar, heap->ar->len - 1);
+ rspamd_min_heap_sink(heap, last);
+ }
+ else {
+ g_ptr_array_remove_index_fast(heap->ar, heap->ar->len - 1);
+ }
+
+
+ return elt;
+}
+
+void rspamd_min_heap_update_elt(struct rspamd_min_heap *heap,
+ struct rspamd_min_heap_elt *elt, guint npri)
+{
+ guint oldpri;
+
+ g_assert(heap != NULL);
+ g_assert(elt->idx > 0 && elt->idx <= heap->ar->len);
+
+ oldpri = elt->pri;
+ elt->pri = npri;
+
+ if (npri > oldpri) {
+ /* We might need to sink */
+ rspamd_min_heap_sink(heap, elt);
+ }
+ else if (npri < oldpri) {
+ /* We might need to swim */
+ rspamd_min_heap_swim(heap, elt);
+ }
+}
+
+void rspamd_min_heap_remove_elt(struct rspamd_min_heap *heap,
+ struct rspamd_min_heap_elt *elt)
+{
+ struct rspamd_min_heap_elt *first;
+
+ g_assert(heap != NULL);
+ g_assert(elt->idx > 0 && elt->idx <= heap->ar->len);
+
+ first = g_ptr_array_index(heap->ar, 0);
+
+ if (elt != first) {
+ elt->pri = first->pri - 1;
+ rspamd_min_heap_swim(heap, elt);
+ }
+
+ /* Now the desired element is on the top of queue */
+ (void) rspamd_min_heap_pop(heap);
+}
+
+void rspamd_min_heap_destroy(struct rspamd_min_heap *heap)
+{
+ if (heap) {
+ g_ptr_array_free(heap->ar, TRUE);
+ g_free(heap);
+ }
+}
+
+struct rspamd_min_heap_elt *
+rspamd_min_heap_index(struct rspamd_min_heap *heap, guint idx)
+{
+ g_assert(heap != NULL);
+ g_assert(idx < heap->ar->len);
+
+ return g_ptr_array_index(heap->ar, idx);
+}
diff --git a/src/libutil/heap.h b/src/libutil/heap.h
new file mode 100644
index 0000000..805f817
--- /dev/null
+++ b/src/libutil/heap.h
@@ -0,0 +1,97 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBUTIL_HEAP_H_
+#define SRC_LIBUTIL_HEAP_H_
+
+#include "config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Binary minimal heap interface based on glib
+ */
+
+struct rspamd_min_heap_elt {
+ gpointer data;
+ guint pri;
+ guint idx;
+};
+
+struct rspamd_min_heap;
+
+/**
+ * Creates min heap with the specified reserved size and compare function
+ * @param reserved_size reserved size in elements
+ * @return opaque minimal heap
+ */
+struct rspamd_min_heap *rspamd_min_heap_create(gsize reserved_size);
+
+/**
+ * Pushes an element to the heap. `pri` should be initialized to use this function,
+ * `idx` is used internally by heap interface
+ * @param heap heap structure
+ * @param elt element to push
+ */
+void rspamd_min_heap_push(struct rspamd_min_heap *heap,
+ struct rspamd_min_heap_elt *elt);
+
+/**
+ * Pops the minimum element from the heap and reorder the queue
+ * @param heap heap structure
+ * @return minimum element
+ */
+struct rspamd_min_heap_elt *rspamd_min_heap_pop(struct rspamd_min_heap *heap);
+
+/**
+ * Updates priority for the element. It must be in queue (so `idx` should be sane)
+ * @param heap heap structure
+ * @param elt element to update
+ * @param npri new priority
+ */
+void rspamd_min_heap_update_elt(struct rspamd_min_heap *heap,
+ struct rspamd_min_heap_elt *elt, guint npri);
+
+
+/**
+ * Removes element from the heap
+ * @param heap
+ * @param elt
+ */
+void rspamd_min_heap_remove_elt(struct rspamd_min_heap *heap,
+ struct rspamd_min_heap_elt *elt);
+
+/**
+ * Destroys heap (elements are not destroyed themselves)
+ * @param heap
+ */
+void rspamd_min_heap_destroy(struct rspamd_min_heap *heap);
+
+/**
+ * Returns element from the heap with the specified index
+ * @param heap
+ * @param idx
+ * @return
+ */
+struct rspamd_min_heap_elt *rspamd_min_heap_index(struct rspamd_min_heap *heap,
+ guint idx);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBUTIL_HEAP_H_ */
diff --git a/src/libutil/libev_helper.c b/src/libutil/libev_helper.c
new file mode 100644
index 0000000..770964b
--- /dev/null
+++ b/src/libutil/libev_helper.c
@@ -0,0 +1,111 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "libev_helper.h"
+
+static void
+rspamd_ev_watcher_io_cb(EV_P_ struct ev_io *w, int revents)
+{
+ struct rspamd_io_ev *ev = (struct rspamd_io_ev *) w->data;
+
+ ev->cb(ev->io.fd, revents, ev->ud);
+}
+
+static void
+rspamd_ev_watcher_timer_cb(EV_P_ struct ev_timer *w, int revents)
+{
+ struct rspamd_io_ev *ev = (struct rspamd_io_ev *) w->data;
+
+ /*
+ * We now call timeout callback in all the cases, as we assume that all
+ * timeouts are final
+ */
+ ev->cb(ev->io.fd, EV_TIMER, ev->ud);
+}
+
+
+void rspamd_ev_watcher_init(struct rspamd_io_ev *ev,
+ int fd,
+ short what,
+ rspamd_ev_cb cb,
+ void *ud)
+{
+ ev_io_init(&ev->io, rspamd_ev_watcher_io_cb, fd, what);
+ ev->io.data = ev;
+ ev_init(&ev->tm, rspamd_ev_watcher_timer_cb);
+ ev->tm.data = ev;
+ ev->ud = ud;
+ ev->cb = cb;
+}
+
+void rspamd_ev_watcher_start(struct ev_loop *loop,
+ struct rspamd_io_ev *ev,
+ ev_tstamp timeout)
+{
+ g_assert(ev->cb != NULL);
+
+ ev_io_start(EV_A_ & ev->io);
+
+ if (timeout > 0) {
+ /* Update timestamp to avoid timers running early */
+ ev_now_update_if_cheap(loop);
+
+ ev->timeout = timeout;
+ ev_timer_set(&ev->tm, timeout, 0.0);
+ ev_timer_start(EV_A_ & ev->tm);
+ }
+}
+
+void rspamd_ev_watcher_stop(struct ev_loop *loop,
+ struct rspamd_io_ev *ev)
+{
+ if (ev_can_stop(&ev->io)) {
+ ev_io_stop(EV_A_ & ev->io);
+ }
+
+ if (ev->timeout > 0) {
+ ev_timer_stop(EV_A_ & ev->tm);
+ }
+}
+
+void rspamd_ev_watcher_reschedule(struct ev_loop *loop,
+ struct rspamd_io_ev *ev,
+ short what)
+{
+ g_assert(ev->cb != NULL);
+
+ if (ev_can_stop(&ev->io)) {
+ ev_io_stop(EV_A_ & ev->io);
+ ev_io_set(&ev->io, ev->io.fd, what);
+ ev_io_start(EV_A_ & ev->io);
+ }
+ else {
+ ev->io.data = ev;
+ ev_io_init(&ev->io, rspamd_ev_watcher_io_cb, ev->io.fd, what);
+ ev_io_start(EV_A_ & ev->io);
+ }
+
+ if (ev->timeout > 0) {
+ if (!(ev_can_stop(&ev->tm))) {
+ /* Update timestamp to avoid timers running early */
+ ev_now_update_if_cheap(loop);
+
+ ev->tm.data = ev;
+ ev_timer_init(&ev->tm, rspamd_ev_watcher_timer_cb, ev->timeout, 0.0);
+ ev_timer_start(EV_A_ & ev->tm);
+ }
+ }
+} \ No newline at end of file
diff --git a/src/libutil/libev_helper.h b/src/libutil/libev_helper.h
new file mode 100644
index 0000000..44d1604
--- /dev/null
+++ b/src/libutil/libev_helper.h
@@ -0,0 +1,86 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_LIBEV_HELPER_H
+#define RSPAMD_LIBEV_HELPER_H
+
+#include "config.h"
+#include "contrib/libev/ev.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * This module is a little helper to simplify libevent->libev transition
+ * It allows to create timed IO watchers utilising both
+ */
+
+typedef void (*rspamd_ev_cb)(int fd, short what, void *ud);
+
+struct rspamd_io_ev {
+ ev_io io;
+ ev_timer tm;
+ rspamd_ev_cb cb;
+ void *ud;
+ ev_tstamp timeout;
+};
+
+/**
+ * Initialize watcher similar to event_init
+ * @param ev
+ * @param fd
+ * @param what
+ * @param cb
+ * @param ud
+ */
+void rspamd_ev_watcher_init(struct rspamd_io_ev *ev,
+ int fd, short what, rspamd_ev_cb cb, void *ud);
+
+/**
+ * Start watcher with the specific timeout
+ * @param loop
+ * @param ev
+ * @param timeout
+ */
+void rspamd_ev_watcher_start(struct ev_loop *loop,
+ struct rspamd_io_ev *ev,
+ ev_tstamp timeout);
+
+/**
+ * Stops watcher and clean it up
+ * @param loop
+ * @param ev
+ */
+void rspamd_ev_watcher_stop(struct ev_loop *loop,
+ struct rspamd_io_ev *ev);
+
+/**
+ * Convenience function to reschedule watcher with different events
+ * @param loop
+ * @param ev
+ * @param what
+ */
+void rspamd_ev_watcher_reschedule(struct ev_loop *loop,
+ struct rspamd_io_ev *ev,
+ short what);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libutil/mem_pool.c b/src/libutil/mem_pool.c
new file mode 100644
index 0000000..119ade3
--- /dev/null
+++ b/src/libutil/mem_pool.c
@@ -0,0 +1,1327 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "mem_pool.h"
+#include "fstring.h"
+#include "logger.h"
+#include "ottery.h"
+#include "unix-std.h"
+#include "khash.h"
+#include "cryptobox.h"
+#include "contrib/uthash/utlist.h"
+#include "mem_pool_internal.h"
+
+#ifdef WITH_JEMALLOC
+#include <jemalloc/jemalloc.h>
+#if (JEMALLOC_VERSION_MAJOR == 3 && JEMALLOC_VERSION_MINOR >= 6) || (JEMALLOC_VERSION_MAJOR > 3)
+#define HAVE_MALLOC_SIZE 1
+#define sys_alloc_size(sz) nallocx(sz, 0)
+#endif
+#elif defined(__APPLE__)
+#include <malloc/malloc.h>
+#define HAVE_MALLOC_SIZE 1
+#define sys_alloc_size(sz) malloc_good_size(sz)
+#endif
+
+#ifdef HAVE_SCHED_YIELD
+#include <sched.h>
+#endif
+
+/* Sleep time for spin lock in nanoseconds */
+#define MUTEX_SLEEP_TIME 10000000L
+#define MUTEX_SPIN_COUNT 100
+
+#define POOL_MTX_LOCK() \
+ do { \
+ } while (0)
+#define POOL_MTX_UNLOCK() \
+ do { \
+ } while (0)
+
+/*
+ * This define specify whether we should check all pools for free space for new object
+ * or just begin scan from current (recently attached) pool
+ * If MEMORY_GREEDY is defined, then we scan all pools to find free space (more CPU usage, slower
+ * but requires less memory). If it is not defined check only current pool and if object is too large
+ * to place in it allocate new one (this may cause huge CPU usage in some cases too, but generally faster than
+ * greedy method)
+ */
+#undef MEMORY_GREEDY
+
+
+static inline uint32_t
+rspamd_entry_hash(const char *str)
+{
+ return (guint) rspamd_cryptobox_fast_hash(str, strlen(str), rspamd_hash_seed());
+}
+
+static inline int
+rspamd_entry_equal(const char *k1, const char *k2)
+{
+ return strcmp(k1, k2) == 0;
+}
+
+
+KHASH_INIT(mempool_entry, const gchar *, struct rspamd_mempool_entry_point *,
+ 1, rspamd_entry_hash, rspamd_entry_equal)
+
+static khash_t(mempool_entry) *mempool_entries = NULL;
+
+
+/* Internal statistic */
+static rspamd_mempool_stat_t *mem_pool_stat = NULL;
+/* Environment variable */
+static gboolean env_checked = FALSE;
+static gboolean always_malloc = FALSE;
+
+/**
+ * Function that return free space in pool page
+ * @param x pool page struct
+ */
+static gsize
+pool_chain_free(struct _pool_chain *chain)
+{
+ gint64 occupied = chain->pos - chain->begin + MIN_MEM_ALIGNMENT;
+
+ return (occupied < (gint64) chain->slice_size ? chain->slice_size - occupied : 0);
+}
+
+/* By default allocate 4Kb chunks of memory */
+#define FIXED_POOL_SIZE 4096
+
+static inline struct rspamd_mempool_entry_point *
+rspamd_mempool_entry_new(const gchar *loc)
+{
+ struct rspamd_mempool_entry_point **pentry, *entry;
+ gint r;
+ khiter_t k;
+
+ k = kh_put(mempool_entry, mempool_entries, loc, &r);
+
+ if (r >= 0) {
+ pentry = &kh_value(mempool_entries, k);
+ entry = g_malloc0(sizeof(*entry));
+ *pentry = entry;
+ memset(entry, 0, sizeof(*entry));
+ rspamd_strlcpy(entry->src, loc, sizeof(entry->src));
+#ifdef HAVE_GETPAGESIZE
+ entry->cur_suggestion = MAX(getpagesize(), FIXED_POOL_SIZE);
+#else
+ entry->cur_suggestion = MAX(sysconf(_SC_PAGESIZE), FIXED_POOL_SIZE);
+#endif
+ }
+ else {
+ g_assert_not_reached();
+ }
+
+ return entry;
+}
+
+RSPAMD_CONSTRUCTOR(rspamd_mempool_entries_ctor)
+{
+ if (mempool_entries == NULL) {
+ mempool_entries = kh_init(mempool_entry);
+ }
+}
+
+RSPAMD_DESTRUCTOR(rspamd_mempool_entries_dtor)
+{
+ struct rspamd_mempool_entry_point *elt;
+
+ kh_foreach_value(mempool_entries, elt, {
+ g_free(elt);
+ });
+
+ kh_destroy(mempool_entry, mempool_entries);
+ mempool_entries = NULL;
+}
+
+static inline struct rspamd_mempool_entry_point *
+rspamd_mempool_get_entry(const gchar *loc)
+{
+ khiter_t k;
+ struct rspamd_mempool_entry_point *elt;
+
+ if (G_UNLIKELY(!mempool_entries)) {
+ rspamd_mempool_entries_ctor();
+ }
+
+ k = kh_get(mempool_entry, mempool_entries, loc);
+
+ if (k != kh_end(mempool_entries)) {
+ elt = kh_value(mempool_entries, k);
+
+ return elt;
+ }
+
+ return rspamd_mempool_entry_new(loc);
+}
+
+static struct _pool_chain *
+rspamd_mempool_chain_new(gsize size, gsize alignment, enum rspamd_mempool_chain_type pool_type)
+{
+ struct _pool_chain *chain;
+ gsize total_size = size + sizeof(struct _pool_chain) + alignment,
+ optimal_size = 0;
+ gpointer map;
+
+ g_assert(size > 0);
+
+ if (pool_type == RSPAMD_MEMPOOL_SHARED) {
+#if defined(HAVE_MMAP_ANON)
+ map = mmap(NULL,
+ total_size,
+ PROT_READ | PROT_WRITE,
+ MAP_ANON | MAP_SHARED,
+ -1,
+ 0);
+ if (map == MAP_FAILED) {
+ g_error("%s: failed to allocate %" G_GSIZE_FORMAT " bytes",
+ G_STRLOC, total_size);
+ abort();
+ }
+ chain = map;
+ chain->begin = ((guint8 *) chain) + sizeof(struct _pool_chain);
+#elif defined(HAVE_MMAP_ZERO)
+ gint fd;
+
+ fd = open("/dev/zero", O_RDWR);
+ if (fd == -1) {
+ return NULL;
+ }
+ map = mmap(NULL,
+ size + sizeof(struct _pool_chain),
+ PROT_READ | PROT_WRITE,
+ MAP_SHARED,
+ fd,
+ 0);
+ if (map == MAP_FAILED) {
+ msg_err("cannot allocate %z bytes, aborting", size +
+ sizeof(struct _pool_chain));
+ abort();
+ }
+ chain = map;
+ chain->begin = ((guint8 *) chain) + sizeof(struct _pool_chain);
+#else
+#error No mmap methods are defined
+#endif
+ g_atomic_int_inc(&mem_pool_stat->shared_chunks_allocated);
+ g_atomic_int_add(&mem_pool_stat->bytes_allocated, total_size);
+ }
+ else {
+#ifdef HAVE_MALLOC_SIZE
+ optimal_size = sys_alloc_size(total_size);
+#endif
+ total_size = MAX(total_size, optimal_size);
+ gint ret = posix_memalign(&map, alignment, total_size);
+
+ if (ret != 0 || map == NULL) {
+ g_error("%s: failed to allocate %" G_GSIZE_FORMAT " bytes: %d - %s",
+ G_STRLOC, total_size, ret, strerror(errno));
+ abort();
+ }
+
+ chain = map;
+ chain->begin = ((guint8 *) chain) + sizeof(struct _pool_chain);
+ g_atomic_int_add(&mem_pool_stat->bytes_allocated, total_size);
+ g_atomic_int_inc(&mem_pool_stat->chunks_allocated);
+ }
+
+ chain->pos = align_ptr(chain->begin, alignment);
+ chain->slice_size = total_size - sizeof(struct _pool_chain);
+
+ return chain;
+}
+
+
+/**
+ * Get the current pool of the specified type, creating the corresponding
+ * array if it's absent
+ * @param pool
+ * @param pool_type
+ * @return
+ */
+static struct _pool_chain *
+rspamd_mempool_get_chain(rspamd_mempool_t *pool,
+ enum rspamd_mempool_chain_type pool_type)
+{
+ g_assert(pool_type >= 0 && pool_type < RSPAMD_MEMPOOL_MAX);
+
+ return pool->priv->pools[pool_type];
+}
+
+static void
+rspamd_mempool_append_chain(rspamd_mempool_t *pool,
+ struct _pool_chain *chain,
+ enum rspamd_mempool_chain_type pool_type)
+{
+ g_assert(pool_type >= 0 && pool_type < RSPAMD_MEMPOOL_MAX);
+ g_assert(chain != NULL);
+
+ LL_PREPEND(pool->priv->pools[pool_type], chain);
+}
+
+/**
+ * Allocate new memory poll
+ * @param size size of pool's page
+ * @return new memory pool object
+ */
+rspamd_mempool_t *
+rspamd_mempool_new_(gsize size, const gchar *tag, gint flags, const gchar *loc)
+{
+ rspamd_mempool_t *new_pool;
+ gpointer map;
+
+ /* Allocate statistic structure if it is not allocated before */
+ if (mem_pool_stat == NULL) {
+#if defined(HAVE_MMAP_ANON)
+ map = mmap(NULL,
+ sizeof(rspamd_mempool_stat_t),
+ PROT_READ | PROT_WRITE,
+ MAP_ANON | MAP_SHARED,
+ -1,
+ 0);
+ if (map == MAP_FAILED) {
+ msg_err("cannot allocate %z bytes, aborting",
+ sizeof(rspamd_mempool_stat_t));
+ abort();
+ }
+ mem_pool_stat = (rspamd_mempool_stat_t *) map;
+#elif defined(HAVE_MMAP_ZERO)
+ gint fd;
+
+ fd = open("/dev/zero", O_RDWR);
+ g_assert(fd != -1);
+ map = mmap(NULL,
+ sizeof(rspamd_mempool_stat_t),
+ PROT_READ | PROT_WRITE,
+ MAP_SHARED,
+ fd,
+ 0);
+ if (map == MAP_FAILED) {
+ msg_err("cannot allocate %z bytes, aborting",
+ sizeof(rspamd_mempool_stat_t));
+ abort();
+ }
+ mem_pool_stat = (rspamd_mempool_stat_t *) map;
+#else
+#error No mmap methods are defined
+#endif
+ memset(map, 0, sizeof(rspamd_mempool_stat_t));
+ }
+
+ if (!env_checked) {
+ /* Check G_SLICE=always-malloc to allow memory pool debug */
+ const char *g_slice;
+
+ g_slice = getenv("VALGRIND");
+ if (g_slice != NULL) {
+ always_malloc = TRUE;
+ }
+ env_checked = TRUE;
+ }
+
+ struct rspamd_mempool_entry_point *entry = rspamd_mempool_get_entry(loc);
+ gsize total_size;
+
+ if (size == 0 && entry) {
+ size = entry->cur_suggestion;
+ }
+
+ total_size = sizeof(rspamd_mempool_t) +
+ sizeof(struct rspamd_mempool_specific) +
+ MIN_MEM_ALIGNMENT +
+ sizeof(struct _pool_chain) +
+ size;
+
+ if (G_UNLIKELY(flags & RSPAMD_MEMPOOL_DEBUG)) {
+ total_size += sizeof(GHashTable *);
+ }
+ /*
+ * Memory layout:
+ * struct rspamd_mempool_t
+ * <optional debug hash table>
+ * struct rspamd_mempool_specific
+ * struct _pool_chain
+ * alignment (if needed)
+ * memory chunk
+ */
+ guchar *mem_chunk;
+ gint ret = posix_memalign((void **) &mem_chunk, MIN_MEM_ALIGNMENT,
+ total_size);
+ gsize priv_offset;
+
+ if (ret != 0 || mem_chunk == NULL) {
+ g_error("%s: failed to allocate %" G_GSIZE_FORMAT " bytes: %d - %s",
+ G_STRLOC, total_size, ret, strerror(errno));
+ abort();
+ }
+
+ /* Set memory layout */
+ new_pool = (rspamd_mempool_t *) mem_chunk;
+ if (G_UNLIKELY(flags & RSPAMD_MEMPOOL_DEBUG)) {
+ /* Allocate debug table */
+ GHashTable *debug_tbl;
+
+ debug_tbl = g_hash_table_new(rspamd_str_hash, rspamd_str_equal);
+ memcpy(mem_chunk + sizeof(rspamd_mempool_t), &debug_tbl,
+ sizeof(GHashTable *));
+ priv_offset = sizeof(rspamd_mempool_t) + sizeof(GHashTable *);
+ }
+ else {
+ priv_offset = sizeof(rspamd_mempool_t);
+ }
+
+ new_pool->priv = (struct rspamd_mempool_specific *) (mem_chunk +
+ priv_offset);
+ /* Zero memory for specific and for the first chain */
+ memset(new_pool->priv, 0, sizeof(struct rspamd_mempool_specific) + sizeof(struct _pool_chain));
+
+ new_pool->priv->entry = entry;
+ new_pool->priv->elt_len = size;
+ new_pool->priv->flags = flags;
+
+ if (tag) {
+ rspamd_strlcpy(new_pool->tag.tagname, tag, sizeof(new_pool->tag.tagname));
+ }
+ else {
+ new_pool->tag.tagname[0] = '\0';
+ }
+
+ /* Generate new uid */
+ uint64_t uid = rspamd_random_uint64_fast();
+ rspamd_encode_hex_buf((unsigned char *) &uid, sizeof(uid),
+ new_pool->tag.uid, sizeof(new_pool->tag.uid) - 1);
+ new_pool->tag.uid[sizeof(new_pool->tag.uid) - 1] = '\0';
+
+ mem_pool_stat->pools_allocated++;
+
+ /* Now we can attach one chunk to speed up simple allocations */
+ struct _pool_chain *nchain;
+
+ nchain = (struct _pool_chain *) (mem_chunk +
+ priv_offset +
+ sizeof(struct rspamd_mempool_specific));
+
+ guchar *unaligned = mem_chunk +
+ priv_offset +
+ sizeof(struct rspamd_mempool_specific) +
+ sizeof(struct _pool_chain);
+
+ nchain->slice_size = size;
+ nchain->begin = unaligned;
+ nchain->slice_size = size;
+ nchain->pos = align_ptr(unaligned, MIN_MEM_ALIGNMENT);
+ new_pool->priv->pools[RSPAMD_MEMPOOL_NORMAL] = nchain;
+ new_pool->priv->used_memory = size;
+
+ /* Adjust stats */
+ g_atomic_int_add(&mem_pool_stat->bytes_allocated,
+ (gint) size);
+ g_atomic_int_add(&mem_pool_stat->chunks_allocated, 1);
+
+ return new_pool;
+}
+
+static void *
+memory_pool_alloc_common(rspamd_mempool_t *pool, gsize size, gsize alignment,
+ enum rspamd_mempool_chain_type pool_type,
+ const gchar *loc)
+ RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL;
+
+
+void rspamd_mempool_notify_alloc_(rspamd_mempool_t *pool, gsize size, const gchar *loc)
+{
+ if (pool && G_UNLIKELY(pool->priv->flags & RSPAMD_MEMPOOL_DEBUG)) {
+ GHashTable *debug_tbl = *(GHashTable **) (((guchar *) pool + sizeof(*pool)));
+ gpointer ptr;
+
+ ptr = g_hash_table_lookup(debug_tbl, loc);
+
+ if (ptr) {
+ ptr = GSIZE_TO_POINTER(GPOINTER_TO_SIZE(ptr) + size);
+ }
+ else {
+ ptr = GSIZE_TO_POINTER(size);
+ }
+
+ g_hash_table_insert(debug_tbl, (gpointer) loc, ptr);
+ }
+}
+
+static void *
+memory_pool_alloc_common(rspamd_mempool_t *pool, gsize size, gsize alignment,
+ enum rspamd_mempool_chain_type pool_type, const gchar *loc)
+{
+ guint8 *tmp;
+ struct _pool_chain *new, *cur;
+ gsize free = 0;
+
+ if (pool) {
+ POOL_MTX_LOCK();
+ pool->priv->used_memory += size;
+
+ if (G_UNLIKELY(pool->priv->flags & RSPAMD_MEMPOOL_DEBUG)) {
+ rspamd_mempool_notify_alloc_(pool, size, loc);
+ }
+
+ if (always_malloc && pool_type != RSPAMD_MEMPOOL_SHARED) {
+ void *ptr;
+
+ if (alignment <= G_MEM_ALIGN) {
+ ptr = g_malloc(size);
+ }
+ else {
+ ptr = g_malloc(size + alignment);
+ ptr = align_ptr(ptr, alignment);
+ }
+ POOL_MTX_UNLOCK();
+
+ if (pool->priv->trash_stack == NULL) {
+ pool->priv->trash_stack = g_ptr_array_sized_new(128);
+ }
+
+ g_ptr_array_add(pool->priv->trash_stack, ptr);
+
+ return ptr;
+ }
+
+ cur = rspamd_mempool_get_chain(pool, pool_type);
+
+ /* Find free space in pool chain */
+ if (cur) {
+ free = pool_chain_free(cur);
+ }
+
+ if (cur == NULL || free < size + alignment) {
+ if (free < size) {
+ pool->priv->wasted_memory += free;
+ }
+
+ /* Allocate new chain element */
+ if (pool->priv->elt_len >= size + alignment) {
+ pool->priv->entry->elts[pool->priv->entry->cur_elts].fragmentation += size;
+ new = rspamd_mempool_chain_new(pool->priv->elt_len, alignment,
+ pool_type);
+ }
+ else {
+ mem_pool_stat->oversized_chunks++;
+ g_atomic_int_add(&mem_pool_stat->fragmented_size,
+ free);
+ pool->priv->entry->elts[pool->priv->entry->cur_elts].fragmentation += free;
+ new = rspamd_mempool_chain_new(size + pool->priv->elt_len, alignment,
+ pool_type);
+ }
+
+ /* Connect to pool subsystem */
+ rspamd_mempool_append_chain(pool, new, pool_type);
+ /* No need to align again, aligned by rspamd_mempool_chain_new */
+ tmp = new->pos;
+ new->pos = tmp + size;
+ POOL_MTX_UNLOCK();
+
+ return tmp;
+ }
+
+ /* No need to allocate page */
+ tmp = align_ptr(cur->pos, alignment);
+ cur->pos = tmp + size;
+ POOL_MTX_UNLOCK();
+
+ return tmp;
+ }
+
+ abort();
+}
+
+
+void *
+rspamd_mempool_alloc_(rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc)
+{
+ return memory_pool_alloc_common(pool, size, alignment, RSPAMD_MEMPOOL_NORMAL, loc);
+}
+
+/*
+ * This is sqrt(SIZE_MAX+1), as s1*s2 <= SIZE_MAX
+ * if both s1 < MUL_NO_OVERFLOW and s2 < MUL_NO_OVERFLOW
+ */
+#define MUL_NO_OVERFLOW (1UL << (sizeof(gsize) * 4))
+
+void *
+rspamd_mempool_alloc_array_(rspamd_mempool_t *pool, gsize nmemb, gsize size, gsize alignment, const gchar *loc)
+{
+ if ((nmemb >= MUL_NO_OVERFLOW || size >= MUL_NO_OVERFLOW) &&
+ nmemb > 0 && G_MAXSIZE / nmemb < size) {
+
+ g_error("alloc_array: overflow %" G_GSIZE_FORMAT " * %" G_GSIZE_FORMAT "",
+ nmemb, size);
+ g_abort();
+ }
+ return memory_pool_alloc_common(pool, size * nmemb, alignment, RSPAMD_MEMPOOL_NORMAL, loc);
+}
+
+void *
+rspamd_mempool_alloc0_(rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc)
+{
+ void *pointer = rspamd_mempool_alloc_(pool, size, alignment, loc);
+ memset(pointer, 0, size);
+
+ return pointer;
+}
+void *
+rspamd_mempool_alloc0_shared_(rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc)
+{
+ void *pointer = rspamd_mempool_alloc_shared_(pool, size, alignment, loc);
+
+ memset(pointer, 0, size);
+ return pointer;
+}
+
+void *
+rspamd_mempool_alloc_shared_(rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc)
+{
+ return memory_pool_alloc_common(pool, size, alignment, RSPAMD_MEMPOOL_SHARED, loc);
+}
+
+
+gchar *
+rspamd_mempool_strdup_(rspamd_mempool_t *pool, const gchar *src, const gchar *loc)
+{
+ if (src == NULL) {
+ return NULL;
+ }
+ return rspamd_mempool_strdup_len_(pool, src, strlen(src), loc);
+}
+
+gchar *
+rspamd_mempool_strdup_len_(rspamd_mempool_t *pool, const gchar *src, gsize len, const gchar *loc)
+{
+ gchar *newstr;
+
+ if (src == NULL) {
+ return NULL;
+ }
+
+ newstr = rspamd_mempool_alloc_(pool, len + 1, MIN_MEM_ALIGNMENT, loc);
+ memcpy(newstr, src, len);
+ newstr[len] = '\0';
+
+ return newstr;
+}
+
+gchar *
+rspamd_mempool_ftokdup_(rspamd_mempool_t *pool, const rspamd_ftok_t *src,
+ const gchar *loc)
+{
+ gchar *newstr;
+
+ if (src == NULL) {
+ return NULL;
+ }
+
+ newstr = rspamd_mempool_alloc_(pool, src->len + 1, MIN_MEM_ALIGNMENT, loc);
+ memcpy(newstr, src->begin, src->len);
+ newstr[src->len] = '\0';
+
+ return newstr;
+}
+
+void rspamd_mempool_add_destructor_full(rspamd_mempool_t *pool,
+ rspamd_mempool_destruct_t func,
+ void *data,
+ const gchar *function,
+ const gchar *line)
+{
+ struct _pool_destructors *cur;
+
+ POOL_MTX_LOCK();
+ cur = rspamd_mempool_alloc_(pool, sizeof(*cur),
+ RSPAMD_ALIGNOF(struct _pool_destructors), line);
+ cur->func = func;
+ cur->data = data;
+ cur->function = function;
+ cur->loc = line;
+ cur->next = NULL;
+
+ if (pool->priv->dtors_tail) {
+ pool->priv->dtors_tail->next = cur;
+ pool->priv->dtors_tail = cur;
+ }
+ else {
+ pool->priv->dtors_head = cur;
+ pool->priv->dtors_tail = cur;
+ }
+
+ POOL_MTX_UNLOCK();
+}
+
+void rspamd_mempool_replace_destructor(rspamd_mempool_t *pool,
+ rspamd_mempool_destruct_t func,
+ void *old_data,
+ void *new_data)
+{
+ struct _pool_destructors *tmp;
+
+ LL_FOREACH(pool->priv->dtors_head, tmp)
+ {
+ if (tmp->func == func && tmp->data == old_data) {
+ tmp->func = func;
+ tmp->data = new_data;
+ break;
+ }
+ }
+}
+
+static gint
+cmp_int(gconstpointer a, gconstpointer b)
+{
+ gint i1 = *(const gint *) a, i2 = *(const gint *) b;
+
+ return i1 - i2;
+}
+
+static void
+rspamd_mempool_adjust_entry(struct rspamd_mempool_entry_point *e)
+{
+ gint sz[G_N_ELEMENTS(e->elts)], sel_pos, sel_neg;
+ guint i, jitter;
+
+ for (i = 0; i < G_N_ELEMENTS(sz); i++) {
+ sz[i] = e->elts[i].fragmentation - (gint) e->elts[i].leftover;
+ }
+
+ qsort(sz, G_N_ELEMENTS(sz), sizeof(gint), cmp_int);
+ jitter = rspamd_random_uint64_fast() % 10;
+ /*
+ * Take stochastic quantiles
+ */
+ sel_pos = sz[50 + jitter];
+ sel_neg = sz[4 + jitter];
+
+ if (-sel_neg > sel_pos) {
+ /* We need to reduce current suggestion */
+ e->cur_suggestion /= (1 + (((double) -sel_neg) / e->cur_suggestion)) * 1.5;
+ }
+ else {
+ /* We still want to grow */
+ e->cur_suggestion *= (1 + (((double) sel_pos) / e->cur_suggestion)) * 1.5;
+ }
+
+ /* Some sane limits counting mempool architecture */
+ if (e->cur_suggestion < 1024) {
+ e->cur_suggestion = 1024;
+ }
+ else if (e->cur_suggestion > 1024 * 1024 * 10) {
+ e->cur_suggestion = 1024 * 1024 * 10;
+ }
+
+ memset(e->elts, 0, sizeof(e->elts));
+}
+
+static void
+rspamd_mempool_variables_cleanup(rspamd_mempool_t *pool)
+{
+ if (pool->priv->variables) {
+ struct rspamd_mempool_variable *var;
+ kh_foreach_value_ptr(pool->priv->variables, var, {
+ if (var->dtor) {
+ var->dtor(var->data);
+ }
+ });
+
+ if (pool->priv->entry && pool->priv->entry->cur_vars <
+ kh_size(pool->priv->variables)) {
+ /*
+ * Increase preallocated size in two cases:
+ * 1) Our previous guess was zero
+ * 2) Our new variables count is not more than twice larger than
+ * previous count
+ * 3) Our variables count is less than some hard limit
+ */
+ static const guint max_preallocated_vars = 512;
+
+ guint cur_size = kh_size(pool->priv->variables);
+ guint old_guess = pool->priv->entry->cur_vars;
+ guint new_guess;
+
+ if (old_guess == 0) {
+ new_guess = MIN(cur_size, max_preallocated_vars);
+ }
+ else {
+ if (old_guess * 2 < cur_size) {
+ new_guess = MIN(cur_size, max_preallocated_vars);
+ }
+ else {
+ /* Too large step */
+ new_guess = MIN(old_guess * 2, max_preallocated_vars);
+ }
+ }
+
+ pool->priv->entry->cur_vars = new_guess;
+ }
+
+ kh_destroy(rspamd_mempool_vars_hash, pool->priv->variables);
+ pool->priv->variables = NULL;
+ }
+}
+
+void rspamd_mempool_destructors_enforce(rspamd_mempool_t *pool)
+{
+ struct _pool_destructors *destructor;
+
+ POOL_MTX_LOCK();
+
+ LL_FOREACH(pool->priv->dtors_head, destructor)
+ {
+ /* Avoid calling destructors for NULL pointers */
+ if (destructor->data != NULL) {
+ destructor->func(destructor->data);
+ }
+ }
+
+ pool->priv->dtors_head = pool->priv->dtors_tail = NULL;
+
+ rspamd_mempool_variables_cleanup(pool);
+
+ POOL_MTX_UNLOCK();
+}
+
+struct mempool_debug_elt {
+ gsize sz;
+ const gchar *loc;
+};
+
+static gint
+rspamd_mempool_debug_elt_cmp(const void *a, const void *b)
+{
+ const struct mempool_debug_elt *e1 = a, *e2 = b;
+
+ /* Inverse order */
+ return (gint) ((gssize) e2->sz) - ((gssize) e1->sz);
+}
+
+void rspamd_mempool_delete(rspamd_mempool_t *pool)
+{
+ struct _pool_chain *cur, *tmp;
+ struct _pool_destructors *destructor;
+ gpointer ptr;
+ guint i;
+ gsize len;
+
+ POOL_MTX_LOCK();
+
+ cur = pool->priv->pools[RSPAMD_MEMPOOL_NORMAL];
+
+ if (G_UNLIKELY(pool->priv->flags & RSPAMD_MEMPOOL_DEBUG)) {
+ GHashTable *debug_tbl = *(GHashTable **) (((guchar *) pool) + sizeof(*pool));
+ /* Show debug info */
+ gsize ndtor = 0;
+ LL_COUNT(pool->priv->dtors_head, destructor, ndtor);
+ msg_info_pool("destructing of the memory pool %p; elt size = %z; "
+ "used memory = %Hz; wasted memory = %Hd; "
+ "vars = %z; destructors = %z",
+ pool,
+ pool->priv->elt_len,
+ pool->priv->used_memory,
+ pool->priv->wasted_memory,
+ pool->priv->variables ? (gsize) kh_size(pool->priv->variables) : (gsize) 0,
+ ndtor);
+
+ GHashTableIter it;
+ gpointer k, v;
+ GArray *sorted_debug_size = g_array_sized_new(FALSE, FALSE,
+ sizeof(struct mempool_debug_elt),
+ g_hash_table_size(debug_tbl));
+
+ g_hash_table_iter_init(&it, debug_tbl);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ struct mempool_debug_elt e;
+ e.loc = (const gchar *) k;
+ e.sz = GPOINTER_TO_SIZE(v);
+ g_array_append_val(sorted_debug_size, e);
+ }
+
+ g_array_sort(sorted_debug_size, rspamd_mempool_debug_elt_cmp);
+
+ for (guint _i = 0; _i < sorted_debug_size->len; _i++) {
+ struct mempool_debug_elt *e;
+
+ e = &g_array_index(sorted_debug_size, struct mempool_debug_elt, _i);
+ msg_info_pool("allocated %Hz from %s", e->sz, e->loc);
+ }
+
+ g_array_free(sorted_debug_size, TRUE);
+ g_hash_table_unref(debug_tbl);
+ }
+
+ if (cur && mempool_entries) {
+ pool->priv->entry->elts[pool->priv->entry->cur_elts].leftover =
+ pool_chain_free(cur);
+
+ pool->priv->entry->cur_elts = (pool->priv->entry->cur_elts + 1) %
+ G_N_ELEMENTS(pool->priv->entry->elts);
+
+ if (pool->priv->entry->cur_elts == 0) {
+ rspamd_mempool_adjust_entry(pool->priv->entry);
+ }
+ }
+
+ /* Call all pool destructors */
+ LL_FOREACH(pool->priv->dtors_head, destructor)
+ {
+ /* Avoid calling destructors for NULL pointers */
+ if (destructor->data != NULL) {
+ destructor->func(destructor->data);
+ }
+ }
+
+ rspamd_mempool_variables_cleanup(pool);
+
+ if (pool->priv->trash_stack) {
+ for (i = 0; i < pool->priv->trash_stack->len; i++) {
+ ptr = g_ptr_array_index(pool->priv->trash_stack, i);
+ g_free(ptr);
+ }
+
+ g_ptr_array_free(pool->priv->trash_stack, TRUE);
+ }
+
+ for (i = 0; i < G_N_ELEMENTS(pool->priv->pools); i++) {
+ if (pool->priv->pools[i]) {
+ LL_FOREACH_SAFE(pool->priv->pools[i], cur, tmp)
+ {
+ g_atomic_int_add(&mem_pool_stat->bytes_allocated,
+ -((gint) cur->slice_size));
+ g_atomic_int_add(&mem_pool_stat->chunks_allocated, -1);
+
+ len = cur->slice_size + sizeof(struct _pool_chain);
+
+ if (i == RSPAMD_MEMPOOL_SHARED) {
+ munmap((void *) cur, len);
+ }
+ else {
+ /* The last pool is special, it is a part of the initial chunk */
+ if (cur->next != NULL) {
+ free(cur); /* Not g_free as we use system allocator */
+ }
+ }
+ }
+ }
+ }
+
+ g_atomic_int_inc(&mem_pool_stat->pools_freed);
+ POOL_MTX_UNLOCK();
+ free(pool); /* allocated by posix_memalign */
+}
+
+void rspamd_mempool_stat(rspamd_mempool_stat_t *st)
+{
+ if (mem_pool_stat != NULL) {
+ st->pools_allocated = mem_pool_stat->pools_allocated;
+ st->pools_freed = mem_pool_stat->pools_freed;
+ st->shared_chunks_allocated = mem_pool_stat->shared_chunks_allocated;
+ st->bytes_allocated = mem_pool_stat->bytes_allocated;
+ st->chunks_allocated = mem_pool_stat->chunks_allocated;
+ st->chunks_freed = mem_pool_stat->chunks_freed;
+ st->oversized_chunks = mem_pool_stat->oversized_chunks;
+ }
+}
+
+void rspamd_mempool_stat_reset(void)
+{
+ if (mem_pool_stat != NULL) {
+ memset(mem_pool_stat, 0, sizeof(rspamd_mempool_stat_t));
+ }
+}
+
+gsize rspamd_mempool_suggest_size_(const char *loc)
+{
+ return 0;
+}
+
+#if !defined(HAVE_PTHREAD_PROCESS_SHARED) || defined(DISABLE_PTHREAD_MUTEX)
+/*
+ * Own emulation
+ */
+static inline gint
+__mutex_spin(rspamd_mempool_mutex_t *mutex)
+{
+ /* check spin count */
+ if (g_atomic_int_dec_and_test(&mutex->spin)) {
+ /* This may be deadlock, so check owner of this lock */
+ if (mutex->owner == getpid()) {
+ /* This mutex was locked by calling process, so it is just double lock and we can easily unlock it */
+ g_atomic_int_set(&mutex->spin, MUTEX_SPIN_COUNT);
+ return 0;
+ }
+ else if (kill(mutex->owner, 0) == -1) {
+ /* Owner process was not found, so release lock */
+ g_atomic_int_set(&mutex->spin, MUTEX_SPIN_COUNT);
+ return 0;
+ }
+ /* Spin again */
+ g_atomic_int_set(&mutex->spin, MUTEX_SPIN_COUNT);
+ }
+
+#ifdef HAVE_SCHED_YIELD
+ (void) sched_yield();
+#elif defined(HAVE_NANOSLEEP)
+ struct timespec ts;
+ ts.tv_sec = 0;
+ ts.tv_nsec = MUTEX_SLEEP_TIME;
+ /* Spin */
+ while (nanosleep(&ts, &ts) == -1 && errno == EINTR)
+ ;
+#else
+#error No methods to spin are defined
+#endif
+ return 1;
+}
+
+static void
+memory_pool_mutex_spin(rspamd_mempool_mutex_t *mutex)
+{
+ while (!g_atomic_int_compare_and_exchange(&mutex->lock, 0, 1)) {
+ if (!__mutex_spin(mutex)) {
+ return;
+ }
+ }
+}
+
+rspamd_mempool_mutex_t *
+rspamd_mempool_get_mutex(rspamd_mempool_t *pool)
+{
+ rspamd_mempool_mutex_t *res;
+ if (pool != NULL) {
+ res =
+ rspamd_mempool_alloc_shared(pool, sizeof(rspamd_mempool_mutex_t));
+ res->lock = 0;
+ res->owner = 0;
+ res->spin = MUTEX_SPIN_COUNT;
+ return res;
+ }
+ return NULL;
+}
+
+void rspamd_mempool_lock_mutex(rspamd_mempool_mutex_t *mutex)
+{
+ memory_pool_mutex_spin(mutex);
+ mutex->owner = getpid();
+}
+
+void rspamd_mempool_unlock_mutex(rspamd_mempool_mutex_t *mutex)
+{
+ mutex->owner = 0;
+ (void) g_atomic_int_compare_and_exchange(&mutex->lock, 1, 0);
+}
+
+rspamd_mempool_rwlock_t *
+rspamd_mempool_get_rwlock(rspamd_mempool_t *pool)
+{
+ rspamd_mempool_rwlock_t *lock;
+
+ lock = rspamd_mempool_alloc_shared(pool, sizeof(rspamd_mempool_rwlock_t));
+ lock->__r_lock = rspamd_mempool_get_mutex(pool);
+ lock->__w_lock = rspamd_mempool_get_mutex(pool);
+
+ return lock;
+}
+
+void rspamd_mempool_rlock_rwlock(rspamd_mempool_rwlock_t *lock)
+{
+ /* Spin on write lock */
+ while (g_atomic_int_get(&lock->__w_lock->lock)) {
+ if (!__mutex_spin(lock->__w_lock)) {
+ break;
+ }
+ }
+
+ g_atomic_int_inc(&lock->__r_lock->lock);
+ lock->__r_lock->owner = getpid();
+}
+
+void rspamd_mempool_wlock_rwlock(rspamd_mempool_rwlock_t *lock)
+{
+ /* Spin on write lock first */
+ rspamd_mempool_lock_mutex(lock->__w_lock);
+ /* Now we have write lock set up */
+ /* Wait all readers */
+ while (g_atomic_int_get(&lock->__r_lock->lock)) {
+ __mutex_spin(lock->__r_lock);
+ }
+}
+
+void rspamd_mempool_runlock_rwlock(rspamd_mempool_rwlock_t *lock)
+{
+ if (g_atomic_int_get(&lock->__r_lock->lock)) {
+ (void) g_atomic_int_dec_and_test(&lock->__r_lock->lock);
+ }
+}
+
+void rspamd_mempool_wunlock_rwlock(rspamd_mempool_rwlock_t *lock)
+{
+ rspamd_mempool_unlock_mutex(lock->__w_lock);
+}
+#else
+
+/*
+ * Pthread bases shared mutexes
+ */
+rspamd_mempool_mutex_t *
+rspamd_mempool_get_mutex(rspamd_mempool_t *pool)
+{
+ rspamd_mempool_mutex_t *res;
+ pthread_mutexattr_t mattr;
+
+ if (pool != NULL) {
+ res =
+ rspamd_mempool_alloc_shared(pool, sizeof(rspamd_mempool_mutex_t));
+
+ pthread_mutexattr_init(&mattr);
+ pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
+ pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST);
+ pthread_mutex_init(res, &mattr);
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) pthread_mutex_destroy, res);
+ pthread_mutexattr_destroy(&mattr);
+
+ return res;
+ }
+ return NULL;
+}
+
+void rspamd_mempool_lock_mutex(rspamd_mempool_mutex_t *mutex)
+{
+ pthread_mutex_lock(mutex);
+}
+
+void rspamd_mempool_unlock_mutex(rspamd_mempool_mutex_t *mutex)
+{
+ pthread_mutex_unlock(mutex);
+}
+
+rspamd_mempool_rwlock_t *
+rspamd_mempool_get_rwlock(rspamd_mempool_t *pool)
+{
+ rspamd_mempool_rwlock_t *res;
+ pthread_rwlockattr_t mattr;
+
+ if (pool != NULL) {
+ res =
+ rspamd_mempool_alloc_shared(pool, sizeof(rspamd_mempool_rwlock_t));
+
+ pthread_rwlockattr_init(&mattr);
+ pthread_rwlockattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
+ pthread_rwlock_init(res, &mattr);
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) pthread_rwlock_destroy, res);
+ pthread_rwlockattr_destroy(&mattr);
+
+ return res;
+ }
+ return NULL;
+}
+
+void rspamd_mempool_rlock_rwlock(rspamd_mempool_rwlock_t *lock)
+{
+ pthread_rwlock_rdlock(lock);
+}
+
+void rspamd_mempool_wlock_rwlock(rspamd_mempool_rwlock_t *lock)
+{
+ pthread_rwlock_wrlock(lock);
+}
+
+void rspamd_mempool_runlock_rwlock(rspamd_mempool_rwlock_t *lock)
+{
+ pthread_rwlock_unlock(lock);
+}
+
+void rspamd_mempool_wunlock_rwlock(rspamd_mempool_rwlock_t *lock)
+{
+ pthread_rwlock_unlock(lock);
+}
+#endif
+
+#define RSPAMD_MEMPOOL_VARS_HASH_SEED 0xb32ad7c55eb2e647ULL
+void rspamd_mempool_set_variable(rspamd_mempool_t *pool,
+ const gchar *name,
+ gpointer value,
+ rspamd_mempool_destruct_t destructor)
+{
+ if (pool->priv->variables == NULL) {
+
+ pool->priv->variables = kh_init(rspamd_mempool_vars_hash);
+
+ if (pool->priv->entry->cur_vars > 0) {
+ /* Preallocate */
+ kh_resize(rspamd_mempool_vars_hash,
+ pool->priv->variables,
+ pool->priv->entry->cur_vars);
+ }
+ }
+
+ gint hv = rspamd_cryptobox_fast_hash(name, strlen(name),
+ RSPAMD_MEMPOOL_VARS_HASH_SEED);
+ khiter_t it;
+ gint r;
+
+ it = kh_put(rspamd_mempool_vars_hash, pool->priv->variables, hv, &r);
+
+ if (it == kh_end(pool->priv->variables)) {
+ g_assert_not_reached();
+ }
+ else {
+ struct rspamd_mempool_variable *pvar;
+
+ if (r == 0) {
+ /* Existing entry, maybe need cleanup */
+ pvar = &kh_val(pool->priv->variables, it);
+
+ if (pvar->dtor) {
+ pvar->dtor(pvar->data);
+ }
+ }
+
+ pvar = &kh_val(pool->priv->variables, it);
+ pvar->data = value;
+ pvar->dtor = destructor;
+ }
+}
+
+gpointer
+rspamd_mempool_get_variable(rspamd_mempool_t *pool, const gchar *name)
+{
+ if (pool->priv->variables == NULL) {
+ return NULL;
+ }
+
+ khiter_t it;
+ gint hv = rspamd_cryptobox_fast_hash(name, strlen(name),
+ RSPAMD_MEMPOOL_VARS_HASH_SEED);
+
+ it = kh_get(rspamd_mempool_vars_hash, pool->priv->variables, hv);
+
+ if (it != kh_end(pool->priv->variables)) {
+ struct rspamd_mempool_variable *pvar;
+
+ pvar = &kh_val(pool->priv->variables, it);
+ return pvar->data;
+ }
+
+ return NULL;
+}
+
+gpointer
+rspamd_mempool_steal_variable(rspamd_mempool_t *pool, const gchar *name)
+{
+ if (pool->priv->variables == NULL) {
+ return NULL;
+ }
+
+ khiter_t it;
+ gint hv = rspamd_cryptobox_fast_hash(name, strlen(name),
+ RSPAMD_MEMPOOL_VARS_HASH_SEED);
+
+ it = kh_get(rspamd_mempool_vars_hash, pool->priv->variables, hv);
+
+ if (it != kh_end(pool->priv->variables)) {
+ struct rspamd_mempool_variable *pvar;
+
+ pvar = &kh_val(pool->priv->variables, it);
+ kh_del(rspamd_mempool_vars_hash, pool->priv->variables, it);
+
+ return pvar->data;
+ }
+
+ return NULL;
+}
+
+void rspamd_mempool_remove_variable(rspamd_mempool_t *pool, const gchar *name)
+{
+ if (pool->priv->variables != NULL) {
+ khiter_t it;
+ gint hv = rspamd_cryptobox_fast_hash(name, strlen(name),
+ RSPAMD_MEMPOOL_VARS_HASH_SEED);
+
+ it = kh_get(rspamd_mempool_vars_hash, pool->priv->variables, hv);
+
+ if (it != kh_end(pool->priv->variables)) {
+ struct rspamd_mempool_variable *pvar;
+
+ pvar = &kh_val(pool->priv->variables, it);
+
+ if (pvar->dtor) {
+ pvar->dtor(pvar->data);
+ }
+
+ kh_del(rspamd_mempool_vars_hash, pool->priv->variables, it);
+ }
+ }
+}
+
+GList *
+rspamd_mempool_glist_prepend(rspamd_mempool_t *pool, GList *l, gpointer p)
+{
+ GList *cell;
+
+ cell = rspamd_mempool_alloc(pool, sizeof(*cell));
+ cell->prev = NULL;
+ cell->data = p;
+
+ if (l == NULL) {
+ cell->next = NULL;
+ }
+ else {
+ cell->next = l;
+ l->prev = cell;
+ }
+
+ return cell;
+}
+
+GList *
+rspamd_mempool_glist_append(rspamd_mempool_t *pool, GList *l, gpointer p)
+{
+ GList *cell, *cur;
+
+ cell = rspamd_mempool_alloc(pool, sizeof(*cell));
+ cell->next = NULL;
+ cell->data = p;
+
+ if (l) {
+ for (cur = l; cur->next != NULL; cur = cur->next) {}
+ cur->next = cell;
+ cell->prev = cur;
+ }
+ else {
+ l = cell;
+ l->prev = NULL;
+ }
+
+ return l;
+}
+
+gsize rspamd_mempool_get_used_size(rspamd_mempool_t *pool)
+{
+ return pool->priv->used_memory;
+}
+
+gsize rspamd_mempool_get_wasted_size(rspamd_mempool_t *pool)
+{
+ return pool->priv->wasted_memory;
+}
diff --git a/src/libutil/mem_pool.h b/src/libutil/mem_pool.h
new file mode 100644
index 0000000..de0fea1
--- /dev/null
+++ b/src/libutil/mem_pool.h
@@ -0,0 +1,470 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @file mem_pool.h
+ * \brief Memory pools library.
+ *
+ * Memory pools library. Library is designed to implement efficient way to
+ * store data in memory avoiding calling of many malloc/free. It has overhead
+ * because of fact that objects live in pool for rather long time and are not freed
+ * immediately after use, but if we know certainly when these objects can be used, we
+ * can use pool for them
+ */
+
+#ifndef RSPAMD_MEM_POOL_H
+#define RSPAMD_MEM_POOL_H
+
+#include "config.h"
+
+
+#if defined(HAVE_PTHREAD_PROCESS_SHARED) && !defined(DISABLE_PTHREAD_MUTEX)
+#include <pthread.h>
+#endif
+
+#ifdef __cplusplus
+#define MEMPOOL_STR_FUNC __FUNCTION__
+#else
+#define MEMPOOL_STR_FUNC G_STRFUNC
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct f_str_s;
+
+#ifdef __has_attribute
+#if __has_attribute(alloc_size)
+#define RSPAMD_ATTR_ALLOC_SIZE(pos) __attribute__((alloc_size(pos)))
+#else
+#define RSPAMD_ATTR_ALLOC_SIZE(pos)
+#endif
+
+#if __has_attribute(assume_aligned)
+#define RSPAMD_ATTR_ALLOC_ALIGN(al) __attribute__((assume_aligned(al)))
+#else
+#define RSPAMD_ATTR_ALLOC_ALIGN(al)
+#endif
+#if __has_attribute(returns_nonnull)
+#define RSPAMD_ATTR_RETURNS_NONNUL __attribute__((returns_nonnull))
+#else
+#define RSPAMD_ATTR_RETURNS_NONNUL
+#endif
+#else
+#define RSPAMD_ATTR_ALLOC_SIZE(pos)
+#define RSPAMD_ATTR_ALLOC_ALIGN(al)
+#define RSPAMD_ATTR_RETURNS_NONNUL
+#endif
+
+#define MEMPOOL_TAG_LEN 16
+#define MEMPOOL_UID_LEN 16
+/* All pointers are aligned as this variable */
+#define MIN_MEM_ALIGNMENT G_MEM_ALIGN
+
+/**
+ * Destructor type definition
+ */
+typedef void (*rspamd_mempool_destruct_t)(void *ptr);
+
+/**
+ * Pool mutex structure
+ */
+#if !defined(HAVE_PTHREAD_PROCESS_SHARED) || defined(DISABLE_PTHREAD_MUTEX)
+typedef struct memory_pool_mutex_s {
+ gint lock;
+ pid_t owner;
+ guint spin;
+} rspamd_mempool_mutex_t;
+/**
+ * Rwlock for locking shared memory regions
+ */
+typedef struct memory_pool_rwlock_s {
+ rspamd_mempool_mutex_t *__r_lock; /**< read mutex (private) */
+ rspamd_mempool_mutex_t *__w_lock; /**< write mutex (private) */
+} rspamd_mempool_rwlock_t;
+#else
+typedef pthread_mutex_t rspamd_mempool_mutex_t;
+typedef pthread_rwlock_t rspamd_mempool_rwlock_t;
+#endif
+
+/**
+ * Tag to use for logging purposes
+ */
+struct rspamd_mempool_tag {
+ gchar tagname[MEMPOOL_TAG_LEN]; /**< readable name */
+ gchar uid[MEMPOOL_UID_LEN]; /**< unique id */
+};
+
+enum rspamd_mempool_flags {
+ RSPAMD_MEMPOOL_DEBUG = (1u << 0u),
+};
+
+/**
+ * Memory pool type
+ */
+struct rspamd_mempool_entry_point;
+struct rspamd_mutex_s;
+struct rspamd_mempool_specific;
+typedef struct memory_pool_s {
+ struct rspamd_mempool_specific *priv;
+ struct rspamd_mempool_tag tag; /**< memory pool tag */
+} rspamd_mempool_t;
+
+/**
+ * Statistics structure
+ */
+typedef struct memory_pool_stat_s {
+ guint pools_allocated; /**< total number of allocated pools */
+ guint pools_freed; /**< number of freed pools */
+ guint bytes_allocated; /**< bytes that are allocated with pool allocator */
+ guint chunks_allocated; /**< number of chunks that are allocated */
+ guint shared_chunks_allocated; /**< shared chunks allocated */
+ guint chunks_freed; /**< chunks freed */
+ guint oversized_chunks; /**< oversized chunks */
+ guint fragmented_size; /**< fragmentation size */
+} rspamd_mempool_stat_t;
+
+
+/**
+ * Allocate new memory poll
+ * @param size size of pool's page
+ * @return new memory pool object
+ */
+rspamd_mempool_t *rspamd_mempool_new_(gsize size, const gchar *tag, gint flags,
+ const gchar *loc);
+
+#define rspamd_mempool_new(size, tag, flags) \
+ rspamd_mempool_new_((size), (tag), (flags), G_STRLOC)
+#define rspamd_mempool_new_default(tag, flags) \
+ rspamd_mempool_new_(rspamd_mempool_suggest_size_(G_STRLOC), (tag), (flags), G_STRLOC)
+
+/**
+ * Get memory from pool
+ * @param pool memory pool object
+ * @param size bytes to allocate
+ * @return pointer to allocated object
+ */
+void *rspamd_mempool_alloc_(rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc)
+ RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL;
+/**
+ * Allocates array handling potential integer overflow
+ * @param pool
+ * @param nmemb
+ * @param size
+ * @param alignment
+ * @param loc
+ * @return
+ */
+void *rspamd_mempool_alloc_array_(rspamd_mempool_t *pool, gsize nmemb, gsize size, gsize alignment, const gchar *loc)
+ RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL;
+#define rspamd_mempool_alloc(pool, size) \
+ rspamd_mempool_alloc_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC))
+#define rspamd_mempool_alloc_array(pool, nmemb, size) \
+ rspamd_mempool_alloc_array_((pool), (nmemb), (size), MIN_MEM_ALIGNMENT, (G_STRLOC))
+#define rspamd_mempool_alloc_array_type(pool, nmemb, type) \
+ (type *) rspamd_mempool_alloc_array_((pool), (nmemb), sizeof(type), MIN_MEM_ALIGNMENT, (G_STRLOC))
+#define rspamd_mempool_alloc_type(pool, type) \
+ (type *) (rspamd_mempool_alloc_((pool), sizeof(type), \
+ MAX(MIN_MEM_ALIGNMENT, RSPAMD_ALIGNOF(type)), (G_STRLOC)))
+#define rspamd_mempool_alloc_buffer(pool, buflen) \
+ (char *) (rspamd_mempool_alloc_((pool), sizeof(char) * (buflen), MIN_MEM_ALIGNMENT, (G_STRLOC)))
+/**
+ * Notify external memory usage for memory pool
+ * @param pool
+ * @param size
+ * @param loc
+ */
+void rspamd_mempool_notify_alloc_(rspamd_mempool_t *pool, gsize size, const gchar *loc);
+#define rspamd_mempool_notify_alloc(pool, size) \
+ rspamd_mempool_notify_alloc_((pool), (size), (G_STRLOC))
+
+/**
+ * Get memory and set it to zero
+ * @param pool memory pool object
+ * @param size bytes to allocate
+ * @return pointer to allocated object
+ */
+void *rspamd_mempool_alloc0_(rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc)
+ RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL;
+#define rspamd_mempool_alloc0(pool, size) \
+ rspamd_mempool_alloc0_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC))
+#define rspamd_mempool_alloc0_type(pool, type) \
+ (type *) (rspamd_mempool_alloc0_((pool), sizeof(type), \
+ MAX(MIN_MEM_ALIGNMENT, RSPAMD_ALIGNOF(type)), (G_STRLOC)))
+
+/**
+ * Make a copy of string in pool
+ * @param pool memory pool object
+ * @param src source string
+ * @return pointer to newly created string that is copy of src
+ */
+gchar *rspamd_mempool_strdup_(rspamd_mempool_t *pool, const gchar *src, const gchar *loc)
+ RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT);
+#define rspamd_mempool_strdup(pool, src) \
+ rspamd_mempool_strdup_((pool), (src), (G_STRLOC))
+gchar *rspamd_mempool_strdup_len_(rspamd_mempool_t *pool, const gchar *src, gsize len, const gchar *loc)
+ RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT);
+#define rspamd_mempool_strdup_len(pool, src, len) \
+ rspamd_mempool_strdup_len_((pool), (src), (len), (G_STRLOC))
+
+struct f_str_tok;
+
+/**
+ * Make a copy of fixed string token in pool as null terminated string
+ * @param pool memory pool object
+ * @param src source string
+ * @return pointer to newly created string that is copy of src
+ */
+gchar *rspamd_mempool_ftokdup_(rspamd_mempool_t *pool,
+ const struct f_str_tok *src,
+ const gchar *loc)
+ RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT);
+#define rspamd_mempool_ftokdup(pool, src) \
+ rspamd_mempool_ftokdup_((pool), (src), (G_STRLOC))
+
+/**
+ * Allocate piece of shared memory
+ * @param pool memory pool object
+ * @param size bytes to allocate
+ */
+void *rspamd_mempool_alloc_shared_(rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc)
+ RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL;
+#define rspamd_mempool_alloc_shared(pool, size) \
+ rspamd_mempool_alloc_shared_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC))
+#define rspamd_mempool_alloc_shared_type(pool, type) \
+ (type *) (rspamd_mempool_alloc_shared_((pool), sizeof(type), \
+ MAX(MIN_MEM_ALIGNMENT, RSPAMD_ALIGNOF(type)), (G_STRLOC)))
+
+void *rspamd_mempool_alloc0_shared_(rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc)
+ RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL;
+#define rspamd_mempool_alloc0_shared(pool, size) \
+ rspamd_mempool_alloc0_shared_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC))
+#define rspamd_mempool_alloc0_shared_type(pool, type) \
+ (type *) (rspamd_mempool_alloc0_shared_((pool), sizeof(type), \
+ MAX(MIN_MEM_ALIGNMENT, RSPAMD_ALIGNOF(type)), (G_STRLOC)))
+
+/**
+ * Add destructor callback to pool
+ * @param pool memory pool object
+ * @param func pointer to function-destructor
+ * @param data pointer to data that would be passed to destructor
+ */
+void rspamd_mempool_add_destructor_full(rspamd_mempool_t *pool,
+ rspamd_mempool_destruct_t func,
+ void *data,
+ const gchar *function,
+ const gchar *line);
+
+/* Macros for common usage */
+#define rspamd_mempool_add_destructor(pool, func, data) \
+ rspamd_mempool_add_destructor_full(pool, func, data, (MEMPOOL_STR_FUNC), (G_STRLOC))
+
+/**
+ * Replace destructor callback to pool for specified pointer
+ * @param pool memory pool object
+ * @param func pointer to function-destructor
+ * @param old_data pointer to old data
+ * @param new_data pointer to data that would be passed to destructor
+ */
+void rspamd_mempool_replace_destructor(rspamd_mempool_t *pool,
+ rspamd_mempool_destruct_t func,
+ void *old_data, void *new_data);
+
+/**
+ * Calls all destructors associated with the specific memory pool without removing
+ * of the pool itself
+ * @param pool
+ */
+void rspamd_mempool_destructors_enforce(rspamd_mempool_t *pool);
+
+/**
+ * Delete pool, free all its chunks and call destructors chain
+ * @param pool memory pool object
+ */
+void rspamd_mempool_delete(rspamd_mempool_t *pool);
+
+/**
+ * Get new mutex from pool (allocated in shared memory)
+ * @param pool memory pool object
+ * @return mutex object
+ */
+rspamd_mempool_mutex_t *rspamd_mempool_get_mutex(rspamd_mempool_t *pool);
+
+/**
+ * Lock mutex
+ * @param mutex mutex to lock
+ */
+void rspamd_mempool_lock_mutex(rspamd_mempool_mutex_t *mutex);
+
+/**
+ * Unlock mutex
+ * @param mutex mutex to unlock
+ */
+void rspamd_mempool_unlock_mutex(rspamd_mempool_mutex_t *mutex);
+
+/**
+ * Create new rwlock and place it in shared memory
+ * @param pool memory pool object
+ * @return rwlock object
+ */
+rspamd_mempool_rwlock_t *rspamd_mempool_get_rwlock(rspamd_mempool_t *pool);
+
+/**
+ * Acquire read lock
+ * @param lock rwlock object
+ */
+void rspamd_mempool_rlock_rwlock(rspamd_mempool_rwlock_t *lock);
+
+/**
+ * Acquire write lock
+ * @param lock rwlock object
+ */
+void rspamd_mempool_wlock_rwlock(rspamd_mempool_rwlock_t *lock);
+
+/**
+ * Release read lock
+ * @param lock rwlock object
+ */
+void rspamd_mempool_runlock_rwlock(rspamd_mempool_rwlock_t *lock);
+
+/**
+ * Release write lock
+ * @param lock rwlock object
+ */
+void rspamd_mempool_wunlock_rwlock(rspamd_mempool_rwlock_t *lock);
+
+/**
+ * Get pool allocator statistics
+ * @param st stat pool struct
+ */
+void rspamd_mempool_stat(rspamd_mempool_stat_t *st);
+
+/**
+ * Reset memory pool stat
+ */
+void rspamd_mempool_stat_reset(void);
+
+/**
+ * Get optimal pool size based on page size for this system
+ * @return size of memory page in system
+ */
+#define rspamd_mempool_suggest_size() rspamd_mempool_suggest_size_(G_STRLOC)
+
+gsize rspamd_mempool_suggest_size_(const char *loc);
+
+gsize rspamd_mempool_get_used_size(rspamd_mempool_t *pool);
+gsize rspamd_mempool_get_wasted_size(rspamd_mempool_t *pool);
+
+/**
+ * Set memory pool variable
+ * @param pool memory pool object
+ * @param name name of variable
+ * @param gpointer value of variable
+ * @param destructor pointer to function-destructor
+ */
+void rspamd_mempool_set_variable(rspamd_mempool_t *pool,
+ const gchar *name,
+ gpointer value,
+ rspamd_mempool_destruct_t destructor);
+
+/**
+ * Get memory pool variable
+ * @param pool memory pool object
+ * @param name name of variable
+ * @return NULL or pointer to variable data
+ */
+gpointer rspamd_mempool_get_variable(rspamd_mempool_t *pool,
+ const gchar *name);
+/**
+ * Steal memory pool variable
+ * @param pool
+ * @param name
+ * @return
+ */
+gpointer rspamd_mempool_steal_variable(rspamd_mempool_t *pool,
+ const gchar *name);
+
+/**
+ * Removes variable from memory pool
+ * @param pool memory pool object
+ * @param name name of variable
+ */
+void rspamd_mempool_remove_variable(rspamd_mempool_t *pool,
+ const gchar *name);
+
+/**
+ * Prepend element to a list creating it in the memory pool
+ * @param l
+ * @param p
+ * @return
+ */
+GList *rspamd_mempool_glist_prepend(rspamd_mempool_t *pool,
+ GList *l, gpointer p) G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Append element to a list creating it in the memory pool
+ * @param l
+ * @param p
+ * @return
+ */
+GList *rspamd_mempool_glist_append(rspamd_mempool_t *pool,
+ GList *l, gpointer p) G_GNUC_WARN_UNUSED_RESULT;
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef __cplusplus
+#include <stdexcept> /* For std::runtime_error */
+
+namespace rspamd {
+
+template<class T>
+class mempool_allocator {
+public:
+ typedef T value_type;
+
+ mempool_allocator() = delete;
+ template<class U>
+ mempool_allocator(const mempool_allocator<U> &other)
+ : pool(other.pool)
+ {
+ }
+ mempool_allocator(rspamd_mempool_t *_pool)
+ : pool(_pool)
+ {
+ }
+ [[nodiscard]] constexpr T *allocate(std::size_t n)
+ {
+ if (G_MAXSIZE / 2 / sizeof(T) > n) {
+ throw std::runtime_error("integer overflow");
+ }
+ return reinterpret_cast<T *>(rspamd_mempool_alloc(pool, n * sizeof(T)));
+ }
+ constexpr void deallocate(T *p, std::size_t n)
+ {
+ /* Do nothing */
+ }
+
+private:
+ rspamd_mempool_t *pool;
+};
+
+}// namespace rspamd
+#endif
+
+#endif
diff --git a/src/libutil/mem_pool_internal.h b/src/libutil/mem_pool_internal.h
new file mode 100644
index 0000000..4fea839
--- /dev/null
+++ b/src/libutil/mem_pool_internal.h
@@ -0,0 +1,92 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_MEM_POOL_INTERNAL_H
+#define RSPAMD_MEM_POOL_INTERNAL_H
+
+/*
+ * Internal memory pool stuff
+ */
+
+#define align_ptr(p, a) \
+ ((guint8 *) ((uintptr_t) (p) + ((-(intptr_t) (p)) & ((a) -1))))
+
+enum rspamd_mempool_chain_type {
+ RSPAMD_MEMPOOL_NORMAL = 0,
+ RSPAMD_MEMPOOL_SHARED,
+ RSPAMD_MEMPOOL_MAX
+};
+#define ENTRY_LEN 128
+#define ENTRY_NELTS 64
+
+struct entry_elt {
+ guint32 fragmentation;
+ guint32 leftover;
+};
+
+struct rspamd_mempool_entry_point {
+ gchar src[ENTRY_LEN];
+ guint32 cur_suggestion;
+ guint32 cur_elts;
+ guint32 cur_vars;
+ struct entry_elt elts[ENTRY_NELTS];
+};
+
+/**
+ * Destructors list item structure
+ */
+struct _pool_destructors {
+ rspamd_mempool_destruct_t func; /**< pointer to destructor */
+ void *data; /**< data to free */
+ const gchar *function; /**< function from which this destructor was added */
+ const gchar *loc; /**< line number */
+ struct _pool_destructors *next;
+};
+
+
+struct rspamd_mempool_variable {
+ gpointer data;
+ rspamd_mempool_destruct_t dtor;
+};
+
+KHASH_INIT(rspamd_mempool_vars_hash,
+ guint32, struct rspamd_mempool_variable, 1,
+ kh_int_hash_func, kh_int_hash_equal);
+
+struct rspamd_mempool_specific {
+ struct _pool_chain *pools[RSPAMD_MEMPOOL_MAX];
+ struct _pool_destructors *dtors_head, *dtors_tail;
+ GPtrArray *trash_stack;
+ khash_t(rspamd_mempool_vars_hash) * variables;
+ struct rspamd_mempool_entry_point *entry;
+ gsize elt_len; /**< size of an element */
+ gsize used_memory;
+ guint wasted_memory;
+ gint flags;
+};
+
+/**
+ * Pool page structure
+ */
+struct _pool_chain {
+ guint8 *begin; /**< begin of pool chain block */
+ guint8 *pos; /**< current start of free space in block */
+ gsize slice_size; /**< length of block */
+ struct _pool_chain *next;
+};
+
+
+#endif
diff --git a/src/libutil/multipattern.c b/src/libutil/multipattern.c
new file mode 100644
index 0000000..630b1f9
--- /dev/null
+++ b/src/libutil/multipattern.c
@@ -0,0 +1,821 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "libutil/multipattern.h"
+#include "libutil/str_util.h"
+#include "libcryptobox/cryptobox.h"
+
+#ifdef WITH_HYPERSCAN
+#include "logger.h"
+#include "unix-std.h"
+#include "hs.h"
+#include "libserver/hyperscan_tools.h"
+#endif
+#include "acism.h"
+#include "libutil/regexp.h"
+#include <stdalign.h>
+
+#define MAX_SCRATCH 4
+
+enum rspamd_hs_check_state {
+ RSPAMD_HS_UNCHECKED = 0,
+ RSPAMD_HS_SUPPORTED,
+ RSPAMD_HS_UNSUPPORTED
+};
+
+static const char *hs_cache_dir = NULL;
+static enum rspamd_hs_check_state hs_suitable_cpu = RSPAMD_HS_UNCHECKED;
+
+
+struct RSPAMD_ALIGNED(64) rspamd_multipattern {
+#ifdef WITH_HYPERSCAN
+ rspamd_cryptobox_hash_state_t hash_state;
+ rspamd_hyperscan_t *hs_db;
+ hs_scratch_t *scratch[MAX_SCRATCH];
+ GArray *hs_pats;
+ GArray *hs_ids;
+ GArray *hs_flags;
+ guint scratch_used;
+#endif
+ ac_trie_t *t;
+ GArray *pats;
+ GArray *res;
+
+ gboolean compiled;
+ guint cnt;
+ enum rspamd_multipattern_flags flags;
+};
+
+static GQuark
+rspamd_multipattern_quark(void)
+{
+ return g_quark_from_static_string("multipattern");
+}
+
+static inline gboolean
+rspamd_hs_check(void)
+{
+#ifdef WITH_HYPERSCAN
+ if (G_UNLIKELY(hs_suitable_cpu == RSPAMD_HS_UNCHECKED)) {
+ if (hs_valid_platform() == HS_SUCCESS) {
+ hs_suitable_cpu = RSPAMD_HS_SUPPORTED;
+ }
+ else {
+ hs_suitable_cpu = RSPAMD_HS_UNSUPPORTED;
+ }
+ }
+#endif
+
+ return hs_suitable_cpu == RSPAMD_HS_SUPPORTED;
+}
+
+void rspamd_multipattern_library_init(const gchar *cache_dir)
+{
+ hs_cache_dir = cache_dir;
+#ifdef WITH_HYPERSCAN
+ rspamd_hs_check();
+#endif
+}
+
+#ifdef WITH_HYPERSCAN
+static gchar *
+rspamd_multipattern_escape_tld_hyperscan(const gchar *pattern, gsize slen,
+ gsize *dst_len)
+{
+ gsize len;
+ const gchar *p, *prefix, *suffix;
+ gchar *res;
+
+ /*
+ * We understand the following cases
+ * 1) blah -> .blah\b
+ * 2) *.blah -> ..*\\.blah\b|$
+ * 3) ???
+ */
+
+ if (pattern[0] == '*') {
+ p = strchr(pattern, '.');
+
+ if (p == NULL) {
+ /* XXX: bad */
+ p = pattern;
+ }
+ else {
+ p++;
+ }
+
+ prefix = "\\.";
+ len = slen + strlen(prefix);
+ }
+ else {
+ prefix = "\\.";
+ p = pattern;
+ len = slen + strlen(prefix);
+ }
+
+ suffix = "(:?\\b|$)";
+ len += strlen(suffix);
+
+ res = g_malloc(len + 1);
+ slen = rspamd_strlcpy(res, prefix, len + 1);
+ slen += rspamd_strlcpy(res + slen, p, len + 1 - slen);
+ slen += rspamd_strlcpy(res + slen, suffix, len + 1 - slen);
+
+ *dst_len = slen;
+
+ return res;
+}
+
+#endif
+static gchar *
+rspamd_multipattern_escape_tld_acism(const gchar *pattern, gsize len,
+ gsize *dst_len)
+{
+ gsize dlen, slen;
+ const gchar *p, *prefix;
+ gchar *res;
+
+ /*
+ * We understand the following cases
+ * 1) blah -> \\.blah
+ * 2) *.blah -> \\..*\\.blah
+ * 3) ???
+ */
+ slen = len;
+
+ if (pattern[0] == '*') {
+ dlen = slen;
+ p = memchr(pattern, '.', len);
+
+ if (p == NULL) {
+ /* XXX: bad */
+ p = pattern;
+ }
+ else {
+ p++;
+ }
+
+ dlen -= p - pattern;
+ prefix = ".";
+ dlen++;
+ }
+ else {
+ dlen = slen + 1;
+ prefix = ".";
+ p = pattern;
+ }
+
+ res = g_malloc(dlen + 1);
+ slen = strlen(prefix);
+ memcpy(res, prefix, slen);
+ rspamd_strlcpy(res + slen, p, dlen - slen + 1);
+
+ *dst_len = dlen;
+
+ return res;
+}
+
+/*
+ * Escapes special characters from specific pattern
+ */
+static gchar *
+rspamd_multipattern_pattern_filter(const gchar *pattern, gsize len,
+ enum rspamd_multipattern_flags flags,
+ gsize *dst_len)
+{
+ gchar *ret = NULL;
+ gint gl_flags = RSPAMD_REGEXP_ESCAPE_ASCII;
+
+ if (flags & RSPAMD_MULTIPATTERN_UTF8) {
+ gl_flags |= RSPAMD_REGEXP_ESCAPE_UTF;
+ }
+
+#ifdef WITH_HYPERSCAN
+ if (rspamd_hs_check()) {
+ if (flags & RSPAMD_MULTIPATTERN_TLD) {
+ gchar *tmp;
+ gsize tlen;
+ tmp = rspamd_multipattern_escape_tld_hyperscan(pattern, len, &tlen);
+
+ ret = rspamd_str_regexp_escape(tmp, tlen, dst_len,
+ gl_flags | RSPAMD_REGEXP_ESCAPE_RE);
+ g_free(tmp);
+ }
+ else if (flags & RSPAMD_MULTIPATTERN_RE) {
+ ret = rspamd_str_regexp_escape(pattern, len, dst_len, gl_flags | RSPAMD_REGEXP_ESCAPE_RE);
+ }
+ else if (flags & RSPAMD_MULTIPATTERN_GLOB) {
+ ret = rspamd_str_regexp_escape(pattern, len, dst_len,
+ gl_flags | RSPAMD_REGEXP_ESCAPE_GLOB);
+ }
+ else {
+ ret = rspamd_str_regexp_escape(pattern, len, dst_len, gl_flags);
+ }
+
+ return ret;
+ }
+#endif
+
+ if (flags & RSPAMD_MULTIPATTERN_TLD) {
+ ret = rspamd_multipattern_escape_tld_acism(pattern, len, dst_len);
+ }
+ else if (flags & RSPAMD_MULTIPATTERN_RE) {
+ ret = rspamd_str_regexp_escape(pattern, len, dst_len, gl_flags | RSPAMD_REGEXP_ESCAPE_RE);
+ }
+ else if (flags & RSPAMD_MULTIPATTERN_GLOB) {
+ ret = rspamd_str_regexp_escape(pattern, len, dst_len,
+ gl_flags | RSPAMD_REGEXP_ESCAPE_GLOB);
+ }
+ else {
+ ret = malloc(len + 1);
+ *dst_len = rspamd_strlcpy(ret, pattern, len + 1);
+ }
+
+ return ret;
+}
+
+struct rspamd_multipattern *
+rspamd_multipattern_create(enum rspamd_multipattern_flags flags)
+{
+ struct rspamd_multipattern *mp;
+
+ /* Align due to blake2b state */
+ (void) !posix_memalign((void **) &mp, RSPAMD_ALIGNOF(struct rspamd_multipattern),
+ sizeof(*mp));
+ g_assert(mp != NULL);
+ memset(mp, 0, sizeof(*mp));
+ mp->flags = flags;
+
+#ifdef WITH_HYPERSCAN
+ if (rspamd_hs_check()) {
+ mp->hs_pats = g_array_new(FALSE, TRUE, sizeof(gchar *));
+ mp->hs_flags = g_array_new(FALSE, TRUE, sizeof(gint));
+ mp->hs_ids = g_array_new(FALSE, TRUE, sizeof(gint));
+ rspamd_cryptobox_hash_init(&mp->hash_state, NULL, 0);
+
+ return mp;
+ }
+#endif
+
+ mp->pats = g_array_new(FALSE, TRUE, sizeof(ac_trie_pat_t));
+
+ return mp;
+}
+
+struct rspamd_multipattern *
+rspamd_multipattern_create_sized(guint npatterns,
+ enum rspamd_multipattern_flags flags)
+{
+ struct rspamd_multipattern *mp;
+
+ /* Align due to blake2b state */
+ (void) !posix_memalign((void **) &mp, RSPAMD_ALIGNOF(struct rspamd_multipattern), sizeof(*mp));
+ g_assert(mp != NULL);
+ memset(mp, 0, sizeof(*mp));
+ mp->flags = flags;
+
+#ifdef WITH_HYPERSCAN
+ if (rspamd_hs_check()) {
+ mp->hs_pats = g_array_sized_new(FALSE, TRUE, sizeof(gchar *), npatterns);
+ mp->hs_flags = g_array_sized_new(FALSE, TRUE, sizeof(gint), npatterns);
+ mp->hs_ids = g_array_sized_new(FALSE, TRUE, sizeof(gint), npatterns);
+ rspamd_cryptobox_hash_init(&mp->hash_state, NULL, 0);
+
+ return mp;
+ }
+#endif
+
+ mp->pats = g_array_sized_new(FALSE, TRUE, sizeof(ac_trie_pat_t), npatterns);
+
+ return mp;
+}
+
+void rspamd_multipattern_add_pattern(struct rspamd_multipattern *mp,
+ const gchar *pattern, gint flags)
+{
+ g_assert(pattern != NULL);
+
+ rspamd_multipattern_add_pattern_len(mp, pattern, strlen(pattern), flags);
+}
+
+void rspamd_multipattern_add_pattern_len(struct rspamd_multipattern *mp,
+ const gchar *pattern, gsize patlen, gint flags)
+{
+ gsize dlen;
+
+ g_assert(pattern != NULL);
+ g_assert(mp != NULL);
+ g_assert(!mp->compiled);
+
+#ifdef WITH_HYPERSCAN
+ if (rspamd_hs_check()) {
+ gchar *np;
+ gint fl = HS_FLAG_SOM_LEFTMOST;
+ gint adjusted_flags = mp->flags | flags;
+
+ if (adjusted_flags & RSPAMD_MULTIPATTERN_ICASE) {
+ fl |= HS_FLAG_CASELESS;
+ }
+ if (adjusted_flags & RSPAMD_MULTIPATTERN_UTF8) {
+ if (adjusted_flags & RSPAMD_MULTIPATTERN_TLD) {
+ fl |= HS_FLAG_UTF8;
+ }
+ else {
+ fl |= HS_FLAG_UTF8 | HS_FLAG_UCP;
+ }
+ }
+ if (adjusted_flags & RSPAMD_MULTIPATTERN_DOTALL) {
+ fl |= HS_FLAG_DOTALL;
+ }
+ if (adjusted_flags & RSPAMD_MULTIPATTERN_SINGLEMATCH) {
+ fl |= HS_FLAG_SINGLEMATCH;
+ fl &= ~HS_FLAG_SOM_LEFTMOST; /* According to hyperscan docs */
+ }
+ if (adjusted_flags & RSPAMD_MULTIPATTERN_NO_START) {
+ fl &= ~HS_FLAG_SOM_LEFTMOST;
+ }
+
+ g_array_append_val(mp->hs_flags, fl);
+ np = rspamd_multipattern_pattern_filter(pattern, patlen, flags, &dlen);
+ g_array_append_val(mp->hs_pats, np);
+ fl = mp->cnt;
+ g_array_append_val(mp->hs_ids, fl);
+ rspamd_cryptobox_hash_update(&mp->hash_state, np, dlen);
+
+ mp->cnt++;
+
+ return;
+ }
+#endif
+ ac_trie_pat_t pat;
+
+ pat.ptr = rspamd_multipattern_pattern_filter(pattern, patlen, flags, &dlen);
+ pat.len = dlen;
+
+ g_array_append_val(mp->pats, pat);
+
+ mp->cnt++;
+}
+
+struct rspamd_multipattern *
+rspamd_multipattern_create_full(const gchar **patterns,
+ guint npatterns, enum rspamd_multipattern_flags flags)
+{
+ struct rspamd_multipattern *mp;
+ guint i;
+
+ g_assert(npatterns > 0);
+ g_assert(patterns != NULL);
+
+ mp = rspamd_multipattern_create_sized(npatterns, flags);
+
+ for (i = 0; i < npatterns; i++) {
+ rspamd_multipattern_add_pattern(mp, patterns[i], flags);
+ }
+
+ return mp;
+}
+
+#ifdef WITH_HYPERSCAN
+static gboolean
+rspamd_multipattern_try_load_hs(struct rspamd_multipattern *mp,
+ const guchar *hash)
+{
+ gchar fp[PATH_MAX];
+
+ if (hs_cache_dir == NULL) {
+ return FALSE;
+ }
+
+ rspamd_snprintf(fp, sizeof(fp), "%s/%*xs.hsmp", hs_cache_dir,
+ (gint) rspamd_cryptobox_HASHBYTES / 2, hash);
+ mp->hs_db = rspamd_hyperscan_maybe_load(fp, 0);
+
+ return mp->hs_db != NULL;
+}
+
+static void
+rspamd_multipattern_try_save_hs(struct rspamd_multipattern *mp,
+ const guchar *hash)
+{
+ gchar fp[PATH_MAX], np[PATH_MAX];
+ char *bytes = NULL;
+ gsize len;
+ gint fd;
+
+ if (hs_cache_dir == NULL) {
+ return;
+ }
+
+ rspamd_snprintf(fp, sizeof(fp), "%s%shsmp-XXXXXXXXXXXXX", G_DIR_SEPARATOR_S,
+ hs_cache_dir);
+
+ if ((fd = g_mkstemp_full(fp, O_CREAT | O_EXCL | O_WRONLY, 00644)) != -1) {
+ int ret;
+ if ((ret = hs_serialize_database(rspamd_hyperscan_get_database(mp->hs_db), &bytes, &len)) == HS_SUCCESS) {
+ if (write(fd, bytes, len) == -1) {
+ msg_warn("cannot write hyperscan cache to %s: %s",
+ fp, strerror(errno));
+ unlink(fp);
+ free(bytes);
+ }
+ else {
+ free(bytes);
+ fsync(fd);
+
+ rspamd_snprintf(np, sizeof(np), "%s/%*xs.hsmp", hs_cache_dir,
+ (gint) rspamd_cryptobox_HASHBYTES / 2, hash);
+
+ if (rename(fp, np) == -1) {
+ msg_warn("cannot rename hyperscan cache from %s to %s: %s",
+ fp, np, strerror(errno));
+ unlink(fp);
+ }
+ else {
+ rspamd_hyperscan_notice_known(np);
+ }
+ }
+ }
+ else {
+ msg_warn("cannot serialize hyperscan cache to %s: error code %d",
+ fp, ret);
+ unlink(fp);
+ }
+
+
+ close(fd);
+ }
+ else {
+ msg_warn("cannot open a temp file %s to write hyperscan cache: %s", fp, strerror(errno));
+ }
+}
+#endif
+
+gboolean
+rspamd_multipattern_compile(struct rspamd_multipattern *mp, GError **err)
+{
+ g_assert(mp != NULL);
+ g_assert(!mp->compiled);
+
+#ifdef WITH_HYPERSCAN
+ if (rspamd_hs_check()) {
+ guint i;
+ hs_platform_info_t plt;
+ hs_compile_error_t *hs_errors;
+ guchar hash[rspamd_cryptobox_HASHBYTES];
+
+ if (mp->cnt > 0) {
+ g_assert(hs_populate_platform(&plt) == HS_SUCCESS);
+ rspamd_cryptobox_hash_update(&mp->hash_state, (void *) &plt, sizeof(plt));
+ rspamd_cryptobox_hash_final(&mp->hash_state, hash);
+
+ if (!rspamd_multipattern_try_load_hs(mp, hash)) {
+ hs_database_t *db = NULL;
+
+ if (hs_compile_multi((const char *const *) mp->hs_pats->data,
+ (const unsigned int *) mp->hs_flags->data,
+ (const unsigned int *) mp->hs_ids->data,
+ mp->cnt,
+ HS_MODE_BLOCK,
+ &plt,
+ &db,
+ &hs_errors) != HS_SUCCESS) {
+
+ g_set_error(err, rspamd_multipattern_quark(), EINVAL,
+ "cannot create tree of regexp when processing '%s': %s",
+ g_array_index(mp->hs_pats, char *, hs_errors->expression),
+ hs_errors->message);
+ hs_free_compile_error(hs_errors);
+
+ return FALSE;
+ }
+
+ if (hs_cache_dir != NULL) {
+ char fpath[PATH_MAX];
+ rspamd_snprintf(fpath, sizeof(fpath), "%s/%*xs.hsmp", hs_cache_dir,
+ (gint) rspamd_cryptobox_HASHBYTES / 2, hash);
+ mp->hs_db = rspamd_hyperscan_from_raw_db(db, fpath);
+ }
+ else {
+ /* Should not happen in the real life */
+ mp->hs_db = rspamd_hyperscan_from_raw_db(db, NULL);
+ }
+
+ rspamd_multipattern_try_save_hs(mp, hash);
+ }
+
+ for (i = 0; i < MAX_SCRATCH; i++) {
+ mp->scratch[i] = NULL;
+ }
+
+ for (i = 0; i < MAX_SCRATCH; i++) {
+ int ret;
+
+ if ((ret = hs_alloc_scratch(rspamd_hyperscan_get_database(mp->hs_db), &mp->scratch[i])) != HS_SUCCESS) {
+ msg_err("cannot allocate scratch space for hyperscan: error code %d", ret);
+
+ /* Clean all scratches that are non-NULL */
+ for (int ii = 0; ii < MAX_SCRATCH; ii++) {
+ if (mp->scratch[ii] != NULL) {
+ hs_free_scratch(mp->scratch[ii]);
+ }
+ }
+ g_set_error(err, rspamd_multipattern_quark(), EINVAL,
+ "cannot allocate scratch space for hyperscan: error code %d", ret);
+
+ rspamd_hyperscan_free(mp->hs_db, true);
+ mp->hs_db = NULL;
+
+ return FALSE;
+ }
+ }
+ }
+
+ mp->compiled = TRUE;
+
+ return TRUE;
+ }
+#endif
+
+ if (mp->cnt > 0) {
+
+ if (mp->flags & (RSPAMD_MULTIPATTERN_GLOB | RSPAMD_MULTIPATTERN_RE)) {
+ /* Fallback to pcre... */
+ rspamd_regexp_t *re;
+ mp->res = g_array_sized_new(FALSE, TRUE,
+ sizeof(rspamd_regexp_t *), mp->cnt);
+
+ for (guint i = 0; i < mp->cnt; i++) {
+ const ac_trie_pat_t *pat;
+ const gchar *pat_flags = NULL;
+
+ if (mp->flags & RSPAMD_MULTIPATTERN_UTF8) {
+ pat_flags = "u";
+ }
+
+ pat = &g_array_index(mp->pats, ac_trie_pat_t, i);
+ re = rspamd_regexp_new(pat->ptr, pat_flags, err);
+
+ if (re == NULL) {
+ return FALSE;
+ }
+
+ g_array_append_val(mp->res, re);
+ }
+ }
+ else {
+ mp->t = acism_create((const ac_trie_pat_t *) mp->pats->data, mp->cnt);
+ }
+ }
+
+ mp->compiled = TRUE;
+
+ return TRUE;
+}
+
+struct rspamd_multipattern_cbdata {
+ struct rspamd_multipattern *mp;
+ const gchar *in;
+ gsize len;
+ rspamd_multipattern_cb_t cb;
+ gpointer ud;
+ guint nfound;
+ gint ret;
+};
+
+#ifdef WITH_HYPERSCAN
+static gint
+rspamd_multipattern_hs_cb(unsigned int id,
+ unsigned long long from,
+ unsigned long long to,
+ unsigned int flags,
+ void *ud)
+{
+ struct rspamd_multipattern_cbdata *cbd = ud;
+ gint ret = 0;
+
+ if (to > 0) {
+
+ if (from == HS_OFFSET_PAST_HORIZON) {
+ from = 0;
+ }
+
+ ret = cbd->cb(cbd->mp, id, from, to, cbd->in, cbd->len, cbd->ud);
+
+ cbd->nfound++;
+ cbd->ret = ret;
+ }
+
+ return ret;
+}
+#endif
+
+static gint
+rspamd_multipattern_acism_cb(int strnum, int textpos, void *context)
+{
+ struct rspamd_multipattern_cbdata *cbd = context;
+ gint ret;
+ ac_trie_pat_t pat;
+
+ pat = g_array_index(cbd->mp->pats, ac_trie_pat_t, strnum);
+ ret = cbd->cb(cbd->mp, strnum, textpos - pat.len,
+ textpos, cbd->in, cbd->len, cbd->ud);
+
+ cbd->nfound++;
+ cbd->ret = ret;
+
+ return ret;
+}
+
+gint rspamd_multipattern_lookup(struct rspamd_multipattern *mp,
+ const gchar *in, gsize len, rspamd_multipattern_cb_t cb,
+ gpointer ud, guint *pnfound)
+{
+ struct rspamd_multipattern_cbdata cbd;
+ gint ret = 0;
+
+ g_assert(mp != NULL);
+
+ if (mp->cnt == 0 || !mp->compiled || len == 0) {
+ return 0;
+ }
+
+ cbd.mp = mp;
+ cbd.in = in;
+ cbd.len = len;
+ cbd.cb = cb;
+ cbd.ud = ud;
+ cbd.nfound = 0;
+ cbd.ret = 0;
+
+#ifdef WITH_HYPERSCAN
+ if (rspamd_hs_check()) {
+ hs_scratch_t *scr = NULL;
+ guint i;
+
+ for (i = 0; i < MAX_SCRATCH; i++) {
+ if (!(mp->scratch_used & (1 << i))) {
+ mp->scratch_used |= (1 << i);
+ scr = mp->scratch[i];
+ break;
+ }
+ }
+
+ g_assert(scr != NULL);
+
+ ret = hs_scan(rspamd_hyperscan_get_database(mp->hs_db), in, len, 0, scr,
+ rspamd_multipattern_hs_cb, &cbd);
+
+ mp->scratch_used &= ~(1 << i);
+
+ if (ret == HS_SUCCESS) {
+ ret = 0;
+ }
+ else if (ret == HS_SCAN_TERMINATED) {
+ ret = cbd.ret;
+ }
+
+ if (pnfound) {
+ *pnfound = cbd.nfound;
+ }
+
+ return ret;
+ }
+#endif
+
+ gint state = 0;
+
+ if (mp->flags & (RSPAMD_MULTIPATTERN_GLOB | RSPAMD_MULTIPATTERN_RE)) {
+ /* Terribly inefficient, but who cares - just use hyperscan */
+ for (guint i = 0; i < mp->cnt; i++) {
+ rspamd_regexp_t *re = g_array_index(mp->res, rspamd_regexp_t *, i);
+ const gchar *start = NULL, *end = NULL;
+
+ while (rspamd_regexp_search(re,
+ in,
+ len,
+ &start,
+ &end,
+ TRUE,
+ NULL)) {
+ if (rspamd_multipattern_acism_cb(i, end - in, &cbd)) {
+ goto out;
+ }
+ }
+ }
+ out:
+ ret = cbd.ret;
+
+ if (pnfound) {
+ *pnfound = cbd.nfound;
+ }
+ }
+ else {
+ /* Plain trie */
+ ret = acism_lookup(mp->t, in, len, rspamd_multipattern_acism_cb, &cbd,
+ &state, mp->flags & RSPAMD_MULTIPATTERN_ICASE);
+
+ if (pnfound) {
+ *pnfound = cbd.nfound;
+ }
+ }
+
+ return ret;
+}
+
+
+void rspamd_multipattern_destroy(struct rspamd_multipattern *mp)
+{
+ guint i;
+
+ if (mp) {
+#ifdef WITH_HYPERSCAN
+ if (rspamd_hs_check()) {
+ gchar *p;
+
+ if (mp->compiled && mp->cnt > 0) {
+ for (i = 0; i < MAX_SCRATCH; i++) {
+ hs_free_scratch(mp->scratch[i]);
+ }
+
+ if (mp->hs_db) {
+ rspamd_hyperscan_free(mp->hs_db, false);
+ }
+ }
+
+ for (i = 0; i < mp->cnt; i++) {
+ p = g_array_index(mp->hs_pats, gchar *, i);
+ g_free(p);
+ }
+
+ g_array_free(mp->hs_pats, TRUE);
+ g_array_free(mp->hs_ids, TRUE);
+ g_array_free(mp->hs_flags, TRUE);
+ free(mp); /* Due to posix_memalign */
+
+ return;
+ }
+#endif
+ ac_trie_pat_t pat;
+
+ if (mp->compiled && mp->cnt > 0) {
+ acism_destroy(mp->t);
+ }
+
+ for (i = 0; i < mp->cnt; i++) {
+ pat = g_array_index(mp->pats, ac_trie_pat_t, i);
+ g_free((gchar *) pat.ptr);
+ }
+
+ g_array_free(mp->pats, TRUE);
+
+ g_free(mp);
+ }
+}
+
+const gchar *
+rspamd_multipattern_get_pattern(struct rspamd_multipattern *mp,
+ guint index)
+{
+ g_assert(mp != NULL);
+ g_assert(index < mp->cnt);
+
+#ifdef WITH_HYPERSCAN
+ if (rspamd_hs_check()) {
+ return g_array_index(mp->hs_pats, gchar *, index);
+ }
+#endif
+
+ ac_trie_pat_t pat;
+
+ pat = g_array_index(mp->pats, ac_trie_pat_t, index);
+
+ return pat.ptr;
+}
+
+guint rspamd_multipattern_get_npatterns(struct rspamd_multipattern *mp)
+{
+ g_assert(mp != NULL);
+
+ return mp->cnt;
+}
+
+gboolean
+rspamd_multipattern_has_hyperscan(void)
+{
+ return rspamd_hs_check();
+}
diff --git a/src/libutil/multipattern.h b/src/libutil/multipattern.h
new file mode 100644
index 0000000..9302766
--- /dev/null
+++ b/src/libutil/multipattern.h
@@ -0,0 +1,173 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_LIBUTIL_MULTIPATTERN_H_
+#define SRC_LIBUTIL_MULTIPATTERN_H_
+
+#include "config.h"
+
+/**
+ * @file multipattern.h
+ *
+ * This file defines structure that acts like a transparent bridge between
+ * hyperscan and ac-trie
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum rspamd_multipattern_flags {
+ RSPAMD_MULTIPATTERN_DEFAULT = 0,
+ RSPAMD_MULTIPATTERN_ICASE = (1 << 0),
+ RSPAMD_MULTIPATTERN_UTF8 = (1 << 1),
+ RSPAMD_MULTIPATTERN_TLD = (1 << 2),
+ /* Not supported by acism */
+ RSPAMD_MULTIPATTERN_GLOB = (1 << 3),
+ RSPAMD_MULTIPATTERN_RE = (1 << 4),
+ RSPAMD_MULTIPATTERN_DOTALL = (1 << 5),
+ RSPAMD_MULTIPATTERN_SINGLEMATCH = (1 << 6),
+ RSPAMD_MULTIPATTERN_NO_START = (1 << 7),
+};
+
+struct rspamd_multipattern;
+struct rspamd_cryptobox_library_ctx;
+
+/**
+ * Called on pattern match
+ * @param mp multipattern structure
+ * @param strnum number of pattern matched
+ * @param textpos position in the text
+ * @param text input text
+ * @param len length of input text
+ * @param context userdata
+ * @return if 0 then search for another pattern, otherwise return this value to caller
+ */
+typedef gint (*rspamd_multipattern_cb_t)(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint match_pos,
+ const gchar *text,
+ gsize len,
+ void *context);
+
+/**
+ * Init multipart library and set the appropriate cache dir
+ * @param cache_dir
+ */
+void rspamd_multipattern_library_init(const gchar *cache_dir);
+
+/**
+ * Creates empty multipattern structure
+ * @param flags
+ * @return
+ */
+struct rspamd_multipattern *rspamd_multipattern_create(
+ enum rspamd_multipattern_flags flags);
+
+/**
+ * Creates multipattern with preallocated number of patterns to speed up loading
+ * @param flags
+ * @param reserved
+ * @return
+ */
+struct rspamd_multipattern *rspamd_multipattern_create_sized(guint reserved,
+ enum rspamd_multipattern_flags flags);
+
+/**
+ * Creates new multipattern structure
+ * @param patterns vector of null terminated strings
+ * @param npatterns number of patterns
+ * @param flags flags applied to all patterns
+ * @return new multipattern structure
+ */
+struct rspamd_multipattern *rspamd_multipattern_create_full(
+ const gchar **patterns,
+ guint npatterns,
+ enum rspamd_multipattern_flags flags);
+
+/**
+ * Adds new pattern to match engine from zero-terminated string
+ * @param mp
+ * @param pattern
+ */
+void rspamd_multipattern_add_pattern(struct rspamd_multipattern *mp,
+ const gchar *pattern, gint flags);
+
+/**
+ * Adds new pattern from arbitrary string
+ * @param mp
+ * @param pattern
+ * @param patlen
+ * @param flags
+ */
+void rspamd_multipattern_add_pattern_len(struct rspamd_multipattern *mp,
+ const gchar *pattern, gsize patlen, gint flags);
+
+/**
+ * Compiles multipattern structure
+ * @param mp
+ * @return
+ */
+gboolean rspamd_multipattern_compile(struct rspamd_multipattern *mp,
+ GError **err);
+
+/**
+ * Lookups for patterns in a text using the specified callback function
+ * @param mp
+ * @param in
+ * @param len
+ * @param cb if callback returns non-zero, then search is terminated and that value is returned
+ * @param ud callback data
+ * @return
+ */
+gint rspamd_multipattern_lookup(struct rspamd_multipattern *mp,
+ const gchar *in, gsize len, rspamd_multipattern_cb_t cb,
+ gpointer ud, guint *pnfound);
+
+/**
+ * Get pattern string from multipattern identified by index
+ * @param mp
+ * @param index
+ * @return
+ */
+const gchar *rspamd_multipattern_get_pattern(struct rspamd_multipattern *mp,
+ guint index);
+
+/**
+ * Returns number of patterns in a multipattern matcher
+ * @param mp
+ * @return
+ */
+guint rspamd_multipattern_get_npatterns(struct rspamd_multipattern *mp);
+
+/**
+ * Destroys multipattern structure
+ * @param mp
+ */
+void rspamd_multipattern_destroy(struct rspamd_multipattern *mp);
+
+/**
+ * Returns TRUE if hyperscan is supported
+ * @return
+ */
+gboolean rspamd_multipattern_has_hyperscan(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBUTIL_MULTIPATTERN_H_ */
diff --git a/src/libutil/printf.c b/src/libutil/printf.c
new file mode 100644
index 0000000..ba53b56
--- /dev/null
+++ b/src/libutil/printf.c
@@ -0,0 +1,1097 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Copyright (C) 2002-2015 Igor Sysoev
+ * Copyright (C) 2011-2015 Nginx, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "printf.h"
+#include "str_util.h"
+#include "contrib/fpconv/fpconv.h"
+
+/**
+ * From FreeBSD libutil code
+ */
+static const int maxscale = 6;
+static const gchar _hex[] = "0123456789abcdef";
+static const gchar _HEX[] = "0123456789ABCDEF";
+
+static gchar *
+rspamd_humanize_number(gchar *buf, gchar *last, gint64 num, gboolean bytes)
+{
+ const gchar *prefixes;
+ int i, r, remainder, sign;
+ gint64 divisor;
+ gsize len = last - buf;
+
+ remainder = 0;
+
+ if (!bytes) {
+ divisor = 1000;
+ prefixes = "\0\0\0\0k\0\0\0M\0\0\0G\0\0\0T\0\0\0P\0\0\0E";
+ }
+ else {
+ divisor = 1024;
+ prefixes = "B\0\0\0KiB\0MiB\0GiB\0TiB\0PiB\0EiB";
+ }
+
+#define SCALE2PREFIX(scale) (&prefixes[(scale) *4])
+
+ if (num < 0) {
+ sign = -1;
+ num = -num;
+ }
+ else {
+ sign = 1;
+ }
+
+ /*
+ * Divide the number until it fits the given column.
+ * If there will be an overflow by the rounding below,
+ * divide once more.
+ */
+ for (i = 0; i < maxscale && num > divisor; i++) {
+ remainder = num % divisor;
+ num /= divisor;
+ }
+
+ if (remainder == 0 || num > divisor / 2) {
+ r = rspamd_snprintf(buf, len, "%L%s",
+ sign * (num + (remainder + 50) / divisor),
+ SCALE2PREFIX(i));
+ }
+ else {
+ /* Floating point version */
+ r = rspamd_snprintf(buf, len, "%.2f%s",
+ sign * (num + remainder / (gdouble) divisor),
+ SCALE2PREFIX(i));
+ }
+
+#undef SCALE2PREFIX
+
+ return buf + r;
+}
+
+
+static inline unsigned
+rspamd_decimal_digits32(guint32 val)
+{
+ static const guint32 powers_of_10[] = {
+ 0,
+ 10,
+ 100,
+ 1000,
+ 10000,
+ 100000,
+ 1000000,
+ 10000000,
+ 100000000,
+ 1000000000};
+ unsigned tmp;
+
+#if defined(_MSC_VER)
+ unsigned long r = 0;
+ _BitScanReverse(&r, val | 1);
+ tmp = (r + 1) * 1233 >> 12;
+#elif defined(__GNUC__) && (__GNUC__ >= 3)
+ tmp = (32 - __builtin_clz(val | 1U)) * 1233 >> 12;
+
+#else /* Software version */
+ static const unsigned debruijn_tbl[32] = {0, 9, 1, 10, 13, 21, 2, 29,
+ 11, 14, 16, 18, 22, 25, 3, 30,
+ 8, 12, 20, 28, 15, 17, 24, 7,
+ 19, 27, 23, 6, 26, 5, 4, 31};
+ guint32 v = val | 1;
+
+ v |= v >> 1;
+ v |= v >> 2;
+ v |= v >> 4;
+ v |= v >> 8;
+ v |= v >> 16;
+ tmp = (1 + debruijn_tbl[(v * 0x07C4ACDDU) >> 27]) * 1233 >> 12;
+#endif
+ return tmp - (val < powers_of_10[tmp]) + 1;
+}
+
+static inline unsigned
+rspamd_decimal_digits64(guint64 val)
+{
+ static const guint64 powers_of_10[] = {
+ 0,
+ 10ULL,
+ 100ULL,
+ 1000ULL,
+ 10000ULL,
+ 100000ULL,
+ 1000000ULL,
+ 10000000ULL,
+ 100000000ULL,
+ 1000000000ULL,
+ 10000000000ULL,
+ 100000000000ULL,
+ 1000000000000ULL,
+ 10000000000000ULL,
+ 100000000000000ULL,
+ 1000000000000000ULL,
+ 10000000000000000ULL,
+ 100000000000000000ULL,
+ 1000000000000000000ULL,
+ 10000000000000000000ULL};
+ unsigned tmp;
+
+#if defined(_MSC_VER)
+#if _M_IX86
+ unsigned long r = 0;
+ guint64 m = val | 1;
+ if (_BitScanReverse(&r, m >> 32)) {
+ r += 32;
+ }
+ else {
+ _BitScanReverse(&r, m & 0xFFFFFFFF);
+ }
+ tmp = (r + 1) * 1233 >> 12;
+#else
+ unsigned long r = 0;
+ _BitScanReverse64(&r, val | 1);
+ tmp = (r + 1) * 1233 >> 12;
+#endif
+#elif defined(__GNUC__) && (__GNUC__ >= 3)
+ tmp = (64 - __builtin_clzll(val | 1ULL)) * 1233 >> 12;
+#else /* Software version */
+ static const unsigned debruijn_tbl[32] = {0, 9, 1, 10, 13, 21, 2, 29,
+ 11, 14, 16, 18, 22, 25, 3, 30,
+ 8, 12, 20, 28, 15, 17, 24, 7,
+ 19, 27, 23, 6, 26, 5, 4, 31};
+ guint32 v = val >> 32;
+
+ if (v) {
+ v |= 1;
+ v |= v >> 1;
+ v |= v >> 2;
+ v |= v >> 4;
+ v |= v >> 8;
+ v |= v >> 16;
+ tmp = 32 + debruijn_tbl[(v * 0x07C4ACDDU) >> 27];
+ }
+ else {
+ v = val & 0xFFFFFFFF;
+ v |= 1;
+ v |= v >> 1;
+ v |= v >> 2;
+ v |= v >> 4;
+ v |= v >> 8;
+ v |= v >> 16;
+
+ tmp = debruijn_tbl[(v * 0x07C4ACDDU) >> 27];
+ }
+
+
+ tmp = (tmp + 1) * 1233 >> 12;
+#endif
+
+ return tmp - (val < powers_of_10[tmp]) + 1;
+}
+
+/*
+ * Idea from https://github.com/miloyip/itoa-benchmark:
+ * Uses lookup table (LUT) of digit pairs for division/modulo of 100.
+ *
+ * Mentioned in:
+ * https://www.slideshare.net/andreialexandrescu1/three-optimization-tips-for-c-15708507
+ */
+
+static const char int_lookup_table[200] = {
+ '0', '0', '0', '1', '0', '2', '0', '3', '0', '4',
+ '0', '5', '0', '6', '0', '7', '0', '8', '0', '9',
+ '1', '0', '1', '1', '1', '2', '1', '3', '1', '4',
+ '1', '5', '1', '6', '1', '7', '1', '8', '1', '9',
+ '2', '0', '2', '1', '2', '2', '2', '3', '2', '4',
+ '2', '5', '2', '6', '2', '7', '2', '8', '2', '9',
+ '3', '0', '3', '1', '3', '2', '3', '3', '3', '4',
+ '3', '5', '3', '6', '3', '7', '3', '8', '3', '9',
+ '4', '0', '4', '1', '4', '2', '4', '3', '4', '4',
+ '4', '5', '4', '6', '4', '7', '4', '8', '4', '9',
+ '5', '0', '5', '1', '5', '2', '5', '3', '5', '4',
+ '5', '5', '5', '6', '5', '7', '5', '8', '5', '9',
+ '6', '0', '6', '1', '6', '2', '6', '3', '6', '4',
+ '6', '5', '6', '6', '6', '7', '6', '8', '6', '9',
+ '7', '0', '7', '1', '7', '2', '7', '3', '7', '4',
+ '7', '5', '7', '6', '7', '7', '7', '8', '7', '9',
+ '8', '0', '8', '1', '8', '2', '8', '3', '8', '4',
+ '8', '5', '8', '6', '8', '7', '8', '8', '8', '9',
+ '9', '0', '9', '1', '9', '2', '9', '3', '9', '4',
+ '9', '5', '9', '6', '9', '7', '9', '8', '9', '9'};
+
+static inline guint
+rspamd_uint32_print(guint32 in, gchar *out)
+{
+ guint ndigits = rspamd_decimal_digits32(in);
+ gchar *p;
+
+ p = out + ndigits - 1;
+
+ while (in >= 100) {
+ unsigned idx = (in % 100) * 2;
+
+ /* Do two digits at once */
+ *p-- = int_lookup_table[idx + 1];
+ *p-- = int_lookup_table[idx];
+
+ in /= 100;
+ }
+
+ if (in < 10) {
+ *p = ((char) in) + '0';
+ }
+ else {
+ unsigned idx = in * 2;
+
+ *p-- = int_lookup_table[idx + 1];
+ *p = int_lookup_table[idx];
+ }
+
+ return ndigits;
+}
+
+static inline guint
+rspamd_uint64_print(guint64 in, gchar *out)
+{
+ guint ndigits = rspamd_decimal_digits64(in);
+ guint32 v32;
+ gchar *p;
+
+ p = out + ndigits - 1;
+
+ while (in >= 100000000) {
+ v32 = (guint32) (in % 100000000);
+ guint32 a, b, a1, a2, b1, b2;
+
+ /* Initial spill */
+ a = v32 / 10000;
+ b = v32 % 10000;
+ a1 = (a / 100) * 2;
+ a2 = (a % 100) * 2;
+ b1 = (b / 100) * 2;
+ b2 = (b % 100) * 2;
+
+ /* Fill 8 digits at once */
+ *p-- = int_lookup_table[b2 + 1];
+ *p-- = int_lookup_table[b2];
+ *p-- = int_lookup_table[b1 + 1];
+ *p-- = int_lookup_table[b1];
+ *p-- = int_lookup_table[a2 + 1];
+ *p-- = int_lookup_table[a2];
+ *p-- = int_lookup_table[a1 + 1];
+ *p-- = int_lookup_table[a1];
+
+ in /= 100000000;
+ }
+
+ /* Remaining 32 bit */
+ v32 = (guint32) in;
+
+ while (v32 >= 100) {
+ unsigned idx = (v32 % 100) << 1;
+
+ /* Do 2 digits at once */
+ *p-- = int_lookup_table[idx + 1];
+ *p-- = int_lookup_table[idx];
+
+ v32 /= 100;
+ }
+
+ if (v32 < 10) {
+ *p = ((char) v32) + '0';
+ }
+ else {
+ unsigned idx = v32 * 2;
+
+ *p-- = int_lookup_table[idx + 1];
+ *p = int_lookup_table[idx];
+ }
+
+ return ndigits;
+}
+
+static inline int
+rspamd_ffsll(long long n)
+{
+#ifdef __has_builtin
+#if __has_builtin(__builtin_ffsll)
+ return __builtin_ffsll(n);
+#elif __has_builtin(__builtin_ctzll)
+ if (n == 0) {
+ return 0;
+ }
+
+ return __builtin_ctzll(n) + 1;
+#endif
+#endif /* __has_builtin */
+
+#ifdef HAVE_FFSL
+ return ffsl(n);
+#else
+ if (n == 0) {
+ return 0;
+ }
+
+ int bit;
+ for (bit = 1; !(n & 1); bit++) {
+ n = ((unsigned long long) n) >> 1;
+ }
+ return bit;
+#endif
+}
+
+static gchar *
+rspamd_sprintf_num(gchar *buf, gchar *last, guint64 ui64, gchar zero,
+ guint hexadecimal, guint binary, guint width)
+{
+ gchar *p, temp[64];
+ size_t len = 0;
+
+ if (G_LIKELY(hexadecimal == 0 && binary == 0)) {
+ p = temp;
+
+ if (ui64 < G_MAXUINT32) {
+ len = rspamd_uint32_print((guint32) ui64, temp);
+ }
+ else {
+ len = rspamd_uint64_print(ui64, temp);
+ }
+ }
+ else if (hexadecimal == 1) {
+ p = temp + sizeof(temp);
+ do {
+ *--p = _hex[(guint32) (ui64 & 0xf)];
+ } while (ui64 >>= 4);
+
+ len = (temp + sizeof(temp)) - p;
+ }
+ else if (hexadecimal == 2) { /* hexadecimal == 2 */
+ p = temp + sizeof(temp);
+ do {
+ *--p = _HEX[(guint32) (ui64 & 0xf)];
+ } while (ui64 >>= 4);
+
+ len = (temp + sizeof(temp)) - p;
+ }
+ else if (binary > 0) {
+ int first_bit = MIN(sizeof(temp), rspamd_ffsll(ui64));
+
+ p = temp + sizeof(temp);
+ for (int i = 0; i <= first_bit; i++, ui64 >>= 1) {
+ *--p = '0' + (ui64 & 0x1);
+ }
+
+ len = (temp + sizeof(temp)) - p;
+ }
+
+ /* zero or space padding */
+
+ if (len < width) {
+ width -= len;
+
+ while (width-- > 0 && buf < last) {
+ *buf++ = zero;
+ }
+ }
+
+ /* number safe copy */
+
+ if (buf + len > last) {
+ len = last - buf;
+ }
+
+ return ((gchar *) memcpy(buf, p, len)) + len;
+}
+
+struct rspamd_printf_char_buf {
+ char *begin;
+ char *pos;
+ glong remain;
+};
+
+static glong
+rspamd_printf_append_char(const gchar *buf, glong buflen, gpointer ud)
+{
+ struct rspamd_printf_char_buf *dst = (struct rspamd_printf_char_buf *) ud;
+ glong wr;
+
+ if (dst->remain <= 0) {
+ return dst->remain;
+ }
+
+ wr = MIN(dst->remain, buflen);
+ memcpy(dst->pos, buf, wr);
+ dst->remain -= wr;
+ dst->pos += wr;
+
+ return wr;
+}
+
+static glong
+rspamd_printf_append_file(const gchar *buf, glong buflen, gpointer ud)
+{
+ FILE *dst = (FILE *) ud;
+ if (buflen > 0) {
+ return fwrite(buf, 1, buflen, dst);
+ }
+ else {
+ return 0;
+ }
+}
+
+static glong
+rspamd_printf_append_gstring(const gchar *buf, glong buflen, gpointer ud)
+{
+ GString *dst = (GString *) ud;
+
+ if (buflen > 0) {
+ g_string_append_len(dst, buf, buflen);
+ }
+
+ return buflen;
+}
+
+static glong
+rspamd_printf_append_fstring(const gchar *buf, glong buflen, gpointer ud)
+{
+ rspamd_fstring_t **dst = ud;
+
+ if (buflen > 0) {
+ *dst = rspamd_fstring_append(*dst, buf, buflen);
+ }
+
+ return buflen;
+}
+
+glong rspamd_fprintf(FILE *f, const gchar *fmt, ...)
+{
+ va_list args;
+ glong r;
+
+ va_start(args, fmt);
+ r = rspamd_vprintf_common(rspamd_printf_append_file, f, fmt, args);
+ va_end(args);
+
+ return r;
+}
+
+glong rspamd_printf(const gchar *fmt, ...)
+{
+ va_list args;
+ glong r;
+
+ va_start(args, fmt);
+ r = rspamd_vprintf_common(rspamd_printf_append_file, stdout, fmt, args);
+ va_end(args);
+
+ return r;
+}
+
+glong rspamd_log_fprintf(FILE *f, const gchar *fmt, ...)
+{
+ va_list args;
+ glong r;
+
+ va_start(args, fmt);
+ r = rspamd_vprintf_common(rspamd_printf_append_file, f, fmt, args);
+ va_end(args);
+
+ fflush(f);
+
+ return r;
+}
+
+
+glong rspamd_snprintf(gchar *buf, glong max, const gchar *fmt, ...)
+{
+ gchar *r;
+ va_list args;
+
+ va_start(args, fmt);
+ r = rspamd_vsnprintf(buf, max, fmt, args);
+ va_end(args);
+
+ return (r - buf);
+}
+
+gchar *
+rspamd_vsnprintf(gchar *buf, glong max, const gchar *fmt, va_list args)
+{
+ struct rspamd_printf_char_buf dst;
+
+ dst.begin = buf;
+ dst.pos = dst.begin;
+ dst.remain = max - 1;
+ (void) rspamd_vprintf_common(rspamd_printf_append_char, &dst, fmt, args);
+ *dst.pos = '\0';
+
+ return dst.pos;
+}
+
+glong rspamd_printf_gstring(GString *s, const gchar *fmt, ...)
+{
+ va_list args;
+ glong r;
+
+ va_start(args, fmt);
+ r = rspamd_vprintf_gstring(s, fmt, args);
+ va_end(args);
+
+ return r;
+}
+
+glong rspamd_vprintf_gstring(GString *s, const gchar *fmt, va_list args)
+{
+ return rspamd_vprintf_common(rspamd_printf_append_gstring, s, fmt, args);
+}
+
+glong rspamd_printf_fstring(rspamd_fstring_t **s, const gchar *fmt, ...)
+{
+ va_list args;
+ glong r;
+
+ va_start(args, fmt);
+ r = rspamd_vprintf_fstring(s, fmt, args);
+ va_end(args);
+
+ return r;
+}
+
+glong rspamd_vprintf_fstring(rspamd_fstring_t **s, const gchar *fmt, va_list args)
+{
+ return rspamd_vprintf_common(rspamd_printf_append_fstring, s, fmt, args);
+}
+
+#define RSPAMD_PRINTF_APPEND(buf, len) \
+ do { \
+ RSPAMD_PRINTF_APPEND_BUF(buf, len); \
+ fmt++; \
+ buf_start = fmt; \
+ } while (0)
+
+#define RSPAMD_PRINTF_APPEND_BUF(buf, len) \
+ do { \
+ wr = func((buf), (len), apd); \
+ if (wr < (__typeof(wr)) (len)) { \
+ goto oob; \
+ } \
+ written += wr; \
+ } while (0)
+
+glong rspamd_vprintf_common(rspamd_printf_append_func func,
+ gpointer apd,
+ const gchar *fmt,
+ va_list args)
+{
+ gchar zero, numbuf[G_ASCII_DTOSTR_BUF_SIZE], dtoabuf[32], *p, *last;
+ guchar c;
+ const gchar *buf_start = fmt;
+ gint d;
+ gdouble f;
+ glong written = 0, wr, slen;
+ gint64 i64;
+ guint64 ui64;
+ guint width, sign, hex, humanize, bytes, frac_width, b32, b64;
+ rspamd_fstring_t *v;
+ rspamd_ftok_t *tok;
+ GString *gs;
+ GError *err;
+
+ while (*fmt) {
+
+ /*
+ * "buf < last" means that we could copy at least one character:
+ * the plain character, "%%", "%c", and minus without the checking
+ */
+
+ if (*fmt == '%') {
+
+ /* Append what we have in buf */
+ if (fmt > buf_start) {
+ wr = func(buf_start, fmt - buf_start, apd);
+ if (wr <= 0) {
+ goto oob;
+ }
+ written += wr;
+ }
+
+ i64 = 0;
+ ui64 = 0;
+
+ zero = (gchar) ((*++fmt == '0') ? '0' : ' ');
+ width = 0;
+ sign = 1;
+ hex = 0;
+ b32 = 0;
+ b64 = 0;
+ bytes = 0;
+ humanize = 0;
+ frac_width = 0;
+ slen = -1;
+
+ while (*fmt >= '0' && *fmt <= '9') {
+ width = width * 10 + *fmt++ - '0';
+ }
+
+
+ for (;;) {
+ switch (*fmt) {
+
+ case 'u':
+ sign = 0;
+ fmt++;
+ continue;
+
+ case 'm':
+ fmt++;
+ continue;
+
+ case 'X':
+ hex = 2;
+ sign = 0;
+ fmt++;
+ continue;
+
+ case 'x':
+ hex = 1;
+ sign = 0;
+ fmt++;
+ continue;
+ case 'b':
+ b32 = 1;
+ sign = 0;
+ fmt++;
+ continue;
+ case 'B':
+ b64 = 1;
+ sign = 0;
+ fmt++;
+ continue;
+ case 'H':
+ humanize = 1;
+ bytes = 1;
+ sign = 0;
+ fmt++;
+ continue;
+ case 'h':
+ humanize = 1;
+ sign = 0;
+ fmt++;
+ continue;
+ case '.':
+ fmt++;
+
+ if (*fmt == '*') {
+ d = (gint) va_arg(args, gint);
+ if (G_UNLIKELY(d < 0)) {
+ return 0;
+ }
+ frac_width = (guint) d;
+ fmt++;
+ }
+ else {
+ while (*fmt >= '0' && *fmt <= '9') {
+ frac_width = frac_width * 10 + *fmt++ - '0';
+ }
+ }
+
+ break;
+
+ case '*':
+ d = (gint) va_arg(args, gint);
+ if (G_UNLIKELY(d < 0)) {
+ return 0;
+ }
+ slen = (glong) d;
+ fmt++;
+ continue;
+
+ default:
+ break;
+ }
+
+ break;
+ }
+
+
+ switch (*fmt) {
+
+ case 'V':
+ v = va_arg(args, rspamd_fstring_t *);
+
+ if (v) {
+ slen = v->len;
+
+ if (G_UNLIKELY(width != 0)) {
+ slen = MIN(v->len, width);
+ }
+
+ RSPAMD_PRINTF_APPEND(v->str, slen);
+ }
+ else {
+ RSPAMD_PRINTF_APPEND("(NULL)", 6);
+ }
+
+ continue;
+
+ case 'T':
+ tok = va_arg(args, rspamd_ftok_t *);
+
+ if (tok) {
+ slen = tok->len;
+
+ if (G_UNLIKELY(width != 0)) {
+ slen = MIN(tok->len, width);
+ }
+ RSPAMD_PRINTF_APPEND(tok->begin, slen);
+ }
+ else {
+ RSPAMD_PRINTF_APPEND("(NULL)", 6);
+ }
+ continue;
+
+ case 'v':
+ gs = va_arg(args, GString *);
+
+ if (gs) {
+ slen = gs->len;
+
+ if (G_UNLIKELY(width != 0)) {
+ slen = MIN(gs->len, width);
+ }
+
+ RSPAMD_PRINTF_APPEND(gs->str, slen);
+ }
+ else {
+ RSPAMD_PRINTF_APPEND("(NULL)", 6);
+ }
+
+ continue;
+
+ case 'e':
+ err = va_arg(args, GError *);
+
+ if (err) {
+ p = err->message;
+
+ if (p == NULL) {
+ p = "(NULL)";
+ }
+ }
+ else {
+ p = "unknown error";
+ }
+
+ slen = strlen(p);
+ RSPAMD_PRINTF_APPEND(p, slen);
+
+ continue;
+
+ case 's':
+ p = va_arg(args, gchar *);
+ if (p == NULL) {
+ p = "(NULL)";
+ slen = sizeof("(NULL)") - 1;
+ }
+
+ if (G_UNLIKELY(b32)) {
+ gchar *b32buf;
+
+ if (G_UNLIKELY(slen == -1)) {
+ if (G_LIKELY(width != 0)) {
+ slen = width;
+ }
+ else {
+ /* NULL terminated string */
+ slen = strlen(p);
+ }
+ }
+
+ b32buf = rspamd_encode_base32(p, slen, RSPAMD_BASE32_DEFAULT);
+
+ if (b32buf) {
+ RSPAMD_PRINTF_APPEND(b32buf, strlen(b32buf));
+ g_free(b32buf);
+ }
+ else {
+ RSPAMD_PRINTF_APPEND("(NULL)", sizeof("(NULL)") - 1);
+ }
+ }
+ else if (G_UNLIKELY(hex)) {
+ gchar hexbuf[2];
+
+ if (G_UNLIKELY(slen == -1)) {
+ if (G_LIKELY(width != 0)) {
+ slen = width;
+ }
+ else {
+ /* NULL terminated string */
+ slen = strlen(p);
+ }
+ }
+
+ while (slen) {
+ hexbuf[0] = hex == 2 ? _HEX[(*p >> 4u) & 0xfu] : _hex[(*p >> 4u) & 0xfu];
+ hexbuf[1] = hex == 2 ? _HEX[*p & 0xfu] : _hex[*p & 0xfu];
+ RSPAMD_PRINTF_APPEND_BUF(hexbuf, 2);
+ p++;
+ slen--;
+ }
+
+ fmt++;
+ buf_start = fmt;
+ }
+ else if (G_UNLIKELY(b64)) {
+ gchar *b64buf;
+ gsize olen = 0;
+
+ if (G_UNLIKELY(slen == -1)) {
+ if (G_LIKELY(width != 0)) {
+ slen = width;
+ }
+ else {
+ /* NULL terminated string */
+ slen = strlen(p);
+ }
+ }
+
+ b64buf = rspamd_encode_base64(p, slen, 0, &olen);
+
+ if (b64buf) {
+ RSPAMD_PRINTF_APPEND(b64buf, olen);
+ g_free(b64buf);
+ }
+ else {
+ RSPAMD_PRINTF_APPEND("(NULL)", sizeof("(NULL)") - 1);
+ }
+ }
+ else {
+ if (slen == -1) {
+ /* NULL terminated string */
+ slen = strlen(p);
+ }
+
+ if (G_UNLIKELY(width != 0)) {
+ slen = MIN(slen, width);
+ }
+
+ RSPAMD_PRINTF_APPEND(p, slen);
+ }
+
+ continue;
+
+ case 'O':
+ i64 = (gint64) va_arg(args, off_t);
+ sign = 1;
+ break;
+
+ case 'P':
+ i64 = (gint64) va_arg(args, pid_t);
+ sign = 1;
+ break;
+
+ case 't':
+ i64 = (gint64) va_arg(args, time_t);
+ sign = 1;
+ break;
+
+ case 'z':
+ if (sign) {
+ i64 = (gint64) va_arg(args, ssize_t);
+ }
+ else {
+ ui64 = (guint64) va_arg(args, size_t);
+ }
+ break;
+
+ case 'd':
+ if (sign) {
+ i64 = (gint64) va_arg(args, gint);
+ }
+ else {
+ ui64 = (guint64) va_arg(args, guint);
+ }
+ break;
+
+ case 'l':
+ if (sign) {
+ i64 = (gint64) va_arg(args, glong);
+ }
+ else {
+ ui64 = (guint64) va_arg(args, gulong);
+ }
+ break;
+
+ case 'D':
+ if (sign) {
+ i64 = (gint64) va_arg(args, gint32);
+ }
+ else {
+ ui64 = (guint64) va_arg(args, guint32);
+ }
+ break;
+
+ case 'L':
+ if (sign) {
+ i64 = va_arg(args, gint64);
+ }
+ else {
+ ui64 = va_arg(args, guint64);
+ }
+ break;
+
+
+ case 'f':
+ f = (gdouble) va_arg(args, double);
+ slen = fpconv_dtoa(f, dtoabuf, frac_width, false);
+
+ RSPAMD_PRINTF_APPEND(dtoabuf, slen);
+
+ continue;
+
+ case 'g':
+ f = (gdouble) va_arg(args, double);
+ slen = fpconv_dtoa(f, dtoabuf, 0, true);
+ RSPAMD_PRINTF_APPEND(dtoabuf, slen);
+
+ continue;
+
+ case 'F':
+ f = (gdouble) va_arg(args, long double);
+ slen = fpconv_dtoa(f, dtoabuf, frac_width, false);
+
+ RSPAMD_PRINTF_APPEND(dtoabuf, slen);
+
+ continue;
+
+ case 'G':
+ f = (gdouble) va_arg(args, long double);
+ slen = fpconv_dtoa(f, dtoabuf, 0, true);
+ RSPAMD_PRINTF_APPEND(dtoabuf, slen);
+
+ continue;
+
+ case 'p':
+ ui64 = (uintptr_t) va_arg(args, void *);
+ hex = 2;
+ sign = 0;
+ zero = '0';
+ width = sizeof(void *) * 2;
+ break;
+
+ case 'c':
+ c = va_arg(args, gint);
+ c &= 0xffu;
+ if (G_UNLIKELY(hex)) {
+ gchar hexbuf[2];
+ hexbuf[0] = hex == 2 ? _HEX[(c >> 4u) & 0xfu] : _hex[(c >> 4u) & 0xfu];
+ hexbuf[1] = hex == 2 ? _HEX[c & 0xfu] : _hex[c & 0xfu];
+
+ RSPAMD_PRINTF_APPEND(hexbuf, 2);
+ }
+ else {
+ RSPAMD_PRINTF_APPEND(&c, 1);
+ }
+
+ continue;
+
+ case 'Z':
+ c = '\0';
+ RSPAMD_PRINTF_APPEND(&c, 1);
+
+ continue;
+
+ case 'N':
+ c = '\n';
+ RSPAMD_PRINTF_APPEND(&c, 1);
+
+ continue;
+
+ case '%':
+ c = '%';
+ RSPAMD_PRINTF_APPEND(&c, 1);
+
+ continue;
+
+ default:
+ c = *fmt;
+ RSPAMD_PRINTF_APPEND(&c, 1);
+
+ continue;
+ }
+
+ /* Print number */
+ p = numbuf;
+ last = p + sizeof(numbuf);
+ if (sign) {
+ if (i64 < 0) {
+ *p++ = '-';
+ ui64 = (guint64) -i64;
+ }
+ else {
+ ui64 = (guint64) i64;
+ }
+ }
+
+ if (!humanize) {
+ p = rspamd_sprintf_num(p, last, ui64, zero, hex, b64 + b32, width);
+ }
+ else {
+ p = rspamd_humanize_number(p, last, ui64, bytes);
+ }
+ slen = p - numbuf;
+ RSPAMD_PRINTF_APPEND(numbuf, slen);
+ }
+ else {
+ fmt++;
+ }
+ }
+
+ /* Finish buffer */
+ if (fmt > buf_start) {
+ wr = func(buf_start, fmt - buf_start, apd);
+ if (wr <= 0) {
+ goto oob;
+ }
+ written += wr;
+ }
+
+oob:
+ return written;
+}
diff --git a/src/libutil/printf.h b/src/libutil/printf.h
new file mode 100644
index 0000000..a9420b2
--- /dev/null
+++ b/src/libutil/printf.h
@@ -0,0 +1,96 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef PRINTF_H_
+#define PRINTF_H_
+
+#include "config.h"
+#include "fstring.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * supported formats:
+ * %[0][width][x][X]O off_t
+ * %[0][width]T time_t
+ * %[0][width][u][x|X|h|H|b|B]z ssize_t/size_t
+ * %[0][width][u][x|X|h|H|b|B]d gint/guint
+ * %[0][width][u][x|X|h|H|b|B]l long
+ * %[0][width][u][x|X|h|H|b|B]D gint32/guint32
+ * %[0][width][u][x|X|h|H|b|B]L gint64/guint64
+ * %[0][width][.width]f double
+ * %[0][width][.width]F long double
+ * %[0][width][.width]g double
+ * %[0][width][.width]G long double
+ * %P pid_t
+ * %r rlim_t
+ * %p void *
+ * %V rspamd_fstring_t *
+ * %T rspamd_ftok_t
+ * %v GString *
+ * %s null-terminated string
+ * %xs hex encoded string
+ * %bs base32 encoded string
+ * %Bs base64 encoded string
+ * %*s length and string
+ * %Z '\0'
+ * %N '\n'
+ * %c gchar
+ * %t time_t
+ * %e GError *
+ * %% %
+ *
+ */
+
+/**
+ * Callback used for common printf operations
+ * @param buf buffer to append
+ * @param buflen length of the buffer
+ * @param ud opaque pointer
+ * @return number of characters written
+ */
+typedef glong (*rspamd_printf_append_func)(const gchar *buf, glong buflen,
+ gpointer ud);
+
+glong rspamd_fprintf(FILE *f, const gchar *fmt, ...);
+
+glong rspamd_printf(const gchar *fmt, ...);
+
+glong rspamd_log_fprintf(FILE *f, const gchar *fmt, ...);
+
+glong rspamd_snprintf(gchar *buf, glong max, const gchar *fmt, ...);
+
+gchar *rspamd_vsnprintf(gchar *buf, glong max, const gchar *fmt,
+ va_list args);
+
+glong rspamd_printf_gstring(GString *s, const gchar *fmt, ...);
+
+glong rspamd_vprintf_gstring(GString *s, const gchar *fmt, va_list args);
+
+glong rspamd_printf_fstring(rspamd_fstring_t **s, const gchar *fmt, ...);
+
+glong rspamd_vprintf_fstring(rspamd_fstring_t **s, const gchar *fmt, va_list args);
+
+glong rspamd_vprintf_common(rspamd_printf_append_func func,
+ gpointer apd,
+ const gchar *fmt,
+ va_list args);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* PRINTF_H_ */
diff --git a/src/libutil/radix.c b/src/libutil/radix.c
new file mode 100644
index 0000000..93c728c
--- /dev/null
+++ b/src/libutil/radix.c
@@ -0,0 +1,434 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "radix.h"
+#include "rspamd.h"
+#include "mem_pool.h"
+#include "btrie.h"
+
+#define msg_err_radix(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "radix", tree->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_warn_radix(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "radix", tree->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_info_radix(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "radix", tree->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_debug_radix(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_radix_log_id, "radix", tree->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(radix)
+
+struct radix_tree_compressed {
+ rspamd_mempool_t *pool;
+ struct btrie *tree;
+ const gchar *name;
+ size_t size;
+ guint duplicates;
+ gboolean own_pool;
+};
+
+uintptr_t
+radix_find_compressed(radix_compressed_t *tree, const guint8 *key, gsize keylen)
+{
+ gconstpointer ret;
+
+ g_assert(tree != NULL);
+
+ ret = btrie_lookup(tree->tree, key, keylen * NBBY);
+
+ if (ret == NULL) {
+ return RADIX_NO_VALUE;
+ }
+
+ return (uintptr_t) ret;
+}
+
+
+uintptr_t
+radix_insert_compressed(radix_compressed_t *tree,
+ guint8 *key, gsize keylen,
+ gsize masklen,
+ uintptr_t value)
+{
+ static const guint max_duplicates = 32;
+ guint keybits = keylen * NBBY;
+ uintptr_t old;
+ gchar ip_str[INET6_ADDRSTRLEN + 1];
+ int ret;
+
+ g_assert(tree != NULL);
+ g_assert(keybits >= masklen);
+
+ msg_debug_radix("%s: want insert value %p with mask %z, key: %*xs",
+ tree->name, (gpointer) value, keybits - masklen, (int) keylen, key);
+
+ old = radix_find_compressed(tree, key, keylen);
+
+ ret = btrie_add_prefix(tree->tree, key, keybits - masklen,
+ (gconstpointer) value);
+
+ if (ret != BTRIE_OKAY) {
+ tree->duplicates++;
+
+ if (tree->duplicates == max_duplicates) {
+ msg_err_radix("%s: maximum duplicates limit reached: %d, "
+ "suppress further errors",
+ tree->name, max_duplicates);
+ }
+ else if (tree->duplicates < max_duplicates) {
+ memset(ip_str, 0, sizeof(ip_str));
+
+ if (keybits == 32) {
+ msg_err_radix("%s: cannot insert %p, key: %s/%d, duplicate value",
+ tree->name,
+ (gpointer) value,
+ inet_ntop(AF_INET, key, ip_str, sizeof(ip_str) - 1),
+ (gint) (keybits - masklen));
+ }
+ else if (keybits == 128) {
+ msg_err_radix("%s: cannot insert %p, key: [%s]/%d, duplicate value",
+ tree->name,
+ (gpointer) value,
+ inet_ntop(AF_INET6, key, ip_str, sizeof(ip_str) - 1),
+ (gint) (keybits - masklen));
+ }
+ else {
+ msg_err_radix("%s: cannot insert %p with mask %z, key: %*xs, duplicate value",
+ tree->name,
+ (gpointer) value,
+ keybits - masklen,
+ (int) keylen, key);
+ }
+ }
+ }
+ else {
+ tree->size++;
+ }
+
+ return old;
+}
+
+
+radix_compressed_t *
+radix_create_compressed(const gchar *tree_name)
+{
+ radix_compressed_t *tree;
+
+ tree = g_malloc(sizeof(*tree));
+ if (tree == NULL) {
+ return NULL;
+ }
+
+ tree->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), NULL, 0);
+ tree->size = 0;
+ tree->duplicates = 0;
+ tree->tree = btrie_init(tree->pool);
+ tree->own_pool = TRUE;
+ tree->name = tree_name;
+
+ return tree;
+}
+
+radix_compressed_t *
+radix_create_compressed_with_pool(rspamd_mempool_t *pool, const gchar *tree_name)
+{
+ radix_compressed_t *tree;
+
+ tree = rspamd_mempool_alloc(pool, sizeof(*tree));
+ tree->pool = pool;
+ tree->size = 0;
+ tree->duplicates = 0;
+ tree->tree = btrie_init(tree->pool);
+ tree->own_pool = FALSE;
+ tree->name = tree_name;
+
+ return tree;
+}
+
+void radix_destroy_compressed(radix_compressed_t *tree)
+{
+ if (tree) {
+ if (tree->own_pool) {
+ rspamd_mempool_delete(tree->pool);
+ g_free(tree);
+ }
+ }
+}
+
+uintptr_t
+radix_find_compressed_addr(radix_compressed_t *tree,
+ const rspamd_inet_addr_t *addr)
+{
+ const guchar *key;
+ guint klen = 0;
+ guchar buf[16];
+
+ if (addr == NULL) {
+ return RADIX_NO_VALUE;
+ }
+
+ key = rspamd_inet_address_get_hash_key(addr, &klen);
+
+ if (key && klen) {
+ if (klen == 4) {
+ /* Map to ipv6 */
+ memset(buf, 0, 10);
+ buf[10] = 0xffu;
+ buf[11] = 0xffu;
+ memcpy(buf + 12, key, klen);
+
+ key = buf;
+ klen = sizeof(buf);
+ }
+
+ return radix_find_compressed(tree, key, klen);
+ }
+
+ return RADIX_NO_VALUE;
+}
+
+gint rspamd_radix_add_iplist(const gchar *list, const gchar *separators,
+ radix_compressed_t *tree, gconstpointer value,
+ gboolean resolve, const gchar *tree_name)
+{
+ gchar *token, *ipnet, *err_str, **strv, **cur, *brace;
+ union {
+ struct in_addr ina;
+ struct in6_addr ina6;
+ guchar buf[16];
+ } addr_buf;
+ guint k = G_MAXINT;
+ gint af;
+ gint res = 0, r;
+ struct addrinfo hints, *ai_res, *cur_ai;
+
+ /* Split string if there are multiple items inside a single string */
+ strv = g_strsplit_set(list, separators, 0);
+ cur = strv;
+ while (*cur) {
+ af = AF_UNSPEC;
+ if (**cur == '\0') {
+ cur++;
+ continue;
+ }
+
+ /* Extract ipnet */
+ ipnet = g_strstrip(*cur);
+ token = strsep(&ipnet, "/");
+
+ if (ipnet != NULL) {
+ errno = 0;
+ /* Get mask */
+ k = strtoul(ipnet, &err_str, 10);
+ if (errno != 0) {
+ msg_warn_radix(
+ "%s: invalid netmask, error detected on symbol: %s, error: %s",
+ tree_name,
+ err_str,
+ strerror(errno));
+ k = G_MAXINT;
+ }
+ }
+
+ /* Check IP */
+ if (token[0] == '[') {
+ /* Braced IPv6 */
+ brace = strrchr(token, ']');
+
+ if (brace != NULL) {
+ token++;
+ *brace = '\0';
+
+ if (inet_pton(AF_INET6, token, &addr_buf.ina6) == 1) {
+ af = AF_INET6;
+ }
+ else {
+ msg_warn_radix("invalid IP address: %s", token);
+
+ cur++;
+ continue;
+ }
+ }
+ else {
+ msg_warn_radix("invalid IP address: %s", token);
+
+ cur++;
+ continue;
+ }
+ }
+ else {
+ if (inet_pton(AF_INET, token, &addr_buf.ina) == 1) {
+ af = AF_INET;
+ }
+ else if (inet_pton(AF_INET6, token, &addr_buf.ina6) == 1) {
+ af = AF_INET6;
+ }
+ else {
+
+ if (resolve) {
+ memset(&hints, 0, sizeof(hints));
+ hints.ai_socktype = SOCK_STREAM; /* Type of the socket */
+ hints.ai_flags = AI_NUMERICSERV;
+ hints.ai_family = AF_UNSPEC;
+
+ if ((r = getaddrinfo(token, NULL, &hints, &ai_res)) == 0) {
+ for (cur_ai = ai_res; cur_ai != NULL;
+ cur_ai = cur_ai->ai_next) {
+
+ if (cur_ai->ai_family == AF_INET) {
+ struct sockaddr_in *sin;
+
+ sin = (struct sockaddr_in *) cur_ai->ai_addr;
+ if (k > 32) {
+ k = 32;
+ }
+
+ /* Convert to IPv4 mapped IPv6 */
+ memset(addr_buf.buf, 0, 10);
+ addr_buf.buf[10] = 0xffu;
+ addr_buf.buf[11] = 0xffu;
+ memcpy(addr_buf.buf + 12,
+ &sin->sin_addr, 4);
+
+ k += 96;
+
+ radix_insert_compressed(tree,
+ addr_buf.buf,
+ sizeof(addr_buf.buf),
+ 128 - k, (uintptr_t) value);
+ res++;
+ }
+ else if (cur_ai->ai_family == AF_INET6) {
+ struct sockaddr_in6 *sin6;
+
+ sin6 = (struct sockaddr_in6 *) cur_ai->ai_addr;
+ if (k > 128) {
+ k = 128;
+ }
+
+ memcpy(addr_buf.buf, &sin6->sin6_addr,
+ sizeof(sin6->sin6_addr));
+ radix_insert_compressed(tree,
+ addr_buf.buf,
+ sizeof(addr_buf.buf),
+ 128 - k, (uintptr_t) value);
+ res++;
+ }
+ }
+
+ freeaddrinfo(ai_res);
+ }
+ else {
+ msg_warn_radix("getaddrinfo failed for %s: %s", token,
+ gai_strerror(r));
+ }
+
+ cur++;
+ continue;
+ }
+ else {
+ msg_warn_radix("invalid IP address: %s", token);
+
+ cur++;
+ continue;
+ }
+ }
+ }
+
+ if (af == AF_INET) {
+ if (k > 32) {
+ k = 32;
+ }
+
+ /* Move to the last part of the address */
+ memmove(addr_buf.buf + 12, &addr_buf.ina, 4);
+ memset(addr_buf.buf, 0, 10);
+ addr_buf.buf[10] = 0xffu;
+ addr_buf.buf[11] = 0xffu;
+ k += 96;
+ radix_insert_compressed(tree, addr_buf.buf, sizeof(addr_buf.buf),
+ 128 - k, (uintptr_t) value);
+ res++;
+ }
+ else if (af == AF_INET6) {
+ if (k > 128) {
+ k = 128;
+ }
+
+ radix_insert_compressed(tree, addr_buf.buf, sizeof(addr_buf),
+ 128 - k, (uintptr_t) value);
+ res++;
+ }
+ cur++;
+ }
+
+ g_strfreev(strv);
+
+ return res;
+}
+
+gboolean
+radix_add_generic_iplist(const gchar *ip_list, radix_compressed_t **tree,
+ gboolean resolve, const gchar *tree_name)
+{
+ static const char fill_ptr[] = "1";
+
+ if (*tree == NULL) {
+ *tree = radix_create_compressed(tree_name);
+ }
+
+ return (rspamd_radix_add_iplist(ip_list, ",; ", *tree,
+ fill_ptr, resolve, tree_name) > 0);
+}
+
+
+gsize radix_get_size(radix_compressed_t *tree)
+{
+ if (tree != NULL) {
+ return tree->size;
+ }
+
+ return 0;
+}
+
+
+rspamd_mempool_t *
+radix_get_pool(radix_compressed_t *tree)
+{
+
+ if (tree != NULL) {
+ return tree->pool;
+ }
+
+ return NULL;
+}
+
+const gchar *
+radix_get_info(radix_compressed_t *tree)
+{
+ if (tree == NULL) {
+ return NULL;
+ }
+
+ return btrie_stats(tree->tree, tree->duplicates);
+}
diff --git a/src/libutil/radix.h b/src/libutil/radix.h
new file mode 100644
index 0000000..a85da5b
--- /dev/null
+++ b/src/libutil/radix.h
@@ -0,0 +1,123 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RADIX_H
+#define RADIX_H
+
+#include "config.h"
+#include "mem_pool.h"
+#include "util.h"
+
+#define RADIX_NO_VALUE (uintptr_t) - 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct radix_tree_compressed radix_compressed_t;
+
+/**
+ * Insert new key to the radix trie
+ * @param tree radix trie
+ * @param key key to insert (bitstring)
+ * @param keylen length of the key (in bytes)
+ * @param masklen length of mask that should be applied to the key (in bits)
+ * @param value opaque value pointer
+ * @return previous value of the key or `RADIX_NO_VALUE`
+ */
+uintptr_t
+radix_insert_compressed(radix_compressed_t *tree,
+ guint8 *key, gsize keylen,
+ gsize masklen,
+ uintptr_t value);
+
+/**
+ * Find a key in a radix trie
+ * @param tree radix trie
+ * @param key key to find (bitstring)
+ * @param keylen length of a key
+ * @return opaque pointer or `RADIX_NO_VALUE` if no value has been found
+ */
+uintptr_t radix_find_compressed(radix_compressed_t *tree, const guint8 *key,
+ gsize keylen);
+
+/**
+ * Find specified address in tree (works for IPv4 or IPv6 addresses)
+ * @param tree
+ * @param addr
+ * @return
+ */
+uintptr_t radix_find_compressed_addr(radix_compressed_t *tree,
+ const rspamd_inet_addr_t *addr);
+
+/**
+ * Destroy the complete radix trie
+ * @param tree
+ */
+void radix_destroy_compressed(radix_compressed_t *tree);
+
+/**
+ * Create new radix trie
+ * @return
+ */
+radix_compressed_t *radix_create_compressed(const gchar *tree_name);
+
+radix_compressed_t *radix_create_compressed_with_pool(rspamd_mempool_t *pool, const gchar *tree_name);
+
+/**
+ * Insert list of ip addresses and masks to the radix tree
+ * @param list string line of addresses
+ * @param separators string of characters used as separators
+ * @param tree target tree
+ * @return number of elements inserted
+ */
+gint rspamd_radix_add_iplist(const gchar *list, const gchar *separators,
+ radix_compressed_t *tree, gconstpointer value,
+ gboolean resolve, const gchar *tree_name);
+
+/**
+ * Generic version of @see rspamd_radix_add_iplist. This function creates tree
+ * if `tree` is NULL.
+ */
+gboolean
+radix_add_generic_iplist(const gchar *ip_list,
+ radix_compressed_t **tree,
+ gboolean resolve,
+ const gchar *tree_name);
+
+/**
+ * Returns number of elements in the tree
+ * @param tree
+ * @return
+ */
+gsize radix_get_size(radix_compressed_t *tree);
+
+/**
+ * Return string that describes this radix tree (memory, nodes, compression etc)
+ * @param tree
+ * @return constant string
+ */
+const gchar *radix_get_info(radix_compressed_t *tree);
+
+/**
+ * Returns memory pool associated with the radix tree
+ */
+rspamd_mempool_t *radix_get_pool(radix_compressed_t *tree);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libutil/ref.h b/src/libutil/ref.h
new file mode 100644
index 0000000..2a3fd8d
--- /dev/null
+++ b/src/libutil/ref.h
@@ -0,0 +1,91 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef REF_H_
+#define REF_H_
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+
+/**
+ * @file ref.h
+ * A set of macros to handle refcounts
+ */
+
+typedef void (*ref_dtor_cb_t)(void *data);
+
+typedef struct ref_entry_s {
+ unsigned int refcount;
+ ref_dtor_cb_t dtor;
+} ref_entry_t;
+
+#define REF_INIT(obj, dtor_cb) \
+ do { \
+ if ((obj) != NULL) { \
+ (obj)->ref.refcount = 0; \
+ (obj)->ref.dtor = (ref_dtor_cb_t) (dtor_cb); \
+ } \
+ } while (0)
+
+#define REF_INIT_RETAIN(obj, dtor_cb) \
+ do { \
+ if ((obj) != NULL) { \
+ (obj)->ref.refcount = 1; \
+ (obj)->ref.dtor = (ref_dtor_cb_t) (dtor_cb); \
+ } \
+ } while (0)
+
+#ifdef HAVE_ATOMIC_BUILTINS
+#define REF_RETAIN_ATOMIC(obj) \
+ do { \
+ if ((obj) != NULL) { \
+ __atomic_add_fetch(&(obj)->ref.refcount, 1, __ATOMIC_RELEASE); \
+ } \
+ } while (0)
+
+#define REF_RELEASE_ATOMIC(obj) \
+ do { \
+ if ((obj) != NULL) { \
+ unsigned int _rc_priv = __atomic_sub_fetch(&(obj)->ref.refcount, 1, __ATOMIC_ACQ_REL); \
+ if (_rc_priv == 0 && (obj)->ref.dtor) { \
+ (obj)->ref.dtor(obj); \
+ } \
+ } \
+ } while (0)
+
+#else
+#define REF_RETAIN_ATOMIC REF_RETAIN
+#define REF_RELEASE_ATOMIC REF_RELEASE_ATOMIC
+#endif
+
+#define REF_RETAIN(obj) \
+ do { \
+ if ((obj) != NULL) { \
+ (obj)->ref.refcount++; \
+ } \
+ } while (0)
+
+#define REF_RELEASE(obj) \
+ do { \
+ if ((obj) != NULL) { \
+ if (--(obj)->ref.refcount == 0 && (obj)->ref.dtor) { \
+ (obj)->ref.dtor(obj); \
+ } \
+ } \
+ } while (0)
+
+#endif /* REF_H_ */
diff --git a/src/libutil/regexp.c b/src/libutil/regexp.c
new file mode 100644
index 0000000..9f143ac
--- /dev/null
+++ b/src/libutil/regexp.c
@@ -0,0 +1,1359 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "regexp.h"
+#include "cryptobox.h"
+#include "ref.h"
+#include "util.h"
+#include "rspamd.h"
+#include "contrib/fastutf8/fastutf8.h"
+
+#ifndef WITH_PCRE2
+/* Normal pcre path */
+#include <pcre.h>
+#define PCRE_T pcre
+#define PCRE_EXTRA_T pcre_extra
+#define PCRE_JIT_T pcre_jit_stack
+#define PCRE_FREE pcre_free
+#define PCRE_JIT_STACK_FREE pcre_jit_stack_free
+#define PCRE_FLAG(x) G_PASTE(PCRE_, x)
+#else
+/* PCRE 2 path */
+#ifndef PCRE2_CODE_UNIT_WIDTH
+#define PCRE2_CODE_UNIT_WIDTH 8
+#endif
+
+#include <pcre2.h>
+#define PCRE_T pcre2_code
+#define PCRE_JIT_T pcre2_jit_stack
+#define PCRE_FREE pcre2_code_free
+#define PCRE_JIT_STACK_FREE pcre2_jit_stack_free
+
+#define PCRE_FLAG(x) G_PASTE(PCRE2_, x)
+#endif
+
+typedef guchar regexp_id_t[rspamd_cryptobox_HASHBYTES];
+
+#undef DISABLE_JIT_FAST
+
+struct rspamd_regexp_s {
+ gdouble exec_time;
+ gchar *pattern;
+ PCRE_T *re;
+ PCRE_T *raw_re;
+#ifndef WITH_PCRE2
+ PCRE_EXTRA_T *extra;
+ PCRE_EXTRA_T *raw_extra;
+#else
+ pcre2_match_context *mcontext;
+ pcre2_match_context *raw_mcontext;
+#endif
+ regexp_id_t id;
+ ref_entry_t ref;
+ gpointer ud;
+ gpointer re_class;
+ guint64 cache_id;
+ gsize match_limit;
+ guint max_hits;
+ gint flags;
+ gint pcre_flags;
+ gint ncaptures;
+};
+
+struct rspamd_regexp_cache {
+ GHashTable *tbl;
+#ifdef HAVE_PCRE_JIT
+ PCRE_JIT_T *jstack;
+#endif
+};
+
+static struct rspamd_regexp_cache *global_re_cache = NULL;
+static gboolean can_jit = FALSE;
+static gboolean check_jit = TRUE;
+static const int max_re_cache_size = 8192;
+
+#ifdef WITH_PCRE2
+static pcre2_compile_context *pcre2_ctx = NULL;
+#endif
+
+static GQuark
+rspamd_regexp_quark(void)
+{
+ return g_quark_from_static_string("rspamd-regexp");
+}
+
+static void
+rspamd_regexp_generate_id(const gchar *pattern, const gchar *flags,
+ regexp_id_t out)
+{
+ rspamd_cryptobox_hash_state_t st;
+
+ rspamd_cryptobox_hash_init(&st, NULL, 0);
+
+ if (flags) {
+ rspamd_cryptobox_hash_update(&st, flags, strlen(flags));
+ }
+
+ rspamd_cryptobox_hash_update(&st, pattern, strlen(pattern));
+ rspamd_cryptobox_hash_final(&st, out);
+}
+
+static void
+rspamd_regexp_dtor(rspamd_regexp_t *re)
+{
+ if (re) {
+ if (re->raw_re && re->raw_re != re->re) {
+#ifndef WITH_PCRE2
+ /* PCRE1 version */
+#ifdef HAVE_PCRE_JIT
+ if (re->raw_extra) {
+ pcre_free_study(re->raw_extra);
+ }
+#endif
+#else
+ /* PCRE 2 version */
+ if (re->raw_mcontext) {
+ pcre2_match_context_free(re->raw_mcontext);
+ }
+#endif
+ PCRE_FREE(re->raw_re);
+ }
+
+ if (re->re) {
+#ifndef WITH_PCRE2
+ /* PCRE1 version */
+#ifdef HAVE_PCRE_JIT
+ if (re->extra) {
+ pcre_free_study(re->extra);
+ }
+#endif
+#else
+ /* PCRE 2 version */
+ if (re->mcontext) {
+ pcre2_match_context_free(re->mcontext);
+ }
+#endif
+ PCRE_FREE(re->re);
+ }
+
+ if (re->pattern) {
+ g_free(re->pattern);
+ }
+
+ g_free(re);
+ }
+}
+
+static void
+rspamd_regexp_post_process(rspamd_regexp_t *r)
+{
+ if (global_re_cache == NULL) {
+ rspamd_regexp_library_init(NULL);
+ }
+#if defined(WITH_PCRE2)
+ static const guint max_recursion_depth = 100000, max_backtrack = 1000000;
+
+ /* Create match context */
+ r->mcontext = pcre2_match_context_create(NULL);
+ g_assert(r->mcontext != NULL);
+ pcre2_set_recursion_limit(r->mcontext, max_recursion_depth);
+ pcre2_set_match_limit(r->mcontext, max_backtrack);
+
+ if (r->raw_re && r->re != r->raw_re) {
+ r->raw_mcontext = pcre2_match_context_create(NULL);
+ g_assert(r->raw_mcontext != NULL);
+ pcre2_set_recursion_limit(r->raw_mcontext, max_recursion_depth);
+ pcre2_set_match_limit(r->raw_mcontext, max_backtrack);
+ }
+ else if (r->raw_re) {
+ r->raw_mcontext = r->mcontext;
+ }
+ else {
+ r->raw_mcontext = NULL;
+ }
+
+#ifdef HAVE_PCRE_JIT
+ guint jit_flags = can_jit ? PCRE2_JIT_COMPLETE : 0;
+ gsize jsz;
+ PCRE2_UCHAR errstr[128];
+ int errcode;
+
+ if (can_jit) {
+ if ((errcode = pcre2_jit_compile(r->re, jit_flags)) < 0) {
+ pcre2_get_error_message(errcode, errstr, G_N_ELEMENTS(errstr));
+ msg_err("jit compilation is not supported: %s; pattern: \"%s\"", errstr, r->pattern);
+ r->flags |= RSPAMD_REGEXP_FLAG_DISABLE_JIT;
+ }
+ else {
+ if (!(pcre2_pattern_info(r->re, PCRE2_INFO_JITSIZE, &jsz) >= 0 && jsz > 0)) {
+ msg_err("cannot exec pcre2_pattern_info(PCRE2_INFO_JITSIZE) on \"%s\"", r->pattern);
+ r->flags |= RSPAMD_REGEXP_FLAG_DISABLE_JIT;
+ }
+ }
+ }
+ else {
+ r->flags |= RSPAMD_REGEXP_FLAG_DISABLE_JIT;
+ }
+
+ if (!(r->flags & RSPAMD_REGEXP_FLAG_DISABLE_JIT)) {
+ pcre2_jit_stack_assign(r->mcontext, NULL, global_re_cache->jstack);
+ }
+
+ if (r->raw_re && r->re != r->raw_re && !(r->flags & RSPAMD_REGEXP_FLAG_DISABLE_JIT)) {
+ if ((errcode = pcre2_jit_compile(r->raw_re, jit_flags)) < 0) {
+ pcre2_get_error_message(errcode, errstr, G_N_ELEMENTS(errstr));
+ msg_debug("jit compilation is not supported for raw regexp: %s; pattern: \"%s\"", errstr, r->pattern);
+ r->flags |= RSPAMD_REGEXP_FLAG_DISABLE_JIT;
+ }
+ else {
+ if (!(pcre2_pattern_info(r->raw_re, PCRE2_INFO_JITSIZE, &jsz) >= 0 && jsz > 0)) {
+ msg_err("cannot exec pcre2_pattern_info(PCRE2_INFO_JITSIZE) on \"%s\"", r->pattern);
+ }
+ else if (!(r->flags & RSPAMD_REGEXP_FLAG_DISABLE_JIT)) {
+ g_assert(r->raw_mcontext != NULL);
+ pcre2_jit_stack_assign(r->raw_mcontext, NULL, global_re_cache->jstack);
+ }
+ }
+ }
+#endif
+
+#else
+ const gchar *err_str = "unknown";
+ gboolean try_jit = TRUE, try_raw_jit = TRUE;
+ gint study_flags = 0;
+
+#if defined(HAVE_PCRE_JIT)
+ study_flags |= PCRE_STUDY_JIT_COMPILE;
+#endif
+
+ /* Pcre 1 needs study */
+ if (r->re) {
+ r->extra = pcre_study(r->re, study_flags, &err_str);
+
+ if (r->extra == NULL) {
+ msg_debug("cannot optimize regexp pattern: '%s': %s",
+ r->pattern, err_str);
+ try_jit = FALSE;
+ r->flags |= RSPAMD_REGEXP_FLAG_DISABLE_JIT;
+ }
+ }
+ else {
+ g_assert_not_reached();
+ }
+
+ if (r->raw_re && r->raw_re != r->re) {
+ r->raw_extra = pcre_study(r->re, study_flags, &err_str);
+ }
+ else if (r->raw_re == r->re) {
+ r->raw_extra = r->extra;
+ }
+
+ if (r->raw_extra == NULL) {
+
+ msg_debug("cannot optimize raw regexp pattern: '%s': %s",
+ r->pattern, err_str);
+ try_raw_jit = FALSE;
+ }
+ /* JIT path */
+ if (try_jit) {
+#ifdef HAVE_PCRE_JIT
+ gint jit, n;
+
+ if (can_jit) {
+ jit = 0;
+ n = pcre_fullinfo(r->re, r->extra,
+ PCRE_INFO_JIT, &jit);
+
+ if (n != 0 || jit != 1) {
+ msg_debug("jit compilation of %s is not supported", r->pattern);
+ r->flags |= RSPAMD_REGEXP_FLAG_DISABLE_JIT;
+ }
+ else {
+ pcre_assign_jit_stack(r->extra, NULL, global_re_cache->jstack);
+ }
+ }
+#endif
+ }
+ else {
+ msg_debug("cannot optimize regexp pattern: '%s': %s",
+ r->pattern, err_str);
+ r->flags |= RSPAMD_REGEXP_FLAG_DISABLE_JIT;
+ }
+
+ if (try_raw_jit) {
+#ifdef HAVE_PCRE_JIT
+ gint jit, n;
+
+ if (can_jit) {
+
+ if (r->raw_re != r->re) {
+ jit = 0;
+ n = pcre_fullinfo(r->raw_re, r->raw_extra,
+ PCRE_INFO_JIT, &jit);
+
+ if (n != 0 || jit != 1) {
+ msg_debug("jit compilation of %s is not supported", r->pattern);
+ r->flags |= RSPAMD_REGEXP_FLAG_DISABLE_JIT;
+ }
+ else {
+ pcre_assign_jit_stack(r->raw_extra, NULL,
+ global_re_cache->jstack);
+ }
+ }
+ }
+#endif
+ }
+#endif /* WITH_PCRE2 */
+}
+
+rspamd_regexp_t *
+rspamd_regexp_new_len(const gchar *pattern, gsize len, const gchar *flags,
+ GError **err)
+{
+ const gchar *start = pattern, *end = start + len, *flags_str = NULL, *flags_end = NULL;
+ gchar *err_str;
+ rspamd_regexp_t *res;
+ gboolean explicit_utf = FALSE;
+ PCRE_T *r;
+ gchar sep = 0, *real_pattern;
+#ifndef WITH_PCRE2
+ gint err_off;
+#else
+ gsize err_off;
+#endif
+ gint regexp_flags = 0, rspamd_flags = 0, err_code, ncaptures;
+ gboolean strict_flags = FALSE;
+
+ rspamd_regexp_library_init(NULL);
+
+ if (pattern == NULL) {
+ g_set_error(err, rspamd_regexp_quark(), EINVAL,
+ "cannot create regexp from a NULL pattern");
+ return NULL;
+ }
+
+ if (flags == NULL && start + 1 < end) {
+ /* We need to parse pattern and detect flags set */
+ if (*start == '/') {
+ sep = '/';
+ }
+ else if (*start == 'm' && start[1] != '\\' && g_ascii_ispunct(start[1])) {
+ start++;
+ sep = *start;
+
+ /* Paired braces */
+ if (sep == '{') {
+ sep = '}';
+ }
+
+ rspamd_flags |= RSPAMD_REGEXP_FLAG_FULL_MATCH;
+ }
+ if (sep == 0) {
+ /* We have no flags, no separators and just use all line as expr */
+ start = pattern;
+ rspamd_flags &= ~RSPAMD_REGEXP_FLAG_FULL_MATCH;
+ }
+ else {
+ gchar *last_sep = rspamd_memrchr(pattern, sep, len);
+
+ if (last_sep == NULL || last_sep <= start) {
+ g_set_error(err, rspamd_regexp_quark(), EINVAL,
+ "pattern is not enclosed with %c: %s",
+ sep, pattern);
+ return NULL;
+ }
+ flags_str = last_sep + 1;
+ flags_end = end;
+ end = last_sep;
+ start++;
+ }
+ }
+ else {
+ /* Strictly check all flags */
+ strict_flags = TRUE;
+ start = pattern;
+ flags_str = flags;
+ if (flags) {
+ flags_end = flags + strlen(flags);
+ }
+ }
+
+ rspamd_flags |= RSPAMD_REGEXP_FLAG_RAW;
+
+#ifndef WITH_PCRE2
+ regexp_flags &= ~PCRE_FLAG(UTF8);
+ regexp_flags |= PCRE_FLAG(NEWLINE_ANYCRLF);
+#else
+ regexp_flags &= ~PCRE_FLAG(UTF);
+#endif
+
+ if (flags_str != NULL) {
+ while (flags_str < flags_end) {
+ switch (*flags_str) {
+ case 'i':
+ regexp_flags |= PCRE_FLAG(CASELESS);
+ break;
+ case 'm':
+ regexp_flags |= PCRE_FLAG(MULTILINE);
+ break;
+ case 's':
+ regexp_flags |= PCRE_FLAG(DOTALL);
+ break;
+ case 'x':
+ regexp_flags |= PCRE_FLAG(EXTENDED);
+ break;
+ case 'u':
+ rspamd_flags &= ~RSPAMD_REGEXP_FLAG_RAW;
+ rspamd_flags |= RSPAMD_REGEXP_FLAG_UTF;
+#ifndef WITH_PCRE2
+ regexp_flags |= PCRE_FLAG(UTF8);
+#else
+ regexp_flags |= PCRE_FLAG(UTF);
+#endif
+ explicit_utf = TRUE;
+ break;
+ case 'O':
+ /* We optimize all regexps by default */
+ rspamd_flags |= RSPAMD_REGEXP_FLAG_NOOPT;
+ break;
+ case 'L':
+ /* SOM_LEFTMOST hyperscan flag */
+ rspamd_flags |= RSPAMD_REGEXP_FLAG_LEFTMOST;
+ break;
+ case 'r':
+ rspamd_flags |= RSPAMD_REGEXP_FLAG_RAW;
+ rspamd_flags &= ~RSPAMD_REGEXP_FLAG_UTF;
+#ifndef WITH_PCRE2
+ regexp_flags &= ~PCRE_FLAG(UTF8);
+#else
+ regexp_flags &= ~PCRE_FLAG(UTF);
+#endif
+ break;
+ default:
+ if (strict_flags) {
+ g_set_error(err, rspamd_regexp_quark(), EINVAL,
+ "invalid regexp flag: %c in pattern %s",
+ *flags_str, pattern);
+ return NULL;
+ }
+ msg_warn("invalid flag '%c' in pattern %s", *flags_str, pattern);
+ goto fin;
+ break;
+ }
+ flags_str++;
+ }
+ }
+fin:
+
+ real_pattern = g_malloc(end - start + 1);
+ rspamd_strlcpy(real_pattern, start, end - start + 1);
+
+#ifndef WITH_PCRE2
+ r = pcre_compile(real_pattern, regexp_flags,
+ (const char **) &err_str, &err_off, NULL);
+ (void) err_code;
+#else
+ r = pcre2_compile(real_pattern, PCRE2_ZERO_TERMINATED,
+ regexp_flags,
+ &err_code, &err_off, pcre2_ctx);
+
+ if (r == NULL) {
+ err_str = g_alloca(1024);
+ memset(err_str, 0, 1024);
+ pcre2_get_error_message(err_code, err_str, 1024);
+ }
+#endif
+
+ if (r == NULL) {
+ g_set_error(err, rspamd_regexp_quark(), EINVAL,
+ "regexp parsing error: '%s' at position %d; pattern: %s",
+ err_str, (gint) err_off, real_pattern);
+ g_free(real_pattern);
+
+ return NULL;
+ }
+
+ /* Now allocate the target structure */
+ res = g_malloc0(sizeof(*res));
+ REF_INIT_RETAIN(res, rspamd_regexp_dtor);
+ res->flags = rspamd_flags;
+ res->pattern = real_pattern;
+ res->cache_id = RSPAMD_INVALID_ID;
+ res->pcre_flags = regexp_flags;
+ res->max_hits = 0;
+ res->re = r;
+
+ if (rspamd_flags & RSPAMD_REGEXP_FLAG_RAW) {
+ res->raw_re = r;
+ }
+ else if (!explicit_utf) {
+#ifndef WITH_PCRE2
+ res->raw_re = pcre_compile(real_pattern, regexp_flags & ~PCRE_FLAG(UTF8),
+ (const char **) &err_str, &err_off, NULL);
+ (void) err_code;
+#else
+ res->raw_re = pcre2_compile(real_pattern, PCRE2_ZERO_TERMINATED,
+ regexp_flags & ~PCRE_FLAG(UTF),
+ &err_code, &err_off, pcre2_ctx);
+ if (res->raw_re == NULL) {
+ err_str = g_alloca(1024);
+ memset(err_str, 0, 1024);
+ pcre2_get_error_message(err_code, err_str, 1024);
+ }
+#endif
+ if (res->raw_re == NULL) {
+ msg_warn("raw regexp parsing error: '%s': '%s' at position %d",
+ err_str, real_pattern, (gint) err_off);
+ }
+ }
+
+ rspamd_regexp_post_process(res);
+ rspamd_regexp_generate_id(pattern, flags, res->id);
+
+#ifndef WITH_PCRE2
+ /* Check number of captures */
+ if (pcre_fullinfo(res->raw_re, res->extra, PCRE_INFO_CAPTURECOUNT,
+ &ncaptures) == 0) {
+ res->ncaptures = ncaptures;
+ }
+#else
+ /* Check number of captures */
+ if (pcre2_pattern_info(res->raw_re, PCRE2_INFO_CAPTURECOUNT,
+ &ncaptures) == 0) {
+ res->ncaptures = ncaptures;
+ }
+#endif
+
+ return res;
+}
+
+rspamd_regexp_t *
+rspamd_regexp_new(const gchar *pattern, const gchar *flags,
+ GError **err)
+{
+ return rspamd_regexp_new_len(pattern, strlen(pattern), flags, err);
+}
+
+#ifndef WITH_PCRE2
+gboolean
+rspamd_regexp_search(const rspamd_regexp_t *re, const gchar *text, gsize len,
+ const gchar **start, const gchar **end, gboolean raw,
+ GArray *captures)
+{
+ pcre *r;
+ pcre_extra *ext;
+#if defined(HAVE_PCRE_JIT) && defined(HAVE_PCRE_JIT_FAST) && !defined(DISABLE_JIT_FAST)
+ pcre_jit_stack *st = NULL;
+#endif
+ const gchar *mt;
+ gsize remain = 0;
+ gint rc, match_flags = 0, *ovec, ncaptures, i;
+ const int junk = 0xdeadbabe;
+
+ g_assert(re != NULL);
+ g_assert(text != NULL);
+
+ if (len == 0) {
+ len = strlen(text);
+ }
+
+ if (re->match_limit > 0 && len > re->match_limit) {
+ len = re->match_limit;
+ }
+
+ if (end != NULL && *end != NULL) {
+ /* Incremental search */
+ mt = (*end);
+
+ if ((gint) len > (mt - text)) {
+ remain = len - (mt - text);
+ }
+ }
+ else {
+ mt = text;
+ remain = len;
+ }
+
+ if (remain == 0) {
+ return FALSE;
+ }
+
+ match_flags = PCRE_NEWLINE_ANYCRLF;
+
+ if ((re->flags & RSPAMD_REGEXP_FLAG_RAW) || raw) {
+ r = re->raw_re;
+ ext = re->raw_extra;
+#if defined(HAVE_PCRE_JIT) && defined(HAVE_PCRE_JIT_FAST) && !defined(DISABLE_JIT_FAST)
+ st = global_re_cache->jstack;
+#endif
+ }
+ else {
+ r = re->re;
+ ext = re->extra;
+#if defined(HAVE_PCRE_JIT) && defined(HAVE_PCRE_JIT_FAST) && !defined(DISABLE_JIT_FAST)
+ if (rspamd_fast_utf8_validate(mt, remain) == 0) {
+ st = global_re_cache->jstack;
+ }
+ else {
+ msg_err("bad utf8 input for JIT re '%s'", re->pattern);
+ return FALSE;
+ }
+#endif
+ }
+
+ if (r == NULL) {
+ /* Invalid regexp type for the specified input */
+ return FALSE;
+ }
+
+ ncaptures = (re->ncaptures + 1) * 3;
+ ovec = g_alloca(sizeof(gint) * ncaptures);
+
+
+ for (i = 0; i < ncaptures; i++) {
+ ovec[i] = junk;
+ }
+
+ if (!(re->flags & RSPAMD_REGEXP_FLAG_NOOPT)) {
+#ifdef HAVE_PCRE_JIT
+#if defined(HAVE_PCRE_JIT_FAST) && !defined(DISABLE_JIT_FAST)
+ /* XXX: flags seems to be broken with jit fast path */
+ g_assert(remain > 0);
+ g_assert(mt != NULL);
+
+ if (st != NULL && !(re->flags & RSPAMD_REGEXP_FLAG_DISABLE_JIT) && can_jit) {
+ rc = pcre_jit_exec(r, ext, mt, remain, 0, 0, ovec,
+ ncaptures, st);
+ }
+ else {
+ rc = pcre_exec(r, ext, mt, remain, 0, match_flags, ovec,
+ ncaptures);
+ }
+#else
+ rc = pcre_exec(r, ext, mt, remain, 0, match_flags, ovec,
+ ncaptures);
+#endif
+#else
+ rc = pcre_exec(r, ext, mt, remain, 0, match_flags, ovec,
+ ncaptures);
+#endif
+ }
+ else {
+ rc = pcre_exec(r, ext, mt, remain, 0, match_flags, ovec,
+ ncaptures);
+ }
+
+ if (rc >= 0) {
+ if (rc > 0) {
+ if (start) {
+ *start = mt + ovec[0];
+ }
+ if (end) {
+ *end = mt + ovec[1];
+ }
+ }
+ else {
+ if (start) {
+ *start = mt;
+ }
+ if (end) {
+ *end = mt + remain;
+ }
+ }
+
+ if (captures != NULL && rc >= 1) {
+ struct rspamd_re_capture *elt;
+
+ g_assert(g_array_get_element_size(captures) ==
+ sizeof(struct rspamd_re_capture));
+ g_array_set_size(captures, rc);
+
+ for (i = 0; i < rc; i++) {
+ if (ovec[i * 2] != junk && ovec[i * 2] >= 0) {
+ elt = &g_array_index(captures, struct rspamd_re_capture, i);
+ elt->p = mt + ovec[i * 2];
+ elt->len = (mt + ovec[i * 2 + 1]) - elt->p;
+ }
+ else {
+ /* Runtime match returned fewer captures than expected */
+ g_array_set_size(captures, i);
+ break;
+ }
+ }
+ }
+
+ if (re->flags & RSPAMD_REGEXP_FLAG_FULL_MATCH) {
+ /* We also ensure that the match is full */
+ if (ovec[0] != 0 || (guint) ovec[1] < len) {
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+#else
+/* PCRE 2 version */
+gboolean
+rspamd_regexp_search(const rspamd_regexp_t *re, const gchar *text, gsize len,
+ const gchar **start, const gchar **end, gboolean raw,
+ GArray *captures)
+{
+ pcre2_match_data *match_data;
+ pcre2_match_context *mcontext;
+ PCRE_T *r;
+ const gchar *mt;
+ PCRE2_SIZE remain = 0, *ovec;
+ const PCRE2_SIZE junk = 0xdeadbabeeeeeeeeULL;
+ gint rc, match_flags, novec, i;
+ gboolean ret = FALSE;
+
+ g_assert(re != NULL);
+ g_assert(text != NULL);
+
+ if (len == 0) {
+ len = strlen(text);
+ }
+
+ if (re->match_limit > 0 && len > re->match_limit) {
+ len = re->match_limit;
+ }
+
+ if (end != NULL && *end != NULL) {
+ /* Incremental search */
+ mt = (*end);
+
+ if ((gint) len > (mt - text)) {
+ remain = len - (mt - text);
+ }
+ }
+ else {
+ mt = text;
+ remain = len;
+ }
+
+ if (remain == 0) {
+ return FALSE;
+ }
+
+ match_flags = 0;
+
+ if (raw || re->re == re->raw_re) {
+ r = re->raw_re;
+ mcontext = re->raw_mcontext;
+ }
+ else {
+ r = re->re;
+ mcontext = re->mcontext;
+ }
+
+ if (r == NULL) {
+ /* Invalid regexp type for the specified input */
+ return FALSE;
+ }
+
+ match_data = pcre2_match_data_create(re->ncaptures + 1, NULL);
+ novec = pcre2_get_ovector_count(match_data);
+ ovec = pcre2_get_ovector_pointer(match_data);
+
+ /* Fill ovec with crap, so we can stop if actual matches is less than announced */
+ for (i = 0; i < novec; i++) {
+ ovec[i * 2] = junk;
+ ovec[i * 2 + 1] = junk;
+ }
+
+#ifdef HAVE_PCRE_JIT
+ if (!(re->flags & RSPAMD_REGEXP_FLAG_DISABLE_JIT) && can_jit) {
+ if (re->re != re->raw_re && rspamd_fast_utf8_validate(mt, remain) != 0) {
+ msg_err("bad utf8 input for JIT re '%s'", re->pattern);
+ return FALSE;
+ }
+
+ rc = pcre2_jit_match(r, mt, remain, 0, match_flags, match_data,
+ mcontext);
+ }
+ else {
+ rc = pcre2_match(r, mt, remain, 0, match_flags, match_data,
+ mcontext);
+ }
+#else
+ rc = pcre2_match(r, mt, remain, 0, match_flags, match_data,
+ mcontext);
+#endif
+
+ if (rc >= 0) {
+ if (novec > 0) {
+ if (start) {
+ *start = mt + ovec[0];
+ }
+ if (end) {
+ *end = mt + ovec[1];
+ }
+ }
+ else {
+ if (start) {
+ *start = mt;
+ }
+ if (end) {
+ *end = mt + remain;
+ }
+ }
+
+ if (captures != NULL && novec >= 1) {
+ struct rspamd_re_capture *elt;
+
+ g_assert(g_array_get_element_size(captures) ==
+ sizeof(struct rspamd_re_capture));
+ g_array_set_size(captures, novec);
+
+ for (i = 0; i < novec; i++) {
+ if (ovec[i * 2] != junk && ovec[i * 2] != PCRE2_UNSET) {
+ elt = &g_array_index(captures, struct rspamd_re_capture, i);
+ elt->p = mt + ovec[i * 2];
+ elt->len = (mt + ovec[i * 2 + 1]) - elt->p;
+ }
+ else {
+ g_array_set_size(captures, i);
+ break;
+ }
+ }
+ }
+
+ ret = TRUE;
+
+ if (re->flags & RSPAMD_REGEXP_FLAG_FULL_MATCH) {
+ /* We also ensure that the match is full */
+ if (ovec[0] != 0 || (guint) ovec[1] < len) {
+ ret = FALSE;
+ }
+ }
+ }
+
+ pcre2_match_data_free(match_data);
+
+ return ret;
+}
+#endif
+
+const char *
+rspamd_regexp_get_pattern(const rspamd_regexp_t *re)
+{
+ g_assert(re != NULL);
+
+ return re->pattern;
+}
+
+guint rspamd_regexp_set_flags(rspamd_regexp_t *re, guint new_flags)
+{
+ guint old_flags;
+
+ g_assert(re != NULL);
+ old_flags = re->flags;
+ re->flags = new_flags;
+
+ return old_flags;
+}
+
+guint rspamd_regexp_get_flags(const rspamd_regexp_t *re)
+{
+ g_assert(re != NULL);
+
+ return re->flags;
+}
+
+guint rspamd_regexp_get_pcre_flags(const rspamd_regexp_t *re)
+{
+ g_assert(re != NULL);
+
+ return re->pcre_flags;
+}
+
+guint rspamd_regexp_get_maxhits(const rspamd_regexp_t *re)
+{
+ g_assert(re != NULL);
+
+ return re->max_hits;
+}
+
+guint rspamd_regexp_set_maxhits(rspamd_regexp_t *re, guint new_maxhits)
+{
+ guint old_hits;
+
+ g_assert(re != NULL);
+ old_hits = re->max_hits;
+ re->max_hits = new_maxhits;
+
+ return old_hits;
+}
+
+guint64
+rspamd_regexp_get_cache_id(const rspamd_regexp_t *re)
+{
+ g_assert(re != NULL);
+
+ return re->cache_id;
+}
+
+guint64
+rspamd_regexp_set_cache_id(rspamd_regexp_t *re, guint64 id)
+{
+ guint64 old;
+
+ g_assert(re != NULL);
+ old = re->cache_id;
+ re->cache_id = id;
+
+ return old;
+}
+
+gsize rspamd_regexp_get_match_limit(const rspamd_regexp_t *re)
+{
+ g_assert(re != NULL);
+
+ return re->match_limit;
+}
+
+gsize rspamd_regexp_set_match_limit(rspamd_regexp_t *re, gsize lim)
+{
+ gsize old;
+
+ g_assert(re != NULL);
+ old = re->match_limit;
+ re->match_limit = lim;
+
+ return old;
+}
+
+gboolean
+rspamd_regexp_match(const rspamd_regexp_t *re, const gchar *text, gsize len,
+ gboolean raw)
+{
+ const gchar *start = NULL, *end = NULL;
+
+ g_assert(re != NULL);
+ g_assert(text != NULL);
+
+ if (len == 0) {
+ len = strlen(text);
+ }
+
+ if (rspamd_regexp_search(re, text, len, &start, &end, raw, NULL)) {
+ if (start == text && end == text + len) {
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+void rspamd_regexp_unref(rspamd_regexp_t *re)
+{
+ REF_RELEASE(re);
+}
+
+rspamd_regexp_t *
+rspamd_regexp_ref(rspamd_regexp_t *re)
+{
+ g_assert(re != NULL);
+
+ REF_RETAIN(re);
+
+ return re;
+}
+
+void rspamd_regexp_set_ud(rspamd_regexp_t *re, gpointer ud)
+{
+ g_assert(re != NULL);
+
+ re->ud = ud;
+}
+
+gpointer
+rspamd_regexp_get_ud(const rspamd_regexp_t *re)
+{
+ g_assert(re != NULL);
+
+ return re->ud;
+}
+
+gboolean
+rspamd_regexp_equal(gconstpointer a, gconstpointer b)
+{
+ const guchar *ia = a, *ib = b;
+
+ return (memcmp(ia, ib, sizeof(regexp_id_t)) == 0);
+}
+
+guint32
+rspamd_regexp_hash(gconstpointer a)
+{
+ const guchar *ia = a;
+ guint32 res;
+
+ memcpy(&res, ia, sizeof(res));
+
+ return res;
+}
+
+gboolean
+rspamd_regexp_cmp(gconstpointer a, gconstpointer b)
+{
+ const guchar *ia = a, *ib = b;
+
+ return memcmp(ia, ib, sizeof(regexp_id_t));
+}
+
+struct rspamd_regexp_cache *
+rspamd_regexp_cache_new(void)
+{
+ struct rspamd_regexp_cache *ncache;
+
+ ncache = g_malloc0(sizeof(*ncache));
+ ncache->tbl = g_hash_table_new_full(rspamd_regexp_hash, rspamd_regexp_equal,
+ NULL, (GDestroyNotify) rspamd_regexp_unref);
+#ifdef HAVE_PCRE_JIT
+#ifdef WITH_PCRE2
+ ncache->jstack = pcre2_jit_stack_create(32 * 1024, 1024 * 1024, NULL);
+#else
+ ncache->jstack = pcre_jit_stack_alloc(32 * 1024, 1024 * 1024);
+#endif
+#endif
+ return ncache;
+}
+
+
+rspamd_regexp_t *
+rspamd_regexp_cache_query(struct rspamd_regexp_cache *cache,
+ const gchar *pattern,
+ const gchar *flags)
+{
+ rspamd_regexp_t *res = NULL;
+ regexp_id_t id;
+
+ if (cache == NULL) {
+ rspamd_regexp_library_init(NULL);
+ cache = global_re_cache;
+ }
+
+ g_assert(cache != NULL);
+ rspamd_regexp_generate_id(pattern, flags, id);
+
+ res = g_hash_table_lookup(cache->tbl, id);
+
+ return res;
+}
+
+
+rspamd_regexp_t *
+rspamd_regexp_cache_create(struct rspamd_regexp_cache *cache,
+ const gchar *pattern,
+ const gchar *flags, GError **err)
+{
+ rspamd_regexp_t *res;
+
+ if (cache == NULL) {
+ rspamd_regexp_library_init(NULL);
+ cache = global_re_cache;
+ }
+
+ g_assert(cache != NULL);
+ res = rspamd_regexp_cache_query(cache, pattern, flags);
+
+ if (res != NULL) {
+ return res;
+ }
+
+ res = rspamd_regexp_new(pattern, flags, err);
+
+ if (res) {
+ /* REF_RETAIN (res); */
+ if (g_hash_table_size(cache->tbl) < max_re_cache_size) {
+ g_hash_table_insert(cache->tbl, res->id, res);
+ }
+ else {
+ msg_warn("cannot insert regexp to the cache: maximum size is reached (%d expressions); "
+ "it might be cached regexp misuse; regexp pattern: %s",
+ max_re_cache_size, pattern);
+ }
+ }
+
+ return res;
+}
+
+gboolean
+rspamd_regexp_cache_remove(struct rspamd_regexp_cache *cache,
+ rspamd_regexp_t *re)
+{
+ if (cache == NULL) {
+ cache = global_re_cache;
+ }
+
+ g_assert(cache != NULL);
+ g_assert(re != NULL);
+
+ return g_hash_table_remove(cache->tbl, re->id);
+}
+
+void rspamd_regexp_cache_destroy(struct rspamd_regexp_cache *cache)
+{
+ if (cache != NULL) {
+ g_hash_table_destroy(cache->tbl);
+#ifdef HAVE_PCRE_JIT
+#ifdef WITH_PCRE2
+ if (cache->jstack) {
+ pcre2_jit_stack_free(cache->jstack);
+ }
+#else
+ if (cache->jstack) {
+ pcre_jit_stack_free(cache->jstack);
+ }
+#endif
+#endif
+ g_free(cache);
+ }
+}
+
+RSPAMD_CONSTRUCTOR(rspamd_re_static_pool_ctor)
+{
+ global_re_cache = rspamd_regexp_cache_new();
+#ifdef WITH_PCRE2
+ pcre2_ctx = pcre2_compile_context_create(NULL);
+ pcre2_set_newline(pcre2_ctx, PCRE_FLAG(NEWLINE_ANY));
+#endif
+}
+
+RSPAMD_DESTRUCTOR(rspamd_re_static_pool_dtor)
+{
+ rspamd_regexp_cache_destroy(global_re_cache);
+#ifdef WITH_PCRE2
+ pcre2_compile_context_free(pcre2_ctx);
+#endif
+}
+
+
+void rspamd_regexp_library_init(struct rspamd_config *cfg)
+{
+ if (cfg) {
+ if (cfg->disable_pcre_jit) {
+ can_jit = FALSE;
+ check_jit = FALSE;
+ }
+ else if (!can_jit) {
+ check_jit = TRUE;
+ }
+ }
+
+ if (check_jit) {
+#ifdef HAVE_PCRE_JIT
+ gint jit, rc;
+ gchar *str;
+
+#ifndef WITH_PCRE2
+ rc = pcre_config(PCRE_CONFIG_JIT, &jit);
+#else
+ rc = pcre2_config(PCRE2_CONFIG_JIT, &jit);
+#endif
+
+ if (rc == 0 && jit == 1) {
+#ifndef WITH_PCRE2
+#ifdef PCRE_CONFIG_JITTARGET
+ pcre_config(PCRE_CONFIG_JITTARGET, &str);
+ msg_info("pcre is compiled with JIT for %s", str);
+#else
+ msg_info("pcre is compiled with JIT for unknown target");
+#endif
+#else
+ rc = pcre2_config(PCRE2_CONFIG_JITTARGET, NULL);
+
+ if (rc > 0) {
+ str = g_alloca(rc);
+ pcre2_config(PCRE2_CONFIG_JITTARGET, str);
+ msg_info("pcre2 is compiled with JIT for %s", str);
+ }
+ else {
+ msg_info("pcre2 is compiled with JIT for unknown");
+ }
+
+#endif /* WITH_PCRE2 */
+
+ if (getenv("VALGRIND") == NULL) {
+ can_jit = TRUE;
+ }
+ else {
+ msg_info("disabling PCRE jit as it does not play well with valgrind");
+ can_jit = FALSE;
+ }
+ }
+ else {
+ msg_info("pcre is compiled without JIT support, so many optimizations"
+ " are impossible");
+ can_jit = FALSE;
+ }
+#else
+ msg_info("pcre is too old and has no JIT support, so many optimizations"
+ " are impossible");
+ can_jit = FALSE;
+#endif
+ check_jit = FALSE;
+ }
+}
+
+gpointer
+rspamd_regexp_get_id(const rspamd_regexp_t *re)
+{
+ g_assert(re != NULL);
+
+ return (gpointer) re->id;
+}
+
+gpointer
+rspamd_regexp_get_class(const rspamd_regexp_t *re)
+{
+ g_assert(re != NULL);
+
+ return re->re_class;
+}
+
+gpointer
+rspamd_regexp_set_class(rspamd_regexp_t *re, gpointer re_class)
+{
+ gpointer old_class;
+
+ g_assert(re != NULL);
+
+ old_class = re->re_class;
+ re->re_class = re_class;
+
+ return old_class;
+}
+
+rspamd_regexp_t *
+rspamd_regexp_from_glob(const gchar *gl, gsize sz, GError **err)
+{
+ GString *out;
+ rspamd_regexp_t *re;
+ const gchar *end;
+ gboolean escaping = FALSE;
+ gint nbraces = 0;
+
+ g_assert(gl != NULL);
+
+ if (sz == 0) {
+ sz = strlen(gl);
+ }
+
+ end = gl + sz;
+ out = g_string_sized_new(sz + 2);
+ g_string_append_c(out, '^');
+
+ while (gl < end) {
+ switch (*gl) {
+ case '*':
+ if (escaping) {
+ g_string_append(out, "\\*");
+ }
+ else {
+ g_string_append(out, ".*");
+ }
+
+ escaping = FALSE;
+ break;
+ case '?':
+ if (escaping) {
+ g_string_append(out, "\\?");
+ }
+ else {
+ g_string_append(out, ".");
+ }
+
+ escaping = FALSE;
+ break;
+ case '.':
+ case '(':
+ case ')':
+ case '+':
+ case '|':
+ case '^':
+ case '$':
+ case '@':
+ case '%':
+ g_string_append_c(out, '\\');
+ g_string_append_c(out, *gl);
+ escaping = FALSE;
+ break;
+ case '\\':
+ if (escaping) {
+ g_string_append(out, "\\\\");
+ escaping = FALSE;
+ }
+ else {
+ escaping = TRUE;
+ }
+ break;
+ case '{':
+ if (escaping) {
+ g_string_append(out, "\\{");
+ }
+ else {
+ g_string_append_c(out, '(');
+ nbraces++;
+ }
+
+ escaping = FALSE;
+ break;
+ case '}':
+ if (nbraces > 0 && !escaping) {
+ g_string_append_c(out, ')');
+ nbraces--;
+ }
+ else if (escaping) {
+ g_string_append(out, "\\}");
+ }
+ else {
+ g_string_append(out, "}");
+ }
+
+ escaping = FALSE;
+ break;
+ case ',':
+ if (nbraces > 0 && !escaping) {
+ g_string_append_c(out, '|');
+ }
+ else if (escaping) {
+ g_string_append(out, "\\,");
+ }
+ else {
+ g_string_append_c(out, ',');
+ }
+
+ break;
+ default:
+ escaping = FALSE;
+ g_string_append_c(out, *gl);
+ break;
+ }
+
+ gl++;
+ }
+
+ g_string_append_c(out, '$');
+ re = rspamd_regexp_new(out->str, "i", err);
+ g_string_free(out, TRUE);
+
+ return re;
+}
diff --git a/src/libutil/regexp.h b/src/libutil/regexp.h
new file mode 100644
index 0000000..6222ba6
--- /dev/null
+++ b/src/libutil/regexp.h
@@ -0,0 +1,276 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef REGEXP_H_
+#define REGEXP_H_
+
+#include "config.h"
+
+#ifndef WITH_PCRE2
+#define PCRE_FLAG(x) G_PASTE(PCRE_, x)
+#else
+#ifndef PCRE2_CODE_UNIT_WIDTH
+#define PCRE2_CODE_UNIT_WIDTH 8
+#endif
+#define PCRE_FLAG(x) G_PASTE(PCRE2_, x)
+#endif
+
+#define RSPAMD_INVALID_ID ((guint64) -1LL)
+#define RSPAMD_REGEXP_FLAG_RAW (1 << 1)
+#define RSPAMD_REGEXP_FLAG_NOOPT (1 << 2)
+#define RSPAMD_REGEXP_FLAG_FULL_MATCH (1 << 3)
+#define RSPAMD_REGEXP_FLAG_PCRE_ONLY (1 << 4)
+#define RSPAMD_REGEXP_FLAG_DISABLE_JIT (1 << 5)
+#define RSPAMD_REGEXP_FLAG_UTF (1 << 6)
+#define RSPAMD_REGEXP_FLAG_LEFTMOST (1 << 7)
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_config;
+
+typedef struct rspamd_regexp_s rspamd_regexp_t;
+struct rspamd_regexp_cache;
+struct rspamd_re_capture {
+ const char *p;
+ gsize len;
+};
+
+/**
+ * Create new rspamd regexp
+ * @param pattern regexp pattern
+ * @param flags flags (may be enclosed inside pattern)
+ * @param err error pointer set if compilation failed
+ * @return new regexp object
+ */
+rspamd_regexp_t *rspamd_regexp_new(const gchar *pattern, const gchar *flags,
+ GError **err);
+
+/**
+ * Create new rspamd regexp
+ * @param pattern regexp pattern
+ * @param flags flags (may be enclosed inside pattern)
+ * @param err error pointer set if compilation failed
+ * @return new regexp object
+ */
+rspamd_regexp_t *rspamd_regexp_new_len(const gchar *pattern, gsize len, const gchar *flags,
+ GError **err);
+
+/**
+ * Search the specified regexp in the text
+ * @param re
+ * @param text
+ * @param len
+ * @param start position of start of match
+ * @param start position of end of match
+ * @param raw
+ * @param captures array of captured strings of type rspamd_fstring_capture or NULL
+ * @return
+ */
+gboolean rspamd_regexp_search(const rspamd_regexp_t *re,
+ const gchar *text, gsize len,
+ const gchar **start, const gchar **end, gboolean raw,
+ GArray *captures);
+
+
+/**
+ * Exact match of the specified text against the regexp
+ * @param re
+ * @param text
+ * @param len
+ * @return
+ */
+gboolean rspamd_regexp_match(const rspamd_regexp_t *re,
+ const gchar *text, gsize len, gboolean raw);
+
+/**
+ * Increase refcount for a regexp object
+ */
+rspamd_regexp_t *rspamd_regexp_ref(rspamd_regexp_t *re);
+
+/**
+ * Unref regexp object
+ * @param re
+ */
+void rspamd_regexp_unref(rspamd_regexp_t *re);
+
+/**
+ * Set auxiliary userdata for the specified regexp
+ * @param re regexp object
+ * @param ud opaque pointer
+ */
+void rspamd_regexp_set_ud(rspamd_regexp_t *re, gpointer ud);
+
+/**
+ * Get userdata for a regexp object
+ * @param re regexp object
+ * @return opaque pointer
+ */
+gpointer rspamd_regexp_get_ud(const rspamd_regexp_t *re);
+
+/**
+ * Get regexp ID suitable for hashing
+ * @param re
+ * @return
+ */
+gpointer rspamd_regexp_get_id(const rspamd_regexp_t *re);
+
+/**
+ * Get pattern for the specified regexp object
+ * @param re
+ * @return
+ */
+const char *rspamd_regexp_get_pattern(const rspamd_regexp_t *re);
+
+/**
+ * Get PCRE flags for the regexp
+ */
+guint rspamd_regexp_get_pcre_flags(const rspamd_regexp_t *re);
+
+/**
+ * Get rspamd flags for the regexp
+ */
+guint rspamd_regexp_get_flags(const rspamd_regexp_t *re);
+
+/**
+ * Set rspamd flags for the regexp
+ */
+guint rspamd_regexp_set_flags(rspamd_regexp_t *re, guint new_flags);
+
+/**
+ * Set regexp maximum hits
+ */
+guint rspamd_regexp_get_maxhits(const rspamd_regexp_t *re);
+
+/**
+ * Get regexp maximum hits
+ */
+guint rspamd_regexp_set_maxhits(rspamd_regexp_t *re, guint new_maxhits);
+
+/**
+ * Returns cache id for a regexp
+ */
+guint64 rspamd_regexp_get_cache_id(const rspamd_regexp_t *re);
+
+/**
+ * Sets cache id for a regexp
+ */
+guint64 rspamd_regexp_set_cache_id(rspamd_regexp_t *re, guint64 id);
+
+/**
+ * Returns match limit for a regexp
+ */
+gsize rspamd_regexp_get_match_limit(const rspamd_regexp_t *re);
+
+/**
+ * Sets cache id for a regexp
+ */
+gsize rspamd_regexp_set_match_limit(rspamd_regexp_t *re, gsize lim);
+
+/**
+ * Get regexp class for the re object
+ */
+gpointer rspamd_regexp_get_class(const rspamd_regexp_t *re);
+
+/**
+ * Set regexp class for the re object
+ * @return old re class value
+ */
+gpointer rspamd_regexp_set_class(rspamd_regexp_t *re, gpointer re_class);
+
+/**
+ * Create new regexp cache
+ * @return
+ */
+struct rspamd_regexp_cache *rspamd_regexp_cache_new(void);
+
+/**
+ * Query rspamd cache for a specified regexp
+ * @param cache regexp cache. if NULL, the superglobal cache is used (*not* thread-safe)
+ * @param pattern
+ * @param flags
+ * @return
+ */
+rspamd_regexp_t *rspamd_regexp_cache_query(struct rspamd_regexp_cache *cache,
+ const gchar *pattern,
+ const gchar *flags);
+
+/**
+ * Create or get cached regexp from the specified cache
+ * @param cache regexp cache. if NULL, the superglobal cache is used (*not* thread-safe)
+ * @param pattern regexp pattern
+ * @param flags flags (may be enclosed inside pattern)
+ * @param err error pointer set if compilation failed
+ * @return new regexp object
+ */
+rspamd_regexp_t *rspamd_regexp_cache_create(struct rspamd_regexp_cache *cache,
+ const gchar *pattern,
+ const gchar *flags, GError **err);
+
+/**
+ * Remove regexp from the cache
+ * @param cache regexp cache. if NULL, the superglobal cache is used (*not* thread-safe)
+ * @param re re to remove
+ * @return TRUE if a regexp has been removed
+ */
+gboolean rspamd_regexp_cache_remove(struct rspamd_regexp_cache *cache,
+ rspamd_regexp_t *re);
+
+/**
+ * Destroy regexp cache and unref all elements inside it
+ * @param cache
+ */
+void rspamd_regexp_cache_destroy(struct rspamd_regexp_cache *cache);
+
+/**
+ * Return the value for regexp hash based on its ID
+ * @param a
+ * @return
+ */
+guint32 rspamd_regexp_hash(gconstpointer a);
+
+/**
+ * Compare two regexp objects based on theirs ID
+ * @param a
+ * @param b
+ * @return
+ */
+gboolean rspamd_regexp_equal(gconstpointer a, gconstpointer b);
+
+/**
+ * Acts like memcmp but for regexp
+ */
+gint rspamd_regexp_cmp(gconstpointer a, gconstpointer b);
+
+/**
+ * Initialize superglobal regexp cache and library
+ */
+void rspamd_regexp_library_init(struct rspamd_config *cfg);
+
+/**
+ * Create regexp from glob
+ * @param gl
+ * @param err
+ * @return
+ */
+rspamd_regexp_t *rspamd_regexp_from_glob(const gchar *gl, gsize sz, GError **err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* REGEXP_H_ */
diff --git a/src/libutil/rrd.c b/src/libutil/rrd.c
new file mode 100644
index 0000000..451e222
--- /dev/null
+++ b/src/libutil/rrd.c
@@ -0,0 +1,1502 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rrd.h"
+#include "util.h"
+#include "cfg_file.h"
+#include "logger.h"
+#include "unix-std.h"
+#include "cryptobox.h"
+#include <math.h>
+
+#define RSPAMD_RRD_DS_COUNT METRIC_ACTION_MAX
+#define RSPAMD_RRD_OLD_DS_COUNT 4
+#define RSPAMD_RRD_RRA_COUNT 4
+
+#define msg_err_rrd(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "rrd", file->id, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_warn_rrd(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "rrd", file->id, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_info_rrd(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "rrd", file->id, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_debug_rrd(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_rrd_log_id, "rrd", file->id, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(rrd)
+
+static GQuark
+rrd_error_quark(void)
+{
+ return g_quark_from_static_string("rrd-error");
+}
+
+/**
+ * Convert rrd dst type from string to numeric value
+ */
+enum rrd_dst_type
+rrd_dst_from_string(const gchar *str)
+{
+ if (g_ascii_strcasecmp(str, "counter") == 0) {
+ return RRD_DST_COUNTER;
+ }
+ else if (g_ascii_strcasecmp(str, "absolute") == 0) {
+ return RRD_DST_ABSOLUTE;
+ }
+ else if (g_ascii_strcasecmp(str, "gauge") == 0) {
+ return RRD_DST_GAUGE;
+ }
+ else if (g_ascii_strcasecmp(str, "cdef") == 0) {
+ return RRD_DST_CDEF;
+ }
+ else if (g_ascii_strcasecmp(str, "derive") == 0) {
+ return RRD_DST_DERIVE;
+ }
+
+ return RRD_DST_INVALID;
+}
+
+/**
+ * Convert numeric presentation of dst to string
+ */
+const gchar *
+rrd_dst_to_string(enum rrd_dst_type type)
+{
+ switch (type) {
+ case RRD_DST_COUNTER:
+ return "COUNTER";
+ case RRD_DST_ABSOLUTE:
+ return "ABSOLUTE";
+ case RRD_DST_GAUGE:
+ return "GAUGE";
+ case RRD_DST_CDEF:
+ return "CDEF";
+ case RRD_DST_DERIVE:
+ return "DERIVE";
+ default:
+ return "U";
+ }
+
+ return "U";
+}
+
+/**
+ * Convert rrd consolidation function type from string to numeric value
+ */
+enum rrd_cf_type
+rrd_cf_from_string(const gchar *str)
+{
+ if (g_ascii_strcasecmp(str, "average") == 0) {
+ return RRD_CF_AVERAGE;
+ }
+ else if (g_ascii_strcasecmp(str, "minimum") == 0) {
+ return RRD_CF_MINIMUM;
+ }
+ else if (g_ascii_strcasecmp(str, "maximum") == 0) {
+ return RRD_CF_MAXIMUM;
+ }
+ else if (g_ascii_strcasecmp(str, "last") == 0) {
+ return RRD_CF_LAST;
+ }
+ /* XXX: add other CF functions supported by rrd */
+
+ return RRD_CF_INVALID;
+}
+
+/**
+ * Convert numeric presentation of cf to string
+ */
+const gchar *
+rrd_cf_to_string(enum rrd_cf_type type)
+{
+ switch (type) {
+ case RRD_CF_AVERAGE:
+ return "AVERAGE";
+ case RRD_CF_MINIMUM:
+ return "MINIMUM";
+ case RRD_CF_MAXIMUM:
+ return "MAXIMUM";
+ case RRD_CF_LAST:
+ return "LAST";
+ default:
+ return "U";
+ }
+
+ /* XXX: add other CF functions supported by rrd */
+
+ return "U";
+}
+
+void rrd_make_default_rra(const gchar *cf_name,
+ gulong pdp_cnt,
+ gulong rows,
+ struct rrd_rra_def *rra)
+{
+ g_assert(cf_name != NULL);
+ g_assert(rrd_cf_from_string(cf_name) != RRD_CF_INVALID);
+
+ rra->pdp_cnt = pdp_cnt;
+ rra->row_cnt = rows;
+ rspamd_strlcpy(rra->cf_nam, cf_name, sizeof(rra->cf_nam));
+ memset(rra->par, 0, sizeof(rra->par));
+ rra->par[RRA_cdp_xff_val].dv = 0.5;
+}
+
+void rrd_make_default_ds(const gchar *name,
+ const gchar *type,
+ gulong pdp_step,
+ struct rrd_ds_def *ds)
+{
+ g_assert(name != NULL);
+ g_assert(type != NULL);
+ g_assert(rrd_dst_from_string(type) != RRD_DST_INVALID);
+
+ rspamd_strlcpy(ds->ds_nam, name, sizeof(ds->ds_nam));
+ rspamd_strlcpy(ds->dst, type, sizeof(ds->dst));
+ memset(ds->par, 0, sizeof(ds->par));
+ ds->par[RRD_DS_mrhb_cnt].lv = pdp_step * 2;
+ ds->par[RRD_DS_min_val].dv = NAN;
+ ds->par[RRD_DS_max_val].dv = NAN;
+}
+
+/**
+ * Check rrd file for correctness (size, cookies, etc)
+ */
+static gboolean
+rspamd_rrd_check_file(const gchar *filename, gboolean need_data, GError **err)
+{
+ gint fd, i;
+ struct stat st;
+ struct rrd_file_head head;
+ struct rrd_rra_def rra;
+ gint head_size;
+
+ fd = open(filename, O_RDWR);
+ if (fd == -1) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd open error: %s", strerror(errno));
+ return FALSE;
+ }
+
+ if (fstat(fd, &st) == -1) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd stat error: %s", strerror(errno));
+ close(fd);
+ return FALSE;
+ }
+ if (st.st_size < (goffset) sizeof(struct rrd_file_head)) {
+ /* We have trimmed file */
+ g_set_error(err, rrd_error_quark(), EINVAL, "rrd size is bad: %ud",
+ (guint) st.st_size);
+ close(fd);
+ return FALSE;
+ }
+
+ /* Try to read header */
+ if (read(fd, &head, sizeof(head)) != sizeof(head)) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd read head error: %s",
+ strerror(errno));
+ close(fd);
+ return FALSE;
+ }
+ /* Check magic */
+ if (memcmp(head.version, RRD_VERSION, sizeof(head.version)) != 0) {
+ g_set_error(err,
+ rrd_error_quark(), EINVAL, "rrd head error: bad cookie");
+ close(fd);
+ return FALSE;
+ }
+ if (head.float_cookie != RRD_FLOAT_COOKIE) {
+ g_set_error(err,
+ rrd_error_quark(), EINVAL, "rrd head error: another architecture "
+ "(file cookie %g != our cookie %g)",
+ head.float_cookie, RRD_FLOAT_COOKIE);
+ close(fd);
+ return FALSE;
+ }
+ /* Check for other params */
+ if (head.ds_cnt <= 0 || head.rra_cnt <= 0) {
+ g_set_error(err,
+ rrd_error_quark(), EINVAL, "rrd head cookies error: bad rra or ds count");
+ close(fd);
+ return FALSE;
+ }
+ /* Now we can calculate the overall size of rrd */
+ head_size = sizeof(struct rrd_file_head) +
+ sizeof(struct rrd_ds_def) * head.ds_cnt +
+ sizeof(struct rrd_rra_def) * head.rra_cnt +
+ sizeof(struct rrd_live_head) +
+ sizeof(struct rrd_pdp_prep) * head.ds_cnt +
+ sizeof(struct rrd_cdp_prep) * head.ds_cnt * head.rra_cnt +
+ sizeof(struct rrd_rra_ptr) * head.rra_cnt;
+ if (st.st_size < (goffset) head_size) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd file seems to have stripped header: %d",
+ head_size);
+ close(fd);
+ return FALSE;
+ }
+
+ if (need_data) {
+ /* Now check rra */
+ if (lseek(fd, sizeof(struct rrd_ds_def) * head.ds_cnt,
+ SEEK_CUR) == -1) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd head lseek error: %s",
+ strerror(errno));
+ close(fd);
+ return FALSE;
+ }
+ for (i = 0; i < (gint) head.rra_cnt; i++) {
+ if (read(fd, &rra, sizeof(rra)) != sizeof(rra)) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd read rra error: %s",
+ strerror(errno));
+ close(fd);
+ return FALSE;
+ }
+ head_size += rra.row_cnt * head.ds_cnt * sizeof(gdouble);
+ }
+
+ if (st.st_size != head_size) {
+ g_set_error(err,
+ rrd_error_quark(), EINVAL, "rrd file seems to have incorrect size: %d, must be %d",
+ (gint) st.st_size, head_size);
+ close(fd);
+ return FALSE;
+ }
+ }
+
+ close(fd);
+ return TRUE;
+}
+
+/**
+ * Adjust pointers in mmapped rrd file
+ * @param file
+ */
+static void
+rspamd_rrd_adjust_pointers(struct rspamd_rrd_file *file, gboolean completed)
+{
+ guint8 *ptr;
+
+ ptr = file->map;
+ file->stat_head = (struct rrd_file_head *) ptr;
+ ptr += sizeof(struct rrd_file_head);
+ file->ds_def = (struct rrd_ds_def *) ptr;
+ ptr += sizeof(struct rrd_ds_def) * file->stat_head->ds_cnt;
+ file->rra_def = (struct rrd_rra_def *) ptr;
+ ptr += sizeof(struct rrd_rra_def) * file->stat_head->rra_cnt;
+ file->live_head = (struct rrd_live_head *) ptr;
+ ptr += sizeof(struct rrd_live_head);
+ file->pdp_prep = (struct rrd_pdp_prep *) ptr;
+ ptr += sizeof(struct rrd_pdp_prep) * file->stat_head->ds_cnt;
+ file->cdp_prep = (struct rrd_cdp_prep *) ptr;
+ ptr += sizeof(struct rrd_cdp_prep) * file->stat_head->rra_cnt *
+ file->stat_head->ds_cnt;
+ file->rra_ptr = (struct rrd_rra_ptr *) ptr;
+ if (completed) {
+ ptr += sizeof(struct rrd_rra_ptr) * file->stat_head->rra_cnt;
+ file->rrd_value = (gdouble *) ptr;
+ }
+ else {
+ file->rrd_value = NULL;
+ }
+}
+
+static void
+rspamd_rrd_calculate_checksum(struct rspamd_rrd_file *file)
+{
+ guchar sigbuf[rspamd_cryptobox_HASHBYTES];
+ struct rrd_ds_def *ds;
+ guint i;
+ rspamd_cryptobox_hash_state_t st;
+
+ if (file->finalized) {
+ rspamd_cryptobox_hash_init(&st, NULL, 0);
+ rspamd_cryptobox_hash_update(&st, file->filename, strlen(file->filename));
+
+ for (i = 0; i < file->stat_head->ds_cnt; i++) {
+ ds = &file->ds_def[i];
+ rspamd_cryptobox_hash_update(&st, ds->ds_nam, sizeof(ds->ds_nam));
+ }
+
+ rspamd_cryptobox_hash_final(&st, sigbuf);
+
+ file->id = rspamd_encode_base32(sigbuf, sizeof(sigbuf), RSPAMD_BASE32_DEFAULT);
+ }
+}
+
+static int
+rspamd_rrd_open_exclusive(const gchar *filename)
+{
+ struct timespec sleep_ts = {
+ .tv_sec = 0,
+ .tv_nsec = 1000000};
+ gint fd;
+
+ fd = open(filename, O_RDWR);
+
+ if (fd == -1) {
+ return -1;
+ }
+
+ for (;;) {
+ if (rspamd_file_lock(fd, TRUE) == -1) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK) {
+ nanosleep(&sleep_ts, NULL);
+ continue;
+ }
+ else {
+ close(fd);
+ return -1;
+ }
+ }
+ else {
+ break;
+ }
+ }
+
+ return fd;
+};
+
+/**
+ * Open completed or incompleted rrd file
+ * @param filename
+ * @param completed
+ * @param err
+ * @return
+ */
+static struct rspamd_rrd_file *
+rspamd_rrd_open_common(const gchar *filename, gboolean completed, GError **err)
+{
+ struct rspamd_rrd_file *file;
+ gint fd;
+ struct stat st;
+
+ if (!rspamd_rrd_check_file(filename, completed, err)) {
+ return NULL;
+ }
+
+ file = g_malloc0(sizeof(struct rspamd_rrd_file));
+
+ /* Open file */
+ fd = rspamd_rrd_open_exclusive(filename);
+ if (fd == -1) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd open error: %s", strerror(errno));
+ g_free(file);
+ return FALSE;
+ }
+
+ if (fstat(fd, &st) == -1) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd stat error: %s", strerror(errno));
+ rspamd_file_unlock(fd, FALSE);
+ g_free(file);
+ close(fd);
+ return FALSE;
+ }
+ /* Mmap file */
+ file->size = st.st_size;
+ if ((file->map =
+ mmap(NULL, st.st_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED, fd, 0)) == MAP_FAILED) {
+
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+ g_set_error(err,
+ rrd_error_quark(), ENOMEM, "mmap failed: %s", strerror(errno));
+ g_free(file);
+ return NULL;
+ }
+
+ file->fd = fd;
+
+ /* Adjust pointers */
+ rspamd_rrd_adjust_pointers(file, completed);
+
+ /* Mark it as finalized */
+ file->finalized = completed;
+
+ file->filename = g_strdup(filename);
+ rspamd_rrd_calculate_checksum(file);
+
+ return file;
+}
+
+/**
+ * Open (and mmap) existing RRD file
+ * @param filename path
+ * @param err error pointer
+ * @return rrd file structure
+ */
+struct rspamd_rrd_file *
+rspamd_rrd_open(const gchar *filename, GError **err)
+{
+ struct rspamd_rrd_file *file;
+
+ if ((file = rspamd_rrd_open_common(filename, TRUE, err))) {
+ msg_info_rrd("rrd file opened: %s", filename);
+ }
+
+ return file;
+}
+
+/**
+ * Create basic header for rrd file
+ * @param filename file path
+ * @param ds_count number of data sources
+ * @param rra_count number of round robin archives
+ * @param pdp_step step of primary data points
+ * @param err error pointer
+ * @return TRUE if file has been created
+ */
+struct rspamd_rrd_file *
+rspamd_rrd_create(const gchar *filename,
+ gulong ds_count,
+ gulong rra_count,
+ gulong pdp_step,
+ gdouble initial_ticks,
+ GError **err)
+{
+ struct rspamd_rrd_file *new;
+ struct rrd_file_head head;
+ struct rrd_ds_def ds;
+ struct rrd_rra_def rra;
+ struct rrd_live_head lh;
+ struct rrd_pdp_prep pdp;
+ struct rrd_cdp_prep cdp;
+ struct rrd_rra_ptr rra_ptr;
+ gint fd;
+ guint i, j;
+
+ /* Open file */
+ fd = open(filename, O_RDWR | O_CREAT | O_EXCL, 0644);
+ if (fd == -1) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd create error: %s",
+ strerror(errno));
+ return NULL;
+ }
+
+ rspamd_file_lock(fd, FALSE);
+
+ /* Fill header */
+ memset(&head, 0, sizeof(head));
+ head.rra_cnt = rra_count;
+ head.ds_cnt = ds_count;
+ head.pdp_step = pdp_step;
+ memcpy(head.cookie, RRD_COOKIE, sizeof(head.cookie));
+ memcpy(head.version, RRD_VERSION, sizeof(head.version));
+ head.float_cookie = RRD_FLOAT_COOKIE;
+
+ if (write(fd, &head, sizeof(head)) != sizeof(head)) {
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd write error: %s", strerror(errno));
+ return NULL;
+ }
+
+ /* Fill DS section */
+ memset(&ds, 0, sizeof(ds));
+ memset(&ds.ds_nam, 0, sizeof(ds.ds_nam));
+ memcpy(&ds.dst, "COUNTER", sizeof("COUNTER"));
+ memset(&ds.par, 0, sizeof(ds.par));
+ for (i = 0; i < ds_count; i++) {
+ if (write(fd, &ds, sizeof(ds)) != sizeof(ds)) {
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd write error: %s",
+ strerror(errno));
+ return NULL;
+ }
+ }
+
+ /* Fill RRA section */
+ memset(&rra, 0, sizeof(rra));
+ memcpy(&rra.cf_nam, "AVERAGE", sizeof("AVERAGE"));
+ rra.pdp_cnt = 1;
+ memset(&rra.par, 0, sizeof(rra.par));
+ for (i = 0; i < rra_count; i++) {
+ if (write(fd, &rra, sizeof(rra)) != sizeof(rra)) {
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd write error: %s",
+ strerror(errno));
+ return NULL;
+ }
+ }
+
+ /* Fill live header */
+ memset(&lh, 0, sizeof(lh));
+ lh.last_up = (glong) initial_ticks;
+ lh.last_up_usec = (glong) ((initial_ticks - lh.last_up) * 1e6f);
+
+ if (write(fd, &lh, sizeof(lh)) != sizeof(lh)) {
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd write error: %s", strerror(errno));
+ return NULL;
+ }
+
+ /* Fill pdp prep */
+ memset(&pdp, 0, sizeof(pdp));
+ memcpy(&pdp.last_ds, "U", sizeof("U"));
+ memset(&pdp.scratch, 0, sizeof(pdp.scratch));
+ pdp.scratch[PDP_val].dv = NAN;
+ pdp.scratch[PDP_unkn_sec_cnt].lv = 0;
+
+ for (i = 0; i < ds_count; i++) {
+ if (write(fd, &pdp, sizeof(pdp)) != sizeof(pdp)) {
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd write error: %s",
+ strerror(errno));
+ return NULL;
+ }
+ }
+
+ /* Fill cdp prep */
+ memset(&cdp, 0, sizeof(cdp));
+ memset(&cdp.scratch, 0, sizeof(cdp.scratch));
+ cdp.scratch[CDP_val].dv = NAN;
+ cdp.scratch[CDP_unkn_pdp_cnt].lv = 0;
+
+ for (i = 0; i < rra_count; i++) {
+ for (j = 0; j < ds_count; j++) {
+ if (write(fd, &cdp, sizeof(cdp)) != sizeof(cdp)) {
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd write error: %s",
+ strerror(errno));
+ return NULL;
+ }
+ }
+ }
+
+ /* Set row pointers */
+ memset(&rra_ptr, 0, sizeof(rra_ptr));
+ for (i = 0; i < rra_count; i++) {
+ if (write(fd, &rra_ptr, sizeof(rra_ptr)) != sizeof(rra_ptr)) {
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd write error: %s",
+ strerror(errno));
+ return NULL;
+ }
+ }
+
+ rspamd_file_unlock(fd, FALSE);
+ close(fd);
+
+ new = rspamd_rrd_open_common(filename, FALSE, err);
+
+ return new;
+}
+
+/**
+ * Add data sources to rrd file
+ * @param filename path to file
+ * @param ds array of struct rrd_ds_def
+ * @param err error pointer
+ * @return TRUE if data sources were added
+ */
+gboolean
+rspamd_rrd_add_ds(struct rspamd_rrd_file *file, GArray *ds, GError **err)
+{
+
+ if (file == NULL || file->stat_head->ds_cnt * sizeof(struct rrd_ds_def) !=
+ ds->len) {
+ g_set_error(err,
+ rrd_error_quark(), EINVAL, "rrd add ds failed: wrong arguments");
+ return FALSE;
+ }
+
+ /* Straightforward memcpy */
+ memcpy(file->ds_def, ds->data, ds->len);
+
+ return TRUE;
+}
+
+/**
+ * Add round robin archives to rrd file
+ * @param filename path to file
+ * @param ds array of struct rrd_rra_def
+ * @param err error pointer
+ * @return TRUE if archives were added
+ */
+gboolean
+rspamd_rrd_add_rra(struct rspamd_rrd_file *file, GArray *rra, GError **err)
+{
+ if (file == NULL || file->stat_head->rra_cnt *
+ sizeof(struct rrd_rra_def) !=
+ rra->len) {
+ g_set_error(err,
+ rrd_error_quark(), EINVAL, "rrd add rra failed: wrong arguments");
+ return FALSE;
+ }
+
+ /* Straightforward memcpy */
+ memcpy(file->rra_def, rra->data, rra->len);
+
+ return TRUE;
+}
+
+/**
+ * Finalize rrd file header and initialize all RRA in the file
+ * @param filename file path
+ * @param err error pointer
+ * @return TRUE if rrd file is ready for use
+ */
+gboolean
+rspamd_rrd_finalize(struct rspamd_rrd_file *file, GError **err)
+{
+ gint fd;
+ guint i;
+ gint count = 0;
+ gdouble vbuf[1024];
+ struct stat st;
+
+ if (file == NULL || file->filename == NULL || file->fd == -1) {
+ g_set_error(err,
+ rrd_error_quark(), EINVAL, "rrd add rra failed: wrong arguments");
+ return FALSE;
+ }
+
+ fd = file->fd;
+
+ if (lseek(fd, 0, SEEK_END) == -1) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd seek error: %s", strerror(errno));
+ close(fd);
+ return FALSE;
+ }
+
+ /* Adjust CDP */
+ for (i = 0; i < file->stat_head->rra_cnt; i++) {
+ file->cdp_prep->scratch[CDP_unkn_pdp_cnt].lv = 0;
+ /* Randomize row pointer (disabled) */
+ /* file->rra_ptr->cur_row = g_random_int () % file->rra_def[i].row_cnt; */
+ file->rra_ptr->cur_row = file->rra_def[i].row_cnt - 1;
+ /* Calculate values count */
+ count += file->rra_def[i].row_cnt * file->stat_head->ds_cnt;
+ }
+
+ munmap(file->map, file->size);
+ /* Write values */
+ for (i = 0; i < G_N_ELEMENTS(vbuf); i++) {
+ vbuf[i] = NAN;
+ }
+
+ while (count > 0) {
+ /* Write values in buffered matter */
+ if (write(fd, vbuf,
+ MIN((gint) G_N_ELEMENTS(vbuf), count) * sizeof(gdouble)) == -1) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd write error: %s",
+ strerror(errno));
+ close(fd);
+ return FALSE;
+ }
+ count -= G_N_ELEMENTS(vbuf);
+ }
+
+ if (fstat(fd, &st) == -1) {
+ g_set_error(err,
+ rrd_error_quark(), errno, "rrd stat error: %s", strerror(errno));
+ close(fd);
+ return FALSE;
+ }
+
+ /* Mmap again */
+ file->size = st.st_size;
+ if ((file->map =
+ mmap(NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd,
+ 0)) == MAP_FAILED) {
+ close(fd);
+ g_set_error(err,
+ rrd_error_quark(), ENOMEM, "mmap failed: %s", strerror(errno));
+
+ return FALSE;
+ }
+
+ /* Adjust pointers */
+ rspamd_rrd_adjust_pointers(file, TRUE);
+
+ file->finalized = TRUE;
+ rspamd_rrd_calculate_checksum(file);
+ msg_info_rrd("rrd file created: %s", file->filename);
+
+ return TRUE;
+}
+
+/**
+ * Update pdp_prep data
+ * @param file rrd file
+ * @param vals new values
+ * @param pdp_new new pdp array
+ * @param interval time elapsed from the last update
+ * @return
+ */
+static gboolean
+rspamd_rrd_update_pdp_prep(struct rspamd_rrd_file *file,
+ gdouble *vals,
+ gdouble *pdp_new,
+ gdouble interval)
+{
+ guint i;
+ enum rrd_dst_type type;
+
+ for (i = 0; i < file->stat_head->ds_cnt; i++) {
+ type = rrd_dst_from_string(file->ds_def[i].dst);
+
+ if (file->ds_def[i].par[RRD_DS_mrhb_cnt].lv < interval) {
+ rspamd_strlcpy(file->pdp_prep[i].last_ds, "U",
+ sizeof(file->pdp_prep[i].last_ds));
+ pdp_new[i] = NAN;
+ msg_debug_rrd("adding unknown point interval %.3f is less than heartbeat %l",
+ interval, file->ds_def[i].par[RRD_DS_mrhb_cnt].lv);
+ }
+ else {
+ switch (type) {
+ case RRD_DST_COUNTER:
+ case RRD_DST_DERIVE:
+ if (file->pdp_prep[i].last_ds[0] == 'U') {
+ pdp_new[i] = NAN;
+ msg_debug_rrd("last point is NaN for point %ud", i);
+ }
+ else {
+ pdp_new[i] = vals[i] - strtod(file->pdp_prep[i].last_ds,
+ NULL);
+ msg_debug_rrd("new PDP %ud, %.3f", i, pdp_new[i]);
+ }
+ break;
+ case RRD_DST_GAUGE:
+ pdp_new[i] = vals[i] * interval;
+ msg_debug_rrd("new PDP %ud, %.3f", i, pdp_new[i]);
+ break;
+ case RRD_DST_ABSOLUTE:
+ pdp_new[i] = vals[i];
+ msg_debug_rrd("new PDP %ud, %.3f", i, pdp_new[i]);
+ break;
+ default:
+ return FALSE;
+ }
+ }
+
+ /* Copy value to the last_ds */
+ if (!isnan(vals[i])) {
+ rspamd_snprintf(file->pdp_prep[i].last_ds,
+ sizeof(file->pdp_prep[i].last_ds), "%.4f", vals[i]);
+ }
+ else {
+ file->pdp_prep[i].last_ds[0] = 'U';
+ file->pdp_prep[i].last_ds[1] = '\0';
+ }
+ }
+
+
+ return TRUE;
+}
+
+/**
+ * Update step for this pdp
+ * @param file
+ * @param pdp_new new pdp array
+ * @param pdp_temp temp pdp array
+ * @param interval time till last update
+ * @param pre_int pre interval
+ * @param post_int post intervall
+ * @param pdp_diff time till last pdp update
+ */
+static void
+rspamd_rrd_update_pdp_step(struct rspamd_rrd_file *file,
+ gdouble *pdp_new,
+ gdouble *pdp_temp,
+ gdouble interval,
+ gulong pdp_diff)
+{
+ guint i;
+ rrd_value_t *scratch;
+ gulong heartbeat;
+
+
+ for (i = 0; i < file->stat_head->ds_cnt; i++) {
+ scratch = file->pdp_prep[i].scratch;
+ heartbeat = file->ds_def[i].par[RRD_DS_mrhb_cnt].lv;
+
+ if (!isnan(pdp_new[i])) {
+ if (isnan(scratch[PDP_val].dv)) {
+ scratch[PDP_val].dv = 0;
+ }
+ }
+
+ /* Check interval value for heartbeat for this DS */
+ if ((interval > heartbeat) ||
+ (file->stat_head->pdp_step / 2.0 < scratch[PDP_unkn_sec_cnt].lv)) {
+ pdp_temp[i] = NAN;
+ }
+ else {
+ pdp_temp[i] = scratch[PDP_val].dv /
+ ((double) (pdp_diff - scratch[PDP_unkn_sec_cnt].lv));
+ }
+
+ if (isnan(pdp_new[i])) {
+ scratch[PDP_unkn_sec_cnt].lv = interval;
+ scratch[PDP_val].dv = NAN;
+ }
+ else {
+ scratch[PDP_unkn_sec_cnt].lv = 0;
+ scratch[PDP_val].dv = pdp_new[i] / interval;
+ }
+
+ msg_debug_rrd("new temp PDP %ud, %.3f -> %.3f, scratch: %3f",
+ i, pdp_new[i], pdp_temp[i],
+ scratch[PDP_val].dv);
+ }
+}
+
+/**
+ * Update CDP for this rra
+ * @param file rrd file
+ * @param pdp_steps how much pdp steps elapsed from the last update
+ * @param pdp_offset offset from pdp
+ * @param rra_steps how much steps must be updated for this rra
+ * @param rra_index index of desired rra
+ * @param pdp_temp temporary pdp points
+ */
+static void
+rspamd_rrd_update_cdp(struct rspamd_rrd_file *file,
+ gdouble pdp_steps,
+ gdouble pdp_offset,
+ gulong *rra_steps,
+ gulong rra_index,
+ gdouble *pdp_temp)
+{
+ guint i;
+ struct rrd_rra_def *rra;
+ rrd_value_t *scratch;
+ enum rrd_cf_type cf;
+ gdouble last_cdp = INFINITY, cur_cdp = INFINITY;
+ gulong pdp_in_cdp;
+
+ rra = &file->rra_def[rra_index];
+ cf = rrd_cf_from_string(rra->cf_nam);
+
+ /* Iterate over all DS for this RRA */
+ for (i = 0; i < file->stat_head->ds_cnt; i++) {
+ /* Get CDP for this RRA and DS */
+ scratch =
+ file->cdp_prep[rra_index * file->stat_head->ds_cnt + i].scratch;
+ if (rra->pdp_cnt > 1) {
+ /* Do we have any CDP to update for this rra ? */
+ if (rra_steps[rra_index] > 0) {
+
+ if (isnan(pdp_temp[i])) {
+ /* New pdp is nan */
+ /* Increment unknown points count */
+ scratch[CDP_unkn_pdp_cnt].lv += pdp_offset;
+ /* Reset secondary value */
+ scratch[CDP_secondary_val].dv = NAN;
+ }
+ else {
+ scratch[CDP_secondary_val].dv = pdp_temp[i];
+ }
+
+ /* Check XFF for this rra */
+ if (scratch[CDP_unkn_pdp_cnt].lv > rra->pdp_cnt *
+ rra->par[RRA_cdp_xff_val].lv) {
+ /* XFF is reached */
+ scratch[CDP_primary_val].dv = NAN;
+ }
+ else {
+ /* Need to initialize CDP using specified consolidation */
+ switch (cf) {
+ case RRD_CF_AVERAGE:
+ last_cdp =
+ isnan(scratch[CDP_val].dv) ? 0.0 : scratch[CDP_val].dv;
+ cur_cdp = isnan(pdp_temp[i]) ? 0.0 : pdp_temp[i];
+ scratch[CDP_primary_val].dv =
+ (last_cdp + cur_cdp *
+ pdp_offset) /
+ (rra->pdp_cnt - scratch[CDP_unkn_pdp_cnt].lv);
+ break;
+ case RRD_CF_MAXIMUM:
+ last_cdp =
+ isnan(scratch[CDP_val].dv) ? -INFINITY : scratch[CDP_val].dv;
+ cur_cdp = isnan(pdp_temp[i]) ? -INFINITY : pdp_temp[i];
+ scratch[CDP_primary_val].dv = MAX(last_cdp, cur_cdp);
+ break;
+ case RRD_CF_MINIMUM:
+ last_cdp =
+ isnan(scratch[CDP_val].dv) ? INFINITY : scratch[CDP_val].dv;
+ cur_cdp = isnan(pdp_temp[i]) ? INFINITY : pdp_temp[i];
+ scratch[CDP_primary_val].dv = MIN(last_cdp, cur_cdp);
+ break;
+ case RRD_CF_LAST:
+ default:
+ scratch[CDP_primary_val].dv = pdp_temp[i];
+ last_cdp = INFINITY;
+ break;
+ }
+ }
+
+ /* Init carry of this CDP */
+ pdp_in_cdp = (pdp_steps - pdp_offset) / rra->pdp_cnt;
+ if (pdp_in_cdp == 0 || isnan(pdp_temp[i])) {
+ /* Set overflow */
+ switch (cf) {
+ case RRD_CF_AVERAGE:
+ scratch[CDP_val].dv = 0;
+ break;
+ case RRD_CF_MAXIMUM:
+ scratch[CDP_val].dv = -INFINITY;
+ break;
+ case RRD_CF_MINIMUM:
+ scratch[CDP_val].dv = INFINITY;
+ break;
+ default:
+ scratch[CDP_val].dv = NAN;
+ break;
+ }
+ }
+ else {
+ /* Special carry for average */
+ if (cf == RRD_CF_AVERAGE) {
+ scratch[CDP_val].dv = pdp_temp[i] * pdp_in_cdp;
+ }
+ else {
+ scratch[CDP_val].dv = pdp_temp[i];
+ }
+ }
+
+ scratch[CDP_unkn_pdp_cnt].lv = 0;
+
+ msg_debug_rrd("update cdp for DS %d with value %.3f, "
+ "stored value: %.3f, carry: %.3f",
+ i, last_cdp,
+ scratch[CDP_primary_val].dv, scratch[CDP_val].dv);
+ }
+ /* In this case we just need to update cdp_prep for this RRA */
+ else {
+ if (isnan(pdp_temp[i])) {
+ /* Just increase undefined zone */
+ scratch[CDP_unkn_pdp_cnt].lv += pdp_steps;
+ }
+ else {
+ /* Calculate cdp value */
+ last_cdp = scratch[CDP_val].dv;
+ switch (cf) {
+ case RRD_CF_AVERAGE:
+ if (isnan(last_cdp)) {
+ scratch[CDP_val].dv = pdp_temp[i] * pdp_steps;
+ }
+ else {
+ scratch[CDP_val].dv = last_cdp + pdp_temp[i] *
+ pdp_steps;
+ }
+ break;
+ case RRD_CF_MAXIMUM:
+ scratch[CDP_val].dv = MAX(last_cdp, pdp_temp[i]);
+ break;
+ case RRD_CF_MINIMUM:
+ scratch[CDP_val].dv = MIN(last_cdp, pdp_temp[i]);
+ break;
+ case RRD_CF_LAST:
+ scratch[CDP_val].dv = pdp_temp[i];
+ break;
+ default:
+ scratch[CDP_val].dv = NAN;
+ break;
+ }
+ }
+
+ msg_debug_rrd("aggregate cdp %d with pdp %.3f, "
+ "stored value: %.3f",
+ i, pdp_temp[i], scratch[CDP_val].dv);
+ }
+ }
+ else {
+ /* We have nothing to consolidate, but we may miss some pdp */
+ if (pdp_steps > 2) {
+ /* Just write PDP value */
+ scratch[CDP_primary_val].dv = pdp_temp[i];
+ scratch[CDP_secondary_val].dv = pdp_temp[i];
+ }
+ }
+ }
+}
+
+/**
+ * Update RRA in a file
+ * @param file rrd file
+ * @param rra_steps steps for each rra
+ * @param now current time
+ */
+void rspamd_rrd_write_rra(struct rspamd_rrd_file *file, gulong *rra_steps)
+{
+ guint i, j, ds_cnt;
+ struct rrd_rra_def *rra;
+ struct rrd_cdp_prep *cdp;
+ gdouble *rra_row = file->rrd_value, *cur_row;
+
+
+ ds_cnt = file->stat_head->ds_cnt;
+ /* Iterate over all RRA */
+ for (i = 0; i < file->stat_head->rra_cnt; i++) {
+ rra = &file->rra_def[i];
+
+ if (rra_steps[i] > 0) {
+
+ /* Move row ptr */
+ if (++file->rra_ptr[i].cur_row >= rra->row_cnt) {
+ file->rra_ptr[i].cur_row = 0;
+ }
+ /* Calculate seek */
+ cdp = &file->cdp_prep[ds_cnt * i];
+ cur_row = rra_row + ds_cnt * file->rra_ptr[i].cur_row;
+ /* Iterate over DS */
+ for (j = 0; j < ds_cnt; j++) {
+ cur_row[j] = cdp[j].scratch[CDP_primary_val].dv;
+ msg_debug_rrd("write cdp %d: %.3f", j, cur_row[j]);
+ }
+ }
+
+ rra_row += rra->row_cnt * ds_cnt;
+ }
+}
+
+/**
+ * Add record to rrd file
+ * @param file rrd file object
+ * @param points points (must be row suitable for this RRA, depending on ds count)
+ * @param err error pointer
+ * @return TRUE if a row has been added
+ */
+gboolean
+rspamd_rrd_add_record(struct rspamd_rrd_file *file,
+ GArray *points,
+ gdouble ticks,
+ GError **err)
+{
+ gdouble interval, *pdp_new, *pdp_temp;
+ guint i;
+ glong seconds, microseconds;
+ gulong pdp_steps, cur_pdp_count, prev_pdp_step, cur_pdp_step,
+ prev_pdp_age, cur_pdp_age, *rra_steps, pdp_offset;
+
+ if (file == NULL || file->stat_head->ds_cnt * sizeof(gdouble) !=
+ points->len) {
+ g_set_error(err,
+ rrd_error_quark(), EINVAL,
+ "rrd add points failed: wrong arguments");
+ return FALSE;
+ }
+
+ /* Get interval */
+ seconds = (glong) ticks;
+ microseconds = (glong) ((ticks - seconds) * 1000000.);
+ interval = ticks - ((gdouble) file->live_head->last_up +
+ file->live_head->last_up_usec / 1000000.);
+
+ msg_debug_rrd("update rrd record after %.3f seconds", interval);
+
+ /* Update PDP preparation values */
+ pdp_new = g_malloc0(sizeof(gdouble) * file->stat_head->ds_cnt);
+ pdp_temp = g_malloc0(sizeof(gdouble) * file->stat_head->ds_cnt);
+ /* How much steps need to be updated in each RRA */
+ rra_steps = g_malloc0(sizeof(gulong) * file->stat_head->rra_cnt);
+
+ if (!rspamd_rrd_update_pdp_prep(file, (gdouble *) points->data, pdp_new,
+ interval)) {
+ g_set_error(err,
+ rrd_error_quark(), EINVAL,
+ "rrd update pdp failed: wrong arguments");
+ g_free(pdp_new);
+ g_free(pdp_temp);
+ g_free(rra_steps);
+ return FALSE;
+ }
+
+ /* Calculate elapsed steps */
+ /* Age in seconds for previous pdp store */
+ prev_pdp_age = file->live_head->last_up % file->stat_head->pdp_step;
+ /* Time in seconds for last pdp update */
+ prev_pdp_step = file->live_head->last_up - prev_pdp_age;
+ /* Age in seconds from current time to required pdp time */
+ cur_pdp_age = seconds % file->stat_head->pdp_step;
+ /* Time of desired pdp step */
+ cur_pdp_step = seconds - cur_pdp_age;
+ cur_pdp_count = cur_pdp_step / file->stat_head->pdp_step;
+ pdp_steps = (cur_pdp_step - prev_pdp_step) / file->stat_head->pdp_step;
+
+
+ if (pdp_steps == 0) {
+ /* Simple update of pdp prep */
+ for (i = 0; i < file->stat_head->ds_cnt; i++) {
+ if (isnan(pdp_new[i])) {
+ /* Increment unknown period */
+ file->pdp_prep[i].scratch[PDP_unkn_sec_cnt].lv += floor(
+ interval);
+ }
+ else {
+ if (isnan(file->pdp_prep[i].scratch[PDP_val].dv)) {
+ /* Reset pdp to the current value */
+ file->pdp_prep[i].scratch[PDP_val].dv = pdp_new[i];
+ }
+ else {
+ /* Increment pdp value */
+ file->pdp_prep[i].scratch[PDP_val].dv += pdp_new[i];
+ }
+ }
+ }
+ }
+ else {
+ /* Complex update of PDP, CDP and RRA */
+
+ /* Update PDP for this step */
+ rspamd_rrd_update_pdp_step(file,
+ pdp_new,
+ pdp_temp,
+ interval,
+ pdp_steps * file->stat_head->pdp_step);
+
+
+ /* Update CDP points for each RRA*/
+ for (i = 0; i < file->stat_head->rra_cnt; i++) {
+ /* Calculate pdp offset for this RRA */
+ pdp_offset = file->rra_def[i].pdp_cnt - cur_pdp_count %
+ file->rra_def[i].pdp_cnt;
+ /* How much steps we got for this RRA */
+ if (pdp_offset <= pdp_steps) {
+ rra_steps[i] =
+ (pdp_steps - pdp_offset) / file->rra_def[i].pdp_cnt + 1;
+ }
+ else {
+ /* This rra have not passed enough pdp steps */
+ rra_steps[i] = 0;
+ }
+
+ msg_debug_rrd("cdp: %ud, rra steps: %ul(%ul), pdp steps: %ul",
+ i, rra_steps[i], pdp_offset, pdp_steps);
+
+ /* Update this specific CDP */
+ rspamd_rrd_update_cdp(file,
+ pdp_steps,
+ pdp_offset,
+ rra_steps,
+ i,
+ pdp_temp);
+ }
+
+ /* Write RRA */
+ rspamd_rrd_write_rra(file, rra_steps);
+ }
+ file->live_head->last_up = seconds;
+ file->live_head->last_up_usec = microseconds;
+
+ /* Sync and invalidate */
+ msync(file->map, file->size, MS_ASYNC | MS_INVALIDATE);
+
+ g_free(pdp_new);
+ g_free(pdp_temp);
+ g_free(rra_steps);
+
+ return TRUE;
+}
+
+/**
+ * Close rrd file
+ * @param file
+ * @return
+ */
+gint rspamd_rrd_close(struct rspamd_rrd_file *file)
+{
+ if (file == NULL) {
+ errno = EINVAL;
+ return -1;
+ }
+
+ munmap(file->map, file->size);
+ close(file->fd);
+ g_free(file->filename);
+ g_free(file->id);
+
+ g_free(file);
+
+ return 0;
+}
+
+static struct rspamd_rrd_file *
+rspamd_rrd_create_file(const gchar *path, gboolean finalize, GError **err)
+{
+ struct rspamd_rrd_file *file;
+ struct rrd_ds_def ds[RSPAMD_RRD_DS_COUNT];
+ struct rrd_rra_def rra[RSPAMD_RRD_RRA_COUNT];
+ gint i;
+ GArray ar;
+
+ /* Try to create new rrd file */
+
+ file = rspamd_rrd_create(path, RSPAMD_RRD_DS_COUNT, RSPAMD_RRD_RRA_COUNT,
+ 1, rspamd_get_calendar_ticks(), err);
+
+ if (file == NULL) {
+ return NULL;
+ }
+
+ /* Create DS and RRA */
+
+ for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) {
+ rrd_make_default_ds(rspamd_action_to_str(i),
+ rrd_dst_to_string(RRD_DST_COUNTER), 1, &ds[i]);
+ }
+
+ ar.data = (gchar *) ds;
+ ar.len = sizeof(ds);
+
+ if (!rspamd_rrd_add_ds(file, &ar, err)) {
+ rspamd_rrd_close(file);
+ return NULL;
+ }
+
+ /* Once per minute for 1 day */
+ rrd_make_default_rra(rrd_cf_to_string(RRD_CF_AVERAGE),
+ 60, 24 * 60, &rra[0]);
+ /* Once per 5 minutes for 1 week */
+ rrd_make_default_rra(rrd_cf_to_string(RRD_CF_AVERAGE),
+ 5 * 60, 7 * 24 * 60 / 5, &rra[1]);
+ /* Once per 10 mins for 1 month */
+ rrd_make_default_rra(rrd_cf_to_string(RRD_CF_AVERAGE),
+ 60 * 10, 30 * 24 * 6, &rra[2]);
+ /* Once per hour for 1 year */
+ rrd_make_default_rra(rrd_cf_to_string(RRD_CF_AVERAGE),
+ 60 * 60, 365 * 24, &rra[3]);
+ ar.data = (gchar *) rra;
+ ar.len = sizeof(rra);
+
+ if (!rspamd_rrd_add_rra(file, &ar, err)) {
+ rspamd_rrd_close(file);
+ return NULL;
+ }
+
+ if (finalize && !rspamd_rrd_finalize(file, err)) {
+ rspamd_rrd_close(file);
+ return NULL;
+ }
+
+ return file;
+}
+
+static void
+rspamd_rrd_convert_ds(struct rspamd_rrd_file *old,
+ struct rspamd_rrd_file *cur, gint idx_old, gint idx_new)
+{
+ struct rrd_pdp_prep *pdp_prep_old, *pdp_prep_new;
+ struct rrd_cdp_prep *cdp_prep_old, *cdp_prep_new;
+ gdouble *val_old, *val_new;
+ gulong rra_cnt, i, j, points_cnt, old_ds, new_ds;
+
+ rra_cnt = old->stat_head->rra_cnt;
+ pdp_prep_old = &old->pdp_prep[idx_old];
+ pdp_prep_new = &cur->pdp_prep[idx_new];
+ memcpy(pdp_prep_new, pdp_prep_old, sizeof(*pdp_prep_new));
+ val_old = old->rrd_value;
+ val_new = cur->rrd_value;
+ old_ds = old->stat_head->ds_cnt;
+ new_ds = cur->stat_head->ds_cnt;
+
+ for (i = 0; i < rra_cnt; i++) {
+ cdp_prep_old = &old->cdp_prep[i * old_ds] + idx_old;
+ cdp_prep_new = &cur->cdp_prep[i * new_ds] + idx_new;
+ memcpy(cdp_prep_new, cdp_prep_old, sizeof(*cdp_prep_new));
+ points_cnt = old->rra_def[i].row_cnt;
+
+ for (j = 0; j < points_cnt; j++) {
+ val_new[j * new_ds + idx_new] = val_old[j * old_ds + idx_old];
+ }
+
+ val_new += points_cnt * new_ds;
+ val_old += points_cnt * old_ds;
+ }
+}
+
+static struct rspamd_rrd_file *
+rspamd_rrd_convert(const gchar *path, struct rspamd_rrd_file *old,
+ GError **err)
+{
+ struct rspamd_rrd_file *rrd;
+ gchar tpath[PATH_MAX];
+
+ g_assert(old != NULL);
+
+ rspamd_snprintf(tpath, sizeof(tpath), "%s.new", path);
+ rrd = rspamd_rrd_create_file(tpath, TRUE, err);
+
+ if (rrd) {
+ /* Copy old data */
+ memcpy(rrd->live_head, old->live_head, sizeof(*rrd->live_head));
+ memcpy(rrd->rra_ptr, old->rra_ptr,
+ sizeof(*old->rra_ptr) * rrd->stat_head->rra_cnt);
+
+ /*
+ * Old DSes:
+ * 0 - spam -> reject
+ * 1 - probable spam -> add header
+ * 2 - greylist -> greylist
+ * 3 - ham -> ham
+ */
+ rspamd_rrd_convert_ds(old, rrd, 0, METRIC_ACTION_REJECT);
+ rspamd_rrd_convert_ds(old, rrd, 1, METRIC_ACTION_ADD_HEADER);
+ rspamd_rrd_convert_ds(old, rrd, 2, METRIC_ACTION_GREYLIST);
+ rspamd_rrd_convert_ds(old, rrd, 3, METRIC_ACTION_NOACTION);
+
+ if (unlink(path) == -1) {
+ g_set_error(err, rrd_error_quark(), errno, "cannot unlink old rrd file %s: %s",
+ path, strerror(errno));
+ unlink(tpath);
+ rspamd_rrd_close(rrd);
+
+ return NULL;
+ }
+
+ if (rename(tpath, path) == -1) {
+ g_set_error(err, rrd_error_quark(), errno, "cannot rename old rrd file %s: %s",
+ path, strerror(errno));
+ unlink(tpath);
+ rspamd_rrd_close(rrd);
+
+ return NULL;
+ }
+ }
+
+ return rrd;
+}
+
+struct rspamd_rrd_file *
+rspamd_rrd_file_default(const gchar *path,
+ GError **err)
+{
+ struct rspamd_rrd_file *file, *nf;
+
+ g_assert(path != NULL);
+
+ if (access(path, R_OK) != -1) {
+ /* We can open rrd file */
+ file = rspamd_rrd_open(path, err);
+
+ if (file == NULL) {
+ return NULL;
+ }
+
+
+ if (file->stat_head->rra_cnt != RSPAMD_RRD_RRA_COUNT) {
+ msg_err_rrd("rrd file is not suitable for rspamd: it has "
+ "%ul ds and %ul rra",
+ file->stat_head->ds_cnt,
+ file->stat_head->rra_cnt);
+ g_set_error(err, rrd_error_quark(), EINVAL, "bad rrd file");
+ rspamd_rrd_close(file);
+
+ return NULL;
+ }
+ else if (file->stat_head->ds_cnt == RSPAMD_RRD_OLD_DS_COUNT) {
+ /* Old rrd, need to convert */
+ msg_info_rrd("rrd file %s is not suitable for rspamd, convert it",
+ path);
+
+ nf = rspamd_rrd_convert(path, file, err);
+ rspamd_rrd_close(file);
+
+ return nf;
+ }
+ else if (file->stat_head->ds_cnt == RSPAMD_RRD_DS_COUNT) {
+ return file;
+ }
+ else {
+ msg_err_rrd("rrd file is not suitable for rspamd: it has "
+ "%ul ds and %ul rra",
+ file->stat_head->ds_cnt,
+ file->stat_head->rra_cnt);
+ g_set_error(err, rrd_error_quark(), EINVAL, "bad rrd file");
+ rspamd_rrd_close(file);
+
+ return NULL;
+ }
+ }
+
+ file = rspamd_rrd_create_file(path, TRUE, err);
+
+ return file;
+}
+
+struct rspamd_rrd_query_result *
+rspamd_rrd_query(struct rspamd_rrd_file *file,
+ gulong rra_num)
+{
+ struct rspamd_rrd_query_result *res;
+ struct rrd_rra_def *rra;
+ const gdouble *rra_offset = NULL;
+ guint i;
+
+ g_assert(file != NULL);
+
+
+ if (rra_num > file->stat_head->rra_cnt) {
+ msg_err_rrd("requested unexisting rra: %l", rra_num);
+
+ return NULL;
+ }
+
+ res = g_malloc0(sizeof(*res));
+ res->ds_count = file->stat_head->ds_cnt;
+ res->last_update = (gdouble) file->live_head->last_up +
+ ((gdouble) file->live_head->last_up_usec / 1e6f);
+ res->pdp_per_cdp = file->rra_def[rra_num].pdp_cnt;
+ res->rra_rows = file->rra_def[rra_num].row_cnt;
+ rra_offset = file->rrd_value;
+
+ for (i = 0; i < file->stat_head->rra_cnt; i++) {
+ rra = &file->rra_def[i];
+
+ if (i == rra_num) {
+ res->cur_row = file->rra_ptr[i].cur_row % rra->row_cnt;
+ break;
+ }
+
+ rra_offset += rra->row_cnt * res->ds_count;
+ }
+
+ res->data = rra_offset;
+
+ return res;
+}
diff --git a/src/libutil/rrd.h b/src/libutil/rrd.h
new file mode 100644
index 0000000..3d81477
--- /dev/null
+++ b/src/libutil/rrd.h
@@ -0,0 +1,362 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RRD_H_
+#define RRD_H_
+
+#include "config.h"
+
+/**
+ * This file contains basic structure and functions to operate with round-robin databases
+ */
+
+#define RRD_COOKIE "RRD"
+#define RRD_VERSION "0003"
+#define RRD_FLOAT_COOKIE ((double) 8.642135E130)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef union {
+ unsigned long lv;
+ double dv;
+} rrd_value_t;
+
+struct rrd_file_head {
+ /* Data Base Identification Section ** */
+ gchar cookie[4]; /* RRD */
+ gchar version[5]; /* version of the format */
+ gdouble float_cookie; /* is it the correct double representation ? */
+
+ /* Data Base Structure Definition **** */
+ gulong ds_cnt; /* how many different ds provide input to the rrd */
+ gulong rra_cnt; /* how many rras will be maintained in the rrd */
+ gulong pdp_step; /* pdp interval in seconds */
+
+ rrd_value_t par[10]; /* global parameters ... unused
+ at the moment */
+};
+
+enum rrd_dst_type {
+ RRD_DST_INVALID = -1,
+ RRD_DST_COUNTER = 0, /* data source types available */
+ RRD_DST_ABSOLUTE,
+ RRD_DST_GAUGE,
+ RRD_DST_DERIVE,
+ RRD_DST_CDEF
+};
+enum rrd_ds_param {
+ RRD_DS_mrhb_cnt = 0, /* minimum required heartbeat */
+ RRD_DS_min_val, /* the processed input of a ds must */
+ RRD_DS_max_val, /* be between max_val and min_val
+ * both can be set to UNKNOWN if you
+ * do not care. Data outside the limits
+ * set to UNKNOWN */
+ RRD_DS_cdef = RRD_DS_mrhb_cnt
+}; /* pointer to encoded rpn expression only applies to DST_CDEF */
+
+
+/* The magic number here is one less than DS_NAM_SIZE */
+#define RRD_DS_NAM_SIZE 20
+
+#define RRD_DST_SIZE 20
+
+struct rrd_ds_def {
+ gchar ds_nam[RRD_DS_NAM_SIZE]; /* Name of the data source (null terminated) */
+ gchar dst[RRD_DST_SIZE]; /* Type of data source (null terminated) */
+ rrd_value_t par[10]; /* index of this array see ds_param_en */
+};
+
+/* RRA definition */
+
+enum rrd_cf_type {
+ RRD_CF_INVALID = -1,
+ RRD_CF_AVERAGE = 0, /* data consolidation functions */
+ RRD_CF_MINIMUM,
+ RRD_CF_MAXIMUM,
+ RRD_CF_LAST,
+};
+
+
+#define MAX_RRA_PAR_EN 10
+
+enum rrd_rra_param {
+ RRA_cdp_xff_val = 0, /* what part of the consolidated
+ * datapoint must be known, to produce a
+ * valid entry in the rra */
+};
+
+
+#define RRD_CF_NAM_SIZE 20
+
+struct rrd_rra_def {
+ gchar cf_nam[RRD_CF_NAM_SIZE]; /* consolidation function (null term) */
+ gulong row_cnt; /* number of entries in the store */
+ gulong pdp_cnt; /* how many primary data points are
+ * required for a consolidated data point?*/
+ rrd_value_t par[MAX_RRA_PAR_EN]; /* index see rra_param_en */
+};
+
+struct rrd_live_head {
+ time_t last_up; /* when was rrd last updated */
+ glong last_up_usec; /* micro seconds part of the update timestamp. Always >= 0 */
+};
+
+#define RRD_LAST_DS_LEN 30
+
+enum rrd_pdp_param {
+ PDP_unkn_sec_cnt = 0, /* how many seconds of the current
+ * pdp value is unknown data? */
+ PDP_val
+}; /* current value of the pdp.
+ this depends on dst */
+
+struct rrd_pdp_prep {
+ gchar last_ds[RRD_LAST_DS_LEN]; /* the last reading from the data
+ * source. this is stored in ASCII
+ * to cater for very large counters
+ * we might encounter in connection
+ * with SNMP. */
+ rrd_value_t scratch[10]; /* contents according to pdp_par_en */
+};
+
+#define RRD_MAX_CDP_PAR_EN 10
+#define RRD_MAX_CDP_FAILURES_IDX 8
+/* max CDP scratch entries avail to record violations for a FAILURES RRA */
+#define RRD_MAX_FAILURES_WINDOW_LEN 28
+
+enum rrd_cdp_param {
+ CDP_val = 0,
+ /* the base_interval is always an
+ * average */
+ CDP_unkn_pdp_cnt,
+ /* how many unknown pdp were
+ * integrated. This and the cdp_xff
+ * will decide if this is going to
+ * be a UNKNOWN or a valid value */
+ CDP_hw_intercept,
+ /* Current intercept coefficient for the Holt-Winters
+ * prediction algorithm. */
+ CDP_hw_last_intercept,
+ /* Last iteration intercept coefficient for the Holt-Winters
+ * prediction algorithm. */
+ CDP_hw_slope,
+ /* Current slope coefficient for the Holt-Winters
+ * prediction algorithm. */
+ CDP_hw_last_slope,
+ /* Last iteration slope coefficient. */
+ CDP_null_count,
+ /* Number of sequential Unknown (DNAN) values + 1 preceding
+ * the current prediction.
+ * */
+ CDP_last_null_count,
+ /* Last iteration count of Unknown (DNAN) values. */
+ CDP_primary_val = 8,
+ /* optimization for bulk updates: the value of the first CDP
+ * value to be written in the bulk update. */
+ CDP_secondary_val = 9,
+ /* optimization for bulk updates: the value of subsequent
+ * CDP values to be written in the bulk update. */
+ CDP_hw_seasonal = CDP_hw_intercept,
+ /* Current seasonal coefficient for the Holt-Winters
+ * prediction algorithm. This is stored in CDP prep to avoid
+ * redundant seek operations. */
+ CDP_hw_last_seasonal = CDP_hw_last_intercept,
+ /* Last iteration seasonal coefficient. */
+ CDP_seasonal_deviation = CDP_hw_intercept,
+ CDP_last_seasonal_deviation = CDP_hw_last_intercept,
+ CDP_init_seasonal = CDP_null_count
+};
+
+struct rrd_cdp_prep {
+ rrd_value_t scratch[RRD_MAX_CDP_PAR_EN];
+ /* contents according to cdp_par_en *
+ * init state should be NAN */
+};
+
+struct rrd_rra_ptr {
+ gulong cur_row; /* current row in the rra */
+};
+
+/* Final rrd file structure */
+struct rspamd_rrd_file {
+ struct rrd_file_head *stat_head; /* the static header */
+ struct rrd_ds_def *ds_def; /* list of data source definitions */
+ struct rrd_rra_def *rra_def; /* list of round robin archive def */
+ struct rrd_live_head *live_head; /* rrd v >= 3 last_up with us */
+ struct rrd_pdp_prep *pdp_prep; /* pdp data prep area */
+ struct rrd_cdp_prep *cdp_prep; /* cdp prep area */
+ struct rrd_rra_ptr *rra_ptr; /* list of rra pointers */
+ gdouble *rrd_value; /* list of rrd values */
+
+ gchar *filename;
+ guint8 *map; /* mmapped area */
+ gsize size; /* its size */
+ gboolean finalized;
+ gchar *id;
+ gint fd;
+};
+
+
+/* Public API */
+
+/**
+ * Open (and mmap) existing RRD file
+ * @param filename path
+ * @param err error pointer
+ * @return rrd file structure
+ */
+struct rspamd_rrd_file *rspamd_rrd_open(const gchar *filename, GError **err);
+
+/**
+ * Create basic header for rrd file
+ * @param filename file path
+ * @param ds_count number of data sources
+ * @param rra_count number of round robin archives
+ * @param pdp_step step of primary data points
+ * @param err error pointer
+ * @return TRUE if file has been created
+ */
+struct rspamd_rrd_file *rspamd_rrd_create(const gchar *filename,
+ gulong ds_count,
+ gulong rra_count,
+ gulong pdp_step,
+ gdouble initial_ticks,
+ GError **err);
+
+/**
+ * Add data sources to rrd file
+ * @param filename path to file
+ * @param ds array of struct rrd_ds_def
+ * @param err error pointer
+ * @return TRUE if data sources were added
+ */
+gboolean rspamd_rrd_add_ds(struct rspamd_rrd_file *file,
+ GArray *ds,
+ GError **err);
+
+/**
+ * Add round robin archives to rrd file
+ * @param filename path to file
+ * @param ds array of struct rrd_rra_def
+ * @param err error pointer
+ * @return TRUE if archives were added
+ */
+gboolean rspamd_rrd_add_rra(struct rspamd_rrd_file *file,
+ GArray *rra,
+ GError **err);
+
+/**
+ * Finalize rrd file header and initialize all RRA in the file
+ * @param filename file path
+ * @param err error pointer
+ * @return TRUE if rrd file is ready for use
+ */
+gboolean rspamd_rrd_finalize(struct rspamd_rrd_file *file, GError **err);
+
+/**
+ * Add record to rrd file
+ * @param file rrd file object
+ * @param points points (must be row suitable for this RRA, depending on ds count)
+ * @param err error pointer
+ * @return TRUE if a row has been added
+ */
+gboolean rspamd_rrd_add_record(struct rspamd_rrd_file *file,
+ GArray *points,
+ gdouble ticks,
+ GError **err);
+
+/**
+ * Close rrd file
+ * @param file
+ * @return
+ */
+gint rspamd_rrd_close(struct rspamd_rrd_file *file);
+
+/*
+ * Conversion functions
+ */
+
+/**
+ * Convert rrd dst type from string to numeric value
+ */
+enum rrd_dst_type rrd_dst_from_string(const gchar *str);
+
+/**
+ * Convert numeric presentation of dst to string
+ */
+const gchar *rrd_dst_to_string(enum rrd_dst_type type);
+
+/**
+ * Convert rrd consolidation function type from string to numeric value
+ */
+enum rrd_cf_type rrd_cf_from_string(const gchar *str);
+
+/**
+ * Convert numeric presentation of cf to string
+ */
+const gchar *rrd_cf_to_string(enum rrd_cf_type type);
+
+/* Default RRA and DS */
+
+/**
+ * Create default RRA
+ */
+void rrd_make_default_rra(const gchar *cf_name,
+ gulong pdp_cnt,
+ gulong rows,
+ struct rrd_rra_def *rra);
+
+/**
+ * Create default DS
+ */
+void rrd_make_default_ds(const gchar *name,
+ const gchar *type,
+ gulong pdp_step,
+ struct rrd_ds_def *ds);
+
+/**
+ * Open or create the default rspamd rrd file
+ */
+struct rspamd_rrd_file *rspamd_rrd_file_default(const gchar *path,
+ GError **err);
+
+/**
+ * Returned by querying rrd database
+ */
+struct rspamd_rrd_query_result {
+ gulong rra_rows;
+ gulong pdp_per_cdp;
+ gulong ds_count;
+ gdouble last_update;
+ gulong cur_row;
+ const gdouble *data;
+};
+
+/**
+ * Return RRA data
+ * @param file rrd file
+ * @param rra_num number of rra to return data for
+ * @return query result structure, that should be freed (using g_slice_free1) after usage
+ */
+struct rspamd_rrd_query_result *rspamd_rrd_query(struct rspamd_rrd_file *file,
+ gulong rra_num);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RRD_H_ */
diff --git a/src/libutil/shingles.c b/src/libutil/shingles.c
new file mode 100644
index 0000000..42d5168
--- /dev/null
+++ b/src/libutil/shingles.c
@@ -0,0 +1,412 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "shingles.h"
+#include "fstring.h"
+#include "cryptobox.h"
+#include "images.h"
+#include "libstat/stat_api.h"
+
+#define SHINGLES_WINDOW 3
+#define SHINGLES_KEY_SIZE rspamd_cryptobox_SIPKEYBYTES
+
+static guint
+rspamd_shingles_keys_hash(gconstpointer k)
+{
+ return rspamd_cryptobox_fast_hash(k, SHINGLES_KEY_SIZE,
+ rspamd_hash_seed());
+}
+
+static gboolean
+rspamd_shingles_keys_equal(gconstpointer k1, gconstpointer k2)
+{
+ return (memcmp(k1, k2, SHINGLES_KEY_SIZE) == 0);
+}
+
+static void
+rspamd_shingles_keys_free(gpointer p)
+{
+ guchar **k = p;
+ guint i;
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ g_free(k[i]);
+ }
+
+ g_free(k);
+}
+
+static guchar **
+rspamd_shingles_keys_new(void)
+{
+ guchar **k;
+ guint i;
+
+ k = g_malloc0(sizeof(guchar *) * RSPAMD_SHINGLE_SIZE);
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ k[i] = g_malloc0(sizeof(guchar) * SHINGLES_KEY_SIZE);
+ }
+
+ return k;
+}
+
+static guchar **
+rspamd_shingles_get_keys_cached(const guchar key[SHINGLES_KEY_SIZE])
+{
+ static GHashTable *ht = NULL;
+ guchar **keys = NULL, *key_cpy;
+ rspamd_cryptobox_hash_state_t bs;
+ const guchar *cur_key;
+ guchar shabuf[rspamd_cryptobox_HASHBYTES], *out_key;
+ guint i;
+
+ if (ht == NULL) {
+ ht = g_hash_table_new_full(rspamd_shingles_keys_hash,
+ rspamd_shingles_keys_equal, g_free, rspamd_shingles_keys_free);
+ }
+ else {
+ keys = g_hash_table_lookup(ht, key);
+ }
+
+ if (keys == NULL) {
+ keys = rspamd_shingles_keys_new();
+ key_cpy = g_malloc(SHINGLES_KEY_SIZE);
+ memcpy(key_cpy, key, SHINGLES_KEY_SIZE);
+
+ /* Generate keys */
+ rspamd_cryptobox_hash_init(&bs, NULL, 0);
+ cur_key = key;
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ /*
+ * To generate a set of hashes we just apply sha256 to the
+ * initial key as many times as many hashes are required and
+ * xor left and right parts of sha256 to get a single 16 bytes SIP key.
+ */
+ out_key = keys[i];
+ rspamd_cryptobox_hash_update(&bs, cur_key, 16);
+ rspamd_cryptobox_hash_final(&bs, shabuf);
+
+ memcpy(out_key, shabuf, 16);
+ rspamd_cryptobox_hash_init(&bs, NULL, 0);
+ cur_key = out_key;
+ }
+
+ g_hash_table_insert(ht, key_cpy, keys);
+ }
+
+ return keys;
+}
+
+struct rspamd_shingle *RSPAMD_OPTIMIZE("unroll-loops")
+ rspamd_shingles_from_text(GArray *input,
+ const guchar key[16],
+ rspamd_mempool_t *pool,
+ rspamd_shingles_filter filter,
+ gpointer filterd,
+ enum rspamd_shingle_alg alg)
+{
+ struct rspamd_shingle *res;
+ guint64 **hashes;
+ guchar **keys;
+ rspamd_fstring_t *row;
+ rspamd_stat_token_t *word;
+ guint64 val;
+ gint i, j, k;
+ gsize hlen, ilen = 0, beg = 0, widx = 0;
+ enum rspamd_cryptobox_fast_hash_type ht;
+
+ if (pool != NULL) {
+ res = rspamd_mempool_alloc(pool, sizeof(*res));
+ }
+ else {
+ res = g_malloc(sizeof(*res));
+ }
+
+ row = rspamd_fstring_sized_new(256);
+
+ for (i = 0; i < input->len; i++) {
+ word = &g_array_index(input, rspamd_stat_token_t, i);
+
+ if (!((word->flags & RSPAMD_STAT_TOKEN_FLAG_SKIPPED) || word->stemmed.len == 0)) {
+ ilen++;
+ }
+ }
+
+ /* Init hashes pipes and keys */
+ hashes = g_malloc(sizeof(*hashes) * RSPAMD_SHINGLE_SIZE);
+ hlen = ilen > SHINGLES_WINDOW ? (ilen - SHINGLES_WINDOW + 1) : 1;
+ keys = rspamd_shingles_get_keys_cached(key);
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ hashes[i] = g_malloc(hlen * sizeof(guint64));
+ }
+
+ /* Now parse input words into a vector of hashes using rolling window */
+ if (alg == RSPAMD_SHINGLES_OLD) {
+ for (i = 0; i <= (gint) ilen; i++) {
+ if (i - beg >= SHINGLES_WINDOW || i == (gint) ilen) {
+ for (j = beg; j < i; j++) {
+
+ word = NULL;
+ while (widx < input->len) {
+ word = &g_array_index(input, rspamd_stat_token_t, widx);
+
+ if ((word->flags & RSPAMD_STAT_TOKEN_FLAG_SKIPPED) || word->stemmed.len == 0) {
+ widx++;
+ }
+ else {
+ break;
+ }
+ }
+
+ if (word == NULL) {
+ /* Nothing but exceptions */
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ g_free(hashes[i]);
+ }
+
+ g_free(hashes);
+
+ if (pool == NULL) {
+ g_free(res);
+ }
+
+ rspamd_fstring_free(row);
+
+ return NULL;
+ }
+
+ row = rspamd_fstring_append(row, word->stemmed.begin,
+ word->stemmed.len);
+ }
+
+ /* Now we need to create a new row here */
+ for (j = 0; j < RSPAMD_SHINGLE_SIZE; j++) {
+ rspamd_cryptobox_siphash((guchar *) &val, row->str, row->len,
+ keys[j]);
+ g_assert(hlen > beg);
+ hashes[j][beg] = val;
+ }
+
+ beg++;
+ widx++;
+
+ row = rspamd_fstring_assign(row, "", 0);
+ }
+ }
+ }
+ else {
+ guint64 window[SHINGLES_WINDOW * RSPAMD_SHINGLE_SIZE], seed;
+
+ switch (alg) {
+ case RSPAMD_SHINGLES_XXHASH:
+ ht = RSPAMD_CRYPTOBOX_XXHASH64;
+ break;
+ case RSPAMD_SHINGLES_MUMHASH:
+ ht = RSPAMD_CRYPTOBOX_MUMHASH;
+ break;
+ default:
+ ht = RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT;
+ break;
+ }
+
+ memset(window, 0, sizeof(window));
+ for (i = 0; i <= ilen; i++) {
+ if (i - beg >= SHINGLES_WINDOW || i == ilen) {
+
+ for (j = 0; j < RSPAMD_SHINGLE_SIZE; j++) {
+ /* Shift hashes window to right */
+ for (k = 0; k < SHINGLES_WINDOW - 1; k++) {
+ window[j * SHINGLES_WINDOW + k] =
+ window[j * SHINGLES_WINDOW + k + 1];
+ }
+
+ word = NULL;
+
+ while (widx < input->len) {
+ word = &g_array_index(input, rspamd_stat_token_t, widx);
+
+ if ((word->flags & RSPAMD_STAT_TOKEN_FLAG_SKIPPED) || word->stemmed.len == 0) {
+ widx++;
+ }
+ else {
+ break;
+ }
+ }
+
+ if (word == NULL) {
+ /* Nothing but exceptions */
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ g_free(hashes[i]);
+ }
+
+ if (pool == NULL) {
+ g_free(res);
+ }
+
+ g_free(hashes);
+ rspamd_fstring_free(row);
+
+ return NULL;
+ }
+
+ /* Insert the last element to the pipe */
+ memcpy(&seed, keys[j], sizeof(seed));
+ window[j * SHINGLES_WINDOW + SHINGLES_WINDOW - 1] =
+ rspamd_cryptobox_fast_hash_specific(ht,
+ word->stemmed.begin, word->stemmed.len,
+ seed);
+ val = 0;
+ for (k = 0; k < SHINGLES_WINDOW; k++) {
+ val ^= window[j * SHINGLES_WINDOW + k] >>
+ (8 * (SHINGLES_WINDOW - k - 1));
+ }
+
+ g_assert(hlen > beg);
+ hashes[j][beg] = val;
+ }
+
+ beg++;
+ widx++;
+ }
+ }
+ }
+
+ /* Now we need to filter all hashes and make a shingles result */
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ res->hashes[i] = filter(hashes[i], hlen,
+ i, key, filterd);
+ g_free(hashes[i]);
+ }
+
+ g_free(hashes);
+
+ rspamd_fstring_free(row);
+
+ return res;
+}
+
+struct rspamd_shingle *RSPAMD_OPTIMIZE("unroll-loops")
+ rspamd_shingles_from_image(guchar *dct,
+ const guchar key[16],
+ rspamd_mempool_t *pool,
+ rspamd_shingles_filter filter,
+ gpointer filterd,
+ enum rspamd_shingle_alg alg)
+{
+ struct rspamd_shingle *shingle;
+ guint64 **hashes;
+ guchar **keys;
+ guint64 d;
+ guint64 val;
+ gint i, j;
+ gsize hlen, beg = 0;
+ enum rspamd_cryptobox_fast_hash_type ht;
+ guint64 res[SHINGLES_WINDOW * RSPAMD_SHINGLE_SIZE], seed;
+
+ if (pool != NULL) {
+ shingle = rspamd_mempool_alloc(pool, sizeof(*shingle));
+ }
+ else {
+ shingle = g_malloc(sizeof(*shingle));
+ }
+
+ /* Init hashes pipes and keys */
+ hashes = g_malloc(sizeof(*hashes) * RSPAMD_SHINGLE_SIZE);
+ hlen = RSPAMD_DCT_LEN / NBBY + 1;
+ keys = rspamd_shingles_get_keys_cached(key);
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ hashes[i] = g_malloc(hlen * sizeof(guint64));
+ }
+
+ switch (alg) {
+ case RSPAMD_SHINGLES_OLD:
+ ht = RSPAMD_CRYPTOBOX_MUMHASH;
+ break;
+ case RSPAMD_SHINGLES_XXHASH:
+ ht = RSPAMD_CRYPTOBOX_XXHASH64;
+ break;
+ case RSPAMD_SHINGLES_MUMHASH:
+ ht = RSPAMD_CRYPTOBOX_MUMHASH;
+ break;
+ default:
+ ht = RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT;
+ break;
+ }
+
+ memset(res, 0, sizeof(res));
+#define INNER_CYCLE_SHINGLES(s, e) \
+ for (j = (s); j < (e); j++) { \
+ d = dct[beg]; \
+ memcpy(&seed, keys[j], sizeof(seed)); \
+ val = rspamd_cryptobox_fast_hash_specific(ht, \
+ &d, sizeof(d), \
+ seed); \
+ hashes[j][beg] = val; \
+ }
+ for (i = 0; i < RSPAMD_DCT_LEN / NBBY; i++) {
+ INNER_CYCLE_SHINGLES(0, RSPAMD_SHINGLE_SIZE / 4);
+ INNER_CYCLE_SHINGLES(RSPAMD_SHINGLE_SIZE / 4, RSPAMD_SHINGLE_SIZE / 2);
+ INNER_CYCLE_SHINGLES(RSPAMD_SHINGLE_SIZE / 2, 3 * RSPAMD_SHINGLE_SIZE / 4);
+ INNER_CYCLE_SHINGLES(3 * RSPAMD_SHINGLE_SIZE / 4, RSPAMD_SHINGLE_SIZE);
+
+ beg++;
+ }
+#undef INNER_CYCLE_SHINGLES
+ /* Now we need to filter all hashes and make a shingles result */
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ shingle->hashes[i] = filter(hashes[i], hlen,
+ i, key, filterd);
+ g_free(hashes[i]);
+ }
+
+ g_free(hashes);
+
+ return shingle;
+}
+
+guint64
+rspamd_shingles_default_filter(guint64 *input, gsize count,
+ gint shno, const guchar *key, gpointer ud)
+{
+ guint64 minimal = G_MAXUINT64;
+ gsize i;
+
+ for (i = 0; i < count; i++) {
+ if (minimal > input[i]) {
+ minimal = input[i];
+ }
+ }
+
+ return minimal;
+}
+
+
+gdouble rspamd_shingles_compare(const struct rspamd_shingle *a,
+ const struct rspamd_shingle *b)
+{
+ gint i, common = 0;
+
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ if (a->hashes[i] == b->hashes[i]) {
+ common++;
+ }
+ }
+
+ return (gdouble) common / (gdouble) RSPAMD_SHINGLE_SIZE;
+}
diff --git a/src/libutil/shingles.h b/src/libutil/shingles.h
new file mode 100644
index 0000000..9a0ca69
--- /dev/null
+++ b/src/libutil/shingles.h
@@ -0,0 +1,101 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SHINGLES_H_
+#define SHINGLES_H_
+
+#include "config.h"
+#include "mem_pool.h"
+
+#define RSPAMD_SHINGLE_SIZE 32
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_shingle {
+ guint64 hashes[RSPAMD_SHINGLE_SIZE];
+};
+
+enum rspamd_shingle_alg {
+ RSPAMD_SHINGLES_OLD = 0,
+ RSPAMD_SHINGLES_XXHASH,
+ RSPAMD_SHINGLES_MUMHASH,
+ RSPAMD_SHINGLES_FAST
+};
+
+/**
+ * Shingles filtering function
+ * @param input input array of hashes
+ * @param count number of hashes in the vector
+ * @return shingle value
+ */
+typedef guint64 (*rspamd_shingles_filter)(guint64 *input, gsize count,
+ gint shno, const guchar *key, gpointer ud);
+
+/**
+ * Generate shingles from the input of fixed size strings using lemmatizer
+ * if needed
+ * @param input array of `rspamd_fstring_t`
+ * @param key secret key used to generate shingles
+ * @param pool pool to allocate shingles array
+ * @param filter hashes filtering function
+ * @param filterd opaque data for filtering function
+ * @return shingles array
+ */
+struct rspamd_shingle *rspamd_shingles_from_text(GArray *input,
+ const guchar key[16],
+ rspamd_mempool_t *pool,
+ rspamd_shingles_filter filter,
+ gpointer filterd,
+ enum rspamd_shingle_alg alg);
+
+/**
+ * Generate shingles from the DCT matrix of an image
+ * @param dct discrete cosine transfor matrix (must be 64x64)
+ * @param key secret key used to generate shingles
+ * @param pool pool to allocate shingles array
+ * @param filter hashes filtering function
+ * @param filterd opaque data for filtering function
+ * @return shingles array
+ */
+struct rspamd_shingle *rspamd_shingles_from_image(guchar *dct,
+ const guchar key[16],
+ rspamd_mempool_t *pool,
+ rspamd_shingles_filter filter,
+ gpointer filterd,
+ enum rspamd_shingle_alg alg);
+
+/**
+ * Compares two shingles and return result as a floating point value - 1.0
+ * for completely similar shingles and 0.0 for completely different ones
+ * @param a
+ * @param b
+ * @return
+ */
+gdouble rspamd_shingles_compare(const struct rspamd_shingle *a,
+ const struct rspamd_shingle *b);
+
+/**
+ * Default filtering function
+ */
+guint64 rspamd_shingles_default_filter(guint64 *input, gsize count,
+ gint shno, const guchar *key, gpointer ud);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SHINGLES_H_ */
diff --git a/src/libutil/sqlite_utils.c b/src/libutil/sqlite_utils.c
new file mode 100644
index 0000000..8aeb598
--- /dev/null
+++ b/src/libutil/sqlite_utils.c
@@ -0,0 +1,620 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "libserver/logger.h"
+#include "libutil/sqlite_utils.h"
+#include "unix-std.h"
+
+
+static GQuark
+rspamd_sqlite3_quark(void)
+{
+ return g_quark_from_static_string("rspamd-sqlite3");
+}
+
+GArray *
+rspamd_sqlite3_init_prstmt(sqlite3 *db,
+ struct rspamd_sqlite3_prstmt *init_stmt,
+ gint max_idx,
+ GError **err)
+{
+ gint i;
+ GArray *res;
+ struct rspamd_sqlite3_prstmt *nst;
+
+ res = g_array_sized_new(FALSE, TRUE, sizeof(struct rspamd_sqlite3_prstmt),
+ max_idx);
+ g_array_set_size(res, max_idx);
+
+ for (i = 0; i < max_idx; i++) {
+ nst = &g_array_index(res, struct rspamd_sqlite3_prstmt, i);
+ memcpy(nst, &init_stmt[i], sizeof(*nst));
+
+ if (sqlite3_prepare_v2(db, init_stmt[i].sql, -1,
+ &nst->stmt, NULL) != SQLITE_OK) {
+ g_set_error(err, rspamd_sqlite3_quark(),
+ -1, "Cannot initialize prepared sql `%s`: %s",
+ nst->sql, sqlite3_errmsg(db));
+ rspamd_sqlite3_close_prstmt(db, res);
+
+ return NULL;
+ }
+ }
+
+ return res;
+}
+
+int rspamd_sqlite3_run_prstmt(rspamd_mempool_t *pool, sqlite3 *db, GArray *stmts,
+ gint idx, ...)
+{
+ gint retcode;
+ va_list ap;
+ sqlite3_stmt *stmt;
+ gint i, rowid, nargs, j;
+ gint64 len;
+ gpointer p;
+ struct rspamd_sqlite3_prstmt *nst;
+ const char *argtypes;
+
+ if (idx < 0 || idx >= (gint) stmts->len) {
+
+ return -1;
+ }
+
+ nst = &g_array_index(stmts, struct rspamd_sqlite3_prstmt, idx);
+ stmt = nst->stmt;
+
+ g_assert(nst != NULL);
+
+ msg_debug_pool("executing `%s`", nst->sql);
+ argtypes = nst->args;
+ sqlite3_clear_bindings(stmt);
+ sqlite3_reset(stmt);
+ va_start(ap, idx);
+ nargs = 1;
+
+ for (i = 0, rowid = 1; argtypes[i] != '\0'; i++) {
+ switch (argtypes[i]) {
+ case 'T':
+
+ for (j = 0; j < nargs; j++, rowid++) {
+ sqlite3_bind_text(stmt, rowid, va_arg(ap, const char *), -1,
+ SQLITE_STATIC);
+ }
+
+ nargs = 1;
+ break;
+ case 'V':
+ case 'B':
+
+ for (j = 0; j < nargs; j++, rowid++) {
+ len = va_arg(ap, gint64);
+ sqlite3_bind_text(stmt, rowid, va_arg(ap, const char *), len,
+ SQLITE_STATIC);
+ }
+
+ nargs = 1;
+ break;
+ case 'I':
+
+ for (j = 0; j < nargs; j++, rowid++) {
+ sqlite3_bind_int64(stmt, rowid, va_arg(ap, gint64));
+ }
+
+ nargs = 1;
+ break;
+ case 'S':
+
+ for (j = 0; j < nargs; j++, rowid++) {
+ sqlite3_bind_int(stmt, rowid, va_arg(ap, gint));
+ }
+
+ nargs = 1;
+ break;
+ case '*':
+ nargs = va_arg(ap, gint);
+ break;
+ }
+ }
+
+ retcode = sqlite3_step(stmt);
+
+ if (retcode == nst->result) {
+ argtypes = nst->ret;
+
+ for (i = 0; argtypes != NULL && argtypes[i] != '\0'; i++) {
+ switch (argtypes[i]) {
+ case 'T':
+ *va_arg(ap, char **) = g_strdup(sqlite3_column_text(stmt, i));
+ break;
+ case 'I':
+ *va_arg(ap, gint64 *) = sqlite3_column_int64(stmt, i);
+ break;
+ case 'S':
+ *va_arg(ap, int *) = sqlite3_column_int(stmt, i);
+ break;
+ case 'L':
+ *va_arg(ap, gint64 *) = sqlite3_last_insert_rowid(db);
+ break;
+ case 'B':
+ len = sqlite3_column_bytes(stmt, i);
+ g_assert(len >= 0);
+ p = g_malloc(len);
+ memcpy(p, sqlite3_column_blob(stmt, i), len);
+ *va_arg(ap, gint64 *) = len;
+ *va_arg(ap, gpointer *) = p;
+ break;
+ }
+ }
+
+ if (!(nst->flags & RSPAMD_SQLITE3_STMT_MULTIPLE)) {
+ sqlite3_clear_bindings(stmt);
+ sqlite3_reset(stmt);
+ }
+
+ va_end(ap);
+
+ return SQLITE_OK;
+ }
+ else if (retcode != SQLITE_DONE && retcode != SQLITE_OK && retcode != SQLITE_ROW) {
+ msg_warn_pool("failed to execute query %s: %d, %s", nst->sql,
+ retcode, sqlite3_errmsg(db));
+ }
+
+ if (!(nst->flags & RSPAMD_SQLITE3_STMT_MULTIPLE)) {
+ sqlite3_clear_bindings(stmt);
+ sqlite3_reset(stmt);
+ }
+
+ va_end(ap);
+
+ return retcode;
+}
+
+void rspamd_sqlite3_close_prstmt(sqlite3 *db, GArray *stmts)
+{
+ guint i;
+ struct rspamd_sqlite3_prstmt *nst;
+
+ for (i = 0; i < stmts->len; i++) {
+ nst = &g_array_index(stmts, struct rspamd_sqlite3_prstmt, i);
+ if (nst->stmt != NULL) {
+ sqlite3_finalize(nst->stmt);
+ }
+ }
+
+ g_array_free(stmts, TRUE);
+
+ return;
+}
+
+static gboolean
+rspamd_sqlite3_wait(rspamd_mempool_t *pool, const gchar *lock)
+{
+ gint fd;
+ pid_t pid;
+ gssize r;
+ struct timespec sleep_ts = {
+ .tv_sec = 0,
+ .tv_nsec = 1000000};
+
+ while ((fd = open(lock, O_WRONLY | O_CREAT | O_EXCL, 00600)) == -1) {
+ if (errno != EBUSY && errno != EEXIST) {
+ msg_err_pool_check("cannot open lock file %s: %s", lock,
+ strerror(errno));
+
+ return FALSE;
+ }
+
+ fd = open(lock, O_RDONLY);
+
+ if (fd == -1) {
+ msg_err_pool_check("cannot open lock file %s: %s", lock,
+ strerror(errno));
+
+ return FALSE;
+ }
+
+ r = read(fd, &pid, sizeof(pid));
+
+ if (r != sizeof(pid)) {
+ msg_warn_pool_check("stale lock file %s, removing", lock);
+ unlink(lock);
+ close(fd);
+
+ return TRUE;
+ }
+
+ /* Now check for process existence */
+ if (pid == getpid()) {
+ msg_warn_pool_check("lock file %s, belongs to me, removing", lock);
+ unlink(lock);
+ close(fd);
+
+ return TRUE;
+ }
+ else if (kill(pid, 0) == -1) {
+ if (errno == ESRCH) {
+ /* Process is already dead */
+ msg_warn_pool_check("stale lock file %s from pid: %P, removing",
+ lock, pid);
+ unlink(lock);
+ close(fd);
+
+ return TRUE;
+ }
+ }
+
+ close(fd);
+
+ if (nanosleep(&sleep_ts, NULL) == -1 && errno != EINTR) {
+ msg_err_pool_check("cannot sleep open lock file %s: %s", lock,
+ strerror(errno));
+
+ return FALSE;
+ }
+ }
+
+ unlink(lock);
+ close(fd);
+
+ return TRUE;
+}
+
+#define RSPAMD_SQLITE_MMAP_LIMIT 268435456
+#define RSPAMD_SQLITE_CACHE_SIZE 262144
+
+sqlite3 *
+rspamd_sqlite3_open_or_create(rspamd_mempool_t *pool, const gchar *path, const gchar *create_sql, guint version, GError **err)
+{
+ sqlite3 *sqlite;
+ gint rc, flags, lock_fd;
+ gchar lock_path[PATH_MAX], dbdir[PATH_MAX], *pdir;
+ static const char sqlite_wal[] =
+ "PRAGMA journal_mode=\"wal\";"
+ "PRAGMA wal_autocheckpoint = 16;"
+ "PRAGMA journal_size_limit = 1536;",
+ exclusive_lock_sql[] = "PRAGMA locking_mode=\"exclusive\";",
+
+ fsync_sql[] = "PRAGMA synchronous=\"NORMAL\";",
+
+ foreign_keys[] = "PRAGMA foreign_keys=\"ON\";",
+
+#if defined(__LP64__) || defined(_LP64)
+ enable_mmap[] = "PRAGMA mmap_size=" G_STRINGIFY(RSPAMD_SQLITE_MMAP_LIMIT) ";",
+#endif
+
+ other_pragmas[] = "PRAGMA read_uncommitted=\"ON\";"
+ "PRAGMA cache_size=" G_STRINGIFY(RSPAMD_SQLITE_CACHE_SIZE) ";",
+ db_version[] = "PRAGMA user_version;";
+ gboolean create = FALSE, has_lock = FALSE;
+
+ flags = SQLITE_OPEN_READWRITE;
+#ifdef SQLITE_OPEN_SHAREDCACHE
+ flags |= SQLITE_OPEN_SHAREDCACHE;
+#endif
+#ifdef SQLITE_OPEN_WAL
+ flags |= SQLITE_OPEN_WAL;
+#endif
+
+ rspamd_strlcpy(dbdir, path, sizeof(dbdir));
+ pdir = dirname(dbdir);
+
+ if (access(pdir, W_OK) == -1) {
+ g_set_error(err, rspamd_sqlite3_quark(),
+ errno, "cannot open sqlite directory %s: %s",
+ pdir, strerror(errno));
+
+ return NULL;
+ }
+
+ rspamd_snprintf(lock_path, sizeof(lock_path), "%s.lock", path);
+
+ if (access(path, R_OK) == -1) {
+ flags |= SQLITE_OPEN_CREATE;
+ create = TRUE;
+ }
+
+
+ rspamd_snprintf(lock_path, sizeof(lock_path), "%s.lock", path);
+ lock_fd = open(lock_path, O_WRONLY | O_CREAT | O_EXCL, 00600);
+
+ if (lock_fd == -1) {
+ if (errno == EEXIST || errno == EBUSY) {
+ msg_debug_pool_check("checking %s to wait for db being initialized", lock_path);
+
+ if (!rspamd_sqlite3_wait(pool, lock_path)) {
+ g_set_error(err, rspamd_sqlite3_quark(),
+ errno, "cannot create sqlite file %s: %s",
+ path, strerror(errno));
+
+ return NULL;
+ }
+
+
+ /* At this point we have database created */
+ create = FALSE;
+ has_lock = FALSE;
+ }
+ else {
+ g_set_error(err, rspamd_sqlite3_quark(),
+ errno, "cannot lock sqlite file %s: %s",
+ path, strerror(errno));
+ }
+ }
+ else {
+ pid_t myself = getpid();
+ msg_debug_pool_check("locking %s to block other processes", lock_path);
+ (void) write(lock_fd, &myself, sizeof(myself));
+
+ g_assert(rspamd_file_lock(lock_fd, FALSE));
+ has_lock = TRUE;
+ }
+
+ if ((rc = sqlite3_open_v2(path, &sqlite,
+ flags, NULL)) != SQLITE_OK) {
+#if SQLITE_VERSION_NUMBER >= 3008000
+ g_set_error(err, rspamd_sqlite3_quark(),
+ rc, "cannot open sqlite db %s: %s",
+ path, sqlite3_errstr(rc));
+#else
+ g_set_error(err, rspamd_sqlite3_quark(),
+ rc, "cannot open sqlite db %s: %d",
+ path, rc);
+#endif
+
+ if (has_lock && lock_fd != -1) {
+ msg_debug_pool_check("removing lock from %s", lock_path);
+ rspamd_file_unlock(lock_fd, FALSE);
+ unlink(lock_path);
+ close(lock_fd);
+ }
+
+ return NULL;
+ }
+
+ if (create && has_lock) {
+ while ((rc = sqlite3_exec(sqlite, sqlite_wal, NULL, NULL, NULL)) != SQLITE_OK) {
+ if (rc == SQLITE_BUSY) {
+ struct timespec sleep_ts = {
+ .tv_sec = 0,
+ .tv_nsec = 1000000};
+
+ nanosleep(&sleep_ts, NULL);
+
+ continue;
+ }
+
+ msg_warn_pool_check("WAL mode is not supported (%s), locking issues might occur",
+ sqlite3_errmsg(sqlite));
+ break;
+ }
+
+ if (sqlite3_exec(sqlite, exclusive_lock_sql, NULL, NULL, NULL) != SQLITE_OK) {
+ msg_warn_pool_check("cannot exclusively lock database to create schema: %s",
+ sqlite3_errmsg(sqlite));
+ }
+
+ if (create_sql) {
+ while ((rc = sqlite3_exec(sqlite, create_sql, NULL, NULL, NULL)) != SQLITE_OK) {
+ if (rc == SQLITE_BUSY) {
+ struct timespec sleep_ts = {
+ .tv_sec = 0,
+ .tv_nsec = 1000000};
+
+ nanosleep(&sleep_ts, NULL);
+
+ continue;
+ }
+
+ g_set_error(err, rspamd_sqlite3_quark(),
+ -1, "cannot execute create sql `%s`: %s",
+ create_sql, sqlite3_errmsg(sqlite));
+ sqlite3_close(sqlite);
+ rspamd_file_unlock(lock_fd, FALSE);
+ unlink(lock_path);
+ if (lock_fd != -1) {
+ close(lock_fd);
+ }
+
+ return NULL;
+ }
+ }
+
+ sqlite3_close(sqlite);
+
+ /* Reopen in normal mode */
+ msg_debug_pool_check("reopening %s in normal mode", path);
+ flags &= ~SQLITE_OPEN_CREATE;
+
+ if ((rc = sqlite3_open_v2(path, &sqlite,
+ flags, NULL)) != SQLITE_OK) {
+#if SQLITE_VERSION_NUMBER >= 3008000
+ g_set_error(err, rspamd_sqlite3_quark(),
+ rc, "cannot open sqlite db after creation %s: %s",
+ path, sqlite3_errstr(rc));
+#else
+ g_set_error(err, rspamd_sqlite3_quark(),
+ rc, "cannot open sqlite db after creation %s: %d",
+ path, rc);
+#endif
+ rspamd_file_unlock(lock_fd, FALSE);
+ unlink(lock_path);
+
+ if (lock_fd != -1) {
+ close(lock_fd);
+ }
+
+ return NULL;
+ }
+ }
+ else if (has_lock && version > 0) {
+ /* Check user version */
+ sqlite3_stmt *stmt = NULL;
+ guint32 db_ver;
+ GString *new_ver_sql;
+
+ if (sqlite3_prepare(sqlite, db_version, -1, &stmt, NULL) != SQLITE_OK) {
+ msg_warn_pool_check("Cannot get user version pragma: %s",
+ sqlite3_errmsg(sqlite));
+ }
+ else {
+ if (sqlite3_step(stmt) != SQLITE_ROW) {
+ msg_warn_pool_check("Cannot get user version pragma, step failed: %s",
+ sqlite3_errmsg(sqlite));
+ sqlite3_finalize(stmt);
+ }
+ else {
+ db_ver = sqlite3_column_int(stmt, 0);
+ sqlite3_reset(stmt);
+ sqlite3_finalize(stmt);
+
+ if (version > db_ver) {
+ msg_warn_pool_check("Database version %ud is less than "
+ "desired version %ud, run create script",
+ db_ver,
+ version);
+
+ if (create_sql) {
+ if (sqlite3_exec(sqlite, create_sql, NULL, NULL, NULL) != SQLITE_OK) {
+ g_set_error(err, rspamd_sqlite3_quark(),
+ -1, "cannot execute create sql `%s`: %s",
+ create_sql, sqlite3_errmsg(sqlite));
+ sqlite3_close(sqlite);
+ rspamd_file_unlock(lock_fd, FALSE);
+ unlink(lock_path);
+ if (lock_fd != -1) {
+ close(lock_fd);
+ }
+
+ return NULL;
+ }
+ }
+
+ new_ver_sql = g_string_new("PRAGMA user_version=");
+ rspamd_printf_gstring(new_ver_sql, "%ud", version);
+
+ if (sqlite3_exec(sqlite, new_ver_sql->str, NULL, NULL, NULL) != SQLITE_OK) {
+ g_set_error(err, rspamd_sqlite3_quark(),
+ -1, "cannot execute update version sql `%s`: %s",
+ new_ver_sql->str, sqlite3_errmsg(sqlite));
+ sqlite3_close(sqlite);
+ rspamd_file_unlock(lock_fd, FALSE);
+ unlink(lock_path);
+ if (lock_fd != -1) {
+ close(lock_fd);
+ }
+
+ g_string_free(new_ver_sql, TRUE);
+
+ return NULL;
+ }
+
+ g_string_free(new_ver_sql, TRUE);
+ }
+ else if (db_ver > version) {
+ msg_warn_pool_check("Database version %ud is more than "
+ "desired version %ud, this could cause"
+ " unexpected behaviour",
+ db_ver,
+ version);
+ }
+ }
+ }
+ }
+
+ while ((rc = sqlite3_exec(sqlite, sqlite_wal, NULL, NULL, NULL)) != SQLITE_OK) {
+ if (rc == SQLITE_BUSY) {
+ struct timespec sleep_ts = {
+ .tv_sec = 0,
+ .tv_nsec = 1000000};
+
+ nanosleep(&sleep_ts, NULL);
+
+ continue;
+ }
+
+ msg_warn_pool_check("WAL mode is not supported (%s), locking issues might occur",
+ sqlite3_errmsg(sqlite));
+ break;
+ }
+
+ if (sqlite3_exec(sqlite, fsync_sql, NULL, NULL, NULL) != SQLITE_OK) {
+ msg_warn_pool_check("cannot set synchronous: %s",
+ sqlite3_errmsg(sqlite));
+ }
+
+ if ((rc = sqlite3_exec(sqlite, foreign_keys, NULL, NULL, NULL)) !=
+ SQLITE_OK) {
+ msg_warn_pool_check("cannot enable foreign keys: %s",
+ sqlite3_errmsg(sqlite));
+ }
+
+#if defined(__LP64__) || defined(_LP64)
+ if ((rc = sqlite3_exec(sqlite, enable_mmap, NULL, NULL, NULL)) != SQLITE_OK) {
+ msg_warn_pool_check("cannot enable mmap: %s",
+ sqlite3_errmsg(sqlite));
+ }
+#endif
+
+ if ((rc = sqlite3_exec(sqlite, other_pragmas, NULL, NULL, NULL)) !=
+ SQLITE_OK) {
+ msg_warn_pool_check("cannot execute tuning pragmas: %s",
+ sqlite3_errmsg(sqlite));
+ }
+
+ if (has_lock && lock_fd != -1) {
+ msg_debug_pool_check("removing lock from %s", lock_path);
+ rspamd_file_unlock(lock_fd, FALSE);
+ unlink(lock_path);
+ close(lock_fd);
+ }
+
+ return sqlite;
+}
+
+gboolean
+rspamd_sqlite3_sync(sqlite3 *db, gint *wal_frames, gint *wal_checkpoints)
+{
+ gint wf = 0, wc = 0, mode;
+
+#ifdef SQLITE_OPEN_WAL
+#ifdef SQLITE_CHECKPOINT_TRUNCATE
+ mode = SQLITE_CHECKPOINT_TRUNCATE;
+#elif defined(SQLITE_CHECKPOINT_RESTART)
+ mode = SQLITE_CHECKPOINT_RESTART;
+#elif defined(SQLITE_CHECKPOINT_FULL)
+ mode = SQLITE_CHECKPOINT_FULL;
+#endif
+ /* Perform wal checkpoint (might be long) */
+ if (sqlite3_wal_checkpoint_v2(db,
+ NULL,
+ mode,
+ &wf,
+ &wc) != SQLITE_OK) {
+ return FALSE;
+ }
+#endif
+
+ if (wal_frames) {
+ *wal_frames = wf;
+ }
+ if (wal_checkpoints) {
+ *wal_checkpoints = wc;
+ }
+
+ return TRUE;
+}
diff --git a/src/libutil/sqlite_utils.h b/src/libutil/sqlite_utils.h
new file mode 100644
index 0000000..5411a47
--- /dev/null
+++ b/src/libutil/sqlite_utils.h
@@ -0,0 +1,90 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBUTIL_SQLITE_UTILS_H_
+#define SRC_LIBUTIL_SQLITE_UTILS_H_
+
+#include "config.h"
+#include "mem_pool.h"
+#include "sqlite3.h"
+
+#define RSPAMD_SQLITE3_STMT_MULTIPLE (1 << 0)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_sqlite3_prstmt {
+ gint idx;
+ const gchar *sql;
+ const gchar *args;
+ sqlite3_stmt *stmt;
+ gint result;
+ const gchar *ret;
+ gint flags;
+};
+
+/**
+ * Create prepared statements for specified database from init statements
+ * @param db
+ * @param max_idx
+ * @param err
+ * @return new prepared statements array or NULL
+ */
+GArray *rspamd_sqlite3_init_prstmt(sqlite3 *db,
+ struct rspamd_sqlite3_prstmt *init_stmt,
+ gint max_idx,
+ GError **err);
+
+/**
+ * Run prepared statements by its index getting parameters and setting results from
+ * varargs structure
+ * @param db
+ * @param stmts
+ * @param idx
+ * @return
+ */
+gint rspamd_sqlite3_run_prstmt(rspamd_mempool_t *pool, sqlite3 *db, GArray *stmts,
+ gint idx, ...);
+
+/**
+ * Close and free prepared statements
+ * @param db
+ * @param stmts
+ */
+void rspamd_sqlite3_close_prstmt(sqlite3 *db, GArray *stmts);
+
+/**
+ * Creates or opens sqlite database trying to share it between processes
+ * @param path
+ * @param create_sql
+ * @return
+ */
+sqlite3 *rspamd_sqlite3_open_or_create(rspamd_mempool_t *pool,
+ const gchar *path, const gchar *create_sql,
+ guint32 version, GError **err);
+
+
+/**
+ * Sync sqlite3 db ensuring that all wal things are done
+ * @param db
+ */
+gboolean rspamd_sqlite3_sync(sqlite3 *db, gint *wal_frames, gint *wal_checkpoints);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBUTIL_SQLITE_UTILS_H_ */
diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c
new file mode 100644
index 0000000..bc99f2a
--- /dev/null
+++ b/src/libutil/str_util.c
@@ -0,0 +1,3886 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "util.h"
+#include "cryptobox.h"
+#include "url.h"
+#include "str_util.h"
+#include "logger.h"
+#include "contrib/t1ha/t1ha.h"
+#include <unicode/uversion.h>
+#include <unicode/ucnv.h>
+#if U_ICU_VERSION_MAJOR_NUM >= 44
+#include <unicode/unorm2.h>
+#endif
+#include <math.h>
+
+#ifdef __x86_64__
+#include <immintrin.h>
+#endif
+
+#include "contrib/fastutf8/fastutf8.h"
+
+const guchar lc_map[256] = {
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
+ 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
+ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+ 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
+ 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+ 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
+ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
+ 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
+ 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+ 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
+ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
+ 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f,
+ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+ 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+ 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
+ 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+ 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+ 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
+ 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
+ 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+ 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+ 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+ 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
+ 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
+ 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
+ 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+ 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff};
+
+guint rspamd_str_lc(gchar *str, guint size)
+{
+ guint leftover = size % 4;
+ guint fp, i;
+ const uint8_t *s = (const uint8_t *) str;
+ gchar *dest = str;
+ guchar c1, c2, c3, c4;
+
+ fp = size - leftover;
+
+ for (i = 0; i != fp; i += 4) {
+ c1 = s[i], c2 = s[i + 1], c3 = s[i + 2], c4 = s[i + 3];
+ dest[0] = lc_map[c1];
+ dest[1] = lc_map[c2];
+ dest[2] = lc_map[c3];
+ dest[3] = lc_map[c4];
+ dest += 4;
+ }
+
+ switch (leftover) {
+ case 3:
+ *dest++ = lc_map[(guchar) str[i++]];
+ /* FALLTHRU */
+ case 2:
+ *dest++ = lc_map[(guchar) str[i++]];
+ /* FALLTHRU */
+ case 1:
+ *dest = lc_map[(guchar) str[i]];
+ }
+
+ return size;
+}
+
+gsize rspamd_str_copy_lc(const gchar *src, gchar *dst, gsize size)
+{
+ gchar *d = dst;
+
+ /* Find aligned start */
+ while ((0xf & (uintptr_t) src) && size > 0) {
+ *d++ = lc_map[(guchar) *src++];
+ size--;
+ }
+
+ /* Aligned start in src */
+#ifdef __x86_64__
+ while (size >= 16) {
+ __m128i sv = _mm_load_si128((const __m128i *) src);
+ /* From A */
+ __m128i rangeshift = _mm_sub_epi8(sv, _mm_set1_epi8((char) ('A' + 128)));
+ /* To Z */
+ __m128i nomodify = _mm_cmpgt_epi8(rangeshift, _mm_set1_epi8(-128 + 25));
+ /* ^ ' ' */
+ __m128i flip = _mm_andnot_si128(nomodify, _mm_set1_epi8(0x20));
+ __m128i uc = _mm_xor_si128(sv, flip);
+ _mm_storeu_si128((__m128i *) d, uc);
+ d += 16;
+ src += 16;
+ size -= 16;
+ }
+#endif
+
+ /* Leftover */
+ while (size > 0) {
+ *d++ = lc_map[(guchar) *src++];
+ size--;
+ }
+
+ return (d - dst);
+}
+
+gint rspamd_lc_cmp(const gchar *s, const gchar *d, gsize l)
+{
+ gsize fp, i;
+ guchar c1, c2, c3, c4;
+ union {
+ guchar c[4];
+ guint32 n;
+ } cmp1, cmp2;
+ gsize leftover = l % 4;
+ gint ret = 0;
+
+ fp = l - leftover;
+
+ for (i = 0; i != fp; i += 4) {
+ c1 = s[i], c2 = s[i + 1], c3 = s[i + 2], c4 = s[i + 3];
+ cmp1.c[0] = lc_map[c1];
+ cmp1.c[1] = lc_map[c2];
+ cmp1.c[2] = lc_map[c3];
+ cmp1.c[3] = lc_map[c4];
+
+ c1 = d[i], c2 = d[i + 1], c3 = d[i + 2], c4 = d[i + 3];
+ cmp2.c[0] = lc_map[c1];
+ cmp2.c[1] = lc_map[c2];
+ cmp2.c[2] = lc_map[c3];
+ cmp2.c[3] = lc_map[c4];
+
+ if (cmp1.n != cmp2.n) {
+ return cmp1.n - cmp2.n;
+ }
+ }
+
+ while (leftover > 0) {
+ if (g_ascii_tolower(s[i]) != g_ascii_tolower(d[i])) {
+ return s[i] - d[i];
+ }
+
+ leftover--;
+ i++;
+ }
+
+ return ret;
+}
+
+/*
+ * The purpose of this function is fast and in place conversion of a unicode
+ * string to lower case, so some locale peculiarities are simply ignored
+ * If the target string is longer than initial one, then we just trim it
+ */
+guint rspamd_str_lc_utf8(gchar *str, guint size)
+{
+ guchar *d = (guchar *) str, tst[6];
+ gint32 i = 0, prev = 0;
+ UChar32 uc;
+
+ while (i < size) {
+ prev = i;
+
+ U8_NEXT((guint8 *) str, i, size, uc);
+ uc = u_tolower(uc);
+
+ gint32 olen = 0;
+ U8_APPEND_UNSAFE(tst, olen, uc);
+
+ if (olen <= (i - prev)) {
+ memcpy(d, tst, olen);
+ d += olen;
+ }
+ else {
+ /* Lowercasing has increased the length, so we need to ignore it */
+ d += i - prev;
+ }
+ }
+
+ return d - (guchar *) str;
+}
+
+gboolean
+rspamd_strcase_equal(gconstpointer v, gconstpointer v2)
+{
+ if (g_ascii_strcasecmp((const gchar *) v, (const gchar *) v2) == 0) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+guint64
+rspamd_icase_hash(const gchar *in, gsize len, guint64 seed)
+{
+ guint leftover = len % sizeof(guint64);
+ guint fp, i;
+ const uint8_t *s = (const uint8_t *) in;
+ union {
+ struct {
+ guchar c1, c2, c3, c4, c5, c6, c7, c8;
+ } c;
+ guint64 pp;
+ } u;
+ guint64 h = seed;
+
+ fp = len - leftover;
+
+ for (i = 0; i != fp; i += 8) {
+ u.c.c1 = s[i], u.c.c2 = s[i + 1], u.c.c3 = s[i + 2], u.c.c4 = s[i + 3];
+ u.c.c5 = s[i + 4], u.c.c6 = s[i + 5], u.c.c7 = s[i + 6], u.c.c8 = s[i + 7];
+ u.c.c1 = lc_map[u.c.c1];
+ u.c.c2 = lc_map[u.c.c2];
+ u.c.c3 = lc_map[u.c.c3];
+ u.c.c4 = lc_map[u.c.c4];
+ u.c.c5 = lc_map[u.c.c5];
+ u.c.c6 = lc_map[u.c.c6];
+ u.c.c7 = lc_map[u.c.c7];
+ u.c.c8 = lc_map[u.c.c8];
+ h = t1ha(&u.pp, sizeof(u), h);
+ }
+
+ u.pp = 0;
+
+ switch (leftover) {
+ case 7:
+ u.c.c7 = lc_map[(guchar) s[i++]]; /* FALLTHRU */
+ case 6:
+ u.c.c6 = lc_map[(guchar) s[i++]]; /* FALLTHRU */
+ case 5:
+ u.c.c5 = lc_map[(guchar) s[i++]]; /* FALLTHRU */
+ case 4:
+ u.c.c4 = lc_map[(guchar) s[i++]]; /* FALLTHRU */
+ case 3:
+ u.c.c3 = lc_map[(guchar) s[i++]]; /* FALLTHRU */
+ case 2:
+ u.c.c2 = lc_map[(guchar) s[i++]]; /* FALLTHRU */
+ case 1:
+ u.c.c1 = lc_map[(guchar) s[i]];
+ break;
+ }
+
+ h = t1ha(&u.pp, sizeof(u), h);
+
+ return h;
+}
+
+guint rspamd_strcase_hash(gconstpointer key)
+{
+ const gchar *p = key;
+ gsize len;
+
+ len = strlen(p);
+
+ return (guint) rspamd_icase_hash(p, len, rspamd_hash_seed());
+}
+
+guint rspamd_str_hash(gconstpointer key)
+{
+ gsize len;
+
+ len = strlen((const gchar *) key);
+
+ return (guint) rspamd_cryptobox_fast_hash(key, len, rspamd_hash_seed());
+}
+
+gboolean
+rspamd_str_equal(gconstpointer v, gconstpointer v2)
+{
+ return strcmp((const gchar *) v, (const gchar *) v2) == 0;
+}
+
+gboolean
+rspamd_ftok_icase_equal(gconstpointer v, gconstpointer v2)
+{
+ const rspamd_ftok_t *f1 = v, *f2 = v2;
+
+ if (f1->len == f2->len &&
+ rspamd_lc_cmp(f1->begin, f2->begin, f1->len) == 0) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+
+guint rspamd_ftok_icase_hash(gconstpointer key)
+{
+ const rspamd_ftok_t *f = key;
+
+ return (guint) rspamd_icase_hash(f->begin, f->len, rspamd_hash_seed());
+}
+
+gboolean
+rspamd_gstring_icase_equal(gconstpointer v, gconstpointer v2)
+{
+ const GString *f1 = v, *f2 = v2;
+ if (f1->len == f2->len &&
+ rspamd_lc_cmp(f1->str, f2->str, f1->len) == 0) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+guint rspamd_gstring_icase_hash(gconstpointer key)
+{
+ const GString *f = key;
+
+ return (guint) rspamd_icase_hash(f->str, f->len, rspamd_hash_seed());
+}
+
+/* https://graphics.stanford.edu/~seander/bithacks.html#ZeroInWord */
+#define MEM_ALIGN (sizeof(gsize) - 1)
+#if defined(__LP64__) || defined(_LP64)
+#define WORD_TYPE guint64
+#define ZEROMASK 0x7F7F7F7F7F7F7F7FLLU
+#else
+#define WORD_TYPE guint32
+#define ZEROMASK 0x7F7F7F7FU
+#endif
+
+#define HASZERO(x) ~(((((x) &ZEROMASK) + ZEROMASK) | (x)) | ZEROMASK)
+
+gsize rspamd_strlcpy_fast(gchar *dst, const gchar *src, gsize siz)
+{
+ gchar *d = dst;
+ const gchar *s = src;
+ gsize n = siz;
+ WORD_TYPE *wd;
+ const WORD_TYPE *ws;
+
+ /* Copy as many bytes as will fit */
+ if (n-- != 0) {
+ if (((uintptr_t) s & MEM_ALIGN) == ((uintptr_t) d & MEM_ALIGN)) {
+ /* Init copy byte by byte */
+ for (; ((uintptr_t) s & MEM_ALIGN) && n && (*d = *s); n--, s++, d++)
+ ;
+ if (n && *s) {
+ wd = (void *) d;
+ ws = (const void *) s;
+ /*
+ * Copy by 32 or 64 bits (causes valgrind warnings)
+ */
+ for (; n >= sizeof(WORD_TYPE) && !HASZERO(*ws);
+ n -= sizeof(WORD_TYPE), ws++, wd++) {
+ *wd = *ws;
+ }
+
+ d = (void *) wd;
+ s = (const void *) ws;
+ }
+ }
+
+ /* Copy the rest */
+ for (; n && (*d = *s); n--, s++, d++)
+ ;
+
+ *d = 0;
+ }
+ else {
+ return 0;
+ }
+
+ return (d - dst);
+}
+
+gsize rspamd_null_safe_copy(const gchar *src, gsize srclen,
+ gchar *dest, gsize destlen)
+{
+ gsize copied = 0, si = 0, di = 0;
+
+ if (destlen == 0) {
+ return 0;
+ }
+
+ while (si < srclen && di + 1 < destlen) {
+ if (src[si] != '\0') {
+ dest[di++] = src[si++];
+ copied++;
+ }
+ else {
+ si++;
+ }
+ }
+
+ dest[di] = '\0';
+
+ return copied;
+}
+
+
+size_t
+rspamd_strlcpy_safe(gchar *dst, const gchar *src, gsize siz)
+{
+ gchar *d = dst;
+ gsize nleft = siz;
+
+ if (nleft != 0) {
+ while (--nleft != 0) {
+ if ((*d++ = *src++) == '\0') {
+ d--;
+ break;
+ }
+ }
+ }
+
+ if (nleft == 0) {
+ if (siz != 0) {
+ *d = '\0';
+ }
+ }
+
+ return (d - dst);
+}
+
+/*
+ * Try to convert string of length to long
+ */
+gboolean
+rspamd_strtol(const gchar *s, gsize len, glong *value)
+{
+ const gchar *p = s, *end = s + len;
+ gchar c;
+ glong v = 0;
+ const glong cutoff = G_MAXLONG / 10, cutlim = G_MAXLONG % 10;
+ gboolean neg;
+
+ /* Case negative values */
+ if (*p == '-') {
+ neg = TRUE;
+ p++;
+ }
+ else {
+ neg = FALSE;
+ }
+ /* Some preparations for range errors */
+
+ while (p < end) {
+ c = *p;
+ if (c >= '0' && c <= '9') {
+ c -= '0';
+ if (v > cutoff || (v == cutoff && c > cutlim)) {
+ /* Range error */
+ *value = neg ? G_MINLONG : G_MAXLONG;
+ return FALSE;
+ }
+ else {
+ v *= 10;
+ v += c;
+ }
+ }
+ else {
+ return FALSE;
+ }
+ p++;
+ }
+
+ *value = neg ? -(v) : v;
+ return TRUE;
+}
+
+/*
+ * Try to convert string of length to long
+ */
+#define CONV_STR_LIM_DECIMAL(max_num) \
+ do { \
+ while (p < end) { \
+ c = *p; \
+ if (c >= '0' && c <= '9') { \
+ c -= '0'; \
+ if (v > cutoff || (v == cutoff && (guint8) c > cutlim)) { \
+ *value = (max_num); \
+ return FALSE; \
+ } \
+ else { \
+ v *= 10; \
+ v += c; \
+ } \
+ } \
+ else { \
+ *value = v; \
+ return FALSE; \
+ } \
+ p++; \
+ } \
+ } while (0)
+
+gboolean
+rspamd_strtoul(const gchar *s, gsize len, gulong *value)
+{
+ const gchar *p = s, *end = s + len;
+ gchar c;
+ gulong v = 0;
+ const gulong cutoff = G_MAXULONG / 10, cutlim = G_MAXULONG % 10;
+
+ /* Some preparations for range errors */
+ CONV_STR_LIM_DECIMAL(G_MAXULONG);
+
+ *value = v;
+ return TRUE;
+}
+
+gboolean
+rspamd_strtou64(const gchar *s, gsize len, guint64 *value)
+{
+ const gchar *p = s, *end = s + len;
+ gchar c;
+ guint64 v = 0;
+ const guint64 cutoff = G_MAXUINT64 / 10, cutlim = G_MAXUINT64 % 10;
+
+ /* Some preparations for range errors */
+ CONV_STR_LIM_DECIMAL(G_MAXUINT64);
+
+ *value = v;
+ return TRUE;
+}
+
+gboolean
+rspamd_xstrtoul(const gchar *s, gsize len, gulong *value)
+{
+ const gchar *p = s, *end = s + len;
+ gchar c;
+ gulong v = 0;
+ const gulong cutoff = G_MAXULONG / 10, cutlim = G_MAXULONG % 10;
+
+ /* Some preparations for range errors */
+ while (p < end) {
+ c = g_ascii_tolower(*p);
+ if (c >= '0' && c <= '9') {
+ c -= '0';
+ if (v > cutoff || (v == cutoff && (guint8) c > cutlim)) {
+ /* Range error */
+ *value = G_MAXULONG;
+ return FALSE;
+ }
+ else {
+ v *= 16;
+ v += c;
+ }
+ }
+ else if (c >= 'a' || c <= 'f') {
+ c = c - 'a' + 10;
+ if (v > cutoff || (v == cutoff && (guint8) c > cutlim)) {
+ /* Range error */
+ *value = G_MAXULONG;
+ return FALSE;
+ }
+ else {
+ v *= 16;
+ v += c;
+ }
+ }
+ else {
+ *value = v;
+
+ return FALSE;
+ }
+ p++;
+ }
+
+ *value = v;
+ return TRUE;
+}
+
+/**
+ * Utility function to provide mem_pool copy for rspamd_hash_table_copy function
+ * @param data string to copy
+ * @param ud memory pool to use
+ * @return
+ */
+gpointer
+rspamd_str_pool_copy(gconstpointer data, gpointer ud)
+{
+ rspamd_mempool_t *pool = ud;
+
+ return data ? rspamd_mempool_strdup(pool, data) : NULL;
+}
+
+/*
+ * We use here z-base32 encoding described here:
+ * http://philzimmermann.com/docs/human-oriented-base-32-encoding.txt
+ */
+
+gint rspamd_encode_base32_buf(const guchar *in, gsize inlen, gchar *out, gsize outlen,
+ enum rspamd_base32_type type)
+{
+ static const char b32_default[] = "ybndrfg8ejkmcpqxot1uwisza345h769",
+ b32_bleach[] = "qpzry9x8gf2tvdw0s3jn54khce6mua7l",
+ b32_rfc[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567",
+ *b32;
+ gchar *o, *end;
+ gsize i;
+ gint remain = -1, x;
+ bool inverse_order = true;
+
+ end = out + outlen;
+ o = out;
+
+ switch (type) {
+ case RSPAMD_BASE32_DEFAULT:
+ b32 = b32_default;
+ break;
+ case RSPAMD_BASE32_BLEACH:
+ b32 = b32_bleach;
+ inverse_order = false;
+ break;
+ case RSPAMD_BASE32_RFC:
+ b32 = b32_rfc;
+ inverse_order = false;
+ break;
+ default:
+ g_assert_not_reached();
+ abort();
+ }
+
+ if (inverse_order) {
+ /* Zbase32 as used in Rspamd */
+ for (i = 0; i < inlen && o < end - 1; i++) {
+ switch (i % 5) {
+ case 0:
+ /* 8 bits of input and 3 to remain */
+ x = in[i];
+ remain = in[i] >> 5;
+ *o++ = b32[x & 0x1F];
+ break;
+ case 1:
+ /* 11 bits of input, 1 to remain */
+ x = remain | in[i] << 3;
+ *o++ = b32[x & 0x1F];
+ *o++ = b32[x >> 5 & 0x1F];
+ remain = x >> 10;
+ break;
+ case 2:
+ /* 9 bits of input, 4 to remain */
+ x = remain | in[i] << 1;
+ *o++ = b32[x & 0x1F];
+ remain = x >> 5;
+ break;
+ case 3:
+ /* 12 bits of input, 2 to remain */
+ x = remain | in[i] << 4;
+ *o++ = b32[x & 0x1F];
+ *o++ = b32[x >> 5 & 0x1F];
+ remain = x >> 10 & 0x3;
+ break;
+ case 4:
+ /* 10 bits of output, nothing to remain */
+ x = remain | in[i] << 2;
+ *o++ = b32[x & 0x1F];
+ *o++ = b32[x >> 5 & 0x1F];
+ remain = -1;
+ break;
+ default:
+ /* Not to be happen */
+ break;
+ }
+ }
+ }
+ else {
+ /* Traditional base32 with no bits inversion */
+ for (i = 0; i < inlen && o < end - 1; i++) {
+ switch (i % 5) {
+ case 0:
+ /* 8 bits of input and 3 to remain */
+ x = in[i] >> 3;
+ remain = (in[i] & 7) << 2;
+ *o++ = b32[x & 0x1F];
+ break;
+ case 1:
+ /* 11 bits of input, 1 to remain */
+ x = (remain << 6) | in[i];
+ *o++ = b32[(x >> 6) & 0x1F];
+ *o++ = b32[(x >> 1) & 0x1F];
+ remain = (x & 0x1) << 4;
+ break;
+ case 2:
+ /* 9 bits of input, 4 to remain */
+ x = (remain << 4) | in[i];
+ *o++ = b32[(x >> 4) & 0x1F];
+ remain = (x & 15) << 1;
+ break;
+ case 3:
+ /* 12 bits of input, 2 to remain */
+ x = (remain << 7) | in[i];
+ *o++ = b32[(x >> 7) & 0x1F];
+ *o++ = b32[(x >> 2) & 0x1F];
+ remain = (x & 3) << 3;
+ break;
+ case 4:
+ /* 10 bits of output, nothing to remain */
+ x = (remain << 5) | in[i];
+ *o++ = b32[(x >> 5) & 0x1F];
+ *o++ = b32[x & 0x1F];
+ remain = -1;
+ break;
+ default:
+ /* Not to be happen */
+ break;
+ }
+ }
+ }
+ if (remain >= 0 && o < end) {
+ *o++ = b32[remain & 0x1F];
+ }
+
+ if (o <= end) {
+ return (o - out);
+ }
+
+ return -1;
+}
+
+gchar *
+rspamd_encode_base32(const guchar *in, gsize inlen, enum rspamd_base32_type type)
+{
+ gsize allocated_len = inlen * 8 / 5 + 2;
+ gchar *out;
+ gint outlen;
+
+ out = g_malloc(allocated_len);
+ outlen = rspamd_encode_base32_buf(in, inlen, out,
+ allocated_len - 1, type);
+
+ if (outlen >= 0) {
+ out[outlen] = 0;
+
+ return out;
+ }
+
+ g_free(out);
+
+ return NULL;
+}
+
+enum rspamd_base32_type
+rspamd_base32_decode_type_from_str(const gchar *str)
+{
+ enum rspamd_base32_type ret = RSPAMD_BASE32_INVALID;
+
+ if (str == NULL) {
+ return RSPAMD_BASE32_DEFAULT;
+ }
+
+ if (strcmp(str, "default") == 0 || strcmp(str, "zbase") == 0) {
+ ret = RSPAMD_BASE32_ZBASE;
+ }
+ else if (strcmp(str, "bleach") == 0) {
+ ret = RSPAMD_BASE32_BLEACH;
+ }
+ else if (strcmp(str, "rfc") == 0) {
+ ret = RSPAMD_BASE32_RFC;
+ }
+
+ return ret;
+}
+
+static const guchar b32_dec_zbase[] = {
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0x12, 0xff, 0x19, 0x1a, 0x1b, 0x1e, 0x1d,
+ 0x07, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0x18, 0x01, 0x0c, 0x03, 0x08, 0x05, 0x06,
+ 0x1c, 0x15, 0x09, 0x0a, 0xff, 0x0b, 0x02, 0x10,
+ 0x0d, 0x0e, 0x04, 0x16, 0x11, 0x13, 0xff, 0x14,
+ 0x0f, 0x00, 0x17, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+static const guchar b32_dec_bleach[] = {
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0x0f, 0xff, 0x0a, 0x11, 0x15, 0x14, 0x1a, 0x1e,
+ 0x07, 0x05, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0x1d, 0xff, 0x18, 0x0d, 0x19, 0x09, 0x08,
+ 0x17, 0xff, 0x12, 0x16, 0x1f, 0x1b, 0x13, 0xff,
+ 0x01, 0x00, 0x03, 0x10, 0x0b, 0x1c, 0x0c, 0x0e,
+ 0x06, 0x04, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+static const guchar b32_dec_rfc[] = {
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0x1a,
+ 0x1b,
+ 0x1c,
+ 0x1d,
+ 0x1e,
+ 0x1f,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0x00,
+ 0x01,
+ 0x02,
+ 0x03,
+ 0x04,
+ 0x05,
+ 0x06,
+ 0x07,
+ 0x08,
+ 0x09,
+ 0x0a,
+ 0x0b,
+ 0x0c,
+ 0x0d,
+ 0x0e,
+ 0x0f,
+ 0x10,
+ 0x11,
+ 0x12,
+ 0x13,
+ 0x14,
+ 0x15,
+ 0x16,
+ 0x17,
+ 0x18,
+ 0x19,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+};
+
+
+gint rspamd_decode_base32_buf(const gchar *in, gsize inlen, guchar *out, gsize outlen,
+ enum rspamd_base32_type type)
+{
+ guchar *o, *end, decoded;
+ guchar c;
+ guint acc = 0U;
+ guint processed_bits = 0;
+ gsize i;
+ const guchar *b32_dec;
+ bool inverse_bits = true;
+
+ end = out + outlen;
+ o = out;
+
+ switch (type) {
+ case RSPAMD_BASE32_DEFAULT:
+ b32_dec = b32_dec_zbase;
+ break;
+ case RSPAMD_BASE32_BLEACH:
+ b32_dec = b32_dec_bleach;
+ inverse_bits = false;
+ break;
+ case RSPAMD_BASE32_RFC:
+ b32_dec = b32_dec_rfc;
+ inverse_bits = false;
+ break;
+ default:
+ g_assert_not_reached();
+ abort();
+ }
+
+ if (inverse_bits) {
+ for (i = 0; i < inlen; i++) {
+ c = (guchar) in[i];
+
+ if (processed_bits >= 8) {
+ /* Emit from left to right */
+ processed_bits -= 8;
+ *o++ = acc & 0xFF;
+ acc >>= 8;
+ }
+
+ decoded = b32_dec[c];
+ if (decoded == 0xff || o >= end) {
+ return -1;
+ }
+
+ acc = (decoded << processed_bits) | acc;
+ processed_bits += 5;
+ }
+
+ if (processed_bits > 0 && o < end) {
+ *o++ = (acc & 0xFF);
+ }
+ else if (o > end) {
+ return -1;
+ }
+ }
+ else {
+ for (i = 0; i < inlen; i++) {
+ c = (guchar) in[i];
+
+ decoded = b32_dec[c];
+ if (decoded == 0xff) {
+ return -1;
+ }
+
+ acc = (acc << 5) | decoded;
+ processed_bits += 5;
+
+ if (processed_bits >= 8) {
+ /* Emit from right to left */
+ processed_bits -= 8;
+
+ /* Output buffer overflow */
+ if (o >= end) {
+ return -1;
+ }
+
+ *o++ = (acc >> processed_bits) & 0xFF;
+ /* Preserve lowers at the higher parts of the input */
+ acc = (acc & ((1u << processed_bits) - 1));
+ }
+ }
+
+ if (processed_bits > 0 && o < end && acc != 0) {
+ *o++ = (acc & 0xFF);
+ }
+ else if (o > end) {
+ return -1;
+ }
+ }
+
+ return (o - out);
+}
+
+guchar *
+rspamd_decode_base32(const gchar *in, gsize inlen, gsize *outlen,
+ enum rspamd_base32_type type)
+{
+ guchar *res;
+
+ gsize allocated_len = inlen * 5 / 8 + 2;
+ gssize olen;
+
+ res = g_malloc(allocated_len);
+
+ olen = rspamd_decode_base32_buf(in, inlen, res, allocated_len - 1,
+ type);
+
+ if (olen >= 0) {
+ res[olen] = '\0';
+ }
+ else {
+ g_free(res);
+
+ if (outlen) {
+ *outlen = 0;
+ }
+
+ return NULL;
+ }
+
+ if (outlen) {
+ *outlen = olen;
+ }
+
+ return res;
+}
+
+
+gchar *
+rspamd_encode_base64_common(const guchar *in, gsize inlen, gint str_len,
+ gsize *outlen, gboolean fold, enum rspamd_newlines_type how)
+{
+#define ADD_SPLIT \
+ do { \
+ if (how == RSPAMD_TASK_NEWLINES_CR || how == RSPAMD_TASK_NEWLINES_CRLF) *o++ = '\r'; \
+ if (how == RSPAMD_TASK_NEWLINES_LF || how == RSPAMD_TASK_NEWLINES_CRLF) *o++ = '\n'; \
+ if (fold) *o++ = '\t'; \
+ } while (0)
+#define CHECK_SPLIT \
+ do { \
+ if (str_len > 0 && cols >= str_len) { \
+ ADD_SPLIT; \
+ cols = 0; \
+ } \
+ } while (0)
+
+ gsize allocated_len = (inlen / 3) * 4 + 5;
+ gchar *out, *o;
+ guint64 n;
+ guint32 rem, t, carry;
+ gint cols, shift;
+ static const char b64_enc[] =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz"
+ "0123456789+/";
+
+ if (str_len > 0) {
+ g_assert(str_len > 8);
+ if (fold) {
+ switch (how) {
+ case RSPAMD_TASK_NEWLINES_CR:
+ case RSPAMD_TASK_NEWLINES_LF:
+ allocated_len += (allocated_len / str_len + 1) * 2 + 1;
+ break;
+ default:
+ allocated_len += (allocated_len / str_len + 1) * 3 + 1;
+ break;
+ }
+ }
+ else {
+ switch (how) {
+ case RSPAMD_TASK_NEWLINES_CR:
+ case RSPAMD_TASK_NEWLINES_LF:
+ allocated_len += (allocated_len / str_len + 1) * 1 + 1;
+ break;
+ default:
+ allocated_len += (allocated_len / str_len + 1) * 2 + 1;
+ break;
+ }
+ }
+ }
+
+ out = g_malloc(allocated_len);
+ o = out;
+ cols = 0;
+
+ while (inlen > 6) {
+ memcpy(&n, in, sizeof(n));
+ n = GUINT64_TO_BE(n);
+
+ if (str_len <= 0 || cols <= str_len - 8) {
+ *o++ = b64_enc[(n >> 58) & 0x3F];
+ *o++ = b64_enc[(n >> 52) & 0x3F];
+ *o++ = b64_enc[(n >> 46) & 0x3F];
+ *o++ = b64_enc[(n >> 40) & 0x3F];
+ *o++ = b64_enc[(n >> 34) & 0x3F];
+ *o++ = b64_enc[(n >> 28) & 0x3F];
+ *o++ = b64_enc[(n >> 22) & 0x3F];
+ *o++ = b64_enc[(n >> 16) & 0x3F];
+ cols += 8;
+ }
+ else {
+ cols = str_len - cols;
+ shift = 58;
+ while (cols) {
+ *o++ = b64_enc[(n >> shift) & 0x3F];
+ shift -= 6;
+ cols--;
+ }
+
+ ADD_SPLIT;
+
+ /* Remaining bytes */
+ while (shift >= 16) {
+ *o++ = b64_enc[(n >> shift) & 0x3F];
+ shift -= 6;
+ cols++;
+ }
+ }
+
+ in += 6;
+ inlen -= 6;
+ }
+
+ CHECK_SPLIT;
+
+ rem = 0;
+ carry = 0;
+
+ for (;;) {
+ /* Padding + remaining data (0 - 2 bytes) */
+ switch (rem) {
+ case 0:
+ if (inlen-- == 0) {
+ goto end;
+ }
+ t = *in++;
+ *o++ = b64_enc[t >> 2];
+ carry = (t << 4) & 0x30;
+ rem = 1;
+ cols++;
+ case 1:
+ if (inlen-- == 0) {
+ goto end;
+ }
+ CHECK_SPLIT;
+ t = *in++;
+ *o++ = b64_enc[carry | (t >> 4)];
+ carry = (t << 2) & 0x3C;
+ rem = 2;
+ cols++;
+ default:
+ if (inlen-- == 0) {
+ goto end;
+ }
+ CHECK_SPLIT;
+ t = *in++;
+ *o++ = b64_enc[carry | (t >> 6)];
+ cols++;
+ CHECK_SPLIT;
+ *o++ = b64_enc[t & 0x3F];
+ cols++;
+ CHECK_SPLIT;
+ rem = 0;
+ }
+ }
+
+end:
+ if (rem == 1) {
+ *o++ = b64_enc[carry];
+ cols++;
+ CHECK_SPLIT;
+ *o++ = '=';
+ cols++;
+ CHECK_SPLIT;
+ *o++ = '=';
+ cols++;
+ CHECK_SPLIT;
+ }
+ else if (rem == 2) {
+ *o++ = b64_enc[carry];
+ cols++;
+ CHECK_SPLIT;
+ *o++ = '=';
+ cols++;
+ }
+
+ CHECK_SPLIT;
+
+ *o = '\0';
+
+ if (outlen != NULL) {
+ *outlen = o - out;
+ }
+
+ return out;
+}
+
+gchar *
+rspamd_encode_base64(const guchar *in, gsize inlen, gint str_len,
+ gsize *outlen)
+{
+ return rspamd_encode_base64_common(in, inlen, str_len, outlen, FALSE,
+ RSPAMD_TASK_NEWLINES_CRLF);
+}
+
+gchar *
+rspamd_encode_base64_fold(const guchar *in, gsize inlen, gint str_len,
+ gsize *outlen, enum rspamd_newlines_type how)
+{
+ return rspamd_encode_base64_common(in, inlen, str_len, outlen, TRUE, how);
+}
+
+#define QP_RANGE(x) (((x) >= 33 && (x) <= 60) || ((x) >= 62 && (x) <= 126) || (x) == '\r' || (x) == '\n' || (x) == ' ' || (x) == '\t')
+#define QP_SPAN_NORMAL(span, str_len) ((str_len) > 0 && \
+ ((span) + 1) >= (str_len))
+#define QP_SPAN_SPECIAL(span, str_len) ((str_len) > 0 && \
+ ((span) + 4) >= (str_len))
+
+gchar *
+rspamd_encode_qp_fold(const guchar *in, gsize inlen, gint str_len,
+ gsize *outlen, enum rspamd_newlines_type how)
+{
+ gsize olen = 0, span = 0, i = 0, seen_spaces = 0;
+ gchar *out;
+ gint ch, last_sp;
+ const guchar *end = in + inlen, *p = in;
+ static const gchar hexdigests[16] = "0123456789ABCDEF";
+
+ while (p < end) {
+ ch = *p;
+
+ if (QP_RANGE(ch)) {
+ olen++;
+ span++;
+
+ if (ch == '\r' || ch == '\n') {
+ if (seen_spaces > 0) {
+ /* We must encode spaces at the end of line */
+ olen += 3;
+ seen_spaces = 0;
+ /* Special stuff for space character at the end */
+ if (QP_SPAN_SPECIAL(span, str_len)) {
+ if (how == RSPAMD_TASK_NEWLINES_CRLF) {
+ /* =\r\n */
+ olen += 3;
+ }
+ else {
+ olen += 2;
+ }
+ }
+ /* Continue with the same `ch` but without spaces logic */
+ continue;
+ }
+
+ span = 0;
+ }
+ else if (ch == ' ' || ch == '\t') {
+ seen_spaces++;
+ last_sp = ch;
+ }
+ else {
+ seen_spaces = 0;
+ }
+ }
+ else {
+ if (QP_SPAN_SPECIAL(span, str_len)) {
+ if (how == RSPAMD_TASK_NEWLINES_CRLF) {
+ /* =\r\n */
+ olen += 3;
+ }
+ else {
+ olen += 2;
+ }
+ span = 0;
+ }
+
+ olen += 3;
+ span += 3;
+ }
+
+ if (QP_SPAN_NORMAL(span, str_len)) {
+ if (how == RSPAMD_TASK_NEWLINES_CRLF) {
+ /* =\r\n */
+ olen += 3;
+ }
+ else {
+ olen += 2;
+ }
+ span = 0;
+ }
+
+ p++;
+ }
+
+ if (seen_spaces > 0) {
+ /* Reserve length for the last space encoded */
+ olen += 3;
+ }
+
+ out = g_malloc(olen + 1);
+ p = in;
+ i = 0;
+ span = 0;
+ seen_spaces = 0;
+
+ while (p < end) {
+ ch = *p;
+
+ if (QP_RANGE(ch)) {
+ if (ch == '\r' || ch == '\n') {
+ if (seen_spaces > 0) {
+ if (QP_SPAN_SPECIAL(span, str_len)) {
+ /* Add soft newline */
+ i--;
+
+ if (p + 1 < end || span + 3 >= str_len) {
+ switch (how) {
+ default:
+ case RSPAMD_TASK_NEWLINES_CRLF:
+ out[i++] = '=';
+ out[i++] = '\r';
+ out[i++] = '\n';
+ break;
+ case RSPAMD_TASK_NEWLINES_LF:
+ out[i++] = '=';
+ out[i++] = '\n';
+ break;
+ case RSPAMD_TASK_NEWLINES_CR:
+ out[i++] = '=';
+ out[i++] = '\r';
+ break;
+ }
+ }
+
+ /* Now write encoded `last_sp` but after newline */
+ out[i++] = '=';
+ out[i++] = hexdigests[((last_sp >> 4) & 0xF)];
+ out[i++] = hexdigests[(last_sp & 0xF)];
+
+ span = 0;
+ }
+ else {
+ /* Encode last space */
+ --i;
+ out[i++] = '=';
+ out[i++] = hexdigests[((last_sp >> 4) & 0xF)];
+ out[i++] = hexdigests[(last_sp & 0xF)];
+ seen_spaces = 0;
+ }
+
+ continue;
+ }
+ span = 0;
+ }
+ else if (ch == ' ' || ch == '\t') {
+ seen_spaces++;
+ last_sp = ch;
+ span++;
+ }
+ else {
+ seen_spaces = 0;
+ span++;
+ }
+
+ out[i++] = ch;
+ }
+ else {
+ if (QP_SPAN_SPECIAL(span, str_len)) {
+ /* Add new line and then continue */
+ if (p + 1 < end || span + 3 >= str_len) {
+ switch (how) {
+ default:
+ case RSPAMD_TASK_NEWLINES_CRLF:
+ out[i++] = '=';
+ out[i++] = '\r';
+ out[i++] = '\n';
+ break;
+ case RSPAMD_TASK_NEWLINES_LF:
+ out[i++] = '=';
+ out[i++] = '\n';
+ break;
+ case RSPAMD_TASK_NEWLINES_CR:
+ out[i++] = '=';
+ out[i++] = '\r';
+ break;
+ }
+ span = 0;
+ }
+ }
+
+ out[i++] = '=';
+ out[i++] = hexdigests[((ch >> 4) & 0xF)];
+ out[i++] = hexdigests[(ch & 0xF)];
+ span += 3;
+ seen_spaces = 0;
+ }
+
+ if (QP_SPAN_NORMAL(span, str_len)) {
+ /* Add new line and then continue */
+ if (p + 1 < end || span > str_len || seen_spaces) {
+ switch (how) {
+ default:
+ case RSPAMD_TASK_NEWLINES_CRLF:
+ out[i++] = '=';
+ out[i++] = '\r';
+ out[i++] = '\n';
+ break;
+ case RSPAMD_TASK_NEWLINES_LF:
+ out[i++] = '=';
+ out[i++] = '\n';
+ break;
+ case RSPAMD_TASK_NEWLINES_CR:
+ out[i++] = '=';
+ out[i++] = '\r';
+ break;
+ }
+ span = 0;
+ seen_spaces = 0;
+ }
+ }
+
+ g_assert(i <= olen);
+ p++;
+ }
+
+ /* Deal with the last space character */
+ if (seen_spaces > 0) {
+ i--;
+ out[i++] = '=';
+ out[i++] = hexdigests[((last_sp >> 4) & 0xF)];
+ out[i++] = hexdigests[(last_sp & 0xF)];
+ }
+
+ out[i] = '\0';
+
+ if (outlen) {
+ *outlen = i;
+ }
+
+ return out;
+}
+
+#define MIN3(a, b, c) ((a) < (b) ? ((a) < (c) ? (a) : (c)) : ((b) < (c) ? (b) : (c)))
+
+gint rspamd_strings_levenshtein_distance(const gchar *s1, gsize s1len,
+ const gchar *s2, gsize s2len,
+ guint replace_cost)
+{
+ gchar c1, c2, last_c2, last_c1;
+ static GArray *current_row = NULL, *prev_row = NULL, *transp_row = NULL;
+ gint eq;
+ static const guint max_cmp = 8192;
+ gint ret;
+
+ g_assert(s1 != NULL);
+ g_assert(s2 != NULL);
+
+ if (s1len == 0) {
+ s1len = strlen(s1);
+ }
+ if (s2len == 0) {
+ s2len = strlen(s2);
+ }
+
+ if (MAX(s1len, s2len) > max_cmp) {
+ /* Cannot compare too many characters */
+ return max_cmp;
+ }
+
+ if (s1len > s2len) {
+ /* Exchange s1 and s2 */
+ const gchar *tmp;
+ gsize tmplen;
+
+ tmp = s2;
+ s2 = s1;
+ s1 = tmp;
+
+ tmplen = s2len;
+ s2len = s1len;
+ s1len = tmplen;
+ }
+
+ /* Adjust static space */
+ if (current_row == NULL) {
+ current_row = g_array_sized_new(FALSE, FALSE, sizeof(gint), s1len + 1);
+ prev_row = g_array_sized_new(FALSE, FALSE, sizeof(gint), s1len + 1);
+ transp_row = g_array_sized_new(FALSE, FALSE, sizeof(gint), s1len + 1);
+ g_array_set_size(current_row, s1len + 1);
+ g_array_set_size(prev_row, s1len + 1);
+ g_array_set_size(transp_row, s1len + 1);
+ }
+ else if (current_row->len < s1len + 1) {
+ g_array_set_size(current_row, s1len + 1);
+ g_array_set_size(prev_row, s1len + 1);
+ g_array_set_size(transp_row, s1len + 1);
+ }
+
+ memset(current_row->data, 0, (s1len + 1) * sizeof(gint));
+ memset(transp_row->data, 0, (s1len + 1) * sizeof(gint));
+
+ for (gint i = 0; i <= s1len; i++) {
+ g_array_index(prev_row, gint, i) = i;
+ }
+
+ last_c2 = '\0';
+
+ for (gint i = 1; i <= s2len; i++) {
+ c2 = s2[i - 1];
+ g_array_index(current_row, gint, 0) = i;
+ last_c1 = '\0';
+
+ for (gint j = 1; j <= s1len; j++) {
+ c1 = s1[j - 1];
+ eq = c1 == c2 ? 0 : replace_cost;
+ ret = MIN3(g_array_index(current_row, gint, j - 1) + 1, /* Insert */
+ g_array_index(prev_row, gint, j) + 1, /* Remove */
+ g_array_index(prev_row, gint, j - 1) + eq /* Replace */);
+
+ /* Take reordering into account */
+ if (c1 == last_c2 && c2 == last_c1 && j >= 2) {
+ ret = MIN(ret, g_array_index(transp_row, gint, j - 2) + eq);
+ }
+
+ g_array_index(current_row, gint, j) = ret;
+ last_c1 = c1;
+ }
+
+ last_c2 = c2;
+
+ /* Exchange pointers */
+ GArray *tmp;
+ tmp = transp_row;
+ transp_row = prev_row;
+ prev_row = current_row;
+ current_row = tmp;
+ }
+
+ ret = g_array_index(prev_row, gint, s1len);
+
+ return ret;
+}
+
+GString *
+rspamd_header_value_fold(const gchar *name, gsize name_len,
+ const gchar *value,
+ gsize value_len,
+ guint fold_max,
+ enum rspamd_newlines_type how,
+ const gchar *fold_on_chars)
+{
+ GString *res;
+ const guint default_fold_max = 76;
+ guint cur_len;
+ const gchar *p, *c, *end, *fold_sequence;
+ guint nspaces = 0;
+ gboolean first_token = TRUE;
+ enum {
+ fold_before = 0,
+ fold_after
+ } fold_type = fold_before;
+ enum {
+ read_token = 0,
+ read_quoted,
+ after_quote,
+ fold_token,
+ } state = read_token,
+ next_state = read_token;
+
+ g_assert(name != NULL);
+ g_assert(value != NULL);
+
+ /* Filter insane values */
+ if (fold_max < 20) {
+ fold_max = default_fold_max;
+ }
+
+ switch (how) {
+ case RSPAMD_TASK_NEWLINES_LF:
+ fold_sequence = "\n\t";
+ break;
+ case RSPAMD_TASK_NEWLINES_CR:
+ fold_sequence = "\r\t";
+ break;
+ case RSPAMD_TASK_NEWLINES_CRLF:
+ default:
+ fold_sequence = "\r\n\t";
+ break;
+ }
+
+ res = g_string_sized_new(value_len);
+
+ c = value;
+ p = c;
+ end = value + value_len;
+ /* name:<WSP> */
+ cur_len = name_len + 2;
+
+ while (p < end) {
+ switch (state) {
+
+ case read_token:
+ if (fold_on_chars) {
+ if (strchr(fold_on_chars, *p) != NULL) {
+ fold_type = fold_after;
+ state = fold_token;
+ next_state = read_token;
+ }
+
+ p++;
+ }
+ else {
+ if (*p == ',' || *p == ';') {
+ /* We have something similar to the token's end, so check len */
+ if (cur_len > fold_max * 0.8 && cur_len < fold_max) {
+ /* We want fold */
+ fold_type = fold_after;
+ state = fold_token;
+ next_state = read_token;
+ }
+ else if (cur_len > fold_max && !first_token) {
+ fold_type = fold_before;
+ state = fold_token;
+ next_state = read_token;
+ }
+ else {
+ g_string_append_len(res, c, p - c + 1);
+ c = p + 1;
+ first_token = FALSE;
+ }
+ p++;
+ }
+ else if (*p == '"') {
+ /* Fold before quoted tokens */
+ g_string_append_len(res, c, p - c);
+ c = p;
+ state = read_quoted;
+ }
+ else if (*p == '\r' || *p == '\n') {
+ if (cur_len > fold_max && !first_token) {
+ fold_type = fold_before;
+ state = fold_token;
+ next_state = read_token;
+ }
+ else {
+ /* We need to ensure that it is a folding and not something else */
+
+ const char *t = p;
+ bool seen_fold = false;
+
+ while (t < end) {
+ if (*t == ' ' || *t == '\t') {
+ seen_fold = true;
+ break;
+ }
+ else if (!g_ascii_isspace(*t)) {
+ break;
+ }
+
+ t++;
+ }
+
+ if (seen_fold) {
+ /* Reset line length */
+ cur_len = 0;
+
+ while (g_ascii_isspace(*p)) {
+ p++;
+ }
+
+ g_string_append_len(res, c, p - c);
+ c = p;
+ first_token = TRUE;
+ }
+ else {
+ /* Not seen folding, inject it */
+ g_string_append_len(res, c, p - c);
+ g_string_append(res, fold_sequence);
+ p = t; /* Adjust p to ensure that we do not append extra stuff */
+ state = read_token;
+ first_token = TRUE;
+ c = p;
+ }
+ }
+ }
+ else if (g_ascii_isspace(*p)) {
+ if (cur_len > fold_max * 0.8 && cur_len < fold_max) {
+ /* We want fold */
+ fold_type = fold_after;
+ state = fold_token;
+ next_state = read_token;
+ }
+ else if (cur_len > fold_max && !first_token) {
+ fold_type = fold_before;
+ state = fold_token;
+ next_state = read_token;
+ }
+ else {
+ g_string_append_len(res, c, p - c);
+ c = p;
+ first_token = FALSE;
+ p++;
+ cur_len++;
+ }
+ }
+ else {
+ p++;
+ cur_len++;
+ }
+ }
+ break;
+ case fold_token:
+ /* Here, we have token start at 'c' and token end at 'p' */
+ if (fold_type == fold_after) {
+ nspaces = 0;
+ if (p > c) {
+ g_string_append_len(res, c, p - c);
+
+ /*
+ * Check any spaces that are appended to the result
+ * before folding
+ */
+ const gchar *last = &res->str[res->len - 1];
+
+ while (g_ascii_isspace(*last)) {
+ last--;
+ nspaces++;
+ res->len--;
+ }
+ }
+
+ g_string_append(res, fold_sequence);
+
+ /* Skip space if needed */
+ if (g_ascii_isspace(*p)) {
+ p++;
+ }
+
+ /* Move leftover spaces */
+ while (nspaces) {
+ g_string_append_c(res, ' ');
+ nspaces--;
+ }
+
+ cur_len = 0;
+ }
+ else {
+ const gchar *last;
+
+ /* Skip space if needed */
+ if (g_ascii_isspace(*c) && p > c) {
+ c++;
+ }
+
+ /* Avoid double folding */
+ last = &res->str[res->len - 1];
+ last--;
+
+ if (*last != '\r' && *last != '\n') {
+ last++;
+ while (g_ascii_isspace(*last)) {
+ last--;
+ nspaces++;
+ res->len--;
+ }
+
+ g_string_append(res, fold_sequence);
+ }
+
+ /* Move leftover spaces */
+ cur_len = nspaces;
+
+ while (nspaces) {
+ g_string_append_c(res, ' ');
+ nspaces--;
+ }
+
+ if (p > c) {
+ g_string_append_len(res, c, p - c);
+ cur_len += p - c;
+ }
+ else {
+ cur_len = 0;
+ }
+ }
+
+ first_token = TRUE;
+ c = p;
+ state = next_state;
+ break;
+
+ case read_quoted:
+ if (p != c && *p == '"') {
+ state = after_quote;
+ }
+ p++;
+ cur_len++;
+ break;
+
+ case after_quote:
+ state = read_token;
+ /* Skip one more character after the quote */
+ p++;
+ cur_len++;
+ g_string_append_len(res, c, p - c);
+ c = p;
+ first_token = TRUE;
+ break;
+ }
+ }
+
+ /* Last token */
+ switch (state) {
+ case read_token:
+ if (!fold_on_chars && cur_len > fold_max && !first_token) {
+ if (g_ascii_isspace(*c)) {
+ c++;
+ }
+ g_string_append(res, fold_sequence);
+ g_string_append_len(res, c, p - c);
+ }
+ else {
+ g_string_append_len(res, c, p - c);
+ }
+ break;
+ case read_quoted:
+ case after_quote:
+ g_string_append_len(res, c, p - c);
+ break;
+ case fold_token:
+ /* Here, we have token start at 'c' and token end at 'p' */
+ if (g_ascii_isspace(res->str[res->len - 1])) {
+ g_string_append_len(res, c, p - c);
+ }
+ else {
+ if (*c != '\r' && *c != '\n') {
+ /* We need to add folding as well */
+ g_string_append(res, fold_sequence);
+ g_string_append_len(res, c, p - c);
+ }
+ else {
+ g_string_append_len(res, c, p - c);
+ }
+ }
+ break;
+ default:
+ g_assert(p == c);
+ break;
+ }
+
+ return res;
+}
+
+static inline bool rspamd_substring_cmp_func(guchar a, guchar b)
+{
+ return a == b;
+}
+
+static inline bool rspamd_substring_casecmp_func(guchar a, guchar b)
+{
+ return lc_map[a] == lc_map[b];
+}
+
+typedef bool (*rspamd_cmpchar_func_t)(guchar a, guchar b);
+
+static inline void
+rspamd_substring_preprocess_kmp(const gchar *pat, gsize len, goffset *fsm,
+ rspamd_cmpchar_func_t f)
+{
+ goffset i, j;
+
+ i = 0;
+ j = -1;
+ fsm[0] = -1;
+
+ while (i < len) {
+ while (j > -1 && !f(pat[i], pat[j])) {
+ j = fsm[j];
+ }
+
+ i++;
+ j++;
+
+ if (i < len && j < len && f(pat[i], pat[j])) {
+ fsm[i] = fsm[j];
+ }
+ else {
+ fsm[i] = j;
+ }
+ }
+}
+
+static inline goffset
+rspamd_substring_search_preprocessed(const gchar *in, gsize inlen,
+ const gchar *srch,
+ gsize srchlen,
+ const goffset *fsm,
+ rspamd_cmpchar_func_t f)
+{
+ goffset i, j, k, ell;
+
+ for (ell = 1; f(srch[ell - 1], srch[ell]); ell++) {}
+ if (ell == srchlen) {
+ ell = 0;
+ }
+
+ /* Searching */
+ i = ell;
+ j = k = 0;
+
+ while (j <= inlen - srchlen) {
+ while (i < srchlen && f(srch[i], in[i + j])) {
+ ++i;
+ }
+
+ if (i >= srchlen) {
+ while (k < ell && f(srch[k], in[j + k])) {
+ ++k;
+ }
+
+ if (k >= ell) {
+ return j;
+ }
+ }
+
+ j += (i - fsm[i]);
+
+ if (i == ell) {
+ k = MAX(0, k - 1);
+ }
+ else {
+ if (fsm[i] <= ell) {
+ k = MAX(0, fsm[i]);
+ i = ell;
+ }
+ else {
+ k = ell;
+ i = fsm[i];
+ }
+ }
+ }
+
+ return -1;
+}
+
+static inline goffset
+rspamd_substring_search_common(const gchar *in, gsize inlen,
+ const gchar *srch, gsize srchlen, rspamd_cmpchar_func_t f)
+{
+ static goffset st_fsm[128];
+ goffset *fsm, ret;
+
+ if (G_LIKELY(srchlen < G_N_ELEMENTS(st_fsm))) {
+ fsm = st_fsm;
+ }
+ else {
+ fsm = g_malloc((srchlen + 1) * sizeof(*fsm));
+ }
+
+ rspamd_substring_preprocess_kmp(srch, srchlen, fsm, f);
+ ret = rspamd_substring_search_preprocessed(in, inlen, srch, srchlen, fsm, f);
+
+ if (G_UNLIKELY(srchlen >= G_N_ELEMENTS(st_fsm))) {
+ g_free(fsm);
+ }
+
+ return ret;
+}
+
+goffset
+rspamd_substring_search(const gchar *in, gsize inlen,
+ const gchar *srch, gsize srchlen)
+{
+ if (inlen > srchlen) {
+ if (G_UNLIKELY(srchlen == 1)) {
+ const gchar *p;
+
+ p = memchr(in, srch[0], inlen);
+
+ if (p) {
+ return p - in;
+ }
+
+ return (-1);
+ }
+ else if (G_UNLIKELY(srchlen == 0)) {
+ return 0;
+ }
+
+ return rspamd_substring_search_common(in, inlen, srch, srchlen,
+ rspamd_substring_cmp_func);
+ }
+ else if (inlen == srchlen) {
+ return (rspamd_lc_cmp(srch, in, srchlen) == 0 ? 0 : -1);
+ }
+ else {
+ return (-1);
+ }
+
+ return (-1);
+}
+
+goffset
+rspamd_substring_search_caseless(const gchar *in, gsize inlen,
+ const gchar *srch, gsize srchlen)
+{
+ if (inlen > srchlen) {
+ if (G_UNLIKELY(srchlen == 1)) {
+ goffset i;
+ gchar s = lc_map[(guchar) srch[0]];
+
+ for (i = 0; i < inlen; i++) {
+ if (lc_map[(guchar) in[i]] == s) {
+ return i;
+ }
+ }
+
+ return (-1);
+ }
+
+ return rspamd_substring_search_common(in, inlen, srch, srchlen,
+ rspamd_substring_casecmp_func);
+ }
+ else if (inlen == srchlen) {
+ return rspamd_lc_cmp(srch, in, srchlen) == 0 ? 0 : (-1);
+ }
+
+ return (-1);
+}
+
+goffset
+rspamd_string_find_eoh(GString *input, goffset *body_start)
+{
+ const gchar *p, *c = NULL, *end;
+ enum {
+ skip_char = 0,
+ got_cr,
+ got_lf,
+ got_linebreak,
+ got_linebreak_cr,
+ got_linebreak_lf,
+ obs_fws
+ } state = skip_char;
+
+ g_assert(input != NULL);
+
+ p = input->str;
+ end = p + input->len;
+
+ while (p < end) {
+ switch (state) {
+ case skip_char:
+ if (*p == '\r') {
+ p++;
+ state = got_cr;
+ }
+ else if (*p == '\n') {
+ p++;
+ state = got_lf;
+ }
+ else {
+ p++;
+ }
+ break;
+
+ case got_cr:
+ if (*p == '\r') {
+ /*
+ * Double \r\r, so need to check the current char
+ * if it is '\n', then we have \r\r\n sequence, that is NOT
+ * double end of line
+ */
+ if (p < end && p[1] == '\n') {
+ p++;
+ state = got_lf;
+ }
+ else {
+ /* We have \r\r[^\n] */
+ if (body_start) {
+ *body_start = p - input->str + 1;
+ }
+
+ return p - input->str;
+ }
+ }
+ else if (*p == '\n') {
+ p++;
+ state = got_lf;
+ }
+ else if (g_ascii_isspace(*p)) {
+ /* We have \r<space>*, allow to stay in this state */
+ c = p;
+ p++;
+ state = obs_fws;
+ }
+ else {
+ p++;
+ state = skip_char;
+ }
+ break;
+ case got_lf:
+ if (*p == '\n') {
+ /* We have \n\n, which is obviously end of headers */
+ if (body_start) {
+ *body_start = p - input->str + 1;
+ }
+ return p - input->str;
+ }
+ else if (*p == '\r') {
+ state = got_linebreak;
+ }
+ else if (g_ascii_isspace(*p)) {
+ /* We have \n<space>*, allow to stay in this state */
+ c = p;
+ p++;
+ state = obs_fws;
+ }
+ else {
+ p++;
+ state = skip_char;
+ }
+ break;
+ case got_linebreak:
+ if (*p == '\r') {
+ c = p;
+ p++;
+ state = got_linebreak_cr;
+ }
+ else if (*p == '\n') {
+ c = p;
+ p++;
+ state = got_linebreak_lf;
+ }
+ else if (g_ascii_isspace(*p)) {
+ /* We have <linebreak><space>*, allow to stay in this state */
+ c = p;
+ p++;
+ state = obs_fws;
+ }
+ else {
+ p++;
+ state = skip_char;
+ }
+ break;
+ case got_linebreak_cr:
+ if (*p == '\r') {
+ /* Got double \r\r after \n, so does not treat it as EOH */
+ state = got_linebreak_cr;
+ p++;
+ }
+ else if (*p == '\n') {
+ state = got_linebreak_lf;
+ p++;
+ }
+ else if (g_ascii_isspace(*p)) {
+ /* We have \r\n<space>*, allow to keep in this state */
+ c = p;
+ state = obs_fws;
+ p++;
+ }
+ else {
+ p++;
+ state = skip_char;
+ }
+ break;
+ case got_linebreak_lf:
+ g_assert(c != NULL);
+ if (body_start) {
+ /* \r\n\r\n */
+ *body_start = p - input->str;
+ }
+
+ return c - input->str;
+ case obs_fws:
+ if (*p == ' ' || *p == '\t') {
+ p++;
+ }
+ else if (*p == '\r') {
+ /* Perform lookahead due to #2349 */
+ if (end - p > 2) {
+ if (p[1] == '\n' && g_ascii_isspace(p[2])) {
+ /* Real obs_fws state, switch */
+ c = p;
+ p++;
+ state = got_cr;
+ }
+ else if (g_ascii_isspace(p[1])) {
+ p++;
+ state = obs_fws;
+ }
+ else {
+ /*
+ * <nline> <wsp>+ \r <nwsp>.
+ * It is an empty header likely, so we can go further...
+ * https://tools.ietf.org/html/rfc2822#section-4.2
+ */
+ c = p;
+ p++;
+ state = got_cr;
+ }
+ }
+ else {
+ /* shortage */
+ if (body_start) {
+ *body_start = p - input->str + 1;
+ }
+
+ return p - input->str;
+ }
+ }
+ else if (*p == '\n') {
+ /* Perform lookahead due to #2349 */
+ if (end - p > 1) {
+ /* Continue folding with an empty line */
+ if (p[1] == ' ' || p[1] == '\t') {
+ c = p;
+ p++;
+ state = obs_fws;
+ }
+ else if (p[1] == '\r') {
+ /* WTF state: we have seen spaces, \n and then it follows \r */
+ c = p;
+ p++;
+ state = got_lf;
+ }
+ else if (p[1] == '\n') {
+ /*
+ * Switching to got_lf state here will let us to finish
+ * the cycle.
+ */
+ c = p;
+ p++;
+ state = got_lf;
+ }
+ else {
+ /*
+ * <nline> <wsp>+ \n <nwsp>.
+ * It is an empty header likely, so we can go further...
+ * https://tools.ietf.org/html/rfc2822#section-4.2
+ */
+ c = p;
+ p++;
+ state = got_lf;
+ }
+ }
+ else {
+ /* shortage */
+ if (body_start) {
+ *body_start = p - input->str + 1;
+ }
+
+ return p - input->str;
+ }
+ }
+ else {
+ p++;
+ state = skip_char;
+ }
+ break;
+ }
+ }
+
+ if (state == got_linebreak_lf) {
+ if (body_start) {
+ /* \r\n\r\n */
+ *body_start = p - input->str;
+ }
+
+ return c - input->str;
+ }
+
+ return -1;
+}
+
+gint rspamd_encode_hex_buf(const guchar *in, gsize inlen, gchar *out,
+ gsize outlen)
+{
+ gchar *o, *end;
+ const guchar *p;
+ static const gchar hexdigests[16] = "0123456789abcdef";
+
+ end = out + outlen;
+ o = out;
+ p = in;
+
+ while (inlen > 0 && o < end - 1) {
+ *o++ = hexdigests[((*p >> 4) & 0xF)];
+ *o++ = hexdigests[((*p++) & 0xF)];
+ inlen--;
+ }
+
+ if (o <= end) {
+ return (o - out);
+ }
+
+ return -1;
+}
+
+gchar *
+rspamd_encode_hex(const guchar *in, gsize inlen)
+{
+ gchar *out;
+ gsize outlen = inlen * 2 + 1;
+ gint olen;
+
+ if (in == NULL) {
+ return NULL;
+ }
+
+ out = g_malloc(outlen);
+ olen = rspamd_encode_hex_buf(in, inlen, out, outlen - 1);
+
+ if (olen >= 0) {
+ out[olen] = '\0';
+ }
+ else {
+ g_free(out);
+
+ return NULL;
+ }
+
+ return out;
+}
+
+gssize
+rspamd_decode_hex_buf(const gchar *in, gsize inlen,
+ guchar *out, gsize outlen)
+{
+ guchar *o, *end, ret = 0;
+ const gchar *p;
+ gchar c;
+
+ end = out + outlen;
+ o = out;
+ p = in;
+
+ /* We ignore trailing chars if we have not even input */
+ inlen = inlen - inlen % 2;
+
+ while (inlen > 1 && o < end) {
+ c = *p++;
+
+ if (c >= '0' && c <= '9') ret = c - '0';
+ else if (c >= 'A' && c <= 'F')
+ ret = c - 'A' + 10;
+ else if (c >= 'a' && c <= 'f')
+ ret = c - 'a' + 10;
+
+ c = *p++;
+ ret *= 16;
+
+ if (c >= '0' && c <= '9') ret += c - '0';
+ else if (c >= 'A' && c <= 'F')
+ ret += c - 'A' + 10;
+ else if (c >= 'a' && c <= 'f')
+ ret += c - 'a' + 10;
+
+ *o++ = ret;
+
+ inlen -= 2;
+ }
+
+ if (o <= end) {
+ return (o - out);
+ }
+
+ return -1;
+}
+
+guchar *
+rspamd_decode_hex(const gchar *in, gsize inlen)
+{
+ guchar *out;
+ gsize outlen = (inlen / 2 + inlen % 2) + 1;
+ gint olen;
+
+ if (in == NULL) {
+ return NULL;
+ }
+
+ out = g_malloc(outlen);
+
+ olen = rspamd_decode_hex_buf(in, inlen, out, outlen - 1);
+
+ if (olen >= 0) {
+ out[olen] = '\0';
+
+ return out;
+ }
+
+ g_free(out);
+
+ return NULL;
+}
+
+gssize
+rspamd_decode_qp_buf(const gchar *in, gsize inlen,
+ gchar *out, gsize outlen)
+{
+ gchar *o, *end, *pos, c;
+ const gchar *p;
+ guchar ret;
+ gssize remain, processed;
+
+ p = in;
+ o = out;
+ end = out + outlen;
+ remain = inlen;
+
+ while (remain > 0 && o < end) {
+ if (*p == '=') {
+ remain--;
+
+ if (remain == 0) {
+ /* Last '=' character, bugon */
+ if (end - o > 0) {
+ *o++ = *p;
+ }
+ else {
+ /* Buffer overflow */
+ return (-1);
+ }
+
+ break;
+ }
+
+ p++;
+ decode:
+ /* Decode character after '=' */
+ c = *p++;
+ remain--;
+ ret = 0;
+
+ if (c >= '0' && c <= '9') {
+ ret = c - '0';
+ }
+ else if (c >= 'A' && c <= 'F') {
+ ret = c - 'A' + 10;
+ }
+ else if (c >= 'a' && c <= 'f') {
+ ret = c - 'a' + 10;
+ }
+ else if (c == '\r') {
+ /* Eat one more endline */
+ if (remain > 0 && *p == '\n') {
+ p++;
+ remain--;
+ }
+
+ continue;
+ }
+ else if (c == '\n') {
+ /* Soft line break */
+ continue;
+ }
+ else {
+ /* Hack, hack, hack, treat =<garbage> as =<garbage> */
+ if (end - o > 1) {
+ *o++ = '=';
+ *o++ = *(p - 1);
+ }
+ else {
+ return (-1);
+ }
+
+ continue;
+ }
+
+ if (remain > 0) {
+ c = *p++;
+ ret *= 16;
+ remain--;
+
+ if (c >= '0' && c <= '9') {
+ ret += c - '0';
+ }
+ else if (c >= 'A' && c <= 'F') {
+ ret += c - 'A' + 10;
+ }
+ else if (c >= 'a' && c <= 'f') {
+ ret += c - 'a' + 10;
+ }
+ else {
+ /* Treat =<good><rubbish> as =<good><rubbish> */
+ if (end - o > 2) {
+ *o++ = '=';
+ *o++ = *(p - 2);
+ *o++ = *(p - 1);
+ }
+ else {
+ return (-1);
+ }
+
+ continue;
+ }
+
+ if (end - o > 0) {
+ *o++ = (gchar) ret;
+ }
+ else {
+ return (-1);
+ }
+ }
+ }
+ else {
+ if (end - o >= remain) {
+ if ((pos = memccpy(o, p, '=', remain)) == NULL) {
+ /* All copied */
+ o += remain;
+ break;
+ }
+ else {
+ processed = pos - o;
+ remain -= processed;
+ p += processed;
+
+ if (remain > 0) {
+ o = pos - 1;
+ /*
+ * Skip comparison and jump inside decode branch,
+ * as we know that we have found match
+ */
+ goto decode;
+ }
+ else {
+ /* Last '=' character, bugon */
+ o = pos;
+
+ if (end - o > 0) {
+ *o = '=';
+ }
+ else {
+ /* Buffer overflow */
+ return (-1);
+ }
+
+ break;
+ }
+ }
+ }
+ else {
+ /* Buffer overflow */
+ return (-1);
+ }
+ }
+ }
+
+ return (o - out);
+}
+
+gssize
+rspamd_decode_uue_buf(const gchar *in, gsize inlen,
+ gchar *out, gsize outlen)
+{
+ gchar *o, *out_end;
+ const gchar *p;
+ gssize remain;
+ gboolean base64 = FALSE;
+ goffset pos;
+ const gchar *nline = "\r\n";
+
+ p = in;
+ o = out;
+ out_end = out + outlen;
+ remain = inlen;
+
+ /* Skip newlines */
+#define SKIP_NEWLINE \
+ do { \
+ while (remain > 0 && (*p == '\n' || *p == '\r')) { \
+ p++; \
+ remain--; \
+ } \
+ } while (0)
+ SKIP_NEWLINE;
+
+ /* First of all, we need to read the first line (and probably skip it) */
+ if (remain < sizeof("begin-base64 ")) {
+ /* Obviously truncated */
+ return -1;
+ }
+
+ if (memcmp(p, "begin ", sizeof("begin ") - 1) == 0) {
+ p += sizeof("begin ") - 1;
+ remain -= sizeof("begin ") - 1;
+
+ pos = rspamd_memcspn(p, nline, remain);
+ }
+ else if (memcmp(p, "begin-base64 ", sizeof("begin-base64 ") - 1) == 0) {
+ base64 = TRUE;
+ p += sizeof("begin-base64 ") - 1;
+ remain -= sizeof("begin-base64 ") - 1;
+ pos = rspamd_memcspn(p, nline, remain);
+ }
+ else {
+ /* Crap */
+ return (-1);
+ }
+
+ if (pos == -1 || remain == 0) {
+ /* Crap */
+ return (-1);
+ }
+
+#define DEC(c) (((c) - ' ') & 077) /* single character decode */
+#define IS_DEC(c) ((((c) - ' ') >= 0) && (((c) - ' ') <= 077 + 1))
+#define CHAR_OUT(c) \
+ do { \
+ if (o < out_end) { *o++ = c; } \
+ else { \
+ return (-1); \
+ } \
+ } while (0)
+
+ remain -= pos;
+ p = p + pos;
+ SKIP_NEWLINE;
+
+ if (base64) {
+ if (!rspamd_cryptobox_base64_decode(p,
+ remain,
+ out, &outlen)) {
+ return (-1);
+ }
+
+ return outlen;
+ }
+
+ while (remain > 0 && o < out_end) {
+ /* Main cycle */
+ const gchar *eol;
+ gint i, ch;
+
+ pos = rspamd_memcspn(p, nline, remain);
+
+ if (pos == 0) {
+ /* Skip empty lines */
+ SKIP_NEWLINE;
+
+ if (remain == 0) {
+ break;
+ }
+ }
+
+ eol = p + pos;
+ remain -= eol - p;
+
+ if ((i = DEC(*p)) <= 0) {
+ /* Last pos */
+ break;
+ }
+
+ /* i can be less than eol - p, it means uue padding which we ignore */
+ for (++p; i > 0 && p < eol; p += 4, i -= 3) {
+ if (i >= 3 && p + 3 < eol) {
+ /* Process 4 bytes of input */
+ if (!IS_DEC(*p)) {
+ return (-1);
+ }
+ if (!IS_DEC(*(p + 1))) {
+ return (-1);
+ }
+ if (!IS_DEC(*(p + 2))) {
+ return (-1);
+ }
+ if (!IS_DEC(*(p + 3))) {
+ return (-1);
+ }
+ ch = DEC(p[0]) << 2 | DEC(p[1]) >> 4;
+ CHAR_OUT(ch);
+ ch = DEC(p[1]) << 4 | DEC(p[2]) >> 2;
+ CHAR_OUT(ch);
+ ch = DEC(p[2]) << 6 | DEC(p[3]);
+ CHAR_OUT(ch);
+ }
+ else {
+ if (i >= 1 && p + 1 < eol) {
+ if (!IS_DEC(*p)) {
+ return (-1);
+ }
+ if (!IS_DEC(*(p + 1))) {
+ return (-1);
+ }
+
+ ch = DEC(p[0]) << 2 | DEC(p[1]) >> 4;
+ CHAR_OUT(ch);
+ }
+ if (i >= 2 && p + 2 < eol) {
+ if (!IS_DEC(*(p + 1))) {
+ return (-1);
+ }
+ if (!IS_DEC(*(p + 2))) {
+ return (-1);
+ }
+
+ ch = DEC(p[1]) << 4 | DEC(p[2]) >> 2;
+ CHAR_OUT(ch);
+ }
+ }
+ }
+ /* Skip newline */
+ p = eol;
+ SKIP_NEWLINE;
+ }
+
+ return (o - out);
+}
+
+#define BITOP(a, b, op) \
+ ((a)[(gsize) (b) / (8 * sizeof *(a))] op(gsize) 1 << ((gsize) (b) % (8 * sizeof *(a))))
+
+
+gsize rspamd_memcspn(const gchar *s, const gchar *e, gsize len)
+{
+ gsize byteset[32 / sizeof(gsize)];
+ const gchar *p = s, *end = s + len;
+
+ if (!e[1]) {
+ for (; p < end && *p != *e; p++)
+ ;
+ return p - s;
+ }
+
+ memset(byteset, 0, sizeof byteset);
+
+ for (; *e && BITOP(byteset, *(guchar *) e, |=); e++)
+ ;
+ for (; p < end && !BITOP(byteset, *(guchar *) p, &); p++)
+ ;
+
+ return p - s;
+}
+
+gsize rspamd_memspn(const gchar *s, const gchar *e, gsize len)
+{
+ gsize byteset[32 / sizeof(gsize)];
+ const gchar *p = s, *end = s + len;
+
+ if (!e[1]) {
+ for (; p < end && *p == *e; p++)
+ ;
+ return p - s;
+ }
+
+ memset(byteset, 0, sizeof byteset);
+
+ for (; *e && BITOP(byteset, *(guchar *) e, |=); e++)
+ ;
+ for (; p < end && BITOP(byteset, *(guchar *) p, &); p++)
+ ;
+
+ return p - s;
+}
+
+gssize
+rspamd_decode_qp2047_buf(const gchar *in, gsize inlen,
+ gchar *out, gsize outlen)
+{
+ gchar *o, *end, c;
+ const gchar *p;
+ guchar ret;
+ gsize remain, processed;
+
+ p = in;
+ o = out;
+ end = out + outlen;
+ remain = inlen;
+
+ while (remain > 0 && o < end) {
+ if (*p == '=') {
+ p++;
+ remain--;
+
+ if (remain == 0) {
+ if (end - o > 0) {
+ *o++ = *p;
+ break;
+ }
+ }
+ decode:
+ /* Decode character after '=' */
+ c = *p++;
+ remain--;
+ ret = 0;
+
+ if (c >= '0' && c <= '9') { ret = c - '0'; }
+ else if (c >= 'A' && c <= 'F') {
+ ret = c - 'A' + 10;
+ }
+ else if (c >= 'a' && c <= 'f') {
+ ret = c - 'a' + 10;
+ }
+ else if (c == '\r' || c == '\n') {
+ /* Soft line break */
+ while (remain > 0 && (*p == '\r' || *p == '\n')) {
+ remain--;
+ p++;
+ }
+
+ continue;
+ }
+
+ if (remain > 0) {
+ c = *p++;
+ ret *= 16;
+
+ if (c >= '0' && c <= '9') { ret += c - '0'; }
+ else if (c >= 'A' && c <= 'F') {
+ ret += c - 'A' + 10;
+ }
+ else if (c >= 'a' && c <= 'f') {
+ ret += c - 'a' + 10;
+ }
+
+ if (end - o > 0) {
+ *o++ = (gchar) ret;
+ }
+ else {
+ return (-1);
+ }
+
+ remain--;
+ }
+ }
+ else {
+ if (end - o >= remain) {
+ processed = rspamd_memcspn(p, "=_", remain);
+ memcpy(o, p, processed);
+ o += processed;
+
+ if (processed == remain) {
+ break;
+ }
+ else {
+
+ remain -= processed;
+ p += processed;
+
+ if (G_LIKELY(*p == '=')) {
+ p++;
+ /* Skip comparison, as we know that we have found match */
+ remain--;
+ goto decode;
+ }
+ else {
+ *o++ = ' ';
+ p++;
+ remain--;
+ }
+ }
+ }
+ else {
+ /* Buffer overflow */
+ return (-1);
+ }
+ }
+ }
+
+ return (o - out);
+}
+
+gssize
+rspamd_encode_qp2047_buf(const gchar *in, gsize inlen,
+ gchar *out, gsize outlen)
+{
+ gchar *o = out, *end = out + outlen, c;
+ static const gchar hexdigests[16] = "0123456789ABCDEF";
+
+ while (inlen > 0 && o < end) {
+ c = *in;
+
+ if (g_ascii_isalnum(c)) {
+ *o++ = c;
+ }
+ else if (c == ' ') {
+ *o++ = '_';
+ }
+ else if (end - o >= 3) {
+ *o++ = '=';
+ *o++ = hexdigests[((c >> 4) & 0xF)];
+ *o++ = hexdigests[(c & 0xF)];
+ }
+ else {
+ return (-1);
+ }
+
+ in++;
+ inlen--;
+ }
+
+ if (inlen != 0) {
+ return (-1);
+ }
+
+ return (o - out);
+}
+
+
+/*
+ * GString ucl emitting functions
+ */
+static int
+rspamd_gstring_append_character(unsigned char c, size_t len, void *ud)
+{
+ GString *buf = ud;
+ gsize old_len;
+
+ if (len == 1) {
+ g_string_append_c(buf, c);
+ }
+ else {
+ if (buf->allocated_len - buf->len <= len) {
+ old_len = buf->len;
+ g_string_set_size(buf, buf->len + len + 1);
+ buf->len = old_len;
+ }
+ memset(&buf->str[buf->len], c, len);
+ buf->len += len;
+ }
+
+ return 0;
+}
+
+static int
+rspamd_gstring_append_len(const unsigned char *str, size_t len, void *ud)
+{
+ GString *buf = ud;
+
+ g_string_append_len(buf, str, len);
+
+ return 0;
+}
+
+static int
+rspamd_gstring_append_int(int64_t val, void *ud)
+{
+ GString *buf = ud;
+
+ rspamd_printf_gstring(buf, "%L", (intmax_t) val);
+ return 0;
+}
+
+static int
+rspamd_gstring_append_double(double val, void *ud)
+{
+ GString *buf = ud;
+ const double delta = 0.0000001;
+
+ if (isfinite(val)) {
+ if (val == (double) (int) val) {
+ rspamd_printf_gstring(buf, "%.1f", val);
+ }
+ else if (fabs(val - (double) (int) val) < delta) {
+ /* Write at maximum precision */
+ rspamd_printf_gstring(buf, "%.*g", DBL_DIG, val);
+ }
+ else {
+ rspamd_printf_gstring(buf, "%f", val);
+ }
+ }
+ else {
+ rspamd_printf_gstring(buf, "null");
+ }
+
+ return 0;
+}
+
+void rspamd_ucl_emit_gstring_comments(const ucl_object_t *obj,
+ enum ucl_emitter emit_type,
+ GString *target,
+ const ucl_object_t *comments)
+{
+ struct ucl_emitter_functions func = {
+ .ucl_emitter_append_character = rspamd_gstring_append_character,
+ .ucl_emitter_append_len = rspamd_gstring_append_len,
+ .ucl_emitter_append_int = rspamd_gstring_append_int,
+ .ucl_emitter_append_double = rspamd_gstring_append_double};
+
+ func.ud = target;
+ ucl_object_emit_full(obj, emit_type, &func, comments);
+}
+
+/*
+ * FString ucl emitting functions
+ */
+static int
+rspamd_fstring_emit_append_character(unsigned char c, size_t len, void *ud)
+{
+ rspamd_fstring_t **buf = ud;
+
+ *buf = rspamd_fstring_append_chars(*buf, c, len);
+
+ return 0;
+}
+
+static int
+rspamd_fstring_emit_append_len(const unsigned char *str, size_t len, void *ud)
+{
+ rspamd_fstring_t **buf = ud;
+
+ *buf = rspamd_fstring_append(*buf, str, len);
+
+ return 0;
+}
+
+static int
+rspamd_fstring_emit_append_int(int64_t val, void *ud)
+{
+ rspamd_fstring_t **buf = ud;
+
+ rspamd_printf_fstring(buf, "%L", (intmax_t) val);
+ return 0;
+}
+
+static int
+rspamd_fstring_emit_append_double(double val, void *ud)
+{
+ rspamd_fstring_t **buf = ud;
+#define MAX_PRECISION 6
+
+ if (isfinite(val)) {
+ if (val == (double) ((gint) val)) {
+ rspamd_printf_fstring(buf, "%.1f", val);
+ }
+ else {
+ rspamd_printf_fstring(buf, "%." G_STRINGIFY(MAX_PRECISION) "f",
+ val);
+ }
+ }
+ else {
+ rspamd_printf_fstring(buf, "null");
+ }
+
+ return 0;
+}
+
+void rspamd_ucl_emit_fstring_comments(const ucl_object_t *obj,
+ enum ucl_emitter emit_type,
+ rspamd_fstring_t **buf,
+ const ucl_object_t *comments)
+{
+ struct ucl_emitter_functions func = {
+ .ucl_emitter_append_character = rspamd_fstring_emit_append_character,
+ .ucl_emitter_append_len = rspamd_fstring_emit_append_len,
+ .ucl_emitter_append_int = rspamd_fstring_emit_append_int,
+ .ucl_emitter_append_double = rspamd_fstring_emit_append_double};
+
+ func.ud = buf;
+ ucl_object_emit_full(obj, emit_type, &func, comments);
+}
+
+#ifndef HAVE_MEMRCHR
+void *
+rspamd_memrchr(const void *m, gint c, gsize len)
+{
+ const guint8 *p = m;
+
+ for (gsize i = len; i > 0; i--) {
+ if (p[i - 1] == c) {
+ return (void *) (p + i - 1);
+ }
+ }
+
+ return NULL;
+}
+#endif
+
+struct UConverter *
+rspamd_get_utf8_converter(void)
+{
+ static UConverter *utf8_conv = NULL;
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ if (utf8_conv == NULL) {
+ utf8_conv = ucnv_open("UTF-8", &uc_err);
+ if (!U_SUCCESS(uc_err)) {
+ msg_err("FATAL error: cannot open converter for utf8: %s",
+ u_errorName(uc_err));
+
+ g_assert_not_reached();
+ }
+
+ ucnv_setFromUCallBack(utf8_conv,
+ UCNV_FROM_U_CALLBACK_SUBSTITUTE,
+ NULL,
+ NULL,
+ NULL,
+ &uc_err);
+ ucnv_setToUCallBack(utf8_conv,
+ UCNV_TO_U_CALLBACK_SUBSTITUTE,
+ NULL,
+ NULL,
+ NULL,
+ &uc_err);
+ }
+
+ return utf8_conv;
+}
+
+
+const struct UNormalizer2 *
+rspamd_get_unicode_normalizer(void)
+{
+#if U_ICU_VERSION_MAJOR_NUM >= 44
+ UErrorCode uc_err = U_ZERO_ERROR;
+ static const UNormalizer2 *norm = NULL;
+
+ if (norm == NULL) {
+ norm = unorm2_getInstance(NULL, "nfkc", UNORM2_COMPOSE, &uc_err);
+ g_assert(U_SUCCESS(uc_err));
+ }
+
+ return norm;
+#else
+ /* Old libicu */
+ return NULL;
+#endif
+}
+
+gchar *
+rspamd_str_regexp_escape(const gchar *pattern, gsize slen,
+ gsize *dst_len, enum rspamd_regexp_escape_flags flags)
+{
+ const gchar *p, *end = pattern + slen;
+ gchar *res, *d, t, *tmp_utf = NULL, *dend;
+ gsize len;
+ static const gchar hexdigests[16] = "0123456789abcdef";
+
+ len = 0;
+ p = pattern;
+
+ /* [-[\]{}()*+?.,\\^$|#\s] need to be escaped */
+ while (p < end) {
+ t = *p++;
+
+ switch (t) {
+ case '[':
+ case ']':
+ case '-':
+ case '\\':
+ case '{':
+ case '}':
+ case '(':
+ case ')':
+ case '*':
+ case '+':
+ case '?':
+ case '.':
+ case ',':
+ case '^':
+ case '$':
+ case '|':
+ case '#':
+ if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) {
+ len++;
+ }
+ break;
+ default:
+ if (g_ascii_isspace(t)) {
+ len++;
+ }
+ else {
+ if (!g_ascii_isprint(t) || (t & 0x80)) {
+
+ if (flags & RSPAMD_REGEXP_ESCAPE_UTF) {
+ /* \x{code}, where code can be up to 5 digits */
+ len += 4;
+ }
+ else {
+ /* \\xHH -> 4 symbols */
+ len += 3;
+ }
+ }
+ }
+ break;
+ }
+ }
+
+ if (flags & RSPAMD_REGEXP_ESCAPE_UTF) {
+ if (rspamd_fast_utf8_validate(pattern, slen) != 0) {
+ tmp_utf = rspamd_str_make_utf_valid(pattern, slen, NULL, NULL);
+ }
+ }
+
+ if (len == 0) {
+ /* No need to escape anything */
+
+ if (dst_len) {
+ *dst_len = slen;
+ }
+
+ if (tmp_utf) {
+ return tmp_utf;
+ }
+ else {
+ return g_strdup(pattern);
+ }
+ }
+
+ /* Escape logic */
+ if (tmp_utf) {
+ pattern = tmp_utf;
+ }
+
+ len = slen + len;
+ res = g_malloc(len + 1);
+ p = pattern;
+ d = res;
+ dend = d + len;
+
+ while (p < end) {
+ g_assert(d < dend);
+ t = *p++;
+
+ switch (t) {
+ case '[':
+ case ']':
+ case '\\':
+ case '{':
+ case '}':
+ case '(':
+ case ')':
+ case '.':
+ case ',':
+ case '^':
+ case '$':
+ case '|':
+ case '#':
+ if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) {
+ *d++ = '\\';
+ }
+ break;
+ case '-':
+ if (flags & RSPAMD_REGEXP_ESCAPE_GLOB) {
+ /*
+ * For glob patterns, we need to ensure that a previous character is alphanumeric
+ * and there is `[` symbol somewhere before
+ */
+ bool seen_brace = false;
+ const char *search = p;
+
+ while (search > pattern) {
+ if (!g_ascii_isalnum(*search) && *search != '-') {
+ break;
+ }
+ if (*search == '[') {
+ seen_brace = true;
+ break;
+ }
+
+ search--;
+ }
+
+ if (!seen_brace) {
+ /* Escape `-` symbol */
+ *d++ = '\\';
+ }
+ }
+ else if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) {
+ *d++ = '\\';
+ }
+ break;
+ case '*':
+ case '?':
+ case '+':
+ if (flags & RSPAMD_REGEXP_ESCAPE_GLOB) {
+ /* Treat * as .* and ? as .? */
+ *d++ = '.';
+ }
+ else if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) {
+ *d++ = '\\';
+ }
+ break;
+ default:
+ if (g_ascii_isspace(t)) {
+ if (!(flags & RSPAMD_REGEXP_ESCAPE_RE)) {
+ *d++ = '\\';
+ }
+ }
+ else if (t & 0x80 || !g_ascii_isprint(t)) {
+ if (!(flags & RSPAMD_REGEXP_ESCAPE_UTF)) {
+ *d++ = '\\';
+ *d++ = 'x';
+ *d++ = hexdigests[((t >> 4) & 0xF)];
+ *d++ = hexdigests[((t) &0xF)];
+ continue; /* To avoid *d++ = t; */
+ }
+ else {
+ if (flags & (RSPAMD_REGEXP_ESCAPE_RE | RSPAMD_REGEXP_ESCAPE_GLOB)) {
+ UChar32 uc;
+ gint32 off = p - pattern - 1;
+ U8_NEXT(pattern, off, slen, uc);
+
+ if (uc > 0) {
+ d += rspamd_snprintf(d, dend - d,
+ "\\x{%xd}", uc);
+ p = pattern + off;
+ }
+
+ continue; /* To avoid *d++ = t; */
+ }
+ }
+ }
+ break;
+ }
+
+ *d++ = t;
+ }
+
+ *d = '\0';
+
+ if (dst_len) {
+ *dst_len = d - res;
+ }
+
+ if (tmp_utf) {
+ g_free(tmp_utf);
+ }
+
+ return res;
+}
+
+
+gchar *
+rspamd_str_make_utf_valid(const guchar *src, gsize slen,
+ gsize *dstlen,
+ rspamd_mempool_t *pool)
+{
+ UChar32 uc;
+ goffset err_offset;
+ const guchar *p;
+ gchar *dst, *d;
+ gsize remain = slen, dlen = 0;
+
+ if (src == NULL) {
+ return NULL;
+ }
+
+ if (slen == 0) {
+ if (dstlen) {
+ *dstlen = 0;
+ }
+
+ return pool ? rspamd_mempool_strdup(pool, "") : g_strdup("");
+ }
+
+ p = src;
+ dlen = slen + 1; /* As we add '\0' */
+
+ /* Check space required */
+ while (remain > 0 && (err_offset = rspamd_fast_utf8_validate(p, remain)) > 0) {
+ gint i = 0;
+
+ err_offset--; /* As it returns it 1 indexed */
+ p += err_offset;
+ remain -= err_offset;
+ dlen += err_offset;
+
+ /* Each invalid character of input requires 3 bytes of output (+2 bytes) */
+ while (i < remain) {
+ U8_NEXT(p, i, remain, uc);
+
+ if (uc < 0) {
+ dlen += 2;
+ }
+ else {
+ break;
+ }
+ }
+
+ p += i;
+ remain -= i;
+ }
+
+ if (pool) {
+ dst = rspamd_mempool_alloc(pool, dlen + 1);
+ }
+ else {
+ dst = g_malloc(dlen + 1);
+ }
+
+ p = src;
+ d = dst;
+ remain = slen;
+
+ while (remain > 0 && (err_offset = rspamd_fast_utf8_validate(p, remain)) > 0) {
+ /* Copy valid */
+ err_offset--; /* As it returns it 1 indexed */
+ memcpy(d, p, err_offset);
+ d += err_offset;
+
+ /* Append 0xFFFD for each bad character */
+ gint i = 0;
+
+ p += err_offset;
+ remain -= err_offset;
+
+ while (i < remain) {
+ gint old_i = i;
+ U8_NEXT(p, i, remain, uc);
+
+ if (uc < 0) {
+ *d++ = '\357';
+ *d++ = '\277';
+ *d++ = '\275';
+ }
+ else {
+ /* Adjust p and remaining stuff and go to the outer cycle */
+ i = old_i;
+ break;
+ }
+ }
+ /*
+ * Now p is the first valid utf8 character and remain is the rest of the string
+ * so we can continue our loop
+ */
+ p += i;
+ remain -= i;
+ }
+
+ if (err_offset == 0 && remain > 0) {
+ /* Last piece */
+ memcpy(d, p, remain);
+ d += remain;
+ }
+
+ /* Last '\0' */
+ g_assert(dlen > d - dst);
+ *d = '\0';
+
+ if (dstlen) {
+ *dstlen = d - dst;
+ }
+
+ return dst;
+}
+
+gsize rspamd_gstring_strip(GString *s, const gchar *strip_chars)
+{
+ const gchar *p, *sc;
+ gsize strip_len = 0, total = 0;
+
+ p = s->str + s->len - 1;
+
+ while (p >= s->str) {
+ gboolean seen = FALSE;
+
+ sc = strip_chars;
+
+ while (*sc != '\0') {
+ if (*p == *sc) {
+ strip_len++;
+ seen = TRUE;
+ break;
+ }
+
+ sc++;
+ }
+
+ if (!seen) {
+ break;
+ }
+
+ p--;
+ }
+
+ if (strip_len > 0) {
+ s->len -= strip_len;
+ s->str[s->len] = '\0';
+ total += strip_len;
+ }
+
+ if (s->len > 0) {
+ strip_len = rspamd_memspn(s->str, strip_chars, s->len);
+
+ if (strip_len > 0) {
+ memmove(s->str, s->str + strip_len, s->len - strip_len);
+ s->len -= strip_len;
+ total += strip_len;
+ }
+ }
+
+ return total;
+}
+
+const gchar *rspamd_string_len_strip(const gchar *in,
+ gsize *len,
+ const gchar *strip_chars)
+{
+ const gchar *p, *sc;
+ gsize strip_len = 0, old_len = *len;
+
+ p = in + old_len - 1;
+
+ /* Trail */
+ while (p >= in) {
+ gboolean seen = FALSE;
+
+ sc = strip_chars;
+
+ while (*sc != '\0') {
+ if (*p == *sc) {
+ strip_len++;
+ seen = TRUE;
+ break;
+ }
+
+ sc++;
+ }
+
+ if (!seen) {
+ break;
+ }
+
+ p--;
+ }
+
+ if (strip_len > 0) {
+ *len -= strip_len;
+ }
+
+ /* Head */
+ old_len = *len;
+
+ if (old_len > 0) {
+ strip_len = rspamd_memspn(in, strip_chars, old_len);
+
+ if (strip_len > 0) {
+ *len -= strip_len;
+
+ return in + strip_len;
+ }
+ }
+
+ return in;
+}
+
+gchar **
+rspamd_string_len_split(const gchar *in, gsize len, const gchar *spill,
+ gint max_elts, rspamd_mempool_t *pool)
+{
+ const gchar *p = in, *end = in + len;
+ gsize detected_elts = 0;
+ gchar **res;
+
+ /* Detect number of elements */
+ while (p < end) {
+ gsize cur_fragment = rspamd_memcspn(p, spill, end - p);
+
+ if (cur_fragment > 0) {
+ detected_elts++;
+ p += cur_fragment;
+
+ if (max_elts > 0 && detected_elts >= max_elts) {
+ break;
+ }
+ }
+
+ /* Something like a,,b produces {'a', 'b'} not {'a', '', 'b'} */
+ p += rspamd_memspn(p, spill, end - p);
+ }
+
+ res = pool ? rspamd_mempool_alloc(pool, sizeof(gchar *) * (detected_elts + 1)) : g_malloc(sizeof(gchar *) * (detected_elts + 1));
+ /* Last one */
+ res[detected_elts] = NULL;
+ detected_elts = 0;
+ p = in;
+
+ while (p < end) {
+ gsize cur_fragment = rspamd_memcspn(p, spill, end - p);
+
+ if (cur_fragment > 0) {
+ gchar *elt;
+
+ elt = pool ? rspamd_mempool_alloc(pool, cur_fragment + 1) : g_malloc(cur_fragment + 1);
+
+ memcpy(elt, p, cur_fragment);
+ elt[cur_fragment] = '\0';
+
+ res[detected_elts++] = elt;
+ p += cur_fragment;
+
+ if (max_elts > 0 && detected_elts >= max_elts) {
+ break;
+ }
+ }
+
+ p += rspamd_memspn(p, spill, end - p);
+ }
+
+ return res;
+}
+
+#if defined(__x86_64__)
+#include <x86intrin.h>
+#endif
+
+static inline gboolean
+rspamd_str_has_8bit_u64(const guchar *beg, gsize len)
+{
+ guint8 orb = 0;
+
+ if (len >= 16) {
+ const guchar *nextd = beg + sizeof(guint64);
+ guint64 n1 = 0, n2 = 0;
+
+ do {
+ guint64 t;
+ memcpy(&t, beg, sizeof(t));
+ n1 |= t;
+ memcpy(&t, nextd, sizeof(t));
+ n2 |= t;
+ beg += 16;
+ nextd += 16;
+ len -= 16;
+ } while (len >= 16);
+
+ /*
+ * Idea from Benny Halevy <bhalevy@scylladb.com>
+ * - 7-th bit set ==> orb = !(non-zero) - 1 = 0 - 1 = 0xFF
+ * - 7-th bit clear ==> orb = !0 - 1 = 1 - 1 = 0x00
+ */
+ orb = !((n1 | n2) & 0x8080808080808080ULL) - 1;
+ }
+
+ while (len--) {
+ orb |= *beg++;
+ }
+
+ return orb >= 0x80;
+}
+
+gboolean
+rspamd_str_has_8bit(const guchar *beg, gsize len)
+{
+#if defined(__x86_64__)
+ if (len >= 32) {
+ const uint8_t *nextd = beg + 16;
+
+ __m128i n1 = _mm_set1_epi8(0), n2;
+
+ n2 = n1;
+
+ while (len >= 32) {
+ __m128i xmm1 = _mm_loadu_si128((const __m128i *) beg);
+ __m128i xmm2 = _mm_loadu_si128((const __m128i *) nextd);
+
+ n1 = _mm_or_si128(n1, xmm1);
+ n2 = _mm_or_si128(n2, xmm2);
+
+ beg += 32;
+ nextd += 32;
+ len -= 32;
+ }
+
+ n1 = _mm_or_si128(n1, n2);
+
+ /* We assume 2 complement here */
+ if (_mm_movemask_epi8(n1)) {
+ return TRUE;
+ }
+ }
+#endif
+
+ return rspamd_str_has_8bit_u64(beg, len);
+}
diff --git a/src/libutil/str_util.h b/src/libutil/str_util.h
new file mode 100644
index 0000000..07560cc
--- /dev/null
+++ b/src/libutil/str_util.h
@@ -0,0 +1,565 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBUTIL_STR_UTIL_H_
+#define SRC_LIBUTIL_STR_UTIL_H_
+
+#include "config.h"
+#include "ucl.h"
+#include "fstring.h"
+
+#include <stdalign.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum rspamd_newlines_type {
+ RSPAMD_TASK_NEWLINES_CR = 0,
+ RSPAMD_TASK_NEWLINES_LF,
+ RSPAMD_TASK_NEWLINES_CRLF,
+ RSPAMD_TASK_NEWLINES_MAX
+};
+
+/**
+ * Compare two memory regions of size `l` using case insensitive matching
+ */
+gint rspamd_lc_cmp(const gchar *s, const gchar *d, gsize l);
+
+/**
+ * Convert string to lowercase in-place using ASCII conversion
+ */
+guint rspamd_str_lc(gchar *str, guint size);
+
+/**
+ * Performs ascii copy & lowercase
+ * @param src
+ * @param size
+ * @return
+ */
+gsize rspamd_str_copy_lc(const gchar *src, gchar *dst, gsize size);
+
+/**
+ * Convert string to lowercase in-place using utf (limited) conversion
+ */
+guint rspamd_str_lc_utf8(gchar *str, guint size);
+
+/*
+ * Hash table utility functions for case insensitive hashing
+ */
+guint64 rspamd_icase_hash(const gchar *in, gsize len, guint64 seed);
+
+guint rspamd_strcase_hash(gconstpointer key);
+
+gboolean rspamd_strcase_equal(gconstpointer v, gconstpointer v2);
+
+/*
+ * Hash table utility functions for case sensitive hashing
+ */
+guint rspamd_str_hash(gconstpointer key);
+
+gboolean rspamd_str_equal(gconstpointer v, gconstpointer v2);
+
+
+/*
+ * Hash table utility functions for hashing fixed strings
+ */
+guint rspamd_ftok_icase_hash(gconstpointer key);
+
+gboolean rspamd_ftok_icase_equal(gconstpointer v, gconstpointer v2);
+
+/* Use in khash for speed */
+#define rspamd_ftok_hash(key) _wyhash32((key)->begin, (key)->len, 0)
+#define rspamd_ftok_equal(v1, v2) ((v1)->len == (v2)->len && memcmp((v1)->begin, (v2)->begin, (v1)->len) == 0)
+
+guint rspamd_gstring_icase_hash(gconstpointer key);
+
+gboolean rspamd_gstring_icase_equal(gconstpointer v, gconstpointer v2);
+
+/**
+ * Copy src to dest limited to len, in compare with standard strlcpy(3) rspamd strlcpy does not
+ * traverse the whole string and it is possible to use it for non NULL terminated strings. This is
+ * more like memccpy(dst, src, size, '\0')
+ *
+ * @param dst destination string
+ * @param src source string
+ * @param siz length of destination buffer
+ * @return bytes copied
+ */
+gsize rspamd_strlcpy_fast(gchar *dst, const gchar *src, gsize siz);
+
+gsize rspamd_strlcpy_safe(gchar *dst, const gchar *src, gsize siz);
+
+#if defined(__has_feature)
+#if __has_feature(address_sanitizer)
+#define rspamd_strlcpy rspamd_strlcpy_safe
+#else
+#ifdef __SANITIZE_ADDRESS__
+#define rspamd_strlcpy rspamd_strlcpy_safe
+#else
+#define rspamd_strlcpy rspamd_strlcpy_fast
+#endif
+#endif
+#else
+#ifdef __SANITIZE_ADDRESS__
+#define rspamd_strlcpy rspamd_strlcpy_safe
+#else
+#define rspamd_strlcpy rspamd_strlcpy_fast
+#endif
+#endif
+
+/**
+ * Copies `srclen` characters from `src` to `dst` ignoring \0
+ * @param src
+ * @param srclen
+ * @param dest
+ * @param destlen
+ * @return number of bytes copied
+ */
+gsize rspamd_null_safe_copy(const gchar *src, gsize srclen,
+ gchar *dest, gsize destlen);
+
+/*
+ * Try to convert string of length to long
+ */
+gboolean rspamd_strtol(const gchar *s, gsize len, glong *value);
+
+/*
+ * Try to convert a string of length to unsigned long
+ */
+gboolean rspamd_strtoul(const gchar *s, gsize len, gulong *value);
+gboolean rspamd_strtou64(const gchar *s, gsize len, guint64 *value);
+
+/*
+ * Try to convert a hex string of length to unsigned long
+ */
+gboolean rspamd_xstrtoul(const gchar *s, gsize len, gulong *value);
+
+/**
+ * Utility function to provide mem_pool copy for rspamd_hash_table_copy function
+ * @param data string to copy
+ * @param ud memory pool to use
+ * @return
+ */
+gpointer rspamd_str_pool_copy(gconstpointer data, gpointer ud);
+
+/**
+ * Encode string using hex encoding
+ * @param in input
+ * @param inlen input length
+ * @return freshly allocated base32 encoding of a specified string
+ */
+gchar *rspamd_encode_hex(const guchar *in, gsize inlen);
+
+/**
+ * Decode string using hex encoding
+ * @param in input
+ * @param inlen input length
+ * @return freshly allocated base32 decoded value or NULL if input is invalid
+ */
+guchar *rspamd_decode_hex(const gchar *in, gsize inlen);
+
+enum rspamd_base32_type {
+ RSPAMD_BASE32_DEFAULT = 0,
+ RSPAMD_BASE32_ZBASE = 0,
+ RSPAMD_BASE32_BLEACH,
+ RSPAMD_BASE32_RFC,
+ RSPAMD_BASE32_INVALID = -1,
+};
+
+/**
+ * Returns base32 type from a string or RSPAMD_BASE32_INVALID
+ * @param str
+ * @return
+ */
+enum rspamd_base32_type rspamd_base32_decode_type_from_str(const gchar *str);
+
+/**
+ * Encode string using base32 encoding
+ * @param in input
+ * @param inlen input length
+ * @return freshly allocated base32 encoding of a specified string
+ */
+gchar *rspamd_encode_base32(const guchar *in, gsize inlen,
+ enum rspamd_base32_type type);
+
+/**
+ * Decode string using base32 encoding
+ * @param in input
+ * @param inlen input length
+ * @return freshly allocated base32 decoded value or NULL if input is invalid
+ */
+guchar *rspamd_decode_base32(const gchar *in, gsize inlen, gsize *outlen, enum rspamd_base32_type type);
+
+/**
+ * Encode string using base32 encoding
+ * @param in input
+ * @param inlen input length
+ * @param out output buf
+ * @param outlen output buf len
+ * @return encoded len if `outlen` is enough to encode `inlen`
+ */
+gint rspamd_encode_base32_buf(const guchar *in, gsize inlen, gchar *out,
+ gsize outlen, enum rspamd_base32_type type);
+
+/**
+ * Decode string using base32 encoding
+ * @param in input
+ * @param inlen input length
+ * @param out output buf (may overlap with `in`)
+ * @param outlen output buf len
+ * @return decoded len if in is valid base32 and `outlen` is enough to encode `inlen`
+ */
+gint rspamd_decode_base32_buf(const gchar *in, gsize inlen, guchar *out,
+ gsize outlen, enum rspamd_base32_type type);
+
+/**
+ * Encode string using hex encoding
+ * @param in input
+ * @param inlen input length
+ * @param out output buf
+ * @param outlen output buf len
+ * @return encoded len if `outlen` is enough to encode `inlen`
+ */
+gint rspamd_encode_hex_buf(const guchar *in, gsize inlen, gchar *out,
+ gsize outlen);
+
+
+/**
+ * Decode string using hex encoding
+ * @param in input
+ * @param inlen input length
+ * @param out output buf (may overlap with `in`)
+ * @param outlen output buf len
+ * @return decoded len if in is valid hex and `outlen` is enough to encode `inlen`
+ */
+gssize rspamd_decode_hex_buf(const gchar *in, gsize inlen,
+ guchar *out, gsize outlen);
+
+/**
+ * Common version of base64 encoder
+ * @param in
+ * @param inlen
+ * @param str_len
+ * @param outlen
+ * @param fold
+ * @param how
+ * @return
+ */
+gchar *
+rspamd_encode_base64_common(const guchar *in,
+ gsize inlen,
+ gint str_len,
+ gsize *outlen,
+ gboolean fold,
+ enum rspamd_newlines_type how);
+
+/**
+ * Encode string using base64 encoding
+ * @param in input
+ * @param inlen input length
+ * @param str_len maximum string length (if <= 0 then no lines are split)
+ * @return freshly allocated base64 encoded value or NULL if input is invalid
+ */
+gchar *rspamd_encode_base64(const guchar *in, gsize inlen, gint str_len,
+ gsize *outlen);
+
+/**
+ * Encode and fold string using base64 encoding
+ * @param in input
+ * @param inlen input length
+ * @param str_len maximum string length (if <= 0 then no lines are split)
+ * @return freshly allocated base64 encoded value or NULL if input is invalid
+ */
+gchar *rspamd_encode_base64_fold(const guchar *in, gsize inlen, gint str_len,
+ gsize *outlen, enum rspamd_newlines_type how);
+
+/**
+ * Encode and fold string using quoted printable encoding
+ * @param in input
+ * @param inlen input length
+ * @param str_len maximum string length (if <= 0 then no lines are split)
+ * @return freshly allocated base64 encoded value or NULL if input is invalid
+ */
+gchar *rspamd_encode_qp_fold(const guchar *in, gsize inlen, gint str_len,
+ gsize *outlen, enum rspamd_newlines_type how);
+
+/**
+ * Decode quoted-printable encoded buffer, input and output must not overlap
+ * @param in input
+ * @param inlen length of input
+ * @param out output
+ * @param outlen length of output
+ * @return real size of decoded output or (-1) if outlen is not enough
+ */
+gssize rspamd_decode_qp_buf(const gchar *in, gsize inlen,
+ gchar *out, gsize outlen);
+
+/**
+ * Decode uuencode encoded buffer, input and output must not overlap
+ * @param in input
+ * @param inlen length of input
+ * @param out output
+ * @param outlen length of output
+ * @return real size of decoded output or (-1) if outlen is not enough
+ */
+gssize rspamd_decode_uue_buf(const gchar *in, gsize inlen,
+ gchar *out, gsize outlen);
+
+/**
+ * Decode quoted-printable encoded buffer using rfc2047 format, input and output must not overlap
+ * @param in input
+ * @param inlen length of input
+ * @param out output
+ * @param outlen length of output
+ * @return real size of decoded output or (-1) if outlen is not enough
+ */
+gssize rspamd_decode_qp2047_buf(const gchar *in, gsize inlen,
+ gchar *out, gsize outlen);
+
+/**
+ * Encode quoted-printable buffer using rfc2047 format, input and output must not overlap
+ * @param in
+ * @param inlen
+ * @param out
+ * @param outlen
+ * @return
+ */
+gssize rspamd_encode_qp2047_buf(const gchar *in, gsize inlen,
+ gchar *out, gsize outlen);
+
+#ifndef g_tolower
+#define g_tolower(x) (((x) >= 'A' && (x) <= 'Z') ? (x) - 'A' + 'a' : (x))
+#endif
+
+/**
+ * Return levenstein distance between two strings
+ * @param s1
+ * @param s1len
+ * @param s2
+ * @param s2len
+ * @return
+ */
+gint rspamd_strings_levenshtein_distance(const gchar *s1, gsize s1len,
+ const gchar *s2, gsize s2len, guint replace_cost);
+
+/**
+ * Fold header using rfc822 rules, return new GString from the previous one
+ * @param name name of header (used just for folding)
+ * @param value value of header
+ * @param fold_max
+ * @param how
+ * @param fold_on_chars
+ * @return new GString with the folded value
+ */
+GString *rspamd_header_value_fold(const gchar *name,
+ gsize name_len,
+ const gchar *value,
+ gsize value_len,
+ guint fold_max,
+ enum rspamd_newlines_type how,
+ const gchar *fold_on_chars);
+
+/**
+ * Search for a substring `srch` in the text `in` using Apostolico-Crochemore algorithm
+ * http://www-igm.univ-mlv.fr/~lecroq/string/node12.html#SECTION00120
+ * @param in input
+ * @param inlen input len
+ * @param srch search string
+ * @param srchlen length of the search string
+ * @return position of the first substring match or (-1) if not found
+ */
+goffset rspamd_substring_search(const gchar *in, gsize inlen,
+ const gchar *srch, gsize srchlen);
+
+/**
+ * Search for a substring `srch` in the text `in` using Apostolico-Crochemore algorithm in caseless matter (ASCII only)
+ * http://www-igm.univ-mlv.fr/~lecroq/string/node12.html#SECTION00120
+ * @param in input
+ * @param inlen input len
+ * @param srch search string
+ * @param srchlen length of the search string
+ * @return position of the first substring match or (-1) if not found
+ */
+goffset rspamd_substring_search_caseless(const gchar *in, gsize inlen,
+ const gchar *srch, gsize srchlen);
+
+/**
+ * Search for end-of-headers mark in the input string. Returns position just after
+ * the last header in message (but before the last newline character).
+ * Hence, to obtain the real EOH position, it is also required to skip
+ * space characters
+ */
+goffset rspamd_string_find_eoh(GString *input, goffset *body_start);
+
+
+#define rspamd_ucl_emit_gstring(o, t, target) \
+ rspamd_ucl_emit_gstring_comments((o), (t), (target), NULL)
+
+/**
+ * Emit UCL object to gstring
+ * @param obj object to emit
+ * @param emit_type emitter type
+ * @param comments optional comments object
+ * @param target target string
+ */
+void rspamd_ucl_emit_gstring_comments(const ucl_object_t *obj,
+ enum ucl_emitter emit_type,
+ GString *target,
+ const ucl_object_t *comments);
+
+#define rspamd_ucl_emit_fstring(o, t, target) \
+ rspamd_ucl_emit_fstring_comments((o), (t), (target), NULL)
+
+/**
+ * Emit UCL object to fstring
+ * @param obj object to emit
+ * @param emit_type emitter type
+ * * @param comments optional comments object
+ * @param target target string
+ */
+void rspamd_ucl_emit_fstring_comments(const ucl_object_t *obj,
+ enum ucl_emitter emit_type,
+ rspamd_fstring_t **target,
+ const ucl_object_t *comments);
+
+extern const guchar lc_map[256];
+
+/**
+ * Search for the last occurrence of character `c` in memory block of size `len`
+ * @param m
+ * @param c
+ * @param len
+ * @return pointer to the last occurrence or NULL
+ */
+#ifdef HAVE_MEMRCHR
+#define rspamd_memrchr memrchr
+#else
+void *rspamd_memrchr(const void *m, gint c, gsize len);
+#endif
+
+/**
+ * Return length of memory segment starting in `s` that contains no chars from `e`
+ * @param s any input
+ * @param e zero terminated string of exceptions
+ * @param len length of `s`
+ * @return segment size
+ */
+gsize rspamd_memcspn(const gchar *s, const gchar *e, gsize len);
+
+/**
+ * Return length of memory segment starting in `s` that contains only chars from `e`
+ * @param s any input
+ * @param e zero terminated string of inclusions
+ * @param len length of `s`
+ * @return segment size
+ */
+gsize rspamd_memspn(const gchar *s, const gchar *e, gsize len);
+
+/* https://graphics.stanford.edu/~seander/bithacks.html#HasMoreInWord */
+#define rspamd_str_hasmore(x, n) ((((x) + ~0UL / 255 * (127 - (n))) | (x)) & ~0UL / 255 * 128)
+/*
+ * Check if a pointer is aligned; n must be power of two
+ */
+#define rspamd_is_aligned(p, n) (((uintptr_t) (p) & ((uintptr_t) (n) -1)) == 0)
+#define rspamd_is_aligned_as(p, v) rspamd_is_aligned(p, RSPAMD_ALIGNOF(__typeof((v))))
+gboolean rspamd_str_has_8bit(const guchar *beg, gsize len);
+
+struct UConverter;
+
+struct UConverter *rspamd_get_utf8_converter(void);
+
+struct UNormalizer2;
+
+const struct UNormalizer2 *rspamd_get_unicode_normalizer(void);
+
+
+enum rspamd_regexp_escape_flags {
+ RSPAMD_REGEXP_ESCAPE_ASCII = 0,
+ RSPAMD_REGEXP_ESCAPE_UTF = 1u << 0,
+ RSPAMD_REGEXP_ESCAPE_GLOB = 1u << 1,
+ RSPAMD_REGEXP_ESCAPE_RE = 1u << 2,
+};
+
+/**
+ * Escapes special characters when reading plain data to be processed in pcre
+ * @param pattern pattern to process
+ * @param slen source length
+ * @param dst_len destination length pointer (can be NULL)
+ * @param allow_glob allow glob expressions to be translated into pcre
+ * @return newly allocated zero terminated escaped pattern
+ */
+gchar *
+rspamd_str_regexp_escape(const gchar *pattern, gsize slen,
+ gsize *dst_len, enum rspamd_regexp_escape_flags flags) G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Returns copy of src (zero terminated) where all unicode is made valid or replaced
+ * to FFFD characters. Caller must free string after usage
+ * @param src
+ * @param slen
+ * @param dstelen
+ * @return
+ */
+gchar *rspamd_str_make_utf_valid(const guchar *src, gsize slen, gsize *dstlen,
+ rspamd_mempool_t *pool) G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Strips characters in `strip_chars` from start and end of the GString
+ * @param s
+ * @param strip_chars
+ */
+gsize rspamd_gstring_strip(GString *s, const gchar *strip_chars);
+
+/**
+ * Strips characters in `strip_chars` from start and end of the sized string
+ * @param s
+ * @param strip_chars
+ */
+const gchar *rspamd_string_len_strip(const gchar *in,
+ gsize *len, const gchar *strip_chars) G_GNUC_WARN_UNUSED_RESULT;
+
+/**
+ * Returns a NULL terminated list of zero terminated strings based on splitting of
+ * the base string into parts. If pool is not NULL then memory is allocated from
+ * the pool. Otherwise, it is allocated from the heap using `g_malloc` (so
+ * g_strfreev could be used to free stuff)
+ * @param in
+ * @param len
+ * @param spill
+ * @param max_elts
+ * @return
+ */
+gchar **rspamd_string_len_split(const gchar *in, gsize len,
+ const gchar *spill, gint max_elts, rspamd_mempool_t *pool);
+
+#define IS_ZERO_WIDTH_SPACE(uc) ((uc) == 0x200B || \
+ (uc) == 0x200C || \
+ (uc) == 0x200D || \
+ (uc) == 0xFEFF || \
+ (uc) == 0x00AD)
+#define IS_OBSCURED_CHAR(uc) (((uc) >= 0x200B && (uc) <= 0x200F) || \
+ ((uc) >= 0x2028 && (uc) <= 0x202F) || \
+ ((uc) >= 0x205F && (uc) <= 0x206F) || \
+ (uc) == 0xFEFF)
+
+#define RSPAMD_LEN_CHECK_STARTS_WITH(s, len, lit) \
+ ((len) >= sizeof(lit) - 1 && g_ascii_strncasecmp((s), (lit), sizeof(lit) - 1) == 0)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBUTIL_STR_UTIL_H_ */
diff --git a/src/libutil/unix-std.h b/src/libutil/unix-std.h
new file mode 100644
index 0000000..0ce2442
--- /dev/null
+++ b/src/libutil/unix-std.h
@@ -0,0 +1,79 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_UNIX_STD_H_H
+#define RSPAMD_UNIX_STD_H_H
+
+#include "config.h"
+
+/*
+ * Default unix system includes
+ */
+
+/* sys/file.h */
+#ifdef HAVE_SYS_FILE_H
+#include <sys/file.h>
+#endif
+
+/* sys/uio.h */
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+
+/* sys/mman */
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
+
+
+/* timedb */
+#ifdef HAVE_SYS_TIMEB_H
+#include <sys/timeb.h>
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* strings */
+#ifdef HAVE_STRINGS_H
+#include <strings.h>
+#endif
+
+/* fcntl */
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+
+#ifdef HAVE_DIRENT_H
+#include <dirent.h>
+#endif
+
+#include <signal.h>
+
+#ifdef HAVE_LIBGEN_H
+#include <libgen.h>
+#endif
+
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+
+#endif
diff --git a/src/libutil/upstream.c b/src/libutil/upstream.c
new file mode 100644
index 0000000..f536a2c
--- /dev/null
+++ b/src/libutil/upstream.c
@@ -0,0 +1,1761 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "upstream.h"
+#include "ottery.h"
+#include "ref.h"
+#include "cfg_file.h"
+#include "rdns.h"
+#include "cryptobox.h"
+#include "utlist.h"
+#include "contrib/libev/ev.h"
+#include "logger.h"
+#include "contrib/librdns/rdns.h"
+#include "contrib/mumhash/mum.h"
+
+#include <math.h>
+
+
+struct upstream_inet_addr_entry {
+ rspamd_inet_addr_t *addr;
+ guint priority;
+ struct upstream_inet_addr_entry *next;
+};
+
+struct upstream_addr_elt {
+ rspamd_inet_addr_t *addr;
+ guint priority;
+ guint errors;
+};
+
+struct upstream_list_watcher {
+ rspamd_upstream_watch_func func;
+ GFreeFunc dtor;
+ gpointer ud;
+ enum rspamd_upstreams_watch_event events_mask;
+ struct upstream_list_watcher *next, *prev;
+};
+
+struct upstream {
+ guint weight;
+ guint cur_weight;
+ guint errors;
+ guint checked;
+ guint dns_requests;
+ gint active_idx;
+ guint ttl;
+ gchar *name;
+ ev_timer ev;
+ gdouble last_fail;
+ gdouble last_resolve;
+ gpointer ud;
+ enum rspamd_upstream_flag flags;
+ struct upstream_list *ls;
+ GList *ctx_pos;
+ struct upstream_ctx *ctx;
+
+ struct {
+ GPtrArray *addr; /* struct upstream_addr_elt */
+ guint cur;
+ } addrs;
+
+ struct upstream_inet_addr_entry *new_addrs;
+ gpointer data;
+ gchar uid[8];
+ ref_entry_t ref;
+#ifdef UPSTREAMS_THREAD_SAFE
+ rspamd_mutex_t *lock;
+#endif
+};
+
+struct upstream_limits {
+ gdouble revive_time;
+ gdouble revive_jitter;
+ gdouble error_time;
+ gdouble dns_timeout;
+ gdouble lazy_resolve_time;
+ guint max_errors;
+ guint dns_retransmits;
+};
+
+struct upstream_list {
+ gchar *ups_line;
+ struct upstream_ctx *ctx;
+ GPtrArray *ups;
+ GPtrArray *alive;
+ struct upstream_list_watcher *watchers;
+ guint64 hash_seed;
+ const struct upstream_limits *limits;
+ enum rspamd_upstream_flag flags;
+ guint cur_elt;
+ enum rspamd_upstream_rotation rot_alg;
+#ifdef UPSTREAMS_THREAD_SAFE
+ rspamd_mutex_t *lock;
+#endif
+};
+
+struct upstream_ctx {
+ struct rdns_resolver *res;
+ struct ev_loop *event_loop;
+ struct upstream_limits limits;
+ GQueue *upstreams;
+ gboolean configured;
+ rspamd_mempool_t *pool;
+ ref_entry_t ref;
+};
+
+#ifndef UPSTREAMS_THREAD_SAFE
+#define RSPAMD_UPSTREAM_LOCK(x) \
+ do { \
+ } while (0)
+#define RSPAMD_UPSTREAM_UNLOCK(x) \
+ do { \
+ } while (0)
+#else
+#define RSPAMD_UPSTREAM_LOCK(x) rspamd_mutex_lock(x->lock)
+#define RSPAMD_UPSTREAM_UNLOCK(x) rspamd_mutex_unlock(x->lock)
+#endif
+
+#define msg_debug_upstream(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_upstream_log_id, "upstream", upstream->uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_info_upstream(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "upstream", upstream->uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+#define msg_err_upstream(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "upstream", upstream->uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(upstream)
+
+/* 4 errors in 10 seconds */
+#define DEFAULT_MAX_ERRORS 4
+static const guint default_max_errors = DEFAULT_MAX_ERRORS;
+#define DEFAULT_REVIVE_TIME 60
+static const gdouble default_revive_time = DEFAULT_REVIVE_TIME;
+#define DEFAULT_REVIVE_JITTER 0.4
+static const gdouble default_revive_jitter = DEFAULT_REVIVE_JITTER;
+#define DEFAULT_ERROR_TIME 10
+static const gdouble default_error_time = DEFAULT_ERROR_TIME;
+#define DEFAULT_DNS_TIMEOUT 1.0
+static const gdouble default_dns_timeout = DEFAULT_DNS_TIMEOUT;
+#define DEFAULT_DNS_RETRANSMITS 2
+static const guint default_dns_retransmits = DEFAULT_DNS_RETRANSMITS;
+/* TODO: make it configurable */
+#define DEFAULT_LAZY_RESOLVE_TIME 3600.0
+static const gdouble default_lazy_resolve_time = DEFAULT_LAZY_RESOLVE_TIME;
+
+static const struct upstream_limits default_limits = {
+ .revive_time = DEFAULT_REVIVE_TIME,
+ .revive_jitter = DEFAULT_REVIVE_JITTER,
+ .error_time = DEFAULT_ERROR_TIME,
+ .dns_timeout = DEFAULT_DNS_TIMEOUT,
+ .dns_retransmits = DEFAULT_DNS_RETRANSMITS,
+ .max_errors = DEFAULT_MAX_ERRORS,
+ .lazy_resolve_time = DEFAULT_LAZY_RESOLVE_TIME,
+};
+
+static void rspamd_upstream_lazy_resolve_cb(struct ev_loop *, ev_timer *, int);
+
+void rspamd_upstreams_library_config(struct rspamd_config *cfg,
+ struct upstream_ctx *ctx,
+ struct ev_loop *event_loop,
+ struct rdns_resolver *resolver)
+{
+ g_assert(ctx != NULL);
+ g_assert(cfg != NULL);
+
+ if (cfg->upstream_error_time) {
+ ctx->limits.error_time = cfg->upstream_error_time;
+ }
+ if (cfg->upstream_max_errors) {
+ ctx->limits.max_errors = cfg->upstream_max_errors;
+ }
+ if (cfg->upstream_revive_time) {
+ ctx->limits.revive_time = cfg->upstream_revive_time;
+ }
+ if (cfg->upstream_lazy_resolve_time) {
+ ctx->limits.lazy_resolve_time = cfg->upstream_lazy_resolve_time;
+ }
+ if (cfg->dns_retransmits) {
+ ctx->limits.dns_retransmits = cfg->dns_retransmits;
+ }
+ if (cfg->dns_timeout) {
+ ctx->limits.dns_timeout = cfg->dns_timeout;
+ }
+
+ ctx->event_loop = event_loop;
+ ctx->res = resolver;
+ ctx->configured = TRUE;
+
+ /* Start lazy resolving */
+ if (event_loop && resolver) {
+ GList *cur;
+ struct upstream *upstream;
+
+ cur = ctx->upstreams->head;
+
+ while (cur) {
+ upstream = cur->data;
+ if (!ev_can_stop(&upstream->ev) && upstream->ls &&
+ !(upstream->flags & RSPAMD_UPSTREAM_FLAG_NORESOLVE)) {
+ gdouble when;
+
+ if (upstream->flags & RSPAMD_UPSTREAM_FLAG_SRV_RESOLVE) {
+ /* Resolve them immediately ! */
+ when = 0.0;
+ }
+ else {
+ when = rspamd_time_jitter(upstream->ls->limits->lazy_resolve_time,
+ upstream->ls->limits->lazy_resolve_time * .1);
+ }
+
+ ev_timer_init(&upstream->ev, rspamd_upstream_lazy_resolve_cb,
+ when, 0);
+ upstream->ev.data = upstream;
+ ev_timer_start(ctx->event_loop, &upstream->ev);
+ }
+
+ cur = g_list_next(cur);
+ }
+ }
+}
+
+static void
+rspamd_upstream_ctx_dtor(struct upstream_ctx *ctx)
+{
+ GList *cur;
+ struct upstream *u;
+
+ cur = ctx->upstreams->head;
+
+ while (cur) {
+ u = cur->data;
+ u->ctx = NULL;
+ u->ctx_pos = NULL;
+ cur = g_list_next(cur);
+ }
+
+ g_queue_free(ctx->upstreams);
+ rspamd_mempool_delete(ctx->pool);
+ g_free(ctx);
+}
+
+void rspamd_upstreams_library_unref(struct upstream_ctx *ctx)
+{
+ REF_RELEASE(ctx);
+}
+
+struct upstream_ctx *
+rspamd_upstreams_library_init(void)
+{
+ struct upstream_ctx *ctx;
+
+ ctx = g_malloc0(sizeof(*ctx));
+ memcpy(&ctx->limits, &default_limits, sizeof(ctx->limits));
+ ctx->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "upstreams", 0);
+
+ ctx->upstreams = g_queue_new();
+ REF_INIT_RETAIN(ctx, rspamd_upstream_ctx_dtor);
+
+ return ctx;
+}
+
+static gint
+rspamd_upstream_af_to_weight(const rspamd_inet_addr_t *addr)
+{
+ int ret;
+
+ switch (rspamd_inet_address_get_af(addr)) {
+ case AF_UNIX:
+ ret = 2;
+ break;
+ case AF_INET:
+ ret = 1;
+ break;
+ default:
+ ret = 0;
+ break;
+ }
+
+ return ret;
+}
+
+/*
+ * Select IPv4 addresses before IPv6
+ */
+static gint
+rspamd_upstream_addr_sort_func(gconstpointer a, gconstpointer b)
+{
+ const struct upstream_addr_elt *ip1 = *(const struct upstream_addr_elt **) a,
+ *ip2 = *(const struct upstream_addr_elt **) b;
+ gint w1, w2;
+
+ if (ip1->priority == 0 && ip2->priority == 0) {
+ w1 = rspamd_upstream_af_to_weight(ip1->addr);
+ w2 = rspamd_upstream_af_to_weight(ip2->addr);
+ }
+ else {
+ w1 = ip1->priority;
+ w2 = ip2->priority;
+ }
+
+ /* Inverse order */
+ return w2 - w1;
+}
+
+static void
+rspamd_upstream_set_active(struct upstream_list *ls, struct upstream *upstream)
+{
+ RSPAMD_UPSTREAM_LOCK(ls);
+ g_ptr_array_add(ls->alive, upstream);
+ upstream->active_idx = ls->alive->len - 1;
+
+ if (upstream->ctx && upstream->ctx->configured &&
+ !(upstream->flags & RSPAMD_UPSTREAM_FLAG_NORESOLVE)) {
+
+ if (ev_can_stop(&upstream->ev)) {
+ ev_timer_stop(upstream->ctx->event_loop, &upstream->ev);
+ }
+
+ /* Start lazy (or not so lazy) names resolution */
+ gdouble when;
+
+ if (upstream->flags & RSPAMD_UPSTREAM_FLAG_SRV_RESOLVE) {
+ /* Resolve them immediately ! */
+ when = 0.0;
+ }
+ else {
+ when = rspamd_time_jitter(upstream->ls->limits->lazy_resolve_time,
+ upstream->ls->limits->lazy_resolve_time * .1);
+ }
+ ev_timer_init(&upstream->ev, rspamd_upstream_lazy_resolve_cb,
+ when, 0);
+ upstream->ev.data = upstream;
+ msg_debug_upstream("start lazy resolving for %s in %.0f seconds",
+ upstream->name, when);
+ ev_timer_start(upstream->ctx->event_loop, &upstream->ev);
+ }
+
+ RSPAMD_UPSTREAM_UNLOCK(ls);
+}
+
+static void
+rspamd_upstream_addr_elt_dtor(gpointer a)
+{
+ struct upstream_addr_elt *elt = a;
+
+ if (elt) {
+ rspamd_inet_address_free(elt->addr);
+ g_free(elt);
+ }
+}
+
+static void
+rspamd_upstream_update_addrs(struct upstream *upstream)
+{
+ guint addr_cnt, i, port;
+ gboolean seen_addr, reset_errors = FALSE;
+ struct upstream_inet_addr_entry *cur, *tmp;
+ GPtrArray *new_addrs;
+ struct upstream_addr_elt *addr_elt, *naddr;
+
+ /*
+ * We need first of all get the saved port, since DNS gives us no
+ * idea about what port has been used previously
+ */
+ RSPAMD_UPSTREAM_LOCK(upstream);
+
+ if (upstream->addrs.addr->len > 0 && upstream->new_addrs) {
+ addr_elt = g_ptr_array_index(upstream->addrs.addr, 0);
+ port = rspamd_inet_address_get_port(addr_elt->addr);
+
+ /* Now calculate new addrs count */
+ addr_cnt = 0;
+ LL_FOREACH(upstream->new_addrs, cur)
+ {
+ addr_cnt++;
+ }
+
+ /* At 10% probability reset errors on addr elements */
+ if (rspamd_random_double_fast() > 0.9) {
+ reset_errors = TRUE;
+ msg_debug_upstream("reset errors on upstream %s",
+ upstream->name);
+ }
+
+ new_addrs = g_ptr_array_new_full(addr_cnt, rspamd_upstream_addr_elt_dtor);
+
+ /* Copy addrs back */
+ LL_FOREACH(upstream->new_addrs, cur)
+ {
+ seen_addr = FALSE;
+ naddr = NULL;
+ /* Ports are problematic, set to compare in the next block */
+ rspamd_inet_address_set_port(cur->addr, port);
+
+ PTR_ARRAY_FOREACH(upstream->addrs.addr, i, addr_elt)
+ {
+ if (rspamd_inet_address_compare(addr_elt->addr, cur->addr, FALSE) == 0) {
+ naddr = g_malloc0(sizeof(*naddr));
+ naddr->addr = cur->addr;
+ naddr->errors = reset_errors ? 0 : addr_elt->errors;
+ seen_addr = TRUE;
+
+ break;
+ }
+ }
+
+ if (!seen_addr) {
+ naddr = g_malloc0(sizeof(*naddr));
+ naddr->addr = cur->addr;
+ naddr->errors = 0;
+ msg_debug_upstream("new address for %s: %s",
+ upstream->name,
+ rspamd_inet_address_to_string_pretty(naddr->addr));
+ }
+ else {
+ msg_debug_upstream("existing address for %s: %s",
+ upstream->name,
+ rspamd_inet_address_to_string_pretty(cur->addr));
+ }
+
+ g_ptr_array_add(new_addrs, naddr);
+ }
+
+ /* Free old addresses */
+ g_ptr_array_free(upstream->addrs.addr, TRUE);
+
+ upstream->addrs.cur = 0;
+ upstream->addrs.addr = new_addrs;
+ g_ptr_array_sort(upstream->addrs.addr, rspamd_upstream_addr_sort_func);
+ }
+
+ LL_FOREACH_SAFE(upstream->new_addrs, cur, tmp)
+ {
+ /* Do not free inet address pointer since it has been transferred to up */
+ g_free(cur);
+ }
+
+ upstream->new_addrs = NULL;
+ RSPAMD_UPSTREAM_UNLOCK(upstream);
+}
+
+static void
+rspamd_upstream_dns_cb(struct rdns_reply *reply, void *arg)
+{
+ struct upstream *up = (struct upstream *) arg;
+ struct rdns_reply_entry *entry;
+ struct upstream_inet_addr_entry *up_ent;
+
+ if (reply->code == RDNS_RC_NOERROR) {
+ entry = reply->entries;
+
+ RSPAMD_UPSTREAM_LOCK(up);
+ while (entry) {
+
+ if (entry->type == RDNS_REQUEST_A) {
+ up_ent = g_malloc0(sizeof(*up_ent));
+ up_ent->addr = rspamd_inet_address_new(AF_INET,
+ &entry->content.a.addr);
+ LL_PREPEND(up->new_addrs, up_ent);
+ }
+ else if (entry->type == RDNS_REQUEST_AAAA) {
+ up_ent = g_malloc0(sizeof(*up_ent));
+ up_ent->addr = rspamd_inet_address_new(AF_INET6,
+ &entry->content.aaa.addr);
+ LL_PREPEND(up->new_addrs, up_ent);
+ }
+ entry = entry->next;
+ }
+
+ RSPAMD_UPSTREAM_UNLOCK(up);
+ }
+
+ up->dns_requests--;
+
+ if (up->dns_requests == 0) {
+ rspamd_upstream_update_addrs(up);
+ }
+
+ REF_RELEASE(up);
+}
+
+struct rspamd_upstream_srv_dns_cb {
+ struct upstream *up;
+ guint priority;
+ guint port;
+ guint requests_inflight;
+};
+
+/* Used when we have resolved SRV record and resolved addrs */
+static void
+rspamd_upstream_dns_srv_phase2_cb(struct rdns_reply *reply, void *arg)
+{
+ struct rspamd_upstream_srv_dns_cb *cbdata =
+ (struct rspamd_upstream_srv_dns_cb *) arg;
+ struct upstream *up;
+ struct rdns_reply_entry *entry;
+ struct upstream_inet_addr_entry *up_ent;
+
+ up = cbdata->up;
+
+ if (reply->code == RDNS_RC_NOERROR) {
+ entry = reply->entries;
+
+ RSPAMD_UPSTREAM_LOCK(up);
+ while (entry) {
+
+ if (entry->type == RDNS_REQUEST_A) {
+ up_ent = g_malloc0(sizeof(*up_ent));
+ up_ent->addr = rspamd_inet_address_new(AF_INET,
+ &entry->content.a.addr);
+ up_ent->priority = cbdata->priority;
+ rspamd_inet_address_set_port(up_ent->addr, cbdata->port);
+ LL_PREPEND(up->new_addrs, up_ent);
+ }
+ else if (entry->type == RDNS_REQUEST_AAAA) {
+ up_ent = g_malloc0(sizeof(*up_ent));
+ up_ent->addr = rspamd_inet_address_new(AF_INET6,
+ &entry->content.aaa.addr);
+ up_ent->priority = cbdata->priority;
+ rspamd_inet_address_set_port(up_ent->addr, cbdata->port);
+ LL_PREPEND(up->new_addrs, up_ent);
+ }
+ entry = entry->next;
+ }
+
+ RSPAMD_UPSTREAM_UNLOCK(up);
+ }
+
+ up->dns_requests--;
+ cbdata->requests_inflight--;
+
+ if (cbdata->requests_inflight == 0) {
+ g_free(cbdata);
+ }
+
+ if (up->dns_requests == 0) {
+ rspamd_upstream_update_addrs(up);
+ }
+
+ REF_RELEASE(up);
+}
+
+static void
+rspamd_upstream_dns_srv_cb(struct rdns_reply *reply, void *arg)
+{
+ struct upstream *upstream = (struct upstream *) arg;
+ struct rdns_reply_entry *entry;
+ struct rspamd_upstream_srv_dns_cb *ncbdata;
+
+ if (reply->code == RDNS_RC_NOERROR) {
+ entry = reply->entries;
+
+ RSPAMD_UPSTREAM_LOCK(upstream);
+ while (entry) {
+ /* XXX: we ignore weight as it contradicts with upstreams logic */
+ if (entry->type == RDNS_REQUEST_SRV) {
+ msg_debug_upstream("got srv reply for %s: %s "
+ "(weight=%d, priority=%d, port=%d)",
+ upstream->name, entry->content.srv.target,
+ entry->content.srv.weight, entry->content.srv.priority,
+ entry->content.srv.port);
+ ncbdata = g_malloc0(sizeof(*ncbdata));
+ ncbdata->priority = entry->content.srv.weight;
+ ncbdata->port = entry->content.srv.port;
+ /* XXX: for all entries? */
+ upstream->ttl = entry->ttl;
+
+ if (rdns_make_request_full(upstream->ctx->res,
+ rspamd_upstream_dns_srv_phase2_cb, ncbdata,
+ upstream->ls->limits->dns_timeout,
+ upstream->ls->limits->dns_retransmits,
+ 1, entry->content.srv.target, RDNS_REQUEST_A) != NULL) {
+ upstream->dns_requests++;
+ REF_RETAIN(upstream);
+ ncbdata->requests_inflight++;
+ }
+
+ if (rdns_make_request_full(upstream->ctx->res,
+ rspamd_upstream_dns_srv_phase2_cb, ncbdata,
+ upstream->ls->limits->dns_timeout,
+ upstream->ls->limits->dns_retransmits,
+ 1, entry->content.srv.target, RDNS_REQUEST_AAAA) != NULL) {
+ upstream->dns_requests++;
+ REF_RETAIN(upstream);
+ ncbdata->requests_inflight++;
+ }
+
+ if (ncbdata->requests_inflight == 0) {
+ g_free(ncbdata);
+ }
+ }
+ entry = entry->next;
+ }
+
+ RSPAMD_UPSTREAM_UNLOCK(upstream);
+ }
+
+ upstream->dns_requests--;
+ REF_RELEASE(upstream);
+}
+
+static void
+rspamd_upstream_revive_cb(struct ev_loop *loop, ev_timer *w, int revents)
+{
+ struct upstream *upstream = (struct upstream *) w->data;
+
+ RSPAMD_UPSTREAM_LOCK(upstream);
+ ev_timer_stop(loop, w);
+
+ msg_debug_upstream("revive upstream %s", upstream->name);
+
+ if (upstream->ls) {
+ rspamd_upstream_set_active(upstream->ls, upstream);
+ }
+
+ RSPAMD_UPSTREAM_UNLOCK(upstream);
+ g_assert(upstream->ref.refcount > 1);
+ REF_RELEASE(upstream);
+}
+
+static void
+rspamd_upstream_resolve_addrs(const struct upstream_list *ls,
+ struct upstream *upstream)
+{
+ /* XXX: maybe make it configurable */
+ static const gdouble min_resolve_interval = 60.0;
+
+ if (upstream->ctx->res != NULL &&
+ upstream->ctx->configured &&
+ upstream->dns_requests == 0 &&
+ !(upstream->flags & RSPAMD_UPSTREAM_FLAG_NORESOLVE)) {
+
+ gdouble now = ev_now(upstream->ctx->event_loop);
+
+ if (now - upstream->last_resolve < min_resolve_interval) {
+ msg_info_upstream("do not resolve upstream %s as it was checked %.0f "
+ "seconds ago (%.0f is minimum)",
+ upstream->name, now - upstream->last_resolve,
+ min_resolve_interval);
+
+ return;
+ }
+
+ /* Resolve name of the upstream one more time */
+ if (upstream->name[0] != '/') {
+ upstream->last_resolve = now;
+
+ /*
+ * If upstream name has a port, then we definitely need to resolve
+ * merely host part!
+ */
+ char dns_name[253 + 1]; /* 253 == max dns name + \0 */
+ const char *semicolon_pos = strchr(upstream->name, ':');
+
+ if (semicolon_pos != NULL && semicolon_pos > upstream->name) {
+ if (sizeof(dns_name) > semicolon_pos - upstream->name) {
+ rspamd_strlcpy(dns_name, upstream->name,
+ semicolon_pos - upstream->name + 1);
+ }
+ else {
+ /* XXX: truncated */
+ msg_err_upstream("internal error: upstream name is larger than"
+ "max DNS name: %s",
+ upstream->name);
+ rspamd_strlcpy(dns_name, upstream->name, sizeof(dns_name));
+ }
+ }
+ else {
+ rspamd_strlcpy(dns_name, upstream->name, sizeof(dns_name));
+ }
+
+ if (upstream->flags & RSPAMD_UPSTREAM_FLAG_SRV_RESOLVE) {
+ if (rdns_make_request_full(upstream->ctx->res,
+ rspamd_upstream_dns_srv_cb, upstream,
+ ls->limits->dns_timeout, ls->limits->dns_retransmits,
+ 1, dns_name, RDNS_REQUEST_SRV) != NULL) {
+ upstream->dns_requests++;
+ REF_RETAIN(upstream);
+ }
+ }
+ else {
+ if (rdns_make_request_full(upstream->ctx->res,
+ rspamd_upstream_dns_cb, upstream,
+ ls->limits->dns_timeout, ls->limits->dns_retransmits,
+ 1, dns_name, RDNS_REQUEST_A) != NULL) {
+ upstream->dns_requests++;
+ REF_RETAIN(upstream);
+ }
+
+ if (rdns_make_request_full(upstream->ctx->res,
+ rspamd_upstream_dns_cb, upstream,
+ ls->limits->dns_timeout, ls->limits->dns_retransmits,
+ 1, dns_name, RDNS_REQUEST_AAAA) != NULL) {
+ upstream->dns_requests++;
+ REF_RETAIN(upstream);
+ }
+ }
+ }
+ }
+ else if (upstream->dns_requests != 0) {
+ msg_info_upstream("do not resolve upstream %s as another request for "
+ "resolving has been already issued",
+ upstream->name);
+ }
+}
+
+static void
+rspamd_upstream_lazy_resolve_cb(struct ev_loop *loop, ev_timer *w, int revents)
+{
+ struct upstream *up = (struct upstream *) w->data;
+
+ RSPAMD_UPSTREAM_LOCK(up);
+ ev_timer_stop(loop, w);
+
+ if (up->ls) {
+ rspamd_upstream_resolve_addrs(up->ls, up);
+
+ if (up->ttl == 0 || up->ttl > up->ls->limits->lazy_resolve_time) {
+ w->repeat = rspamd_time_jitter(up->ls->limits->lazy_resolve_time,
+ up->ls->limits->lazy_resolve_time * .1);
+ }
+ else {
+ w->repeat = up->ttl;
+ }
+
+ ev_timer_again(loop, w);
+ }
+
+ RSPAMD_UPSTREAM_UNLOCK(up);
+}
+
+static void
+rspamd_upstream_set_inactive(struct upstream_list *ls, struct upstream *upstream)
+{
+ gdouble ntim;
+ guint i;
+ struct upstream *cur;
+ struct upstream_list_watcher *w;
+
+ RSPAMD_UPSTREAM_LOCK(ls);
+ g_ptr_array_remove_index(ls->alive, upstream->active_idx);
+ upstream->active_idx = -1;
+
+ /* We need to update all indices */
+ for (i = 0; i < ls->alive->len; i++) {
+ cur = g_ptr_array_index(ls->alive, i);
+ cur->active_idx = i;
+ }
+
+ if (upstream->ctx) {
+ rspamd_upstream_resolve_addrs(ls, upstream);
+
+ REF_RETAIN(upstream);
+ ntim = rspamd_time_jitter(ls->limits->revive_time,
+ ls->limits->revive_time * ls->limits->revive_jitter);
+
+ if (ev_can_stop(&upstream->ev)) {
+ ev_timer_stop(upstream->ctx->event_loop, &upstream->ev);
+ }
+
+ msg_debug_upstream("mark upstream %s inactive; revive in %.0f seconds",
+ upstream->name, ntim);
+ ev_timer_init(&upstream->ev, rspamd_upstream_revive_cb, ntim, 0);
+ upstream->ev.data = upstream;
+
+ if (upstream->ctx->event_loop != NULL && upstream->ctx->configured) {
+ ev_timer_start(upstream->ctx->event_loop, &upstream->ev);
+ }
+ }
+
+ DL_FOREACH(upstream->ls->watchers, w)
+ {
+ if (w->events_mask & RSPAMD_UPSTREAM_WATCH_OFFLINE) {
+ w->func(upstream, RSPAMD_UPSTREAM_WATCH_OFFLINE, upstream->errors, w->ud);
+ }
+ }
+
+ RSPAMD_UPSTREAM_UNLOCK(ls);
+}
+
+void rspamd_upstream_fail(struct upstream *upstream,
+ gboolean addr_failure,
+ const gchar *reason)
+{
+ gdouble error_rate = 0, max_error_rate = 0;
+ gdouble sec_last, sec_cur;
+ struct upstream_addr_elt *addr_elt;
+ struct upstream_list_watcher *w;
+
+ msg_debug_upstream("upstream %s failed; reason: %s",
+ upstream->name,
+ reason);
+
+ if (upstream->ctx && upstream->active_idx != -1 && upstream->ls) {
+ sec_cur = rspamd_get_ticks(FALSE);
+
+ RSPAMD_UPSTREAM_LOCK(upstream);
+ if (upstream->errors == 0) {
+ /* We have the first error */
+ upstream->last_fail = sec_cur;
+ upstream->errors = 1;
+
+ if (upstream->ls && upstream->dns_requests == 0) {
+ /* Try to re-resolve address immediately */
+ rspamd_upstream_resolve_addrs(upstream->ls, upstream);
+ }
+
+ DL_FOREACH(upstream->ls->watchers, w)
+ {
+ if (w->events_mask & RSPAMD_UPSTREAM_WATCH_FAILURE) {
+ w->func(upstream, RSPAMD_UPSTREAM_WATCH_FAILURE, 1, w->ud);
+ }
+ }
+ }
+ else {
+ sec_last = upstream->last_fail;
+
+ if (sec_cur >= sec_last) {
+ upstream->errors++;
+
+
+ DL_FOREACH(upstream->ls->watchers, w)
+ {
+ if (w->events_mask & RSPAMD_UPSTREAM_WATCH_FAILURE) {
+ w->func(upstream, RSPAMD_UPSTREAM_WATCH_FAILURE,
+ upstream->errors, w->ud);
+ }
+ }
+
+ if (sec_cur - sec_last >= upstream->ls->limits->error_time) {
+ error_rate = ((gdouble) upstream->errors) / (sec_cur - sec_last);
+ max_error_rate = ((gdouble) upstream->ls->limits->max_errors) /
+ upstream->ls->limits->error_time;
+ }
+
+ if (error_rate > max_error_rate) {
+ /* Remove upstream from the active list */
+ if (upstream->ls->ups->len > 1) {
+ msg_debug_upstream("mark upstream %s inactive; "
+ "reason: %s; %.2f "
+ "error rate (%d errors), "
+ "%.2f max error rate, "
+ "%.1f first error time, "
+ "%.1f current ts, "
+ "%d upstreams left",
+ upstream->name,
+ reason,
+ error_rate,
+ upstream->errors,
+ max_error_rate,
+ sec_last,
+ sec_cur,
+ upstream->ls->alive->len - 1);
+ rspamd_upstream_set_inactive(upstream->ls, upstream);
+ upstream->errors = 0;
+ }
+ else {
+ msg_debug_upstream("cannot mark last alive upstream %s "
+ "inactive; reason: %s; %.2f "
+ "error rate (%d errors), "
+ "%.2f max error rate, "
+ "%.1f first error time, "
+ "%.1f current ts",
+ upstream->name,
+ reason,
+ error_rate,
+ upstream->errors,
+ max_error_rate,
+ sec_last,
+ sec_cur);
+ /* Just re-resolve addresses */
+ if (sec_cur - sec_last > upstream->ls->limits->revive_time) {
+ upstream->errors = 0;
+ rspamd_upstream_resolve_addrs(upstream->ls, upstream);
+ }
+ }
+ }
+ else if (sec_cur - sec_last >= upstream->ls->limits->error_time) {
+ /* Forget the whole interval */
+ upstream->last_fail = sec_cur;
+ upstream->errors = 1;
+ }
+ }
+ }
+
+ if (addr_failure) {
+ /* Also increase count of errors for this specific address */
+ if (upstream->addrs.addr) {
+ addr_elt = g_ptr_array_index(upstream->addrs.addr,
+ upstream->addrs.cur);
+ addr_elt->errors++;
+ }
+ }
+
+ RSPAMD_UPSTREAM_UNLOCK(upstream);
+ }
+}
+
+void rspamd_upstream_ok(struct upstream *upstream)
+{
+ struct upstream_addr_elt *addr_elt;
+ struct upstream_list_watcher *w;
+
+ RSPAMD_UPSTREAM_LOCK(upstream);
+ if (upstream->errors > 0 && upstream->active_idx != -1 && upstream->ls) {
+ /* We touch upstream if and only if it is active */
+ msg_debug_upstream("reset errors on upstream %s (was %ud)", upstream->name, upstream->errors);
+ upstream->errors = 0;
+
+ if (upstream->addrs.addr) {
+ addr_elt = g_ptr_array_index(upstream->addrs.addr, upstream->addrs.cur);
+ addr_elt->errors = 0;
+ }
+
+ DL_FOREACH(upstream->ls->watchers, w)
+ {
+ if (w->events_mask & RSPAMD_UPSTREAM_WATCH_SUCCESS) {
+ w->func(upstream, RSPAMD_UPSTREAM_WATCH_SUCCESS, 0, w->ud);
+ }
+ }
+ }
+
+ RSPAMD_UPSTREAM_UNLOCK(upstream);
+}
+
+void rspamd_upstream_set_weight(struct upstream *up, guint weight)
+{
+ RSPAMD_UPSTREAM_LOCK(up);
+ up->weight = weight;
+ RSPAMD_UPSTREAM_UNLOCK(up);
+}
+
+#define SEED_CONSTANT 0xa574de7df64e9b9dULL
+
+struct upstream_list *
+rspamd_upstreams_create(struct upstream_ctx *ctx)
+{
+ struct upstream_list *ls;
+
+ ls = g_malloc0(sizeof(*ls));
+ ls->hash_seed = SEED_CONSTANT;
+ ls->ups = g_ptr_array_new();
+ ls->alive = g_ptr_array_new();
+
+#ifdef UPSTREAMS_THREAD_SAFE
+ ls->lock = rspamd_mutex_new();
+#endif
+ ls->cur_elt = 0;
+ ls->ctx = ctx;
+ ls->rot_alg = RSPAMD_UPSTREAM_UNDEF;
+
+ if (ctx) {
+ ls->limits = &ctx->limits;
+ }
+ else {
+ ls->limits = &default_limits;
+ }
+
+ return ls;
+}
+
+gsize rspamd_upstreams_count(struct upstream_list *ups)
+{
+ return ups != NULL ? ups->ups->len : 0;
+}
+
+gsize rspamd_upstreams_alive(struct upstream_list *ups)
+{
+ return ups != NULL ? ups->alive->len : 0;
+}
+
+static void
+rspamd_upstream_dtor(struct upstream *up)
+{
+ struct upstream_inet_addr_entry *cur, *tmp;
+
+ if (up->new_addrs) {
+ LL_FOREACH_SAFE(up->new_addrs, cur, tmp)
+ {
+ /* Here we need to free pointer as well */
+ rspamd_inet_address_free(cur->addr);
+ g_free(cur);
+ }
+ }
+
+ if (up->addrs.addr) {
+ g_ptr_array_free(up->addrs.addr, TRUE);
+ }
+
+#ifdef UPSTREAMS_THREAD_SAFE
+ rspamd_mutex_free(up->lock);
+#endif
+
+ if (up->ctx) {
+
+ if (ev_can_stop(&up->ev)) {
+ ev_timer_stop(up->ctx->event_loop, &up->ev);
+ }
+
+ g_queue_delete_link(up->ctx->upstreams, up->ctx_pos);
+ REF_RELEASE(up->ctx);
+ }
+
+ g_free(up);
+}
+
+rspamd_inet_addr_t *
+rspamd_upstream_addr_next(struct upstream *up)
+{
+ guint idx, next_idx;
+ struct upstream_addr_elt *e1, *e2;
+
+ do {
+ idx = up->addrs.cur;
+ next_idx = (idx + 1) % up->addrs.addr->len;
+ e1 = g_ptr_array_index(up->addrs.addr, idx);
+ e2 = g_ptr_array_index(up->addrs.addr, next_idx);
+ up->addrs.cur = next_idx;
+ } while (e2->errors > e1->errors);
+
+ return e2->addr;
+}
+
+rspamd_inet_addr_t *
+rspamd_upstream_addr_cur(const struct upstream *up)
+{
+ struct upstream_addr_elt *elt;
+
+ elt = g_ptr_array_index(up->addrs.addr, up->addrs.cur);
+
+ return elt->addr;
+}
+
+const gchar *
+rspamd_upstream_name(struct upstream *up)
+{
+ return up->name;
+}
+
+gint rspamd_upstream_port(struct upstream *up)
+{
+ struct upstream_addr_elt *elt;
+
+ elt = g_ptr_array_index(up->addrs.addr, up->addrs.cur);
+ return rspamd_inet_address_get_port(elt->addr);
+}
+
+gboolean
+rspamd_upstreams_add_upstream(struct upstream_list *ups, const gchar *str,
+ guint16 def_port, enum rspamd_upstream_parse_type parse_type,
+ void *data)
+{
+ struct upstream *upstream;
+ GPtrArray *addrs = NULL;
+ guint i, slen;
+ rspamd_inet_addr_t *addr;
+ enum rspamd_parse_host_port_result ret = RSPAMD_PARSE_ADDR_FAIL;
+
+ upstream = g_malloc0(sizeof(*upstream));
+ slen = strlen(str);
+
+ switch (parse_type) {
+ case RSPAMD_UPSTREAM_PARSE_DEFAULT:
+ if (slen > sizeof("service=") &&
+ RSPAMD_LEN_CHECK_STARTS_WITH(str, slen, "service=")) {
+ const gchar *plus_pos, *service_pos, *semicolon_pos;
+
+ /* Accept service=srv_name+hostname[:priority] */
+ service_pos = str + sizeof("service=") - 1;
+ plus_pos = strchr(service_pos, '+');
+
+ if (plus_pos != NULL) {
+ semicolon_pos = strchr(plus_pos + 1, ':');
+
+ if (semicolon_pos) {
+ upstream->weight = strtoul(semicolon_pos + 1, NULL, 10);
+ }
+ else {
+ semicolon_pos = plus_pos + strlen(plus_pos);
+ }
+
+ /*
+ * Now our name is _service._tcp.<domain>
+ * where <domain> is string between semicolon_pos and plus_pos +1
+ * while service is a string between service_pos and plus_pos
+ */
+ guint namelen = (semicolon_pos - (plus_pos + 1)) +
+ (plus_pos - service_pos) +
+ (sizeof("tcp") - 1) +
+ 4;
+ addrs = g_ptr_array_sized_new(1);
+ upstream->name = ups->ctx ? rspamd_mempool_alloc(ups->ctx->pool, namelen + 1) : g_malloc(namelen + 1);
+
+ rspamd_snprintf(upstream->name, namelen + 1,
+ "_%*s._tcp.%*s",
+ (gint) (plus_pos - service_pos), service_pos,
+ (gint) (semicolon_pos - (plus_pos + 1)), plus_pos + 1);
+ upstream->flags |= RSPAMD_UPSTREAM_FLAG_SRV_RESOLVE;
+ ret = RSPAMD_PARSE_ADDR_RESOLVED;
+ }
+ }
+ else {
+ ret = rspamd_parse_host_port_priority(str, &addrs,
+ &upstream->weight,
+ &upstream->name, def_port,
+ FALSE,
+ ups->ctx ? ups->ctx->pool : NULL);
+ }
+ break;
+ case RSPAMD_UPSTREAM_PARSE_NAMESERVER:
+ addrs = g_ptr_array_sized_new(1);
+ if (rspamd_parse_inet_address(&addr, str, strlen(str),
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ if (ups->ctx) {
+ upstream->name = rspamd_mempool_strdup(ups->ctx->pool, str);
+ }
+ else {
+ upstream->name = g_strdup(str);
+ }
+ if (rspamd_inet_address_get_port(addr) == 0) {
+ rspamd_inet_address_set_port(addr, def_port);
+ }
+
+ g_ptr_array_add(addrs, addr);
+ ret = RSPAMD_PARSE_ADDR_NUMERIC;
+
+ if (ups->ctx) {
+ rspamd_mempool_add_destructor(ups->ctx->pool,
+ (rspamd_mempool_destruct_t) rspamd_inet_address_free,
+ addr);
+ rspamd_mempool_add_destructor(ups->ctx->pool,
+ (rspamd_mempool_destruct_t) rspamd_ptr_array_free_hard,
+ addrs);
+ }
+ }
+ else {
+ g_ptr_array_free(addrs, TRUE);
+ }
+
+ break;
+ }
+
+ if (ret == RSPAMD_PARSE_ADDR_FAIL) {
+ g_free(upstream);
+ return FALSE;
+ }
+ else {
+ upstream->flags |= ups->flags;
+
+ if (ret == RSPAMD_PARSE_ADDR_NUMERIC) {
+ /* Add noresolve flag */
+ upstream->flags |= RSPAMD_UPSTREAM_FLAG_NORESOLVE;
+ }
+ for (i = 0; i < addrs->len; i++) {
+ addr = g_ptr_array_index(addrs, i);
+ rspamd_upstream_add_addr(upstream, rspamd_inet_address_copy(addr, NULL));
+ }
+ }
+
+ if (upstream->weight == 0 && ups->rot_alg == RSPAMD_UPSTREAM_MASTER_SLAVE) {
+ /* Special heuristic for master-slave rotation */
+ if (ups->ups->len == 0) {
+ /* Prioritize the first */
+ upstream->weight = 1;
+ }
+ }
+
+ g_ptr_array_add(ups->ups, upstream);
+ upstream->ud = data;
+ upstream->cur_weight = upstream->weight;
+ upstream->ls = ups;
+ REF_INIT_RETAIN(upstream, rspamd_upstream_dtor);
+#ifdef UPSTREAMS_THREAD_SAFE
+ upstream->lock = rspamd_mutex_new();
+#endif
+ upstream->ctx = ups->ctx;
+
+ if (upstream->ctx) {
+ REF_RETAIN(ups->ctx);
+ g_queue_push_tail(ups->ctx->upstreams, upstream);
+ upstream->ctx_pos = g_queue_peek_tail_link(ups->ctx->upstreams);
+ }
+
+ guint h = rspamd_cryptobox_fast_hash(upstream->name,
+ strlen(upstream->name), 0);
+ memset(upstream->uid, 0, sizeof(upstream->uid));
+ rspamd_encode_base32_buf((const guchar *) &h, sizeof(h),
+ upstream->uid, sizeof(upstream->uid) - 1, RSPAMD_BASE32_DEFAULT);
+
+ msg_debug_upstream("added upstream %s (%s)", upstream->name,
+ upstream->flags & RSPAMD_UPSTREAM_FLAG_NORESOLVE ? "numeric ip" : "DNS name");
+ g_ptr_array_sort(upstream->addrs.addr, rspamd_upstream_addr_sort_func);
+ rspamd_upstream_set_active(ups, upstream);
+
+ return TRUE;
+}
+
+void rspamd_upstreams_set_flags(struct upstream_list *ups,
+ enum rspamd_upstream_flag flags)
+{
+ ups->flags = flags;
+}
+
+void rspamd_upstreams_set_rotation(struct upstream_list *ups,
+ enum rspamd_upstream_rotation rot)
+{
+ ups->rot_alg = rot;
+}
+
+gboolean
+rspamd_upstream_add_addr(struct upstream *up, rspamd_inet_addr_t *addr)
+{
+ struct upstream_addr_elt *elt;
+ /*
+ * XXX: slow and inefficient
+ */
+ if (up->addrs.addr == NULL) {
+ up->addrs.addr = g_ptr_array_new_full(8, rspamd_upstream_addr_elt_dtor);
+ }
+
+ elt = g_malloc0(sizeof(*elt));
+ elt->addr = addr;
+ g_ptr_array_add(up->addrs.addr, elt);
+ g_ptr_array_sort(up->addrs.addr, rspamd_upstream_addr_sort_func);
+
+ return TRUE;
+}
+
+gboolean
+rspamd_upstreams_parse_line_len(struct upstream_list *ups,
+ const gchar *str, gsize len, guint16 def_port, void *data)
+{
+ const gchar *end = str + len, *p = str;
+ const gchar *separators = ";, \n\r\t";
+ gchar *tmp;
+ guint span_len;
+ gboolean ret = FALSE;
+
+ if (RSPAMD_LEN_CHECK_STARTS_WITH(p, len, "random:")) {
+ ups->rot_alg = RSPAMD_UPSTREAM_RANDOM;
+ p += sizeof("random:") - 1;
+ }
+ else if (RSPAMD_LEN_CHECK_STARTS_WITH(p, len, "master-slave:")) {
+ ups->rot_alg = RSPAMD_UPSTREAM_MASTER_SLAVE;
+ p += sizeof("master-slave:") - 1;
+ }
+ else if (RSPAMD_LEN_CHECK_STARTS_WITH(p, len, "round-robin:")) {
+ ups->rot_alg = RSPAMD_UPSTREAM_ROUND_ROBIN;
+ p += sizeof("round-robin:") - 1;
+ }
+ else if (RSPAMD_LEN_CHECK_STARTS_WITH(p, len, "hash:")) {
+ ups->rot_alg = RSPAMD_UPSTREAM_HASHED;
+ p += sizeof("hash:") - 1;
+ }
+
+ while (p < end) {
+ span_len = rspamd_memcspn(p, separators, end - p);
+
+ if (span_len > 0) {
+ tmp = g_malloc(span_len + 1);
+ rspamd_strlcpy(tmp, p, span_len + 1);
+
+ if (rspamd_upstreams_add_upstream(ups, tmp, def_port,
+ RSPAMD_UPSTREAM_PARSE_DEFAULT,
+ data)) {
+ ret = TRUE;
+ }
+
+ g_free(tmp);
+ }
+
+ p += span_len;
+ /* Skip separators */
+ if (p < end) {
+ p += rspamd_memspn(p, separators, end - p);
+ }
+ }
+
+ if (!ups->ups_line) {
+ ups->ups_line = g_malloc(len + 1);
+ rspamd_strlcpy(ups->ups_line, str, len + 1);
+ }
+
+ return ret;
+}
+
+
+gboolean
+rspamd_upstreams_parse_line(struct upstream_list *ups,
+ const gchar *str, guint16 def_port, void *data)
+{
+ return rspamd_upstreams_parse_line_len(ups, str, strlen(str),
+ def_port, data);
+}
+
+gboolean
+rspamd_upstreams_from_ucl(struct upstream_list *ups,
+ const ucl_object_t *in, guint16 def_port, void *data)
+{
+ gboolean ret = FALSE;
+ const ucl_object_t *cur;
+ ucl_object_iter_t it = NULL;
+
+ it = ucl_object_iterate_new(in);
+
+ while ((cur = ucl_object_iterate_safe(it, true)) != NULL) {
+ if (ucl_object_type(cur) == UCL_STRING) {
+ ret = rspamd_upstreams_parse_line(ups, ucl_object_tostring(cur),
+ def_port, data);
+ }
+ }
+
+ ucl_object_iterate_free(it);
+
+ return ret;
+}
+
+void rspamd_upstreams_destroy(struct upstream_list *ups)
+{
+ guint i;
+ struct upstream *up;
+ struct upstream_list_watcher *w, *tmp;
+
+ if (ups != NULL) {
+ g_ptr_array_free(ups->alive, TRUE);
+
+ for (i = 0; i < ups->ups->len; i++) {
+ up = g_ptr_array_index(ups->ups, i);
+ up->ls = NULL;
+ REF_RELEASE(up);
+ }
+
+ DL_FOREACH_SAFE(ups->watchers, w, tmp)
+ {
+ if (w->dtor) {
+ w->dtor(w->ud);
+ }
+ g_free(w);
+ }
+
+ g_free(ups->ups_line);
+ g_ptr_array_free(ups->ups, TRUE);
+#ifdef UPSTREAMS_THREAD_SAFE
+ rspamd_mutex_free(ups->lock);
+#endif
+ g_free(ups);
+ }
+}
+
+static void
+rspamd_upstream_restore_cb(gpointer elt, gpointer ls)
+{
+ struct upstream *up = (struct upstream *) elt;
+ struct upstream_list *ups = (struct upstream_list *) ls;
+ struct upstream_list_watcher *w;
+
+ /* Here the upstreams list is already locked */
+ RSPAMD_UPSTREAM_LOCK(up);
+
+ if (ev_can_stop(&up->ev)) {
+ ev_timer_stop(up->ctx->event_loop, &up->ev);
+ }
+
+ g_ptr_array_add(ups->alive, up);
+ up->active_idx = ups->alive->len - 1;
+ RSPAMD_UPSTREAM_UNLOCK(up);
+
+ DL_FOREACH(up->ls->watchers, w)
+ {
+ if (w->events_mask & RSPAMD_UPSTREAM_WATCH_ONLINE) {
+ w->func(up, RSPAMD_UPSTREAM_WATCH_ONLINE, up->errors, w->ud);
+ }
+ }
+
+ /* For revive event */
+ g_assert(up->ref.refcount > 1);
+ REF_RELEASE(up);
+}
+
+static struct upstream *
+rspamd_upstream_get_random(struct upstream_list *ups,
+ struct upstream *except)
+{
+ for (;;) {
+ guint idx = ottery_rand_range(ups->alive->len - 1);
+ struct upstream *up;
+
+ up = g_ptr_array_index(ups->alive, idx);
+
+ if (except && up == except) {
+ continue;
+ }
+
+ return up;
+ }
+}
+
+static struct upstream *
+rspamd_upstream_get_round_robin(struct upstream_list *ups,
+ struct upstream *except,
+ gboolean use_cur)
+{
+ guint max_weight = 0, min_checked = G_MAXUINT;
+ struct upstream *up = NULL, *selected = NULL, *min_checked_sel = NULL;
+ guint i;
+
+ /* Select upstream with the maximum cur_weight */
+ RSPAMD_UPSTREAM_LOCK(ups);
+
+ for (i = 0; i < ups->alive->len; i++) {
+ up = g_ptr_array_index(ups->alive, i);
+
+ if (except != NULL && up == except) {
+ continue;
+ }
+
+ if (use_cur) {
+ if (up->cur_weight > max_weight) {
+ selected = up;
+ max_weight = up->cur_weight;
+ }
+ }
+ else {
+ if (up->weight > max_weight) {
+ selected = up;
+ max_weight = up->weight;
+ }
+ }
+
+ /*
+ * This code is used when all upstreams have zero weight
+ * The logic is to select least currently used upstream and penalise
+ * upstream with errors. The error penalty should no be too high
+ * to avoid sudden traffic drop in this case.
+ */
+ if (up->checked + up->errors * 2 < min_checked) {
+ min_checked_sel = up;
+ min_checked = up->checked;
+ }
+ }
+
+ if (max_weight == 0) {
+ /* All upstreams have zero weight */
+ if (min_checked > G_MAXUINT / 2) {
+ /* Reset all checked counters to avoid overflow */
+ for (i = 0; i < ups->alive->len; i++) {
+ up = g_ptr_array_index(ups->alive, i);
+ up->checked = 0;
+ }
+ }
+
+ selected = min_checked_sel;
+ }
+
+ if (use_cur && selected) {
+ if (selected->cur_weight > 0) {
+ selected->cur_weight--;
+ }
+ else {
+ selected->cur_weight = selected->weight;
+ }
+ }
+
+ RSPAMD_UPSTREAM_UNLOCK(ups);
+
+ return selected;
+}
+
+/*
+ * The key idea of this function is obtained from the following paper:
+ * A Fast, Minimal Memory, Consistent Hash Algorithm
+ * John Lamping, Eric Veach
+ *
+ * http://arxiv.org/abs/1406.2294
+ */
+static guint32
+rspamd_consistent_hash(guint64 key, guint32 nbuckets)
+{
+ gint64 b = -1, j = 0;
+
+ while (j < nbuckets) {
+ b = j;
+ key *= 2862933555777941757ULL + 1;
+ j = (b + 1) * (double) (1ULL << 31) / (double) ((key >> 33) + 1ULL);
+ }
+
+ return b;
+}
+
+static struct upstream *
+rspamd_upstream_get_hashed(struct upstream_list *ups,
+ struct upstream *except,
+ const guint8 *key, guint keylen)
+{
+ guint64 k;
+ guint32 idx;
+ static const guint max_tries = 20;
+ struct upstream *up = NULL;
+
+ /* Generate 64 bits input key */
+ k = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64,
+ key, keylen, ups->hash_seed);
+
+ RSPAMD_UPSTREAM_LOCK(ups);
+ /*
+ * Select new upstream from all upstreams
+ */
+ for (guint i = 0; i < max_tries; i++) {
+ idx = rspamd_consistent_hash(k, ups->ups->len);
+ up = g_ptr_array_index(ups->ups, idx);
+
+ if (up->active_idx < 0 || (except != NULL && up == except)) {
+ /* Found inactive or excluded upstream */
+ k = mum_hash_step(k, ups->hash_seed);
+ }
+ else {
+ break;
+ }
+ }
+ RSPAMD_UPSTREAM_UNLOCK(ups);
+
+ if (up->active_idx >= 0) {
+ return up;
+ }
+
+ /* We failed to find any active upstream */
+ up = rspamd_upstream_get_random(ups, except);
+ msg_info("failed to find hashed upstream for %s, fallback to random: %s",
+ ups->ups_line, up->name);
+
+ return up;
+}
+
+static struct upstream *
+rspamd_upstream_get_common(struct upstream_list *ups,
+ struct upstream *except,
+ enum rspamd_upstream_rotation default_type,
+ const guchar *key, gsize keylen,
+ gboolean forced)
+{
+ enum rspamd_upstream_rotation type;
+ struct upstream *up = NULL;
+
+ RSPAMD_UPSTREAM_LOCK(ups);
+ if (ups->alive->len == 0) {
+ /* We have no upstreams alive */
+ msg_warn("there are no alive upstreams left for %s, revive all of them",
+ ups->ups_line);
+ g_ptr_array_foreach(ups->ups, rspamd_upstream_restore_cb, ups);
+ }
+ RSPAMD_UPSTREAM_UNLOCK(ups);
+
+ if (ups->alive->len == 1 && default_type != RSPAMD_UPSTREAM_SEQUENTIAL) {
+ /* Fast path */
+ up = g_ptr_array_index(ups->alive, 0);
+ goto end;
+ }
+
+ if (!forced) {
+ type = ups->rot_alg != RSPAMD_UPSTREAM_UNDEF ? ups->rot_alg : default_type;
+ }
+ else {
+ type = default_type != RSPAMD_UPSTREAM_UNDEF ? default_type : ups->rot_alg;
+ }
+
+ if (type == RSPAMD_UPSTREAM_HASHED && (keylen == 0 || key == NULL)) {
+ /* Cannot use hashed rotation when no key is specified, switch to random */
+ type = RSPAMD_UPSTREAM_RANDOM;
+ }
+
+ switch (type) {
+ default:
+ case RSPAMD_UPSTREAM_RANDOM:
+ up = rspamd_upstream_get_random(ups, except);
+ break;
+ case RSPAMD_UPSTREAM_HASHED:
+ up = rspamd_upstream_get_hashed(ups, except, key, keylen);
+ break;
+ case RSPAMD_UPSTREAM_ROUND_ROBIN:
+ up = rspamd_upstream_get_round_robin(ups, except, TRUE);
+ break;
+ case RSPAMD_UPSTREAM_MASTER_SLAVE:
+ up = rspamd_upstream_get_round_robin(ups, except, FALSE);
+ break;
+ case RSPAMD_UPSTREAM_SEQUENTIAL:
+ if (ups->cur_elt >= ups->alive->len) {
+ ups->cur_elt = 0;
+ return NULL;
+ }
+
+ up = g_ptr_array_index(ups->alive, ups->cur_elt++);
+ break;
+ }
+
+end:
+ if (up) {
+ up->checked++;
+ }
+
+ return up;
+}
+
+struct upstream *
+rspamd_upstream_get(struct upstream_list *ups,
+ enum rspamd_upstream_rotation default_type,
+ const guchar *key, gsize keylen)
+{
+ return rspamd_upstream_get_common(ups, NULL, default_type, key, keylen, FALSE);
+}
+
+struct upstream *
+rspamd_upstream_get_forced(struct upstream_list *ups,
+ enum rspamd_upstream_rotation forced_type,
+ const guchar *key, gsize keylen)
+{
+ return rspamd_upstream_get_common(ups, NULL, forced_type, key, keylen, TRUE);
+}
+
+struct upstream *rspamd_upstream_get_except(struct upstream_list *ups,
+ struct upstream *except,
+ enum rspamd_upstream_rotation default_type,
+ const guchar *key, gsize keylen)
+{
+ return rspamd_upstream_get_common(ups, except, default_type, key, keylen, FALSE);
+}
+
+void rspamd_upstream_reresolve(struct upstream_ctx *ctx)
+{
+ GList *cur;
+ struct upstream *up;
+
+ cur = ctx->upstreams->head;
+
+ while (cur) {
+ up = cur->data;
+ REF_RETAIN(up);
+ rspamd_upstream_resolve_addrs(up->ls, up);
+ REF_RELEASE(up);
+ cur = g_list_next(cur);
+ }
+}
+
+gpointer
+rspamd_upstream_set_data(struct upstream *up, gpointer data)
+{
+ gpointer prev_data = up->data;
+ up->data = data;
+
+ return prev_data;
+}
+
+gpointer
+rspamd_upstream_get_data(struct upstream *up)
+{
+ return up->data;
+}
+
+
+void rspamd_upstreams_foreach(struct upstream_list *ups,
+ rspamd_upstream_traverse_func cb, void *ud)
+{
+ struct upstream *up;
+ guint i;
+
+ for (i = 0; i < ups->ups->len; i++) {
+ up = g_ptr_array_index(ups->ups, i);
+
+ cb(up, i, ud);
+ }
+}
+
+void rspamd_upstreams_set_limits(struct upstream_list *ups,
+ gdouble revive_time,
+ gdouble revive_jitter,
+ gdouble error_time,
+ gdouble dns_timeout,
+ guint max_errors,
+ guint dns_retransmits)
+{
+ struct upstream_limits *nlimits;
+ g_assert(ups != NULL);
+
+ nlimits = rspamd_mempool_alloc(ups->ctx->pool, sizeof(*nlimits));
+ memcpy(nlimits, ups->limits, sizeof(*nlimits));
+
+ if (!isnan(revive_time)) {
+ nlimits->revive_time = revive_time;
+ }
+
+ if (!isnan(revive_jitter)) {
+ nlimits->revive_jitter = revive_jitter;
+ }
+
+ if (!isnan(error_time)) {
+ nlimits->error_time = error_time;
+ }
+
+ if (!isnan(dns_timeout)) {
+ nlimits->dns_timeout = dns_timeout;
+ }
+
+ if (max_errors > 0) {
+ nlimits->max_errors = max_errors;
+ }
+
+ if (dns_retransmits > 0) {
+ nlimits->dns_retransmits = dns_retransmits;
+ }
+
+ ups->limits = nlimits;
+}
+
+void rspamd_upstreams_add_watch_callback(struct upstream_list *ups,
+ enum rspamd_upstreams_watch_event events,
+ rspamd_upstream_watch_func func,
+ GFreeFunc dtor,
+ gpointer ud)
+{
+ struct upstream_list_watcher *nw;
+
+ g_assert((events & RSPAMD_UPSTREAM_WATCH_ALL) != 0);
+
+ nw = g_malloc(sizeof(*nw));
+ nw->func = func;
+ nw->events_mask = events;
+ nw->ud = ud;
+ nw->dtor = dtor;
+
+ DL_APPEND(ups->watchers, nw);
+}
+
+struct upstream *
+rspamd_upstream_ref(struct upstream *up)
+{
+ REF_RETAIN(up);
+ return up;
+}
+
+void rspamd_upstream_unref(struct upstream *up)
+{
+ REF_RELEASE(up);
+}
diff --git a/src/libutil/upstream.h b/src/libutil/upstream.h
new file mode 100644
index 0000000..22a020c
--- /dev/null
+++ b/src/libutil/upstream.h
@@ -0,0 +1,344 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef UPSTREAM_H
+#define UPSTREAM_H
+
+#include "config.h"
+#include "util.h"
+#include "rdns.h"
+#include "ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Forward declaration */
+struct ev_loop;
+
+enum rspamd_upstream_rotation {
+ RSPAMD_UPSTREAM_RANDOM = 0,
+ RSPAMD_UPSTREAM_HASHED,
+ RSPAMD_UPSTREAM_ROUND_ROBIN,
+ RSPAMD_UPSTREAM_MASTER_SLAVE,
+ RSPAMD_UPSTREAM_SEQUENTIAL,
+ RSPAMD_UPSTREAM_UNDEF
+};
+
+enum rspamd_upstream_flag {
+ RSPAMD_UPSTREAM_FLAG_NORESOLVE = (1 << 0),
+ RSPAMD_UPSTREAM_FLAG_SRV_RESOLVE = (1 << 1),
+};
+
+struct rspamd_config;
+/* Opaque upstream structures */
+struct upstream;
+struct upstream_list;
+struct upstream_ctx;
+
+/**
+ * Init upstreams library
+ * @param resolver
+ */
+struct upstream_ctx *rspamd_upstreams_library_init(void);
+
+/**
+ * Remove reference from upstreams library
+ */
+void rspamd_upstreams_library_unref(struct upstream_ctx *ctx);
+
+/**
+ * Configure attributes of upstreams library
+ * @param cfg
+ */
+void rspamd_upstreams_library_config(struct rspamd_config *cfg,
+ struct upstream_ctx *ctx, struct ev_loop *event_loop,
+ struct rdns_resolver *resolver);
+
+/**
+ * Upstream error logic
+ * 1. During error time we count upstream_ok and upstream_fail
+ * 2. If failcount is more then maxerrors then we mark upstream as unavailable for dead time
+ * 3. After dead time we mark upstream as alive and go to the step 1
+ * 4. If all upstreams are dead, marks every upstream as alive
+ */
+
+/**
+ * Add an error to an upstream
+ */
+void rspamd_upstream_fail(struct upstream *upstream, gboolean addr_failure, const gchar *reason);
+
+/**
+ * Increase upstream successes count
+ */
+void rspamd_upstream_ok(struct upstream *up);
+
+/**
+ * Set weight for an upstream
+ * @param up
+ */
+void rspamd_upstream_set_weight(struct upstream *up, guint weight);
+
+/**
+ * Create new list of upstreams
+ * @return
+ */
+struct upstream_list *rspamd_upstreams_create(struct upstream_ctx *ctx);
+
+/**
+ * Sets specific flag to the upstream list
+ * @param ups
+ * @param flags
+ */
+void rspamd_upstreams_set_flags(struct upstream_list *ups,
+ enum rspamd_upstream_flag flags);
+
+/**
+ * Sets custom limits for upstreams
+ * This function allocates memory from the upstreams ctx pool and should
+ * not be called in cycles/constantly as this memory is likely persistent
+ * @param ups
+ * @param revive_time
+ * @param revive_jitter
+ * @param error_time
+ * @param dns_timeout
+ * @param max_errors
+ * @param dns_retransmits
+ */
+void rspamd_upstreams_set_limits(struct upstream_list *ups,
+ gdouble revive_time,
+ gdouble revive_jitter,
+ gdouble error_time,
+ gdouble dns_timeout,
+ guint max_errors,
+ guint dns_retransmits);
+
+/**
+ * Sets rotation policy for upstreams list
+ * @param ups
+ * @param rot
+ */
+void rspamd_upstreams_set_rotation(struct upstream_list *ups,
+ enum rspamd_upstream_rotation rot);
+
+/**
+ * Destroy list of upstreams
+ * @param ups
+ */
+void rspamd_upstreams_destroy(struct upstream_list *ups);
+
+/**
+ * Returns count of upstreams in a list
+ * @param ups
+ * @return
+ */
+gsize rspamd_upstreams_count(struct upstream_list *ups);
+
+/**
+ * Returns the number of upstreams in the list
+ * @param ups
+ * @return
+ */
+gsize rspamd_upstreams_alive(struct upstream_list *ups);
+
+enum rspamd_upstream_parse_type {
+ RSPAMD_UPSTREAM_PARSE_DEFAULT = 0,
+ RSPAMD_UPSTREAM_PARSE_NAMESERVER,
+};
+
+/**
+ * Add upstream from the string
+ * @param ups upstream list
+ * @param str string in format "name[:port[:priority]]"
+ * @param def_port default port number
+ * @param data optional userdata
+ * @return TRUE if upstream has been added
+ */
+gboolean rspamd_upstreams_add_upstream(struct upstream_list *ups, const gchar *str,
+ guint16 def_port, enum rspamd_upstream_parse_type parse_type,
+ void *data);
+
+/**
+ * Add multiple upstreams from comma, semicolon or space separated line
+ * @param ups upstream list
+ * @param str string in format "(<ups>([<sep>+]<ups>)*)+"
+ * @param def_port default port number
+ * @param data optional userdata
+ * @return TRUE if **any** of upstreams has been added
+ */
+gboolean rspamd_upstreams_parse_line(struct upstream_list *ups,
+ const gchar *str, guint16 def_port, void *data);
+
+
+gboolean rspamd_upstreams_parse_line_len(struct upstream_list *ups,
+ const gchar *str, gsize len,
+ guint16 def_port,
+ void *data);
+
+/**
+ * Parse upstreams list from the UCL object
+ * @param ups
+ * @param in
+ * @param def_port
+ * @param data
+ * @return
+ */
+gboolean rspamd_upstreams_from_ucl(struct upstream_list *ups,
+ const ucl_object_t *in, guint16 def_port, void *data);
+
+
+typedef void (*rspamd_upstream_traverse_func)(struct upstream *up, guint idx,
+ void *ud);
+
+/**
+ * Traverse upstreams list calling the function specified
+ * @param ups
+ * @param cb
+ * @param ud
+ */
+void rspamd_upstreams_foreach(struct upstream_list *ups,
+ rspamd_upstream_traverse_func cb, void *ud);
+
+enum rspamd_upstreams_watch_event {
+ RSPAMD_UPSTREAM_WATCH_SUCCESS = 1u << 0,
+ RSPAMD_UPSTREAM_WATCH_FAILURE = 1u << 1,
+ RSPAMD_UPSTREAM_WATCH_OFFLINE = 1u << 2,
+ RSPAMD_UPSTREAM_WATCH_ONLINE = 1u << 3,
+ RSPAMD_UPSTREAM_WATCH_ALL = (1u << 0) | (1u << 1) | (1u << 2) | (1u << 3),
+};
+
+typedef void (*rspamd_upstream_watch_func)(struct upstream *up,
+ enum rspamd_upstreams_watch_event event,
+ guint cur_errors,
+ void *ud);
+
+/**
+ * Adds new watcher to the upstreams list
+ * @param ups
+ * @param events
+ * @param func
+ * @param ud
+ */
+void rspamd_upstreams_add_watch_callback(struct upstream_list *ups,
+ enum rspamd_upstreams_watch_event events,
+ rspamd_upstream_watch_func func,
+ GFreeFunc free_func,
+ gpointer ud);
+
+/**
+ * Returns the next IP address of the upstream (internal rotation)
+ * @param up
+ * @return
+ */
+rspamd_inet_addr_t *rspamd_upstream_addr_next(struct upstream *up);
+
+/**
+ * Returns the current IP address of the upstream
+ * @param up
+ * @return
+ */
+rspamd_inet_addr_t *rspamd_upstream_addr_cur(const struct upstream *up);
+
+/**
+ * Add custom address for an upstream (ownership of addr is transferred to upstream)
+ * @param up
+ * @return
+ */
+gboolean rspamd_upstream_add_addr(struct upstream *up,
+ rspamd_inet_addr_t *addr);
+
+/**
+ * Returns the symbolic name of the upstream
+ * @param up
+ * @return
+ */
+const gchar *rspamd_upstream_name(struct upstream *up);
+
+/**
+ * Returns the port of the current address for the upstream
+ * @param up
+ * @return
+ */
+gint rspamd_upstream_port(struct upstream *up);
+
+/**
+ * Sets opaque user data associated with this upstream
+ * @param up
+ * @param data
+ * @return old data
+ */
+gpointer rspamd_upstream_set_data(struct upstream *up, gpointer data);
+
+/**
+ * Gets opaque user data associated with this upstream
+ * @param up
+ * @return
+ */
+gpointer rspamd_upstream_get_data(struct upstream *up);
+
+/**
+ * Get new upstream from the list
+ * @param ups upstream list
+ * @param type type of rotation algorithm, for `RSPAMD_UPSTREAM_HASHED` it is required to specify `key` and `keylen` as arguments
+ * @return
+ */
+struct upstream *rspamd_upstream_get(struct upstream_list *ups,
+ enum rspamd_upstream_rotation default_type,
+ const guchar *key, gsize keylen);
+
+/**
+ * Get new upstream from the list
+ * @param ups upstream list
+ * @param type type of rotation algorithm, for `RSPAMD_UPSTREAM_HASHED` it is required to specify `key` and `keylen` as arguments
+ * @return
+ */
+struct upstream *rspamd_upstream_get_forced(struct upstream_list *ups,
+ enum rspamd_upstream_rotation forced_type,
+ const guchar *key, gsize keylen);
+
+/**
+ * Get new upstream from the list excepting the upstream specified
+ * @param ups upstream list
+ * @param type type of rotation algorithm, for `RSPAMD_UPSTREAM_HASHED` it is required to specify `key` and `keylen` as arguments
+ * @return
+ */
+struct upstream *rspamd_upstream_get_except(struct upstream_list *ups,
+ struct upstream *except,
+ enum rspamd_upstream_rotation default_type,
+ const guchar *key, gsize keylen);
+
+/**
+ * Re-resolve addresses for all upstreams registered
+ */
+void rspamd_upstream_reresolve(struct upstream_ctx *ctx);
+
+/**
+ * Share ownership on upstream
+ * @param up
+ * @return
+ */
+struct upstream *rspamd_upstream_ref(struct upstream *up);
+/**
+ * Unshare ownership on upstream
+ * @param up
+ */
+void rspamd_upstream_unref(struct upstream *up);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* UPSTREAM_H */
diff --git a/src/libutil/uthash_strcase.h b/src/libutil/uthash_strcase.h
new file mode 100644
index 0000000..86075ee
--- /dev/null
+++ b/src/libutil/uthash_strcase.h
@@ -0,0 +1,91 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef UTHASH_STRCASE_H_
+#define UTHASH_STRCASE_H_
+
+#ifdef UTHASH_H
+#error Invalid include order: uthash is already included
+#endif
+
+#include "libcryptobox/cryptobox.h"
+#include "libutil/util.h"
+
+/* Utils for uthash tuning */
+#ifndef HASH_CASELESS
+#define HASH_FUNCTION(key, keylen, num_bkts, hashv, bkt) \
+ do { \
+ hashv = (__typeof(hashv)) rspamd_cryptobox_fast_hash(key, keylen, rspamd_hash_seed()); \
+ bkt = (hashv) & (num_bkts - 1); \
+ } while (0)
+
+#define HASH_KEYCMP(a, b, len) memcmp(a, b, len)
+#else
+#define HASH_FUNCTION(key, keylen, num_bkts, hashv, bkt) \
+ do { \
+ unsigned _len = keylen; \
+ rspamd_cryptobox_fast_hash_state_t _hst; \
+ unsigned _leftover = keylen % 8; \
+ unsigned _fp, _i; \
+ const uint8_t *_s = (const uint8_t *) (key); \
+ union { \
+ struct { \
+ unsigned char c1, c2, c3, c4, c5, c6, c7, c8; \
+ } c; \
+ uint64_t pp; \
+ } _u; \
+ _fp = _len - _leftover; \
+ rspamd_cryptobox_fast_hash_init(&_hst, rspamd_hash_seed()); \
+ for (_i = 0; _i != _fp; _i += 8) { \
+ _u.c.c1 = _s[_i], _u.c.c2 = _s[_i + 1], _u.c.c3 = _s[_i + 2], _u.c.c4 = _s[_i + 3]; \
+ _u.c.c5 = _s[_i + 4], _u.c.c6 = _s[_i + 5], _u.c.c7 = _s[_i + 6], _u.c.c8 = _s[_i + 7]; \
+ _u.c.c1 = lc_map[_u.c.c1]; \
+ _u.c.c2 = lc_map[_u.c.c2]; \
+ _u.c.c3 = lc_map[_u.c.c3]; \
+ _u.c.c4 = lc_map[_u.c.c4]; \
+ _u.c.c1 = lc_map[_u.c.c5]; \
+ _u.c.c2 = lc_map[_u.c.c6]; \
+ _u.c.c3 = lc_map[_u.c.c7]; \
+ _u.c.c4 = lc_map[_u.c.c8]; \
+ rspamd_cryptobox_fast_hash_update(&_hst, &_u, sizeof(_u)); \
+ } \
+ _u.pp = 0; \
+ switch (_leftover) { \
+ case 7: \
+ /* fallthrough */ _u.c.c7 = lc_map[(unsigned char) _s[_i++]]; \
+ case 6: \
+ /* fallthrough */ _u.c.c6 = lc_map[(unsigned char) _s[_i++]]; \
+ case 5: \
+ /* fallthrough */ _u.c.c5 = lc_map[(unsigned char) _s[_i++]]; \
+ case 4: \
+ /* fallthrough */ _u.c.c4 = lc_map[(unsigned char) _s[_i++]]; \
+ case 3: \
+ /* fallthrough */ _u.c.c3 = lc_map[(unsigned char) _s[_i++]]; \
+ case 2: \
+ /* fallthrough */ _u.c.c2 = lc_map[(unsigned char) _s[_i++]]; \
+ case 1: \
+ /* fallthrough */ _u.c.c1 = lc_map[(unsigned char) _s[_i]]; \
+ rspamd_cryptobox_fast_hash_update(&_hst, &_u, sizeof(_u)); \
+ break; \
+ } \
+ hashv = (__typeof(hashv)) rspamd_cryptobox_fast_hash_final(&_hst); \
+ bkt = (hashv) & (num_bkts - 1); \
+ } while (0)
+#define HASH_KEYCMP(a, b, len) rspamd_lc_cmp(a, b, len)
+#endif
+
+#include "uthash.h"
+
+#endif /* UTHASH_STRCASE_H_ */
diff --git a/src/libutil/util.c b/src/libutil/util.c
new file mode 100644
index 0000000..04200e3
--- /dev/null
+++ b/src/libutil/util.c
@@ -0,0 +1,2746 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "util.h"
+#include "unix-std.h"
+
+#include "ottery.h"
+#include "cryptobox.h"
+#include "contrib/libev/ev.h"
+
+#ifdef HAVE_TERMIOS_H
+#include <termios.h>
+#endif
+#ifdef HAVE_READPASSPHRASE_H
+#include <readpassphrase.h>
+#endif
+/* libutil */
+#ifdef HAVE_LIBUTIL_H
+#include <libutil.h>
+#endif
+#ifdef __APPLE__
+#include <mach/mach_time.h>
+#include <mach/mach_init.h>
+#include <mach/thread_act.h>
+#include <mach/mach_port.h>
+#endif
+/* poll */
+#ifdef HAVE_POLL_H
+#include <poll.h>
+#endif
+
+#ifdef HAVE_SIGINFO_H
+#include <siginfo.h>
+#endif
+/* sys/wait */
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+/* sys/resource.h */
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
+#ifdef HAVE_RDTSC
+#ifdef __x86_64__
+#include <x86intrin.h>
+#endif
+#endif
+
+#include <math.h> /* for pow */
+#include <glob.h> /* in fact, we require this file ultimately */
+
+#include "zlib.h"
+#include "contrib/uthash/utlist.h"
+#include "blas-config.h"
+
+/* Check log messages intensity once per minute */
+#define CHECK_TIME 60
+/* More than 2 log messages per second */
+#define BUF_INTENSITY 2
+/* Default connect timeout for sync sockets */
+#define CONNECT_TIMEOUT 3
+
+/*
+ * Should be defined in a single point
+ */
+const struct rspamd_controller_pbkdf pbkdf_list[] = {
+ {.name = "PBKDF2-blake2b",
+ .alias = "pbkdf2",
+ .description = "standard CPU intensive \"slow\" KDF using blake2b hash function",
+ .type = RSPAMD_CRYPTOBOX_PBKDF2,
+ .id = RSPAMD_PBKDF_ID_V1,
+ .complexity = 16000,
+ .salt_len = 20,
+ .key_len = rspamd_cryptobox_HASHBYTES / 2},
+ {.name = "Catena-Butterfly",
+ .alias = "catena",
+ .description = "modern CPU and memory intensive KDF",
+ .type = RSPAMD_CRYPTOBOX_CATENA,
+ .id = RSPAMD_PBKDF_ID_V2,
+ .complexity = 10,
+ .salt_len = 20,
+ .key_len = rspamd_cryptobox_HASHBYTES / 2}};
+
+gint rspamd_socket_nonblocking(gint fd)
+{
+ gint ofl;
+
+ ofl = fcntl(fd, F_GETFL, 0);
+
+ if (fcntl(fd, F_SETFL, ofl | O_NONBLOCK) == -1) {
+ return -1;
+ }
+ return 0;
+}
+
+gint rspamd_socket_blocking(gint fd)
+{
+ gint ofl;
+
+ ofl = fcntl(fd, F_GETFL, 0);
+
+ if (fcntl(fd, F_SETFL, ofl & (~O_NONBLOCK)) == -1) {
+ return -1;
+ }
+ return 0;
+}
+
+gint rspamd_socket_poll(gint fd, gint timeout, short events)
+{
+ gint r;
+ struct pollfd fds[1];
+
+ fds->fd = fd;
+ fds->events = events;
+ fds->revents = 0;
+ while ((r = poll(fds, 1, timeout)) < 0) {
+ if (errno != EINTR) {
+ break;
+ }
+ }
+
+ return r;
+}
+
+gint rspamd_socket_create(gint af, gint type, gint protocol, gboolean async)
+{
+ gint fd;
+
+ fd = socket(af, type, protocol);
+ if (fd == -1) {
+ return -1;
+ }
+
+ /* Set close on exec */
+ if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) {
+ close(fd);
+ return -1;
+ }
+ if (async) {
+ if (rspamd_socket_nonblocking(fd) == -1) {
+ close(fd);
+ return -1;
+ }
+ }
+
+ return fd;
+}
+
+static gint
+rspamd_inet_socket_create(gint type, struct addrinfo *addr, gboolean is_server,
+ gboolean async, GList **list)
+{
+ gint fd = -1, r, on = 1, s_error;
+ struct addrinfo *cur;
+ gpointer ptr;
+ socklen_t optlen;
+
+ cur = addr;
+ while (cur) {
+ /* Create socket */
+ fd = rspamd_socket_create(cur->ai_family, type, cur->ai_protocol, TRUE);
+ if (fd == -1) {
+ goto out;
+ }
+
+ if (is_server) {
+ (void) setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const void *) &on,
+ sizeof(gint));
+#ifdef HAVE_IPV6_V6ONLY
+ if (cur->ai_family == AF_INET6) {
+ setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, (const void *) &on,
+ sizeof(gint));
+ }
+#endif
+ r = bind(fd, cur->ai_addr, cur->ai_addrlen);
+ }
+ else {
+ r = connect(fd, cur->ai_addr, cur->ai_addrlen);
+ }
+
+ if (r == -1) {
+ if (errno != EINPROGRESS) {
+ goto out;
+ }
+ if (!async) {
+ /* Try to poll */
+ if (rspamd_socket_poll(fd, CONNECT_TIMEOUT * 1000,
+ POLLOUT) <= 0) {
+ errno = ETIMEDOUT;
+ goto out;
+ }
+ else {
+ /* Make synced again */
+ if (rspamd_socket_blocking(fd) < 0) {
+ goto out;
+ }
+ }
+ }
+ }
+ else {
+ /* Still need to check SO_ERROR on socket */
+ optlen = sizeof(s_error);
+
+ if (getsockopt(fd, SOL_SOCKET, SO_ERROR, (void *) &s_error, &optlen) != -1) {
+ if (s_error) {
+ errno = s_error;
+ goto out;
+ }
+ }
+ }
+ if (list == NULL) {
+ /* Go out immediately */
+ break;
+ }
+ else if (fd != -1) {
+ ptr = GINT_TO_POINTER(fd);
+ *list = g_list_prepend(*list, ptr);
+ cur = cur->ai_next;
+ continue;
+ }
+ out:
+ if (fd != -1) {
+ close(fd);
+ }
+ fd = -1;
+ cur = cur->ai_next;
+ }
+
+ return (fd);
+}
+
+gint rspamd_socket_tcp(struct addrinfo *addr, gboolean is_server, gboolean async)
+{
+ return rspamd_inet_socket_create(SOCK_STREAM, addr, is_server, async, NULL);
+}
+
+gint rspamd_socket_udp(struct addrinfo *addr, gboolean is_server, gboolean async)
+{
+ return rspamd_inet_socket_create(SOCK_DGRAM, addr, is_server, async, NULL);
+}
+
+gint rspamd_socket_unix(const gchar *path,
+ struct sockaddr_un *addr,
+ gint type,
+ gboolean is_server,
+ gboolean async)
+{
+
+ socklen_t optlen;
+ gint fd = -1, s_error, r, serrno, on = 1;
+ struct stat st;
+
+ if (path == NULL)
+ return -1;
+
+ addr->sun_family = AF_UNIX;
+
+ rspamd_strlcpy(addr->sun_path, path, sizeof(addr->sun_path));
+#ifdef FREEBSD
+ addr->sun_len = SUN_LEN(addr);
+#endif
+
+ if (is_server) {
+ /* Unlink socket if it exists already */
+ if (lstat(addr->sun_path, &st) != -1) {
+ if (S_ISSOCK(st.st_mode)) {
+ if (unlink(addr->sun_path) == -1) {
+ goto out;
+ }
+ }
+ else {
+ goto out;
+ }
+ }
+ }
+ fd = socket(PF_LOCAL, type, 0);
+
+ if (fd == -1) {
+ return -1;
+ }
+
+ if (rspamd_socket_nonblocking(fd) < 0) {
+ goto out;
+ }
+
+ /* Set close on exec */
+ if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) {
+ goto out;
+ }
+ if (is_server) {
+ (void) setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const void *) &on,
+ sizeof(gint));
+ r = bind(fd, (struct sockaddr *) addr, SUN_LEN(addr));
+ }
+ else {
+ r = connect(fd, (struct sockaddr *) addr, SUN_LEN(addr));
+ }
+
+ if (r == -1) {
+ if (errno != EINPROGRESS) {
+ goto out;
+ }
+ if (!async) {
+ /* Try to poll */
+ if (rspamd_socket_poll(fd, CONNECT_TIMEOUT * 1000, POLLOUT) <= 0) {
+ errno = ETIMEDOUT;
+ goto out;
+ }
+ else {
+ /* Make synced again */
+ if (rspamd_socket_blocking(fd) < 0) {
+ goto out;
+ }
+ }
+ }
+ }
+ else {
+ /* Still need to check SO_ERROR on socket */
+ optlen = sizeof(s_error);
+
+ if (getsockopt(fd, SOL_SOCKET, SO_ERROR, (void *) &s_error, &optlen) != -1) {
+ if (s_error) {
+ errno = s_error;
+ goto out;
+ }
+ }
+ }
+
+
+ return (fd);
+
+out:
+ serrno = errno;
+ if (fd != -1) {
+ close(fd);
+ }
+ errno = serrno;
+ return (-1);
+}
+
+static int
+rspamd_prefer_v4_hack(const struct addrinfo *a1, const struct addrinfo *a2)
+{
+ return a1->ai_addr->sa_family - a2->ai_addr->sa_family;
+}
+
+/**
+ * Make a universal socket
+ * @param credits host, ip or path to unix socket
+ * @param port port (used for network sockets)
+ * @param async make this socket async
+ * @param is_server make this socket as server socket
+ * @param try_resolve try name resolution for a socket (BLOCKING)
+ */
+gint rspamd_socket(const gchar *credits, guint16 port,
+ gint type, gboolean async, gboolean is_server, gboolean try_resolve)
+{
+ struct sockaddr_un un;
+ struct stat st;
+ struct addrinfo hints, *res;
+ gint r;
+ gchar portbuf[8];
+
+ if (*credits == '/') {
+ if (is_server) {
+ return rspamd_socket_unix(credits, &un, type, is_server, async);
+ }
+ else {
+ r = stat(credits, &st);
+ if (r == -1) {
+ /* Unix socket doesn't exists it must be created first */
+ errno = ENOENT;
+ return -1;
+ }
+ else {
+ if ((st.st_mode & S_IFSOCK) == 0) {
+ /* Path is not valid socket */
+ errno = EINVAL;
+ return -1;
+ }
+ else {
+ return rspamd_socket_unix(credits,
+ &un,
+ type,
+ is_server,
+ async);
+ }
+ }
+ }
+ }
+ else {
+ /* TCP related part */
+ memset(&hints, 0, sizeof(hints));
+ hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
+ hints.ai_socktype = type; /* Type of the socket */
+ hints.ai_flags = is_server ? AI_PASSIVE : 0;
+ hints.ai_protocol = 0; /* Any protocol */
+ hints.ai_canonname = NULL;
+ hints.ai_addr = NULL;
+ hints.ai_next = NULL;
+
+ if (!try_resolve) {
+ hints.ai_flags |= AI_NUMERICHOST | AI_NUMERICSERV;
+ }
+
+ rspamd_snprintf(portbuf, sizeof(portbuf), "%d", (int) port);
+ if ((r = getaddrinfo(credits, portbuf, &hints, &res)) == 0) {
+ LL_SORT2(res, rspamd_prefer_v4_hack, ai_next);
+ r = rspamd_inet_socket_create(type, res, is_server, async, NULL);
+ freeaddrinfo(res);
+ return r;
+ }
+ else {
+ return -1;
+ }
+ }
+}
+
+gboolean
+rspamd_socketpair(gint pair[2], gint af)
+{
+ gint r = -1, serrno;
+
+#ifdef HAVE_SOCK_SEQPACKET
+ if (af == SOCK_SEQPACKET) {
+ r = socketpair(AF_LOCAL, SOCK_SEQPACKET, 0, pair);
+
+ if (r == -1) {
+ r = socketpair(AF_LOCAL, SOCK_DGRAM, 0, pair);
+ }
+ }
+#endif
+ if (r == -1) {
+ r = socketpair(AF_LOCAL, af, 0, pair);
+ }
+
+ if (r == -1) {
+ return -1;
+ }
+
+ /* Set close on exec */
+ if (fcntl(pair[0], F_SETFD, FD_CLOEXEC) == -1) {
+ goto out;
+ }
+ if (fcntl(pair[1], F_SETFD, FD_CLOEXEC) == -1) {
+ goto out;
+ }
+
+ return TRUE;
+
+out:
+ serrno = errno;
+ close(pair[0]);
+ close(pair[1]);
+ errno = serrno;
+
+ return FALSE;
+}
+
+#ifdef HAVE_SA_SIGINFO
+void rspamd_signals_init(struct sigaction *signals, void (*sig_handler)(gint,
+ siginfo_t *,
+ void *))
+#else
+void rspamd_signals_init(struct sigaction *signals, void (*sig_handler)(gint))
+#endif
+{
+ struct sigaction sigpipe_act;
+ /* Setting up signal handlers */
+ /* SIGUSR1 - reopen config file */
+ /* SIGUSR2 - worker is ready for accept */
+ sigemptyset(&signals->sa_mask);
+ sigaddset(&signals->sa_mask, SIGTERM);
+ sigaddset(&signals->sa_mask, SIGINT);
+ sigaddset(&signals->sa_mask, SIGHUP);
+ sigaddset(&signals->sa_mask, SIGCHLD);
+ sigaddset(&signals->sa_mask, SIGUSR1);
+ sigaddset(&signals->sa_mask, SIGUSR2);
+ sigaddset(&signals->sa_mask, SIGALRM);
+#ifdef SIGPOLL
+ sigaddset(&signals->sa_mask, SIGPOLL);
+#endif
+#ifdef SIGIO
+ sigaddset(&signals->sa_mask, SIGIO);
+#endif
+
+#ifdef HAVE_SA_SIGINFO
+ signals->sa_flags = SA_SIGINFO;
+ signals->sa_handler = NULL;
+ signals->sa_sigaction = sig_handler;
+#else
+ signals->sa_handler = sig_handler;
+ signals->sa_flags = 0;
+#endif
+ sigaction(SIGTERM, signals, NULL);
+ sigaction(SIGINT, signals, NULL);
+ sigaction(SIGHUP, signals, NULL);
+ sigaction(SIGCHLD, signals, NULL);
+ sigaction(SIGUSR1, signals, NULL);
+ sigaction(SIGUSR2, signals, NULL);
+ sigaction(SIGALRM, signals, NULL);
+#ifdef SIGPOLL
+ sigaction(SIGPOLL, signals, NULL);
+#endif
+#ifdef SIGIO
+ sigaction(SIGIO, signals, NULL);
+#endif
+
+ /* Ignore SIGPIPE as we handle write errors manually */
+ sigemptyset(&sigpipe_act.sa_mask);
+ sigaddset(&sigpipe_act.sa_mask, SIGPIPE);
+ sigpipe_act.sa_handler = SIG_IGN;
+ sigpipe_act.sa_flags = 0;
+ sigaction(SIGPIPE, &sigpipe_act, NULL);
+}
+
+#ifndef HAVE_SETPROCTITLE
+
+#ifdef LINUX
+static gchar *title_buffer = NULL;
+static size_t title_buffer_size = 0;
+static gchar *title_progname, *title_progname_full;
+gchar **old_environ = NULL;
+
+static void
+rspamd_title_dtor(gpointer d)
+{
+ /* Restore old environment */
+ if (old_environ != NULL) {
+ environ = old_environ;
+ }
+
+ gchar **env = (gchar **) d;
+ guint i;
+
+ for (i = 0; env[i] != NULL; i++) {
+ g_free(env[i]);
+ }
+
+ g_free(env);
+}
+#endif /* ifdef LINUX */
+
+#endif /* ifndef HAVE_SETPROCTITLE */
+
+gint rspamd_init_title(rspamd_mempool_t *pool,
+ gint argc, gchar *argv[], gchar *envp[])
+{
+#if defined(LINUX) && !defined(HAVE_SETPROCTITLE)
+ gchar *begin_of_buffer = 0, *end_of_buffer = 0;
+ gint i;
+
+ for (i = 0; i < argc; ++i) {
+ if (!begin_of_buffer) {
+ begin_of_buffer = argv[i];
+ }
+ if (!end_of_buffer || end_of_buffer + 1 == argv[i]) {
+ end_of_buffer = argv[i] + strlen(argv[i]);
+ }
+ }
+
+ for (i = 0; envp[i]; ++i) {
+ if (!begin_of_buffer) {
+ begin_of_buffer = envp[i];
+ }
+ if (!end_of_buffer || end_of_buffer + 1 == envp[i]) {
+ end_of_buffer = envp[i] + strlen(envp[i]);
+ }
+ }
+
+ if (!end_of_buffer) {
+ return 0;
+ }
+
+ gchar **new_environ = g_malloc((i + 1) * sizeof(envp[0]));
+
+ for (i = 0; envp[i]; ++i) {
+ new_environ[i] = g_strdup(envp[i]);
+ }
+
+ new_environ[i] = NULL;
+
+ if (program_invocation_name) {
+ title_progname_full = g_strdup(program_invocation_name);
+
+ gchar *p = strrchr(title_progname_full, '/');
+
+ if (p) {
+ title_progname = p + 1;
+ }
+ else {
+ title_progname = title_progname_full;
+ }
+
+ program_invocation_name = title_progname_full;
+ program_invocation_short_name = title_progname;
+ }
+
+ old_environ = environ;
+ environ = new_environ;
+ title_buffer = begin_of_buffer;
+ title_buffer_size = end_of_buffer - begin_of_buffer;
+
+ rspamd_mempool_add_destructor(pool,
+ rspamd_title_dtor,
+ new_environ);
+#endif
+
+ return 0;
+}
+
+gint rspamd_setproctitle(const gchar *fmt, ...)
+{
+#ifdef HAVE_SETPROCTITLE
+ if (fmt) {
+ static char titlebuf[4096];
+ va_list ap;
+
+ va_start(ap, fmt);
+ rspamd_vsnprintf(titlebuf, sizeof(titlebuf), fmt, ap);
+ va_end(ap);
+
+ setproctitle("%s", titlebuf);
+ }
+#else
+#if defined(LINUX)
+ if (!title_buffer || !title_buffer_size) {
+ errno = ENOMEM;
+ return -1;
+ }
+
+ memset(title_buffer, '\0', title_buffer_size);
+
+ ssize_t written;
+
+ if (fmt) {
+ va_list ap;
+
+ written = rspamd_snprintf(title_buffer,
+ title_buffer_size,
+ "%s: ",
+ title_progname);
+ if (written < 0 || (size_t) written >= title_buffer_size)
+ return -1;
+
+ va_start(ap, fmt);
+ rspamd_vsnprintf(title_buffer + written,
+ title_buffer_size - written,
+ fmt,
+ ap);
+ va_end(ap);
+ }
+ else {
+ written = rspamd_snprintf(title_buffer,
+ title_buffer_size,
+ "%s",
+ title_progname);
+ if (written < 0 || (size_t) written >= title_buffer_size)
+ return -1;
+ }
+
+ written = strlen(title_buffer);
+ memset(title_buffer + written, '\0', title_buffer_size - written);
+#elif defined(__APPLE__)
+ /* OSX is broken, ignore this brain damaged system */
+#else
+ /* Last resort (usually broken, but eh...) */
+ GString *dest;
+ va_list ap;
+
+ dest = g_string_new("");
+ va_start(ap, fmt);
+ rspamd_vprintf_gstring(dest, fmt, ap);
+ va_end(ap);
+
+ g_set_prgname(dest->str);
+ g_string_free(dest, TRUE);
+
+#endif /* defined(LINUX) */
+
+#endif /* HAVE_SETPROCTITLE */
+ return 0;
+}
+
+
+#ifndef HAVE_PIDFILE
+static gint _rspamd_pidfile_remove(rspamd_pidfh_t *pfh, gint freeit);
+
+static gint
+rspamd_pidfile_verify(rspamd_pidfh_t *pfh)
+{
+ struct stat sb;
+
+ if (pfh == NULL || pfh->pf_fd == -1)
+ return (-1);
+ /*
+ * Check remembered descriptor.
+ */
+ if (fstat(pfh->pf_fd, &sb) == -1)
+ return (errno);
+ if (sb.st_dev != pfh->pf_dev || sb.st_ino != pfh->pf_ino)
+ return -1;
+ return 0;
+}
+
+static gint
+rspamd_pidfile_read(const gchar *path, pid_t *pidptr)
+{
+ gchar buf[16], *endptr;
+ gint error, fd, i;
+
+ fd = open(path, O_RDONLY);
+ if (fd == -1)
+ return (errno);
+
+ i = read(fd, buf, sizeof(buf) - 1);
+ error = errno; /* Remember errno in case close() wants to change it. */
+ close(fd);
+ if (i == -1)
+ return error;
+ else if (i == 0)
+ return EAGAIN;
+ buf[i] = '\0';
+
+ *pidptr = strtol(buf, &endptr, 10);
+ if (endptr != &buf[i])
+ return EINVAL;
+
+ return 0;
+}
+
+rspamd_pidfh_t *
+rspamd_pidfile_open(const gchar *path, mode_t mode, pid_t *pidptr)
+{
+ rspamd_pidfh_t *pfh;
+ struct stat sb;
+ gint error, fd, len, count;
+ struct timespec rqtp;
+
+ pfh = g_malloc(sizeof(*pfh));
+ if (pfh == NULL)
+ return NULL;
+
+ if (path == NULL)
+ len = snprintf(pfh->pf_path,
+ sizeof(pfh->pf_path),
+ "/var/run/%s.pid",
+ g_get_prgname());
+ else
+ len = snprintf(pfh->pf_path, sizeof(pfh->pf_path), "%s", path);
+ if (len >= (gint) sizeof(pfh->pf_path)) {
+ g_free(pfh);
+ errno = ENAMETOOLONG;
+ return NULL;
+ }
+
+ /*
+ * Open the PID file and obtain exclusive lock.
+ * We truncate PID file here only to remove old PID immediately,
+ * PID file will be truncated again in pidfile_write(), so
+ * pidfile_write() can be called multiple times.
+ */
+ fd = open(pfh->pf_path, O_WRONLY | O_CREAT | O_TRUNC | O_NONBLOCK, mode);
+ rspamd_file_lock(fd, TRUE);
+ if (fd == -1) {
+ count = 0;
+ rqtp.tv_sec = 0;
+ rqtp.tv_nsec = 5000000;
+ if (errno == EWOULDBLOCK && pidptr != NULL) {
+ again:
+ errno = rspamd_pidfile_read(pfh->pf_path, pidptr);
+ if (errno == 0)
+ errno = EEXIST;
+ else if (errno == EAGAIN) {
+ if (++count <= 3) {
+ nanosleep(&rqtp, 0);
+ goto again;
+ }
+ }
+ }
+ g_free(pfh);
+ return NULL;
+ }
+ /*
+ * Remember file information, so in pidfile_write() we are sure we write
+ * to the proper descriptor.
+ */
+ if (fstat(fd, &sb) == -1) {
+ error = errno;
+ unlink(pfh->pf_path);
+ close(fd);
+ g_free(pfh);
+ errno = error;
+ return NULL;
+ }
+
+ pfh->pf_fd = fd;
+ pfh->pf_dev = sb.st_dev;
+ pfh->pf_ino = sb.st_ino;
+
+ return pfh;
+}
+
+gint rspamd_pidfile_write(rspamd_pidfh_t *pfh)
+{
+ gchar pidstr[16];
+ gint error, fd;
+
+ /*
+ * Check remembered descriptor, so we don't overwrite some other
+ * file if pidfile was closed and descriptor reused.
+ */
+ errno = rspamd_pidfile_verify(pfh);
+ if (errno != 0) {
+ /*
+ * Don't close descriptor, because we are not sure if it's ours.
+ */
+ return -1;
+ }
+ fd = pfh->pf_fd;
+
+ /*
+ * Truncate PID file, so multiple calls of pidfile_write() are allowed.
+ */
+ if (ftruncate(fd, 0) == -1) {
+ error = errno;
+ _rspamd_pidfile_remove(pfh, 0);
+ errno = error;
+ return -1;
+ }
+
+ rspamd_snprintf(pidstr, sizeof(pidstr), "%P", getpid());
+ if (pwrite(fd, pidstr, strlen(pidstr), 0) != (ssize_t) strlen(pidstr)) {
+ error = errno;
+ _rspamd_pidfile_remove(pfh, 0);
+ errno = error;
+ return -1;
+ }
+
+ return 0;
+}
+
+gint rspamd_pidfile_close(rspamd_pidfh_t *pfh)
+{
+ gint error;
+
+ error = rspamd_pidfile_verify(pfh);
+ if (error != 0) {
+ errno = error;
+ return -1;
+ }
+
+ if (close(pfh->pf_fd) == -1)
+ error = errno;
+ g_free(pfh);
+ if (error != 0) {
+ errno = error;
+ return -1;
+ }
+ return 0;
+}
+
+static gint
+_rspamd_pidfile_remove(rspamd_pidfh_t *pfh, gint freeit)
+{
+ gint error;
+
+ error = rspamd_pidfile_verify(pfh);
+ if (error != 0) {
+ errno = error;
+ return -1;
+ }
+
+ if (unlink(pfh->pf_path) == -1)
+ error = errno;
+ if (!rspamd_file_unlock(pfh->pf_fd, FALSE)) {
+ if (error == 0)
+ error = errno;
+ }
+ if (close(pfh->pf_fd) == -1) {
+ if (error == 0)
+ error = errno;
+ }
+ if (freeit)
+ g_free(pfh);
+ else
+ pfh->pf_fd = -1;
+ if (error != 0) {
+ errno = error;
+ return -1;
+ }
+ return 0;
+}
+
+gint rspamd_pidfile_remove(rspamd_pidfh_t *pfh)
+{
+
+ return (_rspamd_pidfile_remove(pfh, 1));
+}
+#endif
+
+/* Replace %r with rcpt value and %f with from value, new string is allocated in pool */
+gchar *
+resolve_stat_filename(rspamd_mempool_t *pool,
+ gchar *pattern,
+ gchar *rcpt,
+ gchar *from)
+{
+ gint need_to_format = 0, len = 0;
+ gint rcptlen, fromlen;
+ gchar *c = pattern, *new, *s;
+
+ if (rcpt) {
+ rcptlen = strlen(rcpt);
+ }
+ else {
+ rcptlen = 0;
+ }
+
+ if (from) {
+ fromlen = strlen(from);
+ }
+ else {
+ fromlen = 0;
+ }
+
+ /* Calculate length */
+ while (*c++) {
+ if (*c == '%' && *(c + 1) == 'r') {
+ len += rcptlen;
+ c += 2;
+ need_to_format = 1;
+ continue;
+ }
+ else if (*c == '%' && *(c + 1) == 'f') {
+ len += fromlen;
+ c += 2;
+ need_to_format = 1;
+ continue;
+ }
+ len++;
+ }
+
+ /* Do not allocate extra memory if we do not need to format string */
+ if (!need_to_format) {
+ return pattern;
+ }
+
+ /* Allocate new string */
+ new = rspamd_mempool_alloc(pool, len);
+ c = pattern;
+ s = new;
+
+ /* Format string */
+ while (*c++) {
+ if (*c == '%' && *(c + 1) == 'r') {
+ c += 2;
+ memcpy(s, rcpt, rcptlen);
+ s += rcptlen;
+ continue;
+ }
+ *s++ = *c;
+ }
+
+ *s = '\0';
+
+ return new;
+}
+
+const gchar *
+rspamd_log_check_time(gdouble start, gdouble end, gint resolution)
+{
+ gdouble diff;
+ static gchar res[64];
+ gchar fmt[32];
+
+ diff = (end - start) * 1000.0;
+
+ rspamd_snprintf(fmt, sizeof(fmt), "%%.%dfms", resolution);
+ rspamd_snprintf(res, sizeof(res), fmt, diff);
+
+ return (const gchar *) res;
+}
+
+
+#ifdef HAVE_FLOCK
+/* Flock version */
+gboolean
+rspamd_file_lock(gint fd, gboolean async)
+{
+ gint flags;
+
+ if (async) {
+ flags = LOCK_EX | LOCK_NB;
+ }
+ else {
+ flags = LOCK_EX;
+ }
+
+ if (flock(fd, flags) == -1) {
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_file_unlock(gint fd, gboolean async)
+{
+ gint flags;
+
+ if (async) {
+ flags = LOCK_UN | LOCK_NB;
+ }
+ else {
+ flags = LOCK_UN;
+ }
+
+ if (flock(fd, flags) == -1) {
+ if (async && errno == EAGAIN) {
+ return FALSE;
+ }
+
+ return FALSE;
+ }
+
+ return TRUE;
+}
+#else /* HAVE_FLOCK */
+/* Fctnl version */
+gboolean
+rspamd_file_lock(gint fd, gboolean async)
+{
+ struct flock fl = {
+ .l_type = F_WRLCK,
+ .l_whence = SEEK_SET,
+ .l_start = 0,
+ .l_len = 0};
+
+ if (fcntl(fd, async ? F_SETLK : F_SETLKW, &fl) == -1) {
+ if (async && (errno == EAGAIN || errno == EACCES)) {
+ return FALSE;
+ }
+
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+gboolean
+rspamd_file_unlock(gint fd, gboolean async)
+{
+ struct flock fl = {
+ .l_type = F_UNLCK,
+ .l_whence = SEEK_SET,
+ .l_start = 0,
+ .l_len = 0};
+
+ if (fcntl(fd, async ? F_SETLK : F_SETLKW, &fl) == -1) {
+ if (async && (errno == EAGAIN || errno == EACCES)) {
+ return FALSE;
+ }
+
+ return FALSE;
+ }
+
+ return TRUE;
+}
+#endif /* HAVE_FLOCK */
+
+
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 22))
+void g_ptr_array_unref(GPtrArray *array)
+{
+ g_ptr_array_free(array, TRUE);
+}
+gboolean
+g_int64_equal(gconstpointer v1, gconstpointer v2)
+{
+ return *((const gint64 *) v1) == *((const gint64 *) v2);
+}
+guint g_int64_hash(gconstpointer v)
+{
+ guint64 v64 = *(guint64 *) v;
+
+ return (guint) (v ^ (v >> 32));
+}
+#endif
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 14))
+void g_queue_clear(GQueue *queue)
+{
+ g_return_if_fail(queue != NULL);
+
+ g_list_free(queue->head);
+ queue->head = queue->tail = NULL;
+ queue->length = 0;
+}
+#endif
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 30))
+GPtrArray *
+g_ptr_array_new_full(guint reserved_size,
+ GDestroyNotify element_free_func)
+{
+ GPtrArray *array;
+
+ array = g_ptr_array_sized_new(reserved_size);
+ g_ptr_array_set_free_func(array, element_free_func);
+
+ return array;
+}
+#endif
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 32))
+void g_queue_free_full(GQueue *queue, GDestroyNotify free_func)
+{
+ GList *cur;
+
+ cur = queue->head;
+
+ while (cur) {
+ free_func(cur->data);
+ cur = g_list_next(cur);
+ }
+
+ g_queue_free(queue);
+}
+#endif
+
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 40))
+void g_ptr_array_insert(GPtrArray *array, gint index_, gpointer data)
+{
+ g_return_if_fail(array);
+ g_return_if_fail(index_ >= -1);
+ g_return_if_fail(index_ <= (gint) array->len);
+
+ g_ptr_array_set_size(array, array->len + 1);
+
+ if (index_ < 0) {
+ index_ = array->len;
+ }
+
+ if (index_ < array->len) {
+ memmove(&(array->pdata[index_ + 1]), &(array->pdata[index_]),
+ (array->len - index_) * sizeof(gpointer));
+ }
+
+ array->pdata[index_] = data;
+}
+#endif
+
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 32))
+const gchar *
+g_environ_getenv(gchar **envp, const gchar *variable)
+{
+ gsize len;
+ gint i;
+
+ if (envp == NULL) {
+ return NULL;
+ }
+
+ len = strlen(variable);
+
+ for (i = 0; envp[i]; i++) {
+ if (strncmp(envp[i], variable, len) == 0 && envp[i][len] == '=') {
+ return envp[i] + len + 1;
+ }
+ }
+
+ return NULL;
+}
+#endif
+
+gint rspamd_fallocate(gint fd, off_t offset, off_t len)
+{
+#if defined(HAVE_FALLOCATE)
+ return fallocate(fd, 0, offset, len);
+#elif defined(HAVE_POSIX_FALLOCATE)
+ return posix_fallocate(fd, offset, len);
+#else
+ /* Return 0 as nothing can be done on this system */
+ return 0;
+#endif
+}
+
+
+/**
+ * Create new mutex
+ * @return mutex or NULL
+ */
+inline rspamd_mutex_t *
+rspamd_mutex_new(void)
+{
+ rspamd_mutex_t *new;
+
+ new = g_malloc0(sizeof(rspamd_mutex_t));
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30))
+ g_mutex_init(&new->mtx);
+#else
+ g_static_mutex_init(&new->mtx);
+#endif
+
+ return new;
+}
+
+/**
+ * Lock mutex
+ * @param mtx
+ */
+inline void
+rspamd_mutex_lock(rspamd_mutex_t *mtx)
+{
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30))
+ g_mutex_lock(&mtx->mtx);
+#else
+ g_static_mutex_lock(&mtx->mtx);
+#endif
+}
+
+/**
+ * Unlock mutex
+ * @param mtx
+ */
+inline void
+rspamd_mutex_unlock(rspamd_mutex_t *mtx)
+{
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30))
+ g_mutex_unlock(&mtx->mtx);
+#else
+ g_static_mutex_unlock(&mtx->mtx);
+#endif
+}
+
+void rspamd_mutex_free(rspamd_mutex_t *mtx)
+{
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30))
+ g_mutex_clear(&mtx->mtx);
+#endif
+ g_free(mtx);
+}
+
+struct rspamd_thread_data {
+ gchar *name;
+ gint id;
+ GThreadFunc func;
+ gpointer data;
+};
+
+static gpointer
+rspamd_thread_func(gpointer ud)
+{
+ struct rspamd_thread_data *td = ud;
+ sigset_t s_mask;
+
+ /* Ignore signals in thread */
+ sigemptyset(&s_mask);
+ sigaddset(&s_mask, SIGINT);
+ sigaddset(&s_mask, SIGHUP);
+ sigaddset(&s_mask, SIGCHLD);
+ sigaddset(&s_mask, SIGUSR1);
+ sigaddset(&s_mask, SIGUSR2);
+ sigaddset(&s_mask, SIGALRM);
+ sigaddset(&s_mask, SIGPIPE);
+
+ pthread_sigmask(SIG_BLOCK, &s_mask, NULL);
+
+ ud = td->func(td->data);
+ g_free(td->name);
+ g_free(td);
+
+ return ud;
+}
+
+struct hash_copy_callback_data {
+ gpointer (*key_copy_func)(gconstpointer data, gpointer ud);
+ gpointer (*value_copy_func)(gconstpointer data, gpointer ud);
+ gpointer ud;
+ GHashTable *dst;
+};
+
+static void
+copy_foreach_callback(gpointer key, gpointer value, gpointer ud)
+{
+ struct hash_copy_callback_data *cb = ud;
+ gpointer nkey, nvalue;
+
+ nkey = cb->key_copy_func ? cb->key_copy_func(key, cb->ud) : (gpointer) key;
+ nvalue =
+ cb->value_copy_func ? cb->value_copy_func(value,
+ cb->ud)
+ : (gpointer) value;
+ g_hash_table_insert(cb->dst, nkey, nvalue);
+}
+/**
+ * Deep copy of one hash table to another
+ * @param src source hash
+ * @param dst destination hash
+ * @param key_copy_func function called to copy or modify keys (or NULL)
+ * @param value_copy_func function called to copy or modify values (or NULL)
+ * @param ud user data for copy functions
+ */
+void rspamd_hash_table_copy(GHashTable *src, GHashTable *dst,
+ gpointer (*key_copy_func)(gconstpointer data, gpointer ud),
+ gpointer (*value_copy_func)(gconstpointer data, gpointer ud),
+ gpointer ud)
+{
+ struct hash_copy_callback_data cb;
+ if (src != NULL && dst != NULL) {
+ cb.key_copy_func = key_copy_func;
+ cb.value_copy_func = value_copy_func;
+ cb.ud = ud;
+ cb.dst = dst;
+ g_hash_table_foreach(src, copy_foreach_callback, &cb);
+ }
+}
+
+static volatile sig_atomic_t saved_signo[NSIG];
+
+static void
+read_pass_tmp_sig_handler(int s)
+{
+
+ saved_signo[s] = 1;
+}
+
+#ifndef _PATH_TTY
+#define _PATH_TTY "/dev/tty"
+#endif
+
+gint rspamd_read_passphrase_with_prompt(const gchar *prompt, gchar *buf, gint size, bool echo, gpointer key)
+{
+#ifdef HAVE_READPASSPHRASE_H
+ int flags = echo ? RPP_ECHO_ON : RPP_ECHO_OFF;
+ if (readpassphrase(prompt, buf, size, flags | RPP_REQUIRE_TTY) == NULL) {
+ return 0;
+ }
+
+ return strlen(buf);
+#else
+ struct sigaction sa, savealrm, saveint, savehup, savequit, saveterm;
+ struct sigaction savetstp, savettin, savettou, savepipe;
+ struct termios term, oterm;
+ gint input, output, i;
+ gchar *end, *p, ch;
+
+restart:
+ if ((input = output = open(_PATH_TTY, O_RDWR)) == -1) {
+ errno = ENOTTY;
+ return 0;
+ }
+
+ (void) fcntl(input, F_SETFD, FD_CLOEXEC);
+
+ /* Turn echo off */
+ if (tcgetattr(input, &oterm) != 0) {
+ close(input);
+ errno = ENOTTY;
+ return 0;
+ }
+
+ memcpy(&term, &oterm, sizeof(term));
+
+ if (!echo) {
+ term.c_lflag &= ~(ECHO | ECHONL);
+ }
+
+ if (tcsetattr(input, TCSAFLUSH, &term) == -1) {
+ errno = ENOTTY;
+ close(input);
+ return 0;
+ }
+
+ g_assert(write(output, prompt, sizeof("Enter passphrase: ") - 1) != -1);
+
+ /* Save the current sighandler */
+ for (i = 0; i < NSIG; i++) {
+ saved_signo[i] = 0;
+ }
+ sigemptyset(&sa.sa_mask);
+ sa.sa_flags = 0;
+ sa.sa_handler = read_pass_tmp_sig_handler;
+ (void) sigaction(SIGALRM, &sa, &savealrm);
+ (void) sigaction(SIGHUP, &sa, &savehup);
+ (void) sigaction(SIGINT, &sa, &saveint);
+ (void) sigaction(SIGPIPE, &sa, &savepipe);
+ (void) sigaction(SIGQUIT, &sa, &savequit);
+ (void) sigaction(SIGTERM, &sa, &saveterm);
+ (void) sigaction(SIGTSTP, &sa, &savetstp);
+ (void) sigaction(SIGTTIN, &sa, &savettin);
+ (void) sigaction(SIGTTOU, &sa, &savettou);
+
+ /* Now read a passphrase */
+ p = buf;
+ end = p + size - 1;
+ while (read(input, &ch, 1) == 1 && ch != '\n' && ch != '\r') {
+ if (p < end) {
+ *p++ = ch;
+ }
+ }
+ *p = '\0';
+ g_assert(write(output, "\n", 1) == 1);
+
+ /* Restore terminal state */
+ if (memcmp(&term, &oterm, sizeof(term)) != 0) {
+ while (tcsetattr(input, TCSAFLUSH, &oterm) == -1 &&
+ errno == EINTR && !saved_signo[SIGTTOU])
+ ;
+ }
+
+ /* Restore signal handlers */
+ (void) sigaction(SIGALRM, &savealrm, NULL);
+ (void) sigaction(SIGHUP, &savehup, NULL);
+ (void) sigaction(SIGINT, &saveint, NULL);
+ (void) sigaction(SIGQUIT, &savequit, NULL);
+ (void) sigaction(SIGPIPE, &savepipe, NULL);
+ (void) sigaction(SIGTERM, &saveterm, NULL);
+ (void) sigaction(SIGTSTP, &savetstp, NULL);
+ (void) sigaction(SIGTTIN, &savettin, NULL);
+ (void) sigaction(SIGTTOU, &savettou, NULL);
+
+ close(input);
+
+ /* Send signals pending */
+ for (i = 0; i < NSIG; i++) {
+ if (saved_signo[i]) {
+ kill(getpid(), i);
+ switch (i) {
+ case SIGTSTP:
+ case SIGTTIN:
+ case SIGTTOU:
+ goto restart;
+ }
+ }
+ }
+
+ return (p - buf);
+#endif
+}
+
+#ifdef HAVE_CLOCK_GETTIME
+#ifdef CLOCK_MONOTONIC_COARSE
+#define RSPAMD_FAST_MONOTONIC_CLOCK CLOCK_MONOTONIC_COARSE
+#elif defined(CLOCK_MONOTONIC_FAST)
+#define RSPAMD_FAST_MONOTONIC_CLOCK CLOCK_MONOTONIC_FAST
+#else
+#define RSPAMD_FAST_MONOTONIC_CLOCK CLOCK_MONOTONIC
+#endif
+#endif
+
+gdouble
+rspamd_get_ticks(gboolean rdtsc_ok)
+{
+ gdouble res;
+
+#ifdef HAVE_RDTSC
+#ifdef __x86_64__
+ guint64 r64;
+
+ if (rdtsc_ok) {
+ __builtin_ia32_lfence();
+ r64 = __rdtsc();
+ /* Preserve lower 52 bits */
+ res = r64 & ((1ULL << 53) - 1);
+ return res;
+ }
+#endif
+#endif
+#ifdef HAVE_CLOCK_GETTIME
+ struct timespec ts;
+ gint clk_id = RSPAMD_FAST_MONOTONIC_CLOCK;
+
+ clock_gettime(clk_id, &ts);
+
+ if (rdtsc_ok) {
+ res = (double) ts.tv_sec * 1e9 + ts.tv_nsec;
+ }
+ else {
+ res = (double) ts.tv_sec + ts.tv_nsec / 1000000000.;
+ }
+#elif defined(__APPLE__)
+ if (rdtsc_ok) {
+ res = mach_absolute_time();
+ }
+ else {
+ res = mach_absolute_time() / 1000000000.;
+ }
+#else
+ struct timeval tv;
+
+ (void) gettimeofday(&tv, NULL);
+ if (rdtsc_ok) {
+ res = (double) ts.tv_sec * 1e9 + tv.tv_usec * 1e3;
+ }
+ else {
+ res = (double) tv.tv_sec + tv.tv_usec / 1000000.;
+ }
+#endif
+
+ return res;
+}
+
+gdouble
+rspamd_get_virtual_ticks(void)
+{
+ gdouble res;
+
+#ifdef HAVE_CLOCK_GETTIME
+ struct timespec ts;
+ static clockid_t cid = (clockid_t) -1;
+ if (cid == (clockid_t) -1) {
+#ifdef HAVE_CLOCK_GETCPUCLOCKID
+ if (clock_getcpuclockid(0, &cid) == -1) {
+#endif
+#ifdef CLOCK_PROCESS_CPUTIME_ID
+ cid = CLOCK_PROCESS_CPUTIME_ID;
+#elif defined(CLOCK_PROF)
+ cid = CLOCK_PROF;
+#else
+ cid = CLOCK_REALTIME;
+#endif
+#ifdef HAVE_CLOCK_GETCPUCLOCKID
+ }
+#endif
+ }
+
+ clock_gettime(cid, &ts);
+ res = (double) ts.tv_sec + ts.tv_nsec / 1000000000.;
+#elif defined(__APPLE__)
+ thread_port_t thread = mach_thread_self();
+
+ mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT;
+ thread_basic_info_data_t info;
+ if (thread_info(thread, THREAD_BASIC_INFO, (thread_info_t) &info, &count) != KERN_SUCCESS) {
+ return -1;
+ }
+
+ res = info.user_time.seconds + info.system_time.seconds;
+ res += ((gdouble) (info.user_time.microseconds + info.system_time.microseconds)) / 1e6;
+ mach_port_deallocate(mach_task_self(), thread);
+#elif defined(HAVE_RUSAGE_SELF)
+ struct rusage rusage;
+ if (getrusage(RUSAGE_SELF, &rusage) != -1) {
+ res = (double) rusage.ru_utime.tv_sec +
+ (double) rusage.ru_utime.tv_usec / 1000000.0;
+ }
+#else
+ res = clock() / (double) CLOCKS_PER_SEC;
+#endif
+
+ return res;
+}
+
+gdouble
+rspamd_get_calendar_ticks(void)
+{
+ gdouble res;
+#ifdef HAVE_CLOCK_GETTIME
+ struct timespec ts;
+
+ clock_gettime(CLOCK_REALTIME, &ts);
+ res = ts_to_double(&ts);
+#else
+ struct timeval tv;
+
+ if (gettimeofday(&tv, NULL) == 0) {
+ res = tv_to_double(&tv);
+ }
+ else {
+ res = time(NULL);
+ }
+#endif
+
+ return res;
+}
+
+void rspamd_random_hex(gchar *buf, guint64 len)
+{
+ static const gchar hexdigests[16] = "0123456789abcdef";
+ gint64 i;
+
+ g_assert(len > 0);
+
+ ottery_rand_bytes((void *) buf, ceil(len / 2.0));
+
+ for (i = (gint64) len - 1; i >= 0; i -= 2) {
+ buf[i] = hexdigests[buf[i / 2] & 0xf];
+
+ if (i > 0) {
+ buf[i - 1] = hexdigests[(buf[i / 2] >> 4) & 0xf];
+ }
+ }
+}
+
+gint rspamd_shmem_mkstemp(gchar *pattern)
+{
+ gint fd = -1;
+ gchar *nbuf, *xpos;
+ gsize blen;
+
+ xpos = strchr(pattern, 'X');
+
+ if (xpos == NULL) {
+ errno = EINVAL;
+ return -1;
+ }
+
+ blen = strlen(pattern);
+ nbuf = g_malloc(blen + 1);
+ rspamd_strlcpy(nbuf, pattern, blen + 1);
+ xpos = nbuf + (xpos - pattern);
+
+ for (;;) {
+ rspamd_random_hex(xpos, blen - (xpos - nbuf));
+
+ fd = shm_open(nbuf, O_RDWR | O_EXCL | O_CREAT, 0600);
+
+ if (fd != -1) {
+ rspamd_strlcpy(pattern, nbuf, blen + 1);
+ break;
+ }
+ else if (errno != EEXIST) {
+ g_free(nbuf);
+
+ return -1;
+ }
+ }
+
+ g_free(nbuf);
+
+ return fd;
+}
+
+void rspamd_ptr_array_free_hard(gpointer p)
+{
+ GPtrArray *ar = (GPtrArray *) p;
+
+ g_ptr_array_free(ar, TRUE);
+}
+
+void rspamd_array_free_hard(gpointer p)
+{
+ GArray *ar = (GArray *) p;
+
+ g_array_free(ar, TRUE);
+}
+
+void rspamd_gstring_free_hard(gpointer p)
+{
+ GString *ar = (GString *) p;
+
+ g_string_free(ar, TRUE);
+}
+
+void rspamd_gerror_free_maybe(gpointer p)
+{
+ GError **err;
+
+ if (p) {
+ err = (GError **) p;
+
+ if (*err) {
+ g_error_free(*err);
+ }
+ }
+}
+
+/*
+ * Openblas creates threads that are not supported by
+ * jemalloc allocator (aside of being bloody stupid). So this hack
+ * is intended to set number of threads to one by default.
+ * FIXME: is it legit to do so in ctor?
+ */
+#ifdef HAVE_OPENBLAS_SET_NUM_THREADS
+extern void openblas_set_num_threads(int num_threads);
+RSPAMD_CONSTRUCTOR(openblas_thread_fix_ctor)
+{
+ openblas_set_num_threads(1);
+}
+#endif
+#ifdef HAVE_BLI_THREAD_SET_NUM_THREADS
+extern void bli_thread_set_num_threads(int num_threads);
+RSPAMD_CONSTRUCTOR(blis_thread_fix_ctor)
+{
+ bli_thread_set_num_threads(1);
+}
+#endif
+
+guint64
+rspamd_hash_seed(void)
+{
+#if 0
+ static guint64 seed;
+
+ if (seed == 0) {
+ seed = ottery_rand_uint64 ();
+ }
+#endif
+
+ /* Proved to be random, I promise! */
+ /*
+ * TODO: discover if it worth to use random seed on run
+ * with ordinary hash function or we need to switch to
+ * siphash1-3 or other slow cooker function...
+ */
+ return 0xabf9727ba290690bULL;
+}
+
+static inline gdouble
+rspamd_double_from_int64(guint64 x)
+{
+ const union {
+ guint64 i;
+ double d;
+ } u = {
+ .i = G_GUINT64_CONSTANT(0x3FF) << 52 | x >> 12};
+
+ return u.d - 1.0;
+}
+
+gdouble
+rspamd_random_double(void)
+{
+ guint64 rnd_int;
+
+ rnd_int = ottery_rand_uint64();
+
+ return rspamd_double_from_int64(rnd_int);
+}
+
+
+static guint64 *
+rspamd_fast_random_seed(void)
+{
+ static guint64 seed;
+
+ if (G_UNLIKELY(seed == 0)) {
+ ottery_rand_bytes((void *) &seed, sizeof(seed));
+ }
+
+ return &seed;
+}
+
+/* wyrand */
+inline uint64_t
+rspamd_random_uint64_fast_seed(uint64_t *seed)
+{
+ *seed += UINT64_C(0xa0761d6478bd642f);
+#ifdef __SIZEOF_INT128__
+#if defined(__aarch64__)
+ uint64_t lo, hi, p = *seed ^ UINT64_C(0xe7037ed1a0b428db), v = *seed;
+ lo = v * p;
+ __asm__("umulh %0, %1, %2"
+ : "=r"(hi)
+ : "r"(v), "r"(p));
+ return lo ^ hi;
+#else
+ __uint128_t t = (__uint128_t) *seed * (*seed ^ UINT64_C(0xe7037ed1a0b428db));
+ return (t >> 64) ^ t;
+#endif
+#else
+ /* Implementation of 64x64->128-bit multiplication by four 32x32->64
+ * bit multiplication. */
+ uint64_t lo, hi, p = *seed ^ UINT64_C(0xe7037ed1a0b428db), v = *seed;
+ uint64_t hv = v >> 32, hp = p >> 32;
+ uint64_t lv = (uint32_t) v, lp = (uint32_t) p;
+ uint64_t rh = hv * hp;
+ uint64_t rm_0 = hv * lp;
+ uint64_t rm_1 = hp * lv;
+ uint64_t rl = lv * lp;
+ uint64_t t;
+
+ /* We could ignore a carry bit here if we did not care about the
+ same hash for 32-bit and 64-bit targets. */
+ t = rl + (rm_0 << 32);
+ lo = t + (rm_1 << 32);
+ hi = rh + (rm_0 >> 32) + (rm_1 >> 32);
+ return lo ^ hi;
+#endif
+}
+
+gdouble
+rspamd_random_double_fast(void)
+{
+ return rspamd_random_double_fast_seed(rspamd_fast_random_seed());
+}
+
+/* xoshiro256+ */
+inline gdouble
+rspamd_random_double_fast_seed(guint64 *seed)
+{
+ return rspamd_double_from_int64(rspamd_random_uint64_fast_seed(seed));
+}
+
+guint64
+rspamd_random_uint64_fast(void)
+{
+ return rspamd_random_uint64_fast_seed(rspamd_fast_random_seed());
+}
+
+void rspamd_random_seed_fast(void)
+{
+ (void) rspamd_fast_random_seed();
+}
+
+gdouble
+rspamd_time_jitter(gdouble in, gdouble jitter)
+{
+ if (jitter == 0) {
+ jitter = in;
+ }
+
+ return in + jitter * rspamd_random_double();
+}
+
+gboolean
+rspamd_constant_memcmp(const void *a, const void *b, gsize len)
+{
+ gsize lena, lenb, i;
+ guint16 d, r = 0, m;
+ guint16 v;
+ const guint8 *aa = (const guint8 *) a,
+ *bb = (const guint8 *) b;
+
+ if (len == 0) {
+ lena = strlen((const char *) a);
+ lenb = strlen((const char *) b);
+
+ if (lena != lenb) {
+ return FALSE;
+ }
+
+ len = lena;
+ }
+
+ for (i = 0; i < len; i++) {
+ v = ((guint16) (guint8) r) + 255;
+ m = v / 256 - 1;
+ d = (guint16) ((int) aa[i] - (int) bb[i]);
+ r |= (d & m);
+ }
+
+ return (((gint32) (guint16) ((guint32) r + 0x8000) - 0x8000) == 0);
+}
+
+int rspamd_file_xopen(const char *fname, int oflags, guint mode,
+ gboolean allow_symlink)
+{
+ struct stat sb;
+ int fd, flags = oflags;
+
+ if (!(oflags & O_CREAT)) {
+ if (lstat(fname, &sb) == -1) {
+
+ if (errno != ENOENT) {
+ return (-1);
+ }
+ }
+ else if (!S_ISREG(sb.st_mode)) {
+ if (S_ISLNK(sb.st_mode)) {
+ if (!allow_symlink) {
+ return -1;
+ }
+ }
+ else {
+ return -1;
+ }
+ }
+ }
+
+#ifdef HAVE_OCLOEXEC
+ flags |= O_CLOEXEC;
+#endif
+
+#ifdef HAVE_ONOFOLLOW
+ if (!allow_symlink) {
+ flags |= O_NOFOLLOW;
+ fd = open(fname, flags, mode);
+ }
+ else {
+ fd = open(fname, flags, mode);
+ }
+#else
+ fd = open(fname, flags, mode);
+#endif
+
+#ifndef HAVE_OCLOEXEC
+ int serrno;
+ if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) {
+ serrno = errno;
+ close(fd);
+ errno = serrno;
+
+ return -1;
+ }
+#endif
+
+ return (fd);
+}
+
+gpointer
+rspamd_file_xmap(const char *fname, guint mode, gsize *size,
+ gboolean allow_symlink)
+{
+ gint fd;
+ struct stat sb;
+ gpointer map;
+
+ g_assert(fname != NULL);
+ g_assert(size != NULL);
+
+ if (mode & PROT_WRITE) {
+ fd = rspamd_file_xopen(fname, O_RDWR, 0, allow_symlink);
+ }
+ else {
+ fd = rspamd_file_xopen(fname, O_RDONLY, 0, allow_symlink);
+ }
+
+ if (fd == -1) {
+ return NULL;
+ }
+
+ if (fstat(fd, &sb) == -1 || !S_ISREG(sb.st_mode)) {
+ close(fd);
+ *size = (gsize) -1;
+
+ return NULL;
+ }
+
+ if (sb.st_size == 0) {
+ close(fd);
+ *size = (gsize) 0;
+
+ return NULL;
+ }
+
+ map = mmap(NULL, sb.st_size, mode, MAP_SHARED, fd, 0);
+ close(fd);
+
+ if (map == MAP_FAILED) {
+ return NULL;
+ }
+
+ *size = sb.st_size;
+
+ return map;
+}
+
+
+gpointer
+rspamd_shmem_xmap(const char *fname, guint mode,
+ gsize *size)
+{
+ gint fd;
+ struct stat sb;
+ gpointer map;
+
+ g_assert(fname != NULL);
+ g_assert(size != NULL);
+
+#ifdef HAVE_SANE_SHMEM
+ if (mode & PROT_WRITE) {
+ fd = shm_open(fname, O_RDWR, 0);
+ }
+ else {
+ fd = shm_open(fname, O_RDONLY, 0);
+ }
+#else
+ if (mode & PROT_WRITE) {
+ fd = open(fname, O_RDWR, 0);
+ }
+ else {
+ fd = open(fname, O_RDONLY, 0);
+ }
+#endif
+
+ if (fd == -1) {
+ return NULL;
+ }
+
+ if (fstat(fd, &sb) == -1) {
+ close(fd);
+
+ return NULL;
+ }
+
+ map = mmap(NULL, sb.st_size, mode, MAP_SHARED, fd, 0);
+ close(fd);
+
+ if (map == MAP_FAILED) {
+ return NULL;
+ }
+
+ *size = sb.st_size;
+
+ return map;
+}
+
+/*
+ * A(x - 0.5)^4 + B(x - 0.5)^3 + C(x - 0.5)^2 + D(x - 0.5)
+ * A = 32,
+ * B = -6
+ * C = -7
+ * D = 3
+ * y = 32(x - 0.5)^4 - 6(x - 0.5)^3 - 7(x - 0.5)^2 + 3(x - 0.5)
+ *
+ * New approach:
+ * y = ((x - bias)*2)^8
+ */
+gdouble
+rspamd_normalize_probability(gdouble x, gdouble bias)
+{
+ gdouble xx;
+
+ xx = (x - bias) * 2.0;
+
+ return pow(xx, 8);
+}
+
+/*
+ * Calculations from musl libc
+ */
+guint64
+rspamd_tm_to_time(const struct tm *tm, glong tz)
+{
+ guint64 result;
+ gboolean is_leap = FALSE;
+ gint leaps, y = tm->tm_year, cycles, rem, centuries;
+ glong offset = (tz / 100) * 3600 + (tz % 100) * 60;
+
+ /* How many seconds in each month from the beginning of the year */
+ static const gint secs_through_month[] = {
+ 0, 31 * 86400, 59 * 86400, 90 * 86400,
+ 120 * 86400, 151 * 86400, 181 * 86400, 212 * 86400,
+ 243 * 86400, 273 * 86400, 304 * 86400, 334 * 86400};
+
+ /* Convert year */
+ if (tm->tm_year - 2ULL <= 136) {
+ leaps = (y - 68) / 4;
+
+ if (!((y - 68) & 3)) {
+ leaps--;
+ is_leap = 1;
+ }
+
+ result = 31536000 * (y - 70) + 86400 * leaps;
+ }
+ else {
+ cycles = (y - 100) / 400;
+ rem = (y - 100) % 400;
+ if (rem < 0) {
+ cycles--;
+ rem += 400;
+ }
+
+ if (!rem) {
+ is_leap = 1;
+ centuries = 0;
+ leaps = 0;
+ }
+ else {
+ if (rem >= 200) {
+ if (rem >= 300) {
+ centuries = 3;
+ rem -= 300;
+ }
+ else {
+ centuries = 2;
+ rem -= 200;
+ }
+ }
+ else {
+ if (rem >= 100) {
+ centuries = 1;
+ rem -= 100;
+ }
+ else {
+ centuries = 0;
+ }
+ }
+
+ if (!rem) {
+ is_leap = 1;
+ leaps = 0;
+ }
+ else {
+ leaps = rem / 4U;
+ rem %= 4U;
+ is_leap = !rem;
+ }
+ }
+
+ leaps += 97 * cycles + 24 * centuries - (gint) is_leap;
+ result = (y - 100) * 31536000LL + leaps * 86400LL + 946684800 + 86400;
+ }
+
+ /* Now convert months to seconds */
+ result += secs_through_month[tm->tm_mon];
+ /* One more day */
+ if (is_leap && tm->tm_mon >= 2) {
+ result += 86400;
+ }
+
+ result += 86400LL * (tm->tm_mday - 1);
+ result += 3600LL * tm->tm_hour;
+ result += 60LL * tm->tm_min;
+ result += tm->tm_sec;
+
+ /* Now apply tz offset */
+ result -= offset;
+
+ return result;
+}
+
+
+void rspamd_gmtime(gint64 ts, struct tm *dest)
+{
+ guint64 days, secs, years;
+ int remdays, remsecs, remyears;
+ int leap_400_cycles, leap_100_cycles, leap_4_cycles;
+ int months;
+ int wday, yday, leap;
+ /* From March */
+ static const uint8_t days_in_month[] = {31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 31, 29};
+ static const guint64 leap_epoch = 946684800ULL + 86400 * (31 + 29);
+ static const guint64 days_per_400y = 365 * 400 + 97;
+ static const guint64 days_per_100y = 365 * 100 + 24;
+ static const guint64 days_per_4y = 365 * 4 + 1;
+
+ secs = ts - leap_epoch;
+ days = secs / 86400;
+ remsecs = secs % 86400;
+
+ if (remsecs < 0) {
+ remsecs += 86400;
+ days--;
+ }
+
+ wday = (3 + days) % 7;
+ if (wday < 0) {
+ wday += 7;
+ }
+
+ /* Deal with gregorian adjustments */
+ leap_400_cycles = days / days_per_400y;
+ remdays = days % days_per_400y;
+
+ if (remdays < 0) {
+ remdays += days_per_400y;
+ leap_400_cycles--;
+ }
+
+ leap_100_cycles = remdays / days_per_100y;
+ if (leap_100_cycles == 4) {
+ /* 400 years */
+ leap_100_cycles--;
+ }
+
+ remdays -= leap_100_cycles * days_per_100y;
+
+ leap_4_cycles = remdays / days_per_4y;
+ if (leap_4_cycles == 25) {
+ /* 100 years */
+ leap_4_cycles--;
+ }
+ remdays -= leap_4_cycles * days_per_4y;
+
+ remyears = remdays / 365;
+ if (remyears == 4) {
+ /* Ordinary leap year */
+ remyears--;
+ }
+ remdays -= remyears * 365;
+
+ leap = !remyears && (leap_4_cycles || !leap_100_cycles);
+ yday = remdays + 31 + 28 + leap;
+
+ if (yday >= 365 + leap) {
+ yday -= 365 + leap;
+ }
+
+ years = remyears + 4 * leap_4_cycles + 100 * leap_100_cycles +
+ 400ULL * leap_400_cycles;
+
+ for (months = 0; days_in_month[months] <= remdays; months++) {
+ remdays -= days_in_month[months];
+ }
+
+ if (months >= 10) {
+ months -= 12;
+ years++;
+ }
+
+ dest->tm_year = years + 100;
+ dest->tm_mon = months + 2;
+ dest->tm_mday = remdays + 1;
+ dest->tm_wday = wday;
+ dest->tm_yday = yday;
+
+ dest->tm_hour = remsecs / 3600;
+ dest->tm_min = remsecs / 60 % 60;
+ dest->tm_sec = remsecs % 60;
+#if !defined(__sun)
+ dest->tm_gmtoff = 0;
+ dest->tm_zone = "GMT";
+#endif
+}
+
+void rspamd_localtime(gint64 ts, struct tm *dest)
+{
+ time_t t = ts;
+ localtime_r(&t, dest);
+}
+
+gboolean
+rspamd_fstring_gzip(rspamd_fstring_t **in)
+{
+ z_stream strm;
+ rspamd_fstring_t *buf = *in;
+ int ret;
+ unsigned tmp_remain;
+ unsigned char temp[BUFSIZ];
+
+ memset(&strm, 0, sizeof(strm));
+ ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
+ MAX_WBITS + 16, MAX_MEM_LEVEL - 1, Z_DEFAULT_STRATEGY);
+
+ if (ret != Z_OK) {
+ return FALSE;
+ }
+
+ if (buf->allocated < deflateBound(&strm, buf->len)) {
+ buf = rspamd_fstring_grow(buf, deflateBound(&strm, buf->len));
+ *in = buf;
+ }
+
+ strm.next_in = buf->str;
+ strm.avail_in = buf->len;
+
+ strm.next_out = temp;
+ strm.avail_out = sizeof(temp) > buf->allocated ? buf->allocated : sizeof(temp);
+ ret = deflate(&strm, Z_FINISH);
+ if (ret == Z_STREAM_ERROR) {
+ deflateEnd(&strm);
+ return FALSE;
+ }
+
+ /* Try to compress in-place */
+ tmp_remain = strm.next_out - temp;
+ if (tmp_remain <= (strm.avail_in ? buf->len - strm.avail_in : buf->allocated)) {
+ memcpy(buf->str, temp, tmp_remain);
+ strm.next_out = (unsigned char *) buf->str + tmp_remain;
+ tmp_remain = 0;
+ while (ret == Z_OK) {
+ strm.avail_out = strm.avail_in ? strm.next_in - strm.next_out : ((unsigned char *) buf->str + buf->allocated) - strm.next_out;
+ ret = deflate(&strm, Z_FINISH);
+ }
+ if (ret != Z_BUF_ERROR || strm.avail_in == 0) {
+ buf->len = strm.next_out - (unsigned char *) buf->str;
+ *in = buf;
+ deflateEnd(&strm);
+
+ return ret == Z_STREAM_END;
+ }
+ }
+
+ /*
+ * The case when input and output has caught each other, hold the remaining
+ * in a temporary buffer and compress it separately
+ */
+ unsigned char *hold = g_malloc(strm.avail_in);
+ memcpy(hold, strm.next_in, strm.avail_in);
+ strm.next_in = hold;
+ if (tmp_remain) {
+ memcpy(buf->str, temp, tmp_remain);
+ strm.next_out = (unsigned char *) buf->str + tmp_remain;
+ }
+ strm.avail_out = ((unsigned char *) buf->str + buf->allocated) - strm.next_out;
+ ret = deflate(&strm, Z_FINISH);
+ g_free(hold);
+ buf->len = strm.next_out - (unsigned char *) buf->str;
+ *in = buf;
+ deflateEnd(&strm);
+
+ return ret == Z_STREAM_END;
+}
+
+gboolean
+rspamd_fstring_gunzip(rspamd_fstring_t **in)
+{
+ z_stream strm;
+ rspamd_fstring_t *buf = *in, *out = rspamd_fstring_sized_new((*in)->len);
+ int ret;
+
+ memset(&strm, 0, sizeof(strm));
+ ret = inflateInit2(&strm, MAX_WBITS + 16);
+
+ if (ret != Z_OK) {
+ return FALSE;
+ }
+
+ strm.next_in = buf->str;
+ strm.avail_in = buf->len;
+
+ gsize total_out = 0;
+
+ do {
+ strm.next_out = out->str + total_out;
+ strm.avail_out = out->allocated - total_out;
+
+ ret = inflate(&strm, Z_NO_FLUSH);
+ if (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) {
+ break;
+ }
+
+ gsize out_remain = strm.avail_out;
+ total_out = out->allocated - out_remain;
+ if (out_remain == 0 && ret != Z_STREAM_END) {
+ out = rspamd_fstring_grow(out, out->allocated * 2);
+ }
+
+ } while (ret != Z_STREAM_END);
+
+ if (ret == Z_STREAM_END) {
+ *in = out;
+ out->len = total_out;
+ rspamd_fstring_free(buf);
+ }
+ else {
+ /* Revert */
+ *in = buf;
+ rspamd_fstring_free(out);
+ }
+
+ inflateEnd(&strm);
+
+ return ret == Z_STREAM_END;
+}
+
+static gboolean
+rspamd_glob_dir(const gchar *full_path, const gchar *pattern,
+ gboolean recursive, guint rec_len,
+ GPtrArray *res, GError **err)
+{
+ glob_t globbuf;
+ const gchar *path;
+ static gchar pathbuf[PATH_MAX]; /* Static to help recursion */
+ guint i;
+ gint rc;
+ static const guint rec_lim = 16;
+ struct stat st;
+
+ if (rec_len > rec_lim) {
+ g_set_error(err, g_quark_from_static_string("glob"), EOVERFLOW,
+ "maximum nesting is reached: %d", rec_lim);
+
+ return FALSE;
+ }
+
+ memset(&globbuf, 0, sizeof(globbuf));
+
+ if ((rc = glob(full_path, 0, NULL, &globbuf)) != 0) {
+
+ if (rc != GLOB_NOMATCH) {
+ g_set_error(err, g_quark_from_static_string("glob"), errno,
+ "glob %s failed: %s", full_path, strerror(errno));
+ globfree(&globbuf);
+
+ return FALSE;
+ }
+ else {
+ globfree(&globbuf);
+
+ return TRUE;
+ }
+ }
+
+ for (i = 0; i < globbuf.gl_pathc; i++) {
+ path = globbuf.gl_pathv[i];
+
+ if (stat(path, &st) == -1) {
+ if (errno == EPERM || errno == EACCES || errno == ELOOP) {
+ /* Silently ignore */
+ continue;
+ }
+
+ g_set_error(err, g_quark_from_static_string("glob"), errno,
+ "stat %s failed: %s", path, strerror(errno));
+ globfree(&globbuf);
+
+ return FALSE;
+ }
+
+ if (S_ISREG(st.st_mode)) {
+ g_ptr_array_add(res, g_strdup(path));
+ }
+ else if (recursive && S_ISDIR(st.st_mode)) {
+ rspamd_snprintf(pathbuf, sizeof(pathbuf), "%s%c%s",
+ path, G_DIR_SEPARATOR, pattern);
+
+ if (!rspamd_glob_dir(full_path, pattern, recursive, rec_len + 1,
+ res, err)) {
+ globfree(&globbuf);
+
+ return FALSE;
+ }
+ }
+ }
+
+ globfree(&globbuf);
+
+ return TRUE;
+}
+
+GPtrArray *
+rspamd_glob_path(const gchar *dir,
+ const gchar *pattern,
+ gboolean recursive,
+ GError **err)
+{
+ gchar path[PATH_MAX];
+ GPtrArray *res;
+
+ res = g_ptr_array_new_full(32, (GDestroyNotify) g_free);
+ rspamd_snprintf(path, sizeof(path), "%s%c%s", dir, G_DIR_SEPARATOR, pattern);
+
+ if (!rspamd_glob_dir(path, pattern, recursive, 0, res, err)) {
+ g_ptr_array_free(res, TRUE);
+
+ return NULL;
+ }
+
+ return res;
+}
+
+double
+rspamd_set_counter(struct rspamd_counter_data *cd, gdouble value)
+{
+ gdouble cerr;
+
+ /* Cumulative moving average using per-process counter data */
+ if (cd->number == 0) {
+ cd->mean = 0;
+ cd->stddev = 0;
+ }
+
+ cd->mean += (value - cd->mean) / (gdouble) (++cd->number);
+ cerr = (value - cd->mean) * (value - cd->mean);
+ cd->stddev += (cerr - cd->stddev) / (gdouble) (cd->number);
+
+ return cd->mean;
+}
+
+float rspamd_set_counter_ema(struct rspamd_counter_data *cd,
+ float value,
+ float alpha)
+{
+ float diff, incr;
+
+ /* Cumulative moving average using per-process counter data */
+ if (cd->number == 0) {
+ cd->mean = 0;
+ cd->stddev = 0;
+ }
+
+ diff = value - cd->mean;
+ incr = diff * alpha;
+ cd->mean += incr;
+ cd->stddev = (1.0f - alpha) * (cd->stddev + diff * incr);
+ cd->number++;
+
+ return cd->mean;
+}
+
+void rspamd_ptr_array_shuffle(GPtrArray *ar)
+{
+ if (ar->len < 2) {
+ return;
+ }
+
+ guint n = ar->len;
+
+ for (guint i = 0; i < n - 1; i++) {
+ guint j = i + rspamd_random_uint64_fast() % (n - i);
+ gpointer t = g_ptr_array_index(ar, j);
+ g_ptr_array_index(ar, j) = g_ptr_array_index(ar, i);
+ g_ptr_array_index(ar, i) = t;
+ }
+}
+
+float rspamd_sum_floats(float *ar, gsize *nelts)
+{
+ float sum = 0.0f;
+ volatile float c = 0.0f; /* We don't want any optimisations around c */
+ gsize cnt = 0;
+
+ for (gsize i = 0; i < *nelts; i++) {
+ float elt = ar[i];
+
+ if (!isnan(elt)) {
+ cnt++;
+ float y = elt - c;
+ float t = sum + y;
+ c = (t - sum) - y;
+ sum = t;
+ }
+ }
+
+ *nelts = cnt;
+ return sum;
+}
+
+void rspamd_normalize_path_inplace(gchar *path, guint len, gsize *nlen)
+{
+ const gchar *p, *end, *slash = NULL, *dot = NULL;
+ gchar *o;
+ enum {
+ st_normal = 0,
+ st_got_dot,
+ st_got_dot_dot,
+ st_got_slash,
+ st_got_slash_slash,
+ } state = st_normal;
+
+ p = path;
+ end = path + len;
+ o = path;
+
+ while (p < end) {
+ switch (state) {
+ case st_normal:
+ if (G_UNLIKELY(*p == '/')) {
+ state = st_got_slash;
+ slash = p;
+ }
+ else if (G_UNLIKELY(*p == '.')) {
+ state = st_got_dot;
+ dot = p;
+ }
+ else {
+ *o++ = *p;
+ }
+ p++;
+ break;
+ case st_got_slash:
+ if (G_UNLIKELY(*p == '/')) {
+ /* Ignore double slash */
+ *o++ = *p;
+ state = st_got_slash_slash;
+ }
+ else if (G_UNLIKELY(*p == '.')) {
+ dot = p;
+ state = st_got_dot;
+ }
+ else {
+ *o++ = '/';
+ *o++ = *p;
+ slash = NULL;
+ dot = NULL;
+ state = st_normal;
+ }
+ p++;
+ break;
+ case st_got_slash_slash:
+ if (G_LIKELY(*p != '/')) {
+ slash = p - 1;
+ dot = NULL;
+ state = st_normal;
+ continue;
+ }
+ p++;
+ break;
+ case st_got_dot:
+ if (G_UNLIKELY(*p == '/')) {
+ /* Remove any /./ or ./ paths */
+ if (((o > path && *(o - 1) != '/') || (o == path)) && slash) {
+ /* Preserve one slash */
+ *o++ = '/';
+ }
+
+ slash = p;
+ dot = NULL;
+ /* Ignore last slash */
+ state = st_normal;
+ }
+ else if (*p == '.') {
+ /* Double dot character */
+ state = st_got_dot_dot;
+ }
+ else {
+ /* We have something like .some or /.some */
+ if (dot && p > dot) {
+ if (slash == dot - 1 && (o > path && *(o - 1) != '/')) {
+ /* /.blah */
+ memmove(o, slash, p - slash);
+ o += p - slash;
+ }
+ else {
+ memmove(o, dot, p - dot);
+ o += p - dot;
+ }
+ }
+
+ slash = NULL;
+ dot = NULL;
+ state = st_normal;
+ continue;
+ }
+
+ p++;
+ break;
+ case st_got_dot_dot:
+ if (*p == '/') {
+ /* We have something like /../ or ../ */
+ if (slash) {
+ /* We need to remove the last component from o if it is there */
+ if (o > path + 2 && *(o - 1) == '/') {
+ slash = rspamd_memrchr(path, '/', o - path - 2);
+ }
+ else if (o > path + 1) {
+ slash = rspamd_memrchr(path, '/', o - path - 1);
+ }
+ else {
+ slash = NULL;
+ }
+
+ if (slash) {
+ o = (gchar *) slash;
+ }
+ /* Otherwise we keep these dots */
+ slash = p;
+ state = st_got_slash;
+ }
+ else {
+ /* We have something like bla../, so we need to copy it as is */
+ if (o > path && dot && p > dot) {
+ memmove(o, dot, p - dot);
+ o += p - dot;
+ }
+
+ slash = NULL;
+ dot = NULL;
+ state = st_normal;
+ continue;
+ }
+ }
+ else {
+ /* We have something like ..bla or ... */
+ if (slash) {
+ *o++ = '/';
+ }
+
+ if (dot && p > dot) {
+ memmove(o, dot, p - dot);
+ o += p - dot;
+ }
+
+ slash = NULL;
+ dot = NULL;
+ state = st_normal;
+ continue;
+ }
+
+ p++;
+ break;
+ }
+ }
+
+ /* Leftover */
+ switch (state) {
+ case st_got_dot_dot:
+ /* Trailing .. */
+ if (slash) {
+ /* We need to remove the last component from o if it is there */
+ if (o > path + 2 && *(o - 1) == '/') {
+ slash = rspamd_memrchr(path, '/', o - path - 2);
+ }
+ else if (o > path + 1) {
+ slash = rspamd_memrchr(path, '/', o - path - 1);
+ }
+ else {
+ if (o == path) {
+ /* Corner case */
+ *o++ = '/';
+ }
+
+ slash = NULL;
+ }
+
+ if (slash) {
+ /* Remove last / */
+ o = (gchar *) slash;
+ }
+ }
+ else {
+ /* Corner case */
+ if (o == path) {
+ *o++ = '/';
+ }
+ else {
+ if (dot && p > dot) {
+ memmove(o, dot, p - dot);
+ o += p - dot;
+ }
+ }
+ }
+ break;
+ case st_got_dot:
+ if (slash) {
+ /* /. -> must be / */
+ *o++ = '/';
+ }
+ else {
+ if (o > path) {
+ *o++ = '.';
+ }
+ }
+ break;
+ case st_got_slash:
+ *o++ = '/';
+ break;
+ default:
+#if 0
+ if (o > path + 1 && *(o - 1) == '/') {
+ o --;
+ }
+#endif
+ break;
+ }
+
+ if (nlen) {
+ *nlen = (o - path);
+ }
+}
diff --git a/src/libutil/util.h b/src/libutil/util.h
new file mode 100644
index 0000000..7111a07
--- /dev/null
+++ b/src/libutil/util.h
@@ -0,0 +1,581 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_UTIL_H
+#define RSPAMD_UTIL_H
+
+#include "config.h"
+#include "mem_pool.h"
+#include "printf.h"
+#include "fstring.h"
+#include "addr.h"
+#include "str_util.h"
+
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+
+#include <time.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_config;
+
+enum rspamd_exception_type {
+ RSPAMD_EXCEPTION_NEWLINE = 0,
+ RSPAMD_EXCEPTION_URL,
+ RSPAMD_EXCEPTION_GENERIC,
+};
+/**
+ * Structure to point exception in text from processing
+ */
+struct rspamd_process_exception {
+ goffset pos;
+ guint len;
+ gpointer ptr;
+ enum rspamd_exception_type type;
+};
+
+/**
+ * Create generic socket
+ * @param af address family
+ * @param type socket type
+ * @param protocol socket protocol
+ * @param async set non-blocking on a socket
+ * @return socket FD or -1 in case of error
+ */
+gint rspamd_socket_create(gint af, gint type, gint protocol, gboolean async);
+
+/*
+ * Create socket and bind or connect it to specified address and port
+ */
+gint rspamd_socket_tcp(struct addrinfo *, gboolean is_server, gboolean async);
+
+/*
+ * Create socket and bind or connect it to specified address and port
+ */
+gint rspamd_socket_udp(struct addrinfo *, gboolean is_server, gboolean async);
+
+/*
+ * Create and bind or connect unix socket
+ */
+gint rspamd_socket_unix(const gchar *,
+ struct sockaddr_un *,
+ gint type,
+ gboolean is_server,
+ gboolean async);
+
+/**
+ * Make a universal socket
+ * @param credits host, ip or path to unix socket
+ * @param port port (used for network sockets)
+ * @param type type of socket (SO_STREAM or SO_DGRAM)
+ * @param async make this socket async
+ * @param is_server make this socket as server socket
+ * @param try_resolve try name resolution for a socket (BLOCKING)
+ */
+gint rspamd_socket(const gchar *credits, guint16 port, gint type,
+ gboolean async, gboolean is_server, gboolean try_resolve);
+
+
+/*
+ * Create socketpair
+ */
+gboolean rspamd_socketpair(gint pair[2], gint af);
+
+/*
+ * Make specified socket non-blocking
+ */
+gint rspamd_socket_nonblocking(gint);
+
+/*
+ * Make specified socket blocking
+ */
+gint rspamd_socket_blocking(gint);
+
+/*
+ * Poll a sync socket for specified events
+ */
+gint rspamd_socket_poll(gint fd, gint timeout, short events);
+
+/*
+ * Init signals
+ */
+#ifdef HAVE_SA_SIGINFO
+
+void rspamd_signals_init(struct sigaction *sa, void (*sig_handler)(gint,
+ siginfo_t *,
+ void *));
+
+#else
+void rspamd_signals_init(struct sigaction *sa, void (*sig_handler)(gint));
+#endif
+
+/*
+ * Process title utility functions
+ */
+gint rspamd_init_title(rspamd_mempool_t *pool, gint argc, gchar *argv[], gchar *envp[]);
+gint rspamd_setproctitle(const gchar *fmt, ...);
+
+#ifndef HAVE_PIDFILE
+/*
+ * Pidfile functions from FreeBSD libutil code
+ */
+typedef struct rspamd_pidfh_s {
+ gint pf_fd;
+#ifdef HAVE_PATH_MAX
+ gchar pf_path[PATH_MAX + 1];
+#elif defined(HAVE_MAXPATHLEN)
+ gchar pf_path[MAXPATHLEN + 1];
+#else
+ gchar pf_path[1024 + 1];
+#endif
+ dev_t pf_dev;
+ ino_t pf_ino;
+} rspamd_pidfh_t;
+
+rspamd_pidfh_t *rspamd_pidfile_open(const gchar *path,
+ mode_t mode,
+ pid_t *pidptr);
+
+gint rspamd_pidfile_write(rspamd_pidfh_t *pfh);
+
+gint rspamd_pidfile_close(rspamd_pidfh_t *pfh);
+
+gint rspamd_pidfile_remove(rspamd_pidfh_t *pfh);
+
+#else
+typedef struct pidfh rspamd_pidfh_t;
+#define rspamd_pidfile_open pidfile_open
+#define rspamd_pidfile_write pidfile_write
+#define rspamd_pidfile_close pidfile_close
+#define rspamd_pidfile_remove pidfile_remove
+#endif
+
+/*
+ * Replace %r with rcpt value and %f with from value, new string is allocated in pool
+ */
+gchar *resolve_stat_filename(rspamd_mempool_t *pool,
+ gchar *pattern,
+ gchar *rcpt,
+ gchar *from);
+
+const gchar *
+rspamd_log_check_time(gdouble start, gdouble end, gint resolution);
+
+/*
+ * File locking functions
+ */
+gboolean rspamd_file_lock(gint fd, gboolean async);
+
+gboolean rspamd_file_unlock(gint fd, gboolean async);
+
+/*
+ * Workarounds for older versions of glib
+ */
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 22))
+void g_ptr_array_unref(GPtrArray *array);
+gboolean g_int64_equal(gconstpointer v1, gconstpointer v2);
+guint g_int64_hash(gconstpointer v);
+#endif
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 14))
+void g_queue_clear(GQueue *queue);
+#endif
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 32))
+void g_queue_free_full(GQueue *queue, GDestroyNotify free_func);
+#endif
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 40))
+void g_ptr_array_insert(GPtrArray *array, gint index_, gpointer data);
+#endif
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 30))
+GPtrArray *g_ptr_array_new_full(guint reserved_size,
+ GDestroyNotify element_free_func);
+#endif
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION < 32))
+const gchar *g_environ_getenv(gchar **envp, const gchar *variable);
+#endif
+
+/*
+ * Convert milliseconds to timeval fields
+ */
+#define msec_to_tv(msec, tv) \
+ do { \
+ (tv)->tv_sec = (msec) / 1000; \
+ (tv)->tv_usec = \
+ ((msec) - (tv)->tv_sec * 1000) * 1000; \
+ } while (0)
+#define double_to_tv(dbl, tv) \
+ do { \
+ (tv)->tv_sec = (int) (dbl); \
+ (tv)->tv_usec = \
+ ((dbl) - (int) (dbl)) * 1000 * 1000; \
+ } while (0)
+#define double_to_ts(dbl, ts) \
+ do { \
+ (ts)->tv_sec = (int) (dbl); \
+ (ts)->tv_nsec = \
+ ((dbl) - (int) (dbl)) * 1e9; \
+ } while (0)
+#define tv_to_msec(tv) ((tv)->tv_sec * 1000LLU + (tv)->tv_usec / 1000LLU)
+#define tv_to_double(tv) ((double) (tv)->tv_sec + (tv)->tv_usec / 1.0e6)
+#define ts_to_usec(ts) ((ts)->tv_sec * 1000000LLU + \
+ (ts)->tv_nsec / 1000LLU)
+#define ts_to_double(tv) ((double) (tv)->tv_sec + (tv)->tv_nsec / 1.0e9)
+
+/**
+ * Try to allocate a file on filesystem (using fallocate or posix_fallocate)
+ * @param fd descriptor
+ * @param offset offset of file
+ * @param len length to allocate
+ * @return -1 in case of failure
+ */
+gint rspamd_fallocate(gint fd, off_t offset, off_t len);
+
+/**
+ * Utils for working with threads to be compatible with all glib versions
+ */
+typedef struct rspamd_mutex_s {
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30))
+ GMutex mtx;
+#else
+ GStaticMutex mtx;
+#endif
+} rspamd_mutex_t;
+
+
+/**
+ * Create new mutex
+ * @return mutex or NULL
+ */
+rspamd_mutex_t *rspamd_mutex_new(void);
+
+/**
+ * Lock mutex
+ * @param mtx
+ */
+void rspamd_mutex_lock(rspamd_mutex_t *mtx);
+
+/**
+ * Unlock mutex
+ * @param mtx
+ */
+void rspamd_mutex_unlock(rspamd_mutex_t *mtx);
+
+/**
+ * Clear rspamd mutex
+ * @param mtx
+ */
+void rspamd_mutex_free(rspamd_mutex_t *mtx);
+
+/**
+ * Deep copy of one hash table to another
+ * @param src source hash
+ * @param dst destination hash
+ * @param key_copy_func function called to copy or modify keys (or NULL)
+ * @param value_copy_func function called to copy or modify values (or NULL)
+ * @param ud user data for copy functions
+ */
+void rspamd_hash_table_copy(GHashTable *src, GHashTable *dst,
+ gpointer (*key_copy_func)(gconstpointer data, gpointer ud),
+ gpointer (*value_copy_func)(gconstpointer data, gpointer ud),
+ gpointer ud);
+
+
+/**
+ * Read passphrase from tty
+ * @param buf buffer to fill with a password
+ * @param size size of the buffer
+ * @param echo turn echo on or off
+ * @param key unused key
+ * @return size of password read
+ */
+#define rspamd_read_passphrase(buf, size, echo, key) (rspamd_read_passphrase_with_prompt("Enter passphrase: ", (buf), (size), (echo), (key)))
+
+/**
+ * Read passphrase from tty with prompt
+ * @param prompt prompt to use
+ * @param buf buffer to fill with a password
+ * @param size size of the buffer
+ * @param echo turn echo on or off
+ * @param key unused key
+ * @return
+ */
+gint rspamd_read_passphrase_with_prompt(const gchar *prompt, gchar *buf, gint size, bool echo, gpointer key);
+
+/**
+ * Portably return the current clock ticks as seconds
+ * @return
+ */
+gdouble rspamd_get_ticks(gboolean rdtsc_ok);
+
+/**
+ * Portably return the current virtual clock ticks as seconds
+ * @return
+ */
+gdouble rspamd_get_virtual_ticks(void);
+
+
+/**
+ * Return the real timestamp as unixtime
+ */
+gdouble rspamd_get_calendar_ticks(void);
+
+/**
+ * Special utility to help array freeing in rspamd_mempool
+ * @param p
+ */
+void rspamd_ptr_array_free_hard(gpointer p);
+
+/**
+ * Special utility to help array freeing in rspamd_mempool
+ * @param p
+ */
+void rspamd_array_free_hard(gpointer p);
+
+/**
+ * Special utility to help GString freeing in rspamd_mempool
+ * @param p
+ */
+void rspamd_gstring_free_hard(gpointer p);
+
+/**
+ * Special utility to help GError freeing in rspamd_mempool
+ * @param p
+ */
+void rspamd_gerror_free_maybe(gpointer p);
+
+/**
+ * Special utility to help GString freeing (without freeing the memory segment) in rspamd_mempool
+ * @param p
+ */
+void rspamd_gstring_free_soft(gpointer p);
+
+
+/**
+ * Returns some statically initialized random hash seed
+ * @return hash seed
+ */
+guint64 rspamd_hash_seed(void);
+
+/**
+ * Returns random hex string of the specified length
+ * @param buf
+ * @param len
+ */
+void rspamd_random_hex(gchar *buf, guint64 len);
+
+/**
+ * Returns
+ * @param pattern pattern to create (should end with some number of X symbols), modified by this function
+ * @return
+ */
+gint rspamd_shmem_mkstemp(gchar *pattern);
+
+/**
+ * Return jittered time value
+ */
+gdouble rspamd_time_jitter(gdouble in, gdouble jitter);
+
+/**
+ * Return random double in range [0..1)
+ * @return
+ */
+gdouble rspamd_random_double(void);
+
+/**
+ * Return random double in range [0..1) using xoroshiro128+ algorithm (not crypto secure)
+ * @return
+ */
+gdouble rspamd_random_double_fast(void);
+gdouble rspamd_random_double_fast_seed(guint64 *seed);
+uint64_t rspamd_random_uint64_fast_seed(uint64_t *seed);
+guint64 rspamd_random_uint64_fast(void);
+
+/**
+ * Seed fast rng
+ */
+void rspamd_random_seed_fast(void);
+
+/**
+ * Constant time version of memcmp
+ */
+gboolean rspamd_constant_memcmp(const void *a, const void *b, gsize len);
+
+/**
+ * Open file without following symlinks or special stuff
+ * @param fname filename
+ * @param oflags open flags
+ * @param mode mode to open
+ * @return fd or -1 in case of error
+ */
+int rspamd_file_xopen(const char *fname, int oflags, guint mode,
+ gboolean allow_symlink);
+
+/**
+ * Map file without following symlinks or special stuff
+ * @param fname filename
+ * @param mode mode to open
+ * @param size target size (must NOT be NULL)
+ * @return pointer to memory (should be freed using munmap) or NULL in case of error
+ */
+gpointer rspamd_file_xmap(const char *fname, guint mode, gsize *size,
+ gboolean allow_symlink);
+
+/**
+ * Map named shared memory segment
+ * @param fname filename
+ * @param mode mode to open
+ * @param size target size (must NOT be NULL)
+ * @return pointer to memory (should be freed using munmap) or NULL in case of error
+ */
+gpointer rspamd_shmem_xmap(const char *fname, guint mode,
+ gsize *size);
+
+/**
+ * Normalize probabilities using polynomial function
+ * @param x probability (bias .. 1)
+ * @return
+ */
+gdouble rspamd_normalize_probability(gdouble x, gdouble bias);
+
+/**
+ * Converts struct tm to time_t
+ * @param tm
+ * @param tz timezone in format (hours * 100) + minutes
+ * @return
+ */
+guint64 rspamd_tm_to_time(const struct tm *tm, glong tz);
+
+/**
+ * Splits unix timestamp into struct tm using GMT timezone
+ * @param ts
+ * @param dest
+ */
+void rspamd_gmtime(gint64 ts, struct tm *dest);
+
+/**
+ * Split unix timestamp into struct tm using local timezone
+ * @param ts
+ * @param dest
+ */
+void rspamd_localtime(gint64 ts, struct tm *dest);
+
+#define PTR_ARRAY_FOREACH(ar, i, cur) for ((i) = 0; (ar) != NULL && (i) < (ar)->len && (((cur) = (__typeof__(cur)) g_ptr_array_index((ar), (i))) || 1); ++(i))
+
+/**
+ * Compresses the input string using gzip+zlib. Old string is replaced and freed
+ * if compressed.
+ * @param in
+ * @return TRUE if a string has been compressed
+ */
+gboolean rspamd_fstring_gzip(rspamd_fstring_t **in);
+
+/**
+ * Compresses the input string using gzip+zlib. Old string is replaced and freed
+ * if compressed. If not compressed it is untouched.
+ * @param in
+ * @return TRUE if a string has been compressed
+ */
+gboolean rspamd_fstring_gunzip(rspamd_fstring_t **in);
+
+/**
+ * Perform globbing searching for the specified path. Allow recursion,
+ * returns an error if maximum nesting is reached.
+ * @param pattern
+ * @param recursive
+ * @param err
+ * @return GPtrArray of gchar *, elements are freed when array is freed
+ */
+GPtrArray *rspamd_glob_path(const gchar *dir,
+ const gchar *pattern,
+ gboolean recursive,
+ GError **err);
+
+struct rspamd_counter_data {
+ float mean;
+ float stddev;
+ guint64 number;
+};
+
+/**
+ * Sets counter's data using exponential moving average
+ * @param cd counter
+ * @param value new counter value
+ * @param alpha decay coefficient (0..1)
+ * @return new counter value
+ */
+float rspamd_set_counter_ema(struct rspamd_counter_data *cd,
+ float value,
+ float alpha);
+
+/**
+ * Sets counter's data using flat moving average
+ * @param cd counter
+ * @param value new counter value
+ * @return new counter value
+ */
+double rspamd_set_counter(struct rspamd_counter_data *cd,
+ gdouble value);
+
+/**
+ * Shuffle elements in an array inplace
+ * @param ar
+ */
+void rspamd_ptr_array_shuffle(GPtrArray *ar);
+
+enum rspamd_pbkdf_version_id {
+ RSPAMD_PBKDF_ID_V1 = 1,
+ RSPAMD_PBKDF_ID_V2 = 2,
+ RSPAMD_PBKDF_ID_MAX
+};
+
+struct rspamd_controller_pbkdf {
+ const char *name;
+ const char *alias;
+ const char *description;
+ int type; /* enum rspamd_cryptobox_pbkdf_type */
+ gint id;
+ guint complexity;
+ gsize salt_len;
+ gsize key_len;
+};
+
+extern const struct rspamd_controller_pbkdf pbkdf_list[];
+
+/**
+ * Sum array of floats using Kahan sum algorithm
+ * @param ar
+ * @param nelts
+ * @return
+ */
+float rspamd_sum_floats(float *ar, gsize *nelts);
+
+/**
+ * Normalize file path removing dot sequences and repeating '/' symbols as
+ * per rfc3986#section-5.2
+ * @param path
+ * @param len
+ * @param nlen
+ */
+void rspamd_normalize_path_inplace(gchar *path, guint len, gsize *nlen);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt
new file mode 100644
index 0000000..a504f99
--- /dev/null
+++ b/src/lua/CMakeLists.txt
@@ -0,0 +1,39 @@
+# Lua support makefile
+SET(LUASRC ${CMAKE_CURRENT_SOURCE_DIR}/lua_common.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_logger.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_task.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_config.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_classifier.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_cfg_file.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_regexp.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_cdb.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_xmlrpc.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_http.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_redis.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_upstream.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_mempool.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_dns_resolver.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_rsa.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_ip.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_expression.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_trie.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_mimepart.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_url.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_tcp.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_html.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_sqlite3.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_cryptobox.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_map.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_thread_pool.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_dns.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_udp.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_text.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_worker.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_kann.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_spf.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_tensor.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_parsers.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_compress.c)
+
+SET(RSPAMD_LUA ${LUASRC} PARENT_SCOPE) \ No newline at end of file
diff --git a/src/lua/lua_cdb.c b/src/lua/lua_cdb.c
new file mode 100644
index 0000000..76a5795
--- /dev/null
+++ b/src/lua/lua_cdb.c
@@ -0,0 +1,391 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "cdb.h"
+
+#define CDB_REFRESH_TIME 60
+
+/***
+ * @module rspamd_cdb
+ * Rspamd CDB module is used to read and write key/value pairs to the CDB file
+ *
+ * @example
+local rspamd_cdb = require "rspamd_cdb"
+rspamd_cdb.build('/tmp/test.cdb'):add('test', 'value'):finalize()
+local c = rspamd_cdb.open('/tmp/test.cdb')
+c:find('test')
+-- will return 'value'
+ */
+
+/***
+ * @function rspamd_cdb.open(filename, [ev_base])
+ * Opens an existing CDB for reading. If `ev_base` is specified, then cdb file is added
+ * for monitoring, that will get updates on disk file changes.
+ * @param {string} filename path to file
+ * @param {ev_base} event loop object
+ * @return {rspamd_cdb} cdb object
+ */
+LUA_FUNCTION_DEF(cdb, create);
+/***
+ * @method rspamd_cdb:find(key)
+ * Finds a specific key in cdb and returns a string or nil if a key has not been found
+ * @param {string} key key to find
+ * @return {string/nil} value for the specific key
+ */
+LUA_FUNCTION_DEF(cdb, lookup);
+/***
+ * @method rspamd_cdb:get_name()
+ * Returns filename for the specific cdb
+ * @return {string} filename for cdb
+ */
+LUA_FUNCTION_DEF(cdb, get_name);
+LUA_FUNCTION_DEF(cdb, destroy);
+
+/***
+ * @function rspamd_cdb.build(filename, [mode])
+ * Creates a new cdb in a file (existing one will be overwritten!). The object
+ * returned can be used merely for adding data. Upon finalizing, the data is written to
+ * disk and cdb can no longer be changed.
+ * @param {string} filename path to file
+ * @param {int} mode numeric mode to create a file
+ * @return {rspamd_cdb_builder} cdb builder object (or nil + error message)
+ */
+LUA_FUNCTION_DEF(cdb, build);
+/***
+ * @method rspamd_cdb_builder:add(key, value)
+ * Adds new value to cdb in the builder mode
+ * @param {string} key key to add
+ * @param {string} value value to associate with the key
+ * @return {rspamd_cdb_builder} the same object to allow chaining calls
+ */
+LUA_FUNCTION_DEF(cdb_builder, add);
+/***
+ * @method rspamd_cdb_builder:finalize()
+ * Finalizes the CDB and writes it to disk. This method also closes FD associated with
+ * CDB builder. No further additions are allowed after this point
+ */
+LUA_FUNCTION_DEF(cdb_builder, finalize);
+LUA_FUNCTION_DEF(cdb_builder, dtor);
+
+static const struct luaL_reg cdblib_m[] = {
+ LUA_INTERFACE_DEF(cdb, lookup),
+ {"find", lua_cdb_lookup},
+ LUA_INTERFACE_DEF(cdb, get_name),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_cdb_destroy},
+ {NULL, NULL}};
+
+static const struct luaL_reg cdbbuilderlib_m[] = {
+ LUA_INTERFACE_DEF(cdb_builder, add),
+ LUA_INTERFACE_DEF(cdb_builder, finalize),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_cdb_builder_dtor},
+ {NULL, NULL}};
+
+static const struct luaL_reg cdblib_f[] = {
+ LUA_INTERFACE_DEF(cdb, create),
+ {"open", lua_cdb_create},
+ {"build", lua_cdb_build},
+ {NULL, NULL}};
+
+static struct cdb *
+lua_check_cdb(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cdb}");
+
+ luaL_argcheck(L, ud != NULL, pos, "'cdb' expected");
+ return ud ? *((struct cdb **) ud) : NULL;
+}
+
+static struct cdb_make *
+lua_check_cdb_builder(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cdb_builder}");
+
+ luaL_argcheck(L, ud != NULL, pos, "'cdb_builder' expected");
+ return ud ? ((struct cdb_make *) ud) : NULL;
+}
+
+static const char *
+lua_cdb_get_input(lua_State *L, int pos, gsize *olen)
+{
+ int t = lua_type(L, pos);
+
+ switch (t) {
+ case LUA_TSTRING:
+ return lua_tolstring(L, pos, olen);
+ case LUA_TNUMBER: {
+ static char numbuf[sizeof(lua_Number)];
+ lua_Number n = lua_tonumber(L, pos);
+ memcpy(numbuf, &n, sizeof(numbuf));
+ *olen = sizeof(n);
+ return numbuf;
+ }
+ case LUA_TUSERDATA: {
+ void *p = rspamd_lua_check_udata_maybe(L, pos, "rspamd{text}");
+ if (p) {
+ struct rspamd_lua_text *t = (struct rspamd_lua_text *) p;
+ *olen = t->len;
+ return t->start;
+ }
+
+ p = rspamd_lua_check_udata_maybe(L, pos, "rspamd{int64}");
+ if (p) {
+ static char numbuf[sizeof(gint64)];
+
+ memcpy(numbuf, p, sizeof(numbuf));
+ *olen = sizeof(numbuf);
+ return numbuf;
+ }
+ }
+ default:
+ break;
+ }
+
+ return NULL;
+}
+
+static gint
+lua_cdb_create(lua_State *L)
+{
+ struct cdb *cdb, **pcdb;
+ const gchar *filename;
+ gint fd;
+
+ struct ev_loop *ev_base = NULL;
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ ev_base = lua_check_ev_base(L, 2);
+ }
+
+ filename = luaL_checkstring(L, 1);
+ /* If file begins with cdb://, just skip it */
+ if (g_ascii_strncasecmp(filename, "cdb://", sizeof("cdb://") - 1) == 0) {
+ filename += sizeof("cdb://") - 1;
+ }
+
+ if ((fd = open(filename, O_RDONLY)) == -1) {
+ msg_warn("cannot open cdb: %s, %s", filename, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ cdb = g_malloc0(sizeof(struct cdb));
+ cdb->filename = g_strdup(filename);
+ if (cdb_init(cdb, fd) == -1) {
+ g_free(cdb->filename);
+ g_free(cdb);
+ msg_warn("cannot open cdb: %s, %s", filename, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+#ifdef HAVE_READAHEAD
+ struct stat st;
+ /*
+ * Do not readahead more than 100mb,
+ * which is enough for the vast majority of the use cases
+ */
+ static const size_t max_readahead = 100 * 0x100000;
+
+ if (fstat(cdb_fileno(cdb), &st) != 1) {
+ /* Must always be true because cdb_init calls it as well */
+ if (readahead(cdb_fileno(cdb), 0, MIN(max_readahead, st.st_size)) == -1) {
+ msg_warn("cannot readahead cdb: %s, %s", filename, strerror(errno));
+ }
+ }
+#endif
+ if (ev_base) {
+ cdb_add_timer(cdb, ev_base, CDB_REFRESH_TIME);
+ }
+ pcdb = lua_newuserdata(L, sizeof(struct cdb *));
+ rspamd_lua_setclass(L, "rspamd{cdb}", -1);
+ *pcdb = cdb;
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_cdb_get_name(lua_State *L)
+{
+ struct cdb *cdb = lua_check_cdb(L, 1);
+
+ if (!cdb) {
+ lua_error(L);
+ return 1;
+ }
+ lua_pushstring(L, cdb->filename);
+ return 1;
+}
+
+static gint
+lua_cdb_lookup(lua_State *L)
+{
+ struct cdb *cdb = lua_check_cdb(L, 1);
+ gsize klen;
+ const gchar *what = lua_cdb_get_input(L, 2, &klen);
+
+ if (!cdb || what == NULL) {
+ return lua_error(L);
+ }
+
+ if (cdb_find(cdb, what, klen) > 0) {
+ /* Extract and push value to lua as string */
+ lua_pushlstring(L, cdb_getdata(cdb), cdb_datalen(cdb));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_cdb_destroy(lua_State *L)
+{
+ struct cdb *cdb = lua_check_cdb(L, 1);
+
+ if (cdb) {
+ cdb_free(cdb);
+ if (cdb->cdb_fd != -1) {
+ (void) close(cdb->cdb_fd);
+ }
+ g_free(cdb->filename);
+ g_free(cdb);
+ }
+
+ return 0;
+}
+
+static gint
+lua_cdb_build(lua_State *L)
+{
+ const char *filename = luaL_checkstring(L, 1);
+ int fd, mode = 00755;
+
+ if (filename == NULL) {
+ return luaL_error(L, "invalid arguments, filename expected");
+ }
+
+ /* If file begins with cdb://, just skip it */
+ if (g_ascii_strncasecmp(filename, "cdb://", sizeof("cdb://") - 1) == 0) {
+ filename += sizeof("cdb://") - 1;
+ }
+
+ if (lua_isnumber(L, 2)) {
+ mode = lua_tointeger(L, 2);
+ }
+
+ fd = rspamd_file_xopen(filename, O_RDWR | O_CREAT | O_TRUNC, mode, 0);
+
+ if (fd == -1) {
+ lua_pushnil(L);
+ lua_pushfstring(L, "cannot open cdb: %s, %s", filename, strerror(errno));
+
+ return 2;
+ }
+
+ struct cdb_make *cdbm = lua_newuserdata(L, sizeof(struct cdb_make));
+
+ g_assert(cdb_make_start(cdbm, fd) == 0);
+ rspamd_lua_setclass(L, "rspamd{cdb_builder}", -1);
+
+ return 1;
+}
+
+static gint
+lua_cdb_builder_add(lua_State *L)
+{
+ struct cdb_make *cdbm = lua_check_cdb_builder(L, 1);
+ gsize data_sz, key_sz;
+ const char *key = lua_cdb_get_input(L, 2, &key_sz);
+ const char *data = lua_cdb_get_input(L, 3, &data_sz);
+
+ if (cdbm == NULL || key == NULL || data == NULL || cdbm->cdb_fd == -1) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (cdb_make_add(cdbm, key, key_sz, data, data_sz) == -1) {
+ lua_pushvalue(L, 1);
+ lua_pushfstring(L, "cannot push value to cdb: %s", strerror(errno));
+
+ return 2;
+ }
+
+ /* Allow chaining */
+ lua_pushvalue(L, 1);
+ return 1;
+}
+
+static gint
+lua_cdb_builder_finalize(lua_State *L)
+{
+ struct cdb_make *cdbm = lua_check_cdb_builder(L, 1);
+
+ if (cdbm == NULL || cdbm->cdb_fd == -1) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (cdb_make_finish(cdbm) == -1) {
+ lua_pushvalue(L, 1);
+ lua_pushfstring(L, "cannot finish value to cdb: %s", strerror(errno));
+
+ return 2;
+ }
+
+ close(cdbm->cdb_fd);
+ cdbm->cdb_fd = -1; /* To distinguish finalized object */
+
+ /* Allow chaining */
+ lua_pushvalue(L, 1);
+ return 1;
+}
+
+static gint
+lua_cdb_builder_dtor(lua_State *L)
+{
+ struct cdb_make *cdbm = lua_check_cdb_builder(L, 1);
+
+ if (cdbm == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (cdbm->cdb_fd != -1) {
+ cdb_make_finish(cdbm);
+ close(cdbm->cdb_fd);
+ cdbm->cdb_fd = -1; /* Finalized object */
+ }
+
+ return 0;
+}
+
+static gint
+lua_load_cdb(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cdblib_f);
+
+ return 1;
+}
+
+void luaopen_cdb(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{cdb}", cdblib_m);
+ lua_pop(L, 1);
+ rspamd_lua_new_class(L, "rspamd{cdb_builder}", cdbbuilderlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cdb", lua_load_cdb);
+}
diff --git a/src/lua/lua_cfg_file.c b/src/lua/lua_cfg_file.c
new file mode 100644
index 0000000..75bc380
--- /dev/null
+++ b/src/lua/lua_cfg_file.c
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "expression.h"
+#include "src/libserver/composites/composites.h"
+
+/*
+ * This is implementation of lua routines to handle config file params
+ */
+
+/* Process a single item in 'metrics' table */
+static void
+lua_process_metric(lua_State *L, const gchar *name, struct rspamd_config *cfg)
+{
+ gchar *symbol;
+ const gchar *desc = NULL;
+ gdouble *score;
+ struct rspamd_symbol *s;
+
+ /* Now iterate through module table */
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ /* key - -2, value - -1 */
+ symbol = rspamd_mempool_strdup(cfg->cfg_pool, luaL_checkstring(L, -2));
+ if (symbol != NULL) {
+ if (lua_istable(L, -1)) {
+ /* We got a table, so extract individual attributes */
+ lua_pushstring(L, "weight");
+ lua_gettable(L, -2);
+ if (lua_isnumber(L, -1)) {
+ score = rspamd_mempool_alloc(cfg->cfg_pool, sizeof(double));
+ *score = lua_tonumber(L, -1);
+ }
+ else {
+ msg_warn_config("cannot get weight of symbol: %s", symbol);
+ continue;
+ }
+ lua_pop(L, 1);
+ lua_pushstring(L, "description");
+ lua_gettable(L, -2);
+ if (lua_isstring(L, -1)) {
+ desc = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+ }
+ else if (lua_isnumber(L, -1)) {
+ /* Just got weight */
+ score = rspamd_mempool_alloc(cfg->cfg_pool, sizeof(double));
+ *score = lua_tonumber(L, -1);
+ }
+ else {
+ msg_warn_config("cannot get weight of symbol: %s", symbol);
+ continue;
+ }
+ /* Insert symbol */
+ if ((s =
+ g_hash_table_lookup(cfg->symbols, symbol)) != NULL) {
+ msg_info_config("replacing weight for symbol %s: %.2f -> %.2f",
+ symbol,
+ *s->weight_ptr,
+ *score);
+ s->weight_ptr = score;
+ }
+ else {
+ s = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*s));
+ s->name = symbol;
+ s->weight_ptr = score;
+ g_hash_table_insert(cfg->symbols, symbol, s);
+ }
+
+ if (desc) {
+ s->description = rspamd_mempool_strdup(cfg->cfg_pool, desc);
+ }
+ }
+ }
+}
+
+/* Do post load initialization based on lua */
+void rspamd_lua_post_load_config(struct rspamd_config *cfg)
+{
+ lua_State *L = cfg->lua_state;
+ const gchar *name;
+ ucl_object_t *obj;
+ gsize keylen, i;
+
+ /* First check all module options that may be overridden in 'config' global */
+ lua_getglobal(L, "config");
+
+ if (lua_istable(L, -1)) {
+ /* Iterate to get all keys */
+ GPtrArray *names = g_ptr_array_new_full(rspamd_lua_table_size(L, -1),
+ g_free);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 2)) {
+ gchar *tmp;
+ lua_pushvalue(L, -2);
+ name = luaL_checklstring(L, -1, &keylen);
+
+ if (name && lua_istable(L, -2)) {
+ tmp = g_malloc(keylen + 1);
+ rspamd_strlcpy(tmp, name, keylen + 1);
+ g_ptr_array_add(names, tmp);
+ }
+ }
+
+ PTR_ARRAY_FOREACH(names, i, name)
+ {
+ lua_getfield(L, -1, name);
+
+ if (lua_istable(L, -1)) {
+ obj = ucl_object_lua_import(L, lua_gettop(L));
+
+ if (obj != NULL) {
+ ucl_object_sort_keys(obj, UCL_SORT_KEYS_DEFAULT);
+ ucl_object_insert_key_merged(cfg->cfg_ucl_obj,
+ obj,
+ name,
+ strlen(name),
+ true);
+ }
+ }
+ }
+
+ g_ptr_array_free(names, TRUE);
+ }
+
+ /* Check metrics settings */
+ lua_getglobal(L, "metrics");
+
+ if (lua_istable(L, -1)) {
+ /* Iterate */
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ /* 'key' is at index -2 and 'value' is at index -1 */
+ /* Key must be a string and value must be a table */
+ name = luaL_checkstring(L, -2);
+ if (name != NULL && lua_istable(L, -1)) {
+ lua_process_metric(L, name, cfg);
+ }
+ }
+ }
+
+ lua_settop(L, 0);
+
+ rspamd_lua_start_gc(cfg);
+}
diff --git a/src/lua/lua_classifier.c b/src/lua/lua_classifier.c
new file mode 100644
index 0000000..39580a6
--- /dev/null
+++ b/src/lua/lua_classifier.c
@@ -0,0 +1,230 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+
+/* Classifier methods */
+LUA_FUNCTION_DEF(classifier, get_statfiles);
+LUA_FUNCTION_DEF(classifier, get_statfile_by_label);
+LUA_FUNCTION_DEF(classifier, get_param);
+
+static const struct luaL_reg classifierlib_m[] = {
+ LUA_INTERFACE_DEF(classifier, get_statfiles),
+ LUA_INTERFACE_DEF(classifier, get_param),
+ LUA_INTERFACE_DEF(classifier, get_statfile_by_label),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+LUA_FUNCTION_DEF(statfile, get_symbol);
+LUA_FUNCTION_DEF(statfile, get_label);
+LUA_FUNCTION_DEF(statfile, is_spam);
+LUA_FUNCTION_DEF(statfile, get_param);
+
+static const struct luaL_reg statfilelib_m[] = {
+ LUA_INTERFACE_DEF(statfile, get_symbol),
+ LUA_INTERFACE_DEF(statfile, get_label),
+ LUA_INTERFACE_DEF(statfile, is_spam),
+ LUA_INTERFACE_DEF(statfile, get_param),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static struct rspamd_statfile_config *lua_check_statfile(lua_State *L);
+
+/* Classifier implementation */
+
+
+static struct rspamd_classifier_config *
+lua_check_classifier(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{classifier}");
+ luaL_argcheck(L, ud != NULL, 1, "'classifier' expected");
+ return ud ? *((struct rspamd_classifier_config **) ud) : NULL;
+}
+
+/* Return table of statfiles indexed by name */
+static gint
+lua_classifier_get_statfiles(lua_State *L)
+{
+ struct rspamd_classifier_config *ccf = lua_check_classifier(L);
+ GList *cur;
+ struct rspamd_statfile_config *st, **pst;
+ gint i;
+
+ if (ccf) {
+ lua_newtable(L);
+ cur = g_list_first(ccf->statfiles);
+ i = 1;
+ while (cur) {
+ st = cur->data;
+ pst = lua_newuserdata(L, sizeof(struct rspamd_statfile_config *));
+ rspamd_lua_setclass(L, "rspamd{statfile}", -1);
+ *pst = st;
+ lua_rawseti(L, -2, i++);
+
+ cur = g_list_next(cur);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_classifier_get_param(lua_State *L)
+{
+ struct rspamd_classifier_config *ccf = lua_check_classifier(L);
+ const gchar *param;
+ const ucl_object_t *value;
+
+ param = luaL_checkstring(L, 2);
+
+ if (ccf != NULL && param != NULL) {
+ value = ucl_object_lookup(ccf->opts, param);
+
+ if (value != NULL) {
+ ucl_object_push_lua(L, value, true);
+ return 1;
+ }
+ }
+
+ lua_pushnil(L);
+
+ return 1;
+}
+
+/* Get statfile with specified label */
+static gint
+lua_classifier_get_statfile_by_label(lua_State *L)
+{
+ struct rspamd_classifier_config *ccf = lua_check_classifier(L);
+ struct rspamd_statfile_config *st, **pst;
+ const gchar *label;
+ GList *cur;
+ gint i;
+
+ label = luaL_checkstring(L, 2);
+ if (ccf && label) {
+ cur = g_hash_table_lookup(ccf->labels, label);
+ if (cur) {
+ lua_newtable(L);
+ i = 1;
+ while (cur) {
+ st = cur->data;
+ pst =
+ lua_newuserdata(L,
+ sizeof(struct rspamd_statfile_config *));
+ rspamd_lua_setclass(L, "rspamd{statfile}", -1);
+ *pst = st;
+ lua_rawseti(L, -2, i++);
+ cur = g_list_next(cur);
+ }
+ return 1;
+ }
+ }
+ lua_pushnil(L);
+ return 1;
+}
+
+/* Statfile functions */
+static gint
+lua_statfile_get_symbol(lua_State *L)
+{
+ struct rspamd_statfile_config *st = lua_check_statfile(L);
+
+ if (st != NULL) {
+ lua_pushstring(L, st->symbol);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_statfile_get_label(lua_State *L)
+{
+ struct rspamd_statfile_config *st = lua_check_statfile(L);
+
+ if (st != NULL && st->label != NULL) {
+ lua_pushstring(L, st->label);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_statfile_is_spam(lua_State *L)
+{
+ struct rspamd_statfile_config *st = lua_check_statfile(L);
+
+ if (st != NULL) {
+ lua_pushboolean(L, st->is_spam);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_statfile_get_param(lua_State *L)
+{
+ struct rspamd_statfile_config *st = lua_check_statfile(L);
+ const gchar *param;
+ const ucl_object_t *value;
+
+ param = luaL_checkstring(L, 2);
+
+ if (st != NULL && param != NULL) {
+ value = ucl_object_lookup(st->opts, param);
+ if (value != NULL) {
+ lua_pushstring(L, ucl_object_tostring_forced(value));
+ return 1;
+ }
+ }
+ lua_pushnil(L);
+
+ return 1;
+}
+
+static struct rspamd_statfile_config *
+lua_check_statfile(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{statfile}");
+ luaL_argcheck(L, ud != NULL, 1, "'statfile' expected");
+ return ud ? *((struct rspamd_statfile_config **) ud) : NULL;
+}
+
+
+/* Open functions */
+
+void luaopen_classifier(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{classifier}", classifierlib_m);
+ lua_pop(L, 1); /* remove metatable from stack */
+}
+
+void luaopen_statfile(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{statfile}", statfilelib_m);
+ lua_pop(L, 1); /* remove metatable from stack */
+}
diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c
new file mode 100644
index 0000000..9bf9514
--- /dev/null
+++ b/src/lua/lua_common.c
@@ -0,0 +1,2659 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_compress.h"
+#include "lptree.h"
+#include "utlist.h"
+#include "unix-std.h"
+#include "ottery.h"
+#include "lua_thread_pool.h"
+#include "libstat/stat_api.h"
+#include "libserver/rspamd_control.h"
+
+#include <math.h>
+
+
+/* Lua module init function */
+#define MODULE_INIT_FUNC "module_init"
+
+#ifdef WITH_LUA_TRACE
+ucl_object_t *lua_traces;
+#endif
+
+const luaL_reg null_reg[] = {
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static const char rspamd_modules_state_global[] = "rspamd_plugins_state";
+
+static GQuark
+lua_error_quark(void)
+{
+ return g_quark_from_static_string("lua-routines");
+}
+
+/*
+ * Used to map string to a pointer
+ */
+KHASH_INIT(lua_class_set, const char *, int, 1, rspamd_str_hash, rspamd_str_equal);
+struct rspamd_lua_context {
+ lua_State *L;
+ khash_t(lua_class_set) * classes;
+ struct rspamd_lua_context *prev, *next; /* Expensive but we usually have exactly one lua state */
+};
+struct rspamd_lua_context *rspamd_lua_global_ctx = NULL;
+#define RSPAMD_LUA_NCLASSES 64
+static inline struct rspamd_lua_context *
+rspamd_lua_ctx_by_state(lua_State *L)
+{
+ struct rspamd_lua_context *cur;
+
+ DL_FOREACH(rspamd_lua_global_ctx, cur)
+ {
+ if (cur->L == L) {
+ return cur;
+ }
+ }
+
+ /* When we are using thread pool, this is the case... */
+ return rspamd_lua_global_ctx;
+}
+
+/* Util functions */
+/**
+ * Create new class and store metatable on top of the stack (must be popped if not needed)
+ * @param L
+ * @param classname name of class
+ * @param func table of class methods
+ */
+void rspamd_lua_new_class(lua_State *L,
+ const gchar *classname,
+ const struct luaL_reg *methods)
+{
+ khiter_t k;
+ gint r, nmethods = 0;
+ gboolean seen_index = false;
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ if (methods) {
+ for (;;) {
+ if (methods[nmethods].name != NULL) {
+ if (strcmp(methods[nmethods].name, "__index") == 0) {
+ seen_index = true;
+ }
+ nmethods++;
+ }
+ else {
+ break;
+ }
+ }
+ }
+
+ lua_createtable(L, 0, 3 + nmethods);
+
+ if (!seen_index) {
+ lua_pushstring(L, "__index");
+ lua_pushvalue(L, -2); /* pushes the metatable */
+ lua_settable(L, -3); /* metatable.__index = metatable */
+ }
+
+ lua_pushstring(L, "class");
+ lua_pushstring(L, classname);
+ lua_rawset(L, -3);
+
+ if (methods) {
+ luaL_register(L, NULL, methods); /* pushes all methods as MT fields */
+ }
+
+ lua_pushvalue(L, -1); /* Preserves metatable */
+ int offset = luaL_ref(L, LUA_REGISTRYINDEX);
+ k = kh_put(lua_class_set, ctx->classes, classname, &r);
+ kh_value(ctx->classes, k) = offset;
+ /* MT is left on stack ! */
+}
+
+static const gchar *
+rspamd_lua_class_tostring_buf(lua_State *L, gboolean print_pointer, gint pos)
+{
+ static gchar buf[64];
+ const gchar *ret = NULL;
+ gint pop = 0;
+
+ if (!lua_getmetatable(L, pos)) {
+ goto err;
+ }
+
+ pop++;
+ lua_pushstring(L, "class");
+ lua_gettable(L, -2);
+ pop++;
+
+ if (!lua_isstring(L, -1)) {
+ goto err;
+ }
+
+ if (print_pointer) {
+ rspamd_snprintf(buf, sizeof(buf), "%s(%p)", lua_tostring(L, -1),
+ lua_touserdata(L, 1));
+ }
+ else {
+ rspamd_snprintf(buf, sizeof(buf), "%s", lua_tostring(L, -1));
+ }
+
+ ret = buf;
+
+err:
+ lua_pop(L, pop);
+
+ return ret;
+}
+
+gint rspamd_lua_class_tostring(lua_State *L)
+{
+ const gchar *p;
+
+ p = rspamd_lua_class_tostring_buf(L, TRUE, 1);
+
+ if (!p) {
+ lua_pushstring(L, "invalid object passed to 'lua_common.c:__tostring'");
+ return lua_error(L);
+ }
+
+ lua_pushstring(L, p);
+
+ return 1;
+}
+
+
+void rspamd_lua_setclass(lua_State *L, const gchar *classname, gint objidx)
+{
+ khiter_t k;
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ k = kh_get(lua_class_set, ctx->classes, classname);
+
+ g_assert(k != kh_end(ctx->classes));
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(ctx->classes, k));
+
+ if (objidx < 0) {
+ objidx--;
+ }
+ lua_setmetatable(L, objidx);
+}
+
+void rspamd_lua_class_metatable(lua_State *L, const gchar *classname)
+{
+ khiter_t k;
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ k = kh_get(lua_class_set, ctx->classes, classname);
+
+ g_assert(k != kh_end(ctx->classes));
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(ctx->classes, k));
+}
+
+void rspamd_lua_add_metamethod(lua_State *L, const gchar *classname,
+ luaL_Reg *meth)
+{
+ khiter_t k;
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ k = kh_get(lua_class_set, ctx->classes, classname);
+
+ g_assert(k != kh_end(ctx->classes));
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(ctx->classes, k));
+
+ lua_pushcfunction(L, meth->func);
+ lua_setfield(L, -2, meth->name);
+ lua_pop(L, 1); /* remove metatable */
+}
+
+/* assume that table is at the top */
+void rspamd_lua_table_set(lua_State *L, const gchar *index, const gchar *value)
+{
+ lua_pushstring(L, index);
+ if (value) {
+ lua_pushstring(L, value);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ lua_settable(L, -3);
+}
+
+const gchar *
+rspamd_lua_table_get(lua_State *L, const gchar *index)
+{
+ const gchar *result;
+
+ lua_pushstring(L, index);
+ lua_gettable(L, -2);
+ if (!lua_isstring(L, -1)) {
+ return NULL;
+ }
+ result = lua_tostring(L, -1);
+ lua_pop(L, 1);
+ return result;
+}
+
+static void
+lua_add_actions_global(lua_State *L)
+{
+ gint i;
+
+ lua_newtable(L);
+
+ for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
+ lua_pushstring(L, rspamd_action_to_str(i));
+ lua_pushinteger(L, i);
+ lua_settable(L, -3);
+ }
+ /* Set global table */
+ lua_setglobal(L, "rspamd_actions");
+}
+
+#ifndef __APPLE__
+#define OS_SO_SUFFIX ".so"
+#else
+#define OS_SO_SUFFIX ".dylib"
+#endif
+
+void rspamd_lua_set_path(lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars)
+{
+ const gchar *old_path, *additional_path = NULL;
+ const ucl_object_t *opts = NULL;
+ const gchar *rulesdir = RSPAMD_RULESDIR,
+ *lualibdir = RSPAMD_LUALIBDIR,
+ *libdir = RSPAMD_LIBDIR;
+ const gchar *t;
+
+ gchar path_buf[PATH_MAX];
+
+ lua_getglobal(L, "package");
+ lua_getfield(L, -1, "path");
+ old_path = luaL_checkstring(L, -1);
+
+ if (strstr(old_path, RSPAMD_LUALIBDIR) != NULL) {
+ /* Path has been already set, do not touch it */
+ lua_pop(L, 2);
+ return;
+ }
+
+ if (cfg_obj) {
+ opts = ucl_object_lookup(cfg_obj, "options");
+ if (opts != NULL) {
+ opts = ucl_object_lookup(opts, "lua_path");
+ if (opts != NULL && ucl_object_type(opts) == UCL_STRING) {
+ additional_path = ucl_object_tostring(opts);
+ }
+ }
+ }
+
+ if (additional_path) {
+ rspamd_snprintf(path_buf, sizeof(path_buf),
+ "%s;"
+ "%s",
+ additional_path, old_path);
+ }
+ else {
+ /* Try environment */
+ t = getenv("RULESDIR");
+ if (t) {
+ rulesdir = t;
+ }
+
+ t = getenv("LUALIBDIR");
+ if (t) {
+ lualibdir = t;
+ }
+
+ t = getenv("LIBDIR");
+ if (t) {
+ libdir = t;
+ }
+
+ t = getenv("RSPAMD_LIBDIR");
+ if (t) {
+ libdir = t;
+ }
+
+ if (vars) {
+ t = g_hash_table_lookup(vars, "RULESDIR");
+ if (t) {
+ rulesdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "LUALIBDIR");
+ if (t) {
+ lualibdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "LIBDIR");
+ if (t) {
+ libdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "RSPAMD_LIBDIR");
+ if (t) {
+ libdir = t;
+ }
+ }
+
+ rspamd_snprintf(path_buf, sizeof(path_buf),
+ "%s/lua/?.lua;"
+ "%s/?.lua;"
+ "%s/?.lua;"
+ "%s/?/init.lua;"
+ "%s",
+ RSPAMD_CONFDIR,
+ rulesdir,
+ lualibdir, lualibdir,
+ old_path);
+ }
+
+ lua_pop(L, 1);
+ lua_pushstring(L, path_buf);
+ lua_setfield(L, -2, "path");
+
+ lua_getglobal(L, "package");
+ lua_getfield(L, -1, "cpath");
+ old_path = luaL_checkstring(L, -1);
+
+ additional_path = NULL;
+
+ if (opts != NULL) {
+ opts = ucl_object_lookup(opts, "lua_cpath");
+ if (opts != NULL && ucl_object_type(opts) == UCL_STRING) {
+ additional_path = ucl_object_tostring(opts);
+ }
+ }
+
+ if (additional_path) {
+ rspamd_snprintf(path_buf, sizeof(path_buf),
+ "%s/?%s;"
+ "%s",
+ additional_path,
+ OS_SO_SUFFIX,
+ old_path);
+ }
+ else {
+ rspamd_snprintf(path_buf, sizeof(path_buf),
+ "%s/?%s;"
+ "%s",
+ libdir,
+ OS_SO_SUFFIX,
+ old_path);
+ }
+
+ lua_pop(L, 1);
+ lua_pushstring(L, path_buf);
+ lua_setfield(L, -2, "cpath");
+
+ lua_pop(L, 1);
+}
+
+static gint
+rspamd_lua_cmp_version_components(const gchar *comp1, const gchar *comp2)
+{
+ guint v1, v2;
+
+ v1 = strtoul(comp1, NULL, 10);
+ v2 = strtoul(comp2, NULL, 10);
+
+ return v1 - v2;
+}
+
+static int
+rspamd_lua_rspamd_version_cmp(lua_State *L)
+{
+ const gchar *ver;
+ gchar **components;
+ gint ret = 0;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ ver = lua_tostring(L, 2);
+
+ components = g_strsplit_set(ver, ".-_", -1);
+
+ if (!components) {
+ return luaL_error(L, "invalid arguments to 'cmp': %s", ver);
+ }
+
+ if (components[0]) {
+ ret = rspamd_lua_cmp_version_components(components[0],
+ RSPAMD_VERSION_MAJOR);
+ }
+
+ if (ret) {
+ goto set;
+ }
+
+ if (components[1]) {
+ ret = rspamd_lua_cmp_version_components(components[1],
+ RSPAMD_VERSION_MINOR);
+ }
+
+ if (ret) {
+ goto set;
+ }
+
+ /*
+ * XXX: we don't compare git releases assuming that it is meaningless
+ */
+ }
+ else {
+ return luaL_error(L, "invalid arguments to 'cmp'");
+ }
+
+set:
+ g_strfreev(components);
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static int
+rspamd_lua_rspamd_version_numeric(lua_State *L)
+{
+ static gint64 version_num = RSPAMD_VERSION_NUM;
+ const gchar *type;
+
+ if (lua_gettop(L) >= 2 && lua_type(L, 1) == LUA_TSTRING) {
+ type = lua_tostring(L, 1);
+ if (g_ascii_strcasecmp(type, "short") == 0) {
+ version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 +
+ RSPAMD_VERSION_MINOR_NUM * 100 +
+ RSPAMD_VERSION_PATCH_NUM * 10;
+ }
+ else if (g_ascii_strcasecmp(type, "main") == 0) {
+ version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 +
+ RSPAMD_VERSION_MINOR_NUM * 100 +
+ RSPAMD_VERSION_PATCH_NUM * 10;
+ }
+ else if (g_ascii_strcasecmp(type, "major") == 0) {
+ version_num = RSPAMD_VERSION_MAJOR_NUM;
+ }
+ else if (g_ascii_strcasecmp(type, "patch") == 0) {
+ version_num = RSPAMD_VERSION_PATCH_NUM;
+ }
+ else if (g_ascii_strcasecmp(type, "minor") == 0) {
+ version_num = RSPAMD_VERSION_MINOR_NUM;
+ }
+ }
+
+ lua_pushinteger(L, version_num);
+
+ return 1;
+}
+
+static int
+rspamd_lua_rspamd_version(lua_State *L)
+{
+ const gchar *result = NULL, *type;
+
+ if (lua_gettop(L) == 0) {
+ result = RVERSION;
+ }
+ else if (lua_gettop(L) >= 1 && lua_type(L, 1) == LUA_TSTRING) {
+ /* We got something like string */
+ type = lua_tostring(L, 1);
+
+ if (g_ascii_strcasecmp(type, "short") == 0) {
+ result = RSPAMD_VERSION_MAJOR
+ "." RSPAMD_VERSION_MINOR;
+ }
+ else if (g_ascii_strcasecmp(type, "main") == 0) {
+ result = RSPAMD_VERSION_MAJOR "." RSPAMD_VERSION_MINOR "." RSPAMD_VERSION_PATCH;
+ }
+ else if (g_ascii_strcasecmp(type, "major") == 0) {
+ result = RSPAMD_VERSION_MAJOR;
+ }
+ else if (g_ascii_strcasecmp(type, "minor") == 0) {
+ result = RSPAMD_VERSION_MINOR;
+ }
+ else if (g_ascii_strcasecmp(type, "patch") == 0) {
+ result = RSPAMD_VERSION_PATCH;
+ }
+ else if (g_ascii_strcasecmp(type, "id") == 0) {
+ result = RID;
+ }
+ else if (g_ascii_strcasecmp(type, "num") == 0) {
+ return rspamd_lua_rspamd_version_numeric(L);
+ }
+ else if (g_ascii_strcasecmp(type, "cmp") == 0) {
+ return rspamd_lua_rspamd_version_cmp(L);
+ }
+ }
+
+ lua_pushstring(L, result);
+
+ return 1;
+}
+
+static gboolean
+rspamd_lua_load_env(lua_State *L, const char *fname, gint tbl_pos, GError **err)
+{
+ gint orig_top = lua_gettop(L), err_idx;
+ gboolean ret = TRUE;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ if (luaL_loadfile(L, fname) != 0) {
+ g_set_error(err, g_quark_from_static_string("lua_env"), errno,
+ "cannot load lua file %s: %s",
+ fname,
+ lua_tostring(L, -1));
+ ret = FALSE;
+ }
+
+ if (ret && lua_pcall(L, 0, 1, err_idx) != 0) {
+ g_set_error(err, g_quark_from_static_string("lua_env"), errno,
+ "cannot init lua file %s: %s",
+ fname,
+ lua_tostring(L, -1));
+ ret = FALSE;
+ }
+
+ if (ret && lua_type(L, -1) == LUA_TTABLE) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ lua_pushvalue(L, -2); /* Store key */
+ lua_pushvalue(L, -2); /* Store value */
+ lua_settable(L, tbl_pos);
+ }
+ }
+ else if (ret) {
+ g_set_error(err, g_quark_from_static_string("lua_env"), errno,
+ "invalid return type when loading env from %s: %s",
+ fname,
+ lua_typename(L, lua_type(L, -1)));
+ ret = FALSE;
+ }
+
+ lua_settop(L, orig_top);
+
+ return ret;
+}
+
+gboolean
+rspamd_lua_set_env(lua_State *L, GHashTable *vars, char **lua_env, GError **err)
+{
+ gint orig_top = lua_gettop(L);
+ gchar **env = g_get_environ();
+
+ /* Set known paths as rspamd_paths global */
+ lua_getglobal(L, "rspamd_paths");
+ if (lua_isnil(L, -1)) {
+ const gchar *confdir = RSPAMD_CONFDIR,
+ *local_confdir = RSPAMD_LOCAL_CONFDIR,
+ *rundir = RSPAMD_RUNDIR,
+ *dbdir = RSPAMD_DBDIR,
+ *logdir = RSPAMD_LOGDIR,
+ *wwwdir = RSPAMD_WWWDIR,
+ *pluginsdir = RSPAMD_PLUGINSDIR,
+ *rulesdir = RSPAMD_RULESDIR,
+ *lualibdir = RSPAMD_LUALIBDIR,
+ *prefix = RSPAMD_PREFIX,
+ *sharedir = RSPAMD_SHAREDIR;
+ const gchar *t;
+
+ /* Try environment */
+ t = g_environ_getenv(env, "SHAREDIR");
+ if (t) {
+ sharedir = t;
+ }
+
+ t = g_environ_getenv(env, "PLUGINSDIR");
+ if (t) {
+ pluginsdir = t;
+ }
+
+ t = g_environ_getenv(env, "RULESDIR");
+ if (t) {
+ rulesdir = t;
+ }
+
+ t = g_environ_getenv(env, "DBDIR");
+ if (t) {
+ dbdir = t;
+ }
+
+ t = g_environ_getenv(env, "RUNDIR");
+ if (t) {
+ rundir = t;
+ }
+
+ t = g_environ_getenv(env, "LUALIBDIR");
+ if (t) {
+ lualibdir = t;
+ }
+
+ t = g_environ_getenv(env, "LOGDIR");
+ if (t) {
+ logdir = t;
+ }
+
+ t = g_environ_getenv(env, "WWWDIR");
+ if (t) {
+ wwwdir = t;
+ }
+
+ t = g_environ_getenv(env, "CONFDIR");
+ if (t) {
+ confdir = t;
+ }
+
+ t = g_environ_getenv(env, "LOCAL_CONFDIR");
+ if (t) {
+ local_confdir = t;
+ }
+
+
+ if (vars) {
+ t = g_hash_table_lookup(vars, "SHAREDIR");
+ if (t) {
+ sharedir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "PLUGINSDIR");
+ if (t) {
+ pluginsdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "RULESDIR");
+ if (t) {
+ rulesdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "LUALIBDIR");
+ if (t) {
+ lualibdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "RUNDIR");
+ if (t) {
+ rundir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "WWWDIR");
+ if (t) {
+ wwwdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "CONFDIR");
+ if (t) {
+ confdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "LOCAL_CONFDIR");
+ if (t) {
+ local_confdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "DBDIR");
+ if (t) {
+ dbdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "LOGDIR");
+ if (t) {
+ logdir = t;
+ }
+ }
+
+ lua_createtable(L, 0, 9);
+
+ rspamd_lua_table_set(L, RSPAMD_SHAREDIR_INDEX, sharedir);
+ rspamd_lua_table_set(L, RSPAMD_CONFDIR_INDEX, confdir);
+ rspamd_lua_table_set(L, RSPAMD_LOCAL_CONFDIR_INDEX, local_confdir);
+ rspamd_lua_table_set(L, RSPAMD_RUNDIR_INDEX, rundir);
+ rspamd_lua_table_set(L, RSPAMD_DBDIR_INDEX, dbdir);
+ rspamd_lua_table_set(L, RSPAMD_LOGDIR_INDEX, logdir);
+ rspamd_lua_table_set(L, RSPAMD_WWWDIR_INDEX, wwwdir);
+ rspamd_lua_table_set(L, RSPAMD_PLUGINSDIR_INDEX, pluginsdir);
+ rspamd_lua_table_set(L, RSPAMD_RULESDIR_INDEX, rulesdir);
+ rspamd_lua_table_set(L, RSPAMD_LUALIBDIR_INDEX, lualibdir);
+ rspamd_lua_table_set(L, RSPAMD_PREFIX_INDEX, prefix);
+
+ lua_setglobal(L, "rspamd_paths");
+ }
+
+ lua_getglobal(L, "rspamd_env");
+ if (lua_isnil(L, -1)) {
+ lua_newtable(L);
+
+ if (vars != NULL) {
+ GHashTableIter it;
+ gpointer k, v;
+
+ g_hash_table_iter_init(&it, vars);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ rspamd_lua_table_set(L, k, v);
+ }
+ }
+
+ gint hostlen = sysconf(_SC_HOST_NAME_MAX);
+
+ if (hostlen <= 0) {
+ hostlen = 256;
+ }
+ else {
+ hostlen++;
+ }
+
+ gchar *hostbuf = g_alloca(hostlen);
+ memset(hostbuf, 0, hostlen);
+ gethostname(hostbuf, hostlen - 1);
+
+ rspamd_lua_table_set(L, "hostname", hostbuf);
+
+ rspamd_lua_table_set(L, "version", RVERSION);
+ rspamd_lua_table_set(L, "ver_major", RSPAMD_VERSION_MAJOR);
+ rspamd_lua_table_set(L, "ver_minor", RSPAMD_VERSION_MINOR);
+ rspamd_lua_table_set(L, "ver_id", RID);
+ lua_pushstring(L, "ver_num");
+ lua_pushinteger(L, RSPAMD_VERSION_NUM);
+ lua_settable(L, -3);
+
+ if (env) {
+ gint lim = g_strv_length(env);
+
+ for (gint i = 0; i < lim; i++) {
+ if (RSPAMD_LEN_CHECK_STARTS_WITH(env[i], strlen(env[i]), "RSPAMD_")) {
+ const char *var = env[i] + sizeof("RSPAMD_") - 1, *value;
+ gint varlen;
+
+ varlen = strcspn(var, "=");
+ value = var + varlen;
+
+ if (*value == '=') {
+ value++;
+
+ lua_pushlstring(L, var, varlen);
+ lua_pushstring(L, value);
+ lua_settable(L, -3);
+ }
+ }
+ }
+ }
+
+ if (lua_env) {
+ gint lim = g_strv_length(lua_env);
+
+ for (gint i = 0; i < lim; i++) {
+ if (!rspamd_lua_load_env(L, lua_env[i], lua_gettop(L), err)) {
+ return FALSE;
+ }
+ }
+ }
+
+ lua_setglobal(L, "rspamd_env");
+ }
+
+ lua_settop(L, orig_top);
+ g_strfreev(env);
+
+ return TRUE;
+}
+
+void rspamd_lua_set_globals(struct rspamd_config *cfg, lua_State *L)
+{
+ struct rspamd_config **pcfg;
+ gint orig_top = lua_gettop(L);
+
+ /* First check for global variable 'config' */
+ lua_getglobal(L, "config");
+ if (lua_isnil(L, -1)) {
+ /* Assign global table to set up attributes */
+ lua_newtable(L);
+ lua_setglobal(L, "config");
+ }
+
+ lua_getglobal(L, "metrics");
+ if (lua_isnil(L, -1)) {
+ lua_newtable(L);
+ lua_setglobal(L, "metrics");
+ }
+
+ lua_getglobal(L, "composites");
+ if (lua_isnil(L, -1)) {
+ lua_newtable(L);
+ lua_setglobal(L, "composites");
+ }
+
+ lua_getglobal(L, "rspamd_classifiers");
+ if (lua_isnil(L, -1)) {
+ lua_newtable(L);
+ lua_setglobal(L, "rspamd_classifiers");
+ }
+
+ lua_getglobal(L, "classifiers");
+ if (lua_isnil(L, -1)) {
+ lua_newtable(L);
+ lua_setglobal(L, "classifiers");
+ }
+
+ lua_getglobal(L, "rspamd_version");
+ if (lua_isnil(L, -1)) {
+ lua_pushcfunction(L, rspamd_lua_rspamd_version);
+ lua_setglobal(L, "rspamd_version");
+ }
+
+ if (cfg != NULL) {
+ pcfg = lua_newuserdata(L, sizeof(struct rspamd_config *));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ lua_setglobal(L, "rspamd_config");
+ }
+
+ lua_settop(L, orig_top);
+}
+
+#ifdef WITH_LUA_TRACE
+static gint
+lua_push_trace_data(lua_State *L)
+{
+ if (lua_traces) {
+ ucl_object_push_lua(L, lua_traces, true);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+#endif
+
+
+static void *
+rspamd_lua_wipe_realloc(void *ud,
+ void *ptr,
+ size_t osize,
+ size_t nsize) RSPAMD_ATTR_ALLOC_SIZE(4);
+static void *
+rspamd_lua_wipe_realloc(void *ud,
+ void *ptr,
+ size_t osize,
+ size_t nsize)
+{
+ if (nsize == 0) {
+ if (ptr) {
+ rspamd_explicit_memzero(ptr, osize);
+ }
+
+ free(ptr);
+ }
+ else if (ptr == NULL) {
+ return malloc(nsize);
+ }
+ else {
+ if (nsize < osize) {
+ /* Wipe on shrinking (actually never used) */
+ rspamd_explicit_memzero(((unsigned char *) ptr) + nsize, osize - nsize);
+ }
+
+ return realloc(ptr, nsize);
+ }
+
+ return NULL;
+}
+
+#ifndef WITH_LUAJIT
+extern int luaopen_bit(lua_State *L);
+#endif
+
+static unsigned int lua_initialized = 0;
+
+lua_State *
+rspamd_lua_init(bool wipe_mem)
+{
+ lua_State *L;
+
+ if (wipe_mem) {
+#ifdef WITH_LUAJIT
+ /* TODO: broken on luajit without GC64 */
+ L = luaL_newstate();
+#else
+ L = lua_newstate(rspamd_lua_wipe_realloc, NULL);
+#endif
+ }
+ else {
+ L = luaL_newstate();
+ }
+
+ struct rspamd_lua_context *ctx;
+
+ ctx = (struct rspamd_lua_context *) g_malloc0(sizeof(*ctx));
+ ctx->L = L;
+ ctx->classes = kh_init(lua_class_set);
+ kh_resize(lua_class_set, ctx->classes, RSPAMD_LUA_NCLASSES);
+ DL_APPEND(rspamd_lua_global_ctx, ctx);
+
+ lua_gc(L, LUA_GCSTOP, 0);
+ luaL_openlibs(L);
+ luaopen_logger(L);
+ luaopen_mempool(L);
+ luaopen_config(L);
+ luaopen_map(L);
+ luaopen_trie(L);
+ luaopen_task(L);
+ luaopen_textpart(L);
+ luaopen_mimepart(L);
+ luaopen_image(L);
+ luaopen_url(L);
+ luaopen_classifier(L);
+ luaopen_statfile(L);
+ luaopen_regexp(L);
+ luaopen_cdb(L);
+ luaopen_xmlrpc(L);
+ luaopen_http(L);
+ luaopen_redis(L);
+ luaopen_upstream(L);
+ lua_add_actions_global(L);
+ luaopen_dns_resolver(L);
+ luaopen_rsa(L);
+ luaopen_ip(L);
+ luaopen_expression(L);
+ luaopen_text(L);
+ luaopen_util(L);
+ luaopen_tcp(L);
+ luaopen_html(L);
+ luaopen_sqlite3(L);
+ luaopen_cryptobox(L);
+ luaopen_dns(L);
+ luaopen_udp(L);
+ luaopen_worker(L);
+ luaopen_kann(L);
+ luaopen_spf(L);
+ luaopen_tensor(L);
+ luaopen_parsers(L);
+ luaopen_compress(L);
+#ifndef WITH_LUAJIT
+ rspamd_lua_add_preload(L, "bit", luaopen_bit);
+ lua_settop(L, 0);
+#endif
+
+ rspamd_lua_new_class(L, "rspamd{session}", NULL);
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "lpeg", luaopen_lpeg);
+ luaopen_ucl(L);
+ rspamd_lua_add_preload(L, "ucl", luaopen_ucl);
+
+ /* Add plugins global */
+ lua_newtable(L);
+ lua_setglobal(L, "rspamd_plugins");
+
+ /* Set PRNG */
+ lua_getglobal(L, "math");
+ lua_pushstring(L, "randomseed"); /* Push math.randomseed function on top of the stack */
+ lua_gettable(L, -2);
+ lua_pushinteger(L, ottery_rand_uint64());
+ g_assert(lua_pcall(L, 1, 0, 0) == 0);
+ lua_pop(L, 1); /* math table */
+
+ /* Modules state */
+ lua_newtable(L);
+ /*
+ * rspamd_plugins_state = {
+ * enabled = {},
+ * disabled_unconfigured = {},
+ * disabled_redis = {},
+ * disabled_explicitly = {},
+ * disabled_failed = {},
+ * disabled_experimental = {},
+ * disabled_unknown = {},
+ * }
+ */
+#define ADD_TABLE(name) \
+ do { \
+ lua_pushstring(L, #name); \
+ lua_newtable(L); \
+ lua_settable(L, -3); \
+ } while (0)
+
+ ADD_TABLE(enabled);
+ ADD_TABLE(disabled_unconfigured);
+ ADD_TABLE(disabled_redis);
+ ADD_TABLE(disabled_explicitly);
+ ADD_TABLE(disabled_failed);
+ ADD_TABLE(disabled_experimental);
+ ADD_TABLE(disabled_unknown);
+
+#undef ADD_TABLE
+ lua_setglobal(L, rspamd_modules_state_global);
+
+#ifdef WITH_LUA_TRACE
+ lua_pushcfunction(L, lua_push_trace_data);
+ lua_setglobal(L, "get_traces");
+#endif
+
+ lua_initialized++;
+
+ return L;
+}
+
+void rspamd_lua_close(lua_State *L)
+{
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ /* TODO: we will leak this memory, but I don't know how to resolve
+ * the chicked-egg problem when lua_close calls GC for many
+ * userdata that requires classes metatables to be represented
+ * For now, it is safe to leave it as is, I'm afraid
+ */
+#if 0
+ int ref;
+ kh_foreach_value(ctx->classes, ref, {
+ luaL_unref(L, LUA_REGISTRYINDEX, ref);
+ });
+#endif
+
+ lua_close(L);
+ DL_DELETE(rspamd_lua_global_ctx, ctx);
+ kh_destroy(lua_class_set, ctx->classes);
+ g_free(ctx);
+
+ lua_initialized--;
+}
+
+bool rspamd_lua_is_initialised(void)
+{
+ return lua_initialized != 0;
+}
+
+void rspamd_lua_start_gc(struct rspamd_config *cfg)
+{
+ lua_State *L = (lua_State *) cfg->lua_state;
+
+ lua_settop(L, 0);
+ /* Set up GC */
+ lua_gc(L, LUA_GCCOLLECT, 0);
+ lua_gc(L, LUA_GCSETSTEPMUL, cfg->lua_gc_step);
+ lua_gc(L, LUA_GCSETPAUSE, cfg->lua_gc_pause);
+ lua_gc(L, LUA_GCRESTART, 0);
+}
+
+
+void rspamd_plugins_table_push_elt(lua_State *L, const gchar *field_name,
+ const gchar *new_elt)
+{
+ lua_getglobal(L, rspamd_modules_state_global);
+
+ if (lua_istable(L, -1)) {
+ lua_pushstring(L, field_name);
+ lua_gettable(L, -2);
+
+ if (lua_istable(L, -1)) {
+ lua_pushstring(L, new_elt);
+ lua_newtable(L);
+ lua_settable(L, -3);
+ lua_pop(L, 2); /* Global + element */
+ }
+ else {
+ lua_pop(L, 2); /* Global + element */
+ }
+ }
+ else {
+ lua_pop(L, 1);
+ }
+}
+
+gboolean
+rspamd_init_lua_filters(struct rspamd_config *cfg, bool force_load, bool strict)
+{
+ struct rspamd_config **pcfg;
+ struct script_module *module;
+ lua_State *L = cfg->lua_state;
+ gint err_idx, i;
+
+ pcfg = lua_newuserdata(L, sizeof(struct rspamd_config *));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ lua_setglobal(L, "rspamd_config");
+
+ PTR_ARRAY_FOREACH(cfg->script_modules, i, module)
+ {
+ if (module->path) {
+ if (!force_load) {
+ if (!rspamd_config_is_module_enabled(cfg, module->name)) {
+ continue;
+ }
+ }
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ gsize fsize;
+ guint8 *data = rspamd_file_xmap(module->path,
+ PROT_READ, &fsize, TRUE);
+ guchar digest[rspamd_cryptobox_HASHBYTES];
+ gchar *lua_fname;
+
+ if (data == NULL) {
+ msg_err_config("cannot mmap %s failed: %s", module->path,
+ strerror(errno));
+
+ lua_settop(L, err_idx - 1); /* Error function */
+
+ rspamd_plugins_table_push_elt(L, "disabled_failed",
+ module->name);
+
+ if (strict) {
+ return FALSE;
+ }
+
+ continue;
+ }
+
+ module->digest = rspamd_mempool_alloc(cfg->cfg_pool,
+ rspamd_cryptobox_HASHBYTES * 2 + 1);
+ rspamd_cryptobox_hash(digest, data, fsize, NULL, 0);
+ rspamd_encode_hex_buf(digest, sizeof(digest),
+ module->digest, rspamd_cryptobox_HASHBYTES * 2 + 1);
+ module->digest[rspamd_cryptobox_HASHBYTES * 2] = '\0';
+ lua_fname = g_malloc(strlen(module->path) + 2);
+ rspamd_snprintf(lua_fname, strlen(module->path) + 2, "@%s",
+ module->path);
+
+ if (luaL_loadbuffer(L, data, fsize, lua_fname) != 0) {
+ msg_err_config("load of %s failed: %s", module->path,
+ lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1); /* Error function */
+
+ rspamd_plugins_table_push_elt(L, "disabled_failed",
+ module->name);
+ munmap(data, fsize);
+ g_free(lua_fname);
+
+ if (strict) {
+ return FALSE;
+ }
+
+ continue;
+ }
+
+ munmap(data, fsize);
+ g_free(lua_fname);
+
+ if (lua_pcall(L, 0, 0, err_idx) != 0) {
+ msg_err_config("init of %s failed: %s",
+ module->path,
+ lua_tostring(L, -1));
+
+ lua_settop(L, err_idx - 1);
+ rspamd_plugins_table_push_elt(L, "disabled_failed",
+ module->name);
+
+ if (strict) {
+ return FALSE;
+ }
+
+ continue;
+ }
+
+ if (!force_load) {
+ msg_info_config("init lua module %s from %s; digest: %*s",
+ module->name,
+ module->path,
+ 10, module->digest);
+ }
+
+ lua_pop(L, 1); /* Error function */
+ }
+ }
+
+ return TRUE;
+}
+
+void rspamd_lua_dumpstack(lua_State *L)
+{
+ gint i, t, r = 0;
+ gint top = lua_gettop(L);
+ gchar buf[BUFSIZ];
+
+ r += rspamd_snprintf(buf + r, sizeof(buf) - r, "lua stack: ");
+ for (i = 1; i <= top; i++) { /* repeat for each level */
+ t = lua_type(L, i);
+ switch (t) {
+ case LUA_TSTRING: /* strings */
+ r += rspamd_snprintf(buf + r,
+ sizeof(buf) - r,
+ "str: %s",
+ lua_tostring(L, i));
+ break;
+
+ case LUA_TBOOLEAN: /* booleans */
+ r += rspamd_snprintf(buf + r, sizeof(buf) - r,
+ lua_toboolean(L, i) ? "bool: true" : "bool: false");
+ break;
+
+ case LUA_TNUMBER: /* numbers */
+ r += rspamd_snprintf(buf + r,
+ sizeof(buf) - r,
+ "number: %.2f",
+ lua_tonumber(L, i));
+ break;
+
+ default: /* other values */
+ r += rspamd_snprintf(buf + r,
+ sizeof(buf) - r,
+ "type: %s",
+ lua_typename(L, t));
+ break;
+ }
+ if (i < top) {
+ r += rspamd_snprintf(buf + r, sizeof(buf) - r,
+ " -> "); /* put a separator */
+ }
+ }
+
+ msg_info("%*s", r, buf);
+}
+
+gpointer
+rspamd_lua_check_class(lua_State *L, gint index, const gchar *name)
+{
+ gpointer p;
+ khiter_t k;
+
+ if (lua_type(L, index) == LUA_TUSERDATA) {
+ p = lua_touserdata(L, index);
+ if (p) {
+ if (lua_getmetatable(L, index)) {
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ k = kh_get(lua_class_set, ctx->classes, name);
+
+ if (k == kh_end(ctx->classes)) {
+ lua_pop(L, 1);
+
+ return NULL;
+ }
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(ctx->classes, k));
+
+ if (lua_rawequal(L, -1, -2)) { /* does it have the correct mt? */
+ lua_pop(L, 2); /* remove both metatables */
+ return p;
+ }
+ lua_pop(L, 2);
+ }
+ }
+ }
+ return NULL;
+}
+
+int rspamd_lua_typerror(lua_State *L, int narg, const char *tname)
+{
+ const char *msg = lua_pushfstring(L, "%s expected, got %s", tname,
+ luaL_typename(L, narg));
+ return luaL_argerror(L, narg, msg);
+}
+
+
+void rspamd_lua_add_preload(lua_State *L, const gchar *name, lua_CFunction func)
+{
+ lua_getglobal(L, "package");
+ lua_pushstring(L, "preload");
+ lua_gettable(L, -2);
+ lua_pushcfunction(L, func);
+ lua_setfield(L, -2, name);
+ lua_pop(L, 2); /* preload key + global package */
+}
+
+
+gboolean
+rspamd_lua_parse_table_arguments(lua_State *L, gint pos,
+ GError **err,
+ enum rspamd_lua_parse_arguments_flags how,
+ const gchar *extraction_pattern, ...)
+{
+ const gchar *p, *key = NULL, *end, *cls;
+ va_list ap;
+ gboolean required = FALSE, failed = FALSE, is_table;
+ gchar classbuf[128];
+ enum {
+ read_key = 0,
+ read_arg,
+ read_class_start,
+ read_class,
+ read_semicolon
+ } state = read_key;
+ gsize keylen = 0, *valuelen, clslen;
+ gint idx = 0, t, direct_userdata = 0;
+
+ g_assert(extraction_pattern != NULL);
+
+ if (pos < 0) {
+ /* Get absolute pos */
+ pos = lua_gettop(L) + pos + 1;
+ }
+
+ if (lua_type(L, pos) == LUA_TTABLE) {
+ is_table = TRUE;
+ }
+ else {
+ is_table = FALSE;
+ idx = pos;
+ }
+
+ p = extraction_pattern;
+ end = p + strlen(extraction_pattern);
+
+ va_start(ap, extraction_pattern);
+
+ while (p <= end) {
+ switch (state) {
+ case read_key:
+ if (*p == '=') {
+ if (key == NULL) {
+ g_set_error(err, lua_error_quark(), 1, "cannot read key");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ state = read_arg;
+ keylen = p - key;
+ }
+ else if (*p == '*' && key == NULL) {
+ required = TRUE;
+ }
+ else if (key == NULL) {
+ key = p;
+ }
+ p++;
+ break;
+ case read_arg:
+ g_assert(keylen != 0);
+
+ if (is_table) {
+ lua_pushlstring(L, key, keylen);
+ lua_gettable(L, pos);
+ idx = -1;
+ }
+
+ t = lua_type(L, idx);
+
+ switch (*p) {
+ case 'S':
+ if (t == LUA_TSTRING) {
+ *(va_arg(ap, const gchar **)) = lua_tostring(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, const gchar **)) = NULL;
+ }
+ else {
+ (void) va_arg(ap, gchar **);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)), "string");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'I':
+ if (t == LUA_TNUMBER) {
+ *(va_arg(ap, gint64 *)) = lua_tointeger(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gint64 *)) = 0;
+ }
+ else {
+ (void) va_arg(ap, gint64 *);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "int64");
+ va_end(ap);
+
+ return FALSE;
+ }
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'i':
+ if (t == LUA_TNUMBER) {
+ *(va_arg(ap, gint32 *)) = lua_tointeger(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gint32 *)) = 0;
+ }
+ else {
+ (void) va_arg(ap, gint32 *);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "int64");
+ va_end(ap);
+
+ return FALSE;
+ }
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'F':
+ if (t == LUA_TFUNCTION) {
+ if (!is_table) {
+ lua_pushvalue(L, idx);
+ }
+
+ *(va_arg(ap, gint *)) = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gint *)) = -1;
+ }
+ else {
+ (void) va_arg(ap, gint *);
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "function");
+ va_end(ap);
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+
+ return FALSE;
+ }
+
+ /* luaL_ref pops argument from the stack */
+ break;
+
+ case 'B':
+ if (t == LUA_TBOOLEAN) {
+ *(va_arg(ap, gboolean *)) = lua_toboolean(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gboolean *)) = 0;
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "bool");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'N':
+ if (t == LUA_TNUMBER) {
+ *(va_arg(ap, gdouble *)) = lua_tonumber(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gdouble *)) = 0;
+ }
+ else {
+ (void) va_arg(ap, gdouble *);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "double");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'D':
+ if (t == LUA_TNUMBER) {
+ *(va_arg(ap, gdouble *)) = lua_tonumber(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gdouble *)) = NAN;
+ }
+ else {
+ (void) va_arg(ap, gdouble *);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "double");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'V':
+ valuelen = va_arg(ap, gsize *);
+
+ if (t == LUA_TSTRING) {
+ *(va_arg(ap, const gchar **)) = lua_tolstring(L, idx,
+ valuelen);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, const char **)) = NULL;
+ *valuelen = 0;
+ }
+ else {
+ (void) va_arg(ap, const char **);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "string");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+ case 'O':
+ if (t != LUA_TNONE) {
+ *(va_arg(ap, ucl_object_t **)) = ucl_object_lua_import(L,
+ idx);
+ }
+ else {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, ucl_object_t **)) = NULL;
+ }
+ else {
+ (void) va_arg(ap, ucl_object_t **);
+ }
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+ case 'U':
+ if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, void **)) = NULL;
+ }
+ else {
+ (void) va_arg(ap, void **);
+ }
+ }
+ else if (t != LUA_TUSERDATA) {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "int64");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ state = read_class_start;
+ clslen = 0;
+ direct_userdata = 0;
+ cls = NULL;
+ p++;
+ continue;
+ case 'u':
+ if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, void **)) = NULL;
+ }
+ else {
+ (void) va_arg(ap, void **);
+ }
+ }
+ else if (t != LUA_TUSERDATA) {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "int64");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ state = read_class_start;
+ clslen = 0;
+ direct_userdata = 1;
+ cls = NULL;
+ p++;
+ continue;
+ default:
+ g_assert(0);
+ break;
+ }
+
+ if (failed && required) {
+ g_set_error(err, lua_error_quark(), 2, "required parameter "
+ "%.*s is missing",
+ (gint) keylen, key);
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (!is_table) {
+ idx++;
+ }
+
+ /* Reset read params */
+ state = read_semicolon;
+ failed = FALSE;
+ required = FALSE;
+ keylen = 0;
+ key = NULL;
+ p++;
+ break;
+
+ case read_class_start:
+ if (*p == '{') {
+ cls = p + 1;
+ state = read_class;
+ }
+ else {
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+
+ g_set_error(err, lua_error_quark(), 2, "missing classname for "
+ "%.*s",
+ (gint) keylen, key);
+ va_end(ap);
+
+ return FALSE;
+ }
+ p++;
+ break;
+
+ case read_class:
+ if (*p == '}') {
+ clslen = p - cls;
+ if (clslen == 0) {
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+
+ g_set_error(err,
+ lua_error_quark(),
+ 2,
+ "empty classname for "
+ "%*.s",
+ (gint) keylen,
+ key);
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ rspamd_snprintf(classbuf, sizeof(classbuf), "rspamd{%*s}",
+ (gint) clslen, cls);
+
+
+ /*
+ * We skip class check here for speed in non-table mode
+ */
+ if (!failed && (!is_table ||
+ rspamd_lua_check_class(L, idx, classbuf))) {
+ if (direct_userdata) {
+ void **arg_p = (va_arg(ap, void **));
+ *arg_p = lua_touserdata(L, idx);
+ }
+ else {
+ *(va_arg(ap,
+ void **)) = *(void **) lua_touserdata(L, idx);
+ }
+ }
+ else {
+ if (!failed) {
+ g_set_error(err,
+ lua_error_quark(),
+ 2,
+ "invalid class for key %.*s, expected %s, got %s",
+ (gint) keylen,
+ key,
+ classbuf,
+ rspamd_lua_class_tostring_buf(L, FALSE, idx));
+ va_end(ap);
+
+ return FALSE;
+ }
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ else {
+ idx++;
+ }
+
+ if (failed && required) {
+ g_set_error(err,
+ lua_error_quark(),
+ 2,
+ "required parameter "
+ "%.*s is missing",
+ (gint) keylen,
+ key);
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ /* Reset read params */
+ state = read_semicolon;
+ failed = FALSE;
+ required = FALSE;
+ keylen = 0;
+ key = NULL;
+ }
+ p++;
+ break;
+
+ case read_semicolon:
+ if (*p == ';' || p == end) {
+ state = read_key;
+ key = NULL;
+ keylen = 0;
+ failed = FALSE;
+ }
+ else {
+ g_set_error(err, lua_error_quark(), 2, "bad format string: %s,"
+ " at char %c, position %d",
+ extraction_pattern, *p, (int) (p - extraction_pattern));
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ p++;
+ break;
+ }
+ }
+
+ va_end(ap);
+
+ return TRUE;
+}
+
+static void
+rspamd_lua_traceback_string(lua_State *L, luaL_Buffer *buf)
+{
+ gint i = 1, r;
+ lua_Debug d;
+ gchar tmp[256];
+
+ while (lua_getstack(L, i++, &d)) {
+ lua_getinfo(L, "nSl", &d);
+ r = rspamd_snprintf(tmp, sizeof(tmp), " [%d]:{%s:%d - %s [%s]};",
+ i - 1, d.short_src, d.currentline,
+ (d.name ? d.name : "<unknown>"), d.what);
+ luaL_addlstring(buf, tmp, r);
+ }
+}
+
+gint rspamd_lua_traceback(lua_State *L)
+{
+ luaL_Buffer b;
+
+ luaL_buffinit(L, &b);
+ rspamd_lua_get_traceback_string(L, &b);
+ luaL_pushresult(&b);
+
+ return 1;
+}
+
+void rspamd_lua_get_traceback_string(lua_State *L, luaL_Buffer *buf)
+{
+ const gchar *msg = lua_tostring(L, -1);
+
+ if (msg) {
+ luaL_addstring(buf, msg);
+ lua_pop(L, 1); /* Error string */
+ }
+ else {
+ luaL_addstring(buf, "unknown error");
+ }
+
+ luaL_addstring(buf, "; trace:");
+ rspamd_lua_traceback_string(L, buf);
+}
+
+guint rspamd_lua_table_size(lua_State *L, gint tbl_pos)
+{
+ guint tbl_size = 0;
+
+ if (!lua_istable(L, tbl_pos)) {
+ return 0;
+ }
+
+#if LUA_VERSION_NUM >= 502
+ tbl_size = lua_rawlen(L, tbl_pos);
+#else
+ tbl_size = lua_objlen(L, tbl_pos);
+#endif
+
+ return tbl_size;
+}
+
+static void *
+rspamd_lua_check_udata_common(lua_State *L, gint pos, const gchar *classname,
+ gboolean fatal)
+{
+ void *p = lua_touserdata(L, pos);
+ guint i, top = lua_gettop(L);
+ khiter_t k;
+
+ if (p == NULL) {
+ goto err;
+ }
+ else {
+ /* Match class */
+ if (lua_getmetatable(L, pos)) {
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ k = kh_get(lua_class_set, ctx->classes, classname);
+
+ if (k == kh_end(ctx->classes)) {
+ goto err;
+ }
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(ctx->classes, k));
+
+ if (!lua_rawequal(L, -1, -2)) {
+ goto err;
+ }
+ }
+ else {
+ goto err;
+ }
+ }
+
+ lua_settop(L, top);
+
+ return p;
+
+err:
+ if (fatal) {
+ const gchar *actual_classname = NULL;
+
+ if (lua_type(L, pos) == LUA_TUSERDATA && lua_getmetatable(L, pos)) {
+ lua_pushstring(L, "__index");
+ lua_gettable(L, -2);
+ lua_pushstring(L, "class");
+ lua_gettable(L, -2);
+ actual_classname = lua_tostring(L, -1);
+ }
+ else {
+ actual_classname = lua_typename(L, lua_type(L, pos));
+ }
+
+ luaL_Buffer buf;
+ gchar tmp[512];
+ gint r;
+
+ luaL_buffinit(L, &buf);
+ r = rspamd_snprintf(tmp, sizeof(tmp),
+ "expected %s at position %d, but userdata has "
+ "%s metatable; trace: ",
+ classname, pos, actual_classname);
+ luaL_addlstring(&buf, tmp, r);
+ rspamd_lua_traceback_string(L, &buf);
+ r = rspamd_snprintf(tmp, sizeof(tmp), " stack(%d): ", top);
+ luaL_addlstring(&buf, tmp, r);
+
+ for (i = 1; i <= MIN(top, 10); i++) {
+ if (lua_type(L, i) == LUA_TUSERDATA) {
+ const char *clsname;
+
+ if (lua_getmetatable(L, i)) {
+ lua_pushstring(L, "__index");
+ lua_gettable(L, -2);
+ lua_pushstring(L, "class");
+ lua_gettable(L, -2);
+ clsname = lua_tostring(L, -1);
+ }
+ else {
+ clsname = lua_typename(L, lua_type(L, i));
+ }
+
+ r = rspamd_snprintf(tmp, sizeof(tmp), "[%d: ud=%s] ", i,
+ clsname);
+ luaL_addlstring(&buf, tmp, r);
+ }
+ else {
+ r = rspamd_snprintf(tmp, sizeof(tmp), "[%d: %s] ", i,
+ lua_typename(L, lua_type(L, i)));
+ luaL_addlstring(&buf, tmp, r);
+ }
+ }
+
+ luaL_pushresult(&buf);
+ msg_err("lua type error: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+
+ return NULL;
+}
+
+void *
+rspamd_lua_check_udata(lua_State *L, gint pos, const gchar *classname)
+{
+ return rspamd_lua_check_udata_common(L, pos, classname, TRUE);
+}
+
+void *
+rspamd_lua_check_udata_maybe(lua_State *L, gint pos, const gchar *classname)
+{
+ return rspamd_lua_check_udata_common(L, pos, classname, FALSE);
+}
+
+struct rspamd_async_session *
+lua_check_session(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{session}");
+ luaL_argcheck(L, ud != NULL, pos, "'session' expected");
+ return ud ? *((struct rspamd_async_session **) ud) : NULL;
+}
+
+struct ev_loop *
+lua_check_ev_base(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{ev_base}");
+ luaL_argcheck(L, ud != NULL, pos, "'event_base' expected");
+ return ud ? *((struct ev_loop **) ud) : NULL;
+}
+
+static void rspamd_lua_run_postloads_error(struct thread_entry *thread, int ret, const char *msg);
+
+void rspamd_lua_run_postloads(lua_State *L, struct rspamd_config *cfg,
+ struct ev_loop *ev_base, struct rspamd_worker *w)
+{
+ struct rspamd_config_cfg_lua_script *sc;
+ struct rspamd_config **pcfg;
+ struct ev_loop **pev_base;
+ struct rspamd_worker **pw;
+
+ /* Execute post load scripts */
+ LL_FOREACH(cfg->on_load_scripts, sc)
+ {
+ struct thread_entry *thread = lua_thread_pool_get_for_config(cfg);
+ thread->error_callback = rspamd_lua_run_postloads_error;
+ L = thread->lua_state;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, sc->cbref);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ *pcfg = cfg;
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+
+ pev_base = lua_newuserdata(L, sizeof(*pev_base));
+ *pev_base = ev_base;
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+
+ pw = lua_newuserdata(L, sizeof(*pw));
+ *pw = w;
+ rspamd_lua_setclass(L, "rspamd{worker}", -1);
+
+ lua_thread_call(thread, 3);
+ }
+}
+
+
+void rspamd_lua_run_config_post_init(lua_State *L, struct rspamd_config *cfg)
+{
+ struct rspamd_config_cfg_lua_script *sc;
+ struct rspamd_config **pcfg;
+
+ LL_FOREACH(cfg->post_init_scripts, sc)
+ {
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, sc->cbref);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ *pcfg = cfg;
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+
+ if (lua_pcall(L, 1, 0, err_idx) != 0) {
+ msg_err_config("cannot run config post init script: %s; priority = %d",
+ lua_tostring(L, -1), sc->priority);
+ }
+
+ lua_settop(L, err_idx - 1);
+ }
+}
+
+
+void rspamd_lua_run_config_unload(lua_State *L, struct rspamd_config *cfg)
+{
+ struct rspamd_config_cfg_lua_script *sc;
+ struct rspamd_config **pcfg;
+
+ LL_FOREACH(cfg->config_unload_scripts, sc)
+ {
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, sc->cbref);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ *pcfg = cfg;
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+
+ if (lua_pcall(L, 1, 0, err_idx) != 0) {
+ msg_err_config("cannot run config post init script: %s",
+ lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+ }
+}
+
+static void
+rspamd_lua_run_postloads_error(struct thread_entry *thread, int ret, const char *msg)
+{
+ struct rspamd_config *cfg = thread->cfg;
+
+ msg_err_config("error executing post load code: %s", msg);
+}
+
+
+struct rspamd_lua_ref_cbdata {
+ lua_State *L;
+ gint cbref;
+};
+
+static void
+rspamd_lua_ref_dtor(gpointer p)
+{
+ struct rspamd_lua_ref_cbdata *cbdata = p;
+
+ luaL_unref(cbdata->L, LUA_REGISTRYINDEX, cbdata->cbref);
+}
+
+void rspamd_lua_add_ref_dtor(lua_State *L, rspamd_mempool_t *pool,
+ gint ref)
+{
+ struct rspamd_lua_ref_cbdata *cbdata;
+
+ if (ref != -1) {
+ cbdata = rspamd_mempool_alloc(pool, sizeof(*cbdata));
+ cbdata->cbref = ref;
+ cbdata->L = L;
+
+ rspamd_mempool_add_destructor(pool, rspamd_lua_ref_dtor, cbdata);
+ }
+}
+
+gboolean
+rspamd_lua_require_function(lua_State *L, const gchar *modname,
+ const gchar *funcname)
+{
+ gint table_pos, err_pos;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_pos = lua_gettop(L);
+ lua_getglobal(L, "require");
+
+ if (lua_isnil(L, -1)) {
+ lua_remove(L, err_pos);
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+
+ lua_pushstring(L, modname);
+
+ /* Now try to call */
+ if (lua_pcall(L, 1, 1, 0) != 0) {
+ lua_remove(L, err_pos);
+ msg_warn("require of %s.%s failed: %s", modname,
+ funcname, lua_tostring(L, -1));
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+
+ lua_remove(L, err_pos);
+
+ /* Now we should have a table with results */
+ if (funcname) {
+ if (!lua_istable(L, -1)) {
+ msg_warn("require of %s.%s failed: not a table but %s", modname,
+ funcname, lua_typename(L, lua_type(L, -1)));
+
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+
+ table_pos = lua_gettop(L);
+ lua_pushstring(L, funcname);
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ /* Remove table, preserve just a function */
+ lua_remove(L, table_pos);
+
+ return TRUE;
+ }
+ else {
+ msg_warn("require of %s.%s failed: not a function but %s", modname,
+ funcname, lua_typename(L, lua_type(L, -1)));
+ }
+
+ lua_pop(L, 2);
+
+ return FALSE;
+ }
+ else if (lua_isfunction(L, -1)) {
+ return TRUE;
+ }
+ else {
+ msg_warn("require of %s failed: not a function but %s", modname,
+ lua_typename(L, lua_type(L, -1)));
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+}
+
+gint rspamd_lua_function_ref_from_str(lua_State *L, const gchar *str, gsize slen,
+ const gchar *modname, GError **err)
+{
+ gint err_idx, ref_idx;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ /* Load file */
+ if (luaL_loadbuffer(L, str, slen, modname) != 0) {
+ g_set_error(err,
+ lua_error_quark(),
+ EINVAL,
+ "%s: cannot load lua script: %s",
+ modname,
+ lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1); /* Error function */
+
+ return LUA_NOREF;
+ }
+
+ /* Now call it */
+ if (lua_pcall(L, 0, 1, err_idx) != 0) {
+ g_set_error(err,
+ lua_error_quark(),
+ EINVAL,
+ "%s: cannot init lua script: %s",
+ modname,
+ lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+
+ return LUA_NOREF;
+ }
+
+ if (!lua_isfunction(L, -1)) {
+ g_set_error(err,
+ lua_error_quark(),
+ EINVAL,
+ "%s: cannot init lua script: "
+ "must return function not %s",
+ modname,
+ lua_typename(L, lua_type(L, -1)));
+ lua_settop(L, err_idx - 1);
+
+ return LUA_NOREF;
+ }
+
+ ref_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+ lua_settop(L, err_idx - 1);
+
+ return ref_idx;
+}
+
+
+gboolean
+rspamd_lua_try_load_redis(lua_State *L, const ucl_object_t *obj,
+ struct rspamd_config *cfg, gint *ref_id)
+{
+ gint err_idx;
+ struct rspamd_config **pcfg;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ /* Obtain function */
+ if (!rspamd_lua_require_function(L, "lua_redis", "try_load_redis_servers")) {
+ msg_err_config("cannot require lua_redis");
+ lua_pop(L, 2);
+
+ return FALSE;
+ }
+
+ /* Function arguments */
+ ucl_object_push_lua(L, obj, false);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ lua_pushboolean(L, false); /* no_fallback */
+
+ if (lua_pcall(L, 3, 1, err_idx) != 0) {
+ msg_err_config("cannot call lua try_load_redis_servers script: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return FALSE;
+ }
+
+ if (lua_istable(L, -1)) {
+ if (ref_id) {
+ /* Ref table */
+ lua_pushvalue(L, -1);
+ *ref_id = luaL_ref(L, LUA_REGISTRYINDEX);
+ lua_settop(L, 0);
+ }
+ else {
+ /* Leave it on the stack */
+ lua_insert(L, err_idx);
+ lua_settop(L, err_idx);
+ }
+
+ return TRUE;
+ }
+ else {
+ lua_settop(L, 0);
+ }
+
+ return FALSE;
+}
+
+void rspamd_lua_push_full_word(lua_State *L, rspamd_stat_token_t *w)
+{
+ gint fl_cnt;
+
+ lua_createtable(L, 4, 0);
+
+ if (w->stemmed.len > 0) {
+ lua_pushlstring(L, w->stemmed.begin, w->stemmed.len);
+ lua_rawseti(L, -2, 1);
+ }
+ else {
+ lua_pushstring(L, "");
+ lua_rawseti(L, -2, 1);
+ }
+
+ if (w->normalized.len > 0) {
+ lua_pushlstring(L, w->normalized.begin, w->normalized.len);
+ lua_rawseti(L, -2, 2);
+ }
+ else {
+ lua_pushstring(L, "");
+ lua_rawseti(L, -2, 2);
+ }
+
+ if (w->original.len > 0) {
+ lua_pushlstring(L, w->original.begin, w->original.len);
+ lua_rawseti(L, -2, 3);
+ }
+ else {
+ lua_pushstring(L, "");
+ lua_rawseti(L, -2, 3);
+ }
+
+ /* Flags part */
+ fl_cnt = 1;
+ lua_createtable(L, 4, 0);
+
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_NORMALISED) {
+ lua_pushstring(L, "normalised");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE) {
+ lua_pushstring(L, "broken_unicode");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) {
+ lua_pushstring(L, "utf");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) {
+ lua_pushstring(L, "text");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_HEADER) {
+ lua_pushstring(L, "header");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & (RSPAMD_STAT_TOKEN_FLAG_META | RSPAMD_STAT_TOKEN_FLAG_LUA_META)) {
+ lua_pushstring(L, "meta");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_STOP_WORD) {
+ lua_pushstring(L, "stop_word");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES) {
+ lua_pushstring(L, "invisible_spaces");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_STEMMED) {
+ lua_pushstring(L, "stemmed");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+
+ lua_rawseti(L, -2, 4);
+}
+
+gint rspamd_lua_push_words(lua_State *L, GArray *words,
+ enum rspamd_lua_words_type how)
+{
+ rspamd_stat_token_t *w;
+ guint i, cnt;
+
+ lua_createtable(L, words->len, 0);
+
+ for (i = 0, cnt = 1; i < words->len; i++) {
+ w = &g_array_index(words, rspamd_stat_token_t, i);
+
+ switch (how) {
+ case RSPAMD_LUA_WORDS_STEM:
+ if (w->stemmed.len > 0) {
+ lua_pushlstring(L, w->stemmed.begin, w->stemmed.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ break;
+ case RSPAMD_LUA_WORDS_NORM:
+ if (w->normalized.len > 0) {
+ lua_pushlstring(L, w->normalized.begin, w->normalized.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ break;
+ case RSPAMD_LUA_WORDS_RAW:
+ if (w->original.len > 0) {
+ lua_pushlstring(L, w->original.begin, w->original.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ break;
+ case RSPAMD_LUA_WORDS_FULL:
+ rspamd_lua_push_full_word(L, w);
+ /* Push to the resulting vector */
+ lua_rawseti(L, -2, cnt++);
+ break;
+ default:
+ break;
+ }
+ }
+
+ return 1;
+}
+
+gchar *
+rspamd_lua_get_module_name(lua_State *L)
+{
+ lua_Debug d;
+ gchar *p;
+ gchar func_buf[128];
+
+ if (lua_getstack(L, 1, &d) == 1) {
+ (void) lua_getinfo(L, "Sl", &d);
+ if ((p = strrchr(d.short_src, '/')) == NULL) {
+ p = d.short_src;
+ }
+ else {
+ p++;
+ }
+
+ if (strlen(p) > 20) {
+ rspamd_snprintf(func_buf, sizeof(func_buf), "%10s...]:%d", p,
+ d.currentline);
+ }
+ else {
+ rspamd_snprintf(func_buf, sizeof(func_buf), "%s:%d", p,
+ d.currentline);
+ }
+
+ return g_strdup(func_buf);
+ }
+
+ return NULL;
+}
+
+bool rspamd_lua_universal_pcall(lua_State *L, gint cbref, const gchar *strloc,
+ gint nret, const gchar *args, GError **err, ...)
+{
+ va_list ap;
+ const gchar *argp = args, *classname;
+ gint err_idx, nargs = 0;
+ gpointer *cls_ptr;
+ gsize sz;
+
+ /* Error function */
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ va_start(ap, err);
+ /* Called function */
+ if (cbref > 0) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbref);
+ }
+ else {
+ /* Assume that function was on top of the stack */
+ lua_pushvalue(L, err_idx - 1);
+ }
+ /*
+ * Possible arguments
+ * - i - lua_integer, argument - gint64
+ * - n - lua_number, argument - gdouble
+ * - s - lua_string, argument - const gchar * (zero terminated)
+ * - l - lua_lstring, argument - (size_t + const gchar *) pair
+ * - u - lua_userdata, argument - (const char * + void *) - classname + pointer
+ * - b - lua_boolean, argument - gboolean (not bool due to varargs promotion)
+ * - f - lua_function, argument - int - position of the function on stack (not lua_registry)
+ * - t - lua_text, argument - int - position of the lua_text on stack (not lua_registry)
+ */
+ while (*argp) {
+ switch (*argp) {
+ case 'i':
+ lua_pushinteger(L, va_arg(ap, gint64));
+ nargs++;
+ break;
+ case 'n':
+ lua_pushnumber(L, va_arg(ap, gdouble));
+ nargs++;
+ break;
+ case 's':
+ lua_pushstring(L, va_arg(ap, const gchar *));
+ nargs++;
+ break;
+ case 'l':
+ sz = va_arg(ap, gsize);
+ lua_pushlstring(L, va_arg(ap, const gchar *), sz);
+ nargs++;
+ break;
+ case 'b':
+ lua_pushboolean(L, va_arg(ap, gboolean));
+ nargs++;
+ break;
+ case 'u':
+ classname = va_arg(ap, const gchar *);
+ cls_ptr = (gpointer *) lua_newuserdata(L, sizeof(gpointer));
+ *cls_ptr = va_arg(ap, gpointer);
+ rspamd_lua_setclass(L, classname, -1);
+ nargs++;
+ break;
+ case 'f':
+ case 't':
+ lua_pushvalue(L, va_arg(ap, gint));
+ nargs++;
+ break;
+ default:
+ lua_settop(L, err_idx - 1);
+ g_set_error(err, lua_error_quark(), EINVAL,
+ "invalid argument character: %c at %s",
+ *argp, argp);
+ va_end(ap);
+
+ return false;
+ }
+
+ argp++;
+ }
+
+ if (lua_pcall(L, nargs, nret, err_idx) != 0) {
+ g_set_error(err, lua_error_quark(), EBADF,
+ "error when calling lua function from %s: %s",
+ strloc, lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+ va_end(ap);
+
+ return false;
+ }
+
+ lua_remove(L, err_idx);
+ va_end(ap);
+
+ return true;
+}
+
+#if defined(LUA_VERSION_NUM) && LUA_VERSION_NUM <= 502
+gint rspamd_lua_geti(lua_State *L, int pos, int i)
+{
+ pos = lua_absindex(L, pos);
+ lua_pushinteger(L, i);
+ lua_gettable(L, pos);
+
+ return lua_type(L, -1);
+}
+#endif \ No newline at end of file
diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h
new file mode 100644
index 0000000..cc2b943
--- /dev/null
+++ b/src/lua/lua_common.h
@@ -0,0 +1,729 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_LUA_H
+#define RSPAMD_LUA_H
+
+#include "config.h"
+
+/* Lua headers do not have __cplusplus guards... */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <lua.h>
+#include <lauxlib.h>
+#include <lualib.h>
+#ifdef WITH_LUAJIT
+#include <luajit.h>
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#include <stdbool.h>
+
+
+#include "rspamd.h"
+#include "ucl.h"
+#include "lua_ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef lua_open
+#define lua_open() luaL_newstate()
+#endif
+
+#ifndef luaL_reg
+#define luaL_reg luaL_Reg
+#endif
+
+#define LUA_ENUM(L, name, val) \
+ lua_pushlstring(L, #name, sizeof(#name) - 1); \
+ lua_pushinteger(L, val); \
+ lua_settable(L, -3);
+
+#if LUA_VERSION_NUM > 501 && !defined LUA_COMPAT_MODULE
+static inline void
+luaL_register(lua_State *L, const gchar *name, const struct luaL_reg *methods)
+{
+ if (name != NULL) {
+ lua_newtable(L);
+ }
+ luaL_setfuncs(L, methods, 0);
+ if (name != NULL) {
+ lua_pushvalue(L, -1);
+ lua_setglobal(L, name);
+ }
+}
+#endif
+
+#if defined(LUA_VERSION_NUM) && LUA_VERSION_NUM == 501
+
+/* Special hack to work with moonjit of specific version */
+#if !defined(MOONJIT_VERSION) && (!defined(LUAJIT_VERSION_NUM) || LUAJIT_VERSION_NUM != 20200)
+static inline int lua_absindex(lua_State *L, int i)
+{
+ if (i < 0 && i > LUA_REGISTRYINDEX)
+ i += lua_gettop(L) + 1;
+ return i;
+}
+#endif
+
+#endif
+
+/* Interface definitions */
+#define LUA_FUNCTION_DEF(class, name) static int lua_##class##_##name(lua_State *L)
+#define LUA_PUBLIC_FUNCTION_DEF(class, name) int lua_##class##_##name(lua_State *L)
+#define LUA_INTERFACE_DEF(class, name) \
+ { \
+ #name, lua_##class##_##name \
+ }
+
+extern const luaL_reg null_reg[];
+
+#define RSPAMD_LUA_CFG_STATE(cfg) ((lua_State *) ((cfg)->lua_state))
+/**
+* Lua IP address structure
+*/
+struct rspamd_lua_ip {
+ rspamd_inet_addr_t *addr;
+};
+
+#define RSPAMD_TEXT_FLAG_OWN (1u << 0u)
+#define RSPAMD_TEXT_FLAG_MMAPED (1u << 1u)
+#define RSPAMD_TEXT_FLAG_WIPE (1u << 2u)
+#define RSPAMD_TEXT_FLAG_SYSMALLOC (1u << 3u)
+#define RSPAMD_TEXT_FLAG_FAKE (1u << 4u)
+#define RSPAMD_TEXT_FLAG_BINARY (1u << 5u)
+struct rspamd_lua_text {
+ const gchar *start;
+ guint len;
+ guint flags;
+};
+
+struct rspamd_lua_url {
+ struct rspamd_url *url;
+};
+
+struct rspamd_lua_regexp {
+ rspamd_regexp_t *re;
+ gchar *module;
+ gchar *re_pattern;
+ gint re_flags;
+};
+
+struct rspamd_map;
+struct lua_map_callback_data;
+struct radix_tree_compressed;
+struct rspamd_mime_header;
+
+enum rspamd_lua_map_type {
+ RSPAMD_LUA_MAP_RADIX = 0,
+ RSPAMD_LUA_MAP_SET,
+ RSPAMD_LUA_MAP_HASH,
+ RSPAMD_LUA_MAP_REGEXP,
+ RSPAMD_LUA_MAP_REGEXP_MULTIPLE,
+ RSPAMD_LUA_MAP_CALLBACK,
+ RSPAMD_LUA_MAP_CDB,
+ RSPAMD_LUA_MAP_UNKNOWN,
+};
+
+struct rspamd_lua_map {
+ struct rspamd_map *map;
+ enum rspamd_lua_map_type type;
+ guint flags;
+
+ union {
+ struct rspamd_radix_map_helper *radix;
+ struct rspamd_hash_map_helper *hash;
+ struct rspamd_regexp_map_helper *re_map;
+ struct rspamd_cdb_map_helper *cdb_map;
+ struct lua_map_callback_data *cbdata;
+ } data;
+};
+
+struct rspamd_lua_upstream {
+ struct upstream *up;
+ gint upref;
+};
+
+/* Common utility functions */
+
+/**
+* Create and register new class
+*/
+void rspamd_lua_new_class(lua_State *L,
+ const gchar *classname,
+ const struct luaL_reg *methods);
+
+/**
+* Set class name for object at @param objidx position
+*/
+void rspamd_lua_setclass(lua_State *L, const gchar *classname, gint objidx);
+
+/**
+* Pushes the metatable for specific class on top of the stack
+* @param L
+* @param classname
+*/
+void rspamd_lua_class_metatable(lua_State *L, const gchar *classname);
+
+/**
+* Adds a new field to the class (metatable) identified by `classname`
+* @param L
+* @param classname
+* @param meth
+*/
+void rspamd_lua_add_metamethod(lua_State *L, const gchar *classname,
+ luaL_Reg *meth);
+
+/**
+* Set index of table to value (like t['index'] = value)
+*/
+void rspamd_lua_table_set(lua_State *L, const gchar *index, const gchar *value);
+
+/**
+* Get string value of index in a table (return t['index'])
+*/
+const gchar *rspamd_lua_table_get(lua_State *L, const gchar *index);
+
+/**
+* Convert classname to string
+*/
+gint rspamd_lua_class_tostring(lua_State *L);
+
+/**
+* Check whether the argument at specified index is of the specified class
+*/
+gpointer rspamd_lua_check_class(lua_State *L, gint index, const gchar *name);
+
+/**
+* Initialize lua and bindings
+*/
+lua_State *rspamd_lua_init(bool wipe_mem);
+
+/**
+ * Close lua_state and free remainders
+ * @param L
+ */
+void rspamd_lua_close(lua_State *L);
+
+void rspamd_lua_start_gc(struct rspamd_config *cfg);
+
+/**
+* Sets field in a global variable
+* @param L
+* @param global_name
+* @param field_name
+* @param new_elt
+*/
+void rspamd_plugins_table_push_elt(lua_State *L, const gchar *field_name,
+ const gchar *new_elt);
+
+/**
+* Load and initialize lua plugins
+*/
+gboolean
+rspamd_init_lua_filters(struct rspamd_config *cfg, bool force_load, bool strict);
+
+
+/**
+* Push lua ip address
+*/
+void rspamd_lua_ip_push(lua_State *L, rspamd_inet_addr_t *addr);
+
+/**
+* Push rspamd task structure to lua
+*/
+void rspamd_lua_task_push(lua_State *L, struct rspamd_task *task);
+
+/**
+* Return lua ip structure at the specified address
+*/
+struct rspamd_lua_ip *lua_check_ip(lua_State *L, gint pos);
+
+struct rspamd_lua_text *lua_check_text(lua_State *L, gint pos);
+/**
+* Checks for a text or a string. In case of string a pointer to static structure is returned.
+* So it should not be reused or placed to Lua stack anyhow!
+* However, you can use this function up to 4 times and have distinct static structures
+* @param L
+* @param pos
+* @return
+*/
+struct rspamd_lua_text *lua_check_text_or_string(lua_State *L, gint pos);
+/**
+ * Create new text object
+ * @param L
+ * @param start
+ * @param len
+ * @param own
+ * @return
+ */
+struct rspamd_lua_text *lua_new_text(lua_State *L, const gchar *start,
+ gsize len, gboolean own);
+/**
+ * Create new text object from task pool if allocation is needed
+ * @param task
+ * @param L
+ * @param start
+ * @param len
+ * @param own
+ * @return
+ */
+struct rspamd_lua_text *lua_new_text_task(lua_State *L, struct rspamd_task *task,
+ const gchar *start, gsize len, gboolean own);
+/**
+ * Checks if a text has binary characters (non ascii and non-utf8 characters)
+ * @param t
+ * @return
+ */
+bool lua_is_text_binary(struct rspamd_lua_text *t);
+
+struct rspamd_lua_regexp *lua_check_regexp(lua_State *L, gint pos);
+
+struct rspamd_lua_upstream *lua_check_upstream(lua_State *L, int pos);
+
+enum rspamd_lua_task_header_type {
+ RSPAMD_TASK_HEADER_PUSH_SIMPLE = 0,
+ RSPAMD_TASK_HEADER_PUSH_RAW,
+ RSPAMD_TASK_HEADER_PUSH_FULL,
+ RSPAMD_TASK_HEADER_PUSH_COUNT,
+ RSPAMD_TASK_HEADER_PUSH_HAS,
+};
+
+gint rspamd_lua_push_header(lua_State *L,
+ struct rspamd_mime_header *h,
+ enum rspamd_lua_task_header_type how);
+
+/**
+* Push specific header to lua
+*/
+gint rspamd_lua_push_header_array(lua_State *L,
+ const gchar *name,
+ struct rspamd_mime_header *rh,
+ enum rspamd_lua_task_header_type how,
+ gboolean strong);
+
+/**
+* Check for task at the specified position
+*/
+struct rspamd_task *lua_check_task(lua_State *L, gint pos);
+
+struct rspamd_task *lua_check_task_maybe(lua_State *L, gint pos);
+
+struct rspamd_lua_map *lua_check_map(lua_State *L, gint pos);
+
+/**
+* Push ip address from a string (nil is pushed if a string cannot be converted)
+*/
+void rspamd_lua_ip_push_fromstring(lua_State *L, const gchar *ip_str);
+
+/**
+* Create type error
+*/
+int rspamd_lua_typerror(lua_State *L, int narg, const char *tname);
+/**
+* Open libraries functions
+*/
+
+/**
+* Add preload function
+*/
+void rspamd_lua_add_preload(lua_State *L, const gchar *name, lua_CFunction func);
+
+void luaopen_task(lua_State *L);
+
+void luaopen_config(lua_State *L);
+
+void luaopen_map(lua_State *L);
+
+void luaopen_trie(lua_State *L);
+
+void luaopen_textpart(lua_State *L);
+
+void luaopen_mimepart(lua_State *L);
+
+void luaopen_image(lua_State *L);
+
+void luaopen_url(lua_State *L);
+
+void luaopen_classifier(lua_State *L);
+
+void luaopen_statfile(lua_State *L);
+
+void luaopen_regexp(lua_State *L);
+
+void luaopen_cdb(lua_State *L);
+
+void luaopen_xmlrpc(lua_State *L);
+
+void luaopen_http(lua_State *L);
+
+void luaopen_redis(lua_State *L);
+
+void luaopen_upstream(lua_State *L);
+
+void luaopen_mempool(lua_State *L);
+
+void luaopen_dns_resolver(lua_State *L);
+
+void luaopen_rsa(lua_State *L);
+
+void luaopen_ip(lua_State *L);
+
+void luaopen_expression(lua_State *L);
+
+void luaopen_logger(lua_State *L);
+
+void luaopen_text(lua_State *L);
+
+void luaopen_util(lua_State *L);
+
+void luaopen_tcp(lua_State *L);
+
+void luaopen_html(lua_State *L);
+
+void luaopen_sqlite3(lua_State *L);
+
+void luaopen_cryptobox(lua_State *L);
+
+void luaopen_dns(lua_State *L);
+
+void luaopen_udp(lua_State *L);
+
+void luaopen_worker(lua_State *L);
+
+void luaopen_kann(lua_State *L);
+
+void luaopen_spf(lua_State *L);
+
+void luaopen_tensor(lua_State *L);
+
+void luaopen_parsers(lua_State *L);
+
+void rspamd_lua_dostring(const gchar *line);
+
+double rspamd_lua_normalize(struct rspamd_config *cfg,
+ long double score,
+ void *params);
+
+/* Config file functions */
+void rspamd_lua_post_load_config(struct rspamd_config *cfg);
+
+void rspamd_lua_dumpstack(lua_State *L);
+
+/* Set lua path according to the configuration */
+void rspamd_lua_set_path(lua_State *L, const ucl_object_t *cfg_obj,
+ GHashTable *vars);
+
+/* Set some lua globals */
+gboolean rspamd_lua_set_env(lua_State *L, GHashTable *vars, char **lua_env,
+ GError **err);
+
+void rspamd_lua_set_globals(struct rspamd_config *cfg, lua_State *L);
+
+struct memory_pool_s *rspamd_lua_check_mempool(lua_State *L, gint pos);
+
+struct rspamd_config *lua_check_config(lua_State *L, gint pos);
+
+struct rspamd_async_session *lua_check_session(lua_State *L, gint pos);
+
+struct ev_loop *lua_check_ev_base(lua_State *L, gint pos);
+
+struct rspamd_dns_resolver *lua_check_dns_resolver(lua_State *L, gint pos);
+
+struct rspamd_lua_url *lua_check_url(lua_State *L, gint pos);
+
+enum rspamd_lua_parse_arguments_flags {
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT = 0,
+ RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING,
+};
+
+/**
+* Extract an arguments from lua table according to format string. Supported arguments are:
+* [*]key=S|I|N|B|V|U{a-z};[key=...]
+* - S - const char *
+* - I - gint64_t
+* - i - int32_t
+* - N - double
+* - B - gboolean
+* - V - size_t + const char *
+* - U{classname} - userdata of the following class (stored in gpointer)
+* - F - function
+* - O - ucl_object_t *
+* - D - same as N but argument is set to NAN not to 0.0
+* - u{classname} - userdata of the following class (stored directly)
+*
+* If any of keys is prefixed with `*` then it is treated as required argument
+* @param L lua state
+* @param pos at which pos start extraction
+* @param err error pointer
+* @param how extraction type (IGNORE_MISSING means that default values will not be set)
+* @param extraction_pattern static pattern
+* @return TRUE if a table has been parsed
+*/
+gboolean rspamd_lua_parse_table_arguments(lua_State *L, gint pos,
+ GError **err,
+ enum rspamd_lua_parse_arguments_flags how,
+ const gchar *extraction_pattern, ...);
+
+
+gint rspamd_lua_traceback(lua_State *L);
+
+/**
+* Returns stack trace as a string. Caller should clear memory.
+* @param L
+* @return
+*/
+void rspamd_lua_get_traceback_string(lua_State *L, luaL_Buffer *buf);
+
+/**
+* Returns size of table at position `tbl_pos`
+*/
+guint rspamd_lua_table_size(lua_State *L, gint tbl_pos);
+
+void lua_push_emails_address_list(lua_State *L, GPtrArray *addrs, int flags);
+
+
+#define TRACE_POINTS 6
+
+struct lua_logger_trace {
+ gint cur_level;
+ gconstpointer traces[TRACE_POINTS];
+};
+
+enum lua_logger_escape_type {
+ LUA_ESCAPE_NONE = (0u),
+ LUA_ESCAPE_UNPRINTABLE = (1u << 0u),
+ LUA_ESCAPE_NEWLINES = (1u << 1u),
+ LUA_ESCAPE_8BIT = (1u << 2u),
+};
+
+#define LUA_ESCAPE_LOG (LUA_ESCAPE_UNPRINTABLE | LUA_ESCAPE_NEWLINES)
+#define LUA_ESCAPE_ALL (LUA_ESCAPE_UNPRINTABLE | LUA_ESCAPE_NEWLINES | LUA_ESCAPE_8BIT)
+
+/**
+* Log lua object to string
+* @param L
+* @param pos
+* @param outbuf
+* @param len
+* @return
+*/
+gsize lua_logger_out_type(lua_State *L, gint pos, gchar *outbuf,
+ gsize len, struct lua_logger_trace *trace,
+ enum lua_logger_escape_type esc_type);
+
+/**
+* Safely checks userdata to match specified class
+* @param L
+* @param pos
+* @param classname
+*/
+void *rspamd_lua_check_udata(lua_State *L, gint pos, const gchar *classname);
+
+#define RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, pos, classname, type, dest) \
+ do { \
+ type **_maybe_ptr = (type **) rspamd_lua_check_udata((L), (pos), (classname)); \
+ if (_maybe_ptr == NULL) { \
+ return luaL_error(L, "%s: invalid arguments; pos = %d; expected = %s", G_STRFUNC, (pos), (classname)); \
+ } \
+ (dest) = *(_maybe_ptr); \
+ } while (0)
+
+/**
+* Safely checks userdata to match specified class
+* @param L
+* @param pos
+* @param classname
+*/
+void *rspamd_lua_check_udata_maybe(lua_State *L, gint pos, const gchar *classname);
+
+/**
+* Call finishing script with the specified task
+* @param sc
+* @param task
+*/
+void lua_call_finish_script(struct rspamd_config_cfg_lua_script *sc,
+ struct rspamd_task *task);
+
+/**
+* Run post-load operations
+* @param L
+* @param cfg
+* @param ev_base
+*/
+void rspamd_lua_run_postloads(lua_State *L, struct rspamd_config *cfg,
+ struct ev_loop *ev_base, struct rspamd_worker *w);
+
+void rspamd_lua_run_config_post_init(lua_State *L, struct rspamd_config *cfg);
+
+void rspamd_lua_run_config_unload(lua_State *L, struct rspamd_config *cfg);
+
+/**
+* Adds new destructor for a local function for specific pool
+* @param L
+* @param pool
+* @param ref
+*/
+void rspamd_lua_add_ref_dtor(lua_State *L, rspamd_mempool_t *pool,
+ gint ref);
+
+/**
+ * Returns a lua reference from a function like string, e.g. `return function(...) end`
+ * @param L
+ * @param str
+ * @return
+ */
+gint rspamd_lua_function_ref_from_str(lua_State *L, const gchar *str, gsize slen,
+ const gchar *modname, GError **err);
+
+/**
+* Tries to load some module using `require` and get some method from it
+* @param L
+* @param modname
+* @param funcname
+* @return TRUE if function exists in that module, the function is pushed in stack, otherwise stack is unchanged and FALSE is returned
+*/
+gboolean rspamd_lua_require_function(lua_State *L, const gchar *modname,
+ const gchar *funcname);
+
+/**
+* Tries to load redis server definition from ucl object specified
+* @param L
+* @param obj
+* @param cfg
+* @return
+*/
+gboolean rspamd_lua_try_load_redis(lua_State *L, const ucl_object_t *obj,
+ struct rspamd_config *cfg, gint *ref_id);
+
+struct rspamd_stat_token_s;
+
+/**
+* Pushes a single word into Lua
+* @param L
+* @param word
+*/
+void rspamd_lua_push_full_word(lua_State *L, struct rspamd_stat_token_s *word);
+
+enum rspamd_lua_words_type {
+ RSPAMD_LUA_WORDS_STEM = 0,
+ RSPAMD_LUA_WORDS_NORM,
+ RSPAMD_LUA_WORDS_RAW,
+ RSPAMD_LUA_WORDS_FULL,
+ RSPAMD_LUA_WORDS_MAX
+};
+
+/**
+* Pushes words (rspamd_stat_token_t) to Lua
+* @param L
+* @param words
+* @param how
+*/
+gint rspamd_lua_push_words(lua_State *L, GArray *words,
+ enum rspamd_lua_words_type how);
+
+/**
+* Returns newly allocated name for caller module name
+* @param L
+* @return
+*/
+gchar *rspamd_lua_get_module_name(lua_State *L);
+
+/**
+* Call Lua function in a universal way. Arguments string:
+* - i - lua_integer, argument - gint64
+* - n - lua_number, argument - gdouble
+* - s - lua_string, argument - const gchar * (zero terminated)
+* - l - lua_lstring, argument - (size_t + const gchar *) pair
+* - u - lua_userdata, argument - (const char * + void *) - classname + pointer
+* - b - lua_boolean, argument - gboolean (not bool due to varargs promotion)
+* - f - lua_function, argument - int - position of the function on stack (not lua_registry)
+* - t - lua_text, argument - int - position of the lua_text on stack (not lua_registry)
+* @param L lua state
+* @param cbref LUA_REGISTRY reference (if it is -1 then a function on top of the stack is called - it must be removed by caller manually)
+* @param strloc where this function is called from
+* @param nret number of results (or LUA_MULTRET)
+* @param args arguments format string
+* @param err error to promote
+* @param ... arguments
+* @return true of pcall returned 0, false + err otherwise
+*/
+bool rspamd_lua_universal_pcall(lua_State *L, gint cbref, const gchar *strloc,
+ gint nret, const gchar *args, GError **err, ...);
+
+/**
+ * Returns true if lua is initialised
+ * @return
+ */
+bool rspamd_lua_is_initialised(void);
+
+/**
+* Wrapper for lua_geti from lua 5.3
+* @param L
+* @param index
+* @param i
+* @return
+*/
+#if defined(LUA_VERSION_NUM) && LUA_VERSION_NUM <= 502
+gint rspamd_lua_geti(lua_State *L, int index, int i);
+#else
+#define rspamd_lua_geti lua_geti
+#endif
+
+/* Paths defs */
+#define RSPAMD_CONFDIR_INDEX "CONFDIR"
+#define RSPAMD_LOCAL_CONFDIR_INDEX "LOCAL_CONFDIR"
+#define RSPAMD_RUNDIR_INDEX "RUNDIR"
+#define RSPAMD_DBDIR_INDEX "DBDIR"
+#define RSPAMD_LOGDIR_INDEX "LOGDIR"
+#define RSPAMD_PLUGINSDIR_INDEX "PLUGINSDIR"
+#define RSPAMD_SHAREDIR_INDEX "SHAREDIR"
+#define RSPAMD_RULESDIR_INDEX "RULESDIR"
+#define RSPAMD_LUALIBDIR_INDEX "LUALIBDIR"
+#define RSPAMD_WWWDIR_INDEX "WWWDIR"
+#define RSPAMD_PREFIX_INDEX "PREFIX"
+#define RSPAMD_VERSION_INDEX "VERSION"
+
+#ifdef WITH_LUA_TRACE
+extern ucl_object_t *lua_traces;
+#define LUA_TRACE_POINT \
+ do { \
+ ucl_object_t *func_obj; \
+ if (lua_traces == NULL) { lua_traces = ucl_object_typed_new(UCL_OBJECT); } \
+ func_obj = (ucl_object_t *) ucl_object_lookup(lua_traces, G_STRFUNC); \
+ if (func_obj == NULL) { \
+ func_obj = ucl_object_typed_new(UCL_INT); \
+ ucl_object_insert_key(lua_traces, func_obj, G_STRFUNC, 0, false); \
+ } \
+ func_obj->value.iv++; \
+ } while (0)
+#else
+#define LUA_TRACE_POINT \
+ do { \
+ } while (0)
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RSPAMD_LUA_H */
diff --git a/src/lua/lua_compress.c b/src/lua/lua_compress.c
new file mode 100644
index 0000000..77c82c5
--- /dev/null
+++ b/src/lua/lua_compress.c
@@ -0,0 +1,622 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "unix-std.h"
+#include <zlib.h>
+
+#ifdef SYS_ZSTD
+#include "zstd.h"
+#include "zstd_errors.h"
+#else
+#include "contrib/zstd/zstd.h"
+#include "contrib/zstd/error_public.h"
+#endif
+
+/***
+ * @module rspamd_compress
+ * This module contains compression/decompression routines (zstd and zlib currently)
+ */
+
+/***
+ * @function zstd.compress_ctx()
+ * Creates new compression ctx
+ * @return {compress_ctx} new compress ctx
+ */
+LUA_FUNCTION_DEF(zstd, compress_ctx);
+
+/***
+ * @function zstd.compress_ctx()
+ * Creates new compression ctx
+ * @return {compress_ctx} new compress ctx
+ */
+LUA_FUNCTION_DEF(zstd, decompress_ctx);
+
+LUA_FUNCTION_DEF(zstd_compress, stream);
+LUA_FUNCTION_DEF(zstd_compress, dtor);
+
+LUA_FUNCTION_DEF(zstd_decompress, stream);
+LUA_FUNCTION_DEF(zstd_decompress, dtor);
+
+static const struct luaL_reg zstd_compress_lib_f[] = {
+ LUA_INTERFACE_DEF(zstd, compress_ctx),
+ LUA_INTERFACE_DEF(zstd, decompress_ctx),
+ {NULL, NULL}};
+
+static const struct luaL_reg zstd_compress_lib_m[] = {
+ LUA_INTERFACE_DEF(zstd_compress, stream),
+ {"__gc", lua_zstd_compress_dtor},
+ {NULL, NULL}};
+
+static const struct luaL_reg zstd_decompress_lib_m[] = {
+ LUA_INTERFACE_DEF(zstd_decompress, stream),
+ {"__gc", lua_zstd_decompress_dtor},
+ {NULL, NULL}};
+
+static ZSTD_CStream *
+lua_check_zstd_compress_ctx(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{zstd_compress}");
+ luaL_argcheck(L, ud != NULL, pos, "'zstd_compress' expected");
+ return ud ? *(ZSTD_CStream **) ud : NULL;
+}
+
+static ZSTD_DStream *
+lua_check_zstd_decompress_ctx(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{zstd_decompress}");
+ luaL_argcheck(L, ud != NULL, pos, "'zstd_decompress' expected");
+ return ud ? *(ZSTD_DStream **) ud : NULL;
+}
+
+int lua_zstd_push_error(lua_State *L, int err)
+{
+ lua_pushnil(L);
+ lua_pushfstring(L, "zstd error %d (%s)", err, ZSTD_getErrorString(err));
+
+ return 2;
+}
+
+gint lua_compress_zstd_compress(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = NULL, *res;
+ gsize sz, r;
+ gint comp_level = 1;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ comp_level = lua_tointeger(L, 2);
+ }
+
+ sz = ZSTD_compressBound(t->len);
+
+ if (ZSTD_isError(sz)) {
+ msg_err("cannot compress data: %s", ZSTD_getErrorName(sz));
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ res = lua_newuserdata(L, sizeof(*res));
+ res->start = g_malloc(sz);
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ r = ZSTD_compress((void *) res->start, sz, t->start, t->len, comp_level);
+
+ if (ZSTD_isError(r)) {
+ msg_err("cannot compress data: %s", ZSTD_getErrorName(r));
+ lua_pop(L, 1); /* Text will be freed here */
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ res->len = r;
+
+ return 1;
+}
+
+gint lua_compress_zstd_decompress(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = NULL, *res;
+ gsize outlen, r;
+ ZSTD_DStream *zstream;
+ ZSTD_inBuffer zin;
+ ZSTD_outBuffer zout;
+ gchar *out;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ zstream = ZSTD_createDStream();
+ ZSTD_initDStream(zstream);
+
+ zin.pos = 0;
+ zin.src = t->start;
+ zin.size = t->len;
+
+ if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
+ outlen = ZSTD_DStreamOutSize();
+ }
+
+ out = g_malloc(outlen);
+
+ zout.dst = out;
+ zout.pos = 0;
+ zout.size = outlen;
+
+ while (zin.pos < zin.size) {
+ r = ZSTD_decompressStream(zstream, &zout, &zin);
+
+ if (ZSTD_isError(r)) {
+ msg_err("cannot decompress data: %s", ZSTD_getErrorName(r));
+ ZSTD_freeDStream(zstream);
+ g_free(out);
+ lua_pushstring(L, ZSTD_getErrorName(r));
+ lua_pushnil(L);
+
+ return 2;
+ }
+
+ if (zin.pos < zin.size && zout.pos == zout.size) {
+ /* We need to extend output buffer */
+ zout.size = zout.size * 2;
+ out = g_realloc(zout.dst, zout.size);
+ zout.dst = out;
+ }
+ }
+
+ ZSTD_freeDStream(zstream);
+ lua_pushnil(L); /* Error */
+ res = lua_newuserdata(L, sizeof(*res));
+ res->start = out;
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ res->len = zout.pos;
+
+ return 2;
+}
+
+gint lua_compress_zlib_decompress(lua_State *L, bool is_gzip)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = NULL, *res;
+ gsize sz;
+ z_stream strm;
+ gint rc;
+ guchar *p;
+ gsize remain;
+ gssize size_limit = -1;
+
+ int windowBits = is_gzip ? (MAX_WBITS + 16) : (MAX_WBITS);
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ size_limit = lua_tointeger(L, 2);
+ if (size_limit <= 0) {
+ return luaL_error(L, "invalid arguments (size_limit)");
+ }
+
+ sz = MIN(t->len * 2, size_limit);
+ }
+ else {
+ sz = t->len * 2;
+ }
+
+ memset(&strm, 0, sizeof(strm));
+ /* windowBits +16 to decode gzip, zlib 1.2.0.4+ */
+
+ /* Here are dragons to distinguish between raw deflate and zlib */
+ if (windowBits == MAX_WBITS && t->len > 0) {
+ if ((int) (unsigned char) ((t->start[0] << 4)) != 0x80) {
+ /* Assume raw deflate */
+ windowBits = -windowBits;
+ }
+ }
+
+ rc = inflateInit2(&strm, windowBits);
+
+ if (rc != Z_OK) {
+ return luaL_error(L, "cannot init zlib");
+ }
+
+ strm.avail_in = t->len;
+ strm.next_in = (guchar *) t->start;
+
+ res = lua_newuserdata(L, sizeof(*res));
+ res->start = g_malloc(sz);
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ p = (guchar *) res->start;
+ remain = sz;
+
+ while (strm.avail_in != 0) {
+ strm.avail_out = remain;
+ strm.next_out = p;
+
+ rc = inflate(&strm, Z_NO_FLUSH);
+
+ if (rc != Z_OK && rc != Z_BUF_ERROR) {
+ if (rc == Z_STREAM_END) {
+ break;
+ }
+ else {
+ msg_err("cannot decompress data: %s (last error: %s)",
+ zError(rc), strm.msg);
+ lua_pop(L, 1); /* Text will be freed here */
+ lua_pushnil(L);
+ inflateEnd(&strm);
+
+ return 1;
+ }
+ }
+
+ res->len = strm.total_out;
+
+ if (strm.avail_out == 0 && strm.avail_in != 0) {
+
+ if (size_limit > 0 || res->len >= G_MAXUINT32 / 2) {
+ if (res->len > size_limit || res->len >= G_MAXUINT32 / 2) {
+ lua_pop(L, 1); /* Text will be freed here */
+ lua_pushnil(L);
+ inflateEnd(&strm);
+
+ return 1;
+ }
+ }
+
+ /* Need to allocate more */
+ remain = res->len;
+ res->start = g_realloc((gpointer) res->start, res->len * 2);
+ sz = res->len * 2;
+ p = (guchar *) res->start + remain;
+ remain = sz - remain;
+ }
+ }
+
+ inflateEnd(&strm);
+ res->len = strm.total_out;
+
+ return 1;
+}
+
+gint lua_compress_zlib_compress(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = NULL, *res;
+ gsize sz;
+ z_stream strm;
+ gint rc, comp_level = Z_DEFAULT_COMPRESSION;
+ guchar *p;
+ gsize remain;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_isnumber(L, 2)) {
+ comp_level = lua_tointeger(L, 2);
+
+ if (comp_level > Z_BEST_COMPRESSION || comp_level < Z_BEST_SPEED) {
+ return luaL_error(L, "invalid arguments: compression level must be between %d and %d",
+ Z_BEST_SPEED, Z_BEST_COMPRESSION);
+ }
+ }
+
+
+ memset(&strm, 0, sizeof(strm));
+ rc = deflateInit2(&strm, comp_level, Z_DEFLATED,
+ MAX_WBITS + 16, MAX_MEM_LEVEL - 1, Z_DEFAULT_STRATEGY);
+
+ if (rc != Z_OK) {
+ return luaL_error(L, "cannot init zlib: %s", zError(rc));
+ }
+
+ sz = deflateBound(&strm, t->len);
+
+ strm.avail_in = t->len;
+ strm.next_in = (guchar *) t->start;
+
+ res = lua_newuserdata(L, sizeof(*res));
+ res->start = g_malloc(sz);
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ p = (guchar *) res->start;
+ remain = sz;
+
+ while (strm.avail_in != 0) {
+ strm.avail_out = remain;
+ strm.next_out = p;
+
+ rc = deflate(&strm, Z_FINISH);
+
+ if (rc != Z_OK && rc != Z_BUF_ERROR) {
+ if (rc == Z_STREAM_END) {
+ break;
+ }
+ else {
+ msg_err("cannot compress data: %s (last error: %s)",
+ zError(rc), strm.msg);
+ lua_pop(L, 1); /* Text will be freed here */
+ lua_pushnil(L);
+ deflateEnd(&strm);
+
+ return 1;
+ }
+ }
+
+ res->len = strm.total_out;
+
+ if (strm.avail_out == 0 && strm.avail_in != 0) {
+ /* Need to allocate more */
+ remain = res->len;
+ res->start = g_realloc((gpointer) res->start, strm.avail_in + sz);
+ sz = strm.avail_in + sz;
+ p = (guchar *) res->start + remain;
+ remain = sz - remain;
+ }
+ }
+
+ deflateEnd(&strm);
+ res->len = strm.total_out;
+
+ return 1;
+}
+
+/* Stream API interface for Zstd: both compression and decompression */
+
+/* Operations allowed by zstd stream methods */
+static const char *const zstd_stream_op[] = {
+ "continue",
+ "flush",
+ "end",
+ NULL};
+
+static gint
+lua_zstd_compress_ctx(lua_State *L)
+{
+ ZSTD_CCtx *ctx, **pctx;
+
+ pctx = lua_newuserdata(L, sizeof(*pctx));
+ ctx = ZSTD_createCCtx();
+
+ if (!ctx) {
+ return luaL_error(L, "context create failed");
+ }
+
+ *pctx = ctx;
+ rspamd_lua_setclass(L, "rspamd{zstd_compress}", -1);
+ return 1;
+}
+
+static gint
+lua_zstd_compress_dtor(lua_State *L)
+{
+ ZSTD_CCtx *ctx = lua_check_zstd_compress_ctx(L, 1);
+
+ if (ctx) {
+ ZSTD_freeCCtx(ctx);
+ }
+
+ return 0;
+}
+
+static gint
+lua_zstd_compress_reset(lua_State *L)
+{
+ ZSTD_CCtx *ctx = lua_check_zstd_compress_ctx(L, 1);
+
+ if (ctx) {
+ ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_zstd_compress_stream(lua_State *L)
+{
+ ZSTD_CStream *ctx = lua_check_zstd_compress_ctx(L, 1);
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 2);
+ int op = luaL_checkoption(L, 3, zstd_stream_op[0], zstd_stream_op);
+ int err = 0;
+ ZSTD_inBuffer inb;
+ ZSTD_outBuffer onb;
+
+ if (ctx && t) {
+ gsize dlen = 0;
+
+ inb.size = t->len;
+ inb.pos = 0;
+ inb.src = (const void *) t->start;
+
+ onb.pos = 0;
+ onb.size = ZSTD_CStreamInSize(); /* Initial guess */
+ onb.dst = NULL;
+
+ for (;;) {
+ if ((onb.dst = g_realloc(onb.dst, onb.size)) == NULL) {
+ return lua_zstd_push_error(L, ZSTD_error_memory_allocation);
+ }
+
+ dlen = onb.size;
+
+ int res = ZSTD_compressStream2(ctx, &onb, &inb, op);
+
+ if (res == 0) {
+ /* All done */
+ break;
+ }
+
+ if ((err = ZSTD_getErrorCode(res))) {
+ break;
+ }
+
+ onb.size *= 2;
+ res += dlen; /* Hint returned by compression routine */
+
+ /* Either double the buffer, or use the hint provided */
+ if (onb.size < res) {
+ onb.size = res;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (err) {
+ return lua_zstd_push_error(L, err);
+ }
+
+ lua_new_text(L, onb.dst, onb.pos, TRUE);
+
+ return 1;
+}
+
+static gint
+lua_zstd_decompress_dtor(lua_State *L)
+{
+ ZSTD_DStream *ctx = lua_check_zstd_decompress_ctx(L, 1);
+
+ if (ctx) {
+ ZSTD_freeDStream(ctx);
+ }
+
+ return 0;
+}
+
+
+static gint
+lua_zstd_decompress_ctx(lua_State *L)
+{
+ ZSTD_DStream *ctx, **pctx;
+
+ pctx = lua_newuserdata(L, sizeof(*pctx));
+ ctx = ZSTD_createDStream();
+
+ if (!ctx) {
+ return luaL_error(L, "context create failed");
+ }
+
+ *pctx = ctx;
+ rspamd_lua_setclass(L, "rspamd{zstd_decompress}", -1);
+ return 1;
+}
+
+static gint
+lua_zstd_decompress_stream(lua_State *L)
+{
+ ZSTD_DStream *ctx = lua_check_zstd_decompress_ctx(L, 1);
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 2);
+ int err = 0;
+ ZSTD_inBuffer inb;
+ ZSTD_outBuffer onb;
+
+ if (ctx && t) {
+ gsize dlen = 0;
+
+ if (t->len == 0) {
+ return lua_zstd_push_error(L, ZSTD_error_init_missing);
+ }
+
+ inb.size = t->len;
+ inb.pos = 0;
+ inb.src = (const void *) t->start;
+
+ onb.pos = 0;
+ onb.size = ZSTD_DStreamInSize(); /* Initial guess */
+ onb.dst = NULL;
+
+ for (;;) {
+ if ((onb.dst = g_realloc(onb.dst, onb.size)) == NULL) {
+ return lua_zstd_push_error(L, ZSTD_error_memory_allocation);
+ }
+
+ dlen = onb.size;
+
+ int res = ZSTD_decompressStream(ctx, &onb, &inb);
+
+ if (res == 0) {
+ /* All done */
+ break;
+ }
+
+ if ((err = ZSTD_getErrorCode(res))) {
+ break;
+ }
+
+ onb.size *= 2;
+ res += dlen; /* Hint returned by compression routine */
+
+ /* Either double the buffer, or use the hint provided */
+ if (onb.size < res) {
+ onb.size = res;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (err) {
+ return lua_zstd_push_error(L, err);
+ }
+
+ lua_new_text(L, onb.dst, onb.pos, TRUE);
+
+ return 1;
+}
+
+static gint
+lua_load_zstd(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, zstd_compress_lib_f);
+
+ return 1;
+}
+
+void luaopen_compress(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{zstd_compress}", zstd_compress_lib_m);
+ rspamd_lua_new_class(L, "rspamd{zstd_decompress}", zstd_decompress_lib_m);
+ lua_pop(L, 2);
+
+ rspamd_lua_add_preload(L, "rspamd_zstd", lua_load_zstd);
+}
diff --git a/src/lua/lua_compress.h b/src/lua/lua_compress.h
new file mode 100644
index 0000000..34234de
--- /dev/null
+++ b/src/lua/lua_compress.h
@@ -0,0 +1,37 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_LUA_COMPRESS_H
+#define RSPAMD_LUA_COMPRESS_H
+
+#include "lua_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+gint lua_compress_zstd_compress(lua_State *L);
+gint lua_compress_zstd_decompress(lua_State *L);
+gint lua_compress_zlib_compress(lua_State *L);
+gint lua_compress_zlib_decompress(lua_State *L, bool is_gzip);
+
+void luaopen_compress(lua_State *L);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif//RSPAMD_LUA_COMPRESS_H
diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c
new file mode 100644
index 0000000..a044827
--- /dev/null
+++ b/src/lua/lua_config.c
@@ -0,0 +1,4780 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "libmime/message.h"
+#include "libutil/expression.h"
+#include "src/libserver/composites/composites.h"
+#include "libserver/cfg_file_private.h"
+#include "libmime/lang_detection.h"
+#include "lua/lua_map.h"
+#include "lua/lua_thread_pool.h"
+#include "utlist.h"
+#include <math.h>
+
+/***
+ * This module is used to configure rspamd and is normally available as global
+ * variable named `rspamd_config`. Unlike other modules, it is not necessary to
+ * require it before usage.
+ * @module rspamd_config
+ * @example
+-- Register some callback symbol
+local function foo(task)
+ -- do something
+end
+rspamd_config:register_symbol('SYMBOL', 1.0, foo)
+
+-- Get configuration
+local tab = rspamd_config:get_all_opt('module') -- get table for module's options
+local opts = rspamd_config:get_key('options') -- get content of the specified key in rspamd configuration
+ */
+
+/* Config file methods */
+/***
+ * @method rspamd_config:get_module_opt(mname, optname)
+ * Returns value of specified option `optname` for a module `mname`,
+ * @param {string} mname name of module
+ * @param {string} optname option to get
+ * @return {string or table} value of the option or `nil` if option is not found
+ */
+LUA_FUNCTION_DEF(config, get_module_opt);
+/***
+ * @method rspamd_config:get_all_opt(mname)
+ * Returns value of all options for a module `mname`, flattening values into a single table consisting
+ * of all sections with such a name.
+ * @param {string} mname name of module
+ * @return {table} table of all options for `mname` or `nil` if a module's configuration is not found
+ */
+LUA_FUNCTION_DEF(config, get_all_opt);
+
+/***
+ * @method rspamd_config:get_ucl()
+ * Returns full configuration as a native Lua object (ucl to lua conversion).
+ * This method uses caching if possible.
+ * @return {table} table of all options in the configuration
+ */
+LUA_FUNCTION_DEF(config, get_ucl);
+/***
+ * @method rspamd_config:get_mempool()
+ * Returns static configuration memory pool.
+ * @return {mempool} [memory pool](mempool.md) object
+ */
+LUA_FUNCTION_DEF(config, get_mempool);
+/***
+ * @method rspamd_config:get_resolver()
+ * Returns DNS resolver.
+ * @return {dns_resolver} opaque DNS resolver pointer if any
+ */
+LUA_FUNCTION_DEF(config, get_resolver);
+/***
+ * @method rspamd_config:add_radix_map(mapline[, description])
+ * Creates new dynamic map of IP/mask addresses.
+ * @param {string} mapline URL for a map
+ * @param {string} description optional map description
+ * @return {map} radix tree object
+ * @example
+local ip_map = rspamd_config:add_radix_map ('file:///path/to/file', 'my radix map')
+...
+local function foo(task)
+ local ip = task:get_from_ip()
+ if ip_map:get_key(ip) then
+ return true
+ end
+ return false
+end
+ */
+
+/***
+ * @method rspamd_config:radix_from_config(mname, optname)
+ * Creates new embedded map of IP/mask addresses from config.
+ * @param {string} mname name of module
+ * @param {string} optname option to get
+ * @return {map} radix tree object
+ * @example
+local ip_map = rspamd_config:radix_from_config ('mymodule', 'ips')
+...
+local function foo(task)
+ local ip = task:get_from_ip()
+ if ip_map:get_key(ip) then
+ return true
+ end
+ return false
+end
+ */
+/***
+* @method rspamd_config:radix_from_ucl(obj)
+* Creates new embedded map of IP/mask addresses from object.
+* @param {ucl} obj object
+* @return {map} radix tree object
+*/
+/***
+ * @method rspamd_config:add_hash_map(mapline[, description])
+ * Creates new dynamic map string objects.
+ * @param {string} mapline URL for a map
+ * @param {string} description optional map description
+ * @return {map} hash set object
+ * @example
+local hash_map = rspamd_config:add_hash_map ('file:///path/to/file', 'my hash map')
+...
+local function foo(task)
+ local from = task:get_from()
+ if hash_map:get_key(from['user']) then
+ return true
+ end
+ return false
+end
+ */
+/***
+ * @method rspamd_config:add_kv_map(mapline[, description])
+ * Creates new dynamic map of key/values associations.
+ * @param {string} mapline URL for a map
+ * @param {string} description optional map description
+ * @return {map} hash table object
+ * @example
+local kv_map = rspamd_config:add_kv_map ('file:///path/to/file', 'my kv map')
+...
+local function foo(task)
+ local from = task:get_from()
+ if from then
+ local value = kv_map:get_key(from['user'])
+ if value then
+ return true,value
+ end
+ end
+ return false
+end
+ */
+/***
+ * @method rspamd_config:add_map({args})
+ * Creates new dynamic map according to the attributes passed.
+ *
+ * - `type`: type of map to be created, can be one of the following set:
+ * + `set`: set of strings
+ * + `radix`: map of IP addresses to strings
+ * + `map`: map of strings to strings
+ * + `regexp`: map of regexps to strings
+ * + `callback`: map processed by lua callback
+ * - `url`: url to load map from
+ * - `description`: map's description
+ * - `callback`: lua callback for the map
+ *
+ * @return {map} `true` if map has been added
+ * @example
+
+local str = ''
+local function process_map(in)
+ str = in
+end
+
+rspamd_config:add_map('http://example.com/map', "settings map", process_map)
+ */
+/***
+* @method rspamd_config:get_maps()
+* Get all maps defined as an array of rspamd{map} objects
+ *
+* @return {table|rspamd{map}}
+*/
+/***
+ * @method rspamd_config:get_classifier(name)
+ * Returns classifier config.
+ * @param {string} name name of classifier (e.g. `bayes`)
+ * @return {classifier} classifier object or `nil`
+ */
+LUA_FUNCTION_DEF(config, get_classifier);
+/***
+ * @method rspamd_config:register_symbol(table)
+ * Register symbol of a specified type in rspamd. This function accepts table of arguments:
+ *
+ * - `name`: name of symbol (can be missing for callback symbols)
+ * - `callback`: function to be called for symbol's check (can be absent for virtual symbols)
+ * - `weight`: weight of symbol (should normally be 1 or missing)
+ * - `priority`: priority of symbol (normally 0 or missing)
+ * - `type`: type of symbol:
+ * + `normal`: executed after prefilters, according to dependency graph or in undefined order
+ * + `callback`: a check that merely inserts virtual symbols
+ * + `connfilter`: executed early; before message body is available
+ * + `idempotent`: cannot change result in any way; executed last
+ * + `postfilter`: executed after most other checks
+ * + `prefilter`: executed before most other checks
+ * + `virtual`: a symbol inserted by its parent check
+ * - `flags`: various flags split by commas or spaces:
+ * + `nice` if symbol can produce negative score;
+ * + `empty` if symbol can be called for empty messages
+ * + `skip` if symbol should be skipped now
+ * + `nostat` if symbol should be excluded from stat tokens
+ * + `trivial` symbol is trivial (e.g. no network requests)
+ * + `explicit_disable` requires explicit disabling (e.g. via settings)
+ * + `ignore_passthrough` executed even if passthrough result has been set
+ * - `parent`: id of parent symbol (useful for virtual symbols)
+ *
+ * @return {number} id of symbol registered
+ */
+LUA_FUNCTION_DEF(config, register_symbol);
+/***
+ * @method rspamd_config:register_symbols(callback, [weight], callback_name, [, symbol, ...])
+ * Register callback function to be called for a set of symbols with initial weight.
+ * @param {function} callback callback function to be called for a specified symbol
+ * @param {number} weight initial weight of symbol (can be less than zero to specify non-spam symbols)
+ * @param {string} callback_name symbolic name of callback
+ * @param {list of strings} symbol list of symbols registered by this function
+ */
+LUA_FUNCTION_DEF(config, register_symbols);
+/***
+ * @method rspamd_config:register_virtual_symbol(name, weight,)
+ * Register virtual symbol that is not associated with any callback.
+ *
+ * **This method is deprecated and should not be used in newly written code **
+ * @param {string} virtual name symbol's name
+ * @param {number} weight initial weight of symbol (can be less than zero to specify non-spam symbols)
+ */
+LUA_FUNCTION_DEF(config, register_virtual_symbol);
+/***
+ * @method rspamd_config:register_callback_symbol(name, weight, callback)
+ * Register callback function to be called for a specified symbol with initial weight. Symbol itself is
+ * not registered in the metric and is not intended to be visible by a user.
+ *
+ * **This method is deprecated and should not be used in newly written code **
+ * @param {string} name symbol's name (just for unique id purposes)
+ * @param {number} weight initial weight of symbol (can be less than zero to specify non-spam symbols)
+ * @param {function} callback callback function to be called for a specified symbol
+ */
+LUA_FUNCTION_DEF(config, register_callback_symbol);
+LUA_FUNCTION_DEF(config, register_callback_symbol_priority);
+
+/***
+ * @method rspamd_config:register_dependency(id|name, depname)
+ * Create a dependency on symbol identified by name for symbol identified by ID or name.
+ * This affects order of checks only (a symbol is still checked if its dependencies are disabled).
+ * @param {number|string} id id or name of source (numeric id is returned by all register_*_symbol)
+ * @param {string} depname dependency name
+ * @example
+local function cb(task)
+...
+end
+
+local id = rspamd_config:register_symbol('SYM', 1.0, cb)
+rspamd_config:register_dependency(id, 'OTHER_SYM')
+-- Alternative form
+-- Symbol MY_RULE needs result from SPF_CHECK
+rspamd_config:register_dependency('MY_RULE', 'SPF_CHECK')
+ */
+LUA_FUNCTION_DEF(config, register_dependency);
+
+/***
+ * @method rspamd_config:get_symbol_flags(name)
+ * Returns symbol flags
+ * @param {string} name symbols's name
+ * @return {table|string} list of flags for symbol or nil
+ */
+LUA_FUNCTION_DEF(config, get_symbol_flags);
+
+/***
+ * @method rspamd_config:add_symbol_flags(name, flags)
+ * Adds flags to a symbol
+ * @param {string} name symbols's name
+ * @param {table|string} flags flags to add
+ * @return {table|string} new set of flags
+ */
+LUA_FUNCTION_DEF(config, add_symbol_flags);
+
+/**
+ * @method rspamd_config:register_re_selector(name, selector_str, [delimiter, [flatten]])
+ * Registers selector with the specific name to use in regular expressions in form
+ * name=/re/$ or name=/re/{selector}
+ * @param {string} name name of the selector
+ * @param {string} selector_str selector definition
+ * @param {string} delimiter delimiter to use when joining strings if flatten is false
+ * @param {bool} flatten if true then selector will return a table of captures instead of a single string
+ * @return true if selector has been registered
+ */
+LUA_FUNCTION_DEF(config, register_re_selector);
+
+/**
+ * @method rspamd_config:set_symbol({table})
+ * Sets the value of a specified symbol in a metric. This function accepts table with the following elements:
+ *
+ * - `name`: name of symbol (string)
+ * - `score`: score for symbol (number)
+ * - `metric`: name of metric (string, optional)
+ * - `description`: description of symbol (string, optional)
+ * - `group`: name of group for symbol (string, optional)
+ * - `one_shot`: turn off multiple hits for a symbol (boolean, optional)
+ * - `one_param`: turn off multiple options for a symbol (boolean, optional)
+ * - `flags`: comma separated string of flags:
+ * + `ignore`: do not strictly check validity of symbol and corresponding rule
+ * + `one_shot`: turn off multiple hits for a symbol
+ * + `one_param`: allow only one parameter for a symbol
+ * - `priority`: priority of symbol's definition
+ */
+LUA_FUNCTION_DEF(config, set_metric_symbol);
+
+/**
+ * @method rspamd_config:set_action({table})
+ * Sets the score of a specified action in a metric. This function accepts table with the following elements:
+ *
+ * - `action`: name of action (string)
+ * - `score`: score for action (number)
+ * - `metric`: name of metric (string, optional)
+ * - `priority`: priority of action's definition
+ */
+LUA_FUNCTION_DEF(config, set_metric_action);
+
+/**
+ * @method rspamd_config:get_action(name)
+ * Gets data for a specific action in config. This function returns number representing action's score
+ *
+ * @param {string} name name of action
+ * @return {number} action's score or nil in case of undefined score or action
+ */
+LUA_FUNCTION_DEF(config, get_metric_action);
+
+/**
+ * @method rspamd_config:get_all_actions()
+ * Gets data for all action in config
+ * @return {table|str->num} action's score or nil in case of undefined score or action
+ */
+LUA_FUNCTION_DEF(config, get_all_actions);
+
+/**
+ * @method rspamd_config:add_composite(name, expression)
+ * @param {string} name name of composite symbol
+ * @param {string} expression symbolic expression of the composite rule
+ * @return {bool} true if a composite has been added successfully
+ */
+LUA_FUNCTION_DEF(config, add_composite);
+/***
+ * @method rspamd_config:register_pre_filter(callback[, order])
+ * Register function to be called prior to symbols processing.
+ * @param {function} callback callback function
+ * @param {number} order filters are called from lower orders to higher orders, order is equal to 0 by default
+ * @example
+local function check_function(task)
+ -- It is possible to manipulate the task object here: set settings, set pre-action and so on
+ ...
+end
+
+rspamd_config:register_pre_filter(check_function)
+ */
+LUA_FUNCTION_DEF(config, register_pre_filter);
+/***
+ * @method rspamd_config:register_post_filter(callback[, order])
+ * Register function to be called after symbols are processed.
+ *
+ * @param {function} callback callback function
+ * @param {number} order filters are called from lower orders to higher orders, order is equal to 0 by default
+ */
+LUA_FUNCTION_DEF(config, register_post_filter);
+/* XXX: obsoleted */
+LUA_FUNCTION_DEF(config, register_module_option);
+/* XXX: not needed now */
+LUA_FUNCTION_DEF(config, get_api_version);
+/***
+ * @method rspamd_config:get_key(name)
+ * Returns configuration section with the specified `name`.
+ * @param {string} name name of config section
+ * @return {variant} specific value of section
+ * @example
+
+local set_section = rspamd_config:get_key("settings")
+if type(set_section) == "string" then
+ -- Just a map of ucl
+ if rspamd_config:add_map(set_section, "settings map", process_settings_map) then
+ rspamd_config:register_pre_filter(check_settings)
+ end
+elseif type(set_section) == "table" then
+ if process_settings_table(set_section) then
+ rspamd_config:register_pre_filter(check_settings)
+ end
+end
+ */
+LUA_FUNCTION_DEF(config, get_key);
+
+/***
+ * @method rspamd_config:add_condition(symbol, condition)
+ * Adds condition callback for specified symbol
+ * @param {string} symbol symbol's name
+ * @param {function} condition condition callback
+ * @return {boolean} true if condition has been added
+ * @example
+
+local condition_map = rspamd_config:add_map{
+ type = "hash",
+ urls = ['file:///path/to/file'],
+ description = 'SMTP from map that allows FUZZY_DENIED skip for the listed addresses'
+}
+rspamd_config:add_condition('FUZZY_DENIED', function(task)
+ local E = {}
+ -- Check for the smtp from address adding fail safe checks
+ if condition_map:find_key(((task:get_from('smtp') or E)[1] or E).addr) then
+ return false
+ end
+ -- Allow execution otherwise
+ return true
+end)
+ */
+LUA_FUNCTION_DEF(config, add_condition);
+
+/***
+ * @method rspamd_config:enable_symbol(symbol)
+ * Enables execution for the specified symbol
+ * @param {string} symbol symbol's name
+ */
+LUA_FUNCTION_DEF(config, enable_symbol);
+
+/***
+ * @method rspamd_config:disable_symbol(symbol, [disable_parent=true])
+ * Disables execution for the specified symbol
+ * @param {string} symbol symbol's name
+ * @param {boolean} disable_parent if true then disable parent execution in case of a virtual symbol
+ */
+LUA_FUNCTION_DEF(config, disable_symbol);
+
+/***
+ * @method rspamd_config:get_symbol_parent(symbol)
+ * Returns a parent symbol for specific symbol (or symbol itself if top level)
+ * @param {string} symbol symbol's name
+ */
+LUA_FUNCTION_DEF(config, get_symbol_parent);
+
+/***
+ * @method rspamd_config:get_group_symbols(group)
+ * Returns list of symbols for a specific group
+ * @param {string} group group's name
+ * @available 2.0+
+ * @return {list|string} list of all symbols in a specific group
+ */
+LUA_FUNCTION_DEF(config, get_group_symbols);
+
+/***
+ * @method rspamd_config:get_groups([need_private])
+ * Returns list of all groups defined
+ * @param {boolean} need_private optional flag to include private groups
+ * @available 2.3+
+ * @return {list|table} list of all groups
+ */
+LUA_FUNCTION_DEF(config, get_groups);
+
+/***
+ * @method rspamd_config:register_settings_id(name, symbols_enabled, symbols_disabled)
+ * Register new static settings id in config
+ * @param {string} name id name (not numeric!)
+ * @param {map|string->string} symbols_enabled map from symbol's name to boolean (currently)
+ * @param {map|string->string} symbols_disabled map from symbol's name to boolean (currently)
+ * @available 2.0+
+ */
+LUA_FUNCTION_DEF(config, register_settings_id);
+
+/***
+ * @method rspamd_config:__newindex(name, callback)
+ * This metamethod is called if new indices are added to the `rspamd_config` object.
+ * Technically, it is the equivalent of @see rspamd_config:register_symbol where `weight` is 1.0.
+ * There is also table form invocation that allows to control more things:
+ *
+ * - `callback`: has the same meaning and acts as function of task
+ * - `score`: default score for a symbol
+ * - `group`: default group for a symbol
+ * - `description`: default symbol's description
+ * - `priority`: additional priority value
+ * - `one_shot`: default value for one shot attribute
+ * - `condition`: function of task that can enable or disable this specific rule's execution
+ * @param {string} name index name
+ * @param {function/table} callback callback to be called
+ * @return {number} id of the new symbol added
+ * @example
+rspamd_config.R_EMPTY_IMAGE = function (task)
+ parts = task:get_text_parts()
+ if parts then
+ for _,part in ipairs(parts) do
+ if part:is_empty() then
+ images = task:get_images()
+ if images then
+ -- Symbol `R_EMPTY_IMAGE` is inserted
+ return true
+ end
+ return false
+ end
+ end
+ end
+ return false
+end
+
+rspamd_config.SYMBOL = {
+ callback = function(task)
+ ...
+ end,
+ score = 5.1,
+ description = 'sample symbol',
+ group = 'sample symbols',
+ condition = function(task)
+ if task:get_from()[1]['addr'] == 'user@example.com' then
+ return false
+ end
+ return true
+ end
+}
+ */
+LUA_FUNCTION_DEF(config, newindex);
+
+/***
+ * @method rspamd_config:register_regexp(params)
+ * Registers new re for further cached usage
+ * Params is the table with the following fields (mandatory fields are marked with `*`):
+ * - `re`* : regular expression object
+ * - `type`*: type of regular expression:
+ * + `mime`: mime regexp
+ * + `rawmime`: raw mime regexp
+ * + `header`: header regexp
+ * + `rawheader`: raw header expression
+ * + `body`: raw body regexp
+ * + `url`: url regexp
+ * - `header`: for header and rawheader regexp means the name of header
+ * - `pcre_only`: flag regexp as pcre only regexp
+ */
+LUA_FUNCTION_DEF(config, register_regexp);
+
+/***
+ * @method rspamd_config:replace_regexp(params)
+ * Replaces regexp with a new one
+ * Params is the table with the following fields (mandatory fields are marked with `*`):
+ * - `old_re`* : old regular expression object (must be in the cache)
+ * - `new_re`* : old regular expression object (must not be in the cache)
+ */
+LUA_FUNCTION_DEF(config, replace_regexp);
+
+/***
+ * @method rspamd_config:register_worker_script(worker_type, script)
+ * Registers the following script for workers of a specified type. The exact type
+ * of script function depends on worker type
+ * @param {string} worker_type worker type (e.g. "normal")
+ * @param {function} script script for a worker
+ * @return {boolean} `true` if a script has been registered
+ */
+LUA_FUNCTION_DEF(config, register_worker_script);
+
+/***
+ * @method rspamd_config:add_on_load(function(cfg, ev_base, worker) ... end)
+ * Registers the following script to be executed when configuration is completely loaded
+ * and the worker is already started (forked)
+ * @param {function} script function to be executed
+ * @example
+rspamd_config:add_on_load(function(cfg, ev_base, worker)
+ rspamd_config:add_periodic(ev_base, 1.0, function(cfg, ev_base)
+ local logger = require "rspamd_logger"
+ logger.infox(cfg, "periodic function in worker %s", worker:get_name())
+ return true
+ end)
+end)
+ */
+LUA_FUNCTION_DEF(config, add_on_load);
+
+/***
+ * @method rspamd_config:add_periodic(event_base, timeout, function(cfg, ev_base) ... end, [jitter = false])
+ * Registers function to be periodically executed by Rspamd
+ * @param {ev_base} event_base event base that is needed for async events
+ * @param {number} timeout time in seconds (could be fractional)
+ * @param {function} script function to be executed
+ * @param {boolean} jitter `true` if timeout jittering is needed
+ * @example
+rspamd_config:add_on_load(function(cfg, ev_base)
+ rspamd_config:add_periodic(ev_base, 1.0, function(cfg, ev_base)
+ local logger = require "rspamd_logger"
+ logger.infox(cfg, "periodic function")
+ return true -- if return numeric, a new interval is set. if return false, then the periodic event is removed
+ end)
+end)
+ */
+LUA_FUNCTION_DEF(config, add_periodic);
+
+/***
+ * @method rspamd_config:add_post_init(function(cfg) ... end)
+ * Registers the following script to be executed when configuration is completely loaded
+ * @available 2.0+
+ * @param {function} script function to be executed
+ */
+LUA_FUNCTION_DEF(config, add_post_init);
+
+/***
+ * @method rspamd_config:add_config_unload(function(cfg) ... end)
+ * Registers the following script to be executed when configuration is unloaded
+ * @available 2.0+
+ * @param {function} script function to be executed
+ */
+LUA_FUNCTION_DEF(config, add_config_unload);
+
+/***
+ * @method rspamd_config:get_symbols_count()
+ * Returns number of symbols registered in rspamd configuration
+ * @return {number} number of symbols registered in the configuration
+ */
+LUA_FUNCTION_DEF(config, get_symbols_count);
+
+/***
+ * @method rspamd_config:get_symbols_cksum()
+ * Returns checksum for all symbols in the cache
+ * @return {int64} boxed value of the 64 bit checksum
+ */
+LUA_FUNCTION_DEF(config, get_symbols_cksum);
+
+/***
+ * @method rspamd_config:get_symbols_counters()
+ * Returns table of all counters in the cache (weights, frequencies etc)
+ * @return {table|tables} all symbols indexed by name
+ */
+LUA_FUNCTION_DEF(config, get_symbols_counters);
+
+/***
+ * @method rspamd_config:get_symbols()
+ * Returns table of all scores defined in config. From version 2.0 returns table:
+ * - name
+ * - score
+ * - flags (e.g. `ignore` or `oneparam`)
+ * - nshots (== maxhits)
+ * - group - main group
+ * - groups - array of all groups
+ * @available 2.0+
+ * @return {table|tables} all symbols indexed by name
+ */
+LUA_FUNCTION_DEF(config, get_symbols);
+
+/***
+ * @method rspamd_config:get_symbol(sym_name)
+ * Returns table for a specific symbol getting data from the static config:
+ * - name
+ * - score
+ * - flags (e.g. `ignore` or `oneparam`)
+ * - nshots (== maxhits)
+ * - group - main group
+ * - groups - array of all groups
+ * @available 3.3+
+ * @return {table} symbol data (or nil)
+ */
+LUA_FUNCTION_DEF(config, get_symbol);
+
+/***
+ * @method rspamd_config:get_symbol_callback(name)
+ * Returns callback function for the specified symbol if it is a lua registered callback
+ * @return {function} callback function or nil
+ */
+LUA_FUNCTION_DEF(config, get_symbol_callback);
+
+/***
+ * @method rspamd_config:get_symbol_stat(name)
+ * Returns table with statistics for a specific symbol:
+ * - `frequency`: frequency for symbol's hits
+ * - `stddev`: standard deviation of `frequency`
+ * - `time`: average time in seconds (floating point)
+ * - `count`: total number of hits
+ * @return {table} symbol stats
+ */
+LUA_FUNCTION_DEF(config, get_symbol_stat);
+
+/***
+ * @method rspamd_config:set_symbol_callback(name, callback)
+ * Sets callback for the specified symbol
+ * @return {boolean} true if function has been replaced
+ */
+LUA_FUNCTION_DEF(config, set_symbol_callback);
+
+/***
+ * @method rspamd_config:register_finish_script(callback)
+ * Adds new callback that is called on worker process termination when all
+ * tasks pending are processed
+ *
+ * @param callback {function} a function with one argument (rspamd_task)
+ */
+LUA_FUNCTION_DEF(config, register_finish_script);
+
+/***
+ * @method rspamd_config:register_monitored(url, type, [{params}])
+ * Registers monitored resource to watch its availability. Supported types:
+ *
+ * - `dns`: DNS monitored object
+ *
+ * Params are optional table specific for each type. For DNS it supports the
+ * following options:
+ *
+ * - `prefix`: prefix to add before making request
+ * - `type`: type of request (e.g. 'a' or 'txt')
+ * - `ipnet`: array of ip/networks to expect on reply
+ * - `rcode`: expected return code (e.g. `nxdomain`)
+ *
+ * Returned object has the following methods:
+ *
+ * - `alive`: returns `true` if monitored resource is alive
+ * - `offline`: returns number of seconds of the current offline period (or 0 if alive)
+ * - `total_offline`: returns number of seconds of the overall offline
+ * - `latency`: returns the current average latency in seconds (or 0 if offline)
+ *
+ * @param {string} url resource to monitor
+ * @param {string} type type of monitoring
+ * @param {table} opts optional parameters
+ * @return {rspamd_monitored} rspamd monitored object
+ */
+LUA_FUNCTION_DEF(config, register_monitored);
+
+/***
+ * @method rspamd_config:add_doc(path, option, doc_string, [{params}])
+ * Adds new documentation string for an option `option` at path `path`
+ * Options defines optional params, such as:
+ *
+ * - `default`: default option value
+ * - `type`: type of an option (`string`, `number`, `object`, `array` etc)
+ * - `required`: if an option is required
+ *
+ * @param {string} path documentation path (e.g. module name)
+ * @param {string} option name of the option
+ * @param {string} doc_string documentation string
+ * @param {table} params optional parameters
+ */
+LUA_FUNCTION_DEF(config, add_doc);
+
+/***
+ * @method rspamd_config:add_example(path, option, doc_string, example)
+ * Adds new documentation
+ *
+ * @param {string} path documentation path (e.g. module name or nil for top)
+ * @param {string} option name of the option
+ * @param {string} doc_string documentation string
+ * @param {string} example example in ucl format, comments are also parsed
+ */
+LUA_FUNCTION_DEF(config, add_example);
+
+/***
+ * @method rspamd_config:set_peak_cb(function)
+ * Sets a function that will be called when frequency of some symbol goes out of
+ * stddev * 2 over the last period of refreshment.
+ *
+ * @example
+rspamd_config:set_peak_cb(function(ev_base, sym, mean, stddev, value, error)
+ -- ev_base: event base for async events (e.g. redis)
+ -- sym: symbol's name
+ -- mean: mean frequency value
+ -- stddev: standard deviation of frequency
+ -- value: current frequency value
+ -- error: squared error
+ local logger = require "rspamd_logger"
+ logger.infox(rspamd_config, "symbol %s has changed frequency significantly: %s(%s) over %s(%s)",
+ sym, value, error, mean, stddev)
+end)
+ */
+LUA_FUNCTION_DEF(config, set_peak_cb);
+/***
+ * @method rspamd_config:get_cpu_flags()
+ * Returns architecture dependent flags supported by the CPU
+ * Currently, only x86 flags are supported:
+ * - 'ssse3'
+ * - 'sse42'
+ * - 'avx'
+ * - 'avx2'
+ * @return {table} flag -> true table
+ */
+LUA_FUNCTION_DEF(config, get_cpu_flags);
+
+/***
+ * @method rspamd_config:has_torch()
+ * Returns true if Rspamd is compiled with torch support and the runtime CPU
+ * supports sse4.2 required for torch.
+ * @return {boolean} true if torch is compiled and supported
+ */
+LUA_FUNCTION_DEF(config, has_torch);
+
+/***
+ * @method rspamd_config:experimental_enabled()
+ * Returns true if experimental plugins are enabled
+ * @return {boolean} true if experimental plugins are enabled
+ */
+LUA_FUNCTION_DEF(config, experimental_enabled);
+
+/***
+ * @method rspamd_config:load_ucl(filename[, include_trace])
+ * Loads config from the UCL file (but does not perform parsing using rcl)
+ * @param {string} filename file to load
+ * @return true or false + error message
+ */
+LUA_FUNCTION_DEF(config, load_ucl);
+
+/***
+ * @method rspamd_config:parse_rcl([skip_sections])
+ * Parses RCL using loaded ucl file
+ * @param {table|string} sections to skip
+ * @return true or false + error message
+ */
+LUA_FUNCTION_DEF(config, parse_rcl);
+
+/***
+ * @method rspamd_config:init_modules()
+ * Initialize lua and internal modules
+ * @return true or false
+ */
+LUA_FUNCTION_DEF(config, init_modules);
+
+/***
+ * @method rspamd_config:init_subsystem(str)
+ * Initialize config subsystem from a comma separated list:
+ * - `modules` - init modules
+ * - `langdet` - language detector
+ * - `dns` - DNS resolver
+ * - TODO: add more
+ */
+LUA_FUNCTION_DEF(config, init_subsystem);
+
+/***
+ * @method rspamd_config:get_tld_path()
+ * Returns path to TLD file
+ * @return {string} path to tld file
+ */
+LUA_FUNCTION_DEF(config, get_tld_path);
+
+/***
+ * @method rspamd_config:get_dns_max_requests()
+ * Returns limit of DNS requests per task
+ * @return {number} number of dns requests allowed
+ */
+LUA_FUNCTION_DEF(config, get_dns_max_requests);
+
+/***
+ * @method rspamd_config:get_dns_timeout()
+ * Returns timeout for a DNS request
+ * @return {number} DNS timeout in second or 0 if not defined
+ */
+LUA_FUNCTION_DEF(config, get_dns_timeout);
+
+static const struct luaL_reg configlib_m[] = {
+ LUA_INTERFACE_DEF(config, get_module_opt),
+ LUA_INTERFACE_DEF(config, get_mempool),
+ LUA_INTERFACE_DEF(config, get_resolver),
+ LUA_INTERFACE_DEF(config, get_all_opt),
+ LUA_INTERFACE_DEF(config, get_ucl),
+ LUA_INTERFACE_DEF(config, add_radix_map),
+ LUA_INTERFACE_DEF(config, radix_from_config),
+ LUA_INTERFACE_DEF(config, radix_from_ucl),
+ LUA_INTERFACE_DEF(config, add_hash_map),
+ LUA_INTERFACE_DEF(config, add_kv_map),
+ LUA_INTERFACE_DEF(config, add_map),
+ LUA_INTERFACE_DEF(config, get_maps),
+ LUA_INTERFACE_DEF(config, get_classifier),
+ LUA_INTERFACE_DEF(config, register_symbol),
+ LUA_INTERFACE_DEF(config, register_symbols),
+ LUA_INTERFACE_DEF(config, register_virtual_symbol),
+ LUA_INTERFACE_DEF(config, register_callback_symbol),
+ LUA_INTERFACE_DEF(config, register_callback_symbol_priority),
+ LUA_INTERFACE_DEF(config, register_dependency),
+ LUA_INTERFACE_DEF(config, register_settings_id),
+ LUA_INTERFACE_DEF(config, get_symbol_flags),
+ LUA_INTERFACE_DEF(config, set_metric_symbol),
+ {"set_symbol", lua_config_set_metric_symbol},
+ LUA_INTERFACE_DEF(config, set_metric_action),
+ {"set_action", lua_config_set_metric_action},
+ {"get_metric_symbol", lua_config_get_symbol},
+ LUA_INTERFACE_DEF(config, get_metric_action),
+ {"get_action", lua_config_get_metric_action},
+ LUA_INTERFACE_DEF(config, get_all_actions),
+ LUA_INTERFACE_DEF(config, add_composite),
+ LUA_INTERFACE_DEF(config, register_module_option),
+ LUA_INTERFACE_DEF(config, register_pre_filter),
+ LUA_INTERFACE_DEF(config, register_post_filter),
+ LUA_INTERFACE_DEF(config, get_api_version),
+ LUA_INTERFACE_DEF(config, get_key),
+ LUA_INTERFACE_DEF(config, add_condition),
+ LUA_INTERFACE_DEF(config, enable_symbol),
+ LUA_INTERFACE_DEF(config, disable_symbol),
+ LUA_INTERFACE_DEF(config, register_regexp),
+ LUA_INTERFACE_DEF(config, replace_regexp),
+ LUA_INTERFACE_DEF(config, register_worker_script),
+ LUA_INTERFACE_DEF(config, register_re_selector),
+ LUA_INTERFACE_DEF(config, add_on_load),
+ LUA_INTERFACE_DEF(config, add_periodic),
+ LUA_INTERFACE_DEF(config, add_post_init),
+ LUA_INTERFACE_DEF(config, add_config_unload),
+ LUA_INTERFACE_DEF(config, get_symbols_count),
+ LUA_INTERFACE_DEF(config, get_symbols_cksum),
+ LUA_INTERFACE_DEF(config, get_symbols_counters),
+ {"get_symbols_scores", lua_config_get_symbols},
+ LUA_INTERFACE_DEF(config, get_symbols),
+ LUA_INTERFACE_DEF(config, get_symbol),
+ LUA_INTERFACE_DEF(config, get_groups),
+ LUA_INTERFACE_DEF(config, get_symbol_callback),
+ LUA_INTERFACE_DEF(config, set_symbol_callback),
+ LUA_INTERFACE_DEF(config, get_symbol_stat),
+ LUA_INTERFACE_DEF(config, get_symbol_parent),
+ LUA_INTERFACE_DEF(config, get_group_symbols),
+ LUA_INTERFACE_DEF(config, register_finish_script),
+ LUA_INTERFACE_DEF(config, register_monitored),
+ LUA_INTERFACE_DEF(config, add_doc),
+ LUA_INTERFACE_DEF(config, add_example),
+ LUA_INTERFACE_DEF(config, set_peak_cb),
+ LUA_INTERFACE_DEF(config, get_cpu_flags),
+ LUA_INTERFACE_DEF(config, has_torch),
+ LUA_INTERFACE_DEF(config, experimental_enabled),
+ LUA_INTERFACE_DEF(config, load_ucl),
+ LUA_INTERFACE_DEF(config, parse_rcl),
+ LUA_INTERFACE_DEF(config, init_modules),
+ LUA_INTERFACE_DEF(config, init_subsystem),
+ LUA_INTERFACE_DEF(config, get_tld_path),
+ LUA_INTERFACE_DEF(config, get_dns_max_requests),
+ LUA_INTERFACE_DEF(config, get_dns_timeout),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__newindex", lua_config_newindex},
+ {NULL, NULL}};
+
+LUA_FUNCTION_DEF(monitored, alive);
+LUA_FUNCTION_DEF(monitored, latency);
+LUA_FUNCTION_DEF(monitored, offline);
+LUA_FUNCTION_DEF(monitored, total_offline);
+
+static const struct luaL_reg monitoredlib_m[] = {
+ LUA_INTERFACE_DEF(monitored, alive),
+ LUA_INTERFACE_DEF(monitored, latency),
+ LUA_INTERFACE_DEF(monitored, offline),
+ LUA_INTERFACE_DEF(monitored, total_offline),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static const guint64 rspamd_lua_callback_magic = 0x32c118af1e3263c7ULL;
+
+struct rspamd_config *
+lua_check_config(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{config}");
+ luaL_argcheck(L, ud != NULL, pos, "'config' expected");
+ return ud ? *((struct rspamd_config **) ud) : NULL;
+}
+
+static struct rspamd_monitored *
+lua_check_monitored(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{monitored}");
+ luaL_argcheck(L, ud != NULL, pos, "'monitored' expected");
+ return ud ? *((struct rspamd_monitored **) ud) : NULL;
+}
+
+/*** Config functions ***/
+static gint
+lua_config_get_api_version(lua_State *L)
+{
+ msg_warn("get_api_version is deprecated, do not use it");
+ lua_pushnumber(L, 100);
+
+ return 1;
+}
+
+static gint
+lua_config_get_module_opt(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *mname, *optname;
+ const ucl_object_t *obj;
+
+ if (cfg) {
+ mname = luaL_checkstring(L, 2);
+ optname = luaL_checkstring(L, 3);
+
+ if (mname && optname) {
+ obj = rspamd_config_get_module_opt(cfg, mname, optname);
+ if (obj) {
+ return ucl_object_push_lua(L, obj, TRUE);
+ }
+ }
+ }
+ lua_pushnil(L);
+ return 1;
+}
+
+static int
+lua_config_get_mempool(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_mempool_t **ppool;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ ppool = lua_newuserdata(L, sizeof(rspamd_mempool_t *));
+ rspamd_lua_setclass(L, "rspamd{mempool}", -1);
+ *ppool = cfg->cfg_pool;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static int
+lua_config_get_resolver(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_dns_resolver **pres;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL && cfg->dns_resolver) {
+ pres = lua_newuserdata(L, sizeof(*pres));
+ rspamd_lua_setclass(L, "rspamd{resolver}", -1);
+ *pres = cfg->dns_resolver;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_all_opt(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *mname;
+ const ucl_object_t *obj, *cur, *cur_elt;
+ ucl_object_iter_t it = NULL;
+ gint i;
+
+ if (cfg) {
+ mname = luaL_checkstring(L, 2);
+
+ if (mname) {
+ obj = ucl_obj_get_key(cfg->cfg_ucl_obj, mname);
+ /* Flatten object */
+ if (obj != NULL && (ucl_object_type(obj) == UCL_OBJECT ||
+ ucl_object_type(obj) == UCL_ARRAY)) {
+
+ lua_newtable(L);
+ it = ucl_object_iterate_new(obj);
+
+ LL_FOREACH(obj, cur)
+ {
+ it = ucl_object_iterate_reset(it, cur);
+
+ while ((cur_elt = ucl_object_iterate_safe(it, true))) {
+ lua_pushstring(L, ucl_object_key(cur_elt));
+ ucl_object_push_lua(L, cur_elt, true);
+ lua_settable(L, -3);
+ }
+ }
+
+ ucl_object_iterate_free(it);
+
+ return 1;
+ }
+ else if (obj != NULL) {
+ lua_newtable(L);
+ i = 1;
+
+ LL_FOREACH(obj, cur)
+ {
+ lua_pushinteger(L, i++);
+ ucl_object_push_lua(L, cur, true);
+ lua_settable(L, -3);
+ }
+
+ return 1;
+ }
+ }
+ }
+ lua_pushnil(L);
+
+ return 1;
+}
+
+struct rspamd_lua_cached_config {
+ lua_State *L;
+ gint ref;
+};
+
+static void
+lua_config_ucl_dtor(gpointer p)
+{
+ struct rspamd_lua_cached_config *cached = p;
+
+ luaL_unref(cached->L, LUA_REGISTRYINDEX, cached->ref);
+}
+
+static gint
+lua_config_get_ucl(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_lua_cached_config *cached;
+
+ if (cfg) {
+ cached = rspamd_mempool_get_variable(cfg->cfg_pool, "ucl_cached");
+
+ if (cached) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cached->ref);
+ }
+ else {
+ if (cfg->cfg_ucl_obj) {
+ ucl_object_push_lua(L, cfg->cfg_ucl_obj, true);
+ lua_pushvalue(L, -1);
+ cached = rspamd_mempool_alloc(cfg->cfg_pool, sizeof(*cached));
+ cached->L = L;
+ cached->ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ rspamd_mempool_set_variable(cfg->cfg_pool, "ucl_cached",
+ cached, lua_config_ucl_dtor);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_config_get_classifier(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_classifier_config *clc = NULL, **pclc = NULL;
+ const gchar *name;
+ GList *cur;
+
+ if (cfg) {
+ name = luaL_checkstring(L, 2);
+
+ cur = g_list_first(cfg->classifiers);
+ while (cur) {
+ clc = cur->data;
+ if (g_ascii_strcasecmp(clc->name, name) == 0) {
+ pclc = &clc;
+ break;
+ }
+ cur = g_list_next(cur);
+ }
+ if (pclc) {
+ pclc = lua_newuserdata(L,
+ sizeof(struct rspamd_classifier_config *));
+ rspamd_lua_setclass(L, "rspamd{classifier}", -1);
+ *pclc = clc;
+ return 1;
+ }
+ }
+
+ lua_pushnil(L);
+ return 1;
+}
+
+struct lua_callback_data {
+ guint64 magic;
+ lua_State *L;
+ gchar *symbol;
+
+ union {
+ gchar *name;
+ gint ref;
+ } callback;
+ gboolean cb_is_ref;
+
+ /* Dynamic data */
+ gint stack_level;
+ gint order;
+ struct rspamd_symcache_dynamic_item *item;
+};
+
+/*
+ * Unref symbol if it is local reference
+ */
+static void
+lua_destroy_cfg_symbol(gpointer ud)
+{
+ struct lua_callback_data *cd = ud;
+
+ /* Unref callback */
+ if (cd->cb_is_ref) {
+ luaL_unref(cd->L, LUA_REGISTRYINDEX, cd->callback.ref);
+ }
+}
+
+static gint
+lua_config_register_module_option(lua_State *L)
+{
+ return 0;
+}
+
+static gint
+rspamd_compare_order_func(gconstpointer a, gconstpointer b)
+{
+ const struct lua_callback_data *cb1 = a, *cb2 = b;
+
+ /* order of call goes from lower to higher */
+ return cb2->order - cb1->order;
+}
+
+static void
+lua_metric_symbol_callback(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ gpointer ud)
+{
+ struct lua_callback_data *cd = ud;
+ struct rspamd_task **ptask;
+ gint level = lua_gettop(cd->L), nresults, err_idx, ret;
+ lua_State *L = cd->L;
+ struct rspamd_symbol_result *s;
+
+ cd->item = item;
+ rspamd_symcache_item_async_inc(task, item, "lua symbol");
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ level++;
+
+ if (cd->cb_is_ref) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cd->callback.ref);
+ }
+ else {
+ lua_getglobal(L, cd->callback.name);
+ }
+
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+
+ if ((ret = lua_pcall(L, 1, LUA_MULTRET, err_idx)) != 0) {
+ msg_err_task("call to (%s) failed (%d): %s", cd->symbol, ret,
+ lua_tostring(L, -1));
+ lua_settop(L, err_idx); /* Not -1 here, as err_func is popped below */
+ }
+ else {
+ nresults = lua_gettop(L) - level;
+
+ if (nresults >= 1) {
+ /* Function returned boolean, so maybe we need to insert result? */
+ gint res = 0;
+ gint i;
+ gdouble flag = 1.0;
+ gint type;
+
+ type = lua_type(cd->L, level + 1);
+
+ if (type == LUA_TBOOLEAN) {
+ res = lua_toboolean(L, level + 1);
+ }
+ else if (type == LUA_TNUMBER) {
+ res = lua_tonumber(L, level + 1);
+ }
+ else if (type == LUA_TNIL) {
+ /* Can happen sometimes... */
+ res = FALSE;
+ }
+ else {
+ /* Something bogus has been returned, so we should log it */
+ msg_err_task("invalid return value for %s: %s",
+ cd->symbol, lua_typename(L, type));
+ res = FALSE;
+ }
+
+ if (res) {
+ gint first_opt = 2;
+
+ if (lua_type(L, level + 2) == LUA_TNUMBER) {
+ flag = lua_tonumber(L, level + 2);
+ /* Shift opt index */
+ first_opt = 3;
+ }
+ else {
+ flag = res;
+ }
+
+ s = rspamd_task_insert_result(task, cd->symbol, flag, NULL);
+
+ if (s) {
+ guint last_pos = lua_gettop(L);
+
+ for (i = level + first_opt; i <= last_pos; i++) {
+ if (lua_type(L, i) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, i, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, i) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, i);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+ else if (lua_type(L, i) == LUA_TTABLE) {
+ gsize objlen = rspamd_lua_table_size(L, i);
+
+ for (guint j = 1; j <= objlen; j++) {
+ lua_rawgeti(L, i, j);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, -1, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, -1);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ }
+ }
+ }
+
+ lua_pop(L, nresults);
+ }
+ }
+
+ lua_pop(L, 1); /* Error function */
+ rspamd_symcache_item_async_dec_check(task, cd->item, "lua symbol");
+ g_assert(lua_gettop(L) == level - 1);
+}
+
+static void lua_metric_symbol_callback_return(struct thread_entry *thread_entry,
+ int ret);
+
+static void lua_metric_symbol_callback_error(struct thread_entry *thread_entry,
+ int ret,
+ const char *msg);
+
+static void
+lua_metric_symbol_callback_coro(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ gpointer ud)
+{
+ struct lua_callback_data *cd = ud;
+ struct rspamd_task **ptask;
+ struct thread_entry *thread_entry;
+
+ cd->item = item;
+ rspamd_symcache_item_async_inc(task, item, "lua coro symbol");
+ thread_entry = lua_thread_pool_get_for_task(task);
+
+ g_assert(thread_entry->cd == NULL);
+ thread_entry->cd = cd;
+
+ lua_State *thread = thread_entry->lua_state;
+ cd->stack_level = lua_gettop(thread);
+
+ if (cd->cb_is_ref) {
+ lua_rawgeti(thread, LUA_REGISTRYINDEX, cd->callback.ref);
+ }
+ else {
+ lua_getglobal(thread, cd->callback.name);
+ }
+
+ ptask = lua_newuserdata(thread, sizeof(struct rspamd_task *));
+ rspamd_lua_setclass(thread, "rspamd{task}", -1);
+ *ptask = task;
+
+ thread_entry->finish_callback = lua_metric_symbol_callback_return;
+ thread_entry->error_callback = lua_metric_symbol_callback_error;
+
+ lua_thread_call(thread_entry, 1);
+}
+
+static void
+lua_metric_symbol_callback_error(struct thread_entry *thread_entry,
+ int ret,
+ const char *msg)
+{
+ struct lua_callback_data *cd = thread_entry->cd;
+ struct rspamd_task *task = thread_entry->task;
+ msg_err_task("call to coroutine (%s) failed (%d): %s", cd->symbol, ret, msg);
+
+ rspamd_symcache_item_async_dec_check(task, cd->item, "lua coro symbol");
+}
+
+static void
+lua_metric_symbol_callback_return(struct thread_entry *thread_entry, int ret)
+{
+ struct lua_callback_data *cd = thread_entry->cd;
+ struct rspamd_task *task = thread_entry->task;
+ int nresults;
+ struct rspamd_symbol_result *s;
+
+ (void) ret;
+
+ lua_State *L = thread_entry->lua_state;
+
+ nresults = lua_gettop(L) - cd->stack_level;
+
+ if (nresults >= 1) {
+ /* Function returned boolean, so maybe we need to insert result? */
+ gint res = 0;
+ gint i;
+ gdouble flag = 1.0;
+ gint type;
+
+ type = lua_type(L, cd->stack_level + 1);
+
+ if (type == LUA_TBOOLEAN) {
+ res = lua_toboolean(L, cd->stack_level + 1);
+ }
+ else if (type == LUA_TFUNCTION) {
+ g_assert_not_reached();
+ }
+ else {
+ res = lua_tonumber(L, cd->stack_level + 1);
+ }
+
+ if (res) {
+ gint first_opt = 2;
+
+ if (lua_type(L, cd->stack_level + 2) == LUA_TNUMBER) {
+ flag = lua_tonumber(L, cd->stack_level + 2);
+ /* Shift opt index */
+ first_opt = 3;
+ }
+ else {
+ flag = res;
+ }
+
+ s = rspamd_task_insert_result(task, cd->symbol, flag, NULL);
+
+ if (s) {
+ guint last_pos = lua_gettop(L);
+
+ for (i = cd->stack_level + first_opt; i <= last_pos; i++) {
+ if (lua_type(L, i) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, i, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, i) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, i);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+ else if (lua_type(L, i) == LUA_TTABLE) {
+ gsize objlen = rspamd_lua_table_size(L, i);
+
+ for (guint j = 1; j <= objlen; j++) {
+ lua_rawgeti(L, i, j);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, -1, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, -1);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ }
+ }
+ }
+
+ lua_pop(L, nresults);
+ }
+
+ g_assert(lua_gettop(L) == cd->stack_level); /* we properly cleaned up the stack */
+
+ cd->stack_level = 0;
+ rspamd_symcache_item_async_dec_check(task, cd->item, "lua coro symbol");
+}
+
+static GArray *
+rspamd_process_id_list(const gchar *entries)
+{
+ gchar **sym_elts;
+ GArray *ret;
+
+ sym_elts = g_strsplit_set(entries, ",;", -1);
+ guint nids = g_strv_length(sym_elts);
+ ret = g_array_sized_new(FALSE, FALSE, sizeof(guint32), nids);
+
+ for (guint i = 0; i < nids; i++) {
+ guint32 v = rspamd_config_name_to_id(sym_elts[i], strlen(sym_elts[i]));
+ g_array_append_val(ret, v);
+ }
+
+ g_strfreev(sym_elts);
+
+ return ret;
+}
+
+static gint
+rspamd_register_symbol_fromlua(lua_State *L,
+ struct rspamd_config *cfg,
+ const gchar *name,
+ gint ref,
+ gdouble weight,
+ gint priority,
+ enum rspamd_symbol_type type,
+ gint parent,
+ GArray *allowed_ids,
+ GArray *forbidden_ids,
+ gboolean optional)
+{
+ struct lua_callback_data *cd;
+ gint ret = -1;
+
+ if (priority == 0 && weight < 0) {
+ priority = 1;
+ }
+
+ if ((ret = rspamd_symcache_find_symbol(cfg->cache, name)) != -1) {
+ if (optional) {
+ msg_debug_config("duplicate symbol: %s, skip registering", name);
+
+ return ret;
+ }
+ else {
+ msg_err_config("duplicate symbol: %s, skip registering", name);
+
+ return -1;
+ }
+ }
+
+ if (allowed_ids && !(type & SYMBOL_TYPE_EXPLICIT_DISABLE)) {
+ /* Mark symbol as explicit allow */
+ msg_info_config("mark symbol %s as explicit enable as its execution is"
+ "allowed merely on specific settings ids",
+ name);
+ type |= SYMBOL_TYPE_EXPLICIT_ENABLE;
+ }
+
+ if (ref != -1) {
+ cd = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(struct lua_callback_data));
+ cd->magic = rspamd_lua_callback_magic;
+ cd->cb_is_ref = TRUE;
+ cd->callback.ref = ref;
+ cd->L = L;
+
+ if (name) {
+ cd->symbol = rspamd_mempool_strdup(cfg->cfg_pool, name);
+ }
+
+ if (type & SYMBOL_TYPE_USE_CORO) {
+ ret = rspamd_symcache_add_symbol(cfg->cache,
+ name,
+ priority,
+ lua_metric_symbol_callback_coro,
+ cd,
+ type,
+ parent);
+ }
+ else {
+ ret = rspamd_symcache_add_symbol(cfg->cache,
+ name,
+ priority,
+ lua_metric_symbol_callback,
+ cd,
+ type,
+ parent);
+ }
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) lua_destroy_cfg_symbol,
+ cd);
+ }
+ else {
+ /* No callback */
+ ret = rspamd_symcache_add_symbol(cfg->cache,
+ name,
+ priority,
+ NULL,
+ NULL,
+ type,
+ parent);
+ }
+
+ if (allowed_ids) {
+ rspamd_symcache_set_allowed_settings_ids(cfg->cache, name,
+ &g_array_index(allowed_ids, guint32, 0), allowed_ids->len);
+ }
+
+ if (forbidden_ids) {
+ rspamd_symcache_set_forbidden_settings_ids(cfg->cache, name,
+ &g_array_index(forbidden_ids, guint32, 0), forbidden_ids->len);
+ }
+
+ return ret;
+}
+
+static gint
+lua_config_register_post_filter(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gint order = 0, cbref, ret;
+
+ if (cfg) {
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ order = lua_tonumber(L, 3);
+ }
+
+ if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ /* Get a reference */
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ return luaL_error(L, "invalid type for callback: %s",
+ lua_typename(L, lua_type(L, 2)));
+ }
+
+ msg_warn_config("register_post_filter function is deprecated, "
+ "use register_symbol instead");
+
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ NULL,
+ cbref,
+ 1.0,
+ order,
+ SYMBOL_TYPE_POSTFILTER | SYMBOL_TYPE_CALLBACK,
+ -1,
+ NULL, NULL,
+ FALSE);
+
+ lua_pushboolean(L, ret);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_register_pre_filter(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gint order = 0, cbref, ret;
+
+ if (cfg) {
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ order = lua_tonumber(L, 3);
+ }
+
+ if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ /* Get a reference */
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ return luaL_error(L, "invalid type for callback: %s",
+ lua_typename(L, lua_type(L, 2)));
+ }
+
+ msg_warn_config("register_pre_filter function is deprecated, "
+ "use register_symbol instead");
+
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ NULL,
+ cbref,
+ 1.0,
+ order,
+ SYMBOL_TYPE_PREFILTER | SYMBOL_TYPE_CALLBACK,
+ -1,
+ NULL, NULL,
+ FALSE);
+
+ lua_pushboolean(L, ret);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_key(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name;
+ size_t namelen;
+ const ucl_object_t *val;
+
+ name = luaL_checklstring(L, 2, &namelen);
+ if (name && cfg) {
+ val = ucl_object_lookup_len(cfg->cfg_ucl_obj, name, namelen);
+ if (val != NULL) {
+ ucl_object_push_lua(L, val, val->type != UCL_ARRAY);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static guint
+lua_parse_symbol_flags(const gchar *str)
+{
+ guint ret = 0;
+
+ if (str) {
+ if (strstr(str, "fine") != NULL) {
+ ret |= SYMBOL_TYPE_FINE;
+ }
+ if (strstr(str, "nice") != NULL) {
+ ret |= SYMBOL_TYPE_FINE;
+ }
+ if (strstr(str, "empty") != NULL) {
+ ret |= SYMBOL_TYPE_EMPTY;
+ }
+ if (strstr(str, "skip") != NULL) {
+ ret |= SYMBOL_TYPE_SKIPPED;
+ }
+ if (strstr(str, "nostat") != NULL) {
+ ret |= SYMBOL_TYPE_NOSTAT;
+ }
+ if (strstr(str, "idempotent") != NULL) {
+ ret |= SYMBOL_TYPE_IDEMPOTENT;
+ }
+ if (strstr(str, "trivial") != NULL) {
+ ret |= SYMBOL_TYPE_TRIVIAL;
+ }
+ if (strstr(str, "ghost") != NULL) {
+ ret |= SYMBOL_TYPE_GHOST;
+ }
+ if (strstr(str, "mime") != NULL) {
+ ret |= SYMBOL_TYPE_MIME_ONLY;
+ }
+ if (strstr(str, "ignore_passthrough") != NULL) {
+ ret |= SYMBOL_TYPE_IGNORE_PASSTHROUGH;
+ }
+ if (strstr(str, "explicit_disable") != NULL) {
+ ret |= SYMBOL_TYPE_EXPLICIT_DISABLE;
+ }
+ if (strstr(str, "explicit_enable") != NULL) {
+ ret |= SYMBOL_TYPE_EXPLICIT_ENABLE;
+ }
+ if (strstr(str, "coro") != NULL) {
+ ret |= SYMBOL_TYPE_USE_CORO;
+ }
+ }
+
+ return ret;
+}
+
+static guint
+lua_parse_symbol_type(const gchar *str)
+{
+ guint ret = SYMBOL_TYPE_NORMAL;
+ gchar **vec;
+ guint i, l;
+
+ if (str) {
+ vec = g_strsplit_set(str, ",;", -1);
+
+ if (vec) {
+ l = g_strv_length(vec);
+
+ for (i = 0; i < l; i++) {
+ str = vec[i];
+
+ /* TODO: total shit, rework some day */
+ if (g_ascii_strcasecmp(str, "virtual") == 0) {
+ ret |= SYMBOL_TYPE_VIRTUAL;
+ ret &= ~SYMBOL_TYPE_NORMAL;
+ ret &= ~SYMBOL_TYPE_CALLBACK;
+ }
+ else if (g_ascii_strcasecmp(str, "callback") == 0) {
+ ret |= SYMBOL_TYPE_CALLBACK;
+ ret &= ~SYMBOL_TYPE_NORMAL;
+ ret &= ~SYMBOL_TYPE_VIRTUAL;
+ }
+ else if (g_ascii_strcasecmp(str, "normal") == 0) {
+ ret |= SYMBOL_TYPE_NORMAL;
+ ret &= ~SYMBOL_TYPE_CALLBACK;
+ ret &= ~SYMBOL_TYPE_VIRTUAL;
+ }
+ else if (g_ascii_strcasecmp(str, "prefilter") == 0) {
+ ret |= SYMBOL_TYPE_PREFILTER | SYMBOL_TYPE_GHOST;
+ }
+ else if (g_ascii_strcasecmp(str, "postfilter") == 0) {
+ ret |= SYMBOL_TYPE_POSTFILTER | SYMBOL_TYPE_GHOST;
+ }
+ else if (g_ascii_strcasecmp(str, "connfilter") == 0 ||
+ g_ascii_strcasecmp(str, "conn_filter") == 0) {
+ ret |= SYMBOL_TYPE_CONNFILTER | SYMBOL_TYPE_GHOST;
+ }
+ else if (g_ascii_strcasecmp(str, "idempotent") == 0) {
+ ret |= SYMBOL_TYPE_GHOST |
+ SYMBOL_TYPE_IDEMPOTENT | SYMBOL_TYPE_CALLBACK;
+ }
+ else {
+ gint fl = 0;
+
+ fl = lua_parse_symbol_flags(str);
+
+ if (fl == 0) {
+ msg_warn("bad type: %s", str);
+ }
+ else {
+ ret |= fl;
+ }
+ }
+ }
+
+ g_strfreev(vec);
+ }
+ }
+
+ return ret;
+}
+
+enum lua_push_symbol_flags_opts {
+ LUA_SYMOPT_FLAG_CREATE_ARRAY = 1u << 0u,
+ LUA_SYMOPT_FLAG_CREATE_MAP = 1u << 1u,
+ LUA_SYMOPT_FLAG_USE_MAP = 1u << 2u,
+ LUA_SYMOPT_FLAG_USE_ARRAY = 1u << 3u,
+};
+
+#define LUA_SYMOPT_IS_ARRAY(f) ((f) & (LUA_SYMOPT_FLAG_CREATE_ARRAY | LUA_SYMOPT_FLAG_USE_ARRAY))
+#define LUA_SYMOPT_IS_CREATE(f) ((f) & (LUA_SYMOPT_FLAG_CREATE_ARRAY | LUA_SYMOPT_FLAG_CREATE_MAP))
+#define LUA_OPTION_PUSH(nm) \
+ do { \
+ if (LUA_SYMOPT_IS_ARRAY(fl)) { \
+ lua_pushstring(L, #nm); \
+ lua_rawseti(L, -2, i++); \
+ } \
+ else { \
+ lua_pushboolean(L, true); \
+ lua_setfield(L, -2, #nm); \
+ } \
+ } while (0)
+
+static void
+lua_push_symbol_flags(lua_State *L, guint flags, enum lua_push_symbol_flags_opts fl)
+{
+ guint i = 1;
+
+ if (LUA_SYMOPT_IS_CREATE(fl)) {
+ lua_newtable(L);
+ }
+
+ if (flags & SYMBOL_TYPE_FINE) {
+ LUA_OPTION_PUSH(fine);
+ }
+
+ if (flags & SYMBOL_TYPE_EMPTY) {
+ LUA_OPTION_PUSH(empty);
+ }
+
+ if (flags & SYMBOL_TYPE_EXPLICIT_DISABLE) {
+ LUA_OPTION_PUSH(explicit_disable);
+ }
+
+ if (flags & SYMBOL_TYPE_EXPLICIT_ENABLE) {
+ LUA_OPTION_PUSH(explicit_enable);
+ }
+
+ if (flags & SYMBOL_TYPE_IGNORE_PASSTHROUGH) {
+ LUA_OPTION_PUSH(ignore_passthrough);
+ }
+
+ if (flags & SYMBOL_TYPE_NOSTAT) {
+ LUA_OPTION_PUSH(nostat);
+ }
+
+ if (flags & SYMBOL_TYPE_IDEMPOTENT) {
+ LUA_OPTION_PUSH(idempotent);
+ }
+
+ if (flags & SYMBOL_TYPE_MIME_ONLY) {
+ LUA_OPTION_PUSH(mime);
+ }
+
+ if (flags & SYMBOL_TYPE_TRIVIAL) {
+ LUA_OPTION_PUSH(trivial);
+ }
+
+ if (flags & SYMBOL_TYPE_SKIPPED) {
+ LUA_OPTION_PUSH(skip);
+ }
+
+ if (flags & SYMBOL_TYPE_COMPOSITE) {
+ LUA_OPTION_PUSH(composite);
+ }
+}
+
+static gint
+lua_config_get_symbol_flags(lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = luaL_checkstring(L, 2);
+ guint flags;
+
+ if (cfg && name) {
+ flags = rspamd_symcache_get_symbol_flags(cfg->cache,
+ name);
+
+ if (flags != 0) {
+ lua_push_symbol_flags(L, flags, LUA_SYMOPT_FLAG_CREATE_ARRAY);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_register_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = NULL, *type_str = NULL,
+ *description = NULL, *group = NULL;
+ double weight = 0, score = NAN, parent_float = NAN;
+ gboolean one_shot = FALSE;
+ gint ret = -1, cbref = -1;
+ guint type = 0, flags = 0;
+ gint64 parent = 0, priority = 0, nshots = 0;
+ GArray *allowed_ids = NULL, *forbidden_ids = NULL;
+ GError *err = NULL;
+ int prev_top = lua_gettop(L);
+
+ if (cfg) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "name=S;weight=N;callback=F;type=S;priority=I;parent=D;"
+ "score=D;description=S;group=S;one_shot=B;nshots=I",
+ &name, &weight, &cbref, &type_str,
+ &priority, &parent_float,
+ &score, &description, &group, &one_shot, &nshots)) {
+ msg_err_config("bad arguments: %e", err);
+ g_error_free(err);
+ lua_settop(L, prev_top);
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ /* Deal with flags and ids */
+ lua_pushstring(L, "flags");
+ lua_gettable(L, 2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ flags = lua_parse_symbol_flags(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ flags |= lua_parse_symbol_flags(lua_tostring(L, -1));
+ }
+ }
+ lua_pop(L, 1); /* Clean flags */
+
+ lua_pushstring(L, "allowed_ids");
+ lua_gettable(L, 2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ allowed_ids = rspamd_process_id_list(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ allowed_ids = g_array_sized_new(FALSE, FALSE, sizeof(guint32),
+ rspamd_lua_table_size(L, -1));
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ guint32 v = lua_tointeger(L, -1);
+ g_array_append_val(allowed_ids, v);
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "forbidden_ids");
+ lua_gettable(L, 2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ forbidden_ids = rspamd_process_id_list(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ forbidden_ids = g_array_sized_new(FALSE, FALSE, sizeof(guint32),
+ rspamd_lua_table_size(L, -1));
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ guint32 v = lua_tointeger(L, -1);
+ g_array_append_val(forbidden_ids, v);
+ }
+ }
+ lua_pop(L, 1);
+
+ if (nshots == 0) {
+ nshots = cfg->default_max_shots;
+ }
+
+ type = lua_parse_symbol_type(type_str);
+
+ if (!name && !(type & SYMBOL_TYPE_CALLBACK)) {
+ lua_settop(L, prev_top);
+ return luaL_error(L, "no symbol name but type is not callback");
+ }
+ else if (!(type & SYMBOL_TYPE_VIRTUAL) && cbref == -1) {
+ lua_settop(L, prev_top);
+ return luaL_error(L, "no callback for symbol %s", name);
+ }
+
+ if (isnan(parent_float)) {
+ parent = -1;
+ }
+ else {
+ parent = parent_float;
+ }
+
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ name,
+ cbref,
+ weight == 0 ? 1.0 : weight,
+ priority,
+ type | flags,
+ parent,
+ allowed_ids, forbidden_ids,
+ FALSE);
+
+ if (allowed_ids) {
+ g_array_free(allowed_ids, TRUE);
+ }
+
+ if (forbidden_ids) {
+ g_array_free(forbidden_ids, TRUE);
+ }
+
+ if (ret != -1) {
+ if (!isnan(score) || group) {
+ if (one_shot) {
+ nshots = 1;
+ }
+
+ rspamd_config_add_symbol(cfg, name,
+ score, description, group, flags,
+ 0, nshots);
+
+ lua_pushstring(L, "groups");
+ lua_gettable(L, 2);
+
+ if (lua_istable(L, -1)) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ if (lua_isstring(L, -1)) {
+ rspamd_config_add_symbol_group(cfg, name,
+ lua_tostring(L, -1));
+ }
+ else {
+ lua_settop(L, prev_top);
+ return luaL_error(L, "invalid groups element");
+ }
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "augmentations");
+ lua_gettable(L, 2);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ int tbl_idx = lua_gettop(L);
+ for (lua_pushnil(L); lua_next(L, tbl_idx); lua_pop(L, 1)) {
+ size_t len;
+ const char *augmentation = lua_tolstring(L, -1, &len), *eqsign_pos;
+
+ /* Find `=` symbol and use it as a separator */
+ eqsign_pos = memchr(augmentation, '=', len);
+ if (eqsign_pos != NULL && eqsign_pos + 1 < augmentation + len) {
+ rspamd_ftok_t tok;
+
+ tok.begin = augmentation;
+ tok.len = eqsign_pos - augmentation;
+ char *augentation_name = rspamd_ftokdup(&tok);
+
+ tok.begin = eqsign_pos + 1;
+ tok.len = (augmentation + len) - tok.begin;
+
+ char *augmentation_value = rspamd_ftokdup(&tok);
+
+ if (!rspamd_symcache_add_symbol_augmentation(cfg->cache, ret,
+ augentation_name, augmentation_value)) {
+ lua_settop(L, prev_top);
+ g_free(augmentation_value);
+ g_free(augentation_name);
+
+ return luaL_error(L, "unknown or invalid augmentation %s in symbol %s",
+ augmentation, name);
+ }
+
+ g_free(augmentation_value);
+ g_free(augentation_name);
+ }
+ else {
+ /* Just a value */
+ if (!rspamd_symcache_add_symbol_augmentation(cfg->cache, ret,
+ augmentation, NULL)) {
+ lua_settop(L, prev_top);
+
+ return luaL_error(L, "unknown augmentation %s in symbol %s",
+ augmentation, name);
+ }
+ }
+ }
+ }
+ }
+ }
+ else {
+ lua_settop(L, prev_top);
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_settop(L, prev_top);
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_config_register_symbols(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gint i, top, idx, ret = -1;
+ const gchar *sym;
+ gdouble weight = 1.0;
+
+ if (lua_gettop(L) < 3) {
+ if (cfg) {
+ msg_err_config("not enough arguments to register a function");
+ }
+
+ lua_error(L);
+
+ return 0;
+ }
+ if (cfg) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ lua_getglobal(L, luaL_checkstring(L, 2));
+ }
+ else {
+ lua_pushvalue(L, 2);
+ }
+ idx = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ weight = lua_tonumber(L, 3);
+ top = 4;
+ }
+ else {
+ top = 3;
+ }
+ sym = luaL_checkstring(L, top++);
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ sym,
+ idx,
+ weight,
+ 0,
+ SYMBOL_TYPE_CALLBACK,
+ -1,
+ NULL, NULL,
+ FALSE);
+
+ for (i = top; i <= lua_gettop(L); i++) {
+ if (lua_type(L, i) == LUA_TTABLE) {
+ lua_pushvalue(L, i);
+ lua_pushnil(L);
+ while (lua_next(L, -2)) {
+ lua_pushvalue(L, -2);
+ sym = luaL_checkstring(L, -2);
+ rspamd_symcache_add_symbol(cfg->cache, sym,
+ 0, NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL, ret);
+ lua_pop(L, 2);
+ }
+ lua_pop(L, 1);
+ }
+ else if (lua_type(L, i) == LUA_TSTRING) {
+ sym = luaL_checkstring(L, i);
+ rspamd_symcache_add_symbol(cfg->cache, sym,
+ 0, NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL, ret);
+ }
+ }
+ }
+
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_config_register_virtual_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name;
+ double weight;
+ gint ret = -1, parent = -1;
+
+ if (cfg) {
+ name = luaL_checkstring(L, 2);
+ weight = luaL_checknumber(L, 3);
+
+ if (lua_gettop(L) > 3) {
+ parent = lua_tonumber(L, 4);
+ }
+
+ if (name) {
+ ret = rspamd_symcache_add_symbol(cfg->cache, name,
+ weight > 0 ? 0 : -1, NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL, parent);
+ }
+ }
+
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_config_register_callback_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = NULL;
+ double weight;
+ gint ret = -1, top = 2;
+
+ if (cfg) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* Legacy syntax */
+ name = luaL_checkstring(L, 2);
+ top++;
+ }
+
+ weight = luaL_checknumber(L, top);
+
+ if (lua_type(L, top + 1) == LUA_TSTRING) {
+ lua_getglobal(L, luaL_checkstring(L, top + 1));
+ }
+ else {
+ lua_pushvalue(L, top + 1);
+ }
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ name,
+ luaL_ref(L, LUA_REGISTRYINDEX),
+ weight,
+ 0,
+ SYMBOL_TYPE_CALLBACK,
+ -1,
+ NULL, NULL,
+ FALSE);
+ }
+
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_config_register_callback_symbol_priority(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = NULL;
+ double weight;
+ gint priority, ret = -1, top = 2;
+
+ if (cfg) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* Legacy syntax */
+ name = luaL_checkstring(L, 2);
+ top++;
+ }
+
+ weight = luaL_checknumber(L, top);
+ priority = luaL_checknumber(L, top + 1);
+
+ if (lua_type(L, top + 2) == LUA_TSTRING) {
+ lua_getglobal(L, luaL_checkstring(L, top + 2));
+ }
+ else {
+ lua_pushvalue(L, top + 2);
+ }
+
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ name,
+ luaL_ref(L, LUA_REGISTRYINDEX),
+ weight,
+ priority,
+ SYMBOL_TYPE_CALLBACK,
+ -1,
+ NULL, NULL,
+ FALSE);
+ }
+
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+
+static gint
+lua_config_register_dependency(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *parent = NULL, *child = NULL;
+ gint child_id;
+
+ if (cfg == NULL) {
+ lua_error(L);
+ return 0;
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ child_id = luaL_checknumber(L, 2);
+ parent = luaL_checkstring(L, 3);
+
+ return luaL_error(L, "calling for obsolete method to register deps for symbol %d->%s",
+ child_id, parent);
+ }
+ else {
+ child = luaL_checkstring(L, 2);
+ parent = luaL_checkstring(L, 3);
+
+ if (child != NULL && parent != NULL) {
+ rspamd_symcache_add_delayed_dependency(cfg->cache, child,
+ parent);
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_set_metric_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *description = NULL,
+ *group = NULL, *name = NULL, *flags_str = NULL;
+ double score;
+ gboolean one_shot = FALSE, one_param = FALSE;
+ GError *err = NULL;
+ gdouble priority = 0.0;
+ guint flags = 0;
+ gint64 nshots = 0;
+
+ if (cfg) {
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*name=S;score=N;description=S;"
+ "group=S;one_shot=B;one_param=B;priority=N;flags=S;"
+ "nshots=I",
+ &name, &score, &description,
+ &group, &one_shot, &one_param,
+ &priority, &flags_str, &nshots)) {
+ msg_err_config("bad arguments: %e", err);
+ g_error_free(err);
+
+ return 0;
+ }
+ }
+ else {
+ name = luaL_checkstring(L, 2);
+ score = luaL_checknumber(L, 3);
+
+ if (lua_gettop(L) > 3 && lua_type(L, 4) == LUA_TSTRING) {
+ description = luaL_checkstring(L, 4);
+ }
+ if (lua_gettop(L) > 4 && lua_type(L, 5) == LUA_TSTRING) {
+ /* XXX: metrics */
+ }
+ if (lua_gettop(L) > 5 && lua_type(L, 6) == LUA_TSTRING) {
+ group = luaL_checkstring(L, 6);
+ }
+ if (lua_gettop(L) > 6 && lua_type(L, 7) == LUA_TBOOLEAN) {
+ one_shot = lua_toboolean(L, 7);
+ }
+ }
+
+ if (nshots == 0) {
+ nshots = cfg->default_max_shots;
+ }
+
+ if (one_shot) {
+ nshots = 1;
+ }
+ if (one_param) {
+ flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM;
+ }
+
+ if (flags_str) {
+ if (strstr(flags_str, "one_shot") != NULL) {
+ nshots = 1;
+ }
+ if (strstr(flags_str, "ignore") != NULL) {
+ flags |= RSPAMD_SYMBOL_FLAG_IGNORE_METRIC;
+ }
+ if (strstr(flags_str, "one_param") != NULL) {
+ flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM;
+ }
+ }
+
+ rspamd_config_add_symbol(cfg, name,
+ score, description, group, flags, (guint) priority, nshots);
+
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ lua_pushstring(L, "groups");
+ lua_gettable(L, 2);
+
+ if (lua_istable(L, -1)) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ if (lua_isstring(L, -1)) {
+ rspamd_config_add_symbol_group(cfg, name,
+ lua_tostring(L, -1));
+ }
+ else {
+ return luaL_error(L, "invalid groups element");
+ }
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments, rspamd_config expected");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_set_metric_action(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = NULL;
+ double threshold = NAN;
+ GError *err = NULL;
+ gdouble priority = 0.0;
+ ucl_object_t *obj_tbl = NULL;
+
+ if (cfg) {
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*action=S;score=N;"
+ "priority=N",
+ &name, &threshold,
+ &priority)) {
+ msg_err_config("bad arguments: %e", err);
+ g_error_free(err);
+
+ return 0;
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TSTRING && lua_type(L, 3) == LUA_TTABLE) {
+ name = lua_tostring(L, 2);
+ obj_tbl = ucl_object_lua_import(L, 3);
+
+ if (obj_tbl) {
+ if (name) {
+ rspamd_config_set_action_score(cfg, name, obj_tbl);
+ ucl_object_unref(obj_tbl);
+ }
+ else {
+ ucl_object_unref(obj_tbl);
+ return luaL_error(L, "invalid first argument, action name expected");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid second argument, table expected");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments, table expected");
+ }
+
+ if (name != NULL && !isnan(threshold) && threshold != 0) {
+ obj_tbl = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj_tbl, ucl_object_fromdouble(threshold),
+ "score", 0, false);
+ ucl_object_insert_key(obj_tbl, ucl_object_fromdouble(priority),
+ "priority", 0, false);
+ rspamd_config_set_action_score(cfg, name, obj_tbl);
+ ucl_object_unref(obj_tbl);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments, rspamd_config expected");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_get_metric_action(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *act_name = luaL_checkstring(L, 2);
+ struct rspamd_action *act;
+
+ if (cfg && act_name) {
+ act = rspamd_config_get_action(cfg, act_name);
+
+ if (act) {
+ if (!isnan(act->threshold)) {
+ lua_pushnumber(L, act->threshold);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments, rspamd_config expected");
+ }
+
+ return 1;
+}
+
+static void
+lua_config_actions_cb(struct rspamd_action *act, void *cbd)
+{
+ lua_State *L = (lua_State *) cbd;
+
+ if (!isnan(act->threshold)) {
+ lua_pushstring(L, act->name);
+ lua_pushnumber(L, act->threshold);
+ lua_settable(L, -3);
+ }
+}
+
+static gint
+lua_config_get_all_actions(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg) {
+ lua_createtable(L, 0, rspamd_config_actions_size(cfg));
+ rspamd_config_actions_foreach(cfg, lua_config_actions_cb, L);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, rspamd_config expected");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_add_composite(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gchar *name;
+ const gchar *expr_str;
+ struct rspamd_composite *composite;
+ gboolean ret = FALSE;
+
+ if (cfg) {
+ name = rspamd_mempool_strdup(cfg->cfg_pool, luaL_checkstring(L, 2));
+ expr_str = luaL_checkstring(L, 3);
+
+ if (name && expr_str) {
+ composite = rspamd_composites_manager_add_from_string(cfg->composites_manager,
+ name, expr_str);
+
+ if (composite) {
+ rspamd_symcache_add_symbol(cfg->cache, name,
+ 0, NULL, composite, SYMBOL_TYPE_COMPOSITE, -1);
+ ret = TRUE;
+ }
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_config_newindex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name;
+ GArray *allowed_ids = NULL, *forbidden_ids = NULL;
+ gint id, nshots;
+ guint flags = 0;
+ gboolean optional = FALSE;
+
+ name = luaL_checkstring(L, 2);
+
+ if (cfg != NULL && name != NULL && lua_gettop(L) == 3) {
+
+ if (lua_type(L, 3) == LUA_TFUNCTION) {
+ /* Normal symbol from just a function */
+ lua_pushvalue(L, 3);
+ rspamd_register_symbol_fromlua(L,
+ cfg,
+ name,
+ luaL_ref(L, LUA_REGISTRYINDEX),
+ 1.0,
+ 0,
+ SYMBOL_TYPE_NORMAL,
+ -1,
+ NULL, NULL,
+ FALSE);
+ }
+ else if (lua_type(L, 3) == LUA_TTABLE) {
+ guint type = SYMBOL_TYPE_NORMAL, priority = 0;
+ gint idx;
+ gdouble weight = 1.0, score = NAN;
+ const char *type_str, *group = NULL, *description = NULL;
+
+ /*
+ * Table can have the following attributes:
+ * "callback" - should be a callback function
+ * "weight" - optional weight
+ * "priority" - optional priority
+ * "type" - optional type (normal, virtual, callback)
+ * "flags" - optional flags
+ * -- Metric options
+ * "score" - optional default score (overridden by metric)
+ * "group" - optional default group
+ * "one_shot" - optional one shot mode
+ * "description" - optional description
+ */
+ lua_pushvalue(L, 3);
+ lua_pushstring(L, "callback");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ lua_pop(L, 2);
+ msg_info_config("cannot find callback definition for %s",
+ name);
+ return 0;
+ }
+ idx = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ /* Optional fields */
+ lua_pushstring(L, "weight");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ weight = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "priority");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ priority = lua_tointeger(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "optional");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ optional = lua_toboolean(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "type");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ type_str = lua_tostring(L, -1);
+ type = lua_parse_symbol_type(type_str);
+ }
+ lua_pop(L, 1);
+
+ /* Deal with flags and ids */
+ lua_pushstring(L, "flags");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ flags = lua_parse_symbol_flags(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ flags |= lua_parse_symbol_flags(lua_tostring(L, -1));
+ }
+ }
+ lua_pop(L, 1); /* Clean flags */
+
+ lua_pushstring(L, "allowed_ids");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ allowed_ids = rspamd_process_id_list(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ allowed_ids = g_array_sized_new(FALSE, FALSE, sizeof(guint32),
+ rspamd_lua_table_size(L, -1));
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ guint32 v = lua_tointeger(L, -1);
+ g_array_append_val(allowed_ids, v);
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "forbidden_ids");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ forbidden_ids = rspamd_process_id_list(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ forbidden_ids = g_array_sized_new(FALSE, FALSE, sizeof(guint32),
+ rspamd_lua_table_size(L, -1));
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ guint32 v = lua_tointeger(L, -1);
+ g_array_append_val(forbidden_ids, v);
+ }
+ }
+ lua_pop(L, 1);
+
+ id = rspamd_register_symbol_fromlua(L,
+ cfg,
+ name,
+ idx,
+ weight,
+ priority,
+ type | flags,
+ -1,
+ allowed_ids, forbidden_ids,
+ optional);
+
+ if (allowed_ids) {
+ g_array_free(allowed_ids, TRUE);
+ }
+
+ if (forbidden_ids) {
+ g_array_free(forbidden_ids, TRUE);
+ }
+
+ if (id != -1) {
+ /* Check for condition */
+ lua_pushstring(L, "condition");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ gint condref;
+
+ /* Here we pop function from the stack, so no lua_pop is required */
+ condref = luaL_ref(L, LUA_REGISTRYINDEX);
+ g_assert(name != NULL);
+ rspamd_symcache_add_condition_delayed(cfg->cache,
+ name, L, condref);
+ }
+ else {
+ lua_pop(L, 1);
+ }
+
+ /* Check for augmentations */
+ lua_pushstring(L, "augmentations");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+
+ int tbl_idx = lua_gettop(L);
+ for (lua_pushnil(L); lua_next(L, tbl_idx); lua_pop(L, 1)) {
+ rspamd_symcache_add_symbol_augmentation(cfg->cache, id,
+ lua_tostring(L, -1), NULL);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ /*
+ * Now check if a symbol has not been registered in any metric and
+ * insert default value if applicable
+ */
+ struct rspamd_symbol *sym = g_hash_table_lookup(cfg->symbols, name);
+ if (sym == NULL || (sym->flags & RSPAMD_SYMBOL_FLAG_UNSCORED)) {
+ nshots = cfg->default_max_shots;
+
+ lua_pushstring(L, "score");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ score = lua_tonumber(L, -1);
+
+ if (sym) {
+ /* Reset unscored flag */
+ sym->flags &= ~RSPAMD_SYMBOL_FLAG_UNSCORED;
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "group");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ group = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ if (!isnan(score) || group != NULL) {
+ lua_pushstring(L, "description");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ description = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "one_shot");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ if (lua_toboolean(L, -1)) {
+ nshots = 1;
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "one_param");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ if (lua_toboolean(L, -1)) {
+ flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM;
+ }
+ }
+ lua_pop(L, 1);
+
+ /*
+ * Do not override the existing symbols (using zero priority),
+ * since we are defining default values here
+ */
+ if (!isnan(score)) {
+ rspamd_config_add_symbol(cfg, name, score,
+ description, group, flags, 0, nshots);
+ }
+ else if (group) {
+ /* Add with zero score */
+ rspamd_config_add_symbol(cfg, name, NAN,
+ description, group, flags, 0, nshots);
+ }
+
+ lua_pushstring(L, "groups");
+ lua_gettable(L, -2);
+
+ if (lua_istable(L, -1)) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ if (lua_isstring(L, -1)) {
+ rspamd_config_add_symbol_group(cfg, name,
+ lua_tostring(L, -1));
+ }
+ else {
+ return luaL_error(L, "invalid groups element");
+ }
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ /* Fill in missing fields from lua definition if they are not set */
+ if (sym->description == NULL) {
+ lua_pushstring(L, "description");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ description = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ if (description) {
+ sym->description = rspamd_mempool_strdup(cfg->cfg_pool, description);
+ }
+ }
+
+ /* If ungrouped and there is a group defined in lua, change the primary group
+ * Otherwise, add to the list of groups for this symbol. */
+ lua_pushstring(L, "group");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ group = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+ if (group) {
+ if (sym->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED) {
+ /* Unset the "ungrouped" group */
+ sym->gr = NULL;
+ }
+ /* Add the group. If the symbol was ungrouped, this will
+ * clear RSPAMD_SYMBOL_FLAG_UNGROUPED from the flags. */
+ rspamd_config_add_symbol_group(cfg, name, group);
+ }
+ }
+
+ /* Remove table from stack */
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_add_condition(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym = luaL_checkstring(L, 2);
+ gboolean ret = FALSE;
+ gint condref;
+
+ if (cfg && sym && lua_type(L, 3) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 3);
+ condref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ ret = rspamd_symcache_add_condition_delayed(cfg->cache, sym, L,
+ condref);
+
+ if (!ret) {
+ luaL_unref(L, LUA_REGISTRYINDEX, condref);
+ }
+ }
+
+ lua_pushboolean(L, ret);
+ return 1;
+}
+
+static gint
+lua_config_set_peak_cb(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gint condref;
+
+ if (cfg && lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ condref = luaL_ref(L, LUA_REGISTRYINDEX);
+ rspamd_symcache_set_peak_callback(cfg->cache,
+ condref);
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_enable_symbol(lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const char *sym = luaL_checkstring(L, 2);
+
+ if (!sym || !cfg) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ rspamd_symcache_enable_symbol_static(cfg->cache, sym);
+
+ return 0;
+}
+
+static gint
+lua_config_disable_symbol(lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const char *sym = luaL_checkstring(L, 2);
+
+ if (!sym || !cfg) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ rspamd_symcache_disable_symbol_static(cfg->cache, sym);
+
+ return 0;
+}
+
+static gint
+lua_config_register_regexp(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_lua_regexp *re = NULL;
+ rspamd_regexp_t *cache_re;
+ const gchar *type_str = NULL, *header_str = NULL;
+ gsize header_len = 0;
+ GError *err = NULL;
+ enum rspamd_re_type type = RSPAMD_RE_BODY;
+ gboolean pcre_only = FALSE;
+
+ /*
+ * - `re`* : regular expression object
+ * - `type`*: type of regular expression:
+ * + `mime`: mime regexp
+ * + `rawmime`: raw mime regexp
+ * + `header`: header regexp
+ * + `rawheader`: raw header expression
+ * + `body`: raw body regexp
+ * + `url`: url regexp
+ * - `header`: for header and rawheader regexp means the name of header
+ * - `pcre_only`: allow merely pcre for this regexp
+ */
+ if (cfg != NULL) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*re=U{regexp};*type=S;header=S;pcre_only=B",
+ &re, &type_str, &header_str, &pcre_only)) {
+ msg_err_config("cannot get parameters list: %e", err);
+
+ if (err) {
+ g_error_free(err);
+ }
+ }
+ else {
+ type = rspamd_re_cache_type_from_string(type_str);
+
+ if ((type == RSPAMD_RE_HEADER ||
+ type == RSPAMD_RE_RAWHEADER ||
+ type == RSPAMD_RE_MIMEHEADER) &&
+ header_str == NULL) {
+ msg_err_config(
+ "header argument is mandatory for header/rawheader regexps");
+ }
+ else {
+ if (pcre_only) {
+ rspamd_regexp_set_flags(re->re,
+ rspamd_regexp_get_flags(re->re) | RSPAMD_REGEXP_FLAG_PCRE_ONLY);
+ }
+
+ if (header_str != NULL) {
+ /* Include the last \0 */
+ header_len = strlen(header_str) + 1;
+ }
+
+ cache_re = rspamd_re_cache_add(cfg->re_cache, re->re, type,
+ (gpointer) header_str, header_len, -1);
+
+ /*
+ * XXX: here are dragons!
+ * Actually, lua regexp contains internal rspamd_regexp_t
+ * and it owns it.
+ * However, after this operation we have some OTHER regexp,
+ * which we really would like to use.
+ * So we do the following:
+ * 1) Remove old re and unref it
+ * 2) Replace the internal re with cached one
+ * 3) Increase its refcount to share ownership between cache and
+ * lua object
+ */
+ if (cache_re != re->re) {
+ rspamd_regexp_unref(re->re);
+ re->re = rspamd_regexp_ref(cache_re);
+
+ if (pcre_only) {
+ rspamd_regexp_set_flags(re->re,
+ rspamd_regexp_get_flags(re->re) | RSPAMD_REGEXP_FLAG_PCRE_ONLY);
+ }
+ }
+ }
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_replace_regexp(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_lua_regexp *old_re = NULL, *new_re = NULL;
+ gboolean pcre_only = FALSE;
+ GError *err = NULL;
+
+ if (cfg != NULL) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*old_re=U{regexp};*new_re=U{regexp};pcre_only=B",
+ &old_re, &new_re, &pcre_only)) {
+ gint ret = luaL_error(L, "cannot get parameters list: %s",
+ err ? err->message : "invalid arguments");
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return ret;
+ }
+ else {
+
+ if (pcre_only) {
+ rspamd_regexp_set_flags(new_re->re,
+ rspamd_regexp_get_flags(new_re->re) | RSPAMD_REGEXP_FLAG_PCRE_ONLY);
+ }
+
+ rspamd_re_cache_replace(cfg->re_cache, old_re->re, new_re->re);
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_register_worker_script(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *worker_type = luaL_checkstring(L, 2), *wtype;
+ struct rspamd_worker_conf *cf;
+ GList *cur;
+ struct rspamd_worker_lua_script *sc;
+ gboolean found = FALSE;
+
+ if (cfg == NULL || worker_type == NULL || lua_type(L, 3) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ for (cur = g_list_first(cfg->workers); cur != NULL; cur = g_list_next(cur)) {
+ cf = cur->data;
+ wtype = g_quark_to_string(cf->type);
+
+ if (g_ascii_strcasecmp(wtype, worker_type) == 0) {
+ sc = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*sc));
+ lua_pushvalue(L, 3);
+ sc->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ DL_APPEND(cf->scripts, sc);
+ found = TRUE;
+ }
+ }
+
+ lua_pushboolean(L, found);
+
+ return 1;
+}
+
+static gint
+lua_config_add_on_load(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_config_cfg_lua_script *sc;
+
+ if (cfg == NULL || lua_type(L, 2) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sc = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*sc));
+ lua_pushvalue(L, 2);
+ sc->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ DL_APPEND(cfg->on_load_scripts, sc);
+
+ return 0;
+}
+
+static inline int
+rspamd_post_init_sc_sort(const struct rspamd_config_cfg_lua_script *pra,
+ const struct rspamd_config_cfg_lua_script *prb)
+{
+ /* Inverse sort */
+ return prb->priority - pra->priority;
+}
+
+static gint
+lua_config_add_post_init(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_config_cfg_lua_script *sc;
+ guint priority = 0;
+ lua_Debug d;
+ gchar tmp[256], *p;
+
+ if (cfg == NULL || lua_type(L, 2) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ priority = lua_tointeger(L, 3);
+ }
+
+ if (lua_getstack(L, 1, &d) == 1) {
+ (void) lua_getinfo(L, "Sl", &d);
+ if ((p = strrchr(d.short_src, '/')) == NULL) {
+ p = d.short_src;
+ }
+ else {
+ p++;
+ }
+
+ if (strlen(p) > 200) {
+ rspamd_snprintf(tmp, sizeof(tmp), "%10s...]:%d", p,
+ d.currentline);
+ }
+ else {
+ rspamd_snprintf(tmp, sizeof(tmp), "%s:%d", p,
+ d.currentline);
+ }
+ }
+
+ sc = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*sc));
+ lua_pushvalue(L, 2);
+ sc->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ sc->priority = priority;
+ sc->lua_src_pos = rspamd_mempool_strdup(cfg->cfg_pool, tmp);
+ DL_APPEND(cfg->post_init_scripts, sc);
+ DL_SORT(cfg->post_init_scripts, rspamd_post_init_sc_sort);
+
+ return 0;
+}
+
+static gint
+lua_config_add_config_unload(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_config_cfg_lua_script *sc;
+ lua_Debug d;
+ gchar tmp[256], *p;
+
+ if (cfg == NULL || lua_type(L, 2) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_getstack(L, 1, &d) == 1) {
+ (void) lua_getinfo(L, "Sl", &d);
+ if ((p = strrchr(d.short_src, '/')) == NULL) {
+ p = d.short_src;
+ }
+ else {
+ p++;
+ }
+
+ if (strlen(p) > 20) {
+ rspamd_snprintf(tmp, sizeof(tmp), "%10s...]:%d", p,
+ d.currentline);
+ }
+ else {
+ rspamd_snprintf(tmp, sizeof(tmp), "%s:%d", p,
+ d.currentline);
+ }
+ }
+
+ sc = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*sc));
+ lua_pushvalue(L, 2);
+ sc->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ sc->lua_src_pos = rspamd_mempool_strdup(cfg->cfg_pool, tmp);
+ DL_APPEND(cfg->config_unload_scripts, sc);
+
+ return 0;
+}
+
+
+static void lua_periodic_callback_finish(struct thread_entry *thread, int ret);
+static void lua_periodic_callback_error(struct thread_entry *thread, int ret, const char *msg);
+
+struct rspamd_lua_periodic {
+ struct ev_loop *event_loop;
+ struct rspamd_config *cfg;
+ gchar *lua_src_pos;
+ lua_State *L;
+ gdouble timeout;
+ ev_timer ev;
+ gint cbref;
+ gboolean need_jitter;
+ ref_entry_t ref;
+};
+
+static void
+lua_periodic_dtor(struct rspamd_lua_periodic *periodic)
+{
+ luaL_unref(periodic->L, LUA_REGISTRYINDEX, periodic->cbref);
+ ev_timer_stop(periodic->event_loop, &periodic->ev);
+}
+
+static void
+lua_periodic_fin(gpointer p)
+{
+ struct rspamd_lua_periodic *periodic = (struct rspamd_lua_periodic *) p;
+
+ REF_RELEASE(periodic);
+}
+
+static void
+lua_periodic_callback(struct ev_loop *loop, ev_timer *w, int revents)
+{
+ struct rspamd_lua_periodic *periodic = (struct rspamd_lua_periodic *) w->data;
+ struct rspamd_config **pcfg, *cfg;
+ struct ev_loop **pev_base;
+ struct thread_entry *thread;
+ lua_State *L;
+
+ REF_RETAIN(periodic);
+ thread = lua_thread_pool_get_for_config(periodic->cfg);
+ thread->cd = periodic;
+ thread->finish_callback = lua_periodic_callback_finish;
+ thread->error_callback = lua_periodic_callback_error;
+
+ L = thread->lua_state;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, periodic->cbref);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ cfg = periodic->cfg;
+ *pcfg = cfg;
+ pev_base = lua_newuserdata(L, sizeof(*pev_base));
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+ *pev_base = periodic->event_loop;
+ lua_pushnumber(L, ev_now(periodic->event_loop));
+
+ lua_thread_call(thread, 3);
+}
+
+static void
+lua_periodic_callback_finish(struct thread_entry *thread, int ret)
+{
+ lua_State *L;
+ struct rspamd_lua_periodic *periodic = thread->cd;
+ gboolean plan_more = FALSE;
+ gdouble timeout = 0.0;
+
+ L = thread->lua_state;
+
+ ev_now_update(periodic->event_loop);
+
+ if (ret == 0) {
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ plan_more = lua_toboolean(L, -1);
+ timeout = periodic->timeout;
+ }
+ else if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ plan_more = timeout > 0 ? TRUE : FALSE;
+ }
+
+ lua_pop(L, 1); /* Return value */
+ }
+
+ if (periodic->cfg->cur_worker) {
+ if (periodic->cfg->cur_worker->state != rspamd_worker_state_running) {
+ /* We are terminating, no more periodics */
+ plan_more = FALSE;
+ }
+ }
+
+ if (plan_more) {
+ if (periodic->need_jitter) {
+ timeout = rspamd_time_jitter(timeout, 0.0);
+ }
+
+ periodic->ev.repeat = timeout;
+ ev_timer_again(periodic->event_loop, &periodic->ev);
+ }
+ else {
+ ev_timer_stop(periodic->event_loop, &periodic->ev);
+ }
+
+ REF_RELEASE(periodic);
+}
+
+static void
+lua_periodic_callback_error(struct thread_entry *thread, int ret, const char *msg)
+{
+ struct rspamd_config *cfg;
+ struct rspamd_lua_periodic *periodic = thread->cd;
+ cfg = periodic->cfg;
+
+ msg_err_config("call to periodic script (registered at %s) failed: %s",
+ periodic->lua_src_pos, msg);
+
+ lua_periodic_callback_finish(thread, ret);
+}
+
+
+static gint
+lua_config_add_periodic(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct ev_loop *ev_base = lua_check_ev_base(L, 2);
+ gdouble timeout = lua_tonumber(L, 3);
+ struct rspamd_lua_periodic *periodic;
+ gboolean need_jitter = FALSE;
+ lua_Debug d;
+ gchar tmp[256], *p;
+
+ if (cfg == NULL || timeout < 0 || lua_type(L, 4) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 5) == LUA_TBOOLEAN) {
+ need_jitter = lua_toboolean(L, 5);
+ }
+
+ if (lua_getstack(L, 1, &d) == 1) {
+ (void) lua_getinfo(L, "Sl", &d);
+ if ((p = strrchr(d.short_src, '/')) == NULL) {
+ p = d.short_src;
+ }
+ else {
+ p++;
+ }
+
+ if (strlen(p) > 20) {
+ rspamd_snprintf(tmp, sizeof(tmp), "%10s...]:%d", p,
+ d.currentline);
+ }
+ else {
+ rspamd_snprintf(tmp, sizeof(tmp), "%s:%d", p,
+ d.currentline);
+ }
+ }
+
+ periodic = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*periodic));
+ periodic->timeout = timeout;
+ periodic->L = L;
+ periodic->cfg = cfg;
+ periodic->event_loop = ev_base;
+ periodic->need_jitter = need_jitter;
+ periodic->lua_src_pos = rspamd_mempool_strdup(cfg->cfg_pool, tmp);
+ lua_pushvalue(L, 4);
+ periodic->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ if (need_jitter) {
+ timeout = rspamd_time_jitter(timeout, 0.0);
+ }
+
+ ev_timer_init(&periodic->ev, lua_periodic_callback, timeout, 0.0);
+ periodic->ev.data = periodic;
+ ev_timer_start(ev_base, &periodic->ev);
+ REF_INIT_RETAIN(periodic, lua_periodic_dtor);
+
+ rspamd_mempool_add_destructor(cfg->cfg_pool, lua_periodic_fin,
+ periodic);
+
+ return 0;
+}
+
+static gint
+lua_config_get_symbols_count(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ guint res = 0;
+
+ if (cfg != NULL) {
+ res = rspamd_symcache_stats_symbols_count(cfg->cache);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushinteger(L, res);
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbols_cksum(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ guint64 res = 0, *pres;
+
+ if (cfg != NULL) {
+ res = rspamd_symcache_get_cksum(cfg->cache);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ pres = lua_newuserdata(L, sizeof(res));
+ *pres = res;
+ rspamd_lua_setclass(L, "rspamd{int64}", -1);
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbols_counters(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ ucl_object_t *counters;
+
+ if (cfg != NULL) {
+ counters = rspamd_symcache_counters(cfg->cache);
+ ucl_object_push_lua(L, counters, true);
+ ucl_object_unref(counters);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct lua_metric_symbols_cbdata {
+ lua_State *L;
+ struct rspamd_config *cfg;
+ bool is_table;
+};
+
+static void
+lua_metric_symbol_inserter(gpointer k, gpointer v, gpointer ud)
+{
+ struct lua_metric_symbols_cbdata *cbd = (struct lua_metric_symbols_cbdata *) ud;
+ lua_State *L;
+ const gchar *sym = k;
+ struct rspamd_symbol *s = (struct rspamd_symbol *) v;
+ struct rspamd_symbols_group *gr;
+ gint i;
+
+ L = cbd->L;
+
+ if (cbd->is_table) {
+ lua_pushstring(L, sym); /* Symbol name */
+ }
+
+ lua_createtable(L, 0, 6);
+ lua_pushstring(L, "score");
+ lua_pushnumber(L, s->score);
+ lua_settable(L, -3);
+ lua_pushstring(L, "description");
+ lua_pushstring(L, s->description);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "flags");
+ lua_createtable(L, 0, 3);
+
+ if (s->flags & RSPAMD_SYMBOL_FLAG_IGNORE_METRIC) {
+ lua_pushstring(L, "ignore");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (s->flags & RSPAMD_SYMBOL_FLAG_ONEPARAM) {
+ lua_pushstring(L, "oneparam");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (s->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED) {
+ lua_pushstring(L, "ungrouped");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (s->flags & RSPAMD_SYMBOL_FLAG_DISABLED) {
+ lua_pushstring(L, "disabled");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+
+ if (s->cache_item) {
+ guint sflags = rspamd_symcache_get_symbol_flags(cbd->cfg->cache, sym);
+
+ lua_push_symbol_flags(L, sflags, LUA_SYMOPT_FLAG_USE_MAP);
+
+ guint nids;
+ const guint *allowed_ids = rspamd_symcache_get_allowed_settings_ids(cbd->cfg->cache,
+ sym, &nids);
+
+ if (allowed_ids && nids > 0) {
+ lua_createtable(L, nids, 0);
+
+ for (i = 0; i < nids; i++) {
+ lua_pushinteger(L, allowed_ids[i]);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ lua_setfield(L, -2, "allowed_ids");
+ }
+
+ const guint *forbidden_ids = rspamd_symcache_get_forbidden_settings_ids(
+ cbd->cfg->cache,
+ sym, &nids);
+
+ if (forbidden_ids && nids > 0) {
+ lua_createtable(L, nids, 0);
+
+ for (i = 0; i < nids; i++) {
+ lua_pushinteger(L, forbidden_ids[i]);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ lua_setfield(L, -2, "forbidden_ids");
+ }
+ }
+
+ lua_settable(L, -3); /* Flags -> flags_table */
+
+ lua_pushstring(L, "nshots");
+ lua_pushinteger(L, s->nshots);
+ lua_settable(L, -3);
+
+ if (s->gr) {
+ lua_pushstring(L, "group");
+ lua_pushstring(L, s->gr->name);
+ lua_settable(L, -3);
+ }
+
+ if (s->groups && s->groups->len > 0) {
+ lua_pushstring(L, "groups");
+ lua_createtable(L, s->groups->len, 0);
+
+ PTR_ARRAY_FOREACH(s->groups, i, gr)
+ {
+ lua_pushstring(L, gr->name);
+ lua_rawseti(L, -2, i + 1); /* Groups[i + 1] = group_name */
+ }
+
+ lua_settable(L, -3); /* Groups -> groups_table */
+ }
+ else {
+ lua_createtable(L, 0, 0);
+ lua_setfield(L, -2, "groups");
+ }
+
+ if (cbd->is_table) {
+ lua_settable(L, -3); /* Symname -> table */
+ }
+}
+
+static gint
+lua_config_get_symbols(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ struct lua_metric_symbols_cbdata cbd;
+
+ cbd.L = L;
+ cbd.cfg = cfg;
+ cbd.is_table = true;
+
+ lua_createtable(L, 0, g_hash_table_size(cfg->symbols));
+ g_hash_table_foreach(cfg->symbols,
+ lua_metric_symbol_inserter,
+ &cbd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym_name = luaL_checkstring(L, 2);
+
+ if (cfg != NULL && sym_name != NULL) {
+ struct lua_metric_symbols_cbdata cbd;
+ struct rspamd_symbol *s = g_hash_table_lookup(cfg->symbols, sym_name);
+
+ if (s) {
+ cbd.L = L;
+ cbd.cfg = cfg;
+ cbd.is_table = false;
+ lua_metric_symbol_inserter((void *) sym_name, s, &cbd);
+ }
+ else {
+ /* No config for a symbol */
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbol_callback(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym = luaL_checkstring(L, 2);
+ struct rspamd_abstract_callback_data *abs_cbdata;
+ struct lua_callback_data *cbd;
+
+ if (cfg != NULL && sym != NULL) {
+ abs_cbdata = rspamd_symcache_get_cbdata(cfg->cache, sym);
+
+ if (abs_cbdata == NULL || abs_cbdata->magic != rspamd_lua_callback_magic) {
+ lua_pushnil(L);
+ }
+ else {
+ cbd = (struct lua_callback_data *) abs_cbdata;
+
+ if (cbd->cb_is_ref) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->callback.ref);
+ }
+ else {
+ lua_getglobal(L, cbd->callback.name);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_set_symbol_callback(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym = luaL_checkstring(L, 2);
+ struct rspamd_abstract_callback_data *abs_cbdata;
+ struct lua_callback_data *cbd;
+
+ if (cfg != NULL && sym != NULL && lua_type(L, 3) == LUA_TFUNCTION) {
+ abs_cbdata = rspamd_symcache_get_cbdata(cfg->cache, sym);
+
+ if (abs_cbdata == NULL || abs_cbdata->magic != rspamd_lua_callback_magic) {
+ lua_pushboolean(L, FALSE);
+ }
+ else {
+ cbd = (struct lua_callback_data *) abs_cbdata;
+
+ if (cbd->cb_is_ref) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbd->callback.ref);
+ }
+ else {
+ cbd->cb_is_ref = TRUE;
+ }
+
+ lua_pushvalue(L, 3);
+ cbd->callback.ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ lua_pushboolean(L, TRUE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbol_stat(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym = luaL_checkstring(L, 2);
+ gdouble freq, stddev, tm;
+ guint hits;
+
+ if (cfg != NULL && sym != NULL) {
+ if (!rspamd_symcache_stat_symbol(cfg->cache, sym, &freq,
+ &stddev, &tm, &hits)) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_createtable(L, 0, 4);
+ lua_pushstring(L, "frequency");
+ lua_pushnumber(L, freq);
+ lua_settable(L, -3);
+ lua_pushstring(L, "sttdev");
+ lua_pushnumber(L, stddev);
+ lua_settable(L, -3);
+ lua_pushstring(L, "time");
+ lua_pushnumber(L, tm);
+ lua_settable(L, -3);
+ lua_pushstring(L, "hits");
+ lua_pushinteger(L, hits);
+ lua_settable(L, -3);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbol_parent(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym = luaL_checkstring(L, 2), *parent;
+
+ if (cfg != NULL && sym != NULL) {
+ parent = rspamd_symcache_get_parent(cfg->cache, sym);
+
+ if (parent) {
+ lua_pushstring(L, parent);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_group_symbols(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *gr_name = luaL_checkstring(L, 2);
+
+ if (cfg != NULL && gr_name != NULL) {
+ struct rspamd_symbols_group *group;
+
+ group = g_hash_table_lookup(cfg->groups, gr_name);
+
+ if (group == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ guint i = 1;
+ gpointer k, v;
+ GHashTableIter it;
+
+ lua_createtable(L, g_hash_table_size(group->symbols), 0);
+ g_hash_table_iter_init(&it, group->symbols);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ lua_pushstring(L, k);
+ lua_rawseti(L, -2, i);
+ i++;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_groups(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gboolean need_private;
+ struct rspamd_symbols_group *gr;
+ GHashTableIter it;
+ gpointer k, v;
+
+ if (cfg) {
+ if (lua_isboolean(L, 2)) {
+ need_private = lua_toboolean(L, 2);
+ }
+ else {
+ need_private = !(cfg->public_groups_only);
+ }
+
+ lua_createtable(L, 0, g_hash_table_size(cfg->groups));
+ g_hash_table_iter_init(&it, cfg->groups);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ gr = (struct rspamd_symbols_group *) v;
+
+ if (need_private || (gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC)) {
+ lua_createtable(L, 0, 4);
+
+ lua_pushstring(L, gr->description);
+ lua_setfield(L, -2, "description");
+ lua_pushnumber(L, gr->max_score);
+ lua_setfield(L, -2, "max_score");
+ lua_pushboolean(L, (gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC) != 0);
+ lua_setfield(L, -2, "is_public");
+ /* TODO: maybe push symbols as well */
+
+ /* Parent table indexed by group name */
+ lua_setfield(L, -2, gr->name);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_register_finish_script(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_config_cfg_lua_script *sc;
+
+ if (cfg != NULL && lua_type(L, 2) == LUA_TFUNCTION) {
+ sc = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*sc));
+ lua_pushvalue(L, 2);
+ sc->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ DL_APPEND(cfg->on_term_scripts, sc);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static inline bool
+rspamd_lua_config_check_settings_symbols_object(const ucl_object_t *obj)
+{
+ if (obj == NULL) {
+ /* Semantically valid */
+ return true;
+ }
+
+ if (ucl_object_type(obj) == UCL_OBJECT) {
+ /* Key-value mapping - should be okay */
+ return true;
+ }
+
+ if (ucl_object_type(obj) == UCL_ARRAY) {
+ /* Okay if empty */
+ if (obj->len == 0) {
+ return true;
+ }
+ }
+
+ /* Everything else not okay */
+ return false;
+}
+
+static gint
+lua_config_register_settings_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *settings_name = luaL_checkstring(L, 2);
+
+ if (cfg != NULL && settings_name) {
+ ucl_object_t *sym_enabled, *sym_disabled;
+ enum rspamd_config_settings_policy policy = RSPAMD_SETTINGS_POLICY_DEFAULT;
+
+ sym_enabled = ucl_object_lua_import(L, 3);
+
+ if (!rspamd_lua_config_check_settings_symbols_object(sym_enabled)) {
+ ucl_object_unref(sym_enabled);
+
+ return luaL_error(L, "invalid symbols enabled");
+ }
+
+ sym_disabled = ucl_object_lua_import(L, 4);
+
+ if (!rspamd_lua_config_check_settings_symbols_object(sym_disabled)) {
+ ucl_object_unref(sym_enabled);
+ ucl_object_unref(sym_disabled);
+
+ return luaL_error(L, "invalid symbols enabled");
+ }
+
+ /* Check policy */
+ if (lua_isstring(L, 5)) {
+ const gchar *policy_str = lua_tostring(L, 5);
+
+ if (strcmp(policy_str, "default") == 0) {
+ policy = RSPAMD_SETTINGS_POLICY_DEFAULT;
+ }
+ else if (strcmp(policy_str, "implicit_allow") == 0) {
+ policy = RSPAMD_SETTINGS_POLICY_IMPLICIT_ALLOW;
+ }
+ else if (strcmp(policy_str, "implicit_deny") == 0) {
+ policy = RSPAMD_SETTINGS_POLICY_IMPLICIT_DENY;
+ }
+ else {
+ return luaL_error(L, "invalid settings policy: %s", policy_str);
+ }
+ }
+ else {
+ /* Apply heuristic */
+ if (!sym_enabled) {
+ policy = RSPAMD_SETTINGS_POLICY_IMPLICIT_ALLOW;
+ }
+ }
+
+ rspamd_config_register_settings_id(cfg, settings_name, sym_enabled,
+ sym_disabled, policy);
+
+ if (sym_enabled) {
+ ucl_object_unref(sym_enabled);
+ }
+
+ if (sym_disabled) {
+ ucl_object_unref(sym_disabled);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_register_monitored(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_monitored *m, **pm;
+ const gchar *url, *type;
+ ucl_object_t *params = NULL;
+
+ url = lua_tostring(L, 2);
+ type = lua_tostring(L, 3);
+
+ if (cfg != NULL && url != NULL && type != NULL) {
+ if (g_ascii_strcasecmp(type, "dns") == 0) {
+ lua_Debug ar;
+
+ if (lua_type(L, 4) == LUA_TTABLE) {
+ params = ucl_object_lua_import(L, 4);
+ }
+
+ /* Get lua line and source */
+ lua_getstack(L, 1, &ar);
+ lua_getinfo(L, "nSl", &ar);
+
+ m = rspamd_monitored_create_(cfg->monitored_ctx, url,
+ RSPAMD_MONITORED_DNS, RSPAMD_MONITORED_DEFAULT,
+ params, ar.short_src);
+
+ if (m) {
+ pm = lua_newuserdata(L, sizeof(*pm));
+ *pm = m;
+ rspamd_lua_setclass(L, "rspamd{monitored}", -1);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (params) {
+ ucl_object_unref(params);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid monitored type: %s", type);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_add_doc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg;
+ const gchar *path = NULL, *option, *doc_string;
+ const gchar *type_str = NULL, *default_value = NULL;
+ ucl_type_t type = UCL_NULL;
+ gboolean required = FALSE;
+ GError *err = NULL;
+
+ cfg = lua_check_config(L, 1);
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ path = luaL_checkstring(L, 2);
+ }
+
+ option = luaL_checkstring(L, 3);
+ doc_string = luaL_checkstring(L, 4);
+
+ if (cfg && option && doc_string) {
+ if (lua_type(L, 5) == LUA_TTABLE) {
+ if (!rspamd_lua_parse_table_arguments(L, 5, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "type=S;default=S;required=B",
+ &type_str, &default_value, &required)) {
+ msg_err_config("cannot get parameters list: %e", err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ if (type_str) {
+ if (!ucl_object_string_to_type(type_str, &type)) {
+ msg_err_config("invalid type: %s", type_str);
+ }
+ }
+ }
+ }
+
+ rspamd_rcl_add_doc_by_path(cfg, path, doc_string, option,
+ type, NULL, 0, default_value, required);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_add_example(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg;
+ const gchar *path = NULL, *option, *doc_string, *example;
+ gsize example_len;
+
+ cfg = lua_check_config(L, 1);
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ path = luaL_checkstring(L, 2);
+ }
+
+ option = luaL_checkstring(L, 3);
+ doc_string = luaL_checkstring(L, 4);
+ example = luaL_checklstring(L, 5, &example_len);
+
+ if (cfg && option && doc_string && example) {
+
+ rspamd_rcl_add_doc_by_example(cfg, path, doc_string, option,
+ example, example_len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_get_cpu_flags(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_cryptobox_library_ctx *crypto_ctx;
+
+ if (cfg != NULL) {
+ crypto_ctx = cfg->libs_ctx->crypto_ctx;
+ lua_newtable(L);
+
+ if (crypto_ctx->cpu_config & CPUID_SSSE3) {
+ lua_pushstring(L, "ssse3");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_SSE41) {
+ lua_pushstring(L, "sse41");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_SSE42) {
+ lua_pushstring(L, "sse42");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_SSE2) {
+ lua_pushstring(L, "sse2");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_SSE3) {
+ lua_pushstring(L, "sse3");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_AVX) {
+ lua_pushstring(L, "avx");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_AVX2) {
+ lua_pushstring(L, "avx2");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_has_torch(lua_State *L)
+{
+ msg_warn("use of the obsoleted `has_torch` function");
+ lua_pushboolean(L, false);
+
+ return 1;
+}
+
+static gint
+lua_config_experimental_enabled(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ lua_pushboolean(L, cfg->enable_experimental);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct rspamd_lua_include_trace_cbdata {
+ lua_State *L;
+ gint cbref;
+};
+
+static void
+lua_include_trace_cb(struct ucl_parser *parser,
+ const ucl_object_t *parent,
+ const ucl_object_t *args,
+ const char *path,
+ size_t pathlen,
+ void *user_data)
+{
+ struct rspamd_lua_include_trace_cbdata *cbdata =
+ (struct rspamd_lua_include_trace_cbdata *) user_data;
+ gint err_idx;
+ lua_State *L;
+
+ L = cbdata->L;
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbdata->cbref);
+ /* Current filename */
+ lua_pushstring(L, ucl_parser_get_cur_file(parser));
+ /* Included filename */
+ lua_pushlstring(L, path, pathlen);
+ /* Params */
+ if (args) {
+ ucl_object_push_lua(L, args, true);
+ }
+ else {
+ lua_newtable(L);
+ }
+ /* Parent */
+ if (parent) {
+ lua_pushstring(L, ucl_object_key(parent));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (lua_pcall(L, 4, 0, err_idx) != 0) {
+ msg_err("lua call to local include trace failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+}
+
+#define LUA_TABLE_TO_HASH(htb, idx) \
+ do { \
+ lua_pushstring(L, (idx)); \
+ lua_gettable(L, -2); \
+ if (lua_isstring(L, -1)) { \
+ g_hash_table_insert((htb), (idx), g_strdup(lua_tostring(L, -1))); \
+ } \
+ lua_pop(L, 1); \
+ } while (0)
+
+static gint
+lua_config_load_ucl(lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *filename;
+ GHashTable *paths = g_hash_table_new_full(rspamd_str_hash, rspamd_str_equal,
+ NULL, g_free);
+ GError *err = NULL;
+
+ if (cfg) {
+ if (lua_isstring(L, 2)) {
+ filename = lua_tostring(L, 2);
+ }
+ else {
+ filename = RSPAMD_CONFDIR "/rspamd.conf";
+ }
+
+ /* Convert rspamd_paths */
+ lua_getglobal(L, "rspamd_paths");
+
+ if (lua_istable(L, -1)) {
+ LUA_TABLE_TO_HASH(paths, RSPAMD_CONFDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_LOCAL_CONFDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_RUNDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_DBDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_LOGDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_WWWDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_PLUGINSDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_RULESDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_LUALIBDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_PREFIX_INDEX);
+ }
+
+ lua_pop(L, 1);
+
+ if (lua_isfunction(L, 3)) {
+ struct rspamd_lua_include_trace_cbdata cbd;
+
+ lua_pushvalue(L, 3);
+ cbd.cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ cbd.L = L;
+
+ if (!rspamd_config_parse_ucl(cfg, filename, paths,
+ lua_include_trace_cb, &cbd, lua_toboolean(L, 4), &err)) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbd.cbref);
+ lua_pushboolean(L, false);
+ lua_pushfstring(L, "failed to load config: %s", err->message);
+ g_error_free(err);
+ g_hash_table_unref(paths);
+
+ return 2;
+ }
+
+ luaL_unref(L, LUA_REGISTRYINDEX, cbd.cbref);
+ }
+ else {
+ if (!rspamd_config_parse_ucl(cfg, filename, paths, NULL, NULL,
+ lua_toboolean(L, 3), &err)) {
+ lua_pushboolean(L, false);
+ lua_pushfstring(L, "failed to load config: %s", err->message);
+ g_error_free(err);
+ g_hash_table_unref(paths);
+
+ return 2;
+ }
+ }
+
+ rspamd_rcl_maybe_apply_lua_transform(cfg);
+ rspamd_config_calculate_cksum(cfg);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ g_hash_table_unref(paths);
+ lua_pushboolean(L, true);
+
+ return 1;
+}
+
+#undef IDX_TO_HASH
+
+static gint
+lua_config_parse_rcl(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ GHashTable *excluded = g_hash_table_new_full(rspamd_str_hash, rspamd_str_equal,
+ g_free, NULL);
+ GError *err = NULL;
+ struct rspamd_rcl_sections_map *top;
+
+ if (cfg) {
+ if (lua_istable(L, 2)) {
+ lua_pushvalue(L, 2);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ g_hash_table_insert(excluded, g_strdup(lua_tostring(L, -1)),
+ GINT_TO_POINTER(-1));
+ }
+
+ lua_pop(L, 1);
+ }
+
+ top = rspamd_rcl_config_init(cfg, excluded);
+
+ if (!rspamd_rcl_parse(top, cfg, cfg, cfg->cfg_pool, cfg->cfg_ucl_obj, &err)) {
+ lua_pushboolean(L, false);
+ lua_pushfstring(L, "failed to load config: %s", err->message);
+ g_error_free(err);
+ g_hash_table_unref(excluded);
+ rspamd_rcl_sections_free(top);
+
+ return 2;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ g_hash_table_unref(excluded);
+ rspamd_rcl_sections_free(top);
+ lua_pushboolean(L, true);
+
+ return 1;
+}
+
+static gint
+lua_config_init_modules(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ rspamd_lua_post_load_config(cfg);
+ lua_pushboolean(L, rspamd_init_filters(cfg, false, false));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_init_subsystem(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *subsystem = luaL_checkstring(L, 2);
+ gchar **parts;
+ guint nparts, i;
+
+ if (cfg != NULL && subsystem != NULL) {
+ parts = g_strsplit_set(subsystem, ";,", -1);
+ nparts = g_strv_length(parts);
+
+ for (i = 0; i < nparts; i++) {
+ if (strcmp(parts[i], "filters") == 0) {
+ rspamd_lua_post_load_config(cfg);
+ rspamd_init_filters(cfg, false, false);
+ }
+ else if (strcmp(parts[i], "langdet") == 0) {
+ if (!cfg->lang_det) {
+ cfg->lang_det = rspamd_language_detector_init(cfg);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_language_detector_unref,
+ cfg->lang_det);
+ }
+ }
+ else if (strcmp(parts[i], "stat") == 0) {
+ rspamd_stat_init(cfg, NULL);
+ }
+ else if (strcmp(parts[i], "dns") == 0) {
+ struct ev_loop *ev_base = lua_check_ev_base(L, 3);
+
+ if (ev_base) {
+ cfg->dns_resolver = rspamd_dns_resolver_init(rspamd_log_default_logger(),
+ ev_base,
+ cfg);
+ }
+ else {
+ g_strfreev(parts);
+
+ return luaL_error(L, "no event base specified");
+ }
+ }
+ else if (strcmp(parts[i], "symcache") == 0) {
+ rspamd_symcache_init(cfg->cache);
+ }
+ else {
+ int ret = luaL_error(L, "invalid param: %s", parts[i]);
+ g_strfreev(parts);
+
+ return ret;
+ }
+ }
+
+ g_strfreev(parts);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_register_re_selector(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = luaL_checkstring(L, 2);
+ const gchar *selector_str = luaL_checkstring(L, 3);
+ const gchar *delimiter = "";
+ bool flatten = false;
+ gint top = lua_gettop(L);
+ bool res = false;
+
+ if (cfg && name && selector_str) {
+ if (lua_gettop(L) >= 4) {
+ delimiter = luaL_checkstring(L, 4);
+
+ if (lua_isboolean(L, 5)) {
+ flatten = lua_toboolean(L, 5);
+ }
+ }
+
+ if (luaL_dostring(L, "return require \"lua_selectors\"") != 0) {
+ msg_warn_config("cannot require lua_selectors: %s",
+ lua_tostring(L, -1));
+ }
+ else {
+ if (lua_type(L, -1) != LUA_TTABLE) {
+ msg_warn_config("lua selectors must return "
+ "table and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ else {
+ lua_pushstring(L, "create_selector_closure");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_warn_config("create_selector_closure must return "
+ "function and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ else {
+ gint err_idx, ret;
+ struct rspamd_config **pcfg;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ /* Push function */
+ lua_pushvalue(L, -2);
+
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ lua_pushstring(L, selector_str);
+ lua_pushstring(L, delimiter);
+ lua_pushboolean(L, flatten);
+
+ if ((ret = lua_pcall(L, 4, 1, err_idx)) != 0) {
+ msg_err_config("call to create_selector_closure lua "
+ "script failed (%d): %s",
+ ret,
+ lua_tostring(L, -1));
+ }
+ else {
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_warn_config("create_selector_closure "
+ "invocation must return "
+ "function and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ else {
+ ret = luaL_ref(L, LUA_REGISTRYINDEX);
+ rspamd_re_cache_add_selector(cfg->re_cache,
+ name, ret);
+ res = true;
+ }
+ }
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_settop(L, top);
+ lua_pushboolean(L, res);
+
+ if (res) {
+ msg_info_config("registered regexp selector %s", name);
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_tld_path(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ lua_pushstring(L, cfg->tld_file);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_dns_max_requests(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ lua_pushinteger(L, cfg->dns_max_requests);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_dns_timeout(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ lua_pushnumber(L, cfg->dns_timeout);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_monitored_alive(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_monitored *m = lua_check_monitored(L, 1);
+
+ if (m) {
+ lua_pushboolean(L, rspamd_monitored_alive(m));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_monitored_offline(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_monitored *m = lua_check_monitored(L, 1);
+
+ if (m) {
+ lua_pushnumber(L, rspamd_monitored_offline_time(m));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_monitored_total_offline(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_monitored *m = lua_check_monitored(L, 1);
+
+ if (m) {
+ lua_pushnumber(L, rspamd_monitored_total_offline_time(m));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_monitored_latency(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_monitored *m = lua_check_monitored(L, 1);
+
+ if (m) {
+ lua_pushnumber(L, rspamd_monitored_latency(m));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+void luaopen_config(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{config}", configlib_m);
+
+ lua_pop(L, 1);
+
+ rspamd_lua_new_class(L, "rspamd{monitored}", monitoredlib_m);
+
+ lua_pop(L, 1);
+}
+
+void lua_call_finish_script(struct rspamd_config_cfg_lua_script *sc,
+ struct rspamd_task *task)
+{
+
+ struct rspamd_task **ptask;
+ struct thread_entry *thread;
+
+ thread = lua_thread_pool_get_for_task(task);
+ thread->task = task;
+
+ lua_State *L = thread->lua_state;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, sc->cbref);
+
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+
+ lua_thread_call(thread, 1);
+}
diff --git a/src/lua/lua_cryptobox.c b/src/lua/lua_cryptobox.c
new file mode 100644
index 0000000..70c6f0a
--- /dev/null
+++ b/src/lua/lua_cryptobox.c
@@ -0,0 +1,3065 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/***
+ * @module rspamd_cryptobox
+ * Rspamd cryptobox is a module that operates with digital signatures and
+ * hashes.
+ * @example
+ * local hash = require "rspamd_cryptobox_hash"
+ *
+ * local h = hash.create()
+ * h:update('hello world')
+ * print(h:hex())
+ */
+
+
+#include "lua_common.h"
+#include "libcryptobox/cryptobox.h"
+#include "libcryptobox/keypair.h"
+#include "libcryptobox/keypair_private.h"
+#include "unix-std.h"
+#include "contrib/libottery/ottery.h"
+#include "libutil/ref.h"
+
+#include <stdalign.h>
+#include <openssl/hmac.h>
+
+
+enum lua_cryptobox_hash_type {
+ LUA_CRYPTOBOX_HASH_BLAKE2 = 0,
+ LUA_CRYPTOBOX_HASH_SSL,
+ LUA_CRYPTOBOX_HASH_HMAC,
+ LUA_CRYPTOBOX_HASH_XXHASH64,
+ LUA_CRYPTOBOX_HASH_XXHASH32,
+ LUA_CRYPTOBOX_HASH_XXHASH3,
+ LUA_CRYPTOBOX_HASH_MUM,
+ LUA_CRYPTOBOX_HASH_T1HA,
+};
+
+struct rspamd_lua_cryptobox_hash {
+ union {
+ rspamd_cryptobox_hash_state_t *h;
+ EVP_MD_CTX *c;
+ HMAC_CTX *hmac_c;
+ rspamd_cryptobox_fast_hash_state_t *fh;
+ } content;
+
+ unsigned char out[rspamd_cryptobox_HASHBYTES];
+
+ uint8_t type;
+ uint8_t out_len;
+ uint8_t is_finished;
+
+ ref_entry_t ref;
+};
+
+LUA_FUNCTION_DEF(cryptobox_pubkey, load);
+LUA_FUNCTION_DEF(cryptobox_pubkey, create);
+LUA_FUNCTION_DEF(cryptobox_pubkey, gc);
+LUA_FUNCTION_DEF(cryptobox_keypair, load);
+LUA_FUNCTION_DEF(cryptobox_keypair, create);
+LUA_FUNCTION_DEF(cryptobox_keypair, gc);
+LUA_FUNCTION_DEF(cryptobox_keypair, totable);
+LUA_FUNCTION_DEF(cryptobox_keypair, get_type);
+LUA_FUNCTION_DEF(cryptobox_keypair, get_alg);
+LUA_FUNCTION_DEF(cryptobox_keypair, get_pk);
+LUA_FUNCTION_DEF(cryptobox_signature, create);
+LUA_FUNCTION_DEF(cryptobox_signature, load);
+LUA_FUNCTION_DEF(cryptobox_signature, save);
+LUA_FUNCTION_DEF(cryptobox_signature, gc);
+LUA_FUNCTION_DEF(cryptobox_signature, hex);
+LUA_FUNCTION_DEF(cryptobox_signature, base32);
+LUA_FUNCTION_DEF(cryptobox_signature, base64);
+LUA_FUNCTION_DEF(cryptobox_signature, bin);
+LUA_FUNCTION_DEF(cryptobox_hash, create);
+LUA_FUNCTION_DEF(cryptobox_hash, create_specific);
+LUA_FUNCTION_DEF(cryptobox_hash, create_specific_keyed);
+LUA_FUNCTION_DEF(cryptobox_hash, create_keyed);
+LUA_FUNCTION_DEF(cryptobox_hash, update);
+LUA_FUNCTION_DEF(cryptobox_hash, reset);
+LUA_FUNCTION_DEF(cryptobox_hash, hex);
+LUA_FUNCTION_DEF(cryptobox_hash, base32);
+LUA_FUNCTION_DEF(cryptobox_hash, base64);
+LUA_FUNCTION_DEF(cryptobox_hash, bin);
+LUA_FUNCTION_DEF(cryptobox_hash, gc);
+LUA_FUNCTION_DEF(cryptobox, verify_memory);
+LUA_FUNCTION_DEF(cryptobox, verify_file);
+LUA_FUNCTION_DEF(cryptobox, sign_file);
+LUA_FUNCTION_DEF(cryptobox, sign_memory);
+LUA_FUNCTION_DEF(cryptobox, encrypt_memory);
+LUA_FUNCTION_DEF(cryptobox, encrypt_file);
+LUA_FUNCTION_DEF(cryptobox, decrypt_memory);
+LUA_FUNCTION_DEF(cryptobox, decrypt_file);
+LUA_FUNCTION_DEF(cryptobox, encrypt_cookie);
+LUA_FUNCTION_DEF(cryptobox, decrypt_cookie);
+LUA_FUNCTION_DEF(cryptobox, pbkdf);
+LUA_FUNCTION_DEF(cryptobox, gen_dkim_keypair);
+
+/* Secretbox API: uses libsodium secretbox and blake2b for key derivation */
+LUA_FUNCTION_DEF(cryptobox_secretbox, create);
+LUA_FUNCTION_DEF(cryptobox_secretbox, encrypt);
+LUA_FUNCTION_DEF(cryptobox_secretbox, decrypt);
+LUA_FUNCTION_DEF(cryptobox_secretbox, gc);
+
+static const struct luaL_reg cryptoboxlib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox, verify_memory),
+ LUA_INTERFACE_DEF(cryptobox, verify_file),
+ LUA_INTERFACE_DEF(cryptobox, sign_memory),
+ LUA_INTERFACE_DEF(cryptobox, sign_file),
+ LUA_INTERFACE_DEF(cryptobox, encrypt_memory),
+ LUA_INTERFACE_DEF(cryptobox, encrypt_file),
+ LUA_INTERFACE_DEF(cryptobox, decrypt_memory),
+ LUA_INTERFACE_DEF(cryptobox, decrypt_file),
+ LUA_INTERFACE_DEF(cryptobox, encrypt_cookie),
+ LUA_INTERFACE_DEF(cryptobox, decrypt_cookie),
+ LUA_INTERFACE_DEF(cryptobox, pbkdf),
+ LUA_INTERFACE_DEF(cryptobox, gen_dkim_keypair),
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxpubkeylib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox_pubkey, load),
+ LUA_INTERFACE_DEF(cryptobox_pubkey, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxpubkeylib_m[] = {
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_cryptobox_pubkey_gc},
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxkeypairlib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox_keypair, load),
+ LUA_INTERFACE_DEF(cryptobox_keypair, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxkeypairlib_m[] = {
+ {"__tostring", rspamd_lua_class_tostring},
+ {"totable", lua_cryptobox_keypair_totable},
+ {"get_type", lua_cryptobox_keypair_get_type},
+ {"get_alg", lua_cryptobox_keypair_get_alg},
+ {"type", lua_cryptobox_keypair_get_type},
+ {"alg", lua_cryptobox_keypair_get_alg},
+ {"pk", lua_cryptobox_keypair_get_pk},
+ {"pubkey", lua_cryptobox_keypair_get_pk},
+ {"__gc", lua_cryptobox_keypair_gc},
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxsignlib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox_signature, load),
+ LUA_INTERFACE_DEF(cryptobox_signature, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxsignlib_m[] = {
+ LUA_INTERFACE_DEF(cryptobox_signature, save),
+ LUA_INTERFACE_DEF(cryptobox_signature, hex),
+ LUA_INTERFACE_DEF(cryptobox_signature, base32),
+ LUA_INTERFACE_DEF(cryptobox_signature, base64),
+ LUA_INTERFACE_DEF(cryptobox_signature, bin),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_cryptobox_signature_gc},
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxhashlib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox_hash, create),
+ LUA_INTERFACE_DEF(cryptobox_hash, create_keyed),
+ LUA_INTERFACE_DEF(cryptobox_hash, create_specific),
+ LUA_INTERFACE_DEF(cryptobox_hash, create_specific_keyed),
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxhashlib_m[] = {
+ LUA_INTERFACE_DEF(cryptobox_hash, update),
+ LUA_INTERFACE_DEF(cryptobox_hash, reset),
+ LUA_INTERFACE_DEF(cryptobox_hash, hex),
+ LUA_INTERFACE_DEF(cryptobox_hash, base32),
+ LUA_INTERFACE_DEF(cryptobox_hash, base64),
+ LUA_INTERFACE_DEF(cryptobox_hash, bin),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_cryptobox_hash_gc},
+ {NULL, NULL}};
+
+
+static const struct luaL_reg cryptoboxsecretboxlib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox_secretbox, create),
+ {NULL, NULL},
+};
+
+static const struct luaL_reg cryptoboxsecretboxlib_m[] = {
+ LUA_INTERFACE_DEF(cryptobox_secretbox, encrypt),
+ LUA_INTERFACE_DEF(cryptobox_secretbox, decrypt),
+ {"__gc", lua_cryptobox_secretbox_gc},
+ {NULL, NULL},
+};
+
+struct rspamd_lua_cryptobox_secretbox {
+ guchar sk[crypto_secretbox_KEYBYTES];
+};
+
+static struct rspamd_cryptobox_pubkey *
+lua_check_cryptobox_pubkey(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cryptobox_pubkey}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'cryptobox_pubkey' expected");
+ return ud ? *((struct rspamd_cryptobox_pubkey **) ud) : NULL;
+}
+
+static struct rspamd_cryptobox_keypair *
+lua_check_cryptobox_keypair(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cryptobox_keypair}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'cryptobox_keypair' expected");
+ return ud ? *((struct rspamd_cryptobox_keypair **) ud) : NULL;
+}
+
+static rspamd_fstring_t *
+lua_check_cryptobox_sign(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cryptobox_signature}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'cryptobox_signature' expected");
+ return ud ? *((rspamd_fstring_t **) ud) : NULL;
+}
+
+struct rspamd_lua_cryptobox_hash *
+lua_check_cryptobox_hash(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cryptobox_hash}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'cryptobox_hash' expected");
+ return ud ? *((struct rspamd_lua_cryptobox_hash **) ud) : NULL;
+}
+
+static struct rspamd_lua_cryptobox_secretbox *
+lua_check_cryptobox_secretbox(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cryptobox_secretbox}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'cryptobox_secretbox' expected");
+ return ud ? *((struct rspamd_lua_cryptobox_secretbox **) ud) : NULL;
+}
+
+/***
+ * @function rspamd_cryptobox_pubkey.load(file[, type[, alg]])
+ * Loads public key from base32 encoded file
+ * @param {string} file filename to load
+ * @param {string} type optional 'sign' or 'kex' for signing and encryption
+ * @param {string} alg optional 'default' or 'nist' for curve25519/nistp256 keys
+ * @return {cryptobox_pubkey} new public key
+ */
+static gint
+lua_cryptobox_pubkey_load(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_pubkey *pkey = NULL, **ppkey;
+ const gchar *filename, *arg;
+ gint type = RSPAMD_KEYPAIR_SIGN;
+ gint alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ guchar *map;
+ gsize len;
+
+ filename = luaL_checkstring(L, 1);
+ if (filename != NULL) {
+ map = rspamd_file_xmap(filename, PROT_READ, &len, TRUE);
+
+ if (map == NULL) {
+ msg_err("cannot open pubkey from file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* keypair type */
+ arg = lua_tostring(L, 2);
+
+ if (strcmp(arg, "sign") == 0) {
+ type = RSPAMD_KEYPAIR_SIGN;
+ }
+ else if (strcmp(arg, "kex") == 0) {
+ type = RSPAMD_KEYPAIR_KEX;
+ }
+ }
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ /* algorithm */
+ arg = lua_tostring(L, 3);
+
+ if (strcmp(arg, "default") == 0 || strcmp(arg, "curve25519") == 0) {
+ type = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else if (strcmp(arg, "nist") == 0) {
+ type = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ }
+
+ pkey = rspamd_pubkey_from_base32(map, len, type, alg);
+
+ if (pkey == NULL) {
+ msg_err("cannot open pubkey from file: %s", filename);
+ munmap(map, len);
+ lua_pushnil(L);
+ }
+ else {
+ munmap(map, len);
+ ppkey = lua_newuserdata(L, sizeof(void *));
+ rspamd_lua_setclass(L, "rspamd{cryptobox_pubkey}", -1);
+ *ppkey = pkey;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "bad input arguments");
+ }
+
+ return 1;
+}
+
+
+/***
+ * @function rspamd_cryptobox_pubkey.create(data[, type[, alg]])
+ * Loads public key from base32 encoded string
+ * @param {base32 string} base32 string with the key
+ * @param {string} type optional 'sign' or 'kex' for signing and encryption
+ * @param {string} alg optional 'default' or 'nist' for curve25519/nistp256 keys
+ * @return {cryptobox_pubkey} new public key
+ */
+static gint
+lua_cryptobox_pubkey_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_pubkey *pkey = NULL, **ppkey;
+ const gchar *buf, *arg;
+ gsize len;
+ gint type = RSPAMD_KEYPAIR_SIGN;
+ gint alg = RSPAMD_CRYPTOBOX_MODE_25519;
+
+ buf = luaL_checklstring(L, 1, &len);
+ if (buf != NULL) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* keypair type */
+ arg = lua_tostring(L, 2);
+
+ if (strcmp(arg, "sign") == 0) {
+ type = RSPAMD_KEYPAIR_SIGN;
+ }
+ else if (strcmp(arg, "kex") == 0) {
+ type = RSPAMD_KEYPAIR_KEX;
+ }
+ }
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ /* algorithm */
+ arg = lua_tostring(L, 3);
+
+ if (strcmp(arg, "default") == 0 || strcmp(arg, "curve25519") == 0) {
+ type = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else if (strcmp(arg, "nist") == 0) {
+ type = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ }
+
+ pkey = rspamd_pubkey_from_base32(buf, len, type, alg);
+
+ if (pkey == NULL) {
+ msg_err("cannot load pubkey from string");
+ lua_pushnil(L);
+ }
+ else {
+ ppkey = lua_newuserdata(L, sizeof(void *));
+ rspamd_lua_setclass(L, "rspamd{cryptobox_pubkey}", -1);
+ *ppkey = pkey;
+ }
+ }
+ else {
+ return luaL_error(L, "bad input arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_cryptobox_pubkey_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_pubkey *pkey = lua_check_cryptobox_pubkey(L, 1);
+
+ if (pkey != NULL) {
+ rspamd_pubkey_unref(pkey);
+ }
+
+ return 0;
+}
+
+/***
+ * @function rspamd_cryptobox_keypair.load(file|table)
+ * Loads public key from UCL file or directly from Lua
+ * @param {string} file filename to load
+ * @return {cryptobox_keypair} new keypair
+ */
+static gint
+lua_cryptobox_keypair_load(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp, **pkp;
+ const gchar *buf;
+ gsize len;
+ struct ucl_parser *parser;
+ ucl_object_t *obj;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ buf = luaL_checklstring(L, 1, &len);
+ if (buf != NULL) {
+ parser = ucl_parser_new(0);
+
+ if (!ucl_parser_add_chunk(parser, buf, len)) {
+ msg_err("cannot open keypair from data: %s",
+ ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+ lua_pushnil(L);
+ }
+ else {
+ obj = ucl_parser_get_object(parser);
+ kp = rspamd_keypair_from_ucl(obj);
+ ucl_parser_free(parser);
+
+ if (kp == NULL) {
+ msg_err("cannot load keypair from data");
+ ucl_object_unref(obj);
+ lua_pushnil(L);
+ }
+ else {
+ pkp = lua_newuserdata(L, sizeof(gpointer));
+ *pkp = kp;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_keypair}", -1);
+ ucl_object_unref(obj);
+ }
+ }
+ }
+ else {
+ luaL_error(L, "bad input arguments");
+ }
+ }
+ else {
+ /* Directly import from lua */
+ obj = ucl_object_lua_import(L, 1);
+ kp = rspamd_keypair_from_ucl(obj);
+
+ if (kp == NULL) {
+ msg_err("cannot load keypair from data");
+ ucl_object_unref(obj);
+ lua_pushnil(L);
+ }
+ else {
+ pkp = lua_newuserdata(L, sizeof(gpointer));
+ *pkp = kp;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_keypair}", -1);
+ ucl_object_unref(obj);
+ }
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_keypair.create([type='encryption'[, alg='curve25519']])
+ * Generates new keypair
+ * @param {string} type type of keypair: 'encryption' (default) or 'sign'
+ * @param {string} alg algorithm of keypair: 'curve25519' (default) or 'nist'
+ * @return {cryptobox_keypair} new keypair
+ */
+static gint
+lua_cryptobox_keypair_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp, **pkp;
+ enum rspamd_cryptobox_keypair_type type = RSPAMD_KEYPAIR_KEX;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
+
+ if (lua_isstring(L, 1)) {
+ const gchar *str = lua_tostring(L, 1);
+
+ if (strcmp(str, "sign") == 0) {
+ type = RSPAMD_KEYPAIR_SIGN;
+ }
+ else if (strcmp(str, "encryption") == 0) {
+ type = RSPAMD_KEYPAIR_KEX;
+ }
+ else {
+ return luaL_error(L, "invalid keypair type: %s", str);
+ }
+ }
+
+ if (lua_isstring(L, 2)) {
+ const gchar *str = lua_tostring(L, 2);
+
+ if (strcmp(str, "nist") == 0 || strcmp(str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp(str, "curve25519") == 0 || strcmp(str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ return luaL_error(L, "invalid keypair algorithm: %s", str);
+ }
+ }
+
+ kp = rspamd_keypair_new(type, alg);
+
+ pkp = lua_newuserdata(L, sizeof(gpointer));
+ *pkp = kp;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_keypair}", -1);
+
+ return 1;
+}
+
+static gint
+lua_cryptobox_keypair_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair(L, 1);
+
+ if (kp != NULL) {
+ rspamd_keypair_unref(kp);
+ }
+
+ return 0;
+}
+
+/***
+ * @method keypair:totable([hex=false]])
+ * Converts keypair to table (not very safe due to memory leftovers)
+ */
+static gint
+lua_cryptobox_keypair_totable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair(L, 1);
+ ucl_object_t *obj;
+ gboolean hex = FALSE;
+ gint ret = 1;
+
+ if (kp != NULL) {
+
+ if (lua_isboolean(L, 2)) {
+ hex = lua_toboolean(L, 2);
+ }
+
+ obj = rspamd_keypair_to_ucl(kp, hex ? RSPAMD_KEYPAIR_DUMP_HEX : RSPAMD_KEYPAIR_DUMP_DEFAULT);
+
+ ret = ucl_object_push_lua(L, obj, true);
+ ucl_object_unref(obj);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return ret;
+}
+/***
+ * @method keypair:type()
+ * Returns type of keypair as a string: 'encryption' or 'sign'
+ * @return {string} type of keypair as a string
+ */
+static gint
+lua_cryptobox_keypair_get_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair(L, 1);
+
+ if (kp) {
+ if (kp->type == RSPAMD_KEYPAIR_KEX) {
+ lua_pushstring(L, "encryption");
+ }
+ else {
+ lua_pushstring(L, "sign");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method keypair:alg()
+ * Returns algorithm of keypair as a string: 'encryption' or 'sign'
+ * @return {string} type of keypair as a string
+ */
+static gint
+lua_cryptobox_keypair_get_alg(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair(L, 1);
+
+ if (kp) {
+ if (kp->alg == RSPAMD_CRYPTOBOX_MODE_25519) {
+ lua_pushstring(L, "curve25519");
+ }
+ else {
+ lua_pushstring(L, "nist");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method keypair:pk()
+ * Returns pubkey for a specific keypair
+ * @return {rspamd_pubkey} pubkey for a keypair
+ */
+static gint
+lua_cryptobox_keypair_get_pk(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair(L, 1);
+ struct rspamd_cryptobox_pubkey *pk, **ppk;
+ const guchar *data;
+ guint dlen;
+
+ if (kp) {
+ data = rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_PK, &dlen);
+ pk = rspamd_pubkey_from_bin(data, dlen, kp->type, kp->alg);
+
+ if (pk == NULL) {
+ return luaL_error(L, "invalid keypair");
+ }
+
+ ppk = lua_newuserdata(L, sizeof(*ppk));
+ *ppk = pk;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_pubkey}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_signature.load(file, [alg = 'curve25519'])
+ * Loads signature from raw file
+ * @param {string} file filename to load
+ * @return {cryptobox_signature} new signature
+ */
+static gint
+lua_cryptobox_signature_load(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig, **psig;
+ const gchar *filename;
+ gpointer data;
+ int fd;
+ struct stat st;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
+
+ filename = luaL_checkstring(L, 1);
+ if (filename != NULL) {
+ fd = open(filename, O_RDONLY);
+ if (fd == -1) {
+ msg_err("cannot open signature file: %s, %s", filename,
+ strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (fstat(fd, &st) == -1 ||
+ (data =
+ mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
+ msg_err("cannot mmap file %s: %s", filename, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (lua_isstring(L, 2)) {
+ const gchar *str = lua_tostring(L, 2);
+
+ if (strcmp(str, "nist") == 0 || strcmp(str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp(str, "curve25519") == 0 || strcmp(str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ munmap(data, st.st_size);
+ close(fd);
+
+ return luaL_error(L, "invalid keypair algorithm: %s", str);
+ }
+ }
+ if (st.st_size > 0) {
+ sig = rspamd_fstring_new_init(data, st.st_size);
+ psig = lua_newuserdata(L, sizeof(rspamd_fstring_t *));
+ rspamd_lua_setclass(L, "rspamd{cryptobox_signature}", -1);
+ *psig = sig;
+ }
+ else {
+ msg_err("size of %s mismatches: %d while %d is expected",
+ filename, (int) st.st_size,
+ rspamd_cryptobox_signature_bytes(alg));
+ lua_pushnil(L);
+ }
+
+ munmap(data, st.st_size);
+ }
+ close(fd);
+ }
+ }
+ else {
+ luaL_error(L, "bad input arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method rspamd_cryptobox_signature:save(file)
+ * Stores signature in raw file
+ * @param {string} file filename to use
+ * @return {boolean} true if signature has been saved
+ */
+static gint
+lua_cryptobox_signature_save(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig;
+ gint fd, flags;
+ const gchar *filename;
+ gboolean forced = FALSE, res = TRUE;
+
+ sig = lua_check_cryptobox_sign(L, 1);
+ filename = luaL_checkstring(L, 2);
+
+ if (!sig || !filename) {
+ luaL_error(L, "bad input arguments");
+ return 1;
+ }
+
+ if (lua_gettop(L) > 2) {
+ forced = lua_toboolean(L, 3);
+ }
+
+ if (sig != NULL && filename != NULL) {
+ flags = O_WRONLY | O_CREAT;
+ if (forced) {
+ flags |= O_TRUNC;
+ }
+ else {
+ flags |= O_EXCL;
+ }
+ fd = open(filename, flags, 00644);
+ if (fd == -1) {
+ msg_err("cannot create a signature file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushboolean(L, FALSE);
+ }
+ else {
+ while (write(fd, sig->str, sig->len) == -1) {
+ if (errno == EINTR) {
+ continue;
+ }
+ msg_err("cannot write to a signature file: %s, %s",
+ filename,
+ strerror(errno));
+ res = FALSE;
+ break;
+ }
+ lua_pushboolean(L, res);
+ close(fd);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_signature.create(data)
+ * Creates signature object from raw data
+ * @param {data} raw signature data
+ * @return {cryptobox_signature} signature object
+ */
+static gint
+lua_cryptobox_signature_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig, **psig;
+ struct rspamd_lua_text *t;
+ const gchar *data;
+ gsize dlen;
+
+ if (lua_isuserdata(L, 1)) {
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ dlen = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 1, &dlen);
+ }
+
+ if (data != NULL) {
+ if (dlen == rspamd_cryptobox_signature_bytes(RSPAMD_CRYPTOBOX_MODE_25519)) {
+ sig = rspamd_fstring_new_init(data, dlen);
+ psig = lua_newuserdata(L, sizeof(rspamd_fstring_t *));
+ rspamd_lua_setclass(L, "rspamd{cryptobox_signature}", -1);
+ *psig = sig;
+ }
+ }
+ else {
+ return luaL_error(L, "bad input arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_signature:hex()
+ * Return hex encoded signature string
+ * @return {string} raw value of signature
+ */
+static gint
+lua_cryptobox_signature_hex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig = lua_check_cryptobox_sign(L, 1);
+ gchar *encoded;
+
+ if (sig) {
+ encoded = rspamd_encode_hex(sig->str, sig->len);
+ lua_pushstring(L, encoded);
+ g_free(encoded);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_signature:base32([b32type='default'])
+ * Return base32 encoded signature string
+ * @param {string} b32type base32 type (default, bleach, rfc)
+ * @return {string} raw value of signature
+ */
+static gint
+lua_cryptobox_signature_base32(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig = lua_check_cryptobox_sign(L, 1);
+ gchar *encoded;
+ enum rspamd_base32_type btype = RSPAMD_BASE32_DEFAULT;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ btype = rspamd_base32_decode_type_from_str(lua_tostring(L, 2));
+
+ if (btype == RSPAMD_BASE32_INVALID) {
+ return luaL_error(L, "invalid b32 type: %s", lua_tostring(L, 2));
+ }
+ }
+
+ if (sig) {
+ encoded = rspamd_encode_base32(sig->str, sig->len, btype);
+ lua_pushstring(L, encoded);
+ g_free(encoded);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_signature:base64()
+ * Return base64 encoded signature string
+ * @return {string} raw value of signature
+ */
+static gint
+lua_cryptobox_signature_base64(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig = lua_check_cryptobox_sign(L, 1);
+ gsize dlen;
+ gchar *encoded;
+
+ if (sig) {
+ encoded = rspamd_encode_base64(sig->str, sig->len, 0, &dlen);
+ lua_pushlstring(L, encoded, dlen);
+ g_free(encoded);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_signature:bin()
+ * Return raw signature string
+ * @return {string} raw value of signature
+ */
+static gint
+lua_cryptobox_signature_bin(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig = lua_check_cryptobox_sign(L, 1);
+
+ if (sig) {
+ lua_pushlstring(L, sig->str, sig->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_cryptobox_signature_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig = lua_check_cryptobox_sign(L, 1);
+
+ rspamd_fstring_free(sig);
+
+ return 0;
+}
+
+static void
+rspamd_lua_hash_update(struct rspamd_lua_cryptobox_hash *h,
+ const void *p, gsize len)
+{
+ if (h) {
+ switch (h->type) {
+ case LUA_CRYPTOBOX_HASH_BLAKE2:
+ rspamd_cryptobox_hash_update(h->content.h, p, len);
+ break;
+ case LUA_CRYPTOBOX_HASH_SSL:
+ EVP_DigestUpdate(h->content.c, p, len);
+ break;
+ case LUA_CRYPTOBOX_HASH_HMAC:
+ HMAC_Update(h->content.hmac_c, p, len);
+ break;
+ case LUA_CRYPTOBOX_HASH_XXHASH64:
+ case LUA_CRYPTOBOX_HASH_XXHASH32:
+ case LUA_CRYPTOBOX_HASH_XXHASH3:
+ case LUA_CRYPTOBOX_HASH_MUM:
+ case LUA_CRYPTOBOX_HASH_T1HA:
+ rspamd_cryptobox_fast_hash_update(h->content.fh, p, len);
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ }
+}
+
+static void
+lua_cryptobox_hash_dtor(struct rspamd_lua_cryptobox_hash *h)
+{
+ if (h->type == LUA_CRYPTOBOX_HASH_SSL) {
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+ EVP_MD_CTX_cleanup(h->content.c);
+#else
+ EVP_MD_CTX_reset(h->content.c);
+#endif
+ EVP_MD_CTX_destroy(h->content.c);
+ }
+ else if (h->type == LUA_CRYPTOBOX_HASH_HMAC) {
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
+ (defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000)
+ HMAC_CTX_cleanup(h->content.hmac_c);
+ g_free(h->content.hmac_c);
+#else
+ HMAC_CTX_free(h->content.hmac_c);
+#endif
+ }
+ else if (h->type == LUA_CRYPTOBOX_HASH_BLAKE2) {
+ rspamd_explicit_memzero(h->content.h, sizeof(*h->content.h));
+ free(h->content.h); /* Allocated by posix_memalign */
+ }
+ else {
+ rspamd_cryptobox_fast_hash_free(h->content.fh);
+ }
+
+ g_free(h);
+}
+
+static inline void
+rspamd_lua_hash_init_default(struct rspamd_lua_cryptobox_hash *h,
+ const gchar *key, gsize keylen)
+{
+ h->type = LUA_CRYPTOBOX_HASH_BLAKE2;
+ if (posix_memalign((void **) &h->content.h,
+ RSPAMD_ALIGNOF(rspamd_cryptobox_hash_state_t),
+ sizeof(*h->content.h)) != 0) {
+ g_assert_not_reached();
+ }
+
+ rspamd_cryptobox_hash_init(h->content.h, key, keylen);
+ h->out_len = rspamd_cryptobox_HASHBYTES;
+}
+
+static void
+rspamd_lua_ssl_hash_create(struct rspamd_lua_cryptobox_hash *h, const EVP_MD *htype,
+ bool insecure)
+{
+ h->type = LUA_CRYPTOBOX_HASH_SSL;
+ h->content.c = EVP_MD_CTX_create();
+ h->out_len = EVP_MD_size(htype);
+
+ if (insecure) {
+ /* Should never ever be used for crypto/security purposes! */
+#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
+ EVP_MD_CTX_set_flags(h->content.c, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+#endif
+ }
+
+ EVP_DigestInit_ex(h->content.c, htype, NULL);
+}
+
+static void
+rspamd_lua_ssl_hmac_create(struct rspamd_lua_cryptobox_hash *h, const EVP_MD *htype,
+ const gchar *key, gsize keylen,
+ bool insecure)
+{
+ h->type = LUA_CRYPTOBOX_HASH_HMAC;
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
+ (defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000)
+ h->content.hmac_c = g_malloc0(sizeof(*h->content.hmac_c));
+#else
+ h->content.hmac_c = HMAC_CTX_new();
+#endif
+ h->out_len = EVP_MD_size(htype);
+
+#if OPENSSL_VERSION_NUMBER > 0x10100000L
+ if (insecure) {
+ /* Should never ever be used for crypto/security purposes! */
+#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
+ HMAC_CTX_set_flags(h->content.hmac_c, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+#endif
+ }
+#endif
+
+ HMAC_Init_ex(h->content.hmac_c, key, keylen, htype, NULL);
+}
+
+static struct rspamd_lua_cryptobox_hash *
+rspamd_lua_hash_create(const gchar *type, const gchar *key, gsize keylen)
+{
+ struct rspamd_lua_cryptobox_hash *h;
+
+ h = g_malloc0(sizeof(*h));
+ REF_INIT_RETAIN(h, lua_cryptobox_hash_dtor);
+
+ if (type) {
+ if (g_ascii_strcasecmp(type, "md5") == 0) {
+ if (keylen > 0) {
+ rspamd_lua_ssl_hmac_create(h, EVP_md5(), key, keylen, true);
+ }
+ else {
+ rspamd_lua_ssl_hash_create(h, EVP_md5(), true);
+ }
+ }
+ else if (g_ascii_strcasecmp(type, "sha1") == 0 ||
+ g_ascii_strcasecmp(type, "sha") == 0) {
+ if (keylen > 0) {
+ rspamd_lua_ssl_hmac_create(h, EVP_sha1(), key, keylen, true);
+ }
+ else {
+ rspamd_lua_ssl_hash_create(h, EVP_sha1(), true);
+ }
+ }
+ else if (g_ascii_strcasecmp(type, "sha256") == 0) {
+ if (keylen > 0) {
+ rspamd_lua_ssl_hmac_create(h, EVP_sha256(), key, keylen, true);
+ }
+ else {
+ rspamd_lua_ssl_hash_create(h, EVP_sha256(), true);
+ }
+ }
+ else if (g_ascii_strcasecmp(type, "sha512") == 0) {
+ if (keylen > 0) {
+ rspamd_lua_ssl_hmac_create(h, EVP_sha512(), key, keylen, true);
+ }
+ else {
+ rspamd_lua_ssl_hash_create(h, EVP_sha512(), true);
+ }
+ }
+ else if (g_ascii_strcasecmp(type, "sha384") == 0) {
+ if (keylen > 0) {
+ rspamd_lua_ssl_hmac_create(h, EVP_sha384(), key, keylen, true);
+ }
+ else {
+ rspamd_lua_ssl_hash_create(h, EVP_sha384(), true);
+ }
+ }
+ else if (g_ascii_strcasecmp(type, "xxh64") == 0) {
+ h->type = LUA_CRYPTOBOX_HASH_XXHASH64;
+ h->content.fh = rspamd_cryptobox_fast_hash_new();
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH64, 0);
+ h->out_len = sizeof(guint64);
+ }
+ else if (g_ascii_strcasecmp(type, "xxh32") == 0) {
+ h->type = LUA_CRYPTOBOX_HASH_XXHASH32;
+ h->content.fh = rspamd_cryptobox_fast_hash_new();
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH32, 0);
+ h->out_len = sizeof(guint32);
+ }
+ else if (g_ascii_strcasecmp(type, "xxh3") == 0) {
+ h->type = LUA_CRYPTOBOX_HASH_XXHASH3;
+ h->content.fh = rspamd_cryptobox_fast_hash_new();
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH3, 0);
+ h->out_len = sizeof(guint64);
+ }
+ else if (g_ascii_strcasecmp(type, "mum") == 0) {
+ h->type = LUA_CRYPTOBOX_HASH_MUM;
+ h->content.fh = rspamd_cryptobox_fast_hash_new();
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_MUMHASH, 0);
+ h->out_len = sizeof(guint64);
+ }
+ else if (g_ascii_strcasecmp(type, "t1ha") == 0) {
+ h->type = LUA_CRYPTOBOX_HASH_T1HA;
+ h->content.fh = rspamd_cryptobox_fast_hash_new();
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_T1HA, 0);
+ h->out_len = sizeof(guint64);
+ }
+ else if (g_ascii_strcasecmp(type, "blake2") == 0) {
+ rspamd_lua_hash_init_default(h, key, keylen);
+ }
+ else {
+ g_free(h);
+
+ return NULL;
+ }
+ }
+ else {
+ /* Default hash type */
+ rspamd_lua_hash_init_default(h, key, keylen);
+ }
+
+ return h;
+}
+
+/***
+ * @function rspamd_cryptobox_hash.create([string])
+ * Creates new hash context
+ * @param {string} data optional string to hash
+ * @return {cryptobox_hash} hash object
+ */
+static gint
+lua_cryptobox_hash_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h, **ph;
+ const gchar *s = NULL;
+ struct rspamd_lua_text *t;
+ gsize len = 0;
+
+ h = rspamd_lua_hash_create(NULL, NULL, 0);
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = lua_tolstring(L, 1, &len);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ REF_RELEASE(h);
+ return luaL_error(L, "invalid arguments");
+ }
+
+ s = t->start;
+ len = t->len;
+ }
+
+ if (s) {
+ rspamd_lua_hash_update(h, s, len);
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_hash.create_specific(type, [string])
+ * Creates new hash context
+ * @param {string} type type of hash (blake2, sha256, md5, sha512, mum, xxh64, xxh32, t1ha)
+ * @param {string} string initial data
+ * @return {cryptobox_hash} hash object
+ */
+static gint
+lua_cryptobox_hash_create_specific(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h, **ph;
+ const gchar *s = NULL, *type = luaL_checkstring(L, 1);
+ gsize len = 0;
+ struct rspamd_lua_text *t;
+
+ if (!type) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ h = rspamd_lua_hash_create(type, NULL, 0);
+
+ if (h == NULL) {
+ return luaL_error(L, "invalid hash type: %s", type);
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ s = lua_tolstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ REF_RELEASE(h);
+ return luaL_error(L, "invalid arguments");
+ }
+
+ s = t->start;
+ len = t->len;
+ }
+
+ if (s) {
+ rspamd_lua_hash_update(h, s, len);
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_hash.create_keyed(key, [string])
+ * Creates new hash context with specified key
+ * @param {string} key key
+ * @return {cryptobox_hash} hash object
+ */
+static gint
+lua_cryptobox_hash_create_keyed(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h, **ph;
+ const gchar *key, *s = NULL;
+ struct rspamd_lua_text *t;
+ gsize len = 0;
+ gsize keylen;
+
+ key = luaL_checklstring(L, 1, &keylen);
+
+ if (key != NULL) {
+ h = rspamd_lua_hash_create(NULL, key, keylen);
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ s = lua_tolstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ REF_RELEASE(h);
+ return luaL_error(L, "invalid arguments");
+ }
+
+ s = t->start;
+ len = t->len;
+ }
+
+ if (s) {
+ rspamd_lua_hash_update(h, s, len);
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_hash.create_specific_keyed(key, type, [string])
+ * Creates new hash context with specified key
+ * @param {string} key key
+ * @return {cryptobox_hash} hash object
+ */
+static gint
+lua_cryptobox_hash_create_specific_keyed(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h, **ph;
+ const gchar *key, *s = NULL, *type = luaL_checkstring(L, 2);
+ struct rspamd_lua_text *t;
+ gsize len = 0;
+ gsize keylen;
+
+ key = luaL_checklstring(L, 1, &keylen);
+
+ if (key != NULL && type != NULL) {
+ h = rspamd_lua_hash_create(type, key, keylen);
+
+ if (h == NULL) {
+ return luaL_error(L, "invalid hash type: %s", type);
+ }
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ s = lua_tolstring(L, 3, &len);
+ }
+ else if (lua_type(L, 3) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 3);
+
+ if (!t) {
+ REF_RELEASE(h);
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ s = t->start;
+ len = t->len;
+ }
+
+ if (s) {
+ rspamd_lua_hash_update(h, s, len);
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_hash:update(data)
+ * Updates hash with the specified data (hash should not be finalized using `hex` or `bin` methods)
+ * @param {string} data data to hash
+ */
+static gint
+lua_cryptobox_hash_update(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1), **ph;
+ const gchar *data;
+ struct rspamd_lua_text *t;
+ gsize len;
+
+ if (lua_isuserdata(L, 2)) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 2, &len);
+ }
+
+ if (lua_isnumber(L, 3)) {
+ gsize nlen = lua_tonumber(L, 3);
+
+ if (nlen > len) {
+ return luaL_error(L, "invalid length: %d while %d is available",
+ (int) nlen, (int) len);
+ }
+
+ len = nlen;
+ }
+
+ if (h && data) {
+ if (!h->is_finished) {
+ rspamd_lua_hash_update(h, data, len);
+ }
+ else {
+ return luaL_error(L, "hash is already finalized");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ REF_RETAIN(h);
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_hash:reset()
+ * Resets hash to the initial state
+ */
+static gint
+lua_cryptobox_hash_reset(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1), **ph;
+
+ if (h) {
+ switch (h->type) {
+ case LUA_CRYPTOBOX_HASH_BLAKE2:
+ memset(h->content.h, 0, sizeof(*h->content.h));
+ rspamd_cryptobox_hash_init(h->content.h, NULL, 0);
+ break;
+ case LUA_CRYPTOBOX_HASH_SSL:
+ EVP_DigestInit(h->content.c, EVP_MD_CTX_md(h->content.c));
+ break;
+ case LUA_CRYPTOBOX_HASH_HMAC:
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
+ (defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000)
+ /* Old openssl is awesome... */
+ HMAC_Init_ex(h->content.hmac_c, NULL, 0, h->content.hmac_c->md, NULL);
+#else
+ HMAC_CTX_reset(h->content.hmac_c);
+#endif
+ break;
+ case LUA_CRYPTOBOX_HASH_XXHASH64:
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH64, 0);
+ break;
+ case LUA_CRYPTOBOX_HASH_XXHASH32:
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH32, 0);
+ break;
+ case LUA_CRYPTOBOX_HASH_XXHASH3:
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH3, 0);
+ break;
+ case LUA_CRYPTOBOX_HASH_MUM:
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_MUMHASH, 0);
+ break;
+ case LUA_CRYPTOBOX_HASH_T1HA:
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_T1HA, 0);
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ h->is_finished = FALSE;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ REF_RETAIN(h);
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+
+ return 1;
+}
+
+static void
+lua_cryptobox_hash_finish(struct rspamd_lua_cryptobox_hash *h)
+{
+ guint64 ll;
+ guchar out[rspamd_cryptobox_HASHBYTES];
+ guint ssl_outlen = sizeof(out);
+
+ switch (h->type) {
+ case LUA_CRYPTOBOX_HASH_BLAKE2:
+ rspamd_cryptobox_hash_final(h->content.h, out);
+ memcpy(h->out, out, sizeof(out));
+ break;
+ case LUA_CRYPTOBOX_HASH_SSL:
+ EVP_DigestFinal_ex(h->content.c, out, &ssl_outlen);
+ h->out_len = ssl_outlen;
+ g_assert(ssl_outlen <= sizeof(h->out));
+ memcpy(h->out, out, ssl_outlen);
+ break;
+ case LUA_CRYPTOBOX_HASH_HMAC:
+ HMAC_Final(h->content.hmac_c, out, &ssl_outlen);
+ h->out_len = ssl_outlen;
+ g_assert(ssl_outlen <= sizeof(h->out));
+ memcpy(h->out, out, ssl_outlen);
+ break;
+ case LUA_CRYPTOBOX_HASH_XXHASH64:
+ case LUA_CRYPTOBOX_HASH_XXHASH32:
+ case LUA_CRYPTOBOX_HASH_XXHASH3:
+ case LUA_CRYPTOBOX_HASH_MUM:
+ case LUA_CRYPTOBOX_HASH_T1HA:
+ ll = rspamd_cryptobox_fast_hash_final(h->content.fh);
+ memcpy(h->out, &ll, sizeof(ll));
+ break;
+ default:
+ g_assert_not_reached();
+ }
+
+ h->is_finished = TRUE;
+}
+
+/***
+ * @method cryptobox_hash:hex()
+ * Finalizes hash and return it as hex string
+ * @return {string} hex value of hash
+ */
+static gint
+lua_cryptobox_hash_hex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1);
+ guchar out_hex[rspamd_cryptobox_HASHBYTES * 2 + 1], *r;
+ guint dlen;
+
+ if (h) {
+ if (!h->is_finished) {
+ lua_cryptobox_hash_finish(h);
+ }
+
+ memset(out_hex, 0, sizeof(out_hex));
+ r = h->out;
+ dlen = h->out_len;
+
+ if (lua_isnumber(L, 2)) {
+ guint lim = lua_tonumber(L, 2);
+
+ if (lim < dlen) {
+ r += dlen - lim;
+ dlen = lim;
+ }
+ }
+
+ rspamd_encode_hex_buf(r, dlen, out_hex, sizeof(out_hex));
+ lua_pushstring(L, out_hex);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_hash:base32([b32type])
+ * Finalizes hash and return it as zbase32 (by default) string
+ * @param {string} b32type base32 type (default, bleach, rfc)
+ * @return {string} base32 value of hash
+ */
+static gint
+lua_cryptobox_hash_base32(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1);
+ guchar out_b32[rspamd_cryptobox_HASHBYTES * 2], *r;
+ guint dlen;
+
+ if (h) {
+ enum rspamd_base32_type btype = RSPAMD_BASE32_DEFAULT;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ btype = rspamd_base32_decode_type_from_str(lua_tostring(L, 2));
+
+ if (btype == RSPAMD_BASE32_INVALID) {
+ return luaL_error(L, "invalid b32 type: %s", lua_tostring(L, 2));
+ }
+ }
+
+ if (!h->is_finished) {
+ lua_cryptobox_hash_finish(h);
+ }
+
+ memset(out_b32, 0, sizeof(out_b32));
+ r = h->out;
+ dlen = h->out_len;
+
+ if (lua_isnumber(L, 2)) {
+ guint lim = lua_tonumber(L, 2);
+
+ if (lim < dlen) {
+ r += dlen - lim;
+ dlen = lim;
+ }
+ }
+
+ rspamd_encode_base32_buf(r, dlen, out_b32, sizeof(out_b32), btype);
+ lua_pushstring(L, out_b32);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_hash:base64()
+ * Finalizes hash and return it as base64 string
+ * @return {string} base64 value of hash
+ */
+static gint
+lua_cryptobox_hash_base64(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1);
+ guchar *b64, *r;
+ gsize len;
+ guint dlen;
+
+ if (h) {
+ if (!h->is_finished) {
+ lua_cryptobox_hash_finish(h);
+ }
+
+ r = h->out;
+ dlen = h->out_len;
+
+ if (lua_isnumber(L, 2)) {
+ guint lim = lua_tonumber(L, 2);
+
+ if (lim < dlen) {
+ r += dlen - lim;
+ dlen = lim;
+ }
+ }
+
+ b64 = rspamd_encode_base64(r, dlen, 0, &len);
+ lua_pushlstring(L, b64, len);
+ g_free(b64);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_hash:bin()
+ * Finalizes hash and return it as raw string
+ * @return {string} raw value of hash
+ */
+static gint
+lua_cryptobox_hash_bin(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1);
+ guchar *r;
+ guint dlen;
+
+ if (h) {
+ if (!h->is_finished) {
+ lua_cryptobox_hash_finish(h);
+ }
+
+ r = h->out;
+ dlen = h->out_len;
+
+ if (lua_isnumber(L, 2)) {
+ guint lim = lua_tonumber(L, 2);
+
+ if (lim < dlen) {
+ r += dlen - lim;
+ dlen = lim;
+ }
+ }
+
+ lua_pushlstring(L, r, dlen);
+ h->is_finished = TRUE;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_cryptobox_hash_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1);
+
+ REF_RELEASE(h);
+
+ return 0;
+}
+
+/***
+ * @function rspamd_cryptobox.verify_memory(pk, sig, data, [alg = 'curve25519'])
+ * Check memory using specified cryptobox key and signature
+ * @param {pubkey} pk public key to verify
+ * @param {sig} signature to check
+ * @param {string} data data to check signature against
+ * @return {boolean} `true` - if string matches cryptobox signature
+ */
+static gint
+lua_cryptobox_verify_memory(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_pubkey *pk;
+ rspamd_fstring_t *signature;
+ struct rspamd_lua_text *t;
+ const gchar *data;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ gsize len;
+ gint ret;
+
+ pk = lua_check_cryptobox_pubkey(L, 1);
+ signature = lua_check_cryptobox_sign(L, 2);
+
+ if (lua_isuserdata(L, 3)) {
+ t = lua_check_text(L, 3);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 3, &len);
+ }
+
+ if (lua_isstring(L, 4)) {
+ const gchar *str = lua_tostring(L, 4);
+
+ if (strcmp(str, "nist") == 0 || strcmp(str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp(str, "curve25519") == 0 || strcmp(str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ return luaL_error(L, "invalid algorithm: %s", str);
+ }
+ }
+
+ if (pk != NULL && signature != NULL && data != NULL) {
+ ret = rspamd_cryptobox_verify(signature->str, signature->len, data, len,
+ rspamd_pubkey_get_pk(pk, NULL), alg);
+
+ if (ret) {
+ lua_pushboolean(L, 1);
+ }
+ else {
+ lua_pushboolean(L, 0);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.verify_file(pk, sig, file, [alg = 'curve25519'])
+ * Check file using specified cryptobox key and signature
+ * @param {pubkey} pk public key to verify
+ * @param {sig} signature to check
+ * @param {string} file to load data from
+ * @return {boolean} `true` - if string matches cryptobox signature
+ */
+static gint
+lua_cryptobox_verify_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *fname;
+ struct rspamd_cryptobox_pubkey *pk;
+ rspamd_fstring_t *signature;
+ guchar *map = NULL;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ gsize len;
+ gint ret;
+
+ pk = lua_check_cryptobox_pubkey(L, 1);
+ signature = lua_check_cryptobox_sign(L, 2);
+ fname = luaL_checkstring(L, 3);
+
+ if (lua_isstring(L, 4)) {
+ const gchar *str = lua_tostring(L, 4);
+
+ if (strcmp(str, "nist") == 0 || strcmp(str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp(str, "curve25519") == 0 || strcmp(str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ return luaL_error(L, "invalid algorithm: %s", str);
+ }
+ }
+
+ map = rspamd_file_xmap(fname, PROT_READ, &len, TRUE);
+
+ if (map != NULL && pk != NULL && signature != NULL) {
+ ret = rspamd_cryptobox_verify(signature->str, signature->len,
+ map, len,
+ rspamd_pubkey_get_pk(pk, NULL), alg);
+
+ if (ret) {
+ lua_pushboolean(L, 1);
+ }
+ else {
+ lua_pushboolean(L, 0);
+ }
+ }
+ else {
+ if (map != NULL) {
+ munmap(map, len);
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (map != NULL) {
+ munmap(map, len);
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.sign_memory(kp, data)
+ * Sign data using specified keypair
+ * @param {keypair} kp keypair to sign
+ * @param {string} data
+ * @return {cryptobox_signature} signature object
+ */
+static gint
+lua_cryptobox_sign_memory(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *data;
+ struct rspamd_lua_text *t;
+ gsize len = 0;
+ rspamd_fstring_t *sig, **psig;
+
+ kp = lua_check_cryptobox_keypair(L, 1);
+
+ if (lua_isuserdata(L, 2)) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 2, &len);
+ }
+
+
+ if (!kp || !data || kp->type == RSPAMD_KEYPAIR_KEX) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sig = rspamd_fstring_sized_new(rspamd_cryptobox_signature_bytes(
+ rspamd_keypair_alg(kp)));
+
+ unsigned long long siglen = sig->len;
+ rspamd_cryptobox_sign(sig->str, &siglen, data,
+ len, rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_SK, NULL), rspamd_keypair_alg(kp));
+
+ sig->len = siglen;
+ psig = lua_newuserdata(L, sizeof(void *));
+ *psig = sig;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_signature}", -1);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.sign_file(kp, file)
+ * Sign file using specified keypair
+ * @param {keypair} kp keypair to sign
+ * @param {string} filename
+ * @return {cryptobox_signature} signature object
+ */
+static gint
+lua_cryptobox_sign_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *filename;
+ gchar *data;
+ gsize len = 0;
+ rspamd_fstring_t *sig, **psig;
+
+ kp = lua_check_cryptobox_keypair(L, 1);
+ filename = luaL_checkstring(L, 2);
+
+ if (!kp || !filename) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = rspamd_file_xmap(filename, PROT_READ, &len, TRUE);
+
+ if (data == NULL) {
+ msg_err("cannot mmap file %s: %s", filename, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ sig = rspamd_fstring_sized_new(rspamd_cryptobox_signature_bytes(
+ rspamd_keypair_alg(kp)));
+
+ unsigned long long siglen = sig->len;
+
+ rspamd_cryptobox_sign(sig->str, &siglen, data,
+ len, rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_SK, NULL), rspamd_keypair_alg(kp));
+
+ sig->len = siglen;
+ psig = lua_newuserdata(L, sizeof(void *));
+ *psig = sig;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_signature}", -1);
+ munmap(data, len);
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.encrypt_memory(kp, data[, nist=false])
+ * Encrypt data using specified keypair/pubkey
+ * @param {keypair|string} kp keypair or pubkey in base32 to use
+ * @param {string|text} data
+ * @return {rspamd_text} encrypted text
+ */
+static gint
+lua_cryptobox_encrypt_memory(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = NULL;
+ struct rspamd_cryptobox_pubkey *pk = NULL;
+ const gchar *data;
+ guchar *out = NULL;
+ struct rspamd_lua_text *t, *res;
+ gsize len = 0, outlen = 0;
+ GError *err = NULL;
+ bool owned_pk = false;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ if (rspamd_lua_check_udata_maybe(L, 1, "rspamd{cryptobox_keypair}")) {
+ kp = lua_check_cryptobox_keypair(L, 1);
+ }
+ else if (rspamd_lua_check_udata_maybe(L, 1, "rspamd{cryptobox_pubkey}")) {
+ pk = lua_check_cryptobox_pubkey(L, 1);
+ }
+ }
+ else if (lua_type(L, 1) == LUA_TSTRING) {
+ const gchar *b32;
+ gsize blen;
+
+ b32 = lua_tolstring(L, 1, &blen);
+ pk = rspamd_pubkey_from_base32(b32, blen, RSPAMD_KEYPAIR_KEX,
+ lua_toboolean(L, 3) ? RSPAMD_CRYPTOBOX_MODE_NIST : RSPAMD_CRYPTOBOX_MODE_25519);
+ owned_pk = true;
+ }
+
+ if (lua_isuserdata(L, 2)) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ goto err;
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 2, &len);
+ }
+
+
+ if (!(kp || pk) || !data) {
+ goto err;
+ }
+
+ if (kp) {
+ if (!rspamd_keypair_encrypt(kp, data, len, &out, &outlen, &err)) {
+ gint ret = luaL_error(L, "cannot encrypt data: %s", err->message);
+ g_error_free(err);
+
+ if (owned_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return ret;
+ }
+ }
+ else {
+ if (!rspamd_pubkey_encrypt(pk, data, len, &out, &outlen, &err)) {
+ gint ret = luaL_error(L, "cannot encrypt data: %s", err->message);
+ g_error_free(err);
+
+ if (owned_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return ret;
+ }
+ }
+
+ res = lua_newuserdata(L, sizeof(*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ if (owned_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return 1;
+err:
+
+ if (owned_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return luaL_error(L, "invalid arguments");
+}
+
+/***
+ * @function rspamd_cryptobox.encrypt_file(kp|pk_string, filename[, nist=false])
+ * Encrypt data using specified keypair/pubkey
+ * @param {keypair|string} kp keypair or pubkey in base32 to use
+ * @param {string} filename
+ * @return {rspamd_text} encrypted text
+ */
+static gint
+lua_cryptobox_encrypt_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = NULL;
+ struct rspamd_cryptobox_pubkey *pk = NULL;
+ const gchar *filename;
+ gchar *data = NULL;
+ guchar *out = NULL;
+ struct rspamd_lua_text *res;
+ gsize len = 0, outlen = 0;
+ GError *err = NULL;
+ bool own_pk = false;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ if (rspamd_lua_check_udata_maybe(L, 1, "rspamd{cryptobox_keypair}")) {
+ kp = lua_check_cryptobox_keypair(L, 1);
+ }
+ else if (rspamd_lua_check_udata_maybe(L, 1, "rspamd{cryptobox_pubkey}")) {
+ pk = lua_check_cryptobox_pubkey(L, 1);
+ }
+ }
+ else if (lua_type(L, 1) == LUA_TSTRING) {
+ const gchar *b32;
+ gsize blen;
+
+ b32 = lua_tolstring(L, 1, &blen);
+ pk = rspamd_pubkey_from_base32(b32, blen, RSPAMD_KEYPAIR_KEX,
+ lua_toboolean(L, 3) ? RSPAMD_CRYPTOBOX_MODE_NIST : RSPAMD_CRYPTOBOX_MODE_25519);
+ own_pk = true;
+ }
+
+ filename = luaL_checkstring(L, 2);
+ data = rspamd_file_xmap(filename, PROT_READ, &len, TRUE);
+
+ if (!(kp || pk) || !data) {
+ goto err;
+ }
+
+ if (kp) {
+ if (!rspamd_keypair_encrypt(kp, data, len, &out, &outlen, &err)) {
+ gint ret = luaL_error(L, "cannot encrypt file %s: %s", filename,
+ err->message);
+ g_error_free(err);
+ munmap(data, len);
+ if (own_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return ret;
+ }
+ }
+ else if (pk) {
+ if (!rspamd_pubkey_encrypt(pk, data, len, &out, &outlen, &err)) {
+ gint ret = luaL_error(L, "cannot encrypt file %s: %s", filename,
+ err->message);
+ g_error_free(err);
+ munmap(data, len);
+
+ if (own_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return ret;
+ }
+ }
+
+ res = lua_newuserdata(L, sizeof(*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ munmap(data, len);
+ if (own_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return 1;
+
+err:
+ if (data) {
+ munmap(data, len);
+ }
+ if (own_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+ return luaL_error(L, "invalid arguments");
+}
+
+/***
+ * @function rspamd_cryptobox.decrypt_memory(kp, data[, nist = false])
+ * Encrypt data using specified keypair
+ * @param {keypair} kp keypair to use
+ * @param {string} data
+ * @return status,{rspamd_text}|error status is boolean variable followed by either unencrypted data or an error message
+ */
+static gint
+lua_cryptobox_decrypt_memory(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *data;
+ guchar *out;
+ struct rspamd_lua_text *t, *res;
+ gsize len = 0, outlen;
+ GError *err = NULL;
+
+ kp = lua_check_cryptobox_keypair(L, 1);
+
+ if (lua_isuserdata(L, 2)) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 2, &len);
+ }
+
+
+ if (!kp || !data) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (!rspamd_keypair_decrypt(kp, data, len, &out, &outlen, &err)) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, err->message);
+ g_error_free(err);
+ }
+ else {
+ lua_pushboolean(L, true);
+ res = lua_newuserdata(L, sizeof(*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ }
+
+ return 2;
+}
+
+/***
+ * @function rspamd_cryptobox.decrypt_file(kp, filename)
+ * Encrypt data using specified keypair
+ * @param {keypair} kp keypair to use
+ * @param {string} filename
+ * @return status,{rspamd_text}|error status is boolean variable followed by either unencrypted data or an error message
+ */
+static gint
+lua_cryptobox_decrypt_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *filename;
+ gchar *data;
+ guchar *out;
+ struct rspamd_lua_text *res;
+ gsize len = 0, outlen;
+ GError *err = NULL;
+
+ kp = lua_check_cryptobox_keypair(L, 1);
+ if (!kp) {
+ return luaL_error(L, "invalid arguments; keypair is expected");
+ }
+
+ filename = luaL_checkstring(L, 2);
+ data = rspamd_file_xmap(filename, PROT_READ, &len, TRUE);
+ if (!data) {
+ return luaL_error(L, "invalid arguments; cannot mmap %s: %s",
+ filename, strerror(errno));
+ }
+
+ if (!rspamd_keypair_decrypt(kp, data, len, &out, &outlen, &err)) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, err->message);
+ g_error_free(err);
+ }
+ else {
+ lua_pushboolean(L, true);
+ res = lua_newuserdata(L, sizeof(*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ }
+
+ munmap(data, len);
+
+ return 2;
+}
+
+#define RSPAMD_CRYPTOBOX_AES_BLOCKSIZE 16
+#define RSPAMD_CRYPTOBOX_AES_KEYSIZE 16
+
+/***
+ * @function rspamd_cryptobox.encrypt_cookie(secret_key, secret_cookie)
+ * Specialised function that performs AES-CTR encryption of the provided cookie
+ * ```
+ * e := base64(nonce||aesencrypt(nonce, secret_cookie))
+ * nonce := uint32_le(unix_timestamp)||random_64bit
+ * aesencrypt := aes_ctr(nonce, secret_key) ^ pad(secret_cookie)
+ * pad := secret_cookie || 0^(32-len(secret_cookie))
+ * ```
+ * @param {string} secret_key secret key as a hex string (must be 16 bytes in raw or 32 in hex)
+ * @param {string} secret_cookie secret cookie as a string for up to 31 character
+ * @return {string} e function value for this sk and cookie
+ */
+static gint
+lua_cryptobox_encrypt_cookie(lua_State *L)
+{
+ guchar aes_block[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE], *blk;
+ guchar padded_cookie[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE];
+ guchar nonce[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE];
+ guchar aes_key[RSPAMD_CRYPTOBOX_AES_KEYSIZE];
+ guchar result[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2];
+ guint32 ts;
+
+ const gchar *sk, *cookie;
+ gsize sklen, cookie_len;
+ gint bklen;
+
+ sk = lua_tolstring(L, 1, &sklen);
+ cookie = lua_tolstring(L, 2, &cookie_len);
+
+ if (sk && cookie) {
+ if (sklen == 32) {
+ /* Hex */
+ rspamd_decode_hex_buf(sk, sklen, aes_key, sizeof(aes_key));
+ }
+ else if (sklen == RSPAMD_CRYPTOBOX_AES_KEYSIZE) {
+ /* Raw */
+ memcpy(aes_key, sk, sizeof(aes_key));
+ }
+ else {
+ return luaL_error(L, "invalid keysize %d", (gint) sklen);
+ }
+
+ if (cookie_len > sizeof(padded_cookie) - 1) {
+ return luaL_error(L, "cookie is too long %d", (gint) cookie_len);
+ }
+
+ /* Fill nonce */
+ ottery_rand_bytes(nonce, sizeof(guint64) + sizeof(guint32));
+ ts = (guint32) rspamd_get_calendar_ticks();
+ ts = GUINT32_TO_LE(ts);
+ memcpy(nonce + sizeof(guint64) + sizeof(guint32), &ts, sizeof(ts));
+
+ /* Prepare padded cookie */
+ memset(padded_cookie, 0, sizeof(padded_cookie));
+ memcpy(padded_cookie, cookie, cookie_len);
+
+ /* Perform AES CTR via AES ECB on nonce */
+ EVP_CIPHER_CTX *ctx;
+ ctx = EVP_CIPHER_CTX_new();
+ EVP_EncryptInit_ex(ctx, EVP_aes_128_ecb(), NULL, aes_key, NULL);
+ EVP_CIPHER_CTX_set_padding(ctx, 0);
+
+ bklen = sizeof(aes_block);
+ blk = aes_block;
+ g_assert(EVP_EncryptUpdate(ctx, blk, &bklen, nonce, sizeof(nonce)));
+ blk += bklen;
+ g_assert(EVP_EncryptFinal_ex(ctx, blk, &bklen));
+ EVP_CIPHER_CTX_free(ctx);
+
+ /* Encode result */
+ memcpy(result, nonce, sizeof(nonce));
+ for (guint i = 0; i < sizeof(aes_block); i++) {
+ result[i + sizeof(nonce)] = padded_cookie[i] ^ aes_block[i];
+ }
+
+ gsize rlen;
+ gchar *res = rspamd_encode_base64(result, sizeof(result),
+ 0, &rlen);
+
+ lua_pushlstring(L, res, rlen);
+ g_free(res);
+ rspamd_explicit_memzero(aes_key, sizeof(aes_key));
+ rspamd_explicit_memzero(aes_block, sizeof(aes_block));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.decrypt_cookie(secret_key, encrypted_cookie)
+ * Specialised function that performs AES-CTR decryption of the provided cookie in form
+ * ```
+ * e := base64(nonce||aesencrypt(nonce, secret_cookie))
+ * nonce := int32_le(unix_timestamp)||random_96bit
+ * aesencrypt := aes_ctr(nonce, secret_key) ^ pad(secret_cookie)
+ * pad := secret_cookie || 0^(32-len(secret_cookie))
+ * ```
+ * @param {string} secret_key secret key as a hex string (must be 16 bytes in raw or 32 in hex)
+ * @param {string} encrypted_cookie encrypted cookie as a base64 encoded string
+ * @return {string+number} decrypted value of the cookie and the cookie timestamp
+ */
+static gint
+lua_cryptobox_decrypt_cookie(lua_State *L)
+{
+ guchar *blk;
+ guchar nonce[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE];
+ guchar aes_key[RSPAMD_CRYPTOBOX_AES_KEYSIZE];
+ guchar *src;
+ guint32 ts;
+
+ const gchar *sk, *cookie;
+ gsize sklen, cookie_len;
+ gint bklen;
+
+ sk = lua_tolstring(L, 1, &sklen);
+ cookie = lua_tolstring(L, 2, &cookie_len);
+
+ if (sk && cookie) {
+ if (sklen == 32) {
+ /* Hex */
+ rspamd_decode_hex_buf(sk, sklen, aes_key, sizeof(aes_key));
+ }
+ else if (sklen == RSPAMD_CRYPTOBOX_AES_KEYSIZE) {
+ /* Raw */
+ memcpy(aes_key, sk, sizeof(aes_key));
+ }
+ else {
+ return luaL_error(L, "invalid keysize %d", (gint) sklen);
+ }
+
+ src = g_malloc(cookie_len);
+
+ rspamd_cryptobox_base64_decode(cookie, cookie_len, src, &cookie_len);
+
+ if (cookie_len != RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2) {
+ g_free(src);
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ /* Perform AES CTR via AES ECB on nonce */
+ EVP_CIPHER_CTX *ctx;
+ ctx = EVP_CIPHER_CTX_new();
+ /* As per CTR definition, we use encrypt for both encrypt and decrypt */
+ EVP_EncryptInit_ex(ctx, EVP_aes_128_ecb(), NULL, aes_key, NULL);
+ EVP_CIPHER_CTX_set_padding(ctx, 0);
+
+ /* Copy time */
+ memcpy(&ts, src + sizeof(guint64) + sizeof(guint32), sizeof(ts));
+ ts = GUINT32_FROM_LE(ts);
+ bklen = sizeof(nonce);
+ blk = nonce;
+ g_assert(EVP_EncryptUpdate(ctx, blk, &bklen, src,
+ RSPAMD_CRYPTOBOX_AES_BLOCKSIZE));
+ blk += bklen;
+ g_assert(EVP_EncryptFinal_ex(ctx, blk, &bklen));
+ EVP_CIPHER_CTX_free(ctx);
+
+ /* Decode result */
+ for (guint i = 0; i < RSPAMD_CRYPTOBOX_AES_BLOCKSIZE; i++) {
+ src[i + sizeof(nonce)] ^= nonce[i];
+ }
+
+ if (src[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2 - 1] != '\0') {
+ /* Bad cookie */
+ lua_pushnil(L);
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushstring(L, src + sizeof(nonce));
+ lua_pushnumber(L, ts);
+ }
+
+ rspamd_explicit_memzero(src, RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2);
+ g_free(src);
+ rspamd_explicit_memzero(aes_key, sizeof(aes_key));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+/***
+ * @function rspamd_cryptobox.pbkdf([password, [kdf_alg]])
+ * Function that encrypts password using PBKDF function.
+ * This function either reads password from STDIN or accepts prepared password as
+ * an argument
+ * @param {string} password optional password string
+ * @param {string} kdf_alg algorithm to use (catena or pbkdf2)
+ * @return {string} encrypted password or nil if error occurs
+ */
+static gint
+lua_cryptobox_pbkdf(lua_State *L)
+{
+ const struct rspamd_controller_pbkdf *pbkdf = NULL;
+ const gchar *pbkdf_str = "catena";
+ gchar *password;
+ gsize pwlen;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ pbkdf_str = lua_tostring(L, 2);
+ }
+
+ for (guint i = 0; i < RSPAMD_PBKDF_ID_MAX - 1; i++) {
+ pbkdf = &pbkdf_list[i];
+
+ if (g_ascii_strcasecmp(pbkdf_str, pbkdf->alias) == 0) {
+ break;
+ }
+ if (g_ascii_strcasecmp(pbkdf_str, pbkdf->name) == 0) {
+ break;
+ }
+
+ pbkdf = NULL;
+ }
+
+ if (pbkdf == NULL) {
+ return luaL_error(L, "invalid pbkdf algorithm: %s", pbkdf_str);
+ }
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ password = g_strdup(lua_tolstring(L, 1, &pwlen));
+ }
+ else {
+ pwlen = 8192;
+ password = g_malloc0(pwlen);
+ pwlen = rspamd_read_passphrase(password, pwlen, 0, NULL);
+ }
+
+ if (pwlen == 0) {
+ lua_pushnil(L);
+ g_free(password);
+
+ return 1;
+ }
+
+ guchar *salt, *key;
+ gchar *encoded_salt, *encoded_key;
+ GString *result;
+
+ salt = g_alloca(pbkdf->salt_len);
+ key = g_alloca(pbkdf->key_len);
+ ottery_rand_bytes(salt, pbkdf->salt_len);
+ /* Derive key */
+ rspamd_cryptobox_pbkdf(password, pwlen,
+ salt, pbkdf->salt_len, key, pbkdf->key_len, pbkdf->complexity,
+ pbkdf->type);
+
+ encoded_salt = rspamd_encode_base32(salt, pbkdf->salt_len, RSPAMD_BASE32_DEFAULT);
+ encoded_key = rspamd_encode_base32(key, pbkdf->key_len, RSPAMD_BASE32_DEFAULT);
+
+ result = g_string_new("");
+ rspamd_printf_gstring(result, "$%d$%s$%s", pbkdf->id, encoded_salt,
+ encoded_key);
+
+ g_free(encoded_salt);
+ g_free(encoded_key);
+ rspamd_explicit_memzero(password, pwlen);
+ g_free(password);
+ lua_pushlstring(L, result->str, result->len);
+ g_string_free(result, TRUE);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.gen_dkim_keypair([alg, [nbits]])
+ * Generates DKIM keypair. Returns 2 base64 strings as rspamd_text: privkey and pubkey
+ * @param {string} alg optional algorithm (rsa default, can be ed25519)
+ * @param {number} nbits optional number of bits for rsa (default 1024)
+ * @return {rspamd_text,rspamd_text} private key and public key as base64 encoded strings
+ */
+static gint
+lua_cryptobox_gen_dkim_keypair(lua_State *L)
+{
+ const gchar *alg_str = "rsa";
+ guint nbits = 1024;
+ struct rspamd_lua_text *priv_out, *pub_out;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ alg_str = lua_tostring(L, 1);
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ nbits = lua_tointeger(L, 2);
+ }
+
+ if (strcmp(alg_str, "rsa") == 0) {
+ BIGNUM *e;
+ RSA *r;
+ EVP_PKEY *pk;
+
+ e = BN_new();
+ r = RSA_new();
+ pk = EVP_PKEY_new();
+
+ if (BN_set_word(e, RSA_F4) != 1) {
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+
+ return luaL_error(L, "BN_set_word failed");
+ }
+
+ if (RSA_generate_key_ex(r, nbits, e, NULL) != 1) {
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+
+ return luaL_error(L, "RSA_generate_key_ex failed");
+ }
+
+ if (EVP_PKEY_set1_RSA(pk, r) != 1) {
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+
+ return luaL_error(L, "EVP_PKEY_set1_RSA failed");
+ }
+
+ BIO *mbio;
+ gint rc, len;
+ guchar *data;
+ gchar *b64_data;
+ gsize b64_len;
+
+ mbio = BIO_new(BIO_s_mem());
+
+ /* Process private key */
+ rc = i2d_RSAPrivateKey_bio(mbio, r);
+
+ if (rc == 0) {
+ BIO_free(mbio);
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+
+ return luaL_error(L, "i2d_RSAPrivateKey_bio failed");
+ }
+
+ len = BIO_get_mem_data(mbio, &data);
+
+ b64_data = rspamd_encode_base64(data, len, -1, &b64_len);
+
+ priv_out = lua_newuserdata(L, sizeof(*priv_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ priv_out->start = b64_data;
+ priv_out->len = b64_len;
+ priv_out->flags = RSPAMD_TEXT_FLAG_OWN | RSPAMD_TEXT_FLAG_WIPE;
+
+ /* Process public key */
+ BIO_reset(mbio);
+ rc = i2d_RSA_PUBKEY_bio(mbio, r);
+
+ if (rc == 0) {
+ BIO_free(mbio);
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+
+ return luaL_error(L, "i2d_RSA_PUBKEY_bio failed");
+ }
+
+ len = BIO_get_mem_data(mbio, &data);
+
+ b64_data = rspamd_encode_base64(data, len, -1, &b64_len);
+
+ pub_out = lua_newuserdata(L, sizeof(*pub_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ pub_out->start = b64_data;
+ pub_out->len = b64_len;
+ pub_out->flags = RSPAMD_TEXT_FLAG_OWN;
+
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+ BIO_free(mbio);
+ }
+ else if (strcmp(alg_str, "ed25519") == 0) {
+ rspamd_sig_pk_t pk;
+ rspamd_sig_sk_t sk;
+ gchar *b64_data;
+ gsize b64_len;
+
+ rspamd_cryptobox_keypair_sig(pk, sk, RSPAMD_CRYPTOBOX_MODE_25519);
+
+ /* Process private key */
+ b64_data = rspamd_encode_base64(sk,
+ rspamd_cryptobox_sk_sig_bytes(RSPAMD_CRYPTOBOX_MODE_25519),
+ -1, &b64_len);
+
+ priv_out = lua_newuserdata(L, sizeof(*priv_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ priv_out->start = b64_data;
+ priv_out->len = b64_len;
+ priv_out->flags = RSPAMD_TEXT_FLAG_OWN | RSPAMD_TEXT_FLAG_WIPE;
+
+ /* Process public key */
+ b64_data = rspamd_encode_base64(pk,
+ rspamd_cryptobox_pk_sig_bytes(RSPAMD_CRYPTOBOX_MODE_25519),
+ -1, &b64_len);
+
+ pub_out = lua_newuserdata(L, sizeof(*pub_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ pub_out->start = b64_data;
+ pub_out->len = b64_len;
+ pub_out->flags = RSPAMD_TEXT_FLAG_OWN;
+
+ rspamd_explicit_memzero(pk, sizeof(pk));
+ rspamd_explicit_memzero(sk, sizeof(sk));
+ }
+ else if (strcmp(alg_str, "ed25519-seed") == 0) {
+ rspamd_sig_pk_t pk;
+ rspamd_sig_sk_t sk;
+ gchar *b64_data;
+ gsize b64_len;
+
+ rspamd_cryptobox_keypair_sig(pk, sk, RSPAMD_CRYPTOBOX_MODE_25519);
+
+ /* Process private key */
+ b64_data = rspamd_encode_base64(sk,
+ 32,
+ -1, &b64_len);
+
+ priv_out = lua_newuserdata(L, sizeof(*priv_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ priv_out->start = b64_data;
+ priv_out->len = b64_len;
+ priv_out->flags = RSPAMD_TEXT_FLAG_OWN | RSPAMD_TEXT_FLAG_WIPE;
+
+ /* Process public key */
+ b64_data = rspamd_encode_base64(pk,
+ rspamd_cryptobox_pk_sig_bytes(RSPAMD_CRYPTOBOX_MODE_25519),
+ -1, &b64_len);
+
+ pub_out = lua_newuserdata(L, sizeof(*pub_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ pub_out->start = b64_data;
+ pub_out->len = b64_len;
+ pub_out->flags = RSPAMD_TEXT_FLAG_OWN;
+
+ rspamd_explicit_memzero(pk, sizeof(pk));
+ rspamd_explicit_memzero(sk, sizeof(sk));
+ }
+ else {
+ return luaL_error(L, "invalid algorithm %s", alg_str);
+ }
+
+ return 2;
+}
+
+/*
+ * Secretbox API
+ */
+/* Ensure that KDF output is suitable for crypto_secretbox_KEYBYTES */
+#ifdef crypto_generichash_BYTES_MIN
+G_STATIC_ASSERT(crypto_secretbox_KEYBYTES >= crypto_generichash_BYTES_MIN);
+#endif
+
+/***
+ * @function rspamd_cryptobox_secretbox.create(secret_string, [params])
+ * Generates a secretbox state by expanding secret string
+ * @param {string/text} secret_string secret string (should have high enough entropy)
+ * @param {table} params optional parameters - NYI
+ * @return {rspamd_cryptobox_secretbox} opaque object with the key expanded
+ */
+static gint
+lua_cryptobox_secretbox_create(lua_State *L)
+{
+ const gchar *in;
+ gsize inlen;
+
+
+ if (lua_isstring(L, 1)) {
+ in = lua_tolstring(L, 1, &inlen);
+ }
+ else if (lua_isuserdata(L, 1)) {
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments; userdata is not text");
+ }
+
+ in = t->start;
+ inlen = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments; userdata or string are expected");
+ }
+
+ if (in == NULL || inlen == 0) {
+ return luaL_error(L, "invalid arguments; non empty secret expected");
+ }
+
+ struct rspamd_lua_cryptobox_secretbox *sbox, **psbox;
+
+ sbox = g_malloc0(sizeof(*sbox));
+ crypto_generichash(sbox->sk, sizeof(sbox->sk), in, inlen, NULL, 0);
+ psbox = lua_newuserdata(L, sizeof(*psbox));
+ *psbox = sbox;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_secretbox}", -1);
+
+ return 1;
+}
+
+
+static gint
+lua_cryptobox_secretbox_gc(lua_State *L)
+{
+ struct rspamd_lua_cryptobox_secretbox *sbox =
+ lua_check_cryptobox_secretbox(L, 1);
+
+ if (sbox != NULL) {
+ sodium_memzero(sbox->sk, sizeof(sbox->sk));
+ g_free(sbox);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+/***
+ * @method rspamd_cryptobox_secretbox:encrypt(input, [nonce])
+ * Encrypts data using secretbox. MAC is prepended to the message
+ * @param {string/text} input input to encrypt
+ * @param {string/text} nonce optional nonce (must be 1 - 192 bits length)
+ * @param {table} params optional parameters - NYI
+ * @return {rspamd_text},{rspamd_text} output with mac + nonce or just output if nonce is there
+ */
+static gint
+lua_cryptobox_secretbox_encrypt(lua_State *L)
+{
+ const gchar *in, *nonce;
+ gsize inlen, nlen;
+ struct rspamd_lua_cryptobox_secretbox *sbox =
+ lua_check_cryptobox_secretbox(L, 1);
+ struct rspamd_lua_text *out;
+
+ if (sbox == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_isstring(L, 2)) {
+ in = lua_tolstring(L, 2, &inlen);
+ }
+ else if (lua_isuserdata(L, 2)) {
+ struct rspamd_lua_text *t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments; userdata is not text");
+ }
+
+ in = t->start;
+ inlen = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments; userdata or string are expected");
+ }
+
+ /* Nonce part */
+ if (!lua_isnoneornil(L, 3)) {
+ if (lua_isstring(L, 3)) {
+ nonce = lua_tolstring(L, 3, &nlen);
+ }
+ else if (lua_isuserdata(L, 3)) {
+ struct rspamd_lua_text *t = lua_check_text(L, 3);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments; userdata is not text");
+ }
+
+ nonce = t->start;
+ nlen = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments; userdata or string are expected");
+ }
+
+ if (nlen < 1 || nlen > crypto_secretbox_NONCEBYTES) {
+ return luaL_error(L, "bad nonce");
+ }
+
+ guchar real_nonce[crypto_secretbox_NONCEBYTES];
+
+ memset(real_nonce, 0, sizeof(real_nonce));
+ memcpy(real_nonce, nonce, nlen);
+
+ out = lua_new_text(L, NULL, inlen + crypto_secretbox_MACBYTES,
+ TRUE);
+ crypto_secretbox_easy((guchar *) out->start, in, inlen,
+ nonce, sbox->sk);
+
+ return 1;
+ }
+ else {
+ /* Random nonce */
+ struct rspamd_lua_text *random_nonce;
+
+ out = lua_new_text(L, NULL, inlen + crypto_secretbox_MACBYTES,
+ TRUE);
+ random_nonce = lua_new_text(L, NULL, crypto_secretbox_NONCEBYTES, TRUE);
+
+ randombytes_buf((guchar *) random_nonce->start, random_nonce->len);
+ crypto_secretbox_easy((guchar *) out->start, in, inlen,
+ random_nonce->start, sbox->sk);
+
+ return 2; /* output + random nonce */
+ }
+}
+
+/***
+ * @method rspamd_cryptobox_secretbox:decrypt(input, nonce)
+ * Decrypts data using secretbox
+ * @param {string/text} nonce nonce used to encrypt
+ * @param {string/text} input input to decrypt
+ * @param {table} params optional parameters - NYI
+ * @return {boolean},{rspamd_text} decryption result + decrypted text
+ */
+static gint
+lua_cryptobox_secretbox_decrypt(lua_State *L)
+{
+ const gchar *in, *nonce;
+ gsize inlen, nlen;
+ struct rspamd_lua_cryptobox_secretbox *sbox =
+ lua_check_cryptobox_secretbox(L, 1);
+ struct rspamd_lua_text *out;
+
+ if (sbox == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ /* Input argument */
+ if (lua_isstring(L, 2)) {
+ in = lua_tolstring(L, 2, &inlen);
+ }
+ else if (lua_isuserdata(L, 2)) {
+ struct rspamd_lua_text *t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments; userdata is not text");
+ }
+
+ in = t->start;
+ inlen = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments; userdata or string are expected");
+ }
+
+ /* Nonce argument */
+ if (lua_isstring(L, 3)) {
+ nonce = lua_tolstring(L, 3, &nlen);
+ }
+ else if (lua_isuserdata(L, 3)) {
+ struct rspamd_lua_text *t = lua_check_text(L, 3);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments; userdata is not text");
+ }
+
+ nonce = t->start;
+ nlen = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments; userdata or string are expected");
+ }
+
+
+ if (nlen < 1 || nlen > crypto_secretbox_NONCEBYTES) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, "invalid nonce");
+ return 2;
+ }
+
+ if (inlen < crypto_secretbox_MACBYTES) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, "too short");
+ return 2;
+ }
+
+ guchar real_nonce[crypto_secretbox_NONCEBYTES];
+
+ memset(real_nonce, 0, sizeof(real_nonce));
+ memcpy(real_nonce, nonce, nlen);
+
+ out = lua_new_text(L, NULL, inlen - crypto_secretbox_MACBYTES,
+ TRUE);
+ gint text_pos = lua_gettop(L);
+
+ if (crypto_secretbox_open_easy((guchar *) out->start, in, inlen,
+ nonce, sbox->sk) == 0) {
+ lua_pushboolean(L, true);
+ lua_pushvalue(L, text_pos); /* Prevent gc by copying in stack */
+ }
+ else {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, "authentication error");
+ }
+
+ /* This causes gc method if decryption has failed */
+ lua_remove(L, text_pos);
+
+ return 2;
+}
+
+static gint
+lua_load_pubkey(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxpubkeylib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_keypair(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxkeypairlib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_signature(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxsignlib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_hash(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxhashlib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_cryptobox_secretbox(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxsecretboxlib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_cryptobox(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxlib_f);
+
+ return 1;
+}
+
+void luaopen_cryptobox(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{cryptobox_pubkey}", cryptoboxpubkeylib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cryptobox_pubkey", lua_load_pubkey);
+
+ rspamd_lua_new_class(L, "rspamd{cryptobox_keypair}", cryptoboxkeypairlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cryptobox_keypair", lua_load_keypair);
+
+ rspamd_lua_new_class(L, "rspamd{cryptobox_signature}", cryptoboxsignlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cryptobox_signature", lua_load_signature);
+
+ rspamd_lua_new_class(L, "rspamd{cryptobox_hash}", cryptoboxhashlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cryptobox_hash", lua_load_hash);
+
+ rspamd_lua_new_class(L, "rspamd{cryptobox_secretbox}",
+ cryptoboxsecretboxlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cryptobox_secretbox",
+ lua_load_cryptobox_secretbox);
+
+ rspamd_lua_add_preload(L, "rspamd_cryptobox", lua_load_cryptobox);
+
+ lua_settop(L, 0);
+}
diff --git a/src/lua/lua_dns.c b/src/lua/lua_dns.c
new file mode 100644
index 0000000..cffa312
--- /dev/null
+++ b/src/lua/lua_dns.c
@@ -0,0 +1,198 @@
+/*-
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_dns_resolver.h"
+#include "lua_thread_pool.h"
+
+LUA_FUNCTION_DEF(dns, request);
+
+static const struct luaL_reg dns_f[] = {
+ LUA_INTERFACE_DEF(dns, request),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static const gchar *M = "rspamd lua dns";
+
+void lua_dns_callback(struct rdns_reply *reply, void *arg);
+
+struct lua_rspamd_dns_cbdata {
+ struct thread_entry *thread;
+ struct rspamd_task *task;
+ struct rspamd_dns_resolver *resolver;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_async_session *s;
+};
+
+static gint
+lua_dns_request(lua_State *L)
+{
+ GError *err = NULL;
+ struct rspamd_async_session *session = NULL;
+ struct rspamd_config *cfg = NULL;
+ struct lua_rspamd_dns_cbdata *cbdata = NULL;
+ const gchar *to_resolve = NULL;
+ const gchar *type_str = NULL;
+ struct rspamd_task *task = NULL;
+ rspamd_mempool_t *pool = NULL;
+ gint ret = 0;
+ gboolean forced = FALSE;
+
+ /* Check arguments */
+ if (!rspamd_lua_parse_table_arguments(L, 1, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*name=S;task=U{task};*type=S;forced=B;session=U{session};config=U{config}",
+ &to_resolve,
+ &task,
+ &type_str,
+ &forced,
+ &session,
+ &cfg)) {
+
+ if (err) {
+ ret = luaL_error(L, "invalid arguments: %s", err->message);
+ g_error_free(err);
+
+ return ret;
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (task) {
+ session = task->s;
+ pool = task->task_pool;
+ cfg = task->cfg;
+ }
+ else if (session && cfg) {
+ pool = cfg->cfg_pool;
+ }
+ else {
+ return luaL_error(L, "invalid arguments: either task or session/config should be set");
+ }
+
+ enum rdns_request_type type = rdns_type_fromstr(type_str);
+
+ if (type == RDNS_REQUEST_INVALID) {
+ return luaL_error(L, "invalid arguments: this record type is not supported");
+ }
+
+ cbdata = rspamd_mempool_alloc0(pool, sizeof(*cbdata));
+
+ cbdata->task = task;
+
+ if (type == RDNS_REQUEST_PTR) {
+ char *ptr_str;
+
+ ptr_str = rdns_generate_ptr_from_str(to_resolve);
+
+ if (ptr_str == NULL) {
+ msg_err_task_check("wrong resolve string to PTR request: %s",
+ to_resolve);
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ to_resolve = rspamd_mempool_strdup(pool, ptr_str);
+ free(ptr_str);
+ }
+
+
+ if (task == NULL) {
+ ret = (rspamd_dns_resolver_request(cfg->dns_resolver,
+ session,
+ pool,
+ lua_dns_callback,
+ cbdata,
+ type,
+ to_resolve) != NULL);
+ }
+ else {
+ if (forced) {
+ ret = rspamd_dns_resolver_request_task_forced(task,
+ lua_dns_callback,
+ cbdata,
+ type,
+ to_resolve);
+ }
+ else {
+ ret = rspamd_dns_resolver_request_task(task,
+ lua_dns_callback,
+ cbdata,
+ type,
+ to_resolve);
+ }
+ }
+
+ if (ret) {
+ cbdata->thread = lua_thread_pool_get_running_entry(cfg->lua_thread_pool);
+ cbdata->s = session;
+
+ if (task) {
+ cbdata->item = rspamd_symcache_get_cur_item(task);
+ rspamd_symcache_item_async_inc(task, cbdata->item, M);
+ }
+
+ return lua_thread_yield(cbdata->thread, 0);
+ }
+ else {
+ lua_pushnil(L);
+ return 1;
+ }
+}
+
+void lua_dns_callback(struct rdns_reply *reply, void *arg)
+{
+ struct lua_rspamd_dns_cbdata *cbdata = arg;
+ lua_State *L = cbdata->thread->lua_state;
+
+ if (reply->code != RDNS_RC_NOERROR) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, rdns_strerror(reply->code));
+ }
+ else {
+ lua_push_dns_reply(L, reply);
+
+ lua_pushboolean(L, reply->flags & RDNS_AUTH);
+ lua_setfield(L, -3, "authenticated");
+
+ lua_pushboolean(L, reply->flags & RDNS_TRUNCATED);
+ lua_setfield(L, -3, "truncated");
+
+ /* result 1 - not and error */
+ lua_pushboolean(L, true);
+ /* push table into stack, result 2 - results itself */
+ lua_pushvalue(L, -3);
+ }
+
+ lua_thread_resume(cbdata->thread, 2);
+
+ if (cbdata->item) {
+ rspamd_symcache_item_async_dec_check(cbdata->task, cbdata->item, M);
+ }
+}
+
+static gint
+lua_load_dns(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, dns_f);
+
+ return 1;
+}
+
+void luaopen_dns(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_dns", lua_load_dns);
+}
diff --git a/src/lua/lua_dns_resolver.c b/src/lua/lua_dns_resolver.c
new file mode 100644
index 0000000..b022e13
--- /dev/null
+++ b/src/lua/lua_dns_resolver.c
@@ -0,0 +1,754 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+#include "utlist.h"
+
+
+/***
+ * @module rspamd_resolver
+ * This module allows to resolve DNS names from LUA code. All resolving is executed
+ * asynchronously. Here is an example of name resolution:
+ * @example
+local function symbol_callback(task)
+ local host = 'example.com'
+
+ local function dns_cb(resolver, to_resolve, results, err, _, authenticated)
+ if not results then
+ rspamd_logger.infox('DNS resolving of %1 failed: %2', host, err)
+ return
+ end
+ for _,r in ipairs(results) do
+ -- r is of type rspamd{ip} here, but it can be converted to string
+ rspamd_logger.infox('Resolved %1 to %2', host, tostring(r))
+ end
+ end
+
+ task:get_resolver():resolve_a({task = task, name = host, callback = dns_cb})
+end
+ */
+
+static const gchar *M = "rspamd lua dns resolver";
+
+/* Lua bindings */
+LUA_FUNCTION_DEF(dns_resolver, init);
+LUA_FUNCTION_DEF(dns_resolver, resolve_a);
+LUA_FUNCTION_DEF(dns_resolver, resolve_ptr);
+LUA_FUNCTION_DEF(dns_resolver, resolve_txt);
+LUA_FUNCTION_DEF(dns_resolver, resolve_mx);
+LUA_FUNCTION_DEF(dns_resolver, resolve_ns);
+LUA_FUNCTION_DEF(dns_resolver, resolve);
+LUA_FUNCTION_DEF(dns_resolver, idna_convert_utf8);
+
+void lua_push_dns_reply(lua_State *L, const struct rdns_reply *reply);
+
+static const struct luaL_reg dns_resolverlib_f[] = {
+ LUA_INTERFACE_DEF(dns_resolver, init),
+ {NULL, NULL}};
+
+static const struct luaL_reg dns_resolverlib_m[] = {
+ LUA_INTERFACE_DEF(dns_resolver, resolve_a),
+ LUA_INTERFACE_DEF(dns_resolver, resolve_ptr),
+ LUA_INTERFACE_DEF(dns_resolver, resolve_txt),
+ LUA_INTERFACE_DEF(dns_resolver, resolve_mx),
+ LUA_INTERFACE_DEF(dns_resolver, resolve_ns),
+ LUA_INTERFACE_DEF(dns_resolver, resolve),
+ LUA_INTERFACE_DEF(dns_resolver, idna_convert_utf8),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+struct rspamd_dns_resolver *
+lua_check_dns_resolver(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{resolver}");
+ luaL_argcheck(L, ud != NULL, pos, "'resolver' expected");
+ return ud ? *((struct rspamd_dns_resolver **) ud) : NULL;
+}
+
+struct lua_dns_cbdata {
+ struct rspamd_task *task;
+ rspamd_mempool_t *pool;
+ struct rspamd_dns_resolver *resolver;
+ gint cbref;
+ gchar *to_resolve;
+ gchar *user_str;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_async_session *s;
+};
+
+static int
+lua_dns_get_type(lua_State *L, int argno)
+{
+ int type = RDNS_REQUEST_A;
+ const gchar *strtype;
+
+ if (lua_type(L, argno) != LUA_TSTRING) {
+ lua_pushvalue(L, argno);
+ lua_gettable(L, lua_upvalueindex(1));
+
+ type = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+ if (type == 0) {
+ rspamd_lua_typerror(L, argno, "dns_request_type");
+ }
+ }
+ else {
+ strtype = lua_tostring(L, argno);
+ type = rdns_type_fromstr(strtype);
+ }
+
+ return type;
+}
+
+static void
+lua_dns_resolver_callback(struct rdns_reply *reply, gpointer arg)
+{
+ struct lua_dns_cbdata *cd = arg;
+ struct rspamd_dns_resolver **presolver;
+ lua_State *L;
+ struct lua_callback_state cbs;
+ rspamd_mempool_t *pool;
+ gint err_idx;
+
+ pool = cd->pool;
+ lua_thread_pool_prepare_callback(cd->resolver->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cd->cbref);
+
+ presolver = lua_newuserdata(L, sizeof(gpointer));
+ rspamd_lua_setclass(L, "rspamd{resolver}", -1);
+
+ *presolver = cd->resolver;
+ lua_pushstring(L, cd->to_resolve);
+
+ lua_push_dns_reply(L, reply);
+
+ /*
+ * 1 - resolver
+ * 2 - to_resolve
+ * 3 - entries | nil
+ * 4 - error | nil
+ * 5 - user_str
+ * 6 - reply->flags & RDNS_AUTH
+ * 7 - server
+ */
+ if (reply->code != RDNS_RC_NOERROR) {
+ lua_pushnil(L);
+ lua_pushstring(L, rdns_strerror(reply->code));
+ }
+ if (cd->user_str != NULL) {
+ lua_pushstring(L, cd->user_str);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ lua_pushboolean(L, reply->flags & RDNS_AUTH);
+
+ const gchar *servname = rdns_request_get_server(reply->request);
+
+ if (servname) {
+ lua_pushstring(L, servname);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (cd->item) {
+ /* We also need to restore the item in case there are some chains */
+ rspamd_symcache_set_cur_item(cd->task, cd->item);
+ }
+
+ if (lua_pcall(L, 7, 0, err_idx) != 0) {
+ msg_err_pool_check("call to dns callback failed: %s",
+ lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+
+ /* Unref function */
+ luaL_unref(L, LUA_REGISTRYINDEX, cd->cbref);
+ lua_thread_pool_restore_callback(&cbs);
+
+ if (cd->item) {
+ rspamd_symcache_item_async_dec_check(cd->task, cd->item, M);
+ }
+
+ if (!cd->pool) {
+ g_free(cd->to_resolve);
+ g_free(cd->user_str);
+ g_free(cd);
+ }
+}
+
+void lua_push_dns_reply(lua_State *L, const struct rdns_reply *reply)
+{
+ gint i = 0, naddrs = 0;
+ struct rdns_reply_entry *elt;
+ rspamd_inet_addr_t *addr;
+
+ if (reply->code == RDNS_RC_NOERROR) {
+ LL_FOREACH(reply->entries, elt)
+ {
+ naddrs++;
+ }
+
+ lua_createtable(L, naddrs, 0);
+
+ LL_FOREACH(reply->entries, elt)
+ {
+ if (!rdns_request_has_type(reply->request, elt->type)) {
+ /* Unrequested type has been returned, ignore it */
+ continue;
+ }
+
+ switch (elt->type) {
+ case RDNS_REQUEST_A:
+ addr = rspamd_inet_address_new(AF_INET, &elt->content.a.addr);
+ rspamd_lua_ip_push(L, addr);
+ rspamd_inet_address_free(addr);
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_AAAA:
+ addr = rspamd_inet_address_new(AF_INET6, &elt->content.aaa.addr);
+ rspamd_lua_ip_push(L, addr);
+ rspamd_inet_address_free(addr);
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_NS:
+ lua_pushstring(L, elt->content.ns.name);
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_PTR:
+ lua_pushstring(L, elt->content.ptr.name);
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_TXT:
+ case RDNS_REQUEST_SPF:
+ lua_pushstring(L, elt->content.txt.data);
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_MX:
+ /* mx['name'], mx['priority'] */
+ lua_createtable(L, 0, 2);
+ rspamd_lua_table_set(L, "name", elt->content.mx.name);
+ lua_pushstring(L, "priority");
+ lua_pushinteger(L, elt->content.mx.priority);
+ lua_settable(L, -3);
+
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_SOA:
+ lua_createtable(L, 0, 7);
+ rspamd_lua_table_set(L, "ns", elt->content.soa.mname);
+ rspamd_lua_table_set(L, "contact", elt->content.soa.admin);
+ lua_pushstring(L, "serial");
+ lua_pushinteger(L, elt->content.soa.serial);
+ lua_settable(L, -3);
+ lua_pushstring(L, "refresh");
+ lua_pushinteger(L, elt->content.soa.refresh);
+ lua_settable(L, -3);
+ lua_pushstring(L, "retry");
+ lua_pushinteger(L, elt->content.soa.retry);
+ lua_settable(L, -3);
+ lua_pushstring(L, "expiry");
+ lua_pushinteger(L, elt->content.soa.expire);
+ lua_settable(L, -3);
+ /* Negative TTL */
+ lua_pushstring(L, "nx");
+ lua_pushinteger(L, elt->content.soa.minimum);
+ lua_settable(L, -3);
+
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_CNAME:
+ lua_pushstring(L, elt->content.cname.name);
+ lua_rawseti(L, -2, ++i);
+ break;
+ default:
+ continue;
+ }
+ }
+ lua_pushnil(L);
+ }
+}
+
+/***
+ * @function rspamd_resolver.init(ev_base, config)
+ * @param {event_base} ev_base event base used for asynchronous events
+ * @param {rspamd_config} config rspamd configuration parameters
+ * @return {rspamd_resolver} new resolver object associated with the specified base
+ */
+static int
+lua_dns_resolver_init(lua_State *L)
+{
+ struct rspamd_dns_resolver *resolver, **presolver;
+ struct rspamd_config *cfg, **pcfg;
+ struct ev_loop *base, **pbase;
+
+ /* Check args */
+ pbase = rspamd_lua_check_udata(L, 1, "rspamd{ev_base}");
+ luaL_argcheck(L, pbase != NULL, 1, "'ev_base' expected");
+ base = pbase ? *(pbase) : NULL;
+ pcfg = rspamd_lua_check_udata(L, 2, "rspamd{config}");
+ luaL_argcheck(L, pcfg != NULL, 2, "'config' expected");
+ cfg = pcfg ? *(pcfg) : NULL;
+
+ if (base != NULL && cfg != NULL) {
+ resolver = rspamd_dns_resolver_init(NULL, base, cfg);
+ if (resolver) {
+ presolver = lua_newuserdata(L, sizeof(gpointer));
+ rspamd_lua_setclass(L, "rspamd{resolver}", -1);
+ *presolver = resolver;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static int
+lua_dns_resolver_resolve_common(lua_State *L,
+ struct rspamd_dns_resolver *resolver,
+ enum rdns_request_type type,
+ int first)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_async_session *session = NULL;
+ rspamd_mempool_t *pool = NULL;
+ const gchar *to_resolve = NULL, *user_str = NULL;
+ struct lua_dns_cbdata *cbdata;
+ gint cbref = -1, ret;
+ struct rspamd_task *task = NULL;
+ GError *err = NULL;
+ gboolean forced = FALSE;
+ struct rspamd_symcache_dynamic_item *item = NULL;
+
+ /* Check arguments */
+ if (!rspamd_lua_parse_table_arguments(L, first, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "session=U{session};mempool=U{mempool};*name=S;*callback=F;"
+ "option=S;task=U{task};forced=B",
+ &session, &pool, &to_resolve, &cbref, &user_str, &task, &forced)) {
+
+ if (err) {
+ ret = luaL_error(L, "invalid arguments: %s", err->message);
+ g_error_free(err);
+
+ return ret;
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (task) {
+ pool = task->task_pool;
+ session = task->s;
+ item = rspamd_symcache_get_cur_item(task);
+ }
+
+ if (to_resolve != NULL) {
+ if (pool != NULL) {
+ cbdata = rspamd_mempool_alloc0(pool, sizeof(struct lua_dns_cbdata));
+ cbdata->user_str = rspamd_mempool_strdup(pool, user_str);
+
+ if (type != RDNS_REQUEST_PTR) {
+ cbdata->to_resolve = rspamd_mempool_strdup(pool, to_resolve);
+ }
+ else {
+ char *ptr_str;
+
+ ptr_str = rdns_generate_ptr_from_str(to_resolve);
+
+ if (ptr_str == NULL) {
+ msg_err_task_check("wrong resolve string to PTR request: %s",
+ to_resolve);
+ goto err;
+ }
+
+ cbdata->to_resolve = rspamd_mempool_strdup(pool, ptr_str);
+ to_resolve = cbdata->to_resolve;
+ free(ptr_str);
+ }
+ }
+ else {
+ cbdata = g_malloc0(sizeof(struct lua_dns_cbdata));
+ cbdata->user_str = user_str ? g_strdup(user_str) : NULL;
+
+ if (type != RDNS_REQUEST_PTR) {
+ cbdata->to_resolve = g_strdup(to_resolve);
+ }
+ else {
+ char *ptr_str;
+
+ ptr_str = rdns_generate_ptr_from_str(to_resolve);
+
+ if (ptr_str == NULL) {
+ msg_err_task_check("wrong resolve string to PTR request: %s",
+ to_resolve);
+ goto err;
+ }
+
+ cbdata->to_resolve = g_strdup(ptr_str);
+ free(ptr_str);
+ }
+ }
+
+ cbdata->resolver = resolver;
+ cbdata->cbref = cbref;
+ cbdata->task = task;
+ cbdata->pool = pool;
+
+ if (task == NULL) {
+ if (rspamd_dns_resolver_request(resolver,
+ session,
+ pool,
+ lua_dns_resolver_callback,
+ cbdata,
+ type,
+ to_resolve)) {
+
+ lua_pushboolean(L, TRUE);
+
+ if (session) {
+ cbdata->s = session;
+ }
+ }
+ else {
+ goto err;
+ }
+ }
+ else {
+ /* Fail-safety as this function can, in theory, call
+ * lua_dns_resolver_callback without switching to the event loop
+ */
+ if (item) {
+ rspamd_symcache_item_async_inc(task, item, M);
+ }
+
+ if (forced) {
+ ret = rspamd_dns_resolver_request_task_forced(task,
+ lua_dns_resolver_callback,
+ cbdata,
+ type,
+ to_resolve);
+ }
+ else {
+ ret = rspamd_dns_resolver_request_task(task,
+ lua_dns_resolver_callback,
+ cbdata,
+ type,
+ to_resolve);
+ }
+
+ if (ret) {
+ cbdata->s = session;
+
+ if (item) {
+ cbdata->item = item;
+ rspamd_symcache_item_async_inc(task, item, M);
+ }
+ /* callback was set up */
+ lua_pushboolean(L, TRUE);
+ }
+ else {
+ if (item) {
+ rspamd_symcache_item_async_dec_check(task, item, M);
+ }
+
+ goto err;
+ }
+
+ if (item) {
+ rspamd_symcache_item_async_dec_check(task, item, M);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments to lua_resolve");
+ }
+
+ return 1;
+
+err:
+ /* Callback is not called in this case */
+ if (cbdata->cbref != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->cbref);
+ }
+
+ if (!pool) {
+ /* Free resources */
+ g_free(cbdata->to_resolve);
+ g_free(cbdata->user_str);
+ g_free(cbdata);
+ }
+
+ lua_pushnil(L);
+
+ return 1;
+}
+
+/***
+ * @method resolver:resolve_a(table)
+ * Resolve A record for a specified host.
+ * Table elements:
+ * * `task` - task element (preferred, required to track dependencies) -or-
+ * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`)
+ * * `mempool` - pool memory pool for storing intermediate data
+ * * `name` - host name to resolve
+ * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
+ * * `forced` - true if needed to override normal limit for DNS requests
+ * @return {boolean} `true` if DNS request has been scheduled
+ */
+static int
+lua_dns_resolver_resolve_a(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+
+ if (dns_resolver) {
+ return lua_dns_resolver_resolve_common(L,
+ dns_resolver,
+ RDNS_REQUEST_A,
+ 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method resolver:resolve_ptr(table)
+ * Resolve PTR record for a specified host.
+ * Table elements:
+ * * `task` - task element (preferred, required to track dependencies) -or-
+ * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`)
+ * * `mempool` - pool memory pool for storing intermediate data
+ * * `name` - host name to resolve
+ * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
+ * * `forced` - true if needed to override normal limit for DNS requests
+ * @return {boolean} `true` if DNS request has been scheduled
+ */
+static int
+lua_dns_resolver_resolve_ptr(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+
+ if (dns_resolver) {
+ return lua_dns_resolver_resolve_common(L,
+ dns_resolver,
+ RDNS_REQUEST_PTR,
+ 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method resolver:resolve_txt(table)
+ * Resolve TXT record for a specified host.
+ * Table elements:
+ * * `task` - task element (preferred, required to track dependencies) -or-
+ * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`)
+ * * `mempool` - pool memory pool for storing intermediate data
+ * * `name` - host name to resolve
+ * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
+ * * `forced` - true if needed to override normal limit for DNS requests
+ * @return {boolean} `true` if DNS request has been scheduled
+ */
+static int
+lua_dns_resolver_resolve_txt(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+
+ if (dns_resolver) {
+ return lua_dns_resolver_resolve_common(L,
+ dns_resolver,
+ RDNS_REQUEST_TXT,
+ 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method resolver:resolve_mx(table)
+ * Resolve MX record for a specified host.
+ * Table elements:
+ * * `task` - task element (preferred, required to track dependencies) -or-
+ * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`)
+ * * `mempool` - pool memory pool for storing intermediate data
+ * * `name` - host name to resolve
+ * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
+ * * `forced` - true if needed to override normal limit for DNS requests
+ * @return {boolean} `true` if DNS request has been scheduled
+ */
+static int
+lua_dns_resolver_resolve_mx(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+
+ if (dns_resolver) {
+ return lua_dns_resolver_resolve_common(L,
+ dns_resolver,
+ RDNS_REQUEST_MX,
+ 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method resolver:resolve_ns(table)
+ * Resolve NS records for a specified host.
+ * Table elements:
+ * * `task` - task element (preferred, required to track dependencies) -or-
+ * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`)
+ * * `mempool` - pool memory pool for storing intermediate data
+ * * `name` - host name to resolve
+ * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
+ * * `forced` - true if needed to override normal limit for DNS requests
+ * @return {boolean} `true` if DNS request has been scheduled
+ */
+static int
+lua_dns_resolver_resolve_ns(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+
+ if (dns_resolver) {
+ return lua_dns_resolver_resolve_common(L,
+ dns_resolver,
+ RDNS_REQUEST_NS,
+ 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/* XXX: broken currently */
+static int
+lua_dns_resolver_resolve(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+ int type;
+
+ type = lua_dns_get_type(L, 2);
+
+ if (dns_resolver && type != 0) {
+ return lua_dns_resolver_resolve_common(L, dns_resolver, type, 3);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method resolver:idna_convert_utf8(hostname[, pool])
+ * Converts domain name from IDN (in utf8 format) to punycode
+ * @return {string} new name converted
+ */
+static int
+lua_dns_resolver_idna_convert_utf8(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+ gsize hlen;
+ guint conv_len = 0;
+ const gchar *hname = luaL_checklstring(L, 2, &hlen);
+ gchar *converted;
+ rspamd_mempool_t *pool = rspamd_lua_check_udata_maybe(L, 3, "rspamd{mempool}");
+
+
+ if (dns_resolver && hname) {
+ if (!rspamd_str_has_8bit(hname, hlen)) {
+ /* No 8 bit, no reasons to call idna */
+ lua_pushlstring(L, hname, hlen);
+ }
+ else {
+ converted = rspamd_dns_resolver_idna_convert_utf8(dns_resolver, pool,
+ hname, hlen, &conv_len);
+
+ if (converted == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushlstring(L, converted, conv_len);
+
+ if (pool == NULL) {
+ g_free(converted);
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_dns_resolver(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, dns_resolverlib_f);
+
+ return 1;
+}
+
+void luaopen_dns_resolver(lua_State *L)
+{
+
+ rspamd_lua_new_class(L, "rspamd{resolver}", dns_resolverlib_m);
+ {
+ LUA_ENUM(L, DNS_A, RDNS_REQUEST_A);
+ LUA_ENUM(L, DNS_PTR, RDNS_REQUEST_PTR);
+ LUA_ENUM(L, DNS_MX, RDNS_REQUEST_MX);
+ LUA_ENUM(L, DNS_TXT, RDNS_REQUEST_TXT);
+ LUA_ENUM(L, DNS_SRV, RDNS_REQUEST_SRV);
+ LUA_ENUM(L, DNS_SPF, RDNS_REQUEST_SPF);
+ LUA_ENUM(L, DNS_AAAA, RDNS_REQUEST_AAAA);
+ LUA_ENUM(L, DNS_SOA, RDNS_REQUEST_SOA);
+ LUA_ENUM(L, DNS_CNAME, RDNS_REQUEST_CNAME);
+ }
+
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "rspamd_resolver", lua_load_dns_resolver);
+}
diff --git a/src/lua/lua_dns_resolver.h b/src/lua/lua_dns_resolver.h
new file mode 100644
index 0000000..515e9ac
--- /dev/null
+++ b/src/lua/lua_dns_resolver.h
@@ -0,0 +1,15 @@
+#ifndef RSPAMD_LUA_DNS_H
+#define RSPAMD_LUA_DNS_H
+
+struct lua_State;
+struct rdns_reply;
+
+/**
+ * Pushes dns reply onto Lua stack
+ *
+ * @param L
+ * @param reply
+ */
+void lua_push_dns_reply(struct lua_State *L, const struct rdns_reply *reply);
+
+#endif
diff --git a/src/lua/lua_expression.c b/src/lua/lua_expression.c
new file mode 100644
index 0000000..1ac6f86
--- /dev/null
+++ b/src/lua/lua_expression.c
@@ -0,0 +1,512 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "expression.h"
+
+/***
+ * @module rspamd_expression
+ * This module can be used to implement different logic expressions in lua using
+ * rspamd AST optimizer. There are some examples in individual methods definitions to help understanding of this module.
+ */
+
+/***
+ * @function rspamd_expression.create(line, {parse_func, [process_func]}, pool)
+ * Create expression from the line using atom parsing routines and the specified memory pool
+ * @param {string} line expression line
+ * @param {table} atom_functions parse_atom function and optional process_atom function
+ * @param {rspamd_mempool} memory pool to use for this function
+ * @return {expr, err} expression object and error message of `expr` is nil
+ * @example
+require "fun" ()
+local rspamd_expression = require "rspamd_expression"
+local rspamd_mempool = require "rspamd_mempool"
+
+local function parse_func(str)
+ -- extract token till the first space character
+ local token = table.concat(totable(take_while(function(s) return s ~= ' ' end, iter(str))))
+ -- Return token name
+ return token
+end
+
+local function process_func(token)
+ -- Do something using token and task
+end
+
+local pool = rspamd_mempool.create()
+local expr,err = rspamd_expression.create('A & B | !C', {parse_func, process_func}, pool)
+-- Expression is destroyed when the corresponding pool is destroyed
+pool:destroy()
+ */
+LUA_FUNCTION_DEF(expr, create);
+
+/***
+ * @method rspamd_expression:to_string()
+ * Converts rspamd expression to string
+ * @return {string} string representation of rspamd expression
+ */
+LUA_FUNCTION_DEF(expr, to_string);
+
+/***
+ * @method rspamd_expression:process([callback, ]input[, flags])
+ * Executes the expression and pass input to process atom callbacks
+ * @param {function} callback if not specified on process, then callback must be here
+ * @param {any} input input data for processing callbacks
+ * @return {number} result of the expression evaluation
+ */
+LUA_FUNCTION_DEF(expr, process);
+
+/***
+ * @method rspamd_expression:process_traced([callback, ]input[, flags])
+ * Executes the expression and pass input to process atom callbacks. This function also saves the full trace
+ * @param {function} callback if not specified on process, then callback must be here
+ * @param {any} input input data for processing callbacks
+ * @return {number, table of matched atoms} result of the expression evaluation
+ */
+LUA_FUNCTION_DEF(expr, process_traced);
+
+/***
+ * @method rspamd_expression:atoms()
+ * Extract all atoms from the expression as table of strings
+ * @return {table/strings} list of all atoms in the expression
+ */
+LUA_FUNCTION_DEF(expr, atoms);
+
+static const struct luaL_reg exprlib_m[] = {
+ LUA_INTERFACE_DEF(expr, to_string),
+ LUA_INTERFACE_DEF(expr, atoms),
+ LUA_INTERFACE_DEF(expr, process),
+ LUA_INTERFACE_DEF(expr, process_traced),
+ {"__tostring", lua_expr_to_string},
+ {NULL, NULL}};
+
+static const struct luaL_reg exprlib_f[] = {
+ LUA_INTERFACE_DEF(expr, create),
+ {NULL, NULL}};
+
+static rspamd_expression_atom_t *lua_atom_parse(const gchar *line, gsize len,
+ rspamd_mempool_t *pool, gpointer ud, GError **err);
+static gdouble lua_atom_process(gpointer runtime_ud, rspamd_expression_atom_t *atom);
+
+static const struct rspamd_atom_subr lua_atom_subr = {
+ .parse = lua_atom_parse,
+ .process = lua_atom_process,
+ .priority = NULL,
+ .destroy = NULL};
+
+struct lua_expression {
+ struct rspamd_expression *expr;
+ gint parse_idx;
+ gint process_idx;
+ lua_State *L;
+ rspamd_mempool_t *pool;
+};
+
+static GQuark
+lua_expr_quark(void)
+{
+ return g_quark_from_static_string("lua-expression");
+}
+
+struct lua_expression *
+rspamd_lua_expression(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{expr}");
+ luaL_argcheck(L, ud != NULL, pos, "'expr' expected");
+ return ud ? *((struct lua_expression **) ud) : NULL;
+}
+
+static void
+lua_expr_dtor(gpointer p)
+{
+ struct lua_expression *e = (struct lua_expression *) p;
+
+ if (e->parse_idx != -1) {
+ luaL_unref(e->L, LUA_REGISTRYINDEX, e->parse_idx);
+ }
+
+ if (e->process_idx != -1) {
+ luaL_unref(e->L, LUA_REGISTRYINDEX, e->process_idx);
+ }
+}
+
+static rspamd_expression_atom_t *
+lua_atom_parse(const gchar *line, gsize len,
+ rspamd_mempool_t *pool, gpointer ud, GError **err)
+{
+ struct lua_expression *e = (struct lua_expression *) ud;
+ rspamd_expression_atom_t *atom;
+ gsize rlen;
+ const gchar *tok;
+
+ lua_rawgeti(e->L, LUA_REGISTRYINDEX, e->parse_idx);
+ lua_pushlstring(e->L, line, len);
+
+ if (lua_pcall(e->L, 1, 1, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(e->L, -1));
+ lua_pop(e->L, 1);
+ return NULL;
+ }
+
+ if (lua_type(e->L, -1) != LUA_TSTRING) {
+ g_set_error(err, lua_expr_quark(), 500, "cannot parse lua atom");
+ lua_pop(e->L, 1);
+ return NULL;
+ }
+
+ tok = lua_tolstring(e->L, -1, &rlen);
+ atom = rspamd_mempool_alloc0(e->pool, sizeof(*atom));
+ atom->str = rspamd_mempool_strdup(e->pool, tok);
+ atom->len = rlen;
+ atom->data = ud;
+
+ lua_pop(e->L, 1);
+
+ return atom;
+}
+
+struct lua_atom_process_data {
+ lua_State *L;
+ struct lua_expression *e;
+ gint process_cb_pos;
+ gint stack_item;
+};
+
+static gdouble
+lua_atom_process(gpointer runtime_ud, rspamd_expression_atom_t *atom)
+{
+ struct lua_atom_process_data *pd = (struct lua_atom_process_data *) runtime_ud;
+ gdouble ret = 0;
+ guint nargs;
+ gint err_idx;
+
+ if (pd->stack_item != -1) {
+ nargs = 2;
+ }
+ else {
+ nargs = 1;
+ }
+
+ lua_pushcfunction(pd->L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(pd->L);
+
+ /* Function position */
+ lua_pushvalue(pd->L, pd->process_cb_pos);
+ /* Atom name */
+ lua_pushlstring(pd->L, atom->str, atom->len);
+
+ /* If we have data passed */
+ if (pd->stack_item != -1) {
+ lua_pushvalue(pd->L, pd->stack_item);
+ }
+
+ if (lua_pcall(pd->L, nargs, 1, err_idx) != 0) {
+ msg_info("expression process callback failed: %s", lua_tostring(pd->L, -1));
+ }
+ else {
+ ret = lua_tonumber(pd->L, -1);
+ }
+
+ lua_settop(pd->L, err_idx - 1);
+
+ return ret;
+}
+
+static gint
+lua_expr_process(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_expression *e = rspamd_lua_expression(L, 1);
+ struct lua_atom_process_data pd;
+ gdouble res;
+ gint flags = 0, old_top;
+
+ pd.L = L;
+ pd.e = e;
+ old_top = lua_gettop(L);
+
+ if (e->process_idx == -1) {
+ if (!lua_isfunction(L, 2)) {
+ return luaL_error(L, "expression process is called with no callback function");
+ }
+
+ pd.process_cb_pos = 2;
+
+ if (lua_type(L, 3) != LUA_TNONE && lua_type(L, 3) != LUA_TNIL) {
+ pd.stack_item = 3;
+ }
+ else {
+ pd.stack_item = -1;
+ }
+
+ if (lua_isnumber(L, 4)) {
+ flags = lua_tointeger(L, 4);
+ }
+ }
+ else {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, e->process_idx);
+ pd.process_cb_pos = lua_gettop(L);
+
+ if (lua_type(L, 2) != LUA_TNONE && lua_type(L, 2) != LUA_TNIL) {
+ pd.stack_item = 2;
+ }
+ else {
+ pd.stack_item = -1;
+ }
+
+ if (lua_isnumber(L, 3)) {
+ flags = lua_tointeger(L, 3);
+ }
+ }
+
+ res = rspamd_process_expression(e->expr, flags, &pd);
+
+ lua_settop(L, old_top);
+ lua_pushnumber(L, res);
+
+ return 1;
+}
+
+static gint
+lua_expr_process_traced(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_expression *e = rspamd_lua_expression(L, 1);
+ struct lua_atom_process_data pd;
+ gdouble res;
+ gint flags = 0, old_top;
+ GPtrArray *trace;
+
+ pd.L = L;
+ pd.e = e;
+ old_top = lua_gettop(L);
+
+ if (e->process_idx == -1) {
+ if (!lua_isfunction(L, 2)) {
+ return luaL_error(L, "expression process is called with no callback function");
+ }
+
+ pd.process_cb_pos = 2;
+ pd.stack_item = 3;
+
+ if (lua_isnumber(L, 4)) {
+ flags = lua_tointeger(L, 4);
+ }
+ }
+ else {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, e->process_idx);
+ pd.process_cb_pos = lua_gettop(L);
+ pd.stack_item = 2;
+
+ if (lua_isnumber(L, 3)) {
+ flags = lua_tointeger(L, 3);
+ }
+ }
+
+ res = rspamd_process_expression_track(e->expr, flags, &pd, &trace);
+
+ lua_settop(L, old_top);
+ lua_pushnumber(L, res);
+
+ lua_createtable(L, trace->len, 0);
+
+ for (guint i = 0; i < trace->len; i++) {
+ struct rspamd_expression_atom_s *atom = g_ptr_array_index(trace, i);
+
+ lua_pushlstring(L, atom->str, atom->len);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ g_ptr_array_free(trace, TRUE);
+
+ return 2;
+}
+
+static gint
+lua_expr_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_expression *e, **pe;
+ const char *line;
+ gsize len;
+ gboolean no_process = FALSE;
+ GError *err = NULL;
+ rspamd_mempool_t *pool;
+
+ /* Check sanity of the arguments */
+ if (lua_type(L, 1) != LUA_TSTRING ||
+ (lua_type(L, 2) != LUA_TTABLE && lua_type(L, 2) != LUA_TFUNCTION) ||
+ rspamd_lua_check_mempool(L, 3) == NULL) {
+ lua_pushnil(L);
+ lua_pushstring(L, "bad arguments");
+ }
+ else {
+ line = lua_tolstring(L, 1, &len);
+ pool = rspamd_lua_check_mempool(L, 3);
+
+ e = rspamd_mempool_alloc(pool, sizeof(*e));
+ e->L = L;
+ e->pool = pool;
+
+ /* Check callbacks */
+ if (lua_istable(L, 2)) {
+ lua_pushvalue(L, 2);
+ lua_pushnumber(L, 1);
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ lua_pop(L, 1);
+ lua_pushnil(L);
+ lua_pushstring(L, "bad parse callback");
+
+ return 2;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushnumber(L, 2);
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ if (lua_type(L, -1) != LUA_TNIL && lua_type(L, -1) != LUA_TNONE) {
+ lua_pop(L, 1);
+ lua_pushnil(L);
+ lua_pushstring(L, "bad process callback");
+
+ return 2;
+ }
+ else {
+ no_process = TRUE;
+ }
+ }
+
+ lua_pop(L, 1);
+ /* Table is still on the top of stack */
+
+ lua_pushnumber(L, 1);
+ lua_gettable(L, -2);
+ e->parse_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ if (!no_process) {
+ lua_pushnumber(L, 2);
+ lua_gettable(L, -2);
+ e->process_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ e->process_idx = -1;
+ }
+
+ lua_pop(L, 1); /* Table */
+ }
+ else {
+ /* Process function is just a function, not a table */
+ lua_pushvalue(L, 2);
+ e->parse_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+ e->process_idx = -1;
+ }
+
+ if (!rspamd_parse_expression(line, len, &lua_atom_subr, e, pool, &err,
+ &e->expr)) {
+ lua_pushnil(L);
+ lua_pushstring(L, err->message);
+ g_error_free(err);
+ lua_expr_dtor(e);
+
+ return 2;
+ }
+
+ rspamd_mempool_add_destructor(pool, lua_expr_dtor, e);
+ pe = lua_newuserdata(L, sizeof(struct lua_expression *));
+ rspamd_lua_setclass(L, "rspamd{expr}", -1);
+ *pe = e;
+ lua_pushnil(L);
+ }
+
+ return 2;
+}
+
+static gint
+lua_expr_to_string(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_expression *e = rspamd_lua_expression(L, 1);
+ GString *str;
+
+ if (e != NULL && e->expr != NULL) {
+ str = rspamd_expression_tostring(e->expr);
+ if (str) {
+ lua_pushlstring(L, str->str, str->len);
+ g_string_free(str, TRUE);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+struct lua_expr_atoms_cbdata {
+ lua_State *L;
+ gint idx;
+};
+
+static void
+lua_exr_atom_cb(const rspamd_ftok_t *tok, gpointer ud)
+{
+ struct lua_expr_atoms_cbdata *cbdata = ud;
+
+ lua_pushlstring(cbdata->L, tok->begin, tok->len);
+ lua_rawseti(cbdata->L, -2, cbdata->idx++);
+}
+
+static gint
+lua_expr_atoms(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_expression *e = rspamd_lua_expression(L, 1);
+ struct lua_expr_atoms_cbdata cbdata;
+
+ if (e != NULL && e->expr != NULL) {
+ lua_newtable(L);
+ cbdata.L = L;
+ cbdata.idx = 1;
+ rspamd_expression_atom_foreach(e->expr, lua_exr_atom_cb, &cbdata);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_expression(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, exprlib_f);
+
+ return 1;
+}
+
+void luaopen_expression(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{expr}", exprlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_expression", lua_load_expression);
+}
diff --git a/src/lua/lua_html.cxx b/src/lua/lua_html.cxx
new file mode 100644
index 0000000..6613337
--- /dev/null
+++ b/src/lua/lua_html.cxx
@@ -0,0 +1,738 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "message.h"
+#include "libserver/html/html.h"
+#include "libserver/html/html.hxx"
+#include "libserver/html/html_tag.hxx"
+#include "libserver/html/html_block.hxx"
+#include "images.h"
+
+#include "contrib/ankerl/unordered_dense.h"
+#include <frozen/string.h>
+#include <frozen/unordered_map.h>
+
+/***
+ * @module rspamd_html
+ * This module provides different methods to access HTML tags. To get HTML context
+ * from an HTML part you could use method `part:get_html()`
+ * @example
+rspamd_config.R_EMPTY_IMAGE = function(task)
+ local tp = task:get_text_parts() -- get text parts in a message
+
+ for _,p in ipairs(tp) do -- iterate over text parts array using `ipairs`
+ if p:is_html() then -- if the current part is html part
+ local hc = p:get_html() -- we get HTML context
+ local len = p:get_length() -- and part's length
+
+ if len < 50 then -- if we have a part that has less than 50 bytes of text
+ local images = hc:get_images() -- then we check for HTML images
+
+ if images then -- if there are images
+ for _,i in ipairs(images) do -- then iterate over images in the part
+ if i['height'] + i['width'] >= 400 then -- if we have a large image
+ return true -- add symbol
+ end
+ end
+ end
+ end
+ end
+ end
+end
+ */
+
+/***
+ * @method html:has_tag(name)
+ * Checks if a specified tag `name` is presented in a part
+ * @param {string} name name of tag to check
+ * @return {boolean} `true` if the tag exists in HTML tree
+ */
+LUA_FUNCTION_DEF(html, has_tag);
+
+/***
+ * @method html:check_property(name)
+ * Checks if the HTML has a specific property. Here is the list of available properties:
+ *
+ * - `no_html` - no html tag presented
+ * - `bad_element` - part has some broken elements
+ * - `xml` - part is xhtml
+ * - `unknown_element` - part has some unknown elements
+ * - `duplicate_element` - part has some duplicate elements that should be unique (namely, `title` tag)
+ * - `unbalanced` - part has unbalanced tags
+ * @param {string} name name of property
+ * @return {boolean} true if the part has the specified property
+ */
+LUA_FUNCTION_DEF(html, has_property);
+
+/***
+ * @method html:get_images()
+ * Returns a table of images found in html. Each image is, in turn, a table with the following fields:
+ *
+ * - `src` - link to the source
+ * - `height` - height in pixels
+ * - `width` - width in pixels
+ * - `embedded` - `true` if an image is embedded in a message
+ * @return {table} table of images in html part
+ */
+LUA_FUNCTION_DEF(html, get_images);
+
+/***
+ * @method html:foreach_tag(tagname, callback)
+ * Processes HTML tree calling the specified callback for each tag of the specified
+ * type.
+ *
+ * Callback is called with the following attributes:
+ *
+ * - `tag`: html tag structure
+ * - `content_length`: length of content within a tag
+ *
+ * Callback function should return `true` to **stop** processing and `false` to continue
+ * @return nothing
+ */
+LUA_FUNCTION_DEF(html, foreach_tag);
+
+/***
+ * @method html:get_invisible()
+ * Returns invisible content of the HTML data
+ * @return
+ */
+LUA_FUNCTION_DEF(html, get_invisible);
+
+static const struct luaL_reg htmllib_m[] = {
+ LUA_INTERFACE_DEF(html, has_tag),
+ LUA_INTERFACE_DEF(html, has_property),
+ LUA_INTERFACE_DEF(html, get_images),
+ LUA_INTERFACE_DEF(html, foreach_tag),
+ LUA_INTERFACE_DEF(html, get_invisible),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/***
+ * @method html_tag:get_type()
+ * Returns string representation of HTML type for a tag
+ * @return {string} type of tag
+ */
+LUA_FUNCTION_DEF(html_tag, get_type);
+/***
+ * @method html_tag:get_extra()
+ * Returns extra data associated with the tag
+ * @return {url|image|nil} extra data associated with the tag
+ */
+LUA_FUNCTION_DEF(html_tag, get_extra);
+/***
+ * @method html_tag:get_parent()
+ * Returns parent node for a specified tag
+ * @return {html_tag} parent object for a specified tag
+ */
+LUA_FUNCTION_DEF(html_tag, get_parent);
+
+/***
+ * @method html_tag:get_flags()
+ * Returns flags a specified tag:
+ *
+ * - `closed`: tag is properly closed
+ * - `closing`: tag is a closing tag
+ * - `broken`: tag is somehow broken
+ * - `unbalanced`: tag is unbalanced
+ * - `xml`: tag is xml tag
+ * @return {table} table of flags
+ */
+LUA_FUNCTION_DEF(html_tag, get_flags);
+/***
+ * @method html_tag:get_content()
+ * Returns content of tag (approximate for some cases)
+ * @return {rspamd_text} rspamd text with tag's content
+ */
+LUA_FUNCTION_DEF(html_tag, get_content);
+/***
+ * @method html_tag:get_content_length()
+ * Returns length of a tag's content
+ * @return {number} size of content enclosed within a tag
+ */
+LUA_FUNCTION_DEF(html_tag, get_content_length);
+
+/***
+ * @method html_tag:get_style()
+ * Returns style calculated for the element
+ * @return {table} table associated with the style
+ */
+LUA_FUNCTION_DEF(html_tag, get_style);
+
+/***
+ * @method html_tag:get_attribute(name)
+ * Returns value of attribute for the element
+ * Refer to `html_components_map` in `src/libserver/html/html.cxx` for recognised names
+ * @return {string|nil} value of the attribute
+ */
+LUA_FUNCTION_DEF(html_tag, get_attribute);
+
+static const struct luaL_reg taglib_m[] = {
+ LUA_INTERFACE_DEF(html_tag, get_type),
+ LUA_INTERFACE_DEF(html_tag, get_extra),
+ LUA_INTERFACE_DEF(html_tag, get_parent),
+ LUA_INTERFACE_DEF(html_tag, get_flags),
+ LUA_INTERFACE_DEF(html_tag, get_content),
+ LUA_INTERFACE_DEF(html_tag, get_content_length),
+ LUA_INTERFACE_DEF(html_tag, get_style),
+ LUA_INTERFACE_DEF(html_tag, get_attribute),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static struct rspamd::html::html_content *
+lua_check_html(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{html}");
+ luaL_argcheck(L, ud != NULL, pos, "'html' expected");
+ return ud ? *((struct rspamd::html::html_content **) ud) : NULL;
+}
+
+struct lua_html_tag {
+ rspamd::html::html_content *html;
+ const rspamd::html::html_tag *tag;
+};
+
+static struct lua_html_tag *
+lua_check_html_tag(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{html_tag}");
+ luaL_argcheck(L, ud != NULL, pos, "'html_tag' expected");
+ return ud ? ((struct lua_html_tag *) ud) : NULL;
+}
+
+static gint
+lua_html_has_tag(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ auto *hc = lua_check_html(L, 1);
+ const gchar *tagname = luaL_checkstring(L, 2);
+ gboolean ret = FALSE;
+
+ if (hc && tagname) {
+ if (rspamd_html_tag_seen(hc, tagname)) {
+ ret = TRUE;
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return 1;
+}
+
+constexpr const auto prop_map = frozen::make_unordered_map<frozen::string, int>({
+ {"no_html", RSPAMD_HTML_FLAG_BAD_START},
+ {"bad_start", RSPAMD_HTML_FLAG_BAD_START},
+ {"bad_element", RSPAMD_HTML_FLAG_BAD_ELEMENTS},
+ {"bad_elements", RSPAMD_HTML_FLAG_BAD_ELEMENTS},
+ {"xml", RSPAMD_HTML_FLAG_XML},
+ {"unknown_element", RSPAMD_HTML_FLAG_UNKNOWN_ELEMENTS},
+ {"unknown_elements", RSPAMD_HTML_FLAG_UNKNOWN_ELEMENTS},
+ {"duplicate_element", RSPAMD_HTML_FLAG_DUPLICATE_ELEMENTS},
+ {"duplicate_elements", RSPAMD_HTML_FLAG_DUPLICATE_ELEMENTS},
+ {"unbalanced", RSPAMD_HTML_FLAG_UNBALANCED},
+ {"data_urls", RSPAMD_HTML_FLAG_HAS_DATA_URLS},
+});
+
+static gint
+lua_html_has_property(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ auto *hc = lua_check_html(L, 1);
+ const gchar *propname = luaL_checkstring(L, 2);
+ gboolean ret = FALSE;
+
+ if (hc && propname) {
+ auto found_prop = prop_map.find(frozen::string(propname));
+
+ if (found_prop != prop_map.end()) {
+ ret = hc->flags & found_prop->second;
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return 1;
+}
+
+static void
+lua_html_push_image(lua_State *L, const struct html_image *img)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag;
+ struct rspamd_url **purl;
+
+ lua_createtable(L, 0, 7);
+
+ if (img->src) {
+ lua_pushstring(L, "src");
+
+ if (img->flags & RSPAMD_HTML_FLAG_IMAGE_DATA) {
+ struct rspamd_lua_text *t;
+
+ t = static_cast<rspamd_lua_text *>(lua_newuserdata(L, sizeof(*t)));
+ t->start = img->src;
+ t->len = strlen(img->src);
+ t->flags = 0;
+
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ }
+ else {
+ lua_pushstring(L, img->src);
+ }
+
+ lua_settable(L, -3);
+ }
+
+ if (img->url) {
+ lua_pushstring(L, "url");
+ purl = static_cast<rspamd_url **>(lua_newuserdata(L, sizeof(gpointer)));
+ *purl = img->url;
+ rspamd_lua_setclass(L, "rspamd{url}", -1);
+ lua_settable(L, -3);
+ }
+
+ if (img->tag) {
+ lua_pushstring(L, "tag");
+ ltag = static_cast<lua_html_tag *>(lua_newuserdata(L, sizeof(struct lua_html_tag)));
+ ltag->tag = static_cast<rspamd::html::html_tag *>(img->tag);
+ ltag->html = NULL;
+ rspamd_lua_setclass(L, "rspamd{html_tag}", -1);
+ lua_settable(L, -3);
+ }
+
+ lua_pushstring(L, "height");
+ lua_pushinteger(L, img->height);
+ lua_settable(L, -3);
+ lua_pushstring(L, "width");
+ lua_pushinteger(L, img->width);
+ lua_settable(L, -3);
+ lua_pushstring(L, "embedded");
+ lua_pushboolean(L, img->flags & RSPAMD_HTML_FLAG_IMAGE_EMBEDDED);
+ lua_settable(L, -3);
+ lua_pushstring(L, "data");
+ lua_pushboolean(L, img->flags & RSPAMD_HTML_FLAG_IMAGE_DATA);
+ lua_settable(L, -3);
+}
+
+static gint
+lua_html_get_images(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ auto *hc = lua_check_html(L, 1);
+ guint i = 1;
+
+ if (hc != NULL) {
+ lua_createtable(L, hc->images.size(), 0);
+
+ for (const auto *img: hc->images) {
+ lua_html_push_image(L, img);
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+
+ return 1;
+}
+
+static void
+lua_html_push_block(lua_State *L, const struct rspamd::html::html_block *bl)
+{
+ LUA_TRACE_POINT;
+
+ lua_createtable(L, 0, 6);
+
+ if (bl->fg_color_mask) {
+ lua_pushstring(L, "color");
+ lua_createtable(L, 4, 0);
+ lua_pushinteger(L, bl->fg_color.r);
+ lua_rawseti(L, -2, 1);
+ lua_pushinteger(L, bl->fg_color.g);
+ lua_rawseti(L, -2, 2);
+ lua_pushinteger(L, bl->fg_color.b);
+ lua_rawseti(L, -2, 3);
+ lua_pushinteger(L, bl->fg_color.alpha);
+ lua_rawseti(L, -2, 4);
+ lua_settable(L, -3);
+ }
+ if (bl->bg_color_mask) {
+ lua_pushstring(L, "bgcolor");
+ lua_createtable(L, 4, 0);
+ lua_pushinteger(L, bl->bg_color.r);
+ lua_rawseti(L, -2, 1);
+ lua_pushinteger(L, bl->bg_color.g);
+ lua_rawseti(L, -2, 2);
+ lua_pushinteger(L, bl->bg_color.b);
+ lua_rawseti(L, -2, 3);
+ lua_pushinteger(L, bl->bg_color.alpha);
+ lua_rawseti(L, -2, 4);
+ lua_settable(L, -3);
+ }
+
+ if (bl->font_mask) {
+ lua_pushstring(L, "font_size");
+ lua_pushinteger(L, bl->font_size);
+ lua_settable(L, -3);
+ }
+
+ lua_pushstring(L, "visible");
+ lua_pushboolean(L, bl->is_visible());
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "transparent");
+ lua_pushboolean(L, bl->is_transparent());
+ lua_settable(L, -3);
+}
+
+static gint
+lua_html_foreach_tag(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ auto *hc = lua_check_html(L, 1);
+ const gchar *tagname;
+ gint id;
+ auto any = false;
+ ankerl::unordered_dense::set<int> tags;
+
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ tagname = luaL_checkstring(L, 2);
+ if (strcmp(tagname, "any") == 0) {
+ any = true;
+ }
+ else {
+ id = rspamd_html_tag_by_name(tagname);
+
+ if (id == -1) {
+ return luaL_error(L, "invalid tagname: %s", tagname);
+ }
+
+
+ tags.insert(id);
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TTABLE) {
+ lua_pushvalue(L, 2);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ tagname = luaL_checkstring(L, -1);
+ if (strcmp(tagname, "any") == 0) {
+ any = TRUE;
+ }
+ else {
+ id = rspamd_html_tag_by_name(tagname);
+
+ if (id == -1) {
+ return luaL_error(L, "invalid tagname: %s", tagname);
+ }
+ tags.insert(id);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ if (hc && (any || !tags.empty()) && lua_isfunction(L, 3)) {
+ hc->traverse_all_tags([&](const rspamd::html::html_tag *tag) -> bool {
+ if (tag && (any || tags.contains(tag->id))) {
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ auto err_idx = lua_gettop(L);
+ lua_pushvalue(L, 3);
+
+ auto *ltag = static_cast<lua_html_tag *>(lua_newuserdata(L, sizeof(lua_html_tag)));
+ ltag->tag = tag;
+ ltag->html = hc;
+ auto ct = ltag->tag->get_content(hc);
+ rspamd_lua_setclass(L, "rspamd{html_tag}", -1);
+ lua_pushinteger(L, ct.size());
+
+ /* Leaf flag */
+ if (tag->children.empty()) {
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ if (lua_pcall(L, 3, 1, err_idx) != 0) {
+ msg_err("error in foreach_tag callback: %s", lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+ return false;
+ }
+
+ if (lua_toboolean(L, -1)) {
+ lua_settop(L, err_idx - 1);
+ return false;
+ }
+
+ lua_settop(L, err_idx - 1);
+ }
+
+ return true;
+ });
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_html_get_invisible(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ auto *hc = lua_check_html(L, 1);
+
+ if (hc != NULL) {
+ lua_new_text(L, hc->invisible.c_str(), hc->invisible.size(), false);
+ }
+ else {
+ lua_newtable(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+ const gchar *tagname;
+
+ if (ltag != NULL) {
+ tagname = rspamd_html_tag_by_id(ltag->tag->id);
+
+ if (tagname) {
+ lua_pushstring(L, tagname);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_parent(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1), *ptag;
+
+ if (ltag != NULL) {
+ auto *parent = ltag->tag->parent;
+
+ if (parent) {
+ ptag = static_cast<lua_html_tag *>(lua_newuserdata(L, sizeof(*ptag)));
+ ptag->tag = static_cast<rspamd::html::html_tag *>(parent);
+ ptag->html = ltag->html;
+ rspamd_lua_setclass(L, "rspamd{html_tag}", -1);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_flags(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+ gint i = 1;
+
+ if (ltag && ltag->tag) {
+ /* Push flags */
+ lua_createtable(L, 4, 0);
+ if (ltag->tag->flags & FL_HREF) {
+ lua_pushstring(L, "href");
+ lua_rawseti(L, -2, i++);
+ }
+ if (ltag->tag->flags & FL_CLOSED) {
+ lua_pushstring(L, "closed");
+ lua_rawseti(L, -2, i++);
+ }
+ if (ltag->tag->flags & FL_BROKEN) {
+ lua_pushstring(L, "broken");
+ lua_rawseti(L, -2, i++);
+ }
+ if (ltag->tag->flags & FL_XML) {
+ lua_pushstring(L, "xml");
+ lua_rawseti(L, -2, i++);
+ }
+ if (ltag->tag->flags & RSPAMD_HTML_FLAG_UNBALANCED) {
+ lua_pushstring(L, "unbalanced");
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+ struct rspamd_lua_text *t;
+
+ if (ltag) {
+
+ if (ltag->html) {
+ auto ct = ltag->tag->get_content(ltag->html);
+ if (ct.size() > 0) {
+ t = static_cast<rspamd_lua_text *>(lua_newuserdata(L, sizeof(*t)));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = ct.data();
+ t->len = ct.size();
+ t->flags = 0;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_content_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+
+ if (ltag) {
+ if (ltag->html) {
+ auto ct = ltag->tag->get_content(ltag->html);
+ lua_pushinteger(L, ct.size());
+ }
+ else {
+ lua_pushinteger(L, ltag->tag->get_content_length());
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_extra(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+ struct html_image *img;
+
+ if (ltag) {
+ if (!std::holds_alternative<std::monostate>(ltag->tag->extra)) {
+ if (std::holds_alternative<struct html_image *>(ltag->tag->extra)) {
+ img = std::get<struct html_image *>(ltag->tag->extra);
+ lua_html_push_image(L, img);
+ }
+ else if (std::holds_alternative<struct rspamd_url *>(ltag->tag->extra)) {
+ /* For A that's URL */
+ auto *lua_url = static_cast<rspamd_lua_url *>(lua_newuserdata(L, sizeof(rspamd_lua_url)));
+ lua_url->url = std::get<struct rspamd_url *>(ltag->tag->extra);
+ rspamd_lua_setclass(L, "rspamd{url}", -1);
+ }
+ else {
+ /* Unknown extra ? */
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_style(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+
+ if (ltag) {
+ if (ltag->tag->block) {
+ lua_html_push_block(L, ltag->tag->block);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_attribute(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+ gsize slen;
+ const gchar *attr_name = luaL_checklstring(L, 2, &slen);
+
+ if (ltag && attr_name) {
+ auto maybe_attr = ltag->tag->find_component(
+ rspamd::html::html_component_from_string({attr_name, slen}));
+
+ if (maybe_attr) {
+ lua_pushlstring(L, maybe_attr->data(), maybe_attr->size());
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+void luaopen_html(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{html}", htmllib_m);
+ lua_pop(L, 1);
+ rspamd_lua_new_class(L, "rspamd{html_tag}", taglib_m);
+ lua_pop(L, 1);
+}
diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c
new file mode 100644
index 0000000..713082a
--- /dev/null
+++ b/src/lua/lua_http.c
@@ -0,0 +1,1270 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+#include "libserver/http/http_private.h"
+#include "libutil/upstream.h"
+#include "ref.h"
+#include "unix-std.h"
+#include "zlib.h"
+#include "utlist.h"
+
+/***
+ * @module rspamd_http
+ * Rspamd HTTP module represents HTTP asynchronous client available from LUA code.
+ * This module hides all complexity: DNS resolving, sessions management, zero-copy
+ * text transfers and so on under the hood.
+ * @example
+local rspamd_http = require "rspamd_http"
+
+local function symbol_callback(task)
+ local function http_callback(err_message, code, body, headers)
+ task:insert_result('SYMBOL', 1) -- task is available via closure
+ end
+
+ rspamd_http.request({
+ task=task,
+ url='http://example.com/data',
+ body=task:get_content(),
+ callback=http_callback,
+ headers={Header='Value', OtherHeader='Value'},
+ mime_type='text/plain',
+ })
+ end
+ */
+
+#define MAX_HEADERS_SIZE 8192
+
+static const gchar *M = "rspamd lua http";
+
+LUA_FUNCTION_DEF(http, request);
+
+static const struct luaL_reg httplib_m[] = {
+ LUA_INTERFACE_DEF(http, request),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+#define RSPAMD_LUA_HTTP_FLAG_TEXT (1 << 0)
+#define RSPAMD_LUA_HTTP_FLAG_NOVERIFY (1 << 1)
+#define RSPAMD_LUA_HTTP_FLAG_RESOLVED (1 << 2)
+#define RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE (1 << 3)
+#define RSPAMD_LUA_HTTP_FLAG_YIELDED (1 << 4)
+
+struct lua_http_cbdata {
+ struct rspamd_http_connection *conn;
+ struct rspamd_async_session *session;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_http_message *msg;
+ struct ev_loop *event_loop;
+ struct rspamd_config *cfg;
+ struct rspamd_task *task;
+ ev_tstamp timeout;
+ struct rspamd_cryptobox_keypair *local_kp;
+ struct rspamd_cryptobox_pubkey *peer_pk;
+ rspamd_inet_addr_t *addr;
+ gchar *mime_type;
+ gchar *host;
+ gchar *auth;
+ struct upstream *up;
+ const gchar *url;
+ gsize max_size;
+ gint flags;
+ gint fd;
+ gint cbref;
+ struct thread_entry *thread;
+ ref_entry_t ref;
+};
+
+static const gdouble default_http_timeout = 5.0;
+
+static struct rspamd_dns_resolver *
+lua_http_global_resolver(struct ev_loop *ev_base)
+{
+ static struct rspamd_dns_resolver *global_resolver;
+
+ if (global_resolver == NULL) {
+ global_resolver = rspamd_dns_resolver_init(NULL, ev_base, NULL);
+ }
+
+ return global_resolver;
+}
+
+static void
+lua_http_fin(gpointer arg)
+{
+ struct lua_http_cbdata *cbd = (struct lua_http_cbdata *) arg;
+
+ if (cbd->cbref != -1) {
+ luaL_unref(cbd->cfg->lua_state, LUA_REGISTRYINDEX, cbd->cbref);
+ }
+
+ if (cbd->conn) {
+ /* Here we already have a connection, so we need to unref it */
+ rspamd_http_connection_unref(cbd->conn);
+ }
+ else if (cbd->msg != NULL) {
+ /* We need to free message */
+ rspamd_http_message_unref(cbd->msg);
+ }
+
+ if (cbd->fd != -1) {
+ close(cbd->fd);
+ }
+
+ if (cbd->addr) {
+ rspamd_inet_address_free(cbd->addr);
+ }
+
+ if (cbd->up) {
+ rspamd_upstream_unref(cbd->up);
+ }
+
+ if (cbd->mime_type) {
+ g_free(cbd->mime_type);
+ }
+
+ if (cbd->auth) {
+ g_free(cbd->auth);
+ }
+
+ if (cbd->host) {
+ g_free(cbd->host);
+ }
+
+ if (cbd->local_kp) {
+ rspamd_keypair_unref(cbd->local_kp);
+ }
+
+ if (cbd->peer_pk) {
+ rspamd_pubkey_unref(cbd->peer_pk);
+ }
+
+ g_free(cbd);
+}
+
+static void
+lua_http_cbd_dtor(struct lua_http_cbdata *cbd)
+{
+ if (cbd->session) {
+
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_RESOLVED) {
+ /* Event is added merely for resolved events */
+ if (cbd->item) {
+ rspamd_symcache_item_async_dec_check(cbd->task, cbd->item, M);
+ }
+
+ rspamd_session_remove_event(cbd->session, lua_http_fin, cbd);
+ }
+ }
+ else {
+ lua_http_fin(cbd);
+ }
+}
+
+static void
+lua_http_push_error(struct lua_http_cbdata *cbd, const char *err)
+{
+ struct lua_callback_state lcbd;
+ lua_State *L;
+
+ lua_thread_pool_prepare_callback(cbd->cfg->lua_thread_pool, &lcbd);
+
+ L = lcbd.L;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+ lua_pushstring(L, err);
+
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 1, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ lua_pop(L, 1);
+ }
+
+ lua_thread_pool_restore_callback(&lcbd);
+}
+
+static void lua_http_resume_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg, const char *err);
+
+static void
+lua_http_error_handler(struct rspamd_http_connection *conn, GError *err)
+{
+ struct lua_http_cbdata *cbd = (struct lua_http_cbdata *) conn->ud;
+
+ if (cbd->up) {
+ rspamd_upstream_fail(cbd->up, false, err ? err->message : "unknown error");
+ }
+
+ if (cbd->cbref == -1) {
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_YIELDED) {
+ cbd->flags &= ~RSPAMD_LUA_HTTP_FLAG_YIELDED;
+ lua_http_resume_handler(conn, NULL, err->message);
+ }
+ else {
+ /* TODO: kill me please */
+ msg_info("lost HTTP error from %s in coroutines mess: %s",
+ rspamd_inet_address_to_string_pretty(cbd->addr),
+ err->message);
+ }
+ }
+ else {
+ lua_http_push_error(cbd, err->message);
+ }
+
+ REF_RELEASE(cbd);
+}
+
+static int
+lua_http_finish_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct lua_http_cbdata *cbd = (struct lua_http_cbdata *) conn->ud;
+ struct rspamd_http_header *h;
+ const gchar *body;
+ gsize body_len;
+
+ struct lua_callback_state lcbd;
+ lua_State *L;
+
+ if (cbd->cbref == -1) {
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_YIELDED) {
+ cbd->flags &= ~RSPAMD_LUA_HTTP_FLAG_YIELDED;
+ lua_http_resume_handler(conn, msg, NULL);
+ }
+ else {
+ /* TODO: kill me please */
+ msg_err("lost HTTP data from %s in coroutines mess",
+ rspamd_inet_address_to_string_pretty(cbd->addr));
+ }
+
+ REF_RELEASE(cbd);
+
+ return 0;
+ }
+ lua_thread_pool_prepare_callback(cbd->cfg->lua_thread_pool, &lcbd);
+
+ if (cbd->up) {
+ rspamd_upstream_ok(cbd->up);
+ }
+
+ L = lcbd.L;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+ /* Error */
+ lua_pushnil(L);
+ /* Reply code */
+ lua_pushinteger(L, msg->code);
+ /* Body */
+ body = rspamd_http_message_get_body(msg, &body_len);
+
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_TEXT) {
+ struct rspamd_lua_text *t;
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = body;
+ t->len = body_len;
+ t->flags = 0;
+ }
+ else {
+ if (body_len > 0) {
+ lua_pushlstring(L, body, body_len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ /* Headers */
+ lua_newtable(L);
+
+ kh_foreach_value(msg->headers, h, {
+ /*
+ * Lowercase header name, as Lua cannot search in caseless matter
+ */
+ rspamd_str_lc(h->combined->str, h->name.len);
+ lua_pushlstring(L, h->name.begin, h->name.len);
+ lua_pushlstring(L, h->value.begin, h->value.len);
+ lua_settable(L, -3);
+ });
+
+ if (cbd->item) {
+ /* Replace watcher to deal with nested calls */
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 4, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ lua_pop(L, 1);
+ }
+
+ REF_RELEASE(cbd);
+
+ lua_thread_pool_restore_callback(&lcbd);
+
+ return 0;
+}
+
+/*
+ * resumes yielded thread
+ */
+static void
+lua_http_resume_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg, const char *err)
+{
+ struct lua_http_cbdata *cbd = (struct lua_http_cbdata *) conn->ud;
+ lua_State *L = cbd->thread->lua_state;
+ const gchar *body;
+ gsize body_len;
+ struct rspamd_http_header *h;
+
+ if (err) {
+ lua_pushstring(L, err);
+ lua_pushnil(L);
+ }
+ else {
+ /*
+ * 1 - nil (error)
+ * 2 - table:
+ * code (int)
+ * content (string)
+ * headers (table: header -> value)
+ */
+ lua_pushnil(L);// error code
+
+ lua_createtable(L, 0, 3);
+
+ /* code */
+ lua_pushliteral(L, "code");
+ lua_pushinteger(L, msg->code);
+ lua_settable(L, -3);
+
+ /* content */
+ lua_pushliteral(L, "content");
+
+ body = rspamd_http_message_get_body(msg, &body_len);
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_TEXT) {
+ struct rspamd_lua_text *t;
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = body;
+ t->len = body_len;
+ t->flags = 0;
+ }
+ else {
+ if (body_len > 0) {
+ lua_pushlstring(L, body, body_len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ lua_settable(L, -3);
+
+ /* headers */
+ lua_pushliteral(L, "headers");
+ lua_newtable(L);
+
+ kh_foreach_value(msg->headers, h, {
+ /*
+ * Lowercase header name, as Lua cannot search in caseless matter
+ */
+ rspamd_str_lc(h->combined->str, h->name.len);
+ lua_pushlstring(L, h->name.begin, h->name.len);
+ lua_pushlstring(L, h->value.begin, h->value.len);
+ lua_settable(L, -3);
+ });
+
+ lua_settable(L, -3);
+ }
+
+ if (cbd->item) {
+ /* Replace watcher to deal with nested calls */
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ lua_thread_resume(cbd->thread, 2);
+}
+
+static gboolean
+lua_http_make_connection(struct lua_http_cbdata *cbd)
+{
+ rspamd_inet_address_set_port(cbd->addr, cbd->msg->port);
+ unsigned http_opts = RSPAMD_HTTP_CLIENT_SIMPLE;
+
+ if (cbd->msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL) {
+ http_opts |= RSPAMD_HTTP_CLIENT_SSL;
+ }
+
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE) {
+ cbd->fd = -1; /* FD is owned by keepalive connection */
+ cbd->conn = rspamd_http_connection_new_client_keepalive(
+ NULL, /* Default context */
+ NULL,
+ lua_http_error_handler,
+ lua_http_finish_handler,
+ http_opts,
+ cbd->addr,
+ cbd->host);
+ }
+ else {
+ cbd->fd = -1;
+ cbd->conn = rspamd_http_connection_new_client(
+ NULL, /* Default context */
+ NULL,
+ lua_http_error_handler,
+ lua_http_finish_handler,
+ http_opts,
+ cbd->addr);
+ }
+
+ if (cbd->conn) {
+ if (cbd->local_kp) {
+ rspamd_http_connection_set_key(cbd->conn, cbd->local_kp);
+ }
+
+ if (cbd->peer_pk) {
+ rspamd_http_message_set_peer_key(cbd->msg, cbd->peer_pk);
+ }
+
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_NOVERIFY) {
+ cbd->msg->flags |= RSPAMD_HTTP_FLAG_SSL_NOVERIFY;
+ }
+
+ if (cbd->max_size) {
+ rspamd_http_connection_set_max_size(cbd->conn, cbd->max_size);
+ }
+
+ if (cbd->auth) {
+ rspamd_http_message_add_header(cbd->msg, "Authorization",
+ cbd->auth);
+ }
+
+ if (cbd->session) {
+ if (cbd->item) {
+ rspamd_session_add_event_full(cbd->session,
+ (event_finalizer_t) lua_http_fin, cbd,
+ M,
+ rspamd_symcache_dyn_item_name(cbd->task, cbd->item));
+ }
+ else {
+ rspamd_session_add_event(cbd->session,
+ (event_finalizer_t) lua_http_fin, cbd,
+ M);
+ }
+ cbd->flags |= RSPAMD_LUA_HTTP_FLAG_RESOLVED;
+ }
+
+ if (cbd->task) {
+ cbd->conn->log_tag = cbd->task->task_pool->tag.uid;
+
+ if (cbd->item) {
+ rspamd_symcache_item_async_inc(cbd->task, cbd->item, M);
+ }
+ }
+ else if (cbd->cfg) {
+ cbd->conn->log_tag = cbd->cfg->cfg_pool->tag.uid;
+ }
+
+ struct rspamd_http_message *msg = cbd->msg;
+
+ /* Message is now owned by a connection object */
+ cbd->msg = NULL;
+
+ return rspamd_http_connection_write_message(cbd->conn, msg,
+ cbd->host, cbd->mime_type, cbd,
+ cbd->timeout);
+ }
+
+ return FALSE;
+}
+
+static void
+lua_http_dns_handler(struct rdns_reply *reply, gpointer ud)
+{
+ struct lua_http_cbdata *cbd = (struct lua_http_cbdata *) ud;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_task *task;
+
+ task = cbd->task;
+ item = cbd->item;
+
+ if (reply->code != RDNS_RC_NOERROR) {
+ lua_http_push_error(cbd, "unable to resolve host");
+ REF_RELEASE(cbd);
+ }
+ else {
+ struct rdns_reply_entry *entry;
+
+ DL_FOREACH(reply->entries, entry)
+ {
+ if (entry->type == RDNS_REQUEST_A) {
+ cbd->addr = rspamd_inet_address_new(AF_INET,
+ &entry->content.a.addr);
+ break;
+ }
+ else if (entry->type == RDNS_REQUEST_AAAA) {
+ cbd->addr = rspamd_inet_address_new(AF_INET6,
+ &entry->content.aaa.addr);
+ break;
+ }
+ }
+
+ if (cbd->addr == NULL) {
+ lua_http_push_error(cbd, "unable to resolve host: no records with such name");
+ REF_RELEASE(cbd);
+ }
+ else {
+ REF_RETAIN(cbd);
+ if (!lua_http_make_connection(cbd)) {
+ lua_http_push_error(cbd, "unable to make connection to the host");
+
+ if (cbd->ref.refcount > 1) {
+ REF_RELEASE(cbd);
+ }
+
+ REF_RELEASE(cbd);
+
+ return;
+ }
+ REF_RELEASE(cbd);
+ }
+ }
+
+ if (item) {
+ rspamd_symcache_item_async_dec_check(task, item, M);
+ }
+}
+
+static void
+lua_http_push_headers(lua_State *L, struct rspamd_http_message *msg)
+{
+ const char *name, *value;
+ gint i, sz;
+
+ lua_pushnil(L);
+ while (lua_next(L, -2) != 0) {
+
+ lua_pushvalue(L, -2);
+ name = lua_tostring(L, -1);
+ sz = rspamd_lua_table_size(L, -2);
+ if (sz != 0 && name != NULL) {
+ for (i = 1; i <= sz; i++) {
+ lua_rawgeti(L, -2, i);
+ value = lua_tostring(L, -1);
+ if (value != NULL) {
+ rspamd_http_message_add_header(msg, name, value);
+ }
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ value = lua_tostring(L, -2);
+ if (name != NULL && value != NULL) {
+ rspamd_http_message_add_header(msg, name, value);
+ }
+ }
+ lua_pop(L, 2);
+ }
+}
+
+/***
+ * @function rspamd_http.request({params...})
+ * This function creates HTTP request and accepts several parameters as a table using key=value syntax.
+ * Required params are:
+ *
+ * - `url`
+ * - `task`
+ *
+ * In taskless mode, instead of `task` required are:
+ *
+ * - `ev_base`
+ * - `config`
+ *
+ * @param {string} url specifies URL for a request in the standard URI form (e.g. 'http://example.com/path')
+ * @param {function} callback specifies callback function in format `function (err_message, code, body, headers)` that is called on HTTP request completion. if this parameter is missing, the function performs "pseudo-synchronous" call (see [Synchronous and Asynchronous API overview](/doc/lua/sync_async.html#API-example-http-module)
+ * @param {task} task if called from symbol handler it is generally a good idea to use the common task objects: event base, DNS resolver and events session
+ * @param {table} headers optional headers in form `[name='value', name='value']`
+ * @param {string} mime_type MIME type of the HTTP content (for example, `text/html`)
+ * @param {string/text} body full body content, can be opaque `rspamd{text}` to avoid data copying
+ * @param {number} timeout floating point request timeout value in seconds (default is 5.0 seconds)
+ * @param {resolver} resolver to perform DNS-requests. Usually got from either `task` or `config`
+ * @param {boolean} gzip if true, body of the requests will be compressed
+ * @param {boolean} no_ssl_verify disable SSL peer checks
+ * @param {boolean} keepalive enable keep-alive pool
+ * @param {string} user for HTTP authentication
+ * @param {string} password for HTTP authentication, only if "user" present
+ * @return {boolean} `true`, in **async** mode, if a request has been successfully scheduled. If this value is `false` then some error occurred, the callback thus will not be called.
+ * @return In **sync** mode `string|nil, nil|table` In sync mode error message if any and response as table: `int` _code_, `string` _content_ and `table` _headers_ (header -> value)
+ */
+static gint
+lua_http_request(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct ev_loop *ev_base;
+ struct rspamd_http_message *msg;
+ struct lua_http_cbdata *cbd;
+ struct rspamd_dns_resolver *resolver;
+ struct rspamd_async_session *session = NULL;
+ struct rspamd_lua_text *t;
+ struct rspamd_task *task = NULL;
+ struct rspamd_config *cfg = NULL;
+ struct rspamd_cryptobox_pubkey *peer_key = NULL;
+ struct rspamd_cryptobox_keypair *local_kp = NULL;
+ struct upstream *up = NULL;
+ const gchar *url, *lua_body;
+ rspamd_fstring_t *body = NULL;
+ gint cbref = -1;
+ gsize bodylen;
+ gdouble timeout = default_http_timeout;
+ gint flags = 0;
+ gchar *mime_type = NULL;
+ gchar *auth = NULL;
+ gsize max_size = 0;
+ gboolean gzip = FALSE;
+
+ if (lua_gettop(L) >= 2) {
+ /* url, callback and event_base format */
+ url = luaL_checkstring(L, 1);
+
+ if (url == NULL || lua_type(L, 2) != LUA_TFUNCTION) {
+ msg_err("http request has bad params");
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+
+ lua_pushvalue(L, 2);
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ if (lua_gettop(L) >= 3 && rspamd_lua_check_udata_maybe(L, 3, "rspamd{ev_base}")) {
+ ev_base = *(struct ev_loop **) lua_touserdata(L, 3);
+ }
+ else {
+ ev_base = NULL;
+ }
+
+ if (lua_gettop(L) >= 4 && rspamd_lua_check_udata_maybe(L, 4, "rspamd{resolver}")) {
+ resolver = *(struct rspamd_dns_resolver **) lua_touserdata(L, 4);
+ }
+ else {
+ resolver = lua_http_global_resolver(ev_base);
+ }
+
+ if (lua_gettop(L) >= 5 && rspamd_lua_check_udata_maybe(L, 5, "rspamd{session}")) {
+ session = *(struct rspamd_async_session **) lua_touserdata(L, 5);
+ }
+ else {
+ session = NULL;
+ }
+
+ msg = rspamd_http_message_from_url(url);
+
+ if (msg == NULL) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbref);
+ msg_err("cannot create HTTP message from url %s", url);
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+ }
+ else if (lua_type(L, 1) == LUA_TTABLE) {
+ lua_pushstring(L, "url");
+ lua_gettable(L, 1);
+ url = luaL_checkstring(L, -1);
+ lua_pop(L, 1);
+
+ if (url == NULL) {
+ msg_err("cannot create HTTP message without url");
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+
+ lua_pushstring(L, "callback");
+ lua_gettable(L, 1);
+ if (url == NULL || lua_type(L, -1) != LUA_TFUNCTION) {
+ lua_pop(L, 1);
+ }
+ else {
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+
+ lua_pushstring(L, "task");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ task = lua_check_task(L, -1);
+
+ if (task) {
+ ev_base = task->event_loop;
+ resolver = task->resolver;
+ session = task->s;
+ cfg = task->cfg;
+ }
+ }
+ lua_pop(L, 1);
+
+ if (task == NULL) {
+ lua_pushstring(L, "ev_base");
+ lua_gettable(L, 1);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{ev_base}")) {
+ ev_base = *(struct ev_loop **) lua_touserdata(L, -1);
+ }
+ else {
+ ev_base = NULL;
+ }
+ lua_pop(L, 1);
+
+
+ lua_pushstring(L, "session");
+ lua_gettable(L, 1);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{session}")) {
+ session = *(struct rspamd_async_session **) lua_touserdata(L, -1);
+ }
+ else {
+ session = NULL;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "config");
+ lua_gettable(L, 1);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{config}")) {
+ cfg = *(struct rspamd_config **) lua_touserdata(L, -1);
+ }
+ else {
+ cfg = NULL;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "resolver");
+ lua_gettable(L, 1);
+
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{resolver}")) {
+ resolver = *(struct rspamd_dns_resolver **) lua_touserdata(L, -1);
+ }
+ else {
+ if (cfg && cfg->dns_resolver) {
+ resolver = cfg->dns_resolver;
+ }
+ else {
+ resolver = lua_http_global_resolver(ev_base);
+ }
+ }
+ lua_pop(L, 1);
+ }
+
+ msg = rspamd_http_message_from_url(url);
+ if (msg == NULL) {
+ msg_err_task_check("cannot create HTTP message from url %s", url);
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+
+ lua_pushstring(L, "headers");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_http_push_headers(L, msg);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "mime_type");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ mime_type = g_strdup(lua_tostring(L, -1));
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "body");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ lua_body = lua_tolstring(L, -1, &bodylen);
+ body = rspamd_fstring_new_init(lua_body, bodylen);
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, -1);
+ /* TODO: think about zero-copy possibilities */
+ if (t) {
+ body = rspamd_fstring_new_init(t->start, t->len);
+ }
+ else {
+ rspamd_http_message_unref(msg);
+ g_free(mime_type);
+
+ return luaL_error(L, "invalid body argument type: %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ gsize total_len = 0, nelts = rspamd_lua_table_size(L, -1);
+
+ /* Calculate length and check types */
+ for (gsize i = 0; i < nelts; i++) {
+ lua_rawgeti(L, -1, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+#if LUA_VERSION_NUM >= 502
+ total_len += lua_rawlen(L, -1);
+#else
+ total_len += lua_objlen(L, -1);
+#endif
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ total_len += t->len;
+ }
+ else {
+ rspamd_http_message_unref(msg);
+ if (mime_type) {
+ g_free(mime_type);
+ }
+
+ return luaL_error(L, "invalid body argument: %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ }
+ else {
+ rspamd_http_message_unref(msg);
+ if (mime_type) {
+ g_free(mime_type);
+ }
+
+ return luaL_error(L, "invalid body argument type: %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+
+ lua_pop(L, 1);
+ }
+
+ /* Preallocate body */
+ if (total_len > 0) {
+ body = rspamd_fstring_sized_new(total_len);
+ }
+ else {
+ rspamd_http_message_unref(msg);
+ if (mime_type) {
+ g_free(mime_type);
+ }
+
+ return luaL_error(L, "empty body specified");
+ }
+
+ /* Fill elements */
+ for (gsize i = 0; i < nelts; i++) {
+ lua_rawgeti(L, -1, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ lua_body = lua_tolstring(L, -1, &bodylen);
+ body = rspamd_fstring_append(body, lua_body, bodylen);
+ }
+ else {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ body = rspamd_fstring_append(body, t->start, t->len);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ else if (lua_type(L, -1) != LUA_TNONE && lua_type(L, -1) != LUA_TNIL) {
+ rspamd_http_message_unref(msg);
+ return luaL_error(L, "invalid body argument type: %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "peer_key");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *in;
+ gsize inlen;
+
+ in = lua_tolstring(L, -1, &inlen);
+ peer_key = rspamd_pubkey_from_base32(in, inlen,
+ RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "keypair");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ ucl_object_t *kp_ucl = ucl_object_lua_import(L, -1);
+
+ local_kp = rspamd_keypair_from_ucl(kp_ucl);
+ ucl_object_unref(kp_ucl);
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "opaque_body");
+ lua_gettable(L, 1);
+
+ if (!!lua_toboolean(L, -1)) {
+ flags |= RSPAMD_LUA_HTTP_FLAG_TEXT;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "gzip");
+ lua_gettable(L, 1);
+
+ if (!!lua_toboolean(L, -1)) {
+ gzip = TRUE;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "no_ssl_verify");
+ lua_gettable(L, 1);
+
+ if (!!lua_toboolean(L, -1)) {
+ flags |= RSPAMD_LUA_HTTP_FLAG_NOVERIFY;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "keepalive");
+ lua_gettable(L, 1);
+
+ if (!!lua_toboolean(L, -1)) {
+ flags |= RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "max_size");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ max_size = lua_tointeger(L, -1);
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "method");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ rspamd_http_message_set_method(msg, lua_tostring(L, -1));
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "upstream");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_upstream *lup = lua_check_upstream(L, -1);
+
+ if (lup) {
+ /* Preserve pointer in case if lup is destructed */
+ up = lup->up;
+ }
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "user");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *user = lua_tostring(L, -1);
+
+ lua_pushstring(L, "password");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *password = lua_tostring(L, -1);
+ gchar *tmpbuf;
+ gsize tlen;
+
+ tlen = strlen(user) + strlen(password) + 1;
+ tmpbuf = g_malloc(tlen + 1);
+ rspamd_snprintf(tmpbuf, tlen + 1, "%s:%s", user, password);
+ tlen *= 2;
+ tlen += sizeof("Basic ") - 1;
+ auth = g_malloc(tlen + 1);
+ rspamd_snprintf(auth, tlen + 1, "Basic %Bs", tmpbuf);
+ g_free(tmpbuf);
+ }
+ else {
+ msg_warn("HTTP user must have password, disabling auth");
+ }
+
+ lua_pop(L, 1); /* password */
+ }
+
+ lua_pop(L, 1); /* username */
+ }
+ else {
+ msg_err("http request has bad params");
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+
+ if (session && rspamd_session_blocked(session)) {
+ lua_pushboolean(L, FALSE);
+
+ g_free(auth);
+ rspamd_http_message_unref(msg);
+ if (body) {
+ rspamd_fstring_free(body);
+ }
+ if (local_kp) {
+ rspamd_keypair_unref(local_kp);
+ }
+
+ return 1;
+ }
+ if (task == NULL && cfg == NULL) {
+ g_free(auth);
+ rspamd_http_message_unref(msg);
+ if (body) {
+ rspamd_fstring_free(body);
+ }
+ if (local_kp) {
+ rspamd_keypair_unref(local_kp);
+ }
+
+ return luaL_error(L,
+ "Bad params to rspamd_http:request(): either task or config should be set");
+ }
+
+ if (ev_base == NULL) {
+ g_free(auth);
+ rspamd_http_message_unref(msg);
+ if (body) {
+ rspamd_fstring_free(body);
+ }
+ if (local_kp) {
+ rspamd_keypair_unref(local_kp);
+ }
+
+ return luaL_error(L,
+ "Bad params to rspamd_http:request(): ev_base isn't passed");
+ }
+
+ cbd = g_malloc0(sizeof(*cbd));
+ cbd->cbref = cbref;
+ cbd->msg = msg;
+ cbd->event_loop = ev_base;
+ cbd->mime_type = mime_type;
+ cbd->timeout = timeout;
+ cbd->fd = -1;
+ cbd->cfg = cfg;
+ cbd->peer_pk = peer_key;
+ cbd->local_kp = local_kp;
+ cbd->flags = flags;
+ cbd->max_size = max_size;
+ cbd->url = url;
+ cbd->auth = auth;
+ cbd->task = task;
+
+ if (up) {
+ cbd->up = rspamd_upstream_ref(up);
+ }
+
+ if (cbd->cbref == -1) {
+ cbd->thread = lua_thread_pool_get_running_entry(cfg->lua_thread_pool);
+ }
+
+ REF_INIT_RETAIN(cbd, lua_http_cbd_dtor);
+
+ if (task) {
+ cbd->item = rspamd_symcache_get_cur_item(task);
+ }
+
+
+ if (body) {
+ if (gzip) {
+ if (rspamd_fstring_gzip(&body)) {
+ rspamd_http_message_add_header(msg, "Content-Encoding", "gzip");
+ }
+ }
+
+ rspamd_http_message_set_body_from_fstring_steal(msg, body);
+ }
+
+ if (session) {
+ cbd->session = session;
+ }
+
+ bool numeric_ip = false;
+
+ /* Check if we can skip resolving */
+
+ gsize hostlen = 0;
+ const gchar *host = rspamd_http_message_get_http_host(msg, &hostlen);
+
+ if (host) {
+ cbd->host = g_malloc(hostlen + 1);
+ rspamd_strlcpy(cbd->host, host, hostlen + 1);
+
+ /* Keep-alive entry is available */
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE) {
+ const rspamd_inet_addr_t *ka_addr = rspamd_http_context_has_keepalive(NULL,
+ cbd->host,
+ msg->port,
+ msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL);
+
+ if (ka_addr) {
+ cbd->addr = rspamd_inet_address_copy(ka_addr, NULL);
+ numeric_ip = true;
+ }
+ }
+
+ /*
+ * No keep-alive stuff, check if we have upstream or if we can parse host as
+ * a numeric address
+ */
+ if (!cbd->addr) {
+ if (cbd->up) {
+ numeric_ip = true;
+ cbd->addr = rspamd_inet_address_copy(rspamd_upstream_addr_next(cbd->up), NULL);
+ }
+ else {
+ /* We use msg->host here, not cbd->host ! */
+ if (rspamd_parse_inet_address(&cbd->addr,
+ msg->host->str, msg->host->len,
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ numeric_ip = true;
+ }
+ }
+ }
+ }
+ else {
+ if (cbd->up) {
+ numeric_ip = true;
+ cbd->addr = rspamd_inet_address_copy(rspamd_upstream_addr_next(cbd->up), NULL);
+ }
+ cbd->host = NULL;
+ }
+
+ if (numeric_ip) {
+ /* Host is numeric IP, no need to resolve */
+ gboolean ret;
+
+ REF_RETAIN(cbd);
+ ret = lua_http_make_connection(cbd);
+
+ if (!ret) {
+ if (cbd->up) {
+ rspamd_upstream_fail(cbd->up, true, "HTTP connection failed");
+ }
+ if (cbd->ref.refcount > 1) {
+ /* Not released by make_connection */
+ REF_RELEASE(cbd);
+ }
+
+ REF_RELEASE(cbd);
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+
+ REF_RELEASE(cbd);
+ }
+ else {
+ if (!cbd->host) {
+ REF_RELEASE(cbd);
+
+ return luaL_error(L, "no host has been specified");
+ }
+ if (task == NULL) {
+
+ REF_RETAIN(cbd);
+ if (!rspamd_dns_resolver_request(resolver, session, NULL, lua_http_dns_handler, cbd,
+ RDNS_REQUEST_A,
+ cbd->host)) {
+ if (cbd->ref.refcount > 1) {
+ /* Not released by make_connection */
+ REF_RELEASE(cbd);
+ }
+
+ REF_RELEASE(cbd);
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+
+ REF_RELEASE(cbd);
+ }
+ else {
+ REF_RETAIN(cbd);
+
+ if (!rspamd_dns_resolver_request_task_forced(task, lua_http_dns_handler, cbd,
+ RDNS_REQUEST_A, cbd->host)) {
+ if (cbd->ref.refcount > 1) {
+ /* Not released by make_connection */
+ REF_RELEASE(cbd);
+ }
+
+ REF_RELEASE(cbd);
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+ else if (cbd->item) {
+ rspamd_symcache_item_async_inc(cbd->task, cbd->item, M);
+ }
+
+ REF_RELEASE(cbd);
+ }
+ }
+
+ if (cbd->cbref == -1) {
+ cbd->thread = lua_thread_pool_get_running_entry(cfg->lua_thread_pool);
+ cbd->flags |= RSPAMD_LUA_HTTP_FLAG_YIELDED;
+
+ return lua_thread_yield(cbd->thread, 0);
+ }
+ else {
+ lua_pushboolean(L, TRUE);
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_http(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, httplib_m);
+
+ return 1;
+}
+
+void luaopen_http(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_http", lua_load_http);
+}
diff --git a/src/lua/lua_ip.c b/src/lua/lua_ip.c
new file mode 100644
index 0000000..ac24dc5
--- /dev/null
+++ b/src/lua/lua_ip.c
@@ -0,0 +1,637 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "libserver/maps/map_helpers.h"
+
+/***
+ * @module rspamd_ip
+ * `rspamd_ip` is a helper module to simplify IP addresses manipulations.
+ * @example
+local print_octets = function(ip)
+ print('Normal order octets:')
+ for _,o in ipairs(ip:str_octets()) do
+ print(o)
+ end
+ print('Reversed order octets:')
+ for _,o in ipairs(ip:inversed_str_octets()) do
+ print(o)
+ end
+ print('Numeric octets:')
+ for _,o in ipairs(ip:to_table()) do
+ print(o)
+ end
+end
+
+local rspamd_ip = require "rspamd_ip"
+-- Create ipv4
+local ip4 = rspamd_ip.from_string('127.0.0.1')
+-- Implicit conversion to string
+print(ip4)
+-- Numeric version
+print(ip4:get_version())
+print_octets(ip4)
+
+-- Create a sample ipv6 address
+local ip6 = rspamd_ip.from_string('2001:41d0:8:dd9a::100')
+print(ip6)
+print(ip6:get_version())
+print_octets(ip6)
+ */
+
+/***
+ * @method ip:to_string([pretty=false])
+ * Converts valid IP address to string
+ * @param {bool} pretty print IP address with port and braces (for IPv6)
+ * @return {string or nil} string representation of IP or `nil` if IP is invalid
+ */
+LUA_FUNCTION_DEF(ip, to_string);
+/***
+ * @method ip:to_number()
+ * Converts valid IP address to number or list of numbers in case of IPv6
+ * @return {integer(s) or nil} numeric representation of IP in *host* byte order or `nil` if IP is invalid
+ */
+LUA_FUNCTION_DEF(ip, to_number);
+
+/***
+ * @method ip:to_table()
+ * Converts valid IP address to the table of numeric octets
+ * @return {table or nil} numeric octets of IP address or `nil` if IP is invalid
+ * @example
+local ip = rspamd_ip.from_string('127.0.0.1')
+for _,o in ipairs(ip:to_table()) do
+ print(o)
+end
+-- Output:
+-- 127
+-- 0
+-- 0
+-- 1
+ */
+LUA_FUNCTION_DEF(ip, to_table);
+/***
+ * @method ip:str_octets()
+ * Converts valid IP address to the table of string octets. The difference from
+ * @see ip:to_table() is that this method returns just hex strings for ipv6
+ * addresses.
+ * @return {table or nil} string octets of IP address or `nil` if IP is invalid
+ * @example
+local ip = rspamd_ip.from_string('fe80::11')
+print(table.concat(ip:str_octets(), "."))
+-- Output:
+-- f.e.8.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.1.1
+ */
+LUA_FUNCTION_DEF(ip, str_octets);
+/***
+ * @method ip:inversed_str_octets()
+ * Converts valid IP address to the table of string octets in reversed order. The difference from
+ * @see ip:to_table() is that this method returns just hex strings for ipv6
+ * addresses in reversed order.
+ * @return {table or nil} string octets of IP address or `nil` if IP is invalid
+ * @example
+local ip = rspamd_ip.from_string('fe80::11')
+print(table.concat(ip:inversed_str_octets(), "."))
+-- Output:
+-- 1.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f
+ */
+LUA_FUNCTION_DEF(ip, inversed_str_octets);
+/***
+ * @function rspamd_ip.from_string(line)
+ * Create IP address from its string representation.
+ * @param {string} line valid IP address string (either ipv4 or ipv6)
+ * @return {ip} new ip object or `nil` if input is invalid
+ */
+LUA_FUNCTION_DEF(ip, from_string);
+/***
+ * @method ip:__gc()
+ * Automatically destroys IP object.
+ */
+LUA_FUNCTION_DEF(ip, destroy);
+/***
+ * @method ip:get_version()
+ * Gets numeric version of ip address
+ * @return {number} `4` for IPv4 and `6` for IPv6
+ */
+LUA_FUNCTION_DEF(ip, get_version);
+/***
+ * @method ip:is_valid()
+ * Checks if an IP object is a valid IP address.
+ * @return {boolean} `true` if IP is valid and `false` otherwise
+ */
+LUA_FUNCTION_DEF(ip, is_valid);
+/***
+ * @method ip:apply_mask(mask)
+ * Applies mask to IP address, resetting up to `mask` least significant bits to zero.
+ * @param {integer} mask how many bits to reset
+ * @return {ip} new IP object with `mask` bits reset
+ */
+LUA_FUNCTION_DEF(ip, apply_mask);
+/***
+ * @method ip:__eq(other)
+ * Compares two IP addresses
+ * @param {ip} other IP to compare
+ * @return {boolean} `true` if two objects are the same
+ */
+LUA_FUNCTION_DEF(ip, equal);
+/***
+ * @method ip:copy()
+ * Performs deep copy of IP address.
+ * @return {ip} a fresh copy of IP address
+ */
+LUA_FUNCTION_DEF(ip, copy);
+
+/**
+ * @method ip:get_port()
+ * Returns associated port for this IP address
+ * @return {number} port number or nil
+ */
+LUA_FUNCTION_DEF(ip, get_port);
+/***
+ * @method ip:is_local()
+ * Returns true if address is local one
+ * @return {boolean} `true` if address is local
+ */
+LUA_FUNCTION_DEF(ip, is_local);
+
+/***
+ * @method ip:less_than(other)
+ * Returns true if address is less than other
+ * @return {boolean}
+ */
+LUA_FUNCTION_DEF(ip, less_than);
+
+static const struct luaL_reg iplib_m[] = {
+ LUA_INTERFACE_DEF(ip, to_string),
+ LUA_INTERFACE_DEF(ip, to_table),
+ LUA_INTERFACE_DEF(ip, to_number),
+ LUA_INTERFACE_DEF(ip, str_octets),
+ LUA_INTERFACE_DEF(ip, inversed_str_octets),
+ LUA_INTERFACE_DEF(ip, get_version),
+ LUA_INTERFACE_DEF(ip, get_port),
+ LUA_INTERFACE_DEF(ip, is_valid),
+ LUA_INTERFACE_DEF(ip, apply_mask),
+ LUA_INTERFACE_DEF(ip, copy),
+ LUA_INTERFACE_DEF(ip, is_local),
+ {"tostring", lua_ip_to_string},
+ {"totable", lua_ip_to_table},
+ {"tonumber", lua_ip_to_number},
+ {"__tostring", lua_ip_to_string},
+ {"__eq", lua_ip_equal},
+ {"__gc", lua_ip_destroy},
+ {"__lt", lua_ip_less_than},
+ {NULL, NULL}};
+
+static const struct luaL_reg iplib_f[] = {
+ LUA_INTERFACE_DEF(ip, from_string),
+ {"fromstring", lua_ip_from_string},
+ {"fromip", lua_ip_copy},
+ {"from_ip", lua_ip_copy},
+ {NULL, NULL}};
+
+static struct rspamd_lua_ip *
+lua_ip_new(lua_State *L, struct rspamd_lua_ip *old)
+{
+ struct rspamd_lua_ip *ip, **pip;
+
+ ip = g_malloc0(sizeof(*ip));
+
+ if (old != NULL && old->addr != NULL) {
+ ip->addr = rspamd_inet_address_copy(old->addr, NULL);
+ }
+
+ pip = lua_newuserdata(L, sizeof(struct rspamd_lua_ip *));
+ rspamd_lua_setclass(L, "rspamd{ip}", -1);
+ *pip = ip;
+
+
+ return ip;
+}
+
+struct rspamd_lua_ip *
+lua_check_ip(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{ip}");
+
+ luaL_argcheck(L, ud != NULL, pos, "'ip' expected");
+ return ud ? *((struct rspamd_lua_ip **) ud) : NULL;
+}
+
+static gint
+lua_ip_to_table(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+ guint max, i;
+ guint8 *ptr;
+
+ if (ip != NULL && ip->addr) {
+ ptr = rspamd_inet_address_get_hash_key(ip->addr, &max);
+ lua_createtable(L, max, 0);
+
+ for (i = 1; i <= max; i++, ptr++) {
+ lua_pushinteger(L, *ptr);
+ lua_rawseti(L, -2, i);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_str_octets(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+ guint max, i;
+ guint8 *ptr;
+ gint af;
+ char numbuf[8];
+
+ if (ip != NULL && ip->addr) {
+ af = rspamd_inet_address_get_af(ip->addr);
+ ptr = rspamd_inet_address_get_hash_key(ip->addr, &max);
+ lua_createtable(L, max * 2, 0);
+
+ for (i = 1; i <= max; i++, ptr++) {
+ if (af == AF_INET) {
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%d", *ptr);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i);
+ }
+ else {
+ rspamd_snprintf(numbuf,
+ sizeof(numbuf),
+ "%xd",
+ (*ptr & 0xf0) >> 4);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i * 2 - 1);
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%xd", *ptr & 0x0f);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i * 2);
+ }
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_inversed_str_octets(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+ guint max, i;
+ guint8 *ptr;
+ char numbuf[4];
+ gint af;
+
+ if (ip != NULL && ip->addr) {
+ ptr = rspamd_inet_address_get_hash_key(ip->addr, &max);
+ af = rspamd_inet_address_get_af(ip->addr);
+ lua_createtable(L, max * 2, 0);
+
+ ptr += max - 1;
+ for (i = 1; i <= max; i++, ptr--) {
+ if (af == AF_INET) {
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%d", *ptr);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i);
+ }
+ else {
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%xd", *ptr & 0x0f);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i * 2 - 1);
+ rspamd_snprintf(numbuf,
+ sizeof(numbuf),
+ "%xd",
+ (*ptr & 0xf0) >> 4);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i * 2);
+ }
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_to_string(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip != NULL && ip->addr) {
+ if (lua_isboolean(L, 2) && lua_toboolean(L, 2) == true) {
+ lua_pushstring(L, rspamd_inet_address_to_string_pretty(ip->addr));
+ }
+ else {
+ lua_pushstring(L, rspamd_inet_address_to_string(ip->addr));
+ }
+ }
+ else {
+ luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_get_port(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip != NULL && ip->addr) {
+ lua_pushinteger(L, rspamd_inet_address_get_port(ip->addr));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_from_string(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip;
+ const gchar *ip_str;
+ gsize len;
+
+ ip_str = luaL_checklstring(L, 1, &len);
+ if (ip_str) {
+ ip = lua_ip_new(L, NULL);
+
+ if (!rspamd_parse_inet_address(&ip->addr,
+ ip_str, len, RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ msg_warn("cannot parse ip: %*s", (gint) len, ip_str);
+ ip->addr = NULL;
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_to_number(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+ guint32 c;
+ guint max, i;
+ guchar *ptr;
+
+ if (ip != NULL && ip->addr) {
+ ptr = rspamd_inet_address_get_hash_key(ip->addr, &max);
+
+ for (i = 0; i < max / sizeof(c); i++) {
+ memcpy(&c, ptr + i * sizeof(c), sizeof(c));
+ lua_pushinteger(L, ntohl(c));
+ }
+
+ return max / sizeof(c);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_ip_destroy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip) {
+ if (ip->addr) {
+ rspamd_inet_address_free(ip->addr);
+ }
+ g_free(ip);
+ }
+
+ return 0;
+}
+
+static gint
+lua_ip_get_version(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip && ip->addr) {
+ lua_pushinteger(L, rspamd_inet_address_get_af(ip->addr) == AF_INET6 ? 6 : 4);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_is_valid(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip) {
+ lua_pushboolean(L, ip->addr != NULL);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_apply_mask(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1), *nip;
+ gint mask;
+
+ mask = lua_tonumber(L, 2);
+ if (mask > 0 && ip != NULL && ip->addr) {
+ nip = lua_ip_new(L, ip);
+ rspamd_inet_address_apply_mask(nip->addr, mask);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_equal(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip1 = lua_check_ip(L, 1),
+ *ip2 = lua_check_ip(L, 2);
+ gboolean res = FALSE;
+
+ if (ip1 && ip2 && ip1->addr && ip2->addr) {
+ res = rspamd_inet_address_compare(ip1->addr, ip2->addr, TRUE) == 0;
+ }
+
+ lua_pushboolean(L, res);
+
+ return 1;
+}
+
+static gint
+lua_ip_copy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip) {
+ lua_ip_new(L, ip);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_is_local(lua_State *L)
+{
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+ gboolean check_laddrs = TRUE;
+
+ if (ip && ip->addr) {
+
+ if (lua_type(L, 2) == LUA_TBOOLEAN) {
+ check_laddrs = lua_toboolean(L, 2);
+ }
+
+ if (rspamd_inet_address_is_local(ip->addr)) {
+ lua_pushboolean(L, true);
+
+ return 1;
+ }
+ else if (check_laddrs) {
+ struct rspamd_radix_map_helper *local_addrs =
+ rspamd_inet_library_get_lib_ctx();
+ if (local_addrs) {
+ if (rspamd_match_radix_map_addr(local_addrs, ip->addr) != NULL) {
+ lua_pushboolean(L, true);
+
+ return 1;
+ }
+ }
+ }
+
+ lua_pushboolean(L, false);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_less_than(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1),
+ *other = lua_check_ip(L, 2);
+
+ if (ip && other) {
+ lua_pushboolean(L,
+ rspamd_inet_address_compare(ip->addr, other->addr, true) < 0);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+void rspamd_lua_ip_push(lua_State *L, rspamd_inet_addr_t *addr)
+{
+ struct rspamd_lua_ip *ip, **pip;
+
+ if (addr) {
+ ip = g_malloc0(sizeof(struct rspamd_lua_ip));
+ ip->addr = rspamd_inet_address_copy(addr, NULL);
+ pip = lua_newuserdata(L, sizeof(struct rspamd_lua_ip *));
+ rspamd_lua_setclass(L, "rspamd{ip}", -1);
+ *pip = ip;
+ }
+ else {
+ lua_pushnil(L);
+ }
+}
+
+void rspamd_lua_ip_push_fromstring(lua_State *L, const gchar *ip_str)
+{
+ struct rspamd_lua_ip *ip, **pip;
+
+ if (ip_str == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ ip = g_malloc0(sizeof(struct rspamd_lua_ip));
+
+ if (rspamd_parse_inet_address(&ip->addr,
+ ip_str, strlen(ip_str), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+
+ pip = lua_newuserdata(L, sizeof(struct rspamd_lua_ip *));
+ rspamd_lua_setclass(L, "rspamd{ip}", -1);
+ *pip = ip;
+ }
+ else {
+ g_free(ip);
+ lua_pushnil(L);
+ }
+ }
+}
+
+static gint
+lua_load_ip(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, iplib_f);
+
+ return 1;
+}
+
+void luaopen_ip(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{ip}", iplib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_ip", lua_load_ip);
+}
diff --git a/src/lua/lua_kann.c b/src/lua/lua_kann.c
new file mode 100644
index 0000000..e42fbfb
--- /dev/null
+++ b/src/lua/lua_kann.c
@@ -0,0 +1,1361 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "lua_tensor.h"
+#include "contrib/kann/kann.h"
+
+/***
+ * @module rspamd_kann
+ * `rspamd_kann` is a Lua interface to kann library
+ */
+
+#define KANN_NODE_CLASS "rspamd{kann_node}"
+#define KANN_NETWORK_CLASS "rspamd{kann}"
+
+/* Simple macros to define behaviour */
+#define KANN_LAYER_DEF(name) static int lua_kann_layer_##name(lua_State *L)
+#define KANN_LAYER_INTERFACE(name) \
+ { \
+ #name, lua_kann_layer_##name \
+ }
+
+#define KANN_TRANSFORM_DEF(name) static int lua_kann_transform_##name(lua_State *L)
+#define KANN_TRANSFORM_INTERFACE(name) \
+ { \
+ #name, lua_kann_transform_##name \
+ }
+
+#define KANN_LOSS_DEF(name) static int lua_kann_loss_##name(lua_State *L)
+#define KANN_LOSS_INTERFACE(name) \
+ { \
+ #name, lua_kann_loss_##name \
+ }
+
+#define KANN_NEW_DEF(name) static int lua_kann_new_##name(lua_State *L)
+#define KANN_NEW_INTERFACE(name) \
+ { \
+ #name, lua_kann_new_##name \
+ }
+
+
+/*
+ * Forwarded declarations
+ */
+static kad_node_t *lua_check_kann_node(lua_State *L, int pos);
+
+/* Layers */
+KANN_LAYER_DEF(input);
+KANN_LAYER_DEF(dense);
+KANN_LAYER_DEF(layernorm);
+KANN_LAYER_DEF(rnn);
+KANN_LAYER_DEF(lstm);
+KANN_LAYER_DEF(gru);
+KANN_LAYER_DEF(conv2d);
+KANN_LAYER_DEF(conv1d);
+KANN_LAYER_DEF(cost);
+
+static luaL_reg rspamd_kann_layers_f[] = {
+ KANN_LAYER_INTERFACE(input),
+ KANN_LAYER_INTERFACE(dense),
+ KANN_LAYER_INTERFACE(layernorm),
+ KANN_LAYER_INTERFACE(rnn),
+ KANN_LAYER_INTERFACE(lstm),
+ KANN_LAYER_INTERFACE(gru),
+ KANN_LAYER_INTERFACE(conv2d),
+ KANN_LAYER_INTERFACE(conv1d),
+ KANN_LAYER_INTERFACE(cost),
+ {NULL, NULL},
+};
+
+/* Transition and composition functions */
+
+/* General transform */
+KANN_TRANSFORM_DEF(add);
+KANN_TRANSFORM_DEF(sub);
+KANN_TRANSFORM_DEF(mul);
+KANN_TRANSFORM_DEF(cmul);
+KANN_TRANSFORM_DEF(matmul);
+
+KANN_TRANSFORM_DEF(square);
+KANN_TRANSFORM_DEF(sigm);
+KANN_TRANSFORM_DEF(tanh);
+KANN_TRANSFORM_DEF(relu);
+KANN_TRANSFORM_DEF(softmax);
+KANN_TRANSFORM_DEF(1minus);
+KANN_TRANSFORM_DEF(exp);
+KANN_TRANSFORM_DEF(log);
+KANN_TRANSFORM_DEF(sin);
+static luaL_reg rspamd_kann_transform_f[] = {
+ KANN_TRANSFORM_INTERFACE(add),
+ KANN_TRANSFORM_INTERFACE(sub),
+ KANN_TRANSFORM_INTERFACE(mul),
+ KANN_TRANSFORM_INTERFACE(cmul),
+ KANN_TRANSFORM_INTERFACE(matmul),
+
+ KANN_TRANSFORM_INTERFACE(square),
+ KANN_TRANSFORM_INTERFACE(sigm),
+ KANN_TRANSFORM_INTERFACE(tanh),
+ KANN_TRANSFORM_INTERFACE(relu),
+ KANN_TRANSFORM_INTERFACE(softmax),
+ KANN_TRANSFORM_INTERFACE(1minus),
+ KANN_TRANSFORM_INTERFACE(exp),
+ KANN_TRANSFORM_INTERFACE(log),
+ KANN_TRANSFORM_INTERFACE(sin),
+ {NULL, NULL},
+};
+
+/* Loss functions */
+KANN_LOSS_DEF(mse);
+KANN_LOSS_DEF(ce_multi);
+KANN_LOSS_DEF(ce_bin);
+KANN_LOSS_DEF(ce_bin_neg);
+KANN_LOSS_DEF(ce_multi_weighted);
+static luaL_reg rspamd_kann_loss_f[] = {
+ KANN_LOSS_INTERFACE(mse),
+ KANN_LOSS_INTERFACE(ce_multi),
+ KANN_LOSS_INTERFACE(ce_bin),
+ KANN_LOSS_INTERFACE(ce_bin_neg),
+ KANN_LOSS_INTERFACE(ce_multi_weighted),
+ {NULL, NULL},
+};
+
+/* Creation functions */
+KANN_NEW_DEF(leaf);
+KANN_NEW_DEF(scalar);
+KANN_NEW_DEF(weight);
+KANN_NEW_DEF(bias);
+KANN_NEW_DEF(weight_conv2d);
+KANN_NEW_DEF(weight_conv1d);
+KANN_NEW_DEF(kann);
+
+static luaL_reg rspamd_kann_new_f[] = {
+ KANN_NEW_INTERFACE(leaf),
+ KANN_NEW_INTERFACE(scalar),
+ KANN_NEW_INTERFACE(weight),
+ KANN_NEW_INTERFACE(bias),
+ KANN_NEW_INTERFACE(weight_conv2d),
+ KANN_NEW_INTERFACE(weight_conv1d),
+ KANN_NEW_INTERFACE(kann),
+ {NULL, NULL},
+};
+
+LUA_FUNCTION_DEF(kann, load);
+LUA_FUNCTION_DEF(kann, destroy);
+LUA_FUNCTION_DEF(kann, save);
+LUA_FUNCTION_DEF(kann, train1);
+LUA_FUNCTION_DEF(kann, apply1);
+
+static luaL_reg rspamd_kann_m[] = {
+ LUA_INTERFACE_DEF(kann, save),
+ LUA_INTERFACE_DEF(kann, train1),
+ LUA_INTERFACE_DEF(kann, apply1),
+ {"__gc", lua_kann_destroy},
+ {NULL, NULL},
+};
+
+static int
+rspamd_kann_table_to_flags(lua_State *L, int table_pos)
+{
+ int result = 0;
+
+ lua_pushvalue(L, table_pos);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ int fl = lua_tointeger(L, -1);
+
+ result |= fl;
+ }
+
+ lua_pop(L, 1);
+
+ return result;
+}
+
+static gint
+lua_load_kann(lua_State *L)
+{
+ lua_newtable(L);
+
+ /* Flags */
+ lua_pushstring(L, "flag");
+ lua_newtable(L);
+ lua_pushinteger(L, KANN_F_IN);
+ lua_setfield(L, -2, "in");
+ lua_pushinteger(L, KANN_F_COST);
+ lua_setfield(L, -2, "cost");
+ lua_pushinteger(L, KANN_F_OUT);
+ lua_setfield(L, -2, "out");
+ lua_pushinteger(L, KANN_F_TRUTH);
+ lua_setfield(L, -2, "truth");
+ lua_settable(L, -3);
+
+ /* Cost type */
+ lua_pushstring(L, "cost");
+ lua_newtable(L);
+ /* binary cross-entropy cost, used with sigmoid */
+ lua_pushinteger(L, KANN_C_CEB);
+ lua_setfield(L, -2, "ceb");
+ /* multi-class cross-entropy cost, used with softmax */
+ lua_pushinteger(L, KANN_C_CEM);
+ lua_setfield(L, -2, "cem");
+ /* binary cross-entropy-like cost, used with tanh */
+ lua_pushinteger(L, KANN_C_CEB_NEG);
+ lua_setfield(L, -2, "ceb_neg");
+ lua_pushinteger(L, KANN_C_MSE);
+ lua_setfield(L, -2, "mse");
+ lua_settable(L, -3);
+
+ /* RNN flag */
+ lua_pushstring(L, "rnn");
+ lua_newtable(L);
+ /* apply layer normalization */
+ lua_pushinteger(L, KANN_RNN_NORM);
+ lua_setfield(L, -2, "norm");
+ /* take the initial hidden values as variables */
+ lua_pushinteger(L, KANN_RNN_VAR_H0);
+ lua_setfield(L, -2, "var_h0");
+ lua_settable(L, -3);
+
+ /* Layers */
+ lua_pushstring(L, "layer");
+ lua_newtable(L);
+ luaL_register(L, NULL, rspamd_kann_layers_f);
+ lua_settable(L, -3);
+
+ /* Transforms */
+ lua_pushstring(L, "transform");
+ lua_newtable(L);
+ luaL_register(L, NULL, rspamd_kann_transform_f);
+ lua_settable(L, -3);
+
+ /* Cost */
+ lua_pushstring(L, "loss");
+ lua_newtable(L);
+ luaL_register(L, NULL, rspamd_kann_loss_f);
+ lua_settable(L, -3);
+
+ /* Create functions */
+ lua_pushstring(L, "new");
+ lua_newtable(L);
+ luaL_register(L, NULL, rspamd_kann_new_f);
+ lua_settable(L, -3);
+
+ /* Load ann from memory or file */
+ lua_pushstring(L, "load");
+ lua_pushcfunction(L, lua_kann_load);
+ lua_settable(L, -3);
+
+ return 1;
+}
+
+static kad_node_t *
+lua_check_kann_node(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, KANN_NODE_CLASS);
+ luaL_argcheck(L, ud != NULL, pos, "'kann_node' expected");
+ return ud ? *((kad_node_t **) ud) : NULL;
+}
+
+static kann_t *
+lua_check_kann(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, KANN_NETWORK_CLASS);
+ luaL_argcheck(L, ud != NULL, pos, "'kann' expected");
+ return ud ? *((kann_t **) ud) : NULL;
+}
+
+void luaopen_kann(lua_State *L)
+{
+ /* Metatables */
+ rspamd_lua_new_class(L, KANN_NODE_CLASS, NULL); /* TODO: add methods */
+ lua_pop(L, 1); /* No need in metatable... */
+ rspamd_lua_new_class(L, KANN_NETWORK_CLASS, rspamd_kann_m);
+ lua_pop(L, 1); /* No need in metatable... */
+ rspamd_lua_add_preload(L, "rspamd_kann", lua_load_kann);
+ lua_settop(L, 0);
+}
+
+/* Layers implementation */
+#define PUSH_KAD_NODE(n) \
+ do { \
+ kad_node_t **pt; \
+ pt = lua_newuserdata(L, sizeof(kad_node_t *)); \
+ *pt = (n); \
+ rspamd_lua_setclass(L, KANN_NODE_CLASS, -1); \
+ } while (0)
+
+#define PUSH_KAN_NETWORK(n) \
+ do { \
+ kann_t **pn; \
+ pn = lua_newuserdata(L, sizeof(kann_t *)); \
+ *pn = (n); \
+ rspamd_lua_setclass(L, KANN_NETWORK_CLASS, -1); \
+ } while (0)
+
+#define PROCESS_KAD_FLAGS(n, pos) \
+ do { \
+ int fl = 0; \
+ if (lua_type(L, (pos)) == LUA_TTABLE) { fl = rspamd_kann_table_to_flags(L, (pos)); } \
+ else if (lua_type(L, (pos)) == LUA_TNUMBER) { \
+ fl = lua_tointeger(L, (pos)); \
+ } \
+ (n)->ext_flag |= fl; \
+ } while (0)
+
+/***
+ * @function kann.layer.input(ninputs[, flags])
+ * Creates an input layer for ANN
+ * @param {int} ninputs number of inputs
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_input(lua_State *L)
+{
+ gint nnodes = luaL_checkinteger(L, 1);
+
+ if (nnodes > 0) {
+ kad_node_t *t;
+
+ t = kann_layer_input(nnodes);
+
+ PROCESS_KAD_FLAGS(t, 2);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, nnodes required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.dense(in, ninputs[, flags])
+ * Creates a dense layer (e.g. for hidden layer)
+ * @param {kann_node} in kann node
+ * @param {int} ninputs number of dense nodes
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_dense(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ gint nnodes = luaL_checkinteger(L, 2);
+
+ if (in != NULL && nnodes > 0) {
+ kad_node_t *t;
+
+ t = kann_layer_dense(in, nnodes);
+
+ PROCESS_KAD_FLAGS(t, 3);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input + nnodes required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.dropout(in, ratio[, flags])
+ * Creates a dropout layer
+ * @param {kann_node} in kann node
+ * @param {float} ratio drop ratio
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_layerdropout(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ double r = luaL_checknumber(L, 2);
+
+ if (in != NULL) {
+ kad_node_t *t;
+
+ t = kann_layer_dropout(in, r);
+
+ PROCESS_KAD_FLAGS(t, 3);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input + rate required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.dropout(in [, flags])
+ * Creates a normalisation layer
+ * @param {kann_node} in kann node
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_layernorm(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+
+ if (in != NULL) {
+ kad_node_t *t;
+
+ t = kann_layer_layernorm(in);
+
+ PROCESS_KAD_FLAGS(t, 2);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.rnn(in, nnodes[, rnn_flags, [, flags]])
+ * Creates a recursive NN layer
+ * @param {kann_node} in kann node
+ * @param {int} nnodes number of cells
+ * @param {int} rnnflags rnn flags
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_rnn(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ gint nnodes = luaL_checkinteger(L, 2);
+ gint rnnflags = 0;
+
+ if (in != NULL && nnodes > 0) {
+ kad_node_t *t;
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ rnnflags = lua_tointeger(L, 3);
+ }
+
+ t = kann_layer_rnn(in, nnodes, rnnflags);
+
+ PROCESS_KAD_FLAGS(t, 4);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input + nnodes required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.lstm(in, nnodes[, rnn_flags, [, flags]])
+ * Creates a recursive NN layer using LSTM cells
+ * @param {kann_node} in kann node
+ * @param {int} nnodes number of cells
+ * @param {int} rnnflags rnn flags
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_lstm(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ gint nnodes = luaL_checkinteger(L, 2);
+ gint rnnflags = 0;
+
+ if (in != NULL && nnodes > 0) {
+ kad_node_t *t;
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ rnnflags = lua_tointeger(L, 3);
+ }
+
+ t = kann_layer_lstm(in, nnodes, rnnflags);
+
+ PROCESS_KAD_FLAGS(t, 4);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input + nnodes required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.rnn(in, nnodes[, rnn_flags, [, flags]])
+ * Creates a recursive NN layer using GRU cells
+ * @param {kann_node} in kann node
+ * @param {int} nnodes number of cells
+ * @param {int} rnnflags rnn flags
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_gru(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ gint nnodes = luaL_checkinteger(L, 2);
+ gint rnnflags = 0;
+
+ if (in != NULL && nnodes > 0) {
+ kad_node_t *t;
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ rnnflags = lua_tointeger(L, 3);
+ }
+
+ t = kann_layer_gru(in, nnodes, rnnflags);
+
+ PROCESS_KAD_FLAGS(t, 4);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input + nnodes required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.conv2d(in, n_flt, k_rows, k_cols, stride_rows, stride_cols, pad_rows, pad_columns[, flags])
+ * Creates a 2D convolution layer
+ * @param {kann_node} in kann node
+ * @param {int} n_flt number of filters
+ * @param {int} k_rows kernel rows
+ * @param {int} k_cols kernel columns
+ * @param {int} stride_rows stride rows
+ * @param {int} stride_cols stride columns
+ * @param {int} pad_rows padding rows
+ * @param {int} pad_columns padding columns
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_conv2d(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ int n_flt = luaL_checkinteger(L, 2);
+ int k_rows = luaL_checkinteger(L, 3);
+ int k_cols = luaL_checkinteger(L, 4);
+ int stride_r = luaL_checkinteger(L, 5);
+ int stride_c = luaL_checkinteger(L, 6);
+ int pad_r = luaL_checkinteger(L, 7);
+ int pad_c = luaL_checkinteger(L, 8);
+
+ if (in != NULL) {
+ kad_node_t *t;
+ t = kann_layer_conv2d(in, n_flt, k_rows, k_cols, stride_r, stride_c,
+ pad_r, pad_c);
+
+ PROCESS_KAD_FLAGS(t, 9);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input, nflt, kx, ky, stridex, stridey, padx, pady are required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.conv1d(in, n_flt, kern_size, stride_size, pad_size[, flags])
+ * Creates 1D convolution layer
+ * @param {kann_node} in kann node
+ * @param {int} n_flt number of filters
+ * @param {int} kern_size kernel rows
+ * @param {int} stride_size stride rows
+ * @param {int} pad_size padding rows
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_conv1d(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ int n_flt = luaL_checkinteger(L, 2);
+ int k_size = luaL_checkinteger(L, 3);
+ int stride = luaL_checkinteger(L, 4);
+ int pad = luaL_checkinteger(L, 5);
+
+ if (in != NULL) {
+ kad_node_t *t;
+ t = kann_layer_conv1d(in, n_flt, k_size, stride, pad);
+
+ PROCESS_KAD_FLAGS(t, 6);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input, nflt, k, stride, pad required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.cost(in, nout, cost_type[, flags])
+ * Creates 1D convolution layer
+ * @param {kann_node} in kann node
+ * @param {int} nout number of outputs
+ * @param {int} cost_type see kann.cost table
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_cost(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ int nout = luaL_checkinteger(L, 2);
+ int cost_type = luaL_checkinteger(L, 3);
+
+ if (in != NULL && nout > 0) {
+ kad_node_t *t;
+ t = kann_layer_cost(in, nout, cost_type);
+
+ PROCESS_KAD_FLAGS(t, 4);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input, nout and cost_type are required");
+ }
+
+ return 1;
+}
+
+/* Generic helpers */
+static int
+lua_kann_call_unary_function(lua_State *L, const char *name,
+ kad_node_t *(*func)(kad_node_t *) )
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+
+ if (in != NULL) {
+ kad_node_t *t;
+ t = func(in);
+
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments for %s, input required", name);
+ }
+
+ return 1;
+}
+static int
+lua_kann_call_binary_function(lua_State *L, const char *name,
+ kad_node_t *(*func)(kad_node_t *, kad_node_t *) )
+{
+ kad_node_t *x = lua_check_kann_node(L, 1);
+ kad_node_t *y = lua_check_kann_node(L, 2);
+
+ if (x != NULL && y != NULL) {
+ kad_node_t *t;
+ t = func(x, y);
+
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments for %s, 2 inputs required", name);
+ }
+
+ return 1;
+}
+
+#define LUA_UNARY_TRANSFORM_FUNC_IMPL(name) \
+ static int lua_kann_transform_##name(lua_State *L) \
+ { \
+ return lua_kann_call_unary_function(L, #name, kad_##name); \
+ }
+
+#define LUA_BINARY_TRANSFORM_FUNC_IMPL(name) \
+ static int lua_kann_transform_##name(lua_State *L) \
+ { \
+ return lua_kann_call_binary_function(L, #name, kad_##name); \
+ }
+
+#define LUA_LOSS_FUNC_IMPL(name) \
+ static int lua_kann_loss_##name(lua_State *L) \
+ { \
+ return lua_kann_call_binary_function(L, #name, kad_##name); \
+ }
+
+/* Transform functions registered via macro helpers */
+LUA_BINARY_TRANSFORM_FUNC_IMPL(add)
+LUA_BINARY_TRANSFORM_FUNC_IMPL(sub)
+LUA_BINARY_TRANSFORM_FUNC_IMPL(mul)
+LUA_BINARY_TRANSFORM_FUNC_IMPL(cmul)
+LUA_BINARY_TRANSFORM_FUNC_IMPL(matmul)
+
+LUA_UNARY_TRANSFORM_FUNC_IMPL(square)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(sigm)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(tanh)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(relu)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(softmax)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(1minus)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(exp)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(log)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(sin)
+
+/* Generic cost functions */
+LUA_LOSS_FUNC_IMPL(mse)
+LUA_LOSS_FUNC_IMPL(ce_multi)
+LUA_LOSS_FUNC_IMPL(ce_bin)
+LUA_LOSS_FUNC_IMPL(ce_bin_neg)
+
+/* The only case of ternary weight function */
+static int
+lua_kann_loss_ce_multi_weighted(lua_State *L)
+{
+ kad_node_t *pred = lua_check_kann_node(L, 1);
+ kad_node_t *truth = lua_check_kann_node(L, 2);
+ kad_node_t *weight = lua_check_kann_node(L, 3);
+
+ if (pred != NULL && truth != NULL && weight != NULL) {
+ kad_node_t *t;
+ t = kad_ce_multi_weighted(pred, truth, weight);
+
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments for ce_multi_weighted, 3 inputs required");
+ }
+
+ return 1;
+}
+
+/* Creation functions */
+static int
+lua_kann_new_scalar(lua_State *L)
+{
+ gint flag = luaL_checkinteger(L, 1);
+ double x = luaL_checknumber(L, 2);
+ kad_node_t *t;
+
+ t = kann_new_scalar(flag, x);
+
+ PROCESS_KAD_FLAGS(t, 3);
+ PUSH_KAD_NODE(t);
+
+ return 1;
+}
+
+static int
+lua_kann_new_weight(lua_State *L)
+{
+ gint nrow = luaL_checkinteger(L, 1);
+ gint ncol = luaL_checkinteger(L, 2);
+ kad_node_t *t;
+
+ t = kann_new_weight(nrow, ncol);
+
+ PROCESS_KAD_FLAGS(t, 3);
+ PUSH_KAD_NODE(t);
+
+ return 1;
+}
+
+static int
+lua_kann_new_bias(lua_State *L)
+{
+ gint n = luaL_checkinteger(L, 1);
+ kad_node_t *t;
+
+ t = kann_new_bias(n);
+
+ PROCESS_KAD_FLAGS(t, 2);
+ PUSH_KAD_NODE(t);
+
+ return 1;
+}
+
+static int
+lua_kann_new_weight_conv2d(lua_State *L)
+{
+ gint nout = luaL_checkinteger(L, 1);
+ gint nin = luaL_checkinteger(L, 2);
+ gint krow = luaL_checkinteger(L, 3);
+ gint kcol = luaL_checkinteger(L, 4);
+ kad_node_t *t;
+
+ t = kann_new_weight_conv2d(nout, nin, krow, kcol);
+
+ PROCESS_KAD_FLAGS(t, 5);
+ PUSH_KAD_NODE(t);
+
+ return 1;
+}
+
+static int
+lua_kann_new_weight_conv1d(lua_State *L)
+{
+ gint nout = luaL_checkinteger(L, 1);
+ gint nin = luaL_checkinteger(L, 2);
+ gint klen = luaL_checkinteger(L, 3);
+ kad_node_t *t;
+
+ t = kann_new_weight_conv1d(nout, nin, klen);
+
+ PROCESS_KAD_FLAGS(t, 4);
+ PUSH_KAD_NODE(t);
+
+ return 1;
+}
+
+static int
+lua_kann_new_leaf(lua_State *L)
+{
+ int dim = luaL_checkinteger(L, 1), i, *ar;
+ kad_node_t *t;
+
+ if (dim >= 1 && dim < KAD_MAX_DIM && lua_istable(L, 2)) {
+ ar = g_new0(int, KAD_MAX_DIM);
+
+ for (i = 0; i < dim; i++) {
+ lua_rawgeti(L, 2, i + 1);
+ ar[i] = lua_tointeger(L, -1);
+ lua_pop(L, 1);
+ }
+
+ t = kann_new_leaf_array(NULL, NULL, 0, 0.0, dim, ar);
+
+ PROCESS_KAD_FLAGS(t, 3);
+ PUSH_KAD_NODE(t);
+
+ g_free(ar);
+ }
+ else {
+ return luaL_error(L, "invalid arguments for new.leaf, "
+ "dim and vector of elements are required");
+ }
+
+ return 1;
+}
+
+static int
+lua_kann_new_kann(lua_State *L)
+{
+ kad_node_t *cost = lua_check_kann_node(L, 1);
+ kann_t *k;
+
+ if (cost) {
+ k = kann_new(cost, 0);
+
+ PUSH_KAN_NETWORK(k);
+ }
+ else {
+ return luaL_error(L, "invalid arguments for new.kann, "
+ "cost node is required");
+ }
+
+ return 1;
+}
+
+static int
+lua_kann_destroy(lua_State *L)
+{
+ kann_t *k = lua_check_kann(L, 1);
+
+ kann_delete(k);
+
+ return 0;
+}
+
+static int
+lua_kann_save(lua_State *L)
+{
+ kann_t *k = lua_check_kann(L, 1);
+
+ if (k) {
+ if (lua_istable(L, 2)) {
+ lua_getfield(L, 2, "filename");
+
+ if (lua_isstring(L, -1)) {
+ const gchar *fname = lua_tostring(L, -1);
+ FILE *f;
+
+ f = fopen(fname, "w");
+
+ if (!f) {
+ lua_pop(L, 1);
+
+ return luaL_error(L, "cannot open %s for writing: %s",
+ fname, strerror(errno));
+ }
+
+ kann_save_fp(f, k);
+ fclose(f);
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pop(L, 1);
+
+ return luaL_error(L, "invalid arguments: missing filename");
+ }
+
+ lua_pop(L, 1);
+ }
+ else {
+ /* Save to Rspamd text */
+#ifndef HAVE_OPENMEMSTREAM
+ return luaL_error(L, "no support of saving to memory on your system");
+#endif
+ FILE *f;
+ char *buf = NULL;
+ size_t buflen;
+ struct rspamd_lua_text *t;
+
+ f = open_memstream(&buf, &buflen);
+ g_assert(f != NULL);
+
+ kann_save_fp(f, k);
+ fclose(f);
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ t->start = (const gchar *) buf;
+ t->len = buflen;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_kann_load(lua_State *L)
+{
+ kann_t *k;
+ FILE *f = NULL;
+
+ if (lua_istable(L, 1)) {
+ lua_getfield(L, 2, "filename");
+
+ if (lua_isstring(L, -1)) {
+ const gchar *fname = lua_tostring(L, -1);
+
+ f = fopen(fname, "rb");
+ }
+ else {
+ lua_pop(L, 1);
+
+ return luaL_error(L, "invalid arguments: missing filename");
+ }
+
+ lua_pop(L, 1);
+ }
+ else if (lua_isstring(L, 1)) {
+ gsize dlen;
+ const gchar *data;
+
+ data = lua_tolstring(L, 1, &dlen);
+
+#ifndef HAVE_FMEMOPEN
+ return luaL_error(L, "no support of loading from memory on your system");
+#endif
+ f = fmemopen((void *) data, dlen, "rb");
+ }
+ else if (lua_isuserdata(L, 1)) {
+ struct rspamd_lua_text *t;
+
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+#ifndef HAVE_FMEMOPEN
+ return luaL_error(L, "no support of loading from memory on your system");
+#endif
+ f = fmemopen((void *) t->start, t->len, "rb");
+ }
+
+ if (f == NULL) {
+ return luaL_error(L, "invalid arguments or cannot open file");
+ }
+
+ k = kann_load_fp(f);
+ fclose(f);
+
+ if (k == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ PUSH_KAN_NETWORK(k);
+ }
+
+ return 1;
+}
+
+struct rspamd_kann_train_cbdata {
+ lua_State *L;
+ kann_t *k;
+ gint cbref;
+};
+
+static void
+lua_kann_train_cb(int iter, float train_cost, float val_cost, void *ud)
+{
+ struct rspamd_kann_train_cbdata *cbd = (struct rspamd_kann_train_cbdata *) ud;
+
+ if (cbd->cbref != -1) {
+ gint err_idx;
+ lua_State *L = cbd->L;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+ lua_pushinteger(L, iter);
+ lua_pushnumber(L, train_cost);
+ lua_pushnumber(L, val_cost);
+
+ if (lua_pcall(L, 3, 0, err_idx) != 0) {
+ msg_err("cannot run lua train callback: %s",
+ lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+ }
+}
+
+#define FREE_VEC(a, n) \
+ do { \
+ for (int i = 0; i < (n); i++) g_free((a)[i]); \
+ g_free(a); \
+ } while (0)
+
+static int
+lua_kann_train1(lua_State *L)
+{
+ kann_t *k = lua_check_kann(L, 1);
+ struct rspamd_lua_tensor *pca = NULL;
+
+ /* Default train params */
+ double lr = 0.001;
+ gint64 mini_size = 64;
+ gint64 max_epoch = 25;
+ gint64 max_drop_streak = 10;
+ double frac_val = 0.1;
+ gint cbref = -1;
+
+ if (k && lua_istable(L, 2) && lua_istable(L, 3)) {
+ int n = rspamd_lua_table_size(L, 2);
+ int n_in = kann_dim_in(k);
+ int n_out = kann_dim_out(k);
+
+ if (n_in <= 0) {
+ return luaL_error(L, "invalid inputs count: %d", n_in);
+ }
+
+ if (n_out <= 0) {
+ return luaL_error(L, "invalid outputs count: %d", n_out);
+ }
+
+ if (n != rspamd_lua_table_size(L, 3) || n == 0) {
+ return luaL_error(L, "invalid dimensions: outputs size must be "
+ "equal to inputs and non zero");
+ }
+
+ if (lua_istable(L, 4)) {
+ GError *err = NULL;
+
+ if (!rspamd_lua_parse_table_arguments(L, 4, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING,
+ "lr=N;mini_size=I;max_epoch=I;max_drop_streak=I;frac_val=N;cb=F;pca=u{tensor}",
+ &lr, &mini_size, &max_epoch, &max_drop_streak, &frac_val, &cbref, &pca)) {
+ n = luaL_error(L, "invalid params: %s",
+ err ? err->message : "unknown error");
+ g_error_free(err);
+
+ return n;
+ }
+ }
+
+ if (pca) {
+ /* Check pca matrix validity */
+ if (pca->ndims != 2) {
+ return luaL_error(L, "invalid pca tensor: matrix expected, got a row");
+ }
+
+ if (pca->dim[0] != n_in) {
+ return luaL_error(L, "invalid pca tensor: "
+ "matrix must have %d rows and it has %d rows instead",
+ n_in, pca->dim[0]);
+ }
+ }
+
+ float **x, **y, *tmp_row = NULL;
+
+ /* Fill vectors row by row */
+ x = (float **) g_malloc0(sizeof(float *) * n);
+ y = (float **) g_malloc0(sizeof(float *) * n);
+
+ if (pca) {
+ tmp_row = g_malloc(sizeof(float) * pca->dim[1]);
+ }
+
+ for (int s = 0; s < n; s++) {
+ /* Inputs */
+ lua_rawgeti(L, 2, s + 1);
+ x[s] = (float *) g_malloc(sizeof(float) * n_in);
+
+ if (pca == NULL) {
+ if (rspamd_lua_table_size(L, -1) != n_in) {
+ FREE_VEC(x, n);
+ FREE_VEC(y, n);
+
+ n = luaL_error(L, "invalid params at pos %d: "
+ "bad input dimension %d; %d expected",
+ s + 1,
+ (int) rspamd_lua_table_size(L, -1),
+ n_in);
+ lua_pop(L, 1);
+
+ return n;
+ }
+
+ for (int i = 0; i < n_in; i++) {
+ lua_rawgeti(L, -1, i + 1);
+ x[s][i] = lua_tonumber(L, -1);
+
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ if (rspamd_lua_table_size(L, -1) != pca->dim[1]) {
+ FREE_VEC(x, n);
+ FREE_VEC(y, n);
+ g_free(tmp_row);
+
+ n = luaL_error(L, "(pca on) invalid params at pos %d: "
+ "bad input dimension %d; %d expected",
+ s + 1,
+ (int) rspamd_lua_table_size(L, -1),
+ pca->dim[1]);
+ lua_pop(L, 1);
+
+ return n;
+ }
+
+
+ for (int i = 0; i < pca->dim[1]; i++) {
+ lua_rawgeti(L, -1, i + 1);
+ tmp_row[i] = lua_tonumber(L, -1);
+
+ lua_pop(L, 1);
+ }
+
+ kad_sgemm_simple(0, 1, 1, n_in,
+ pca->dim[1], tmp_row, pca->data,
+ x[s]);
+ }
+
+ lua_pop(L, 1);
+
+ /* Outputs */
+ y[s] = (float *) g_malloc(sizeof(float) * n_out);
+ lua_rawgeti(L, 3, s + 1);
+
+ if (rspamd_lua_table_size(L, -1) != n_out) {
+ FREE_VEC(x, n);
+ FREE_VEC(y, n);
+ g_free(tmp_row);
+
+ n = luaL_error(L, "invalid params at pos %d: "
+ "bad output dimension %d; "
+ "%d expected",
+ s + 1,
+ (int) rspamd_lua_table_size(L, -1),
+ n_out);
+ lua_pop(L, 1);
+
+ return n;
+ }
+
+ for (int i = 0; i < n_out; i++) {
+ lua_rawgeti(L, -1, i + 1);
+ y[s][i] = lua_tonumber(L, -1);
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ struct rspamd_kann_train_cbdata cbd;
+
+ cbd.cbref = cbref;
+ cbd.k = k;
+ cbd.L = L;
+
+ int niters = kann_train_fnn1(k, lr,
+ mini_size, max_epoch, max_drop_streak,
+ frac_val, n, x, y, lua_kann_train_cb, &cbd);
+
+ lua_pushinteger(L, niters);
+
+ FREE_VEC(x, n);
+ FREE_VEC(y, n);
+ g_free(tmp_row);
+ }
+ else {
+ return luaL_error(L, "invalid arguments: kann, inputs, outputs and"
+ " optional params are expected");
+ }
+
+ return 1;
+}
+
+static int
+lua_kann_apply1(lua_State *L)
+{
+ kann_t *k = lua_check_kann(L, 1);
+ struct rspamd_lua_tensor *pca = NULL;
+
+ if (k) {
+ if (lua_istable(L, 2)) {
+ gsize vec_len = rspamd_lua_table_size(L, 2);
+ float *vec = (float *) g_malloc(sizeof(float) * vec_len),
+ *pca_out = NULL;
+ int i_out;
+ int n_in = kann_dim_in(k);
+
+ if (n_in <= 0) {
+ g_free(vec);
+ return luaL_error(L, "invalid inputs count: %d", n_in);
+ }
+
+ if (lua_isuserdata(L, 3)) {
+ pca = lua_check_tensor(L, 3);
+
+ if (pca) {
+ if (pca->ndims != 2) {
+ g_free(vec);
+ return luaL_error(L, "invalid pca tensor: matrix expected, got a row");
+ }
+
+ if (pca->dim[0] != n_in) {
+ g_free(vec);
+ return luaL_error(L, "invalid pca tensor: "
+ "matrix must have %d rows and it has %d rows instead",
+ n_in, pca->dim[0]);
+ }
+ }
+ else {
+ g_free(vec);
+ return luaL_error(L, "invalid params: pca matrix expected");
+ }
+ }
+ else {
+ if (n_in != vec_len) {
+ g_free(vec);
+ return luaL_error(L, "invalid params: bad input dimension %d; %d expected",
+ (int) vec_len, n_in);
+ }
+ }
+
+ for (gsize i = 0; i < vec_len; i++) {
+ lua_rawgeti(L, 2, i + 1);
+ vec[i] = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+ }
+
+ i_out = kann_find(k, KANN_F_OUT, 0);
+
+ if (i_out <= 0) {
+ g_free(vec);
+ return luaL_error(L, "invalid ANN: output layer is missing or is "
+ "at the input pos");
+ }
+
+ kann_set_batch_size(k, 1);
+ if (pca) {
+ pca_out = g_malloc(sizeof(float) * n_in);
+
+ kad_sgemm_simple(0, 1, 1, n_in,
+ vec_len, vec, pca->data,
+ pca_out);
+
+ kann_feed_bind(k, KANN_F_IN, 0, &pca_out);
+ }
+ else {
+ kann_feed_bind(k, KANN_F_IN, 0, &vec);
+ }
+
+ kad_eval_at(k->n, k->v, i_out);
+
+ gsize outlen = kad_len(k->v[i_out]);
+ lua_createtable(L, outlen, 0);
+
+ for (gsize i = 0; i < outlen; i++) {
+ lua_pushnumber(L, k->v[i_out]->x[i]);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ g_free(vec);
+ g_free(pca_out);
+ }
+ else if (lua_isuserdata(L, 2)) {
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 2);
+
+ if (t && t->ndims == 1) {
+ int i_out;
+ int n_in = kann_dim_in(k);
+
+ if (n_in != t->dim[0]) {
+ return luaL_error(L, "invalid params: bad input dimension %d; %d expected",
+ (int) t->dim[0], n_in);
+ }
+
+ i_out = kann_find(k, KANN_F_OUT, 0);
+
+ if (i_out <= 0) {
+ return luaL_error(L, "invalid ANN: output layer is missing or is "
+ "at the input pos");
+ }
+
+ kann_set_batch_size(k, 1);
+ kann_feed_bind(k, KANN_F_IN, 0, &t->data);
+ kad_eval_at(k->n, k->v, i_out);
+
+ gint outlen = kad_len(k->v[i_out]);
+ struct rspamd_lua_tensor *out;
+ out = lua_newtensor(L, 1, &outlen, false, false);
+ /* Ensure that kann and tensor have the same understanding of floats */
+ G_STATIC_ASSERT(sizeof(float) == sizeof(rspamd_tensor_num_t));
+ memcpy(out->data, k->v[i_out]->x, outlen * sizeof(float));
+ }
+ else {
+ return luaL_error(L, "invalid arguments: 1D rspamd{tensor} expected");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments: 1D rspamd{tensor} expected");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments: rspamd{kann} expected");
+ }
+
+ return 1;
+} \ No newline at end of file
diff --git a/src/lua/lua_logger.c b/src/lua/lua_logger.c
new file mode 100644
index 0000000..f4f8f3d
--- /dev/null
+++ b/src/lua/lua_logger.c
@@ -0,0 +1,1068 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_private.h"
+
+/***
+ * @module rspamd_logger
+ * Rspamd logger module is used to log messages from LUA API to the main rspamd logger.
+ * It supports legacy and modern interfaces allowing highly customized an convenient log functions.
+ * Here is an example of logger usage:
+ * @example
+local rspamd_logger = require "rspamd_logger"
+
+local a = 'string'
+local b = 1.5
+local c = 1
+local d = {
+ 'aa',
+ 1,
+ 'bb'
+}
+local e = {
+ key = 'value',
+ key2 = 1.0
+}
+
+-- New extended interface
+-- %<number> means numeric arguments and %s means the next argument
+-- for example %1, %2, %s: %s would mean the third argument
+
+rspamd_logger.info('a=%1, b=%2, c=%3, d=%4, e=%s', a, b, c, d, e)
+-- Output: a=string, b=1.50000, c=1, d={[1] = aa, [2] = 1, [3] = bb} e={[key]=value, [key2]=1.0}
+
+-- Create string using logger API
+local str = rspamd_logger.slog('a=%1, b=%2, c=%3, d=%4, e=%5', a, b, c, d, e)
+
+print(str)
+-- Output: a=string, b=1.50000, c=1, d={[1] = aa, [2] = 1, [3] = bb} e={[key]=value, [key2]=1.0}
+ */
+
+/* Logger methods */
+/***
+ * @function logger.err(msg)
+ * Log message as an error
+ * @param {string} msg string to be logged
+ */
+LUA_FUNCTION_DEF(logger, err);
+/***
+ * @function logger.warn(msg)
+ * Log message as a warning
+ * @param {string} msg string to be logged
+ */
+LUA_FUNCTION_DEF(logger, warn);
+/***
+ * @function logger.info(msg)
+ * Log message as an informational message
+ * @param {string} msg string to be logged
+ */
+LUA_FUNCTION_DEF(logger, info);
+/***
+ * @function logger.message(msg)
+ * Log message as an notice message
+ * @param {string} msg string to be logged
+ */
+LUA_FUNCTION_DEF(logger, message);
+/***
+ * @function logger.debug(msg)
+ * Log message as a debug message
+ * @param {string} msg string to be logged
+ */
+LUA_FUNCTION_DEF(logger, debug);
+/***
+ * @function logger.errx(fmt[, args)
+ * Extended interface to make an error log message
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, errx);
+/***
+ * @function logger.warn(fmt[, args)
+ * Extended interface to make a warning log message
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, warnx);
+/***
+ * @function logger.infox(fmt[, args)
+ * Extended interface to make an informational log message
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, infox);
+/***
+ * @function logger.infox(fmt[, args)
+ * Extended interface to make an informational log message
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, messagex);
+/***
+ * @function logger.debugx(fmt[, args)
+ * Extended interface to make a debug log message
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, debugx);
+
+/***
+ * @function logger.debugm(module, id, fmt[, args)
+ * Extended interface to make a debug log message
+ * @param {string} module debug module
+ * @param {task|cfg|pool|string} id id to log
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, debugm);
+/***
+ * @function logger.slog(fmt[, args)
+ * Create string replacing percent params with corresponding arguments
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ * @return {string} string with percent parameters substituted
+ */
+LUA_FUNCTION_DEF(logger, slog);
+
+/***
+ * @function logger.logx(level, module, id, fmt[, args)
+ * Extended interface to make a generic log message on any level
+ * @param {number} log level as a number (see GLogLevelFlags enum for values)
+ * @param {task|cfg|pool|string} id id to log
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, logx);
+
+/***
+ * @function logger.log_level()
+ * Returns log level for a logger
+ * @return {string} current log level
+ */
+LUA_FUNCTION_DEF(logger, log_level);
+
+static const struct luaL_reg loggerlib_f[] = {
+ LUA_INTERFACE_DEF(logger, err),
+ LUA_INTERFACE_DEF(logger, warn),
+ LUA_INTERFACE_DEF(logger, message),
+ {"msg", lua_logger_message},
+ LUA_INTERFACE_DEF(logger, info),
+ LUA_INTERFACE_DEF(logger, debug),
+ LUA_INTERFACE_DEF(logger, errx),
+ LUA_INTERFACE_DEF(logger, warnx),
+ LUA_INTERFACE_DEF(logger, infox),
+ LUA_INTERFACE_DEF(logger, messagex),
+ {"msgx", lua_logger_messagex},
+ LUA_INTERFACE_DEF(logger, debugx),
+ LUA_INTERFACE_DEF(logger, debugm),
+ LUA_INTERFACE_DEF(logger, slog),
+ LUA_INTERFACE_DEF(logger, logx),
+ LUA_INTERFACE_DEF(logger, log_level),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static void
+lua_common_log_line(GLogLevelFlags level,
+ lua_State *L,
+ const gchar *msg,
+ const gchar *uid,
+ const gchar *module,
+ gint stack_level)
+{
+ lua_Debug d;
+ gchar func_buf[128], *p;
+
+ if (lua_getstack(L, stack_level, &d) == 1) {
+ (void) lua_getinfo(L, "Sl", &d);
+ if ((p = strrchr(d.short_src, '/')) == NULL) {
+ p = d.short_src;
+ }
+ else {
+ p++;
+ }
+
+ if (strlen(p) > 30) {
+ rspamd_snprintf(func_buf, sizeof(func_buf), "%27s...:%d", p,
+ d.currentline);
+ }
+ else {
+ rspamd_snprintf(func_buf, sizeof(func_buf), "%s:%d", p,
+ d.currentline);
+ }
+
+ rspamd_common_log_function(NULL,
+ level,
+ module,
+ uid,
+ func_buf,
+ "%s",
+ msg);
+ }
+ else {
+ rspamd_common_log_function(NULL,
+ level,
+ module,
+ uid,
+ G_STRFUNC,
+ "%s",
+ msg);
+ }
+}
+
+/*** Logger interface ***/
+static gint
+lua_logger_err(lua_State *L)
+{
+ return lua_logger_errx(L);
+}
+
+static gint
+lua_logger_warn(lua_State *L)
+{
+ return lua_logger_warnx(L);
+}
+
+static gint
+lua_logger_info(lua_State *L)
+{
+ return lua_logger_infox(L);
+}
+
+static gint
+lua_logger_message(lua_State *L)
+{
+ return lua_logger_messagex(L);
+}
+
+static gint
+lua_logger_debug(lua_State *L)
+{
+ return lua_logger_debugx(L);
+}
+
+static inline bool
+lua_logger_char_safe(int t, unsigned int esc_type)
+{
+ if (t & 0x80) {
+ if (esc_type & LUA_ESCAPE_8BIT) {
+ return false;
+ }
+
+ return true;
+ }
+
+ if (esc_type & LUA_ESCAPE_UNPRINTABLE) {
+ if (!g_ascii_isprint(t) && !g_ascii_isspace(t)) {
+ return false;
+ }
+ }
+
+ if (esc_type & LUA_ESCAPE_NEWLINES) {
+ if (t == '\r' || t == '\n') {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static gsize
+lua_logger_out_str(lua_State *L, gint pos,
+ gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace,
+ enum lua_logger_escape_type esc_type)
+{
+ gsize slen, flen;
+ const gchar *str = lua_tolstring(L, pos, &slen);
+ static const gchar hexdigests[16] = "0123456789abcdef";
+ gsize r = 0, s;
+
+ if (str) {
+ gboolean normal = TRUE;
+ flen = MIN(slen, len - 1);
+
+ for (r = 0; r < flen; r++) {
+ if (!lua_logger_char_safe(str[r], esc_type)) {
+ normal = FALSE;
+ break;
+ }
+ }
+
+ if (normal) {
+ r = rspamd_strlcpy(outbuf, str, flen + 1);
+ }
+ else {
+ /* Need to escape non-printed characters */
+ r = 0;
+ s = 0;
+
+ while (slen > 0 && len > 1) {
+ if (!lua_logger_char_safe(str[s], esc_type)) {
+ if (len >= 3) {
+ outbuf[r++] = '\\';
+ outbuf[r++] = hexdigests[((str[s] >> 4) & 0xF)];
+ outbuf[r++] = hexdigests[((str[s]) & 0xF)];
+
+ len -= 2;
+ }
+ else {
+ outbuf[r++] = '?';
+ }
+ }
+ else {
+ outbuf[r++] = str[s];
+ }
+
+ s++;
+ slen--;
+ len--;
+ }
+
+ outbuf[r] = '\0';
+ }
+ }
+
+ return r;
+}
+
+static gsize
+lua_logger_out_num(lua_State *L, gint pos, gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace)
+{
+ gdouble num = lua_tonumber(L, pos);
+ glong inum;
+ gsize r = 0;
+
+ if ((gdouble) (glong) num == num) {
+ inum = num;
+ r = rspamd_snprintf(outbuf, len + 1, "%l", inum);
+ }
+ else {
+ r = rspamd_snprintf(outbuf, len + 1, "%f", num);
+ }
+
+ return r;
+}
+
+static gsize
+lua_logger_out_boolean(lua_State *L, gint pos, gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace)
+{
+ gboolean val = lua_toboolean(L, pos);
+ gsize r = 0;
+
+ r = rspamd_strlcpy(outbuf, val ? "true" : "false", len + 1);
+
+ return r;
+}
+
+static gsize
+lua_logger_out_userdata(lua_State *L, gint pos, gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace)
+{
+ gint r = 0, top;
+ const gchar *str = NULL;
+ gboolean converted_to_str = FALSE;
+
+ top = lua_gettop(L);
+
+ if (!lua_getmetatable(L, pos)) {
+ return 0;
+ }
+
+ lua_pushstring(L, "__index");
+ lua_gettable(L, -2);
+
+ if (!lua_istable(L, -1)) {
+
+ if (lua_isfunction(L, -1)) {
+ /* Functional metatable, try to get __tostring directly */
+ lua_pushstring(L, "__tostring");
+ lua_gettable(L, -3);
+
+ if (lua_isfunction(L, -1)) {
+ lua_pushvalue(L, pos);
+
+ if (lua_pcall(L, 1, 1, 0) != 0) {
+ lua_settop(L, top);
+
+ return 0;
+ }
+
+ str = lua_tostring(L, -1);
+
+ if (str) {
+ r = rspamd_snprintf(outbuf, len, "%s", str);
+ }
+
+ lua_settop(L, top);
+
+ return r;
+ }
+ }
+ lua_settop(L, top);
+
+ return 0;
+ }
+
+ lua_pushstring(L, "__tostring");
+ lua_gettable(L, -2);
+
+ if (lua_isfunction(L, -1)) {
+ lua_pushvalue(L, pos);
+
+ if (lua_pcall(L, 1, 1, 0) != 0) {
+ lua_settop(L, top);
+
+ return 0;
+ }
+
+ str = lua_tostring(L, -1);
+
+ if (str) {
+ converted_to_str = TRUE;
+ }
+ }
+ else {
+ lua_pop(L, 1);
+ lua_pushstring(L, "class");
+ lua_gettable(L, -2);
+
+ if (lua_isstring(L, -1)) {
+ str = lua_tostring(L, -1);
+ converted_to_str = TRUE;
+ }
+ }
+
+ if (converted_to_str) {
+ r = rspamd_snprintf(outbuf, len, "%s", str);
+ }
+ else {
+ /* Print raw pointer */
+ r = rspamd_snprintf(outbuf, len, "%s(%p)", str, lua_touserdata(L, pos));
+ }
+
+ lua_settop(L, top);
+
+ return r;
+}
+
+#define MOVE_BUF(d, remain, r) \
+ (d) += (r); \
+ (remain) -= (r); \
+ if ((remain) == 0) { \
+ lua_settop(L, old_top); \
+ break; \
+ }
+
+static gsize
+lua_logger_out_table(lua_State *L, gint pos, gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace,
+ enum lua_logger_escape_type esc_type)
+{
+ gchar *d = outbuf;
+ gsize remain = len, r;
+ gboolean first = TRUE;
+ gconstpointer self = NULL;
+ gint i, tpos, last_seq = -1, old_top;
+
+ if (!lua_istable(L, pos) || remain == 0) {
+ return 0;
+ }
+
+ old_top = lua_gettop(L);
+ self = lua_topointer(L, pos);
+
+ /* Check if we have seen this pointer */
+ for (i = 0; i < TRACE_POINTS; i++) {
+ if (trace->traces[i] == self) {
+ r = rspamd_snprintf(d, remain + 1, "ref(%p)", self);
+
+ d += r;
+
+ return (d - outbuf);
+ }
+ }
+
+ trace->traces[trace->cur_level % TRACE_POINTS] = self;
+
+ lua_pushvalue(L, pos);
+ r = rspamd_snprintf(d, remain + 1, "{");
+ remain -= r;
+ d += r;
+
+ /* Get numeric keys (ipairs) */
+ for (i = 1;; i++) {
+ lua_rawgeti(L, -1, i);
+
+ if (lua_isnil(L, -1)) {
+ lua_pop(L, 1);
+ break;
+ }
+
+ last_seq = i;
+
+ if (!first) {
+ r = rspamd_snprintf(d, remain + 1, ", ");
+ MOVE_BUF(d, remain, r);
+ }
+
+ r = rspamd_snprintf(d, remain + 1, "[%d] = ", i);
+ MOVE_BUF(d, remain, r);
+ tpos = lua_gettop(L);
+
+ if (lua_topointer(L, tpos) == self) {
+ r = rspamd_snprintf(d, remain + 1, "__self");
+ }
+ else {
+ r = lua_logger_out_type(L, tpos, d, remain, trace, esc_type);
+ }
+ MOVE_BUF(d, remain, r);
+
+ first = FALSE;
+ lua_pop(L, 1);
+ }
+
+ /* Get string keys (pairs) */
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ /* 'key' is at index -2 and 'value' is at index -1 */
+ if (lua_type(L, -2) == LUA_TNUMBER) {
+ if (last_seq > 0) {
+ lua_pushvalue(L, -2);
+
+ if (lua_tonumber(L, -1) <= last_seq + 1) {
+ lua_pop(L, 1);
+ /* Already seen */
+ continue;
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+
+ if (!first) {
+ r = rspamd_snprintf(d, remain + 1, ", ");
+ MOVE_BUF(d, remain, r);
+ }
+
+ /* Preserve key */
+ lua_pushvalue(L, -2);
+ r = rspamd_snprintf(d, remain + 1, "[%s] = ",
+ lua_tostring(L, -1));
+ lua_pop(L, 1); /* Remove key */
+ MOVE_BUF(d, remain, r);
+ tpos = lua_gettop(L);
+
+ if (lua_topointer(L, tpos) == self) {
+ r = rspamd_snprintf(d, remain + 1, "__self");
+ }
+ else {
+ r = lua_logger_out_type(L, tpos, d, remain, trace, esc_type);
+ }
+ MOVE_BUF(d, remain, r);
+
+ first = FALSE;
+ }
+
+ lua_settop(L, old_top);
+
+ r = rspamd_snprintf(d, remain + 1, "}");
+ d += r;
+
+ return (d - outbuf);
+}
+
+#undef MOVE_BUF
+
+gsize lua_logger_out_type(lua_State *L, gint pos,
+ gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace,
+ enum lua_logger_escape_type esc_type)
+{
+ gint type;
+ gsize r = 0;
+
+ if (len == 0) {
+ return 0;
+ }
+
+ type = lua_type(L, pos);
+ trace->cur_level++;
+
+ switch (type) {
+ case LUA_TNUMBER:
+ r = lua_logger_out_num(L, pos, outbuf, len, trace);
+ break;
+ case LUA_TBOOLEAN:
+ r = lua_logger_out_boolean(L, pos, outbuf, len, trace);
+ break;
+ case LUA_TTABLE:
+ r = lua_logger_out_table(L, pos, outbuf, len, trace, esc_type);
+ break;
+ case LUA_TUSERDATA:
+ r = lua_logger_out_userdata(L, pos, outbuf, len, trace);
+ break;
+ case LUA_TFUNCTION:
+ r = rspamd_snprintf(outbuf, len + 1, "function");
+ break;
+ case LUA_TLIGHTUSERDATA:
+ r = rspamd_snprintf(outbuf, len + 1, "0x%p", lua_topointer(L, pos));
+ break;
+ case LUA_TNIL:
+ r = rspamd_snprintf(outbuf, len + 1, "nil");
+ break;
+ case LUA_TNONE:
+ r = rspamd_snprintf(outbuf, len + 1, "no value");
+ break;
+ default:
+ /* Try to push everything as string using tostring magic */
+ r = lua_logger_out_str(L, pos, outbuf, len, trace, esc_type);
+ break;
+ }
+
+ trace->cur_level--;
+
+ return r;
+}
+
+static const gchar *
+lua_logger_get_id(lua_State *L, gint pos, GError **err)
+{
+ const gchar *uid = NULL, *clsname;
+
+ if (lua_getmetatable(L, pos) != 0) {
+ uid = "";
+ lua_pushstring(L, "__index");
+ lua_gettable(L, -2);
+
+ lua_pushstring(L, "class");
+ lua_gettable(L, -2);
+
+ clsname = lua_tostring(L, -1);
+
+ if (strcmp(clsname, "rspamd{task}") == 0) {
+ struct rspamd_task *task = lua_check_task(L, pos);
+
+ if (task) {
+ uid = task->task_pool->tag.uid;
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "invalid rspamd{task}");
+ }
+ }
+ else if (strcmp(clsname, "rspamd{mempool}") == 0) {
+ rspamd_mempool_t *pool;
+
+ pool = rspamd_lua_check_mempool(L, pos);
+
+ if (pool) {
+ uid = pool->tag.uid;
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "invalid rspamd{mempool}");
+ }
+ }
+ else if (strcmp(clsname, "rspamd{config}") == 0) {
+ struct rspamd_config *cfg;
+
+ cfg = lua_check_config(L, pos);
+
+ if (cfg) {
+ if (cfg->checksum) {
+ uid = cfg->checksum;
+ }
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "invalid rspamd{config}");
+ }
+ }
+ else if (strcmp(clsname, "rspamd{map}") == 0) {
+ struct rspamd_lua_map *map;
+
+ map = lua_check_map(L, pos);
+
+ if (map) {
+ if (map->map) {
+ uid = map->map->tag;
+ }
+ else {
+ uid = "embedded";
+ }
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "invalid rspamd{map}");
+ }
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "unknown class: %s", clsname);
+ }
+
+
+ /* Metatable, __index, classname */
+ lua_pop(L, 3);
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "no metatable found for userdata");
+ }
+
+ return uid;
+}
+
+static gboolean
+lua_logger_log_format(lua_State *L, gint fmt_pos, gboolean is_string,
+ gchar *logbuf, gsize remain)
+{
+ gchar *d;
+ const gchar *s, *c;
+ gsize r, cpylen = 0;
+ guint arg_num = 0, cur_arg;
+ bool num_arg = false;
+ struct lua_logger_trace tr;
+ enum {
+ copy_char = 0,
+ got_percent,
+ parse_arg_num
+ } state = copy_char;
+
+ d = logbuf;
+ s = lua_tostring(L, fmt_pos);
+ c = s;
+ cur_arg = fmt_pos;
+
+ if (s == NULL) {
+ return FALSE;
+ }
+
+ while (remain > 0 && *s != '\0') {
+ switch (state) {
+ case copy_char:
+ if (*s == '%') {
+ state = got_percent;
+ s++;
+ if (cpylen > 0) {
+ memcpy(d, c, cpylen);
+ d += cpylen;
+ }
+ cpylen = 0;
+ }
+ else {
+ s++;
+ cpylen++;
+ remain--;
+ }
+ break;
+ case got_percent:
+ if (g_ascii_isdigit(*s) || *s == 's') {
+ state = parse_arg_num;
+ c = s;
+ }
+ else {
+ *d++ = *s++;
+ c = s;
+ state = copy_char;
+ }
+ break;
+ case parse_arg_num:
+ if (g_ascii_isdigit(*s)) {
+ s++;
+ num_arg = true;
+ }
+ else {
+ if (num_arg) {
+ arg_num = strtoul(c, NULL, 10);
+ arg_num += fmt_pos - 1;
+ /* Update the current argument */
+ cur_arg = arg_num;
+ }
+ else {
+ /* We have non numeric argument, e.g. %s */
+ arg_num = cur_arg++;
+ s++;
+ }
+
+ if (arg_num < 1 || arg_num > (guint) lua_gettop(L) + 1) {
+ msg_err("wrong argument number: %ud", arg_num);
+
+ return FALSE;
+ }
+
+ memset(&tr, 0, sizeof(tr));
+ r = lua_logger_out_type(L, arg_num + 1, d, remain, &tr,
+ is_string ? LUA_ESCAPE_UNPRINTABLE : LUA_ESCAPE_LOG);
+ g_assert(r <= remain);
+ remain -= r;
+ d += r;
+ state = copy_char;
+ c = s;
+ }
+ break;
+ }
+ }
+
+ if (state == parse_arg_num) {
+ if (num_arg) {
+ arg_num = strtoul(c, NULL, 10);
+ arg_num += fmt_pos - 1;
+ }
+ else {
+ /* We have non numeric argument, e.g. %s */
+ arg_num = cur_arg;
+ }
+
+ if (arg_num < 1 || arg_num > (guint) lua_gettop(L) + 1) {
+ msg_err("wrong argument number: %ud", arg_num);
+
+ return FALSE;
+ }
+
+ memset(&tr, 0, sizeof(tr));
+ r = lua_logger_out_type(L, arg_num + 1, d, remain, &tr,
+ is_string ? LUA_ESCAPE_UNPRINTABLE : LUA_ESCAPE_LOG);
+ g_assert(r <= remain);
+ remain -= r;
+ d += r;
+ }
+ else if (state == copy_char) {
+ if (cpylen > 0 && remain > 0) {
+ memcpy(d, c, cpylen);
+ d += cpylen;
+ }
+ }
+
+ *d = '\0';
+
+
+ return TRUE;
+}
+
+static gint
+lua_logger_do_log(lua_State *L,
+ GLogLevelFlags level,
+ gboolean is_string,
+ gint start_pos)
+{
+ gchar logbuf[RSPAMD_LOGBUF_SIZE - 128];
+ const gchar *uid = NULL;
+ gint fmt_pos = start_pos;
+ gint ret;
+ GError *err = NULL;
+
+ if (lua_type(L, start_pos) == LUA_TSTRING) {
+ fmt_pos = start_pos;
+ }
+ else if (lua_type(L, start_pos) == LUA_TUSERDATA) {
+ fmt_pos = start_pos + 1;
+
+ uid = lua_logger_get_id(L, start_pos, &err);
+
+ if (uid == NULL) {
+ ret = luaL_error(L, "bad userdata for logging: %s",
+ err ? err->message : "unknown error");
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return ret;
+ }
+ }
+ else {
+ /* Bad argument type */
+ return luaL_error(L, "bad format string type: %s",
+ lua_typename(L, lua_type(L, start_pos)));
+ }
+
+ ret = lua_logger_log_format(L, fmt_pos, is_string,
+ logbuf, sizeof(logbuf) - 1);
+
+ if (ret) {
+ if (is_string) {
+ lua_pushstring(L, logbuf);
+ return 1;
+ }
+ else {
+ lua_common_log_line(level, L, logbuf, uid, "lua", 1);
+ }
+ }
+ else {
+ if (is_string) {
+ lua_pushnil(L);
+
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_logger_errx(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_logger_do_log(L, G_LOG_LEVEL_CRITICAL, FALSE, 1);
+}
+
+static gint
+lua_logger_warnx(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_logger_do_log(L, G_LOG_LEVEL_WARNING, FALSE, 1);
+}
+
+static gint
+lua_logger_infox(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_logger_do_log(L, G_LOG_LEVEL_INFO, FALSE, 1);
+}
+
+static gint
+lua_logger_messagex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_logger_do_log(L, G_LOG_LEVEL_MESSAGE, FALSE, 1);
+}
+
+static gint
+lua_logger_debugx(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_logger_do_log(L, G_LOG_LEVEL_DEBUG, FALSE, 1);
+}
+
+static gint
+lua_logger_logx(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ GLogLevelFlags flags = lua_tonumber(L, 1);
+ const gchar *modname = lua_tostring(L, 2), *uid = NULL;
+ gchar logbuf[RSPAMD_LOGBUF_SIZE - 128];
+ gboolean ret;
+ gint stack_pos = 1;
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ uid = luaL_checkstring(L, 3);
+ }
+ else if (lua_type(L, 3) == LUA_TUSERDATA) {
+ uid = lua_logger_get_id(L, 3, NULL);
+ }
+ else {
+ uid = "???";
+ }
+
+ if (uid && modname) {
+ if (lua_type(L, 4) == LUA_TSTRING) {
+ ret = lua_logger_log_format(L, 4, FALSE, logbuf, sizeof(logbuf) - 1);
+ }
+ else if (lua_type(L, 4) == LUA_TNUMBER) {
+ stack_pos = lua_tonumber(L, 4);
+ ret = lua_logger_log_format(L, 5, FALSE, logbuf, sizeof(logbuf) - 1);
+ }
+ else {
+ return luaL_error(L, "invalid argument on pos 4");
+ }
+
+ if (ret) {
+ lua_common_log_line(flags, L, logbuf, uid, modname, stack_pos);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+
+static gint
+lua_logger_debugm(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gchar logbuf[RSPAMD_LOGBUF_SIZE - 128];
+ const gchar *uid = NULL, *module = NULL;
+ gint stack_pos = 1;
+ gboolean ret;
+
+ module = luaL_checkstring(L, 1);
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ uid = luaL_checkstring(L, 2);
+ }
+ else {
+ uid = lua_logger_get_id(L, 2, NULL);
+ }
+
+ if (uid && module) {
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ ret = lua_logger_log_format(L, 3, FALSE, logbuf, sizeof(logbuf) - 1);
+ }
+ else if (lua_type(L, 3) == LUA_TNUMBER) {
+ stack_pos = lua_tonumber(L, 3);
+ ret = lua_logger_log_format(L, 4, FALSE, logbuf, sizeof(logbuf) - 1);
+ }
+ else {
+ return luaL_error(L, "invalid argument on pos 3");
+ }
+
+ if (ret) {
+ lua_common_log_line(G_LOG_LEVEL_DEBUG, L, logbuf, uid, module, stack_pos);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+
+static gint
+lua_logger_slog(lua_State *L)
+{
+ return lua_logger_do_log(L, 0, TRUE, 1);
+}
+
+static gint
+lua_logger_log_level(lua_State *L)
+{
+ gint log_level = rspamd_log_get_log_level(NULL);
+
+ lua_pushstring(L, rspamd_get_log_severity_string(log_level));
+
+ return 1;
+}
+
+/*** Init functions ***/
+
+static gint
+lua_load_logger(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, loggerlib_f);
+
+ return 1;
+}
+
+void luaopen_logger(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_logger", lua_load_logger);
+}
diff --git a/src/lua/lua_map.c b/src/lua/lua_map.c
new file mode 100644
index 0000000..54cfb4b
--- /dev/null
+++ b/src/lua/lua_map.c
@@ -0,0 +1,1421 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
+#include "libserver/maps/map_private.h"
+#include "contrib/libucl/lua_ucl.h"
+
+/***
+ * This module is used to manage rspamd maps and map like objects
+ *
+ * @module rspamd_map
+ *
+ * All maps could be obtained by function `rspamd_config:get_maps()`
+ * Also see [`lua_maps` module description](lua_maps.html).
+ *
+ * **Important notice** maps cannot be queried outside of the worker context.
+ * For example, you cannot add even a file map and query some keys from it during
+ * some module initialisation, you need to add the appropriate event loop context
+ * for a worker (e.g. you cannot use `get_key` outside of the symbols callbacks or
+ * a worker `on_load` scripts).
+ *
+@example
+
+local hash_map = rspamd_config:add_map{
+ type = "hash",
+ urls = ['file:///path/to/file'],
+ description = 'sample map'
+}
+
+local function sample_symbol_cb(task)
+ -- Check whether hash map contains from address of message
+ if hash_map:get_key((task:get_from() or {})[1]) then
+ -- key found
+ end
+end
+
+rspamd_config:register_symbol{
+ name = 'SAMPLE_SYMBOL',
+ type = 'normal',
+ score = 1.0,
+ description = "A sample symbol",
+ callback = sample_symbol_cb,
+}
+ */
+
+/***
+ * @method map:get_key(in)
+ * Variable method for different types of maps:
+ *
+ * - For hash maps it returns boolean and accepts string
+ * - For kv maps it returns string (or nil) and accepts string
+ * - For radix maps it returns boolean and accepts IP address (as object, string or number)
+ *
+ * @param {vary} in input to check
+ * @return {bool|string} if a value is found then this function returns string or `True` if not - then it returns `nil` or `False`
+ */
+LUA_FUNCTION_DEF(map, get_key);
+
+
+/***
+ * @method map:is_signed()
+ * Returns `True` if a map is signed
+ * @return {bool} signed value
+ */
+LUA_FUNCTION_DEF(map, is_signed);
+
+/***
+ * @method map:get_proto()
+ * Returns protocol of map as string:
+ *
+ * - `http`: for HTTP map
+ * - `file`: for file map
+ * @return {string} string representation of the map protocol
+ */
+LUA_FUNCTION_DEF(map, get_proto);
+
+/***
+ * @method map:get_sign_key()
+ * Returns pubkey used for signing as base32 string or nil
+ * @return {string} base32 encoded string or nil
+ */
+LUA_FUNCTION_DEF(map, get_sign_key);
+
+/***
+ * @method map:set_sign_key(key)
+ * Set trusted key for signatures for this map
+ * @param {string} key base32 encoded string or nil
+ */
+LUA_FUNCTION_DEF(map, set_sign_key);
+
+/***
+ * @method map:set_callback(cb)
+ * Set callback for a specified callback map.
+ * @param {function} cb map callback function
+ */
+LUA_FUNCTION_DEF(map, set_callback);
+
+/***
+ * @method map:get_uri()
+ * Get uri for a specified map
+ * @return {string} map's URI
+ */
+LUA_FUNCTION_DEF(map, get_uri);
+
+/***
+ * @method map:get_stats(reset)
+ * Get statistics for specific map. It returns table in form:
+ * [key] => [nhits]
+ * @param {boolean} reset reset stats if true
+ * @return {table} map's stat
+ */
+LUA_FUNCTION_DEF(map, get_stats);
+
+/***
+ * @method map:foreach(callback, is_text)
+ * Iterate over map elements and call callback for each element.
+ * @param {function} callback callback function, that accepts two arguments: key and value, if it returns true then iteration is stopped
+ * @param {boolean} is_text if true then callback accepts rspamd_text instead of Lua strings
+ * @return {number} number of elements iterated
+ */
+LUA_FUNCTION_DEF(map, foreach);
+
+/***
+ * @method map:on_load(callback)
+ * Sets a callback for a map that is called when map is loaded
+ * @param {function} callback callback function, that accepts no arguments (pass maps in a closure if needed)
+ */
+LUA_FUNCTION_DEF(map, on_load);
+
+/***
+ * @method map:get_data_digest()
+ * Get data digest for specific map
+ * @return {string} 64 bit number represented as string (due to Lua limitations)
+ */
+LUA_FUNCTION_DEF(map, get_data_digest);
+
+/***
+ * @method map:get_nelts()
+ * Get number of elements for specific map
+ * @return {number} number of elements in the map
+ */
+LUA_FUNCTION_DEF(map, get_nelts);
+
+static const struct luaL_reg maplib_m[] = {
+ LUA_INTERFACE_DEF(map, get_key),
+ LUA_INTERFACE_DEF(map, is_signed),
+ LUA_INTERFACE_DEF(map, get_proto),
+ LUA_INTERFACE_DEF(map, get_sign_key),
+ LUA_INTERFACE_DEF(map, set_sign_key),
+ LUA_INTERFACE_DEF(map, set_callback),
+ LUA_INTERFACE_DEF(map, get_uri),
+ LUA_INTERFACE_DEF(map, get_stats),
+ LUA_INTERFACE_DEF(map, foreach),
+ LUA_INTERFACE_DEF(map, on_load),
+ LUA_INTERFACE_DEF(map, get_data_digest),
+ LUA_INTERFACE_DEF(map, get_nelts),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+struct lua_map_callback_data {
+ lua_State *L;
+ gint ref;
+ gboolean opaque;
+ rspamd_fstring_t *data;
+ struct rspamd_lua_map *lua_map;
+};
+
+struct rspamd_lua_map *
+lua_check_map(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{map}");
+ luaL_argcheck(L, ud != NULL, pos, "'map' expected");
+ return ud ? *((struct rspamd_lua_map **) ud) : NULL;
+}
+
+gint lua_config_add_radix_map(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *map_line, *description;
+ struct rspamd_lua_map *map, **pmap;
+ struct rspamd_map *m;
+
+ if (cfg) {
+ map_line = luaL_checkstring(L, 2);
+ description = lua_tostring(L, 3);
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.radix = NULL;
+ map->type = RSPAMD_LUA_MAP_RADIX;
+
+ if ((m = rspamd_map_add(cfg, map_line, description,
+ rspamd_radix_read,
+ rspamd_radix_fin,
+ rspamd_radix_dtor,
+ (void **) &map->data.radix,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_warn_config("invalid radix map %s", map_line);
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ map->map = m;
+ m->lua_map = map;
+ pmap = lua_newuserdata(L, sizeof(void *));
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+gint lua_config_radix_from_config(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *mname, *optname;
+ const ucl_object_t *obj;
+ struct rspamd_lua_map *map, **pmap;
+ ucl_object_t *fake_obj;
+ struct rspamd_map *m;
+
+ if (!cfg) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ mname = luaL_checkstring(L, 2);
+ optname = luaL_checkstring(L, 3);
+
+ if (mname && optname) {
+ obj = rspamd_config_get_module_opt(cfg, mname, optname);
+
+ if (obj) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.radix = NULL;
+ map->type = RSPAMD_LUA_MAP_RADIX;
+
+ fake_obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(fake_obj, ucl_object_ref(obj),
+ "data", 0, false);
+ ucl_object_insert_key(fake_obj, ucl_object_fromstring("static"),
+ "url", 0, false);
+
+ if ((m = rspamd_map_add_from_ucl(cfg, fake_obj, "static radix map",
+ rspamd_radix_read,
+ rspamd_radix_fin,
+ rspamd_radix_dtor,
+ (void **) &map->data.radix,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_err_config("invalid radix map static");
+ lua_pushnil(L);
+ ucl_object_unref(fake_obj);
+
+ return 1;
+ }
+
+ ucl_object_unref(fake_obj);
+ pmap = lua_newuserdata(L, sizeof(void *));
+ map->map = m;
+ m->lua_map = map;
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ msg_warn_config("Couldnt find config option [%s][%s]", mname,
+ optname);
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+gint lua_config_radix_from_ucl(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ ucl_object_t *obj;
+ struct rspamd_lua_map *map, **pmap;
+ ucl_object_t *fake_obj;
+ struct rspamd_map *m;
+
+ if (!cfg) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ obj = ucl_object_lua_import(L, 2);
+
+ if (obj) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.radix = NULL;
+ map->type = RSPAMD_LUA_MAP_RADIX;
+
+ fake_obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(fake_obj, ucl_object_ref(obj),
+ "data", 0, false);
+ ucl_object_insert_key(fake_obj, ucl_object_fromstring("static"),
+ "url", 0, false);
+
+ if ((m = rspamd_map_add_from_ucl(cfg, fake_obj, "static radix map",
+ rspamd_radix_read,
+ rspamd_radix_fin,
+ rspamd_radix_dtor,
+ (void **) &map->data.radix,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_err_config("invalid radix map static");
+ lua_pushnil(L);
+ ucl_object_unref(fake_obj);
+ ucl_object_unref(obj);
+
+ return 1;
+ }
+
+ ucl_object_unref(fake_obj);
+ ucl_object_unref(obj);
+ pmap = lua_newuserdata(L, sizeof(void *));
+ map->map = m;
+ m->lua_map = map;
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+gint lua_config_add_hash_map(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *map_line, *description;
+ struct rspamd_lua_map *map, **pmap;
+ struct rspamd_map *m;
+
+ if (cfg) {
+ map_line = luaL_checkstring(L, 2);
+ description = lua_tostring(L, 3);
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.hash = NULL;
+ map->type = RSPAMD_LUA_MAP_SET;
+
+ if ((m = rspamd_map_add(cfg, map_line, description,
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &map->data.hash,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_warn_config("invalid set map %s", map_line);
+ lua_pushnil(L);
+ return 1;
+ }
+
+ map->map = m;
+ m->lua_map = map;
+ pmap = lua_newuserdata(L, sizeof(void *));
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+gint lua_config_add_kv_map(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *map_line, *description;
+ struct rspamd_lua_map *map, **pmap;
+ struct rspamd_map *m;
+
+ if (cfg) {
+ map_line = luaL_checkstring(L, 2);
+ description = lua_tostring(L, 3);
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.hash = NULL;
+ map->type = RSPAMD_LUA_MAP_HASH;
+
+ if ((m = rspamd_map_add(cfg, map_line, description,
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &map->data.hash,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_warn_config("invalid hash map %s", map_line);
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ map->map = m;
+ m->lua_map = map;
+ pmap = lua_newuserdata(L, sizeof(void *));
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gchar *
+lua_map_read(gchar *chunk, gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct lua_map_callback_data *cbdata, *old;
+
+ if (data->cur_data == NULL) {
+ old = (struct lua_map_callback_data *) data->prev_data;
+ cbdata = old;
+ cbdata->L = old->L;
+ cbdata->ref = old->ref;
+ cbdata->lua_map = old->lua_map;
+ data->cur_data = cbdata;
+ data->prev_data = NULL;
+ }
+ else {
+ cbdata = (struct lua_map_callback_data *) data->cur_data;
+ }
+
+ if (cbdata->data == NULL) {
+ cbdata->data = rspamd_fstring_new_init(chunk, len);
+ }
+ else {
+ cbdata->data = rspamd_fstring_append(cbdata->data, chunk, len);
+ }
+
+ return NULL;
+}
+
+static void
+lua_map_fin(struct map_cb_data *data, void **target)
+{
+ struct lua_map_callback_data *cbdata;
+ struct rspamd_lua_map **pmap;
+ struct rspamd_map *map;
+
+ map = data->map;
+
+ if (data->errored) {
+ if (data->cur_data) {
+ cbdata = (struct lua_map_callback_data *) data->cur_data;
+ if (cbdata->ref != -1) {
+ luaL_unref(cbdata->L, LUA_REGISTRYINDEX, cbdata->ref);
+ }
+
+ if (cbdata->data) {
+ rspamd_fstring_free(cbdata->data);
+ }
+
+ data->cur_data = NULL;
+ }
+ }
+ else {
+ if (data->cur_data) {
+ cbdata = (struct lua_map_callback_data *) data->cur_data;
+ }
+ else {
+ msg_err_map("no data read for map");
+ return;
+ }
+
+ if (cbdata->ref == -1) {
+ msg_err_map("map has no callback set");
+ }
+ else if (cbdata->data != NULL && cbdata->data->len != 0) {
+
+ lua_pushcfunction(cbdata->L, &rspamd_lua_traceback);
+ int err_idx = lua_gettop(cbdata->L);
+
+ lua_rawgeti(cbdata->L, LUA_REGISTRYINDEX, cbdata->ref);
+
+ if (!cbdata->opaque) {
+ lua_pushlstring(cbdata->L, cbdata->data->str, cbdata->data->len);
+ }
+ else {
+ struct rspamd_lua_text *t;
+
+ t = lua_newuserdata(cbdata->L, sizeof(*t));
+ rspamd_lua_setclass(cbdata->L, "rspamd{text}", -1);
+ t->flags = 0;
+ t->len = cbdata->data->len;
+ t->start = cbdata->data->str;
+ }
+
+ pmap = lua_newuserdata(cbdata->L, sizeof(void *));
+ *pmap = cbdata->lua_map;
+ rspamd_lua_setclass(cbdata->L, "rspamd{map}", -1);
+
+ gint ret = lua_pcall(cbdata->L, 2, 0, err_idx);
+
+ if (ret != 0) {
+ msg_info_map("call to %s failed (%d): %s", "map fin function",
+ ret,
+ lua_tostring(cbdata->L, -1));
+ }
+
+ lua_settop(cbdata->L, err_idx - 1);
+ }
+
+ cbdata->data = rspamd_fstring_assign(cbdata->data, "", 0);
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ data->prev_data = NULL;
+ }
+ }
+}
+
+static void
+lua_map_dtor(struct map_cb_data *data)
+{
+ struct lua_map_callback_data *cbdata;
+
+ if (data->cur_data) {
+ cbdata = (struct lua_map_callback_data *) data->cur_data;
+ if (cbdata->ref != -1) {
+ luaL_unref(cbdata->L, LUA_REGISTRYINDEX, cbdata->ref);
+ }
+
+ if (cbdata->data) {
+ rspamd_fstring_free(cbdata->data);
+ }
+ }
+}
+
+gint lua_config_add_map(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const char *description = NULL;
+ const gchar *type = NULL;
+ ucl_object_t *map_obj = NULL;
+ struct lua_map_callback_data *cbdata;
+ struct rspamd_lua_map *map, **pmap;
+ struct rspamd_map *m;
+ gboolean opaque_data = FALSE;
+ int cbidx = -1, ret;
+ GError *err = NULL;
+
+ if (cfg) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*url=O;description=S;callback=F;type=S;opaque_data=B",
+ &map_obj, &description, &cbidx, &type, &opaque_data)) {
+ ret = luaL_error(L, "invalid table arguments: %s", err->message);
+ g_error_free(err);
+ if (map_obj) {
+ ucl_object_unref(map_obj);
+ }
+
+ return ret;
+ }
+
+ g_assert(map_obj != NULL);
+
+ if (type == NULL && cbidx != -1) {
+ type = "callback";
+ }
+ else if (type == NULL) {
+ return luaL_error(L, "invalid map type");
+ }
+
+ if (strcmp(type, "callback") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->type = RSPAMD_LUA_MAP_CALLBACK;
+ map->data.cbdata = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(*map->data.cbdata));
+ cbdata = map->data.cbdata;
+ cbdata->L = L;
+ cbdata->data = NULL;
+ cbdata->lua_map = map;
+ cbdata->ref = cbidx;
+ cbdata->opaque = opaque_data;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ lua_map_read,
+ lua_map_fin,
+ lua_map_dtor,
+ (void **) &map->data.cbdata,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+
+ if (cbidx != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbidx);
+ }
+
+ if (map_obj) {
+ ucl_object_unref(map_obj);
+ }
+
+ lua_pushnil(L);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "set") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.hash = NULL;
+ map->type = RSPAMD_LUA_MAP_SET;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &map->data.hash,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "map") == 0 || strcmp(type, "hash") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.hash = NULL;
+ map->type = RSPAMD_LUA_MAP_HASH;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &map->data.hash,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "radix") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.radix = NULL;
+ map->type = RSPAMD_LUA_MAP_RADIX;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_radix_read,
+ rspamd_radix_fin,
+ rspamd_radix_dtor,
+ (void **) &map->data.radix,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "regexp") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.re_map = NULL;
+ map->type = RSPAMD_LUA_MAP_REGEXP;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_regexp_list_read_single,
+ rspamd_regexp_list_fin,
+ rspamd_regexp_list_dtor,
+ (void **) &map->data.re_map,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "regexp_multi") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.re_map = NULL;
+ map->type = RSPAMD_LUA_MAP_REGEXP_MULTIPLE;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_regexp_list_read_multiple,
+ rspamd_regexp_list_fin,
+ rspamd_regexp_list_dtor,
+ (void **) &map->data.re_map,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "glob") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.re_map = NULL;
+ map->type = RSPAMD_LUA_MAP_REGEXP;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_glob_list_read_single,
+ rspamd_regexp_list_fin,
+ rspamd_regexp_list_dtor,
+ (void **) &map->data.re_map,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "glob_multi") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.re_map = NULL;
+ map->type = RSPAMD_LUA_MAP_REGEXP_MULTIPLE;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_glob_list_read_multiple,
+ rspamd_regexp_list_fin,
+ rspamd_regexp_list_dtor,
+ (void **) &map->data.re_map,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "cdb") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.cdb_map = NULL;
+ map->type = RSPAMD_LUA_MAP_CDB;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_cdb_list_read,
+ rspamd_cdb_list_fin,
+ rspamd_cdb_list_dtor,
+ (void **) &map->data.cdb_map,
+ NULL, RSPAMD_MAP_FILE_ONLY | RSPAMD_MAP_FILE_NO_READ)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else {
+ ret = luaL_error(L, "invalid arguments: unknown type '%s'", type);
+ ucl_object_unref(map_obj);
+
+ return ret;
+ }
+
+ map->map = m;
+ pmap = lua_newuserdata(L, sizeof(void *));
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ ucl_object_unref(map_obj);
+
+ return 1;
+}
+
+gint lua_config_get_maps(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_lua_map *map, **pmap;
+ struct rspamd_map *m;
+ gint i = 1;
+ GList *cur;
+
+ if (cfg) {
+ lua_newtable(L);
+ cur = g_list_first(cfg->maps);
+
+ while (cur) {
+ m = cur->data;
+
+ if (m->lua_map) {
+ map = m->lua_map;
+ }
+ else {
+ /* Implement heuristic */
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+
+ if (m->read_callback == rspamd_radix_read) {
+ map->type = RSPAMD_LUA_MAP_RADIX;
+ map->data.radix = *m->user_data;
+ }
+ else if (m->read_callback == rspamd_kv_list_read) {
+ map->type = RSPAMD_LUA_MAP_HASH;
+ map->data.hash = *m->user_data;
+ }
+ else {
+ map->type = RSPAMD_LUA_MAP_UNKNOWN;
+ }
+
+ map->map = m;
+ m->lua_map = map;
+ }
+
+ pmap = lua_newuserdata(L, sizeof(*pmap));
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ lua_rawseti(L, -2, i);
+
+ cur = g_list_next(cur);
+ i++;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static const gchar *
+lua_map_process_string_key(lua_State *L, gint pos, gsize *len)
+{
+ struct rspamd_lua_text *t;
+
+ if (lua_type(L, pos) == LUA_TSTRING) {
+ return lua_tolstring(L, pos, len);
+ }
+ else if (lua_type(L, pos) == LUA_TUSERDATA) {
+ t = lua_check_text(L, pos);
+
+ if (t) {
+ *len = t->len;
+ return t->start;
+ }
+ }
+
+ return NULL;
+}
+
+/* Radix and hash table functions */
+static gint
+lua_map_get_key(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ struct rspamd_radix_map_helper *radix;
+ struct rspamd_lua_ip *addr = NULL;
+ const gchar *key, *value = NULL;
+ gpointer ud;
+ gsize len;
+ guint32 key_num = 0;
+ gboolean ret = FALSE;
+
+ if (map) {
+ if (map->type == RSPAMD_LUA_MAP_RADIX) {
+ radix = map->data.radix;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ const gchar *addr_str;
+
+ addr_str = luaL_checklstring(L, 2, &len);
+ addr = g_alloca(sizeof(*addr));
+ addr->addr = g_alloca(rspamd_inet_address_storage_size());
+
+ if (!rspamd_parse_inet_address_ip(addr_str, len, addr->addr)) {
+ addr = NULL;
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ ud = rspamd_lua_check_udata(L, 2, "rspamd{ip}");
+ if (ud != NULL) {
+ addr = *((struct rspamd_lua_ip **) ud);
+
+ if (addr->addr == NULL) {
+ addr = NULL;
+ }
+ }
+ else {
+ msg_err("invalid userdata type provided, rspamd{ip} expected");
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TNUMBER) {
+ key_num = luaL_checkinteger(L, 2);
+ key_num = htonl(key_num);
+ }
+
+ if (radix) {
+ gconstpointer p = NULL;
+
+ if (addr != NULL) {
+ if ((p = rspamd_match_radix_map_addr(radix, addr->addr)) != NULL) {
+ ret = TRUE;
+ }
+ else {
+ p = 0;
+ }
+ }
+ else if (key_num != 0) {
+ if ((p = rspamd_match_radix_map(radix,
+ (guint8 *) &key_num, sizeof(key_num))) != NULL) {
+ ret = TRUE;
+ }
+ else {
+ p = 0;
+ }
+ }
+
+ value = (const char *) p;
+ }
+
+ if (ret) {
+ lua_pushstring(L, value);
+ return 1;
+ }
+ }
+ else if (map->type == RSPAMD_LUA_MAP_SET) {
+ key = lua_map_process_string_key(L, 2, &len);
+
+ if (key && map->data.hash) {
+ ret = rspamd_match_hash_map(map->data.hash, key, len) != NULL;
+ }
+ }
+ else if (map->type == RSPAMD_LUA_MAP_REGEXP) {
+ key = lua_map_process_string_key(L, 2, &len);
+
+ if (key && map->data.re_map) {
+ value = rspamd_match_regexp_map_single(map->data.re_map, key,
+ len);
+
+ if (value) {
+ lua_pushstring(L, value);
+ return 1;
+ }
+ }
+ }
+ else if (map->type == RSPAMD_LUA_MAP_REGEXP_MULTIPLE) {
+ GPtrArray *ar;
+ guint i;
+ const gchar *val;
+
+ key = lua_map_process_string_key(L, 2, &len);
+
+ if (key && map->data.re_map) {
+ ar = rspamd_match_regexp_map_all(map->data.re_map, key,
+ len);
+
+ if (ar) {
+ lua_createtable(L, ar->len, 0);
+
+ PTR_ARRAY_FOREACH(ar, i, val)
+ {
+ lua_pushstring(L, val);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ g_ptr_array_free(ar, TRUE);
+
+ return 1;
+ }
+ }
+ }
+ else if (map->type == RSPAMD_LUA_MAP_HASH) {
+ /* key-value map */
+ key = lua_map_process_string_key(L, 2, &len);
+
+ if (key && map->data.hash) {
+ value = rspamd_match_hash_map(map->data.hash, key, len);
+ }
+
+ if (value) {
+ lua_pushstring(L, value);
+ return 1;
+ }
+ }
+ else if (map->type == RSPAMD_LUA_MAP_CDB) {
+ /* cdb map */
+ const rspamd_ftok_t *tok = NULL;
+
+ key = lua_map_process_string_key(L, 2, &len);
+
+ if (key && map->data.cdb_map) {
+ tok = rspamd_match_cdb_map(map->data.cdb_map, key, len);
+ }
+
+ if (tok) {
+ lua_pushlstring(L, tok->begin, tok->len);
+ return 1;
+ }
+ }
+ else {
+ /* callback map or unknown type map */
+ lua_pushnil(L);
+ return 1;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, ret);
+ return 1;
+}
+
+static gboolean
+lua_map_traverse_cb(gconstpointer key,
+ gconstpointer value, gsize hits, gpointer ud)
+{
+ lua_State *L = (lua_State *) ud;
+
+ lua_pushstring(L, key);
+ lua_pushinteger(L, hits);
+ lua_settable(L, -3);
+
+ return TRUE;
+}
+
+static gint
+lua_map_get_stats(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ gboolean do_reset = FALSE;
+
+ if (map != NULL) {
+ if (lua_isboolean(L, 2)) {
+ do_reset = lua_toboolean(L, 2);
+ }
+
+ lua_createtable(L, 0, map->map->nelts);
+
+ if (map->map->traverse_function) {
+ rspamd_map_traverse(map->map, lua_map_traverse_cb, L, do_reset);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct lua_map_traverse_cbdata {
+ lua_State *L;
+ gint cbref;
+ gboolean use_text;
+};
+
+static gboolean
+lua_map_foreach_cb(gconstpointer key, gconstpointer value, gsize _hits, gpointer ud)
+{
+ struct lua_map_traverse_cbdata *cbdata = ud;
+ lua_State *L = cbdata->L;
+
+ lua_pushvalue(L, cbdata->cbref);
+
+ if (cbdata->use_text) {
+ lua_new_text(L, key, strlen(key), 0);
+ lua_new_text(L, value, strlen(value), 0);
+ }
+ else {
+ lua_pushstring(L, key);
+ lua_pushstring(L, value);
+ }
+
+ if (lua_pcall(L, 2, 1, 0) != 0) {
+ msg_err("call to map foreach callback failed: %s", lua_tostring(L, -1));
+ lua_pop(L, 1); /* error */
+
+ return FALSE;
+ }
+ else {
+ if (lua_isboolean(L, -1)) {
+ lua_pop(L, 2);
+
+ return lua_toboolean(L, -1);
+ }
+
+ lua_pop(L, 1); /* result */
+ }
+
+ return TRUE;
+}
+
+static gint
+lua_map_foreach(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ gboolean use_text = FALSE;
+
+ if (map != NULL && lua_isfunction(L, 2)) {
+ if (lua_isboolean(L, 3)) {
+ use_text = lua_toboolean(L, 3);
+ }
+
+ struct lua_map_traverse_cbdata cbdata;
+ cbdata.L = L;
+ lua_pushvalue(L, 2); /* func */
+ cbdata.cbref = lua_gettop(L);
+ cbdata.use_text = use_text;
+
+ if (map->map->traverse_function) {
+ rspamd_map_traverse(map->map, lua_map_foreach_cb, &cbdata, FALSE);
+ }
+
+ /* Remove callback */
+ lua_pop(L, 1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_map_get_data_digest(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ gchar numbuf[64];
+
+ if (map != NULL) {
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%uL", map->map->digest);
+ lua_pushstring(L, numbuf);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_map_get_nelts(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+
+ if (map != NULL) {
+ lua_pushinteger(L, map->map->nelts);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_map_is_signed(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ gboolean ret = FALSE;
+ struct rspamd_map_backend *bk;
+ guint i;
+
+ if (map != NULL) {
+ if (map->map) {
+ for (i = 0; i < map->map->backends->len; i++) {
+ bk = g_ptr_array_index(map->map->backends, i);
+ if (bk->is_signed && bk->protocol == MAP_PROTO_FILE) {
+ ret = TRUE;
+ break;
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, ret);
+ return 1;
+}
+
+static int
+lua_map_get_proto(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ const gchar *ret = "undefined";
+ struct rspamd_map_backend *bk;
+ guint i;
+
+ if (map != NULL) {
+ for (i = 0; i < map->map->backends->len; i++) {
+ bk = g_ptr_array_index(map->map->backends, i);
+ switch (bk->protocol) {
+ case MAP_PROTO_FILE:
+ ret = "file";
+ break;
+ case MAP_PROTO_HTTP:
+ ret = "http";
+ break;
+ case MAP_PROTO_HTTPS:
+ ret = "https";
+ break;
+ case MAP_PROTO_STATIC:
+ ret = "static";
+ break;
+ }
+ lua_pushstring(L, ret);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return map->map->backends->len;
+}
+
+static int
+lua_map_get_sign_key(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ struct rspamd_map_backend *bk;
+ guint i;
+ GString *ret = NULL;
+
+ if (map != NULL) {
+ for (i = 0; i < map->map->backends->len; i++) {
+ bk = g_ptr_array_index(map->map->backends, i);
+
+ if (bk->trusted_pubkey) {
+ ret = rspamd_pubkey_print(bk->trusted_pubkey,
+ RSPAMD_KEYPAIR_PUBKEY | RSPAMD_KEYPAIR_BASE32);
+ }
+ else {
+ ret = NULL;
+ }
+
+ if (ret) {
+ lua_pushlstring(L, ret->str, ret->len);
+ g_string_free(ret, TRUE);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return map->map->backends->len;
+}
+
+static int
+lua_map_set_sign_key(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ struct rspamd_map_backend *bk;
+ const gchar *pk_str;
+ struct rspamd_cryptobox_pubkey *pk;
+ gsize len;
+ guint i;
+
+ pk_str = lua_tolstring(L, 2, &len);
+
+ if (map && pk_str) {
+ pk = rspamd_pubkey_from_base32(pk_str, len, RSPAMD_KEYPAIR_SIGN,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+
+ if (!pk) {
+ return luaL_error(L, "invalid pubkey string");
+ }
+
+ for (i = 0; i < map->map->backends->len; i++) {
+ bk = g_ptr_array_index(map->map->backends, i);
+ if (bk->trusted_pubkey) {
+ /* Unref old pk */
+ rspamd_pubkey_unref(bk->trusted_pubkey);
+ }
+
+ bk->trusted_pubkey = rspamd_pubkey_ref(pk);
+ }
+
+ rspamd_pubkey_unref(pk);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static int
+lua_map_set_callback(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+
+ if (!map || map->type != RSPAMD_LUA_MAP_CALLBACK || map->data.cbdata == NULL) {
+ return luaL_error(L, "invalid map");
+ }
+
+ if (lua_type(L, 2) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid callback");
+ }
+
+ lua_pushvalue(L, 2);
+ /* Get a reference */
+ map->data.cbdata->ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ return 0;
+}
+
+static int
+lua_map_get_uri(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ struct rspamd_map_backend *bk;
+ guint i;
+
+ if (map != NULL) {
+ for (i = 0; i < map->map->backends->len; i++) {
+ bk = g_ptr_array_index(map->map->backends, i);
+ lua_pushstring(L, bk->uri);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return map->map->backends->len;
+}
+
+struct lua_map_on_load_cbdata {
+ lua_State *L;
+ gint ref;
+};
+
+static void
+lua_map_on_load_dtor(gpointer p)
+{
+ struct lua_map_on_load_cbdata *cbd = p;
+
+ luaL_unref(cbd->L, LUA_REGISTRYINDEX, cbd->ref);
+ g_free(cbd);
+}
+
+static void
+lua_map_on_load_handler(struct rspamd_map *map, gpointer ud)
+{
+ struct lua_map_on_load_cbdata *cbd = ud;
+ lua_State *L = cbd->L;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->ref);
+
+ if (lua_pcall(L, 0, 0, 0) != 0) {
+ msg_err_map("call to on_load function failed: %s", lua_tostring(L, -1));
+ }
+}
+
+static gint
+lua_map_on_load(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+
+ if (map == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ struct lua_map_on_load_cbdata *cbd = g_malloc(sizeof(struct lua_map_on_load_cbdata));
+ cbd->L = L;
+ cbd->ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ rspamd_map_set_on_load_function(map->map, lua_map_on_load_handler, cbd, lua_map_on_load_dtor);
+ }
+ else {
+ return luaL_error(L, "invalid callback");
+ }
+
+ return 0;
+}
+
+void luaopen_map(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{map}", maplib_m);
+
+ lua_pop(L, 1);
+}
diff --git a/src/lua/lua_map.h b/src/lua/lua_map.h
new file mode 100644
index 0000000..70677de
--- /dev/null
+++ b/src/lua/lua_map.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LUA_LUA_MAP_H_
+#define SRC_LUA_LUA_MAP_H_
+
+#include "lua_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+LUA_PUBLIC_FUNCTION_DEF(config, add_radix_map);
+LUA_PUBLIC_FUNCTION_DEF(config, radix_from_config);
+LUA_PUBLIC_FUNCTION_DEF(config, radix_from_ucl);
+LUA_PUBLIC_FUNCTION_DEF(config, add_map);
+LUA_PUBLIC_FUNCTION_DEF(config, add_hash_map);
+LUA_PUBLIC_FUNCTION_DEF(config, add_kv_map);
+LUA_PUBLIC_FUNCTION_DEF(config, add_map);
+LUA_PUBLIC_FUNCTION_DEF(config, get_maps);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LUA_LUA_MAP_H_ */
diff --git a/src/lua/lua_mempool.c b/src/lua/lua_mempool.c
new file mode 100644
index 0000000..4897d15
--- /dev/null
+++ b/src/lua/lua_mempool.c
@@ -0,0 +1,612 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+
+/***
+ * @module rspamd_mempool
+ * Rspamd memory pool is used to allocate memory attached to specific objects,
+ * namely it was initially used for memory allocation for rspamd_task.
+ *
+ * All memory allocated by the pool is destroyed when the associated object is
+ * destroyed. This allows a sort of controlled garbage collection for memory
+ * allocated from the pool. Memory pools are extensively used by rspamd internal
+ * components and provide some powerful features, such as destructors or
+ * persistent variables.
+ * @example
+local mempool = require "rspamd_mempool"
+local pool = mempool.create()
+
+pool:set_variable('a', 'bcd', 1, 1.01, false)
+local v1, v2, v3, v4 = pool:get_variable('a', 'string,double,double,bool')
+pool:destroy()
+ */
+
+/* Lua bindings */
+/***
+ * @function mempool.create([size])
+ * Creates a memory pool of a specified `size` or platform dependent optimal size (normally, a page size)
+ * @param {number} size size of a page inside pool
+ * @return {rspamd_mempool} new pool object (that should be removed by explicit call to `pool:destroy()`)
+ */
+LUA_FUNCTION_DEF(mempool, create);
+/***
+ * @method mempool:add_destructor(func)
+ * Adds new destructor function to the pool
+ * @param {function} func function to be called when the pool is destroyed
+ */
+LUA_FUNCTION_DEF(mempool, add_destructor);
+/***
+ * @method mempool:destroy()
+ * Destroys memory pool cleaning all variables and calling all destructors registered (both C and Lua ones)
+ */
+LUA_FUNCTION_DEF(mempool, delete);
+LUA_FUNCTION_DEF(mempool, stat);
+LUA_FUNCTION_DEF(mempool, suggest_size);
+/***
+ * @method mempool:set_variable(name, [value1[, value2 ...]])
+ * Sets a variable that's valid during memory pool lifetime. This function allows
+ * to pack multiple values inside a single variable. Currently supported types are:
+ *
+ * - `string`: packed as null terminated C string (so no `\0` are allowed)
+ * - `number`: packed as C double
+ * - `boolean`: packed as bool
+ * @param {string} name variable's name to set
+ */
+LUA_FUNCTION_DEF(mempool, set_variable);
+/***
+ * @method mempool:set_bucket(name, num_values, [value1...valuen]|[table])
+ * Stores a variable bucket of numbers where the first number is number of elements to pack
+ * and then there should be either n numeric values or a plain table of numeric values
+ * @param {string} name variable's name to set
+ * @param {number} num_values number of variables in the bucket
+ * @param {table|list} values values
+ */
+LUA_FUNCTION_DEF(mempool, set_bucket);
+/***
+ * @method mempool:get_variable(name[, type])
+ * Unpacks mempool variable to lua If `type` is not specified, then a variable is
+ * assumed to be zero-terminated C string. Otherwise, `type` is a comma separated (spaces are ignored)
+ * list of types that should be unpacked from a variable's content. The following types
+ * are supported:
+ *
+ * - `string`: null terminated C string (so no `\0` are allowed)
+ * - `double`: returned as lua number
+ * - `int`: unpack a single integer
+ * - `int64`: unpack 64-bits integer
+ * - `boolean`: unpack boolean
+ * - `bucket`: bucket of numbers represented as a Lua table
+ * - `fstrings`: list of rspamd_fstring_t (GList) represented as a Lua table
+ * @param {string} name variable's name to get
+ * @param {string} type list of types to be extracted
+ * @return {variable list} list of variables extracted (but **not** a table)
+ */
+LUA_FUNCTION_DEF(mempool, get_variable);
+/***
+ * @method mempool:has_variable(name)
+ * Checks if the specified variable `name` exists in the memory pool
+ * @param {string} name variable's name to get
+ * @return {boolean} `true` if variable exists and `false` otherwise
+ */
+LUA_FUNCTION_DEF(mempool, has_variable);
+
+/***
+ * @method mempool:delete_variable(name)
+ * Removes the specified variable `name` from the memory pool
+ * @param {string} name variable's name to remove
+ * @return {boolean} `true` if variable exists and has been removed
+ */
+LUA_FUNCTION_DEF(mempool, delete_variable);
+/**
+ * @method mempool:topointer()
+ *
+ * Returns raw C pointer (lightuserdata) associated with mempool. This might be
+ * broken with luajit and GC64, use with caution.
+ */
+LUA_FUNCTION_DEF(mempool, topointer);
+
+static const struct luaL_reg mempoollib_m[] = {
+ LUA_INTERFACE_DEF(mempool, add_destructor),
+ LUA_INTERFACE_DEF(mempool, stat),
+ LUA_INTERFACE_DEF(mempool, suggest_size),
+ LUA_INTERFACE_DEF(mempool, set_variable),
+ LUA_INTERFACE_DEF(mempool, set_bucket),
+ LUA_INTERFACE_DEF(mempool, get_variable),
+ LUA_INTERFACE_DEF(mempool, has_variable),
+ LUA_INTERFACE_DEF(mempool, delete_variable),
+ LUA_INTERFACE_DEF(mempool, topointer),
+ LUA_INTERFACE_DEF(mempool, delete),
+ {"destroy", lua_mempool_delete},
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static const struct luaL_reg mempoollib_f[] = {
+ LUA_INTERFACE_DEF(mempool, create),
+ {NULL, NULL}};
+
+/*
+ * Struct for lua destructor
+ */
+
+struct lua_mempool_udata {
+ lua_State *L;
+ gint cbref;
+ rspamd_mempool_t *mempool;
+};
+
+struct memory_pool_s *
+rspamd_lua_check_mempool(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{mempool}");
+ luaL_argcheck(L, ud != NULL, pos, "'mempool' expected");
+ return ud ? *((struct memory_pool_s **) ud) : NULL;
+}
+
+
+static int
+lua_mempool_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_mempool_new(
+ rspamd_mempool_suggest_size(), "lua", 0),
+ **pmempool;
+
+ if (mempool) {
+ pmempool = lua_newuserdata(L, sizeof(struct memory_pool_s *));
+ rspamd_lua_setclass(L, "rspamd{mempool}", -1);
+ *pmempool = mempool;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static void
+lua_mempool_destructor_func(gpointer p)
+{
+ struct lua_mempool_udata *ud = p;
+
+ lua_rawgeti(ud->L, LUA_REGISTRYINDEX, ud->cbref);
+ if (lua_pcall(ud->L, 0, 0, 0) != 0) {
+ msg_info("call to destructor failed: %s", lua_tostring(ud->L, -1));
+ lua_pop(ud->L, 1);
+ }
+ luaL_unref(ud->L, LUA_REGISTRYINDEX, ud->cbref);
+}
+
+static int
+lua_mempool_add_destructor(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ struct lua_mempool_udata *ud;
+
+ if (mempool) {
+ if (lua_isfunction(L, 2)) {
+ ud = rspamd_mempool_alloc(mempool,
+ sizeof(struct lua_mempool_udata));
+ lua_pushvalue(L, 2);
+ /* Get a reference */
+ ud->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ ud->L = L;
+ ud->mempool = mempool;
+ rspamd_mempool_add_destructor(mempool,
+ lua_mempool_destructor_func,
+ ud);
+ }
+ else {
+ msg_err("trying to add destructor without function");
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static int
+lua_mempool_delete(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+
+ if (mempool) {
+ rspamd_mempool_delete(mempool);
+ return 0;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static int
+lua_mempool_stat(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+
+ if (mempool) {
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static int
+lua_mempool_suggest_size(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+
+ if (mempool) {
+ lua_pushinteger(L, rspamd_mempool_suggest_size());
+ return 0;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+struct lua_numbers_bucket {
+ guint nelts;
+ gdouble elts[0];
+};
+
+static int
+lua_mempool_set_bucket(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ const gchar *var = luaL_checkstring(L, 2);
+ struct lua_numbers_bucket *bucket;
+ gint nelts = luaL_checknumber(L, 3), i;
+
+ if (var && nelts > 0) {
+ bucket = rspamd_mempool_alloc(mempool,
+ sizeof(*bucket) + sizeof(gdouble) * nelts);
+ bucket->nelts = nelts;
+
+ if (lua_type(L, 4) == LUA_TTABLE) {
+ /* Table version */
+ for (i = 1; i <= nelts; i++) {
+ lua_rawgeti(L, 4, i);
+ bucket->elts[i - 1] = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ for (i = 0; i <= nelts; i++) {
+ bucket->elts[i] = lua_tonumber(L, 4 + i);
+ }
+ }
+
+ rspamd_mempool_set_variable(mempool, var, bucket, NULL);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static int
+lua_mempool_set_variable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ const gchar *var = luaL_checkstring(L, 2);
+ gpointer value;
+ struct lua_numbers_bucket *bucket;
+ gchar *vp;
+ union {
+ gdouble d;
+ const gchar *s;
+ gboolean b;
+ } val;
+ gsize slen;
+ gint i, j, len = 0, type;
+
+ if (mempool && var) {
+
+ for (i = 3; i <= lua_gettop(L); i++) {
+ type = lua_type(L, i);
+
+ if (type == LUA_TNUMBER) {
+ /* We have some ambiguity here between integer and double */
+ len += sizeof(gdouble);
+ }
+ else if (type == LUA_TBOOLEAN) {
+ len += sizeof(gboolean);
+ }
+ else if (type == LUA_TSTRING) {
+ (void) lua_tolstring(L, i, &slen);
+ len += slen + 1;
+ }
+ else if (type == LUA_TTABLE) {
+ /* We assume it as a bucket of numbers so far */
+ slen = rspamd_lua_table_size(L, i);
+ len += sizeof(gdouble) * slen + sizeof(*bucket);
+ }
+ else {
+ msg_err("cannot handle lua type %s", lua_typename(L, type));
+ }
+ }
+
+ if (len == 0) {
+ msg_err("no values specified");
+ }
+ else {
+ value = rspamd_mempool_alloc(mempool, len);
+ vp = value;
+
+ for (i = 3; i <= lua_gettop(L); i++) {
+ type = lua_type(L, i);
+
+ if (type == LUA_TNUMBER) {
+ val.d = lua_tonumber(L, i);
+ memcpy(vp, &val, sizeof(gdouble));
+ vp += sizeof(gdouble);
+ }
+ else if (type == LUA_TBOOLEAN) {
+ val.b = lua_toboolean(L, i);
+ memcpy(vp, &val, sizeof(gboolean));
+ vp += sizeof(gboolean);
+ }
+ else if (type == LUA_TSTRING) {
+ val.s = lua_tolstring(L, i, &slen);
+ memcpy(vp, val.s, slen + 1);
+ vp += slen + 1;
+ }
+ else if (type == LUA_TTABLE) {
+ slen = rspamd_lua_table_size(L, i);
+ /* XXX: Ret, ret, ret: alignment issues */
+ bucket = (struct lua_numbers_bucket *) vp;
+ bucket->nelts = slen;
+
+ for (j = 0; j < slen; j++) {
+ lua_rawgeti(L, i, j + 1);
+ bucket->elts[j] = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+ }
+
+ vp += sizeof(gdouble) * slen + sizeof(*bucket);
+ }
+ else {
+ msg_err("cannot handle lua type %s", lua_typename(L, type));
+ }
+ }
+
+ rspamd_mempool_set_variable(mempool, var, value, NULL);
+ }
+
+ return 0;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static int
+lua_mempool_get_variable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ const gchar *var = luaL_checkstring(L, 2);
+ const gchar *type = NULL, *pt;
+ struct lua_numbers_bucket bucket;
+ const gchar *value, *pv;
+ guint len, nvar, slen, i;
+
+ if (mempool && var) {
+ value = rspamd_mempool_get_variable(mempool, var);
+
+ if (lua_gettop(L) >= 3) {
+ type = luaL_checkstring(L, 3);
+ }
+
+ if (value) {
+
+ if (type) {
+ pt = type;
+ pv = value;
+ nvar = 0;
+
+ while ((len = strcspn(pt, ", ")) > 0) {
+ if (len == sizeof("double") - 1 &&
+ g_ascii_strncasecmp(pt, "double", len) == 0) {
+ gdouble num;
+ memcpy(&num, pv, sizeof(gdouble));
+ lua_pushnumber(L, num);
+ pv += sizeof(gdouble);
+ }
+ else if (len == sizeof("int") - 1 &&
+ g_ascii_strncasecmp(pt, "int", len) == 0) {
+ gint num;
+ memcpy(&num, pv, sizeof(gint));
+ lua_pushinteger(L, num);
+ pv += sizeof(gint);
+ }
+ else if (len == sizeof("int64") - 1 &&
+ g_ascii_strncasecmp(pt, "int64", len) == 0) {
+ gint64 num;
+ memcpy(&num, pv, sizeof(gint64));
+ lua_pushinteger(L, num);
+ pv += sizeof(gint64);
+ }
+ else if (len == sizeof("bool") - 1 &&
+ g_ascii_strncasecmp(pt, "bool", len) == 0) {
+ gboolean num;
+ memcpy(&num, pv, sizeof(gboolean));
+ lua_pushboolean(L, num);
+ pv += sizeof(gboolean);
+ }
+ else if (len == sizeof("string") - 1 &&
+ g_ascii_strncasecmp(pt, "string", len) == 0) {
+ slen = strlen((const gchar *) pv);
+ lua_pushlstring(L, (const gchar *) pv, slen);
+ pv += slen + 1;
+ }
+ else if (len == sizeof("gstring") - 1 &&
+ g_ascii_strncasecmp(pt, "gstring", len) == 0) {
+ GString *st = (GString *) pv;
+ lua_pushlstring(L, st->str, st->len);
+ pv += sizeof(GString *);
+ }
+ else if (len == sizeof("bucket") - 1 &&
+ g_ascii_strncasecmp(pt, "bucket", len) == 0) {
+ memcpy(&bucket, pv, sizeof(bucket));
+ lua_createtable(L, bucket.nelts, 0);
+ pv += sizeof(struct lua_numbers_bucket);
+
+ for (i = 0; i < bucket.nelts; i++) {
+ gdouble num;
+ memcpy(&num, pv, sizeof(num));
+ lua_pushnumber(L, num);
+ lua_rawseti(L, -2, i + 1);
+ pv += sizeof(num);
+ }
+ }
+ else if (len == sizeof("fstrings") - 1 &&
+ g_ascii_strncasecmp(pt, "fstrings", len) == 0) {
+ GList *cur;
+ rspamd_fstring_t *fstr;
+
+ cur = (GList *) pv;
+ lua_newtable(L);
+
+ i = 1;
+ while (cur != NULL) {
+ fstr = cur->data;
+ lua_pushlstring(L, fstr->str, fstr->len);
+ lua_rawseti(L, -2, i);
+ i++;
+ cur = g_list_next(cur);
+ }
+
+ pv += sizeof(GList *);
+ }
+ else {
+ msg_err("unknown type for get_variable: %s", pt);
+ lua_pushnil(L);
+ }
+
+ pt += len;
+ pt += strspn(pt, ", ");
+
+ nvar++;
+ }
+
+ return nvar;
+ }
+ else {
+ /* No type specified, return string */
+ lua_pushstring(L, value);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static int
+lua_mempool_has_variable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ const gchar *var = luaL_checkstring(L, 2);
+ gboolean ret = FALSE;
+
+ if (mempool && var) {
+ if (rspamd_mempool_get_variable(mempool, var) != NULL) {
+ ret = TRUE;
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return 1;
+}
+
+static int
+lua_mempool_delete_variable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ const gchar *var = luaL_checkstring(L, 2);
+ gboolean ret = FALSE;
+
+ if (mempool && var) {
+ if (rspamd_mempool_get_variable(mempool, var) != NULL) {
+ ret = TRUE;
+
+ rspamd_mempool_remove_variable(mempool, var);
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_mempool_topointer(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_mempool_t *pool = rspamd_lua_check_mempool(L, 1);
+
+ if (pool) {
+ /* XXX: this might cause issues on arm64 and LuaJIT */
+ lua_pushlightuserdata(L, pool);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_mempool(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, mempoollib_f);
+
+ return 1;
+}
+
+void luaopen_mempool(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{mempool}", mempoollib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_mempool", lua_load_mempool);
+}
diff --git a/src/lua/lua_mimepart.c b/src/lua/lua_mimepart.c
new file mode 100644
index 0000000..5d4b8b7
--- /dev/null
+++ b/src/lua/lua_mimepart.c
@@ -0,0 +1,2304 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "lua_url.h"
+#include "libmime/message.h"
+#include "libmime/lang_detection.h"
+#include "libstat/stat_api.h"
+#include "libcryptobox/cryptobox.h"
+#include "libutil/shingles.h"
+
+#include "contrib/uthash/utlist.h"
+
+/* Textpart methods */
+/***
+ * @module rspamd_textpart
+ * This module provides different methods to manipulate text parts data. Text parts
+ * could be obtained from the `rspamd_task` by using of method `task:get_text_parts()`
+@example
+rspamd_config.R_EMPTY_IMAGE = function (task)
+ parts = task:get_text_parts()
+ if parts then
+ for _,part in ipairs(parts) do
+ if part:is_empty() then
+ texts = task:get_texts()
+ if texts then
+ return true
+ end
+ return false
+ end
+ end
+ end
+ return false
+end
+ */
+
+/***
+ * @method text_part:is_utf()
+ * Return TRUE if part is a valid utf text
+ * @return {boolean} true if part is valid `UTF8` part
+ */
+LUA_FUNCTION_DEF(textpart, is_utf);
+
+/***
+ * @method text_part:has_8bit_raw()
+ * Return TRUE if a part has raw 8bit characters
+ * @return {boolean} true if a part has raw 8bit characters
+ */
+LUA_FUNCTION_DEF(textpart, has_8bit_raw);
+
+/***
+ * @method text_part:has_8bit()
+ * Return TRUE if a part has raw 8bit characters
+ * @return {boolean} true if a part has encoded 8bit characters
+ */
+LUA_FUNCTION_DEF(textpart, has_8bit);
+
+/***
+ * @method text_part:get_content([type])
+ * Get the text of the part (html tags stripped). Optional `type` defines type of content to get:
+ * - `content` (default): utf8 content with HTML tags stripped and newlines preserved
+ * - `content_oneline`: utf8 content with HTML tags and newlines stripped
+ * - `raw`: raw content, not mime decoded nor utf8 converted
+ * - `raw_parsed`: raw content, mime decoded, not utf8 converted
+ * - `raw_utf`: raw content, mime decoded, utf8 converted (but with HTML tags and newlines)
+ * @return {text} `UTF8` encoded content of the part (zero-copy if not converted to a lua string)
+ */
+LUA_FUNCTION_DEF(textpart, get_content);
+/***
+ * @method text_part:get_raw_content()
+ * Get the original text of the part
+ * @return {text} `UTF8` encoded content of the part (zero-copy if not converted to a lua string)
+ */
+LUA_FUNCTION_DEF(textpart, get_raw_content);
+/***
+ * @method text_part:get_content_oneline()
+ *Get the text of the part (html tags and newlines stripped)
+ * @return {text} `UTF8` encoded content of the part (zero-copy if not converted to a lua string)
+ */
+LUA_FUNCTION_DEF(textpart, get_content_oneline);
+/***
+ * @method text_part:get_length()
+ * Get length of the text of the part
+ * @return {integer} length of part in **bytes**
+ */
+LUA_FUNCTION_DEF(textpart, get_length);
+/***
+ * @method mime_part:get_raw_length()
+ * Get length of the **raw** content of the part (e.g. HTML with tags unstripped)
+ * @return {integer} length of part in **bytes**
+ */
+LUA_FUNCTION_DEF(textpart, get_raw_length);
+/***
+ * @method mime_part:get_urls_length()
+ * Get length of the urls within the part
+ * @return {integer} length of urls in **bytes**
+ */
+LUA_FUNCTION_DEF(textpart, get_urls_length);
+/***
+ * @method mime_part:get_lines_count()
+ * Get lines number in the part
+ * @return {integer} number of lines in the part
+ */
+LUA_FUNCTION_DEF(textpart, get_lines_count);
+/***
+ * @method mime_part:get_stats()
+ * Returns a table with the following data:
+ * - `lines`: number of lines
+ * - `spaces`: number of spaces
+ * - `double_spaces`: double spaces
+ * - `empty_lines`: number of empty lines
+ * - `non_ascii_characters`: number of non ascii characters
+ * - `ascii_characters`: number of ascii characters
+ * @return {table} table of stats
+ */
+LUA_FUNCTION_DEF(textpart, get_stats);
+/***
+ * @method mime_part:get_words_count()
+ * Get words number in the part
+ * @return {integer} number of words in the part
+ */
+LUA_FUNCTION_DEF(textpart, get_words_count);
+
+/***
+ * @method mime_part:get_words([how])
+ * Get words in the part. Optional `how` argument defines type of words returned:
+ * - `stem`: stemmed words (default)
+ * - `norm`: normalised words (utf normalised + lowercased)
+ * - `raw`: raw words in utf (if possible)
+ * - `full`: list of tables, each table has the following fields:
+ * - [1] - stemmed word
+ * - [2] - normalised word
+ * - [3] - raw word
+ * - [4] - flags (table of strings)
+ * @return {table/strings} words in the part
+ */
+LUA_FUNCTION_DEF(textpart, get_words);
+
+/***
+ * @method mime_part:filter_words(regexp, [how][, max]])
+ * Filter words using some regexp:
+ * - `stem`: stemmed words (default)
+ * - `norm`: normalised words (utf normalised + lowercased)
+ * - `raw`: raw words in utf (if possible)
+ * - `full`: list of tables, each table has the following fields:
+ * - [1] - stemmed word
+ * - [2] - normalised word
+ * - [3] - raw word
+ * - [4] - flags (table of strings)
+ * @param {rspamd_regexp} regexp regexp to match
+ * @param {string} how what words to extract
+ * @param {number} max maximum number of hits returned (all hits if <= 0 or nil)
+ * @return {table/strings} words matching regexp
+ */
+LUA_FUNCTION_DEF(textpart, filter_words);
+
+/***
+ * @method text_part:is_empty()
+ * Returns `true` if the specified part is empty
+ * @return {bool} whether a part is empty
+ */
+LUA_FUNCTION_DEF(textpart, is_empty);
+/***
+ * @method text_part:is_html()
+ * Returns `true` if the specified part has HTML content
+ * @return {bool} whether a part is HTML part
+ */
+LUA_FUNCTION_DEF(textpart, is_html);
+/***
+ * @method text_part:get_html()
+ * Returns html content of the specified part
+ * @return {html} html content
+ */
+LUA_FUNCTION_DEF(textpart, get_html);
+/***
+ * @method text_part:get_language()
+ * Returns the code of the most used unicode script in the text part. Does not work with raw parts
+ * @return {string} short abbreviation (such as `ru`) for the script's language
+ */
+LUA_FUNCTION_DEF(textpart, get_language);
+
+/***
+ * @method text_part:get_charset()
+ * Returns part real charset
+ * @return {string} charset of the part
+ */
+LUA_FUNCTION_DEF(textpart, get_charset);
+/***
+ * @method text_part:get_languages()
+ * Returns array of tables of all languages detected for a part:
+ * - 'code': language code (short string)
+ * - 'prob': logarithm of probability
+ * @return {array|tables} all languages detected for the part
+ */
+LUA_FUNCTION_DEF(textpart, get_languages);
+/***
+ * @method text_part:get_fuzzy_hashes(mempool)
+ * @param {rspamd_mempool} mempool - memory pool (usually task pool)
+ * Returns direct hash of textpart as a string and array [1..32] of shingles each represented as a following table:
+ * - [1] - 64 bit fuzzy hash represented as a string
+ * - [2..4] - strings used to generate this hash
+ * @return {string,array|tables} fuzzy hashes calculated
+ */
+LUA_FUNCTION_DEF(textpart, get_fuzzy_hashes);
+/***
+ * @method text_part:get_mimepart()
+ * Returns the mime part object corresponding to this text part
+ * @return {mimepart} mimepart object
+ */
+LUA_FUNCTION_DEF(textpart, get_mimepart);
+
+static const struct luaL_reg textpartlib_m[] = {
+ LUA_INTERFACE_DEF(textpart, is_utf),
+ LUA_INTERFACE_DEF(textpart, has_8bit_raw),
+ LUA_INTERFACE_DEF(textpart, has_8bit),
+ LUA_INTERFACE_DEF(textpart, get_content),
+ LUA_INTERFACE_DEF(textpart, get_raw_content),
+ LUA_INTERFACE_DEF(textpart, get_content_oneline),
+ LUA_INTERFACE_DEF(textpart, get_length),
+ LUA_INTERFACE_DEF(textpart, get_raw_length),
+ LUA_INTERFACE_DEF(textpart, get_urls_length),
+ LUA_INTERFACE_DEF(textpart, get_lines_count),
+ LUA_INTERFACE_DEF(textpart, get_words_count),
+ LUA_INTERFACE_DEF(textpart, get_words),
+ LUA_INTERFACE_DEF(textpart, filter_words),
+ LUA_INTERFACE_DEF(textpart, is_empty),
+ LUA_INTERFACE_DEF(textpart, is_html),
+ LUA_INTERFACE_DEF(textpart, get_html),
+ LUA_INTERFACE_DEF(textpart, get_language),
+ LUA_INTERFACE_DEF(textpart, get_charset),
+ LUA_INTERFACE_DEF(textpart, get_languages),
+ LUA_INTERFACE_DEF(textpart, get_mimepart),
+ LUA_INTERFACE_DEF(textpart, get_stats),
+ LUA_INTERFACE_DEF(textpart, get_fuzzy_hashes),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/* Mimepart methods */
+
+/***
+ * @module rspamd_mimepart
+ * This module provides access to mime parts found in a message
+@example
+rspamd_config.MISSING_CONTENT_TYPE = function(task)
+ local parts = task:get_parts()
+ if parts and #parts > 1 then
+ -- We have more than one part
+ for _,p in ipairs(parts) do
+ local ct = p:get_header('Content-Type')
+ -- And some parts have no Content-Type header
+ if not ct then
+ return true
+ end
+ end
+ end
+ return false
+end
+ */
+
+/***
+ * @method mime_part:get_header(name[, case_sensitive])
+ * Get decoded value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {string} decoded value of a header
+ */
+LUA_FUNCTION_DEF(mimepart, get_header);
+/***
+ * @method mime_part:get_header_raw(name[, case_sensitive])
+ * Get raw value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {string} raw value of a header
+ */
+LUA_FUNCTION_DEF(mimepart, get_header_raw);
+/***
+ * @method mime_part:get_header_full(name[, case_sensitive])
+ * Get raw value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter. This method returns more
+ * information about the header as a list of tables with the following structure:
+ *
+ * - `name` - name of a header
+ * - `value` - raw value of a header
+ * - `decoded` - decoded value of a header
+ * - `tab_separated` - `true` if a header and a value are separated by `tab` character
+ * - `empty_separator` - `true` if there are no separator between a header and a value
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {list of tables} all values of a header as specified above
+@example
+function check_header_delimiter_tab(task, header_name)
+ for _,rh in ipairs(task:get_header_full(header_name)) do
+ if rh['tab_separated'] then return true end
+ end
+ return false
+end
+ */
+LUA_FUNCTION_DEF(mimepart, get_header_full);
+/***
+ * @method mimepart:get_header_count(name[, case_sensitive])
+ * Lightweight version if you need just a header's count
+ * * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {number} number of header's occurrences or 0 if not found
+ */
+LUA_FUNCTION_DEF(mimepart, get_header_count);
+
+/***
+ * @method mimepart:get_raw_headers()
+ * Get all undecoded headers of a mime part as a string
+ * @return {rspamd_text} all raw headers for a message as opaque text
+ */
+LUA_FUNCTION_DEF(mimepart, get_raw_headers);
+
+/***
+ * @method mimepart:get_headers()
+ * Get all undecoded headers of a mime part as a string
+ * @return {rspamd_text} all raw headers for a message as opaque text
+ */
+LUA_FUNCTION_DEF(mimepart, get_headers);
+
+/***
+ * @method mime_part:get_content()
+ * Get the parsed content of part
+ * @return {text} opaque text object (zero-copy if not casted to lua string)
+ */
+LUA_FUNCTION_DEF(mimepart, get_content);
+/***
+ * @method mime_part:get_raw_content()
+ * Get the raw content of part
+ * @return {text} opaque text object (zero-copy if not casted to lua string)
+ */
+LUA_FUNCTION_DEF(mimepart, get_raw_content);
+/***
+ * @method mime_part:get_length()
+ * Get length of the content of the part
+ * @return {integer} length of part in **bytes**
+ */
+LUA_FUNCTION_DEF(mimepart, get_length);
+/***
+ * @method mime_part:get_type()
+ * Extract content-type string of the mime part
+ * @return {string,string} content type in form 'type','subtype'
+ */
+LUA_FUNCTION_DEF(mimepart, get_type);
+
+/***
+ * @method mime_part:get_type_full()
+ * Extract content-type string of the mime part with all attributes
+ * @return {string,string,table} content type in form 'type','subtype', {attrs}
+ */
+LUA_FUNCTION_DEF(mimepart, get_type_full);
+
+/***
+ * @method mime_part:get_detected_type()
+ * Extract content-type string of the mime part. Use lua_magic detection
+ * @return {string,string} content type in form 'type','subtype'
+ */
+LUA_FUNCTION_DEF(mimepart, get_detected_type);
+
+/***
+ * @method mime_part:get_detected_type_full()
+ * Extract content-type string of the mime part with all attributes. Use lua_magic detection
+ * @return {string,string,table} content type in form 'type','subtype', {attrs}
+ */
+LUA_FUNCTION_DEF(mimepart, get_detected_type_full);
+
+/***
+ * @method mime_part:get_detected_ext()
+ * Returns a msdos extension name according to lua_magic detection
+ * @return {string} detected extension (see lua_magic.types)
+ */
+LUA_FUNCTION_DEF(mimepart, get_detected_ext);
+
+/***
+ * @method mime_part:get_cte()
+ * Extract content-transfer-encoding for a part
+ * @return {string} content transfer encoding (e.g. `base64` or `7bit`)
+ */
+LUA_FUNCTION_DEF(mimepart, get_cte);
+
+/***
+ * @method mime_part:get_filename()
+ * Extract filename associated with mime part if it is an attachment
+ * @return {string} filename or `nil` if no file is associated with this part
+ */
+LUA_FUNCTION_DEF(mimepart, get_filename);
+/***
+ * @method mime_part:is_image()
+ * Returns true if mime part is an image
+ * @return {bool} true if a part is an image
+ */
+LUA_FUNCTION_DEF(mimepart, is_image);
+/***
+ * @method mime_part:get_image()
+ * Returns rspamd_image structure associated with this part. This structure has
+ * the following methods:
+ *
+ * * `get_width` - return width of an image in pixels
+ * * `get_height` - return height of an image in pixels
+ * * `get_type` - return string representation of image's type (e.g. 'jpeg')
+ * * `get_filename` - return string with image's file name
+ * * `get_size` - return size in bytes
+ * @return {rspamd_image} image structure or nil if a part is not an image
+ */
+LUA_FUNCTION_DEF(mimepart, get_image);
+/***
+ * @method mime_part:is_archive()
+ * Returns true if mime part is an archive
+ * @return {bool} true if a part is an archive
+ */
+LUA_FUNCTION_DEF(mimepart, is_archive);
+/***
+ * @method mime_part:is_attachment()
+ * Returns true if mime part looks like an attachment
+ * @return {bool} true if a part looks like an attachment
+ */
+LUA_FUNCTION_DEF(mimepart, is_attachment);
+
+/***
+ * @method mime_part:get_archive()
+ * Returns rspamd_archive structure associated with this part. This structure has
+ * the following methods:
+ *
+ * * `get_files` - return list of strings with filenames inside archive
+ * * `get_files_full` - return list of tables with all information about files
+ * * `is_encrypted` - return true if an archive is encrypted
+ * * `get_type` - return string representation of image's type (e.g. 'zip')
+ * * `get_filename` - return string with archive's file name
+ * * `get_size` - return size in bytes
+ * @return {rspamd_archive} archive structure or nil if a part is not an archive
+ */
+LUA_FUNCTION_DEF(mimepart, get_archive);
+/***
+ * @method mime_part:is_multipart()
+ * Returns true if mime part is a multipart part
+ * @return {bool} true if a part is is a multipart part
+ */
+LUA_FUNCTION_DEF(mimepart, is_multipart);
+/***
+ * @method mime_part:is_message()
+ * Returns true if mime part is a message part (message/rfc822)
+ * @return {bool} true if a part is is a message part
+ */
+LUA_FUNCTION_DEF(mimepart, is_message);
+/***
+ * @method mime_part:get_boundary()
+ * Returns boundary for a part (extracted from parent multipart for normal parts and
+ * from the part itself for multipart)
+ * @return {string} boundary value or nil
+ */
+LUA_FUNCTION_DEF(mimepart, get_boundary);
+
+/***
+ * @method mime_part:get_enclosing_boundary()
+ * Returns an enclosing boundary for a part even for multiparts. For normal parts
+ * this method is identical to `get_boundary`
+ * @return {string} boundary value or nil
+ */
+LUA_FUNCTION_DEF(mimepart, get_enclosing_boundary);
+
+/***
+ * @method mime_part:get_children()
+ * Returns rspamd_mimepart table of part's childer. Returns nil if mime part is not multipart
+ * or a message part.
+ * @return {rspamd_mimepart} table of children
+ */
+LUA_FUNCTION_DEF(mimepart, get_children);
+/***
+ * @method mime_part:is_text()
+ * Returns true if mime part is a text part
+ * @return {bool} true if a part is a text part
+ */
+LUA_FUNCTION_DEF(mimepart, is_text);
+/***
+ * @method mime_part:get_text()
+ * Returns rspamd_textpart structure associated with this part.
+ * @return {rspamd_textpart} textpart structure or nil if a part is not an text
+ */
+LUA_FUNCTION_DEF(mimepart, get_text);
+
+/***
+ * @method mime_part:get_digest()
+ * Returns the unique digest for this mime part
+ * @return {string} 128 characters hex string with digest of the part
+ */
+LUA_FUNCTION_DEF(mimepart, get_digest);
+
+/***
+ * @method mime_part:get_id()
+ * Returns the order of the part in parts list
+ * @return {number} index of the part (starting from 1 as it is Lua API)
+ */
+LUA_FUNCTION_DEF(mimepart, get_id);
+/***
+ * @method mime_part:is_broken()
+ * Returns true if mime part has incorrectly specified content type
+ * @return {bool} true if a part has bad content type
+ */
+LUA_FUNCTION_DEF(mimepart, is_broken);
+/***
+ * @method mime_part:headers_foreach(callback, [params])
+ * This method calls `callback` for each header that satisfies some condition.
+ * By default, all headers are iterated unless `callback` returns `true`. Nil or
+ * false means continue of iterations.
+ * Params could be as following:
+ *
+ * - `full`: header value is full table of all attributes @see task:get_header_full for details
+ * - `regexp`: return headers that satisfies the specified regexp
+ * @param {function} callback function from header name and header value
+ * @param {table} params optional parameters
+ */
+LUA_FUNCTION_DEF(mimepart, headers_foreach);
+/***
+ * @method mime_part:get_parent()
+ * Returns parent part for this part
+ * @return {rspamd_mimepart} parent part or nil
+ */
+LUA_FUNCTION_DEF(mimepart, get_parent);
+
+/***
+ * @method mime_part:get_specific()
+ * Returns specific lua content for this part
+ * @return {any} specific lua content
+ */
+LUA_FUNCTION_DEF(mimepart, get_specific);
+
+/***
+ * @method mime_part:set_specific(<any>)
+ * Sets a specific content for this part
+ * @return {any} previous specific lua content (or nil)
+ */
+LUA_FUNCTION_DEF(mimepart, set_specific);
+
+/***
+ * @method mime_part:is_specific(<any>)
+ * Returns true if part has specific lua content
+ * @return {boolean} flag
+ */
+LUA_FUNCTION_DEF(mimepart, is_specific);
+
+/***
+ * @method mime_part:get_urls([need_emails|list_protos][, need_images])
+ * Get all URLs found in a mime part. Telephone urls and emails are not included unless explicitly asked in `list_protos`
+ * @param {boolean} need_emails if `true` then return also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`)
+ * @param {boolean} need_images return urls from images (<img src=...>) as well
+ * @return {table rspamd_url} list of all urls found
+ */
+LUA_FUNCTION_DEF(mimepart, get_urls);
+
+static const struct luaL_reg mimepartlib_m[] = {
+ LUA_INTERFACE_DEF(mimepart, get_content),
+ LUA_INTERFACE_DEF(mimepart, get_raw_content),
+ LUA_INTERFACE_DEF(mimepart, get_length),
+ LUA_INTERFACE_DEF(mimepart, get_type),
+ LUA_INTERFACE_DEF(mimepart, get_type_full),
+ LUA_INTERFACE_DEF(mimepart, get_detected_type),
+ LUA_INTERFACE_DEF(mimepart, get_detected_ext),
+ LUA_INTERFACE_DEF(mimepart, get_detected_type_full),
+ LUA_INTERFACE_DEF(mimepart, get_cte),
+ LUA_INTERFACE_DEF(mimepart, get_filename),
+ LUA_INTERFACE_DEF(mimepart, get_boundary),
+ LUA_INTERFACE_DEF(mimepart, get_enclosing_boundary),
+ LUA_INTERFACE_DEF(mimepart, get_header),
+ LUA_INTERFACE_DEF(mimepart, get_header_raw),
+ LUA_INTERFACE_DEF(mimepart, get_header_full),
+ LUA_INTERFACE_DEF(mimepart, get_header_count),
+ LUA_INTERFACE_DEF(mimepart, get_raw_headers),
+ LUA_INTERFACE_DEF(mimepart, get_headers),
+ LUA_INTERFACE_DEF(mimepart, is_image),
+ LUA_INTERFACE_DEF(mimepart, get_image),
+ LUA_INTERFACE_DEF(mimepart, is_archive),
+ LUA_INTERFACE_DEF(mimepart, get_archive),
+ LUA_INTERFACE_DEF(mimepart, is_multipart),
+ LUA_INTERFACE_DEF(mimepart, is_message),
+ LUA_INTERFACE_DEF(mimepart, get_children),
+ LUA_INTERFACE_DEF(mimepart, get_parent),
+ LUA_INTERFACE_DEF(mimepart, get_urls),
+ LUA_INTERFACE_DEF(mimepart, is_text),
+ LUA_INTERFACE_DEF(mimepart, is_broken),
+ LUA_INTERFACE_DEF(mimepart, is_attachment),
+ LUA_INTERFACE_DEF(mimepart, get_text),
+ LUA_INTERFACE_DEF(mimepart, get_digest),
+ LUA_INTERFACE_DEF(mimepart, get_id),
+ LUA_INTERFACE_DEF(mimepart, headers_foreach),
+ LUA_INTERFACE_DEF(mimepart, get_specific),
+ LUA_INTERFACE_DEF(mimepart, set_specific),
+ LUA_INTERFACE_DEF(mimepart, is_specific),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+
+static struct rspamd_mime_text_part *
+lua_check_textpart(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{textpart}");
+ luaL_argcheck(L, ud != NULL, 1, "'textpart' expected");
+ return ud ? *((struct rspamd_mime_text_part **) ud) : NULL;
+}
+
+static struct rspamd_mime_part *
+lua_check_mimepart(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{mimepart}");
+ luaL_argcheck(L, ud != NULL, 1, "'mimepart' expected");
+ return ud ? *((struct rspamd_mime_part **) ud) : NULL;
+}
+
+
+static gint
+lua_textpart_is_utf(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL || IS_TEXT_PART_EMPTY(part)) {
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+
+ lua_pushboolean(L, IS_TEXT_PART_UTF(part));
+
+ return 1;
+}
+
+
+static gint
+lua_textpart_has_8bit_raw(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part) {
+ if (part->flags & RSPAMD_MIME_TEXT_PART_FLAG_8BIT_RAW) {
+ lua_pushboolean(L, TRUE);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_has_8bit(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part) {
+ if (part->flags & RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED) {
+ lua_pushboolean(L, TRUE);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_textpart_get_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ struct rspamd_lua_text *t;
+ gsize len;
+ const gchar *start, *type = NULL;
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ type = lua_tostring(L, 2);
+ }
+
+ if (!type) {
+ if (IS_TEXT_PART_EMPTY(part)) {
+ lua_pushnil(L);
+ return 1;
+ }
+ start = part->utf_content.begin;
+ len = part->utf_content.len;
+ }
+ else if (strcmp(type, "content") == 0) {
+ if (IS_TEXT_PART_EMPTY(part)) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ start = part->utf_content.begin;
+ len = part->utf_content.len;
+ }
+ else if (strcmp(type, "content_oneline") == 0) {
+ if (IS_TEXT_PART_EMPTY(part)) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ start = part->utf_stripped_content->data;
+ len = part->utf_stripped_content->len;
+ }
+ else if (strcmp(type, "raw_parsed") == 0) {
+ if (part->parsed.len == 0) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ start = part->parsed.begin;
+ len = part->parsed.len;
+ }
+ else if (strcmp(type, "raw_utf") == 0) {
+ if (part->utf_raw_content == NULL || part->utf_raw_content->len == 0) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ start = part->utf_raw_content->data;
+ len = part->utf_raw_content->len;
+ }
+ else if (strcmp(type, "raw") == 0) {
+ if (part->raw.len == 0) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ start = part->raw.begin;
+ len = part->raw.len;
+ }
+ else {
+ return luaL_error(L, "invalid content type: %s", type);
+ }
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ t->start = start;
+ t->len = len;
+ t->flags = 0;
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_raw_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ struct rspamd_lua_text *t;
+
+ if (part == NULL || IS_TEXT_PART_EMPTY(part)) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = part->raw.begin;
+ t->len = part->raw.len;
+ t->flags = 0;
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_content_oneline(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL || IS_TEXT_PART_EMPTY(part)) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_new_text(L, part->utf_stripped_content->data, part->utf_stripped_content->len, FALSE);
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ if (IS_TEXT_PART_EMPTY(part) || part->utf_content.len == 0) {
+ lua_pushinteger(L, 0);
+ }
+ else {
+ lua_pushinteger(L, part->utf_content.len);
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_raw_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushinteger(L, part->raw.len);
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_urls_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ GList *cur;
+ guint total = 0;
+ struct rspamd_process_exception *ex;
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ for (cur = part->exceptions; cur != NULL; cur = g_list_next(cur)) {
+ ex = cur->data;
+
+ if (ex->type == RSPAMD_EXCEPTION_URL) {
+ total += ex->len;
+ }
+ }
+
+ lua_pushinteger(L, total);
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_lines_count(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ if (IS_TEXT_PART_EMPTY(part)) {
+ lua_pushinteger(L, 0);
+ }
+ else {
+ lua_pushinteger(L, part->nlines);
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_words_count(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ if (IS_TEXT_PART_EMPTY(part) || part->utf_words == NULL) {
+ lua_pushinteger(L, 0);
+ }
+ else {
+ lua_pushinteger(L, part->nwords);
+ }
+
+ return 1;
+}
+
+static inline enum rspamd_lua_words_type
+word_extract_type_from_string(const gchar *how_str)
+{
+ enum rspamd_lua_words_type how = RSPAMD_LUA_WORDS_MAX;
+
+ if (strcmp(how_str, "stem") == 0) {
+ how = RSPAMD_LUA_WORDS_STEM;
+ }
+ else if (strcmp(how_str, "norm") == 0) {
+ how = RSPAMD_LUA_WORDS_NORM;
+ }
+ else if (strcmp(how_str, "raw") == 0) {
+ how = RSPAMD_LUA_WORDS_RAW;
+ }
+ else if (strcmp(how_str, "full") == 0) {
+ how = RSPAMD_LUA_WORDS_FULL;
+ }
+
+ return how;
+}
+
+static gint
+lua_textpart_get_words(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ enum rspamd_lua_words_type how = RSPAMD_LUA_WORDS_STEM;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (IS_TEXT_PART_EMPTY(part) || part->utf_words == NULL) {
+ lua_createtable(L, 0, 0);
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 2);
+
+ how = word_extract_type_from_string(how_str);
+
+ if (how == RSPAMD_LUA_WORDS_MAX) {
+ return luaL_error(L, "invalid extraction type: %s", how_str);
+ }
+ }
+
+ return rspamd_lua_push_words(L, part->utf_words, how);
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_filter_words(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 2);
+ gint lim = -1;
+ enum rspamd_lua_words_type how = RSPAMD_LUA_WORDS_STEM;
+
+ if (part == NULL || re == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (IS_TEXT_PART_EMPTY(part) || part->utf_words == NULL) {
+ lua_createtable(L, 0, 0);
+ }
+ else {
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 3);
+
+ how = word_extract_type_from_string(how_str);
+
+ if (how == RSPAMD_LUA_WORDS_MAX) {
+ return luaL_error(L, "invalid extraction type: %s", how_str);
+ }
+ }
+
+ if (lua_type(L, 4) == LUA_TNUMBER) {
+ lim = lua_tointeger(L, 4);
+ }
+
+ guint cnt, i;
+
+ lua_createtable(L, 8, 0);
+
+ for (i = 0, cnt = 1; i < part->utf_words->len; i++) {
+ rspamd_stat_token_t *w = &g_array_index(part->utf_words,
+ rspamd_stat_token_t, i);
+
+ switch (how) {
+ case RSPAMD_LUA_WORDS_STEM:
+ if (w->stemmed.len > 0) {
+ if (rspamd_regexp_match(re->re, w->stemmed.begin,
+ w->stemmed.len, FALSE)) {
+ lua_pushlstring(L, w->stemmed.begin, w->stemmed.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ }
+ break;
+ case RSPAMD_LUA_WORDS_NORM:
+ if (w->normalized.len > 0) {
+ if (rspamd_regexp_match(re->re, w->normalized.begin,
+ w->normalized.len, FALSE)) {
+ lua_pushlstring(L, w->normalized.begin, w->normalized.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ }
+ break;
+ case RSPAMD_LUA_WORDS_RAW:
+ if (w->original.len > 0) {
+ if (rspamd_regexp_match(re->re, w->original.begin,
+ w->original.len, TRUE)) {
+ lua_pushlstring(L, w->original.begin, w->original.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ }
+ break;
+ case RSPAMD_LUA_WORDS_FULL:
+ if (rspamd_regexp_match(re->re, w->normalized.begin,
+ w->normalized.len, FALSE)) {
+ rspamd_lua_push_full_word(L, w);
+ /* Push to the resulting vector */
+ lua_rawseti(L, -2, cnt++);
+ }
+ break;
+ default:
+ break;
+ }
+
+ if (lim > 0 && cnt >= lim) {
+ break;
+ }
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_is_empty(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushboolean(L, IS_TEXT_PART_EMPTY(part));
+
+ return 1;
+}
+
+static gint
+lua_textpart_is_html(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushboolean(L, IS_TEXT_PART_HTML(part));
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_html(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ struct html_content **phc;
+
+ if (part == NULL || part->html == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ phc = lua_newuserdata(L, sizeof(*phc));
+ rspamd_lua_setclass(L, "rspamd{html}", -1);
+ *phc = part->html;
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_language(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part != NULL) {
+ if (part->language != NULL && part->language[0] != '\0') {
+ lua_pushstring(L, part->language);
+ return 1;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_charset(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part != NULL) {
+ if (part->real_charset != NULL) {
+ lua_pushstring(L, part->real_charset);
+ return 1;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_languages(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ guint i;
+ struct rspamd_lang_detector_res *cur;
+
+ if (part != NULL) {
+ if (part->languages != NULL) {
+ lua_createtable(L, part->languages->len, 0);
+
+ PTR_ARRAY_FOREACH(part->languages, i, cur)
+ {
+ lua_createtable(L, 0, 2);
+ lua_pushstring(L, "code");
+ lua_pushstring(L, cur->lang);
+ lua_settable(L, -3);
+ lua_pushstring(L, "prob");
+ lua_pushnumber(L, cur->prob);
+ lua_settable(L, -3);
+
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct lua_shingle_data {
+ guint64 hash;
+ rspamd_ftok_t t1;
+ rspamd_ftok_t t2;
+ rspamd_ftok_t t3;
+};
+
+struct lua_shingle_filter_cbdata {
+ struct rspamd_mime_text_part *part;
+ rspamd_mempool_t *pool;
+};
+
+#define STORE_TOKEN(i, t) \
+ do { \
+ if ((i) < part->utf_words->len) { \
+ word = &g_array_index(part->utf_words, rspamd_stat_token_t, (i)); \
+ sd->t.begin = word->stemmed.begin; \
+ sd->t.len = word->stemmed.len; \
+ } \
+ } while (0)
+
+static guint64
+lua_shingles_filter(guint64 *input, gsize count,
+ gint shno, const guchar *key, gpointer ud)
+{
+ guint64 minimal = G_MAXUINT64;
+ gsize i, min_idx = 0;
+ struct lua_shingle_data *sd;
+ rspamd_stat_token_t *word;
+ struct lua_shingle_filter_cbdata *cbd = (struct lua_shingle_filter_cbdata *) ud;
+ struct rspamd_mime_text_part *part;
+
+ part = cbd->part;
+
+ for (i = 0; i < count; i++) {
+ if (minimal > input[i]) {
+ minimal = input[i];
+ min_idx = i;
+ }
+ }
+
+ sd = rspamd_mempool_alloc0(cbd->pool, sizeof(*sd));
+ sd->hash = minimal;
+
+
+ STORE_TOKEN(min_idx, t1);
+ STORE_TOKEN(min_idx + 1, t2);
+ STORE_TOKEN(min_idx + 2, t3);
+
+ return GPOINTER_TO_SIZE(sd);
+}
+
+#undef STORE_TOKEN
+
+static gint
+lua_textpart_get_fuzzy_hashes(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ rspamd_mempool_t *pool = rspamd_lua_check_mempool(L, 2);
+ guchar key[rspamd_cryptobox_HASHBYTES], digest[rspamd_cryptobox_HASHBYTES],
+ hexdigest[rspamd_cryptobox_HASHBYTES * 2 + 1], numbuf[64];
+ struct rspamd_shingle *sgl;
+ guint i;
+ struct lua_shingle_data *sd;
+ rspamd_cryptobox_hash_state_t st;
+ rspamd_stat_token_t *word;
+ struct lua_shingle_filter_cbdata cbd;
+
+
+ if (part == NULL || pool == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (IS_TEXT_PART_EMPTY(part) || part->utf_words == NULL) {
+ lua_pushnil(L);
+ lua_pushnil(L);
+ }
+ else {
+ /* TODO: add keys and algorithms support */
+ rspamd_cryptobox_hash(key, "rspamd", strlen("rspamd"), NULL, 0);
+
+ /* TODO: add short text support */
+
+ /* Calculate direct hash */
+ rspamd_cryptobox_hash_init(&st, key, rspamd_cryptobox_HASHKEYBYTES);
+
+ for (i = 0; i < part->utf_words->len; i++) {
+ word = &g_array_index(part->utf_words, rspamd_stat_token_t, i);
+ rspamd_cryptobox_hash_update(&st,
+ word->stemmed.begin, word->stemmed.len);
+ }
+
+ rspamd_cryptobox_hash_final(&st, digest);
+
+ rspamd_encode_hex_buf(digest, sizeof(digest), hexdigest,
+ sizeof(hexdigest));
+ lua_pushlstring(L, hexdigest, sizeof(hexdigest) - 1);
+
+ cbd.pool = pool;
+ cbd.part = part;
+ sgl = rspamd_shingles_from_text(part->utf_words, key,
+ pool, lua_shingles_filter, &cbd, RSPAMD_SHINGLES_MUMHASH);
+
+ if (sgl == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_createtable(L, G_N_ELEMENTS(sgl->hashes), 0);
+
+ for (i = 0; i < G_N_ELEMENTS(sgl->hashes); i++) {
+ sd = GSIZE_TO_POINTER(sgl->hashes[i]);
+
+ lua_createtable(L, 4, 0);
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%uL", sd->hash);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, 1);
+
+ /* Tokens */
+ lua_pushlstring(L, sd->t1.begin, sd->t1.len);
+ lua_rawseti(L, -2, 2);
+
+ lua_pushlstring(L, sd->t2.begin, sd->t2.len);
+ lua_rawseti(L, -2, 3);
+
+ lua_pushlstring(L, sd->t3.begin, sd->t3.len);
+ lua_rawseti(L, -2, 4);
+
+ lua_rawseti(L, -2, i + 1); /* Store table */
+ }
+ }
+ }
+
+ return 2;
+}
+
+static gint
+lua_textpart_get_mimepart(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ struct rspamd_mime_part **pmime;
+
+ if (part != NULL) {
+ if (part->mime_part != NULL) {
+ pmime = lua_newuserdata(L, sizeof(struct rspamd_mime_part *));
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+ *pmime = part->mime_part;
+
+ return 1;
+ }
+ }
+
+ lua_pushnil(L);
+ return 1;
+}
+
+/***
+ * @method mime_part:get_stats()
+ * Returns a table with the following data:
+ * -
+ * - `lines`: number of lines
+ * - `spaces`: number of spaces
+ * - `double_spaces`: double spaces
+ * - `empty_lines`: number of empty lines
+ * - `non_ascii_characters`: number of non ascii characters
+ * - `ascii_characters`: number of ascii characters
+ * @return {table} table of stats
+ */
+static gint
+lua_textpart_get_stats(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part != NULL) {
+ lua_createtable(L, 0, 9);
+
+ lua_pushstring(L, "lines");
+ lua_pushinteger(L, part->nlines);
+ lua_settable(L, -3);
+ lua_pushstring(L, "empty_lines");
+ lua_pushinteger(L, part->empty_lines);
+ lua_settable(L, -3);
+ lua_pushstring(L, "spaces");
+ lua_pushinteger(L, part->spaces);
+ lua_settable(L, -3);
+ lua_pushstring(L, "non_spaces");
+ lua_pushinteger(L, part->non_spaces);
+ lua_settable(L, -3);
+ lua_pushstring(L, "double_spaces");
+ lua_pushinteger(L, part->double_spaces);
+ lua_settable(L, -3);
+ lua_pushstring(L, "ascii_characters");
+ lua_pushinteger(L, part->ascii_chars);
+ lua_settable(L, -3);
+ lua_pushstring(L, "non_ascii_characters");
+ lua_pushinteger(L, part->non_ascii_chars);
+ lua_settable(L, -3);
+ lua_pushstring(L, "capital_letters");
+ lua_pushinteger(L, part->capital_letters);
+ lua_settable(L, -3);
+ lua_pushstring(L, "numeric_characters");
+ lua_pushinteger(L, part->numeric_characters);
+ lua_settable(L, -3);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/* Mimepart implementation */
+
+static gint
+lua_mimepart_get_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_lua_text *t;
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = part->parsed_data.begin;
+ t->len = part->parsed_data.len;
+ t->flags = 0;
+
+ if (lua_is_text_binary(t)) {
+ t->flags |= RSPAMD_TEXT_FLAG_BINARY;
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_raw_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_lua_text *t;
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = part->raw_data.begin;
+ t->len = part->raw_data.len;
+ t->flags = 0;
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushinteger(L, part->parsed_data.len);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_type_common(lua_State *L, struct rspamd_content_type *ct,
+ gboolean full)
+{
+
+ GHashTableIter it;
+ gpointer k, v;
+ struct rspamd_content_type_param *param;
+
+ if (ct == NULL) {
+ lua_pushnil(L);
+ lua_pushnil(L);
+ return 2;
+ }
+
+ lua_pushlstring(L, ct->type.begin, ct->type.len);
+ lua_pushlstring(L, ct->subtype.begin, ct->subtype.len);
+
+ if (!full) {
+ return 2;
+ }
+
+ lua_createtable(L, 0, 2 + (ct->attrs ? g_hash_table_size(ct->attrs) : 0));
+
+ if (ct->charset.len > 0) {
+ lua_pushstring(L, "charset");
+ lua_pushlstring(L, ct->charset.begin, ct->charset.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->boundary.len > 0) {
+ lua_pushstring(L, "boundary");
+ lua_pushlstring(L, ct->boundary.begin, ct->boundary.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->attrs) {
+ g_hash_table_iter_init(&it, ct->attrs);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ param = v;
+
+ if (param->name.len > 0 && param->value.len > 0) {
+ /* TODO: think about multiple values here */
+ lua_pushlstring(L, param->name.begin, param->name.len);
+ lua_pushlstring(L, param->value.begin, param->value.len);
+ lua_settable(L, -3);
+ }
+ }
+ }
+
+ return 3;
+}
+
+static gint
+lua_mimepart_get_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return lua_mimepart_get_type_common(L, part->ct, FALSE);
+}
+
+static gint
+lua_mimepart_get_type_full(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return lua_mimepart_get_type_common(L, part->ct, TRUE);
+}
+
+static gint
+lua_mimepart_get_detected_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return lua_mimepart_get_type_common(L, part->detected_ct, FALSE);
+}
+
+static gint
+lua_mimepart_get_detected_type_full(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return lua_mimepart_get_type_common(L, part->detected_ct, TRUE);
+}
+
+static gint
+lua_mimepart_get_detected_ext(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->detected_ext) {
+ lua_pushstring(L, part->detected_ext);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_cte(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushstring(L, rspamd_cte_to_string(part->cte));
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_filename(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL || part->cd == NULL || part->cd->filename.len == 0) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushlstring(L, part->cd->filename.begin, part->cd->filename.len);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_boundary(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L), *parent;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (IS_PART_MULTIPART(part)) {
+ lua_pushlstring(L, part->specific.mp->boundary.begin,
+ part->specific.mp->boundary.len);
+ }
+ else {
+ parent = part->parent_part;
+
+ if (!parent || !IS_PART_MULTIPART(parent)) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushlstring(L, parent->specific.mp->boundary.begin,
+ parent->specific.mp->boundary.len);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_enclosing_boundary(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L), *parent;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ parent = part->parent_part;
+
+ if (!parent || !IS_PART_MULTIPART(parent)) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushlstring(L, parent->specific.mp->boundary.begin,
+ parent->specific.mp->boundary.len);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_header_common(lua_State *L, enum rspamd_lua_task_header_type how)
+{
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ const gchar *name;
+ gboolean strong = FALSE;
+
+ name = luaL_checkstring(L, 2);
+
+ if (name && part) {
+
+ if (lua_isboolean(L, 3)) {
+ strong = lua_toboolean(L, 3);
+ }
+
+ return rspamd_lua_push_header_array(L,
+ name,
+ rspamd_message_get_header_from_hash(part->raw_headers, name, FALSE),
+ how,
+ strong);
+ }
+
+ lua_pushnil(L);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_header_full(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_mimepart_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_FULL);
+}
+
+static gint
+lua_mimepart_get_header(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_mimepart_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_SIMPLE);
+}
+
+static gint
+lua_mimepart_get_header_raw(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_mimepart_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_RAW);
+}
+
+static gint
+lua_mimepart_get_header_count(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_mimepart_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_COUNT);
+}
+
+static gint
+lua_mimepart_get_raw_headers(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_lua_text *t;
+
+ if (part) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = part->raw_headers_str;
+ t->len = part->raw_headers_len;
+ t->flags = 0;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_headers(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ bool need_modified = lua_isnoneornil(L, 2) ? false : lua_toboolean(L, 2);
+
+ if (part) {
+ struct rspamd_mime_header *cur;
+ int i = 1;
+
+ lua_createtable(L, rspamd_mime_headers_count(part->raw_headers), 0);
+ LL_FOREACH2(part->headers_order, cur, ord_next)
+ {
+ if (need_modified && cur->modified_chain) {
+ struct rspamd_mime_header *cur_modified;
+
+ LL_FOREACH(cur->modified_chain, cur_modified)
+ {
+ rspamd_lua_push_header(L, cur_modified, RSPAMD_TASK_HEADER_PUSH_FULL);
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ else {
+ rspamd_lua_push_header(L, cur, RSPAMD_TASK_HEADER_PUSH_FULL);
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+
+static gint
+lua_mimepart_is_image(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, part->part_type == RSPAMD_MIME_PART_IMAGE);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_archive(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, part->part_type == RSPAMD_MIME_PART_ARCHIVE);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_multipart(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, IS_PART_MULTIPART(part) ? true : false);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, IS_PART_MESSAGE(part) ? true : false);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_attachment(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->cd && part->cd->type == RSPAMD_CT_ATTACHMENT) {
+ lua_pushboolean(L, true);
+ }
+ else {
+ /* if has_name and not (image and Content-ID_header_present) */
+ if (part->cd && part->cd->filename.len > 0) {
+ if (part->part_type != RSPAMD_MIME_PART_IMAGE &&
+ rspamd_message_get_header_from_hash(part->raw_headers,
+ "Content-Id", FALSE) == NULL) {
+ /* Filename is presented but no content id and not image */
+ lua_pushboolean(L, true);
+ }
+ else {
+ /* Image or an embedded object */
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ /* No filename */
+ lua_pushboolean(L, false);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_text(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, part->part_type == RSPAMD_MIME_PART_TEXT);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_broken(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->ct) {
+ lua_pushboolean(L, (part->ct->flags & RSPAMD_CONTENT_TYPE_BROKEN) ? true : false);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_image(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_image **pimg;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->part_type != RSPAMD_MIME_PART_IMAGE || part->specific.img == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ pimg = lua_newuserdata(L, sizeof(*pimg));
+ *pimg = part->specific.img;
+ rspamd_lua_setclass(L, "rspamd{image}", -1);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_archive(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_archive **parch;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->part_type != RSPAMD_MIME_PART_ARCHIVE || part->specific.arch == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ parch = lua_newuserdata(L, sizeof(*parch));
+ *parch = part->specific.arch;
+ rspamd_lua_setclass(L, "rspamd{archive}", -1);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_children(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_mime_part **pcur, *cur;
+ guint i;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (!IS_PART_MULTIPART(part) || part->specific.mp->children == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_createtable(L, part->specific.mp->children->len, 0);
+
+ PTR_ARRAY_FOREACH(part->specific.mp->children, i, cur)
+ {
+ pcur = lua_newuserdata(L, sizeof(*pcur));
+ *pcur = cur;
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_parent(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_mime_part **pparent;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->parent_part) {
+ pparent = lua_newuserdata(L, sizeof(*pparent));
+ *pparent = part->parent_part;
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_mimepart_get_text(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_mime_text_part **ppart;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->part_type != RSPAMD_MIME_PART_TEXT || part->specific.txt == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ ppart = lua_newuserdata(L, sizeof(*ppart));
+ *ppart = part->specific.txt;
+ rspamd_lua_setclass(L, "rspamd{textpart}", -1);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_digest(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ gchar digestbuf[rspamd_cryptobox_HASHBYTES * 2 + 1];
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ memset(digestbuf, 0, sizeof(digestbuf));
+ rspamd_encode_hex_buf(part->digest, sizeof(part->digest),
+ digestbuf, sizeof(digestbuf));
+ lua_pushstring(L, digestbuf);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushinteger(L, part->part_number);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_headers_foreach(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ enum rspamd_lua_task_header_type how = RSPAMD_TASK_HEADER_PUSH_SIMPLE;
+ struct rspamd_lua_regexp *re = NULL;
+ struct rspamd_mime_header *hdr, *cur;
+ gint old_top;
+
+ if (part && lua_isfunction(L, 2)) {
+ if (lua_istable(L, 3)) {
+ lua_pushstring(L, "full");
+ lua_gettable(L, 3);
+
+ if (lua_isboolean(L, -1) && lua_toboolean(L, -1)) {
+ how = RSPAMD_TASK_HEADER_PUSH_FULL;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "raw");
+ lua_gettable(L, 3);
+
+ if (lua_isboolean(L, -1) && lua_toboolean(L, -1)) {
+ how = RSPAMD_TASK_HEADER_PUSH_RAW;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "regexp");
+ lua_gettable(L, 3);
+
+ if (lua_isuserdata(L, -1)) {
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, -1, "rspamd{regexp}",
+ struct rspamd_lua_regexp, re);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ if (part->headers_order) {
+ hdr = part->headers_order;
+
+ LL_FOREACH2(hdr, cur, ord_next)
+ {
+ if (re && re->re) {
+ if (!rspamd_regexp_match(re->re, cur->name,
+ strlen(cur->name), FALSE)) {
+ continue;
+ }
+ }
+
+ old_top = lua_gettop(L);
+ lua_pushvalue(L, 2);
+ lua_pushstring(L, cur->name);
+ rspamd_lua_push_header(L, cur, how);
+
+ if (lua_pcall(L, 2, LUA_MULTRET, 0) != 0) {
+ msg_err("call to header_foreach failed: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, old_top);
+ break;
+ }
+ else {
+ if (lua_gettop(L) > old_top) {
+ if (lua_isboolean(L, old_top + 1)) {
+ if (lua_toboolean(L, old_top + 1)) {
+ lua_settop(L, old_top);
+ break;
+ }
+ }
+ }
+ }
+
+ lua_settop(L, old_top);
+ }
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_mimepart_get_specific(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->part_type != RSPAMD_MIME_PART_CUSTOM_LUA) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, part->specific.lua_specific.cbref);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_urls(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ struct lua_tree_cb_data cb;
+ struct rspamd_url *u;
+ static const gint default_protocols_mask = PROTOCOL_HTTP | PROTOCOL_HTTPS |
+ PROTOCOL_FILE | PROTOCOL_FTP;
+ gsize sz, max_urls = 0, i;
+
+ if (part->urls == NULL) {
+ lua_newtable(L);
+
+ return 1;
+ }
+
+ if (!lua_url_cbdata_fill(L, 2, &cb, default_protocols_mask,
+ ~(0), max_urls)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sz = part->urls->len;
+
+ lua_createtable(L, sz, 0);
+
+ PTR_ARRAY_FOREACH(part->urls, i, u)
+ {
+ lua_tree_url_callback(u, u, &cb);
+ }
+
+ lua_url_cbdata_dtor(&cb);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_specific(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, part->part_type == RSPAMD_MIME_PART_CUSTOM_LUA);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_set_specific(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL || lua_isnil(L, 2)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->part_type != RSPAMD_MIME_PART_UNDEFINED &&
+ part->part_type != RSPAMD_MIME_PART_CUSTOM_LUA) {
+ return luaL_error(L,
+ "internal error: trying to set specific lua content on part of type %d",
+ part->part_type);
+ }
+
+ if (part->part_type == RSPAMD_MIME_PART_CUSTOM_LUA) {
+ /* Push old specific data */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, part->specific.lua_specific.cbref);
+ luaL_unref(L, LUA_REGISTRYINDEX, part->specific.lua_specific.cbref);
+ }
+ else {
+ part->part_type = RSPAMD_MIME_PART_CUSTOM_LUA;
+ lua_pushnil(L);
+ }
+
+ /* Now, we push argument on the position 2 and save its reference */
+ lua_pushvalue(L, 2);
+ part->specific.lua_specific.cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ /* Now stack has just a return value as luaL_ref removes value from stack */
+
+ gint ltype = lua_type(L, 2);
+
+ switch (ltype) {
+ case LUA_TTABLE:
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_TABLE;
+ break;
+ case LUA_TSTRING:
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_STRING;
+ break;
+ case LUA_TUSERDATA:
+ if (rspamd_lua_check_udata_maybe(L, 2, "rspamd{text}")) {
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_TEXT;
+ }
+ else {
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_UNKNOWN;
+ }
+ break;
+ case LUA_TFUNCTION:
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_FUNCTION;
+ break;
+ default:
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_UNKNOWN;
+ break;
+ }
+
+ return 1;
+}
+
+void luaopen_textpart(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{textpart}", textpartlib_m);
+ lua_pop(L, 1);
+}
+
+void luaopen_mimepart(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{mimepart}", mimepartlib_m);
+ lua_pop(L, 1);
+}
diff --git a/src/lua/lua_parsers.c b/src/lua/lua_parsers.c
new file mode 100644
index 0000000..1fc71db
--- /dev/null
+++ b/src/lua/lua_parsers.c
@@ -0,0 +1,410 @@
+/*-
+ * Copyright 2020 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "tokenizers/tokenizers.h"
+#include "contrib/uthash/utlist.h"
+#include "libserver/html/html.h"
+#include "libmime/email_addr.h"
+#include "libmime/content_type.h"
+#include "libmime/mime_headers.h"
+#include "libmime/smtp_parsers.h"
+#include "lua_parsers.h"
+
+/***
+ * @module rspamd_parsers
+ * This module contains Lua-C interfaces to Rspamd parsers of different kind.
+ */
+
+/***
+ * @function parsers.tokenize_text(input[, exceptions])
+ * Create tokens from a text using optional exceptions list
+ * @param {text/string} input input data
+ * @param {table} exceptions, a table of pairs containing <start_pos,length> of exceptions in the input
+ * @return {table/strings} list of strings representing words in the text
+ */
+
+
+/***
+ * @function parsers.parse_html(input)
+ * Parses HTML and returns the according text
+ * @param {string|text} in input HTML
+ * @return {rspamd_text} processed text with no HTML tags
+ */
+
+/***
+ * @function parsers.parse_mail_address(str, [pool])
+ * Parses email address and returns a table of tables in the following format:
+ *
+ * - `raw` - the original value without any processing
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * - `flags` - table with following keys set to true if given condition fulfilled:
+ * - [valid] - valid SMTP address in conformity with https://tools.ietf.org/html/rfc5321#section-4.1.
+ * - [ip] - domain is IPv4/IPv6 address
+ * - [braced] - angled `<blah@foo.com>` address
+ * - [quoted] - quoted user part
+ * - [empty] - empty address
+ * - [backslash] - user part contains backslash
+ * - [8bit] - contains 8bit characters
+ *
+ * @param {string} str input string
+ * @param {rspamd_mempool} pool memory pool to use
+ * @return {table/tables} parsed list of mail addresses
+ */
+
+/***
+ * @function parsers.parse_content_type(ct_string, mempool)
+ * Parses content-type string to a table:
+ * - `type`
+ * - `subtype`
+ * - `charset`
+ * - `boundary`
+ * - other attributes
+ *
+ * @param {string} ct_string content type as string
+ * @param {rspamd_mempool} mempool needed to store temporary data (e.g. task pool)
+ * @return table or nil if cannot parse content type
+ */
+
+/***
+ * @function parsers.parse_smtp_date(str[, local_tz])
+ * Converts an SMTP date string to unix timestamp
+ * @param {string} str input string
+ * @param {boolean} local_tz convert to local tz if `true`
+ * @return {number} time as unix timestamp (converted to float)
+ */
+
+static const struct luaL_reg parserslib_f[] = {
+ LUA_INTERFACE_DEF(parsers, tokenize_text),
+ LUA_INTERFACE_DEF(parsers, parse_html),
+ LUA_INTERFACE_DEF(parsers, parse_mail_address),
+ LUA_INTERFACE_DEF(parsers, parse_content_type),
+ LUA_INTERFACE_DEF(parsers, parse_smtp_date),
+
+ {NULL, NULL}};
+
+gint lua_parsers_tokenize_text(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *in = NULL;
+ gsize len = 0, pos, ex_len, i;
+ GList *exceptions = NULL, *cur;
+ struct rspamd_lua_text *t;
+ struct rspamd_process_exception *ex;
+ UText utxt = UTEXT_INITIALIZER;
+ GArray *res;
+ rspamd_stat_token_t *w;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ in = luaL_checklstring(L, 1, &len);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t) {
+ in = t->start;
+ len = t->len;
+ }
+ }
+
+ if (in == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ if (lua_gettop(L) > 1 && lua_type(L, 2) == LUA_TTABLE) {
+ lua_pushvalue(L, 2);
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_rawgeti(L, -1, 1);
+ pos = luaL_checknumber(L, -1);
+ lua_pop(L, 1);
+ lua_rawgeti(L, -1, 2);
+ ex_len = luaL_checknumber(L, -1);
+ lua_pop(L, 1);
+
+ if (ex_len > 0) {
+ ex = g_malloc0(sizeof(*ex));
+ ex->pos = pos;
+ ex->len = ex_len;
+ ex->type = RSPAMD_EXCEPTION_GENERIC;
+ exceptions = g_list_prepend(exceptions, ex);
+ }
+ }
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ if (exceptions) {
+ exceptions = g_list_reverse(exceptions);
+ }
+
+ UErrorCode uc_err = U_ZERO_ERROR;
+ utext_openUTF8(&utxt,
+ in,
+ len,
+ &uc_err);
+
+ res = rspamd_tokenize_text((gchar *) in, len,
+ &utxt,
+ RSPAMD_TOKENIZE_UTF, NULL,
+ exceptions,
+ NULL, NULL, NULL);
+
+ if (res == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_createtable(L, res->len, 0);
+
+ for (i = 0; i < res->len; i++) {
+ w = &g_array_index(res, rspamd_stat_token_t, i);
+ lua_pushlstring(L, w->original.begin, w->original.len);
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+
+ cur = exceptions;
+ while (cur) {
+ ex = cur->data;
+ g_free(ex);
+ cur = g_list_next(cur);
+ }
+
+ g_list_free(exceptions);
+ utext_close(&utxt);
+
+ return 1;
+}
+
+gint lua_parsers_parse_html(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ const gchar *start = NULL;
+ gsize len;
+ GByteArray *in;
+ rspamd_mempool_t *pool;
+ void *hc;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ start = t->start;
+ len = t->len;
+ }
+ }
+ else if (lua_type(L, 1) == LUA_TSTRING) {
+ start = luaL_checklstring(L, 1, &len);
+ }
+
+ if (start != NULL) {
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), NULL, 0);
+ in = g_byte_array_sized_new(len);
+ g_byte_array_append(in, start, len);
+
+ hc = rspamd_html_process_part(pool, in);
+
+ rspamd_ftok_t res;
+ rspamd_html_get_parsed_content(hc, &res);
+ lua_new_text(L, res.begin, res.len, TRUE);
+
+ g_byte_array_free(in, TRUE);
+ rspamd_mempool_delete(pool);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+gint lua_parsers_parse_mail_address(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ GPtrArray *addrs;
+ gsize len;
+ const gchar *str = luaL_checklstring(L, 1, &len);
+ gint max_addrs = luaL_optinteger(L, 3, 10240);
+ rspamd_mempool_t *pool;
+ gboolean own_pool = FALSE;
+
+ if (str) {
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ pool = rspamd_lua_check_mempool(L, 2);
+
+ if (pool == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+ }
+ else {
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "lua parsers", 0);
+ own_pool = TRUE;
+ }
+
+ addrs = rspamd_email_address_from_mime(pool, str, len, NULL, max_addrs);
+
+ if (addrs == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_push_emails_address_list(L, addrs, 0);
+ }
+
+ if (own_pool) {
+ rspamd_mempool_delete(pool);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+gint lua_parsers_parse_content_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gsize len;
+ const gchar *ct_str = luaL_checklstring(L, 1, &len);
+ rspamd_mempool_t *pool = rspamd_lua_check_mempool(L, 2);
+ struct rspamd_content_type *ct;
+
+ if (!ct_str || !pool) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ ct = rspamd_content_type_parse(ct_str, len, pool);
+
+ if (ct == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ GHashTableIter it;
+ gpointer k, v;
+
+ lua_createtable(L, 0, 4 + (ct->attrs ? g_hash_table_size(ct->attrs) : 0));
+
+ if (ct->type.len > 0) {
+ lua_pushstring(L, "type");
+ lua_pushlstring(L, ct->type.begin, ct->type.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->subtype.len > 0) {
+ lua_pushstring(L, "subtype");
+ lua_pushlstring(L, ct->subtype.begin, ct->subtype.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->charset.len > 0) {
+ lua_pushstring(L, "charset");
+ lua_pushlstring(L, ct->charset.begin, ct->charset.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->orig_boundary.len > 0) {
+ lua_pushstring(L, "boundary");
+ lua_pushlstring(L, ct->orig_boundary.begin, ct->orig_boundary.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->attrs) {
+ g_hash_table_iter_init(&it, ct->attrs);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ struct rspamd_content_type_param *param =
+ (struct rspamd_content_type_param *) v,
+ *cur;
+ guint i = 1;
+
+ lua_pushlstring(L, param->name.begin, param->name.len);
+ lua_createtable(L, 1, 0);
+
+ DL_FOREACH(param, cur)
+ {
+ lua_pushlstring(L, cur->value.begin, cur->value.len);
+ lua_rawseti(L, -2, i++);
+ }
+
+ lua_settable(L, -3);
+ }
+ }
+ }
+
+ return 1;
+}
+
+int lua_parsers_parse_smtp_date(lua_State *L)
+{
+ gsize slen;
+ const gchar *str = lua_tolstring(L, 1, &slen);
+ GError *err = NULL;
+
+ if (str == NULL) {
+ return luaL_argerror(L, 1, "invalid argument");
+ }
+
+ time_t tt = rspamd_parse_smtp_date(str, slen, &err);
+
+ if (err == NULL) {
+ if (lua_isboolean(L, 2) && !!lua_toboolean(L, 2)) {
+ struct tm t;
+
+ rspamd_localtime(tt, &t);
+#if !defined(__sun)
+ t.tm_gmtoff = 0;
+#endif
+ t.tm_isdst = 0;
+ tt = mktime(&t);
+ }
+
+ lua_pushnumber(L, tt);
+ }
+ else {
+ lua_pushnil(L);
+ lua_pushstring(L, err->message);
+ g_error_free(err);
+
+ return 2;
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_parsers(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, parserslib_f);
+
+ return 1;
+}
+
+void luaopen_parsers(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_parsers", lua_load_parsers);
+} \ No newline at end of file
diff --git a/src/lua/lua_parsers.h b/src/lua/lua_parsers.h
new file mode 100644
index 0000000..2466938
--- /dev/null
+++ b/src/lua/lua_parsers.h
@@ -0,0 +1,88 @@
+/*-
+ * Copyright 2020 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_LUA_PARSERS_H
+#define RSPAMD_LUA_PARSERS_H
+
+#include "lua_common.h"
+
+/***
+ * @function parsers.tokenize_text(input[, exceptions])
+ * Create tokens from a text using optional exceptions list
+ * @param {text/string} input input data
+ * @param {table} exceptions, a table of pairs containing <start_pos,length> of exceptions in the input
+ * @return {table/strings} list of strings representing words in the text
+ */
+LUA_PUBLIC_FUNCTION_DEF(parsers, tokenize_text);
+
+/***
+ * @function parsers.parse_html(input)
+ * Parses HTML and returns the according text
+ * @param {string|text} in input HTML
+ * @return {rspamd_text} processed text with no HTML tags
+ */
+LUA_PUBLIC_FUNCTION_DEF(parsers, parse_html);
+
+/***
+ * @function parsers.parse_mail_address(str, [pool])
+ * Parses email address and returns a table of tables in the following format:
+ *
+ * - `raw` - the original value without any processing
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * - `flags` - table with following keys set to true if given condition fulfilled:
+ * - [valid] - valid SMTP address in conformity with https://tools.ietf.org/html/rfc5321#section-4.1.
+ * - [ip] - domain is IPv4/IPv6 address
+ * - [braced] - angled `<blah@foo.com>` address
+ * - [quoted] - quoted user part
+ * - [empty] - empty address
+ * - [backslash] - user part contains backslash
+ * - [8bit] - contains 8bit characters
+ *
+ * @param {string} str input string
+ * @param {rspamd_mempool} pool memory pool to use
+ * @return {table/tables} parsed list of mail addresses
+ */
+LUA_PUBLIC_FUNCTION_DEF(parsers, parse_mail_address);
+
+/***
+ * @function parsers.parse_content_type(ct_string, mempool)
+ * Parses content-type string to a table:
+ * - `type`
+ * - `subtype`
+ * - `charset`
+ * - `boundary`
+ * - other attributes
+ *
+ * @param {string} ct_string content type as string
+ * @param {rspamd_mempool} mempool needed to store temporary data (e.g. task pool)
+ * @return table or nil if cannot parse content type
+ */
+LUA_PUBLIC_FUNCTION_DEF(parsers, parse_content_type);
+
+/***
+ * @function parsers.parse_smtp_date(str[, local_tz])
+ * Converts an SMTP date string to unix timestamp
+ * @param {string} str input string
+ * @param {boolean} local_tz convert to local tz if `true`
+ * @return {number} time as unix timestamp (converted to float)
+ */
+LUA_PUBLIC_FUNCTION_DEF(parsers, parse_smtp_date);
+
+
+#endif//RSPAMD_LUA_PARSERS_H
diff --git a/src/lua/lua_redis.c b/src/lua/lua_redis.c
new file mode 100644
index 0000000..1ad3b3d
--- /dev/null
+++ b/src/lua/lua_redis.c
@@ -0,0 +1,1662 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+#include "utlist.h"
+
+#include "contrib/hiredis/hiredis.h"
+#include "contrib/hiredis/async.h"
+
+#define REDIS_DEFAULT_TIMEOUT 1.0
+
+static const gchar *M = "rspamd lua redis";
+static void *redis_null;
+
+/***
+ * @module rspamd_redis
+ * This module implements redis asynchronous client for rspamd LUA API.
+ * Here is an example of using of this module:
+ * @example
+local rspamd_redis = require "rspamd_redis"
+local rspamd_logger = require "rspamd_logger"
+
+local function symbol_callback(task)
+ local redis_key = 'some_key'
+ local function redis_cb(err, data)
+ if not err then
+ rspamd_logger.infox('redis returned %1=%2', redis_key, data)
+ end
+ end
+
+ rspamd_redis.make_request(task, "127.0.0.1:6379", redis_cb,
+ 'GET', {redis_key})
+ -- or in table form:
+ -- rspamd_redis.make_request({task=task, host="127.0.0.1:6379,
+ -- callback=redis_cb, timeout=2.0, cmd='GET', args={redis_key}})
+end
+ */
+
+LUA_FUNCTION_DEF(redis, make_request);
+LUA_FUNCTION_DEF(redis, make_request_sync);
+LUA_FUNCTION_DEF(redis, connect);
+LUA_FUNCTION_DEF(redis, connect_sync);
+LUA_FUNCTION_DEF(redis, add_cmd);
+LUA_FUNCTION_DEF(redis, exec);
+LUA_FUNCTION_DEF(redis, gc);
+
+static const struct luaL_reg redislib_f[] = {
+ LUA_INTERFACE_DEF(redis, make_request),
+ LUA_INTERFACE_DEF(redis, make_request_sync),
+ LUA_INTERFACE_DEF(redis, connect),
+ LUA_INTERFACE_DEF(redis, connect_sync),
+ {NULL, NULL}};
+
+static const struct luaL_reg redislib_m[] = {
+ LUA_INTERFACE_DEF(redis, add_cmd),
+ LUA_INTERFACE_DEF(redis, exec),
+ {"__gc", lua_redis_gc},
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+#undef REDIS_DEBUG_REFS
+#ifdef REDIS_DEBUG_REFS
+#define REDIS_RETAIN(x) \
+ do { \
+ msg_err("retain ref %p, refcount: %d", (x), (x)->ref.refcount); \
+ REF_RETAIN(x); \
+ } while (0)
+
+#define REDIS_RELEASE(x) \
+ do { \
+ msg_err("release ref %p, refcount: %d", (x), (x)->ref.refcount); \
+ REF_RELEASE(x); \
+ } while (0)
+#else
+#define REDIS_RETAIN REF_RETAIN
+#define REDIS_RELEASE REF_RELEASE
+#endif
+
+struct lua_redis_request_specific_userdata;
+/**
+ * Struct for userdata representation
+ */
+struct lua_redis_userdata {
+ redisAsyncContext *ctx;
+ struct rspamd_task *task;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_async_session *s;
+ struct ev_loop *event_loop;
+ struct rspamd_config *cfg;
+ struct rspamd_redis_pool *pool;
+ gchar *server;
+ gchar log_tag[RSPAMD_LOG_ID_LEN + 1];
+ struct lua_redis_request_specific_userdata *specific;
+ gdouble timeout;
+ guint16 port;
+ guint16 terminated;
+};
+
+#define msg_debug_lua_redis(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_lua_redis_log_id, "lua_redis", ud->log_tag, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+INIT_LOG_MODULE(lua_redis)
+
+#define LUA_REDIS_SPECIFIC_REPLIED (1 << 0)
+/* session was finished */
+#define LUA_REDIS_SPECIFIC_FINISHED (1 << 1)
+#define LUA_REDIS_ASYNC (1 << 0)
+#define LUA_REDIS_TEXTDATA (1 << 1)
+#define LUA_REDIS_TERMINATED (1 << 2)
+#define LUA_REDIS_NO_POOL (1 << 3)
+#define LUA_REDIS_SUBSCRIBED (1 << 4)
+#define IS_ASYNC(ctx) ((ctx)->flags & LUA_REDIS_ASYNC)
+
+struct lua_redis_request_specific_userdata {
+ gint cbref;
+ guint nargs;
+ gchar **args;
+ gsize *arglens;
+ struct lua_redis_userdata *c;
+ struct lua_redis_ctx *ctx;
+ struct lua_redis_request_specific_userdata *next;
+ ev_timer timeout_ev;
+ guint flags;
+};
+
+struct lua_redis_ctx {
+ guint flags;
+ struct lua_redis_userdata async;
+ guint cmds_pending;
+ ref_entry_t ref;
+ GQueue *replies; /* for sync connection only */
+ GQueue *events_cleanup; /* for sync connection only */
+ struct thread_entry *thread; /* for sync mode, set only if there was yield */
+};
+
+struct lua_redis_result {
+ gboolean is_error;
+ gint result_ref;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_async_session *s;
+ struct rspamd_task *task;
+ struct lua_redis_request_specific_userdata *sp_ud;
+};
+
+static struct lua_redis_ctx *
+lua_check_redis(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{redis}");
+ luaL_argcheck(L, ud != NULL, pos, "'redis' expected");
+ return ud ? *((struct lua_redis_ctx **) ud) : NULL;
+}
+
+static void
+lua_redis_free_args(char **args, gsize *arglens, guint nargs)
+{
+ guint i;
+
+ if (args) {
+ for (i = 0; i < nargs; i++) {
+ g_free(args[i]);
+ }
+
+ g_free(args);
+ g_free(arglens);
+ }
+}
+
+static void
+lua_redis_dtor(struct lua_redis_ctx *ctx)
+{
+ struct lua_redis_userdata *ud;
+ struct lua_redis_request_specific_userdata *cur, *tmp;
+ gboolean is_successful = TRUE;
+ struct redisAsyncContext *ac;
+
+ ud = &ctx->async;
+ msg_debug_lua_redis("destructing %p", ctx);
+
+ if (ud->ctx) {
+
+ LL_FOREACH_SAFE(ud->specific, cur, tmp)
+ {
+ ev_timer_stop(ud->event_loop, &cur->timeout_ev);
+
+ if (!(cur->flags & LUA_REDIS_SPECIFIC_REPLIED)) {
+ is_successful = FALSE;
+ }
+
+ cur->flags |= LUA_REDIS_SPECIFIC_FINISHED;
+ }
+
+ ctx->flags |= LUA_REDIS_TERMINATED;
+
+ ud->terminated = 1;
+ ac = ud->ctx;
+ ud->ctx = NULL;
+
+ if (!is_successful) {
+ rspamd_redis_pool_release_connection(ud->pool, ac,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ }
+ else {
+ rspamd_redis_pool_release_connection(ud->pool, ac,
+ (ctx->flags & LUA_REDIS_NO_POOL) ? RSPAMD_REDIS_RELEASE_ENFORCE : RSPAMD_REDIS_RELEASE_DEFAULT);
+ }
+ }
+
+ LL_FOREACH_SAFE(ud->specific, cur, tmp)
+ {
+ lua_redis_free_args(cur->args, cur->arglens, cur->nargs);
+
+ if (cur->cbref != -1) {
+ luaL_unref(ud->cfg->lua_state, LUA_REGISTRYINDEX, cur->cbref);
+ }
+
+ g_free(cur);
+ }
+
+ if (ctx->events_cleanup) {
+ g_queue_free(ctx->events_cleanup);
+ ctx->events_cleanup = NULL;
+ }
+ if (ctx->replies) {
+ g_queue_free(ctx->replies);
+ ctx->replies = NULL;
+ }
+
+ g_free(ctx);
+}
+
+static gint
+lua_redis_gc(lua_State *L)
+{
+ struct lua_redis_ctx *ctx = lua_check_redis(L, 1);
+
+ if (ctx) {
+ REDIS_RELEASE(ctx);
+ }
+
+ return 0;
+}
+
+static void
+lua_redis_fin(void *arg)
+{
+ struct lua_redis_request_specific_userdata *sp_ud = arg;
+ struct lua_redis_userdata *ud;
+ struct lua_redis_ctx *ctx;
+
+ ctx = sp_ud->ctx;
+ ud = sp_ud->c;
+
+ if (ev_can_stop(&sp_ud->timeout_ev)) {
+ ev_timer_stop(sp_ud->ctx->async.event_loop, &sp_ud->timeout_ev);
+ }
+
+ msg_debug_lua_redis("finished redis query %p from session %p; refcount=%d",
+ sp_ud, ctx, ctx->ref.refcount);
+ sp_ud->flags |= LUA_REDIS_SPECIFIC_FINISHED;
+
+ REDIS_RELEASE(ctx);
+}
+
+/**
+ * Push error of redis request to lua callback
+ * @param code
+ * @param ud
+ */
+static void
+lua_redis_push_error(const gchar *err,
+ struct lua_redis_ctx *ctx,
+ struct lua_redis_request_specific_userdata *sp_ud,
+ gboolean connected)
+{
+ struct lua_redis_userdata *ud = sp_ud->c;
+ struct lua_callback_state cbs;
+ lua_State *L;
+
+ if (!(sp_ud->flags & (LUA_REDIS_SPECIFIC_REPLIED | LUA_REDIS_SPECIFIC_FINISHED))) {
+ if (sp_ud->cbref != -1) {
+
+ lua_thread_pool_prepare_callback(ud->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ int err_idx = lua_gettop(L);
+ /* Push error */
+ lua_rawgeti(cbs.L, LUA_REGISTRYINDEX, sp_ud->cbref);
+
+ /* String of error */
+ lua_pushstring(cbs.L, err);
+ /* Data is nil */
+ lua_pushnil(cbs.L);
+
+ if (ud->item) {
+ rspamd_symcache_set_cur_item(ud->task, ud->item);
+ }
+
+ if (lua_pcall(cbs.L, 2, 0, err_idx) != 0) {
+ msg_info("call to callback failed: %s", lua_tostring(cbs.L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+ lua_thread_pool_restore_callback(&cbs);
+ }
+
+ sp_ud->flags |= LUA_REDIS_SPECIFIC_REPLIED;
+
+ if (connected && ud->s) {
+ if (ud->item) {
+ rspamd_symcache_item_async_dec_check(ud->task, ud->item, M);
+ }
+
+ rspamd_session_remove_event(ud->s, lua_redis_fin, sp_ud);
+ }
+ else {
+ lua_redis_fin(sp_ud);
+ }
+ }
+}
+
+static void
+lua_redis_push_reply(lua_State *L, const redisReply *r, gboolean text_data)
+{
+ guint i;
+ struct rspamd_lua_text *t;
+
+ switch (r->type) {
+ case REDIS_REPLY_INTEGER:
+ lua_pushinteger(L, r->integer);
+ break;
+ case REDIS_REPLY_NIL:
+ lua_getfield(L, LUA_REGISTRYINDEX, "redis.null");
+ break;
+ case REDIS_REPLY_STRING:
+ case REDIS_REPLY_STATUS:
+ if (text_data) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->flags = 0;
+ t->start = r->str;
+ t->len = r->len;
+ }
+ else {
+ lua_pushlstring(L, r->str, r->len);
+ }
+ break;
+ case REDIS_REPLY_ARRAY:
+ lua_createtable(L, r->elements, 0);
+ for (i = 0; i < r->elements; ++i) {
+ lua_redis_push_reply(L, r->element[i], text_data);
+ lua_rawseti(L, -2, i + 1); /* Store sub-reply */
+ }
+ break;
+ default: /* should not happen */
+ msg_info("unknown reply type: %d", r->type);
+ break;
+ }
+}
+
+/**
+ * Push data of redis request to lua callback
+ * @param r redis reply data
+ * @param ud
+ */
+static void
+lua_redis_push_data(const redisReply *r, struct lua_redis_ctx *ctx,
+ struct lua_redis_request_specific_userdata *sp_ud)
+{
+ struct lua_redis_userdata *ud = sp_ud->c;
+ struct lua_callback_state cbs;
+ lua_State *L;
+
+ if (!(sp_ud->flags & (LUA_REDIS_SPECIFIC_REPLIED | LUA_REDIS_SPECIFIC_FINISHED)) ||
+ (sp_ud->flags & LUA_REDIS_SUBSCRIBED)) {
+ if (sp_ud->cbref != -1) {
+ lua_thread_pool_prepare_callback(ud->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ int err_idx = lua_gettop(L);
+ /* Push error */
+ lua_rawgeti(cbs.L, LUA_REGISTRYINDEX, sp_ud->cbref);
+ /* Error is nil */
+ lua_pushnil(cbs.L);
+ /* Data */
+ lua_redis_push_reply(cbs.L, r, ctx->flags & LUA_REDIS_TEXTDATA);
+
+ if (ud->item) {
+ rspamd_symcache_set_cur_item(ud->task, ud->item);
+ }
+
+ gint ret = lua_pcall(cbs.L, 2, 0, err_idx);
+
+ if (ret != 0) {
+ msg_info("call to lua_redis callback failed (%d): %s",
+ ret, lua_tostring(cbs.L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+ lua_thread_pool_restore_callback(&cbs);
+ }
+
+ if (sp_ud->flags & LUA_REDIS_SUBSCRIBED) {
+ if (!(sp_ud->flags & LUA_REDIS_SPECIFIC_REPLIED)) {
+ if (ev_can_stop(&sp_ud->timeout_ev)) {
+ ev_timer_stop(sp_ud->ctx->async.event_loop,
+ &sp_ud->timeout_ev);
+ }
+ }
+ }
+
+ sp_ud->flags |= LUA_REDIS_SPECIFIC_REPLIED;
+
+ if (!(sp_ud->flags & LUA_REDIS_SUBSCRIBED)) {
+ if (ud->s) {
+ if (ud->item) {
+ rspamd_symcache_item_async_dec_check(ud->task,
+ ud->item, M);
+ }
+
+ rspamd_session_remove_event(ud->s, lua_redis_fin, sp_ud);
+ }
+ else {
+ lua_redis_fin(sp_ud);
+ }
+ }
+ }
+}
+
+/**
+ * Callback for redis replies
+ * @param c context of redis connection
+ * @param r redis reply
+ * @param priv userdata
+ */
+static void
+lua_redis_callback(redisAsyncContext *c, gpointer r, gpointer priv)
+{
+ redisReply *reply = r;
+ struct lua_redis_request_specific_userdata *sp_ud = priv;
+ struct lua_redis_ctx *ctx;
+ struct lua_redis_userdata *ud;
+ redisAsyncContext *ac;
+
+ ctx = sp_ud->ctx;
+ ud = sp_ud->c;
+
+ if (ud->terminated || !rspamd_lua_is_initialised()) {
+ /* We are already at the termination stage, just go out */
+ return;
+ }
+
+ msg_debug_lua_redis("got reply from redis %p for query %p", sp_ud->c->ctx,
+ sp_ud);
+
+ REDIS_RETAIN(ctx);
+
+ /* If session is finished, we cannot call lua callbacks */
+ if (!(sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED) ||
+ (sp_ud->flags & LUA_REDIS_SUBSCRIBED)) {
+ if (c->err == 0) {
+ if (r != NULL) {
+ if (reply->type != REDIS_REPLY_ERROR) {
+ lua_redis_push_data(reply, ctx, sp_ud);
+ }
+ else {
+ lua_redis_push_error(reply->str, ctx, sp_ud, TRUE);
+ }
+ }
+ else {
+ lua_redis_push_error("received no data from server", ctx, sp_ud, TRUE);
+ }
+ }
+ else {
+ if (c->err == REDIS_ERR_IO) {
+ lua_redis_push_error(strerror(errno), ctx, sp_ud, TRUE);
+ }
+ else {
+ lua_redis_push_error(c->errstr, ctx, sp_ud, TRUE);
+ }
+ }
+ }
+
+ if (!(sp_ud->flags & LUA_REDIS_SUBSCRIBED)) {
+ ctx->cmds_pending--;
+
+ if (ctx->cmds_pending == 0 && !ud->terminated) {
+ /* Disconnect redis early as we don't need it anymore */
+ ud->terminated = 1;
+ ac = ud->ctx;
+ ud->ctx = NULL;
+
+ if (ac) {
+ msg_debug_lua_redis("release redis connection ud=%p; ctx=%p; refcount=%d",
+ ud, ctx, ctx->ref.refcount);
+ rspamd_redis_pool_release_connection(ud->pool, ac,
+ (ctx->flags & LUA_REDIS_NO_POOL) ? RSPAMD_REDIS_RELEASE_ENFORCE : RSPAMD_REDIS_RELEASE_DEFAULT);
+ }
+ }
+ }
+
+ REDIS_RELEASE(ctx);
+}
+
+static gint
+lua_redis_push_results(struct lua_redis_ctx *ctx, lua_State *L)
+{
+ gint results = g_queue_get_length(ctx->replies);
+ gint i;
+ gboolean can_use_lua = TRUE;
+
+ results = g_queue_get_length(ctx->replies);
+
+ if (!lua_checkstack(L, (results * 2) + 1)) {
+ luaL_error(L, "cannot resize stack to fit %d commands",
+ ctx->cmds_pending);
+
+ can_use_lua = FALSE;
+ }
+
+ for (i = 0; i < results; i++) {
+ struct lua_redis_result *result = g_queue_pop_head(ctx->replies);
+
+ if (can_use_lua) {
+ lua_pushboolean(L, !result->is_error);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, result->result_ref);
+ }
+
+ luaL_unref(L, LUA_REGISTRYINDEX, result->result_ref);
+
+ g_queue_push_tail(ctx->events_cleanup, result);
+ }
+
+ return can_use_lua ? results * 2 : 0;
+}
+
+static void
+lua_redis_cleanup_events(struct lua_redis_ctx *ctx)
+{
+ REDIS_RETAIN(ctx); /* To avoid preliminary destruction */
+
+ while (!g_queue_is_empty(ctx->events_cleanup)) {
+ struct lua_redis_result *result = g_queue_pop_head(ctx->events_cleanup);
+
+ if (result->item) {
+ rspamd_symcache_item_async_dec_check(result->task, result->item, M);
+ }
+
+ if (result->s) {
+ rspamd_session_remove_event(result->s, lua_redis_fin, result->sp_ud);
+ }
+ else {
+ lua_redis_fin(result->sp_ud);
+ }
+
+ g_free(result);
+ }
+
+ REDIS_RELEASE(ctx);
+}
+
+/**
+ * Callback for redis replies
+ * @param c context of redis connection
+ * @param r redis reply
+ * @param priv userdata
+ */
+static void
+lua_redis_callback_sync(redisAsyncContext *ac, gpointer r, gpointer priv)
+{
+ redisReply *reply = r;
+
+ struct lua_redis_request_specific_userdata *sp_ud = priv;
+ struct lua_redis_ctx *ctx;
+ struct lua_redis_userdata *ud;
+ struct thread_entry *thread;
+ gint results;
+
+ ctx = sp_ud->ctx;
+ ud = sp_ud->c;
+ lua_State *L = ctx->async.cfg->lua_state;
+
+ sp_ud->flags |= LUA_REDIS_SPECIFIC_REPLIED;
+
+ if (ud->terminated) {
+ /* We are already at the termination stage, just go out */
+ /* TODO:
+ if somebody is waiting for us (ctx->thread), return result,
+ otherwise, indeed, ignore
+ */
+ return;
+ }
+
+ if (ev_can_stop(&sp_ud->timeout_ev)) {
+ ev_timer_stop(ud->event_loop, &sp_ud->timeout_ev);
+ }
+
+ if (!(sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED)) {
+ msg_debug_lua_redis("got reply from redis: %p for query %p", ac, sp_ud);
+
+ struct lua_redis_result *result = g_malloc0(sizeof *result);
+
+ if (ac->err == 0) {
+ if (r != NULL) {
+ if (reply->type != REDIS_REPLY_ERROR) {
+ result->is_error = FALSE;
+ lua_redis_push_reply(L, reply, ctx->flags & LUA_REDIS_TEXTDATA);
+ }
+ else {
+ result->is_error = TRUE;
+ lua_pushstring(L, reply->str);
+ }
+ }
+ else {
+ result->is_error = TRUE;
+ lua_pushliteral(L, "received no data from server");
+ }
+ }
+ else {
+ result->is_error = TRUE;
+ if (ac->err == REDIS_ERR_IO) {
+ lua_pushstring(L, strerror(errno));
+ }
+ else {
+ lua_pushstring(L, ac->errstr);
+ }
+ }
+
+ /* if error happened, we should terminate the connection,
+ and release it */
+
+ if (result->is_error && sp_ud->c->ctx) {
+ ac = sp_ud->c->ctx;
+ /* Set to NULL to avoid double free in dtor */
+ sp_ud->c->ctx = NULL;
+ ctx->flags |= LUA_REDIS_TERMINATED;
+
+ /*
+ * This will call all callbacks pending so the entire context
+ * will be destructed
+ */
+ rspamd_redis_pool_release_connection(sp_ud->c->pool, ac,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ }
+
+ result->result_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ result->s = ud->s;
+ result->item = ud->item;
+ result->task = ud->task;
+ result->sp_ud = sp_ud;
+
+ g_queue_push_tail(ctx->replies, result);
+ }
+
+ ctx->cmds_pending--;
+
+ if (ctx->cmds_pending == 0) {
+ if (ctx->thread) {
+ if (!(sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED)) {
+ /* somebody yielded and waits for results */
+ thread = ctx->thread;
+ ctx->thread = NULL;
+
+ results = lua_redis_push_results(ctx, thread->lua_state);
+
+ if (ud->item) {
+ rspamd_symcache_set_cur_item(ud->task, ud->item);
+ }
+
+ lua_thread_resume(thread, results);
+ lua_redis_cleanup_events(ctx);
+ }
+ else {
+ /* We cannot resume the thread as the associated task has gone */
+ lua_thread_pool_terminate_entry_full(ud->cfg->lua_thread_pool,
+ ctx->thread, G_STRLOC, true);
+ ctx->thread = NULL;
+ }
+ }
+ }
+}
+
+static void
+lua_redis_timeout_sync(EV_P_ ev_timer *w, int revents)
+{
+ struct lua_redis_request_specific_userdata *sp_ud =
+ (struct lua_redis_request_specific_userdata *) w->data;
+ struct lua_redis_ctx *ctx;
+ struct lua_redis_userdata *ud;
+ redisAsyncContext *ac;
+
+ if (sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED) {
+ return;
+ }
+
+ ud = sp_ud->c;
+ ctx = sp_ud->ctx;
+ msg_debug_lua_redis("timeout while querying redis server: %p, redis: %p", sp_ud,
+ sp_ud->c->ctx);
+
+ if (sp_ud->c->ctx) {
+ ac = sp_ud->c->ctx;
+
+ /* Set to NULL to avoid double free in dtor */
+ sp_ud->c->ctx = NULL;
+ ac->err = REDIS_ERR_IO;
+ errno = ETIMEDOUT;
+ ctx->flags |= LUA_REDIS_TERMINATED;
+
+ /*
+ * This will call all callbacks pending so the entire context
+ * will be destructed
+ */
+ rspamd_redis_pool_release_connection(sp_ud->c->pool, ac,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ }
+}
+
+static void
+lua_redis_timeout(EV_P_ ev_timer *w, int revents)
+{
+ struct lua_redis_request_specific_userdata *sp_ud =
+ (struct lua_redis_request_specific_userdata *) w->data;
+ struct lua_redis_userdata *ud;
+ struct lua_redis_ctx *ctx;
+ redisAsyncContext *ac;
+
+ if (sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED) {
+ return;
+ }
+
+ ctx = sp_ud->ctx;
+ ud = sp_ud->c;
+
+ REDIS_RETAIN(ctx);
+ msg_debug_lua_redis("timeout while querying redis server: %p, redis: %p", sp_ud,
+ sp_ud->c->ctx);
+ lua_redis_push_error("timeout while connecting the server", ctx, sp_ud, TRUE);
+
+ if (sp_ud->c->ctx) {
+ ac = sp_ud->c->ctx;
+ /* Set to NULL to avoid double free in dtor */
+ sp_ud->c->ctx = NULL;
+ ac->err = REDIS_ERR_IO;
+ errno = ETIMEDOUT;
+ /*
+ * This will call all callbacks pending so the entire context
+ * will be destructed
+ */
+ rspamd_redis_pool_release_connection(sp_ud->c->pool, ac,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ }
+
+ REDIS_RELEASE(ctx);
+}
+
+
+static void
+lua_redis_parse_args(lua_State *L, gint idx, const gchar *cmd,
+ gchar ***pargs, gsize **parglens, guint *nargs)
+{
+ gchar **args = NULL;
+ gsize *arglens;
+ gint top;
+
+ if (idx != 0 && lua_type(L, idx) == LUA_TTABLE) {
+ /* Get all arguments */
+ lua_pushvalue(L, idx);
+ lua_pushnil(L);
+ top = 0;
+
+ while (lua_next(L, -2) != 0) {
+ gint type = lua_type(L, -1);
+
+ if (type == LUA_TNUMBER || type == LUA_TSTRING ||
+ type == LUA_TUSERDATA) {
+ top++;
+ }
+ lua_pop(L, 1);
+ }
+
+ args = g_malloc((top + 1) * sizeof(gchar *));
+ arglens = g_malloc((top + 1) * sizeof(gsize));
+ arglens[0] = strlen(cmd);
+ args[0] = g_malloc(arglens[0]);
+ memcpy(args[0], cmd, arglens[0]);
+ top = 1;
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ gint type = lua_type(L, -1);
+
+ if (type == LUA_TSTRING) {
+ const gchar *s;
+
+ s = lua_tolstring(L, -1, &arglens[top]);
+ args[top] = g_malloc(arglens[top]);
+ memcpy(args[top], s, arglens[top]);
+ top++;
+ }
+ else if (type == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t;
+
+ t = lua_check_text(L, -1);
+
+ if (t && t->start) {
+ arglens[top] = t->len;
+ args[top] = g_malloc(arglens[top]);
+ memcpy(args[top], t->start, arglens[top]);
+ top++;
+ }
+ }
+ else if (type == LUA_TNUMBER) {
+ gdouble val = lua_tonumber(L, -1);
+ gint r;
+ gchar numbuf[64];
+
+ if (val == (gdouble) ((gint64) val)) {
+ r = rspamd_snprintf(numbuf, sizeof(numbuf), "%L",
+ (gint64) val);
+ }
+ else {
+ r = rspamd_snprintf(numbuf, sizeof(numbuf), "%f",
+ val);
+ }
+
+ arglens[top] = r;
+ args[top] = g_malloc(arglens[top]);
+ memcpy(args[top], numbuf, arglens[top]);
+ top++;
+ }
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+ else {
+ /* Use merely cmd */
+
+ args = g_malloc(sizeof(gchar *));
+ arglens = g_malloc(sizeof(gsize));
+ arglens[0] = strlen(cmd);
+ args[0] = g_malloc(arglens[0]);
+ memcpy(args[0], cmd, arglens[0]);
+ top = 1;
+ }
+
+ *pargs = args;
+ *parglens = arglens;
+ *nargs = top;
+}
+
+static struct lua_redis_ctx *
+rspamd_lua_redis_prepare_connection(lua_State *L, gint *pcbref, gboolean is_async)
+{
+ struct lua_redis_ctx *ctx = NULL;
+ rspamd_inet_addr_t *ip = NULL;
+ struct lua_redis_userdata *ud = NULL;
+ struct rspamd_lua_ip *addr = NULL;
+ struct rspamd_task *task = NULL;
+ const gchar *host = NULL;
+ const gchar *username = NULL, *password = NULL, *dbname = NULL, *log_tag = NULL;
+ gint cbref = -1;
+ struct rspamd_config *cfg = NULL;
+ struct rspamd_async_session *session = NULL;
+ struct ev_loop *ev_base = NULL;
+ gboolean ret = FALSE;
+ guint flags = 0;
+
+ if (lua_istable(L, 1)) {
+ /* Table version */
+ lua_pushvalue(L, 1);
+ lua_pushstring(L, "task");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ task = lua_check_task_maybe(L, -1);
+ }
+ lua_pop(L, 1);
+
+ if (!task) {
+ /* We need to get ev_base, config and session separately */
+ lua_pushstring(L, "config");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ cfg = lua_check_config(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "session");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ session = lua_check_session(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "ev_base");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ ev_base = lua_check_ev_base(L, -1);
+ }
+ lua_pop(L, 1);
+
+ if (cfg && ev_base) {
+ ret = TRUE;
+ }
+ else if (!cfg) {
+ msg_err_task_check("config is not passed");
+ }
+ else {
+ msg_err_task_check("ev_base is not set");
+ }
+ }
+ else {
+ cfg = task->cfg;
+ session = task->s;
+ ev_base = task->event_loop;
+ log_tag = task->task_pool->tag.uid;
+ ret = TRUE;
+ }
+
+ if (pcbref) {
+ lua_pushstring(L, "callback");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ /* This also pops function from the stack */
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ *pcbref = cbref;
+ }
+ else {
+ *pcbref = -1;
+ lua_pop(L, 1);
+ }
+ }
+
+ lua_pushstring(L, "host");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ addr = lua_check_ip(L, -1);
+ host = rspamd_inet_address_to_string_pretty(addr->addr);
+ }
+ else if (lua_type(L, -1) == LUA_TSTRING) {
+ host = lua_tostring(L, -1);
+
+ if (rspamd_parse_inet_address(&ip,
+ host, strlen(host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ addr = g_alloca(sizeof(*addr));
+ addr->addr = ip;
+
+ if (rspamd_inet_address_get_port(ip) == 0) {
+ rspamd_inet_address_set_port(ip, 6379);
+ }
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "username");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ username = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "password");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ password = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "dbname");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ dbname = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "opaque_data");
+ lua_gettable(L, -2);
+ if (!!lua_toboolean(L, -1)) {
+ flags |= LUA_REDIS_TEXTDATA;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "no_pool");
+ lua_gettable(L, -2);
+ if (!!lua_toboolean(L, -1)) {
+ flags |= LUA_REDIS_NO_POOL;
+ }
+ lua_pop(L, 1);
+
+ lua_pop(L, 1); /* table */
+
+ if (session && rspamd_session_blocked(session)) {
+ msg_err_task_check("Session is being destroying");
+ ret = FALSE;
+ }
+
+ if (ret && addr != NULL) {
+ ctx = g_malloc0(sizeof(struct lua_redis_ctx));
+ REF_INIT_RETAIN(ctx, lua_redis_dtor);
+ if (is_async) {
+ ctx->flags |= flags | LUA_REDIS_ASYNC;
+ ud = &ctx->async;
+ }
+ else {
+ ud = &ctx->async;
+ ctx->replies = g_queue_new();
+ ctx->events_cleanup = g_queue_new();
+ }
+
+ ud->s = session;
+ ud->cfg = cfg;
+ ud->pool = cfg->redis_pool;
+ ud->event_loop = ev_base;
+ ud->task = task;
+
+ if (log_tag) {
+ rspamd_strlcpy(ud->log_tag, log_tag, sizeof(ud->log_tag));
+ }
+ else {
+ /* Use pointer itself as a tag */
+ rspamd_snprintf(ud->log_tag, sizeof(ud->log_tag),
+ "%ud",
+ (int) rspamd_cryptobox_fast_hash(&ud, sizeof(ud), 0));
+ }
+
+ if (task) {
+ ud->item = rspamd_symcache_get_cur_item(task);
+ }
+
+ ret = TRUE;
+ }
+ else {
+ if (cbref != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbref);
+ }
+
+ msg_err_task_check("incorrect function invocation");
+ ret = FALSE;
+ }
+ }
+
+ if (ret) {
+ ud->terminated = 0;
+ ud->ctx = rspamd_redis_pool_connect(ud->pool,
+ dbname, username, password,
+ rspamd_inet_address_to_string(addr->addr),
+ rspamd_inet_address_get_port(addr->addr));
+
+ if (ip) {
+ rspamd_inet_address_free(ip);
+ }
+
+ if (ud->ctx == NULL || ud->ctx->err) {
+ if (ud->ctx) {
+ msg_err_task_check("cannot connect to redis: %s",
+ ud->ctx->errstr);
+ rspamd_redis_pool_release_connection(ud->pool, ud->ctx,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ ud->ctx = NULL;
+ }
+ else {
+ msg_err_task_check("cannot connect to redis (OS error): %s",
+ strerror(errno));
+ }
+
+ REDIS_RELEASE(ctx);
+
+ return NULL;
+ }
+
+ msg_debug_lua_redis("opened redis connection host=%s; ctx=%p; ud=%p",
+ host, ctx, ud);
+
+ return ctx;
+ }
+
+ if (ip) {
+ rspamd_inet_address_free(ip);
+ }
+
+ return NULL;
+}
+
+/***
+ * @function rspamd_redis.make_request({params})
+ * Make request to redis server, params is a table of key=value arguments in any order
+ * @param {task} task worker task object
+ * @param {ip|string} host server address
+ * @param {function} callback callback to be called in form `function (task, err, data)`
+ * @param {string} cmd command to be sent to redis
+ * @param {table} args numeric array of strings used as redis arguments
+ * @param {number} timeout timeout in seconds for request (1.0 by default)
+ * @return {boolean} `true` if a request has been scheduled
+ */
+static int
+lua_redis_make_request(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_redis_request_specific_userdata *sp_ud;
+ struct lua_redis_userdata *ud;
+ struct lua_redis_ctx *ctx, **pctx;
+ const gchar *cmd = NULL;
+ gdouble timeout = REDIS_DEFAULT_TIMEOUT;
+ gint cbref = -1;
+ gboolean ret = FALSE;
+
+ ctx = rspamd_lua_redis_prepare_connection(L, &cbref, TRUE);
+
+ if (ctx) {
+ ud = &ctx->async;
+ sp_ud = g_malloc0(sizeof(*sp_ud));
+ sp_ud->cbref = cbref;
+ sp_ud->c = ud;
+ sp_ud->ctx = ctx;
+
+ lua_pushstring(L, "cmd");
+ lua_gettable(L, -2);
+ cmd = lua_tostring(L, -1);
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+ ud->timeout = timeout;
+
+
+ lua_pushstring(L, "args");
+ lua_gettable(L, 1);
+ lua_redis_parse_args(L, -1, cmd, &sp_ud->args, &sp_ud->arglens,
+ &sp_ud->nargs);
+ lua_pop(L, 1);
+ LL_PREPEND(ud->specific, sp_ud);
+
+ ret = redisAsyncCommandArgv(ud->ctx,
+ lua_redis_callback,
+ sp_ud,
+ sp_ud->nargs,
+ (const gchar **) sp_ud->args,
+ sp_ud->arglens);
+
+ if (ret == REDIS_OK) {
+ if (ud->s) {
+ rspamd_session_add_event(ud->s,
+ lua_redis_fin, sp_ud,
+ M);
+
+ if (ud->item) {
+ rspamd_symcache_item_async_inc(ud->task, ud->item, M);
+ }
+ }
+
+ REDIS_RETAIN(ctx); /* Cleared by fin event */
+ ctx->cmds_pending++;
+
+ if (ud->ctx->c.flags & REDIS_SUBSCRIBED) {
+ msg_debug_lua_redis("subscribe command, never unref/timeout");
+ sp_ud->flags |= LUA_REDIS_SUBSCRIBED;
+ }
+
+ sp_ud->timeout_ev.data = sp_ud;
+ ev_now_update_if_cheap((struct ev_loop *) ud->event_loop);
+ ev_timer_init(&sp_ud->timeout_ev, lua_redis_timeout, timeout, 0.0);
+ ev_timer_start(ud->event_loop, &sp_ud->timeout_ev);
+
+ ret = TRUE;
+ }
+ else {
+ msg_info("call to redis failed: %s", ud->ctx->errstr);
+ rspamd_redis_pool_release_connection(ud->pool, ud->ctx,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ ud->ctx = NULL;
+ REDIS_RELEASE(ctx);
+ ret = FALSE;
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ lua_pushnil(L);
+
+ return 2;
+ }
+
+ lua_pushboolean(L, ret);
+
+ if (ret) {
+ pctx = lua_newuserdata(L, sizeof(ctx));
+ *pctx = ctx;
+ rspamd_lua_setclass(L, "rspamd{redis}", -1);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 2;
+}
+
+/***
+ * @function rspamd_redis.make_request_sync({params})
+ * Make blocking request to redis server, params is a table of key=value arguments in any order
+ * @param {ip|string} host server address
+ * @param {string} cmd command to be sent to redis
+ * @param {table} args numeric array of strings used as redis arguments
+ * @param {number} timeout timeout in seconds for request (1.0 by default)
+ * @return {boolean + result} `true` and a result if a request has been successful
+ */
+static int
+lua_redis_make_request_sync(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *addr = NULL;
+ rspamd_inet_addr_t *ip = NULL;
+ const gchar *cmd = NULL, *host;
+ struct timeval tv;
+ gboolean ret = FALSE;
+ gdouble timeout = REDIS_DEFAULT_TIMEOUT;
+ gchar **args = NULL;
+ gsize *arglens = NULL;
+ guint nargs = 0, flags = 0;
+ redisContext *ctx;
+ redisReply *r;
+
+ if (lua_istable(L, 1)) {
+ lua_pushvalue(L, 1);
+
+ lua_pushstring(L, "cmd");
+ lua_gettable(L, -2);
+ cmd = lua_tostring(L, -1);
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "host");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ addr = lua_check_ip(L, -1);
+ }
+ else if (lua_type(L, -1) == LUA_TSTRING) {
+ host = lua_tostring(L, -1);
+ if (rspamd_parse_inet_address(&ip,
+ host, strlen(host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ addr = g_alloca(sizeof(*addr));
+ addr->addr = ip;
+
+ if (rspamd_inet_address_get_port(ip) == 0) {
+ rspamd_inet_address_set_port(ip, 6379);
+ }
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "opaque_data");
+ lua_gettable(L, -2);
+ if (!!lua_toboolean(L, -1)) {
+ flags |= LUA_REDIS_TEXTDATA;
+ }
+ lua_pop(L, 1);
+
+
+ if (cmd) {
+ lua_pushstring(L, "args");
+ lua_gettable(L, -2);
+ lua_redis_parse_args(L, -1, cmd, &args, &arglens, &nargs);
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+
+ if (addr && cmd) {
+ ret = TRUE;
+ }
+ }
+
+ if (ret) {
+ double_to_tv(timeout, &tv);
+
+ if (rspamd_inet_address_get_af(addr->addr) == AF_UNIX) {
+ ctx = redisConnectUnixWithTimeout(
+ rspamd_inet_address_to_string(addr->addr), tv);
+ }
+ else {
+ ctx = redisConnectWithTimeout(
+ rspamd_inet_address_to_string(addr->addr),
+ rspamd_inet_address_get_port(addr->addr), tv);
+ }
+
+ if (ip) {
+ rspamd_inet_address_free(ip);
+ }
+
+ if (ctx == NULL || ctx->err) {
+ redisFree(ctx);
+ lua_redis_free_args(args, arglens, nargs);
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+
+ r = redisCommandArgv(ctx,
+ nargs,
+ (const gchar **) args,
+ arglens);
+
+ if (r != NULL) {
+ if (r->type != REDIS_REPLY_ERROR) {
+ lua_pushboolean(L, TRUE);
+ lua_redis_push_reply(L, r, flags & LUA_REDIS_TEXTDATA);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ lua_pushstring(L, r->str);
+ }
+
+ freeReplyObject(r);
+ redisFree(ctx);
+ lua_redis_free_args(args, arglens, nargs);
+
+ return 2;
+ }
+ else {
+ msg_info("call to redis failed: %s", ctx->errstr);
+ redisFree(ctx);
+ lua_redis_free_args(args, arglens, nargs);
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ if (ip) {
+ rspamd_inet_address_free(ip);
+ }
+ msg_err("bad arguments for redis request");
+ lua_redis_free_args(args, arglens, nargs);
+
+ lua_pushboolean(L, FALSE);
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_redis.connect({params})
+ * Make request to redis server, params is a table of key=value arguments in any order
+ * @param {task} task worker task object
+ * @param {ip|string} host server address
+ * @param {number} timeout timeout in seconds for request (1.0 by default)
+ * @return {boolean,redis} new connection object or nil if connection failed
+ */
+static int
+lua_redis_connect(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_redis_userdata *ud;
+ struct lua_redis_ctx *ctx, **pctx;
+ gdouble timeout = REDIS_DEFAULT_TIMEOUT;
+
+ ctx = rspamd_lua_redis_prepare_connection(L, NULL, TRUE);
+
+ if (ctx) {
+ ud = &ctx->async;
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+
+ lua_pop(L, 1);
+ ud->timeout = timeout;
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ lua_pushnil(L);
+
+ return 2;
+ }
+
+ lua_pushboolean(L, TRUE);
+ pctx = lua_newuserdata(L, sizeof(ctx));
+ *pctx = ctx;
+ rspamd_lua_setclass(L, "rspamd{redis}", -1);
+
+ return 2;
+}
+
+/***
+ * @function rspamd_redis.connect_sync({params})
+ * Make blocking request to redis server, params is a table of key=value arguments in any order
+ * @param {ip|string} host server address
+ * @param {number} timeout timeout in seconds for request (1.0 by default)
+ * @return {redis} redis object if a request has been successful
+ */
+static int
+lua_redis_connect_sync(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gdouble timeout = REDIS_DEFAULT_TIMEOUT;
+ struct lua_redis_ctx *ctx, **pctx;
+
+ ctx = rspamd_lua_redis_prepare_connection(L, NULL, FALSE);
+
+ if (ctx) {
+ if (lua_istable(L, 1)) {
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+ }
+
+ ctx->async.timeout = timeout;
+
+ lua_pushboolean(L, TRUE);
+ pctx = lua_newuserdata(L, sizeof(ctx));
+ *pctx = ctx;
+ rspamd_lua_setclass(L, "rspamd{redis}", -1);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ lua_pushstring(L, "bad arguments for redis request");
+ return 2;
+ }
+
+ return 2;
+}
+
+/***
+ * @method rspamd_redis:add_cmd(cmd, {args})
+ * Append new cmd to redis pipeline
+ * @param {string} cmd command to be sent to redis
+ * @param {table} args array of strings used as redis arguments
+ * @return {boolean} `true` if a request has been successful
+ */
+static int
+lua_redis_add_cmd(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_redis_ctx *ctx = lua_check_redis(L, 1);
+ struct lua_redis_request_specific_userdata *sp_ud;
+ struct lua_redis_userdata *ud;
+ const gchar *cmd = NULL;
+ gint args_pos = 2;
+ gint cbref = -1, ret;
+
+ if (ctx) {
+ if (ctx->flags & LUA_REDIS_TERMINATED) {
+ lua_pushboolean(L, FALSE);
+ lua_pushstring(L, "Connection is terminated");
+
+ return 2;
+ }
+
+ /* Async version */
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* No callback version */
+ cmd = lua_tostring(L, 2);
+ args_pos = 3;
+ }
+ else if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ cmd = lua_tostring(L, 3);
+ args_pos = 4;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sp_ud = g_malloc0(sizeof(*sp_ud));
+ if (IS_ASYNC(ctx)) {
+ sp_ud->c = &ctx->async;
+ ud = &ctx->async;
+ sp_ud->cbref = cbref;
+ }
+ else {
+ sp_ud->c = &ctx->async;
+ ud = &ctx->async;
+ }
+ sp_ud->ctx = ctx;
+
+ lua_redis_parse_args(L, args_pos, cmd, &sp_ud->args,
+ &sp_ud->arglens, &sp_ud->nargs);
+
+ LL_PREPEND(sp_ud->c->specific, sp_ud);
+
+ if (ud->s && rspamd_session_blocked(ud->s)) {
+ lua_pushboolean(L, 0);
+ lua_pushstring(L, "session is terminating");
+
+ return 2;
+ }
+
+ if (IS_ASYNC(ctx)) {
+ ret = redisAsyncCommandArgv(sp_ud->c->ctx,
+ lua_redis_callback,
+ sp_ud,
+ sp_ud->nargs,
+ (const gchar **) sp_ud->args,
+ sp_ud->arglens);
+ }
+ else {
+ ret = redisAsyncCommandArgv(sp_ud->c->ctx,
+ lua_redis_callback_sync,
+ sp_ud,
+ sp_ud->nargs,
+ (const gchar **) sp_ud->args,
+ sp_ud->arglens);
+ }
+
+ if (ret == REDIS_OK) {
+ if (ud->s) {
+ rspamd_session_add_event(ud->s,
+ lua_redis_fin,
+ sp_ud,
+ M);
+
+ if (ud->item) {
+ rspamd_symcache_item_async_inc(ud->task, ud->item, M);
+ }
+ }
+
+ sp_ud->timeout_ev.data = sp_ud;
+
+ if (IS_ASYNC(ctx)) {
+ ev_timer_init(&sp_ud->timeout_ev, lua_redis_timeout,
+ sp_ud->c->timeout, 0.0);
+ }
+ else {
+ ev_timer_init(&sp_ud->timeout_ev, lua_redis_timeout_sync,
+ sp_ud->c->timeout, 0.0);
+ }
+
+ ev_timer_start(ud->event_loop, &sp_ud->timeout_ev);
+ REDIS_RETAIN(ctx);
+ ctx->cmds_pending++;
+ }
+ else {
+ msg_info("call to redis failed: %s",
+ sp_ud->c->ctx->errstr);
+ lua_pushboolean(L, 0);
+ lua_pushstring(L, sp_ud->c->ctx->errstr);
+
+ return 2;
+ }
+ }
+
+ lua_pushboolean(L, true);
+
+ return 1;
+}
+
+/***
+ * @method rspamd_redis:exec()
+ * Executes pending commands (suitable for blocking IO only for now)
+ * @return {boolean}, {table}, ...: pairs in format [bool, result] for each request pending
+ */
+static int
+lua_redis_exec(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_redis_ctx *ctx = lua_check_redis(L, 1);
+
+ if (ctx == NULL) {
+ lua_error(L);
+
+ return 1;
+ }
+
+ if (IS_ASYNC(ctx)) {
+ lua_pushstring(L, "Async redis pipelining is not implemented");
+ lua_error(L);
+ return 0;
+ }
+ else {
+ if (ctx->cmds_pending == 0 && g_queue_get_length(ctx->replies) == 0) {
+ lua_pushstring(L, "No pending commands to execute");
+ lua_error(L);
+ }
+ if (ctx->cmds_pending == 0 && g_queue_get_length(ctx->replies) > 0) {
+ gint results = lua_redis_push_results(ctx, L);
+ return results;
+ }
+ else {
+ ctx->thread = lua_thread_pool_get_running_entry(ctx->async.cfg->lua_thread_pool);
+ return lua_thread_yield(ctx->thread, 0);
+ }
+ }
+}
+
+static gint
+lua_load_redis(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, redislib_f);
+
+ return 1;
+}
+
+static gint
+lua_redis_null_idx(lua_State *L)
+{
+ lua_pushnil(L);
+
+ return 1;
+}
+
+static void
+lua_redis_null_mt(lua_State *L)
+{
+ luaL_newmetatable(L, "redis{null}");
+
+ lua_pushcfunction(L, lua_redis_null_idx);
+ lua_setfield(L, -2, "__index");
+ lua_pushcfunction(L, lua_redis_null_idx);
+ lua_setfield(L, -2, "__tostring");
+
+ lua_pop(L, 1);
+}
+
+/**
+ * Open redis library
+ * @param L lua stack
+ * @return
+ */
+void luaopen_redis(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{redis}", redislib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_redis", lua_load_redis);
+
+ /* Set null element */
+ lua_redis_null_mt(L);
+ redis_null = lua_newuserdata(L, 0);
+ luaL_getmetatable(L, "redis{null}");
+ lua_setmetatable(L, -2);
+ lua_setfield(L, LUA_REGISTRYINDEX, "redis.null");
+}
diff --git a/src/lua/lua_regexp.c b/src/lua/lua_regexp.c
new file mode 100644
index 0000000..7e638ca
--- /dev/null
+++ b/src/lua/lua_regexp.c
@@ -0,0 +1,858 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+
+/***
+ * @module rspamd_regexp
+ * Rspamd regexp is an utility module that handles rspamd perl compatible
+ * regular expressions
+ * @example
+ * local rspamd_regexp = require "rspamd_regexp"
+ *
+ * local re = rspamd_regexp.create_cached('/^\\s*some_string\\s*$/i')
+ * re:match('some_string')
+ * local re = rspamd_regexp.create_cached('/\\s+/i')
+ * re:split('word word word') -- returns ['word', 'word', 'word']
+ */
+
+LUA_FUNCTION_DEF(regexp, create);
+LUA_FUNCTION_DEF(regexp, import_glob);
+LUA_FUNCTION_DEF(regexp, import_plain);
+LUA_FUNCTION_DEF(regexp, create_cached);
+LUA_FUNCTION_DEF(regexp, get_cached);
+LUA_FUNCTION_DEF(regexp, get_pattern);
+LUA_FUNCTION_DEF(regexp, set_limit);
+LUA_FUNCTION_DEF(regexp, set_max_hits);
+LUA_FUNCTION_DEF(regexp, get_max_hits);
+LUA_FUNCTION_DEF(regexp, search);
+LUA_FUNCTION_DEF(regexp, match);
+LUA_FUNCTION_DEF(regexp, matchn);
+LUA_FUNCTION_DEF(regexp, split);
+LUA_FUNCTION_DEF(regexp, destroy);
+LUA_FUNCTION_DEF(regexp, gc);
+
+static const struct luaL_reg regexplib_m[] = {
+ LUA_INTERFACE_DEF(regexp, get_pattern),
+ LUA_INTERFACE_DEF(regexp, set_limit),
+ LUA_INTERFACE_DEF(regexp, set_max_hits),
+ LUA_INTERFACE_DEF(regexp, get_max_hits),
+ LUA_INTERFACE_DEF(regexp, match),
+ LUA_INTERFACE_DEF(regexp, matchn),
+ LUA_INTERFACE_DEF(regexp, search),
+ LUA_INTERFACE_DEF(regexp, split),
+ LUA_INTERFACE_DEF(regexp, destroy),
+ {"__tostring", lua_regexp_get_pattern},
+ {"__gc", lua_regexp_gc},
+ {NULL, NULL}};
+static const struct luaL_reg regexplib_f[] = {
+ LUA_INTERFACE_DEF(regexp, create),
+ LUA_INTERFACE_DEF(regexp, import_glob),
+ LUA_INTERFACE_DEF(regexp, import_plain),
+ LUA_INTERFACE_DEF(regexp, get_cached),
+ LUA_INTERFACE_DEF(regexp, create_cached),
+ {NULL, NULL}};
+
+#define LUA_REGEXP_FLAG_DESTROYED (1 << 0)
+#define IS_DESTROYED(re) ((re)->re_flags & LUA_REGEXP_FLAG_DESTROYED)
+
+rspamd_mempool_t *regexp_static_pool = NULL;
+
+struct rspamd_lua_regexp *
+lua_check_regexp(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{regexp}");
+
+ luaL_argcheck(L, ud != NULL, pos, "'regexp' expected");
+ return ud ? *((struct rspamd_lua_regexp **) ud) : NULL;
+}
+
+/***
+ * @function rspamd_regexp.create(pattern[, flags])
+ * Creates new rspamd_regexp
+ * @param {string} pattern pattern to build regexp. If this pattern is enclosed in `//` then it is possible to specify flags after it
+ * @param {string} flags optional flags to create regular expression
+ * @return {regexp} regexp argument that is *not* automatically destroyed
+ * @example
+ * local regexp = require "rspamd_regexp"
+ *
+ * local re = regexp.create('/^test.*[0-9]\\s*$/i')
+ */
+static int
+lua_regexp_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_regexp_t *re;
+ struct rspamd_lua_regexp *new, **pnew;
+ const gchar *string, *flags_str = NULL;
+ GError *err = NULL;
+
+ string = luaL_checkstring(L, 1);
+ if (lua_gettop(L) == 2) {
+ flags_str = luaL_checkstring(L, 2);
+ }
+
+ if (string) {
+ re = rspamd_regexp_new(string, flags_str, &err);
+ if (re == NULL) {
+ lua_pushnil(L);
+ msg_info("cannot parse regexp: %s, error: %s",
+ string,
+ err == NULL ? "undefined" : err->message);
+ g_error_free(err);
+ }
+ else {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = re;
+ new->re_pattern = g_strdup(string);
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_regexp.import_glob(glob_pattern[, flags])
+ * Creates new rspamd_regexp from glob
+ * @param {string} pattern pattern to build regexp.
+ * @param {string} flags optional flags to create regular expression
+ * @return {regexp} regexp argument that is *not* automatically destroyed
+ * @example
+ * local regexp = require "rspamd_regexp"
+ *
+ * local re = regexp.import_glob('ab*', 'i')
+ */
+static int
+lua_regexp_import_glob(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_regexp_t *re;
+ struct rspamd_lua_regexp *new, **pnew;
+ const gchar *string, *flags_str = NULL;
+ gchar *escaped;
+ gsize pat_len;
+ GError *err = NULL;
+
+ string = luaL_checklstring(L, 1, &pat_len);
+
+ if (lua_gettop(L) == 2) {
+ flags_str = luaL_checkstring(L, 2);
+ }
+
+ if (string) {
+ escaped = rspamd_str_regexp_escape(string, pat_len, NULL,
+ RSPAMD_REGEXP_ESCAPE_GLOB | RSPAMD_REGEXP_ESCAPE_UTF);
+
+ re = rspamd_regexp_new(escaped, flags_str, &err);
+
+ if (re == NULL) {
+ lua_pushnil(L);
+ msg_info("cannot parse regexp: %s, error: %s",
+ string,
+ err == NULL ? "undefined" : err->message);
+ g_error_free(err);
+ g_free(escaped);
+ }
+ else {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = re;
+ new->re_pattern = escaped;
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_regexp.import_plain(plain_string[, flags])
+ * Creates new rspamd_regexp from plain string (escaping specials)
+ * @param {string} pattern pattern to build regexp.
+ * @param {string} flags optional flags to create regular expression
+ * @return {regexp} regexp argument that is *not* automatically destroyed
+ * @example
+ * local regexp = require "rspamd_regexp"
+ *
+ * local re = regexp.import_plain('exact_string_with*', 'i')
+ */
+static int
+lua_regexp_import_plain(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_regexp_t *re;
+ struct rspamd_lua_regexp *new, **pnew;
+ const gchar *string, *flags_str = NULL;
+ gchar *escaped;
+ gsize pat_len;
+ GError *err = NULL;
+
+ string = luaL_checklstring(L, 1, &pat_len);
+
+ if (lua_gettop(L) == 2) {
+ flags_str = luaL_checkstring(L, 2);
+ }
+
+ if (string) {
+ escaped = rspamd_str_regexp_escape(string, pat_len, NULL,
+ RSPAMD_REGEXP_ESCAPE_ASCII);
+
+ re = rspamd_regexp_new(escaped, flags_str, &err);
+
+ if (re == NULL) {
+ lua_pushnil(L);
+ msg_info("cannot parse regexp: %s, error: %s",
+ string,
+ err == NULL ? "undefined" : err->message);
+ g_error_free(err);
+ g_free(escaped);
+ }
+ else {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = re;
+ new->re_pattern = escaped;
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_regexp.get_cached(pattern)
+ * This function gets cached and pre-compiled regexp created by either `create`
+ * or `create_cached` methods. If no cached regexp is found then `nil` is returned.
+ *
+ * @param {string} pattern regexp pattern
+ * @return {regexp} cached regexp structure or `nil`
+ */
+static int
+lua_regexp_get_cached(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_regexp_t *re;
+ struct rspamd_lua_regexp *new, **pnew;
+ const gchar *string, *flags_str = NULL;
+
+ string = luaL_checkstring(L, 1);
+ if (lua_gettop(L) == 2) {
+ flags_str = luaL_checkstring(L, 2);
+ }
+
+ if (string) {
+ re = rspamd_regexp_cache_query(NULL, string, flags_str);
+
+ if (re) {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = rspamd_regexp_ref(re);
+ new->re_pattern = g_strdup(string);
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_regexp.create_cached(pattern[, flags])
+ * This function is similar to `create` but it tries to search for regexp in the
+ * cache first.
+ * @param {string} pattern pattern to build regexp. If this pattern is enclosed in `//` then it is possible to specify flags after it
+ * @param {string} flags optional flags to create regular expression
+ * @return {regexp} regexp argument that is *not* automatically destroyed
+ * @example
+ * local regexp = require "rspamd_regexp"
+ *
+ * local re = regexp.create_cached('/^test.*[0-9]\\s*$/i')
+ * ...
+ * -- This doesn't create new regexp object
+ * local other_re = regexp.create_cached('/^test.*[0-9]\\s*$/i')
+ */
+static int
+lua_regexp_create_cached(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_regexp_t *re;
+ struct rspamd_lua_regexp *new, **pnew;
+ const gchar *string, *flags_str = NULL;
+ GError *err = NULL;
+
+ string = luaL_checkstring(L, 1);
+ if (lua_gettop(L) == 2) {
+ flags_str = luaL_checkstring(L, 2);
+ }
+
+ if (string) {
+ re = rspamd_regexp_cache_query(NULL, string, flags_str);
+
+ if (re) {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = rspamd_regexp_ref(re);
+ new->re_pattern = g_strdup(string);
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ else {
+ re = rspamd_regexp_cache_create(NULL, string, flags_str, &err);
+ if (re == NULL) {
+ lua_pushnil(L);
+ msg_info("cannot parse regexp: %s, error: %s",
+ string,
+ err == NULL ? "undefined" : err->message);
+ g_error_free(err);
+ }
+ else {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = rspamd_regexp_ref(re);
+ new->re_pattern = g_strdup(string);
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:get_pattern()
+ * Get a pattern for specified regexp object
+ * @return {string} pattern line
+ */
+static int
+lua_regexp_get_pattern(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+
+ if (re && re->re && !IS_DESTROYED(re)) {
+ lua_pushstring(L, rspamd_regexp_get_pattern(re->re));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:set_limit(lim)
+ * Set maximum size of text length to be matched with this regexp (if `lim` is
+ * less or equal to zero then all texts are checked)
+ * @param {number} lim limit in bytes
+ */
+static int
+lua_regexp_set_limit(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ gint64 lim;
+
+ lim = lua_tointeger(L, 2);
+
+ if (re && re->re && !IS_DESTROYED(re)) {
+ if (lim > 0) {
+ rspamd_regexp_set_match_limit(re->re, lim);
+ }
+ else {
+ rspamd_regexp_set_match_limit(re->re, 0);
+ }
+ }
+
+ return 0;
+}
+
+/***
+ * @method re:set_max_hits(lim)
+ * Set maximum number of hits returned by a regexp
+ * @param {number} lim limit in hits count
+ * @return {number} old number of max hits
+ */
+static int
+lua_regexp_set_max_hits(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ guint lim;
+
+ lim = luaL_checkinteger(L, 2);
+
+ if (re && re->re && !IS_DESTROYED(re)) {
+ lua_pushinteger(L, rspamd_regexp_set_maxhits(re->re, lim));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:get_max_hits(lim)
+ * Get maximum number of hits returned by a regexp
+ * @return {number} number of max hits
+ */
+static int
+lua_regexp_get_max_hits(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+
+ if (re && re->re && !IS_DESTROYED(re)) {
+ lua_pushinteger(L, rspamd_regexp_get_maxhits(re->re));
+ }
+ else {
+ lua_pushinteger(L, 1);
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:search(line[, raw[, capture]])
+ * Search line in regular expression object. If line matches then this
+ * function returns the table of captured strings. Otherwise, nil is returned.
+ * If `raw` is specified, then input is treated as raw data not encoded in `utf-8`.
+ * If `capture` is true, then this function saves all captures to the table of
+ * values, so the first element is the whole matched string and the
+ * subsequent elements are ordered captures defined within pattern.
+ *
+ * @param {string} line match the specified line against regexp object
+ * @param {bool} match raw regexp instead of utf8 one
+ * @param {bool} capture perform subpatterns capturing
+ * @return {table or nil} table of strings or tables (if `capture` is true) or nil if not matched
+ * @example
+ * local re = regexp.create_cached('/^\s*([0-9]+)\s*$/')
+ * -- returns nil
+ * local m1 = re:search('blah')
+ * local m2 = re:search(' 190 ')
+ * -- prints ' 190 '
+ * print(m2[1])
+ *
+ * local m3 = re:search(' 100500 ')
+ * -- prints ' 100500 '
+ * print(m3[1][1])
+ * -- prints '100500' capture
+ * print(m3[1][2])
+ */
+static int
+lua_regexp_search(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ const gchar *data = NULL;
+ struct rspamd_lua_text *t;
+ const gchar *start = NULL, *end = NULL;
+ gint i;
+ gsize len = 0, capn;
+ gboolean matched = FALSE, capture = FALSE, raw = FALSE;
+ GArray *captures = NULL;
+ struct rspamd_re_capture *cap;
+
+ if (re && !IS_DESTROYED(re)) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ data = luaL_checklstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+ if (t != NULL) {
+ data = t->start;
+ len = t->len;
+ }
+ }
+
+ if (lua_gettop(L) >= 3) {
+ raw = lua_toboolean(L, 3);
+ }
+
+ if (data && len > 0) {
+ if (lua_gettop(L) >= 4 && lua_toboolean(L, 4)) {
+ capture = TRUE;
+ captures = g_array_new(FALSE, TRUE,
+ sizeof(struct rspamd_re_capture));
+ }
+
+ lua_newtable(L);
+ i = 0;
+
+ while (rspamd_regexp_search(re->re, data, len, &start, &end, raw,
+ captures)) {
+
+ if (capture) {
+ lua_createtable(L, captures->len, 0);
+
+ for (capn = 0; capn < captures->len; capn++) {
+ cap = &g_array_index(captures, struct rspamd_re_capture,
+ capn);
+ lua_pushlstring(L, cap->p, cap->len);
+ lua_rawseti(L, -2, capn + 1);
+ }
+
+ lua_rawseti(L, -2, ++i);
+ }
+ else {
+ lua_pushlstring(L, start, end - start);
+ lua_rawseti(L, -2, ++i);
+ }
+
+ matched = TRUE;
+ }
+
+ if (!matched) {
+ lua_pop(L, 1);
+ lua_pushnil(L);
+ }
+
+ if (capture) {
+ g_array_free(captures, TRUE);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:match(line[, raw_match])
+ * Matches line against the regular expression and return true if line matches
+ * (partially or completely)
+ *
+ * @param {string} line match the specified line against regexp object
+ * @param {bool} match raw regexp instead of utf8 one
+ * @return {bool} true if `line` matches
+ */
+static int
+lua_regexp_match(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ struct rspamd_lua_text *t;
+ const gchar *data = NULL;
+ gsize len = 0;
+ gboolean raw = FALSE;
+
+ if (re && !IS_DESTROYED(re)) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ data = luaL_checklstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+ if (t != NULL) {
+ data = t->start;
+ len = t->len;
+ }
+ }
+
+ if (lua_gettop(L) == 3) {
+ raw = lua_toboolean(L, 3);
+ }
+
+ if (data && len > 0) {
+ if (rspamd_regexp_search(re->re, data, len, NULL, NULL, raw, NULL)) {
+ lua_pushboolean(L, TRUE);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:matchn(line, max_matches, [, raw_match])
+ * Matches line against the regular expression and return number of matches if line matches
+ * (partially or completely). This process stop when `max_matches` is reached.
+ * If `max_matches` is zero, then only a single match is counted which is equal to
+ * @see re:match If `max_matches` is negative, then all matches are considered.
+ *
+ * @param {string} line match the specified line against regexp object
+ * @param {number} max_matches maximum number of matches
+ * @param {bool} match raw regexp instead of utf8 one
+ * @return {number} number of matches found in the `line` argument
+ */
+static int
+lua_regexp_matchn(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ struct rspamd_lua_text *t;
+ const gchar *data = NULL, *start = NULL, *end = NULL;
+ gint max_matches, matches;
+ gsize len = 0;
+ gboolean raw = FALSE;
+
+ if (re && !IS_DESTROYED(re)) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ data = luaL_checklstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+ if (t != NULL) {
+ data = t->start;
+ len = t->len;
+ }
+ }
+
+ max_matches = lua_tointeger(L, 3);
+ matches = 0;
+
+ if (lua_gettop(L) == 4) {
+ raw = lua_toboolean(L, 4);
+ }
+
+ if (data && len > 0) {
+ for (;;) {
+ if (rspamd_regexp_search(re->re, data, len, &start, &end, raw,
+ NULL)) {
+ matches++;
+ }
+ else {
+ break;
+ }
+
+ if (max_matches >= 0 && matches >= max_matches) {
+ break;
+ }
+ }
+ }
+
+ lua_pushinteger(L, matches);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+/***
+ * @method re:split(line)
+ * Split line using the specified regular expression.
+ * Breaks the string on the pattern, and returns an array of the tokens.
+ * If the pattern contains capturing parentheses, then the text for each
+ * of the substrings will also be returned. If the pattern does not match
+ * anywhere in the string, then the whole string is returned as the first
+ * token.
+ * @param {string/text} line line to split
+ * @return {table} table of split line portions (if text was the input, then text is used for return parts)
+ */
+static int
+lua_regexp_split(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ const gchar *data = NULL;
+ struct rspamd_lua_text *t;
+ gboolean matched = FALSE, is_text = FALSE;
+ gsize len = 0;
+ const gchar *start = NULL, *end = NULL, *old_start;
+ gint i;
+
+ if (re && !IS_DESTROYED(re)) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ data = luaL_checklstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+
+ if (t == NULL) {
+ lua_error(L);
+ return 0;
+ }
+
+ data = t->start;
+ len = t->len;
+ is_text = TRUE;
+ }
+
+ if (data && len > 0) {
+ lua_newtable(L);
+ i = 0;
+ old_start = data;
+
+ while (rspamd_regexp_search(re->re, data, len, &start, &end, FALSE,
+ NULL)) {
+ if (start - old_start > 0) {
+ if (!is_text) {
+ lua_pushlstring(L, old_start, start - old_start);
+ }
+ else {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = old_start;
+ t->len = start - old_start;
+ t->flags = 0;
+ }
+
+ lua_rawseti(L, -2, ++i);
+ matched = TRUE;
+ }
+ else if (start == end) {
+ break;
+ }
+ old_start = end;
+ }
+
+ if (len > 0 && (end == NULL || end < data + len)) {
+ if (end == NULL) {
+ end = data;
+ }
+
+ if (!is_text) {
+ lua_pushlstring(L, end, (data + len) - end);
+ }
+ else {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = end;
+ t->len = (data + len) - end;
+ t->flags = 0;
+ }
+
+ lua_rawseti(L, -2, ++i);
+ matched = TRUE;
+ }
+
+ if (!matched) {
+ lua_pop(L, 1);
+ lua_pushnil(L);
+ }
+ return 1;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushnil(L);
+ return 1;
+}
+
+/***
+ * @method re:destroy()
+ * Destroy regexp from caches if needed (the pointer is removed by garbage collector)
+ */
+static gint
+lua_regexp_destroy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *to_del = lua_check_regexp(L, 1);
+
+ if (to_del) {
+ rspamd_regexp_cache_remove(NULL, to_del->re);
+ rspamd_regexp_unref(to_del->re);
+ to_del->re = NULL;
+ to_del->re_flags |= LUA_REGEXP_FLAG_DESTROYED;
+ }
+
+ return 0;
+}
+
+static gint
+lua_regexp_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *to_del = lua_check_regexp(L, 1);
+
+ if (to_del) {
+ if (!IS_DESTROYED(to_del)) {
+ rspamd_regexp_unref(to_del->re);
+ }
+
+ g_free(to_del->re_pattern);
+ g_free(to_del->module);
+ g_free(to_del);
+ }
+
+ return 0;
+}
+
+static gint
+lua_load_regexp(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, regexplib_f);
+
+ return 1;
+}
+
+void luaopen_regexp(lua_State *L)
+{
+ if (!regexp_static_pool) {
+ regexp_static_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "regexp_lua_pool", 0);
+ }
+
+ rspamd_lua_new_class(L, "rspamd{regexp}", regexplib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_regexp", lua_load_regexp);
+}
+
+RSPAMD_DESTRUCTOR(lua_re_static_pool_dtor)
+{
+ if (regexp_static_pool) {
+ rspamd_mempool_delete(regexp_static_pool);
+ }
+} \ No newline at end of file
diff --git a/src/lua/lua_rsa.c b/src/lua/lua_rsa.c
new file mode 100644
index 0000000..ae5acc8
--- /dev/null
+++ b/src/lua/lua_rsa.c
@@ -0,0 +1,867 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file lua_rsa.c
+ * This module exports routines to load rsa keys, check inline or external
+ * rsa signatures. It assumes sha256 based signatures.
+ */
+
+#include "lua_common.h"
+#include "unix-std.h"
+#include <openssl/err.h>
+#include <openssl/pem.h>
+#include <openssl/sha.h>
+#include <openssl/rsa.h>
+
+LUA_FUNCTION_DEF(rsa_pubkey, load);
+LUA_FUNCTION_DEF(rsa_pubkey, create);
+LUA_FUNCTION_DEF(rsa_pubkey, gc);
+LUA_FUNCTION_DEF(rsa_pubkey, tostring);
+
+LUA_FUNCTION_DEF(rsa_privkey, load_file);
+LUA_FUNCTION_DEF(rsa_privkey, load_pem);
+LUA_FUNCTION_DEF(rsa_privkey, load_raw);
+LUA_FUNCTION_DEF(rsa_privkey, load_base64);
+LUA_FUNCTION_DEF(rsa_privkey, create);
+LUA_FUNCTION_DEF(rsa_privkey, gc);
+LUA_FUNCTION_DEF(rsa_privkey, save);
+
+LUA_FUNCTION_DEF(rsa_signature, create);
+LUA_FUNCTION_DEF(rsa_signature, load);
+LUA_FUNCTION_DEF(rsa_signature, save);
+LUA_FUNCTION_DEF(rsa_signature, base64);
+LUA_FUNCTION_DEF(rsa_signature, gc);
+
+LUA_FUNCTION_DEF(rsa, verify_memory);
+LUA_FUNCTION_DEF(rsa, sign_memory);
+LUA_FUNCTION_DEF(rsa, keypair);
+
+static const struct luaL_reg rsalib_f[] = {
+ LUA_INTERFACE_DEF(rsa, verify_memory),
+ LUA_INTERFACE_DEF(rsa, sign_memory),
+ LUA_INTERFACE_DEF(rsa, keypair),
+ {NULL, NULL}};
+
+static const struct luaL_reg rsapubkeylib_f[] = {
+ LUA_INTERFACE_DEF(rsa_pubkey, load),
+ LUA_INTERFACE_DEF(rsa_pubkey, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg rsapubkeylib_m[] = {
+ {"__tostring", lua_rsa_pubkey_tostring},
+ {"__gc", lua_rsa_pubkey_gc},
+ {NULL, NULL}};
+
+static const struct luaL_reg rsaprivkeylib_f[] = {
+ LUA_INTERFACE_DEF(rsa_privkey, load_file),
+ LUA_INTERFACE_DEF(rsa_privkey, load_pem),
+ LUA_INTERFACE_DEF(rsa_privkey, load_raw),
+ LUA_INTERFACE_DEF(rsa_privkey, load_base64),
+ LUA_INTERFACE_DEF(rsa_privkey, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg rsaprivkeylib_m[] = {
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_rsa_privkey_gc},
+ LUA_INTERFACE_DEF(rsa_privkey, save),
+ {NULL, NULL}};
+
+static const struct luaL_reg rsasignlib_f[] = {
+ LUA_INTERFACE_DEF(rsa_signature, load),
+ LUA_INTERFACE_DEF(rsa_signature, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg rsasignlib_m[] = {
+ LUA_INTERFACE_DEF(rsa_signature, save),
+ LUA_INTERFACE_DEF(rsa_signature, base64),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_rsa_signature_gc},
+ {NULL, NULL}};
+
+static RSA *
+lua_check_rsa_pubkey(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{rsa_pubkey}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'rsa_pubkey' expected");
+ return ud ? *((RSA **) ud) : NULL;
+}
+
+static RSA *
+lua_check_rsa_privkey(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{rsa_privkey}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'rsa_privkey' expected");
+ return ud ? *((RSA **) ud) : NULL;
+}
+
+static rspamd_fstring_t *
+lua_check_rsa_sign(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{rsa_signature}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'rsa_signature' expected");
+ return ud ? *((rspamd_fstring_t **) ud) : NULL;
+}
+
+static gint
+lua_rsa_pubkey_load(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ const gchar *filename;
+ FILE *f;
+
+ filename = luaL_checkstring(L, 1);
+ if (filename != NULL) {
+ f = fopen(filename, "r");
+ if (f == NULL) {
+ msg_err("cannot open pubkey from file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (!PEM_read_RSA_PUBKEY(f, &rsa, NULL, NULL)) {
+ msg_err("cannot open pubkey from file: %s, %s", filename,
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_pubkey}", -1);
+ *prsa = rsa;
+ }
+ fclose(f);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_save(lua_State *L)
+{
+ const gchar *filename;
+ const gchar *type = "pem";
+ FILE *f;
+ int ret;
+
+ RSA *rsa = lua_check_rsa_privkey(L, 1);
+
+ filename = luaL_checkstring(L, 2);
+ if (lua_gettop(L) > 2) {
+ type = luaL_checkstring(L, 3);
+ }
+
+ if (rsa != NULL && filename != NULL) {
+ if (strcmp(filename, "-") == 0) {
+ f = stdout;
+ }
+ else {
+ f = fopen(filename, "wb");
+ }
+ if (f == NULL) {
+ msg_err("cannot save privkey to file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushboolean(L, FALSE);
+ }
+ else {
+ if (f != stdout) {
+ /* Set secure permissions for the private key file */
+ chmod(filename, S_IRUSR | S_IWUSR);
+ }
+
+ if (strcmp(type, "der") == 0) {
+ ret = i2d_RSAPrivateKey_fp(f, rsa);
+ }
+ else {
+ ret = PEM_write_RSAPrivateKey(f, rsa, NULL, NULL, 0, NULL, NULL);
+ }
+
+ if (!ret) {
+ msg_err("cannot save privkey to file: %s, %s", filename,
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushboolean(L, FALSE);
+ }
+ else {
+ lua_pushboolean(L, TRUE);
+ }
+
+ if (f != stdout) {
+ fclose(f);
+ }
+ else {
+ fflush(f);
+ }
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_rsa_pubkey_create(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ const gchar *buf;
+ BIO *bp;
+
+ buf = luaL_checkstring(L, 1);
+ if (buf != NULL) {
+ bp = BIO_new_mem_buf((void *) buf, -1);
+
+ if (!PEM_read_bio_RSA_PUBKEY(bp, &rsa, NULL, NULL)) {
+ msg_err("cannot parse pubkey: %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_pubkey}", -1);
+ *prsa = rsa;
+ }
+ BIO_free(bp);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static gint
+lua_rsa_pubkey_gc(lua_State *L)
+{
+ RSA *rsa = lua_check_rsa_pubkey(L, 1);
+
+ if (rsa != NULL) {
+ RSA_free(rsa);
+ }
+
+ return 0;
+}
+
+static gint
+lua_rsa_pubkey_tostring(lua_State *L)
+{
+ RSA *rsa = lua_check_rsa_pubkey(L, 1);
+
+ if (rsa != NULL) {
+ BIO *pubout = BIO_new(BIO_s_mem());
+ const gchar *pubdata;
+ gsize publen;
+ int rc = i2d_RSA_PUBKEY_bio(pubout, rsa);
+
+ if (rc != 1) {
+ BIO_free(pubout);
+
+ return luaL_error(L, "i2d_RSA_PUBKEY_bio failed");
+ }
+
+ publen = BIO_get_mem_data(pubout, &pubdata);
+ lua_pushlstring(L, pubdata, publen);
+ BIO_free(pubout);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_load_file(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ const gchar *filename;
+ FILE *f;
+
+ filename = luaL_checkstring(L, 1);
+ if (filename != NULL) {
+ f = fopen(filename, "r");
+ if (f == NULL) {
+ msg_err("cannot open private key from file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (!PEM_read_RSAPrivateKey(f, &rsa, NULL, NULL)) {
+ msg_err("cannot open private key from file: %s, %s", filename,
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = rsa;
+ }
+ fclose(f);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_load_pem(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ BIO *b;
+ struct rspamd_lua_text *t;
+ const gchar *data;
+ gsize len;
+
+ if (lua_isuserdata(L, 1)) {
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 1, &len);
+ }
+
+ if (data != NULL) {
+ b = BIO_new_mem_buf(data, len);
+
+ if (!PEM_read_bio_RSAPrivateKey(b, &rsa, NULL, NULL)) {
+ msg_err("cannot open private key from data, %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = rsa;
+ }
+
+ BIO_free(b);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_load_raw(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ BIO *b;
+ struct rspamd_lua_text *t;
+ const gchar *data;
+ gsize len;
+
+ if (lua_isuserdata(L, 1)) {
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 1, &len);
+ }
+
+ if (data != NULL) {
+ b = BIO_new_mem_buf(data, len);
+ rsa = d2i_RSAPrivateKey_bio(b, NULL);
+
+ if (rsa == NULL) {
+ msg_err("cannot open private key from data, %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = rsa;
+ }
+
+ BIO_free(b);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_load_base64(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ BIO *b;
+ EVP_PKEY *evp = NULL;
+ struct rspamd_lua_text *t;
+ const gchar *data;
+ guchar *decoded;
+ gsize len, dec_len;
+
+ if (lua_isuserdata(L, 1)) {
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 1, &len);
+ }
+
+ if (data != NULL) {
+ decoded = g_malloc(len);
+
+ if (!rspamd_cryptobox_base64_decode(data, len, decoded, &dec_len)) {
+ g_free(decoded);
+
+ return luaL_error(L, "invalid base64 encoding");
+ }
+
+ b = BIO_new_mem_buf(decoded, dec_len);
+
+ if (d2i_PrivateKey_bio(b, &evp) != NULL) {
+ rsa = EVP_PKEY_get1_RSA(evp);
+
+ if (rsa == NULL) {
+ msg_err("cannot open RSA private key from data, %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = rsa;
+ }
+
+ EVP_PKEY_free(evp);
+ }
+ else {
+ msg_err("cannot open EVP private key from data, %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+
+ BIO_free(b);
+ g_free(decoded);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_create(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ const gchar *buf;
+ BIO *bp;
+
+ buf = luaL_checkstring(L, 1);
+ if (buf != NULL) {
+ bp = BIO_new_mem_buf((void *) buf, -1);
+
+ if (!PEM_read_bio_RSAPrivateKey(bp, &rsa, NULL, NULL)) {
+ msg_err("cannot parse private key: %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = rsa;
+ }
+ BIO_free(bp);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_gc(lua_State *L)
+{
+ RSA *rsa = lua_check_rsa_privkey(L, 1);
+
+ if (rsa != NULL) {
+ RSA_free(rsa);
+ }
+
+ return 0;
+}
+
+static gint
+lua_rsa_signature_load(lua_State *L)
+{
+ rspamd_fstring_t *sig, **psig;
+ const gchar *filename;
+ gpointer data;
+ int fd;
+ struct stat st;
+
+ filename = luaL_checkstring(L, 1);
+ if (filename != NULL) {
+ fd = open(filename, O_RDONLY);
+ if (fd == -1) {
+ msg_err("cannot open signature file: %s, %s", filename,
+ strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (fstat(fd, &st) == -1 ||
+ (data =
+ mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
+ msg_err("cannot mmap file %s: %s", filename, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ sig = rspamd_fstring_new_init(data, st.st_size);
+ psig = lua_newuserdata(L, sizeof(rspamd_fstring_t *));
+ rspamd_lua_setclass(L, "rspamd{rsa_signature}", -1);
+ *psig = sig;
+ munmap(data, st.st_size);
+ }
+ close(fd);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static gint
+lua_rsa_signature_save(lua_State *L)
+{
+ rspamd_fstring_t *sig;
+ gint fd, flags;
+ const gchar *filename;
+ gboolean forced = FALSE, res = TRUE;
+
+ sig = lua_check_rsa_sign(L, 1);
+ filename = luaL_checkstring(L, 2);
+ if (lua_gettop(L) > 2) {
+ forced = lua_toboolean(L, 3);
+ }
+
+ if (sig != NULL && filename != NULL) {
+ flags = O_WRONLY | O_CREAT;
+ if (forced) {
+ flags |= O_TRUNC;
+ }
+ else {
+ flags |= O_EXCL;
+ }
+ fd = open(filename, flags, 00644);
+ if (fd == -1) {
+ msg_err("cannot create a signature file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushboolean(L, FALSE);
+ }
+ else {
+ while (write(fd, sig->str, sig->len) == -1) {
+ if (errno == EINTR) {
+ continue;
+ }
+ msg_err("cannot write to a signature file: %s, %s",
+ filename,
+ strerror(errno));
+ res = FALSE;
+ break;
+ }
+ lua_pushboolean(L, res);
+ close(fd);
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_signature_create(lua_State *L)
+{
+ rspamd_fstring_t *sig, **psig;
+ const gchar *data;
+ gsize dlen;
+
+ data = luaL_checklstring(L, 1, &dlen);
+ if (data != NULL) {
+ sig = rspamd_fstring_new_init(data, dlen);
+ psig = lua_newuserdata(L, sizeof(rspamd_fstring_t *));
+ rspamd_lua_setclass(L, "rspamd{rsa_signature}", -1);
+ *psig = sig;
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_signature_gc(lua_State *L)
+{
+ rspamd_fstring_t *sig = lua_check_rsa_sign(L, 1);
+
+ rspamd_fstring_free(sig);
+
+ return 0;
+}
+
+static gint
+lua_rsa_signature_base64(lua_State *L)
+{
+ rspamd_fstring_t *sig = lua_check_rsa_sign(L, 1);
+ guint boundary = 0;
+ gchar *b64;
+ gsize outlen;
+ enum rspamd_newlines_type how = RSPAMD_TASK_NEWLINES_CRLF;
+
+ if (lua_isnumber(L, 2)) {
+ boundary = lua_tonumber(L, 2);
+ }
+
+ if (lua_isstring(L, 3)) {
+ const gchar *how_str = lua_tostring(L, 3);
+
+ if (strcmp(how_str, "cr") == 0) {
+ how = RSPAMD_TASK_NEWLINES_CR;
+ }
+ else if (strcmp(how_str, "lf") == 0) {
+ how = RSPAMD_TASK_NEWLINES_LF;
+ }
+ else {
+ how = RSPAMD_TASK_NEWLINES_CRLF;
+ }
+ }
+
+ b64 = rspamd_encode_base64_fold(sig->str, sig->len, boundary, &outlen, how);
+
+ if (b64) {
+ lua_pushlstring(L, b64, outlen);
+ g_free(b64);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/**
+ * Check memory using specified rsa key and signature
+ *
+ * arguments:
+ * (rsa_pubkey, rsa_signature, string)
+ *
+ * returns:
+ * true - if string match rsa signature
+ * false - otherwise
+ */
+static gint
+lua_rsa_verify_memory(lua_State *L)
+{
+ RSA *rsa;
+ rspamd_fstring_t *signature;
+ const gchar *data;
+ gsize sz;
+ gint ret;
+
+ rsa = lua_check_rsa_pubkey(L, 1);
+ signature = lua_check_rsa_sign(L, 2);
+ data = luaL_checklstring(L, 3, &sz);
+
+ if (rsa != NULL && signature != NULL && data != NULL) {
+ ret = RSA_verify(NID_sha256, data, sz,
+ signature->str, signature->len, rsa);
+
+ if (ret == 0) {
+ lua_pushboolean(L, FALSE);
+ lua_pushstring(L, ERR_error_string(ERR_get_error(), NULL));
+
+ return 2;
+ }
+ else {
+ lua_pushboolean(L, TRUE);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/**
+ * Sign memory using specified rsa key and signature
+ *
+ * arguments:
+ * (rsa_privkey, string)
+ *
+ * returns:
+ * rspamd_signature object
+ * nil - otherwise
+ */
+static gint
+lua_rsa_sign_memory(lua_State *L)
+{
+ RSA *rsa;
+ rspamd_fstring_t *signature, **psig;
+ const gchar *data;
+ gsize sz;
+ gint ret;
+
+ rsa = lua_check_rsa_privkey(L, 1);
+ data = luaL_checklstring(L, 2, &sz);
+
+ if (rsa != NULL && data != NULL) {
+ signature = rspamd_fstring_sized_new(RSA_size(rsa));
+
+ guint siglen = signature->len;
+ ret = RSA_sign(NID_sha256, data, sz,
+ signature->str, &siglen, rsa);
+
+ if (ret != 1) {
+ rspamd_fstring_free(signature);
+
+ return luaL_error(L, "cannot sign: %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ }
+ else {
+ signature->len = siglen;
+ psig = lua_newuserdata(L, sizeof(rspamd_fstring_t *));
+ rspamd_lua_setclass(L, "rspamd{rsa_signature}", -1);
+ *psig = signature;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_keypair(lua_State *L)
+{
+ BIGNUM *e;
+ RSA *rsa, *pub_rsa, *priv_rsa, **prsa;
+ gint bits = lua_gettop(L) > 0 ? lua_tointeger(L, 1) : 1024;
+
+ if (bits > 4096 || bits < 512) {
+ return luaL_error(L, "invalid bits count");
+ }
+
+ e = BN_new();
+ rsa = RSA_new();
+ g_assert(BN_set_word(e, RSA_F4) == 1);
+ g_assert(RSA_generate_key_ex(rsa, bits, e, NULL) == 1);
+
+ priv_rsa = RSAPrivateKey_dup(rsa);
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = priv_rsa;
+
+ pub_rsa = RSAPublicKey_dup(rsa);
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_pubkey}", -1);
+ *prsa = pub_rsa;
+
+ RSA_free(rsa);
+ BN_free(e);
+
+ return 2;
+}
+
+static gint
+lua_load_pubkey(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, rsapubkeylib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_privkey(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, rsaprivkeylib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_signature(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, rsasignlib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_rsa(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, rsalib_f);
+
+ return 1;
+}
+
+void luaopen_rsa(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{rsa_pubkey}", rsapubkeylib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_rsa_pubkey", lua_load_pubkey);
+
+ rspamd_lua_new_class(L, "rspamd{rsa_privkey}", rsaprivkeylib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_rsa_privkey", lua_load_privkey);
+
+ rspamd_lua_new_class(L, "rspamd{rsa_signature}", rsasignlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_rsa_signature", lua_load_signature);
+
+ rspamd_lua_add_preload(L, "rspamd_rsa", lua_load_rsa);
+
+ lua_settop(L, 0);
+}
diff --git a/src/lua/lua_spf.c b/src/lua/lua_spf.c
new file mode 100644
index 0000000..a67a267
--- /dev/null
+++ b/src/lua/lua_spf.c
@@ -0,0 +1,620 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file lua_spf.c
+ * This module exports spf functions to Lua
+ */
+
+#include "lua_common.h"
+#include "libserver/spf.h"
+#include "libutil/ref.h"
+
+#define SPF_RECORD_CLASS "rspamd{spf_record}"
+
+LUA_FUNCTION_DEF(spf, resolve);
+LUA_FUNCTION_DEF(spf, config);
+
+LUA_FUNCTION_DEF(spf_record, check_ip);
+LUA_FUNCTION_DEF(spf_record, dtor);
+LUA_FUNCTION_DEF(spf_record, get_domain);
+LUA_FUNCTION_DEF(spf_record, get_elts);
+LUA_FUNCTION_DEF(spf_record, get_ttl);
+LUA_FUNCTION_DEF(spf_record, get_timestamp);
+LUA_FUNCTION_DEF(spf_record, get_digest);
+
+static luaL_reg rspamd_spf_f[] = {
+ LUA_INTERFACE_DEF(spf, resolve),
+ LUA_INTERFACE_DEF(spf, config),
+ {NULL, NULL},
+};
+
+static luaL_reg rspamd_spf_record_m[] = {
+ LUA_INTERFACE_DEF(spf_record, check_ip),
+ LUA_INTERFACE_DEF(spf_record, get_domain),
+ LUA_INTERFACE_DEF(spf_record, get_ttl),
+ LUA_INTERFACE_DEF(spf_record, get_digest),
+ LUA_INTERFACE_DEF(spf_record, get_elts),
+ LUA_INTERFACE_DEF(spf_record, get_timestamp),
+ {"__gc", lua_spf_record_dtor},
+ {NULL, NULL},
+};
+
+struct rspamd_lua_spf_cbdata {
+ struct rspamd_task *task;
+ lua_State *L;
+ struct rspamd_symcache_dynamic_item *item;
+ gint cbref;
+ ref_entry_t ref;
+};
+
+static gint
+lua_load_spf(lua_State *L)
+{
+ lua_newtable(L);
+
+ /* Create integer arguments to check SPF results */
+ lua_newtable(L);
+ lua_pushinteger(L, SPF_FAIL);
+ lua_setfield(L, -2, "fail");
+ lua_pushinteger(L, SPF_PASS);
+ lua_setfield(L, -2, "pass");
+ lua_pushinteger(L, SPF_NEUTRAL);
+ lua_setfield(L, -2, "neutral");
+ lua_pushinteger(L, SPF_SOFT_FAIL);
+ lua_setfield(L, -2, "soft_fail");
+
+ lua_setfield(L, -2, "policy");
+
+ /* Flags stuff */
+ lua_newtable(L);
+
+ lua_pushinteger(L, RSPAMD_SPF_RESOLVED_TEMP_FAILED);
+ lua_setfield(L, -2, "temp_fail");
+ lua_pushinteger(L, RSPAMD_SPF_RESOLVED_NA);
+ lua_setfield(L, -2, "na");
+ lua_pushinteger(L, RSPAMD_SPF_RESOLVED_PERM_FAILED);
+ lua_setfield(L, -2, "perm_fail");
+ lua_pushinteger(L, RSPAMD_SPF_FLAG_CACHED);
+ lua_setfield(L, -2, "cached");
+
+ lua_setfield(L, -2, "flags");
+
+ luaL_register(L, NULL, rspamd_spf_f);
+
+ return 1;
+}
+
+void luaopen_spf(lua_State *L)
+{
+ rspamd_lua_new_class(L, SPF_RECORD_CLASS, rspamd_spf_record_m);
+ lua_pop(L, 1); /* No need in metatable... */
+
+ rspamd_lua_add_preload(L, "rspamd_spf", lua_load_spf);
+ lua_settop(L, 0);
+}
+
+static void
+lua_spf_push_result(struct rspamd_lua_spf_cbdata *cbd, gint code_flags,
+ struct spf_resolved *resolved, const gchar *err)
+{
+ g_assert(cbd != NULL);
+ REF_RETAIN(cbd);
+
+ lua_pushcfunction(cbd->L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(cbd->L);
+
+ lua_rawgeti(cbd->L, LUA_REGISTRYINDEX, cbd->cbref);
+
+ if (resolved) {
+ struct spf_resolved **presolved;
+
+ presolved = lua_newuserdata(cbd->L, sizeof(*presolved));
+ rspamd_lua_setclass(cbd->L, SPF_RECORD_CLASS, -1);
+ *presolved = spf_record_ref(resolved);
+ }
+ else {
+ lua_pushnil(cbd->L);
+ }
+
+ lua_pushinteger(cbd->L, code_flags);
+
+ if (err) {
+ lua_pushstring(cbd->L, err);
+ }
+ else {
+ lua_pushnil(cbd->L);
+ }
+
+ if (lua_pcall(cbd->L, 3, 0, err_idx) != 0) {
+ struct rspamd_task *task = cbd->task;
+
+ msg_err_task("cannot call callback function for spf: %s",
+ lua_tostring(cbd->L, -1));
+ }
+
+ lua_settop(cbd->L, err_idx - 1);
+
+ REF_RELEASE(cbd);
+}
+
+static void
+lua_spf_dtor(struct rspamd_lua_spf_cbdata *cbd)
+{
+ if (cbd) {
+ luaL_unref(cbd->L, LUA_REGISTRYINDEX, cbd->cbref);
+ if (cbd->item) {
+ rspamd_symcache_item_async_dec_check(cbd->task, cbd->item,
+ "lua_spf");
+ }
+ }
+}
+
+static void
+spf_lua_lib_callback(struct spf_resolved *record, struct rspamd_task *task,
+ gpointer ud)
+{
+ struct rspamd_lua_spf_cbdata *cbd = (struct rspamd_lua_spf_cbdata *) ud;
+
+ if (record) {
+ if ((record->flags & RSPAMD_SPF_RESOLVED_NA)) {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_NA, NULL,
+ "no SPF record");
+ }
+ else if (record->elts->len == 0) {
+ if (record->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED) {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL,
+ "bad SPF record");
+ }
+ else if ((record->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED)) {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_TEMP_FAILED, NULL,
+ "temporary DNS error");
+ }
+ else {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL,
+ "empty SPF record");
+ }
+ }
+ else if (record->domain) {
+ spf_record_ref(record);
+ lua_spf_push_result(cbd, record->flags, record, NULL);
+ spf_record_unref(record);
+ }
+ else {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL,
+ "internal error: non empty record for no domain");
+ }
+ }
+ else {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL,
+ "internal error: no record");
+ }
+
+ REF_RELEASE(cbd);
+}
+
+/***
+ * @function rspamd_spf.resolve(task, callback)
+ * Resolves SPF credentials for a task
+ * @param {rspamd_task} task task
+ * @param {function} callback callback that is called on spf resolution
+*/
+gint lua_spf_resolve(lua_State *L)
+{
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task && lua_isfunction(L, 2)) {
+ struct rspamd_lua_spf_cbdata *cbd = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(*cbd));
+ struct rspamd_spf_cred *spf_cred;
+
+ cbd->task = task;
+ cbd->L = L;
+ lua_pushvalue(L, 2);
+ cbd->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ /* TODO: make it as an optional parameter */
+ spf_cred = rspamd_spf_get_cred(task);
+ cbd->item = rspamd_symcache_get_cur_item(task);
+
+ if (cbd->item) {
+ rspamd_symcache_item_async_inc(task, cbd->item, "lua_spf");
+ }
+ REF_INIT_RETAIN(cbd, lua_spf_dtor);
+
+ if (!rspamd_spf_resolve(task, spf_lua_lib_callback, cbd, spf_cred)) {
+ msg_info_task("cannot make spf request for %s",
+ spf_cred ? spf_cred->domain : "empty domain");
+ if (spf_cred) {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_TEMP_FAILED,
+ NULL, "DNS failed");
+ }
+ else {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_NA,
+ NULL, "No domain");
+ }
+ REF_RELEASE(cbd);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_spf_record_dtor(lua_State *L)
+{
+ struct spf_resolved *record;
+
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ spf_record_unref(record);
+ }
+
+ return 0;
+}
+
+static void
+lua_spf_push_spf_addr(lua_State *L, struct spf_addr *addr)
+{
+ gchar *addr_mask;
+
+ lua_createtable(L, 0, 4);
+
+ lua_pushinteger(L, addr->mech);
+ lua_setfield(L, -2, "result");
+ lua_pushinteger(L, addr->flags);
+ lua_setfield(L, -2, "flags");
+
+ if (addr->spf_string) {
+ lua_pushstring(L, addr->spf_string);
+ lua_setfield(L, -2, "str");
+ }
+
+ addr_mask = spf_addr_mask_to_string(addr);
+
+ if (addr_mask) {
+ lua_pushstring(L, addr_mask);
+ lua_setfield(L, -2, "addr");
+ g_free(addr_mask);
+ }
+}
+
+static gint
+spf_check_element(lua_State *L, struct spf_resolved *rec, struct spf_addr *addr,
+ struct rspamd_lua_ip *ip)
+{
+ gboolean res = FALSE;
+ const guint8 *s, *d;
+ guint af, mask, bmask, addrlen;
+
+
+ if (addr->flags & RSPAMD_SPF_FLAG_TEMPFAIL) {
+ /* Ignore failed addresses */
+
+ return -1;
+ }
+
+ af = rspamd_inet_address_get_af(ip->addr);
+ /* Basic comparing algorithm */
+ if (((addr->flags & RSPAMD_SPF_FLAG_IPV6) && af == AF_INET6) ||
+ ((addr->flags & RSPAMD_SPF_FLAG_IPV4) && af == AF_INET)) {
+ d = rspamd_inet_address_get_hash_key(ip->addr, &addrlen);
+
+ if (af == AF_INET6) {
+ s = (const guint8 *) addr->addr6;
+ mask = addr->m.dual.mask_v6;
+ }
+ else {
+ s = (const guint8 *) addr->addr4;
+ mask = addr->m.dual.mask_v4;
+ }
+
+ /* Compare the first bytes */
+ bmask = mask / CHAR_BIT;
+ if (mask > addrlen * CHAR_BIT) {
+ /* XXX: add logging */
+ }
+ else if (memcmp(s, d, bmask) == 0) {
+ if (bmask * CHAR_BIT < mask) {
+ /* Compare the remaining bits */
+ s += bmask;
+ d += bmask;
+ mask = (0xff << (CHAR_BIT - (mask - bmask * 8))) & 0xff;
+
+ if ((*s & mask) == (*d & mask)) {
+ res = TRUE;
+ }
+ }
+ else {
+ res = TRUE;
+ }
+ }
+ }
+ else {
+ if (addr->flags & RSPAMD_SPF_FLAG_ANY) {
+ res = TRUE;
+ }
+ else {
+ res = FALSE;
+ }
+ }
+
+ if (res) {
+ if (addr->flags & RSPAMD_SPF_FLAG_ANY) {
+ if (rec->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED) {
+ lua_pushboolean(L, false);
+ lua_pushinteger(L, RSPAMD_SPF_RESOLVED_PERM_FAILED);
+ lua_pushfstring(L, "%cany", spf_mech_char(addr->mech));
+ }
+ else if (rec->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED) {
+ lua_pushboolean(L, false);
+ lua_pushinteger(L, RSPAMD_SPF_RESOLVED_TEMP_FAILED);
+ lua_pushfstring(L, "%cany", spf_mech_char(addr->mech));
+ }
+ else {
+ lua_pushboolean(L, true);
+ lua_pushinteger(L, addr->mech);
+ lua_spf_push_spf_addr(L, addr);
+ }
+ }
+ else {
+ lua_pushboolean(L, true);
+ lua_pushinteger(L, addr->mech);
+ lua_spf_push_spf_addr(L, addr);
+ }
+
+ return 3;
+ }
+
+ return -1;
+}
+
+/***
+ * @method rspamd_spf_record:check_ip(ip)
+ * Checks the processed record versus a specific IP address. This function
+ * returns 3 values normally:
+ * 1. Boolean check result
+ * 2. If result is `false` then the second value is the error flag (e.g. rspamd_spf.flags.temp_fail), otherwise it will be an SPF method
+ * 3. If result is `false` then this will be an error string, otherwise - an SPF string (e.g. `mx` or `ip4:x.y.z.1`)
+ * @param {rspamd_ip|string} ip address
+ * @return {result,flag_or_policy,error_or_addr} - triplet
+*/
+static gint
+lua_spf_record_check_ip(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+ struct rspamd_lua_ip *ip = NULL;
+ gint nres = 0;
+ gboolean need_free_ip = FALSE;
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ ip = lua_check_ip(L, 2);
+ }
+ else if (lua_type(L, 2) == LUA_TSTRING) {
+ const gchar *ip_str;
+ gsize iplen;
+
+ ip = g_malloc0(sizeof(struct rspamd_lua_ip));
+ ip_str = lua_tolstring(L, 2, &iplen);
+
+ if (!rspamd_parse_inet_address(&ip->addr,
+ ip_str, iplen, RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ g_free(ip);
+ ip = NULL;
+ }
+ else {
+ need_free_ip = TRUE;
+ }
+ }
+
+ if (record && ip && ip->addr) {
+ for (guint i = 0; i < record->elts->len; i++) {
+ struct spf_addr *addr = &g_array_index(record->elts, struct spf_addr, i);
+ if ((nres = spf_check_element(L, record, addr, ip)) > 0) {
+ if (need_free_ip) {
+ g_free(ip);
+ }
+
+ return nres;
+ }
+ }
+ }
+ else {
+ if (need_free_ip) {
+ g_free(ip);
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (need_free_ip) {
+ g_free(ip);
+ }
+
+ /* If we are here it means that there is no ALL record */
+ /*
+ * According to https://tools.ietf.org/html/rfc7208#section-4.7 it means
+ * SPF neutral
+ */
+ struct spf_addr fake_all;
+
+ fake_all.mech = SPF_NEUTRAL;
+ fake_all.flags = RSPAMD_SPF_FLAG_ANY;
+ fake_all.spf_string = "all";
+
+ lua_pushboolean(L, true);
+ lua_pushinteger(L, SPF_NEUTRAL);
+ lua_spf_push_spf_addr(L, &fake_all);
+
+ return 3;
+}
+
+/***
+ * @method rspamd_spf_record:get_domain()
+ * Returns domain for the specific spf record
+*/
+static gint
+lua_spf_record_get_domain(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ lua_pushstring(L, record->domain);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method rspamd_spf_record:get_ttl()
+ * Returns ttl for the specific spf record
+*/
+static gint
+lua_spf_record_get_ttl(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ lua_pushinteger(L, record->ttl);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method rspamd_spf_record:get_timestamp()
+ * Returns ttl for the specific spf record
+*/
+static gint
+lua_spf_record_get_timestamp(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ lua_pushnumber(L, record->timestamp);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method rspamd_spf_record:get_digest()
+ * Returns string hex representation of the record digest (fast hash function)
+*/
+static gint
+lua_spf_record_get_digest(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ gchar hexbuf[64];
+
+ rspamd_snprintf(hexbuf, sizeof(hexbuf), "%xuL", record->digest);
+ lua_pushstring(L, hexbuf);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method rspamd_spf_record:get_elts()
+ * Returns a list of all elements in an SPF record. Each element is a table with the
+ * following fields:
+ *
+ * - result - mech flag from rspamd_spf.results
+ * - flags - all flags
+ * - addr - address and mask as a string
+ * - str - string representation (if available)
+*/
+static gint
+lua_spf_record_get_elts(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ guint i;
+ struct spf_addr *addr;
+
+ lua_createtable(L, record->elts->len, 0);
+
+ for (i = 0; i < record->elts->len; i++) {
+ addr = (struct spf_addr *) &g_array_index(record->elts,
+ struct spf_addr, i);
+ lua_spf_push_spf_addr(L, addr);
+
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_spf.config(object)
+ * Configures SPF library according to the UCL config
+ * @param {table} object configuration object
+*/
+gint lua_spf_config(lua_State *L)
+{
+ ucl_object_t *config_obj = ucl_object_lua_import(L, 1);
+
+ if (config_obj) {
+ spf_library_config(config_obj);
+ ucl_object_unref(config_obj); /* As we copy data all the time */
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+} \ No newline at end of file
diff --git a/src/lua/lua_sqlite3.c b/src/lua/lua_sqlite3.c
new file mode 100644
index 0000000..be7a9ae
--- /dev/null
+++ b/src/lua/lua_sqlite3.c
@@ -0,0 +1,379 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "sqlite_utils.h"
+
+/***
+ * @module rspamd_sqlite3
+ * This module provides routines to query sqlite3 databases
+@example
+local sqlite3 = require "rspamd_sqlite3"
+
+local db = sqlite3.open("/tmp/db.sqlite")
+
+if db then
+ db:exec([[ CREATE TABLE x (id INT, value TEXT); ]])
+
+ db:exec([[ INSERT INTO x VALUES (?1, ?2); ]], 1, 'test')
+
+ for row in db:rows([[ SELECT * FROM x ]]) do
+ print(string.format('%d -> %s', row.id, row.value))
+ end
+end
+ */
+
+LUA_FUNCTION_DEF(sqlite3, open);
+LUA_FUNCTION_DEF(sqlite3, sql);
+LUA_FUNCTION_DEF(sqlite3, rows);
+LUA_FUNCTION_DEF(sqlite3, close);
+LUA_FUNCTION_DEF(sqlite3_stmt, close);
+
+static const struct luaL_reg sqlitelib_f[] = {
+ LUA_INTERFACE_DEF(sqlite3, open),
+ {NULL, NULL}};
+
+static const struct luaL_reg sqlitelib_m[] = {
+ LUA_INTERFACE_DEF(sqlite3, sql),
+ {"query", lua_sqlite3_sql},
+ {"exec", lua_sqlite3_sql},
+ LUA_INTERFACE_DEF(sqlite3, rows),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_sqlite3_close},
+ {NULL, NULL}};
+
+static const struct luaL_reg sqlitestmtlib_m[] = {
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_sqlite3_stmt_close},
+ {NULL, NULL}};
+
+static void lua_sqlite3_push_row(lua_State *L, sqlite3_stmt *stmt);
+
+static sqlite3 *
+lua_check_sqlite3(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{sqlite3}");
+ luaL_argcheck(L, ud != NULL, pos, "'sqlite3' expected");
+ return ud ? *((sqlite3 **) ud) : NULL;
+}
+
+static sqlite3_stmt *
+lua_check_sqlite3_stmt(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{sqlite3_stmt}");
+ luaL_argcheck(L, ud != NULL, pos, "'sqlite3_stmt' expected");
+ return ud ? *((sqlite3_stmt **) ud) : NULL;
+}
+
+
+/***
+ * @function rspamd_sqlite3.open(path)
+ * Opens sqlite3 database at the specified path. DB is created if not exists.
+ * @param {string} path path to db
+ * @return {sqlite3} sqlite3 handle
+ */
+static gint
+lua_sqlite3_open(lua_State *L)
+{
+ const gchar *path = luaL_checkstring(L, 1);
+ sqlite3 *db, **pdb;
+ GError *err = NULL;
+
+ if (path == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ db = rspamd_sqlite3_open_or_create(NULL, path, NULL, 0, &err);
+
+ if (db == NULL) {
+ if (err) {
+ msg_err("cannot open db: %e", err);
+ g_error_free(err);
+ }
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ pdb = lua_newuserdata(L, sizeof(db));
+ *pdb = db;
+ rspamd_lua_setclass(L, "rspamd{sqlite3}", -1);
+
+ return 1;
+}
+
+static void
+lua_sqlite3_bind_statements(lua_State *L, gint start, gint end,
+ sqlite3_stmt *stmt)
+{
+ gint i, type, num = 1;
+ const gchar *str;
+ gsize slen;
+ gdouble n;
+
+ g_assert(start <= end && start > 0 && end > 0);
+
+ for (i = start; i <= end; i++) {
+ type = lua_type(L, i);
+
+ switch (type) {
+ case LUA_TNUMBER:
+ n = lua_tonumber(L, i);
+
+ if (n == (gdouble) ((gint64) n)) {
+ sqlite3_bind_int64(stmt, num, n);
+ }
+ else {
+ sqlite3_bind_double(stmt, num, n);
+ }
+ num++;
+ break;
+ case LUA_TSTRING:
+ str = lua_tolstring(L, i, &slen);
+ sqlite3_bind_text(stmt, num, str, slen, SQLITE_TRANSIENT);
+ num++;
+ break;
+ default:
+ msg_err("invalid type at position %d: %s", i, lua_typename(L, type));
+ break;
+ }
+ }
+}
+
+/***
+ * @function rspamd_sqlite3:sql(query[, args..])
+ * Performs sqlite3 query replacing '?1', '?2' and so on with the subsequent args
+ * of the function
+ *
+ * @param {string} query SQL query
+ * @param {string|number} args... variable number of arguments
+ * @return {boolean} `true` if a statement has been successfully executed
+ */
+static gint
+lua_sqlite3_sql(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ sqlite3 *db = lua_check_sqlite3(L, 1);
+ const gchar *query = luaL_checkstring(L, 2);
+ sqlite3_stmt *stmt;
+ gboolean ret = FALSE;
+ gint top = 1, rc;
+
+ if (db && query) {
+ if (sqlite3_prepare_v2(db, query, -1, &stmt, NULL) != SQLITE_OK) {
+ msg_err("cannot prepare query %s: %s", query, sqlite3_errmsg(db));
+ return luaL_error(L, sqlite3_errmsg(db));
+ }
+ else {
+ top = lua_gettop(L);
+
+ if (top > 2) {
+ /* Push additional arguments to sqlite3 */
+ lua_sqlite3_bind_statements(L, 3, top, stmt);
+ }
+
+ rc = sqlite3_step(stmt);
+ top = 1;
+
+ if (rc == SQLITE_ROW || rc == SQLITE_OK || rc == SQLITE_DONE) {
+ ret = TRUE;
+
+ if (rc == SQLITE_ROW) {
+ lua_sqlite3_push_row(L, stmt);
+ top = 2;
+ }
+ }
+ else {
+ msg_warn("sqlite3 error: %s", sqlite3_errmsg(db));
+ }
+
+ sqlite3_finalize(stmt);
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return top;
+}
+
+static void
+lua_sqlite3_push_row(lua_State *L, sqlite3_stmt *stmt)
+{
+ const gchar *str;
+ gsize slen;
+ gint64 num;
+ gchar numbuf[32];
+ gint nresults, i, type;
+
+ nresults = sqlite3_column_count(stmt);
+ lua_createtable(L, 0, nresults);
+
+ for (i = 0; i < nresults; i++) {
+ lua_pushstring(L, sqlite3_column_name(stmt, i));
+ type = sqlite3_column_type(stmt, i);
+
+ switch (type) {
+ case SQLITE_INTEGER:
+ /*
+ * XXX: we represent int64 as strings, as we can nothing else to do
+ * about it portably
+ */
+ num = sqlite3_column_int64(stmt, i);
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%uL", num);
+ lua_pushstring(L, numbuf);
+ break;
+ case SQLITE_FLOAT:
+ lua_pushnumber(L, sqlite3_column_double(stmt, i));
+ break;
+ case SQLITE_TEXT:
+ slen = sqlite3_column_bytes(stmt, i);
+ str = sqlite3_column_text(stmt, i);
+ lua_pushlstring(L, str, slen);
+ break;
+ case SQLITE_BLOB:
+ slen = sqlite3_column_bytes(stmt, i);
+ str = sqlite3_column_blob(stmt, i);
+ lua_pushlstring(L, str, slen);
+ break;
+ default:
+ lua_pushboolean(L, 0);
+ break;
+ }
+
+ lua_settable(L, -3);
+ }
+}
+
+static gint
+lua_sqlite3_next_row(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ sqlite3_stmt *stmt = *(sqlite3_stmt **) lua_touserdata(L, lua_upvalueindex(1));
+ gint rc;
+
+ if (stmt != NULL) {
+ rc = sqlite3_step(stmt);
+
+ if (rc == SQLITE_ROW) {
+ lua_sqlite3_push_row(L, stmt);
+ return 1;
+ }
+ }
+
+ lua_pushnil(L);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_sqlite3:rows(query[, args..])
+ * Performs sqlite3 query replacing '?1', '?2' and so on with the subsequent args
+ * of the function. This function returns iterator suitable for loop construction:
+ *
+ * @param {string} query SQL query
+ * @param {string|number} args... variable number of arguments
+ * @return {function} iterator to get all rows
+@example
+for row in db:rows([[ SELECT * FROM x ]]) do
+ print(string.format('%d -> %s', row.id, row.value))
+end
+ */
+static gint
+lua_sqlite3_rows(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ sqlite3 *db = lua_check_sqlite3(L, 1);
+ const gchar *query = luaL_checkstring(L, 2);
+ sqlite3_stmt *stmt, **pstmt;
+ gint top;
+
+ if (db && query) {
+ if (sqlite3_prepare_v2(db, query, -1, &stmt, NULL) != SQLITE_OK) {
+ msg_err("cannot prepare query %s: %s", query, sqlite3_errmsg(db));
+ lua_pushstring(L, sqlite3_errmsg(db));
+ return lua_error(L);
+ }
+ else {
+ top = lua_gettop(L);
+
+ if (top > 2) {
+ /* Push additional arguments to sqlite3 */
+ lua_sqlite3_bind_statements(L, 3, top, stmt);
+ }
+
+ /* Create C closure */
+ pstmt = lua_newuserdata(L, sizeof(stmt));
+ *pstmt = stmt;
+ rspamd_lua_setclass(L, "rspamd{sqlite3_stmt}", -1);
+
+ lua_pushcclosure(L, lua_sqlite3_next_row, 1);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_sqlite3_close(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ sqlite3 *db = lua_check_sqlite3(L, 1);
+
+ if (db) {
+ sqlite3_close(db);
+ }
+
+ return 0;
+}
+
+static gint
+lua_sqlite3_stmt_close(lua_State *L)
+{
+ sqlite3_stmt *stmt = lua_check_sqlite3_stmt(L, 1);
+
+ if (stmt) {
+ sqlite3_finalize(stmt);
+ }
+
+ return 0;
+}
+
+static gint
+lua_load_sqlite3(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, sqlitelib_f);
+
+ return 1;
+}
+/**
+ * Open redis library
+ * @param L lua stack
+ * @return
+ */
+void luaopen_sqlite3(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{sqlite3}", sqlitelib_m);
+ lua_pop(L, 1);
+
+ rspamd_lua_new_class(L, "rspamd{sqlite3_stmt}", sqlitestmtlib_m);
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "rspamd_sqlite3", lua_load_sqlite3);
+}
diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c
new file mode 100644
index 0000000..7278602
--- /dev/null
+++ b/src/lua/lua_task.c
@@ -0,0 +1,7295 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_url.h"
+
+#include "message.h"
+#include "images.h"
+#include "archives.h"
+#include "utlist.h"
+#include "unix-std.h"
+#include "libmime/smtp_parsers.h"
+#include "libserver/mempool_vars_internal.h"
+#include "libserver/dkim.h"
+#include "libserver/task.h"
+#include "libserver/cfg_file_private.h"
+#include "libmime/scan_result_private.h"
+#include "libstat/stat_api.h"
+#include "libserver/maps/map_helpers.h"
+
+#include <math.h>
+#include "libmime/received.h"
+
+/***
+ * @module rspamd_task
+ * This module provides routines for tasks manipulation in rspamd. Tasks usually
+ * represent messages being scanned, and this API provides access to such elements
+ * as headers, symbols, metrics and so on and so forth. Normally, task objects
+ * are passed to the lua callbacks allowing to check specific properties of messages
+ * and add the corresponding symbols to the scan's results.
+@example
+rspamd_config.DATE_IN_PAST = function(task)
+ local dm = task:get_date{format = 'message', gmt = true}
+ local dt = task:get_date{format = 'connect', gmt = true}
+ -- A day
+ if dt - dm > 86400 then
+ return true
+ end
+
+ return false
+end
+ */
+
+/* Task methods */
+
+/***
+ * @function rspamd_task.create([cfg])
+ * Create a new empty task
+ * @return {rspamd_task} new task
+ */
+LUA_FUNCTION_DEF(task, create);
+/***
+ * @function rspamd_task.load_from_file(filename[, cfg])
+ * Loads a message from specific file
+ * @return {boolean,rspamd_task|error} status + new task or error message
+ */
+LUA_FUNCTION_DEF(task, load_from_file);
+/***
+ * @function rspamd_task.load_from_string(message[, cfg])
+ * Loads a message from specific file
+ * @return {boolean,rspamd_task|error} status + new task or error message
+ */
+LUA_FUNCTION_DEF(task, load_from_string);
+/***
+ * @method task:get_message()
+ * Returns task raw message content as opaque text
+ * @return {rspamd_text} task raw content
+ */
+LUA_FUNCTION_DEF(task, get_message);
+/***
+ * @method task:set_message(msg)
+ * Updates task message with another message; It also parses a message to
+ * fill the internal structures.
+ * Input might be a string, a lua_text or a table of the former stuff.
+ * @param {string/text/table} msg new message to set
+ * @return {boolean,number} if a message has been set + its raw size
+ */
+LUA_FUNCTION_DEF(task, set_message);
+/***
+ * @method task:process_message()
+ * Parses message
+ */
+LUA_FUNCTION_DEF(task, process_message);
+/***
+ * @method task:get_cfg()
+ * Get configuration object for a task.
+ * @return {rspamd_config} (config.md)[configuration object] for the task
+ */
+LUA_FUNCTION_DEF(task, get_cfg);
+LUA_FUNCTION_DEF(task, set_cfg);
+LUA_FUNCTION_DEF(task, destroy);
+/***
+ * @method task:get_mempool()
+ * Returns memory pool valid for a lifetime of task. It is used internally by
+ * many rspamd routines.
+ * @return {rspamd_mempool} memory pool object
+ */
+LUA_FUNCTION_DEF(task, get_mempool);
+/***
+ * @method task:get_session()
+ * Returns asynchronous session object that is used by many rspamd asynchronous
+ * utilities internally.
+ * @return {rspamd_session} session object
+ */
+LUA_FUNCTION_DEF(task, get_session);
+/***
+ * @method task:set_session(session)
+ * Sets new async session for a task
+ */
+LUA_FUNCTION_DEF(task, set_session);
+/***
+ * @method task:get_ev_base()
+ * Return asynchronous event base for using in callbacks and resolver.
+ * @return {rspamd_ev_base} event base
+ */
+LUA_FUNCTION_DEF(task, get_ev_base);
+/***
+ * @method task:get_worker()
+ * Returns a worker object associated with the task
+ * @return {rspamd_worker} worker object
+ */
+LUA_FUNCTION_DEF(task, get_worker);
+/***
+ * @method task:insert_result([enforce_symbol,]symbol, weight[, option1, ...])
+ * Insert specific symbol to the tasks scanning results assigning the initial
+ * weight to it.
+ * @param {boolean} enforce_symbol if represented and true, then insert symbol even if it is not registered in the metric
+ * @param {string} symbol symbol to insert
+ * @param {number} weight initial weight (this weight is multiplied by the metric weight)
+ * @param {string} options list of optional options attached to a symbol inserted
+@example
+local function cb(task)
+ if task:get_header('Some header') then
+ task:insert_result('SOME_HEADER', 1.0, 'Got some header')
+ end
+end
+ */
+LUA_FUNCTION_DEF(task, insert_result);
+/***
+ * @method task:insert_result_named(shadow_result, [enforce_symbol,]symbol, weight[, option1, ...])
+ * Insert specific symbol to the tasks scanning results assigning the initial
+ * weight to it.
+ * @param {string} shadow_result name of shadow result
+ * @param {boolean} enforce_symbol if represented and true, then insert symbol even if it is not registered in the metric
+ * @param {string} symbol symbol to insert
+ * @param {number} weight initial weight (this weight is multiplied by the metric weight)
+ * @param {string} options list of optional options attached to a symbol inserted
+ */
+LUA_FUNCTION_DEF(task, insert_result_named);
+
+/***
+ * @method task:adjust_result(symbol, score[, option1, ...])
+ * Alters the existing symbol's score to a new score. It is not affected by
+ * metric score or grow factor. You can also add new options
+ * using this method. Symbol must be already inserted into metric or an error
+ * will be emitted.
+ * @param {string} symbol symbol to adjust
+ * @param {number} score this value is NOT multiplied by the metric score
+ * @param {string/table} options list of optional options attached to a symbol adjusted
+ */
+LUA_FUNCTION_DEF(task, adjust_result);
+
+/***
+ * @method task:remove_result(symbol[, shadow_result])
+ * Removes the symbol from a named or unnamed/default result
+ * @param {string} symbol symbol to remove
+ * @param {string} shadow_result name of shadow result
+ * @return {boolean} true if a symbol has been removed
+ */
+LUA_FUNCTION_DEF(task, remove_result);
+/***
+ * @method task:set_pre_result(action, [message, [module], [score], [priority], [flags])
+ * Sets pre-result for a task. It is used in pre-filters to specify early result
+ * of the task scanned. If a pre-filter sets some result, then further processing
+ * may be skipped. For selecting action it is possible to use global table
+ * `rspamd_actions` or a string value:
+ *
+ * - `reject`: reject message permanently
+ * - `add header`: add spam header
+ * - `rewrite subject`: rewrite subject to spam subject
+ * - `greylist`: greylist message
+ * - `accept` or `no action`: whitelist message
+ *
+ * This function also accepts a table from Rspamd 2.6 with the following keys:
+ * - action: string required
+ * - message: string
+ * - module: string
+ * - score: number
+ * - priority: integer
+ * - flags: flags string
+ * - result: named result if needed
+ *
+ * @param {rspamd_action or string} action a numeric or string action value
+ * @param {string} message action message
+ * @param {string} module optional module name
+ * @param {number/nil} score optional explicit score
+ * @param {number/nil} priority optional explicit priority
+ * @param {string/nil} flags optional flags (e.g. 'least' for least action)
+@example
+local function cb(task)
+ local gr = task:get_header('Greylist')
+ if gr and gr == 'greylist' then
+ task:set_pre_result('soft reject', 'Greylisting required')
+ end
+end
+ */
+LUA_FUNCTION_DEF(task, set_pre_result);
+
+/***
+ * @method task:has_pre_result()
+ * Returns true if task has some pre-result being set.
+ * If result has been set this function also returns pre result action,
+ * message and module as strings in this order.
+ *
+ * @return {boolean,[string,string,string]} true if task has some pre-result being set
+ */
+LUA_FUNCTION_DEF(task, has_pre_result);
+/***
+ * @method task:append_message(message, [category])
+ * Adds a message to scanning output.
+ * @param {string} message
+ * @param {category} message category
+@example
+local function cb(task)
+ task:append_message('Example message')
+end
+ */
+LUA_FUNCTION_DEF(task, append_message);
+/***
+ * @method task:get_urls([need_emails|list_protos][, need_images])
+ * Get all URLs found in a message. Telephone urls and emails are not included unless explicitly asked in `list_protos`
+ * @param {boolean} need_emails if `true` then return also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`)
+ * @param {boolean} need_images return urls from images (<img src=...>) as well
+ * @return {table rspamd_url} list of all urls found
+@example
+local function phishing_cb(task)
+ local urls = task:get_urls({'https', 'http'});
+
+ if urls then
+ for _,url in ipairs(urls) do
+ if url:is_phished() then
+ return true
+ end
+ end
+ end
+ return false
+end
+ */
+LUA_FUNCTION_DEF(task, get_urls);
+/***
+ * @method task:get_urls_filtered([{flags_include}, [{flags_exclude}]], [{protocols_mask}])
+ * Get urls managed by either exclude or include flags list
+ * - If flags include are nil then all but excluded urls are returned
+ * - If flags exclude are nil then only included explicitly urls are returned
+ * - If both parameters are nil then all urls are included
+ * @param {table} flags_include included flags
+ * @param {table} flags_exclude excluded flags
+ * @param {table} protocols_mask include only specific protocols
+ * @return {table rspamd_url} list of urls matching conditions
+ */
+LUA_FUNCTION_DEF(task, get_urls_filtered);
+/***
+ * @method task:has_urls([need_emails])
+ * Returns 'true' if a task has urls listed
+ * @param {boolean} need_emails if `true` then return also email urls
+ * @return {boolean} true if a task has urls (urls or emails if `need_emails` is true)
+ */
+LUA_FUNCTION_DEF(task, has_urls);
+/***
+ * @method task:inject_url(url)
+ * Inserts an url into a task (useful for redirected urls)
+ * @param {lua_url} url url to inject
+ */
+LUA_FUNCTION_DEF(task, inject_url);
+/***
+ * @method task:get_content()
+ * Get raw content for the specified task
+ * @return {text} the data contained in the task
+ */
+LUA_FUNCTION_DEF(task, get_content);
+
+/***
+ * @method task:get_filename()
+ * Returns filename for a specific task
+ * @return {string|nil} filename or nil if unknown
+ */
+LUA_FUNCTION_DEF(task, get_filename);
+
+/***
+ * @method task:get_rawbody()
+ * Get raw body for the specified task
+ * @return {text} the data contained in the task
+ */
+LUA_FUNCTION_DEF(task, get_rawbody);
+
+/***
+ * @method task:get_emails()
+ * Get all email addresses found in a message.
+ * @return {table rspamd_url} list of all email addresses found
+ */
+LUA_FUNCTION_DEF(task, get_emails);
+/***
+ * @method task:get_text_parts()
+ * Get all text (and HTML) parts found in a message
+ * @return {table rspamd_text_part} list of text parts
+ */
+LUA_FUNCTION_DEF(task, get_text_parts);
+/***
+ * @method task:get_parts()
+ * Get all mime parts found in a message
+ * @return {table rspamd_mime_part} list of mime parts
+ */
+LUA_FUNCTION_DEF(task, get_parts);
+/***
+ * @method task:get_meta_words([how='stem'])
+ * Get meta words from task (subject and displayed names)
+ * - `stem`: stemmed words (default)
+ * - `norm`: normalised words (utf normalised + lowercased)
+ * - `raw`: raw words in utf (if possible)
+ * - `full`: list of tables, each table has the following fields:
+ * - [1] - stemmed word
+ * - [2] - normalised word
+ * - [3] - raw word
+ * - [4] - flags (table of strings)
+ */
+LUA_FUNCTION_DEF(task, get_meta_words);
+/***
+ * @method task:get_request_header(name)
+ * Get value of a HTTP request header.
+ * @param {string} name name of header to get
+ * @return {rspamd_text} value of an HTTP header
+ */
+LUA_FUNCTION_DEF(task, get_request_header);
+/***
+ * @method task:set_request_header(name, value)
+ * Set value of a HTTP request header. If value is omitted, then a header is removed
+ * @param {string} name name of header to get
+ * @param {rspamd_text/string} value new header's value
+ */
+LUA_FUNCTION_DEF(task, set_request_header);
+/***
+ * @method task:get_subject()
+ * Returns task subject (either from the protocol override or from a header)
+ * @return {string} value of a subject (decoded)
+ */
+LUA_FUNCTION_DEF(task, get_subject);
+/***
+ * @method task:get_header(name[, case_sensitive])
+ * Get decoded value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {string} decoded value of a header
+ */
+LUA_FUNCTION_DEF(task, get_header);
+/***
+ * @method task:has_header(name[, case_sensitive])
+ * Get decoded value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in the case insensitive matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {boolean} true if header exists
+ */
+LUA_FUNCTION_DEF(task, has_header);
+/***
+ * @method task:get_header_raw(name[, case_sensitive])
+ * Get raw value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {string} raw value of a header
+ */
+LUA_FUNCTION_DEF(task, get_header_raw);
+/***
+ * @method task:get_header_full(name[, case_sensitive[, need_modified]])
+ * Get raw value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter. This method returns more
+ * information about the header as a list of tables with the following structure:
+ *
+ * - `name` - name of a header
+ * - `value` - raw value of a header
+ * - `decoded` - decoded value of a header
+ * - `tab_separated` - `true` if a header and a value are separated by `tab` character
+ * - `empty_separator` - `true` if there are no separator between a header and a value
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @param {boolean} need_modified return a modified value of a header if presented
+ * @return {list of tables} all values of a header as specified above
+@example
+function check_header_delimiter_tab(task, header_name)
+ for _,rh in ipairs(task:get_header_full(header_name)) do
+ if rh['tab_separated'] then return true end
+ end
+ return false
+end
+ */
+LUA_FUNCTION_DEF(task, get_header_full);
+/***
+ * @method task:get_header_count(name[, case_sensitive])
+ * Lightweight version if you need just a header's count
+ * * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {number} number of header's occurrences or 0 if not found
+ */
+LUA_FUNCTION_DEF(task, get_header_count);
+/***
+ * @method task:get_raw_headers()
+ * Get all undecoded headers of a message as a string
+ * @return {rspamd_text} all raw headers for a message as opaque text
+ */
+LUA_FUNCTION_DEF(task, get_raw_headers);
+
+/***
+ * @method task:get_headers([need_modified=false])
+ * Get all headers of a message in the same format as get_header_full
+ * @return {table of headers data} all headers for a message
+ */
+LUA_FUNCTION_DEF(task, get_headers);
+
+/***
+ * @method task:modify_header(name, mods)
+ * Modify an existing or non-existing header with the name `name`
+ * Mods is a table with the following structure:
+ * {
+ * "add" = { {order, value}, {order, value} },
+ * "remove" = {order, order, order}
+ * }
+ * Modifications are evaluated in order: remove, add, so headers are first
+ * removed (if any) and then added
+ * Order in remove starts from 1, where 0 means 'remove all', and negative value means
+ * remove from the end
+ * Order in addition means addition from the top: 0 means the most top header, 1 one after, etc
+ * negative order means addition to the end, e.g. -1 is appending header.
+ * @return {bool} true if header could be modified (always true unless we don't have an unparsed message)
+ */
+LUA_FUNCTION_DEF(task, modify_header);
+
+/***
+ * @method task:get_received_headers()
+ * Returns a list of tables of parsed received headers. A tables returned have
+ * the following structure:
+ *
+ * - `from_hostname` - string that represents hostname provided by a peer
+ * - `from_ip` - string representation of sending IP address
+ * - `real_hostname` - hostname as resolved by MTA
+ * - `real_ip` - rspamd_ip object representing sending IP address
+ * - `by_hostname` - MTA hostname
+ * - `proto` - protocol, e.g. ESMTP or ESMTPS
+ * - `timestamp` - received timestamp
+ * - `for` - for value (unparsed mailbox)
+ *
+ * Please note that in some situations rspamd cannot parse all the fields of received headers.
+ * In that case you should check all strings for validity.
+ * @return {table of tables} list of received headers described above
+ */
+LUA_FUNCTION_DEF(task, get_received_headers);
+/***
+ * @method task:get_queue_id()
+ * Returns queue ID of the message being processed.
+ */
+LUA_FUNCTION_DEF(task, get_queue_id);
+/***
+ * @method task:get_uid()
+ * Returns ID of the task being processed.
+ */
+LUA_FUNCTION_DEF(task, get_uid);
+/***
+ * @method task:get_resolver()
+ * Returns ready to use rspamd_resolver object suitable for making asynchronous DNS requests.
+ * @return {rspamd_resolver} resolver object associated with the task's session
+ * @example
+local logger = require "rspamd_logger"
+
+local function task_cb(task)
+ local function dns_cb(resolver, to_resolve, results, err)
+ -- task object is available due to closure
+ task:inc_dns_req()
+ if results then
+ logger.info(string.format('<%s> [%s] resolved for symbol: %s',
+ task:get_message_id(), to_resolve, 'EXAMPLE_SYMBOL'))
+ task:insert_result('EXAMPLE_SYMBOL', 1)
+ end
+ end
+ local r = task:get_resolver()
+ r:resolve_a(task:get_session(), task:get_mempool(), 'example.com', dns_cb)
+end
+ */
+LUA_FUNCTION_DEF(task, get_resolver);
+/***
+ * @method task:set_resolver(resolver)
+ * Sets rspamd_resolver for a specified task.
+ */
+LUA_FUNCTION_DEF(task, set_resolver);
+/***
+* @method task:inc_dns_req()
+* Increment number of DNS requests for the task. Is used just for logging purposes.
+*/
+LUA_FUNCTION_DEF(task, inc_dns_req);
+/***
+ * @method task:get_dns_req()
+ * Get number of dns requests being sent in the task
+ * @return {number} number of DNS requests
+ */
+LUA_FUNCTION_DEF(task, get_dns_req);
+
+/***
+ * @method task:has_recipients([type])
+ * Return true if there are SMTP or MIME recipients for a task.
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP recipients and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP recipients and `2` or `mime` means MIME recipients only
+ * @return {bool,integer} `true` if there are recipients of the following type and a number of such a recipients excluding artificial ones
+ */
+LUA_FUNCTION_DEF(task, has_recipients);
+
+/***
+ * @method task:get_recipients([type])
+ * Return SMTP or MIME recipients for a task. This function returns list of internet addresses each one is a table with the following structure:
+ *
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP recipients and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP recipients and `2` or `mime` means MIME recipients only
+ * @return {list of addresses} list of recipients or `nil`
+ */
+LUA_FUNCTION_DEF(task, get_recipients);
+
+/***
+ * @method task:get_principal_recipient()
+ * Returns a single string with so called `principal recipient` for a message. The order
+ * of check is the following:
+ *
+ * - deliver-to request header
+ * - the first recipient (envelope)
+ * - the first recipient (mime)
+ * @return {string} principal recipient
+ */
+LUA_FUNCTION_DEF(task, get_principal_recipient);
+/***
+ * @method task:get_reply_sender()
+ * Returns a single string with address that should be used to reply on a message
+ *
+ * - reply-to header
+ * - from header
+ * - smtp from as a last resort
+ * @return {address} email address
+ */
+LUA_FUNCTION_DEF(task, get_reply_sender);
+
+/***
+ * @method task:set_recipients([type], {rcpt1, rcpt2...}, [how='add'])
+ * Sets recipients for a task. This function accepts table that will be converted to the address.
+ * If some fields are missing they are subsequently reconstructed by this function. E.g. if you
+ * specify 'user' and 'domain', then address and raw string will be reconstructed
+ *
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP recipients and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP recipients and `2` or `mime` means MIME recipients only
+ * @param {list of tables} recipients recipients to set
+ * @param {string} how define how to set recipients: `rewrite` - rewrite existing recipients, `alias` - treat existing recipients as aliased recipients, `add` - add new recipients
+ * @return {boolean} result of the operation
+ */
+LUA_FUNCTION_DEF(task, set_recipients);
+
+/***
+ * @method task:has_from([type])
+ * Return true if there is SMTP or MIME sender for a task.
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP recipients and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP recipients and `2` or `mime` means MIME recipients only
+ * @return {bool} `true` if there is sender of the following type
+ */
+LUA_FUNCTION_DEF(task, has_from);
+
+/***
+ * @method task:get_from([type])
+ * Return SMTP or MIME sender for a task. This function returns an internet address which one is a table with the following structure:
+ *
+ * - `raw` - the original value without any processing
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * - `flags` - table with following keys set to true if given condition fulfilled:
+ * - [valid] - valid SMTP address in conformity with https://tools.ietf.org/html/rfc5321#section-4.1.
+ * - [ip] - domain is IPv4/IPv6 address
+ * - [braced] - angled `<blah@foo.com>` address
+ * - [quoted] - quoted user part
+ * - [empty] - empty address
+ * - [backslash] - user part contains backslash
+ * - [8bit] - contains 8bit characters
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP sender and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP sender and `2` or `mime` means MIME `From:` only
+ * @return {address} sender or `nil`
+ */
+LUA_FUNCTION_DEF(task, get_from);
+
+/***
+ * @method task:set_from(type, addr)
+ * Sets sender for a task. This function accepts table that will be converted to the address.
+ * If some fields are missing they are subsequently reconstructed by this function. E.g. if you
+ * specify 'user' and 'domain', then address and raw string will be reconstructed
+ *
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP sender and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP sender and `2` or `mime` means MIME `From:` only
+ * @param {table
+ * @return {boolean} success or not
+ */
+LUA_FUNCTION_DEF(task, set_from);
+/***
+ * @method task:get_user()
+ * Returns authenticated user name for this task if specified by an MTA.
+ * @return {string} username or nil
+ */
+LUA_FUNCTION_DEF(task, get_user);
+/***
+ * @method task:set_user([username])
+ * Sets or resets (if username is not specified) authenticated user name for this task.
+ * @return {string} the previously set username or nil
+ */
+LUA_FUNCTION_DEF(task, set_user);
+/***
+ * @method task:get_from_ip()
+ * Returns [ip_addr](ip.md) object of a sender that is provided by MTA
+ * @return {rspamd_ip} ip address object
+ */
+LUA_FUNCTION_DEF(task, get_from_ip);
+/***
+ * @method task:set_from_ip(str)
+ * Set tasks's IP address based on the passed string
+ * @param {string} str string representation of ip
+ */
+LUA_FUNCTION_DEF(task, set_from_ip);
+LUA_FUNCTION_DEF(task, get_from_ip_num);
+/***
+ * @method task:get_client_ip()
+ * Returns [ip_addr](ip.md) object of a client connected to rspamd (normally, it is an IP address of MTA)
+ * @return {rspamd_ip} ip address object
+ */
+LUA_FUNCTION_DEF(task, get_client_ip);
+/***
+ * @method task:get_helo()
+ * Returns the value of SMTP helo provided by MTA.
+ * @return {string} HELO value
+ */
+LUA_FUNCTION_DEF(task, get_helo);
+LUA_FUNCTION_DEF(task, set_helo);
+/***
+ * @method task:get_hostname()
+ * Returns the value of sender's hostname provided by MTA
+ * @return {string} hostname value
+ */
+LUA_FUNCTION_DEF(task, get_hostname);
+LUA_FUNCTION_DEF(task, set_hostname);
+/***
+ * @method task:get_images()
+ * Returns list of all images found in a task as a table of `rspamd_image`.
+ * Each image has the following methods:
+ *
+ * * `get_width` - return width of an image in pixels
+ * * `get_height` - return height of an image in pixels
+ * * `get_type` - return string representation of image's type (e.g. 'jpeg')
+ * * `get_filename` - return string with image's file name
+ * * `get_size` - return size in bytes
+ * @return {list of rspamd_image} images found in a message
+ */
+LUA_FUNCTION_DEF(task, get_images);
+/***
+ * @method task:get_archives()
+ * Returns list of all archives found in a task as a table of `rspamd_archive`.
+ * Each archive has the following methods available:
+ *
+ * * `get_files` - return list of strings with filenames inside archive
+ * * `get_files_full` - return list of tables with all information about files
+ * * `is_encrypted` - return true if an archive is encrypted
+ * * `get_type` - return string representation of image's type (e.g. 'zip')
+ * * `get_filename` - return string with archive's file name
+ * * `get_size` - return size in bytes
+ * @return {list of rspamd_archive} archives found in a message
+ */
+LUA_FUNCTION_DEF(task, get_archives);
+/***
+ * @method task:get_dkim_results()
+ * Returns list of all dkim check results as table of maps. Callee must ensure that
+ * dkim checks have been completed by adding dependency on `DKIM_TRACE` symbol.
+ * Fields in map:
+ *
+ * * `result` - string result of check:
+ * - `reject`
+ * - `allow`
+ * - `tempfail`
+ * - `permfail`
+ * - `not found`
+ * - `bad record`
+ * * `domain` - dkim domain
+ * * `selector` - dkim selector
+ * * `bhash` - short version of b tag (8 base64 symbols)
+ * * `fail_reason` - reason of failure (if applicable)
+ * @return {list of maps} dkim check results
+ */
+LUA_FUNCTION_DEF(task, get_dkim_results);
+/***
+ * @method task:get_symbol(name, [shadow_result_name])
+ * Searches for a symbol `name` in all metrics results and returns a list of tables
+ * one per metric that describes the symbol inserted.
+ * Please note, that for using this function you need to ensure that the symbol
+ * being queried is already checked. This is guaranteed if there is a dependency
+ * between the caller symbol and the checked symbol (either virtual or real).
+ * Please check `rspamd_config:register_dependency` method for details.
+ * The symbols are returned as the list of the following tables:
+ *
+ * - `metric` - name of metric
+ * - `score` - score of a symbol in that metric
+ * - `options` - a table of strings representing options of a symbol
+ * - `group` - a group of symbol (or 'ungrouped')
+ * @param {string} name symbol's name
+ * @return {list of tables} list of tables or nil if symbol was not found
+ */
+LUA_FUNCTION_DEF(task, get_symbol);
+/***
+ * @method task:get_symbols_all()
+ * Returns array of symbols matched in default metric with all metadata
+ * @return {table} table of tables formatted as in `task:get_symbol()` except that `metric` is absent and `name` is added
+ */
+LUA_FUNCTION_DEF(task, get_symbols_all);
+/***
+ * @method task:get_symbols([shadow_result_name])
+ * Returns array of all symbols matched for this task
+ * @return {table, table} table of strings with symbols names + table of theirs scores
+ */
+LUA_FUNCTION_DEF(task, get_symbols);
+
+/***
+ * @method task:get_groups([need_private])
+ * Returns a map [group -> group_score] for matched group. If `need_private` is
+ * unspecified, then the global option `public_groups_only` is used for default.
+ * @return {table, number} a map [group -> group_score]
+ */
+LUA_FUNCTION_DEF(task, get_groups);
+
+/***
+ * @method task:get_symbols_numeric()
+ * Returns array of all symbols matched for this task
+ * @return {table|number, table|number} table of numbers with symbols ids + table of theirs scores
+ */
+LUA_FUNCTION_DEF(task, get_symbols_numeric);
+
+/***
+ * @method task:get_symbols_tokens()
+ * Returns array of all symbols as statistical tokens
+ * @return {table|number} table of numbers
+ */
+LUA_FUNCTION_DEF(task, get_symbols_tokens);
+
+/***
+ * @method task:process_ann_tokens(symbols, ann_tokens, offset, [min])
+ * Processes ann tokens
+ * @param {table|string} symbols list of symbols in this profile
+ * @param {table|number} ann_tokens list of tokens (including metatokens)
+ * @param {integer} offset offset for symbols token (#metatokens)
+ * @param {number} min minimum value for symbols found (e.g. for 0 score symbols)
+ * @return nothing
+ */
+LUA_FUNCTION_DEF(task, process_ann_tokens);
+
+/***
+ * @method task:has_symbol(name, [shadow_result_name])
+ * Fast path to check if a specified symbol is in the task's results.
+ * Please note, that for using this function you need to ensure that the symbol
+ * being queried is already checked. This is guaranteed if there is a dependency
+ * between the caller symbol and the checked symbol (either virtual or real).
+ * Please check `rspamd_config:register_dependency` method for details.
+ * @param {string} name symbol's name
+ * @return {boolean} `true` if symbol has been found
+ */
+LUA_FUNCTION_DEF(task, has_symbol);
+/***
+ * @method task:enable_symbol(name)
+ * Enable specified symbol for this particular task
+ * @param {string} name symbol's name
+ * @return {boolean} `true` if symbol has been found
+ */
+LUA_FUNCTION_DEF(task, enable_symbol);
+/***
+ * @method task:disable_symbol(name)
+ * Disable specified symbol for this particular task
+ * @param {string} name symbol's name
+ * @return {boolean} `true` if symbol has been found
+ */
+LUA_FUNCTION_DEF(task, disable_symbol);
+/***
+ * @method task:get_date(type[, gmt])
+ * Returns timestamp for a connection or for a MIME message. This function can be called with a
+ * single table arguments with the following fields:
+ *
+ * * `format` - a format of date returned:
+ * - `message` - returns a mime date as integer (unix timestamp)
+ * - `connect` - returns a unix timestamp of a connection to rspamd
+ * * `gmt` - returns date in `GMT` timezone (normal for unix timestamps)
+ *
+ * By default this function returns connection time in numeric format.
+ * @param {string} type date format as described above
+ * @param {boolean} gmt gmt flag as described above
+ * @return {string/number} date representation according to format
+ * @example
+rspamd_config.DATE_IN_PAST = function(task)
+ local dm = task:get_date{format = 'message', gmt = true}
+ local dt = task:get_date{format = 'connect', gmt = true}
+ -- A day
+ if dt - dm > 86400 then
+ return true
+ end
+
+ return false
+end
+ */
+LUA_FUNCTION_DEF(task, get_date);
+/***
+ * @method task:get_message_id()
+ * Returns message identifier from the `Message-ID` header. Angle brackets (`<>`)
+ * are stripped off if present. If a Message-ID header is missing `undef` is
+ * returned.
+ * @return {string} ID of the message
+ */
+LUA_FUNCTION_DEF(task, get_message_id);
+/***
+ * @method task:get_timeval([raw])
+ * Returns the timestamp for a task start processing time.
+ * @param {boolean} raw if true then two float numbers are returned: task start timestamp and timeout event timestamp
+ * @return {table} table with fields as described in `struct timeval` in C
+ */
+LUA_FUNCTION_DEF(task, get_timeval);
+/***
+ * @method task:get_scan_time([set])
+ * Returns 2 floating point numbers: scan real time and scan virtual time.
+ * If `set` is `true`, then the finishing time is also set (enabled by default).
+ * This function should be normally called on idempotent phase.
+ * @return {number,number} real and virtual times in seconds with floating point
+ */
+LUA_FUNCTION_DEF(task, get_scan_time);
+/***
+ * @method task:get_metric_result()
+ * Get full result of a metric as a table:
+ * - `score`: current score
+ * - `action`: current action as a string
+ * - `nnegative`: number of negative rules matched
+ * - `npositive`: number of positive rules matched
+ * - `positive_score`: total score for positive rules
+ * - `negative_score`: total score for negative rules
+ * - `passthrough`: set to true if message has a passthrough result
+ * @return {table} metric result
+ */
+LUA_FUNCTION_DEF(task, get_metric_result);
+/***
+ * @method task:get_metric_score(name)
+ * Get the current score of metric `name` (must be nil or 'default') . Should be used in idempotent filters only.
+ * @param {string} name name of a metric
+ * @return {number,number} 2 numbers containing the current score and required score of the metric
+ */
+LUA_FUNCTION_DEF(task, get_metric_score);
+/***
+ * @method task:get_metric_action(name)
+ * Get the current action of metric `name` (must be nil or 'default'). Should be used in idempotent filters only.
+ * @param {string} name name of a metric
+ * @return {string} the current action of the metric as a string
+ */
+LUA_FUNCTION_DEF(task, get_metric_action);
+/***
+ * @method task:set_metric_score(name, score)
+ * Set the current score of metric `name`. Should be used in post-filters only.
+ * @param {string} name name of a metric
+ * @param {number} score the current score of the metric
+ */
+LUA_FUNCTION_DEF(task, set_metric_score);
+/***
+ * @method task:set_metric_subject(subject)
+ * Set the subject in the default metric
+ * @param {string} subject subject to set
+ */
+LUA_FUNCTION_DEF(task, set_metric_subject);
+
+/***
+ * @method task:learn(is_spam[, classifier)
+ * Learn classifier `classifier` with the task. If `is_spam` is true then message
+ * is learnt as spam. Otherwise HAM is learnt. By default, this function learns
+ * `bayes` classifier.
+ * @param {boolean} is_spam learn spam or ham
+ * @param {string} classifier classifier's name
+ * @return {boolean} `true` if classifier has been learnt successfully
+ */
+LUA_FUNCTION_DEF(task, learn);
+/***
+ * @method task:set_settings(obj)
+ * Set users settings object for a task. The format of this object is described
+ * [here](https://rspamd.com/doc/configuration/settings.html).
+ * @param {any} obj any lua object that corresponds to the settings format
+ */
+LUA_FUNCTION_DEF(task, set_settings);
+
+/***
+ * @method task:set_settings_id(id)
+ * Set users settings id for a task (must be registered previously)
+ * @available 2.0+
+ * @param {number} id numeric id
+ * @return {boolean} true if settings id has been replaced from the existing one
+ */
+LUA_FUNCTION_DEF(task, set_settings_id);
+
+/***
+ * @method task:get_settings()
+ * Gets users settings object for a task. The format of this object is described
+ * [here](https://rspamd.com/doc/configuration/settings.html).
+ * @return {lua object} lua object generated from UCL
+ */
+LUA_FUNCTION_DEF(task, get_settings);
+
+/***
+ * @method task:lookup_settings(key)
+ * Gets users settings object with the specified key for a task.
+ * @param {string} key key to lookup
+ * @return {lua object} lua object generated from UCL
+ */
+LUA_FUNCTION_DEF(task, lookup_settings);
+
+/***
+ * @method task:get_settings_id()
+ * Get numeric hash of settings id if specified for this task. 0 is returned otherwise.
+ * @return {number} settings-id hash
+ */
+LUA_FUNCTION_DEF(task, get_settings_id);
+
+/***
+ * @method task:set_milter_reply(obj)
+ * Set special reply for milter
+ * @param {any} obj any lua object that corresponds to the settings format
+ * @example
+task:set_milter_reply({
+ add_headers = {['X-Lua'] = 'test'},
+ -- 1 is the position of header to remove
+ remove_headers = {['DKIM-Signature'] = 1},
+})
+ */
+LUA_FUNCTION_DEF(task, set_milter_reply);
+
+/***
+ * @method task:process_re(params)
+ * Processes the specified regexp and returns number of captures (cached or new)
+ * Params is the table with the following fields (mandatory fields are marked with `*`):
+ * - `re`* : regular expression object
+ * - `type`*: type of regular expression:
+ * + `mime`: mime regexp
+ * + `header`: header regexp
+ * + `rawheader`: raw header expression
+ * + `rawmime`: raw mime regexp
+ * + `body`: raw body regexp
+ * + `url`: url regexp
+ * - `header`: for header and rawheader regexp means the name of header
+ * - `strong`: case sensitive match for headers
+ * @return {number} number of regexp occurrences in the task (limited by 255 so far)
+ */
+LUA_FUNCTION_DEF(task, process_regexp);
+
+/***
+ * @method task:cache_set(key, value)
+ * Store some value to the task cache
+ * @param {string} key key to use
+ * @param {any} value any value (including functions and tables)
+ */
+LUA_FUNCTION_DEF(task, cache_set);
+/***
+ * @method task:cache_get(key)
+ * Returns cached value or nil if nothing is cached
+ * @param {string} key key to use
+ * @return {any} cached value
+ */
+LUA_FUNCTION_DEF(task, cache_get);
+
+/***
+ * @method task:get_size()
+ * Returns size of the task in bytes (that includes headers + parts size)
+ * @return {number} size in bytes
+ */
+LUA_FUNCTION_DEF(task, get_size);
+
+/***
+ * @method task:set_flag(flag_name[, set])
+ * Set specific flag for task:
+ *
+ * - `no_log`: do not log task summary
+ * - `no_stat`: do not include task into scanned stats
+ * - `pass_all`: check all filters for task
+ * - `extended_urls`: output extended info about urls
+ * - `skip`: skip task processing
+ * - `learn_spam`: learn message as spam
+ * - `learn_ham`: learn message as ham
+ * - `broken_headers`: header data is broken for a message
+ * @param {string} flag to set
+ * @param {boolean} set set or clear flag (default is set)
+@example
+--[[
+For messages with undefined queue ID (scanned with rspamc or WebUI)
+do not include results into statistics and do not log task summary
+(it will not appear in the WebUI history as well).
+]]--
+
+-- Callback function to set flags
+local function no_log_stat_cb(task)
+ if not task:get_queue_id() then
+ task:set_flag('no_log')
+ task:set_flag('no_stat')
+ end
+end
+
+rspamd_config:register_symbol({
+ name = 'LOCAL_NO_LOG_STAT',
+ type = 'postfilter',
+ callback = no_log_stat_cb
+})
+ */
+LUA_FUNCTION_DEF(task, set_flag);
+
+
+/***
+ * @method task:has_flag(flag_name)
+ * Checks for a specific flag in task:
+ *
+ * - `no_log`: do not log task summary
+ * - `no_stat`: do not include task into scanned stats
+ * - `pass_all`: check all filters for task
+ * - `extended_urls`: output extended info about urls
+ * - `skip`: skip task processing
+ * - `learn_spam`: learn message as spam
+ * - `learn_ham`: learn message as ham
+ * - `broken_headers`: header data is broken for a message
+ * @param {string} flag to check
+ * @return {boolean} true if flags is set
+ */
+LUA_FUNCTION_DEF(task, has_flag);
+
+
+/***
+ * @method task:get_flags()
+ * Get list of flags for task:
+ *
+ * - `no_log`: do not log task summary
+ * - `no_stat`: do not include task into scanned stats
+ * - `pass_all`: check all filters for task
+ * - `extended_urls`: output extended info about urls
+ * - `skip`: skip task processing
+ * - `learn_spam`: learn message as spam
+ * - `learn_ham`: learn message as ham
+ * - `broken_headers`: header data is broken for a message
+ * - `milter`: task is initiated by milter connection
+ * @return {array of strings} table with all flags as strings
+ */
+LUA_FUNCTION_DEF(task, get_flags);
+
+/***
+ * @method task:get_digest()
+ * Returns message's unique digest (32 hex symbols)
+ * @return {string} hex digest
+ */
+LUA_FUNCTION_DEF(task, get_digest);
+
+/***
+ * @method task:store_in_file([mode|table])
+ * If task was loaded using file scan, then this method just returns its name,
+ * otherwise, a fresh temporary file is created and its name is returned. Default
+ * mode is 0600. To convert lua number to the octal mode you can use the following
+ * trick: `tonumber("0644", 8)`. The file is automatically removed when task is
+ * destroyed.
+ *
+ * If table argument is specified, the following extra fields are allowed:
+ *
+ * - `mode`: same as mode argument
+ * - `force_new`: always create a new file
+ * - `filename`: use specific filename instead of a temporary one
+ * - `tmpmask`: use specific tempmask, e.g. '/tmp/file-XXXXX', where XXXX will be replaced by some random letters
+ * - `keep`: do not remove file after task is processed
+ *
+ * @param {number} mode mode for new file
+ * @return {string} file name with task content
+ */
+LUA_FUNCTION_DEF(task, store_in_file);
+
+/***
+ * @method task:get_protocol_reply([flags])
+ * This method being called from a **postfilter** will return reply for a message
+ * as it is returned to a client. This method returns the Lua table corresponding
+ * to the UCL object. Flags is a table that specify which information should be
+ * there in a reply:
+ *
+ * - `basic`: basic info, such as message-id
+ * - `metrics`: metrics and symbols
+ * - `messages`: messages
+ * - `dkim`: dkim signature
+ * - `milter`: milter control block
+ * - `extra`: extra data, such as profiling
+ * - `urls`: list of all urls in a message
+ *
+ * @param {table} flags table of flags (default is all flags but `urls`)
+ * @return {table} ucl object corresponding to the reply
+ */
+LUA_FUNCTION_DEF(task, get_protocol_reply);
+
+/***
+ * @method task:headers_foreach(callback, [params])
+ * This method calls `callback` for each header that satisfies some condition.
+ * By default, all headers are iterated unless `callback` returns `true`. Nil or
+ * false means continue of iterations.
+ * Params could be as following:
+ *
+ * - `full`: header value is full table of all attributes @see task:get_header_full for details
+ * - `regexp`: return headers that satisfies the specified regexp
+ * @param {function} callback function from header name and header value
+ * @param {table} params optional parameters
+ */
+LUA_FUNCTION_DEF(task, headers_foreach);
+
+/***
+ * @method task:disable_action(action)
+ * Disables some action for this task (e.g. 'greylist')
+ *
+ * @param {string} action action to disable
+ * @return {boolean} true if an action was enabled and is disabled after the method call
+ */
+LUA_FUNCTION_DEF(task, disable_action);
+
+/***
+ * @method task:get_newlines_type()
+ * Returns the most frequent newlines type met in a task
+ *
+ * @return {string} "cr" for \r, "lf" for \n, "crlf" for \r\n
+ */
+LUA_FUNCTION_DEF(task, get_newlines_type);
+
+/***
+ * @method task:get_stat_tokens()
+ * Returns list of tables the statistical tokens:
+ * - `data`: 64 bit number encoded as a string
+ * - `t1`: the first token (if any)
+ * - `t2`: the second token (if any)
+ * - `win`: window index
+ * - `flag`: table of strings:
+ * - `text`: text token
+ * - `meta`: meta token
+ * - `lua`: lua meta token
+ * - `exception`: exception
+ * - `subject`: subject token
+ * - `unigram`: unigram token
+ *
+ * @return {table of tables}
+ */
+LUA_FUNCTION_DEF(task, get_stat_tokens);
+
+/***
+ * @method task:lookup_words(map, function({o, n, s, f}) ... end)
+ * Matches words in a task (including meta words) against some map (set, regexp and so on)
+ * and call the specified function with a table containing 4 values:
+ * - [1] - stemmed word
+ * - [2] - normalised word
+ * - [3] - raw word
+ * - [4] - flags (table of strings)
+ */
+LUA_FUNCTION_DEF(task, lookup_words);
+
+/**
+ * @method task:topointer()
+ *
+ * Returns raw C pointer (lightuserdata) associated with task. This might be
+ * broken with luajit and GC64, use with caution.
+ */
+LUA_FUNCTION_DEF(task, topointer);
+
+/**
+ * @method task:add_named_result(name, symbol_control_function)
+ *
+ * Adds a new named result for this task. symbol_control_function is a callback
+ * called with 3 parameters:
+ * `function(task, symbol, result_name)` and it should return boolean that
+ * specifies if this symbol should be added to this named result.
+ * @param {string} name for this result
+ * @param {function} symbol_control_function predicate for symbols
+ */
+LUA_FUNCTION_DEF(task, add_named_result);
+
+/**
+ * @method task:get_all_named_results()
+ *
+ * Returns all named results registered for the task as a table of strings
+ * @return {table|string} all named results starting from `default`
+ */
+LUA_FUNCTION_DEF(task, get_all_named_results);
+
+/***
+ * @method task:get_dns_req()
+ * Get number of dns requests being sent in the task
+ * @return {number} number of DNS requests
+ */
+LUA_FUNCTION_DEF(task, get_dns_req);
+
+static const struct luaL_reg tasklib_f[] = {
+ LUA_INTERFACE_DEF(task, create),
+ LUA_INTERFACE_DEF(task, load_from_file),
+ LUA_INTERFACE_DEF(task, load_from_string),
+ {NULL, NULL}};
+
+static const struct luaL_reg tasklib_m[] = {
+ LUA_INTERFACE_DEF(task, get_message),
+ LUA_INTERFACE_DEF(task, set_message),
+ LUA_INTERFACE_DEF(task, destroy),
+ LUA_INTERFACE_DEF(task, process_message),
+ LUA_INTERFACE_DEF(task, set_cfg),
+ LUA_INTERFACE_DEF(task, get_cfg),
+ LUA_INTERFACE_DEF(task, get_mempool),
+ LUA_INTERFACE_DEF(task, get_session),
+ LUA_INTERFACE_DEF(task, set_session),
+ LUA_INTERFACE_DEF(task, get_ev_base),
+ LUA_INTERFACE_DEF(task, get_worker),
+ LUA_INTERFACE_DEF(task, insert_result),
+ LUA_INTERFACE_DEF(task, insert_result_named),
+ LUA_INTERFACE_DEF(task, adjust_result),
+ LUA_INTERFACE_DEF(task, remove_result),
+ LUA_INTERFACE_DEF(task, set_pre_result),
+ LUA_INTERFACE_DEF(task, has_pre_result),
+ LUA_INTERFACE_DEF(task, append_message),
+ LUA_INTERFACE_DEF(task, has_urls),
+ LUA_INTERFACE_DEF(task, get_urls),
+ LUA_INTERFACE_DEF(task, get_urls_filtered),
+ LUA_INTERFACE_DEF(task, inject_url),
+ LUA_INTERFACE_DEF(task, get_content),
+ LUA_INTERFACE_DEF(task, get_filename),
+ LUA_INTERFACE_DEF(task, get_rawbody),
+ LUA_INTERFACE_DEF(task, get_emails),
+ LUA_INTERFACE_DEF(task, get_text_parts),
+ LUA_INTERFACE_DEF(task, get_parts),
+ LUA_INTERFACE_DEF(task, get_request_header),
+ LUA_INTERFACE_DEF(task, set_request_header),
+ LUA_INTERFACE_DEF(task, get_header),
+ LUA_INTERFACE_DEF(task, has_header),
+ LUA_INTERFACE_DEF(task, get_header_raw),
+ LUA_INTERFACE_DEF(task, get_header_full),
+ LUA_INTERFACE_DEF(task, get_header_count),
+ LUA_INTERFACE_DEF(task, get_raw_headers),
+ LUA_INTERFACE_DEF(task, get_headers),
+ LUA_INTERFACE_DEF(task, modify_header),
+ LUA_INTERFACE_DEF(task, get_received_headers),
+ LUA_INTERFACE_DEF(task, get_queue_id),
+ LUA_INTERFACE_DEF(task, get_uid),
+ LUA_INTERFACE_DEF(task, get_resolver),
+ LUA_INTERFACE_DEF(task, set_resolver),
+ LUA_INTERFACE_DEF(task, inc_dns_req),
+ LUA_INTERFACE_DEF(task, get_dns_req),
+ LUA_INTERFACE_DEF(task, has_recipients),
+ LUA_INTERFACE_DEF(task, get_recipients),
+ LUA_INTERFACE_DEF(task, set_recipients),
+ LUA_INTERFACE_DEF(task, get_principal_recipient),
+ LUA_INTERFACE_DEF(task, get_reply_sender),
+ LUA_INTERFACE_DEF(task, has_from),
+ LUA_INTERFACE_DEF(task, get_from),
+ LUA_INTERFACE_DEF(task, set_from),
+ LUA_INTERFACE_DEF(task, get_user),
+ LUA_INTERFACE_DEF(task, set_user),
+ {"get_addr", lua_task_get_from_ip},
+ {"get_ip", lua_task_get_from_ip},
+ {"get_from_addr", lua_task_get_from_ip},
+ LUA_INTERFACE_DEF(task, get_from_ip),
+ LUA_INTERFACE_DEF(task, set_from_ip),
+ LUA_INTERFACE_DEF(task, get_from_ip_num),
+ LUA_INTERFACE_DEF(task, get_client_ip),
+ LUA_INTERFACE_DEF(task, get_subject),
+ LUA_INTERFACE_DEF(task, get_helo),
+ LUA_INTERFACE_DEF(task, set_helo),
+ LUA_INTERFACE_DEF(task, get_hostname),
+ LUA_INTERFACE_DEF(task, set_hostname),
+ LUA_INTERFACE_DEF(task, get_images),
+ LUA_INTERFACE_DEF(task, get_archives),
+ LUA_INTERFACE_DEF(task, get_dkim_results),
+ LUA_INTERFACE_DEF(task, get_symbol),
+ LUA_INTERFACE_DEF(task, get_symbols),
+ LUA_INTERFACE_DEF(task, get_symbols_all),
+ LUA_INTERFACE_DEF(task, get_symbols_numeric),
+ LUA_INTERFACE_DEF(task, get_symbols_tokens),
+ LUA_INTERFACE_DEF(task, get_groups),
+ LUA_INTERFACE_DEF(task, process_ann_tokens),
+ LUA_INTERFACE_DEF(task, has_symbol),
+ LUA_INTERFACE_DEF(task, enable_symbol),
+ LUA_INTERFACE_DEF(task, disable_symbol),
+ LUA_INTERFACE_DEF(task, get_date),
+ LUA_INTERFACE_DEF(task, get_message_id),
+ LUA_INTERFACE_DEF(task, get_timeval),
+ LUA_INTERFACE_DEF(task, get_scan_time),
+ LUA_INTERFACE_DEF(task, get_metric_result),
+ LUA_INTERFACE_DEF(task, get_metric_score),
+ LUA_INTERFACE_DEF(task, get_metric_action),
+ LUA_INTERFACE_DEF(task, set_metric_score),
+ LUA_INTERFACE_DEF(task, set_metric_subject),
+ LUA_INTERFACE_DEF(task, learn),
+ LUA_INTERFACE_DEF(task, set_settings),
+ LUA_INTERFACE_DEF(task, get_settings),
+ LUA_INTERFACE_DEF(task, lookup_settings),
+ LUA_INTERFACE_DEF(task, get_settings_id),
+ LUA_INTERFACE_DEF(task, set_settings_id),
+ LUA_INTERFACE_DEF(task, cache_get),
+ LUA_INTERFACE_DEF(task, cache_set),
+ LUA_INTERFACE_DEF(task, process_regexp),
+ LUA_INTERFACE_DEF(task, get_size),
+ LUA_INTERFACE_DEF(task, set_flag),
+ LUA_INTERFACE_DEF(task, get_flags),
+ LUA_INTERFACE_DEF(task, has_flag),
+ {"set_rmilter_reply", lua_task_set_milter_reply},
+ LUA_INTERFACE_DEF(task, set_milter_reply),
+ LUA_INTERFACE_DEF(task, get_digest),
+ LUA_INTERFACE_DEF(task, store_in_file),
+ LUA_INTERFACE_DEF(task, get_protocol_reply),
+ LUA_INTERFACE_DEF(task, headers_foreach),
+ LUA_INTERFACE_DEF(task, disable_action),
+ LUA_INTERFACE_DEF(task, get_newlines_type),
+ LUA_INTERFACE_DEF(task, get_stat_tokens),
+ LUA_INTERFACE_DEF(task, get_meta_words),
+ LUA_INTERFACE_DEF(task, lookup_words),
+ LUA_INTERFACE_DEF(task, add_named_result),
+ LUA_INTERFACE_DEF(task, get_all_named_results),
+ LUA_INTERFACE_DEF(task, topointer),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/* Image methods */
+LUA_FUNCTION_DEF(image, get_width);
+LUA_FUNCTION_DEF(image, get_height);
+LUA_FUNCTION_DEF(image, get_type);
+LUA_FUNCTION_DEF(image, get_filename);
+LUA_FUNCTION_DEF(image, get_size);
+
+static const struct luaL_reg imagelib_m[] = {
+ LUA_INTERFACE_DEF(image, get_width),
+ LUA_INTERFACE_DEF(image, get_height),
+ LUA_INTERFACE_DEF(image, get_type),
+ LUA_INTERFACE_DEF(image, get_filename),
+ LUA_INTERFACE_DEF(image, get_size),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/* Archive methods */
+LUA_FUNCTION_DEF(archive, get_type);
+LUA_FUNCTION_DEF(archive, get_files);
+LUA_FUNCTION_DEF(archive, get_files_full);
+/* TODO: Export archive flags as integers to use bitops for that */
+LUA_FUNCTION_DEF(archive, is_encrypted);
+LUA_FUNCTION_DEF(archive, is_obfuscated);
+LUA_FUNCTION_DEF(archive, is_unreadable);
+LUA_FUNCTION_DEF(archive, get_filename);
+LUA_FUNCTION_DEF(archive, get_size);
+
+static const struct luaL_reg archivelib_m[] = {
+ LUA_INTERFACE_DEF(archive, get_type),
+ LUA_INTERFACE_DEF(archive, get_files),
+ LUA_INTERFACE_DEF(archive, get_files_full),
+ LUA_INTERFACE_DEF(archive, is_encrypted),
+ LUA_INTERFACE_DEF(archive, is_obfuscated),
+ LUA_INTERFACE_DEF(archive, is_unreadable),
+ LUA_INTERFACE_DEF(archive, get_filename),
+ LUA_INTERFACE_DEF(archive, get_size),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/* Utility functions */
+struct rspamd_task *
+lua_check_task(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{task}");
+ luaL_argcheck(L, ud != NULL, pos, "'task' expected");
+ return ud ? *((struct rspamd_task **) ud) : NULL;
+}
+
+struct rspamd_task *
+lua_check_task_maybe(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata_maybe(L, pos, "rspamd{task}");
+
+ return ud ? *((struct rspamd_task **) ud) : NULL;
+}
+
+static struct rspamd_image *
+lua_check_image(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{image}");
+ luaL_argcheck(L, ud != NULL, 1, "'image' expected");
+ return ud ? *((struct rspamd_image **) ud) : NULL;
+}
+
+static struct rspamd_archive *
+lua_check_archive(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{archive}");
+ luaL_argcheck(L, ud != NULL, 1, "'archive' expected");
+ return ud ? *((struct rspamd_archive **) ud) : NULL;
+}
+
+static void
+lua_task_set_cached(lua_State *L, struct rspamd_task *task, const gchar *key,
+ gint pos)
+{
+ LUA_TRACE_POINT;
+ khiter_t k;
+
+ lua_pushvalue(L, pos);
+
+ k = kh_get(rspamd_task_lua_cache, &task->lua_cache, (char *) key);
+
+ if (G_UNLIKELY(k != kh_end(&task->lua_cache))) {
+ /* Unref previous value */
+ luaL_unref(L, LUA_REGISTRYINDEX, kh_value(&task->lua_cache, k).ref);
+ }
+ else {
+ int r;
+
+ k = kh_put(rspamd_task_lua_cache, &task->lua_cache, rspamd_mempool_strdup(task->task_pool, key), &r);
+ }
+
+ kh_value(&task->lua_cache, k).ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ kh_value(&task->lua_cache, k).id = GPOINTER_TO_UINT(task->message);
+}
+
+
+static gboolean
+lua_task_get_cached(lua_State *L, struct rspamd_task *task, const gchar *key)
+{
+ LUA_TRACE_POINT;
+ khiter_t k;
+
+ k = kh_get(rspamd_task_lua_cache, &task->lua_cache, (char *) key);
+
+ if (k != kh_end(&task->lua_cache) && (kh_value(&task->lua_cache, k).id == GPOINTER_TO_UINT(task->message))) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(&task->lua_cache, k).ref);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+/* Task methods */
+static int
+lua_task_process_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean enforce = FALSE;
+
+ if (task != NULL) {
+ if (task->msg.len > 0) {
+ if (lua_isboolean(L, 2)) {
+ enforce = lua_toboolean(L, 2);
+ }
+
+ if (rspamd_message_parse(task)) {
+ if (enforce ||
+ (!(task->flags & RSPAMD_TASK_FLAG_SKIP_PROCESS) &&
+ !(task->processed_stages & RSPAMD_TASK_STAGE_PROCESS_MESSAGE))) {
+
+ lua_pushboolean(L, TRUE);
+ rspamd_message_process(task);
+ task->processed_stages |= RSPAMD_TASK_STAGE_PROCESS_MESSAGE;
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_task_get_cfg(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_config **pcfg;
+
+ if (task) {
+ pcfg = lua_newuserdata(L, sizeof(gpointer));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = task->cfg;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_task_set_cfg(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ void *ud = rspamd_lua_check_udata(L, 2, "rspamd{config}");
+
+ if (task) {
+ luaL_argcheck(L, ud != NULL, 1, "'config' expected");
+ task->cfg = ud ? *((struct rspamd_config **) ud) : NULL;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static int
+lua_task_destroy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ rspamd_task_free(task);
+ }
+
+ return 0;
+}
+
+static int
+lua_task_get_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->flags = 0;
+ t->start = task->msg.begin;
+ t->len = task->msg.len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_task_set_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean message_set = FALSE;
+
+ if (task) {
+ gsize final_len = 0;
+ gchar *buf = NULL;
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ /* Piecewise construct */
+ guint vec_len = rspamd_lua_table_size(L, 2);
+
+
+ for (guint i = 0; i < vec_len; i++) {
+ lua_rawgeti(L, 2, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize l;
+
+ (void) lua_tolstring(L, -1, &l);
+ final_len += l;
+ }
+ else {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ final_len += t->len;
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ if (final_len > 0) {
+ gchar *pos;
+
+ buf = rspamd_mempool_alloc(task->task_pool, final_len);
+ pos = buf;
+
+ for (guint i = 0; i < vec_len; i++) {
+ lua_rawgeti(L, 2, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize l;
+ const gchar *s;
+
+ s = lua_tolstring(L, -1, &l);
+ memcpy(pos, s, l);
+ pos += l;
+ }
+ else {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ memcpy(pos, t->start, t->len);
+ pos += t->len;
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ task->flags |= RSPAMD_TASK_FLAG_MESSAGE_REWRITE;
+ task->msg.begin = buf;
+ task->msg.len = final_len;
+ message_set = TRUE;
+ }
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ const gchar *s;
+
+ s = lua_tolstring(L, -1, &final_len);
+ buf = rspamd_mempool_alloc(task->task_pool, final_len);
+ memcpy(buf, s, final_len);
+ }
+ else {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ final_len = t->len;
+ buf = rspamd_mempool_alloc(task->task_pool, final_len);
+ memcpy(buf, t->start, final_len);
+ }
+ }
+
+ if (buf) {
+ task->msg.begin = buf;
+ task->msg.len = final_len;
+ task->flags |= RSPAMD_TASK_FLAG_MESSAGE_REWRITE;
+ message_set = TRUE;
+ }
+ }
+
+ if (message_set) {
+ if (rspamd_message_parse(task)) {
+ rspamd_message_process(task);
+ lua_pushboolean(L, TRUE);
+ lua_pushinteger(L, final_len);
+
+ return 2;
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static void
+lua_task_unmap_dtor(gpointer p)
+{
+ struct rspamd_task *task = (struct rspamd_task *) p;
+
+ if (task->msg.begin) {
+ munmap((gpointer) task->msg.begin, task->msg.len);
+ }
+}
+
+static void
+lua_task_free_dtor(gpointer p)
+{
+ g_free(p);
+}
+
+static gint
+lua_task_load_from_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = NULL, **ptask;
+ const gchar *fname = luaL_checkstring(L, 1), *err = NULL;
+ struct rspamd_config *cfg = NULL;
+ gboolean res = FALSE;
+ gpointer map;
+ gsize sz;
+
+ if (fname) {
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ gpointer p;
+ p = rspamd_lua_check_udata_maybe(L, 2, "rspamd{config}");
+
+ if (p) {
+ cfg = *(struct rspamd_config **) p;
+ }
+ }
+
+ if (strcmp(fname, "-") == 0) {
+ /* Read from stdin */
+ gint fd = STDIN_FILENO;
+ GString *data = g_string_sized_new(BUFSIZ);
+ gchar buf[BUFSIZ];
+ gssize r;
+
+ for (;;) {
+ r = read(fd, buf, sizeof(buf));
+
+ if (r == -1) {
+ err = strerror(errno);
+ break;
+ }
+ else if (r == 0) {
+ break;
+ }
+ else {
+ g_string_append_len(data, buf, r);
+ }
+ }
+
+ task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE);
+ task->msg.begin = data->str;
+ task->msg.len = data->len;
+ rspamd_mempool_add_destructor(task->task_pool,
+ lua_task_free_dtor, data->str);
+ res = TRUE;
+ g_string_free(data, FALSE); /* Buffer is still valid */
+ }
+ else {
+ map = rspamd_file_xmap(fname, PROT_READ, &sz, TRUE);
+
+ if (!map) {
+ err = strerror(errno);
+ }
+ else {
+ task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE);
+ task->msg.begin = map;
+ task->msg.len = sz;
+ rspamd_mempool_add_destructor(task->task_pool,
+ lua_task_unmap_dtor, task);
+ res = TRUE;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, res);
+
+ if (res) {
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ }
+ else {
+ if (err) {
+ lua_pushstring(L, err);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ return 2;
+}
+
+static gint
+lua_task_load_from_string(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = NULL, **ptask;
+ const gchar *str_message;
+ gsize message_len;
+ struct rspamd_config *cfg = NULL;
+
+ str_message = luaL_checklstring(L, 1, &message_len);
+
+ if (str_message) {
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ gpointer p;
+ p = rspamd_lua_check_udata_maybe(L, 2, "rspamd{config}");
+
+ if (p) {
+ cfg = *(struct rspamd_config **) p;
+ }
+ }
+
+ task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE);
+ task->msg.begin = g_malloc(message_len);
+ memcpy((gchar *) task->msg.begin, str_message, message_len);
+ task->msg.len = message_len;
+ rspamd_mempool_add_destructor(task->task_pool, lua_task_free_dtor,
+ (gpointer) task->msg.begin);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, true);
+
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ return 2;
+}
+
+static gint
+lua_task_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = NULL, **ptask;
+ struct rspamd_config *cfg = NULL;
+ struct ev_loop *ev_base = NULL;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ gpointer p;
+ p = rspamd_lua_check_udata_maybe(L, 1, "rspamd{config}");
+
+ if (p) {
+ cfg = *(struct rspamd_config **) p;
+ }
+ }
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ gpointer p;
+ p = rspamd_lua_check_udata_maybe(L, 2, "rspamd{ev_base}");
+
+ if (p) {
+ ev_base = *(struct ev_loop **) p;
+ }
+ }
+
+ task = rspamd_task_new(NULL, cfg, NULL, NULL, ev_base, FALSE);
+ task->flags |= RSPAMD_TASK_FLAG_EMPTY;
+
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ return 1;
+}
+
+static int
+lua_task_get_mempool(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_mempool_t **ppool;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ ppool = lua_newuserdata(L, sizeof(rspamd_mempool_t *));
+ rspamd_lua_setclass(L, "rspamd{mempool}", -1);
+ *ppool = task->task_pool;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_task_get_session(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_async_session **psession;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ psession = lua_newuserdata(L, sizeof(void *));
+ rspamd_lua_setclass(L, "rspamd{session}", -1);
+ *psession = task->s;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+ return 1;
+}
+
+static int
+lua_task_set_session(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_async_session *session = lua_check_session(L, 2);
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL && session != NULL) {
+ task->s = session;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+ return 1;
+}
+
+static int
+lua_task_get_ev_base(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct ev_loop **pbase;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ pbase = lua_newuserdata(L, sizeof(struct ev_loop *));
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+ *pbase = task->event_loop;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+ return 1;
+}
+
+static int
+lua_task_get_worker(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_worker **pworker;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ if (task->worker) {
+ pworker = lua_newuserdata(L, sizeof(struct rspamd_worker *));
+ rspamd_lua_setclass(L, "rspamd{worker}", -1);
+ *pworker = task->worker;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+ return 1;
+}
+
+
+static gint
+lua_task_insert_result_common(lua_State *L, struct rspamd_scan_result *result,
+ gint common_args_pos)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol_name;
+ double weight;
+ struct rspamd_symbol_result *s;
+ enum rspamd_symbol_insert_flags flags = RSPAMD_SYMBOL_INSERT_DEFAULT;
+ gint i, top, args_start;
+
+ if (task != NULL) {
+ if (lua_isboolean(L, common_args_pos)) {
+ args_start = common_args_pos + 1;
+
+ if (lua_toboolean(L, common_args_pos)) {
+ flags |= RSPAMD_SYMBOL_INSERT_ENFORCE;
+ }
+ }
+ else {
+ args_start = common_args_pos;
+ }
+
+ symbol_name = rspamd_mempool_strdup(task->task_pool,
+ luaL_checkstring(L, args_start));
+ weight = luaL_checknumber(L, args_start + 1);
+ top = lua_gettop(L);
+ s = rspamd_task_insert_result_full(task, symbol_name, weight,
+ NULL, flags, result);
+
+ /* Get additional options */
+ if (s) {
+ if (s->sym == NULL) {
+ /* Unknown symbol, print traceback */
+ lua_pushfstring(L, "unknown symbol %s", symbol_name);
+ rspamd_lua_traceback(L);
+
+ msg_info_task("symbol insertion issue: %s", lua_tostring(L, -1));
+
+ lua_pop(L, 1); /* Traceback string */
+ }
+ for (i = args_start + 2; i <= top; i++) {
+ gint ltype = lua_type(L, i);
+
+ if (ltype == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, i, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (ltype == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, i);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+ else if (ltype == LUA_TTABLE) {
+ gsize objlen = rspamd_lua_table_size(L, i);
+
+ for (guint j = 1; j <= objlen; j++) {
+ lua_rawgeti(L, i, j);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, -1, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, -1);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ else {
+ return luaL_error(L, "not rspamd_text option in a table "
+ "when adding symbol %s: %s type",
+ s->name);
+ }
+ }
+ else {
+ const gchar *tname = lua_typename(L, lua_type(L, -1));
+ lua_pop(L, 2);
+
+ return luaL_error(L, "not a string option in a table "
+ "when adding symbol %s: %s type",
+ s->name, tname);
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ else if (ltype == LUA_TNIL) {
+ /* We have received a NULL option, it is not good but not a fatal error */
+ msg_info_task("nil option when adding symbol %s at pos %d",
+ s->name, i);
+ continue;
+ }
+ else {
+ const gchar *tname = lua_typename(L, ltype);
+
+ return luaL_error(L, "not a string/table option "
+ "when adding symbol %s: %s type",
+ s->name, tname);
+ }
+ }
+ }
+ else if (task->settings == NULL && task->settings_elt == NULL) {
+ lua_pushfstring(L, "insertion failed for %s", symbol_name);
+ rspamd_lua_traceback(L);
+
+ msg_info_task("symbol insertion issue: %s", lua_tostring(L, -1));
+
+ lua_pop(L, 2); /* Traceback string + error string */
+ }
+ else {
+ /* Usually denied by settings */
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_insert_result(lua_State *L)
+{
+ return lua_task_insert_result_common(L, NULL, 2);
+}
+
+static gint
+lua_task_insert_result_named(lua_State *L)
+{
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *named_result = luaL_checkstring(L, 2);
+ struct rspamd_scan_result *res;
+
+ if (task && named_result) {
+ res = rspamd_find_metric_result(task, named_result);
+
+ if (res == NULL) {
+ return luaL_error(L, "invalid arguments: bad named result: %s",
+ named_result);
+ }
+
+ return lua_task_insert_result_common(L, res, 3);
+ }
+
+ return luaL_error(L, "invalid arguments");
+}
+
+static gint
+lua_task_adjust_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol_name;
+ struct rspamd_scan_result *metric_res;
+ struct rspamd_symbol_result *s = NULL;
+ double weight;
+ gint i, top;
+
+ if (task != NULL) {
+
+ symbol_name = luaL_checkstring(L, 2);
+ weight = luaL_checknumber(L, 3);
+ top = lua_gettop(L);
+ metric_res = task->result;
+
+ if (metric_res) {
+ s = rspamd_task_find_symbol_result(task, symbol_name, NULL);
+ }
+ else {
+ return luaL_error(L, "no metric result");
+ }
+
+ if (s) {
+ if (!isnan(weight)) {
+ metric_res->score -= s->score;
+ s->score = weight;
+ metric_res->score += s->score;
+ }
+ }
+ else {
+ return luaL_error(L, "symbol not found: %s", symbol_name);
+ }
+
+ /* Get additional options */
+ if (s) {
+ for (i = 4; i <= top; i++) {
+ if (lua_type(L, i) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, i, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, i) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, i);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+ else if (lua_type(L, i) == LUA_TTABLE) {
+ gsize objlen = rspamd_lua_table_size(L, i);
+
+ for (guint j = 1; j <= objlen; j++) {
+ lua_rawgeti(L, i, j);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, -1, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, -1);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_remove_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol_name = luaL_checkstring(L, 2);
+ struct rspamd_scan_result *metric_res;
+ const gchar *named_result = luaL_optstring(L, 3, NULL);
+
+ if (task != NULL) {
+ metric_res = rspamd_find_metric_result(task, named_result);
+
+ if (metric_res == NULL) {
+ return luaL_error(L, "invalid arguments: bad named result: %s",
+ named_result);
+ }
+
+ lua_pushboolean(L, (rspamd_task_remove_symbol_result(task, symbol_name,
+ metric_res)) != NULL);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_pre_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *message = NULL, *module = NULL, *fl_str = NULL, *act_str = NULL,
+ *res_name = NULL;
+ gdouble score = NAN;
+ struct rspamd_action *action;
+ guint priority = RSPAMD_PASSTHROUGH_NORMAL, flags = 0;
+
+ if (task != NULL) {
+
+ if (RSPAMD_TASK_IS_SKIPPED(task)) {
+ /* Do not set pre-result for a skipped task */
+ return 0;
+ }
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ GError *err = NULL;
+
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*action=S;message=S;module=S;score=D;priority=i;flags=S;result=S",
+ &act_str, &message, &module, &score, &priority, &fl_str, &res_name)) {
+ gint ret = luaL_error(L, "invalid arguments: %s", err->message);
+ g_error_free(err);
+
+ return ret;
+ }
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ act_str = lua_tostring(L, 2);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ message = lua_tostring(L, 3);
+ }
+
+ if (lua_type(L, 4) == LUA_TSTRING) {
+ module = lua_tostring(L, 4);
+ }
+
+ if (lua_type(L, 5) == LUA_TNUMBER) {
+ score = lua_tonumber(L, 5);
+ }
+
+ if (lua_type(L, 6) == LUA_TNUMBER) {
+ priority = lua_tointeger(L, 6);
+ }
+
+ if (lua_type(L, 7) == LUA_TSTRING) {
+ fl_str = lua_tostring(L, 7);
+ }
+ }
+
+ enum rspamd_action_type internal_type;
+
+ if (strcmp(act_str, "accept") == 0) {
+ /* Compatibility! */
+ act_str = "no action";
+ }
+ else if (rspamd_action_from_str(act_str, &internal_type)) {
+ /* Compatibility! */
+ act_str = rspamd_action_to_str(internal_type);
+ }
+
+ action = rspamd_config_get_action(task->cfg, act_str);
+
+ if (action == NULL) {
+ return luaL_error(L, "unknown action %s", act_str);
+ }
+
+ if (module == NULL) {
+ module = "Unknown lua";
+ }
+
+ if (message == NULL) {
+ message = "unknown reason";
+ flags |= RSPAMD_PASSTHROUGH_NO_SMTP_MESSAGE;
+ }
+
+ if (fl_str != NULL) {
+ /*
+ * TODO: convert to a set of string and split by `,` + add table support
+ * once this legacy code is migrated to C++
+ */
+ if (strstr(fl_str, "least") != NULL) {
+ flags |= RSPAMD_PASSTHROUGH_LEAST;
+ }
+ else if (strstr(fl_str, "no_smtp_message") != NULL) {
+ flags |= RSPAMD_PASSTHROUGH_NO_SMTP_MESSAGE;
+ }
+ else if (strstr(fl_str, "process_all") != NULL) {
+ flags |= RSPAMD_PASSTHROUGH_PROCESS_ALL;
+ }
+ }
+
+
+ rspamd_add_passthrough_result(task,
+ action,
+ priority,
+ score,
+ rspamd_mempool_strdup(task->task_pool, message),
+ rspamd_mempool_strdup(task->task_pool, module),
+ flags,
+ rspamd_find_metric_result(task, res_name));
+
+ /* Don't classify or filter message if pre-filter sets results */
+
+ if (res_name == NULL && !(flags & (RSPAMD_PASSTHROUGH_LEAST | RSPAMD_PASSTHROUGH_PROCESS_ALL))) {
+ task->processed_stages |= (RSPAMD_TASK_STAGE_CLASSIFIERS |
+ RSPAMD_TASK_STAGE_CLASSIFIERS_PRE |
+ RSPAMD_TASK_STAGE_CLASSIFIERS_POST);
+ rspamd_symcache_disable_all_symbols(task, task->cfg->cache,
+ SYMBOL_TYPE_IDEMPOTENT | SYMBOL_TYPE_IGNORE_PASSTHROUGH);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_has_pre_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gint nret = 1;
+
+ if (task) {
+ if (task->result->passthrough_result) {
+ struct rspamd_passthrough_result *pr = task->result->passthrough_result;
+
+ lua_pushboolean(L, true);
+ nret = 4;
+ /* bool, action, message, module */
+
+ if (pr->action) {
+ lua_pushstring(L, rspamd_action_to_str(pr->action->action_type));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (pr->message) {
+ lua_pushstring(L, pr->message);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ if (pr->module) {
+ lua_pushstring(L, pr->module);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return nret;
+}
+
+static gint
+lua_task_append_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *category;
+
+ if (task != NULL) {
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ category = luaL_checkstring(L, 3);
+ }
+ else {
+ category = "unknown";
+ }
+
+ ucl_object_insert_key(task->messages,
+ ucl_object_lua_import(L, 2),
+ category, 0,
+ true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+
+static gint
+lua_task_get_urls(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct lua_tree_cb_data cb;
+ struct rspamd_url *u;
+ static const gint default_protocols_mask = PROTOCOL_HTTP | PROTOCOL_HTTPS |
+ PROTOCOL_FILE | PROTOCOL_FTP;
+ gsize sz, max_urls = 0;
+
+ if (task) {
+ if (task->cfg) {
+ max_urls = task->cfg->max_lua_urls;
+ }
+
+ if (task->message == NULL) {
+ lua_newtable(L);
+
+ return 1;
+ }
+
+ /* Exclude RSPAMD_URL_FLAG_CONTENT to preserve backward compatibility */
+ if (!lua_url_cbdata_fill(L, 2, &cb, default_protocols_mask,
+ ~(RSPAMD_URL_FLAG_CONTENT | RSPAMD_URL_FLAG_IMAGE),
+ max_urls)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sz = kh_size(MESSAGE_FIELD(task, urls));
+ sz = lua_url_adjust_skip_prob(task->task_timestamp,
+ MESSAGE_FIELD(task, digest), &cb, sz);
+
+ lua_createtable(L, sz, 0);
+
+ if (cb.sort) {
+ struct rspamd_url **urls_sorted;
+ gint i = 0;
+
+ urls_sorted = g_new0(struct rspamd_url *, sz);
+
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ if (i < sz) {
+ urls_sorted[i] = u;
+ i++;
+ }
+ });
+
+ qsort(urls_sorted, i, sizeof(struct rspamd_url *), rspamd_url_cmp_qsort);
+
+ for (int j = 0; j < i; j++) {
+ lua_tree_url_callback(urls_sorted[j], urls_sorted[j], &cb);
+ }
+
+ g_free(urls_sorted);
+ }
+ else {
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ lua_tree_url_callback(u, u, &cb);
+ });
+ }
+
+ lua_url_cbdata_dtor(&cb);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, no task");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_urls_filtered(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct lua_tree_cb_data cb;
+ struct rspamd_url *u;
+ static const gint default_protocols_mask = PROTOCOL_HTTP | PROTOCOL_HTTPS |
+ PROTOCOL_FILE | PROTOCOL_FTP;
+ gsize sz, max_urls = 0;
+
+ if (task) {
+ if (task->cfg) {
+ max_urls = task->cfg->max_lua_urls;
+ }
+
+ if (task->message == NULL) {
+ lua_newtable(L);
+
+ return 1;
+ }
+
+ if (!lua_url_cbdata_fill_exclude_include(L, 2, &cb, default_protocols_mask, max_urls)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sz = kh_size(MESSAGE_FIELD(task, urls));
+ sz = lua_url_adjust_skip_prob(task->task_timestamp,
+ MESSAGE_FIELD(task, digest), &cb, sz);
+
+ lua_createtable(L, sz, 0);
+
+ if (cb.sort) {
+ struct rspamd_url **urls_sorted;
+ gint i = 0;
+
+ urls_sorted = g_new0(struct rspamd_url *, sz);
+
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ if (i < sz) {
+ urls_sorted[i] = u;
+ i++;
+ }
+ });
+
+ qsort(urls_sorted, i, sizeof(struct rspamd_url *), rspamd_url_cmp_qsort);
+
+ for (int j = 0; j < i; j++) {
+ lua_tree_url_callback(urls_sorted[j], urls_sorted[j], &cb);
+ }
+
+ g_free(urls_sorted);
+ }
+ else {
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ lua_tree_url_callback(u, u, &cb);
+ });
+ }
+
+ lua_url_cbdata_dtor(&cb);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, no task");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_has_urls(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ bool need_emails = false;
+ gboolean ret = FALSE;
+ gsize sz = 0;
+
+ if (task) {
+ if (task->message) {
+ if (lua_gettop(L) >= 2) {
+ need_emails = lua_toboolean(L, 2);
+ }
+
+ if (need_emails) {
+ /* Simplified check */
+ if (kh_size(MESSAGE_FIELD(task, urls)) > 0) {
+ sz += kh_size(MESSAGE_FIELD(task, urls));
+ ret = TRUE;
+ }
+ }
+ else {
+ /* Linear scan */
+ struct rspamd_url *u;
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ if (u->protocol != PROTOCOL_MAILTO) {
+ sz++;
+ ret = TRUE;
+ }
+ });
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, ret);
+ lua_pushinteger(L, sz);
+
+ return 2;
+}
+
+static gint
+lua_task_inject_url(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_url *url = lua_check_url(L, 2);
+ struct rspamd_mime_part *mpart = NULL;
+
+ if (lua_isuserdata(L, 3)) {
+ /* We also have a mime part there */
+ mpart = *((struct rspamd_mime_part **) rspamd_lua_check_udata_maybe(L,
+ 3, "rspamd{mimepart}"));
+ }
+
+ if (task && task->message && url && url->url) {
+ if (rspamd_url_set_add_or_increase(MESSAGE_FIELD(task, urls), url->url, false)) {
+ if (mpart && mpart->urls) {
+ /* Also add url to the mime part */
+ g_ptr_array_add(mpart->urls, url->url);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_text *t;
+
+ if (task) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->len = task->msg.len;
+ t->start = task->msg.begin;
+ t->flags = 0;
+
+ if (lua_is_text_binary(t)) {
+ t->flags |= RSPAMD_TEXT_FLAG_BINARY;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_filename(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->msg.fpath) {
+ lua_pushstring(L, task->msg.fpath);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_rawbody(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_text *t;
+
+ if (task) {
+ if (task->message != NULL) {
+
+
+ if (MESSAGE_FIELD(task, raw_headers_content).len > 0) {
+ g_assert(MESSAGE_FIELD(task, raw_headers_content).len <= task->msg.len);
+ t = lua_new_text_task(L, task, task->msg.begin + MESSAGE_FIELD(task, raw_headers_content).len,
+ task->msg.len - MESSAGE_FIELD(task, raw_headers_content).len, FALSE);
+ }
+ else {
+ t = lua_new_text_task(L, task, task->msg.begin,
+ task->msg.len, FALSE);
+ }
+
+ t->flags = 0;
+ }
+ else {
+ /* Push body it it is there */
+ if (task->msg.len > 0 && task->msg.begin != NULL) {
+ lua_new_text_task(L, task, task->msg.begin, task->msg.len, FALSE);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_emails(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct lua_tree_cb_data cb;
+ struct rspamd_url *u;
+ gsize max_urls = 0, sz;
+
+ if (task) {
+ if (task->message) {
+ if (task->cfg) {
+ max_urls = task->cfg->max_lua_urls;
+ }
+
+ if (!lua_url_cbdata_fill(L, 2, &cb, PROTOCOL_MAILTO,
+ ~(RSPAMD_URL_FLAG_CONTENT | RSPAMD_URL_FLAG_IMAGE),
+ max_urls)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sz = kh_size(MESSAGE_FIELD(task, urls));
+ sz = lua_url_adjust_skip_prob(task->task_timestamp,
+ MESSAGE_FIELD(task, digest), &cb, sz);
+
+ lua_createtable(L, sz, 0);
+
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ lua_tree_url_callback(u, u, &cb);
+ });
+
+ lua_url_cbdata_dtor(&cb);
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_text_parts(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ guint i;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_mime_text_part *part, **ppart;
+
+ if (task != NULL) {
+
+ if (task->message) {
+ if (!lua_task_get_cached(L, task, "text_parts")) {
+ lua_createtable(L, MESSAGE_FIELD(task, text_parts)->len, 0);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, part)
+ {
+ ppart = lua_newuserdata(L, sizeof(struct rspamd_mime_text_part *));
+ *ppart = part;
+ rspamd_lua_setclass(L, "rspamd{textpart}", -1);
+ /* Make it array */
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ lua_task_set_cached(L, task, "text_parts", -1);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_parts(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ guint i;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_mime_part *part, **ppart;
+
+ if (task != NULL) {
+ if (task->message) {
+ lua_createtable(L, MESSAGE_FIELD(task, parts)->len, 0);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ ppart = lua_newuserdata(L, sizeof(struct rspamd_mime_part *));
+ *ppart = part;
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+ /* Make it array */
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_request_header(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_ftok_t *hdr;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *s;
+ struct rspamd_lua_text *t;
+
+ s = luaL_checkstring(L, 2);
+
+ if (s && task) {
+ hdr = rspamd_task_get_request_header(task, s);
+
+ if (hdr) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = hdr->begin;
+ t->len = hdr->len;
+ t->flags = 0;
+
+ return 1;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_request_header(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *s, *v = NULL;
+ rspamd_fstring_t *buf;
+ struct rspamd_lua_text *t;
+ rspamd_ftok_t *hdr, *new_name;
+ gsize len, vlen = 0;
+
+ s = luaL_checklstring(L, 2, &len);
+
+ if (s && task) {
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ v = luaL_checklstring(L, 3, &vlen);
+ }
+ else if (lua_type(L, 3) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 3);
+
+ if (t != NULL) {
+ v = t->start;
+ vlen = t->len;
+ }
+ }
+
+ if (v != NULL) {
+ buf = rspamd_fstring_new_init(v, vlen);
+ hdr = rspamd_ftok_map(buf);
+ buf = rspamd_fstring_new_init(s, len);
+ new_name = rspamd_ftok_map(buf);
+
+ rspamd_task_add_request_header(task, new_name, hdr);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 0;
+}
+
+
+gint rspamd_lua_push_header(lua_State *L, struct rspamd_mime_header *rh,
+ enum rspamd_lua_task_header_type how)
+{
+ LUA_TRACE_POINT;
+ switch (how) {
+ case RSPAMD_TASK_HEADER_PUSH_FULL:
+ /* Create new associated table for a header */
+ lua_createtable(L, 0, 7);
+ rspamd_lua_table_set(L, "name", rh->name);
+
+ if (rh->value) {
+ rspamd_lua_table_set(L, "value", rh->value);
+ }
+
+ if (rh->raw_len > 0) {
+ lua_pushstring(L, "raw");
+ lua_pushlstring(L, rh->raw_value, rh->raw_len);
+ lua_settable(L, -3);
+ }
+
+ if (rh->decoded) {
+ rspamd_lua_table_set(L, "decoded", rh->decoded);
+ }
+
+ lua_pushstring(L, "tab_separated");
+ lua_pushboolean(L, rh->flags & RSPAMD_HEADER_TAB_SEPARATED);
+ lua_settable(L, -3);
+ lua_pushstring(L, "empty_separator");
+ lua_pushboolean(L, rh->flags & RSPAMD_HEADER_EMPTY_SEPARATOR);
+ lua_settable(L, -3);
+ rspamd_lua_table_set(L, "separator", rh->separator);
+ lua_pushstring(L, "order");
+ lua_pushinteger(L, rh->order);
+ lua_settable(L, -3);
+ break;
+ case RSPAMD_TASK_HEADER_PUSH_RAW:
+ if (rh->value) {
+ lua_pushstring(L, rh->value);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ break;
+ case RSPAMD_TASK_HEADER_PUSH_SIMPLE:
+ if (rh->decoded) {
+ lua_pushstring(L, rh->decoded);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ break;
+ case RSPAMD_TASK_HEADER_PUSH_COUNT:
+ default:
+ g_assert_not_reached();
+ break;
+ }
+
+ return 1;
+}
+
+gint rspamd_lua_push_header_array(lua_State *L,
+ const gchar *name,
+ struct rspamd_mime_header *rh,
+ enum rspamd_lua_task_header_type how,
+ gboolean strong)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_header *cur;
+ guint i;
+ gint nret = 1;
+
+ if (rh == NULL) {
+ if (how == RSPAMD_TASK_HEADER_PUSH_HAS) {
+ lua_pushboolean(L, false);
+ nret = 1;
+ }
+ else if (how == RSPAMD_TASK_HEADER_PUSH_COUNT) {
+ lua_pushnumber(L, 0);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return nret;
+ }
+
+ if (how == RSPAMD_TASK_HEADER_PUSH_FULL) {
+ lua_createtable(L, 0, 0);
+ i = 0;
+
+ DL_FOREACH(rh, cur)
+ {
+ if (!strong || strcmp(name, cur->name) == 0) {
+ rspamd_lua_push_header(L, cur, how);
+ lua_rawseti(L, -2, ++i);
+ }
+ }
+ }
+ else if (how == RSPAMD_TASK_HEADER_PUSH_COUNT) {
+ i = 0;
+
+ DL_FOREACH(rh, cur)
+ {
+ if (!strong || strcmp(name, cur->name) == 0) {
+ i++;
+ }
+ }
+
+ lua_pushinteger(L, i);
+ }
+ else if (how == RSPAMD_TASK_HEADER_PUSH_HAS) {
+ nret = 1;
+ bool found = false;
+
+ if (strong) {
+ /* We still have to check all headers in the chain */
+ DL_FOREACH(rh, cur)
+ {
+ if (strcmp(name, cur->name) == 0) {
+ found = true;
+ break;
+ }
+ }
+ }
+ else {
+ found = true;
+ }
+
+ lua_pushboolean(L, found);
+ }
+ else {
+ DL_FOREACH(rh, cur)
+ {
+ if (!strong || strcmp(name, cur->name) == 0) {
+ return rspamd_lua_push_header(L, cur, how);
+ }
+ }
+
+ /* Not found with this case */
+ lua_pushnil(L);
+ }
+
+ return nret;
+}
+
+static gint
+lua_task_get_header_common(lua_State *L, enum rspamd_lua_task_header_type how)
+{
+ LUA_TRACE_POINT;
+ gboolean strong = FALSE, need_modified = FALSE;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_mime_header *rh;
+ const gchar *name;
+
+ name = luaL_checkstring(L, 2);
+
+ if (name && task) {
+ if (lua_gettop(L) >= 3) {
+ strong = lua_toboolean(L, 3);
+ if (lua_isboolean(L, 4)) {
+ need_modified = lua_toboolean(L, 4);
+ }
+ }
+
+
+ rh = rspamd_message_get_header_array(task, name, need_modified);
+
+ return rspamd_lua_push_header_array(L, name, rh, how, strong);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+}
+
+static gint
+lua_task_get_header_full(lua_State *L)
+{
+ return lua_task_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_FULL);
+}
+
+static gint
+lua_task_get_header(lua_State *L)
+{
+ return lua_task_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_SIMPLE);
+}
+
+static gint
+lua_task_get_header_raw(lua_State *L)
+{
+ return lua_task_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_RAW);
+}
+
+static gint
+lua_task_get_header_count(lua_State *L)
+{
+ return lua_task_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_COUNT);
+}
+
+static gint
+lua_task_has_header(lua_State *L)
+{
+ return lua_task_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_HAS);
+}
+
+static gint
+lua_task_get_headers(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ bool need_modified = lua_isnoneornil(L, 2) ? false : lua_toboolean(L, 2);
+
+ if (task && task->message) {
+ struct rspamd_mime_header *cur;
+ int i = 1;
+
+ lua_createtable(L, rspamd_mime_headers_count(MESSAGE_FIELD(task, raw_headers)), 0);
+ LL_FOREACH2(MESSAGE_FIELD(task, headers_order), cur, ord_next)
+ {
+ if (need_modified && cur->modified_chain) {
+ struct rspamd_mime_header *cur_modified;
+
+ LL_FOREACH(cur->modified_chain, cur_modified)
+ {
+ rspamd_lua_push_header(L, cur_modified, RSPAMD_TASK_HEADER_PUSH_FULL);
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ else {
+ rspamd_lua_push_header(L, cur, RSPAMD_TASK_HEADER_PUSH_FULL);
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+static gint
+lua_task_get_raw_headers(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_text *t;
+
+ if (task && task->message) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = MESSAGE_FIELD(task, raw_headers_content).begin;
+ t->len = MESSAGE_FIELD(task, raw_headers_content).len;
+ t->flags = 0;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+static gint
+lua_task_get_received_headers(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (!task->message) {
+ /* No message - no received */
+ lua_newtable(L);
+ return 1;
+ }
+
+ if (!lua_task_get_cached(L, task, "received")) {
+
+ if (rspamd_received_export_to_lua(task, L)) {
+ lua_task_set_cached(L, task, "received", -1);
+ }
+ else {
+ /* no received, preserve compatibility */
+ lua_newtable(L);
+ return 1;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_queue_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->queue_id != NULL && strcmp(task->queue_id, "undef") != 0) {
+ lua_pushstring(L, task->queue_id);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_uid(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ lua_pushstring(L, task->task_pool->tag.uid);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_resolver(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_dns_resolver **presolver;
+
+ if (task != NULL && task->resolver != NULL) {
+ presolver = lua_newuserdata(L, sizeof(void *));
+ rspamd_lua_setclass(L, "rspamd{resolver}", -1);
+ *presolver = task->resolver;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_resolver(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_dns_resolver *resolver = lua_check_dns_resolver(L, 2);
+
+ if (task != NULL && resolver != NULL) {
+ task->resolver = resolver;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_inc_dns_req(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ static guint warning_shown = 0;
+
+ if (warning_shown < 100) {
+ warning_shown++;
+ msg_warn_task_check("task:inc_dns_req is deprecated and should not be used");
+ }
+
+ if (task != NULL) {
+ /* Deprecation: already done in rspamd_dns_resolver_request */
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_dns_req(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ lua_pushinteger(L, task->dns_requests);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+enum lua_email_address_type {
+ LUA_ADDRESS_ANY = 0u,
+ LUA_ADDRESS_SMTP = 1,
+ LUA_ADDRESS_MIME = 2,
+ LUA_ADDRESS_MASK = 0x3FF,
+ LUA_ADDRESS_RAW = (1u << 10),
+ LUA_ADDRESS_ORIGINAL = (1u << 11),
+ LUA_ADDRESS_MAX = LUA_ADDRESS_MASK,
+};
+
+/*
+ * Convert element at the specified position to the type
+ * for get_from/get_recipients
+ */
+static enum lua_email_address_type
+lua_task_str_to_get_type(lua_State *L, struct rspamd_task *task, gint pos, gint last_pos)
+{
+ const gchar *type = NULL;
+ gint ret = LUA_ADDRESS_ANY;
+ guint64 h;
+ gsize sz;
+
+ /* Get what value */
+ do {
+ if (lua_type(L, pos) == LUA_TNUMBER) {
+ ret = lua_tonumber(L, pos);
+
+ if (ret >= LUA_ADDRESS_ANY && ret < LUA_ADDRESS_MAX) {
+ return ret;
+ }
+
+ return LUA_ADDRESS_ANY;
+ }
+ else if (lua_type(L, pos) == LUA_TSTRING) {
+ type = lua_tolstring(L, pos, &sz);
+
+ if (type && sz > 0) {
+ h = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64,
+ type, sz, 0xdeadbabe);
+
+ switch (h) {
+ case 0xDA081341FB600389ULL: /* mime */
+ ret = LUA_ADDRESS_MIME;
+ break;
+ case 0xEEC8A7832F8C43ACULL: /* any */
+ ret = LUA_ADDRESS_ANY;
+ break;
+ case 0x472274D5193B2A80ULL: /* smtp */
+ case 0xEFE0F586CC9F14A9ULL: /* envelope */
+ ret = LUA_ADDRESS_SMTP;
+ break;
+ default:
+ msg_err_task("invalid email type: %*s", (gint) sz, type);
+ break;
+ }
+ }
+ }
+ else if (lua_type(L, pos) == LUA_TTABLE) {
+ for (lua_pushnil(L); lua_next(L, pos); lua_pop(L, 1)) {
+ type = lua_tolstring(L, -1, &sz);
+
+ if (type && sz > 0) {
+ h = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64,
+ type, sz, 0xdeadbabe);
+
+ switch (h) {
+ case 0xDA081341FB600389ULL: /* mime */
+ ret |= LUA_ADDRESS_MIME;
+ break;
+ case 0xEEC8A7832F8C43ACULL: /* any */
+ ret |= LUA_ADDRESS_ANY;
+ break;
+ case 0x472274D5193B2A80ULL: /* smtp */
+ case 0xEFE0F586CC9F14A9ULL: /* envelope */
+ ret |= LUA_ADDRESS_SMTP;
+ break;
+ case 0xAF4DE083D9AD0132: /* raw */
+ ret |= LUA_ADDRESS_RAW;
+ break;
+ case 0xC7AB6C7B7B0F5A8A: /* orig */
+ case 0x1778AE905589E431: /* original */
+ ret |= LUA_ADDRESS_ORIGINAL;
+ break;
+ default:
+ msg_err_task("invalid email type: %*s", (gint) sz, type);
+ break;
+ }
+ }
+ }
+ }
+ pos++;
+ } while (pos <= last_pos);
+
+ return ret;
+}
+
+#define EMAIL_CHECK_FLAG(fl, str) \
+ do { \
+ if (addr->flags & (fl)) { \
+ lua_pushstring(L, (str)); \
+ lua_pushboolean(L, true); \
+ lua_settable(L, -3); \
+ } \
+ } while (0)
+
+static void
+lua_push_email_address(lua_State *L, struct rspamd_email_address *addr)
+{
+ if (addr) {
+ lua_createtable(L, 0, 5);
+
+ if (addr->raw_len > 0) {
+ lua_pushstring(L, "raw");
+ lua_pushlstring(L, addr->raw, addr->raw_len);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "raw");
+ lua_pushstring(L, "");
+ lua_settable(L, -3);
+ }
+ if (addr->addr_len > 0) {
+ lua_pushstring(L, "addr");
+ lua_pushlstring(L, addr->addr, addr->addr_len);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "addr");
+ lua_pushstring(L, "");
+ lua_settable(L, -3);
+ }
+ if (addr->domain_len > 0) {
+ lua_pushstring(L, "domain");
+ lua_pushlstring(L, addr->domain, addr->domain_len);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "domain");
+ lua_pushstring(L, "");
+ lua_settable(L, -3);
+ }
+ if (addr->user_len > 0) {
+ lua_pushstring(L, "user");
+ lua_pushlstring(L, addr->user, addr->user_len);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "user");
+ lua_pushstring(L, "");
+ lua_settable(L, -3);
+ }
+
+ if (addr->name) {
+ lua_pushstring(L, "name");
+ lua_pushstring(L, addr->name);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "name");
+ lua_pushstring(L, "");
+ lua_settable(L, -3);
+ }
+
+ lua_pushstring(L, "flags");
+ lua_createtable(L, 0, 7);
+
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_VALID, "valid");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_IP, "ip");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_BRACED, "braced");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_QUOTED, "quoted");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_EMPTY, "empty");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_HAS_BACKSLASH, "backslash");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_HAS_8BIT, "8bit");
+
+ lua_settable(L, -3);
+ }
+}
+
+void lua_push_emails_address_list(lua_State *L, GPtrArray *addrs, int flags)
+{
+ struct rspamd_email_address *addr;
+ guint i, pos = 1;
+
+ lua_createtable(L, addrs->len, 0);
+
+ for (i = 0; i < addrs->len; i++) {
+ addr = g_ptr_array_index(addrs, i);
+
+ if (addr->flags & RSPAMD_EMAIL_ADDR_ORIGINAL) {
+ if (flags & LUA_ADDRESS_ORIGINAL) {
+ lua_push_email_address(L, addr);
+ lua_rawseti(L, -2, pos);
+ pos++;
+ }
+ }
+ else {
+ lua_push_email_address(L, addr);
+ lua_rawseti(L, -2, pos);
+ pos++;
+ }
+ }
+}
+
+static gboolean
+lua_import_email_address(lua_State *L, struct rspamd_task *task,
+ gint pos,
+ struct rspamd_email_address **paddr)
+{
+ struct rspamd_email_address *addr;
+ const gchar *p;
+ gchar *dst;
+ gsize len;
+
+ g_assert(paddr != NULL);
+
+ if (!lua_istable(L, pos)) {
+ return FALSE;
+ }
+
+ addr = g_malloc0(sizeof(*addr));
+
+ lua_pushstring(L, "name");
+ lua_gettable(L, pos);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ p = lua_tolstring(L, -1, &len);
+ dst = rspamd_mempool_alloc(task->task_pool, len + 1);
+ rspamd_strlcpy(dst, p, len + 1);
+ addr->name = dst;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "user");
+ lua_gettable(L, pos);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ p = lua_tolstring(L, -1, &len);
+ addr->user = (const gchar *) rspamd_mempool_alloc(task->task_pool, len);
+ memcpy((gchar *) addr->user, p, len);
+ addr->user_len = len;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "domain");
+ lua_gettable(L, pos);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ p = lua_tolstring(L, -1, &len);
+ addr->domain = (const gchar *) rspamd_mempool_alloc(task->task_pool, len);
+ memcpy((gchar *) addr->domain, p, len);
+ addr->domain_len = len;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "addr");
+ lua_gettable(L, pos);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ p = lua_tolstring(L, -1, &len);
+ addr->addr = (const gchar *) rspamd_mempool_alloc(task->task_pool, len);
+ memcpy((gchar *) addr->addr, p, len);
+ addr->addr_len = len;
+ }
+ else {
+ /* Construct addr */
+ len = addr->domain_len + addr->user_len + 1;
+ addr->addr = (const gchar *) rspamd_mempool_alloc(task->task_pool, len);
+ addr->addr_len = rspamd_snprintf((gchar *) addr->addr, len, "%*s@%*s",
+ (int) addr->user_len, addr->user,
+ (int) addr->domain_len, addr->domain);
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "raw");
+ lua_gettable(L, pos);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gchar *cpy;
+ p = lua_tolstring(L, -1, &len);
+ cpy = rspamd_mempool_alloc(task->task_pool, len + 1);
+ memcpy(cpy, p, len);
+ cpy[len] = '\0';
+ addr->raw_len = len;
+ addr->raw = cpy;
+ }
+ else {
+ /* Construct raw addr */
+ len = addr->addr_len + 3;
+
+ if (addr->name) {
+ len += strlen(addr->name) + 1;
+ dst = rspamd_mempool_alloc(task->task_pool, len + 1);
+
+ addr->raw_len = rspamd_snprintf(dst, len, "%s <%*s>",
+ addr->name,
+ (int) addr->addr_len, addr->addr);
+ }
+ else {
+ dst = rspamd_mempool_alloc(task->task_pool, len + 1);
+
+ addr->raw_len = rspamd_snprintf(dst, len, "<%*s@%*s>",
+ (int) addr->user_len, addr->user,
+ (int) addr->domain_len, addr->domain);
+ }
+
+ addr->raw = dst;
+ }
+
+ lua_pop(L, 1);
+ addr->flags = RSPAMD_EMAIL_ADDR_VALID;
+
+ *paddr = addr;
+
+ return TRUE;
+}
+
+static gint
+lua_task_get_recipients(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ GPtrArray *ptrs = NULL;
+ gint what = 0;
+
+ if (task) {
+ if (lua_gettop(L) == 2) {
+ /* Get what value */
+ what = lua_task_str_to_get_type(L, task, 2, lua_gettop(L));
+ }
+
+ switch (what & LUA_ADDRESS_MASK) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ ptrs = task->rcpt_envelope;
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ ptrs = MESSAGE_FIELD_CHECK(task, rcpt_mime);
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ if (task->rcpt_envelope) {
+ ptrs = task->rcpt_envelope;
+ }
+ else {
+ ptrs = MESSAGE_FIELD_CHECK(task, rcpt_mime);
+ }
+ break;
+ }
+ if (ptrs) {
+ lua_push_emails_address_list(L, ptrs, what & ~LUA_ADDRESS_MASK);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_recipients(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ GPtrArray *ptrs = NULL;
+ struct rspamd_email_address *addr = NULL;
+ gint what = 0, pos = 3;
+ const gchar *how = "add";
+ gboolean need_update_digest = FALSE;
+
+ if (task && lua_gettop(L) >= 3) {
+
+ /* Get what value */
+ what = lua_task_str_to_get_type(L, task, 2, -1);
+
+ if (lua_isstring(L, 4)) {
+ how = lua_tostring(L, 4);
+ }
+
+ switch (what) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ if (task->rcpt_envelope) {
+ ptrs = task->rcpt_envelope;
+ }
+ else {
+ ptrs = g_ptr_array_new();
+ task->rcpt_envelope = ptrs;
+ }
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ ptrs = MESSAGE_FIELD_CHECK(task, rcpt_mime);
+ need_update_digest = TRUE;
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ if (task->rcpt_envelope) {
+ if (task->rcpt_envelope) {
+ ptrs = task->rcpt_envelope;
+ }
+ else {
+ ptrs = g_ptr_array_new();
+ task->rcpt_envelope = ptrs;
+ }
+ }
+ else {
+ ptrs = MESSAGE_FIELD_CHECK(task, rcpt_mime);
+ need_update_digest = TRUE;
+ }
+ break;
+ }
+ if (ptrs) {
+ guint i, flags_existing = RSPAMD_EMAIL_ADDR_ORIGINAL, flags_add = 0;
+ struct rspamd_email_address *tmp;
+
+ if (strcmp(how, "alias") == 0) {
+ flags_add |= RSPAMD_EMAIL_ADDR_ALIASED;
+ }
+ else if (strcmp(how, "rewrite") == 0) {
+ /* Clear old addresses */
+ PTR_ARRAY_FOREACH(ptrs, i, tmp)
+ {
+ rspamd_email_address_free(addr);
+ }
+
+ g_ptr_array_set_size(ptrs, 0);
+ }
+
+ PTR_ARRAY_FOREACH(ptrs, i, tmp)
+ {
+ tmp->flags |= flags_existing;
+ }
+
+ lua_pushvalue(L, pos);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ if (lua_import_email_address(L, task, lua_gettop(L), &addr)) {
+
+ if (need_update_digest) {
+ rspamd_message_update_digest(task->message,
+ addr->addr, addr->addr_len);
+ }
+
+ addr->flags |= flags_add;
+ g_ptr_array_add(ptrs, addr);
+ }
+ }
+
+ lua_pop(L, 1);
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+#define CHECK_EMAIL_ADDR(addr) \
+ do { \
+ if (addr == NULL) { \
+ ret = 0; \
+ } \
+ else { \
+ ret = addr->flags & RSPAMD_EMAIL_ADDR_VALID; \
+ } \
+ } while (0)
+
+#define CHECK_EMAIL_ADDR_LIST(addr) \
+ do { \
+ if (addr == NULL) { \
+ ret = 0; \
+ } \
+ else { \
+ ret = addr->len > 0; \
+ nrcpt = addr->len; \
+ } \
+ } while (0)
+
+static gint
+lua_task_has_from(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gint what = 0, nrcpt = 0;
+ gboolean ret = FALSE;
+
+ if (task) {
+ if (lua_gettop(L) == 2) {
+ /* Get what value */
+ what = lua_task_str_to_get_type(L, task, 2, lua_gettop(L));
+ }
+
+ switch (what & LUA_ADDRESS_MASK) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ CHECK_EMAIL_ADDR(task->from_envelope);
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ CHECK_EMAIL_ADDR_LIST(MESSAGE_FIELD_CHECK(task, from_mime));
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ CHECK_EMAIL_ADDR(task->from_envelope);
+
+ if (!ret) {
+ CHECK_EMAIL_ADDR_LIST(MESSAGE_FIELD_CHECK(task, from_mime));
+ }
+ break;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, ret);
+ (void) nrcpt; /* Silence warning */
+
+ return 1;
+}
+
+static inline int
+rspamd_check_real_recipients_array_size(GPtrArray *ar)
+{
+ gint ret = 0, i;
+ struct rspamd_email_address *addr;
+
+ PTR_ARRAY_FOREACH(ar, i, addr)
+ {
+ if (!(addr->flags & RSPAMD_EMAIL_ADDR_ORIGINAL)) {
+ ret++;
+ }
+ }
+
+ return ret;
+}
+
+static gint
+lua_task_has_recipients(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gint what = 0, nrcpt = 0;
+ gboolean ret = FALSE;
+
+ if (task) {
+ if (lua_gettop(L) == 2) {
+ /* Get what value */
+ what = lua_task_str_to_get_type(L, task, 2, lua_gettop(L));
+ }
+
+ switch (what & LUA_ADDRESS_MASK) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ nrcpt = rspamd_check_real_recipients_array_size(task->rcpt_envelope);
+ ret = nrcpt > 0;
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ nrcpt = rspamd_check_real_recipients_array_size(MESSAGE_FIELD_CHECK(task, rcpt_mime));
+ ret = nrcpt > 0;
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ nrcpt = rspamd_check_real_recipients_array_size(task->rcpt_envelope);
+ ret = nrcpt > 0;
+
+ if (!ret) {
+ nrcpt = rspamd_check_real_recipients_array_size(MESSAGE_FIELD_CHECK(task, rcpt_mime));
+ ret = nrcpt > 0;
+ }
+ break;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, ret);
+ lua_pushinteger(L, nrcpt);
+
+ return 2;
+}
+
+static gint
+lua_task_get_from(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ GPtrArray *addrs = NULL;
+ struct rspamd_email_address *addr = NULL;
+ gint what = 0;
+
+ if (task) {
+ if (lua_gettop(L) == 2) {
+ /* Get what value */
+ what = lua_task_str_to_get_type(L, task, 2, lua_gettop(L));
+ }
+
+ switch (what & LUA_ADDRESS_MASK) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ addr = task->from_envelope;
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ addrs = MESSAGE_FIELD_CHECK(task, from_mime);
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ if (task->from_envelope) {
+ addr = task->from_envelope;
+ }
+ else {
+ addrs = MESSAGE_FIELD_CHECK(task, from_mime);
+ }
+ break;
+ }
+
+ if (addrs && addrs->len > 0) {
+ lua_push_emails_address_list(L, addrs, what & ~LUA_ADDRESS_MASK);
+ }
+ else if (addr) {
+ /* Create table to preserve compatibility */
+ if (addr->addr) {
+ lua_createtable(L, 1, 0);
+ if (what & LUA_ADDRESS_ORIGINAL) {
+ if (task->from_envelope_orig) {
+ lua_push_email_address(L, task->from_envelope_orig);
+ }
+ else {
+ lua_push_email_address(L, addr);
+ }
+ }
+ else {
+ lua_push_email_address(L, addr);
+ }
+
+ lua_rawseti(L, -2, 1);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_from(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *how = "rewrite";
+ GPtrArray *addrs = NULL;
+ struct rspamd_email_address **paddr = NULL, *addr;
+ gboolean need_update_digest = FALSE;
+ gint what = 0;
+
+ if (task && lua_gettop(L) >= 3) {
+ what = lua_task_str_to_get_type(L, task, 2, -1);
+
+ if (lua_isstring(L, 4)) {
+ how = lua_tostring(L, 4);
+ }
+
+ switch (what & LUA_ADDRESS_MASK) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ paddr = &task->from_envelope;
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ addrs = MESSAGE_FIELD_CHECK(task, from_mime);
+ need_update_digest = TRUE;
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ if (task->from_envelope) {
+ paddr = &task->from_envelope;
+ }
+ else {
+ addrs = MESSAGE_FIELD_CHECK(task, from_mime);
+ need_update_digest = TRUE;
+ }
+ break;
+ }
+
+ if (addrs) {
+ if (lua_import_email_address(L, task, 3, &addr)) {
+ guint i, flags_add = RSPAMD_EMAIL_ADDR_ORIGINAL;
+ struct rspamd_email_address *tmp;
+
+ if (strcmp(how, "alias") == 0) {
+ flags_add |= RSPAMD_EMAIL_ADDR_ALIASED;
+ }
+
+ PTR_ARRAY_FOREACH(addrs, i, tmp)
+ {
+ tmp->flags |= flags_add;
+ }
+
+ if (need_update_digest) {
+ rspamd_message_update_digest(task->message,
+ addr->addr, addr->addr_len);
+ }
+
+ g_ptr_array_add(addrs, addr);
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else if (paddr) {
+ /* SMTP from case */
+ if (lua_import_email_address(L, task, 3, &addr)) {
+ task->from_envelope_orig = *paddr;
+ task->from_envelope = addr;
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_principal_recipient(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *r;
+
+ if (task) {
+ r = rspamd_task_get_principal_recipient(task);
+ if (r != NULL) {
+ lua_pushstring(L, r);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_reply_sender(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_mime_header *rh;
+
+ if (task) {
+
+ rh = rspamd_message_get_header_array(task, "Reply-To", FALSE);
+
+ if (rh) {
+ GPtrArray *addrs;
+
+ addrs = rspamd_email_address_from_mime(task->task_pool, rh->decoded,
+ strlen(rh->decoded), NULL, -1);
+
+ if (addrs == NULL || addrs->len == 0) {
+ lua_pushnil(L);
+ }
+ else {
+ struct rspamd_email_address *addr;
+
+ addr = (struct rspamd_email_address *) g_ptr_array_index(addrs, 0);
+ lua_pushlstring(L, addr->addr, addr->addr_len);
+ }
+ }
+ else if (MESSAGE_FIELD_CHECK(task, from_mime) &&
+ MESSAGE_FIELD(task, from_mime)->len >= 1) {
+ struct rspamd_email_address *addr;
+
+ addr = (struct rspamd_email_address *) g_ptr_array_index(
+ MESSAGE_FIELD(task, from_mime), 0);
+
+ lua_pushlstring(L, addr->addr, addr->addr_len);
+ }
+ else if (task->from_envelope) {
+ lua_pushlstring(L, task->from_envelope->addr,
+ task->from_envelope->addr_len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_user(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->auth_user != NULL) {
+ lua_pushstring(L, task->auth_user);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_user(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *new_user;
+
+ if (task) {
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ new_user = lua_tostring(L, 2);
+
+ if (task->auth_user) {
+ /* Push old user */
+ lua_pushstring(L, task->auth_user);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ task->auth_user = rspamd_mempool_strdup(task->task_pool, new_user);
+ }
+ else {
+ /* Reset user */
+ if (task->auth_user) {
+ /* Push old user */
+ lua_pushstring(L, task->auth_user);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ task->auth_user = NULL;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_from_ip(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->from_addr) {
+ rspamd_lua_ip_push(L, task->from_addr);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_from_ip(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ rspamd_inet_addr_t *addr = NULL;
+
+ if (!task) {
+ return luaL_error(L, "no task");
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ gsize len;
+ const gchar *ip_str = lua_tolstring(L, 2, &len);
+
+ if (!rspamd_parse_inet_address(&addr,
+ ip_str,
+ len,
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ return luaL_error(L, "invalid IP string: %s", ip_str);
+ }
+ else {
+ if (task->from_addr) {
+ rspamd_inet_address_free(task->from_addr);
+ }
+
+ task->from_addr = addr;
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 2);
+
+ if (ip && ip->addr) {
+ if (task->from_addr) {
+ rspamd_inet_address_free(task->from_addr);
+ }
+
+ task->from_addr = rspamd_inet_address_copy(ip->addr, NULL);
+ }
+ else {
+ return luaL_error(L, "invalid IP object");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid IP argument type: %s", lua_typename(L, lua_type(L, 2)));
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_from_ip_num(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ msg_err("this function is deprecated and should no longer be used");
+ lua_pushnil(L);
+ return 1;
+}
+
+static gint
+lua_task_get_client_ip(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->client_addr) {
+ rspamd_lua_ip_push(L, task->client_addr);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_helo(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->helo != NULL) {
+ lua_pushstring(L, task->helo);
+ return 1;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_subject(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (MESSAGE_FIELD_CHECK(task, subject) != NULL) {
+ lua_pushstring(L, MESSAGE_FIELD(task, subject));
+ return 1;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_helo(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *new_helo;
+
+ if (task) {
+ new_helo = luaL_checkstring(L, 2);
+ if (new_helo) {
+ task->helo = rspamd_mempool_strdup(task->task_pool, new_helo);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_hostname(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->hostname != NULL) {
+ /* Check whether it looks like an IP address */
+ if (*task->hostname == '[') {
+ /*
+ * From the milter documentation:
+ * If the reverse lookup fails or if none of the IP
+ * addresses of the resolved host name matches the
+ * original IP address, hostname will contain the
+ * message sender's IP address enclosed in square
+ * brackets (e.g. `[a.b.c.d]')
+ */
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushstring(L, task->hostname);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_hostname(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *new_hostname;
+
+ if (task) {
+ new_hostname = luaL_checkstring(L, 2);
+ if (new_hostname) {
+ task->hostname = rspamd_mempool_strdup(task->task_pool,
+ new_hostname);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_images(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint nelt = 0, i;
+ struct rspamd_mime_part *part;
+ struct rspamd_image **pimg;
+
+ if (task) {
+ if (task->message) {
+ if (!lua_task_get_cached(L, task, "images")) {
+ lua_createtable(L, MESSAGE_FIELD(task, parts)->len, 0);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ if (part->part_type == RSPAMD_MIME_PART_IMAGE) {
+ pimg = lua_newuserdata(L, sizeof(struct rspamd_image *));
+ rspamd_lua_setclass(L, "rspamd{image}", -1);
+ *pimg = part->specific.img;
+ lua_rawseti(L, -2, ++nelt);
+ }
+ }
+
+ lua_task_set_cached(L, task, "images", -1);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_archives(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint nelt = 0, i;
+ struct rspamd_mime_part *part;
+ struct rspamd_archive **parch;
+
+ if (task) {
+ if (task->message) {
+ if (!lua_task_get_cached(L, task, "archives")) {
+ lua_createtable(L, MESSAGE_FIELD(task, parts)->len, 0);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ if (part->part_type == RSPAMD_MIME_PART_ARCHIVE) {
+ parch = lua_newuserdata(L, sizeof(struct rspamd_archive *));
+ rspamd_lua_setclass(L, "rspamd{archive}", -1);
+ *parch = part->specific.arch;
+ lua_rawseti(L, -2, ++nelt);
+ }
+ }
+
+ lua_task_set_cached(L, task, "archives", -1);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_dkim_results(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint nelt = 0, i;
+ struct rspamd_dkim_check_result **pres, **cur;
+
+ if (task) {
+ if (!lua_task_get_cached(L, task, "dkim_results")) {
+ pres = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_DKIM_CHECK_RESULTS);
+
+ if (pres == NULL) {
+ lua_newtable(L);
+ }
+ else {
+ for (cur = pres; *cur != NULL; cur++) {
+ nelt++;
+ }
+
+ lua_createtable(L, nelt, 0);
+
+ for (i = 0; i < nelt; i++) {
+ struct rspamd_dkim_check_result *res = pres[i];
+ const gchar *result_str = "unknown";
+
+ lua_createtable(L, 0, 4);
+
+ switch (res->rcode) {
+ case DKIM_CONTINUE:
+ result_str = "allow";
+ break;
+ case DKIM_REJECT:
+ result_str = "reject";
+ break;
+ case DKIM_TRYAGAIN:
+ result_str = "tempfail";
+ break;
+ case DKIM_NOTFOUND:
+ result_str = "not found";
+ break;
+ case DKIM_RECORD_ERROR:
+ result_str = "bad record";
+ break;
+ case DKIM_PERM_ERROR:
+ result_str = "permanent error";
+ break;
+ default:
+ break;
+ }
+
+ rspamd_lua_table_set(L, "result", result_str);
+
+ if (res->domain) {
+ rspamd_lua_table_set(L, "domain", res->domain);
+ }
+
+ if (res->selector) {
+ rspamd_lua_table_set(L, "selector", res->selector);
+ }
+
+ if (res->short_b) {
+ rspamd_lua_table_set(L, "bhash", res->short_b);
+ }
+
+ if (res->fail_reason) {
+ rspamd_lua_table_set(L, "fail_reason", res->fail_reason);
+ }
+
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+
+ lua_task_set_cached(L, task, "dkim_results", -1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static inline gboolean
+lua_push_symbol_result(lua_State *L,
+ struct rspamd_task *task,
+ const gchar *symbol,
+ struct rspamd_symbol_result *symbol_result,
+ struct rspamd_scan_result *metric_res,
+ gboolean add_metric,
+ gboolean add_name)
+{
+
+ struct rspamd_symbol_result *s = NULL;
+ struct rspamd_symbol_option *opt;
+ struct rspamd_symbols_group *sym_group;
+ guint i;
+ gint j = 1, table_fields_cnt = 4;
+
+ if (!metric_res) {
+ metric_res = task->result;
+ }
+
+ if (!symbol_result) {
+ s = rspamd_task_find_symbol_result(task, symbol, metric_res);
+ }
+ else {
+ s = symbol_result;
+ }
+
+ if (s && !(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ if (add_metric) {
+ table_fields_cnt++;
+ }
+ if (add_name) {
+ table_fields_cnt++;
+ }
+
+ lua_createtable(L, 0, table_fields_cnt);
+
+ if (add_name) {
+ lua_pushstring(L, "name");
+ lua_pushstring(L, symbol);
+ lua_settable(L, -3);
+ }
+ lua_pushstring(L, "score");
+ lua_pushnumber(L, s->score);
+ lua_settable(L, -3);
+
+ if (s->sym && s->sym->gr) {
+ lua_pushstring(L, "group");
+ lua_pushstring(L, s->sym->gr->name);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "groups");
+ lua_createtable(L, s->sym->groups->len, 0);
+
+ PTR_ARRAY_FOREACH(s->sym->groups, i, sym_group)
+ {
+ lua_pushstring(L, sym_group->name);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "group");
+ lua_pushstring(L, "ungrouped");
+ lua_settable(L, -3);
+ }
+
+ if (s->options) {
+ lua_pushstring(L, "options");
+ lua_createtable(L, kh_size(s->options), 0);
+
+ DL_FOREACH(s->opts_head, opt)
+ {
+ lua_pushlstring(L, opt->option, opt->optlen);
+ lua_rawseti(L, -2, j++);
+ }
+
+ lua_settable(L, -3);
+ }
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static gint
+lua_task_get_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol;
+ gboolean found = FALSE;
+
+ symbol = luaL_checkstring(L, 2);
+
+ if (task && symbol) {
+ struct rspamd_scan_result *sres = NULL;
+
+ if (lua_isstring(L, 3)) {
+ sres = rspamd_find_metric_result(task, lua_tostring(L, 3));
+
+ if (sres == NULL) {
+ return luaL_error(L, "invalid scan result: %s",
+ lua_tostring(L, 3));
+ }
+ }
+
+ /* Always push as a table for compatibility :( */
+ lua_createtable(L, 1, 0);
+
+ if ((found = lua_push_symbol_result(L, task, symbol,
+ NULL, sres, TRUE, FALSE))) {
+ lua_rawseti(L, -2, 1);
+ }
+ else {
+ /* Pop table */
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (!found) {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_has_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_symbol_result *s;
+ const gchar *symbol;
+ gboolean found = FALSE;
+
+ symbol = luaL_checkstring(L, 2);
+
+ if (task && symbol) {
+ if (lua_isstring(L, 3)) {
+ s = rspamd_task_find_symbol_result(task, symbol,
+ rspamd_find_metric_result(task, lua_tostring(L, 3)));
+
+ if (s && !(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ found = TRUE;
+ }
+ }
+ else {
+ s = rspamd_task_find_symbol_result(task, symbol, NULL);
+
+ if (s && !(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ found = TRUE;
+ }
+ }
+ lua_pushboolean(L, found);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_enable_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol;
+ gboolean found = FALSE;
+
+ symbol = luaL_checkstring(L, 2);
+
+ if (task && symbol) {
+ found = rspamd_symcache_enable_symbol(task, task->cfg->cache, symbol);
+ lua_pushboolean(L, found);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_disable_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol;
+ gboolean found = FALSE;
+
+ symbol = luaL_checkstring(L, 2);
+
+ if (task && symbol) {
+ found = rspamd_symcache_disable_symbol(task, task->cfg->cache, symbol);
+ lua_pushboolean(L, found);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_symbols(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_scan_result *mres;
+ gint i = 1;
+ struct rspamd_symbol_result *s;
+
+ if (task) {
+ mres = task->result;
+
+ if (lua_isstring(L, 2)) {
+ mres = rspamd_find_metric_result(task, lua_tostring(L, 2));
+ }
+
+ if (mres) {
+ lua_createtable(L, kh_size(mres->symbols), 0);
+ lua_createtable(L, kh_size(mres->symbols), 0);
+
+ kh_foreach_value(mres->symbols, s, {
+ if (!(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ lua_pushstring(L, s->name);
+ lua_rawseti(L, -3, i);
+ lua_pushnumber(L, s->score);
+ lua_rawseti(L, -2, i);
+ i++;
+ }
+ });
+ }
+ else {
+ lua_createtable(L, 0, 0);
+ lua_createtable(L, 0, 0);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+static gint
+lua_task_get_symbols_all(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_scan_result *mres;
+ struct rspamd_symbol_result *s;
+ gboolean found = FALSE;
+ gint i = 1;
+
+ if (task) {
+ mres = task->result;
+
+ if (lua_isstring(L, 2)) {
+ mres = rspamd_find_metric_result(task, lua_tostring(L, 2));
+ }
+
+ if (mres) {
+ found = TRUE;
+ lua_createtable(L, kh_size(mres->symbols), 0);
+
+ kh_foreach_value(mres->symbols, s, {
+ if (!(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ lua_push_symbol_result(L, task, s->name, s, mres, FALSE, TRUE);
+ lua_rawseti(L, -2, i++);
+ }
+ });
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (!found) {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_task_get_symbols_numeric(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_scan_result *mres;
+ gint i = 1, id;
+ struct rspamd_symbol_result *s;
+
+ if (task) {
+ mres = task->result;
+
+ if (lua_isstring(L, 2)) {
+ mres = rspamd_find_metric_result(task, lua_tostring(L, 2));
+ }
+
+ if (mres) {
+ lua_createtable(L, kh_size(mres->symbols), 0);
+ lua_createtable(L, kh_size(mres->symbols), 0);
+
+ lua_createtable(L, kh_size(mres->symbols), 0);
+
+ kh_foreach_value(mres->symbols, s, {
+ if (!(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ id = rspamd_symcache_find_symbol(task->cfg->cache,
+ s->name);
+ lua_pushinteger(L, id);
+ lua_rawseti(L, -3, i);
+ lua_pushnumber(L, s->score);
+ lua_rawseti(L, -2, i);
+ i++;
+ }
+ });
+ }
+ else {
+ lua_createtable(L, 0, 0);
+ lua_createtable(L, 0, 0);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+static gint
+lua_task_get_groups(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean need_private;
+ struct rspamd_scan_result *mres;
+ struct rspamd_symbols_group *gr;
+ gdouble gr_score;
+
+ if (task) {
+ mres = task->result;
+
+ if (lua_isboolean(L, 2)) {
+ need_private = lua_toboolean(L, 2);
+ }
+ else {
+ need_private = !(task->cfg->public_groups_only);
+ }
+
+ if (lua_isstring(L, 3)) {
+ mres = rspamd_find_metric_result(task, lua_tostring(L, 3));
+ }
+
+ if (mres == NULL) {
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ lua_createtable(L, 0, kh_size(mres->sym_groups));
+
+ kh_foreach(mres->sym_groups, gr, gr_score, {
+ if (!(gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC)) {
+ if (!need_private) {
+ continue;
+ }
+ }
+
+ lua_pushnumber(L, gr_score);
+ lua_setfield(L, -2, gr->name);
+ });
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct tokens_foreach_cbdata {
+ struct rspamd_task *task;
+ lua_State *L;
+ gint idx;
+ gboolean normalize;
+};
+
+static void
+tokens_foreach_cb(struct rspamd_symcache_item *item, gpointer ud)
+{
+ struct tokens_foreach_cbdata *cbd = ud;
+ struct rspamd_symbol_result *s;
+ gint flags;
+ const gchar *sym;
+
+ sym = rspamd_symcache_item_name(item);
+ flags = rspamd_symcache_item_flags(item);
+
+ if (flags & SYMBOL_TYPE_NOSTAT) {
+ return;
+ }
+
+ if ((s = rspamd_task_find_symbol_result(cbd->task, sym, NULL)) != NULL) {
+ if (s->flags & RSPAMD_SYMBOL_RESULT_IGNORED) {
+ lua_pushnumber(cbd->L, 0.0);
+ }
+ else {
+ if (cbd->normalize) {
+ lua_pushnumber(cbd->L, tanh(s->score));
+ }
+ else {
+ lua_pushnumber(cbd->L, s->score);
+ }
+ }
+ }
+ else {
+ lua_pushnumber(cbd->L, 0.0);
+ }
+
+ lua_rawseti(cbd->L, -2, cbd->idx++);
+}
+
+static gint
+lua_task_get_symbols_tokens(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct tokens_foreach_cbdata cbd;
+
+ if (task) {
+ cbd.task = task;
+ cbd.L = L;
+ cbd.idx = 1;
+ cbd.normalize = TRUE;
+
+ if (lua_type(L, 2) == LUA_TBOOLEAN) {
+ cbd.normalize = lua_toboolean(L, 2);
+ }
+ else {
+ cbd.normalize = TRUE;
+ }
+
+ lua_createtable(L,
+ rspamd_symcache_stats_symbols_count(task->cfg->cache), 0);
+ rspamd_symcache_foreach(task->cfg->cache, tokens_foreach_cb, &cbd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ /* Return type is table created */
+ return 1;
+}
+
+static gint
+lua_task_process_ann_tokens(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gint offset = luaL_checkinteger(L, 4);
+ gdouble min_score = 0.0;
+
+ if (task && lua_istable(L, 2) && lua_istable(L, 3)) {
+ guint symlen = rspamd_lua_table_size(L, 2);
+ if (lua_isnumber(L, 5)) {
+ min_score = lua_tonumber(L, 5);
+ }
+
+ for (guint i = 1; i <= symlen; i++, offset++) {
+ const gchar *sym;
+ struct rspamd_symbol_result *sres;
+
+ lua_rawgeti(L, 2, i);
+ sym = lua_tostring(L, -1);
+
+ /*
+ * TODO: this cycle involves one hash lookup per symbol in a profile
+ * Basically, in a common case that would be a table of all symbols
+ * So we need to do N_symbols hash lookups which is not optimal
+ * The optimal solution is to convert [sym1, sym2, ... symn] profile
+ * to a set {sym1 = true, sym2 = true, ...} and then for each
+ * resulting symbol check this table.
+ *
+ * That would lead to N_results lookups which is usually MUCH smaller
+ */
+ sres = rspamd_task_find_symbol_result(task, sym, NULL);
+
+ if (sres && !(sres->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+
+ if (!isnan(sres->score) && !isinf(sres->score) &&
+ (!sres->sym ||
+ !(rspamd_symcache_item_flags(sres->sym->cache_item) & SYMBOL_TYPE_NOSTAT))) {
+
+ gdouble norm_score;
+
+ if (sres->sym && !isnan(sres->sym->score)) {
+ if (sres->sym->score == 0) {
+
+ if (sres->score == 0) {
+ /* Binary symbol */
+ norm_score = 1.0;
+ }
+ else {
+ norm_score = fabs(tanh(sres->score));
+ }
+ }
+ else {
+ /* Get dynamic weight */
+ norm_score = fabs(sres->score / sres->sym->score);
+
+ if (norm_score > 1.0) {
+ /* Multiple hits, we assume them as a single one */
+ norm_score = 1.0;
+ }
+ }
+ }
+ else {
+ norm_score = fabs(tanh(sres->score));
+ }
+
+ lua_pushnumber(L, MAX(min_score, norm_score));
+ lua_rawseti(L, 3, offset + 1);
+ }
+ }
+
+ lua_pop(L, 1); /* Symbol name */
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+enum lua_date_type {
+ DATE_CONNECT = 0,
+ DATE_MESSAGE,
+ DATE_INVALID
+};
+
+static enum lua_date_type
+lua_task_detect_date_type(struct rspamd_task *task,
+ lua_State *L, gint idx, gboolean *gmt)
+{
+ enum lua_date_type type = DATE_CONNECT;
+
+ if (lua_type(L, idx) == LUA_TNUMBER) {
+ gint num = lua_tonumber(L, idx);
+ if (num >= DATE_CONNECT && num < DATE_INVALID) {
+ return num;
+ }
+ }
+ else if (lua_type(L, idx) == LUA_TTABLE) {
+ const gchar *str;
+
+ lua_pushvalue(L, idx);
+ lua_pushstring(L, "format");
+ lua_gettable(L, -2);
+
+ str = lua_tostring(L, -1);
+
+ if (str) {
+ if (g_ascii_strcasecmp(str, "message") == 0) {
+ type = DATE_MESSAGE;
+ }
+ }
+ else {
+ msg_warn_task("date format has not been specified");
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "gmt");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ *gmt = lua_toboolean(L, -1);
+ }
+
+ /* Value and table */
+ lua_pop(L, 2);
+ }
+
+ return type;
+}
+
+static gint
+lua_task_get_date(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_mime_header *h;
+ gdouble tim;
+ enum lua_date_type type = DATE_CONNECT;
+ gboolean gmt = TRUE;
+
+ if (task != NULL) {
+ if (lua_gettop(L) > 1) {
+ type = lua_task_detect_date_type(task, L, 2, &gmt);
+ }
+ /* Get GMT date and store it to time_t */
+ if (type == DATE_CONNECT) {
+ tim = task->task_timestamp;
+
+ if (!gmt) {
+ struct tm t;
+ time_t tt;
+
+ tt = tim;
+ rspamd_localtime(tt, &t);
+#if !defined(__sun)
+ t.tm_gmtoff = 0;
+#endif
+ t.tm_isdst = 0;
+ /* Preserve fractional part as Lua is aware of it */
+ tim = mktime(&t) + (tim - tt);
+ }
+ }
+ else {
+ h = rspamd_message_get_header_array(task, "Date", FALSE);
+
+ if (h) {
+ time_t tt;
+ struct tm t;
+ GError *err = NULL;
+
+ tt = rspamd_parse_smtp_date(h->decoded, strlen(h->decoded),
+ &err);
+
+ if (err == NULL) {
+ if (!gmt) {
+ rspamd_localtime(tt, &t);
+#if !defined(__sun)
+ t.tm_gmtoff = 0;
+#endif
+ t.tm_isdst = 0;
+ tim = mktime(&t);
+ }
+ else {
+ tim = tt;
+ }
+ }
+ else {
+ g_error_free(err);
+ tim = 0.0;
+ }
+ }
+ else {
+ tim = 0.0;
+ }
+ }
+
+ lua_pushnumber(L, tim);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_message_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ if (MESSAGE_FIELD_CHECK(task, message_id) != NULL) {
+ lua_pushstring(L, MESSAGE_FIELD(task, message_id));
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_timeval(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct timeval tv;
+
+ if (task != NULL) {
+ if (lua_isboolean(L, 2) && !!lua_toboolean(L, 2)) {
+ lua_pushnumber(L, task->task_timestamp);
+ }
+ else {
+ double_to_tv(task->task_timestamp, &tv);
+ lua_createtable(L, 0, 2);
+ lua_pushstring(L, "tv_sec");
+ lua_pushinteger(L, (lua_Integer) tv.tv_sec);
+ lua_settable(L, -3);
+ lua_pushstring(L, "tv_usec");
+ lua_pushinteger(L, (lua_Integer) tv.tv_usec);
+ lua_settable(L, -3);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_scan_time(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean set = TRUE;
+
+ if (task != NULL) {
+ if (lua_isboolean(L, 2)) {
+ set = lua_toboolean(L, 2);
+ }
+
+ rspamd_task_set_finish_time(task);
+ gdouble diff = task->time_real_finish - task->task_timestamp;
+ lua_pushnumber(L, diff);
+ lua_pushnumber(L, diff);
+
+ if (!set) {
+ /* Reset to nan to allow further calcs in rspamd_task_set_finish_time */
+ task->time_real_finish = NAN;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+static gint
+lua_task_get_size(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ lua_pushinteger(L, task->msg.len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/**
+* - `no_log`: do not log task summary
+* - `no_stat`: do not include task into scanned stats
+* - `pass_all`: check all filters for task
+* - `extended_urls`: output extended info about urls
+* - `skip`: skip task processing
+*/
+
+#define LUA_TASK_FLAG_WRITE(flag, set) \
+ do { \
+ task->flags = (set) ? (task->flags | (flag)) : (task->flags & ~(flag)); \
+ } while (0)
+
+#define LUA_TASK_SET_FLAG(flag, strname, macro, set) \
+ do { \
+ if (!found && strcmp((flag), strname) == 0) { \
+ LUA_TASK_FLAG_WRITE((macro), set); \
+ found = TRUE; \
+ } \
+ } while (0)
+
+#define LUA_TASK_FLAG_READ(flag) \
+ do { \
+ lua_pushboolean(L, !!(task->flags & (flag))); \
+ } while (0)
+
+#define LUA_TASK_GET_FLAG(flag, strname, macro) \
+ do { \
+ if (!found && strcmp((flag), strname) == 0) { \
+ LUA_TASK_FLAG_READ((macro)); \
+ found = TRUE; \
+ } \
+ } while (0)
+
+#define LUA_TASK_PROTOCOL_FLAG_READ(flag) \
+ do { \
+ lua_pushboolean(L, !!(task->protocol_flags & (flag))); \
+ } while (0)
+
+#define LUA_TASK_GET_PROTOCOL_FLAG(flag, strname, macro) \
+ do { \
+ if (!found && strcmp((flag), strname) == 0) { \
+ LUA_TASK_PROTOCOL_FLAG_READ((macro)); \
+ found = TRUE; \
+ } \
+ } while (0)
+
+static gint
+lua_task_set_flag(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *flag = luaL_checkstring(L, 2);
+ gboolean set = TRUE, found = FALSE;
+
+ if (lua_gettop(L) >= 3) {
+ set = lua_toboolean(L, 3);
+ }
+
+ if (task != NULL && flag != NULL) {
+ LUA_TASK_SET_FLAG(flag, "pass_all", RSPAMD_TASK_FLAG_PASS_ALL, set);
+ LUA_TASK_SET_FLAG(flag, "no_log", RSPAMD_TASK_FLAG_NO_LOG, set);
+ LUA_TASK_SET_FLAG(flag, "no_stat", RSPAMD_TASK_FLAG_NO_STAT, set);
+ LUA_TASK_SET_FLAG(flag, "skip", RSPAMD_TASK_FLAG_SKIP, set);
+ LUA_TASK_SET_FLAG(flag, "learn_spam", RSPAMD_TASK_FLAG_LEARN_SPAM, set);
+ LUA_TASK_SET_FLAG(flag, "learn_ham", RSPAMD_TASK_FLAG_LEARN_HAM, set);
+ LUA_TASK_SET_FLAG(flag, "broken_headers",
+ RSPAMD_TASK_FLAG_BROKEN_HEADERS, set);
+ LUA_TASK_SET_FLAG(flag, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED, set);
+ LUA_TASK_SET_FLAG(flag, "skip_process", RSPAMD_TASK_FLAG_SKIP_PROCESS, set);
+ LUA_TASK_SET_FLAG(flag, "message_rewrite", RSPAMD_TASK_FLAG_MESSAGE_REWRITE, set);
+
+ if (!found) {
+ msg_warn_task("unknown flag requested: %s", flag);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_has_flag(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *flag = luaL_checkstring(L, 2);
+ gboolean found = FALSE;
+
+ if (task != NULL && flag != NULL) {
+ LUA_TASK_GET_FLAG(flag, "pass_all", RSPAMD_TASK_FLAG_PASS_ALL);
+ LUA_TASK_GET_FLAG(flag, "no_log", RSPAMD_TASK_FLAG_NO_LOG);
+ LUA_TASK_GET_FLAG(flag, "no_stat", RSPAMD_TASK_FLAG_NO_STAT);
+ LUA_TASK_GET_FLAG(flag, "skip", RSPAMD_TASK_FLAG_SKIP);
+ LUA_TASK_GET_FLAG(flag, "learn_spam", RSPAMD_TASK_FLAG_LEARN_SPAM);
+ LUA_TASK_GET_FLAG(flag, "learn_ham", RSPAMD_TASK_FLAG_LEARN_HAM);
+ LUA_TASK_GET_FLAG(flag, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED);
+ LUA_TASK_GET_FLAG(flag, "broken_headers",
+ RSPAMD_TASK_FLAG_BROKEN_HEADERS);
+ LUA_TASK_GET_FLAG(flag, "skip_process",
+ RSPAMD_TASK_FLAG_SKIP_PROCESS);
+ LUA_TASK_GET_FLAG(flag, "bad_unicode",
+ RSPAMD_TASK_FLAG_BAD_UNICODE);
+ LUA_TASK_GET_FLAG(flag, "mime",
+ RSPAMD_TASK_FLAG_MIME);
+ LUA_TASK_GET_FLAG(flag, "message_rewrite",
+ RSPAMD_TASK_FLAG_MESSAGE_REWRITE);
+ LUA_TASK_GET_PROTOCOL_FLAG(flag, "milter",
+ RSPAMD_TASK_PROTOCOL_FLAG_MILTER);
+
+ if (!found) {
+ msg_warn_task("unknown flag requested: %s", flag);
+ lua_pushboolean(L, 0);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_flags(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gint idx = 1;
+ guint flags, bit, i;
+
+ if (task) {
+ lua_createtable(L, 8, 0);
+
+ flags = task->flags;
+
+ for (i = 0; i <= RSPAMD_TASK_FLAG_MAX_SHIFT; i++) {
+ bit = (1U << i);
+
+ if (flags & bit) {
+ switch (bit) {
+ case RSPAMD_TASK_FLAG_PASS_ALL:
+ lua_pushstring(L, "pass_all");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_NO_LOG:
+ lua_pushstring(L, "no_log");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_NO_STAT:
+ lua_pushstring(L, "no_stat");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_SKIP:
+ lua_pushstring(L, "skip");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_BROKEN_HEADERS:
+ lua_pushstring(L, "broken_headers");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_LEARN_SPAM:
+ lua_pushstring(L, "learn_spam");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_LEARN_HAM:
+ lua_pushstring(L, "learn_ham");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_GREYLISTED:
+ lua_pushstring(L, "greylisted");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_SKIP_PROCESS:
+ lua_pushstring(L, "skip_process");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_MESSAGE_REWRITE:
+ lua_pushstring(L, "message_rewrite");
+ lua_rawseti(L, -2, idx++);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_MILTER) {
+ lua_pushstring(L, "milter");
+ lua_rawseti(L, -2, idx++);
+ }
+ if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_BODY_BLOCK) {
+ lua_pushstring(L, "body_block");
+ lua_rawseti(L, -2, idx++);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_digest(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gchar hexbuf[sizeof(MESSAGE_FIELD(task, digest)) * 2 + 1];
+ gint r;
+
+ if (task) {
+ if (task->message) {
+ r = rspamd_encode_hex_buf(MESSAGE_FIELD(task, digest),
+ sizeof(MESSAGE_FIELD(task, digest)),
+ hexbuf, sizeof(hexbuf) - 1);
+
+ if (r > 0) {
+ hexbuf[r] = '\0';
+ lua_pushstring(L, hexbuf);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_learn(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean is_spam = FALSE;
+ const gchar *clname = NULL;
+ GError *err = NULL;
+ int ret = 1;
+
+ if (task == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ is_spam = lua_toboolean(L, 2);
+ if (lua_gettop(L) > 2) {
+ clname = luaL_checkstring(L, 3);
+ }
+
+ if (!rspamd_learn_task_spam(task, is_spam, clname, &err)) {
+ lua_pushboolean(L, FALSE);
+ if (err != NULL) {
+ lua_pushstring(L, err->message);
+ ret = 2;
+ }
+ }
+ else {
+ lua_pushboolean(L, TRUE);
+ }
+
+ return ret;
+}
+
+static gint
+lua_task_set_settings(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ ucl_object_t *settings;
+ const ucl_object_t *act, *metric_elt, *vars, *cur;
+ ucl_object_iter_t it = NULL;
+ struct rspamd_scan_result *mres;
+ guint i;
+
+ settings = ucl_object_lua_import(L, 2);
+
+ if (settings != NULL && task != NULL) {
+
+ if (task->settings) {
+ /* Do not allow to set settings on top of the existing ones */
+ ucl_object_unref(settings);
+
+ return luaL_error(L, "invalid invocation: settings has been already set");
+ }
+
+ metric_elt = ucl_object_lookup(settings, DEFAULT_METRIC);
+
+ if (metric_elt) {
+ task->settings = ucl_object_ref(metric_elt);
+ ucl_object_unref(settings);
+ }
+ else {
+ task->settings = settings;
+ }
+
+ act = ucl_object_lookup(task->settings, "actions");
+
+ if (act && ucl_object_type(act) == UCL_OBJECT) {
+ /* Adjust desired actions */
+ mres = task->result;
+
+ it = NULL;
+
+ while ((cur = ucl_object_iterate(act, &it, true)) != NULL) {
+ const gchar *act_name = ucl_object_key(cur);
+ struct rspamd_action_config *action_config = NULL;
+ double act_score;
+ enum rspamd_action_type act_type;
+
+ if (!rspamd_action_from_str(act_name, &act_type)) {
+ act_type = -1;
+ }
+
+ for (i = 0; i < mres->nactions; i++) {
+ struct rspamd_action_config *cur_act = &mres->actions_config[i];
+
+ if (cur_act->action->action_type == METRIC_ACTION_CUSTOM &&
+ act_type == -1) {
+ /* Compare by name */
+ if (g_ascii_strcasecmp(act_name, cur_act->action->name) == 0) {
+ action_config = cur_act;
+ break;
+ }
+ }
+ else {
+ if (cur_act->action->action_type == act_type) {
+ action_config = cur_act;
+ break;
+ }
+ }
+ }
+
+ if (!action_config) {
+ act_score = ucl_object_todouble(cur);
+ if (!isnan(act_score)) {
+ struct rspamd_action *new_act;
+
+ new_act = rspamd_config_get_action(task->cfg, act_name);
+
+ if (new_act == NULL) {
+ /* New action! */
+ msg_info_task("added new action %s with threshold %.2f "
+ "due to settings",
+ act_name,
+ act_score);
+ new_act = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(*new_act));
+ new_act->name = rspamd_mempool_strdup(task->task_pool, act_name);
+ new_act->action_type = METRIC_ACTION_CUSTOM;
+ new_act->threshold = act_score;
+ }
+ else {
+ /* A disabled action that is enabled */
+ msg_info_task("enabled disabled action %s with threshold %.2f "
+ "due to settings",
+ act_name,
+ act_score);
+ }
+
+ /* Insert it to the mres structure */
+ gsize new_actions_cnt = mres->nactions + 1;
+ struct rspamd_action_config *old_actions = mres->actions_config;
+
+ mres->actions_config = rspamd_mempool_alloc(task->task_pool,
+ sizeof(struct rspamd_action_config) * new_actions_cnt);
+ memcpy(mres->actions_config, old_actions,
+ sizeof(struct rspamd_action_config) * mres->nactions);
+ mres->actions_config[mres->nactions].action = new_act;
+ mres->actions_config[mres->nactions].cur_limit = act_score;
+ mres->nactions++;
+ }
+ /* Disabled/missing action is disabled one more time, not an error */
+ }
+ else {
+ /* Found the existing configured action */
+ if (ucl_object_type(cur) == UCL_NULL) {
+ /* Disable action completely */
+ action_config->flags |= RSPAMD_ACTION_RESULT_DISABLED;
+ msg_info_task("disabled action %s due to settings",
+ action_config->action->name);
+ }
+ else {
+ act_score = ucl_object_todouble(cur);
+ if (isnan(act_score)) {
+ msg_info_task("disabled action %s threshold (was %.2f) due to settings",
+ action_config->action->name,
+ action_config->cur_limit);
+ action_config->flags |= RSPAMD_ACTION_RESULT_NO_THRESHOLD;
+ }
+ else {
+ action_config->cur_limit = act_score;
+ msg_debug_task("adjusted action %s: %.2f -> %.2f",
+ act_name,
+ action_config->cur_limit,
+ act_score);
+ }
+ }
+ }
+ }
+ }
+
+ vars = ucl_object_lookup(task->settings, "variables");
+ if (vars && ucl_object_type(vars) == UCL_OBJECT) {
+ /* Set memory pool variables */
+ it = NULL;
+
+ while ((cur = ucl_object_iterate(vars, &it, true)) != NULL) {
+ if (ucl_object_type(cur) == UCL_STRING) {
+ rspamd_mempool_set_variable(task->task_pool,
+ ucl_object_key(cur), rspamd_mempool_strdup(task->task_pool, ucl_object_tostring(cur)), NULL);
+ }
+ }
+ }
+
+ rspamd_symcache_process_settings(task, task->cfg->cache);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_set_milter_reply(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ ucl_object_t *reply, *prev;
+
+ reply = ucl_object_lua_import(L, 2);
+
+ if (reply != NULL && task != NULL) {
+ prev = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_MILTER_REPLY);
+
+ if (prev) {
+ /*
+ * We need to be very special about the add_headers part
+ * If we want to insert some existing object, such as
+ * add_headers = {
+ * hdr = {value = val1, order = 1},
+ * }
+ *
+ * and new header has something similar:
+ * add_headers = {
+ * hdr = {value = val2, order = 1},
+ * }
+ *
+ * then we need to convert it to an array...
+ *
+ * add_headers = {
+ * hdr = [{value = val1, order = 1}, {value = val2, order = 1}],
+ * }
+ *
+ * UCL itself cannot do it directly. So the trick is to extract the
+ * original object, pack it into an array and then insert it back.
+ *
+ * I wish there was a simpler way to do it...
+ */
+ const ucl_object_t *add_hdrs = ucl_object_lookup(prev, "add_headers");
+ const ucl_object_t *nadd_hdrs = ucl_object_lookup(reply, "add_headers");
+
+ if (add_hdrs && nadd_hdrs) {
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur;
+
+ while ((cur = ucl_object_iterate(nadd_hdrs, &it, true)) != NULL) {
+ gsize klen;
+ const gchar *key = ucl_object_keyl(cur, &klen);
+ const ucl_object_t *existing;
+
+ existing = ucl_object_lookup_len(add_hdrs, key, klen);
+
+ if (existing && ucl_object_type(existing) != UCL_ARRAY) {
+ ucl_object_t *ar = ucl_object_typed_new(UCL_ARRAY);
+
+ ucl_array_append(ar, ucl_object_ref(existing));
+ /* Avoid double refcount */
+ key = ucl_object_keyl(existing, &klen);
+ ucl_object_delete_keyl((ucl_object_t *) add_hdrs, key, klen);
+ ucl_object_insert_key((ucl_object_t *) add_hdrs,
+ ar, key, klen, false);
+ }
+ }
+ }
+
+ if (!ucl_object_merge(prev, reply, false)) {
+ msg_err_task("internal error: cannot merge two objects when setting milter reply!");
+ }
+ ucl_object_unref(reply);
+ }
+ else {
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_MILTER_REPLY,
+ reply,
+ (rspamd_mempool_destruct_t) ucl_object_unref);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_settings(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+
+ if (task->settings) {
+ return ucl_object_push_lua(L, task->settings, true);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_lookup_settings(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *key = NULL;
+ const ucl_object_t *elt;
+
+ if (task != NULL) {
+
+ if (lua_isstring(L, 2)) {
+ key = lua_tostring(L, 2);
+ }
+
+ if (task->settings) {
+ if (key == NULL) {
+ return ucl_object_push_lua(L, task->settings, true);
+ }
+ else {
+ elt = ucl_object_lookup(task->settings, key);
+
+ if (elt) {
+ return ucl_object_push_lua(L, elt, true);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_settings_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+
+ if (task->settings_elt) {
+ lua_pushinteger(L, task->settings_elt->id);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_settings_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint32 id = lua_tointeger(L, 2);
+
+ if (task != NULL && id != 0) {
+
+ struct rspamd_config_settings_elt *selt =
+ rspamd_config_find_settings_id_ref(task->cfg, id);
+
+ if (selt == NULL) {
+ return luaL_error(L, "settings id %f is unknown", (lua_Number) id);
+ }
+ if (task->settings_elt) {
+ /* Overwrite existing settings from Lua */
+ REF_RELEASE(task->settings_elt);
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ task->settings_elt = selt;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_cache_get(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *key = luaL_checkstring(L, 2);
+
+ if (task && key) {
+ if (!lua_task_get_cached(L, task, key)) {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_cache_set(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *key = luaL_checkstring(L, 2);
+
+ if (task && key && lua_gettop(L) >= 3) {
+ lua_task_set_cached(L, task, key, 3);
+ }
+ else {
+ luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+struct lua_file_cbdata {
+ gchar *fname;
+ gint fd;
+ gboolean keep;
+};
+
+static void
+lua_tmp_file_dtor(gpointer p)
+{
+ struct lua_file_cbdata *cbdata = p;
+
+ if (!cbdata->keep) {
+ unlink(cbdata->fname);
+ }
+
+ close(cbdata->fd);
+}
+
+static gint
+lua_task_store_in_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean force_new = FALSE, keep = FALSE;
+ gchar fpath[PATH_MAX];
+ const gchar *tmpmask = NULL, *fname = NULL;
+ guint mode = 00600;
+ gint fd;
+ struct lua_file_cbdata *cbdata;
+ GError *err = NULL;
+
+ if (task) {
+ if (lua_istable(L, 2)) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "filename=S;tmpmask=S;mode=I;force_new=B;keep=B",
+ &fname, &tmpmask, &mode, &force_new, &keep)) {
+ msg_err_task("cannot get parameters list: %e", err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+ }
+ else if (lua_isnumber(L, 2)) {
+ mode = lua_tointeger(L, 2);
+ }
+
+ if (!force_new && (task->flags & RSPAMD_TASK_FLAG_FILE) &&
+ task->msg.fpath) {
+ lua_pushstring(L, task->msg.fpath);
+ }
+ else {
+ if (fname == NULL) {
+ if (tmpmask == NULL) {
+ rspamd_snprintf(fpath, sizeof(fpath), "%s%c%s",
+ task->cfg->temp_dir,
+ G_DIR_SEPARATOR, "rmsg-XXXXXXXXXX");
+ }
+ else {
+ rspamd_snprintf(fpath, sizeof(fpath), "%s", tmpmask);
+ }
+
+ fd = g_mkstemp_full(fpath, O_WRONLY | O_CREAT | O_EXCL, mode);
+ fname = fpath;
+
+ if (fd != -1) {
+ fchmod(fd, mode);
+ }
+ }
+ else {
+ fd = rspamd_file_xopen(fname, O_WRONLY | O_CREAT | O_EXCL,
+ (guint) mode, FALSE);
+ }
+
+ if (fd == -1) {
+ msg_err_task("cannot save file: %s", strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (write(fd, task->msg.begin, task->msg.len) == -1) {
+ msg_err_task("cannot write file %s: %s", fpath,
+ strerror(errno));
+ unlink(fname);
+ close(fd);
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ cbdata = rspamd_mempool_alloc(task->task_pool, sizeof(*cbdata));
+ cbdata->fd = fd;
+ cbdata->fname = rspamd_mempool_strdup(task->task_pool, fname);
+ cbdata->keep = keep;
+ lua_pushstring(L, cbdata->fname);
+ rspamd_mempool_add_destructor(task->task_pool,
+ lua_tmp_file_dtor, cbdata);
+ }
+ }
+ }
+ else {
+ luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_process_regexp(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_regexp *re = NULL;
+ gboolean strong = FALSE;
+ const gchar *type_str = NULL, *header_str = NULL;
+ gsize header_len = 0;
+ GError *err = NULL;
+ gint ret = 0;
+ enum rspamd_re_type type = RSPAMD_RE_BODY;
+
+ /*
+ * - `re`* : regular expression object
+ * - `type`*: type of regular expression:
+ * + `mime`: mime regexp
+ * + `rawmime`: raw mime regexp
+ * + `header`: header regexp
+ * + `rawheader`: raw header expression
+ * + `body`: raw body regexp
+ * + `url`: url regexp
+ * - `header`: for header and rawheader regexp means the name of header
+ * - `strong`: case sensitive match for headers
+ */
+ if (task != NULL) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*re=U{regexp};*type=S;header=V;strong=B",
+ &re, &type_str, &header_len, &header_str,
+ &strong)) {
+ msg_err_task("cannot get parameters list: %e", err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+ else {
+ type = rspamd_re_cache_type_from_string(type_str);
+
+ if ((type == RSPAMD_RE_HEADER || type == RSPAMD_RE_RAWHEADER) && header_str == NULL) {
+ msg_err_task(
+ "header argument is mandatory for header/rawheader regexps");
+ }
+ else {
+ ret = rspamd_re_cache_process(task, re->re, type,
+ (gpointer) header_str, header_len, strong);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_task_get_metric_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_scan_result *metric_res;
+ struct rspamd_action *action;
+
+ if (task) {
+ metric_res = task->result;
+
+ if (lua_isstring(L, 2)) {
+ metric_res = rspamd_find_metric_result(task, lua_tostring(L, 2));
+
+ if (metric_res == NULL) {
+ lua_pushnil(L);
+
+ return 1;
+ }
+ }
+
+ /* Fields added:
+ * - `score`: current score
+ * - `action`: current action as a string
+ * - `nnegative`: number of negative rules matched
+ * - `npositive`: number of positive rules matched
+ * - `positive_score`: total score for positive rules
+ * - `negative_score`: total score for negative rules
+ * - `passthrough`: set to true if message has a passthrough result
+ */
+ lua_createtable(L, 0, 7);
+
+ lua_pushstring(L, "score");
+ lua_pushnumber(L, metric_res->score);
+ lua_settable(L, -3);
+
+ action = rspamd_check_action_metric(task, NULL, metric_res);
+
+ if (action) {
+ lua_pushstring(L, "action");
+ lua_pushstring(L, action->name);
+ lua_settable(L, -3);
+ }
+
+ lua_pushstring(L, "nnegative");
+ lua_pushnumber(L, metric_res->nnegative);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "npositive");
+ lua_pushnumber(L, metric_res->npositive);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "positive_score");
+ lua_pushnumber(L, metric_res->positive_score);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "negative_score");
+ lua_pushnumber(L, metric_res->negative_score);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "passthrough");
+ lua_pushboolean(L, !!(metric_res->passthrough_result != NULL));
+ lua_settable(L, -3);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_metric_score(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gdouble rs;
+ struct rspamd_scan_result *metric_res;
+
+ if (task) {
+ metric_res = task->result;
+
+ if (lua_isstring(L, 2)) {
+ metric_res = rspamd_find_metric_result(task, lua_tostring(L, 2));
+ }
+
+ if (metric_res != NULL) {
+ lua_createtable(L, 2, 0);
+ lua_pushnumber(L, isnan(metric_res->score) ? 0.0 : metric_res->score);
+ rs = rspamd_task_get_required_score(task, metric_res);
+ lua_rawseti(L, -2, 1);
+ lua_pushnumber(L, rs);
+ lua_rawseti(L, -2, 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_metric_action(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_action *action;
+
+ if (task) {
+ struct rspamd_scan_result *mres = task->result;
+
+ if (lua_isstring(L, 2)) {
+ mres = rspamd_find_metric_result(task, lua_tostring(L, 2));
+ }
+
+ if (mres == NULL) {
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ action = rspamd_check_action_metric(task, NULL, mres);
+ lua_pushstring(L, action->name);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_metric_score(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_scan_result *metric_res;
+ gdouble nscore;
+
+ if (lua_isnumber(L, 2)) {
+ nscore = luaL_checknumber(L, 2);
+ }
+ else {
+ nscore = luaL_checknumber(L, 3);
+ }
+
+ if (task) {
+ metric_res = task->result;
+
+ if (lua_isstring(L, 4)) {
+ metric_res = rspamd_find_metric_result(task, lua_tostring(L, 4));
+ }
+
+ if (metric_res != NULL) {
+ msg_debug_task("set metric score from %.2f to %.2f",
+ metric_res->score, nscore);
+ metric_res->score = nscore;
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_disable_action(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *action_name;
+ struct rspamd_action_config *action_res;
+
+ action_name = luaL_checkstring(L, 2);
+
+ if (task && action_name) {
+
+ for (guint i = 0; i < task->result->nactions; i++) {
+ action_res = &task->result->actions_config[i];
+
+ if (strcmp(action_name, action_res->action->name) == 0) {
+ if (isnan(action_res->cur_limit)) {
+ lua_pushboolean(L, false);
+ }
+ else {
+ action_res->cur_limit = NAN;
+ lua_pushboolean(L, true);
+ }
+
+ break;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_newlines_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->message) {
+ switch (MESSAGE_FIELD(task, nlines_type)) {
+ case RSPAMD_TASK_NEWLINES_CR:
+ lua_pushstring(L, "cr");
+ break;
+ case RSPAMD_TASK_NEWLINES_LF:
+ lua_pushstring(L, "lf");
+ break;
+ case RSPAMD_TASK_NEWLINES_CRLF:
+ default:
+ lua_pushstring(L, "crlf");
+ break;
+ }
+ }
+ else {
+ lua_pushstring(L, "crlf");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static void
+lua_push_stat_token(lua_State *L, rspamd_token_t *tok)
+{
+ gchar numbuf[64];
+
+ /* Table values
+ * - `data`: 64 bit number encoded as a string
+ * - `t1`: the first token (if any)
+ * - `t2`: the second token (if any)
+ * - `win`: window index
+ * - `flag`: table of strings:
+ * - `text`: text token
+ * - `meta`: meta token
+ * - `lua`: lua meta token
+ * - `exception`: exception
+ * - `subject`: subject token
+ * - `unigram`: unigram token
+ */
+ lua_createtable(L, 0, 5);
+
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%uL", tok->data);
+ lua_pushstring(L, "data");
+ lua_pushstring(L, numbuf);
+ lua_settable(L, -3);
+
+ if (tok->t1) {
+ lua_pushstring(L, "t1");
+ lua_pushlstring(L, tok->t1->stemmed.begin, tok->t1->stemmed.len);
+ lua_settable(L, -3);
+ }
+
+ if (tok->t2) {
+ lua_pushstring(L, "t2");
+ lua_pushlstring(L, tok->t2->stemmed.begin, tok->t2->stemmed.len);
+ lua_settable(L, -3);
+ }
+
+ lua_pushstring(L, "win");
+ lua_pushinteger(L, tok->window_idx);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "flags");
+ lua_createtable(L, 0, 5);
+
+ /* Flags */
+ {
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) {
+ lua_pushstring(L, "text");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_META) {
+ lua_pushstring(L, "meta");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_LUA_META) {
+ lua_pushstring(L, "lua");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_EXCEPTION) {
+ lua_pushstring(L, "exception");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_HEADER) {
+ lua_pushstring(L, "header");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ }
+ lua_settable(L, -3);
+}
+
+static gint
+lua_task_get_stat_tokens(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint i;
+ rspamd_token_t *tok;
+
+ if (task) {
+ if (!task->tokens) {
+ rspamd_stat_process_tokenize(NULL, task);
+ }
+
+ if (!task->tokens) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_createtable(L, task->tokens->len, 0);
+
+ PTR_ARRAY_FOREACH(task->tokens, i, tok)
+ {
+ lua_push_stat_token(L, tok);
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_metric_subject(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *subject;
+
+ subject = luaL_checkstring(L, 2);
+
+ if (task && subject) {
+ rspamd_mempool_set_variable(task->task_pool, "metric_subject",
+ rspamd_mempool_strdup(task->task_pool, subject), NULL);
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_protocol_reply(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint flags = 0;
+ ucl_object_t *obj;
+
+ if (!task) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (!(task->processed_stages & (RSPAMD_TASK_STAGE_POST_FILTERS >> 1))) {
+ return luaL_error(L, "must not be called before post-filters");
+ }
+
+ if (lua_istable(L, 2)) {
+ for (lua_pushnil(L); lua_next(L, 2); lua_pop(L, 1)) {
+ if (lua_isstring(L, -1)) {
+ const gchar *str = lua_tostring(L, -1);
+
+ if (strcmp(str, "default") == 0) {
+ flags |= RSPAMD_PROTOCOL_DEFAULT;
+ }
+ else if (strcmp(str, "basic") == 0) {
+ flags |= RSPAMD_PROTOCOL_BASIC;
+ }
+ else if (strcmp(str, "metrics") == 0) {
+ flags |= RSPAMD_PROTOCOL_METRICS;
+ }
+ else if (strcmp(str, "messages") == 0) {
+ flags |= RSPAMD_PROTOCOL_MESSAGES;
+ }
+ else if (strcmp(str, "rmilter") == 0) {
+ flags |= RSPAMD_PROTOCOL_RMILTER;
+ }
+ else if (strcmp(str, "dkim") == 0) {
+ flags |= RSPAMD_PROTOCOL_DKIM;
+ }
+ else if (strcmp(str, "extra") == 0) {
+ flags |= RSPAMD_PROTOCOL_EXTRA;
+ }
+ else {
+ msg_err_task("invalid protocol flag: %s", str);
+ }
+ }
+ }
+ }
+ else {
+ flags = RSPAMD_PROTOCOL_DEFAULT;
+ }
+
+ obj = rspamd_protocol_write_ucl(task, flags);
+
+ if (obj) {
+ ucl_object_push_lua(L, obj, true);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_headers_foreach(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ enum rspamd_lua_task_header_type how = RSPAMD_TASK_HEADER_PUSH_SIMPLE;
+ struct rspamd_lua_regexp *re = NULL;
+ struct rspamd_mime_header *hdr, *cur;
+ gint old_top;
+
+ if (task && lua_isfunction(L, 2)) {
+ if (task->message) {
+ if (lua_istable(L, 3)) {
+ lua_pushstring(L, "full");
+ lua_gettable(L, 3);
+
+ if (lua_isboolean(L, -1) && lua_toboolean(L, -1)) {
+ how = RSPAMD_TASK_HEADER_PUSH_FULL;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "raw");
+ lua_gettable(L, 3);
+
+ if (lua_isboolean(L, -1) && lua_toboolean(L, -1)) {
+ how = RSPAMD_TASK_HEADER_PUSH_RAW;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "regexp");
+ lua_gettable(L, 3);
+
+ if (lua_isuserdata(L, -1)) {
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, -1, "rspamd{regexp}",
+ struct rspamd_lua_regexp, re);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ if (MESSAGE_FIELD(task, headers_order)) {
+ hdr = MESSAGE_FIELD(task, headers_order);
+
+ LL_FOREACH2(hdr, cur, ord_next)
+ {
+ if (re && re->re) {
+ if (!rspamd_regexp_match(re->re, cur->name,
+ strlen(cur->name), FALSE)) {
+ continue;
+ }
+ }
+
+ old_top = lua_gettop(L);
+ lua_pushvalue(L, 2);
+ lua_pushstring(L, cur->name);
+ rspamd_lua_push_header(L, cur, how);
+
+ if (lua_pcall(L, 2, LUA_MULTRET, 0) != 0) {
+ msg_err("call to header_foreach failed: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, old_top);
+ break;
+ }
+ else {
+ if (lua_gettop(L) > old_top) {
+ if (lua_isboolean(L, old_top + 1)) {
+ if (lua_toboolean(L, old_top + 1)) {
+ lua_settop(L, old_top);
+ break;
+ }
+ }
+ }
+ }
+
+ lua_settop(L, old_top);
+ }
+ }
+ } /* if (task->message) */
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_modify_header(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *hname = luaL_checkstring(L, 2);
+
+ if (hname && task && lua_type(L, 3) == LUA_TTABLE) {
+ if (task->message) {
+ ucl_object_t *mods = ucl_object_lua_import(L, 3);
+
+ rspamd_message_set_modified_header(task,
+ MESSAGE_FIELD(task, raw_headers),
+ hname,
+ mods,
+ &(MESSAGE_FIELD(task, headers_order)));
+ ucl_object_unref(mods);
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_meta_words(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ enum rspamd_lua_words_type how = RSPAMD_LUA_WORDS_STEM;
+
+ if (task == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (task->meta_words == NULL) {
+ lua_createtable(L, 0, 0);
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 2);
+
+ if (strcmp(how_str, "stem") == 0) {
+ how = RSPAMD_LUA_WORDS_STEM;
+ }
+ else if (strcmp(how_str, "norm") == 0) {
+ how = RSPAMD_LUA_WORDS_NORM;
+ }
+ else if (strcmp(how_str, "raw") == 0) {
+ how = RSPAMD_LUA_WORDS_RAW;
+ }
+ else if (strcmp(how_str, "full") == 0) {
+ how = RSPAMD_LUA_WORDS_FULL;
+ }
+ else {
+ return luaL_error(L, "unknown words type: %s", how_str);
+ }
+ }
+
+ return rspamd_lua_push_words(L, task->meta_words, how);
+ }
+
+ return 1;
+}
+
+static guint
+lua_lookup_words_array(lua_State *L,
+ gint cbpos,
+ struct rspamd_task *task,
+ struct rspamd_lua_map *map,
+ GArray *words)
+{
+ rspamd_stat_token_t *tok;
+ guint i, nmatched = 0;
+ gint err_idx;
+ gboolean matched;
+ const gchar *key;
+ gsize keylen;
+
+ for (i = 0; i < words->len; i++) {
+ tok = &g_array_index(words, rspamd_stat_token_t, i);
+
+ matched = FALSE;
+
+ if (tok->normalized.len == 0) {
+ continue;
+ }
+
+ key = tok->normalized.begin;
+ keylen = tok->normalized.len;
+
+ switch (map->type) {
+ case RSPAMD_LUA_MAP_SET:
+ case RSPAMD_LUA_MAP_HASH:
+ /* We know that tok->normalized is zero terminated in fact */
+ if (rspamd_match_hash_map(map->data.hash, key, keylen)) {
+ matched = TRUE;
+ }
+ break;
+ case RSPAMD_LUA_MAP_REGEXP:
+ case RSPAMD_LUA_MAP_REGEXP_MULTIPLE:
+ if (rspamd_match_regexp_map_single(map->data.re_map, key,
+ keylen)) {
+ matched = TRUE;
+ }
+ break;
+ default:
+ g_assert_not_reached();
+ break;
+ }
+
+ if (matched) {
+ nmatched++;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_pushvalue(L, cbpos); /* Function */
+ rspamd_lua_push_full_word(L, tok);
+
+ if (lua_pcall(L, 1, 0, err_idx) != 0) {
+ msg_err_task("cannot call callback function for lookup words: %s",
+ lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+ }
+ }
+
+ return nmatched;
+}
+
+static gint
+lua_task_lookup_words(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_map *map = lua_check_map(L, 2);
+ struct rspamd_mime_text_part *tp;
+
+ guint i, matches = 0;
+
+ if (task == NULL || map == NULL || task->message == NULL || lua_type(L, 3) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (map->type != RSPAMD_LUA_MAP_SET &&
+ map->type != RSPAMD_LUA_MAP_REGEXP &&
+ map->type != RSPAMD_LUA_MAP_HASH &&
+ map->type != RSPAMD_LUA_MAP_REGEXP_MULTIPLE) {
+ return luaL_error(L, "invalid map type");
+ }
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, tp)
+ {
+ if (tp->utf_words) {
+ matches += lua_lookup_words_array(L, 3, task, map, tp->utf_words);
+ }
+ }
+
+ if (task->meta_words) {
+ matches += lua_lookup_words_array(L, 3, task, map, task->meta_words);
+ }
+
+ lua_pushinteger(L, matches);
+
+ return 1;
+}
+
+static gint
+lua_task_topointer(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ /* XXX: this might cause issues on arm64 and LuaJIT */
+ lua_pushlightuserdata(L, task);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_add_named_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *name = luaL_checkstring(L, 2);
+ gint cbref;
+
+ if (task && name && lua_isfunction(L, 3)) {
+ lua_pushvalue(L, 3);
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ rspamd_create_metric_result(task, name, cbref);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_all_named_results(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ gint n = 0;
+ struct rspamd_scan_result *res;
+
+ DL_COUNT(task->result, res, n);
+ lua_createtable(L, n, 0);
+ n = 1;
+
+ DL_FOREACH(task->result, res)
+ {
+ if (res->name != NULL) {
+ lua_pushstring(L, res->name);
+ }
+ else {
+ lua_pushstring(L, DEFAULT_METRIC);
+ }
+
+ lua_rawseti(L, -2, n++);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+/* Image functions */
+static gint
+lua_image_get_width(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_image *img = lua_check_image(L);
+
+ if (img != NULL) {
+ lua_pushinteger(L, img->width);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_image_get_height(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_image *img = lua_check_image(L);
+
+ if (img != NULL) {
+ lua_pushinteger(L, img->height);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_image_get_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_image *img = lua_check_image(L);
+
+ if (img != NULL) {
+ lua_pushstring(L, rspamd_image_type_str(img->type));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_image_get_size(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_image *img = lua_check_image(L);
+
+ if (img != NULL) {
+ lua_pushinteger(L, img->data->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_image_get_filename(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_image *img = lua_check_image(L);
+
+ if (img != NULL) {
+ if (img->filename != NULL) {
+ lua_pushlstring(L, img->filename->begin, img->filename->len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/* Archive methods */
+static gint
+lua_archive_get_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushstring(L, rspamd_archive_type_str(arch->type));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_get_files(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+ guint i, max_files = 0;
+ struct rspamd_archive_file *f;
+
+ if (arch != NULL) {
+ if (lua_isnumber(L, 2)) {
+ max_files = lua_tointeger(L, 2);
+ max_files = MIN(arch->files->len, max_files);
+ }
+ else {
+ max_files = arch->files->len;
+ }
+
+ lua_createtable(L, max_files, 0);
+
+ for (i = 0; i < max_files; i++) {
+ f = g_ptr_array_index(arch->files, i);
+
+ lua_pushlstring(L, f->fname->str, f->fname->len);
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_get_files_full(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+ guint i, max_files = 0;
+ struct rspamd_archive_file *f;
+
+ if (arch != NULL) {
+ if (lua_isnumber(L, 2)) {
+ max_files = lua_tointeger(L, 2);
+ max_files = MIN(arch->files->len, max_files);
+ }
+ else {
+ max_files = arch->files->len;
+ }
+
+ lua_createtable(L, max_files, 0);
+
+ for (i = 0; i < max_files; i++) {
+ f = g_ptr_array_index(arch->files, i);
+
+ lua_createtable(L, 0, 4);
+
+ lua_pushstring(L, "name");
+ lua_pushlstring(L, f->fname->str, f->fname->len);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "compressed_size");
+ lua_pushinteger(L, f->compressed_size);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "uncompressed_size");
+ lua_pushinteger(L, f->uncompressed_size);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "encrypted");
+ lua_pushboolean(L, (f->flags & RSPAMD_ARCHIVE_FILE_ENCRYPTED) ? true : false);
+ lua_settable(L, -3);
+
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_is_encrypted(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushboolean(L, (arch->flags & RSPAMD_ARCHIVE_ENCRYPTED) ? true : false);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_is_obfuscated(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushboolean(L,
+ (arch->flags & RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES) ? true : false);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_is_unreadable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushboolean(L, (arch->flags & RSPAMD_ARCHIVE_CANNOT_READ) ? true : false);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_get_size(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushinteger(L, arch->size);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_get_filename(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushlstring(L, arch->archive_name->begin, arch->archive_name->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/* Init part */
+
+static gint
+lua_load_task(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, tasklib_f);
+
+ return 1;
+}
+
+static void
+luaopen_archive(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{archive}", archivelib_m);
+ lua_pop(L, 1);
+}
+
+void luaopen_task(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{task}", tasklib_m);
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "rspamd_task", lua_load_task);
+
+ luaopen_archive(L);
+}
+
+void luaopen_image(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{image}", imagelib_m);
+ lua_pop(L, 1);
+}
+
+void rspamd_lua_task_push(lua_State *L, struct rspamd_task *task)
+{
+ struct rspamd_task **ptask;
+
+ ptask = lua_newuserdata(L, sizeof(gpointer));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+}
diff --git a/src/lua/lua_tcp.c b/src/lua/lua_tcp.c
new file mode 100644
index 0000000..45faa79
--- /dev/null
+++ b/src/lua/lua_tcp.c
@@ -0,0 +1,2566 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+#include "libserver/ssl_util.h"
+#include "utlist.h"
+#include "unix-std.h"
+#include <math.h>
+
+static const gchar *M = "rspamd lua tcp";
+
+/***
+ * @module rspamd_tcp
+ * Rspamd TCP module represents generic TCP asynchronous client available from LUA code.
+ * This module hides all complexity: DNS resolving, sessions management, zero-copy
+ * text transfers and so on under the hood. It can work in partial or complete modes:
+ *
+ * - partial mode is used when you need to call a continuation routine each time data is available for read
+ * - complete mode calls for continuation merely when all data is read from socket (e.g. when a server sends reply and closes a connection)
+ * @example
+local logger = require "rspamd_logger"
+local tcp = require "rspamd_tcp"
+
+rspamd_config.SYM = function(task)
+
+ local function cb(err, data)
+ logger.infox('err: %1, data: %2', err, tostring(data))
+ end
+
+ tcp.request({
+ task = task,
+ host = "google.com",
+ port = 80,
+ data = {"GET / HTTP/1.0\r\n", "Host: google.com\r\n", "\r\n"},
+ callback = cb})
+end
+
+-- New TCP syntax test
+rspamd_config:register_symbol({
+ name = 'TCP_TEST',
+ type = "normal",
+ callback = function(task)
+ local logger = require "rspamd_logger"
+ local function rcpt_done_cb(err, data, conn)
+ logger.errx(task, 'RCPT: got reply: %s, error: %s', data, err)
+ conn:close()
+ end
+ local function rcpt_cb(err, conn)
+ logger.errx(task, 'written rcpt, error: %s', err)
+ conn:add_read(rcpt_done_cb, '\r\n')
+ end
+ local function from_done_cb(err, data, conn)
+ logger.errx(task, 'FROM: got reply: %s, error: %s', data, err)
+ conn:add_write(rcpt_cb, 'RCPT TO: <test@yandex.ru>\r\n')
+ end
+ local function from_cb(err, conn)
+ logger.errx(task, 'written from, error: %s', err)
+ conn:add_read(from_done_cb, '\r\n')
+ end
+ local function hello_done_cb(err, data, conn)
+ logger.errx(task, 'HELO: got reply: %s, error: %s', data, err)
+ conn:add_write(from_cb, 'MAIL FROM: <>\r\n')
+ end
+ local function hello_cb(err, conn)
+ logger.errx(task, 'written hello, error: %s', err)
+ conn:add_read(hello_done_cb, '\r\n')
+ end
+ local function init_cb(err, data, conn)
+ logger.errx(task, 'got reply: %s, error: %s', data, err)
+ conn:add_write(hello_cb, 'HELO example.com\r\n')
+ end
+ tcp.request{
+ task = task,
+ callback = init_cb,
+ stop_pattern = '\r\n',
+ host = 'mx.yandex.ru',
+ port = 25
+ }
+ end,
+ priority = 10,
+})
+ */
+
+LUA_FUNCTION_DEF(tcp, request);
+
+/***
+ * @function rspamd_tcp.connect_sync()
+ *
+ * Creates pseudo-synchronous TCP connection.
+ * Each method of the connection requiring IO, becomes a yielding point,
+ * i.e. current thread Lua thread is get suspended and resumes as soon as IO is done
+ *
+ * This class represents low-level API, using of "lua_tcp_sync" module is recommended.
+ *
+ * @example
+
+local rspamd_tcp = require "rspamd_tcp"
+local logger = require "rspamd_logger"
+
+local function http_simple_tcp_symbol(task)
+
+ local err
+ local is_ok, connection = rspamd_tcp.connect_sync {
+ task = task,
+ host = '127.0.0.1',
+ timeout = 20,
+ port = 18080,
+ ssl = false, -- If SSL connection is needed
+ ssl_verify = true, -- set to false if verify is not needed
+ }
+
+ is_ok, err = connection:write('GET /request_sync HTTP/1.1\r\nConnection: keep-alive\r\n\r\n')
+
+ logger.errx(task, 'write %1, %2', is_ok, err)
+ if not is_ok then
+ logger.errx(task, 'write error: %1', err)
+ end
+
+ local data
+ is_ok, data = connection:read_once();
+
+ logger.errx(task, 'read_once: is_ok: %1, data: %2', is_ok, data)
+
+ is_ok, err = connection:write("POST /request2 HTTP/1.1\r\n\r\n")
+ logger.errx(task, 'write[2] %1, %2', is_ok, err)
+
+ is_ok, data = connection:read_once();
+ logger.errx(task, 'read_once[2]: is_ok %1, data: %2', is_ok, data)
+
+ connection:close()
+end
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE_TCP_TEST',
+ score = 1.0,
+ callback = http_simple_tcp_symbol,
+ no_squeeze = true
+})
+ *
+ */
+LUA_FUNCTION_DEF(tcp, connect_sync);
+
+/***
+ * @method tcp:close()
+ *
+ * Closes TCP connection
+ */
+LUA_FUNCTION_DEF(tcp, close);
+
+/***
+ * @method tcp:add_read(callback, [pattern])
+ *
+ * Adds new read event to the tcp connection
+ * @param {function} callback to be called when data is read
+ * @param {string} pattern optional stop pattern
+ */
+LUA_FUNCTION_DEF(tcp, add_read);
+
+/***
+ * @method tcp:add_write(callback, data)
+ *
+ * Adds new write event to the tcp connection
+ * @param {function} optional callback to be called when data is completely written
+ * @param {table/string/text} data to send to a remote server
+ */
+LUA_FUNCTION_DEF(tcp, add_write);
+
+/***
+ * @method tcp:shift_callback()
+ *
+ * Shifts the current callback and go to the next one (if any)
+ */
+LUA_FUNCTION_DEF(tcp, shift_callback);
+
+/***
+ * @method tcp:starttls([no_verify])
+ *
+ * Starts tls connection
+ * @param {boolean} no_verify used to skip ssl verification
+ */
+LUA_FUNCTION_DEF(tcp, starttls);
+
+static const struct luaL_reg tcp_libf[] = {
+ LUA_INTERFACE_DEF(tcp, request),
+ {"new", lua_tcp_request},
+ {"connect", lua_tcp_request},
+ {"connect_sync", lua_tcp_connect_sync},
+ {NULL, NULL}};
+
+static const struct luaL_reg tcp_libm[] = {
+ LUA_INTERFACE_DEF(tcp, close),
+ LUA_INTERFACE_DEF(tcp, add_read),
+ LUA_INTERFACE_DEF(tcp, add_write),
+ LUA_INTERFACE_DEF(tcp, shift_callback),
+ LUA_INTERFACE_DEF(tcp, starttls),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/***
+ * @method tcp:close()
+ *
+ * Closes TCP connection
+ */
+LUA_FUNCTION_DEF(tcp_sync, close);
+
+/***
+ * @method read_once()
+ *
+ * Performs one read operation. If syscall returned with EAGAIN/EINT,
+ * restarts the operation, so it always returns either data or error.
+ */
+LUA_FUNCTION_DEF(tcp_sync, read_once);
+
+/***
+ * @method eof()
+ *
+ * True if last IO operation ended with EOF, i.e. endpoint closed connection
+ */
+LUA_FUNCTION_DEF(tcp_sync, eof);
+
+/***
+ * @method shutdown()
+ *
+ * Half-shutdown TCP connection
+ */
+LUA_FUNCTION_DEF(tcp_sync, shutdown);
+
+/***
+ * @method write()
+ *
+ * Writes data into the stream. If syscall returned with EAGAIN/EINT
+ * restarts the operation. If performs write() until all the passed
+ * data is written completely.
+ */
+LUA_FUNCTION_DEF(tcp_sync, write);
+
+LUA_FUNCTION_DEF(tcp_sync, gc);
+
+static void lua_tcp_sync_session_dtor(gpointer ud);
+
+static const struct luaL_reg tcp_sync_libm[] = {
+ LUA_INTERFACE_DEF(tcp_sync, close),
+ LUA_INTERFACE_DEF(tcp_sync, read_once),
+ LUA_INTERFACE_DEF(tcp_sync, write),
+ LUA_INTERFACE_DEF(tcp_sync, eof),
+ LUA_INTERFACE_DEF(tcp_sync, shutdown),
+ {"__gc", lua_tcp_sync_gc},
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+struct lua_tcp_read_handler {
+ gchar *stop_pattern;
+ guint plen;
+ gint cbref;
+};
+
+struct lua_tcp_write_handler {
+ struct iovec *iov;
+ guint iovlen;
+ gint cbref;
+ gsize pos;
+ gsize total_bytes;
+};
+
+enum lua_tcp_handler_type {
+ LUA_WANT_WRITE = 0,
+ LUA_WANT_READ,
+ LUA_WANT_CONNECT
+};
+
+struct lua_tcp_handler {
+ union {
+ struct lua_tcp_read_handler r;
+ struct lua_tcp_write_handler w;
+ } h;
+ enum lua_tcp_handler_type type;
+};
+
+struct lua_tcp_dtor {
+ rspamd_mempool_destruct_t dtor;
+ void *data;
+ struct lua_tcp_dtor *next;
+};
+
+#define LUA_TCP_FLAG_PARTIAL (1u << 0u)
+#define LUA_TCP_FLAG_SHUTDOWN (1u << 2u)
+#define LUA_TCP_FLAG_CONNECTED (1u << 3u)
+#define LUA_TCP_FLAG_FINISHED (1u << 4u)
+#define LUA_TCP_FLAG_SYNC (1u << 5u)
+#define LUA_TCP_FLAG_RESOLVED (1u << 6u)
+#define LUA_TCP_FLAG_SSL (1u << 7u)
+#define LUA_TCP_FLAG_SSL_NOVERIFY (1u << 8u)
+
+#undef TCP_DEBUG_REFS
+#ifdef TCP_DEBUG_REFS
+#define TCP_RETAIN(x) \
+ do { \
+ msg_info("retain ref %p, refcount: %d", (x), (x)->ref.refcount); \
+ REF_RETAIN(x); \
+ } while (0)
+
+#define TCP_RELEASE(x) \
+ do { \
+ msg_info("release ref %p, refcount: %d", (x), (x)->ref.refcount); \
+ REF_RELEASE(x); \
+ } while (0)
+#else
+#define TCP_RETAIN(x) REF_RETAIN(x)
+#define TCP_RELEASE(x) REF_RELEASE(x)
+#endif
+
+struct lua_tcp_cbdata {
+ struct rspamd_async_session *session;
+ struct rspamd_async_event *async_ev;
+ struct ev_loop *event_loop;
+ rspamd_inet_addr_t *addr;
+ GByteArray *in;
+ GQueue *handlers;
+ gint fd;
+ gint connect_cb;
+ guint port;
+ guint flags;
+ gchar tag[7];
+ struct rspamd_io_ev ev;
+ struct lua_tcp_dtor *dtors;
+ ref_entry_t ref;
+ struct rspamd_task *task;
+ struct rspamd_symcache_dynamic_item *item;
+ struct thread_entry *thread;
+ struct rspamd_config *cfg;
+ struct rspamd_ssl_connection *ssl_conn;
+ gchar *hostname;
+ struct upstream *up;
+ gboolean eof;
+};
+
+#define IS_SYNC(c) (((c)->flags & LUA_TCP_FLAG_SYNC) != 0)
+
+#define msg_debug_tcp(...) rspamd_conditional_debug_fast(NULL, cbd->addr, \
+ rspamd_lua_tcp_log_id, "lua_tcp", cbd->tag, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(lua_tcp)
+
+static void lua_tcp_handler(int fd, short what, gpointer ud);
+static void lua_tcp_plan_handler_event(struct lua_tcp_cbdata *cbd,
+ gboolean can_read, gboolean can_write);
+static void lua_tcp_unregister_event(struct lua_tcp_cbdata *cbd);
+
+static void
+lua_tcp_void_finalyser(gpointer arg)
+{
+}
+
+static const gdouble default_tcp_timeout = 5.0;
+
+static struct rspamd_dns_resolver *
+lua_tcp_global_resolver(struct ev_loop *ev_base,
+ struct rspamd_config *cfg)
+{
+ static struct rspamd_dns_resolver *global_resolver;
+
+ if (cfg && cfg->dns_resolver) {
+ return cfg->dns_resolver;
+ }
+
+ if (global_resolver == NULL) {
+ global_resolver = rspamd_dns_resolver_init(NULL, ev_base, cfg);
+ }
+
+ return global_resolver;
+}
+
+static gboolean
+lua_tcp_shift_handler(struct lua_tcp_cbdata *cbd)
+{
+ struct lua_tcp_handler *hdl;
+
+ hdl = g_queue_pop_head(cbd->handlers);
+
+ if (hdl == NULL) {
+ /* We are done */
+ return FALSE;
+ }
+
+ if (hdl->type == LUA_WANT_READ) {
+ msg_debug_tcp("switch from read handler %d", hdl->h.r.cbref);
+ if (hdl->h.r.cbref && hdl->h.r.cbref != -1) {
+ luaL_unref(cbd->cfg->lua_state, LUA_REGISTRYINDEX, hdl->h.r.cbref);
+ }
+
+ if (hdl->h.r.stop_pattern) {
+ g_free(hdl->h.r.stop_pattern);
+ }
+ }
+ else if (hdl->type == LUA_WANT_WRITE) {
+ msg_debug_tcp("switch from write handler %d", hdl->h.r.cbref);
+ if (hdl->h.w.cbref && hdl->h.w.cbref != -1) {
+ luaL_unref(cbd->cfg->lua_state, LUA_REGISTRYINDEX, hdl->h.w.cbref);
+ }
+
+ if (hdl->h.w.iov) {
+ g_free(hdl->h.w.iov);
+ }
+ }
+ else {
+ msg_debug_tcp("removing connect handler");
+ /* LUA_WANT_CONNECT: it doesn't allocate anything, nothing to do here */
+ }
+
+ g_free(hdl);
+
+ return TRUE;
+}
+
+static void
+lua_tcp_fin(gpointer arg)
+{
+ struct lua_tcp_cbdata *cbd = (struct lua_tcp_cbdata *) arg;
+ struct lua_tcp_dtor *dtor, *dttmp;
+
+ if (IS_SYNC(cbd) && cbd->task) {
+ /*
+ pointer is now becoming invalid, we should remove registered destructor,
+ all the necessary steps are done here
+ */
+ rspamd_mempool_replace_destructor(cbd->task->task_pool,
+ lua_tcp_sync_session_dtor, cbd, NULL);
+ }
+
+ msg_debug_tcp("finishing TCP %s connection", IS_SYNC(cbd) ? "sync" : "async");
+
+ if (cbd->connect_cb != -1) {
+ luaL_unref(cbd->cfg->lua_state, LUA_REGISTRYINDEX, cbd->connect_cb);
+ }
+
+ if (cbd->ssl_conn) {
+ /* TODO: postpone close in case ssl is used ! */
+ rspamd_ssl_connection_free(cbd->ssl_conn);
+ }
+
+ if (cbd->fd != -1) {
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+ close(cbd->fd);
+ cbd->fd = -1;
+ }
+
+ if (cbd->addr) {
+ rspamd_inet_address_free(cbd->addr);
+ }
+
+ if (cbd->up) {
+ rspamd_upstream_unref(cbd->up);
+ }
+
+ while (lua_tcp_shift_handler(cbd)) {}
+ g_queue_free(cbd->handlers);
+
+ LL_FOREACH_SAFE(cbd->dtors, dtor, dttmp)
+ {
+ dtor->dtor(dtor->data);
+ g_free(dtor);
+ }
+
+ g_byte_array_unref(cbd->in);
+ g_free(cbd->hostname);
+ g_free(cbd);
+}
+
+static struct lua_tcp_cbdata *
+lua_check_tcp(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{tcp}");
+ luaL_argcheck(L, ud != NULL, pos, "'tcp' expected");
+ return ud ? *((struct lua_tcp_cbdata **) ud) : NULL;
+}
+
+static void
+lua_tcp_maybe_free(struct lua_tcp_cbdata *cbd)
+{
+ if (IS_SYNC(cbd)) {
+ /*
+ * in this mode, we don't remove object, we only remove the event
+ * Object is owned by lua and will be destroyed on __gc()
+ */
+
+ if (cbd->item) {
+ rspamd_symcache_item_async_dec_check(cbd->task, cbd->item, M);
+ cbd->item = NULL;
+ }
+
+ if (cbd->async_ev) {
+ rspamd_session_remove_event(cbd->session, lua_tcp_void_finalyser, cbd);
+ }
+
+ cbd->async_ev = NULL;
+ }
+ else {
+ if (cbd->item) {
+ rspamd_symcache_item_async_dec_check(cbd->task, cbd->item, M);
+ cbd->item = NULL;
+ }
+
+ if (cbd->async_ev) {
+ rspamd_session_remove_event(cbd->session, lua_tcp_fin, cbd);
+ }
+ else {
+ lua_tcp_fin(cbd);
+ }
+ }
+}
+
+#ifdef __GNUC__
+static void
+lua_tcp_push_error(struct lua_tcp_cbdata *cbd, gboolean is_fatal,
+ const char *err, ...) __attribute__((format(printf, 3, 4)));
+#endif
+
+static void lua_tcp_resume_thread_error_argp(struct lua_tcp_cbdata *cbd, const gchar *error, va_list argp);
+
+static void
+lua_tcp_push_error(struct lua_tcp_cbdata *cbd, gboolean is_fatal,
+ const char *err, ...)
+{
+ va_list ap, ap_copy;
+ struct lua_tcp_cbdata **pcbd;
+ struct lua_tcp_handler *hdl;
+ gint cbref, top;
+ struct lua_callback_state cbs;
+ lua_State *L;
+ gboolean callback_called = FALSE;
+
+ if (is_fatal && cbd->up) {
+ rspamd_upstream_fail(cbd->up, false, err);
+ }
+
+ if (cbd->thread) {
+ va_start(ap, err);
+ lua_tcp_resume_thread_error_argp(cbd, err, ap);
+ va_end(ap);
+
+ return;
+ }
+
+ lua_thread_pool_prepare_callback(cbd->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ va_start(ap, err);
+
+ for (;;) {
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ if (hdl == NULL) {
+ break;
+ }
+
+ if (hdl->type == LUA_WANT_READ) {
+ cbref = hdl->h.r.cbref;
+ }
+ else {
+ cbref = hdl->h.w.cbref;
+ }
+
+ if (cbref != -1) {
+ top = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbref);
+
+ /* Error message */
+ va_copy(ap_copy, ap);
+ lua_pushvfstring(L, err, ap_copy);
+ va_end(ap_copy);
+
+ /* Body */
+ lua_pushnil(L);
+ /* Connection */
+ pcbd = lua_newuserdata(L, sizeof(*pcbd));
+ *pcbd = cbd;
+ rspamd_lua_setclass(L, "rspamd{tcp}", -1);
+ TCP_RETAIN(cbd);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 3, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+
+ TCP_RELEASE(cbd);
+
+ if ((cbd->flags & (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) ==
+ (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+
+ callback_called = TRUE;
+ }
+
+ if (!is_fatal) {
+ if (callback_called) {
+ /* Stop on the first callback found */
+ break;
+ }
+ else {
+ /* Shift to another callback to inform about non fatal error */
+ msg_debug_tcp("non fatal error find matching callback");
+ lua_tcp_shift_handler(cbd);
+ continue;
+ }
+ }
+ else {
+ msg_debug_tcp("fatal error rollback all handlers");
+ lua_tcp_shift_handler(cbd);
+ }
+ }
+
+ va_end(ap);
+
+ lua_thread_pool_restore_callback(&cbs);
+}
+
+static void lua_tcp_resume_thread(struct lua_tcp_cbdata *cbd, const guint8 *str, gsize len);
+
+static void
+lua_tcp_push_data(struct lua_tcp_cbdata *cbd, const guint8 *str, gsize len)
+{
+ struct rspamd_lua_text *t;
+ struct lua_tcp_cbdata **pcbd;
+ struct lua_tcp_handler *hdl;
+ gint cbref, arg_cnt, top;
+ struct lua_callback_state cbs;
+ lua_State *L;
+
+ if (cbd->thread) {
+ lua_tcp_resume_thread(cbd, str, len);
+ return;
+ }
+
+ lua_thread_pool_prepare_callback(cbd->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ g_assert(hdl != NULL);
+
+ if (hdl->type == LUA_WANT_READ) {
+ cbref = hdl->h.r.cbref;
+ }
+ else {
+ cbref = hdl->h.w.cbref;
+ }
+
+ if (cbref != -1) {
+ top = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbref);
+ /* Error */
+ lua_pushnil(L);
+ /* Body */
+
+ if (hdl->type == LUA_WANT_READ) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = (const gchar *) str;
+ t->len = len;
+ t->flags = 0;
+ arg_cnt = 3;
+ }
+ else {
+ arg_cnt = 2;
+ }
+ /* Connection */
+ pcbd = lua_newuserdata(L, sizeof(*pcbd));
+ *pcbd = cbd;
+ rspamd_lua_setclass(L, "rspamd{tcp}", -1);
+
+ TCP_RETAIN(cbd);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, arg_cnt, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+ TCP_RELEASE(cbd);
+
+ if ((cbd->flags & (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) ==
+ (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+ }
+
+ lua_thread_pool_restore_callback(&cbs);
+}
+
+static void
+lua_tcp_resume_thread_error_argp(struct lua_tcp_cbdata *cbd, const gchar *error, va_list argp)
+{
+ struct thread_entry *thread = cbd->thread;
+ lua_State *L = thread->lua_state;
+
+ lua_pushboolean(L, FALSE);
+ lua_pushvfstring(L, error, argp);
+
+ lua_tcp_shift_handler(cbd);
+ // lua_tcp_unregister_event (cbd);
+ lua_thread_pool_set_running_entry(cbd->cfg->lua_thread_pool, cbd->thread);
+ lua_thread_resume(thread, 2);
+ TCP_RELEASE(cbd);
+}
+
+static void
+lua_tcp_resume_thread(struct lua_tcp_cbdata *cbd, const guint8 *str, gsize len)
+{
+ /*
+ * typical call returns:
+ *
+ * read:
+ * error:
+ * (nil, error message)
+ * got data:
+ * (true, data)
+ * write/connect:
+ * error:
+ * (nil, error message)
+ * wrote
+ * (true)
+ */
+
+ lua_State *L = cbd->thread->lua_state;
+ struct lua_tcp_handler *hdl;
+
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ lua_pushboolean(L, TRUE);
+ if (hdl->type == LUA_WANT_READ) {
+ lua_pushlstring(L, str, len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ lua_tcp_shift_handler(cbd);
+ lua_thread_pool_set_running_entry(cbd->cfg->lua_thread_pool,
+ cbd->thread);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ lua_thread_resume(cbd->thread, 2);
+
+ TCP_RELEASE(cbd);
+}
+
+static void
+lua_tcp_plan_read(struct lua_tcp_cbdata *cbd)
+{
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev, EV_READ);
+}
+
+static void
+lua_tcp_connect_helper(struct lua_tcp_cbdata *cbd)
+{
+ /* This is used for sync mode only */
+ lua_State *L = cbd->thread->lua_state;
+
+ struct lua_tcp_cbdata **pcbd;
+
+ lua_pushboolean(L, TRUE);
+
+ lua_thread_pool_set_running_entry(cbd->cfg->lua_thread_pool, cbd->thread);
+ pcbd = lua_newuserdata(L, sizeof(*pcbd));
+ *pcbd = cbd;
+ rspamd_lua_setclass(L, "rspamd{tcp_sync}", -1);
+ msg_debug_tcp("tcp connected");
+
+ lua_tcp_shift_handler(cbd);
+
+ // lua_tcp_unregister_event (cbd);
+ lua_thread_resume(cbd->thread, 2);
+ TCP_RELEASE(cbd);
+}
+
+static void
+lua_tcp_write_helper(struct lua_tcp_cbdata *cbd)
+{
+ struct iovec *start;
+ guint niov, i;
+ gint flags = 0;
+ bool allocated_iov = false;
+ gsize remain;
+ gssize r;
+ struct iovec *cur_iov;
+ struct lua_tcp_handler *hdl;
+ struct lua_tcp_write_handler *wh;
+ struct msghdr msg;
+
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ g_assert(hdl != NULL && hdl->type == LUA_WANT_WRITE);
+ wh = &hdl->h.w;
+
+ if (wh->pos == wh->total_bytes) {
+ goto call_finish_handler;
+ }
+
+ start = &wh->iov[0];
+ niov = wh->iovlen;
+ remain = wh->pos;
+ /* We know that niov is small enough for that */
+
+ if (niov < 1024) {
+ cur_iov = g_alloca(niov * sizeof(struct iovec));
+ }
+ else {
+ cur_iov = g_malloc0(niov * sizeof(struct iovec));
+ allocated_iov = true;
+ }
+
+ memcpy(cur_iov, wh->iov, niov * sizeof(struct iovec));
+
+ for (i = 0; i < wh->iovlen && remain > 0; i++) {
+ /* Find out the first iov required */
+ start = &cur_iov[i];
+ if (start->iov_len <= remain) {
+ remain -= start->iov_len;
+ start = &cur_iov[i + 1];
+ niov--;
+ }
+ else {
+ start->iov_base = (void *) ((char *) start->iov_base + remain);
+ start->iov_len -= remain;
+ remain = 0;
+ }
+ }
+
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = start;
+ msg.msg_iovlen = MIN(IOV_MAX, niov);
+ g_assert(niov > 0);
+#ifdef MSG_NOSIGNAL
+ flags = MSG_NOSIGNAL;
+#endif
+
+ msg_debug_tcp("want write %d io vectors of %d", (int) msg.msg_iovlen,
+ (int) niov);
+
+ if (cbd->ssl_conn) {
+ r = rspamd_ssl_writev(cbd->ssl_conn, msg.msg_iov, msg.msg_iovlen);
+ }
+ else {
+ r = sendmsg(cbd->fd, &msg, flags);
+ }
+
+ if (allocated_iov) {
+ g_free(cur_iov);
+ }
+
+ if (r == -1) {
+ if (!(cbd->ssl_conn)) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
+ msg_debug_tcp("got temporary failure, retry write");
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+ return;
+ }
+ else {
+ lua_tcp_push_error(cbd, TRUE,
+ "IO write error while trying to write %d bytes: %s",
+ (gint) remain, strerror(errno));
+
+ msg_debug_tcp("write error, terminate connection");
+ TCP_RELEASE(cbd);
+ }
+ }
+
+ return;
+ }
+ else {
+ wh->pos += r;
+ }
+
+ msg_debug_tcp("written %z bytes: %z/%z", r,
+ wh->pos, wh->total_bytes);
+
+ if (wh->pos >= wh->total_bytes) {
+ goto call_finish_handler;
+ }
+ else {
+ /* Want to write more */
+ if (r > 0) {
+ /* XXX: special case: we know that we want to write more data
+ * than it is available in iov function.
+ *
+ * Hence, we need to check if we can write more at some point...
+ */
+ lua_tcp_write_helper(cbd);
+ }
+ }
+
+ return;
+
+call_finish_handler:
+
+ msg_debug_tcp("finishing TCP write, calling TCP handler");
+
+ if ((cbd->flags & LUA_TCP_FLAG_SHUTDOWN)) {
+ /* Half close the connection */
+ shutdown(cbd->fd, SHUT_WR);
+ cbd->flags &= ~LUA_TCP_FLAG_SHUTDOWN;
+ }
+
+ if (cbd->up) {
+ rspamd_upstream_ok(cbd->up);
+ }
+
+ lua_tcp_push_data(cbd, NULL, 0);
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_shift_handler(cbd);
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+ }
+}
+
+static gboolean
+lua_tcp_process_read_handler(struct lua_tcp_cbdata *cbd,
+ struct lua_tcp_read_handler *rh, gboolean eof)
+{
+ guint slen;
+ goffset pos;
+
+ if (rh->stop_pattern) {
+ slen = rh->plen;
+
+ if (cbd->in->len >= slen) {
+ if ((pos = rspamd_substring_search(cbd->in->data, cbd->in->len,
+ rh->stop_pattern, slen)) != -1) {
+ msg_debug_tcp("found TCP stop pattern");
+ lua_tcp_push_data(cbd, cbd->in->data, pos);
+
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_shift_handler(cbd);
+ }
+ if (pos + slen < cbd->in->len) {
+ /* We have a leftover */
+ memmove(cbd->in->data, cbd->in->data + pos + slen,
+ cbd->in->len - (pos + slen));
+ cbd->in->len = cbd->in->len - (pos + slen);
+ }
+ else {
+ cbd->in->len = 0;
+ }
+
+ return TRUE;
+ }
+ else {
+ /* Plan new read */
+ msg_debug_tcp("NOT found TCP stop pattern");
+
+ if (!cbd->eof) {
+ lua_tcp_plan_read(cbd);
+ }
+ else {
+ /* Got session finished but no stop pattern */
+ lua_tcp_push_error(cbd, TRUE,
+ "IO read error: connection terminated");
+ }
+ }
+ }
+ }
+ else {
+ msg_debug_tcp("read TCP partial data %d bytes", cbd->in->len);
+ slen = cbd->in->len;
+
+ /* we have eaten all the data, handler should not know that there is something */
+ cbd->in->len = 0;
+ lua_tcp_push_data(cbd, cbd->in->data, slen);
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_shift_handler(cbd);
+ }
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static void
+lua_tcp_process_read(struct lua_tcp_cbdata *cbd,
+ guchar *in, gssize r)
+{
+ struct lua_tcp_handler *hdl;
+ struct lua_tcp_read_handler *rh;
+
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ g_assert(hdl != NULL && hdl->type == LUA_WANT_READ);
+ rh = &hdl->h.r;
+
+ if (r > 0) {
+ if (cbd->flags & LUA_TCP_FLAG_PARTIAL) {
+ lua_tcp_push_data(cbd, in, r);
+ /* Plan next event */
+ lua_tcp_plan_read(cbd);
+ }
+ else {
+ g_byte_array_append(cbd->in, in, r);
+
+ if (!lua_tcp_process_read_handler(cbd, rh, FALSE)) {
+ /* Plan more read */
+ lua_tcp_plan_read(cbd);
+ }
+ else {
+ /* Go towards the next handler */
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+ }
+ }
+ }
+ }
+ else if (r == 0) {
+ /* EOF */
+ cbd->eof = TRUE;
+ if (cbd->in->len > 0) {
+ /* We have some data to process */
+ lua_tcp_process_read_handler(cbd, rh, TRUE);
+ }
+ else {
+ lua_tcp_push_error(cbd, TRUE, "IO read error: connection terminated");
+
+ if ((cbd->flags & LUA_TCP_FLAG_FINISHED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+ }
+
+ lua_tcp_plan_handler_event(cbd, FALSE, FALSE);
+ }
+ else {
+ /* An error occurred */
+ if (errno == EAGAIN || errno == EINTR) {
+ /* Restart call */
+ lua_tcp_plan_read(cbd);
+
+ return;
+ }
+
+ /* Fatal error */
+ cbd->eof = TRUE;
+ if (cbd->in->len > 0) {
+ /* We have some data to process */
+ lua_tcp_process_read_handler(cbd, rh, TRUE);
+ }
+ else {
+ lua_tcp_push_error(cbd, TRUE,
+ "IO read error while trying to read data: %s",
+ strerror(errno));
+
+ if ((cbd->flags & LUA_TCP_FLAG_FINISHED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+ }
+
+ lua_tcp_plan_handler_event(cbd, FALSE, FALSE);
+ }
+}
+
+static void
+lua_tcp_handler(int fd, short what, gpointer ud)
+{
+ struct lua_tcp_cbdata *cbd = ud;
+ guchar inbuf[8192];
+ gssize r;
+ gint so_error = 0;
+ socklen_t so_len = sizeof(so_error);
+ struct lua_callback_state cbs;
+ lua_State *L;
+ enum lua_tcp_handler_type event_type;
+ TCP_RETAIN(cbd);
+
+ msg_debug_tcp("processed TCP event: %d", what);
+
+ struct lua_tcp_handler *rh = g_queue_peek_head(cbd->handlers);
+ event_type = rh->type;
+
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+
+ if (what == EV_READ) {
+ if (cbd->ssl_conn) {
+ r = rspamd_ssl_read(cbd->ssl_conn, inbuf, sizeof(inbuf));
+ }
+ else {
+ r = read(cbd->fd, inbuf, sizeof(inbuf));
+ }
+
+ lua_tcp_process_read(cbd, inbuf, r);
+ }
+ else if (what == EV_WRITE) {
+
+ if (!(cbd->flags & LUA_TCP_FLAG_CONNECTED)) {
+ if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &so_error, &so_len) == -1) {
+ lua_tcp_push_error(cbd, TRUE, "Cannot get socket error: %s",
+ strerror(errno));
+ TCP_RELEASE(cbd);
+ goto out;
+ }
+ else if (so_error != 0) {
+ lua_tcp_push_error(cbd, TRUE, "Socket error detected: %s",
+ strerror(so_error));
+ TCP_RELEASE(cbd);
+ goto out;
+ }
+ else {
+ cbd->flags |= LUA_TCP_FLAG_CONNECTED;
+
+ if (cbd->connect_cb != -1) {
+ struct lua_tcp_cbdata **pcbd;
+ gint top;
+
+ lua_thread_pool_prepare_callback(cbd->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ top = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->connect_cb);
+ pcbd = lua_newuserdata(L, sizeof(*pcbd));
+ *pcbd = cbd;
+ TCP_RETAIN(cbd);
+ rspamd_lua_setclass(L, "rspamd{tcp}", -1);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 1, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+ TCP_RELEASE(cbd);
+ lua_thread_pool_restore_callback(&cbs);
+
+ if ((cbd->flags & (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) ==
+ (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+ }
+ }
+ }
+
+ if (event_type == LUA_WANT_WRITE) {
+ lua_tcp_write_helper(cbd);
+ }
+ else if (event_type == LUA_WANT_CONNECT) {
+ lua_tcp_connect_helper(cbd);
+ }
+ else {
+ g_assert_not_reached();
+ }
+
+ if ((cbd->flags & (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) ==
+ (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+ }
+ else {
+ lua_tcp_push_error(cbd, TRUE, "IO timeout");
+ TCP_RELEASE(cbd);
+ }
+
+out:
+ TCP_RELEASE(cbd);
+}
+
+static void
+lua_tcp_plan_handler_event(struct lua_tcp_cbdata *cbd, gboolean can_read,
+ gboolean can_write)
+{
+ struct lua_tcp_handler *hdl;
+
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ if (hdl == NULL) {
+ if (!(cbd->flags & LUA_TCP_FLAG_FINISHED)) {
+ /* We are finished with a connection */
+ msg_debug_tcp("no handlers left, finish session");
+ cbd->flags |= LUA_TCP_FLAG_FINISHED;
+ TCP_RELEASE(cbd);
+ }
+ }
+ else {
+ if (hdl->type == LUA_WANT_READ) {
+
+ /* We need to check if we have some leftover in the buffer */
+ if (cbd->in->len > 0) {
+ msg_debug_tcp("process read buffer leftover");
+ if (lua_tcp_process_read_handler(cbd, &hdl->h.r, FALSE)) {
+ if (!IS_SYNC(cbd)) {
+ /* We can go to the next handler */
+ lua_tcp_plan_handler_event(cbd, can_read, can_write);
+ }
+ }
+ }
+ else {
+ if (can_read) {
+ /* We need to plan a new event */
+ msg_debug_tcp("plan new read");
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev,
+ EV_READ);
+ }
+ else {
+ /* Cannot read more */
+ msg_debug_tcp("cannot read more");
+ lua_tcp_push_error(cbd, FALSE, "EOF, cannot read more data");
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_shift_handler(cbd);
+ lua_tcp_plan_handler_event(cbd, can_read, can_write);
+ }
+ }
+ }
+ }
+ else if (hdl->type == LUA_WANT_WRITE) {
+ /*
+ * We need to plan write event if there is something in the
+ * write request
+ */
+
+ if (hdl->h.w.pos < hdl->h.w.total_bytes) {
+ msg_debug_tcp("plan new write");
+ if (can_write) {
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev,
+ EV_WRITE);
+ }
+ else {
+ /* Cannot write more */
+ lua_tcp_push_error(cbd, FALSE, "EOF, cannot write more data");
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_shift_handler(cbd);
+ lua_tcp_plan_handler_event(cbd, can_read, can_write);
+ }
+ }
+ }
+ else {
+ /* We shouldn't have empty write handlers */
+ g_assert_not_reached();
+ }
+ }
+ else { /* LUA_WANT_CONNECT */
+ msg_debug_tcp("plan new connect");
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev,
+ EV_WRITE);
+ }
+ }
+}
+
+static gboolean
+lua_tcp_register_event(struct lua_tcp_cbdata *cbd)
+{
+ if (cbd->session) {
+ event_finalizer_t fin = IS_SYNC(cbd) ? lua_tcp_void_finalyser : lua_tcp_fin;
+
+ if (cbd->item) {
+ cbd->async_ev = rspamd_session_add_event_full(cbd->session, fin, cbd, M,
+ rspamd_symcache_dyn_item_name(cbd->task, cbd->item));
+ }
+ else {
+ cbd->async_ev = rspamd_session_add_event(cbd->session, fin, cbd, M);
+ }
+
+ if (!cbd->async_ev) {
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+static void
+lua_tcp_register_watcher(struct lua_tcp_cbdata *cbd)
+{
+ if (cbd->item && cbd->task) {
+ rspamd_symcache_item_async_inc(cbd->task, cbd->item, M);
+ }
+}
+
+static void
+lua_tcp_ssl_on_error(gpointer ud, GError *err)
+{
+ struct lua_tcp_cbdata *cbd = (struct lua_tcp_cbdata *) ud;
+
+ if (err) {
+ lua_tcp_push_error(cbd, TRUE, "ssl error: %s", err->message);
+ }
+ else {
+ lua_tcp_push_error(cbd, TRUE, "ssl error: unknown error");
+ }
+
+ TCP_RELEASE(cbd);
+}
+
+static gboolean
+lua_tcp_make_connection(struct lua_tcp_cbdata *cbd)
+{
+ int fd;
+
+ rspamd_inet_address_set_port(cbd->addr, cbd->port);
+ fd = rspamd_inet_address_connect(cbd->addr, SOCK_STREAM, TRUE);
+
+ if (fd == -1) {
+ if (cbd->session) {
+ rspamd_mempool_t *pool = rspamd_session_mempool(cbd->session);
+ msg_info_pool("cannot connect to %s (%s): %s",
+ rspamd_inet_address_to_string(cbd->addr),
+ cbd->hostname,
+ strerror(errno));
+ }
+ else {
+ msg_info("cannot connect to %s (%s): %s",
+ rspamd_inet_address_to_string(cbd->addr),
+ cbd->hostname,
+ strerror(errno));
+ }
+
+ return FALSE;
+ }
+
+ cbd->fd = fd;
+
+#if 0
+ if (!(cbd->flags & LUA_TCP_FLAG_RESOLVED)) {
+ /* We come here without resolving, so we need to add a watcher */
+ lua_tcp_register_watcher (cbd);
+ }
+ else {
+ cbd->flags |= LUA_TCP_FLAG_RESOLVED;
+ }
+#endif
+
+ if (cbd->flags & LUA_TCP_FLAG_SSL) {
+ gpointer ssl_ctx;
+ gboolean verify_peer;
+
+ if (cbd->flags & LUA_TCP_FLAG_SSL_NOVERIFY) {
+ ssl_ctx = cbd->cfg->libs_ctx->ssl_ctx_noverify;
+ verify_peer = FALSE;
+ }
+ else {
+ ssl_ctx = cbd->cfg->libs_ctx->ssl_ctx;
+ verify_peer = TRUE;
+ }
+
+ cbd->ssl_conn = rspamd_ssl_connection_new(ssl_ctx,
+ cbd->event_loop,
+ verify_peer,
+ cbd->tag);
+
+ if (!rspamd_ssl_connect_fd(cbd->ssl_conn, fd, cbd->hostname, &cbd->ev,
+ cbd->ev.timeout, lua_tcp_handler, lua_tcp_ssl_on_error, cbd)) {
+ lua_tcp_push_error(cbd, TRUE, "ssl connection failed: %s",
+ strerror(errno));
+
+ return FALSE;
+ }
+ else {
+ lua_tcp_register_event(cbd);
+ }
+ }
+ else {
+ rspamd_ev_watcher_init(&cbd->ev, cbd->fd, EV_WRITE,
+ lua_tcp_handler, cbd);
+ lua_tcp_register_event(cbd);
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+ }
+
+
+ return TRUE;
+}
+
+static void
+lua_tcp_dns_handler(struct rdns_reply *reply, gpointer ud)
+{
+ struct lua_tcp_cbdata *cbd = (struct lua_tcp_cbdata *) ud;
+ const struct rdns_request_name *rn;
+
+ if (reply->code != RDNS_RC_NOERROR) {
+ rn = rdns_request_get_name(reply->request, NULL);
+ lua_tcp_push_error(cbd, TRUE, "unable to resolve host: %s",
+ rn->name);
+ TCP_RELEASE(cbd);
+ }
+ else {
+ /*
+ * We set this flag as it means that we have already registered the watcher
+ * when started DNS query
+ */
+ struct rdns_reply_entry *entry;
+
+ DL_FOREACH(reply->entries, entry)
+ {
+ if (entry->type == RDNS_REQUEST_A) {
+ cbd->addr = rspamd_inet_address_new(AF_INET,
+ &entry->content.a.addr);
+ break;
+ }
+ else if (entry->type == RDNS_REQUEST_AAAA) {
+ cbd->addr = rspamd_inet_address_new(AF_INET6,
+ &entry->content.aaa.addr);
+ break;
+ }
+ }
+
+ if (cbd->addr == NULL) {
+ rn = rdns_request_get_name(reply->request, NULL);
+ lua_tcp_push_error(cbd, TRUE, "unable to resolve host: %s; no records with this name",
+ rn->name);
+ TCP_RELEASE(cbd);
+ return;
+ }
+
+ cbd->flags |= LUA_TCP_FLAG_RESOLVED;
+ rspamd_inet_address_set_port(cbd->addr, cbd->port);
+
+ if (!lua_tcp_make_connection(cbd)) {
+ lua_tcp_push_error(cbd, TRUE, "unable to make connection to the host %s",
+ rspamd_inet_address_to_string(cbd->addr));
+ TCP_RELEASE(cbd);
+ }
+ }
+}
+
+static gboolean
+lua_tcp_arg_toiovec(lua_State *L, gint pos, struct lua_tcp_cbdata *cbd,
+ struct iovec *vec)
+{
+ struct rspamd_lua_text *t;
+ gsize len;
+ const gchar *str;
+ struct lua_tcp_dtor *dtor;
+
+ if (lua_type(L, pos) == LUA_TUSERDATA) {
+ t = lua_check_text(L, pos);
+
+ if (t) {
+ vec->iov_base = (void *) t->start;
+ vec->iov_len = t->len;
+
+ if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ /* Steal ownership */
+ t->flags = 0;
+ dtor = g_malloc0(sizeof(*dtor));
+ dtor->dtor = g_free;
+ dtor->data = (void *) t->start;
+ LL_PREPEND(cbd->dtors, dtor);
+ }
+ }
+ else {
+ msg_err("bad userdata argument at position %d", pos);
+ return FALSE;
+ }
+ }
+ else if (lua_type(L, pos) == LUA_TSTRING) {
+ str = luaL_checklstring(L, pos, &len);
+ vec->iov_base = g_malloc(len);
+ dtor = g_malloc0(sizeof(*dtor));
+ dtor->dtor = g_free;
+ dtor->data = vec->iov_base;
+ LL_PREPEND(cbd->dtors, dtor);
+ memcpy(vec->iov_base, str, len);
+ vec->iov_len = len;
+ }
+ else {
+ msg_err("bad argument at position %d", pos);
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+/***
+ * @function rspamd_tcp.request({params})
+ * This function creates and sends TCP request to the specified host and port,
+ * resolves hostname (if needed) and invokes continuation callback upon data received
+ * from the remote peer. This function accepts table of arguments with the following
+ * attributes
+ *
+ * - `task`: rspamd task objects (implies `pool`, `session`, `ev_base` and `resolver` arguments)
+ * - `ev_base`: event base (if no task specified)
+ * - `resolver`: DNS resolver (no task)
+ * - `session`: events session (no task)
+ * - `host`: IP or name of the peer (required)
+ * - `port`: remote port to use
+ * - `data`: a table of strings or `rspamd_text` objects that contains data pieces
+ * - `callback`: continuation function (required)
+ * - `on_connect`: callback called on connection success
+ * - `timeout`: floating point value that specifies timeout for IO operations in **seconds**
+ * - `partial`: boolean flag that specifies that callback should be called on any data portion received
+ * - `stop_pattern`: stop reading on finding a certain pattern (e.g. \r\n.\r\n for smtp)
+ * - `shutdown`: half-close socket after writing (boolean: default false)
+ * - `read`: read response after sending request (boolean: default true)
+ * - `upstream`: optional upstream object that would be used to get an address
+ * @return {boolean} true if request has been sent
+ */
+static gint
+lua_tcp_request(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *host;
+ gchar *stop_pattern = NULL;
+ guint port;
+ gint cbref, tp, conn_cbref = -1;
+ gsize plen = 0;
+ struct ev_loop *event_loop = NULL;
+ struct lua_tcp_cbdata *cbd;
+ struct rspamd_dns_resolver *resolver = NULL;
+ struct rspamd_async_session *session = NULL;
+ struct rspamd_task *task = NULL;
+ struct rspamd_config *cfg = NULL;
+ struct iovec *iov = NULL;
+ struct upstream *up = NULL;
+ guint niov = 0, total_out;
+ guint64 h;
+ gdouble timeout = default_tcp_timeout;
+ gboolean partial = FALSE, do_shutdown = FALSE, do_read = TRUE,
+ ssl = FALSE, ssl_noverify = FALSE;
+
+ if (lua_type(L, 1) == LUA_TTABLE) {
+ lua_pushstring(L, "host");
+ lua_gettable(L, -2);
+ host = luaL_checkstring(L, -1);
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "port");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ port = lua_tointeger(L, -1);
+ }
+ else {
+ /* We assume that it is a unix socket */
+ port = 0;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "callback");
+ lua_gettable(L, -2);
+ if (host == NULL || lua_type(L, -1) != LUA_TFUNCTION) {
+ lua_pop(L, 1);
+ msg_err("tcp request has bad params");
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ cbd = g_malloc0(sizeof(*cbd));
+
+ lua_pushstring(L, "task");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ task = lua_check_task(L, -1);
+ event_loop = task->event_loop;
+ resolver = task->resolver;
+ session = task->s;
+ cfg = task->cfg;
+ }
+ lua_pop(L, 1);
+
+ if (task == NULL) {
+ lua_pushstring(L, "ev_base");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{ev_base}")) {
+ event_loop = *(struct ev_loop **) lua_touserdata(L, -1);
+ }
+ else {
+ g_free(cbd);
+
+ return luaL_error(L, "event loop is required");
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "session");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{session}")) {
+ session = *(struct rspamd_async_session **) lua_touserdata(L, -1);
+ }
+ else {
+ session = NULL;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "config");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{config}")) {
+ cfg = *(struct rspamd_config **) lua_touserdata(L, -1);
+ }
+ else {
+ cfg = NULL;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "resolver");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{resolver}")) {
+ resolver = *(struct rspamd_dns_resolver **) lua_touserdata(L, -1);
+ }
+ else {
+ resolver = lua_tcp_global_resolver(event_loop, cfg);
+ }
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "stop_pattern");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *p;
+
+ p = lua_tolstring(L, -1, &plen);
+
+ if (p && plen > 0) {
+ stop_pattern = g_malloc(plen);
+ memcpy(stop_pattern, p, plen);
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "partial");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ partial = lua_toboolean(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "shutdown");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ do_shutdown = lua_toboolean(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "read");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ do_read = lua_toboolean(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "ssl");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ ssl = lua_toboolean(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "ssl_noverify");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ ssl_noverify = lua_toboolean(L, -1);
+ lua_pop(L, 1);
+ }
+ else {
+ lua_pop(L, 1); /* Previous nil... */
+ /* Similar to lua http, meh... */
+ lua_pushstring(L, "no_ssl_verify");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ ssl_noverify = lua_toboolean(L, -1);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "on_connect");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ conn_cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "upstream");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_upstream *lup = lua_check_upstream(L, -1);
+
+ if (lup) {
+ /* Preserve pointer in case if lup is destructed */
+ up = lup->up;
+ }
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "data");
+ lua_gettable(L, -2);
+ total_out = 0;
+
+ tp = lua_type(L, -1);
+ if (tp == LUA_TSTRING || tp == LUA_TUSERDATA) {
+ iov = g_malloc(sizeof(*iov));
+ niov = 1;
+
+ if (!lua_tcp_arg_toiovec(L, -1, cbd, iov)) {
+ lua_pop(L, 1);
+ msg_err("tcp request has bad data argument");
+ lua_pushboolean(L, FALSE);
+ g_free(iov);
+ g_free(cbd);
+
+ return 1;
+ }
+
+ total_out = iov[0].iov_len;
+ }
+ else if (tp == LUA_TTABLE) {
+ /* Count parts */
+ lua_pushnil(L);
+ while (lua_next(L, -2) != 0) {
+ niov++;
+ lua_pop(L, 1);
+ }
+
+ iov = g_malloc(sizeof(*iov) * niov);
+ lua_pushnil(L);
+ niov = 0;
+
+ while (lua_next(L, -2) != 0) {
+ if (!lua_tcp_arg_toiovec(L, -1, cbd, &iov[niov])) {
+ lua_pop(L, 2);
+ msg_err("tcp request has bad data argument at pos %d", niov);
+ lua_pushboolean(L, FALSE);
+ g_free(iov);
+ g_free(cbd);
+
+ return 1;
+ }
+
+ total_out += iov[niov].iov_len;
+ niov++;
+
+ lua_pop(L, 1);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ else {
+ return luaL_error(L, "tcp request has bad params");
+ }
+
+ if (resolver == NULL && cfg == NULL && task == NULL) {
+ g_free(cbd);
+ g_free(iov);
+
+ return luaL_error(L, "tcp request has bad params: one of "
+ "{resolver,task,config} should be set");
+ }
+
+ cbd->task = task;
+
+ if (task) {
+ cbd->item = rspamd_symcache_get_cur_item(task);
+ }
+
+ cbd->cfg = cfg;
+ h = rspamd_random_uint64_fast();
+ rspamd_snprintf(cbd->tag, sizeof(cbd->tag), "%uxL", h);
+ cbd->handlers = g_queue_new();
+ cbd->hostname = g_strdup(host);
+
+ if (total_out > 0) {
+ struct lua_tcp_handler *wh;
+
+ wh = g_malloc0(sizeof(*wh));
+ wh->type = LUA_WANT_WRITE;
+ wh->h.w.iov = iov;
+ wh->h.w.iovlen = niov;
+ wh->h.w.total_bytes = total_out;
+ wh->h.w.pos = 0;
+ /* Cannot set write handler here */
+ wh->h.w.cbref = -1;
+
+ if (cbref != -1 && !do_read) {
+ /* We have write only callback */
+ wh->h.w.cbref = cbref;
+ }
+ else {
+ /* We have simple client callback */
+ wh->h.w.cbref = -1;
+ }
+
+ g_queue_push_tail(cbd->handlers, wh);
+ }
+
+ cbd->event_loop = event_loop;
+ cbd->fd = -1;
+ cbd->port = port;
+ cbd->ev.timeout = timeout;
+
+ if (ssl) {
+ cbd->flags |= LUA_TCP_FLAG_SSL;
+
+ if (ssl_noverify) {
+ cbd->flags |= LUA_TCP_FLAG_SSL_NOVERIFY;
+ }
+ }
+
+ if (do_read) {
+ cbd->in = g_byte_array_sized_new(8192);
+ }
+ else {
+ /* Save some space... */
+ cbd->in = g_byte_array_new();
+ }
+
+ if (partial) {
+ cbd->flags |= LUA_TCP_FLAG_PARTIAL;
+ }
+
+ if (do_shutdown) {
+ cbd->flags |= LUA_TCP_FLAG_SHUTDOWN;
+ }
+
+ if (do_read) {
+ struct lua_tcp_handler *rh;
+
+ rh = g_malloc0(sizeof(*rh));
+ rh->type = LUA_WANT_READ;
+ rh->h.r.cbref = cbref;
+ rh->h.r.stop_pattern = stop_pattern;
+ rh->h.r.plen = plen;
+ g_queue_push_tail(cbd->handlers, rh);
+ }
+
+ cbd->connect_cb = conn_cbref;
+ REF_INIT_RETAIN(cbd, lua_tcp_maybe_free);
+
+ if (up) {
+ cbd->up = rspamd_upstream_ref(up);
+ }
+
+ if (session) {
+ cbd->session = session;
+
+ if (rspamd_session_blocked(session)) {
+ lua_tcp_push_error(cbd, TRUE, "async session is the blocked state");
+ TCP_RELEASE(cbd);
+ cbd->item = NULL; /* To avoid decrease with no watcher */
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+ }
+
+ if (cbd->up) {
+ /* Use upstream to get addr */
+ cbd->addr = rspamd_inet_address_copy(rspamd_upstream_addr_next(cbd->up), NULL);
+
+ /* Host is numeric IP, no need to resolve */
+ lua_tcp_register_watcher(cbd);
+
+ if (!lua_tcp_make_connection(cbd)) {
+ lua_tcp_push_error(cbd, TRUE, "cannot connect to the host: %s", host);
+ lua_pushboolean(L, FALSE);
+
+ rspamd_upstream_fail(cbd->up, true, "failed to connect");
+
+ /* No reset of the item as watcher has been registered */
+ TCP_RELEASE(cbd);
+
+ return 1;
+ }
+ }
+ else if (rspamd_parse_inet_address(&cbd->addr,
+ host, strlen(host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ rspamd_inet_address_set_port(cbd->addr, port);
+ /* Host is numeric IP, no need to resolve */
+ lua_tcp_register_watcher(cbd);
+
+ if (!lua_tcp_make_connection(cbd)) {
+ lua_tcp_push_error(cbd, TRUE, "cannot connect to the host: %s", host);
+ lua_pushboolean(L, FALSE);
+
+ /* No reset of the item as watcher has been registered */
+ TCP_RELEASE(cbd);
+
+ return 1;
+ }
+ }
+ else {
+ if (task == NULL) {
+ if (!rspamd_dns_resolver_request(resolver, session, NULL, lua_tcp_dns_handler, cbd,
+ RDNS_REQUEST_A, host)) {
+ lua_tcp_push_error(cbd, TRUE, "cannot resolve host: %s", host);
+ lua_pushboolean(L, FALSE);
+ cbd->item = NULL; /* To avoid decrease with no watcher */
+ TCP_RELEASE(cbd);
+
+ return 1;
+ }
+ else {
+ lua_tcp_register_watcher(cbd);
+ }
+ }
+ else {
+ if (!rspamd_dns_resolver_request_task(task, lua_tcp_dns_handler, cbd,
+ RDNS_REQUEST_A, host)) {
+ lua_tcp_push_error(cbd, TRUE, "cannot resolve host: %s", host);
+ lua_pushboolean(L, FALSE);
+ cbd->item = NULL; /* To avoid decrease with no watcher */
+
+ TCP_RELEASE(cbd);
+
+ return 1;
+ }
+ else {
+ lua_tcp_register_watcher(cbd);
+ }
+ }
+ }
+
+ lua_pushboolean(L, TRUE);
+ return 1;
+}
+
+/***
+ * @function rspamd_tcp.connect_sync({params})
+ * Creates new pseudo-synchronous connection to the specific address:port
+ *
+ * - `task`: rspamd task objects (implies `pool`, `session`, `ev_base` and `resolver` arguments)
+ * - `ev_base`: event base (if no task specified)
+ * - `resolver`: DNS resolver (no task)
+ * - `session`: events session (no task)
+ * - `config`: config (no task)
+ * - `host`: IP or name of the peer (required)
+ * - `port`: remote port to use
+ * - `timeout`: floating point value that specifies timeout for IO operations in **seconds**
+ * @return {boolean} true if request has been sent
+ */
+static gint
+lua_tcp_connect_sync(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ GError *err = NULL;
+
+ gint64 port = -1;
+ gdouble timeout = default_tcp_timeout;
+ const gchar *host = NULL;
+ gint ret;
+ guint64 h;
+
+ struct rspamd_task *task = NULL;
+ struct rspamd_async_session *session = NULL;
+ struct rspamd_dns_resolver *resolver = NULL;
+ struct rspamd_config *cfg = NULL;
+ struct ev_loop *ev_base = NULL;
+ struct lua_tcp_cbdata *cbd;
+
+
+ int arguments_validated = rspamd_lua_parse_table_arguments(L, 1, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "task=U{task};session=U{session};resolver=U{resolver};ev_base=U{ev_base};"
+ "*host=S;*port=I;timeout=D;config=U{config}",
+ &task, &session, &resolver, &ev_base,
+ &host, &port, &timeout, &cfg);
+
+ if (!arguments_validated) {
+ if (err) {
+ ret = luaL_error(L, "invalid arguments: %s", err->message);
+ g_error_free(err);
+
+ return ret;
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (0 > port || port > 65535) {
+ return luaL_error(L, "invalid port given (correct values: 1..65535)");
+ }
+
+ if (task == NULL && (cfg == NULL || ev_base == NULL || session == NULL)) {
+ return luaL_error(L, "invalid arguments: either task or config+ev_base+session should be set");
+ }
+
+ if (isnan(timeout)) {
+ /* rspamd_lua_parse_table_arguments() sets missing N field to zero */
+ timeout = default_tcp_timeout;
+ }
+
+ cbd = g_new0(struct lua_tcp_cbdata, 1);
+
+ if (task) {
+ static const gchar hexdigests[16] = "0123456789abcdef";
+
+ cfg = task->cfg;
+ ev_base = task->event_loop;
+ session = task->s;
+ /* Make a readable tag */
+ memcpy(cbd->tag, task->task_pool->tag.uid, sizeof(cbd->tag) - 2);
+ cbd->tag[sizeof(cbd->tag) - 2] = hexdigests[GPOINTER_TO_INT(cbd) & 0xf];
+ cbd->tag[sizeof(cbd->tag) - 1] = 0;
+ }
+ else {
+ h = rspamd_random_uint64_fast();
+ rspamd_snprintf(cbd->tag, sizeof(cbd->tag), "%uxL", h);
+ }
+
+ if (resolver == NULL) {
+ if (task) {
+ resolver = task->resolver;
+ }
+ else {
+ resolver = lua_tcp_global_resolver(ev_base, cfg);
+ }
+ }
+
+ cbd->task = task;
+ cbd->cfg = cfg;
+ cbd->thread = lua_thread_pool_get_running_entry(cfg->lua_thread_pool);
+
+
+ cbd->handlers = g_queue_new();
+
+ cbd->event_loop = ev_base;
+ cbd->flags |= LUA_TCP_FLAG_SYNC;
+ cbd->fd = -1;
+ cbd->port = (guint16) port;
+
+ cbd->in = g_byte_array_new();
+
+ cbd->connect_cb = -1;
+
+ REF_INIT_RETAIN(cbd, lua_tcp_maybe_free);
+
+ if (task) {
+ rspamd_mempool_add_destructor(task->task_pool, lua_tcp_sync_session_dtor, cbd);
+ }
+
+ struct lua_tcp_handler *wh;
+
+ wh = g_malloc0(sizeof(*wh));
+ wh->type = LUA_WANT_CONNECT;
+
+ g_queue_push_tail(cbd->handlers, wh);
+
+ if (session) {
+ cbd->session = session;
+
+ if (rspamd_session_blocked(session)) {
+ TCP_RELEASE(cbd);
+ lua_pushboolean(L, FALSE);
+ lua_pushliteral(L, "Session is being destroyed, requests are not allowed");
+
+ return 2;
+ }
+ }
+
+ if (rspamd_parse_inet_address(&cbd->addr,
+ host, strlen(host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ rspamd_inet_address_set_port(cbd->addr, (guint16) port);
+ /* Host is numeric IP, no need to resolve */
+ if (!lua_tcp_make_connection(cbd)) {
+ lua_pushboolean(L, FALSE);
+ lua_pushliteral(L, "Failed to initiate connection");
+
+ TCP_RELEASE(cbd);
+
+ return 2;
+ }
+ }
+ else {
+ if (task == NULL) {
+ if (!rspamd_dns_resolver_request(resolver, session, NULL, lua_tcp_dns_handler, cbd,
+ RDNS_REQUEST_A, host)) {
+ lua_pushboolean(L, FALSE);
+ lua_pushliteral(L, "Failed to initiate dns request");
+
+ TCP_RELEASE(cbd);
+
+ return 2;
+ }
+ else {
+ lua_tcp_register_watcher(cbd);
+ }
+ }
+ else {
+ cbd->item = rspamd_symcache_get_cur_item(task);
+
+ if (!rspamd_dns_resolver_request_task(task, lua_tcp_dns_handler, cbd,
+ RDNS_REQUEST_A, host)) {
+ cbd->item = NULL; /* We have not registered watcher */
+ lua_pushboolean(L, FALSE);
+ lua_pushliteral(L, "Failed to initiate dns request");
+ TCP_RELEASE(cbd);
+
+ return 2;
+ }
+ else {
+ lua_tcp_register_watcher(cbd);
+ }
+ }
+ }
+
+ return lua_thread_yield(cbd->thread, 0);
+}
+
+static gint
+lua_tcp_close(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_tcp(L, 1);
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ cbd->flags |= LUA_TCP_FLAG_FINISHED;
+
+ if (cbd->ssl_conn) {
+ /* TODO: postpone close in case ssl is used ! */
+ rspamd_ssl_connection_free(cbd->ssl_conn);
+ cbd->ssl_conn = NULL;
+ }
+
+ if (cbd->fd != -1) {
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+ close(cbd->fd);
+ cbd->fd = -1;
+ }
+
+ if (cbd->addr) {
+ rspamd_inet_address_free(cbd->addr);
+ cbd->addr = NULL;
+ }
+
+ if (cbd->up) {
+ rspamd_upstream_unref(cbd->up);
+ cbd->up = NULL;
+ }
+ /* Do not release refcount as it will be handled elsewhere */
+ /* TCP_RELEASE (cbd); */
+
+ return 0;
+}
+
+static gint
+lua_tcp_add_read(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_tcp(L, 1);
+ struct lua_tcp_handler *rh;
+ gchar *stop_pattern = NULL;
+ const gchar *p;
+ gsize plen = 0;
+ gint cbref = -1;
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ p = lua_tolstring(L, 3, &plen);
+
+ if (p && plen > 0) {
+ stop_pattern = g_malloc(plen);
+ memcpy(stop_pattern, p, plen);
+ }
+ }
+
+ rh = g_malloc0(sizeof(*rh));
+ rh->type = LUA_WANT_READ;
+ rh->h.r.cbref = cbref;
+ rh->h.r.stop_pattern = stop_pattern;
+ rh->h.r.plen = plen;
+ msg_debug_tcp("added read event, cbref: %d", cbref);
+
+ g_queue_push_tail(cbd->handlers, rh);
+
+ return 0;
+}
+
+static gint
+lua_tcp_add_write(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_tcp(L, 1);
+ struct lua_tcp_handler *wh;
+ gint cbref = -1, tp;
+ struct iovec *iov = NULL;
+ guint niov = 0, total_out = 0;
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+
+ tp = lua_type(L, 3);
+ if (tp == LUA_TSTRING || tp == LUA_TUSERDATA) {
+ iov = g_malloc(sizeof(*iov));
+ niov = 1;
+
+ if (!lua_tcp_arg_toiovec(L, 3, cbd, iov)) {
+ msg_err("tcp request has bad data argument");
+ lua_pushboolean(L, FALSE);
+ g_free(iov);
+
+ return 1;
+ }
+
+ total_out = iov[0].iov_len;
+ }
+ else if (tp == LUA_TTABLE) {
+ /* Count parts */
+ lua_pushvalue(L, 3);
+
+ lua_pushnil(L);
+ while (lua_next(L, -2) != 0) {
+ niov++;
+ lua_pop(L, 1);
+ }
+
+ iov = g_malloc(sizeof(*iov) * niov);
+ lua_pushnil(L);
+ niov = 0;
+
+ while (lua_next(L, -2) != 0) {
+ if (!lua_tcp_arg_toiovec(L, -1, cbd, &iov[niov])) {
+ lua_pop(L, 2);
+ msg_err("tcp request has bad data argument at pos %d", niov);
+ lua_pushboolean(L, FALSE);
+ g_free(iov);
+ g_free(cbd);
+
+ return 1;
+ }
+
+ total_out += iov[niov].iov_len;
+ niov++;
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ wh = g_malloc0(sizeof(*wh));
+ wh->type = LUA_WANT_WRITE;
+ wh->h.w.iov = iov;
+ wh->h.w.iovlen = niov;
+ wh->h.w.total_bytes = total_out;
+ wh->h.w.pos = 0;
+ /* Cannot set write handler here */
+ wh->h.w.cbref = cbref;
+ msg_debug_tcp("added write event, cbref: %d", cbref);
+
+ g_queue_push_tail(cbd->handlers, wh);
+ lua_pushboolean(L, TRUE);
+
+ return 1;
+}
+
+static gint
+lua_tcp_shift_callback(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_tcp(L, 1);
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_tcp_shift_handler(cbd);
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+
+ return 0;
+}
+
+static struct lua_tcp_cbdata *
+lua_check_sync_tcp(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{tcp_sync}");
+ luaL_argcheck(L, ud != NULL, pos, "'tcp' expected");
+ return ud ? *((struct lua_tcp_cbdata **) ud) : NULL;
+}
+
+static int
+lua_tcp_sync_close(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+ cbd->flags |= LUA_TCP_FLAG_FINISHED;
+
+ if (cbd->fd != -1) {
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+ close(cbd->fd);
+ cbd->fd = -1;
+ }
+
+ return 0;
+}
+
+static void
+lua_tcp_sync_session_dtor(gpointer ud)
+{
+ struct lua_tcp_cbdata *cbd = ud;
+ cbd->flags |= LUA_TCP_FLAG_FINISHED;
+
+ if (cbd->fd != -1) {
+ msg_debug("closing sync TCP connection");
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+ close(cbd->fd);
+ cbd->fd = -1;
+ }
+
+ /* Task is gone, we should not try use it anymore */
+ cbd->task = NULL;
+
+ /* All events are removed when task is done, we should not refer them */
+ cbd->async_ev = NULL;
+}
+
+static int
+lua_tcp_sync_read_once(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+ struct lua_tcp_handler *rh;
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+
+ struct thread_entry *thread = lua_thread_pool_get_running_entry(cbd->cfg->lua_thread_pool);
+
+ rh = g_malloc0(sizeof(*rh));
+ rh->type = LUA_WANT_READ;
+ rh->h.r.cbref = -1;
+
+ msg_debug_tcp("added read sync event, thread: %p", thread);
+
+ g_queue_push_tail(cbd->handlers, rh);
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+
+ TCP_RETAIN(cbd);
+
+ return lua_thread_yield(thread, 0);
+}
+
+static int
+lua_tcp_sync_write(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+ struct lua_tcp_handler *wh;
+ gint tp;
+ struct iovec *iov = NULL;
+ guint niov = 0;
+ gsize total_out = 0;
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+
+ struct thread_entry *thread = lua_thread_pool_get_running_entry(cbd->cfg->lua_thread_pool);
+
+ tp = lua_type(L, 2);
+ if (tp == LUA_TSTRING || tp == LUA_TUSERDATA) {
+ iov = g_malloc(sizeof(*iov));
+ niov = 1;
+
+ if (!lua_tcp_arg_toiovec(L, 2, cbd, iov)) {
+ msg_err("tcp request has bad data argument");
+ g_free(iov);
+ g_free(cbd);
+
+ return luaL_error(L, "invalid arguments second parameter (data) is expected to be either string or rspamd{text}");
+ }
+
+ total_out = iov[0].iov_len;
+ }
+ else if (tp == LUA_TTABLE) {
+ /* Count parts */
+ lua_pushvalue(L, 3);
+
+ lua_pushnil(L);
+ while (lua_next(L, -2) != 0) {
+ niov++;
+ lua_pop(L, 1);
+ }
+
+ iov = g_malloc(sizeof(*iov) * niov);
+ lua_pushnil(L);
+ niov = 0;
+
+ while (lua_next(L, -2) != 0) {
+ if (!lua_tcp_arg_toiovec(L, -1, cbd, &iov[niov])) {
+ msg_err("tcp request has bad data argument at pos %d", niov);
+ g_free(iov);
+ g_free(cbd);
+
+ return luaL_error(L, "invalid arguments second parameter (data) is expected to be either string or rspamd{text}");
+ }
+
+ total_out += iov[niov].iov_len;
+ niov++;
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ wh = g_malloc0(sizeof(*wh));
+ wh->type = LUA_WANT_WRITE;
+ wh->h.w.iov = iov;
+ wh->h.w.iovlen = niov;
+ wh->h.w.total_bytes = total_out;
+ wh->h.w.pos = 0;
+ wh->h.w.cbref = -1;
+ msg_debug_tcp("added sync write event, thread: %p", thread);
+
+ g_queue_push_tail(cbd->handlers, wh);
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+
+ TCP_RETAIN(cbd);
+
+ return lua_thread_yield(thread, 0);
+}
+
+static gint
+lua_tcp_sync_eof(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+
+ lua_pushboolean(L, cbd->eof);
+
+ return 1;
+}
+
+static gint
+lua_tcp_sync_shutdown(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+
+ shutdown(cbd->fd, SHUT_WR);
+
+ return 0;
+}
+
+static gint
+lua_tcp_starttls(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_tcp(L, 1);
+ gpointer ssl_ctx;
+ gboolean verify_peer;
+
+ if (cbd == NULL || cbd->ssl_conn != NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (cbd->flags & LUA_TCP_FLAG_SSL_NOVERIFY) {
+ ssl_ctx = cbd->cfg->libs_ctx->ssl_ctx_noverify;
+ verify_peer = FALSE;
+ }
+ else {
+ ssl_ctx = cbd->cfg->libs_ctx->ssl_ctx;
+ verify_peer = TRUE;
+ }
+
+ cbd->ssl_conn = rspamd_ssl_connection_new(ssl_ctx,
+ cbd->event_loop,
+ verify_peer,
+ cbd->tag);
+
+ if (!rspamd_ssl_connect_fd(cbd->ssl_conn, cbd->fd, cbd->hostname, &cbd->ev,
+ cbd->ev.timeout, lua_tcp_handler, lua_tcp_ssl_on_error, cbd)) {
+ lua_tcp_push_error(cbd, TRUE, "ssl connection failed: %s",
+ strerror(errno));
+ }
+
+ return 0;
+}
+
+static gint
+lua_tcp_sync_gc(lua_State *L)
+{
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+ if (!cbd) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+
+ lua_tcp_maybe_free(cbd);
+ lua_tcp_fin(cbd);
+
+ return 0;
+}
+
+static gint
+lua_load_tcp(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, tcp_libf);
+
+ return 1;
+}
+
+void luaopen_tcp(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_tcp", lua_load_tcp);
+ rspamd_lua_new_class(L, "rspamd{tcp}", tcp_libm);
+ rspamd_lua_new_class(L, "rspamd{tcp_sync}", tcp_sync_libm);
+ lua_pop(L, 1);
+}
diff --git a/src/lua/lua_tensor.c b/src/lua/lua_tensor.c
new file mode 100644
index 0000000..75e6139
--- /dev/null
+++ b/src/lua/lua_tensor.c
@@ -0,0 +1,817 @@
+/*-
+ * Copyright 2020 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "lua_tensor.h"
+#include "contrib/kann/kautodiff.h"
+#include "blas-config.h"
+
+/***
+ * @module rspamd_tensor
+ * `rspamd_tensor` is a simple Lua library to abstract matrices and vectors
+ * Internally, they are represented as arrays of float variables
+ * So far, merely 1D and 2D tensors are supported
+ */
+
+LUA_FUNCTION_DEF(tensor, load);
+LUA_FUNCTION_DEF(tensor, save);
+LUA_FUNCTION_DEF(tensor, new);
+LUA_FUNCTION_DEF(tensor, fromtable);
+LUA_FUNCTION_DEF(tensor, destroy);
+LUA_FUNCTION_DEF(tensor, mul);
+LUA_FUNCTION_DEF(tensor, tostring);
+LUA_FUNCTION_DEF(tensor, index);
+LUA_FUNCTION_DEF(tensor, newindex);
+LUA_FUNCTION_DEF(tensor, len);
+LUA_FUNCTION_DEF(tensor, eigen);
+LUA_FUNCTION_DEF(tensor, mean);
+LUA_FUNCTION_DEF(tensor, transpose);
+LUA_FUNCTION_DEF(tensor, has_blas);
+LUA_FUNCTION_DEF(tensor, scatter_matrix);
+
+static luaL_reg rspamd_tensor_f[] = {
+ LUA_INTERFACE_DEF(tensor, load),
+ LUA_INTERFACE_DEF(tensor, new),
+ LUA_INTERFACE_DEF(tensor, fromtable),
+ LUA_INTERFACE_DEF(tensor, has_blas),
+ LUA_INTERFACE_DEF(tensor, scatter_matrix),
+ {NULL, NULL},
+};
+
+static luaL_reg rspamd_tensor_m[] = {
+ LUA_INTERFACE_DEF(tensor, save),
+ {"__gc", lua_tensor_destroy},
+ {"__mul", lua_tensor_mul},
+ {"mul", lua_tensor_mul},
+ {"tostring", lua_tensor_tostring},
+ {"__tostring", lua_tensor_tostring},
+ {"__index", lua_tensor_index},
+ {"__newindex", lua_tensor_newindex},
+ {"__len", lua_tensor_len},
+ LUA_INTERFACE_DEF(tensor, eigen),
+ LUA_INTERFACE_DEF(tensor, mean),
+ LUA_INTERFACE_DEF(tensor, transpose),
+ {NULL, NULL},
+};
+
+struct rspamd_lua_tensor *
+lua_newtensor(lua_State *L, int ndims, const int *dim, bool zero_fill, bool own)
+{
+ struct rspamd_lua_tensor *res;
+
+ res = lua_newuserdata(L, sizeof(struct rspamd_lua_tensor));
+ memset(res, 0, sizeof(*res));
+
+ res->ndims = ndims;
+ res->size = 1;
+
+ for (guint i = 0; i < ndims; i++) {
+ res->size *= dim[i];
+ res->dim[i] = dim[i];
+ }
+
+ /* To avoid allocating large stuff in Lua */
+ if (own) {
+ res->data = g_malloc(sizeof(rspamd_tensor_num_t) * res->size);
+
+ if (zero_fill) {
+ memset(res->data, 0, sizeof(rspamd_tensor_num_t) * res->size);
+ }
+ }
+ else {
+ /* Mark size negative to distinguish */
+ res->size = -(res->size);
+ }
+
+ rspamd_lua_setclass(L, TENSOR_CLASS, -1);
+
+ return res;
+}
+
+/***
+ * @function tensor.new(ndims, [dim1, ... dimN])
+ * Creates a new zero filled tensor with the specific number of dimensions
+ * @return
+ */
+static gint
+lua_tensor_new(lua_State *L)
+{
+ gint ndims = luaL_checkinteger(L, 1);
+
+ if (ndims > 0 && ndims <= 2) {
+ gint *dims = g_alloca(sizeof(gint) * ndims);
+
+ for (guint i = 0; i < ndims; i++) {
+ dims[i] = lua_tointeger(L, i + 2);
+ }
+
+ (void) lua_newtensor(L, ndims, dims, true, true);
+ }
+ else {
+ return luaL_error(L, "incorrect dimensions number: %d", ndims);
+ }
+
+ return 1;
+}
+
+/***
+ * @function tensor.fromtable(tbl)
+ * Creates a new zero filled tensor with the specific number of dimensions
+ * @return
+ */
+static gint
+lua_tensor_fromtable(lua_State *L)
+{
+ if (lua_istable(L, 1)) {
+ lua_rawgeti(L, 1, 1);
+
+ if (lua_isnumber(L, -1)) {
+ lua_pop(L, 1);
+ /* Input vector */
+ gint dims[2];
+ dims[0] = 1;
+ dims[1] = rspamd_lua_table_size(L, 1);
+
+ struct rspamd_lua_tensor *res = lua_newtensor(L, 2,
+ dims, false, true);
+
+ for (guint i = 0; i < dims[1]; i++) {
+ lua_rawgeti(L, 1, i + 1);
+ res->data[i] = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+ }
+ }
+ else if (lua_istable(L, -1)) {
+ /* Input matrix */
+ lua_pop(L, 1);
+
+ /* Calculate the overall size */
+ gint nrows = rspamd_lua_table_size(L, 1), ncols = 0;
+ gint err;
+
+ for (gint i = 0; i < nrows; i++) {
+ lua_rawgeti(L, 1, i + 1);
+
+ if (ncols == 0) {
+ ncols = rspamd_lua_table_size(L, -1);
+
+ if (ncols == 0) {
+ lua_pop(L, 1);
+ err = luaL_error(L, "invalid params at pos %d: "
+ "bad input dimension %d",
+ i,
+ (int) ncols);
+
+ return err;
+ }
+ }
+ else {
+ if (ncols != rspamd_lua_table_size(L, -1)) {
+ gint t = rspamd_lua_table_size(L, -1);
+
+ lua_pop(L, 1);
+ err = luaL_error(L, "invalid params at pos %d: "
+ "bad input dimension %d; %d expected",
+ i,
+ t,
+ ncols);
+
+ return err;
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ gint dims[2];
+ dims[0] = nrows;
+ dims[1] = ncols;
+
+ struct rspamd_lua_tensor *res = lua_newtensor(L, 2,
+ dims, false, true);
+
+ for (gint i = 0; i < nrows; i++) {
+ lua_rawgeti(L, 1, i + 1);
+
+ for (gint j = 0; j < ncols; j++) {
+ lua_rawgeti(L, -1, j + 1);
+
+ res->data[i * ncols + j] = lua_tonumber(L, -1);
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ lua_pop(L, 1);
+ return luaL_error(L, "incorrect table");
+ }
+ }
+ else {
+ return luaL_error(L, "incorrect input");
+ }
+
+ return 1;
+}
+
+
+/***
+ * @method tensor:destroy()
+ * Tensor destructor
+ * @return
+ */
+static gint
+lua_tensor_destroy(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+
+ if (t) {
+ if (t->size > 0) {
+ g_free(t->data);
+ }
+ }
+
+ return 0;
+}
+
+/***
+ * @method tensor:save()
+ * Tensor serialisation function
+ * @return
+ */
+static gint
+lua_tensor_save(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+ gint size;
+
+ if (t) {
+ if (t->size > 0) {
+ size = t->size;
+ }
+ else {
+ size = -(t->size);
+ }
+
+ gsize sz = sizeof(gint) * 4 + size * sizeof(rspamd_tensor_num_t);
+ guchar *data;
+
+ struct rspamd_lua_text *out = lua_new_text(L, NULL, 0, TRUE);
+
+ data = g_malloc(sz);
+ memcpy(data, &t->ndims, sizeof(int));
+ memcpy(data + sizeof(int), &size, sizeof(int));
+ memcpy(data + 2 * sizeof(int), t->dim, sizeof(int) * 2);
+ memcpy(data + 4 * sizeof(int), t->data,
+ size * sizeof(rspamd_tensor_num_t));
+
+ out->start = (const gchar *) data;
+ out->len = sz;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_tensor_tostring(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+
+ if (t) {
+ GString *out = g_string_sized_new(128);
+
+ if (t->ndims == 1) {
+ /* Print as a vector */
+ for (gint i = 0; i < t->dim[0]; i++) {
+ rspamd_printf_gstring(out, "%.4f ", t->data[i]);
+ }
+ /* Trim last space */
+ out->len--;
+ }
+ else {
+ for (gint i = 0; i < t->dim[0]; i++) {
+ for (gint j = 0; j < t->dim[1]; j++) {
+ rspamd_printf_gstring(out, "%.4f ",
+ t->data[i * t->dim[1] + j]);
+ }
+ /* Trim last space */
+ out->len--;
+ rspamd_printf_gstring(out, "\n");
+ }
+ /* Trim last ; */
+ out->len--;
+ }
+
+ lua_pushlstring(L, out->str, out->len);
+
+ g_string_free(out, TRUE);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_tensor_index(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+ gint idx;
+
+ if (t) {
+ if (lua_isnumber(L, 2)) {
+ idx = lua_tointeger(L, 2);
+
+ if (t->ndims == 1) {
+ /* Individual element */
+ if (idx <= t->dim[0]) {
+ lua_pushnumber(L, t->data[idx - 1]);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ /* Push row */
+ gint dim = t->dim[1];
+
+
+ if (idx <= t->dim[0]) {
+ /* Non-owning tensor */
+ struct rspamd_lua_tensor *res =
+ lua_newtensor(L, 1, &dim, false, false);
+ res->data = &t->data[(idx - 1) * t->dim[1]];
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ }
+ else if (lua_isstring(L, 2)) {
+ /* Access to methods */
+ lua_getmetatable(L, 1);
+ lua_pushvalue(L, 2);
+ lua_rawget(L, -2);
+ }
+ }
+
+ return 1;
+}
+static gint
+lua_tensor_newindex(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+ gint idx;
+
+ if (t) {
+ if (lua_isnumber(L, 2)) {
+ idx = lua_tointeger(L, 2);
+
+ if (t->ndims == 1) {
+ /* Individual element */
+ if (idx <= t->dim[0] && idx > 0) {
+ rspamd_tensor_num_t value = lua_tonumber(L, 3), old;
+
+ old = t->data[idx - 1];
+ t->data[idx - 1] = value;
+ lua_pushnumber(L, old);
+ }
+ else {
+ return luaL_error(L, "invalid index: %d", idx);
+ }
+ }
+ else {
+ if (lua_isnumber(L, 3)) {
+ return luaL_error(L, "cannot assign number to a row");
+ }
+ else if (lua_isuserdata(L, 3)) {
+ /* Tensor assignment */
+ struct rspamd_lua_tensor *row = lua_check_tensor(L, 3);
+
+ if (row) {
+ if (row->ndims == 1) {
+ if (row->dim[0] == t->dim[1]) {
+ if (idx > 0 && idx <= t->dim[0]) {
+ idx--; /* Zero based index */
+ memcpy(&t->data[idx * t->dim[1]],
+ row->data,
+ t->dim[1] * sizeof(rspamd_tensor_num_t));
+
+ return 0;
+ }
+ else {
+ return luaL_error(L, "invalid index: %d", idx);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "cannot assign matrix to row");
+ }
+ }
+ else {
+ return luaL_error(L, "cannot assign row, invalid tensor");
+ }
+ }
+ else {
+ /* TODO: add table assignment */
+ return luaL_error(L, "cannot assign row, not a tensor");
+ }
+ }
+ }
+ else {
+ /* Access to methods? NYI */
+ return luaL_error(L, "cannot assign method of a tensor");
+ }
+ }
+
+ return 1;
+}
+
+/***
+ * @method tensor:mul(other, [transA, [transB]])
+ * Multiply two tensors (optionally transposed) and return a new tensor
+ * @return
+ */
+static gint
+lua_tensor_mul(lua_State *L)
+{
+ struct rspamd_lua_tensor *t1 = lua_check_tensor(L, 1),
+ *t2 = lua_check_tensor(L, 2), *res;
+ int transA = 0, transB = 0;
+
+ if (lua_isboolean(L, 3)) {
+ transA = lua_toboolean(L, 3);
+ }
+
+ if (lua_isboolean(L, 4)) {
+ transB = lua_toboolean(L, 4);
+ }
+
+ if (t1 && t2) {
+ gint dims[2], shadow_dims[2];
+ dims[0] = abs(transA ? t1->dim[1] : t1->dim[0]);
+ shadow_dims[0] = abs(transB ? t2->dim[1] : t2->dim[0]);
+ dims[1] = abs(transB ? t2->dim[0] : t2->dim[1]);
+ shadow_dims[1] = abs(transA ? t1->dim[0] : t1->dim[1]);
+
+ if (shadow_dims[0] != shadow_dims[1]) {
+ return luaL_error(L, "incompatible dimensions %d x %d * %d x %d",
+ dims[0], shadow_dims[1], shadow_dims[0], dims[1]);
+ }
+ else if (shadow_dims[0] == 0) {
+ /* Row * Column -> matrix */
+ shadow_dims[0] = 1;
+ shadow_dims[1] = 1;
+ }
+
+ if (dims[0] == 0) {
+ /* Column */
+ dims[0] = 1;
+
+ if (dims[1] == 0) {
+ /* Column * row -> number */
+ dims[1] = 1;
+ }
+ res = lua_newtensor(L, 2, dims, true, true);
+ }
+ else if (dims[1] == 0) {
+ /* Row */
+ res = lua_newtensor(L, 1, dims, true, true);
+ dims[1] = 1;
+ }
+ else {
+ res = lua_newtensor(L, 2, dims, true, true);
+ }
+
+ kad_sgemm_simple(transA, transB, dims[0], dims[1], shadow_dims[0],
+ t1->data, t2->data, res->data);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function tensor.load(rspamd_text)
+ * Deserialize tensor
+ * @return
+ */
+static gint
+lua_tensor_load(lua_State *L)
+{
+ const guchar *data;
+ gsize sz;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid argument");
+ }
+
+ data = (const guchar *) t->start;
+ sz = t->len;
+ }
+ else {
+ data = (const guchar *) lua_tolstring(L, 1, &sz);
+ }
+
+ if (sz >= sizeof(gint) * 4) {
+ int ndims, nelts, dims[2];
+
+ memcpy(&ndims, data, sizeof(int));
+ memcpy(&nelts, data + sizeof(int), sizeof(int));
+ memcpy(dims, data + sizeof(int) * 2, sizeof(int) * 2);
+
+ if (sz == nelts * sizeof(rspamd_tensor_num_t) + sizeof(int) * 4) {
+ if (ndims == 1) {
+ if (nelts == dims[0]) {
+ struct rspamd_lua_tensor *t = lua_newtensor(L, ndims, dims, false, true);
+ memcpy(t->data, data + sizeof(int) * 4, nelts * sizeof(rspamd_tensor_num_t));
+ }
+ else {
+ return luaL_error(L, "invalid argument: bad dims: %d x %d != %d",
+ dims[0], 1, nelts);
+ }
+ }
+ else if (ndims == 2) {
+ if (nelts == dims[0] * dims[1]) {
+ struct rspamd_lua_tensor *t = lua_newtensor(L, ndims, dims, false, true);
+ memcpy(t->data, data + sizeof(int) * 4, nelts * sizeof(rspamd_tensor_num_t));
+ }
+ else {
+ return luaL_error(L, "invalid argument: bad dims: %d x %d != %d",
+ dims[0], dims[1], nelts);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid argument: bad ndims: %d", ndims);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid size: %d, %d required, %d elts", (int) sz,
+ (int) (nelts * sizeof(rspamd_tensor_num_t) + sizeof(int) * 4),
+ nelts);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments; sz = %d", (int) sz);
+ }
+
+ return 1;
+}
+
+static gint
+lua_tensor_len(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+ gint nret = 1;
+
+ if (t) {
+ /* Return the main dimension first */
+ if (t->ndims == 1) {
+ lua_pushinteger(L, t->dim[0]);
+ }
+ else {
+ lua_pushinteger(L, t->dim[0]);
+ lua_pushinteger(L, t->dim[1]);
+ nret = 2;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return nret;
+}
+
+static gint
+lua_tensor_eigen(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1), *eigen;
+
+ if (t) {
+ if (t->ndims != 2 || t->dim[0] != t->dim[1]) {
+ return luaL_error(L, "expected square matrix NxN but got %dx%d",
+ t->dim[0], t->dim[1]);
+ }
+
+ eigen = lua_newtensor(L, 1, &t->dim[0], true, true);
+
+ if (!kad_ssyev_simple(t->dim[0], t->data, eigen->data)) {
+ lua_pop(L, 1);
+ return luaL_error(L, "kad_ssyev_simple failed (no blas?)");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static inline rspamd_tensor_num_t
+mean_vec(rspamd_tensor_num_t *x, gsize n)
+{
+ float sum = rspamd_sum_floats(x, &n);
+ return sum / (rspamd_tensor_num_t) n;
+}
+
+static gint
+lua_tensor_mean(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+
+ if (t) {
+ if (t->ndims == 1) {
+ /* Mean of all elements in a vector */
+ lua_pushnumber(L, mean_vec(t->data, t->dim[0]));
+ }
+ else {
+ /* Row-wise mean vector output */
+ struct rspamd_lua_tensor *res;
+
+ res = lua_newtensor(L, 1, &t->dim[0], false, true);
+
+ for (int i = 0; i < t->dim[0]; i++) {
+ res->data[i] = mean_vec(&t->data[i * t->dim[1]], t->dim[1]);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_tensor_transpose(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1), *res;
+ int dims[2];
+
+ if (t) {
+ if (t->ndims == 1) {
+ /* Row to column */
+ dims[0] = 1;
+ dims[1] = t->dim[0];
+ res = lua_newtensor(L, 2, dims, false, true);
+ memcpy(res->data, t->data, t->dim[0] * sizeof(rspamd_tensor_num_t));
+ }
+ else {
+ /* Cache friendly algorithm */
+ struct rspamd_lua_tensor *res;
+
+ dims[0] = t->dim[1];
+ dims[1] = t->dim[0];
+ res = lua_newtensor(L, 2, dims, false, true);
+
+ static const int block = 32;
+
+ for (int i = 0; i < t->dim[0]; i += block) {
+ for (int j = 0; j < t->dim[1]; ++j) {
+ for (int boff = 0; boff < block && i + boff < t->dim[0]; ++boff) {
+ res->data[j * t->dim[0] + i + boff] =
+ t->data[(i + boff) * t->dim[1] + j];
+ }
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_tensor_has_blas(lua_State *L)
+{
+#ifdef HAVE_CBLAS
+ lua_pushboolean(L, true);
+#else
+ lua_pushboolean(L, false);
+#endif
+
+ return 1;
+}
+
+static gint
+lua_tensor_scatter_matrix(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1), *res;
+ int dims[2];
+
+ if (t) {
+ if (t->ndims != 2) {
+ return luaL_error(L, "matrix required");
+ }
+
+ /* X * X square matrix */
+ dims[0] = t->dim[1];
+ dims[1] = t->dim[1];
+ res = lua_newtensor(L, 2, dims, true, true);
+
+ /* Auxiliary vars */
+ rspamd_tensor_num_t *means, /* means vector */
+ *tmp_row, /* temp row for Kahan's algorithm */
+ *tmp_square /* temp matrix for multiplications */;
+ means = g_malloc0(sizeof(rspamd_tensor_num_t) * t->dim[1]);
+ tmp_row = g_malloc0(sizeof(rspamd_tensor_num_t) * t->dim[1]);
+ tmp_square = g_malloc(sizeof(rspamd_tensor_num_t) * t->dim[1] * t->dim[1]);
+
+ /*
+ * Column based means
+ * means will have s, tmp_row will have c
+ */
+ for (int i = 0; i < t->dim[0]; i++) {
+ /* Cycle by rows */
+ for (int j = 0; j < t->dim[1]; j++) {
+ rspamd_tensor_num_t v = t->data[i * t->dim[1] + j];
+ rspamd_tensor_num_t y = v - tmp_row[j];
+ rspamd_tensor_num_t st = means[j] + y;
+ tmp_row[j] = (st - means[j]) - y;
+ means[j] = st;
+ }
+ }
+
+ for (int j = 0; j < t->dim[1]; j++) {
+ means[j] /= t->dim[0];
+ }
+
+ for (int i = 0; i < t->dim[0]; i++) {
+ /* Update for each sample */
+ for (int j = 0; j < t->dim[1]; j++) {
+ tmp_row[j] = t->data[i * t->dim[1] + j] - means[j];
+ }
+
+ memset(tmp_square, 0, t->dim[1] * t->dim[1] * sizeof(rspamd_tensor_num_t));
+ kad_sgemm_simple(1, 0, t->dim[1], t->dim[1], 1,
+ tmp_row, tmp_row, tmp_square);
+
+ for (int j = 0; j < t->dim[1]; j++) {
+ kad_saxpy(t->dim[1], 1.0, &tmp_square[j * t->dim[1]],
+ &res->data[j * t->dim[1]]);
+ }
+ }
+
+ g_free(tmp_row);
+ g_free(means);
+ g_free(tmp_square);
+ }
+ else {
+ return luaL_error(L, "tensor required");
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_tensor(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, rspamd_tensor_f);
+
+ return 1;
+}
+
+
+void luaopen_tensor(lua_State *L)
+{
+ /* Metatables */
+ rspamd_lua_new_class(L, TENSOR_CLASS, rspamd_tensor_m);
+ lua_pop(L, 1); /* No need in metatable... */
+ rspamd_lua_add_preload(L, "rspamd_tensor", lua_load_tensor);
+ lua_settop(L, 0);
+}
+
+struct rspamd_lua_tensor *
+lua_check_tensor(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, TENSOR_CLASS);
+ luaL_argcheck(L, ud != NULL, pos, "'tensor' expected");
+ return ud ? ((struct rspamd_lua_tensor *) ud) : NULL;
+}
diff --git a/src/lua/lua_tensor.h b/src/lua/lua_tensor.h
new file mode 100644
index 0000000..2103868
--- /dev/null
+++ b/src/lua/lua_tensor.h
@@ -0,0 +1,34 @@
+/*-
+ * Copyright 2020 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_LUA_TENSOR_H
+#define RSPAMD_LUA_TENSOR_H
+
+#define TENSOR_CLASS "rspamd{tensor}"
+
+typedef float rspamd_tensor_num_t;
+
+struct rspamd_lua_tensor {
+ int ndims;
+ int size; /* overall size (product of dims) */
+ int dim[2];
+ rspamd_tensor_num_t *data;
+};
+
+struct rspamd_lua_tensor *lua_check_tensor(lua_State *L, int pos);
+struct rspamd_lua_tensor *lua_newtensor(lua_State *L, int ndims,
+ const int *dim, bool zero_fill, bool own);
+
+#endif
diff --git a/src/lua/lua_text.c b/src/lua/lua_text.c
new file mode 100644
index 0000000..26a5c08
--- /dev/null
+++ b/src/lua/lua_text.c
@@ -0,0 +1,1789 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "libcryptobox/cryptobox.h"
+#include "contrib/fastutf8/fastutf8.h"
+#include "unix-std.h"
+
+/***
+ * @module rspamd_text
+ * This module provides access to opaque text structures used widely to prevent
+ * copying between Lua and C for various concerns: performance, security etc...
+ *
+ * You can convert rspamd_text into string but it will copy data.
+ */
+
+/***
+ * @function rspamd_text.fromstring(str)
+ * Creates rspamd_text from Lua string (copied to the text)
+ * @param {string} str string to use
+ * @return {rspamd_text} resulting text
+ */
+LUA_FUNCTION_DEF(text, fromstring);
+
+/***
+ * @function rspamd_text.null()
+ * Creates rspamd_text with NULL pointer for testing purposes
+ * @param {string} str string to use
+ * @return {rspamd_text} resulting text
+ */
+LUA_FUNCTION_DEF(text, null);
+/***
+ * @function rspamd_text.randombytes(nbytes)
+ * Creates rspamd_text with random bytes inside (raw bytes)
+ * @param {number} nbytes number of random bytes generated
+ * @return {rspamd_text} random bytes text
+ */
+LUA_FUNCTION_DEF(text, randombytes);
+
+/***
+ * @function rspamd_text.fromtable(tbl[, delim])
+ * Same as `table.concat` but generates rspamd_text instead of the Lua string
+ * @param {table} tbl table to use
+ * @param {string} delim optional delimiter
+ * @return {rspamd_text} resulting text
+ */
+LUA_FUNCTION_DEF(text, fromtable);
+/***
+ * @method rspamd_text:byte(pos[, pos2])
+ * Returns a byte at the position `pos` or bytes from `pos` to `pos2` if specified
+ * @param {integer} pos index
+ * @param {integer} pos2 index
+ * @return {integer} byte at the position `pos` or varargs of bytes
+ */
+LUA_FUNCTION_DEF(text, byte);
+/***
+ * @method rspamd_text:len()
+ * Returns length of a string
+ * @return {number} length of string in **bytes**
+ */
+LUA_FUNCTION_DEF(text, len);
+/***
+ * @method rspamd_text:str()
+ * Converts text to string by copying its content
+ * @return {string} copy of text as Lua string
+ */
+LUA_FUNCTION_DEF(text, str);
+/***
+ * @method rspamd_text:ptr()
+ * Converts text to lightuserdata
+ * @return {lightuserdata} pointer value of rspamd_text
+ */
+LUA_FUNCTION_DEF(text, ptr);
+/***
+ * @method rspamd_text:save_in_file(fname[, mode])
+ * Saves text in file
+ * @return {boolean} true if save has been completed
+ */
+LUA_FUNCTION_DEF(text, save_in_file);
+/***
+ * @method rspamd_text:span(start[, len])
+ * Returns a span for lua_text starting at pos [start] (1 indexed) and with
+ * length `len` (or to the end of the text)
+ * @param {integer} start start index
+ * @param {integer} len length of span
+ * @return {rspamd_text} new rspamd_text with span (must be careful when using with owned texts...)
+ */
+LUA_FUNCTION_DEF(text, span);
+/***
+ * @method rspamd_text:sub(start[, len])
+ * Returns a substring for lua_text similar to string.sub from Lua
+ * @return {rspamd_text} new rspamd_text with span (must be careful when using with owned texts...)
+ */
+LUA_FUNCTION_DEF(text, sub);
+/***
+ * @method rspamd_text:lines([stringify])
+ * Returns an iter over all lines as rspamd_text objects or as strings if `stringify` is true
+ * @param {boolean} stringify stringify lines
+ * @return {iterator} iterator triplet
+ */
+LUA_FUNCTION_DEF(text, lines);
+/***
+ * @method rspamd_text:split(regexp, [stringify])
+ * Returns an iter over all encounters of the specific regexp as rspamd_text objects or as strings if `stringify` is true
+ * @param {rspamd_regexp} regexp regexp (pcre syntax) used for splitting
+ * @param {boolean} stringify stringify lines
+ * @return {iterator} iterator triplet
+ */
+LUA_FUNCTION_DEF(text, split);
+/***
+ * @method rspamd_text:at(pos)
+ * Returns a byte at the position `pos`
+ * @param {integer} pos index
+ * @return {integer} byte at the position `pos` or nil if pos out of bound
+ */
+LUA_FUNCTION_DEF(text, at);
+/***
+ * @method rspamd_text:memchr(chr, [reverse])
+ * Returns the first or the last position of the character `chr` in the text or
+ * -1 in case if a character has not been found. Indexes start from `1`
+ * @param {string/number} chr character or a character code to find
+ * @param {boolean} reverse last character if `true`
+ * @return {integer} position of the character or `-1`
+ */
+LUA_FUNCTION_DEF(text, memchr);
+/***
+ * @method rspamd_text:bytes()
+ * Converts text to an array of bytes
+ * @return {table|integer} bytes in the array (as unsigned char)
+ */
+LUA_FUNCTION_DEF(text, bytes);
+/***
+ * @method rspamd_text:lower([is_utf, [inplace]])
+ * Return a new text with lowercased characters, if is_utf is true then Rspamd applies utf8 lowercase
+ * @param {boolean} is_utf apply utf8 lowercase
+ * @param {boolean} inplace lowercase the original text
+ * @return {rspamd_text} new rspamd_text (or the original text if inplace) with lowercased letters
+ */
+LUA_FUNCTION_DEF(text, lower);
+LUA_FUNCTION_DEF(text, take_ownership);
+/***
+ * @method rspamd_text:exclude_chars(set_to_exclude, [always_copy])
+ * Returns a text (if owned, then the original text is modified, if not, then it is copied and owned)
+ * where all chars from `set_to_exclude` are removed
+ * Patterns supported:
+ *
+ * - %s - all space characters
+ * - %n - all newline characters
+ * - %c - all control characters (it includes 8bit characters and spaces)
+ * - %8 - all 8 bit characters
+ * - %% - just a percent character
+ *
+ * @param {string} set_to_exclude characters to exclude
+ * @param {boolean} always_copy always copy the source text
+ * @return {rspamd_text} modified or copied text
+ */
+LUA_FUNCTION_DEF(text, exclude_chars);
+/***
+ * @method rspamd_text:oneline([always_copy])
+ * Returns a text (if owned, then the original text is modified, if not, then it is copied and owned)
+ * where the following transformations are made:
+ * - All spaces sequences are replaced with a single space
+ * - All newlines sequences are replaced with a single space
+ * - Trailing and leading spaces are removed
+ * - Control characters are excluded
+ * - UTF8 sequences are normalised
+ *
+ * @param {boolean} always_copy always copy the source text
+ * @return {rspamd_text} modified or copied text
+ */
+LUA_FUNCTION_DEF(text, oneline);
+/***
+ * @method rspamd_text:base32([b32type])
+ * Returns a text encoded in base32 (new rspamd_text is allocated)
+ *
+ * @param {string} b32type base32 type (default, bleach, rfc)
+ * @return {rspamd_text} new text encoded in base32
+ */
+LUA_FUNCTION_DEF(text, base32);
+/***
+ * @method rspamd_text:base64([line_length, [nline, [fold]]])
+ * Returns a text encoded in base64 (new rspamd_text is allocated)
+ *
+ * @param {number} line_length return text split with newlines up to this attribute
+ * @param {string} nline newline type: `cr`, `lf`, `crlf`
+ * @param {boolean} fold use folding when splitting into lines (false by default)
+ * @return {rspamd_text} new text encoded in base64
+ */
+LUA_FUNCTION_DEF(text, base64);
+/***
+ * @method rspamd_text:hex()
+ * Returns a text encoded in hex (new rspamd_text is allocated)
+ *
+ * @return {rspamd_text} new text encoded in hex
+ */
+LUA_FUNCTION_DEF(text, hex);
+/***
+ * @method rspamd_text:find(pattern [, init])
+ * Looks for the first match of pattern in the string s.
+ * If it finds a match, then find returns the indices of s where this occurrence
+ * starts and ends; otherwise, it returns nil. A third,
+ * optional numerical argument init specifies where to start the search;
+ * its default value is 1 and can be negative.
+ * This method currently supports merely a plain search, no patterns.
+ *
+ * @param {string} pattern pattern to find
+ * @param {number} init specifies where to start the search (1 default)
+ * @return {number,number/nil} If it finds a match, then find returns the indices of s where this occurrence starts and ends; otherwise, it returns nil
+ */
+LUA_FUNCTION_DEF(text, find);
+LUA_FUNCTION_DEF(text, gc);
+LUA_FUNCTION_DEF(text, eq);
+LUA_FUNCTION_DEF(text, lt);
+LUA_FUNCTION_DEF(text, concat);
+LUA_FUNCTION_DEF(text, strtoul);
+
+static const struct luaL_reg textlib_f[] = {
+ LUA_INTERFACE_DEF(text, fromstring),
+ {"from_string", lua_text_fromstring},
+ LUA_INTERFACE_DEF(text, fromtable),
+ {"from_table", lua_text_fromtable},
+ LUA_INTERFACE_DEF(text, null),
+ LUA_INTERFACE_DEF(text, randombytes),
+ {NULL, NULL}};
+
+static const struct luaL_reg textlib_m[] = {
+ LUA_INTERFACE_DEF(text, len),
+ LUA_INTERFACE_DEF(text, str),
+ LUA_INTERFACE_DEF(text, ptr),
+ LUA_INTERFACE_DEF(text, take_ownership),
+ LUA_INTERFACE_DEF(text, save_in_file),
+ LUA_INTERFACE_DEF(text, span),
+ LUA_INTERFACE_DEF(text, sub),
+ LUA_INTERFACE_DEF(text, lines),
+ LUA_INTERFACE_DEF(text, split),
+ LUA_INTERFACE_DEF(text, at),
+ LUA_INTERFACE_DEF(text, memchr),
+ LUA_INTERFACE_DEF(text, byte),
+ LUA_INTERFACE_DEF(text, bytes),
+ LUA_INTERFACE_DEF(text, lower),
+ LUA_INTERFACE_DEF(text, exclude_chars),
+ LUA_INTERFACE_DEF(text, oneline),
+ LUA_INTERFACE_DEF(text, base32),
+ LUA_INTERFACE_DEF(text, base64),
+ LUA_INTERFACE_DEF(text, hex),
+ LUA_INTERFACE_DEF(text, find),
+ LUA_INTERFACE_DEF(text, strtoul),
+ {"write", lua_text_save_in_file},
+ {"__len", lua_text_len},
+ {"__tostring", lua_text_str},
+ {"__gc", lua_text_gc},
+ {"__eq", lua_text_eq},
+ {"__lt", lua_text_lt},
+ {"__concat", lua_text_concat},
+ {NULL, NULL}};
+
+struct rspamd_lua_text *
+lua_check_text(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{text}");
+ luaL_argcheck(L, ud != NULL, pos, "'text' expected");
+ return ud ? (struct rspamd_lua_text *) ud : NULL;
+}
+
+struct rspamd_lua_text *
+lua_check_text_or_string(lua_State *L, gint pos)
+{
+ gint pos_type = lua_type(L, pos);
+
+ if (pos_type == LUA_TUSERDATA) {
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{text}");
+ luaL_argcheck(L, ud != NULL, pos, "'text' expected");
+ return ud ? (struct rspamd_lua_text *) ud : NULL;
+ }
+ else if (pos_type == LUA_TSTRING) {
+ /*
+ * Fake static lua_text, we allow to use this function multiple times
+ * by having a small array of static structures.
+ */
+ static unsigned cur_txt_idx = 0;
+ static struct rspamd_lua_text fake_text[4];
+ gsize len;
+ int sel_idx;
+
+ sel_idx = cur_txt_idx++ % G_N_ELEMENTS(fake_text);
+ fake_text[sel_idx].start = lua_tolstring(L, pos, &len);
+
+ if (len >= G_MAXUINT) {
+ return NULL;
+ }
+
+ fake_text[sel_idx].len = len;
+ fake_text[sel_idx].flags = RSPAMD_TEXT_FLAG_FAKE;
+
+ return &fake_text[sel_idx];
+ }
+
+ return NULL;
+}
+
+struct rspamd_lua_text *
+lua_new_text(lua_State *L, const gchar *start, gsize len, gboolean own)
+{
+ struct rspamd_lua_text *t;
+
+ t = lua_newuserdata(L, sizeof(*t));
+ t->flags = 0;
+
+ if (own) {
+ gchar *storage;
+
+ if (len > 0) {
+ storage = g_malloc(len);
+
+ if (start != NULL) {
+ memcpy(storage, start, len);
+ }
+
+ t->start = storage;
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ t->start = "";
+ }
+ }
+ else {
+ t->start = start;
+ }
+
+ t->len = len;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ return t;
+}
+
+struct rspamd_lua_text *
+lua_new_text_task(lua_State *L, struct rspamd_task *task,
+ const gchar *start, gsize len, gboolean own)
+{
+ struct rspamd_lua_text *t;
+
+ t = lua_newuserdata(L, sizeof(*t));
+ t->flags = 0;
+
+ if (own) {
+ gchar *storage;
+
+ if (len > 0) {
+ storage = rspamd_mempool_alloc(task->task_pool, len);
+
+ if (start != NULL) {
+ memcpy(storage, start, len);
+ }
+
+ t->start = storage;
+ }
+ else {
+ t->start = "";
+ }
+ }
+ else {
+ t->start = start;
+ }
+
+ t->len = len;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ return t;
+}
+
+bool lua_is_text_binary(struct rspamd_lua_text *t)
+{
+ if (t == NULL || t->len == 0) {
+ return false;
+ }
+
+ if (rspamd_str_has_8bit(t->start, t->len)) {
+ if (rspamd_fast_utf8_validate(t->start, t->len) == 0) {
+ return false;
+ }
+ return true;
+ }
+
+ return false;
+}
+
+
+static gint
+lua_text_fromstring(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *str;
+ gsize l = 0;
+ gboolean transparent = FALSE;
+
+ str = luaL_checklstring(L, 1, &l);
+
+ if (str) {
+ if (lua_isboolean(L, 2)) {
+ transparent = lua_toboolean(L, 2);
+ }
+
+ lua_new_text(L, str, l, !transparent);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+static gint
+lua_text_null(lua_State *L)
+{
+ LUA_TRACE_POINT;
+
+ lua_new_text(L, NULL, 0, false);
+
+ return 1;
+}
+
+static gint
+lua_text_randombytes(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ guint nbytes = luaL_checkinteger(L, 1);
+ struct rspamd_lua_text *out;
+
+ out = lua_new_text(L, NULL, nbytes, TRUE);
+ randombytes_buf((char *) out->start, nbytes);
+ out->len = nbytes;
+
+ return 1;
+}
+
+#define MAX_REC 10
+
+static void
+lua_text_tbl_length(lua_State *L, gsize dlen, gsize *dest, guint rec)
+{
+ gsize tblen, stlen;
+ struct rspamd_lua_text *elt;
+
+ if (rec > MAX_REC) {
+ luaL_error(L, "lua_text_tbl_length: recursion limit exceeded");
+
+ return;
+ }
+
+ tblen = rspamd_lua_table_size(L, -1);
+
+ for (gsize i = 0; i < tblen; i++) {
+ lua_rawgeti(L, -1, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+#if LUA_VERSION_NUM >= 502
+ stlen = lua_rawlen(L, -1);
+#else
+ stlen = lua_objlen(L, -1);
+#endif
+ (*dest) += stlen;
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ elt = (struct rspamd_lua_text *) lua_touserdata(L, -1);
+
+ if (elt) {
+ (*dest) += elt->len;
+ }
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_text_tbl_length(L, dlen, dest, rec + 1);
+ }
+
+ if (i != tblen - 1) {
+ (*dest) += dlen;
+ }
+
+ lua_pop(L, 1);
+ }
+}
+
+static void
+lua_text_tbl_append(lua_State *L,
+ const gchar *delim,
+ gsize dlen,
+ gchar **dest,
+ guint rec)
+{
+ const gchar *st;
+ gsize tblen, stlen;
+ struct rspamd_lua_text *elt;
+
+ if (rec > MAX_REC) {
+ luaL_error(L, "lua_text_tbl_length: recursion limit exceeded");
+
+ return;
+ }
+
+ tblen = rspamd_lua_table_size(L, -1);
+
+ for (guint i = 0; i < tblen; i++) {
+ lua_rawgeti(L, -1, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ st = lua_tolstring(L, -1, &stlen);
+ memcpy((*dest), st, stlen);
+ (*dest) += stlen;
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ elt = (struct rspamd_lua_text *) lua_touserdata(L, -1);
+
+ if (elt) {
+ memcpy((*dest), elt->start, elt->len);
+ (*dest) += elt->len;
+ }
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_text_tbl_append(L, delim, dlen, dest, rec + 1);
+ }
+
+ if (dlen && i != tblen - 1) {
+ memcpy((*dest), delim, dlen);
+ (*dest) += dlen;
+ }
+
+ lua_pop(L, 1);
+ }
+}
+
+static gint
+lua_text_fromtable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *delim = "";
+ struct rspamd_lua_text *t;
+ gsize textlen = 0, dlen, oldtop = lua_gettop(L);
+ gchar *dest;
+
+ if (!lua_istable(L, 1)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ delim = lua_tolstring(L, 2, &dlen);
+ }
+ else {
+ dlen = 0;
+ }
+
+ /* Calculate length needed */
+ lua_pushvalue(L, 1);
+ lua_text_tbl_length(L, dlen, &textlen, 0);
+ lua_pop(L, 1);
+
+ /* Allocate new text */
+ t = lua_newuserdata(L, sizeof(*t));
+ dest = g_malloc(textlen);
+ t->start = dest;
+ t->len = textlen;
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ lua_pushvalue(L, 1);
+ lua_text_tbl_append(L, delim, dlen, &dest, 0);
+ lua_pop(L, 1); /* Table arg */
+
+ gint newtop = lua_gettop(L);
+ g_assert(newtop == oldtop + 1);
+
+ return 1;
+}
+
+static gint
+lua_text_len(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gsize l = 0;
+
+ if (t != NULL) {
+ l = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushinteger(L, l);
+
+ return 1;
+}
+
+static gint
+lua_text_str(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ lua_pushlstring(L, t->start, t->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_ptr(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ lua_pushlightuserdata(L, (gpointer) t->start);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_take_ownership(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gchar *dest;
+
+ if (t != NULL) {
+ if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ /* We already own it */
+ lua_pushboolean(L, true);
+ }
+ else {
+ dest = g_malloc(t->len);
+ memcpy(dest, t->start, t->len);
+ t->start = dest;
+ t->flags |= RSPAMD_TEXT_FLAG_OWN;
+ lua_pushboolean(L, true);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_span(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gint64 start = lua_tointeger(L, 2), len = -1;
+
+ if (t && start >= 1 && start <= t->len) {
+ if (lua_isnumber(L, 3)) {
+ len = lua_tonumber(L, 3);
+ }
+
+ if (len == -1) {
+ len = t->len - (start - 1);
+ }
+
+ if (len < 0 || (len > (t->len - (start - 1)))) {
+ return luaL_error(L, "invalid length");
+ }
+
+ lua_new_text(L, t->start + (start - 1), len, FALSE);
+ }
+ else {
+ if (!t) {
+ return luaL_error(L, "invalid arguments, text required");
+ }
+ else {
+ return luaL_error(L, "invalid arguments: start offset %d "
+ "is larger than text len %d",
+ (int) start, (int) t->len);
+ }
+ }
+
+ return 1;
+}
+
+/* Helpers to behave exactly as Lua does */
+static inline gsize
+relative_pos_start(gint pos, gsize len)
+{
+ if (pos > 0) {
+ return pos;
+ }
+ else if (pos == 0) {
+ return 1;
+ }
+ else if (pos < -((gint) len)) {
+ return 1;
+ }
+
+ /* Negative pos inside str */
+ return len + ((gsize) pos) + 1;
+}
+
+static inline gsize
+relative_pos_end(gint pos, gsize len)
+{
+ if (pos > (gint) len) {
+ return len;
+ }
+ else if (pos >= 0) {
+ return (size_t) pos;
+ }
+ else if (pos < -((gint) len)) {
+ return 0;
+ }
+
+ return len + ((gsize) pos) + 1;
+}
+
+static gint
+lua_text_sub(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t) {
+ size_t start = relative_pos_start(luaL_checkinteger(L, 2),
+ t->len);
+ size_t end = relative_pos_end(luaL_optinteger(L, 3, -1),
+ t->len);
+
+
+ if (start <= end) {
+ lua_new_text(L, t->start + (start - 1),
+ (end - start) + 1, FALSE);
+ }
+ else {
+ lua_new_text(L, "", 0, TRUE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint64
+rspamd_lua_text_push_line(lua_State *L,
+ struct rspamd_lua_text *t,
+ gint64 start_offset,
+ const gchar *sep_pos,
+ gboolean stringify)
+{
+ const gchar *start;
+ gsize len;
+ gint64 ret;
+
+ start = t->start + start_offset;
+ len = sep_pos ? (sep_pos - start) : (t->len - start_offset);
+ ret = start_offset + len;
+
+ /* Trim line */
+ while (len > 0) {
+ if (start[len - 1] == '\r' || start[len - 1] == '\n') {
+ len--;
+ }
+ else {
+ break;
+ }
+ }
+
+ if (stringify) {
+ lua_pushlstring(L, start, len);
+ }
+ else {
+ struct rspamd_lua_text *ntext;
+
+ ntext = lua_newuserdata(L, sizeof(*ntext));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ ntext->start = start;
+ ntext->len = len;
+ ntext->flags = 0; /* Not own as it must be owned by a top object */
+ }
+
+ return ret;
+}
+
+static gint
+rspamd_lua_text_readline(lua_State *L)
+{
+ struct rspamd_lua_text *t = lua_touserdata(L, lua_upvalueindex(1));
+ gboolean stringify = lua_toboolean(L, lua_upvalueindex(2));
+ gint64 pos = lua_tointeger(L, lua_upvalueindex(3));
+
+ if (pos < 0) {
+ return luaL_error(L, "invalid pos: %d", (gint) pos);
+ }
+
+ if (pos >= t->len) {
+ /* We are done */
+ return 0;
+ }
+
+ const gchar *sep_pos;
+
+ /* We look just for `\n` ignoring `\r` as it is very rare nowadays */
+ sep_pos = memchr(t->start + pos, '\n', t->len - pos);
+
+ if (sep_pos == NULL) {
+ /* Either last `\n` or `\r` separated text */
+ sep_pos = memchr(t->start + pos, '\r', t->len - pos);
+ }
+
+ pos = rspamd_lua_text_push_line(L, t, pos, sep_pos, stringify);
+
+ /* Skip separators */
+ while (pos < t->len) {
+ if (t->start[pos] == '\n' || t->start[pos] == '\r') {
+ pos++;
+ }
+ else {
+ break;
+ }
+ }
+
+ /* Update pos */
+ lua_pushinteger(L, pos);
+ lua_replace(L, lua_upvalueindex(3));
+
+ return 1;
+}
+
+static gint
+lua_text_lines(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gboolean stringify = FALSE;
+
+ if (t) {
+ if (lua_isboolean(L, 2)) {
+ stringify = lua_toboolean(L, 2);
+ }
+
+ lua_pushvalue(L, 1);
+ lua_pushboolean(L, stringify);
+ lua_pushinteger(L, 0); /* Current pos */
+ lua_pushcclosure(L, rspamd_lua_text_readline, 3);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+rspamd_lua_text_regexp_split(lua_State *L)
+{
+ struct rspamd_lua_text *t = lua_touserdata(L, lua_upvalueindex(1)),
+ *new_t;
+ struct rspamd_lua_regexp *re = *(struct rspamd_lua_regexp **)
+ lua_touserdata(L, lua_upvalueindex(2));
+ gboolean stringify = lua_toboolean(L, lua_upvalueindex(3));
+ gint64 pos = lua_tointeger(L, lua_upvalueindex(4));
+ gboolean matched;
+
+ if (pos < 0) {
+ return luaL_error(L, "invalid pos: %d", (gint) pos);
+ }
+
+ if (pos >= t->len) {
+ /* We are done */
+ return 0;
+ }
+
+ const gchar *start, *end, *old_start;
+
+ end = t->start + pos;
+
+ for (;;) {
+ old_start = end;
+
+ matched = rspamd_regexp_search(re->re, t->start, t->len, &start, &end, FALSE,
+ NULL);
+
+ if (matched) {
+ if (start - old_start > 0) {
+ if (stringify) {
+ lua_pushlstring(L, old_start, start - old_start);
+ }
+ else {
+ new_t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ new_t->start = old_start;
+ new_t->len = start - old_start;
+ new_t->flags = 0;
+ }
+
+ break;
+ }
+ else {
+ if (start == end) {
+ matched = FALSE;
+ break;
+ }
+ /*
+ * All match separators (e.g. starting separator,
+ * we need to skip it). Continue iterations.
+ */
+ }
+ }
+ else {
+ /* No match, stop */
+ break;
+ }
+ }
+
+ if (!matched && (t->len > 0 && (end == NULL || end < t->start + t->len))) {
+ /* No more matches, but we might need to push the last element */
+ if (end == NULL) {
+ end = t->start;
+ }
+ /* No separators, need to push the whole remaining part */
+ if (stringify) {
+ lua_pushlstring(L, end, (t->start + t->len) - end);
+ }
+ else {
+ new_t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ new_t->start = end;
+ new_t->len = (t->start + t->len) - end;
+ new_t->flags = 0;
+ }
+
+ pos = t->len;
+ }
+ else {
+
+ pos = end - t->start;
+ }
+
+ /* Update pos */
+ lua_pushinteger(L, pos);
+ lua_replace(L, lua_upvalueindex(4));
+
+ return 1;
+}
+
+static gint
+lua_text_split(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ struct rspamd_lua_regexp *re;
+ gboolean stringify = FALSE, own_re = FALSE;
+
+ if (t == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ re = lua_check_regexp(L, 2);
+ }
+ else {
+ rspamd_regexp_t *c_re;
+ GError *err = NULL;
+
+ c_re = rspamd_regexp_new(lua_tostring(L, 2), NULL, &err);
+ if (c_re == NULL) {
+
+ gint ret = luaL_error(L, "cannot parse regexp: %s, error: %s",
+ lua_tostring(L, 2),
+ err == NULL ? "undefined" : err->message);
+ if (err) {
+ g_error_free(err);
+ }
+
+ return ret;
+ }
+
+ re = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ re->re = c_re;
+ re->re_pattern = g_strdup(lua_tostring(L, 2));
+ re->module = rspamd_lua_get_module_name(L);
+ own_re = TRUE;
+ }
+
+ if (re) {
+ if (lua_isboolean(L, 3)) {
+ stringify = lua_toboolean(L, 3);
+ }
+
+ /* Upvalues */
+ lua_pushvalue(L, 1); /* text */
+
+ if (own_re) {
+ struct rspamd_lua_regexp **pre;
+ pre = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pre = re;
+ }
+ else {
+ lua_pushvalue(L, 2); /* regexp */
+ }
+
+ lua_pushboolean(L, stringify);
+ lua_pushinteger(L, 0); /* Current pos */
+ lua_pushcclosure(L, rspamd_lua_text_regexp_split, 4);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_text_at(lua_State *L)
+{
+ return lua_text_byte(L);
+}
+
+static gint
+lua_text_byte(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ gsize start = relative_pos_start(luaL_optinteger(L, 2, 1), t->len);
+ gsize end = relative_pos_end(luaL_optinteger(L, 3, start), t->len);
+ start--;
+
+ if (start >= end) {
+ return 0;
+ }
+
+ for (gsize i = start; i < end; i++) {
+ lua_pushinteger(L, t->start[i]);
+ }
+ return end - start;
+}
+
+static gint
+lua_text_memchr(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ int c;
+ bool reverse = false;
+
+ if (lua_isnumber(L, 2)) {
+ c = lua_tonumber(L, 2);
+ }
+ else {
+ gsize l;
+ const gchar *str = lua_tolstring(L, 2, &l);
+
+ if (str) {
+ c = str[0];
+
+ if (l != 1) {
+ return luaL_error(L, "need exactly one character to search");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+ }
+
+ if (t) {
+ void *f;
+
+ if (lua_isboolean(L, 3)) {
+ reverse = lua_toboolean(L, 3);
+ }
+
+ if (reverse) {
+ f = rspamd_memrchr(t->start, c, t->len);
+ }
+ else {
+ f = memchr(t->start, c, t->len);
+ }
+
+ if (f) {
+ lua_pushinteger(L, ((const char *) f) - t->start + 1);
+ }
+ else {
+ lua_pushinteger(L, -1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_bytes(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t) {
+ lua_createtable(L, t->len, 0);
+
+ for (gsize i = 0; i < t->len; i++) {
+ lua_pushinteger(L, (guchar) t->start[i]);
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_save_in_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ const gchar *fname = NULL;
+ guint mode = 00644;
+ gint fd = -1;
+ gboolean need_close = FALSE;
+
+ if (t != NULL) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ fname = luaL_checkstring(L, 2);
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ mode = lua_tointeger(L, 3);
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TNUMBER) {
+ /* Created fd */
+ fd = lua_tointeger(L, 2);
+ }
+
+ if (fd == -1) {
+ if (fname) {
+ fd = rspamd_file_xopen(fname, O_CREAT | O_WRONLY | O_EXCL, mode, 0);
+
+ if (fd == -1) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+ need_close = TRUE;
+ }
+ else {
+ fd = STDOUT_FILENO;
+ }
+ }
+
+ if (write(fd, t->start, t->len) == -1) {
+ if (fd != STDOUT_FILENO) {
+ close(fd);
+ }
+
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ if (need_close) {
+ close(fd);
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ g_assert(!(t->flags & RSPAMD_TEXT_FLAG_FAKE));
+
+ if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ if (t->flags & RSPAMD_TEXT_FLAG_WIPE) {
+ rspamd_explicit_memzero((guchar *) t->start, t->len);
+ }
+
+ if (t->flags & RSPAMD_TEXT_FLAG_MMAPED) {
+ munmap((gpointer) t->start, t->len);
+ }
+ else {
+ if (t->flags & RSPAMD_TEXT_FLAG_SYSMALLOC) {
+ free((gpointer) t->start);
+ }
+ else {
+ g_free((gpointer) t->start);
+ }
+ }
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_text_eq(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1 = lua_check_text_or_string(L, 1),
+ *t2 = lua_check_text_or_string(L, 2);
+
+ if (t1->len == t2->len) {
+ lua_pushboolean(L, memcmp(t1->start, t2->start, t1->len) == 0);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_lt(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1 = lua_check_text_or_string(L, 1),
+ *t2 = lua_check_text_or_string(L, 2);
+
+ if (t1 && t2) {
+ if (t1->len == t2->len) {
+ lua_pushboolean(L, memcmp(t1->start, t2->start, t1->len) < 0);
+ }
+ else {
+ lua_pushboolean(L, t1->len < t2->len);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_concat(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1 = lua_check_text_or_string(L, 1),
+ *t2 = lua_check_text_or_string(L, 2);
+
+ if (t1 && t2) {
+ struct rspamd_lua_text *final;
+
+ final = lua_new_text(L, NULL, t1->len + t2->len, TRUE);
+ memcpy((void *) final->start, t1->start, t1->len);
+ memcpy((void *) (final->start + t1->len), t2->start, t2->len);
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_wipe(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ rspamd_explicit_memzero((guchar *) t->start, t->len);
+ }
+ else {
+ return luaL_error(L, "cannot wipe not owned text");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_text_base32(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1), *out;
+ enum rspamd_base32_type btype = RSPAMD_BASE32_DEFAULT;
+
+ if (t != NULL) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ btype = rspamd_base32_decode_type_from_str(lua_tostring(L, 2));
+
+ if (btype == RSPAMD_BASE32_INVALID) {
+ return luaL_error(L, "invalid b32 type: %s", lua_tostring(L, 2));
+ }
+ }
+
+ out = lua_new_text(L, NULL, t->len * 8 / 5 + 2, TRUE);
+ out->len = rspamd_encode_base32_buf(t->start, t->len, (gchar *) out->start,
+ out->len, btype);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_base64(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1), *out;
+ gsize line_len = 0;
+ gboolean fold = FALSE;
+
+ if (t != NULL) {
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ line_len = lua_tointeger(L, 2);
+
+ if (line_len <= 8) {
+ return luaL_error(L, "too small line length (at least 8 is required)");
+ }
+ }
+
+ enum rspamd_newlines_type how = RSPAMD_TASK_NEWLINES_CRLF;
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 3);
+
+ if (g_ascii_strcasecmp(how_str, "cr") == 0) {
+ how = RSPAMD_TASK_NEWLINES_CR;
+ }
+ else if (g_ascii_strcasecmp(how_str, "lf") == 0) {
+ how = RSPAMD_TASK_NEWLINES_LF;
+ }
+ else if (g_ascii_strcasecmp(how_str, "crlf") != 0) {
+ return luaL_error(L, "invalid newline style: %s", how_str);
+ }
+ }
+
+ if (lua_type(L, 4) == LUA_TBOOLEAN) {
+ fold = lua_toboolean(L, 4);
+ }
+
+ gsize sz_len;
+
+ out = lua_newuserdata(L, sizeof(*t));
+ out->flags = RSPAMD_TEXT_FLAG_OWN;
+ out->start = rspamd_encode_base64_common(t->start, t->len,
+ line_len, &sz_len, fold, how);
+ out->len = sz_len;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_hex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1), *out;
+
+ if (t != NULL) {
+
+ out = lua_new_text(L, NULL, t->len * 2, TRUE);
+ out->len = rspamd_encode_hex_buf(t->start, t->len, (gchar *) out->start,
+ out->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_find(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gsize patlen, init = 1;
+ const gchar *pat = luaL_checklstring(L, 2, &patlen);
+
+ if (t != NULL && pat != NULL) {
+
+ if (lua_isnumber(L, 3)) {
+ init = relative_pos_start(lua_tointeger(L, 3), t->len);
+ }
+
+ init--;
+
+ if (init > t->len) {
+ return luaL_error(L, "invalid arguments to find: init too large");
+ }
+
+ goffset pos = rspamd_substring_search(t->start + init,
+ t->len - init,
+ pat, patlen);
+
+ if (pos == -1) {
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ lua_pushinteger(L, pos + 1);
+ lua_pushinteger(L, pos + patlen);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+#define BITOP(a, b, op) \
+ ((a)[(guint64) (b) / (8u * sizeof *(a))] op(guint64) 1 << ((guint64) (b) % (8u * sizeof *(a))))
+
+static gint
+lua_text_exclude_chars(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gssize patlen;
+ const gchar *pat = lua_tolstring(L, 2, &patlen), *p, *end;
+ gchar *dest, *d;
+ guint64 byteset[32 / sizeof(guint64)]; /* Bitset for ascii */
+ gboolean copy = TRUE;
+ guint *plen;
+
+ if (t != NULL && pat && patlen > 0) {
+ if (lua_isboolean(L, 3)) {
+ copy = lua_toboolean(L, 3);
+ }
+ else if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ copy = FALSE;
+ }
+
+ if (!copy) {
+ dest = (gchar *) t->start;
+ plen = &t->len;
+ lua_pushvalue(L, 1); /* Push text as a result */
+ }
+ else {
+ /* We need to copy read only text */
+ struct rspamd_lua_text *nt;
+
+ dest = g_malloc(t->len);
+ nt = lua_newuserdata(L, sizeof(*nt));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ nt->len = t->len;
+ nt->flags = RSPAMD_TEXT_FLAG_OWN;
+ memcpy(dest, t->start, t->len);
+ nt->start = dest;
+ plen = &nt->len;
+ }
+
+ /* Fill pattern bitset */
+ memset(byteset, 0, sizeof byteset);
+
+ while (patlen > 0) {
+ if (*pat == '%') {
+ pat++;
+ patlen--;
+
+ if (patlen > 0) {
+ /*
+ * This stuff assumes little endian, but GUINT64_FROM_LE should
+ * deal with proper conversion
+ */
+ switch (*pat) {
+ case '%':
+ BITOP(byteset, *(guchar *) pat, |=);
+ break;
+ case 's':
+ /* "\r\n\t\f " */
+ byteset[0] |= GUINT64_FROM_LE(0x100003600LLU);
+ break;
+ case 'n':
+ /* newlines: "\r\n" */
+ byteset[0] |= GUINT64_FROM_LE(0x2400LLU);
+ break;
+ case '8':
+ /* 8 bit characters */
+ byteset[2] |= GUINT64_FROM_LE(0xffffffffffffffffLLU);
+ byteset[3] |= GUINT64_FROM_LE(0xffffffffffffffffLLU);
+ break;
+ case 'c':
+ /* Non printable (control) characters */
+ byteset[0] |= GUINT64_FROM_LE(0xffffffffLLU);
+ /* Del character */
+ byteset[1] |= GUINT64_FROM_LE(0x8000000000000000LLU);
+ break;
+ }
+ }
+ else {
+ /* Last '%' */
+ BITOP(byteset, (guchar) '%', |=);
+ }
+ }
+ else {
+ BITOP(byteset, *(guchar *) pat, |=);
+ }
+
+ pat++;
+ patlen--;
+ }
+ for (; patlen > 0 && BITOP(byteset, *(guchar *) pat, |=); pat++, patlen--)
+ ;
+
+ p = t->start;
+ end = t->start + t->len;
+ d = dest;
+
+ while (p < end) {
+ if (!BITOP(byteset, *(guchar *) p, &)) {
+ *d++ = *p;
+ }
+
+ p++;
+ }
+
+ *(plen) = d - dest;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_oneline(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ const gchar *p, *end;
+ gchar *dest, *d;
+ guint64 byteset[32 / sizeof(guint64)]; /* Bitset for ascii */
+ gboolean copy = TRUE, seen_8bit = FALSE;
+ guint *plen;
+
+ if (t != NULL) {
+ if (lua_isboolean(L, 2)) {
+ copy = lua_toboolean(L, 2);
+ }
+ else if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ copy = FALSE;
+ }
+
+ if (!copy) {
+ dest = (gchar *) t->start;
+ plen = &t->len;
+ lua_pushvalue(L, 1); /* Push text as a result */
+ }
+ else {
+ /* We need to copy read only text */
+ struct rspamd_lua_text *nt;
+
+ dest = g_malloc(t->len);
+ nt = lua_newuserdata(L, sizeof(*nt));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ nt->len = t->len;
+ nt->flags = RSPAMD_TEXT_FLAG_OWN;
+ memcpy(dest, t->start, t->len);
+ nt->start = dest;
+ plen = &nt->len;
+ }
+
+ /* Fill pattern bitset */
+ memset(byteset, 0, sizeof byteset);
+ /* All spaces */
+ byteset[0] |= GUINT64_FROM_LE(0x100003600LLU);
+ /* Control characters */
+ byteset[0] |= GUINT64_FROM_LE(0xffffffffLLU);
+ /* Del character */
+ byteset[1] |= GUINT64_FROM_LE(0x8000000000000000LLU);
+ /* 8 bit characters */
+ byteset[2] |= GUINT64_FROM_LE(0xffffffffffffffffLLU);
+ byteset[3] |= GUINT64_FROM_LE(0xffffffffffffffffLLU);
+
+ p = t->start;
+ end = t->start + t->len;
+ d = dest;
+
+ while (p < end) {
+ if (!BITOP(byteset, *(guchar *) p, &)) {
+ *d++ = *p;
+ }
+ else {
+ if ((*(guchar *) p) & 0x80) {
+ seen_8bit = TRUE;
+ *d++ = *p;
+ }
+ else {
+ if (*p == ' ') {
+ if (d != dest) {
+ *d++ = *p++;
+ }
+
+ while (p < end && g_ascii_isspace(*p)) {
+ p++;
+ }
+
+ continue; /* To avoid p++ */
+ }
+ else if (*p == '\r' || *p == '\n') {
+ if (d != dest) {
+ *d++ = ' ';
+ p++;
+ }
+
+ while (p < end && g_ascii_isspace(*p)) {
+ p++;
+ }
+
+ continue; /* To avoid p++ */
+ }
+ }
+ }
+
+ p++;
+ }
+
+ while (d > dest && g_ascii_isspace(*(d - 1))) {
+ d--;
+ }
+
+ if (seen_8bit) {
+ if (rspamd_fast_utf8_validate(dest, d - dest) != 0) {
+ /* Need to make it valid :( */
+ UChar32 uc;
+ goffset err_offset;
+ gsize remain = d - dest;
+ gchar *nd = dest;
+
+ while (remain > 0 && (err_offset = rspamd_fast_utf8_validate(nd, remain)) > 0) {
+ gint i = 0;
+
+ err_offset--; /* As it returns it 1 indexed */
+ nd += err_offset;
+ remain -= err_offset;
+
+ /* Each invalid character of input requires 3 bytes of output (+2 bytes) */
+ while (i < remain) {
+ gint old_pos = i;
+ U8_NEXT(nd, i, remain, uc);
+
+ if (uc < 0) {
+ nd[old_pos] = '?';
+ }
+ else {
+ break;
+ }
+ }
+
+ nd += i;
+ remain -= i;
+ }
+ }
+ }
+
+ *(plen) = d - dest;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_lower(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1), *nt;
+ gboolean is_utf8 = FALSE, is_inplace = FALSE;
+
+ if (t != NULL) {
+ if (lua_isboolean(L, 2)) {
+ is_utf8 = lua_toboolean(L, 2);
+ }
+ if (lua_isboolean(L, 3)) {
+ is_inplace = lua_toboolean(L, 3);
+ }
+
+ if (is_inplace) {
+ nt = t;
+ lua_pushvalue(L, 1);
+ }
+ else {
+ nt = lua_new_text(L, t->start, t->len, TRUE);
+ }
+
+ if (!is_utf8) {
+ rspamd_str_lc((gchar *) nt->start, nt->len);
+ }
+ else {
+ rspamd_str_lc_utf8((gchar *) nt->start, nt->len);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_strtoul(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t) {
+ unsigned long ll;
+
+ if (rspamd_strtoul(t->start, t->len, &ll)) {
+ lua_pushinteger(L, ll);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/* Used to distinguish lua text metatable */
+static const guint rspamd_lua_text_cookie = 0x2b21ef6fU;
+
+static gint
+lua_load_text(lua_State *L)
+{
+ lua_newtable(L);
+ lua_pushstring(L, "cookie");
+ lua_pushnumber(L, rspamd_lua_text_cookie);
+ lua_settable(L, -3);
+ luaL_register(L, NULL, textlib_f);
+
+ return 1;
+}
+
+void luaopen_text(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{text}", textlib_m);
+ lua_pushstring(L, "cookie");
+ lua_pushnumber(L, rspamd_lua_text_cookie);
+ lua_settable(L, -3);
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "rspamd_text", lua_load_text);
+}
diff --git a/src/lua/lua_thread_pool.cxx b/src/lua/lua_thread_pool.cxx
new file mode 100644
index 0000000..295f33d
--- /dev/null
+++ b/src/lua/lua_thread_pool.cxx
@@ -0,0 +1,369 @@
+/*-
+ * Copyright 2018 Mikhail Galanin
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+
+#include <vector>
+
+#define msg_debug_lua_threads(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_lua_threads_log_id, "lua_threads", NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(lua_threads)
+
+static struct thread_entry *thread_entry_new(lua_State *L);
+static void thread_entry_free(lua_State *L, struct thread_entry *ent);
+
+#define CFG_POOL_GET(cfg) (reinterpret_cast<lua_thread_pool *>((cfg)->lua_thread_pool))
+
+struct lua_thread_pool {
+ std::vector<struct thread_entry *> available_items;
+ lua_State *L;
+ gint max_items;
+ struct thread_entry *running_entry;
+ static const int default_max_items = 100;
+
+ lua_thread_pool(lua_State *L, gint max_items = default_max_items)
+ : L(L), max_items(max_items)
+ {
+ running_entry = nullptr;
+ available_items.reserve(max_items);
+
+ for (auto i = 0; i < MAX(2, max_items / 10); i++) {
+ auto *ent = thread_entry_new(L);
+ available_items.push_back(ent);
+ }
+ }
+
+ ~lua_thread_pool()
+ {
+ for (auto *ent: available_items) {
+ thread_entry_free(L, ent);
+ }
+ }
+
+ auto get_thread() -> struct thread_entry *
+ {
+ struct thread_entry *ent;
+
+ if (!available_items.empty()) {
+ ent = available_items.back();
+ available_items.pop_back();
+ }
+ else {
+ ent = thread_entry_new(L);
+ }
+
+ running_entry = ent;
+
+ return ent;
+ }
+
+ auto return_thread(struct thread_entry *thread_entry, const gchar *loc) -> void
+ {
+ /* we can't return a running/yielded thread into the pool */
+ g_assert(lua_status(thread_entry->lua_state) == 0);
+
+ if (running_entry == thread_entry) {
+ running_entry = NULL;
+ }
+
+ if (available_items.size() <= max_items) {
+ thread_entry->cd = NULL;
+ thread_entry->finish_callback = NULL;
+ thread_entry->error_callback = NULL;
+ thread_entry->task = NULL;
+ thread_entry->cfg = NULL;
+
+ msg_debug_lua_threads("%s: returned thread to the threads pool %ud items",
+ loc,
+ available_items.size());
+
+ available_items.push_back(thread_entry);
+ }
+ else {
+ msg_debug_lua_threads("%s: removed thread as thread pool has %ud items",
+ loc,
+ available_items.size());
+ thread_entry_free(L, thread_entry);
+ }
+ }
+
+ auto terminate_thread(struct thread_entry *thread_entry,
+ const gchar *loc,
+ bool enforce) -> void
+ {
+ struct thread_entry *ent = NULL;
+
+ if (!enforce) {
+ /* we should only terminate failed threads */
+ g_assert(lua_status(thread_entry->lua_state) != 0 &&
+ lua_status(thread_entry->lua_state) != LUA_YIELD);
+ }
+
+ if (running_entry == thread_entry) {
+ running_entry = NULL;
+ }
+
+ msg_debug_lua_threads("%s: terminated thread entry", loc);
+ thread_entry_free(L, thread_entry);
+
+ if (available_items.size() <= max_items) {
+ ent = thread_entry_new(L);
+ available_items.push_back(ent);
+ }
+ }
+
+ auto get_running_entry(void) -> struct thread_entry *
+ {
+ return running_entry;
+ };
+
+ auto set_running_entry(struct thread_entry *ent) -> struct thread_entry *
+ {
+ auto *old_entry = running_entry;
+ running_entry = ent;
+ return old_entry;
+ };
+};
+
+static struct thread_entry *
+thread_entry_new(lua_State *L)
+{
+ struct thread_entry *ent;
+ ent = g_new0(struct thread_entry, 1);
+ ent->lua_state = lua_newthread(L);
+ ent->thread_index = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ return ent;
+}
+
+static void
+thread_entry_free(lua_State *L, struct thread_entry *ent)
+{
+ luaL_unref(L, LUA_REGISTRYINDEX, ent->thread_index);
+ g_free(ent);
+}
+
+struct lua_thread_pool *
+lua_thread_pool_new(lua_State *L)
+{
+ auto *pool = new lua_thread_pool(L);
+ return pool;
+}
+
+void lua_thread_pool_free(struct lua_thread_pool *pool)
+{
+ delete pool;
+}
+
+
+struct thread_entry *
+lua_thread_pool_get_for_task(struct rspamd_task *task)
+{
+ struct thread_entry *ent = CFG_POOL_GET(task->cfg)->get_thread();
+
+ ent->task = task;
+
+ return ent;
+}
+
+struct thread_entry *
+lua_thread_pool_get_for_config(struct rspamd_config *cfg)
+{
+ struct thread_entry *ent = CFG_POOL_GET(cfg)->get_thread();
+
+ ent->cfg = cfg;
+
+ return ent;
+}
+
+void lua_thread_pool_return_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry, const gchar *loc)
+{
+ pool->return_thread(thread_entry, loc);
+}
+
+void lua_thread_pool_terminate_entry_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry, const gchar *loc,
+ bool enforce)
+{
+ pool->terminate_thread(thread_entry, loc, enforce);
+}
+
+struct thread_entry *
+lua_thread_pool_get_running_entry_full(struct lua_thread_pool *pool,
+ const gchar *loc)
+{
+ msg_debug_lua_threads("%s: lua_thread_pool_get_running_entry_full", loc);
+ return pool->get_running_entry();
+}
+
+void lua_thread_pool_set_running_entry_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry,
+ const gchar *loc)
+{
+ msg_debug_lua_threads("%s: lua_thread_pool_set_running_entry_full", loc);
+ pool->set_running_entry(thread_entry);
+}
+
+static void
+lua_thread_pool_set_running_entry_for_thread(struct thread_entry *thread_entry,
+ const gchar *loc)
+{
+ struct lua_thread_pool *pool;
+
+ if (thread_entry->task) {
+ pool = CFG_POOL_GET(thread_entry->task->cfg);
+ }
+ else {
+ pool = CFG_POOL_GET(thread_entry->cfg);
+ }
+
+ lua_thread_pool_set_running_entry_full(pool, thread_entry, loc);
+}
+
+void lua_thread_pool_prepare_callback_full(struct lua_thread_pool *pool,
+ struct lua_callback_state *cbs,
+ const gchar *loc)
+{
+ msg_debug_lua_threads("%s: lua_thread_pool_prepare_callback_full", loc);
+ cbs->thread_pool = pool;
+ cbs->previous_thread = lua_thread_pool_get_running_entry_full(pool, loc);
+ cbs->my_thread = pool->get_thread();
+ cbs->L = cbs->my_thread->lua_state;
+}
+
+void lua_thread_pool_restore_callback_full(struct lua_callback_state *cbs,
+ const gchar *loc)
+{
+ lua_thread_pool_return_full(cbs->thread_pool, cbs->my_thread, loc);
+ lua_thread_pool_set_running_entry_full(cbs->thread_pool,
+ cbs->previous_thread, loc);
+}
+
+static gint
+lua_do_resume_full(lua_State *L, gint narg, const gchar *loc)
+{
+#if LUA_VERSION_NUM >= 504
+ int nres;
+#endif
+ msg_debug_lua_threads("%s: lua_do_resume_full", loc);
+#if LUA_VERSION_NUM < 502
+ return lua_resume(L, narg);
+#else
+#if LUA_VERSION_NUM >= 504
+ return lua_resume(L, NULL, narg, &nres);
+#else
+ return lua_resume(L, NULL, narg);
+#endif
+#endif
+}
+
+static void
+lua_resume_thread_internal_full(struct thread_entry *thread_entry,
+ gint narg, const gchar *loc)
+{
+ gint ret;
+ struct lua_thread_pool *pool;
+ struct rspamd_task *task;
+
+ msg_debug_lua_threads("%s: lua_resume_thread_internal_full", loc);
+ ret = lua_do_resume_full(thread_entry->lua_state, narg, loc);
+
+ if (ret != LUA_YIELD) {
+ /*
+ LUA_YIELD state should not be handled here.
+ It should only happen when the thread initiated a asynchronous event and it will be restored as soon
+ the event is finished
+ */
+
+ if (thread_entry->task) {
+ pool = CFG_POOL_GET(thread_entry->task->cfg);
+ }
+ else {
+ pool = CFG_POOL_GET(thread_entry->cfg);
+ }
+
+ if (ret == 0) {
+ if (thread_entry->finish_callback) {
+ thread_entry->finish_callback(thread_entry, ret);
+ }
+
+ pool->return_thread(thread_entry, loc);
+ }
+ else {
+ rspamd_lua_traceback(thread_entry->lua_state);
+ if (thread_entry->error_callback) {
+ thread_entry->error_callback(thread_entry, ret,
+ lua_tostring(thread_entry->lua_state, -1));
+ }
+ else if (thread_entry->task) {
+ task = thread_entry->task;
+ msg_err_task("lua call failed (%d): %s", ret,
+ lua_tostring(thread_entry->lua_state, -1));
+ }
+ else {
+ msg_err("lua call failed (%d): %s", ret,
+ lua_tostring(thread_entry->lua_state, -1));
+ }
+
+ /*
+ * Maybe there is a way to recover here.
+ * For now, just remove faulty thread
+ */
+ pool->terminate_thread(thread_entry, loc, false);
+ }
+ }
+}
+
+void lua_thread_resume_full(struct thread_entry *thread_entry, gint narg,
+ const gchar *loc)
+{
+ /*
+ * The only state where we can resume from is LUA_YIELD
+ * Another acceptable status is OK (0) but in that case we should push function on stack
+ * to start the thread from, which is happening in lua_thread_call(), not in this function.
+ */
+ g_assert(lua_status(thread_entry->lua_state) == LUA_YIELD);
+ msg_debug_lua_threads("%s: lua_thread_resume_full", loc);
+ lua_thread_pool_set_running_entry_for_thread(thread_entry, loc);
+ lua_resume_thread_internal_full(thread_entry, narg, loc);
+}
+
+void lua_thread_call_full(struct thread_entry *thread_entry,
+ int narg, const gchar *loc)
+{
+ g_assert(lua_status(thread_entry->lua_state) == 0); /* we can't call running/yielded thread */
+ g_assert(thread_entry->task != NULL || thread_entry->cfg != NULL); /* we can't call without pool */
+
+ lua_resume_thread_internal_full(thread_entry, narg, loc);
+}
+
+gint lua_thread_yield_full(struct thread_entry *thread_entry,
+ gint nresults,
+ const gchar *loc)
+{
+ g_assert(lua_status(thread_entry->lua_state) == 0);
+
+ msg_debug_lua_threads("%s: lua_thread_yield_full", loc);
+ return lua_yield(thread_entry->lua_state, nresults);
+}
diff --git a/src/lua/lua_thread_pool.h b/src/lua/lua_thread_pool.h
new file mode 100644
index 0000000..b612ac3
--- /dev/null
+++ b/src/lua/lua_thread_pool.h
@@ -0,0 +1,194 @@
+#ifndef LUA_THREAD_POOL_H_
+#define LUA_THREAD_POOL_H_
+
+#include <lua.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct thread_entry;
+struct lua_thread_pool;
+
+typedef void (*lua_thread_finish_t)(struct thread_entry *thread, int ret);
+
+typedef void (*lua_thread_error_t)(struct thread_entry *thread, int ret, const char *msg);
+
+struct thread_entry {
+ lua_State *lua_state;
+ gint thread_index;
+ gpointer cd;
+
+ /* function to handle result of called method, can be NULL */
+ lua_thread_finish_t finish_callback;
+
+ /* function to log result, i.e. if you want to modify error logging message or somehow process this state, can be NUL */
+ lua_thread_error_t error_callback;
+ struct rspamd_task *task;
+ struct rspamd_config *cfg;
+};
+
+struct lua_callback_state {
+ lua_State *L;
+ struct thread_entry *my_thread;
+ struct thread_entry *previous_thread;
+ struct lua_thread_pool *thread_pool;
+};
+
+/**
+ * Allocates new thread pool on state L. Pre-creates number of lua-threads to use later on
+ *
+ * @param L
+ * @return
+ */
+struct lua_thread_pool *
+lua_thread_pool_new(lua_State *L);
+
+/**
+ * Destroys the pool
+ * @param pool
+ */
+void lua_thread_pool_free(struct lua_thread_pool *pool);
+
+/**
+ * Extracts a thread from the list of available ones.
+ * It immediately becomes the running one and should be used to run a Lua script/function straight away.
+ * as soon as the code is finished, it should be either returned into list of available threads by
+ * calling lua_thread_pool_return() or terminated by calling lua_thread_pool_terminate_entry()
+ * if the code finished with error.
+ *
+ * If the code performed YIELD, the thread is still running and it's live should be controlled by the callee
+ *
+ * @param task
+ * @return
+ */
+struct thread_entry *
+lua_thread_pool_get_for_task(struct rspamd_task *task);
+
+/**
+ * The same, but used when task is not available
+ *
+ * @param cfg
+ * @return
+ */
+struct thread_entry *
+lua_thread_pool_get_for_config(struct rspamd_config *cfg);
+
+/**
+ * Return thread into the list of available ones. It can't be done with yielded or dead threads.
+ *
+ * @param pool
+ * @param thread_entry
+ */
+void lua_thread_pool_return_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry,
+ const gchar *loc);
+
+#define lua_thread_pool_return(pool, thread_entry) \
+ lua_thread_pool_return_full(pool, thread_entry, G_STRLOC)
+
+/**
+ * Currently running thread. Typically needed in yielding point - to fill-up continuation.
+ *
+ * @param pool
+ * @return
+ */
+struct thread_entry *
+lua_thread_pool_get_running_entry_full(struct lua_thread_pool *pool,
+ const gchar *loc);
+
+#define lua_thread_pool_get_running_entry(pool) \
+ lua_thread_pool_get_running_entry_full(pool, G_STRLOC)
+
+/**
+ * Updates currently running thread
+ *
+ * @param pool
+ * @param thread_entry
+ */
+void lua_thread_pool_set_running_entry_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry,
+ const gchar *loc);
+
+#define lua_thread_pool_set_running_entry(pool, thread_entry) \
+ lua_thread_pool_set_running_entry_full(pool, thread_entry, G_STRLOC)
+
+/**
+ * Prevents yielded thread to be used for callback execution. lua_thread_pool_restore_callback() should be called afterwards.
+ *
+ * @param pool
+ * @param cbs
+ */
+void lua_thread_pool_prepare_callback_full(struct lua_thread_pool *pool,
+ struct lua_callback_state *cbs, const gchar *loc);
+
+#define lua_thread_pool_prepare_callback(pool, cbs) \
+ lua_thread_pool_prepare_callback_full(pool, cbs, G_STRLOC)
+
+/**
+ * Restores state after lua_thread_pool_prepare_callback () usage
+ *
+ * @param cbs
+ */
+void lua_thread_pool_restore_callback_full(struct lua_callback_state *cbs,
+ const gchar *loc);
+
+#define lua_thread_pool_restore_callback(cbs) \
+ lua_thread_pool_restore_callback_full(cbs, G_STRLOC)
+
+/**
+ * Acts like lua_call but the tread is able to suspend execution.
+ * As soon as the call is over, call either thread_entry::finish_callback or thread_entry::error_callback.
+ *
+ * @param thread_entry
+ * @param narg
+ */
+void lua_thread_call_full(struct thread_entry *thread_entry,
+ int narg,
+ const gchar *loc);
+
+#define lua_thread_call(thread_entry, narg) \
+ lua_thread_call_full(thread_entry, narg, G_STRLOC)
+
+/**
+ * Yields thread. should be only called in return statement
+ * @param thread_entry
+ * @param nresults
+ * @return
+ */
+int lua_thread_yield_full(struct thread_entry *thread_entry, int nresults,
+ const gchar *loc);
+
+#define lua_thread_yield(thread_entry, narg) \
+ lua_thread_yield_full(thread_entry, narg, G_STRLOC)
+
+/**
+ * Resumes suspended by lua_yield_thread () thread
+ * @param task
+ * @param thread_entry
+ * @param narg
+ */
+void lua_thread_resume_full(struct thread_entry *thread_entry,
+ int narg,
+ const gchar *loc);
+
+#define lua_thread_resume(thread_entry, narg) \
+ lua_thread_resume_full(thread_entry, narg, G_STRLOC)
+
+/**
+ * Terminates thread pool entry and fill the pool with another thread entry if needed
+ * @param pool
+ * @param thread_entry
+ * @param loc
+ */
+void lua_thread_pool_terminate_entry_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry,
+ const gchar *loc, bool enforce);
+#define lua_thread_pool_terminate_entry(pool, thread_entry) \
+ lua_thread_pool_terminate_entry_full(pool, thread_entry, G_STRLOC, false)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* LUA_THREAD_POOL_H_ */
diff --git a/src/lua/lua_trie.c b/src/lua/lua_trie.c
new file mode 100644
index 0000000..3b0b55e
--- /dev/null
+++ b/src/lua/lua_trie.c
@@ -0,0 +1,500 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "message.h"
+#include "libutil/multipattern.h"
+
+/***
+ * @module rspamd_trie
+ * Rspamd trie module provides the data structure suitable for searching of many
+ * patterns in arbitrary texts (or binary chunks). The algorithmic complexity of
+ * this algorithm is at most O(n + m + z), where `n` is the length of text, `m` is a length of pattern and `z` is a number of patterns in the text.
+ *
+ * Here is a typical example of trie usage:
+ * @example
+local rspamd_trie = require "rspamd_trie"
+local patterns = {'aab', 'ab', 'bcd\0ef'}
+
+local trie = rspamd_trie.create(patterns)
+
+local function trie_callback(number, pos)
+ print('Matched pattern number ' .. tostring(number) .. ' at pos: ' .. tostring(pos))
+end
+
+trie:match('some big text', trie_callback)
+ */
+
+/* Suffix trie */
+LUA_FUNCTION_DEF(trie, create);
+LUA_FUNCTION_DEF(trie, has_hyperscan);
+LUA_FUNCTION_DEF(trie, match);
+LUA_FUNCTION_DEF(trie, search_mime);
+LUA_FUNCTION_DEF(trie, search_rawmsg);
+LUA_FUNCTION_DEF(trie, search_rawbody);
+LUA_FUNCTION_DEF(trie, destroy);
+
+static const struct luaL_reg trielib_m[] = {
+ LUA_INTERFACE_DEF(trie, match),
+ LUA_INTERFACE_DEF(trie, search_mime),
+ LUA_INTERFACE_DEF(trie, search_rawmsg),
+ LUA_INTERFACE_DEF(trie, search_rawbody),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_trie_destroy},
+ {NULL, NULL}};
+static const struct luaL_reg trielib_f[] = {
+ LUA_INTERFACE_DEF(trie, create),
+ LUA_INTERFACE_DEF(trie, has_hyperscan),
+ {NULL, NULL}};
+
+static struct rspamd_multipattern *
+lua_check_trie(lua_State *L, gint idx)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{trie}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'trie' expected");
+ return ud ? *((struct rspamd_multipattern **) ud) : NULL;
+}
+
+static gint
+lua_trie_destroy(lua_State *L)
+{
+ struct rspamd_multipattern *trie = lua_check_trie(L, 1);
+
+ if (trie) {
+ rspamd_multipattern_destroy(trie);
+ }
+
+ return 0;
+}
+
+/***
+ * function trie.has_hyperscan()
+ * Checks for hyperscan support
+ *
+ * @return {bool} true if hyperscan is supported
+ */
+static gint
+lua_trie_has_hyperscan(lua_State *L)
+{
+ lua_pushboolean(L, rspamd_multipattern_has_hyperscan());
+ return 1;
+}
+
+/***
+ * function trie.create(patterns, [flags])
+ * Creates new trie data structure
+ * @param {table} array of string patterns
+ * @return {trie} new trie object
+ */
+static gint
+lua_trie_create(lua_State *L)
+{
+ struct rspamd_multipattern *trie, **ptrie;
+ gint npat = 0, flags = RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_GLOB;
+ GError *err = NULL;
+
+ if (lua_isnumber(L, 2)) {
+ flags = lua_tointeger(L, 2);
+ }
+
+ if (!lua_istable(L, 1)) {
+ return luaL_error(L, "lua trie expects array of patterns for now");
+ }
+ else {
+ lua_pushvalue(L, 1);
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ if (lua_isstring(L, -1)) {
+ npat++;
+ }
+
+ lua_pop(L, 1);
+ }
+
+ trie = rspamd_multipattern_create_sized(npat, flags);
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ if (lua_isstring(L, -1)) {
+ const gchar *pat;
+ gsize patlen;
+
+ pat = lua_tolstring(L, -1, &patlen);
+ rspamd_multipattern_add_pattern_len(trie, pat, patlen, flags);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1); /* table */
+
+ if (!rspamd_multipattern_compile(trie, &err)) {
+ msg_err("cannot compile multipattern: %e", err);
+ g_error_free(err);
+ rspamd_multipattern_destroy(trie);
+ lua_pushnil(L);
+ }
+ else {
+ ptrie = lua_newuserdata(L, sizeof(void *));
+ rspamd_lua_setclass(L, "rspamd{trie}", -1);
+ *ptrie = trie;
+ }
+ }
+
+ return 1;
+}
+
+#define PUSH_TRIE_MATCH(L, start, end, report_start) \
+ do { \
+ if (report_start) { \
+ lua_createtable(L, 2, 0); \
+ lua_pushinteger(L, (start)); \
+ lua_rawseti(L, -2, 1); \
+ lua_pushinteger(L, (end)); \
+ lua_rawseti(L, -2, 2); \
+ } \
+ else { \
+ lua_pushinteger(L, (end)); \
+ } \
+ } while (0)
+
+/* Normal callback type */
+static gint
+lua_trie_lua_cb_callback(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint textpos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ lua_State *L = context;
+ gint ret;
+
+ gboolean report_start = lua_toboolean(L, -1);
+
+ /* Function */
+ lua_pushvalue(L, 3);
+ lua_pushinteger(L, strnum + 1);
+
+ PUSH_TRIE_MATCH(L, match_start, textpos, report_start);
+
+ if (lua_pcall(L, 2, 1, 0) != 0) {
+ msg_info("call to trie callback has failed: %s",
+ lua_tostring(L, -1));
+ lua_pop(L, 1);
+
+ return 1;
+ }
+
+ ret = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+
+ return ret;
+}
+
+/* Table like callback, expect result table on top of the stack */
+static gint
+lua_trie_table_callback(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint textpos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ lua_State *L = context;
+
+ gint report_start = lua_toboolean(L, -2);
+ /* Set table, indexed by pattern number */
+ lua_rawgeti(L, -1, strnum + 1);
+
+ if (lua_istable(L, -1)) {
+ /* Already have table, add offset */
+ gsize last = rspamd_lua_table_size(L, -1);
+ PUSH_TRIE_MATCH(L, match_start, textpos, report_start);
+ lua_rawseti(L, -2, last + 1);
+ /* Remove table from the stack */
+ lua_pop(L, 1);
+ }
+ else {
+ /* Pop none */
+ lua_pop(L, 1);
+ /* New table */
+ lua_newtable(L);
+ PUSH_TRIE_MATCH(L, match_start, textpos, report_start);
+ lua_rawseti(L, -2, 1);
+ lua_rawseti(L, -2, strnum + 1);
+ }
+
+ return 0;
+}
+
+/*
+ * We assume that callback argument is at pos 3 and icase is in position 4
+ */
+static gint
+lua_trie_search_str(lua_State *L, struct rspamd_multipattern *trie,
+ const gchar *str, gsize len, rspamd_multipattern_cb_t cb)
+{
+ gint ret;
+ guint nfound = 0;
+
+ if ((ret = rspamd_multipattern_lookup(trie, str, len,
+ cb, L, &nfound)) == 0) {
+ return nfound;
+ }
+
+ return ret;
+}
+
+/***
+ * @method trie:match(input, [cb][, report_start])
+ * Search for patterns in `input` invoking `cb` optionally ignoring case
+ * @param {table or string} input one or several (if `input` is an array) strings of input text
+ * @param {function} cb callback called on each pattern match in form `function (idx, pos)` where `idx` is a numeric index of pattern (starting from 1) and `pos` is a numeric offset where the pattern ends
+ * @param {boolean} report_start report both start and end offset when matching patterns
+ * @return {boolean} `true` if any pattern has been found (`cb` might be called multiple times however). If `cb` is not defined then it returns a table of match positions indexed by pattern number
+ */
+static gint
+lua_trie_match(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_multipattern *trie = lua_check_trie(L, 1);
+ const gchar *text;
+ gsize len;
+ gboolean found = FALSE, report_start = FALSE;
+ struct rspamd_lua_text *t;
+ rspamd_multipattern_cb_t cb = lua_trie_lua_cb_callback;
+
+ gint old_top = lua_gettop(L);
+
+ if (trie) {
+ if (lua_type(L, 3) != LUA_TFUNCTION) {
+ if (lua_isboolean(L, 3)) {
+ report_start = lua_toboolean(L, 3);
+ }
+
+ lua_pushboolean(L, report_start);
+ /* Table like match */
+ lua_newtable(L);
+ cb = lua_trie_table_callback;
+ }
+ else {
+ if (lua_isboolean(L, 4)) {
+ report_start = lua_toboolean(L, 4);
+ }
+ lua_pushboolean(L, report_start);
+ }
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ lua_pushvalue(L, 2);
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ if (lua_isstring(L, -1)) {
+ text = lua_tolstring(L, -1, &len);
+
+ if (lua_trie_search_str(L, trie, text, len, cb)) {
+ found = TRUE;
+ }
+ }
+ else if (lua_isuserdata(L, -1)) {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ if (lua_trie_search_str(L, trie, t->start, t->len, cb)) {
+ found = TRUE;
+ }
+ }
+ }
+ lua_pop(L, 1);
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TSTRING) {
+ text = lua_tolstring(L, 2, &len);
+
+ if (lua_trie_search_str(L, trie, text, len, cb)) {
+ found = TRUE;
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+
+ if (t && lua_trie_search_str(L, trie, t->start, t->len, cb)) {
+ found = TRUE;
+ }
+ }
+ }
+
+ if (lua_type(L, 3) == LUA_TFUNCTION) {
+ lua_settop(L, old_top);
+ lua_pushboolean(L, found);
+ }
+ else {
+ lua_remove(L, -2);
+ }
+
+ return 1;
+}
+
+/***
+ * @method trie:search_mime(task, cb)
+ * This is a helper mehthod to search pattern within text parts of a message in rspamd task
+ * @param {task} task object
+ * @param {function} cb callback called on each pattern match @see trie:match
+ * @param {boolean} caseless if `true` then match ignores symbols case (ASCII only)
+ * @return {boolean} `true` if any pattern has been found (`cb` might be called multiple times however)
+ */
+static gint
+lua_trie_search_mime(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_multipattern *trie = lua_check_trie(L, 1);
+ struct rspamd_task *task = lua_check_task(L, 2);
+ struct rspamd_mime_text_part *part;
+ const gchar *text;
+ gsize len, i;
+ gboolean found = FALSE;
+ rspamd_multipattern_cb_t cb = lua_trie_lua_cb_callback;
+
+ if (trie && task) {
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, part)
+ {
+ if (!IS_TEXT_PART_EMPTY(part) && part->utf_content.len > 0) {
+ text = part->utf_content.begin;
+ len = part->utf_content.len;
+
+ if (lua_trie_search_str(L, trie, text, len, cb) != 0) {
+ found = TRUE;
+ }
+ }
+ }
+ }
+
+ lua_pushboolean(L, found);
+ return 1;
+}
+
+/***
+ * @method trie:search_rawmsg(task, cb[, caseless])
+ * This is a helper mehthod to search pattern within the whole undecoded content of rspamd task
+ * @param {task} task object
+ * @param {function} cb callback called on each pattern match @see trie:match
+ * @param {boolean} caseless if `true` then match ignores symbols case (ASCII only)
+ * @return {boolean} `true` if any pattern has been found (`cb` might be called multiple times however)
+ */
+static gint
+lua_trie_search_rawmsg(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_multipattern *trie = lua_check_trie(L, 1);
+ struct rspamd_task *task = lua_check_task(L, 2);
+ const gchar *text;
+ gsize len;
+ gboolean found = FALSE;
+
+ if (trie && task) {
+ text = task->msg.begin;
+ len = task->msg.len;
+
+ if (lua_trie_search_str(L, trie, text, len, lua_trie_lua_cb_callback) != 0) {
+ found = TRUE;
+ }
+ }
+
+ lua_pushboolean(L, found);
+ return 1;
+}
+
+/***
+ * @method trie:search_rawbody(task, cb[, caseless])
+ * This is a helper mehthod to search pattern within the whole undecoded content of task's body (not including headers)
+ * @param {task} task object
+ * @param {function} cb callback called on each pattern match @see trie:match
+ * @param {boolean} caseless if `true` then match ignores symbols case (ASCII only)
+ * @return {boolean} `true` if any pattern has been found (`cb` might be called multiple times however)
+ */
+static gint
+lua_trie_search_rawbody(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_multipattern *trie = lua_check_trie(L, 1);
+ struct rspamd_task *task = lua_check_task(L, 2);
+ const gchar *text;
+ gsize len;
+ gboolean found = FALSE;
+
+ if (trie && task) {
+ if (MESSAGE_FIELD(task, raw_headers_content).len > 0) {
+ text = task->msg.begin + MESSAGE_FIELD(task, raw_headers_content).len;
+ len = task->msg.len - MESSAGE_FIELD(task, raw_headers_content).len;
+ }
+ else {
+ /* Treat as raw message */
+ text = task->msg.begin;
+ len = task->msg.len;
+ }
+
+ if (lua_trie_search_str(L, trie, text, len, lua_trie_lua_cb_callback) != 0) {
+ found = TRUE;
+ }
+ }
+
+ lua_pushboolean(L, found);
+ return 1;
+}
+
+static gint
+lua_load_trie(lua_State *L)
+{
+ lua_newtable(L);
+
+ /* Flags */
+ lua_pushstring(L, "flags");
+ lua_newtable(L);
+
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_GLOB);
+ lua_setfield(L, -2, "glob");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_RE);
+ lua_setfield(L, -2, "re");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_ICASE);
+ lua_setfield(L, -2, "icase");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_UTF8);
+ lua_setfield(L, -2, "utf8");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_TLD);
+ lua_setfield(L, -2, "tld");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_DOTALL);
+ lua_setfield(L, -2, "dot_all");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_SINGLEMATCH);
+ lua_setfield(L, -2, "single_match");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_NO_START);
+ lua_setfield(L, -2, "no_start");
+ lua_settable(L, -3);
+
+ /* Main content */
+ luaL_register(L, NULL, trielib_f);
+
+ return 1;
+}
+
+void luaopen_trie(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{trie}", trielib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_trie", lua_load_trie);
+}
diff --git a/src/lua/lua_udp.c b/src/lua/lua_udp.c
new file mode 100644
index 0000000..c79e35a
--- /dev/null
+++ b/src/lua/lua_udp.c
@@ -0,0 +1,594 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+#include "utlist.h"
+#include "unix-std.h"
+#include <math.h>
+#include <src/libutil/libev_helper.h>
+
+static const gchar *M = "rspamd lua udp";
+
+/***
+ * @module rspamd_udp
+ * Rspamd UDP module is available from the version 1.9.0 and represents a generic
+ * UDP asynchronous client available from the LUA code.
+ * This module is quite simple: it can either send requests to some address or
+ * it can send requests and wait for replies, potentially handling retransmits.
+ * @example
+local logger = require "rspamd_logger"
+local udp = require "rspamd_udp"
+
+rspamd_config.SYM = function(task)
+ udp.sento{
+ host = addr, -- must be ip address object (e.g. received by upstream module)
+ port = 500,
+ data = {'str1', 'str2'}, -- can be table, string or rspamd_text
+ timeout = 0.5, -- default = 1s
+ task = task, -- if has task
+ session = session, -- optional
+ ev_base = ev_base, -- if no task available
+ -- You can include callback and then Rspamd will try to read replies
+ callback = function(success, data)
+ -- success is bool, data is either data or an error (string)
+ end,
+ retransmits = 0, -- Or more if retransmitting is necessary
+ }
+end
+ */
+
+static const double default_udp_timeout = 1.0;
+
+LUA_FUNCTION_DEF(udp, sendto);
+
+static const struct luaL_reg udp_libf[] = {
+ LUA_INTERFACE_DEF(udp, sendto),
+ {NULL, NULL}};
+
+struct lua_udp_cbdata {
+ struct ev_loop *event_loop;
+ struct rspamd_io_ev ev;
+ struct rspamd_async_event *async_ev;
+ struct rspamd_task *task;
+ rspamd_mempool_t *pool;
+ rspamd_inet_addr_t *addr;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_async_session *s;
+ struct iovec *iov;
+ lua_State *L;
+ guint retransmits;
+ guint iovlen;
+ gint sock;
+ gint cbref;
+ gboolean sent;
+};
+
+#define msg_debug_udp(...) rspamd_conditional_debug_fast(NULL, cbd->addr, \
+ rspamd_lua_udp_log_id, "lua_udp", cbd->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(lua_udp)
+
+static inline void
+lua_fill_iov(lua_State *L, rspamd_mempool_t *pool,
+ struct iovec *iov, gint pos)
+{
+ if (lua_type(L, pos) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, pos);
+
+ if (t) {
+ iov->iov_base = rspamd_mempool_alloc(pool, t->len);
+ iov->iov_len = t->len;
+ memcpy(iov->iov_base, t->start, t->len);
+ }
+ }
+ else {
+ const gchar *s;
+ gsize len;
+
+ s = lua_tolstring(L, pos, &len);
+
+ iov->iov_base = rspamd_mempool_alloc(pool, len);
+ iov->iov_len = len;
+ memcpy(iov->iov_base, s, len);
+ }
+}
+
+static void
+lua_udp_cbd_fin(gpointer p)
+{
+ struct lua_udp_cbdata *cbd = (struct lua_udp_cbdata *) p;
+
+ if (cbd->sock != -1) {
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+ close(cbd->sock);
+ }
+
+ if (cbd->addr) {
+ rspamd_inet_address_free(cbd->addr);
+ }
+
+ if (cbd->cbref) {
+ luaL_unref(cbd->L, LUA_REGISTRYINDEX, cbd->cbref);
+ }
+}
+
+static void
+lua_udp_maybe_free(struct lua_udp_cbdata *cbd)
+{
+ if (cbd->item) {
+ rspamd_symcache_item_async_dec_check(cbd->task, cbd->item, M);
+ cbd->item = NULL;
+ }
+
+ if (cbd->async_ev) {
+ rspamd_session_remove_event(cbd->s, lua_udp_cbd_fin, cbd);
+ }
+ else {
+ lua_udp_cbd_fin(cbd);
+ }
+}
+
+
+enum rspamd_udp_send_result {
+ RSPAMD_SENT_OK,
+ RSPAMD_SENT_RETRY,
+ RSPAMD_SENT_FAILURE
+};
+
+static enum rspamd_udp_send_result
+lua_try_send_request(struct lua_udp_cbdata *cbd)
+{
+ struct msghdr msg;
+ gint r;
+
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = cbd->iov;
+ msg.msg_iovlen = cbd->iovlen;
+ msg.msg_name = rspamd_inet_address_get_sa(cbd->addr, &msg.msg_namelen);
+
+ r = sendmsg(cbd->sock, &msg, 0);
+
+ if (r != -1) {
+ return RSPAMD_SENT_OK;
+ }
+
+ if (errno == EAGAIN || errno == EINTR) {
+ return RSPAMD_SENT_RETRY;
+ }
+
+ return RSPAMD_SENT_FAILURE;
+}
+
+static void
+lua_udp_maybe_push_error(struct lua_udp_cbdata *cbd, const gchar *err)
+{
+ if (cbd->cbref != -1) {
+ gint top;
+ lua_State *L = cbd->L;
+
+ top = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+
+ /* Error message */
+ lua_pushboolean(L, false);
+ lua_pushstring(L, err);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 2, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+ }
+
+ lua_udp_maybe_free(cbd);
+}
+
+static void
+lua_udp_push_data(struct lua_udp_cbdata *cbd, const gchar *data,
+ gssize len)
+{
+ if (cbd->cbref != -1) {
+ gint top;
+ lua_State *L = cbd->L;
+
+ top = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+
+ /* Error message */
+ lua_pushboolean(L, true);
+ lua_pushlstring(L, data, len);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 2, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+ }
+
+ lua_udp_maybe_free(cbd);
+}
+
+static gboolean
+lua_udp_maybe_register_event(struct lua_udp_cbdata *cbd)
+{
+ if (cbd->s && !cbd->async_ev) {
+ if (cbd->item) {
+ cbd->async_ev = rspamd_session_add_event_full(cbd->s, lua_udp_cbd_fin,
+ cbd, M,
+ rspamd_symcache_dyn_item_name(cbd->task, cbd->item));
+ }
+ else {
+ cbd->async_ev = rspamd_session_add_event(cbd->s, lua_udp_cbd_fin,
+ cbd, M);
+ }
+
+ if (!cbd->async_ev) {
+ return FALSE;
+ }
+ }
+
+ if (cbd->task && !cbd->item) {
+ cbd->item = rspamd_symcache_get_cur_item(cbd->task);
+ rspamd_symcache_item_async_inc(cbd->task, cbd->item, M);
+ }
+
+ return TRUE;
+}
+
+static void
+lua_udp_io_handler(gint fd, short what, gpointer p)
+{
+ struct lua_udp_cbdata *cbd = (struct lua_udp_cbdata *) p;
+ gssize r;
+
+ if (what == EV_TIMEOUT) {
+ if (cbd->sent && cbd->retransmits > 0) {
+ r = lua_try_send_request(cbd);
+
+ if (r == RSPAMD_SENT_OK) {
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev, EV_READ);
+ lua_udp_maybe_register_event(cbd);
+ cbd->retransmits--;
+ }
+ else if (r == RSPAMD_SENT_FAILURE) {
+ lua_udp_maybe_push_error(cbd, "write error");
+ }
+ else {
+ cbd->retransmits--;
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev, EV_WRITE);
+ }
+ }
+ else {
+ if (!cbd->sent) {
+ lua_udp_maybe_push_error(cbd, "sent timeout");
+ }
+ else {
+ lua_udp_maybe_push_error(cbd, "read timeout");
+ }
+ }
+ }
+ else if (what == EV_WRITE) {
+ r = lua_try_send_request(cbd);
+
+ if (r == RSPAMD_SENT_OK) {
+ if (cbd->cbref != -1) {
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev, EV_READ);
+ cbd->sent = TRUE;
+ }
+ else {
+ lua_udp_maybe_free(cbd);
+ }
+ }
+ else if (r == RSPAMD_SENT_FAILURE) {
+ lua_udp_maybe_push_error(cbd, "write error");
+ }
+ else {
+ cbd->retransmits--;
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev, EV_WRITE);
+ }
+ }
+ else if (what == EV_READ) {
+ guchar udpbuf[4096];
+ socklen_t slen;
+ struct sockaddr *sa;
+
+ sa = rspamd_inet_address_get_sa(cbd->addr, &slen);
+
+ r = recvfrom(cbd->sock, udpbuf, sizeof(udpbuf), 0, sa, &slen);
+
+ if (r == -1) {
+ lua_udp_maybe_push_error(cbd, strerror(errno));
+ }
+ else {
+ lua_udp_push_data(cbd, udpbuf, r);
+ }
+ }
+}
+
+/***
+ * @function rspamd_udp.sendto({params})
+ * This function simply sends data to an external UDP service
+ *
+ * - `task`: rspamd task objects (implies `pool`, `session` and `ev_base` arguments)
+ * - `ev_base`: event base (if no task specified)
+ * - `session`: events session (no task, optional)
+ * - `pool`: memory pool (if no task specified)
+ * - `host`: IP or name of the peer (required)
+ * - `port`: remote port to use (if `host` has no port part this is required)
+ * - `data`: a table of strings or `rspamd_text` objects that contains data pieces
+ * - `retransmits`: number of retransmits if needed
+ * - `callback`: optional callback if reply should be read
+ * @return {boolean} true if request has been sent (additional string if it has not)
+ */
+static gint
+lua_udp_sendto(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *host;
+ guint port;
+ struct ev_loop *ev_base = NULL;
+ struct lua_udp_cbdata *cbd;
+ struct rspamd_async_session *session = NULL;
+ struct rspamd_task *task = NULL;
+ rspamd_inet_addr_t *addr;
+ rspamd_mempool_t *pool = NULL;
+ gdouble timeout = default_udp_timeout;
+
+ if (lua_type(L, 1) == LUA_TTABLE) {
+ lua_pushstring(L, "port");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ port = lua_tointeger(L, -1);
+ }
+ else {
+ /* We assume that it is a unix socket */
+ port = 0;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "host");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ host = luaL_checkstring(L, -1);
+
+ if (rspamd_parse_inet_address(&addr,
+ host, strlen(host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ if (port != 0) {
+ rspamd_inet_address_set_port(addr, port);
+ }
+ }
+ else {
+ lua_pop(L, 1);
+ return luaL_error(L, "invalid host: %s", host);
+ }
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_ip *lip;
+
+ lip = lua_check_ip(L, -1);
+
+ if (lip == NULL || lip->addr == NULL) {
+ lua_pop(L, 1);
+ return luaL_error(L, "invalid host class");
+ }
+
+ addr = rspamd_inet_address_copy(lip->addr, NULL);
+
+ if (port != 0) {
+ rspamd_inet_address_set_port(addr, port);
+ }
+ }
+ else {
+ lua_pop(L, 1);
+ return luaL_error(L, "invalid host");
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "task");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ task = lua_check_task(L, -1);
+ ev_base = task->event_loop;
+ session = task->s;
+ pool = task->task_pool;
+ }
+ lua_pop(L, 1);
+
+ if (task == NULL) {
+ lua_pushstring(L, "ev_base");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{ev_base}")) {
+ ev_base = *(struct ev_loop **) lua_touserdata(L, -1);
+ }
+ else {
+ ev_base = NULL;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "session");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{session}")) {
+ session = *(struct rspamd_async_session **) lua_touserdata(L, -1);
+ }
+ else {
+ session = NULL;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "pool");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{mempool}")) {
+ pool = *(rspamd_mempool_t **) lua_touserdata(L, -1);
+ }
+ else {
+ pool = NULL;
+ }
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ if (!ev_base || !pool) {
+ rspamd_inet_address_free(addr);
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ cbd = rspamd_mempool_alloc0(pool, sizeof(*cbd));
+ cbd->event_loop = ev_base;
+ cbd->pool = pool;
+ cbd->s = session;
+ cbd->addr = addr;
+ cbd->sock = rspamd_socket_create(rspamd_inet_address_get_af(addr),
+ SOCK_DGRAM, 0, TRUE);
+ cbd->cbref = -1;
+ cbd->ev.timeout = timeout;
+
+ if (cbd->sock == -1) {
+ rspamd_inet_address_free(addr);
+
+ return luaL_error(L, "cannot open socket: %s", strerror(errno));
+ }
+
+ cbd->L = L;
+
+ gsize data_len;
+
+ lua_pushstring(L, "data");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ data_len = rspamd_lua_table_size(L, -1);
+ cbd->iov = rspamd_mempool_alloc(pool,
+ sizeof(*cbd->iov) * data_len);
+
+ for (int i = 0; i < data_len; i++) {
+ lua_rawgeti(L, -1, i + 1);
+ lua_fill_iov(L, pool, &cbd->iov[i], -1);
+ lua_pop(L, 1);
+ }
+
+ cbd->iovlen = data_len;
+ }
+ else {
+ cbd->iov = rspamd_mempool_alloc(pool, sizeof(*cbd->iov));
+ cbd->iovlen = 1;
+ lua_fill_iov(L, pool, cbd->iov, -1);
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "callback");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ cbd->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "retransmits");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ cbd->retransmits = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ enum rspamd_udp_send_result r;
+
+ r = lua_try_send_request(cbd);
+ if (r == RSPAMD_SENT_OK) {
+ if (cbd->cbref == -1) {
+ lua_udp_maybe_free(cbd);
+ }
+ else {
+ if (!lua_udp_maybe_register_event(cbd)) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, "session error");
+ lua_udp_maybe_free(cbd);
+
+ return 2;
+ }
+
+ rspamd_ev_watcher_init(&cbd->ev, cbd->sock, EV_READ,
+ lua_udp_io_handler, cbd);
+ rspamd_ev_watcher_start(cbd->event_loop, &cbd->ev, timeout);
+ cbd->sent = TRUE;
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else if (r == RSPAMD_SENT_FAILURE) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+ lua_udp_maybe_free(cbd);
+
+ return 2;
+ }
+ else {
+ rspamd_ev_watcher_init(&cbd->ev, cbd->sock, EV_WRITE,
+ lua_udp_io_handler, cbd);
+ rspamd_ev_watcher_start(cbd->event_loop, &cbd->ev, timeout);
+
+ if (!lua_udp_maybe_register_event(cbd)) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, "session error");
+ lua_udp_maybe_free(cbd);
+
+ return 2;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_udp(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, udp_libf);
+
+ return 1;
+}
+
+void luaopen_udp(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_udp", lua_load_udp);
+}
diff --git a/src/lua/lua_upstream.c b/src/lua/lua_upstream.c
new file mode 100644
index 0000000..583ee6a
--- /dev/null
+++ b/src/lua/lua_upstream.c
@@ -0,0 +1,672 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "lua_common.h"
+
+
+/***
+ * @module rspamd_upstream_list
+ * This module implements upstreams manipulation from LUA API. This functionality
+ * can be used for load balancing using different strategies including:
+ *
+ * - round-robin: balance upstreams one by one selecting accordingly to their weight
+ * - hash: use stable hashing algorithm to distribute values according to some static strings
+ * - master-slave: always prefer upstream with higher priority unless it is not available
+ *
+ * Here is an example of upstreams manipulations:
+ * @example
+local rspamd_logger = require "rspamd_logger"
+local rspamd_redis = require "rspamd_redis"
+local upstream_list = require "rspamd_upstream_list"
+local upstreams = upstream_list.create('127.0.0.1,10.0.0.1,10.0.0.2', 6379)
+
+local function sym_callback(task)
+ local upstream = upstreams:get_upstream_by_hash(task:get_from()[1]['domain'])
+
+ local function cb(task, err, data)
+ if err then
+ upstream:fail()
+ else
+ upstream:ok()
+ end
+ end
+
+ local addr = upstream:get_addr()
+ rspamd_redis.make_request(task, addr, cb,
+ 'PUSH', {'key', 'value'})
+end
+ */
+/* Upstream list functions */
+LUA_FUNCTION_DEF(upstream_list, create);
+LUA_FUNCTION_DEF(upstream_list, destroy);
+LUA_FUNCTION_DEF(upstream_list, all_upstreams);
+LUA_FUNCTION_DEF(upstream_list, get_upstream_by_hash);
+LUA_FUNCTION_DEF(upstream_list, get_upstream_round_robin);
+LUA_FUNCTION_DEF(upstream_list, get_upstream_master_slave);
+LUA_FUNCTION_DEF(upstream_list, add_watcher);
+
+static const struct luaL_reg upstream_list_m[] = {
+
+ LUA_INTERFACE_DEF(upstream_list, get_upstream_by_hash),
+ LUA_INTERFACE_DEF(upstream_list, get_upstream_round_robin),
+ LUA_INTERFACE_DEF(upstream_list, get_upstream_master_slave),
+ LUA_INTERFACE_DEF(upstream_list, all_upstreams),
+ LUA_INTERFACE_DEF(upstream_list, add_watcher),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_upstream_list_destroy},
+ {NULL, NULL}};
+static const struct luaL_reg upstream_list_f[] = {
+ LUA_INTERFACE_DEF(upstream_list, create),
+ {NULL, NULL}};
+
+/* Upstream functions */
+LUA_FUNCTION_DEF(upstream, ok);
+LUA_FUNCTION_DEF(upstream, fail);
+LUA_FUNCTION_DEF(upstream, get_addr);
+LUA_FUNCTION_DEF(upstream, get_name);
+LUA_FUNCTION_DEF(upstream, get_port);
+LUA_FUNCTION_DEF(upstream, destroy);
+
+static const struct luaL_reg upstream_m[] = {
+ LUA_INTERFACE_DEF(upstream, ok),
+ LUA_INTERFACE_DEF(upstream, fail),
+ LUA_INTERFACE_DEF(upstream, get_addr),
+ LUA_INTERFACE_DEF(upstream, get_port),
+ LUA_INTERFACE_DEF(upstream, get_name),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_upstream_destroy},
+ {NULL, NULL}};
+
+/* Upstream class */
+
+struct rspamd_lua_upstream *
+lua_check_upstream(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{upstream}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'upstream' expected");
+ return ud ? (struct rspamd_lua_upstream *) ud : NULL;
+}
+
+/***
+ * @method upstream:get_addr()
+ * Get ip of upstream
+ * @return {ip} ip address object
+ */
+static gint
+lua_upstream_get_addr(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+
+ if (up) {
+ rspamd_lua_ip_push(L, rspamd_upstream_addr_next(up->up));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method upstream:get_name()
+ * Get name of upstream
+ * @return {string} name of the upstream
+ */
+static gint
+lua_upstream_get_name(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+
+ if (up) {
+ lua_pushstring(L, rspamd_upstream_name(up->up));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method upstream:get_port()
+ * Get port of upstream
+ * @return {int} port of the upstream
+ */
+static gint
+lua_upstream_get_port(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+
+ if (up) {
+ lua_pushinteger(L, rspamd_upstream_port(up->up));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method upstream:fail()
+ * Indicate upstream failure. After certain amount of failures during specified time frame, an upstream is marked as down and does not participate in rotations.
+ */
+static gint
+lua_upstream_fail(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+ gboolean fail_addr = FALSE;
+ const gchar *reason = "unknown";
+
+ if (up) {
+
+ if (lua_isboolean(L, 2)) {
+ fail_addr = lua_toboolean(L, 2);
+
+ if (lua_isstring(L, 3)) {
+ reason = lua_tostring(L, 3);
+ }
+ }
+ else if (lua_isstring(L, 2)) {
+ reason = lua_tostring(L, 2);
+ }
+
+ rspamd_upstream_fail(up->up, fail_addr, reason);
+ }
+
+ return 0;
+}
+
+/***
+ * @method upstream:ok()
+ * Indicates upstream success. Resets errors count for an upstream.
+ */
+static gint
+lua_upstream_ok(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+
+ if (up) {
+ rspamd_upstream_ok(up->up);
+ }
+
+ return 0;
+}
+
+static gint
+lua_upstream_destroy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+
+ if (up) {
+ /* Remove reference to the parent */
+ luaL_unref(L, LUA_REGISTRYINDEX, up->upref);
+ /* Upstream belongs to the upstream list, so no free here */
+ }
+
+ return 0;
+}
+
+/* Upstream list class */
+
+static struct upstream_list *
+lua_check_upstream_list(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{upstream_list}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'upstream_list' expected");
+ return ud ? *((struct upstream_list **) ud) : NULL;
+}
+
+static struct rspamd_lua_upstream *
+lua_push_upstream(lua_State *L, gint up_idx, struct upstream *up)
+{
+ struct rspamd_lua_upstream *lua_ups;
+
+ if (up_idx < 0) {
+ up_idx = lua_gettop(L) + up_idx + 1;
+ }
+
+ lua_ups = lua_newuserdata(L, sizeof(*lua_ups));
+ lua_ups->up = up;
+ rspamd_lua_setclass(L, "rspamd{upstream}", -1);
+ /* Store parent in the upstream to prevent gc */
+ lua_pushvalue(L, up_idx);
+ lua_ups->upref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ return lua_ups;
+}
+
+/***
+ * @function upstream_list.create(cfg, def, [default_port])
+ * Create new upstream list from its string definition in form `<upstream>,<upstream>;<upstream>`
+ * @param {rspamd_config} cfg configuration reference
+ * @param {string} def upstream list definition
+ * @param {number} default_port default port for upstreams
+ * @return {upstream_list} upstream list structure
+ */
+static gint
+lua_upstream_list_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *new = NULL, **pnew;
+ struct rspamd_config *cfg = NULL;
+ const gchar *def;
+ guint default_port = 0;
+ gint top;
+
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ cfg = lua_check_config(L, 1);
+ top = 2;
+ }
+ else {
+ top = 1;
+ }
+
+ if (lua_gettop(L) >= top + 1) {
+ default_port = luaL_checknumber(L, top + 1);
+ }
+
+ if (lua_type(L, top) == LUA_TSTRING) {
+ def = luaL_checkstring(L, top);
+
+ new = rspamd_upstreams_create(cfg ? cfg->ups_ctx : NULL);
+
+ if (rspamd_upstreams_parse_line(new, def, default_port, NULL)) {
+ pnew = lua_newuserdata(L, sizeof(struct upstream_list *));
+ rspamd_lua_setclass(L, "rspamd{upstream_list}", -1);
+ *pnew = new;
+ }
+ else {
+ rspamd_upstreams_destroy(new);
+ lua_pushnil(L);
+ }
+ }
+ else if (lua_type(L, top) == LUA_TTABLE) {
+ new = rspamd_upstreams_create(cfg ? cfg->ups_ctx : NULL);
+ pnew = lua_newuserdata(L, sizeof(struct upstream_list *));
+ rspamd_lua_setclass(L, "rspamd{upstream_list}", -1);
+ *pnew = new;
+
+ lua_pushvalue(L, top);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ def = lua_tostring(L, -1);
+
+ if (!def || !rspamd_upstreams_parse_line(new, def, default_port, NULL)) {
+ msg_warn("cannot parse upstream %s", def);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/**
+ * Destroy a single upstream list object
+ * @param L
+ * @return
+ */
+static gint
+lua_upstream_list_destroy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl = lua_check_upstream_list(L);
+
+ rspamd_upstreams_destroy(upl);
+
+ return 0;
+}
+
+/***
+ * @method upstream_list:get_upstream_by_hash(key)
+ * Get upstream by hash from key
+ * @param {string} key a string used as input for stable hash algorithm
+ * @return {upstream} upstream from a list corresponding to the given key
+ */
+static gint
+lua_upstream_list_get_upstream_by_hash(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl;
+ struct upstream *selected;
+ const gchar *key;
+ gsize keyl;
+
+ upl = lua_check_upstream_list(L);
+ if (upl) {
+ key = luaL_checklstring(L, 2, &keyl);
+ if (key) {
+ selected = rspamd_upstream_get(upl, RSPAMD_UPSTREAM_HASHED, key,
+ (guint) keyl);
+
+ if (selected) {
+ lua_push_upstream(L, 1, selected);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method upstream_list:get_upstream_round_robin()
+ * Get upstream round robin (by current weight)
+ * @return {upstream} upstream from a list in round-robin matter
+ */
+static gint
+lua_upstream_list_get_upstream_round_robin(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl;
+ struct upstream *selected;
+
+ upl = lua_check_upstream_list(L);
+ if (upl) {
+
+ selected = rspamd_upstream_get(upl, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0);
+ if (selected) {
+ lua_push_upstream(L, 1, selected);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method upstream_list:get_upstream_master_slave()
+ * Get upstream master slave order (by static priority)
+ * @return {upstream} upstream from a list in master-slave order
+ */
+static gint
+lua_upstream_list_get_upstream_master_slave(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl;
+ struct upstream *selected;
+
+ upl = lua_check_upstream_list(L);
+ if (upl) {
+
+ selected = rspamd_upstream_get(upl, RSPAMD_UPSTREAM_MASTER_SLAVE,
+ NULL,
+ 0);
+ if (selected) {
+ lua_push_upstream(L, 1, selected);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct upstream_foreach_cbdata {
+ lua_State *L;
+ gint ups_pos;
+};
+
+static void lua_upstream_inserter(struct upstream *up, guint idx, void *ud)
+{
+ struct upstream_foreach_cbdata *cbd = (struct upstream_foreach_cbdata *) ud;
+
+ lua_push_upstream(cbd->L, cbd->ups_pos, up);
+ lua_rawseti(cbd->L, -2, idx + 1);
+}
+/***
+ * @method upstream_list:all_upstreams()
+ * Returns all upstreams for this list
+ * @return {table|upstream} all upstreams defined
+ */
+static gint
+lua_upstream_list_all_upstreams(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl;
+ struct upstream_foreach_cbdata cbd;
+
+ upl = lua_check_upstream_list(L);
+ if (upl) {
+ cbd.L = L;
+ cbd.ups_pos = 1;
+
+ lua_createtable(L, rspamd_upstreams_count(upl), 0);
+ rspamd_upstreams_foreach(upl, lua_upstream_inserter, &cbd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static inline enum rspamd_upstreams_watch_event
+lua_str_to_upstream_flag(const gchar *str)
+{
+ enum rspamd_upstreams_watch_event fl = 0;
+
+ if (strcmp(str, "success") == 0) {
+ fl = RSPAMD_UPSTREAM_WATCH_SUCCESS;
+ }
+ else if (strcmp(str, "failure") == 0) {
+ fl = RSPAMD_UPSTREAM_WATCH_FAILURE;
+ }
+ else if (strcmp(str, "online") == 0) {
+ fl = RSPAMD_UPSTREAM_WATCH_ONLINE;
+ }
+ else if (strcmp(str, "offline") == 0) {
+ fl = RSPAMD_UPSTREAM_WATCH_OFFLINE;
+ }
+ else {
+ msg_err("invalid flag: %s", str);
+ }
+
+ return fl;
+}
+
+static inline const gchar *
+lua_upstream_flag_to_str(enum rspamd_upstreams_watch_event fl)
+{
+ const gchar *res = "unknown";
+
+ /* Works with single flags, not combinations */
+ if (fl & RSPAMD_UPSTREAM_WATCH_SUCCESS) {
+ res = "success";
+ }
+ else if (fl & RSPAMD_UPSTREAM_WATCH_FAILURE) {
+ res = "failure";
+ }
+ else if (fl & RSPAMD_UPSTREAM_WATCH_ONLINE) {
+ res = "online";
+ }
+ else if (fl & RSPAMD_UPSTREAM_WATCH_OFFLINE) {
+ res = "offline";
+ }
+ else {
+ msg_err("invalid flag: %d", fl);
+ }
+
+ return res;
+}
+
+struct rspamd_lua_upstream_watcher_cbdata {
+ lua_State *L;
+ gint cbref;
+ gint parent_cbref; /* Reference to the upstream list */
+ struct upstream_list *upl;
+};
+
+static void
+lua_upstream_watch_func(struct upstream *up,
+ enum rspamd_upstreams_watch_event event,
+ guint cur_errors,
+ void *ud)
+{
+ struct rspamd_lua_upstream_watcher_cbdata *cdata =
+ (struct rspamd_lua_upstream_watcher_cbdata *) ud;
+ lua_State *L;
+ const gchar *what;
+ gint err_idx;
+
+ L = cdata->L;
+ what = lua_upstream_flag_to_str(event);
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cdata->cbref);
+ lua_pushstring(L, what);
+
+ struct rspamd_lua_upstream *lua_ups = lua_newuserdata(L, sizeof(*lua_ups));
+ lua_ups->up = up;
+ rspamd_lua_setclass(L, "rspamd{upstream}", -1);
+ /* Store parent in the upstream to prevent gc */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cdata->parent_cbref);
+ lua_ups->upref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ lua_pushinteger(L, cur_errors);
+
+ if (lua_pcall(L, 3, 0, err_idx) != 0) {
+ msg_err("cannot call watch function for upstream: %s", lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return;
+ }
+
+ lua_settop(L, 0);
+}
+
+static void
+lua_upstream_watch_dtor(gpointer ud)
+{
+ struct rspamd_lua_upstream_watcher_cbdata *cdata =
+ (struct rspamd_lua_upstream_watcher_cbdata *) ud;
+
+ luaL_unref(cdata->L, LUA_REGISTRYINDEX, cdata->cbref);
+ luaL_unref(cdata->L, LUA_REGISTRYINDEX, cdata->parent_cbref);
+ g_free(cdata);
+}
+
+/***
+ * @method upstream_list:add_watcher(what, cb)
+ * Add new watcher to the upstream lists events (table or a string):
+ * - `success` - called whenever upstream successfully used
+ * - `failure` - called on upstream error
+ * - `online` - called when upstream is being taken online from offline
+ * - `offline` - called when upstream is being taken offline from online
+ * Callback is a function: function(what, upstream, cur_errors) ... end
+ * @example
+ups:add_watcher('success', function(what, up, cur_errors) ... end)
+ups:add_watcher({'online', 'offline'}, function(what, up, cur_errors) ... end)
+ * @return nothing
+ */
+static gint
+lua_upstream_list_add_watcher(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl;
+
+ upl = lua_check_upstream_list(L);
+ if (upl &&
+ (lua_type(L, 2) == LUA_TTABLE || lua_type(L, 2) == LUA_TSTRING) &&
+ lua_type(L, 3) == LUA_TFUNCTION) {
+
+ enum rspamd_upstreams_watch_event flags = 0;
+ struct rspamd_lua_upstream_watcher_cbdata *cdata;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ flags = lua_str_to_upstream_flag(lua_tostring(L, 2));
+ }
+ else {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ if (lua_isstring(L, -1)) {
+ flags |= lua_str_to_upstream_flag(lua_tostring(L, -1));
+ }
+ else {
+ lua_pop(L, 1);
+
+ return luaL_error(L, "invalid arguments");
+ }
+ }
+ }
+
+ cdata = g_malloc0(sizeof(*cdata));
+ lua_pushvalue(L, 3); /* callback */
+ cdata->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ cdata->L = L;
+ cdata->upl = upl;
+ lua_pushvalue(L, 1); /* upstream list itself */
+ cdata->parent_cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ rspamd_upstreams_add_watch_callback(upl, flags,
+ lua_upstream_watch_func, lua_upstream_watch_dtor, cdata);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_load_upstream_list(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, upstream_list_f);
+
+ return 1;
+}
+
+void luaopen_upstream(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{upstream_list}", upstream_list_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_upstream_list", lua_load_upstream_list);
+
+ rspamd_lua_new_class(L, "rspamd{upstream}", upstream_m);
+ lua_pop(L, 1);
+}
diff --git a/src/lua/lua_url.c b/src/lua/lua_url.c
new file mode 100644
index 0000000..913469f
--- /dev/null
+++ b/src/lua/lua_url.c
@@ -0,0 +1,1481 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_url.h"
+
+
+/***
+ * @module rspamd_url
+ * This module provides routines to handle URL's and extract URL's from the text.
+ * Objects of this class are returned, for example, by `task:get_urls()` or `task:get_emails()`.
+ * You can also create `rspamd_url` from any text.
+ * @example
+local url = require "rspamd_url"
+local mpool = require "rspamd_mempool"
+
+url.init("/usr/share/rspamd/effective_tld_names.dat")
+local pool = mpool.create()
+local res = url.create(pool, 'Look at: http://user@test.example.com/test?query")
+local t = res:to_table()
+-- Content of t:
+-- url = ['http://test.example.com/test?query']
+-- host = ['test.example.com']
+-- user = ['user']
+-- path = ['test']
+-- tld = ['example.com']
+
+pool:destroy() -- res is destroyed here, so you should not use it afterwards
+
+local mistake = res:to_table() -- INVALID! as pool is destroyed
+ */
+
+/* URL methods */
+LUA_FUNCTION_DEF(url, get_length);
+LUA_FUNCTION_DEF(url, get_host);
+LUA_FUNCTION_DEF(url, get_port);
+LUA_FUNCTION_DEF(url, get_user);
+LUA_FUNCTION_DEF(url, get_path);
+LUA_FUNCTION_DEF(url, get_query);
+LUA_FUNCTION_DEF(url, get_fragment);
+LUA_FUNCTION_DEF(url, get_text);
+LUA_FUNCTION_DEF(url, tostring);
+LUA_FUNCTION_DEF(url, get_raw);
+LUA_FUNCTION_DEF(url, get_tld);
+LUA_FUNCTION_DEF(url, get_flags);
+LUA_FUNCTION_DEF(url, get_flags_num);
+LUA_FUNCTION_DEF(url, get_protocol);
+LUA_FUNCTION_DEF(url, to_table);
+LUA_FUNCTION_DEF(url, is_phished);
+LUA_FUNCTION_DEF(url, is_redirected);
+LUA_FUNCTION_DEF(url, is_obscured);
+LUA_FUNCTION_DEF(url, is_html_displayed);
+LUA_FUNCTION_DEF(url, is_subject);
+LUA_FUNCTION_DEF(url, get_phished);
+LUA_FUNCTION_DEF(url, set_redirected);
+LUA_FUNCTION_DEF(url, get_count);
+LUA_FUNCTION_DEF(url, get_visible);
+LUA_FUNCTION_DEF(url, create);
+LUA_FUNCTION_DEF(url, init);
+LUA_FUNCTION_DEF(url, all);
+LUA_FUNCTION_DEF(url, lt);
+LUA_FUNCTION_DEF(url, eq);
+LUA_FUNCTION_DEF(url, get_order);
+LUA_FUNCTION_DEF(url, get_part_order);
+
+static const struct luaL_reg urllib_m[] = {
+ LUA_INTERFACE_DEF(url, get_length),
+ LUA_INTERFACE_DEF(url, get_host),
+ LUA_INTERFACE_DEF(url, get_port),
+ LUA_INTERFACE_DEF(url, get_user),
+ LUA_INTERFACE_DEF(url, get_path),
+ LUA_INTERFACE_DEF(url, get_query),
+ LUA_INTERFACE_DEF(url, get_fragment),
+ LUA_INTERFACE_DEF(url, get_text),
+ LUA_INTERFACE_DEF(url, get_tld),
+ LUA_INTERFACE_DEF(url, get_raw),
+ LUA_INTERFACE_DEF(url, get_protocol),
+ LUA_INTERFACE_DEF(url, to_table),
+ LUA_INTERFACE_DEF(url, is_phished),
+ LUA_INTERFACE_DEF(url, is_redirected),
+ LUA_INTERFACE_DEF(url, is_obscured),
+ LUA_INTERFACE_DEF(url, is_html_displayed),
+ LUA_INTERFACE_DEF(url, is_subject),
+ LUA_INTERFACE_DEF(url, get_phished),
+
+ LUA_INTERFACE_DEF(url, get_visible),
+ LUA_INTERFACE_DEF(url, get_count),
+ LUA_INTERFACE_DEF(url, get_flags),
+ LUA_INTERFACE_DEF(url, get_flags_num),
+ LUA_INTERFACE_DEF(url, get_order),
+ LUA_INTERFACE_DEF(url, get_part_order),
+ {"get_redirected", lua_url_get_phished},
+ LUA_INTERFACE_DEF(url, set_redirected),
+ {"__tostring", lua_url_tostring},
+ {"__eq", lua_url_eq},
+ {"__lt", lua_url_lt},
+ {NULL, NULL}};
+
+static const struct luaL_reg urllib_f[] = {
+ LUA_INTERFACE_DEF(url, init),
+ LUA_INTERFACE_DEF(url, create),
+ LUA_INTERFACE_DEF(url, all),
+ {NULL, NULL}};
+
+struct rspamd_lua_url *
+lua_check_url(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{url}");
+ luaL_argcheck(L, ud != NULL, pos, "'url' expected");
+ return ud ? ((struct rspamd_lua_url *) ud) : NULL;
+}
+
+static gboolean
+lua_url_single_inserter(struct rspamd_url *url, gsize start_offset,
+ gsize end_offset, gpointer ud)
+{
+ lua_State *L = ud;
+ struct rspamd_lua_url *lua_url;
+
+ lua_url = lua_newuserdata(L, sizeof(struct rspamd_lua_url));
+ rspamd_lua_setclass(L, "rspamd{url}", -1);
+ lua_url->url = url;
+
+ return TRUE;
+}
+
+/***
+ * @method url:get_length()
+ * Get length of the url
+ * @return {number} length of url in bytes
+ */
+static gint
+lua_url_get_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushinteger(L, url->url->urllen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+/***
+ * @method url:get_host()
+ * Get domain part of the url
+ * @return {string} domain part of URL
+ */
+static gint
+lua_url_get_host(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url && url->url->hostlen > 0) {
+ lua_pushlstring(L, rspamd_url_host(url->url), url->url->hostlen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+/***
+ * @method url:get_port()
+ * Get port of the url
+ * @return {number} url port
+ */
+static gint
+lua_url_get_port(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ if (rspamd_url_get_port_if_special(url->url) == 0) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushinteger(L, rspamd_url_get_port_if_special(url->url));
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+/***
+ * @method url:get_user()
+ * Get user part of the url (e.g. username in email)
+ * @return {string} user part of URL
+ */
+static gint
+lua_url_get_user(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && rspamd_url_user(url->url) != NULL) {
+ lua_pushlstring(L, rspamd_url_user(url->url), url->url->userlen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_path()
+ * Get path of the url
+ * @return {string} path part of URL
+ */
+static gint
+lua_url_get_path(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->datalen > 0) {
+ lua_pushlstring(L, rspamd_url_data_unsafe(url->url), url->url->datalen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_query()
+ * Get query of the url
+ * @return {string} query part of URL
+ */
+static gint
+lua_url_get_query(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->querylen > 0) {
+ lua_pushlstring(L, rspamd_url_query_unsafe(url->url), url->url->querylen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_fragment()
+ * Get fragment of the url
+ * @return {string} fragment part of URL
+ */
+static gint
+lua_url_get_fragment(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->fragmentlen > 0) {
+ lua_pushlstring(L, rspamd_url_fragment_unsafe(url->url), url->url->fragmentlen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_text()
+ * Get full content of the url
+ * @return {string} url string
+ */
+static gint
+lua_url_get_text(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushlstring(L, url->url->string, url->url->urllen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:tostring()
+ * Get full content of the url or user@domain in case of email
+ * @return {string} url as a string
+ */
+static gint
+lua_url_tostring(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url != NULL) {
+ if (url->url->protocol == PROTOCOL_MAILTO) {
+ gchar *tmp = g_malloc(url->url->userlen + 1 +
+ url->url->hostlen);
+ if (url->url->userlen) {
+ memcpy(tmp, url->url->string + url->url->usershift, url->url->userlen);
+ }
+
+ tmp[url->url->userlen] = '@';
+ memcpy(tmp + url->url->userlen + 1, rspamd_url_host_unsafe(url->url),
+ url->url->hostlen);
+
+ lua_pushlstring(L, tmp, url->url->userlen + 1 + url->url->hostlen);
+ g_free(tmp);
+ }
+ else {
+ lua_pushlstring(L, url->url->string, url->url->urllen);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_raw()
+ * Get full content of the url as it was parsed (e.g. with urldecode)
+ * @return {string} url string
+ */
+static gint
+lua_url_get_raw(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushlstring(L, url->url->raw, url->url->rawlen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:is_phished()
+ * Check whether URL is treated as phished
+ * @return {boolean} `true` if URL is phished
+ */
+static gint
+lua_url_is_phished(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushboolean(L, url->url->flags & RSPAMD_URL_FLAG_PHISHED);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:is_redirected()
+ * Check whether URL was redirected
+ * @return {boolean} `true` if URL is redirected
+ */
+static gint
+lua_url_is_redirected(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushboolean(L, url->url->flags & RSPAMD_URL_FLAG_REDIRECTED);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:is_obscured()
+ * Check whether URL is treated as obscured or obfuscated (e.g. numbers in IP address or other hacks)
+ * @return {boolean} `true` if URL is obscured
+ */
+static gint
+lua_url_is_obscured(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushboolean(L, url->url->flags & RSPAMD_URL_FLAG_OBSCURED);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+/***
+ * @method url:is_html_displayed()
+ * Check whether URL is just displayed in HTML (e.g. NOT a real href)
+ * @return {boolean} `true` if URL is displayed only
+ */
+static gint
+lua_url_is_html_displayed(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushboolean(L, url->url->flags & RSPAMD_URL_FLAG_HTML_DISPLAYED);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:is_subject()
+ * Check whether URL is found in subject
+ * @return {boolean} `true` if URL is found in subject
+ */
+static gint
+lua_url_is_subject(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushboolean(L, url->url->flags & RSPAMD_URL_FLAG_SUBJECT);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_phished()
+ * Get another URL that pretends to be this URL (e.g. used in phishing)
+ * @return {url} phished URL
+ */
+static gint
+lua_url_get_phished(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *purl, *url = lua_check_url(L, 1);
+
+ if (url) {
+ if (url->url->ext && url->url->ext->linked_url != NULL) {
+ /* XXX: in fact, this is the only possible combination of flags, so this check is redundant */
+ if (url->url->flags &
+ (RSPAMD_URL_FLAG_PHISHED | RSPAMD_URL_FLAG_REDIRECTED)) {
+ purl = lua_newuserdata(L, sizeof(struct rspamd_lua_url));
+ rspamd_lua_setclass(L, "rspamd{url}", -1);
+ purl->url = url->url->ext->linked_url;
+
+ return 1;
+ }
+ }
+ }
+
+ lua_pushnil(L);
+ return 1;
+}
+
+/***
+ * @method url:set_redirected(url, pool)
+ * Set url as redirected to another url
+ * @param {string|url} url new url that is redirecting an old one
+ * @param {pool} pool memory pool to allocate memory if needed
+ * @return {url} parsed redirected url (if needed)
+ */
+static gint
+lua_url_set_redirected(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1), *redir;
+ rspamd_mempool_t *pool = NULL;
+
+ if (url == NULL) {
+ return luaL_error(L, "url is required as the first argument");
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* Parse url */
+ if (lua_type(L, 3) != LUA_TUSERDATA) {
+ return luaL_error(L, "mempool is required as the third argument");
+ }
+
+ pool = rspamd_lua_check_mempool(L, 3);
+
+ if (pool == NULL) {
+ return luaL_error(L, "mempool is required as the third argument");
+ }
+
+ gsize len;
+ const gchar *urlstr = lua_tolstring(L, 2, &len);
+
+ rspamd_url_find_single(pool, urlstr, len, RSPAMD_URL_FIND_ALL,
+ lua_url_single_inserter, L);
+
+ if (lua_type(L, -1) != LUA_TUSERDATA) {
+ /* URL is actually not found */
+ lua_pushnil(L);
+ }
+ else {
+ redir = lua_check_url(L, -1);
+
+ url->url->flags |= RSPAMD_URL_FLAG_REDIRECTED;
+
+ if (url->url->ext == NULL) {
+ url->url->ext = rspamd_mempool_alloc0_type(pool, struct rspamd_url_ext);
+ }
+ url->url->ext->linked_url = redir->url;
+ }
+ }
+ else {
+ redir = lua_check_url(L, 2);
+
+ if (redir == NULL) {
+ return luaL_error(L, "url is required as the second argument");
+ }
+
+ pool = rspamd_lua_check_mempool(L, 3);
+
+ if (pool == NULL) {
+ return luaL_error(L, "mempool is required as the third argument");
+ }
+
+ url->url->flags |= RSPAMD_URL_FLAG_REDIRECTED;
+ if (url->url->ext == NULL) {
+ url->url->ext = rspamd_mempool_alloc0_type(pool, struct rspamd_url_ext);
+ }
+ url->url->ext->linked_url = redir->url;
+
+ /* Push back on stack */
+ lua_pushvalue(L, 2);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_tld()
+ * Get effective second level domain part (eSLD) of the url host
+ * @return {string} effective second level domain part (eSLD) of the url host
+ */
+static gint
+lua_url_get_tld(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->tldlen > 0) {
+ lua_pushlstring(L, rspamd_url_tld_unsafe(url->url), url->url->tldlen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_protocol()
+ * Get protocol name
+ * @return {string} protocol as a string
+ */
+static gint
+lua_url_get_protocol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->protocol != PROTOCOL_UNKNOWN) {
+ lua_pushstring(L, rspamd_url_protocol_name(url->url->protocol));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_count()
+ * Return number of occurrences for this particular URL
+ * @return {number} number of occurrences
+ */
+static gint
+lua_url_get_count(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url != NULL) {
+ lua_pushinteger(L, url->url->count);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+* @method url:get_visible()
+* Get visible part of the url with html tags stripped
+* @return {string} url string
+*/
+static gint
+lua_url_get_visible(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->ext && url->url->ext->visible_part) {
+ lua_pushstring(L, url->url->ext->visible_part);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:to_table()
+ * Return url as a table with the following fields:
+ *
+ * - `url`: full content
+ * - `host`: hostname part
+ * - `user`: user part
+ * - `path`: path part
+ * - `tld`: top level domain
+ * - `protocol`: url protocol
+ * @return {table} URL as a table
+ */
+static gint
+lua_url_to_table(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+ struct rspamd_url *u;
+
+ if (url != NULL) {
+ u = url->url;
+ lua_createtable(L, 0, 12);
+ lua_pushstring(L, "url");
+ lua_pushlstring(L, u->string, u->urllen);
+ lua_settable(L, -3);
+
+ if (u->hostlen > 0) {
+ lua_pushstring(L, "host");
+ lua_pushlstring(L, rspamd_url_host_unsafe(u), u->hostlen);
+ lua_settable(L, -3);
+ }
+
+ if (rspamd_url_get_port_if_special(u) != 0) {
+ lua_pushstring(L, "port");
+ lua_pushinteger(L, rspamd_url_get_port_if_special(u));
+ lua_settable(L, -3);
+ }
+
+ if (u->tldlen > 0) {
+ lua_pushstring(L, "tld");
+ lua_pushlstring(L, rspamd_url_tld_unsafe(u), u->tldlen);
+ lua_settable(L, -3);
+ }
+
+ if (u->userlen > 0) {
+ lua_pushstring(L, "user");
+ lua_pushlstring(L, rspamd_url_user(u), u->userlen);
+ lua_settable(L, -3);
+ }
+
+ if (u->datalen > 0) {
+ lua_pushstring(L, "path");
+ lua_pushlstring(L, rspamd_url_data_unsafe(u), u->datalen);
+ lua_settable(L, -3);
+ }
+
+ if (u->querylen > 0) {
+ lua_pushstring(L, "query");
+ lua_pushlstring(L, rspamd_url_query_unsafe(u), u->querylen);
+ lua_settable(L, -3);
+ }
+
+ if (u->fragmentlen > 0) {
+ lua_pushstring(L, "fragment");
+ lua_pushlstring(L, rspamd_url_fragment_unsafe(u), u->fragmentlen);
+ lua_settable(L, -3);
+ }
+
+
+ lua_pushstring(L, "protocol");
+ lua_pushstring(L, rspamd_url_protocol_name(u->protocol));
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static rspamd_mempool_t *static_lua_url_pool;
+
+RSPAMD_CONSTRUCTOR(rspamd_urls_static_pool_ctor)
+{
+ static_lua_url_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "static_lua_url", 0);
+}
+
+RSPAMD_DESTRUCTOR(rspamd_urls_static_pool_dtor)
+{
+ rspamd_mempool_delete(static_lua_url_pool);
+}
+
+/***
+ * @function url.create([mempool,] str, [{flags_table}])
+ * @param {rspamd_mempool} memory pool for URL, e.g. `task:get_mempool()`
+ * @param {string} text that contains URL (can also contain other stuff)
+ * @return {url} new url object that exists as long as the corresponding mempool exists
+ */
+static gint
+lua_url_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_mempool_t *pool;
+ struct rspamd_lua_text *t;
+ struct rspamd_lua_url *u;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ pool = rspamd_lua_check_mempool(L, 1);
+ t = lua_check_text_or_string(L, 2);
+ }
+ else {
+ pool = static_lua_url_pool;
+ t = lua_check_text_or_string(L, 2);
+ }
+
+ if (pool == NULL || t == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+ else {
+ rspamd_url_find_single(pool, t->start, t->len, RSPAMD_URL_FIND_ALL,
+ lua_url_single_inserter, L);
+
+ if (lua_type(L, -1) != LUA_TUSERDATA) {
+ /* URL is actually not found */
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ u = (struct rspamd_lua_url *) lua_touserdata(L, -1);
+
+ if (lua_type(L, 3) == LUA_TTABLE) {
+ /* Add flags */
+ for (lua_pushnil(L); lua_next(L, 3); lua_pop(L, 1)) {
+ int nmask = 0;
+ const gchar *fname = lua_tostring(L, -1);
+
+ if (rspamd_url_flag_from_string(fname, &nmask)) {
+ u->url->flags |= nmask;
+ }
+ else {
+ lua_pop(L, 1);
+ return luaL_error(L, "invalid flag: %s", fname);
+ }
+ }
+ }
+ }
+
+ return 1;
+}
+
+/***
+ * @function url.init(tld_file)
+ * Initialize url library if not initialized yet by Rspamd
+ * @param {string} tld_file path to effective_tld_names.dat file (public suffix list)
+ * @return nothing
+ */
+static gint
+lua_url_init(lua_State *L)
+{
+ const gchar *tld_path;
+
+ tld_path = luaL_checkstring(L, 1);
+
+ rspamd_url_init(tld_path);
+
+ return 0;
+}
+
+static gboolean
+lua_url_table_inserter(struct rspamd_url *url, gsize start_offset,
+ gsize end_offset, gpointer ud)
+{
+ lua_State *L = ud;
+ struct rspamd_lua_url *lua_url;
+ gint n;
+
+ n = rspamd_lua_table_size(L, -1);
+ lua_url = lua_newuserdata(L, sizeof(struct rspamd_lua_url));
+ rspamd_lua_setclass(L, "rspamd{url}", -1);
+ lua_url->url = url;
+ lua_rawseti(L, -2, n + 1);
+
+ return TRUE;
+}
+
+
+static gint
+lua_url_all(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_mempool_t *pool = rspamd_lua_check_mempool(L, 1);
+ const gchar *text;
+ size_t length;
+
+ if (pool == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ text = luaL_checklstring(L, 2, &length);
+
+ if (text != NULL) {
+ lua_newtable(L);
+ rspamd_url_find_multiple(pool, text, length,
+ RSPAMD_URL_FIND_ALL, NULL,
+ lua_url_table_inserter, L);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_flags()
+ * Return flags for a specified URL as map 'flag'->true for all flags set,
+ * possible flags are:
+ *
+ * - `phished`: URL is likely phished
+ * - `numeric`: URL is numeric (e.g. IP address)
+ * - `obscured`: URL was obscured
+ * - `redirected`: URL comes from redirector
+ * - `html_displayed`: URL is used just for displaying purposes
+ * - `text`: URL comes from the text
+ * - `subject`: URL comes from the subject
+ * - `host_encoded`: URL host part is encoded
+ * - `schema_encoded`: URL schema part is encoded
+ * - `query_encoded`: URL query part is encoded
+ * - `missing_slashes`: URL has some slashes missing
+ * - `idn`: URL has international characters
+ * - `has_port`: URL has port
+ * - `has_user`: URL has user part
+ * - `schemaless`: URL has no schema
+ * - `unnormalised`: URL has some unicode unnormalities
+ * - `zw_spaces`: URL has some zero width spaces
+ * - `url_displayed`: URL has some other url-like string in visible part
+ * - `image`: URL is from src attribute of img HTML tag
+ * @return {table} URL flags
+ */
+#define PUSH_FLAG(fl) \
+ do { \
+ if (flags & (fl)) { \
+ lua_pushstring(L, rspamd_url_flag_to_string(fl)); \
+ lua_pushboolean(L, true); \
+ lua_settable(L, -3); \
+ } \
+ } while (0)
+
+static gint
+lua_url_get_flags(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+ enum rspamd_url_flags flags;
+
+ if (url != NULL) {
+ flags = url->url->flags;
+
+ lua_createtable(L, 0, 4);
+
+ for (gint i = 0; i < RSPAMD_URL_MAX_FLAG_SHIFT; i++) {
+ PUSH_FLAG(1u << i);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+#undef PUSH_FLAG
+
+static gint
+lua_url_get_flags_num(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url) {
+ lua_pushinteger(L, url->url->flags);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_url_get_order(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url) {
+ if (url->url->order != (uint16_t) -1) {
+ lua_pushinteger(L, url->url->order);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_url_get_part_order(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url) {
+ if (url->url->part_order != (uint16_t) -1) {
+ lua_pushinteger(L, url->url->part_order);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+void lua_tree_url_callback(gpointer key, gpointer value, gpointer ud)
+{
+ struct rspamd_lua_url *lua_url;
+ struct rspamd_url *url = (struct rspamd_url *) value;
+ struct lua_tree_cb_data *cb = ud;
+
+ if ((url->protocol & cb->protocols_mask) == url->protocol) {
+
+ /* Handle different flags application logic */
+ switch (cb->flags_mode) {
+ case url_flags_mode_include_any:
+ if (url->flags != (url->flags & cb->flags_mask)) {
+ return;
+ }
+ break;
+ case url_flags_mode_include_explicit:
+ if ((url->flags & cb->flags_mask) != cb->flags_mask) {
+ return;
+ }
+ break;
+ case url_flags_mode_exclude_include:
+ if ((url->flags & cb->flags_exclude_mask) != 0) {
+ return;
+ }
+ if ((url->flags & cb->flags_mask) == 0) {
+ return;
+ }
+ break;
+ }
+
+ if (cb->skip_prob > 0) {
+ gdouble coin = rspamd_random_double_fast_seed(&cb->random_seed);
+
+ if (coin < cb->skip_prob) {
+ return;
+ }
+ }
+
+ lua_url = lua_newuserdata(cb->L, sizeof(struct rspamd_lua_url));
+ lua_pushvalue(cb->L, cb->metatable_pos);
+ lua_setmetatable(cb->L, -2);
+ lua_url->url = url;
+ lua_rawseti(cb->L, -2, cb->i++);
+ }
+}
+
+gboolean
+lua_url_cbdata_fill(lua_State *L,
+ gint pos,
+ struct lua_tree_cb_data *cbd,
+ guint default_protocols,
+ guint default_flags,
+ gsize max_urls)
+{
+ gint protocols_mask = 0;
+
+ gint pos_arg_type = lua_type(L, pos);
+ guint flags_mask = default_flags;
+ gboolean seen_flags = FALSE, seen_protocols = FALSE;
+
+ memset(cbd, 0, sizeof(*cbd));
+ cbd->flags_mode = url_flags_mode_include_any;
+
+ if (pos_arg_type == LUA_TBOOLEAN) {
+ protocols_mask = default_protocols;
+ if (lua_toboolean(L, 2)) {
+ protocols_mask |= PROTOCOL_MAILTO;
+ }
+ }
+ else if (pos_arg_type == LUA_TTABLE) {
+ if (rspamd_lua_geti(L, 1, pos) == LUA_TNIL) {
+ /* New method: indexed table */
+
+ lua_getfield(L, pos, "flags");
+ if (lua_istable(L, -1)) {
+ gint top = lua_gettop(L);
+
+ lua_getfield(L, pos, "flags_mode");
+ if (lua_isstring(L, -1)) {
+ const gchar *mode_str = lua_tostring(L, -1);
+
+ if (strcmp(mode_str, "explicit") == 0) {
+ cbd->flags_mode = url_flags_mode_include_explicit;
+ /*
+ * Ignore default flags in this mode and include
+ * merely flags specified by a caller
+ */
+ flags_mask = 0;
+ }
+ }
+ lua_pop(L, 1);
+
+ for (lua_pushnil(L); lua_next(L, top); lua_pop(L, 1)) {
+ int nmask = 0;
+
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *fname = lua_tostring(L, -1);
+
+
+ if (rspamd_url_flag_from_string(fname, &nmask)) {
+ flags_mask |= nmask;
+ }
+ else {
+ msg_info("bad url flag: %s", fname);
+ return FALSE;
+ }
+ }
+ else {
+ flags_mask |= lua_tointeger(L, -1);
+ }
+ }
+
+ seen_flags = TRUE;
+ }
+ else {
+ flags_mask |= default_flags;
+ }
+ lua_pop(L, 1);
+
+ lua_getfield(L, pos, "protocols");
+ if (lua_istable(L, -1)) {
+ gint top = lua_gettop(L);
+
+ for (lua_pushnil(L); lua_next(L, top); lua_pop(L, 1)) {
+ int nmask;
+ const gchar *pname = lua_tostring(L, -1);
+
+ nmask = rspamd_url_protocol_from_string(pname);
+
+ if (nmask != PROTOCOL_UNKNOWN) {
+ protocols_mask |= nmask;
+ }
+ else {
+ msg_info("bad url protocol: %s", pname);
+ return FALSE;
+ }
+ }
+ seen_protocols = TRUE;
+ }
+ else {
+ protocols_mask = default_protocols;
+ }
+ lua_pop(L, 1);
+
+ if (!seen_protocols) {
+ lua_getfield(L, pos, "emails");
+ if (lua_isboolean(L, -1)) {
+ if (lua_toboolean(L, -1)) {
+ protocols_mask |= PROTOCOL_MAILTO;
+ }
+ }
+ lua_pop(L, 1);
+ }
+
+ if (!seen_flags) {
+ lua_getfield(L, pos, "images");
+ if (lua_isboolean(L, -1)) {
+ if (lua_toboolean(L, -1)) {
+ flags_mask |= RSPAMD_URL_FLAG_IMAGE;
+ }
+ else {
+ flags_mask &= ~RSPAMD_URL_FLAG_IMAGE;
+ }
+ }
+ else {
+ flags_mask &= ~RSPAMD_URL_FLAG_IMAGE;
+ }
+ lua_pop(L, 1);
+ }
+
+ if (!seen_flags) {
+ lua_getfield(L, pos, "content");
+ if (lua_isboolean(L, -1)) {
+ if (lua_toboolean(L, -1)) {
+ flags_mask |= RSPAMD_URL_FLAG_CONTENT;
+ }
+ else {
+ flags_mask &= ~RSPAMD_URL_FLAG_CONTENT;
+ }
+ }
+ else {
+ flags_mask &= ~RSPAMD_URL_FLAG_CONTENT;
+ }
+ lua_pop(L, 1);
+ }
+
+ lua_getfield(L, pos, "max_urls");
+ if (lua_isnumber(L, -1)) {
+ max_urls = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_getfield(L, pos, "sort");
+ if (lua_isboolean(L, -1)) {
+ cbd->sort = TRUE;
+ }
+ lua_pop(L, 1);
+ }
+ else {
+ /* Plain table of the protocols */
+ for (lua_pushnil(L); lua_next(L, pos); lua_pop(L, 1)) {
+ int nmask;
+ const gchar *pname = lua_tostring(L, -1);
+
+ nmask = rspamd_url_protocol_from_string(pname);
+
+ if (nmask != PROTOCOL_UNKNOWN) {
+ protocols_mask |= nmask;
+ }
+ else {
+ msg_info("bad url protocol: %s", pname);
+ return FALSE;
+ }
+ }
+ }
+
+ lua_pop(L, 1); /* After rspamd_lua_geti */
+ }
+ else if (pos_arg_type == LUA_TSTRING) {
+ const gchar *plist = lua_tostring(L, pos);
+ gchar **strvec;
+ gchar *const *cvec;
+
+ strvec = g_strsplit_set(plist, ",;", -1);
+ cvec = strvec;
+
+ while (*cvec) {
+ int nmask;
+
+ nmask = rspamd_url_protocol_from_string(*cvec);
+
+ if (nmask != PROTOCOL_UNKNOWN) {
+ protocols_mask |= nmask;
+ }
+ else {
+ msg_info("bad url protocol: %s", *cvec);
+ g_strfreev(strvec);
+
+ return FALSE;
+ }
+
+ cvec++;
+ }
+
+ g_strfreev(strvec);
+ }
+ else if (pos_arg_type == LUA_TNONE || pos_arg_type == LUA_TNIL) {
+ protocols_mask = default_protocols;
+ flags_mask = default_flags;
+ }
+ else {
+ return FALSE;
+ }
+
+ if (lua_type(L, pos + 1) == LUA_TBOOLEAN) {
+ if (lua_toboolean(L, pos + 1)) {
+ flags_mask |= RSPAMD_URL_FLAG_IMAGE;
+ }
+ else {
+ flags_mask &= ~RSPAMD_URL_FLAG_IMAGE;
+ }
+ }
+
+ cbd->i = 1;
+ cbd->L = L;
+ cbd->max_urls = max_urls;
+ cbd->protocols_mask = protocols_mask;
+ cbd->flags_mask = flags_mask;
+
+ /* This needs to be removed from the stack */
+ rspamd_lua_class_metatable(L, "rspamd{url}");
+ cbd->metatable_pos = lua_gettop(L);
+ (void) lua_checkstack(L, cbd->metatable_pos + 4);
+
+ return TRUE;
+}
+
+gboolean
+lua_url_cbdata_fill_exclude_include(lua_State *L,
+ gint pos,
+ struct lua_tree_cb_data *cbd,
+ guint default_protocols,
+ gsize max_urls)
+{
+ guint protocols_mask = default_protocols;
+ guint include_flags_mask, exclude_flags_mask;
+
+ gint pos_arg_type = lua_type(L, pos);
+
+ memset(cbd, 0, sizeof(*cbd));
+ cbd->flags_mode = url_flags_mode_exclude_include;
+
+ /* Include flags */
+ if (pos_arg_type == LUA_TTABLE) {
+ include_flags_mask = 0; /* Reset to no flags */
+
+ for (lua_pushnil(L); lua_next(L, pos); lua_pop(L, 1)) {
+ int nmask = 0;
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *fname = lua_tostring(L, -1);
+
+ if (rspamd_url_flag_from_string(fname, &nmask)) {
+ include_flags_mask |= nmask;
+ }
+ else {
+ msg_info("bad url include flag: %s", fname);
+ return FALSE;
+ }
+ }
+ else {
+ include_flags_mask |= lua_tointeger(L, -1);
+ }
+ }
+ }
+ else if (pos_arg_type == LUA_TNIL || pos_arg_type == LUA_TNONE) {
+ /* Include all flags */
+ include_flags_mask = ~0U;
+ }
+ else {
+ msg_info("bad arguments: wrong include mask");
+ return FALSE;
+ }
+
+ /* Exclude flags */
+ pos_arg_type = lua_type(L, pos + 1);
+ if (pos_arg_type == LUA_TTABLE) {
+ exclude_flags_mask = 0; /* Reset to no flags */
+
+ for (lua_pushnil(L); lua_next(L, pos + 1); lua_pop(L, 1)) {
+ int nmask = 0;
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *fname = lua_tostring(L, -1);
+
+ if (rspamd_url_flag_from_string(fname, &nmask)) {
+ exclude_flags_mask |= nmask;
+ }
+ else {
+ msg_info("bad url exclude flag: %s", fname);
+ return FALSE;
+ }
+ }
+ else {
+ exclude_flags_mask |= lua_tointeger(L, -1);
+ }
+ }
+ }
+ else if (pos_arg_type == LUA_TNIL || pos_arg_type == LUA_TNONE) {
+ /* Empty all exclude flags */
+ exclude_flags_mask = 0U;
+ }
+ else {
+ msg_info("bad arguments: wrong exclude mask");
+ return FALSE;
+ }
+
+ if (lua_type(L, pos + 2) == LUA_TTABLE) {
+ protocols_mask = 0U; /* Reset all protocols */
+
+ for (lua_pushnil(L); lua_next(L, pos + 2); lua_pop(L, 1)) {
+ int nmask;
+ const gchar *pname = lua_tostring(L, -1);
+
+ nmask = rspamd_url_protocol_from_string(pname);
+
+ if (nmask != PROTOCOL_UNKNOWN) {
+ protocols_mask |= nmask;
+ }
+ else {
+ msg_info("bad url protocol: %s", pname);
+ return FALSE;
+ }
+ }
+ }
+ else {
+ protocols_mask = default_protocols;
+ }
+
+ cbd->i = 1;
+ cbd->L = L;
+ cbd->max_urls = max_urls;
+ cbd->protocols_mask = protocols_mask;
+ cbd->flags_mask = include_flags_mask;
+ cbd->flags_exclude_mask = exclude_flags_mask;
+
+ /* This needs to be removed from the stack */
+ rspamd_lua_class_metatable(L, "rspamd{url}");
+ cbd->metatable_pos = lua_gettop(L);
+ (void) lua_checkstack(L, cbd->metatable_pos + 4);
+
+ return TRUE;
+}
+
+
+void lua_url_cbdata_dtor(struct lua_tree_cb_data *cbd)
+{
+ if (cbd->metatable_pos != -1) {
+ lua_remove(cbd->L, cbd->metatable_pos);
+ }
+}
+
+gsize lua_url_adjust_skip_prob(float timestamp,
+ guchar digest[16],
+ struct lua_tree_cb_data *cb,
+ gsize sz)
+{
+ if (cb->max_urls > 0 && sz > cb->max_urls) {
+ cb->skip_prob = 1.0 - ((gdouble) cb->max_urls) / (gdouble) sz;
+ /*
+ * Use task dependent probabilistic seed to ensure that
+ * consequent task:get_urls return the same list of urls
+ * We use both digest and timestamp here to avoid attack surface
+ * based just on digest.
+ */
+ memcpy(&cb->random_seed, digest, 4);
+ memcpy(((unsigned char *) &cb->random_seed) + 4, &timestamp, 4);
+ sz = cb->max_urls;
+ }
+
+ return sz;
+}
+
+static gint
+lua_url_eq(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *u1 = lua_check_url(L, 1),
+ *u2 = lua_check_url(L, 2);
+
+ if (u1 && u2) {
+ lua_pushboolean(L, (rspamd_url_cmp(u1->url, u2->url) == 0));
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ return 1;
+}
+
+static gint
+lua_url_lt(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *u1 = lua_check_url(L, 1),
+ *u2 = lua_check_url(L, 2);
+
+ if (u1 && u2) {
+ lua_pushinteger(L, rspamd_url_cmp(u1->url, u2->url));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_url(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, urllib_f);
+
+ /* Push flags */
+ lua_createtable(L, 0, RSPAMD_URL_MAX_FLAG_SHIFT);
+ for (int i = 0; i < RSPAMD_URL_MAX_FLAG_SHIFT; i++) {
+ guint flag = 1u << i;
+
+ lua_pushinteger(L, flag);
+ lua_setfield(L, -2, rspamd_url_flag_to_string(flag));
+ }
+
+ lua_setfield(L, -2, "flags");
+
+ return 1;
+}
+
+void luaopen_url(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{url}", urllib_m);
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "rspamd_url", lua_load_url);
+}
diff --git a/src/lua/lua_url.h b/src/lua/lua_url.h
new file mode 100644
index 0000000..a78dffa
--- /dev/null
+++ b/src/lua/lua_url.h
@@ -0,0 +1,87 @@
+/*-
+ * Copyright 2020 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_LUA_URL_H
+#define RSPAMD_LUA_URL_H
+
+#include "lua_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct lua_tree_cb_data {
+ lua_State *L;
+ int i;
+ int metatable_pos;
+ guint flags_mask;
+ guint flags_exclude_mask;
+ guint protocols_mask;
+ enum {
+ url_flags_mode_include_any,
+ url_flags_mode_include_explicit,
+ url_flags_mode_exclude_include,
+ } flags_mode;
+ gboolean sort;
+ gsize max_urls;
+ gdouble skip_prob;
+ guint64 random_seed;
+};
+
+void lua_tree_url_callback(gpointer key, gpointer value, gpointer ud);
+
+/**
+ * Fills a cbdata table based on the parameter at position pos
+ * @param L
+ * @param pos
+ * @param cbd
+ * @return
+ */
+gboolean lua_url_cbdata_fill(lua_State *L, gint pos,
+ struct lua_tree_cb_data *cbd,
+ guint default_protocols,
+ guint default_flags,
+ gsize max_urls);
+
+gboolean lua_url_cbdata_fill_exclude_include(lua_State *L, gint pos,
+ struct lua_tree_cb_data *cbd,
+ guint default_protocols,
+ gsize max_urls);
+
+/**
+ * Cleanup url cbdata
+ * @param cbd
+ */
+void lua_url_cbdata_dtor(struct lua_tree_cb_data *cbd);
+
+/**
+ * Adjust probabilistic skip of the urls
+ * @param timestamp
+ * @param digest
+ * @param cb
+ * @param sz
+ * @param max_urls
+ * @return
+ */
+gsize lua_url_adjust_skip_prob(float timestamp,
+ guchar digest[16],
+ struct lua_tree_cb_data *cb,
+ gsize sz);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c
new file mode 100644
index 0000000..152c02d
--- /dev/null
+++ b/src/lua/lua_util.c
@@ -0,0 +1,3585 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "unix-std.h"
+#include "lua_compress.h"
+#include "libmime/email_addr.h"
+#include "libmime/content_type.h"
+#include "libmime/mime_headers.h"
+#include "libutil/hash.h"
+
+#include "lua_parsers.h"
+
+#ifdef WITH_LUA_REPL
+#include "replxx.h"
+#endif
+
+#include <math.h>
+#include <glob.h>
+
+#include "unicode/uspoof.h"
+#include "unicode/uscript.h"
+#include "contrib/fastutf8/fastutf8.h"
+
+/***
+ * @module rspamd_util
+ * This module contains some generic purpose utilities that could be useful for
+ * testing and production rules.
+ */
+
+/***
+ * @function util.create_event_base()
+ * Creates new event base for processing asynchronous events
+ * @return {ev_base} new event processing base
+ */
+LUA_FUNCTION_DEF(util, create_event_base);
+/***
+ * @function util.load_rspamd_config(filename)
+ * Load rspamd config from the specified file
+ * @return {confg} new configuration object suitable for access
+ */
+LUA_FUNCTION_DEF(util, load_rspamd_config);
+/***
+ * @function util.config_from_ucl(any, string)
+ * Load rspamd config from ucl represented by any lua table
+ * @return {confg} new configuration object suitable for access
+ */
+LUA_FUNCTION_DEF(util, config_from_ucl);
+/***
+ * @function util.encode_base64(input[, str_len, [newlines_type]])
+ * Encodes data in base64 breaking lines if needed
+ * @param {text or string} input input data
+ * @param {number} str_len optional size of lines or 0 if split is not needed
+ * @return {rspamd_text} encoded data chunk
+ */
+LUA_FUNCTION_DEF(util, encode_base64);
+/***
+ * @function util.encode_qp(input[, str_len, [newlines_type]])
+ * Encodes data in quoted printable breaking lines if needed
+ * @param {text or string} input input data
+ * @param {number} str_len optional size of lines or 0 if split is not needed
+ * @return {rspamd_text} encoded data chunk
+ */
+LUA_FUNCTION_DEF(util, encode_qp);
+
+/***
+ * @function util.decode_qp(input)
+ * Decodes data from quoted printable
+ * @param {text or string} input input data
+ * @return {rspamd_text} decoded data chunk
+ */
+LUA_FUNCTION_DEF(util, decode_qp);
+
+/***
+ * @function util.decode_base64(input)
+ * Decodes data from base64 ignoring whitespace characters
+ * @param {text or string} input data to decode; if `rspamd{text}` is used then the string is modified **in-place**
+ * @return {rspamd_text} decoded data chunk
+ */
+LUA_FUNCTION_DEF(util, decode_base64);
+
+/***
+ * @function util.encode_base32(input, [b32type = 'default'])
+ * Encodes data in base32 breaking lines if needed
+ * @param {text or string} input input data
+ * @param {string} b32type base32 type (default, bleach, rfc)
+ * @return {rspamd_text} encoded data chunk
+ */
+LUA_FUNCTION_DEF(util, encode_base32);
+/***
+ * @function util.decode_base32(input, [b32type = 'default'])
+ * Decodes data from base32 ignoring whitespace characters
+ * @param {text or string} input data to decode
+ * @param {string} b32type base32 type (default, bleach, rfc)
+ * @return {rspamd_text} decoded data chunk
+ */
+LUA_FUNCTION_DEF(util, decode_base32);
+
+/***
+ * @function util.decode_url(input)
+ * Decodes data from url encoding
+ * @param {text or string} input data to decode
+ * @return {rspamd_text} decoded data chunk
+ */
+LUA_FUNCTION_DEF(util, decode_url);
+
+/***
+ * @function util.tokenize_text(input[, exceptions])
+ * Create tokens from a text using optional exceptions list
+ * @param {text/string} input input data
+ * @param {table} exceptions, a table of pairs containing <start_pos,length> of exceptions in the input
+ * @return {table/strings} list of strings representing words in the text
+ */
+LUA_FUNCTION_DEF(util, tokenize_text);
+LUA_FUNCTION_DEF(util, process_message);
+/***
+ * @function util.tanh(num)
+ * Calculates hyperbolic tangent of the specified floating point value
+ * @param {number} num input number
+ * @return {number} hyperbolic tangent of the variable
+ */
+LUA_FUNCTION_DEF(util, tanh);
+
+/***
+ * @function util.parse_html(input)
+ * Parses HTML and returns the according text
+ * @param {string|text} in input HTML
+ * @return {rspamd_text} processed text with no HTML tags
+ */
+LUA_FUNCTION_DEF(util, parse_html);
+
+/***
+ * @function util.levenshtein_distance(s1, s2)
+ * Returns levenstein distance between two strings
+ * @param {string} s1 the first string
+ * @param {string} s2 the second string
+ * @return {number} number of differences in two strings
+ */
+LUA_FUNCTION_DEF(util, levenshtein_distance);
+
+/***
+ * @function util.fold_header(name, value, [how, [stop_chars]])
+ * Fold rfc822 header according to the folding rules
+ *
+ * @param {string} name name of the header
+ * @param {string} value value of the header
+ * @param {string} how "cr" for \r, "lf" for \n and "crlf" for \r\n (default)
+ * @param {string} stop_chars also fold header when the
+ * @return {string} Folded value of the header
+ */
+LUA_FUNCTION_DEF(util, fold_header);
+
+/***
+ * @function util.is_uppercase(str)
+ * Returns true if a string is all uppercase
+ *
+ * @param {string} str input string
+ * @return {bool} true if a string is all uppercase
+ */
+LUA_FUNCTION_DEF(util, is_uppercase);
+
+/***
+ * @function util.humanize_number(num)
+ * Returns humanized representation of given number (like 1k instead of 1000)
+ *
+ * @param {number} num number to humanize
+ * @return {string} humanized representation of a number
+ */
+LUA_FUNCTION_DEF(util, humanize_number);
+
+/***
+ * @function util.get_tld(host)
+ * Returns effective second level domain part (eSLD) for the specified host
+ *
+ * @param {string} host hostname
+ * @return {string} eSLD part of the hostname or the full hostname if eSLD was not found
+ */
+LUA_FUNCTION_DEF(util, get_tld);
+
+/***
+ * @function util.glob(pattern)
+ * Returns results for the glob match for the specified pattern
+ *
+ * @param {string} pattern glob pattern to match ('?' and '*' are supported)
+ * @return {table/string} list of matched files
+ */
+LUA_FUNCTION_DEF(util, glob);
+
+/***
+ * @function util.parse_mail_address(str, [pool])
+ * Parses email address and returns a table of tables in the following format:
+ *
+ * - `raw` - the original value without any processing
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * - `flags` - table with following keys set to true if given condition fulfilled:
+ * - [valid] - valid SMTP address in conformity with https://tools.ietf.org/html/rfc5321#section-4.1.
+ * - [ip] - domain is IPv4/IPv6 address
+ * - [braced] - angled `<blah@foo.com>` address
+ * - [quoted] - quoted user part
+ * - [empty] - empty address
+ * - [backslash] - user part contains backslash
+ * - [8bit] - contains 8bit characters
+ *
+ * @param {string} str input string
+ * @param {rspamd_mempool} pool memory pool to use
+ * @return {table/tables} parsed list of mail addresses
+ */
+LUA_FUNCTION_DEF(util, parse_mail_address);
+
+/***
+ * @function util.strlen_utf8(str)
+ * Returns length of string encoded in utf-8 in characters.
+ * If invalid characters are found, then this function returns number of bytes.
+ * @param {string} str utf8 encoded string
+ * @return {number} number of characters in string
+ */
+LUA_FUNCTION_DEF(util, strlen_utf8);
+
+/***
+ * @function util.lower_utf8(str)
+ * Converts utf8 string to lower case
+ * @param {string} str utf8 encoded string
+ * @return {string} lowercased utf8 string
+ */
+LUA_FUNCTION_DEF(util, lower_utf8);
+
+/***
+ * @function util.normalize_utf8(str)
+ * Gets a string in UTF8 and normalises it to NFKC_Casefold form
+ * @param {string} str utf8 encoded string
+ * @return {string,integer} lowercased utf8 string + result of the normalisation (use bit.band to check):
+ * RSPAMD_UNICODE_NORM_NORMAL = 0,
+ * RSPAMD_UNICODE_NORM_UNNORMAL = (1 << 0),
+ * RSPAMD_UNICODE_NORM_ZERO_SPACES = (1 << 1),
+ * RSPAMD_UNICODE_NORM_ERROR = (1 << 2),
+ * RSPAMD_UNICODE_NORM_OVERFLOW = (1 << 3)
+ */
+LUA_FUNCTION_DEF(util, normalize_utf8);
+
+
+/***
+ * @function util.transliterate(str)
+ * Converts utf8 encoded string to latin transliteration
+ * @param {string/text} str utf8 encoded string
+ * @return {text} transliterated string
+ */
+LUA_FUNCTION_DEF(util, transliterate);
+
+/***
+ * @function util.strequal_caseless(str1, str2)
+ * Compares two strings regardless of their case using ascii comparison.
+ * Returns `true` if `str1` is equal to `str2`
+ * @param {string} str1 utf8 encoded string
+ * @param {string} str2 utf8 encoded string
+ * @return {bool} result of comparison
+ */
+LUA_FUNCTION_DEF(util, strequal_caseless);
+
+
+/***
+ * @function util.strequal_caseless_utf8(str1, str2)
+ * Compares two utf8 strings regardless of their case using utf8 collation rules.
+ * Returns `true` if `str1` is equal to `str2`
+ * @param {string} str1 utf8 encoded string
+ * @param {string} str2 utf8 encoded string
+ * @return {bool} result of comparison
+ */
+LUA_FUNCTION_DEF(util, strequal_caseless_utf8);
+
+
+/***
+ * @function util.get_ticks()
+ * Returns current number of ticks as floating point number
+ * @return {number} number of current clock ticks (monotonically increasing)
+ */
+LUA_FUNCTION_DEF(util, get_ticks);
+
+/***
+ * @function util.get_time()
+ * Returns current time as unix time in floating point representation
+ * @return {number} number of seconds since 01.01.1970
+ */
+LUA_FUNCTION_DEF(util, get_time);
+
+/***
+ * @function util.time_to_string(seconds)
+ * Converts time from Unix time to HTTP date format
+ * @param {number} seconds unix timestamp
+ * @return {string} date as HTTP date
+ */
+LUA_FUNCTION_DEF(util, time_to_string);
+
+/***
+ * @function util.stat(fname)
+ * Performs stat(2) on a specified filepath and returns table of values
+ *
+ * - `size`: size of file in bytes
+ * - `type`: type of filepath: `regular`, `directory`, `special`
+ * - `mtime`: modification time as unix time
+ *
+ * @return {string,table} string is returned when error is occurred
+ * @example
+ *
+ * local err,st = util.stat('/etc/password')
+ *
+ * if err then
+ * -- handle error
+ * else
+ * print(st['size'])
+ * end
+ */
+LUA_FUNCTION_DEF(util, stat);
+
+/***
+ * @function util.unlink(fname)
+ * Removes the specified file from the filesystem
+ *
+ * @param {string} fname filename to remove
+ * @return {boolean,[string]} true if file has been deleted or false,'error string'
+ */
+LUA_FUNCTION_DEF(util, unlink);
+
+/***
+ * @function util.lock_file(fname, [fd])
+ * Lock the specified file. This function returns {number} which must be passed to `util.unlock_file` after usage
+ * or you'll have a resource leak
+ *
+ * @param {string} fname filename to lock
+ * @param {number} fd use the specified fd instead of opening one
+ * @return {number|nil,string} number if locking was successful or nil + error otherwise
+ */
+LUA_FUNCTION_DEF(util, lock_file);
+
+/***
+ * @function util.unlock_file(fd, [close_fd])
+ * Unlock the specified file closing the file descriptor associated.
+ *
+ * @param {number} fd descriptor to unlock
+ * @param {boolean} close_fd close descriptor on unlocking (default: TRUE)
+ * @return {boolean[,string]} true if a file was unlocked
+ */
+LUA_FUNCTION_DEF(util, unlock_file);
+
+/***
+ * @function util.create_file(fname, [mode])
+ * Creates the specified file with the default mode 0644
+ *
+ * @param {string} fname filename to create
+ * @param {number} mode open mode (you should use octal number here)
+ * @return {number|nil,string} file descriptor or pair nil + error string
+ */
+LUA_FUNCTION_DEF(util, create_file);
+
+/***
+ * @function util.close_file(fd)
+ * Closes descriptor fd
+ *
+ * @param {number} fd descriptor to close
+ * @return {boolean[,string]} true if a file was closed
+ */
+LUA_FUNCTION_DEF(util, close_file);
+
+/***
+ * @function util.random_hex(size)
+ * Returns random hex string of the specified size
+ *
+ * @param {number} len length of desired string in bytes
+ * @return {string} string with random hex digests
+ */
+LUA_FUNCTION_DEF(util, random_hex);
+
+/***
+ * @function util.zstd_compress(data, [level=1])
+ * Compresses input using zstd compression
+ *
+ * @param {string/rspamd_text} data input data
+ * @return {rspamd_text} compressed data
+ */
+LUA_FUNCTION_DEF(util, zstd_compress);
+
+/***
+ * @function util.zstd_decompress(data)
+ * Decompresses input using zstd algorithm
+ *
+ * @param {string/rspamd_text} data compressed data
+ * @return {error,rspamd_text} pair of error + decompressed text
+ */
+LUA_FUNCTION_DEF(util, zstd_decompress);
+
+/***
+ * @function util.gzip_decompress(data, [size_limit])
+ * Decompresses input using gzip algorithm
+ *
+ * @param {string/rspamd_text} data compressed data
+ * @param {integer} size_limit optional size limit
+ * @return {rspamd_text} decompressed text
+ */
+LUA_FUNCTION_DEF(util, gzip_decompress);
+
+/***
+ * @function util.inflate(data, [size_limit])
+ * Decompresses input using inflate algorithm
+ *
+ * @param {string/rspamd_text} data compressed data
+ * @param {integer} size_limit optional size limit
+ * @return {rspamd_text} decompressed text
+ */
+LUA_FUNCTION_DEF(util, inflate);
+
+/***
+ * @function util.gzip_compress(data, [level=1])
+ * Compresses input using gzip compression
+ *
+ * @param {string/rspamd_text} data input data
+ * @return {rspamd_text} compressed data
+ */
+LUA_FUNCTION_DEF(util, gzip_compress);
+
+/***
+ * @function util.normalize_prob(prob, [bias = 0.5])
+ * Normalize probabilities using polynom
+ *
+ * @param {number} prob probability param
+ * @param {number} bias number to subtract for making the final solution
+ * @return {number} normalized number
+ */
+LUA_FUNCTION_DEF(util, normalize_prob);
+/***
+ * @function util.is_utf_spoofed(str, [str2])
+ * Returns true if a string is spoofed (possibly with another string `str2`)
+ * @return {boolean} true if a string is spoofed
+ */
+LUA_FUNCTION_DEF(util, is_utf_spoofed);
+
+/**
+* @function util.is_utf_mixed_script(str)
+* Returns true if a string contains mixed unicode scripts
+* @param {string} String to check
+* @return {boolean} true if a string contains chars with mixed unicode script
+*/
+LUA_FUNCTION_DEF(util, is_utf_mixed_script);
+
+/**
+* @function util.is_utf_outside_range(str, range_start, range_end)
+* Returns true if a string contains chars outside range
+* @param {string} String to check
+* @param {number} start of character range similar to uset_addRange
+* @param {number} end of character range similar to uset_addRange
+* @return {boolean} true if a string contains chars outside selected utf range
+*/
+LUA_FUNCTION_DEF(util, is_utf_outside_range);
+
+/***
+* @function util.get_string_stats(str)
+* Returns table with number of letters and digits in string
+* @return {table} with string stats keys are "digits" and "letters"
+*/
+LUA_FUNCTION_DEF(util, get_string_stats);
+
+/***
+ * @function util.is_valid_utf8(str)
+ * Returns true if a string is valid UTF8 string
+ * @return {boolean} true if a string is spoofed
+ */
+LUA_FUNCTION_DEF(util, is_valid_utf8);
+
+/***
+ * @function util.has_obscured_unicode(str)
+ * Returns true if a string has obscure UTF symbols (zero width spaces, order marks), ignores invalid utf characters
+ * @return {boolean} true if a has obscured unicode characters (+ character and offset if found)
+ */
+LUA_FUNCTION_DEF(util, has_obscured_unicode);
+
+/***
+ * @function util.readline([prompt])
+ * Returns string read from stdin with history and editing support
+ * @return {string} string read from the input (with line endings stripped)
+ */
+LUA_FUNCTION_DEF(util, readline);
+
+/***
+ * @function util.readpassphrase([prompt])
+ * Returns string read from stdin disabling echo
+ * @return {string} string read from the input (with line endings stripped)
+ */
+LUA_FUNCTION_DEF(util, readpassphrase);
+
+/***
+ * @function util.file_exists(file)
+ * Checks if a specified file exists and is available for reading
+ * @return {boolean,string} true if file exists + string error if not
+ */
+LUA_FUNCTION_DEF(util, file_exists);
+
+/***
+ * @function util.mkdir(dir[, recursive])
+ * Creates a specified directory
+ * @return {boolean[,error]} true if directory has been created
+ */
+LUA_FUNCTION_DEF(util, mkdir);
+
+/***
+ * @function util.umask(mask)
+ * Sets new umask. Accepts either numeric octal string, e.g. '022' or a plain
+ * number, e.g. 0x12 (since Lua does not support octal integrals)
+ * @return {number} old umask
+ */
+LUA_FUNCTION_DEF(util, umask);
+
+/***
+ * @function util.isatty()
+ * Returns if stdout is a tty
+ * @return {boolean} true in case of output being tty
+ */
+LUA_FUNCTION_DEF(util, isatty);
+
+/***
+ * @function util.pack(fmt, ...)
+ *
+ * Backport of Lua 5.3 `string.pack` function:
+ * Returns a binary string containing the values v1, v2, etc. packed (that is,
+ * serialized in binary form) according to the format string `fmt`
+ * A format string is a sequence of conversion options. The conversion
+ * options are as follows:
+ *
+ * * <: sets little endian
+ * * >: sets big endian
+ * * =: sets native endian
+ * * ![n]: sets maximum alignment to n (default is native alignment)
+ * * b: a signed byte (char)
+ * * B: an unsigned byte (char)
+ * * h: a signed short (native size)
+ * * H: an unsigned short (native size)
+ * * l: a signed long (native size)
+ * * L: an unsigned long (native size)
+ * * j: a lua_Integer
+ * * J: a lua_Unsigned
+ * * T: a size_t (native size)
+ * * i[n]: a signed int with n bytes (default is native size)
+ * * I[n]: an unsigned int with n bytes (default is native size)
+ * * f: a float (native size)
+ * * d: a double (native size)
+ * * n: a lua_Number
+ * * cn: a fixed-sized string with n bytes
+ * * z: a zero-terminated string
+ * * s[n]: a string preceded by its length coded as an unsigned integer with
+ * * n bytes (default is a size_t)
+ * * x: one byte of padding
+ * * Xop: an empty item that aligns according to option op (which is otherwise ignored)
+ * * ' ': (empty space) ignored
+ *
+ * (A "[n]" means an optional integral numeral.) Except for padding, spaces,
+ * and configurations (options "xX <=>!"), each option corresponds to an
+ * argument (in string.pack) or a result (in string.unpack).
+ *
+ * For options "!n", "sn", "in", and "In", n can be any integer between 1 and
+ * All integral options check overflows; string.pack checks whether the given
+ * value fits in the given size; string.unpack checks whether the read value
+ * fits in a Lua integer.
+ *
+ * Any format string starts as if prefixed by "!1=", that is, with maximum
+ * alignment of 1 (no alignment) and native endianness.
+ *
+ * Alignment works as follows: For each option, the format gets extra padding
+ * until the data starts at an offset that is a multiple of the minimum
+ * between the option size and the maximum alignment; this minimum must be a
+ * power of 2. Options "c" and "z" are not aligned; option "s" follows the
+ * alignment of its starting integer.
+ *
+ * All padding is filled with zeros by string.pack (and ignored by unpack).
+ */
+LUA_FUNCTION_DEF(util, pack);
+
+/***
+ * @function util.packsize(fmt)
+ *
+ * Returns size of the packed binary string returned for the same `fmt` argument
+ * by @see util.pack
+ */
+LUA_FUNCTION_DEF(util, packsize);
+
+/***
+ * @function util.unpack(fmt, s [, pos])
+ * Unpacks string `s` according to the format string `fmt` as described in
+ * @see util.pack
+ *
+ * @returns {multiple} list of unpacked values according to `fmt`
+ */
+LUA_FUNCTION_DEF(util, unpack);
+
+/***
+ * @function util.caseless_hash(str[, seed])
+ * Calculates caseless non-crypto hash from a string or rspamd text
+ * @param str string or lua_text
+ * @param seed mandatory seed (0xdeadbabe by default)
+ * @return {int64} boxed int64_t
+ */
+LUA_FUNCTION_DEF(util, caseless_hash);
+
+/***
+ * @function util.caseless_hash_fast(str[, seed])
+ * Calculates caseless non-crypto hash from a string or rspamd text
+ * @param str string or lua_text
+ * @param seed mandatory seed (0xdeadbabe by default)
+ * @return {number} number from int64_t
+ */
+LUA_FUNCTION_DEF(util, caseless_hash_fast);
+
+/***
+ * @function util.get_hostname()
+ * Returns hostname for this machine
+ * @return {string} hostname
+ */
+LUA_FUNCTION_DEF(util, get_hostname);
+
+/***
+ * @function util.parse_content_type(ct_string, mempool)
+ * Parses content-type string to a table:
+ * - `type`
+ * - `subtype`
+ * - `charset`
+ * - `boundary`
+ * - other attributes
+ *
+ * @param {string} ct_string content type as string
+ * @param {rspamd_mempool} mempool needed to store temporary data (e.g. task pool)
+ * @return table or nil if cannot parse content type
+ */
+LUA_FUNCTION_DEF(util, parse_content_type);
+
+/***
+ * @function util.mime_header_encode(hdr)
+ * Encodes header if needed
+ * @param {string} hdr input header
+ * @return encoded header
+ */
+LUA_FUNCTION_DEF(util, mime_header_encode);
+
+/***
+ * @function util.btc_polymod(input_values)
+ * Performs bitcoin polymod function
+ * @param {table|numbers} input_values
+ * @return {boolean} true if polymod has been successful
+ */
+LUA_FUNCTION_DEF(util, btc_polymod);
+
+/***
+ * @function util.parse_smtp_date(str[, local_tz])
+ * Converts an SMTP date string to unix timestamp
+ * @param {string} str input string
+ * @param {boolean} local_tz convert to local tz if `true`
+ * @return {number} time as unix timestamp (converted to float)
+ */
+LUA_FUNCTION_DEF(util, parse_smtp_date);
+
+
+static const struct luaL_reg utillib_f[] = {
+ LUA_INTERFACE_DEF(util, create_event_base),
+ LUA_INTERFACE_DEF(util, load_rspamd_config),
+ LUA_INTERFACE_DEF(util, config_from_ucl),
+ LUA_INTERFACE_DEF(util, process_message),
+ LUA_INTERFACE_DEF(util, encode_base64),
+ LUA_INTERFACE_DEF(util, encode_qp),
+ LUA_INTERFACE_DEF(util, decode_qp),
+ LUA_INTERFACE_DEF(util, decode_base64),
+ LUA_INTERFACE_DEF(util, encode_base32),
+ LUA_INTERFACE_DEF(util, decode_base32),
+ LUA_INTERFACE_DEF(util, decode_url),
+ LUA_INTERFACE_DEF(util, tokenize_text),
+ LUA_INTERFACE_DEF(util, tanh),
+ LUA_INTERFACE_DEF(util, parse_html),
+ LUA_INTERFACE_DEF(util, levenshtein_distance),
+ LUA_INTERFACE_DEF(util, fold_header),
+ LUA_INTERFACE_DEF(util, is_uppercase),
+ LUA_INTERFACE_DEF(util, humanize_number),
+ LUA_INTERFACE_DEF(util, get_tld),
+ LUA_INTERFACE_DEF(util, glob),
+ {"parse_addr", lua_util_parse_mail_address},
+ LUA_INTERFACE_DEF(util, parse_mail_address),
+ LUA_INTERFACE_DEF(util, strlen_utf8),
+ LUA_INTERFACE_DEF(util, lower_utf8),
+ LUA_INTERFACE_DEF(util, normalize_utf8),
+ LUA_INTERFACE_DEF(util, transliterate),
+ LUA_INTERFACE_DEF(util, strequal_caseless),
+ LUA_INTERFACE_DEF(util, strequal_caseless_utf8),
+ LUA_INTERFACE_DEF(util, get_ticks),
+ LUA_INTERFACE_DEF(util, get_time),
+ LUA_INTERFACE_DEF(util, time_to_string),
+ LUA_INTERFACE_DEF(util, stat),
+ LUA_INTERFACE_DEF(util, unlink),
+ LUA_INTERFACE_DEF(util, lock_file),
+ LUA_INTERFACE_DEF(util, unlock_file),
+ LUA_INTERFACE_DEF(util, create_file),
+ LUA_INTERFACE_DEF(util, close_file),
+ LUA_INTERFACE_DEF(util, random_hex),
+ LUA_INTERFACE_DEF(util, zstd_compress),
+ LUA_INTERFACE_DEF(util, zstd_decompress),
+ LUA_INTERFACE_DEF(util, gzip_compress),
+ LUA_INTERFACE_DEF(util, gzip_decompress),
+ LUA_INTERFACE_DEF(util, inflate),
+ LUA_INTERFACE_DEF(util, normalize_prob),
+ LUA_INTERFACE_DEF(util, caseless_hash),
+ LUA_INTERFACE_DEF(util, caseless_hash_fast),
+ LUA_INTERFACE_DEF(util, is_utf_spoofed),
+ LUA_INTERFACE_DEF(util, is_utf_mixed_script),
+ LUA_INTERFACE_DEF(util, is_utf_outside_range),
+ LUA_INTERFACE_DEF(util, get_string_stats),
+ LUA_INTERFACE_DEF(util, is_valid_utf8),
+ LUA_INTERFACE_DEF(util, has_obscured_unicode),
+ LUA_INTERFACE_DEF(util, readline),
+ LUA_INTERFACE_DEF(util, readpassphrase),
+ LUA_INTERFACE_DEF(util, file_exists),
+ LUA_INTERFACE_DEF(util, mkdir),
+ LUA_INTERFACE_DEF(util, umask),
+ LUA_INTERFACE_DEF(util, isatty),
+ LUA_INTERFACE_DEF(util, get_hostname),
+ LUA_INTERFACE_DEF(util, parse_content_type),
+ LUA_INTERFACE_DEF(util, mime_header_encode),
+ LUA_INTERFACE_DEF(util, pack),
+ LUA_INTERFACE_DEF(util, unpack),
+ LUA_INTERFACE_DEF(util, packsize),
+ LUA_INTERFACE_DEF(util, btc_polymod),
+ LUA_INTERFACE_DEF(util, parse_smtp_date),
+ {NULL, NULL}};
+
+LUA_FUNCTION_DEF(int64, tostring);
+LUA_FUNCTION_DEF(int64, fromstring);
+LUA_FUNCTION_DEF(int64, tonumber);
+LUA_FUNCTION_DEF(int64, hex);
+
+static const struct luaL_reg int64lib_f[] = {
+ LUA_INTERFACE_DEF(int64, fromstring),
+ {NULL, NULL}};
+static const struct luaL_reg int64lib_m[] = {
+ LUA_INTERFACE_DEF(int64, tostring),
+ LUA_INTERFACE_DEF(int64, tonumber),
+ LUA_INTERFACE_DEF(int64, hex),
+ {"__tostring", lua_int64_tostring},
+ {NULL, NULL}};
+
+LUA_FUNCTION_DEF(ev_base, loop);
+
+static const struct luaL_reg ev_baselib_m[] = {
+ LUA_INTERFACE_DEF(ev_base, loop),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static gint64
+lua_check_int64(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{int64}");
+ luaL_argcheck(L, ud != NULL, pos, "'int64' expected");
+ return ud ? *((gint64 *) ud) : 0LL;
+}
+
+
+static gint
+lua_util_create_event_base(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct ev_loop **pev_base;
+
+ pev_base = lua_newuserdata(L, sizeof(struct ev_loop *));
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+ *pev_base = ev_loop_new(EVFLAG_SIGNALFD | EVBACKEND_ALL);
+
+ return 1;
+}
+
+static gint
+lua_util_load_rspamd_config(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg, **pcfg;
+ const gchar *cfg_name;
+
+ cfg_name = luaL_checkstring(L, 1);
+
+ if (cfg_name) {
+ cfg = rspamd_config_new(RSPAMD_CONFIG_INIT_SKIP_LUA);
+ cfg->lua_state = L;
+
+ if (rspamd_config_read(cfg, cfg_name, NULL, NULL, NULL, FALSE, NULL)) {
+ msg_err_config("cannot load config from %s", cfg_name);
+ lua_pushnil(L);
+ }
+ else {
+ rspamd_config_post_load(cfg, 0);
+ pcfg = lua_newuserdata(L, sizeof(struct rspamd_config *));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ }
+ }
+
+ return 1;
+}
+
+static gint
+parse_config_options(const char *str_options)
+{
+ gint ret = 0;
+ gchar **vec;
+ const gchar *str;
+ guint i, l;
+
+ vec = g_strsplit_set(str_options, ",;", -1);
+ if (vec) {
+ l = g_strv_length(vec);
+ for (i = 0; i < l; i++) {
+ str = vec[i];
+
+ if (g_ascii_strcasecmp(str, "INIT_URL") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_URL;
+ }
+ else if (g_ascii_strcasecmp(str, "INIT_LIBS") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_LIBS;
+ }
+ else if (g_ascii_strcasecmp(str, "INIT_SYMCACHE") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_SYMCACHE;
+ }
+ else if (g_ascii_strcasecmp(str, "INIT_VALIDATE") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_VALIDATE;
+ }
+ else if (g_ascii_strcasecmp(str, "INIT_NO_TLD") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_NO_TLD;
+ }
+ else if (g_ascii_strcasecmp(str, "INIT_PRELOAD_MAPS") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_PRELOAD_MAPS;
+ }
+ else {
+ msg_warn("bad type: %s", str);
+ }
+ }
+
+ g_strfreev(vec);
+ }
+
+ return ret;
+}
+
+static gint
+lua_util_config_from_ucl(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = NULL, **pcfg;
+ struct rspamd_rcl_sections_map *top;
+ GError *err = NULL;
+ ucl_object_t *obj;
+ const char *str_options = NULL;
+ gint int_options = 0;
+
+
+ obj = ucl_object_lua_import(L, 1);
+ if (lua_gettop(L) == 2) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ str_options = lua_tostring(L, 2);
+ int_options = parse_config_options(str_options);
+ }
+ else {
+ msg_err("config_from_ucl: second parameter is expected to be string");
+ ucl_object_unref(obj);
+ lua_pushnil(L);
+ }
+ }
+
+ if (obj) {
+ cfg = rspamd_config_new(RSPAMD_CONFIG_INIT_SKIP_LUA);
+ cfg->lua_state = L;
+
+ cfg->cfg_ucl_obj = obj;
+ top = rspamd_rcl_config_init(cfg, NULL);
+
+ if (!rspamd_rcl_parse(top, cfg, cfg, cfg->cfg_pool, cfg->cfg_ucl_obj, &err)) {
+ msg_err("rcl parse error: %s", err->message);
+ ucl_object_unref(obj);
+ lua_pushnil(L);
+ }
+ else {
+
+ if (int_options & RSPAMD_CONFIG_INIT_LIBS) {
+ cfg->libs_ctx = rspamd_init_libs();
+ }
+
+ rspamd_config_post_load(cfg, int_options);
+ pcfg = lua_newuserdata(L, sizeof(struct rspamd_config *));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ }
+
+ rspamd_rcl_sections_free(top);
+ }
+
+ return 1;
+}
+
+static gboolean
+lua_util_task_fin(struct rspamd_task *task, void *ud)
+{
+ ucl_object_t **target = ud;
+
+ *target = rspamd_protocol_write_ucl(task, RSPAMD_PROTOCOL_DEFAULT);
+ rdns_resolver_release(task->resolver->r);
+
+ return TRUE;
+}
+
+static gint
+lua_util_process_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *message;
+ gsize mlen;
+ struct rspamd_task *task;
+ struct ev_loop *base;
+ ucl_object_t *res = NULL;
+
+ message = luaL_checklstring(L, 2, &mlen);
+
+ if (cfg != NULL && message != NULL) {
+ base = ev_loop_new(EVFLAG_SIGNALFD | EVBACKEND_ALL);
+ rspamd_init_filters(cfg, false, false);
+ task = rspamd_task_new(NULL, cfg, NULL, NULL, base, FALSE);
+ task->msg.begin = rspamd_mempool_alloc(task->task_pool, mlen);
+ rspamd_strlcpy((gpointer) task->msg.begin, message, mlen);
+ task->msg.len = mlen;
+ task->fin_callback = lua_util_task_fin;
+ task->fin_arg = &res;
+ task->resolver = rspamd_dns_resolver_init(NULL, base, cfg);
+ task->s = rspamd_session_create(task->task_pool, rspamd_task_fin,
+ NULL, (event_finalizer_t) rspamd_task_free, task);
+
+ if (!rspamd_task_load_message(task, NULL, message, mlen)) {
+ lua_pushnil(L);
+ }
+ else {
+ if (rspamd_task_process(task, RSPAMD_TASK_PROCESS_ALL)) {
+ ev_loop(base, 0);
+
+ if (res != NULL) {
+ ucl_object_push_lua(L, res, true);
+
+ ucl_object_unref(res);
+ }
+ else {
+ ucl_object_push_lua(L,
+ rspamd_protocol_write_ucl(task, RSPAMD_PROTOCOL_DEFAULT),
+ true);
+ rdns_resolver_release(task->resolver->r);
+ rspamd_session_destroy(task->s);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ ev_loop_destroy(base);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_encode_base64(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ gchar *out;
+ gsize outlen;
+ long str_lim = 0;
+ gboolean fold = FALSE;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (lua_gettop(L) > 1) {
+ str_lim = luaL_checkinteger(L, 2);
+ fold = str_lim > 0;
+ }
+
+ if (t == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+ else {
+
+ if (fold) {
+ out = rspamd_encode_base64(t->start, t->len, str_lim, &outlen);
+ }
+ else {
+ enum rspamd_newlines_type how = RSPAMD_TASK_NEWLINES_CRLF;
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 3);
+
+ if (g_ascii_strcasecmp(how_str, "cr") == 0) {
+ how = RSPAMD_TASK_NEWLINES_CR;
+ }
+ else if (g_ascii_strcasecmp(how_str, "lf") == 0) {
+ how = RSPAMD_TASK_NEWLINES_LF;
+ }
+ else if (g_ascii_strcasecmp(how_str, "crlf") != 0) {
+ return luaL_error(L, "invalid newline style: %s", how_str);
+ }
+ }
+
+ out = rspamd_encode_base64_fold(t->start, t->len, str_lim, &outlen, how);
+ }
+
+ if (out != NULL) {
+ lua_new_text(L, out, outlen, TRUE);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_encode_qp(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ const gchar *s = NULL;
+ gchar *out;
+ gsize inlen, outlen;
+ guint str_lim = 0;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = luaL_checklstring(L, 1, &inlen);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ s = t->start;
+ inlen = t->len;
+ }
+ }
+
+ if (lua_gettop(L) > 1) {
+ str_lim = luaL_checknumber(L, 2);
+ }
+
+ if (s == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ enum rspamd_newlines_type how = RSPAMD_TASK_NEWLINES_CRLF;
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 3);
+
+ if (g_ascii_strcasecmp(how_str, "cr") == 0) {
+ how = RSPAMD_TASK_NEWLINES_CR;
+ }
+ else if (g_ascii_strcasecmp(how_str, "lf") == 0) {
+ how = RSPAMD_TASK_NEWLINES_LF;
+ }
+ else if (g_ascii_strcasecmp(how_str, "crlf") != 0) {
+ return luaL_error(L, "invalid newline style: %s", how_str);
+ }
+ }
+
+ out = rspamd_encode_qp_fold(s, inlen, str_lim, &outlen, how);
+
+ if (out != NULL) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = out;
+ t->len = outlen;
+ /* Need destruction */
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_decode_qp(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t, *out;
+ const gchar *s = NULL;
+ gsize inlen = 0;
+ gssize outlen;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = luaL_checklstring(L, 1, &inlen);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ s = t->start;
+ inlen = t->len;
+ }
+ }
+
+ if (s == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ out = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ out->start = g_malloc(inlen + 1);
+ out->flags = RSPAMD_TEXT_FLAG_OWN;
+ outlen = rspamd_decode_qp_buf(s, inlen, (gchar *) out->start, inlen + 1);
+
+ if (outlen > 0) {
+ out->len = outlen;
+ }
+ else {
+ /*
+ * It removes out and frees memory on gc due to RSPAMD_TEXT_FLAG_OWN
+ */
+ lua_pop(L, 1);
+ lua_pushnil(L);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_decode_base64(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ const gchar *s = NULL;
+ gsize inlen = 0, outlen;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = luaL_checklstring(L, 1, &inlen);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ s = t->start;
+ inlen = t->len;
+ }
+ }
+
+ if (s != NULL) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->len = (inlen / 4) * 3 + 3;
+ t->start = g_malloc(t->len);
+
+ rspamd_cryptobox_base64_decode(s, inlen, (guchar *) t->start,
+ &outlen);
+ t->len = outlen;
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_encode_base32(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ const gchar *s = NULL;
+ gchar *out;
+ enum rspamd_base32_type btype = RSPAMD_BASE32_DEFAULT;
+ gsize inlen, outlen;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = luaL_checklstring(L, 1, &inlen);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ s = t->start;
+ inlen = t->len;
+ }
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ btype = rspamd_base32_decode_type_from_str(lua_tostring(L, 2));
+
+ if (btype == RSPAMD_BASE32_INVALID) {
+ return luaL_error(L, "invalid b32 type: %s", lua_tostring(L, 2));
+ }
+ }
+
+ if (s == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+ else {
+ out = rspamd_encode_base32(s, inlen, btype);
+
+ if (out != NULL) {
+ t = lua_newuserdata(L, sizeof(*t));
+ outlen = strlen(out);
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = out;
+ t->len = outlen;
+ /* Need destruction */
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_decode_base32(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ const gchar *s = NULL;
+ gsize inlen, outlen;
+ enum rspamd_base32_type btype = RSPAMD_BASE32_DEFAULT;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = luaL_checklstring(L, 1, &inlen);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ s = t->start;
+ inlen = t->len;
+ }
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ btype = rspamd_base32_decode_type_from_str(lua_tostring(L, 2));
+
+ if (btype == RSPAMD_BASE32_INVALID) {
+ return luaL_error(L, "invalid b32 type: %s", lua_tostring(L, 2));
+ }
+ }
+
+ if (s != NULL) {
+ guchar *decoded;
+
+ decoded = rspamd_decode_base32(s, inlen, &outlen, btype);
+
+ if (decoded) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = (const gchar *) decoded;
+ t->len = outlen;
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_decode_url(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t != NULL) {
+ struct rspamd_lua_text *out = lua_new_text(L, NULL, t->len, TRUE);
+
+ out->len = rspamd_url_decode((char *) out->start, t->start, t->len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_util_tokenize_text(lua_State *L)
+{
+ return lua_parsers_tokenize_text(L);
+}
+
+static gint
+lua_util_tanh(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gdouble in = luaL_checknumber(L, 1);
+
+ lua_pushnumber(L, tanh(in));
+
+ return 1;
+}
+
+static gint
+lua_util_parse_html(lua_State *L)
+{
+ return lua_parsers_parse_html(L);
+}
+
+static gint
+lua_util_levenshtein_distance(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1, *t2;
+ gint dist = 0;
+ guint replace_cost = 1;
+
+ t1 = lua_check_text_or_string(L, 1);
+ t2 = lua_check_text_or_string(L, 2);
+ if (lua_isnumber(L, 3)) {
+ replace_cost = lua_tointeger(L, 3);
+ }
+
+ if (t1 && t2) {
+ dist = rspamd_strings_levenshtein_distance(t1->start, t1->len, t2->start, t2->len,
+ replace_cost);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushinteger(L, dist);
+
+ return 1;
+}
+
+static gint
+lua_util_fold_header(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *how, *stop_chars = NULL;
+ struct rspamd_lua_text *name, *value;
+ GString *folded;
+
+ name = lua_check_text_or_string(L, 1);
+ value = lua_check_text_or_string(L, 2);
+
+ if (name && value) {
+
+ if (lua_isstring(L, 3)) {
+
+ how = lua_tostring(L, 3);
+
+ if (lua_isstring(L, 4)) {
+ stop_chars = lua_tostring(L, 4);
+ }
+
+ if (strcmp(how, "cr") == 0) {
+ folded = rspamd_header_value_fold(name->start, name->len,
+ value->start, value->len,
+ 0,
+ RSPAMD_TASK_NEWLINES_CR, stop_chars);
+ }
+ else if (strcmp(how, "lf") == 0) {
+ folded = rspamd_header_value_fold(name->start, name->len,
+ value->start, value->len, 0,
+ RSPAMD_TASK_NEWLINES_LF, stop_chars);
+ }
+ else {
+ folded = rspamd_header_value_fold(name->start, name->len,
+ value->start, value->len, 0,
+ RSPAMD_TASK_NEWLINES_CRLF, stop_chars);
+ }
+ }
+ else {
+ folded = rspamd_header_value_fold(name->start, name->len,
+ value->start, value->len, 0,
+ RSPAMD_TASK_NEWLINES_CRLF, stop_chars);
+ }
+
+ if (folded) {
+ lua_pushlstring(L, folded->str, folded->len);
+ g_string_free(folded, TRUE);
+
+ return 1;
+ }
+ }
+
+ lua_pushnil(L);
+ return 1;
+}
+
+static gint
+lua_util_is_uppercase(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint32 i = 0;
+ UChar32 uc;
+ guint nlc = 0, nuc = 0;
+
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 1);
+ if (t) {
+ while (i >= 0 && i < t->len) {
+ U8_NEXT(t->start, i, t->len, uc);
+
+ if (uc < 0) {
+ break;
+ }
+
+ if (u_isupper(uc)) {
+ nuc++;
+ }
+ else if (u_islower(uc)) {
+ nlc++;
+ }
+ }
+ }
+
+ if (nuc > 0 && nlc == 0) {
+ lua_pushboolean(L, TRUE);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_humanize_number(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint64 number = luaL_checkinteger(L, 1);
+ gchar numbuf[32];
+
+
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%hL", number);
+ lua_pushstring(L, numbuf);
+
+ return 1;
+}
+
+static gint
+lua_util_get_tld(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *host;
+ gsize hostlen;
+ rspamd_ftok_t tld;
+
+ host = luaL_checklstring(L, 1, &hostlen);
+
+ if (host) {
+ if (!rspamd_url_find_tld(host, hostlen, &tld)) {
+ lua_pushlstring(L, host, hostlen);
+ }
+ else {
+ lua_pushlstring(L, tld.begin, tld.len);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_util_glob(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *pattern;
+ glob_t gl;
+ gint top, i, flags = 0;
+
+ top = lua_gettop(L);
+ memset(&gl, 0, sizeof(gl));
+
+ for (i = 1; i <= top; i++, flags |= GLOB_APPEND) {
+ pattern = luaL_checkstring(L, i);
+
+ if (pattern) {
+ if (glob(pattern, flags, NULL, &gl) != 0) {
+ /* There is no way to return error here, so just create an table */
+ lua_createtable(L, 0, 0);
+ globfree(&gl);
+
+ return 1;
+ }
+ }
+ }
+
+ lua_createtable(L, gl.gl_pathc, 0);
+ /* Push results */
+ for (i = 0; i < (gint) gl.gl_pathc; i++) {
+ lua_pushstring(L, gl.gl_pathv[i]);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ globfree(&gl);
+
+ return 1;
+}
+
+static gint
+lua_util_parse_mail_address(lua_State *L)
+{
+ return lua_parsers_parse_mail_address(L);
+}
+
+static gint
+lua_util_strlen_utf8(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t) {
+ gint32 i = 0, nchars = 0;
+ UChar32 uc;
+
+ while (i < t->len) {
+ U8_NEXT((guint8 *) t->start, i, t->len, uc);
+ nchars++;
+ }
+
+ lua_pushinteger(L, nchars);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_lower_utf8(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+
+ gchar *dst;
+ UChar32 uc;
+ UBool err = 0;
+ gint32 i = 0, j = 0;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t) {
+ dst = g_malloc(t->len);
+
+ while (i < t->len && err == 0) {
+ U8_NEXT((guint8 *) t->start, i, t->len, uc);
+ uc = u_tolower(uc);
+ U8_APPEND(dst, j, t->len, uc, err);
+ }
+
+ if (lua_isstring(L, 1)) {
+ lua_pushlstring(L, dst, j);
+ g_free(dst);
+ }
+ else {
+ t = lua_new_text(L, dst, j, FALSE);
+ /* We have actually allocated text data before */
+ t->flags |= RSPAMD_TEXT_FLAG_OWN;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_normalize_utf8(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ bool is_text = lua_type(L, 1) == LUA_TUSERDATA;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ char *cpy = g_malloc(t->len + 1);
+ memcpy(cpy, t->start, t->len);
+ cpy[t->len] = '\0';
+ gsize len = t->len;
+ enum rspamd_utf8_normalise_result res = rspamd_normalise_unicode_inplace(cpy, &len);
+
+ if (is_text) {
+ struct rspamd_lua_text *out = lua_new_text(L, cpy, len, FALSE);
+ out->flags |= RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ lua_pushlstring(L, cpy, len);
+ g_free(cpy);
+ }
+
+ lua_pushinteger(L, res);
+
+ return 2;
+}
+
+static gint
+lua_util_transliterate(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ t = lua_check_text_or_string(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ gsize outlen;
+ char *transliterated = rspamd_utf8_transliterate(t->start, t->len, &outlen);
+ lua_new_text(L, transliterated, outlen, TRUE);
+
+ return 1;
+}
+
+static gint
+lua_util_strequal_caseless(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1, *t2;
+ gint ret = -1;
+
+ t1 = lua_check_text_or_string(L, 1);
+ t2 = lua_check_text_or_string(L, 2);
+
+ if (t1 && t2) {
+
+ if (t1->len == t2->len) {
+ ret = rspamd_lc_cmp(t1->start, t2->start, t1->len);
+ }
+ else {
+ ret = t1->len - t2->len;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, (ret == 0) ? true : false);
+ return 1;
+}
+
+static gint
+lua_util_strequal_caseless_utf8(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1, *t2;
+ gint ret = -1;
+
+ t1 = lua_check_text_or_string(L, 1);
+ t2 = lua_check_text_or_string(L, 2);
+
+ if (t1 && t2) {
+ ret = rspamd_utf8_strcmp_sizes(t1->start, t1->len, t2->start, t2->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, (ret == 0) ? true : false);
+
+ return 1;
+}
+
+static gint
+lua_util_get_ticks(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gdouble ticks;
+ gboolean rdtsc = FALSE;
+
+ if (lua_isboolean(L, 1)) {
+ rdtsc = lua_toboolean(L, 1);
+ }
+
+ ticks = rspamd_get_ticks(rdtsc);
+ lua_pushnumber(L, ticks);
+
+ return 1;
+}
+
+static gint
+lua_util_get_time(lua_State *L)
+{
+ LUA_TRACE_POINT;
+
+ lua_pushnumber(L, ev_time());
+
+ return 1;
+}
+
+static gint
+lua_util_time_to_string(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gdouble seconds;
+ char timebuf[128];
+
+ if (lua_isnumber(L, 1)) {
+ seconds = lua_tonumber(L, 1);
+ }
+ else {
+ seconds = ev_time();
+ }
+
+ rspamd_http_date_format(timebuf, sizeof(timebuf), seconds);
+ lua_pushstring(L, timebuf);
+
+ return 1;
+}
+
+static gint
+lua_util_stat(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *fpath;
+ struct stat st;
+
+ fpath = luaL_checkstring(L, 1);
+
+ if (fpath) {
+ if (stat(fpath, &st) == -1) {
+ lua_pushstring(L, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushnil(L);
+ lua_createtable(L, 0, 3);
+
+ lua_pushstring(L, "size");
+ lua_pushinteger(L, st.st_size);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "mtime");
+ lua_pushinteger(L, st.st_mtime);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "type");
+ if (S_ISREG(st.st_mode)) {
+ lua_pushstring(L, "regular");
+ }
+ else if (S_ISDIR(st.st_mode)) {
+ lua_pushstring(L, "directory");
+ }
+ else {
+ lua_pushstring(L, "special");
+ }
+ lua_settable(L, -3);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+static gint
+lua_util_unlink(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *fpath;
+ gint ret;
+
+ fpath = luaL_checkstring(L, 1);
+
+ if (fpath) {
+ ret = unlink(fpath);
+
+ if (ret == -1) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_lock_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *fpath;
+ gint fd = -1;
+ gboolean own = FALSE;
+
+#if !HAVE_FLOCK
+ struct flock fl = {
+ .l_type = F_WRLCK,
+ .l_whence = SEEK_SET,
+ .l_start = 0,
+ .l_len = 0};
+#endif
+
+ fpath = luaL_checkstring(L, 1);
+
+ if (fpath) {
+ if (lua_isnumber(L, 2)) {
+ fd = lua_tointeger(L, 2);
+ }
+ else {
+ fd = open(fpath, O_RDONLY);
+ own = TRUE;
+ }
+
+ if (fd == -1) {
+ lua_pushnil(L);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+#if HAVE_FLOCK
+ if (flock(fd, LOCK_EX) == -1) {
+#else
+ if (fcntl(fd, F_SETLKW, &fl) == -1) {
+#endif
+ lua_pushnil(L);
+ lua_pushstring(L, strerror(errno));
+
+ if (own) {
+ close(fd);
+ }
+
+ return 2;
+ }
+
+ lua_pushinteger(L, fd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_unlock_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint fd = -1, ret, serrno;
+ gboolean do_close = TRUE;
+
+#if !HAVE_FLOCK
+ struct flock fl = {
+ .l_type = F_UNLCK,
+ .l_whence = SEEK_SET,
+ .l_start = 0,
+ .l_len = 0};
+#endif
+
+ if (lua_isnumber(L, 1)) {
+ fd = lua_tointeger(L, 1);
+
+ if (lua_isboolean(L, 2)) {
+ do_close = lua_toboolean(L, 2);
+ }
+
+#if HAVE_FLOCK
+ ret = flock(fd, LOCK_UN);
+#else
+ ret = fcntl(fd, F_SETLKW, &fl);
+#endif
+
+ if (do_close) {
+ serrno = errno;
+ close(fd);
+ errno = serrno;
+ }
+
+ if (ret == -1) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_create_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint fd, mode = 00644;
+ const gchar *fpath;
+
+ fpath = luaL_checkstring(L, 1);
+
+ if (fpath) {
+ if (lua_isnumber(L, 2)) {
+ mode = lua_tointeger(L, 2);
+ }
+
+ fd = rspamd_file_xopen(fpath, O_RDWR | O_CREAT | O_TRUNC, mode, 0);
+
+ if (fd == -1) {
+ lua_pushnil(L);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ lua_pushinteger(L, fd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_close_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint fd = -1;
+
+ if (lua_isnumber(L, 1)) {
+ fd = lua_tointeger(L, 1);
+
+ if (close(fd) == -1) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_random_hex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gchar *buf;
+ gint buflen;
+
+ buflen = lua_tointeger(L, 1);
+
+ if (buflen <= 0) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ buf = g_malloc(buflen);
+ rspamd_random_hex(buf, buflen);
+ lua_pushlstring(L, buf, buflen);
+ g_free(buf);
+
+ return 1;
+}
+
+static gint
+lua_util_zstd_compress(lua_State *L)
+{
+ return lua_compress_zstd_compress(L);
+}
+
+static gint
+lua_util_zstd_decompress(lua_State *L)
+{
+ return lua_compress_zstd_decompress(L);
+}
+
+static gint
+lua_util_gzip_compress(lua_State *L)
+{
+ return lua_compress_zlib_compress(L);
+}
+
+static gint
+lua_util_gzip_decompress(lua_State *L)
+{
+ return lua_compress_zlib_decompress(L, true);
+}
+
+static gint
+lua_util_inflate(lua_State *L)
+{
+ return lua_compress_zlib_decompress(L, false);
+}
+
+static gint
+lua_util_normalize_prob(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gdouble x, bias = 0.5;
+
+ x = lua_tonumber(L, 1);
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ bias = lua_tonumber(L, 2);
+ }
+
+ lua_pushnumber(L, rspamd_normalize_probability(x, bias));
+
+ return 1;
+}
+
+static gint
+lua_util_caseless_hash(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ guint64 seed = 0xdeadbabe, h;
+ struct rspamd_lua_text *t = NULL;
+ gint64 *r;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ seed = lua_tointeger(L, 2);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ seed = lua_check_int64(L, 2);
+ }
+
+ h = rspamd_icase_hash(t->start, t->len, seed);
+ r = lua_newuserdata(L, sizeof(*r));
+ *r = h;
+ rspamd_lua_setclass(L, "rspamd{int64}", -1);
+
+ return 1;
+}
+
+static gint
+lua_util_caseless_hash_fast(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ guint64 seed = 0xdeadbabe, h;
+ struct rspamd_lua_text *t = NULL;
+ union {
+ guint64 i;
+ double d;
+ } u;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ seed = lua_tointeger(L, 2);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ seed = lua_check_int64(L, 2);
+ }
+
+ /*
+ * Here, we loose entropy from 64 bits to 52 bits roughly, however,
+ * it is still fine for practical applications
+ */
+
+ h = rspamd_icase_hash(t->start, t->len, seed);
+ u.i = G_GUINT64_CONSTANT(0x3FF) << 52 | h >> 12;
+ lua_pushnumber(L, u.d - 1.0);
+
+ return 1;
+}
+
+static gint
+lua_util_is_utf_spoofed(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gsize l1, l2;
+ gint ret, nres = 2;
+ const gchar *s1 = lua_tolstring(L, 1, &l1),
+ *s2 = lua_tolstring(L, 2, &l2);
+ static USpoofChecker *spc, *spc_sgl;
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ if (s1 && s2) {
+ if (spc == NULL) {
+ spc = uspoof_open(&uc_err);
+
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("cannot init spoof checker: %s", u_errorName(uc_err));
+ lua_pushboolean(L, false);
+
+ return 1;
+ }
+ }
+
+ ret = uspoof_areConfusableUTF8(spc, s1, l1, s2, l2, &uc_err);
+ }
+ else if (s1) {
+ /* We have just s1, not s2 */
+ if (spc_sgl == NULL) {
+ spc_sgl = uspoof_open(&uc_err);
+
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("cannot init spoof checker: %s", u_errorName(uc_err));
+ lua_pushboolean(L, false);
+
+ return 1;
+ }
+
+ uspoof_setChecks(spc_sgl,
+ USPOOF_INVISIBLE | USPOOF_MIXED_SCRIPT_CONFUSABLE | USPOOF_ANY_CASE,
+ &uc_err);
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("Cannot set proper checks for uspoof: %s", u_errorName(uc_err));
+ lua_pushboolean(L, false);
+ uspoof_close(spc);
+ return 1;
+ }
+ }
+
+ ret = uspoof_checkUTF8(spc_sgl, s1, l1, NULL, &uc_err);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, !!(ret != 0));
+
+ switch (ret) {
+ case 0:
+ nres = 1;
+ break;
+ case USPOOF_SINGLE_SCRIPT_CONFUSABLE:
+ lua_pushstring(L, "single");
+ break;
+ case USPOOF_MIXED_SCRIPT_CONFUSABLE:
+ lua_pushstring(L, "multiple");
+ break;
+ case USPOOF_WHOLE_SCRIPT_CONFUSABLE:
+ lua_pushstring(L, "whole");
+ break;
+ default:
+ lua_pushstring(L, "unknown");
+ break;
+ }
+
+ return nres;
+}
+
+static gint
+lua_util_is_utf_mixed_script(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gsize len_of_string;
+ const guchar *string_to_check = lua_tolstring(L, 1, &len_of_string);
+ UScriptCode last_script_code = USCRIPT_INVALID_CODE;
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ if (string_to_check) {
+ uint index = 0;
+ UChar32 char_to_check = 0;
+
+ while (index < len_of_string) {
+ U8_NEXT(string_to_check, index, len_of_string, char_to_check);
+
+ if (char_to_check < 0) {
+ return luaL_error(L, "passed string is not valid utf");
+ }
+
+ UScriptCode current_script_code = uscript_getScript(char_to_check, &uc_err);
+
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("cannot get unicode script for character, error: %s",
+ u_errorName(uc_err));
+ lua_pushboolean(L, false);
+
+ return 1;
+ }
+
+ if (current_script_code != USCRIPT_COMMON &&
+ current_script_code != USCRIPT_INHERITED) {
+
+ if (last_script_code == USCRIPT_INVALID_CODE) {
+ last_script_code = current_script_code;
+ }
+ else {
+ if (last_script_code != current_script_code) {
+ lua_pushboolean(L, true);
+
+ return 1;
+ }
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, false);
+
+ return 1;
+}
+
+static gint
+lua_util_get_string_stats(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint num_of_digits = 0, num_of_letters = 0;
+ struct rspamd_lua_text *t;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t) {
+ const gchar *p = t->start, *end = t->start + t->len;
+ while (p < end) {
+ if (g_ascii_isdigit(*p)) {
+ num_of_digits++;
+ }
+ else if (g_ascii_isalpha(*p)) {
+ num_of_letters++;
+ }
+ p++;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_createtable(L, 0, 2);
+ lua_pushstring(L, "digits");
+ lua_pushinteger(L, num_of_digits);
+ lua_settable(L, -3);
+ lua_pushstring(L, "letters");
+ lua_pushinteger(L, num_of_letters);
+ lua_settable(L, -3);
+
+ return 1;
+}
+
+
+static gint
+lua_util_is_utf_outside_range(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint ret;
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 1);
+ guint32 range_start = lua_tointeger(L, 2);
+ guint32 range_end = lua_tointeger(L, 3);
+
+ static rspamd_lru_hash_t *validators;
+
+ if (validators == NULL) {
+ validators = rspamd_lru_hash_new_full(16, g_free, (GDestroyNotify) uspoof_close, g_int64_hash, g_int64_equal);
+ }
+
+ if (t) {
+ guint64 hash_key = (guint64) range_end << 32 || range_start;
+
+ USpoofChecker *validator = rspamd_lru_hash_lookup(validators, &hash_key, 0);
+
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ if (validator == NULL) {
+ USet *allowed_chars;
+ guint64 *creation_hash_key = g_malloc(sizeof(guint64));
+ *creation_hash_key = hash_key;
+
+ validator = uspoof_open(&uc_err);
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("cannot init spoof checker: %s", u_errorName(uc_err));
+ lua_pushboolean(L, false);
+ uspoof_close(validator);
+ g_free(creation_hash_key);
+ return 1;
+ }
+
+ allowed_chars = uset_openEmpty();
+ uset_addRange(allowed_chars, range_start, range_end);
+ uspoof_setAllowedChars(validator, allowed_chars, &uc_err);
+
+ uspoof_setChecks(validator,
+ USPOOF_CHAR_LIMIT | USPOOF_ANY_CASE, &uc_err);
+
+ uset_close(allowed_chars);
+
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("Cannot configure uspoof: %s", u_errorName(uc_err));
+ lua_pushboolean(L, false);
+ uspoof_close(validator);
+ g_free(creation_hash_key);
+ return 1;
+ }
+
+ rspamd_lru_hash_insert(validators, creation_hash_key, validator,
+ 0, 0);
+ }
+
+ gint32 pos = 0;
+ ret = uspoof_checkUTF8(validator, t->start, t->len, &pos,
+ &uc_err);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, !!(ret != 0));
+
+ return 1;
+}
+
+
+static gint
+lua_util_get_hostname(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gchar *hostbuf;
+ gsize hostlen;
+
+ hostlen = sysconf(_SC_HOST_NAME_MAX);
+
+ if (hostlen <= 0) {
+ hostlen = 256;
+ }
+ else {
+ hostlen++;
+ }
+
+ hostbuf = g_alloca(hostlen);
+ memset(hostbuf, 0, hostlen);
+ gethostname(hostbuf, hostlen - 1);
+
+ lua_pushstring(L, hostbuf);
+
+ return 1;
+}
+
+static gint
+lua_util_parse_content_type(lua_State *L)
+{
+ return lua_parsers_parse_content_type(L);
+}
+
+
+static gint
+lua_util_mime_header_encode(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gsize len;
+ const gchar *hdr = luaL_checklstring(L, 1, &len);
+ gchar *encoded;
+
+ if (!hdr) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ encoded = rspamd_mime_header_encode(hdr, len);
+ lua_pushstring(L, encoded);
+ g_free(encoded);
+
+ return 1;
+}
+
+static gint
+lua_util_is_valid_utf8(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 1);
+
+ if (t) {
+ goffset error_offset = rspamd_fast_utf8_validate(t->start, t->len);
+
+ if (error_offset == 0) {
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ lua_pushinteger(L, error_offset);
+
+ return 2;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_has_obscured_unicode(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint32 i = 0, prev_i;
+ UChar32 uc;
+
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 1);
+
+ while (i < t->len) {
+ prev_i = i;
+ U8_NEXT(t->start, i, t->len, uc);
+
+ if (uc > 0) {
+ if (IS_OBSCURED_CHAR(uc)) {
+ lua_pushboolean(L, true);
+ lua_pushinteger(L, uc); /* Character */
+ lua_pushinteger(L, prev_i); /* Offset */
+
+ return 3;
+ }
+ }
+ }
+
+ lua_pushboolean(L, false);
+
+ return 1;
+}
+
+static gint
+lua_util_readline(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *prompt = "";
+ gchar *input;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ prompt = lua_tostring(L, 1);
+ }
+#ifdef WITH_LUA_REPL
+ static Replxx *rx_instance = NULL;
+
+ if (rx_instance == NULL) {
+ rx_instance = replxx_init();
+ /* See https://github.com/AmokHuginnsson/replxx/issues/137 */
+ replxx_history_add(rx_instance, "");
+ }
+
+ input = (gchar *) replxx_input(rx_instance, prompt);
+
+ if (input) {
+ lua_pushstring(L, input);
+ }
+ else {
+ lua_pushnil(L);
+ }
+#else
+ size_t linecap = 0;
+ ssize_t linelen;
+
+ fprintf(stdout, "%s ", prompt);
+
+ linelen = getline(&input, &linecap, stdin);
+
+ if (linelen > 0) {
+ if (input[linelen - 1] == '\n') {
+ linelen--;
+ }
+
+ lua_pushlstring(L, input, linelen);
+ free(input);
+ }
+ else {
+ lua_pushnil(L);
+ }
+#endif
+
+ return 1;
+}
+
+static gint
+lua_util_readpassphrase(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gchar test_password[8192];
+ gsize r;
+
+ r = rspamd_read_passphrase(test_password, sizeof(test_password), 0, NULL);
+
+ if (r > 0) {
+ lua_pushlstring(L, test_password, r);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ /* In fact, we still pass it to Lua which is not very safe */
+ rspamd_explicit_memzero(test_password, sizeof(test_password));
+
+ return 1;
+}
+
+static gint
+lua_util_file_exists(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *fname = luaL_checkstring(L, 1);
+ gint serrno;
+
+ if (fname) {
+ if (access(fname, R_OK) == -1) {
+ serrno = errno;
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(serrno));
+ }
+ else {
+ lua_pushboolean(L, true);
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+static gint
+lua_util_mkdir(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *dname = luaL_checkstring(L, 1);
+ gboolean recursive = FALSE;
+ gint r = -1;
+
+ if (dname) {
+ if (lua_isboolean(L, 2)) {
+ recursive = lua_toboolean(L, 2);
+ }
+
+ if (recursive) {
+ char path[PATH_MAX];
+ gsize len, i;
+
+ len = rspamd_strlcpy(path, dname, sizeof(path));
+
+ /* Strip last / */
+ if (path[len - 1] == '/') {
+ path[len - 1] = '\0';
+ len--;
+ }
+
+ for (i = 1; i < len; i++) {
+ if (path[i] == '/') {
+ path[i] = '\0';
+
+ errno = 0;
+ r = mkdir(path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
+
+ if (r == -1 && errno != EEXIST) {
+ break;
+ }
+
+ path[i] = '/';
+ }
+ }
+
+ /* Final path component */
+ r = mkdir(path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
+ }
+ else {
+ r = mkdir(dname, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
+ }
+
+ if (r == -1 && errno != EEXIST) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_util_umask(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ mode_t mask = 0, old;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ const gchar *str = lua_tostring(L, 1);
+
+ if (str[0] == '0') {
+ /* e.g. '022' */
+ mask = strtol(str, NULL, 8);
+ }
+ else {
+ /* XXX: implement modestring parsing at some point */
+ return luaL_error(L, "invalid arguments");
+ }
+ }
+ else if (lua_type(L, 1) == LUA_TNUMBER) {
+ mask = lua_tointeger(L, 1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ old = umask(mask);
+
+ lua_pushinteger(L, old);
+
+ return 1;
+}
+
+static gint
+lua_util_isatty(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ if (isatty(STDOUT_FILENO)) {
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ return 1;
+}
+
+/* Backport from Lua 5.3 */
+
+/******************************************************************************
+* Copyright (C) 1994-2016 Lua.org, PUC-Rio.
+*
+* Permission is hereby granted, free of charge, to any person obtaining
+* a copy of this software and associated documentation files (the
+* "Software"), to deal in the Software without restriction, including
+* without limitation the rights to use, copy, modify, merge, publish,
+* distribute, sublicense, and/or sell copies of the Software, and to
+* permit persons to whom the Software is furnished to do so, subject to
+* the following conditions:
+*
+* The above copyright notice and this permission notice shall be
+* included in all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+/*
+** {======================================================
+** PACK/UNPACK
+** =======================================================
+*/
+
+
+/* value used for padding */
+#if !defined(LUA_PACKPADBYTE)
+#define LUA_PACKPADBYTE 0x00
+#endif
+
+/* maximum size for the binary representation of an integer */
+#define MAXINTSIZE 16
+
+/* number of bits in a character */
+#define NB CHAR_BIT
+
+/* mask for one character (NB 1's) */
+#define MC ((1 << NB) - 1)
+
+/* size of a lua_Integer */
+#define SZINT ((int) sizeof(lua_Integer))
+
+#define MAX_SIZET ((size_t) (~(size_t) 0))
+
+#define MAXSIZE \
+ (sizeof(size_t) < sizeof(int) ? MAX_SIZET : (size_t) (INT_MAX))
+
+
+/* dummy union to get native endianness */
+static const union {
+ int dummy;
+ char little; /* true if machine is little endian */
+} nativeendian = {1};
+
+
+/* dummy structure to get native alignment requirements */
+struct cD {
+ char c;
+ union {
+ double d;
+ void *p;
+ lua_Integer i;
+ lua_Number n;
+ } u;
+};
+
+#define MAXALIGN (offsetof(struct cD, u))
+
+/*
+** Union for serializing floats
+*/
+typedef union Ftypes {
+ float f;
+ double d;
+ lua_Number n;
+ char buff[5 * sizeof(lua_Number)]; /* enough for any float type */
+} Ftypes;
+
+
+/*
+** information to pack/unpack stuff
+*/
+typedef struct Header {
+ lua_State *L;
+ int islittle;
+ int maxalign;
+} Header;
+
+/*
+** options for pack/unpack
+*/
+typedef enum KOption {
+ Kint, /* signed integers */
+ Kuint, /* unsigned integers */
+ Kfloat, /* floating-point numbers */
+ Kchar, /* fixed-length strings */
+ Kstring, /* strings with prefixed length */
+ Kzstr, /* zero-terminated strings */
+ Kpadding, /* padding */
+ Kpaddalign, /* padding for alignment */
+ Knop /* no-op (configuration or spaces) */
+} KOption;
+
+#if LUA_VERSION_NUM <= 502
+#define lua_Unsigned size_t
+#endif
+
+#if LUA_VERSION_NUM < 502
+
+#define lua_Unsigned size_t
+
+typedef struct luaL_Buffer_53 {
+ luaL_Buffer b; /* make incorrect code crash! */
+ char *ptr;
+ size_t nelems;
+ size_t capacity;
+ lua_State *L2;
+} luaL_Buffer_53;
+
+#define luaL_Buffer luaL_Buffer_53
+#define COMPAT53_PREFIX lua
+#undef COMPAT53_API
+
+#if defined(__GNUC__) || defined(__clang__)
+#define COMPAT53_API __attribute__((__unused__)) static
+#else
+#define COMPAT53_API static
+#endif
+
+#define COMPAT53_CONCAT_HELPER(a, b) a##b
+#define COMPAT53_CONCAT(a, b) COMPAT53_CONCAT_HELPER(a, b)
+
+#define luaL_buffinit COMPAT53_CONCAT(COMPAT53_PREFIX, _buffinit_53)
+COMPAT53_API void luaL_buffinit(lua_State *L, luaL_Buffer_53 *B);
+#define luaL_prepbuffsize COMPAT53_CONCAT(COMPAT53_PREFIX, _prepbufsize_53)
+COMPAT53_API char *luaL_prepbuffsize(luaL_Buffer_53 *B, size_t s);
+#define luaL_addlstring COMPAT53_CONCAT(COMPAT53_PREFIX, _addlstring_53)
+COMPAT53_API void luaL_addlstring(luaL_Buffer_53 *B, const char *s, size_t l);
+#define luaL_addvalue COMPAT53_CONCAT(COMPAT53_PREFIX, _addvalue_53)
+COMPAT53_API void luaL_addvalue(luaL_Buffer_53 *B);
+#define luaL_pushresult COMPAT53_CONCAT(COMPAT53_PREFIX, _pushresult_53)
+COMPAT53_API void luaL_pushresult(luaL_Buffer_53 *B);
+#undef luaL_buffinitsize
+#define luaL_buffinitsize(L, B, s) \
+ (luaL_buffinit(L, B), luaL_prepbuffsize(B, s))
+
+#undef luaL_prepbuffer
+#define luaL_prepbuffer(B) \
+ luaL_prepbuffsize(B, LUAL_BUFFERSIZE)
+
+#undef luaL_addchar
+#define luaL_addchar(B, c) \
+ ((void) ((B)->nelems < (B)->capacity || luaL_prepbuffsize(B, 1)), \
+ ((B)->ptr[(B)->nelems++] = (c)))
+
+#undef luaL_addsize
+#define luaL_addsize(B, s) \
+ ((B)->nelems += (s))
+
+#undef luaL_addstring
+#define luaL_addstring(B, s) \
+ luaL_addlstring(B, s, strlen(s))
+
+#undef luaL_pushresultsize
+#define luaL_pushresultsize(B, s) \
+ (luaL_addsize(B, s), luaL_pushresult(B))
+
+COMPAT53_API void
+luaL_buffinit(lua_State *L, luaL_Buffer_53 *B)
+{
+ /* make it crash if used via pointer to a 5.1-style luaL_Buffer */
+ B->b.p = NULL;
+ B->b.L = NULL;
+ B->b.lvl = 0;
+ /* reuse the buffer from the 5.1-style luaL_Buffer though! */
+ B->ptr = B->b.buffer;
+ B->nelems = 0;
+ B->capacity = LUAL_BUFFERSIZE;
+ B->L2 = L;
+}
+
+
+COMPAT53_API char *
+luaL_prepbuffsize(luaL_Buffer_53 *B, size_t s)
+{
+ if (B->capacity - B->nelems < s) { /* needs to grow */
+ char *newptr = NULL;
+ size_t newcap = B->capacity * 2;
+ if (newcap - B->nelems < s)
+ newcap = B->nelems + s;
+ if (newcap < B->capacity) /* overflow */
+ luaL_error(B->L2, "buffer too large");
+ newptr = (char *) lua_newuserdata(B->L2, newcap);
+ memcpy(newptr, B->ptr, B->nelems);
+ if (B->ptr != B->b.buffer) {
+ lua_replace(B->L2, -2); /* remove old buffer */
+ }
+ B->ptr = newptr;
+ B->capacity = newcap;
+ }
+ return B->ptr + B->nelems;
+}
+
+
+COMPAT53_API void
+luaL_addlstring(luaL_Buffer_53 *B, const char *s, size_t l)
+{
+ memcpy(luaL_prepbuffsize(B, l), s, l);
+ luaL_addsize(B, l);
+}
+
+
+COMPAT53_API void
+luaL_addvalue(luaL_Buffer_53 *B)
+{
+ size_t len = 0;
+ const char *s = lua_tolstring(B->L2, -1, &len);
+ if (!s)
+ luaL_error(B->L2, "cannot convert value to string");
+ if (B->ptr != B->b.buffer) {
+ lua_insert(B->L2, -2); /* userdata buffer must be at stack top */
+ }
+ luaL_addlstring(B, s, len);
+ lua_remove(B->L2, B->ptr != B->b.buffer ? -2 : -1);
+}
+
+
+COMPAT53_API void
+luaL_pushresult(luaL_Buffer_53 *B)
+{
+ lua_pushlstring(B->L2, B->ptr, B->nelems);
+ if (B->ptr != B->b.buffer) {
+ lua_replace(B->L2, -2); /* remove userdata buffer */
+ }
+}
+
+#endif
+
+/*
+** Read an integer numeral from string 'fmt' or return 'df' if
+** there is no numeral
+*/
+static int
+digit(int c)
+{
+ return '0' <= c && c <= '9';
+}
+
+static int
+getnum(const char **fmt, int df)
+{
+ if (!digit(**fmt)) /* no number? */
+ return df; /* return default value */
+ else {
+ int a = 0;
+ do {
+ a = a * 10 + (*((*fmt)++) - '0');
+ } while (digit(**fmt) && a <= ((int) MAXSIZE - 9) / 10);
+ return a;
+ }
+}
+
+
+/*
+** Read an integer numeral and raises an error if it is larger
+** than the maximum size for integers.
+*/
+static int
+getnumlimit(Header *h, const char **fmt, int df)
+{
+ int sz = getnum(fmt, df);
+ if (sz > MAXINTSIZE || sz <= 0)
+ luaL_error(h->L, "integral size (%d) out of limits [1,%d]",
+ sz, MAXINTSIZE);
+ return sz;
+}
+
+
+/*
+** Initialize Header
+*/
+static void
+initheader(lua_State *L, Header *h)
+{
+ h->L = L;
+ h->islittle = nativeendian.little;
+ h->maxalign = 1;
+}
+
+
+/*
+** Read and classify next option. 'size' is filled with option's size.
+*/
+static KOption
+getoption(Header *h, const char **fmt, int *size)
+{
+ int opt = *((*fmt)++);
+ *size = 0; /* default */
+ switch (opt) {
+ case 'b':
+ *size = sizeof(char);
+ return Kint;
+ case 'B':
+ *size = sizeof(char);
+ return Kuint;
+ case 'h':
+ *size = sizeof(short);
+ return Kint;
+ case 'H':
+ *size = sizeof(short);
+ return Kuint;
+ case 'l':
+ *size = sizeof(long);
+ return Kint;
+ case 'L':
+ *size = sizeof(long);
+ return Kuint;
+ case 'j':
+ *size = sizeof(lua_Integer);
+ return Kint;
+ case 'J':
+ *size = sizeof(lua_Integer);
+ return Kuint;
+ case 'T':
+ *size = sizeof(size_t);
+ return Kuint;
+ case 'f':
+ *size = sizeof(float);
+ return Kfloat;
+ case 'd':
+ *size = sizeof(double);
+ return Kfloat;
+ case 'n':
+ *size = sizeof(lua_Number);
+ return Kfloat;
+ case 'i':
+ *size = getnumlimit(h, fmt, sizeof(int));
+ return Kint;
+ case 'I':
+ *size = getnumlimit(h, fmt, sizeof(int));
+ return Kuint;
+ case 's':
+ *size = getnumlimit(h, fmt, sizeof(size_t));
+ return Kstring;
+ case 'c':
+ *size = getnum(fmt, -1);
+ if (*size == -1)
+ luaL_error(h->L, "missing size for format option 'c'");
+ return Kchar;
+ case 'z':
+ return Kzstr;
+ case 'x':
+ *size = 1;
+ return Kpadding;
+ case 'X':
+ return Kpaddalign;
+ case ' ':
+ break;
+ case '<':
+ h->islittle = 1;
+ break;
+ case '>':
+ h->islittle = 0;
+ break;
+ case '=':
+ h->islittle = nativeendian.little;
+ break;
+ case '!':
+ h->maxalign = getnumlimit(h, fmt, MAXALIGN);
+ break;
+ default:
+ luaL_error(h->L, "invalid format option '%c'", opt);
+ }
+ return Knop;
+}
+
+
+/*
+** Read, classify, and fill other details about the next option.
+** 'psize' is filled with option's size, 'notoalign' with its
+** alignment requirements.
+** Local variable 'size' gets the size to be aligned. (Kpadal option
+** always gets its full alignment, other options are limited by
+** the maximum alignment ('maxalign'). Kchar option needs no alignment
+** despite its size.
+*/
+static KOption
+getdetails(Header *h, size_t totalsize,
+ const char **fmt, int *psize, int *ntoalign)
+{
+ KOption opt = getoption(h, fmt, psize);
+ int align = *psize; /* usually, alignment follows size */
+ if (opt == Kpaddalign) { /* 'X' gets alignment from following option */
+ if (**fmt == '\0' || getoption(h, fmt, &align) == Kchar || align == 0)
+ luaL_argerror(h->L, 1, "invalid next option for option 'X'");
+ }
+ if (align <= 1 || opt == Kchar) /* need no alignment? */
+ *ntoalign = 0;
+ else {
+ if (align > h->maxalign) /* enforce maximum alignment */
+ align = h->maxalign;
+ if ((align & (align - 1)) != 0) /* is 'align' not a power of 2? */
+ luaL_argerror(h->L, 1, "format asks for alignment not power of 2");
+ *ntoalign = (align - (int) (totalsize & (align - 1))) & (align - 1);
+ }
+ return opt;
+}
+
+
+/*
+** Pack integer 'n' with 'size' bytes and 'islittle' endianness.
+** The final 'if' handles the case when 'size' is larger than
+** the size of a Lua integer, correcting the extra sign-extension
+** bytes if necessary (by default they would be zeros).
+*/
+static void
+packint(luaL_Buffer *b, lua_Unsigned n,
+ int islittle, int size, int neg)
+{
+ char *buff = luaL_prepbuffsize(b, size);
+ int i;
+ buff[islittle ? 0 : size - 1] = (char) (n & MC); /* first byte */
+ for (i = 1; i < size; i++) {
+ n >>= NB;
+ buff[islittle ? i : size - 1 - i] = (char) (n & MC);
+ }
+ if (neg && size > SZINT) { /* negative number need sign extension? */
+ for (i = SZINT; i < size; i++) /* correct extra bytes */
+ buff[islittle ? i : size - 1 - i] = (char) MC;
+ }
+ luaL_addsize(b, size); /* add result to buffer */
+}
+
+
+/*
+** Copy 'size' bytes from 'src' to 'dest', correcting endianness if
+** given 'islittle' is different from native endianness.
+*/
+static void
+copywithendian(volatile char *dest, volatile const char *src,
+ int size, int islittle)
+{
+ if (islittle == nativeendian.little) {
+ while (size-- != 0)
+ *(dest++) = *(src++);
+ }
+ else {
+ dest += size - 1;
+ while (size-- != 0)
+ *(dest--) = *(src++);
+ }
+}
+
+
+static int
+lua_util_pack(lua_State *L)
+{
+ luaL_Buffer b;
+ Header h;
+ const char *fmt = luaL_checkstring(L, 1); /* format string */
+ int arg = 1; /* current argument to pack */
+ size_t totalsize = 0; /* accumulate total size of result */
+ initheader(L, &h);
+ lua_pushnil(L); /* mark to separate arguments from string buffer */
+ luaL_buffinit(L, &b);
+
+ while (*fmt != '\0') {
+ int size, ntoalign;
+ KOption opt = getdetails(&h, totalsize, &fmt, &size, &ntoalign);
+ totalsize += ntoalign + size;
+ while (ntoalign-- > 0)
+ luaL_addchar(&b, LUA_PACKPADBYTE); /* fill alignment */
+ arg++;
+ switch (opt) {
+ case Kint: { /* signed integers */
+ lua_Integer n = luaL_checkinteger(L, arg);
+ if (size < SZINT) { /* need overflow check? */
+ lua_Integer lim = (lua_Integer) 1 << ((size * NB) - 1);
+ luaL_argcheck(L, -lim <= n && n < lim, arg, "integer overflow");
+ }
+ packint(&b, (lua_Unsigned) n, h.islittle, size, (n < 0));
+ break;
+ }
+ case Kuint: { /* unsigned integers */
+ lua_Integer n = luaL_checkinteger(L, arg);
+ if (size < SZINT) /* need overflow check? */
+ luaL_argcheck(L,
+ (lua_Unsigned) n < ((lua_Unsigned) 1 << (size * NB)),
+ arg,
+ "unsigned overflow");
+ packint(&b, (lua_Unsigned) n, h.islittle, size, 0);
+ break;
+ }
+ case Kfloat: { /* floating-point options */
+ volatile Ftypes u;
+ char *buff = luaL_prepbuffsize(&b, size);
+ lua_Number n = luaL_checknumber(L, arg); /* get argument */
+ if (size == sizeof(u.f))
+ u.f = (float) n; /* copy it into 'u' */
+ else if (size == sizeof(u.d))
+ u.d = (double) n;
+ else
+ u.n = n;
+ /* move 'u' to final result, correcting endianness if needed */
+ copywithendian(buff, u.buff, size, h.islittle);
+ luaL_addsize(&b, size);
+ break;
+ }
+ case Kchar: { /* fixed-size string */
+ size_t len;
+ const char *s = luaL_checklstring(L, arg, &len);
+ if ((size_t) size <=
+ len) /* string larger than (or equal to) needed? */
+ luaL_addlstring(&b,
+ s,
+ size); /* truncate string to asked size */
+ else { /* string smaller than needed */
+ luaL_addlstring(&b, s, len); /* add it all */
+ while (len++ < (size_t) size) /* pad extra space */
+ luaL_addchar(&b, LUA_PACKPADBYTE);
+ }
+ break;
+ }
+ case Kstring: { /* strings with length count */
+ size_t len;
+ const char *s = luaL_checklstring(L, arg, &len);
+ luaL_argcheck(L, size >= (int) sizeof(size_t) || len < ((size_t) 1 << (size * NB)),
+ arg, "string length does not fit in given size");
+ packint(&b,
+ (lua_Unsigned) len,
+ h.islittle,
+ size,
+ 0); /* pack length */
+ luaL_addlstring(&b, s, len);
+ totalsize += len;
+ break;
+ }
+ case Kzstr: { /* zero-terminated string */
+ size_t len;
+ const char *s = luaL_checklstring(L, arg, &len);
+ luaL_argcheck(L, strlen(s) == len, arg, "string contains zeros");
+ luaL_addlstring(&b, s, len);
+ luaL_addchar(&b, '\0'); /* add zero at the end */
+ totalsize += len + 1;
+ break;
+ }
+ case Kpadding:
+ luaL_addchar(&b, LUA_PACKPADBYTE); /* FALLTHROUGH */
+ case Kpaddalign:
+ case Knop:
+ arg--; /* undo increment */
+ break;
+ }
+ }
+ luaL_pushresult(&b);
+ return 1;
+}
+
+
+static int
+lua_util_packsize(lua_State *L)
+{
+ Header h;
+ const char *fmt = luaL_checkstring(L, 1); /* format string */
+ size_t totalsize = 0; /* accumulate total size of result */
+ initheader(L, &h);
+ while (*fmt != '\0') {
+ int size, ntoalign;
+ KOption opt = getdetails(&h, totalsize, &fmt, &size, &ntoalign);
+ size += ntoalign; /* total space used by option */
+ luaL_argcheck(L, totalsize <= MAXSIZE - size, 1,
+ "format result too large");
+ totalsize += size;
+ switch (opt) {
+ case Kstring: /* strings with length count */
+ case Kzstr: /* zero-terminated string */
+ luaL_argerror(L, 1, "variable-length format");
+ /* call never return, but to avoid warnings: */ /* FALLTHROUGH */
+ default:
+ break;
+ }
+ }
+ lua_pushinteger(L, (lua_Integer) totalsize);
+ return 1;
+}
+
+
+/*
+** Unpack an integer with 'size' bytes and 'islittle' endianness.
+** If size is smaller than the size of a Lua integer and integer
+** is signed, must do sign extension (propagating the sign to the
+** higher bits); if size is larger than the size of a Lua integer,
+** it must check the unread bytes to see whether they do not cause an
+** overflow.
+*/
+static lua_Integer
+unpackint(lua_State *L, const char *str,
+ int islittle, int size, int issigned)
+{
+ lua_Unsigned res = 0;
+ int i;
+ int limit = (size <= SZINT) ? size : SZINT;
+ for (i = limit - 1; i >= 0; i--) {
+ res <<= NB;
+ res |= (lua_Unsigned) (unsigned char) str[islittle ? i : size - 1 - i];
+ }
+ if (size < SZINT) { /* real size smaller than lua_Integer? */
+ if (issigned) { /* needs sign extension? */
+ lua_Unsigned mask = (lua_Unsigned) 1 << (size * NB - 1);
+ res = ((res ^ mask) - mask); /* do sign extension */
+ }
+ }
+ else if (size > SZINT) { /* must check unread bytes */
+ int mask = (!issigned || (lua_Integer) res >= 0) ? 0 : MC;
+ for (i = limit; i < size; i++) {
+ if ((unsigned char) str[islittle ? i : size - 1 - i] != mask)
+ luaL_error(L,
+ "%d-byte integer does not fit into Lua Integer",
+ size);
+ }
+ }
+ return (lua_Integer) res;
+}
+
+static lua_Integer
+posrelat(lua_Integer pos, size_t len)
+{
+ if (pos >= 0)
+ return pos;
+ else if (0u - (size_t) pos > len)
+ return 0;
+ else
+ return (lua_Integer) len + pos + 1;
+}
+
+static int
+lua_util_unpack(lua_State *L)
+{
+ Header h;
+ const char *fmt = luaL_checkstring(L, 1);
+ size_t ld;
+ const char *data;
+ int n = 0; /* number of results */
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ ld = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 2, &ld);
+ }
+
+ size_t pos = (size_t) posrelat(luaL_optinteger(L, 3, 1), ld) - 1;
+ luaL_argcheck(L, pos <= ld, 3, "initial position out of string");
+
+ initheader(L, &h);
+
+ while (*fmt != '\0') {
+ int size, ntoalign;
+ KOption opt = getdetails(&h, pos, &fmt, &size, &ntoalign);
+ if ((size_t) ntoalign + size > ~pos || pos + ntoalign + size > ld)
+ luaL_argerror(L, 2, "data string too short");
+ pos += ntoalign; /* skip alignment */
+ /* stack space for item + next position */
+ luaL_checkstack(L, 2, "too many results");
+ n++;
+ switch (opt) {
+ case Kint:
+ case Kuint: {
+ lua_Integer res = unpackint(L, data + pos, h.islittle, size,
+ (opt == Kint));
+ lua_pushinteger(L, res);
+ break;
+ }
+ case Kfloat: {
+ volatile Ftypes u;
+ lua_Number num;
+ copywithendian(u.buff, data + pos, size, h.islittle);
+ if (size == sizeof(u.f))
+ num = (lua_Number) u.f;
+ else if (size == sizeof(u.d))
+ num = (lua_Number) u.d;
+ else
+ num = u.n;
+ lua_pushnumber(L, num);
+ break;
+ }
+ case Kchar: {
+ lua_pushlstring(L, data + pos, size);
+ break;
+ }
+ case Kstring: {
+ size_t len = (size_t) unpackint(L,
+ data + pos,
+ h.islittle,
+ size,
+ 0);
+ luaL_argcheck(L,
+ pos + len + size <= ld,
+ 2,
+ "data string too short");
+ lua_pushlstring(L, data + pos + size, len);
+ pos += len; /* skip string */
+ break;
+ }
+ case Kzstr: {
+ size_t len = (int) strlen(data + pos);
+ lua_pushlstring(L, data + pos, len);
+ pos += len + 1; /* skip string plus final '\0' */
+ break;
+ }
+ case Kpaddalign:
+ case Kpadding:
+ case Knop:
+ n--; /* undo increment */
+ break;
+ }
+ pos += size;
+ }
+ lua_pushinteger(L, pos + 1); /* next position */
+ return n + 1;
+}
+
+static int
+lua_util_btc_polymod(lua_State *L)
+{
+ guint64 c = 1;
+
+ if (lua_type(L, 1) != LUA_TTABLE) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ for (lua_pushnil(L); lua_next(L, 1); lua_pop(L, 1)) {
+ guint8 c0 = c >> 35;
+ guint64 d = lua_tointeger(L, -1);
+
+ c = ((c & 0x07ffffffff) << 5) ^ d;
+
+ if (c0 & 0x01) c ^= 0x98f2bc8e61;
+ if (c0 & 0x02) c ^= 0x79b76d99e2;
+ if (c0 & 0x04) c ^= 0xf33e5fb3c4;
+ if (c0 & 0x08) c ^= 0xae2eabe2a8;
+ if (c0 & 0x10) c ^= 0x1e4f43e470;
+ }
+
+ if ((c ^ 1) == 0) {
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ return 1;
+}
+
+static int
+lua_util_parse_smtp_date(lua_State *L)
+{
+ return lua_parsers_parse_smtp_date(L);
+}
+
+
+static gint
+lua_load_util(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, utillib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_int64(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, int64lib_f);
+
+ return 1;
+}
+
+
+void luaopen_util(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{ev_base}", ev_baselib_m);
+ lua_pop(L, 1);
+ rspamd_lua_new_class(L, "rspamd{int64}", int64lib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_util", lua_load_util);
+ rspamd_lua_add_preload(L, "rspamd_int64", lua_load_int64);
+}
+
+static int
+lua_int64_tostring(lua_State *L)
+{
+ gint64 n = lua_check_int64(L, 1);
+ gchar buf[32];
+ bool is_signed = false;
+
+ if (lua_isboolean(L, 2)) {
+ is_signed = lua_toboolean(L, 2);
+ }
+
+ if (is_signed) {
+ rspamd_snprintf(buf, sizeof(buf), "%L", n);
+ }
+ else {
+ rspamd_snprintf(buf, sizeof(buf), "%uL", n);
+ }
+ lua_pushstring(L, buf);
+
+ return 1;
+}
+
+static int
+lua_int64_fromstring(lua_State *L)
+{
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 1);
+
+ if (t && t->len > 0) {
+ guint64 u64;
+ const char *p = t->start;
+ gsize len = t->len;
+ bool neg = false;
+
+ /*
+ * We use complicated negation to allow both signed and unsinged values to
+ * fit into result.
+ * So we read int64 as unsigned and copy it to signed number.
+ * If we wanted u64 this allows to have the same memory representation of
+ * signed and unsigned.
+ * If we wanted signed i64 we still can use -1000500 and it will be parsed
+ * properly
+ */
+ if (*p == '-') {
+ neg = true;
+ p++;
+ len--;
+ }
+ if (!rspamd_strtou64(p, len, &u64)) {
+ lua_pushnil(L);
+ lua_pushstring(L, "invalid number");
+ return 2;
+ }
+
+ gint64 *i64_p = lua_newuserdata(L, sizeof(gint64));
+ rspamd_lua_setclass(L, "rspamd{int64}", -1);
+ memcpy(i64_p, &u64, sizeof(u64));
+
+ if (neg) {
+ *i64_p = -(*i64_p);
+ }
+ }
+ else {
+ }
+
+ return 1;
+}
+
+static int
+lua_int64_tonumber(lua_State *L)
+{
+ gint64 n = lua_check_int64(L, 1);
+ gdouble d;
+
+ d = n;
+ lua_pushinteger(L, d);
+
+ return 1;
+}
+
+static int
+lua_int64_hex(lua_State *L)
+{
+ gint64 n = lua_check_int64(L, 1);
+ gchar buf[32];
+
+ rspamd_snprintf(buf, sizeof(buf), "%XL", n);
+ lua_pushstring(L, buf);
+
+ return 1;
+}
+
+static int
+lua_ev_base_loop(lua_State *L)
+{
+ int flags = 0;
+ struct ev_loop *ev_base;
+
+ ev_base = lua_check_ev_base(L, 1);
+ if (lua_isnumber(L, 2)) {
+ flags = lua_tointeger(L, 2);
+ }
+
+ int ret = ev_run(ev_base, flags);
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
diff --git a/src/lua/lua_worker.c b/src/lua/lua_worker.c
new file mode 100644
index 0000000..025b97b
--- /dev/null
+++ b/src/lua/lua_worker.c
@@ -0,0 +1,883 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "unix-std.h"
+#include "worker_util.h"
+#include "rspamd_control.h"
+#include "ottery.h"
+
+#ifdef WITH_JEMALLOC
+#include <jemalloc/jemalloc.h>
+#endif
+
+#include <sys/wait.h>
+#include <src/libserver/rspamd_control.h>
+
+/***
+ * @module rspamd_worker
+ * This module provides methods to access worker related functions in various
+ * places, such as periodic events or on_load events.
+ */
+
+
+LUA_FUNCTION_DEF(worker, get_name);
+LUA_FUNCTION_DEF(worker, get_stat);
+LUA_FUNCTION_DEF(worker, get_index);
+LUA_FUNCTION_DEF(worker, get_count);
+LUA_FUNCTION_DEF(worker, get_pid);
+LUA_FUNCTION_DEF(worker, is_scanner);
+LUA_FUNCTION_DEF(worker, is_primary_controller);
+LUA_FUNCTION_DEF(worker, spawn_process);
+LUA_FUNCTION_DEF(worker, get_mem_stats);
+LUA_FUNCTION_DEF(worker, add_control_handler);
+
+const luaL_reg worker_reg[] = {
+ LUA_INTERFACE_DEF(worker, get_name),
+ {"get_type", lua_worker_get_name},
+ LUA_INTERFACE_DEF(worker, get_stat),
+ LUA_INTERFACE_DEF(worker, get_index),
+ LUA_INTERFACE_DEF(worker, get_count),
+ LUA_INTERFACE_DEF(worker, get_pid),
+ LUA_INTERFACE_DEF(worker, spawn_process),
+ LUA_INTERFACE_DEF(worker, is_scanner),
+ LUA_INTERFACE_DEF(worker, is_primary_controller),
+ LUA_INTERFACE_DEF(worker, get_mem_stats),
+ LUA_INTERFACE_DEF(worker, add_control_handler),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static struct rspamd_worker *
+lua_check_worker(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{worker}");
+ luaL_argcheck(L, ud != NULL, pos, "'worker' expected");
+ return ud ? *((struct rspamd_worker **) ud) : NULL;
+}
+
+static gint
+lua_worker_get_stat(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ rspamd_mempool_stat_t mem_st;
+ struct rspamd_stat *stat, stat_copy;
+ ucl_object_t *top, *sub;
+ gint i;
+ guint64 spam = 0, ham = 0;
+
+ memset(&mem_st, 0, sizeof(mem_st));
+ rspamd_mempool_stat(&mem_st);
+ memcpy(&stat_copy, w->srv->stat, sizeof(stat_copy));
+ stat = &stat_copy;
+ top = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(top, ucl_object_fromint(stat->messages_scanned), "scanned", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(stat->messages_learned), "learned", 0, false);
+ if (stat->messages_scanned > 0) {
+ sub = ucl_object_typed_new(UCL_OBJECT);
+ for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
+ ucl_object_insert_key(sub,
+ ucl_object_fromint(stat->actions_stat[i]),
+ rspamd_action_to_str(i), 0, false);
+ if (i < METRIC_ACTION_GREYLIST) {
+ spam += stat->actions_stat[i];
+ }
+ else {
+ ham += stat->actions_stat[i];
+ }
+ }
+ ucl_object_insert_key(top, sub, "actions", 0, false);
+ }
+ else {
+ sub = ucl_object_typed_new(UCL_OBJECT);
+ for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
+ ucl_object_insert_key(sub,
+ 0,
+ rspamd_action_to_str(i), 0, false);
+ }
+ ucl_object_insert_key(top, sub, "actions", 0, false);
+ }
+ ucl_object_insert_key(top, ucl_object_fromint(spam), "spam_count", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(ham), "ham_count", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(stat->connections_count), "connections", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(stat->control_connections_count),
+ "control_connections", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.pools_allocated), "pools_allocated", 0,
+ false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.pools_freed), "pools_freed", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.bytes_allocated), "bytes_allocated", 0,
+ false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(
+ mem_st.chunks_allocated),
+ "chunks_allocated", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.shared_chunks_allocated),
+ "shared_chunks_allocated", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.chunks_freed), "chunks_freed", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(
+ mem_st.oversized_chunks),
+ "chunks_oversized", 0, false);
+
+ ucl_object_push_lua(L, top, true);
+ ucl_object_unref(top);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_worker_get_name(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushstring(L, g_quark_to_string(w->type));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_worker_get_index(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushinteger(L, w->index);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_worker_get_count(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushinteger(L, w->cf->count);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_worker_get_pid(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushinteger(L, w->pid);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_worker_is_scanner(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushboolean(L, rspamd_worker_is_scanner(w));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_worker_is_primary_controller(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushboolean(L, rspamd_worker_is_primary_controller(w));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct rspamd_control_cbdata {
+ lua_State *L;
+ rspamd_mempool_t *pool;
+ struct rspamd_worker *w;
+ struct rspamd_config *cfg;
+ struct ev_loop *event_loop;
+ struct rspamd_async_session *session;
+ enum rspamd_control_type cmd;
+ gint cbref;
+ gint fd;
+};
+
+static gboolean
+lua_worker_control_fin_session(void *ud)
+{
+ struct rspamd_control_reply rep;
+ struct rspamd_control_cbdata *cbd = (struct rspamd_control_cbdata *) ud;
+ rspamd_mempool_t *pool;
+
+ pool = cbd->pool;
+
+ memset(&rep, 0, sizeof(rep));
+ rep.type = cbd->cmd;
+
+ if (write(cbd->fd, &rep, sizeof(rep)) != sizeof(rep)) {
+ msg_err_pool("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ return TRUE;
+}
+
+static void
+lua_worker_control_session_dtor(void *ud)
+{
+ struct rspamd_control_cbdata *cbd = (struct rspamd_control_cbdata *) ud;
+
+ rspamd_mempool_delete(cbd->pool);
+}
+
+static gboolean
+lua_worker_control_handler(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker,
+ gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud)
+{
+ struct rspamd_async_session *session, **psession;
+ struct rspamd_control_cbdata *cbd = (struct rspamd_control_cbdata *) ud;
+ rspamd_mempool_t *pool;
+ lua_State *L;
+ gint err_idx, status;
+
+ L = cbd->L;
+ pool = cbd->pool;
+ session = rspamd_session_create(cbd->pool,
+ lua_worker_control_fin_session,
+ NULL,
+ lua_worker_control_session_dtor,
+ cbd);
+ cbd->session = session;
+ cbd->fd = fd;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+ psession = lua_newuserdata(L, sizeof(*psession));
+ rspamd_lua_setclass(L, "rspamd{session}", -1);
+ *psession = session;
+
+ /* Command name */
+ lua_pushstring(L, rspamd_control_command_to_string(cmd->type));
+
+ /* Command's extras */
+ lua_newtable(L);
+
+ switch (cmd->type) {
+ case RSPAMD_CONTROL_CHILD_CHANGE:
+ lua_pushinteger(L, cmd->cmd.child_change.pid);
+ lua_setfield(L, -2, "pid");
+ switch (cmd->cmd.child_change.what) {
+ case rspamd_child_offline:
+ lua_pushstring(L, "offline");
+ lua_setfield(L, -2, "what");
+ break;
+ case rspamd_child_online:
+ lua_pushstring(L, "online");
+ lua_setfield(L, -2, "what");
+ break;
+ case rspamd_child_terminated:
+ lua_pushstring(L, "terminated");
+ lua_setfield(L, -2, "what");
+ status = cmd->cmd.child_change.additional;
+
+ if (WIFEXITED(status)) {
+ lua_pushinteger(L, WEXITSTATUS(status));
+ lua_setfield(L, -2, "exit_code");
+ }
+
+ if (WIFSIGNALED(status)) {
+ lua_pushinteger(L, WTERMSIG(status));
+ lua_setfield(L, -2, "signal");
+#ifdef WCOREDUMP
+ lua_pushboolean(L, WCOREDUMP(status));
+ lua_setfield(L, -2, "core");
+#endif
+ }
+ break;
+ }
+ break;
+ case RSPAMD_CONTROL_MONITORED_CHANGE:
+ lua_pushinteger(L, cmd->cmd.monitored_change.sender);
+ lua_setfield(L, -2, "sender");
+ lua_pushboolean(L, cmd->cmd.monitored_change.alive);
+ lua_setfield(L, -2, "alive");
+ lua_pushlstring(L, cmd->cmd.monitored_change.tag,
+ sizeof(cmd->cmd.monitored_change.tag));
+ lua_setfield(L, -2, "tag");
+ break;
+ case RSPAMD_CONTROL_HYPERSCAN_LOADED:
+ lua_pushstring(L, cmd->cmd.hs_loaded.cache_dir);
+ lua_setfield(L, -2, "cache_dir");
+ lua_pushboolean(L, cmd->cmd.hs_loaded.forced);
+ lua_setfield(L, -2, "forced");
+ break;
+ case RSPAMD_CONTROL_STAT:
+ case RSPAMD_CONTROL_RELOAD:
+ case RSPAMD_CONTROL_RERESOLVE:
+ case RSPAMD_CONTROL_RECOMPILE:
+ case RSPAMD_CONTROL_LOG_PIPE:
+ case RSPAMD_CONTROL_FUZZY_STAT:
+ case RSPAMD_CONTROL_FUZZY_SYNC:
+ default:
+ break;
+ }
+
+ if (lua_pcall(L, 3, 0, err_idx) != 0) {
+ msg_err_pool("cannot init lua parser script: %s", lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+
+ struct rspamd_control_reply rep;
+
+ memset(&rep, 0, sizeof(rep));
+ rep.type = cbd->cmd;
+ rep.reply.monitored_change.status = -1;
+
+ if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
+ msg_err_pool("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ rspamd_session_destroy(session);
+ }
+ else {
+ lua_settop(L, err_idx - 1);
+ rspamd_session_pending(session);
+ }
+
+ return TRUE;
+}
+
+static gint
+lua_worker_add_control_handler(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+ struct rspamd_config *cfg = lua_check_config(L, 2);
+ struct ev_loop *event_loop = lua_check_ev_base(L, 3);
+ const gchar *cmd_name = luaL_checkstring(L, 4);
+ enum rspamd_control_type cmd;
+ struct rspamd_control_cbdata *cbd;
+
+ if (w && cfg && event_loop && cmd_name && lua_isfunction(L, 5)) {
+ cmd = rspamd_control_command_from_string(cmd_name);
+
+ if (cmd == RSPAMD_CONTROL_MAX) {
+ return luaL_error(L, "invalid command type: %s", cmd_name);
+ }
+
+ rspamd_mempool_t *pool = rspamd_mempool_new(
+ rspamd_mempool_suggest_size(), "lua_control", 0);
+ cbd = rspamd_mempool_alloc0(pool, sizeof(*cbd));
+ cbd->pool = pool;
+ cbd->event_loop = event_loop;
+ cbd->w = w;
+ cbd->cfg = cfg;
+ cbd->cmd = cmd;
+ cbd->L = L;
+ /* Refcount callback */
+ lua_pushvalue(L, 5);
+ cbd->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ rspamd_control_worker_add_cmd_handler(w, cmd, lua_worker_control_handler,
+ cbd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, need worker, cfg, "
+ "ev_loop, cmd_name and callback function");
+ }
+
+ return 0;
+}
+
+#ifdef WITH_JEMALLOC
+static void
+lua_worker_jemalloc_stats_cb(void *ud, const char *msg)
+{
+ lua_State *L = (lua_State *) ud;
+
+ lua_pushstring(L, msg);
+}
+#endif
+
+static gint
+lua_worker_get_mem_stats(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+#ifdef WITH_JEMALLOC
+ malloc_stats_print(lua_worker_jemalloc_stats_cb, (void *) L, NULL);
+#else
+ lua_pushstring(L, "no stats, jemalloc support is required");
+#endif
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct rspamd_lua_process_cbdata {
+ gint sp[2];
+ gint func_cbref;
+ gint cb_cbref;
+ gboolean replied;
+ gboolean is_error;
+ pid_t cpid;
+ lua_State *L;
+ guint64 sz;
+ GString *io_buf;
+ GString *out_buf;
+ goffset out_pos;
+ struct rspamd_worker *wrk;
+ struct ev_loop *event_loop;
+ ev_io ev;
+};
+
+static void
+rspamd_lua_execute_lua_subprocess(lua_State *L,
+ struct rspamd_lua_process_cbdata *cbdata)
+{
+ gint err_idx, r;
+ guint64 wlen = 0;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbdata->func_cbref);
+
+ if (lua_pcall(L, 0, 1, err_idx) != 0) {
+ const gchar *s = lua_tostring(L, -1);
+ gsize slen = strlen(s);
+
+ msg_err("call to subprocess failed: %s", s);
+ /* Indicate error */
+ wlen = (1ULL << 63u) + slen;
+
+ r = write(cbdata->sp[1], &wlen, sizeof(wlen));
+ if (r == -1) {
+ msg_err("write failed: %s", strerror(errno));
+ }
+
+ r = write(cbdata->sp[1], s, slen);
+ if (r == -1) {
+ msg_err("write failed: %s", strerror(errno));
+ }
+ }
+ else {
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, -1);
+
+ if (t) {
+ wlen = t->len;
+ r = write(cbdata->sp[1], &wlen, sizeof(wlen));
+
+ if (r == -1) {
+ msg_err("write failed: %s", strerror(errno));
+ }
+
+ r = write(cbdata->sp[1], t->start, wlen);
+
+ if (r == -1) {
+ msg_err("write failed: %s", strerror(errno));
+ }
+ }
+ else {
+ msg_err("subprocess: invalid return value: %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ }
+
+ lua_settop(L, err_idx - 1);
+}
+
+static void
+rspamd_lua_call_on_complete(lua_State *L,
+ struct rspamd_lua_process_cbdata *cbdata,
+ const gchar *err_msg,
+ const gchar *data, gsize datalen)
+{
+ gint err_idx;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbdata->cb_cbref);
+
+ if (err_msg) {
+ lua_pushstring(L, err_msg);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (data) {
+ lua_pushlstring(L, data, datalen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (lua_pcall(L, 2, 0, err_idx) != 0) {
+ msg_err("call to on_complete script failed: %s",
+ lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+}
+
+static gboolean
+rspamd_lua_cld_handler(struct rspamd_worker_signal_handler *sigh, void *ud)
+{
+ struct rspamd_lua_process_cbdata *cbdata = ud;
+ struct rspamd_srv_command srv_cmd;
+ lua_State *L;
+ pid_t died;
+ gint res = 0;
+
+ /* Are we called by a correct children ? */
+ died = waitpid(cbdata->cpid, &res, WNOHANG);
+
+ if (died <= 0) {
+ /* Wait more */
+ return TRUE;
+ }
+
+ L = cbdata->L;
+ msg_info("handled SIGCHLD from %P", cbdata->cpid);
+
+ if (!cbdata->replied) {
+ /* We still need to call on_complete callback */
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ "Worker has died without reply", NULL, 0);
+ }
+
+ /* Free structures */
+ close(cbdata->sp[0]);
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->func_cbref);
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->cb_cbref);
+ g_string_free(cbdata->io_buf, TRUE);
+
+ if (cbdata->out_buf) {
+ g_string_free(cbdata->out_buf, TRUE);
+ }
+
+ /* Notify main */
+ memset(&srv_cmd, 0, sizeof(srv_cmd));
+ srv_cmd.type = RSPAMD_SRV_ON_FORK;
+ srv_cmd.cmd.on_fork.state = child_dead;
+ srv_cmd.cmd.on_fork.cpid = cbdata->cpid;
+ srv_cmd.cmd.on_fork.ppid = getpid();
+ rspamd_srv_send_command(cbdata->wrk, cbdata->event_loop, &srv_cmd, -1,
+ NULL, NULL);
+ g_free(cbdata);
+
+ /* We are done with this SIGCHLD */
+ return FALSE;
+}
+
+static void
+rspamd_lua_subprocess_io(EV_P_ ev_io *w, int revents)
+{
+ struct rspamd_lua_process_cbdata *cbdata =
+ (struct rspamd_lua_process_cbdata *) w->data;
+ gssize r;
+
+ if (cbdata->sz == (guint64) -1) {
+ guint64 sz;
+
+ /* We read size of reply + flags first */
+ r = read(cbdata->sp[0], cbdata->io_buf->str + cbdata->io_buf->len,
+ sizeof(guint64) - cbdata->io_buf->len);
+
+ if (r == 0) {
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ "Unexpected EOF", NULL, 0);
+ cbdata->replied = TRUE;
+ kill(cbdata->cpid, SIGTERM);
+
+ return;
+ }
+ else if (r == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ return;
+ }
+ else {
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ strerror(errno), NULL, 0);
+ cbdata->replied = TRUE;
+ kill(cbdata->cpid, SIGTERM);
+
+ return;
+ }
+ }
+
+ cbdata->io_buf->len += r;
+
+ if (cbdata->io_buf->len == sizeof(guint64)) {
+ memcpy((guchar *) &sz, cbdata->io_buf->str, sizeof(sz));
+
+ if (sz & (1ULL << 63)) {
+ cbdata->is_error = TRUE;
+ sz &= ~(1ULL << 63);
+ }
+
+ cbdata->io_buf->len = 0;
+ cbdata->sz = sz;
+ g_string_set_size(cbdata->io_buf, sz + 1);
+ cbdata->io_buf->len = 0;
+ }
+ }
+ else {
+ /* Read data */
+ r = read(cbdata->sp[0], cbdata->io_buf->str + cbdata->io_buf->len,
+ cbdata->sz - cbdata->io_buf->len);
+
+ if (r == 0) {
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ "Unexpected EOF", NULL, 0);
+ cbdata->replied = TRUE;
+ kill(cbdata->cpid, SIGTERM);
+
+ return;
+ }
+ else if (r == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ return;
+ }
+ else {
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ strerror(errno), NULL, 0);
+ cbdata->replied = TRUE;
+ kill(cbdata->cpid, SIGTERM);
+
+ return;
+ }
+ }
+
+ cbdata->io_buf->len += r;
+
+ if (cbdata->io_buf->len == cbdata->sz) {
+ gchar rep[4];
+
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ /* Finished reading data */
+ if (cbdata->is_error) {
+ cbdata->io_buf->str[cbdata->io_buf->len] = '\0';
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ cbdata->io_buf->str, NULL, 0);
+ }
+ else {
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ NULL, cbdata->io_buf->str, cbdata->io_buf->len);
+ }
+
+ cbdata->replied = TRUE;
+
+ /* Write reply to the child */
+ rspamd_socket_blocking(cbdata->sp[0]);
+ memset(rep, 0, sizeof(rep));
+ (void) !write(cbdata->sp[0], rep, sizeof(rep));
+ }
+ }
+}
+
+static gint
+lua_worker_spawn_process(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+ struct rspamd_lua_process_cbdata *cbdata;
+ struct rspamd_abstract_worker_ctx *actx;
+ struct rspamd_srv_command srv_cmd;
+ const gchar *cmdline = NULL, *input = NULL, *proctitle = NULL;
+ gsize inputlen = 0;
+ pid_t pid;
+ GError *err = NULL;
+ gint func_cbref, cb_cbref;
+
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "func=F;exec=S;stdin=V;*on_complete=F;proctitle=S", &func_cbref,
+ &cmdline, &inputlen, &input, &cb_cbref, &proctitle)) {
+ msg_err("cannot get parameters list: %e", err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return 0;
+ }
+
+ cbdata = g_malloc0(sizeof(*cbdata));
+ cbdata->cb_cbref = cb_cbref;
+ cbdata->func_cbref = func_cbref;
+
+ if (input) {
+ cbdata->out_buf = g_string_new_len(input, inputlen);
+ cbdata->out_pos = 0;
+ }
+
+ if (rspamd_socketpair(cbdata->sp, SOCK_STREAM) == -1) {
+ msg_err("cannot spawn socketpair: %s", strerror(errno));
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->func_cbref);
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->cb_cbref);
+ g_free(cbdata);
+
+ return 0;
+ }
+
+ actx = w->ctx;
+ cbdata->wrk = w;
+ cbdata->L = L;
+ cbdata->event_loop = actx->event_loop;
+ cbdata->sz = (guint64) -1;
+
+ pid = fork();
+
+ if (pid == -1) {
+ msg_err("cannot spawn process: %s", strerror(errno));
+ close(cbdata->sp[0]);
+ close(cbdata->sp[1]);
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->func_cbref);
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->cb_cbref);
+ g_free(cbdata);
+
+ return 0;
+ }
+ else if (pid == 0) {
+ /* Child */
+ gint rc;
+ gchar inbuf[4];
+
+ rspamd_log_on_fork(w->cf->type, w->srv->cfg, w->srv->logger);
+ rc = ottery_init(w->srv->cfg->libs_ctx->ottery_cfg);
+
+ if (rc != OTTERY_ERR_NONE) {
+ msg_err("cannot initialize PRNG: %d", rc);
+ abort();
+ }
+ rspamd_random_seed_fast();
+#ifdef HAVE_EVUTIL_RNG_INIT
+ evutil_secure_rng_init();
+#endif
+
+ close(cbdata->sp[0]);
+ /* Here we assume that we can block on writing results */
+ rspamd_socket_blocking(cbdata->sp[1]);
+ g_hash_table_remove_all(w->signal_events);
+ ev_loop_destroy(cbdata->event_loop);
+
+ if (proctitle) {
+ rspamd_setproctitle("lua process: %s", proctitle);
+ }
+ else {
+ rspamd_setproctitle("lua process: unnamed");
+ }
+
+ cbdata->event_loop = ev_loop_new(EVFLAG_SIGNALFD);
+ rspamd_worker_unblock_signals();
+ rspamd_lua_execute_lua_subprocess(L, cbdata);
+
+ /* Wait for parent to reply and exit */
+ rc = read(cbdata->sp[1], inbuf, sizeof(inbuf));
+
+ if (rc >= sizeof(inbuf) &&
+ memcmp(inbuf, "\0\0\0\0", sizeof(inbuf)) == 0) {
+ exit(EXIT_SUCCESS);
+ }
+ else {
+ msg_err("got invalid reply from parent");
+
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ cbdata->cpid = pid;
+ cbdata->io_buf = g_string_sized_new(8);
+ /* Notify main */
+ memset(&srv_cmd, 0, sizeof(srv_cmd));
+ srv_cmd.type = RSPAMD_SRV_ON_FORK;
+ srv_cmd.cmd.on_fork.state = child_create;
+ srv_cmd.cmd.on_fork.cpid = pid;
+ srv_cmd.cmd.on_fork.ppid = getpid();
+ rspamd_srv_send_command(w, cbdata->event_loop, &srv_cmd, -1, NULL, NULL);
+
+ close(cbdata->sp[1]);
+ rspamd_socket_nonblocking(cbdata->sp[0]);
+ /* Parent */
+ rspamd_worker_set_signal_handler(SIGCHLD, w, cbdata->event_loop,
+ rspamd_lua_cld_handler,
+ cbdata);
+
+ /* Add result pipe waiting */
+ ev_io_init(&cbdata->ev, rspamd_lua_subprocess_io, cbdata->sp[0], EV_READ);
+ cbdata->ev.data = cbdata;
+ ev_io_start(cbdata->event_loop, &cbdata->ev);
+
+ return 0;
+}
+
+void luaopen_worker(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{worker}", worker_reg);
+}
diff --git a/src/lua/lua_xmlrpc.c b/src/lua/lua_xmlrpc.c
new file mode 100644
index 0000000..efb2b22
--- /dev/null
+++ b/src/lua/lua_xmlrpc.c
@@ -0,0 +1,796 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+
+
+LUA_FUNCTION_DEF(xmlrpc, parse_reply);
+LUA_FUNCTION_DEF(xmlrpc, make_request);
+
+static const struct luaL_reg xmlrpclib_m[] = {
+ LUA_INTERFACE_DEF(xmlrpc, parse_reply),
+ LUA_INTERFACE_DEF(xmlrpc, make_request),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+#define msg_debug_xmlrpc(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_xmlrpc_log_id, "xmlrpc", "", \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(xmlrpc)
+
+enum lua_xmlrpc_state {
+ read_method_response = 0,
+ read_params = 1,
+ read_param = 2,
+ read_param_value = 3,
+ read_param_element = 4,
+ read_struct = 5,
+ read_struct_member_name = 6,
+ read_struct_member_value = 7,
+ read_struct_element = 8,
+ read_string = 9,
+ read_int = 10,
+ read_double = 11,
+ read_array = 12,
+ read_array_value = 13,
+ read_array_element = 14,
+ error_state = 99,
+ success_state = 100,
+};
+
+enum lua_xmlrpc_stack {
+ st_array = 1,
+ st_struct = 2,
+};
+
+struct lua_xmlrpc_ud {
+ enum lua_xmlrpc_state parser_state;
+ GQueue *st;
+ gint param_count;
+ gboolean got_text;
+ lua_State *L;
+};
+
+static void xmlrpc_start_element(GMarkupParseContext *context,
+ const gchar *name,
+ const gchar **attribute_names,
+ const gchar **attribute_values,
+ gpointer user_data,
+ GError **error);
+static void xmlrpc_end_element(GMarkupParseContext *context,
+ const gchar *element_name,
+ gpointer user_data,
+ GError **error);
+static void xmlrpc_error(GMarkupParseContext *context,
+ GError *error,
+ gpointer user_data);
+static void xmlrpc_text(GMarkupParseContext *context,
+ const gchar *text,
+ gsize text_len,
+ gpointer user_data,
+ GError **error);
+
+static GMarkupParser xmlrpc_parser = {
+ .start_element = xmlrpc_start_element,
+ .end_element = xmlrpc_end_element,
+ .passthrough = NULL,
+ .text = xmlrpc_text,
+ .error = xmlrpc_error,
+};
+
+static GQuark
+xmlrpc_error_quark(void)
+{
+ return g_quark_from_static_string("xmlrpc-error-quark");
+}
+
+static void
+xmlrpc_start_element(GMarkupParseContext *context,
+ const gchar *name,
+ const gchar **attribute_names,
+ const gchar **attribute_values,
+ gpointer user_data,
+ GError **error)
+{
+ struct lua_xmlrpc_ud *ud = user_data;
+ enum lua_xmlrpc_state last_state;
+
+ last_state = ud->parser_state;
+
+ msg_debug_xmlrpc("got start element %s on state %d", name, last_state);
+
+ switch (ud->parser_state) {
+ case read_method_response:
+ /* Expect tag methodResponse */
+ if (g_ascii_strcasecmp(name, "methodResponse") == 0) {
+ ud->parser_state = read_params;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_params:
+ /* Expect tag params */
+ if (g_ascii_strcasecmp(name, "params") == 0) {
+ ud->parser_state = read_param;
+ /* result -> table of params indexed by int */
+ lua_newtable(ud->L);
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param:
+ /* Expect tag param */
+ if (g_ascii_strcasecmp(name, "param") == 0) {
+ ud->parser_state = read_param_value;
+ /* Create new param */
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param_value:
+ /* Expect tag value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ ud->parser_state = read_param_element;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param_element:
+ /* Expect tag struct */
+ if (g_ascii_strcasecmp(name, "struct") == 0) {
+ ud->parser_state = read_struct;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_struct));
+ msg_debug_xmlrpc("push struct");
+ }
+ else if (g_ascii_strcasecmp(name, "array") == 0) {
+ ud->parser_state = read_array;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_array));
+ msg_debug_xmlrpc("push array");
+ }
+ else if (g_ascii_strcasecmp(name, "string") == 0) {
+ ud->parser_state = read_string;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "int") == 0) {
+ ud->parser_state = read_int;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "double") == 0) {
+ ud->parser_state = read_double;
+ ud->got_text = FALSE;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct:
+ /* Parse structure */
+ /* Expect tag member */
+ if (g_ascii_strcasecmp(name, "member") == 0) {
+ ud->parser_state = read_struct_member_name;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_member_name:
+ /* Expect tag name */
+ if (g_ascii_strcasecmp(name, "name") == 0) {
+ ud->parser_state = read_struct_member_value;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_member_value:
+ /* Accept value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ ud->parser_state = read_struct_element;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_element:
+ /* Parse any values */
+ /* Primitives */
+ if (g_ascii_strcasecmp(name, "string") == 0) {
+ ud->parser_state = read_string;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "int") == 0) {
+ ud->parser_state = read_int;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "double") == 0) {
+ ud->parser_state = read_double;
+ ud->got_text = FALSE;
+ }
+ /* Structure */
+ else if (g_ascii_strcasecmp(name, "struct") == 0) {
+ ud->parser_state = read_struct;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_struct));
+ msg_debug_xmlrpc("push struct");
+ }
+ else if (g_ascii_strcasecmp(name, "array") == 0) {
+ ud->parser_state = read_array;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_array));
+ msg_debug_xmlrpc("push array");
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array:
+ /* Parse array */
+ /* Expect data */
+ if (g_ascii_strcasecmp(name, "data") == 0) {
+ ud->parser_state = read_array_value;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array_value:
+ /* Accept array value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ ud->parser_state = read_array_element;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array_element:
+ /* Parse any values */
+ /* Primitives */
+ if (g_ascii_strcasecmp(name, "string") == 0) {
+ ud->parser_state = read_string;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "int") == 0) {
+ ud->parser_state = read_int;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "double") == 0) {
+ ud->parser_state = read_double;
+ ud->got_text = FALSE;
+ }
+ /* Structure */
+ else if (g_ascii_strcasecmp(name, "struct") == 0) {
+ ud->parser_state = read_struct;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_struct));
+ msg_debug_xmlrpc("push struct");
+ }
+ else if (g_ascii_strcasecmp(name, "array") == 0) {
+ ud->parser_state = read_array;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_array));
+ msg_debug_xmlrpc("push array");
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ default:
+ break;
+ }
+
+ msg_debug_xmlrpc("switched state on start tag %d->%d", last_state,
+ ud->parser_state);
+
+ if (ud->parser_state == error_state) {
+ g_set_error(error,
+ xmlrpc_error_quark(), 1, "xmlrpc parse error on state: %d, while parsing start tag: %s",
+ last_state, name);
+ }
+}
+
+static void
+xmlrpc_end_element(GMarkupParseContext *context,
+ const gchar *name,
+ gpointer user_data,
+ GError **error)
+{
+ struct lua_xmlrpc_ud *ud = user_data;
+ enum lua_xmlrpc_state last_state;
+ int last_queued;
+
+ last_state = ud->parser_state;
+
+ msg_debug_xmlrpc("got end element %s on state %d", name, last_state);
+
+ switch (ud->parser_state) {
+ case read_method_response:
+ ud->parser_state = error_state;
+ break;
+ case read_params:
+ /* Got methodResponse */
+ if (g_ascii_strcasecmp(name, "methodResponse") == 0) {
+ /* End processing */
+ ud->parser_state = success_state;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param:
+ /* Got tag params */
+ if (g_ascii_strcasecmp(name, "params") == 0) {
+ ud->parser_state = read_params;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param_value:
+ /* Got tag param */
+ if (g_ascii_strcasecmp(name, "param") == 0) {
+ ud->parser_state = read_param;
+ lua_rawseti(ud->L, -2, ++ud->param_count);
+ msg_debug_xmlrpc("set param element idx: %d", ud->param_count);
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param_element:
+ /* Got tag value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ if (g_queue_get_length(ud->st) == 0) {
+ ud->parser_state = read_param_value;
+ }
+ else {
+ if (GPOINTER_TO_INT(g_queue_peek_head(ud->st)) == st_struct) {
+ ud->parser_state = read_struct_member_name;
+ }
+ else {
+ ud->parser_state = read_array_value;
+ }
+ }
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct:
+ /* Got tag struct */
+ if (g_ascii_strcasecmp(name, "struct") == 0) {
+ g_assert(GPOINTER_TO_INT(g_queue_pop_head(ud->st)) == st_struct);
+
+ if (g_queue_get_length(ud->st) == 0) {
+ ud->parser_state = read_param_element;
+ }
+ else {
+ last_queued = GPOINTER_TO_INT(g_queue_peek_head(ud->st));
+ if (last_queued == st_struct) {
+ ud->parser_state = read_struct_element;
+ }
+ else {
+ ud->parser_state = read_array_element;
+ }
+ }
+
+ msg_debug_xmlrpc("pop struct");
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_member_name:
+ /* Got tag member */
+ if (g_ascii_strcasecmp(name, "member") == 0) {
+ ud->parser_state = read_struct;
+ /* Set table */
+ msg_debug_xmlrpc("set struct element idx: %s",
+ lua_tostring(ud->L, -2));
+ lua_settable(ud->L, -3);
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_member_value:
+ /* Got tag name */
+ if (g_ascii_strcasecmp(name, "name") == 0) {
+ ud->parser_state = read_struct_member_value;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_element:
+ /* Got tag value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ ud->parser_state = read_struct_member_name;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_string:
+ case read_int:
+ case read_double:
+ /* Parse any values */
+ /* Handle empty tags */
+ if (!ud->got_text) {
+ lua_pushnil(ud->L);
+ }
+ else {
+ ud->got_text = FALSE;
+ }
+ /* Primitives */
+ if (g_ascii_strcasecmp(name, "string") == 0 ||
+ g_ascii_strcasecmp(name, "int") == 0 ||
+ g_ascii_strcasecmp(name, "double") == 0) {
+ if (GPOINTER_TO_INT(g_queue_peek_head(ud->st)) == st_struct) {
+ ud->parser_state = read_struct_element;
+ }
+ else {
+ ud->parser_state = read_array_element;
+ }
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array:
+ /* Got tag array */
+ if (g_ascii_strcasecmp(name, "array") == 0) {
+ g_assert(GPOINTER_TO_INT(g_queue_pop_head(ud->st)) == st_array);
+
+ if (g_queue_get_length(ud->st) == 0) {
+ ud->parser_state = read_param_element;
+ }
+ else {
+ last_queued = GPOINTER_TO_INT(g_queue_peek_head(ud->st));
+ if (last_queued == st_struct) {
+ ud->parser_state = read_struct_element;
+ }
+ else {
+ ud->parser_state = read_array_element;
+ }
+ }
+
+ msg_debug_xmlrpc("pop array");
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array_value:
+ /* Got tag data */
+ if (g_ascii_strcasecmp(name, "data") == 0) {
+ ud->parser_state = read_array;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array_element:
+ /* Got tag value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ guint tbl_len = rspamd_lua_table_size(ud->L, -2);
+ lua_rawseti(ud->L, -2, tbl_len + 1);
+ msg_debug_xmlrpc("set array element idx: %d", tbl_len + 1);
+ ud->parser_state = read_array_value;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ default:
+ break;
+ }
+
+ msg_debug_xmlrpc("switched state on end tag %d->%d",
+ last_state, ud->parser_state);
+
+ if (ud->parser_state == error_state) {
+ g_set_error(error,
+ xmlrpc_error_quark(), 1, "xmlrpc parse error on state: %d, while parsing end tag: %s",
+ last_state, name);
+ }
+}
+
+static void
+xmlrpc_text(GMarkupParseContext *context,
+ const gchar *text,
+ gsize text_len,
+ gpointer user_data,
+ GError **error)
+{
+ struct lua_xmlrpc_ud *ud = user_data;
+ gulong num;
+ gdouble dnum;
+
+ /* Strip line */
+ while (text_len > 0 && g_ascii_isspace(*text)) {
+ text++;
+ text_len--;
+ }
+ while (text_len > 0 && g_ascii_isspace(text[text_len - 1])) {
+ text_len--;
+ }
+
+ if (text_len > 0) {
+ msg_debug_xmlrpc("got data on state %d", ud->parser_state);
+ switch (ud->parser_state) {
+ case read_struct_member_value:
+ /* Push key */
+ lua_pushlstring(ud->L, text, text_len);
+ break;
+ case read_string:
+ /* Push string value */
+ lua_pushlstring(ud->L, text, text_len);
+ break;
+ case read_int:
+ /* Push integer value */
+ rspamd_strtoul(text, text_len, &num);
+ lua_pushinteger(ud->L, num);
+ break;
+ case read_double:
+ /* Push integer value */
+ dnum = strtod(text, NULL);
+ lua_pushnumber(ud->L, dnum);
+ break;
+ default:
+ break;
+ }
+ ud->got_text = TRUE;
+ }
+}
+
+static void
+xmlrpc_error(GMarkupParseContext *context, GError *error, gpointer user_data)
+{
+ msg_err("xmlrpc parser error: %s", error->message);
+}
+
+static gint
+lua_xmlrpc_parse_reply(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *data;
+ GMarkupParseContext *ctx;
+ GError *err = NULL;
+ struct lua_xmlrpc_ud ud;
+ gsize s;
+ gboolean res;
+
+ data = luaL_checklstring(L, 1, &s);
+
+ if (data != NULL) {
+ ud.L = L;
+ ud.parser_state = read_method_response;
+ ud.param_count = 0;
+ ud.st = g_queue_new();
+
+ ctx = g_markup_parse_context_new(&xmlrpc_parser,
+ G_MARKUP_TREAT_CDATA_AS_TEXT, &ud, NULL);
+ res = g_markup_parse_context_parse(ctx, data, s, &err);
+
+ g_markup_parse_context_free(ctx);
+ if (!res) {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ /* Return table or nil */
+ return 1;
+}
+
+static gint
+lua_xmlrpc_parse_table(lua_State *L,
+ gint pos,
+ gchar *databuf,
+ gint pr,
+ gsize size)
+{
+ gint r = pr, num;
+ double dnum;
+
+ r += rspamd_snprintf(databuf + r, size - r, "<struct>");
+ lua_pushnil(L); /* first key */
+ while (lua_next(L, pos) != 0) {
+ /* uses 'key' (at index -2) and 'value' (at index -1) */
+ if (lua_type(L, -2) != LUA_TSTRING) {
+ /* Ignore non sting keys */
+ lua_pop(L, 1);
+ continue;
+ }
+ r += rspamd_snprintf(databuf + r,
+ size - r,
+ "<member><name>%s</name><value>",
+ lua_tostring(L, -2));
+ switch (lua_type(L, -1)) {
+ case LUA_TNUMBER:
+ num = lua_tointeger(L, -1);
+ dnum = lua_tonumber(L, -1);
+
+ /* Try to avoid conversion errors */
+ if (dnum != (double) num) {
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<double>%f</double>",
+ dnum);
+ }
+ else {
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<int>%d</int>",
+ num);
+ }
+ break;
+ case LUA_TBOOLEAN:
+ r += rspamd_snprintf(databuf + r,
+ size - r,
+ "<boolean>%d</boolean>",
+ lua_toboolean(L, -1) ? 1 : 0);
+ break;
+ case LUA_TSTRING:
+ r += rspamd_snprintf(databuf + r, size - r, "<string>%s</string>",
+ lua_tostring(L, -1));
+ break;
+ case LUA_TTABLE:
+ /* Recursive call */
+ r += lua_xmlrpc_parse_table(L, -1, databuf + r, r, size);
+ break;
+ }
+ r += rspamd_snprintf(databuf + r, size - r, "</value></member>");
+ /* removes 'value'; keeps 'key' for next iteration */
+ lua_pop(L, 1);
+ }
+ r += rspamd_snprintf(databuf + r, size - r, "</struct>");
+
+ return r - pr;
+}
+
+/*
+ * Internal limitation: xmlrpc request must NOT be more than
+ * BUFSIZ * 2 (16384 bytes)
+ */
+static gint
+lua_xmlrpc_make_request(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gchar databuf[BUFSIZ * 2];
+ const gchar *func;
+ gint r, top, i, num;
+ double dnum;
+
+ func = luaL_checkstring(L, 1);
+
+ if (func) {
+ r = rspamd_snprintf(databuf, sizeof(databuf),
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+ "<methodCall><methodName>%s</methodName><params>",
+ func);
+ /* Extract arguments */
+ top = lua_gettop(L);
+ /* Get additional options */
+ for (i = 2; i <= top; i++) {
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<param><value>");
+ switch (lua_type(L, i)) {
+ case LUA_TNUMBER:
+ num = lua_tointeger(L, i);
+ dnum = lua_tonumber(L, i);
+
+ /* Try to avoid conversion errors */
+ if (dnum != (double) num) {
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<double>%f</double>",
+ dnum);
+ }
+ else {
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<int>%d</int>",
+ num);
+ }
+ break;
+ case LUA_TBOOLEAN:
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<boolean>%d</boolean>",
+ lua_toboolean(L, i) ? 1 : 0);
+ break;
+ case LUA_TSTRING:
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<string>%s</string>",
+ lua_tostring(L, i));
+ break;
+ case LUA_TTABLE:
+ r +=
+ lua_xmlrpc_parse_table(L, i, databuf, r, sizeof(databuf));
+ break;
+ }
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "</value></param>");
+ }
+
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "</params></methodCall>");
+ lua_pushlstring(L, databuf, r);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_xmlrpc(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, xmlrpclib_m);
+
+ return 1;
+}
+
+void luaopen_xmlrpc(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_xmlrpc", lua_load_xmlrpc);
+}
diff --git a/src/lua/rspamd.luadoc b/src/lua/rspamd.luadoc
new file mode 100644
index 0000000..7f2c5cc
--- /dev/null
+++ b/src/lua/rspamd.luadoc
@@ -0,0 +1,124 @@
+--- Rspamd interaction package
+-- contains several subclasses:
+-- config - for parsing config files
+-- metric - for handling metrics callbacks
+-- task - for interaction with task object
+-- message - gate to GMime functions
+-- textpart - a single textual part of message
+module Rspamd
+
+--- Each lua module has global rspamd_config that can be used for getting config
+-- options and registering callbacks (via metric interface)
+
+------------------------------------- CONFIG -----------------------------------------
+--
+--- Get module option from config
+-- @param mname module name
+-- @param option option
+-- @return string with value
+function config:get_module_opt (mname, option)
+
+--- Get all module options as a table like ['param' => 'value']
+-- @param mname module name
+-- @return table with options
+function config:get_all_opt (mname)
+
+--- Get specified metric
+-- @param name metric name
+-- @return metric object
+function config:get_metric (name)
+
+------------------------------------- METRIC -----------------------------------------
+
+--- Register symbol in metric
+-- @param symbol name of symbol
+-- @param weight weight of symbol
+-- @param callback function that would be called as callback for symbol
+function metric:register_symbol (symbol, weight, callback)
+
+------------------------------------- TASK -------------------------------------------
+
+--- Get message object from task
+-- @return message object
+function task:get_message ()
+
+--- Insert result to specified metric with specified weight (obsoleted)
+-- @param metric metric name
+-- @param symbol symbol name
+-- @param weight weight of symbol
+function task:insert_result (metric, symbol, weight)
+
+--- Get all urls as array
+-- @return array of urls in textual form
+function task:get_urls ()
+
+--- Get all text parts
+-- @return array of textpart objects
+function task:get_text_parts ()
+
+--- Get raw headers
+-- @return string that contains raw headers
+function task:get_raw_headers ()
+
+--- Get array of received headers
+-- @return array of received headers that are tables itself
+function task:get_received_headers ()
+
+--- Resolve A record using rspamd async resolver
+-- @param host host to resolve
+-- @param callback name of callback function
+function task:resolve_dns_a (host, callback)
+
+--- Resolve PTR record using rspamd async resolver
+-- @param host host to resolve
+-- @param callback name of callback function
+function task:resolve_dns_ptr (host, callback)
+
+--- Callback function for dns resolving
+-- @param task task object
+-- @param to_resolve ptr or a record that was resolved
+-- @param results results of dns query (array or nil)
+-- @param err resolver error or nil
+function dns_cb(task, to_resolve, results, err)
+
+------------------------------------- TEXTPART ---------------------------------------
+
+--- Get part's content
+-- @return string that contains part's content
+function textpart:get_content ()
+
+--- Check if part is empty
+-- @return boolean value
+function textpart:is_empty ()
+
+--- Check if part is html
+-- @return boolean value
+function textpart:is_html ()
+
+--- Get part's fuzzy
+-- @return string that contains part's fuzzy
+function textpart:get_fuzzy ()
+
+------------------------------------- MESSAGE ----------------------------------------
+
+--- Get message subject
+-- @return message subject
+function message:get_subject ()
+
+--- Get message id
+-- @return message id
+function message:get_message_id ()
+
+--- Get sender of message
+-- @return sender's credits
+function message:get_sender ()
+
+--- Get reply-to field
+-- @return value of reply-to header
+function message:get_reply_to ()
+
+--- Get header
+-- @param header_name name of header
+-- @return array of headers with specified name
+function message:get_header (header_name)
+
diff --git a/src/plugins/chartable.cxx b/src/plugins/chartable.cxx
new file mode 100644
index 0000000..704f12a
--- /dev/null
+++ b/src/plugins/chartable.cxx
@@ -0,0 +1,2122 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/***MODULE:chartable
+ * rspamd module that make marks based on symbol chains
+ *
+ * Allowed options:
+ * - symbol (string): symbol to insert (default: 'R_BAD_CHARSET')
+ * - threshold (double): value that would be used as threshold in expression characters_changed / total_characters
+ * (e.g. if threshold is 0.1 than charset change should occur more often than in 10 symbols), default: 0.1
+ */
+
+
+#include "config.h"
+#include "libmime/message.h"
+#include "rspamd.h"
+#include "libstat/stat_api.h"
+#include "libmime/lang_detection.h"
+
+#include "unicode/utf8.h"
+#include "unicode/uchar.h"
+#include "contrib/ankerl/unordered_dense.h"
+
+#define DEFAULT_SYMBOL "R_MIXED_CHARSET"
+#define DEFAULT_URL_SYMBOL "R_MIXED_CHARSET_URL"
+#define DEFAULT_THRESHOLD 0.1
+
+#define msg_debug_chartable(...) rspamd_conditional_debug_fast(nullptr, task->from_addr, \
+ rspamd_chartable_log_id, "chartable", task->task_pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(chartable)
+
+/* Initialization */
+gint chartable_module_init(struct rspamd_config *cfg, struct module_ctx **ctx);
+
+gint chartable_module_config(struct rspamd_config *cfg, bool validate);
+
+gint chartable_module_reconfig(struct rspamd_config *cfg);
+
+module_t chartable_module = {
+ "chartable",
+ chartable_module_init,
+ chartable_module_config,
+ chartable_module_reconfig,
+ nullptr,
+ RSPAMD_MODULE_VER,
+ (guint) -1,
+};
+
+struct chartable_ctx {
+ struct module_ctx ctx;
+ const gchar *symbol;
+ const gchar *url_symbol;
+ double threshold;
+ guint max_word_len;
+};
+
+static inline struct chartable_ctx *
+chartable_get_context(struct rspamd_config *cfg)
+{
+ return (struct chartable_ctx *) g_ptr_array_index(cfg->c_modules,
+ chartable_module.ctx_offset);
+}
+
+static void chartable_symbol_callback(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ void *unused);
+
+static void chartable_url_symbol_callback(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ void *unused);
+
+gint chartable_module_init(struct rspamd_config *cfg, struct module_ctx **ctx)
+{
+ struct chartable_ctx *chartable_module_ctx;
+
+ chartable_module_ctx = rspamd_mempool_alloc0_type(cfg->cfg_pool,
+ struct chartable_ctx);
+ chartable_module_ctx->max_word_len = 10;
+
+ *ctx = (struct module_ctx *) chartable_module_ctx;
+
+ return 0;
+}
+
+
+gint chartable_module_config(struct rspamd_config *cfg, bool _)
+{
+ const ucl_object_t *value;
+ gint res = TRUE;
+ struct chartable_ctx *chartable_module_ctx = chartable_get_context(cfg);
+
+ if (!rspamd_config_is_module_enabled(cfg, "chartable")) {
+ return TRUE;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "chartable", "symbol")) != nullptr) {
+ chartable_module_ctx->symbol = ucl_obj_tostring(value);
+ }
+ else {
+ chartable_module_ctx->symbol = DEFAULT_SYMBOL;
+ }
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "chartable", "url_symbol")) != nullptr) {
+ chartable_module_ctx->url_symbol = ucl_obj_tostring(value);
+ }
+ else {
+ chartable_module_ctx->url_symbol = DEFAULT_URL_SYMBOL;
+ }
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "chartable", "threshold")) != nullptr) {
+ if (!ucl_obj_todouble_safe(value, &chartable_module_ctx->threshold)) {
+ msg_warn_config("invalid numeric value");
+ chartable_module_ctx->threshold = DEFAULT_THRESHOLD;
+ }
+ }
+ else {
+ chartable_module_ctx->threshold = DEFAULT_THRESHOLD;
+ }
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "chartable", "max_word_len")) != nullptr) {
+ chartable_module_ctx->max_word_len = ucl_object_toint(value);
+ }
+ else {
+ chartable_module_ctx->threshold = DEFAULT_THRESHOLD;
+ }
+
+ rspamd_symcache_add_symbol(cfg->cache,
+ chartable_module_ctx->symbol,
+ 0,
+ chartable_symbol_callback,
+ nullptr,
+ SYMBOL_TYPE_NORMAL,
+ -1);
+ rspamd_symcache_add_symbol(cfg->cache,
+ chartable_module_ctx->url_symbol,
+ 0,
+ chartable_url_symbol_callback,
+ nullptr,
+ SYMBOL_TYPE_NORMAL,
+ -1);
+
+ msg_info_config("init internal chartable module");
+
+ return res;
+}
+
+gint chartable_module_reconfig(struct rspamd_config *cfg)
+{
+ return chartable_module_config(cfg, false);
+}
+
+static const auto latin_confusable = ankerl::unordered_dense::set<int>{
+ 0x02028,
+ 0x02029,
+ 0x01680,
+ 0x02000,
+ 0x02001,
+ 0x02002,
+ 0x02003,
+ 0x02004,
+ 0x02005,
+ 0x02006,
+ 0x02008,
+ 0x02009,
+ 0x0200a,
+ 0x0205f,
+ 0x000a0,
+ 0x02007,
+ 0x0202f,
+ 0x007fa,
+ 0x0fe4d,
+ 0x0fe4e,
+ 0x0fe4f,
+ 0x02010,
+ 0x02011,
+ 0x02012,
+ 0x02013,
+ 0x0fe58,
+ 0x006d4,
+ 0x02043,
+ 0x002d7,
+ 0x02212,
+ 0x02796,
+ 0x02cba,
+ 0x0060d,
+ 0x0066b,
+ 0x0201a,
+ 0x000b8,
+ 0x0a4f9,
+ 0x0037e,
+ 0x00903,
+ 0x00a83,
+ 0x0ff1a,
+ 0x00589,
+ 0x00703,
+ 0x00704,
+ 0x016ec,
+ 0x0fe30,
+ 0x01803,
+ 0x01809,
+ 0x0205a,
+ 0x005c3,
+ 0x002f8,
+ 0x0a789,
+ 0x02236,
+ 0x002d0,
+ 0x0a4fd,
+ 0x0ff01,
+ 0x001c3,
+ 0x02d51,
+ 0x00294,
+ 0x00241,
+ 0x0097d,
+ 0x013ae,
+ 0x0a6eb,
+ 0x1d16d,
+ 0x02024,
+ 0x00701,
+ 0x00702,
+ 0x0a60e,
+ 0x10a50,
+ 0x00660,
+ 0x006f0,
+ 0x0a4f8,
+ 0x0055d,
+ 0x0ff07,
+ 0x02018,
+ 0x02019,
+ 0x0201b,
+ 0x02032,
+ 0x02035,
+ 0x0055a,
+ 0x005f3,
+ 0x00060,
+ 0x01fef,
+ 0x0ff40,
+ 0x000b4,
+ 0x00384,
+ 0x01ffd,
+ 0x01fbd,
+ 0x01fbf,
+ 0x01ffe,
+ 0x002b9,
+ 0x00374,
+ 0x002c8,
+ 0x002ca,
+ 0x002cb,
+ 0x002f4,
+ 0x002bb,
+ 0x002bd,
+ 0x002bc,
+ 0x002be,
+ 0x0a78c,
+ 0x005d9,
+ 0x007f4,
+ 0x007f5,
+ 0x0144a,
+ 0x016cc,
+ 0x16f51,
+ 0x16f52,
+ 0x0ff3b,
+ 0x02768,
+ 0x02772,
+ 0x03014,
+ 0x0fd3e,
+ 0x0ff3d,
+ 0x02769,
+ 0x02773,
+ 0x03015,
+ 0x0fd3f,
+ 0x02774,
+ 0x1d114,
+ 0x02775,
+ 0x0204e,
+ 0x0066d,
+ 0x02217,
+ 0x1031f,
+ 0x01735,
+ 0x02041,
+ 0x02215,
+ 0x02044,
+ 0x02571,
+ 0x027cb,
+ 0x029f8,
+ 0x1d23a,
+ 0x031d3,
+ 0x03033,
+ 0x02cc6,
+ 0x030ce,
+ 0x04e3f,
+ 0x02f03,
+ 0x0ff3c,
+ 0x0fe68,
+ 0x02216,
+ 0x027cd,
+ 0x029f5,
+ 0x029f9,
+ 0x1d20f,
+ 0x1d23b,
+ 0x031d4,
+ 0x04e36,
+ 0x02f02,
+ 0x0a778,
+ 0x002c4,
+ 0x002c6,
+ 0x016ed,
+ 0x02795,
+ 0x1029b,
+ 0x02039,
+ 0x0276e,
+ 0x002c2,
+ 0x1d236,
+ 0x01438,
+ 0x016b2,
+ 0x01400,
+ 0x02e40,
+ 0x030a0,
+ 0x0a4ff,
+ 0x0203a,
+ 0x0276f,
+ 0x002c3,
+ 0x1d237,
+ 0x01433,
+ 0x16f3f,
+ 0x02053,
+ 0x002dc,
+ 0x01fc0,
+ 0x0223c,
+ 0x1d7d0,
+ 0x1d7da,
+ 0x1d7e4,
+ 0x1d7ee,
+ 0x1d7f8,
+ 0x0a75a,
+ 0x001a7,
+ 0x003e8,
+ 0x0a644,
+ 0x014bf,
+ 0x0a6ef,
+ 0x1d206,
+ 0x1d7d1,
+ 0x1d7db,
+ 0x1d7e5,
+ 0x1d7ef,
+ 0x1d7f9,
+ 0x0a7ab,
+ 0x0021c,
+ 0x001b7,
+ 0x0a76a,
+ 0x02ccc,
+ 0x00417,
+ 0x004e0,
+ 0x16f3b,
+ 0x118ca,
+ 0x1d7d2,
+ 0x1d7dc,
+ 0x1d7e6,
+ 0x1d7f0,
+ 0x1d7fa,
+ 0x013ce,
+ 0x118af,
+ 0x1d7d3,
+ 0x1d7dd,
+ 0x1d7e7,
+ 0x1d7f1,
+ 0x1d7fb,
+ 0x001bc,
+ 0x118bb,
+ 0x1d7d4,
+ 0x1d7de,
+ 0x1d7e8,
+ 0x1d7f2,
+ 0x1d7fc,
+ 0x02cd2,
+ 0x00431,
+ 0x013ee,
+ 0x118d5,
+ 0x1d212,
+ 0x1d7d5,
+ 0x1d7df,
+ 0x1d7e9,
+ 0x1d7f3,
+ 0x1d7fd,
+ 0x104d2,
+ 0x118c6,
+ 0x00b03,
+ 0x009ea,
+ 0x00a6a,
+ 0x1e8cb,
+ 0x1d7d6,
+ 0x1d7e0,
+ 0x1d7ea,
+ 0x1d7f4,
+ 0x1d7fe,
+ 0x00223,
+ 0x00222,
+ 0x1031a,
+ 0x00a67,
+ 0x00b68,
+ 0x009ed,
+ 0x00d6d,
+ 0x1d7d7,
+ 0x1d7e1,
+ 0x1d7eb,
+ 0x1d7f5,
+ 0x1d7ff,
+ 0x0a76e,
+ 0x02cca,
+ 0x118cc,
+ 0x118ac,
+ 0x118d6,
+ 0x0237a,
+ 0x0ff41,
+ 0x1d41a,
+ 0x1d44e,
+ 0x1d482,
+ 0x1d4b6,
+ 0x1d4ea,
+ 0x1d51e,
+ 0x1d552,
+ 0x1d586,
+ 0x1d5ba,
+ 0x1d5ee,
+ 0x1d622,
+ 0x1d656,
+ 0x1d68a,
+ 0x00251,
+ 0x003b1,
+ 0x1d6c2,
+ 0x1d6fc,
+ 0x1d736,
+ 0x1d770,
+ 0x1d7aa,
+ 0x00430,
+ 0x0ff21,
+ 0x1d400,
+ 0x1d434,
+ 0x1d468,
+ 0x1d49c,
+ 0x1d4d0,
+ 0x1d504,
+ 0x1d538,
+ 0x1d56c,
+ 0x1d5a0,
+ 0x1d5d4,
+ 0x1d608,
+ 0x1d63c,
+ 0x1d670,
+ 0x00391,
+ 0x1d6a8,
+ 0x1d6e2,
+ 0x1d71c,
+ 0x1d756,
+ 0x1d790,
+ 0x00410,
+ 0x013aa,
+ 0x015c5,
+ 0x0a4ee,
+ 0x16f40,
+ 0x102a0,
+ 0x1d41b,
+ 0x1d44f,
+ 0x1d483,
+ 0x1d4b7,
+ 0x1d4eb,
+ 0x1d51f,
+ 0x1d553,
+ 0x1d587,
+ 0x1d5bb,
+ 0x1d5ef,
+ 0x1d623,
+ 0x1d657,
+ 0x1d68b,
+ 0x00184,
+ 0x0042c,
+ 0x013cf,
+ 0x015af,
+ 0x0ff22,
+ 0x0212c,
+ 0x1d401,
+ 0x1d435,
+ 0x1d469,
+ 0x1d4d1,
+ 0x1d505,
+ 0x1d539,
+ 0x1d56d,
+ 0x1d5a1,
+ 0x1d5d5,
+ 0x1d609,
+ 0x1d63d,
+ 0x1d671,
+ 0x0a7b4,
+ 0x00392,
+ 0x1d6a9,
+ 0x1d6e3,
+ 0x1d71d,
+ 0x1d757,
+ 0x1d791,
+ 0x00412,
+ 0x013f4,
+ 0x015f7,
+ 0x0a4d0,
+ 0x10282,
+ 0x102a1,
+ 0x10301,
+ 0x0ff43,
+ 0x0217d,
+ 0x1d41c,
+ 0x1d450,
+ 0x1d484,
+ 0x1d4b8,
+ 0x1d4ec,
+ 0x1d520,
+ 0x1d554,
+ 0x1d588,
+ 0x1d5bc,
+ 0x1d5f0,
+ 0x1d624,
+ 0x1d658,
+ 0x1d68c,
+ 0x01d04,
+ 0x003f2,
+ 0x02ca5,
+ 0x00441,
+ 0x0abaf,
+ 0x1043d,
+ 0x1f74c,
+ 0x118f2,
+ 0x118e9,
+ 0x0ff23,
+ 0x0216d,
+ 0x02102,
+ 0x0212d,
+ 0x1d402,
+ 0x1d436,
+ 0x1d46a,
+ 0x1d49e,
+ 0x1d4d2,
+ 0x1d56e,
+ 0x1d5a2,
+ 0x1d5d6,
+ 0x1d60a,
+ 0x1d63e,
+ 0x1d672,
+ 0x003f9,
+ 0x02ca4,
+ 0x00421,
+ 0x013df,
+ 0x0a4da,
+ 0x102a2,
+ 0x10302,
+ 0x10415,
+ 0x1051c,
+ 0x0217e,
+ 0x02146,
+ 0x1d41d,
+ 0x1d451,
+ 0x1d485,
+ 0x1d4b9,
+ 0x1d4ed,
+ 0x1d521,
+ 0x1d555,
+ 0x1d589,
+ 0x1d5bd,
+ 0x1d5f1,
+ 0x1d625,
+ 0x1d659,
+ 0x1d68d,
+ 0x00501,
+ 0x013e7,
+ 0x0146f,
+ 0x0a4d2,
+ 0x0216e,
+ 0x02145,
+ 0x1d403,
+ 0x1d437,
+ 0x1d46b,
+ 0x1d49f,
+ 0x1d4d3,
+ 0x1d507,
+ 0x1d53b,
+ 0x1d56f,
+ 0x1d5a3,
+ 0x1d5d7,
+ 0x1d60b,
+ 0x1d63f,
+ 0x1d673,
+ 0x013a0,
+ 0x015de,
+ 0x015ea,
+ 0x0a4d3,
+ 0x0212e,
+ 0x0ff45,
+ 0x0212f,
+ 0x02147,
+ 0x1d41e,
+ 0x1d452,
+ 0x1d486,
+ 0x1d4ee,
+ 0x1d522,
+ 0x1d556,
+ 0x1d58a,
+ 0x1d5be,
+ 0x1d5f2,
+ 0x1d626,
+ 0x1d65a,
+ 0x1d68e,
+ 0x0ab32,
+ 0x00435,
+ 0x004bd,
+ 0x022ff,
+ 0x0ff25,
+ 0x02130,
+ 0x1d404,
+ 0x1d438,
+ 0x1d46c,
+ 0x1d4d4,
+ 0x1d508,
+ 0x1d53c,
+ 0x1d570,
+ 0x1d5a4,
+ 0x1d5d8,
+ 0x1d60c,
+ 0x1d640,
+ 0x1d674,
+ 0x00395,
+ 0x1d6ac,
+ 0x1d6e6,
+ 0x1d720,
+ 0x1d75a,
+ 0x1d794,
+ 0x00415,
+ 0x02d39,
+ 0x013ac,
+ 0x0a4f0,
+ 0x118a6,
+ 0x118ae,
+ 0x10286,
+ 0x1d41f,
+ 0x1d453,
+ 0x1d487,
+ 0x1d4bb,
+ 0x1d4ef,
+ 0x1d523,
+ 0x1d557,
+ 0x1d58b,
+ 0x1d5bf,
+ 0x1d5f3,
+ 0x1d627,
+ 0x1d65b,
+ 0x1d68f,
+ 0x0ab35,
+ 0x0a799,
+ 0x0017f,
+ 0x01e9d,
+ 0x00584,
+ 0x1d213,
+ 0x02131,
+ 0x1d405,
+ 0x1d439,
+ 0x1d46d,
+ 0x1d4d5,
+ 0x1d509,
+ 0x1d53d,
+ 0x1d571,
+ 0x1d5a5,
+ 0x1d5d9,
+ 0x1d60d,
+ 0x1d641,
+ 0x1d675,
+ 0x0a798,
+ 0x003dc,
+ 0x1d7ca,
+ 0x015b4,
+ 0x0a4dd,
+ 0x118c2,
+ 0x118a2,
+ 0x10287,
+ 0x102a5,
+ 0x10525,
+ 0x0ff47,
+ 0x0210a,
+ 0x1d420,
+ 0x1d454,
+ 0x1d488,
+ 0x1d4f0,
+ 0x1d524,
+ 0x1d558,
+ 0x1d58c,
+ 0x1d5c0,
+ 0x1d5f4,
+ 0x1d628,
+ 0x1d65c,
+ 0x1d690,
+ 0x00261,
+ 0x01d83,
+ 0x0018d,
+ 0x00581,
+ 0x1d406,
+ 0x1d43a,
+ 0x1d46e,
+ 0x1d4a2,
+ 0x1d4d6,
+ 0x1d50a,
+ 0x1d53e,
+ 0x1d572,
+ 0x1d5a6,
+ 0x1d5da,
+ 0x1d60e,
+ 0x1d642,
+ 0x1d676,
+ 0x0050c,
+ 0x013c0,
+ 0x013f3,
+ 0x0a4d6,
+ 0x0ff48,
+ 0x0210e,
+ 0x1d421,
+ 0x1d489,
+ 0x1d4bd,
+ 0x1d4f1,
+ 0x1d525,
+ 0x1d559,
+ 0x1d58d,
+ 0x1d5c1,
+ 0x1d5f5,
+ 0x1d629,
+ 0x1d65d,
+ 0x1d691,
+ 0x004bb,
+ 0x00570,
+ 0x013c2,
+ 0x0ff28,
+ 0x0210b,
+ 0x0210c,
+ 0x0210d,
+ 0x1d407,
+ 0x1d43b,
+ 0x1d46f,
+ 0x1d4d7,
+ 0x1d573,
+ 0x1d5a7,
+ 0x1d5db,
+ 0x1d60f,
+ 0x1d643,
+ 0x1d677,
+ 0x00397,
+ 0x1d6ae,
+ 0x1d6e8,
+ 0x1d722,
+ 0x1d75c,
+ 0x1d796,
+ 0x02c8e,
+ 0x0041d,
+ 0x013bb,
+ 0x0157c,
+ 0x0a4e7,
+ 0x102cf,
+ 0x002db,
+ 0x02373,
+ 0x0ff49,
+ 0x02170,
+ 0x02139,
+ 0x02148,
+ 0x1d422,
+ 0x1d456,
+ 0x1d48a,
+ 0x1d4be,
+ 0x1d4f2,
+ 0x1d526,
+ 0x1d55a,
+ 0x1d58e,
+ 0x1d5c2,
+ 0x1d5f6,
+ 0x1d62a,
+ 0x1d65e,
+ 0x1d692,
+ 0x00131,
+ 0x1d6a4,
+ 0x0026a,
+ 0x00269,
+ 0x003b9,
+ 0x01fbe,
+ 0x0037a,
+ 0x1d6ca,
+ 0x1d704,
+ 0x1d73e,
+ 0x1d778,
+ 0x1d7b2,
+ 0x00456,
+ 0x0a647,
+ 0x004cf,
+ 0x0ab75,
+ 0x013a5,
+ 0x118c3,
+ 0x0ff4a,
+ 0x02149,
+ 0x1d423,
+ 0x1d457,
+ 0x1d48b,
+ 0x1d4bf,
+ 0x1d4f3,
+ 0x1d527,
+ 0x1d55b,
+ 0x1d58f,
+ 0x1d5c3,
+ 0x1d5f7,
+ 0x1d62b,
+ 0x1d65f,
+ 0x1d693,
+ 0x003f3,
+ 0x00458,
+ 0x0ff2a,
+ 0x1d409,
+ 0x1d43d,
+ 0x1d471,
+ 0x1d4a5,
+ 0x1d4d9,
+ 0x1d50d,
+ 0x1d541,
+ 0x1d575,
+ 0x1d5a9,
+ 0x1d5dd,
+ 0x1d611,
+ 0x1d645,
+ 0x1d679,
+ 0x0a7b2,
+ 0x0037f,
+ 0x00408,
+ 0x013ab,
+ 0x0148d,
+ 0x0a4d9,
+ 0x1d424,
+ 0x1d458,
+ 0x1d48c,
+ 0x1d4c0,
+ 0x1d4f4,
+ 0x1d528,
+ 0x1d55c,
+ 0x1d590,
+ 0x1d5c4,
+ 0x1d5f8,
+ 0x1d62c,
+ 0x1d660,
+ 0x1d694,
+ 0x0212a,
+ 0x0ff2b,
+ 0x1d40a,
+ 0x1d43e,
+ 0x1d472,
+ 0x1d4a6,
+ 0x1d4da,
+ 0x1d50e,
+ 0x1d542,
+ 0x1d576,
+ 0x1d5aa,
+ 0x1d5de,
+ 0x1d612,
+ 0x1d646,
+ 0x1d67a,
+ 0x0039a,
+ 0x1d6b1,
+ 0x1d6eb,
+ 0x1d725,
+ 0x1d75f,
+ 0x1d799,
+ 0x02c94,
+ 0x0041a,
+ 0x013e6,
+ 0x016d5,
+ 0x0a4d7,
+ 0x10518,
+ 0x005c0,
+ 0x0007c,
+ 0x02223,
+ 0x023fd,
+ 0x0ffe8,
+ 0x00031,
+ 0x00661,
+ 0x006f1,
+ 0x10320,
+ 0x1e8c7,
+ 0x1d7cf,
+ 0x1d7d9,
+ 0x1d7e3,
+ 0x1d7ed,
+ 0x1d7f7,
+ 0x00049,
+ 0x0ff29,
+ 0x02160,
+ 0x02110,
+ 0x02111,
+ 0x1d408,
+ 0x1d43c,
+ 0x1d470,
+ 0x1d4d8,
+ 0x1d540,
+ 0x1d574,
+ 0x1d5a8,
+ 0x1d5dc,
+ 0x1d610,
+ 0x1d644,
+ 0x1d678,
+ 0x00196,
+ 0x0ff4c,
+ 0x0217c,
+ 0x02113,
+ 0x1d425,
+ 0x1d459,
+ 0x1d48d,
+ 0x1d4c1,
+ 0x1d4f5,
+ 0x1d529,
+ 0x1d55d,
+ 0x1d591,
+ 0x1d5c5,
+ 0x1d5f9,
+ 0x1d62d,
+ 0x1d661,
+ 0x1d695,
+ 0x001c0,
+ 0x00399,
+ 0x1d6b0,
+ 0x1d6ea,
+ 0x1d724,
+ 0x1d75e,
+ 0x1d798,
+ 0x02c92,
+ 0x00406,
+ 0x004c0,
+ 0x005d5,
+ 0x005df,
+ 0x00627,
+ 0x1ee00,
+ 0x1ee80,
+ 0x0fe8e,
+ 0x0fe8d,
+ 0x007ca,
+ 0x02d4f,
+ 0x016c1,
+ 0x0a4f2,
+ 0x16f28,
+ 0x1028a,
+ 0x10309,
+ 0x1d22a,
+ 0x0216c,
+ 0x02112,
+ 0x1d40b,
+ 0x1d43f,
+ 0x1d473,
+ 0x1d4db,
+ 0x1d50f,
+ 0x1d543,
+ 0x1d577,
+ 0x1d5ab,
+ 0x1d5df,
+ 0x1d613,
+ 0x1d647,
+ 0x1d67b,
+ 0x02cd0,
+ 0x013de,
+ 0x014aa,
+ 0x0a4e1,
+ 0x16f16,
+ 0x118a3,
+ 0x118b2,
+ 0x1041b,
+ 0x10526,
+ 0x0ff2d,
+ 0x0216f,
+ 0x02133,
+ 0x1d40c,
+ 0x1d440,
+ 0x1d474,
+ 0x1d4dc,
+ 0x1d510,
+ 0x1d544,
+ 0x1d578,
+ 0x1d5ac,
+ 0x1d5e0,
+ 0x1d614,
+ 0x1d648,
+ 0x1d67c,
+ 0x0039c,
+ 0x1d6b3,
+ 0x1d6ed,
+ 0x1d727,
+ 0x1d761,
+ 0x1d79b,
+ 0x003fa,
+ 0x02c98,
+ 0x0041c,
+ 0x013b7,
+ 0x015f0,
+ 0x016d6,
+ 0x0a4df,
+ 0x102b0,
+ 0x10311,
+ 0x1d427,
+ 0x1d45b,
+ 0x1d48f,
+ 0x1d4c3,
+ 0x1d4f7,
+ 0x1d52b,
+ 0x1d55f,
+ 0x1d593,
+ 0x1d5c7,
+ 0x1d5fb,
+ 0x1d62f,
+ 0x1d663,
+ 0x1d697,
+ 0x00578,
+ 0x0057c,
+ 0x0ff2e,
+ 0x02115,
+ 0x1d40d,
+ 0x1d441,
+ 0x1d475,
+ 0x1d4a9,
+ 0x1d4dd,
+ 0x1d511,
+ 0x1d579,
+ 0x1d5ad,
+ 0x1d5e1,
+ 0x1d615,
+ 0x1d649,
+ 0x1d67d,
+ 0x0039d,
+ 0x1d6b4,
+ 0x1d6ee,
+ 0x1d728,
+ 0x1d762,
+ 0x1d79c,
+ 0x02c9a,
+ 0x0a4e0,
+ 0x10513,
+ 0x00c02,
+ 0x00c82,
+ 0x00d02,
+ 0x00d82,
+ 0x00966,
+ 0x00a66,
+ 0x00ae6,
+ 0x00be6,
+ 0x00c66,
+ 0x00ce6,
+ 0x00d66,
+ 0x00e50,
+ 0x00ed0,
+ 0x01040,
+ 0x00665,
+ 0x006f5,
+ 0x0ff4f,
+ 0x02134,
+ 0x1d428,
+ 0x1d45c,
+ 0x1d490,
+ 0x1d4f8,
+ 0x1d52c,
+ 0x1d560,
+ 0x1d594,
+ 0x1d5c8,
+ 0x1d5fc,
+ 0x1d630,
+ 0x1d664,
+ 0x1d698,
+ 0x01d0f,
+ 0x01d11,
+ 0x0ab3d,
+ 0x003bf,
+ 0x1d6d0,
+ 0x1d70a,
+ 0x1d744,
+ 0x1d77e,
+ 0x1d7b8,
+ 0x003c3,
+ 0x1d6d4,
+ 0x1d70e,
+ 0x1d748,
+ 0x1d782,
+ 0x1d7bc,
+ 0x02c9f,
+ 0x0043e,
+ 0x010ff,
+ 0x00585,
+ 0x005e1,
+ 0x00647,
+ 0x1ee24,
+ 0x1ee64,
+ 0x1ee84,
+ 0x0feeb,
+ 0x0feec,
+ 0x0feea,
+ 0x0fee9,
+ 0x006be,
+ 0x0fbac,
+ 0x0fbad,
+ 0x0fbab,
+ 0x0fbaa,
+ 0x006c1,
+ 0x0fba8,
+ 0x0fba9,
+ 0x0fba7,
+ 0x0fba6,
+ 0x006d5,
+ 0x00d20,
+ 0x0101d,
+ 0x104ea,
+ 0x118c8,
+ 0x118d7,
+ 0x1042c,
+ 0x00030,
+ 0x007c0,
+ 0x009e6,
+ 0x00b66,
+ 0x03007,
+ 0x114d0,
+ 0x118e0,
+ 0x1d7ce,
+ 0x1d7d8,
+ 0x1d7e2,
+ 0x1d7ec,
+ 0x1d7f6,
+ 0x0ff2f,
+ 0x1d40e,
+ 0x1d442,
+ 0x1d476,
+ 0x1d4aa,
+ 0x1d4de,
+ 0x1d512,
+ 0x1d546,
+ 0x1d57a,
+ 0x1d5ae,
+ 0x1d5e2,
+ 0x1d616,
+ 0x1d64a,
+ 0x1d67e,
+ 0x0039f,
+ 0x1d6b6,
+ 0x1d6f0,
+ 0x1d72a,
+ 0x1d764,
+ 0x1d79e,
+ 0x02c9e,
+ 0x0041e,
+ 0x00555,
+ 0x02d54,
+ 0x012d0,
+ 0x00b20,
+ 0x104c2,
+ 0x0a4f3,
+ 0x118b5,
+ 0x10292,
+ 0x102ab,
+ 0x10404,
+ 0x10516,
+ 0x02374,
+ 0x0ff50,
+ 0x1d429,
+ 0x1d45d,
+ 0x1d491,
+ 0x1d4c5,
+ 0x1d4f9,
+ 0x1d52d,
+ 0x1d561,
+ 0x1d595,
+ 0x1d5c9,
+ 0x1d5fd,
+ 0x1d631,
+ 0x1d665,
+ 0x1d699,
+ 0x003c1,
+ 0x003f1,
+ 0x1d6d2,
+ 0x1d6e0,
+ 0x1d70c,
+ 0x1d71a,
+ 0x1d746,
+ 0x1d754,
+ 0x1d780,
+ 0x1d78e,
+ 0x1d7ba,
+ 0x1d7c8,
+ 0x02ca3,
+ 0x00440,
+ 0x0ff30,
+ 0x02119,
+ 0x1d40f,
+ 0x1d443,
+ 0x1d477,
+ 0x1d4ab,
+ 0x1d4df,
+ 0x1d513,
+ 0x1d57b,
+ 0x1d5af,
+ 0x1d5e3,
+ 0x1d617,
+ 0x1d64b,
+ 0x1d67f,
+ 0x003a1,
+ 0x1d6b8,
+ 0x1d6f2,
+ 0x1d72c,
+ 0x1d766,
+ 0x1d7a0,
+ 0x02ca2,
+ 0x00420,
+ 0x013e2,
+ 0x0146d,
+ 0x0a4d1,
+ 0x10295,
+ 0x1d42a,
+ 0x1d45e,
+ 0x1d492,
+ 0x1d4c6,
+ 0x1d4fa,
+ 0x1d52e,
+ 0x1d562,
+ 0x1d596,
+ 0x1d5ca,
+ 0x1d5fe,
+ 0x1d632,
+ 0x1d666,
+ 0x1d69a,
+ 0x0051b,
+ 0x00563,
+ 0x00566,
+ 0x0211a,
+ 0x1d410,
+ 0x1d444,
+ 0x1d478,
+ 0x1d4ac,
+ 0x1d4e0,
+ 0x1d514,
+ 0x1d57c,
+ 0x1d5b0,
+ 0x1d5e4,
+ 0x1d618,
+ 0x1d64c,
+ 0x1d680,
+ 0x02d55,
+ 0x1d42b,
+ 0x1d45f,
+ 0x1d493,
+ 0x1d4c7,
+ 0x1d4fb,
+ 0x1d52f,
+ 0x1d563,
+ 0x1d597,
+ 0x1d5cb,
+ 0x1d5ff,
+ 0x1d633,
+ 0x1d667,
+ 0x1d69b,
+ 0x0ab47,
+ 0x0ab48,
+ 0x01d26,
+ 0x02c85,
+ 0x00433,
+ 0x0ab81,
+ 0x1d216,
+ 0x0211b,
+ 0x0211c,
+ 0x0211d,
+ 0x1d411,
+ 0x1d445,
+ 0x1d479,
+ 0x1d4e1,
+ 0x1d57d,
+ 0x1d5b1,
+ 0x1d5e5,
+ 0x1d619,
+ 0x1d64d,
+ 0x1d681,
+ 0x001a6,
+ 0x013a1,
+ 0x013d2,
+ 0x104b4,
+ 0x01587,
+ 0x0a4e3,
+ 0x16f35,
+ 0x0ff53,
+ 0x1d42c,
+ 0x1d460,
+ 0x1d494,
+ 0x1d4c8,
+ 0x1d4fc,
+ 0x1d530,
+ 0x1d564,
+ 0x1d598,
+ 0x1d5cc,
+ 0x1d600,
+ 0x1d634,
+ 0x1d668,
+ 0x1d69c,
+ 0x0a731,
+ 0x001bd,
+ 0x00455,
+ 0x0abaa,
+ 0x118c1,
+ 0x10448,
+ 0x0ff33,
+ 0x1d412,
+ 0x1d446,
+ 0x1d47a,
+ 0x1d4ae,
+ 0x1d4e2,
+ 0x1d516,
+ 0x1d54a,
+ 0x1d57e,
+ 0x1d5b2,
+ 0x1d5e6,
+ 0x1d61a,
+ 0x1d64e,
+ 0x1d682,
+ 0x00405,
+ 0x0054f,
+ 0x013d5,
+ 0x013da,
+ 0x0a4e2,
+ 0x16f3a,
+ 0x10296,
+ 0x10420,
+ 0x1d42d,
+ 0x1d461,
+ 0x1d495,
+ 0x1d4c9,
+ 0x1d4fd,
+ 0x1d531,
+ 0x1d565,
+ 0x1d599,
+ 0x1d5cd,
+ 0x1d601,
+ 0x1d635,
+ 0x1d669,
+ 0x1d69d,
+ 0x022a4,
+ 0x027d9,
+ 0x1f768,
+ 0x0ff34,
+ 0x1d413,
+ 0x1d447,
+ 0x1d47b,
+ 0x1d4af,
+ 0x1d4e3,
+ 0x1d517,
+ 0x1d54b,
+ 0x1d57f,
+ 0x1d5b3,
+ 0x1d5e7,
+ 0x1d61b,
+ 0x1d64f,
+ 0x1d683,
+ 0x003a4,
+ 0x1d6bb,
+ 0x1d6f5,
+ 0x1d72f,
+ 0x1d769,
+ 0x1d7a3,
+ 0x02ca6,
+ 0x00422,
+ 0x013a2,
+ 0x0a4d4,
+ 0x16f0a,
+ 0x118bc,
+ 0x10297,
+ 0x102b1,
+ 0x10315,
+ 0x1d42e,
+ 0x1d462,
+ 0x1d496,
+ 0x1d4ca,
+ 0x1d4fe,
+ 0x1d532,
+ 0x1d566,
+ 0x1d59a,
+ 0x1d5ce,
+ 0x1d602,
+ 0x1d636,
+ 0x1d66a,
+ 0x1d69e,
+ 0x0a79f,
+ 0x01d1c,
+ 0x0ab4e,
+ 0x0ab52,
+ 0x0028b,
+ 0x003c5,
+ 0x1d6d6,
+ 0x1d710,
+ 0x1d74a,
+ 0x1d784,
+ 0x1d7be,
+ 0x0057d,
+ 0x104f6,
+ 0x118d8,
+ 0x0222a,
+ 0x022c3,
+ 0x1d414,
+ 0x1d448,
+ 0x1d47c,
+ 0x1d4b0,
+ 0x1d4e4,
+ 0x1d518,
+ 0x1d54c,
+ 0x1d580,
+ 0x1d5b4,
+ 0x1d5e8,
+ 0x1d61c,
+ 0x1d650,
+ 0x1d684,
+ 0x0054d,
+ 0x01200,
+ 0x104ce,
+ 0x0144c,
+ 0x0a4f4,
+ 0x16f42,
+ 0x118b8,
+ 0x02228,
+ 0x022c1,
+ 0x0ff56,
+ 0x02174,
+ 0x1d42f,
+ 0x1d463,
+ 0x1d497,
+ 0x1d4cb,
+ 0x1d4ff,
+ 0x1d533,
+ 0x1d567,
+ 0x1d59b,
+ 0x1d5cf,
+ 0x1d603,
+ 0x1d637,
+ 0x1d66b,
+ 0x1d69f,
+ 0x01d20,
+ 0x003bd,
+ 0x1d6ce,
+ 0x1d708,
+ 0x1d742,
+ 0x1d77c,
+ 0x1d7b6,
+ 0x00475,
+ 0x005d8,
+ 0x11706,
+ 0x0aba9,
+ 0x118c0,
+ 0x1d20d,
+ 0x00667,
+ 0x006f7,
+ 0x02164,
+ 0x1d415,
+ 0x1d449,
+ 0x1d47d,
+ 0x1d4b1,
+ 0x1d4e5,
+ 0x1d519,
+ 0x1d54d,
+ 0x1d581,
+ 0x1d5b5,
+ 0x1d5e9,
+ 0x1d61d,
+ 0x1d651,
+ 0x1d685,
+ 0x00474,
+ 0x02d38,
+ 0x013d9,
+ 0x0142f,
+ 0x0a6df,
+ 0x0a4e6,
+ 0x16f08,
+ 0x118a0,
+ 0x1051d,
+ 0x0026f,
+ 0x1d430,
+ 0x1d464,
+ 0x1d498,
+ 0x1d4cc,
+ 0x1d500,
+ 0x1d534,
+ 0x1d568,
+ 0x1d59c,
+ 0x1d5d0,
+ 0x1d604,
+ 0x1d638,
+ 0x1d66c,
+ 0x1d6a0,
+ 0x01d21,
+ 0x00461,
+ 0x0051d,
+ 0x00561,
+ 0x1170a,
+ 0x1170e,
+ 0x1170f,
+ 0x0ab83,
+ 0x118ef,
+ 0x118e6,
+ 0x1d416,
+ 0x1d44a,
+ 0x1d47e,
+ 0x1d4b2,
+ 0x1d4e6,
+ 0x1d51a,
+ 0x1d54e,
+ 0x1d582,
+ 0x1d5b6,
+ 0x1d5ea,
+ 0x1d61e,
+ 0x1d652,
+ 0x1d686,
+ 0x0051c,
+ 0x013b3,
+ 0x013d4,
+ 0x0a4ea,
+ 0x0166e,
+ 0x000d7,
+ 0x0292b,
+ 0x0292c,
+ 0x02a2f,
+ 0x0ff58,
+ 0x02179,
+ 0x1d431,
+ 0x1d465,
+ 0x1d499,
+ 0x1d4cd,
+ 0x1d501,
+ 0x1d535,
+ 0x1d569,
+ 0x1d59d,
+ 0x1d5d1,
+ 0x1d605,
+ 0x1d639,
+ 0x1d66d,
+ 0x1d6a1,
+ 0x00445,
+ 0x01541,
+ 0x0157d,
+ 0x0166d,
+ 0x02573,
+ 0x10322,
+ 0x118ec,
+ 0x0ff38,
+ 0x02169,
+ 0x1d417,
+ 0x1d44b,
+ 0x1d47f,
+ 0x1d4b3,
+ 0x1d4e7,
+ 0x1d51b,
+ 0x1d54f,
+ 0x1d583,
+ 0x1d5b7,
+ 0x1d5eb,
+ 0x1d61f,
+ 0x1d653,
+ 0x1d687,
+ 0x0a7b3,
+ 0x003a7,
+ 0x1d6be,
+ 0x1d6f8,
+ 0x1d732,
+ 0x1d76c,
+ 0x1d7a6,
+ 0x02cac,
+ 0x00425,
+ 0x02d5d,
+ 0x016b7,
+ 0x0a4eb,
+ 0x10290,
+ 0x102b4,
+ 0x10317,
+ 0x10527,
+ 0x00263,
+ 0x01d8c,
+ 0x0ff59,
+ 0x1d432,
+ 0x1d466,
+ 0x1d49a,
+ 0x1d4ce,
+ 0x1d502,
+ 0x1d536,
+ 0x1d56a,
+ 0x1d59e,
+ 0x1d5d2,
+ 0x1d606,
+ 0x1d63a,
+ 0x1d66e,
+ 0x1d6a2,
+ 0x0028f,
+ 0x01eff,
+ 0x0ab5a,
+ 0x003b3,
+ 0x0213d,
+ 0x1d6c4,
+ 0x1d6fe,
+ 0x1d738,
+ 0x1d772,
+ 0x1d7ac,
+ 0x00443,
+ 0x004af,
+ 0x010e7,
+ 0x118dc,
+ 0x0ff39,
+ 0x1d418,
+ 0x1d44c,
+ 0x1d480,
+ 0x1d4b4,
+ 0x1d4e8,
+ 0x1d51c,
+ 0x1d550,
+ 0x1d584,
+ 0x1d5b8,
+ 0x1d5ec,
+ 0x1d620,
+ 0x1d654,
+ 0x1d688,
+ 0x003a5,
+ 0x003d2,
+ 0x1d6bc,
+ 0x1d6f6,
+ 0x1d730,
+ 0x1d76a,
+ 0x1d7a4,
+ 0x02ca8,
+ 0x00423,
+ 0x004ae,
+ 0x013a9,
+ 0x013bd,
+ 0x0a4ec,
+ 0x16f43,
+ 0x118a4,
+ 0x102b2,
+ 0x1d433,
+ 0x1d467,
+ 0x1d49b,
+ 0x1d4cf,
+ 0x1d503,
+ 0x1d537,
+ 0x1d56b,
+ 0x1d59f,
+ 0x1d5d3,
+ 0x1d607,
+ 0x1d63b,
+ 0x1d66f,
+ 0x1d6a3,
+ 0x01d22,
+ 0x0ab93,
+ 0x118c4,
+ 0x102f5,
+ 0x118e5,
+ 0x0ff3a,
+ 0x02124,
+ 0x02128,
+ 0x1d419,
+ 0x1d44d,
+ 0x1d481,
+ 0x1d4b5,
+ 0x1d4e9,
+ 0x1d585,
+ 0x1d5b9,
+ 0x1d5ed,
+ 0x1d621,
+ 0x1d655,
+ 0x1d689,
+ 0x00396,
+ 0x1d6ad,
+ 0x1d6e7,
+ 0x1d721,
+ 0x1d75b,
+ 0x1d795,
+ 0x013c3,
+ 0x0a4dc,
+ 0x118a9,
+};
+
+static gboolean
+rspamd_can_alias_latin(gint ch)
+{
+ return latin_confusable.contains(ch);
+}
+
+static gdouble
+rspamd_chartable_process_word_utf(struct rspamd_task *task,
+ rspamd_stat_token_t *w,
+ gboolean is_url,
+ guint *ncap,
+ struct chartable_ctx *chartable_module_ctx,
+ gboolean ignore_diacritics)
+{
+ const UChar32 *p, *end;
+ gdouble badness = 0.0;
+ UChar32 uc;
+ UBlockCode sc;
+ guint cat;
+ gint last_is_latin = -1;
+ guint same_script_count = 0, nsym = 0, nspecial = 0;
+ enum {
+ start_process = 0,
+ got_alpha,
+ got_digit,
+ got_unknown,
+ } state = start_process,
+ prev_state = start_process;
+
+ p = w->unicode.begin;
+ end = p + w->unicode.len;
+
+ /* We assume that w is normalized */
+
+ while (p < end) {
+ uc = *p++;
+
+ if (((gint32) uc) < 0) {
+ break;
+ }
+
+ sc = ublock_getCode(uc);
+ cat = u_charType(uc);
+
+ if (!ignore_diacritics) {
+ if (cat == U_NON_SPACING_MARK ||
+ (sc == UBLOCK_LATIN_1_SUPPLEMENT) ||
+ (sc == UBLOCK_LATIN_EXTENDED_A) ||
+ (sc == UBLOCK_LATIN_EXTENDED_ADDITIONAL) ||
+ (sc == UBLOCK_LATIN_EXTENDED_B) ||
+ (sc == UBLOCK_COMBINING_DIACRITICAL_MARKS)) {
+ nspecial++;
+ }
+ }
+
+ if (u_isalpha(uc)) {
+
+ if (sc <= UBLOCK_COMBINING_DIACRITICAL_MARKS ||
+ sc == UBLOCK_LATIN_EXTENDED_ADDITIONAL) {
+ /*
+ * Assume all latin, IPA, diacritic and space modifiers
+ * characters as basic latin
+ */
+ sc = UBLOCK_BASIC_LATIN;
+ }
+
+ if (sc != UBLOCK_BASIC_LATIN && u_isupper(uc)) {
+ if (ncap) {
+ (*ncap)++;
+ }
+ }
+
+ if (state == got_digit) {
+ /* Penalize digit -> alpha translations */
+ if (!is_url && sc != UBLOCK_BASIC_LATIN &&
+ prev_state != start_process) {
+ badness += 0.25;
+ }
+ }
+ else if (state == got_alpha) {
+ /* Check script */
+ if (same_script_count > 0) {
+ if (sc != UBLOCK_BASIC_LATIN && last_is_latin) {
+
+ if (rspamd_can_alias_latin(uc)) {
+ badness += 1.0 / (gdouble) same_script_count;
+ }
+
+ last_is_latin = 0;
+ same_script_count = 1;
+ }
+ else {
+ same_script_count++;
+ }
+ }
+ else {
+ last_is_latin = sc == UBLOCK_BASIC_LATIN;
+ same_script_count = 1;
+ }
+ }
+
+ prev_state = state;
+ state = got_alpha;
+ }
+ else if (u_isdigit(uc)) {
+ if (state != got_digit) {
+ prev_state = state;
+ }
+
+ state = got_digit;
+ same_script_count = 0;
+ }
+ else {
+ /* We don't care about unknown characters here */
+ if (state != got_unknown) {
+ prev_state = state;
+ }
+
+ state = got_unknown;
+ same_script_count = 0;
+ }
+
+ nsym++;
+ }
+
+ if (nspecial > 0) {
+ if (!ignore_diacritics) {
+ /* Count diacritics */
+ badness += nspecial;
+ }
+ else if (nspecial > 1) {
+ badness += (nspecial - 1.0) / 2.0;
+ }
+ }
+
+ /* Try to avoid FP for long words */
+ if (nsym > chartable_module_ctx->max_word_len) {
+ badness = 0;
+ }
+ else {
+ if (badness > 4.0) {
+ badness = 4.0;
+ }
+ }
+
+ msg_debug_chartable("word %*s, badness: %.2f",
+ (gint) w->normalized.len, w->normalized.begin,
+ badness);
+
+ return badness;
+}
+
+static gdouble
+rspamd_chartable_process_word_ascii(struct rspamd_task *task,
+ rspamd_stat_token_t *w,
+ gboolean is_url,
+ struct chartable_ctx *chartable_module_ctx)
+{
+ gdouble badness = 0.0;
+ enum {
+ ascii = 1,
+ non_ascii
+ } sc,
+ last_sc;
+ gint same_script_count = 0, seen_alpha = FALSE;
+ enum {
+ start_process = 0,
+ got_alpha,
+ got_digit,
+ got_unknown,
+ } state = start_process;
+
+ const auto *p = (const unsigned char *) w->normalized.begin;
+ const auto *end = p + w->normalized.len;
+ last_sc = non_ascii;
+
+ if (w->normalized.len > chartable_module_ctx->max_word_len) {
+ return 0.0;
+ }
+
+ /* We assume that w is normalized */
+ while (p < end) {
+ if (g_ascii_isalpha(*p) || *p > 0x7f) {
+
+ if (state == got_digit) {
+ /* Penalize digit -> alpha translations */
+ if (seen_alpha && !is_url && !g_ascii_isxdigit(*p)) {
+ badness += 0.25;
+ }
+ }
+ else if (state == got_alpha) {
+ /* Check script */
+ sc = (*p > 0x7f) ? ascii : non_ascii;
+
+ if (same_script_count > 0) {
+ if (sc != last_sc) {
+ badness += 1.0 / (gdouble) same_script_count;
+ last_sc = sc;
+ same_script_count = 1;
+ }
+ else {
+ same_script_count++;
+ }
+ }
+ else {
+ last_sc = sc;
+ same_script_count = 1;
+ }
+ }
+
+ seen_alpha = TRUE;
+ state = got_alpha;
+ }
+ else if (g_ascii_isdigit(*p)) {
+ state = got_digit;
+ same_script_count = 0;
+ }
+ else {
+ /* We don't care about unknown characters here */
+ state = got_unknown;
+ same_script_count = 0;
+ }
+
+ p++;
+ }
+
+ if (badness > 4.0) {
+ badness = 4.0;
+ }
+
+ msg_debug_chartable("word %*s, badness: %.2f",
+ (gint) w->normalized.len, w->normalized.begin,
+ badness);
+
+ return badness;
+}
+
+static gboolean
+rspamd_chartable_process_part(struct rspamd_task *task,
+ struct rspamd_mime_text_part *part,
+ struct chartable_ctx *chartable_module_ctx,
+ gboolean ignore_diacritics)
+{
+ rspamd_stat_token_t *w;
+ guint i, ncap = 0;
+ gdouble cur_score = 0.0;
+
+ if (part == nullptr || part->utf_words == nullptr ||
+ part->utf_words->len == 0 || part->nwords == 0) {
+ return FALSE;
+ }
+
+ for (i = 0; i < part->utf_words->len; i++) {
+ w = &g_array_index(part->utf_words, rspamd_stat_token_t, i);
+
+ if ((w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT)) {
+
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) {
+ cur_score += rspamd_chartable_process_word_utf(task, w, FALSE,
+ &ncap, chartable_module_ctx, ignore_diacritics);
+ }
+ else {
+ cur_score += rspamd_chartable_process_word_ascii(task, w,
+ FALSE, chartable_module_ctx);
+ }
+ }
+ }
+
+ /*
+ * TODO: perhaps, we should do this analysis somewhere else and get
+ * something like: <SYM_SC><SYM_SC><SYM_SC> representing classes for all
+ * symbols in the text
+ */
+ part->capital_letters += ncap;
+
+ cur_score /= (gdouble) part->nwords;
+
+ if (cur_score > 1.0) {
+ cur_score = 1.0;
+ }
+
+ if (cur_score > chartable_module_ctx->threshold) {
+ rspamd_task_insert_result(task, chartable_module_ctx->symbol,
+ cur_score, nullptr);
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static void
+chartable_symbol_callback(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ void *_)
+{
+ guint i;
+ struct rspamd_mime_text_part *part;
+ struct chartable_ctx *chartable_module_ctx = chartable_get_context(task->cfg);
+ gboolean ignore_diacritics = TRUE, seen_violated_part = FALSE;
+
+ /* Check if we have parts with diacritic symbols language */
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, part)
+ {
+ if (part->languages && part->languages->len > 0) {
+ auto *lang = (struct rspamd_lang_detector_res *) g_ptr_array_index(part->languages, 0);
+ gint flags;
+
+ flags = rspamd_language_detector_elt_flags(lang->elt);
+
+ if ((flags & RS_LANGUAGE_DIACRITICS)) {
+ ignore_diacritics = TRUE;
+ }
+ else if (lang->prob > 0.75) {
+ ignore_diacritics = FALSE;
+ }
+ }
+
+ if (rspamd_chartable_process_part(task, part, chartable_module_ctx, ignore_diacritics)) {
+ seen_violated_part = TRUE;
+ }
+ }
+
+ if (MESSAGE_FIELD(task, text_parts)->len == 0) {
+ /* No text parts, assume that we should ignore diacritics checks for metatokens */
+ ignore_diacritics = TRUE;
+ }
+
+ if (task->meta_words != nullptr && task->meta_words->len > 0) {
+ rspamd_stat_token_t *w;
+ gdouble cur_score = 0;
+ gsize arlen = task->meta_words->len;
+
+ for (i = 0; i < arlen; i++) {
+ w = &g_array_index(task->meta_words, rspamd_stat_token_t, i);
+ cur_score += rspamd_chartable_process_word_utf(task, w, FALSE,
+ nullptr, chartable_module_ctx, ignore_diacritics);
+ }
+
+ cur_score /= (gdouble) (arlen + 1);
+
+ if (cur_score > 1.0) {
+ cur_score = 1.0;
+ }
+
+ if (cur_score > chartable_module_ctx->threshold) {
+ if (!seen_violated_part) {
+ /* Further penalise */
+ if (cur_score > 0.25) {
+ cur_score = 0.25;
+ }
+ }
+
+ rspamd_task_insert_result(task, chartable_module_ctx->symbol,
+ cur_score, "subject");
+ }
+ }
+
+ rspamd_symcache_finalize_item(task, item);
+}
+
+static void
+chartable_url_symbol_callback(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ void *unused)
+{
+ /* XXX: TODO: unbreak module once URLs unicode project is over */
+#if 0
+ struct rspamd_url *u;
+ GHashTableIter it;
+ gpointer k, v;
+ rspamd_stat_token_t w;
+ gdouble cur_score = 0.0;
+ struct chartable_ctx *chartable_module_ctx = chartable_get_context (task->cfg);
+
+ g_hash_table_iter_init (&it, task->urls);
+
+ while (g_hash_table_iter_next (&it, &k, &v)) {
+ u = v;
+
+ if (cur_score > 2.0) {
+ cur_score = 2.0;
+ break;
+ }
+
+ if (u->hostlen > 0) {
+ w.stemmed.begin = u->host;
+ w.stemmed.len = u->hostlen;
+
+ if (g_utf8_validate (w.stemmed.begin, w.stemmed.len, nullptr)) {
+ cur_score += rspamd_chartable_process_word_utf (task, &w,
+ TRUE, nullptr, chartable_module_ctx);
+ }
+ else {
+ cur_score += rspamd_chartable_process_word_ascii (task, &w,
+ TRUE, chartable_module_ctx);
+ }
+ }
+ }
+
+ g_hash_table_iter_init (&it, task->emails);
+
+ while (g_hash_table_iter_next (&it, &k, &v)) {
+ u = v;
+
+ if (cur_score > 2.0) {
+ cur_score = 2.0;
+ break;
+ }
+
+ if (u->hostlen > 0) {
+ w.stemmed.begin = u->host;
+ w.stemmed.len = u->hostlen;
+
+ if (g_utf8_validate (w.stemmed.begin, w.stemmed.len, nullptr)) {
+ cur_score += rspamd_chartable_process_word_utf (task, &w,
+ TRUE, nullptr, chartable_module_ctx);
+ }
+ else {
+ cur_score += rspamd_chartable_process_word_ascii (task, &w,
+ TRUE, chartable_module_ctx);
+ }
+ }
+ }
+
+ if (cur_score > chartable_module_ctx->threshold) {
+ rspamd_task_insert_result (task, chartable_module_ctx->symbol,
+ cur_score, nullptr);
+
+ }
+#endif
+ rspamd_symcache_finalize_item(task, item);
+}
diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c
new file mode 100644
index 0000000..29ab34d
--- /dev/null
+++ b/src/plugins/dkim_check.c
@@ -0,0 +1,1620 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/***MODULE:dkim
+ * rspamd module that checks dkim records of incoming email
+ *
+ * Allowed options:
+ * - symbol_allow (string): symbol to insert in case of allow (default: 'R_DKIM_ALLOW')
+ * - symbol_reject (string): symbol to insert (default: 'R_DKIM_REJECT')
+ * - symbol_tempfail (string): symbol to insert in case of temporary fail (default: 'R_DKIM_TEMPFAIL')
+ * - symbol_permfail (string): symbol to insert in case of permanent failure (default: 'R_DKIM_PERMFAIL')
+ * - symbol_na (string): symbol to insert in case of no signing (default: 'R_DKIM_NA')
+ * - whitelist (map): map of whitelisted networks
+ * - domains (map): map of domains to check
+ * - strict_multiplier (number): multiplier for strict domains
+ * - time_jitter (number): jitter in seconds to allow time diff while checking
+ * - trusted_only (flag): check signatures only for domains in 'domains' map
+ */
+
+
+#include "config.h"
+#include "libmime/message.h"
+#include "libserver/dkim.h"
+#include "libutil/hash.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
+#include "rspamd.h"
+#include "utlist.h"
+#include "unix-std.h"
+#include "lua/lua_common.h"
+#include "libserver/mempool_vars_internal.h"
+
+#define DEFAULT_SYMBOL_REJECT "R_DKIM_REJECT"
+#define DEFAULT_SYMBOL_TEMPFAIL "R_DKIM_TEMPFAIL"
+#define DEFAULT_SYMBOL_ALLOW "R_DKIM_ALLOW"
+#define DEFAULT_SYMBOL_NA "R_DKIM_NA"
+#define DEFAULT_SYMBOL_PERMFAIL "R_DKIM_PERMFAIL"
+#define DEFAULT_CACHE_SIZE 2048
+#define DEFAULT_TIME_JITTER 60
+#define DEFAULT_MAX_SIGS 5
+
+static const gchar *M = "rspamd dkim plugin";
+
+static const gchar default_sign_headers[] = ""
+ "(o)from:(x)sender:(o)reply-to:(o)subject:(x)date:(x)message-id:"
+ "(o)to:(o)cc:(x)mime-version:(x)content-type:(x)content-transfer-encoding:"
+ "resent-to:resent-cc:resent-from:resent-sender:resent-message-id:"
+ "(x)in-reply-to:(x)references:list-id:list-help:list-owner:list-unsubscribe:"
+ "list-unsubscribe-post:list-subscribe:list-post:(x)openpgp:(x)autocrypt";
+static const gchar default_arc_sign_headers[] = ""
+ "(o)from:(x)sender:(o)reply-to:(o)subject:(x)date:(x)message-id:"
+ "(o)to:(o)cc:(x)mime-version:(x)content-type:(x)content-transfer-encoding:"
+ "resent-to:resent-cc:resent-from:resent-sender:resent-message-id:"
+ "(x)in-reply-to:(x)references:list-id:list-help:list-owner:list-unsubscribe:"
+ "list-unsubscribe-post:list-subscribe:list-post:dkim-signature:(x)openpgp:"
+ "(x)autocrypt";
+
+struct dkim_ctx {
+ struct module_ctx ctx;
+ const gchar *symbol_reject;
+ const gchar *symbol_tempfail;
+ const gchar *symbol_allow;
+ const gchar *symbol_na;
+ const gchar *symbol_permfail;
+
+ struct rspamd_radix_map_helper *whitelist_ip;
+ struct rspamd_hash_map_helper *dkim_domains;
+ guint strict_multiplier;
+ guint time_jitter;
+ rspamd_lru_hash_t *dkim_hash;
+ rspamd_lru_hash_t *dkim_sign_hash;
+ const gchar *sign_headers;
+ const gchar *arc_sign_headers;
+ guint max_sigs;
+ gboolean trusted_only;
+ gboolean check_local;
+ gboolean check_authed;
+};
+
+struct dkim_check_result {
+ rspamd_dkim_context_t *ctx;
+ rspamd_dkim_key_t *key;
+ struct rspamd_task *task;
+ struct rspamd_dkim_check_result *res;
+ gdouble mult_allow;
+ gdouble mult_deny;
+ struct rspamd_symcache_dynamic_item *item;
+ struct dkim_check_result *next, *prev, *first;
+};
+
+static void dkim_symbol_callback(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ void *unused);
+
+static gint lua_dkim_sign_handler(lua_State *L);
+static gint lua_dkim_verify_handler(lua_State *L);
+static gint lua_dkim_canonicalize_handler(lua_State *L);
+
+/* Initialization */
+gint dkim_module_init(struct rspamd_config *cfg, struct module_ctx **ctx);
+gint dkim_module_config(struct rspamd_config *cfg, bool validate);
+gint dkim_module_reconfig(struct rspamd_config *cfg);
+
+module_t dkim_module = {
+ "dkim",
+ dkim_module_init,
+ dkim_module_config,
+ dkim_module_reconfig,
+ NULL,
+ RSPAMD_MODULE_VER,
+ (guint) -1,
+};
+
+static inline struct dkim_ctx *
+dkim_get_context(struct rspamd_config *cfg)
+{
+ return (struct dkim_ctx *) g_ptr_array_index(cfg->c_modules,
+ dkim_module.ctx_offset);
+}
+
+static void
+dkim_module_key_dtor(gpointer k)
+{
+ rspamd_dkim_key_t *key = k;
+
+ rspamd_dkim_key_unref(key);
+}
+
+static void
+dkim_module_free_list(gpointer k)
+{
+ g_list_free_full((GList *) k, rspamd_gstring_free_hard);
+}
+
+gint dkim_module_init(struct rspamd_config *cfg, struct module_ctx **ctx)
+{
+ struct dkim_ctx *dkim_module_ctx;
+
+ dkim_module_ctx = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(*dkim_module_ctx));
+ dkim_module_ctx->sign_headers = default_sign_headers;
+ dkim_module_ctx->arc_sign_headers = default_arc_sign_headers;
+ dkim_module_ctx->max_sigs = DEFAULT_MAX_SIGS;
+
+ *ctx = (struct module_ctx *) dkim_module_ctx;
+
+ rspamd_rcl_add_doc_by_path(cfg,
+ NULL,
+ "DKIM check plugin",
+ "dkim",
+ UCL_OBJECT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Map of IP addresses that should be excluded from DKIM checks",
+ "whitelist",
+ UCL_STRING,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Symbol that is added if DKIM check is successful",
+ "symbol_allow",
+ UCL_STRING,
+ NULL,
+ 0,
+ DEFAULT_SYMBOL_ALLOW,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Symbol that is added if DKIM check is unsuccessful",
+ "symbol_reject",
+ UCL_STRING,
+ NULL,
+ 0,
+ DEFAULT_SYMBOL_REJECT,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Symbol that is added if DKIM check can't be completed (e.g. DNS failure)",
+ "symbol_tempfail",
+ UCL_STRING,
+ NULL,
+ 0,
+ DEFAULT_SYMBOL_TEMPFAIL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Symbol that is added if mail is not signed",
+ "symbol_na",
+ UCL_STRING,
+ NULL,
+ 0,
+ DEFAULT_SYMBOL_NA,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Symbol that is added if permanent failure encountered",
+ "symbol_permfail",
+ UCL_STRING,
+ NULL,
+ 0,
+ DEFAULT_SYMBOL_PERMFAIL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Size of DKIM keys cache",
+ "dkim_cache_size",
+ UCL_INT,
+ NULL,
+ 0,
+ G_STRINGIFY(DEFAULT_CACHE_SIZE),
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Allow this time difference when checking DKIM signature time validity",
+ "time_jitter",
+ UCL_TIME,
+ NULL,
+ 0,
+ G_STRINGIFY(DEFAULT_TIME_JITTER),
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Domains to check DKIM for (check all domains if this option is empty)",
+ "domains",
+ UCL_STRING,
+ NULL,
+ 0,
+ "empty",
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Map of domains that are treated as 'trusted' meaning that DKIM policy failure has more significant score",
+ "trusted_domains",
+ UCL_STRING,
+ NULL,
+ 0,
+ "empty",
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Multiply dkim score by this factor for trusted domains",
+ "strict_multiplier",
+ UCL_FLOAT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Check DKIM policies merely for `trusted_domains`",
+ "trusted_only",
+ UCL_BOOLEAN,
+ NULL,
+ 0,
+ "false",
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Lua script that tells if a message should be signed and with what params (obsoleted)",
+ "sign_condition",
+ UCL_STRING,
+ NULL,
+ 0,
+ "empty",
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Obsoleted: maximum number of DKIM signatures to check",
+ "max_sigs",
+ UCL_INT,
+ NULL,
+ 0,
+ "n/a",
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "dkim",
+ "Headers used in signing",
+ "sign_headers",
+ UCL_STRING,
+ NULL,
+ 0,
+ default_sign_headers,
+ 0);
+
+ return 0;
+}
+
+gint dkim_module_config(struct rspamd_config *cfg, bool validate)
+{
+ const ucl_object_t *value;
+ gint res = TRUE, cb_id = -1;
+ guint cache_size, sign_cache_size;
+ gboolean got_trusted = FALSE;
+ struct dkim_ctx *dkim_module_ctx = dkim_get_context(cfg);
+
+ /* Register global methods */
+ lua_getglobal(cfg->lua_state, "rspamd_plugins");
+
+ if (lua_type(cfg->lua_state, -1) == LUA_TTABLE) {
+ lua_pushstring(cfg->lua_state, "dkim");
+ lua_createtable(cfg->lua_state, 0, 1);
+ /* Set methods */
+ lua_pushstring(cfg->lua_state, "sign");
+ lua_pushcfunction(cfg->lua_state, lua_dkim_sign_handler);
+ lua_settable(cfg->lua_state, -3);
+ lua_pushstring(cfg->lua_state, "verify");
+ lua_pushcfunction(cfg->lua_state, lua_dkim_verify_handler);
+ lua_settable(cfg->lua_state, -3);
+ lua_pushstring(cfg->lua_state, "canon_header_relaxed");
+ lua_pushcfunction(cfg->lua_state, lua_dkim_canonicalize_handler);
+ lua_settable(cfg->lua_state, -3);
+ /* Finish dkim key */
+ lua_settable(cfg->lua_state, -3);
+ }
+
+ lua_pop(cfg->lua_state, 1); /* Remove global function */
+ dkim_module_ctx->whitelist_ip = NULL;
+
+ value = rspamd_config_get_module_opt(cfg, "dkim", "check_local");
+
+ if (value == NULL) {
+ value = rspamd_config_get_module_opt(cfg, "options", "check_local");
+ }
+
+ if (value != NULL) {
+ dkim_module_ctx->check_local = ucl_object_toboolean(value);
+ }
+ else {
+ dkim_module_ctx->check_local = FALSE;
+ }
+
+ value = rspamd_config_get_module_opt(cfg, "dkim",
+ "check_authed");
+
+ if (value == NULL) {
+ value = rspamd_config_get_module_opt(cfg, "options",
+ "check_authed");
+ }
+
+ if (value != NULL) {
+ dkim_module_ctx->check_authed = ucl_object_toboolean(value);
+ }
+ else {
+ dkim_module_ctx->check_authed = FALSE;
+ }
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim", "symbol_reject")) != NULL) {
+ dkim_module_ctx->symbol_reject = ucl_object_tostring(value);
+ }
+ else {
+ dkim_module_ctx->symbol_reject = DEFAULT_SYMBOL_REJECT;
+ }
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim",
+ "symbol_tempfail")) != NULL) {
+ dkim_module_ctx->symbol_tempfail = ucl_object_tostring(value);
+ }
+ else {
+ dkim_module_ctx->symbol_tempfail = DEFAULT_SYMBOL_TEMPFAIL;
+ }
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim", "symbol_allow")) != NULL) {
+ dkim_module_ctx->symbol_allow = ucl_object_tostring(value);
+ }
+ else {
+ dkim_module_ctx->symbol_allow = DEFAULT_SYMBOL_ALLOW;
+ }
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim", "symbol_na")) != NULL) {
+ dkim_module_ctx->symbol_na = ucl_object_tostring(value);
+ }
+ else {
+ dkim_module_ctx->symbol_na = DEFAULT_SYMBOL_NA;
+ }
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim", "symbol_permfail")) != NULL) {
+ dkim_module_ctx->symbol_permfail = ucl_object_tostring(value);
+ }
+ else {
+ dkim_module_ctx->symbol_permfail = DEFAULT_SYMBOL_PERMFAIL;
+ }
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim",
+ "dkim_cache_size")) != NULL) {
+ cache_size = ucl_object_toint(value);
+ }
+ else {
+ cache_size = DEFAULT_CACHE_SIZE;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim",
+ "sign_cache_size")) != NULL) {
+ sign_cache_size = ucl_object_toint(value);
+ }
+ else {
+ sign_cache_size = 128;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim", "time_jitter")) != NULL) {
+ dkim_module_ctx->time_jitter = ucl_object_todouble(value);
+ }
+ else {
+ dkim_module_ctx->time_jitter = DEFAULT_TIME_JITTER;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim", "max_sigs")) != NULL) {
+ dkim_module_ctx->max_sigs = ucl_object_toint(value);
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim", "whitelist")) != NULL) {
+
+ rspamd_config_radix_from_ucl(cfg, value, "DKIM whitelist",
+ &dkim_module_ctx->whitelist_ip, NULL, NULL, "dkim whitelist");
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim", "domains")) != NULL) {
+ if (!rspamd_map_add_from_ucl(cfg, value,
+ "DKIM domains",
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &dkim_module_ctx->dkim_domains,
+ NULL, RSPAMD_MAP_DEFAULT)) {
+ msg_warn_config("cannot load dkim domains list from %s",
+ ucl_object_tostring(value));
+ }
+ else {
+ got_trusted = TRUE;
+ }
+ }
+
+ if (!got_trusted && (value =
+ rspamd_config_get_module_opt(cfg, "dkim", "trusted_domains")) != NULL) {
+ if (!rspamd_map_add_from_ucl(cfg, value,
+ "DKIM domains",
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &dkim_module_ctx->dkim_domains,
+ NULL, RSPAMD_MAP_DEFAULT)) {
+ msg_warn_config("cannot load dkim domains list from %s",
+ ucl_object_tostring(value));
+
+ if (validate) {
+ return FALSE;
+ }
+ }
+ else {
+ got_trusted = TRUE;
+ }
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim",
+ "strict_multiplier")) != NULL) {
+ dkim_module_ctx->strict_multiplier = ucl_object_toint(value);
+ }
+ else {
+ dkim_module_ctx->strict_multiplier = 1;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim", "trusted_only")) != NULL) {
+ dkim_module_ctx->trusted_only = ucl_object_toboolean(value);
+ }
+ else {
+ dkim_module_ctx->trusted_only = FALSE;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "dkim", "sign_headers")) != NULL) {
+ dkim_module_ctx->sign_headers = ucl_object_tostring(value);
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "arc", "sign_headers")) != NULL) {
+ dkim_module_ctx->arc_sign_headers = ucl_object_tostring(value);
+ }
+
+ if (cache_size > 0) {
+ dkim_module_ctx->dkim_hash = rspamd_lru_hash_new(
+ cache_size,
+ g_free,
+ dkim_module_key_dtor);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_lru_hash_destroy,
+ dkim_module_ctx->dkim_hash);
+ }
+
+ if (sign_cache_size > 0) {
+ dkim_module_ctx->dkim_sign_hash = rspamd_lru_hash_new(
+ sign_cache_size,
+ g_free,
+ (GDestroyNotify) rspamd_dkim_sign_key_unref);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_lru_hash_destroy,
+ dkim_module_ctx->dkim_sign_hash);
+ }
+
+ if (dkim_module_ctx->trusted_only && !got_trusted) {
+ msg_err_config("trusted_only option is set and no trusted domains are defined");
+ if (validate) {
+ return FALSE;
+ }
+ }
+ else {
+ if (!rspamd_config_is_module_enabled(cfg, "dkim")) {
+ return TRUE;
+ }
+
+ cb_id = rspamd_symcache_add_symbol(cfg->cache,
+ "DKIM_CHECK",
+ 0,
+ dkim_symbol_callback,
+ NULL,
+ SYMBOL_TYPE_CALLBACK,
+ -1);
+ rspamd_config_add_symbol(cfg,
+ "DKIM_CHECK",
+ 0.0,
+ "DKIM check callback",
+ "policies",
+ RSPAMD_SYMBOL_FLAG_IGNORE_METRIC,
+ 1,
+ 1);
+ rspamd_config_add_symbol_group(cfg, "DKIM_CHECK", "dkim");
+ rspamd_symcache_add_symbol(cfg->cache,
+ dkim_module_ctx->symbol_reject,
+ 0,
+ NULL,
+ NULL,
+ SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE,
+ cb_id);
+ rspamd_symcache_add_symbol(cfg->cache,
+ dkim_module_ctx->symbol_na,
+ 0,
+ NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE,
+ cb_id);
+ rspamd_symcache_add_symbol(cfg->cache,
+ dkim_module_ctx->symbol_permfail,
+ 0,
+ NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE,
+ cb_id);
+ rspamd_symcache_add_symbol(cfg->cache,
+ dkim_module_ctx->symbol_tempfail,
+ 0,
+ NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE,
+ cb_id);
+ rspamd_symcache_add_symbol(cfg->cache,
+ dkim_module_ctx->symbol_allow,
+ 0,
+ NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE,
+ cb_id);
+
+ rspamd_symcache_add_symbol(cfg->cache,
+ "DKIM_TRACE",
+ 0,
+ NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_NOSTAT,
+ cb_id);
+ rspamd_config_add_symbol(cfg,
+ "DKIM_TRACE",
+ 0.0,
+ "DKIM trace symbol",
+ "policies",
+ RSPAMD_SYMBOL_FLAG_IGNORE_METRIC,
+ 1,
+ 1);
+ rspamd_config_add_symbol_group(cfg, "DKIM_TRACE", "dkim");
+
+ msg_info_config("init internal dkim module");
+#ifndef HAVE_OPENSSL
+ msg_warn_config(
+ "openssl is not found so dkim rsa check is disabled, only check body hash, it is NOT safe to trust these results");
+#endif
+ }
+
+ return res;
+}
+
+
+/**
+ * Grab a private key from the cache
+ * or from the key content provided
+ */
+rspamd_dkim_sign_key_t *
+dkim_module_load_key_format(struct rspamd_task *task,
+ struct dkim_ctx *dkim_module_ctx,
+ const gchar *key, gsize keylen,
+ enum rspamd_dkim_key_format key_format)
+
+{
+ guchar h[rspamd_cryptobox_HASHBYTES],
+ hex_hash[rspamd_cryptobox_HASHBYTES * 2 + 1];
+ rspamd_dkim_sign_key_t *ret = NULL;
+ GError *err = NULL;
+ struct stat st;
+
+ memset(hex_hash, 0, sizeof(hex_hash));
+ rspamd_cryptobox_hash(h, key, keylen, NULL, 0);
+ rspamd_encode_hex_buf(h, sizeof(h), hex_hash, sizeof(hex_hash));
+
+ if (dkim_module_ctx->dkim_sign_hash) {
+ ret = rspamd_lru_hash_lookup(dkim_module_ctx->dkim_sign_hash,
+ hex_hash, time(NULL));
+ }
+
+ /*
+ * This fails for paths that are also valid base64.
+ * Maybe the caller should have specified a format.
+ */
+ if (key_format == RSPAMD_DKIM_KEY_UNKNOWN) {
+ if (key[0] == '.' || key[0] == '/') {
+ if (!rspamd_cryptobox_base64_is_valid(key, keylen)) {
+ key_format = RSPAMD_DKIM_KEY_FILE;
+ }
+ }
+ else if (rspamd_cryptobox_base64_is_valid(key, keylen)) {
+ key_format = RSPAMD_DKIM_KEY_BASE64;
+ }
+ }
+
+
+ if (ret != NULL && key_format == RSPAMD_DKIM_KEY_FILE) {
+ msg_debug_task("checking for stale file key");
+
+ if (stat(key, &st) != 0) {
+ msg_err_task("cannot stat key file: %s", strerror(errno));
+ return NULL;
+ }
+
+ if (rspamd_dkim_sign_key_maybe_invalidate(ret, st.st_mtime)) {
+ msg_debug_task("removing stale file key");
+ /*
+ * Invalidate DKIM key
+ * removal from lru cache also cleanup the key and value
+ */
+ if (dkim_module_ctx->dkim_sign_hash) {
+ rspamd_lru_hash_remove(dkim_module_ctx->dkim_sign_hash,
+ hex_hash);
+ }
+ ret = NULL;
+ }
+ }
+
+ /* found key; done */
+ if (ret != NULL) {
+ return ret;
+ }
+
+ ret = rspamd_dkim_sign_key_load(key, keylen, key_format, &err);
+
+ if (ret == NULL) {
+ msg_err_task("cannot load dkim key %s: %e",
+ key, err);
+ g_error_free(err);
+ }
+ else if (dkim_module_ctx->dkim_sign_hash) {
+ rspamd_lru_hash_insert(dkim_module_ctx->dkim_sign_hash,
+ g_strdup(hex_hash), ret, time(NULL), 0);
+ }
+
+ return ret;
+}
+
+static gint
+lua_dkim_sign_handler(lua_State *L)
+{
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gint64 arc_idx = 0, expire = 0;
+ enum rspamd_dkim_type sign_type = RSPAMD_DKIM_NORMAL;
+ GError *err = NULL;
+ GString *hdr;
+ GList *sigs = NULL;
+ const gchar *selector = NULL, *domain = NULL, *key = NULL, *rawkey = NULL,
+ *headers = NULL, *sign_type_str = NULL, *arc_cv = NULL,
+ *pubkey = NULL;
+ rspamd_dkim_sign_context_t *ctx;
+ rspamd_dkim_sign_key_t *dkim_key;
+ gsize rawlen = 0, keylen = 0;
+ gboolean no_cache = FALSE, strict_pubkey_check = FALSE;
+ struct dkim_ctx *dkim_module_ctx;
+
+ luaL_argcheck(L, lua_type(L, 2) == LUA_TTABLE, 2, "'table' expected");
+ /*
+ * Get the following elements:
+ * - selector
+ * - domain
+ * - key
+ */
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "key=V;rawkey=V;*domain=S;*selector=S;no_cache=B;headers=S;"
+ "sign_type=S;arc_idx=I;arc_cv=S;expire=I;pubkey=S;"
+ "strict_pubkey_check=B",
+ &keylen, &key, &rawlen, &rawkey, &domain,
+ &selector, &no_cache, &headers,
+ &sign_type_str, &arc_idx, &arc_cv, &expire, &pubkey,
+ &strict_pubkey_check)) {
+ msg_err_task("cannot parse table arguments: %e",
+ err);
+ g_error_free(err);
+
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+
+ dkim_module_ctx = dkim_get_context(task->cfg);
+
+ if (key) {
+ dkim_key = dkim_module_load_key_format(task, dkim_module_ctx, key,
+ keylen, RSPAMD_DKIM_KEY_UNKNOWN);
+ }
+ else if (rawkey) {
+ dkim_key = dkim_module_load_key_format(task, dkim_module_ctx, rawkey,
+ rawlen, RSPAMD_DKIM_KEY_UNKNOWN);
+ }
+ else {
+ msg_err_task("neither key nor rawkey are specified");
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+
+ if (dkim_key == NULL) {
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+
+ if (sign_type_str) {
+ if (strcmp(sign_type_str, "dkim") == 0) {
+ sign_type = RSPAMD_DKIM_NORMAL;
+
+ if (headers == NULL) {
+ headers = dkim_module_ctx->sign_headers;
+ }
+ }
+ else if (strcmp(sign_type_str, "arc-sign") == 0) {
+ sign_type = RSPAMD_DKIM_ARC_SIG;
+
+ if (headers == NULL) {
+ headers = dkim_module_ctx->arc_sign_headers;
+ }
+
+ if (arc_idx == 0) {
+ lua_settop(L, 0);
+ return luaL_error(L, "no arc idx specified");
+ }
+ }
+ else if (strcmp(sign_type_str, "arc-seal") == 0) {
+ sign_type = RSPAMD_DKIM_ARC_SEAL;
+ if (arc_cv == NULL) {
+ lua_settop(L, 0);
+ return luaL_error(L, "no arc cv specified");
+ }
+ if (arc_idx == 0) {
+ lua_settop(L, 0);
+ return luaL_error(L, "no arc idx specified");
+ }
+ }
+ else {
+ lua_settop(L, 0);
+ return luaL_error(L, "unknown sign type: %s",
+ sign_type_str);
+ }
+ }
+ else {
+ /* Unspecified sign type, assume plain dkim */
+ if (headers == NULL) {
+ headers = dkim_module_ctx->sign_headers;
+ }
+ }
+
+ if (pubkey != NULL) {
+ /* Also check if private and public keys match */
+ rspamd_dkim_key_t *pk;
+ keylen = strlen(pubkey);
+
+ pk = rspamd_dkim_parse_key(pubkey, &keylen, NULL);
+
+ if (pk == NULL) {
+ if (strict_pubkey_check) {
+ msg_err_task("cannot parse pubkey from string: %s, skip signing",
+ pubkey);
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+ else {
+ msg_warn_task("cannot parse pubkey from string: %s",
+ pubkey);
+ }
+ }
+ else {
+ GError *te = NULL;
+
+ /* We have parsed the key, so try to check keys */
+ if (!rspamd_dkim_match_keys(pk, dkim_key, &te)) {
+ if (strict_pubkey_check) {
+ msg_err_task("public key for %s/%s does not match private "
+ "key: %e, skip signing",
+ domain, selector, te);
+ g_error_free(te);
+ lua_pushboolean(L, FALSE);
+ rspamd_dkim_key_unref(pk);
+
+ return 1;
+ }
+ else {
+ msg_warn_task("public key for %s/%s does not match private "
+ "key: %e",
+ domain, selector, te);
+ g_error_free(te);
+ }
+ }
+
+ rspamd_dkim_key_unref(pk);
+ }
+ }
+
+ ctx = rspamd_create_dkim_sign_context(task, dkim_key,
+ DKIM_CANON_RELAXED, DKIM_CANON_RELAXED,
+ headers, sign_type, &err);
+
+ if (ctx == NULL) {
+ msg_err_task("cannot create sign context: %e",
+ err);
+ g_error_free(err);
+
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+
+ hdr = rspamd_dkim_sign(task, selector, domain, 0,
+ expire, arc_idx, arc_cv, ctx);
+
+ if (hdr) {
+
+ if (!no_cache) {
+ sigs = rspamd_mempool_get_variable(task->task_pool, "dkim-signature");
+
+ if (sigs == NULL) {
+ sigs = g_list_append(sigs, hdr);
+ rspamd_mempool_set_variable(task->task_pool, "dkim-signature",
+ sigs, dkim_module_free_list);
+ }
+ else {
+ sigs = g_list_append(sigs, hdr);
+ (void) sigs;
+ }
+ }
+
+ lua_pushboolean(L, TRUE);
+ lua_pushlstring(L, hdr->str, hdr->len);
+
+ if (no_cache) {
+ g_string_free(hdr, TRUE);
+ }
+
+ return 2;
+ }
+
+
+ lua_pushboolean(L, FALSE);
+ lua_pushnil(L);
+
+ return 2;
+}
+
+gint dkim_module_reconfig(struct rspamd_config *cfg)
+{
+ return dkim_module_config(cfg, false);
+}
+
+/*
+ * Parse strict value for domain in format: 'reject_multiplier:deny_multiplier'
+ */
+static gboolean
+dkim_module_parse_strict(const gchar *value, gdouble *allow, gdouble *deny)
+{
+ const gchar *colon;
+ gchar *err = NULL;
+ gdouble val;
+ gchar numbuf[64];
+
+ colon = strchr(value, ':');
+ if (colon) {
+ rspamd_strlcpy(numbuf, value,
+ MIN(sizeof(numbuf), (colon - value) + 1));
+ val = strtod(numbuf, &err);
+
+ if (err == NULL || *err == '\0') {
+ *deny = val;
+ colon++;
+ rspamd_strlcpy(numbuf, colon, sizeof(numbuf));
+ err = NULL;
+ val = strtod(numbuf, &err);
+
+ if (err == NULL || *err == '\0') {
+ *allow = val;
+ return TRUE;
+ }
+ }
+ }
+ return FALSE;
+}
+
+static void
+dkim_module_check(struct dkim_check_result *res)
+{
+ gboolean all_done = TRUE;
+ const gchar *strict_value;
+ struct dkim_check_result *first, *cur = NULL;
+ struct dkim_ctx *dkim_module_ctx = dkim_get_context(res->task->cfg);
+ struct rspamd_task *task = res->task;
+
+ first = res->first;
+
+ DL_FOREACH(first, cur)
+ {
+ if (cur->ctx == NULL) {
+ continue;
+ }
+
+ if (cur->key != NULL && cur->res == NULL) {
+ cur->res = rspamd_dkim_check(cur->ctx, cur->key, task);
+
+ if (dkim_module_ctx->dkim_domains != NULL) {
+ /* Perform strict check */
+ const gchar *domain = rspamd_dkim_get_domain(cur->ctx);
+
+ if ((strict_value =
+ rspamd_match_hash_map(dkim_module_ctx->dkim_domains,
+ domain,
+ strlen(domain))) != NULL) {
+ if (!dkim_module_parse_strict(strict_value, &cur->mult_allow,
+ &cur->mult_deny)) {
+ cur->mult_allow = dkim_module_ctx->strict_multiplier;
+ cur->mult_deny = dkim_module_ctx->strict_multiplier;
+ }
+ }
+ }
+ }
+ }
+
+ DL_FOREACH(first, cur)
+ {
+ if (cur->ctx == NULL) {
+ continue;
+ }
+ if (cur->res == NULL) {
+ /* Still need a key */
+ all_done = FALSE;
+ }
+ }
+
+ if (all_done) {
+ /* Create zero terminated array of results */
+ struct rspamd_dkim_check_result **pres;
+ guint nres = 0, i = 0;
+
+ DL_FOREACH(first, cur)
+ {
+ if (cur->ctx == NULL || cur->res == NULL) {
+ continue;
+ }
+
+ nres++;
+ }
+
+ pres = rspamd_mempool_alloc(task->task_pool, sizeof(*pres) * (nres + 1));
+ pres[nres] = NULL;
+
+ DL_FOREACH(first, cur)
+ {
+ const gchar *symbol = NULL, *trace = NULL;
+ gdouble symbol_weight = 1.0;
+
+ if (cur->ctx == NULL || cur->res == NULL) {
+ continue;
+ }
+
+ pres[i++] = cur->res;
+
+ if (cur->res->rcode == DKIM_REJECT) {
+ symbol = dkim_module_ctx->symbol_reject;
+ trace = "-";
+ symbol_weight = cur->mult_deny * 1.0;
+ }
+ else if (cur->res->rcode == DKIM_CONTINUE) {
+ symbol = dkim_module_ctx->symbol_allow;
+ trace = "+";
+ symbol_weight = cur->mult_allow * 1.0;
+ }
+ else if (cur->res->rcode == DKIM_PERM_ERROR) {
+ trace = "~";
+ symbol = dkim_module_ctx->symbol_permfail;
+ }
+ else if (cur->res->rcode == DKIM_TRYAGAIN) {
+ trace = "?";
+ symbol = dkim_module_ctx->symbol_tempfail;
+ }
+
+ if (symbol != NULL) {
+ const gchar *domain = rspamd_dkim_get_domain(cur->ctx);
+ const gchar *selector = rspamd_dkim_get_selector(cur->ctx);
+ gsize tracelen;
+ gchar *tracebuf;
+
+ tracelen = strlen(domain) + strlen(selector) + 4;
+ tracebuf = rspamd_mempool_alloc(task->task_pool,
+ tracelen);
+ rspamd_snprintf(tracebuf, tracelen, "%s:%s", domain, trace);
+
+ rspamd_task_insert_result(cur->task,
+ "DKIM_TRACE",
+ 0.0,
+ tracebuf);
+
+ rspamd_snprintf(tracebuf, tracelen, "%s:s=%s", domain, selector);
+ rspamd_task_insert_result(task,
+ symbol,
+ symbol_weight,
+ tracebuf);
+ }
+ }
+
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_DKIM_CHECK_RESULTS,
+ pres, NULL);
+ }
+}
+
+static void
+dkim_module_key_handler(rspamd_dkim_key_t *key,
+ gsize keylen,
+ rspamd_dkim_context_t *ctx,
+ gpointer ud,
+ GError *err)
+{
+ struct dkim_check_result *res = ud;
+ struct rspamd_task *task;
+ struct dkim_ctx *dkim_module_ctx;
+
+ task = res->task;
+ dkim_module_ctx = dkim_get_context(task->cfg);
+
+ if (key != NULL) {
+ /* Another ref belongs to the check context */
+ res->key = rspamd_dkim_key_ref(key);
+ /*
+ * We actually receive key with refcount = 1, so we just assume that
+ * lru hash owns this object now
+ */
+ /* Release key when task is processed */
+ rspamd_mempool_add_destructor(res->task->task_pool,
+ dkim_module_key_dtor, res->key);
+
+ if (dkim_module_ctx->dkim_hash) {
+ rspamd_lru_hash_insert(dkim_module_ctx->dkim_hash,
+ g_strdup(rspamd_dkim_get_dns_key(ctx)),
+ key, res->task->task_timestamp, rspamd_dkim_key_get_ttl(key));
+
+ msg_info_task("stored DKIM key for %s in LRU cache for %d seconds, "
+ "%d/%d elements in the cache",
+ rspamd_dkim_get_dns_key(ctx),
+ rspamd_dkim_key_get_ttl(key),
+ rspamd_lru_hash_size(dkim_module_ctx->dkim_hash),
+ rspamd_lru_hash_capacity(dkim_module_ctx->dkim_hash));
+ }
+ }
+ else {
+ /* Insert tempfail symbol */
+ msg_info_task("cannot get key for domain %s: %e",
+ rspamd_dkim_get_dns_key(ctx), err);
+
+ if (err != NULL) {
+ if (err->code == DKIM_SIGERROR_NOKEY) {
+ res->res = rspamd_dkim_create_result(ctx, DKIM_TRYAGAIN, task);
+ res->res->fail_reason = "DNS error when getting key";
+ }
+ else {
+ res->res = rspamd_dkim_create_result(ctx, DKIM_PERM_ERROR, task);
+ res->res->fail_reason = "invalid DKIM record";
+ }
+ }
+ }
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ dkim_module_check(res);
+}
+
+static void
+dkim_symbol_callback(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ void *unused)
+{
+ rspamd_dkim_context_t *ctx;
+ rspamd_dkim_key_t *key;
+ GError *err = NULL;
+ struct rspamd_mime_header *rh, *rh_cur;
+ struct dkim_check_result *res = NULL, *cur;
+ guint checked = 0;
+ gdouble *dmarc_checks;
+ struct dkim_ctx *dkim_module_ctx = dkim_get_context(task->cfg);
+
+ /* Allow dmarc */
+ dmarc_checks = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_DMARC_CHECKS);
+
+ if (dmarc_checks) {
+ (*dmarc_checks)++;
+ }
+ else {
+ dmarc_checks = rspamd_mempool_alloc(task->task_pool,
+ sizeof(*dmarc_checks));
+ *dmarc_checks = 1;
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_DMARC_CHECKS,
+ dmarc_checks, NULL);
+ }
+
+ /* First check if plugin should be enabled */
+ if ((!dkim_module_ctx->check_authed && task->auth_user != NULL) || (!dkim_module_ctx->check_local &&
+ rspamd_ip_is_local_cfg(task->cfg, task->from_addr))) {
+ msg_info_task("skip DKIM checks for local networks and authorized users");
+ rspamd_symcache_finalize_item(task, item);
+
+ return;
+ }
+ /* Check whitelist */
+ if (rspamd_match_radix_map_addr(dkim_module_ctx->whitelist_ip,
+ task->from_addr) != NULL) {
+ msg_info_task("skip DKIM checks for whitelisted address");
+ rspamd_symcache_finalize_item(task, item);
+
+ return;
+ }
+
+ rspamd_symcache_item_async_inc(task, item, M);
+
+ /* Now check if a message has its signature */
+ rh = rspamd_message_get_header_array(task, RSPAMD_DKIM_SIGNHEADER, FALSE);
+ if (rh) {
+ msg_debug_task("dkim signature found");
+
+ DL_FOREACH(rh, rh_cur)
+ {
+ if (rh_cur->decoded == NULL || rh_cur->decoded[0] == '\0') {
+ msg_info_task("cannot load empty DKIM signature");
+ continue;
+ }
+
+ cur = rspamd_mempool_alloc0(task->task_pool, sizeof(*cur));
+ cur->first = res;
+ cur->res = NULL;
+ cur->task = task;
+ cur->mult_allow = 1.0;
+ cur->mult_deny = 1.0;
+ cur->item = item;
+
+ ctx = rspamd_create_dkim_context(rh_cur->decoded,
+ task->task_pool,
+ task->resolver,
+ dkim_module_ctx->time_jitter,
+ RSPAMD_DKIM_NORMAL,
+ &err);
+
+ if (res == NULL) {
+ res = cur;
+ res->first = res;
+ res->prev = res;
+ }
+ else {
+ DL_APPEND(res, cur);
+ }
+
+ if (ctx == NULL) {
+ if (err != NULL) {
+ msg_info_task("cannot parse DKIM signature: %e",
+ err);
+ g_error_free(err);
+ err = NULL;
+ }
+ else {
+ msg_info_task("cannot parse DKIM signature: "
+ "unknown error");
+ }
+
+ continue;
+ }
+ else {
+ /* Get key */
+ cur->ctx = ctx;
+ const gchar *domain = rspamd_dkim_get_domain(cur->ctx);
+
+ if (dkim_module_ctx->trusted_only &&
+ (dkim_module_ctx->dkim_domains == NULL ||
+ rspamd_match_hash_map(dkim_module_ctx->dkim_domains,
+ domain, strlen(domain)) == NULL)) {
+ msg_debug_task("skip dkim check for %s domain",
+ rspamd_dkim_get_domain(ctx));
+
+ continue;
+ }
+
+ if (dkim_module_ctx->dkim_hash) {
+ key = rspamd_lru_hash_lookup(dkim_module_ctx->dkim_hash,
+ rspamd_dkim_get_dns_key(ctx),
+ task->task_timestamp);
+ }
+ else {
+ key = NULL;
+ }
+
+ if (key != NULL) {
+ cur->key = rspamd_dkim_key_ref(key);
+ /* Release key when task is processed */
+ rspamd_mempool_add_destructor(task->task_pool,
+ dkim_module_key_dtor, cur->key);
+ }
+ else {
+ if (!rspamd_get_dkim_key(ctx,
+ task,
+ dkim_module_key_handler,
+ cur)) {
+ continue;
+ }
+ }
+ }
+
+ checked++;
+
+ if (checked > dkim_module_ctx->max_sigs) {
+ msg_info_task("message has multiple signatures but we"
+ " stopped after %d checked signatures as limit"
+ " is reached",
+ checked);
+ break;
+ }
+ }
+ }
+ else {
+ rspamd_task_insert_result(task,
+ dkim_module_ctx->symbol_na,
+ 1.0,
+ NULL);
+ }
+
+ if (res != NULL) {
+ dkim_module_check(res);
+ }
+
+ rspamd_symcache_item_async_dec_check(task, item, M);
+}
+
+struct rspamd_dkim_lua_verify_cbdata {
+ rspamd_dkim_context_t *ctx;
+ struct rspamd_task *task;
+ lua_State *L;
+ rspamd_dkim_key_t *key;
+ gint cbref;
+};
+
+static void
+dkim_module_lua_push_verify_result(struct rspamd_dkim_lua_verify_cbdata *cbd,
+ struct rspamd_dkim_check_result *res, GError *err)
+{
+ struct rspamd_task **ptask, *task;
+ const gchar *error_str = "unknown error";
+ gboolean success = FALSE;
+
+ task = cbd->task;
+
+ switch (res->rcode) {
+ case DKIM_CONTINUE:
+ error_str = NULL;
+ success = TRUE;
+ break;
+ case DKIM_REJECT:
+ if (err) {
+ error_str = err->message;
+ }
+ else {
+ error_str = "reject";
+ }
+ break;
+ case DKIM_TRYAGAIN:
+ if (err) {
+ error_str = err->message;
+ }
+ else {
+ error_str = "tempfail";
+ }
+ break;
+ case DKIM_NOTFOUND:
+ if (err) {
+ error_str = err->message;
+ }
+ else {
+ error_str = "not found";
+ }
+ break;
+ case DKIM_RECORD_ERROR:
+ if (err) {
+ error_str = err->message;
+ }
+ else {
+ error_str = "bad record";
+ }
+ break;
+ case DKIM_PERM_ERROR:
+ if (err) {
+ error_str = err->message;
+ }
+ else {
+ error_str = "permanent error";
+ }
+ break;
+ default:
+ break;
+ }
+
+ lua_rawgeti(cbd->L, LUA_REGISTRYINDEX, cbd->cbref);
+ ptask = lua_newuserdata(cbd->L, sizeof(*ptask));
+ *ptask = task;
+ lua_pushboolean(cbd->L, success);
+
+ if (error_str) {
+ lua_pushstring(cbd->L, error_str);
+ }
+ else {
+ lua_pushnil(cbd->L);
+ }
+
+ if (cbd->ctx) {
+ if (res->domain) {
+ lua_pushstring(cbd->L, res->domain);
+ }
+ else {
+ lua_pushnil(cbd->L);
+ }
+
+ if (res->selector) {
+ lua_pushstring(cbd->L, res->selector);
+ }
+ else {
+ lua_pushnil(cbd->L);
+ }
+
+ if (res->short_b) {
+ lua_pushstring(cbd->L, res->short_b);
+ }
+ else {
+ lua_pushnil(cbd->L);
+ }
+
+ if (res->fail_reason) {
+ lua_pushstring(cbd->L, res->fail_reason);
+ }
+ else {
+ lua_pushnil(cbd->L);
+ }
+ }
+ else {
+ lua_pushnil(cbd->L);
+ lua_pushnil(cbd->L);
+ lua_pushnil(cbd->L);
+ lua_pushnil(cbd->L);
+ }
+
+ if (lua_pcall(cbd->L, 7, 0, 0) != 0) {
+ msg_err_task("call to verify callback failed: %s",
+ lua_tostring(cbd->L, -1));
+ lua_pop(cbd->L, 1);
+ }
+
+ luaL_unref(cbd->L, LUA_REGISTRYINDEX, cbd->cbref);
+}
+
+static void
+dkim_module_lua_on_key(rspamd_dkim_key_t *key,
+ gsize keylen,
+ rspamd_dkim_context_t *ctx,
+ gpointer ud,
+ GError *err)
+{
+ struct rspamd_dkim_lua_verify_cbdata *cbd = ud;
+ struct rspamd_task *task;
+ struct rspamd_dkim_check_result *res;
+ struct dkim_ctx *dkim_module_ctx;
+
+ task = cbd->task;
+ dkim_module_ctx = dkim_get_context(task->cfg);
+
+ if (key != NULL) {
+ /* Another ref belongs to the check context */
+ cbd->key = rspamd_dkim_key_ref(key);
+ /*
+ * We actually receive key with refcount = 1, so we just assume that
+ * lru hash owns this object now
+ */
+
+ if (dkim_module_ctx->dkim_hash) {
+ rspamd_lru_hash_insert(dkim_module_ctx->dkim_hash,
+ g_strdup(rspamd_dkim_get_dns_key(ctx)),
+ key, cbd->task->task_timestamp, rspamd_dkim_key_get_ttl(key));
+ }
+ /* Release key when task is processed */
+ rspamd_mempool_add_destructor(cbd->task->task_pool,
+ dkim_module_key_dtor, cbd->key);
+ }
+ else {
+ /* Insert tempfail symbol */
+ msg_info_task("cannot get key for domain %s: %e",
+ rspamd_dkim_get_dns_key(ctx), err);
+
+ if (err != NULL) {
+ if (err->code == DKIM_SIGERROR_NOKEY) {
+ res = rspamd_dkim_create_result(ctx, DKIM_TRYAGAIN, task);
+ res->fail_reason = "DNS error when getting key";
+ }
+ else {
+ res = rspamd_dkim_create_result(ctx, DKIM_PERM_ERROR, task);
+ res->fail_reason = "invalid DKIM record";
+ }
+ }
+ else {
+ res = rspamd_dkim_create_result(ctx, DKIM_TRYAGAIN, task);
+ res->fail_reason = "DNS error when getting key";
+ }
+
+ dkim_module_lua_push_verify_result(cbd, res, err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return;
+ }
+
+ res = rspamd_dkim_check(cbd->ctx, cbd->key, cbd->task);
+ dkim_module_lua_push_verify_result(cbd, res, NULL);
+}
+
+static gint
+lua_dkim_verify_handler(lua_State *L)
+{
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *sig = luaL_checkstring(L, 2);
+ rspamd_dkim_context_t *ctx;
+ struct rspamd_dkim_lua_verify_cbdata *cbd;
+ rspamd_dkim_key_t *key;
+ struct rspamd_dkim_check_result *ret;
+ GError *err = NULL;
+ const gchar *type_str = NULL;
+ enum rspamd_dkim_type type = RSPAMD_DKIM_NORMAL;
+ struct dkim_ctx *dkim_module_ctx;
+
+ if (task && sig && lua_isfunction(L, 3)) {
+ if (lua_isstring(L, 4)) {
+ type_str = lua_tostring(L, 4);
+
+ if (type_str) {
+ if (strcmp(type_str, "dkim") == 0) {
+ type = RSPAMD_DKIM_NORMAL;
+ }
+ else if (strcmp(type_str, "arc-sign") == 0) {
+ type = RSPAMD_DKIM_ARC_SIG;
+ }
+ else if (strcmp(type_str, "arc-seal") == 0) {
+ type = RSPAMD_DKIM_ARC_SEAL;
+ }
+ else {
+ lua_settop(L, 0);
+ return luaL_error(L, "unknown sign type: %s",
+ type_str);
+ }
+ }
+ }
+
+ dkim_module_ctx = dkim_get_context(task->cfg);
+
+ ctx = rspamd_create_dkim_context(sig,
+ task->task_pool,
+ task->resolver,
+ dkim_module_ctx->time_jitter,
+ type,
+ &err);
+
+ if (ctx == NULL) {
+ lua_pushboolean(L, false);
+
+ if (err) {
+ lua_pushstring(L, err->message);
+ g_error_free(err);
+ }
+ else {
+ lua_pushstring(L, "unknown error");
+ }
+
+ return 2;
+ }
+
+ cbd = rspamd_mempool_alloc(task->task_pool, sizeof(*cbd));
+ cbd->L = L;
+ cbd->task = task;
+ lua_pushvalue(L, 3);
+ cbd->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ cbd->ctx = ctx;
+ cbd->key = NULL;
+
+ if (dkim_module_ctx->dkim_hash) {
+ key = rspamd_lru_hash_lookup(dkim_module_ctx->dkim_hash,
+ rspamd_dkim_get_dns_key(ctx),
+ task->task_timestamp);
+ }
+ else {
+ key = NULL;
+ }
+
+ if (key != NULL) {
+ cbd->key = rspamd_dkim_key_ref(key);
+ /* Release key when task is processed */
+ rspamd_mempool_add_destructor(task->task_pool,
+ dkim_module_key_dtor, cbd->key);
+ ret = rspamd_dkim_check(cbd->ctx, cbd->key, cbd->task);
+ dkim_module_lua_push_verify_result(cbd, ret, NULL);
+ }
+ else {
+ rspamd_get_dkim_key(ctx,
+ task,
+ dkim_module_lua_on_key,
+ cbd);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, TRUE);
+ lua_pushnil(L);
+
+ return 2;
+}
+
+static gint
+lua_dkim_canonicalize_handler(lua_State *L)
+{
+ gsize nlen, vlen;
+ const gchar *hname = luaL_checklstring(L, 1, &nlen),
+ *hvalue = luaL_checklstring(L, 2, &vlen);
+ static gchar st_buf[8192];
+ gchar *buf;
+ guint inlen;
+ gboolean allocated = FALSE;
+ goffset r;
+
+ if (hname && hvalue && nlen > 0) {
+ inlen = nlen + vlen + sizeof(":" CRLF);
+
+ if (inlen > sizeof(st_buf)) {
+ buf = g_malloc(inlen);
+ allocated = TRUE;
+ }
+ else {
+ /* Faster */
+ buf = st_buf;
+ }
+
+ r = rspamd_dkim_canonize_header_relaxed_str(hname, hvalue, buf, inlen);
+
+ if (r == -1) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushlstring(L, buf, r);
+ }
+
+ if (allocated) {
+ g_free(buf);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
new file mode 100644
index 0000000..85db83d
--- /dev/null
+++ b/src/plugins/fuzzy_check.c
@@ -0,0 +1,4695 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/***MODULE:fuzzy
+ * rspamd module that checks fuzzy checksums for messages
+ *
+ * Allowed options:
+ * - symbol (string): symbol to insert (default: 'R_FUZZY')
+ * - max_score (double): maximum score to that weights of hashes would be normalized (default: 0 - no normalization)
+ *
+ * - fuzzy_map (string): a string that contains map in format { fuzzy_key => [ symbol, weight ] } where fuzzy_key is number of
+ * fuzzy list. This string itself should be in format 1:R_FUZZY_SAMPLE1:10,2:R_FUZZY_SAMPLE2:1 etc, where first number is fuzzy
+ * key, second is symbol to insert and third - weight for normalization
+ *
+ * - min_length (integer): minimum length (in characters) for text part to be checked for fuzzy hash (default: 0 - no limit)
+ * - whitelist (map string): map of ip addresses that should not be checked with this module
+ * - servers (string): list of fuzzy servers in format "server1:port,server2:port" - these servers would be used for checking and storing
+ * fuzzy hashes
+ */
+
+#include "config.h"
+#include "libmime/message.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
+#include "libmime/images.h"
+#include "libserver/worker_util.h"
+#include "libserver/mempool_vars_internal.h"
+#include "fuzzy_wire.h"
+#include "utlist.h"
+#include "ottery.h"
+#include "lua/lua_common.h"
+#include "unix-std.h"
+#include "libserver/http/http_private.h"
+#include "libserver/http/http_router.h"
+#include "libstat/stat_api.h"
+#include <math.h>
+#include "libutil/libev_helper.h"
+
+#define DEFAULT_SYMBOL "R_FUZZY_HASH"
+
+#define DEFAULT_IO_TIMEOUT 1.0
+#define DEFAULT_RETRANSMITS 3
+#define DEFAULT_MAX_ERRORS 4
+#define DEFAULT_REVIVE_TIME 60
+#define DEFAULT_PORT 11335
+
+#define RSPAMD_FUZZY_PLUGIN_VERSION RSPAMD_FUZZY_VERSION
+
+static const gint rspamd_fuzzy_hash_len = 5;
+static const gchar *M = "fuzzy check";
+struct fuzzy_ctx;
+
+struct fuzzy_mapping {
+ guint64 fuzzy_flag;
+ const gchar *symbol;
+ double weight;
+};
+
+struct fuzzy_rule {
+ struct upstream_list *servers;
+ const gchar *symbol;
+ const gchar *algorithm_str;
+ const gchar *name;
+ const ucl_object_t *ucl_obj;
+ enum rspamd_shingle_alg alg;
+ GHashTable *mappings;
+ GPtrArray *fuzzy_headers;
+ GString *hash_key;
+ GString *shingles_key;
+ gdouble io_timeout;
+ struct rspamd_cryptobox_keypair *local_key;
+ struct rspamd_cryptobox_pubkey *peer_key;
+ double max_score;
+ double weight_threshold;
+ gboolean read_only;
+ gboolean skip_unknown;
+ gboolean no_share;
+ gboolean no_subject;
+ gint learn_condition_cb;
+ guint32 retransmits;
+ struct rspamd_hash_map_helper *skip_map;
+ struct fuzzy_ctx *ctx;
+ gint lua_id;
+};
+
+struct fuzzy_ctx {
+ struct module_ctx ctx;
+ rspamd_mempool_t *fuzzy_pool;
+ GPtrArray *fuzzy_rules;
+ struct rspamd_config *cfg;
+ const gchar *default_symbol;
+ struct rspamd_radix_map_helper *whitelist;
+ struct rspamd_keypair_cache *keypairs_cache;
+ guint max_errors;
+ gdouble revive_time;
+ gdouble io_timeout;
+ gint check_mime_part_ref; /* Lua callback */
+ gint process_rule_ref; /* Lua callback */
+ gint cleanup_rules_ref;
+ guint32 retransmits;
+ gboolean enabled;
+};
+
+enum fuzzy_result_type {
+ FUZZY_RESULT_TXT,
+ FUZZY_RESULT_IMG,
+ FUZZY_RESULT_CONTENT,
+ FUZZY_RESULT_BIN
+};
+
+struct fuzzy_client_result {
+ const gchar *symbol;
+ gchar *option;
+ gdouble score;
+ gdouble prob;
+ enum fuzzy_result_type type;
+};
+
+struct fuzzy_client_session {
+ GPtrArray *commands;
+ GPtrArray *results;
+ struct rspamd_task *task;
+ struct rspamd_symcache_dynamic_item *item;
+ struct upstream *server;
+ struct fuzzy_rule *rule;
+ struct ev_loop *event_loop;
+ struct rspamd_io_ev ev;
+ gint state;
+ gint fd;
+ guint retransmits;
+};
+
+struct fuzzy_learn_session {
+ GPtrArray *commands;
+ gint *saved;
+ struct {
+ const gchar *error_message;
+ gint error_code;
+ } err;
+ struct rspamd_http_connection_entry *http_entry;
+ struct rspamd_async_session *session;
+ struct upstream *server;
+ struct fuzzy_rule *rule;
+ struct rspamd_task *task;
+ struct ev_loop *event_loop;
+ struct rspamd_io_ev ev;
+ gint fd;
+ guint retransmits;
+};
+
+#define FUZZY_CMD_FLAG_REPLIED (1 << 0)
+#define FUZZY_CMD_FLAG_SENT (1 << 1)
+#define FUZZY_CMD_FLAG_IMAGE (1 << 2)
+#define FUZZY_CMD_FLAG_CONTENT (1 << 3)
+
+#define FUZZY_CHECK_FLAG_NOIMAGES (1 << 0)
+#define FUZZY_CHECK_FLAG_NOATTACHMENTS (1 << 1)
+#define FUZZY_CHECK_FLAG_NOTEXT (1 << 2)
+
+struct fuzzy_cmd_io {
+ guint32 tag;
+ guint32 flags;
+ struct iovec io;
+ struct rspamd_mime_part *part;
+ struct rspamd_fuzzy_cmd cmd;
+};
+
+
+static const char *default_headers = "Subject,Content-Type,Reply-To,X-Mailer";
+
+static void fuzzy_symbol_callback(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ void *unused);
+
+/* Initialization */
+gint fuzzy_check_module_init(struct rspamd_config *cfg,
+ struct module_ctx **ctx);
+gint fuzzy_check_module_config(struct rspamd_config *cfg, bool valdate);
+gint fuzzy_check_module_reconfig(struct rspamd_config *cfg);
+static gint fuzzy_attach_controller(struct module_ctx *ctx,
+ GHashTable *commands);
+static gint fuzzy_lua_learn_handler(lua_State *L);
+static gint fuzzy_lua_unlearn_handler(lua_State *L);
+static gint fuzzy_lua_gen_hashes_handler(lua_State *L);
+static gint fuzzy_lua_hex_hashes_handler(lua_State *L);
+static gint fuzzy_lua_list_storages(lua_State *L);
+static gint fuzzy_lua_ping_storage(lua_State *L);
+
+module_t fuzzy_check_module = {
+ "fuzzy_check",
+ fuzzy_check_module_init,
+ fuzzy_check_module_config,
+ fuzzy_check_module_reconfig,
+ fuzzy_attach_controller,
+ RSPAMD_MODULE_VER,
+ (guint) -1,
+};
+
+static inline struct fuzzy_ctx *
+fuzzy_get_context(struct rspamd_config *cfg)
+{
+ return (struct fuzzy_ctx *) g_ptr_array_index(cfg->c_modules,
+ fuzzy_check_module.ctx_offset);
+}
+
+static void
+parse_flags(struct fuzzy_rule *rule,
+ struct rspamd_config *cfg,
+ const ucl_object_t *val,
+ gint cb_id)
+{
+ const ucl_object_t *elt;
+ struct fuzzy_mapping *map;
+ const gchar *sym = NULL;
+
+ if (val->type == UCL_STRING) {
+ msg_err_config(
+ "string mappings are deprecated and no longer supported, use new style configuration");
+ }
+ else if (val->type == UCL_OBJECT) {
+ elt = ucl_object_lookup(val, "symbol");
+ if (elt == NULL || !ucl_object_tostring_safe(elt, &sym)) {
+ sym = ucl_object_key(val);
+ }
+ if (sym != NULL) {
+ map =
+ rspamd_mempool_alloc(cfg->cfg_pool,
+ sizeof(struct fuzzy_mapping));
+ map->symbol = sym;
+ elt = ucl_object_lookup(val, "flag");
+
+ if (elt != NULL) {
+ map->fuzzy_flag = ucl_obj_toint(elt);
+
+ elt = ucl_object_lookup(val, "max_score");
+
+ if (elt != NULL) {
+ map->weight = ucl_obj_todouble(elt);
+ }
+ else {
+ map->weight = rule->max_score;
+ }
+ /* Add flag to hash table */
+ g_hash_table_insert(rule->mappings,
+ GINT_TO_POINTER(map->fuzzy_flag), map);
+ rspamd_symcache_add_symbol(cfg->cache,
+ map->symbol, 0,
+ NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE,
+ cb_id);
+ }
+ else {
+ msg_err_config("fuzzy_map parameter has no flag definition");
+ }
+ }
+ else {
+ msg_err_config("fuzzy_map parameter has no symbol definition");
+ }
+ }
+ else {
+ msg_err_config("fuzzy_map parameter is of an unsupported type");
+ }
+}
+
+static GPtrArray *
+parse_fuzzy_headers(struct rspamd_config *cfg, const gchar *str)
+{
+ gchar **strvec;
+ gint num, i;
+ GPtrArray *res;
+
+ strvec = g_strsplit_set(str, ",", 0);
+ num = g_strv_length(strvec);
+ res = g_ptr_array_sized_new(num);
+
+ for (i = 0; i < num; i++) {
+ g_strstrip(strvec[i]);
+ g_ptr_array_add(res, rspamd_mempool_strdup(
+ cfg->cfg_pool, strvec[i]));
+ }
+
+ g_strfreev(strvec);
+
+ return res;
+}
+
+static double
+fuzzy_normalize(gint32 in, double weight)
+{
+ if (weight == 0) {
+ return 0;
+ }
+#ifdef HAVE_TANH
+ return tanh(G_E * (double) in / weight);
+#else
+ return (in < weight ? in / weight : weight);
+#endif
+}
+
+static struct fuzzy_rule *
+fuzzy_rule_new(const char *default_symbol, rspamd_mempool_t *pool)
+{
+ struct fuzzy_rule *rule;
+
+ rule = rspamd_mempool_alloc0(pool, sizeof(struct fuzzy_rule));
+
+ rule->mappings = g_hash_table_new(g_direct_hash, g_direct_equal);
+ rule->symbol = default_symbol;
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) g_hash_table_unref,
+ rule->mappings);
+ rule->read_only = FALSE;
+ rule->weight_threshold = NAN;
+
+ return rule;
+}
+
+static void
+fuzzy_free_rule(gpointer r)
+{
+ struct fuzzy_rule *rule = (struct fuzzy_rule *) r;
+
+ g_string_free(rule->hash_key, TRUE);
+ g_string_free(rule->shingles_key, TRUE);
+
+ if (rule->local_key) {
+ rspamd_keypair_unref(rule->local_key);
+ }
+
+ if (rule->peer_key) {
+ rspamd_pubkey_unref(rule->peer_key);
+ }
+}
+
+static gint
+fuzzy_parse_rule(struct rspamd_config *cfg, const ucl_object_t *obj,
+ const gchar *name, gint cb_id)
+{
+ const ucl_object_t *value, *cur;
+ struct fuzzy_rule *rule;
+ ucl_object_iter_t it = NULL;
+ const char *k = NULL, *key_str = NULL, *shingles_key_str = NULL, *lua_script;
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(cfg);
+
+ if (obj->type != UCL_OBJECT) {
+ msg_err_config("invalid rule definition");
+ return -1;
+ }
+
+ if ((value = ucl_object_lookup_any(obj, "enabled", "enable", NULL)) != NULL) {
+ if (!ucl_object_toboolean(value)) {
+ msg_info_config("fuzzy rule %s is disabled by configuration", name);
+
+ return 0;
+ }
+ }
+
+ rule = fuzzy_rule_new(fuzzy_module_ctx->default_symbol,
+ cfg->cfg_pool);
+ rule->ucl_obj = obj;
+ rule->ctx = fuzzy_module_ctx;
+ rule->learn_condition_cb = -1;
+ rule->alg = RSPAMD_SHINGLES_OLD;
+ rule->skip_map = NULL;
+
+ if ((value = ucl_object_lookup(obj, "skip_hashes")) != NULL) {
+ rspamd_map_add_from_ucl(cfg, value,
+ "Fuzzy hashes whitelist",
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &rule->skip_map,
+ NULL, RSPAMD_MAP_DEFAULT);
+ }
+
+ if ((value = ucl_object_lookup(obj, "headers")) != NULL) {
+ it = NULL;
+ while ((cur = ucl_object_iterate(value, &it, value->type == UCL_ARRAY)) != NULL) {
+ GPtrArray *tmp;
+ guint i;
+ gpointer ptr;
+
+ tmp = parse_fuzzy_headers(cfg, ucl_obj_tostring(cur));
+
+ if (tmp) {
+ if (rule->fuzzy_headers) {
+ PTR_ARRAY_FOREACH(tmp, i, ptr)
+ {
+ g_ptr_array_add(rule->fuzzy_headers, ptr);
+ }
+
+ g_ptr_array_free(tmp, TRUE);
+ }
+ else {
+ rule->fuzzy_headers = tmp;
+ }
+ }
+ }
+ }
+ else {
+ rule->fuzzy_headers = parse_fuzzy_headers(cfg, default_headers);
+ }
+
+ if (rule->fuzzy_headers != NULL) {
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_ptr_array_free_hard,
+ rule->fuzzy_headers);
+ }
+
+
+ if ((value = ucl_object_lookup(obj, "max_score")) != NULL) {
+ rule->max_score = ucl_obj_todouble(value);
+ }
+
+ if ((value = ucl_object_lookup(obj, "retransmits")) != NULL) {
+ rule->retransmits = ucl_obj_toint(value);
+ }
+ else {
+ rule->retransmits = fuzzy_module_ctx->retransmits;
+ }
+
+ if ((value = ucl_object_lookup(obj, "timeout")) != NULL) {
+ rule->io_timeout = ucl_obj_todouble(value);
+ }
+ else {
+ rule->io_timeout = fuzzy_module_ctx->io_timeout;
+ }
+
+ if ((value = ucl_object_lookup(obj, "symbol")) != NULL) {
+ rule->symbol = ucl_obj_tostring(value);
+ }
+
+ if (name) {
+ rule->name = name;
+ }
+ else {
+ rule->name = rule->symbol;
+ }
+
+
+ if ((value = ucl_object_lookup(obj, "read_only")) != NULL) {
+ rule->read_only = ucl_obj_toboolean(value);
+ }
+
+ if ((value = ucl_object_lookup(obj, "skip_unknown")) != NULL) {
+ rule->skip_unknown = ucl_obj_toboolean(value);
+ }
+
+ if ((value = ucl_object_lookup(obj, "no_share")) != NULL) {
+ rule->no_share = ucl_obj_toboolean(value);
+ }
+
+ if ((value = ucl_object_lookup(obj, "no_subject")) != NULL) {
+ rule->no_subject = ucl_obj_toboolean(value);
+ }
+
+ if ((value = ucl_object_lookup(obj, "algorithm")) != NULL) {
+ rule->algorithm_str = ucl_object_tostring(value);
+
+ if (rule->algorithm_str) {
+ if (g_ascii_strcasecmp(rule->algorithm_str, "old") == 0 ||
+ g_ascii_strcasecmp(rule->algorithm_str, "siphash") == 0) {
+ rule->alg = RSPAMD_SHINGLES_OLD;
+ }
+ else if (g_ascii_strcasecmp(rule->algorithm_str, "xxhash") == 0) {
+ rule->alg = RSPAMD_SHINGLES_XXHASH;
+ }
+ else if (g_ascii_strcasecmp(rule->algorithm_str, "mumhash") == 0) {
+ rule->alg = RSPAMD_SHINGLES_MUMHASH;
+ }
+ else if (g_ascii_strcasecmp(rule->algorithm_str, "fasthash") == 0 ||
+ g_ascii_strcasecmp(rule->algorithm_str, "fast") == 0) {
+ rule->alg = RSPAMD_SHINGLES_FAST;
+ }
+ else {
+ msg_warn_config("unknown algorithm: %s, use siphash by default",
+ rule->algorithm_str);
+ }
+ }
+ }
+
+ /* Set a consistent and short string name */
+ switch (rule->alg) {
+ case RSPAMD_SHINGLES_OLD:
+ rule->algorithm_str = "sip";
+ break;
+ case RSPAMD_SHINGLES_XXHASH:
+ rule->algorithm_str = "xx";
+ break;
+ case RSPAMD_SHINGLES_MUMHASH:
+ rule->algorithm_str = "mum";
+ break;
+ case RSPAMD_SHINGLES_FAST:
+ rule->algorithm_str = "fast";
+ break;
+ }
+
+ if ((value = ucl_object_lookup(obj, "servers")) != NULL) {
+ rule->servers = rspamd_upstreams_create(cfg->ups_ctx);
+ /* pass max_error and revive_time configuration in upstream for fuzzy storage
+ * it allows to configure error_rate threshold and upstream dead timer
+ */
+ rspamd_upstreams_set_limits(rule->servers,
+ (gdouble) fuzzy_module_ctx->revive_time, NAN, NAN, NAN,
+ (guint) fuzzy_module_ctx->max_errors, 0);
+
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_upstreams_destroy,
+ rule->servers);
+ if (!rspamd_upstreams_from_ucl(rule->servers, value, DEFAULT_PORT, NULL)) {
+ msg_err_config("cannot read servers definition");
+ return -1;
+ }
+ }
+ if ((value = ucl_object_lookup(obj, "fuzzy_map")) != NULL) {
+ it = NULL;
+ while ((cur = ucl_object_iterate(value, &it, true)) != NULL) {
+ parse_flags(rule, cfg, cur, cb_id);
+ }
+ }
+
+ if ((value = ucl_object_lookup(obj, "encryption_key")) != NULL) {
+ /* Create key from user's input */
+ k = ucl_object_tostring(value);
+
+ if (k == NULL || (rule->peer_key =
+ rspamd_pubkey_from_base32(k, 0, RSPAMD_KEYPAIR_KEX,
+ RSPAMD_CRYPTOBOX_MODE_25519)) == NULL) {
+ msg_err_config("bad encryption key value: %s",
+ k);
+ return -1;
+ }
+
+ rule->local_key = rspamd_keypair_new(RSPAMD_KEYPAIR_KEX,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+ }
+
+ if ((value = ucl_object_lookup(obj, "learn_condition")) != NULL) {
+ lua_script = ucl_object_tostring(value);
+
+ if (lua_script) {
+ if (luaL_dostring(cfg->lua_state, lua_script) != 0) {
+ msg_err_config("cannot execute lua script for fuzzy "
+ "learn condition: %s",
+ lua_tostring(cfg->lua_state, -1));
+ }
+ else {
+ if (lua_type(cfg->lua_state, -1) == LUA_TFUNCTION) {
+ rule->learn_condition_cb = luaL_ref(cfg->lua_state,
+ LUA_REGISTRYINDEX);
+ msg_info_config("loaded learn condition script for fuzzy rule:"
+ " %s",
+ rule->name);
+ }
+ else {
+ msg_err_config("lua script must return "
+ "function(task) and not %s",
+ lua_typename(cfg->lua_state,
+ lua_type(cfg->lua_state, -1)));
+ }
+ }
+ }
+ }
+
+ key_str = NULL;
+ if ((value = ucl_object_lookup(obj, "fuzzy_key")) != NULL) {
+ /* Create key from user's input */
+ key_str = ucl_object_tostring(value);
+ }
+
+ /* Setup keys */
+ if (key_str == NULL) {
+ /* Use some default key for all ops */
+ key_str = "rspamd";
+ }
+
+ rule->hash_key = g_string_sized_new(rspamd_cryptobox_HASHBYTES);
+ rspamd_cryptobox_hash(rule->hash_key->str, key_str, strlen(key_str), NULL, 0);
+ rule->hash_key->len = rspamd_cryptobox_HASHKEYBYTES;
+
+ shingles_key_str = NULL;
+ if ((value = ucl_object_lookup(obj, "fuzzy_shingles_key")) != NULL) {
+ shingles_key_str = ucl_object_tostring(value);
+ }
+ if (shingles_key_str == NULL) {
+ shingles_key_str = "rspamd";
+ }
+
+ rule->shingles_key = g_string_sized_new(rspamd_cryptobox_HASHBYTES);
+ rspamd_cryptobox_hash(rule->shingles_key->str, shingles_key_str,
+ strlen(shingles_key_str), NULL, 0);
+ rule->shingles_key->len = 16;
+
+ if (rspamd_upstreams_count(rule->servers) == 0) {
+ msg_err_config("no servers defined for fuzzy rule with name: %s",
+ rule->name);
+ return -1;
+ }
+ else {
+ g_ptr_array_add(fuzzy_module_ctx->fuzzy_rules, rule);
+
+ if (rule->symbol != fuzzy_module_ctx->default_symbol) {
+ int vid = rspamd_symcache_add_symbol(cfg->cache, rule->symbol,
+ 0,
+ NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL | SYMBOL_TYPE_FINE,
+ cb_id);
+
+ if (rule->io_timeout > 0) {
+ char timeout_buf[32];
+ rspamd_snprintf(timeout_buf, sizeof(timeout_buf), "%f",
+ rule->io_timeout);
+ rspamd_symcache_add_symbol_augmentation(cfg->cache,
+ vid, "timeout",
+ timeout_buf);
+ }
+ }
+
+ msg_info_config("added fuzzy rule %s, key: %*xs, "
+ "shingles_key: %*xs, algorithm: %s",
+ rule->symbol,
+ 6, rule->hash_key->str,
+ 6, rule->shingles_key->str,
+ rule->algorithm_str);
+ }
+
+ if ((value = ucl_object_lookup(obj, "weight_threshold")) != NULL) {
+ rule->weight_threshold = ucl_object_todouble(value);
+ }
+
+ /*
+ * Process rule in Lua
+ */
+ gint err_idx, ret;
+ lua_State *L = (lua_State *) cfg->lua_state;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, fuzzy_module_ctx->process_rule_ref);
+ ucl_object_push_lua(L, obj, true);
+
+ if ((ret = lua_pcall(L, 1, 1, err_idx)) != 0) {
+ msg_err_config("call to process_rule lua "
+ "script failed (%d): %s",
+ ret, lua_tostring(L, -1));
+
+ rule->lua_id = -1;
+ }
+ else {
+ rule->lua_id = lua_tonumber(L, -1);
+ }
+
+ lua_settop(L, err_idx - 1);
+
+ rspamd_mempool_add_destructor(cfg->cfg_pool, fuzzy_free_rule,
+ rule);
+
+ return 0;
+}
+
+gint fuzzy_check_module_init(struct rspamd_config *cfg, struct module_ctx **ctx)
+{
+ struct fuzzy_ctx *fuzzy_module_ctx;
+
+ fuzzy_module_ctx = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(struct fuzzy_ctx));
+
+ fuzzy_module_ctx->fuzzy_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ NULL, 0);
+ /* TODO: this should match rules count actually */
+ fuzzy_module_ctx->keypairs_cache = rspamd_keypair_cache_new(32);
+ fuzzy_module_ctx->fuzzy_rules = g_ptr_array_new();
+ fuzzy_module_ctx->cfg = cfg;
+ fuzzy_module_ctx->process_rule_ref = -1;
+ fuzzy_module_ctx->check_mime_part_ref = -1;
+ fuzzy_module_ctx->cleanup_rules_ref = -1;
+
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_mempool_delete,
+ fuzzy_module_ctx->fuzzy_pool);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_keypair_cache_destroy,
+ fuzzy_module_ctx->keypairs_cache);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_ptr_array_free_hard,
+ fuzzy_module_ctx->fuzzy_rules);
+
+ *ctx = (struct module_ctx *) fuzzy_module_ctx;
+
+ rspamd_rcl_add_doc_by_path(cfg,
+ NULL,
+ "Fuzzy check plugin",
+ "fuzzy_check",
+ UCL_OBJECT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Default symbol",
+ "symbol",
+ UCL_STRING,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Minimum number of *words* to check a text part",
+ "min_length",
+ UCL_INT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Minimum number of *bytes* to check a non-text part",
+ "min_bytes",
+ UCL_INT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Multiplier for bytes limit when checking for text parts",
+ "text_multiplier",
+ UCL_FLOAT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Minimum height in pixels for embedded images to check using fuzzy storage",
+ "min_height",
+ UCL_INT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Minimum width in pixels for embedded images to check using fuzzy storage",
+ "min_width",
+ UCL_INT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Timeout for waiting reply from a fuzzy server",
+ "timeout",
+ UCL_TIME,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Maximum number of retransmits for a single request",
+ "retransmits",
+ UCL_INT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Maximum number of upstream errors, affects error rate threshold",
+ "max_errors",
+ UCL_INT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Time to lapse before re-resolve faulty upstream",
+ "revive_time",
+ UCL_FLOAT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Whitelisted IPs map",
+ "whitelist",
+ UCL_STRING,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ /* Rules doc strings */
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check",
+ "Fuzzy check rule",
+ "rule",
+ UCL_OBJECT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Headers that are used to make a separate hash",
+ "headers",
+ UCL_ARRAY,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Whitelisted hashes map",
+ "skip_hashes",
+ UCL_STRING,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Set of mime types (in form type/subtype, or type/*, or *) to check with fuzzy",
+ "mime_types",
+ UCL_ARRAY,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Maximum value for fuzzy hash when weight of symbol is exactly 1.0 (if value is higher then score is still 1.0)",
+ "max_score",
+ UCL_INT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "List of servers to check (or learn)",
+ "servers",
+ UCL_STRING,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "If true then never try to learn this fuzzy storage",
+ "read_only",
+ UCL_BOOLEAN,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "If true then ignore unknown flags and not add the default fuzzy symbol",
+ "skip_unknown",
+ UCL_BOOLEAN,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Default symbol for rule (if no flags defined or matched)",
+ "symbol",
+ UCL_STRING,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Base32 value for the protocol encryption public key",
+ "encryption_key",
+ UCL_STRING,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Base32 value for the hashing key (for private storages)",
+ "fuzzy_key",
+ UCL_STRING,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Base32 value for the shingles hashing key (for private storages)",
+ "fuzzy_shingles_key",
+ UCL_STRING,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Lua script that returns boolean function to check if this task "
+ "should be considered when learning fuzzy storage",
+ "learn_condition",
+ UCL_STRING,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Map of SYMBOL -> data for flags configuration",
+ "fuzzy_map",
+ UCL_OBJECT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Use direct hash for short texts",
+ "short_text_direct_hash",
+ UCL_BOOLEAN,
+ NULL,
+ 0,
+ "true",
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Override module default min bytes for this rule",
+ "min_bytes",
+ UCL_INT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ /* Fuzzy map doc strings */
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule.fuzzy_map",
+ "Maximum score for this flag",
+ "max_score",
+ UCL_INT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule.fuzzy_map",
+ "Flag number",
+ "flag",
+ UCL_INT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Do no use subject to distinguish short text hashes",
+ "no_subject",
+ UCL_BOOLEAN,
+ NULL,
+ 0,
+ "false",
+ 0);
+ rspamd_rcl_add_doc_by_path(cfg,
+ "fuzzy_check.rule",
+ "Disable sharing message stats with the fuzzy server",
+ "no_share",
+ UCL_BOOLEAN,
+ NULL,
+ 0,
+ "false",
+ 0);
+
+ return 0;
+}
+
+gint fuzzy_check_module_config(struct rspamd_config *cfg, bool validate)
+{
+ const ucl_object_t *value, *cur, *elt;
+ ucl_object_iter_t it;
+ gint res = TRUE, cb_id, nrules = 0;
+ lua_State *L = cfg->lua_state;
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(cfg);
+
+ if (!rspamd_config_is_module_enabled(cfg, "fuzzy_check")) {
+ return TRUE;
+ }
+
+ fuzzy_module_ctx->enabled = TRUE;
+ fuzzy_module_ctx->check_mime_part_ref = -1;
+ fuzzy_module_ctx->process_rule_ref = -1;
+ fuzzy_module_ctx->cleanup_rules_ref = -1;
+
+ /* Interact with lua_fuzzy */
+ if (luaL_dostring(L, "return require \"lua_fuzzy\"") != 0) {
+ msg_err_config("cannot require lua_fuzzy: %s",
+ lua_tostring(L, -1));
+ fuzzy_module_ctx->enabled = FALSE;
+ }
+ else {
+#if LUA_VERSION_NUM >= 504
+ lua_settop(L, -2);
+#endif
+ if (lua_type(L, -1) != LUA_TTABLE) {
+ msg_err_config("lua fuzzy must return "
+ "table and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ fuzzy_module_ctx->enabled = FALSE;
+ }
+ else {
+ lua_pushstring(L, "process_rule");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_err_config("process_rule must return "
+ "function and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ fuzzy_module_ctx->enabled = FALSE;
+ }
+ else {
+ fuzzy_module_ctx->process_rule_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+
+ lua_pushstring(L, "check_mime_part");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_err_config("check_mime_part must return "
+ "function and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ fuzzy_module_ctx->enabled = FALSE;
+ }
+ else {
+ fuzzy_module_ctx->check_mime_part_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+
+ lua_pushstring(L, "cleanup_rules");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_err_config("cleanup_rules must return "
+ "function and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ fuzzy_module_ctx->enabled = FALSE;
+ }
+ else {
+ fuzzy_module_ctx->cleanup_rules_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ }
+ }
+
+ lua_settop(L, 0);
+
+ if (!fuzzy_module_ctx->enabled) {
+ return TRUE;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "fuzzy_check", "symbol")) != NULL) {
+ fuzzy_module_ctx->default_symbol = ucl_obj_tostring(value);
+ }
+ else {
+ fuzzy_module_ctx->default_symbol = DEFAULT_SYMBOL;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "fuzzy_check", "timeout")) != NULL) {
+ fuzzy_module_ctx->io_timeout = ucl_obj_todouble(value);
+ }
+ else {
+ fuzzy_module_ctx->io_timeout = DEFAULT_IO_TIMEOUT;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg,
+ "fuzzy_check",
+ "retransmits")) != NULL) {
+ fuzzy_module_ctx->retransmits = ucl_obj_toint(value);
+ }
+ else {
+ fuzzy_module_ctx->retransmits = DEFAULT_RETRANSMITS;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "fuzzy_check",
+ "max_errors")) != NULL) {
+ fuzzy_module_ctx->max_errors = ucl_obj_toint(value);
+ }
+ else {
+ fuzzy_module_ctx->max_errors = DEFAULT_MAX_ERRORS;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "fuzzy_check",
+ "revive_time")) != NULL) {
+ fuzzy_module_ctx->revive_time = ucl_obj_todouble(value);
+ }
+ else {
+ fuzzy_module_ctx->revive_time = DEFAULT_REVIVE_TIME;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "fuzzy_check",
+ "whitelist")) != NULL) {
+ rspamd_config_radix_from_ucl(cfg, value, "Fuzzy whitelist",
+ &fuzzy_module_ctx->whitelist,
+ NULL,
+ NULL, "fuzzy ip whitelist");
+ }
+ else {
+ fuzzy_module_ctx->whitelist = NULL;
+ }
+
+ if ((value =
+ rspamd_config_get_module_opt(cfg, "fuzzy_check", "rule")) != NULL) {
+
+ cb_id = rspamd_symcache_add_symbol(cfg->cache,
+ "FUZZY_CALLBACK", 0, fuzzy_symbol_callback, NULL,
+ SYMBOL_TYPE_CALLBACK | SYMBOL_TYPE_FINE,
+ -1);
+ rspamd_config_add_symbol(cfg,
+ "FUZZY_CALLBACK",
+ 0.0,
+ "Fuzzy check callback",
+ "fuzzy",
+ RSPAMD_SYMBOL_FLAG_IGNORE_METRIC,
+ 1,
+ 1);
+
+ /*
+ * Here we can have 2 possibilities:
+ *
+ * unnamed rules:
+ *
+ * rule {
+ * ...
+ * }
+ * rule {
+ * ...
+ * }
+ *
+ * - or - named rules:
+ *
+ * rule {
+ * "rule1": {
+ * ...
+ * }
+ * "rule2": {
+ * ...
+ * }
+ * }
+ *
+ * So, for each element, we check, if there 'servers' key. If 'servers' is
+ * presented, then we treat it as unnamed rule, otherwise we treat it as
+ * named rule.
+ */
+ LL_FOREACH(value, cur)
+ {
+
+ if (ucl_object_lookup(cur, "servers")) {
+ /* Unnamed rule */
+ fuzzy_parse_rule(cfg, cur, NULL, cb_id);
+ nrules++;
+ }
+ else {
+ /* Named rule */
+ it = NULL;
+
+ while ((elt = ucl_object_iterate(cur, &it, true)) != NULL) {
+ fuzzy_parse_rule(cfg, elt, ucl_object_key(elt), cb_id);
+ nrules++;
+ }
+ }
+ }
+
+ /* We want that to check bad mime attachments */
+ rspamd_symcache_add_delayed_dependency(cfg->cache,
+ "FUZZY_CALLBACK", "MIME_TYPES_CALLBACK");
+ }
+
+ if (fuzzy_module_ctx->fuzzy_rules == NULL) {
+ msg_warn_config("fuzzy module is enabled but no rules are defined");
+ }
+
+ msg_info_config("init internal fuzzy_check module, %d rules loaded",
+ nrules);
+
+ /* Register global methods */
+ lua_getglobal(L, "rspamd_plugins");
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_pushstring(L, "fuzzy_check");
+ lua_createtable(L, 0, 3);
+ /* Set methods */
+ lua_pushstring(L, "unlearn");
+ lua_pushcfunction(L, fuzzy_lua_unlearn_handler);
+ lua_settable(L, -3);
+ lua_pushstring(L, "learn");
+ lua_pushcfunction(L, fuzzy_lua_learn_handler);
+ lua_settable(L, -3);
+ lua_pushstring(L, "gen_hashes");
+ lua_pushcfunction(L, fuzzy_lua_gen_hashes_handler);
+ lua_settable(L, -3);
+ lua_pushstring(L, "hex_hashes");
+ lua_pushcfunction(L, fuzzy_lua_hex_hashes_handler);
+ lua_settable(L, -3);
+ lua_pushstring(L, "list_storages");
+ lua_pushcfunction(L, fuzzy_lua_list_storages);
+ lua_settable(L, -3);
+ lua_pushstring(L, "ping_storage");
+ lua_pushcfunction(L, fuzzy_lua_ping_storage);
+ lua_settable(L, -3);
+ /* Finish fuzzy_check key */
+ lua_settable(L, -3);
+ }
+
+ lua_settop(L, 0);
+
+ return res;
+}
+
+gint fuzzy_check_module_reconfig(struct rspamd_config *cfg)
+{
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(cfg);
+
+ if (fuzzy_module_ctx->cleanup_rules_ref != -1) {
+ /* Sync lua_fuzzy rules */
+ gint err_idx, ret;
+ lua_State *L = (lua_State *) cfg->lua_state;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, fuzzy_module_ctx->cleanup_rules_ref);
+
+ if ((ret = lua_pcall(L, 0, 0, err_idx)) != 0) {
+ msg_err_config("call to cleanup_rules lua "
+ "script failed (%d): %s",
+ ret, lua_tostring(L, -1));
+ }
+
+ luaL_unref(cfg->lua_state, LUA_REGISTRYINDEX,
+ fuzzy_module_ctx->cleanup_rules_ref);
+ lua_settop(L, 0);
+ }
+
+ if (fuzzy_module_ctx->check_mime_part_ref != -1) {
+ luaL_unref(cfg->lua_state, LUA_REGISTRYINDEX,
+ fuzzy_module_ctx->check_mime_part_ref);
+ }
+
+ if (fuzzy_module_ctx->process_rule_ref != -1) {
+ luaL_unref(cfg->lua_state, LUA_REGISTRYINDEX,
+ fuzzy_module_ctx->process_rule_ref);
+ }
+
+ return fuzzy_check_module_config(cfg, false);
+}
+
+/* Finalize IO */
+static void
+fuzzy_io_fin(void *ud)
+{
+ struct fuzzy_client_session *session = ud;
+
+ if (session->commands) {
+ g_ptr_array_free(session->commands, TRUE);
+ }
+
+ if (session->results) {
+ g_ptr_array_free(session->results, TRUE);
+ }
+
+ rspamd_ev_watcher_stop(session->event_loop, &session->ev);
+ close(session->fd);
+}
+
+static GArray *
+fuzzy_preprocess_words(struct rspamd_mime_text_part *part, rspamd_mempool_t *pool)
+{
+ return part->utf_words;
+}
+
+static void
+fuzzy_encrypt_cmd(struct fuzzy_rule *rule,
+ struct rspamd_fuzzy_encrypted_req_hdr *hdr,
+ guchar *data, gsize datalen)
+{
+ const guchar *pk;
+ guint pklen;
+
+ g_assert(hdr != NULL);
+ g_assert(data != NULL);
+ g_assert(rule != NULL);
+
+ /* Encrypt data */
+ memcpy(hdr->magic,
+ fuzzy_encrypted_magic,
+ sizeof(hdr->magic));
+ ottery_rand_bytes(hdr->nonce, sizeof(hdr->nonce));
+ pk = rspamd_keypair_component(rule->local_key,
+ RSPAMD_KEYPAIR_COMPONENT_PK, &pklen);
+ memcpy(hdr->pubkey, pk, MIN(pklen, sizeof(hdr->pubkey)));
+ pk = rspamd_pubkey_get_pk(rule->peer_key, &pklen);
+ memcpy(hdr->key_id, pk, MIN(sizeof(hdr->key_id), pklen));
+ rspamd_keypair_cache_process(rule->ctx->keypairs_cache,
+ rule->local_key, rule->peer_key);
+ rspamd_cryptobox_encrypt_nm_inplace(data, datalen,
+ hdr->nonce, rspamd_pubkey_get_nm(rule->peer_key, rule->local_key),
+ hdr->mac,
+ rspamd_pubkey_alg(rule->peer_key));
+}
+
+static struct fuzzy_cmd_io *
+fuzzy_cmd_stat(struct fuzzy_rule *rule,
+ int c,
+ gint flag,
+ guint32 weight,
+ rspamd_mempool_t *pool)
+{
+ struct rspamd_fuzzy_cmd *cmd;
+ struct rspamd_fuzzy_encrypted_cmd *enccmd = NULL;
+ struct fuzzy_cmd_io *io;
+
+ if (rule->peer_key) {
+ enccmd = rspamd_mempool_alloc0(pool, sizeof(*enccmd));
+ cmd = &enccmd->cmd;
+ }
+ else {
+ cmd = rspamd_mempool_alloc0(pool, sizeof(*cmd));
+ }
+
+ cmd->cmd = c;
+ cmd->version = RSPAMD_FUZZY_PLUGIN_VERSION;
+ cmd->shingles_count = 0;
+ cmd->tag = ottery_rand_uint32();
+
+ io = rspamd_mempool_alloc(pool, sizeof(*io));
+ io->flags = 0;
+ io->tag = cmd->tag;
+ memcpy(&io->cmd, cmd, sizeof(io->cmd));
+
+ if (rule->peer_key && enccmd) {
+ fuzzy_encrypt_cmd(rule, &enccmd->hdr, (guchar *) cmd, sizeof(*cmd));
+ io->io.iov_base = enccmd;
+ io->io.iov_len = sizeof(*enccmd);
+ }
+ else {
+ io->io.iov_base = cmd;
+ io->io.iov_len = sizeof(*cmd);
+ }
+
+ return io;
+}
+
+static inline double
+fuzzy_milliseconds_since_midnight(void)
+{
+ double now = rspamd_get_calendar_ticks();
+ double ms = now - (int64_t) now;
+ now = (((int64_t) now % 86400) + ms) * 1000;
+
+ return now;
+}
+
+static struct fuzzy_cmd_io *
+fuzzy_cmd_ping(struct fuzzy_rule *rule,
+ rspamd_mempool_t *pool)
+{
+ struct rspamd_fuzzy_cmd *cmd;
+ struct rspamd_fuzzy_encrypted_cmd *enccmd = NULL;
+ struct fuzzy_cmd_io *io;
+
+ if (rule->peer_key) {
+ enccmd = rspamd_mempool_alloc0(pool, sizeof(*enccmd));
+ cmd = &enccmd->cmd;
+ }
+ else {
+ cmd = rspamd_mempool_alloc0(pool, sizeof(*cmd));
+ }
+
+ /* Get milliseconds since midnight */
+
+
+ cmd->cmd = FUZZY_PING;
+ cmd->version = RSPAMD_FUZZY_PLUGIN_VERSION;
+ cmd->shingles_count = 0;
+ cmd->value = fuzzy_milliseconds_since_midnight(); /* Record timestamp */
+ cmd->tag = ottery_rand_uint32();
+
+ io = rspamd_mempool_alloc(pool, sizeof(*io));
+ io->flags = 0;
+ io->tag = cmd->tag;
+ memcpy(&io->cmd, cmd, sizeof(io->cmd));
+
+ if (rule->peer_key && enccmd) {
+ fuzzy_encrypt_cmd(rule, &enccmd->hdr, (guchar *) cmd, sizeof(*cmd));
+ io->io.iov_base = enccmd;
+ io->io.iov_len = sizeof(*enccmd);
+ }
+ else {
+ io->io.iov_base = cmd;
+ io->io.iov_len = sizeof(*cmd);
+ }
+
+ return io;
+}
+
+static struct fuzzy_cmd_io *
+fuzzy_cmd_hash(struct fuzzy_rule *rule,
+ int c,
+ const rspamd_ftok_t *hash,
+ gint flag,
+ guint32 weight,
+ rspamd_mempool_t *pool)
+{
+ struct rspamd_fuzzy_cmd *cmd;
+ struct rspamd_fuzzy_encrypted_cmd *enccmd = NULL;
+ struct fuzzy_cmd_io *io;
+
+ if (rule->peer_key) {
+ enccmd = rspamd_mempool_alloc0(pool, sizeof(*enccmd));
+ cmd = &enccmd->cmd;
+ }
+ else {
+ cmd = rspamd_mempool_alloc0(pool, sizeof(*cmd));
+ }
+
+ if (hash->len == sizeof(cmd->digest) * 2) {
+ /* It is hex encoding */
+ if (rspamd_decode_hex_buf(hash->begin, hash->len, cmd->digest,
+ sizeof(cmd->digest)) == -1) {
+ msg_err_pool("cannot decode hash, wrong encoding");
+ return NULL;
+ }
+ }
+ else {
+ msg_err_pool("cannot decode hash, wrong length: %z", hash->len);
+ return NULL;
+ }
+
+ cmd->cmd = c;
+ cmd->version = RSPAMD_FUZZY_PLUGIN_VERSION;
+ cmd->shingles_count = 0;
+ cmd->tag = ottery_rand_uint32();
+
+ io = rspamd_mempool_alloc(pool, sizeof(*io));
+ io->flags = 0;
+ io->tag = cmd->tag;
+
+ memcpy(&io->cmd, cmd, sizeof(io->cmd));
+
+ if (rule->peer_key && enccmd) {
+ fuzzy_encrypt_cmd(rule, &enccmd->hdr, (guchar *) cmd, sizeof(*cmd));
+ io->io.iov_base = enccmd;
+ io->io.iov_len = sizeof(*enccmd);
+ }
+ else {
+ io->io.iov_base = cmd;
+ io->io.iov_len = sizeof(*cmd);
+ }
+
+ return io;
+}
+
+struct rspamd_cached_shingles {
+ struct rspamd_shingle *sh;
+ guchar digest[rspamd_cryptobox_HASHBYTES];
+ guint additional_length;
+ guchar *additional_data;
+};
+
+
+static struct rspamd_cached_shingles *
+fuzzy_cmd_get_cached(struct fuzzy_rule *rule,
+ struct rspamd_task *task,
+ struct rspamd_mime_part *mp)
+{
+ gchar key[32];
+ gint key_part;
+ struct rspamd_cached_shingles **cached;
+
+ memcpy(&key_part, rule->shingles_key->str, sizeof(key_part));
+ rspamd_snprintf(key, sizeof(key), "%s%d", rule->algorithm_str,
+ key_part);
+
+ cached = (struct rspamd_cached_shingles **) rspamd_mempool_get_variable(
+ task->task_pool, key);
+
+ if (cached && cached[mp->part_number]) {
+ return cached[mp->part_number];
+ }
+
+ return NULL;
+}
+
+static void
+fuzzy_cmd_set_cached(struct fuzzy_rule *rule,
+ struct rspamd_task *task,
+ struct rspamd_mime_part *mp,
+ struct rspamd_cached_shingles *data)
+{
+ gchar key[32];
+ gint key_part;
+ struct rspamd_cached_shingles **cached;
+
+ memcpy(&key_part, rule->shingles_key->str, sizeof(key_part));
+ rspamd_snprintf(key, sizeof(key), "%s%d", rule->algorithm_str,
+ key_part);
+
+ cached = (struct rspamd_cached_shingles **) rspamd_mempool_get_variable(
+ task->task_pool, key);
+
+ if (cached) {
+ cached[mp->part_number] = data;
+ }
+ else {
+ cached = rspamd_mempool_alloc0(task->task_pool, sizeof(*cached) *
+ (MESSAGE_FIELD(task, parts)->len + 1));
+ cached[mp->part_number] = data;
+
+ rspamd_mempool_set_variable(task->task_pool, key, cached, NULL);
+ }
+}
+
+static gboolean
+fuzzy_rule_check_mimepart(struct rspamd_task *task,
+ struct fuzzy_rule *rule,
+ struct rspamd_mime_part *part,
+ gboolean *need_check,
+ gboolean *fuzzy_check)
+{
+ lua_State *L = (lua_State *) task->cfg->lua_state;
+
+ gint old_top = lua_gettop(L);
+
+ if (rule->lua_id != -1 && rule->ctx->check_mime_part_ref != -1) {
+ gint err_idx, ret;
+
+ struct rspamd_task **ptask;
+ struct rspamd_mime_part **ppart;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, rule->ctx->check_mime_part_ref);
+
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ ppart = lua_newuserdata(L, sizeof(*ppart));
+ *ppart = part;
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+
+ lua_pushnumber(L, rule->lua_id);
+
+ if ((ret = lua_pcall(L, 3, 2, err_idx)) != 0) {
+ msg_err_task("call to check_mime_part lua "
+ "script failed (%d): %s",
+ ret, lua_tostring(L, -1));
+
+ ret = FALSE;
+ }
+ else {
+ ret = TRUE;
+ *need_check = lua_toboolean(L, -2);
+ *fuzzy_check = lua_toboolean(L, -1);
+ }
+
+ lua_settop(L, old_top);
+
+ return ret;
+ }
+
+ return FALSE;
+}
+
+#define MAX_FUZZY_DOMAIN 64
+
+static guint
+fuzzy_cmd_extension_length(struct rspamd_task *task,
+ struct fuzzy_rule *rule)
+{
+ guint total = 0;
+
+ if (rule->no_share) {
+ return 0;
+ }
+
+ /* From domain */
+ if (MESSAGE_FIELD(task, from_mime) && MESSAGE_FIELD(task, from_mime)->len > 0) {
+ struct rspamd_email_address *addr = g_ptr_array_index(MESSAGE_FIELD(task,
+ from_mime),
+ 0);
+
+ if (addr->domain_len > 0) {
+ total += 2; /* 2 bytes: type + length */
+ total += MIN(MAX_FUZZY_DOMAIN, addr->domain_len);
+ }
+ }
+
+ if (task->from_addr && rspamd_inet_address_get_af(task->from_addr) == AF_INET) {
+ total += sizeof(struct in_addr) + 1;
+ }
+ else if (task->from_addr && rspamd_inet_address_get_af(task->from_addr) == AF_INET6) {
+ total += sizeof(struct in6_addr) + 1;
+ }
+
+ return total;
+}
+
+static guint
+fuzzy_cmd_write_extensions(struct rspamd_task *task,
+ struct fuzzy_rule *rule,
+ guchar *dest,
+ gsize available)
+{
+ guint written = 0;
+
+ if (rule->no_share) {
+ return 0;
+ }
+
+ if (MESSAGE_FIELD(task, from_mime) && MESSAGE_FIELD(task, from_mime)->len > 0) {
+ struct rspamd_email_address *addr = g_ptr_array_index(MESSAGE_FIELD(task,
+ from_mime),
+ 0);
+ guint to_write = MIN(MAX_FUZZY_DOMAIN, addr->domain_len) + 2;
+
+ if (to_write > 0 && to_write <= available) {
+ *dest++ = RSPAMD_FUZZY_EXT_SOURCE_DOMAIN;
+ *dest++ = to_write - 2;
+
+ if (addr->domain_len < MAX_FUZZY_DOMAIN) {
+ memcpy(dest, addr->domain, addr->domain_len);
+ dest += addr->domain_len;
+ }
+ else {
+ /* Trim from left */
+ memcpy(dest,
+ addr->domain + (addr->domain_len - MAX_FUZZY_DOMAIN),
+ MAX_FUZZY_DOMAIN);
+ dest += MAX_FUZZY_DOMAIN;
+ }
+
+ available -= to_write;
+ written += to_write;
+ }
+ }
+
+ if (task->from_addr && rspamd_inet_address_get_af(task->from_addr) == AF_INET) {
+ if (available >= sizeof(struct in_addr) + 1) {
+ guint klen;
+ guchar *inet_data = rspamd_inet_address_get_hash_key(task->from_addr, &klen);
+
+ *dest++ = RSPAMD_FUZZY_EXT_SOURCE_IP4;
+
+ memcpy(dest, inet_data, klen);
+ dest += klen;
+
+ available -= klen + 1;
+ written += klen + 1;
+ }
+ }
+ else if (task->from_addr && rspamd_inet_address_get_af(task->from_addr) == AF_INET6) {
+ if (available >= sizeof(struct in6_addr) + 1) {
+ guint klen;
+ guchar *inet_data = rspamd_inet_address_get_hash_key(task->from_addr, &klen);
+
+ *dest++ = RSPAMD_FUZZY_EXT_SOURCE_IP6;
+
+ memcpy(dest, inet_data, klen);
+ dest += klen;
+
+ available -= klen + 1;
+ written += klen + 1;
+ }
+ }
+
+ return written;
+}
+
+/*
+ * Create fuzzy command from a text part
+ */
+static struct fuzzy_cmd_io *
+fuzzy_cmd_from_text_part(struct rspamd_task *task,
+ struct fuzzy_rule *rule,
+ int c,
+ gint flag,
+ guint32 weight,
+ gboolean short_text,
+ struct rspamd_mime_text_part *part,
+ struct rspamd_mime_part *mp)
+{
+ struct rspamd_fuzzy_shingle_cmd *shcmd = NULL;
+ struct rspamd_fuzzy_cmd *cmd = NULL;
+ struct rspamd_fuzzy_encrypted_shingle_cmd *encshcmd = NULL;
+ struct rspamd_fuzzy_encrypted_cmd *enccmd = NULL;
+ struct rspamd_cached_shingles *cached = NULL;
+ struct rspamd_shingle *sh = NULL;
+ guint i;
+ rspamd_cryptobox_hash_state_t st;
+ rspamd_stat_token_t *word;
+ GArray *words;
+ struct fuzzy_cmd_io *io;
+ guint additional_length;
+ guchar *additional_data;
+
+ cached = fuzzy_cmd_get_cached(rule, task, mp);
+
+ /*
+ * Important note:
+ *
+ * We assume that fuzzy io is a consistent memory layout to fit into
+ * iov structure of size 1
+ *
+ * However, there are 4 possibilities:
+ * 1) non encrypted, non shingle command - just one cmd
+ * 2) encrypted, non shingle command - encryption hdr + cmd
+ * 3) non encrypted, shingle command - cmd + shingle
+ * 4) encrypted, shingle command - encryption hdr + cmd + shingle
+ *
+ * Extensions are always at the end, but since we also have caching (sigh, meh...)
+ * then we have one piece that looks like cmd (+ shingle) + extensions
+ * To encrypt it optionally we take this memory and prepend encryption header
+ *
+ * In case of cached version we do the same: allocate, copy from cached (including extra)
+ * and optionally encrypt.
+ *
+ * However, there should be no extensions in case of unencrypted connection
+ * (for sanity + privacy).
+ */
+ if (cached) {
+ additional_length = cached->additional_length;
+ additional_data = cached->additional_data;
+
+ /* Copy cached */
+ if (short_text) {
+ enccmd = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(*enccmd) + additional_length);
+ cmd = &enccmd->cmd;
+ memcpy(cmd->digest, cached->digest,
+ sizeof(cached->digest));
+ cmd->shingles_count = 0;
+ memcpy(((guchar *) enccmd) + sizeof(*enccmd), additional_data,
+ additional_length);
+ }
+ else if (cached->sh) {
+ encshcmd = rspamd_mempool_alloc0(task->task_pool,
+ additional_length + sizeof(*encshcmd));
+ shcmd = &encshcmd->cmd;
+ memcpy(&shcmd->sgl, cached->sh, sizeof(struct rspamd_shingle));
+ memcpy(shcmd->basic.digest, cached->digest,
+ sizeof(cached->digest));
+ memcpy(((guchar *) encshcmd) + sizeof(*encshcmd), additional_data,
+ additional_length);
+ shcmd->basic.shingles_count = RSPAMD_SHINGLE_SIZE;
+ }
+ else {
+ return NULL;
+ }
+ }
+ else {
+ additional_length = fuzzy_cmd_extension_length(task, rule);
+ cached = rspamd_mempool_alloc0(task->task_pool, sizeof(*cached) +
+ additional_length);
+ /*
+ * Allocate extensions and never touch it except copying to avoid
+ * occasional encryption
+ */
+ cached->additional_length = additional_length;
+ cached->additional_data = ((guchar *) cached) + sizeof(*cached);
+
+ if (additional_length > 0) {
+ fuzzy_cmd_write_extensions(task, rule, cached->additional_data,
+ additional_length);
+ }
+
+ if (short_text) {
+ enccmd = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(*enccmd) + additional_length);
+ cmd = &enccmd->cmd;
+ rspamd_cryptobox_hash_init(&st, rule->hash_key->str,
+ rule->hash_key->len);
+
+ rspamd_cryptobox_hash_update(&st, part->utf_stripped_content->data,
+ part->utf_stripped_content->len);
+
+ if (!rule->no_subject && (MESSAGE_FIELD(task, subject))) {
+ /* We also include subject */
+ rspamd_cryptobox_hash_update(&st, MESSAGE_FIELD(task, subject),
+ strlen(MESSAGE_FIELD(task, subject)));
+ }
+
+ rspamd_cryptobox_hash_final(&st, cmd->digest);
+ memcpy(cached->digest, cmd->digest, sizeof(cached->digest));
+ cached->sh = NULL;
+
+ additional_data = ((guchar *) enccmd) + sizeof(*enccmd);
+ memcpy(additional_data, cached->additional_data, additional_length);
+ }
+ else {
+ encshcmd = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(*encshcmd) + additional_length);
+ shcmd = &encshcmd->cmd;
+
+ /*
+ * Generate hash from all words in the part
+ */
+ rspamd_cryptobox_hash_init(&st, rule->hash_key->str, rule->hash_key->len);
+ words = fuzzy_preprocess_words(part, task->task_pool);
+
+ for (i = 0; i < words->len; i++) {
+ word = &g_array_index(words, rspamd_stat_token_t, i);
+
+ if (!((word->flags & RSPAMD_STAT_TOKEN_FLAG_SKIPPED) || word->stemmed.len == 0)) {
+ rspamd_cryptobox_hash_update(&st, word->stemmed.begin,
+ word->stemmed.len);
+ }
+ }
+
+ rspamd_cryptobox_hash_final(&st, shcmd->basic.digest);
+
+ msg_debug_task("loading shingles of type %s with key %*xs",
+ rule->algorithm_str,
+ 16, rule->shingles_key->str);
+ sh = rspamd_shingles_from_text(words,
+ rule->shingles_key->str, task->task_pool,
+ rspamd_shingles_default_filter, NULL,
+ rule->alg);
+ if (sh != NULL) {
+ memcpy(&shcmd->sgl, sh, sizeof(shcmd->sgl));
+ shcmd->basic.shingles_count = RSPAMD_SHINGLE_SIZE;
+ }
+ else {
+ /* No shingles, no check */
+ return NULL;
+ }
+
+ cached->sh = sh;
+ memcpy(cached->digest, shcmd->basic.digest, sizeof(cached->digest));
+ additional_data = ((guchar *) encshcmd) + sizeof(*encshcmd);
+ memcpy(additional_data, cached->additional_data, additional_length);
+ }
+
+ /*
+ * We always save encrypted command as it can handle both
+ * encrypted and unencrypted requests.
+ *
+ * Since it is copied when obtained from the cache, it is safe to use
+ * it this way.
+ */
+ fuzzy_cmd_set_cached(rule, task, mp, cached);
+ }
+
+ io = rspamd_mempool_alloc(task->task_pool, sizeof(*io));
+ io->part = mp;
+
+ if (!short_text) {
+ shcmd->basic.tag = ottery_rand_uint32();
+ shcmd->basic.cmd = c;
+ shcmd->basic.version = RSPAMD_FUZZY_PLUGIN_VERSION;
+
+ if (c != FUZZY_CHECK) {
+ shcmd->basic.flag = flag;
+ shcmd->basic.value = weight;
+ }
+ io->tag = shcmd->basic.tag;
+ memcpy(&io->cmd, &shcmd->basic, sizeof(io->cmd));
+ }
+ else {
+ cmd->tag = ottery_rand_uint32();
+ cmd->cmd = c;
+ cmd->version = RSPAMD_FUZZY_PLUGIN_VERSION;
+
+ if (c != FUZZY_CHECK) {
+ cmd->flag = flag;
+ cmd->value = weight;
+ }
+ io->tag = cmd->tag;
+ memcpy(&io->cmd, cmd, sizeof(io->cmd));
+ }
+
+ io->flags = 0;
+
+
+ if (rule->peer_key) {
+ /* Encrypt data */
+ if (!short_text) {
+ fuzzy_encrypt_cmd(rule, &encshcmd->hdr, (guchar *) shcmd,
+ sizeof(*shcmd) + additional_length);
+ io->io.iov_base = encshcmd;
+ io->io.iov_len = sizeof(*encshcmd) + additional_length;
+ }
+ else {
+ fuzzy_encrypt_cmd(rule, &enccmd->hdr, (guchar *) cmd,
+ sizeof(*cmd) + additional_length);
+ io->io.iov_base = enccmd;
+ io->io.iov_len = sizeof(*enccmd) + additional_length;
+ }
+ }
+ else {
+
+ if (!short_text) {
+ io->io.iov_base = shcmd;
+ io->io.iov_len = sizeof(*shcmd) + additional_length;
+ }
+ else {
+ io->io.iov_base = cmd;
+ io->io.iov_len = sizeof(*cmd) + additional_length;
+ }
+ }
+
+ return io;
+}
+
+#if 0
+static struct fuzzy_cmd_io *
+fuzzy_cmd_from_image_part (struct fuzzy_rule *rule,
+ int c,
+ gint flag,
+ guint32 weight,
+ struct rspamd_task *task,
+ struct rspamd_image *img,
+ struct rspamd_mime_part *mp)
+{
+ struct rspamd_fuzzy_shingle_cmd *shcmd;
+ struct rspamd_fuzzy_encrypted_shingle_cmd *encshcmd;
+ struct fuzzy_cmd_io *io;
+ struct rspamd_shingle *sh;
+ struct rspamd_cached_shingles *cached;
+
+ cached = fuzzy_cmd_get_cached (rule, task, mp);
+
+ if (cached) {
+ /* Copy cached */
+ encshcmd = rspamd_mempool_alloc0 (task->task_pool, sizeof (*encshcmd));
+ shcmd = &encshcmd->cmd;
+ memcpy (&shcmd->sgl, cached->sh, sizeof (struct rspamd_shingle));
+ memcpy (shcmd->basic.digest, cached->digest,
+ sizeof (cached->digest));
+ shcmd->basic.shingles_count = RSPAMD_SHINGLE_SIZE;
+ }
+ else {
+ encshcmd = rspamd_mempool_alloc0 (task->task_pool, sizeof (*encshcmd));
+ shcmd = &encshcmd->cmd;
+
+ /*
+ * Generate shingles
+ */
+ sh = rspamd_shingles_from_image (img->dct,
+ rule->shingles_key->str, task->task_pool,
+ rspamd_shingles_default_filter, NULL,
+ rule->alg);
+ if (sh != NULL) {
+ memcpy (&shcmd->sgl, sh->hashes, sizeof (shcmd->sgl));
+ shcmd->basic.shingles_count = RSPAMD_SHINGLE_SIZE;
+#if 0
+ for (unsigned int i = 0; i < RSPAMD_SHINGLE_SIZE; i ++) {
+ msg_err ("shingle %d: %L", i, sh->hashes[i]);
+ }
+#endif
+ }
+
+ rspamd_cryptobox_hash (shcmd->basic.digest,
+ (const guchar *)img->dct, RSPAMD_DCT_LEN / NBBY,
+ rule->hash_key->str, rule->hash_key->len);
+
+ msg_debug_task ("loading shingles of type %s with key %*xs",
+ rule->algorithm_str,
+ 16, rule->shingles_key->str);
+
+ /*
+ * We always save encrypted command as it can handle both
+ * encrypted and unencrypted requests.
+ *
+ * Since it is copied when obtained from the cache, it is safe to use
+ * it this way.
+ */
+ cached = rspamd_mempool_alloc (task->task_pool, sizeof (*cached));
+ cached->sh = sh;
+ memcpy (cached->digest, shcmd->basic.digest, sizeof (cached->digest));
+ fuzzy_cmd_set_cached (rule, task, mp, cached);
+ }
+
+ shcmd->basic.tag = ottery_rand_uint32 ();
+ shcmd->basic.cmd = c;
+ shcmd->basic.version = RSPAMD_FUZZY_PLUGIN_VERSION;
+
+ if (c != FUZZY_CHECK) {
+ shcmd->basic.flag = flag;
+ shcmd->basic.value = weight;
+ }
+
+ io = rspamd_mempool_alloc (task->task_pool, sizeof (*io));
+ io->part = mp;
+ io->tag = shcmd->basic.tag;
+ io->flags = FUZZY_CMD_FLAG_IMAGE;
+ memcpy (&io->cmd, &shcmd->basic, sizeof (io->cmd));
+
+ if (rule->peer_key) {
+ /* Encrypt data */
+ fuzzy_encrypt_cmd (rule, &encshcmd->hdr, (guchar *) shcmd, sizeof (*shcmd));
+ io->io.iov_base = encshcmd;
+ io->io.iov_len = sizeof (*encshcmd);
+ }
+ else {
+ io->io.iov_base = shcmd;
+ io->io.iov_len = sizeof (*shcmd);
+ }
+
+ return io;
+}
+#endif
+
+static struct fuzzy_cmd_io *
+fuzzy_cmd_from_data_part(struct fuzzy_rule *rule,
+ int c,
+ gint flag,
+ guint32 weight,
+ struct rspamd_task *task,
+ guchar digest[rspamd_cryptobox_HASHBYTES],
+ struct rspamd_mime_part *mp)
+{
+ struct rspamd_fuzzy_cmd *cmd;
+ struct rspamd_fuzzy_encrypted_cmd *enccmd = NULL;
+ struct fuzzy_cmd_io *io;
+ guint additional_length;
+ guchar *additional_data;
+
+ additional_length = fuzzy_cmd_extension_length(task, rule);
+
+ if (rule->peer_key) {
+ enccmd = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(*enccmd) + additional_length);
+ cmd = &enccmd->cmd;
+ additional_data = ((guchar *) enccmd) + sizeof(*enccmd);
+ }
+ else {
+ cmd = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(*cmd) + additional_length);
+ additional_data = ((guchar *) cmd) + sizeof(*cmd);
+ }
+
+ cmd->cmd = c;
+ cmd->version = RSPAMD_FUZZY_PLUGIN_VERSION;
+ if (c != FUZZY_CHECK) {
+ cmd->flag = flag;
+ cmd->value = weight;
+ }
+ cmd->shingles_count = 0;
+ cmd->tag = ottery_rand_uint32();
+ memcpy(cmd->digest, digest, sizeof(cmd->digest));
+
+ io = rspamd_mempool_alloc(task->task_pool, sizeof(*io));
+ io->flags = 0;
+ io->tag = cmd->tag;
+ io->part = mp;
+ memcpy(&io->cmd, cmd, sizeof(io->cmd));
+
+ if (additional_length > 0) {
+ fuzzy_cmd_write_extensions(task, rule, additional_data,
+ additional_length);
+ }
+
+ if (rule->peer_key) {
+ g_assert(enccmd != NULL);
+ fuzzy_encrypt_cmd(rule, &enccmd->hdr, (guchar *) cmd,
+ sizeof(*cmd) + additional_length);
+ io->io.iov_base = enccmd;
+ io->io.iov_len = sizeof(*enccmd) + additional_length;
+ }
+ else {
+ io->io.iov_base = cmd;
+ io->io.iov_len = sizeof(*cmd) + additional_length;
+ }
+
+ return io;
+}
+
+static gboolean
+fuzzy_cmd_to_wire(gint fd, struct iovec *io)
+{
+ struct msghdr msg;
+
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = io;
+ msg.msg_iovlen = 1;
+
+ while (sendmsg(fd, &msg, 0) == -1) {
+ if (errno == EINTR) {
+ continue;
+ }
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static gboolean
+fuzzy_cmd_vector_to_wire(gint fd, GPtrArray *v)
+{
+ guint i;
+ gboolean all_sent = TRUE, all_replied = TRUE;
+ struct fuzzy_cmd_io *io;
+ gboolean processed = FALSE;
+
+ /* First try to resend unsent commands */
+ for (i = 0; i < v->len; i++) {
+ io = g_ptr_array_index(v, i);
+
+ if (io->flags & FUZZY_CMD_FLAG_REPLIED) {
+ continue;
+ }
+
+ all_replied = FALSE;
+
+ if (!(io->flags & FUZZY_CMD_FLAG_SENT)) {
+ if (!fuzzy_cmd_to_wire(fd, &io->io)) {
+ return FALSE;
+ }
+ processed = TRUE;
+ io->flags |= FUZZY_CMD_FLAG_SENT;
+ all_sent = FALSE;
+ }
+ }
+
+ if (all_sent && !all_replied) {
+ /* Now try to resend each command in the vector */
+ for (i = 0; i < v->len; i++) {
+ io = g_ptr_array_index(v, i);
+
+ if (!(io->flags & FUZZY_CMD_FLAG_REPLIED)) {
+ io->flags &= ~FUZZY_CMD_FLAG_SENT;
+ }
+ }
+
+ return fuzzy_cmd_vector_to_wire(fd, v);
+ }
+
+ return processed;
+}
+
+/*
+ * Read replies one-by-one and remove them from req array
+ */
+static const struct rspamd_fuzzy_reply *
+fuzzy_process_reply(guchar **pos, gint *r, GPtrArray *req,
+ struct fuzzy_rule *rule, struct rspamd_fuzzy_cmd **pcmd,
+ struct fuzzy_cmd_io **pio)
+{
+ guchar *p = *pos;
+ gint remain = *r;
+ guint i, required_size;
+ struct fuzzy_cmd_io *io;
+ const struct rspamd_fuzzy_reply *rep;
+ struct rspamd_fuzzy_encrypted_reply encrep;
+ gboolean found = FALSE;
+
+ if (rule->peer_key) {
+ required_size = sizeof(encrep);
+ }
+ else {
+ required_size = sizeof(*rep);
+ }
+
+ if (remain <= 0 || (guint) remain < required_size) {
+ return NULL;
+ }
+
+ if (rule->peer_key) {
+ memcpy(&encrep, p, sizeof(encrep));
+ *pos += required_size;
+ *r -= required_size;
+
+ /* Try to decrypt reply */
+ rspamd_keypair_cache_process(rule->ctx->keypairs_cache,
+ rule->local_key, rule->peer_key);
+
+ if (!rspamd_cryptobox_decrypt_nm_inplace((guchar *) &encrep.rep,
+ sizeof(encrep.rep),
+ encrep.hdr.nonce,
+ rspamd_pubkey_get_nm(rule->peer_key, rule->local_key),
+ encrep.hdr.mac,
+ rspamd_pubkey_alg(rule->peer_key))) {
+ msg_info("cannot decrypt reply");
+ return NULL;
+ }
+
+ /* Copy decrypted over the input wire */
+ memcpy(p, &encrep.rep, sizeof(encrep.rep));
+ }
+ else {
+
+ *pos += required_size;
+ *r -= required_size;
+ }
+
+ rep = (const struct rspamd_fuzzy_reply *) p;
+ /*
+ * Search for tag
+ */
+ for (i = 0; i < req->len; i++) {
+ io = g_ptr_array_index(req, i);
+
+ if (io->tag == rep->v1.tag) {
+ if (!(io->flags & FUZZY_CMD_FLAG_REPLIED)) {
+ io->flags |= FUZZY_CMD_FLAG_REPLIED;
+
+ if (pcmd) {
+ *pcmd = &io->cmd;
+ }
+
+ if (pio) {
+ *pio = io;
+ }
+
+ return rep;
+ }
+ found = TRUE;
+ }
+ }
+
+ if (!found) {
+ msg_info("unexpected tag: %ud", rep->v1.tag);
+ }
+
+ return NULL;
+}
+
+static void
+fuzzy_insert_result(struct fuzzy_client_session *session,
+ const struct rspamd_fuzzy_reply *rep,
+ struct rspamd_fuzzy_cmd *cmd,
+ struct fuzzy_cmd_io *io,
+ guint flag)
+{
+ const gchar *symbol;
+ struct fuzzy_mapping *map;
+ struct rspamd_task *task = session->task;
+ double weight;
+ double nval;
+ guchar buf[2048];
+ const gchar *type = "bin";
+ struct fuzzy_client_result *res;
+ gboolean is_fuzzy = FALSE;
+ gchar hexbuf[rspamd_cryptobox_HASHBYTES * 2 + 1];
+ /* Discriminate scores for small images */
+ static const guint short_image_limit = 32 * 1024;
+
+ /* Get mapping by flag */
+ if ((map =
+ g_hash_table_lookup(session->rule->mappings,
+ GINT_TO_POINTER(rep->v1.flag))) == NULL) {
+ /* Default symbol and default weight */
+ symbol = session->rule->symbol;
+ weight = session->rule->max_score;
+ }
+ else {
+ /* Get symbol and weight from map */
+ symbol = map->symbol;
+ weight = map->weight;
+ }
+
+ res = rspamd_mempool_alloc0(task->task_pool, sizeof(*res));
+ res->prob = rep->v1.prob;
+ res->symbol = symbol;
+ /*
+ * Hash is assumed to be found if probability is more than 0.5
+ * In that case `value` means number of matches
+ * Otherwise `value` means error code
+ */
+
+ nval = fuzzy_normalize(rep->v1.value, weight);
+
+ if (io) {
+ if ((io->flags & FUZZY_CMD_FLAG_IMAGE)) {
+ if (!io->part || io->part->parsed_data.len <= short_image_limit) {
+ nval *= rspamd_normalize_probability(rep->v1.prob, 0.5);
+ }
+
+ type = "img";
+ res->type = FUZZY_RESULT_IMG;
+ }
+ else {
+ /* Calc real probability */
+ nval *= sqrtf(rep->v1.prob);
+
+ if (cmd->shingles_count > 0) {
+ type = "txt";
+ res->type = FUZZY_RESULT_TXT;
+ }
+ else {
+ if (io->flags & FUZZY_CMD_FLAG_CONTENT) {
+ type = "content";
+ res->type = FUZZY_RESULT_CONTENT;
+ }
+ else {
+ res->type = FUZZY_RESULT_BIN;
+ }
+ }
+ }
+ }
+
+ res->score = nval;
+
+ if (memcmp(rep->digest, cmd->digest, sizeof(rep->digest)) != 0) {
+ is_fuzzy = TRUE;
+ }
+
+ if (map != NULL || !session->rule->skip_unknown) {
+ GList *fuzzy_var;
+ rspamd_fstring_t *hex_result;
+ gchar timebuf[64];
+ struct tm tm_split;
+
+ if (session->rule->skip_map) {
+ rspamd_encode_hex_buf(cmd->digest, sizeof(cmd->digest),
+ hexbuf, sizeof(hexbuf) - 1);
+ hexbuf[sizeof(hexbuf) - 1] = '\0';
+ if (rspamd_match_hash_map(session->rule->skip_map, hexbuf,
+ sizeof(hexbuf) - 1)) {
+ return;
+ }
+ }
+
+ rspamd_encode_hex_buf(rep->digest, sizeof(rep->digest),
+ hexbuf, sizeof(hexbuf) - 1);
+ hexbuf[sizeof(hexbuf) - 1] = '\0';
+
+ rspamd_gmtime(rep->ts, &tm_split);
+ rspamd_snprintf(timebuf, sizeof(timebuf), "%02d.%02d.%4d %02d:%02d:%02d GMT",
+ tm_split.tm_mday,
+ tm_split.tm_mon + 1,
+ tm_split.tm_year + 1900,
+ tm_split.tm_hour, tm_split.tm_min, tm_split.tm_sec);
+
+ if (is_fuzzy) {
+ msg_notice_task(
+ "found fuzzy hash(%s) %s (%*xs requested) with weight: "
+ "%.2f, probability %.2f, in list: %s:%d%s; added on %s",
+ type,
+ hexbuf,
+ (gint) sizeof(cmd->digest), cmd->digest,
+ nval,
+ (gdouble) rep->v1.prob,
+ symbol,
+ rep->v1.flag,
+ map == NULL ? "(unknown)" : "",
+ timebuf);
+ }
+ else {
+ msg_notice_task(
+ "found exact fuzzy hash(%s) %s with weight: "
+ "%.2f, probability %.2f, in list: %s:%d%s; added on %s",
+ type,
+ hexbuf,
+ nval,
+ (gdouble) rep->v1.prob,
+ symbol,
+ rep->v1.flag,
+ map == NULL ? "(unknown)" : "",
+ timebuf);
+ }
+
+ rspamd_snprintf(buf,
+ sizeof(buf),
+ "%d:%*s:%.2f:%s",
+ rep->v1.flag,
+ (gint) MIN(rspamd_fuzzy_hash_len * 2, sizeof(rep->digest) * 2), hexbuf,
+ rep->v1.prob,
+ type);
+ res->option = rspamd_mempool_strdup(task->task_pool, buf);
+ g_ptr_array_add(session->results, res);
+
+ /* Store hex string in pool variable */
+ hex_result = rspamd_mempool_alloc(task->task_pool,
+ sizeof(rspamd_fstring_t) + sizeof(hexbuf));
+ memcpy(hex_result->str, hexbuf, sizeof(hexbuf));
+ hex_result->len = sizeof(hexbuf) - 1;
+ hex_result->allocated = (gsize) -1;
+ fuzzy_var = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_FUZZY_RESULT);
+
+ if (fuzzy_var == NULL) {
+ fuzzy_var = g_list_prepend(NULL, hex_result);
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_FUZZY_RESULT, fuzzy_var,
+ (rspamd_mempool_destruct_t) g_list_free);
+ }
+ else {
+ /* Not very efficient, but we don't really use it intensively */
+ fuzzy_var = g_list_append(fuzzy_var, hex_result);
+ }
+ }
+}
+
+static gint
+fuzzy_check_try_read(struct fuzzy_client_session *session)
+{
+ struct rspamd_task *task;
+ const struct rspamd_fuzzy_reply *rep;
+ struct rspamd_fuzzy_cmd *cmd = NULL;
+ struct fuzzy_cmd_io *io = NULL;
+ gint r, ret;
+ guchar buf[2048], *p;
+
+ task = session->task;
+
+ if ((r = read(session->fd, buf, sizeof(buf) - 1)) == -1) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
+ return 0;
+ }
+ else {
+ return -1;
+ }
+ }
+ else {
+ p = buf;
+
+ ret = 0;
+
+ while ((rep = fuzzy_process_reply(&p, &r,
+ session->commands, session->rule, &cmd, &io)) != NULL) {
+ if (rep->v1.prob > 0.5) {
+ if (cmd->cmd == FUZZY_CHECK) {
+ fuzzy_insert_result(session, rep, cmd, io, rep->v1.flag);
+ }
+ else if (cmd->cmd == FUZZY_STAT) {
+ /*
+ * We store fuzzy stat in the following way:
+ * 1) We store fuzzy hashes as a hash of rspamd_fuzzy_stat_entry
+ * 2) We store the resulting hash table inside pool variable `fuzzy_stat`
+ */
+ struct rspamd_fuzzy_stat_entry *pval;
+ GHashTable *stats_hash;
+
+ stats_hash = (GHashTable *) rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_FUZZY_STAT);
+
+ if (stats_hash == NULL) {
+ stats_hash = g_hash_table_new(rspamd_str_hash, rspamd_str_equal);
+ rspamd_mempool_set_variable(task->task_pool, RSPAMD_MEMPOOL_FUZZY_STAT,
+ stats_hash,
+ (rspamd_mempool_destruct_t) g_hash_table_destroy);
+ }
+
+ pval = g_hash_table_lookup(stats_hash, session->rule->name);
+
+ if (pval == NULL) {
+ pval = rspamd_mempool_alloc(task->task_pool,
+ sizeof(*pval));
+ pval->name = rspamd_mempool_strdup(task->task_pool,
+ session->rule->name);
+ /* Safe, as pval->name is owned by the pool */
+ g_hash_table_insert(stats_hash, (char *) pval->name, pval);
+ }
+
+ pval->fuzzy_cnt = (((guint64) rep->v1.value) << 32) + rep->v1.flag;
+ }
+ }
+ else if (rep->v1.value == 403) {
+ rspamd_task_insert_result(task, "FUZZY_BLOCKED", 0.0,
+ session->rule->name);
+ }
+ else if (rep->v1.value == 401) {
+ if (cmd->cmd != FUZZY_CHECK) {
+ msg_info_task(
+ "fuzzy check error for %d: skipped by server",
+ rep->v1.flag);
+ }
+ }
+ else if (rep->v1.value != 0) {
+ msg_info_task(
+ "fuzzy check error for %d: unknown error (%d)",
+ rep->v1.flag,
+ rep->v1.value);
+ }
+
+ ret = 1;
+ }
+ }
+
+ return ret;
+}
+
+static void
+fuzzy_insert_metric_results(struct rspamd_task *task, struct fuzzy_rule *rule,
+ GPtrArray *results)
+{
+ struct fuzzy_client_result *res;
+ guint i;
+ gboolean seen_text_hash = FALSE,
+ seen_img_hash = FALSE,
+ seen_text_part = FALSE,
+ seen_long_text = FALSE;
+ gdouble prob_txt = 0.0, mult;
+ struct rspamd_mime_text_part *tp;
+
+ /* About 5 words */
+ static const unsigned int text_length_cutoff = 25;
+
+ PTR_ARRAY_FOREACH(results, i, res)
+ {
+ if (res->type == FUZZY_RESULT_TXT) {
+ seen_text_hash = TRUE;
+ prob_txt = MAX(prob_txt, res->prob);
+ }
+ else if (res->type == FUZZY_RESULT_IMG) {
+ seen_img_hash = TRUE;
+ }
+ }
+
+ if (task->message) {
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, tp)
+ {
+ if (!IS_TEXT_PART_EMPTY(tp) && tp->utf_words != NULL && tp->utf_words->len > 0) {
+ seen_text_part = TRUE;
+
+ if (tp->utf_stripped_text.magic == UTEXT_MAGIC) {
+ if (utext_isLengthExpensive(&tp->utf_stripped_text)) {
+ seen_long_text =
+ utext_nativeLength(&tp->utf_stripped_text) >
+ text_length_cutoff;
+ }
+ else {
+ /* Cannot directly calculate length */
+ seen_long_text =
+ (tp->utf_stripped_content->len / 2) >
+ text_length_cutoff;
+ }
+ }
+ }
+ }
+ }
+
+ PTR_ARRAY_FOREACH(results, i, res)
+ {
+ mult = 1.0;
+
+ if (res->type == FUZZY_RESULT_IMG) {
+ if (!seen_text_hash) {
+ if (seen_long_text) {
+ mult *= 0.25;
+ }
+ else if (seen_text_part) {
+ /* We have some short text + image */
+ mult *= 0.9;
+ }
+ /* Otherwise apply full score */
+ }
+ else if (prob_txt < 0.75) {
+ /* Penalize sole image without matching text */
+ if (prob_txt > 0.5) {
+ mult *= prob_txt;
+ }
+ else {
+ mult *= 0.5; /* cutoff */
+ }
+ }
+ }
+ else if (res->type == FUZZY_RESULT_TXT) {
+ if (seen_img_hash) {
+ /* Slightly increase score */
+ mult = 1.1;
+ }
+ }
+
+ gdouble weight = res->score * mult;
+
+ if (!isnan(rule->weight_threshold)) {
+ if (weight >= rule->weight_threshold) {
+ rspamd_task_insert_result_single(task, res->symbol,
+ weight, res->option);
+ }
+ else {
+ msg_info_task("%s is not added: weight=%.4f below threshold",
+ res->symbol, weight);
+ }
+ }
+ else {
+ rspamd_task_insert_result_single(task, res->symbol,
+ weight, res->option);
+ }
+ }
+}
+
+static gboolean
+fuzzy_check_session_is_completed(struct fuzzy_client_session *session)
+{
+ struct fuzzy_cmd_io *io;
+ guint nreplied = 0, i;
+
+ rspamd_upstream_ok(session->server);
+
+ for (i = 0; i < session->commands->len; i++) {
+ io = g_ptr_array_index(session->commands, i);
+
+ if (io->flags & FUZZY_CMD_FLAG_REPLIED) {
+ nreplied++;
+ }
+ }
+
+ if (nreplied == session->commands->len) {
+ fuzzy_insert_metric_results(session->task, session->rule, session->results);
+
+ if (session->item) {
+ rspamd_symcache_item_async_dec_check(session->task, session->item, M);
+ }
+
+ rspamd_session_remove_event(session->task->s, fuzzy_io_fin, session);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+/* Fuzzy check timeout callback */
+static void
+fuzzy_check_timer_callback(gint fd, short what, void *arg)
+{
+ struct fuzzy_client_session *session = arg;
+ struct rspamd_task *task;
+
+ task = session->task;
+
+ /* We might be here because of other checks being slow */
+ if (fuzzy_check_try_read(session) > 0) {
+ if (fuzzy_check_session_is_completed(session)) {
+ return;
+ }
+ }
+
+ if (session->retransmits >= session->rule->retransmits) {
+ msg_err_task("got IO timeout with server %s(%s), after %d/%d retransmits",
+ rspamd_upstream_name(session->server),
+ rspamd_inet_address_to_string_pretty(
+ rspamd_upstream_addr_cur(session->server)),
+ session->retransmits,
+ session->rule->retransmits);
+ rspamd_upstream_fail(session->server, TRUE, "timeout");
+
+ if (session->item) {
+ rspamd_symcache_item_async_dec_check(session->task, session->item, M);
+ }
+ rspamd_session_remove_event(session->task->s, fuzzy_io_fin, session);
+ }
+ else {
+ /* Plan write event */
+ rspamd_ev_watcher_reschedule(session->event_loop,
+ &session->ev, EV_READ | EV_WRITE);
+ session->retransmits++;
+ }
+}
+
+/* Fuzzy check callback */
+static void
+fuzzy_check_io_callback(gint fd, short what, void *arg)
+{
+ struct fuzzy_client_session *session = arg;
+ struct rspamd_task *task;
+ gint r;
+
+ enum {
+ return_error = 0,
+ return_want_more,
+ return_finished
+ } ret = return_error;
+
+ task = session->task;
+
+ if ((what & EV_READ) || session->state == 1) {
+ /* Try to read reply */
+ r = fuzzy_check_try_read(session);
+
+ switch (r) {
+ case 0:
+ if (what & EV_READ) {
+ ret = return_want_more;
+ }
+ else {
+ if (what & EV_WRITE) {
+ /* Retransmit attempt */
+ if (!fuzzy_cmd_vector_to_wire(fd, session->commands)) {
+ ret = return_error;
+ }
+ else {
+ session->state = 1;
+ ret = return_want_more;
+ }
+ }
+ else {
+ /* It is actually time out */
+ fuzzy_check_timer_callback(fd, what, arg);
+ return;
+ }
+ }
+ break;
+ case 1:
+ ret = return_finished;
+ break;
+ default:
+ ret = return_error;
+ break;
+ }
+ }
+ else if (what & EV_WRITE) {
+ if (!fuzzy_cmd_vector_to_wire(fd, session->commands)) {
+ ret = return_error;
+ }
+ else {
+ session->state = 1;
+ ret = return_want_more;
+ }
+ }
+ else {
+ fuzzy_check_timer_callback(fd, what, arg);
+ return;
+ }
+
+ if (ret == return_want_more) {
+ /* Processed write, switch to reading */
+ rspamd_ev_watcher_reschedule(session->event_loop,
+ &session->ev, EV_READ);
+ }
+ else if (ret == return_error) {
+ /* Error state */
+ msg_err_task("got error on IO with server %s(%s), on %s, %d, %s",
+ rspamd_upstream_name(session->server),
+ rspamd_inet_address_to_string_pretty(
+ rspamd_upstream_addr_cur(session->server)),
+ session->state == 1 ? "read" : "write",
+ errno,
+ strerror(errno));
+ rspamd_upstream_fail(session->server, TRUE, strerror(errno));
+
+ if (session->item) {
+ rspamd_symcache_item_async_dec_check(session->task, session->item, M);
+ }
+
+ rspamd_session_remove_event(session->task->s, fuzzy_io_fin, session);
+ }
+ else {
+ /* Read something from network */
+ if (!fuzzy_check_session_is_completed(session)) {
+ /* Need to read more */
+ rspamd_ev_watcher_reschedule(session->event_loop,
+ &session->ev, EV_READ);
+ }
+ }
+}
+
+
+static void
+fuzzy_controller_lua_fin(void *ud)
+{
+ struct fuzzy_learn_session *session = ud;
+
+ (*session->saved)--;
+
+ rspamd_ev_watcher_stop(session->event_loop, &session->ev);
+ close(session->fd);
+}
+
+/* Controller IO */
+
+static void
+fuzzy_controller_timer_callback(gint fd, short what, void *arg)
+{
+ struct fuzzy_learn_session *session = arg;
+ struct rspamd_task *task;
+
+ task = session->task;
+
+ if (session->retransmits >= session->rule->retransmits) {
+ rspamd_upstream_fail(session->server, TRUE, "timeout");
+ msg_err_task_check("got IO timeout with server %s(%s), "
+ "after %d/%d retransmits",
+ rspamd_upstream_name(session->server),
+ rspamd_inet_address_to_string_pretty(
+ rspamd_upstream_addr_cur(session->server)),
+ session->retransmits,
+ session->rule->retransmits);
+
+ if (session->session) {
+ rspamd_session_remove_event(session->session, fuzzy_controller_lua_fin,
+ session);
+ }
+ else {
+ if (session->http_entry) {
+ rspamd_controller_send_error(session->http_entry,
+ 500, "IO timeout with fuzzy storage");
+ }
+
+ if (*session->saved > 0) {
+ (*session->saved)--;
+ if (*session->saved == 0) {
+ if (session->http_entry) {
+ rspamd_task_free(session->task);
+ }
+
+ session->task = NULL;
+ }
+ }
+
+ if (session->http_entry) {
+ rspamd_http_connection_unref(session->http_entry->conn);
+ }
+
+ rspamd_ev_watcher_stop(session->event_loop,
+ &session->ev);
+ close(session->fd);
+ }
+ }
+ else {
+ /* Plan write event */
+ rspamd_ev_watcher_reschedule(session->event_loop,
+ &session->ev, EV_READ | EV_WRITE);
+ session->retransmits++;
+ }
+}
+
+static void
+fuzzy_controller_io_callback(gint fd, short what, void *arg)
+{
+ struct fuzzy_learn_session *session = arg;
+ const struct rspamd_fuzzy_reply *rep;
+ struct fuzzy_mapping *map;
+ struct rspamd_task *task;
+ guchar buf[2048], *p;
+ struct fuzzy_cmd_io *io;
+ struct rspamd_fuzzy_cmd *cmd = NULL;
+ const gchar *symbol, *ftype;
+ gint r;
+ enum {
+ return_error = 0,
+ return_want_more,
+ return_finished
+ } ret = return_want_more;
+ guint i, nreplied;
+ const gchar *op = "process";
+
+ task = session->task;
+
+ if (what & EV_READ) {
+ if ((r = read(fd, buf, sizeof(buf) - 1)) == -1) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
+ rspamd_ev_watcher_reschedule(session->event_loop,
+ &session->ev, EV_READ);
+ return;
+ }
+
+ msg_info_task("cannot process fuzzy hash for message: %s",
+ strerror(errno));
+ session->err.error_message = "read socket error";
+ session->err.error_code = errno;
+
+ ret = return_error;
+ }
+ else {
+ p = buf;
+ ret = return_want_more;
+
+ while ((rep = fuzzy_process_reply(&p, &r,
+ session->commands, session->rule, &cmd, &io)) != NULL) {
+ if ((map =
+ g_hash_table_lookup(session->rule->mappings,
+ GINT_TO_POINTER(rep->v1.flag))) == NULL) {
+ /* Default symbol and default weight */
+ symbol = session->rule->symbol;
+ }
+ else {
+ /* Get symbol and weight from map */
+ symbol = map->symbol;
+ }
+
+ ftype = "bin";
+
+ if (io) {
+ if ((io->flags & FUZZY_CMD_FLAG_IMAGE)) {
+ ftype = "img";
+ }
+ else if (io->flags & FUZZY_CMD_FLAG_CONTENT) {
+ ftype = "content";
+ }
+ else if (cmd->shingles_count > 0) {
+ ftype = "txt";
+ }
+
+ if (io->cmd.cmd == FUZZY_WRITE) {
+ op = "added";
+ }
+ else if (io->cmd.cmd == FUZZY_DEL) {
+ op = "deleted";
+ }
+ }
+
+ if (rep->v1.prob > 0.5) {
+ msg_info_task("%s fuzzy hash (%s) %*xs, list: %s:%d for "
+ "message <%s>",
+ op,
+ ftype,
+ (gint) sizeof(rep->digest), rep->digest,
+ symbol,
+ rep->v1.flag,
+ MESSAGE_FIELD_CHECK(session->task, message_id));
+ }
+ else {
+ if (rep->v1.value == 401) {
+ msg_info_task(
+ "fuzzy hash (%s) for message cannot be %s"
+ "<%s>, %*xs, "
+ "list %s:%d, skipped by server",
+ ftype,
+ op,
+ MESSAGE_FIELD_CHECK(session->task, message_id),
+ (gint) sizeof(rep->digest), rep->digest,
+ symbol,
+ rep->v1.flag);
+
+ session->err.error_message = "fuzzy hash is skipped";
+ session->err.error_code = rep->v1.value;
+ }
+ else {
+ msg_info_task(
+ "fuzzy hash (%s) for message cannot be %s"
+ "<%s>, %*xs, "
+ "list %s:%d, error: %d",
+ ftype,
+ op,
+ MESSAGE_FIELD_CHECK(session->task, message_id),
+ (gint) sizeof(rep->digest), rep->digest,
+ symbol,
+ rep->v1.flag,
+ rep->v1.value);
+
+ session->err.error_message = "process fuzzy error";
+ session->err.error_code = rep->v1.value;
+ }
+
+ ret = return_finished;
+ }
+ }
+
+ nreplied = 0;
+
+ for (i = 0; i < session->commands->len; i++) {
+ io = g_ptr_array_index(session->commands, i);
+
+ if (io->flags & FUZZY_CMD_FLAG_REPLIED) {
+ nreplied++;
+ }
+ }
+
+ if (nreplied == session->commands->len) {
+ ret = return_finished;
+ }
+ }
+ }
+ else if (what & EV_WRITE) {
+ /* Send commands to storage */
+ if (!fuzzy_cmd_vector_to_wire(fd, session->commands)) {
+ session->err.error_message = "write socket error";
+ session->err.error_code = errno;
+ ret = return_error;
+ }
+ }
+ else {
+ fuzzy_controller_timer_callback(fd, what, arg);
+
+ return;
+ }
+
+ if (ret == return_want_more) {
+ rspamd_ev_watcher_reschedule(session->event_loop,
+ &session->ev, EV_READ);
+
+ return;
+ }
+ else if (ret == return_error) {
+ msg_err_task("got error in IO with server %s(%s), %d, %s",
+ rspamd_upstream_name(session->server),
+ rspamd_inet_address_to_string_pretty(
+ rspamd_upstream_addr_cur(session->server)),
+ errno, strerror(errno));
+ rspamd_upstream_fail(session->server, FALSE, strerror(errno));
+ }
+
+ /*
+ * XXX: actually, we check merely a single reply, which is not correct...
+ * XXX: when we send a command, we do not check if *all* commands have been
+ * written
+ * XXX: please, please, change this code some day
+ */
+
+ if (session->session == NULL) {
+ (*session->saved)--;
+
+ if (session->http_entry) {
+ rspamd_http_connection_unref(session->http_entry->conn);
+ }
+
+ rspamd_ev_watcher_stop(session->event_loop, &session->ev);
+ close(session->fd);
+
+ if (*session->saved == 0) {
+ goto cleanup;
+ }
+ }
+ else {
+ /* Lua handler */
+ rspamd_session_remove_event(session->session, fuzzy_controller_lua_fin, session);
+ }
+
+ return;
+
+cleanup:
+ /*
+ * When we send learn commands to fuzzy storages, this code is executed
+ * *once* when we have queried all storages. We also don't know which
+ * storage has been failed.
+ *
+ * Therefore, we cleanup sessions earlier and actually this code is wrong.
+ */
+
+ if (session->err.error_code != 0) {
+ if (session->http_entry) {
+ rspamd_controller_send_error(session->http_entry,
+ session->err.error_code, session->err.error_message);
+ }
+ }
+ else {
+ rspamd_upstream_ok(session->server);
+
+ if (session->http_entry) {
+ ucl_object_t *reply, *hashes;
+ gchar hexbuf[rspamd_cryptobox_HASHBYTES * 2 + 1];
+
+ reply = ucl_object_typed_new(UCL_OBJECT);
+
+ ucl_object_insert_key(reply, ucl_object_frombool(true),
+ "success", 0, false);
+ hashes = ucl_object_typed_new(UCL_ARRAY);
+
+ for (i = 0; i < session->commands->len; i++) {
+ io = g_ptr_array_index(session->commands, i);
+
+ rspamd_snprintf(hexbuf, sizeof(hexbuf), "%*xs",
+ (gint) sizeof(io->cmd.digest), io->cmd.digest);
+ ucl_array_append(hashes, ucl_object_fromstring(hexbuf));
+ }
+
+ ucl_object_insert_key(reply, hashes, "hashes", 0, false);
+ rspamd_controller_send_ucl(session->http_entry, reply);
+ ucl_object_unref(reply);
+ }
+ }
+
+ if (session->task != NULL) {
+ if (session->http_entry) {
+ rspamd_task_free(session->task);
+ }
+
+ session->task = NULL;
+ }
+}
+
+static GPtrArray *
+fuzzy_generate_commands(struct rspamd_task *task, struct fuzzy_rule *rule,
+ gint c, gint flag, guint32 value, guint flags)
+{
+ struct rspamd_mime_text_part *part;
+ struct rspamd_mime_part *mime_part;
+ struct rspamd_image *image;
+ struct fuzzy_cmd_io *io, *cur;
+ guint i, j;
+ GPtrArray *res = NULL;
+ gboolean check_part, fuzzy_check;
+
+ if (c == FUZZY_STAT) {
+ res = g_ptr_array_sized_new(1);
+
+ io = fuzzy_cmd_stat(rule, c, flag, value, task->task_pool);
+ if (io) {
+ g_ptr_array_add(res, io);
+ }
+
+ goto end;
+ }
+ else if (c == FUZZY_PING) {
+ res = g_ptr_array_sized_new(1);
+
+ io = fuzzy_cmd_ping(rule, task->task_pool);
+ if (io) {
+ g_ptr_array_add(res, io);
+ }
+
+ goto end;
+ }
+
+ if (task->message == NULL) {
+ goto end;
+ }
+
+ res = g_ptr_array_sized_new(MESSAGE_FIELD(task, parts)->len + 1);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, mime_part)
+ {
+ check_part = FALSE;
+ fuzzy_check = FALSE;
+
+ if (fuzzy_rule_check_mimepart(task, rule, mime_part, &check_part,
+ &fuzzy_check)) {
+ io = NULL;
+
+ if (check_part) {
+ if (mime_part->part_type == RSPAMD_MIME_PART_TEXT &&
+ !(flags & FUZZY_CHECK_FLAG_NOTEXT)) {
+ part = mime_part->specific.txt;
+
+ io = fuzzy_cmd_from_text_part(task, rule,
+ c,
+ flag,
+ value,
+ !fuzzy_check,
+ part,
+ mime_part);
+ }
+ else if (mime_part->part_type == RSPAMD_MIME_PART_IMAGE &&
+ !(flags & FUZZY_CHECK_FLAG_NOIMAGES)) {
+ image = mime_part->specific.img;
+
+ io = fuzzy_cmd_from_data_part(rule, c, flag, value,
+ task,
+ image->parent->digest,
+ mime_part);
+ io->flags |= FUZZY_CMD_FLAG_IMAGE;
+ }
+ else if (mime_part->part_type == RSPAMD_MIME_PART_CUSTOM_LUA) {
+ const struct rspamd_lua_specific_part *lua_spec;
+
+ lua_spec = &mime_part->specific.lua_specific;
+
+ if (lua_spec->type == RSPAMD_LUA_PART_TABLE) {
+ lua_State *L = (lua_State *) task->cfg->lua_state;
+ gint old_top;
+
+ old_top = lua_gettop(L);
+ /* Push table */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, lua_spec->cbref);
+ lua_pushstring(L, "fuzzy_hashes");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ gint tbl_pos = lua_gettop(L);
+
+ for (lua_pushnil(L); lua_next(L, tbl_pos);
+ lua_pop(L, 1)) {
+ const gchar *h = NULL;
+ gsize hlen = 0;
+
+ if (lua_isstring(L, -1)) {
+ h = lua_tolstring(L, -1, &hlen);
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t;
+
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ h = t->start;
+ hlen = t->len;
+ }
+ }
+
+ if (hlen == rspamd_cryptobox_HASHBYTES) {
+ io = fuzzy_cmd_from_data_part(rule, c,
+ flag, value,
+ task,
+ (guchar *) h,
+ mime_part);
+
+ if (io) {
+ io->flags |= FUZZY_CMD_FLAG_CONTENT;
+ g_ptr_array_add(res, io);
+ }
+ }
+ }
+ }
+
+ lua_settop(L, old_top);
+
+ /*
+ * Add part itself as well
+ */
+ io = fuzzy_cmd_from_data_part(rule, c,
+ flag, value,
+ task,
+ mime_part->digest,
+ mime_part);
+ }
+ }
+ else {
+ io = fuzzy_cmd_from_data_part(rule, c, flag, value,
+ task,
+ mime_part->digest, mime_part);
+ }
+
+ if (io) {
+ gboolean skip_existing = FALSE;
+
+ PTR_ARRAY_FOREACH(res, j, cur)
+ {
+ if (memcmp(cur->cmd.digest, io->cmd.digest,
+ sizeof(io->cmd.digest)) == 0) {
+ skip_existing = TRUE;
+ break;
+ }
+ }
+
+ if (!skip_existing) {
+ g_ptr_array_add(res, io);
+ }
+ }
+ }
+ }
+ }
+
+end:
+ if (res && res->len == 0) {
+ g_ptr_array_free(res, TRUE);
+
+ return NULL;
+ }
+
+ return res;
+}
+
+
+static inline void
+register_fuzzy_client_call(struct rspamd_task *task,
+ struct fuzzy_rule *rule,
+ GPtrArray *commands)
+{
+ struct fuzzy_client_session *session;
+ struct upstream *selected;
+ rspamd_inet_addr_t *addr;
+ gint sock;
+
+ if (!rspamd_session_blocked(task->s)) {
+ /* Get upstream */
+ selected = rspamd_upstream_get(rule->servers, RSPAMD_UPSTREAM_ROUND_ROBIN,
+ NULL, 0);
+ if (selected) {
+ addr = rspamd_upstream_addr_next(selected);
+ if ((sock = rspamd_inet_address_connect(addr, SOCK_DGRAM, TRUE)) == -1) {
+ msg_warn_task("cannot connect to %s(%s), %d, %s",
+ rspamd_upstream_name(selected),
+ rspamd_inet_address_to_string_pretty(addr),
+ errno,
+ strerror(errno));
+ rspamd_upstream_fail(selected, TRUE, strerror(errno));
+ g_ptr_array_free(commands, TRUE);
+ }
+ else {
+ /* Create session for a socket */
+ session =
+ rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct fuzzy_client_session));
+ session->state = 0;
+ session->commands = commands;
+ session->task = task;
+ session->fd = sock;
+ session->server = selected;
+ session->rule = rule;
+ session->results = g_ptr_array_sized_new(32);
+ session->event_loop = task->event_loop;
+
+ rspamd_ev_watcher_init(&session->ev,
+ sock,
+ EV_WRITE,
+ fuzzy_check_io_callback,
+ session);
+ rspamd_ev_watcher_start(session->event_loop, &session->ev,
+ rule->io_timeout);
+
+ rspamd_session_add_event(task->s, fuzzy_io_fin, session, M);
+ session->item = rspamd_symcache_get_cur_item(task);
+
+ if (session->item) {
+ rspamd_symcache_item_async_inc(task, session->item, M);
+ }
+ }
+ }
+ }
+}
+
+/* This callback is called when we check message in fuzzy hashes storage */
+static void
+fuzzy_symbol_callback(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ void *unused)
+{
+ struct fuzzy_rule *rule;
+ guint i;
+ GPtrArray *commands;
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(task->cfg);
+
+ if (!fuzzy_module_ctx->enabled) {
+ rspamd_symcache_finalize_item(task, item);
+
+ return;
+ }
+
+ /* Check whitelist */
+ if (fuzzy_module_ctx->whitelist) {
+ if (rspamd_match_radix_map_addr(fuzzy_module_ctx->whitelist,
+ task->from_addr) != NULL) {
+ msg_info_task("<%s>, address %s is whitelisted, skip fuzzy check",
+ MESSAGE_FIELD(task, message_id),
+ rspamd_inet_address_to_string(task->from_addr));
+ rspamd_symcache_finalize_item(task, item);
+
+ return;
+ }
+ }
+
+ rspamd_symcache_item_async_inc(task, item, M);
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ commands = fuzzy_generate_commands(task, rule, FUZZY_CHECK, 0, 0, 0);
+
+ if (commands != NULL) {
+ register_fuzzy_client_call(task, rule, commands);
+ }
+ }
+
+ rspamd_symcache_item_async_dec_check(task, item, M);
+}
+
+void fuzzy_stat_command(struct rspamd_task *task)
+{
+ struct fuzzy_rule *rule;
+ guint i;
+ GPtrArray *commands;
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(task->cfg);
+
+ if (!fuzzy_module_ctx->enabled) {
+ return;
+ }
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ commands = fuzzy_generate_commands(task, rule, FUZZY_STAT, 0, 0, 0);
+ if (commands != NULL) {
+ register_fuzzy_client_call(task, rule, commands);
+ }
+ }
+}
+
+static inline gint
+register_fuzzy_controller_call(struct rspamd_http_connection_entry *entry,
+ struct fuzzy_rule *rule,
+ struct rspamd_task *task,
+ GPtrArray *commands,
+ gint *saved)
+{
+ struct fuzzy_learn_session *s;
+ struct upstream *selected;
+ rspamd_inet_addr_t *addr;
+ struct rspamd_controller_session *session = entry->ud;
+ gint sock;
+ gint ret = -1;
+
+ /* Get upstream */
+
+ while ((selected = rspamd_upstream_get_forced(rule->servers,
+ RSPAMD_UPSTREAM_SEQUENTIAL, NULL, 0))) {
+ /* Create UDP socket */
+ addr = rspamd_upstream_addr_next(selected);
+
+ if ((sock = rspamd_inet_address_connect(addr,
+ SOCK_DGRAM, TRUE)) == -1) {
+ msg_warn_task("cannot connect to fuzzy storage %s (%s rule): %s",
+ rspamd_inet_address_to_string_pretty(addr),
+ rule->name,
+ strerror(errno));
+ rspamd_upstream_fail(selected, TRUE, strerror(errno));
+ }
+ else {
+ s =
+ rspamd_mempool_alloc0(session->pool,
+ sizeof(struct fuzzy_learn_session));
+
+ s->task = task;
+ s->commands = commands;
+ s->http_entry = entry;
+ s->server = selected;
+ s->saved = saved;
+ s->fd = sock;
+ s->rule = rule;
+ s->event_loop = task->event_loop;
+ /* We ref connection to avoid freeing before we process fuzzy rule */
+ rspamd_http_connection_ref(entry->conn);
+
+ rspamd_ev_watcher_init(&s->ev,
+ sock,
+ EV_WRITE,
+ fuzzy_controller_io_callback,
+ s);
+ rspamd_ev_watcher_start(s->event_loop, &s->ev, rule->io_timeout);
+
+ (*saved)++;
+ ret = 1;
+ }
+ }
+
+ return ret;
+}
+
+static void
+fuzzy_process_handler(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg, gint cmd, gint value, gint flag,
+ struct fuzzy_ctx *ctx, gboolean is_hash, guint flags)
+{
+ struct fuzzy_rule *rule;
+ struct rspamd_controller_session *session = conn_ent->ud;
+ struct rspamd_task *task, **ptask;
+ gboolean processed = FALSE, skip = FALSE;
+ gint res = 0;
+ guint i;
+ GPtrArray *commands;
+ lua_State *L;
+ gint r, *saved, rules = 0, err_idx;
+ struct fuzzy_ctx *fuzzy_module_ctx;
+
+ /* Prepare task */
+ task = rspamd_task_new(session->wrk, session->cfg, NULL,
+ session->lang_det, conn_ent->rt->event_loop, FALSE);
+ task->cfg = ctx->cfg;
+ saved = rspamd_mempool_alloc0(session->pool, sizeof(gint));
+ fuzzy_module_ctx = fuzzy_get_context(ctx->cfg);
+
+ if (!is_hash) {
+ /* Allocate message from string */
+ /* XXX: what about encrypted messages ? */
+ task->msg.begin = msg->body_buf.begin;
+ task->msg.len = msg->body_buf.len;
+
+ r = rspamd_message_parse(task);
+
+ if (r == -1) {
+ msg_warn_task("<%s>: cannot process message for fuzzy",
+ MESSAGE_FIELD(task, message_id));
+ rspamd_task_free(task);
+ rspamd_controller_send_error(conn_ent, 400,
+ "Message processing error");
+
+ return;
+ }
+
+ rspamd_message_process(task);
+ }
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ if (rule->read_only) {
+ continue;
+ }
+
+ /* Check for flag */
+ if (g_hash_table_lookup(rule->mappings,
+ GINT_TO_POINTER(flag)) == NULL) {
+ msg_info_task("skip rule %s as it has no flag %d defined"
+ " false",
+ rule->name, flag);
+ continue;
+ }
+
+ /* Check learn condition */
+ if (rule->learn_condition_cb != -1) {
+ skip = FALSE;
+ L = session->cfg->lua_state;
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, rule->learn_condition_cb);
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ if (lua_pcall(L, 1, LUA_MULTRET, err_idx) != 0) {
+ msg_err_task("call to fuzzy learn condition failed: %s",
+ lua_tostring(L, -1));
+ }
+ else {
+ if (lua_gettop(L) > err_idx + 1) {
+ /* 2 return values */
+ skip = !(lua_toboolean(L, err_idx + 1));
+
+ if (lua_isnumber(L, err_idx + 2)) {
+ msg_info_task("learn condition changed flag from %d to "
+ "%d",
+ flag,
+ (gint) lua_tonumber(L, err_idx + 2));
+ flag = lua_tonumber(L, err_idx + 2);
+ }
+ }
+ else {
+ if (lua_isboolean(L, err_idx + 1)) {
+ skip = !(lua_toboolean(L, err_idx + 1));
+ }
+ else {
+ msg_warn_task("set skip for rule %s as its condition "
+ "callback returned"
+ " a valid boolean",
+ rule->name);
+ skip = TRUE;
+ }
+ }
+ }
+
+ /* Result + error function */
+ lua_settop(L, err_idx - 1);
+
+ if (skip) {
+ msg_info_task("skip rule %s by condition callback",
+ rule->name);
+ continue;
+ }
+ }
+
+ rules++;
+
+ res = 0;
+
+ if (is_hash) {
+ GPtrArray *args;
+ const rspamd_ftok_t *arg;
+ guint j;
+
+ args = rspamd_http_message_find_header_multiple(msg, "Hash");
+
+ if (args) {
+ struct fuzzy_cmd_io *io;
+ commands = g_ptr_array_sized_new(args->len);
+
+ for (j = 0; j < args->len; j++) {
+ arg = g_ptr_array_index(args, j);
+ io = fuzzy_cmd_hash(rule, cmd, arg, flag, value,
+ task->task_pool);
+
+ if (io) {
+ g_ptr_array_add(commands, io);
+ }
+ }
+
+ res = register_fuzzy_controller_call(conn_ent,
+ rule,
+ task,
+ commands,
+ saved);
+ rspamd_mempool_add_destructor(task->task_pool,
+ rspamd_ptr_array_free_hard, commands);
+ g_ptr_array_free(args, TRUE);
+ }
+ else {
+ rspamd_controller_send_error(conn_ent, 400,
+ "No hash defined");
+ rspamd_task_free(task);
+ return;
+ }
+ }
+ else {
+ commands = fuzzy_generate_commands(task, rule, cmd, flag, value,
+ flags);
+ if (commands != NULL) {
+ res = register_fuzzy_controller_call(conn_ent,
+ rule,
+ task,
+ commands,
+ saved);
+ rspamd_mempool_add_destructor(task->task_pool,
+ rspamd_ptr_array_free_hard, commands);
+ }
+ }
+
+ if (res > 0) {
+ processed = TRUE;
+ }
+ }
+
+ if (res == -1) {
+ if (!processed) {
+ msg_warn_task("cannot send fuzzy request: %s",
+ strerror(errno));
+ rspamd_controller_send_error(conn_ent, 400, "Message sending error");
+ rspamd_task_free(task);
+
+ return;
+ }
+ else {
+ /* Some rules failed and some rules are OK */
+ msg_warn_task("some rules are not processed, but we still sent this request");
+ }
+ }
+ else if (!processed) {
+ if (rules) {
+ msg_warn_task("no content to generate fuzzy");
+ rspamd_controller_send_error(conn_ent, 404,
+ "No content to generate fuzzy for flag %d", flag);
+ }
+ else {
+ if (skip) {
+ rspamd_controller_send_error(conn_ent, 403,
+ "Message is conditionally skipped for flag %d", flag);
+ }
+ else {
+ msg_warn_task("no fuzzy rules found for flag %d", flag);
+ rspamd_controller_send_error(conn_ent, 404,
+ "No fuzzy rules matched for flag %d", flag);
+ }
+ }
+ rspamd_task_free(task);
+ }
+}
+
+static int
+fuzzy_controller_handler(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg, struct module_ctx *ctx, gint cmd,
+ gboolean is_hash)
+{
+ const rspamd_ftok_t *arg;
+ glong value = 1, flag = 0, send_flags = 0;
+ struct fuzzy_ctx *fuzzy_module_ctx = (struct fuzzy_ctx *) ctx;
+
+ if (!fuzzy_module_ctx->enabled) {
+ msg_err("fuzzy_check module is not enabled");
+ rspamd_controller_send_error(conn_ent, 500, "Module disabled");
+ return 0;
+ }
+
+ if (fuzzy_module_ctx->fuzzy_rules == NULL) {
+ msg_err("fuzzy_check module has no rules defined");
+ rspamd_controller_send_error(conn_ent, 500, "Module has no rules");
+ return 0;
+ }
+
+ /* Get size */
+ arg = rspamd_http_message_find_header(msg, "Weight");
+ if (arg) {
+ errno = 0;
+
+ if (!rspamd_strtol(arg->begin, arg->len, &value)) {
+ msg_info("error converting numeric argument %T", arg);
+ }
+ }
+
+ arg = rspamd_http_message_find_header(msg, "Flag");
+ if (arg) {
+ errno = 0;
+
+ if (!rspamd_strtol(arg->begin, arg->len, &flag)) {
+ msg_info("error converting numeric argument %T", arg);
+ flag = 0;
+ }
+ }
+ else {
+ flag = 0;
+ arg = rspamd_http_message_find_header(msg, "Symbol");
+
+ /* Search flag by symbol */
+ if (arg) {
+ struct fuzzy_rule *rule;
+ guint i;
+ GHashTableIter it;
+ gpointer k, v;
+ struct fuzzy_mapping *map;
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ if (flag != 0) {
+ break;
+ }
+
+ g_hash_table_iter_init(&it, rule->mappings);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ map = v;
+
+ if (strlen(map->symbol) == arg->len &&
+ rspamd_lc_cmp(map->symbol, arg->begin, arg->len) == 0) {
+ flag = map->fuzzy_flag;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ if (flag == 0) {
+ msg_err("no flag defined to learn fuzzy");
+ rspamd_controller_send_error(conn_ent, 404, "Unknown or missing flag");
+ return 0;
+ }
+
+ arg = rspamd_http_message_find_header(msg, "Skip-Images");
+ if (arg) {
+ send_flags |= FUZZY_CHECK_FLAG_NOIMAGES;
+ }
+
+ arg = rspamd_http_message_find_header(msg, "Skip-Attachments");
+ if (arg) {
+ send_flags |= FUZZY_CHECK_FLAG_NOATTACHMENTS;
+ }
+
+ arg = rspamd_http_message_find_header(msg, "Skip-Text");
+ if (arg) {
+ send_flags |= FUZZY_CHECK_FLAG_NOTEXT;
+ }
+
+ fuzzy_process_handler(conn_ent, msg, cmd, value, flag,
+ (struct fuzzy_ctx *) ctx, is_hash, send_flags);
+
+ return 0;
+}
+
+static inline gint
+fuzzy_check_send_lua_learn(struct fuzzy_rule *rule,
+ struct rspamd_task *task,
+ GPtrArray *commands,
+ gint *saved)
+{
+ struct fuzzy_learn_session *s;
+ struct upstream *selected;
+ rspamd_inet_addr_t *addr;
+ gint sock;
+ gint ret = -1;
+
+ /* Get upstream */
+ if (!rspamd_session_blocked(task->s)) {
+ while ((selected = rspamd_upstream_get(rule->servers,
+ RSPAMD_UPSTREAM_SEQUENTIAL, NULL, 0))) {
+ /* Create UDP socket */
+ addr = rspamd_upstream_addr_next(selected);
+
+ if ((sock = rspamd_inet_address_connect(addr,
+ SOCK_DGRAM, TRUE)) == -1) {
+ rspamd_upstream_fail(selected, TRUE, strerror(errno));
+ }
+ else {
+ s =
+ rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct fuzzy_learn_session));
+ s->task = task;
+ s->commands = commands;
+ s->http_entry = NULL;
+ s->server = selected;
+ s->saved = saved;
+ s->fd = sock;
+ s->rule = rule;
+ s->session = task->s;
+ s->event_loop = task->event_loop;
+
+ rspamd_ev_watcher_init(&s->ev,
+ sock,
+ EV_WRITE,
+ fuzzy_controller_io_callback,
+ s);
+ rspamd_ev_watcher_start(s->event_loop, &s->ev,
+ rule->io_timeout);
+
+ rspamd_session_add_event(task->s,
+ fuzzy_controller_lua_fin,
+ s,
+ M);
+
+ (*saved)++;
+ ret = 1;
+ }
+ }
+ }
+
+ return ret;
+}
+
+static gboolean
+fuzzy_check_lua_process_learn(struct rspamd_task *task,
+ gint cmd, gint value, gint flag, guint send_flags)
+{
+ struct fuzzy_rule *rule;
+ gboolean processed = FALSE, res = TRUE;
+ guint i;
+ GPtrArray *commands;
+ gint *saved, rules = 0;
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(task->cfg);
+
+ saved = rspamd_mempool_alloc0(task->task_pool, sizeof(gint));
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ if (!res) {
+ break;
+ }
+ if (rule->read_only) {
+ continue;
+ }
+
+ /* Check for flag */
+ if (g_hash_table_lookup(rule->mappings,
+ GINT_TO_POINTER(flag)) == NULL) {
+ msg_info_task("skip rule %s as it has no flag %d defined"
+ " false",
+ rule->name, flag);
+ continue;
+ }
+
+ rules++;
+
+ res = 0;
+ commands = fuzzy_generate_commands(task, rule, cmd, flag,
+ value, send_flags);
+
+ if (commands != NULL) {
+ res = fuzzy_check_send_lua_learn(rule, task, commands,
+ saved);
+ rspamd_mempool_add_destructor(task->task_pool,
+ rspamd_ptr_array_free_hard, commands);
+ }
+
+ if (res) {
+ processed = TRUE;
+ }
+ }
+
+ if (res == -1) {
+ msg_warn_task("cannot send fuzzy request: %s",
+ strerror(errno));
+ }
+ else if (!processed) {
+ if (rules) {
+ msg_warn_task("no content to generate fuzzy");
+
+ return FALSE;
+ }
+ else {
+ msg_warn_task("no fuzzy rules found for flag %d", flag);
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+static gint
+fuzzy_lua_learn_handler(lua_State *L)
+{
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ guint flag = 0, weight = 1, send_flags = 0;
+ const gchar *symbol;
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(task->cfg);
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ flag = lua_tointeger(L, 2);
+ }
+ else if (lua_type(L, 2) == LUA_TSTRING) {
+ struct fuzzy_rule *rule;
+ guint i;
+ GHashTableIter it;
+ gpointer k, v;
+ struct fuzzy_mapping *map;
+
+ symbol = lua_tostring(L, 2);
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ if (flag != 0) {
+ break;
+ }
+
+ g_hash_table_iter_init(&it, rule->mappings);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ map = v;
+
+ if (g_ascii_strcasecmp(symbol, map->symbol) == 0) {
+ flag = map->fuzzy_flag;
+ break;
+ }
+ }
+ }
+ }
+
+ if (flag == 0) {
+ return luaL_error(L, "bad flag");
+ }
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ weight = lua_tonumber(L, 3);
+ }
+
+ if (lua_type(L, 4) == LUA_TTABLE) {
+ const gchar *sf;
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ sf = lua_tostring(L, -1);
+
+ if (sf) {
+ if (g_ascii_strcasecmp(sf, "noimages") == 0) {
+ send_flags |= FUZZY_CHECK_FLAG_NOIMAGES;
+ }
+ else if (g_ascii_strcasecmp(sf, "noattachments") == 0) {
+ send_flags |= FUZZY_CHECK_FLAG_NOATTACHMENTS;
+ }
+ else if (g_ascii_strcasecmp(sf, "notext") == 0) {
+ send_flags |= FUZZY_CHECK_FLAG_NOTEXT;
+ }
+ }
+ }
+ }
+
+ lua_pushboolean(L,
+ fuzzy_check_lua_process_learn(task, FUZZY_WRITE, weight, flag,
+ send_flags));
+ return 1;
+}
+
+static gint
+fuzzy_lua_unlearn_handler(lua_State *L)
+{
+ struct rspamd_task *task = lua_check_task(L, 1);
+ if (task == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ guint flag = 0, weight = 1.0, send_flags = 0;
+ const gchar *symbol;
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(task->cfg);
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ flag = lua_tointeger(L, 2);
+ }
+ else if (lua_type(L, 2) == LUA_TSTRING) {
+ struct fuzzy_rule *rule;
+ guint i;
+ GHashTableIter it;
+ gpointer k, v;
+ struct fuzzy_mapping *map;
+
+ symbol = lua_tostring(L, 2);
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+
+ if (flag != 0) {
+ break;
+ }
+
+ g_hash_table_iter_init(&it, rule->mappings);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ map = v;
+
+ if (g_ascii_strcasecmp(symbol, map->symbol) == 0) {
+ flag = map->fuzzy_flag;
+ break;
+ }
+ }
+ }
+ }
+
+ if (flag == 0) {
+ return luaL_error(L, "bad flag");
+ }
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ weight = lua_tonumber(L, 3);
+ }
+
+ if (lua_type(L, 4) == LUA_TTABLE) {
+ const gchar *sf;
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ sf = lua_tostring(L, -1);
+
+ if (sf) {
+ if (g_ascii_strcasecmp(sf, "noimages") == 0) {
+ send_flags |= FUZZY_CHECK_FLAG_NOIMAGES;
+ }
+ else if (g_ascii_strcasecmp(sf, "noattachments") == 0) {
+ send_flags |= FUZZY_CHECK_FLAG_NOATTACHMENTS;
+ }
+ else if (g_ascii_strcasecmp(sf, "notext") == 0) {
+ send_flags |= FUZZY_CHECK_FLAG_NOTEXT;
+ }
+ }
+ }
+ }
+
+ lua_pushboolean(L,
+ fuzzy_check_lua_process_learn(task, FUZZY_DEL, weight, flag,
+ send_flags));
+
+ return 1;
+}
+
+static gint
+fuzzy_lua_gen_hashes_handler(lua_State *L)
+{
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ guint flag = 0, weight = 1, send_flags = 0;
+ const gchar *symbol;
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(task->cfg);
+ struct fuzzy_rule *rule;
+ GPtrArray *commands;
+ gint cmd = FUZZY_WRITE;
+ gint i;
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ flag = lua_tonumber(L, 2);
+ }
+ else if (lua_type(L, 2) == LUA_TSTRING) {
+ struct fuzzy_rule *rule;
+ GHashTableIter it;
+ gpointer k, v;
+ struct fuzzy_mapping *map;
+
+ symbol = lua_tostring(L, 2);
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ if (flag != 0) {
+ break;
+ }
+
+ g_hash_table_iter_init(&it, rule->mappings);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ map = v;
+
+ if (g_ascii_strcasecmp(symbol, map->symbol) == 0) {
+ flag = map->fuzzy_flag;
+ break;
+ }
+ }
+ }
+ }
+
+ if (flag == 0) {
+ return luaL_error(L, "bad flag");
+ }
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ weight = lua_tonumber(L, 3);
+ }
+
+ /* Flags */
+ if (lua_type(L, 4) == LUA_TTABLE) {
+ const gchar *sf;
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ sf = lua_tostring(L, -1);
+
+ if (sf) {
+ if (g_ascii_strcasecmp(sf, "noimages") == 0) {
+ send_flags |= FUZZY_CHECK_FLAG_NOIMAGES;
+ }
+ else if (g_ascii_strcasecmp(sf, "noattachments") == 0) {
+ send_flags |= FUZZY_CHECK_FLAG_NOATTACHMENTS;
+ }
+ else if (g_ascii_strcasecmp(sf, "notext") == 0) {
+ send_flags |= FUZZY_CHECK_FLAG_NOTEXT;
+ }
+ }
+ }
+ }
+
+ /* Type */
+ if (lua_type(L, 5) == LUA_TSTRING) {
+ const gchar *cmd_name = lua_tostring(L, 5);
+
+ if (strcmp(cmd_name, "add") == 0 || strcmp(cmd_name, "write") == 0) {
+ cmd = FUZZY_WRITE;
+ }
+ else if (strcmp(cmd_name, "delete") == 0 || strcmp(cmd_name, "remove") == 0) {
+ cmd = FUZZY_DEL;
+ }
+ else {
+ return luaL_error(L, "invalid command: %s", cmd_name);
+ }
+ }
+
+ lua_createtable(L, 0, fuzzy_module_ctx->fuzzy_rules->len);
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ if (rule->read_only) {
+ continue;
+ }
+
+ /* Check for flag */
+ if (g_hash_table_lookup(rule->mappings,
+ GINT_TO_POINTER(flag)) == NULL) {
+ msg_info_task("skip rule %s as it has no flag %d defined"
+ " false",
+ rule->name, flag);
+ continue;
+ }
+
+ commands = fuzzy_generate_commands(task, rule, cmd, flag,
+ weight, send_flags);
+
+ if (commands != NULL) {
+ struct fuzzy_cmd_io *io;
+ gint j;
+
+ lua_pushstring(L, rule->name);
+ lua_createtable(L, commands->len, 0);
+
+ PTR_ARRAY_FOREACH(commands, j, io)
+ {
+ lua_pushlstring(L, io->io.iov_base, io->io.iov_len);
+ lua_rawseti(L, -2, j + 1);
+ }
+
+ lua_settable(L, -3); /* ret[rule->name] = {raw_fuzzy1, ..., raw_fuzzyn} */
+
+ g_ptr_array_free(commands, TRUE);
+ }
+ }
+
+
+ return 1;
+}
+
+static gint
+fuzzy_lua_hex_hashes_handler(lua_State *L)
+{
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ guint flag = 0, weight = 1, send_flags = 0;
+ const gchar *symbol;
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(task->cfg);
+ struct fuzzy_rule *rule;
+ GPtrArray *commands;
+ gint i;
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ flag = lua_tonumber(L, 2);
+ }
+ else if (lua_type(L, 2) == LUA_TSTRING) {
+ struct fuzzy_rule *rule;
+ GHashTableIter it;
+ gpointer k, v;
+ struct fuzzy_mapping *map;
+
+ symbol = lua_tostring(L, 2);
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ if (flag != 0) {
+ break;
+ }
+
+ g_hash_table_iter_init(&it, rule->mappings);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ map = v;
+
+ if (g_ascii_strcasecmp(symbol, map->symbol) == 0) {
+ flag = map->fuzzy_flag;
+ break;
+ }
+ }
+ }
+ }
+
+ if (flag == 0) {
+ return luaL_error(L, "bad flag");
+ }
+
+ lua_createtable(L, 0, fuzzy_module_ctx->fuzzy_rules->len);
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ /* Check for flag */
+ if (g_hash_table_lookup(rule->mappings,
+ GINT_TO_POINTER(flag)) == NULL) {
+ msg_debug_task("skip rule %s as it has no flag %d defined"
+ " false",
+ rule->name, flag);
+ continue;
+ }
+
+ commands = fuzzy_generate_commands(task, rule, FUZZY_CHECK, flag,
+ weight, send_flags);
+
+ lua_pushstring(L, rule->name);
+
+ if (commands != NULL) {
+ lua_createtable(L, commands->len, 0);
+ /*
+ * We have all commands cached, so we can just read their cached value to
+ * get hex hashes
+ */
+ struct rspamd_mime_part *mp;
+ gint j, part_idx = 1;
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), j, mp)
+ {
+ struct rspamd_cached_shingles *cached;
+
+ cached = fuzzy_cmd_get_cached(rule, task, mp);
+
+ if (cached) {
+ gchar hexbuf[rspamd_cryptobox_HASHBYTES * 2 + 1];
+ gint r = rspamd_encode_hex_buf(cached->digest, sizeof(cached->digest), hexbuf,
+ sizeof(hexbuf));
+ lua_pushlstring(L, hexbuf, r);
+ lua_rawseti(L, -2, part_idx++);
+ }
+ }
+
+ g_ptr_array_free(commands, TRUE);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ /* res[rule->name] = {hex_hash1, ..., hex_hashn} */
+ lua_settable(L, -3);
+ }
+
+ return 1;
+}
+
+static gboolean
+fuzzy_add_handler(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg, struct module_ctx *ctx)
+{
+ return fuzzy_controller_handler(conn_ent, msg,
+ ctx, FUZZY_WRITE, FALSE);
+}
+
+static gboolean
+fuzzy_delete_handler(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg, struct module_ctx *ctx)
+{
+ return fuzzy_controller_handler(conn_ent, msg,
+ ctx, FUZZY_DEL, FALSE);
+}
+
+static gboolean
+fuzzy_deletehash_handler(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg, struct module_ctx *ctx)
+{
+ return fuzzy_controller_handler(conn_ent, msg,
+ ctx, FUZZY_DEL, TRUE);
+}
+
+static int
+fuzzy_attach_controller(struct module_ctx *ctx, GHashTable *commands)
+{
+ struct fuzzy_ctx *fctx = (struct fuzzy_ctx *) ctx;
+ struct rspamd_custom_controller_command *cmd;
+
+ cmd = rspamd_mempool_alloc(fctx->fuzzy_pool, sizeof(*cmd));
+ cmd->privileged = TRUE;
+ cmd->require_message = TRUE;
+ cmd->handler = fuzzy_add_handler;
+ cmd->ctx = ctx;
+ g_hash_table_insert(commands, "/fuzzyadd", cmd);
+
+ cmd = rspamd_mempool_alloc(fctx->fuzzy_pool, sizeof(*cmd));
+ cmd->privileged = TRUE;
+ cmd->require_message = TRUE;
+ cmd->handler = fuzzy_delete_handler;
+ cmd->ctx = ctx;
+ g_hash_table_insert(commands, "/fuzzydel", cmd);
+
+ cmd = rspamd_mempool_alloc(fctx->fuzzy_pool, sizeof(*cmd));
+ cmd->privileged = TRUE;
+ cmd->require_message = FALSE;
+ cmd->handler = fuzzy_deletehash_handler;
+ cmd->ctx = ctx;
+ g_hash_table_insert(commands, "/fuzzydelhash", cmd);
+
+ return 0;
+}
+
+/* Lua handlers */
+/* TODO: move to a separate unit, as this file is now a bit too hard to read */
+
+static void
+lua_upstream_str_inserter(struct upstream *up, guint idx, void *ud)
+{
+ lua_State *L = (lua_State *) ud;
+
+ lua_pushstring(L, rspamd_upstream_name(up));
+ lua_rawseti(L, -2, idx + 1);
+}
+
+static gint
+fuzzy_lua_list_storages(lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(cfg);
+ struct fuzzy_rule *rule;
+ guint i;
+
+ lua_createtable(L, 0, fuzzy_module_ctx->fuzzy_rules->len);
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ lua_newtable(L);
+
+ lua_pushboolean(L, rule->read_only);
+ lua_setfield(L, -2, "read_only");
+
+ /* Push servers */
+ lua_createtable(L, rspamd_upstreams_count(rule->servers), 0);
+ rspamd_upstreams_foreach(rule->servers, lua_upstream_str_inserter, L);
+ lua_setfield(L, -2, "servers");
+
+ /* Push flags */
+ GHashTableIter it;
+
+ lua_createtable(L, 0, g_hash_table_size(rule->mappings));
+ gpointer k, v;
+ struct fuzzy_mapping *map;
+
+ g_hash_table_iter_init(&it, rule->mappings);
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ map = v;
+
+ lua_pushinteger(L, map->fuzzy_flag);
+ lua_setfield(L, -2, map->symbol);
+ }
+ lua_setfield(L, -2, "flags");
+
+ /* Final table */
+ lua_setfield(L, -2, rule->name);
+ }
+
+ return 1;
+}
+
+struct fuzzy_lua_session {
+ struct rspamd_task *task;
+ lua_State *L;
+ rspamd_inet_addr_t *addr;
+ GPtrArray *commands;
+ struct fuzzy_rule *rule;
+ struct rspamd_io_ev ev;
+ gint cbref;
+ gint fd;
+};
+
+static void
+fuzzy_lua_session_fin(void *ud)
+{
+ struct fuzzy_lua_session *session = ud;
+
+ if (session->commands) {
+ g_ptr_array_free(session->commands, TRUE);
+ }
+
+ rspamd_ev_watcher_stop(session->task->event_loop, &session->ev);
+ luaL_unref(session->L, LUA_REGISTRYINDEX, session->cbref);
+}
+
+static gboolean
+fuzzy_lua_session_is_completed(struct fuzzy_lua_session *session)
+{
+ struct fuzzy_cmd_io *io;
+ guint nreplied = 0, i;
+
+
+ for (i = 0; i < session->commands->len; i++) {
+ io = g_ptr_array_index(session->commands, i);
+
+ if (io->flags & FUZZY_CMD_FLAG_REPLIED) {
+ nreplied++;
+ }
+ }
+
+ if (nreplied == session->commands->len) {
+
+ rspamd_session_remove_event(session->task->s, fuzzy_lua_session_fin, session);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static void
+fuzzy_lua_push_result(struct fuzzy_lua_session *session, gdouble latency)
+{
+ lua_rawgeti(session->L, LUA_REGISTRYINDEX, session->cbref);
+ lua_pushboolean(session->L, TRUE);
+ rspamd_lua_ip_push(session->L, session->addr);
+ lua_pushnumber(session->L, latency);
+
+ /* TODO: check results maybe? */
+ lua_pcall(session->L, 3, 0, 0);
+}
+
+#ifdef __GNUC__
+static void
+fuzzy_lua_push_error(struct fuzzy_lua_session *session, const gchar *err_fmt, ...) __attribute__((format(printf, 2, 3)));
+#endif
+
+static void
+fuzzy_lua_push_error(struct fuzzy_lua_session *session, const gchar *err_fmt, ...)
+{
+ va_list v;
+
+ va_start(v, err_fmt);
+ lua_rawgeti(session->L, LUA_REGISTRYINDEX, session->cbref);
+ lua_pushboolean(session->L, FALSE);
+ rspamd_lua_ip_push(session->L, session->addr);
+ lua_pushvfstring(session->L, err_fmt, v);
+ va_end(v);
+
+ /* TODO: check results maybe? */
+ lua_pcall(session->L, 3, 0, 0);
+}
+
+static gint
+fuzzy_lua_try_read(struct fuzzy_lua_session *session)
+{
+ const struct rspamd_fuzzy_reply *rep;
+ struct rspamd_fuzzy_cmd *cmd = NULL;
+ struct fuzzy_cmd_io *io = NULL;
+ gint r, ret;
+ guchar buf[2048], *p;
+
+ if ((r = read(session->fd, buf, sizeof(buf) - 1)) == -1) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
+ return 0;
+ }
+ else {
+ fuzzy_lua_push_error(session, "cannot read from socket: %s", strerror(errno));
+ return -1;
+ }
+ }
+ else {
+ p = buf;
+
+ ret = 0;
+
+ while ((rep = fuzzy_process_reply(&p, &r,
+ session->commands, session->rule, &cmd, &io)) != NULL) {
+
+ if (rep->v1.prob > 0.5) {
+ if (cmd->cmd == FUZZY_PING) {
+ fuzzy_lua_push_result(session, fuzzy_milliseconds_since_midnight() - rep->v1.value);
+ }
+ else {
+ fuzzy_lua_push_error(session, "unsupported");
+ }
+ }
+ else {
+ fuzzy_lua_push_error(session, "invalid reply from server: %d", rep->v1.value);
+ }
+
+ ret = 1;
+ }
+ }
+
+ return ret;
+}
+
+/* Fuzzy check callback */
+static void
+fuzzy_lua_io_callback(gint fd, short what, void *arg)
+{
+ struct fuzzy_lua_session *session = arg;
+ gint r;
+
+ enum {
+ return_error = 0,
+ return_want_more,
+ return_finished
+ } ret = return_error;
+
+ if (what & EV_READ) {
+ /* Try to read reply */
+ r = fuzzy_lua_try_read(session);
+
+ switch (r) {
+ case 0:
+ if (what & EV_READ) {
+ ret = return_want_more;
+ }
+ else {
+ if (what & EV_WRITE) {
+ /* Retransmit attempt */
+ if (!fuzzy_cmd_vector_to_wire(fd, session->commands)) {
+ fuzzy_lua_push_error(session, "cannot write to socket");
+ ret = return_error;
+ }
+ else {
+ ret = return_want_more;
+ }
+ }
+ }
+ break;
+ case 1:
+ ret = return_finished;
+ break;
+ default:
+ ret = return_error;
+ break;
+ }
+ }
+ else if (what & EV_WRITE) {
+ if (!fuzzy_cmd_vector_to_wire(fd, session->commands)) {
+ fuzzy_lua_push_error(session, "cannot write to socket");
+ ret = return_error;
+ }
+ else {
+ ret = return_want_more;
+ }
+ }
+ else {
+ /* Timeout */
+ fuzzy_lua_push_error(session, "timeout waiting for the reply");
+ ret = return_error;
+ }
+
+ if (ret == return_want_more) {
+ /* Processed write, switch to reading */
+ rspamd_ev_watcher_reschedule(session->task->event_loop,
+ &session->ev, EV_READ);
+ }
+ else if (ret == return_error) {
+ rspamd_session_remove_event(session->task->s, fuzzy_lua_session_fin, session);
+ }
+ else {
+ /* Read something from network */
+ if (!fuzzy_lua_session_is_completed(session)) {
+ /* Need to read more */
+ rspamd_ev_watcher_reschedule(session->task->event_loop,
+ &session->ev, EV_READ);
+ }
+ }
+}
+
+/***
+ * @function fuzzy_check.ping_storage(task, callback, rule, timeout[, server_override])
+ * @return
+ */
+static gint
+fuzzy_lua_ping_storage(lua_State *L)
+{
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task == NULL) {
+ return luaL_error(L, "invalid arguments: task");
+ }
+
+ /* Other arguments sanity */
+ if (lua_type(L, 2) != LUA_TFUNCTION || lua_type(L, 3) != LUA_TSTRING || lua_type(L, 4) != LUA_TNUMBER) {
+ return luaL_error(L, "invalid arguments: callback/rule/timeout argument");
+ }
+
+ struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context(task->cfg);
+ struct fuzzy_rule *rule, *rule_found = NULL;
+ int i;
+ const char *rule_name = lua_tostring(L, 3);
+
+ PTR_ARRAY_FOREACH(fuzzy_module_ctx->fuzzy_rules, i, rule)
+ {
+ if (strcmp(rule->name, rule_name) == 0) {
+ rule_found = rule;
+ break;
+ }
+ }
+
+ if (rule_found == NULL) {
+ return luaL_error(L, "invalid arguments: no such rule defined");
+ }
+
+ rspamd_inet_addr_t *addr = NULL;
+
+ if (lua_type(L, 5) == LUA_TSTRING) {
+ const gchar *server_name = lua_tostring(L, 5);
+ enum rspamd_parse_host_port_result res;
+ GPtrArray *addrs = g_ptr_array_new();
+
+ /* We resolve address synchronously here! Why? Because it is an override... */
+ res = rspamd_parse_host_port_priority(server_name, &addrs, 0, NULL,
+ 11335, FALSE, task->task_pool);
+
+ if (res == RSPAMD_PARSE_ADDR_FAIL) {
+ lua_pushboolean(L, FALSE);
+ lua_pushfstring(L, "invalid arguments: cannot resolve %s", server_name);
+ return 2;
+ }
+
+ /* Get random address */
+ addr = rspamd_inet_address_copy(g_ptr_array_index(addrs, rspamd_random_uint64_fast() % addrs->len),
+ task->task_pool);
+ rspamd_mempool_add_destructor(task->task_pool,
+ rspamd_ptr_array_free_hard, addrs);
+ }
+ else {
+ struct upstream *selected = rspamd_upstream_get(rule_found->servers,
+ RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0);
+ addr = rspamd_upstream_addr_next(selected);
+ }
+
+ if (addr != NULL) {
+ int sock;
+ GPtrArray *commands = fuzzy_generate_commands(task, rule, FUZZY_PING, 0, 0, 0);
+
+ if ((sock = rspamd_inet_address_connect(addr, SOCK_DGRAM, TRUE)) == -1) {
+ lua_pushboolean(L, FALSE);
+ lua_pushfstring(L, "cannot connect to %s, %s",
+ rspamd_inet_address_to_string_pretty(addr),
+ strerror(errno));
+ return 2;
+ }
+ else {
+ /* Create a dedicated ping session for a socket */
+ struct fuzzy_lua_session *session =
+ rspamd_mempool_alloc0(task->task_pool,
+ sizeof(struct fuzzy_lua_session));
+ session->task = task;
+ session->fd = sock;
+ session->addr = addr;
+ session->commands = commands;
+ session->L = L;
+ session->rule = rule_found;
+ /* Store callback */
+ lua_pushvalue(L, 2);
+ session->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ rspamd_session_add_event(task->s, fuzzy_lua_session_fin, session, M);
+ rspamd_ev_watcher_init(&session->ev,
+ sock,
+ EV_WRITE,
+ fuzzy_lua_io_callback,
+ session);
+ rspamd_ev_watcher_start(session->task->event_loop, &session->ev,
+ lua_tonumber(L, 4));
+ }
+ }
+
+ lua_pushboolean(L, TRUE);
+ return 1;
+} \ No newline at end of file
diff --git a/src/plugins/lua/antivirus.lua b/src/plugins/lua/antivirus.lua
new file mode 100644
index 0000000..e39ddc5
--- /dev/null
+++ b/src/plugins/lua/antivirus.lua
@@ -0,0 +1,348 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]] --
+
+local rspamd_logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local lua_redis = require "lua_redis"
+local fun = require "fun"
+local lua_antivirus = require("lua_scanners").filter('antivirus')
+local common = require "lua_scanners/common"
+local redis_params
+
+local N = "antivirus"
+
+if confighelp then
+ rspamd_config:add_example(nil, 'antivirus',
+ "Check messages for viruses",
+ [[
+ antivirus {
+ # multiple scanners could be checked, for each we create a configuration block with an arbitrary name
+ clamav {
+ # If set force this action if any virus is found (default unset: no action is forced)
+ # action = "reject";
+ # If set, then rejection message is set to this value (mention single quotes)
+ # message = '${SCANNER}: virus found: "${VIRUS}"';
+ # Scan mime_parts separately - otherwise the complete mail will be transferred to AV Scanner
+ #scan_mime_parts = true;
+ # Scanning Text is suitable for some av scanner databases (e.g. Sanesecurity)
+ #scan_text_mime = false;
+ #scan_image_mime = false;
+ # If `max_size` is set, messages > n bytes in size are not scanned
+ max_size = 20000000;
+ # symbol to add (add it to metric if you want non-zero weight)
+ symbol = "CLAM_VIRUS";
+ # type of scanner: "clamav", "fprot", "sophos" or "savapi"
+ type = "clamav";
+ # For "savapi" you must also specify the following variable
+ product_id = 12345;
+ # You can enable logging for clean messages
+ log_clean = true;
+ # servers to query (if port is unspecified, scanner-specific default is used)
+ # can be specified multiple times to pool servers
+ # can be set to a path to a unix socket
+ # Enable this in local.d/antivirus.conf
+ servers = "127.0.0.1:3310";
+ # if `patterns` is specified virus name will be matched against provided regexes and the related
+ # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+ patterns {
+ # symbol_name = "pattern";
+ JUST_EICAR = "^Eicar-Test-Signature$";
+ }
+ # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+ whitelist = "/etc/rspamd/antivirus.wl";
+ # Replace content that exactly matches the following string to the EICAR pattern
+ # Useful for E2E testing when another party removes/blocks EICAR attachments
+ #eicar_fake_pattern = 'testpatterneicar';
+ }
+ }
+ ]])
+ return
+end
+
+-- Encode as base32 in the source to avoid crappy stuff
+local eicar_pattern = rspamd_util.decode_base32(
+ [[akp6woykfbonrepmwbzyfpbmibpone3mj3pgwbffzj9e1nfjdkorisckwkohrnfe1nt41y3jwk1cirjki4w4nkieuni4ndfjcktnn1yjmb1wn]]
+)
+
+local function add_antivirus_rule(sym, opts)
+ if not opts.type then
+ rspamd_logger.errx(rspamd_config, 'unknown type for AV rule %s', sym)
+ return nil
+ end
+
+ if not opts.symbol then
+ opts.symbol = sym:upper()
+ end
+ local cfg = lua_antivirus[opts.type]
+
+ if not cfg then
+ rspamd_logger.errx(rspamd_config, 'unknown antivirus type: %s',
+ opts.type)
+ return nil
+ end
+
+ if not opts.symbol_fail then
+ opts.symbol_fail = opts.symbol .. '_FAIL'
+ end
+ if not opts.symbol_encrypted then
+ opts.symbol_encrypted = opts.symbol .. '_ENCRYPTED'
+ end
+ if not opts.symbol_macro then
+ opts.symbol_macro = opts.symbol .. '_MACRO'
+ end
+
+ -- WORKAROUND for deprecated attachments_only
+ if opts.attachments_only ~= nil then
+ opts.scan_mime_parts = opts.attachments_only
+ rspamd_logger.warnx(rspamd_config, '%s [%s]: Using attachments_only is deprecated. ' ..
+ 'Please use scan_mime_parts = %s instead', opts.symbol, opts.type, opts.attachments_only)
+ end
+ -- WORKAROUND for deprecated attachments_only
+
+ local rule = cfg.configure(opts)
+ if not rule then
+ return nil
+ end
+
+ rule.type = opts.type
+ rule.symbol_fail = opts.symbol_fail
+ rule.symbol_encrypted = opts.symbol_encrypted
+ rule.redis_params = redis_params
+
+ if not rule then
+ rspamd_logger.errx(rspamd_config, 'cannot configure %s for %s',
+ opts.type, opts.symbol)
+ return nil
+ end
+
+ rule.patterns = common.create_regex_table(opts.patterns or {})
+ rule.patterns_fail = common.create_regex_table(opts.patterns_fail or {})
+
+ lua_redis.register_prefix(rule.prefix .. '_*', N,
+ string.format('Antivirus cache for rule "%s"',
+ rule.type), {
+ type = 'string',
+ })
+
+ -- if any mime_part filter defined, do not scan all attachments
+ if opts.mime_parts_filter_regex ~= nil
+ or opts.mime_parts_filter_ext ~= nil then
+ rule.scan_all_mime_parts = false
+ else
+ rule.scan_all_mime_parts = true
+ end
+
+ rule.patterns = common.create_regex_table(opts.patterns or {})
+ rule.patterns_fail = common.create_regex_table(opts.patterns_fail or {})
+
+ rule.mime_parts_filter_regex = common.create_regex_table(opts.mime_parts_filter_regex or {})
+
+ rule.mime_parts_filter_ext = common.create_regex_table(opts.mime_parts_filter_ext or {})
+
+ if opts.whitelist then
+ rule.whitelist = rspamd_config:add_hash_map(opts.whitelist)
+ end
+
+ return function(task)
+ if rule.scan_mime_parts then
+
+ fun.each(function(p)
+ local content = p:get_content()
+ local clen = #content
+ if content and clen > 0 then
+ if opts.eicar_fake_pattern then
+ if type(opts.eicar_fake_pattern) == 'string' then
+ -- Convert it to Rspamd text
+ local rspamd_text = require "rspamd_text"
+ opts.eicar_fake_pattern = rspamd_text.fromstring(opts.eicar_fake_pattern)
+ end
+
+ if clen == #opts.eicar_fake_pattern and content == opts.eicar_fake_pattern then
+ rspamd_logger.infox(task, 'found eicar fake replacement part in the part (filename="%s")',
+ p:get_filename())
+ content = eicar_pattern
+ end
+ end
+ cfg.check(task, content, p:get_digest(), rule, p)
+ end
+ end, common.check_parts_match(task, rule))
+
+ else
+ cfg.check(task, task:get_content(), task:get_digest(), rule)
+ end
+ end
+end
+
+-- Registration
+local opts = rspamd_config:get_all_opt(N)
+if opts and type(opts) == 'table' then
+ redis_params = lua_redis.parse_redis_server(N)
+ local has_valid = false
+ for k, m in pairs(opts) do
+ if type(m) == 'table' then
+ if not m.type then
+ m.type = k
+ end
+ if not m.name then
+ m.name = k
+ end
+ local cb = add_antivirus_rule(k, m)
+
+ if not cb then
+ rspamd_logger.errx(rspamd_config, 'cannot add rule: "' .. k .. '"')
+ lua_util.config_utils.push_config_error(N, 'cannot add AV rule: "' .. k .. '"')
+ else
+ rspamd_logger.infox(rspamd_config, 'added antivirus engine %s -> %s', k, m.symbol)
+ local t = {
+ name = m.symbol,
+ callback = cb,
+ score = 0.0,
+ group = N
+ }
+
+ if m.symbol_type == 'postfilter' then
+ t.type = 'postfilter'
+ t.priority = lua_util.symbols_priorities.medium
+ else
+ t.type = 'normal'
+ end
+
+ t.augmentations = {}
+
+ if type(m.timeout) == 'number' then
+ -- Here, we ignore possible DNS timeout and timeout from multiple retries
+ -- as these situations are not usual nor likely for the antivirus module
+ table.insert(t.augmentations, string.format("timeout=%f", m.timeout))
+ end
+
+ local id = rspamd_config:register_symbol(t)
+
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = m['symbol_fail'],
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = m['symbol_encrypted'],
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = m['symbol_macro'],
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ has_valid = true
+ if type(m['patterns']) == 'table' then
+ if m['patterns'][1] then
+ for _, p in ipairs(m['patterns']) do
+ if type(p) == 'table' then
+ for sym in pairs(p) do
+ rspamd_logger.debugm(N, rspamd_config, 'registering: %1', {
+ type = 'virtual',
+ name = sym,
+ parent = m['symbol'],
+ parent_id = id,
+ group = N
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ end
+ end
+ end
+ else
+ for sym in pairs(m['patterns']) do
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ end
+ end
+ end
+ if type(m['patterns_fail']) == 'table' then
+ if m['patterns_fail'][1] then
+ for _, p in ipairs(m['patterns_fail']) do
+ if type(p) == 'table' then
+ for sym in pairs(p) do
+ rspamd_logger.debugm(N, rspamd_config, 'registering: %1', {
+ type = 'virtual',
+ name = sym,
+ parent = m['symbol'],
+ parent_id = id,
+ group = N
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ end
+ end
+ end
+ else
+ for sym in pairs(m['patterns_fail']) do
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ end
+ end
+ end
+ if m['score'] then
+ -- Register metric symbol
+ local description = 'antivirus symbol'
+ local group = N
+ if m['description'] then
+ description = m['description']
+ end
+ if m['group'] then
+ group = m['group']
+ end
+ rspamd_config:set_metric_symbol({
+ name = m['symbol'],
+ score = m['score'],
+ description = description,
+ group = group or 'antivirus'
+ })
+ end
+ end
+ end
+ end
+
+ if not has_valid then
+ lua_util.disable_module(N, 'config')
+ end
+end
diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua
new file mode 100644
index 0000000..ff19aef
--- /dev/null
+++ b/src/plugins/lua/arc.lua
@@ -0,0 +1,853 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]] --
+
+local rspamd_logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local dkim_sign_tools = require "lua_dkim_tools"
+local rspamd_util = require "rspamd_util"
+local rspamd_rsa_privkey = require "rspamd_rsa_privkey"
+local rspamd_rsa = require "rspamd_rsa"
+local fun = require "fun"
+local lua_auth_results = require "lua_auth_results"
+local hash = require "rspamd_cryptobox_hash"
+local lua_mime = require "lua_mime"
+
+if confighelp then
+ return
+end
+
+local N = 'arc'
+local AR_TRUSTED_CACHE_KEY = 'arc_trusted_aar'
+
+if not rspamd_plugins.dkim then
+ rspamd_logger.errx(rspamd_config, "cannot enable arc plugin: dkim is disabled")
+ return
+end
+
+local dkim_verify = rspamd_plugins.dkim.verify
+local dkim_sign = rspamd_plugins.dkim.sign
+local dkim_canonicalize = rspamd_plugins.dkim.canon_header_relaxed
+local redis_params
+
+if not dkim_verify or not dkim_sign or not dkim_canonicalize then
+ rspamd_logger.errx(rspamd_config, "cannot enable arc plugin: dkim is disabled")
+ return
+end
+
+local arc_symbols = {
+ allow = 'ARC_ALLOW',
+ invalid = 'ARC_INVALID',
+ dnsfail = 'ARC_DNSFAIL',
+ na = 'ARC_NA',
+ reject = 'ARC_REJECT',
+}
+
+local settings = {
+ allow_envfrom_empty = true,
+ allow_hdrfrom_mismatch = false,
+ allow_hdrfrom_mismatch_local = false,
+ allow_hdrfrom_mismatch_sign_networks = false,
+ allow_hdrfrom_multiple = false,
+ allow_username_mismatch = false,
+ sign_authenticated = true,
+ domain = {},
+ path = string.format('%s/%s/%s', rspamd_paths['DBDIR'], 'arc', '$domain.$selector.key'),
+ sign_local = true,
+ selector = 'arc',
+ sign_symbol = 'ARC_SIGNED',
+ try_fallback = true,
+ use_domain = 'header',
+ use_esld = true,
+ use_redis = false,
+ key_prefix = 'arc_keys', -- default hash name
+ reuse_auth_results = false, -- Reuse the existing authentication results
+ whitelisted_signers_map = nil, -- Trusted signers domains
+ adjust_dmarc = true, -- Adjust DMARC rejected policy for trusted forwarders
+ allowed_ids = nil, -- Allowed settings id
+ forbidden_ids = nil, -- Banned settings id
+}
+
+-- To match normal AR
+local ar_settings = lua_auth_results.default_settings
+
+local function parse_arc_header(hdr, target, is_aar)
+ -- Split elements by ';' and trim spaces
+ local arr = fun.totable(fun.map(
+ function(val)
+ return fun.totable(fun.map(lua_util.rspamd_str_trim,
+ fun.filter(function(v)
+ return v and #v > 0
+ end,
+ lua_util.rspamd_str_split(val.decoded, ';')
+ )
+ ))
+ end, hdr
+ ))
+
+ -- v[1] is the key and v[2] is the value
+ local function fill_arc_header_table(v, t)
+ if v[1] and v[2] then
+ local key = lua_util.rspamd_str_trim(v[1])
+ local value = lua_util.rspamd_str_trim(v[2])
+ t[key] = value
+ end
+ end
+
+ -- Now we have two tables in format:
+ -- [arc_header] -> [{arc_header1_elts}, {arc_header2_elts}...]
+ for i, elts in ipairs(arr) do
+ if not target[i] then
+ target[i] = {}
+ end
+ if not is_aar then
+ -- For normal ARC headers we split by kv pair, like k=v
+ fun.each(function(v)
+ fill_arc_header_table(v, target[i])
+ end,
+ fun.map(function(elt)
+ return lua_util.rspamd_str_split(elt, '=')
+ end, elts)
+ )
+ else
+ -- For AAR we check special case of i=%d and pass everything else to
+ -- AAR specific parser
+ for _, elt in ipairs(elts) do
+ if string.match(elt, "%s*i%s*=%s*%d+%s*") then
+ local pair = lua_util.rspamd_str_split(elt, '=')
+ fill_arc_header_table(pair, target[i])
+ else
+ -- Normal element
+ local ar_elt = lua_auth_results.parse_ar_element(elt)
+
+ if ar_elt then
+ if not target[i].ar then
+ target[i].ar = {}
+ end
+ table.insert(target[i].ar, ar_elt)
+ end
+ end
+ end
+ end
+ target[i].header = hdr[i].decoded
+ target[i].raw_header = hdr[i].value
+ end
+
+ -- sort by i= attribute
+ table.sort(target, function(a, b)
+ return (a.i or 0) < (b.i or 0)
+ end)
+end
+
+local function arc_validate_seals(task, seals, sigs, seal_headers, sig_headers)
+ local fail_reason
+ for i = 1, #seals do
+ if (sigs[i].i or 0) ~= i then
+ fail_reason = string.format('bad i for signature: %d, expected %d; d=%s',
+ sigs[i].i, i, sigs[i].d)
+ rspamd_logger.infox(task, fail_reason)
+ task:insert_result(arc_symbols['invalid'], 1.0, fail_reason)
+ return false, fail_reason
+ end
+ if (seals[i].i or 0) ~= i then
+ fail_reason = string.format('bad i for seal: %d, expected %d; d=%s',
+ seals[i].i, i, seals[i].d)
+ rspamd_logger.infox(task, fail_reason)
+ task:insert_result(arc_symbols['invalid'], 1.0, fail_reason)
+ return false, fail_reason
+ end
+
+ if not seals[i].cv then
+ fail_reason = string.format('no cv on i=%d', i)
+ task:insert_result(arc_symbols['invalid'], 1.0, fail_reason)
+ return false, fail_reason
+ end
+
+ if i == 1 then
+ -- We need to ensure that cv of seal is equal to 'none'
+ if seals[i].cv ~= 'none' then
+ fail_reason = 'cv is not "none" for i=1'
+ task:insert_result(arc_symbols['invalid'], 1.0, fail_reason)
+ return false, fail_reason
+ end
+ else
+ if seals[i].cv ~= 'pass' then
+ fail_reason = string.format('cv is %s on i=%d', seals[i].cv, i)
+ task:insert_result(arc_symbols['reject'], 1.0, fail_reason)
+ return true, fail_reason
+ end
+ end
+ end
+
+ return true, nil
+end
+
+local function arc_callback(task)
+ local arc_sig_headers = task:get_header_full('ARC-Message-Signature')
+ local arc_seal_headers = task:get_header_full('ARC-Seal')
+ local arc_ar_headers = task:get_header_full('ARC-Authentication-Results')
+
+ if not arc_sig_headers or not arc_seal_headers then
+ task:insert_result(arc_symbols['na'], 1.0)
+ return
+ end
+
+ if #arc_sig_headers ~= #arc_seal_headers then
+ -- We mandate that count of seals is equal to count of signatures
+ rspamd_logger.infox(task, 'number of seals (%s) is not equal to number of signatures (%s)',
+ #arc_seal_headers, #arc_sig_headers)
+ task:insert_result(arc_symbols['invalid'], 1.0, 'invalid count of seals and signatures')
+ return
+ end
+
+ local cbdata = {
+ seals = {},
+ sigs = {},
+ ars = {},
+ res = 'success',
+ errors = {},
+ allowed_by_trusted = false
+ }
+
+ parse_arc_header(arc_seal_headers, cbdata.seals, false)
+ parse_arc_header(arc_sig_headers, cbdata.sigs, false)
+
+ if arc_ar_headers then
+ parse_arc_header(arc_ar_headers, cbdata.ars, true)
+ end
+
+ -- Fix i type
+ fun.each(function(hdr)
+ hdr.i = tonumber(hdr.i) or 0
+ end, cbdata.seals)
+
+ fun.each(function(hdr)
+ hdr.i = tonumber(hdr.i) or 0
+ end, cbdata.sigs)
+
+ -- Now we need to sort elements according to their [i] value
+ table.sort(cbdata.seals, function(e1, e2)
+ return (e1.i or 0) < (e2.i or 0)
+ end)
+ table.sort(cbdata.sigs, function(e1, e2)
+ return (e1.i or 0) < (e2.i or 0)
+ end)
+
+ lua_util.debugm(N, task, 'got %s arc sections', #cbdata.seals)
+
+ -- Now check sanity of what we have
+ local valid, validation_error = arc_validate_seals(task, cbdata.seals, cbdata.sigs,
+ arc_seal_headers, arc_sig_headers)
+ if not valid then
+ task:cache_set('arc-failure', validation_error)
+ return
+ end
+
+ task:cache_set('arc-sigs', cbdata.sigs)
+ task:cache_set('arc-seals', cbdata.seals)
+ task:cache_set('arc-authres', cbdata.ars)
+
+ if validation_error then
+ -- ARC rejection but no strong failure for signing
+ return
+ end
+
+ local function gen_arc_seal_cb(index, sig)
+ return function(_, res, err, domain)
+ lua_util.debugm(N, task, 'checked arc seal: %s(%s), %s processed',
+ res, err, index)
+
+ if not res then
+ cbdata.res = 'fail'
+ if err and domain then
+ table.insert(cbdata.errors, string.format('sig:%s:%s', domain, err))
+ end
+ end
+
+ if settings.whitelisted_signers_map and cbdata.res == 'success' then
+ if settings.whitelisted_signers_map:get_key(sig.d) then
+ -- Whitelisted signer has been found in a valid chain
+ local mult = 1.0
+ local cur_aar = cbdata.ars[index]
+ if not cur_aar then
+ rspamd_logger.warnx(task, "cannot find Arc-Authentication-Results for trusted " ..
+ "forwarder %s on i=%s", domain, cbdata.index)
+ else
+ task:cache_set(AR_TRUSTED_CACHE_KEY, cur_aar)
+ local seen_dmarc
+ for _, ar in ipairs(cur_aar.ar) do
+ if ar.dmarc then
+ local dmarc_fwd = ar.dmarc
+ seen_dmarc = true
+ if dmarc_fwd == 'reject' or dmarc_fwd == 'fail' or dmarc_fwd == 'quarantine' then
+ lua_util.debugm(N, "found rejected dmarc on forwarding")
+ mult = 0.0
+ elseif dmarc_fwd == 'pass' then
+ mult = 1.0
+ end
+ elseif ar.spf then
+ local spf_fwd = ar.spf
+ if spf_fwd == 'reject' or spf_fwd == 'fail' or spf_fwd == 'quarantine' then
+ lua_util.debugm(N, "found rejected spf on forwarding")
+ if not seen_dmarc then
+ mult = mult * 0.5
+ end
+ end
+ end
+ end
+ end
+ task:insert_result(arc_symbols.trusted_allow, mult,
+ string.format('%s:s=%s:i=%d', domain, sig.s, index))
+ end
+ end
+
+ if index == #arc_sig_headers then
+ if cbdata.res == 'success' then
+ local arc_allow_result = string.format('%s:s=%s:i=%d',
+ domain, sig.s, index)
+ task:insert_result(arc_symbols.allow, 1.0, arc_allow_result)
+ task:cache_set('arc-allow', arc_allow_result)
+ else
+ task:insert_result(arc_symbols.reject, 1.0,
+ rspamd_logger.slog('seal check failed: %s, %s', cbdata.res,
+ cbdata.errors))
+ end
+ end
+ end
+ end
+
+ local function arc_signature_cb(_, res, err, domain)
+ lua_util.debugm(N, task, 'checked arc signature %s: %s(%s)',
+ domain, res, err)
+
+ if not res then
+ cbdata.res = 'fail'
+ if err and domain then
+ table.insert(cbdata.errors, string.format('sig:%s:%s', domain, err))
+ end
+ end
+ if cbdata.res == 'success' then
+ -- Verify seals
+ for i, sig in ipairs(cbdata.seals) do
+ local ret, lerr = dkim_verify(task, sig.header, gen_arc_seal_cb(i, sig), 'arc-seal')
+ if not ret then
+ cbdata.res = 'fail'
+ table.insert(cbdata.errors, string.format('seal:%s:s=%s:i=%s:%s',
+ sig.d or '', sig.s or '', sig.i or '', lerr))
+ lua_util.debugm(N, task, 'checked arc seal %s: %s(%s), %s processed',
+ sig.d, ret, lerr, i)
+ end
+ end
+ else
+ task:insert_result(arc_symbols['reject'], 1.0,
+ rspamd_logger.slog('signature check failed: %s, %s', cbdata.res,
+ cbdata.errors))
+ end
+ end
+
+ --[[
+ 1. Collect all ARC Sets currently attached to the message. If there
+ are none, the Chain Validation Status is "none" and the algorithm
+ stops here. The maximum number of ARC Sets that can be attached
+ to a message is 50. If more than the maximum number exist the
+ Chain Validation Status is "fail" and the algorithm stops here.
+ In the following algorithm, the maximum ARC instance value is
+ referred to as "N".
+
+ 2. If the Chain Validation Status of the highest instance value ARC
+ Set is "fail", then the Chain Validation status is "fail" and the
+ algorithm stops here.
+
+ 3. Validate the structure of the Authenticated Received Chain. A
+ valid ARC has the following conditions:
+
+ 1. Each ARC Set MUST contain exactly one each of the three ARC
+ header fields (AAR, AMS, and AS).
+
+ 2. The instance values of the ARC Sets MUST form a continuous
+ sequence from 1..N with no gaps or repetition.
+
+ 3. The "cv" value for all ARC-Seal header fields must be non-
+ failing. For instance values > 1, the value must be "pass".
+ For instance value = 1, the value must be "none".
+
+ * If any of these conditions are not met, the Chain Validation
+ Status is "fail" and the algorithm stops here.
+
+ 4. Validate the AMS with the greatest instance value (most recent).
+ If validation fails, then the Chain Validation Status is "fail"
+ and the algorithm stops here.
+
+ 5 - 7. Optional, not implemented
+ 8. Validate each AS beginning with the greatest instance value and
+ proceeding in decreasing order to the AS with the instance value
+ of 1. If any AS fails to validate, the Chain Validation Status
+ is "fail" and the algorithm stops here.
+ 9. If the algorithm reaches this step, then the Chain Validation
+ Status is "pass", and the algorithm is complete.
+ ]]--
+
+ local processed = 0
+ local sig = cbdata.sigs[#cbdata.sigs] -- last AMS
+ local ret, err = dkim_verify(task, sig.header, arc_signature_cb, 'arc-sign')
+
+ if not ret then
+ cbdata.res = 'fail'
+ table.insert(cbdata.errors, string.format('sig:%s:%s', sig.d or '', err))
+ else
+ processed = processed + 1
+ lua_util.debugm(N, task, 'processed arc signature %s[%s]: %s(%s), %s total',
+ sig.d, sig.i, ret, err, #cbdata.seals)
+ end
+
+ if processed == 0 then
+ task:insert_result(arc_symbols['reject'], 1.0,
+ rspamd_logger.slog('cannot verify %s of %s signatures: %s',
+ #arc_sig_headers - processed, #arc_sig_headers, cbdata.errors))
+ end
+end
+
+local opts = rspamd_config:get_all_opt('arc')
+if not opts or type(opts) ~= 'table' then
+ return
+end
+
+if opts['symbols'] then
+ for k, _ in pairs(arc_symbols) do
+ if opts['symbols'][k] then
+ arc_symbols[k] = opts['symbols'][k]
+ end
+ end
+end
+
+local id = rspamd_config:register_symbol({
+ name = 'ARC_CHECK',
+ type = 'callback',
+ group = 'policies',
+ groups = { 'arc' },
+ callback = arc_callback,
+ augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) },
+})
+rspamd_config:register_symbol({
+ name = 'ARC_CALLBACK', -- compatibility symbol
+ type = 'virtual,skip',
+ parent = id,
+})
+
+rspamd_config:register_symbol({
+ name = arc_symbols['allow'],
+ parent = id,
+ type = 'virtual',
+ score = -1.0,
+ group = 'policies',
+ groups = { 'arc' },
+})
+rspamd_config:register_symbol({
+ name = arc_symbols['reject'],
+ parent = id,
+ type = 'virtual',
+ score = 2.0,
+ group = 'policies',
+ groups = { 'arc' },
+})
+rspamd_config:register_symbol({
+ name = arc_symbols['invalid'],
+ parent = id,
+ type = 'virtual',
+ score = 1.0,
+ group = 'policies',
+ groups = { 'arc' },
+})
+rspamd_config:register_symbol({
+ name = arc_symbols['dnsfail'],
+ parent = id,
+ type = 'virtual',
+ score = 0.0,
+ group = 'policies',
+ groups = { 'arc' },
+})
+rspamd_config:register_symbol({
+ name = arc_symbols['na'],
+ parent = id,
+ type = 'virtual',
+ score = 0.0,
+ group = 'policies',
+ groups = { 'arc' },
+})
+
+rspamd_config:register_dependency('ARC_CHECK', 'SPF_CHECK')
+rspamd_config:register_dependency('ARC_CHECK', 'DKIM_CHECK')
+
+local function arc_sign_seal(task, params, header)
+ local arc_sigs = task:cache_get('arc-sigs')
+ local arc_seals = task:cache_get('arc-seals')
+ local arc_auth_results = task:cache_get('arc-authres')
+ local cur_auth_results
+ local privkey
+
+ if params.rawkey then
+ -- Distinguish between pem and base64
+ if string.match(params.rawkey, '^-----BEGIN') then
+ privkey = rspamd_rsa_privkey.load_pem(params.rawkey)
+ else
+ privkey = rspamd_rsa_privkey.load_base64(params.rawkey)
+ end
+ elseif params.key then
+ privkey = rspamd_rsa_privkey.load_file(params.key)
+ end
+
+ if not privkey then
+ rspamd_logger.errx(task, 'cannot load private key for signing')
+ return
+ end
+
+ if settings.reuse_auth_results then
+ local ar_header = task:get_header('Authentication-Results')
+
+ if ar_header then
+ rspamd_logger.debugm(N, task, 'reuse authentication results header for ARC')
+ cur_auth_results = ar_header
+ else
+ rspamd_logger.debugm(N, task, 'cannot reuse authentication results, header is missing')
+ cur_auth_results = lua_auth_results.gen_auth_results(task, ar_settings) or ''
+ end
+ else
+ cur_auth_results = lua_auth_results.gen_auth_results(task, ar_settings) or ''
+ end
+
+ local sha_ctx = hash.create_specific('sha256')
+
+ -- Update using previous seals + sigs + AAR
+ local cur_idx = 1
+ if arc_seals then
+ cur_idx = #arc_seals + 1
+ -- We use the cached version per each ARC-* header field individually, already sorted by instance
+ -- value in ascending order
+ for i = 1, #arc_seals, 1 do
+ if arc_auth_results[i] then
+ local s = dkim_canonicalize('ARC-Authentication-Results',
+ arc_auth_results[i].raw_header)
+ sha_ctx:update(s)
+ lua_util.debugm(N, task, 'update signature with header: %s', s)
+ end
+ if arc_sigs[i] then
+ local s = dkim_canonicalize('ARC-Message-Signature',
+ arc_sigs[i].raw_header)
+ sha_ctx:update(s)
+ lua_util.debugm(N, task, 'update signature with header: %s', s)
+ end
+ if arc_seals[i] then
+ local s = dkim_canonicalize('ARC-Seal', arc_seals[i].raw_header)
+ sha_ctx:update(s)
+ lua_util.debugm(N, task, 'update signature with header: %s', s)
+ end
+ end
+ end
+
+ header = lua_util.fold_header(task,
+ 'ARC-Message-Signature',
+ header)
+
+ cur_auth_results = string.format('i=%d; %s', cur_idx, cur_auth_results)
+ cur_auth_results = lua_util.fold_header(task,
+ 'ARC-Authentication-Results',
+ cur_auth_results, ';')
+
+ local s = dkim_canonicalize('ARC-Authentication-Results',
+ cur_auth_results)
+ sha_ctx:update(s)
+ lua_util.debugm(N, task, 'update signature with header: %s', s)
+ s = dkim_canonicalize('ARC-Message-Signature', header)
+ sha_ctx:update(s)
+ lua_util.debugm(N, task, 'update signature with header: %s', s)
+
+ local cur_arc_seal = string.format('i=%d; s=%s; d=%s; t=%d; a=rsa-sha256; cv=%s; b=',
+ cur_idx,
+ params.selector,
+ params.domain,
+ math.floor(rspamd_util.get_time()), params.arc_cv)
+ s = string.format('%s:%s', 'arc-seal', cur_arc_seal)
+ sha_ctx:update(s)
+ lua_util.debugm(N, task, 'initial update signature with header: %s', s)
+
+ local nl_type
+ if task:has_flag("milter") then
+ nl_type = "lf"
+ else
+ nl_type = task:get_newlines_type()
+ end
+
+ local sig = rspamd_rsa.sign_memory(privkey, sha_ctx:bin())
+ cur_arc_seal = string.format('%s%s', cur_arc_seal,
+ sig:base64(70, nl_type))
+
+ lua_mime.modify_headers(task, {
+ add = {
+ ['ARC-Authentication-Results'] = { order = 1, value = cur_auth_results },
+ ['ARC-Message-Signature'] = { order = 1, value = header },
+ ['ARC-Seal'] = { order = 1, value = lua_util.fold_header(task,
+ 'ARC-Seal', cur_arc_seal) }
+ },
+ -- RFC requires a strict order for these headers to be inserted
+ order = { 'ARC-Authentication-Results', 'ARC-Message-Signature', 'ARC-Seal' },
+ })
+ task:insert_result(settings.sign_symbol, 1.0,
+ string.format('%s:s=%s:i=%d', params.domain, params.selector, cur_idx))
+end
+
+local function prepare_arc_selector(task, sel)
+ local arc_seals = task:cache_get('arc-seals')
+
+ if not arc_seals then
+ -- Check if our arc is broken
+ local failure_reason = task:cache_get('arc-failure')
+ if failure_reason then
+ rspamd_logger.infox(task, 'skip ARC as the existing chain is broken: %s', failure_reason)
+ return false
+ end
+ end
+
+ sel.arc_cv = 'none'
+ sel.arc_idx = 1
+ sel.no_cache = true
+ sel.sign_type = 'arc-sign'
+
+ if arc_seals then
+ sel.arc_idx = #arc_seals + 1
+
+ local function default_arc_cv()
+ if task:cache_get('arc-allow') then
+ sel.arc_cv = 'pass'
+ else
+ sel.arc_cv = 'fail'
+ end
+ end
+
+ if settings.reuse_auth_results then
+ local ar_header = task:get_header('Authentication-Results')
+
+ if ar_header then
+ local arc_match = string.match(ar_header, 'arc=(%w+)')
+
+ if arc_match then
+ if arc_match == 'none' or arc_match == 'pass' then
+ -- none should be converted to `pass`
+ sel.arc_cv = 'pass'
+ else
+ sel.arc_cv = 'fail'
+ end
+ else
+ default_arc_cv()
+ end
+ else
+ -- Cannot reuse, use normal path
+ default_arc_cv()
+ end
+ else
+ default_arc_cv()
+ end
+
+ end
+
+ return true
+end
+
+local function do_sign(task, sign_params)
+ if sign_params.alg and sign_params.alg ~= 'rsa' then
+ -- No support for ed25519 keys
+ return
+ end
+
+ if not prepare_arc_selector(task, sign_params) then
+ -- Broken arc
+ return
+ end
+
+ if settings.check_pubkey then
+ local resolve_name = sign_params.selector .. "._domainkey." .. sign_params.domain
+ task:get_resolver():resolve_txt({
+ task = task,
+ name = resolve_name,
+ callback = function(_, _, results, err)
+ if not err and results and results[1] then
+ sign_params.pubkey = results[1]
+ sign_params.strict_pubkey_check = not settings.allow_pubkey_mismatch
+ elseif not settings.allow_pubkey_mismatch then
+ rspamd_logger.errx('public key for domain %s/%s is not found: %s, skip signing',
+ sign_params.domain, sign_params.selector, err)
+ return
+ else
+ rspamd_logger.infox('public key for domain %s/%s is not found: %s',
+ sign_params.domain, sign_params.selector, err)
+ end
+
+ local dret, hdr = dkim_sign(task, sign_params)
+ if dret then
+ arc_sign_seal(task, sign_params, hdr)
+ end
+
+ end,
+ forced = true
+ })
+ else
+ local dret, hdr = dkim_sign(task, sign_params)
+ if dret then
+ arc_sign_seal(task, sign_params, hdr)
+ end
+ end
+end
+
+local function sign_error(task, msg)
+ rspamd_logger.errx(task, 'signing failure: %s', msg)
+end
+
+local function arc_signing_cb(task)
+ local ret, selectors = dkim_sign_tools.prepare_dkim_signing(N, task, settings)
+
+ if not ret then
+ return
+ end
+
+ if settings.use_redis then
+ dkim_sign_tools.sign_using_redis(N, task, settings, selectors, do_sign, sign_error)
+ else
+ if selectors.vault then
+ dkim_sign_tools.sign_using_vault(N, task, settings, selectors, do_sign, sign_error)
+ else
+ -- TODO: no support for multiple sigs
+ local cur_selector = selectors[1]
+ prepare_arc_selector(task, cur_selector)
+ if ((cur_selector.key or cur_selector.rawkey) and cur_selector.selector) then
+ if cur_selector.key then
+ cur_selector.key = lua_util.template(cur_selector.key, {
+ domain = cur_selector.domain,
+ selector = cur_selector.selector
+ })
+
+ local exists, err = rspamd_util.file_exists(cur_selector.key)
+ if not exists then
+ if err and err == 'No such file or directory' then
+ lua_util.debugm(N, task, 'cannot read key from %s: %s', cur_selector.key, err)
+ else
+ rspamd_logger.warnx(task, 'cannot read key from %s: %s', cur_selector.key, err)
+ end
+ return false
+ end
+ end
+
+ do_sign(task, cur_selector)
+ else
+ rspamd_logger.infox(task, 'key path or dkim selector unconfigured; no signing')
+ return false
+ end
+ end
+ end
+end
+
+dkim_sign_tools.process_signing_settings(N, settings, opts)
+
+if not dkim_sign_tools.validate_signing_settings(settings) then
+ rspamd_logger.infox(rspamd_config, 'mandatory parameters missing, disable arc signing')
+ return
+end
+
+local ar_opts = rspamd_config:get_all_opt('milter_headers')
+
+if ar_opts and ar_opts.routines then
+ local routines = ar_opts.routines
+
+ if routines['authentication-results'] then
+ ar_settings = lua_util.override_defaults(ar_settings,
+ routines['authentication-results'])
+ end
+end
+
+if settings.use_redis then
+ redis_params = rspamd_parse_redis_server('arc')
+
+ if not redis_params then
+ rspamd_logger.errx(rspamd_config, 'no servers are specified, ' ..
+ 'but module is configured to load keys from redis, disable arc signing')
+ return
+ end
+
+ settings.redis_params = redis_params
+end
+
+local sym_reg_tbl = {
+ name = settings['sign_symbol'],
+ callback = arc_signing_cb,
+ groups = { "policies", "arc" },
+ flags = 'ignore_passthrough',
+ score = 0.0,
+}
+if type(settings.allowed_ids) == 'table' then
+ sym_reg_tbl.allowed_ids = settings.allowed_ids
+end
+if type(settings.forbidden_ids) == 'table' then
+ sym_reg_tbl.forbidden_ids = settings.forbidden_ids
+end
+
+if settings.whitelisted_signers_map then
+ arc_symbols.trusted_allow = arc_symbols.trusted_allow or 'ARC_ALLOW_TRUSTED'
+ rspamd_config:register_symbol({
+ name = arc_symbols.trusted_allow,
+ parent = id,
+ type = 'virtual',
+ score = -2.0,
+ group = 'policies',
+ groups = { 'arc' },
+ })
+end
+
+rspamd_config:register_symbol(sym_reg_tbl)
+
+-- Do not sign unless checked
+rspamd_config:register_dependency(settings['sign_symbol'], 'ARC_CHECK')
+-- We need to check dmarc before signing as we have to produce valid AAR header
+-- see #3613
+rspamd_config:register_dependency(settings['sign_symbol'], 'DMARC_CHECK')
+
+if settings.adjust_dmarc and settings.whitelisted_signers_map then
+ local function arc_dmarc_adjust_cb(task)
+ local trusted_arc_ar = task:cache_get(AR_TRUSTED_CACHE_KEY)
+ local sym_to_adjust
+ if task:has_symbol(ar_settings.dmarc_symbols.reject) then
+ sym_to_adjust = ar_settings.dmarc_symbols.reject
+ elseif task:has_symbol(ar_settings.dmarc_symbols.quarantine) then
+ sym_to_adjust = ar_settings.dmarc_symbols.quarantine
+ end
+ if sym_to_adjust and trusted_arc_ar and trusted_arc_ar.ar then
+ for _, ar in ipairs(trusted_arc_ar.ar) do
+ if ar.dmarc then
+ local dmarc_fwd = ar.dmarc
+ if dmarc_fwd == 'pass' then
+ rspamd_logger.infox(task, "adjust dmarc reject score as trusted forwarder "
+ .. "proved DMARC validity for %s", ar['header.from'])
+ task:adjust_result(sym_to_adjust, 0.1,
+ 'ARC trusted')
+ end
+ end
+ end
+ end
+ end
+ rspamd_config:register_symbol({
+ name = 'ARC_DMARC_ADJUSTMENT',
+ callback = arc_dmarc_adjust_cb,
+ type = 'callback',
+ })
+ rspamd_config:register_dependency('ARC_DMARC_ADJUSTMENT', 'DMARC_CHECK')
+ rspamd_config:register_dependency('ARC_DMARC_ADJUSTMENT', 'ARC_CHECK')
+end
diff --git a/src/plugins/lua/asn.lua b/src/plugins/lua/asn.lua
new file mode 100644
index 0000000..24da19e
--- /dev/null
+++ b/src/plugins/lua/asn.lua
@@ -0,0 +1,168 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_regexp = require "rspamd_regexp"
+local lua_util = require "lua_util"
+local N = "asn"
+
+if confighelp then
+ return
+end
+
+local options = {
+ provider_type = 'rspamd',
+ provider_info = {
+ ip4 = 'asn.rspamd.com',
+ ip6 = 'asn6.rspamd.com',
+ },
+ symbol = 'ASN',
+ check_local = false,
+}
+
+local rspamd_re = rspamd_regexp.create_cached("[\\|\\s]")
+
+local function asn_check(task)
+
+ local function asn_set(asn, ipnet, country)
+ local descr_t = {}
+ local mempool = task:get_mempool()
+ if asn then
+ if tonumber(asn) ~= nil then
+ mempool:set_variable("asn", asn)
+ table.insert(descr_t, "asn:" .. asn)
+ else
+ rspamd_logger.errx(task, 'malformed ASN "%s" for ip %s', asn, task:get_from_ip())
+ end
+ end
+ if ipnet then
+ mempool:set_variable("ipnet", ipnet)
+ table.insert(descr_t, "ipnet:" .. ipnet)
+ end
+ if country then
+ mempool:set_variable("country", country)
+ table.insert(descr_t, "country:" .. country)
+ end
+ if options['symbol'] then
+ task:insert_result(options['symbol'], 0.0, table.concat(descr_t, ', '))
+ end
+ end
+
+ local asn_check_func = {}
+ asn_check_func.rspamd = function(ip)
+ local dnsbl = options['provider_info']['ip' .. ip:get_version()]
+ local req_name = string.format("%s.%s",
+ table.concat(ip:inversed_str_octets(), '.'), dnsbl)
+ local function rspamd_dns_cb(_, _, results, dns_err, _, _, serv)
+ if dns_err and (dns_err ~= 'requested record is not found' and dns_err ~= 'no records with this name') then
+ rspamd_logger.errx(task, 'error querying dns "%s" on %s: %s',
+ req_name, serv, dns_err)
+ task:insert_result(options['symbol_fail'], 0, string.format('%s:%s', req_name, dns_err))
+ return
+ end
+ if not results or not results[1] then
+ rspamd_logger.infox(task, 'no ASN information is available for the IP address "%s" on %s',
+ req_name, serv)
+ return
+ end
+
+ lua_util.debugm(N, task, 'got reply from %s when requesting %s: %s',
+ serv, req_name, results[1])
+
+ local parts = rspamd_re:split(results[1])
+ -- "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
+ asn_set(parts[1], parts[2], parts[3])
+ end
+
+ task:get_resolver():resolve_txt({
+ task = task,
+ name = req_name,
+ callback = rspamd_dns_cb
+ })
+ end
+
+ local ip = task:get_from_ip()
+ if not (ip and ip:is_valid()) or
+ (not options.check_local and ip:is_local()) then
+ return
+ end
+
+ asn_check_func[options['provider_type']](ip)
+end
+
+-- Configuration options
+local configure_asn_module = function()
+ local opts = rspamd_config:get_all_opt('asn')
+ if opts then
+ for k, v in pairs(opts) do
+ options[k] = v
+ end
+ end
+
+ local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N,
+ false, true)
+ options.check_local = auth_and_local_conf[1]
+ options.check_authed = auth_and_local_conf[2]
+
+ if options['provider_type'] == 'rspamd' then
+ if not options['provider_info'] and options['provider_info']['ip4'] and
+ options['provider_info']['ip6'] then
+ rspamd_logger.errx("Missing required provider_info for rspamd")
+ return false
+ end
+ else
+ rspamd_logger.errx("Unknown provider_type: %s", options['provider_type'])
+ return false
+ end
+
+ if options['symbol'] then
+ options['symbol_fail'] = options['symbol'] .. '_FAIL'
+ else
+ options['symbol_fail'] = 'ASN_FAIL'
+ end
+
+ return true
+end
+
+if configure_asn_module() then
+ local id = rspamd_config:register_symbol({
+ name = 'ASN_CHECK',
+ type = 'prefilter',
+ callback = asn_check,
+ priority = lua_util.symbols_priorities.high,
+ flags = 'empty,nostat',
+ augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) },
+ })
+ if options['symbol'] then
+ rspamd_config:register_symbol({
+ name = options['symbol'],
+ parent = id,
+ type = 'virtual',
+ flags = 'empty,nostat',
+ score = 0,
+ })
+ end
+ rspamd_config:register_symbol {
+ name = options['symbol_fail'],
+ parent = id,
+ type = 'virtual',
+ flags = 'empty,nostat',
+ score = 0,
+ }
+else
+ lua_util.disable_module(N, 'config')
+end
diff --git a/src/plugins/lua/aws_s3.lua b/src/plugins/lua/aws_s3.lua
new file mode 100644
index 0000000..30e88d2
--- /dev/null
+++ b/src/plugins/lua/aws_s3.lua
@@ -0,0 +1,269 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local N = "aws_s3"
+local lua_util = require "lua_util"
+local lua_aws = require "lua_aws"
+local rspamd_logger = require "rspamd_logger"
+local ts = (require "tableshape").types
+local rspamd_text = require "rspamd_text"
+local rspamd_http = require "rspamd_http"
+local rspamd_util = require "rspamd_util"
+
+local settings = {
+ s3_bucket = nil,
+ s3_region = 'us-east-1',
+ s3_host = 's3.amazonaws.com',
+ s3_secret_key = nil,
+ s3_key_id = nil,
+ s3_timeout = 10,
+ save_raw = true,
+ save_structure = false,
+ inline_content_limit = nil,
+}
+
+local settings_schema = ts.shape {
+ s3_bucket = ts.string,
+ s3_region = ts.string,
+ s3_host = ts.string,
+ s3_secret_key = ts.string,
+ s3_key_id = ts.string,
+ s3_timeout = ts.number + ts.string / lua_util.parse_time_interval,
+ enabled = ts.boolean:is_optional(),
+ fail_action = ts.string:is_optional(),
+ zstd_compress = ts.boolean:is_optional(),
+ save_raw = ts.boolean:is_optional(),
+ save_structure = ts.boolean:is_optional(),
+ inline_content_limit = ts.number:is_optional(),
+}
+
+local function raw_data(task, nonce, queue_id)
+ local ext, content, content_type
+
+ if settings.zstd_compress then
+ ext = 'eml.zst'
+ content = rspamd_util.zstd_compress(task:get_content())
+ content_type = 'application/zstd'
+ else
+ ext = 'eml'
+ content = task:get_content()
+ content_type = 'message/rfc-822'
+ end
+
+ local path = string.format('/%s-%s.%s', queue_id, nonce, ext)
+
+ return path, content, content_type
+end
+
+local function gen_ext(base)
+ local ext = base
+ if settings.zstd_compress then
+ ext = base .. '.zst'
+ end
+
+ return ext
+end
+
+local function convert_to_ref(task, nonce, queue_id, part, external_refs)
+ local path = string.format('/%s-%s-%s.%s', queue_id, nonce,
+ rspamd_text.randombytes(8):base32(), gen_ext('raw'))
+ local content = part.content
+
+ if settings.zstd_compress then
+ external_refs[path] = rspamd_util.zstd_compress(content)
+ else
+ external_refs[path] = content
+ end
+
+ part.content = nil
+ part.content_path = path
+
+ return path
+end
+
+local function structured_data(task, nonce, queue_id)
+ local content, content_type
+ local external_refs = {}
+ local lua_mime = require "lua_mime"
+ local ucl = require "ucl"
+
+ local message_split = lua_mime.message_to_ucl(task)
+ if settings.inline_content_limit and settings.inline_content_limit > 0 then
+
+ for i, part in ipairs(message_split.parts or {}) do
+ if part.content and #part.content >= settings.inline_content_limit then
+ local ref = convert_to_ref(task, nonce, queue_id, part, external_refs)
+ lua_util.debugm(N, task, "convert part number %s to a reference %s",
+ i, ref)
+ end
+ end
+ end
+
+ if settings.zstd_compress then
+ content = rspamd_util.zstd_compress(ucl.to_format(message_split, 'msgpack'))
+ content_type = 'application/zstd'
+ else
+ content = ucl.to_format(message_split, 'msgpack')
+ content_type = 'application/msgpack'
+ end
+
+ local path = string.format('/%s-%s.%s', queue_id, nonce, gen_ext('msgpack'))
+
+ return path, content, content_type, external_refs
+end
+
+local function s3_aws_callback(task)
+ local uri = string.format('https://%s.%s', settings.s3_bucket, settings.s3_host)
+ -- Create a nonce
+ local nonce = rspamd_text.randombytes(16):base32()
+ local queue_id = task:get_queue_id()
+ if not queue_id then
+ queue_id = rspamd_text.randombytes(8):base32()
+ end
+ -- Hack to pass host
+ local aws_host = string.format('%s.%s', settings.s3_bucket, settings.s3_host)
+
+ local function gen_s3_http_callback(path, what)
+ return function(http_err, code, body, headers)
+
+ if http_err then
+ if settings.fail_action then
+ task:set_pre_result(settings.fail_action,
+ string.format('S3 save failed: %s', http_err), N,
+ nil, nil, 'least')
+ end
+ rspamd_logger.errx(task, 'cannot save %s to AWS S3: %s', path, http_err)
+ else
+ rspamd_logger.messagex(task, 'saved %s successfully in S3 object %s', what, path)
+ end
+ lua_util.debugm(N, task, 'obj=%s, err=%s, code=%s, body=%s, headers=%s',
+ path, http_err, code, body, headers)
+ end
+ end
+
+ if settings.save_raw then
+ local path, content, content_type = raw_data(task, nonce, queue_id)
+ local hdrs = lua_aws.aws_request_enrich({
+ region = settings.s3_region,
+ headers = {
+ ['Content-Type'] = content_type,
+ ['Host'] = aws_host
+ },
+ uri = path,
+ key_id = settings.s3_key_id,
+ secret_key = settings.s3_secret_key,
+ method = 'PUT',
+ }, content)
+ rspamd_http.request({
+ url = uri .. path,
+ task = task,
+ method = 'PUT',
+ body = content,
+ callback = gen_s3_http_callback(path, 'raw message'),
+ headers = hdrs,
+ timeout = settings.s3_timeout,
+ })
+ end
+ if settings.save_structure then
+ local path, content, content_type, external_refs = structured_data(task, nonce, queue_id)
+ local hdrs = lua_aws.aws_request_enrich({
+ region = settings.s3_region,
+ headers = {
+ ['Content-Type'] = content_type,
+ ['Host'] = aws_host
+ },
+ uri = path,
+ key_id = settings.s3_key_id,
+ secret_key = settings.s3_secret_key,
+ method = 'PUT',
+ }, content)
+ rspamd_http.request({
+ url = uri .. path,
+ task = task,
+ method = 'PUT',
+ body = content,
+ callback = gen_s3_http_callback(path, 'structured message'),
+ headers = hdrs,
+ upstream = settings.upstreams:get_upstream_round_robin(),
+ timeout = settings.s3_timeout,
+ })
+
+ for ref, part_content in pairs(external_refs) do
+ local part_hdrs = lua_aws.aws_request_enrich({
+ region = settings.s3_region,
+ headers = {
+ ['Content-Type'] = content_type,
+ ['Host'] = aws_host
+ },
+ uri = ref,
+ key_id = settings.s3_key_id,
+ secret_key = settings.s3_secret_key,
+ method = 'PUT',
+ }, part_content)
+ rspamd_http.request({
+ url = uri .. ref,
+ task = task,
+ upstream = settings.upstreams:get_upstream_round_robin(),
+ method = 'PUT',
+ body = part_content,
+ callback = gen_s3_http_callback(ref, 'part content'),
+ headers = part_hdrs,
+ timeout = settings.s3_timeout,
+ })
+ end
+ end
+
+
+end
+
+local opts = rspamd_config:get_all_opt('aws_s3')
+if not opts then
+ return
+end
+
+settings = lua_util.override_defaults(settings, opts)
+local res, err = settings_schema:transform(settings)
+
+if not res then
+ rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err)
+ lua_util.disable_module(N, "config")
+ return
+end
+
+rspamd_logger.infox(rspamd_config, 'enabled AWS s3 dump to %s', res.s3_bucket)
+
+settings = res
+
+settings.upstreams = lua_util.http_upstreams_by_url(rspamd_config:get_mempool(),
+ string.format('https://%s.%s', settings.s3_bucket, settings.s3_host))
+
+if not settings.upstreams then
+ rspamd_logger.warnx(rspamd_config, 'cannot parse hostname: %s',
+ string.format('https://%s.%s', settings.s3_bucket, settings.s3_host))
+ lua_util.disable_module(N, "config")
+ return
+end
+
+local is_postfilter = settings.fail_action ~= nil
+
+rspamd_config:register_symbol({
+ name = 'EXPORT_AWS_S3',
+ type = is_postfilter and 'postfilter' or 'idempotent',
+ callback = s3_aws_callback,
+ augmentations = { string.format("timeout=%f", settings.s3_timeout) },
+ priority = is_postfilter and lua_util.symbols_priorities.high or nil,
+ flags = 'empty,explicit_disable,ignore_passthrough,nostat',
+}) \ No newline at end of file
diff --git a/src/plugins/lua/bayes_expiry.lua b/src/plugins/lua/bayes_expiry.lua
new file mode 100644
index 0000000..44ff9da
--- /dev/null
+++ b/src/plugins/lua/bayes_expiry.lua
@@ -0,0 +1,503 @@
+--[[
+Copyright (c) 2017, Andrew Lewis <nerf@judo.za.org>
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]] --
+
+if confighelp then
+ return
+end
+
+local N = 'bayes_expiry'
+local E = {}
+local logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local lutil = require "lua_util"
+local lredis = require "lua_redis"
+
+local settings = {
+ interval = 60, -- one iteration step per minute
+ count = 1000, -- check up to 1000 keys on each iteration
+ epsilon_common = 0.01, -- eliminate common if spam to ham rate is equal to this epsilon
+ common_ttl = 10 * 86400, -- TTL of discriminated common elements
+ significant_factor = 3.0 / 4.0, -- which tokens should we update
+ classifiers = {},
+ cluster_nodes = 0,
+}
+
+local template = {}
+
+local function check_redis_classifier(cls, cfg)
+ -- Skip old classifiers
+ if cls.new_schema then
+ local symbol_spam, symbol_ham
+ local expiry = (cls.expiry or cls.expire)
+ if type(expiry) == 'table' then
+ expiry = expiry[1]
+ end
+
+ -- Load symbols from statfiles
+
+ local function check_statfile_table(tbl, def_sym)
+ local symbol = tbl.symbol or def_sym
+
+ local spam
+ if tbl.spam then
+ spam = tbl.spam
+ else
+ if string.match(symbol:upper(), 'SPAM') then
+ spam = true
+ else
+ spam = false
+ end
+ end
+
+ if spam then
+ symbol_spam = symbol
+ else
+ symbol_ham = symbol
+ end
+ end
+
+ local statfiles = cls.statfile
+ if statfiles[1] then
+ for _, stf in ipairs(statfiles) do
+ if not stf.symbol then
+ for k, v in pairs(stf) do
+ check_statfile_table(v, k)
+ end
+ else
+ check_statfile_table(stf, 'undefined')
+ end
+ end
+ else
+ for stn, stf in pairs(statfiles) do
+ check_statfile_table(stf, stn)
+ end
+ end
+
+ if not symbol_spam or not symbol_ham or type(expiry) ~= 'number' then
+ logger.debugm(N, rspamd_config,
+ 'disable expiry for classifier %s: no expiry %s',
+ symbol_spam, cls)
+ return
+ end
+ -- Now try to load redis_params if needed
+
+ local redis_params
+ redis_params = lredis.try_load_redis_servers(cls, rspamd_config, false, 'bayes')
+ if not redis_params then
+ redis_params = lredis.try_load_redis_servers(cfg[N] or E, rspamd_config, false, 'bayes')
+ if not redis_params then
+ redis_params = lredis.try_load_redis_servers(cfg[N] or E, rspamd_config, true)
+ if not redis_params then
+ return false
+ end
+ end
+ end
+
+ if redis_params['read_only'] then
+ logger.infox(rspamd_config, 'disable expiry for classifier %s: read only redis configuration',
+ symbol_spam)
+ return
+ end
+
+ logger.debugm(N, rspamd_config, "enabled expiry for %s/%s -> %s expiry",
+ symbol_spam, symbol_ham, expiry)
+
+ table.insert(settings.classifiers, {
+ symbol_spam = symbol_spam,
+ symbol_ham = symbol_ham,
+ redis_params = redis_params,
+ expiry = expiry
+ })
+ end
+end
+
+-- Check classifiers and try find the appropriate ones
+local obj = rspamd_config:get_ucl()
+
+local classifier = obj.classifier
+
+if classifier then
+ if classifier[1] then
+ for _, cls in ipairs(classifier) do
+ if cls.bayes then
+ cls = cls.bayes
+ end
+ if cls.backend and cls.backend == 'redis' then
+ check_redis_classifier(cls, obj)
+ end
+ end
+ else
+ if classifier.bayes then
+
+ classifier = classifier.bayes
+ if classifier[1] then
+ for _, cls in ipairs(classifier) do
+ if cls.backend and cls.backend == 'redis' then
+ check_redis_classifier(cls, obj)
+ end
+ end
+ else
+ if classifier.backend and classifier.backend == 'redis' then
+ check_redis_classifier(classifier, obj)
+ end
+ end
+ end
+ end
+end
+
+local opts = rspamd_config:get_all_opt(N)
+
+if opts then
+ for k, v in pairs(opts) do
+ settings[k] = v
+ end
+end
+
+-- In clustered setup, we need to increase interval of expiration
+-- according to number of nodes in a cluster
+if settings.cluster_nodes == 0 then
+ local neighbours = obj.neighbours or {}
+ local n_neighbours = 0
+ for _, _ in pairs(neighbours) do
+ n_neighbours = n_neighbours + 1
+ end
+ settings.cluster_nodes = n_neighbours
+end
+
+-- Fill template
+template.count = settings.count
+template.threshold = settings.threshold
+template.common_ttl = settings.common_ttl
+template.epsilon_common = settings.epsilon_common
+template.significant_factor = settings.significant_factor
+template.expire_step = settings.interval
+template.hostname = rspamd_util.get_hostname()
+
+for k, v in pairs(template) do
+ template[k] = tostring(v)
+end
+
+-- Arguments:
+-- [1] = symbol pattern
+-- [2] = expire value
+-- [3] = cursor
+-- returns {cursor for the next step, step number, step statistic counters, cycle statistic counters, tokens occurrences distribution}
+local expiry_script = [[
+ local unpack_function = table.unpack or unpack
+
+ local hash2list = function (hash)
+ local res = {}
+ for k, v in pairs(hash) do
+ table.insert(res, k)
+ table.insert(res, v)
+ end
+ return res
+ end
+
+ local function merge_list(table, list)
+ local k
+ for i, v in ipairs(list) do
+ if i % 2 == 1 then
+ k = v
+ else
+ table[k] = v
+ end
+ end
+ end
+
+ local expire = math.floor(KEYS[2])
+ local pattern_sha1 = redis.sha1hex(KEYS[1])
+
+ local lock_key = pattern_sha1 .. '_lock' -- Check locking
+ local lock = redis.call('GET', lock_key)
+
+ if lock then
+ if lock ~= '${hostname}' then
+ return 'locked by ' .. lock
+ end
+ end
+
+ redis.replicate_commands()
+ redis.call('SETEX', lock_key, ${expire_step}, '${hostname}')
+
+ local cursor_key = pattern_sha1 .. '_cursor'
+ local cursor = tonumber(redis.call('GET', cursor_key) or 0)
+
+ local step = 1
+ local step_key = pattern_sha1 .. '_step'
+ if cursor > 0 then
+ step = redis.call('GET', step_key)
+ step = step and (tonumber(step) + 1) or 1
+ end
+
+ local ret = redis.call('SCAN', cursor, 'MATCH', KEYS[1], 'COUNT', '${count}')
+ local next_cursor = ret[1]
+ local keys = ret[2]
+ local tokens = {}
+
+ -- Tokens occurrences distribution counters
+ local occur = {
+ ham = {},
+ spam = {},
+ total = {}
+ }
+
+ -- Expiry step statistics counters
+ local nelts, extended, discriminated, sum, sum_squares, common, significant,
+ infrequent, infrequent_ttls_set, insignificant, insignificant_ttls_set =
+ 0,0,0,0,0,0,0,0,0,0,0
+
+ for _,key in ipairs(keys) do
+ local t = redis.call('TYPE', key)["ok"]
+ if t == 'hash' then
+ local values = redis.call('HMGET', key, 'H', 'S')
+ local ham = tonumber(values[1]) or 0
+ local spam = tonumber(values[2]) or 0
+ local ttl = redis.call('TTL', key)
+ tokens[key] = {
+ ham,
+ spam,
+ ttl
+ }
+ local total = spam + ham
+ sum = sum + total
+ sum_squares = sum_squares + total * total
+ nelts = nelts + 1
+
+ for k,v in pairs({['ham']=ham, ['spam']=spam, ['total']=total}) do
+ if tonumber(v) > 19 then v = 20 end
+ occur[k][v] = occur[k][v] and occur[k][v] + 1 or 1
+ end
+ end
+ end
+
+ local mean, stddev = 0, 0
+
+ if nelts > 0 then
+ mean = sum / nelts
+ stddev = math.sqrt(sum_squares / nelts - mean * mean)
+ end
+
+ for key,token in pairs(tokens) do
+ local ham, spam, ttl = token[1], token[2], tonumber(token[3])
+ local threshold = mean
+ local total = spam + ham
+
+ local function set_ttl()
+ if expire < 0 then
+ if ttl ~= -1 then
+ redis.call('PERSIST', key)
+ return 1
+ end
+ elseif ttl == -1 or ttl > expire then
+ redis.call('EXPIRE', key, expire)
+ return 1
+ end
+ return 0
+ end
+
+ if total == 0 or math.abs(ham - spam) <= total * ${epsilon_common} then
+ common = common + 1
+ if ttl > ${common_ttl} then
+ discriminated = discriminated + 1
+ redis.call('EXPIRE', key, ${common_ttl})
+ end
+ elseif total >= threshold and total > 0 then
+ if ham / total > ${significant_factor} or spam / total > ${significant_factor} then
+ significant = significant + 1
+ if ttl ~= -1 then
+ redis.call('PERSIST', key)
+ extended = extended + 1
+ end
+ else
+ insignificant = insignificant + 1
+ insignificant_ttls_set = insignificant_ttls_set + set_ttl()
+ end
+ else
+ infrequent = infrequent + 1
+ infrequent_ttls_set = infrequent_ttls_set + set_ttl()
+ end
+ end
+
+ -- Expiry cycle statistics counters
+ local c = {nelts = 0, extended = 0, discriminated = 0, sum = 0, sum_squares = 0,
+ common = 0, significant = 0, infrequent = 0, infrequent_ttls_set = 0, insignificant = 0, insignificant_ttls_set = 0}
+
+ local counters_key = pattern_sha1 .. '_counters'
+
+ if cursor ~= 0 then
+ merge_list(c, redis.call('HGETALL', counters_key))
+ end
+
+ c.nelts = c.nelts + nelts
+ c.extended = c.extended + extended
+ c.discriminated = c.discriminated + discriminated
+ c.sum = c.sum + sum
+ c.sum_squares = c.sum_squares + sum_squares
+ c.common = c.common + common
+ c.significant = c.significant + significant
+ c.infrequent = c.infrequent + infrequent
+ c.infrequent_ttls_set = c.infrequent_ttls_set + infrequent_ttls_set
+ c.insignificant = c.insignificant + insignificant
+ c.insignificant_ttls_set = c.insignificant_ttls_set + insignificant_ttls_set
+
+ redis.call('HMSET', counters_key, unpack_function(hash2list(c)))
+ redis.call('SET', cursor_key, tostring(next_cursor))
+ redis.call('SET', step_key, tostring(step))
+ redis.call('DEL', lock_key)
+
+ local occ_distr = {}
+ for _,cl in pairs({'ham', 'spam', 'total'}) do
+ local occur_key = pattern_sha1 .. '_occurrence_' .. cl
+
+ if cursor ~= 0 then
+ local n
+ for i,v in ipairs(redis.call('HGETALL', occur_key)) do
+ if i % 2 == 1 then
+ n = tonumber(v)
+ else
+ occur[cl][n] = occur[cl][n] and occur[cl][n] + v or v
+ end
+ end
+
+ local str = ''
+ if occur[cl][0] ~= nil then
+ str = '0:' .. occur[cl][0] .. ','
+ end
+ for k,v in ipairs(occur[cl]) do
+ if k == 20 then k = '>19' end
+ str = str .. k .. ':' .. v .. ','
+ end
+ table.insert(occ_distr, str)
+ else
+ redis.call('DEL', occur_key)
+ end
+
+ if next(occur[cl]) ~= nil then
+ redis.call('HMSET', occur_key, unpack_function(hash2list(occur[cl])))
+ end
+ end
+
+ return {
+ next_cursor, step,
+ {nelts, extended, discriminated, mean, stddev, common, significant, infrequent,
+ infrequent_ttls_set, insignificant, insignificant_ttls_set},
+ {c.nelts, c.extended, c.discriminated, c.sum, c.sum_squares, c.common,
+ c.significant, c.infrequent, c.infrequent_ttls_set, c.insignificant, c.insignificant_ttls_set},
+ occ_distr
+ }
+]]
+
+local function expire_step(cls, ev_base, worker)
+ local function redis_step_cb(err, args)
+ if err then
+ logger.errx(rspamd_config, 'cannot perform expiry step: %s', err)
+ elseif type(args) == 'table' then
+ local cur = tonumber(args[1])
+ local step = args[2]
+ local data = args[3]
+ local c_data = args[4]
+ local occ_distr = args[5]
+
+ local function log_stat(cycle)
+ local infrequent_action = (cls.expiry < 0) and 'made persistent' or 'ttls set'
+
+ local c_mean, c_stddev = 0, 0
+ if cycle and c_data[1] ~= 0 then
+ c_mean = c_data[4] / c_data[1]
+ c_stddev = math.floor(.5 + math.sqrt(c_data[5] / c_data[1] - c_mean * c_mean))
+ c_mean = math.floor(.5 + c_mean)
+ end
+
+ local d = cycle and {
+ 'cycle in ' .. step .. ' steps', c_data[1],
+ c_data[7], c_data[2], 'made persistent',
+ c_data[10], c_data[11], infrequent_action,
+ c_data[6], c_data[3],
+ c_data[8], c_data[9], infrequent_action,
+ c_mean,
+ c_stddev
+ } or {
+ 'step ' .. step, data[1],
+ data[7], data[2], 'made persistent',
+ data[10], data[11], infrequent_action,
+ data[6], data[3],
+ data[8], data[9], infrequent_action,
+ data[4],
+ data[5]
+ }
+ logger.infox(rspamd_config,
+ 'finished expiry %s: %s items checked, %s significant (%s %s), ' ..
+ '%s insignificant (%s %s), %s common (%s discriminated), ' ..
+ '%s infrequent (%s %s), %s mean, %s std',
+ lutil.unpack(d))
+ if cycle then
+ for i, cl in ipairs({ 'in ham', 'in spam', 'total' }) do
+ logger.infox(rspamd_config, 'tokens occurrences, %s: {%s}', cl, occ_distr[i])
+ end
+ end
+ end
+ log_stat(false)
+ if cur == 0 then
+ log_stat(true)
+ end
+ elseif type(args) == 'string' then
+ logger.infox(rspamd_config, 'skip expiry step: %s', args)
+ end
+ end
+ lredis.exec_redis_script(cls.script,
+ { ev_base = ev_base, is_write = true },
+ redis_step_cb,
+ { 'RS*_*', cls.expiry }
+ )
+end
+
+rspamd_config:add_on_load(function(_, ev_base, worker)
+ -- Exit unless we're the first 'controller' worker
+ if not worker:is_primary_controller() then
+ return
+ end
+
+ local unique_redis_params = {}
+ -- Push redis script to all unique redis servers
+ for _, cls in ipairs(settings.classifiers) do
+ if not unique_redis_params[cls.redis_params.hash] then
+ unique_redis_params[cls.redis_params.hash] = cls.redis_params
+ end
+ end
+
+ for h, rp in pairs(unique_redis_params) do
+ local script_id = lredis.add_redis_script(lutil.template(expiry_script,
+ template), rp)
+
+ for _, cls in ipairs(settings.classifiers) do
+ if cls.redis_params.hash == h then
+ cls.script = script_id
+ end
+ end
+ end
+
+ -- Expire tokens at regular intervals
+ for _, cls in ipairs(settings.classifiers) do
+ rspamd_config:add_periodic(ev_base,
+ settings['interval'],
+ function()
+ expire_step(cls, ev_base, worker)
+ return true
+ end, true)
+ end
+end)
diff --git a/src/plugins/lua/bimi.lua b/src/plugins/lua/bimi.lua
new file mode 100644
index 0000000..2783590
--- /dev/null
+++ b/src/plugins/lua/bimi.lua
@@ -0,0 +1,391 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local N = "bimi"
+local lua_util = require "lua_util"
+local rspamd_logger = require "rspamd_logger"
+local ts = (require "tableshape").types
+local lua_redis = require "lua_redis"
+local ucl = require "ucl"
+local lua_mime = require "lua_mime"
+local rspamd_http = require "rspamd_http"
+local rspamd_util = require "rspamd_util"
+
+local settings = {
+ helper_url = "http://127.0.0.1:3030",
+ helper_timeout = 5,
+ helper_sync = true,
+ vmc_only = true,
+ redis_prefix = 'rs_bimi',
+ redis_min_expiry = 24 * 3600,
+}
+local redis_params
+
+local settings_schema = lua_redis.enrich_schema({
+ helper_url = ts.string,
+ helper_timeout = ts.number + ts.string / lua_util.parse_time_interval,
+ helper_sync = ts.boolean,
+ vmc_only = ts.boolean,
+ redis_min_expiry = ts.number + ts.string / lua_util.parse_time_interval,
+ redis_prefix = ts.string,
+ enabled = ts.boolean:is_optional(),
+})
+
+local function check_dmarc_policy(task)
+ local dmarc_sym = task:get_symbol('DMARC_POLICY_ALLOW')
+
+ if not dmarc_sym then
+ lua_util.debugm(N, task, "no DMARC allow symbol")
+ return nil
+ end
+
+ local opts = dmarc_sym[1].options or {}
+ if not opts[1] or #opts ~= 2 then
+ lua_util.debugm(N, task, "DMARC options are bogus: %s", opts)
+ return nil
+ end
+
+ -- opts[1] - domain; opts[2] - policy
+ local dom, policy = opts[1], opts[2]
+
+ if policy ~= 'reject' and policy ~= 'quarantine' then
+ lua_util.debugm(N, task, "DMARC policy for domain %s is not strict: %s",
+ dom, policy)
+ return nil
+ end
+
+ return dom
+end
+
+local function gen_bimi_grammar()
+ local lpeg = require "lpeg"
+ lpeg.locale(lpeg)
+ local space = lpeg.space ^ 0
+ local name = lpeg.C(lpeg.alpha ^ 1) * space
+ local sep = (lpeg.S("\\;") * space) + (lpeg.space ^ 1)
+ local value = lpeg.C(lpeg.P(lpeg.graph - sep) ^ 1)
+ local pair = lpeg.Cg(name * "=" * space * value) * sep ^ -1
+ local list = lpeg.Cf(lpeg.Ct("") * pair ^ 0, rawset)
+ local version = lpeg.P("v") * space * lpeg.P("=") * space * lpeg.P("BIMI1")
+ local record = version * sep * list
+
+ return record
+end
+
+local bimi_grammar = gen_bimi_grammar()
+
+local function check_bimi_record(task, rec)
+ local elts = bimi_grammar:match(rec)
+
+ if elts then
+ lua_util.debugm(N, task, "got BIMI record: %s, processed=%s",
+ rec, elts)
+ local res = {}
+
+ if type(elts.l) == 'string' then
+ res.l = elts.l
+ end
+ if type(elts.a) == 'string' then
+ res.a = elts.a
+ end
+
+ if res.l or res.a then
+ return res
+ end
+ end
+end
+
+local function insert_bimi_headers(task, domain, bimi_content)
+ local hdr_name = 'BIMI-Indicator'
+ -- Re-encode base64...
+ local content = rspamd_util.encode_base64(rspamd_util.decode_base64(bimi_content),
+ 73, task:get_newlines_type())
+ lua_mime.modify_headers(task, {
+ remove = { [hdr_name] = 0 },
+ add = {
+ [hdr_name] = {
+ order = 0,
+ value = rspamd_util.fold_header(hdr_name, content,
+ task:get_newlines_type())
+ }
+ }
+ })
+ task:insert_result('BIMI_VALID', 1.0, { domain })
+end
+
+local function process_bimi_json(task, domain, redis_data)
+ local parser = ucl.parser()
+ local _, err = parser:parse_string(redis_data)
+
+ if err then
+ rspamd_logger.errx(task, "cannot parse BIMI result from Redis for %s: %s",
+ domain, err)
+ else
+ local d = parser:get_object()
+ if d.content then
+ insert_bimi_headers(task, domain, d.content)
+ elseif d.error then
+ lua_util.debugm(N, task, "invalid BIMI for %s: %s",
+ domain, d.error)
+ end
+ end
+end
+
+local function make_helper_request(task, domain, record, redis_server)
+ local is_sync = settings.helper_sync
+ local helper_url = string.format('%s/v1/check', settings.helper_url)
+ local redis_key = string.format('%s%s', settings.redis_prefix,
+ domain)
+
+ local function http_helper_callback(http_err, code, body, _)
+ if http_err then
+ rspamd_logger.warnx(task, 'got error reply from helper %s: code=%s; reply=%s',
+ helper_url, code, http_err)
+ return
+ end
+ if code ~= 200 then
+ rspamd_logger.warnx(task, 'got non 200 reply from helper %s: code=%s; reply=%s',
+ helper_url, code, http_err)
+ return
+ end
+ if is_sync then
+ local parser = ucl.parser()
+ local _, err = parser:parse_string(body)
+
+ if err then
+ rspamd_logger.errx(task, "cannot parse BIMI result from helper for %s: %s",
+ domain, err)
+ else
+ local d = parser:get_object()
+ if d.content then
+ insert_bimi_headers(task, domain, d.content)
+ elseif d.error then
+ lua_util.debugm(N, task, "invalid BIMI for %s: %s",
+ domain, d.error)
+ end
+
+ local ret, upstream
+ local function redis_set_cb(redis_err, _)
+ if redis_err then
+ rspamd_logger.warnx(task, 'cannot get reply from Redis when storing image %s: %s',
+ upstream:get_addr():to_string(), redis_err)
+ upstream:fail()
+ else
+ lua_util.debugm(N, task, 'stored bimi image in Redis for domain %s; key=%s',
+ domain, redis_key)
+ end
+ end
+
+ ret, _, upstream = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ redis_key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'PSETEX', -- command
+ { redis_key, tostring(settings.redis_min_expiry * 1000.0),
+ ucl.to_format(d, "json-compact") })
+
+ if not ret then
+ rspamd_logger.warnx(task, 'cannot make request to Redis when storing image; domain %s',
+ domain)
+ end
+ end
+ else
+ -- In async mode we skip request and use merely Redis to insert indicators
+ lua_util.debugm(N, task, "sent request to resolve %s to %s",
+ domain, helper_url)
+ end
+ end
+
+ local request_data = {
+ url = record.a,
+ sync = is_sync,
+ domain = domain
+ }
+
+ if not is_sync then
+ -- Allow bimi helper to save data in Redis
+ request_data.redis_server = redis_server
+ request_data.redis_prefix = settings.redis_prefix
+ request_data.redis_expiry = settings.redis_min_expiry * 1000.0
+ else
+ request_data.skip_redis = true
+ end
+
+ local serialised = ucl.to_format(request_data, 'json-compact')
+ lua_util.debugm(N, task, "send request to BIMI helper: %s",
+ serialised)
+ rspamd_http.request({
+ task = task,
+ mime_type = 'application/json',
+ timeout = settings.helper_timeout,
+ body = serialised,
+ url = helper_url,
+ callback = http_helper_callback,
+ keepalive = true,
+ })
+end
+
+local function check_bimi_vmc(task, domain, record)
+ local redis_key = string.format('%s%s', settings.redis_prefix,
+ domain)
+ local ret, _, upstream
+
+ local function redis_cached_cb(err, data)
+ if err then
+ rspamd_logger.warnx(task, 'cannot get reply from Redis %s: %s',
+ upstream:get_addr():to_string(), err)
+ upstream:fail()
+ else
+ if type(data) == 'string' then
+ -- We got a cached record, good stuff
+ lua_util.debugm(N, task, "got valid cached BIMI result for domain: %s",
+ domain)
+ process_bimi_json(task, domain, data)
+ else
+ -- Get server addr + port
+ -- We need to fix IPv6 address as redis-rs has no support of
+ -- the braced IPv6 addresses
+ local db, password = '', ''
+ if redis_params.db then
+ db = string.format('/%s', redis_params.db)
+ end
+ if redis_params.username then
+ if redis_params.password then
+ password = string.format( '%s:%s@', redis_params.username, redis_params.password)
+ else
+ rspamd_logger.warnx(task, "Redis requires a password when username is supplied")
+ end
+ elseif redis_params.password then
+ password = string.format(':%s@', redis_params.password)
+ end
+ local redis_server = string.format('redis://%s%s:%s%s',
+ password,
+ upstream:get_name(), upstream:get_port(),
+ db)
+ make_helper_request(task, domain, record, redis_server)
+ end
+ end
+ end
+
+ -- We first check Redis and then try to use helper
+ ret, _, upstream = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ redis_key, -- hash key
+ false, -- is write
+ redis_cached_cb, --callback
+ 'GET', -- command
+ { redis_key })
+
+ if not ret then
+ rspamd_logger.warnx(task, 'cannot make request to Redis; domain %s', domain)
+ end
+end
+
+local function check_bimi_dns(task, domain)
+ local resolve_name = string.format('default._bimi.%s', domain)
+ local dns_cb = function(_, _, results, err)
+ if err then
+ lua_util.debugm(N, task, "cannot resolve bimi for %s: %s",
+ domain, err)
+ else
+ for _, rec in ipairs(results) do
+ local res = check_bimi_record(task, rec)
+
+ if res then
+ if settings.vmc_only and not res.a then
+ lua_util.debugm(N, task, "BIMI for domain %s has no VMC, skip it",
+ domain)
+
+ return
+ end
+
+ if res.a then
+ check_bimi_vmc(task, domain, res)
+ elseif res.l then
+ -- TODO: add l check
+ lua_util.debugm(N, task, "l only BIMI for domain %s is not implemented yet",
+ domain)
+ end
+ end
+ end
+ end
+ end
+ task:get_resolver():resolve_txt({
+ task = task,
+ name = resolve_name,
+ callback = dns_cb,
+ forced = true
+ })
+end
+
+local function bimi_callback(task)
+ local dmarc_domain_maybe = check_dmarc_policy(task)
+
+ if not dmarc_domain_maybe then
+ return
+ end
+
+
+ -- We can either check BIMI via DNS or check Redis cache
+ -- BIMI check is an external check, so we might prefer Redis to be checked
+ -- first. On the other hand, DNS request is cheaper and counting low BIMI
+ -- adaptation we would need to have both Redis and DNS request to hit no
+ -- result. So, it might be better to check DNS first at this stage...
+ check_bimi_dns(task, dmarc_domain_maybe)
+end
+
+local opts = rspamd_config:get_all_opt('bimi')
+if not opts then
+ lua_util.disable_module(N, "config")
+ return
+end
+
+settings = lua_util.override_defaults(settings, opts)
+local res, err = settings_schema:transform(settings)
+
+if not res then
+ rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err)
+ local err_msg = string.format("schema error: %s", res)
+ lua_util.config_utils.push_config_error(N, err_msg)
+ lua_util.disable_module(N, "failed", err_msg)
+ return
+end
+
+rspamd_logger.infox(rspamd_config, 'enabled BIMI plugin')
+
+settings = res
+redis_params = lua_redis.parse_redis_server(N, opts)
+
+if redis_params then
+ local id = rspamd_config:register_symbol({
+ name = 'BIMI_CHECK',
+ type = 'normal',
+ callback = bimi_callback,
+ augmentations = { string.format("timeout=%f", settings.helper_timeout or
+ redis_params.timeout or 0.0) }
+ })
+ rspamd_config:register_symbol {
+ name = 'BIMI_VALID',
+ type = 'virtual',
+ parent = id,
+ score = 0.0
+ }
+
+ rspamd_config:register_dependency('BIMI_CHECK', 'DMARC_CHECK')
+else
+ lua_util.disable_module(N, "redis")
+end
diff --git a/src/plugins/lua/clickhouse.lua b/src/plugins/lua/clickhouse.lua
new file mode 100644
index 0000000..25eabc7
--- /dev/null
+++ b/src/plugins/lua/clickhouse.lua
@@ -0,0 +1,1556 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local rspamd_logger = require 'rspamd_logger'
+local upstream_list = require "rspamd_upstream_list"
+local lua_util = require "lua_util"
+local lua_clickhouse = require "lua_clickhouse"
+local lua_settings = require "lua_settings"
+local fun = require "fun"
+
+local N = "clickhouse"
+
+if confighelp then
+ return
+end
+
+local data_rows = {}
+local custom_rows = {}
+local nrows = 0
+local used_memory = 0
+local last_collection = 0
+local final_call = false -- If the final collection has been started
+local schema_version = 9 -- Current schema version
+
+local settings = {
+ limits = { -- Collection limits
+ max_rows = 1000, -- How many rows are allowed (0 for disable this)
+ max_memory = 50 * 1024 * 1024, -- How many memory should be occupied before sending collection
+ max_interval = 60, -- Maximum collection interval
+ },
+ collect_garbage = false, -- Perform GC collection after sending the data
+ check_timeout = 10.0, -- Periodic timeout
+ timeout = 5.0,
+ bayes_spam_symbols = { 'BAYES_SPAM' },
+ bayes_ham_symbols = { 'BAYES_HAM' },
+ ann_symbols_spam = { 'NEURAL_SPAM' },
+ ann_symbols_ham = { 'NEURAL_HAM' },
+ fuzzy_symbols = { 'FUZZY_DENIED' },
+ whitelist_symbols = { 'WHITELIST_DKIM', 'WHITELIST_SPF_DKIM', 'WHITELIST_DMARC' },
+ dkim_allow_symbols = { 'R_DKIM_ALLOW' },
+ dkim_reject_symbols = { 'R_DKIM_REJECT' },
+ dkim_dnsfail_symbols = { 'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL' },
+ dkim_na_symbols = { 'R_DKIM_NA' },
+ dmarc_allow_symbols = { 'DMARC_POLICY_ALLOW' },
+ dmarc_reject_symbols = { 'DMARC_POLICY_REJECT' },
+ dmarc_quarantine_symbols = { 'DMARC_POLICY_QUARANTINE' },
+ dmarc_softfail_symbols = { 'DMARC_POLICY_SOFTFAIL' },
+ dmarc_na_symbols = { 'DMARC_NA' },
+ spf_allow_symbols = { 'R_SPF_ALLOW' },
+ spf_reject_symbols = { 'R_SPF_FAIL' },
+ spf_dnsfail_symbols = { 'R_SPF_DNSFAIL', 'R_SPF_PERMFAIL' },
+ spf_neutral_symbols = { 'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL' },
+ spf_na_symbols = { 'R_SPF_NA' },
+ stop_symbols = {},
+ ipmask = 19,
+ ipmask6 = 48,
+ full_urls = false,
+ from_tables = nil,
+ enable_symbols = false,
+ database = 'default',
+ use_https = false,
+ use_gzip = true,
+ allow_local = false,
+ insert_subject = false,
+ subject_privacy = false, -- subject privacy is off
+ subject_privacy_alg = 'blake2', -- default hash-algorithm to obfuscate subject
+ subject_privacy_prefix = 'obf', -- prefix to show it's obfuscated
+ subject_privacy_length = 16, -- cut the length of the hash
+ schema_additions = {}, -- additional SQL statements to be executed when schema is uploaded
+ user = nil,
+ password = nil,
+ no_ssl_verify = false,
+ custom_rules = {},
+ enable_digest = false,
+ exceptions = nil,
+ retention = {
+ enable = false,
+ method = 'detach',
+ period_months = 3,
+ run_every = '7d',
+ },
+ extra_columns = {},
+}
+
+--- @language SQL
+local clickhouse_schema = { [[
+CREATE TABLE IF NOT EXISTS rspamd
+(
+ Date Date COMMENT 'Date (used for partitioning)',
+ TS DateTime COMMENT 'Date and time of the request start (UTC)',
+ From String COMMENT 'Domain part of the return address (RFC5321.MailFrom)',
+ MimeFrom String COMMENT 'Domain part of the address in From: header (RFC5322.From)',
+ IP String COMMENT 'SMTP client IP as provided by MTA or from Received: header',
+ Helo String COMMENT 'Full hostname as sent by the SMTP client (RFC5321.HELO/.EHLO)',
+ Score Float32 COMMENT 'Message score',
+ NRcpt UInt8 COMMENT 'Number of envelope recipients (RFC5321.RcptTo)',
+ Size UInt32 COMMENT 'Message size in bytes',
+ IsWhitelist Enum8('blacklist' = 0, 'whitelist' = 1, 'unknown' = 2) DEFAULT 'unknown' COMMENT 'Based on symbols configured in `whitelist_symbols` module option',
+ IsBayes Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2) DEFAULT 'unknown' COMMENT 'Based on symbols configured in `bayes_spam_symbols` and `bayes_ham_symbols` module options',
+ IsFuzzy Enum8('whitelist' = 0, 'deny' = 1, 'unknown' = 2) DEFAULT 'unknown' COMMENT 'Based on symbols configured in `fuzzy_symbols` module option',
+ IsFann Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2) DEFAULT 'unknown' COMMENT 'Based on symbols configured in `ann_symbols_spam` and `ann_symbols_ham` module options',
+ IsDkim Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2, 'dnsfail' = 3, 'na' = 4) DEFAULT 'unknown' COMMENT 'Based on symbols configured in dkim_* module options',
+ IsDmarc Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2, 'softfail' = 3, 'na' = 4, 'quarantine' = 5) DEFAULT 'unknown' COMMENT 'Based on symbols configured in dmarc_* module options',
+ IsSpf Enum8('reject' = 0, 'allow' = 1, 'neutral' = 2, 'dnsfail' = 3, 'na' = 4, 'unknown' = 5) DEFAULT 'unknown' COMMENT 'Based on symbols configured in spf_* module options',
+ NUrls Int32 COMMENT 'Number of URLs and email extracted from the message',
+ Action Enum8('reject' = 0, 'rewrite subject' = 1, 'add header' = 2, 'greylist' = 3, 'no action' = 4, 'soft reject' = 5, 'custom' = 6) DEFAULT 'no action' COMMENT 'Action returned for the message; if action is not predefined actual action will be in `CustomAction` field',
+ CustomAction LowCardinality(String) COMMENT 'Action string for custom action',
+ FromUser String COMMENT 'Local part of the return address (RFC5321.MailFrom)',
+ MimeUser String COMMENT 'Local part of the address in From: header (RFC5322.From)',
+ RcptUser String COMMENT '[Deprecated] Local part of the first envelope recipient (RFC5321.RcptTo)',
+ RcptDomain String COMMENT '[Deprecated] Domain part of the first envelope recipient (RFC5321.RcptTo)',
+ SMTPRecipients Array(String) COMMENT 'List of envelope recipients (RFC5321.RcptTo)',
+ MimeRecipients Array(String) COMMENT 'List of recipients from headers (RFC5322.To/.CC/.BCC)',
+ MessageId String COMMENT 'Message-ID header',
+ ListId String COMMENT 'List-Id header',
+ Subject String COMMENT 'Subject header (or hash if `subject_privacy` module option enabled)',
+ `Attachments.FileName` Array(String) COMMENT 'Attachment name',
+ `Attachments.ContentType` Array(String) COMMENT 'Attachment Content-Type',
+ `Attachments.Length` Array(UInt32) COMMENT 'Attachment size in bytes',
+ `Attachments.Digest` Array(FixedString(16)) COMMENT 'First 16 characters of hash returned by mime_part:get_digest()',
+ `Urls.Tld` Array(String) COMMENT 'Effective second level domain part of the URL host',
+ `Urls.Url` Array(String) COMMENT 'Full URL if `full_urls` module option enabled, host part of URL otherwise',
+ `Urls.Flags` Array(UInt32) COMMENT 'Corresponding url flags, see `enum rspamd_url_flags` in libserver/url.h for details',
+ Emails Array(String) COMMENT 'List of emails extracted from the message',
+ ASN UInt32 COMMENT 'BGP AS number for SMTP client IP (returned by asn.rspamd.com or asn6.rspamd.com)',
+ Country FixedString(2) COMMENT 'Country for SMTP client IP (returned by asn.rspamd.com or asn6.rspamd.com)',
+ IPNet String,
+ `Symbols.Names` Array(LowCardinality(String)) COMMENT 'Symbol name',
+ `Symbols.Scores` Array(Float32) COMMENT 'Symbol score',
+ `Symbols.Options` Array(String) COMMENT 'Symbol options (comma separated list)',
+ `Groups.Names` Array(LowCardinality(String)) COMMENT 'Group name',
+ `Groups.Scores` Array(Float32) COMMENT 'Group score',
+ ScanTimeReal UInt32 COMMENT 'Request time in milliseconds',
+ ScanTimeVirtual UInt32 COMMENT 'Deprecated do not use',
+ AuthUser String COMMENT 'Username for authenticated SMTP client',
+ SettingsId LowCardinality(String) COMMENT 'ID for the settings profile',
+ Digest FixedString(32) COMMENT '[Deprecated]',
+ SMTPFrom ALIAS if(From = '', '', concat(FromUser, '@', From)) COMMENT 'Return address (RFC5321.MailFrom)',
+ SMTPRcpt ALIAS SMTPRecipients[1] COMMENT 'The first envelope recipient (RFC5321.RcptTo)',
+ MIMEFrom ALIAS if(MimeFrom = '', '', concat(MimeUser, '@', MimeFrom)) COMMENT 'Address in From: header (RFC5322.From)',
+ MIMERcpt ALIAS MimeRecipients[1] COMMENT 'The first recipient from headers (RFC5322.To/.CC/.BCC)'
+) ENGINE = MergeTree()
+PARTITION BY toMonday(Date)
+ORDER BY TS
+]],
+ [[CREATE TABLE IF NOT EXISTS rspamd_version ( Version UInt32) ENGINE = TinyLog]],
+ { [[INSERT INTO rspamd_version (Version) Values (${SCHEMA_VERSION})]], true },
+}
+
+-- This describes SQL queries to migrate between versions
+local migrations = {
+ [1] = {
+ -- Move to a wide fat table
+ [[ALTER TABLE rspamd
+ ADD COLUMN IF NOT EXISTS `Attachments.FileName` Array(String) AFTER ListId,
+ ADD COLUMN IF NOT EXISTS `Attachments.ContentType` Array(String) AFTER `Attachments.FileName`,
+ ADD COLUMN IF NOT EXISTS `Attachments.Length` Array(UInt32) AFTER `Attachments.ContentType`,
+ ADD COLUMN IF NOT EXISTS `Attachments.Digest` Array(FixedString(16)) AFTER `Attachments.Length`,
+ ADD COLUMN IF NOT EXISTS `Urls.Tld` Array(String) AFTER `Attachments.Digest`,
+ ADD COLUMN IF NOT EXISTS `Urls.Url` Array(String) AFTER `Urls.Tld`,
+ ADD COLUMN IF NOT EXISTS Emails Array(String) AFTER `Urls.Url`,
+ ADD COLUMN IF NOT EXISTS ASN UInt32 AFTER Emails,
+ ADD COLUMN IF NOT EXISTS Country FixedString(2) AFTER ASN,
+ ADD COLUMN IF NOT EXISTS IPNet String AFTER Country,
+ ADD COLUMN IF NOT EXISTS `Symbols.Names` Array(String) AFTER IPNet,
+ ADD COLUMN IF NOT EXISTS `Symbols.Scores` Array(Float64) AFTER `Symbols.Names`,
+ ADD COLUMN IF NOT EXISTS `Symbols.Options` Array(String) AFTER `Symbols.Scores`]],
+ -- Add explicit version
+ [[CREATE TABLE rspamd_version ( Version UInt32) ENGINE = TinyLog]],
+ [[INSERT INTO rspamd_version (Version) Values (2)]],
+ },
+ [2] = {
+ -- Add `Subject` column
+ [[ALTER TABLE rspamd
+ ADD COLUMN IF NOT EXISTS Subject String AFTER ListId]],
+ -- New version
+ [[INSERT INTO rspamd_version (Version) Values (3)]],
+ },
+ [3] = {
+ [[ALTER TABLE rspamd
+ ADD COLUMN IF NOT EXISTS IsSpf Enum8('reject' = 0, 'allow' = 1, 'neutral' = 2, 'dnsfail' = 3, 'na' = 4, 'unknown' = 5) DEFAULT 'unknown' AFTER IsDmarc,
+ MODIFY COLUMN IsDkim Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2, 'dnsfail' = 3, 'na' = 4) DEFAULT 'unknown',
+ MODIFY COLUMN IsDmarc Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2, 'softfail' = 3, 'na' = 4, 'quarantine' = 5) DEFAULT 'unknown',
+ ADD COLUMN IF NOT EXISTS MimeRecipients Array(String) AFTER RcptDomain,
+ ADD COLUMN IF NOT EXISTS MessageId String AFTER MimeRecipients,
+ ADD COLUMN IF NOT EXISTS ScanTimeReal UInt32 AFTER `Symbols.Options`,
+ ADD COLUMN IF NOT EXISTS ScanTimeVirtual UInt32 AFTER ScanTimeReal]],
+ -- Add aliases
+ [[ALTER TABLE rspamd
+ ADD COLUMN IF NOT EXISTS SMTPFrom ALIAS if(From = '', '', concat(FromUser, '@', From)),
+ ADD COLUMN IF NOT EXISTS SMTPRcpt ALIAS if(RcptDomain = '', '', concat(RcptUser, '@', RcptDomain)),
+ ADD COLUMN IF NOT EXISTS MIMEFrom ALIAS if(MimeFrom = '', '', concat(MimeUser, '@', MimeFrom)),
+ ADD COLUMN IF NOT EXISTS MIMERcpt ALIAS MimeRecipients[1]
+ ]],
+ -- New version
+ [[INSERT INTO rspamd_version (Version) Values (4)]],
+ },
+ [4] = {
+ [[ALTER TABLE rspamd
+ MODIFY COLUMN Action Enum8('reject' = 0, 'rewrite subject' = 1, 'add header' = 2, 'greylist' = 3, 'no action' = 4, 'soft reject' = 5, 'custom' = 6) DEFAULT 'no action',
+ ADD COLUMN IF NOT EXISTS CustomAction String AFTER Action
+ ]],
+ -- New version
+ [[INSERT INTO rspamd_version (Version) Values (5)]],
+ },
+ [5] = {
+ [[ALTER TABLE rspamd
+ ADD COLUMN IF NOT EXISTS AuthUser String AFTER ScanTimeVirtual,
+ ADD COLUMN IF NOT EXISTS SettingsId LowCardinality(String) AFTER AuthUser
+ ]],
+ -- New version
+ [[INSERT INTO rspamd_version (Version) Values (6)]],
+ },
+ [6] = {
+ -- Add new columns
+ [[ALTER TABLE rspamd
+ ADD COLUMN IF NOT EXISTS Helo String AFTER IP,
+ ADD COLUMN IF NOT EXISTS SMTPRecipients Array(String) AFTER RcptDomain
+ ]],
+ -- Modify SMTPRcpt alias
+ [[
+ ALTER TABLE rspamd
+ MODIFY COLUMN SMTPRcpt ALIAS SMTPRecipients[1]
+ ]],
+ -- New version
+ [[INSERT INTO rspamd_version (Version) Values (7)]],
+ },
+ [7] = {
+ -- Add new columns
+ [[ALTER TABLE rspamd
+ ADD COLUMN IF NOT EXISTS `Groups.Names` Array(LowCardinality(String)) AFTER `Symbols.Options`,
+ ADD COLUMN IF NOT EXISTS `Groups.Scores` Array(Float32) AFTER `Groups.Names`
+ ]],
+ -- New version
+ [[INSERT INTO rspamd_version (Version) Values (8)]],
+ },
+ [8] = {
+ -- Add new columns
+ [[ALTER TABLE rspamd
+ ADD COLUMN IF NOT EXISTS `Urls.Flags` Array(UInt32) AFTER `Urls.Url`
+ ]],
+ -- New version
+ [[INSERT INTO rspamd_version (Version) Values (9)]],
+ },
+}
+
+local predefined_actions = {
+ ['reject'] = true,
+ ['rewrite subject'] = true,
+ ['add header'] = true,
+ ['greylist'] = true,
+ ['no action'] = true,
+ ['soft reject'] = true
+}
+
+local function clickhouse_main_row(res)
+ local fields = {
+ 'Date',
+ 'TS',
+ 'From',
+ 'MimeFrom',
+ 'IP',
+ 'Helo',
+ 'Score',
+ 'NRcpt',
+ 'Size',
+ 'IsWhitelist',
+ 'IsBayes',
+ 'IsFuzzy',
+ 'IsFann',
+ 'IsDkim',
+ 'IsDmarc',
+ 'NUrls',
+ 'Action',
+ 'FromUser',
+ 'MimeUser',
+ 'RcptUser',
+ 'RcptDomain',
+ 'SMTPRecipients',
+ 'ListId',
+ 'Subject',
+ 'Digest',
+ -- 1.9.2 +
+ 'IsSpf',
+ 'MimeRecipients',
+ 'MessageId',
+ 'ScanTimeReal',
+ -- 1.9.3 +
+ 'CustomAction',
+ -- 2.0 +
+ 'AuthUser',
+ 'SettingsId',
+ }
+
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
+end
+
+local function clickhouse_attachments_row(res)
+ local fields = {
+ 'Attachments.FileName',
+ 'Attachments.ContentType',
+ 'Attachments.Length',
+ 'Attachments.Digest',
+ }
+
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
+end
+
+local function clickhouse_urls_row(res)
+ local fields = {
+ 'Urls.Tld',
+ 'Urls.Url',
+ 'Urls.Flags',
+ }
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
+end
+
+local function clickhouse_emails_row(res)
+ local fields = {
+ 'Emails',
+ }
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
+end
+
+local function clickhouse_symbols_row(res)
+ local fields = {
+ 'Symbols.Names',
+ 'Symbols.Scores',
+ 'Symbols.Options',
+ }
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
+end
+
+local function clickhouse_groups_row(res)
+ local fields = {
+ 'Groups.Names',
+ 'Groups.Scores',
+ }
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
+end
+
+local function clickhouse_asn_row(res)
+ local fields = {
+ 'ASN',
+ 'Country',
+ 'IPNet',
+ }
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
+end
+
+local function clickhouse_extra_columns(res)
+ for _, v in ipairs(settings.extra_columns) do
+ table.insert(res, v.name)
+ end
+end
+
+local function today(ts)
+ return os.date('!%Y-%m-%d', ts)
+end
+
+local function clickhouse_check_symbol(task, settings_field_name, fields_table,
+ field_name, value, value_negative)
+ for _, s in ipairs(settings[settings_field_name] or {}) do
+ if task:has_symbol(s) then
+ if value_negative then
+ local sym = task:get_symbol(s)[1]
+ if sym['score'] > 0 then
+ fields_table[field_name] = value
+ else
+ fields_table[field_name] = value_negative
+ end
+ else
+ fields_table[field_name] = value
+ end
+
+ return true
+ end
+ end
+
+ return false
+end
+
+local function clickhouse_send_data(task, ev_base, why, gen_rows, cust_rows)
+ local log_object = task or rspamd_config
+ local upstream = settings.upstream:get_upstream_round_robin()
+ local ip_addr = upstream:get_addr():to_string(true)
+ rspamd_logger.infox(log_object, "trying to send %s rows to clickhouse server %s; started as %s",
+ #gen_rows + #cust_rows, ip_addr, why)
+
+ local function gen_success_cb(what, how_many)
+ return function(_, _)
+ rspamd_logger.messagex(log_object, "sent %s rows of %s to clickhouse server %s; started as %s",
+ how_many, what, ip_addr, why)
+ upstream:ok()
+ end
+ end
+
+ local function gen_fail_cb(what, how_many)
+ return function(_, err)
+ rspamd_logger.errx(log_object, "cannot send %s rows of %s data to clickhouse server %s: %s; started as %s",
+ how_many, what, ip_addr, err, why)
+ upstream:fail()
+ end
+ end
+
+ local function send_data(what, tbl, query)
+ local ch_params = {}
+ if task then
+ ch_params.task = task
+ else
+ ch_params.config = rspamd_config
+ ch_params.ev_base = ev_base
+ end
+
+ local ret = lua_clickhouse.insert(upstream, settings, ch_params,
+ query, tbl,
+ gen_success_cb(what, #tbl),
+ gen_fail_cb(what, #tbl))
+ if not ret then
+ rspamd_logger.errx(log_object, "cannot send %s rows of %s data to clickhouse server %s: %s",
+ #tbl, what, ip_addr, 'cannot make HTTP request')
+ end
+ end
+
+ local fields = {}
+ clickhouse_main_row(fields)
+ clickhouse_attachments_row(fields)
+ clickhouse_urls_row(fields)
+ clickhouse_emails_row(fields)
+ clickhouse_asn_row(fields)
+
+ if settings.enable_symbols then
+ clickhouse_symbols_row(fields)
+ clickhouse_groups_row(fields)
+ end
+
+ if #settings.extra_columns > 0 then
+ clickhouse_extra_columns(fields)
+ end
+
+ send_data('generic data', gen_rows,
+ string.format('INSERT INTO rspamd (%s)',
+ table.concat(fields, ',')))
+
+ for k, crows in pairs(cust_rows) do
+ if #crows > 1 then
+ send_data('custom data (' .. k .. ')', crows,
+ settings.custom_rules[k].first_row())
+ end
+ end
+end
+
+local function clickhouse_collect(task)
+ if task:has_flag('skip') then
+ return
+ end
+
+ if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then
+ return
+ end
+
+ for _, sym in ipairs(settings.stop_symbols) do
+ if task:has_symbol(sym) then
+ rspamd_logger.infox(task, 'skip Clickhouse storage for message: symbol %s has fired', sym)
+ return
+ end
+ end
+
+ if settings.exceptions then
+ local excepted, trace = settings.exceptions:process(task)
+ if excepted then
+ rspamd_logger.infox(task, 'skipped Clickhouse storage for message: excepted (%s)',
+ trace)
+ -- Excepted
+ return
+ end
+ end
+
+ local from_domain = ''
+ local from_user = ''
+ if task:has_from('smtp') then
+ local from = task:get_from({ 'smtp', 'orig' })[1]
+
+ if from then
+ from_domain = from['domain']:lower()
+ from_user = from['user']
+ end
+ end
+
+ local mime_domain = ''
+ local mime_user = ''
+ if task:has_from('mime') then
+ local from = task:get_from({ 'mime', 'orig' })[1]
+ if from then
+ mime_domain = from['domain']:lower()
+ mime_user = from['user']
+ end
+ end
+
+ local mime_recipients = {}
+ if task:has_recipients('mime') then
+ local recipients = task:get_recipients({ 'mime', 'orig' })
+ for _, rcpt in ipairs(recipients) do
+ table.insert(mime_recipients, rcpt['user'] .. '@' .. rcpt['domain']:lower())
+ end
+ end
+
+ local ip_str = 'undefined'
+ local ip = task:get_from_ip()
+ if ip and ip:is_valid() then
+ local ipnet
+ if ip:get_version() == 4 then
+ ipnet = ip:apply_mask(settings['ipmask'])
+ else
+ ipnet = ip:apply_mask(settings['ipmask6'])
+ end
+ ip_str = ipnet:to_string()
+ end
+
+ local helo = task:get_helo() or ''
+
+ local rcpt_user = ''
+ local rcpt_domain = ''
+ local smtp_recipients = {}
+ if task:has_recipients('smtp') then
+ local recipients = task:get_recipients('smtp')
+ -- for compatibility with an old table structure
+ rcpt_user = recipients[1]['user']
+ rcpt_domain = recipients[1]['domain']:lower()
+
+ for _, rcpt in ipairs(recipients) do
+ table.insert(smtp_recipients, rcpt['user'] .. '@' .. rcpt['domain']:lower())
+ end
+ end
+
+ local list_id = task:get_header('List-Id') or ''
+ local message_id = lua_util.maybe_obfuscate_string(task:get_message_id() or '',
+ settings, 'mid')
+
+ local score = task:get_metric_score()[1];
+ local fields = {
+ bayes = 'unknown',
+ fuzzy = 'unknown',
+ ann = 'unknown',
+ whitelist = 'unknown',
+ dkim = 'unknown',
+ dmarc = 'unknown',
+ spf = 'unknown',
+ }
+
+ local ret
+
+ ret = clickhouse_check_symbol(task, 'bayes_spam_symbols', fields,
+ 'bayes', 'spam')
+ if not ret then
+ clickhouse_check_symbol(task, 'bayes_ham_symbols', fields,
+ 'bayes', 'ham')
+ end
+
+ clickhouse_check_symbol(task, 'ann_symbols_spam', fields,
+ 'ann', 'spam')
+ if not ret then
+ clickhouse_check_symbol(task, 'ann_symbols_ham', fields,
+ 'ann', 'ham')
+ end
+
+ clickhouse_check_symbol(task, 'whitelist_symbols', fields,
+ 'whitelist', 'blacklist', 'whitelist')
+
+ clickhouse_check_symbol(task, 'fuzzy_symbols', fields,
+ 'fuzzy', 'deny')
+
+ ret = clickhouse_check_symbol(task, 'dkim_allow_symbols', fields,
+ 'dkim', 'allow')
+ if not ret then
+ ret = clickhouse_check_symbol(task, 'dkim_reject_symbols', fields,
+ 'dkim', 'reject')
+ end
+ if not ret then
+ ret = clickhouse_check_symbol(task, 'dkim_dnsfail_symbols', fields,
+ 'dkim', 'dnsfail')
+ end
+ if not ret then
+ clickhouse_check_symbol(task, 'dkim_na_symbols', fields,
+ 'dkim', 'na')
+ end
+
+ ret = clickhouse_check_symbol(task, 'dmarc_allow_symbols', fields,
+ 'dmarc', 'allow')
+ if not ret then
+ ret = clickhouse_check_symbol(task, 'dmarc_reject_symbols', fields,
+ 'dmarc', 'reject')
+ end
+ if not ret then
+ ret = clickhouse_check_symbol(task, 'dmarc_quarantine_symbols', fields,
+ 'dmarc', 'quarantine')
+ end
+ if not ret then
+ ret = clickhouse_check_symbol(task, 'dmarc_softfail_symbols', fields,
+ 'dmarc', 'softfail')
+ end
+ if not ret then
+ clickhouse_check_symbol(task, 'dmarc_na_symbols', fields,
+ 'dmarc', 'na')
+ end
+
+ ret = clickhouse_check_symbol(task, 'spf_allow_symbols', fields,
+ 'spf', 'allow')
+ if not ret then
+ ret = clickhouse_check_symbol(task, 'spf_reject_symbols', fields,
+ 'spf', 'reject')
+ end
+ if not ret then
+ ret = clickhouse_check_symbol(task, 'spf_neutral_symbols', fields,
+ 'spf', 'neutral')
+ end
+ if not ret then
+ ret = clickhouse_check_symbol(task, 'spf_dnsfail_symbols', fields,
+ 'spf', 'dnsfail')
+ end
+ if not ret then
+ clickhouse_check_symbol(task, 'spf_na_symbols', fields,
+ 'spf', 'na')
+ end
+
+ local nrcpts = 0
+ if task:has_recipients('smtp') then
+ nrcpts = #task:get_recipients('smtp')
+ end
+
+ local nurls = 0
+ local task_urls = task:get_urls({
+ content = true,
+ images = true,
+ emails = false,
+ sort = true,
+ }) or {}
+
+ nurls = #task_urls
+
+ local timestamp = math.floor(task:get_date({
+ format = 'connect',
+ gmt = true, -- The only sane way to sync stuff with different timezones
+ }))
+
+ local action = task:get_metric_action()
+ local custom_action = ''
+
+ if not predefined_actions[action] then
+ custom_action = action
+ action = 'custom'
+ end
+
+ local digest = ''
+
+ if settings.enable_digest then
+ digest = task:get_digest()
+ end
+
+ local subject = ''
+ if settings.insert_subject then
+ subject = lua_util.maybe_obfuscate_string(task:get_subject() or '', settings, 'subject')
+ end
+
+ local scan_real = task:get_scan_time()
+ scan_real = math.floor(scan_real * 1000)
+ if scan_real < 0 then
+ rspamd_logger.messagex(task,
+ 'clock skew detected for message: %s ms real scan time (reset to 0)',
+ scan_real)
+ scan_real = 0
+ end
+
+ local auth_user = task:get_user() or ''
+ local settings_id = task:get_settings_id()
+
+ if settings_id then
+ -- Convert to string
+ settings_id = lua_settings.settings_by_id(settings_id)
+
+ if settings_id then
+ settings_id = settings_id.name
+ end
+ end
+
+ if not settings_id then
+ settings_id = ''
+ end
+
+ local row = {
+ today(timestamp),
+ timestamp,
+ from_domain,
+ mime_domain,
+ ip_str,
+ helo,
+ score,
+ nrcpts,
+ task:get_size(),
+ fields.whitelist,
+ fields.bayes,
+ fields.fuzzy,
+ fields.ann,
+ fields.dkim,
+ fields.dmarc,
+ nurls,
+ action,
+ from_user,
+ mime_user,
+ rcpt_user,
+ rcpt_domain,
+ smtp_recipients,
+ list_id,
+ subject,
+ digest,
+ fields.spf,
+ mime_recipients,
+ message_id,
+ scan_real,
+ custom_action,
+ auth_user,
+ settings_id
+ }
+
+ -- Attachments step
+ local attachments_fnames = {}
+ local attachments_ctypes = {}
+ local attachments_lengths = {}
+ local attachments_digests = {}
+ for _, part in ipairs(task:get_parts()) do
+ if part:is_attachment() then
+ table.insert(attachments_fnames, part:get_filename() or '')
+ local mime_type, mime_subtype = part:get_type()
+ table.insert(attachments_ctypes, string.format("%s/%s", mime_type, mime_subtype))
+ table.insert(attachments_lengths, part:get_length())
+ table.insert(attachments_digests, string.sub(part:get_digest(), 1, 16))
+ end
+ end
+
+ if #attachments_fnames > 0 then
+ table.insert(row, attachments_fnames)
+ table.insert(row, attachments_ctypes)
+ table.insert(row, attachments_lengths)
+ table.insert(row, attachments_digests)
+ else
+ table.insert(row, {})
+ table.insert(row, {})
+ table.insert(row, {})
+ table.insert(row, {})
+ end
+
+ -- Urls step
+ local urls_urls = {}
+ local urls_tlds = {}
+ local urls_flags = {}
+
+ if settings.full_urls then
+ for i, u in ipairs(task_urls) do
+ urls_urls[i] = u:get_text()
+ urls_tlds[i] = u:get_tld() or u:get_host()
+ urls_flags[i] = u:get_flags_num()
+ end
+ else
+ -- We need to store unique
+ local mt = {
+ ord_tbl = {}, -- ordered list of urls
+ idx_tbl = {}, -- indexed by host + flags, reference to an index in ord_tbl
+ __newindex = function(t, k, v)
+ local idx = getmetatable(t).idx_tbl
+ local ord = getmetatable(t).ord_tbl
+ local key = k:get_host() .. tostring(k:get_flags_num())
+ if idx[key] then
+ ord[idx[key]] = v -- replace
+ else
+ ord[#ord + 1] = v
+ idx[key] = #ord
+ end
+ end,
+ __index = function(t, k)
+ local ord = getmetatable(t).ord_tbl
+ if type(k) == 'number' then
+ return ord[k]
+ else
+ local idx = getmetatable(t).idx_tbl
+ local key = k:get_host() .. tostring(k:get_flags_num())
+ if idx[key] then
+ return ord[idx[key]]
+ end
+ end
+ end,
+ }
+ -- Extra index needed for making this unique
+ local urls_idx = {}
+ setmetatable(urls_idx, mt)
+ for _, u in ipairs(task_urls) do
+ if not urls_idx[u] then
+ urls_idx[u] = u
+ urls_urls[#urls_urls + 1] = u:get_host()
+ urls_tlds[#urls_tlds + 1] = u:get_tld() or u:get_host()
+ urls_flags[#urls_flags + 1] = u:get_flags_num()
+ end
+ end
+ end
+
+
+ -- Get tlds
+ table.insert(row, urls_tlds)
+ -- Get hosts/full urls
+ table.insert(row, urls_urls)
+ -- Numeric flags
+ table.insert(row, urls_flags)
+
+ -- Emails step
+ if task:has_urls(true) then
+ local emails = task:get_emails() or {}
+ local emails_formatted = {}
+ for i, u in ipairs(emails) do
+ emails_formatted[i] = string.format('%s@%s', u:get_user(), u:get_host())
+ end
+ table.insert(row, emails_formatted)
+ else
+ table.insert(row, {})
+ end
+
+ -- ASN information
+ local asn, country, ipnet = 0, '--', '--'
+ local pool = task:get_mempool()
+ ret = pool:get_variable("asn")
+ if ret then
+ asn = ret
+ end
+ ret = pool:get_variable("country")
+ if ret then
+ country = ret:sub(1, 2)
+ end
+ ret = pool:get_variable("ipnet")
+ if ret then
+ ipnet = ret
+ end
+ table.insert(row, asn)
+ table.insert(row, country)
+ table.insert(row, ipnet)
+
+ -- Symbols info
+ if settings.enable_symbols then
+ local symbols = task:get_symbols_all()
+ local syms_tab = {}
+ local scores_tab = {}
+ local options_tab = {}
+
+ for _, s in ipairs(symbols) do
+ table.insert(syms_tab, s.name or '')
+ table.insert(scores_tab, s.score)
+
+ if s.options then
+ table.insert(options_tab, table.concat(s.options, ','))
+ else
+ table.insert(options_tab, '');
+ end
+ end
+ table.insert(row, syms_tab)
+ table.insert(row, scores_tab)
+ table.insert(row, options_tab)
+
+ -- Groups data
+ local groups = task:get_groups()
+ local groups_tab = {}
+ local gr_scores_tab = {}
+ for gr, sc in pairs(groups) do
+ table.insert(groups_tab, gr)
+ table.insert(gr_scores_tab, sc)
+ end
+ table.insert(row, groups_tab)
+ table.insert(row, gr_scores_tab)
+ end
+
+ -- Extra columns
+ if #settings.extra_columns > 0 then
+ for _, col in ipairs(settings.extra_columns) do
+ local elts = col.real_selector(task)
+
+ if elts then
+ table.insert(row, elts)
+ else
+ table.insert(row, col.default_value)
+ end
+ end
+ end
+
+ -- Custom data
+ for k, rule in pairs(settings.custom_rules) do
+ if not custom_rows[k] then
+ custom_rows[k] = {}
+ end
+ table.insert(custom_rows[k], lua_clickhouse.row_to_tsv(rule.get_row(task)))
+ end
+
+ local tsv_row = lua_clickhouse.row_to_tsv(row)
+ used_memory = used_memory + #tsv_row
+ data_rows[#data_rows + 1] = tsv_row
+ nrows = nrows + 1
+ lua_util.debugm(N, task,
+ "add clickhouse row %s / %s; used memory: %s / %s",
+ nrows, settings.limits.max_rows,
+ used_memory, settings.limits.max_memory)
+end
+
+local function do_remove_partition(ev_base, cfg, table_name, partition)
+ lua_util.debugm(N, rspamd_config, "removing partition %s.%s", table_name, partition)
+ local upstream = settings.upstream:get_upstream_round_robin()
+ local remove_partition_sql = "ALTER TABLE ${table_name} ${remove_method} PARTITION '${partition}'"
+ local remove_method = (settings.retention.method == 'drop') and 'DROP' or 'DETACH'
+ local sql_params = {
+ ['table_name'] = table_name,
+ ['remove_method'] = remove_method,
+ ['partition'] = partition
+ }
+
+ local sql = lua_util.template(remove_partition_sql, sql_params)
+
+ local ch_params = {
+ body = sql,
+ ev_base = ev_base,
+ config = cfg,
+ }
+
+ local err, _ = lua_clickhouse.generic_sync(upstream, settings, ch_params, sql)
+ if err then
+ rspamd_logger.errx(rspamd_config,
+ "cannot detach partition %s:%s from server %s: %s",
+ table_name, partition,
+ settings['server'], err)
+ return
+ end
+
+ rspamd_logger.infox(rspamd_config,
+ 'detached partition %s:%s on server %s', table_name, partition,
+ settings['server'])
+
+end
+
+--[[
+ nil - file is not writable, do not perform removal
+ 0 - it's time to perform removal
+ <int> - how many seconds wait until next run
+]]
+local function get_last_removal_ago()
+ local ts_file = string.format('%s/%s', rspamd_paths['DBDIR'], 'clickhouse_retention_run')
+ local last_ts
+ local current_ts = os.time()
+
+ local function write_ts_to_file()
+ local write_file, err = io.open(ts_file, 'w')
+ if err then
+ rspamd_logger.errx(rspamd_config, 'Failed to open %s, will not perform retention: %s', ts_file, err)
+ return nil
+ end
+
+ local res
+ res, err = write_file:write(tostring(current_ts))
+ if err or res == nil then
+ write_file:close()
+ rspamd_logger.errx(rspamd_config, 'Failed to write %s, will not perform retention: %s', ts_file, err)
+ return nil
+ end
+ write_file:close()
+
+ return true
+ end
+
+ local f, err = io.open(ts_file, 'r')
+ if err then
+ lua_util.debugm(N, rspamd_config, 'Failed to open %s: %s', ts_file, err)
+ else
+ last_ts = tonumber(f:read('*number'))
+ f:close()
+ end
+
+ if last_ts == nil or (last_ts + settings.retention.period) <= current_ts then
+ return write_ts_to_file() and 0
+ end
+
+ if last_ts > current_ts then
+ -- Clock skew detected, overwrite last_ts with current_ts and wait for the next
+ -- retention period
+ rspamd_logger.errx(rspamd_config, 'Last collection time is in future: %s; overwrite it with %s in %s',
+ last_ts, current_ts, ts_file)
+ return write_ts_to_file() and -1
+ end
+
+ return (last_ts + settings.retention.period) - current_ts
+end
+
+local function clickhouse_maybe_send_data_periodic(cfg, ev_base, now)
+ local need_collect = false
+ local reason
+
+ if nrows == 0 then
+ lua_util.debugm(N, cfg, "no need to send data, as there are no rows to collect")
+ return settings.check_timeout
+ end
+
+ if final_call then
+ lua_util.debugm(N, cfg, "no need to send data, final call has been issued")
+ return 0
+ end
+
+ if settings.limits.max_rows > 0 then
+ if nrows > settings.limits.max_rows then
+ need_collect = true
+ reason = string.format('limit of rows has been reached: %d', nrows)
+ end
+ end
+
+ if last_collection > 0 and settings.limits.max_interval > 0 then
+ if now - last_collection > settings.limits.max_interval then
+ need_collect = true
+ reason = string.format('limit of time since last collection has been reached: %d seconds passed ' ..
+ '(%d seconds trigger)',
+ (now - last_collection), settings.limits.max_interval)
+ end
+ end
+
+ if settings.limits.max_memory > 0 then
+ if used_memory >= settings.limits.max_memory then
+ need_collect = true
+ reason = string.format('limit of memory has been reached: %d bytes used',
+ used_memory)
+ end
+ end
+
+ if last_collection == 0 then
+ last_collection = now
+ end
+
+ if need_collect then
+ -- Do it atomic
+ local saved_rows = data_rows
+ local saved_custom = custom_rows
+ nrows = 0
+ last_collection = now
+ used_memory = 0
+ data_rows = {}
+ custom_rows = {}
+
+ clickhouse_send_data(nil, ev_base, reason, saved_rows, saved_custom)
+
+ if settings.collect_garbage then
+ collectgarbage()
+ end
+ end
+
+ return settings.check_timeout
+end
+
+local function clickhouse_remove_old_partitions(cfg, ev_base)
+ local last_time_ago = get_last_removal_ago()
+ if last_time_ago == nil then
+ rspamd_logger.errx(rspamd_config, "Failed to get last run time. Disabling retention")
+ return false
+ elseif last_time_ago ~= 0 then
+ return last_time_ago
+ end
+
+ local upstream = settings.upstream:get_upstream_round_robin()
+ local partition_to_remove_sql = "SELECT partition, table " ..
+ "FROM system.parts WHERE table IN ('${tables}') " ..
+ "GROUP BY partition, table " ..
+ "HAVING max(max_date) < toDate(now() - interval ${month} month)"
+
+ local table_names = { 'rspamd' }
+ local tables = table.concat(table_names, "', '")
+ local sql_params = {
+ tables = tables,
+ month = settings.retention.period_months,
+ }
+ local sql = lua_util.template(partition_to_remove_sql, sql_params)
+
+ local ch_params = {
+ ev_base = ev_base,
+ config = cfg,
+ }
+ local err, rows = lua_clickhouse.select_sync(upstream, settings, ch_params, sql)
+ if err then
+ rspamd_logger.errx(rspamd_config,
+ "cannot send data to clickhouse server %s: %s",
+ settings['server'], err)
+ else
+ fun.each(function(row)
+ do_remove_partition(ev_base, cfg, row.table, row.partition)
+ end, rows)
+ end
+
+ -- settings.retention.period is added on initialisation, see below
+ return settings.retention.period
+end
+
+local function upload_clickhouse_schema(upstream, ev_base, cfg, initial)
+ local ch_params = {
+ ev_base = ev_base,
+ config = cfg,
+ }
+
+ local errored = false
+
+ -- Upload a single element of the schema
+ local function upload_schema_elt(v)
+ if errored then
+ rspamd_logger.errx(rspamd_config, "cannot upload schema '%s' on clickhouse server %s: due to previous errors",
+ v, upstream:get_addr():to_string(true))
+ return
+ end
+ local sql = v
+ local err, reply = lua_clickhouse.generic_sync(upstream, settings, ch_params, sql)
+
+ if err then
+ rspamd_logger.errx(rspamd_config, "cannot upload schema '%s' on clickhouse server %s: %s",
+ sql, upstream:get_addr():to_string(true), err)
+ errored = true
+ return
+ end
+ rspamd_logger.debugm(N, rspamd_config, 'uploaded clickhouse schema element %s to %s: %s',
+ v, upstream:get_addr():to_string(true), reply)
+ end
+
+ -- Process element and return nil if statement should be skipped
+ local function preprocess_schema_elt(v)
+ if type(v) == 'string' then
+ return lua_util.template(v, { SCHEMA_VERSION = tostring(schema_version) })
+ elseif type(v) == 'table' then
+ -- Pair of statement + boolean
+ if initial == v[2] then
+ return lua_util.template(v[1], { SCHEMA_VERSION = tostring(schema_version) })
+ else
+ rspamd_logger.debugm(N, rspamd_config, 'skip clickhouse schema element %s: schema already exists',
+ v)
+ end
+ end
+
+ return nil
+ end
+
+ -- Apply schema elements sequentially, users additions are concatenated to the tail
+ fun.each(upload_schema_elt,
+ -- Also template schema version
+ fun.filter(function(v)
+ return v ~= nil
+ end,
+ fun.map(preprocess_schema_elt,
+ fun.chain(clickhouse_schema, settings.schema_additions)
+ )
+ )
+ )
+end
+
+local function maybe_apply_migrations(upstream, ev_base, cfg, version)
+ local ch_params = {
+ ev_base = ev_base,
+ config = cfg,
+ }
+ -- Apply migrations sequentially
+ local function migration_recursor(i)
+ if i < schema_version then
+ if migrations[i] then
+ -- We also need to apply statements sequentially
+ local function sql_recursor(j)
+ if migrations[i][j] then
+ local sql = migrations[i][j]
+ local ret = lua_clickhouse.generic(upstream, settings, ch_params, sql,
+ function(_, _)
+ rspamd_logger.infox(rspamd_config,
+ 'applied migration to version %s from version %s: %s',
+ i + 1, version, sql:gsub('[\n%s]+', ' '))
+ if j == #migrations[i] then
+ -- Go to the next migration
+ migration_recursor(i + 1)
+ else
+ -- Apply the next statement
+ sql_recursor(j + 1)
+ end
+ end,
+ function(_, err)
+ rspamd_logger.errx(rspamd_config,
+ "cannot apply migration %s: '%s' on clickhouse server %s: %s",
+ i, sql, upstream:get_addr():to_string(true), err)
+ end)
+ if not ret then
+ rspamd_logger.errx(rspamd_config,
+ "cannot apply migration %s: '%s' on clickhouse server %s: cannot make request",
+ i, sql, upstream:get_addr():to_string(true))
+ end
+ end
+ end
+
+ sql_recursor(1)
+ else
+ -- Try another migration
+ migration_recursor(i + 1)
+ end
+ end
+ end
+
+ migration_recursor(version)
+end
+
+local function add_extra_columns(upstream, ev_base, cfg)
+ local ch_params = {
+ ev_base = ev_base,
+ config = cfg,
+ }
+ -- Apply migrations sequentially
+ local function columns_recursor(i)
+ if i <= #settings.extra_columns then
+ local col = settings.extra_columns[i]
+ local prev_column
+ if i == 1 then
+ prev_column = 'MIMERcpt'
+ else
+ prev_column = settings.extra_columns[i - 1].name
+ end
+ local sql = string.format('ALTER TABLE rspamd ADD COLUMN IF NOT EXISTS `%s` %s AFTER `%s`',
+ col.name, col.type, prev_column)
+ if col.comment then
+ sql = sql .. string.format(", COMMENT COLUMN IF EXISTS `%s` '%s'", col.name, col.comment)
+ end
+
+ local ret = lua_clickhouse.generic(upstream, settings, ch_params, sql,
+ function(_, _)
+ rspamd_logger.infox(rspamd_config,
+ 'added extra column %s (%s) after %s',
+ col.name, col.type, prev_column)
+ -- Apply the next statement
+ columns_recursor(i + 1)
+ end,
+ function(_, err)
+ rspamd_logger.errx(rspamd_config,
+ "cannot apply add column alter %s: '%s' on clickhouse server %s: %s",
+ i, sql, upstream:get_addr():to_string(true), err)
+ end)
+ if not ret then
+ rspamd_logger.errx(rspamd_config,
+ "cannot apply add column alter %s: '%s' on clickhouse server %s: cannot make request",
+ i, sql, upstream:get_addr():to_string(true))
+ end
+ end
+ end
+
+ columns_recursor(1)
+end
+
+local function check_rspamd_table(upstream, ev_base, cfg)
+ local ch_params = {
+ ev_base = ev_base,
+ config = cfg,
+ }
+ local sql = [[EXISTS TABLE rspamd]]
+ local err, rows = lua_clickhouse.select_sync(upstream, settings, ch_params, sql)
+ if err then
+ rspamd_logger.errx(rspamd_config, "cannot check rspamd table in clickhouse server %s: %s",
+ upstream:get_addr():to_string(true), err)
+ return
+ end
+
+ if rows[1] and rows[1].result then
+ if tonumber(rows[1].result) == 1 then
+ -- Apply migration
+ upload_clickhouse_schema(upstream, ev_base, cfg, false)
+ rspamd_logger.infox(rspamd_config, 'table rspamd exists, check if we need to apply migrations')
+ maybe_apply_migrations(upstream, ev_base, cfg, 1)
+ else
+ -- Upload schema
+ rspamd_logger.infox(rspamd_config, 'table rspamd does not exists, upload full schema')
+ upload_clickhouse_schema(upstream, ev_base, cfg, true)
+ end
+ else
+ rspamd_logger.errx(rspamd_config,
+ "unexpected reply on EXISTS command from server %s: %s",
+ upstream:get_addr():to_string(true), rows)
+ end
+end
+
+local function check_clickhouse_upstream(upstream, ev_base, cfg)
+ local ch_params = {
+ ev_base = ev_base,
+ config = cfg,
+ }
+ -- If we have some custom rules, we just send its schema to the upstream
+ for k, rule in pairs(settings.custom_rules) do
+ if rule.schema then
+ local sql = lua_util.template(rule.schema, settings)
+ local err, _ = lua_clickhouse.generic_sync(upstream, settings, ch_params, sql)
+ if err then
+ rspamd_logger.errx(rspamd_config, 'cannot send custom schema %s to clickhouse server %s: ' ..
+ 'cannot make request (%s)',
+ k, upstream:get_addr():to_string(true), err)
+ end
+ end
+ end
+
+ -- Now check the main schema and apply migrations if needed
+ local sql = [[SELECT MAX(Version) as v FROM rspamd_version]]
+ local err, rows = lua_clickhouse.select_sync(upstream, settings, ch_params, sql)
+ if err then
+ if rows and rows.code == 404 then
+ rspamd_logger.infox(rspamd_config,
+ 'table rspamd_version does not exist, check rspamd table')
+ check_rspamd_table(upstream, ev_base, cfg)
+ else
+ rspamd_logger.errx(rspamd_config,
+ "cannot get version on clickhouse server %s: %s",
+ upstream:get_addr():to_string(true), err)
+ end
+ else
+ upload_clickhouse_schema(upstream, ev_base, cfg, false)
+ local version = tonumber(rows[1].v)
+ maybe_apply_migrations(upstream, ev_base, cfg, version)
+ end
+
+ if #settings.extra_columns > 0 then
+ add_extra_columns(upstream, ev_base, cfg)
+ end
+end
+
+local opts = rspamd_config:get_all_opt('clickhouse')
+if opts then
+ -- Legacy `limit` options
+ if opts.limit and not opts.limits then
+ settings.limits.max_rows = opts.limit
+ end
+ for k, v in pairs(opts) do
+ if k == 'custom_rules' then
+ if not v[1] then
+ v = { v }
+ end
+
+ for i, rule in ipairs(v) do
+ if rule.schema and rule.first_row and rule.get_row then
+ local first_row, get_row
+ local loadstring = loadstring or load
+ local ret, res_or_err = pcall(loadstring(rule.first_row))
+
+ if not ret or type(res_or_err) ~= 'function' then
+ rspamd_logger.errx(rspamd_config, 'invalid first_row (%s) - must be a function',
+ res_or_err)
+ else
+ first_row = res_or_err
+ end
+
+ ret, res_or_err = pcall(loadstring(rule.get_row))
+
+ if not ret or type(res_or_err) ~= 'function' then
+ rspamd_logger.errx(rspamd_config,
+ 'invalid get_row (%s) - must be a function',
+ res_or_err)
+ else
+ get_row = res_or_err
+ end
+
+ if first_row and get_row then
+ local name = rule.name or tostring(i)
+ settings.custom_rules[name] = {
+ schema = rule.schema,
+ first_row = first_row,
+ get_row = get_row,
+ }
+ end
+ else
+ rspamd_logger.errx(rspamd_config, 'custom rule has no required attributes: schema, first_row and get_row')
+ end
+ end
+ else
+ settings[k] = lua_util.deepcopy(v)
+ end
+ end
+
+ if not settings['server'] and not settings['servers'] then
+ rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
+ lua_util.disable_module(N, "config")
+ else
+ local lua_maps = require "lua_maps"
+ settings['from_map'] = lua_maps.map_add('clickhouse', 'from_tables',
+ 'regexp', 'clickhouse specific domains')
+
+ settings.upstream = upstream_list.create(rspamd_config,
+ settings['server'] or settings['servers'], 8123)
+
+ if not settings.upstream then
+ rspamd_logger.errx(rspamd_config, 'cannot parse clickhouse address: %s',
+ settings['server'] or settings['servers'])
+ lua_util.disable_module(N, "config")
+ return
+ end
+
+ if settings.exceptions then
+ local maps_expressions = require "lua_maps_expressions"
+
+ settings.exceptions = maps_expressions.create(rspamd_config,
+ settings.exceptions, N)
+ end
+
+ if settings.extra_columns then
+ -- Check sanity and create selector closures
+ local lua_selectors = require "lua_selectors"
+ local columns_transformed = {}
+ local need_sort = false
+ -- Select traverse function depending on what we have
+ local iter_func = settings.extra_columns[1] and ipairs or pairs
+
+ for col_name, col_data in iter_func(settings.extra_columns) do
+ -- Array based extra columns
+ if col_data.name then
+ col_name = col_data.name
+ end
+ if not col_data.selector or not col_data.type then
+ rspamd_logger.errx(rspamd_config, 'cannot add clickhouse extra row %s: no type or no selector',
+ col_name)
+ else
+ local is_array = false
+
+ if col_data.type:lower():match('^array') then
+ is_array = true
+ end
+
+ local selector = lua_selectors.create_selector_closure(rspamd_config,
+ col_data.selector, col_data.delimiter or '', is_array)
+
+ if not selector then
+ rspamd_logger.errx(rspamd_config, 'cannot add clickhouse extra row %s: bad selector: %s',
+ col_name, col_data.selector)
+ else
+ if not col_data.default_value then
+ if is_array then
+ col_data.default_value = {}
+ else
+ col_data.default_value = ''
+ end
+ end
+ col_data.real_selector = selector
+ if not col_data.name then
+ col_data.name = col_name
+ need_sort = true
+ end
+ table.insert(columns_transformed, col_data)
+ end
+ end
+ end
+
+ -- Convert extra columns from a map to an array sorted by column name to
+ -- preserve strict order when doing altering
+ if need_sort then
+ rspamd_logger.infox(rspamd_config, 'sort extra columns as they are not configured as an array')
+ table.sort(columns_transformed, function(c1, c2)
+ return c1.name < c2.name
+ end)
+ end
+ settings.extra_columns = columns_transformed
+ end
+
+ rspamd_config:register_symbol({
+ name = 'CLICKHOUSE_COLLECT',
+ type = 'idempotent',
+ callback = clickhouse_collect,
+ flags = 'empty,explicit_disable,ignore_passthrough',
+ augmentations = { string.format("timeout=%f", settings.timeout) },
+ })
+ rspamd_config:register_finish_script(function(task)
+ if nrows > 0 then
+ final_call = true
+ local saved_rows = data_rows
+ local saved_custom = custom_rows
+
+ nrows = 0
+ data_rows = {}
+ used_memory = 0
+ custom_rows = {}
+
+ clickhouse_send_data(task, nil, 'final collection',
+ saved_rows, saved_custom)
+
+ if settings.collect_garbage then
+ collectgarbage()
+ end
+ end
+ end)
+ -- Create tables on load
+ rspamd_config:add_on_load(function(cfg, ev_base, worker)
+ if worker:is_scanner() then
+ rspamd_config:add_periodic(ev_base, 0,
+ clickhouse_maybe_send_data_periodic, true)
+ end
+ if worker:is_primary_controller() then
+ local upstreams = settings.upstream:all_upstreams()
+
+ for _, up in ipairs(upstreams) do
+ check_clickhouse_upstream(up, ev_base, cfg)
+ end
+
+ if settings.retention.enable and settings.retention.method ~= 'drop' and
+ settings.retention.method ~= 'detach' then
+ rspamd_logger.errx(rspamd_config,
+ "retention.method should be either 'drop' or 'detach' (now: %s). Disabling retention",
+ settings.retention.method)
+ settings.retention.enable = false
+ end
+ if settings.retention.enable and settings.retention.period_months < 1 or
+ settings.retention.period_months > 1000 then
+ rspamd_logger.errx(rspamd_config,
+ "please, set retention.period_months between 1 and 1000 (now: %s). Disabling retention",
+ settings.retention.period_months)
+ settings.retention.enable = false
+ end
+ local period = lua_util.parse_time_interval(settings.retention.run_every)
+ if settings.retention.enable and period == nil then
+ rspamd_logger.errx(rspamd_config, "invalid value for retention.run_every (%s). Disabling retention",
+ settings.retention.run_every)
+ settings.retention.enable = false
+ end
+
+ if settings.retention.enable then
+ settings.retention.period = period
+ rspamd_logger.infox(rspamd_config,
+ "retention will be performed each %s seconds for %s month with method %s",
+ period, settings.retention.period_months, settings.retention.method)
+ rspamd_config:add_periodic(ev_base, 0, clickhouse_remove_old_partitions, false)
+ end
+ end
+ end)
+ end
+end
diff --git a/src/plugins/lua/clustering.lua b/src/plugins/lua/clustering.lua
new file mode 100644
index 0000000..d97bdb9
--- /dev/null
+++ b/src/plugins/lua/clustering.lua
@@ -0,0 +1,322 @@
+--[[
+Copyright (c) 2018, Vsevolod Stakhov
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- Plugin for finding patterns in email flows
+
+local N = 'clustering'
+
+local rspamd_logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local lua_verdict = require "lua_verdict"
+local lua_redis = require "lua_redis"
+local lua_selectors = require "lua_selectors"
+local ts = require("tableshape").types
+
+local redis_params
+
+local rules = {} -- Rules placement
+
+local default_rule = {
+ max_elts = 100, -- Maximum elements in a cluster
+ expire = 3600, -- Expire for a bucket when limit is not reached
+ expire_overflow = 36000, -- Expire for a bucket when limit is reached
+ spam_mult = 1.0, -- Increase on spam hit
+ junk_mult = 0.5, -- Increase on junk
+ ham_mult = -0.1, -- Increase on ham
+ size_mult = 0.01, -- Reaches 1.0 on `max_elts`
+ score_mult = 0.1,
+}
+
+local rule_schema = ts.shape {
+ max_elts = ts.number + ts.string / tonumber,
+ expire = ts.number + ts.string / lua_util.parse_time_interval,
+ expire_overflow = ts.number + ts.string / lua_util.parse_time_interval,
+ spam_mult = ts.number,
+ junk_mult = ts.number,
+ ham_mult = ts.number,
+ size_mult = ts.number,
+ score_mult = ts.number,
+ source_selector = ts.string,
+ cluster_selector = ts.string,
+ symbol = ts.string:is_optional(),
+ prefix = ts.string:is_optional(),
+}
+
+-- Redis scripts
+
+-- Queries for a cluster's data
+-- Arguments:
+-- 1. Source selector (string)
+-- 2. Cluster selector (string)
+-- Returns: {cur_elts, total_score, element_score}
+local query_cluster_script = [[
+local sz = redis.call('HLEN', KEYS[1])
+
+if not sz or not tonumber(sz) then
+ -- New bucket, will update on idempotent phase
+ return {0, '0', '0'}
+end
+
+local total_score = redis.call('HGET', KEYS[1], '__s')
+total_score = tonumber(total_score) or 0
+local score = redis.call('HGET', KEYS[1], KEYS[2])
+if not score or not tonumber(score) then
+ return {sz, tostring(total_score), '0'}
+end
+return {sz, tostring(total_score), tostring(score)}
+]]
+local query_cluster_id
+
+-- Updates cluster's data
+-- Arguments:
+-- 1. Source selector (string)
+-- 2. Cluster selector (string)
+-- 3. Score (number)
+-- 4. Max buckets (number)
+-- 5. Expire (number)
+-- 6. Expire overflow (number)
+-- Returns: nothing
+local update_cluster_script = [[
+local sz = redis.call('HLEN', KEYS[1])
+
+if not sz or not tonumber(sz) then
+ -- Create bucket
+ redis.call('HSET', KEYS[1], KEYS[2], math.abs(KEYS[3]))
+ redis.call('HSET', KEYS[1], '__s', KEYS[3])
+ redis.call('EXPIRE', KEYS[1], KEYS[5])
+
+ return
+end
+
+sz = tonumber(sz)
+local lim = tonumber(KEYS[4])
+
+if sz > lim then
+
+ if k then
+ -- Existing key
+ redis.call('HINCRBYFLOAT', KEYS[1], KEYS[2], math.abs(KEYS[3]))
+ end
+else
+ redis.call('HINCRBYFLOAT', KEYS[1], KEYS[2], math.abs(KEYS[3]))
+ redis.call('EXPIRE', KEYS[1], KEYS[6])
+end
+
+redis.call('HINCRBYFLOAT', KEYS[1], '__s', KEYS[3])
+redis.call('EXPIRE', KEYS[1], KEYS[5])
+]]
+local update_cluster_id
+
+-- Callbacks and logic
+
+local function clusterting_filter_cb(task, rule)
+ local source_selector = rule.source_selector(task)
+ local cluster_selector
+
+ if source_selector then
+ cluster_selector = rule.cluster_selector(task)
+ end
+
+ if not cluster_selector or not source_selector then
+ rspamd_logger.debugm(N, task, 'skip rule %s, selectors: source="%s", cluster="%s"',
+ rule.name, source_selector, cluster_selector)
+ return
+ end
+
+ local function combine_scores(cur_elts, total_score, element_score)
+ local final_score
+
+ local size_score = cur_elts * rule.size_mult
+ local cluster_score = total_score * rule.score_mult
+
+ if element_score > 0 then
+ -- We have seen this element mostly in junk/spam
+ final_score = math.min(1.0, size_score + cluster_score)
+ else
+ -- We have seen this element in ham mostly, so subtract average it from the size score
+ final_score = math.min(1.0, size_score - cluster_score / cur_elts)
+ end
+ rspamd_logger.debugm(N, task,
+ 'processed rule %s, selectors: source="%s", cluster="%s"; data: %s elts, %s score, %s elt score',
+ rule.name, source_selector, cluster_selector, cur_elts, total_score, element_score)
+ if final_score > 0.1 then
+ task:insert_result(rule.symbol, final_score, { source_selector,
+ tostring(size_score),
+ tostring(cluster_score) })
+ end
+ end
+
+ local function redis_get_cb(err, data)
+ if data then
+ if type(data) == 'table' then
+ combine_scores(tonumber(data[1]), tonumber(data[2]), tonumber(data[3]))
+ else
+ rspamd_logger.errx(task, 'invalid type while getting clustering keys %s: %s',
+ source_selector, type(data))
+ end
+
+ elseif err then
+ rspamd_logger.errx(task, 'got error while getting clustering keys %s: %s',
+ source_selector, err)
+ else
+ rspamd_logger.errx(task, 'got error while getting clustering keys %s: %s',
+ source_selector, "unknown error")
+ end
+ end
+
+ lua_redis.exec_redis_script(query_cluster_id,
+ { task = task, is_write = false, key = source_selector },
+ redis_get_cb,
+ { source_selector, cluster_selector })
+end
+
+local function clusterting_idempotent_cb(task, rule)
+ if task:has_flag('skip') then
+ return
+ end
+ if not rule.allow_local and lua_util.is_rspamc_or_controller(task) then
+ return
+ end
+
+ local verdict = lua_verdict.get_specific_verdict(N, task)
+ local score
+
+ if verdict == 'ham' then
+ score = rule.ham_mult
+ elseif verdict == 'spam' then
+ score = rule.spam_mult
+ elseif verdict == 'junk' then
+ score = rule.junk_mult
+ else
+ rspamd_logger.debugm(N, task, 'skip rule %s, verdict=%s',
+ rule.name, verdict)
+ return
+ end
+
+ local source_selector = rule.source_selector(task)
+ local cluster_selector
+
+ if source_selector then
+ cluster_selector = rule.cluster_selector(task)
+ end
+
+ if not cluster_selector or not source_selector then
+ rspamd_logger.debugm(N, task, 'skip rule %s, selectors: source="%s", cluster="%s"',
+ rule.name, source_selector, cluster_selector)
+ return
+ end
+
+ local function redis_set_cb(err, data)
+ if err then
+ rspamd_logger.errx(task, 'got error while getting clustering keys %s: %s',
+ source_selector, err)
+ else
+ rspamd_logger.debugm(N, task, 'set clustering key for %s: %s{%s} = %s',
+ source_selector, "unknown error")
+ end
+ end
+
+ lua_redis.exec_redis_script(update_cluster_id,
+ { task = task, is_write = true, key = source_selector },
+ redis_set_cb,
+ {
+ source_selector,
+ cluster_selector,
+ tostring(score),
+ tostring(rule.max_elts),
+ tostring(rule.expire),
+ tostring(rule.expire_overflow)
+ }
+ )
+end
+-- Init part
+redis_params = lua_redis.parse_redis_server('clustering')
+local opts = rspamd_config:get_all_opt("clustering")
+
+-- Initialization part
+if not (opts and type(opts) == 'table') then
+ lua_util.disable_module(N, "config")
+ return
+end
+
+if not redis_params then
+ lua_util.disable_module(N, "redis")
+ return
+end
+
+if opts['rules'] then
+ for k, v in pairs(opts['rules']) do
+ local raw_rule = lua_util.override_defaults(default_rule, v)
+
+ local rule, err = rule_schema:transform(raw_rule)
+
+ if not rule then
+ rspamd_logger.errx(rspamd_config, 'invalid clustering rule %s: %s',
+ k, err)
+ else
+
+ if not rule.symbol then
+ rule.symbol = k
+ end
+ if not rule.prefix then
+ rule.prefix = k .. "_"
+ end
+
+ rule.source_selector = lua_selectors.create_selector_closure(rspamd_config,
+ rule.source_selector, '')
+ rule.cluster_selector = lua_selectors.create_selector_closure(rspamd_config,
+ rule.cluster_selector, '')
+ if rule.source_selector and rule.cluster_selector then
+ rule.name = k
+ table.insert(rules, rule)
+ end
+ end
+ end
+
+ if #rules > 0 then
+
+ query_cluster_id = lua_redis.add_redis_script(query_cluster_script, redis_params)
+ update_cluster_id = lua_redis.add_redis_script(update_cluster_script, redis_params)
+ local function callback_gen(f, rule)
+ return function(task)
+ return f(task, rule)
+ end
+ end
+
+ for _, rule in ipairs(rules) do
+ rspamd_config:register_symbol {
+ name = rule.symbol,
+ type = 'normal',
+ callback = callback_gen(clusterting_filter_cb, rule),
+ }
+ rspamd_config:register_symbol {
+ name = rule.symbol .. '_STORE',
+ type = 'idempotent',
+ flags = 'empty,explicit_disable,ignore_passthrough',
+ callback = callback_gen(clusterting_idempotent_cb, rule),
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }
+ }
+ end
+ else
+ lua_util.disable_module(N, "config")
+ end
+else
+ lua_util.disable_module(N, "config")
+end
diff --git a/src/plugins/lua/dcc.lua b/src/plugins/lua/dcc.lua
new file mode 100644
index 0000000..8508320
--- /dev/null
+++ b/src/plugins/lua/dcc.lua
@@ -0,0 +1,119 @@
+--[[
+Copyright (c) 2016, Steve Freegard <steve.freegard@fsl.com>
+Copyright (c) 2016, Vsevolod Stakhov
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Check messages for 'bulkiness' using DCC
+
+local N = 'dcc'
+local symbol_bulk = "DCC_BULK"
+local symbol = "DCC_REJECT"
+local opts = rspamd_config:get_all_opt(N)
+local lua_util = require "lua_util"
+local rspamd_logger = require "rspamd_logger"
+local dcc = require("lua_scanners").filter('dcc').dcc
+
+if confighelp then
+ rspamd_config:add_example(nil, 'dcc',
+ "Check messages for 'bulkiness' using DCC",
+ [[
+ dcc {
+ socket = "/var/dcc/dccifd"; # Unix socket
+ servers = "127.0.0.1:10045" # OR TCP upstreams
+ timeout = 2s; # Timeout to wait for checks
+ body_max = 999999; # Bulkness threshold for body
+ fuz1_max = 999999; # Bulkness threshold for fuz1
+ fuz2_max = 999999; # Bulkness threshold for fuz2
+ }
+ ]])
+ return
+end
+
+local rule
+
+local function check_dcc (task)
+ dcc.check(task, task:get_content(), nil, rule)
+end
+
+-- Configuration
+
+-- WORKAROUND for deprecated host and port settings
+if opts['host'] ~= nil and opts['port'] ~= nil then
+ opts['servers'] = opts['host'] .. ':' .. opts['port']
+ rspamd_logger.warnx(rspamd_config, 'Using host and port parameters is deprecated. ' ..
+ 'Please use servers = "%s:%s"; instead', opts['host'], opts['port'])
+end
+if opts['host'] ~= nil and not opts['port'] then
+ opts['socket'] = opts['host']
+ rspamd_logger.warnx(rspamd_config, 'Using host parameters is deprecated. ' ..
+ 'Please use socket = "%s"; instead', opts['host'])
+end
+-- WORKAROUND for deprecated host and port settings
+
+if not opts.symbol_bulk then
+ opts.symbol_bulk = symbol_bulk
+end
+if not opts.symbol then
+ opts.symbol = symbol
+end
+
+rule = dcc.configure(opts)
+
+if rule then
+ local id = rspamd_config:register_symbol({
+ name = 'DCC_CHECK',
+ callback = check_dcc,
+ type = 'callback',
+ })
+ rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = id,
+ name = opts.symbol
+ }
+ rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = id,
+ name = opts.symbol_bulk
+ }
+ rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = id,
+ name = 'DCC_FAIL'
+ }
+ rspamd_config:set_metric_symbol({
+ group = N,
+ score = 1.0,
+ description = 'Detected as bulk mail by DCC',
+ one_shot = true,
+ name = opts.symbol_bulk,
+ })
+ rspamd_config:set_metric_symbol({
+ group = N,
+ score = 2.0,
+ description = 'Rejected by DCC',
+ one_shot = true,
+ name = opts.symbol,
+ })
+ rspamd_config:set_metric_symbol({
+ group = N,
+ score = 0.0,
+ description = 'DCC failure',
+ one_shot = true,
+ name = 'DCC_FAIL',
+ })
+else
+ lua_util.disable_module(N, "config")
+ rspamd_logger.infox('DCC module not configured');
+end
diff --git a/src/plugins/lua/dkim_signing.lua b/src/plugins/lua/dkim_signing.lua
new file mode 100644
index 0000000..6c05520
--- /dev/null
+++ b/src/plugins/lua/dkim_signing.lua
@@ -0,0 +1,186 @@
+--[[
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local lua_util = require "lua_util"
+local rspamd_logger = require "rspamd_logger"
+local dkim_sign_tools = require "lua_dkim_tools"
+local lua_redis = require "lua_redis"
+local lua_mime = require "lua_mime"
+
+if confighelp then
+ return
+end
+
+local settings = {
+ allow_envfrom_empty = true,
+ allow_hdrfrom_mismatch = false,
+ allow_hdrfrom_mismatch_local = false,
+ allow_hdrfrom_mismatch_sign_networks = false,
+ allow_hdrfrom_multiple = false,
+ allow_username_mismatch = false,
+ allow_pubkey_mismatch = true,
+ sign_authenticated = true,
+ allowed_ids = nil,
+ forbidden_ids = nil,
+ check_pubkey = false,
+ domain = {},
+ path = string.format('%s/%s/%s', rspamd_paths['DBDIR'], 'dkim', '$domain.$selector.key'),
+ sign_local = true,
+ selector = 'dkim',
+ symbol = 'DKIM_SIGNED',
+ try_fallback = true,
+ use_domain = 'header',
+ use_esld = true,
+ use_redis = false,
+ key_prefix = 'dkim_keys', -- default hash name
+ use_milter_headers = false, -- use milter headers instead of `dkim_signature`
+}
+
+local N = 'dkim_signing'
+local redis_params
+local sign_func = rspamd_plugins.dkim.sign
+
+local function insert_sign_results(task, ret, hdr, dkim_params)
+ if settings.use_milter_headers then
+ lua_mime.modify_headers(task, {
+ add = {
+ ['DKIM-Signature'] = { order = 1, value = hdr },
+ }
+ })
+ end
+ if ret then
+ task:insert_result(settings.symbol, 1.0, string.format('%s:s=%s',
+ dkim_params.domain, dkim_params.selector))
+ end
+end
+
+local function do_sign(task, p)
+ if settings.use_milter_headers then
+ p.no_cache = true -- Disable caching in rspamd_mempool
+ end
+ if settings.check_pubkey then
+ local resolve_name = p.selector .. "._domainkey." .. p.domain
+ task:get_resolver():resolve_txt({
+ task = task,
+ name = resolve_name,
+ callback = function(_, _, results, err)
+ if not err and results and results[1] then
+ p.pubkey = results[1]
+ p.strict_pubkey_check = not settings.allow_pubkey_mismatch
+ elseif not settings.allow_pubkey_mismatch then
+ rspamd_logger.infox(task, 'public key for domain %s/%s is not found: %s, skip signing',
+ p.domain, p.selector, err)
+ return
+ else
+ rspamd_logger.infox(task, 'public key for domain %s/%s is not found: %s',
+ p.domain, p.selector, err)
+ end
+
+ local sret, hdr = sign_func(task, p)
+ insert_sign_results(task, sret, hdr, p)
+ end,
+ forced = true
+ })
+ else
+ local sret, hdr = sign_func(task, p)
+ insert_sign_results(task, sret, hdr, p)
+ end
+end
+
+local function sign_error(task, msg)
+ rspamd_logger.errx(task, 'signing failure: %s', msg)
+end
+
+local function dkim_signing_cb(task)
+ local ret, selectors = dkim_sign_tools.prepare_dkim_signing(N, task, settings)
+
+ if not ret then
+ return
+ end
+
+ if settings.use_redis then
+ dkim_sign_tools.sign_using_redis(N, task, settings, selectors, do_sign, sign_error)
+ else
+ if selectors.vault then
+ dkim_sign_tools.sign_using_vault(N, task, settings, selectors, do_sign, sign_error)
+ else
+ if #selectors > 0 then
+ for _, k in ipairs(selectors) do
+ -- templates
+ if k.key then
+ k.key = lua_util.template(k.key, {
+ domain = k.domain,
+ selector = k.selector
+ })
+ lua_util.debugm(N, task, 'using key "%s", use selector "%s" for domain "%s"',
+ k.key, k.selector, k.domain)
+ end
+
+ do_sign(task, k)
+ end
+ else
+ rspamd_logger.infox(task, 'key path or dkim selector unconfigured; no signing')
+ return false
+ end
+ end
+ end
+end
+
+local opts = rspamd_config:get_all_opt('dkim_signing')
+if not opts then
+ return
+end
+
+dkim_sign_tools.process_signing_settings(N, settings, opts)
+
+if not dkim_sign_tools.validate_signing_settings(settings) then
+ rspamd_logger.infox(rspamd_config, 'mandatory parameters missing, disable dkim signing')
+ lua_util.disable_module(N, "config")
+ return
+end
+
+if settings.use_redis then
+ redis_params = lua_redis.parse_redis_server('dkim_signing')
+
+ if not redis_params then
+ rspamd_logger.errx(rspamd_config,
+ 'no servers are specified, but module is configured to load keys from redis, disable dkim signing')
+ lua_util.disable_module(N, "redis")
+ return
+ end
+
+ settings.redis_params = redis_params
+end
+
+local sym_reg_tbl = {
+ name = settings['symbol'],
+ callback = dkim_signing_cb,
+ groups = { "policies", "dkim" },
+ flags = 'ignore_passthrough',
+ score = 0.0,
+}
+
+if type(settings.allowed_ids) == 'table' then
+ sym_reg_tbl.allowed_ids = settings.allowed_ids
+end
+if type(settings.forbidden_ids) == 'table' then
+ sym_reg_tbl.forbidden_ids = settings.forbidden_ids
+end
+
+rspamd_config:register_symbol(sym_reg_tbl)
+-- Add dependency on DKIM checks
+rspamd_config:register_dependency(settings['symbol'], 'DKIM_CHECK')
diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua
new file mode 100644
index 0000000..792672b
--- /dev/null
+++ b/src/plugins/lua/dmarc.lua
@@ -0,0 +1,685 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2015-2016, Andrew Lewis <nerf@judo.za.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Dmarc policy filter
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local lua_redis = require "lua_redis"
+local lua_util = require "lua_util"
+local dmarc_common = require "plugins/dmarc"
+
+if confighelp then
+ return
+end
+
+local N = 'dmarc'
+
+local settings = dmarc_common.default_settings
+
+local redis_params = nil
+
+local E = {}
+
+-- Keys:
+-- 1 = index key (string)
+-- 2 = report key (string)
+-- 3 = max report elements (number)
+-- 4 = expiry time for elements (number)
+-- Arguments
+-- 1 = dmarc domain
+-- 2 = dmarc report
+local take_report_id
+local take_report_script = [[
+local index_key = KEYS[1]
+local report_key = KEYS[2]
+local max_entries = -(tonumber(KEYS[3]) + 1)
+local keys_expiry = tonumber(KEYS[4])
+local dmarc_domain = ARGV[1]
+local report = ARGV[2]
+redis.call('SADD', index_key, report_key)
+redis.call('EXPIRE', index_key, 172800)
+redis.call('ZINCRBY', report_key, 1, report)
+redis.call('ZREMRANGEBYRANK', report_key, 0, max_entries)
+redis.call('EXPIRE', report_key, 172800)
+]]
+
+local function maybe_force_action(task, disposition)
+ if disposition then
+ local force_action = settings.actions[disposition]
+ if force_action then
+ -- Set least action
+ task:set_pre_result(force_action, 'Action set by DMARC', N, nil, nil, 'least')
+ end
+ end
+end
+
+local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld)
+ local reason = {}
+
+ -- Check dkim and spf symbols
+ local spf_ok = false
+ local dkim_ok = false
+ local spf_tmpfail = false
+ local dkim_tmpfail = false
+
+ local spf_domain = ((task:get_from(1) or E)[1] or E).domain
+
+ if not spf_domain or spf_domain == '' then
+ spf_domain = task:get_helo() or ''
+ end
+
+ if task:has_symbol(settings.symbols['spf_allow_symbol']) then
+ if policy.strict_spf then
+ if rspamd_util.strequal_caseless(spf_domain, hdrfromdom) then
+ spf_ok = true
+ else
+ table.insert(reason, "SPF not aligned (strict)")
+ end
+ else
+ local spf_tld = rspamd_util.get_tld(spf_domain)
+ if rspamd_util.strequal_caseless(spf_tld, dmarc_esld) then
+ spf_ok = true
+ else
+ table.insert(reason, "SPF not aligned (relaxed)")
+ end
+ end
+ else
+ if task:has_symbol(settings.symbols['spf_tempfail_symbol']) then
+ if policy.strict_spf then
+ if rspamd_util.strequal_caseless(spf_domain, hdrfromdom) then
+ spf_tmpfail = true
+ end
+ else
+ local spf_tld = rspamd_util.get_tld(spf_domain)
+ if rspamd_util.strequal_caseless(spf_tld, dmarc_esld) then
+ spf_tmpfail = true
+ end
+ end
+ end
+
+ table.insert(reason, "No valid SPF")
+ end
+
+ local opts = ((task:get_symbol('DKIM_TRACE') or E)[1] or E).options
+ local dkim_results = {
+ pass = {},
+ temperror = {},
+ permerror = {},
+ fail = {},
+ }
+
+ if opts then
+ dkim_results.pass = {}
+ local dkim_violated
+
+ for _, opt in ipairs(opts) do
+ local check_res = string.sub(opt, -1)
+ local domain = string.sub(opt, 1, -3):lower()
+
+ if check_res == '+' then
+ table.insert(dkim_results.pass, domain)
+
+ if policy.strict_dkim then
+ if rspamd_util.strequal_caseless(hdrfromdom, domain) then
+ dkim_ok = true
+ else
+ dkim_violated = "DKIM not aligned (strict)"
+ end
+ else
+ local dkim_tld = rspamd_util.get_tld(domain)
+
+ if rspamd_util.strequal_caseless(dkim_tld, dmarc_esld) then
+ dkim_ok = true
+ else
+ dkim_violated = "DKIM not aligned (relaxed)"
+ end
+ end
+ elseif check_res == '?' then
+ -- Check for dkim tempfail
+ if not dkim_ok then
+ if policy.strict_dkim then
+ if rspamd_util.strequal_caseless(hdrfromdom, domain) then
+ dkim_tmpfail = true
+ end
+ else
+ local dkim_tld = rspamd_util.get_tld(domain)
+
+ if rspamd_util.strequal_caseless(dkim_tld, dmarc_esld) then
+ dkim_tmpfail = true
+ end
+ end
+ end
+ table.insert(dkim_results.temperror, domain)
+ elseif check_res == '-' then
+ table.insert(dkim_results.fail, domain)
+ else
+ table.insert(dkim_results.permerror, domain)
+ end
+ end
+
+ if not dkim_ok and dkim_violated then
+ table.insert(reason, dkim_violated)
+ end
+ else
+ table.insert(reason, "No valid DKIM")
+ end
+
+ lua_util.debugm(N, task,
+ "validated dmarc policy for %s: %s; dkim_ok=%s, dkim_tempfail=%s, spf_ok=%s, spf_tempfail=%s",
+ policy.domain, policy.dmarc_policy,
+ dkim_ok, dkim_tmpfail,
+ spf_ok, spf_tmpfail)
+
+ local disposition = 'none'
+ local sampled_out = false
+
+ local function handle_dmarc_failure(what, reason_str)
+ if not policy.pct or policy.pct == 100 then
+ task:insert_result(settings.symbols[what], 1.0,
+ policy.domain .. ' : ' .. reason_str, policy.dmarc_policy)
+ disposition = what
+ else
+ local coin = math.random(100)
+ if (coin > policy.pct) then
+ if (not settings.no_sampling_domains or
+ not settings.no_sampling_domains:get_key(policy.domain)) then
+
+ if what == 'reject' then
+ disposition = 'quarantine'
+ else
+ disposition = 'softfail'
+ end
+
+ task:insert_result(settings.symbols[disposition], 1.0,
+ policy.domain .. ' : ' .. reason_str, policy.dmarc_policy, "sampled_out")
+ sampled_out = true
+ lua_util.debugm(N, task,
+ 'changed dmarc policy from %s to %s, sampled out: %s < %s',
+ what, disposition, coin, policy.pct)
+ else
+ task:insert_result(settings.symbols[what], 1.0,
+ policy.domain .. ' : ' .. reason_str, policy.dmarc_policy, "local_policy")
+ disposition = what
+ end
+ else
+ task:insert_result(settings.symbols[what], 1.0,
+ policy.domain .. ' : ' .. reason_str, policy.dmarc_policy)
+ disposition = what
+ end
+ end
+
+ maybe_force_action(task, disposition)
+ end
+
+ if spf_ok or dkim_ok then
+ --[[
+ https://tools.ietf.org/html/rfc7489#section-6.6.2
+ DMARC evaluation can only yield a "pass" result after one of the
+ underlying authentication mechanisms passes for an aligned
+ identifier.
+ ]]--
+ task:insert_result(settings.symbols['allow'], 1.0, policy.domain,
+ policy.dmarc_policy)
+ else
+ --[[
+ https://tools.ietf.org/html/rfc7489#section-6.6.2
+
+ If neither passes and one or both of them fail due to a
+ temporary error, the Receiver evaluating the message is unable to
+ conclude that the DMARC mechanism had a permanent failure; they
+ therefore cannot apply the advertised DMARC policy.
+ ]]--
+ if spf_tmpfail or dkim_tmpfail then
+ task:insert_result(settings.symbols['dnsfail'], 1.0, policy.domain ..
+ ' : ' .. 'SPF/DKIM temp error', policy.dmarc_policy)
+ else
+ -- We can now check the failed policy and maybe send report data elt
+ local reason_str = table.concat(reason, ', ')
+
+ if policy.dmarc_policy == 'quarantine' then
+ handle_dmarc_failure('quarantine', reason_str)
+ elseif policy.dmarc_policy == 'reject' then
+ handle_dmarc_failure('reject', reason_str)
+ else
+ task:insert_result(settings.symbols['softfail'], 1.0,
+ policy.domain .. ' : ' .. reason_str,
+ policy.dmarc_policy)
+ end
+ end
+ end
+
+ if policy.rua and redis_params and settings.reporting.enabled then
+ if settings.reporting.exclude_domains then
+ if settings.reporting.exclude_domains:get_key(policy.domain) or
+ settings.reporting.exclude_domains:get_key(rspamd_util.get_tld(policy.domain)) then
+ rspamd_logger.info(task, 'DMARC reporting suppressed for sender domain %s', policy.domain)
+ return
+ end
+ end
+ if settings.reporting.exclude_recipients then
+ local rcpt = task:get_principal_recipient()
+ if rcpt and settings.reporting.exclude_recipients:get_key(rcpt) then
+ rspamd_logger.info(task, 'DMARC reporting suppressed for recipient %s', rcpt)
+ return
+ end
+ end
+
+ local function dmarc_report_cb(err)
+ if not err then
+ rspamd_logger.infox(task, 'dmarc report saved for %s (rua = %s)',
+ hdrfromdom, policy.rua)
+ else
+ rspamd_logger.errx(task, 'dmarc report is not saved for %s: %s',
+ hdrfromdom, err)
+ end
+ end
+
+ local spf_result
+ if spf_ok then
+ spf_result = 'pass'
+ elseif spf_tmpfail then
+ spf_result = 'temperror'
+ else
+ if task:has_symbol(settings.symbols.spf_deny_symbol) then
+ spf_result = 'fail'
+ elseif task:has_symbol(settings.symbols.spf_softfail_symbol) then
+ spf_result = 'softfail'
+ elseif task:has_symbol(settings.symbols.spf_neutral_symbol) then
+ spf_result = 'neutral'
+ elseif task:has_symbol(settings.symbols.spf_permfail_symbol) then
+ spf_result = 'permerror'
+ else
+ spf_result = 'none'
+ end
+ end
+
+ -- Prepare and send redis report element
+ local period = os.date('%Y%m%d',
+ task:get_date({ format = 'connect', gmt = false }))
+
+ -- Dmarc domain key must include dmarc domain, rua and period
+ local dmarc_domain_key = table.concat(
+ { settings.reporting.redis_keys.report_prefix, policy.domain, policy.rua, period },
+ settings.reporting.redis_keys.join_char)
+ local report_data = dmarc_common.dmarc_report(task, settings, {
+ spf_ok = spf_ok and 'pass' or 'fail',
+ dkim_ok = dkim_ok and 'pass' or 'fail',
+ disposition = (disposition == "softfail") and "none" or disposition,
+ sampled_out = sampled_out,
+ domain = hdrfromdom,
+ spf_domain = spf_domain,
+ dkim_results = dkim_results,
+ spf_result = spf_result
+ })
+
+ local idx_key = table.concat({ settings.reporting.redis_keys.index_prefix, period },
+ settings.reporting.redis_keys.join_char)
+
+ if report_data then
+ lua_redis.exec_redis_script(take_report_id,
+ { task = task, is_write = true },
+ dmarc_report_cb,
+ { idx_key, dmarc_domain_key,
+ tostring(settings.reporting.max_entries), tostring(settings.reporting.keys_expire) },
+ { hdrfromdom, report_data })
+ end
+ end
+end
+
+local function dmarc_callback(task)
+ local from = task:get_from(2)
+ local hfromdom = ((from or E)[1] or E).domain
+ local dmarc_domain
+ local ip_addr = task:get_ip()
+ local dmarc_checks = task:get_mempool():get_variable('dmarc_checks', 'double') or 0
+ local seen_invalid = false
+
+ if dmarc_checks ~= 2 then
+ rspamd_logger.infox(task, "skip DMARC checks as either SPF or DKIM were not checked")
+ return
+ end
+
+ if lua_util.is_skip_local_or_authed(task, settings.auth_and_local_conf, ip_addr) then
+ rspamd_logger.infox(task, "skip DMARC checks for local networks and authorized users")
+ return
+ end
+
+ -- Do some initial sanity checks, detect tld domain if different
+ if hfromdom and hfromdom ~= '' and not (from or E)[2] then
+ -- Lowercase domain as per #3940
+ hfromdom = hfromdom:lower()
+ dmarc_domain = rspamd_util.get_tld(hfromdom)
+ elseif (from or E)[2] then
+ task:insert_result(settings.symbols['na'], 1.0, 'Duplicate From header')
+ return maybe_force_action(task, 'na')
+ elseif (from or E)[1] then
+ task:insert_result(settings.symbols['na'], 1.0, 'No domain in From header')
+ return maybe_force_action(task, 'na')
+ else
+ task:insert_result(settings.symbols['na'], 1.0, 'No From header')
+ return maybe_force_action(task, 'na')
+ end
+
+ local dns_checks_inflight = 0
+ local dmarc_domain_policy = {}
+ local dmarc_tld_policy = {}
+
+ local function process_dmarc_policy(policy, final)
+ lua_util.debugm(N, task, "validate DMARC policy (final=%s): %s",
+ true, policy)
+ if policy.err and policy.symbol then
+ -- In case of fatal errors or final check for tld, we give up and
+ -- insert result
+ if final or policy.fatal then
+ task:insert_result(policy.symbol, 1.0, policy.err)
+ maybe_force_action(task, policy.disposition)
+
+ return true
+ end
+ elseif policy.dmarc_policy then
+ dmarc_validate_policy(task, policy, hfromdom, dmarc_domain)
+
+ return true -- We have a more specific version, use it
+ end
+
+ return false -- Missing record
+ end
+
+ local function gen_dmarc_cb(lookup_domain, is_tld)
+ local policy_target = dmarc_domain_policy
+ if is_tld then
+ policy_target = dmarc_tld_policy
+ end
+
+ return function(_, _, results, err)
+ dns_checks_inflight = dns_checks_inflight - 1
+
+ if not seen_invalid then
+ policy_target.domain = lookup_domain
+
+ if err then
+ if (err ~= 'requested record is not found' and
+ err ~= 'no records with this name') then
+ policy_target.err = lookup_domain .. ' : ' .. err
+ policy_target.symbol = settings.symbols['dnsfail']
+ else
+ policy_target.err = lookup_domain
+ policy_target.symbol = settings.symbols['na']
+ end
+ else
+ local has_valid_policy = false
+
+ for _, rec in ipairs(results) do
+ local ret, results_or_err = dmarc_common.dmarc_check_record(task, rec, is_tld)
+
+ if not ret then
+ if results_or_err then
+ -- We have a fatal parsing error, give up
+ policy_target.err = lookup_domain .. ' : ' .. results_or_err
+ policy_target.symbol = settings.symbols['badpolicy']
+ policy_target.fatal = true
+ seen_invalid = true
+ end
+ else
+ if has_valid_policy then
+ policy_target.err = lookup_domain .. ' : ' ..
+ 'Multiple policies defined in DNS'
+ policy_target.symbol = settings.symbols['badpolicy']
+ policy_target.fatal = true
+ seen_invalid = true
+ end
+ has_valid_policy = true
+
+ for k, v in pairs(results_or_err) do
+ policy_target[k] = v
+ end
+ end
+ end
+
+ if not has_valid_policy and not seen_invalid then
+ policy_target.err = lookup_domain .. ':' .. ' no valid DMARC record'
+ policy_target.symbol = settings.symbols['na']
+ end
+ end
+ end
+
+ if dns_checks_inflight == 0 then
+ lua_util.debugm(N, task, "finished DNS queries, validate policies")
+ -- We have checked both tld and real domain (if different)
+ if not process_dmarc_policy(dmarc_domain_policy, false) then
+ -- Try tld policy as well
+ if not process_dmarc_policy(dmarc_tld_policy, true) then
+ process_dmarc_policy(dmarc_domain_policy, true)
+ end
+ end
+ end
+ end
+ end
+
+ local resolve_name = '_dmarc.' .. hfromdom
+
+ task:get_resolver():resolve_txt({
+ task = task,
+ name = resolve_name,
+ callback = gen_dmarc_cb(hfromdom, false),
+ forced = true
+ })
+ dns_checks_inflight = dns_checks_inflight + 1
+
+ if dmarc_domain ~= hfromdom then
+ resolve_name = '_dmarc.' .. dmarc_domain
+
+ task:get_resolver():resolve_txt({
+ task = task,
+ name = resolve_name,
+ callback = gen_dmarc_cb(dmarc_domain, true),
+ forced = true
+ })
+
+ dns_checks_inflight = dns_checks_inflight + 1
+ end
+end
+
+local opts = rspamd_config:get_all_opt('dmarc')
+settings = lua_util.override_defaults(settings, opts)
+
+settings.auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N,
+ false, false)
+
+-- Legacy...
+if settings.reporting and not settings.reporting.exclude_domains and settings.no_reporting_domains then
+ settings.reporting.exclude_domains = settings.no_reporting_domains
+end
+
+local lua_maps = require "lua_maps"
+lua_maps.fill_config_maps(N, settings, {
+ no_sampling_domains = {
+ optional = true,
+ type = 'map',
+ description = 'Domains not to apply DMARC sampling to'
+ },
+})
+
+if type(settings.reporting) == 'table' then
+ lua_maps.fill_config_maps(N, settings.reporting, {
+ exclude_domains = {
+ optional = true,
+ type = 'map',
+ description = 'Domains not to store DMARC reports about'
+ },
+ exclude_recipients = {
+ optional = true,
+ type = 'map',
+ description = 'Recipients not to store DMARC reports for'
+ },
+ })
+end
+
+if settings.reporting == true then
+ rspamd_logger.errx(rspamd_config, 'old style dmarc reporting is NO LONGER supported, please read the documentation')
+elseif settings.reporting.enabled then
+ redis_params = lua_redis.parse_redis_server('dmarc', opts)
+ if not redis_params then
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers parameter')
+ else
+ rspamd_logger.infox(rspamd_config, 'dmarc reporting is enabled')
+ take_report_id = lua_redis.add_redis_script(take_report_script, redis_params)
+ end
+end
+
+-- Check spf and dkim sections for changed symbols
+local function check_mopt(var, m_opts, name)
+ if m_opts[name] then
+ settings.symbols[var] = tostring(m_opts[name])
+ end
+end
+
+local spf_opts = rspamd_config:get_all_opt('spf')
+if spf_opts then
+ check_mopt('spf_deny_symbol', spf_opts, 'symbol_fail')
+ check_mopt('spf_allow_symbol', spf_opts, 'symbol_allow')
+ check_mopt('spf_softfail_symbol', spf_opts, 'symbol_softfail')
+ check_mopt('spf_neutral_symbol', spf_opts, 'symbol_neutral')
+ check_mopt('spf_tempfail_symbol', spf_opts, 'symbol_dnsfail')
+ check_mopt('spf_na_symbol', spf_opts, 'symbol_na')
+end
+
+local dkim_opts = rspamd_config:get_all_opt('dkim')
+if dkim_opts then
+ check_mopt('dkim_deny_symbol', dkim_opts, 'symbol_reject')
+ check_mopt('dkim_allow_symbol', dkim_opts, 'symbol_allow')
+ check_mopt('dkim_tempfail_symbol', dkim_opts, 'symbol_tempfail')
+ check_mopt('dkim_na_symbol', dkim_opts, 'symbol_na')
+end
+
+local id = rspamd_config:register_symbol({
+ name = 'DMARC_CHECK',
+ type = 'callback',
+ callback = dmarc_callback
+})
+rspamd_config:register_symbol({
+ name = 'DMARC_CALLBACK', -- compatibility symbol
+ type = 'virtual,skip',
+ parent = id,
+})
+rspamd_config:register_symbol({
+ name = settings.symbols['allow'],
+ parent = id,
+ group = 'policies',
+ groups = { 'dmarc' },
+ type = 'virtual'
+})
+rspamd_config:register_symbol({
+ name = settings.symbols['reject'],
+ parent = id,
+ group = 'policies',
+ groups = { 'dmarc' },
+ type = 'virtual'
+})
+rspamd_config:register_symbol({
+ name = settings.symbols['quarantine'],
+ parent = id,
+ group = 'policies',
+ groups = { 'dmarc' },
+ type = 'virtual'
+})
+rspamd_config:register_symbol({
+ name = settings.symbols['softfail'],
+ parent = id,
+ group = 'policies',
+ groups = { 'dmarc' },
+ type = 'virtual'
+})
+rspamd_config:register_symbol({
+ name = settings.symbols['dnsfail'],
+ parent = id,
+ group = 'policies',
+ groups = { 'dmarc' },
+ type = 'virtual'
+})
+rspamd_config:register_symbol({
+ name = settings.symbols['badpolicy'],
+ parent = id,
+ group = 'policies',
+ groups = { 'dmarc' },
+ type = 'virtual'
+})
+rspamd_config:register_symbol({
+ name = settings.symbols['na'],
+ parent = id,
+ group = 'policies',
+ groups = { 'dmarc' },
+ type = 'virtual'
+})
+
+rspamd_config:register_dependency('DMARC_CHECK', settings.symbols['spf_allow_symbol'])
+rspamd_config:register_dependency('DMARC_CHECK', settings.symbols['dkim_allow_symbol'])
+
+-- DMARC munging support
+if settings.munging then
+ local lua_maps_expressions = require "lua_maps_expressions"
+
+ local munging_defaults = {
+ reply_goes_to_list = false,
+ mitigate_allow_only = true, -- perform munging based on DMARC_POLICY_ALLOW only
+ mitigate_strict_only = false, -- perform mugning merely for reject/quarantine policies
+ munge_from = true, -- replace from with something like <orig name> via <rcpt user>
+ list_map = nil, -- map of maillist domains
+ munge_map_condition = nil, -- maps expression to enable munging
+ }
+
+ local munging_opts = lua_util.override_defaults(munging_defaults, settings.munging)
+
+ if not munging_opts.list_map then
+ rspamd_logger.errx(rspamd_config, 'cannot enable DMARC munging with no list_map parameter')
+
+ return
+ end
+
+ munging_opts.list_map = lua_maps.map_add_from_ucl(munging_opts.list_map,
+ 'set', 'DMARC munging map of the recipients addresses to munge')
+
+ if not munging_opts.list_map then
+ rspamd_logger.errx(rspamd_config, 'cannot enable DMARC munging with invalid list_map (invalid map)')
+
+ return
+ end
+
+ if munging_opts.munge_map_condition then
+ munging_opts.munge_map_condition = lua_maps_expressions.create(rspamd_config,
+ munging_opts.munge_map_condition, N)
+ end
+
+ rspamd_config:register_symbol({
+ name = 'DMARC_MUNGED',
+ type = 'normal',
+ flags = 'nostat',
+ score = 0,
+ group = 'policies',
+ groups = { 'dmarc' },
+ callback = dmarc_common.gen_munging_callback(munging_opts, settings),
+ augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) },
+ })
+
+ rspamd_config:register_dependency('DMARC_MUNGED', 'DMARC_CHECK')
+ -- To avoid dkim signing issues
+ rspamd_config:register_dependency('DKIM_SIGNED', 'DMARC_MUNGED')
+ rspamd_config:register_dependency('ARC_SIGNED', 'DMARC_MUNGED')
+
+ rspamd_logger.infox(rspamd_config, 'enabled DMARC munging')
+end
diff --git a/src/plugins/lua/dynamic_conf.lua b/src/plugins/lua/dynamic_conf.lua
new file mode 100644
index 0000000..5af26a9
--- /dev/null
+++ b/src/plugins/lua/dynamic_conf.lua
@@ -0,0 +1,333 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local rspamd_logger = require "rspamd_logger"
+local redis_params
+local ucl = require "ucl"
+local fun = require "fun"
+local lua_util = require "lua_util"
+local rspamd_redis = require "lua_redis"
+local N = "dynamic_conf"
+
+if confighelp then
+ return
+end
+
+local settings = {
+ redis_key = "dynamic_conf",
+ redis_watch_interval = 10.0,
+ priority = 10
+}
+
+local cur_settings = {
+ version = 0,
+ updates = {
+ symbols = {},
+ actions = {},
+ has_updates = false
+ }
+}
+
+local function alpha_cmp(v1, v2)
+ local math = math
+ if math.abs(v1 - v2) < 0.001 then
+ return true
+ end
+
+ return false
+end
+
+local function apply_dynamic_actions(_, acts)
+ fun.each(function(k, v)
+ if type(v) == 'table' then
+ v['name'] = k
+ if not v['priority'] then
+ v['priority'] = settings.priority
+ end
+ rspamd_config:set_metric_action(v)
+ else
+ rspamd_config:set_metric_symbol({
+ name = k,
+ score = v,
+ priority = settings.priority
+ })
+ end
+ end, fun.filter(function(k, v)
+ local act = rspamd_config:get_metric_action(k)
+ if (act and alpha_cmp(act, v)) or cur_settings.updates.actions[k] then
+ return false
+ end
+
+ return true
+ end, acts))
+end
+
+local function apply_dynamic_scores(_, sc)
+ fun.each(function(k, v)
+ if type(v) == 'table' then
+ v['name'] = k
+ if not v['priority'] then
+ v['priority'] = settings.priority
+ end
+ rspamd_config:set_metric_symbol(v)
+ else
+ rspamd_config:set_metric_symbol({
+ name = k,
+ score = v,
+ priority = settings.priority
+ })
+ end
+ end, fun.filter(function(k, v)
+ -- Select elts with scores that are different from local ones
+ local sym = rspamd_config:get_symbol(k)
+ if (sym and alpha_cmp(sym.score, v)) or cur_settings.updates.symbols[k] then
+ return false
+ end
+
+ return true
+ end, sc))
+end
+
+local function apply_dynamic_conf(cfg, data)
+ if data['scores'] then
+ -- Apply scores changes
+ apply_dynamic_scores(cfg, data['scores'])
+ end
+
+ if data['actions'] then
+ apply_dynamic_actions(cfg, data['actions'])
+ end
+
+ if data['symbols_enabled'] then
+ fun.each(function(_, v)
+ cfg:enable_symbol(v)
+ end, data['symbols_enabled'])
+ end
+
+ if data['symbols_disabled'] then
+ fun.each(function(_, v)
+ cfg:disable_symbol(v)
+ end, data['symbols_disabled'])
+ end
+end
+
+local function update_dynamic_conf(cfg, ev_base, recv)
+ local function redis_version_set_cb(err, data)
+ if err then
+ rspamd_logger.errx(cfg, "cannot save dynamic conf version to redis: %s", err)
+ else
+ rspamd_logger.infox(cfg, "saved dynamic conf version: %s", data)
+ cur_settings.updates.has_updates = false
+ cur_settings.updates.symbols = {}
+ cur_settings.updates.actions = {}
+ end
+ end
+ local function redis_data_set_cb(err)
+ if err then
+ rspamd_logger.errx(cfg, "cannot save dynamic conf to redis: %s", err)
+ else
+ rspamd_redis.redis_make_request_taskless(ev_base,
+ cfg,
+ redis_params,
+ settings.redis_key,
+ true,
+ redis_version_set_cb,
+ 'HINCRBY', { settings.redis_key, 'v', '1' })
+ end
+ end
+
+ if recv then
+ -- We need to merge two configs
+ if recv['scores'] then
+ if not cur_settings.data.scores then
+ cur_settings.data.scores = {}
+ end
+ fun.each(function(k, v)
+ cur_settings.data.scores[k] = v
+ end,
+ fun.filter(function(k)
+ if cur_settings.updates.symbols[k] then
+ return false
+ end
+ return true
+ end, recv['scores']))
+ end
+ if recv['actions'] then
+ if not cur_settings.data.actions then
+ cur_settings.data.actions = {}
+ end
+ fun.each(function(k, v)
+ cur_settings.data.actions[k] = v
+ end,
+ fun.filter(function(k)
+ if cur_settings.updates.actions[k] then
+ return false
+ end
+ return true
+ end, recv['actions']))
+ end
+ end
+ local newdata = ucl.to_format(cur_settings.data, 'json-compact')
+ rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params,
+ settings.redis_key, true,
+ redis_data_set_cb, 'HSET', { settings.redis_key, 'd', newdata })
+end
+
+local function check_dynamic_conf(cfg, ev_base)
+ local function redis_load_cb(redis_err, data)
+ if redis_err then
+ rspamd_logger.errx(cfg, "cannot read dynamic conf from redis: %s", redis_err)
+ elseif data and type(data) == 'string' then
+ local parser = ucl.parser()
+ local _, err = parser:parse_string(data)
+
+ if err then
+ rspamd_logger.errx(cfg, "cannot load dynamic conf from redis: %s", err)
+ else
+ local d = parser:get_object()
+ apply_dynamic_conf(cfg, d)
+ if cur_settings.updates.has_updates then
+ -- Need to send our updates to Redis
+ update_dynamic_conf(cfg, ev_base, d)
+ else
+ cur_settings.data = d
+ end
+ end
+ end
+ end
+ local function redis_check_cb(err, data)
+ if not err and type(data) == 'string' then
+ local rver = tonumber(data)
+
+ if not cur_settings.version or (rver and rver > cur_settings.version) then
+ rspamd_logger.infox(cfg, "need to load fresh dynamic settings with version %s, local version is %s",
+ rver, cur_settings.version)
+ cur_settings.version = rver
+ rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params,
+ settings.redis_key, false,
+ redis_load_cb, 'HGET', { settings.redis_key, 'd' })
+ elseif cur_settings.updates.has_updates then
+ -- Need to send our updates to Redis
+ update_dynamic_conf(cfg, ev_base)
+ end
+ elseif cur_settings.updates.has_updates then
+ -- Need to send our updates to Redis
+ update_dynamic_conf(cfg, ev_base)
+ end
+ end
+
+ rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params,
+ settings.redis_key, false,
+ redis_check_cb, 'HGET', { settings.redis_key, 'v' })
+end
+
+local section = rspamd_config:get_all_opt("dynamic_conf")
+if section then
+ redis_params = rspamd_redis.parse_redis_server('dynamic_conf')
+ if not redis_params then
+ rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
+ return
+ end
+
+ for k, v in pairs(section) do
+ settings[k] = v
+ end
+
+ rspamd_config:add_on_load(function(_, ev_base, worker)
+ if worker:is_scanner() then
+ rspamd_config:add_periodic(ev_base, 0.0,
+ function(cfg, _)
+ check_dynamic_conf(cfg, ev_base)
+ return settings.redis_watch_interval
+ end, true)
+ end
+ end)
+end
+
+-- Updates part
+local function add_dynamic_symbol(_, sym, score)
+ local add = false
+ if not cur_settings.data then
+ cur_settings.data = {}
+ end
+
+ if not cur_settings.data.scores then
+ cur_settings.data.scores = {}
+ cur_settings.data.scores[sym] = score
+ add = true
+ else
+ if cur_settings.data.scores[sym] then
+ if cur_settings.data.scores[sym] ~= score then
+ add = true
+ end
+ else
+ cur_settings.data.scores[sym] = score
+ add = true
+ end
+ end
+
+ if add then
+ cur_settings.data.scores[sym] = score
+ table.insert(cur_settings.updates.symbols, sym)
+ cur_settings.updates.has_updates = true
+ end
+
+ return add
+end
+
+local function add_dynamic_action(_, act, score)
+ local add = false
+ if not cur_settings.data then
+ cur_settings.data = {}
+ cur_settings.version = 0
+ end
+
+ if not cur_settings.data.actions then
+ cur_settings.data.actions = {}
+ cur_settings.data.actions[act] = score
+ add = true
+ else
+ if cur_settings.data.actions[act] then
+ if cur_settings.data.actions[act] ~= score then
+ add = true
+ end
+ else
+ cur_settings.data.actions[act] = score
+ add = true
+ end
+ end
+
+ if add then
+ cur_settings.data.actions[act] = score
+ table.insert(cur_settings.updates.actions, act)
+ cur_settings.updates.has_updates = true
+ end
+
+ return add
+end
+
+if section then
+ if redis_params then
+ rspamd_plugins["dynamic_conf"] = {
+ add_symbol = add_dynamic_symbol,
+ add_action = add_dynamic_action,
+ }
+ else
+ lua_util.disable_module(N, "redis")
+ end
+else
+ lua_util.disable_module(N, "config")
+end \ No newline at end of file
diff --git a/src/plugins/lua/elastic.lua b/src/plugins/lua/elastic.lua
new file mode 100644
index 0000000..ccbb7c1
--- /dev/null
+++ b/src/plugins/lua/elastic.lua
@@ -0,0 +1,544 @@
+--[[
+Copyright (c) 2017, Veselin Iordanov
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local rspamd_logger = require 'rspamd_logger'
+local rspamd_http = require "rspamd_http"
+local lua_util = require "lua_util"
+local util = require "rspamd_util"
+local ucl = require "ucl"
+local rspamd_redis = require "lua_redis"
+local upstream_list = require "rspamd_upstream_list"
+local lua_settings = require "lua_settings"
+
+if confighelp then
+ return
+end
+
+local rows = {}
+local nrows = 0
+local failed_sends = 0
+local elastic_template
+local redis_params
+local N = "elastic"
+local E = {}
+local HOSTNAME = util.get_hostname()
+local connect_prefix = 'http://'
+local enabled = true
+local ingest_geoip_type = 'plugins'
+local settings = {
+ limit = 500,
+ index_pattern = 'rspamd-%Y.%m.%d',
+ template_file = rspamd_paths['SHAREDIR'] .. '/elastic/rspamd_template.json',
+ kibana_file = rspamd_paths['SHAREDIR'] .. '/elastic/kibana.json',
+ key_prefix = 'elastic-',
+ expire = 3600,
+ timeout = 5.0,
+ failover = false,
+ import_kibana = false,
+ use_https = false,
+ use_gzip = true,
+ allow_local = false,
+ user = nil,
+ password = nil,
+ no_ssl_verify = false,
+ max_fail = 3,
+ ingest_module = false,
+ elasticsearch_version = 6,
+}
+
+local function read_file(path)
+ local file = io.open(path, "rb")
+ if not file then
+ return nil
+ end
+ local content = file:read "*a"
+ file:close()
+ return content
+end
+
+local function elastic_send_data(task)
+ local es_index = os.date(settings['index_pattern'])
+ local tbl = {}
+ for _, value in pairs(rows) do
+ if settings.elasticsearch_version >= 7 then
+ table.insert(tbl, '{ "index" : { "_index" : "' .. es_index ..
+ '","pipeline": "rspamd-geoip"} }')
+ else
+ table.insert(tbl, '{ "index" : { "_index" : "' .. es_index ..
+ '", "_type" : "_doc" ,"pipeline": "rspamd-geoip"} }')
+ end
+ table.insert(tbl, ucl.to_format(value, 'json-compact'))
+ end
+
+ table.insert(tbl, '') -- For last \n
+
+ local upstream = settings.upstream:get_upstream_round_robin()
+ local ip_addr = upstream:get_addr():to_string(true)
+
+ local push_url = connect_prefix .. ip_addr .. '/' .. es_index .. '/_bulk'
+ local bulk_json = table.concat(tbl, "\n")
+
+ local function http_callback(err, code, _, _)
+ if err then
+ rspamd_logger.infox(task, "cannot push data to elastic backend (%s): %s; failed attempts: %s/%s",
+ push_url, err, failed_sends, settings.max_fail)
+ else
+ if code ~= 200 then
+ rspamd_logger.infox(task,
+ "cannot push data to elastic backend (%s): wrong http code %s (%s); failed attempts: %s/%s",
+ push_url, err, code, failed_sends, settings.max_fail)
+ else
+ lua_util.debugm(N, task, "successfully sent %s (%s bytes) rows to ES",
+ nrows, #bulk_json)
+ end
+ end
+ end
+
+ return rspamd_http.request({
+ url = push_url,
+ headers = {
+ ['Content-Type'] = 'application/x-ndjson',
+ },
+ body = bulk_json,
+ callback = http_callback,
+ task = task,
+ method = 'post',
+ gzip = settings.use_gzip,
+ no_ssl_verify = settings.no_ssl_verify,
+ user = settings.user,
+ password = settings.password,
+ timeout = settings.timeout,
+ })
+end
+
+local function get_general_metadata(task)
+ local r = {}
+ local ip_addr = task:get_ip()
+
+ if ip_addr and ip_addr:is_valid() then
+ r.is_local = ip_addr:is_local()
+ r.ip = tostring(ip_addr)
+ else
+ r.ip = '127.0.0.1'
+ end
+
+ r.webmail = false
+ r.sender_ip = 'unknown'
+ local origin = task:get_header('X-Originating-IP')
+ if origin then
+ origin = origin:gsub('%[', ''):gsub('%]', '')
+ local rspamd_ip = require "rspamd_ip"
+ local origin_ip = rspamd_ip.from_string(origin)
+ if origin_ip and origin_ip:is_valid() then
+ r.webmail = true
+ r.sender_ip = origin -- use string here
+ end
+ end
+
+ r.direction = "Inbound"
+ r.user = task:get_user() or 'unknown'
+ r.qid = task:get_queue_id() or 'unknown'
+ r.action = task:get_metric_action()
+ r.rspamd_server = HOSTNAME
+ if r.user ~= 'unknown' then
+ r.direction = "Outbound"
+ end
+ local s = task:get_metric_score()[1]
+ r.score = s
+
+ local rcpt = task:get_recipients('smtp')
+ if rcpt then
+ local l = {}
+ for _, a in ipairs(rcpt) do
+ table.insert(l, a['addr'])
+ end
+ r.rcpt = l
+ else
+ r.rcpt = 'unknown'
+ end
+
+ local from = task:get_from { 'smtp', 'orig' }
+ if ((from or E)[1] or E).addr then
+ r.from = from[1].addr
+ else
+ r.from = 'unknown'
+ end
+
+ local mime_from = task:get_from { 'mime', 'orig' }
+ if ((mime_from or E)[1] or E).addr then
+ r.mime_from = mime_from[1].addr
+ else
+ r.mime_from = 'unknown'
+ end
+
+ local syminf = task:get_symbols_all()
+ r.symbols = syminf
+ r.asn = {}
+ local pool = task:get_mempool()
+ r.asn.country = pool:get_variable("country") or 'unknown'
+ r.asn.asn = pool:get_variable("asn") or 0
+ r.asn.ipnet = pool:get_variable("ipnet") or 'unknown'
+
+ local function process_header(name)
+ local hdr = task:get_header_full(name)
+ if hdr then
+ local l = {}
+ for _, h in ipairs(hdr) do
+ table.insert(l, h.decoded)
+ end
+ return l
+ else
+ return 'unknown'
+ end
+ end
+
+ r.header_from = process_header('from')
+ r.header_to = process_header('to')
+ r.header_subject = process_header('subject')
+ r.header_date = process_header('date')
+ r.message_id = task:get_message_id()
+ local hname = task:get_hostname() or 'unknown'
+ r.hostname = hname
+
+ local settings_id = task:get_settings_id()
+
+ if settings_id then
+ -- Convert to string
+ settings_id = lua_settings.settings_by_id(settings_id)
+
+ if settings_id then
+ settings_id = settings_id.name
+ end
+ end
+
+ if not settings_id then
+ settings_id = ''
+ end
+
+ r.settings_id = settings_id
+
+ local scan_real = task:get_scan_time()
+ scan_real = math.floor(scan_real * 1000)
+ if scan_real < 0 then
+ rspamd_logger.messagex(task,
+ 'clock skew detected for message: %s ms real scan time (reset to 0)',
+ scan_real)
+ scan_real = 0
+ end
+
+ r.scan_time = scan_real
+
+ return r
+end
+
+local function elastic_collect(task)
+ if not enabled then
+ return
+ end
+ if task:has_flag('skip') then
+ return
+ end
+ if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then
+ return
+ end
+
+ local row = { ['rspamd_meta'] = get_general_metadata(task),
+ ['@timestamp'] = tostring(util.get_time() * 1000) }
+ table.insert(rows, row)
+ nrows = nrows + 1
+ if nrows > settings['limit'] then
+ lua_util.debugm(N, task, 'send elastic search rows: %s', nrows)
+ if elastic_send_data(task) then
+ nrows = 0
+ rows = {}
+ failed_sends = 0;
+ else
+ failed_sends = failed_sends + 1
+
+ if failed_sends > settings.max_fail then
+ rspamd_logger.errx(task, 'cannot send %s rows to ES %s times, stop trying',
+ nrows, failed_sends)
+ nrows = 0
+ rows = {}
+ failed_sends = 0;
+ end
+ end
+ end
+end
+
+local opts = rspamd_config:get_all_opt('elastic')
+
+local function check_elastic_server(cfg, ev_base, _)
+ local upstream = settings.upstream:get_upstream_round_robin()
+ local ip_addr = upstream:get_addr():to_string(true)
+ local plugins_url = connect_prefix .. ip_addr .. '/_nodes/' .. ingest_geoip_type
+ local function http_callback(err, code, body, _)
+ if code == 200 then
+ local parser = ucl.parser()
+ local res, ucl_err = parser:parse_string(body)
+ if not res then
+ rspamd_logger.infox(rspamd_config, 'failed to parse reply from %s: %s',
+ plugins_url, ucl_err)
+ enabled = false;
+ return
+ end
+ local obj = parser:get_object()
+ for node, value in pairs(obj['nodes']) do
+ local plugin_found = false
+ for _, plugin in pairs(value['plugins']) do
+ if plugin['name'] == 'ingest-geoip' then
+ plugin_found = true
+ lua_util.debugm(N, "ingest-geoip plugin has been found")
+ end
+ end
+ if not plugin_found then
+ rspamd_logger.infox(rspamd_config,
+ 'Unable to find ingest-geoip on %1 node, disabling module', node)
+ enabled = false
+ return
+ end
+ end
+ else
+ rspamd_logger.errx('cannot get plugins from %s: %s(%s) (%s)', plugins_url,
+ err, code, body)
+ enabled = false
+ end
+ end
+ rspamd_http.request({
+ url = plugins_url,
+ ev_base = ev_base,
+ config = cfg,
+ method = 'get',
+ callback = http_callback,
+ no_ssl_verify = settings.no_ssl_verify,
+ user = settings.user,
+ password = settings.password,
+ timeout = settings.timeout,
+ })
+end
+
+-- import ingest pipeline and kibana dashboard/visualization
+local function initial_setup(cfg, ev_base, worker)
+ if not worker:is_primary_controller() then
+ return
+ end
+
+ local upstream = settings.upstream:get_upstream_round_robin()
+ local ip_addr = upstream:get_addr():to_string(true)
+
+ local function push_kibana_template()
+ -- add kibana dashboard and visualizations
+ if settings['import_kibana'] then
+ local kibana_mappings = read_file(settings['kibana_file'])
+ if kibana_mappings then
+ local parser = ucl.parser()
+ local res, parser_err = parser:parse_string(kibana_mappings)
+ if not res then
+ rspamd_logger.infox(rspamd_config, 'kibana template cannot be parsed: %s',
+ parser_err)
+ enabled = false
+
+ return
+ end
+ local obj = parser:get_object()
+ local tbl = {}
+ for _, item in ipairs(obj) do
+ table.insert(tbl, '{ "index" : { "_index" : ".kibana", "_type" : "doc" ,"_id": "' ..
+ item['_type'] .. ':' .. item["_id"] .. '"} }')
+ table.insert(tbl, ucl.to_format(item['_source'], 'json-compact'))
+ end
+ table.insert(tbl, '') -- For last \n
+
+ local kibana_url = connect_prefix .. ip_addr .. '/.kibana/_bulk'
+ local function kibana_template_callback(err, code, body, _)
+ if code ~= 200 then
+ rspamd_logger.errx('cannot put template to %s: %s(%s) (%s)', kibana_url,
+ err, code, body)
+ enabled = false
+ else
+ lua_util.debugm(N, 'pushed kibana template: %s', body)
+ end
+ end
+
+ rspamd_http.request({
+ url = kibana_url,
+ ev_base = ev_base,
+ config = cfg,
+ headers = {
+ ['Content-Type'] = 'application/x-ndjson',
+ },
+ body = table.concat(tbl, "\n"),
+ method = 'post',
+ gzip = settings.use_gzip,
+ callback = kibana_template_callback,
+ no_ssl_verify = settings.no_ssl_verify,
+ user = settings.user,
+ password = settings.password,
+ timeout = settings.timeout,
+ })
+ else
+ rspamd_logger.infox(rspamd_config, 'kibana template file %s not found', settings['kibana_file'])
+ end
+ end
+ end
+
+ if enabled then
+ -- create ingest pipeline
+ local geoip_url = connect_prefix .. ip_addr .. '/_ingest/pipeline/rspamd-geoip'
+ local function geoip_cb(err, code, body, _)
+ if code ~= 200 then
+ rspamd_logger.errx('cannot get data from %s: %s(%s) (%s)',
+ geoip_url, err, code, body)
+ enabled = false
+ end
+ end
+ local template = {
+ description = "Add geoip info for rspamd",
+ processors = {
+ {
+ geoip = {
+ field = "rspamd_meta.ip",
+ target_field = "rspamd_meta.geoip"
+ }
+ }
+ }
+ }
+ rspamd_http.request({
+ url = geoip_url,
+ ev_base = ev_base,
+ config = cfg,
+ callback = geoip_cb,
+ headers = {
+ ['Content-Type'] = 'application/json',
+ },
+ gzip = settings.use_gzip,
+ body = ucl.to_format(template, 'json-compact'),
+ method = 'put',
+ no_ssl_verify = settings.no_ssl_verify,
+ user = settings.user,
+ password = settings.password,
+ timeout = settings.timeout,
+ })
+ -- create template mappings if not exist
+ local template_url = connect_prefix .. ip_addr .. '/_template/rspamd'
+ local function http_template_put_callback(err, code, body, _)
+ if code ~= 200 then
+ rspamd_logger.errx('cannot put template to %s: %s(%s) (%s)',
+ template_url, err, code, body)
+ enabled = false
+ else
+ lua_util.debugm(N, 'pushed rspamd template: %s', body)
+ push_kibana_template()
+ end
+ end
+ local function http_template_exist_callback(_, code, _, _)
+ if code ~= 200 then
+ rspamd_http.request({
+ url = template_url,
+ ev_base = ev_base,
+ config = cfg,
+ body = elastic_template,
+ method = 'put',
+ headers = {
+ ['Content-Type'] = 'application/json',
+ },
+ gzip = settings.use_gzip,
+ callback = http_template_put_callback,
+ no_ssl_verify = settings.no_ssl_verify,
+ user = settings.user,
+ password = settings.password,
+ timeout = settings.timeout,
+ })
+ else
+ push_kibana_template()
+ end
+ end
+
+ rspamd_http.request({
+ url = template_url,
+ ev_base = ev_base,
+ config = cfg,
+ method = 'head',
+ callback = http_template_exist_callback,
+ no_ssl_verify = settings.no_ssl_verify,
+ user = settings.user,
+ password = settings.password,
+ timeout = settings.timeout,
+ })
+
+ end
+end
+
+redis_params = rspamd_redis.parse_redis_server('elastic')
+
+if redis_params and opts then
+ for k, v in pairs(opts) do
+ settings[k] = v
+ end
+
+ if not settings['server'] and not settings['servers'] then
+ rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
+ lua_util.disable_module(N, "config")
+ else
+ if settings.use_https then
+ connect_prefix = 'https://'
+ end
+
+ if settings.ingest_module then
+ ingest_geoip_type = 'modules'
+ end
+
+ settings.upstream = upstream_list.create(rspamd_config,
+ settings['server'] or settings['servers'], 9200)
+
+ if not settings.upstream then
+ rspamd_logger.errx('cannot parse elastic address: %s',
+ settings['server'] or settings['servers'])
+ lua_util.disable_module(N, "config")
+ return
+ end
+ if not settings['template_file'] then
+ rspamd_logger.infox(rspamd_config, 'elastic template_file is required, disabling module')
+ lua_util.disable_module(N, "config")
+ return
+ end
+
+ elastic_template = read_file(settings['template_file']);
+ if not elastic_template then
+ rspamd_logger.infox(rspamd_config, 'elastic unable to read %s, disabling module',
+ settings['template_file'])
+ lua_util.disable_module(N, "config")
+ return
+ end
+
+ rspamd_config:register_symbol({
+ name = 'ELASTIC_COLLECT',
+ type = 'idempotent',
+ callback = elastic_collect,
+ flags = 'empty,explicit_disable,ignore_passthrough',
+ augmentations = { string.format("timeout=%f", settings.timeout) },
+ })
+
+ rspamd_config:add_on_load(function(cfg, ev_base, worker)
+ if worker:is_scanner() then
+ check_elastic_server(cfg, ev_base, worker) -- check for elasticsearch requirements
+ initial_setup(cfg, ev_base, worker) -- import mappings pipeline and visualizations
+ end
+ end)
+ end
+
+end
diff --git a/src/plugins/lua/emails.lua b/src/plugins/lua/emails.lua
new file mode 100644
index 0000000..5f25e69
--- /dev/null
+++ b/src/plugins/lua/emails.lua
@@ -0,0 +1,4 @@
+-- This module is deprecated and must not be used.
+-- This file serves as a tombstone to prevent old emails to be loaded
+
+return \ No newline at end of file
diff --git a/src/plugins/lua/external_relay.lua b/src/plugins/lua/external_relay.lua
new file mode 100644
index 0000000..3660f92
--- /dev/null
+++ b/src/plugins/lua/external_relay.lua
@@ -0,0 +1,285 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[
+external_relay plugin - sets IP/hostname from Received headers
+]]--
+
+if confighelp then
+ return
+end
+
+local lua_maps = require "lua_maps"
+local lua_util = require "lua_util"
+local rspamd_logger = require "rspamd_logger"
+local ts = require("tableshape").types
+
+local E = {}
+local N = "external_relay"
+
+local settings = {
+ rules = {},
+}
+
+local config_schema = ts.shape {
+ enabled = ts.boolean:is_optional(),
+ rules = ts.map_of(
+ ts.string, ts.one_of {
+ ts.shape {
+ priority = ts.number:is_optional(),
+ strategy = 'authenticated',
+ symbol = ts.string:is_optional(),
+ user_map = lua_maps.map_schema:is_optional(),
+ },
+ ts.shape {
+ count = ts.number,
+ priority = ts.number:is_optional(),
+ strategy = 'count',
+ symbol = ts.string:is_optional(),
+ },
+ ts.shape {
+ priority = ts.number:is_optional(),
+ strategy = 'local',
+ symbol = ts.string:is_optional(),
+ },
+ ts.shape {
+ hostname_map = lua_maps.map_schema,
+ priority = ts.number:is_optional(),
+ strategy = 'hostname_map',
+ symbol = ts.string:is_optional(),
+ },
+ ts.shape {
+ ip_map = lua_maps.map_schema,
+ priority = ts.number:is_optional(),
+ strategy = 'ip_map',
+ symbol = ts.string:is_optional(),
+ },
+ }
+ ),
+}
+
+local function set_from_rcvd(task, rcvd)
+ local rcvd_ip = rcvd.real_ip
+ if not (rcvd_ip and rcvd_ip:is_valid()) then
+ rspamd_logger.errx(task, 'no IP in header: %s', rcvd)
+ return
+ end
+ task:set_from_ip(rcvd_ip)
+ if rcvd.from_hostname then
+ task:set_hostname(rcvd.from_hostname)
+ task:set_helo(rcvd.from_hostname) -- use fake value for HELO
+ else
+ rspamd_logger.warnx(task, "couldn't get hostname from headers")
+ local ipstr = string.format('[%s]', rcvd_ip)
+ task:set_hostname(ipstr) -- returns nil from task:get_hostname()
+ task:set_helo(ipstr)
+ end
+ return true
+end
+
+local strategies = {}
+
+strategies.authenticated = function(rule)
+ local user_map
+ if rule.user_map then
+ user_map = lua_maps.map_add_from_ucl(rule.user_map, 'set', 'external relay usernames')
+ if not user_map then
+ rspamd_logger.errx(rspamd_config, "couldn't add map %s; won't register symbol %s",
+ rule.user_map, rule.symbol)
+ return
+ end
+ end
+
+ return function(task)
+ local user = task:get_user()
+ if not user then
+ lua_util.debugm(N, task, 'sender is unauthenticated')
+ return
+ end
+ if user_map then
+ if not user_map:get_key(user) then
+ lua_util.debugm(N, task, 'sender (%s) is not in user_map', user)
+ return
+ end
+ end
+
+ local rcvd_hdrs = task:get_received_headers()
+ -- Try find end of authentication chain
+ for _, rcvd in ipairs(rcvd_hdrs) do
+ if not rcvd.flags.authenticated then
+ -- Found unauthenticated hop, use this header
+ return set_from_rcvd(task, rcvd)
+ end
+ end
+
+ rspamd_logger.errx(task, 'found nothing useful in Received headers')
+ end
+end
+
+strategies.count = function(rule)
+ return function(task)
+ local rcvd_hdrs = task:get_received_headers()
+ -- Reduce count by 1 if artificial header is present
+ local hdr_count
+ if ((rcvd_hdrs[1] or E).flags or E).artificial then
+ hdr_count = rule.count - 1
+ else
+ hdr_count = rule.count
+ end
+
+ local rcvd = rcvd_hdrs[hdr_count]
+ if not rcvd then
+ rspamd_logger.errx(task, 'found no received header #%s', hdr_count)
+ return
+ end
+
+ return set_from_rcvd(task, rcvd)
+ end
+end
+
+strategies.hostname_map = function(rule)
+ local hostname_map = lua_maps.map_add_from_ucl(rule.hostname_map, 'map', 'external relay hostnames')
+ if not hostname_map then
+ rspamd_logger.errx(rspamd_config, "couldn't add map %s; won't register symbol %s",
+ rule.hostname_map, rule.symbol)
+ return
+ end
+
+ return function(task)
+ local from_hn = task:get_hostname()
+ if not from_hn then
+ lua_util.debugm(N, task, 'sending hostname is missing')
+ return
+ end
+
+ if not hostname_map:get_key(from_hn) then
+ lua_util.debugm(N, task, 'sender\'s hostname (%s) is not a relay', from_hn)
+ return
+ end
+
+ local rcvd_hdrs = task:get_received_headers()
+ -- Try find sending hostname in Received headers
+ for _, rcvd in ipairs(rcvd_hdrs) do
+ if rcvd.by_hostname == from_hn and rcvd.real_ip then
+ if not hostname_map:get_key(rcvd.from_hostname) then
+ -- Remote hostname is not another relay, use this header
+ return set_from_rcvd(task, rcvd)
+ else
+ -- Keep checking with new hostname
+ from_hn = rcvd.from_hostname
+ end
+ end
+ end
+
+ rspamd_logger.errx(task, 'found nothing useful in Received headers')
+ end
+end
+
+strategies.ip_map = function(rule)
+ local ip_map = lua_maps.map_add_from_ucl(rule.ip_map, 'radix', 'external relay IPs')
+ if not ip_map then
+ rspamd_logger.errx(rspamd_config, "couldn't add map %s; won't register symbol %s",
+ rule.ip_map, rule.symbol)
+ return
+ end
+
+ return function(task)
+ local from_ip = task:get_from_ip()
+ if not (from_ip and from_ip:is_valid()) then
+ lua_util.debugm(N, task, 'sender\'s IP is missing')
+ return
+ end
+
+ if not ip_map:get_key(from_ip) then
+ lua_util.debugm(N, task, 'sender\'s ip (%s) is not a relay', from_ip)
+ return
+ end
+
+ local rcvd_hdrs = task:get_received_headers()
+ local num_rcvd = #rcvd_hdrs
+ -- Try find sending IP in Received headers
+ for i, rcvd in ipairs(rcvd_hdrs) do
+ if rcvd.real_ip then
+ local rcvd_ip = rcvd.real_ip
+ if rcvd_ip:is_valid() and (not ip_map:get_key(rcvd_ip) or i == num_rcvd) then
+ return set_from_rcvd(task, rcvd)
+ end
+ end
+ end
+
+ rspamd_logger.errx(task, 'found nothing useful in Received headers')
+ end
+end
+
+strategies['local'] = function(rule)
+ return function(task)
+ local from_ip = task:get_from_ip()
+ if not from_ip then
+ lua_util.debugm(N, task, 'sending IP is missing')
+ return
+ end
+
+ if not from_ip:is_local() then
+ lua_util.debugm(N, task, 'sending IP (%s) is non-local', from_ip)
+ return
+ end
+
+ local rcvd_hdrs = task:get_received_headers()
+ local num_rcvd = #rcvd_hdrs
+ -- Try find first non-local IP in Received headers
+ for i, rcvd in ipairs(rcvd_hdrs) do
+ if rcvd.real_ip then
+ local rcvd_ip = rcvd.real_ip
+ if rcvd_ip and rcvd_ip:is_valid() and (not rcvd_ip:is_local() or i == num_rcvd) then
+ return set_from_rcvd(task, rcvd)
+ end
+ end
+ end
+
+ rspamd_logger.errx(task, 'found nothing useful in Received headers')
+ end
+end
+
+local opts = rspamd_config:get_all_opt(N)
+if opts then
+ settings = lua_util.override_defaults(settings, opts)
+
+ local ok, schema_err = config_schema:transform(settings)
+ if not ok then
+ rspamd_logger.errx(rspamd_config, 'config schema error: %s', schema_err)
+ lua_util.disable_module(N, "config")
+ return
+ end
+
+ for k, rule in pairs(settings.rules) do
+
+ if not rule.symbol then
+ rule.symbol = k
+ end
+
+ local cb = strategies[rule.strategy](rule)
+
+ if cb then
+ rspamd_config:register_symbol({
+ name = rule.symbol,
+ type = 'prefilter',
+ priority = rule.priority or lua_util.symbols_priorities.top + 1,
+ group = N,
+ callback = cb,
+ })
+ end
+ end
+end
diff --git a/src/plugins/lua/external_services.lua b/src/plugins/lua/external_services.lua
new file mode 100644
index 0000000..e299d9f
--- /dev/null
+++ b/src/plugins/lua/external_services.lua
@@ -0,0 +1,408 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]] --
+
+local rspamd_logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local lua_redis = require "lua_redis"
+local fun = require "fun"
+local lua_scanners = require("lua_scanners").filter('scanner')
+local common = require "lua_scanners/common"
+local redis_params
+
+local N = "external_services"
+
+if confighelp then
+ rspamd_config:add_example(nil, 'external_services',
+ "Check messages using external services (e.g. OEM AS engines, DCC, Pyzor etc)",
+ [[
+ external_services {
+ # multiple scanners could be checked, for each we create a configuration block with an arbitrary name
+
+ oletools {
+ # If set force this action if any virus is found (default unset: no action is forced)
+ # action = "reject";
+ # If set, then rejection message is set to this value (mention single quotes)
+ # If `max_size` is set, messages > n bytes in size are not scanned
+ # max_size = 20000000;
+ # log_clean = true;
+ # servers = "127.0.0.1:10050";
+ # cache_expire = 86400;
+ # scan_mime_parts = true;
+ # extended = false;
+ # if `patterns` is specified virus name will be matched against provided regexes and the related
+ # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+ patterns {
+ # symbol_name = "pattern";
+ JUST_EICAR = "^Eicar-Test-Signature$";
+ }
+ # mime-part regex matching in content-type or filename
+ mime_parts_filter_regex {
+ #GEN1 = "application\/octet-stream";
+ DOC2 = "application\/msword";
+ DOC3 = "application\/vnd\.ms-word.*";
+ XLS = "application\/vnd\.ms-excel.*";
+ PPT = "application\/vnd\.ms-powerpoint.*";
+ GEN2 = "application\/vnd\.openxmlformats-officedocument.*";
+ }
+ # Mime-Part filename extension matching (no regex)
+ mime_parts_filter_ext {
+ doc = "doc";
+ dot = "dot";
+ docx = "docx";
+ dotx = "dotx";
+ docm = "docm";
+ dotm = "dotm";
+ xls = "xls";
+ xlt = "xlt";
+ xla = "xla";
+ xlsx = "xlsx";
+ xltx = "xltx";
+ xlsm = "xlsm";
+ xltm = "xltm";
+ xlam = "xlam";
+ xlsb = "xlsb";
+ ppt = "ppt";
+ pot = "pot";
+ pps = "pps";
+ ppa = "ppa";
+ pptx = "pptx";
+ potx = "potx";
+ ppsx = "ppsx";
+ ppam = "ppam";
+ pptm = "pptm";
+ potm = "potm";
+ ppsm = "ppsm";
+ }
+ # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+ whitelist = "/etc/rspamd/antivirus.wl";
+ }
+ dcc {
+ # If set force this action if any virus is found (default unset: no action is forced)
+ # action = "reject";
+ # If set, then rejection message is set to this value (mention single quotes)
+ # If `max_size` is set, messages > n bytes in size are not scanned
+ max_size = 20000000;
+ #servers = "127.0.0.1:10045;
+ # if `patterns` is specified virus name will be matched against provided regexes and the related
+ # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+ patterns {
+ # symbol_name = "pattern";
+ JUST_EICAR = "^Eicar-Test-Signature$";
+ }
+ # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+ whitelist = "/etc/rspamd/antivirus.wl";
+ }
+ }
+ ]])
+ return
+end
+
+local function add_scanner_rule(sym, opts)
+ if not opts.type then
+ rspamd_logger.errx(rspamd_config, 'unknown type for external scanner rule %s', sym)
+ return nil
+ end
+
+ local cfg = lua_scanners[opts.type]
+
+ if not cfg then
+ rspamd_logger.errx(rspamd_config, 'unknown external scanner type: %s',
+ opts.type)
+ return nil
+ end
+
+ local rule = cfg.configure(opts)
+
+ if not rule then
+ rspamd_logger.errx(rspamd_config, 'cannot configure %s for %s',
+ opts.type, rule.symbol or sym:upper())
+ return nil
+ end
+
+ rule.type = opts.type
+ -- Fill missing symbols
+ if not rule.symbol then
+ rule.symbol = sym:upper()
+ end
+ if not rule.symbol_fail then
+ rule.symbol_fail = rule.symbol .. '_FAIL'
+ end
+ if not rule.symbol_encrypted then
+ rule.symbol_encrypted = rule.symbol .. '_ENCRYPTED'
+ end
+ if not rule.symbol_macro then
+ rule.symbol_macro = rule.symbol .. '_MACRO'
+ end
+
+ rule.redis_params = redis_params
+
+ lua_redis.register_prefix(rule.prefix .. '_*', N,
+ string.format('External services cache for rule "%s"',
+ rule.type), {
+ type = 'string',
+ })
+
+ -- if any mime_part filter defined, do not scan all attachments
+ if opts.mime_parts_filter_regex ~= nil
+ or opts.mime_parts_filter_ext ~= nil then
+ rule.scan_all_mime_parts = false
+ else
+ rule.scan_all_mime_parts = true
+ end
+
+ rule.patterns = common.create_regex_table(opts.patterns or {})
+ rule.patterns_fail = common.create_regex_table(opts.patterns_fail or {})
+
+ rule.mime_parts_filter_regex = common.create_regex_table(opts.mime_parts_filter_regex or {})
+
+ rule.mime_parts_filter_ext = common.create_regex_table(opts.mime_parts_filter_ext or {})
+
+ if opts.whitelist then
+ rule.whitelist = rspamd_config:add_hash_map(opts.whitelist)
+ end
+
+ local function scan_cb(task)
+ if rule.scan_mime_parts then
+
+ fun.each(function(p)
+ local content = p:get_content()
+ if content and #content > 0 then
+ cfg.check(task, content, p:get_digest(), rule, p)
+ end
+ end, common.check_parts_match(task, rule))
+
+ else
+ cfg.check(task, task:get_content(), task:get_digest(), rule, nil)
+ end
+ end
+
+ rspamd_logger.infox(rspamd_config, 'registered external services rule: symbol %s; type %s',
+ rule.symbol, rule.type)
+
+ return scan_cb, rule
+end
+
+-- Registration
+local opts = rspamd_config:get_all_opt(N)
+if opts and type(opts) == 'table' then
+ redis_params = lua_redis.parse_redis_server(N)
+ local has_valid = false
+ for k, m in pairs(opts) do
+ if type(m) == 'table' and m.servers then
+ if not m.type then
+ m.type = k
+ end
+ if not m.name then
+ m.name = k
+ end
+ local cb, nrule = add_scanner_rule(k, m)
+
+ if not cb then
+ rspamd_logger.errx(rspamd_config, 'cannot add rule: "' .. k .. '"')
+ else
+ m = nrule
+
+ local t = {
+ name = m.symbol,
+ callback = cb,
+ score = 0.0,
+ group = N
+ }
+
+ if m.symbol_type == 'postfilter' then
+ t.type = 'postfilter'
+ t.priority = lua_util.symbols_priorities.medium
+ else
+ t.type = 'normal'
+ end
+
+ t.augmentations = {}
+
+ if type(m.timeout) == 'number' then
+ -- Here, we ignore possible DNS timeout and timeout from multiple retries
+ -- as these situations are not usual nor likely for the external_services module
+ table.insert(t.augmentations, string.format("timeout=%f", m.timeout))
+ end
+
+ local id = rspamd_config:register_symbol(t)
+
+ if m.symbol_fail then
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = m['symbol_fail'],
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ end
+
+ if m.symbol_encrypted then
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = m['symbol_encrypted'],
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ end
+ if m.symbol_macro then
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = m['symbol_macro'],
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ end
+ has_valid = true
+ if type(m['patterns']) == 'table' then
+ if m['patterns'][1] then
+ for _, p in ipairs(m['patterns']) do
+ if type(p) == 'table' then
+ for sym in pairs(p) do
+ rspamd_logger.debugm(N, rspamd_config, 'registering: %1', {
+ type = 'virtual',
+ name = sym,
+ parent = m['symbol'],
+ parent_id = id,
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ end
+ end
+ end
+ else
+ for sym in pairs(m['patterns']) do
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ end
+ end
+ end
+ if type(m['patterns_fail']) == 'table' then
+ if m['patterns_fail'][1] then
+ for _, p in ipairs(m['patterns_fail']) do
+ if type(p) == 'table' then
+ for sym in pairs(p) do
+ rspamd_logger.debugm(N, rspamd_config, 'registering: %1', {
+ type = 'virtual',
+ name = sym,
+ parent = m['symbol'],
+ parent_id = id,
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ end
+ end
+ end
+ else
+ for sym in pairs(m['patterns_fail']) do
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ score = 0.0,
+ group = N
+ })
+ end
+ end
+ end
+ if m.symbols then
+ local function reg_symbols(tbl)
+ for _, sym in pairs(tbl) do
+ if type(sym) == 'string' then
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ group = N
+ })
+ elseif type(sym) == 'table' then
+ if sym.symbol then
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym.symbol,
+ parent = id,
+ group = N
+ })
+
+ if sym.score then
+ rspamd_config:set_metric_symbol({
+ name = sym.symbol,
+ score = sym.score,
+ description = sym.description,
+ group = sym.group or N,
+ })
+ end
+ else
+ reg_symbols(sym)
+ end
+ end
+ end
+ end
+
+ reg_symbols(m.symbols)
+ end
+
+ if m['score'] then
+ -- Register metric symbol
+ local description = 'external services symbol'
+ local group = N
+ if m['description'] then
+ description = m['description']
+ end
+ if m['group'] then
+ group = m['group']
+ end
+ rspamd_config:set_metric_symbol({
+ name = m['symbol'],
+ score = m['score'],
+ description = description,
+ group = group
+ })
+ end
+
+ -- Add preloads if a module requires that
+ if type(m.preloads) == 'table' then
+ for _, preload in ipairs(m.preloads) do
+ rspamd_config:add_on_load(function(cfg, ev_base, worker)
+ preload(m, cfg, ev_base, worker)
+ end)
+ end
+ end
+ end
+ end
+ end
+
+ if not has_valid then
+ lua_util.disable_module(N, 'config')
+ end
+end
diff --git a/src/plugins/lua/force_actions.lua b/src/plugins/lua/force_actions.lua
new file mode 100644
index 0000000..4a87cf5
--- /dev/null
+++ b/src/plugins/lua/force_actions.lua
@@ -0,0 +1,227 @@
+--[[
+Copyright (c) 2017, Andrew Lewis <nerf@judo.za.org>
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- A plugin that forces actions
+
+if confighelp then
+ return
+end
+
+local E = {}
+local N = 'force_actions'
+local selector_cache = {}
+
+local fun = require "fun"
+local lua_util = require "lua_util"
+local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash"
+local rspamd_expression = require "rspamd_expression"
+local rspamd_logger = require "rspamd_logger"
+local lua_selectors = require "lua_selectors"
+
+-- Params table fields:
+-- expr, act, pool, message, subject, raction, honor, limit, flags
+local function gen_cb(params)
+
+ local function parse_atom(str)
+ local atom = table.concat(fun.totable(fun.take_while(function(c)
+ if string.find(', \t()><+!|&\n', c, 1, true) then
+ return false
+ end
+ return true
+ end, fun.iter(str))), '')
+ return atom
+ end
+
+ local function process_atom(atom, task)
+ local f_ret = task:has_symbol(atom)
+ if f_ret then
+ f_ret = math.abs(task:get_symbol(atom)[1].score)
+ if f_ret < 0.001 then
+ -- Adjust some low score to distinguish from pure zero
+ f_ret = 0.001
+ end
+ return f_ret
+ end
+ return 0
+ end
+
+ local e, err = rspamd_expression.create(params.expr, { parse_atom, process_atom }, params.pool)
+ if err then
+ rspamd_logger.errx(rspamd_config, 'Couldnt create expression [%1]: %2', params.expr, err)
+ return
+ end
+
+ return function(task)
+
+ local function process_message_selectors(repl, selector_expr)
+ -- create/reuse selector to extract value for this placeholder
+ local selector = selector_cache[selector_expr]
+ if not selector then
+ selector_cache[selector_expr] = lua_selectors.create_selector_closure(rspamd_config, selector_expr, '', true)
+ selector = selector_cache[selector_expr]
+ if not selector then
+ rspamd_logger.errx(task, 'could not create selector [%1]', selector_expr)
+ return "((could not create selector))"
+ end
+ end
+ local extracted = selector(task)
+ if extracted then
+ if type(extracted) == 'table' then
+ extracted = table.concat(extracted, ',')
+ end
+ else
+ rspamd_logger.errx(task, 'could not extract value with selector [%1]', selector_expr)
+ extracted = '((error extracting value))'
+ end
+ return extracted
+ end
+
+ local cact = task:get_metric_action()
+ if not params.message and not params.subject and params.act and cact == params.act then
+ return false
+ end
+ if params.honor and params.honor[cact] then
+ return false
+ elseif params.raction and not params.raction[cact] then
+ return false
+ end
+
+ local ret = e:process(task)
+ lua_util.debugm(N, task, "expression %s returned %s", params.expr, ret)
+ if (not params.limit and ret > 0) or (ret > (params.limit or 0)) then
+ if params.subject then
+ task:set_metric_subject(params.subject)
+ end
+
+ local flags = params.flags or ""
+
+ if type(params.message) == 'string' then
+ -- process selector expressions in the message
+ local message = string.gsub(params.message, '(${(.-)})', process_message_selectors)
+ task:set_pre_result { action = params.act, message = message, module = N, flags = flags }
+ else
+ task:set_pre_result { action = params.act, module = N, flags = flags }
+ end
+ return true, params.act
+ end
+
+ end, e:atoms()
+
+end
+
+local function configure_module()
+ local opts = rspamd_config:get_all_opt(N)
+ if not opts then
+ return false
+ end
+ if type(opts.actions) == 'table' then
+ rspamd_logger.warnx(rspamd_config, 'Processing legacy config')
+ for action, expressions in pairs(opts.actions) do
+ if type(expressions) == 'table' then
+ for _, expr in ipairs(expressions) do
+ local message, subject
+ if type(expr) == 'table' then
+ subject = expr[3]
+ message = expr[2]
+ expr = expr[1]
+ else
+ message = (opts.messages or E)[expr]
+ end
+ if type(expr) == 'string' then
+ -- expr, act, pool, message, subject, raction, honor, limit, flags
+ local cb, atoms = gen_cb { expr = expr,
+ act = action,
+ pool = rspamd_config:get_mempool(),
+ message = message,
+ subject = subject }
+ if cb and atoms then
+ local h = rspamd_cryptobox_hash.create()
+ h:update(expr)
+ local name = 'FORCE_ACTION_' .. string.upper(string.sub(h:hex(), 1, 12))
+ rspamd_config:register_symbol({
+ type = 'normal',
+ name = name,
+ callback = cb,
+ flags = 'empty',
+ group = N,
+ })
+ for _, a in ipairs(atoms) do
+ rspamd_config:register_dependency(name, a)
+ end
+ rspamd_logger.infox(rspamd_config, 'Registered symbol %1 <%2> with dependencies [%3]',
+ name, expr, table.concat(atoms, ','))
+ end
+ end
+ end
+ end
+ end
+ elseif type(opts.rules) == 'table' then
+ for name, sett in pairs(opts.rules) do
+ local action = sett.action
+ local expr = sett.expression
+
+ if action and expr then
+ local flags = {}
+ if sett.least then
+ table.insert(flags, "least")
+ end
+ if sett.process_all then
+ table.insert(flags, "process_all")
+ end
+ local raction = lua_util.list_to_hash(sett.require_action)
+ local honor = lua_util.list_to_hash(sett.honor_action)
+ local cb, atoms = gen_cb { expr = expr,
+ act = action,
+ pool = rspamd_config:get_mempool(),
+ message = sett.message,
+ subject = sett.subject,
+ raction = raction,
+ honor = honor,
+ limit = sett.limit,
+ flags = table.concat(flags, ',') }
+ if cb and atoms then
+ local t = {}
+ if (raction or honor) then
+ t.type = 'postfilter'
+ t.priority = lua_util.symbols_priorities.high
+ else
+ t.type = 'normal'
+ if not sett.least then
+ t.augmentations = { 'passthrough', 'important' }
+ end
+ end
+ t.name = 'FORCE_ACTION_' .. name
+ t.callback = cb
+ t.flags = 'empty, ignore_passthrough'
+ t.group = N
+ rspamd_config:register_symbol(t)
+ if t.type == 'normal' then
+ for _, a in ipairs(atoms) do
+ rspamd_config:register_dependency(t.name, a)
+ end
+ rspamd_logger.infox(rspamd_config, 'Registered symbol %1 <%2> with dependencies [%3]',
+ t.name, expr, table.concat(atoms, ','))
+ else
+ rspamd_logger.infox(rspamd_config, 'Registered symbol %1 <%2> as postfilter', t.name, expr)
+ end
+ end
+ end
+ end
+ end
+end
+
+configure_module()
diff --git a/src/plugins/lua/forged_recipients.lua b/src/plugins/lua/forged_recipients.lua
new file mode 100644
index 0000000..0d51db3
--- /dev/null
+++ b/src/plugins/lua/forged_recipients.lua
@@ -0,0 +1,183 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Plugin for comparing smtp dialog recipients and sender with recipients and sender
+-- in mime headers
+
+if confighelp then
+ rspamd_config:add_example(nil, 'forged_recipients',
+ "Check forged recipients and senders (e.g. mime and smtp recipients mismatch)",
+ [[
+ forged_recipients {
+ symbol_sender = "FORGED_SENDER"; # Symbol for a forged sender
+ symbol_rcpt = "FORGED_RECIPIENTS"; # Symbol for a forged recipients
+ }
+ ]])
+end
+
+local symbol_rcpt = 'FORGED_RECIPIENTS'
+local symbol_sender = 'FORGED_SENDER'
+local rspamd_util = require "rspamd_util"
+
+local E = {}
+
+local function check_forged_headers(task)
+ local auser = task:get_user()
+ local delivered_to = task:get_header('Delivered-To')
+ local smtp_rcpts = task:get_recipients(1)
+ local smtp_from = task:get_from(1)
+
+ if not smtp_rcpts then
+ return
+ end
+ if #smtp_rcpts == 0 then
+ return
+ end
+
+ local mime_rcpts = task:get_recipients({ 'mime', 'orig' })
+
+ if not mime_rcpts then
+ return
+ elseif #mime_rcpts == 0 then
+ return
+ end
+
+ -- Find pair for each smtp recipient in To or Cc headers
+ if #smtp_rcpts > 100 or #mime_rcpts > 100 then
+ -- Trim array, suggested by Anton Yuzhaninov
+ smtp_rcpts[100] = nil
+ mime_rcpts[100] = nil
+ end
+
+ -- map smtp recipient domains to a list of addresses for this domain
+ local smtp_rcpt_domain_map = {}
+ local smtp_rcpt_map = {}
+ for _, smtp_rcpt in ipairs(smtp_rcpts) do
+ local addr = smtp_rcpt.addr
+
+ if addr and addr ~= '' then
+ local dom = string.lower(smtp_rcpt.domain)
+ addr = addr:lower()
+
+ local dom_map = smtp_rcpt_domain_map[dom]
+ if not dom_map then
+ dom_map = {}
+ smtp_rcpt_domain_map[dom] = dom_map
+ end
+
+ dom_map[addr] = smtp_rcpt
+ smtp_rcpt_map[addr] = smtp_rcpt
+
+ if auser and auser == addr then
+ smtp_rcpt.matched = true
+ end
+ if ((smtp_from or E)[1] or E).addr and
+ smtp_from[1]['addr'] == addr then
+ -- allow sender to BCC themselves
+ smtp_rcpt.matched = true
+ end
+ end
+ end
+
+ for _, mime_rcpt in ipairs(mime_rcpts) do
+ if mime_rcpt.addr and mime_rcpt.addr ~= '' then
+ local addr = string.lower(mime_rcpt.addr)
+ local dom = string.lower(mime_rcpt.domain)
+ local matched_smtp_addr = smtp_rcpt_map[addr]
+ if matched_smtp_addr then
+ -- Direct match, go forward
+ matched_smtp_addr.matched = true
+ mime_rcpt.matched = true
+ elseif delivered_to and delivered_to == addr then
+ mime_rcpt.matched = true
+ elseif auser and auser == addr then
+ -- allow user to BCC themselves
+ mime_rcpt.matched = true
+ else
+ local matched_smtp_domain = smtp_rcpt_domain_map[dom]
+
+ if matched_smtp_domain then
+ -- Same domain but another user, it is likely okay due to aliases substitution
+ mime_rcpt.matched = true
+ -- Special field
+ matched_smtp_domain._seen_mime_domain = true
+ end
+ end
+ end
+ end
+
+ -- Now go through all lists one more time and find unmatched stuff
+ local opts = {}
+ local seen_mime_unmatched = false
+ local seen_smtp_unmatched = false
+ for _, mime_rcpt in ipairs(mime_rcpts) do
+ if not mime_rcpt.matched then
+ seen_mime_unmatched = true
+ table.insert(opts, 'm:' .. mime_rcpt.addr)
+ end
+ end
+ for _, smtp_rcpt in ipairs(smtp_rcpts) do
+ if not smtp_rcpt.matched then
+ if not smtp_rcpt_domain_map[smtp_rcpt.domain:lower()]._seen_mime_domain then
+ seen_smtp_unmatched = true
+ table.insert(opts, 's:' .. smtp_rcpt.addr)
+ end
+ end
+ end
+
+ if seen_smtp_unmatched and seen_mime_unmatched then
+ task:insert_result(symbol_rcpt, 1.0, opts)
+ end
+
+ -- Check sender
+ if smtp_from and smtp_from[1] and smtp_from[1]['addr'] ~= '' then
+ local mime_from = task:get_from(2)
+ if not mime_from or not mime_from[1] or
+ not rspamd_util.strequal_caseless_utf8(mime_from[1]['addr'], smtp_from[1]['addr']) then
+ task:insert_result(symbol_sender, 1, ((mime_from or E)[1] or E).addr or '', smtp_from[1].addr)
+ end
+ end
+end
+
+-- Configuration
+local opts = rspamd_config:get_all_opt('forged_recipients')
+if opts then
+ if opts['symbol_rcpt'] or opts['symbol_sender'] then
+ local id = rspamd_config:register_symbol({
+ name = 'FORGED_CALLBACK',
+ callback = check_forged_headers,
+ type = 'callback',
+ group = 'headers',
+ score = 0.0,
+ })
+ if opts['symbol_rcpt'] then
+ symbol_rcpt = opts['symbol_rcpt']
+ rspamd_config:register_symbol({
+ name = symbol_rcpt,
+ type = 'virtual',
+ parent = id,
+ })
+ end
+ if opts['symbol_sender'] then
+ symbol_sender = opts['symbol_sender']
+ rspamd_config:register_symbol({
+ name = symbol_sender,
+ type = 'virtual',
+ parent = id,
+ })
+ end
+ end
+end
diff --git a/src/plugins/lua/fuzzy_collect.lua b/src/plugins/lua/fuzzy_collect.lua
new file mode 100644
index 0000000..132ace9
--- /dev/null
+++ b/src/plugins/lua/fuzzy_collect.lua
@@ -0,0 +1,193 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]] --
+
+if confighelp then
+ return
+end
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local rspamd_http = require "rspamd_http"
+local rspamd_keypairlib = require "rspamd_cryptobox_keypair"
+local rspamd_cryptolib = require "rspamd_cryptobox"
+local fun = require "fun"
+
+local settings = {
+ sync_time = 60.0,
+ saved_cookie = '',
+ timeout = 10.0,
+}
+
+local function send_data_mirror(m, cfg, ev_base, body)
+ local function store_callback(err, _, _, _)
+ if err then
+ rspamd_logger.errx(cfg, 'cannot save data on %(%s): %s', m.server, m.name, err)
+ else
+ rspamd_logger.infox(cfg, 'saved data on %s(%s)', m.server, m.name)
+ end
+ end
+ rspamd_http.request {
+ url = string.format('http://%s//update_v1/%s', m.server, m.name),
+ resolver = cfg:get_resolver(),
+ config = cfg,
+ ev_base = ev_base,
+ timeout = settings.timeout,
+ callback = store_callback,
+ body = body,
+ peer_key = m.pubkey,
+ keypair = m.keypair,
+ }
+end
+
+local function collect_fuzzy_hashes(cfg, ev_base)
+ local function data_callback(err, _, body, _)
+ if not body or err then
+ rspamd_logger.errx(cfg, 'cannot load data: %s', err)
+ else
+ -- Here, we actually copy body once for each mirror
+ fun.each(function(_, v)
+ send_data_mirror(v, cfg, ev_base, body)
+ end,
+ settings.mirrors)
+ end
+ end
+
+ local function cookie_callback(err, _, body, _)
+ if not body or err then
+ rspamd_logger.errx(cfg, 'cannot load cookie: %s', err)
+ else
+ if settings.saved_cookie ~= tostring(body) then
+ settings.saved_cookie = tostring(body)
+ rspamd_logger.infox(cfg, 'received collection cookie %s',
+ tostring(rspamd_util.encode_base32(settings.saved_cookie:sub(1, 6))))
+ local sig = rspamd_cryptolib.sign_memory(settings.sign_keypair,
+ settings.saved_cookie)
+ if not sig then
+ rspamd_logger.info(cfg, 'cannot sign cookie')
+ else
+ rspamd_http.request {
+ url = string.format('http://%s/data', settings.collect_server),
+ resolver = cfg:get_resolver(),
+ config = cfg,
+ ev_base = ev_base,
+ timeout = settings.timeout,
+ callback = data_callback,
+ peer_key = settings.collect_pubkey,
+ headers = {
+ Signature = sig:hex()
+ },
+ opaque_body = true,
+ }
+ end
+ else
+ rspamd_logger.info(cfg, 'cookie has not changed, do not update')
+ end
+ end
+ end
+ rspamd_logger.infox(cfg, 'start fuzzy collection, next sync in %s seconds',
+ settings.sync_time)
+ rspamd_http.request {
+ url = string.format('http://%s/cookie', settings.collect_server),
+ resolver = cfg:get_resolver(),
+ config = cfg,
+ ev_base = ev_base,
+ timeout = settings.timeout,
+ callback = cookie_callback,
+ peer_key = settings.collect_pubkey,
+ }
+
+ return settings.sync_time
+end
+
+local function test_mirror_config(k, m)
+ if not m.server then
+ rspamd_logger.errx(rspamd_config, 'server is missing for the mirror')
+ return false
+ end
+
+ if not m.pubkey then
+ rspamd_logger.errx(rspamd_config, 'pubkey is missing for the mirror')
+ return false
+ end
+
+ if type(k) ~= 'string' and not m.name then
+ rspamd_logger.errx(rspamd_config, 'name is missing for the mirror')
+ return false
+ end
+
+ if not m.keypair then
+ rspamd_logger.errx(rspamd_config, 'keypair is missing for the mirror')
+ return false
+ end
+
+ if not m.name then
+ m.name = k
+ end
+
+ return true
+end
+
+local opts = rspamd_config:get_all_opt('fuzzy_collect')
+
+if opts and type(opts) == 'table' then
+ for k, v in pairs(opts) do
+ settings[k] = v
+ end
+ local sane_config = true
+
+ if not settings['sign_keypair'] then
+ rspamd_logger.errx(rspamd_config, 'sign_keypair is missing')
+ sane_config = false
+ end
+
+ settings['sign_keypair'] = rspamd_keypairlib.create(settings['sign_keypair'])
+ if not settings['sign_keypair'] then
+ rspamd_logger.errx(rspamd_config, 'sign_keypair is invalid')
+ sane_config = false
+ end
+
+ if not settings['collect_server'] then
+ rspamd_logger.errx(rspamd_config, 'collect_server is missing')
+ sane_config = false
+ end
+
+ if not settings['collect_pubkey'] then
+ rspamd_logger.errx(rspamd_config, 'collect_pubkey is missing')
+ sane_config = false
+ end
+
+ if not settings['mirrors'] then
+ rspamd_logger.errx(rspamd_config, 'collect_pubkey is missing')
+ sane_config = false
+ end
+
+ if not fun.all(test_mirror_config, settings['mirrors']) then
+ sane_config = false
+ end
+
+ if sane_config then
+ rspamd_config:add_on_load(function(_, ev_base, worker)
+ if worker:is_primary_controller() then
+ rspamd_config:add_periodic(ev_base, 0.0,
+ function(cfg, _)
+ return collect_fuzzy_hashes(cfg, ev_base)
+ end)
+ end
+ end)
+ else
+ rspamd_logger.errx(rspamd_config, 'module is not configured properly')
+ end
+end
diff --git a/src/plugins/lua/greylist.lua b/src/plugins/lua/greylist.lua
new file mode 100644
index 0000000..6e221b3
--- /dev/null
+++ b/src/plugins/lua/greylist.lua
@@ -0,0 +1,542 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2016, Alexey Savelyev <info@homeweb.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[
+Example domains whitelist config:
+greylist {
+ # Search "example.com" and "mail.example.com" for "mx.out.mail.example.com":
+ whitelist_domains_url = [
+ "$LOCAL_CONFDIR/local.d/maps.d/greylist-whitelist-domains.inc",
+ "${CONFDIR}/maps.d/maillist.inc",
+ "${CONFDIR}/maps.d/redirectors.inc",
+ "${CONFDIR}/maps.d/dmarc_whitelist.inc",
+ "${CONFDIR}/maps.d/spf_dkim_whitelist.inc",
+ "${CONFDIR}/maps.d/surbl-whitelist.inc",
+ "https://maps.rspamd.com/freemail/free.txt.zst"
+ ];
+}
+Example config for exim users:
+greylist {
+ action = "greylist";
+}
+--]]
+
+if confighelp then
+ rspamd_config:add_example(nil, 'greylist',
+ "Performs adaptive greylisting using Redis",
+ [[
+greylist {
+ # Buckets expire (1 day by default)
+ expire = 1d;
+ # Greylisting timeout
+ timeout = 5m;
+ # Redis prefix
+ key_prefix = 'rg';
+ # Use body hash up to this value of bytes for greylisting
+ max_data_len = 10k;
+ # Default greylisting message
+ message = 'Try again later';
+ # Append symbol on greylisting
+ symbol = 'GREYLIST';
+ # Default action change (for Exim use `greylist`)
+ action = 'soft reject';
+ # Skip greylisting if one of the following symbols has been found
+ whitelist_symbols = [];
+ # Mask bits for ipv4
+ ipv4_mask = 19;
+ # Mask bits for ipv6
+ ipv6_mask = 64;
+ # Tell when greylisting is expired (appended to `message`)
+ report_time = false;
+ # Greylist local messages
+ check_local = false;
+ # Greylist messages from authenticated users
+ check_authed = false;
+}
+ ]])
+ return
+end
+
+-- A plugin that implements greylisting using redis
+
+local redis_params
+local whitelisted_ip
+local whitelist_domains_map
+local toint = math.ifloor or math.floor
+local settings = {
+ expire = 86400, -- 1 day by default
+ timeout = 300, -- 5 minutes by default
+ key_prefix = 'rg', -- default hash name
+ max_data_len = 10240, -- default data limit to hash
+ message = 'Try again later', -- default greylisted message
+ symbol = 'GREYLIST',
+ action = 'soft reject', -- default greylisted action
+ whitelist_symbols = {}, -- whitelist when specific symbols have been found
+ ipv4_mask = 19, -- Mask bits for ipv4
+ ipv6_mask = 64, -- Mask bits for ipv6
+ report_time = false, -- Tell when greylisting is expired (appended to `message`)
+ check_local = false,
+ check_authed = false,
+}
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local lua_redis = require "lua_redis"
+local lua_util = require "lua_util"
+local fun = require "fun"
+local hash = require "rspamd_cryptobox_hash"
+local rspamd_lua_utils = require "lua_util"
+local lua_map = require "lua_maps"
+local N = "greylist"
+
+local function data_key(task)
+ local cached = task:get_mempool():get_variable("grey_bodyhash")
+ if cached then
+ return cached
+ end
+
+ local body = task:get_rawbody()
+
+ if not body then
+ return nil
+ end
+
+ local len = body:len()
+ if len > settings['max_data_len'] then
+ len = settings['max_data_len']
+ end
+
+ local h = hash.create()
+ h:update(body, len)
+
+ local b32 = settings['key_prefix'] .. 'b' .. h:base32():sub(1, 20)
+ task:get_mempool():set_variable("grey_bodyhash", b32)
+ return b32
+end
+
+local function envelope_key(task)
+ local cached = task:get_mempool():get_variable("grey_metahash")
+ if cached then
+ return cached
+ end
+
+ local from = task:get_from('smtp')
+ local h = hash.create()
+
+ local addr = '<>'
+ if from and from[1] then
+ addr = from[1]['addr']
+ end
+
+ h:update(addr)
+ local rcpt = task:get_recipients('smtp')
+ if rcpt then
+ table.sort(rcpt, function(r1, r2)
+ return r1['addr'] < r2['addr']
+ end)
+
+ fun.each(function(r)
+ h:update(r['addr'])
+ end, rcpt)
+ end
+
+ local ip = task:get_ip()
+
+ if ip and ip:is_valid() then
+ local s
+ if ip:get_version() == 4 then
+ s = tostring(ip:apply_mask(settings['ipv4_mask']))
+ else
+ s = tostring(ip:apply_mask(settings['ipv6_mask']))
+ end
+ h:update(s)
+ end
+
+ local b32 = settings['key_prefix'] .. 'm' .. h:base32():sub(1, 20)
+ task:get_mempool():set_variable("grey_metahash", b32)
+ return b32
+end
+
+-- Returns pair of booleans: found,greylisted
+local function check_time(task, tm, type, now)
+ local t = tonumber(tm)
+
+ if not t then
+ rspamd_logger.errx(task, 'not a valid number: %s', tm)
+ return false, false
+ end
+
+ if now - t < settings['timeout'] then
+ return true, true
+ else
+ -- We just set variable to pass when in post-filter stage
+ task:get_mempool():set_variable("grey_whitelisted", type)
+
+ return true, false
+ end
+end
+
+local function greylist_message(task, end_time, why)
+ task:insert_result(settings['symbol'], 0.0, 'greylisted', end_time, why)
+
+ if not settings.check_local and rspamd_lua_utils.is_rspamc_or_controller(task) then
+ return
+ end
+
+ if settings.message_func then
+ task:set_pre_result(settings['action'],
+ settings.message_func(task, end_time), N)
+ else
+ local message = settings['message']
+ if settings.report_time then
+ message = string.format("%s: %s", message, end_time)
+ end
+ task:set_pre_result(settings['action'], message, N)
+ end
+
+ task:set_flag('greylisted')
+end
+
+local function greylist_check(task)
+ local ip = task:get_ip()
+
+ if ((not settings.check_authed and task:get_user()) or
+ (not settings.check_local and ip and ip:is_local())) then
+ rspamd_logger.infox(task, "skip greylisting for local networks and/or authorized users");
+ return
+ end
+
+ if ip and ip:is_valid() and whitelisted_ip then
+ if whitelisted_ip:get_key(ip) then
+ -- Do not check whitelisted ip
+ rspamd_logger.infox(task, 'skip greylisting for whitelisted IP')
+ return
+ end
+ end
+
+ local body_key = data_key(task)
+ local meta_key = envelope_key(task)
+ local hash_key = body_key .. meta_key
+
+ local function redis_get_cb(err, data)
+ local ret_body = false
+ local greylisted_body = false
+ local ret_meta = false
+ local greylisted_meta = false
+
+ if data then
+ local end_time_body, end_time_meta
+ local now = rspamd_util.get_time()
+
+ if data[1] and type(data[1]) ~= 'userdata' then
+ local tm = tonumber(data[1]) or now
+ ret_body, greylisted_body = check_time(task, data[1], 'body', now)
+ if greylisted_body then
+ end_time_body = tm + settings['timeout']
+ task:get_mempool():set_variable("grey_greylisted_body",
+ rspamd_util.time_to_string(end_time_body))
+ end
+ end
+
+ if data[2] and type(data[2]) ~= 'userdata' then
+ if not ret_body or greylisted_body then
+ local tm = tonumber(data[2]) or now
+ ret_meta, greylisted_meta = check_time(task, data[2], 'meta', now)
+
+ if greylisted_meta then
+ end_time_meta = tm + settings['timeout']
+ task:get_mempool():set_variable("grey_greylisted_meta",
+ rspamd_util.time_to_string(end_time_meta))
+ end
+ end
+ end
+
+ local how
+ local end_time_str
+
+ if not ret_body and not ret_meta then
+ -- no record found
+ task:get_mempool():set_variable("grey_greylisted", 'true')
+ elseif greylisted_body and greylisted_meta then
+ end_time_str = rspamd_util.time_to_string(
+ math.min(end_time_body, end_time_meta))
+ how = 'meta and body'
+ elseif greylisted_body then
+ end_time_str = rspamd_util.time_to_string(end_time_body)
+ how = 'body only'
+ elseif greylisted_meta then
+ end_time_str = rspamd_util.time_to_string(end_time_meta)
+ how = 'meta only'
+ end
+
+ if how and end_time_str then
+ rspamd_logger.infox(task, 'greylisted until "%s" (%s)',
+ end_time_str, how)
+ greylist_message(task, end_time_str, 'too early')
+ end
+ elseif err then
+ rspamd_logger.errx(task, 'got error while getting greylisting keys: %1', err)
+ return
+ end
+ end
+
+ local ret = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ hash_key, -- hash key
+ false, -- is write
+ redis_get_cb, --callback
+ 'MGET', -- command
+ { body_key, meta_key } -- arguments
+ )
+ if not ret then
+ rspamd_logger.errx(task, 'cannot make redis request to check results')
+ end
+end
+
+local function greylist_set(task)
+ local action = task:get_metric_action()
+ local ip = task:get_ip()
+
+ -- Don't do anything if pre-result has been already set
+ if task:has_pre_result() then
+ return
+ end
+
+ -- Check whitelist_symbols
+ for _, sym in ipairs(settings.whitelist_symbols) do
+ if task:has_symbol(sym) then
+ rspamd_logger.infox(task, 'skip greylisting as we have found symbol %s', sym)
+ if action == 'greylist' then
+ -- We are going to accept message
+ rspamd_logger.infox(task, 'downgrading metric action from "greylist" to "no action"')
+ task:disable_action('greylist')
+ end
+ return
+ end
+ end
+
+ if settings.greylist_min_score then
+ local score = task:get_metric_score('default')[1]
+ if score < settings.greylist_min_score then
+ rspamd_logger.infox(task, 'Score too low - skip greylisting')
+ if action == 'greylist' then
+ -- We are going to accept message
+ rspamd_logger.infox(task, 'Downgrading metric action from "greylist" to "no action"')
+ task:disable_action('greylist')
+ end
+ return
+ end
+ end
+
+ if ((not settings.check_authed and task:get_user()) or
+ (not settings.check_local and ip and ip:is_local())) then
+ if action == 'greylist' then
+ -- We are going to accept message
+ rspamd_logger.infox(task, 'Downgrading metric action from "greylist" to "no action"')
+ task:disable_action('greylist')
+ end
+ return
+ end
+
+ if ip and ip:is_valid() and whitelisted_ip then
+ if whitelisted_ip:get_key(ip) then
+ if action == 'greylist' then
+ -- We are going to accept message
+ rspamd_logger.infox(task, 'Downgrading metric action from "greylist" to "no action"')
+ task:disable_action('greylist')
+ end
+ return
+ end
+ end
+
+ local is_whitelisted = task:get_mempool():get_variable("grey_whitelisted")
+ local do_greylisting = task:get_mempool():get_variable("grey_greylisted")
+ local do_greylisting_required = task:get_mempool():get_variable("grey_greylisted_required")
+
+ -- Third and second level domains whitelist
+ if not is_whitelisted and whitelist_domains_map then
+ local hostname = task:get_hostname()
+ if hostname then
+ local domain = rspamd_util.get_tld(hostname)
+ if whitelist_domains_map:get_key(hostname) or (domain and whitelist_domains_map:get_key(domain)) then
+ is_whitelisted = 'meta'
+ rspamd_logger.infox(task, 'skip greylisting for whitelisted domain')
+ end
+ end
+ end
+
+ if action == 'reject' or
+ not do_greylisting_required and action == 'no action' then
+ return
+ end
+ local body_key = data_key(task)
+ local meta_key = envelope_key(task)
+ local upstream, ret, conn
+ local hash_key = body_key .. meta_key
+
+ local function redis_set_cb(err)
+ if err then
+ rspamd_logger.errx(task, 'got error %s when setting greylisting record on server %s',
+ err, upstream:get_addr())
+ end
+ end
+
+ local is_rspamc = rspamd_lua_utils.is_rspamc_or_controller(task)
+
+ if is_whitelisted then
+ if action == 'greylist' then
+ -- We are going to accept message
+ rspamd_logger.infox(task, 'Downgrading metric action from "greylist" to "no action"')
+ task:disable_action('greylist')
+ end
+
+ task:insert_result(settings['symbol'], 0.0, 'pass', is_whitelisted)
+ rspamd_logger.infox(task, 'greylisting pass (%s) until %s',
+ is_whitelisted,
+ rspamd_util.time_to_string(rspamd_util.get_time() + settings['expire']))
+
+ if not settings.check_local and is_rspamc then
+ return
+ end
+
+ ret, conn, upstream = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ hash_key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'EXPIRE', -- command
+ { body_key, tostring(toint(settings['expire'])) } -- arguments
+ )
+ -- Update greylisting record expire
+ if ret then
+ conn:add_cmd('EXPIRE', {
+ meta_key, tostring(toint(settings['expire']))
+ })
+ else
+ rspamd_logger.errx(task, 'got error while connecting to redis')
+ end
+ elseif do_greylisting or do_greylisting_required then
+ if not settings.check_local and is_rspamc then
+ return
+ end
+ local t = tostring(toint(rspamd_util.get_time()))
+ local end_time = rspamd_util.time_to_string(t + settings['timeout'])
+ rspamd_logger.infox(task, 'greylisted until "%s", new record', end_time)
+ greylist_message(task, end_time, 'new record')
+ -- Create new record
+ ret, conn, upstream = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ hash_key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'SETEX', -- command
+ { body_key, tostring(toint(settings['expire'])), t } -- arguments
+ )
+
+ if ret then
+ conn:add_cmd('SETEX', {
+ meta_key, tostring(toint(settings['expire'])), t
+ })
+ else
+ rspamd_logger.errx(task, 'got error while connecting to redis')
+ end
+ else
+ if action ~= 'no action' and action ~= 'reject' then
+ local grey_res = task:get_mempool():get_variable("grey_greylisted_body")
+
+ if grey_res then
+ -- We need to delay message, hence set a temporary result
+ rspamd_logger.infox(task, 'greylisting delayed until "%s": body', grey_res)
+ greylist_message(task, grey_res, 'body')
+ else
+ grey_res = task:get_mempool():get_variable("grey_greylisted_meta")
+ if grey_res then
+ greylist_message(task, grey_res, 'meta')
+ end
+ end
+ else
+ task:insert_result(settings['symbol'], 0.0, 'greylisted', 'passed')
+ end
+ end
+end
+
+local opts = rspamd_config:get_all_opt('greylist')
+if opts then
+ if opts['message_func'] then
+ settings.message_func = assert(load(opts['message_func']))()
+ end
+
+ for k, v in pairs(opts) do
+ if k ~= 'message_func' then
+ settings[k] = v
+ end
+ end
+
+ local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N,
+ false, false)
+ settings.check_local = auth_and_local_conf[1]
+ settings.check_authed = auth_and_local_conf[2]
+
+ if settings['greylist_min_score'] then
+ settings['greylist_min_score'] = tonumber(settings['greylist_min_score'])
+ else
+ local greylist_threshold = rspamd_config:get_metric_action('greylist')
+ if greylist_threshold then
+ settings['greylist_min_score'] = greylist_threshold
+ end
+ end
+
+ whitelisted_ip = lua_map.rspamd_map_add(N, 'whitelisted_ip', 'radix',
+ 'Greylist whitelist ip map')
+ whitelist_domains_map = lua_map.rspamd_map_add(N, 'whitelist_domains_url',
+ 'map', 'Greylist whitelist domains map')
+
+ redis_params = lua_redis.parse_redis_server(N)
+ if not redis_params then
+ rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
+ rspamd_lua_utils.disable_module(N, "redis")
+ else
+ lua_redis.register_prefix(settings.key_prefix .. 'b[a-z0-9]{20}', N,
+ 'Greylisting elements (body hashes)"', {
+ type = 'string',
+ })
+ lua_redis.register_prefix(settings.key_prefix .. 'm[a-z0-9]{20}', N,
+ 'Greylisting elements (meta hashes)"', {
+ type = 'string',
+ })
+ rspamd_config:register_symbol({
+ name = 'GREYLIST_SAVE',
+ type = 'postfilter',
+ callback = greylist_set,
+ priority = lua_util.symbols_priorities.medium,
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
+ })
+ local id = rspamd_config:register_symbol({
+ name = 'GREYLIST_CHECK',
+ type = 'prefilter',
+ callback = greylist_check,
+ priority = lua_util.symbols_priorities.medium,
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }
+ })
+ rspamd_config:register_symbol({
+ name = settings.symbol,
+ type = 'virtual',
+ parent = id,
+ score = 0,
+ })
+ end
+end
diff --git a/src/plugins/lua/hfilter.lua b/src/plugins/lua/hfilter.lua
new file mode 100644
index 0000000..8c132f5
--- /dev/null
+++ b/src/plugins/lua/hfilter.lua
@@ -0,0 +1,622 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2013-2015, Alexey Savelyev <info@homeweb.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+
+-- Weight for checks_hellohost and checks_hello: 5 - very hard, 4 - hard, 3 - medium, 2 - low, 1 - very low.
+-- From HFILTER_HELO_* and HFILTER_HOSTNAME_* symbols the maximum weight is selected in case of their actuating.
+
+if confighelp then
+ return
+end
+
+local rspamd_regexp = require "rspamd_regexp"
+local lua_util = require "lua_util"
+local rspamc_local_helo = "rspamc.local"
+local checks_hellohost = [[
+/[-.0-9][0-9][.-]?nat/i 5
+/homeuser[.-][0-9]/i 5
+/[-.0-9][0-9][.-]?unused-addr/i 3
+/[-.0-9][0-9][.-]?pppoe/i 5
+/[-.0-9][0-9][.-]?dynamic/i 5
+/[.-]catv[.-]/i 5
+/unused-addr[.-][0-9]/i 3
+/comcast[.-][0-9]/i 5
+/[.-]broadband[.-]/i 5
+/[0-9][.-]?fbx/i 4
+/[.-]peer[.-]/i 1
+/[.-]homeuser[.-]/i 5
+/[-.0-9][0-9][.-]?catv/i 5
+/customers?[.-][0-9]/i 1
+/[.-]wifi[.-]/i 5
+/[0-9][.-]?kabel/i 3
+/dynip[.-][0-9]/i 5
+/[.-]broad[.-]/i 5
+/[a|x]?dsl-line[.-]?[0-9]/i 4
+/[-.0-9][0-9][.-]?ppp/i 5
+/pool[.-][0-9]/i 4
+/[.-]nat[.-]/i 5
+/gprs[.-][0-9]/i 5
+/brodband[.-][0-9]/i 5
+/[.-]gprs[.-]/i 5
+/[.-]user[.-]/i 1
+/[-.0-9][0-9][.-]?in-?addr/i 4
+/[.-]host[.-]/i 2
+/[.-]fbx[.-]/i 4
+/dynamic[.-][0-9]/i 5
+/[-.0-9][0-9][.-]?peer/i 1
+/[-.0-9][0-9][.-]?pool/i 4
+/[-.0-9][0-9][.-]?user/i 1
+/[.-]cdma[.-]/i 5
+/user[.-][0-9]/i 1
+/[-.0-9][0-9][.-]?customers?/i 1
+/ppp[.-][0-9]/i 5
+/kabel[.-][0-9]/i 3
+/dhcp[.-][0-9]/i 5
+/peer[.-][0-9]/i 1
+/[-.0-9][0-9][.-]?host/i 2
+/clients?[.-][0-9]{2,}/i 5
+/host[.-][0-9]/i 2
+/[.-]ppp[.-]/i 5
+/[.-]dhcp[.-]/i 5
+/[.-]comcast[.-]/i 5
+/cable[.-][0-9]/i 3
+/[-.0-9][0-9][.-]?dial-?up/i 5
+/[-.0-9][0-9][.-]?bredband/i 5
+/[-.0-9][0-9][.-]?[a|x]?dsl-line/i 4
+/[.-]dial-?up[.-]/i 5
+/[.-]cablemodem[.-]/i 5
+/pppoe[.-][0-9]/i 5
+/[.-]unused-addr[.-]/i 3
+/pptp[.-][0-9]/i 5
+/broadband[.-][0-9]/i 5
+/[.-][a|x]?dsl-line[.-]/i 4
+/[.-]customers?[.-]/i 1
+/[-.0-9][0-9][.-]?fibertel/i 4
+/[-.0-9][0-9][.-]?comcast/i 5
+/[.-]dynamic[.-]/i 5
+/cdma[.-][0-9]/i 5
+/[0-9][.-]?broad/i 5
+/fbx[.-][0-9]/i 4
+/catv[.-][0-9]/i 5
+/[-.0-9][0-9][.-]?homeuser/i 5
+/[-.0-9][.-]pppoe[.-]/i 5
+/[-.0-9][.-]dynip[.-]/i 5
+/[-.0-9][0-9][.-]?[a|x]?dsl/i 4
+/[-.0-9][0-9]{3,}[.-]?clients?/i 5
+/[-.0-9][0-9][.-]?pptp/i 5
+/[.-]clients?[.-]/i 1
+/[.-]in-?addr[.-]/i 4
+/[.-]pool[.-]/i 4
+/[a|x]?dsl[.-]?[0-9]/i 4
+/[.-][a|x]?dsl[.-]/i 4
+/[-.0-9][0-9][.-]?[a|x]?dsl-dynamic/i 5
+/dial-?up[.-][0-9]/i 5
+/[-.0-9][0-9][.-]?cablemodem/i 5
+/[a|x]?dsl-dynamic[.-]?[0-9]/i 5
+/[.-]pptp[.-]/i 5
+/[.-][a|x]?dsl-dynamic[.-]/i 5
+/[0-9][.-]?wifi/i 5
+/fibertel[.-][0-9]/i 4
+/dyn[.-][0-9][-.0-9]/i 5
+/[-.0-9][0-9][.-]broadband/i 5
+/[-.0-9][0-9][.-]cable/i 3
+/broad[.-][0-9]/i 5
+/[-.0-9][0-9][.-]gprs/i 5
+/cablemodem[.-][0-9]/i 5
+/[-.0-9][0-9][.-]modem/i 5
+/[-.0-9][0-9][.-]dyn/i 5
+/[-.0-9][0-9][.-]dynip/i 5
+/[-.0-9][0-9][.-]cdma/i 5
+/[.-]modem[.-]/i 5
+/[.-]kabel[.-]/i 3
+/[.-]cable[.-]/i 3
+/in-?addr[.-][0-9]/i 4
+/nat[.-][0-9]/i 5
+/[.-]fibertel[.-]/i 4
+/[.-]bredband[.-]/i 5
+/modem[.-][0-9]/i 5
+/[0-9][.-]?dhcp/i 5
+/wifi[.-][0-9]/i 5
+]]
+local checks_hellohost_map
+
+local checks_hello = [[
+/^[^\.]+$/i 5 # for helo=COMPUTER, ANNA, etc... Without dot in helo
+/^(dsl)?(device|speedtouch)\.lan$/i 5
+/\.(lan|local|home|localdomain|intra|in-addr.arpa|priv|user|veloxzon)$/i 5
+]]
+local checks_hello_map
+
+local checks_hello_badip = [[
+/^\d\.\d\.\d\.255$/i 1
+/^192\.0\.0\./i 1
+/^2001:db8::/i 1
+/^10\./i 1
+/^192\.0\.2\./i 1
+/^172\.1[6-9]\./i 1
+/^192\.168\./i 1
+/^::1$/i 1 # loopback ipv4, ipv6
+/^ffxx::/i 1
+/^fc00::/i 1
+/^203\.0\.113\./i 1
+/^fe[cdf][0-9a-f]:/i 1
+/^100.12[0-7]\d\./i 1
+/^fe[89ab][0-9a-f]::/i 1
+/^169\.254\./i 1
+/^0\./i 1
+/^198\.51\.100\./i 1
+/^172\.3[01]\./i 1
+/^100.[7-9]\d\./i 1
+/^100.1[01]\d\./i 1
+/^127\./i 1
+/^100.6[4-9]\./i 1
+/^192\.88\.99\./i 1
+/^172\.2[0-9]\./i 1
+]]
+local checks_hello_badip_map
+
+local checks_hello_bareip = [[
+/^\d+[x.-]\d+[x.-]\d+[x.-]\d+$/
+/^[0-9a-f]+:/
+]]
+local checks_hello_bareip_map
+
+local config = {
+ ['helo_enabled'] = false,
+ ['hostname_enabled'] = false,
+ ['from_enabled'] = false,
+ ['rcpt_enabled'] = false,
+ ['mid_enabled'] = false,
+ ['url_enabled'] = false
+}
+
+local compiled_regexp = {} -- cache of regexps
+local check_local = false
+local check_authed = false
+local N = "hfilter"
+
+local function check_regexp(str, regexp_text)
+ local re = compiled_regexp[regexp_text]
+ if not re then
+ re = rspamd_regexp.create(regexp_text, 'i')
+ compiled_regexp[regexp_text] = re
+ end
+
+ return re:match(str)
+end
+
+local function add_static_map(data)
+ return rspamd_config:add_map {
+ type = 'regexp_multi',
+ url = {
+ upstreams = 'static',
+ data = data,
+ }
+ }
+end
+
+local function check_fqdn(domain)
+ if check_regexp(domain,
+ '(?=^.{4,253}$)(^((?!-)[a-zA-Z0-9-]{1,63}(?<!-)\\.)+[a-zA-Z0-9-]{2,63}\\.?$)') then
+ return true
+ end
+ return false
+end
+
+-- host: host for check
+-- symbol_suffix: suffix for symbol
+-- eq_ip: ip for comparing or empty string
+-- eq_host: host for comparing or empty string
+local function check_host(task, host, symbol_suffix, eq_ip, eq_host)
+ local failed_address = 0
+ local resolved_address = {}
+
+ local function check_host_cb_mx(_, to_resolve, results, err)
+ if err and (err ~= 'requested record is not found' and err ~= 'no records with this name') then
+ lua_util.debugm(N, task, 'error looking up %s: %s', to_resolve, err)
+ end
+ if not results then
+ task:insert_result('HFILTER_' .. symbol_suffix .. '_NORES_A_OR_MX', 1.0,
+ to_resolve)
+ else
+ for _, mx in pairs(results) do
+ if mx['name'] then
+ local failed_mx_address = 0
+ -- Capture failed_mx_address
+ local function check_host_cb_mx_a(_, _, mx_results)
+ if not mx_results then
+ failed_mx_address = failed_mx_address + 1
+ end
+
+ if failed_mx_address >= 2 then
+ task:insert_result('HFILTER_' .. symbol_suffix .. '_NORESOLVE_MX',
+ 1.0, mx['name'])
+ end
+ end
+
+ task:get_resolver():resolve('a', {
+ task = task,
+ name = mx['name'],
+ callback = check_host_cb_mx_a
+ })
+ task:get_resolver():resolve('aaaa', {
+ task = task,
+ name = mx['name'],
+ callback = check_host_cb_mx_a
+ })
+ end
+ end
+ end
+ end
+ local function check_host_cb_a(_, _, results)
+ if not results then
+ failed_address = failed_address + 1
+ else
+ for _, result in pairs(results) do
+ table.insert(resolved_address, result:to_string())
+ end
+ end
+
+ if failed_address >= 2 then
+ -- No A or AAAA records
+ if eq_ip and eq_ip ~= '' then
+ for _, result in pairs(resolved_address) do
+ if result == eq_ip then
+ return true
+ end
+ end
+ task:insert_result('HFILTER_' .. symbol_suffix .. '_IP_A', 1.0, host)
+ end
+ task:get_resolver():resolve_mx({
+ task = task,
+ name = host,
+ callback = check_host_cb_mx
+ })
+ end
+ end
+
+ if host then
+ host = string.lower(host)
+ else
+ return false
+ end
+ if eq_host then
+ eq_host = string.lower(eq_host)
+ else
+ eq_host = ''
+ end
+
+ if check_fqdn(host) then
+ if eq_host == '' or eq_host ~= host then
+ task:get_resolver():resolve('a', {
+ task = task,
+ name = host,
+ callback = check_host_cb_a
+ })
+ -- Check ipv6 as well
+ task:get_resolver():resolve('aaaa', {
+ task = task,
+ name = host,
+ callback = check_host_cb_a
+ })
+ end
+ else
+ task:insert_result('HFILTER_' .. symbol_suffix .. '_NOT_FQDN', 1.0, host)
+ end
+
+ return true
+end
+
+--
+local function hfilter_callback(task)
+ -- Links checks
+ if config['url_enabled'] then
+ local parts = task:get_text_parts()
+ if parts then
+ local plain_text_part, html_text_part
+
+ for _, p in ipairs(parts) do
+ if p:is_html() then
+ html_text_part = p
+ else
+ plain_text_part = p
+ end
+ end
+
+ local function check_text_part(part, ty)
+ local url_len = part:get_urls_length()
+ local plen = part:get_length()
+
+ if plen > 0 and url_len > 0 then
+ local rel = url_len / plen
+ if rel > 0.8 then
+ local sc = (rel - 0.8) * 5.0
+ if sc > 1.0 then
+ sc = 1.0
+ end
+ task:insert_result('HFILTER_URL_ONLY', sc, tostring(sc))
+ local lines = part:get_lines_count()
+ if lines > 0 and lines < 2 then
+ task:insert_result('HFILTER_URL_ONELINE', 1.00,
+ string.format('%s:%d:%d', ty, math.floor(rel), lines))
+ end
+ end
+ end
+ end
+ if html_text_part then
+ check_text_part(html_text_part, 'html')
+ elseif plain_text_part then
+ check_text_part(plain_text_part, 'plain')
+ end
+ end
+ end
+
+ --No more checks for auth user or local network
+ local rip = task:get_from_ip()
+ if ((not check_authed and task:get_user()) or
+ (not check_local and rip and rip:is_local())) then
+ return false
+ end
+
+ --local message = task:get_message()
+ local ip = false
+ if rip and rip:is_valid() then
+ ip = rip:to_string()
+ end
+
+ -- Check's HELO
+ local weight_helo = 0
+ local helo
+ if config['helo_enabled'] then
+ helo = task:get_helo()
+ if helo then
+ if helo ~= rspamc_local_helo then
+ helo = string.gsub(helo, '[%[%]]', '')
+ -- Regexp check HELO (checks_hello_badip)
+ local find_badip = false
+ local values = checks_hello_badip_map:get_key(helo)
+ if values then
+ task:insert_result('HFILTER_HELO_BADIP', 1.0, helo, values)
+ find_badip = true
+ end
+
+ -- Regexp check HELO (checks_hello_bareip)
+ local find_bareip = false
+ if not find_badip then
+ values = checks_hello_bareip_map:get_key(helo)
+ if values then
+ task:insert_result('HFILTER_HELO_BAREIP', 1.0, helo, values)
+ find_bareip = true
+ end
+ end
+
+ if not find_badip and not find_bareip then
+ -- Regexp check HELO (checks_hello)
+ local weights = checks_hello_map:get_key(helo)
+ for _, weight in ipairs(weights or {}) do
+ weight = tonumber(weight) or 0
+ if weight > weight_helo then
+ weight_helo = weight
+ end
+ end
+ -- Regexp check HELO (checks_hellohost)
+ weights = checks_hellohost_map:get_key(helo)
+ for _, weight in ipairs(weights or {}) do
+ weight = tonumber(weight) or 0
+ if weight > weight_helo then
+ weight_helo = weight
+ end
+ end
+ --FQDN check HELO
+ if ip and helo and weight_helo == 0 then
+ check_host(task, helo, 'HELO', ip)
+ end
+ end
+ end
+ end
+ end
+
+ -- Check's HOSTNAME
+ local weight_hostname = 0
+ local hostname = task:get_hostname()
+
+ if config['hostname_enabled'] then
+ if hostname then
+ -- Check regexp HOSTNAME
+ local weights = checks_hellohost_map:get_key(hostname)
+ for _, weight in ipairs(weights or {}) do
+ weight = tonumber(weight) or 0
+ if weight > weight_hostname then
+ weight_hostname = weight
+ end
+ end
+ else
+ task:insert_result('HFILTER_HOSTNAME_UNKNOWN', 1.00)
+ end
+ end
+
+ --Insert weight's for HELO or HOSTNAME
+ if weight_helo > 0 and weight_helo >= weight_hostname then
+ task:insert_result('HFILTER_HELO_' .. weight_helo, 1.0, helo)
+ elseif weight_hostname > 0 and weight_hostname > weight_helo then
+ task:insert_result('HFILTER_HOSTNAME_' .. weight_hostname, 1.0, hostname)
+ end
+
+ -- MAILFROM checks --
+ local frombounce = false
+ if config['from_enabled'] then
+ local from = task:get_from(1)
+ if from then
+ --FROM host check
+ for _, fr in ipairs(from) do
+ local fr_split = rspamd_str_split(fr['addr'], '@')
+ if #fr_split == 2 then
+ check_host(task, fr_split[2], 'FROMHOST', '', '')
+ if fr_split[1] == 'postmaster' then
+ frombounce = true
+ end
+ end
+ end
+ else
+ if helo and helo ~= rspamc_local_helo then
+ task:insert_result('HFILTER_FROM_BOUNCE', 1.00, helo)
+ frombounce = true
+ end
+ end
+ end
+
+ -- Recipients checks --
+ if config['rcpt_enabled'] then
+ local rcpt = task:get_recipients()
+ if rcpt then
+ local count_rcpt = #rcpt
+ if frombounce then
+ if count_rcpt > 1 then
+ task:insert_result('HFILTER_RCPT_BOUNCEMOREONE', 1.00,
+ tostring(count_rcpt))
+ end
+ end
+ end
+ end
+
+ --Message ID host check
+ if config['mid_enabled'] then
+ local message_id = task:get_message_id()
+ if message_id then
+ local mid_split = rspamd_str_split(message_id, '@')
+ if #mid_split == 2 and not string.find(mid_split[2], 'local') then
+ check_host(task, mid_split[2], 'MID')
+ end
+ end
+ end
+
+ return false
+end
+
+local symbols_enabled = {}
+
+local symbols_helo = {
+ "HFILTER_HELO_BAREIP",
+ "HFILTER_HELO_BADIP",
+ "HFILTER_HELO_1",
+ "HFILTER_HELO_2",
+ "HFILTER_HELO_3",
+ "HFILTER_HELO_4",
+ "HFILTER_HELO_5",
+ "HFILTER_HELO_NORESOLVE_MX",
+ "HFILTER_HELO_NORES_A_OR_MX",
+ "HFILTER_HELO_IP_A",
+ "HFILTER_HELO_NOT_FQDN"
+}
+local symbols_hostname = {
+ "HFILTER_HOSTNAME_1",
+ "HFILTER_HOSTNAME_2",
+ "HFILTER_HOSTNAME_3",
+ "HFILTER_HOSTNAME_4",
+ "HFILTER_HOSTNAME_5",
+ "HFILTER_HOSTNAME_UNKNOWN"
+}
+local symbols_rcpt = {
+ "HFILTER_RCPT_BOUNCEMOREONE"
+}
+local symbols_mid = {
+ "HFILTER_MID_NORESOLVE_MX",
+ "HFILTER_MID_NORES_A_OR_MX",
+ "HFILTER_MID_NOT_FQDN"
+}
+local symbols_url = {
+ "HFILTER_URL_ONLY",
+ "HFILTER_URL_ONELINE"
+}
+local symbols_from = {
+ "HFILTER_FROMHOST_NORESOLVE_MX",
+ "HFILTER_FROMHOST_NORES_A_OR_MX",
+ "HFILTER_FROMHOST_NOT_FQDN",
+ "HFILTER_FROM_BOUNCE"
+}
+
+local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N,
+ false, false)
+check_local = auth_and_local_conf[1]
+check_authed = auth_and_local_conf[2]
+local timeout = 0.0
+
+local opts = rspamd_config:get_all_opt('hfilter')
+if opts then
+ for k, v in pairs(opts) do
+ config[k] = v
+ end
+end
+
+local function append_t(t, a)
+ for _, v in ipairs(a) do
+ table.insert(t, v)
+ end
+end
+if config['helo_enabled'] then
+ checks_hello_bareip_map = add_static_map(checks_hello_bareip)
+ checks_hello_badip_map = add_static_map(checks_hello_badip)
+ checks_hellohost_map = add_static_map(checks_hellohost)
+ checks_hello_map = add_static_map(checks_hello)
+ append_t(symbols_enabled, symbols_helo)
+ timeout = math.max(timeout, rspamd_config:get_dns_timeout() * 3)
+end
+if config['hostname_enabled'] then
+ if not checks_hellohost_map then
+ checks_hellohost_map = add_static_map(checks_hellohost)
+ end
+ append_t(symbols_enabled, symbols_hostname)
+ timeout = math.max(timeout, rspamd_config:get_dns_timeout())
+end
+if config['from_enabled'] then
+ append_t(symbols_enabled, symbols_from)
+ timeout = math.max(timeout, rspamd_config:get_dns_timeout())
+end
+if config['rcpt_enabled'] then
+ append_t(symbols_enabled, symbols_rcpt)
+end
+if config['mid_enabled'] then
+ append_t(symbols_enabled, symbols_mid)
+end
+if config['url_enabled'] then
+ append_t(symbols_enabled, symbols_url)
+end
+
+--dumper(symbols_enabled)
+if #symbols_enabled > 0 then
+ local id = rspamd_config:register_symbol {
+ name = 'HFILTER_CHECK',
+ callback = hfilter_callback,
+ type = 'callback',
+ augmentations = { string.format("timeout=%f", timeout) },
+ }
+ for _, sym in ipairs(symbols_enabled) do
+ rspamd_config:register_symbol {
+ type = 'virtual',
+ score = 1.0,
+ parent = id,
+ name = sym,
+ }
+ rspamd_config:set_metric_symbol({
+ name = sym,
+ score = 0.0,
+ group = 'hfilter'
+ })
+ end
+else
+ lua_util.disable_module(N, "config")
+end
diff --git a/src/plugins/lua/history_redis.lua b/src/plugins/lua/history_redis.lua
new file mode 100644
index 0000000..d0aa5ae
--- /dev/null
+++ b/src/plugins/lua/history_redis.lua
@@ -0,0 +1,314 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ rspamd_config:add_example(nil, 'history_redis',
+ "Store history of checks for WebUI using Redis",
+ [[
+redis_history {
+ # History key name
+ key_prefix = 'rs_history';
+ # History expire in seconds
+ expire = 0;
+ # History rows limit
+ nrows = 200;
+ # Use zstd compression when storing data in redis
+ compress = true;
+ # Obfuscate subjects for privacy
+ subject_privacy = false;
+ # Default hash-algorithm to obfuscate subject
+ subject_privacy_alg = 'blake2';
+ # Prefix to show it's obfuscated
+ subject_privacy_prefix = 'obf';
+ # Cut the length of the hash if desired
+ subject_privacy_length = 16;
+}
+ ]])
+ return
+end
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local lua_util = require "lua_util"
+local lua_redis = require "lua_redis"
+local fun = require "fun"
+local ucl = require "ucl"
+local ts = (require "tableshape").types
+local lua_verdict = require "lua_verdict"
+local E = {}
+local N = "history_redis"
+local hostname = rspamd_util.get_hostname()
+
+local redis_params
+
+local settings = {
+ key_prefix = 'rs_history', -- default key name
+ expire = nil, -- default no expire
+ nrows = 200, -- default rows limit
+ compress = true, -- use zstd compression when storing data in redis
+ subject_privacy = false, -- subject privacy is off
+ subject_privacy_alg = 'blake2', -- default hash-algorithm to obfuscate subject
+ subject_privacy_prefix = 'obf', -- prefix to show it's obfuscated
+ subject_privacy_length = 16, -- cut the length of the hash
+}
+
+local settings_schema = lua_redis.enrich_schema({
+ key_prefix = ts.string,
+ expire = (ts.number + ts.string / lua_util.parse_time_interval):is_optional(),
+ nrows = ts.number,
+ compress = ts.boolean,
+ subject_privacy = ts.boolean:is_optional(),
+ subject_privacy_alg = ts.string:is_optional(),
+ subject_privacy_prefix = ts.string:is_optional(),
+ subject_privacy_length = ts.number:is_optional(),
+})
+
+local function process_addr(addr)
+ if addr then
+ return addr.addr
+ end
+
+ return 'unknown'
+end
+
+local function normalise_results(tbl, task)
+ local metric = tbl.default
+
+ -- Convert stupid metric object
+ if metric then
+ tbl.symbols = {}
+ local symbols, others = fun.partition(function(_, v)
+ return type(v) == 'table' and v.score
+ end, metric)
+
+ fun.each(function(k, v)
+ v.name = nil;
+ tbl.symbols[k] = v;
+ end, symbols)
+ fun.each(function(k, v)
+ tbl[k] = v
+ end, others)
+
+ -- Reset the original metric
+ tbl.default = nil
+ end
+
+ -- Now, add recipients and senders
+ tbl.sender_smtp = process_addr((task:get_from('smtp') or E)[1])
+ tbl.sender_mime = process_addr((task:get_from('mime') or E)[1])
+ tbl.rcpt_smtp = fun.totable(fun.map(process_addr, task:get_recipients('smtp') or {}))
+ tbl.rcpt_mime = fun.totable(fun.map(process_addr, task:get_recipients('mime') or {}))
+ tbl.user = task:get_user() or 'unknown'
+ tbl.rmilter = nil
+ tbl.messages = nil
+ tbl.urls = nil
+ tbl.action = lua_verdict.adjust_passthrough_action(task)
+
+ local seconds = task:get_timeval()['tv_sec']
+ tbl.unix_time = seconds
+
+ local subject = task:get_header('subject') or 'unknown'
+ tbl.subject = lua_util.maybe_obfuscate_string(subject, settings, 'subject')
+ tbl.size = task:get_size()
+ local ip = task:get_from_ip()
+ if ip and ip:is_valid() then
+ tbl.ip = tostring(ip)
+ else
+ tbl.ip = 'unknown'
+ end
+
+ tbl.user = task:get_user() or 'unknown'
+end
+
+local function history_save(task)
+ local function redis_llen_cb(err, _)
+ if err then
+ rspamd_logger.errx(task, 'got error %s when writing history row: %s',
+ err)
+ end
+ end
+
+ -- We skip saving it to the history
+ if task:has_flag('no_log') then
+ return
+ end
+
+ local data = task:get_protocol_reply { 'metrics', 'basic' }
+ local prefix = settings.key_prefix .. hostname
+
+ if data then
+ normalise_results(data, task)
+ else
+ rspamd_logger.errx('cannot get protocol reply, skip saving in history')
+ return
+ end
+
+ local json = ucl.to_format(data, 'json-compact')
+
+ if settings.compress then
+ json = rspamd_util.zstd_compress(json)
+ -- Distinguish between compressed and non-compressed options
+ prefix = prefix .. '_zst'
+ end
+
+ local ret, conn, _ = lua_redis.rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ nil, -- hash key
+ true, -- is write
+ redis_llen_cb, --callback
+ 'LPUSH', -- command
+ { prefix, json } -- arguments
+ )
+
+ if ret then
+ conn:add_cmd('LTRIM', { prefix, '0', string.format('%d', settings.nrows - 1) })
+
+ if settings.expire and settings.expire > 0 then
+ conn:add_cmd('EXPIRE', { prefix, string.format('%d', settings.expire) })
+ end
+ end
+end
+
+local function handle_history_request(task, conn, from, to, reset)
+ local prefix = settings.key_prefix .. hostname
+ if settings.compress then
+ -- Distinguish between compressed and non-compressed options
+ prefix = prefix .. '_zst'
+ end
+
+ if reset then
+ local function redis_ltrim_cb(err, _)
+ if err then
+ rspamd_logger.errx(task, 'got error %s when resetting history: %s',
+ err)
+ conn:send_error(504, '{"error": "' .. err .. '"}')
+ else
+ conn:send_string('{"success":true}')
+ end
+ end
+ lua_redis.rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ nil, -- hash key
+ true, -- is write
+ redis_ltrim_cb, --callback
+ 'LTRIM', -- command
+ { prefix, '0', '0' } -- arguments
+ )
+ else
+ local function redis_lrange_cb(err, data)
+ if data then
+ local reply = {
+ version = 2,
+ }
+ if settings.compress then
+ local t1 = rspamd_util:get_ticks()
+
+ data = fun.totable(fun.filter(function(e)
+ return e ~= nil
+ end,
+ fun.map(function(e)
+ local _, dec = rspamd_util.zstd_decompress(e)
+ if dec then
+ return dec
+ end
+ return nil
+ end, data)))
+ lua_util.debugm(N, task, 'decompress took %s ms',
+ (rspamd_util:get_ticks() - t1) * 1000.0)
+ collectgarbage()
+ end
+ -- Parse elements using ucl
+ local t1 = rspamd_util:get_ticks()
+ data = fun.totable(
+ fun.map(function(_, obj)
+ return obj
+ end,
+ fun.filter(function(res, obj)
+ if res then
+ return true
+ end
+ return false
+ end,
+ fun.map(function(elt)
+ local parser = ucl.parser()
+ local res, _ = parser:parse_text(elt)
+
+ if res then
+ return true, parser:get_object()
+ else
+ return false, nil
+ end
+ end, data))))
+ lua_util.debugm(N, task, 'parse took %s ms',
+ (rspamd_util:get_ticks() - t1) * 1000.0)
+ collectgarbage()
+ t1 = rspamd_util:get_ticks()
+ reply.rows = data
+ conn:send_ucl(reply)
+ lua_util.debugm(N, task, 'process + sending took %s ms',
+ (rspamd_util:get_ticks() - t1) * 1000.0)
+ collectgarbage()
+ else
+ rspamd_logger.errx(task, 'got error %s when getting history: %s',
+ err)
+ conn:send_error(504, '{"error": "' .. err .. '"}')
+ end
+ end
+ lua_redis.rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ nil, -- hash key
+ false, -- is write
+ redis_lrange_cb, --callback
+ 'LRANGE', -- command
+ { prefix, string.format('%d', from), string.format('%d', to) }, -- arguments
+ { opaque_data = true }
+ )
+ end
+end
+
+local opts = rspamd_config:get_all_opt('history_redis')
+if opts then
+ settings = lua_util.override_defaults(settings, opts)
+ local res, err = settings_schema:transform(settings)
+
+ if not res then
+ rspamd_logger.warnx(rspamd_config, '%s: plugin is misconfigured: %s', N, err)
+ lua_util.disable_module(N, "config")
+ return
+ end
+ settings = res
+
+ redis_params = lua_redis.parse_redis_server('history_redis')
+ if not redis_params then
+ rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
+ lua_util.disable_module(N, "redis")
+ else
+ rspamd_config:register_symbol({
+ name = 'HISTORY_SAVE',
+ type = 'idempotent',
+ callback = history_save,
+ flags = 'empty,explicit_disable,ignore_passthrough',
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }
+ })
+ lua_redis.register_prefix(settings.key_prefix .. hostname, N,
+ "Redis history", {
+ type = 'list',
+ })
+ rspamd_plugins['history'] = {
+ handler = handle_history_request
+ }
+ end
+end
diff --git a/src/plugins/lua/http_headers.lua b/src/plugins/lua/http_headers.lua
new file mode 100644
index 0000000..1c6494a
--- /dev/null
+++ b/src/plugins/lua/http_headers.lua
@@ -0,0 +1,198 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local logger = require "rspamd_logger"
+local ucl = require "ucl"
+
+local spf_symbols = {
+ symbol_allow = 'R_SPF_ALLOW',
+ symbol_deny = 'R_SPF_FAIL',
+ symbol_softfail = 'R_SPF_SOFTFAIL',
+ symbol_neutral = 'R_SPF_NEUTRAL',
+ symbol_tempfail = 'R_SPF_DNSFAIL',
+ symbol_na = 'R_SPF_NA',
+ symbol_permfail = 'R_SPF_PERMFAIL',
+}
+
+local dkim_symbols = {
+ symbol_allow = 'R_DKIM_ALLOW',
+ symbol_deny = 'R_DKIM_REJECT',
+ symbol_tempfail = 'R_DKIM_TEMPFAIL',
+ symbol_na = 'R_DKIM_NA',
+ symbol_permfail = 'R_DKIM_PERMFAIL',
+ symbol_trace = 'DKIM_TRACE',
+}
+
+local dkim_trace = {
+ pass = '+',
+ fail = '-',
+ temperror = '?',
+ permerror = '~',
+}
+
+local dmarc_symbols = {
+ allow = 'DMARC_POLICY_ALLOW',
+ badpolicy = 'DMARC_BAD_POLICY',
+ dnsfail = 'DMARC_DNSFAIL',
+ na = 'DMARC_NA',
+ reject = 'DMARC_POLICY_REJECT',
+ softfail = 'DMARC_POLICY_SOFTFAIL',
+ quarantine = 'DMARC_POLICY_QUARANTINE',
+}
+
+local opts = rspamd_config:get_all_opt('dmarc')
+if opts and opts['symbols'] then
+ for k, _ in pairs(dmarc_symbols) do
+ if opts['symbols'][k] then
+ dmarc_symbols[k] = opts['symbols'][k]
+ end
+ end
+end
+
+opts = rspamd_config:get_all_opt('dkim')
+if opts then
+ for k, _ in pairs(dkim_symbols) do
+ if opts[k] then
+ dkim_symbols[k] = opts[k]
+ end
+ end
+end
+
+opts = rspamd_config:get_all_opt('spf')
+if opts then
+ for k, _ in pairs(spf_symbols) do
+ if opts[k] then
+ spf_symbols[k] = opts[k]
+ end
+ end
+end
+
+-- Disable DKIM checks if passed via HTTP headers
+rspamd_config:add_condition("DKIM_CHECK", function(task)
+ local hdr = task:get_request_header('DKIM')
+
+ if hdr then
+ local parser = ucl.parser()
+ local res, err = parser:parse_string(tostring(hdr))
+ if not res then
+ logger.infox(task, "cannot parse DKIM header: %1", err)
+ return true
+ end
+
+ local p_obj = parser:get_object()
+ local results = p_obj['results']
+ if not results and p_obj['result'] then
+ results = { { result = p_obj['result'], domain = 'unknown' } }
+ end
+
+ if results then
+ for _, obj in ipairs(results) do
+ local dkim_domain = obj['domain'] or 'unknown'
+ if obj['result'] == 'pass' or obj['result'] == 'allow' then
+ task:insert_result(dkim_symbols['symbol_allow'], 1.0, 'http header')
+ task:insert_result(dkim_symbols['symbol_trace'], 1.0,
+ string.format('%s:%s', dkim_domain, dkim_trace.pass))
+ elseif obj['result'] == 'fail' or obj['result'] == 'reject' then
+ task:insert_result(dkim_symbols['symbol_deny'], 1.0, 'http header')
+ task:insert_result(dkim_symbols['symbol_trace'], 1.0,
+ string.format('%s:%s', dkim_domain, dkim_trace.fail))
+ elseif obj['result'] == 'tempfail' or obj['result'] == 'softfail' then
+ task:insert_result(dkim_symbols['symbol_tempfail'], 1.0, 'http header')
+ task:insert_result(dkim_symbols['symbol_trace'], 1.0,
+ string.format('%s:%s', dkim_domain, dkim_trace.temperror))
+ elseif obj['result'] == 'permfail' then
+ task:insert_result(dkim_symbols['symbol_permfail'], 1.0, 'http header')
+ task:insert_result(dkim_symbols['symbol_trace'], 1.0,
+ string.format('%s:%s', dkim_domain, dkim_trace.permerror))
+ elseif obj['result'] == 'na' then
+ task:insert_result(dkim_symbols['symbol_na'], 1.0, 'http header')
+ end
+ end
+ end
+ end
+
+ return false
+end)
+
+-- Disable SPF checks if passed via HTTP headers
+rspamd_config:add_condition("SPF_CHECK", function(task)
+ local hdr = task:get_request_header('SPF')
+
+ if hdr then
+ local parser = ucl.parser()
+ local res, err = parser:parse_string(tostring(hdr))
+ if not res then
+ logger.infox(task, "cannot parse SPF header: %1", err)
+ return true
+ end
+
+ local obj = parser:get_object()
+
+ if obj['result'] then
+ if obj['result'] == 'pass' or obj['result'] == 'allow' then
+ task:insert_result(spf_symbols['symbol_allow'], 1.0, 'http header')
+ elseif obj['result'] == 'fail' or obj['result'] == 'reject' then
+ task:insert_result(spf_symbols['symbol_deny'], 1.0, 'http header')
+ elseif obj['result'] == 'neutral' then
+ task:insert_result(spf_symbols['symbol_neutral'], 1.0, 'http header')
+ elseif obj['result'] == 'softfail' then
+ task:insert_result(spf_symbols['symbol_softfail'], 1.0, 'http header')
+ elseif obj['result'] == 'permfail' then
+ task:insert_result(spf_symbols['symbol_permfail'], 1.0, 'http header')
+ elseif obj['result'] == 'na' then
+ task:insert_result(spf_symbols['symbol_na'], 1.0, 'http header')
+ end
+ end
+ end
+
+ return false
+end)
+
+rspamd_config:add_condition("DMARC_CALLBACK", function(task)
+ local hdr = task:get_request_header('DMARC')
+
+ if hdr then
+ local parser = ucl.parser()
+ local res, err = parser:parse_string(tostring(hdr))
+ if not res then
+ logger.infox(task, "cannot parse DMARC header: %1", err)
+ return true
+ end
+
+ local obj = parser:get_object()
+
+ if obj['result'] then
+ if obj['result'] == 'pass' or obj['result'] == 'allow' then
+ task:insert_result(dmarc_symbols['allow'], 1.0, 'http header')
+ elseif obj['result'] == 'fail' or obj['result'] == 'reject' then
+ task:insert_result(dmarc_symbols['reject'], 1.0, 'http header')
+ elseif obj['result'] == 'quarantine' then
+ task:insert_result(dmarc_symbols['quarantine'], 1.0, 'http header')
+ elseif obj['result'] == 'tempfail' then
+ task:insert_result(dmarc_symbols['dnsfail'], 1.0, 'http header')
+ elseif obj['result'] == 'softfail' or obj['result'] == 'none' then
+ task:insert_result(dmarc_symbols['softfail'], 1.0, 'http header')
+ elseif obj['result'] == 'permfail' or obj['result'] == 'badpolicy' then
+ task:insert_result(dmarc_symbols['badpolicy'], 1.0, 'http header')
+ elseif obj['result'] == 'na' then
+ task:insert_result(dmarc_symbols['na'], 1.0, 'http header')
+ end
+ end
+ end
+
+ return false
+end)
+
diff --git a/src/plugins/lua/ip_score.lua b/src/plugins/lua/ip_score.lua
new file mode 100644
index 0000000..e43fa3b
--- /dev/null
+++ b/src/plugins/lua/ip_score.lua
@@ -0,0 +1,4 @@
+-- This module is deprecated and must not be used.
+-- This file serves as a tombstone to prevent old ip_score to be loaded
+
+return \ No newline at end of file
diff --git a/src/plugins/lua/known_senders.lua b/src/plugins/lua/known_senders.lua
new file mode 100644
index 0000000..d26a1df
--- /dev/null
+++ b/src/plugins/lua/known_senders.lua
@@ -0,0 +1,245 @@
+--[[
+Copyright (c) 2023, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- This plugin implements known senders logic for Rspamd
+
+local rspamd_logger = require "rspamd_logger"
+local ts = (require "tableshape").types
+local N = 'known_senders'
+local lua_util = require "lua_util"
+local lua_redis = require "lua_redis"
+local lua_maps = require "lua_maps"
+local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash"
+
+if confighelp then
+ rspamd_config:add_example(nil, 'known_senders',
+ "Maintain a list of known senders using Redis",
+ [[
+known_senders {
+ # Domains to track senders
+ domains = "https://maps.rspamd.com/freemail/free.txt.zst";
+ # Maximum number of elements
+ max_senders = 100000;
+ # Maximum time to live (when not using bloom filters)
+ max_ttl = 30d;
+ # Use bloom filters (must be enabled in Redis as a plugin)
+ use_bloom = false;
+ # Insert symbol for new senders from the specific domains
+ symbol_unknown = 'UNKNOWN_SENDER';
+}
+ ]])
+ return
+end
+
+local redis_params
+local settings = {
+ domains = {},
+ max_senders = 100000,
+ max_ttl = 30 * 86400,
+ use_bloom = false,
+ symbol = 'KNOWN_SENDER',
+ symbol_unknown = 'UNKNOWN_SENDER',
+ redis_key = 'rs_known_senders',
+}
+
+local settings_schema = lua_redis.enrich_schema({
+ domains = lua_maps.map_schema,
+ enabled = ts.boolean:is_optional(),
+ max_senders = (ts.integer + ts.string / tonumber):is_optional(),
+ max_ttl = (ts.integer + ts.string / tonumber):is_optional(),
+ use_bloom = ts.boolean:is_optional(),
+ redis_key = ts.string:is_optional(),
+ symbol = ts.string:is_optional(),
+ symbol_unknown = ts.string:is_optional(),
+})
+
+local function make_key(input)
+ local hash = rspamd_cryptobox_hash.create_specific('md5')
+ hash:update(input.addr)
+ return hash:hex()
+end
+
+local function check_redis_key(task, key, key_ty)
+ lua_util.debugm(N, task, 'check key %s, type: %s', key, key_ty)
+ local function redis_zset_callback(err, data)
+ lua_util.debugm(N, task, 'got data: %s', data)
+ if err then
+ rspamd_logger.errx(task, 'redis error: %s', err)
+ elseif data then
+ if type(data) ~= 'userdata' then
+ -- non-null reply
+ task:insert_result(settings.symbol, 1.0, string.format("%s:%s", key_ty, key))
+ else
+ if settings.symbol_unknown then
+ task:insert_result(settings.symbol_unknown, 1.0, string.format("%s:%s", key_ty, key))
+ end
+ lua_util.debugm(N, task, 'insert key %s, type: %s', key, key_ty)
+ -- Insert key to zset and trim it's cardinality
+ lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ nil, --callback
+ 'ZADD', -- command
+ { settings.redis_key, tostring(task:get_timeval(true)), key } -- arguments
+ )
+ lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ nil, --callback
+ 'ZREMRANGEBYRANK', -- command
+ { settings.redis_key, '0',
+ tostring(-(settings.max_senders + 1)) } -- arguments
+ )
+ end
+ end
+ end
+
+ local function redis_bloom_callback(err, data)
+ lua_util.debugm(N, task, 'got data: %s', data)
+ if err then
+ rspamd_logger.errx(task, 'redis error: %s', err)
+ elseif data then
+ if type(data) ~= 'userdata' and data == 1 then
+ -- non-null reply equal to `1`
+ task:insert_result(settings.symbol, 1.0, string.format("%s:%s", key_ty, key))
+ else
+ if settings.symbol_unknown then
+ task:insert_result(settings.symbol_unknown, 1.0, string.format("%s:%s", key_ty, key))
+ end
+ lua_util.debugm(N, task, 'insert key %s, type: %s', key, key_ty)
+ -- Reserve bloom filter space
+ lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ nil, --callback
+ 'BF.RESERVE', -- command
+ { settings.redis_key, tostring(settings.max_senders), '0.01', '1000', 'NONSCALING' } -- arguments
+ )
+ -- Insert key and adjust bloom filter
+ lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ nil, --callback
+ 'BF.ADD', -- command
+ { settings.redis_key, key } -- arguments
+ )
+ end
+ end
+ end
+
+ if settings.use_bloom then
+ lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_bloom_callback, --callback
+ 'BF.EXISTS', -- command
+ { settings.redis_key, key } -- arguments
+ )
+ else
+ lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_zset_callback, --callback
+ 'ZSCORE', -- command
+ { settings.redis_key, key } -- arguments
+ )
+ end
+end
+
+local function known_senders_callback(task)
+ local mime_from = (task:get_from('mime') or {})[1]
+ local smtp_from = (task:get_from('smtp') or {})[1]
+ local mime_key, smtp_key
+ if mime_from and mime_from.addr then
+ if settings.domains:get_key(mime_from.domain) then
+ mime_key = make_key(mime_from)
+ else
+ lua_util.debugm(N, task, 'skip mime from domain %s', mime_from.domain)
+ end
+ end
+ if smtp_from and smtp_from.addr then
+ if settings.domains:get_key(smtp_from.domain) then
+ smtp_key = make_key(smtp_from)
+ else
+ lua_util.debugm(N, task, 'skip smtp from domain %s', smtp_from.domain)
+ end
+ end
+
+ if mime_key and smtp_key and mime_key ~= smtp_key then
+ -- Check both keys
+ check_redis_key(task, mime_key, 'mime')
+ check_redis_key(task, smtp_key, 'smtp')
+ elseif mime_key then
+ -- Check mime key
+ check_redis_key(task, mime_key, 'mime')
+ elseif smtp_key then
+ -- Check smtp key
+ check_redis_key(task, smtp_key, 'smtp')
+ end
+end
+
+local opts = rspamd_config:get_all_opt('known_senders')
+if opts then
+ settings = lua_util.override_defaults(settings, opts)
+ local res, err = settings_schema:transform(settings)
+ if not res then
+ rspamd_logger.errx(rspamd_config, 'cannot parse known_senders options: %1', err)
+ else
+ settings = res
+ end
+ redis_params = lua_redis.parse_redis_server(N, opts)
+
+ if redis_params then
+ local map_conf = settings.domains
+ settings.domains = lua_maps.map_add_from_ucl(settings.domains, 'set', 'domains to track senders from')
+ if not settings.domains then
+ rspamd_logger.errx(rspamd_config, "couldn't add map %s, disable module",
+ map_conf)
+ lua_util.disable_module(N, "config")
+ return
+ end
+ lua_redis.register_prefix(settings.redis_key, N,
+ 'Known elements redis key', {
+ type = 'zset/bloom filter',
+ })
+ local id = rspamd_config:register_symbol({
+ name = settings.symbol,
+ type = 'normal',
+ callback = known_senders_callback,
+ one_shot = true,
+ score = -1.0,
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }
+ })
+
+ if settings.symbol_unknown and #settings.symbol_unknown > 0 then
+ rspamd_config:register_symbol({
+ name = settings.symbol_unknown,
+ type = 'virtual',
+ parent = id,
+ one_shot = true,
+ score = 0.5,
+ })
+ end
+ else
+ lua_util.disable_module(N, "redis")
+ end
+end
diff --git a/src/plugins/lua/maillist.lua b/src/plugins/lua/maillist.lua
new file mode 100644
index 0000000..be1401c
--- /dev/null
+++ b/src/plugins/lua/maillist.lua
@@ -0,0 +1,235 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- Module for checking mail list headers
+local N = 'maillist'
+local symbol = 'MAILLIST'
+local lua_util = require "lua_util"
+-- EZMLM
+-- Mailing-List: .*run by ezmlm
+-- Precedence: bulk
+-- List-Post: <mailto:
+-- List-Help: <mailto:
+-- List-Unsubscribe: <mailto:[a-zA-Z\.-]+-unsubscribe@
+-- List-Subscribe: <mailto:[a-zA-Z\.-]+-subscribe@
+-- RFC 2919 headers exist
+local function check_ml_ezmlm(task)
+ -- Mailing-List
+ local header = task:get_header('mailing-list')
+ if not header or not string.find(header, 'ezmlm$') then
+ return false
+ end
+ -- Precedence
+ header = task:get_header('precedence')
+ if not header or not string.match(header, '^bulk$') then
+ return false
+ end
+ -- Other headers
+ header = task:get_header('list-post')
+ if not header or not string.find(header, '^<mailto:') then
+ return false
+ end
+ header = task:get_header('list-help')
+ if not header or not string.find(header, '^<mailto:') then
+ return false
+ end
+ -- Subscribe and unsubscribe
+ header = task:get_header('list-subscribe')
+ if not header or not string.find(header, '<mailto:[a-zA-Z.-]+-subscribe@') then
+ return false
+ end
+ header = task:get_header('list-unsubscribe')
+ if not header or not string.find(header, '<mailto:[a-zA-Z.-]+-unsubscribe@') then
+ return false
+ end
+
+ return true
+end
+
+-- GNU Mailman
+-- Two major versions currently in use and they use slightly different headers
+-- Mailman2: https://code.launchpad.net/~mailman-coders/mailman/2.1
+-- Mailman3: https://gitlab.com/mailman/mailman
+local function check_ml_mailman(task)
+ local header = task:get_header('X-Mailman-Version')
+ if not header then
+ return false
+ end
+ local mm_version = header:match('^([23])%.')
+ if not mm_version then
+ lua_util.debugm(N, task, 'unknown Mailman version: %s', header)
+ return false
+ end
+ lua_util.debugm(N, task, 'checking Mailman %s headers', mm_version)
+
+ -- XXX Some messages may not contain Precedence, but they are rare:
+ -- http://bazaar.launchpad.net/~mailman-coders/mailman/2.1/revision/1339
+ header = task:get_header('Precedence')
+ if not header or (header ~= 'bulk' and header ~= 'list') then
+ return false
+ end
+
+ -- Mailman 3 allows to disable all List-* headers in settings, but by default it adds them.
+ -- In all other cases all Mailman message should have List-Id header
+ if not task:has_header('List-Id') then
+ return false
+ end
+
+ if mm_version == '2' then
+ -- X-BeenThere present in all Mailman2 messages
+ if not task:has_header('X-BeenThere') then
+ return false
+ end
+ -- X-List-Administrivia: is only added to messages Mailman creates and
+ -- sends out of its own accord
+ header = task:get_header('X-List-Administrivia')
+ if header and header == 'yes' then
+ -- not much elase we can check, Subjects can be changed in settings
+ return true
+ end
+ else
+ -- Mailman 3
+ -- XXX not Mailman3 admin messages have this headers, but one
+ -- which don't usually have List-* headers examined below
+ if task:has_header('List-Administrivia') then
+ return true
+ end
+ end
+
+ -- List-Archive and List-Post are optional, check other headers
+ for _, h in ipairs({ 'List-Help', 'List-Subscribe', 'List-Unsubscribe' }) do
+ header = task:get_header(h)
+ if not (header and header:find('<mailto:', 1, true)) then
+ return false
+ end
+ end
+
+ return true
+end
+
+-- Google groups detector
+-- header exists X-Google-Loop
+-- RFC 2919 headers exist
+--
+local function check_ml_googlegroup(task)
+ return task:has_header('X-Google-Loop') or task:has_header('X-Google-Group-Id')
+end
+
+-- CGP detector
+-- X-Listserver = CommuniGate Pro LIST
+-- RFC 2919 headers exist
+--
+local function check_ml_cgp(task)
+ local header = task:get_header('X-Listserver')
+
+ if not header or string.sub(header, 0, 20) ~= 'CommuniGate Pro LIST' then
+ return false
+ end
+
+ return true
+end
+
+-- RFC 2919 headers
+local function check_generic_list_headers(task)
+ local score = 0
+ local has_subscribe, has_unsubscribe
+
+ local common_list_headers = {
+ ['List-Id'] = 0.75,
+ ['List-Archive'] = 0.125,
+ ['List-Owner'] = 0.125,
+ ['List-Help'] = 0.125,
+ ['List-Post'] = 0.125,
+ ['X-Loop'] = 0.125,
+ ['List-Subscribe'] = function()
+ has_subscribe = true
+ return 0.125
+ end,
+ ['List-Unsubscribe'] = function()
+ has_unsubscribe = true
+ return 0.125
+ end,
+ ['Precedence'] = function()
+ local header = task:get_header('Precedence')
+ if header and (header == 'list' or header == 'bulk') then
+ return 0.25
+ end
+ end,
+ }
+
+ for hname, hscore in pairs(common_list_headers) do
+ if task:has_header(hname) then
+ if type(hscore) == 'number' then
+ score = score + hscore
+ lua_util.debugm(N, task, 'has %s header, score = %s', hname, score)
+ else
+ local score_change = hscore()
+ if score and score_change then
+ score = score + score_change
+ lua_util.debugm(N, task, 'has %s header, score = %s', hname, score)
+ end
+ end
+ end
+ end
+
+ if has_subscribe and has_unsubscribe then
+ score = score + 0.25
+ end
+
+ lua_util.debugm(N, task, 'final maillist score %s', score)
+ return score
+end
+
+
+-- RFC 2919 headers exist
+local function check_maillist(task)
+ local score = check_generic_list_headers(task)
+ if score >= 1 then
+ if check_ml_ezmlm(task) then
+ task:insert_result(symbol, 1, 'ezmlm')
+ elseif check_ml_mailman(task) then
+ task:insert_result(symbol, 1, 'mailman')
+ elseif check_ml_googlegroup(task) then
+ task:insert_result(symbol, 1, 'googlegroups')
+ elseif check_ml_cgp(task) then
+ task:insert_result(symbol, 1, 'cgp')
+ else
+ if score > 2 then
+ score = 2
+ end
+ task:insert_result(symbol, 0.5 * score, 'generic')
+ end
+ end
+end
+
+
+
+-- Configuration
+local opts = rspamd_config:get_all_opt('maillist')
+if opts then
+ if opts['symbol'] then
+ symbol = opts['symbol']
+ rspamd_config:register_symbol({
+ name = symbol,
+ callback = check_maillist,
+ flags = 'nice'
+ })
+ end
+end
diff --git a/src/plugins/lua/maps_stats.lua b/src/plugins/lua/maps_stats.lua
new file mode 100644
index 0000000..d418810
--- /dev/null
+++ b/src/plugins/lua/maps_stats.lua
@@ -0,0 +1,133 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ rspamd_config:add_example(nil, 'maps_stats',
+ "Stores maps statistics in Redis", [[
+maps_stats {
+ # one iteration step per 2 minutes
+ interval = 2m;
+ # how many elements to store in Redis
+ count = 1k;
+ # common prefix for elements
+ prefix = 'rm_';
+}
+]])
+end
+
+local redis_params
+local lua_util = require "lua_util"
+local rspamd_logger = require "rspamd_logger"
+local lua_redis = require "lua_redis"
+local N = "maps_stats"
+
+local settings = {
+ interval = 120, -- one iteration step per 2 minutes
+ count = 1000, -- how many elements to store in Redis
+ prefix = 'rm_', -- common prefix for elements
+}
+
+local function process_map(map, ev_base, _)
+ if map:get_nelts() > 0 and map:get_uri() ~= 'static' then
+ local key = settings.prefix .. map:get_uri()
+
+ local function redis_zrange_cb(err, data)
+ if err then
+ rspamd_logger.errx(rspamd_config, 'cannot delete extra elements in %s: %s',
+ key, err)
+ elseif data then
+ rspamd_logger.infox(rspamd_config, 'cleared %s elements from %s',
+ data, key)
+ end
+ end
+ local function redis_card_cb(err, data)
+ if err then
+ rspamd_logger.errx(rspamd_config, 'cannot get number of elements in %s: %s',
+ key, err)
+ elseif data then
+ if settings.count > 0 and tonumber(data) > settings.count then
+ lua_redis.rspamd_redis_make_request_taskless(ev_base,
+ rspamd_config,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_zrange_cb, --callback
+ 'ZREMRANGEBYRANK', -- command
+ { key, '0', tostring(-(settings.count) - 1) } -- arguments
+ )
+ end
+ end
+ end
+ local ret, conn, _ = lua_redis.rspamd_redis_make_request_taskless(ev_base,
+ rspamd_config,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_card_cb, --callback
+ 'ZCARD', -- command
+ { key } -- arguments
+ )
+
+ if ret and conn then
+ local stats = map:get_stats(true)
+ for k, s in pairs(stats) do
+ if s > 0 then
+ conn:add_cmd('ZINCRBY', { key, tostring(s), k })
+ end
+ end
+ end
+ end
+end
+
+if not lua_util.check_experimental(N) then
+ return
+end
+
+local opts = rspamd_config:get_all_opt(N)
+
+if opts then
+ for k, v in pairs(opts) do
+ settings[k] = v
+ end
+end
+
+redis_params = lua_redis.parse_redis_server(N, opts)
+-- XXX, this is a poor approach as not all maps are defined here...
+local tmaps = rspamd_config:get_maps()
+for _, m in ipairs(tmaps) do
+ if m:get_uri() ~= 'static' then
+ lua_redis.register_prefix(settings.prefix .. m:get_uri(), N,
+ 'Maps stats data', {
+ type = 'zlist',
+ persistent = true,
+ })
+ end
+end
+
+if redis_params then
+ rspamd_config:add_on_load(function(_, ev_base, worker)
+ local maps = rspamd_config:get_maps()
+
+ for _, m in ipairs(maps) do
+ rspamd_config:add_periodic(ev_base,
+ settings['interval'],
+ function()
+ process_map(m, ev_base, worker)
+ return true
+ end, true)
+ end
+ end)
+end \ No newline at end of file
diff --git a/src/plugins/lua/metadata_exporter.lua b/src/plugins/lua/metadata_exporter.lua
new file mode 100644
index 0000000..7b353b8
--- /dev/null
+++ b/src/plugins/lua/metadata_exporter.lua
@@ -0,0 +1,707 @@
+--[[
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- A plugin that pushes metadata (or whole messages) to external services
+
+local redis_params
+local lua_util = require "lua_util"
+local rspamd_http = require "rspamd_http"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local rspamd_tcp = require "rspamd_tcp"
+local ucl = require "ucl"
+local E = {}
+local N = 'metadata_exporter'
+local HOSTNAME = rspamd_util.get_hostname()
+
+local settings = {
+ pusher_enabled = {},
+ pusher_format = {},
+ pusher_select = {},
+ mime_type = 'text/plain',
+ defer = false,
+ mail_from = '',
+ mail_to = 'postmaster@localhost',
+ helo = 'rspamd',
+ email_template = [[From: "Rspamd" <$mail_from>
+To: $mail_to
+Subject: Spam alert
+Date: $date
+MIME-Version: 1.0
+Message-ID: <$our_message_id>
+Content-type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+Authenticated username: $user
+IP: $ip
+Queue ID: $qid
+SMTP FROM: $from
+SMTP RCPT: $rcpt
+MIME From: $header_from
+MIME To: $header_to
+MIME Date: $header_date
+Subject: $header_subject
+Message-ID: $message_id
+Action: $action
+Score: $score
+Symbols: $symbols]],
+}
+
+local function get_general_metadata(task, flatten, no_content)
+ local r = {}
+ local ip = task:get_from_ip()
+ if ip and ip:is_valid() then
+ r.ip = tostring(ip)
+ else
+ r.ip = 'unknown'
+ end
+ r.user = task:get_user() or 'unknown'
+ r.qid = task:get_queue_id() or 'unknown'
+ r.subject = task:get_subject() or 'unknown'
+ r.action = task:get_metric_action()
+ r.rspamd_server = HOSTNAME
+
+ local s = task:get_metric_score()[1]
+ r.score = flatten and string.format('%.2f', s) or s
+
+ local fuzzy = task:get_mempool():get_variable("fuzzy_hashes", "fstrings")
+ if fuzzy and #fuzzy > 0 then
+ local fz = {}
+ for _, h in ipairs(fuzzy) do
+ table.insert(fz, h)
+ end
+ if not flatten then
+ r.fuzzy = fz
+ else
+ r.fuzzy = table.concat(fz, ', ')
+ end
+ else
+ if not flatten then
+ r.fuzzy = {}
+ else
+ r.fuzzy = ''
+ end
+ end
+
+ local rcpt = task:get_recipients('smtp')
+ if rcpt then
+ local l = {}
+ for _, a in ipairs(rcpt) do
+ table.insert(l, a['addr'])
+ end
+ if not flatten then
+ r.rcpt = l
+ else
+ r.rcpt = table.concat(l, ', ')
+ end
+ else
+ r.rcpt = 'unknown'
+ end
+ local from = task:get_from('smtp')
+ if ((from or E)[1] or E).addr then
+ r.from = from[1].addr
+ else
+ r.from = 'unknown'
+ end
+ local syminf = task:get_symbols_all()
+ if flatten then
+ local l = {}
+ for _, sym in ipairs(syminf) do
+ local txt
+ if sym.options then
+ local topt = table.concat(sym.options, ', ')
+ txt = sym.name .. '(' .. string.format('%.2f', sym.score) .. ')' .. ' [' .. topt .. ']'
+ else
+ txt = sym.name .. '(' .. string.format('%.2f', sym.score) .. ')'
+ end
+ table.insert(l, txt)
+ end
+ r.symbols = table.concat(l, '\n\t')
+ else
+ r.symbols = syminf
+ end
+ local function process_header(name)
+ local hdr = task:get_header_full(name)
+ if hdr then
+ local l = {}
+ for _, h in ipairs(hdr) do
+ table.insert(l, h.decoded)
+ end
+ if not flatten then
+ return l
+ else
+ return table.concat(l, '\n')
+ end
+ else
+ return 'unknown'
+ end
+ end
+
+ local scan_real = task:get_scan_time()
+ scan_real = math.floor(scan_real * 1000)
+ if scan_real < 0 then
+ rspamd_logger.messagex(task,
+ 'clock skew detected for message: %s ms real sca time (reset to 0)',
+ scan_real)
+ scan_real = 0
+ end
+
+ r.scan_time = scan_real
+ local content = task:get_content()
+ r.size = content and content:len() or 0
+
+ if not no_content then
+ r.header_from = process_header('from')
+ r.header_to = process_header('to')
+ r.header_subject = process_header('subject')
+ r.header_date = process_header('date')
+ r.message_id = task:get_message_id()
+ end
+ return r
+end
+
+local formatters = {
+ default = function(task)
+ return task:get_content(), {}
+ end,
+ email_alert = function(task, rule, extra)
+ local meta = get_general_metadata(task, true)
+ local display_emails = {}
+ local mail_targets = {}
+ meta.mail_from = rule.mail_from or settings.mail_from
+ local mail_rcpt = rule.mail_to or settings.mail_to
+ if type(mail_rcpt) ~= 'table' then
+ table.insert(display_emails, string.format('<%s>', mail_rcpt))
+ table.insert(mail_targets, mail_rcpt)
+ else
+ for _, e in ipairs(mail_rcpt) do
+ table.insert(display_emails, string.format('<%s>', e))
+ table.insert(mail_targets, e)
+ end
+ end
+ if rule.email_alert_sender then
+ local x = task:get_from('smtp')
+ if x and string.len(x[1].addr) > 0 then
+ table.insert(mail_targets, x)
+ table.insert(display_emails, string.format('<%s>', x[1].addr))
+ end
+ end
+ if rule.email_alert_user then
+ local x = task:get_user()
+ if x then
+ table.insert(mail_targets, x)
+ table.insert(display_emails, string.format('<%s>', x))
+ end
+ end
+ if rule.email_alert_recipients then
+ local x = task:get_recipients('smtp')
+ if x then
+ for _, e in ipairs(x) do
+ if string.len(e.addr) > 0 then
+ table.insert(mail_targets, e.addr)
+ table.insert(display_emails, string.format('<%s>', e.addr))
+ end
+ end
+ end
+ end
+ meta.mail_to = table.concat(display_emails, ', ')
+ meta.our_message_id = rspamd_util.random_hex(12) .. '@rspamd'
+ meta.date = rspamd_util.time_to_string(rspamd_util.get_time())
+ return lua_util.template(rule.email_template or settings.email_template, meta), { mail_targets = mail_targets }
+ end,
+ json = function(task)
+ return ucl.to_format(get_general_metadata(task), 'json-compact')
+ end
+}
+
+local function is_spam(action)
+ return (action == 'reject' or action == 'add header' or action == 'rewrite subject')
+end
+
+local selectors = {
+ default = function(task)
+ return true
+ end,
+ is_spam = function(task)
+ local action = task:get_metric_action()
+ return is_spam(action)
+ end,
+ is_spam_authed = function(task)
+ if not task:get_user() then
+ return false
+ end
+ local action = task:get_metric_action()
+ return is_spam(action)
+ end,
+ is_reject = function(task)
+ local action = task:get_metric_action()
+ return (action == 'reject')
+ end,
+ is_reject_authed = function(task)
+ if not task:get_user() then
+ return false
+ end
+ local action = task:get_metric_action()
+ return (action == 'reject')
+ end,
+ is_not_soft_reject = function(task)
+ local action = task:get_metric_action()
+ return (action ~= 'soft reject')
+ end,
+}
+
+local function maybe_defer(task, rule)
+ if rule.defer then
+ rspamd_logger.warnx(task, 'deferring message')
+ task:set_pre_result('soft reject', 'deferred', N)
+ end
+end
+
+local pushers = {
+ redis_pubsub = function(task, formatted, rule)
+ local _, ret, upstream
+ local function redis_pub_cb(err)
+ if err then
+ rspamd_logger.errx(task, 'got error %s when publishing on server %s',
+ err, upstream:get_addr())
+ return maybe_defer(task, rule)
+ end
+ return true
+ end
+ ret, _, upstream = rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ nil, -- hash key
+ true, -- is write
+ redis_pub_cb, --callback
+ 'PUBLISH', -- command
+ { rule.channel, formatted } -- arguments
+ )
+ if not ret then
+ rspamd_logger.errx(task, 'error connecting to redis')
+ maybe_defer(task, rule)
+ end
+ end,
+ http = function(task, formatted, rule)
+ local function http_callback(err, code)
+ local valid_status = { 200, 201, 202, 204 }
+
+ if err then
+ rspamd_logger.errx(task, 'got error %s in http callback', err)
+ return maybe_defer(task, rule)
+ end
+ for _, v in ipairs(valid_status) do
+ if v == code then
+ return true
+ end
+ end
+ rspamd_logger.errx(task, 'got unexpected http status: %s', code)
+ return maybe_defer(task, rule)
+ end
+ local hdrs = {}
+ if rule.meta_headers then
+ local gm = get_general_metadata(task, false, true)
+ local pfx = rule.meta_header_prefix or 'X-Rspamd-'
+ for k, v in pairs(gm) do
+ if type(v) == 'table' then
+ hdrs[pfx .. k] = ucl.to_format(v, 'json-compact')
+ else
+ hdrs[pfx .. k] = v
+ end
+ end
+ end
+ rspamd_http.request({
+ task = task,
+ url = rule.url,
+ user = rule.user,
+ password = rule.password,
+ body = formatted,
+ callback = http_callback,
+ mime_type = rule.mime_type or settings.mime_type,
+ headers = hdrs,
+ })
+ end,
+ send_mail = function(task, formatted, rule, extra)
+ local lua_smtp = require "lua_smtp"
+ local function sendmail_cb(ret, err)
+ if not ret then
+ rspamd_logger.errx(task, 'SMTP export error: %s', err)
+ maybe_defer(task, rule)
+ end
+ end
+
+ lua_smtp.sendmail({
+ task = task,
+ host = rule.smtp,
+ port = rule.smtp_port or settings.smtp_port or 25,
+ from = rule.mail_from or settings.mail_from,
+ recipients = extra.mail_targets or rule.mail_to or settings.mail_to,
+ helo = rule.helo or settings.helo,
+ timeout = rule.timeout or settings.timeout,
+ }, formatted, sendmail_cb)
+ end,
+ json_raw_tcp = function(task, formatted, rule)
+ local function json_raw_tcp_callback(err, code)
+ if err then
+ rspamd_logger.errx(task, 'got error %s in json_raw_tcp callback', err)
+ return maybe_defer(task, rule)
+ end
+ return true
+ end
+ rspamd_tcp.request({
+ task=task,
+ host=rule.host,
+ port=rule.port,
+ data=formatted,
+ callback=json_raw_tcp_callback,
+ read=false,
+ })
+ end,
+}
+
+local opts = rspamd_config:get_all_opt(N)
+if not opts then
+ return
+end
+local process_settings = {
+ select = function(val)
+ selectors.custom = assert(load(val))()
+ end,
+ format = function(val)
+ formatters.custom = assert(load(val))()
+ end,
+ push = function(val)
+ pushers.custom = assert(load(val))()
+ end,
+ custom_push = function(val)
+ if type(val) == 'table' then
+ for k, v in pairs(val) do
+ pushers[k] = assert(load(v))()
+ end
+ end
+ end,
+ custom_select = function(val)
+ if type(val) == 'table' then
+ for k, v in pairs(val) do
+ selectors[k] = assert(load(v))()
+ end
+ end
+ end,
+ custom_format = function(val)
+ if type(val) == 'table' then
+ for k, v in pairs(val) do
+ formatters[k] = assert(load(v))()
+ end
+ end
+ end,
+ pusher_enabled = function(val)
+ if type(val) == 'string' then
+ if pushers[val] then
+ settings.pusher_enabled[val] = true
+ else
+ rspamd_logger.errx(rspamd_config, 'Pusher type: %s is invalid', val)
+ end
+ elseif type(val) == 'table' then
+ for _, v in ipairs(val) do
+ if pushers[v] then
+ settings.pusher_enabled[v] = true
+ else
+ rspamd_logger.errx(rspamd_config, 'Pusher type: %s is invalid', val)
+ end
+ end
+ end
+ end,
+}
+for k, v in pairs(opts) do
+ local f = process_settings[k]
+ if f then
+ f(opts[k])
+ else
+ settings[k] = v
+ end
+end
+if type(settings.rules) ~= 'table' then
+ -- Legacy config
+ settings.rules = {}
+ if not next(settings.pusher_enabled) then
+ if pushers.custom then
+ rspamd_logger.infox(rspamd_config, 'Custom pusher implicitly enabled')
+ settings.pusher_enabled.custom = true
+ else
+ -- Check legacy options
+ if settings.url then
+ rspamd_logger.warnx(rspamd_config, 'HTTP pusher implicitly enabled')
+ settings.pusher_enabled.http = true
+ end
+ if settings.channel then
+ rspamd_logger.warnx(rspamd_config, 'Redis Pubsub pusher implicitly enabled')
+ settings.pusher_enabled.redis_pubsub = true
+ end
+ if settings.smtp and settings.mail_to then
+ rspamd_logger.warnx(rspamd_config, 'SMTP pusher implicitly enabled')
+ settings.pusher_enabled.send_mail = true
+ end
+ end
+ end
+ if not next(settings.pusher_enabled) then
+ rspamd_logger.errx(rspamd_config, 'No push backend enabled')
+ return
+ end
+ if settings.formatter then
+ settings.format = formatters[settings.formatter]
+ if not settings.format then
+ rspamd_logger.errx(rspamd_config, 'No such formatter: %s', settings.formatter)
+ return
+ end
+ end
+ if settings.selector then
+ settings.select = selectors[settings.selector]
+ if not settings.select then
+ rspamd_logger.errx(rspamd_config, 'No such selector: %s', settings.selector)
+ return
+ end
+ end
+ for k in pairs(settings.pusher_enabled) do
+ local formatter = settings.pusher_format[k]
+ local selector = settings.pusher_select[k]
+ if not formatter then
+ settings.pusher_format[k] = settings.formatter or 'default'
+ rspamd_logger.infox(rspamd_config, 'Using default formatter for %s pusher', k)
+ else
+ if not formatters[formatter] then
+ rspamd_logger.errx(rspamd_config, 'No such formatter: %s - disabling %s', formatter, k)
+ settings.pusher_enabled.k = nil
+ end
+ end
+ if not selector then
+ settings.pusher_select[k] = settings.selector or 'default'
+ rspamd_logger.infox(rspamd_config, 'Using default selector for %s pusher', k)
+ else
+ if not selectors[selector] then
+ rspamd_logger.errx(rspamd_config, 'No such selector: %s - disabling %s', selector, k)
+ settings.pusher_enabled.k = nil
+ end
+ end
+ end
+ if settings.pusher_enabled.redis_pubsub then
+ redis_params = rspamd_parse_redis_server(N)
+ if not redis_params then
+ rspamd_logger.errx(rspamd_config, 'No redis servers are specified')
+ settings.pusher_enabled.redis_pubsub = nil
+ else
+ local r = {}
+ r.backend = 'redis_pubsub'
+ r.channel = settings.channel
+ r.defer = settings.defer
+ r.selector = settings.pusher_select.redis_pubsub
+ r.formatter = settings.pusher_format.redis_pubsub
+ r.timeout = redis_params.timeout
+ settings.rules[r.backend:upper()] = r
+ end
+ end
+ if settings.pusher_enabled.http then
+ if not settings.url then
+ rspamd_logger.errx(rspamd_config, 'No URL is specified')
+ settings.pusher_enabled.http = nil
+ else
+ local r = {}
+ r.backend = 'http'
+ r.url = settings.url
+ r.mime_type = settings.mime_type
+ r.defer = settings.defer
+ r.selector = settings.pusher_select.http
+ r.formatter = settings.pusher_format.http
+ r.timeout = settings.timeout or 0.0
+ settings.rules[r.backend:upper()] = r
+ end
+ end
+ if settings.pusher_enabled.send_mail then
+ if not (settings.mail_to and settings.smtp) then
+ rspamd_logger.errx(rspamd_config, 'No mail_to and/or smtp setting is specified')
+ settings.pusher_enabled.send_mail = nil
+ else
+ local r = {}
+ r.backend = 'send_mail'
+ r.mail_to = settings.mail_to
+ r.mail_from = settings.mail_from
+ r.helo = settings.hello
+ r.smtp = settings.smtp
+ r.smtp_port = settings.smtp_port
+ r.email_template = settings.email_template
+ r.defer = settings.defer
+ r.timeout = settings.timeout or 0.0
+ r.selector = settings.pusher_select.send_mail
+ r.formatter = settings.pusher_format.send_mail
+ settings.rules[r.backend:upper()] = r
+ end
+ end
+ if settings.pusher_enabled.json_raw_tcp then
+ if not (settings.host and settings.port) then
+ rspamd_logger.errx(rspamd_config, 'No host and/or port is specified')
+ settings.pusher_enabled.json_raw_tcp = nil
+ else
+ local r = {}
+ r.backend = 'json_raw_tcp'
+ r.host = settings.host
+ r.port = settings.port
+ r.defer = settings.defer
+ r.selector = settings.pusher_select.json_raw_tcp
+ r.formatter = settings.pusher_format.json_raw_tcp
+ settings.rules[r.backend:upper()] = r
+ end
+ end
+ if not next(settings.pusher_enabled) then
+ rspamd_logger.errx(rspamd_config, 'No push backend enabled')
+ return
+ end
+elseif not next(settings.rules) then
+ lua_util.debugm(N, rspamd_config, 'No rules enabled')
+ return
+end
+if not settings.rules or not next(settings.rules) then
+ rspamd_logger.errx(rspamd_config, 'No rules enabled')
+ return
+end
+local backend_required_elements = {
+ http = {
+ 'url',
+ },
+ smtp = {
+ 'mail_to',
+ 'smtp',
+ },
+ redis_pubsub = {
+ 'channel',
+ },
+ json_raw_tcp = {
+ 'host',
+ 'port',
+ },
+}
+local check_element = {
+ selector = function(k, v)
+ if not selectors[v] then
+ rspamd_logger.errx(rspamd_config, 'Rule %s has invalid selector %s', k, v)
+ return false
+ else
+ return true
+ end
+ end,
+ formatter = function(k, v)
+ if not formatters[v] then
+ rspamd_logger.errx(rspamd_config, 'Rule %s has invalid formatter %s', k, v)
+ return false
+ else
+ return true
+ end
+ end,
+}
+local backend_check = {
+ default = function(k, rule)
+ local reqset = backend_required_elements[rule.backend]
+ if reqset then
+ for _, e in ipairs(reqset) do
+ if not rule[e] then
+ rspamd_logger.errx(rspamd_config, 'Rule %s misses required setting %s', k, e)
+ settings.rules[k] = nil
+ end
+ end
+ end
+ for sett, v in pairs(rule) do
+ local f = check_element[sett]
+ if f then
+ if not f(sett, v) then
+ settings.rules[k] = nil
+ end
+ end
+ end
+ end,
+}
+backend_check.redis_pubsub = function(k, rule)
+ if not redis_params then
+ redis_params = rspamd_parse_redis_server(N)
+ end
+ if not redis_params then
+ rspamd_logger.errx(rspamd_config, 'No redis servers are specified')
+ settings.rules[k] = nil
+ else
+ backend_check.default(k, rule)
+ rule.timeout = redis_params.timeout
+ end
+end
+setmetatable(backend_check, {
+ __index = function()
+ return backend_check.default
+ end,
+})
+for k, v in pairs(settings.rules) do
+ if type(v) == 'table' then
+ local backend = v.backend
+ if not backend then
+ rspamd_logger.errx(rspamd_config, 'Rule %s has no backend', k)
+ settings.rules[k] = nil
+ elseif not pushers[backend] then
+ rspamd_logger.errx(rspamd_config, 'Rule %s has invalid backend %s', k, backend)
+ settings.rules[k] = nil
+ else
+ local f = backend_check[backend]
+ f(k, v)
+ end
+ else
+ rspamd_logger.errx(rspamd_config, 'Rule %s has bad type: %s', k, type(v))
+ settings.rules[k] = nil
+ end
+end
+
+local function gen_exporter(rule)
+ return function(task)
+ if task:has_flag('skip') then
+ return
+ end
+ local selector = rule.selector or 'default'
+ local selected = selectors[selector](task)
+ if selected then
+ lua_util.debugm(N, task, 'Message selected for processing')
+ local formatter = rule.formatter or 'default'
+ local formatted, extra = formatters[formatter](task, rule)
+ if formatted then
+ pushers[rule.backend](task, formatted, rule, extra)
+ else
+ lua_util.debugm(N, task, 'Formatter [%s] returned non-truthy value [%s]', formatter, formatted)
+ end
+ else
+ lua_util.debugm(N, task, 'Selector [%s] returned non-truthy value [%s]', selector, selected)
+ end
+ end
+end
+
+if not next(settings.rules) then
+ rspamd_logger.errx(rspamd_config, 'No rules enabled')
+ lua_util.disable_module(N, "config")
+end
+for k, r in pairs(settings.rules) do
+ rspamd_config:register_symbol({
+ name = 'EXPORT_METADATA_' .. k,
+ type = 'idempotent',
+ callback = gen_exporter(r),
+ flags = 'empty,explicit_disable,ignore_passthrough',
+ augmentations = { string.format("timeout=%f", r.timeout or 0.0) }
+ })
+end
diff --git a/src/plugins/lua/metric_exporter.lua b/src/plugins/lua/metric_exporter.lua
new file mode 100644
index 0000000..7588551
--- /dev/null
+++ b/src/plugins/lua/metric_exporter.lua
@@ -0,0 +1,252 @@
+--[[
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]] --
+
+if confighelp then
+ return
+end
+
+local N = 'metric_exporter'
+local logger = require "rspamd_logger"
+local mempool = require "rspamd_mempool"
+local util = require "rspamd_util"
+local tcp = require "rspamd_tcp"
+local lua_util = require "lua_util"
+
+local pool
+local settings = {
+ interval = 120,
+ timeout = 15,
+ statefile = string.format('%s/%s', rspamd_paths['DBDIR'], 'metric_exporter_last_push')
+}
+
+local VAR_NAME = 'metric_exporter_last_push'
+
+local valid_metrics = {
+ 'actions.add header',
+ 'actions.greylist',
+ 'actions.no action',
+ 'actions.reject',
+ 'actions.rewrite subject',
+ 'actions.soft reject',
+ 'bytes_allocated',
+ 'chunks_allocated',
+ 'chunks_freed',
+ 'chunks_oversized',
+ 'connections',
+ 'control_connections',
+ 'ham_count',
+ 'learned',
+ 'pools_allocated',
+ 'pools_freed',
+ 'scanned',
+ 'shared_chunks_allocated',
+ 'spam_count',
+}
+
+local function validate_metrics(settings_metrics)
+ if type(settings_metrics) ~= 'table' or #settings_metrics == 0 then
+ logger.errx(rspamd_config, 'No metrics specified for collection')
+ return false
+ end
+ for _, v in ipairs(settings_metrics) do
+ local isvalid = false
+ for _, vm in ipairs(valid_metrics) do
+ if vm == v then
+ isvalid = true
+ break
+ end
+ end
+ if not isvalid then
+ logger.errx('Invalid metric: %s', v)
+ return false
+ end
+ local split = rspamd_str_split(v, '.')
+ if #split > 2 then
+ logger.errx('Too many dots in metric name: %s', v)
+ return false
+ end
+ end
+ return true
+end
+
+local function load_defaults(defaults)
+ for k, v in pairs(defaults) do
+ if settings[k] == nil then
+ settings[k] = v
+ end
+ end
+end
+
+local function graphite_config()
+ load_defaults({
+ host = 'localhost',
+ port = 2003,
+ metric_prefix = 'rspamd'
+ })
+ return validate_metrics(settings['metrics'])
+end
+
+local function graphite_push(kwargs)
+ local stamp
+ if kwargs['time'] then
+ stamp = math.floor(kwargs['time'])
+ else
+ stamp = math.floor(util.get_time())
+ end
+ local metrics_str = {}
+ for _, v in ipairs(settings['metrics']) do
+ local mvalue
+ local mname = string.format('%s.%s', settings['metric_prefix'], v:gsub(' ', '_'))
+ local split = rspamd_str_split(v, '.')
+ if #split == 1 then
+ mvalue = kwargs['stats'][v]
+ elseif #split == 2 then
+ mvalue = kwargs['stats'][split[1]][split[2]]
+ end
+ table.insert(metrics_str, string.format('%s %s %s', mname, mvalue, stamp))
+ end
+
+ metrics_str = table.concat(metrics_str, '\n')
+
+ tcp.request({
+ ev_base = kwargs['ev_base'],
+ config = rspamd_config,
+ host = settings['host'],
+ port = settings['port'],
+ timeout = settings['timeout'],
+ read = false,
+ data = {
+ metrics_str, '\n',
+ },
+ callback = (function(err)
+ if err then
+ logger.errx('Push failed: %1', err)
+ return
+ end
+ pool:set_variable(VAR_NAME, stamp)
+ end)
+ })
+end
+
+local backends = {
+ graphite = {
+ configure = graphite_config,
+ push = graphite_push,
+ },
+}
+
+local function configure_metric_exporter()
+ local opts = rspamd_config:get_all_opt(N)
+ local be = opts['backend']
+ if not be then
+ logger.debugm(N, rspamd_config, 'Backend is unspecified')
+ return
+ end
+ if not backends[be] then
+ logger.errx(rspamd_config, 'Backend is invalid: ' .. be)
+ return false
+ end
+ for k, v in pairs(opts) do
+ settings[k] = v
+ end
+ return backends[be]['configure']()
+end
+
+if not configure_metric_exporter() then
+ lua_util.disable_module(N, "config")
+ return
+end
+
+rspamd_config:add_on_load(function(_, ev_base, worker)
+ -- Exit unless we're the first 'controller' worker
+ if not worker:is_primary_controller() then
+ return
+ end
+ -- Persist mempool variable to statefile on shutdown
+ pool = mempool.create()
+ rspamd_config:register_finish_script(function()
+ local stamp = pool:get_variable(VAR_NAME, 'double')
+ if not stamp then
+ logger.warn('No last metric exporter push to persist to disk')
+ return
+ end
+ local f, err = io.open(settings['statefile'], 'w')
+ if err then
+ logger.errx('Unable to write statefile to disk: %s', err)
+ return
+ end
+ if f then
+ f:write(pool:get_variable(VAR_NAME, 'double'))
+ f:close()
+ end
+ pool:destroy()
+ end)
+ -- Push metrics to backend
+ local function push_metrics(time)
+ logger.infox('Pushing metrics to %s backend', settings['backend'])
+ local args = {
+ ev_base = ev_base,
+ stats = worker:get_stat(),
+ }
+ if time then
+ table.insert(args, time)
+ end
+ backends[settings['backend']]['push'](args)
+ end
+ -- Push metrics at regular intervals
+ local function schedule_regular_push()
+ rspamd_config:add_periodic(ev_base, settings['interval'], function()
+ push_metrics()
+ return true
+ end)
+ end
+ -- Push metrics to backend and reschedule check
+ local function schedule_intermediate_push(when)
+ rspamd_config:add_periodic(ev_base, when, function()
+ push_metrics()
+ schedule_regular_push()
+ return false
+ end)
+ end
+ -- Try read statefile on startup
+ local stamp
+ local f, err = io.open(settings['statefile'], 'r')
+ if err then
+ logger.errx('Failed to open statefile: %s', err)
+ end
+ if f then
+ io.input(f)
+ stamp = tonumber(io.read())
+ pool:set_variable(VAR_NAME, stamp)
+ end
+ if not stamp then
+ logger.debugm(N, rspamd_config, 'No state found - pushing stats immediately')
+ push_metrics()
+ schedule_regular_push()
+ return
+ end
+ local time = util.get_time()
+ local delta = stamp - time + settings['interval']
+ if delta <= 0 then
+ logger.debugm(N, rspamd_config, 'Last push is too old - pushing stats immediately')
+ push_metrics(time)
+ schedule_regular_push()
+ return
+ end
+ logger.debugm(N, rspamd_config, 'Scheduling next push in %s seconds', delta)
+ schedule_intermediate_push(delta)
+end)
diff --git a/src/plugins/lua/mid.lua b/src/plugins/lua/mid.lua
new file mode 100644
index 0000000..b8650c8
--- /dev/null
+++ b/src/plugins/lua/mid.lua
@@ -0,0 +1,123 @@
+--[[
+Copyright (c) 2016, Alexander Moisseev <moiseev@mezonplus.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[
+MID plugin - suppress INVALID_MSGID and MISSING_MID for messages originating
+from listed valid DKIM domains with missed or known proprietary Message-IDs
+]]--
+
+if confighelp then
+ return
+end
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_regexp = require "rspamd_regexp"
+local lua_util = require "lua_util"
+local N = "mid"
+
+local settings = {
+ url = '',
+ symbol_known_mid = 'KNOWN_MID',
+ symbol_known_no_mid = 'KNOWN_NO_MID',
+ symbol_invalid_msgid = 'INVALID_MSGID',
+ symbol_missing_mid = 'MISSING_MID',
+ symbol_dkim_allow = 'R_DKIM_ALLOW',
+ csymbol_invalid_msgid_allowed = 'INVALID_MSGID_ALLOWED',
+ csymbol_missing_mid_allowed = 'MISSING_MID_ALLOWED',
+}
+
+local map
+
+local E = {}
+
+local function known_mid_cb(task)
+ local re = {}
+ local header = task:get_header('Message-Id')
+ local das = task:get_symbol(settings['symbol_dkim_allow'])
+ if ((das or E)[1] or E).options then
+ for _, dkim_domain in ipairs(das[1]['options']) do
+ if dkim_domain then
+ local v = map:get_key(dkim_domain:match "[^:]+")
+ if v then
+ if v == '' then
+ if not header then
+ task:insert_result(settings['symbol_known_no_mid'], 1, dkim_domain)
+ return
+ end
+ else
+ re[dkim_domain] = rspamd_regexp.create_cached(v)
+ if header and re[dkim_domain] and re[dkim_domain]:match(header) then
+ task:insert_result(settings['symbol_known_mid'], 1, dkim_domain)
+ return
+ end
+ end
+ end
+ end
+ end
+ end
+end
+
+local opts = rspamd_config:get_all_opt('mid')
+if opts then
+ for k, v in pairs(opts) do
+ settings[k] = v
+ end
+
+ if not opts.source then
+ rspamd_logger.infox(rspamd_config, 'mid module requires "source" parameter')
+ lua_util.disable_module(N, "config")
+ return
+ end
+
+ map = rspamd_config:add_map {
+ url = opts.source,
+ description = "Message-IDs map",
+ type = 'map'
+ }
+ if map then
+ local id = rspamd_config:register_symbol({
+ name = 'KNOWN_MID_CALLBACK',
+ type = 'callback',
+ group = 'mid',
+ callback = known_mid_cb
+ })
+ rspamd_config:register_symbol({
+ name = settings['symbol_known_mid'],
+ parent = id,
+ group = 'mid',
+ type = 'virtual'
+ })
+ rspamd_config:register_symbol({
+ name = settings['symbol_known_no_mid'],
+ parent = id,
+ group = 'mid',
+ type = 'virtual'
+ })
+ rspamd_config:add_composite(settings['csymbol_invalid_msgid_allowed'],
+ string.format('~%s & ^%s',
+ settings['symbol_known_mid'],
+ settings['symbol_invalid_msgid']))
+ rspamd_config:add_composite(settings['csymbol_missing_mid_allowed'],
+ string.format('~%s & ^%s',
+ settings['symbol_known_no_mid'],
+ settings['symbol_missing_mid']))
+
+ rspamd_config:register_dependency('KNOWN_MID_CALLBACK', 'DKIM_CHECK')
+ else
+ rspamd_logger.infox(rspamd_config, 'source is not a valid map definition, disabling module')
+ lua_util.disable_module(N, "config")
+ end
+end
diff --git a/src/plugins/lua/milter_headers.lua b/src/plugins/lua/milter_headers.lua
new file mode 100644
index 0000000..b53a454
--- /dev/null
+++ b/src/plugins/lua/milter_headers.lua
@@ -0,0 +1,762 @@
+--[[
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- A plugin that provides common header manipulations
+
+local logger = require "rspamd_logger"
+local util = require "rspamd_util"
+local N = 'milter_headers'
+local lua_util = require "lua_util"
+local lua_maps = require "lua_maps"
+local lua_mime = require "lua_mime"
+local ts = require("tableshape").types
+local E = {}
+
+local HOSTNAME = util.get_hostname()
+
+local settings = {
+ remove_upstream_spam_flag = true;
+ skip_local = true,
+ skip_authenticated = true,
+ skip_all = false,
+ local_headers = {},
+ authenticated_headers = {},
+ headers_modify_mode = 'compat', -- To avoid compatibility issues on upgrade
+ default_headers_order = nil, -- Insert at the end (set 1 to insert just after the first received)
+ routines = {
+ ['remove-headers'] = {
+ headers = {},
+ },
+ ['add-headers'] = {
+ headers = {},
+ remove = 0,
+ },
+ ['remove-header'] = {
+ remove = 0,
+ },
+ ['x-spamd-result'] = {
+ header = 'X-Spamd-Result',
+ remove = 0,
+ stop_chars = ' ',
+ sort_by = 'score',
+ },
+ ['x-rspamd-server'] = {
+ header = 'X-Rspamd-Server',
+ remove = 0,
+ hostname = nil, -- Get the local computer host name
+ },
+ ['x-rspamd-queue-id'] = {
+ header = 'X-Rspamd-Queue-Id',
+ remove = 0,
+ },
+ ['x-rspamd-pre-result'] = {
+ header = 'X-Rspamd-Pre-Result',
+ remove = 0,
+ },
+ ['x-rspamd-action'] = {
+ header = 'X-Rspamd-Action',
+ remove = 0,
+ },
+ ['remove-spam-flag'] = {
+ header = 'X-Spam',
+ },
+ ['spam-header'] = {
+ header = 'Deliver-To',
+ value = 'Junk',
+ remove = 0,
+ },
+ ['x-virus'] = {
+ header = 'X-Virus',
+ remove = 0,
+ status_clean = nil,
+ status_infected = nil,
+ status_fail = nil,
+ symbols_fail = {},
+ symbols = {}, -- needs config
+ },
+ ['x-os-fingerprint'] = {
+ header = 'X-OS-Fingerprint',
+ remove = 0,
+ },
+ ['x-spamd-bar'] = {
+ header = 'X-Spamd-Bar',
+ positive = '+',
+ negative = '-',
+ neutral = '/',
+ remove = 0,
+ },
+ ['x-spam-level'] = {
+ header = 'X-Spam-Level',
+ char = '*',
+ remove = 0,
+ },
+ ['x-spam-status'] = {
+ header = 'X-Spam-Status',
+ remove = 0,
+ },
+ ['authentication-results'] = {
+ header = 'Authentication-Results',
+ remove = 0,
+ add_smtp_user = true,
+ stop_chars = ';',
+ },
+ ['stat-signature'] = {
+ header = 'X-Stat-Signature',
+ remove = 0,
+ },
+ ['fuzzy-hashes'] = {
+ header = 'X-Rspamd-Fuzzy',
+ },
+ },
+}
+
+local active_routines = {}
+local custom_routines = {}
+
+local function milter_headers(task)
+
+ -- Used to override wanted stuff by means of settings
+ local settings_override = false
+
+ local function skip_wanted(hdr)
+ if settings_override then
+ return true
+ end
+ -- Normal checks
+ local function match_extended_headers_rcpt()
+ local rcpts = task:get_recipients('smtp')
+ if not rcpts then
+ return false
+ end
+ local found
+ for _, r in ipairs(rcpts) do
+ found = false
+ -- Try full addr match
+ if r.addr and r.domain and r.user then
+ if settings.extended_headers_rcpt:get_key(r.addr) then
+ lua_util.debugm(N, task, 'found full addr in recipients for extended headers: %s',
+ r.addr)
+ found = true
+ end
+ -- Try user as plain match
+ if not found and settings.extended_headers_rcpt:get_key(r.user) then
+ lua_util.debugm(N, task, 'found user in recipients for extended headers: %s (%s)',
+ r.user, r.addr)
+ found = true
+ end
+ -- Try @domain to match domain
+ if not found and settings.extended_headers_rcpt:get_key('@' .. r.domain) then
+ lua_util.debugm(N, task, 'found domain in recipients for extended headers: @%s (%s)',
+ r.domain, r.addr)
+ found = true
+ end
+ end
+ if found then
+ break
+ end
+ end
+ return found
+ end
+
+ if settings.extended_headers_rcpt and match_extended_headers_rcpt() then
+ return false
+ end
+
+ if settings.skip_local and not settings.local_headers[hdr] then
+ local ip = task:get_ip()
+ if (ip and ip:is_local()) then
+ return true
+ end
+ end
+
+ if settings.skip_authenticated and not settings.authenticated_headers[hdr] then
+ if task:get_user() ~= nil then
+ return true
+ end
+ end
+
+ if settings.skip_all then
+ return true
+ end
+
+ return false
+
+ end
+
+ -- XXX: fix this crap one day
+ -- routines - are closures that encloses all environment including task
+ -- common - a common environment shared between routines
+ -- add - add headers table (filled by routines)
+ -- remove - remove headers table (filled by routines)
+ local routines, common, add, remove = {}, {}, {}, {}
+
+ local function add_header(name, value, stop_chars, order)
+ local hname = settings.routines[name].header
+ if not add[hname] then
+ add[hname] = {}
+ end
+ table.insert(add[hname], {
+ order = (order or settings.default_headers_order or -1),
+ value = lua_util.fold_header(task, hname, value, stop_chars)
+ })
+ end
+
+ routines['x-spamd-result'] = function()
+ local local_mod = settings.routines['x-spamd-result']
+ if skip_wanted('x-spamd-result') then
+ return
+ end
+ if not common.symbols then
+ common.symbols = task:get_symbols_all()
+ end
+ if not common['metric_score'] then
+ common['metric_score'] = task:get_metric_score()
+ end
+ if not common['metric_action'] then
+ common['metric_action'] = task:get_metric_action()
+ end
+ if local_mod.remove then
+ remove[local_mod.header] = local_mod.remove
+ end
+
+ local buf = {}
+ local verdict = string.format('default: %s [%.2f / %.2f]',
+ --TODO: (common.metric_action == 'no action') and 'False' or 'True',
+ (common.metric_action == 'reject') and 'True' or 'False',
+ common.metric_score[1], common.metric_score[2])
+ table.insert(buf, verdict)
+
+ -- Deal with symbols
+ table.sort(common.symbols, function(s1, s2)
+ local res
+ if local_mod.sort_by == 'name' then
+ res = s1.name < s2.name
+ else
+ -- inverse order to show important symbols first
+ res = math.abs(s1.score) > math.abs(s2.score)
+ end
+
+ return res
+ end)
+
+ for _, s in ipairs(common.symbols) do
+ local sym_str = string.format('%s(%.2f)[%s]',
+ s.name, s.score, table.concat(s.options or {}, ','))
+ table.insert(buf, sym_str)
+ end
+ add_header('x-spamd-result', table.concat(buf, '; '), ';')
+
+ local has_pr, action, message, module = task:has_pre_result()
+
+ if has_pr then
+ local pr_header = {}
+ if action then
+ table.insert(pr_header, string.format('action=%s', action))
+ end
+ if module then
+ table.insert(pr_header, string.format('module=%s', module))
+ end
+ if message then
+ table.insert(pr_header, message)
+ end
+ add_header('x-rspamd-pre-result', table.concat(pr_header, '; '), ';')
+ end
+ end
+
+ routines['x-rspamd-queue-id'] = function()
+ if skip_wanted('x-rspamd-queue-id') then
+ return
+ end
+ if common.queue_id ~= false then
+ common.queue_id = task:get_queue_id()
+ if not common.queue_id then
+ common.queue_id = false
+ end
+ end
+ if settings.routines['x-rspamd-queue-id'].remove then
+ remove[settings.routines['x-rspamd-queue-id'].header] = settings.routines['x-rspamd-queue-id'].remove
+ end
+ if common.queue_id then
+ add[settings.routines['x-rspamd-queue-id'].header] = common.queue_id
+ end
+ end
+
+ routines['remove-header'] = function()
+ if skip_wanted('remove-header') then
+ return
+ end
+ if settings.routines['remove-header'].header and settings.routines['remove-header'].remove then
+ remove[settings.routines['remove-header'].header] = settings.routines['remove-header'].remove
+ end
+ end
+
+ routines['remove-headers'] = function()
+ if skip_wanted('remove-headers') then
+ return
+ end
+ for h, r in pairs(settings.routines['remove-headers'].headers) do
+ remove[h] = r
+ end
+ end
+
+ routines['add-headers'] = function()
+ if skip_wanted('add-headers') then
+ return
+ end
+ for h, r in pairs(settings.routines['add-headers'].headers) do
+ add[h] = r
+ remove[h] = settings.routines['add-headers'].remove
+ end
+ end
+
+ routines['x-rspamd-server'] = function()
+ local local_mod = settings.routines['x-rspamd-server']
+ if skip_wanted('x-rspamd-server') then
+ return
+ end
+ if local_mod.remove then
+ remove[local_mod.header] = local_mod.remove
+ end
+ local hostname = local_mod.hostname
+ add[local_mod.header] = hostname and hostname or HOSTNAME
+ end
+
+ routines['x-spamd-bar'] = function()
+ local local_mod = settings.routines['x-spamd-bar']
+ if skip_wanted('x-rspamd-bar') then
+ return
+ end
+ if not common['metric_score'] then
+ common['metric_score'] = task:get_metric_score()
+ end
+ local score = common['metric_score'][1]
+ local spambar
+ if score <= -1 then
+ spambar = string.rep(local_mod.negative, math.floor(score * -1))
+ elseif score >= 1 then
+ spambar = string.rep(local_mod.positive, math.floor(score))
+ else
+ spambar = local_mod.neutral
+ end
+ if local_mod.remove then
+ remove[local_mod.header] = local_mod.remove
+ end
+ if spambar ~= '' then
+ add[local_mod.header] = spambar
+ end
+ end
+
+ routines['x-spam-level'] = function()
+ local local_mod = settings.routines['x-spam-level']
+ if skip_wanted('x-spam-level') then
+ return
+ end
+ if not common['metric_score'] then
+ common['metric_score'] = task:get_metric_score()
+ end
+ local score = common['metric_score'][1]
+ if score < 1 then
+ return nil, {}, {}
+ end
+ if local_mod.remove then
+ remove[local_mod.header] = local_mod.remove
+ end
+ add[local_mod.header] = string.rep(local_mod.char, math.floor(score))
+ end
+
+ routines['x-rspamd-action'] = function()
+ local local_mod = settings.routines['x-rspamd-action']
+ if skip_wanted('x-rspamd-action') then
+ return
+ end
+ if not common['metric_action'] then
+ common['metric_action'] = task:get_metric_action()
+ end
+ local action = common['metric_action']
+ if local_mod.remove then
+ remove[local_mod.header] = local_mod.remove
+ end
+ add[local_mod.header] = action
+ end
+
+ local function spam_header (class, name, value, remove_v)
+ if skip_wanted(class) then
+ return
+ end
+ if not common['metric_action'] then
+ common['metric_action'] = task:get_metric_action()
+ end
+ if remove_v then
+ remove[name] = remove_v
+ end
+ local action = common['metric_action']
+ if action ~= 'no action' and action ~= 'greylist' then
+ add[name] = value
+ end
+ end
+
+ routines['spam-header'] = function()
+ spam_header('spam-header',
+ settings.routines['spam-header'].header,
+ settings.routines['spam-header'].value,
+ settings.routines['spam-header'].remove)
+ end
+
+ routines['remove-spam-flag'] = function()
+ remove[settings.routines['remove-spam-flag'].header] = 0
+ end
+
+ routines['x-virus'] = function()
+ local local_mod = settings.routines['x-virus']
+ if skip_wanted('x-virus') then
+ return
+ end
+ if not common.symbols_hash then
+ if not common.symbols then
+ common.symbols = task:get_symbols_all()
+ end
+ local h = {}
+ for _, s in ipairs(common.symbols) do
+ h[s.name] = s
+ end
+ common.symbols_hash = h
+ end
+ if local_mod.remove then
+ remove[local_mod.header] = local_mod.remove
+ end
+ local virii = {}
+ for _, sym in ipairs(local_mod.symbols) do
+ local s = common.symbols_hash[sym]
+ if s then
+ if (s.options or E)[1] then
+ table.insert(virii, table.concat(s.options, ','))
+ elseif s then
+ table.insert(virii, 'unknown')
+ end
+ end
+ end
+ if #virii > 0 then
+ local virusstatus = table.concat(virii, ',')
+ if local_mod.status_infected then
+ virusstatus = local_mod.status_infected .. ', ' .. virusstatus
+ end
+ add_header('x-virus', virusstatus)
+ else
+ local failed = false
+ local fail_reason = 'unknown'
+ for _, sym in ipairs(local_mod.symbols_fail) do
+ local s = common.symbols_hash[sym]
+ if s then
+ failed = true
+ if (s.options or E)[1] then
+ fail_reason = table.concat(s.options, ',')
+ end
+ end
+ end
+ if not failed then
+ if local_mod.status_clean then
+ add_header('x-virus', local_mod.status_clean)
+ end
+ else
+ if local_mod.status_clean then
+ add_header('x-virus', string.format('%s(%s)',
+ local_mod.status_fail, fail_reason))
+ end
+ end
+ end
+ end
+
+ routines['x-os-fingerprint'] = function()
+ if skip_wanted('x-os-fingerprint') then
+ return
+ end
+ local local_mod = settings.routines['x-os-fingerprint']
+
+ local os_string, link_type, uptime_min, distance = task:get_mempool():get_variable('os_fingerprint',
+ 'string, string, double, double');
+
+ if not os_string then
+ return
+ end
+
+ local value = string.format('%s, (up: %i min), (distance %i, link: %s)',
+ os_string, uptime_min, distance, link_type)
+
+ if local_mod.remove then
+ remove[local_mod.header] = local_mod.remove
+ end
+
+ add_header('x-os-fingerprint', value)
+ end
+
+ routines['x-spam-status'] = function()
+ if skip_wanted('x-spam-status') then
+ return
+ end
+ if not common['metric_score'] then
+ common['metric_score'] = task:get_metric_score()
+ end
+ if not common['metric_action'] then
+ common['metric_action'] = task:get_metric_action()
+ end
+ local score = common['metric_score'][1]
+ local action = common['metric_action']
+ local is_spam
+ local spamstatus
+ if action ~= 'no action' and action ~= 'greylist' then
+ is_spam = 'Yes'
+ else
+ is_spam = 'No'
+ end
+ spamstatus = is_spam .. ', score=' .. string.format('%.2f', score)
+
+ if settings.routines['x-spam-status'].remove then
+ remove[settings.routines['x-spam-status'].header] = settings.routines['x-spam-status'].remove
+ end
+ add_header('x-spam-status', spamstatus)
+ end
+
+ routines['authentication-results'] = function()
+ if skip_wanted('authentication-results') then
+ return
+ end
+ local ar = require "lua_auth_results"
+
+ if settings.routines['authentication-results'].remove then
+ remove[settings.routines['authentication-results'].header] = settings.routines['authentication-results'].remove
+ end
+
+ local res = ar.gen_auth_results(task,
+ lua_util.override_defaults(ar.default_settings,
+ settings.routines['authentication-results']))
+
+ if res then
+ add_header('authentication-results', res, ';', 1)
+ end
+ end
+
+ routines['stat-signature'] = function()
+ if skip_wanted('stat-signature') then
+ return
+ end
+ if settings.routines['stat-signature'].remove then
+ remove[settings.routines['stat-signature'].header] = settings.routines['stat-signature'].remove
+ end
+ local res = task:get_mempool():get_variable("stat_signature")
+ if res then
+ add[settings.routines['stat-signature'].header] = res
+ end
+ end
+
+ routines['fuzzy-hashes'] = function()
+ local res = task:get_mempool():get_variable("fuzzy_hashes", "fstrings")
+
+ if res and #res > 0 then
+ for _, h in ipairs(res) do
+ add_header('fuzzy-hashes', h)
+ end
+ end
+ end
+
+ local routines_enabled = active_routines
+ local user_settings = task:cache_get('settings')
+ if user_settings and user_settings.plugins then
+ user_settings = user_settings.plugins.milter_headers or E
+ end
+
+ if user_settings and type(user_settings.routines) == 'table' then
+ lua_util.debugm(N, task, 'override routines to %s from user settings',
+ user_settings.routines)
+ routines_enabled = user_settings.routines
+ settings_override = true
+ end
+
+ for _, n in ipairs(routines_enabled) do
+ local ok, err
+ if custom_routines[n] then
+ local to_add, to_remove, common_in
+ ok, err, to_add, to_remove, common_in = pcall(custom_routines[n], task, common)
+ if ok then
+ for k, v in pairs(to_add) do
+ add[k] = v
+ end
+ for k, v in pairs(to_remove) do
+ remove[k] = v
+ end
+ for k, v in pairs(common_in) do
+ if type(v) == 'table' then
+ if not common[k] then
+ common[k] = {}
+ end
+ for kk, vv in pairs(v) do
+ common[k][kk] = vv
+ end
+ else
+ common[k] = v
+ end
+ end
+ end
+ else
+ ok, err = pcall(routines[n])
+ end
+ if not ok then
+ logger.errx(task, 'call to %s failed: %s', n, err)
+ end
+ end
+
+ if not next(add) then
+ add = nil
+ end
+ if not next(remove) then
+ remove = nil
+ end
+ if add or remove then
+
+ lua_mime.modify_headers(task, {
+ add = add,
+ remove = remove
+ }, settings.headers_modify_mode)
+ end
+end
+
+local config_schema = ts.shape({
+ use = ts.array_of(ts.string) + ts.string / function(s)
+ return { s }
+ end,
+ remove_upstream_spam_flag = ts.boolean:is_optional(),
+ extended_spam_headers = ts.boolean:is_optional(),
+ skip_local = ts.boolean:is_optional(),
+ skip_authenticated = ts.boolean:is_optional(),
+ local_headers = ts.array_of(ts.string):is_optional(),
+ authenticated_headers = ts.array_of(ts.string):is_optional(),
+ extended_headers_rcpt = lua_maps.map_schema:is_optional(),
+ custom = ts.map_of(ts.string, ts.string):is_optional(),
+}, {
+ extra_fields = ts.map_of(ts.string, ts.any)
+})
+
+local opts = rspamd_config:get_all_opt(N) or
+ rspamd_config:get_all_opt('rmilter_headers')
+
+if not opts then
+ return
+end
+
+-- Process config
+do
+ local res, err = config_schema:transform(opts)
+ if not res then
+ logger.errx(rspamd_config, 'invalid config for %s: %s', N, err)
+ return
+ else
+ opts = res
+ end
+end
+
+local have_routine = {}
+local function activate_routine(s)
+ if settings.routines[s] or custom_routines[s] then
+ if not have_routine[s] then
+ have_routine[s] = true
+ table.insert(active_routines, s)
+ if (opts.routines and opts.routines[s]) then
+ settings.routines[s] = lua_util.override_defaults(settings.routines[s],
+ opts.routines[s])
+ end
+ end
+ else
+ logger.errx(rspamd_config, 'routine "%s" does not exist', s)
+ end
+end
+
+if opts.remove_upstream_spam_flag ~= nil then
+ settings.remove_upstream_spam_flag = opts.remove_upstream_spam_flag
+end
+
+if opts.extended_spam_headers then
+ activate_routine('x-spamd-result')
+ activate_routine('x-rspamd-server')
+ activate_routine('x-rspamd-queue-id')
+ activate_routine('x-rspamd-action')
+end
+
+if opts.local_headers then
+ for _, h in ipairs(opts.local_headers) do
+ settings.local_headers[h] = true
+ end
+end
+if opts.authenticated_headers then
+ for _, h in ipairs(opts.authenticated_headers) do
+ settings.authenticated_headers[h] = true
+ end
+end
+if opts.custom then
+ for k, v in pairs(opts['custom']) do
+ local f, err = load(v)
+ if not f then
+ logger.errx(rspamd_config, 'could not load "%s": %s', k, err)
+ else
+ custom_routines[k] = f()
+ end
+ end
+end
+
+if type(opts['skip_local']) == 'boolean' then
+ settings.skip_local = opts['skip_local']
+end
+
+if type(opts['skip_authenticated']) == 'boolean' then
+ settings.skip_authenticated = opts['skip_authenticated']
+end
+
+if type(opts['skip_all']) == 'boolean' then
+ settings.skip_all = opts['skip_all']
+end
+
+for _, s in ipairs(opts['use']) do
+ if not have_routine[s] then
+ activate_routine(s)
+ end
+end
+
+if settings.remove_upstream_spam_flag then
+ activate_routine('remove-spam-flag')
+end
+
+if (#active_routines < 1) then
+ logger.errx(rspamd_config, 'no active routines')
+ return
+end
+
+logger.infox(rspamd_config, 'active routines [%s]',
+ table.concat(active_routines, ','))
+
+if opts.extended_headers_rcpt then
+ settings.extended_headers_rcpt = lua_maps.rspamd_map_add_from_ucl(opts.extended_headers_rcpt,
+ 'set', 'Extended headers recipients')
+end
+
+rspamd_config:register_symbol({
+ name = 'MILTER_HEADERS',
+ type = 'idempotent',
+ callback = milter_headers,
+ flags = 'empty,explicit_disable,ignore_passthrough',
+})
diff --git a/src/plugins/lua/mime_types.lua b/src/plugins/lua/mime_types.lua
new file mode 100644
index 0000000..167ed38
--- /dev/null
+++ b/src/plugins/lua/mime_types.lua
@@ -0,0 +1,737 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- This plugin implements mime types checks for mail messages
+local logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
+local lua_maps = require "lua_maps"
+local lua_mime_types = require "lua_mime_types"
+local lua_magic_types = require "lua_magic/types"
+local fun = require "fun"
+
+local N = "mime_types"
+local settings = {
+ file = '',
+ symbol_unknown = 'MIME_UNKNOWN',
+ symbol_bad = 'MIME_BAD',
+ symbol_good = 'MIME_GOOD',
+ symbol_attachment = 'MIME_BAD_ATTACHMENT',
+ symbol_encrypted_archive = 'MIME_ENCRYPTED_ARCHIVE',
+ symbol_obfuscated_archive = 'MIME_OBFUSCATED_ARCHIVE',
+ symbol_exe_in_gen_split_rar = 'MIME_EXE_IN_GEN_SPLIT_RAR',
+ symbol_archive_in_archive = 'MIME_ARCHIVE_IN_ARCHIVE',
+ symbol_double_extension = 'MIME_DOUBLE_BAD_EXTENSION',
+ symbol_bad_extension = 'MIME_BAD_EXTENSION',
+ symbol_bad_unicode = 'MIME_BAD_UNICODE',
+ regexp = false,
+ extension_map = { -- extension -> mime_type
+ html = 'text/html',
+ htm = 'text/html',
+ pdf = 'application/pdf',
+ shtm = 'text/html',
+ shtml = 'text/html',
+ txt = 'text/plain'
+ },
+
+ bad_extensions = {
+ cue = 2,
+ exe = 1,
+ iso = 4,
+ jar = 2,
+ zpaq = 2,
+ -- In contrast to HTML MIME parts, dedicated HTML attachments are considered harmful
+ htm = 1,
+ html = 1,
+ shtm = 1,
+ shtml = 1,
+ -- Have you ever seen that in legit email?
+ ace = 4,
+ arj = 2,
+ aspx = 1,
+ asx = 2,
+ cab = 3,
+ dll = 4,
+ dqy = 2,
+ iqy = 2,
+ mht = 2,
+ mhtml = 2,
+ oqy = 2,
+ rqy = 2,
+ sfx = 2,
+ slk = 2,
+ vst = 2,
+ vss = 2,
+ wim = 2,
+ -- Additional bad extensions from Gmail
+ ade = 4,
+ adp = 4,
+ cmd = 4,
+ cpl = 4,
+ ins = 4,
+ isp = 4,
+ js = 4,
+ jse = 4,
+ lib = 4,
+ mde = 4,
+ msc = 4,
+ msi = 4,
+ msp = 4,
+ mst = 4,
+ nsh = 4,
+ pif = 4,
+ sct = 4,
+ shb = 4,
+ sys = 4,
+ vb = 4,
+ vbe = 4,
+ vbs = 4,
+ vxd = 4,
+ wsc = 4,
+ wsh = 4,
+ -- Additional bad extensions from Outlook
+ app = 4,
+ asp = 4,
+ bas = 4,
+ bat = 4,
+ chm = 4,
+ cnt = 4,
+ com = 4,
+ csh = 4,
+ diagcab = 4,
+ fxp = 4,
+ gadget = 4,
+ grp = 4,
+ hlp = 4,
+ hpj = 4,
+ hta = 4,
+ htc = 4,
+ inf = 4,
+ its = 4,
+ jnlp = 4,
+ lnk = 4,
+ ksh = 4,
+ mad = 4,
+ maf = 4,
+ mag = 4,
+ mam = 4,
+ maq = 4,
+ mar = 4,
+ mas = 4,
+ mat = 4,
+ mau = 4,
+ mav = 4,
+ maw = 4,
+ mcf = 4,
+ mda = 4,
+ mdb = 4,
+ mdt = 4,
+ mdw = 4,
+ mdz = 4,
+ msh = 4,
+ msh1 = 4,
+ msh2 = 4,
+ mshxml = 4,
+ msh1xml = 4,
+ msh2xml = 4,
+ msu = 4,
+ ops = 4,
+ osd = 4,
+ pcd = 4,
+ pl = 4,
+ plg = 4,
+ prf = 4,
+ prg = 4,
+ printerexport = 4,
+ ps1 = 4,
+ ps1xml = 4,
+ ps2 = 4,
+ ps2xml = 4,
+ psc1 = 4,
+ psc2 = 4,
+ psd1 = 4,
+ psdm1 = 4,
+ pst = 4,
+ pyc = 4,
+ pyo = 4,
+ pyw = 4,
+ pyz = 4,
+ pyzw = 4,
+ reg = 4,
+ scf = 4,
+ scr = 4,
+ shs = 4,
+ theme = 4,
+ url = 4,
+ vbp = 4,
+ vhd = 4,
+ vhdx = 4,
+ vsmacros = 4,
+ vsw = 4,
+ webpnp = 4,
+ website = 4,
+ ws = 4,
+ wsf = 4,
+ xbap = 4,
+ xll = 4,
+ xnk = 4,
+ },
+
+ -- Something that should not be in archive
+ bad_archive_extensions = {
+ docx = 0.1,
+ hta = 4,
+ jar = 3,
+ js = 0.5,
+ pdf = 0.1,
+ pptx = 0.1,
+ vbs = 4,
+ wsf = 4,
+ xlsx = 0.1,
+ },
+
+ archive_extensions = {
+ ['7z'] = 1,
+ ace = 1,
+ alz = 1,
+ arj = 1,
+ bz2 = 1,
+ cab = 1,
+ egg = 1,
+ lz = 1,
+ rar = 1,
+ xz = 1,
+ zip = 1,
+ zpaq = 1,
+ },
+
+ -- Not really archives
+ archive_exceptions = {
+ docx = true,
+ odp = true,
+ ods = true,
+ odt = true,
+ pptx = true,
+ vsdx = true,
+ xlsx = true,
+ -- jar = true,
+ },
+
+ -- Multiplier for full extension_map mismatch
+ other_extensions_mult = 0.4,
+}
+
+local map = nil
+
+local function check_mime_type(task)
+ local function gen_extension(fname)
+ local parts = lua_util.str_split(fname or '', '.')
+
+ local ext = {}
+ for n = 1, 2 do
+ ext[n] = #parts > n and string.lower(parts[#parts + 1 - n]) or nil
+ end
+
+ return ext[1], ext[2], parts
+ end
+
+ local function check_filename(fname, ct, is_archive, part, detected_ext, nfiles)
+
+ lua_util.debugm(N, task, "check filename: %s, ct=%s, is_archive=%s, detected_ext=%s, nfiles=%s",
+ fname, ct, is_archive, detected_ext, nfiles)
+ local has_bad_unicode, char, ch_pos = rspamd_util.has_obscured_unicode(fname)
+ if has_bad_unicode then
+ task:insert_result(settings.symbol_bad_unicode, 1.0,
+ string.format("0x%xd after %s", char,
+ fname:sub(1, ch_pos)))
+ end
+
+ -- Decode hex encoded characters
+ fname = string.gsub(fname, '%%(%x%x)',
+ function(hex)
+ return string.char(tonumber(hex, 16))
+ end)
+
+ -- Replace potentially bad characters with '?'
+ fname = fname:gsub('[^%s%g]', '?')
+
+ -- Check file is in filename whitelist
+ if settings.filename_whitelist and
+ settings.filename_whitelist:get_key(fname) then
+ logger.debugm("mime_types", task, "skip checking of %s - file is in filename whitelist",
+ fname)
+ return
+ end
+
+ local ext, ext2, parts = gen_extension(fname)
+ -- ext is the last extension, LOWERCASED
+ -- ext2 is the one before last extension LOWERCASED
+
+ local detected
+
+ if not is_archive and detected_ext then
+ detected = lua_magic_types[detected_ext]
+ end
+
+ if detected_ext and ((not ext) or ext ~= detected_ext) then
+ -- Try to find extension by real content type
+ check_filename('detected.' .. detected_ext, detected.ct,
+ false, part, nil, 1)
+ end
+
+ if not ext then
+ return
+ end
+
+ local function check_extension(badness_mult, badness_mult2)
+ if not badness_mult and not badness_mult2 then
+ return
+ end
+ if #parts > 2 then
+ -- We need to ensure that next-to-last extension is an extension,
+ -- so we check for its length and if it is not a number or date
+ if #ext2 > 0 and #ext2 <= 4 and not string.match(ext2, '^%d+[%]%)]?$') then
+
+ -- Use the greatest badness multiplier
+ if not badness_mult or
+ (badness_mult2 and badness_mult < badness_mult2) then
+ badness_mult = badness_mult2
+ end
+
+ -- Double extension + bad extension == VERY bad
+ task:insert_result(settings['symbol_double_extension'], badness_mult,
+ string.format(".%s.%s", ext2, ext))
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", part:get_id(), '-'))
+ return
+ end
+ end
+ if badness_mult then
+ -- Just bad extension
+ task:insert_result(settings['symbol_bad_extension'], badness_mult, ext)
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", part:get_id(), '-'))
+ end
+ end
+
+ -- Process settings
+ local extra_table = {}
+ local extra_archive_table = {}
+ local user_settings = task:cache_get('settings')
+ if user_settings and user_settings.plugins then
+ user_settings = user_settings.plugins.mime_types
+ end
+
+ if user_settings then
+ logger.infox(task, 'using special tables from user settings')
+ if user_settings.bad_extensions then
+ if user_settings.bad_extensions[1] then
+ -- Convert to a key-value map
+ extra_table = fun.tomap(
+ fun.map(function(e)
+ return e, 1.0
+ end,
+ user_settings.bad_extensions))
+ else
+ extra_table = user_settings.bad_extensions
+ end
+ end
+ if user_settings.bad_archive_extensions then
+ if user_settings.bad_archive_extensions[1] then
+ -- Convert to a key-value map
+ extra_archive_table = fun.tomap(fun.map(
+ function(e)
+ return e, 1.0
+ end,
+ user_settings.bad_archive_extensions))
+ else
+ extra_archive_table = user_settings.bad_archive_extensions
+ end
+ end
+ end
+
+ local function check_tables(e)
+ if is_archive then
+ return extra_archive_table[e] or (nfiles < 2 and settings.bad_archive_extensions[e]) or
+ extra_table[e] or settings.bad_extensions[e]
+ end
+
+ return extra_table[e] or settings.bad_extensions[e]
+ end
+
+ -- Also check for archive bad extension
+ if is_archive then
+ if ext2 then
+ local score1 = check_tables(ext)
+ local score2 = check_tables(ext2)
+ check_extension(score1, score2)
+ else
+ local score1 = check_tables(ext)
+ check_extension(score1, nil)
+ end
+
+ if settings['archive_extensions'][ext] then
+ -- Archive in archive
+ task:insert_result(settings['symbol_archive_in_archive'], 1.0, ext)
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", part:get_id(), '-'))
+ end
+ else
+ if ext2 then
+ local score1 = check_tables(ext)
+ local score2 = check_tables(ext2)
+ check_extension(score1, score2)
+ -- Check for archive cloaking like .zip.gz
+ if settings['archive_extensions'][ext2]
+ -- Exclude multipart archive extensions, e.g. .zip.001
+ and not string.match(ext, '^%d+$')
+ then
+ task:insert_result(settings['symbol_archive_in_archive'],
+ 1.0, string.format(".%s.%s", ext2, ext))
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", part:get_id(), '-'))
+ end
+ else
+ local score1 = check_tables(ext)
+ check_extension(score1, nil)
+ end
+ end
+
+ local mt = settings['extension_map'][ext]
+ if mt and ct and ct ~= 'application/octet-stream' then
+ local found
+ local mult
+ for _, v in ipairs(mt) do
+ mult = v.mult
+ if ct == v.ct then
+ found = true
+ break
+ end
+ end
+
+ if not found then
+ task:insert_result(settings['symbol_attachment'], mult, string.format('%s:%s',
+ ext, ct))
+ end
+ end
+ end
+
+ local parts = task:get_parts()
+
+ if parts then
+ for _, p in ipairs(parts) do
+ local mtype, subtype = p:get_type()
+
+ if not mtype then
+ lua_util.debugm(N, task, "no content type for part: %s", p:get_id())
+ task:insert_result(settings['symbol_unknown'], 1.0, 'missing content type')
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", p:get_id(), '~'))
+ else
+ -- Check for attachment
+ local filename = p:get_filename()
+ local ct = string.format('%s/%s', mtype, subtype):lower()
+ local detected_ext = p:get_detected_ext()
+
+ if filename then
+ check_filename(filename, ct, false, p, detected_ext, 1)
+ end
+
+ if p:is_archive() then
+ local check = true
+ if detected_ext then
+ local detected_type = lua_magic_types[detected_ext]
+
+ if detected_type.type ~= 'archive' then
+ logger.debugm("mime_types", task, "skip checking of %s as archive, %s is not archive but %s",
+ filename, detected_type.type)
+ check = false
+ end
+ end
+ if check and filename then
+ local ext = gen_extension(filename)
+
+ if ext and settings.archive_exceptions[ext] then
+ check = false
+ logger.debugm("mime_types", task, "skip checking of %s as archive, %s is whitelisted",
+ filename, ext)
+ end
+ end
+ local arch = p:get_archive()
+
+ -- TODO: migrate to flags once C part is ready
+ if arch:is_encrypted() then
+ task:insert_result(settings.symbol_encrypted_archive, 1.0, filename)
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", p:get_id(), '-'))
+ elseif arch:is_unreadable() then
+ task:insert_result(settings.symbol_encrypted_archive, 0.5, {
+ 'compressed header',
+ filename,
+ })
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", p:get_id(), '-'))
+ elseif arch:is_obfuscated() then
+ task:insert_result(settings.symbol_obfuscated_archive, 1.0, {
+ 'obfuscated archive',
+ filename,
+ })
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", p:get_id(), '-'))
+ end
+
+ if check then
+ local is_gen_split_rar = false
+ if filename then
+ local ext = gen_extension(filename)
+ is_gen_split_rar = ext and (string.match(ext, '^%d%d%d$')) and (arch:get_type() == 'rar')
+ end
+
+ local fl = arch:get_files_full(1000)
+
+ local nfiles = #fl
+
+ for _, f in ipairs(fl) do
+ if f['encrypted'] then
+ task:insert_result(settings['symbol_encrypted_archive'],
+ 1.0, f['name'])
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", p:get_id(), '-'))
+ end
+
+ if f['name'] then
+ if is_gen_split_rar and (gen_extension(f['name']) or '') == 'exe' then
+ task:insert_result(settings['symbol_exe_in_gen_split_rar'], 1.0, f['name'])
+ else
+ check_filename(f['name'], nil,
+ true, p, nil, nfiles)
+ end
+ end
+ end
+
+ if nfiles == 1 and fl[1].name then
+ -- We check that extension of the file inside archive is
+ -- the same as double extension of the file
+ local _, ext2 = gen_extension(filename)
+
+ if ext2 and #ext2 > 0 then
+ local enc_ext = gen_extension(fl[1].name)
+
+ if enc_ext
+ and settings['bad_extensions'][enc_ext]
+ and not tonumber(ext2)
+ and enc_ext ~= ext2 then
+ task:insert_result(settings['symbol_double_extension'], 2.0,
+ string.format("%s!=%s", ext2, enc_ext))
+ end
+ end
+ end
+ end
+ end
+
+ if map then
+ local v = map:get_key(ct)
+ local detected_different = false
+
+ local detected_type
+ if detected_ext then
+ detected_type = lua_magic_types[detected_ext]
+ end
+
+ if detected_type and detected_type.ct ~= ct then
+ local v_detected = map:get_key(detected_type.ct)
+ if not v or v_detected and v_detected > v then
+ v = v_detected
+ end
+ detected_different = true
+ end
+ if v then
+ local n = tonumber(v)
+
+ if n then
+ if n > 0 then
+ if detected_different then
+ -- Penalize case
+ n = n * 1.5
+ task:insert_result(settings['symbol_bad'], n,
+ string.format('%s:%s', ct, detected_type.ct))
+ else
+ task:insert_result(settings['symbol_bad'], n, ct)
+ end
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", p:get_id(), '-'))
+ elseif n < 0 then
+ task:insert_result(settings['symbol_good'], -n, ct)
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", p:get_id(), '+'))
+ else
+ -- Neutral content type
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", p:get_id(), '~'))
+ end
+ else
+ logger.warnx(task, 'unknown value: "%s" for content type %s in the map',
+ v, ct)
+ end
+ else
+ task:insert_result(settings['symbol_unknown'], 1.0, ct)
+ task:insert_result('MIME_TRACE', 0.0,
+ string.format("%s:%s", p:get_id(), '~'))
+ end
+ end
+ end
+ end
+ end
+end
+
+local opts = rspamd_config:get_all_opt('mime_types')
+if opts then
+ for k, v in pairs(opts) do
+ settings[k] = v
+ end
+
+ settings.filename_whitelist = lua_maps.rspamd_map_add('mime_types', 'filename_whitelist', 'regexp',
+ 'filename whitelist')
+
+ local function change_extension_map_entry(ext, ct, mult)
+ if type(ct) == 'table' then
+ local tbl = {}
+ for _, elt in ipairs(ct) do
+ table.insert(tbl, {
+ ct = elt,
+ mult = mult,
+ })
+ end
+ settings.extension_map[ext] = tbl
+ else
+ settings.extension_map[ext] = { [1] = {
+ ct = ct,
+ mult = mult
+ } }
+ end
+ end
+
+ -- Transform extension_map
+ for ext, ct in pairs(settings.extension_map) do
+ change_extension_map_entry(ext, ct, 1.0)
+ end
+
+ -- Add all extensions
+ for _, pair in ipairs(lua_mime_types.full_extensions_map) do
+ local ext, ct = pair[1], pair[2]
+ if not settings.extension_map[ext] then
+ change_extension_map_entry(ext, ct, settings.other_extensions_mult)
+ end
+ end
+
+ local map_type = 'map'
+ if settings['regexp'] then
+ map_type = 'regexp'
+ end
+ map = lua_maps.rspamd_map_add('mime_types', 'file', map_type,
+ 'mime types map')
+ if map then
+ local id = rspamd_config:register_symbol({
+ name = 'MIME_TYPES_CALLBACK',
+ callback = check_mime_type,
+ type = 'callback',
+ flags = 'nostat',
+ group = 'mime_types',
+ })
+
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = settings['symbol_unknown'],
+ parent = id,
+ group = 'mime_types',
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = settings['symbol_bad'],
+ parent = id,
+ group = 'mime_types',
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = settings['symbol_good'],
+ flags = 'nice',
+ parent = id,
+ group = 'mime_types',
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = settings['symbol_attachment'],
+ parent = id,
+ group = 'mime_types',
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = settings['symbol_encrypted_archive'],
+ parent = id,
+ group = 'mime_types',
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = settings['symbol_obfuscated_archive'],
+ parent = id,
+ group = 'mime_types',
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = settings['symbol_exe_in_gen_split_rar'],
+ parent = id,
+ group = 'mime_types',
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = settings['symbol_archive_in_archive'],
+ parent = id,
+ group = 'mime_types',
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = settings['symbol_double_extension'],
+ parent = id,
+ group = 'mime_types',
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = settings['symbol_bad_extension'],
+ parent = id,
+ group = 'mime_types',
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = settings['symbol_bad_unicode'],
+ parent = id,
+ group = 'mime_types',
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = 'MIME_TRACE',
+ parent = id,
+ group = 'mime_types',
+ flags = 'nostat',
+ score = 0,
+ })
+ else
+ lua_util.disable_module(N, "config")
+ end
+end
diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua
new file mode 100644
index 0000000..53b2732
--- /dev/null
+++ b/src/plugins/lua/multimap.lua
@@ -0,0 +1,1403 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- Multimap is rspamd module designed to define and operate with different maps
+
+local rules = {}
+local rspamd_logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local rspamd_regexp = require "rspamd_regexp"
+local rspamd_expression = require "rspamd_expression"
+local rspamd_ip = require "rspamd_ip"
+local lua_util = require "lua_util"
+local lua_selectors = require "lua_selectors"
+local lua_maps = require "lua_maps"
+local redis_params
+local fun = require "fun"
+local N = 'multimap'
+
+local multimap_grammar
+-- Parse result in form: <symbol>:<score>|<symbol>|<score>
+local function parse_multimap_value(parse_rule, p_ret)
+ if p_ret and type(p_ret) == 'string' then
+ local lpeg = require "lpeg"
+
+ if not multimap_grammar then
+ local number = {}
+
+ local digit = lpeg.R("09")
+ number.integer = (lpeg.S("+-") ^ -1) *
+ (digit ^ 1)
+
+ -- Matches: .6, .899, .9999873
+ number.fractional = (lpeg.P(".")) *
+ (digit ^ 1)
+
+ -- Matches: 55.97, -90.8, .9
+ number.decimal = (number.integer * -- Integer
+ (number.fractional ^ -1)) + -- Fractional
+ (lpeg.S("+-") * number.fractional) -- Completely fractional number
+
+ local sym_start = lpeg.R("az", "AZ") + lpeg.S("_")
+ local sym_elt = sym_start + lpeg.R("09")
+ local symbol = sym_start * sym_elt ^ 0
+ local symbol_cap = lpeg.Cg(symbol, 'symbol')
+ local score_cap = lpeg.Cg(number.decimal, 'score')
+ local opts_cap = lpeg.Cg(lpeg.Ct(lpeg.C(symbol) * (lpeg.P(",") * lpeg.C(symbol)) ^ 0), 'opts')
+ local symscore_cap = (symbol_cap * lpeg.P(":") * score_cap)
+ local symscoreopt_cap = symscore_cap * lpeg.P(":") * opts_cap
+ local grammar = symscoreopt_cap + symscore_cap + symbol_cap + score_cap
+ multimap_grammar = lpeg.Ct(grammar)
+ end
+ local tbl = multimap_grammar:match(p_ret)
+
+ if tbl then
+ local sym
+ local score = 1.0
+ local opts = {}
+
+ if tbl.symbol then
+ sym = tbl.symbol
+ end
+ if tbl.score then
+ score = tonumber(tbl.score)
+ end
+ if tbl.opts then
+ opts = tbl.opts
+ end
+
+ return true, sym, score, opts
+ else
+ if p_ret ~= '' then
+ rspamd_logger.infox(rspamd_config, '%s: cannot parse string "%s"',
+ parse_rule.symbol, p_ret)
+ end
+
+ return true, nil, 1.0, {}
+ end
+ elseif type(p_ret) == 'boolean' then
+ return p_ret, nil, 1.0, {}
+ end
+
+ return false, nil, 0.0, {}
+end
+
+local value_types = {
+ ip = {
+ get_value = function(ip)
+ return ip:to_string()
+ end,
+ },
+ from = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ helo = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ header = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ rcpt = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ user = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ url = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ dnsbl = {
+ get_value = function(ip)
+ return ip:to_string()
+ end,
+ },
+ filename = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ content = {
+ get_value = function()
+ return nil
+ end,
+ },
+ hostname = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ asn = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ country = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ received = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ mempool = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ selector = {
+ get_value = function(val)
+ return val
+ end,
+ },
+ symbol_options = {
+ get_value = function(val)
+ return val
+ end,
+ },
+}
+
+local function ip_to_rbl(ip, rbl)
+ return table.concat(ip:inversed_str_octets(), ".") .. '.' .. rbl
+end
+
+local function apply_hostname_filter(task, filter, hostname, r)
+ if filter == 'tld' then
+ local tld = rspamd_util.get_tld(hostname)
+ return tld
+ elseif filter == 'top' then
+ local tld = rspamd_util.get_tld(hostname)
+ return tld:match('[^.]*$') or tld
+ else
+ if not r['re_filter'] then
+ local pat = string.match(filter, 'tld:regexp:(.+)')
+ if not pat then
+ rspamd_logger.errx(task, 'bad search filter: %s', filter)
+ return
+ end
+ r['re_filter'] = rspamd_regexp.create_cached(pat)
+ if not r['re_filter'] then
+ rspamd_logger.errx(task, 'couldnt create regex: %s', pat)
+ return
+ end
+ end
+ local tld = rspamd_util.get_tld(hostname)
+ local res = r['re_filter']:search(tld)
+ if res then
+ return res[1]
+ else
+ return nil
+ end
+ end
+end
+
+local function apply_url_filter(task, filter, url, r)
+ if not filter then
+ return url:get_host()
+ end
+
+ if filter == 'tld' then
+ return url:get_tld()
+ elseif filter == 'top' then
+ local tld = url:get_tld()
+ return tld:match('[^.]*$') or tld
+ elseif filter == 'full' then
+ return url:get_text()
+ elseif filter == 'is_phished' then
+ if url:is_phished() then
+ return url:get_host()
+ else
+ return nil
+ end
+ elseif filter == 'is_redirected' then
+ if url:is_redirected() then
+ return url:get_host()
+ else
+ return nil
+ end
+ elseif filter == 'is_obscured' then
+ if url:is_obscured() then
+ return url:get_host()
+ else
+ return nil
+ end
+ elseif filter == 'path' then
+ return url:get_path()
+ elseif filter == 'query' then
+ return url:get_query()
+ elseif string.find(filter, 'tag:') then
+ local tags = url:get_tags()
+ local want_tag = string.match(filter, 'tag:(.*)')
+ for _, t in ipairs(tags) do
+ if t == want_tag then
+ return url:get_host()
+ end
+ end
+ return nil
+ elseif string.find(filter, 'tld:regexp:') then
+ if not r['re_filter'] then
+ local type, pat = string.match(filter, '(regexp:)(.+)')
+ if type and pat then
+ r['re_filter'] = rspamd_regexp.create_cached(pat)
+ end
+ end
+
+ if not r['re_filter'] then
+ rspamd_logger.errx(task, 'bad search filter: %s', filter)
+ else
+ local results = r['re_filter']:search(url:get_tld())
+ if results then
+ return results[1]
+ else
+ return nil
+ end
+ end
+ elseif string.find(filter, 'full:regexp:') then
+ if not r['re_filter'] then
+ local type, pat = string.match(filter, '(regexp:)(.+)')
+ if type and pat then
+ r['re_filter'] = rspamd_regexp.create_cached(pat)
+ end
+ end
+
+ if not r['re_filter'] then
+ rspamd_logger.errx(task, 'bad search filter: %s', filter)
+ else
+ local results = r['re_filter']:search(url:get_text())
+ if results then
+ return results[1]
+ else
+ return nil
+ end
+ end
+ elseif string.find(filter, 'regexp:') then
+ if not r['re_filter'] then
+ local type, pat = string.match(filter, '(regexp:)(.+)')
+ if type and pat then
+ r['re_filter'] = rspamd_regexp.create_cached(pat)
+ end
+ end
+
+ if not r['re_filter'] then
+ rspamd_logger.errx(task, 'bad search filter: %s', filter)
+ else
+ local results = r['re_filter']:search(url:get_host())
+ if results then
+ return results[1]
+ else
+ return nil
+ end
+ end
+ elseif string.find(filter, '^template:') then
+ if not r['template'] then
+ r['template'] = string.match(filter, '^template:(.+)')
+ end
+
+ if r['template'] then
+ return lua_util.template(r['template'], url:to_table())
+ end
+ end
+
+ return url:get_host()
+end
+
+local function apply_addr_filter(task, filter, input, rule)
+ if filter == 'email:addr' or filter == 'email' then
+ local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
+ if addr and addr[1] then
+ return fun.totable(fun.map(function(a)
+ return a.addr
+ end, addr))
+ end
+ elseif filter == 'email:user' then
+ local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
+ if addr and addr[1] then
+ return fun.totable(fun.map(function(a)
+ return a.user
+ end, addr))
+ end
+ elseif filter == 'email:domain' then
+ local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
+ if addr and addr[1] then
+ return fun.totable(fun.map(function(a)
+ return a.domain
+ end, addr))
+ end
+ elseif filter == 'email:domain:tld' then
+ local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
+ if addr and addr[1] then
+ return fun.totable(fun.map(function(a)
+ return rspamd_util.get_tld(a.domain)
+ end, addr))
+ end
+ elseif filter == 'email:name' then
+ local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
+ if addr and addr[1] then
+ return fun.totable(fun.map(function(a)
+ return a.name
+ end, addr))
+ end
+ elseif filter == 'ip_addr' then
+ local ip_addr = rspamd_ip.from_string(input)
+
+ if ip_addr and ip_addr:is_valid() then
+ return ip_addr
+ end
+ else
+ -- regexp case
+ if not rule['re_filter'] then
+ local type, pat = string.match(filter, '(regexp:)(.+)')
+ if type and pat then
+ rule['re_filter'] = rspamd_regexp.create_cached(pat)
+ end
+ end
+
+ if not rule['re_filter'] then
+ rspamd_logger.errx(task, 'bad search filter: %s', filter)
+ else
+ local results = rule['re_filter']:search(input)
+ if results then
+ return results[1]
+ end
+ end
+ end
+
+ return input
+end
+local function apply_filename_filter(task, filter, fn, r)
+ if filter == 'extension' or filter == 'ext' then
+ return string.match(fn, '%.([^.]+)$')
+ elseif string.find(filter, 'regexp:') then
+ if not r['re_filter'] then
+ local type, pat = string.match(filter, '(regexp:)(.+)')
+ if type and pat then
+ r['re_filter'] = rspamd_regexp.create_cached(pat)
+ end
+ end
+
+ if not r['re_filter'] then
+ rspamd_logger.errx(task, 'bad search filter: %s', filter)
+ else
+ local results = r['re_filter']:search(fn)
+ if results then
+ return results[1]
+ else
+ return nil
+ end
+ end
+ end
+
+ return fn
+end
+
+local function apply_regexp_filter(task, filter, fn, r)
+ if string.find(filter, 'regexp:') then
+ if not r['re_filter'] then
+ local type, pat = string.match(filter, '(regexp:)(.+)')
+ if type and pat then
+ r['re_filter'] = rspamd_regexp.create_cached(pat)
+ end
+ end
+
+ if not r['re_filter'] then
+ rspamd_logger.errx(task, 'bad search filter: %s', filter)
+ else
+ local results = r['re_filter']:search(fn, false, true)
+ if results then
+ return results[1][2]
+ else
+ return nil
+ end
+ end
+ end
+
+ return fn
+end
+
+local function apply_content_filter(task, filter)
+ if filter == 'body' then
+ return { task:get_rawbody() }
+ elseif filter == 'full' then
+ return { task:get_content() }
+ elseif filter == 'headers' then
+ return { task:get_raw_headers() }
+ elseif filter == 'text' then
+ local ret = {}
+ for _, p in ipairs(task:get_text_parts()) do
+ table.insert(ret, p:get_content())
+ end
+ return ret
+ elseif filter == 'rawtext' then
+ local ret = {}
+ for _, p in ipairs(task:get_text_parts()) do
+ table.insert(ret, p:get_content('raw_parsed'))
+ end
+ return ret
+ elseif filter == 'oneline' then
+ local ret = {}
+ for _, p in ipairs(task:get_text_parts()) do
+ table.insert(ret, p:get_content_oneline())
+ end
+ return ret
+ else
+ rspamd_logger.errx(task, 'bad search filter: %s', filter)
+ end
+
+ return {}
+end
+
+local multimap_filters = {
+ from = apply_addr_filter,
+ rcpt = apply_addr_filter,
+ helo = apply_hostname_filter,
+ symbol_options = apply_regexp_filter,
+ header = apply_addr_filter,
+ url = apply_url_filter,
+ filename = apply_filename_filter,
+ mempool = apply_regexp_filter,
+ selector = apply_regexp_filter,
+ hostname = apply_hostname_filter,
+ --content = apply_content_filter, -- Content filters are special :(
+}
+
+local function multimap_query_redis(key, task, value, callback)
+ local cmd = 'HGET'
+ if type(value) == 'userdata' and value.class == 'rspamd{ip}' then
+ cmd = 'HMGET'
+ end
+
+ local srch = { key }
+
+ -- Insert all ips for some mask :(
+ if type(value) == 'userdata' and value.class == 'rspamd{ip}' then
+ srch[#srch + 1] = tostring(value)
+ -- IPv6 case
+ local maxbits = 128
+ local minbits = 64
+ if value:get_version() == 4 then
+ maxbits = 32
+ minbits = 8
+ end
+ for i = maxbits, minbits, -1 do
+ local nip = value:apply_mask(i):tostring() .. "/" .. i
+ srch[#srch + 1] = nip
+ end
+ else
+ srch[#srch + 1] = value
+ end
+
+ local function redis_map_cb(err, data)
+ lua_util.debugm(N, task, 'got reply from Redis when trying to get key %s: err=%s, data=%s',
+ key, err, data)
+ if not err and type(data) ~= 'userdata' then
+ callback(data)
+ end
+ end
+
+ return rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_map_cb, --callback
+ cmd, -- command
+ srch -- arguments
+ )
+end
+
+local function multimap_callback(task, rule)
+ local function match_element(r, value, callback)
+ if not value then
+ return false
+ end
+
+ local function get_key_callback(ret, err_or_data, err_code)
+ lua_util.debugm(N, task, 'got return "%s" (err code = %s) for multimap %s',
+ err_or_data,
+ err_code,
+ rule.symbol)
+
+ if ret then
+ if type(err_or_data) == 'table' then
+ for _, elt in ipairs(err_or_data) do
+ callback(elt)
+ end
+ else
+ callback(err_or_data)
+ end
+ elseif err_code ~= 404 then
+ rspamd_logger.infox(task, "map %s: get key returned error %s: %s",
+ rule.symbol, err_code, err_or_data)
+ end
+ end
+
+ lua_util.debugm(N, task, 'check value %s for multimap %s', value,
+ rule.symbol)
+
+ local ret = false
+
+ if r.redis_key then
+ -- Deal with hash name here: it can be either plain string or a selector
+ if type(r.redis_key) == 'string' then
+ ret = multimap_query_redis(r.redis_key, task, value, callback)
+ else
+ -- Here we have a selector
+ local results = r.redis_key(task)
+
+ -- Here we need to spill this function into multiple queries
+ if type(results) == 'table' then
+ for _, res in ipairs(results) do
+ ret = multimap_query_redis(res, task, value, callback)
+
+ if not ret then
+ break
+ end
+ end
+ else
+ ret = multimap_query_redis(results, task, value, callback)
+ end
+ end
+
+ return ret
+ elseif r.map_obj then
+ r.map_obj:get_key(value, get_key_callback, task)
+ end
+ end
+
+ local function insert_results(result, opt)
+ local _, symbol, score, opts = parse_multimap_value(rule, result)
+ local forced = false
+ if symbol then
+ if rule.symbols_set then
+ if not rule.symbols_set[symbol] then
+ rspamd_logger.infox(task, 'symbol %s is not registered for map %s, ' ..
+ 'replace it with just %s',
+ symbol, rule.symbol, rule.symbol)
+ symbol = rule.symbol
+ end
+ elseif rule.disable_multisymbol then
+ symbol = rule.symbol
+ if type(opt) == 'table' then
+ table.insert(opt, result)
+ elseif type(opt) ~= nil then
+ opt = { opt, result }
+ else
+ opt = { result }
+ end
+ else
+ forced = not rule.dynamic_symbols
+ end
+ else
+ symbol = rule.symbol
+ end
+
+ if opts and #opts > 0 then
+ -- Options come from the map itself
+ task:insert_result(forced, symbol, score, opts)
+ else
+ if opt then
+ if type(opt) == 'table' then
+ task:insert_result(forced, symbol, score, fun.totable(fun.map(tostring, opt)))
+ else
+ task:insert_result(forced, symbol, score, tostring(opt))
+ end
+
+ else
+ task:insert_result(forced, symbol, score)
+ end
+ end
+
+ if rule.action then
+ local message = rule.message
+ if rule.message_func then
+ message = rule.message_func(task, rule.symbol, opt)
+ end
+ if message then
+ task:set_pre_result(rule.action, message, N)
+ else
+ task:set_pre_result(rule.action, 'Matched map: ' .. rule.symbol, N)
+ end
+ end
+ end
+
+ -- Match a single value for against a single rule
+ local function match_rule(r, value)
+ local function rule_callback(result)
+ if result then
+ if type(result) == 'table' then
+ for _, rs in ipairs(result) do
+ if type(rs) ~= 'userdata' then
+ rule_callback(rs)
+ end
+ end
+ return
+ end
+ local opt = value_types[r['type']].get_value(value)
+ insert_results(result, opt)
+ end
+ end
+
+ if r.filter or r.type == 'url' then
+ local fn = multimap_filters[r.type]
+
+ if fn then
+
+ local filtered_value = fn(task, r.filter, value, r)
+ lua_util.debugm(N, task, 'apply filter %s for rule %s: %s -> %s',
+ r.filter, r.symbol, value, filtered_value)
+ value = filtered_value
+ end
+ end
+
+ if type(value) == 'table' then
+ fun.each(function(elt)
+ match_element(r, elt, rule_callback)
+ end, value)
+ else
+ match_element(r, value, rule_callback)
+ end
+ end
+
+ -- Match list of values according to the field
+ local function match_list(r, ls, fields)
+ if ls then
+ if fields then
+ fun.each(function(e)
+ local match = e[fields[1]]
+ if match then
+ if fields[2] then
+ match = fields[2](match)
+ end
+ match_rule(r, match)
+ end
+ end, ls)
+ else
+ fun.each(function(e)
+ match_rule(r, e)
+ end, ls)
+ end
+ end
+ end
+
+ local function match_addr(r, addr)
+ match_list(r, addr, { 'addr' })
+
+ if not r.filter then
+ match_list(r, addr, { 'domain' })
+ match_list(r, addr, { 'user' })
+ end
+ end
+
+ local function match_url(r, url)
+ match_rule(r, url)
+ end
+
+ local function match_hostname(r, hostname)
+ match_rule(r, hostname)
+ end
+
+ local function match_filename(r, fn)
+ match_rule(r, fn)
+ end
+
+ local function match_received_header(r, pos, total, h)
+ local use_tld = false
+ local filter = r['filter'] or 'real_ip'
+ if filter:match('^tld:') then
+ filter = filter:sub(5)
+ use_tld = true
+ end
+ local v = h[filter]
+ if v then
+ local min_pos = tonumber(r['min_pos'])
+ local max_pos = tonumber(r['max_pos'])
+ if min_pos then
+ if min_pos < 0 then
+ if min_pos == -1 then
+ if (pos ~= total) then
+ return
+ end
+ else
+ if pos <= (total - (min_pos * -1)) then
+ return
+ end
+ end
+ elseif pos < min_pos then
+ return
+ end
+ end
+ if max_pos then
+ if max_pos < -1 then
+ if (total - (max_pos * -1)) >= pos then
+ return
+ end
+ elseif max_pos > 0 then
+ if pos > max_pos then
+ return
+ end
+ end
+ end
+ local match_flags = r['flags']
+ local nmatch_flags = r['nflags']
+ if match_flags or nmatch_flags then
+ local got_flags = h['flags']
+ if match_flags then
+ for _, flag in ipairs(match_flags) do
+ if not got_flags[flag] then
+ return
+ end
+ end
+ end
+ if nmatch_flags then
+ for _, flag in ipairs(nmatch_flags) do
+ if got_flags[flag] then
+ return
+ end
+ end
+ end
+ end
+ if filter == 'real_ip' or filter == 'from_ip' then
+ if type(v) == 'string' then
+ v = rspamd_ip.from_string(v)
+ end
+ if v and v:is_valid() then
+ match_rule(r, v)
+ end
+ else
+ if use_tld and type(v) == 'string' then
+ v = rspamd_util.get_tld(v)
+ end
+ match_rule(r, v)
+ end
+ end
+ end
+
+ local function match_content(r)
+ local data
+
+ if r['filter'] then
+ data = apply_content_filter(task, r['filter'], r)
+ else
+ data = { task:get_content() }
+ end
+
+ for _, v in ipairs(data) do
+ match_rule(r, v)
+ end
+ end
+
+ if rule.expression and not rule.combined then
+ local res, trace = rule['expression']:process_traced(task)
+
+ if not res or res == 0 then
+ lua_util.debugm(N, task, 'condition is false for %s',
+ rule.symbol)
+ return
+ else
+ lua_util.debugm(N, task, 'condition is true for %s: %s',
+ rule.symbol,
+ trace)
+ end
+ end
+
+ local process_rule_funcs = {
+ ip = function()
+ local ip = task:get_from_ip()
+ if ip and ip:is_valid() then
+ match_rule(rule, ip)
+ end
+ end,
+ dnsbl = function()
+ local ip = task:get_from_ip()
+ if ip and ip:is_valid() then
+ local to_resolve = ip_to_rbl(ip, rule['map'])
+ local function dns_cb(_, _, results, err)
+ lua_util.debugm(N, rspamd_config,
+ 'resolve() finished: results=%1, err=%2, to_resolve=%3',
+ results, err, to_resolve)
+
+ if err and
+ (err ~= 'requested record is not found' and
+ err ~= 'no records with this name') then
+ rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, results)
+ elseif results then
+ task:insert_result(rule['symbol'], 1, rule['map'])
+ if rule.action then
+ task:set_pre_result(rule['action'],
+ 'Matched map: ' .. rule['symbol'], N)
+ end
+ end
+ end
+
+ task:get_resolver():resolve_a({
+ task = task,
+ name = to_resolve,
+ callback = dns_cb,
+ forced = true
+ })
+ end
+ end,
+ header = function()
+ if type(rule['header']) == 'table' then
+ for _, rh in ipairs(rule['header']) do
+ local hv = task:get_header_full(rh)
+ match_list(rule, hv, { 'decoded' })
+ end
+ else
+ local hv = task:get_header_full(rule['header'])
+ match_list(rule, hv, { 'decoded' })
+ end
+ end,
+ rcpt = function()
+ if task:has_recipients('smtp') then
+ local rcpts = task:get_recipients('smtp')
+ match_addr(rule, rcpts)
+ elseif task:has_recipients('mime') then
+ local rcpts = task:get_recipients('mime')
+ match_addr(rule, rcpts)
+ end
+ end,
+ from = function()
+ if task:has_from('smtp') then
+ local from = task:get_from('smtp')
+ match_addr(rule, from)
+ elseif task:has_from('mime') then
+ local from = task:get_from('mime')
+ match_addr(rule, from)
+ end
+ end,
+ helo = function()
+ local helo = task:get_helo()
+ if helo then
+ match_hostname(rule, helo)
+ end
+ end,
+ url = function()
+ if task:has_urls() then
+ local msg_urls = task:get_urls()
+
+ for _, url in ipairs(msg_urls) do
+ match_url(rule, url)
+ end
+ end
+ end,
+ user = function()
+ local user = task:get_user()
+ if user then
+ match_rule(rule, user)
+ end
+ end,
+ filename = function()
+ local parts = task:get_parts()
+
+ local function filter_parts(p)
+ return p:is_attachment() or (not p:is_text()) and (not p:is_multipart())
+ end
+
+ local function filter_archive(p)
+ local ext = p:get_detected_ext()
+ local det_type = 'unknown'
+
+ if ext then
+ local lua_magic_types = require "lua_magic/types"
+ local det_t = lua_magic_types[ext]
+
+ if det_t then
+ det_type = det_t.type
+ end
+ end
+
+ return p:is_archive() and det_type == 'archive' and not rule.skip_archives
+ end
+
+ for _, p in fun.iter(fun.filter(filter_parts, parts)) do
+ if filter_archive(p) then
+ local fnames = p:get_archive():get_files(1000)
+
+ for _, fn in ipairs(fnames) do
+ match_filename(rule, fn)
+ end
+ end
+
+ local fn = p:get_filename()
+ if fn then
+ match_filename(rule, fn)
+ end
+ -- Also deal with detected content type
+ if not rule.skip_detected then
+ local ext = p:get_detected_ext()
+
+ if ext then
+ local fake_fname = string.format('detected.%s', ext)
+ lua_util.debugm(N, task, 'detected filename %s',
+ fake_fname)
+ match_filename(rule, fake_fname)
+ end
+ end
+ end
+ end,
+
+ content = function()
+ match_content(rule)
+ end,
+ hostname = function()
+ local hostname = task:get_hostname()
+ if hostname then
+ match_hostname(rule, hostname)
+ end
+ end,
+ asn = function()
+ local asn = task:get_mempool():get_variable('asn')
+ if asn then
+ match_rule(rule, asn)
+ end
+ end,
+ country = function()
+ local country = task:get_mempool():get_variable('country')
+ if country then
+ match_rule(rule, country)
+ end
+ end,
+ mempool = function()
+ local var = task:get_mempool():get_variable(rule['variable'])
+ if var then
+ match_rule(rule, var)
+ end
+ end,
+ symbol_options = function()
+ local sym = task:get_symbol(rule['target_symbol'])
+ if sym and sym[1].options then
+ for _, o in ipairs(sym[1].options) do
+ match_rule(rule, o)
+ end
+ end
+ end,
+ received = function()
+ local hdrs = task:get_received_headers()
+ if hdrs and hdrs[1] then
+ if not rule['artificial'] then
+ hdrs = fun.filter(function(h)
+ return not h['flags']['artificial']
+ end, hdrs):totable()
+ end
+ for pos, h in ipairs(hdrs) do
+ match_received_header(rule, pos, #hdrs, h)
+ end
+ end
+ end,
+ selector = function()
+ local elts = rule.selector(task)
+
+ if elts then
+ if type(elts) == 'table' then
+ for _, elt in ipairs(elts) do
+ match_rule(rule, elt)
+ end
+ else
+ match_rule(rule, elts)
+ end
+ end
+ end,
+ combined = function()
+ local ret, trace = rule.combined:process(task)
+ if ret and ret ~= 0 then
+ for n, t in pairs(trace) do
+ insert_results(t.value, string.format("%s=%s",
+ n, t.matched))
+ end
+ end
+ end,
+ }
+
+ local rt = rule.type
+ local process_func = process_rule_funcs[rt]
+ if process_func then
+ process_func()
+ else
+ rspamd_logger.errx(task, 'Unrecognised rule type: %s', rt)
+ end
+end
+
+local function gen_multimap_callback(rule)
+ return function(task)
+ multimap_callback(task, rule)
+ end
+end
+
+local function multimap_on_load_gen(rule)
+ return function()
+ lua_util.debugm(N, rspamd_config, "loaded map object for rule %s", rule['symbol'])
+ local known_symbols = {}
+ rule.map_obj:foreach(function(key, value)
+ local r, symbol, score, _ = parse_multimap_value(rule, value)
+
+ if r and symbol and not known_symbols[symbol] then
+ lua_util.debugm(N, rspamd_config, "%s: adding new symbol %s (score = %s), triggered by %s",
+ rule.symbol, symbol, score, key)
+ rspamd_config:register_symbol {
+ name = value,
+ parent = rule.callback_id,
+ type = 'virtual',
+ score = score,
+ }
+ rspamd_config:set_metric_symbol({
+ group = N,
+ score = 1.0, -- In future, we will parse score from `get_value` and use it as multiplier
+ description = 'Automatic symbol generated by rule: ' .. rule.symbol,
+ name = value,
+ })
+ known_symbols[value] = true
+ end
+ end)
+ end
+end
+
+local function add_multimap_rule(key, newrule)
+ local ret = false
+
+ local function multimap_load_kv_map(rule)
+ if rule['regexp'] then
+ if rule['multi'] then
+ rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'regexp_multi',
+ rule.description)
+ else
+ rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'regexp',
+ rule.description)
+ end
+ elseif rule['glob'] then
+ if rule['multi'] then
+ rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'glob_multi',
+ rule.description)
+ else
+ rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'glob',
+ rule.description)
+ end
+ else
+ rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'hash',
+ rule.description)
+ end
+ end
+
+ local known_generic_types = {
+ header = true,
+ rcpt = true,
+ from = true,
+ helo = true,
+ symbol_options = true,
+ filename = true,
+ url = true,
+ user = true,
+ content = true,
+ hostname = true,
+ asn = true,
+ country = true,
+ mempool = true,
+ selector = true,
+ combined = true
+ }
+
+ if newrule['message_func'] then
+ newrule['message_func'] = assert(load(newrule['message_func']))()
+ end
+ if newrule['url'] and not newrule['map'] then
+ newrule['map'] = newrule['url']
+ end
+ if not (newrule.map or newrule.rules) then
+ rspamd_logger.errx(rspamd_config, 'incomplete rule, missing map')
+ return nil
+ end
+ if not newrule['symbol'] and key then
+ newrule['symbol'] = key
+ elseif not newrule['symbol'] then
+ rspamd_logger.errx(rspamd_config, 'incomplete rule, missing symbol')
+ return nil
+ end
+ if not newrule['description'] then
+ newrule['description'] = string.format('multimap, type %s: %s', newrule['type'],
+ newrule['symbol'])
+ end
+ if newrule['type'] == 'mempool' and not newrule['variable'] then
+ rspamd_logger.errx(rspamd_config, 'mempool map requires variable')
+ return nil
+ end
+ if newrule['type'] == 'selector' then
+ if not newrule['selector'] then
+ rspamd_logger.errx(rspamd_config, 'selector map requires selector definition')
+ return nil
+ else
+ local selector = lua_selectors.create_selector_closure(
+ rspamd_config, newrule['selector'], newrule['delimiter'] or "")
+
+ if not selector then
+ rspamd_logger.errx(rspamd_config, 'selector map has invalid selector: "%s", symbol: %s',
+ newrule['selector'], newrule['symbol'])
+ return nil
+ end
+
+ newrule.selector = selector
+ end
+ end
+ if type(newrule['map']) == 'string' and
+ string.find(newrule['map'], '^redis://.*$') then
+ if not redis_params then
+ rspamd_logger.infox(rspamd_config, 'no redis servers are specified, ' ..
+ 'cannot add redis map %s: %s', newrule['symbol'], newrule['map'])
+ return nil
+ end
+
+ newrule['redis_key'] = string.match(newrule['map'], '^redis://(.*)$')
+
+ if newrule['redis_key'] then
+ ret = true
+ end
+ elseif type(newrule['map']) == 'string' and
+ string.find(newrule['map'], '^redis%+selector://.*$') then
+ if not redis_params then
+ rspamd_logger.infox(rspamd_config, 'no redis servers are specified, ' ..
+ 'cannot add redis map %s: %s', newrule['symbol'], newrule['map'])
+ return nil
+ end
+
+ local selector_str = string.match(newrule['map'], '^redis%+selector://(.*)$')
+ local selector = lua_selectors.create_selector_closure(
+ rspamd_config, selector_str, newrule['delimiter'] or "")
+
+ if not selector then
+ rspamd_logger.errx(rspamd_config, 'redis selector map has invalid selector: "%s", symbol: %s',
+ selector_str, newrule['symbol'])
+ return nil
+ end
+
+ newrule['redis_key'] = selector
+ ret = true
+ elseif newrule.type == 'combined' then
+ local lua_maps_expressions = require "lua_maps_expressions"
+ newrule.combined = lua_maps_expressions.create(rspamd_config,
+ {
+ rules = newrule.rules,
+ expression = newrule.expression,
+ on_load = newrule.dynamic_symbols and multimap_on_load_gen(newrule) or nil,
+ }, N, 'Combined map for ' .. newrule.symbol)
+ if not newrule.combined then
+ rspamd_logger.errx(rspamd_config, 'cannot add combined map for %s', newrule.symbol)
+ else
+ ret = true
+ end
+ else
+ if newrule['type'] == 'ip' then
+ newrule.map_obj = lua_maps.map_add_from_ucl(newrule.map, 'radix',
+ newrule.description)
+ if newrule.map_obj then
+ ret = true
+ else
+ rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1',
+ newrule['map'])
+ end
+ elseif newrule['type'] == 'received' then
+ if type(newrule['flags']) == 'table' and newrule['flags'][1] then
+ newrule['flags'] = newrule['flags']
+ elseif type(newrule['flags']) == 'string' then
+ newrule['flags'] = { newrule['flags'] }
+ end
+ if type(newrule['nflags']) == 'table' and newrule['nflags'][1] then
+ newrule['nflags'] = newrule['nflags']
+ elseif type(newrule['nflags']) == 'string' then
+ newrule['nflags'] = { newrule['nflags'] }
+ end
+ local filter = newrule['filter'] or 'real_ip'
+ if filter == 'real_ip' or filter == 'from_ip' then
+ newrule.map_obj = lua_maps.map_add_from_ucl(newrule.map, 'radix',
+ newrule.description)
+ if newrule.map_obj then
+ ret = true
+ else
+ rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1',
+ newrule['map'])
+ end
+ else
+ multimap_load_kv_map(newrule)
+
+ if newrule.map_obj then
+ ret = true
+ else
+ rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1',
+ newrule['map'])
+ end
+ end
+ elseif known_generic_types[newrule.type] then
+
+ if newrule.filter == 'ip_addr' then
+ newrule.map_obj = lua_maps.map_add_from_ucl(newrule.map, 'radix',
+ newrule.description)
+ elseif not newrule.combined then
+ multimap_load_kv_map(newrule)
+ end
+
+ if newrule.map_obj then
+ ret = true
+ else
+ rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1',
+ newrule['map'])
+ end
+ elseif newrule['type'] == 'dnsbl' then
+ ret = true
+ end
+ end
+
+ if ret then
+ if newrule.map_obj and newrule.dynamic_symbols then
+ newrule.map_obj:on_load(multimap_on_load_gen(newrule))
+ end
+ if newrule['type'] == 'symbol_options' then
+ rspamd_config:register_dependency(newrule['symbol'], newrule['target_symbol'])
+ end
+ if newrule['require_symbols'] then
+ local atoms = {}
+
+ local function parse_atom(str)
+ local atom = table.concat(fun.totable(fun.take_while(function(c)
+ if string.find(', \t()><+!|&\n', c, 1, true) then
+ return false
+ end
+ return true
+ end, fun.iter(str))), '')
+ table.insert(atoms, atom)
+ return atom
+ end
+
+ local function process_atom(atom, task)
+ local f_ret = task:has_symbol(atom)
+ lua_util.debugm(N, rspamd_config, 'check for symbol %s: %s', atom, f_ret)
+
+ if f_ret then
+ return 1
+ end
+
+ return 0
+ end
+
+ local expression = rspamd_expression.create(newrule['require_symbols'],
+ { parse_atom, process_atom }, rspamd_config:get_mempool())
+ if expression then
+ newrule['expression'] = expression
+
+ fun.each(function(v)
+ lua_util.debugm(N, rspamd_config, 'add dependency %s -> %s',
+ newrule['symbol'], v)
+ rspamd_config:register_dependency(newrule['symbol'], v)
+ end, atoms)
+ end
+ end
+ return newrule
+ end
+
+ return nil
+end
+
+-- Registration
+local opts = rspamd_config:get_all_opt(N)
+if opts and type(opts) == 'table' then
+ redis_params = rspamd_parse_redis_server(N)
+ for k, m in pairs(opts) do
+ if type(m) == 'table' and m['type'] then
+ local rule = add_multimap_rule(k, m)
+ if not rule then
+ rspamd_logger.errx(rspamd_config, 'cannot add rule: "' .. k .. '"')
+ else
+ rspamd_logger.infox(rspamd_config, 'added multimap rule: %s (%s)',
+ k, rule.type)
+ table.insert(rules, rule)
+ end
+ end
+ end
+ -- add fake symbol to check all maps inside a single callback
+ fun.each(function(rule)
+ local augmentations = {}
+
+ if rule.action then
+ table.insert(augmentations, 'passthrough')
+ end
+
+ local id = rspamd_config:register_symbol({
+ type = 'normal',
+ name = rule['symbol'],
+ augmentations = augmentations,
+ callback = gen_multimap_callback(rule),
+ })
+
+ rule.callback_id = id
+
+ if rule['symbols'] then
+ -- Find allowed symbols by this map
+ rule['symbols_set'] = {}
+ fun.each(function(s)
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = s,
+ parent = id,
+ score = tonumber(rule.score or "0") or 0, -- Default score
+ })
+ rule['symbols_set'][s] = 1
+ end, rule['symbols'])
+ end
+ if not rule.score then
+ rspamd_logger.infox(rspamd_config, 'set default score 0 for multimap rule %s', rule.symbol)
+ rule.score = 0
+ end
+ if rule.score then
+ -- Register metric symbol
+ rule.name = rule.symbol
+ rule.description = rule.description or 'multimap symbol'
+ rule.group = rule.group or N
+
+ local tmp_flags
+ tmp_flags = rule.flags
+
+ if rule.type == 'received' and rule.flags then
+ -- XXX: hack to allow received flags/nflags
+ -- See issue #3526 on GH
+ rule.flags = nil
+ end
+
+ -- XXX: for combined maps we use trace, so flags must include one_shot to avoid scores multiplication
+ if rule.combined and not rule.flags then
+ rule.flags = 'one_shot'
+ end
+ rspamd_config:set_metric_symbol(rule)
+ rule.flags = tmp_flags
+ end
+ end, rules)
+
+ if #rules == 0 then
+ lua_util.disable_module(N, "config")
+ end
+end
diff --git a/src/plugins/lua/mx_check.lua b/src/plugins/lua/mx_check.lua
new file mode 100644
index 0000000..71892b9
--- /dev/null
+++ b/src/plugins/lua/mx_check.lua
@@ -0,0 +1,392 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- MX check plugin
+local rspamd_logger = require "rspamd_logger"
+local rspamd_tcp = require "rspamd_tcp"
+local rspamd_util = require "rspamd_util"
+local lua_util = require "lua_util"
+local lua_redis = require "lua_redis"
+local N = "mx_check"
+local fun = require "fun"
+
+local settings = {
+ timeout = 1.0, -- connect timeout
+ symbol_bad_mx = 'MX_INVALID',
+ symbol_no_mx = 'MX_MISSING',
+ symbol_good_mx = 'MX_GOOD',
+ symbol_white_mx = 'MX_WHITE',
+ expire = 86400, -- 1 day by default
+ expire_novalid = 7200, -- 2 hours by default for no valid mxes
+ greylist_invalid = true, -- Greylist first message with invalid MX (require greylist plugin)
+ key_prefix = 'rmx',
+ max_mx_a_records = 5, -- Maximum number of A records to check per MX request
+ wait_for_greeting = false, -- Wait for SMTP greeting and emit `quit` command
+}
+local redis_params
+local exclude_domains
+
+local E = {}
+local CRLF = '\r\n'
+local mx_miss_cache_prefix = 'mx_miss:'
+
+local function mx_check(task)
+ local ip_addr = task:get_ip()
+ if task:get_user() or (ip_addr and ip_addr:is_local()) then
+ return
+ end
+
+ local from = task:get_from('smtp')
+ local mx_domain
+ if ((from or E)[1] or E).domain and not from[2] then
+ mx_domain = from[1]['domain']
+ else
+ mx_domain = task:get_helo()
+
+ if mx_domain then
+ mx_domain = rspamd_util.get_tld(mx_domain)
+ end
+ end
+
+ if not mx_domain then
+ return
+ end
+
+ if exclude_domains then
+ if exclude_domains:get_key(mx_domain) then
+ rspamd_logger.infox(task, 'skip mx check for %s, excluded', mx_domain)
+ task:insert_result(settings.symbol_white_mx, 1.0, mx_domain)
+ return
+ end
+ end
+
+ local valid = false
+
+ local function check_results(mxes)
+ if fun.all(function(_, elt)
+ return elt.checked
+ end, mxes) then
+ -- Save cache
+ local key = settings.key_prefix .. mx_domain
+ local function redis_cache_cb(err)
+ if err ~= nil then
+ rspamd_logger.errx(task, 'redis_cache_cb received error: %1', err)
+ return
+ end
+ end
+ if not valid then
+ -- Greylist message
+ if settings.greylist_invalid then
+ task:get_mempool():set_variable("grey_greylisted_required", "1")
+ lua_util.debugm(N, task, "advice to greylist a message")
+ task:insert_result(settings.symbol_bad_mx, 1.0, "greylisted")
+ else
+ task:insert_result(settings.symbol_bad_mx, 1.0)
+ end
+ local ret = rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_cache_cb, --callback
+ 'SETEX', -- command
+ { key, tostring(settings.expire_novalid), '0' } -- arguments
+ )
+ lua_util.debugm(N, task, "set redis cache key: %s; invalid MX", key)
+ if not ret then
+ rspamd_logger.errx(task, 'got error connecting to redis')
+ end
+ else
+ local valid_mx = {}
+ fun.each(function(k)
+ table.insert(valid_mx, k)
+ end, fun.filter(function(_, elt)
+ return elt.working
+ end, mxes))
+ task:insert_result(settings.symbol_good_mx, 1.0, valid_mx)
+ local value = table.concat(valid_mx, ';')
+ if mxes[mx_domain] and type(mxes[mx_domain]) == 'table' and mxes[mx_domain].mx_missing then
+ value = mx_miss_cache_prefix .. value
+ end
+ local ret = rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_cache_cb, --callback
+ 'SETEX', -- command
+ { key, tostring(settings.expire), value } -- arguments
+ )
+ lua_util.debugm(N, task, "set redis cache key: %s; %s", key, value)
+ if not ret then
+ rspamd_logger.errx(task, 'error connecting to redis')
+ end
+ end
+ end
+ end
+
+ local function gen_mx_a_callback(name, mxes)
+ return function(_, _, results, err)
+ lua_util.debugm(N, task, "got DNS results for %s: %s", name, results)
+ mxes[name].ips = results
+
+ local function io_cb(io_err, _, conn)
+ lua_util.debugm(N, task, "TCP IO callback for %s, error: %s", name, io_err)
+ if io_err then
+ mxes[name].checked = true
+ conn:close()
+ else
+ mxes[name].checked = true
+ mxes[name].working = true
+ valid = true
+ if settings.wait_for_greeting then
+ conn:add_write(function(_)
+ conn:close()
+ end, string.format('QUIT%s', CRLF))
+ end
+ end
+ check_results(mxes)
+ end
+ local function on_connect_cb(conn)
+ lua_util.debugm(N, task, "TCP connect callback for %s, error: %s", name, err)
+ if err then
+ mxes[name].checked = true
+ conn:close()
+ check_results(mxes)
+ else
+ mxes[name].checked = true
+ valid = true
+ mxes[name].working = true
+ end
+
+ -- Disconnect without SMTP dialog
+ if not settings.wait_for_greeting then
+ check_results(mxes)
+ conn:close()
+ end
+ end
+
+ if err or not results or #results == 0 then
+ mxes[name].checked = true
+ else
+ -- Try to open TCP connection to port 25 for a random IP address
+ -- see #3839 on GitHub
+ lua_util.shuffle(results)
+ local str_ip = results[1]:to_string()
+ lua_util.debugm(N, task, "trying to connect to IP %s", str_ip)
+ local t_ret = rspamd_tcp.new({
+ task = task,
+ host = str_ip,
+ callback = io_cb,
+ stop_pattern = CRLF,
+ on_connect = on_connect_cb,
+ timeout = settings.timeout,
+ port = 25
+ })
+
+ if not t_ret then
+ mxes[name].checked = true
+ end
+ end
+ check_results(mxes)
+ end
+ end
+
+ local function mx_callback(_, _, results, err)
+ local mxes = {}
+ if err or not results then
+ local r = task:get_resolver()
+ -- XXX: maybe add ipv6?
+ -- fallback to implicit mx
+ if not err and not results then
+ err = 'no MX records found'
+ end
+
+ lua_util.debugm(N, task, "cannot find MX record for %s: %s, use implicit fallback",
+ mx_domain, err)
+ mxes[mx_domain] = { checked = false, working = false, ips = {}, mx_missing = true }
+ r:resolve('a', {
+ name = mx_domain,
+ callback = gen_mx_a_callback(mx_domain, mxes),
+ task = task,
+ forced = true
+ })
+ task:insert_result(settings.symbol_no_mx, 1.0, err)
+ else
+ -- Inverse sort by priority
+ table.sort(results, function(r1, r2)
+ return r1['priority'] > r2['priority']
+ end)
+
+ local max_mx_to_resolve = math.min(#results, settings.max_mx_a_records)
+ lua_util.debugm(N, task, 'check %s MX records (%d actually returned)',
+ max_mx_to_resolve, #results)
+ for i = 1, max_mx_to_resolve do
+ local mx = results[i]
+ mxes[mx.name] = { checked = false, working = false, ips = {} }
+ local r = task:get_resolver()
+ -- XXX: maybe add ipv6?
+ r:resolve('a', {
+ name = mx.name,
+ callback = gen_mx_a_callback(mx.name, mxes),
+ task = task,
+ forced = true
+ })
+ end
+ check_results(mxes)
+ end
+ end
+
+ if not redis_params then
+ local r = task:get_resolver()
+ r:resolve('mx', {
+ name = mx_domain,
+ callback = mx_callback,
+ task = task,
+ forced = true
+ })
+ else
+ local function redis_cache_get_cb(err, data)
+ if err or type(data) ~= 'string' then
+ local r = task:get_resolver()
+ r:resolve('mx', {
+ name = mx_domain,
+ callback = mx_callback,
+ task = task,
+ forced = true
+ })
+ else
+ if data == '0' then
+ task:insert_result(settings.symbol_bad_mx, 1.0, 'cached')
+ else
+ if lua_util.str_startswith(data, mx_miss_cache_prefix) then
+ task:insert_result(settings.symbol_no_mx, 1.0, 'cached')
+ data = string.sub(data, #mx_miss_cache_prefix + 1)
+ end
+ local mxes = lua_util.str_split(data, ';')
+ task:insert_result(settings.symbol_good_mx, 1.0, 'cached: ' .. mxes[1])
+ end
+ end
+ end
+
+ local key = settings.key_prefix .. mx_domain
+ local ret = rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_cache_get_cb, --callback
+ 'GET', -- command
+ { key } -- arguments
+ )
+
+ if not ret then
+ local r = task:get_resolver()
+ r:resolve('mx', {
+ name = mx_domain,
+ callback = mx_callback,
+ task = task,
+ forced = true
+ })
+ end
+ end
+end
+
+-- Module setup
+local opts = rspamd_config:get_all_opt('mx_check')
+if not (opts and type(opts) == 'table') then
+ rspamd_logger.infox(rspamd_config, 'module is unconfigured')
+ return
+end
+if opts then
+ redis_params = lua_redis.parse_redis_server('mx_check')
+ if not redis_params then
+ rspamd_logger.errx(rspamd_config, 'no redis servers are specified, disabling module')
+ lua_util.disable_module(N, "redis")
+ return
+ end
+
+ settings = lua_util.override_defaults(settings, opts)
+ lua_redis.register_prefix(settings.key_prefix .. '*', N,
+ 'MX check cache', {
+ type = 'string',
+ })
+
+ local id = rspamd_config:register_symbol({
+ name = settings.symbol_bad_mx,
+ type = 'normal',
+ callback = mx_check,
+ flags = 'empty',
+ augmentations = { string.format("timeout=%f", settings.timeout + rspamd_config:get_dns_timeout() or 0.0) },
+ })
+ rspamd_config:register_symbol({
+ name = settings.symbol_no_mx,
+ type = 'virtual',
+ parent = id
+ })
+ rspamd_config:register_symbol({
+ name = settings.symbol_good_mx,
+ type = 'virtual',
+ parent = id
+ })
+ rspamd_config:register_symbol({
+ name = settings.symbol_white_mx,
+ type = 'virtual',
+ parent = id
+ })
+
+ rspamd_config:set_metric_symbol({
+ name = settings.symbol_bad_mx,
+ score = 0.5,
+ description = 'Domain has no working MX',
+ group = 'MX',
+ one_shot = true,
+ one_param = true,
+ })
+ rspamd_config:set_metric_symbol({
+ name = settings.symbol_good_mx,
+ score = -0.01,
+ description = 'Domain has working MX',
+ group = 'MX',
+ one_shot = true,
+ one_param = true,
+ })
+ rspamd_config:set_metric_symbol({
+ name = settings.symbol_white_mx,
+ score = 0.0,
+ description = 'Domain is whitelisted from MX check',
+ group = 'MX',
+ one_shot = true,
+ one_param = true,
+ })
+ rspamd_config:set_metric_symbol({
+ name = settings.symbol_no_mx,
+ score = 3.5,
+ description = 'Domain has no resolvable MX',
+ group = 'MX',
+ one_shot = true,
+ one_param = true,
+ })
+
+ if settings.exclude_domains then
+ exclude_domains = rspamd_config:add_map {
+ type = 'set',
+ description = 'Exclude specific domains from MX checks',
+ url = settings.exclude_domains,
+ }
+ end
+end
diff --git a/src/plugins/lua/neural.lua b/src/plugins/lua/neural.lua
new file mode 100644
index 0000000..f3b26f1
--- /dev/null
+++ b/src/plugins/lua/neural.lua
@@ -0,0 +1,1000 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+
+if confighelp then
+ return
+end
+
+local fun = require "fun"
+local lua_redis = require "lua_redis"
+local lua_util = require "lua_util"
+local lua_verdict = require "lua_verdict"
+local neural_common = require "plugins/neural"
+local rspamd_kann = require "rspamd_kann"
+local rspamd_logger = require "rspamd_logger"
+local rspamd_tensor = require "rspamd_tensor"
+local rspamd_text = require "rspamd_text"
+local rspamd_util = require "rspamd_util"
+local ts = require("tableshape").types
+
+local N = "neural"
+
+local settings = neural_common.settings
+
+local redis_profile_schema = ts.shape {
+ digest = ts.string,
+ symbols = ts.array_of(ts.string),
+ version = ts.number,
+ redis_key = ts.string,
+ distance = ts.number:is_optional(),
+}
+
+local has_blas = rspamd_tensor.has_blas()
+local text_cookie = rspamd_text.cookie
+
+-- Creates and stores ANN profile in Redis
+local function new_ann_profile(task, rule, set, version)
+ local ann_key = neural_common.new_ann_key(rule, set, version, settings)
+
+ local profile = {
+ symbols = set.symbols,
+ redis_key = ann_key,
+ version = version,
+ digest = set.digest,
+ distance = 0 -- Since we are using our own profile
+ }
+
+ local ucl = require "ucl"
+ local profile_serialized = ucl.to_format(profile, 'json-compact', true)
+
+ local function add_cb(err, _)
+ if err then
+ rspamd_logger.errx(task, 'cannot store ANN profile for %s:%s at %s : %s',
+ rule.prefix, set.name, profile.redis_key, err)
+ else
+ rspamd_logger.infox(task, 'created new ANN profile for %s:%s, data stored at prefix %s',
+ rule.prefix, set.name, profile.redis_key)
+ end
+ end
+
+ lua_redis.redis_make_request(task,
+ rule.redis,
+ nil,
+ true, -- is write
+ add_cb, --callback
+ 'ZADD', -- command
+ { set.prefix, tostring(rspamd_util.get_time()), profile_serialized }
+ )
+
+ return profile
+end
+
+
+-- ANN filter function, used to insert scores based on the existing symbols
+local function ann_scores_filter(task)
+
+ for _, rule in pairs(settings.rules) do
+ local sid = task:get_settings_id() or -1
+ local ann
+ local profile
+
+ local set = neural_common.get_rule_settings(task, rule)
+ if set then
+ if set.ann then
+ ann = set.ann.ann
+ profile = set.ann
+ else
+ lua_util.debugm(N, task, 'no ann loaded for %s:%s',
+ rule.prefix, set.name)
+ end
+ else
+ lua_util.debugm(N, task, 'no ann defined in %s for settings id %s',
+ rule.prefix, sid)
+ end
+
+ if ann then
+ local vec = neural_common.result_to_vector(task, profile)
+
+ local score
+ local out = ann:apply1(vec, set.ann.pca)
+ score = out[1]
+
+ local symscore = string.format('%.3f', score)
+ task:cache_set(rule.prefix .. '_neural_score', score)
+ lua_util.debugm(N, task, '%s:%s:%s ann score: %s',
+ rule.prefix, set.name, set.ann.version, symscore)
+
+ if score > 0 then
+ local result = score
+
+ -- If spam_score_threshold is defined, override all other thresholds.
+ local spam_threshold = 0
+ if rule.spam_score_threshold then
+ spam_threshold = rule.spam_score_threshold
+ elseif rule.roc_enabled and not set.ann.roc_thresholds then
+ spam_threshold = set.ann.roc_thresholds[1]
+ end
+
+ if result >= spam_threshold then
+ if rule.flat_threshold_curve then
+ task:insert_result(rule.symbol_spam, 1.0, symscore)
+ else
+ task:insert_result(rule.symbol_spam, result, symscore)
+ end
+ else
+ lua_util.debugm(N, task, '%s:%s:%s ann score: %s < %s (spam threshold)',
+ rule.prefix, set.name, set.ann.version, symscore,
+ spam_threshold)
+ end
+ else
+ local result = -(score)
+
+ -- If ham_score_threshold is defined, override all other thresholds.
+ local ham_threshold = 0
+ if rule.ham_score_threshold then
+ ham_threshold = rule.ham_score_threshold
+ elseif rule.roc_enabled and not set.ann.roc_thresholds then
+ ham_threshold = set.ann.roc_thresholds[2]
+ end
+
+ if result >= ham_threshold then
+ if rule.flat_threshold_curve then
+ task:insert_result(rule.symbol_ham, 1.0, symscore)
+ else
+ task:insert_result(rule.symbol_ham, result, symscore)
+ end
+ else
+ lua_util.debugm(N, task, '%s:%s:%s ann score: %s < %s (ham threshold)',
+ rule.prefix, set.name, set.ann.version, result,
+ ham_threshold)
+ end
+ end
+ end
+ end
+end
+
+local function ann_push_task_result(rule, task, verdict, score, set)
+ local train_opts = rule.train
+ local learn_spam, learn_ham
+ local skip_reason = 'unknown'
+
+ if not train_opts.store_pool_only and train_opts.autotrain then
+ if train_opts.spam_score then
+ learn_spam = score >= train_opts.spam_score
+
+ if not learn_spam then
+ skip_reason = string.format('score < spam_score: %f < %f',
+ score, train_opts.spam_score)
+ end
+ else
+ learn_spam = verdict == 'spam' or verdict == 'junk'
+
+ if not learn_spam then
+ skip_reason = string.format('verdict: %s',
+ verdict)
+ end
+ end
+
+ if train_opts.ham_score then
+ learn_ham = score <= train_opts.ham_score
+ if not learn_ham then
+ skip_reason = string.format('score > ham_score: %f > %f',
+ score, train_opts.ham_score)
+ end
+ else
+ learn_ham = verdict == 'ham'
+
+ if not learn_ham then
+ skip_reason = string.format('verdict: %s',
+ verdict)
+ end
+ end
+ else
+ -- Train by request header
+ local hdr = task:get_request_header('ANN-Train')
+
+ if hdr then
+ if hdr:lower() == 'spam' then
+ learn_spam = true
+ elseif hdr:lower() == 'ham' then
+ learn_ham = true
+ else
+ skip_reason = 'no explicit header'
+ end
+ elseif train_opts.store_pool_only then
+ local ucl = require "ucl"
+ learn_ham = false
+ learn_spam = false
+
+ -- Explicitly store tokens in cache
+ local vec = neural_common.result_to_vector(task, set)
+ task:cache_set(rule.prefix .. '_neural_vec_mpack', ucl.to_format(vec, 'msgpack'))
+ task:cache_set(rule.prefix .. '_neural_profile_digest', set.digest)
+ skip_reason = 'store_pool_only has been set'
+ end
+ end
+
+ if learn_spam or learn_ham then
+ local learn_type
+ if learn_spam then
+ learn_type = 'spam'
+ else
+ learn_type = 'ham'
+ end
+
+ local function vectors_len_cb(err, data)
+ if not err and type(data) == 'table' then
+ local nspam, nham = data[1], data[2]
+
+ if neural_common.can_push_train_vector(rule, task, learn_type, nspam, nham) then
+ local vec = neural_common.result_to_vector(task, set)
+
+ local str = rspamd_util.zstd_compress(table.concat(vec, ';'))
+ local target_key = set.ann.redis_key .. '_' .. learn_type .. '_set'
+
+ local function learn_vec_cb(redis_err)
+ if redis_err then
+ rspamd_logger.errx(task, 'cannot store train vector for %s:%s: %s',
+ rule.prefix, set.name, redis_err)
+ else
+ lua_util.debugm(N, task,
+ "add train data for ANN rule " ..
+ "%s:%s, save %s vector of %s elts in %s key; %s bytes compressed",
+ rule.prefix, set.name, learn_type, #vec, target_key, #str)
+ end
+ end
+
+ lua_redis.redis_make_request(task,
+ rule.redis,
+ nil,
+ true, -- is write
+ learn_vec_cb, --callback
+ 'SADD', -- command
+ { target_key, str } -- arguments
+ )
+ else
+ lua_util.debugm(N, task,
+ "do not add %s train data for ANN rule " ..
+ "%s:%s",
+ learn_type, rule.prefix, set.name)
+ end
+ else
+ if err then
+ rspamd_logger.errx(task, 'cannot check if we can train %s:%s : %s',
+ rule.prefix, set.name, err)
+ elseif type(data) == 'string' then
+ -- nil return value
+ rspamd_logger.infox(task, "cannot learn %s ANN %s:%s; redis_key: %s: locked for learning: %s",
+ learn_type, rule.prefix, set.name, set.ann.redis_key, data)
+ else
+ rspamd_logger.errx(task, 'cannot check if we can train %s:%s : type of Redis key %s is %s, expected table' ..
+ 'please remove this key from Redis manually if you perform upgrade from the previous version',
+ rule.prefix, set.name, set.ann.redis_key, type(data))
+ end
+ end
+ end
+
+ -- Check if we can learn
+ if set.can_store_vectors then
+ if not set.ann then
+ -- Need to create or load a profile corresponding to the current configuration
+ set.ann = new_ann_profile(task, rule, set, 0)
+ lua_util.debugm(N, task,
+ 'requested new profile for %s, set.ann is missing',
+ set.name)
+ end
+
+ lua_redis.exec_redis_script(neural_common.redis_script_id.vectors_len,
+ { task = task, is_write = false },
+ vectors_len_cb,
+ {
+ set.ann.redis_key,
+ })
+ else
+ lua_util.debugm(N, task,
+ 'do not push data: train condition not satisfied; reason: not checked existing ANNs')
+ end
+ else
+ lua_util.debugm(N, task,
+ 'do not push data to key %s: train condition not satisfied; reason: %s',
+ (set.ann or {}).redis_key,
+ skip_reason)
+ end
+end
+
+--- Offline training logic
+
+-- Utility to extract and split saved training vectors to a table of tables
+local function process_training_vectors(data)
+ return fun.totable(fun.map(function(tok)
+ local _, str = rspamd_util.zstd_decompress(tok)
+ return fun.totable(fun.map(tonumber, lua_util.str_split(tostring(str), ';')))
+ end, data))
+end
+
+-- This function does the following:
+-- * Tries to lock ANN
+-- * Loads spam and ham vectors
+-- * Spawn learning process
+local function do_train_ann(worker, ev_base, rule, set, ann_key)
+ local spam_elts = {}
+ local ham_elts = {}
+
+ local function redis_ham_cb(err, data)
+ if err or type(data) ~= 'table' then
+ rspamd_logger.errx(rspamd_config, 'cannot get ham tokens for ANN %s from redis: %s',
+ ann_key, err)
+ -- Unlock on error
+ lua_redis.redis_make_request_taskless(ev_base,
+ rspamd_config,
+ rule.redis,
+ nil,
+ true, -- is write
+ neural_common.gen_unlock_cb(rule, set, ann_key), --callback
+ 'HDEL', -- command
+ { ann_key, 'lock' }
+ )
+ else
+ -- Decompress and convert to numbers each training vector
+ ham_elts = process_training_vectors(data)
+ neural_common.spawn_train({ worker = worker, ev_base = ev_base,
+ rule = rule, set = set, ann_key = ann_key, ham_vec = ham_elts,
+ spam_vec = spam_elts })
+ end
+ end
+
+ -- Spam vectors received
+ local function redis_spam_cb(err, data)
+ if err or type(data) ~= 'table' then
+ rspamd_logger.errx(rspamd_config, 'cannot get spam tokens for ANN %s from redis: %s',
+ ann_key, err)
+ -- Unlock ANN on error
+ lua_redis.redis_make_request_taskless(ev_base,
+ rspamd_config,
+ rule.redis,
+ nil,
+ true, -- is write
+ neural_common.gen_unlock_cb(rule, set, ann_key), --callback
+ 'HDEL', -- command
+ { ann_key, 'lock' }
+ )
+ else
+ -- Decompress and convert to numbers each training vector
+ spam_elts = process_training_vectors(data)
+ -- Now get ham vectors...
+ lua_redis.redis_make_request_taskless(ev_base,
+ rspamd_config,
+ rule.redis,
+ nil,
+ false, -- is write
+ redis_ham_cb, --callback
+ 'SMEMBERS', -- command
+ { ann_key .. '_ham_set' }
+ )
+ end
+ end
+
+ local function redis_lock_cb(err, data)
+ if err then
+ rspamd_logger.errx(rspamd_config, 'cannot call lock script for ANN %s from redis: %s',
+ ann_key, err)
+ elseif type(data) == 'number' and data == 1 then
+ -- ANN is locked, so we can extract SPAM and HAM vectors and spawn learning
+ lua_redis.redis_make_request_taskless(ev_base,
+ rspamd_config,
+ rule.redis,
+ nil,
+ false, -- is write
+ redis_spam_cb, --callback
+ 'SMEMBERS', -- command
+ { ann_key .. '_spam_set' }
+ )
+
+ rspamd_logger.infox(rspamd_config, 'lock ANN %s:%s (key name %s) for learning',
+ rule.prefix, set.name, ann_key)
+ else
+ local lock_tm = tonumber(data[1])
+ rspamd_logger.infox(rspamd_config, 'do not learn ANN %s:%s (key name %s), ' ..
+ 'locked by another host %s at %s', rule.prefix, set.name, ann_key,
+ data[2], os.date('%c', lock_tm))
+ end
+ end
+
+ -- Check if we are already learning this network
+ if set.learning_spawned then
+ rspamd_logger.infox(rspamd_config, 'do not learn ANN %s, already learning another ANN',
+ ann_key)
+ return
+ end
+
+ -- Call Redis script that tries to acquire a lock
+ -- This script returns either a boolean or a pair {'lock_time', 'hostname'} when
+ -- ANN is locked by another host (or a process, meh)
+ lua_redis.exec_redis_script(neural_common.redis_script_id.maybe_lock,
+ { ev_base = ev_base, is_write = true },
+ redis_lock_cb,
+ {
+ ann_key,
+ tostring(os.time()),
+ tostring(math.max(10.0, rule.watch_interval * 2)),
+ rspamd_util.get_hostname()
+ })
+end
+
+-- This function loads new ann from Redis
+-- This is based on `profile` attribute.
+-- ANN is loaded from `profile.redis_key`
+-- Rank of `profile` key is also increased, unfortunately, it means that we need to
+-- serialize profile one more time and set its rank to the current time
+-- set.ann fields are set according to Redis data received
+local function load_new_ann(rule, ev_base, set, profile, min_diff)
+ local ann_key = profile.redis_key
+
+ local function data_cb(err, data)
+ if err then
+ rspamd_logger.errx(rspamd_config, 'cannot get ANN data from key: %s; %s',
+ ann_key, err)
+ else
+ if type(data) == 'table' then
+ if type(data[1]) == 'userdata' and data[1].cookie == text_cookie then
+ local _err, ann_data = rspamd_util.zstd_decompress(data[1])
+ local ann
+
+ if _err or not ann_data then
+ rspamd_logger.errx(rspamd_config, 'cannot decompress ANN for %s from Redis key %s: %s',
+ rule.prefix .. ':' .. set.name, ann_key, _err)
+ return
+ else
+ ann = rspamd_kann.load(ann_data)
+
+ if ann then
+ set.ann = {
+ digest = profile.digest,
+ version = profile.version,
+ symbols = profile.symbols,
+ distance = min_diff,
+ redis_key = profile.redis_key
+ }
+
+ local ucl = require "ucl"
+ local profile_serialized = ucl.to_format(profile, 'json-compact', true)
+ set.ann.ann = ann -- To avoid serialization
+
+ local function rank_cb(_, _)
+ -- TODO: maybe add some logging
+ end
+ -- Also update rank for the loaded ANN to avoid removal
+ lua_redis.redis_make_request_taskless(ev_base,
+ rspamd_config,
+ rule.redis,
+ nil,
+ true, -- is write
+ rank_cb, --callback
+ 'ZADD', -- command
+ { set.prefix, tostring(rspamd_util.get_time()), profile_serialized }
+ )
+ rspamd_logger.infox(rspamd_config,
+ 'loaded ANN for %s:%s from %s; %s bytes compressed; version=%s',
+ rule.prefix, set.name, ann_key, #data[1], profile.version)
+ else
+ rspamd_logger.errx(rspamd_config,
+ 'cannot unpack/deserialise ANN for %s:%s from Redis key %s',
+ rule.prefix, set.name, ann_key)
+ end
+ end
+ else
+ lua_util.debugm(N, rspamd_config, 'missing ANN for %s:%s in Redis key %s',
+ rule.prefix, set.name, ann_key)
+ end
+
+ if set.ann and set.ann.ann and type(data[2]) == 'userdata' and data[2].cookie == text_cookie then
+ if rule.roc_enabled then
+ local ucl = require "ucl"
+ local parser = ucl.parser()
+ local ok, parse_err = parser:parse_text(data[2])
+ assert(ok, parse_err)
+ local roc_thresholds = parser:get_object()
+ set.ann.roc_thresholds = roc_thresholds
+ rspamd_logger.infox(rspamd_config,
+ 'loaded ROC thresholds for %s:%s; version=%s',
+ rule.prefix, set.name, profile.version)
+ rspamd_logger.debugx("ROC thresholds: %s", roc_thresholds)
+ end
+ end
+
+ if set.ann and set.ann.ann and type(data[3]) == 'userdata' and data[3].cookie == text_cookie then
+ -- PCA table
+ local _err, pca_data = rspamd_util.zstd_decompress(data[3])
+ if pca_data then
+ if rule.max_inputs then
+ -- We can use PCA
+ set.ann.pca = rspamd_tensor.load(pca_data)
+ rspamd_logger.infox(rspamd_config,
+ 'loaded PCA for ANN for %s:%s from %s; %s bytes compressed; version=%s',
+ rule.prefix, set.name, ann_key, #data[3], profile.version)
+ else
+ -- no need in pca, why is it there?
+ rspamd_logger.warnx(rspamd_config,
+ 'extra PCA for ANN for %s:%s from Redis key %s: no max inputs defined',
+ rule.prefix, set.name, ann_key)
+ end
+ else
+ -- pca can be missing merely if we have no max_inputs
+ if rule.max_inputs then
+ rspamd_logger.errx(rspamd_config, 'cannot unpack/deserialise ANN for %s:%s from Redis key %s: no PCA: %s',
+ rule.prefix, set.name, ann_key, _err)
+ set.ann.ann = nil
+ else
+ -- It is okay
+ set.ann.pca = nil
+ end
+ end
+ end
+
+ else
+ lua_util.debugm(N, rspamd_config, 'no ANN key for %s:%s in Redis key %s',
+ rule.prefix, set.name, ann_key)
+ end
+ end
+ end
+ lua_redis.redis_make_request_taskless(ev_base,
+ rspamd_config,
+ rule.redis,
+ nil,
+ false, -- is write
+ data_cb, --callback
+ 'HMGET', -- command
+ { ann_key, 'ann', 'roc_thresholds', 'pca' }, -- arguments
+ { opaque_data = true }
+ )
+end
+
+-- Used to check an element in Redis serialized as JSON
+-- for some specific rule + some specific setting
+-- This function tries to load more fresh or more specific ANNs in lieu of
+-- the existing ones.
+-- Use this function to load ANNs as `callback` parameter for `check_anns` function
+local function process_existing_ann(_, ev_base, rule, set, profiles)
+ local my_symbols = set.symbols
+ local min_diff = math.huge
+ local sel_elt
+
+ for _, elt in fun.iter(profiles) do
+ if elt and elt.symbols then
+ local dist = lua_util.distance_sorted(elt.symbols, my_symbols)
+ -- Check distance
+ if dist < #my_symbols * .3 then
+ if dist < min_diff then
+ min_diff = dist
+ sel_elt = elt
+ end
+ end
+ end
+ end
+
+ if sel_elt then
+ -- We can load element from ANN
+ if set.ann then
+ -- We have an existing ANN, probably the same...
+ if set.ann.digest == sel_elt.digest then
+ -- Same ANN, check version
+ if set.ann.version < sel_elt.version then
+ -- Load new ann
+ rspamd_logger.infox(rspamd_config, 'ann %s is changed, ' ..
+ 'our version = %s, remote version = %s',
+ rule.prefix .. ':' .. set.name,
+ set.ann.version,
+ sel_elt.version)
+ load_new_ann(rule, ev_base, set, sel_elt, min_diff)
+ else
+ lua_util.debugm(N, rspamd_config, 'ann %s is not changed, ' ..
+ 'our version = %s, remote version = %s',
+ rule.prefix .. ':' .. set.name,
+ set.ann.version,
+ sel_elt.version)
+ end
+ else
+ -- We have some different ANN, so we need to compare distance
+ if set.ann.distance > min_diff then
+ -- Load more specific ANN
+ rspamd_logger.infox(rspamd_config, 'more specific ann is available for %s, ' ..
+ 'our distance = %s, remote distance = %s',
+ rule.prefix .. ':' .. set.name,
+ set.ann.distance,
+ min_diff)
+ load_new_ann(rule, ev_base, set, sel_elt, min_diff)
+ else
+ lua_util.debugm(N, rspamd_config, 'ann %s is not changed or less specific, ' ..
+ 'our distance = %s, remote distance = %s',
+ rule.prefix .. ':' .. set.name,
+ set.ann.distance,
+ min_diff)
+ end
+ end
+ else
+ -- We have no ANN, load new one
+ load_new_ann(rule, ev_base, set, sel_elt, min_diff)
+ end
+ end
+end
+
+
+-- This function checks all profiles and selects if we can train our
+-- ANN. By our we mean that it has exactly the same symbols in profile.
+-- Use this function to train ANN as `callback` parameter for `check_anns` function
+local function maybe_train_existing_ann(worker, ev_base, rule, set, profiles)
+ local my_symbols = set.symbols
+ local sel_elt
+ local lens = {
+ spam = 0,
+ ham = 0,
+ }
+
+ for _, elt in fun.iter(profiles) do
+ if elt and elt.symbols then
+ local dist = lua_util.distance_sorted(elt.symbols, my_symbols)
+ -- Check distance
+ if dist == 0 then
+ sel_elt = elt
+ break
+ end
+ end
+ end
+
+ if sel_elt then
+ -- We have our ANN and that's train vectors, check if we can learn
+ local ann_key = sel_elt.redis_key
+
+ lua_util.debugm(N, rspamd_config, "check if ANN %s needs to be trained",
+ ann_key)
+
+ -- Create continuation closure
+ local redis_len_cb_gen = function(cont_cb, what, is_final)
+ return function(err, data)
+ if err then
+ rspamd_logger.errx(rspamd_config,
+ 'cannot get ANN %s trains %s from redis: %s', what, ann_key, err)
+ elseif data and type(data) == 'number' or type(data) == 'string' then
+ local ntrains = tonumber(data) or 0
+ lens[what] = ntrains
+ if is_final then
+ -- Ensure that we have the following:
+ -- one class has reached max_trains
+ -- other class(es) are at least as full as classes_bias
+ -- e.g. if classes_bias = 0.25 and we have 10 max_trains then
+ -- one class must have 10 or more trains whilst another should have
+ -- at least (10 * (1 - 0.25)) = 8 trains
+
+ local max_len = math.max(lua_util.unpack(lua_util.values(lens)))
+ local min_len = math.min(lua_util.unpack(lua_util.values(lens)))
+
+ if rule.train.learn_type == 'balanced' then
+ local len_bias_check_pred = function(_, l)
+ return l >= rule.train.max_trains * (1.0 - rule.train.classes_bias)
+ end
+ if max_len >= rule.train.max_trains and fun.all(len_bias_check_pred, lens) then
+ rspamd_logger.debugm(N, rspamd_config,
+ 'can start ANN %s learn as it has %s learn vectors; %s required, after checking %s vectors',
+ ann_key, lens, rule.train.max_trains, what)
+ cont_cb()
+ else
+ rspamd_logger.debugm(N, rspamd_config,
+ 'cannot learn ANN %s now: there are not enough %s learn vectors (has %s vectors; %s required)',
+ ann_key, what, lens, rule.train.max_trains)
+ end
+ else
+ -- Probabilistic mode, just ensure that at least one vector is okay
+ if min_len > 0 and max_len >= rule.train.max_trains then
+ rspamd_logger.debugm(N, rspamd_config,
+ 'can start ANN %s learn as it has %s learn vectors; %s required, after checking %s vectors',
+ ann_key, lens, rule.train.max_trains, what)
+ cont_cb()
+ else
+ rspamd_logger.debugm(N, rspamd_config,
+ 'cannot learn ANN %s now: there are not enough %s learn vectors (has %s vectors; %s required)',
+ ann_key, what, lens, rule.train.max_trains)
+ end
+ end
+
+ else
+ rspamd_logger.debugm(N, rspamd_config,
+ 'checked %s vectors in ANN %s: %s vectors; %s required, need to check other class vectors',
+ what, ann_key, ntrains, rule.train.max_trains)
+ cont_cb()
+ end
+ end
+ end
+
+ end
+
+ local function initiate_train()
+ rspamd_logger.infox(rspamd_config,
+ 'need to learn ANN %s after %s required learn vectors',
+ ann_key, lens)
+ do_train_ann(worker, ev_base, rule, set, ann_key)
+ end
+
+ -- Spam vector is OK, check ham vector length
+ local function check_ham_len()
+ lua_redis.redis_make_request_taskless(ev_base,
+ rspamd_config,
+ rule.redis,
+ nil,
+ false, -- is write
+ redis_len_cb_gen(initiate_train, 'ham', true), --callback
+ 'SCARD', -- command
+ { ann_key .. '_ham_set' }
+ )
+ end
+
+ lua_redis.redis_make_request_taskless(ev_base,
+ rspamd_config,
+ rule.redis,
+ nil,
+ false, -- is write
+ redis_len_cb_gen(check_ham_len, 'spam', false), --callback
+ 'SCARD', -- command
+ { ann_key .. '_spam_set' }
+ )
+ end
+end
+
+-- Used to deserialise ANN element from a list
+local function load_ann_profile(element)
+ local ucl = require "ucl"
+
+ local parser = ucl.parser()
+ local res, ucl_err = parser:parse_string(element)
+ if not res then
+ rspamd_logger.warnx(rspamd_config, 'cannot parse ANN from redis: %s',
+ ucl_err)
+ return nil
+ else
+ local profile = parser:get_object()
+ local checked, schema_err = redis_profile_schema:transform(profile)
+ if not checked then
+ rspamd_logger.errx(rspamd_config, "cannot parse profile schema: %s", schema_err)
+
+ return nil
+ end
+ return checked
+ end
+end
+
+-- Function to check or load ANNs from Redis
+local function check_anns(worker, cfg, ev_base, rule, process_callback, what)
+ for _, set in pairs(rule.settings) do
+ local function members_cb(err, data)
+ if err then
+ rspamd_logger.errx(cfg, 'cannot get ANNs list from redis: %s',
+ err)
+ set.can_store_vectors = true
+ elseif type(data) == 'table' then
+ lua_util.debugm(N, cfg, '%s: process element %s:%s',
+ what, rule.prefix, set.name)
+ process_callback(worker, ev_base, rule, set, fun.map(load_ann_profile, data))
+ set.can_store_vectors = true
+ end
+ end
+
+ if type(set) == 'table' then
+ -- Extract all profiles for some specific settings id
+ -- Get the last `max_profiles` recently used
+ -- Select the most appropriate to our profile but it should not differ by more
+ -- than 30% of symbols
+ lua_redis.redis_make_request_taskless(ev_base,
+ cfg,
+ rule.redis,
+ nil,
+ false, -- is write
+ members_cb, --callback
+ 'ZREVRANGE', -- command
+ { set.prefix, '0', tostring(settings.max_profiles) } -- arguments
+ )
+ end
+ end -- Cycle over all settings
+
+ return rule.watch_interval
+end
+
+-- Function to clean up old ANNs
+local function cleanup_anns(rule, cfg, ev_base)
+ for _, set in pairs(rule.settings) do
+ local function invalidate_cb(err, data)
+ if err then
+ rspamd_logger.errx(cfg, 'cannot exec invalidate script in redis: %s',
+ err)
+ elseif type(data) == 'table' then
+ for _, expired in ipairs(data) do
+ local profile = load_ann_profile(expired)
+ rspamd_logger.infox(cfg, 'invalidated ANN for %s; redis key: %s; version=%s',
+ rule.prefix .. ':' .. set.name,
+ profile.redis_key,
+ profile.version)
+ end
+ end
+ end
+
+ if type(set) == 'table' then
+ lua_redis.exec_redis_script(neural_common.redis_script_id.maybe_invalidate,
+ { ev_base = ev_base, is_write = true },
+ invalidate_cb,
+ { set.prefix, tostring(settings.max_profiles) })
+ end
+ end
+end
+
+local function ann_push_vector(task)
+ if task:has_flag('skip') then
+ lua_util.debugm(N, task, 'do not push data for skipped task')
+ return
+ end
+ if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then
+ lua_util.debugm(N, task, 'do not push data for manual scan')
+ return
+ end
+
+ local verdict, score = lua_verdict.get_specific_verdict(N, task)
+
+ if verdict == 'passthrough' then
+ lua_util.debugm(N, task, 'ignore task as its verdict is %s(%s)',
+ verdict, score)
+
+ return
+ end
+
+ if score ~= score then
+ lua_util.debugm(N, task, 'ignore task as its score is nan (%s verdict)',
+ verdict)
+
+ return
+ end
+
+ for _, rule in pairs(settings.rules) do
+ local set = neural_common.get_rule_settings(task, rule)
+
+ if set then
+ ann_push_task_result(rule, task, verdict, score, set)
+ else
+ lua_util.debugm(N, task, 'settings not found in rule %s', rule.prefix)
+ end
+
+ end
+end
+
+
+-- Initialization part
+if not (neural_common.module_config and type(neural_common.module_config) == 'table')
+ or not neural_common.redis_params then
+ rspamd_logger.infox(rspamd_config, 'Module is unconfigured')
+ lua_util.disable_module(N, "redis")
+ return
+end
+
+local rules = neural_common.module_config['rules']
+
+if not rules then
+ -- Use legacy configuration
+ rules = {}
+ rules['default'] = neural_common.module_config
+end
+
+local id = rspamd_config:register_symbol({
+ name = 'NEURAL_CHECK',
+ type = 'postfilter,callback',
+ flags = 'nostat',
+ priority = lua_util.symbols_priorities.medium,
+ callback = ann_scores_filter
+})
+
+neural_common.settings.rules = {} -- Reset unless validated further in the cycle
+
+if settings.blacklisted_symbols and settings.blacklisted_symbols[1] then
+ -- Transform to hash for simplicity
+ settings.blacklisted_symbols = lua_util.list_to_hash(settings.blacklisted_symbols)
+end
+
+-- Check all rules
+for k, r in pairs(rules) do
+ local rule_elt = lua_util.override_defaults(neural_common.default_options, r)
+ rule_elt['redis'] = neural_common.redis_params
+ rule_elt['anns'] = {} -- Store ANNs here
+
+ if not rule_elt.prefix then
+ rule_elt.prefix = k
+ end
+ if not rule_elt.name then
+ rule_elt.name = k
+ end
+ if rule_elt.train.max_train and not rule_elt.train.max_trains then
+ rule_elt.train.max_trains = rule_elt.train.max_train
+ end
+
+ if not rule_elt.profile then
+ rule_elt.profile = {}
+ end
+
+ if rule_elt.max_inputs and not has_blas then
+ rspamd_logger.errx('cannot set max inputs to %s as BLAS is not compiled in',
+ rule_elt.name, rule_elt.max_inputs)
+ rule_elt.max_inputs = nil
+ end
+
+ rspamd_logger.infox(rspamd_config, "register ann rule %s", k)
+ settings.rules[k] = rule_elt
+ rspamd_config:set_metric_symbol({
+ name = rule_elt.symbol_spam,
+ score = 0.0,
+ description = 'Neural network SPAM',
+ group = 'neural'
+ })
+ rspamd_config:register_symbol({
+ name = rule_elt.symbol_spam,
+ type = 'virtual',
+ flags = 'nostat',
+ parent = id
+ })
+
+ rspamd_config:set_metric_symbol({
+ name = rule_elt.symbol_ham,
+ score = -0.0,
+ description = 'Neural network HAM',
+ group = 'neural'
+ })
+ rspamd_config:register_symbol({
+ name = rule_elt.symbol_ham,
+ type = 'virtual',
+ flags = 'nostat',
+ parent = id
+ })
+end
+
+rspamd_config:register_symbol({
+ name = 'NEURAL_LEARN',
+ type = 'idempotent,callback',
+ flags = 'nostat,explicit_disable,ignore_passthrough',
+ callback = ann_push_vector
+})
+
+-- We also need to deal with settings
+rspamd_config:add_post_init(neural_common.process_rules_settings)
+
+-- Add training scripts
+for _, rule in pairs(settings.rules) do
+ neural_common.load_scripts(rule.redis)
+ -- This function will check ANNs in Redis when a worker is loaded
+ rspamd_config:add_on_load(function(cfg, ev_base, worker)
+ if worker:is_scanner() then
+ rspamd_config:add_periodic(ev_base, 0.0,
+ function(_, _)
+ return check_anns(worker, cfg, ev_base, rule, process_existing_ann,
+ 'try_load_ann')
+ end)
+ end
+
+ if worker:is_primary_controller() then
+ -- We also want to train neural nets when they have enough data
+ rspamd_config:add_periodic(ev_base, 0.0,
+ function(_, _)
+ -- Clean old ANNs
+ cleanup_anns(rule, cfg, ev_base)
+ return check_anns(worker, cfg, ev_base, rule, maybe_train_existing_ann,
+ 'try_train_ann')
+ end)
+ end
+ end)
+end
diff --git a/src/plugins/lua/once_received.lua b/src/plugins/lua/once_received.lua
new file mode 100644
index 0000000..2a5552a
--- /dev/null
+++ b/src/plugins/lua/once_received.lua
@@ -0,0 +1,230 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- 0 or 1 received: = spam
+
+local symbol = 'ONCE_RECEIVED'
+local symbol_rdns = 'RDNS_NONE'
+local symbol_rdns_dnsfail = 'RDNS_DNSFAIL'
+local symbol_mx = 'DIRECT_TO_MX'
+-- Symbol for strict checks
+local symbol_strict = nil
+local bad_hosts = {}
+local good_hosts = {}
+local whitelist = nil
+
+local rspamd_logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local fun = require "fun"
+local N = 'once_received'
+
+local check_local = false
+local check_authed = false
+
+local function check_quantity_received (task)
+ local recvh = task:get_received_headers()
+
+ local nreceived = fun.reduce(function(acc, _)
+ return acc + 1
+ end, 0, fun.filter(function(h)
+ return not h['flags']['artificial']
+ end, recvh))
+
+ local function recv_dns_cb(_, to_resolve, results, err)
+ if err and (err ~= 'requested record is not found' and err ~= 'no records with this name') then
+ rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, err)
+ task:insert_result(symbol_rdns_dnsfail, 1.0)
+ end
+
+ if not results then
+ if nreceived <= 1 then
+ task:insert_result(symbol, 1)
+ -- Avoid strict symbol inserting as the remaining symbols have already
+ -- quote a significant weight, so a message could be rejected by just
+ -- this property.
+ --task:insert_result(symbol_strict, 1)
+ -- Check for MUAs
+ local ua = task:get_header('User-Agent')
+ local xm = task:get_header('X-Mailer')
+ if (ua or xm) then
+ task:insert_result(symbol_mx, 1, (ua or xm))
+ end
+ end
+ task:insert_result(symbol_rdns, 1)
+ else
+ rspamd_logger.infox(task, 'source hostname has not been passed to Rspamd from MTA, ' ..
+ 'but we could resolve source IP address PTR %s as "%s"',
+ to_resolve, results[1])
+ task:set_hostname(results[1])
+
+ if good_hosts then
+ for _, gh in ipairs(good_hosts) do
+ if string.find(results[1], gh) then
+ return
+ end
+ end
+ end
+
+ if nreceived <= 1 then
+ task:insert_result(symbol, 1)
+ for _, h in ipairs(bad_hosts) do
+ if string.find(results[1], h) then
+
+ task:insert_result(symbol_strict, 1, h)
+ return
+ end
+ end
+ end
+ end
+ end
+
+ local task_ip = task:get_ip()
+
+ if ((not check_authed and task:get_user()) or
+ (not check_local and task_ip and task_ip:is_local())) then
+ rspamd_logger.infox(task, 'Skipping once_received for authenticated user or local network')
+ return
+ end
+ if whitelist and task_ip and whitelist:get_key(task_ip) then
+ rspamd_logger.infox(task, 'whitelisted mail from %s',
+ task_ip:to_string())
+ return
+ end
+
+ local hn = task:get_hostname()
+ -- Here we don't care about received
+ if (not hn) and task_ip and task_ip:is_valid() then
+ task:get_resolver():resolve_ptr({ task = task,
+ name = task_ip:to_string(),
+ callback = recv_dns_cb,
+ forced = true
+ })
+ return
+ end
+
+ if nreceived <= 1 then
+ local ret = true
+ local r = recvh[1]
+
+ if not r then
+ return
+ end
+
+ if r['real_hostname'] then
+ local rhn = string.lower(r['real_hostname'])
+ -- Check for good hostname
+ if rhn and good_hosts then
+ for _, gh in ipairs(good_hosts) do
+ if string.find(rhn, gh) then
+ ret = false
+ break
+ end
+ end
+ end
+ end
+
+ if ret then
+ -- Strict checks
+ if symbol_strict then
+ -- Unresolved host
+ task:insert_result(symbol, 1)
+
+ if not hn then
+ return
+ end
+ for _, h in ipairs(bad_hosts) do
+ if string.find(hn, h) then
+ task:insert_result(symbol_strict, 1, h)
+ return
+ end
+ end
+ else
+ task:insert_result(symbol, 1)
+ end
+ end
+ end
+end
+
+local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N,
+ false, false)
+check_local = auth_and_local_conf[1]
+check_authed = auth_and_local_conf[2]
+
+-- Configuration
+local opts = rspamd_config:get_all_opt(N)
+if opts then
+ if opts['symbol'] then
+ symbol = opts['symbol']
+
+ local id = rspamd_config:register_symbol({
+ name = symbol,
+ callback = check_quantity_received,
+ })
+
+ for n, v in pairs(opts) do
+ if n == 'symbol_strict' then
+ symbol_strict = v
+ elseif n == 'symbol_rdns' then
+ symbol_rdns = v
+ elseif n == 'symbol_rdns_dnsfail' then
+ symbol_rdns_dnsfail = v
+ elseif n == 'bad_host' then
+ if type(v) == 'string' then
+ bad_hosts[1] = v
+ else
+ bad_hosts = v
+ end
+ elseif n == 'good_host' then
+ if type(v) == 'string' then
+ good_hosts[1] = v
+ else
+ good_hosts = v
+ end
+ elseif n == 'whitelist' then
+ local lua_maps = require "lua_maps"
+ whitelist = lua_maps.map_add('once_received', 'whitelist', 'radix',
+ 'once received whitelist')
+ elseif n == 'symbol_mx' then
+ symbol_mx = v
+ end
+ end
+
+ rspamd_config:register_symbol({
+ name = symbol_rdns,
+ type = 'virtual',
+ parent = id
+ })
+ rspamd_config:register_symbol({
+ name = symbol_rdns_dnsfail,
+ type = 'virtual',
+ parent = id
+ })
+ rspamd_config:register_symbol({
+ name = symbol_strict,
+ type = 'virtual',
+ parent = id
+ })
+ rspamd_config:register_symbol({
+ name = symbol_mx,
+ type = 'virtual',
+ parent = id
+ })
+ end
+end
diff --git a/src/plugins/lua/p0f.lua b/src/plugins/lua/p0f.lua
new file mode 100644
index 0000000..97757c2
--- /dev/null
+++ b/src/plugins/lua/p0f.lua
@@ -0,0 +1,124 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2019, Denis Paavilainen <denpa@denpa.pro>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Detect remote OS via passive fingerprinting
+
+local lua_util = require "lua_util"
+local lua_redis = require "lua_redis"
+local rspamd_logger = require "rspamd_logger"
+local p0f = require("lua_scanners").filter('p0f').p0f
+
+local N = 'p0f'
+
+if confighelp then
+ rspamd_config:add_example(nil, N,
+ 'Detect remote OS via passive fingerprinting',
+ [[
+ p0f {
+ # Enable module
+ enabled = true
+
+ # Path to the unix socket that p0f listens on
+ socket = '/var/run/p0f.sock';
+
+ # Connection timeout
+ timeout = 5s;
+
+ # If defined, insert symbol with lookup results
+ symbol = 'P0F';
+
+ # Patterns to match against results returned by p0f
+ # Symbol will be yielded on OS string, link type or distance matches
+ patterns = {
+ WINDOWS = '^Windows.*';
+ #DSL = '^DSL$';
+ #DISTANCE10 = '^distance:10$';
+ }
+
+ # Cache lifetime in seconds (default - 2 hours)
+ expire = 7200;
+
+ # Cache key prefix
+ prefix = 'p0f';
+ }
+ ]])
+ return
+end
+
+local rule
+
+local function check_p0f(task)
+ local ip = task:get_from_ip()
+
+ if not (ip and ip:is_valid()) or ip:is_local() then
+ return
+ end
+
+ p0f.check(task, ip, rule)
+end
+
+local opts = rspamd_config:get_all_opt(N)
+
+rule = p0f.configure(opts)
+
+if rule then
+ rule.redis_params = lua_redis.parse_redis_server(N)
+
+ lua_redis.register_prefix(rule.prefix .. '*', N,
+ 'P0f check cache', {
+ type = 'string',
+ })
+
+ local id = rspamd_config:register_symbol({
+ name = 'P0F_CHECK',
+ type = 'prefilter',
+ callback = check_p0f,
+ priority = lua_util.symbols_priorities.medium,
+ flags = 'empty,nostat',
+ group = N,
+ augmentations = { string.format("timeout=%f", rule.timeout or 0.0) },
+
+ })
+
+ if rule.symbol then
+ rspamd_config:register_symbol({
+ name = rule.symbol,
+ parent = id,
+ type = 'virtual',
+ flags = 'empty',
+ group = N
+ })
+ end
+
+ for sym in pairs(rule.patterns) do
+ rspamd_logger.debugm(N, rspamd_config, 'registering: %1', {
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ group = N
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ group = N
+ })
+ end
+else
+ lua_util.disable_module(N, 'config')
+ rspamd_logger.infox('p0f module not configured');
+end
diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua
new file mode 100644
index 0000000..05e08c0
--- /dev/null
+++ b/src/plugins/lua/phishing.lua
@@ -0,0 +1,667 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+local rspamd_logger = require "rspamd_logger"
+local util = require "rspamd_util"
+local lua_util = require "lua_util"
+local lua_maps = require "lua_maps"
+
+-- Phishing detection interface for selecting phished urls and inserting corresponding symbol
+--
+--
+local N = 'phishing'
+local symbol = 'PHISHED_URL'
+local phishing_feed_exclusion_symbol = 'PHISHED_EXCLUDED'
+local generic_service_symbol = 'PHISHED_GENERIC_SERVICE'
+local openphish_symbol = 'PHISHED_OPENPHISH'
+local phishtank_symbol = 'PHISHED_PHISHTANK'
+local generic_service_name = 'generic service'
+local domains = nil
+local phishing_exceptions_maps = {}
+local anchor_exceptions_maps = {}
+local strict_domains_maps = {}
+local phishing_feed_exclusion_map = nil
+local generic_service_map = nil
+local openphish_map = 'https://www.openphish.com/feed.txt'
+local phishtank_suffix = 'phishtank.rspamd.com'
+-- Not enabled by default as their feed is quite large
+local openphish_premium = false
+-- Published via DNS
+local phishtank_enabled = false
+local phishing_feed_exclusion_hash
+local generic_service_hash
+local openphish_hash
+local phishing_feed_exclusion_data = {}
+local generic_service_data = {}
+local openphish_data = {}
+
+local opts = rspamd_config:get_all_opt(N)
+if not (opts and type(opts) == 'table') then
+ rspamd_logger.infox(rspamd_config, 'Module is unconfigured')
+ return
+end
+
+local function is_host_excluded(exclusion_map, host)
+ if exclusion_map and host then
+ local excluded = exclusion_map[host]
+ if excluded then
+ return true
+ end
+ return false
+ end
+end
+
+local function phishing_cb(task)
+ local function check_phishing_map(table)
+ local phishing_data = {}
+ for k,v in pairs(table) do
+ phishing_data[k] = v
+ end
+ local url = phishing_data.url
+ local host = url:get_host()
+
+ if is_host_excluded(phishing_data.exclusion_map, host) then
+ task:insert_result(phishing_data.excl_symbol, 1.0, host)
+ return
+ end
+
+ if host then
+ local elt = phishing_data.map[host]
+ local found_path = false
+ local found_query = false
+ local data = nil
+
+ if elt then
+ local path = url:get_path()
+ local query = url:get_query()
+
+ if path then
+ for _, d in ipairs(elt) do
+ if d['path'] == path then
+ found_path = true
+ data = d['data']
+
+ if query and d['query'] and query == d['query'] then
+ found_query = true
+ elseif not d['query'] then
+ found_query = true
+ end
+ end
+ end
+ else
+ for _, d in ipairs(elt) do
+ if not d['path'] then
+ found_path = true
+ end
+
+ if query and d['query'] and query == d['query'] then
+ found_query = true
+ elseif not d['query'] then
+ found_query = true
+ end
+ end
+ end
+
+ if found_path then
+ local args
+
+ if type(data) == 'table' then
+ args = {
+ data['tld'],
+ data['sector'],
+ data['brand'],
+ }
+ elseif type(data) == 'string' then
+ args = data
+ else
+ args = host
+ end
+
+ if found_query then
+ -- Query + path match
+ task:insert_result(phishing_data.phish_symbol, 1.0, args)
+ else
+ -- Host + path match
+ if path then
+ task:insert_result(phishing_data.phish_symbol, 0.3, args)
+ end
+ -- No path, no symbol
+ end
+ else
+ if url:is_phished() then
+ -- Only host matches
+ task:insert_result(phishing_data.phish_symbol, 0.1, host)
+ end
+ end
+ end
+ end
+ end
+
+ local function check_phishing_dns(table)
+ local phishing_data = {}
+ for k,v in pairs(table) do
+ phishing_data[k] = v
+ end
+ local url = phishing_data.url
+ local host = url:get_host()
+
+ if is_host_excluded(phishing_data.exclusion_map, host) then
+ task:insert_result(phishing_data.excl_symbol, 1.0, host)
+ return
+ end
+
+ local function compose_dns_query(elts)
+ local cr = require "rspamd_cryptobox_hash"
+ local h = cr.create()
+ for _, elt in ipairs(elts) do
+ h:update(elt)
+ end
+ return string.format("%s.%s", h:base32():sub(1, 32), phishing_data.dns_suffix)
+ end
+
+ local r = task:get_resolver()
+ local path = url:get_path()
+ local query = url:get_query()
+
+ if host and path then
+ local function host_host_path_cb(_, _, results, err)
+ if not err and results then
+ if not query then
+ task:insert_result(phishing_data.phish_symbol, 1.0, results)
+ else
+ task:insert_result(phishing_data.phish_symbol, 0.3, results)
+ end
+ end
+ end
+
+ local to_resolve_hp = compose_dns_query({ host, path })
+ rspamd_logger.debugm(N, task, 'try to resolve {%s, %s} -> %s',
+ host, path, to_resolve_hp)
+ r:resolve_txt({
+ task = task,
+ name = to_resolve_hp,
+ callback = host_host_path_cb })
+
+ if query then
+ local function host_host_path_query_cb(_, _, results, err)
+ if not err and results then
+ task:insert_result(phishing_data.phish_symbol, 1.0, results)
+ end
+ end
+
+ local to_resolve_hpq = compose_dns_query({ host, path, query })
+ rspamd_logger.debugm(N, task, 'try to resolve {%s, %s, %s} -> %s',
+ host, path, query, to_resolve_hpq)
+ r:resolve_txt({
+ task = task,
+ name = to_resolve_hpq,
+ callback = host_host_path_query_cb })
+ end
+
+ end
+ end
+
+ -- Process all urls
+ local dmarc_dom
+ local dsym = task:get_symbol('DMARC_POLICY_ALLOW')
+ if dsym then
+ dsym = dsym[1] -- legacy stuff, need to take the first element
+ if dsym.options then
+ dmarc_dom = dsym.options[1]
+ end
+ end
+
+ local urls = task:get_urls() or {}
+ for _, url_iter in ipairs(urls) do
+ local function do_loop_iter()
+ -- to emulate continue
+ local url = url_iter
+ local phishing_data = {}
+ phishing_data.url = url
+ phishing_data.exclusion_map = phishing_feed_exclusion_data
+ phishing_data.excl_symbol = phishing_feed_exclusion_symbol
+ if generic_service_hash then
+ phishing_data.map = generic_service_data
+ phishing_data.phish_symbol = generic_service_symbol
+ check_phishing_map(phishing_data)
+ end
+
+ if openphish_hash then
+ phishing_data.map = openphish_data
+ phishing_data.phish_symbol = openphish_symbol
+ check_phishing_map(phishing_data)
+ end
+
+ if phishtank_enabled then
+ phishing_data.dns_suffix = phishtank_suffix
+ phishing_data.phish_symbol = phishtank_symbol
+ check_phishing_dns(phishing_data)
+ end
+
+ if url:is_phished() then
+ local purl
+
+ if url:is_redirected() then
+ local rspamd_url = require "rspamd_url"
+ -- Examine the real redirect target instead of the url
+ local redirected_url = url:get_redirected()
+ if not redirected_url then
+ return
+ end
+
+ purl = rspamd_url.create(task:get_mempool(), url:get_visible())
+ url = redirected_url
+ else
+ purl = url:get_phished()
+ end
+
+ if not purl then
+ return
+ end
+
+ local tld = url:get_tld()
+ local ptld = purl:get_tld()
+
+ if not ptld or not tld then
+ return
+ end
+
+ if dmarc_dom and tld == dmarc_dom then
+ lua_util.debugm(N, 'exclude phishing from %s -> %s by dmarc domain', tld,
+ ptld)
+ return
+ end
+
+ -- Now we can safely remove the last dot component if it is the same
+ local b, _ = string.find(tld, '%.[^%.]+$')
+ local b1, _ = string.find(ptld, '%.[^%.]+$')
+
+ local stripped_tld, stripped_ptld = tld, ptld
+ if b1 and b then
+ if string.sub(tld, b) == string.sub(ptld, b1) then
+ stripped_ptld = string.gsub(ptld, '%.[^%.]+$', '')
+ stripped_tld = string.gsub(tld, '%.[^%.]+$', '')
+ end
+
+ if #ptld == 0 or #tld == 0 then
+ return false
+ end
+ end
+
+ local weight = 1.0
+ local spoofed, why = util.is_utf_spoofed(tld, ptld)
+ if spoofed then
+ lua_util.debugm(N, task, "confusable: %1 -> %2: %3", tld, ptld, why)
+ weight = 1.0
+ else
+ local dist = util.levenshtein_distance(stripped_tld, stripped_ptld, 2)
+ dist = 2 * dist / (#stripped_tld + #stripped_ptld)
+
+ if dist > 0.3 and dist <= 1.0 then
+ -- Use distance to penalize the total weight
+ weight = util.tanh(3 * (1 - dist + 0.1))
+ elseif dist > 1 then
+ -- We also check if two labels are in the same ascii/non-ascii representation
+ local a1, a2 = false, false
+
+ if string.match(tld, '^[\001-\127]*$') then
+ a1 = true
+ end
+ if string.match(ptld, '^[\001-\127]*$') then
+ a2 = true
+ end
+
+ if a1 ~= a2 then
+ weight = 1
+ lua_util.debugm(N, task, "confusable: %1 -> %2: different characters",
+ tld, ptld, why)
+ else
+ -- We have totally different strings in tld, so penalize it somehow
+ weight = 0.5
+ end
+ end
+
+ lua_util.debugm(N, task, "distance: %1 -> %2: %3", tld, ptld, dist)
+ end
+
+ local function is_url_in_map(map, furl)
+ for _, dn in ipairs({ furl:get_tld(), furl:get_host() }) do
+ if map:get_key(dn) then
+ return true, dn
+ end
+ end
+
+ return false
+ end
+ local function found_in_map(map, furl, sweight)
+ if not furl then
+ furl = url
+ end
+ if not sweight then
+ sweight = weight
+ end
+ if #map > 0 then
+ for _, rule in ipairs(map) do
+ local found, dn = is_url_in_map(rule.map, furl)
+ if found then
+ task:insert_result(rule.symbol, sweight, string.format("%s->%s:%s", ptld, tld, dn))
+ return true
+ end
+ end
+ end
+ end
+
+ found_in_map(strict_domains_maps, purl, 1.0)
+ if not found_in_map(anchor_exceptions_maps) then
+ if not found_in_map(phishing_exceptions_maps, purl, 1.0) then
+ if domains then
+ if is_url_in_map(domains, purl) then
+ task:insert_result(symbol, weight, ptld .. '->' .. tld)
+ end
+ else
+ task:insert_result(symbol, weight, ptld .. '->' .. tld)
+ end
+ end
+ end
+ end
+ end
+
+ do_loop_iter()
+ end
+end
+
+local function phishing_map(mapname, phishmap, id)
+ if opts[mapname] then
+ local xd
+ if type(opts[mapname]) == 'table' then
+ xd = opts[mapname]
+ else
+ rspamd_logger.errx(rspamd_config, 'invalid exception table')
+ end
+
+ for sym, map_data in pairs(xd) do
+ local rmap = lua_maps.map_add_from_ucl(map_data, 'set',
+ 'Phishing ' .. mapname .. ' map')
+ if rmap then
+ rspamd_config:register_virtual_symbol(sym, 1, id)
+ local rule = { symbol = sym, map = rmap }
+ table.insert(phishmap, rule)
+ else
+ rspamd_logger.infox(rspamd_config, 'cannot add map for symbol: %s', sym)
+ end
+ end
+ end
+end
+
+local function rspamd_str_split_fun(s, sep, func)
+ local lpeg = require "lpeg"
+ sep = lpeg.P(sep)
+ local elem = lpeg.P((1 - sep) ^ 0 / func)
+ local p = lpeg.P(elem * (sep * elem) ^ 0)
+ return p:match(s)
+end
+
+local function insert_url_from_string(pool, tbl, str, data)
+ local rspamd_url = require "rspamd_url"
+
+ local u = rspamd_url.create(pool, str)
+
+ if u then
+ local host = u:get_host()
+ if host then
+ local elt = {
+ data = data,
+ path = u:get_path(),
+ query = u:get_query()
+ }
+
+ if tbl[host] then
+ table.insert(tbl[host], elt)
+ else
+ tbl[host] = { elt }
+ end
+
+ return true
+ end
+ end
+
+ return false
+end
+
+local function phishing_feed_exclusion_plain_cb(string)
+ local nelts = 0
+ local new_data = {}
+ local rspamd_mempool = require "rspamd_mempool"
+ local pool = rspamd_mempool.create()
+
+ local function phishing_feed_exclusion_elt_parser(cap)
+ if insert_url_from_string(pool, new_data, cap, nil) then
+ nelts = nelts + 1
+ end
+ end
+
+ rspamd_str_split_fun(string, '\n', phishing_feed_exclusion_elt_parser)
+
+ phishing_feed_exclusion_data = new_data
+ rspamd_logger.infox(phishing_feed_exclusion_hash, "parsed %s elements from phishing feed exclusions",
+ nelts)
+ pool:destroy()
+end
+
+local function generic_service_plain_cb(string)
+ local nelts = 0
+ local new_data = {}
+ local rspamd_mempool = require "rspamd_mempool"
+ local pool = rspamd_mempool.create()
+
+ local function generic_service_elt_parser(cap)
+ if insert_url_from_string(pool, new_data, cap, nil) then
+ nelts = nelts + 1
+ end
+ end
+
+ rspamd_str_split_fun(string, '\n', generic_service_elt_parser)
+
+ generic_service_data = new_data
+ rspamd_logger.infox(generic_service_hash, "parsed %s elements from %s feed",
+ nelts, generic_service_name)
+ pool:destroy()
+end
+
+local function openphish_json_cb(string)
+ local ucl = require "ucl"
+ local rspamd_mempool = require "rspamd_mempool"
+ local nelts = 0
+ local new_json_map = {}
+ local valid = true
+
+ local pool = rspamd_mempool.create()
+
+ local function openphish_elt_parser(cap)
+ if valid then
+ local parser = ucl.parser()
+ local res, err = parser:parse_string(cap)
+ if not res then
+ valid = false
+ rspamd_logger.warnx(openphish_hash, 'cannot parse openphish map: ' .. err)
+ else
+ local obj = parser:get_object()
+
+ if obj['url'] then
+ if insert_url_from_string(pool, new_json_map, obj['url'], obj) then
+ nelts = nelts + 1
+ end
+ end
+ end
+ end
+ end
+
+ rspamd_str_split_fun(string, '\n', openphish_elt_parser)
+
+ if valid then
+ openphish_data = new_json_map
+ rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed",
+ nelts)
+ end
+
+ pool:destroy()
+end
+
+local function openphish_plain_cb(s)
+ local nelts = 0
+ local new_data = {}
+ local rspamd_mempool = require "rspamd_mempool"
+ local pool = rspamd_mempool.create()
+
+ local function openphish_elt_parser(cap)
+ if insert_url_from_string(pool, new_data, cap, nil) then
+ nelts = nelts + 1
+ end
+ end
+
+ rspamd_str_split_fun(s, '\n', openphish_elt_parser)
+
+ openphish_data = new_data
+ rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed",
+ nelts)
+ pool:destroy()
+end
+
+if opts then
+ local id
+ if opts['symbol'] then
+ symbol = opts['symbol']
+ -- Register symbol's callback
+ id = rspamd_config:register_symbol({
+ name = symbol,
+ callback = phishing_cb
+ })
+
+ -- To exclude from domains for dmarc verified messages
+ rspamd_config:register_dependency(symbol, 'DMARC_CHECK')
+
+ if opts['phishing_feed_exclusion_symbol'] then
+ phishing_feed_exclusion_symbol = opts['phishing_feed_exclusion_symbol']
+ end
+ if opts['phishing_feed_exclusion_map'] then
+ phishing_feed_exclusion_map = opts['phishing_feed_exclusion_map']
+ end
+
+ if opts['phishing_feed_exclusion_enabled'] then
+ phishing_feed_exclusion_hash = rspamd_config:add_map({
+ type = 'callback',
+ url = phishing_feed_exclusion_map,
+ callback = phishing_feed_exclusion_plain_cb,
+ description = 'Phishing feed exclusions'
+ })
+ end
+
+ if opts['generic_service_symbol'] then
+ generic_service_symbol = opts['generic_service_symbol']
+ end
+ if opts['generic_service_map'] then
+ generic_service_map = opts['generic_service_map']
+ end
+ if opts['generic_service_url'] then
+ generic_service_map = opts['generic_service_url']
+ end
+ if opts['generic_service_name'] then
+ generic_service_name = opts['generic_service_name']
+ end
+
+ if opts['generic_service_enabled'] then
+ generic_service_hash = rspamd_config:add_map({
+ type = 'callback',
+ url = generic_service_map,
+ callback = generic_service_plain_cb,
+ description = 'Generic feed'
+ })
+ end
+
+ if opts['openphish_map'] then
+ openphish_map = opts['openphish_map']
+ end
+ if opts['openphish_url'] then
+ openphish_map = opts['openphish_url']
+ end
+
+ if opts['openphish_premium'] then
+ openphish_premium = true
+ end
+
+ if opts['openphish_enabled'] then
+ if not openphish_premium then
+ openphish_hash = rspamd_config:add_map({
+ type = 'callback',
+ url = openphish_map,
+ callback = openphish_plain_cb,
+ description = 'Open phishing feed map (see https://www.openphish.com for details)',
+ opaque_data = true,
+ })
+ else
+ openphish_hash = rspamd_config:add_map({
+ type = 'callback',
+ url = openphish_map,
+ callback = openphish_json_cb,
+ opaque_data = true,
+ description = 'Open phishing premium feed map (see https://www.openphish.com for details)'
+ })
+ end
+ end
+
+ if opts['phishtank_enabled'] then
+ phishtank_enabled = true
+ if opts['phishtank_suffix'] then
+ phishtank_suffix = opts['phishtank_suffix']
+ end
+ end
+
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ parent = id,
+ name = generic_service_symbol,
+ })
+
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ parent = id,
+ name = phishing_feed_exclusion_symbol,
+ })
+
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ parent = id,
+ name = openphish_symbol,
+ })
+
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ parent = id,
+ name = phishtank_symbol,
+ })
+ end
+ if opts['domains'] and type(opts['domains']) == 'string' then
+ domains = lua_maps.map_add_from_ucl(opts['domains'], 'set',
+ 'Phishing domains')
+ end
+ phishing_map('phishing_exceptions', phishing_exceptions_maps, id)
+ phishing_map('exceptions', anchor_exceptions_maps, id)
+ phishing_map('strict_domains', strict_domains_maps, id)
+end
diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua
new file mode 100644
index 0000000..add5741
--- /dev/null
+++ b/src/plugins/lua/ratelimit.lua
@@ -0,0 +1,868 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2016-2017, Andrew Lewis <nerf@judo.za.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local rspamd_lua_utils = require "lua_util"
+local lua_redis = require "lua_redis"
+local fun = require "fun"
+local lua_maps = require "lua_maps"
+local lua_util = require "lua_util"
+local lua_verdict = require "lua_verdict"
+local rspamd_hash = require "rspamd_cryptobox_hash"
+local lua_selectors = require "lua_selectors"
+local ts = require("tableshape").types
+
+-- A plugin that implements ratelimits using redis
+
+local E = {}
+local N = 'ratelimit'
+local redis_params
+-- Senders that are considered as bounce
+local settings = {
+ bounce_senders = { 'postmaster', 'mailer-daemon', '', 'null', 'fetchmail-daemon', 'mdaemon' },
+ -- Do not check ratelimits for these recipients
+ whitelisted_rcpts = { 'postmaster', 'mailer-daemon' },
+ prefix = 'RL',
+ ham_factor_rate = 1.01,
+ spam_factor_rate = 0.99,
+ ham_factor_burst = 1.02,
+ spam_factor_burst = 0.98,
+ max_rate_mult = 5,
+ max_bucket_mult = 10,
+ expire = 60 * 60 * 24 * 2, -- 2 days by default
+ limits = {},
+ allow_local = false,
+ prefilter = true,
+}
+
+local bucket_check_script = "ratelimit_check.lua"
+local bucket_check_id
+
+local bucket_update_script = "ratelimit_update.lua"
+local bucket_update_id
+
+local bucket_cleanup_script = "ratelimit_cleanup_pending.lua"
+local bucket_cleanup_id
+
+-- message_func(task, limit_type, prefix, bucket, limit_key)
+local message_func = function(_, limit_type, _, _, _)
+ return string.format('Ratelimit "%s" exceeded', limit_type)
+end
+
+local function load_scripts(_, _)
+ bucket_check_id = lua_redis.load_redis_script_from_file(bucket_check_script, redis_params)
+ bucket_update_id = lua_redis.load_redis_script_from_file(bucket_update_script, redis_params)
+ bucket_cleanup_id = lua_redis.load_redis_script_from_file(bucket_cleanup_script, redis_params)
+end
+
+local limit_parser
+local function parse_string_limit(lim, no_error)
+ local function parse_time_suffix(s)
+ if s == 's' then
+ return 1
+ elseif s == 'm' then
+ return 60
+ elseif s == 'h' then
+ return 3600
+ elseif s == 'd' then
+ return 86400
+ end
+ end
+ local function parse_num_suffix(s)
+ if s == '' then
+ return 1
+ elseif s == 'k' then
+ return 1000
+ elseif s == 'm' then
+ return 1000000
+ elseif s == 'g' then
+ return 1000000000
+ end
+ end
+ local lpeg = require "lpeg"
+
+ if not limit_parser then
+ local digit = lpeg.R("09")
+ limit_parser = {}
+ limit_parser.integer = (lpeg.S("+-") ^ -1) *
+ (digit ^ 1)
+ limit_parser.fractional = (lpeg.P(".")) *
+ (digit ^ 1)
+ limit_parser.number = (limit_parser.integer *
+ (limit_parser.fractional ^ -1)) +
+ (lpeg.S("+-") * limit_parser.fractional)
+ limit_parser.time = lpeg.Cf(lpeg.Cc(1) *
+ (limit_parser.number / tonumber) *
+ ((lpeg.S("smhd") / parse_time_suffix) ^ -1),
+ function(acc, val)
+ return acc * val
+ end)
+ limit_parser.suffixed_number = lpeg.Cf(lpeg.Cc(1) *
+ (limit_parser.number / tonumber) *
+ ((lpeg.S("kmg") / parse_num_suffix) ^ -1),
+ function(acc, val)
+ return acc * val
+ end)
+ limit_parser.limit = lpeg.Ct(limit_parser.suffixed_number *
+ (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) *
+ limit_parser.time)
+ end
+ local t = lpeg.match(limit_parser.limit, lim)
+
+ if t and t[1] and t[2] and t[2] ~= 0 then
+ return t[2], t[1]
+ end
+
+ if not no_error then
+ rspamd_logger.errx(rspamd_config, 'bad limit: %s', lim)
+ end
+
+ return nil
+end
+
+local function str_to_rate(str)
+ local divider, divisor = parse_string_limit(str, false)
+
+ if not divisor then
+ rspamd_logger.errx(rspamd_config, 'bad rate string: %s', str)
+
+ return nil
+ end
+
+ return divisor / divider
+end
+
+local bucket_schema = ts.shape {
+ burst = ts.number + ts.string / lua_util.dehumanize_number,
+ rate = ts.number + ts.string / str_to_rate,
+ skip_recipients = ts.boolean:is_optional(),
+ symbol = ts.string:is_optional(),
+ message = ts.string:is_optional(),
+ skip_soft_reject = ts.boolean:is_optional(),
+}
+
+local function parse_limit(name, data)
+ if type(data) == 'table' then
+ -- 2 cases here:
+ -- * old limit in format [burst, rate]
+ -- * vector of strings in Andrew's string format (removed from 1.8.2)
+ -- * proper bucket table
+ if #data == 2 and tonumber(data[1]) and tonumber(data[2]) then
+ -- Old style ratelimit
+ rspamd_logger.warnx(rspamd_config, 'old style ratelimit for %s', name)
+ if tonumber(data[1]) > 0 and tonumber(data[2]) > 0 then
+ return {
+ burst = data[1],
+ rate = data[2]
+ }
+ elseif data[1] ~= 0 then
+ rspamd_logger.warnx(rspamd_config, 'invalid numbers for %s', name)
+ else
+ rspamd_logger.infox(rspamd_config, 'disable limit %s, burst is zero', name)
+ end
+
+ return nil
+ else
+ local parsed_bucket, err = bucket_schema:transform(data)
+
+ if not parsed_bucket or err then
+ rspamd_logger.errx(rspamd_config, 'cannot parse bucket for %s: %s; original value: %s',
+ name, err, data)
+ else
+ return parsed_bucket
+ end
+ end
+ elseif type(data) == 'string' then
+ local rep_rate, burst = parse_string_limit(data)
+ rspamd_logger.warnx(rspamd_config, 'old style rate bucket config detected for %s: %s',
+ name, data)
+ if rep_rate and burst then
+ return {
+ burst = burst,
+ rate = burst / rep_rate -- reciprocal
+ }
+ end
+ end
+
+ return nil
+end
+
+--- Check whether this addr is bounce
+local function check_bounce(from)
+ return fun.any(function(b)
+ return b == from
+ end, settings.bounce_senders)
+end
+
+local keywords = {
+ ['ip'] = {
+ ['get_value'] = function(task)
+ local ip = task:get_ip()
+ if ip and ip:is_valid() then
+ return tostring(ip)
+ end
+ return nil
+ end,
+ },
+ ['rip'] = {
+ ['get_value'] = function(task)
+ local ip = task:get_ip()
+ if ip and ip:is_valid() and not ip:is_local() then
+ return tostring(ip)
+ end
+ return nil
+ end,
+ },
+ ['from'] = {
+ ['get_value'] = function(task)
+ local from = task:get_from(0)
+ if ((from or E)[1] or E).addr then
+ return string.lower(from[1]['addr'])
+ end
+ return nil
+ end,
+ },
+ ['bounce'] = {
+ ['get_value'] = function(task)
+ local from = task:get_from(0)
+ if not ((from or E)[1] or E).user then
+ return '_'
+ end
+ if check_bounce(from[1]['user']) then
+ return '_'
+ else
+ return nil
+ end
+ end,
+ },
+ ['asn'] = {
+ ['get_value'] = function(task)
+ local asn = task:get_mempool():get_variable('asn')
+ if not asn then
+ return nil
+ else
+ return asn
+ end
+ end,
+ },
+ ['user'] = {
+ ['get_value'] = function(task)
+ local auser = task:get_user()
+ if not auser then
+ return nil
+ else
+ return auser
+ end
+ end,
+ },
+ ['to'] = {
+ ['get_value'] = function(task)
+ return task:get_principal_recipient()
+ end,
+ },
+ ['digest'] = {
+ ['get_value'] = function(task)
+ return task:get_digest()
+ end,
+ },
+ ['attachments'] = {
+ ['get_value'] = function(task)
+ local parts = task:get_parts() or E
+ local digests = {}
+
+ for _, p in ipairs(parts) do
+ if p:get_filename() then
+ table.insert(digests, p:get_digest())
+ end
+ end
+
+ if #digests > 0 then
+ return table.concat(digests, '')
+ end
+
+ return nil
+ end,
+ },
+ ['files'] = {
+ ['get_value'] = function(task)
+ local parts = task:get_parts() or E
+ local files = {}
+
+ for _, p in ipairs(parts) do
+ local fname = p:get_filename()
+ if fname then
+ table.insert(files, fname)
+ end
+ end
+
+ if #files > 0 then
+ return table.concat(files, ':')
+ end
+
+ return nil
+ end,
+ },
+}
+
+local function gen_rate_key(task, rtype, bucket)
+ local key_t = { tostring(lua_util.round(100000.0 / bucket.burst)) }
+ local key_keywords = lua_util.str_split(rtype, '_')
+ local have_user = false
+
+ for _, v in ipairs(key_keywords) do
+ local ret
+
+ if keywords[v] and type(keywords[v]['get_value']) == 'function' then
+ ret = keywords[v]['get_value'](task)
+ end
+ if not ret then
+ return nil
+ end
+ if v == 'user' then
+ have_user = true
+ end
+ if type(ret) ~= 'string' then
+ ret = tostring(ret)
+ end
+ table.insert(key_t, ret)
+ end
+
+ if have_user and not task:get_user() then
+ return nil
+ end
+
+ return table.concat(key_t, ":")
+end
+
+local function make_prefix(redis_key, name, bucket)
+ local hash_len = 24
+ if hash_len > #redis_key then
+ hash_len = #redis_key
+ end
+ local hash = settings.prefix ..
+ string.sub(rspamd_hash.create(redis_key):base32(), 1, hash_len)
+ -- Fill defaults
+ if not bucket.spam_factor_rate then
+ bucket.spam_factor_rate = settings.spam_factor_rate
+ end
+ if not bucket.ham_factor_rate then
+ bucket.ham_factor_rate = settings.ham_factor_rate
+ end
+ if not bucket.spam_factor_burst then
+ bucket.spam_factor_burst = settings.spam_factor_burst
+ end
+ if not bucket.ham_factor_burst then
+ bucket.ham_factor_burst = settings.ham_factor_burst
+ end
+
+ return {
+ bucket = bucket,
+ name = name,
+ hash = hash
+ }
+end
+
+local function limit_to_prefixes(task, k, v, prefixes)
+ local n = 0
+ for _, bucket in ipairs(v.buckets) do
+ if v.selector then
+ local selectors = lua_selectors.process_selectors(task, v.selector)
+ if selectors then
+ local combined = lua_selectors.combine_selectors(task, selectors, ':')
+ if type(combined) == 'string' then
+ prefixes[combined] = make_prefix(combined, k, bucket)
+ n = n + 1
+ else
+ fun.each(function(p)
+ prefixes[p] = make_prefix(p, k, bucket)
+ n = n + 1
+ end, combined)
+ end
+ end
+ else
+ local prefix = gen_rate_key(task, k, bucket)
+ if prefix then
+ if type(prefix) == 'string' then
+ prefixes[prefix] = make_prefix(prefix, k, bucket)
+ n = n + 1
+ else
+ fun.each(function(p)
+ prefixes[p] = make_prefix(p, k, bucket)
+ n = n + 1
+ end, prefix)
+ end
+ end
+ end
+ end
+
+ return n
+end
+
+local function ratelimit_cb(task)
+ if not settings.allow_local and
+ rspamd_lua_utils.is_rspamc_or_controller(task) then
+ lua_util.debugm(N, task, 'skip ratelimit for local request')
+ return
+ end
+
+ -- Get initial task data
+ local ip = task:get_from_ip()
+ if ip and ip:is_valid() and settings.whitelisted_ip then
+ if settings.whitelisted_ip:get_key(ip) then
+ -- Do not check whitelisted ip
+ rspamd_logger.infox(task, 'skip ratelimit for whitelisted IP')
+ return
+ end
+ end
+ -- Parse all rcpts
+ local rcpts = task:get_recipients()
+ local rcpts_user = {}
+ if rcpts then
+ fun.each(function(r)
+ fun.each(function(type)
+ table.insert(rcpts_user, r[type])
+ end, { 'user', 'addr' })
+ end, rcpts)
+
+ if fun.any(function(r)
+ return settings.whitelisted_rcpts:get_key(r)
+ end, rcpts_user) then
+ rspamd_logger.infox(task, 'skip ratelimit for whitelisted recipient')
+ return
+ end
+ end
+ -- Get user (authuser)
+ if settings.whitelisted_user then
+ local auser = task:get_user()
+ if settings.whitelisted_user:get_key(auser) then
+ rspamd_logger.infox(task, 'skip ratelimit for whitelisted user')
+ return
+ end
+ end
+ -- Now create all ratelimit prefixes
+ local prefixes = {}
+ local nprefixes = 0
+
+ for k, v in pairs(settings.limits) do
+ nprefixes = nprefixes + limit_to_prefixes(task, k, v, prefixes)
+ end
+
+ for k, hdl in pairs(settings.custom_keywords or E) do
+ local ret, redis_key, bd = pcall(hdl, task)
+
+ if ret then
+ local bucket = parse_limit(k, bd)
+ if bucket then
+ prefixes[redis_key] = make_prefix(redis_key, k, bucket)
+ end
+ nprefixes = nprefixes + 1
+ else
+ rspamd_logger.errx(task, 'cannot call handler for %s: %s',
+ k, redis_key)
+ end
+ end
+
+ local function gen_check_cb(prefix, bucket, lim_name, lim_key)
+ return function(err, data)
+ if err then
+ rspamd_logger.errx('cannot check limit %s: %s %s', prefix, err, data)
+ elseif type(data) == 'table' and data[1] then
+ lua_util.debugm(N, task,
+ "got reply for limit %s (%s / %s); %s burst, %s:%s dyn, %s leaked",
+ prefix, bucket.burst, bucket.rate,
+ data[2], data[3], data[4], data[5])
+
+ task:cache_set('ratelimit_bucket_touched', true)
+ if data[1] == 1 then
+ -- set symbol only and do NOT soft reject
+ if bucket.symbol then
+ -- Per bucket symbol
+ task:insert_result(bucket.symbol, 1.0,
+ string.format('%s(%s)', lim_name, lim_key))
+ else
+ if settings.symbol then
+ task:insert_result(settings.symbol, 1.0,
+ string.format('%s(%s)', lim_name, lim_key))
+ elseif settings.info_symbol then
+ task:insert_result(settings.info_symbol, 1.0,
+ string.format('%s(%s)', lim_name, lim_key))
+ end
+ end
+ rspamd_logger.infox(task,
+ 'ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn); redis key: %s',
+ lim_name, prefix,
+ bucket.burst, bucket.rate,
+ data[2], data[3], data[4], lim_key)
+
+ if not (bucket.symbol or settings.symbol) and not bucket.skip_soft_reject then
+ if not bucket.message then
+ task:set_pre_result('soft reject',
+ message_func(task, lim_name, prefix, bucket, lim_key), N)
+ else
+ task:set_pre_result('soft reject', bucket.message)
+ end
+ end
+ end
+ end
+ end
+ end
+
+ -- Don't do anything if pre-result has been already set
+ if task:has_pre_result() then
+ return
+ end
+
+ local _, nrcpt = task:has_recipients('smtp')
+ if not nrcpt or nrcpt <= 0 then
+ nrcpt = 1
+ end
+
+ if nprefixes > 0 then
+ -- Save prefixes to the cache to allow update
+ task:cache_set('ratelimit_prefixes', prefixes)
+ local now = rspamd_util.get_time()
+ now = lua_util.round(now * 1000.0) -- Get milliseconds
+ -- Now call check script for all defined prefixes
+
+ for pr, value in pairs(prefixes) do
+ local bucket = value.bucket
+ local rate = (bucket.rate) / 1000.0 -- Leak rate in messages/ms
+ local bincr = nrcpt
+ if bucket.skip_recipients then
+ bincr = 1
+ end
+
+ lua_util.debugm(N, task, "check limit %s:%s -> %s (%s/%s)",
+ value.name, pr, value.hash, bucket.burst, bucket.rate)
+ lua_redis.exec_redis_script(bucket_check_id,
+ { key = value.hash, task = task, is_write = true },
+ gen_check_cb(pr, bucket, value.name, value.hash),
+ { value.hash, tostring(now), tostring(rate), tostring(bucket.burst),
+ tostring(settings.expire), tostring(bincr) })
+ end
+ end
+end
+
+
+-- This function is used to clean up pending bucket when
+-- the task is somehow being skipped (e.g. greylisting/ratelimit/whatever)
+-- but the ratelimit buckets for this task are touched (e.g. pending has been increased)
+-- See https://github.com/rspamd/rspamd/issues/4467 for more context
+local function maybe_cleanup_pending(task)
+ if task:cache_get('ratelimit_bucket_touched') then
+ local prefixes = task:cache_get('ratelimit_prefixes')
+ if prefixes then
+ for k, v in pairs(prefixes) do
+ local bucket = v.bucket
+ local function cleanup_cb(err, data)
+ if err then
+ rspamd_logger.errx('cannot cleanup limit %s: %s %s', k, err, data)
+ else
+ lua_util.debugm(N, task, 'cleaned pending bucked for %s: %s', k, data)
+ end
+ end
+ local _, nrcpt = task:has_recipients('smtp')
+ if not nrcpt or nrcpt <= 0 then
+ nrcpt = 1
+ end
+ local bincr = nrcpt
+ if bucket.skip_recipients then
+ bincr = 1
+ end
+ local now = task:get_timeval(true)
+ now = lua_util.round(now * 1000.0) -- Get milliseconds
+ lua_redis.exec_redis_script(bucket_cleanup_id,
+ { key = v.hash, task = task, is_write = true },
+ cleanup_cb,
+ { v.hash, tostring(now), tostring(settings.expire), tostring(bincr) })
+ end
+ end
+ end
+end
+
+local function ratelimit_update_cb(task)
+ if task:has_flag('skip') then
+ maybe_cleanup_pending(task)
+ return
+ end
+ if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then
+ maybe_cleanup_pending(task)
+ end
+
+ local prefixes = task:cache_get('ratelimit_prefixes')
+
+ if prefixes then
+ if task:has_pre_result() then
+ -- Already rate limited/greylisted, do nothing
+ lua_util.debugm(N, task, 'pre-action has been set, do not update')
+ maybe_cleanup_pending(task)
+ return
+ end
+
+ local verdict = lua_verdict.get_specific_verdict(N, task)
+ local _, nrcpt = task:has_recipients('smtp')
+ if not nrcpt or nrcpt <= 0 then
+ nrcpt = 1
+ end
+
+ -- Update each bucket
+ for k, v in pairs(prefixes) do
+ local bucket = v.bucket
+ local function update_bucket_cb(err, data)
+ if err then
+ rspamd_logger.errx(task, 'cannot update rate bucket %s: %s',
+ k, err)
+ else
+ lua_util.debugm(N, task,
+ "updated limit %s:%s -> %s (%s/%s), burst: %s, dyn_rate: %s, dyn_burst: %s",
+ v.name, k, v.hash,
+ bucket.burst, bucket.rate,
+ data[1], data[2], data[3])
+ end
+ end
+ local now = task:get_timeval(true)
+ now = lua_util.round(now * 1000.0) -- Get milliseconds
+ local mult_burst = 1.0
+ local mult_rate = 1.0
+
+ if verdict == 'spam' or verdict == 'junk' then
+ mult_burst = bucket.spam_factor_burst or 1.0
+ mult_rate = bucket.spam_factor_rate or 1.0
+ elseif verdict == 'ham' then
+ mult_burst = bucket.ham_factor_burst or 1.0
+ mult_rate = bucket.ham_factor_rate or 1.0
+ end
+
+ local bincr = nrcpt
+ if bucket.skip_recipients then
+ bincr = 1
+ end
+
+ lua_redis.exec_redis_script(bucket_update_id,
+ { key = v.hash, task = task, is_write = true },
+ update_bucket_cb,
+ { v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst),
+ tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult),
+ tostring(settings.expire), tostring(bincr) })
+ end
+ end
+end
+
+local opts = rspamd_config:get_all_opt(N)
+if opts then
+
+ settings = lua_util.override_defaults(settings, opts)
+
+ if opts['limit'] then
+ rspamd_logger.errx(rspamd_config, 'Legacy ratelimit config format no longer supported')
+ end
+
+ if opts['rates'] and type(opts['rates']) == 'table' then
+ -- new way of setting limits
+ fun.each(function(t, lim)
+ local buckets = {}
+
+ if type(lim) == 'table' and lim.bucket then
+
+ if lim.bucket[1] then
+ for _, bucket in ipairs(lim.bucket) do
+ local b = parse_limit(t, bucket)
+
+ if not b then
+ rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"',
+ t, b)
+ return
+ end
+
+ table.insert(buckets, b)
+ end
+ else
+ local bucket = parse_limit(t, lim.bucket)
+
+ if not bucket then
+ rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"',
+ t, lim.bucket)
+ return
+ end
+
+ buckets = { bucket }
+ end
+
+ settings.limits[t] = {
+ buckets = buckets
+ }
+
+ if lim.selector then
+ local selector = lua_selectors.parse_selector(rspamd_config, lim.selector)
+ if not selector then
+ rspamd_logger.errx(rspamd_config, 'bad ratelimit selector for %s: "%s"',
+ t, lim.selector)
+ settings.limits[t] = nil
+ return
+ end
+
+ settings.limits[t].selector = selector
+ end
+ else
+ rspamd_logger.warnx(rspamd_config, 'old syntax for ratelimits: %s', lim)
+ buckets = parse_limit(t, lim)
+ if buckets then
+ settings.limits[t] = {
+ buckets = { buckets }
+ }
+ end
+ end
+ end, opts['rates'])
+ end
+
+ -- Display what's enabled
+ fun.each(function(s)
+ rspamd_logger.infox(rspamd_config, 'enabled ratelimit: %s', s)
+ end, fun.map(function(n, d)
+ return string.format('%s [%s]', n,
+ table.concat(fun.totable(fun.map(function(v)
+ return string.format('symbol: %s, %s msgs burst, %s msgs/sec rate',
+ v.symbol, v.burst, v.rate)
+ end, d.buckets)), '; ')
+ )
+ end, settings.limits))
+
+ -- Ret, ret, ret: stupid legacy stuff:
+ -- If we have a string with commas then load it as as static map
+ -- otherwise, apply normal logic of Rspamd maps
+
+ local wrcpts = opts['whitelisted_rcpts']
+ if type(wrcpts) == 'string' then
+ if string.find(wrcpts, ',') then
+ settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(
+ lua_util.rspamd_str_split(wrcpts, ','), 'set', 'Ratelimit whitelisted rcpts')
+ else
+ settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set',
+ 'Ratelimit whitelisted rcpts')
+ end
+ elseif type(opts['whitelisted_rcpts']) == 'table' then
+ settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set',
+ 'Ratelimit whitelisted rcpts')
+ else
+ -- Stupid default...
+ settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(
+ settings.whitelisted_rcpts, 'set', 'Ratelimit whitelisted rcpts')
+ end
+
+ if opts['whitelisted_ip'] then
+ settings.whitelisted_ip = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_ip', 'radix',
+ 'Ratelimit whitelist ip map')
+ end
+
+ if opts['whitelisted_user'] then
+ settings.whitelisted_user = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_user', 'set',
+ 'Ratelimit whitelist user map')
+ end
+
+ settings.custom_keywords = {}
+ if opts['custom_keywords'] then
+ local ret, res_or_err = pcall(loadfile(opts['custom_keywords']))
+
+ if ret then
+ opts['custom_keywords'] = {}
+ if type(res_or_err) == 'table' then
+ for k, hdl in pairs(res_or_err) do
+ settings['custom_keywords'][k] = hdl
+ end
+ elseif type(res_or_err) == 'function' then
+ settings['custom_keywords']['custom'] = res_or_err
+ end
+ else
+ rspamd_logger.errx(rspamd_config, 'cannot execute %s: %s',
+ opts['custom_keywords'], res_or_err)
+ settings['custom_keywords'] = {}
+ end
+ end
+
+ if opts['message_func'] then
+ message_func = assert(load(opts['message_func']))()
+ end
+
+ redis_params = lua_redis.parse_redis_server('ratelimit')
+
+ if not redis_params then
+ rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
+ lua_util.disable_module(N, "redis")
+ else
+ local s = {
+ type = settings.prefilter and 'prefilter' or 'callback',
+ name = 'RATELIMIT_CHECK',
+ priority = lua_util.symbols_priorities.medium,
+ callback = ratelimit_cb,
+ flags = 'empty,nostat',
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
+ }
+
+ local id = rspamd_config:register_symbol(s)
+
+ -- Register per bucket symbols
+ -- Display what's enabled
+ fun.each(function(set, lim)
+ if type(lim.buckets) == 'table' then
+ for _, b in ipairs(lim.buckets) do
+ if b.symbol then
+ rspamd_config:register_symbol {
+ type = 'virtual',
+ name = b.symbol,
+ score = 0.0,
+ parent = id
+ }
+ end
+ end
+ end
+ end, settings.limits)
+
+ if settings.info_symbol then
+ rspamd_config:register_symbol {
+ type = 'virtual',
+ name = settings.info_symbol,
+ score = 0.0,
+ parent = id
+ }
+ end
+ if settings.symbol then
+ rspamd_config:register_symbol {
+ type = 'virtual',
+ name = settings.symbol,
+ score = 0.0, -- Might be overridden if needed
+ parent = id
+ }
+ end
+
+ rspamd_config:register_symbol {
+ type = 'idempotent',
+ name = 'RATELIMIT_UPDATE',
+ flags = 'explicit_disable,ignore_passthrough',
+ callback = ratelimit_update_cb,
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
+ }
+ end
+end
+
+rspamd_config:add_on_load(function(cfg, ev_base, _)
+ load_scripts(cfg, ev_base)
+end)
diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua
new file mode 100644
index 0000000..b2ccf86
--- /dev/null
+++ b/src/plugins/lua/rbl.lua
@@ -0,0 +1,1425 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2013-2015, Andrew Lewis <nerf@judo.za.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+local hash = require 'rspamd_cryptobox_hash'
+local rspamd_logger = require 'rspamd_logger'
+local rspamd_util = require 'rspamd_util'
+local rspamd_ip = require "rspamd_ip"
+local fun = require 'fun'
+local lua_util = require 'lua_util'
+local selectors = require "lua_selectors"
+local bit = require 'bit'
+local lua_maps = require "lua_maps"
+local rbl_common = require "plugins/rbl"
+local rspamd_url = require "rspamd_url"
+
+-- This plugin implements various types of RBL checks
+-- Documentation can be found here:
+-- https://rspamd.com/doc/modules/rbl.html
+
+local E = {}
+local N = 'rbl'
+
+-- Checks that could be performed by rbl module
+local local_exclusions
+local white_symbols = {}
+local black_symbols = {}
+local monitored_addresses = {}
+local known_selectors = {} -- map from selector string to selector id
+local url_flag_bits = rspamd_url.flags
+
+local function get_monitored(rbl)
+ local function is_random_monitored()
+ -- Explicit definition
+ if type(rbl.random_monitored) == 'boolean' then
+ return rbl.random_monitored
+ end
+
+ -- We check 127.0.0.1 for merely RBLs with `from` or `received` and only if
+ -- they don't have `no_ip` attribute at the same time
+ --
+ -- Convert to a boolean variable using the common idiom
+ return (not (rbl.from or rbl.received)
+ or rbl.no_ip)
+ and true or false
+ end
+
+ local default_monitored = '1.0.0.127'
+ local ret = {
+ rcode = 'nxdomain',
+ prefix = default_monitored,
+ random = is_random_monitored(),
+ }
+
+ if rbl.monitored_address then
+ ret.prefix = rbl.monitored_address
+ end
+
+ lua_util.debugm(N, rspamd_config,
+ 'added monitored address: %s (%s random)',
+ ret.prefix, ret.random)
+
+ return ret
+end
+
+local function validate_dns(lstr)
+ if lstr:match('%.%.') then
+ -- two dots in a row
+ return false, "two dots in a row"
+ end
+ if not rspamd_util.is_valid_utf8(lstr) then
+ -- invalid utf8 detected
+ return false, "invalid utf8"
+ end
+ for v in lstr:gmatch('[^%.]+') do
+ if v:len() > 63 then
+ -- too long label
+ return false, "too long label"
+ end
+ if v:match('^-') or v:match('-$') then
+ -- dash at the beginning or end of label
+ return false, "dash at the beginning or end of label"
+ end
+ end
+ return true
+end
+
+local function maybe_make_hash(data, rule)
+ if rule.hash then
+ local h = hash.create_specific(rule.hash, data)
+ local s
+ if rule.hash_format then
+ if rule.hash_format == 'base32' then
+ s = h:base32()
+ elseif rule.hash_format == 'base64' then
+ s = h:base64()
+ else
+ s = h:hex()
+ end
+ else
+ s = h:hex()
+ end
+
+ if rule.hash_len then
+ s = s:sub(1, rule.hash_len)
+ end
+
+ return s
+ else
+ return data
+ end
+end
+
+local function is_excluded_ip(rip)
+ if local_exclusions and local_exclusions:get_key(rip) then
+ return true
+ end
+ return false
+end
+
+local function ip_to_rbl(ip)
+ return table.concat(ip:inversed_str_octets(), '.')
+end
+
+local function gen_check_rcvd_conditions(rbl, received_total)
+ local min_pos = tonumber(rbl.received_min_pos)
+ local max_pos = tonumber(rbl.received_max_pos)
+ local match_flags = rbl.received_flags
+ local nmatch_flags = rbl.received_nflags
+
+ local function basic_received_check(rh)
+ if not (rh.real_ip and rh.real_ip:is_valid()) then
+ return false
+ end
+ if ((rh.real_ip:get_version() == 6 and rbl.ipv6) or
+ (rh.real_ip:get_version() == 4 and rbl.ipv4)) and
+ ((rbl.exclude_local and not rh.real_ip:is_local() or is_excluded_ip(rh.real_ip)) or not rbl.exclude_local) then
+ return true
+ else
+ return false
+ end
+ end
+
+ local function positioned_received_check(rh, pos)
+ if not rh or not basic_received_check(rh) then
+ return false
+ end
+ local got_flags = rh.flags or E
+ if min_pos then
+ if min_pos < 0 then
+ if min_pos == -1 then
+ if (pos ~= received_total) then
+ return false
+ end
+ else
+ if pos <= (received_total - math.abs(min_pos)) then
+ return false
+ end
+ end
+ elseif pos < min_pos then
+ return false
+ end
+ end
+ if max_pos then
+ if max_pos < -1 then
+ if (received_total - math.abs(max_pos)) >= pos then
+ return false
+ end
+ elseif max_pos > 0 then
+ if pos > max_pos then
+ return false
+ end
+ end
+ end
+ if match_flags then
+ for _, flag in ipairs(match_flags) do
+ if not got_flags[flag] then
+ return false
+ end
+ end
+ end
+ if nmatch_flags then
+ for _, flag in ipairs(nmatch_flags) do
+ if got_flags[flag] then
+ return false
+ end
+ end
+ end
+ return true
+ end
+
+ if not (max_pos or min_pos or match_flags or nmatch_flags) then
+ return basic_received_check
+ else
+ return positioned_received_check
+ end
+end
+
+local matchers = {}
+
+matchers.radix = function(_, _, real_ip, map)
+ return map and map:get_key(real_ip) or false
+end
+
+matchers.equality = function(codes, to_match)
+ if type(codes) ~= 'table' then return codes == to_match end
+ for _, ip in ipairs(codes) do
+ if to_match == ip then
+ return true
+ end
+ end
+ return false
+end
+
+matchers.luapattern = function(codes, to_match)
+ if type(codes) ~= 'table' then
+ return string.find(to_match, '^' .. codes .. '$') and true or false
+ end
+ for _, pattern in ipairs(codes) do
+ if string.find(to_match, '^' .. pattern .. '$') then
+ return true
+ end
+ end
+ return false
+end
+
+matchers.regexp = function(_, to_match, _, map)
+ return map and map:get_key(to_match) or false
+end
+
+matchers.glob = function(_, to_match, _, map)
+ return map and map:get_key(to_match) or false
+end
+
+local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_table_elt, match)
+ local function make_option(ip, label)
+ if ip then
+ return string.format('%s:%s:%s',
+ resolve_table_elt.orig,
+ label,
+ ip)
+ else
+ return string.format('%s:%s',
+ resolve_table_elt.orig,
+ label)
+ end
+ end
+
+ local function insert_result(s, ip, label)
+ if rbl.symbols_prefixes then
+ local prefix = rbl.symbols_prefixes[label]
+
+ if not prefix then
+ rspamd_logger.warnx(task, 'unlisted symbol prefix for %s', label)
+ task:insert_result(s, 1.0, make_option(ip, label))
+ else
+ task:insert_result(prefix .. '_' .. s, 1.0, make_option(ip, label))
+ end
+ else
+ task:insert_result(s, 1.0, make_option(ip, label))
+ end
+ end
+
+ local function insert_results(s, ip)
+ for label in pairs(resolve_table_elt.what) do
+ insert_result(s, ip, label)
+ end
+ end
+
+ if err and (err ~= 'requested record is not found' and
+ err ~= 'no records with this name') then
+ rspamd_logger.infox(task, 'error looking up %s: %s', to_resolve, err)
+ task:insert_result(rbl.symbol .. '_FAIL', 1, string.format('%s:%s',
+ resolve_table_elt.orig, err))
+ return
+ end
+
+ if not results then
+ lua_util.debugm(N, task,
+ 'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4',
+ to_resolve, false, err, rbl.symbol)
+ return
+ else
+ lua_util.debugm(N, task,
+ 'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4',
+ to_resolve, true, err, rbl.symbol)
+ end
+
+ if rbl.returncodes == nil and rbl.returnbits == nil and rbl.symbol ~= nil then
+ insert_results(rbl.symbol)
+ return
+ end
+
+ local returncodes_maps = rbl.returncodes_maps or {}
+
+ for _, result in ipairs(results) do
+ local ipstr = result:to_string()
+ lua_util.debugm(N, task, '%s DNS result %s', to_resolve, ipstr)
+ local foundrc = false
+ -- Check return codes
+ if rbl.returnbits then
+ local ipnum = result:to_number()
+ for s, bits in pairs(rbl.returnbits) do
+ for _, check_bit in ipairs(bits) do
+ if bit.band(ipnum, check_bit) == check_bit then
+ foundrc = true
+ insert_results(s)
+ -- Here, we continue with other bits
+ end
+ end
+ end
+ elseif rbl.returncodes then
+ for s, codes in pairs(rbl.returncodes) do
+ local res = match(codes, ipstr, result, returncodes_maps[s])
+ if res then
+ foundrc = true
+ insert_results(s)
+ end
+ end
+ end
+
+ if not foundrc then
+ if rbl.unknown and rbl.symbol then
+ insert_results(rbl.symbol, ipstr)
+ else
+ lua_util.debugm(N, task, '%1 returned unknown result: %2',
+ to_resolve, ipstr)
+ end
+ end
+ end
+
+end
+
+local function gen_rbl_callback(rule)
+ local function is_whitelisted(task, req, req_str, whitelist, what)
+ if rule.ignore_whitelist then
+ lua_util.debugm(N, task,
+ 'ignore whitelisting checks to %s by %s: ignore whitelist is being set',
+ req_str, rule.symbol)
+ return false
+ end
+
+ if rule.whitelist then
+ if rule.whitelist:get_key(req) then
+ lua_util.debugm(N, task,
+ 'whitelisted %s on %s',
+ req_str, rule.symbol)
+
+ return true
+ end
+ end
+
+ -- Maybe whitelisted by some other rbl rule
+ if whitelist then
+ local wl = whitelist[req_str]
+ if wl then
+ lua_util.debugm(N, task,
+ 'whitelisted request to %s by %s (%s) rbl rule (%s checked type, %s whitelist type)',
+ req_str, wl.type, wl.symbol, what, wl.type)
+ if wl.type == what then
+ -- This was decided to be a bad idea as in case of whitelisting a request to blacklist
+ -- is not even sent
+ --task:adjust_result(wl.symbol, 0.0 / 0.0, rule.symbol)
+
+ return true
+ end
+ end
+ end
+
+ return false
+ end
+
+ local function add_dns_request(task, req, forced, is_ip, requests_table, label, whitelist)
+ local req_str = req
+ if is_ip then
+ req_str = tostring(req)
+ end
+
+ if whitelist and is_whitelisted(task, req, req_str, whitelist, label) then
+ return
+ end
+
+ if is_ip then
+ req = ip_to_rbl(req)
+ end
+
+ if requests_table[req] then
+ -- Duplicate request
+ local nreq = requests_table[req]
+ if forced and not nreq.forced then
+ nreq.forced = true
+ end
+ if not nreq.what[label] then
+ nreq.what[label] = true
+ end
+
+ return true, nreq -- Duplicate
+ else
+ local nreq
+
+ local resolve_ip = rule.resolve_ip and not is_ip
+ if rule.process_script then
+ local processed = rule.process_script(req, rule.rbl, task, resolve_ip)
+
+ if processed then
+ nreq = {
+ forced = forced,
+ n = processed,
+ orig = req_str,
+ resolve_ip = resolve_ip,
+ what = { [label] = true },
+ }
+ requests_table[req] = nreq
+ end
+ else
+ local to_resolve
+ local origin = req
+
+ if not resolve_ip then
+ origin = maybe_make_hash(req, rule)
+ to_resolve = string.format('%s.%s',
+ origin,
+ rule.rbl)
+ else
+ -- First, resolve origin stuff without hashing or anything
+ to_resolve = origin
+ end
+
+ nreq = {
+ forced = forced,
+ n = to_resolve,
+ orig = req_str,
+ resolve_ip = resolve_ip,
+ what = { [label] = true },
+ }
+ requests_table[req] = nreq
+ end
+ return false, nreq
+ end
+ end
+
+ -- Here, we have functional approach: we form a pipeline of functions
+ -- f1, f2, ... fn. Each function accepts task and return boolean value
+ -- that allows to process pipeline further
+ -- Each function in the pipeline can add something to `dns_req` vector as a side effect
+ local function is_alive(_, _)
+ if rule.monitored then
+ if not rule.monitored:alive() then
+ return false
+ end
+ end
+
+ return true
+ end
+
+ local function check_required_symbols(task, _)
+ if rule.require_symbols then
+ return fun.all(function(sym)
+ task:has_symbol(sym)
+ end, rule.require_symbols)
+ end
+
+ return true
+ end
+
+ local function check_user(task, _)
+ if task:get_user() then
+ return false
+ end
+
+ return true
+ end
+
+ local function check_local(task, _)
+ local ip = task:get_from_ip()
+
+ if ip and not ip:is_valid() then
+ ip = nil
+ end
+
+ if ip and ip:is_local() or is_excluded_ip(ip) then
+ return false
+ end
+
+ return true
+ end
+
+ local function check_helo(task, requests_table, whitelist)
+ local helo = task:get_helo()
+
+ if not helo then
+ -- Avoid pipeline breaking
+ return true
+ end
+
+ add_dns_request(task, helo, true, false, requests_table,
+ 'helo', whitelist)
+
+ return true
+ end
+
+ local function check_dkim(task, requests_table, whitelist)
+ local das = task:get_symbol('DKIM_TRACE')
+ local mime_from_domain
+
+ if das and das[1] and das[1].options then
+
+ if rule.dkim_match_from then
+ -- We check merely mime from
+ mime_from_domain = ((task:get_from('mime') or E)[1] or E).domain
+ if mime_from_domain then
+ local mime_from_domain_tld = rule.url_full_hostname and
+ mime_from_domain or rspamd_util.get_tld(mime_from_domain)
+
+ if rule.url_compose_map then
+ mime_from_domain = rule.url_compose_map:process_url(task, mime_from_domain_tld, mime_from_domain)
+ else
+ mime_from_domain = mime_from_domain_tld
+ end
+ end
+ end
+
+ for _, d in ipairs(das[1].options) do
+
+ local domain, result = d:match('^([^%:]*):([%+%-%~])$')
+
+ -- We must ignore bad signatures, omg
+ if domain and result and result == '+' then
+ if rule.dkim_match_from then
+ -- We check merely mime from
+ local domain_tld = domain
+ if not rule.dkim_domainonly then
+ -- Adjust
+ domain_tld = rspamd_util.get_tld(domain)
+
+ if rule.url_compose_map then
+ domain_tld = rule.url_compose_map:process_url(task, domain_tld, domain)
+ elseif rule.url_full_hostname then
+ domain_tld = domain
+ end
+ end
+
+ if mime_from_domain and mime_from_domain == domain_tld then
+ add_dns_request(task, domain_tld, true, false, requests_table,
+ 'dkim', whitelist)
+ end
+ else
+ if rule.dkim_domainonly then
+ local domain_tld = rspamd_util.get_tld(domain)
+ if rule.url_compose_map then
+ domain_tld = rule.url_compose_map:process_url(task, domain_tld, domain)
+ elseif rule.url_full_hostname then
+ domain_tld = domain
+ end
+ add_dns_request(task, domain_tld,
+ false, false, requests_table, 'dkim', whitelist)
+ else
+ add_dns_request(task, domain, false, false, requests_table,
+ 'dkim', whitelist)
+ end
+ end
+ end
+ end
+ end
+
+ return true
+ end
+
+ local function check_urls(task, requests_table, whitelist)
+ local esld_lim = 1
+
+ if rule.url_compose_map then
+ esld_lim = nil -- Avoid esld limit as we use custom composition rules
+ end
+ local ex_params = {
+ task = task,
+ limit = rule.requests_limit,
+ ignore_redirected = true,
+ ignore_ip = rule.no_ip,
+ need_images = rule.images,
+ need_emails = false,
+ need_content = rule.content_urls or false,
+ esld_limit = esld_lim,
+ no_cache = true,
+ }
+
+ if rule.numeric_urls then
+ if rule.content_urls then
+ if not rule.images then
+ ex_params.flags_mode = 'explicit'
+ ex_params.flags = { 'numeric' }
+ ex_params.filter = function(url)
+ return (bit.band(url:get_flags_num(), url_flag_bits.image) == 0)
+ end
+ else
+ ex_params.filter = function(url)
+ return (bit.band(url:get_flags_num(), url_flag_bits.numeric) ~= 0)
+ end
+ end
+ elseif rule.images then
+ ex_params.filter = function(url)
+ return (bit.band(url:get_flags_num(), url_flag_bits.numeric) ~= 0)
+ end
+ else
+ ex_params.flags_mode = 'explicit'
+ ex_params.flags = { 'numeric' }
+ ex_params.filter = function(url)
+ return (bit.band(url:get_flags_num(), url_flag_bits.content) == 0)
+ end
+ end
+ elseif not rule.urls and (rule.content_urls or rule.images) then
+ ex_params.flags_mode = 'explicit'
+ ex_params.flags = {}
+ if rule.content_urls then
+ table.insert(ex_params.flags, 'content')
+ end
+ if rule.images then
+ table.insert(ex_params.flags, 'image')
+ end
+ end
+
+ local urls = lua_util.extract_specific_urls(ex_params)
+
+ for _, u in ipairs(urls) do
+ local flags = u:get_flags_num()
+
+ if bit.band(flags, url_flag_bits.numeric) ~= 0 then
+ -- For numeric urls we convert data to the ip address and
+ -- reverse octets. See #3948 for details
+ local to_resolve = u:get_host()
+ local addr = rspamd_ip.from_string(to_resolve)
+
+ if addr then
+ to_resolve = table.concat(addr:inversed_str_octets(), ".")
+ end
+ add_dns_request(task, to_resolve, false,
+ false, requests_table, 'url', whitelist)
+ else
+ local url_hostname = u:get_host()
+ local url_tld = rule.url_full_hostname and url_hostname or u:get_tld()
+ if rule.url_compose_map then
+ url_tld = rule.url_compose_map:process_url(task, url_tld, url_hostname)
+ end
+ add_dns_request(task, url_tld, false,
+ false, requests_table, 'url', whitelist)
+ end
+ end
+
+ return true
+ end
+
+ local function check_from(task, requests_table, whitelist)
+ local ip = task:get_from_ip()
+
+ if not ip or not ip:is_valid() then
+ return true
+ end
+ if (ip:get_version() == 6 and rule.ipv6) or
+ (ip:get_version() == 4 and rule.ipv4) then
+ add_dns_request(task, ip, true, true,
+ requests_table, 'from',
+ whitelist)
+ end
+
+ return true
+ end
+
+ local function check_received(task, requests_table, whitelist)
+ local received = fun .filter(function(h)
+ return not h['flags']['artificial']
+ end, task:get_received_headers()):totable()
+
+ local received_total = #received
+ local check_conditions = gen_check_rcvd_conditions(rule, received_total)
+
+ for pos, rh in ipairs(received) do
+ if check_conditions(rh, pos) then
+ add_dns_request(task, rh.real_ip, false, true,
+ requests_table, 'received',
+ whitelist)
+ end
+ end
+
+ return true
+ end
+
+ local function check_rdns(task, requests_table, whitelist)
+ local hostname = task:get_hostname()
+ if hostname == nil or hostname == 'unknown' then
+ return true
+ end
+
+ add_dns_request(task, hostname, true, false,
+ requests_table, 'rdns', whitelist)
+
+ return true
+ end
+
+ local function check_selector(task, requests_table, whitelist)
+ for selector_label, selector in pairs(rule.selectors) do
+ local res = selector(task)
+
+ if res and type(res) == 'table' then
+ for _, r in ipairs(res) do
+ add_dns_request(task, r, false, false, requests_table,
+ selector_label, whitelist)
+ end
+ elseif res then
+ add_dns_request(task, res, false, false,
+ requests_table, selector_label, whitelist)
+ end
+ end
+
+ return true
+ end
+
+ local function check_email_table(task, email_tbl, requests_table, whitelist, what)
+ lua_util.remove_email_aliases(email_tbl)
+ email_tbl.domain = email_tbl.domain:lower()
+ email_tbl.user = email_tbl.user:lower()
+
+ if email_tbl.domain == '' or email_tbl.user == '' then
+ rspamd_logger.infox(task, "got an email with some empty parts: '%s@%s'; skip it in the checks",
+ email_tbl.user, email_tbl.domain)
+ return
+ end
+
+ if rule.emails_domainonly then
+ add_dns_request(task, email_tbl.domain, false, false, requests_table,
+ what, whitelist)
+ else
+ -- Also check WL for domain only
+ if is_whitelisted(task,
+ email_tbl.domain,
+ email_tbl.domain,
+ whitelist,
+ what) then
+ return
+ end
+ local delimiter = '.'
+ if rule.emails_delimiter then
+ delimiter = rule.emails_delimiter
+ else
+ if rule.hash then
+ delimiter = '@'
+ end
+ end
+ add_dns_request(task, string.format('%s%s%s',
+ email_tbl.user, delimiter, email_tbl.domain), false, false,
+ requests_table, what, whitelist)
+ end
+ end
+
+ local function check_emails(task, requests_table, whitelist)
+ local ex_params = {
+ task = task,
+ limit = rule.requests_limit,
+ filter = function(u)
+ return u:get_protocol() == 'mailto'
+ end,
+ need_emails = true,
+ prefix = 'rbl_email'
+ }
+
+ if rule.emails_domainonly then
+ if not rule.url_compose_map then
+ ex_params.esld_limit = 1
+ end
+ ex_params.prefix = 'rbl_email_domainonly'
+ end
+
+ local emails = lua_util.extract_specific_urls(ex_params)
+
+ for _, email in ipairs(emails) do
+ local domain
+ if rule.emails_domainonly and not rule.url_full_hostname then
+ if rule.url_compose_map then
+ domain = rule.url_compose_map:process_url(task, email:get_tld(), email:get_host())
+ else
+ domain = email:get_tld()
+ end
+ else
+ domain = email:get_host()
+ end
+
+ local email_tbl = {
+ domain = domain or '',
+ user = email:get_user() or '',
+ addr = tostring(email),
+ }
+ check_email_table(task, email_tbl, requests_table, whitelist, 'email')
+ end
+
+ return true
+ end
+
+ local function check_replyto(task, requests_table, whitelist)
+ local function get_raw_header(name)
+ return ((task:get_header_full(name) or {})[1] or {})['value']
+ end
+
+ local replyto = get_raw_header('Reply-To')
+ if replyto then
+ local rt = rspamd_util.parse_mail_address(replyto, task:get_mempool())
+ lua_util.debugm(N, task, 'check replyto %s', rt[1])
+
+ if rt and rt[1] and (rt[1].addr and #rt[1].addr > 0) then
+ check_email_table(task, rt[1], requests_table, whitelist, 'replyto')
+ end
+ end
+
+ return true
+ end
+
+ -- Create function pipeline depending on rbl settings
+ local pipeline = {
+ is_alive, -- check monitored status
+ check_required_symbols -- if we have require_symbols then check those symbols
+ }
+ local description = {
+ 'alive',
+ }
+
+ if rule.exclude_users then
+ pipeline[#pipeline + 1] = check_user
+ description[#description + 1] = 'user'
+ end
+
+ if rule.exclude_local then
+ pipeline[#pipeline + 1] = check_local
+ description[#description + 1] = 'local'
+ end
+
+ if rule.helo then
+ pipeline[#pipeline + 1] = check_helo
+ description[#description + 1] = 'helo'
+ end
+
+ if rule.dkim then
+ pipeline[#pipeline + 1] = check_dkim
+ description[#description + 1] = 'dkim'
+ end
+
+ if rule.emails then
+ pipeline[#pipeline + 1] = check_emails
+ description[#description + 1] = 'emails'
+ end
+ if rule.replyto then
+ pipeline[#pipeline + 1] = check_replyto
+ description[#description + 1] = 'replyto'
+ end
+
+ if rule.urls or rule.content_urls or rule.images or rule.numeric_urls then
+ pipeline[#pipeline + 1] = check_urls
+ description[#description + 1] = 'urls'
+ end
+
+ if rule.from then
+ pipeline[#pipeline + 1] = check_from
+ description[#description + 1] = 'ip'
+ end
+
+ if rule.received then
+ pipeline[#pipeline + 1] = check_received
+ description[#description + 1] = 'received'
+ end
+
+ if rule.rdns then
+ pipeline[#pipeline + 1] = check_rdns
+ description[#description + 1] = 'rdns'
+ end
+
+ if rule.selector then
+ pipeline[#pipeline + 1] = check_selector
+ description[#description + 1] = 'selector'
+ end
+
+ if not rule.returncodes_matcher then
+ rule.returncodes_matcher = 'equality'
+ end
+ local match = matchers[rule.returncodes_matcher]
+
+ local callback_f = function(task)
+ -- DNS requests to issue (might be hashed afterwards)
+ local dns_req = {}
+ local whitelist = task:cache_get('rbl_whitelisted') or {}
+
+ local function gen_rbl_dns_callback(resolve_table_elt)
+ return function(_, to_resolve, results, err)
+ rbl_dns_process(task, rule, to_resolve, results, err, resolve_table_elt, match)
+ end
+ end
+
+ -- Execute functions pipeline
+ for i, f in ipairs(pipeline) do
+ if not f(task, dns_req, whitelist) then
+ lua_util.debugm(N, task,
+ "skip rbl check: %s; pipeline condition %s returned false",
+ rule.symbol, i)
+ return
+ end
+ end
+
+ -- Now check all DNS requests pending and emit them
+ local r = task:get_resolver()
+ -- Used for 2 passes ip resolution
+ local resolved_req = {}
+ local nresolved = 0
+
+ -- This is called when doing resolve_ip phase...
+ local function gen_rbl_ip_dns_callback(orig_resolve_table_elt)
+ return function(_, _, results, err)
+ if not err then
+ for _, dns_res in ipairs(results) do
+ -- Check if we have rspamd{ip} userdata
+ if type(dns_res) == 'userdata' then
+ -- Add result as an actual RBL request
+ local label = next(orig_resolve_table_elt.what)
+ local dup, nreq = add_dns_request(task, dns_res, false, true,
+ resolved_req, label)
+ -- Add original name
+ if not dup then
+ nreq.orig = nreq.orig .. ':' .. orig_resolve_table_elt.n
+ end
+ end
+ end
+ end
+
+ nresolved = nresolved - 1
+
+ if nresolved == 0 then
+ -- Emit real RBL requests as there are no ip resolution requests
+ for name, req in pairs(resolved_req) do
+ local val_res, val_error = validate_dns(req.n)
+ if val_res then
+ lua_util.debugm(N, task, "rbl %s; resolve %s -> %s",
+ rule.symbol, name, req.n)
+ r:resolve_a({
+ task = task,
+ name = req.n,
+ callback = gen_rbl_dns_callback(req),
+ forced = req.forced
+ })
+ else
+ rspamd_logger.warnx(task, 'cannot send invalid DNS request %s for %s: %s',
+ req.n, rule.symbol, val_error)
+ end
+ end
+ end
+ end
+ end
+
+ for name, req in pairs(dns_req) do
+ local val_res, val_error = validate_dns(req.n)
+ if val_res then
+ lua_util.debugm(N, task, "rbl %s; resolve %s -> %s",
+ rule.symbol, name, req.n)
+
+ if req.resolve_ip then
+ -- Deal with both ipv4 and ipv6
+ -- Resolve names first
+ if r:resolve_a({
+ task = task,
+ name = req.n,
+ callback = gen_rbl_ip_dns_callback(req),
+ forced = req.forced
+ }) then
+ nresolved = nresolved + 1
+ end
+ if r:resolve('aaaa', {
+ task = task,
+ name = req.n,
+ callback = gen_rbl_ip_dns_callback(req),
+ forced = req.forced
+ }) then
+ nresolved = nresolved + 1
+ end
+ else
+ r:resolve_a({
+ task = task,
+ name = req.n,
+ callback = gen_rbl_dns_callback(req),
+ forced = req.forced
+ })
+ end
+
+ else
+ rspamd_logger.warnx(task, 'cannot send invalid DNS request %s for %s: %s',
+ req.n, rule.symbol, val_error)
+ end
+ end
+ end
+
+ return callback_f, string.format('checks: %s', table.concat(description, ','))
+end
+
+local map_match_types = {
+ glob = true,
+ radix = true,
+ regexp = true,
+}
+
+local function add_rbl(key, rbl, global_opts)
+ if not rbl.symbol then
+ rbl.symbol = key:upper()
+ end
+
+ local flags_tbl = { 'no_squeeze' }
+ if rbl.is_whitelist then
+ flags_tbl[#flags_tbl + 1] = 'nice'
+ end
+
+ -- Check if rbl is available for empty tasks
+ if not (rbl.emails or rbl.urls or rbl.dkim or rbl.received or rbl.selector or rbl.replyto) or
+ rbl.is_empty then
+ flags_tbl[#flags_tbl + 1] = 'empty'
+ end
+
+ if rbl.selector then
+
+ rbl.selectors = {}
+ if type(rbl.selector) ~= 'table' then
+ rbl.selector = { ['selector'] = rbl.selector }
+ end
+
+ for selector_label, selector in pairs(rbl.selector) do
+ if known_selectors[selector] then
+ lua_util.debugm(N, rspamd_config, 'reuse selector id %s',
+ known_selectors[selector].id)
+ rbl.selectors[selector_label] = known_selectors[selector].selector
+ else
+
+ if type(rbl.selector_flatten) ~= 'boolean' then
+ -- Fail-safety
+ rbl.selector_flatten = true
+ end
+ local sel = selectors.create_selector_closure(rspamd_config, selector, '',
+ rbl.selector_flatten)
+
+ if not sel then
+ rspamd_logger.errx('invalid selector for rbl rule %s: %s', key, selector)
+ return false
+ end
+
+ rbl.selector = sel
+ known_selectors[selector] = {
+ selector = sel,
+ id = #lua_util.keys(known_selectors) + 1,
+ }
+ rbl.selectors[selector_label] = known_selectors[selector].selector
+ end
+ end
+
+ end
+
+ if rbl.process_script then
+ local ret, f = lua_util.callback_from_string(rbl.process_script)
+
+ if ret then
+ rbl.process_script = f
+ else
+ rspamd_logger.errx(rspamd_config,
+ 'invalid process script for rbl rule %s: %s; %s',
+ key, rbl.process_script, f)
+ return false
+ end
+ end
+
+ if rbl.whitelist then
+ local def_type = 'set'
+ if rbl.from or rbl.received then
+ def_type = 'radix'
+ end
+ rbl.whitelist = lua_maps.map_add_from_ucl(rbl.whitelist, def_type,
+ 'RBL whitelist for ' .. rbl.symbol)
+ rspamd_logger.infox(rspamd_config, 'added %s whitelist for RBL %s',
+ def_type, rbl.symbol)
+ end
+
+ local match_type = rbl.returncodes_matcher
+ if match_type and rbl.returncodes and map_match_types[match_type] then
+ if not rbl.returncodes_maps then
+ rbl.returncodes_maps = {}
+ end
+ for label, v in pairs(rbl.returncodes) do
+ if type(v) ~= 'table' then
+ v = {v}
+ end
+ rbl.returncodes_maps[label] = lua_maps.map_add_from_ucl(v, match_type, string.format('%s_%s RBL returncodes', label, rbl.symbol))
+ end
+ end
+
+ if rbl.url_compose_map then
+ local lua_urls_compose = require "lua_urls_compose"
+ rbl.url_compose_map = lua_urls_compose.add_composition_map(rspamd_config, rbl.url_compose_map)
+
+ if rbl.url_compose_map then
+ rspamd_logger.infox(rspamd_config, 'added url composition map for RBL %s',
+ rbl.symbol)
+ end
+ end
+
+ if not rbl.whitelist and not rbl.ignore_url_whitelist and (global_opts.url_whitelist or rbl.url_whitelist) and
+ (rbl.urls or rbl.emails or rbl.dkim or rbl.replyto) and
+ not (rbl.from or rbl.received) then
+ local def_type = 'set'
+ rbl.whitelist = lua_maps.map_add_from_ucl(rbl.url_whitelist or global_opts.url_whitelist, def_type,
+ 'RBL url whitelist for ' .. rbl.symbol)
+ rspamd_logger.infox(rspamd_config, 'added URL whitelist for RBL %s',
+ rbl.symbol)
+ end
+
+ local callback, description = gen_rbl_callback(rbl)
+
+ if callback then
+ local id
+
+ if rbl.symbols_prefixes then
+ id = rspamd_config:register_symbol {
+ type = 'callback',
+ callback = callback,
+ groups = { 'rbl' },
+ name = rbl.symbol .. '_CHECK',
+ flags = table.concat(flags_tbl, ',')
+ }
+
+ for _, prefix in pairs(rbl.symbols_prefixes) do
+ -- For unknown results...
+ rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = id,
+ group = 'rbl',
+ score = 0,
+ name = prefix .. '_' .. rbl.symbol,
+ }
+ end
+ if not (rbl.is_whitelist or rbl.ignore_whitelist) then
+ table.insert(black_symbols, rbl.symbol .. '_CHECK')
+ else
+ lua_util.debugm(N, rspamd_config, 'rule %s ignores whitelists: rbl.is_whitelist = %s, ' ..
+ 'rbl.ignore_whitelist = %s',
+ rbl.symbol, rbl.is_whitelist, rbl.ignore_whitelist)
+ end
+ else
+ id = rspamd_config:register_symbol {
+ type = 'callback',
+ callback = callback,
+ name = rbl.symbol,
+ groups = { 'rbl' },
+ group = 'rbl',
+ score = 0,
+ flags = table.concat(flags_tbl, ',')
+ }
+ if not (rbl.is_whitelist or rbl.ignore_whitelist) then
+ table.insert(black_symbols, rbl.symbol)
+ else
+ lua_util.debugm(N, rspamd_config, 'rule %s ignores whitelists: rbl.is_whitelist = %s, ' ..
+ 'rbl.ignore_whitelist = %s',
+ rbl.symbol, rbl.is_whitelist, rbl.ignore_whitelist)
+ end
+ end
+
+ rspamd_logger.infox(rspamd_config, 'added rbl rule %s: %s',
+ rbl.symbol, description)
+ lua_util.debugm(N, rspamd_config, 'rule dump for %s: %s',
+ rbl.symbol, rbl)
+
+ local check_sym = rbl.symbols_prefixes and rbl.symbol .. '_CHECK' or rbl.symbol
+
+ if rbl.dkim then
+ rspamd_config:register_dependency(check_sym, 'DKIM_CHECK')
+ end
+
+ if rbl.require_symbols then
+ for _, dep in ipairs(rbl.require_symbols) do
+ rspamd_config:register_dependency(check_sym, dep)
+ end
+ end
+
+ -- Failure symbol
+ rspamd_config:register_symbol {
+ type = 'virtual',
+ flags = 'nostat',
+ name = rbl.symbol .. '_FAIL',
+ parent = id,
+ score = 0.0,
+ }
+
+ local function process_return_code(suffix)
+ local function process_specific_suffix(s)
+ if s ~= rbl.symbol then
+ -- hack
+
+ rspamd_config:register_symbol {
+ type = 'virtual',
+ parent = id,
+ name = s,
+ group = 'rbl',
+ score = 0,
+ }
+ end
+ if rbl.is_whitelist then
+ if rbl.whitelist_exception then
+ local found_exception = false
+ for _, e in ipairs(rbl.whitelist_exception) do
+ if e == s then
+ found_exception = true
+ break
+ end
+ end
+ if not found_exception then
+ table.insert(white_symbols, s)
+ end
+ else
+ table.insert(white_symbols, s)
+ end
+ else
+ if not rbl.ignore_whitelist then
+ table.insert(black_symbols, s)
+ end
+ end
+ end
+
+ if rbl.symbols_prefixes then
+ for _, prefix in pairs(rbl.symbols_prefixes) do
+ process_specific_suffix(prefix .. '_' .. suffix)
+ end
+ else
+ process_specific_suffix(suffix)
+ end
+
+ end
+
+ if rbl.returncodes then
+ for s, _ in pairs(rbl.returncodes) do
+ process_return_code(s)
+ end
+ end
+
+ if rbl.returnbits then
+ for s, _ in pairs(rbl.returnbits) do
+ process_return_code(s)
+ end
+ end
+
+ -- Process monitored
+ if not rbl.disable_monitoring then
+ if not monitored_addresses[rbl.rbl] then
+ monitored_addresses[rbl.rbl] = true
+ rbl.monitored = rspamd_config:register_monitored(rbl.rbl, 'dns',
+ get_monitored(rbl))
+ end
+ end
+ return true
+ end
+
+ return false
+end
+
+-- Configuration
+local opts = rspamd_config:get_all_opt(N)
+if not (opts and type(opts) == 'table') then
+ rspamd_logger.infox(rspamd_config, 'Module is unconfigured')
+ lua_util.disable_module(N, "config")
+ return
+end
+
+-- Plugin defaults should not be changed - override these in config
+-- New defaults should not alter behaviour
+
+
+opts = lua_util.override_defaults(rbl_common.default_options, opts)
+
+if opts.rules and opts.rbls then
+ -- Common issue :(
+ rspamd_logger.infox(rspamd_config, 'merging `rules` and `rbls` keys for compatibility')
+ opts.rbls = lua_util.override_defaults(opts.rbls, opts.rules)
+end
+
+if (opts['local_exclude_ip_map'] ~= nil) then
+ local_exclusions = lua_maps.map_add(N, 'local_exclude_ip_map', 'radix',
+ 'RBL exclusions map')
+end
+
+-- TODO: this code should be universal for all modules that use selectors to allow
+-- maps usage from selectors registered for a specific module
+if type(opts.attached_maps) == 'table' then
+ opts.attached_maps_processed = {}
+ for i, map in ipairs(opts.attached_maps) do
+ -- Store maps in the configuration table to keep lifetime track
+ opts.attached_maps_processed[i] = lua_maps.map_add_from_ucl(map)
+ if opts.attached_maps_processed[i] == nil then
+ rspamd_logger.warnx(rspamd_config, "cannot parse attached map: %s", map)
+ end
+ end
+end
+
+for key, rbl in pairs(opts.rbls) do
+ if type(rbl) ~= 'table' or rbl.disabled == true or rbl.enabled == false then
+ rspamd_logger.infox(rspamd_config, 'disable rbl "%s"', key)
+ else
+ -- Aliases
+ if type(rbl.ignore_default) == 'boolean' then
+ rbl.ignore_defaults = rbl.ignore_default
+ end
+ if type(rbl.ignore_whitelists) == 'boolean' then
+ rbl.ignore_whitelist = rbl.ignore_whitelists
+ end
+ -- Propagate default options from opts to rule
+ if not rbl.ignore_defaults then
+ for default_opt_key, _ in pairs(rbl_common.default_options) do
+ local rbl_opt = default_opt_key:sub(#('default_') + 1)
+ if rbl[rbl_opt] == nil then
+ rbl[rbl_opt] = opts[default_opt_key]
+ end
+ end
+ end
+
+ if not rbl.requests_limit then
+ rbl.requests_limit = rspamd_config:get_dns_max_requests()
+ end
+
+ local res, err = rbl_common.rule_schema:transform(rbl)
+ if not res then
+ rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED',
+ key, err)
+ else
+ res = rbl_common.convert_checks(res, rbl.symbol or key:upper())
+ -- Aliases
+ if res.return_codes then
+ res.returncodes = res.return_codes
+ end
+ if res.return_bits then
+ res.returnbits = res.return_bits
+ end
+
+ if not res then
+ rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED',
+ key, err)
+ else
+ add_rbl(key, res, opts)
+ end
+ end
+ end -- rbl.enabled
+end
+
+-- We now create two symbols:
+-- * RBL_CALLBACK_WHITE that depends on all symbols white
+-- * RBL_CALLBACK that depends on all symbols black to participate in depends chains
+local function rbl_callback_white(task)
+ local whitelisted_elements = {}
+ for _, w in ipairs(white_symbols) do
+ local ws = task:get_symbol(w)
+ if ws and ws[1] then
+ ws = ws[1]
+ if not ws.options then
+ ws.options = {}
+ end
+ for _, opt in ipairs(ws.options) do
+ local elt, what = opt:match('^([^:]+):([^:]+)')
+ lua_util.debugm(N, task, 'found whitelist from %s: %s(%s)', w,
+ elt, what)
+ if elt and what then
+ whitelisted_elements[elt] = {
+ type = what,
+ symbol = w,
+ }
+ end
+ end
+ end
+ end
+
+ task:cache_set('rbl_whitelisted', whitelisted_elements)
+
+ lua_util.debugm(N, task, "finished rbl whitelists processing")
+end
+
+local function rbl_callback_fin(task)
+ -- Do nothing
+ lua_util.debugm(N, task, "finished rbl processing")
+end
+
+rspamd_config:register_symbol {
+ type = 'callback',
+ callback = rbl_callback_white,
+ name = 'RBL_CALLBACK_WHITE',
+ flags = 'nice,empty,no_squeeze',
+ groups = { 'rbl' },
+ augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0) },
+}
+
+rspamd_config:register_symbol {
+ type = 'callback',
+ callback = rbl_callback_fin,
+ name = 'RBL_CALLBACK',
+ flags = 'empty,no_squeeze',
+ groups = { 'rbl' },
+ augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0) },
+}
+
+for _, w in ipairs(white_symbols) do
+ rspamd_config:register_dependency('RBL_CALLBACK_WHITE', w)
+end
+
+for _, b in ipairs(black_symbols) do
+ rspamd_config:register_dependency(b, 'RBL_CALLBACK_WHITE')
+ rspamd_config:register_dependency('RBL_CALLBACK', b)
+end
diff --git a/src/plugins/lua/replies.lua b/src/plugins/lua/replies.lua
new file mode 100644
index 0000000..c4df9c9
--- /dev/null
+++ b/src/plugins/lua/replies.lua
@@ -0,0 +1,328 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+local rspamd_logger = require 'rspamd_logger'
+local hash = require 'rspamd_cryptobox_hash'
+local lua_util = require 'lua_util'
+local lua_redis = require 'lua_redis'
+local fun = require "fun"
+
+-- A plugin that implements replies check using redis
+
+-- Default port for redis upstreams
+local redis_params
+local settings = {
+ action = nil,
+ expire = 86400, -- 1 day by default
+ key_prefix = 'rr',
+ key_size = 20,
+ message = 'Message is reply to one we originated',
+ symbol = 'REPLY',
+ score = -4, -- Default score
+ use_auth = true,
+ use_local = true,
+ cookie = nil,
+ cookie_key = nil,
+ cookie_is_pattern = false,
+ cookie_valid_time = '2w', -- 2 weeks by default
+ min_message_id = 2, -- minimum length of the message-id header
+}
+
+local N = "replies"
+
+local function make_key(goop, sz, prefix)
+ local h = hash.create()
+ h:update(goop)
+ local key
+ if sz then
+ key = h:base32():sub(1, sz)
+ else
+ key = h:base32()
+ end
+
+ if prefix then
+ key = prefix .. key
+ end
+
+ return key
+end
+
+local function replies_check(task)
+ local in_reply_to
+ local function check_recipient(stored_rcpt)
+ local rcpts = task:get_recipients('mime')
+
+ if rcpts then
+ local filter_predicate = function(input_rcpt)
+ local real_rcpt_h = make_key(input_rcpt:lower(), 8)
+
+ return real_rcpt_h == stored_rcpt
+ end
+
+ if fun.any(filter_predicate, fun.map(function(rcpt)
+ return rcpt.addr or ''
+ end, rcpts)) then
+ lua_util.debugm(N, task, 'reply to %s validated', in_reply_to)
+ return true
+ end
+
+ rspamd_logger.infox(task, 'ignoring reply to %s as no recipients are matching hash %s',
+ in_reply_to, stored_rcpt)
+ else
+ rspamd_logger.infox(task, 'ignoring reply to %s as recipient cannot be detected for hash %s',
+ in_reply_to, stored_rcpt)
+ end
+
+ return false
+ end
+
+ local function redis_get_cb(err, data, addr)
+ if err ~= nil then
+ rspamd_logger.errx(task, 'redis_get_cb error when reading data from %s: %s', addr:get_addr(), err)
+ return
+ end
+ if data and type(data) == 'string' and check_recipient(data) then
+ -- Hash was found
+ task:insert_result(settings['symbol'], 1.0)
+ if settings['action'] ~= nil then
+ local ip_addr = task:get_ip()
+ if (settings.use_auth and
+ task:get_user()) or
+ (settings.use_local and ip_addr and ip_addr:is_local()) then
+ rspamd_logger.infox(task, "not forcing action for local network or authorized user");
+ else
+ task:set_pre_result(settings['action'], settings['message'], N)
+ end
+ end
+ end
+ end
+ -- If in-reply-to header not present return
+ in_reply_to = task:get_header_raw('in-reply-to')
+ if not in_reply_to then
+ return
+ end
+ -- Create hash of in-reply-to and query redis
+ local key = make_key(in_reply_to, settings.key_size, settings.key_prefix)
+
+ local ret = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_get_cb, --callback
+ 'GET', -- command
+ { key } -- arguments
+ )
+
+ if not ret then
+ rspamd_logger.errx(task, "redis request wasn't scheduled")
+ end
+end
+
+local function replies_set(task)
+ local function redis_set_cb(err, _, addr)
+ if err ~= nil then
+ rspamd_logger.errx(task, 'redis_set_cb error when writing data to %s: %s', addr:get_addr(), err)
+ end
+ end
+ -- If sender is unauthenticated return
+ local ip = task:get_ip()
+ if settings.use_auth and task:get_user() then
+ lua_util.debugm(N, task, 'sender is authenticated')
+ elseif settings.use_local and (ip and ip:is_local()) then
+ lua_util.debugm(N, task, 'sender is from local network')
+ else
+ return
+ end
+ -- If no message-id present return
+ local msg_id = task:get_header_raw('message-id')
+ if msg_id == nil or msg_id:len() <= (settings.min_message_id or 2) then
+ return
+ end
+ -- Create hash of message-id and store to redis
+ local key = make_key(msg_id, settings.key_size, settings.key_prefix)
+
+ local sender = task:get_reply_sender()
+
+ if sender then
+ local sender_hash = make_key(sender:lower(), 8)
+ lua_util.debugm(N, task, 'storing id: %s (%s), reply-to: %s (%s) for replies check',
+ msg_id, key, sender, sender_hash)
+ local ret = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'PSETEX', -- command
+ { key, tostring(math.floor(settings['expire'] * 1000)), sender_hash } -- arguments
+ )
+ if not ret then
+ rspamd_logger.errx(task, "redis request wasn't scheduled")
+ end
+ else
+ rspamd_logger.infox(task, "cannot find reply sender address")
+ end
+end
+
+local function replies_check_cookie(task)
+ local function cookie_matched(extra, ts)
+ local dt = task:get_date { format = 'connect', gmt = true }
+
+ if dt < ts then
+ rspamd_logger.infox(task, 'ignore cookie as its date is in future')
+
+ return
+ end
+
+ if settings.cookie_valid_time then
+ if dt - ts > settings.cookie_valid_time then
+ rspamd_logger.infox(task,
+ 'ignore cookie as its timestamp is too old: %s (%s current time)',
+ ts, dt)
+
+ return
+ end
+ end
+
+ if extra then
+ task:insert_result(settings['symbol'], 1.0,
+ string.format('cookie:%s:%s', extra, ts))
+ else
+ task:insert_result(settings['symbol'], 1.0,
+ string.format('cookie:%s', ts))
+ end
+ if settings['action'] ~= nil then
+ local ip_addr = task:get_ip()
+ if (settings.use_auth and
+ task:get_user()) or
+ (settings.use_local and ip_addr and ip_addr:is_local()) then
+ rspamd_logger.infox(task, "not forcing action for local network or authorized user");
+ else
+ task:set_pre_result(settings['action'], settings['message'], N)
+ end
+ end
+ end
+
+ -- If in-reply-to header not present return
+ local irt = task:get_header('in-reply-to')
+ if irt == nil then
+ return
+ end
+
+ local cr = require "rspamd_cryptobox"
+ -- Extract user part if needed
+ local extracted_cookie = irt:match('^%<?([^@]+)@.*$')
+ if not extracted_cookie then
+ -- Assume full message id as a cookie
+ extracted_cookie = irt
+ end
+
+ local dec_cookie, ts = cr.decrypt_cookie(settings.cookie_key, extracted_cookie)
+
+ if dec_cookie then
+ -- We have something that looks like a cookie
+ if settings.cookie_is_pattern then
+ local m = dec_cookie:match(settings.cookie)
+
+ if m then
+ cookie_matched(m, ts)
+ end
+ else
+ -- Direct match
+ if dec_cookie == settings.cookie then
+ cookie_matched(nil, ts)
+ end
+ end
+ end
+end
+
+local opts = rspamd_config:get_all_opt('replies')
+if not (opts and type(opts) == 'table') then
+ rspamd_logger.infox(rspamd_config, 'module is unconfigured')
+ return
+end
+if opts then
+ settings = lua_util.override_defaults(settings, opts)
+ redis_params = lua_redis.parse_redis_server('replies')
+ if not redis_params then
+ if not (settings.cookie and settings.cookie_key) then
+ rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
+ lua_util.disable_module(N, "redis")
+ else
+ -- Cookies mode
+ -- Check key sanity:
+ local pattern = { '^' }
+ for i = 1, 32 do
+ pattern[i + 1] = '[a-zA-Z0-9]'
+ end
+ pattern[34] = '$'
+ if not settings.cookie_key:match(table.concat(pattern, '')) then
+ rspamd_logger.errx(rspamd_config,
+ 'invalid cookies key: %s, must be 32 hex digits', settings.cookie_key)
+ lua_util.disable_module(N, "config")
+
+ return
+ end
+
+ if settings.cookie_valid_time then
+ settings.cookie_valid_time = lua_util.parse_time_interval(settings.cookie_valid_time)
+ end
+
+ local id = rspamd_config:register_symbol({
+ name = 'REPLIES_CHECK',
+ type = 'prefilter',
+ callback = replies_check_cookie,
+ flags = 'nostat',
+ priority = lua_util.symbols_priorities.medium,
+ group = "replies"
+ })
+ rspamd_config:register_symbol({
+ name = settings['symbol'],
+ parent = id,
+ type = 'virtual',
+ score = settings.score,
+ group = "replies",
+ })
+ end
+ else
+ rspamd_config:register_symbol({
+ name = 'REPLIES_SET',
+ type = 'idempotent',
+ callback = replies_set,
+ group = 'replies',
+ flags = 'explicit_disable,ignore_passthrough',
+ })
+ local id = rspamd_config:register_symbol({
+ name = 'REPLIES_CHECK',
+ type = 'prefilter',
+ flags = 'nostat',
+ callback = replies_check,
+ priority = lua_util.symbols_priorities.medium,
+ group = "replies"
+ })
+ rspamd_config:register_symbol({
+ name = settings['symbol'],
+ parent = id,
+ type = 'virtual',
+ score = settings.score,
+ group = "replies",
+ })
+ end
+end
diff --git a/src/plugins/lua/reputation.lua b/src/plugins/lua/reputation.lua
new file mode 100644
index 0000000..a3af26c
--- /dev/null
+++ b/src/plugins/lua/reputation.lua
@@ -0,0 +1,1390 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- A generic plugin for reputation handling
+
+local E = {}
+local N = 'reputation'
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local lua_util = require "lua_util"
+local lua_maps = require "lua_maps"
+local lua_maps_exprs = require "lua_maps_expressions"
+local hash = require 'rspamd_cryptobox_hash'
+local lua_redis = require "lua_redis"
+local fun = require "fun"
+local lua_selectors = require "lua_selectors"
+local ts = require("tableshape").types
+
+local redis_params = nil
+local default_expiry = 864000 -- 10 day by default
+local default_prefix = 'RR:' -- Rspamd Reputation
+
+local tanh = math.tanh or rspamd_util.tanh
+
+-- Get reputation from ham/spam/probable hits
+local function generic_reputation_calc(token, rule, mult, task)
+ local cfg = rule.selector.config or E
+ local reject_threshold = task:get_metric_score()[2] or 10.0
+
+ if cfg.score_calc_func then
+ return cfg.score_calc_func(rule, token, mult)
+ end
+
+ if tonumber(token[1]) < cfg.lower_bound then
+ lua_util.debugm(N, task, "not enough matches %s < %s for rule %s",
+ token[1], cfg.lower_bound, rule.symbol)
+ return 0
+ end
+
+ -- Get average score
+ local avg_score = fun.foldl(function(acc, v)
+ return acc + v
+ end, 0.0, fun.map(tonumber, token[2])) / #token[2]
+
+ -- Apply function tanh(x / reject_score * atanh(0.95) - atanh(0.5))
+ -- 1.83178 0.5493
+ local score = tanh(avg_score / reject_threshold * 1.83178 - 0.5493) * mult
+ lua_util.debugm(N, task, "got generic average score %s (reject threshold=%s, mult=%s) -> %s for rule %s",
+ avg_score, reject_threshold, mult, score, rule.symbol)
+ return score
+end
+
+local function add_symbol_score(task, rule, mult, params)
+ if not params then
+ params = { tostring(mult) }
+ end
+
+ if rule.selector.config.split_symbols then
+ local sym_spam = rule.symbol .. '_SPAM'
+ local sym_ham = rule.symbol .. '_HAM'
+ if not rule.static_symbols then
+ rule.static_symbols = {}
+ rule.static_symbols.ham = rspamd_config:get_symbol(sym_ham)
+ rule.static_symbols.spam = rspamd_config:get_symbol(sym_spam)
+ end
+ if mult >= 0 then
+ task:insert_result(sym_spam, mult, params)
+ else
+ -- Avoid multiplication of negative the `mult` by negative static score of the
+ -- ham symbol
+ if rule.static_symbols.ham and rule.static_symbols.ham.score then
+ if rule.static_symbols.ham.score < 0 then
+ mult = math.abs(mult)
+ end
+ end
+ task:insert_result(sym_ham, mult, params)
+ end
+ else
+ task:insert_result(rule.symbol, mult, params)
+ end
+end
+
+local function sub_symbol_score(task, rule, score)
+ local function sym_score(sym)
+ local s = task:get_symbol(sym)[1]
+ return s.score
+ end
+ if rule.selector.config.split_symbols then
+ local spam_sym = rule.symbol .. '_SPAM'
+ local ham_sym = rule.symbol .. '_HAM'
+
+ if task:has_symbol(spam_sym) then
+ score = score - sym_score(spam_sym)
+ elseif task:has_symbol(ham_sym) then
+ score = score - sym_score(ham_sym)
+ end
+ else
+ if task:has_symbol(rule.symbol) then
+ score = score - sym_score(rule.symbol)
+ end
+ end
+
+ return score
+end
+
+-- Extracts task score and subtracts score of the rule itself
+local function extract_task_score(task, rule)
+ local lua_verdict = require "lua_verdict"
+ local verdict, score = lua_verdict.get_specific_verdict(N, task)
+
+ if not score or verdict == 'passthrough' then
+ return nil
+ end
+
+ return sub_symbol_score(task, rule, score)
+end
+
+-- DKIM Selector functions
+local gr
+local function gen_dkim_queries(task, rule)
+ local dkim_trace = (task:get_symbol('DKIM_TRACE') or E)[1]
+ local lpeg = require 'lpeg'
+ local ret = {}
+
+ if not gr then
+ local semicolon = lpeg.P(':')
+ local domain = lpeg.C((1 - semicolon) ^ 1)
+ local res = lpeg.S '+-?~'
+
+ local function res_to_label(ch)
+ if ch == '+' then
+ return 'a'
+ elseif ch == '-' then
+ return 'r'
+ end
+
+ return 'u'
+ end
+
+ gr = domain * semicolon * (lpeg.C(res ^ 1) / res_to_label)
+ end
+
+ if dkim_trace and dkim_trace.options then
+ for _, opt in ipairs(dkim_trace.options) do
+ local dom, res = lpeg.match(gr, opt)
+
+ if dom and res then
+ local tld = rspamd_util.get_tld(dom)
+ ret[tld] = res
+ end
+ end
+ end
+
+ return ret
+end
+
+local function dkim_reputation_filter(task, rule)
+ local requests = gen_dkim_queries(task, rule)
+ local results = {}
+ local dkim_tlds = lua_util.keys(requests)
+ local requests_left = #dkim_tlds
+ local rep_accepted = 0.0
+
+ lua_util.debugm(N, task, 'dkim reputation tokens: %s', requests)
+
+ local function tokens_cb(err, token, values)
+ requests_left = requests_left - 1
+
+ if values then
+ results[token] = values
+ end
+
+ if requests_left == 0 then
+ for k, v in pairs(results) do
+ -- `k` in results is a prefixed and suffixed tld, so we need to look through
+ -- all requests to find any request with the matching tld
+ local sel_tld
+ for _, tld in ipairs(dkim_tlds) do
+ if k:find(tld, 1, true) then
+ sel_tld = tld
+ break
+ end
+ end
+
+ if sel_tld and requests[sel_tld] then
+ if requests[sel_tld] == 'a' then
+ rep_accepted = rep_accepted + generic_reputation_calc(v, rule, 1.0, task)
+ end
+ else
+ rspamd_logger.warnx(task, "cannot find the requested tld for a request: %s (%s tlds noticed)",
+ k, dkim_tlds)
+ end
+ end
+
+ -- Set local reputation symbol
+ local rep_accepted_abs = math.abs(rep_accepted or 0)
+ lua_util.debugm(N, task, "dkim reputation accepted: %s",
+ rep_accepted_abs)
+ if rep_accepted_abs then
+ local final_rep = rep_accepted
+ if rep_accepted > 1.0 then
+ final_rep = 1.0
+ end
+ if rep_accepted < -1.0 then
+ final_rep = -1.0
+ end
+ add_symbol_score(task, rule, final_rep)
+
+ -- Store results for future DKIM results adjustments
+ task:get_mempool():set_variable("dkim_reputation_accept", tostring(rep_accepted))
+ end
+ end
+ end
+
+ for dom, res in pairs(requests) do
+ -- tld + "." + check_result, e.g. example.com.+ - reputation for valid sigs
+ local query = string.format('%s.%s', dom, res)
+ rule.backend.get_token(task, rule, nil, query, tokens_cb, 'string')
+ end
+end
+
+local function dkim_reputation_idempotent(task, rule)
+ local requests = gen_dkim_queries(task, rule)
+ local sc = extract_task_score(task, rule)
+
+ if sc then
+ for dom, res in pairs(requests) do
+ -- tld + "." + check_result, e.g. example.com.+ - reputation for valid sigs
+ local query = string.format('%s.%s', dom, res)
+ rule.backend.set_token(task, rule, nil, query, sc)
+ end
+ end
+end
+
+local function dkim_reputation_postfilter(task, rule)
+ local sym_accepted = (task:get_symbol('R_DKIM_ALLOW') or E)[1]
+ local accept_adjustment = task:get_mempool():get_variable("dkim_reputation_accept")
+ local cfg = rule.selector.config or E
+
+ if sym_accepted and sym_accepted.score and
+ accept_adjustment and type(cfg.max_accept_adjustment) == 'number' then
+ local final_adjustment = cfg.max_accept_adjustment *
+ rspamd_util.tanh(tonumber(accept_adjustment) or 0)
+ lua_util.debugm(N, task, "adjust DKIM_ALLOW: " ..
+ "cfg.max_accept_adjustment=%s accept_adjustment=%s final_adjustment=%s sym_accepted.score=%s",
+ cfg.max_accept_adjustment, accept_adjustment, final_adjustment,
+ sym_accepted.score)
+
+ task:adjust_result('R_DKIM_ALLOW', sym_accepted.score + final_adjustment)
+ end
+end
+
+local dkim_selector = {
+ config = {
+ symbol = 'DKIM_SCORE', -- symbol to be inserted
+ lower_bound = 10, -- minimum number of messages to be scored
+ min_score = nil,
+ max_score = nil,
+ outbound = true,
+ inbound = true,
+ max_accept_adjustment = 2.0, -- How to adjust accepted DKIM score
+ },
+ dependencies = { "DKIM_TRACE" },
+ filter = dkim_reputation_filter, -- used to get scores
+ postfilter = dkim_reputation_postfilter, -- used to adjust DKIM scores
+ idempotent = dkim_reputation_idempotent, -- used to set scores
+}
+
+-- URL Selector functions
+
+local function gen_url_queries(task, rule)
+ local domains = {}
+
+ fun.each(function(u)
+ if u:is_redirected() then
+ local redir = u:get_redirected() -- get the original url
+ local redir_tld = redir:get_tld()
+ if domains[redir_tld] then
+ domains[redir_tld] = domains[redir_tld] - 1
+ end
+ end
+ local dom = u:get_tld()
+ if not domains[dom] then
+ domains[dom] = 1
+ else
+ domains[dom] = domains[dom] + 1
+ end
+ end, fun.filter(function(u)
+ return not u:is_html_displayed()
+ end,
+ task:get_urls(true)))
+
+ local results = {}
+ for k, v in lua_util.spairs(domains,
+ function(t, a, b)
+ return t[a] > t[b]
+ end, rule.selector.config.max_urls) do
+ if v > 0 then
+ table.insert(results, { k, v })
+ end
+ end
+
+ return results
+end
+
+local function url_reputation_filter(task, rule)
+ local requests = gen_url_queries(task, rule)
+ local url_keys = lua_util.keys(requests)
+ local requests_left = #url_keys
+ local results = {}
+
+ local function indexed_tokens_cb(err, index, values)
+ requests_left = requests_left - 1
+
+ if values then
+ results[index] = values
+ end
+
+ if requests_left == 0 then
+ -- Check the url with maximum hits
+ local mhits = 0
+
+ for i, res in ipairs(results) do
+ local req = requests[i]
+ if req then
+ local hits = tonumber(res[1])
+ if hits > mhits then
+ mhits = hits
+ end
+ else
+ rspamd_logger.warnx(task, "cannot find the requested response for a request: %s (%s requests noticed)",
+ i, #requests)
+ end
+ end
+
+ if mhits > 0 then
+ local score = 0
+ for i, res in pairs(results) do
+ local req = requests[i]
+ if req then
+ local url_score = generic_reputation_calc(res, rule,
+ req[2] / mhits, task)
+ lua_util.debugm(N, task, "score for url %s is %s, score=%s", req[1], url_score, score)
+ score = score + url_score
+ end
+ end
+
+ if math.abs(score) > 1e-3 then
+ -- TODO: add description
+ add_symbol_score(task, rule, score)
+ end
+ end
+ end
+ end
+
+ for i, req in ipairs(requests) do
+ local function tokens_cb(err, token, values)
+ indexed_tokens_cb(err, i, values)
+ end
+
+ rule.backend.get_token(task, rule, nil, req[1], tokens_cb, 'string')
+ end
+end
+
+local function url_reputation_idempotent(task, rule)
+ local requests = gen_url_queries(task, rule)
+ local sc = extract_task_score(task, rule)
+
+ if sc then
+ for _, tld in ipairs(requests) do
+ rule.backend.set_token(task, rule, nil, tld[1], sc)
+ end
+ end
+end
+
+local url_selector = {
+ config = {
+ symbol = 'URL_SCORE', -- symbol to be inserted
+ lower_bound = 10, -- minimum number of messages to be scored
+ min_score = nil,
+ max_score = nil,
+ max_urls = 10,
+ check_from = true,
+ outbound = true,
+ inbound = true,
+ },
+ filter = url_reputation_filter, -- used to get scores
+ idempotent = url_reputation_idempotent -- used to set scores
+}
+-- IP Selector functions
+
+local function ip_reputation_init(rule)
+ local cfg = rule.selector.config
+
+ if cfg.asn_cc_whitelist then
+ cfg.asn_cc_whitelist = lua_maps.map_add('reputation',
+ 'asn_cc_whitelist',
+ 'map',
+ 'IP score whitelisted ASNs/countries')
+ end
+
+ return true
+end
+
+local function ip_reputation_filter(task, rule)
+
+ local ip = task:get_from_ip()
+
+ if not ip or not ip:is_valid() then
+ return
+ end
+ if lua_util.is_rspamc_or_controller(task) then
+ return
+ end
+
+ local cfg = rule.selector.config
+
+ if ip:get_version() == 4 and cfg.ipv4_mask then
+ ip = ip:apply_mask(cfg.ipv4_mask)
+ elseif cfg.ipv6_mask then
+ ip = ip:apply_mask(cfg.ipv6_mask)
+ end
+
+ local pool = task:get_mempool()
+ local asn = pool:get_variable("asn")
+ local country = pool:get_variable("country")
+
+ if country and cfg.asn_cc_whitelist then
+ if cfg.asn_cc_whitelist:get_key(country) then
+ return
+ end
+ if asn and cfg.asn_cc_whitelist:get_key(asn) then
+ return
+ end
+ end
+
+ -- These variables are used to define if we have some specific token
+ local has_asn = not asn
+ local has_country = not country
+ local has_ip = false
+
+ local asn_stats, country_stats, ip_stats
+
+ local function ipstats_check()
+ local score = 0.0
+ local description_t = {}
+
+ if asn_stats then
+ local asn_score = generic_reputation_calc(asn_stats, rule, cfg.scores.asn, task)
+ score = score + asn_score
+ table.insert(description_t, string.format('asn: %s(%.2f)',
+ asn, asn_score))
+ end
+ if country_stats then
+ local country_score = generic_reputation_calc(country_stats, rule,
+ cfg.scores.country, task)
+ score = score + country_score
+ table.insert(description_t, string.format('country: %s(%.2f)',
+ country, country_score))
+ end
+ if ip_stats then
+ local ip_score = generic_reputation_calc(ip_stats, rule, cfg.scores.ip,
+ task)
+ score = score + ip_score
+ table.insert(description_t, string.format('ip: %s(%.2f)',
+ tostring(ip), ip_score))
+ end
+
+ if math.abs(score) > 0.001 then
+ add_symbol_score(task, rule, score, table.concat(description_t, ', '))
+ end
+ end
+
+ local function gen_token_callback(what)
+ return function(err, _, values)
+ if not err and values then
+ if what == 'asn' then
+ has_asn = true
+ asn_stats = values
+ elseif what == 'country' then
+ has_country = true
+ country_stats = values
+ elseif what == 'ip' then
+ has_ip = true
+ ip_stats = values
+ end
+ else
+ if what == 'asn' then
+ has_asn = true
+ elseif what == 'country' then
+ has_country = true
+ elseif what == 'ip' then
+ has_ip = true
+ end
+ end
+
+ if has_asn and has_country and has_ip then
+ -- Check reputation
+ ipstats_check()
+ end
+ end
+ end
+
+ if asn then
+ rule.backend.get_token(task, rule, cfg.asn_prefix, asn,
+ gen_token_callback('asn'), 'string')
+ end
+ if country then
+ rule.backend.get_token(task, rule, cfg.country_prefix, country,
+ gen_token_callback('country'), 'string')
+ end
+
+ rule.backend.get_token(task, rule, cfg.ip_prefix, ip,
+ gen_token_callback('ip'), 'ip')
+end
+
+-- Used to set scores
+local function ip_reputation_idempotent(task, rule)
+ if not rule.backend.set_token then
+ return
+ end -- Read only backend
+ local ip = task:get_from_ip()
+ local cfg = rule.selector.config
+
+ if not ip or not ip:is_valid() then
+ return
+ end
+
+ if lua_util.is_rspamc_or_controller(task) then
+ return
+ end
+
+ if ip:get_version() == 4 and cfg.ipv4_mask then
+ ip = ip:apply_mask(cfg.ipv4_mask)
+ elseif cfg.ipv6_mask then
+ ip = ip:apply_mask(cfg.ipv6_mask)
+ end
+
+ local pool = task:get_mempool()
+ local asn = pool:get_variable("asn")
+ local country = pool:get_variable("country")
+
+ if country and cfg.asn_cc_whitelist then
+ if cfg.asn_cc_whitelist:get_key(country) then
+ return
+ end
+ if asn and cfg.asn_cc_whitelist:get_key(asn) then
+ return
+ end
+ end
+ local sc = extract_task_score(task, rule)
+ if sc then
+ if asn then
+ rule.backend.set_token(task, rule, cfg.asn_prefix, asn, sc, nil, 'string')
+ end
+ if country then
+ rule.backend.set_token(task, rule, cfg.country_prefix, country, sc, nil, 'string')
+ end
+
+ rule.backend.set_token(task, rule, cfg.ip_prefix, ip, sc, nil, 'ip')
+ end
+end
+
+-- Selectors are used to extract reputation tokens
+local ip_selector = {
+ config = {
+ scores = { -- how each component is evaluated
+ ['asn'] = 0.4,
+ ['country'] = 0.01,
+ ['ip'] = 1.0
+ },
+ symbol = 'SENDER_REP', -- symbol to be inserted
+ split_symbols = true,
+ asn_prefix = 'a:', -- prefix for ASN hashes
+ country_prefix = 'c:', -- prefix for country hashes
+ ip_prefix = 'i:',
+ lower_bound = 10, -- minimum number of messages to be scored
+ min_score = nil,
+ max_score = nil,
+ score_divisor = 1,
+ outbound = false,
+ inbound = true,
+ ipv4_mask = 32, -- Mask bits for ipv4
+ ipv6_mask = 64, -- Mask bits for ipv6
+ },
+ --dependencies = {"ASN"}, -- ASN is a prefilter now...
+ init = ip_reputation_init,
+ filter = ip_reputation_filter, -- used to get scores
+ idempotent = ip_reputation_idempotent, -- used to set scores
+}
+
+-- SPF Selector functions
+
+local function spf_reputation_filter(task, rule)
+ local spf_record = task:get_mempool():get_variable('spf_record')
+ local spf_allow = task:has_symbol('R_SPF_ALLOW')
+
+ -- Don't care about bad/missing spf
+ if not spf_record or not spf_allow then
+ return
+ end
+
+ local cr = require "rspamd_cryptobox_hash"
+ local hkey = cr.create(spf_record):base32():sub(1, 32)
+
+ lua_util.debugm(N, task, 'check spf record %s -> %s', spf_record, hkey)
+
+ local function tokens_cb(err, token, values)
+ if values then
+ local score = generic_reputation_calc(values, rule, 1.0, task)
+
+ if math.abs(score) > 1e-3 then
+ -- TODO: add description
+ add_symbol_score(task, rule, score)
+ end
+ end
+ end
+
+ rule.backend.get_token(task, rule, nil, hkey, tokens_cb, 'string')
+end
+
+local function spf_reputation_idempotent(task, rule)
+ local sc = extract_task_score(task, rule)
+ local spf_record = task:get_mempool():get_variable('spf_record')
+ local spf_allow = task:has_symbol('R_SPF_ALLOW')
+
+ if not spf_record or not spf_allow or not sc then
+ return
+ end
+
+ local cr = require "rspamd_cryptobox_hash"
+ local hkey = cr.create(spf_record):base32():sub(1, 32)
+
+ lua_util.debugm(N, task, 'set spf record %s -> %s = %s',
+ spf_record, hkey, sc)
+ rule.backend.set_token(task, rule, nil, hkey, sc)
+end
+
+local spf_selector = {
+ config = {
+ symbol = 'SPF_REP', -- symbol to be inserted
+ split_symbols = true,
+ lower_bound = 10, -- minimum number of messages to be scored
+ min_score = nil,
+ max_score = nil,
+ outbound = true,
+ inbound = true,
+ },
+ dependencies = { "R_SPF_ALLOW" },
+ filter = spf_reputation_filter, -- used to get scores
+ idempotent = spf_reputation_idempotent, -- used to set scores
+}
+
+-- Generic selector based on lua_selectors framework
+
+local function generic_reputation_init(rule)
+ local cfg = rule.selector.config
+
+ if not cfg.selector then
+ rspamd_logger.errx(rspamd_config, 'cannot configure generic rule: no selector specified')
+ return false
+ end
+
+ local selector = lua_selectors.create_selector_closure(rspamd_config,
+ cfg.selector, cfg.delimiter)
+
+ if not selector then
+ rspamd_logger.errx(rspamd_config, 'cannot configure generic rule: bad selector: %s',
+ cfg.selector)
+ return false
+ end
+
+ cfg.selector = selector -- Replace with closure
+
+ if cfg.whitelist then
+ cfg.whitelist = lua_maps.map_add('reputation',
+ 'generic_whitelist',
+ 'map',
+ 'Whitelisted selectors')
+ end
+
+ return true
+end
+
+local function generic_reputation_filter(task, rule)
+ local cfg = rule.selector.config
+ local selector_res = cfg.selector(task)
+
+ local function tokens_cb(err, token, values)
+ if values then
+ local score = generic_reputation_calc(values, rule, 1.0, task)
+
+ if math.abs(score) > 1e-3 then
+ -- TODO: add description
+ add_symbol_score(task, rule, score)
+ end
+ end
+ end
+
+ if selector_res then
+ if type(selector_res) == 'table' then
+ fun.each(function(e)
+ lua_util.debugm(N, task, 'check generic reputation (%s) %s',
+ rule['symbol'], e)
+ rule.backend.get_token(task, rule, nil, e, tokens_cb, 'string')
+ end, selector_res)
+ else
+ lua_util.debugm(N, task, 'check generic reputation (%s) %s',
+ rule['symbol'], selector_res)
+ rule.backend.get_token(task, rule, nil, selector_res, tokens_cb, 'string')
+ end
+ end
+end
+
+local function generic_reputation_idempotent(task, rule)
+ local sc = extract_task_score(task, rule)
+ local cfg = rule.selector.config
+
+ local selector_res = cfg.selector(task)
+ if not selector_res then
+ return
+ end
+
+ if sc then
+ if type(selector_res) == 'table' then
+ fun.each(function(e)
+ lua_util.debugm(N, task, 'set generic selector (%s) %s = %s',
+ rule['symbol'], e, sc)
+ rule.backend.set_token(task, rule, nil, e, sc)
+ end, selector_res)
+ else
+ lua_util.debugm(N, task, 'set generic selector (%s) %s = %s',
+ rule['symbol'], selector_res, sc)
+ rule.backend.set_token(task, rule, nil, selector_res, sc)
+ end
+ end
+end
+
+local generic_selector = {
+ schema = ts.shape {
+ lower_bound = ts.number + ts.string / tonumber,
+ max_score = ts.number:is_optional(),
+ min_score = ts.number:is_optional(),
+ outbound = ts.boolean,
+ inbound = ts.boolean,
+ selector = ts.string,
+ delimiter = ts.string,
+ whitelist = ts.one_of(lua_maps.map_schema, lua_maps_exprs.schema):is_optional(),
+ },
+ config = {
+ lower_bound = 10, -- minimum number of messages to be scored
+ min_score = nil,
+ max_score = nil,
+ outbound = true,
+ inbound = true,
+ selector = nil,
+ delimiter = ':',
+ whitelist = nil
+ },
+ init = generic_reputation_init,
+ filter = generic_reputation_filter, -- used to get scores
+ idempotent = generic_reputation_idempotent -- used to set scores
+}
+
+local selectors = {
+ ip = ip_selector,
+ sender = ip_selector, -- Better name
+ url = url_selector,
+ dkim = dkim_selector,
+ spf = spf_selector,
+ generic = generic_selector
+}
+
+local function reputation_dns_init(rule, _, _, _)
+ if not rule.backend.config.list then
+ rspamd_logger.errx(rspamd_config, "rule %s with DNS backend has no `list` parameter defined",
+ rule.symbol)
+ return false
+ end
+
+ return true
+end
+
+local function gen_token_key(prefix, token, rule)
+ if prefix then
+ token = prefix .. token
+ end
+ local res = token
+ if rule.backend.config.hashed then
+ local hash_alg = rule.backend.config.hash_alg or "blake2"
+ local encoding = "base32"
+
+ if rule.backend.config.hash_encoding then
+ encoding = rule.backend.config.hash_encoding
+ end
+
+ local h = hash.create_specific(hash_alg, res)
+ if encoding == 'hex' then
+ res = h:hex()
+ elseif encoding == 'base64' then
+ res = h:base64()
+ else
+ res = h:base32()
+ end
+ end
+
+ if rule.backend.config.hashlen then
+ res = string.sub(res, 1, rule.backend.config.hashlen)
+ end
+
+ if rule.backend.config.prefix then
+ res = rule.backend.config.prefix .. res
+ end
+
+ return res
+end
+
+--[[
+-- Generic interface for get and set tokens functions:
+-- get_token(task, rule, prefix, token, continuation, token_type), where `continuation` is the following function:
+--
+-- function(err, token, values) ... end
+-- `err`: string value for error (similar to redis or DNS callbacks)
+-- `token`: string value of a token
+-- `values`: table of key=number, parsed from backend. It is selector's duty
+-- to deal with missing, invalid or other values
+--
+-- set_token(task, rule, token, values, continuation_cb)
+-- This function takes values, encodes them using whatever suitable format
+-- and calls for continuation:
+--
+-- function(err, token) ... end
+-- `err`: string value for error (similar to redis or DNS callbacks)
+-- `token`: string value of a token
+--
+-- example of tokens: {'s': 0, 'h': 0, 'p': 1}
+--]]
+
+local function reputation_dns_get_token(task, rule, prefix, token, continuation_cb, token_type)
+ -- local r = task:get_resolver()
+ -- In DNS we never ever use prefix as prefix, we use if as a suffix!
+ if token_type == 'ip' then
+ token = table.concat(token:inversed_str_octets(), '.')
+ end
+
+ local key = gen_token_key(nil, token, rule)
+ local dns_name = key .. '.' .. rule.backend.config.list
+
+ if prefix then
+ dns_name = string.format('%s.%s.%s', key, prefix,
+ rule.backend.config.list)
+ else
+ dns_name = string.format('%s.%s', key, rule.backend.config.list)
+ end
+
+ local function dns_cb(_, _, results, err)
+ if err and (err ~= 'requested record is not found' and
+ err ~= 'no records with this name') then
+ rspamd_logger.warnx(task, 'error looking up %s: %s', dns_name, err)
+ end
+
+ lua_util.debugm(N, task, 'DNS RESPONSE: label=%1 results=%2 err=%3 list=%4',
+ dns_name, results, err, rule.backend.config.list)
+
+ -- Now split tokens to list of values
+ if results and results[1] then
+ -- Format: num_messages;sc1;sc2...scn
+ local dns_tokens = lua_util.rspamd_str_split(results[1], ";")
+ -- Convert all to numbers excluding any possible non-numbers
+ dns_tokens = fun.totable(fun.filter(function(e)
+ return type(e) == 'number'
+ end,
+ fun.map(function(e)
+ local n = tonumber(e)
+ if n then
+ return n
+ end
+ return "BAD"
+ end, dns_tokens)))
+
+ if #dns_tokens < 2 then
+ rspamd_logger.warnx(task, 'cannot parse response for reputation token %s: %s',
+ dns_name, results[1])
+ continuation_cb(results, dns_name, nil)
+ else
+ local cnt = table.remove(dns_tokens, 1)
+ continuation_cb(nil, dns_name, { cnt, dns_tokens })
+ end
+ else
+ rspamd_logger.messagex(task, 'invalid response for reputation token %s: %s',
+ dns_name, results[1])
+ continuation_cb(results, dns_name, nil)
+ end
+ end
+
+ task:get_resolver():resolve_a({
+ task = task,
+ name = dns_name,
+ callback = dns_cb,
+ forced = true,
+ })
+end
+
+local function reputation_redis_init(rule, cfg, ev_base, worker)
+ local our_redis_params = {}
+
+ our_redis_params = lua_redis.try_load_redis_servers(rule.backend.config, rspamd_config,
+ true)
+ if not our_redis_params then
+ our_redis_params = redis_params
+ end
+ if not our_redis_params then
+ rspamd_logger.errx(rspamd_config, 'cannot init redis for reputation rule: %s',
+ rule)
+ return false
+ end
+ -- Init scripts for buckets
+ -- Redis script to extract data from Redis buckets
+ -- KEYS[1] - key to extract
+ -- Value returned - table of scores as a strings vector + number of scores
+ local redis_get_script_tpl = [[
+ local cnt = redis.call('HGET', KEYS[1], 'n')
+ local results = {}
+ if cnt then
+ {% for w in windows %}
+ local sc = tonumber(redis.call('HGET', KEYS[1], 'v' .. '{= w.name =}'))
+ table.insert(results, tostring(sc * {= w.mult =}))
+ {% endfor %}
+ else
+ {% for w in windows %}
+ table.insert(results, '0')
+ {% endfor %}
+ end
+
+ return {cnt or 0, results}
+ ]]
+
+ local get_script = lua_util.jinja_template(redis_get_script_tpl,
+ { windows = rule.backend.config.buckets })
+ rspamd_logger.debugm(N, rspamd_config, 'added extraction script %s', get_script)
+ rule.backend.script_get = lua_redis.add_redis_script(get_script, our_redis_params)
+
+ -- Redis script to update Redis buckets
+ -- KEYS[1] - key to update
+ -- KEYS[2] - current time in milliseconds
+ -- KEYS[3] - message score
+ -- KEYS[4] - expire for a bucket
+ -- Value returned - table of scores as a strings vector
+ local redis_adaptive_emea_script_tpl = [[
+ local last = redis.call('HGET', KEYS[1], 'l')
+ local score = tonumber(KEYS[3])
+ local now = tonumber(KEYS[2])
+ local scores = {}
+
+ if last then
+ {% for w in windows %}
+ local last_value = tonumber(redis.call('HGET', KEYS[1], 'v' .. '{= w.name =}'))
+ local window = {= w.time =}
+ -- Adjust alpha
+ local time_diff = now - last
+ if time_diff < 0 then
+ time_diff = 0
+ end
+ local alpha = 1.0 - math.exp((-time_diff) / (1000 * window))
+ local nscore = alpha * score + (1.0 - alpha) * last_value
+ table.insert(scores, tostring(nscore * {= w.mult =}))
+ {% endfor %}
+ else
+ {% for w in windows %}
+ table.insert(scores, tostring(score * {= w.mult =}))
+ {% endfor %}
+ end
+
+ local i = 1
+ {% for w in windows %}
+ redis.call('HSET', KEYS[1], 'v' .. '{= w.name =}', scores[i])
+ i = i + 1
+ {% endfor %}
+ redis.call('HSET', KEYS[1], 'l', now)
+ redis.call('HINCRBY', KEYS[1], 'n', 1)
+ redis.call('EXPIRE', KEYS[1], tonumber(KEYS[4]))
+
+ return scores
+]]
+
+ local set_script = lua_util.jinja_template(redis_adaptive_emea_script_tpl,
+ { windows = rule.backend.config.buckets })
+ rspamd_logger.debugm(N, rspamd_config, 'added emea update script %s', set_script)
+ rule.backend.script_set = lua_redis.add_redis_script(set_script, our_redis_params)
+
+ return true
+end
+
+local function reputation_redis_get_token(task, rule, prefix, token, continuation_cb, token_type)
+ if token_type and token_type == 'ip' then
+ token = tostring(token)
+ end
+ local key = gen_token_key(prefix, token, rule)
+
+ local function redis_get_cb(err, data)
+ if data then
+ if type(data) == 'table' then
+ lua_util.debugm(N, task, 'rule %s - got values for key %s -> %s',
+ rule['symbol'], key, data)
+ continuation_cb(nil, key, data)
+ else
+ rspamd_logger.errx(task, 'rule %s - invalid type while getting reputation keys %s: %s',
+ rule['symbol'], key, type(data))
+ continuation_cb("invalid type", key, nil)
+ end
+
+ elseif err then
+ rspamd_logger.errx(task, 'rule %s - got error while getting reputation keys %s: %s',
+ rule['symbol'], key, err)
+ continuation_cb(err, key, nil)
+ else
+ rspamd_logger.errx(task, 'rule %s - got error while getting reputation keys %s: %s',
+ rule['symbol'], key, "unknown error")
+ continuation_cb("unknown error", key, nil)
+ end
+ end
+
+ local ret = lua_redis.exec_redis_script(rule.backend.script_get,
+ { task = task, is_write = false },
+ redis_get_cb,
+ { key })
+ if not ret then
+ rspamd_logger.errx(task, 'cannot make redis request to check results')
+ end
+end
+
+local function reputation_redis_set_token(task, rule, prefix, token, sc, continuation_cb, token_type)
+ if token_type and token_type == 'ip' then
+ token = tostring(token)
+ end
+ local key = gen_token_key(prefix, token, rule)
+
+ local function redis_set_cb(err, data)
+ if err then
+ rspamd_logger.errx(task, 'rule %s - got error while setting reputation keys %s: %s',
+ rule['symbol'], key, err)
+ if continuation_cb then
+ continuation_cb(err, key)
+ end
+ else
+ if continuation_cb then
+ continuation_cb(nil, key)
+ end
+ end
+ end
+
+ lua_util.debugm(N, task, 'rule %s - set values for key %s -> %s',
+ rule['symbol'], key, sc)
+ local ret = lua_redis.exec_redis_script(rule.backend.script_set,
+ { task = task, is_write = true },
+ redis_set_cb,
+ { key, tostring(os.time() * 1000),
+ tostring(sc),
+ tostring(rule.backend.config.expiry) })
+ if not ret then
+ rspamd_logger.errx(task, 'got error while connecting to redis')
+ end
+end
+
+--[[ Backends are responsible for getting reputation tokens
+ -- Common config options:
+ -- `hashed`: if `true` then apply hash function to the key
+ -- `hash_alg`: use specific hash type (`blake2` by default)
+ -- `hash_len`: strip hash to this amount of bytes (no strip by default)
+ -- `hash_encoding`: use specific hash encoding (base32 by default)
+--]]
+local backends = {
+ redis = {
+ schema = lua_redis.enrich_schema({
+ prefix = ts.string:is_optional(),
+ expiry = (ts.number + ts.string / lua_util.parse_time_interval):is_optional(),
+ buckets = ts.array_of(ts.shape {
+ time = ts.number + ts.string / lua_util.parse_time_interval,
+ name = ts.string,
+ mult = ts.number + ts.string / tonumber
+ }) :is_optional(),
+ }),
+ config = {
+ expiry = default_expiry,
+ prefix = default_prefix,
+ buckets = {
+ {
+ time = 60 * 60 * 24 * 30,
+ name = '1m',
+ mult = 1.0,
+ }
+ }, -- What buckets should be used, default 1h and 1month
+ },
+ init = reputation_redis_init,
+ get_token = reputation_redis_get_token,
+ set_token = reputation_redis_set_token,
+ },
+ dns = {
+ schema = ts.shape {
+ list = ts.string,
+ },
+ config = {
+ -- list = rep.example.com
+ },
+ get_token = reputation_dns_get_token,
+ -- No set token for DNS
+ init = reputation_dns_init,
+ }
+}
+
+local function is_rule_applicable(task, rule)
+ local ip = task:get_from_ip()
+ if not (rule.selector.config.outbound and rule.selector.config.inbound) then
+ if rule.selector.config.outbound then
+ if not (task:get_user() or (ip and ip:is_local())) then
+ return false
+ end
+ elseif rule.selector.config.inbound then
+ if task:get_user() or (ip and ip:is_local()) then
+ return false
+ end
+ end
+ end
+
+ if rule.config.whitelist_map then
+ if rule.config.whitelist_map:process(task) then
+ return false
+ end
+ end
+
+ return true
+end
+
+local function reputation_filter_cb(task, rule)
+ if (is_rule_applicable(task, rule)) then
+ rule.selector.filter(task, rule, rule.backend)
+ end
+end
+
+local function reputation_postfilter_cb(task, rule)
+ if (is_rule_applicable(task, rule)) then
+ rule.selector.postfilter(task, rule, rule.backend)
+ end
+end
+
+local function reputation_idempotent_cb(task, rule)
+ if (is_rule_applicable(task, rule)) then
+ rule.selector.idempotent(task, rule, rule.backend)
+ end
+end
+
+local function callback_gen(cb, rule)
+ return function(task)
+ if rule.enabled then
+ cb(task, rule)
+ end
+ end
+end
+
+local function parse_rule(name, tbl)
+ local sel_type, sel_conf = fun.head(tbl.selector)
+ local selector = selectors[sel_type]
+
+ if not selector then
+ rspamd_logger.errx(rspamd_config, "unknown selector defined for rule %s: %s", name,
+ sel_type)
+ return false
+ end
+
+ local bk_type, bk_conf = fun.head(tbl.backend)
+
+ local backend = backends[bk_type]
+ if not backend then
+ rspamd_logger.errx(rspamd_config, "unknown backend defined for rule %s: %s", name,
+ tbl.backend.type)
+ return false
+ end
+ -- Allow config override
+ local rule = {
+ selector = lua_util.shallowcopy(selector),
+ backend = lua_util.shallowcopy(backend),
+ config = {}
+ }
+
+ -- Override default config params
+ rule.backend.config = lua_util.override_defaults(rule.backend.config, bk_conf)
+ if backend.schema then
+ local checked, schema_err = backend.schema:transform(rule.backend.config)
+ if not checked then
+ rspamd_logger.errx(rspamd_config, "cannot parse backend config for %s: %s",
+ sel_type, schema_err)
+
+ return false
+ end
+
+ rule.backend.config = checked
+ end
+
+ rule.selector.config = lua_util.override_defaults(rule.selector.config, sel_conf)
+ if selector.schema then
+ local checked, schema_err = selector.schema:transform(rule.selector.config)
+
+ if not checked then
+ rspamd_logger.errx(rspamd_config, "cannot parse selector config for %s: %s (%s)",
+ sel_type,
+ schema_err, sel_conf)
+ return
+ end
+
+ rule.selector.config = checked
+ end
+ -- Generic options
+ tbl.selector = nil
+ tbl.backend = nil
+ rule.config = lua_util.override_defaults(rule.config, tbl)
+
+ if rule.config.whitelist then
+ if lua_maps_exprs.schema(rule.config.whitelist) then
+ rule.config.whitelist_map = lua_maps_exprs.create(rspamd_config,
+ rule.config.whitelist, N)
+ elseif lua_maps.map_schema(rule.config.whitelist) then
+ local map = lua_maps.map_add_from_ucl(rule.config.whitelist,
+ 'radix',
+ sel_type .. ' reputation whitelist')
+
+ if not map then
+ rspamd_logger.errx(rspamd_config, "cannot parse whitelist map config for %s: (%s)",
+ sel_type,
+ rule.config.whitelist)
+ return
+ end
+
+ rule.config.whitelist_map = {
+ process = function(_, task)
+ -- Hack: we assume that it is an ip whitelist :(
+ local ip = task:get_from_ip()
+
+ if ip and map:get_key(ip) then
+ return true
+ end
+ return false
+ end
+ }
+ else
+ rspamd_logger.errx(rspamd_config, "cannot parse whitelist map config for %s: (%s)",
+ sel_type,
+ rule.config.whitelist)
+ return false
+ end
+ end
+
+ local symbol = rule.selector.config.symbol or name
+ if tbl.symbol then
+ symbol = tbl.symbol
+ end
+
+ rule.symbol = symbol
+ rule.enabled = true
+ if rule.selector.init then
+ rule.enabled = false
+ end
+ if rule.backend.init then
+ rule.enabled = false
+ end
+ -- Perform additional initialization if needed
+ rspamd_config:add_on_load(function(cfg, ev_base, worker)
+ if rule.selector.init then
+ if not rule.selector.init(rule, cfg, ev_base, worker) then
+ rule.enabled = false
+ rspamd_logger.errx(rspamd_config, 'Cannot init selector %s (backend %s) for symbol %s',
+ sel_type, bk_type, rule.symbol)
+ else
+ rule.enabled = true
+ end
+ end
+ if rule.backend.init then
+ if not rule.backend.init(rule, cfg, ev_base, worker) then
+ rule.enabled = false
+ rspamd_logger.errx(rspamd_config, 'Cannot init backend (%s) for rule %s for symbol %s',
+ bk_type, sel_type, rule.symbol)
+ else
+ rule.enabled = true
+ end
+ end
+
+ if rule.enabled then
+ rspamd_logger.infox(rspamd_config, 'Enable %s (%s backend) rule for symbol %s (split symbols: %s)',
+ sel_type, bk_type, rule.symbol,
+ rule.selector.config.split_symbols)
+ end
+ end)
+
+ -- We now generate symbol for checking
+ local rule_type = 'normal'
+ if rule.selector.config.split_symbols then
+ rule_type = 'callback'
+ end
+
+ local id = rspamd_config:register_symbol {
+ name = rule.symbol,
+ type = rule_type,
+ callback = callback_gen(reputation_filter_cb, rule),
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
+ }
+
+ if rule.selector.config.split_symbols then
+ rspamd_config:register_symbol {
+ name = rule.symbol .. '_HAM',
+ type = 'virtual',
+ parent = id,
+ }
+ rspamd_config:register_symbol {
+ name = rule.symbol .. '_SPAM',
+ type = 'virtual',
+ parent = id,
+ }
+ end
+
+ if rule.selector.dependencies then
+ fun.each(function(d)
+ rspamd_config:register_dependency(symbol, d)
+ end, rule.selector.dependencies)
+ end
+
+ if rule.selector.postfilter then
+ -- Also register a postfilter
+ rspamd_config:register_symbol {
+ name = rule.symbol .. '_POST',
+ type = 'postfilter',
+ flags = 'nostat,explicit_disable,ignore_passthrough',
+ callback = callback_gen(reputation_postfilter_cb, rule),
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
+ }
+ end
+
+ if rule.selector.idempotent then
+ -- Has also idempotent component (e.g. saving data to the backend)
+ rspamd_config:register_symbol {
+ name = rule.symbol .. '_IDEMPOTENT',
+ type = 'idempotent',
+ flags = 'explicit_disable,ignore_passthrough',
+ callback = callback_gen(reputation_idempotent_cb, rule),
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
+ }
+ end
+
+ return true
+end
+
+redis_params = lua_redis.parse_redis_server('reputation')
+local opts = rspamd_config:get_all_opt("reputation")
+
+-- Initialization part
+if not (opts and type(opts) == 'table') then
+ rspamd_logger.infox(rspamd_config, 'Module is not configured, disabling it')
+ return
+end
+
+if opts['rules'] then
+ for k, v in pairs(opts['rules']) do
+ if not ((v or E).selector) then
+ rspamd_logger.errx(rspamd_config, "no selector defined for rule %s", k)
+ lua_util.config_utils.push_config_error(N, "no selector defined for rule: " .. k)
+ else
+ if not parse_rule(k, v) then
+ lua_util.config_utils.push_config_error(N, "reputation rule is misconfigured: " .. k)
+ end
+ end
+ end
+else
+ lua_util.disable_module(N, "config")
+end
diff --git a/src/plugins/lua/rspamd_update.lua b/src/plugins/lua/rspamd_update.lua
new file mode 100644
index 0000000..deda038
--- /dev/null
+++ b/src/plugins/lua/rspamd_update.lua
@@ -0,0 +1,161 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- This plugin implements dynamic updates for rspamd
+
+local ucl = require "ucl"
+local fun = require "fun"
+local rspamd_logger = require "rspamd_logger"
+local rspamd_config = rspamd_config
+local hash = require "rspamd_cryptobox_hash"
+local lua_util = require "lua_util"
+local N = "rspamd_update"
+local rspamd_version = rspamd_version
+local maps = {}
+local allow_rules = false -- Deny for now
+local global_priority = 1 -- Default for local rules
+
+local function process_symbols(obj, priority)
+ fun.each(function(sym, score)
+ rspamd_config:set_metric_symbol({
+ name = sym,
+ score = score,
+ priority = priority
+ })
+ end, obj)
+end
+
+local function process_actions(obj, priority)
+ fun.each(function(act, score)
+ rspamd_config:set_metric_action({
+ action = act,
+ score = score,
+ priority = priority
+ })
+ end, obj)
+end
+
+local function process_rules(obj)
+ fun.each(function(key, code)
+ local f = load(code)
+ if f then
+ f()
+ else
+ rspamd_logger(rspamd_config, 'cannot load rules for %s', key)
+ end
+ end, obj)
+end
+
+local function check_version(obj)
+ local ret = true
+
+ if not obj then
+ return false
+ end
+
+ if obj['min_version'] then
+ if rspamd_version('cmp', obj['min_version']) > 0 then
+ ret = false
+ rspamd_logger.errx(rspamd_config, 'updates require at least %s version of rspamd',
+ obj['min_version'])
+ end
+ end
+ if obj['max_version'] then
+ if rspamd_version('cmp', obj['max_version']) < 0 then
+ ret = false
+ rspamd_logger.errx(rspamd_config, 'updates require maximum %s version of rspamd',
+ obj['max_version'])
+ end
+ end
+
+ return ret
+end
+
+local function gen_callback()
+
+ return function(data)
+ local parser = ucl.parser()
+ local res, err = parser:parse_string(data)
+
+ if not res then
+ rspamd_logger.warnx(rspamd_config, 'cannot parse updates map: ' .. err)
+ else
+ local h = hash.create()
+ h:update(data)
+ local obj = parser:get_object()
+
+ if check_version(obj) then
+
+ if obj['symbols'] then
+ process_symbols(obj['symbols'], global_priority)
+ end
+ if obj['actions'] then
+ process_actions(obj['actions'], global_priority)
+ end
+ if allow_rules and obj['rules'] then
+ process_rules(obj['rules'])
+ end
+
+ rspamd_logger.infox(rspamd_config, 'loaded new rules with hash "%s"',
+ h:hex())
+ end
+ end
+
+ return res
+ end
+end
+
+-- Configuration part
+local section = rspamd_config:get_all_opt("rspamd_update")
+if section and section.rules then
+ local trusted_key
+ if section.key then
+ trusted_key = section.key
+ end
+
+ if type(section.rules) ~= 'table' then
+ section.rules = { section.rules }
+ end
+
+ fun.each(function(elt)
+ local map = rspamd_config:add_map(elt, "rspamd updates map", nil, "callback")
+ if not map then
+ rspamd_logger.errx(rspamd_config, 'cannot load updates from %1', elt)
+ else
+ map:set_callback(gen_callback(map))
+ maps['elt'] = map
+ end
+ end, section.rules)
+
+ fun.each(function(k, map)
+ -- Check sanity for maps
+ local proto = map:get_proto()
+ if (proto == 'http' or proto == 'https') and not map:get_sign_key() then
+ if trusted_key then
+ map:set_sign_key(trusted_key)
+ else
+ rspamd_logger.warnx(rspamd_config, 'Map %s is loaded by HTTP and it is not signed', k)
+ end
+ end
+ end, maps)
+else
+ rspamd_logger.infox(rspamd_config, 'Module is unconfigured')
+ lua_util.disable_module(N, "config")
+end
diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua
new file mode 100644
index 0000000..69d31d3
--- /dev/null
+++ b/src/plugins/lua/settings.lua
@@ -0,0 +1,1437 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- This plugin implements user dynamic settings
+-- Settings documentation can be found here:
+-- https://rspamd.com/doc/configuration/settings.html
+
+local rspamd_logger = require "rspamd_logger"
+local lua_maps = require "lua_maps"
+local lua_util = require "lua_util"
+local rspamd_ip = require "rspamd_ip"
+local rspamd_regexp = require "rspamd_regexp"
+local lua_selectors = require "lua_selectors"
+local lua_settings = require "lua_settings"
+local ucl = require "ucl"
+local fun = require "fun"
+local rspamd_mempool = require "rspamd_mempool"
+
+local redis_params
+
+local settings = {}
+local N = "settings"
+local settings_initialized = false
+local max_pri = 0
+local module_sym_id -- Main module symbol
+
+local function apply_settings(task, to_apply, id, name)
+ local cached_name = task:cache_get('settings_name')
+ if cached_name then
+ local cached_settings = task:cache_get('settings')
+ rspamd_logger.warnx(task, "cannot apply settings rule %s (id=%s):" ..
+ " settings has been already applied by rule %s (id=%s)",
+ name, id, cached_name, cached_settings.id)
+ return false
+ end
+
+ task:set_settings(to_apply)
+ task:cache_set('settings', to_apply)
+ task:cache_set('settings_name', name or 'unknown')
+
+ if id then
+ task:set_settings_id(id)
+ end
+
+ if to_apply['add_headers'] or to_apply['remove_headers'] then
+ local rep = {
+ add_headers = to_apply['add_headers'] or {},
+ remove_headers = to_apply['remove_headers'] or {},
+ }
+ task:set_rmilter_reply(rep)
+ end
+
+ if to_apply.flags and type(to_apply.flags) == 'table' then
+ for _, fl in ipairs(to_apply.flags) do
+ task:set_flag(fl)
+ end
+ end
+
+ if to_apply.symbols then
+ -- Add symbols, specified in the settings
+ if #to_apply.symbols > 0 then
+ -- Array like symbols
+ for _, val in ipairs(to_apply.symbols) do
+ task:insert_result(val, 1.0)
+ end
+ else
+ -- Object like symbols
+ for k, v in pairs(to_apply.symbols) do
+ if type(v) == 'table' then
+ task:insert_result(k, v.score or 1.0, v.options or {})
+ elseif tonumber(v) then
+ task:insert_result(k, tonumber(v))
+ end
+ end
+ end
+ end
+
+ if to_apply.subject then
+ task:set_metric_subject(to_apply.subject)
+ end
+
+ -- E.g.
+ -- messages = { smtp_message = "5.3.1 Go away" }
+ if to_apply.messages and type(to_apply.messages) == 'table' then
+ fun.each(function(category, message)
+ task:append_message(message, category)
+ end, to_apply.messages)
+ end
+
+ return true
+end
+
+-- Checks for overridden settings within query params and returns 3 values:
+-- * Apply element
+-- * Settings ID element if found
+-- * Priority of the settings according to the place where it is found
+--
+-- If no override has been found, it returns `false`
+local function check_query_settings(task)
+ -- Try 'settings' attribute
+ local settings_id = task:get_settings_id()
+ local query_set = task:get_request_header('settings')
+ if query_set then
+
+ local parser = ucl.parser()
+ local res, err = parser:parse_text(query_set)
+ if res then
+ if settings_id then
+ rspamd_logger.warnx(task, "both settings-id '%s' and settings headers are presented, ignore settings-id; ",
+ tostring(settings_id))
+ end
+ local settings_obj = parser:get_object()
+
+ -- Treat as low priority
+ return settings_obj, nil, 1
+ else
+ rspamd_logger.errx(task, 'Parse error: %s', err)
+ end
+ end
+
+ local query_maxscore = task:get_request_header('maxscore')
+ local nset
+
+ if query_maxscore then
+ if settings_id then
+ rspamd_logger.infox(task, "both settings id '%s' and maxscore '%s' headers are presented, merge them; " ..
+ "settings id has priority",
+ tostring(settings_id), tostring(query_maxscore))
+ end
+ -- We have score limits redefined by request
+ local ms = tonumber(tostring(query_maxscore))
+ if ms then
+ nset = {
+ actions = {
+ reject = ms
+ }
+ }
+
+ local query_softscore = task:get_request_header('softscore')
+ if query_softscore then
+ local ss = tonumber(tostring(query_softscore))
+ nset.actions['add header'] = ss
+ end
+
+ if not settings_id then
+ rspamd_logger.infox(task, 'apply maxscore = %s', nset.actions)
+ -- Maxscore is low priority
+ return nset, nil, 1
+ end
+ end
+ end
+
+ if settings_id and settings_initialized then
+ local cached = lua_settings.settings_by_id(settings_id)
+ lua_util.debugm(N, task, "check settings id for %s", settings_id)
+
+ if cached then
+ local elt = cached.settings
+ if elt['whitelist'] then
+ elt['apply'] = { whitelist = true }
+ end
+
+ if elt.apply then
+ if nset then
+ elt.apply = lua_util.override_defaults(nset, elt.apply)
+ end
+ end
+ return elt.apply, cached, cached.priority or 1
+ else
+ rspamd_logger.warnx(task, 'no settings id "%s" has been found', settings_id)
+ if nset then
+ rspamd_logger.infox(task, 'apply maxscore = %s', nset.actions)
+ return nset, nil, 1
+ end
+ end
+ else
+ if nset then
+ rspamd_logger.infox(task, 'apply maxscore = %s', nset.actions)
+ return nset, nil, 1
+ end
+ end
+
+ return false
+end
+
+local function check_addr_setting(expected, addr)
+ local function check_specific_addr(elt)
+ if expected.name then
+ if lua_maps.rspamd_maybe_check_map(expected.name, elt.addr) then
+ return true
+ end
+ end
+ if expected.user then
+ if lua_maps.rspamd_maybe_check_map(expected.user, elt.user) then
+ return true
+ end
+ end
+ if expected.domain and elt.domain then
+ if lua_maps.rspamd_maybe_check_map(expected.domain, elt.domain) then
+ return true
+ end
+ end
+ if expected.regexp then
+ if expected.regexp:match(elt.addr) then
+ return true
+ end
+ end
+ return false
+ end
+
+ for _, e in ipairs(addr) do
+ if check_specific_addr(e) then
+ return true
+ end
+ end
+
+ return false
+end
+
+local function check_string_setting(expected, str)
+ if expected.regexp then
+ if expected.regexp:match(str) then
+ return true
+ end
+ elseif expected.check then
+ if lua_maps.rspamd_maybe_check_map(expected.check, str) then
+ return true
+ end
+ end
+ return false
+end
+
+local function check_ip_setting(expected, ip)
+ if not expected[2] then
+ if lua_maps.rspamd_maybe_check_map(expected[1], ip:to_string()) then
+ return true
+ end
+ else
+ if expected[2] ~= 0 then
+ local nip = ip:apply_mask(expected[2])
+ if nip and nip:to_string() == expected[1] then
+ return true
+ end
+ elseif ip:to_string() == expected[1] then
+ return true
+ end
+ end
+
+ return false
+end
+
+local function check_map_setting(map, input)
+ return map:get_key(input)
+end
+
+local function priority_to_string(pri)
+ if pri then
+ if pri >= 3 then
+ return "high"
+ elseif pri >= 2 then
+ return "medium"
+ end
+ end
+
+ return "low"
+end
+
+-- Check limit for a task
+local function check_settings(task)
+ local function check_specific_setting(rule, matched)
+ local function process_atom(atom)
+ local elt = rule.checks[atom]
+
+ if elt then
+ local input = elt.extract(task)
+ if not input then
+ return false
+ end
+
+ if elt.check(input) then
+ matched[#matched + 1] = atom
+ return 1.0
+ end
+ else
+ rspamd_logger.errx(task, 'error in settings: check %s is not defined!', atom)
+ end
+
+ return 0
+ end
+
+ local res = rule.expression and rule.expression:process(process_atom) or rule.implicit
+
+ if res and res > 0 then
+ if rule['whitelist'] then
+ rule['apply'] = { whitelist = true }
+ end
+
+ return rule
+ end
+
+ return nil
+ end
+
+ -- Check if we have override as query argument
+ local query_apply, id_elt, priority = check_query_settings(task)
+
+ local function maybe_apply_query_settings()
+ if query_apply then
+ if id_elt then
+ apply_settings(task, query_apply, id_elt.id, id_elt.name)
+ rspamd_logger.infox(task, "applied settings id %s(%s); priority %s",
+ id_elt.name, id_elt.id, priority_to_string(priority))
+ else
+ apply_settings(task, query_apply, nil, 'HTTP query')
+ rspamd_logger.infox(task, "applied settings from query; priority %s",
+ priority_to_string(priority))
+ end
+ end
+ end
+
+ local min_pri = 1
+ if query_apply then
+ if priority >= min_pri then
+ -- Do not check lower or equal priorities
+ min_pri = priority + 1
+ end
+
+ if priority > max_pri then
+ -- Our internal priorities are lower then a priority from query, so no need to check
+ maybe_apply_query_settings()
+
+ return
+ end
+ elseif id_elt and type(id_elt.settings) == 'table' and id_elt.settings.external_map then
+ local external_map = id_elt.settings.external_map
+ local selector_result = external_map.selector(task)
+
+ if selector_result then
+ external_map.map:get_key(selector_result, nil, task)
+ -- No more selection logic
+ return
+ else
+ rspamd_logger.infox("cannot query selector to make external map request")
+ end
+ end
+
+ -- Do not waste resources
+ if not settings_initialized then
+ maybe_apply_query_settings()
+ return
+ end
+
+ -- Match rules according their order
+ local applied = false
+
+ for pri = max_pri, min_pri, -1 do
+ if not applied and settings[pri] then
+ for _, s in ipairs(settings[pri]) do
+ local matched = {}
+
+ local result = check_specific_setting(s.rule, matched)
+ lua_util.debugm(N, task, "check for settings element %s; result = %s",
+ s.name, result)
+ -- Can use xor here but more complicated for reading
+ if result then
+ if s.rule.apply then
+ if s.rule.id then
+ -- Extract static settings
+ local cached = lua_settings.settings_by_id(s.rule.id)
+
+ if not cached or not cached.settings or not cached.settings.apply then
+ rspamd_logger.errx(task, 'unregistered settings id found: %s!', s.rule.id)
+ else
+ rspamd_logger.infox(task, "<%s> apply static settings %s (id = %s); %s matched; priority %s",
+ task:get_message_id(),
+ cached.name, s.rule.id,
+ table.concat(matched, ','),
+ priority_to_string(pri))
+ apply_settings(task, cached.settings.apply, s.rule.id, s.name)
+ end
+
+ else
+ -- Dynamic settings
+ rspamd_logger.infox(task, "<%s> apply settings according to rule %s (%s matched)",
+ task:get_message_id(), s.name, table.concat(matched, ','))
+ apply_settings(task, s.rule.apply, nil, s.name)
+ end
+
+ applied = true
+ elseif s.rule.external_map then
+ local external_map = s.rule.external_map
+ local selector_result = external_map.selector(task)
+
+ if selector_result then
+ external_map.map:get_key(selector_result, nil, task)
+ -- No more selection logic
+ return
+ else
+ rspamd_logger.infox("cannot query selector to make external map request")
+ end
+ end
+ if s.rule['symbols'] then
+ -- Add symbols, specified in the settings
+ fun.each(function(val)
+ task:insert_result(val, 1.0)
+ end, s.rule['symbols'])
+ end
+ end
+ end
+ end
+ end
+
+ if not applied then
+ maybe_apply_query_settings()
+ end
+
+end
+
+local function convert_to_table(chk_elt, out)
+ if type(chk_elt) == 'string' then
+ return { out }
+ end
+
+ return out
+end
+
+local function gen_settings_external_cb(name)
+ return function(result, err_or_data, code, task)
+ if result then
+ local parser = ucl.parser()
+
+ local res, ucl_err = parser:parse_text(err_or_data)
+ if not res then
+ rspamd_logger.warnx(task, 'cannot parse settings from the external map %s: %s',
+ name, ucl_err)
+ else
+ local obj = parser:get_object()
+ rspamd_logger.infox(task, "<%s> apply settings according to the external map %s",
+ name, task:get_message_id())
+ apply_settings(task, obj, nil, 'external_map')
+ end
+ else
+ rspamd_logger.infox(task, "<%s> no settings returned from the external map %s: %s (code = %s)",
+ task:get_message_id(), name, err_or_data, code)
+ end
+ end
+end
+
+-- Process IP address: converted to a table {ip, mask}
+local function process_ip_condition(ip)
+ local out = {}
+
+ if type(ip) == "table" then
+ for _, v in ipairs(ip) do
+ table.insert(out, process_ip_condition(v))
+ end
+ elseif type(ip) == "string" then
+ local slash = string.find(ip, '/')
+
+ if not slash then
+ -- Just a plain IP address
+ local res = rspamd_ip.from_string(ip)
+
+ if res:is_valid() then
+ out[1] = res:to_string()
+ out[2] = 0
+ else
+ -- It can still be a map
+ out[1] = ip
+ end
+ else
+ local res = rspamd_ip.from_string(string.sub(ip, 1, slash - 1))
+ local mask = tonumber(string.sub(ip, slash + 1))
+
+ if res:is_valid() then
+ out[1] = res:to_string()
+ out[2] = mask
+ else
+ rspamd_logger.errx(rspamd_config, "bad IP address: " .. ip)
+ return nil
+ end
+ end
+ else
+ return nil
+ end
+
+ return out
+end
+
+-- Process email like condition, converted to a table with fields:
+-- name - full email (surprise!)
+-- user - user part
+-- domain - domain part
+-- regexp - full email regexp (yes, it sucks)
+local function process_email_condition(addr)
+ local out = {}
+ if type(addr) == "table" then
+ for _, v in ipairs(addr) do
+ table.insert(out, process_email_condition(v))
+ end
+ elseif type(addr) == "string" then
+ if string.sub(addr, 1, 4) == "map:" then
+ -- It is map, don't apply any extra logic
+ out['name'] = addr
+ else
+ local start = string.sub(addr, 1, 1)
+ if start == '/' then
+ -- It is a regexp
+ local re = rspamd_regexp.create(addr)
+ if re then
+ out['regexp'] = re
+ else
+ rspamd_logger.errx(rspamd_config, "bad regexp: " .. addr)
+ return nil
+ end
+
+ elseif start == '@' then
+ -- It is a domain if form @domain
+ out['domain'] = string.sub(addr, 2)
+ else
+ -- Check user@domain parts
+ local at = string.find(addr, '@')
+ if at then
+ -- It is full address
+ out['name'] = addr
+ else
+ -- It is a user
+ out['user'] = addr
+ end
+ end
+ end
+ else
+ return nil
+ end
+
+ return out
+end
+
+-- Convert a plain string condition to a table:
+-- check - string to match
+-- regexp - regexp to match
+local function process_string_condition(addr)
+ local out = {}
+ if type(addr) == "table" then
+ for _, v in ipairs(addr) do
+ table.insert(out, process_string_condition(v))
+ end
+ elseif type(addr) == "string" then
+ if string.sub(addr, 1, 4) == "map:" then
+ -- It is map, don't apply any extra logic
+ out['check'] = addr
+ else
+ local start = string.sub(addr, 1, 1)
+ if start == '/' then
+ -- It is a regexp
+ local re = rspamd_regexp.create(addr)
+ if re then
+ out['regexp'] = re
+ else
+ rspamd_logger.errx(rspamd_config, "bad regexp: " .. addr)
+ return nil
+ end
+
+ else
+ out['check'] = addr
+ end
+ end
+ else
+ return nil
+ end
+
+ return out
+end
+
+local function get_priority (elt)
+ local pri_tonum = function(p)
+ if p then
+ if type(p) == "number" then
+ return tonumber(p)
+ elseif type(p) == "string" then
+ if p == "high" then
+ return 3
+ elseif p == "medium" then
+ return 2
+ end
+
+ end
+
+ end
+
+ return 1
+ end
+
+ return pri_tonum(elt['priority'])
+end
+
+-- Used to create a checking closure: if value matches expected somehow, return true
+local function gen_check_closure(expected, check_func)
+ return function(value)
+ if not value then
+ return false
+ end
+
+ if type(value) == 'function' then
+ value = value()
+ end
+
+ if value then
+
+ if not check_func then
+ check_func = function(a, b)
+ return a == b
+ end
+ end
+
+ local ret
+ if type(expected) == 'table' then
+ ret = fun.any(function(d)
+ return check_func(d, value)
+ end, expected)
+ else
+ ret = check_func(expected, value)
+ end
+ if ret then
+ return true
+ end
+ end
+
+ return false
+ end
+end
+
+-- Process settings based on their priority
+local function process_settings_table(tbl, allow_ids, mempool, is_static)
+
+ -- Check the setting element internal data
+ local process_setting_elt = function(name, elt)
+
+ lua_util.debugm(N, rspamd_config, 'process settings "%s"', name)
+
+ local out = {}
+
+ local checks = {}
+ if elt.ip then
+ local ips_table = process_ip_condition(elt['ip'])
+
+ if ips_table then
+ lua_util.debugm(N, rspamd_config, 'added ip condition to "%s": %s',
+ name, ips_table)
+ checks.ip = {
+ check = gen_check_closure(convert_to_table(elt.ip, ips_table), check_ip_setting),
+ extract = function(task)
+ local ip = task:get_from_ip()
+ if ip and ip:is_valid() then
+ return ip
+ end
+ return nil
+ end,
+ }
+ end
+ end
+ if elt.ip_map then
+ local ips_map = lua_maps.map_add_from_ucl(elt.ip_map, 'radix',
+ 'settings ip map for ' .. name)
+
+ if ips_map then
+ lua_util.debugm(N, rspamd_config, 'added ip_map condition to "%s"',
+ name)
+ checks.ip_map = {
+ check = gen_check_closure(ips_map, check_map_setting),
+ extract = function(task)
+ local ip = task:get_from_ip()
+ if ip and ip:is_valid() then
+ return ip
+ end
+ return nil
+ end,
+ }
+ end
+ end
+
+ if elt.client_ip then
+ local client_ips_table = process_ip_condition(elt.client_ip)
+
+ if client_ips_table then
+ lua_util.debugm(N, rspamd_config, 'added client_ip condition to "%s": %s',
+ name, client_ips_table)
+ checks.client_ip = {
+ check = gen_check_closure(convert_to_table(elt.client_ip, client_ips_table),
+ check_ip_setting),
+ extract = function(task)
+ local ip = task:get_client_ip()
+ if ip:is_valid() then
+ return ip
+ end
+ return nil
+ end,
+ }
+ end
+ end
+ if elt.client_ip_map then
+ local ips_map = lua_maps.map_add_from_ucl(elt.ip_map, 'radix',
+ 'settings client ip map for ' .. name)
+
+ if ips_map then
+ lua_util.debugm(N, rspamd_config, 'added client ip_map condition to "%s"',
+ name)
+ checks.client_ip_map = {
+ check = gen_check_closure(ips_map, check_map_setting),
+ extract = function(task)
+ local ip = task:get_client_ip()
+ if ip and ip:is_valid() then
+ return ip
+ end
+ return nil
+ end,
+ }
+ end
+ end
+
+ if elt.from then
+ local from_condition = process_email_condition(elt.from)
+
+ if from_condition then
+ lua_util.debugm(N, rspamd_config, 'added from condition to "%s": %s',
+ name, from_condition)
+ checks.from = {
+ check = gen_check_closure(convert_to_table(elt.from, from_condition),
+ check_addr_setting),
+ extract = function(task)
+ return task:get_from(1)
+ end,
+ }
+ end
+ end
+
+ if elt.rcpt then
+ local rcpt_condition = process_email_condition(elt.rcpt)
+ if rcpt_condition then
+ lua_util.debugm(N, rspamd_config, 'added rcpt condition to "%s": %s',
+ name, rcpt_condition)
+ checks.rcpt = {
+ check = gen_check_closure(convert_to_table(elt.rcpt, rcpt_condition),
+ check_addr_setting),
+ extract = function(task)
+ return task:get_recipients(1)
+ end,
+ }
+ end
+ end
+
+ if elt.from_mime then
+ local from_mime_condition = process_email_condition(elt.from_mime)
+
+ if from_mime_condition then
+ lua_util.debugm(N, rspamd_config, 'added from_mime condition to "%s": %s',
+ name, from_mime_condition)
+ checks.from_mime = {
+ check = gen_check_closure(convert_to_table(elt.from_mime, from_mime_condition),
+ check_addr_setting),
+ extract = function(task)
+ return task:get_from(2)
+ end,
+ }
+ end
+ end
+
+ if elt.rcpt_mime then
+ local rcpt_mime_condition = process_email_condition(elt.rcpt_mime)
+ if rcpt_mime_condition then
+ lua_util.debugm(N, rspamd_config, 'added rcpt mime condition to "%s": %s',
+ name, rcpt_mime_condition)
+ checks.rcpt_mime = {
+ check = gen_check_closure(convert_to_table(elt.rcpt_mime, rcpt_mime_condition),
+ check_addr_setting),
+ extract = function(task)
+ return task:get_recipients(2)
+ end,
+ }
+ end
+ end
+
+ if elt.user then
+ local user_condition = process_email_condition(elt.user)
+ if user_condition then
+ lua_util.debugm(N, rspamd_config, 'added user condition to "%s": %s',
+ name, user_condition)
+ checks.user = {
+ check = gen_check_closure(convert_to_table(elt.user, user_condition),
+ check_addr_setting),
+ extract = function(task)
+ local uname = task:get_user()
+ local user = {}
+ if uname then
+ user[1] = {}
+ local localpart, domainpart = string.gmatch(uname, "(.+)@(.+)")()
+ if localpart then
+ user[1]["user"] = localpart
+ user[1]["domain"] = domainpart
+ user[1]["addr"] = uname
+ else
+ user[1]["user"] = uname
+ user[1]["addr"] = uname
+ end
+
+ return user
+ end
+
+ return nil
+ end,
+ }
+ end
+ end
+
+ if elt.hostname then
+ local hostname_condition = process_string_condition(elt.hostname)
+ if hostname_condition then
+ lua_util.debugm(N, rspamd_config, 'added hostname condition to "%s": %s',
+ name, hostname_condition)
+ checks.hostname = {
+ check = gen_check_closure(convert_to_table(elt.hostname, hostname_condition),
+ check_string_setting),
+ extract = function(task)
+ return task:get_hostname() or ''
+ end,
+ }
+ end
+ end
+
+ if elt.authenticated then
+ lua_util.debugm(N, rspamd_config, 'added authenticated condition to "%s"',
+ name)
+ checks.authenticated = {
+ check = function(value)
+ if value then
+ return true
+ end
+ return false
+ end,
+ extract = function(task)
+ return task:get_user()
+ end
+ }
+ end
+
+ if elt['local'] then
+ lua_util.debugm(N, rspamd_config, 'added local condition to "%s"',
+ name)
+ checks['local'] = {
+ check = function(value)
+ if value then
+ return true
+ end
+ return false
+ end,
+ extract = function(task)
+ local ip = task:get_from_ip()
+ if not ip or not ip:is_valid() then
+ return nil
+ end
+
+ if ip:is_local() then
+ return true
+ else
+ return nil
+ end
+ end
+ }
+ end
+
+ local aliases = {}
+ -- This function is used to convert compound condition with
+ -- generic type and specific part (e.g. `header`, `Content-Transfer-Encoding`)
+ -- to a set of usable check elements:
+ -- `generic:specific` - most common part
+ -- `generic:<order>` - e.g. `header:1` for the first header
+ -- `generic:safe` - replace unsafe stuff with safe + lowercase
+ -- also aliases entry is set to avoid implicit expression
+ local function process_compound_condition(cond, generic, specific)
+ local full_key = generic .. ':' .. specific
+ checks[full_key] = cond
+
+ -- Try numeric key
+ for i = 1, 1000 do
+ local num_key = generic .. ':' .. tostring(i)
+ if not checks[num_key] then
+ checks[num_key] = cond
+ aliases[num_key] = true
+ break
+ end
+ end
+
+ local safe_key = generic .. ':' ..
+ specific:gsub('[:%-+&|><]', '_')
+ :gsub('%(', '[')
+ :gsub('%)', ']')
+ :lower()
+
+ if not checks[safe_key] then
+ checks[safe_key] = cond
+ aliases[full_key] = true
+ end
+
+ return safe_key
+ end
+ -- Headers are tricky:
+ -- We create an closure with extraction function depending on header name
+ -- We also inserts it into `checks` table as an atom in form header:<hname>
+ -- Check function depends on the input:
+ -- * for something that looks like `header = "/bar/"` we create a regexp
+ -- * for something that looks like `header = true` we just check the existence
+ local function process_header_elt(table_element, extractor_func)
+ if elt[table_element] then
+ for k, v in pairs(elt[table_element]) do
+ if type(v) == 'string' then
+ local re = rspamd_regexp.create(v)
+ if re then
+ local cond = {
+ check = function(values)
+ return fun.any(function(c)
+ return re:match(c)
+ end, values)
+ end,
+ extract = extractor_func(k),
+ }
+ local skey = process_compound_condition(cond, table_element,
+ k)
+ lua_util.debugm(N, rspamd_config, 'added %s condition to "%s": %s =~ %s',
+ skey, name, k, v)
+ end
+ elseif type(v) == 'boolean' then
+ local cond = {
+ check = function(values)
+ if #values == 0 then
+ return (not v)
+ end
+ return v
+ end,
+ extract = extractor_func(k),
+ }
+
+ local skey = process_compound_condition(cond, table_element,
+ k)
+ lua_util.debugm(N, rspamd_config, 'added %s condition to "%s": %s == %s',
+ skey, name, k, v)
+ else
+ rspamd_logger.errx(rspamd_config, 'invalid %s %s = %s', table_element, k, v)
+ end
+ end
+ end
+ end
+
+ process_header_elt('request_header', function(hname)
+ return function(task)
+ local rh = task:get_request_header(hname)
+ if rh then
+ return { rh }
+ end
+ return {}
+ end
+ end)
+ process_header_elt('header', function(hname)
+ return function(task)
+ local rh = task:get_header_full(hname)
+ if rh then
+ return fun.totable(fun.map(function(h)
+ return h.decoded
+ end, rh))
+ end
+ return {}
+ end
+ end)
+
+ if elt.selector then
+ local sel = lua_selectors.create_selector_closure(rspamd_config, elt.selector,
+ elt.delimiter or "")
+
+ if sel then
+ local cond = {
+ check = function(values)
+ return fun.any(function(c)
+ return c
+ end, values)
+ end,
+ extract = sel,
+ }
+ local skey = process_compound_condition(cond, 'selector', elt.selector)
+ lua_util.debugm(N, rspamd_config, 'added selector condition to "%s": %s',
+ name, skey)
+ end
+
+ end
+
+ -- Special, special case!
+ local inverse = false
+ if elt.inverse then
+ lua_util.debugm(N, rspamd_config, 'added inverse condition to "%s"',
+ name)
+ inverse = true
+ end
+
+ -- Count checks and create Rspamd expression from a set of rules
+ local nchecks = 0
+ for k, _ in pairs(checks) do
+ if not aliases[k] then
+ nchecks = nchecks + 1
+ end
+ end
+
+ if nchecks > 0 then
+ -- Now we can deal with the expression!
+ if not elt.expression then
+ -- Artificial & expression to deal with the legacy parts
+ -- Here we get all keys and concatenate them with '&&'
+ local s = ' && '
+ -- By De Morgan laws
+ if inverse then
+ s = ' || '
+ end
+ -- Exclude aliases and join all checks by key
+ local expr_str = table.concat(lua_util.keys(fun.filter(
+ function(k, _)
+ return not aliases[k]
+ end,
+ checks)), s)
+
+ if inverse then
+ expr_str = string.format('!(%s)', expr_str)
+ end
+
+ elt.expression = expr_str
+ lua_util.debugm(N, rspamd_config, 'added implicit settings expression for %s: %s',
+ name, expr_str)
+ end
+
+ -- Parse expression's sanity
+ local function parse_atom(str)
+ local atom = table.concat(fun.totable(fun.take_while(function(c)
+ if string.find(', \t()><+!|&\n', c, 1, true) then
+ return false
+ end
+ return true
+ end, fun.iter(str))), '')
+
+ if checks[atom] then
+ return atom
+ end
+
+ rspamd_logger.errx(rspamd_config,
+ 'use of undefined element "%s" when parsing settings expression, known checks: %s',
+ atom, table.concat(fun.totable(fun.map(function(k, _)
+ return k
+ end, checks)), ','))
+
+ return nil
+ end
+
+ local rspamd_expression = require "rspamd_expression"
+ out.expression = rspamd_expression.create(elt.expression, parse_atom,
+ mempool)
+ out.checks = checks
+
+ if not out.expression then
+ rspamd_logger.errx(rspamd_config, 'cannot parse expression %s for %s',
+ elt.expression, name)
+ else
+ lua_util.debugm(N, rspamd_config, 'registered settings %s with %s checks',
+ name, nchecks)
+ end
+ else
+ if not elt.disabled and elt.external_map then
+ lua_util.debugm(N, rspamd_config, 'registered settings %s with no checks, assume it as implicit',
+ name)
+ out.implicit = 1
+ end
+ end
+
+ -- Process symbols part/apply part
+ if elt['symbols'] then
+ lua_util.debugm(N, rspamd_config, 'added symbols condition to "%s": %s',
+ name, elt.symbols)
+ out['symbols'] = elt['symbols']
+ end
+
+ --[[
+ external_map = {
+ map = { ... };
+ selector = "...";
+ }
+ --]]
+ if type(elt.external_map) == 'table'
+ and elt.external_map.map and elt.external_map.selector then
+ local maybe_external_map = {}
+ maybe_external_map.map = lua_maps.map_add_from_ucl(elt.external_map.map, "",
+ string.format("External map for settings element %s", name),
+ gen_settings_external_cb(name))
+ maybe_external_map.selector = lua_selectors.create_selector_closure_fn(rspamd_config,
+ rspamd_config, elt.external_map.selector, ";", lua_selectors.kv_table_from_pairs)
+
+ if maybe_external_map.map and maybe_external_map.selector then
+ rspamd_logger.infox(rspamd_config, "added external map for user's settings %s", name)
+ out.external_map = maybe_external_map
+ else
+ local incorrect_element
+ if not maybe_external_map.map then
+ incorrect_element = "map definition"
+ else
+ incorrect_element = "selector definition"
+ end
+ rspamd_logger.warnx(rspamd_config, "cannot add external map for user's settings; incorrect element: %s",
+ incorrect_element)
+ out.external_map = nil
+ end
+ end
+
+ if not elt.external_map then
+ if elt['apply'] then
+ -- Just insert all metric results to the action key
+ out['apply'] = elt['apply']
+ elseif elt['whitelist'] or elt['want_spam'] then
+ out['whitelist'] = true
+ else
+ rspamd_logger.errx(rspamd_config, "no actions in settings: " .. name)
+ return nil
+ end
+ end
+
+ if allow_ids then
+ if not elt.id then
+ elt.id = name
+ end
+
+ if elt['id'] then
+ -- We are here from a postload script
+ out.id = lua_settings.register_settings_id(elt.id, out, true)
+ lua_util.debugm(N, rspamd_config,
+ 'added settings id to "%s": %s -> %s',
+ name, elt.id, out.id)
+ end
+
+ if not is_static then
+ -- If we apply that from map
+ -- In fact, it is useless and evil but who cares...
+ if elt.apply and elt.apply.symbols then
+ -- Register virtual symbols
+ for k, v in pairs(elt.apply.symbols) do
+ local rtb = {
+ type = 'virtual',
+ parent = module_sym_id,
+ }
+ if type(k) == 'number' and type(v) == 'string' then
+ rtb.name = v
+ elseif type(k) == 'string' then
+ rtb.name = k
+ end
+ if out.id then
+ rtb.allowed_ids = tostring(elt.id)
+ end
+ rspamd_config:register_symbol(rtb)
+ end
+ end
+ end
+ else
+ if elt['id'] then
+ rspamd_logger.errx(rspamd_config,
+ 'cannot set static IDs from dynamic settings, please read the docs')
+ end
+ end
+
+ return out
+ end
+
+ settings_initialized = false
+ -- filter trash in the input
+ local ft = fun.filter(
+ function(_, elt)
+ if type(elt) == "table" then
+ return true
+ end
+ return false
+ end, tbl)
+
+ -- clear all settings
+ max_pri = 0
+ local nrules = 0
+ for k in pairs(settings) do
+ settings[k] = {}
+ end
+ -- fill new settings by priority
+ fun.for_each(function(k, v)
+ local pri = get_priority(v)
+ if pri > max_pri then
+ max_pri = pri
+ end
+ if not settings[pri] then
+ settings[pri] = {}
+ end
+ local s = process_setting_elt(k, v)
+ if s then
+ table.insert(settings[pri], { name = k, rule = s })
+ nrules = nrules + 1
+ end
+ end, ft)
+ -- sort settings with equal priorities in alphabetical order
+ for pri, _ in pairs(settings) do
+ table.sort(settings[pri], function(a, b)
+ return a.name < b.name
+ end)
+ end
+
+ settings_initialized = true
+ lua_settings.load_all_settings(true)
+ rspamd_logger.infox(rspamd_config, 'loaded %s elements of settings', nrules)
+
+ return true
+end
+
+-- Parse settings map from the ucl line
+local settings_map_pool
+
+local function process_settings_map(map_text)
+ local parser = ucl.parser()
+ local res, err = parser:parse_text(map_text)
+
+ if not res then
+ rspamd_logger.warnx(rspamd_config, 'cannot parse settings map: ' .. err)
+ else
+ if settings_map_pool then
+ settings_map_pool:destroy()
+ end
+
+ settings_map_pool = rspamd_mempool.create()
+ local obj = parser:get_object()
+ if obj['settings'] then
+ process_settings_table(obj['settings'], false,
+ settings_map_pool, false)
+ else
+ process_settings_table(obj, false, settings_map_pool,
+ false)
+ end
+ end
+
+ return res
+end
+
+local function gen_redis_callback(handler, id)
+ return function(task)
+ local key = handler(task)
+
+ local function redis_settings_cb(err, data)
+ if not err and type(data) == 'table' then
+ for _, d in ipairs(data) do
+ if type(d) == 'string' then
+ local parser = ucl.parser()
+ local res, ucl_err = parser:parse_text(d)
+ if not res then
+ rspamd_logger.warnx(rspamd_config, 'cannot parse settings from redis: %s',
+ ucl_err)
+ else
+ local obj = parser:get_object()
+ rspamd_logger.infox(task, "<%1> apply settings according to redis rule %2",
+ task:get_message_id(), id)
+ apply_settings(task, obj, nil, 'redis')
+ break
+ end
+ end
+ end
+ elseif err then
+ rspamd_logger.errx(task, 'Redis error: %1', err)
+ end
+ end
+
+ if not key then
+ lua_util.debugm(N, task, 'handler number %s returned nil', id)
+ return
+ end
+
+ local keys
+ if type(key) == 'table' then
+ keys = key
+ else
+ keys = { key }
+ end
+ key = keys[1]
+
+ local ret, _, _ = rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_settings_cb, --callback
+ 'MGET', -- command
+ keys -- arguments
+ )
+ if not ret then
+ rspamd_logger.errx(task, 'Redis MGET failed: %s', ret)
+ end
+ end
+end
+
+local redis_section = rspamd_config:get_all_opt("settings_redis")
+local redis_key_handlers = {}
+
+if redis_section then
+ redis_params = rspamd_parse_redis_server('settings_redis')
+ if redis_params then
+ local handlers = redis_section.handlers
+
+ for id, h in pairs(handlers) do
+ local chunk, err = load(h)
+
+ if not chunk then
+ rspamd_logger.errx(rspamd_config, 'Cannot load handler from string: %s',
+ tostring(err))
+ else
+ local res, func = pcall(chunk)
+ if not res then
+ rspamd_logger.errx(rspamd_config, 'Cannot add handler from string: %s',
+ tostring(func))
+ else
+ redis_key_handlers[id] = func
+ end
+ end
+ end
+ end
+
+ fun.each(function(id, h)
+ rspamd_config:register_symbol({
+ name = 'REDIS_SETTINGS' .. tostring(id),
+ type = 'prefilter',
+ callback = gen_redis_callback(h, id),
+ priority = lua_util.symbols_priorities.top,
+ flags = 'empty,nostat',
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
+ })
+ end, redis_key_handlers)
+end
+
+module_sym_id = rspamd_config:register_symbol({
+ name = 'SETTINGS_CHECK',
+ type = 'prefilter',
+ callback = check_settings,
+ priority = lua_util.symbols_priorities.top,
+ flags = 'empty,nostat,explicit_disable,ignore_passthrough',
+})
+
+local set_section = rspamd_config:get_all_opt("settings")
+
+if set_section and set_section[1] and type(set_section[1]) == "string" then
+ -- Just a map of ucl
+ local map_attrs = {
+ url = set_section[1],
+ description = "settings map",
+ callback = process_settings_map,
+ opaque_data = true
+ }
+ if not rspamd_config:add_map(map_attrs) then
+ rspamd_logger.errx(rspamd_config, 'cannot load settings from %1', set_section)
+ end
+elseif set_section and type(set_section) == "table" then
+ settings_map_pool = rspamd_mempool.create()
+ -- We need to check this table and register static symbols first
+ -- Postponed settings init is needed to ensure that all symbols have been
+ -- registered BEFORE settings plugin. Otherwise, we can have inconsistent settings expressions
+ fun.each(function(_, elt)
+ if elt.register_symbols then
+ for k, v in pairs(elt.register_symbols) do
+ local rtb = {
+ type = 'virtual',
+ parent = module_sym_id,
+ }
+ if type(k) == 'number' and type(v) == 'string' then
+ rtb.name = v
+ elseif type(k) == 'string' then
+ rtb.name = k
+ if type(v) == 'table' then
+ for kk, vv in pairs(v) do
+ -- Enrich table wih extra values
+ rtb[kk] = vv
+ end
+ end
+ end
+ rspamd_config:register_symbol(rtb)
+ end
+ end
+ if elt.apply and elt.apply.symbols then
+ -- Register virtual symbols
+ for k, v in pairs(elt.apply.symbols) do
+ local rtb = {
+ type = 'virtual',
+ parent = module_sym_id,
+ }
+ if type(k) == 'number' and type(v) == 'string' then
+ rtb.name = v
+ elseif type(k) == 'string' then
+ rtb.name = k
+ end
+ rspamd_config:register_symbol(rtb)
+ end
+ end
+ end,
+ -- Include only settings, exclude all maps
+ fun.filter(
+ function(_, elt)
+ if type(elt) == "table" then
+ return true
+ end
+ return false
+ end, set_section)
+ )
+
+ rspamd_config:add_post_init(function()
+ process_settings_table(set_section, true, settings_map_pool, true)
+ end, 100)
+end
+
+rspamd_config:add_config_unload(function()
+ if settings_map_pool then
+ settings_map_pool:destroy()
+ end
+end)
diff --git a/src/plugins/lua/spamassassin.lua b/src/plugins/lua/spamassassin.lua
new file mode 100644
index 0000000..3ea7944
--- /dev/null
+++ b/src/plugins/lua/spamassassin.lua
@@ -0,0 +1,1774 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- This plugin is intended to read and parse spamassassin rules with regexp
+-- rules. SA plugins or statistics are not supported
+
+local E = {}
+local N = 'spamassassin'
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_regexp = require "rspamd_regexp"
+local rspamd_expression = require "rspamd_expression"
+local rspamd_trie = require "rspamd_trie"
+local util = require "rspamd_util"
+local lua_util = require "lua_util"
+local fun = require "fun"
+
+-- Known plugins
+local known_plugins = {
+ 'Mail::SpamAssassin::Plugin::FreeMail',
+ 'Mail::SpamAssassin::Plugin::HeaderEval',
+ 'Mail::SpamAssassin::Plugin::ReplaceTags',
+ 'Mail::SpamAssassin::Plugin::RelayEval',
+ 'Mail::SpamAssassin::Plugin::MIMEEval',
+ 'Mail::SpamAssassin::Plugin::BodyEval',
+ 'Mail::SpamAssassin::Plugin::MIMEHeader',
+ 'Mail::SpamAssassin::Plugin::WLBLEval',
+ 'Mail::SpamAssassin::Plugin::HTMLEval',
+}
+
+-- Table that replaces SA symbol with rspamd equivalent
+-- Used for dependency resolution
+local symbols_replacements = {
+ -- SPF replacements
+ USER_IN_SPF_WHITELIST = 'WHITELIST_SPF',
+ USER_IN_DEF_SPF_WL = 'WHITELIST_SPF',
+ SPF_PASS = 'R_SPF_ALLOW',
+ SPF_FAIL = 'R_SPF_FAIL',
+ SPF_SOFTFAIL = 'R_SPF_SOFTFAIL',
+ SPF_HELO_PASS = 'R_SPF_ALLOW',
+ SPF_HELLO_FAIL = 'R_SPF_FAIL',
+ SPF_HELLO_SOFTFAIL = 'R_SPF_SOFTFAIL',
+ -- DKIM replacements
+ USER_IN_DKIM_WHITELIST = 'WHITELIST_DKIM',
+ USER_IN_DEF_DKIM_WL = 'WHITELIST_DKIM',
+ DKIM_VALID = 'R_DKIM_ALLOW',
+ -- SURBL replacements
+ URIBL_SBL_A = 'URIBL_SBL',
+ URIBL_DBL_SPAM = 'DBL_SPAM',
+ URIBL_DBL_PHISH = 'DBL_PHISH',
+ URIBL_DBL_MALWARE = 'DBL_MALWARE',
+ URIBL_DBL_BOTNETCC = 'DBL_BOTNET',
+ URIBL_DBL_ABUSE_SPAM = 'DBL_ABUSE',
+ URIBL_DBL_ABUSE_REDIR = 'DBL_ABUSE_REDIR',
+ URIBL_DBL_ABUSE_MALW = 'DBL_ABUSE_MALWARE',
+ URIBL_DBL_ABUSE_BOTCC = 'DBL_ABUSE_BOTNET',
+ URIBL_WS_SURBL = 'WS_SURBL_MULTI',
+ URIBL_PH_SURBL = 'PH_SURBL_MULTI',
+ URIBL_MW_SURBL = 'MW_SURBL_MULTI',
+ URIBL_CR_SURBL = 'CRACKED_SURBL',
+ URIBL_ABUSE_SURBL = 'ABUSE_SURBL',
+ -- Misc rules
+ BODY_URI_ONLY = 'R_EMPTY_IMAGE',
+ HTML_IMAGE_ONLY_04 = 'HTML_SHORT_LINK_IMG_1',
+ HTML_IMAGE_ONLY_08 = 'HTML_SHORT_LINK_IMG_1',
+ HTML_IMAGE_ONLY_12 = 'HTML_SHORT_LINK_IMG_1',
+ HTML_IMAGE_ONLY_16 = 'HTML_SHORT_LINK_IMG_2',
+ HTML_IMAGE_ONLY_20 = 'HTML_SHORT_LINK_IMG_2',
+ HTML_IMAGE_ONLY_24 = 'HTML_SHORT_LINK_IMG_3',
+ HTML_IMAGE_ONLY_28 = 'HTML_SHORT_LINK_IMG_3',
+ HTML_IMAGE_ONLY_32 = 'HTML_SHORT_LINK_IMG_3',
+}
+
+-- Internal variables
+local rules = {}
+local atoms = {}
+local scores = {}
+local scores_added = {}
+local external_deps = {}
+local freemail_domains = {}
+local pcre_only_regexps = {}
+local freemail_trie
+local replace = {
+ tags = {},
+ pre = {},
+ inter = {},
+ post = {},
+ rules = {},
+}
+local internal_regexp = {
+ date_shift = rspamd_regexp.create("^\\(\\s*'((?:-?\\d+)|(?:undef))'\\s*,\\s*'((?:-?\\d+)|(?:undef))'\\s*\\)$")
+}
+
+-- Mail::SpamAssassin::Plugin::WLBLEval plugin
+local sa_lists = {
+ from_blacklist = {},
+ from_whitelist = {},
+ from_def_whitelist = {},
+ to_blacklist = {},
+ to_whitelist = {},
+ elts = 0,
+}
+
+local func_cache = {}
+local section = rspamd_config:get_all_opt("spamassassin")
+if not (section and type(section) == 'table') then
+ rspamd_logger.infox(rspamd_config, 'Module is unconfigured')
+end
+
+-- Minimum score to treat symbols as meta
+local meta_score_alpha = 0.5
+
+-- Maximum size of regexp checked
+local match_limit = 0
+
+-- Default priority of the scores registered in the metric
+-- Historically this is set to 2 allowing SA scores to override Rspamd scores
+local scores_priority = 2
+
+local function split(str, delim)
+ local result = {}
+
+ if not delim then
+ delim = '[^%s]+'
+ end
+
+ for token in string.gmatch(str, delim) do
+ table.insert(result, token)
+ end
+
+ return result
+end
+
+local function replace_symbol(s)
+ local rspamd_symbol = symbols_replacements[s]
+ if not rspamd_symbol then
+ return s, false
+ end
+ return rspamd_symbol, true
+end
+
+local ffi
+if type(jit) == 'table' then
+ ffi = require("ffi")
+ ffi.cdef [[
+ int rspamd_re_cache_type_from_string (const char *str);
+ int rspamd_re_cache_process_ffi (void *ptask,
+ void *pre,
+ int type,
+ const char *type_data,
+ int is_strong);
+]]
+end
+
+local function process_regexp_opt(re, task, re_type, header, strong)
+ --[[
+ -- This is now broken with lua regexp conditions!
+ if type(jit) == 'table' then
+ -- Use ffi call
+ local itype = ffi.C.rspamd_re_cache_type_from_string(re_type)
+
+ if not strong then
+ strong = 0
+ else
+ strong = 1
+ end
+ local iret = ffi.C.rspamd_re_cache_process_ffi (task, re, itype, header, strong)
+
+ return tonumber(iret)
+ else
+ return task:process_regexp(re, re_type, header, strong)
+ end
+ --]]
+ return task:process_regexp(re, re_type, header, strong)
+end
+
+local function is_pcre_only(name)
+ if pcre_only_regexps[name] then
+ rspamd_logger.infox(rspamd_config, 'mark re %s as PCRE only', name)
+ return true
+ end
+ return false
+end
+
+local function handle_header_def(hline, cur_rule)
+ --Now check for modifiers inside header's name
+ local hdrs = split(hline, '[^|]+')
+ local hdr_params = {}
+ local cur_param = {}
+ -- Check if an re is an ordinary re
+ local ordinary = true
+
+ for _, h in ipairs(hdrs) do
+ if h == 'ALL' or h == 'ALL:raw' then
+ ordinary = false
+ cur_rule['type'] = 'function'
+ -- Pack closure
+ local re = cur_rule['re']
+ -- Rule to match all headers
+ rspamd_config:register_regexp({
+ re = re,
+ type = 'allheader',
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ cur_rule['function'] = function(task)
+ if not re then
+ rspamd_logger.errx(task, 're is missing for rule %1', h)
+ return 0
+ end
+
+ return process_regexp_opt(re, task, 'allheader')
+ end
+ else
+ local args = split(h, '[^:]+')
+ cur_param['strong'] = false
+ cur_param['raw'] = false
+ cur_param['header'] = args[1]
+
+ if args[2] then
+ -- We have some ops that are required for the header, so it's not ordinary
+ ordinary = false
+ end
+
+ fun.each(function(func)
+ if func == 'addr' then
+ cur_param['function'] = function(str)
+ local addr_parsed = util.parse_mail_address(str)
+ local ret = {}
+ if addr_parsed then
+ for _, elt in ipairs(addr_parsed) do
+ if elt['addr'] then
+ table.insert(ret, elt['addr'])
+ end
+ end
+ end
+
+ return ret
+ end
+ elseif func == 'name' then
+ cur_param['function'] = function(str)
+ local addr_parsed = util.parse_mail_address(str)
+ local ret = {}
+ if addr_parsed then
+ for _, elt in ipairs(addr_parsed) do
+ if elt['name'] then
+ table.insert(ret, elt['name'])
+ end
+ end
+ end
+
+ return ret
+ end
+ elseif func == 'raw' then
+ cur_param['raw'] = true
+ elseif func == 'case' then
+ cur_param['strong'] = true
+ else
+ rspamd_logger.warnx(rspamd_config, 'Function %1 is not supported in %2',
+ func, cur_rule['symbol'])
+ end
+ end, fun.tail(args))
+
+ local function split_hdr_param(param, headers)
+ for _, hh in ipairs(headers) do
+ local nparam = {}
+ for k, v in pairs(param) do
+ if k ~= 'header' then
+ nparam[k] = v
+ end
+ end
+
+ nparam['header'] = hh
+ table.insert(hdr_params, nparam)
+ end
+ end
+ -- Some header rules require splitting to check of multiple headers
+ if cur_param['header'] == 'MESSAGEID' then
+ -- Special case for spamassassin
+ ordinary = false
+ split_hdr_param(cur_param, {
+ 'Message-ID',
+ 'X-Message-ID',
+ 'Resent-Message-ID' })
+ elseif cur_param['header'] == 'ToCc' then
+ ordinary = false
+ split_hdr_param(cur_param, { 'To', 'Cc', 'Bcc' })
+ else
+ table.insert(hdr_params, cur_param)
+ end
+ end
+
+ cur_rule['ordinary'] = ordinary
+ cur_rule['header'] = hdr_params
+ end
+end
+
+local function freemail_search(input)
+ local res = 0
+ local function trie_callback(number, pos)
+ lua_util.debugm(N, rspamd_config, 'Matched pattern %1 at pos %2', freemail_domains[number], pos)
+ res = res + 1
+ end
+
+ if input then
+ freemail_trie:match(input, trie_callback, true)
+ end
+
+ return res
+end
+
+local function gen_eval_rule(arg)
+ local eval_funcs = {
+ { 'check_freemail_from', function(task)
+ local from = task:get_from('mime')
+ if from and from[1] then
+ return freemail_search(string.lower(from[1]['addr']))
+ end
+ return 0
+ end },
+ { 'check_freemail_replyto',
+ function(task)
+ return freemail_search(task:get_header('Reply-To'))
+ end
+ },
+ { 'check_freemail_header',
+ function(task, remain)
+ -- Remain here contains one or two args: header and regexp to match
+ local larg = string.match(remain, "^%(%s*['\"]([^%s]+)['\"]%s*%)$")
+ local re = nil
+ if not larg then
+ larg, re = string.match(remain, "^%(%s*['\"]([^%s]+)['\"]%s*,%s*['\"]([^%s]+)['\"]%s*%)$")
+ end
+
+ if larg then
+ local h
+ if larg == 'EnvelopeFrom' then
+ h = task:get_from('smtp')
+ if h then
+ h = h[1]['addr']
+ end
+ else
+ h = task:get_header(larg)
+ end
+ if h then
+ local hdr_freemail = freemail_search(string.lower(h))
+
+ if hdr_freemail > 0 and re then
+ local r = rspamd_regexp.create_cached(re)
+ if r then
+ if r:match(h) then
+ return 1
+ end
+ return 0
+ else
+ rspamd_logger.infox(rspamd_config, 'cannot create regexp %1', re)
+ return 0
+ end
+ end
+
+ return hdr_freemail
+ end
+ end
+
+ return 0
+ end
+ },
+ {
+ 'check_for_missing_to_header',
+ function(task)
+ local th = task:get_recipients('mime')
+ if not th or #th == 0 then
+ return 1
+ end
+
+ return 0
+ end
+ },
+ {
+ 'check_relays_unparseable',
+ function(task)
+ local rh_mime = task:get_header_full('Received')
+ local rh_parsed = task:get_received_headers()
+
+ local rh_cnt = 0
+ if rh_mime then
+ rh_cnt = #rh_mime
+ end
+ local parsed_cnt = 0
+ if rh_parsed then
+ parsed_cnt = #rh_parsed
+ end
+
+ return rh_cnt - parsed_cnt
+ end
+ },
+ {
+ 'check_for_shifted_date',
+ function(task, remain)
+ -- Remain here contains two args: start and end hours shift
+ local matches = internal_regexp['date_shift']:search(remain, true, true)
+ if matches and matches[1] then
+ local min_diff = matches[1][2]
+ local max_diff = matches[1][3]
+
+ if min_diff == 'undef' then
+ min_diff = 0
+ else
+ min_diff = tonumber(min_diff) * 3600
+ end
+ if max_diff == 'undef' then
+ max_diff = 0
+ else
+ max_diff = tonumber(max_diff) * 3600
+ end
+
+ -- Now get the difference between Date and message received date
+ local dm = task:get_date { format = 'message', gmt = true }
+ local dt = task:get_date { format = 'connect', gmt = true }
+ local diff = dm - dt
+
+ if (max_diff == 0 and diff >= min_diff) or
+ (min_diff == 0 and diff <= max_diff) or
+ (diff >= min_diff and diff <= max_diff) then
+ return 1
+ end
+ end
+
+ return 0
+ end
+ },
+ {
+ 'check_for_mime',
+ function(task, remain)
+ local larg = string.match(remain, "^%(%s*['\"]([^%s]+)['\"]%s*%)$")
+
+ if larg then
+ if larg == 'mime_attachment' then
+ local parts = task:get_parts()
+ if parts then
+ for _, p in ipairs(parts) do
+ if p:get_filename() then
+ return 1
+ end
+ end
+ end
+ else
+ rspamd_logger.infox(task, 'unimplemented mime check %1', arg)
+ end
+ end
+
+ return 0
+ end
+ },
+ {
+ 'check_from_in_blacklist',
+ function(task)
+ local from = task:get_from('mime')
+ if ((from or E)[1] or E).addr then
+ if sa_lists['from_blacklist'][string.lower(from[1]['addr'])] then
+ return 1
+ end
+ end
+
+ return 0
+ end
+ },
+ {
+ 'check_from_in_whitelist',
+ function(task)
+ local from = task:get_from('mime')
+ if ((from or E)[1] or E).addr then
+ if sa_lists['from_whitelist'][string.lower(from[1]['addr'])] then
+ return 1
+ end
+ end
+
+ return 0
+ end
+ },
+ {
+ 'check_from_in_default_whitelist',
+ function(task)
+ local from = task:get_from('mime')
+ if ((from or E)[1] or E).addr then
+ if sa_lists['from_def_whitelist'][string.lower(from[1]['addr'])] then
+ return 1
+ end
+ end
+
+ return 0
+ end
+ },
+ {
+ 'check_to_in_blacklist',
+ function(task)
+ local rcpt = task:get_recipients('mime')
+ if rcpt then
+ for _, r in ipairs(rcpt) do
+ if sa_lists['to_blacklist'][string.lower(r['addr'])] then
+ return 1
+ end
+ end
+ end
+
+ return 0
+ end
+ },
+ {
+ 'check_to_in_whitelist',
+ function(task)
+ local rcpt = task:get_recipients('mime')
+ if rcpt then
+ for _, r in ipairs(rcpt) do
+ if sa_lists['to_whitelist'][string.lower(r['addr'])] then
+ return 1
+ end
+ end
+ end
+
+ return 0
+ end
+ },
+ {
+ 'html_tag_exists',
+ function(task, remain)
+ local tp = task:get_text_parts()
+
+ for _, p in ipairs(tp) do
+ if p:is_html() then
+ local hc = p:get_html()
+
+ if hc:has_tag(remain) then
+ return 1
+ end
+ end
+ end
+
+ return 0
+ end
+ }
+ }
+
+ for _, f in ipairs(eval_funcs) do
+ local pat = string.format('^%s', f[1])
+ local first, last = string.find(arg, pat)
+
+ if first then
+ local func_arg = string.sub(arg, last + 1)
+ return function(task)
+ return f[2](task, func_arg)
+ end
+ end
+ end
+end
+
+-- Returns parser function or nil
+local function maybe_parse_sa_function(line)
+ local arg
+ local elts = split(line, '[^:]+')
+ arg = elts[2]
+
+ lua_util.debugm(N, rspamd_config, 'trying to parse SA function %1 with args %2',
+ elts[1], elts[2])
+ local substitutions = {
+ { '^exists:',
+ function(task)
+ -- filter
+ local hdrs_check
+ if arg == 'MESSAGEID' then
+ hdrs_check = {
+ 'Message-ID',
+ 'X-Message-ID',
+ 'Resent-Message-ID'
+ }
+ elseif arg == 'ToCc' then
+ hdrs_check = { 'To', 'Cc', 'Bcc' }
+ else
+ hdrs_check = { arg }
+ end
+
+ for _, h in ipairs(hdrs_check) do
+ if task:has_header(h) then
+ return 1
+ end
+ end
+ return 0
+ end,
+ },
+ { '^eval:',
+ function(task)
+ local func = func_cache[arg]
+ if not func then
+ func = gen_eval_rule(arg)
+ func_cache[arg] = func
+ end
+
+ if not func then
+ rspamd_logger.errx(task, 'cannot find appropriate eval rule for function %1',
+ arg)
+ else
+ return func(task)
+ end
+
+ return 0
+ end
+ },
+ }
+
+ for _, s in ipairs(substitutions) do
+ if string.find(line, s[1]) then
+ return s[2]
+ end
+ end
+
+ return nil
+end
+
+local function words_to_re(words, start)
+ return table.concat(fun.totable(fun.drop_n(start, words)), " ");
+end
+
+local function process_tflags(rule, flags)
+ fun.each(function(flag)
+ if flag == 'publish' then
+ rule['publish'] = true
+ elseif flag == 'multiple' then
+ rule['multiple'] = true
+ elseif string.match(flag, '^maxhits=(%d+)$') then
+ rule['maxhits'] = tonumber(string.match(flag, '^maxhits=(%d+)$'))
+ elseif flag == 'nice' then
+ rule['nice'] = true
+ end
+ end, fun.drop_n(1, flags))
+
+ if rule['re'] then
+ if rule['maxhits'] then
+ rule['re']:set_max_hits(rule['maxhits'])
+ elseif rule['multiple'] then
+ rule['re']:set_max_hits(0)
+ else
+ rule['re']:set_max_hits(1)
+ end
+ end
+end
+
+local function process_replace(words, tbl)
+ local re = words_to_re(words, 2)
+ tbl[words[2]] = re
+end
+
+local function process_sa_conf(f)
+ local cur_rule = {}
+ local valid_rule = false
+
+ local function insert_cur_rule()
+ if cur_rule['type'] ~= 'meta' and cur_rule['publish'] then
+ -- Create meta rule from this rule
+ local nsym = '__fake' .. cur_rule['symbol']
+ local nrule = {
+ type = 'meta',
+ symbol = cur_rule['symbol'],
+ score = cur_rule['score'],
+ meta = nsym,
+ description = cur_rule['description'],
+ }
+ rules[nrule['symbol']] = nrule
+ cur_rule['symbol'] = nsym
+ end
+ -- We have previous rule valid
+ if not cur_rule['symbol'] then
+ rspamd_logger.errx(rspamd_config, 'bad rule definition: %1', cur_rule)
+ end
+ rules[cur_rule['symbol']] = cur_rule
+ cur_rule = {}
+ valid_rule = false
+ end
+
+ local function parse_score(words)
+ if #words == 3 then
+ -- score rule <x>
+ lua_util.debugm(N, rspamd_config, 'found score for %1: %2', words[2], words[3])
+ return tonumber(words[3])
+ elseif #words == 6 then
+ -- score rule <x1> <x2> <x3> <x4>
+ -- we assume here that bayes and network are enabled and select <x4>
+ lua_util.debugm(N, rspamd_config, 'found score for %1: %2', words[2], words[6])
+ return tonumber(words[6])
+ else
+ rspamd_logger.errx(rspamd_config, 'invalid score for %1', words[2])
+ end
+
+ return 0
+ end
+
+ local skip_to_endif = false
+ local if_nested = 0
+ for l in f:lines() do
+ (function()
+ l = lua_util.rspamd_str_trim(l)
+ -- Replace bla=~/re/ with bla =~ /re/ (#2372)
+ l = l:gsub('([^%s])%s*([=!]~)%s*([^%s])', '%1 %2 %3')
+
+ if string.len(l) == 0 or string.sub(l, 1, 1) == '#' then
+ return
+ end
+
+ -- Unbalanced if/endif
+ if if_nested < 0 then
+ if_nested = 0
+ end
+ if skip_to_endif then
+ if string.match(l, '^endif') then
+ if_nested = if_nested - 1
+
+ if if_nested == 0 then
+ skip_to_endif = false
+ end
+ elseif string.match(l, '^if') then
+ if_nested = if_nested + 1
+ elseif string.match(l, '^else') then
+ -- Else counterpart for if
+ skip_to_endif = false
+ end
+ return
+ else
+ if string.match(l, '^ifplugin') then
+ local ls = split(l)
+
+ if not fun.any(function(pl)
+ if pl == ls[2] then
+ return true
+ end
+ return false
+ end, known_plugins) then
+ skip_to_endif = true
+ end
+ if_nested = if_nested + 1
+ elseif string.match(l, '^if !plugin%(') then
+ local pname = string.match(l, '^if !plugin%(([A-Za-z:]+)%)')
+ if fun.any(function(pl)
+ if pl == pname then
+ return true
+ end
+ return false
+ end, known_plugins) then
+ skip_to_endif = true
+ end
+ if_nested = if_nested + 1
+ elseif string.match(l, '^if') then
+ -- Unknown if
+ skip_to_endif = true
+ if_nested = if_nested + 1
+ elseif string.match(l, '^else') then
+ -- Else counterpart for if
+ skip_to_endif = true
+ elseif string.match(l, '^endif') then
+ if_nested = if_nested - 1
+ end
+ end
+
+ -- Skip comments
+ local words = fun.totable(fun.take_while(
+ function(w)
+ return string.sub(w, 1, 1) ~= '#'
+ end,
+ fun.filter(function(w)
+ return w ~= ""
+ end,
+ fun.iter(split(l)))))
+
+ if words[1] == "header" or words[1] == 'mimeheader' then
+ -- header SYMBOL Header ~= /regexp/
+ if valid_rule then
+ insert_cur_rule()
+ end
+ if words[4] and (words[4] == '=~' or words[4] == '!~') then
+ cur_rule['type'] = 'header'
+ cur_rule['symbol'] = words[2]
+
+ if words[4] == '!~' then
+ cur_rule['not'] = true
+ end
+
+ cur_rule['re_expr'] = words_to_re(words, 4)
+ local unset_comp = string.find(cur_rule['re_expr'], '%s+%[if%-unset:')
+ if unset_comp then
+ -- We have optional part that needs to be processed
+ local unset = string.match(string.sub(cur_rule['re_expr'], unset_comp),
+ '%[if%-unset:%s*([^%]%s]+)]')
+ cur_rule['unset'] = unset
+ -- Cut it down
+ cur_rule['re_expr'] = string.sub(cur_rule['re_expr'], 1, unset_comp - 1)
+ end
+
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+
+ if not cur_rule['re'] then
+ rspamd_logger.warnx(rspamd_config, "Cannot parse regexp '%1' for %2",
+ cur_rule['re_expr'], cur_rule['symbol'])
+ else
+ cur_rule['re']:set_max_hits(1)
+ handle_header_def(words[3], cur_rule)
+ end
+
+ if cur_rule['unset'] then
+ cur_rule['ordinary'] = false
+ end
+
+ if words[1] == 'mimeheader' then
+ cur_rule['mime'] = true
+ else
+ cur_rule['mime'] = false
+ end
+
+ if cur_rule['re'] and cur_rule['symbol'] and
+ (cur_rule['header'] or cur_rule['function']) then
+ valid_rule = true
+ cur_rule['re']:set_max_hits(1)
+ if cur_rule['header'] and cur_rule['ordinary'] then
+ for _, h in ipairs(cur_rule['header']) do
+ if type(h) == 'string' then
+ if cur_rule['mime'] then
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'mimeheader',
+ header = h,
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ else
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'header',
+ header = h,
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ end
+ else
+ h['mime'] = cur_rule['mime']
+ if cur_rule['mime'] then
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'mimeheader',
+ header = h['header'],
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ else
+ if h['raw'] then
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'rawheader',
+ header = h['header'],
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ else
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'header',
+ header = h['header'],
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ end
+ end
+ end
+ end
+ cur_rule['re']:set_limit(match_limit)
+ cur_rule['re']:set_max_hits(1)
+ end
+ end
+ else
+ -- Maybe we know the function and can convert it
+ local args = words_to_re(words, 2)
+ local func = maybe_parse_sa_function(args)
+
+ if func then
+ cur_rule['type'] = 'function'
+ cur_rule['symbol'] = words[2]
+ cur_rule['function'] = func
+ valid_rule = true
+ else
+ rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ end
+ end
+ elseif words[1] == "body" then
+ -- body SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
+ end
+
+ cur_rule['symbol'] = words[2]
+ if words[3] and (string.sub(words[3], 1, 1) == '/'
+ or string.sub(words[3], 1, 1) == 'm') then
+ cur_rule['type'] = 'sabody'
+ cur_rule['re_expr'] = words_to_re(words, 2)
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ if cur_rule['re'] then
+
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'sabody',
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ valid_rule = true
+ cur_rule['re']:set_limit(match_limit)
+ cur_rule['re']:set_max_hits(1)
+ end
+ else
+ -- might be function
+ local args = words_to_re(words, 2)
+ local func = maybe_parse_sa_function(args)
+
+ if func then
+ cur_rule['type'] = 'function'
+ cur_rule['symbol'] = words[2]
+ cur_rule['function'] = func
+ valid_rule = true
+ else
+ rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ end
+ end
+ elseif words[1] == "rawbody" then
+ -- body SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
+ end
+
+ cur_rule['symbol'] = words[2]
+ if words[3] and (string.sub(words[3], 1, 1) == '/'
+ or string.sub(words[3], 1, 1) == 'm') then
+ cur_rule['type'] = 'sarawbody'
+ cur_rule['re_expr'] = words_to_re(words, 2)
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ if cur_rule['re'] then
+
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'sarawbody',
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ valid_rule = true
+ cur_rule['re']:set_limit(match_limit)
+ cur_rule['re']:set_max_hits(1)
+ end
+ else
+ -- might be function
+ local args = words_to_re(words, 2)
+ local func = maybe_parse_sa_function(args)
+
+ if func then
+ cur_rule['type'] = 'function'
+ cur_rule['symbol'] = words[2]
+ cur_rule['function'] = func
+ valid_rule = true
+ else
+ rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ end
+ end
+ elseif words[1] == "full" then
+ -- body SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
+ end
+
+ cur_rule['symbol'] = words[2]
+
+ if words[3] and (string.sub(words[3], 1, 1) == '/'
+ or string.sub(words[3], 1, 1) == 'm') then
+ cur_rule['type'] = 'message'
+ cur_rule['re_expr'] = words_to_re(words, 2)
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ cur_rule['raw'] = true
+ if cur_rule['re'] then
+ valid_rule = true
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'body',
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ cur_rule['re']:set_limit(match_limit)
+ cur_rule['re']:set_max_hits(1)
+ end
+ else
+ -- might be function
+ local args = words_to_re(words, 2)
+ local func = maybe_parse_sa_function(args)
+
+ if func then
+ cur_rule['type'] = 'function'
+ cur_rule['symbol'] = words[2]
+ cur_rule['function'] = func
+ valid_rule = true
+ else
+ rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ end
+ end
+ elseif words[1] == "uri" then
+ -- uri SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
+ end
+ cur_rule['type'] = 'uri'
+ cur_rule['symbol'] = words[2]
+ cur_rule['re_expr'] = words_to_re(words, 2)
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ if cur_rule['re'] and cur_rule['symbol'] then
+ valid_rule = true
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'url',
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ cur_rule['re']:set_limit(match_limit)
+ cur_rule['re']:set_max_hits(1)
+ end
+ elseif words[1] == "meta" then
+ -- meta SYMBOL expression
+ if valid_rule then
+ insert_cur_rule()
+ end
+ cur_rule['type'] = 'meta'
+ cur_rule['symbol'] = words[2]
+ cur_rule['meta'] = words_to_re(words, 2)
+ if cur_rule['meta'] and cur_rule['symbol']
+ and cur_rule['meta'] ~= '0' then
+ valid_rule = true
+ end
+ elseif words[1] == "describe" and valid_rule then
+ cur_rule['description'] = words_to_re(words, 2)
+ elseif words[1] == "score" then
+ scores[words[2]] = parse_score(words)
+ elseif words[1] == 'freemail_domains' then
+ fun.each(function(dom)
+ table.insert(freemail_domains, '@' .. dom)
+ end, fun.drop_n(1, words))
+ elseif words[1] == 'blacklist_from' then
+ sa_lists['from_blacklist'][words[2]] = 1
+ sa_lists['elts'] = sa_lists['elts'] + 1
+ elseif words[1] == 'whitelist_from' then
+ sa_lists['from_whitelist'][words[2]] = 1
+ sa_lists['elts'] = sa_lists['elts'] + 1
+ elseif words[1] == 'whitelist_to' then
+ sa_lists['to_whitelist'][words[2]] = 1
+ sa_lists['elts'] = sa_lists['elts'] + 1
+ elseif words[1] == 'blacklist_to' then
+ sa_lists['to_blacklist'][words[2]] = 1
+ sa_lists['elts'] = sa_lists['elts'] + 1
+ elseif words[1] == 'tflags' then
+ process_tflags(cur_rule, words)
+ elseif words[1] == 'replace_tag' then
+ process_replace(words, replace['tags'])
+ elseif words[1] == 'replace_pre' then
+ process_replace(words, replace['pre'])
+ elseif words[1] == 'replace_inter' then
+ process_replace(words, replace['inter'])
+ elseif words[1] == 'replace_post' then
+ process_replace(words, replace['post'])
+ elseif words[1] == 'replace_rules' then
+ fun.each(function(r)
+ table.insert(replace['rules'], r)
+ end,
+ fun.drop_n(1, words))
+ end
+ end)()
+ end
+ if valid_rule then
+ insert_cur_rule()
+ end
+end
+
+-- Now check all valid rules and add the according rspamd rules
+
+local function calculate_score(sym, rule)
+ if fun.all(function(c)
+ return c == '_'
+ end, fun.take_n(2, fun.iter(sym))) then
+ return 0.0
+ end
+
+ if rule['nice'] or (rule['score'] and rule['score'] < 0.0) then
+ return -1.0
+ end
+
+ return 1.0
+end
+
+local function add_sole_meta(sym, rule)
+ local r = {
+ type = 'meta',
+ meta = rule['symbol'],
+ score = rule['score'],
+ description = rule['description']
+ }
+ rules[sym] = r
+end
+
+local function sa_regexp_match(data, re, raw, rule)
+ local res = 0
+ if not re then
+ return 0
+ end
+ if rule['multiple'] then
+ local lim = -1
+ if rule['maxhits'] then
+ lim = rule['maxhits']
+ end
+ res = res + re:matchn(data, lim, raw)
+ else
+ if re:match(data, raw) then
+ res = 1
+ end
+ end
+
+ return res
+end
+
+local function apply_replacements(str)
+ local pre = ""
+ local post = ""
+ local inter = ""
+
+ local function check_specific_tag(prefix, s, tbl)
+ local replacement = nil
+ local ret = s
+ fun.each(function(n, t)
+ local ns, matches = string.gsub(s, string.format("<%s%s>", prefix, n), "")
+ if matches > 0 then
+ replacement = t
+ ret = ns
+ end
+ end, tbl)
+
+ return ret, replacement
+ end
+
+ local repl
+ str, repl = check_specific_tag("pre ", str, replace['pre'])
+ if repl then
+ pre = repl
+ end
+ str, repl = check_specific_tag("inter ", str, replace['inter'])
+ if repl then
+ inter = repl
+ end
+ str, repl = check_specific_tag("post ", str, replace['post'])
+ if repl then
+ post = repl
+ end
+
+ -- XXX: ugly hack
+ if inter then
+ str = string.gsub(str, "><", string.format(">%s<", inter))
+ end
+
+ local function replace_all_tags(s)
+ local sstr
+ sstr = s
+ fun.each(function(n, t)
+ local rep = string.format("%s%s%s", pre, t, post)
+ rep = string.gsub(rep, '%%', '%%%%')
+ sstr = string.gsub(sstr, string.format("<%s>", n), rep)
+ end, replace['tags'])
+
+ return sstr
+ end
+
+ local s = replace_all_tags(str)
+
+ if str ~= s then
+ return true, s
+ end
+
+ return false, str
+end
+
+local function parse_atom(str)
+ local atom = table.concat(fun.totable(fun.take_while(function(c)
+ if string.find(', \t()><+!|&\n', c, 1, true) then
+ return false
+ end
+ return true
+ end, fun.iter(str))), '')
+
+ return atom
+end
+
+local function gen_process_atom_cb(result_name, task)
+ return function(atom)
+ local atom_cb = atoms[atom]
+
+ if atom_cb then
+ local res = atom_cb(task, result_name)
+
+ if not res then
+ lua_util.debugm(N, task, 'metric: %s, atom: %s, NULL result', result_name, atom)
+ elseif res > 0 then
+ lua_util.debugm(N, task, 'metric: %s, atom: %s, result: %s', result_name, atom, res)
+ end
+ return res
+ else
+ -- This is likely external atom
+ local real_sym = atom
+ if symbols_replacements[atom] then
+ real_sym = symbols_replacements[atom]
+ end
+ if task:has_symbol(real_sym, result_name) then
+ lua_util.debugm(N, task, 'external atom: %s, result: 1, named_result: %s', real_sym, result_name)
+ return 1
+ end
+ lua_util.debugm(N, task, 'external atom: %s, result: 0, , named_result: %s', real_sym, result_name)
+ end
+ return 0
+ end
+end
+
+local function post_process()
+ -- Replace rule tags
+ local ntags = {}
+ local function rec_replace_tags(tag, tagv)
+ if ntags[tag] then
+ return ntags[tag]
+ end
+ fun.each(function(n, t)
+ if n ~= tag then
+ local s, matches = string.gsub(tagv, string.format("<%s>", n), t)
+ if matches > 0 then
+ ntags[tag] = rec_replace_tags(tag, s)
+ end
+ end
+ end, replace['tags'])
+
+ if not ntags[tag] then
+ ntags[tag] = tagv
+ end
+ return ntags[tag]
+ end
+
+ fun.each(function(n, t)
+ rec_replace_tags(n, t)
+ end, replace['tags'])
+ fun.each(function(n, t)
+ replace['tags'][n] = t
+ end, ntags)
+
+ fun.each(function(r)
+ local rule = rules[r]
+
+ if rule['re_expr'] and rule['re'] then
+ local res, nexpr = apply_replacements(rule['re_expr'])
+ if res then
+ local nre = rspamd_regexp.create(nexpr)
+ if not nre then
+ rspamd_logger.errx(rspamd_config, 'cannot apply replacement for rule %1', r)
+ --rule['re'] = nil
+ else
+ local old_max_hits = rule['re']:get_max_hits()
+ lua_util.debugm(N, rspamd_config, 'replace %1 -> %2', r, nexpr)
+ rspamd_config:replace_regexp({
+ old_re = rule['re'],
+ new_re = nre,
+ pcre_only = is_pcre_only(rule['symbol']),
+ })
+ rule['re'] = nre
+ rule['re_expr'] = nexpr
+ nre:set_limit(match_limit)
+ nre:set_max_hits(old_max_hits)
+ end
+ end
+ end
+ end, replace['rules'])
+
+ fun.each(function(key, score)
+ if rules[key] then
+ rules[key]['score'] = score
+ end
+ end, scores)
+
+ -- Header rules
+ fun.each(function(k, r)
+ local f = function(task)
+
+ local raw = false
+ local check = {}
+ -- Cached path for ordinary expressions
+ if r['ordinary'] then
+ local h = r['header'][1]
+ local t = 'header'
+
+ if h['raw'] then
+ t = 'rawheader'
+ end
+
+ if not r['re'] then
+ rspamd_logger.errx(task, 're is missing for rule %1 (%2 header)', k,
+ h['header'])
+ return 0
+ end
+
+ local ret = process_regexp_opt(r.re, task, t, h.header, h.strong)
+
+ if r['not'] then
+ if ret ~= 0 then
+ ret = 0
+ else
+ ret = 1
+ end
+ end
+
+ return ret
+ end
+
+ -- Slow path
+ fun.each(function(h)
+ local hname = h['header']
+
+ local hdr
+ if h['mime'] then
+ local parts = task:get_parts()
+ for _, p in ipairs(parts) do
+ local m_hdr = p:get_header_full(hname, h['strong'])
+
+ if m_hdr then
+ if not hdr then
+ hdr = {}
+ end
+ for _, mh in ipairs(m_hdr) do
+ table.insert(hdr, mh)
+ end
+ end
+ end
+ else
+ hdr = task:get_header_full(hname, h['strong'])
+ end
+
+ if hdr then
+ for _, rh in ipairs(hdr) do
+ -- Subject for optimization
+ local str
+ if h['raw'] then
+ str = rh['value']
+ raw = true
+ else
+ str = rh['decoded']
+ end
+ if not str then
+ return 0
+ end
+
+ if h['function'] then
+ str = h['function'](str)
+ end
+
+ if type(str) == 'string' then
+ table.insert(check, str)
+ else
+ for _, c in ipairs(str) do
+ table.insert(check, c)
+ end
+ end
+ end
+ elseif r['unset'] then
+ table.insert(check, r['unset'])
+ end
+ end, r['header'])
+
+ if #check == 0 then
+ if r['not'] then
+ return 1
+ end
+ return 0
+ end
+
+ local ret = 0
+ for _, c in ipairs(check) do
+ local match = sa_regexp_match(c, r['re'], raw, r)
+ if (match > 0 and not r['not']) or (match == 0 and r['not']) then
+ ret = 1
+ end
+ end
+
+ return ret
+ end
+ if r['score'] then
+ local real_score = r['score'] * calculate_score(k, r)
+ if math.abs(real_score) > meta_score_alpha then
+ add_sole_meta(k, r)
+ end
+ end
+ atoms[k] = f
+ end,
+ fun.filter(function(_, r)
+ return r['type'] == 'header' and r['header']
+ end,
+ rules))
+
+ -- Custom function rules
+ fun.each(function(k, r)
+ local f = function(task)
+ local res = r['function'](task)
+ if res and res > 0 then
+ return res
+ end
+ return 0
+ end
+ if r['score'] then
+ local real_score = r['score'] * calculate_score(k, r)
+ if math.abs(real_score) > meta_score_alpha then
+ add_sole_meta(k, r)
+ end
+ end
+ atoms[k] = f
+ end,
+ fun.filter(function(_, r)
+ return r['type'] == 'function' and r['function']
+ end,
+ rules))
+
+ -- Parts rules
+ fun.each(function(k, r)
+ local f = function(task)
+ if not r['re'] then
+ rspamd_logger.errx(task, 're is missing for rule %1', k)
+ return 0
+ end
+
+ local t = 'mime'
+ if r['raw'] then
+ t = 'rawmime'
+ end
+
+ return process_regexp_opt(r.re, task, t)
+ end
+ if r['score'] then
+ local real_score = r['score'] * calculate_score(k, r)
+ if math.abs(real_score) > meta_score_alpha then
+ add_sole_meta(k, r)
+ end
+ end
+ atoms[k] = f
+ end,
+ fun.filter(function(_, r)
+ return r['type'] == 'part'
+ end, rules))
+
+ -- SA body rules
+ fun.each(function(k, r)
+ local f = function(task)
+ if not r['re'] then
+ rspamd_logger.errx(task, 're is missing for rule %1', k)
+ return 0
+ end
+
+ local t = r['type']
+
+ local ret = process_regexp_opt(r.re, task, t)
+ return ret
+ end
+ if r['score'] then
+ local real_score = r['score'] * calculate_score(k, r)
+ if math.abs(real_score) > meta_score_alpha then
+ add_sole_meta(k, r)
+ end
+ end
+ atoms[k] = f
+ end,
+ fun.filter(function(_, r)
+ return r['type'] == 'sabody' or r['type'] == 'message' or r['type'] == 'sarawbody'
+ end, rules))
+
+ -- URL rules
+ fun.each(function(k, r)
+ local f = function(task)
+ if not r['re'] then
+ rspamd_logger.errx(task, 're is missing for rule %1', k)
+ return 0
+ end
+
+ return process_regexp_opt(r.re, task, 'url')
+ end
+ if r['score'] then
+ local real_score = r['score'] * calculate_score(k, r)
+ if math.abs(real_score) > meta_score_alpha then
+ add_sole_meta(k, r)
+ end
+ end
+ atoms[k] = f
+ end,
+ fun.filter(function(_, r)
+ return r['type'] == 'uri'
+ end,
+ rules))
+ -- Meta rules
+ fun.each(function(k, r)
+ local expression = nil
+ -- Meta function callback
+ -- Here are dragons!
+ -- This function can be called from 2 DIFFERENT type of invocations:
+ -- 1) Invocation from Rspamd itself where `res_name` will be nil
+ -- 2) Invocation from other meta during expression:process_traced call
+ -- So we need to distinguish that and return different stuff to be able to deal with atoms
+ local meta_cb = function(task, res_name)
+ lua_util.debugm(N, task, 'meta callback for %s; result name: %s', k, res_name)
+ local cached = task:cache_get('sa_metas_processed')
+
+ -- We avoid many task methods invocations here (likely)
+ if not cached then
+ cached = {}
+ task:cache_set('sa_metas_processed', cached)
+ end
+
+ local already_processed = cached[k]
+
+ -- Exclude elements that are named in the same way as the symbol itself
+ local function exclude_sym_filter(sopt)
+ return sopt ~= k
+ end
+
+ if not (already_processed and already_processed[res_name or 'default']) then
+ -- Execute symbol
+ local function exec_symbol(cur_res)
+ local res, trace = expression:process_traced(gen_process_atom_cb(cur_res, task))
+ lua_util.debugm(N, task, 'meta result for %s: %s; result name: %s', k, res, cur_res)
+ if res > 0 then
+ -- Symbol should be one shot to make it working properly
+ task:insert_result_named(cur_res, k, res, fun.totable(fun.filter(exclude_sym_filter, trace)))
+ end
+
+ if not cached[k] then
+ cached[k] = {}
+ end
+
+ cached[k][cur_res] = res
+ end
+
+ if not res_name then
+ -- Invoke for all named results
+ local named_results = task:get_all_named_results()
+ for _, cur_res in ipairs(named_results) do
+ exec_symbol(cur_res)
+ end
+ else
+ -- Invoked from another meta
+ exec_symbol(res_name)
+ return cached[k][res_name] or 0
+ end
+ else
+ -- We have cached the result
+ local res = already_processed[res_name or 'default'] or 0
+ lua_util.debugm(N, task, 'cached meta result for %s: %s; result name: %s',
+ k, res, res_name)
+
+ if res_name then
+ return res
+ end
+ end
+
+ -- No return if invoked directly from Rspamd as we use task:insert_result_named directly
+ end
+
+ expression = rspamd_expression.create(r['meta'], parse_atom, rspamd_config:get_mempool())
+ if not expression then
+ rspamd_logger.errx(rspamd_config, 'Cannot parse expression ' .. r['meta'])
+ else
+
+ if r['score'] then
+ rspamd_config:set_metric_symbol {
+ name = k, score = r['score'],
+ description = r['description'],
+ priority = scores_priority,
+ one_shot = true
+ }
+ scores_added[k] = 1
+ rspamd_config:register_symbol {
+ name = k,
+ weight = calculate_score(k, r),
+ callback = meta_cb
+ }
+ else
+ -- Add 0 score to avoid issues
+ rspamd_config:register_symbol {
+ name = k,
+ weight = calculate_score(k, r),
+ callback = meta_cb,
+ score = 0,
+ }
+ end
+
+ r['expression'] = expression
+
+ if not atoms[k] then
+ atoms[k] = meta_cb
+ end
+ end
+ end,
+ fun.filter(function(_, r)
+ return r['type'] == 'meta'
+ end,
+ rules))
+
+ -- Check meta rules for foreign symbols and register dependencies
+ -- First direct dependencies:
+ fun.each(function(k, r)
+ if r['expression'] then
+ local expr_atoms = r['expression']:atoms()
+
+ for _, a in ipairs(expr_atoms) do
+ if not atoms[a] then
+ local rspamd_symbol = replace_symbol(a)
+ if not external_deps[k] then
+ external_deps[k] = {}
+ end
+
+ if not external_deps[k][rspamd_symbol] then
+ rspamd_config:register_dependency(k, rspamd_symbol)
+ external_deps[k][rspamd_symbol] = true
+ lua_util.debugm(N, rspamd_config,
+ 'atom %1 is a direct foreign dependency, ' ..
+ 'register dependency for %2 on %3',
+ a, k, rspamd_symbol)
+ end
+ end
+ end
+ end
+ end,
+ fun.filter(function(_, r)
+ return r['type'] == 'meta'
+ end,
+ rules))
+
+ -- ... And then indirect ones ...
+ local nchanges
+ repeat
+ nchanges = 0
+ fun.each(function(k, r)
+ if r['expression'] then
+ local expr_atoms = r['expression']:atoms()
+ for _, a in ipairs(expr_atoms) do
+ if type(external_deps[a]) == 'table' then
+ for dep in pairs(external_deps[a]) do
+ if not external_deps[k] then
+ external_deps[k] = {}
+ end
+ if not external_deps[k][dep] then
+ rspamd_config:register_dependency(k, dep)
+ external_deps[k][dep] = true
+ lua_util.debugm(N, rspamd_config,
+ 'atom %1 is an indirect foreign dependency, ' ..
+ 'register dependency for %2 on %3',
+ a, k, dep)
+ nchanges = nchanges + 1
+ end
+ end
+ else
+ local rspamd_symbol, replaced_symbol = replace_symbol(a)
+ if replaced_symbol then
+ external_deps[a] = { [rspamd_symbol] = true }
+ else
+ external_deps[a] = {}
+ end
+ end
+ end
+ end
+ end,
+ fun.filter(function(_, r)
+ return r['type'] == 'meta'
+ end,
+ rules))
+ until nchanges == 0
+
+ -- Set missing symbols
+ fun.each(function(key, score)
+ if not scores_added[key] then
+ rspamd_config:set_metric_symbol({
+ name = key, score = score,
+ priority = 2, flags = 'ignore' })
+ end
+ end, scores)
+
+ -- Logging output
+ if freemail_domains then
+ freemail_trie = rspamd_trie.create(freemail_domains)
+ rspamd_logger.infox(rspamd_config, 'loaded %1 freemail domains definitions',
+ #freemail_domains)
+ end
+ rspamd_logger.infox(rspamd_config, 'loaded %1 blacklist/whitelist elements',
+ sa_lists['elts'])
+end
+
+local has_rules = false
+
+if type(section) == "table" then
+ local keywords = {
+ pcre_only = { 'table', function(v)
+ pcre_only_regexps = lua_util.list_to_hash(v)
+ end },
+ alpha = { 'number', function(v)
+ meta_score_alpha = tonumber(v)
+ end },
+ match_limit = { 'number', function(v)
+ match_limit = tonumber(v)
+ end },
+ scores_priority = { 'number', function(v)
+ scores_priority = tonumber(v)
+ end },
+ }
+
+ for k, fn in pairs(section) do
+ local kw = keywords[k]
+ if kw and type(fn) == kw[1] then
+ kw[2](fn)
+ else
+ -- SA rule file
+ if type(fn) == 'table' then
+ for _, elt in ipairs(fn) do
+ local files = util.glob(elt)
+
+ if not files or #files == 0 then
+ rspamd_logger.errx(rspamd_config, "cannot find any files matching pattern %s", elt)
+ else
+ for _, matched in ipairs(files) do
+ local f = io.open(matched, "r")
+ if f then
+ rspamd_logger.infox(rspamd_config, 'loading SA rules from %s', matched)
+ process_sa_conf(f)
+ has_rules = true
+ else
+ rspamd_logger.errx(rspamd_config, "cannot open %1", matched)
+ end
+ end
+ end
+ end
+ else
+ -- assume string
+ local files = util.glob(fn)
+
+ if not files or #files == 0 then
+ rspamd_logger.errx(rspamd_config, "cannot find any files matching pattern %s", fn)
+ else
+ for _, matched in ipairs(files) do
+ local f = io.open(matched, "r")
+ if f then
+ rspamd_logger.infox(rspamd_config, 'loading SA rules from %s', matched)
+ process_sa_conf(f)
+ has_rules = true
+ else
+ rspamd_logger.errx(rspamd_config, "cannot open %1", matched)
+ end
+ end
+ end
+ end
+ end
+ end
+end
+
+if has_rules then
+ post_process()
+else
+ lua_util.disable_module(N, "config")
+end
diff --git a/src/plugins/lua/spamtrap.lua b/src/plugins/lua/spamtrap.lua
new file mode 100644
index 0000000..cd3b296
--- /dev/null
+++ b/src/plugins/lua/spamtrap.lua
@@ -0,0 +1,200 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- A plugin that triggers, if a spam trapped email address was detected
+
+local rspamd_logger = require "rspamd_logger"
+local redis_params
+local use_redis = false;
+local M = 'spamtrap'
+local lua_util = require "lua_util"
+local fun = require "fun"
+
+local settings = {
+ symbol = 'SPAMTRAP',
+ score = 0.0,
+ learn_fuzzy = false,
+ learn_spam = false,
+ fuzzy_flag = 1,
+ fuzzy_weight = 10.0,
+ key_prefix = 'sptr_',
+ allow_multiple_rcpts = false,
+}
+
+local check_authed = true
+local check_local = true
+
+local function spamtrap_cb(task)
+ local rcpts = task:get_recipients('smtp')
+ local authed_user = task:get_user()
+ local ip_addr = task:get_ip()
+ local called_for_domain = false
+
+ if ((not check_authed and authed_user) or
+ (not check_local and ip_addr and ip_addr:is_local())) then
+ rspamd_logger.infox(task, "skip spamtrap checks for local networks or authenticated user");
+ return
+ end
+
+ local function do_action(rcpt)
+ if settings['learn_fuzzy'] then
+ rspamd_plugins.fuzzy_check.learn(task,
+ settings['fuzzy_flag'],
+ settings['fuzzy_weight'])
+ end
+ local act_flags = ''
+ if settings['learn_spam'] then
+ task:set_flag("learn_spam")
+ -- Allow processing as we still need to learn and do other stuff
+ act_flags = 'process_all'
+ end
+ task:insert_result(settings['symbol'], 1, rcpt)
+
+ if settings.action then
+ rspamd_logger.infox(task, 'spamtrap found: <%s>', rcpt)
+ local smtp_message
+ if settings.smtp_message then
+ smtp_message = lua_util.template(settings.smtp_message, { rcpt = rcpt })
+ else
+ smtp_message = 'unknown error'
+ if settings.action == 'no action' then
+ smtp_message = 'message accepted'
+ elseif settings.action == 'reject' then
+ smtp_message = 'message rejected'
+ end
+ end
+ task:set_pre_result { action = settings.action,
+ message = smtp_message,
+ module = 'spamtrap',
+ flags = act_flags }
+ end
+
+ return true
+ end
+
+ local function gen_redis_spamtrap_cb(target)
+ return function(err, data)
+ if err ~= nil then
+ rspamd_logger.errx(task, 'redis_spamtrap_cb received error: %1', err)
+ return
+ end
+
+ if data and type(data) ~= 'userdata' then
+ do_action(target)
+ else
+ if not called_for_domain then
+ -- Recurse for @catchall domain
+ target = rcpts[1]['domain']:lower()
+ local key = settings['key_prefix'] .. '@' .. target
+ local ret = rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ gen_redis_spamtrap_cb(target), -- callback
+ 'GET', -- command
+ { key } -- arguments
+ )
+ if not ret then
+ rspamd_logger.errx(task, "redis request wasn't scheduled")
+ end
+ called_for_domain = true
+ else
+ lua_util.debugm(M, task, 'skip spamtrap for %s', target)
+ end
+ end
+ end
+ end
+
+ -- Do not risk a FP by checking for more than one recipient
+ if rcpts and (#rcpts == 1 or (#rcpts > 0 and settings.allow_multiple_rcpts)) then
+ local targets = fun.map(function(r)
+ return r['addr']:lower()
+ end, rcpts)
+ if use_redis then
+ fun.each(function(target)
+ local key = settings['key_prefix'] .. target
+ local ret = rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ gen_redis_spamtrap_cb(target), -- callback
+ 'GET', -- command
+ { key } -- arguments
+ )
+ if not ret then
+ rspamd_logger.errx(task, "redis request wasn't scheduled")
+ end
+ end, targets)
+
+ elseif settings['map'] then
+ local function check_map_functor(target)
+ if settings['map']:get_key(target) then
+ return do_action(target)
+ end
+ end
+ if not fun.any(check_map_functor, targets) then
+ lua_util.debugm(M, task, 'skip spamtrap')
+ end
+ end
+ end
+end
+
+-- Module setup
+
+local opts = rspamd_config:get_all_opt('spamtrap')
+if not (opts and type(opts) == 'table') then
+ rspamd_logger.infox(rspamd_config, 'module is unconfigured')
+ return
+end
+
+local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, 'spamtrap',
+ false, false)
+check_local = auth_and_local_conf[1]
+check_authed = auth_and_local_conf[2]
+
+if opts then
+ for k, v in pairs(opts) do
+ settings[k] = v
+ end
+ if settings['map'] then
+ settings['map'] = rspamd_config:add_map {
+ url = settings['map'],
+ description = string.format("Spamtrap map for %s", settings['symbol']),
+ type = "regexp"
+ }
+ else
+ redis_params = rspamd_parse_redis_server('spamtrap')
+ if not redis_params then
+ rspamd_logger.errx(
+ rspamd_config, 'no redis servers are specified, disabling module')
+ return
+ end
+ use_redis = true;
+ end
+
+ local id = rspamd_config:register_symbol({
+ name = "SPAMTRAP_CHECK",
+ type = "callback,postfilter",
+ callback = spamtrap_cb
+ })
+ rspamd_config:register_symbol({
+ name = settings['symbol'],
+ parent = id,
+ type = 'virtual',
+ score = settings.score
+ })
+end
diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua
new file mode 100644
index 0000000..5e15128
--- /dev/null
+++ b/src/plugins/lua/spf.lua
@@ -0,0 +1,242 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local N = "spf"
+local lua_util = require "lua_util"
+local rspamd_spf = require "rspamd_spf"
+local bit = require "bit"
+local rspamd_logger = require "rspamd_logger"
+
+if confighelp then
+ rspamd_config:add_example(nil, N,
+ 'Performs SPF checks',
+ [[
+spf {
+ # Enable module
+ enabled = true
+ # Number of elements in the cache of parsed SPF records
+ spf_cache_size = 2048;
+ # Default max expire for an element in this cache
+ spf_cache_expire = 1d;
+ # Whitelist IPs from checks
+ whitelist = "/path/to/some/file";
+ # Maximum number of recursive DNS subrequests (e.g. includes chanin length)
+ max_dns_nesting = 10;
+ # Maximum count of DNS requests per record
+ max_dns_requests = 30;
+ # Minimum TTL enforced for all elements in SPF records
+ min_cache_ttl = 5m;
+ # Disable all IPv6 lookups
+ disable_ipv6 = false;
+ # Use IP address from a received header produced by this relay (using by attribute)
+ external_relay = ["192.168.1.1"];
+}
+ ]])
+ return
+end
+
+local symbols = {
+ fail = "R_SPF_FAIL",
+ softfail = "R_SPF_SOFTFAIL",
+ neutral = "R_SPF_NEUTRAL",
+ allow = "R_SPF_ALLOW",
+ dnsfail = "R_SPF_DNSFAIL",
+ permfail = "R_SPF_PERMFAIL",
+ na = "R_SPF_NA",
+}
+
+local default_config = {
+ spf_cache_size = 2048,
+ max_dns_nesting = 10,
+ max_dns_requests = 30,
+ whitelist = nil,
+ min_cache_ttl = 60 * 5,
+ disable_ipv6 = false,
+ symbols = symbols,
+ external_relay = nil,
+}
+
+local local_config = rspamd_config:get_all_opt('spf')
+local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N,
+ false, false)
+
+if local_config then
+ local_config = lua_util.override_defaults(default_config, local_config)
+else
+ local_config = default_config
+end
+
+local function spf_check_callback(task)
+
+ local ip
+
+ if local_config.external_relay then
+ -- Search received headers to get header produced by an external relay
+ local rh = task:get_received_headers() or {}
+ local found = false
+
+ for i, hdr in ipairs(rh) do
+ if hdr.real_ip and local_config.external_relay:get_key(hdr.real_ip) then
+ -- We can use the next header as a source of IP address
+ if rh[i + 1] then
+ local nhdr = rh[i + 1]
+ lua_util.debugm(N, task, 'found external relay %s at received header number %s -> %s',
+ local_config.external_relay, i, nhdr.real_ip)
+
+ if nhdr.real_ip then
+ ip = nhdr.real_ip
+ found = true
+ end
+ end
+
+ break
+ end
+ end
+ if not found then
+ ip = task:get_from_ip()
+ rspamd_logger.warnx(task,
+ "cannot find external relay for SPF checks in received headers; use the original IP: %s",
+ tostring(ip))
+ end
+ else
+ ip = task:get_from_ip()
+ end
+
+ local function flag_to_symbol(fl)
+ if bit.band(fl, rspamd_spf.flags.temp_fail) ~= 0 then
+ return local_config.symbols.dnsfail
+ elseif bit.band(fl, rspamd_spf.flags.perm_fail) ~= 0 then
+ return local_config.symbols.permfail
+ elseif bit.band(fl, rspamd_spf.flags.na) ~= 0 then
+ return local_config.symbols.na
+ end
+
+ return 'SPF_UNKNOWN'
+ end
+
+ local function policy_decode(res)
+ if res == rspamd_spf.policy.fail then
+ return local_config.symbols.fail, '-'
+ elseif res == rspamd_spf.policy.pass then
+ return local_config.symbols.allow, '+'
+ elseif res == rspamd_spf.policy.soft_fail then
+ return local_config.symbols.softfail, '~'
+ elseif res == rspamd_spf.policy.neutral then
+ return local_config.symbols.neutral, '?'
+ end
+
+ return 'SPF_UNKNOWN', '?'
+ end
+
+ local function spf_resolved_cb(record, flags, err)
+ lua_util.debugm(N, task, 'got spf results: %s flags, %s err',
+ flags, err)
+
+ if record then
+ local result, flag_or_policy, error_or_addr = record:check_ip(ip)
+
+ lua_util.debugm(N, task,
+ 'checked ip %s: result=%s, flag_or_policy=%s, error_or_addr=%s',
+ ip, flags, err, error_or_addr)
+
+ if result then
+ local sym, code = policy_decode(flag_or_policy)
+ local opt = string.format('%s%s', code, error_or_addr.str or '???')
+ if bit.band(flags, rspamd_spf.flags.cached) ~= 0 then
+ opt = opt .. ':c'
+ rspamd_logger.infox(task,
+ "use cached record for %s (0x%s) in LRU cache for %s seconds",
+ record:get_domain(),
+ record:get_digest(),
+ record:get_ttl() - math.floor(task:get_timeval(true) -
+ record:get_timestamp()));
+ end
+ task:insert_result(sym, 1.0, opt)
+ else
+ local sym = flag_to_symbol(flag_or_policy)
+ task:insert_result(sym, 1.0, error_or_addr)
+ end
+ else
+ local sym = flag_to_symbol(flags)
+ task:insert_result(sym, 1.0, err)
+ end
+ end
+
+ if ip then
+ if local_config.whitelist and ip and local_config.whitelist:get_key(ip) then
+ rspamd_logger.infox(task, 'whitelisted SPF checks from %s',
+ tostring(ip))
+ return
+ end
+
+ if lua_util.is_skip_local_or_authed(task, auth_and_local_conf, ip) then
+ rspamd_logger.infox(task, 'skip SPF checks for local networks and authorized users')
+ return
+ end
+
+ rspamd_spf.resolve(task, spf_resolved_cb)
+ else
+ lua_util.debugm(N, task, "spf checks are not possible as no source IP address is defined")
+ end
+
+ -- FIXME: we actually need to set this variable when we really checked SPF
+ -- However, the old C module has set it all the times
+ -- Hence, we follow the same rule for now. It should be better designed at some day
+ local mpool = task:get_mempool()
+ local dmarc_checks = mpool:get_variable('dmarc_checks', 'double') or 0
+ dmarc_checks = dmarc_checks + 1
+ mpool:set_variable('dmarc_checks', dmarc_checks)
+end
+
+-- Register all symbols and init rspamd_spf library
+rspamd_spf.config(local_config)
+local sym_id = rspamd_config:register_symbol {
+ name = 'SPF_CHECK',
+ type = 'callback',
+ flags = 'fine,empty',
+ groups = { 'policies', 'spf' },
+ score = 0.0,
+ callback = spf_check_callback,
+ -- We can merely estimate timeout here, as it is possible to construct an SPF record that would cause
+ -- many DNS requests. But we won't like to set the maximum value for that all the time, as
+ -- the majority of requests will typically have 1-4 subrequests
+ augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() * 4 or 0.0) },
+}
+
+if local_config.whitelist then
+ local lua_maps = require "lua_maps"
+
+ local_config.whitelist = lua_maps.map_add_from_ucl(local_config.whitelist,
+ "radix", "SPF whitelist map")
+end
+
+if local_config.external_relay then
+ local lua_maps = require "lua_maps"
+
+ local_config.external_relay = lua_maps.map_add_from_ucl(local_config.external_relay,
+ "radix", "External IP SPF map")
+end
+
+for _, sym in pairs(local_config.symbols) do
+ rspamd_config:register_symbol {
+ name = sym,
+ type = 'virtual',
+ parent = sym_id,
+ groups = { 'policies', 'spf' },
+ }
+end
+
+
diff --git a/src/plugins/lua/trie.lua b/src/plugins/lua/trie.lua
new file mode 100644
index 0000000..7ba4552
--- /dev/null
+++ b/src/plugins/lua/trie.lua
@@ -0,0 +1,184 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+-- Trie is rspamd module designed to define and operate with suffix trie
+
+local N = 'trie'
+local rspamd_logger = require "rspamd_logger"
+local rspamd_trie = require "rspamd_trie"
+local fun = require "fun"
+local lua_util = require "lua_util"
+
+local mime_trie
+local raw_trie
+local body_trie
+
+-- here we store all patterns as text
+local mime_patterns = {}
+local raw_patterns = {}
+local body_patterns = {}
+
+-- here we store params for each pattern, so for each i = 1..n patterns[i]
+-- should have corresponding params[i]
+local mime_params = {}
+local raw_params = {}
+local body_params = {}
+
+local function tries_callback(task)
+
+ local matched = {}
+
+ local function gen_trie_cb(type)
+ local patterns = mime_patterns
+ local params = mime_params
+ if type == 'rawmessage' then
+ patterns = raw_patterns
+ params = raw_params
+ elseif type == 'rawbody' then
+ patterns = body_patterns
+ params = body_params
+ end
+
+ return function(idx, pos)
+ local param = params[idx]
+ local pattern = patterns[idx]
+ local pattern_idx = pattern .. tostring(idx) .. type
+
+ if param['multi'] or not matched[pattern_idx] then
+ lua_util.debugm(N, task, "<%1> matched pattern %2 at pos %3",
+ task:get_message_id(), pattern, pos)
+ task:insert_result(param['symbol'], 1.0, type)
+ if not param['multi'] then
+ matched[pattern_idx] = true
+ end
+ end
+ end
+ end
+
+ if mime_trie then
+ mime_trie:search_mime(task, gen_trie_cb('mime'))
+ end
+ if raw_trie then
+ raw_trie:search_rawmsg(task, gen_trie_cb('rawmessage'))
+ end
+ if body_trie then
+ body_trie:search_rawbody(task, gen_trie_cb('rawbody'))
+ end
+end
+
+local function process_single_pattern(pat, symbol, cf)
+ if pat then
+ local multi = false
+ if cf['multi'] then
+ multi = true
+ end
+
+ if cf['raw'] then
+ table.insert(raw_patterns, pat)
+ table.insert(raw_params, { symbol = symbol, multi = multi })
+ elseif cf['body'] then
+ table.insert(body_patterns, pat)
+ table.insert(body_params, { symbol = symbol, multi = multi })
+ else
+ table.insert(mime_patterns, pat)
+ table.insert(mime_params, { symbol = symbol, multi = multi })
+ end
+ end
+end
+
+local function process_trie_file(symbol, cf)
+ local file = io.open(cf['file'])
+
+ if not file then
+ rspamd_logger.errx(rspamd_config, 'Cannot open trie file %1', cf['file'])
+ else
+ if cf['binary'] then
+ rspamd_logger.errx(rspamd_config, 'binary trie patterns are not implemented yet: %1',
+ cf['file'])
+ else
+ for line in file:lines() do
+ local pat = string.match(line, '^([^#].*[^%s])%s*$')
+ process_single_pattern(pat, symbol, cf)
+ end
+ end
+ end
+end
+
+local function process_trie_conf(symbol, cf)
+ if type(cf) ~= 'table' then
+ rspamd_logger.errx(rspamd_config, 'invalid value for symbol %1: "%2", expected table',
+ symbol, cf)
+ return
+ end
+
+ if cf['file'] then
+ process_trie_file(symbol, cf)
+ elseif cf['patterns'] then
+ fun.each(function(pat)
+ process_single_pattern(pat, symbol, cf)
+ end, cf['patterns'])
+ end
+end
+
+local opts = rspamd_config:get_all_opt("trie")
+if opts then
+ for sym, opt in pairs(opts) do
+ process_trie_conf(sym, opt)
+ end
+
+ if #raw_patterns > 0 then
+ raw_trie = rspamd_trie.create(raw_patterns)
+ rspamd_logger.infox(rspamd_config, 'registered raw search trie from %1 patterns', #raw_patterns)
+ end
+
+ if #mime_patterns > 0 then
+ mime_trie = rspamd_trie.create(mime_patterns)
+ rspamd_logger.infox(rspamd_config, 'registered mime search trie from %1 patterns', #mime_patterns)
+ end
+
+ if #body_patterns > 0 then
+ body_trie = rspamd_trie.create(body_patterns)
+ rspamd_logger.infox(rspamd_config, 'registered body search trie from %1 patterns', #body_patterns)
+ end
+
+ local id = -1
+ if mime_trie or raw_trie or body_trie then
+ id = rspamd_config:register_symbol({
+ name = 'TRIE_CALLBACK',
+ type = 'callback',
+ callback = tries_callback
+ })
+ else
+ rspamd_logger.infox(rspamd_config, 'no tries defined')
+ end
+
+ if id ~= -1 then
+ for sym in pairs(opts) do
+ rspamd_config:register_symbol({
+ name = sym,
+ type = 'virtual',
+ parent = id
+ })
+ end
+ end
+else
+ rspamd_logger.infox(rspamd_config, "Module is unconfigured")
+ lua_util.disable_module(N, "config")
+end
diff --git a/src/plugins/lua/url_redirector.lua b/src/plugins/lua/url_redirector.lua
new file mode 100644
index 0000000..10b5fb2
--- /dev/null
+++ b/src/plugins/lua/url_redirector.lua
@@ -0,0 +1,422 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_http = require "rspamd_http"
+local hash = require "rspamd_cryptobox_hash"
+local rspamd_url = require "rspamd_url"
+local lua_util = require "lua_util"
+local lua_redis = require "lua_redis"
+local N = "url_redirector"
+
+-- Some popular UA
+local default_ua = {
+ 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)',
+ 'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)',
+ 'Wget/1.9.1',
+ 'Mozilla/5.0 (Android; Linux armv7l; rv:9.0) Gecko/20111216 Firefox/9.0 Fennec/9.0',
+ 'Mozilla/5.0 (Windows NT 5.2; RW; rv:7.0a1) Gecko/20091211 SeaMonkey/9.23a1pre',
+ 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko',
+ 'W3C-checklink/4.5 [4.160] libwww-perl/5.823',
+ 'Lynx/2.8.8dev.3 libwww-FM/2.14 SSL-MM/1.4.1',
+}
+
+local redis_params
+
+local settings = {
+ expire = 86400, -- 1 day by default
+ timeout = 10, -- 10 seconds by default
+ nested_limit = 5, -- How many redirects to follow
+ --proxy = "http://example.com:3128", -- Send request through proxy
+ key_prefix = 'rdr:', -- default hash name
+ check_ssl = false, -- check ssl certificates
+ max_urls = 5, -- how many urls to check
+ max_size = 10 * 1024, -- maximum body to process
+ user_agent = default_ua,
+ redirector_symbol = nil, -- insert symbol if redirected url has been found
+ redirector_symbol_nested = "URL_REDIRECTOR_NESTED", -- insert symbol if nested limit has been reached
+ redirectors_only = true, -- follow merely redirectors
+ top_urls_key = 'rdr:top_urls', -- key for top urls
+ top_urls_count = 200, -- how many top urls to save
+ redirector_hosts_map = nil -- check only those redirectors
+}
+
+local function adjust_url(task, orig_url, redir_url)
+ local mempool = task:get_mempool()
+ if type(redir_url) == 'string' then
+ redir_url = rspamd_url.create(mempool, redir_url, { 'redirect_target' })
+ end
+
+ if redir_url then
+ orig_url:set_redirected(redir_url, mempool)
+ task:inject_url(redir_url)
+ if settings.redirector_symbol then
+ task:insert_result(settings.redirector_symbol, 1.0,
+ string.format('%s->%s', orig_url:get_host(), redir_url:get_host()))
+ end
+ else
+ rspamd_logger.infox(task, 'bad url %s as redirection for %s', redir_url, orig_url)
+ end
+end
+
+local function cache_url(task, orig_url, url, key, prefix)
+ -- String representation
+ local str_orig_url = tostring(orig_url)
+ local str_url = tostring(url)
+
+ if str_url ~= str_orig_url then
+ -- Set redirected url
+ adjust_url(task, orig_url, url)
+ end
+
+ local function redis_trim_cb(err, _)
+ if err then
+ rspamd_logger.errx(task, 'got error while getting top urls count: %s', err)
+ else
+ rspamd_logger.infox(task, 'trimmed url set to %s elements',
+ settings.top_urls_count)
+ end
+ end
+
+ -- Cleanup logic
+ local function redis_card_cb(err, data)
+ if err then
+ rspamd_logger.errx(task, 'got error while getting top urls count: %s', err)
+ else
+ if data then
+ if tonumber(data) > settings.top_urls_count * 2 then
+ local ret = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_trim_cb, --callback
+ 'ZREMRANGEBYRANK', -- command
+ { settings.top_urls_key, '0',
+ tostring(-(settings.top_urls_count + 1)) } -- arguments
+ )
+ if not ret then
+ rspamd_logger.errx(task, 'cannot trim top urls set')
+ else
+ rspamd_logger.infox(task, 'need to trim urls set from %s to %s elements',
+ data,
+ settings.top_urls_count)
+ return
+ end
+ end
+ end
+ end
+ end
+
+ local function redis_set_cb(err, _)
+ if err then
+ rspamd_logger.errx(task, 'got error while setting redirect keys: %s', err)
+ else
+ local ret = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_card_cb, --callback
+ 'ZCARD', -- command
+ { settings.top_urls_key } -- arguments
+ )
+ if not ret then
+ rspamd_logger.errx(task, 'cannot make redis request to cache results')
+ end
+ end
+ end
+
+ if prefix then
+ -- Save url with prefix
+ str_url = string.format('^%s:%s', prefix, str_url)
+ end
+ local ret, conn, _ = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'SETEX', -- command
+ { key, tostring(settings.expire), str_url } -- arguments
+ )
+
+ if not ret then
+ rspamd_logger.errx(task, 'cannot make redis request to cache results')
+ else
+ conn:add_cmd('ZINCRBY', { settings.top_urls_key, '1', str_url })
+ end
+end
+
+-- Reduce length of a string to a given length (16 by default)
+local function maybe_trim_url(url, limit)
+ if not limit then
+ limit = 16
+ end
+ if #url > limit then
+ return string.sub(url, 1, limit) .. '...'
+ else
+ return url
+ end
+end
+
+-- Resolve maybe cached url
+-- Orig url is the original url object
+-- url should be a new url object...
+local function resolve_cached(task, orig_url, url, key, ntries)
+ local str_url = tostring(url or "")
+ local function resolve_url()
+ if ntries > settings.nested_limit then
+ -- We cannot resolve more, stop
+ rspamd_logger.debugm(N, task, 'cannot get more requests to resolve %s, stop on %s after %s attempts',
+ orig_url, url, ntries)
+ cache_url(task, orig_url, url, key, 'nested')
+ local str_orig_url = tostring(orig_url)
+ task:insert_result(settings.redirector_symbol_nested, 1.0,
+ string.format('%s->%s:%d', maybe_trim_url(str_orig_url), maybe_trim_url(str_url), ntries))
+
+ return
+ end
+
+ local redirection_codes = {
+ [301] = true, -- moved permanently
+ [302] = true, -- found
+ [303] = true, -- see other
+ [307] = true, -- temporary redirect
+ [308] = true, -- permanent redirect
+ }
+
+ local function http_callback(err, code, _, headers)
+ if err then
+ rspamd_logger.infox(task, 'found redirect error from %s to %s, err message: %s',
+ orig_url, url, err)
+ cache_url(task, orig_url, url, key)
+ else
+ if code == 200 then
+ if orig_url == url then
+ rspamd_logger.infox(task, 'direct url %s, err code 200',
+ url)
+ else
+ rspamd_logger.infox(task, 'found redirect from %s to %s, err code 200',
+ orig_url, url)
+ end
+
+ cache_url(task, orig_url, url, key)
+
+ elseif redirection_codes[code] then
+ local loc = headers['location']
+ local redir_url
+ if loc then
+ redir_url = rspamd_url.create(task:get_mempool(), loc)
+ end
+ rspamd_logger.debugm(N, task, 'found redirect from %s to %s, err code %s',
+ orig_url, loc, code)
+
+ if redir_url then
+ if settings.redirectors_only then
+ if settings.redirector_hosts_map:get_key(redir_url:get_host()) then
+ resolve_cached(task, orig_url, redir_url, key, ntries + 1)
+ else
+ lua_util.debugm(N, task,
+ "stop resolving redirects as %s is not a redirector", loc)
+ cache_url(task, orig_url, redir_url, key)
+ end
+ else
+ resolve_cached(task, orig_url, redir_url, key, ntries + 1)
+ end
+ else
+ rspamd_logger.debugm(N, task, "no location, headers: %s", headers)
+ cache_url(task, orig_url, url, key)
+ end
+ else
+ rspamd_logger.debugm(N, task, 'found redirect error from %s to %s, err code: %s',
+ orig_url, url, code)
+ cache_url(task, orig_url, url, key)
+ end
+ end
+ end
+
+ local ua
+ if type(settings.user_agent) == 'string' then
+ ua = settings.user_agent
+ else
+ ua = settings.user_agent[math.random(#settings.user_agent)]
+ end
+
+ lua_util.debugm(N, task, 'select user agent %s', ua)
+
+ rspamd_http.request {
+ headers = {
+ ['User-Agent'] = ua,
+ },
+ url = str_url,
+ task = task,
+ method = 'head',
+ max_size = settings.max_size,
+ timeout = settings.timeout,
+ opaque_body = true,
+ no_ssl_verify = not settings.check_ssl,
+ callback = http_callback
+ }
+ end
+ local function redis_get_cb(err, data)
+ if not err then
+ if type(data) == 'string' then
+ if data ~= 'processing' then
+ -- Got cached result
+ rspamd_logger.debugm(N, task, 'found cached redirect from %s to %s',
+ url, data)
+ if data:sub(1, 1) == '^' then
+ -- Prefixed url stored
+ local prefix, new_url = data:match('^%^(%a+):(.+)$')
+ if prefix == 'nested' then
+ task:insert_result(settings.redirector_symbol_nested, 1.0,
+ string.format('%s->%s:cached', maybe_trim_url(str_url), maybe_trim_url(new_url)))
+ end
+ data = new_url
+ end
+ if data ~= tostring(orig_url) then
+ adjust_url(task, orig_url, data)
+ end
+ return
+ end
+ end
+ end
+ local function redis_reserve_cb(nerr, ndata)
+ if nerr then
+ rspamd_logger.errx(task, 'got error while setting redirect keys: %s', nerr)
+ elseif ndata == 'OK' then
+ resolve_url()
+ end
+ end
+
+ if ntries == 1 then
+ -- Reserve key in Redis that we are processing this redirection
+ local ret = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_reserve_cb, --callback
+ 'SET', -- command
+ { key, 'processing', 'EX', tostring(settings.timeout * 2), 'NX' } -- arguments
+ )
+ if not ret then
+ rspamd_logger.errx(task, 'Couldn\'t schedule SET')
+ end
+ else
+ -- Just continue resolving
+ resolve_url()
+ end
+
+ end
+ local ret = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_get_cb, --callback
+ 'GET', -- command
+ { key } -- arguments
+ )
+ if not ret then
+ rspamd_logger.errx(task, 'cannot make redis request to check results')
+ end
+end
+
+local function url_redirector_process_url(task, url)
+ local url_str = url:get_raw()
+ -- 32 base32 characters are roughly 20 bytes of data or 160 bits
+ local key = settings.key_prefix .. hash.create(url_str):base32():sub(1, 32)
+ resolve_cached(task, url, url, key, 1)
+end
+
+local function url_redirector_handler(task)
+ local sp_urls = lua_util.extract_specific_urls({
+ task = task,
+ limit = settings.max_urls,
+ filter = function(url)
+ local host = url:get_host()
+ if settings.redirector_hosts_map:get_key(host) then
+ lua_util.debugm(N, task, 'check url %s', tostring(url))
+ return true
+ end
+ end,
+ no_cache = true,
+ need_content = true,
+ })
+
+ if sp_urls then
+ for _, u in ipairs(sp_urls) do
+ url_redirector_process_url(task, u)
+ end
+ end
+end
+
+local opts = rspamd_config:get_all_opt('url_redirector')
+if opts then
+ settings = lua_util.override_defaults(settings, opts)
+ redis_params = lua_redis.parse_redis_server('url_redirector', settings)
+
+ if not redis_params then
+ rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
+ lua_util.disable_module(N, "redis")
+ else
+
+ if not settings.redirector_hosts_map then
+ rspamd_logger.infox(rspamd_config, 'no redirector_hosts_map option is specified, disabling module')
+ lua_util.disable_module(N, "config")
+ else
+ local lua_maps = require "lua_maps"
+ settings.redirector_hosts_map = lua_maps.map_add_from_ucl(settings.redirector_hosts_map,
+ 'set', 'Redirectors definitions')
+
+ lua_redis.register_prefix(settings.key_prefix .. '[a-z0-9]{32}', N,
+ 'URL redirector hashes', {
+ type = 'string',
+ })
+ if settings.top_urls_key then
+ lua_redis.register_prefix(settings.top_urls_key, N,
+ 'URL redirector top urls', {
+ type = 'zlist',
+ })
+ end
+ local id = rspamd_config:register_symbol {
+ name = 'URL_REDIRECTOR_CHECK',
+ type = 'callback,prefilter',
+ priority = lua_util.symbols_priorities.medium,
+ callback = url_redirector_handler,
+ -- In fact, the real timeout is nested_limit * timeout...
+ augmentations = { string.format("timeout=%f", settings.timeout) }
+ }
+
+ rspamd_config:register_symbol {
+ name = settings.redirector_symbol_nested,
+ type = 'virtual',
+ parent = id,
+ score = 0,
+ }
+
+ if settings.redirector_symbol then
+ rspamd_config:register_symbol {
+ name = settings.redirector_symbol,
+ type = 'virtual',
+ parent = id,
+ score = 0,
+ }
+ end
+ end
+ end
+end
diff --git a/src/plugins/lua/whitelist.lua b/src/plugins/lua/whitelist.lua
new file mode 100644
index 0000000..fa76da8
--- /dev/null
+++ b/src/plugins/lua/whitelist.lua
@@ -0,0 +1,443 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+ return
+end
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local fun = require "fun"
+local lua_util = require "lua_util"
+
+local N = "whitelist"
+
+local options = {
+ dmarc_allow_symbol = 'DMARC_POLICY_ALLOW',
+ spf_allow_symbol = 'R_SPF_ALLOW',
+ dkim_allow_symbol = 'R_DKIM_ALLOW',
+ check_local = false,
+ check_authed = false,
+ rules = {}
+}
+
+local E = {}
+
+local function whitelist_cb(symbol, rule, task)
+
+ local domains = {}
+
+ local function find_domain(dom, check)
+ local mult
+ local how = 'wl'
+
+ -- Can be overridden
+ if rule.blacklist then
+ how = 'bl'
+ end
+
+ local function parse_val(val)
+ local how_override
+ -- Strict is 'special'
+ if rule.strict then
+ how_override = 'both'
+ end
+ if val then
+ lua_util.debugm(N, task, "found whitelist key: %s=%s", dom, val)
+ if val == '' then
+ return (how_override or how), 1.0
+ elseif val:match('^bl:') then
+ return (how_override or 'bl'), (tonumber(val:sub(4)) or 1.0)
+ elseif val:match('^wl:') then
+ return (how_override or 'wl'), (tonumber(val:sub(4)) or 1.0)
+ elseif val:match('^both:') then
+ return (how_override or 'both'), (tonumber(val:sub(6)) or 1.0)
+ else
+ return (how_override or how), (tonumber(val) or 1.0)
+ end
+ end
+
+ return (how_override or how), 1.0
+ end
+
+ if rule['map'] then
+ local val = rule['map']:get_key(dom)
+ if val then
+ how, mult = parse_val(val)
+
+ if not domains[check] then
+ domains[check] = {}
+ end
+
+ domains[check] = {
+ [dom] = { how, mult }
+ }
+
+ lua_util.debugm(N, task, "final result: %s: %s->%s",
+ dom, how, mult)
+ return true, mult, how
+ end
+ elseif rule['maps'] then
+ for _, v in pairs(rule['maps']) do
+ local map = v.map
+ if map then
+ local val = map:get_key(dom)
+ if val then
+ how, mult = parse_val(val)
+
+ if not domains[check] then
+ domains[check] = {}
+ end
+
+ domains[check] = {
+ [dom] = { how, mult }
+ }
+
+ lua_util.debugm(N, task, "final result: %s: %s->%s",
+ dom, how, mult)
+ return true, mult, how
+ end
+ end
+ end
+ else
+ mult = rule['domains'][dom]
+ if mult then
+ if not domains[check] then
+ domains[check] = {}
+ end
+
+ domains[check] = {
+ [dom] = { how, mult }
+ }
+
+ return true, mult, how
+ end
+ end
+
+ return false, 0.0, how
+ end
+
+ local spf_violated = false
+ local dmarc_violated = false
+ local dkim_violated = false
+ local ip_addr = task:get_ip()
+
+ if rule.valid_spf then
+ if not task:has_symbol(options['spf_allow_symbol']) then
+ -- Not whitelisted
+ spf_violated = true
+ end
+ -- Now we can check from domain or helo
+ local from = task:get_from(1)
+
+ if ((from or E)[1] or E).domain then
+ local tld = rspamd_util.get_tld(from[1]['domain'])
+
+ if tld then
+ find_domain(tld, 'spf')
+ end
+ else
+ local helo = task:get_helo()
+
+ if helo then
+ local tld = rspamd_util.get_tld(helo)
+
+ if tld then
+ find_domain(tld, 'spf')
+ end
+ end
+ end
+ end
+
+ if rule.valid_dkim then
+ if task:has_symbol('DKIM_TRACE') then
+ local sym = task:get_symbol('DKIM_TRACE')
+ local dkim_opts = sym[1]['options']
+ if dkim_opts then
+ fun.each(function(val)
+ if val[2] == '+' then
+ local tld = rspamd_util.get_tld(val[1])
+ find_domain(tld, 'dkim_success')
+ elseif val[2] == '-' then
+ local tld = rspamd_util.get_tld(val[1])
+ find_domain(tld, 'dkim_fail')
+ end
+ end,
+ fun.map(function(s)
+ return lua_util.rspamd_str_split(s, ':')
+ end, dkim_opts))
+ end
+ end
+ end
+
+ if rule.valid_dmarc then
+ if not task:has_symbol(options.dmarc_allow_symbol) then
+ dmarc_violated = true
+ end
+
+ local from = task:get_from(2)
+
+ if ((from or E)[1] or E).domain then
+ local tld = rspamd_util.get_tld(from[1]['domain'])
+
+ if tld then
+ local found = find_domain(tld, 'dmarc')
+ if not found then
+ find_domain(from[1]['domain'], 'dmarc')
+ end
+ end
+ end
+ end
+
+ local final_mult = 1.0
+ local found_wl, found_bl = false, false
+ local opts = {}
+
+ if rule.valid_dkim then
+ dkim_violated = true
+
+ for dom, val in pairs(domains.dkim_success or E) do
+ if val[1] == 'wl' or val[1] == 'both' then
+ -- We have valid and whitelisted signature
+ table.insert(opts, dom .. ':d:+')
+ found_wl = true
+ dkim_violated = false
+
+ if not found_bl then
+ final_mult = val[2]
+ end
+ end
+ end
+
+ -- Blacklist counterpart
+ for dom, val in pairs(domains.dkim_fail or E) do
+ if val[1] == 'bl' or val[1] == 'both' then
+ -- We have valid and whitelisted signature
+ table.insert(opts, dom .. ':d:-')
+ found_bl = true
+ final_mult = val[2]
+ else
+ -- Even in the case of whitelisting we need to indicate dkim failure
+ dkim_violated = true
+ end
+ end
+ end
+
+ local function check_domain_violation(what, dom, val, violated)
+ if violated then
+ if val[1] == 'both' or val[1] == 'bl' then
+ found_bl = true
+ final_mult = val[2]
+ table.insert(opts, string.format("%s:%s:-", dom, what))
+ end
+ else
+ if val[1] == 'both' or val[1] == 'wl' then
+ found_wl = true
+ table.insert(opts, string.format("%s:%s:+", dom, what))
+ if not found_bl then
+ final_mult = val[2]
+ end
+ end
+ end
+ end
+
+ if rule.valid_dmarc then
+
+ found_wl = false
+
+ for dom, val in pairs(domains.dmarc or E) do
+ check_domain_violation('D', dom, val,
+ (dmarc_violated or dkim_violated))
+ end
+ end
+
+ if rule.valid_spf then
+ found_wl = false
+
+ for dom, val in pairs(domains.spf or E) do
+ check_domain_violation('s', dom, val,
+ (spf_violated or dkim_violated))
+ end
+ end
+
+ lua_util.debugm(N, task, "final mult: %s", final_mult)
+
+ local function add_symbol(violated, mult)
+ local sym = symbol
+
+ if violated then
+ if rule.inverse_symbol then
+ sym = rule.inverse_symbol
+ elseif not rule.blacklist then
+ mult = -mult
+ end
+
+ if rule.inverse_multiplier then
+ mult = mult * rule.inverse_multiplier
+ end
+
+ task:insert_result(sym, mult, opts)
+ else
+ task:insert_result(sym, mult, opts)
+ end
+ end
+
+ if found_bl then
+ if not ((not options.check_authed and task:get_user()) or
+ (not options.check_local and ip_addr and ip_addr:is_local())) then
+ add_symbol(true, final_mult)
+ else
+ if rule.valid_spf or rule.valid_dmarc then
+ rspamd_logger.infox(task, "skip DMARC/SPF blacklists for local networks and/or authorized users")
+ else
+ add_symbol(true, final_mult)
+ end
+ end
+ elseif found_wl then
+ add_symbol(false, final_mult)
+ end
+
+end
+
+local function gen_whitelist_cb(symbol, rule)
+ return function(task)
+ whitelist_cb(symbol, rule, task)
+ end
+end
+
+local configure_whitelist_module = function()
+ local opts = rspamd_config:get_all_opt('whitelist')
+ if opts then
+ for k, v in pairs(opts) do
+ options[k] = v
+ end
+
+ local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N,
+ false, false)
+ options.check_local = auth_and_local_conf[1]
+ options.check_authed = auth_and_local_conf[2]
+ else
+ rspamd_logger.infox(rspamd_config, 'Module is unconfigured')
+ return
+ end
+
+ if options['rules'] then
+ fun.each(function(symbol, rule)
+ if rule['domains'] then
+ if type(rule['domains']) == 'string' then
+ rule['map'] = rspamd_config:add_map {
+ url = rule['domains'],
+ description = "Whitelist map for " .. symbol,
+ type = 'map'
+ }
+ elseif type(rule['domains']) == 'table' then
+ -- Transform ['domain1', 'domain2' ...] to indexes:
+ -- {'domain1' = 1, 'domain2' = 1 ...]
+ local is_domains_list = fun.all(function(v)
+ if type(v) == 'table' then
+ return true
+ elseif type(v) == 'string' and not (string.match(v, '^https?://') or
+ string.match(v, '^ftp://') or string.match(v, '^[./]')) then
+ return true
+ end
+
+ return false
+ end, rule.domains)
+
+ if is_domains_list then
+ rule['domains'] = fun.tomap(fun.map(function(d)
+ if type(d) == 'table' then
+ return d[1], d[2]
+ end
+
+ return d, 1.0
+ end, rule['domains']))
+ else
+ rule['map'] = rspamd_config:add_map {
+ url = rule['domains'],
+ description = "Whitelist map for " .. symbol,
+ type = 'map'
+ }
+ end
+ else
+ rspamd_logger.errx(rspamd_config, 'whitelist %s has bad "domains" value',
+ symbol)
+ return
+ end
+
+ local flags = 'nice,empty'
+ if rule['blacklist'] then
+ flags = 'empty'
+ end
+
+ local id = rspamd_config:register_symbol({
+ name = symbol,
+ flags = flags,
+ callback = gen_whitelist_cb(symbol, rule),
+ score = rule.score or 0,
+ })
+
+ if rule.inverse_symbol then
+ rspamd_config:register_symbol({
+ name = rule.inverse_symbol,
+ type = 'virtual',
+ parent = id,
+ score = rule.score and -(rule.score) or 0,
+ })
+ end
+
+ local spf_dep = false
+ local dkim_dep = false
+ if rule['valid_spf'] then
+ rspamd_config:register_dependency(symbol, options['spf_allow_symbol'])
+ spf_dep = true
+ end
+ if rule['valid_dkim'] then
+ rspamd_config:register_dependency(symbol, options['dkim_allow_symbol'])
+ dkim_dep = true
+ end
+ if rule['valid_dmarc'] then
+ if not spf_dep then
+ rspamd_config:register_dependency(symbol, options['spf_allow_symbol'])
+ end
+ if not dkim_dep then
+ rspamd_config:register_dependency(symbol, options['dkim_allow_symbol'])
+ end
+ rspamd_config:register_dependency(symbol, 'DMARC_CALLBACK')
+ end
+
+ if rule['score'] then
+ if not rule['group'] then
+ rule['group'] = 'whitelist'
+ end
+ rule['name'] = symbol
+ rspamd_config:set_metric_symbol(rule)
+
+ if rule.inverse_symbol then
+ local inv_rule = lua_util.shallowcopy(rule)
+ inv_rule.name = rule.inverse_symbol
+ inv_rule.score = -rule.score
+ rspamd_config:set_metric_symbol(inv_rule)
+ end
+ end
+ end
+ end, options['rules'])
+ else
+ lua_util.disable_module(N, "config")
+ end
+end
+
+configure_whitelist_module()
diff --git a/src/plugins/regexp.c b/src/plugins/regexp.c
new file mode 100644
index 0000000..59a84c5
--- /dev/null
+++ b/src/plugins/regexp.c
@@ -0,0 +1,564 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/***MODULE:regexp
+ * rspamd module that implements different regexp rules
+ */
+
+
+#include "config.h"
+#include "libmime/message.h"
+#include "expression.h"
+#include "mime_expressions.h"
+#include "libserver/maps/map.h"
+#include "lua/lua_common.h"
+
+static const guint64 rspamd_regexp_cb_magic = 0xca9d9649fc3e2659ULL;
+
+struct regexp_module_item {
+ guint64 magic;
+ struct rspamd_expression *expr;
+ const gchar *symbol;
+ struct ucl_lua_funcdata *lua_function;
+};
+
+struct regexp_ctx {
+ struct module_ctx ctx;
+ gsize max_size;
+};
+
+static void process_regexp_item(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ void *user_data);
+
+
+/* Initialization */
+gint regexp_module_init(struct rspamd_config *cfg, struct module_ctx **ctx);
+gint regexp_module_config(struct rspamd_config *cfg, bool validate);
+gint regexp_module_reconfig(struct rspamd_config *cfg);
+
+module_t regexp_module = {
+ "regexp",
+ regexp_module_init,
+ regexp_module_config,
+ regexp_module_reconfig,
+ NULL,
+ RSPAMD_MODULE_VER,
+ (guint) -1,
+};
+
+
+static inline struct regexp_ctx *
+regexp_get_context(struct rspamd_config *cfg)
+{
+ return (struct regexp_ctx *) g_ptr_array_index(cfg->c_modules,
+ regexp_module.ctx_offset);
+}
+
+/* Process regexp expression */
+static gboolean
+read_regexp_expression(rspamd_mempool_t *pool,
+ struct regexp_module_item *chain,
+ const gchar *symbol,
+ const gchar *line,
+ struct rspamd_mime_expr_ud *ud)
+{
+ struct rspamd_expression *e = NULL;
+ GError *err = NULL;
+
+ if (!rspamd_parse_expression(line, 0, &mime_expr_subr, ud, pool, &err,
+ &e)) {
+ msg_warn_pool("%s = \"%s\" is invalid regexp expression: %e", symbol,
+ line,
+ err);
+ g_error_free(err);
+
+ return FALSE;
+ }
+
+ g_assert(e != NULL);
+ chain->expr = e;
+
+ return TRUE;
+}
+
+
+/* Init function */
+gint regexp_module_init(struct rspamd_config *cfg, struct module_ctx **ctx)
+{
+ struct regexp_ctx *regexp_module_ctx;
+
+ regexp_module_ctx = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(*regexp_module_ctx));
+
+ *ctx = (struct module_ctx *) regexp_module_ctx;
+
+ rspamd_rcl_add_doc_by_path(cfg,
+ NULL,
+ "Regular expressions rules plugin",
+ "regexp",
+ UCL_OBJECT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+
+ rspamd_rcl_add_doc_by_path(cfg,
+ "regexp",
+ "Maximum size of data chunk scanned with any regexp (further data is truncated)",
+ "max_size",
+ UCL_INT,
+ NULL,
+ 0,
+ NULL,
+ 0);
+
+ return 0;
+}
+
+gint regexp_module_config(struct rspamd_config *cfg, bool validate)
+{
+ struct regexp_ctx *regexp_module_ctx = regexp_get_context(cfg);
+ struct regexp_module_item *cur_item = NULL;
+ const ucl_object_t *sec, *value, *elt;
+ ucl_object_iter_t it = NULL;
+ gint res = TRUE, nre = 0, nlua = 0, nshots = cfg->default_max_shots;
+
+ if (!rspamd_config_is_module_enabled(cfg, "regexp")) {
+ return TRUE;
+ }
+
+ sec = ucl_object_lookup(cfg->cfg_ucl_obj, "regexp");
+ if (sec == NULL) {
+ msg_err_config("regexp module enabled, but no rules are defined");
+ return TRUE;
+ }
+
+ regexp_module_ctx->max_size = 0;
+
+ while ((value = ucl_object_iterate(sec, &it, true)) != NULL) {
+ if (g_ascii_strncasecmp(ucl_object_key(value), "max_size",
+ sizeof("max_size") - 1) == 0) {
+ regexp_module_ctx->max_size = ucl_obj_toint(value);
+ rspamd_re_cache_set_limit(cfg->re_cache, regexp_module_ctx->max_size);
+ }
+ else if (g_ascii_strncasecmp(ucl_object_key(value), "max_threads",
+ sizeof("max_threads") - 1) == 0) {
+ msg_warn_config("regexp module is now single threaded, max_threads is ignored");
+ }
+ else if (value->type == UCL_STRING) {
+ struct rspamd_mime_expr_ud ud;
+
+ cur_item = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(struct regexp_module_item));
+ cur_item->symbol = ucl_object_key(value);
+ cur_item->magic = rspamd_regexp_cb_magic;
+
+ ud.conf_obj = NULL;
+ ud.cfg = cfg;
+
+ if (!read_regexp_expression(cfg->cfg_pool,
+ cur_item, ucl_object_key(value),
+ ucl_obj_tostring(value), &ud)) {
+ if (validate) {
+ return FALSE;
+ }
+ }
+ else {
+ rspamd_symcache_add_symbol(cfg->cache,
+ cur_item->symbol,
+ 0,
+ process_regexp_item,
+ cur_item,
+ SYMBOL_TYPE_NORMAL, -1);
+ nre++;
+ }
+ }
+ else if (value->type == UCL_USERDATA) {
+ /* Just a lua function */
+ cur_item = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(struct regexp_module_item));
+ cur_item->magic = rspamd_regexp_cb_magic;
+ cur_item->symbol = ucl_object_key(value);
+ cur_item->lua_function = ucl_object_toclosure(value);
+
+ rspamd_symcache_add_symbol(cfg->cache,
+ cur_item->symbol,
+ 0,
+ process_regexp_item,
+ cur_item,
+ SYMBOL_TYPE_NORMAL, -1);
+ nlua++;
+ }
+ else if (value->type == UCL_OBJECT) {
+ const gchar *description = NULL, *group = NULL;
+ gdouble score = 0.0;
+ guint flags = 0, priority = 0;
+ gboolean is_lua = FALSE, valid_expression = TRUE;
+ struct rspamd_mime_expr_ud ud;
+
+ /* We have some lua table, extract its arguments */
+ elt = ucl_object_lookup(value, "callback");
+
+ if (elt == NULL || elt->type != UCL_USERDATA) {
+
+ /* Try plain regexp expression */
+ elt = ucl_object_lookup_any(value, "regexp", "re", NULL);
+
+ if (elt != NULL && ucl_object_type(elt) == UCL_STRING) {
+ cur_item = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(struct regexp_module_item));
+ cur_item->symbol = ucl_object_key(value);
+ cur_item->magic = rspamd_regexp_cb_magic;
+ ud.cfg = cfg;
+ ud.conf_obj = value;
+
+ if (!read_regexp_expression(cfg->cfg_pool,
+ cur_item, ucl_object_key(value),
+ ucl_obj_tostring(elt), &ud)) {
+ if (validate) {
+ return FALSE;
+ }
+ }
+ else {
+ valid_expression = TRUE;
+ nre++;
+ }
+ }
+ else {
+ msg_err_config(
+ "no callback/expression defined for regexp symbol: "
+ "%s",
+ ucl_object_key(value));
+ }
+ }
+ else {
+ is_lua = TRUE;
+ nlua++;
+ cur_item = rspamd_mempool_alloc0(
+ cfg->cfg_pool,
+ sizeof(struct regexp_module_item));
+ cur_item->magic = rspamd_regexp_cb_magic;
+ cur_item->symbol = ucl_object_key(value);
+ cur_item->lua_function = ucl_object_toclosure(value);
+ }
+
+ if (cur_item && (is_lua || valid_expression)) {
+
+ flags = SYMBOL_TYPE_NORMAL;
+ elt = ucl_object_lookup(value, "mime_only");
+
+ if (elt) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ msg_err_config(
+ "mime_only attribute is not boolean for symbol: '%s'",
+ cur_item->symbol);
+
+ if (validate) {
+ return FALSE;
+ }
+ }
+ else {
+ if (ucl_object_toboolean(elt)) {
+ flags |= SYMBOL_TYPE_MIME_ONLY;
+ }
+ }
+ }
+
+ rspamd_symcache_add_symbol(cfg->cache,
+ cur_item->symbol,
+ 0,
+ process_regexp_item,
+ cur_item,
+ flags, -1);
+
+ /* Reset flags */
+ flags = 0;
+
+ elt = ucl_object_lookup(value, "condition");
+
+ if (elt != NULL && ucl_object_type(elt) == UCL_USERDATA) {
+ struct ucl_lua_funcdata *conddata;
+
+ g_assert(cur_item->symbol != NULL);
+ conddata = ucl_object_toclosure(elt);
+ rspamd_symcache_add_condition_delayed(cfg->cache,
+ cur_item->symbol,
+ conddata->L, conddata->idx);
+ }
+
+ elt = ucl_object_lookup(value, "description");
+
+ if (elt) {
+ description = ucl_object_tostring(elt);
+ }
+
+ elt = ucl_object_lookup(value, "group");
+
+ if (elt) {
+ group = ucl_object_tostring(elt);
+ }
+
+ elt = ucl_object_lookup(value, "score");
+
+ if (elt) {
+ if (ucl_object_type(elt) != UCL_FLOAT && ucl_object_type(elt) != UCL_INT) {
+ msg_err_config(
+ "score attribute is not numeric for symbol: '%s'",
+ cur_item->symbol);
+
+ if (validate) {
+ return FALSE;
+ }
+ }
+ else {
+ score = ucl_object_todouble(elt);
+ }
+ }
+
+ elt = ucl_object_lookup(value, "one_shot");
+
+ if (elt) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ msg_err_config(
+ "one_shot attribute is not boolean for symbol: '%s'",
+ cur_item->symbol);
+
+ if (validate) {
+ return FALSE;
+ }
+ }
+ else {
+ if (ucl_object_toboolean(elt)) {
+ nshots = 1;
+ }
+ }
+ }
+
+ if ((elt = ucl_object_lookup(value, "any_shot")) != NULL) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ msg_err_config(
+ "any_shot attribute is not boolean for symbol: '%s'",
+ cur_item->symbol);
+
+ if (validate) {
+ return FALSE;
+ }
+ }
+ else {
+ if (ucl_object_toboolean(elt)) {
+ nshots = -1;
+ }
+ }
+ }
+
+ if ((elt = ucl_object_lookup(value, "nshots")) != NULL) {
+ if (ucl_object_type(elt) != UCL_FLOAT && ucl_object_type(elt) != UCL_INT) {
+ msg_err_config(
+ "nshots attribute is not numeric for symbol: '%s'",
+ cur_item->symbol);
+
+ if (validate) {
+ return FALSE;
+ }
+ }
+ else {
+ nshots = ucl_object_toint(elt);
+ }
+ }
+
+ elt = ucl_object_lookup(value, "one_param");
+
+ if (elt) {
+ if (ucl_object_type(elt) != UCL_BOOLEAN) {
+ msg_err_config(
+ "one_param attribute is not boolean for symbol: '%s'",
+ cur_item->symbol);
+
+ if (validate) {
+ return FALSE;
+ }
+ }
+ else {
+ if (ucl_object_toboolean(elt)) {
+ flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM;
+ }
+ }
+ }
+
+ elt = ucl_object_lookup(value, "priority");
+
+ if (elt) {
+ if (ucl_object_type(elt) != UCL_FLOAT && ucl_object_type(elt) != UCL_INT) {
+ msg_err_config(
+ "priority attribute is not numeric for symbol: '%s'",
+ cur_item->symbol);
+
+ if (validate) {
+ return FALSE;
+ }
+ }
+ else {
+ priority = ucl_object_toint(elt);
+ }
+ }
+ else {
+ priority = 0;
+ }
+
+ rspamd_config_add_symbol(cfg, cur_item->symbol,
+ score, description, group, flags, priority, nshots);
+
+ elt = ucl_object_lookup(value, "groups");
+
+ if (elt) {
+ ucl_object_iter_t gr_it;
+ const ucl_object_t *cur_gr;
+
+ gr_it = ucl_object_iterate_new(elt);
+
+ while ((cur_gr = ucl_object_iterate_safe(gr_it, true)) != NULL) {
+ rspamd_config_add_symbol_group(cfg, cur_item->symbol,
+ ucl_object_tostring(cur_gr));
+ }
+
+ ucl_object_iterate_free(gr_it);
+ }
+ }
+ }
+ else {
+ msg_warn_config("unknown type of attribute %s for regexp module",
+ ucl_object_key(value));
+ }
+ }
+
+ if (res) {
+ msg_info_config("init internal regexp module, %d regexp rules and %d "
+ "lua rules are loaded",
+ nre, nlua);
+ }
+ else {
+ msg_err_config("fatal regexp module error");
+ }
+
+ return res;
+}
+
+gint regexp_module_reconfig(struct rspamd_config *cfg)
+{
+ return regexp_module_config(cfg, false);
+}
+
+static gboolean
+rspamd_lua_call_expression_func(struct ucl_lua_funcdata *lua_data,
+ struct rspamd_task *task,
+ GArray *args, gdouble *res,
+ const gchar *symbol)
+{
+ lua_State *L = lua_data->L;
+ struct rspamd_task **ptask;
+ struct expression_argument *arg;
+ gint pop = 0, i, nargs = 0;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, lua_data->idx);
+ /* Now we got function in top of stack */
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+
+ /* Now push all arguments */
+ if (args) {
+ for (i = 0; i < (gint) args->len; i++) {
+ arg = &g_array_index(args, struct expression_argument, i);
+ if (arg) {
+ switch (arg->type) {
+ case EXPRESSION_ARGUMENT_NORMAL:
+ lua_pushstring(L, (const gchar *) arg->data);
+ break;
+ case EXPRESSION_ARGUMENT_BOOL:
+ lua_pushboolean(L, (gboolean) GPOINTER_TO_SIZE(arg->data));
+ break;
+ default:
+ msg_err_task("%s: cannot pass custom params to lua function",
+ symbol);
+ return FALSE;
+ }
+ }
+ }
+ nargs = args->len;
+ }
+
+ if (lua_pcall(L, nargs + 1, 1, 0) != 0) {
+ msg_info_task("%s: call to lua function failed: %s", symbol,
+ lua_tostring(L, -1));
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+
+ pop++;
+
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ *res = lua_tonumber(L, -1);
+ }
+ else if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ *res = lua_toboolean(L, -1);
+ }
+ else {
+ msg_info_task("%s: lua function must return a boolean", symbol);
+ *res = FALSE;
+ }
+
+ lua_pop(L, pop);
+
+ return TRUE;
+}
+
+
+static void
+process_regexp_item(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *symcache_item,
+ void *user_data)
+{
+ struct regexp_module_item *item = user_data;
+ gdouble res = FALSE;
+
+ /* Non-threaded version */
+ if (item->lua_function) {
+ /* Just call function */
+ res = FALSE;
+ if (!rspamd_lua_call_expression_func(item->lua_function, task, NULL,
+ &res, item->symbol)) {
+ msg_err_task("error occurred when checking symbol %s",
+ item->symbol);
+ }
+ }
+ else {
+ /* Process expression */
+ if (item->expr) {
+ res = rspamd_process_expression(item->expr, 0, task);
+ }
+ else {
+ msg_warn_task("FIXME: %s symbol is broken with new expressions",
+ item->symbol);
+ }
+ }
+
+ if (res != 0) {
+ rspamd_task_insert_result(task, item->symbol, res, NULL);
+ }
+
+ rspamd_symcache_finalize_item(task, symcache_item);
+}
diff --git a/src/ragel/content_disposition.rl b/src/ragel/content_disposition.rl
new file mode 100644
index 0000000..862015e
--- /dev/null
+++ b/src/ragel/content_disposition.rl
@@ -0,0 +1,36 @@
+%%{
+ machine content_disposition;
+
+ # https://tools.ietf.org/html/rfc2045#section-5.1
+
+ ccontent = ctext | FWS | '(' @{ fcall balanced_ccontent; };
+ balanced_ccontent := ccontent* ')' @{ fret; };
+ comment = "(" (FWS? ccontent)* FWS? ")";
+ CFWS = ((FWS? comment)+ FWS?) | FWS;
+ qcontent = qtextSMTP | quoted_pairSMTP | textUTF8;
+ quoted_string = CFWS?
+ (DQUOTE
+ (((FWS? qcontent)* FWS?) >Quoted_Str_Start %Quoted_Str_End)
+ DQUOTE) CFWS?;
+ token = 0x21..0x27 | 0x2a..0x2b | 0x2c..0x2e | 0x30..0x39 | 0x41..0x5a | 0x5e..0x7e;
+ value = (quoted_string | (token -- ('"' | 0x3d | utf8_2c | utf8_3c | utf8_4c))+) >Param_Value_Start %Param_Value_End;
+ attribute = (quoted_string | (token -- ('"' | '='))+) >Param_Name_Start %Param_Name_End;
+ parameter = CFWS? attribute FWS? "=" FWS? value CFWS?;
+
+ ietf_token = token+;
+ custom_x_token = /x/i "-" token+;
+ extension_token = ietf_token | custom_x_token;
+ disposition_type = /inline/i %Disposition_Inline | /attachment/i %Disposition_Attachment
+ | extension_token >Disposition_Start %Disposition_End;
+ disposition_parm = parameter;
+ content_disposition = disposition_type (";" disposition_parm)*;
+
+ prepush {
+ if (top >= st_storage.size) {
+ st_storage.size = (top + 1) * 2;
+ st_storage.data = realloc (st_storage.data, st_storage.size * sizeof (int));
+ g_assert (st_storage.data != NULL);
+ stack = st_storage.data;
+ }
+ }
+}%%
diff --git a/src/ragel/content_disposition_parser.rl b/src/ragel/content_disposition_parser.rl
new file mode 100644
index 0000000..f1b0172
--- /dev/null
+++ b/src/ragel/content_disposition_parser.rl
@@ -0,0 +1,126 @@
+%%{
+ machine content_type_parser;
+ alphtype unsigned char;
+
+ action Disposition_Start {
+ }
+
+ action Disposition_End {
+ }
+
+ action Disposition_Inline {
+ cd->type = RSPAMD_CT_INLINE;
+ }
+
+ action Disposition_Attachment {
+ cd->type = RSPAMD_CT_ATTACHMENT;
+ }
+
+ action Param_Name_Start {
+ qstart = NULL;
+ qend = NULL;
+ pname_start = p;
+ pname_end = NULL;
+ }
+
+ action Param_Name_End {
+ if (qstart) {
+ pname_start = qstart;
+ }
+ if (qend && qend >= qstart) {
+ pname_end = qend;
+ }
+ else if (p >= pname_start) {
+ pname_end = p;
+ }
+ qstart = NULL;
+ qend = NULL;
+ }
+
+
+ action Param_Value_Start {
+ qstart = NULL;
+ qend = NULL;
+
+ if (pname_end) {
+ pvalue_start = p;
+ pvalue_end = NULL;
+ }
+ }
+
+
+ action Param_Value_End {
+ if (pname_end) {
+ if (qstart) {
+ pvalue_start = qstart;
+ }
+ if (qend && qend >= qstart) {
+ pvalue_end = qend;
+ }
+ else if (p >= pvalue_start) {
+ pvalue_end = p;
+ }
+ qstart = NULL;
+ qend = NULL;
+
+ if (pvalue_end && pvalue_end > pvalue_start && pname_end > pname_start) {
+ rspamd_content_disposition_add_param (pool, cd, pname_start, pname_end, pvalue_start, pvalue_end);
+ }
+ }
+
+ pname_start = NULL;
+ pname_end = NULL;
+ pvalue_start = NULL;
+ pvalue_end = NULL;
+ qend = NULL;
+ qstart = NULL;
+ }
+
+ action Quoted_Str_Start {
+ qstart = p;
+ qend = NULL;
+ }
+
+ action Quoted_Str_End {
+ if (qstart) {
+ qend = p;
+ }
+ }
+
+ include smtp_base "smtp_base.rl";
+ include content_disposition "content_disposition.rl";
+
+ main := content_disposition;
+
+}%%
+
+#include "smtp_parsers.h"
+#include "content_type.h"
+
+%% write data;
+
+gboolean
+rspamd_content_disposition_parser (const char *data, size_t len, struct rspamd_content_disposition *cd, rspamd_mempool_t *pool)
+{
+ const unsigned char *p = data, *pe = data + len, *eof, *qstart = NULL, *qend = NULL,
+ *pname_start = NULL, *pname_end = NULL, *pvalue_start = NULL, *pvalue_end = NULL;
+ int cs, *stack = NULL;
+ gsize top = 0;
+ struct _ragel_st_storage {
+ int *data;
+ gsize size;
+ } st_storage;
+
+ memset (&st_storage, 0, sizeof (st_storage));
+ memset (cd, 0, sizeof (*cd));
+ eof = pe;
+
+ %% write init;
+ %% write exec;
+
+ if (st_storage.data) {
+ free (st_storage.data);
+ }
+
+ return cd->attrs != NULL || cd->type != RSPAMD_CT_UNKNOWN;
+}
diff --git a/src/ragel/content_type.rl b/src/ragel/content_type.rl
new file mode 100644
index 0000000..2988920
--- /dev/null
+++ b/src/ragel/content_type.rl
@@ -0,0 +1,36 @@
+%%{
+ machine content_type;
+ include smtp_whitespace "smtp_whitespace.rl";
+
+ # https://tools.ietf.org/html/rfc2045#section-5.1
+
+ ccontent = ctext | FWS | '(' @{ fcall balanced_ccontent; };
+ balanced_ccontent := ccontent* ')' @{ fret; };
+ comment = "(" (FWS? ccontent)* FWS? ")";
+ CFWS = ((FWS? comment)+ FWS?) | FWS;
+ qcontent = qtextSMTP | quoted_pairSMTP;
+ quoted_string = (DQUOTE
+ (((FWS? qcontent)* FWS?) >Quoted_Str_Start %Quoted_Str_End)
+ DQUOTE);
+ token = 0x21..0x27 | 0x2a..0x2b | 0x2c..0x2e | 0x30..0x39 | 0x41..0x5a | 0x5e..0x7e;
+ value = (quoted_string | (token)+) >Param_Value_Start %Param_Value_End;
+ attribute = (token+) >Param_Name_Start %Param_Name_End;
+ parameter = CFWS? attribute FWS? "=" FWS? value CFWS?;
+
+ ietf_token = token+;
+ custom_x_token = 'x'i "-" token+;
+ extension_token = ietf_token | custom_x_token;
+ iana_token = token+;
+ main_type = (extension_token) >Type_Start %Type_End;
+ sub_type = (extension_token | iana_token) >Subtype_Start %Subtype_End;
+ content_type = main_type ("/" sub_type)? (((CFWS? ";"+) | CFWS) parameter CFWS?)*;
+
+ prepush {
+ if (top >= st_storage.size) {
+ st_storage.size = (top + 1) * 2;
+ st_storage.data = realloc (st_storage.data, st_storage.size * sizeof (int));
+ g_assert (st_storage.data != NULL);
+ stack = st_storage.data;
+ }
+ }
+}%% \ No newline at end of file
diff --git a/src/ragel/rfc2047_parser.rl b/src/ragel/rfc2047_parser.rl
new file mode 100644
index 0000000..88e107b
--- /dev/null
+++ b/src/ragel/rfc2047_parser.rl
@@ -0,0 +1,85 @@
+%%{
+ # It actually implements rfc2047 + rfc2231 extension
+ machine rfc2047_parser;
+
+ action Start_Charset {
+ charset_start = p;
+ }
+
+ action End_Charset {
+ if (charset_start && p > charset_start) {
+ charset_end = p;
+ }
+ }
+
+ action End_Encoding {
+ if (p > in) {
+ switch (*(p - 1)) {
+ case 'B':
+ case 'b':
+ encoding = RSPAMD_RFC2047_BASE64;
+ break;
+ default:
+ encoding = RSPAMD_RFC2047_QP;
+ break;
+ }
+ }
+ }
+
+ action Start_Encoded {
+ encoded_start = p;
+ }
+
+ action End_Encoded {
+ if (encoded_start && p > encoded_start) {
+ encoded_end = p;
+ }
+ }
+
+ primary_tag = alpha{1,8};
+ subtag = alpha{1,8};
+ language = primary_tag ( "-" subtag )*;
+ especials = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\"" | "/" | "[" | "]" | "?" | "." | "=" | "*";
+ token = (graph - especials)+;
+ charset = token;
+ encoding = "Q" | "q" | "B" | "b";
+ encoded_text = (print+ -- ("?="));
+ encoded_word = "=?" charset >Start_Charset %End_Charset
+ ("*" language)? "?"
+ encoding %End_Encoding "?"
+ encoded_text >Start_Encoded %End_Encoded
+ "?="?;
+ main := encoded_word;
+}%%
+
+#include "smtp_parsers.h"
+#include "mime_headers.h"
+
+%% write data;
+
+gboolean
+rspamd_rfc2047_parser (const gchar *in, gsize len, gint *pencoding,
+ const gchar **charset, gsize *charset_len,
+ const gchar **encoded, gsize *encoded_len)
+{
+ const char *p = in, *pe = in + len,
+ *encoded_start = NULL, *encoded_end = NULL,
+ *charset_start = NULL, *charset_end = NULL,
+ *eof = in + len;
+ gint encoding = RSPAMD_RFC2047_QP, cs = 0;
+
+ %% write init;
+ %% write exec;
+
+ if (encoded_end) {
+ *pencoding = encoding;
+ *charset = charset_start;
+ *charset_len = charset_end - charset_start;
+ *encoded = encoded_start;
+ *encoded_len = encoded_end - encoded_start;
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
diff --git a/src/ragel/smtp_addr_parser.rl b/src/ragel/smtp_addr_parser.rl
new file mode 100644
index 0000000..b5b4863
--- /dev/null
+++ b/src/ragel/smtp_addr_parser.rl
@@ -0,0 +1,103 @@
+%%{
+
+ machine smtp_addr_parser;
+
+ action IP6_start {}
+ action IP6_end {}
+ action IP4_start {}
+ action IP4_end {}
+
+ action User_start {
+ addr->user = p;
+ }
+
+ action User_end {
+ if (addr->user) {
+ addr->user_len = p - addr->user;
+ }
+ }
+
+ action Domain_start {
+ addr->domain = p;
+ }
+
+ action Domain_end {
+ if (addr->domain) {
+ addr->domain_len = p - addr->domain;
+ }
+ }
+
+ action Domain_addr_start {
+ addr->domain = p;
+ addr->flags |= RSPAMD_EMAIL_ADDR_IP;
+ }
+
+ action Domain_addr_end {
+ if (addr->domain) {
+ addr->domain_len = p - addr->domain;
+ }
+ }
+
+ action User_has_backslash {
+ addr->flags |= RSPAMD_EMAIL_ADDR_HAS_BACKSLASH;
+ }
+
+ action Quoted_addr {
+ addr->flags |= RSPAMD_EMAIL_ADDR_QUOTED;
+ }
+
+ action Empty_addr {
+ addr->flags |= RSPAMD_EMAIL_ADDR_EMPTY|RSPAMD_EMAIL_ADDR_VALID;
+ addr->addr = "";
+ addr->user = addr->addr;
+ addr->domain = addr->addr;
+ }
+
+ action Valid_addr {
+ if (addr->addr_len > 0) {
+ addr->flags |= RSPAMD_EMAIL_ADDR_VALID;
+ }
+ }
+
+ action Addr_has_angle {
+ addr->flags |= RSPAMD_EMAIL_ADDR_BRACED;
+ }
+
+ action Addr_start {
+ addr->addr = p;
+ }
+
+ action Addr_end {
+ if (addr->addr) {
+ addr->addr_len = p - addr->addr;
+ }
+ }
+
+ include smtp_base "smtp_base.rl";
+ include smtp_ip "smtp_ip.rl";
+ include smtp_address "smtp_address.rl";
+
+ main := SMTPAddr;
+}%%
+
+#include "smtp_parsers.h"
+
+%% write data;
+
+int
+rspamd_smtp_addr_parse (const char *data, size_t len, struct rspamd_email_address *addr)
+{
+ const char *p = data, *pe = data + len, *eof;
+ int cs;
+
+ g_assert (addr != NULL);
+ memset (addr, 0, sizeof (*addr));
+ addr->raw = data;
+ addr->raw_len = len;
+ eof = pe;
+
+ %% write init;
+ %% write exec;
+
+ return cs;
+}
diff --git a/src/ragel/smtp_address.rl b/src/ragel/smtp_address.rl
new file mode 100644
index 0000000..0caf1a6
--- /dev/null
+++ b/src/ragel/smtp_address.rl
@@ -0,0 +1,36 @@
+%%{
+ machine smtp_address;
+
+ # SMTP address spec
+ # Source: https://tools.ietf.org/html/rfc5321#section-4.1.2
+ # Dependencies: smtp_base + smtp_ip
+ # Required actions:
+ # - User_has_backslash
+ # - User_end
+ # - Quoted_addr
+ # - Domain_start
+ # - Domain_end
+ # - Addr_end
+ # - Addr_has_angle
+ # - Valid_addr
+ # - Empty_addr
+ # + from deps:
+ # - IP4_start
+ # - IP4_end
+ # - IP6_start
+ # - IP6_end
+ # - Domain_addr_start
+ # - Domain_addr_end
+
+ # SMTP address spec
+ # Obtained from: https://tools.ietf.org/html/rfc5321#section-4.1.2
+
+ QcontentSMTP = qtextSMTP | quoted_pairSMTP %User_has_backslash;
+ Quoted_string = ( DQUOTE QcontentSMTP* >User_start %User_end DQUOTE ) %Quoted_addr;
+ Local_part = Dot_string >User_start %User_end | Quoted_string;
+ Mailbox = Local_part "@" (address_literal | Domain >Domain_start %Domain_end);
+ UnangledPath = ( Adl ":" )? Mailbox >Addr_start %Addr_end "."?;
+ AngledPath = "<" space* UnangledPath space* ">" %Addr_has_angle;
+ Path = AngledPath | UnangledPath;
+ SMTPAddr = space* (Path | "<>" %Empty_addr ) %Valid_addr space*;
+}%%
diff --git a/src/ragel/smtp_base.rl b/src/ragel/smtp_base.rl
new file mode 100644
index 0000000..cb4f066
--- /dev/null
+++ b/src/ragel/smtp_base.rl
@@ -0,0 +1,44 @@
+%%{
+ machine smtp_base;
+
+ # Base SMTP definitions
+ # Dependencies: none
+ # Required actions: none
+
+ WSP = " ";
+ CRLF = "\r\n" | ("\r" [^\n]) | ([^\r] "\n");
+ DQUOTE = '"';
+
+ # Printable US-ASCII characters not including specials
+ atext = alpha | digit | "!" | "#" | "$" | "%" | "&" |
+ "'" | "*" | "+" | "_" | "/" | "=" | "?" | "^" |
+ "-" | "`" | "{" | "|" | "}" | "~";
+ # Printable US-ASCII characters not including "[", "]", or "\"
+ dtext = 33..90 | 94..126;
+ # Printable US-ASCII characters not including "(", ")", or "\"
+ ctext = 33..39 | 42..91 | 93..126;
+
+ dcontent = 33..90 | 94..126;
+ Let_dig = alpha | digit;
+ Ldh_str = ( alpha | digit | "_" | "-" )* Let_dig;
+
+ quoted_pairSMTP = "\\" 32..126;
+ qtextSMTP = 32..33 | 35..91 | 93..126;
+ utf8_cont = 0x80..0xbf;
+ utf8_2c = 0xc0..0xdf utf8_cont;
+ utf8_3c = 0xe0..0xef utf8_cont utf8_cont;
+ utf8_4c = 0xf0..0xf7 utf8_cont utf8_cont utf8_cont;
+ textUTF8 = qtextSMTP | utf8_2c | utf8_3c | utf8_4c;
+ Atom = atext+;
+ Dot_string = Atom ("." Atom)*;
+ dot_atom_text = atext+ ("." atext+)*;
+ #FWS = ((WSP* CRLF)? WSP+);
+ FWS = WSP+; # We work with unfolded headers, so we can simplify machine
+
+ sub_domain = Let_dig Ldh_str?;
+ Domain = sub_domain ("." sub_domain)*;
+ Atdomain = "@" Domain;
+ Adl = Atdomain ( "," Atdomain )*;
+
+ Standardized_tag = Ldh_str;
+}%% \ No newline at end of file
diff --git a/src/ragel/smtp_date.rl b/src/ragel/smtp_date.rl
new file mode 100644
index 0000000..125ae8a
--- /dev/null
+++ b/src/ragel/smtp_date.rl
@@ -0,0 +1,230 @@
+%%{
+ machine smtp_date;
+
+ # SMTP date spec
+ # Obtained from: http://tools.ietf.org/html/rfc5322#section_3.3
+
+
+ action Day_Start {
+ tmp = p;
+ }
+ action Day_End {
+ if (p > tmp) {
+ gulong n;
+ if (rspamd_strtoul (tmp, p - tmp, &n)) {
+ if (n > 0 && n <= 31) {
+ tm.tm_mday = n;
+ }
+ else {
+ fbreak;
+ }
+ }
+ }
+ }
+ action Month_End {
+
+ }
+ action Year_Start {
+ tmp = p;
+ }
+ action Year_End {
+ if (p > tmp) {
+ gulong n;
+ if (rspamd_strtoul (tmp, p - tmp, &n)) {
+ if (n < 1000) {
+ if (n < 50) {
+ tm.tm_year = n - 1900 + 2000;
+ }
+ else {
+ tm.tm_year = n;
+ }
+ }
+ else {
+ tm.tm_year = n - 1900;
+ }
+ }
+ }
+ }
+ action Hour_Start {
+ tmp = p;
+ }
+ action Hour_End {
+ if (p > tmp) {
+ gulong n;
+ if (rspamd_strtoul (tmp, p - tmp, &n)) {
+ if (n < 24) {
+ tm.tm_hour = n;
+ }
+ else {
+ fbreak;
+ }
+ }
+ }
+ else {
+ fbreak;
+ }
+ }
+ action Minute_Start {
+ tmp = p;
+ }
+ action Minute_End {
+ if (p > tmp) {
+ gulong n;
+ if (rspamd_strtoul (tmp, p - tmp, &n)) {
+ if (n < 60) {
+ tm.tm_min = n;
+ }
+ else {
+ fbreak;
+ }
+ }
+ }
+ else {
+ fbreak;
+ }
+ }
+ action Second_Start {
+ tmp = p;
+ }
+ action Second_End {
+ if (p > tmp) {
+ gulong n;
+ if (rspamd_strtoul (tmp, p - tmp, &n)) {
+ if (n <= 60) { /* Leap second */
+ tm.tm_sec = n;
+ }
+ else {
+ fbreak;
+ }
+ }
+ }
+ else {
+ fbreak;
+ }
+ }
+ action TZ_Sign {
+ tmp = p;
+ }
+ action TZ_Offset_Start {
+
+ }
+ action TZ_Offset_End {
+ if (p > tmp) {
+ rspamd_strtoul (tmp, p - tmp, (gulong *)&tz);
+
+ if (*(tmp - 1) == '-') {
+ tz = -(tz);
+ }
+ }
+ }
+ action Obs_Zone_End {
+ }
+ action DT_End {
+ }
+
+ # Specific actions
+ # Months
+ action Month_Jan {
+ tm.tm_mon = 0;
+ }
+ action Month_Feb {
+ tm.tm_mon = 1;
+ }
+ action Month_Mar {
+ tm.tm_mon = 2;
+ }
+ action Month_Apr {
+ tm.tm_mon = 3;
+ }
+ action Month_May {
+ tm.tm_mon = 4;
+ }
+ action Month_Jun {
+ tm.tm_mon = 5;
+ }
+ action Month_Jul {
+ tm.tm_mon = 6;
+ }
+ action Month_Aug {
+ tm.tm_mon = 7;
+ }
+ action Month_Sep {
+ tm.tm_mon = 8;
+ }
+ action Month_Oct {
+ tm.tm_mon = 9;
+ }
+ action Month_Nov {
+ tm.tm_mon = 10;
+ }
+ action Month_Dec {
+ tm.tm_mon = 11;
+ }
+ # Obsoleted timezones
+ action TZ_UT {
+ tz = 0;
+ }
+ action TZ_GMT {
+ tz = 0;
+ }
+ action TZ_EST {
+ tz = -500;
+ }
+ action TZ_EDT {
+ tz = -400;
+ }
+ action TZ_CST {
+ tz = -600;
+ }
+ action TZ_CDT {
+ tz = -500;
+ }
+ action TZ_MST {
+ tz = -700;
+ }
+ action TZ_MDT {
+ tz = -600;
+ }
+ action TZ_PST {
+ tz = -800;
+ }
+ action TZ_PDT {
+ tz = -700;
+ }
+ prepush {
+ if (top >= st_storage.size) {
+ st_storage.size = (top + 1) * 2;
+ st_storage.data = realloc (st_storage.data, st_storage.size * sizeof (int));
+ g_assert (st_storage.data != NULL);
+ stack = st_storage.data;
+ }
+ }
+ ccontent = ctext | FWS | '(' @{ fcall balanced_ccontent; };
+ balanced_ccontent := ccontent* ')' @{ fret; };
+ comment = "(" (FWS? ccontent)* FWS? ")";
+ CFWS = ((FWS? comment)+ FWS?) | FWS;
+ digit_2 = digit{2};
+ digit_4 = digit{4};
+ day_name = "Mon" | "Tue" | "Wed" | "Thu" |
+ "Fri" | "Sat" | "Sun";
+ day_of_week = FWS? day_name;
+ day = FWS? digit{1,2} >Day_Start %Day_End FWS;
+ month = "Jan" %Month_Jan | "Feb" %Month_Feb | "Mar" %Month_Mar | "Apr" %Month_Apr |
+ "May" %Month_May | "Jun" %Month_Jun | "Jul" %Month_Jul | "Aug" %Month_Aug |
+ "Sep" %Month_Sep | "Oct" %Month_Oct | "Nov" %Month_Nov | "Dec" %Month_Dec;
+ year = FWS digit{2,4} >Year_Start %Year_End FWS;
+ date = day month %Month_End year;
+ hour = digit_2;
+ minute = digit_2;
+ second = digit_2;
+ time_of_day = hour >Hour_Start %Hour_End ":" minute >Minute_Start %Minute_End (":" second >Second_Start %Second_End )?;
+ zone = ("+" | "-") %TZ_Sign digit_4 >TZ_Offset_Start %TZ_Offset_End;
+ obs_zone = "UT" %TZ_UT | "GMT" %TZ_GMT |
+ "EST" %TZ_EST | "EDT" %TZ_EDT |
+ "CST" %TZ_CST | "CDT" %TZ_CDT |
+ "MST" %TZ_MST | "MDT" %TZ_MDT |
+ "PST" %TZ_PST | "PDT" %TZ_PDT |
+ [a-iA-I] | [k-zK-Z];
+ time = time_of_day %DT_End FWS (zone | obs_zone %Obs_Zone_End) FWS*;
+ date_time = (day_of_week ",")? date time CFWS?;
+}%%
diff --git a/src/ragel/smtp_date_parser.rl b/src/ragel/smtp_date_parser.rl
new file mode 100644
index 0000000..8d99ea9
--- /dev/null
+++ b/src/ragel/smtp_date_parser.rl
@@ -0,0 +1,47 @@
+%%{
+
+ machine smtp_date_parser;
+ alphtype unsigned char;
+ include smtp_base "smtp_base.rl";
+ include smtp_date "smtp_date.rl";
+
+ main := date_time;
+}%%
+
+#include "smtp_parsers.h"
+#include "util.h"
+
+%% write data;
+
+guint64
+rspamd_parse_smtp_date (const unsigned char *data, size_t len, GError **err)
+{
+ const unsigned char *p = data, *pe = data + len, *eof = data + len, *tmp = data;
+ struct tm tm;
+ glong tz = 0;
+ gint cs = 0, *stack = NULL;;
+ gsize top = 0;
+
+ memset (&tm, 0, sizeof (tm));
+
+ struct _ragel_st_storage {
+ int *data;
+ gsize size;
+ } st_storage;
+ memset (&st_storage, 0, sizeof (st_storage));
+
+ %% write init;
+ %% write exec;
+
+ if (st_storage.data) {
+ free (st_storage.data);
+ }
+
+ if ( cs < %%{ write first_final; }%% ) {
+ g_set_error (err, g_quark_from_static_string ("smtp_date"), cs, "invalid date at offset %d (%c), state %d",
+ (int)(p - data), (*p > 0 && *p < 128) ? *p : '?', cs);
+ return (guint64)(-1);
+ }
+
+ return rspamd_tm_to_time (&tm, tz);
+} \ No newline at end of file
diff --git a/src/ragel/smtp_ip.rl b/src/ragel/smtp_ip.rl
new file mode 100644
index 0000000..ed10c95
--- /dev/null
+++ b/src/ragel/smtp_ip.rl
@@ -0,0 +1,36 @@
+%%{
+ machine smtp_ip;
+
+ # Parses IPv4/IPv6 address
+ # Source: https://tools.ietf.org/html/rfc5321#section-4.1.3
+ # Dependencies: none
+ # Required actions:
+ # - IP4_start
+ # - IP4_end
+ # - IP6_start
+ # - IP6_end
+ # - Domain_addr_start
+ # - Domain_addr_end
+
+ Snum = digit{1,3};
+ IPv4_addr = (Snum ("." Snum){3});
+ IPv4_address_literal = IPv4_addr >IP4_start %IP4_end;
+ IPv6_hex = xdigit{1,4};
+ IPv6_full = IPv6_hex (":" IPv6_hex){7};
+ IPv6_comp = (IPv6_hex (":" IPv6_hex){0,5})? "::"
+ (IPv6_hex (":" IPv6_hex){0,5})?;
+ IPv6v4_full = IPv6_hex (":" IPv6_hex){5} ":" IPv4_address_literal;
+ IPv6v4_comp = (IPv6_hex (":" IPv6_hex){0,3})? "::"
+ (IPv6_hex (":" IPv6_hex){0,3} ":")?
+ IPv4_address_literal;
+ IPv6_simple = IPv6_full | IPv6_comp;
+ IPv6_addr = IPv6_simple | IPv6v4_full | IPv6v4_comp;
+ IPv6_address_literal = "IPv6:" %IP6_start IPv6_addr %IP6_end;
+
+ General_address_literal = Standardized_tag ":" dcontent+;
+ address_literal = "[" ( IPv4_address_literal |
+ IPv6_address_literal |
+ General_address_literal ) >Domain_addr_start %Domain_addr_end "]";
+ non_conformant_address_literal = IPv4_address_literal >Domain_addr_start %Domain_addr_end;
+
+}%% \ No newline at end of file
diff --git a/src/ragel/smtp_ip_parser.rl b/src/ragel/smtp_ip_parser.rl
new file mode 100644
index 0000000..617f731
--- /dev/null
+++ b/src/ragel/smtp_ip_parser.rl
@@ -0,0 +1,56 @@
+%%{
+
+ machine smtp_ip_parser;
+
+ action IP6_start {
+ in_v6 = 1;
+ ip_start = p;
+ }
+ action IP6_end {
+ in_v6 = 0;
+ ip_end = p;
+ }
+ action IP4_start {
+ if (!in_v6) {
+ ip_start = p;
+ }
+ }
+ action IP4_end {
+ if (!in_v6) {
+ ip_end = p;
+ }
+ }
+
+ action Domain_addr_start {}
+ action Domain_addr_end {}
+
+ include smtp_base "smtp_base.rl";
+ include smtp_ip "smtp_ip.rl";
+
+ main := address_literal | non_conformant_address_literal;
+}%%
+
+#include "smtp_parsers.h"
+#include "util.h"
+#include "addr.h"
+
+%% write data;
+
+rspamd_inet_addr_t *
+rspamd_parse_smtp_ip (const char *data, size_t len, rspamd_mempool_t *pool)
+{
+ const char *p = data, *pe = data + len, *eof = data + len;
+ const char *ip_start = NULL, *ip_end = NULL;
+ gboolean in_v6 = FALSE;
+ gint cs = 0;
+
+ %% write init;
+ %% write exec;
+
+ if (ip_start && ip_end && ip_end > ip_start) {
+ return rspamd_parse_inet_address_pool (ip_start, ip_end - ip_start, pool,
+ RSPAMD_INET_ADDRESS_PARSE_NO_UNIX|RSPAMD_INET_ADDRESS_PARSE_REMOTE);
+ }
+
+ return NULL;
+} \ No newline at end of file
diff --git a/src/rspamadm/CMakeLists.txt b/src/rspamadm/CMakeLists.txt
new file mode 100644
index 0000000..5e88ec8
--- /dev/null
+++ b/src/rspamadm/CMakeLists.txt
@@ -0,0 +1,31 @@
+SET(RSPAMADMSRC rspamadm.c
+ commands.c
+ pw.c
+ configtest.c
+ fuzzy_convert.c
+ configdump.c
+ control.c
+ confighelp.c
+ stat_convert.c
+ signtool.c
+ lua_repl.c
+ ${CMAKE_BINARY_DIR}/src/workers.c
+ #${CMAKE_BINARY_DIR}/src/modules.c - defined in rspamdserver
+ ${CMAKE_SOURCE_DIR}/src/controller.c
+ ${CMAKE_SOURCE_DIR}/src/fuzzy_storage.c
+ ${CMAKE_SOURCE_DIR}/src/worker.c
+ ${CMAKE_SOURCE_DIR}/src/rspamd_proxy.c)
+INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR})
+IF (ENABLE_HYPERSCAN MATCHES "ON")
+ LIST(APPEND RSPAMADMSRC "${CMAKE_SOURCE_DIR}/src/hs_helper.c")
+ENDIF()
+ADD_EXECUTABLE(rspamadm ${RSPAMADMSRC})
+TARGET_LINK_LIBRARIES(rspamadm rspamd-server)
+
+IF (NOT DEBIAN_BUILD)
+ SET_TARGET_PROPERTIES(rspamadm PROPERTIES VERSION ${RSPAMD_VERSION})
+ENDIF (NOT DEBIAN_BUILD)
+
+SET_TARGET_PROPERTIES(rspamadm PROPERTIES LINKER_LANGUAGE CXX)
+ADD_BACKWARD(rspamadm)
+INSTALL(TARGETS rspamadm RUNTIME DESTINATION bin)
diff --git a/src/rspamadm/commands.c b/src/rspamadm/commands.c
new file mode 100644
index 0000000..d64b172
--- /dev/null
+++ b/src/rspamadm/commands.c
@@ -0,0 +1,280 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "rspamadm.h"
+#include "libutil/util.h"
+#include "libserver/logger.h"
+#include "lua/lua_common.h"
+#include "lua/lua_thread_pool.h"
+
+extern struct rspamadm_command pw_command;
+extern struct rspamadm_command configtest_command;
+extern struct rspamadm_command configdump_command;
+extern struct rspamadm_command control_command;
+extern struct rspamadm_command confighelp_command;
+extern struct rspamadm_command statconvert_command;
+extern struct rspamadm_command fuzzyconvert_command;
+extern struct rspamadm_command signtool_command;
+extern struct rspamadm_command lua_command;
+
+const struct rspamadm_command *commands[] = {
+ &help_command,
+ &pw_command,
+ &configtest_command,
+ &configdump_command,
+ &control_command,
+ &confighelp_command,
+ &statconvert_command,
+ &fuzzyconvert_command,
+ &signtool_command,
+ &lua_command,
+ NULL};
+
+
+const struct rspamadm_command *
+rspamadm_search_command(const gchar *name, GPtrArray *all_commands)
+{
+ const struct rspamadm_command *ret = NULL, *cmd;
+ const gchar *alias;
+ guint i, j;
+
+ if (name == NULL) {
+ name = "help";
+ }
+
+ PTR_ARRAY_FOREACH(all_commands, i, cmd)
+ {
+ if (strcmp(name, cmd->name) == 0) {
+ ret = cmd;
+ break;
+ }
+
+ PTR_ARRAY_FOREACH(cmd->aliases, j, alias)
+ {
+ if (strcmp(name, alias) == 0) {
+ ret = cmd;
+ break;
+ }
+ }
+ }
+
+ return ret;
+}
+
+void rspamadm_fill_internal_commands(GPtrArray *dest)
+{
+ guint i;
+
+ for (i = 0; i < G_N_ELEMENTS(commands); i++) {
+ if (commands[i]) {
+ g_ptr_array_add(dest, (gpointer) commands[i]);
+ }
+ }
+}
+
+static void
+lua_thread_str_error_cb(struct thread_entry *thread, int ret, const char *msg)
+{
+ msg_err("call to rspamadm lua script failed (%d): %s",
+ ret, msg);
+}
+
+static void
+rspamadm_lua_command_run(gint argc, gchar **argv,
+ const struct rspamadm_command *cmd)
+{
+ struct thread_entry *thread = lua_thread_pool_get_for_config(rspamd_main->cfg);
+
+ lua_State *L = thread->lua_state;
+
+ gint table_idx = GPOINTER_TO_INT(cmd->command_data);
+ gint i;
+
+ /* Function */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, table_idx);
+ lua_pushstring(L, "handler");
+ lua_gettable(L, -2);
+
+ /* Args */
+ lua_createtable(L, argc + 1, 0);
+
+ for (i = 0; i < argc; i++) {
+ lua_pushstring(L, argv[i]);
+ lua_rawseti(L, -2, i); /* Starting from zero ! */
+ }
+
+ if (lua_repl_thread_call(thread, 1, (void *) cmd, lua_thread_str_error_cb) != 0) {
+ exit(EXIT_FAILURE);
+ }
+
+ lua_settop(L, 0);
+}
+
+static const gchar *
+rspamadm_lua_command_help(gboolean full_help,
+ const struct rspamadm_command *cmd)
+{
+ gint table_idx = GPOINTER_TO_INT(cmd->command_data);
+
+ if (full_help) {
+ struct thread_entry *thread = lua_thread_pool_get_for_config(rspamd_main->cfg);
+
+ lua_State *L = thread->lua_state;
+ lua_rawgeti(L, LUA_REGISTRYINDEX, table_idx);
+ /* Function */
+ lua_pushstring(L, "handler");
+ lua_gettable(L, -2);
+
+ /* Args */
+ lua_createtable(L, 2, 0);
+ lua_pushstring(L, cmd->name);
+ lua_rawseti(L, -2, 0); /* Starting from zero ! */
+
+ lua_pushstring(L, "--help");
+ lua_rawseti(L, -2, 1);
+
+ if (lua_repl_thread_call(thread, 1, (void *) cmd, lua_thread_str_error_cb) != 0) {
+ exit(EXIT_FAILURE);
+ }
+
+ lua_settop(L, 0);
+ }
+ else {
+ lua_State *L = rspamd_main->cfg->lua_state;
+ lua_rawgeti(L, LUA_REGISTRYINDEX, table_idx);
+ lua_pushstring(L, "description");
+ lua_gettable(L, -2);
+
+ if (lua_isstring(L, -1)) {
+ printf(" %-18s %-60s\n", cmd->name, lua_tostring(L, -1));
+ }
+ else {
+ printf(" %-18s %-60s\n", cmd->name, "no description available");
+ }
+
+ lua_settop(L, 0);
+ }
+
+ return NULL; /* Must be handled in rspamadm itself */
+}
+
+void rspamadm_fill_lua_commands(lua_State *L, GPtrArray *dest)
+{
+ gint i;
+
+ GPtrArray *lua_paths;
+ GError *err = NULL;
+ const gchar *lualibdir = RSPAMD_LUALIBDIR, *path;
+ struct rspamadm_command *lua_cmd;
+ gchar search_dir[PATH_MAX];
+
+ if (g_hash_table_lookup(ucl_vars, "LUALIBDIR")) {
+ lualibdir = g_hash_table_lookup(ucl_vars, "LUALIBDIR");
+ }
+
+ rspamd_snprintf(search_dir, sizeof(search_dir), "%s%crspamadm%c",
+ lualibdir, G_DIR_SEPARATOR, G_DIR_SEPARATOR);
+
+ if ((lua_paths = rspamd_glob_path(search_dir, "*.lua", FALSE, &err)) == NULL) {
+ msg_err("cannot glob files in %s/*.lua: %e", search_dir, err);
+ g_error_free(err);
+
+ return;
+ }
+
+ PTR_ARRAY_FOREACH(lua_paths, i, path)
+ {
+ if (luaL_dofile(L, path) != 0) {
+ msg_err("cannot execute lua script %s: %s",
+ path, lua_tostring(L, -1));
+ lua_settop(L, 0);
+ continue;
+ }
+ else {
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_pushstring(L, "handler");
+ lua_gettable(L, -2);
+ }
+ else {
+ continue; /* Something goes wrong, huh */
+ }
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_err("rspamadm script %s does not have 'handler' field with type "
+ "function",
+ path);
+ continue;
+ }
+
+ /* Pop handler */
+ lua_pop(L, 1);
+ lua_cmd = g_malloc0(sizeof(*lua_cmd));
+
+ lua_pushstring(L, "name");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ lua_cmd->name = g_strdup(lua_tostring(L, -1));
+ }
+ else {
+ goffset ext_pos;
+ gchar *name;
+
+ name = g_path_get_basename(path);
+ /* Remove .lua */
+ ext_pos = rspamd_substring_search(path, strlen(path), ".lua", 4);
+
+ if (ext_pos != -1) {
+ name[ext_pos] = '\0';
+ }
+
+ lua_cmd->name = name;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "aliases");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_cmd->aliases = g_ptr_array_new_full(
+ rspamd_lua_table_size(L, -1),
+ g_free);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ if (lua_isstring(L, -1)) {
+ g_ptr_array_add(lua_cmd->aliases,
+ g_strdup(lua_tostring(L, -1)));
+ }
+ }
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushvalue(L, -1);
+ /* Reference table itself */
+ lua_cmd->command_data = GINT_TO_POINTER(luaL_ref(L, LUA_REGISTRYINDEX));
+ lua_cmd->flags |= RSPAMADM_FLAG_LUA | RSPAMADM_FLAG_DYNAMIC;
+ lua_cmd->run = rspamadm_lua_command_run;
+ lua_cmd->help = rspamadm_lua_command_help;
+
+ g_ptr_array_add(dest, lua_cmd);
+ }
+
+ lua_settop(L, 0);
+ }
+
+ g_ptr_array_free(lua_paths, TRUE);
+}
diff --git a/src/rspamadm/configdump.c b/src/rspamadm/configdump.c
new file mode 100644
index 0000000..dc8b822
--- /dev/null
+++ b/src/rspamadm/configdump.c
@@ -0,0 +1,555 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamadm.h"
+#include "cfg_file.h"
+#include "cfg_rcl.h"
+#include "utlist.h"
+#include "rspamd.h"
+#include "lua/lua_common.h"
+#include "utlist.h"
+
+static gboolean json = FALSE;
+static gboolean compact = FALSE;
+static gboolean show_help = FALSE;
+static gboolean show_comments = FALSE;
+static gboolean modules_state = FALSE;
+static gboolean symbol_groups_only = FALSE;
+static gboolean symbol_full_details = FALSE;
+static gboolean skip_template = FALSE;
+static gchar *config = NULL;
+extern struct rspamd_main *rspamd_main;
+/* Defined in modules.c */
+extern module_t *modules[];
+extern worker_t *workers[];
+
+static void rspamadm_configdump(gint argc, gchar **argv, const struct rspamadm_command *);
+static const char *rspamadm_configdump_help(gboolean full_help, const struct rspamadm_command *);
+
+struct rspamadm_command configdump_command = {
+ .name = "configdump",
+ .flags = 0,
+ .help = rspamadm_configdump_help,
+ .run = rspamadm_configdump,
+ .lua_subrs = NULL,
+};
+
+static GOptionEntry entries[] = {
+ {"json", 'j', 0, G_OPTION_ARG_NONE, &json,
+ "Json output (pretty formatted)", NULL},
+ {"compact", 'C', 0, G_OPTION_ARG_NONE, &compact,
+ "Compacted json output", NULL},
+ {"config", 'c', 0, G_OPTION_ARG_STRING, &config,
+ "Config file to test", NULL},
+ {"show-help", 'h', 0, G_OPTION_ARG_NONE, &show_help,
+ "Show help as comments for each option", NULL},
+ {"show-comments", 's', 0, G_OPTION_ARG_NONE, &show_comments,
+ "Show saved comments from the configuration file", NULL},
+ {"modules-state", 'm', 0, G_OPTION_ARG_NONE, &modules_state,
+ "Show modules state only", NULL},
+ {"groups", 'g', 0, G_OPTION_ARG_NONE, &symbol_groups_only,
+ "Show symbols groups only", NULL},
+ {"symbol-details", 'd', 0, G_OPTION_ARG_NONE, &symbol_full_details,
+ "Show full symbol details only", NULL},
+ {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template,
+ "Do not apply Jinja templates", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+static const char *
+rspamadm_configdump_help(gboolean full_help, const struct rspamadm_command *cmd)
+{
+ const char *help_str;
+
+ if (full_help) {
+ help_str = "Perform configuration file dump\n\n"
+ "Usage: rspamadm configdump [-c <config_name> [-j --compact -m] [<path1> [<path2> ...]]]\n"
+ "Where options are:\n\n"
+ "-j: output plain json\n"
+ "--compact: output compacted json\n"
+ "-c: config file to test\n"
+ "-m: show state of modules only\n"
+ "-h: show help for dumped options\n"
+ "--help: shows available options and commands";
+ }
+ else {
+ help_str = "Perform configuration file dump";
+ }
+
+ return help_str;
+}
+
+static void
+config_logger(rspamd_mempool_t *pool, gpointer ud)
+{
+}
+
+static void
+rspamadm_add_doc_elt(const ucl_object_t *obj, const ucl_object_t *doc_obj,
+ ucl_object_t *comment_obj)
+{
+ rspamd_fstring_t *comment = rspamd_fstring_new();
+ const ucl_object_t *elt;
+ ucl_object_t *nobj, *cur_comment;
+
+ if (ucl_object_lookup_len(comment_obj, (const char *) &obj,
+ sizeof(void *))) {
+ rspamd_fstring_free(comment);
+ /* Do not rewrite the existing comment */
+ return;
+ }
+
+ if (doc_obj != NULL) {
+ /* Create doc comment */
+ nobj = ucl_object_fromstring_common("/*", 0, 0);
+ }
+ else {
+ rspamd_fstring_free(comment);
+ return;
+ }
+
+ /* We create comments as a list of parts */
+ elt = ucl_object_lookup(doc_obj, "data");
+ if (elt) {
+ rspamd_printf_fstring(&comment, " * %s", ucl_object_tostring(elt));
+ cur_comment = ucl_object_fromstring_common(comment->str, comment->len, 0);
+ rspamd_fstring_erase(comment, 0, comment->len);
+ DL_APPEND(nobj, cur_comment);
+ }
+
+ elt = ucl_object_lookup(doc_obj, "type");
+ if (elt) {
+ rspamd_printf_fstring(&comment, " * Type: %s", ucl_object_tostring(elt));
+ cur_comment = ucl_object_fromstring_common(comment->str, comment->len, 0);
+ rspamd_fstring_erase(comment, 0, comment->len);
+ DL_APPEND(nobj, cur_comment);
+ }
+
+ elt = ucl_object_lookup(doc_obj, "required");
+ if (elt) {
+ rspamd_printf_fstring(&comment, " * Required: %s",
+ ucl_object_toboolean(elt) ? "true" : "false");
+ cur_comment = ucl_object_fromstring_common(comment->str, comment->len, 0);
+ rspamd_fstring_erase(comment, 0, comment->len);
+ DL_APPEND(nobj, cur_comment);
+ }
+
+ cur_comment = ucl_object_fromstring(" */");
+ DL_APPEND(nobj, cur_comment);
+ rspamd_fstring_free(comment);
+
+ ucl_object_insert_key(comment_obj, ucl_object_ref(nobj),
+ (const char *) &obj,
+ sizeof(void *), true);
+
+ ucl_object_unref(nobj);
+}
+
+static void
+rspamadm_gen_comments(const ucl_object_t *obj, const ucl_object_t *doc_obj,
+ ucl_object_t *comments)
+{
+ const ucl_object_t *cur_obj, *cur_doc, *cur_elt;
+ ucl_object_iter_t it = NULL;
+
+ if (obj == NULL || doc_obj == NULL) {
+ return;
+ }
+
+ if (obj->keylen > 0) {
+ rspamadm_add_doc_elt(obj, doc_obj, comments);
+ }
+
+ if (ucl_object_type(obj) == UCL_OBJECT) {
+ while ((cur_obj = ucl_object_iterate(obj, &it, true))) {
+ cur_doc = ucl_object_lookup_len(doc_obj, cur_obj->key,
+ cur_obj->keylen);
+
+ if (cur_doc != NULL) {
+ LL_FOREACH(cur_obj, cur_elt)
+ {
+ if (ucl_object_lookup_len(comments, (const char *) &cur_elt,
+ sizeof(void *)) == NULL) {
+ rspamadm_gen_comments(cur_elt, cur_doc, comments);
+ }
+ }
+ }
+ }
+ }
+}
+
+static void
+rspamadm_dump_section_obj(struct rspamd_config *cfg,
+ const ucl_object_t *obj, const ucl_object_t *doc_obj)
+{
+ rspamd_fstring_t *output;
+ ucl_object_t *comments = NULL;
+
+ output = rspamd_fstring_new();
+
+ if (show_help) {
+ if (show_comments) {
+ comments = cfg->config_comments;
+ }
+ else {
+ comments = ucl_object_typed_new(UCL_OBJECT);
+ }
+
+ rspamadm_gen_comments(obj, doc_obj, comments);
+ }
+ else if (show_comments) {
+ comments = cfg->config_comments;
+ }
+
+ if (json) {
+ rspamd_ucl_emit_fstring_comments(obj, UCL_EMIT_JSON, &output, comments);
+ }
+ else if (compact) {
+ rspamd_ucl_emit_fstring_comments(obj, UCL_EMIT_JSON_COMPACT, &output,
+ comments);
+ }
+ else {
+ rspamd_ucl_emit_fstring_comments(obj, UCL_EMIT_CONFIG, &output,
+ comments);
+ }
+
+ rspamd_printf("%V", output);
+ rspamd_fstring_free(output);
+
+ if (comments != NULL) {
+ ucl_object_unref(comments);
+ }
+}
+
+__attribute__((noreturn)) static void
+rspamadm_configdump(gint argc, gchar **argv, const struct rspamadm_command *cmd)
+{
+ GOptionContext *context;
+ GError *error = NULL;
+ const gchar *confdir;
+ const ucl_object_t *obj = NULL, *cur, *doc_obj;
+ struct rspamd_config *cfg = rspamd_main->cfg;
+ gboolean ret = TRUE;
+ worker_t **pworker;
+ gint i;
+
+ context = g_option_context_new(
+ "configdump - dumps Rspamd configuration");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd administration utility version " RVERSION
+ "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, NULL);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ g_option_context_free(context);
+
+ if (config == NULL) {
+ if ((confdir = g_hash_table_lookup(ucl_vars, "CONFDIR")) == NULL) {
+ confdir = RSPAMD_CONFDIR;
+ }
+
+ config = g_strdup_printf("%s%c%s", confdir, G_DIR_SEPARATOR,
+ "rspamd.conf");
+ }
+
+ pworker = &workers[0];
+ while (*pworker) {
+ /* Init string quarks */
+ (void) g_quark_from_static_string((*pworker)->name);
+ pworker++;
+ }
+
+ cfg->compiled_modules = modules;
+ cfg->compiled_workers = workers;
+ cfg->cfg_name = config;
+
+ if (!rspamd_config_read(cfg, cfg->cfg_name, config_logger, rspamd_main,
+ ucl_vars, skip_template, lua_env)) {
+ ret = FALSE;
+ }
+ else {
+ /* Do post-load actions */
+ rspamd_lua_post_load_config(cfg);
+
+ (void) rspamd_init_filters(rspamd_main->cfg, false, false);
+ rspamd_config_post_load(cfg, RSPAMD_CONFIG_INIT_SYMCACHE);
+ }
+
+ if (ret) {
+ if (modules_state) {
+
+ rspamadm_execute_lua_ucl_subr(argc,
+ argv,
+ cfg->cfg_ucl_obj,
+ "plugins_stats",
+ FALSE);
+
+ exit(EXIT_SUCCESS);
+ }
+
+ if (symbol_full_details) {
+ /*
+ * Create object from symbols groups and output it using the
+ * specified format
+ */
+ ucl_object_t *out = ucl_object_typed_new(UCL_OBJECT);
+ GHashTableIter it;
+ gpointer sk, sv;
+
+ g_hash_table_iter_init(&it, cfg->symbols);
+ ucl_object_t *sym_ucl = ucl_object_typed_new(UCL_OBJECT);
+ const ucl_object_t *all_symbols_ucl = ucl_object_lookup(cfg->cfg_ucl_obj, "symbols");
+
+ while (g_hash_table_iter_next(&it, &sk, &sv)) {
+ const gchar *sym_name = (const gchar *) sk;
+ struct rspamd_symbol *s = (struct rspamd_symbol *) sv;
+ ucl_object_t *this_sym_ucl = ucl_object_typed_new(UCL_OBJECT);
+
+ ucl_object_insert_key(this_sym_ucl,
+ ucl_object_fromdouble(s->score),
+ "score", strlen("score"),
+ false);
+
+ ucl_object_insert_key(this_sym_ucl,
+ ucl_object_fromstring(s->description),
+ "description", strlen("description"), false);
+
+ rspamd_symcache_get_symbol_details(cfg->cache, sym_name, this_sym_ucl);
+
+ ucl_object_insert_key(this_sym_ucl,
+ ucl_object_frombool(!!(s->flags & RSPAMD_SYMBOL_FLAG_DISABLED)),
+ "disabled", strlen("disabled"),
+ false);
+
+ if (s->nshots == 1) {
+ ucl_object_insert_key(this_sym_ucl,
+ ucl_object_frombool(true),
+ "one_shot", strlen("one_shot"),
+ false);
+ }
+ else {
+ ucl_object_insert_key(this_sym_ucl,
+ ucl_object_frombool(false),
+ "one_shot", strlen("one_shot"),
+ false);
+ }
+
+ if (s->gr != NULL) {
+ struct rspamd_symbols_group *gr = s->gr;
+ const char *gr_name = gr->name;
+ if (strcmp(gr_name, "ungrouped") != 0) {
+ ucl_object_insert_key(this_sym_ucl,
+ ucl_object_fromstring(gr_name),
+ "group", strlen("group"),
+ false);
+ }
+
+ if (s->groups) {
+ ucl_object_t *add_groups = ucl_object_typed_new(UCL_ARRAY);
+ guint j;
+ struct rspamd_symbols_group *add_gr;
+ bool has_extra_groups = false;
+
+ PTR_ARRAY_FOREACH(s->groups, j, add_gr)
+ {
+ if (add_gr->name && strcmp(add_gr->name, gr_name) != 0) {
+ ucl_array_append(add_groups,
+ ucl_object_fromstring(add_gr->name));
+ has_extra_groups = true;
+ }
+ }
+
+ if (has_extra_groups == true) {
+ ucl_object_insert_key(this_sym_ucl,
+ add_groups,
+ "groups", strlen("groups"),
+ false);
+ }
+ }
+ }
+
+ const ucl_object_t *loaded_symbol_ucl = ucl_object_lookup(all_symbols_ucl, sym_name);
+ if (loaded_symbol_ucl) {
+ ucl_object_iter_t it = NULL;
+ while ((cur = ucl_iterate_object(loaded_symbol_ucl, &it, true)) != NULL) {
+ const char *key = ucl_object_key(cur);
+ /* If this key isn't something we have direct in the symbol item, grab the key/value */
+ if ((strcmp(key, "score") != 0) &&
+ (strcmp(key, "description") != 0) &&
+ (strcmp(key, "disabled") != 0) &&
+ (strcmp(key, "condition") != 0) &&
+ (strcmp(key, "one_shot") != 0) &&
+ (strcmp(key, "any_shot") != 0) &&
+ (strcmp(key, "nshots") != 0) &&
+ (strcmp(key, "one_param") != 0) &&
+ (strcmp(key, "priority") != 0)) {
+ ucl_object_insert_key(this_sym_ucl, (ucl_object_t *) cur, key, strlen(key), false);
+ }
+ }
+ }
+
+ ucl_object_insert_key(sym_ucl, this_sym_ucl, sym_name,
+ strlen(sym_name), true);
+ }
+ ucl_object_insert_key(out, sym_ucl, "symbols",
+ strlen("symbols"), true);
+
+ rspamadm_dump_section_obj(cfg, out, NULL);
+ exit(EXIT_SUCCESS);
+ }
+
+ if (symbol_groups_only) {
+ /*
+ * Create object from symbols groups and output it using the
+ * specified format
+ */
+ ucl_object_t *out = ucl_object_typed_new(UCL_OBJECT);
+ GHashTableIter it;
+ gpointer k, v;
+
+ g_hash_table_iter_init(&it, cfg->groups);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ const gchar *gr_name = (const gchar *) k;
+ struct rspamd_symbols_group *gr = (struct rspamd_symbols_group *) v;
+ ucl_object_t *gr_ucl = ucl_object_typed_new(UCL_OBJECT);
+
+ ucl_object_insert_key(gr_ucl,
+ ucl_object_frombool(!!(gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC)),
+ "public", strlen("public"), false);
+ ucl_object_insert_key(gr_ucl,
+ ucl_object_frombool(!!(gr->flags & RSPAMD_SYMBOL_GROUP_DISABLED)),
+ "disabled", strlen("disabled"), false);
+ ucl_object_insert_key(gr_ucl,
+ ucl_object_frombool(!!(gr->flags & RSPAMD_SYMBOL_GROUP_ONE_SHOT)),
+ "one_shot", strlen("one_shot"), false);
+ ucl_object_insert_key(gr_ucl,
+ ucl_object_fromdouble(gr->max_score),
+ "max_score", strlen("max_score"), false);
+ ucl_object_insert_key(gr_ucl,
+ ucl_object_fromstring(gr->description),
+ "description", strlen("description"), false);
+
+ if (gr->symbols) {
+ GHashTableIter sit;
+ gpointer sk, sv;
+
+ g_hash_table_iter_init(&sit, gr->symbols);
+ ucl_object_t *sym_ucl = ucl_object_typed_new(UCL_OBJECT);
+
+ while (g_hash_table_iter_next(&sit, &sk, &sv)) {
+ const gchar *sym_name = (const gchar *) sk;
+ struct rspamd_symbol *s = (struct rspamd_symbol *) sv;
+ ucl_object_t *spec_sym = ucl_object_typed_new(UCL_OBJECT);
+
+ ucl_object_insert_key(spec_sym,
+ ucl_object_fromdouble(s->score),
+ "score", strlen("score"),
+ false);
+ ucl_object_insert_key(spec_sym,
+ ucl_object_fromstring(s->description),
+ "description", strlen("description"), false);
+ ucl_object_insert_key(spec_sym,
+ ucl_object_frombool(!!(s->flags & RSPAMD_SYMBOL_FLAG_DISABLED)),
+ "disabled", strlen("disabled"),
+ false);
+
+ if (s->nshots == 1) {
+ ucl_object_insert_key(spec_sym,
+ ucl_object_frombool(true),
+ "one_shot", strlen("one_shot"),
+ false);
+ }
+ else {
+ ucl_object_insert_key(spec_sym,
+ ucl_object_frombool(false),
+ "one_shot", strlen("one_shot"),
+ false);
+ }
+
+ ucl_object_t *add_groups = ucl_object_typed_new(UCL_ARRAY);
+ guint j;
+ struct rspamd_symbols_group *add_gr;
+
+ PTR_ARRAY_FOREACH(s->groups, j, add_gr)
+ {
+ if (add_gr->name && strcmp(add_gr->name, gr_name) != 0) {
+ ucl_array_append(add_groups,
+ ucl_object_fromstring(add_gr->name));
+ }
+ }
+
+ ucl_object_insert_key(spec_sym,
+ add_groups,
+ "extra_groups", strlen("extra_groups"),
+ false);
+
+ ucl_object_insert_key(sym_ucl, spec_sym, sym_name,
+ strlen(sym_name), true);
+ }
+
+ ucl_object_insert_key(gr_ucl, sym_ucl, "symbols",
+ strlen("symbols"), false);
+ }
+
+ ucl_object_insert_key(out, gr_ucl, gr_name, strlen(gr_name),
+ true);
+ }
+
+ rspamadm_dump_section_obj(cfg, out, NULL);
+
+ exit(EXIT_SUCCESS);
+ }
+
+ /* Output configuration */
+ if (argc == 1) {
+ rspamadm_dump_section_obj(cfg, cfg->cfg_ucl_obj, cfg->doc_strings);
+ }
+ else {
+ for (i = 1; i < argc; i++) {
+ obj = ucl_object_lookup_path(cfg->cfg_ucl_obj, argv[i]);
+ doc_obj = ucl_object_lookup_path(cfg->doc_strings, argv[i]);
+
+ if (!obj) {
+ rspamd_printf("Section %s NOT FOUND\n", argv[i]);
+ }
+ else {
+ LL_FOREACH(obj, cur)
+ {
+ if (!json && !compact) {
+ rspamd_printf("*** Section %s ***\n", argv[i]);
+ }
+ rspamadm_dump_section_obj(cfg, cur, doc_obj);
+
+ if (!json && !compact) {
+ rspamd_printf("\n*** End of section %s ***\n", argv[i]);
+ }
+ else {
+ rspamd_printf("\n");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ exit(ret ? EXIT_SUCCESS : EXIT_FAILURE);
+}
diff --git a/src/rspamadm/confighelp.c b/src/rspamadm/confighelp.c
new file mode 100644
index 0000000..2ad07c0
--- /dev/null
+++ b/src/rspamadm/confighelp.c
@@ -0,0 +1,301 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <ucl.h>
+#include "config.h"
+#include "rspamadm.h"
+#include "cfg_file.h"
+#include "cfg_rcl.h"
+#include "rspamd.h"
+#include "lua/lua_common.h"
+
+static gboolean json = FALSE;
+static gboolean compact = FALSE;
+static gboolean keyword = FALSE;
+static const gchar *plugins_path = RSPAMD_PLUGINSDIR;
+extern struct rspamd_main *rspamd_main;
+/* Defined in modules.c */
+extern module_t *modules[];
+extern worker_t *workers[];
+
+static void rspamadm_confighelp(gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+
+static const char *rspamadm_confighelp_help(gboolean full_help,
+ const struct rspamadm_command *cmd);
+
+struct rspamadm_command confighelp_command = {
+ .name = "confighelp",
+ .flags = 0,
+ .help = rspamadm_confighelp_help,
+ .run = rspamadm_confighelp,
+ .lua_subrs = NULL,
+};
+
+static GOptionEntry entries[] = {
+ {"json", 'j', 0, G_OPTION_ARG_NONE, &json,
+ "Output json", NULL},
+ {"compact", 'c', 0, G_OPTION_ARG_NONE, &compact,
+ "Output compacted", NULL},
+ {"keyword", 'k', 0, G_OPTION_ARG_NONE, &keyword,
+ "Search by keyword", NULL},
+ {"plugins", 'P', 0, G_OPTION_ARG_STRING, &plugins_path,
+ "Use the following plugin path (" RSPAMD_PLUGINSDIR ")", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+static const char *
+rspamadm_confighelp_help(gboolean full_help, const struct rspamadm_command *cmd)
+{
+ const char *help_str;
+
+ if (full_help) {
+ help_str = "Shows help for the specified configuration options\n\n"
+ "Usage: rspamadm confighelp [option[, option...]]\n"
+ "Where options are:\n\n"
+ "-c: output compacted JSON\n"
+ "-j: output pretty formatted JSON\n"
+ "-k: search by keyword in doc string\n"
+ "-P: use specific Lua plugins path\n"
+ "--no-color: disable coloured output\n"
+ "--short: show only option names\n"
+ "--no-examples: do not show examples (implied by --short)\n"
+ "--help: shows available options and commands";
+ }
+ else {
+ help_str = "Shows help for configuration options";
+ }
+
+ return help_str;
+}
+
+static void
+rspamadm_confighelp_show(struct rspamd_config *cfg, gint argc, gchar **argv,
+ const char *key, const ucl_object_t *obj)
+{
+ rspamd_fstring_t *out;
+
+ rspamd_lua_set_path(cfg->lua_state, NULL, ucl_vars);
+ out = rspamd_fstring_new();
+
+ if (json) {
+ rspamd_ucl_emit_fstring(obj, UCL_EMIT_JSON, &out);
+ }
+ else if (compact) {
+ rspamd_ucl_emit_fstring(obj, UCL_EMIT_JSON_COMPACT, &out);
+ }
+ else {
+ /* TODO: add lua helper for output */
+ if (key) {
+ rspamd_fprintf(stdout, "Showing help for %s%s:\n",
+ keyword ? "keyword " : "", key);
+ }
+ else {
+ rspamd_fprintf(stdout, "Showing help for all options:\n");
+ }
+
+ rspamadm_execute_lua_ucl_subr(argc,
+ argv,
+ obj,
+ "confighelp",
+ TRUE);
+
+ rspamd_fstring_free(out);
+ return;
+ }
+
+ rspamd_fprintf(stdout, "%V", out);
+ rspamd_fprintf(stdout, "\n");
+
+ rspamd_fstring_free(out);
+}
+
+static void
+rspamadm_confighelp_search_word_step(const ucl_object_t *obj,
+ ucl_object_t *res,
+ const gchar *str,
+ gsize len,
+ GString *path)
+{
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur, *elt;
+ const gchar *dot_pos;
+
+ while ((cur = ucl_object_iterate(obj, &it, true)) != NULL) {
+ if (cur->keylen > 0) {
+ rspamd_printf_gstring(path, ".%*s", (int) cur->keylen, cur->key);
+
+ if (rspamd_substring_search_caseless(cur->key,
+ cur->keylen,
+ str,
+ len) != -1) {
+ ucl_object_insert_key(res, ucl_object_ref(cur),
+ path->str, path->len, true);
+ goto fin;
+ }
+ }
+
+ if (ucl_object_type(cur) == UCL_OBJECT) {
+ elt = ucl_object_lookup(cur, "data");
+
+ if (elt != NULL && ucl_object_type(elt) == UCL_STRING) {
+ if (rspamd_substring_search_caseless(elt->value.sv,
+ elt->len,
+ str,
+ len) != -1) {
+ ucl_object_insert_key(res, ucl_object_ref(cur),
+ path->str, path->len, true);
+ goto fin;
+ }
+ }
+
+ rspamadm_confighelp_search_word_step(cur, res, str, len, path);
+ }
+
+ fin:
+ /* Remove the last component of the path */
+ dot_pos = strrchr(path->str, '.');
+
+ if (dot_pos) {
+ g_string_erase(path, dot_pos - path->str,
+ path->len - (dot_pos - path->str));
+ }
+ }
+}
+
+static ucl_object_t *
+rspamadm_confighelp_search_word(const ucl_object_t *obj, const gchar *str)
+{
+ gsize len = strlen(str);
+ GString *path = g_string_new("");
+ ucl_object_t *res;
+
+
+ res = ucl_object_typed_new(UCL_OBJECT);
+
+ rspamadm_confighelp_search_word_step(obj, res, str, len, path);
+
+ return res;
+}
+
+__attribute__((noreturn)) static void
+rspamadm_confighelp(gint argc, gchar **argv, const struct rspamadm_command *cmd)
+{
+ struct rspamd_config *cfg;
+ ucl_object_t *doc_obj;
+ const ucl_object_t *elt;
+ GOptionContext *context;
+ GError *error = NULL;
+ module_t *mod, **pmod;
+ worker_t **pworker;
+ struct module_ctx *mod_ctx;
+ gint i, ret = 0, processed_args = 0;
+
+ context = g_option_context_new(
+ "confighelp - displays help for the configuration options");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd administration utility version " RVERSION
+ "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, NULL);
+ g_option_context_set_ignore_unknown_options(context, TRUE);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ rspamd_fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ g_option_context_free(context);
+ pworker = &workers[0];
+ while (*pworker) {
+ /* Init string quarks */
+ (void) g_quark_from_static_string((*pworker)->name);
+ pworker++;
+ }
+
+ cfg = rspamd_config_new(RSPAMD_CONFIG_INIT_SKIP_LUA);
+ cfg->lua_state = rspamd_main->cfg->lua_state;
+ cfg->compiled_modules = modules;
+ cfg->compiled_workers = workers;
+
+ rspamd_rcl_config_init(cfg, NULL);
+ lua_pushboolean(cfg->lua_state, true);
+ lua_setglobal(cfg->lua_state, "confighelp");
+ rspamd_rcl_add_lua_plugins_path(cfg->rcl_top_section, cfg, plugins_path, FALSE, NULL);
+
+ /* Init modules to get documentation strings */
+ i = 0;
+ for (pmod = cfg->compiled_modules; pmod != NULL && *pmod != NULL; pmod++) {
+ mod = *pmod;
+ mod_ctx = g_malloc0(sizeof(struct module_ctx));
+
+ if (mod->module_init_func(cfg, &mod_ctx) == 0) {
+ g_ptr_array_add(cfg->c_modules, mod_ctx);
+ mod_ctx->mod = mod;
+ mod->ctx_offset = i++;
+ mod_ctx->mod = mod;
+ }
+ }
+ /* Also init all workers */
+ for (pworker = cfg->compiled_workers; *pworker != NULL; pworker++) {
+ (*pworker)->worker_init_func(cfg);
+ }
+
+ /* Init lua modules */
+ rspamd_lua_set_path(cfg->lua_state, cfg->cfg_ucl_obj, ucl_vars);
+ rspamd_init_lua_filters(cfg, true, false);
+
+ if (argc > 1) {
+ for (i = 1; i < argc; i++) {
+ if (argv[i][0] != '-') {
+
+ if (keyword) {
+ doc_obj = rspamadm_confighelp_search_word(cfg->doc_strings,
+ argv[i]);
+ }
+ else {
+ doc_obj = ucl_object_typed_new(UCL_OBJECT);
+ elt = ucl_object_lookup_path(cfg->doc_strings, argv[i]);
+
+ if (elt) {
+ ucl_object_insert_key(doc_obj, ucl_object_ref(elt),
+ argv[i], 0, false);
+ }
+ }
+
+ if (doc_obj != NULL) {
+ rspamadm_confighelp_show(cfg, argc, argv, argv[i], doc_obj);
+ ucl_object_unref(doc_obj);
+ }
+ else {
+ rspamd_fprintf(stderr,
+ "Cannot find help for %s\n",
+ argv[i]);
+ ret = EXIT_FAILURE;
+ }
+ processed_args++;
+ }
+ }
+ }
+
+ if (processed_args == 0) {
+ /* Show all documentation strings */
+ rspamadm_confighelp_show(cfg, argc, argv, NULL, cfg->doc_strings);
+ }
+
+ rspamd_config_free(cfg);
+
+ exit(ret);
+}
diff --git a/src/rspamadm/configtest.c b/src/rspamadm/configtest.c
new file mode 100644
index 0000000..8f1482f
--- /dev/null
+++ b/src/rspamadm/configtest.c
@@ -0,0 +1,190 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamadm.h"
+#include "cfg_file.h"
+#include "cfg_rcl.h"
+#include "rspamd.h"
+#include "lua/lua_common.h"
+
+static gboolean quiet = FALSE;
+static gchar *config = NULL;
+static gboolean strict = FALSE;
+static gboolean skip_template = FALSE;
+extern struct rspamd_main *rspamd_main;
+/* Defined in modules.c */
+extern module_t *modules[];
+extern worker_t *workers[];
+
+static void rspamadm_configtest(gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_configtest_help(gboolean full_help,
+ const struct rspamadm_command *cmd);
+
+struct rspamadm_command configtest_command = {
+ .name = "configtest",
+ .flags = 0,
+ .help = rspamadm_configtest_help,
+ .run = rspamadm_configtest,
+ .lua_subrs = NULL,
+};
+
+static GOptionEntry entries[] = {
+ {"quiet", 'q', 0, G_OPTION_ARG_NONE, &quiet,
+ "Suppress output", NULL},
+ {"config", 'c', 0, G_OPTION_ARG_STRING, &config,
+ "Config file to test", NULL},
+ {"strict", 's', 0, G_OPTION_ARG_NONE, &strict,
+ "Stop on any error in config", NULL},
+ {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template,
+ "Do not apply Jinja templates", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+static const char *
+rspamadm_configtest_help(gboolean full_help, const struct rspamadm_command *cmd)
+{
+ const char *help_str;
+
+ if (full_help) {
+ help_str = "Perform configuration file test\n\n"
+ "Usage: rspamadm configtest [-q -c <config_name>]\n"
+ "Where options are:\n\n"
+ "-q: quiet output\n"
+ "-c: config file to test\n"
+ "--help: shows available options and commands";
+ }
+ else {
+ help_str = "Perform configuration file test";
+ }
+
+ return help_str;
+}
+
+static void
+config_logger(rspamd_mempool_t *pool, gpointer ud)
+{
+}
+
+static void
+rspamadm_configtest(gint argc, gchar **argv, const struct rspamadm_command *cmd)
+{
+ GOptionContext *context;
+ GError *error = NULL;
+ const gchar *confdir;
+ struct rspamd_config *cfg = rspamd_main->cfg;
+ gboolean ret = TRUE;
+ worker_t **pworker;
+ const guint64 *log_cnt;
+
+ context = g_option_context_new(
+ "configtest - perform configuration file test");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd administration utility version " RVERSION
+ "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, NULL);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ g_option_context_free(context);
+
+ if (config == NULL) {
+ static gchar fbuf[PATH_MAX];
+
+ if ((confdir = g_hash_table_lookup(ucl_vars, "CONFDIR")) == NULL) {
+ confdir = RSPAMD_CONFDIR;
+ }
+
+ rspamd_snprintf(fbuf, sizeof(fbuf), "%s%c%s",
+ confdir, G_DIR_SEPARATOR,
+ "rspamd.conf");
+ config = fbuf;
+ }
+
+ pworker = &workers[0];
+ while (*pworker) {
+ /* Init string quarks */
+ (void) g_quark_from_static_string((*pworker)->name);
+ pworker++;
+ }
+
+ cfg->compiled_modules = modules;
+ cfg->compiled_workers = workers;
+ cfg->cfg_name = config;
+
+ if (!rspamd_config_read(cfg, cfg->cfg_name, config_logger, rspamd_main,
+ ucl_vars, skip_template, lua_env)) {
+ ret = FALSE;
+ }
+ else {
+ /* Do post-load actions */
+ rspamd_lua_post_load_config(cfg);
+
+ if (!rspamd_init_filters(rspamd_main->cfg, false, strict)) {
+ ret = FALSE;
+ }
+
+ if (ret) {
+ ret = rspamd_config_post_load(cfg, RSPAMD_CONFIG_INIT_SYMCACHE);
+ }
+
+ if (ret && !rspamd_symcache_validate(cfg->cache,
+ cfg,
+ FALSE)) {
+ ret = FALSE;
+ }
+
+ if (ret) {
+ if (rspamd_lua_require_function(cfg->lua_state, "lua_cfg_utils", "check_configuration_errors")) {
+ GError *err = NULL;
+
+ if (!rspamd_lua_universal_pcall(cfg->lua_state, -1, G_STRLOC, 1, "", &err)) {
+ msg_err_config("call to lua function failed: %s",
+ lua_tostring(cfg->lua_state, -1));
+ lua_pop(cfg->lua_state, 2);
+ ret = FALSE;
+ }
+ else {
+ ret = lua_toboolean(cfg->lua_state, -1);
+ lua_pop(cfg->lua_state, 2);
+ }
+ }
+ }
+ }
+
+ if (strict && ret) {
+ log_cnt = rspamd_log_counters(rspamd_main->logger);
+
+ if (log_cnt && log_cnt[0] > 0) {
+ if (!quiet) {
+ rspamd_printf("%L errors found\n", log_cnt[0]);
+ }
+ ret = FALSE;
+ }
+ }
+
+ if (!quiet) {
+ rspamd_printf("syntax %s\n", ret ? "OK" : "BAD");
+ }
+
+ if (!ret) {
+ exit(EXIT_FAILURE);
+ }
+}
diff --git a/src/rspamadm/control.c b/src/rspamadm/control.c
new file mode 100644
index 0000000..c82d4ac
--- /dev/null
+++ b/src/rspamadm/control.c
@@ -0,0 +1,258 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamadm.h"
+#include "cryptobox.h"
+#include "printf.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
+#include "addr.h"
+#include "unix-std.h"
+#include "contrib/libev/ev.h"
+#include "libutil/util.h"
+#include "lua/lua_common.h"
+
+static gchar *control_path = RSPAMD_DBDIR "/rspamd.sock";
+static gboolean json = FALSE;
+static gboolean ucl = TRUE;
+static gboolean compact = FALSE;
+static gdouble timeout = 1.0;
+
+static void rspamadm_control(gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_control_help(gboolean full_help,
+ const struct rspamadm_command *cmd);
+
+struct rspamadm_command control_command = {
+ .name = "control",
+ .flags = 0,
+ .help = rspamadm_control_help,
+ .run = rspamadm_control,
+ .lua_subrs = NULL,
+};
+
+struct rspamadm_control_cbdata {
+ const gchar *path;
+ gint argc;
+ gchar **argv;
+};
+
+static GOptionEntry entries[] = {
+ {"json", 'j', 0, G_OPTION_ARG_NONE, &json,
+ "Output json", NULL},
+ {"compact", 'c', 0, G_OPTION_ARG_NONE, &compact,
+ "Output compacted", NULL},
+ {"ucl", 'u', 0, G_OPTION_ARG_NONE, &ucl,
+ "Output ucl (default)", NULL},
+ {"socket", 's', 0, G_OPTION_ARG_STRING, &control_path,
+ "Use the following socket path", NULL},
+ {"timeout", 't', 0, G_OPTION_ARG_DOUBLE, &timeout,
+ "Set IO timeout (1s by default)", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+static const char *
+rspamadm_control_help(gboolean full_help, const struct rspamadm_command *cmd)
+{
+ const char *help_str;
+
+ if (full_help) {
+ help_str = "Manage rspamd main control interface\n\n"
+ "Usage: rspamadm control [-c] [-j] [-u] [-s path] command\n"
+ "Where options are:\n\n"
+ "-c: output compacted json\n"
+ "-j: output linted json\n"
+ "-u: output ucl (default)\n"
+ "-s: use the following socket instead of " RSPAMD_DBDIR "/rspamd.sock\n"
+ "-t: set IO timeout (1.0 seconds default)\n"
+ "--help: shows available options and commands\n\n"
+ "Supported commands:\n"
+ "stat - show statistics\n"
+ "reload - reload workers dynamic data\n"
+ "reresolve - resolve upstreams addresses\n"
+ "recompile - recompile hyperscan regexes\n"
+ "fuzzystat - show fuzzy statistics\n"
+ "fuzzysync - immediately sync fuzzy database to storage\n";
+ }
+ else {
+ help_str = "Manage rspamd main control interface";
+ }
+
+ return help_str;
+}
+
+static void
+rspamd_control_error_handler(struct rspamd_http_connection *conn, GError *err)
+{
+ rspamd_fprintf(stderr, "Cannot make HTTP request: %e\n", err);
+ ev_break(rspamd_main->event_loop, EVBREAK_ALL);
+}
+
+static gint
+rspamd_control_finish_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct ucl_parser *parser;
+ ucl_object_t *obj;
+ rspamd_fstring_t *out;
+ const gchar *body;
+ gsize body_len;
+ struct rspamadm_control_cbdata *cbdata = conn->ud;
+
+ body = rspamd_http_message_get_body(msg, &body_len);
+ parser = ucl_parser_new(0);
+
+ if (!body || !ucl_parser_add_chunk(parser, body, body_len)) {
+ rspamd_fprintf(stderr, "cannot parse server's reply: %s\n",
+ ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+ }
+ else {
+ obj = ucl_parser_get_object(parser);
+ out = rspamd_fstring_new();
+
+ if (json) {
+ rspamd_ucl_emit_fstring(obj, UCL_EMIT_JSON, &out);
+ }
+ else if (compact) {
+ rspamd_ucl_emit_fstring(obj, UCL_EMIT_JSON_COMPACT, &out);
+ }
+ else {
+ if (strcmp(cbdata->path, "/fuzzystat") == 0) {
+ rspamadm_execute_lua_ucl_subr(cbdata->argc - 1,
+ &cbdata->argv[1],
+ obj,
+ "fuzzy_stat",
+ TRUE);
+
+ rspamd_fstring_free(out);
+ ucl_object_unref(obj);
+ ucl_parser_free(parser);
+ goto end;
+ }
+ else {
+ rspamd_ucl_emit_fstring(obj, UCL_EMIT_CONFIG, &out);
+ }
+ }
+
+ rspamd_fprintf(stdout, "%V", out);
+
+ rspamd_fstring_free(out);
+ ucl_object_unref(obj);
+ ucl_parser_free(parser);
+ }
+
+end:
+ ev_break(rspamd_main->event_loop, EVBREAK_ALL);
+
+ return 0;
+}
+
+static void
+rspamadm_control(gint argc, gchar **argv, const struct rspamadm_command *_cmd)
+{
+ GOptionContext *context;
+ GError *error = NULL;
+ const gchar *cmd, *path = NULL;
+ struct rspamd_http_connection *conn;
+ struct rspamd_http_message *msg;
+ rspamd_inet_addr_t *addr;
+ static struct rspamadm_control_cbdata cbdata;
+
+ context = g_option_context_new(
+ "control - manage rspamd main control interface");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd administration utility version " RVERSION
+ "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, NULL);
+ g_option_context_set_ignore_unknown_options(context, TRUE);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ rspamd_fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ g_option_context_free(context);
+
+ if (argc <= 1) {
+ rspamd_fprintf(stderr, "command required\n");
+ exit(EXIT_FAILURE);
+ }
+
+ cmd = argv[1];
+
+ if (g_ascii_strcasecmp(cmd, "stat") == 0) {
+ path = "/stat";
+ }
+ else if (g_ascii_strcasecmp(cmd, "reload") == 0) {
+ path = "/reload";
+ }
+ else if (g_ascii_strcasecmp(cmd, "reresolve") == 0) {
+ path = "/reresolve";
+ }
+ else if (g_ascii_strcasecmp(cmd, "recompile") == 0) {
+ path = "/recompile";
+ }
+ else if (g_ascii_strcasecmp(cmd, "fuzzystat") == 0 ||
+ g_ascii_strcasecmp(cmd, "fuzzy_stat") == 0) {
+ path = "/fuzzystat";
+ }
+ else if (g_ascii_strcasecmp(cmd, "fuzzysync") == 0 ||
+ g_ascii_strcasecmp(cmd, "fuzzy_sync") == 0) {
+ path = "/fuzzysync";
+ }
+ else {
+ rspamd_fprintf(stderr, "unknown command: %s\n", cmd);
+ exit(EXIT_FAILURE);
+ }
+
+ if (!rspamd_parse_inet_address(&addr,
+ control_path, strlen(control_path), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ rspamd_fprintf(stderr, "bad control path: %s\n", control_path);
+ exit(EXIT_FAILURE);
+ }
+
+
+ conn = rspamd_http_connection_new_client(
+ rspamd_main->http_ctx, /* Default context */
+ NULL,
+ rspamd_control_error_handler,
+ rspamd_control_finish_handler,
+ RSPAMD_HTTP_CLIENT_SIMPLE,
+ addr);
+
+ if (!conn) {
+ rspamd_fprintf(stderr, "cannot open connection to %s: %s\n",
+ control_path, strerror(errno));
+ exit(-errno);
+ }
+
+ msg = rspamd_http_new_message(HTTP_REQUEST);
+ msg->url = rspamd_fstring_new_init(path, strlen(path));
+
+ cbdata.argc = argc;
+ cbdata.argv = argv;
+ cbdata.path = path;
+
+ rspamd_http_connection_write_message(conn, msg, NULL, NULL, &cbdata,
+ timeout);
+
+ ev_loop(rspamd_main->event_loop, 0);
+
+ rspamd_http_connection_unref(conn);
+ rspamd_inet_address_free(addr);
+}
diff --git a/src/rspamadm/fuzzy_convert.c b/src/rspamadm/fuzzy_convert.c
new file mode 100644
index 0000000..a4da8bd
--- /dev/null
+++ b/src/rspamadm/fuzzy_convert.c
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*-
+ * Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+ * Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamadm.h"
+#include "lua/lua_common.h"
+
+static gchar *source_db = NULL;
+static gchar *redis_host = NULL;
+static gchar *redis_db = NULL;
+static gchar *redis_username = NULL;
+static gchar *redis_password = NULL;
+static int64_t fuzzy_expiry = 0;
+
+static void rspamadm_fuzzyconvert(gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_fuzzyconvert_help(gboolean full_help,
+ const struct rspamadm_command *cmd);
+
+struct rspamadm_command fuzzyconvert_command = {
+ .name = "fuzzyconvert",
+ .flags = 0,
+ .help = rspamadm_fuzzyconvert_help,
+ .run = rspamadm_fuzzyconvert,
+ .lua_subrs = NULL,
+};
+
+static GOptionEntry entries[] = {
+ {"database", 'd', 0, G_OPTION_ARG_FILENAME, &source_db,
+ "Input sqlite", NULL},
+ {"expiry", 'e', 0, G_OPTION_ARG_INT, &fuzzy_expiry,
+ "Time in seconds after which hashes should be expired", NULL},
+ {"host", 'h', 0, G_OPTION_ARG_STRING, &redis_host,
+ "Output redis ip (in format ip:port)", NULL},
+ {"dbname", 'D', 0, G_OPTION_ARG_STRING, &redis_db,
+ "Database in redis (should be numeric)", NULL},
+ {"username", 'u', 0, G_OPTION_ARG_STRING, &redis_username,
+ "Username to connect to redis", NULL},
+ {"password", 'p', 0, G_OPTION_ARG_STRING, &redis_password,
+ "Password to connect to redis", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+
+static const char *
+rspamadm_fuzzyconvert_help(gboolean full_help, const struct rspamadm_command *cmd)
+{
+ const char *help_str;
+
+ if (full_help) {
+ help_str = "Convert fuzzy hashes from sqlite3 to redis\n\n"
+ "Usage: rspamadm fuzzyconvert -d <sqlite_db> -h <redis_ip>\n"
+ "Where options are:\n\n"
+ "-d: input sqlite\n"
+ "-h: output redis ip (in format ip:port)\n"
+ "-D: output redis database\n"
+ "-u: redis username\n"
+ "-p: redis password\n";
+ }
+ else {
+ help_str = "Convert fuzzy hashes from sqlite3 to redis";
+ }
+
+ return help_str;
+}
+
+static void
+rspamadm_fuzzyconvert(gint argc, gchar **argv, const struct rspamadm_command *cmd)
+{
+ GOptionContext *context;
+ GError *error = NULL;
+ ucl_object_t *obj;
+
+ context = g_option_context_new(
+ "fuzzyconvert - converts fuzzy hashes from sqlite3 to redis");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd administration utility version " RVERSION
+ "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, NULL);
+ g_option_context_set_ignore_unknown_options(context, TRUE);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ rspamd_fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ g_option_context_free(context);
+
+ if (!source_db) {
+ rspamd_fprintf(stderr, "source db is missing\n");
+ exit(EXIT_FAILURE);
+ }
+ if (!redis_host) {
+ rspamd_fprintf(stderr, "redis host is missing\n");
+ exit(EXIT_FAILURE);
+ }
+ if (!fuzzy_expiry) {
+ rspamd_fprintf(stderr, "expiry is missing\n");
+ exit(EXIT_FAILURE);
+ }
+
+ obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj, ucl_object_fromstring(source_db),
+ "source_db", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromstring(redis_host),
+ "redis_host", 0, false);
+ ucl_object_insert_key(obj, ucl_object_fromint(fuzzy_expiry),
+ "expiry", 0, false);
+
+ if (redis_username) {
+ ucl_object_insert_key(obj, ucl_object_fromstring(redis_username),
+ "redis_username", 0, false);
+ }
+ if (redis_password) {
+ ucl_object_insert_key(obj, ucl_object_fromstring(redis_password),
+ "redis_password", 0, false);
+ }
+
+ if (redis_db) {
+ ucl_object_insert_key(obj, ucl_object_fromstring(redis_db),
+ "redis_db", 0, false);
+ }
+
+ rspamadm_execute_lua_ucl_subr(argc,
+ argv,
+ obj,
+ "fuzzy_convert",
+ TRUE);
+
+ ucl_object_unref(obj);
+}
diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c
new file mode 100644
index 0000000..432c4de
--- /dev/null
+++ b/src/rspamadm/lua_repl.c
@@ -0,0 +1,1026 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "rspamadm.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
+#include "libserver/http/http_router.h"
+#include "printf.h"
+#include "lua/lua_common.h"
+#include "lua/lua_thread_pool.h"
+#include "message.h"
+#include "unix-std.h"
+#ifdef WITH_LUA_REPL
+#include "replxx.h"
+#endif
+#include "worker_util.h"
+#ifdef WITH_LUAJIT
+#include <luajit.h>
+#endif
+
+static gchar **paths = NULL;
+static gchar **scripts = NULL;
+static gchar **lua_args = NULL;
+static gchar *histfile = NULL;
+static guint max_history = 2000;
+static gchar *serve = NULL;
+static gchar *exec_line = NULL;
+static gint batch = -1;
+extern struct rspamd_async_session *rspamadm_session;
+
+static const char *default_history_file = ".rspamd_repl.hist";
+
+#ifdef WITH_LUA_REPL
+static Replxx *rx_instance = NULL;
+#endif
+
+#ifdef WITH_LUAJIT
+#define MAIN_PROMPT LUAJIT_VERSION "> "
+#else
+#define MAIN_PROMPT LUA_VERSION "> "
+#endif
+#define MULTILINE_PROMPT "... "
+
+static void rspamadm_lua(gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_lua_help(gboolean full_help,
+ const struct rspamadm_command *cmd);
+
+struct rspamadm_command lua_command = {
+ .name = "lua",
+ .flags = 0,
+ .help = rspamadm_lua_help,
+ .run = rspamadm_lua,
+ .lua_subrs = NULL,
+};
+
+/*
+ * Dot commands
+ */
+typedef void (*rspamadm_lua_dot_handler)(lua_State *L, gint argc, gchar **argv);
+struct rspamadm_lua_dot_command {
+ const gchar *name;
+ const gchar *description;
+ rspamadm_lua_dot_handler handler;
+};
+
+static void rspamadm_lua_help_handler(lua_State *L, gint argc, gchar **argv);
+static void rspamadm_lua_load_handler(lua_State *L, gint argc, gchar **argv);
+static void rspamadm_lua_exec_handler(lua_State *L, gint argc, gchar **argv);
+static void rspamadm_lua_message_handler(lua_State *L, gint argc, gchar **argv);
+
+static void lua_thread_error_cb(struct thread_entry *thread, int ret, const char *msg);
+static void lua_thread_finish_cb(struct thread_entry *thread, int ret);
+
+static struct rspamadm_lua_dot_command cmds[] = {
+ {.name = "help",
+ .description = "shows help for commands",
+ .handler = rspamadm_lua_help_handler},
+ {.name = "load",
+ .description = "load lua file",
+ .handler = rspamadm_lua_load_handler},
+ {.name = "exec",
+ .description = "exec lua file",
+ .handler = rspamadm_lua_exec_handler},
+ {.name = "message",
+ .description = "scans message using specified callback: .message <callback_name> <file>...",
+ .handler = rspamadm_lua_message_handler},
+};
+
+static GHashTable *cmds_hash = NULL;
+
+static GOptionEntry entries[] = {
+ {"script", 's', 0, G_OPTION_ARG_STRING_ARRAY, &scripts,
+ "Load specified scripts", NULL},
+ {"path", 'P', 0, G_OPTION_ARG_STRING_ARRAY, &paths,
+ "Add specified paths to lua paths", NULL},
+ {"history-file", 'H', 0, G_OPTION_ARG_FILENAME, &histfile,
+ "Load history from the specified file", NULL},
+ {"max-history", 'm', 0, G_OPTION_ARG_INT, &max_history,
+ "Store this number of history entries", NULL},
+ {"serve", 'S', 0, G_OPTION_ARG_STRING, &serve,
+ "Serve http lua server", NULL},
+ {"batch", 'b', 0, G_OPTION_ARG_NONE, &batch,
+ "Batch execution mode", NULL},
+ {"exec", 'e', 0, G_OPTION_ARG_STRING, &exec_line,
+ "Execute specified script", NULL},
+ {"args", 'a', 0, G_OPTION_ARG_STRING_ARRAY, &lua_args,
+ "Arguments to pass to Lua", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+static const char *
+rspamadm_lua_help(gboolean full_help, const struct rspamadm_command *cmd)
+{
+ const char *help_str;
+
+ if (full_help) {
+ help_str = "Run lua read/execute/print loop\n\n"
+ "Usage: rspamadm lua [-P paths] [-s scripts]\n"
+ "Where options are:\n\n"
+ "-P: add additional lua paths (may be repeated)\n"
+ "-p: split input to lines and feed each line to the script\n"
+ "-s: load scripts on start from specified files (may be repeated)\n"
+ "-S: listen on a specified address as HTTP server\n"
+ "-a: pass argument to lua (may be repeated)\n"
+ "-e: execute script specified in command line"
+ "--help: shows available options and commands";
+ }
+ else {
+ help_str = "Run LUA interpreter";
+ }
+
+ return help_str;
+}
+
+static void
+rspamadm_lua_add_path(lua_State *L, const gchar *path)
+{
+ const gchar *old_path;
+ gsize len;
+ GString *new_path;
+
+ lua_getglobal(L, "package");
+ lua_getfield(L, -1, "path");
+ old_path = luaL_checklstring(L, -1, &len);
+
+ new_path = g_string_sized_new(len + strlen(path) + sizeof("/?.lua"));
+
+ if (strstr(path, "?.lua") == NULL) {
+ rspamd_printf_gstring(new_path, "%s/?.lua;%s", path, old_path);
+ }
+ else {
+ rspamd_printf_gstring(new_path, "%s;%s", path, old_path);
+ }
+
+ lua_pushlstring(L, new_path->str, new_path->len);
+ lua_setfield(L, -2, "path");
+ lua_settop(L, 0);
+ g_string_free(new_path, TRUE);
+}
+
+
+static void
+lua_thread_finish_cb(struct thread_entry *thread, int ret)
+{
+ struct lua_call_data *cd = thread->cd;
+
+ cd->ret = ret;
+}
+
+static void
+lua_thread_error_cb(struct thread_entry *thread, int ret, const char *msg)
+{
+ struct lua_call_data *cd = thread->cd;
+
+ rspamd_fprintf(stderr, "call failed: %s\n", msg);
+
+ cd->ret = ret;
+}
+
+static void
+lua_thread_str_error_cb(struct thread_entry *thread, int ret, const char *msg)
+{
+ struct lua_call_data *cd = thread->cd;
+ const char *what = cd->ud;
+
+ rspamd_fprintf(stderr, "call to %s failed: %s\n", what, msg);
+
+ cd->ret = ret;
+}
+
+static gboolean
+rspamadm_lua_load_script(lua_State *L, const gchar *path)
+{
+ struct thread_entry *thread = lua_thread_pool_get_for_config(rspamd_main->cfg);
+ L = thread->lua_state;
+
+ if (luaL_loadfile(L, path) != 0) {
+ rspamd_fprintf(stderr, "cannot load script %s: %s\n",
+ path, lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return FALSE;
+ }
+
+ if (lua_repl_thread_call(thread, 0, (void *) path, lua_thread_str_error_cb) != 0) {
+ return FALSE;
+ }
+
+ lua_settop(L, 0);
+
+ return TRUE;
+}
+
+static void
+rspamadm_exec_input(lua_State *L, const gchar *input)
+{
+ GString *tb;
+ gint i, cbref;
+ int top = 0;
+ gchar outbuf[8192];
+ struct lua_logger_trace tr;
+
+ struct thread_entry *thread = lua_thread_pool_get_for_config(rspamd_main->cfg);
+ L = thread->lua_state;
+
+ /* First try return + input */
+ tb = g_string_sized_new(strlen(input) + sizeof("return "));
+ rspamd_printf_gstring(tb, "return %s", input);
+
+ int r = luaL_loadstring(L, tb->str);
+ if (r != 0) {
+ /* Reset stack */
+ lua_settop(L, 0);
+ /* Try with no return */
+ if (luaL_loadstring(L, input) != 0) {
+ rspamd_fprintf(stderr, "cannot load string %s\n",
+ input);
+ g_string_free(tb, TRUE);
+ lua_settop(L, 0);
+
+ lua_thread_pool_return(rspamd_main->cfg->lua_thread_pool, thread);
+ return;
+ }
+ }
+
+ g_string_free(tb, TRUE);
+
+
+ top = lua_gettop(L);
+
+ if (lua_repl_thread_call(thread, 0, NULL, NULL) == 0) {
+ /* Print output */
+ for (i = top; i <= lua_gettop(L); i++) {
+ if (lua_isfunction(L, i)) {
+ lua_pushvalue(L, i);
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ rspamd_printf("local function: %d\n", cbref);
+ }
+ else {
+ memset(&tr, 0, sizeof(tr));
+ lua_logger_out_type(L, i, outbuf, sizeof(outbuf) - 1, &tr,
+ LUA_ESCAPE_UNPRINTABLE);
+ rspamd_printf("%s\n", outbuf);
+ }
+ }
+ }
+}
+
+static void
+wait_session_events(void)
+{
+ /* XXX: it's probably worth to add timeout here - not to wait forever */
+ while (rspamd_session_events_pending(rspamadm_session) > 0) {
+ ev_loop(rspamd_main->event_loop, EVRUN_ONCE);
+ }
+
+ msg_debug("finished events waiting, terminating session");
+}
+
+gint lua_repl_thread_call(struct thread_entry *thread, gint narg, gpointer ud, lua_thread_error_t error_func)
+{
+ int ret;
+ struct lua_call_data *cd = g_new0(struct lua_call_data, 1);
+ cd->top = lua_gettop(thread->lua_state);
+ cd->ud = ud;
+
+ thread->finish_callback = lua_thread_finish_cb;
+ if (error_func) {
+ thread->error_callback = error_func;
+ }
+ else {
+ thread->error_callback = lua_thread_error_cb;
+ }
+ thread->cd = cd;
+
+ lua_thread_call(thread, narg);
+
+ wait_session_events();
+
+ ret = cd->ret;
+
+ g_free(cd);
+
+ return ret;
+}
+
+static void
+rspamadm_lua_help_handler(lua_State *L, gint argc, gchar **argv)
+{
+ guint i;
+ struct rspamadm_lua_dot_command *cmd;
+
+ if (argv[1] == NULL) {
+ /* Print all commands */
+ for (i = 0; i < G_N_ELEMENTS(cmds); i++) {
+ rspamd_printf("%s: %s\n", cmds[i].name, cmds[i].description);
+ }
+
+ rspamd_printf("{{: start multiline input\n");
+ rspamd_printf("}}: end multiline input\n");
+ }
+ else {
+ for (i = 1; argv[i] != NULL; i++) {
+ cmd = g_hash_table_lookup(cmds_hash, argv[i]);
+
+ if (cmd) {
+ rspamd_printf("%s: %s\n", cmds->name, cmds->description);
+ }
+ else {
+ rspamd_printf("%s: no such command\n", argv[i]);
+ }
+ }
+ }
+}
+
+static void
+rspamadm_lua_load_handler(lua_State *L, gint argc, gchar **argv)
+{
+ guint i;
+ gboolean ret;
+
+ for (i = 1; argv[i] != NULL; i++) {
+ ret = rspamadm_lua_load_script(L, argv[i]);
+ rspamd_printf("%s: %sloaded\n", argv[i], ret ? "" : "NOT ");
+ }
+}
+
+static void
+rspamadm_lua_exec_handler(lua_State *L, gint argc, gchar **argv)
+{
+ gint i;
+
+ struct thread_entry *thread = lua_thread_pool_get_for_config(rspamd_main->cfg);
+ L = thread->lua_state;
+
+ for (i = 1; argv[i] != NULL; i++) {
+
+ if (luaL_loadfile(L, argv[i]) != 0) {
+ rspamd_fprintf(stderr, "cannot load script %s: %s\n",
+ argv[i], lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return;
+ }
+
+ if (lua_repl_thread_call(thread, 0, argv[i], lua_thread_str_error_cb) != 0) {
+ return;
+ }
+ }
+}
+
+static void
+rspamadm_lua_message_handler(lua_State *L, gint argc, gchar **argv)
+{
+ gulong cbref;
+ gint old_top, func_idx, i, j;
+ struct rspamd_task *task, **ptask;
+ gpointer map;
+ gsize len;
+ gchar outbuf[8192];
+ struct lua_logger_trace tr;
+
+ if (argv[1] == NULL) {
+ rspamd_printf("no callback is specified\n");
+ return;
+ }
+
+ for (i = 2; argv[i] != NULL; i++) {
+ struct thread_entry *thread = lua_thread_pool_get_for_config(rspamd_main->cfg);
+ L = thread->lua_state;
+
+ if (rspamd_strtoul(argv[1], strlen(argv[1]), &cbref)) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbref);
+ }
+ else {
+ lua_getglobal(L, argv[1]);
+ }
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ rspamd_printf("bad callback type: %s\n", lua_typename(L, lua_type(L, -1)));
+ lua_thread_pool_return(rspamd_main->cfg->lua_thread_pool, thread);
+ return;
+ }
+
+ /* Save index to reuse */
+ func_idx = lua_gettop(L);
+
+ map = rspamd_file_xmap(argv[i], PROT_READ, &len, TRUE);
+
+ if (map == NULL) {
+ rspamd_printf("cannot open %s: %s\n", argv[i], strerror(errno));
+ }
+ else {
+ task = rspamd_task_new(NULL, rspamd_main->cfg, NULL, NULL, NULL, FALSE);
+
+ if (!rspamd_task_load_message(task, NULL, map, len)) {
+ rspamd_printf("cannot load %s\n", argv[i]);
+ rspamd_task_free(task);
+ munmap(map, len);
+ continue;
+ }
+
+ if (!rspamd_message_parse(task)) {
+ rspamd_printf("cannot parse %s: %e\n", argv[i], task->err);
+ rspamd_task_free(task);
+ munmap(map, len);
+ continue;
+ }
+
+ rspamd_message_process(task);
+ old_top = lua_gettop(L);
+
+ lua_pushvalue(L, func_idx);
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+
+ if (lua_repl_thread_call(thread, 1, argv[i], lua_thread_str_error_cb) == 0) {
+ rspamd_printf("lua callback for %s returned:\n", argv[i]);
+
+ for (j = old_top + 1; j <= lua_gettop(L); j++) {
+ memset(&tr, 0, sizeof(tr));
+ lua_logger_out_type(L, j, outbuf, sizeof(outbuf), &tr,
+ LUA_ESCAPE_UNPRINTABLE);
+ rspamd_printf("%s\n", outbuf);
+ }
+ }
+
+ rspamd_task_free(task);
+ munmap(map, len);
+ /* Pop all but the original function */
+ lua_settop(L, func_idx);
+ }
+ }
+
+ lua_settop(L, 0);
+}
+
+
+static gboolean
+rspamadm_lua_try_dot_command(lua_State *L, const gchar *input)
+{
+ struct rspamadm_lua_dot_command *cmd;
+ gchar **argv;
+
+ argv = g_strsplit_set(input + 1, " ", -1);
+
+ if (argv == NULL || argv[0] == NULL) {
+ if (argv) {
+ g_strfreev(argv);
+ }
+
+ return FALSE;
+ }
+
+ cmd = g_hash_table_lookup(cmds_hash, argv[0]);
+
+ if (cmd) {
+ cmd->handler(L, g_strv_length(argv), argv);
+ g_strfreev(argv);
+
+ return TRUE;
+ }
+
+ g_strfreev(argv);
+
+ return FALSE;
+}
+
+#ifdef WITH_LUA_REPL
+static gint lex_ref_idx = -1;
+
+static void
+lua_syntax_highlighter(const char *str, ReplxxColor *colours, int size, void *ud)
+{
+ lua_State *L = (lua_State *) ud;
+
+ if (lex_ref_idx == -1) {
+ if (!rspamd_lua_require_function(L, "lua_lexer", "lex_to_table")) {
+ fprintf(stderr, "cannot require lua_lexer!\n");
+
+ exit(EXIT_FAILURE);
+ }
+
+ lex_ref_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, lex_ref_idx);
+ lua_pushstring(L, str);
+
+ if (lua_pcall(L, 1, 1, 0) != 0) {
+ fprintf(stderr, "cannot lex a string!\n");
+ }
+ else {
+ /* Process what we have after lexing */
+ gsize nelts = rspamd_lua_table_size(L, -1);
+
+ for (gsize i = 0; i < nelts; i++) {
+ /*
+ * Indexes in the table:
+ * 1 - type of element (string)
+ * 2 - text (string)
+ * 3 - line num (int), always 1...
+ * 4 - column num (must be less than size)
+ */
+ const gchar *what;
+ gsize column, tlen, cur_top, elt_pos;
+ ReplxxColor elt_color = REPLXX_COLOR_DEFAULT;
+
+ cur_top = lua_gettop(L);
+ lua_rawgeti(L, -1, i + 1);
+ elt_pos = lua_gettop(L);
+ lua_rawgeti(L, elt_pos, 1);
+ what = lua_tostring(L, -1);
+ lua_rawgeti(L, elt_pos, 2);
+ lua_tolstring(L, -1, &tlen);
+ lua_rawgeti(L, elt_pos, 4);
+ column = lua_tointeger(L, -1);
+
+ g_assert(column > 0);
+ column--; /* Start from 0 */
+
+ if (column + tlen > size) {
+ /* Likely utf8 case, too complicated to match */
+ lua_settop(L, cur_top);
+ continue;
+ }
+
+ /* Check what and adjust color */
+ if (strcmp(what, "identifier") == 0) {
+ elt_color = REPLXX_COLOR_NORMAL;
+ }
+ else if (strcmp(what, "number") == 0) {
+ elt_color = REPLXX_COLOR_BLUE;
+ }
+ else if (strcmp(what, "string") == 0) {
+ elt_color = REPLXX_COLOR_GREEN;
+ }
+ else if (strcmp(what, "keyword") == 0) {
+ elt_color = REPLXX_COLOR_WHITE;
+ }
+ else if (strcmp(what, "constant") == 0) {
+ elt_color = REPLXX_COLOR_WHITE;
+ }
+ else if (strcmp(what, "operator") == 0) {
+ elt_color = REPLXX_COLOR_CYAN;
+ }
+ else if (strcmp(what, "comment") == 0) {
+ elt_color = REPLXX_COLOR_BRIGHTGREEN;
+ }
+ else if (strcmp(what, "error") == 0) {
+ elt_color = REPLXX_COLOR_ERROR;
+ }
+
+ for (gsize j = column; j < column + tlen; j++) {
+ colours[j] = elt_color;
+ }
+
+ /* Restore stack */
+ lua_settop(L, cur_top);
+ }
+ }
+
+ lua_settop(L, 0);
+}
+#endif
+
+static void
+rspamadm_lua_run_repl(lua_State *L, bool is_batch)
+{
+ gchar *input;
+#ifdef WITH_LUA_REPL
+ gboolean is_multiline = FALSE;
+ GString *tb = NULL;
+ gsize i;
+#else
+ /* Always set is_batch */
+ is_batch = TRUE;
+#endif
+
+ for (;;) {
+ if (is_batch) {
+ size_t linecap = 0;
+ ssize_t linelen;
+
+ linelen = getline(&input, &linecap, stdin);
+
+ if (linelen > 0) {
+ if (input[linelen - 1] == '\n') {
+ input[linelen - 1] = '\0';
+ linelen--;
+ }
+
+ if (linelen > 0) {
+ if (input[0] == '.') {
+ if (rspamadm_lua_try_dot_command(L, input)) {
+ continue;
+ }
+ }
+
+ rspamadm_exec_input(L, input);
+ }
+ }
+ else {
+ break;
+ }
+
+ lua_settop(L, 0);
+ }
+ else {
+#ifdef WITH_LUA_REPL
+ replxx_set_highlighter_callback(rx_instance, lua_syntax_highlighter,
+ L);
+
+ if (!is_multiline) {
+ input = (gchar *) replxx_input(rx_instance, MAIN_PROMPT);
+
+ if (input == NULL) {
+ return;
+ }
+
+ if (input[0] == '.') {
+ if (rspamadm_lua_try_dot_command(L, input)) {
+ if (!is_batch) {
+ replxx_history_add(rx_instance, input);
+ }
+ continue;
+ }
+ }
+
+ if (strcmp(input, "{{") == 0) {
+ is_multiline = TRUE;
+ tb = g_string_sized_new(8192);
+ continue;
+ }
+
+ rspamadm_exec_input(L, input);
+ if (!is_batch) {
+ replxx_history_add(rx_instance, input);
+ }
+ lua_settop(L, 0);
+ }
+ else {
+ input = (gchar *) replxx_input(rx_instance, MULTILINE_PROMPT);
+
+ if (input == NULL) {
+ g_string_free(tb, TRUE);
+ return;
+ }
+
+ if (strcmp(input, "}}") == 0) {
+ is_multiline = FALSE;
+ rspamadm_exec_input(L, tb->str);
+
+ /* Replace \n with ' ' for sanity */
+ for (i = 0; i < tb->len; i++) {
+ if (tb->str[i] == '\n') {
+ tb->str[i] = ' ';
+ }
+ }
+
+ if (!is_batch) {
+ replxx_history_add(rx_instance, tb->str);
+ }
+ g_string_free(tb, TRUE);
+ }
+ else {
+ g_string_append(tb, input);
+ g_string_append(tb, " \n");
+ }
+ }
+ }
+#endif
+ }
+}
+
+struct rspamadm_lua_repl_context {
+ struct rspamd_http_connection_router *rt;
+ lua_State *L;
+};
+
+struct rspamadm_lua_repl_session {
+ struct rspamd_http_connection_router *rt;
+ rspamd_inet_addr_t *addr;
+ struct rspamadm_lua_repl_context *ctx;
+ gint sock;
+};
+
+static void
+rspamadm_lua_accept_cb(EV_P_ ev_io *w, int revents)
+{
+ struct rspamadm_lua_repl_context *ctx =
+ (struct rspamadm_lua_repl_context *) w->data;
+ rspamd_inet_addr_t *addr = NULL;
+ struct rspamadm_lua_repl_session *session;
+ gint nfd;
+
+ if ((nfd =
+ rspamd_accept_from_socket(w->fd, &addr, NULL, NULL)) == -1) {
+ rspamd_fprintf(stderr, "accept failed: %s", strerror(errno));
+ return;
+ }
+ /* Check for EAGAIN */
+ if (nfd == 0) {
+ rspamd_inet_address_free(addr);
+ return;
+ }
+
+ session = g_malloc0(sizeof(*session));
+ session->rt = ctx->rt;
+ session->ctx = ctx;
+ session->addr = addr;
+ session->sock = nfd;
+
+ rspamd_http_router_handle_socket(ctx->rt, nfd, session);
+}
+
+static void
+rspamadm_lua_error_handler(struct rspamd_http_connection_entry *conn_ent,
+ GError *err)
+{
+ rspamd_fprintf(stderr, "http error occurred: %s\n", err->message);
+}
+
+static void
+rspamadm_lua_finish_handler(struct rspamd_http_connection_entry *conn_ent)
+{
+ struct rspamadm_lua_repl_session *session = conn_ent->ud;
+
+ g_free(session);
+}
+
+static void
+lua_thread_http_error_cb(struct thread_entry *thread, int ret, const char *msg)
+{
+ struct lua_call_data *cd = thread->cd;
+ struct rspamd_http_connection_entry *conn_ent = cd->ud;
+
+ rspamd_controller_send_error(conn_ent, 500, "call failed: %s\n", msg);
+
+ cd->ret = ret;
+}
+
+
+/*
+ * Exec command handler:
+ * request: /exec
+ * body: lua script
+ * reply: json {"status": "ok", "reply": {<lua json object>}}
+ */
+static int
+rspamadm_lua_handle_exec(struct rspamd_http_connection_entry *conn_ent,
+ struct rspamd_http_message *msg)
+{
+ GString *tb;
+ gint err_idx, i;
+ lua_State *L;
+ ucl_object_t *obj, *elt;
+ const gchar *body;
+ gsize body_len;
+ struct thread_entry *thread = lua_thread_pool_get_for_config(rspamd_main->cfg);
+
+ L = thread->lua_state;
+
+ body = rspamd_http_message_get_body(msg, &body_len);
+
+ if (body == NULL) {
+ rspamd_controller_send_error(conn_ent, 400, "Empty lua script");
+
+ return 0;
+ }
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ /* First try return + input */
+ tb = g_string_sized_new(body_len + sizeof("return "));
+ rspamd_printf_gstring(tb, "return %*s", (gint) body_len, body);
+
+ if (luaL_loadstring(L, tb->str) != 0) {
+ /* Reset stack */
+ lua_settop(L, 0);
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ /* Try with no return */
+ if (luaL_loadbuffer(L, body, body_len, "http input") != 0) {
+ rspamd_controller_send_error(conn_ent, 400, "Invalid lua script");
+
+ return 0;
+ }
+ }
+
+ g_string_free(tb, TRUE);
+
+ if (lua_repl_thread_call(thread, 0, conn_ent, lua_thread_http_error_cb) != 0) {
+ return 0;
+ }
+
+ obj = ucl_object_typed_new(UCL_ARRAY);
+
+ for (i = err_idx + 1; i <= lua_gettop(L); i++) {
+ if (lua_isfunction(L, i)) {
+ /* XXX: think about API */
+ }
+ else {
+ elt = ucl_object_lua_import(L, i);
+
+ if (elt) {
+ ucl_array_append(obj, elt);
+ }
+ }
+ }
+
+ rspamd_controller_send_ucl(conn_ent, obj);
+ ucl_object_unref(obj);
+ lua_settop(L, 0);
+
+ return 0;
+}
+
+static void
+rspamadm_lua(gint argc, gchar **argv, const struct rspamadm_command *cmd)
+{
+ GOptionContext *context;
+ GError *error = NULL;
+ gchar **elt;
+ guint i;
+ lua_State *L = rspamd_main->cfg->lua_state;
+
+ context = g_option_context_new("lua - run lua interpreter");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd administration utility version " RVERSION
+ "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, NULL);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ g_option_context_free(context);
+
+ if (batch == -1) {
+ if (isatty(STDIN_FILENO)) {
+ batch = 0;
+ }
+ else {
+ batch = 1;
+ }
+ }
+
+ if (paths) {
+ for (elt = paths; *elt != NULL; elt++) {
+ rspamadm_lua_add_path(L, *elt);
+ }
+ }
+
+ if (lua_args) {
+ i = 1;
+
+ lua_newtable(L);
+
+ for (elt = lua_args; *elt != NULL; elt++) {
+ lua_pushinteger(L, i);
+ lua_pushstring(L, *elt);
+ lua_settable(L, -3);
+ i++;
+ }
+
+ lua_setglobal(L, "arg");
+ }
+
+ if (scripts) {
+ for (elt = scripts; *elt != NULL; elt++) {
+ if (!rspamadm_lua_load_script(L, *elt)) {
+ exit(EXIT_FAILURE);
+ }
+ }
+ }
+
+ if (exec_line) {
+ rspamadm_exec_input(L, exec_line);
+ }
+
+ if (serve) {
+ /* HTTP Server mode */
+ GPtrArray *addrs = NULL;
+ gchar *name = NULL;
+ struct ev_loop *ev_base;
+ struct rspamd_http_connection_router *http;
+ gint fd;
+ struct rspamadm_lua_repl_context *ctx;
+
+ if (rspamd_parse_host_port_priority(serve, &addrs, NULL, &name,
+ 10000, TRUE, NULL) == RSPAMD_PARSE_ADDR_FAIL) {
+ fprintf(stderr, "cannot listen on %s", serve);
+ exit(EXIT_FAILURE);
+ }
+
+ ev_base = rspamd_main->event_loop;
+ ctx = g_malloc0(sizeof(*ctx));
+ http = rspamd_http_router_new(rspamadm_lua_error_handler,
+ rspamadm_lua_finish_handler,
+ 0.0,
+ NULL,
+ rspamd_main->http_ctx);
+ ctx->L = L;
+ ctx->rt = http;
+ rspamd_http_router_add_path(http,
+ "/exec",
+ rspamadm_lua_handle_exec);
+
+ for (i = 0; i < addrs->len; i++) {
+ rspamd_inet_addr_t *addr = g_ptr_array_index(addrs, i);
+
+ fd = rspamd_inet_address_listen(addr, SOCK_STREAM,
+ RSPAMD_INET_ADDRESS_LISTEN_ASYNC, -1);
+
+ if (fd != -1) {
+ static ev_io ev;
+
+ ev.data = ctx;
+ ev_io_init(&ev, rspamadm_lua_accept_cb, fd, EV_READ);
+ ev_io_start(ev_base, &ev);
+ rspamd_printf("listen on %s\n",
+ rspamd_inet_address_to_string_pretty(addr));
+ }
+ }
+
+ ev_loop(ev_base, 0);
+
+ exit(EXIT_SUCCESS);
+ }
+
+ if (histfile == NULL) {
+ const gchar *homedir;
+ GString *hist_path;
+
+ homedir = getenv("HOME");
+
+ if (homedir) {
+ hist_path = g_string_sized_new(strlen(homedir) +
+ strlen(default_history_file) + 1);
+ rspamd_printf_gstring(hist_path, "%s/%s", homedir,
+ default_history_file);
+ }
+ else {
+ hist_path = g_string_sized_new(strlen(default_history_file) + 2);
+ rspamd_printf_gstring(hist_path, "./%s", default_history_file);
+ }
+
+ histfile = hist_path->str;
+ g_string_free(hist_path, FALSE);
+ }
+
+ if (argc > 1) {
+ for (i = 1; i < argc; i++) {
+ if (!rspamadm_lua_load_script(L, argv[i])) {
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ exit(EXIT_SUCCESS);
+ }
+
+ /* Init dot commands */
+ cmds_hash = g_hash_table_new(rspamd_strcase_hash, rspamd_strcase_equal);
+
+ for (i = 0; i < G_N_ELEMENTS(cmds); i++) {
+ g_hash_table_insert(cmds_hash, (gpointer) cmds[i].name, &cmds[i]);
+ }
+
+ if (!batch) {
+#ifdef WITH_LUA_REPL
+ rx_instance = replxx_init();
+ replxx_set_max_history_size(rx_instance, max_history);
+ replxx_history_load(rx_instance, histfile);
+#endif
+ rspamadm_lua_run_repl(L, false);
+#ifdef WITH_LUA_REPL
+ replxx_history_save(rx_instance, histfile);
+ replxx_end(rx_instance);
+#endif
+ }
+ else {
+ rspamadm_lua_run_repl(L, true);
+ }
+}
diff --git a/src/rspamadm/pw.c b/src/rspamadm/pw.c
new file mode 100644
index 0000000..9fe9cd7
--- /dev/null
+++ b/src/rspamadm/pw.c
@@ -0,0 +1,392 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "util.h"
+#include "ottery.h"
+#include "cryptobox.h"
+#include "rspamd.h"
+#include "rspamadm.h"
+#include "unix-std.h"
+
+static void rspamadm_pw(gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_pw_help(gboolean full_help,
+ const struct rspamadm_command *cmd);
+static void rspamadm_pw_lua_subrs(gpointer pL);
+
+static gboolean do_encrypt = FALSE;
+static gboolean do_check = FALSE;
+static gboolean quiet = FALSE;
+static gboolean list = FALSE;
+static gchar *type = "catena";
+static gchar *password = NULL;
+
+struct rspamadm_command pw_command = {
+ .name = "pw",
+ .flags = 0,
+ .help = rspamadm_pw_help,
+ .run = rspamadm_pw,
+ .lua_subrs = rspamadm_pw_lua_subrs,
+};
+
+static GOptionEntry entries[] = {
+ {"encrypt", 'e', 0, G_OPTION_ARG_NONE, &do_encrypt,
+ "Encrypt password", NULL},
+ {"check", 'c', 0, G_OPTION_ARG_NONE, &do_check,
+ "Check password", NULL},
+ {"quiet", 'q', 0, G_OPTION_ARG_NONE, &quiet,
+ "Suppress output", NULL},
+ {"password", 'p', 0, G_OPTION_ARG_STRING, &password,
+ "Input password", NULL},
+ {"type", 't', 0, G_OPTION_ARG_STRING, &type,
+ "PBKDF type", NULL},
+ {"list", 'l', 0, G_OPTION_ARG_NONE, &list,
+ "List available algorithms", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+static const char *
+rspamadm_pw_help(gboolean full_help, const struct rspamadm_command *cmd)
+{
+ const char *help_str;
+
+ if (full_help) {
+ help_str = "Manipulate with passwords in Rspamd\n\n"
+ "Usage: rspamadm pw [command]\n"
+ "Where commands are:\n\n"
+ "--encrypt: encrypt password (this is a default command)\n"
+ "--check: check encrypted password using encrypted password\n"
+ "--list: list available pbkdf algorithms\n"
+ "--password: input password\n"
+ "--type: select the specified pbkdf type\n"
+ "--help: shows available options and commands";
+ }
+ else {
+ help_str = "Manage rspamd passwords";
+ }
+
+ return help_str;
+}
+
+static const struct rspamd_controller_pbkdf *
+rspamadm_get_pbkdf(void)
+{
+ const struct rspamd_controller_pbkdf *pbkdf;
+ guint i;
+
+ for (i = 0; i < RSPAMD_PBKDF_ID_MAX - 1; i++) {
+ pbkdf = &pbkdf_list[i];
+
+ if (strcmp(type, pbkdf->alias) == 0) {
+ return pbkdf;
+ }
+ }
+
+ rspamd_fprintf(stderr, "Unknown PKDF type: %s\n", type);
+ exit(EXIT_FAILURE);
+
+ return NULL;
+}
+
+static char *
+rspamadm_pw_encrypt(char *password)
+{
+ const struct rspamd_controller_pbkdf *pbkdf;
+ guchar *salt, *key;
+ gchar *encoded_salt, *encoded_key;
+ GString *result;
+ gsize plen;
+
+ pbkdf = rspamadm_get_pbkdf();
+ g_assert(pbkdf != NULL);
+
+ if (password == NULL) {
+ plen = 8192;
+ password = g_malloc0(plen);
+ plen = rspamd_read_passphrase(password, plen, 0, NULL);
+ }
+ else {
+ plen = strlen(password);
+ }
+
+ if (plen == 0) {
+ fprintf(stderr, "Invalid password\n");
+ exit(EXIT_FAILURE);
+ }
+
+ salt = g_alloca(pbkdf->salt_len);
+ key = g_alloca(pbkdf->key_len);
+ ottery_rand_bytes(salt, pbkdf->salt_len);
+ /* Derive key */
+ rspamd_cryptobox_pbkdf(password, strlen(password),
+ salt, pbkdf->salt_len, key, pbkdf->key_len, pbkdf->complexity,
+ pbkdf->type);
+
+ encoded_salt = rspamd_encode_base32(salt, pbkdf->salt_len, RSPAMD_BASE32_DEFAULT);
+ encoded_key = rspamd_encode_base32(key, pbkdf->key_len, RSPAMD_BASE32_DEFAULT);
+
+ result = g_string_new("");
+ rspamd_printf_gstring(result, "$%d$%s$%s", pbkdf->id, encoded_salt,
+ encoded_key);
+
+ g_free(encoded_salt);
+ g_free(encoded_key);
+ rspamd_explicit_memzero(password, plen);
+ g_free(password);
+ password = result->str;
+ g_string_free(result, FALSE); /* Not freeing memory */
+
+ return password;
+}
+
+static const gchar *
+rspamd_encrypted_password_get_str(const gchar *password, gsize skip,
+ gsize *length)
+{
+ const gchar *str, *start, *end;
+ gsize size;
+
+ start = password + skip;
+ end = start;
+ size = 0;
+
+ while (*end != '\0' && g_ascii_isalnum(*end)) {
+ size++;
+ end++;
+ }
+
+ if (size) {
+ str = start;
+ *length = size;
+ }
+ else {
+ str = NULL;
+ }
+
+ return str;
+}
+
+static void
+rspamadm_pw_check(void)
+{
+ const struct rspamd_controller_pbkdf *pbkdf = NULL;
+ const gchar *salt, *hash;
+ const gchar *start, *end;
+ guchar *salt_decoded, *key_decoded, *local_key;
+ gsize salt_len, key_len, size;
+ gchar test_password[8192], encrypted_password[8192];
+ gsize plen, i;
+ gint id;
+ gboolean ret = FALSE;
+
+ if (password == NULL) {
+ plen = rspamd_read_passphrase_with_prompt("Enter encrypted password: ", encrypted_password,
+ sizeof(encrypted_password), 1, NULL);
+ }
+ else {
+ plen = rspamd_strlcpy(encrypted_password, password, sizeof(encrypted_password));
+ }
+
+ if (encrypted_password[0] == '$') {
+ /* Parse id */
+ start = encrypted_password + 1;
+ end = start;
+ size = 0;
+
+ while (*end != '\0' && g_ascii_isdigit(*end)) {
+ size++;
+ end++;
+ }
+
+ if (size > 0) {
+ gchar *endptr;
+ id = strtoul(start, &endptr, 10);
+
+ if ((endptr == NULL || *endptr == *end)) {
+ for (i = 0; i < RSPAMD_PBKDF_ID_MAX - 1; i++) {
+ pbkdf = &pbkdf_list[i];
+
+ if (pbkdf->id == id) {
+ ret = TRUE;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ if (!ret) {
+ rspamd_fprintf(stderr, "Invalid password format\n");
+ rspamd_explicit_memzero(encrypted_password, sizeof(encrypted_password));
+ exit(EXIT_FAILURE);
+ }
+
+ if (plen < pbkdf->salt_len + pbkdf->key_len + 3) {
+ msg_err("incorrect salt: password length: %z, must be at least %z characters",
+ plen, pbkdf->salt_len);
+ rspamd_explicit_memzero(encrypted_password, sizeof(encrypted_password));
+ exit(EXIT_FAILURE);
+ }
+
+ /* get salt */
+ salt = rspamd_encrypted_password_get_str(encrypted_password, 3, &salt_len);
+ /* get hash */
+ hash = rspamd_encrypted_password_get_str(encrypted_password,
+ 3 + salt_len + 1,
+ &key_len);
+ if (salt != NULL && hash != NULL) {
+
+ /* decode salt */
+ salt_decoded = rspamd_decode_base32(salt, salt_len, &salt_len, RSPAMD_BASE32_DEFAULT);
+
+ if (salt_decoded == NULL || salt_len != pbkdf->salt_len) {
+ /* We have some unknown salt here */
+ rspamd_explicit_memzero(encrypted_password, sizeof(encrypted_password));
+ msg_err("incorrect salt: %z, while %z expected",
+ salt_len, pbkdf->salt_len);
+ exit(EXIT_FAILURE);
+ }
+
+ key_decoded = rspamd_decode_base32(hash, key_len, &key_len, RSPAMD_BASE32_DEFAULT);
+
+ if (key_decoded == NULL || key_len != pbkdf->key_len) {
+ /* We have some unknown salt here */
+ rspamd_explicit_memzero(encrypted_password, sizeof(encrypted_password));
+ msg_err("incorrect key: %z, while %z expected",
+ key_len, pbkdf->key_len);
+ exit(EXIT_FAILURE);
+ }
+
+ plen = rspamd_read_passphrase(test_password, sizeof(test_password),
+ 0, NULL);
+ if (plen == 0) {
+ rspamd_explicit_memzero(encrypted_password, sizeof(encrypted_password));
+ fprintf(stderr, "Invalid password\n");
+ exit(EXIT_FAILURE);
+ }
+
+ local_key = g_alloca(pbkdf->key_len);
+ rspamd_cryptobox_pbkdf(test_password, plen,
+ salt_decoded, salt_len,
+ local_key, pbkdf->key_len,
+ pbkdf->complexity,
+ pbkdf->type);
+ rspamd_explicit_memzero(test_password, plen);
+ rspamd_explicit_memzero(encrypted_password, sizeof(encrypted_password));
+
+ if (!rspamd_constant_memcmp(key_decoded, local_key, pbkdf->key_len)) {
+ if (!quiet) {
+ rspamd_printf("password incorrect\n");
+ }
+ exit(EXIT_FAILURE);
+ }
+
+ g_free(salt_decoded);
+ g_free(key_decoded);
+ }
+ else {
+ msg_err("bad encrypted password format");
+ rspamd_explicit_memzero(encrypted_password, sizeof(encrypted_password));
+ exit(EXIT_FAILURE);
+ }
+
+ if (!quiet) {
+ rspamd_printf("password correct\n");
+ }
+}
+
+static gint
+rspamadm_pw_lua_encrypt(lua_State *L)
+{
+ const gchar *pw_in = NULL;
+ gchar *ret, *tmp = NULL;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ pw_in = lua_tostring(L, 1);
+ tmp = g_strdup(pw_in);
+ }
+
+ ret = rspamadm_pw_encrypt(tmp);
+
+ lua_pushstring(L, ret);
+ g_free(ret);
+
+ return 1;
+}
+
+
+static void
+rspamadm_pw_lua_subrs(gpointer pL)
+{
+ lua_State *L = pL;
+
+ lua_pushstring(L, "pw_encrypt");
+ lua_pushcfunction(L, rspamadm_pw_lua_encrypt);
+ lua_settable(L, -3);
+}
+
+static void
+rspamadm_alg_list(void)
+{
+ const struct rspamd_controller_pbkdf *pbkdf;
+ guint i;
+
+ for (i = 0; i < RSPAMD_PBKDF_ID_MAX - 1; i++) {
+ pbkdf = &pbkdf_list[i];
+
+ rspamd_printf("%s: %s - %s\n", pbkdf->alias, pbkdf->name,
+ pbkdf->description);
+ }
+}
+
+static void
+rspamadm_pw(gint argc, gchar **argv, const struct rspamadm_command *cmd)
+{
+ GOptionContext *context;
+ GError *error = NULL;
+
+ context = g_option_context_new("pw [--encrypt | --check] - manage rspamd passwords");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd administration utility version " RVERSION
+ "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, NULL);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ g_option_context_free(context);
+
+ if (list) {
+ rspamadm_alg_list();
+ exit(EXIT_SUCCESS);
+ }
+
+ if (!do_encrypt && !do_check) {
+ do_encrypt = TRUE;
+ }
+
+ if (do_encrypt) {
+ gchar *encr = rspamadm_pw_encrypt(password);
+ rspamd_printf("%s\n", encr);
+ g_free(encr);
+ }
+ else if (do_check) {
+ rspamadm_pw_check();
+ }
+}
diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c
new file mode 100644
index 0000000..0e38dc3
--- /dev/null
+++ b/src/rspamadm/rspamadm.c
@@ -0,0 +1,621 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamadm.h"
+#include "rspamd.h"
+#include "ottery.h"
+#include "lua/lua_common.h"
+#include "lua/lua_thread_pool.h"
+#include "lua_ucl.h"
+#include "unix-std.h"
+#include "contrib/libev/ev.h"
+
+#ifdef HAVE_LIBUTIL_H
+#include <libutil.h>
+#endif
+
+static gboolean verbose = FALSE;
+static gboolean list_commands = FALSE;
+static gboolean show_help = FALSE;
+static gboolean show_version = FALSE;
+GHashTable *ucl_vars = NULL;
+gchar **lua_env = NULL;
+struct rspamd_main *rspamd_main = NULL;
+struct rspamd_async_session *rspamadm_session = NULL;
+lua_State *L = NULL;
+
+/* Defined in modules.c */
+extern module_t *modules[];
+extern worker_t *workers[];
+
+static void rspamadm_help(gint argc, gchar **argv, const struct rspamadm_command *);
+static const char *rspamadm_help_help(gboolean full_help, const struct rspamadm_command *);
+
+struct rspamadm_command help_command = {
+ .name = "help",
+ .flags = RSPAMADM_FLAG_NOHELP,
+ .help = rspamadm_help_help,
+ .run = rspamadm_help};
+
+static gboolean rspamadm_parse_ucl_var(const gchar *option_name,
+ const gchar *value, gpointer data,
+ GError **error);
+
+
+static GOptionEntry entries[] = {
+ {"verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose,
+ "Enable verbose logging", NULL},
+ {"list-commands", 'l', 0, G_OPTION_ARG_NONE, &list_commands,
+ "List available commands", NULL},
+ {"var", 0, 0, G_OPTION_ARG_CALLBACK, (gpointer) &rspamadm_parse_ucl_var,
+ "Redefine/define environment variable", NULL},
+ {"help", 'h', 0, G_OPTION_ARG_NONE, &show_help,
+ "Show help", NULL},
+ {"version", 'V', 0, G_OPTION_ARG_NONE, &show_version,
+ "Show version", NULL},
+ {"lua-env", '\0', 0, G_OPTION_ARG_FILENAME_ARRAY, &lua_env,
+ "Load lua environment from the specified files", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+GQuark
+rspamadm_error(void)
+{
+ return g_quark_from_static_string("rspamadm");
+}
+
+static void
+rspamadm_version(void)
+{
+ rspamd_printf("Rspamadm %s\n", RVERSION);
+}
+
+static void
+rspamadm_usage(GOptionContext *context)
+{
+ gchar *help_str;
+
+ help_str = g_option_context_get_help(context, TRUE, NULL);
+ rspamd_printf("%s", help_str);
+}
+
+static void
+rspamadm_commands(GPtrArray *all_commands)
+{
+ const struct rspamadm_command *cmd;
+ guint i;
+
+ rspamd_printf("Rspamadm %s\n", RVERSION);
+ rspamd_printf("Usage: rspamadm [global_options] command [command_options]\n");
+ rspamd_printf("\nAvailable commands:\n");
+
+ PTR_ARRAY_FOREACH(all_commands, i, cmd)
+ {
+ if (!(cmd->flags & RSPAMADM_FLAG_NOHELP)) {
+ if (cmd->flags & RSPAMADM_FLAG_LUA) {
+ (void) cmd->help(FALSE, cmd);
+ }
+ else {
+ printf(" %-18s %-60s\n", cmd->name, cmd->help(FALSE, cmd));
+ }
+ }
+ }
+}
+
+static const char *
+rspamadm_help_help(gboolean full_help, const struct rspamadm_command *cmd)
+{
+ const char *help_str;
+
+ if (full_help) {
+ help_str = "Shows help for a specified command\n"
+ "Usage: rspamadm help <command>";
+ }
+ else {
+ help_str = "Shows help for a specified command";
+ }
+
+ return help_str;
+}
+
+static void
+rspamadm_help(gint argc, gchar **argv, const struct rspamadm_command *command)
+{
+ const gchar *cmd_name;
+ const struct rspamadm_command *cmd;
+ GPtrArray *all_commands = (GPtrArray *) command->command_data;
+
+ rspamd_printf("Rspamadm %s\n", RVERSION);
+ rspamd_printf("Usage: rspamadm [global_options] command [command_options]\n\n");
+
+ if (argc <= 1) {
+ cmd_name = "help";
+ }
+ else {
+ cmd_name = argv[1];
+ rspamd_printf("Showing help for %s command\n\n", cmd_name);
+ }
+
+ cmd = rspamadm_search_command(cmd_name, all_commands);
+
+ if (cmd == NULL) {
+ fprintf(stderr, "Invalid command name: %s\n", cmd_name);
+ exit(EXIT_FAILURE);
+ }
+
+ if (strcmp(cmd_name, "help") == 0) {
+ guint i;
+ rspamd_printf("Available commands:\n");
+
+ PTR_ARRAY_FOREACH(all_commands, i, cmd)
+ {
+ if (!(cmd->flags & RSPAMADM_FLAG_NOHELP)) {
+ if (!(cmd->flags & RSPAMADM_FLAG_LUA)) {
+ printf(" %-18s %-60s\n", cmd->name,
+ cmd->help(FALSE, cmd));
+ }
+ else {
+ /* Just call lua subr */
+ (void) cmd->help(FALSE, cmd);
+ }
+ }
+ }
+ }
+ else {
+ if (!(cmd->flags & RSPAMADM_FLAG_LUA)) {
+ rspamd_printf("%s\n", cmd->help(TRUE, cmd));
+ }
+ else {
+ /* Just call lua subr */
+ (void) cmd->help(TRUE, cmd);
+ }
+ }
+}
+
+static gboolean
+rspamadm_parse_ucl_var(const gchar *option_name,
+ const gchar *value, gpointer data,
+ GError **error)
+{
+ gchar *k, *v, *t;
+
+ t = strchr(value, '=');
+
+ if (t != NULL) {
+ k = g_strdup(value);
+ t = k + (t - value);
+ v = g_strdup(t + 1);
+ *t = '\0';
+
+ g_hash_table_insert(ucl_vars, k, v);
+ }
+ else {
+ g_set_error(error, rspamadm_error(), EINVAL,
+ "Bad variable format: %s", value);
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static void
+lua_thread_str_error_cb(struct thread_entry *thread, int ret, const char *msg)
+{
+ struct lua_call_data *cd = thread->cd;
+
+ msg_err("call to rspamadm lua script failed (%d): %s", ret, msg);
+
+ cd->ret = ret;
+}
+
+gboolean
+rspamadm_execute_lua_ucl_subr(gint argc, gchar **argv,
+ const ucl_object_t *res,
+ const gchar *script_name,
+ gboolean rspamadm_subcommand)
+{
+ struct thread_entry *thread = lua_thread_pool_get_for_config(rspamd_main->cfg);
+
+ lua_State *L = thread->lua_state;
+
+ gint i;
+ gchar str[PATH_MAX];
+
+ g_assert(script_name != NULL);
+ g_assert(res != NULL);
+ g_assert(L != NULL);
+
+ /* Init internal rspamadm routines */
+
+ if (rspamadm_subcommand) {
+ rspamd_snprintf(str, sizeof(str), "return require \"%s.%s\"", "rspamadm",
+ script_name);
+ }
+ else {
+ rspamd_snprintf(str, sizeof(str), "return require \"%s\"",
+ script_name);
+ }
+
+ if (luaL_dostring(L, str) != 0) {
+ msg_err("cannot execute lua script %s: %s",
+ str, lua_tostring(L, -1));
+ return FALSE;
+ }
+ else {
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_pushstring(L, "handler");
+ lua_gettable(L, -2);
+ }
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_err("lua script must return "
+ "function and not %s",
+ lua_typename(L, lua_type(L, -1)));
+
+ return FALSE;
+ }
+ }
+
+ /* Push function */
+ lua_pushvalue(L, -1);
+
+ /* Push argv */
+ lua_newtable(L);
+
+ for (i = 1; i < argc; i++) {
+ lua_pushstring(L, argv[i]);
+ lua_rawseti(L, -2, i);
+ }
+
+ /* Push results */
+ ucl_object_push_lua(L, res, TRUE);
+
+ if (lua_repl_thread_call(thread, 2, NULL, lua_thread_str_error_cb) != 0) {
+
+ return FALSE;
+ }
+
+ /* error function */
+ lua_settop(L, 0);
+
+ return TRUE;
+}
+
+static gint
+rspamdadm_commands_sort_func(gconstpointer a, gconstpointer b)
+{
+ const struct rspamadm_command *cmda = *((struct rspamadm_command const **) a),
+ *cmdb = *((struct rspamadm_command const **) b);
+
+ return strcmp(cmda->name, cmdb->name);
+}
+
+static gboolean
+rspamadm_command_maybe_match_name(const gchar *cmd, const gchar *input)
+{
+ gsize clen, inplen;
+
+ clen = strlen(cmd);
+ inplen = strlen(input);
+
+ if (rspamd_strings_levenshtein_distance(cmd, clen,
+ input, inplen, 1) == 1) {
+ return TRUE;
+ }
+ else if ((clen > inplen &&
+ rspamd_substring_search(cmd, clen, input, inplen) != -1) ||
+ (inplen > clen &&
+ rspamd_substring_search(input, inplen, cmd, clen) != -1)) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+
+static void
+rspamadm_add_lua_globals(struct rspamd_dns_resolver *resolver)
+{
+ struct rspamd_async_session **psession;
+ struct ev_loop **pev_base;
+ struct rspamd_dns_resolver **presolver;
+
+ rspamadm_session = rspamd_session_create(rspamd_main->cfg->cfg_pool, NULL,
+ NULL, (event_finalizer_t) NULL, NULL);
+
+ psession = lua_newuserdata(L, sizeof(struct rspamd_async_session *));
+ rspamd_lua_setclass(L, "rspamd{session}", -1);
+ *psession = rspamadm_session;
+ lua_setglobal(L, "rspamadm_session");
+
+ pev_base = lua_newuserdata(L, sizeof(struct ev_loop *));
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+ *pev_base = rspamd_main->event_loop;
+ lua_setglobal(L, "rspamadm_ev_base");
+
+ presolver = lua_newuserdata(L, sizeof(struct rspamd_dns_resolver *));
+ rspamd_lua_setclass(L, "rspamd{resolver}", -1);
+ *presolver = resolver;
+ lua_setglobal(L, "rspamadm_dns_resolver");
+}
+
+static void
+rspamadm_cmd_dtor(gpointer p)
+{
+ struct rspamadm_command *cmd = (struct rspamadm_command *) p;
+
+ if (cmd->flags & RSPAMADM_FLAG_DYNAMIC) {
+ if (cmd->aliases) {
+ g_ptr_array_free(cmd->aliases, TRUE);
+ }
+
+ g_free((gpointer) cmd->name);
+ g_free(cmd);
+ }
+}
+
+gint main(gint argc, gchar **argv, gchar **env)
+{
+ GError *error = NULL;
+ GOptionContext *context;
+ GOptionGroup *og;
+ struct rspamd_config *cfg;
+ GQuark process_quark;
+ gchar **nargv, **targv;
+ const gchar *cmd_name;
+ const struct rspamadm_command *cmd;
+ struct rspamd_dns_resolver *resolver;
+ GPtrArray *all_commands = g_ptr_array_new_full(32,
+ rspamadm_cmd_dtor); /* Discovered during check */
+ gint i, nargc, targc;
+ worker_t **pworker;
+ gboolean lua_file = FALSE;
+ gint retcode = 0;
+
+ ucl_vars = g_hash_table_new_full(rspamd_strcase_hash,
+ rspamd_strcase_equal, g_free, g_free);
+ process_quark = g_quark_from_static_string("rspamadm");
+ cfg = rspamd_config_new(RSPAMD_CONFIG_INIT_DEFAULT | RSPAMD_CONFIG_INIT_WIPE_LUA_MEM);
+ cfg->libs_ctx = rspamd_init_libs();
+ rspamd_main = g_malloc0(sizeof(*rspamd_main));
+ rspamd_main->cfg = cfg;
+ rspamd_main->pid = getpid();
+ rspamd_main->type = process_quark;
+ rspamd_main->server_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "rspamadm", 0);
+
+ rspamadm_fill_internal_commands(all_commands);
+ help_command.command_data = all_commands;
+
+ /* Now read options and store everything till the first non-dash argument */
+ nargv = g_malloc0(sizeof(gchar *) * (argc + 1));
+ nargv[0] = g_strdup(argv[0]);
+
+ for (i = 1, nargc = 1; i < argc; i++) {
+ if (argv[i] && argv[i][0] == '-') {
+ /* Copy to nargv */
+ nargv[nargc] = g_strdup(argv[i]);
+ nargc++;
+ }
+ else {
+ break;
+ }
+ }
+
+ context = g_option_context_new("command - rspamd administration utility");
+ og = g_option_group_new("global", "global options", "global options",
+ NULL, NULL);
+ g_option_context_set_help_enabled(context, FALSE);
+ g_option_group_add_entries(og, entries);
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd administration utility version " RVERSION
+ "\n Release id: " RID);
+ g_option_context_set_main_group(context, og);
+
+ targv = nargv;
+ targc = nargc;
+
+ if (!g_option_context_parse(context, &targc, &targv, &error)) {
+ fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+
+ /* Setup logger */
+ if (verbose) {
+ rspamd_main->logger = rspamd_log_open_emergency(rspamd_main->server_pool,
+ RSPAMD_LOG_FLAG_USEC | RSPAMD_LOG_FLAG_ENFORCED | RSPAMD_LOG_FLAG_RSPAMADM);
+ rspamd_log_set_log_level(rspamd_main->logger, G_LOG_LEVEL_DEBUG);
+ }
+ else {
+ rspamd_main->logger = rspamd_log_open_emergency(rspamd_main->server_pool,
+ RSPAMD_LOG_FLAG_RSPAMADM);
+ rspamd_log_set_log_level(rspamd_main->logger, G_LOG_LEVEL_MESSAGE);
+ }
+
+ rspamd_main->event_loop = ev_default_loop(rspamd_config_ev_backend_get(cfg));
+
+ resolver = rspamd_dns_resolver_init(rspamd_main->logger,
+ rspamd_main->event_loop,
+ cfg);
+ rspamd_main->http_ctx = rspamd_http_context_create(cfg, rspamd_main->event_loop,
+ NULL);
+
+ g_log_set_default_handler(rspamd_glib_log_function, rspamd_main->logger);
+ g_set_printerr_handler(rspamd_glib_printerr_function);
+ rspamd_config_post_load(cfg,
+ RSPAMD_CONFIG_INIT_LIBS | RSPAMD_CONFIG_INIT_URL | RSPAMD_CONFIG_INIT_NO_TLD);
+
+ pworker = &workers[0];
+ while (*pworker) {
+ /* Init string quarks */
+ (void) g_quark_from_static_string((*pworker)->name);
+ pworker++;
+ }
+
+ cfg->compiled_modules = modules;
+ cfg->compiled_workers = workers;
+
+ rspamd_setproctitle("rspamdadm");
+
+ L = cfg->lua_state;
+ rspamd_lua_set_path(L, NULL, ucl_vars);
+
+ if (!rspamd_lua_set_env(L, ucl_vars, lua_env, &error)) {
+ rspamd_fprintf(stderr, "Cannot load lua environment: %e", error);
+ g_error_free(error);
+
+ goto end;
+ }
+
+ rspamd_lua_set_globals(cfg, L);
+ rspamadm_add_lua_globals(resolver);
+ rspamd_redis_pool_config(cfg->redis_pool, cfg, rspamd_main->event_loop);
+
+ /* Init rspamadm global */
+ lua_newtable(L);
+
+ PTR_ARRAY_FOREACH(all_commands, i, cmd)
+ {
+ if (cmd->lua_subrs != NULL) {
+ cmd->lua_subrs(L);
+ }
+
+ cmd++;
+ }
+
+ lua_setglobal(L, "rspamadm");
+
+ rspamadm_fill_lua_commands(L, all_commands);
+ rspamd_lua_start_gc(cfg);
+ g_ptr_array_sort(all_commands, rspamdadm_commands_sort_func);
+
+ g_strfreev(nargv);
+
+ if (show_version) {
+ rspamadm_version();
+ goto end;
+ }
+ if (show_help) {
+ rspamadm_usage(context);
+ goto end;
+ }
+ if (list_commands) {
+ rspamadm_commands(all_commands);
+ goto end;
+ }
+
+ cmd_name = argv[nargc];
+
+ if (cmd_name == NULL) {
+ cmd_name = "help";
+ }
+
+ gsize cmdlen = strlen(cmd_name);
+
+ if (cmdlen > 4 && memcmp(cmd_name + (cmdlen - 4), ".lua", 4) == 0) {
+ cmd_name = "lua";
+ lua_file = TRUE;
+ }
+
+ cmd = rspamadm_search_command(cmd_name, all_commands);
+
+ if (cmd == NULL) {
+ rspamd_fprintf(stderr, "Invalid command name: %s\n", cmd_name);
+
+ /* Try fuzz search */
+ rspamd_fprintf(stderr, "Suggested commands:\n");
+ PTR_ARRAY_FOREACH(all_commands, i, cmd)
+ {
+ guint j;
+ const gchar *alias;
+
+ if (rspamadm_command_maybe_match_name(cmd->name, cmd_name)) {
+ rspamd_fprintf(stderr, "%s\n", cmd->name);
+ }
+ else {
+ PTR_ARRAY_FOREACH(cmd->aliases, j, alias)
+ {
+ if (rspamadm_command_maybe_match_name(alias, cmd_name)) {
+ rspamd_fprintf(stderr, "%s\n", alias);
+ }
+ }
+ }
+ }
+
+ retcode = EXIT_FAILURE;
+ goto end;
+ }
+
+ if (nargc < argc) {
+
+ if (lua_file) {
+ nargv = g_malloc0(sizeof(gchar *) * (argc - nargc + 2));
+ nargv[1] = g_strdup(argv[nargc]);
+ i = 2;
+ argc++;
+ }
+ else {
+ nargv = g_malloc0(sizeof(gchar *) * (argc - nargc + 1));
+ i = 1;
+ }
+
+ nargv[0] = g_strdup_printf("%s %s", argv[0], cmd_name);
+
+ for (; i < argc - nargc; i++) {
+ if (lua_file) {
+ /*
+ * We append prefix '--arg=' to each argument and shift argv index
+ */
+ gsize arglen = strlen(argv[i + nargc - 1]);
+
+ arglen += sizeof("--args="); /* Including \0 */
+ nargv[i] = g_malloc(arglen);
+ rspamd_snprintf(nargv[i], arglen, "--args=%s", argv[i + nargc - 1]);
+ }
+ else {
+ nargv[i] = g_strdup(argv[i + nargc]);
+ }
+ }
+
+ targc = argc - nargc;
+ targv = nargv;
+ cmd->run(targc, targv, cmd);
+ g_strfreev(nargv);
+ }
+ else {
+ cmd->run(0, NULL, cmd);
+ }
+
+ ev_break(rspamd_main->event_loop, EVBREAK_ALL);
+
+end:
+ rspamd_session_destroy(rspamadm_session);
+ g_option_context_free(context);
+ rspamd_dns_resolver_deinit(resolver);
+ REF_RELEASE(rspamd_main->cfg);
+ rspamd_http_context_free(rspamd_main->http_ctx);
+ rspamd_log_close(rspamd_main->logger);
+ rspamd_url_deinit();
+ g_ptr_array_free(all_commands, TRUE);
+ ev_loop_destroy(rspamd_main->event_loop);
+ g_hash_table_unref(ucl_vars);
+ rspamd_mempool_delete(rspamd_main->server_pool);
+ g_free(rspamd_main);
+
+ return retcode;
+}
diff --git a/src/rspamadm/rspamadm.h b/src/rspamadm/rspamadm.h
new file mode 100644
index 0000000..5fe51c3
--- /dev/null
+++ b/src/rspamadm/rspamadm.h
@@ -0,0 +1,92 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_RSPAMDADM_H
+#define RSPAMD_RSPAMDADM_H
+
+#include "config.h"
+#include "ucl.h"
+#include <lua.h>
+#include <lauxlib.h>
+#include <lualib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern GHashTable *ucl_vars;
+extern gchar **lua_env;
+extern struct rspamd_main *rspamd_main;
+
+GQuark rspamadm_error(void);
+
+struct rspamadm_command;
+
+typedef const gchar *(*rspamadm_help_func)(gboolean full_help,
+ const struct rspamadm_command *cmd);
+
+typedef void (*rspamadm_run_func)(gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+
+typedef void (*rspamadm_lua_exports_func)(gpointer lua_state);
+
+#define RSPAMADM_FLAG_NOHELP (1u << 0u)
+#define RSPAMADM_FLAG_LUA (1u << 1u)
+#define RSPAMADM_FLAG_DYNAMIC (1u << 2u)
+
+struct rspamadm_command {
+ const gchar *name;
+ guint flags;
+ rspamadm_help_func help;
+ rspamadm_run_func run;
+ rspamadm_lua_exports_func lua_subrs;
+ GPtrArray *aliases;
+ gpointer command_data; /* Opaque data */
+};
+
+extern const struct rspamadm_command *commands[];
+extern struct rspamadm_command help_command;
+
+const struct rspamadm_command *rspamadm_search_command(const gchar *name,
+ GPtrArray *all_commands);
+
+void rspamadm_fill_internal_commands(GPtrArray *dest);
+
+void rspamadm_fill_lua_commands(lua_State *L, GPtrArray *dest);
+
+gboolean rspamadm_execute_lua_ucl_subr(gint argc, gchar **argv,
+ const ucl_object_t *res,
+ const gchar *script_name,
+ gboolean rspamadm_subcommand);
+
+struct thread_entry;
+
+typedef void (*lua_thread_error_t)(struct thread_entry *thread, int ret, const char *msg);
+
+
+struct lua_call_data {
+ gint top;
+ gint ret;
+ gpointer ud;
+};
+
+gint lua_repl_thread_call(struct thread_entry *thread, gint narg,
+ gpointer ud, lua_thread_error_t error_func);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/rspamadm/signtool.c b/src/rspamadm/signtool.c
new file mode 100644
index 0000000..b39b870
--- /dev/null
+++ b/src/rspamadm/signtool.c
@@ -0,0 +1,623 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "rspamadm.h"
+#include "cryptobox.h"
+#include "printf.h"
+#include "ucl.h"
+#include "libcryptobox/keypair.h"
+#include "libutil/str_util.h"
+#include "libutil/util.h"
+#include "unix-std.h"
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+
+static gboolean openssl = FALSE;
+static gboolean verify = FALSE;
+static gboolean quiet = FALSE;
+static gchar *suffix = NULL;
+static gchar *pubkey_file = NULL;
+static gchar *pubkey = NULL;
+static gchar *pubout = NULL;
+static gchar *keypair_file = NULL;
+static gchar *editor = NULL;
+static gboolean edit = FALSE;
+enum rspamd_cryptobox_mode mode = RSPAMD_CRYPTOBOX_MODE_25519;
+
+static void rspamadm_signtool(gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_signtool_help(gboolean full_help,
+ const struct rspamadm_command *cmd);
+
+struct rspamadm_command signtool_command = {
+ .name = "signtool",
+ .flags = 0,
+ .help = rspamadm_signtool_help,
+ .run = rspamadm_signtool,
+ .lua_subrs = NULL,
+};
+
+static GOptionEntry entries[] = {
+ {"openssl", 'o', 0, G_OPTION_ARG_NONE, &openssl,
+ "Generate openssl nistp256 keypair not curve25519 one", NULL},
+ {"verify", 'v', 0, G_OPTION_ARG_NONE, &verify,
+ "Verify signatures and not sign", NULL},
+ {"suffix", 'S', 0, G_OPTION_ARG_STRING, &suffix,
+ "Save signatures in file<suffix> files", NULL},
+ {"pubkey", 'p', 0, G_OPTION_ARG_STRING, &pubkey,
+ "Base32 encoded pubkey to verify", NULL},
+ {"pubout", '\0', 0, G_OPTION_ARG_FILENAME, &pubout,
+ "Output public key to the specified file", NULL},
+ {"pubfile", 'P', 0, G_OPTION_ARG_FILENAME, &pubkey_file,
+ "Load base32 encoded pubkey to verify from the file", NULL},
+ {"keypair", 'k', 0, G_OPTION_ARG_STRING, &keypair_file,
+ "UCL with keypair to load for signing", NULL},
+ {"quiet", 'q', 0, G_OPTION_ARG_NONE, &quiet,
+ "Be quiet", NULL},
+ {"edit", 'e', 0, G_OPTION_ARG_NONE, &edit,
+ "Run editor and sign the edited file", NULL},
+ {"editor", '\0', 0, G_OPTION_ARG_STRING, &editor,
+ "Use the specified editor instead of $EDITOR environment var", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+static const char *
+rspamadm_signtool_help(gboolean full_help,
+ const struct rspamadm_command *cmd)
+{
+ const char *help_str;
+
+ if (full_help) {
+ help_str = "Manage digital signatures\n\n"
+ "Usage: rspamadm signtool [-o] -k <keypair_file> [-v -p <pubkey> | -P <pubkey_file>] [-S <suffix>] file1 ...\n"
+ "Where options are:\n\n"
+ "-v: verify against pubkey instead of \n"
+ "-o: use ECDSA instead of EdDSA\n"
+ "-p: load pubkey as base32 string\n"
+ "-P: load pubkey paced in file\n"
+ "-k: load signing keypair from ucl file\n"
+ "-S: append suffix for signatures and store them in files\n"
+ "-q: be quiet\n"
+ "-e: opens file for editing and sign the result\n"
+ "--editor: use the specified editor instead of $EDITOR environment var\n"
+ "--help: shows available options and commands";
+ }
+ else {
+ help_str = "Sign and verify files tool";
+ }
+
+ return help_str;
+}
+
+static gint
+rspamadm_edit_file(const gchar *fname)
+{
+ gchar tmppath[PATH_MAX], run_cmdline[PATH_MAX];
+ guchar *map;
+ gsize len = 0;
+ gint fd_out, retcode, child_argc;
+ GPid child_pid;
+ gchar *tmpdir, **child_argv = NULL;
+ struct stat st;
+ GError *err = NULL;
+
+ if (editor == NULL) {
+ editor = getenv("EDITOR");
+ }
+
+ if (editor == NULL) {
+ rspamd_fprintf(stderr, "cannot find editor: specify $EDITOR "
+ "environment variable or pass --editor argument\n");
+ exit(EXIT_FAILURE);
+ }
+
+ tmpdir = getenv("TMPDIR");
+ if (tmpdir == NULL) {
+ tmpdir = "/tmp";
+ }
+
+ if (stat(fname, &st) == -1 || st.st_size == 0) {
+ /* The source does not exist, but that shouldn't be a problem */
+ len = 0;
+ map = NULL;
+
+ /* Try to touch source anyway */
+ fd_out = rspamd_file_xopen(fname, O_WRONLY | O_CREAT | O_EXCL, 00644,
+ 0);
+
+ if (fd_out == -1) {
+ rspamd_fprintf(stderr, "cannot open %s: %s\n", fname,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ close(fd_out);
+ }
+ else {
+ map = rspamd_file_xmap(fname, PROT_READ, &len, TRUE);
+
+ if (map == NULL) {
+ rspamd_fprintf(stderr, "cannot open %s: %s\n", fname,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ rspamd_snprintf(tmppath, sizeof(tmppath),
+ "%s/rspamd_sign-XXXXXXXXXX", tmpdir);
+ mode_t cur_umask = umask(S_IRWXO | S_IRWXG);
+ fd_out = mkstemp(tmppath);
+ (void) umask(cur_umask);
+
+ if (fd_out == -1) {
+ rspamd_fprintf(stderr, "cannot open tempfile %s: %s\n", tmppath,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ if (len > 0 && write(fd_out, map, len) == -1) {
+ rspamd_fprintf(stderr, "cannot write to tempfile %s: %s\n", tmppath,
+ strerror(errno));
+ unlink(tmppath);
+ munmap(map, len);
+ close(fd_out);
+ exit(EXIT_FAILURE);
+ }
+
+ if (len > 0) {
+ munmap(map, len);
+ }
+
+ fsync(fd_out);
+ close(fd_out);
+
+ /* Now we spawn editor with the filename as argument */
+ rspamd_snprintf(run_cmdline, sizeof(run_cmdline), "%s %s", editor, tmppath);
+ if (!g_shell_parse_argv(run_cmdline, &child_argc,
+ &child_argv, &err)) {
+ rspamd_fprintf(stderr, "cannot exec %s: %e\n", editor,
+ err);
+ unlink(tmppath);
+ exit(EXIT_FAILURE);
+ }
+
+ if (!g_spawn_async(NULL, child_argv, NULL,
+ G_SPAWN_CHILD_INHERITS_STDIN | G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD,
+ NULL, NULL, &child_pid, &err)) {
+ rspamd_fprintf(stderr, "cannot exec %s: %e\n", editor,
+ err);
+ unlink(tmppath);
+ exit(EXIT_FAILURE);
+ }
+
+ g_strfreev(child_argv);
+
+ for (;;) {
+ if (waitpid((pid_t) child_pid, &retcode, 0) != -1) {
+ break;
+ }
+
+ if (errno != EINTR) {
+ rspamd_fprintf(stderr, "failed to wait for %s: %s\n", editor,
+ strerror(errno));
+ unlink(tmppath);
+ exit(EXIT_FAILURE);
+ }
+ }
+
+#if GLIB_MAJOR_VERSION >= 2 && GLIB_MINOR_VERSION >= 34
+#if GLIB_MINOR_VERSION >= 70
+ if (!g_spawn_check_wait_status(retcode, &err)) {
+#else
+ if (!g_spawn_check_exit_status(retcode, &err)) {
+#endif
+ unlink(tmppath);
+ rspamd_fprintf(stderr, "%s returned error code: %d - %e\n", editor,
+ retcode, err);
+ exit(EXIT_FAILURE);
+ }
+#else
+ if (retcode != 0) {
+ unlink(tmppath);
+ rspamd_fprintf(stderr, "%s returned error code: %d\n", editor,
+ retcode);
+ exit(retcode);
+ }
+#endif
+
+ map = rspamd_file_xmap(tmppath, PROT_READ, &len, TRUE);
+
+ if (map == NULL) {
+ rspamd_fprintf(stderr, "cannot map %s: %s\n", tmppath,
+ strerror(errno));
+ unlink(tmppath);
+ exit(EXIT_FAILURE);
+ }
+
+ rspamd_snprintf(run_cmdline, sizeof(run_cmdline), "%s.new", fname);
+ fd_out = rspamd_file_xopen(run_cmdline, O_RDWR | O_CREAT | O_TRUNC, 00600,
+ 0);
+
+ if (fd_out == -1) {
+ rspamd_fprintf(stderr, "cannot open new file %s: %s\n", run_cmdline,
+ strerror(errno));
+ unlink(tmppath);
+ munmap(map, len);
+ exit(EXIT_FAILURE);
+ }
+
+ if (write(fd_out, map, len) == -1) {
+ rspamd_fprintf(stderr, "cannot write new file %s: %s\n", run_cmdline,
+ strerror(errno));
+ unlink(tmppath);
+ unlink(run_cmdline);
+ close(fd_out);
+ munmap(map, len);
+ exit(EXIT_FAILURE);
+ }
+
+ unlink(tmppath);
+ (void) lseek(fd_out, 0, SEEK_SET);
+ munmap(map, len);
+
+ return fd_out;
+}
+
+static bool
+rspamadm_sign_file(const gchar *fname, struct rspamd_cryptobox_keypair *kp)
+{
+ gint fd_sig, fd_input;
+ guchar sig[rspamd_cryptobox_MAX_SIGBYTES], *map;
+ gchar sigpath[PATH_MAX];
+ FILE *pub_fp;
+ struct stat st;
+ const guchar *sk;
+
+ if (suffix == NULL) {
+ suffix = ".sig";
+ }
+
+ if (edit) {
+ /* We need to open editor and then sign the temporary file */
+ fd_input = rspamadm_edit_file(fname);
+ }
+ else {
+ fd_input = rspamd_file_xopen(fname, O_RDONLY, 0, TRUE);
+ }
+
+ if (fd_input == -1) {
+ rspamd_fprintf(stderr, "cannot open %s: %s\n", fname,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ g_assert(fstat(fd_input, &st) != -1);
+
+ rspamd_snprintf(sigpath, sizeof(sigpath), "%s%s", fname, suffix);
+ fd_sig = rspamd_file_xopen(sigpath, O_WRONLY | O_CREAT | O_TRUNC, 00644, 0);
+
+ if (fd_sig == -1) {
+ close(fd_input);
+ rspamd_fprintf(stderr, "cannot open %s: %s\n", sigpath,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ map = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd_input, 0);
+ close(fd_input);
+
+ if (map == MAP_FAILED) {
+ close(fd_sig);
+ rspamd_fprintf(stderr, "cannot map %s: %s\n", fname,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ g_assert(rspamd_cryptobox_MAX_SIGBYTES >=
+ rspamd_cryptobox_signature_bytes(mode));
+
+ sk = rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_SK, NULL);
+ rspamd_cryptobox_sign(sig, NULL, map, st.st_size, sk, mode);
+
+ if (edit) {
+ /* We also need to rename .new file */
+ rspamd_snprintf(sigpath, sizeof(sigpath), "%s.new", fname);
+
+ if (rename(sigpath, fname) == -1) {
+ rspamd_fprintf(stderr, "cannot rename %s to %s: %s\n", sigpath, fname,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ unlink(sigpath);
+ }
+
+ rspamd_snprintf(sigpath, sizeof(sigpath), "%s%s", fname, suffix);
+
+ if (write(fd_sig, sig, rspamd_cryptobox_signature_bytes(mode)) == -1) {
+ rspamd_fprintf(stderr, "cannot write signature to %s: %s\n", sigpath,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ close(fd_sig);
+ munmap(map, st.st_size);
+
+ if (!quiet) {
+ rspamd_fprintf(stdout, "signed %s; stored hash in %s\n",
+ fname, sigpath);
+ }
+
+ if (pubout) {
+ GString *b32_pk;
+
+ pub_fp = fopen(pubout, "w");
+
+ if (pub_fp == NULL) {
+ rspamd_fprintf(stderr, "cannot write pubkey to %s: %s",
+ pubout, strerror(errno));
+ }
+ else {
+ b32_pk = rspamd_keypair_print(kp,
+ RSPAMD_KEYPAIR_PUBKEY | RSPAMD_KEYPAIR_BASE32);
+
+ if (b32_pk) {
+ rspamd_fprintf(pub_fp, "%v", b32_pk);
+ g_string_free(b32_pk, TRUE);
+ }
+
+ fclose(pub_fp);
+ }
+ if (!quiet) {
+ rspamd_fprintf(stdout, "stored pubkey in %s\n",
+ pubout);
+ }
+ }
+
+ return true;
+}
+
+static bool
+rspamadm_verify_file(const gchar *fname, const guchar *pk)
+{
+ gint fd_sig, fd_input;
+ guchar *map, *map_sig;
+ gchar sigpath[PATH_MAX];
+ struct stat st, st_sig;
+ bool ret;
+
+ g_assert(rspamd_cryptobox_MAX_SIGBYTES >=
+ rspamd_cryptobox_signature_bytes(mode));
+
+ if (suffix == NULL) {
+ suffix = ".sig";
+ }
+
+ fd_input = rspamd_file_xopen(fname, O_RDONLY, 0, TRUE);
+
+ if (fd_input == -1) {
+ rspamd_fprintf(stderr, "cannot open %s: %s\n", fname,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ g_assert(fstat(fd_input, &st) != -1);
+
+ rspamd_snprintf(sigpath, sizeof(sigpath), "%s%s", fname, suffix);
+ fd_sig = rspamd_file_xopen(sigpath, O_RDONLY, 0, TRUE);
+
+ if (fd_sig == -1) {
+ close(fd_input);
+ rspamd_fprintf(stderr, "cannot open %s: %s\n", sigpath,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ map = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd_input, 0);
+ close(fd_input);
+
+ if (map == MAP_FAILED) {
+ close(fd_sig);
+ rspamd_fprintf(stderr, "cannot open %s: %s\n", sigpath,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ g_assert(fstat(fd_sig, &st_sig) != -1);
+
+ if (st_sig.st_size != rspamd_cryptobox_signature_bytes(mode)) {
+ close(fd_sig);
+ rspamd_fprintf(stderr, "invalid signature size %s: %ud\n", fname,
+ (guint) st_sig.st_size);
+ munmap(map, st.st_size);
+ exit(EXIT_FAILURE);
+ }
+
+ map_sig = mmap(NULL, st_sig.st_size, PROT_READ, MAP_SHARED, fd_sig, 0);
+ close(fd_sig);
+
+ if (map_sig == MAP_FAILED) {
+ munmap(map, st.st_size);
+ rspamd_fprintf(stderr, "cannot map %s: %s\n", sigpath,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ ret = rspamd_cryptobox_verify(map_sig, st_sig.st_size,
+ map, st.st_size, pk, mode);
+ munmap(map, st.st_size);
+ munmap(map_sig, st_sig.st_size);
+
+ if (!ret) {
+ rspamd_fprintf(stderr, "cannot verify %s using %s: invalid signature\n",
+ fname, sigpath);
+ }
+ else if (!quiet) {
+ rspamd_fprintf(stdout, "verified %s using %s\n",
+ fname, sigpath);
+ }
+
+ return ret;
+}
+
+
+static void
+rspamadm_signtool(gint argc, gchar **argv, const struct rspamadm_command *cmd)
+{
+ GOptionContext *context;
+ GError *error = NULL;
+ struct ucl_parser *parser;
+ ucl_object_t *top;
+ struct rspamd_cryptobox_pubkey *pk;
+ struct rspamd_cryptobox_keypair *kp;
+ gsize fsize, flen;
+ gint i;
+
+ context = g_option_context_new(
+ "keypair - create encryption keys");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd administration utility version " RVERSION
+ "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, NULL);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ rspamd_fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ g_option_context_free(context);
+
+ if (openssl) {
+ mode = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+
+ if (verify && (!pubkey && !pubkey_file)) {
+ rspamd_fprintf(stderr, "no pubkey for verification\n");
+ exit(EXIT_FAILURE);
+ }
+ else if (!verify && (!keypair_file)) {
+ rspamd_fprintf(stderr, "no keypair for signing\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (verify) {
+ g_assert(pubkey || pubkey_file);
+
+ if (pubkey_file) {
+ gint fd;
+ gchar *map;
+ struct stat st;
+
+ fd = open(pubkey_file, O_RDONLY);
+
+ if (fd == -1) {
+ rspamd_fprintf(stderr, "cannot open %s: %s\n", pubkey_file,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ g_assert(fstat(fd, &st) != -1);
+ fsize = st.st_size;
+ flen = fsize;
+ map = mmap(NULL, fsize, PROT_READ, MAP_SHARED, fd, 0);
+ close(fd);
+
+ if (map == MAP_FAILED) {
+ rspamd_fprintf(stderr, "cannot read %s: %s\n", pubkey_file,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ /* XXX: assume base32 pubkey now */
+ while (flen > 0 && g_ascii_isspace(map[flen - 1])) {
+ flen--;
+ }
+
+ pk = rspamd_pubkey_from_base32(map, flen,
+ RSPAMD_KEYPAIR_SIGN, mode);
+
+ if (pk == NULL) {
+ rspamd_fprintf(stderr, "bad size %s: %ud, %ud expected\n",
+ pubkey_file,
+ (guint) flen,
+ rspamd_cryptobox_pk_sig_bytes(mode));
+ exit(EXIT_FAILURE);
+ }
+
+ munmap(map, fsize);
+ }
+ else {
+ pk = rspamd_pubkey_from_base32(pubkey, strlen(pubkey),
+ RSPAMD_KEYPAIR_SIGN, mode);
+
+ if (pk == NULL) {
+ rspamd_fprintf(stderr, "bad size %s: %ud, %ud expected\n",
+ pubkey_file,
+ (guint) strlen(pubkey),
+ rspamd_cryptobox_pk_sig_bytes(mode));
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ for (i = 1; i < argc; i++) {
+ /* XXX: support cmd line signature */
+ if (!rspamadm_verify_file(argv[i], rspamd_pubkey_get_pk(pk, NULL))) {
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ g_free(pk);
+ }
+ else {
+ g_assert(keypair_file != NULL);
+
+ parser = ucl_parser_new(0);
+
+ if (!ucl_parser_add_file(parser, keypair_file) ||
+ (top = ucl_parser_get_object(parser)) == NULL) {
+ rspamd_fprintf(stderr, "cannot load keypair: %s\n",
+ ucl_parser_get_error(parser));
+ exit(EXIT_FAILURE);
+ }
+
+ ucl_parser_free(parser);
+
+ kp = rspamd_keypair_from_ucl(top);
+
+ if (kp == NULL) {
+ rspamd_fprintf(stderr, "invalid signing key\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (rspamd_keypair_type(kp) != RSPAMD_KEYPAIR_SIGN) {
+ rspamd_fprintf(stderr, "unsuitable for signing key\n");
+ exit(EXIT_FAILURE);
+ }
+
+ for (i = 1; i < argc; i++) {
+ /* XXX: support cmd line signature */
+ if (!rspamadm_sign_file(argv[i], kp)) {
+ rspamd_keypair_unref(kp);
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ rspamd_keypair_unref(kp);
+ }
+}
diff --git a/src/rspamadm/stat_convert.c b/src/rspamadm/stat_convert.c
new file mode 100644
index 0000000..0741279
--- /dev/null
+++ b/src/rspamadm/stat_convert.c
@@ -0,0 +1,262 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamadm.h"
+#include "lua/lua_common.h"
+
+#include "contrib/uthash/utlist.h"
+
+/* Common */
+static gchar *config_file = NULL;
+static gchar *symbol_ham = NULL;
+static gchar *symbol_spam = NULL;
+
+static gdouble expire = 0.0;
+
+/* Inputs */
+static gchar *spam_db = NULL;
+static gchar *ham_db = NULL;
+static gchar *cache_db = NULL;
+
+/* Outputs */
+static gchar *redis_host = NULL;
+static gchar *redis_db = NULL;
+static gchar *redis_username = NULL;
+static gchar *redis_password = NULL;
+static gboolean reset_previous = FALSE;
+
+static void rspamadm_statconvert(gint argc, gchar **argv,
+ const struct rspamadm_command *cmd);
+static const char *rspamadm_statconvert_help(gboolean full_help,
+ const struct rspamadm_command *cmd);
+
+struct rspamadm_command statconvert_command = {
+ .name = "statconvert",
+ .flags = 0,
+ .help = rspamadm_statconvert_help,
+ .run = rspamadm_statconvert,
+ .lua_subrs = NULL,
+};
+
+static GOptionEntry entries[] = {
+ {"config", 'c', 0, G_OPTION_ARG_FILENAME, &config_file,
+ "Config file to read data from", NULL},
+ {"reset", 'r', 0, G_OPTION_ARG_NONE, &reset_previous,
+ "Reset previous data instead of appending values", NULL},
+ {"expire", 'e', 0, G_OPTION_ARG_DOUBLE, &expire,
+ "Set expiration in seconds (can be fractional)", NULL},
+
+ {"symbol-spam", 0, 0, G_OPTION_ARG_STRING, &symbol_spam,
+ "Symbol for spam (e.g. BAYES_SPAM)", NULL},
+ {"symbol-ham", 0, 0, G_OPTION_ARG_STRING, &symbol_ham,
+ "Symbol for ham (e.g. BAYES_HAM)", NULL},
+ {"spam-db", 0, 0, G_OPTION_ARG_STRING, &spam_db,
+ "Input spam file (sqlite3)", NULL},
+ {"ham-db", 0, 0, G_OPTION_ARG_STRING, &ham_db,
+ "Input ham file (sqlite3)", NULL},
+ {"cache", 0, 0, G_OPTION_ARG_FILENAME, &cache_db,
+ "Input learn cache", NULL},
+ {"redis-host", 'h', 0, G_OPTION_ARG_STRING, &redis_host,
+ "Output redis ip (in format ip:port)", NULL},
+ {"redis-username", 'u', 0, G_OPTION_ARG_STRING, &redis_username,
+ "Username to connect to redis", NULL},
+ {"redis-password", 'p', 0, G_OPTION_ARG_STRING, &redis_password,
+ "Password to connect to redis", NULL},
+ {"redis-db", 'd', 0, G_OPTION_ARG_STRING, &redis_db,
+ "Redis database (should be numeric)", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+
+static const char *
+rspamadm_statconvert_help(gboolean full_help, const struct rspamadm_command *cmd)
+{
+ const char *help_str;
+
+ if (full_help) {
+ help_str = "Convert statistics from sqlite3 to redis\n\n"
+ "Usage: rspamadm statconvert -c /etc/rspamd.conf [-r]\n"
+ "Where options are:\n\n"
+ "-c: config file to read data from\n"
+ "-r: reset previous data instead of increasing values\n"
+ "-e: set expire to that amount of seconds\n"
+ "** Or specify options directly **\n"
+ "--redis-host: output redis ip (in format ip:port)\n"
+ "--redis-db: output redis database\n"
+ "--redis-username: redis username\n"
+ "--redis-password: redis password\n"
+ "--cache: sqlite3 file for learn cache\n"
+ "--spam-db: sqlite3 input file for spam data\n"
+ "--ham-db: sqlite3 input file for ham data\n"
+ "--symbol-spam: symbol in redis for spam (e.g. BAYES_SPAM)\n"
+ "--symbol-ham: symbol in redis for ham (e.g. BAYES_HAM)\n";
+ }
+ else {
+ help_str = "Convert statistics from sqlite3 to redis";
+ }
+
+ return help_str;
+}
+
+static void
+rspamadm_statconvert(gint argc, gchar **argv, const struct rspamadm_command *cmd)
+{
+ GOptionContext *context;
+ GError *error = NULL;
+ ucl_object_t *obj;
+
+ context = g_option_context_new(
+ "statconvert - converts statistics from sqlite3 to redis");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd administration utility version " RVERSION
+ "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, NULL);
+ g_option_context_set_ignore_unknown_options(context, TRUE);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ rspamd_fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ g_option_context_free(context);
+
+ if (config_file) {
+ /* Load config file, assuming that it has all information required */
+ struct ucl_parser *parser;
+
+ parser = ucl_parser_new(0);
+ rspamd_ucl_add_conf_variables(parser, ucl_vars);
+
+ if (!ucl_parser_add_file(parser, config_file)) {
+ msg_err("ucl parser error: %s", ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+
+ exit(EXIT_FAILURE);
+ }
+
+ obj = ucl_parser_get_object(parser);
+ ucl_parser_free(parser);
+ }
+ else {
+ /* We need to get all information from the command line */
+ ucl_object_t *classifier, *statfile_ham, *statfile_spam, *tmp, *redis;
+
+ /* Check arguments sanity */
+ if (spam_db == NULL) {
+ msg_err("No spam-db specified");
+ exit(EXIT_FAILURE);
+ }
+ if (ham_db == NULL) {
+ msg_err("No ham-db specified");
+ exit(EXIT_FAILURE);
+ }
+ if (redis_host == NULL) {
+ msg_err("No redis-host specified");
+ exit(EXIT_FAILURE);
+ }
+ if (symbol_ham == NULL) {
+ msg_err("No symbol-ham specified");
+ exit(EXIT_FAILURE);
+ }
+ if (symbol_spam == NULL) {
+ msg_err("No symbol-spam specified");
+ exit(EXIT_FAILURE);
+ }
+
+ obj = ucl_object_typed_new(UCL_OBJECT);
+
+ classifier = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj, classifier, "classifier", 0, false);
+ /* Now we need to create "bayes" key in it */
+ tmp = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(classifier, tmp, "bayes", 0, false);
+ classifier = tmp;
+ ucl_object_insert_key(classifier, ucl_object_fromstring("sqlite3"),
+ "backend", 0, false);
+
+ if (cache_db != NULL) {
+ ucl_object_t *cache;
+
+ cache = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(cache, ucl_object_fromstring("sqlite3"),
+ "type", 0, false);
+ ucl_object_insert_key(cache, ucl_object_fromstring(cache_db),
+ "file", 0, false);
+
+ ucl_object_insert_key(classifier, cache, "cache", 0, false);
+ }
+
+ statfile_ham = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(statfile_ham, ucl_object_fromstring(symbol_ham),
+ "symbol", 0, false);
+ ucl_object_insert_key(statfile_ham, ucl_object_frombool(false),
+ "spam", 0, false);
+ ucl_object_insert_key(statfile_ham, ucl_object_fromstring(ham_db),
+ "db", 0, false);
+
+ statfile_spam = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(statfile_spam, ucl_object_fromstring(symbol_spam),
+ "symbol", 0, false);
+ ucl_object_insert_key(statfile_spam, ucl_object_frombool(true),
+ "spam", 0, false);
+ ucl_object_insert_key(statfile_spam, ucl_object_fromstring(spam_db),
+ "db", 0, false);
+
+ DL_APPEND(statfile_ham, statfile_spam);
+ ucl_object_insert_key(classifier, statfile_ham,
+ "statfile", 0, false);
+
+ /* Deal with redis */
+
+ redis = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj, redis, "redis", 0, false);
+
+ ucl_object_insert_key(redis, ucl_object_fromstring(redis_host),
+ "servers", 0, false);
+
+ if (redis_db) {
+ ucl_object_insert_key(redis, ucl_object_fromstring(redis_db),
+ "dbname", 0, false);
+ }
+
+ if (redis_username) {
+ ucl_object_insert_key(redis, ucl_object_fromstring(redis_username),
+ "username", 0, false);
+ }
+
+ if (redis_password) {
+ ucl_object_insert_key(redis, ucl_object_fromstring(redis_password),
+ "password", 0, false);
+ }
+ }
+
+ ucl_object_insert_key(obj, ucl_object_frombool(reset_previous),
+ "reset_previous", 0, false);
+
+ if (expire != 0) {
+ ucl_object_insert_key(obj, ucl_object_fromdouble(expire),
+ "expire", 0, false);
+ }
+
+ rspamadm_execute_lua_ucl_subr(argc,
+ argv,
+ obj,
+ "stat_convert",
+ TRUE);
+
+ ucl_object_unref(obj);
+}
diff --git a/src/rspamd.c b/src/rspamd.c
new file mode 100644
index 0000000..b8523b4
--- /dev/null
+++ b/src/rspamd.c
@@ -0,0 +1,1729 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "blas-config.h"
+#include "rspamd.h"
+#include "libserver/maps/map.h"
+#include "lua/lua_common.h"
+#include "libserver/worker_util.h"
+#include "libserver/rspamd_control.h"
+#include "ottery.h"
+#include "cryptobox.h"
+#include "utlist.h"
+#include "unix-std.h"
+/* pwd and grp */
+#ifdef HAVE_PWD_H
+#include <pwd.h>
+#endif
+#ifdef HAVE_GRP_H
+#include <grp.h>
+#endif
+
+#ifdef HAVE_NFTW
+#include <ftw.h>
+#endif
+
+#include <signal.h>
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
+#ifdef HAVE_LIBUTIL_H
+#include <libutil.h>
+#endif
+
+#ifdef HAVE_OPENSSL
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <math.h>
+
+#endif
+
+#include "sqlite3.h"
+#include "contrib/libev/ev.h"
+
+#ifdef WITH_HYPERSCAN
+#include "libserver/hyperscan_tools.h"
+#endif
+
+/* 2 seconds to fork new process in place of dead one */
+#define SOFT_FORK_TIME 2
+
+/* 10 seconds after getting termination signal to terminate all workers with SIGKILL */
+#define TERMINATION_INTERVAL (0.2)
+
+static gboolean load_rspamd_config(struct rspamd_main *rspamd_main,
+ struct rspamd_config *cfg,
+ gboolean init_modules,
+ enum rspamd_post_load_options opts,
+ gboolean reload);
+static void rspamd_cld_handler(EV_P_ ev_child *w,
+ struct rspamd_main *rspamd_main,
+ struct rspamd_worker *wrk);
+
+/* Control socket */
+static gint control_fd;
+static ev_io control_ev;
+static struct rspamd_stat old_stat;
+static ev_timer stat_ev;
+
+static gboolean valgrind_mode = FALSE;
+
+/* Cmdline options */
+static gboolean no_fork = FALSE;
+static gboolean show_version = FALSE;
+static gchar **cfg_names = NULL;
+static gchar *rspamd_user = NULL;
+static gchar *rspamd_group = NULL;
+static gchar *rspamd_pidfile = NULL;
+static gboolean is_debug = FALSE;
+static gboolean is_insecure = FALSE;
+static GHashTable *ucl_vars = NULL;
+static gchar **lua_env = NULL;
+static gboolean skip_template = FALSE;
+
+static gint term_attempts = 0;
+
+/* List of active listen sockets indexed by worker type */
+static GHashTable *listen_sockets = NULL;
+
+/* Defined in modules.c */
+extern module_t *modules[];
+extern worker_t *workers[];
+
+/* Command line options */
+static gboolean rspamd_parse_var(const gchar *option_name,
+ const gchar *value, gpointer data,
+ GError **error);
+static GOptionEntry entries[] =
+ {
+ {"no-fork", 'f', 0, G_OPTION_ARG_NONE, &no_fork,
+ "Do not daemonize main process", NULL},
+ {"config", 'c', 0, G_OPTION_ARG_FILENAME_ARRAY, &cfg_names,
+ "Specify config file(s)", NULL},
+ {"user", 'u', 0, G_OPTION_ARG_STRING, &rspamd_user,
+ "User to run rspamd as", NULL},
+ {"group", 'g', 0, G_OPTION_ARG_STRING, &rspamd_group,
+ "Group to run rspamd as", NULL},
+ {"pid", 'p', 0, G_OPTION_ARG_STRING, &rspamd_pidfile, "Path to pidfile",
+ NULL},
+ {"debug", 'd', 0, G_OPTION_ARG_NONE, &is_debug, "Force debug output",
+ NULL},
+ {"insecure", 'i', 0, G_OPTION_ARG_NONE, &is_insecure,
+ "Ignore running workers as privileged users (insecure)", NULL},
+ {"version", 'v', 0, G_OPTION_ARG_NONE, &show_version,
+ "Show version and exit", NULL},
+ {"var", 0, 0, G_OPTION_ARG_CALLBACK, (gpointer) &rspamd_parse_var,
+ "Redefine/define environment variable", NULL},
+ {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template,
+ "Do not apply Jinja templates", NULL},
+ {"lua-env", '\0', 0, G_OPTION_ARG_FILENAME_ARRAY, &lua_env,
+ "Load lua environment from the specified files", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+static gboolean
+rspamd_parse_var(const gchar *option_name,
+ const gchar *value, gpointer data,
+ GError **error)
+{
+ gchar *k, *v, *t;
+
+ t = strchr(value, '=');
+
+ if (t != NULL) {
+ k = g_strdup(value);
+ t = k + (t - value);
+ v = g_strdup(t + 1);
+ *t = '\0';
+
+ if (ucl_vars == NULL) {
+ ucl_vars = g_hash_table_new_full(rspamd_strcase_hash,
+ rspamd_strcase_equal, g_free, g_free);
+ }
+
+ g_hash_table_insert(ucl_vars, k, v);
+ }
+ else {
+ g_set_error(error, g_quark_try_string("main"), EINVAL,
+ "Bad variable format: %s", value);
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static void
+read_cmd_line(gint *argc, gchar ***argv, struct rspamd_config *cfg)
+{
+ GError *error = NULL;
+ GOptionContext *context;
+ guint cfg_num;
+
+ context = g_option_context_new("- run rspamd daemon");
+#if defined(GIT_VERSION) && GIT_VERSION == 1
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd daemon version " RVERSION "-git\n Git id: " RID);
+#else
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd daemon version " RVERSION);
+#endif
+ g_option_context_add_main_entries(context, entries, NULL);
+
+ if (!g_option_context_parse(context, argc, argv, &error)) {
+ fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ cfg->rspamd_user = rspamd_user;
+ cfg->rspamd_group = rspamd_group;
+ cfg_num = cfg_names != NULL ? g_strv_length(cfg_names) : 0;
+
+ if (cfg_num == 0) {
+ cfg->cfg_name = FIXED_CONFIG_FILE;
+ }
+ else {
+ cfg->cfg_name = cfg_names[0];
+ g_assert(cfg_num == 1);
+ }
+
+ cfg->pid_file = rspamd_pidfile;
+ g_option_context_free(context);
+}
+
+static int
+rspamd_write_pid(struct rspamd_main *main)
+{
+ pid_t pid;
+
+ if (main->cfg->pid_file == NULL) {
+ return -1;
+ }
+ main->pfh = rspamd_pidfile_open(main->cfg->pid_file, 0644, &pid);
+
+ if (main->pfh == NULL) {
+ return -1;
+ }
+
+ if (main->is_privileged) {
+ /* Force root user as owner of pid file */
+#ifdef HAVE_PIDFILE_FILENO
+ if (fchown(pidfile_fileno(main->pfh), 0, 0) == -1) {
+#else
+ if (fchown(main->pfh->pf_fd, 0, 0) == -1) {
+#endif
+ }
+ }
+
+ rspamd_pidfile_write(main->pfh);
+
+ return 0;
+}
+
+/* Detect privileged mode */
+static void
+detect_priv(struct rspamd_main *rspamd_main)
+{
+ struct passwd *pwd;
+ struct group *grp;
+ uid_t euid;
+
+ euid = geteuid();
+
+ if (euid == 0) {
+ if (!rspamd_main->cfg->rspamd_user && !is_insecure) {
+ msg_err_main(
+ "cannot run rspamd workers as root user, please add -u and -g options to select a proper unprivileged user or specify --insecure flag");
+ exit(EXIT_FAILURE);
+ }
+ else if (is_insecure) {
+ rspamd_main->is_privileged = TRUE;
+ rspamd_main->workers_uid = 0;
+ rspamd_main->workers_gid = 0;
+ }
+ else {
+ rspamd_main->is_privileged = TRUE;
+ pwd = getpwnam(rspamd_main->cfg->rspamd_user);
+ if (pwd == NULL) {
+ msg_err_main("user specified does not exists (%s), aborting",
+ strerror(errno));
+ exit(-errno);
+ }
+ if (rspamd_main->cfg->rspamd_group) {
+ grp = getgrnam(rspamd_main->cfg->rspamd_group);
+ if (grp == NULL) {
+ msg_err_main("group specified does not exists (%s), aborting",
+ strerror(errno));
+ exit(-errno);
+ }
+ rspamd_main->workers_gid = grp->gr_gid;
+ }
+ else {
+ /* Use the main group of user */
+ rspamd_main->workers_gid = pwd->pw_gid;
+ }
+ rspamd_main->workers_uid = pwd->pw_uid;
+ }
+ }
+ else {
+ rspamd_main->is_privileged = FALSE;
+ rspamd_main->workers_uid = (uid_t) -1;
+ rspamd_main->workers_gid = (gid_t) -1;
+ }
+}
+
+static void
+config_logger(rspamd_mempool_t *pool, gpointer ud)
+{
+ struct rspamd_main *rspamd_main = ud;
+
+ rspamd_main->logger = rspamd_log_open_specific(rspamd_main->server_pool,
+ rspamd_main->cfg,
+ "main",
+ rspamd_main->workers_uid,
+ rspamd_main->workers_gid);
+
+ if (rspamd_main->logger == NULL) {
+ /*
+ * XXX:
+ * Error has been already logged (in fact,
+ * we might fall back to console logger here)
+ */
+ exit(EXIT_FAILURE);
+ }
+
+ rspamd_logger_configure_modules(rspamd_main->cfg->debug_modules);
+}
+
+static gboolean
+reread_config(struct rspamd_main *rspamd_main)
+{
+ struct rspamd_config *tmp_cfg, *old_cfg;
+ gchar *cfg_file;
+ int load_opts = RSPAMD_CONFIG_INIT_VALIDATE | RSPAMD_CONFIG_INIT_SYMCACHE |
+ RSPAMD_CONFIG_INIT_LIBS | RSPAMD_CONFIG_INIT_URL;
+
+ rspamd_symcache_save(rspamd_main->cfg->cache);
+ tmp_cfg = rspamd_config_new(RSPAMD_CONFIG_INIT_DEFAULT);
+ tmp_cfg->libs_ctx = rspamd_main->cfg->libs_ctx;
+ REF_RETAIN(tmp_cfg->libs_ctx);
+ cfg_file = rspamd_mempool_strdup(tmp_cfg->cfg_pool,
+ rspamd_main->cfg->cfg_name);
+ /* Save some variables */
+ tmp_cfg->cfg_name = cfg_file;
+ old_cfg = rspamd_main->cfg;
+ rspamd_main->cfg = tmp_cfg;
+ rspamd_logger_t *old_logger = rspamd_main->logger;
+
+ if (!load_rspamd_config(rspamd_main, tmp_cfg, TRUE, load_opts, TRUE)) {
+ rspamd_main->cfg = old_cfg;
+ rspamd_main->logger = old_logger;
+ msg_err_main("cannot parse new config file, revert to old one");
+ REF_RELEASE(tmp_cfg);
+
+ return FALSE;
+ }
+ else {
+ rspamd_log_close(old_logger);
+ msg_info_main("replacing config");
+ REF_RELEASE(old_cfg);
+ rspamd_main->cfg->rspamd_user = rspamd_user;
+ rspamd_main->cfg->rspamd_group = rspamd_group;
+ /* Here, we can do post actions with the existing config */
+ /*
+ * As some rules are defined in lua, we need to process them, then init
+ * modules and merely afterwards to init modules
+ */
+ rspamd_lua_post_load_config(tmp_cfg);
+ rspamd_init_filters(tmp_cfg, true, false);
+
+ /* Do post-load actions */
+ rspamd_config_post_load(tmp_cfg,
+ load_opts | RSPAMD_CONFIG_INIT_POST_LOAD_LUA | RSPAMD_CONFIG_INIT_PRELOAD_MAPS);
+ msg_info_main("config has been reread successfully");
+ }
+
+ return TRUE;
+}
+
+struct waiting_worker {
+ struct rspamd_main *rspamd_main;
+ struct ev_timer wait_ev;
+ struct rspamd_worker_conf *cf;
+ guint oldindex;
+};
+
+static void
+rspamd_fork_delayed_cb(EV_P_ ev_timer *w, int revents)
+{
+ struct waiting_worker *waiting_worker = (struct waiting_worker *) w->data;
+
+ ev_timer_stop(EV_A_ & waiting_worker->wait_ev);
+ rspamd_fork_worker(waiting_worker->rspamd_main, waiting_worker->cf,
+ waiting_worker->oldindex,
+ waiting_worker->rspamd_main->event_loop,
+ rspamd_cld_handler, listen_sockets);
+ REF_RELEASE(waiting_worker->cf);
+ g_free(waiting_worker);
+}
+
+static void
+rspamd_fork_delayed(struct rspamd_worker_conf *cf,
+ guint index,
+ struct rspamd_main *rspamd_main)
+{
+ struct waiting_worker *nw;
+
+ nw = g_malloc0(sizeof(*nw));
+ nw->cf = cf;
+ nw->oldindex = index;
+ nw->rspamd_main = rspamd_main;
+ REF_RETAIN(cf);
+ nw->wait_ev.data = nw;
+ ev_timer_init(&nw->wait_ev, rspamd_fork_delayed_cb, SOFT_FORK_TIME, 0.0);
+ ev_timer_start(rspamd_main->event_loop, &nw->wait_ev);
+}
+
+static GList *
+create_listen_socket(GPtrArray *addrs, guint cnt,
+ enum rspamd_worker_socket_type listen_type)
+{
+ GList *result = NULL;
+ gint fd;
+ guint i;
+ static const int listen_opts = RSPAMD_INET_ADDRESS_LISTEN_ASYNC;
+ struct rspamd_worker_listen_socket *ls;
+
+ g_ptr_array_sort(addrs, rspamd_inet_address_compare_ptr);
+ for (i = 0; i < cnt; i++) {
+
+ /*
+ * Copy address to avoid reload issues
+ */
+ if (listen_type & RSPAMD_WORKER_SOCKET_TCP) {
+ fd = rspamd_inet_address_listen(g_ptr_array_index(addrs, i),
+ SOCK_STREAM,
+ listen_opts, -1);
+ if (fd != -1) {
+ ls = g_malloc0(sizeof(*ls));
+ ls->addr = rspamd_inet_address_copy(g_ptr_array_index(addrs, i), NULL);
+ ls->fd = fd;
+ ls->type = RSPAMD_WORKER_SOCKET_TCP;
+ result = g_list_prepend(result, ls);
+ }
+ }
+ if (listen_type & RSPAMD_WORKER_SOCKET_UDP) {
+ fd = rspamd_inet_address_listen(g_ptr_array_index(addrs, i),
+ SOCK_DGRAM,
+ listen_opts | RSPAMD_INET_ADDRESS_LISTEN_REUSEPORT, -1);
+ if (fd != -1) {
+ ls = g_malloc0(sizeof(*ls));
+ ls->addr = rspamd_inet_address_copy(g_ptr_array_index(addrs, i), NULL);
+ ls->fd = fd;
+ ls->type = RSPAMD_WORKER_SOCKET_UDP;
+ result = g_list_prepend(result, ls);
+ }
+ }
+ }
+
+ return result;
+}
+
+static GList *
+systemd_get_socket(struct rspamd_main *rspamd_main, const gchar *fdname)
+{
+ int number, sock, num_passed, flags;
+ GList *result = NULL;
+ const gchar *e;
+ gchar **fdnames;
+ gchar *end;
+ struct stat st;
+ static const int sd_listen_fds_start = 3; /* SD_LISTEN_FDS_START */
+ struct rspamd_worker_listen_socket *ls;
+
+ union {
+ struct sockaddr_storage ss;
+ struct sockaddr sa;
+ struct sockaddr_un sun;
+ struct sockaddr_in6 s6;
+ } addr_storage;
+ socklen_t slen = sizeof(addr_storage);
+ gint stype;
+
+ number = strtoul(fdname, &end, 10);
+ if (end != NULL && *end != '\0') {
+ /* Cannot parse as number, assume a name in LISTEN_FDNAMES. */
+ e = getenv("LISTEN_FDNAMES");
+ if (!e) {
+ msg_err_main("cannot get systemd variable 'LISTEN_FDNAMES'");
+ errno = ENOENT;
+ return NULL;
+ }
+
+ fdnames = g_strsplit(e, ":", -1);
+ for (number = 0; fdnames[number]; number++) {
+ if (!strcmp(fdnames[number], fdname)) {
+ break;
+ }
+ }
+ if (!fdnames[number]) {
+ number = -1;
+ }
+ g_strfreev(fdnames);
+ }
+
+ if (number < 0) {
+ msg_warn_main("cannot find systemd socket: %s", fdname);
+ errno = ENOENT;
+ return NULL;
+ }
+
+ e = getenv("LISTEN_FDS");
+ if (e != NULL) {
+ errno = 0;
+ num_passed = strtoul(e, &end, 10);
+ if ((end == NULL || *end == '\0') && num_passed > number) {
+ sock = number + sd_listen_fds_start;
+ if (fstat(sock, &st) == -1) {
+ msg_warn_main("cannot stat systemd descriptor %d", sock);
+ return NULL;
+ }
+ if (!S_ISSOCK(st.st_mode)) {
+ msg_warn_main("systemd descriptor %d is not a socket", sock);
+ errno = EINVAL;
+ return NULL;
+ }
+ flags = fcntl(sock, F_GETFD);
+
+ if (flags != -1) {
+ (void) fcntl(sock, F_SETFD, flags | FD_CLOEXEC);
+ }
+
+ rspamd_socket_nonblocking(sock);
+
+ if (getsockname(sock, &addr_storage.sa, &slen) == -1) {
+ msg_warn_main("cannot get name for systemd descriptor %d: %s",
+ sock, strerror(errno));
+ errno = EINVAL;
+ return NULL;
+ }
+
+ ls = g_malloc0(sizeof(*ls));
+ ls->addr = rspamd_inet_address_from_sa(&addr_storage.sa, slen);
+ ls->fd = sock;
+ ls->is_systemd = true;
+
+ slen = sizeof(stype);
+ if (getsockopt(sock, SOL_SOCKET, SO_TYPE, &stype, &slen) != -1) {
+ if (stype == SOCK_STREAM) {
+ ls->type = RSPAMD_WORKER_SOCKET_TCP;
+ }
+ else {
+ ls->type = RSPAMD_WORKER_SOCKET_UDP;
+ }
+ }
+ else {
+ msg_warn_main("cannot get type for systemd descriptor %d: %s",
+ sock, strerror(errno));
+ ls->type = RSPAMD_WORKER_SOCKET_TCP;
+ }
+
+
+ result = g_list_prepend(result, ls);
+ }
+ else if (num_passed <= number) {
+ msg_err_main("systemd LISTEN_FDS does not contain the expected fd: %d",
+ num_passed);
+ errno = EINVAL;
+ }
+ }
+ else {
+ msg_err_main("cannot get systemd variable 'LISTEN_FDS'");
+ errno = ENOENT;
+ }
+
+ return result;
+}
+
+static void
+pass_signal_cb(gpointer key, gpointer value, gpointer ud)
+{
+ struct rspamd_worker *cur = value;
+ gint signo = GPOINTER_TO_INT(ud);
+
+ kill(cur->pid, signo);
+}
+
+static void
+rspamd_pass_signal(GHashTable *workers, gint signo)
+{
+ g_hash_table_foreach(workers, pass_signal_cb, GINT_TO_POINTER(signo));
+}
+
+static inline uintptr_t
+make_listen_key(struct rspamd_worker_bind_conf *cf)
+{
+ rspamd_cryptobox_fast_hash_state_t st;
+ guint i, keylen = 0;
+ guint8 *key;
+ rspamd_inet_addr_t *addr;
+ guint16 port;
+
+ rspamd_cryptobox_fast_hash_init(&st, rspamd_hash_seed());
+ if (cf->is_systemd) {
+ /* Something like 'systemd:0' or 'systemd:controller'. */
+ rspamd_cryptobox_fast_hash_update(&st, cf->name, strlen(cf->name));
+ }
+ else {
+ rspamd_cryptobox_fast_hash_update(&st, cf->name, strlen(cf->name));
+ for (i = 0; i < cf->cnt; i++) {
+ addr = g_ptr_array_index(cf->addrs, i);
+ key = rspamd_inet_address_get_hash_key(
+ addr, &keylen);
+ rspamd_cryptobox_fast_hash_update(&st, key, keylen);
+ port = rspamd_inet_address_get_port(addr);
+ rspamd_cryptobox_fast_hash_update(&st, &port, sizeof(port));
+ }
+ }
+
+ return rspamd_cryptobox_fast_hash_final(&st);
+}
+
+static void
+spawn_worker_type(struct rspamd_main *rspamd_main, struct ev_loop *event_loop,
+ struct rspamd_worker_conf *cf)
+{
+ gint i;
+
+ if (cf->count < 0) {
+ msg_info_main("skip spawning of worker %s: disabled in configuration",
+ cf->worker->name);
+
+ return;
+ }
+ if (cf->worker->flags & RSPAMD_WORKER_UNIQUE) {
+ if (cf->count > 1) {
+ msg_warn_main(
+ "cannot spawn more than 1 %s worker, so spawn one",
+ cf->worker->name);
+ }
+ rspamd_fork_worker(rspamd_main, cf, 0, event_loop, rspamd_cld_handler,
+ listen_sockets);
+ }
+ else if (cf->worker->flags & RSPAMD_WORKER_THREADED) {
+ rspamd_fork_worker(rspamd_main, cf, 0, event_loop, rspamd_cld_handler,
+ listen_sockets);
+ }
+ else {
+ for (i = 0; i < cf->count; i++) {
+ rspamd_fork_worker(rspamd_main, cf, i, event_loop,
+ rspamd_cld_handler, listen_sockets);
+ }
+ }
+}
+
+static void
+spawn_workers(struct rspamd_main *rspamd_main, struct ev_loop *ev_base)
+{
+ GList *cur, *ls;
+ struct rspamd_worker_conf *cf;
+ gpointer p;
+ guintptr key;
+ struct rspamd_worker_bind_conf *bcf;
+ gboolean listen_ok = FALSE;
+ GPtrArray *seen_mandatory_workers;
+ worker_t **cw, *wrk;
+ guint i;
+
+ /* Special hack for hs_helper if it's not defined in a config */
+ seen_mandatory_workers = g_ptr_array_new();
+ cur = rspamd_main->cfg->workers;
+
+ while (cur) {
+ cf = cur->data;
+ listen_ok = FALSE;
+
+ if (cf->worker == NULL) {
+ msg_err_main("type of worker is unspecified, skip spawning");
+ }
+ else {
+ if (!cf->enabled || cf->count <= 0) {
+ msg_info_main("worker of type %s(%s) is disabled in the config, "
+ "skip spawning",
+ g_quark_to_string(cf->type),
+ cf->bind_conf ? cf->bind_conf->name : "none");
+ cur = g_list_next(cur);
+
+ continue;
+ }
+
+ if (cf->worker->flags & RSPAMD_WORKER_ALWAYS_START) {
+ g_ptr_array_add(seen_mandatory_workers, cf->worker);
+ }
+ if (cf->worker->flags & RSPAMD_WORKER_HAS_SOCKET) {
+ LL_FOREACH(cf->bind_conf, bcf)
+ {
+ key = make_listen_key(bcf);
+
+ if ((p =
+ g_hash_table_lookup(listen_sockets,
+ GINT_TO_POINTER(key))) == NULL) {
+
+ if (!bcf->is_systemd) {
+ /* Create listen socket */
+ ls = create_listen_socket(bcf->addrs, bcf->cnt,
+ cf->worker->listen_type);
+ }
+ else {
+ ls = systemd_get_socket(rspamd_main,
+ g_ptr_array_index(bcf->addrs, 0));
+ }
+
+ if (ls == NULL) {
+ msg_err_main("cannot listen on %s socket %s: %s",
+ bcf->is_systemd ? "systemd" : "normal",
+ bcf->name,
+ strerror(errno));
+ }
+ else {
+ g_hash_table_insert(listen_sockets, (gpointer) key, ls);
+ listen_ok = TRUE;
+ }
+ }
+ else {
+ /* We had socket for this type of worker */
+ ls = p;
+ listen_ok = TRUE;
+ }
+ /* Do not add existing lists as it causes loops */
+ if (g_list_position(cf->listen_socks, ls) == -1) {
+ cf->listen_socks = g_list_concat(cf->listen_socks, ls);
+ }
+ }
+
+ if (listen_ok) {
+ spawn_worker_type(rspamd_main, ev_base, cf);
+ }
+ else {
+ if (cf->bind_conf == NULL) {
+ msg_err_main("cannot create listen socket for %s",
+ g_quark_to_string(cf->type));
+ }
+ else {
+ msg_err_main("cannot create listen socket for %s at %s",
+ g_quark_to_string(cf->type), cf->bind_conf->name);
+ }
+
+ rspamd_hard_terminate(rspamd_main);
+ g_assert_not_reached();
+ }
+ }
+ else {
+ spawn_worker_type(rspamd_main, ev_base, cf);
+ }
+ }
+
+ cur = g_list_next(cur);
+ }
+
+ for (cw = workers; *cw != NULL; cw++) {
+ gboolean seen = FALSE;
+
+ wrk = *cw;
+
+ if (wrk->flags & RSPAMD_WORKER_ALWAYS_START) {
+ for (i = 0; i < seen_mandatory_workers->len; i++) {
+ if (wrk == g_ptr_array_index(seen_mandatory_workers, i)) {
+ seen = TRUE;
+ break;
+ }
+ }
+
+ if (!seen) {
+ cf = rspamd_config_new_worker(rspamd_main->cfg, NULL);
+ cf->count = 1;
+ cf->worker = wrk;
+ cf->type = g_quark_from_static_string(wrk->name);
+
+ if (cf->worker->worker_init_func) {
+ cf->ctx = cf->worker->worker_init_func(rspamd_main->cfg);
+ }
+
+ spawn_worker_type(rspamd_main, ev_base, cf);
+ }
+ }
+ }
+
+ g_ptr_array_free(seen_mandatory_workers, TRUE);
+}
+
+static void
+kill_old_workers(gpointer key, gpointer value, gpointer unused)
+{
+ struct rspamd_worker *w = value;
+ struct rspamd_main *rspamd_main;
+
+ rspamd_main = w->srv;
+
+ if (w->state == rspamd_worker_state_wanna_die) {
+ w->state = rspamd_worker_state_terminating;
+ kill(w->pid, SIGUSR2);
+ ev_io_stop(rspamd_main->event_loop, &w->srv_ev);
+ g_hash_table_remove_all(w->control_events_pending);
+ msg_info_main("send signal to worker %P", w->pid);
+ }
+ else if (w->state != rspamd_worker_state_running) {
+ msg_info_main("do not send signal to worker %P, already sent", w->pid);
+ }
+}
+
+static void
+mark_old_workers(gpointer key, gpointer value, gpointer unused)
+{
+ struct rspamd_worker *w = value;
+
+ if (w->state == rspamd_worker_state_running) {
+ w->state = rspamd_worker_state_wanna_die;
+ }
+
+ w->flags |= RSPAMD_WORKER_OLD_CONFIG;
+}
+
+static void
+rspamd_worker_wait(struct rspamd_worker *w)
+{
+ struct rspamd_main *rspamd_main;
+ rspamd_main = w->srv;
+
+ if (term_attempts < 0) {
+ if (w->cf->worker->flags & RSPAMD_WORKER_KILLABLE) {
+ if (kill(w->pid, SIGKILL) == -1) {
+ if (errno == ESRCH) {
+ /* We have actually killed the process */
+ return;
+ }
+ }
+ else {
+ msg_warn_main("terminate worker %s(%P) with SIGKILL",
+ g_quark_to_string(w->type), w->pid);
+ }
+ }
+ else {
+ kill(w->pid, SIGKILL);
+ if (errno == ESRCH) {
+ /* We have actually killed the process */
+ return;
+ }
+ else {
+ msg_err_main("data corruption warning: terminating "
+ "special worker %s(%P) with SIGKILL",
+ g_quark_to_string(w->type), w->pid);
+ }
+ }
+ }
+}
+
+static void
+hash_worker_wait_callback(gpointer key, gpointer value, gpointer unused)
+{
+ rspamd_worker_wait((struct rspamd_worker *) value);
+}
+
+struct core_check_cbdata {
+ struct rspamd_config *cfg;
+ gsize total_count;
+ gsize total_size;
+};
+
+#ifdef HAVE_NFTW
+
+static struct core_check_cbdata cores_cbdata;
+
+static gint
+rspamd_check_core_cb(const gchar *path, const struct stat *st,
+ gint flag, struct FTW *ft)
+{
+ if (S_ISREG(st->st_mode)) {
+ cores_cbdata.total_count++;
+ /* Use physical size instead of displayed one */
+ cores_cbdata.total_size += st->st_blocks * 512;
+ }
+
+ return 0;
+}
+
+#endif
+
+static void
+rspamd_check_core_limits(struct rspamd_main *rspamd_main)
+{
+#ifdef HAVE_NFTW
+ struct rspamd_config *cfg = rspamd_main->cfg;
+
+ cores_cbdata.cfg = cfg;
+ cores_cbdata.total_count = 0;
+ cores_cbdata.total_size = 0;
+
+ if (cfg->cores_dir && (cfg->max_cores_count || cfg->max_cores_size)) {
+ if (nftw(cfg->cores_dir, rspamd_check_core_cb, 1, FTW_MOUNT | FTW_PHYS) == -1) {
+ msg_err_main("nftw failed for path %s: %s", cfg->cores_dir,
+ strerror(errno));
+ }
+ else {
+ if (!rspamd_main->cores_throttling) {
+ if (cfg->max_cores_size &&
+ cores_cbdata.total_size > cfg->max_cores_size) {
+ msg_warn_main(
+ "enable cores throttling as size of cores in"
+ " %s is %Hz, limit is %Hz",
+ cfg->cores_dir,
+ cores_cbdata.total_size,
+ cfg->max_cores_size);
+ rspamd_main->cores_throttling = TRUE;
+ }
+ if (cfg->max_cores_count &&
+ cores_cbdata.total_count > cfg->max_cores_count) {
+ msg_warn_main(
+ "enable cores throttling as count of cores in"
+ " %s is %z, limit is %z",
+ cfg->cores_dir,
+ cores_cbdata.total_count,
+ cfg->max_cores_count);
+ rspamd_main->cores_throttling = TRUE;
+ }
+ }
+ else {
+ if (cfg->max_cores_size &&
+ cores_cbdata.total_size < cfg->max_cores_size) {
+ msg_info_main(
+ "disable cores throttling as size of cores in"
+ " %s is now %Hz, limit is %Hz",
+ cfg->cores_dir,
+ cores_cbdata.total_size,
+ cfg->max_cores_size);
+ rspamd_main->cores_throttling = FALSE;
+ }
+ if (cfg->max_cores_count &&
+ cores_cbdata.total_count < cfg->max_cores_count) {
+ msg_info_main(
+ "disable cores throttling as count of cores in"
+ " %s is %z, limit is %z",
+ cfg->cores_dir,
+ cores_cbdata.total_count,
+ cfg->max_cores_count);
+ rspamd_main->cores_throttling = FALSE;
+ }
+ }
+ }
+ }
+#endif
+}
+
+static void
+reopen_log_handler(gpointer key, gpointer value, gpointer unused)
+{
+ struct rspamd_worker *w = value;
+ struct rspamd_main *rspamd_main;
+
+ rspamd_main = w->srv;
+
+ if (kill(w->pid, SIGUSR1) == -1) {
+ msg_err_main("kill failed for pid %P: %s", w->pid, strerror(errno));
+ }
+}
+
+static gboolean
+load_rspamd_config(struct rspamd_main *rspamd_main,
+ struct rspamd_config *cfg, gboolean init_modules,
+ enum rspamd_post_load_options opts,
+ gboolean reload)
+{
+ cfg->compiled_modules = modules;
+ cfg->compiled_workers = workers;
+
+ if (!rspamd_config_read(cfg, cfg->cfg_name, config_logger, rspamd_main,
+ ucl_vars, skip_template, lua_env)) {
+ return FALSE;
+ }
+
+ /* Strictly set temp dir */
+ if (!cfg->temp_dir) {
+ msg_warn_main("tempdir is not set, trying to use $TMPDIR");
+ cfg->temp_dir =
+ rspamd_mempool_strdup(cfg->cfg_pool, getenv("TMPDIR"));
+
+ if (!cfg->temp_dir) {
+ msg_warn_main("$TMPDIR is empty too, using /tmp as default");
+ cfg->temp_dir = rspamd_mempool_strdup(cfg->cfg_pool, "/tmp");
+ }
+ }
+
+ if (!reload) {
+ /*
+ * As some rules are defined in lua, we need to process them, then init
+ * modules and merely afterwards to init modules
+ */
+ rspamd_lua_post_load_config(cfg);
+
+ if (init_modules) {
+ if (!rspamd_init_filters(cfg, reload, false)) {
+ return FALSE;
+ }
+ }
+
+ /* Do post-load actions */
+ if (!rspamd_config_post_load(cfg, opts)) {
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+static void
+rspamd_detach_worker(struct rspamd_main *rspamd_main, struct rspamd_worker *wrk)
+{
+ ev_io_stop(rspamd_main->event_loop, &wrk->srv_ev);
+ ev_timer_stop(rspamd_main->event_loop, &wrk->hb.heartbeat_ev);
+}
+
+static void
+rspamd_attach_worker(struct rspamd_main *rspamd_main, struct rspamd_worker *wrk)
+{
+ ev_io_start(rspamd_main->event_loop, &wrk->srv_ev);
+ ev_timer_start(rspamd_main->event_loop, &wrk->hb.heartbeat_ev);
+}
+
+static void
+stop_srv_ev(gpointer key, gpointer value, gpointer ud)
+{
+ struct rspamd_worker *cur = (struct rspamd_worker *) value;
+ struct rspamd_main *rspamd_main = (struct rspamd_main *) ud;
+
+ rspamd_detach_worker(rspamd_main, cur);
+}
+
+static void
+start_srv_ev(gpointer key, gpointer value, gpointer ud)
+{
+ struct rspamd_worker *cur = (struct rspamd_worker *) value;
+ struct rspamd_main *rspamd_main = (struct rspamd_main *) ud;
+
+ rspamd_attach_worker(rspamd_main, cur);
+}
+
+static void
+rspamd_final_timer_handler(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_main *rspamd_main = (struct rspamd_main *) w->data;
+
+ term_attempts--;
+
+ g_hash_table_foreach(rspamd_main->workers, hash_worker_wait_callback,
+ NULL);
+
+ if (g_hash_table_size(rspamd_main->workers) == 0) {
+ ev_break(rspamd_main->event_loop, EVBREAK_ALL);
+ }
+}
+
+/* Signal handlers */
+static void
+rspamd_term_handler(struct ev_loop *loop, ev_signal *w, int revents)
+{
+ struct rspamd_main *rspamd_main = (struct rspamd_main *) w->data;
+ static ev_timer ev_finale;
+ ev_tstamp shutdown_ts;
+
+ if (!rspamd_main->wanna_die) {
+ rspamd_main->wanna_die = TRUE;
+ shutdown_ts = MAX(SOFT_SHUTDOWN_TIME,
+ rspamd_main->cfg->task_timeout * 2.0);
+ msg_info_main("catch termination signal, waiting for %d children for %.2f seconds",
+ (gint) g_hash_table_size(rspamd_main->workers),
+ valgrind_mode ? shutdown_ts * 10 : shutdown_ts);
+ /* Stop srv events to avoid false notifications */
+ g_hash_table_foreach(rspamd_main->workers, stop_srv_ev, rspamd_main);
+ rspamd_pass_signal(rspamd_main->workers, SIGTERM);
+
+ if (control_fd != -1) {
+ ev_io_stop(rspamd_main->event_loop, &control_ev);
+ close(control_fd);
+ }
+
+ if (valgrind_mode) {
+ /* Special case if we are likely running with valgrind */
+ term_attempts = shutdown_ts / TERMINATION_INTERVAL * 10;
+ }
+ else {
+ term_attempts = shutdown_ts / TERMINATION_INTERVAL;
+ }
+
+ ev_finale.data = rspamd_main;
+ ev_timer_init(&ev_finale, rspamd_final_timer_handler,
+ TERMINATION_INTERVAL, TERMINATION_INTERVAL);
+ ev_timer_start(rspamd_main->event_loop, &ev_finale);
+ }
+}
+
+static void
+rspamd_usr1_handler(struct ev_loop *loop, ev_signal *w, int revents)
+{
+ struct rspamd_main *rspamd_main = (struct rspamd_main *) w->data;
+
+ if (!rspamd_main->wanna_die) {
+ rspamd_log_reopen(rspamd_main->logger,
+ rspamd_main->cfg,
+ rspamd_main->workers_uid,
+ rspamd_main->workers_gid);
+ msg_info_main("logging reinitialised");
+ g_hash_table_foreach(rspamd_main->workers, reopen_log_handler,
+ NULL);
+ }
+}
+
+static void
+rspamd_stat_update_handler(struct ev_loop *loop, ev_timer *w, int revents)
+{
+ struct rspamd_main *rspamd_main = (struct rspamd_main *) w->data;
+ struct rspamd_stat cur_stat;
+ gchar proctitle[128];
+
+ memcpy(&cur_stat, rspamd_main->stat, sizeof(cur_stat));
+
+ if (old_stat.messages_scanned > 0 &&
+ cur_stat.messages_scanned > old_stat.messages_scanned) {
+ gdouble rate = (double) (cur_stat.messages_scanned - old_stat.messages_scanned) /
+ w->repeat;
+ gdouble old_spam = old_stat.actions_stat[METRIC_ACTION_REJECT] +
+ old_stat.actions_stat[METRIC_ACTION_ADD_HEADER] +
+ old_stat.actions_stat[METRIC_ACTION_REWRITE_SUBJECT];
+ gdouble old_ham = old_stat.actions_stat[METRIC_ACTION_NOACTION];
+ gdouble new_spam = cur_stat.actions_stat[METRIC_ACTION_REJECT] +
+ cur_stat.actions_stat[METRIC_ACTION_ADD_HEADER] +
+ cur_stat.actions_stat[METRIC_ACTION_REWRITE_SUBJECT];
+ gdouble new_ham = cur_stat.actions_stat[METRIC_ACTION_NOACTION];
+
+ gsize cnt = MAX_AVG_TIME_SLOTS;
+ float sum = rspamd_sum_floats(cur_stat.avg_time.avg_time, &cnt);
+
+ rspamd_snprintf(proctitle, sizeof(proctitle),
+ "main process; %.1f msg/sec, %.1f msg/sec spam, %.1f msg/sec ham; %.2fs avg processing time",
+ rate,
+ (new_spam - old_spam) / w->repeat,
+ (new_ham - old_ham) / w->repeat,
+ cnt > 0 ? sum / cnt : 0);
+ rspamd_setproctitle("%s", proctitle);
+ }
+
+ memcpy(&old_stat, &cur_stat, sizeof(cur_stat));
+}
+
+static void
+rspamd_hup_handler(struct ev_loop *loop, ev_signal *w, int revents)
+{
+ struct rspamd_main *rspamd_main = (struct rspamd_main *) w->data;
+
+ if (!rspamd_main->wanna_die) {
+ msg_info_main("rspamd " RVERSION
+ " is requested to reload configuration");
+ /* Detach existing workers and stop their heartbeats */
+ g_hash_table_foreach(rspamd_main->workers, stop_srv_ev, rspamd_main);
+
+ if (reread_config(rspamd_main)) {
+ rspamd_check_core_limits(rspamd_main);
+ /* Mark old workers */
+ g_hash_table_foreach(rspamd_main->workers, mark_old_workers, NULL);
+ msg_info_main("spawn workers with a new config");
+ spawn_workers(rspamd_main, rspamd_main->event_loop);
+ msg_info_main("workers spawning has been finished");
+ /* Kill marked */
+ msg_info_main("kill old workers");
+ g_hash_table_foreach(rspamd_main->workers, kill_old_workers, NULL);
+ }
+ else {
+ /* Reattach old workers */
+ msg_info_main("restore old workers with a old config");
+ g_hash_table_foreach(rspamd_main->workers, start_srv_ev, rspamd_main);
+ }
+ }
+}
+
+/* Called when a dead child has been found */
+
+static void
+rspamd_cld_handler(EV_P_ ev_child *w, struct rspamd_main *rspamd_main,
+ struct rspamd_worker *wrk)
+{
+ gboolean need_refork;
+ static struct rspamd_control_command cmd;
+
+ /* Turn off locking for logger */
+ ev_child_stop(EV_A_ w);
+
+ /* Remove dead child form children list */
+ g_hash_table_remove(rspamd_main->workers, GSIZE_TO_POINTER(wrk->pid));
+ g_hash_table_remove_all(wrk->control_events_pending);
+
+ if (wrk->srv_pipe[0] != -1) {
+ /* Ugly workaround */
+ if (wrk->tmp_data) {
+ g_free(wrk->tmp_data);
+ }
+
+ rspamd_detach_worker(rspamd_main, wrk);
+ }
+
+ if (wrk->control_pipe[0] != -1) {
+ /* We also need to clean descriptors left */
+ close(wrk->control_pipe[0]);
+ close(wrk->srv_pipe[0]);
+ }
+
+ if (!rspamd_main->wanna_die) {
+ cmd.type = RSPAMD_CONTROL_CHILD_CHANGE;
+ cmd.cmd.child_change.what = rspamd_child_terminated;
+ cmd.cmd.child_change.pid = wrk->pid;
+ cmd.cmd.child_change.additional = w->rstatus;
+ rspamd_control_broadcast_srv_cmd(rspamd_main, &cmd, wrk->pid);
+ }
+
+ need_refork = rspamd_check_termination_clause(wrk->srv, wrk, w->rstatus);
+
+ if (need_refork) {
+ /* Fork another worker in replace of dead one */
+ msg_info_main("respawn process %s in lieu of terminated process with pid %P",
+ g_quark_to_string(wrk->type),
+ wrk->pid);
+ rspamd_check_core_limits(rspamd_main);
+ rspamd_fork_delayed(wrk->cf, wrk->index, rspamd_main);
+ }
+ else {
+ msg_info_main("do not respawn process %s after found terminated process with pid %P",
+ g_quark_to_string(wrk->type),
+ wrk->pid);
+ }
+
+ REF_RELEASE(wrk->cf);
+ g_hash_table_unref(wrk->control_events_pending);
+ g_free(wrk);
+}
+
+/* Control socket handler */
+static void
+rspamd_control_handler(EV_P_ ev_io *w, int revents)
+{
+ struct rspamd_main *rspamd_main = (struct rspamd_main *) w->data;
+ rspamd_inet_addr_t *addr = NULL;
+ gint nfd;
+
+ if ((nfd =
+ rspamd_accept_from_socket(w->fd, &addr, NULL, NULL)) == -1) {
+ msg_warn_main("accept failed: %s", strerror(errno));
+ return;
+ }
+ /* Check for EAGAIN */
+ if (nfd == 0) {
+ rspamd_inet_address_free(addr);
+ return;
+ }
+
+ msg_info_main("accepted control connection from %s",
+ rspamd_inet_address_to_string(addr));
+
+ rspamd_control_process_client_socket(rspamd_main, nfd, addr);
+}
+
+static guint
+rspamd_spair_hash(gconstpointer p)
+{
+ return rspamd_cryptobox_fast_hash(p, PAIR_ID_LEN, rspamd_hash_seed());
+}
+
+static gboolean
+rspamd_spair_equal(gconstpointer a, gconstpointer b)
+{
+ return memcmp(a, b, PAIR_ID_LEN) == 0;
+}
+
+static void
+rspamd_spair_close(gpointer p)
+{
+ gint *fds = p;
+
+ close(fds[0]);
+ close(fds[1]);
+ g_free(p);
+}
+
+const char *
+get_cpu_architecture(void)
+{
+#if defined(__x86_64__) || defined(_M_X64)
+ return "x86_64";
+#elif defined(__i386) || defined(_M_IX86)
+ return "x86";
+#elif defined(__aarch64__)
+ return "ARM64";
+#elif defined(__arm__) || defined(_M_ARM)
+ return "ARM";
+#elif defined(__loongarch__) || defined(__loongarch64)
+ return "LOONGARCH64";
+#elif defined(__mips__)
+ return "MIPS";
+#elif defined(__powerpc__) || defined(_M_PPC)
+ return "PowerPC";
+#elif defined(__sparc__)
+ return "SPARC";
+#else
+ return "Unknown";
+#endif
+}
+
+static void
+version(struct rspamd_main *rspamd_main)
+{
+#if defined(GIT_VERSION) && GIT_VERSION == 1
+ rspamd_printf("Rspamd daemon version " RVERSION "-git." RID "\n\n");
+#else
+ rspamd_printf("Rspamd daemon version " RVERSION "\n\n");
+#endif
+ rspamd_printf("CPU architecture %s; features: %s\n",
+ get_cpu_architecture(),
+ rspamd_main->cfg->libs_ctx->crypto_ctx->cpu_extensions);
+#ifdef WITH_HYPERSCAN
+ rspamd_printf("Hyperscan enabled: TRUE\n");
+#else
+ rspamd_printf("Hyperscan enabled: FALSE\n");
+#endif
+
+#ifdef WITH_JEMALLOC
+ rspamd_printf("Jemalloc enabled: TRUE\n");
+#else
+ rspamd_printf("Jemalloc enabled: FALSE\n");
+#endif
+#ifdef WITH_LUAJIT
+ rspamd_printf("LuaJIT enabled: TRUE (LuaJIT version: %s)\n", LUAJIT_VERSION);
+#else
+ rspamd_printf("LuaJIT enabled: FALSE (Lua version: %s)\n", LUA_VERSION);
+#endif
+#ifndef __has_feature
+#define __has_feature(x) 0
+#endif
+#if (defined(__has_feature) && __has_feature(address_sanitizer)) || defined(ADDRESS_SANITIZER)
+ rspamd_printf("ASAN enabled: TRUE\n");
+#else
+ rspamd_printf("ASAN enabled: FALSE\n");
+#endif
+#ifdef HAVE_CBLAS
+ rspamd_printf("BLAS enabled: TRUE\n");
+#else
+ rspamd_printf("BLAS enabled: FALSE\n");
+#endif
+#ifdef WITH_FASTTEXT
+ rspamd_printf("Fasttext enabled: TRUE\n");
+#else
+ rspamd_printf("Fasttext enabled: FALSE\n");
+#endif
+}
+
+static gboolean
+rspamd_main_daemon(struct rspamd_main *rspamd_main)
+{
+ int fd;
+ pid_t old_pid = getpid();
+
+ switch (fork()) {
+ case -1:
+ msg_err_main("fork() failed: %s", strerror(errno));
+ return FALSE;
+
+ case 0:
+ break;
+
+ default:
+ /* Old process */
+ exit(0);
+ }
+
+ rspamd_log_on_fork(g_quark_from_static_string("main"),
+ rspamd_main->cfg,
+ rspamd_main->logger);
+
+ if (setsid() == -1) {
+ msg_err_main("setsid () failed: %s", strerror(errno));
+ return FALSE;
+ }
+
+ umask(0);
+
+ fd = open("/dev/null", O_RDWR);
+ if (fd == -1) {
+ msg_err_main("open(\"/dev/null\") failed: %s", strerror(errno));
+ return FALSE;
+ }
+
+ if (dup2(fd, STDIN_FILENO) == -1) {
+ msg_err_main("dup2(STDIN) failed: %s", strerror(errno));
+ return FALSE;
+ }
+
+ if (dup2(fd, STDOUT_FILENO) == -1) {
+ msg_err_main("dup2(STDOUT) failed: %s", strerror(errno));
+ return FALSE;
+ }
+
+ if (fd > STDERR_FILENO) {
+ if (close(fd) == -1) {
+ msg_err_main("close() failed: %s", strerror(errno));
+ return FALSE;
+ }
+ }
+
+ msg_info_main("daemonized successfully; old pid %P, new pid %P; pid file: %s",
+ old_pid, getpid(),
+ rspamd_main->cfg->pid_file);
+
+ return TRUE;
+}
+
+gint main(gint argc, gchar **argv, gchar **env)
+{
+ gint i, res = 0;
+ struct sigaction signals, sigpipe_act;
+ worker_t **pworker;
+ GQuark type;
+ rspamd_inet_addr_t *control_addr = NULL;
+ struct ev_loop *event_loop;
+ struct rspamd_main *rspamd_main;
+ gboolean skip_pid = FALSE;
+ sigset_t control_signals;
+
+ /* Block special signals on loading */
+ sigemptyset(&control_signals);
+ sigaddset(&control_signals, SIGHUP);
+ sigaddset(&control_signals, SIGUSR1);
+ sigaddset(&control_signals, SIGUSR2);
+ sigprocmask(SIG_BLOCK, &control_signals, NULL);
+
+ rspamd_main = (struct rspamd_main *) g_malloc0(sizeof(struct rspamd_main));
+
+ rspamd_main->server_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "main", 0);
+ rspamd_main->stat = rspamd_mempool_alloc0_shared_(rspamd_main->server_pool,
+ sizeof(struct rspamd_stat),
+ RSPAMD_ALIGNOF(struct rspamd_stat),
+ G_STRLOC);
+ /* Set all time slots to nan */
+ for (i = 0; i < MAX_AVG_TIME_SLOTS; i++) {
+ rspamd_main->stat->avg_time.avg_time[i] = NAN;
+ }
+
+ rspamd_main->cfg = rspamd_config_new(RSPAMD_CONFIG_INIT_DEFAULT);
+ rspamd_main->spairs = g_hash_table_new_full(rspamd_spair_hash,
+ rspamd_spair_equal, g_free, rspamd_spair_close);
+ rspamd_main->start_mtx = rspamd_mempool_get_mutex(rspamd_main->server_pool);
+
+ if (getenv("VALGRIND") != NULL) {
+ valgrind_mode = TRUE;
+ }
+
+#ifndef HAVE_SETPROCTITLE
+ rspamd_init_title(rspamd_main->server_pool, argc, argv, env);
+#endif
+
+ rspamd_main->cfg->libs_ctx = rspamd_init_libs();
+ memset(&signals, 0, sizeof(struct sigaction));
+
+ read_cmd_line(&argc, &argv, rspamd_main->cfg);
+
+ if (show_version) {
+ version(rspamd_main);
+ exit(EXIT_SUCCESS);
+ }
+
+ if (argc > 0) {
+ /* Parse variables */
+ for (i = 0; i < argc; i++) {
+ if (strchr(argv[i], '=') != NULL) {
+ gchar *k, *v, *t;
+
+ k = g_strdup(argv[i]);
+ t = strchr(k, '=');
+ v = g_strdup(t + 1);
+ *t = '\0';
+
+ if (ucl_vars == NULL) {
+ ucl_vars = g_hash_table_new_full(rspamd_strcase_hash,
+ rspamd_strcase_equal, g_free, g_free);
+ }
+
+ g_hash_table_insert(ucl_vars, k, v);
+ }
+ }
+ }
+
+ if (is_debug) {
+ rspamd_main->cfg->log_level = G_LOG_LEVEL_DEBUG;
+ }
+ else {
+ rspamd_main->cfg->log_level = G_LOG_LEVEL_MESSAGE;
+ }
+
+ type = g_quark_from_static_string("main");
+
+ /* First set logger to console logger */
+ rspamd_main->logger = rspamd_log_open_emergency(rspamd_main->server_pool, 0);
+ g_assert(rspamd_main->logger != NULL);
+
+ if (is_debug) {
+ rspamd_log_set_log_level(rspamd_main->logger, G_LOG_LEVEL_DEBUG);
+ }
+ else {
+ rspamd_log_set_log_level(rspamd_main->logger, G_LOG_LEVEL_MESSAGE);
+ }
+
+ g_log_set_default_handler(rspamd_glib_log_function, rspamd_main->logger);
+ g_set_printerr_handler(rspamd_glib_printerr_function);
+
+ detect_priv(rspamd_main);
+
+ msg_notice_main("rspamd " RVERSION
+ " is loading configuration, build id: " RID);
+
+ pworker = &workers[0];
+ while (*pworker) {
+ /* Init string quarks */
+ (void) g_quark_from_static_string((*pworker)->name);
+ pworker++;
+ }
+
+ /* Init listen sockets hash */
+ listen_sockets = g_hash_table_new(g_direct_hash, g_direct_equal);
+ sqlite3_initialize();
+
+ /* Load config */
+ if (!load_rspamd_config(rspamd_main, rspamd_main->cfg, TRUE,
+ RSPAMD_CONFIG_LOAD_ALL, FALSE)) {
+ exit(EXIT_FAILURE);
+ }
+
+ /* Override pidfile from configuration by command line argument */
+ if (rspamd_pidfile != NULL) {
+ rspamd_main->cfg->pid_file = rspamd_pidfile;
+ }
+
+ /* Force debug log */
+ if (is_debug) {
+ rspamd_log_set_log_level(rspamd_main->logger, G_LOG_LEVEL_DEBUG);
+ }
+
+ /* Create rolling history */
+ rspamd_main->history = rspamd_roll_history_new(rspamd_main->server_pool,
+ rspamd_main->cfg->history_rows, rspamd_main->cfg);
+
+ msg_info_main("rspamd " RVERSION
+ " is starting, build id: " RID);
+ rspamd_main->cfg->cfg_name = rspamd_mempool_strdup(
+ rspamd_main->cfg->cfg_pool,
+ rspamd_main->cfg->cfg_name);
+ msg_info_main("cpu features: %s",
+ rspamd_main->cfg->libs_ctx->crypto_ctx->cpu_extensions);
+ msg_info_main("cryptobox configuration: curve25519(libsodium), "
+ "chacha20(%s), poly1305(libsodium), siphash(libsodium), blake2(libsodium), base64(%s)",
+ rspamd_main->cfg->libs_ctx->crypto_ctx->chacha20_impl,
+ rspamd_main->cfg->libs_ctx->crypto_ctx->base64_impl);
+ msg_info_main("libottery prf: %s", ottery_get_impl_name());
+
+ /* Daemonize */
+ if (!no_fork) {
+ if (!rspamd_main_daemon(rspamd_main)) {
+ exit(EXIT_FAILURE);
+ }
+
+ /* Close emergency logger */
+ rspamd_log_close(rspamd_log_emergency_logger());
+ }
+
+ /* Write info */
+ rspamd_main->pid = getpid();
+ rspamd_main->type = type;
+
+ rspamd_set_crash_handler(rspamd_main);
+
+ /* Ignore SIGPIPE as we handle write errors manually */
+ sigemptyset(&sigpipe_act.sa_mask);
+ sigaddset(&sigpipe_act.sa_mask, SIGPIPE);
+ sigpipe_act.sa_handler = SIG_IGN;
+ sigpipe_act.sa_flags = 0;
+ sigaction(SIGPIPE, &sigpipe_act, NULL);
+
+ if (rspamd_main->cfg->pid_file == NULL) {
+ msg_info_main("pid file is not specified, skipping writing it");
+ skip_pid = TRUE;
+ }
+ else if (no_fork) {
+ msg_info_main("skip writing pid in no-fork mode");
+ skip_pid = TRUE;
+ }
+ else if (rspamd_write_pid(rspamd_main) == -1) {
+ msg_err_main("cannot write pid file %s", rspamd_main->cfg->pid_file);
+ exit(-errno);
+ }
+
+ sigprocmask(SIG_BLOCK, &signals.sa_mask, NULL);
+
+ /* Set title */
+ rspamd_setproctitle("main process");
+
+ /* Open control socket if needed */
+ control_fd = -1;
+ if (rspamd_main->cfg->control_socket_path) {
+ if (!rspamd_parse_inet_address(&control_addr,
+ rspamd_main->cfg->control_socket_path,
+ strlen(rspamd_main->cfg->control_socket_path),
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ msg_err_main("cannot parse inet address %s",
+ rspamd_main->cfg->control_socket_path);
+ }
+ else {
+ control_fd = rspamd_inet_address_listen(control_addr, SOCK_STREAM,
+ RSPAMD_INET_ADDRESS_LISTEN_ASYNC, -1);
+ if (control_fd == -1) {
+ msg_err_main("cannot open control socket at path: %s",
+ rspamd_main->cfg->control_socket_path);
+ }
+ }
+ }
+
+ /* Maybe read roll history */
+ if (rspamd_main->cfg->history_file) {
+ rspamd_roll_history_load(rspamd_main->history,
+ rspamd_main->cfg->history_file);
+ }
+
+ /* Init workers hash */
+ rspamd_main->workers = g_hash_table_new(g_direct_hash, g_direct_equal);
+
+ /* Unblock control signals */
+ sigprocmask(SIG_UNBLOCK, &control_signals, NULL);
+ /* Init event base */
+ event_loop = ev_default_loop(rspamd_config_ev_backend_get(rspamd_main->cfg));
+ rspamd_main->event_loop = event_loop;
+
+ if (event_loop) {
+ int loop_type = ev_backend(event_loop);
+ gboolean effective_backend;
+ const gchar *loop_str;
+
+ loop_str =
+ rspamd_config_ev_backend_to_string(loop_type, &effective_backend);
+
+ if (!effective_backend) {
+ msg_warn_main("event loop uses non-optimal backend: %s", loop_str);
+ }
+ else {
+ msg_info_main("event loop initialised with backend: %s", loop_str);
+ }
+ }
+ else {
+ msg_err("cannot init event loop! exiting");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Unblock signals */
+ sigemptyset(&signals.sa_mask);
+ sigprocmask(SIG_SETMASK, &signals.sa_mask, NULL);
+
+ /* Set events for signals */
+ ev_signal_init(&rspamd_main->term_ev, rspamd_term_handler, SIGTERM);
+ rspamd_main->term_ev.data = rspamd_main;
+ ev_signal_start(event_loop, &rspamd_main->term_ev);
+
+ ev_signal_init(&rspamd_main->int_ev, rspamd_term_handler, SIGINT);
+ rspamd_main->int_ev.data = rspamd_main;
+ ev_signal_start(event_loop, &rspamd_main->int_ev);
+
+ ev_signal_init(&rspamd_main->hup_ev, rspamd_hup_handler, SIGHUP);
+ rspamd_main->hup_ev.data = rspamd_main;
+ ev_signal_start(event_loop, &rspamd_main->hup_ev);
+
+ ev_signal_init(&rspamd_main->usr1_ev, rspamd_usr1_handler, SIGUSR1);
+ rspamd_main->usr1_ev.data = rspamd_main;
+ ev_signal_start(event_loop, &rspamd_main->usr1_ev);
+
+ /* Update proctitle according to number of messages processed */
+ static const ev_tstamp stat_update_time = 10.0;
+
+ memset(&old_stat, 0, sizeof(old_stat));
+ stat_ev.data = rspamd_main;
+ ev_timer_init(&stat_ev, rspamd_stat_update_handler,
+ stat_update_time, stat_update_time);
+ ev_timer_start(event_loop, &stat_ev);
+
+ rspamd_check_core_limits(rspamd_main);
+ rspamd_mempool_lock_mutex(rspamd_main->start_mtx);
+ spawn_workers(rspamd_main, event_loop);
+ rspamd_mempool_unlock_mutex(rspamd_main->start_mtx);
+
+ rspamd_main->http_ctx = rspamd_http_context_create(rspamd_main->cfg,
+ event_loop, rspamd_main->cfg->ups_ctx);
+
+ if (control_fd != -1) {
+ msg_info_main("listening for control commands on %s",
+ rspamd_inet_address_to_string(control_addr));
+ ev_io_init(&control_ev, rspamd_control_handler, control_fd, EV_READ);
+ control_ev.data = rspamd_main;
+ ev_io_start(event_loop, &control_ev);
+ }
+
+ ev_loop(event_loop, 0);
+
+ /* Maybe save roll history */
+ if (rspamd_main->cfg->history_file) {
+ rspamd_roll_history_save(rspamd_main->history,
+ rspamd_main->cfg->history_file);
+ }
+
+ if (rspamd_main->cfg->cache) {
+ rspamd_symcache_save(rspamd_main->cfg->cache);
+ }
+
+ msg_info_main("terminating...");
+
+#ifdef WITH_HYPERSCAN
+ rspamd_hyperscan_cleanup_maybe();
+#endif
+ REF_RELEASE(rspamd_main->cfg);
+ rspamd_log_close(rspamd_main->logger);
+ g_hash_table_unref(rspamd_main->spairs);
+ g_hash_table_unref(rspamd_main->workers);
+ rspamd_mempool_delete(rspamd_main->server_pool);
+
+ if (!skip_pid) {
+ rspamd_pidfile_close(rspamd_main->pfh);
+ }
+
+ rspamd_unset_crash_handler(rspamd_main);
+ g_free(rspamd_main);
+ ev_unref(event_loop);
+ sqlite3_shutdown();
+
+ if (control_addr) {
+ rspamd_inet_address_free(control_addr);
+ }
+
+ return (res);
+}
diff --git a/src/rspamd.h b/src/rspamd.h
new file mode 100644
index 0000000..523ea79
--- /dev/null
+++ b/src/rspamd.h
@@ -0,0 +1,378 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_MAIN_H
+#define RSPAMD_MAIN_H
+
+#include "config.h"
+#include "libutil/fstring.h"
+#include "libutil/mem_pool.h"
+#include "libutil/util.h"
+#include "libserver/logger.h"
+#include "libserver/http/http_connection.h"
+#include "libutil/upstream.h"
+#include "libutil/radix.h"
+#include "libserver/cfg_file.h"
+#include "libserver/url.h"
+#include "libserver/protocol.h"
+#include "libserver/async_session.h"
+#include "libserver/roll_history.h"
+#include "libserver/task.h"
+
+#include <openssl/ssl.h>
+
+/* Default values */
+#define FIXED_CONFIG_FILE RSPAMD_CONFDIR "/rspamd.conf"
+/* Time in seconds to exit for old worker */
+#define SOFT_SHUTDOWN_TIME 10
+
+/* Spam subject */
+#define SPAM_SUBJECT "*** SPAM *** %s"
+
+#ifdef CRLF
+#undef CRLF
+#undef CR
+#undef LF
+#endif
+
+#define CRLF "\r\n"
+#define CR '\r'
+#define LF '\n'
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_main;
+
+enum rspamd_worker_flags {
+ RSPAMD_WORKER_HAS_SOCKET = (1 << 0),
+ RSPAMD_WORKER_UNIQUE = (1 << 1),
+ RSPAMD_WORKER_THREADED = (1 << 2),
+ RSPAMD_WORKER_KILLABLE = (1 << 3),
+ RSPAMD_WORKER_ALWAYS_START = (1 << 4),
+ RSPAMD_WORKER_SCANNER = (1 << 5),
+ RSPAMD_WORKER_CONTROLLER = (1 << 6),
+ RSPAMD_WORKER_NO_TERMINATE_DELAY = (1 << 7),
+ RSPAMD_WORKER_OLD_CONFIG = (1 << 8),
+ RSPAMD_WORKER_NO_STRICT_CONFIG = (1 << 9),
+};
+
+struct rspamd_worker_accept_event {
+ ev_io accept_ev;
+ ev_timer throttling_ev;
+ struct ev_loop *event_loop;
+ struct rspamd_worker_accept_event *prev, *next;
+};
+
+typedef void (*rspamd_worker_term_cb)(EV_P_ ev_child *, struct rspamd_main *,
+ struct rspamd_worker *);
+
+struct rspamd_worker_heartbeat {
+ ev_timer heartbeat_ev; /**< used by main for checking heartbeats and by workers to send heartbeats */
+ ev_tstamp last_event; /**< last heartbeat received timestamp */
+ gint64 nbeats; /**< positive for beats received, negative for beats missed */
+};
+
+enum rspamd_worker_state {
+ rspamd_worker_state_running = 0,
+ rspamd_worker_state_wanna_die,
+ rspamd_worker_state_terminating,
+ rspamd_worker_wait_connections,
+ rspamd_worker_wait_final_scripts,
+ rspamd_worker_wanna_die
+};
+
+/**
+ * Worker process structure
+ */
+struct rspamd_worker {
+ pid_t pid; /**< pid of worker */
+ pid_t ppid; /**< pid of parent */
+ guint index; /**< index number */
+ guint nconns; /**< current connections count */
+ enum rspamd_worker_state state; /**< current worker state */
+ gboolean cores_throttled; /**< set to true if cores throttling took place */
+ gdouble start_time; /**< start time */
+ struct rspamd_main *srv; /**< pointer to server structure */
+ GQuark type; /**< process type */
+ GHashTable *signal_events; /**< signal events */
+ struct rspamd_worker_accept_event *accept_events; /**< socket events */
+ struct rspamd_worker_conf *cf; /**< worker config data */
+ gpointer ctx; /**< worker's specific data */
+ gint flags; /**< worker's flags (enum rspamd_worker_flags) */
+ gint control_pipe[2]; /**< control pipe. [0] is used by main process,
+ [1] is used by a worker */
+ gint srv_pipe[2]; /**< used by workers to request something from the
+ main process. [0] - main, [1] - worker */
+ ev_io srv_ev; /**< used by main for read workers' requests */
+ struct rspamd_worker_heartbeat hb; /**< heartbeat data */
+ gpointer control_data; /**< used by control protocol to handle commands */
+ gpointer tmp_data; /**< used to avoid race condition to deal with control messages */
+ ev_child cld_ev; /**< to allow reaping */
+ rspamd_worker_term_cb term_handler; /**< custom term handler */
+ GHashTable *control_events_pending; /**< control events pending indexed by ptr */
+};
+
+struct rspamd_abstract_worker_ctx {
+ guint64 magic;
+ /* Events base */
+ struct ev_loop *event_loop;
+ /* DNS resolver */
+ struct rspamd_dns_resolver *resolver;
+ /* Config */
+ struct rspamd_config *cfg;
+ char data[];
+};
+
+struct rspamd_worker_signal_handler;
+
+typedef gboolean (*rspamd_worker_signal_cb_t)(
+ struct rspamd_worker_signal_handler *, void *ud);
+
+struct rspamd_worker_signal_handler_elt {
+ rspamd_worker_signal_cb_t handler;
+ void *handler_data;
+ struct rspamd_worker_signal_handler_elt *next, *prev;
+};
+
+struct rspamd_worker_signal_handler {
+ gint signo;
+ gboolean enabled;
+ ev_signal ev_sig;
+ struct ev_loop *event_loop;
+ struct rspamd_worker *worker;
+ struct rspamd_worker_signal_handler_elt *cb;
+};
+
+/**
+ * Common structure representing C module context
+ */
+struct module_s;
+
+struct module_ctx {
+ gint (*filter)(struct rspamd_task *task); /**< pointer to headers process function */
+ struct module_s *mod; /**< module pointer */
+ gboolean enabled; /**< true if module is enabled in configuration */
+};
+
+#ifndef WITH_HYPERSCAN
+#define RSPAMD_FEATURE_HYPERSCAN "0"
+#else
+#define RSPAMD_FEATURE_HYPERSCAN "1"
+#endif
+#ifndef WITH_PCRE2
+#define RSPAMD_FEATURE_PCRE2 "0"
+#else
+#define RSPAMD_FEATURE_PCRE2 "1"
+#endif
+#ifndef WITH_FANN
+#define RSPAMD_FEATURE_FANN "0"
+#else
+#define RSPAMD_FEATURE_FANN "1"
+#endif
+#ifndef WITH_SNOWBALL
+#define RSPAMD_FEATURE_SNOWBALL "0"
+#else
+#define RSPAMD_FEATURE_SNOWBALL "1"
+#endif
+
+#define RSPAMD_CUR_MODULE_VERSION 0x1
+#define RSPAMD_CUR_WORKER_VERSION 0x2
+
+#define RSPAMD_FEATURES \
+ RSPAMD_FEATURE_HYPERSCAN RSPAMD_FEATURE_PCRE2 \
+ RSPAMD_FEATURE_FANN RSPAMD_FEATURE_SNOWBALL
+
+#define RSPAMD_MODULE_VER \
+ RSPAMD_CUR_MODULE_VERSION, /* Module version */ \
+ RSPAMD_VERSION_NUM, /* Rspamd version */ \
+ RSPAMD_FEATURES /* Compilation features */
+
+#define RSPAMD_WORKER_VER \
+ RSPAMD_CUR_WORKER_VERSION, /* Worker version */ \
+ RSPAMD_VERSION_NUM, /* Rspamd version */ \
+ RSPAMD_FEATURES /* Compilation features */ /**
+ * Module
+ */
+typedef struct module_s {
+ const gchar *name;
+
+ int (*module_init_func)(struct rspamd_config *cfg, struct module_ctx **ctx);
+
+ int (*module_config_func)(struct rspamd_config *cfg, bool validate);
+
+ int (*module_reconfig_func)(struct rspamd_config *cfg);
+
+ int (*module_attach_controller_func)(struct module_ctx *ctx,
+ GHashTable *custom_commands);
+
+ guint module_version;
+ guint64 rspamd_version;
+ const gchar *rspamd_features;
+ guint ctx_offset;
+} module_t;
+
+enum rspamd_worker_socket_type {
+ RSPAMD_WORKER_SOCKET_NONE = 0,
+ RSPAMD_WORKER_SOCKET_TCP = (1 << 0),
+ RSPAMD_WORKER_SOCKET_UDP = (1 << 1),
+};
+
+struct rspamd_worker_listen_socket {
+ const rspamd_inet_addr_t *addr;
+ gint fd;
+ enum rspamd_worker_socket_type type;
+ bool is_systemd;
+};
+
+typedef struct worker_s {
+ const gchar *name;
+
+ gpointer (*worker_init_func)(struct rspamd_config *cfg);
+
+ void (*worker_start_func)(struct rspamd_worker *worker);
+
+ int flags;
+ int listen_type;
+ guint worker_version;
+ guint64 rspamd_version;
+ const gchar *rspamd_features;
+} worker_t;
+
+/**
+ * Check if loaded worker is compatible with rspamd
+ * @param cfg
+ * @param wrk
+ * @return
+ */
+gboolean rspamd_check_worker(struct rspamd_config *cfg, worker_t *wrk);
+
+/**
+ * Check if loaded module is compatible with rspamd
+ * @param cfg
+ * @param wrk
+ * @return
+ */
+gboolean rspamd_check_module(struct rspamd_config *cfg, module_t *wrk);
+
+struct pidfh;
+struct rspamd_config;
+struct tokenizer;
+struct rspamd_stat_classifier;
+struct rspamd_classifier_config;
+struct rspamd_mime_part;
+struct rspamd_dns_resolver;
+struct rspamd_task;
+struct rspamd_cryptobox_library_ctx;
+
+#define MAX_AVG_TIME_SLOTS 31
+struct RSPAMD_ALIGNED(64) rspamd_avg_time {
+ guint32 cur_slot;
+ float avg_time[MAX_AVG_TIME_SLOTS];
+};
+/**
+ * Server statistics
+ */
+struct RSPAMD_ALIGNED(64) rspamd_stat {
+ guint messages_scanned; /**< total number of messages scanned */
+ guint actions_stat[METRIC_ACTION_MAX]; /**< statistic for each action */
+ guint connections_count; /**< total connections count */
+ guint control_connections_count; /**< connections count to control interface */
+ guint messages_learned; /**< messages learned */
+ struct rspamd_avg_time avg_time; /**< average time stats */
+};
+
+/**
+ * Struct that determine main server object (for logging purposes)
+ */
+struct rspamd_main {
+ struct rspamd_config *cfg; /**< pointer to config structure */
+ pid_t pid; /**< main pid */
+ /* Pid file structure */
+ rspamd_pidfh_t *pfh; /**< struct pidfh for pidfile */
+ GQuark type; /**< process type */
+ struct rspamd_stat *stat; /**< pointer to statistics */
+
+ rspamd_mempool_t *server_pool; /**< server's memory pool */
+ rspamd_mempool_mutex_t *start_mtx; /**< server is starting up */
+ GHashTable *workers; /**< workers pool indexed by pid */
+ GHashTable *spairs; /**< socket pairs requested by workers */
+ rspamd_logger_t *logger;
+ uid_t workers_uid; /**< worker's uid running to */
+ gid_t workers_gid; /**< worker's gid running to */
+ gboolean is_privileged; /**< true if run in privileged mode */
+ gboolean wanna_die; /**< no respawn of processes */
+ gboolean cores_throttling; /**< turn off cores when limits are exceeded */
+ struct roll_history *history; /**< rolling history */
+ struct ev_loop *event_loop;
+ ev_signal term_ev, int_ev, hup_ev, usr1_ev; /**< signals */
+ struct rspamd_http_context *http_ctx;
+};
+
+/**
+ * Control session object
+ */
+struct controller_command;
+struct controller_session;
+
+typedef gboolean (*controller_func_t)(gchar **args,
+ struct controller_session *session);
+
+struct controller_session {
+ struct rspamd_worker *worker; /**< pointer to worker structure (controller in fact) */
+ gint sock; /**< socket descriptor */
+ struct controller_command *cmd; /**< real command */
+ struct rspamd_config *cfg; /**< pointer to config file */
+ GList *parts; /**< extracted mime parts */
+ struct rspamd_async_session *s; /**< async session object */
+ struct rspamd_dns_resolver *resolver; /**< DNS resolver */
+ struct ev_loop *ev_base; /**< Event base */
+};
+
+struct zstd_dictionary {
+ void *dict;
+ gsize size;
+ guint id;
+};
+
+struct rspamd_external_libs_ctx {
+ void **local_addrs;
+ struct rspamd_cryptobox_library_ctx *crypto_ctx;
+ struct ottery_config *ottery_cfg;
+ void *ssl_ctx;
+ void *ssl_ctx_noverify;
+ struct zstd_dictionary *in_dict;
+ struct zstd_dictionary *out_dict;
+ void *out_zstream;
+ void *in_zstream;
+ ref_entry_t ref;
+};
+
+
+/**
+ * Register custom controller function
+ */
+void register_custom_controller_command(const gchar *name,
+ controller_func_t handler,
+ gboolean privileged,
+ gboolean require_message);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif \ No newline at end of file
diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c
new file mode 100644
index 0000000..838de06
--- /dev/null
+++ b/src/rspamd_proxy.c
@@ -0,0 +1,2441 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "libutil/util.h"
+#include "libserver/maps/map.h"
+#include "libutil/upstream.h"
+#include "libserver/http/http_connection.h"
+#include "libserver/http/http_private.h"
+#include "libserver/protocol.h"
+#include "libserver/protocol_internal.h"
+#include "libserver/cfg_file.h"
+#include "libserver/url.h"
+#include "libserver/dns.h"
+#include "libmime/message.h"
+#include "rspamd.h"
+#include "libserver/worker_util.h"
+#include "worker_private.h"
+#include "lua/lua_common.h"
+#include "keypairs_cache.h"
+#include "libstat/stat_api.h"
+#include "ottery.h"
+#include "unix-std.h"
+#include "libserver/milter.h"
+#include "libserver/milter_internal.h"
+#include "libmime/lang_detection.h"
+
+#include <math.h>
+
+#ifdef HAVE_NETINET_TCP_H
+#include <netinet/tcp.h> /* for TCP_NODELAY */
+#endif
+
+#ifdef SYS_ZSTD
+#include "zstd.h"
+#else
+#include "contrib/zstd/zstd.h"
+#endif
+
+/* Rotate keys each minute by default */
+#define DEFAULT_ROTATION_TIME 60.0
+#define DEFAULT_RETRIES 5
+
+#define msg_err_session(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ session->pool->tag.tagname, session->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_session(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ session->pool->tag.tagname, session->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_session(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ session->pool->tag.tagname, session->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+#define msg_debug_session(...) rspamd_conditional_debug_fast(NULL, session->client_addr, \
+ rspamd_proxy_log_id, "proxy", session->pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(proxy)
+
+gpointer init_rspamd_proxy(struct rspamd_config *cfg);
+void start_rspamd_proxy(struct rspamd_worker *worker);
+
+worker_t rspamd_proxy_worker = {
+ "rspamd_proxy", /* Name */
+ init_rspamd_proxy, /* Init function */
+ start_rspamd_proxy, /* Start function */
+ RSPAMD_WORKER_HAS_SOCKET | RSPAMD_WORKER_KILLABLE | RSPAMD_WORKER_SCANNER,
+ RSPAMD_WORKER_SOCKET_TCP, /* TCP socket */
+ RSPAMD_WORKER_VER};
+
+struct rspamd_http_upstream {
+ gchar *name;
+ gchar *settings_id;
+ struct upstream_list *u;
+ struct rspamd_cryptobox_pubkey *key;
+ gdouble timeout;
+ gint parser_from_ref;
+ gint parser_to_ref;
+ gboolean local;
+ gboolean self_scan;
+ gboolean compress;
+};
+
+struct rspamd_http_mirror {
+ gchar *name;
+ gchar *settings_id;
+ struct upstream_list *u;
+ struct rspamd_cryptobox_pubkey *key;
+ gdouble prob;
+ gdouble timeout;
+ gint parser_from_ref;
+ gint parser_to_ref;
+ gboolean local;
+ gboolean compress;
+};
+
+static const guint64 rspamd_rspamd_proxy_magic = 0xcdeb4fd1fc351980ULL;
+
+struct rspamd_proxy_ctx {
+ guint64 magic;
+ /* Events base */
+ struct ev_loop *event_loop;
+ /* DNS resolver */
+ struct rspamd_dns_resolver *resolver;
+ /* Config */
+ struct rspamd_config *cfg;
+ /* END OF COMMON PART */
+ gdouble timeout;
+ /* Encryption key for clients */
+ struct rspamd_cryptobox_keypair *key;
+ /* HTTP context */
+ struct rspamd_http_context *http_ctx;
+ /* Upstreams to use */
+ GHashTable *upstreams;
+ /* Mirrors to send traffic to */
+ GPtrArray *mirrors;
+ /* Default upstream */
+ struct rspamd_http_upstream *default_upstream;
+ lua_State *lua_state;
+ /* Array of callback functions called on end of scan to compare results */
+ GArray *cmp_refs;
+ /* Maximum count for retries */
+ guint max_retries;
+ gboolean encrypted_only;
+ /* If we have self_scanning backends, we need to work as a normal worker */
+ gboolean has_self_scan;
+ /* It is not HTTP but milter proxy */
+ gboolean milter;
+ /* Discard messages instead of rejecting them */
+ gboolean discard_on_reject;
+ /* Quarantine messages instead of rejecting them */
+ gboolean quarantine_on_reject;
+ /* Milter spam header */
+ gchar *spam_header;
+ /* CA name that can be used for client certificates */
+ gchar *client_ca_name;
+ /* Milter rejection message */
+ gchar *reject_message;
+ /* Sessions cache */
+ void *sessions_cache;
+ struct rspamd_milter_context milter_ctx;
+ /* Language detector */
+ struct rspamd_lang_detector *lang_det;
+ gdouble task_timeout;
+};
+
+enum rspamd_backend_flags {
+ RSPAMD_BACKEND_REPLIED = 1 << 0,
+ RSPAMD_BACKEND_CLOSED = 1 << 1,
+ RSPAMD_BACKEND_PARSED = 1 << 2,
+};
+
+struct rspamd_proxy_session;
+
+struct rspamd_proxy_backend_connection {
+ const gchar *name;
+ struct rspamd_cryptobox_keypair *local_key;
+ struct rspamd_cryptobox_pubkey *remote_key;
+ struct upstream *up;
+ struct rspamd_http_connection *backend_conn;
+ ucl_object_t *results;
+ const gchar *err;
+ struct rspamd_proxy_session *s;
+ gint backend_sock;
+ ev_tstamp timeout;
+ enum rspamd_backend_flags flags;
+ gint parser_from_ref;
+ gint parser_to_ref;
+ struct rspamd_task *task;
+};
+
+enum rspamd_proxy_legacy_support {
+ LEGACY_SUPPORT_NO = 0,
+ LEGACY_SUPPORT_RSPAMC,
+ LEGACY_SUPPORT_SPAMC
+};
+
+struct rspamd_proxy_session {
+ struct rspamd_worker *worker;
+ rspamd_mempool_t *pool;
+ struct rspamd_proxy_ctx *ctx;
+ rspamd_inet_addr_t *client_addr;
+ struct rspamd_http_connection *client_conn;
+ struct rspamd_milter_session *client_milter_conn;
+ struct rspamd_http_upstream *backend;
+ gpointer map;
+ gchar *fname;
+ gpointer shmem_ref;
+ struct rspamd_proxy_backend_connection *master_conn;
+ struct rspamd_http_message *client_message;
+ GPtrArray *mirror_conns;
+ gsize map_len;
+ gint client_sock;
+ enum rspamd_proxy_legacy_support legacy_support;
+ gint retries;
+ ref_entry_t ref;
+};
+
+static gboolean proxy_send_master_message(struct rspamd_proxy_session *session);
+
+static GQuark
+rspamd_proxy_quark(void)
+{
+ return g_quark_from_static_string("rspamd-proxy");
+}
+
+static gboolean
+rspamd_proxy_parse_lua_parser(lua_State *L, const ucl_object_t *obj,
+ gint *ref_from, gint *ref_to, GError **err)
+{
+ const gchar *lua_script;
+ gsize slen;
+ gint err_idx, ref_idx;
+ gboolean has_ref = FALSE;
+
+ g_assert(obj != NULL);
+ g_assert(ref_from != NULL);
+ g_assert(ref_to != NULL);
+
+ *ref_from = -1;
+ *ref_to = -1;
+
+ lua_script = ucl_object_tolstring(obj, &slen);
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ /* Load data */
+ if (luaL_loadbuffer(L, lua_script, slen, "proxy parser") != 0) {
+ g_set_error(err,
+ rspamd_proxy_quark(),
+ EINVAL,
+ "cannot load lua parser script: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0); /* Error function */
+
+ return FALSE;
+ }
+
+ /* Now do it */
+ if (lua_pcall(L, 0, 1, err_idx) != 0) {
+ g_set_error(err,
+ rspamd_proxy_quark(),
+ EINVAL,
+ "cannot init lua parser script: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return FALSE;
+ }
+
+ if (lua_istable(L, -1)) {
+ /*
+ * We have a table, so we check for two keys:
+ * 'from' -> function
+ * 'to' -> function
+ *
+ * From converts parent request to a client one
+ * To converts client request to a parent one
+ */
+ lua_pushstring(L, "from");
+ lua_gettable(L, -2);
+
+ if (lua_isfunction(L, -1)) {
+ ref_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+ *ref_from = ref_idx;
+ has_ref = TRUE;
+ }
+
+ lua_pushstring(L, "to");
+ lua_gettable(L, -2);
+
+ if (lua_isfunction(L, -1)) {
+ ref_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+ *ref_to = ref_idx;
+ has_ref = TRUE;
+ }
+ }
+ else if (!lua_isfunction(L, -1)) {
+ g_set_error(err,
+ rspamd_proxy_quark(),
+ EINVAL,
+ "cannot init lua parser script: "
+ "must return function");
+ lua_settop(L, 0);
+
+ return FALSE;
+ }
+ else {
+ /* Just parser from protocol */
+ ref_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+ *ref_from = ref_idx;
+ lua_settop(L, 0);
+ has_ref = TRUE;
+ }
+
+ return has_ref;
+}
+
+static gboolean
+rspamd_proxy_parse_upstream(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ const ucl_object_t *elt;
+ struct rspamd_http_upstream *up = NULL;
+ struct rspamd_proxy_ctx *ctx;
+ struct rspamd_rcl_struct_parser *pd = ud;
+ lua_State *L;
+
+ ctx = pd->user_struct;
+ L = ctx->lua_state;
+
+ if (ucl_object_type(obj) != UCL_OBJECT) {
+ if (ucl_object_type(obj) != UCL_NULL) {
+ msg_info_pool_check("upstream %s is disabled by setting it to NULL",
+ ucl_object_key(obj));
+ return TRUE;
+ }
+
+ g_set_error(err, rspamd_proxy_quark(), 100,
+ "upstream option must be an object");
+
+ return FALSE;
+ }
+
+ if (!rspamd_config_is_enabled_from_ucl(pool, obj)) {
+ /* Upstream is valid but disabled */
+ msg_info_pool_check("upstream %s is disabled",
+ ucl_object_lookup(obj, "name") ? ucl_object_tostring(ucl_object_lookup(obj, "name")) : ucl_object_key(obj));
+ return TRUE;
+ }
+
+ up = rspamd_mempool_alloc0(pool, sizeof(*up));
+
+ elt = ucl_object_lookup(obj, "name");
+ if (elt == NULL) {
+
+ if (ucl_object_key(obj)) {
+ if (strcmp(ucl_object_key(obj), "upstream") == 0) {
+ /* Iterate over the object and find upstream elements */
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur;
+ gboolean ret = TRUE;
+
+ while ((cur = ucl_object_iterate(obj, &it, true)) != NULL) {
+ if (!rspamd_proxy_parse_upstream(pool, cur, ud,
+ section, err)) {
+ ret = FALSE;
+ }
+ }
+
+ return ret;
+ }
+ else {
+ /* Inside upstream */
+ up->name = rspamd_mempool_strdup(pool, ucl_object_key(obj));
+ }
+ }
+ else {
+ g_set_error(err, rspamd_proxy_quark(), 100,
+ "upstream option must have some name definition");
+
+ return FALSE;
+ }
+ }
+ else {
+ up->name = rspamd_mempool_strdup(pool, ucl_object_tostring(elt));
+ }
+
+ up->parser_from_ref = -1;
+ up->parser_to_ref = -1;
+ up->timeout = ctx->timeout;
+
+ elt = ucl_object_lookup(obj, "key");
+ if (elt != NULL) {
+ up->key = rspamd_pubkey_from_base32(ucl_object_tostring(elt), 0,
+ RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
+
+ if (up->key == NULL) {
+ g_set_error(err, rspamd_proxy_quark(), 100,
+ "cannot read upstream key");
+
+ goto err;
+ }
+
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) rspamd_pubkey_unref, up->key);
+ }
+
+ elt = ucl_object_lookup(obj, "self_scan");
+ if (elt && ucl_object_toboolean(elt)) {
+ up->self_scan = TRUE;
+ ctx->has_self_scan = TRUE;
+ }
+
+ elt = ucl_object_lookup_any(obj, "compress", "compression", NULL);
+ if (elt && ucl_object_toboolean(elt)) {
+ up->compress = TRUE;
+ }
+
+ elt = ucl_object_lookup(obj, "hosts");
+
+ if (elt == NULL && !up->self_scan) {
+ g_set_error(err, rspamd_proxy_quark(), 100,
+ "upstream option must have some hosts definition");
+
+ goto err;
+ }
+
+ if (elt) {
+ up->u = rspamd_upstreams_create(ctx->cfg->ups_ctx);
+
+ if (!rspamd_upstreams_from_ucl(up->u, elt, 11333, NULL)) {
+ g_set_error(err, rspamd_proxy_quark(), 100,
+ "upstream has bad hosts definition");
+
+ goto err;
+ }
+
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) rspamd_upstreams_destroy, up->u);
+ }
+
+ elt = ucl_object_lookup(obj, "default");
+ if (elt) {
+ if (ucl_object_toboolean(elt)) {
+ ctx->default_upstream = up;
+ }
+ }
+ else if (up->self_scan) {
+ ctx->default_upstream = up;
+ }
+
+
+ elt = ucl_object_lookup(obj, "local");
+ if (elt && ucl_object_toboolean(elt)) {
+ up->local = TRUE;
+ }
+
+ elt = ucl_object_lookup(obj, "timeout");
+ if (elt) {
+ ucl_object_todouble_safe(elt, &up->timeout);
+ }
+
+ elt = ucl_object_lookup_any(obj, "settings", "settings_id", NULL);
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ up->settings_id = rspamd_mempool_strdup(pool, ucl_object_tostring(elt));
+ }
+
+ /*
+ * Accept lua function here in form
+ * fun :: String -> UCL
+ */
+ elt = ucl_object_lookup(obj, "parser");
+ if (elt) {
+ if (!rspamd_proxy_parse_lua_parser(L, elt, &up->parser_from_ref,
+ &up->parser_to_ref, err)) {
+ goto err;
+ }
+
+ rspamd_lua_add_ref_dtor(L, pool, up->parser_from_ref);
+ rspamd_lua_add_ref_dtor(L, pool, up->parser_to_ref);
+ }
+
+ g_hash_table_insert(ctx->upstreams, up->name, up);
+
+ return TRUE;
+
+err:
+ return FALSE;
+}
+
+static gboolean
+rspamd_proxy_parse_mirror(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ const ucl_object_t *elt;
+ struct rspamd_http_mirror *up = NULL;
+ struct rspamd_proxy_ctx *ctx;
+ struct rspamd_rcl_struct_parser *pd = ud;
+ lua_State *L;
+
+ ctx = pd->user_struct;
+ L = ctx->lua_state;
+
+ if (ucl_object_type(obj) != UCL_OBJECT) {
+ if (ucl_object_type(obj) != UCL_NULL) {
+ msg_info_pool_check("mirror %s is disabled by setting it to NULL",
+ ucl_object_key(obj));
+ return TRUE;
+ }
+
+ g_set_error(err, rspamd_proxy_quark(), 100,
+ "mirror option must be an object");
+
+ return FALSE;
+ }
+
+ if (!rspamd_config_is_enabled_from_ucl(pool, obj)) {
+ /* Upstream is valid but disabled */
+ msg_info_pool_check("mirror %s is disabled",
+ ucl_object_lookup(obj, "name") ? ucl_object_tostring(ucl_object_lookup(obj, "name")) : ucl_object_key(obj));
+ return TRUE;
+ }
+
+ up = rspamd_mempool_alloc0(pool, sizeof(*up));
+
+ elt = ucl_object_lookup(obj, "name");
+ if (elt == NULL) {
+
+ if (ucl_object_key(obj)) {
+ if (strcmp(ucl_object_key(obj), "mirror") == 0) {
+ /* Iterate over the object and find upstream elements */
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur;
+ gboolean ret = TRUE;
+
+ while ((cur = ucl_object_iterate(obj, &it, true)) != NULL) {
+ if (!rspamd_proxy_parse_mirror(pool, cur, ud,
+ section, err)) {
+ ret = FALSE;
+ }
+ }
+
+ return ret;
+ }
+ else {
+ /* Inside upstream */
+ up->name = rspamd_mempool_strdup(pool, ucl_object_key(obj));
+ }
+ }
+ else {
+ g_set_error(err, rspamd_proxy_quark(), 100,
+ "mirror option must have some name definition");
+
+ return FALSE;
+ }
+ }
+ else {
+ up->name = rspamd_mempool_strdup(pool, ucl_object_tostring(elt));
+ }
+
+ up->parser_to_ref = -1;
+ up->parser_from_ref = -1;
+ up->timeout = ctx->timeout;
+
+ elt = ucl_object_lookup(obj, "key");
+ if (elt != NULL) {
+ up->key = rspamd_pubkey_from_base32(ucl_object_tostring(elt), 0,
+ RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
+
+ if (up->key == NULL) {
+ g_set_error(err, rspamd_proxy_quark(), 100,
+ "cannot read mirror key");
+
+ goto err;
+ }
+
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) rspamd_pubkey_unref, up->key);
+ }
+
+ elt = ucl_object_lookup(obj, "hosts");
+
+ if (elt == NULL) {
+ g_set_error(err, rspamd_proxy_quark(), 100,
+ "mirror option must have some hosts definition");
+
+ goto err;
+ }
+
+ up->u = rspamd_upstreams_create(ctx->cfg->ups_ctx);
+ if (!rspamd_upstreams_from_ucl(up->u, elt, 11333, NULL)) {
+ g_set_error(err, rspamd_proxy_quark(), 100,
+ "mirror has bad hosts definition");
+
+ goto err;
+ }
+
+ rspamd_mempool_add_destructor(pool,
+ (rspamd_mempool_destruct_t) rspamd_upstreams_destroy, up->u);
+
+ elt = ucl_object_lookup_any(obj, "probability", "prob", NULL);
+ if (elt) {
+ up->prob = ucl_object_todouble(elt);
+ }
+ else {
+ up->prob = 1.0;
+ }
+
+ elt = ucl_object_lookup(obj, "local");
+ if (elt && ucl_object_toboolean(elt)) {
+ up->local = TRUE;
+ }
+
+ elt = ucl_object_lookup_any(obj, "compress", "compression", NULL);
+ if (elt && ucl_object_toboolean(elt)) {
+ up->compress = TRUE;
+ }
+
+ elt = ucl_object_lookup(obj, "timeout");
+ if (elt) {
+ ucl_object_todouble_safe(elt, &up->timeout);
+ }
+
+ /*
+ * Accept lua function here in form
+ * fun :: String -> UCL
+ */
+ elt = ucl_object_lookup(obj, "parser");
+ if (elt) {
+ if (!rspamd_proxy_parse_lua_parser(L, elt, &up->parser_from_ref,
+ &up->parser_to_ref, err)) {
+ goto err;
+ }
+
+ rspamd_lua_add_ref_dtor(L, pool, up->parser_from_ref);
+ rspamd_lua_add_ref_dtor(L, pool, up->parser_to_ref);
+ }
+
+ elt = ucl_object_lookup_any(obj, "settings", "settings_id", NULL);
+ if (elt && ucl_object_type(elt) == UCL_STRING) {
+ up->settings_id = rspamd_mempool_strdup(pool, ucl_object_tostring(elt));
+ }
+
+ g_ptr_array_add(ctx->mirrors, up);
+
+ return TRUE;
+
+err:
+
+ return FALSE;
+}
+
+static gboolean
+rspamd_proxy_parse_script(rspamd_mempool_t *pool,
+ const ucl_object_t *obj,
+ gpointer ud,
+ struct rspamd_rcl_section *section,
+ GError **err)
+{
+ struct rspamd_proxy_ctx *ctx;
+ struct rspamd_rcl_struct_parser *pd = ud;
+ lua_State *L;
+ const gchar *lua_script;
+ gsize slen;
+ gint err_idx, ref_idx;
+ struct stat st;
+
+ ctx = pd->user_struct;
+ L = ctx->lua_state;
+
+ if (ucl_object_type(obj) != UCL_STRING) {
+ g_set_error(err, rspamd_proxy_quark(), 100,
+ "script option must be a string with file or lua chunk");
+
+ return FALSE;
+ }
+
+ lua_script = ucl_object_tolstring(obj, &slen);
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ if (stat(lua_script, &st) != -1) {
+ /* Load file */
+ if (luaL_loadfile(L, lua_script) != 0) {
+ g_set_error(err,
+ rspamd_proxy_quark(),
+ EINVAL,
+ "cannot load lua parser script: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0); /* Error function */
+
+ goto err;
+ }
+ }
+ else {
+ /* Load data directly */
+ if (luaL_loadbuffer(L, lua_script, slen, "proxy parser") != 0) {
+ g_set_error(err,
+ rspamd_proxy_quark(),
+ EINVAL,
+ "cannot load lua parser script: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0); /* Error function */
+
+ goto err;
+ }
+ }
+
+ /* Now do it */
+ if (lua_pcall(L, 0, 1, err_idx) != 0) {
+ g_set_error(err,
+ rspamd_proxy_quark(),
+ EINVAL,
+ "cannot init lua parser script: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ goto err;
+ }
+
+ if (!lua_isfunction(L, -1)) {
+ g_set_error(err,
+ rspamd_proxy_quark(),
+ EINVAL,
+ "cannot init lua parser script: "
+ "must return function, %s returned",
+ lua_typename(L, lua_type(L, -1)));
+ lua_settop(L, 0);
+
+ goto err;
+ }
+
+ ref_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+ lua_settop(L, 0);
+ g_array_append_val(ctx->cmp_refs, ref_idx);
+
+ return TRUE;
+
+err:
+ return FALSE;
+}
+
+gpointer
+init_rspamd_proxy(struct rspamd_config *cfg)
+{
+ struct rspamd_proxy_ctx *ctx;
+ GQuark type;
+
+ type = g_quark_try_string("rspamd_proxy");
+
+ ctx = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(struct rspamd_proxy_ctx));
+ ctx->magic = rspamd_rspamd_proxy_magic;
+ ctx->timeout = 120.0;
+ ctx->upstreams = g_hash_table_new(rspamd_strcase_hash, rspamd_strcase_equal);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) g_hash_table_unref, ctx->upstreams);
+ ctx->mirrors = g_ptr_array_new();
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_ptr_array_free_hard, ctx->mirrors);
+ ctx->cfg = cfg;
+ ctx->lua_state = cfg->lua_state;
+ ctx->cmp_refs = g_array_new(FALSE, FALSE, sizeof(gint));
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_array_free_hard, ctx->cmp_refs);
+ ctx->max_retries = DEFAULT_RETRIES;
+ ctx->spam_header = RSPAMD_MILTER_SPAM_HEADER;
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "timeout",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_proxy_ctx, timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "IO timeout");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "keypair",
+ rspamd_rcl_parse_struct_keypair,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_proxy_ctx, key),
+ 0,
+ "Server's keypair");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "encrypted_only",
+ rspamd_rcl_parse_struct_boolean,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_proxy_ctx, encrypted_only),
+ 0,
+ "Allow only encrypted connections");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "upstream",
+ rspamd_proxy_parse_upstream,
+ ctx,
+ 0,
+ 0,
+ "List of upstreams");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "mirror",
+ rspamd_proxy_parse_mirror,
+ ctx,
+ 0,
+ RSPAMD_CL_FLAG_MULTIPLE,
+ "List of mirrors");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "script",
+ rspamd_proxy_parse_script,
+ ctx,
+ 0,
+ RSPAMD_CL_FLAG_MULTIPLE,
+ "Compare script to be executed");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "max_retries",
+ rspamd_rcl_parse_struct_integer,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_proxy_ctx, max_retries),
+ RSPAMD_CL_FLAG_UINT,
+ "Maximum number of retries for master connection");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "milter",
+ rspamd_rcl_parse_struct_boolean,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_proxy_ctx, milter),
+ 0,
+ "Accept milter connections, not HTTP");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "discard_on_reject",
+ rspamd_rcl_parse_struct_boolean,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_proxy_ctx, discard_on_reject),
+ 0,
+ "Tell MTA to discard rejected messages silently");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "quarantine_on_reject",
+ rspamd_rcl_parse_struct_boolean,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_proxy_ctx, quarantine_on_reject),
+ 0,
+ "Tell MTA to quarantine rejected messages");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "spam_header",
+ rspamd_rcl_parse_struct_string,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_proxy_ctx, spam_header),
+ 0,
+ "Use the specific spam header (default: X-Spam)");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "client_ca_name",
+ rspamd_rcl_parse_struct_string,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_proxy_ctx, client_ca_name),
+ 0,
+ "Allow certificates issued by this CA to be treated as client certificates");
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "reject_message",
+ rspamd_rcl_parse_struct_string,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_proxy_ctx, reject_message),
+ 0,
+ "Use custom rejection message");
+
+ return ctx;
+}
+
+static void
+proxy_backend_close_connection(struct rspamd_proxy_backend_connection *conn)
+{
+ if (conn && !(conn->flags & RSPAMD_BACKEND_CLOSED)) {
+ if (conn->backend_conn) {
+ rspamd_http_connection_reset(conn->backend_conn);
+ rspamd_http_connection_unref(conn->backend_conn);
+ close(conn->backend_sock);
+ }
+
+ conn->flags |= RSPAMD_BACKEND_CLOSED;
+ }
+}
+
+static gboolean
+proxy_backend_parse_results(struct rspamd_proxy_session *session,
+ struct rspamd_proxy_backend_connection *conn,
+ lua_State *L, gint parser_ref,
+ struct rspamd_http_message *msg,
+ goffset *body_offset,
+ const rspamd_ftok_t *ct)
+{
+ struct ucl_parser *parser;
+ gint err_idx;
+ const gchar *in = msg->body_buf.begin;
+ gsize inlen = msg->body_buf.len;
+ const rspamd_ftok_t *offset_hdr;
+
+ if (inlen == 0 || in == NULL) {
+ return FALSE;
+ }
+
+ offset_hdr = rspamd_http_message_find_header(msg, MESSAGE_OFFSET_HEADER);
+
+ if (offset_hdr) {
+ gulong val;
+
+ if (rspamd_strtoul(offset_hdr->begin, offset_hdr->len, &val) && val < inlen) {
+
+ if (body_offset) {
+ *body_offset = val;
+ }
+ inlen = val;
+ }
+ }
+
+ if (parser_ref != -1) {
+ /* Call parser function */
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, parser_ref);
+ /* XXX: copies all data */
+ lua_pushlstring(L, in, inlen);
+
+ if (lua_pcall(L, 1, 1, err_idx) != 0) {
+ msg_err_session(
+ "cannot run lua parser script: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return FALSE;
+ }
+
+ conn->results = ucl_object_lua_import(L, -1);
+ lua_settop(L, 0);
+ }
+ else {
+ rspamd_ftok_t json_ct;
+ RSPAMD_FTOK_ASSIGN(&json_ct, "application/json");
+
+ if (ct && rspamd_ftok_casecmp(ct, &json_ct) == 0) {
+ parser = ucl_parser_new(0);
+
+ if (!ucl_parser_add_chunk(parser, in, inlen)) {
+ gchar *encoded;
+
+ encoded = rspamd_encode_base64(in, inlen, 0, NULL);
+ msg_err_session("cannot parse input: %s", ucl_parser_get_error(
+ parser));
+ msg_err_session("input encoded: %s", encoded);
+ ucl_parser_free(parser);
+ g_free(encoded);
+
+ return FALSE;
+ }
+
+ conn->results = ucl_parser_get_object(parser);
+ ucl_parser_free(parser);
+ }
+ }
+
+ return TRUE;
+}
+
+static void
+proxy_call_cmp_script(struct rspamd_proxy_session *session, gint cbref)
+{
+ gint err_idx;
+ guint i;
+ struct rspamd_proxy_backend_connection *conn;
+ lua_State *L;
+
+ L = session->ctx->lua_state;
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbref);
+
+ lua_createtable(L, 0, session->mirror_conns->len + 1);
+ /* Now push master results */
+ if (session->master_conn && session->master_conn->results) {
+ lua_pushstring(L, "master");
+ ucl_object_push_lua(L, session->master_conn->results, true);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "master");
+ lua_pushstring(L, "no results");
+ lua_settable(L, -3);
+ }
+
+ for (i = 0; i < session->mirror_conns->len; i++) {
+ conn = g_ptr_array_index(session->mirror_conns, i);
+
+ if (conn->results) {
+ lua_pushstring(L, conn->name);
+ ucl_object_push_lua(L, conn->results, true);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, conn->name);
+ lua_pushstring(L, conn->err ? conn->err : "unknown error");
+ lua_settable(L, -3);
+ }
+ }
+
+ gchar log_tag[RSPAMD_LOG_ID_LEN + 1];
+ rspamd_strlcpy(log_tag, session->pool->tag.uid, sizeof(log_tag));
+ lua_pushstring(L, log_tag);
+
+ if (lua_pcall(L, 2, 0, err_idx) != 0) {
+ msg_err_session(
+ "cannot run lua compare script: %s",
+ lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+}
+
+static void
+proxy_session_dtor(struct rspamd_proxy_session *session)
+{
+ guint i;
+ gint cbref;
+ struct rspamd_proxy_backend_connection *conn;
+
+ if (session->master_conn && session->master_conn->results) {
+ for (i = 0; i < session->ctx->cmp_refs->len; i++) {
+ cbref = g_array_index(session->ctx->cmp_refs, gint, i);
+ proxy_call_cmp_script(session, cbref);
+ }
+ }
+
+ if (session->master_conn) {
+ proxy_backend_close_connection(session->master_conn);
+ }
+
+ if (session->client_milter_conn) {
+ rspamd_milter_session_unref(session->client_milter_conn);
+ }
+ else if (session->client_conn) {
+ rspamd_http_connection_reset(session->client_conn);
+ rspamd_http_connection_unref(session->client_conn);
+ }
+
+ if (session->map && session->map_len) {
+ munmap(session->map, session->map_len);
+ }
+
+ for (i = 0; i < session->mirror_conns->len; i++) {
+ conn = g_ptr_array_index(session->mirror_conns, i);
+
+ if (!(conn->flags & RSPAMD_BACKEND_CLOSED)) {
+ proxy_backend_close_connection(conn);
+ }
+
+ if (conn->results) {
+ ucl_object_unref(conn->results);
+ }
+ }
+
+ if (session->master_conn) {
+ if (session->master_conn->results) {
+ ucl_object_unref(session->master_conn->results);
+ }
+
+ if (session->master_conn->task) {
+ rspamd_session_destroy(session->master_conn->task->s);
+ }
+ }
+
+ g_ptr_array_free(session->mirror_conns, TRUE);
+ rspamd_http_message_shmem_unref(session->shmem_ref);
+ rspamd_http_message_unref(session->client_message);
+
+ if (session->client_addr) {
+ rspamd_inet_address_free(session->client_addr);
+ }
+
+ if (session->client_sock != -1) {
+ close(session->client_sock);
+ }
+
+ if (session->ctx->sessions_cache) {
+ rspamd_worker_session_cache_remove(session->ctx->sessions_cache,
+ session);
+ }
+
+ if (session->pool) {
+ rspamd_mempool_delete(session->pool);
+ }
+
+ g_free(session);
+}
+
+static void
+proxy_request_compress(struct rspamd_http_message *msg)
+{
+ guint flags;
+ ZSTD_CCtx *zctx;
+ rspamd_fstring_t *body;
+ const gchar *in;
+ gsize inlen;
+
+ flags = rspamd_http_message_get_flags(msg);
+
+ if (!rspamd_http_message_find_header(msg, COMPRESSION_HEADER)) {
+ if ((flags & RSPAMD_HTTP_FLAG_SHMEM) ||
+ !(flags & RSPAMD_HTTP_FLAG_HAS_BODY)) {
+ /* Cannot compress shared or empty message */
+ return;
+ }
+
+ in = rspamd_http_message_get_body(msg, &inlen);
+
+ if (in == NULL || inlen == 0) {
+ return;
+ }
+
+ body = rspamd_fstring_sized_new(ZSTD_compressBound(inlen));
+ zctx = ZSTD_createCCtx();
+ body->len = ZSTD_compressCCtx(zctx, body->str, body->allocated,
+ in, inlen, 1);
+
+ if (ZSTD_isError(body->len)) {
+ msg_err("compression error");
+ rspamd_fstring_free(body);
+ ZSTD_freeCCtx(zctx);
+
+ return;
+ }
+
+ ZSTD_freeCCtx(zctx);
+ rspamd_http_message_set_body_from_fstring_steal(msg, body);
+ rspamd_http_message_add_header(msg, COMPRESSION_HEADER, "zstd");
+ }
+}
+
+static void
+proxy_request_decompress(struct rspamd_http_message *msg)
+{
+ rspamd_fstring_t *body;
+ const gchar *in;
+ gsize inlen, outlen, r;
+ ZSTD_DStream *zstream;
+ ZSTD_inBuffer zin;
+ ZSTD_outBuffer zout;
+
+ if (rspamd_http_message_find_header(msg, COMPRESSION_HEADER)) {
+ in = rspamd_http_message_get_body(msg, &inlen);
+
+ if (in == NULL || inlen == 0) {
+ return;
+ }
+
+ zstream = ZSTD_createDStream();
+ ZSTD_initDStream(zstream);
+
+ zin.pos = 0;
+ zin.src = in;
+ zin.size = inlen;
+
+ if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
+ outlen = ZSTD_DStreamOutSize();
+ }
+
+ body = rspamd_fstring_sized_new(outlen);
+ zout.dst = body->str;
+ zout.pos = 0;
+ zout.size = outlen;
+
+ while (zin.pos < zin.size) {
+ r = ZSTD_decompressStream(zstream, &zout, &zin);
+
+ if (ZSTD_isError(r)) {
+ msg_err("Decompression error: %s", ZSTD_getErrorName(r));
+ ZSTD_freeDStream(zstream);
+ rspamd_fstring_free(body);
+
+ return;
+ }
+
+ if (zout.pos == zout.size) {
+ /* We need to extend output buffer */
+ zout.size = zout.size * 2 + 1;
+ body = rspamd_fstring_grow(body, zout.size);
+ zout.size = body->allocated;
+ zout.dst = body->str;
+ }
+ }
+
+ body->len = zout.pos;
+ ZSTD_freeDStream(zstream);
+ rspamd_http_message_set_body_from_fstring_steal(msg, body);
+ rspamd_http_message_remove_header(msg, COMPRESSION_HEADER);
+ }
+}
+
+static struct rspamd_proxy_session *
+proxy_session_refresh(struct rspamd_proxy_session *session)
+{
+ struct rspamd_proxy_session *nsession;
+
+ nsession = g_malloc0(sizeof(*nsession));
+ nsession->client_milter_conn = session->client_milter_conn;
+ session->client_milter_conn = NULL;
+ rspamd_milter_update_userdata(nsession->client_milter_conn,
+ nsession);
+ nsession->client_addr = session->client_addr;
+ session->client_addr = NULL;
+ nsession->ctx = session->ctx;
+ nsession->worker = session->worker;
+ nsession->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), "proxy", 0);
+ nsession->client_sock = session->client_sock;
+ session->client_sock = -1;
+ nsession->mirror_conns = g_ptr_array_sized_new(nsession->ctx->mirrors->len);
+
+ REF_INIT_RETAIN(nsession, proxy_session_dtor);
+
+ if (nsession->ctx->sessions_cache) {
+ rspamd_worker_session_cache_add(nsession->ctx->sessions_cache,
+ nsession->pool->tag.uid, &nsession->ref.refcount, nsession);
+ }
+
+ return nsession;
+}
+
+static gboolean
+proxy_check_file(struct rspamd_http_message *msg,
+ struct rspamd_proxy_session *session)
+{
+ const rspamd_ftok_t *tok, *key_tok;
+ rspamd_ftok_t srch;
+ gchar *file_str;
+ GHashTable *query_args;
+ GHashTableIter it;
+ gpointer k, v;
+ struct http_parser_url u;
+ rspamd_fstring_t *new_url;
+
+ tok = rspamd_http_message_find_header(msg, "File");
+
+ if (tok) {
+ file_str = rspamd_mempool_ftokdup(session->pool, tok);
+ session->map = rspamd_file_xmap(file_str, PROT_READ, &session->map_len,
+ TRUE);
+
+ if (session->map == NULL) {
+ if (session->map_len != 0) {
+ msg_err_session("cannot map %s: %s", file_str,
+ strerror(errno));
+
+ return FALSE;
+ }
+ }
+ /* Remove header after processing */
+ rspamd_http_message_remove_header(msg, "File");
+ session->fname = file_str;
+ }
+ else {
+ /* Need to parse query URL */
+ if (http_parser_parse_url(RSPAMD_FSTRING_DATA(msg->url),
+ RSPAMD_FSTRING_LEN(msg->url), 0, &u) != 0) {
+ msg_err_session("bad request url: %V", msg->url);
+
+ return FALSE;
+ }
+
+ if (u.field_set & (1 << UF_QUERY)) {
+ /* In case if we have a query, we need to store it somewhere */
+ query_args = rspamd_http_message_parse_query(msg);
+ srch.begin = "File";
+ srch.len = strlen("File");
+ tok = g_hash_table_lookup(query_args, &srch);
+
+ if (tok) {
+ file_str = rspamd_mempool_ftokdup(session->pool, tok);
+ session->map = rspamd_file_xmap(file_str, PROT_READ,
+ &session->map_len, TRUE);
+
+ if (session->map == NULL) {
+ if (session->map_len != 0) {
+ msg_err_session("cannot map %s: %s", file_str,
+ strerror(errno));
+ g_hash_table_unref(query_args);
+
+ return FALSE;
+ }
+ }
+
+ /* We need to create a new URL with file attribute removed */
+ new_url = rspamd_fstring_new_init(RSPAMD_FSTRING_DATA(msg->url),
+ u.field_data[UF_QUERY].off);
+ new_url = rspamd_fstring_append(new_url, "?", 1);
+
+ g_hash_table_iter_init(&it, query_args);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ key_tok = k;
+ tok = v;
+
+ if (!rspamd_ftok_icase_equal(key_tok, &srch)) {
+ rspamd_printf_fstring(&new_url, "%T=%T&",
+ key_tok, tok);
+ }
+ }
+
+ /* Erase last character (might be either & or ?) */
+ rspamd_fstring_erase(new_url, new_url->len - 1, 1);
+
+ rspamd_fstring_free(msg->url);
+ msg->url = new_url;
+ session->fname = file_str;
+ }
+
+ g_hash_table_unref(query_args);
+ }
+ }
+
+ return TRUE;
+}
+
+static void
+proxy_backend_mirror_error_handler(struct rspamd_http_connection *conn, GError *err)
+{
+ struct rspamd_proxy_backend_connection *bk_conn = conn->ud;
+ struct rspamd_proxy_session *session;
+
+ session = bk_conn->s;
+ msg_info_session("abnormally closing connection from backend: %s:%s, "
+ "error: %e",
+ bk_conn->name,
+ rspamd_inet_address_to_string_pretty(
+ rspamd_upstream_addr_cur(bk_conn->up)),
+ err);
+
+ if (err) {
+ bk_conn->err = rspamd_mempool_strdup(session->pool, err->message);
+ }
+
+ rspamd_upstream_fail(bk_conn->up, FALSE, err ? err->message : "unknown");
+
+ proxy_backend_close_connection(bk_conn);
+ REF_RELEASE(bk_conn->s);
+}
+
+static gint
+proxy_backend_mirror_finish_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_proxy_backend_connection *bk_conn = conn->ud;
+ struct rspamd_proxy_session *session;
+ const rspamd_ftok_t *orig_ct;
+
+ session = bk_conn->s;
+
+ proxy_request_decompress(msg);
+ orig_ct = rspamd_http_message_find_header(msg, "Content-Type");
+
+ if (!proxy_backend_parse_results(session, bk_conn, session->ctx->lua_state,
+ bk_conn->parser_from_ref, msg, NULL, orig_ct)) {
+ msg_warn_session("cannot parse results from the mirror backend %s:%s",
+ bk_conn->name,
+ rspamd_inet_address_to_string(
+ rspamd_upstream_addr_cur(bk_conn->up)));
+ bk_conn->err = "cannot parse ucl";
+ }
+
+ msg_info_session("finished mirror connection to %s", bk_conn->name);
+ rspamd_upstream_ok(bk_conn->up);
+
+ proxy_backend_close_connection(bk_conn);
+ REF_RELEASE(bk_conn->s);
+
+ return 0;
+}
+
+static void
+proxy_open_mirror_connections(struct rspamd_proxy_session *session)
+{
+ gdouble coin;
+ struct rspamd_http_mirror *m;
+ guint i;
+ struct rspamd_proxy_backend_connection *bk_conn;
+ struct rspamd_http_message *msg;
+ GError *err = NULL;
+
+ coin = rspamd_random_double();
+
+ for (i = 0; i < session->ctx->mirrors->len; i++) {
+ m = g_ptr_array_index(session->ctx->mirrors, i);
+
+ if (m->prob < coin) {
+ /* No luck */
+ continue;
+ }
+
+ bk_conn = rspamd_mempool_alloc0(session->pool,
+ sizeof(*bk_conn));
+ bk_conn->s = session;
+ bk_conn->name = m->name;
+ bk_conn->timeout = m->timeout;
+
+ bk_conn->up = rspamd_upstream_get(m->u,
+ RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0);
+ bk_conn->parser_from_ref = m->parser_from_ref;
+ bk_conn->parser_to_ref = m->parser_to_ref;
+
+ if (bk_conn->up == NULL) {
+ msg_err_session("cannot select upstream for %s", m->name);
+ continue;
+ }
+
+ bk_conn->backend_sock = rspamd_inet_address_connect(
+ rspamd_upstream_addr_next(bk_conn->up),
+ SOCK_STREAM, TRUE);
+
+ if (bk_conn->backend_sock == -1) {
+ msg_err_session("cannot connect upstream for %s", m->name);
+ rspamd_upstream_fail(bk_conn->up, TRUE, strerror(errno));
+ continue;
+ }
+
+ msg = rspamd_http_connection_copy_msg(session->client_message, &err);
+
+ if (msg == NULL) {
+ msg_err_session("cannot copy message to send to a mirror %s: %e",
+ m->name, err);
+ if (err) {
+ g_error_free(err);
+ }
+ continue;
+ }
+
+ if (msg->url->len == 0) {
+ msg->url = rspamd_fstring_append(msg->url, "/check", strlen("/check"));
+ }
+
+ if (m->settings_id != NULL) {
+ rspamd_http_message_remove_header(msg, "Settings-ID");
+ rspamd_http_message_add_header(msg, "Settings-ID", m->settings_id);
+ }
+
+ bk_conn->backend_conn = rspamd_http_connection_new_client_socket(
+ session->ctx->http_ctx,
+ NULL,
+ proxy_backend_mirror_error_handler,
+ proxy_backend_mirror_finish_handler,
+ RSPAMD_HTTP_CLIENT_SIMPLE,
+ bk_conn->backend_sock);
+
+ if (m->key) {
+ msg->peer_key = rspamd_pubkey_ref(m->key);
+ }
+
+ if (m->local ||
+ rspamd_inet_address_is_local(rspamd_upstream_addr_cur(bk_conn->up))) {
+
+ if (session->fname) {
+ rspamd_http_message_add_header(msg, "File", session->fname);
+ }
+
+ msg->method = HTTP_GET;
+ rspamd_http_connection_write_message_shared(bk_conn->backend_conn,
+ msg, rspamd_upstream_name(bk_conn->up), NULL, bk_conn,
+ bk_conn->timeout);
+ }
+ else {
+ if (session->fname) {
+ msg->flags &= ~RSPAMD_HTTP_FLAG_SHMEM;
+ rspamd_http_message_set_body(msg, session->map, session->map_len);
+ }
+
+ msg->method = HTTP_POST;
+
+ if (m->compress) {
+ proxy_request_compress(msg);
+
+ if (session->client_milter_conn) {
+ rspamd_http_message_add_header(msg, "Content-Type",
+ "application/octet-stream");
+ }
+ }
+ else {
+ if (session->client_milter_conn) {
+ rspamd_http_message_add_header(msg, "Content-Type",
+ "text/plain");
+ }
+ }
+
+ rspamd_http_connection_write_message(bk_conn->backend_conn,
+ msg, rspamd_upstream_name(bk_conn->up), NULL, bk_conn,
+ bk_conn->timeout);
+ }
+
+ g_ptr_array_add(session->mirror_conns, bk_conn);
+ REF_RETAIN(session);
+ msg_info_session("send request to %s", m->name);
+ }
+}
+
+static void
+proxy_client_write_error(struct rspamd_proxy_session *session, gint code,
+ const gchar *status)
+{
+ struct rspamd_http_message *reply;
+
+ if (session->client_milter_conn) {
+ rspamd_milter_send_action(session->client_milter_conn,
+ RSPAMD_MILTER_TEMPFAIL);
+ REF_RELEASE(session);
+ }
+ else {
+ reply = rspamd_http_new_message(HTTP_RESPONSE);
+
+ switch (code) {
+ case ETIMEDOUT:
+ reply->code = 504;
+ reply->status = RSPAMD_FSTRING_LIT("Gateway timeout");
+ break;
+ case ECONNRESET:
+ case ECONNABORTED:
+ reply->code = 502;
+ reply->status = RSPAMD_FSTRING_LIT("Gateway connection reset");
+ break;
+ case ECONNREFUSED:
+ reply->code = 502;
+ reply->status = RSPAMD_FSTRING_LIT("Gateway connection refused");
+ break;
+ default:
+ if (code >= 300) {
+ /* Likely HTTP error */
+ reply->code = code;
+ reply->status = rspamd_fstring_new_init(status, strlen(status));
+ }
+ else {
+ reply->code = 502;
+ reply->status = RSPAMD_FSTRING_LIT("Unknown gateway error: ");
+ reply->status = rspamd_fstring_append(reply->status,
+ status, strlen(status));
+ }
+ break;
+ }
+
+ rspamd_http_connection_write_message(session->client_conn,
+ reply, NULL, NULL, session,
+ session->ctx->timeout);
+ }
+}
+
+static void
+proxy_backend_master_error_handler(struct rspamd_http_connection *conn, GError *err)
+{
+ struct rspamd_proxy_backend_connection *bk_conn = conn->ud;
+ struct rspamd_proxy_session *session;
+
+ session = bk_conn->s;
+ session->retries++;
+ msg_info_session("abnormally closing connection from backend: %s, error: %e,"
+ " retries left: %d",
+ rspamd_inet_address_to_string_pretty(
+ rspamd_upstream_addr_cur(session->master_conn->up)),
+ err,
+ session->ctx->max_retries - session->retries);
+ rspamd_upstream_fail(bk_conn->up, FALSE, err ? err->message : "unknown");
+ proxy_backend_close_connection(session->master_conn);
+
+ if (session->ctx->max_retries > 0 &&
+ session->retries >= session->ctx->max_retries) {
+ msg_err_session("cannot connect to upstream, maximum retries "
+ "has been reached: %d",
+ session->retries);
+ /* Terminate session immediately */
+ if (err) {
+ proxy_client_write_error(session, err->code, err->message);
+ }
+ else {
+ proxy_client_write_error(session, 503, "Unknown error after no retries left");
+ }
+ }
+ else {
+ if (!proxy_send_master_message(session)) {
+ if (err) {
+ proxy_client_write_error(session, err->code, err->message);
+ }
+ else {
+ proxy_client_write_error(session, 503, "Unknown error on write");
+ }
+ }
+ else {
+ msg_info_session("retry connection to: %s"
+ " retries left: %d",
+ rspamd_inet_address_to_string(
+ rspamd_upstream_addr_cur(session->master_conn->up)),
+ session->ctx->max_retries - session->retries);
+ }
+ }
+}
+
+static gint
+proxy_backend_master_finish_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_proxy_backend_connection *bk_conn = conn->ud;
+ struct rspamd_proxy_session *session, *nsession;
+ rspamd_fstring_t *reply;
+ const rspamd_ftok_t *orig_ct;
+ goffset body_offset = -1;
+
+ session = bk_conn->s;
+ rspamd_http_connection_steal_msg(session->master_conn->backend_conn);
+ proxy_request_decompress(msg);
+
+ /*
+ * These are likely set by an http library, so we will double these headers
+ * if they are not removed
+ */
+ rspamd_http_message_remove_header(msg, "Content-Length");
+ rspamd_http_message_remove_header(msg, "Connection");
+ rspamd_http_message_remove_header(msg, "Date");
+ rspamd_http_message_remove_header(msg, "Server");
+ rspamd_http_message_remove_header(msg, "Key");
+ orig_ct = rspamd_http_message_find_header(msg, "Content-Type");
+ rspamd_http_connection_reset(session->master_conn->backend_conn);
+
+ if (!proxy_backend_parse_results(session, bk_conn, session->ctx->lua_state,
+ bk_conn->parser_from_ref, msg, &body_offset, orig_ct)) {
+ msg_warn_session("cannot parse results from the master backend");
+ }
+
+
+ if (session->legacy_support > LEGACY_SUPPORT_NO) {
+ /* We need to reformat ucl to fit with legacy spamc protocol */
+ if (bk_conn->results) {
+ reply = rspamd_fstring_new();
+
+ if (session->legacy_support == LEGACY_SUPPORT_SPAMC) {
+ rspamd_ucl_tospamc_output(bk_conn->results, &reply);
+ msg->flags |= RSPAMD_HTTP_FLAG_SPAMC;
+ }
+ else {
+ rspamd_ucl_torspamc_output(bk_conn->results, &reply);
+ }
+
+ rspamd_http_message_set_body_from_fstring_steal(msg, reply);
+ msg->method = HTTP_SYMBOLS;
+ }
+ else {
+ msg_warn_session("cannot parse results from the master backend, "
+ "return them as is");
+ }
+ }
+
+ rspamd_upstream_ok(bk_conn->up);
+
+ if (session->client_milter_conn) {
+ nsession = proxy_session_refresh(session);
+
+ if (body_offset > 0) {
+ rspamd_milter_send_task_results(nsession->client_milter_conn,
+ session->master_conn->results,
+ msg->body_buf.begin + body_offset,
+ msg->body_buf.len - body_offset);
+ }
+ else {
+ rspamd_milter_send_task_results(nsession->client_milter_conn,
+ session->master_conn->results, NULL, 0);
+ }
+ REF_RELEASE(session);
+ rspamd_http_message_free(msg);
+ }
+ else {
+ const gchar *passed_ct = NULL;
+
+ if (orig_ct) {
+ passed_ct = rspamd_mempool_ftokdup(session->pool, orig_ct);
+ /* Remove original */
+ rspamd_http_message_remove_header(msg, "Content-Type");
+ }
+
+ rspamd_http_connection_write_message(session->client_conn,
+ msg, NULL, passed_ct, session,
+ bk_conn->timeout);
+ }
+
+ return 0;
+}
+
+static void
+rspamd_proxy_scan_self_reply(struct rspamd_task *task)
+{
+ struct rspamd_http_message *msg;
+ struct rspamd_proxy_session *session = task->fin_arg, *nsession;
+ ucl_object_t *rep = NULL;
+ const char *ctype = "application/json";
+
+ msg = rspamd_http_new_message(HTTP_RESPONSE);
+ msg->date = time(NULL);
+ msg->code = 200;
+
+ switch (task->cmd) {
+ case CMD_CHECK:
+ case CMD_SKIP:
+ case CMD_CHECK_RSPAMC:
+ case CMD_CHECK_SPAMC:
+ case CMD_CHECK_V2:
+ rspamd_task_set_finish_time(task);
+ rspamd_protocol_http_reply(msg, task, &rep);
+ rspamd_protocol_write_log_pipe(task);
+ break;
+ case CMD_PING:
+ rspamd_http_message_set_body(msg, "pong" CRLF, 6);
+ ctype = "text/plain";
+ break;
+ default:
+ msg_err_task("BROKEN");
+ break;
+ }
+
+ session->master_conn->flags |= RSPAMD_BACKEND_CLOSED;
+
+ if (rep) {
+ session->master_conn->results = ucl_object_ref(rep);
+ }
+
+ if (session->client_milter_conn) {
+ nsession = proxy_session_refresh(session);
+
+ if (task->flags & RSPAMD_TASK_FLAG_MESSAGE_REWRITE) {
+ const gchar *start;
+ goffset len, hdr_off;
+
+ start = task->msg.begin;
+ len = task->msg.len;
+
+ hdr_off = MESSAGE_FIELD(task, raw_headers_content).len;
+
+ if (hdr_off < len) {
+ start += hdr_off;
+ len -= hdr_off;
+
+ /* The problem here is that we need not end of headers, we need
+ * start of body.
+ *
+ * Hence, we need to skip one \r\n till there is anything else in
+ * a line.
+ */
+
+ if (*start == '\r' && len > 0) {
+ start++;
+ len--;
+ }
+
+ if (*start == '\n' && len > 0) {
+ start++;
+ len--;
+ }
+
+ rspamd_milter_send_task_results(nsession->client_milter_conn,
+ session->master_conn->results, start, len);
+ }
+ else {
+ /* XXX: should never happen! */
+ rspamd_milter_send_task_results(nsession->client_milter_conn,
+ session->master_conn->results, NULL, 0);
+ }
+ }
+ else {
+ rspamd_milter_send_task_results(nsession->client_milter_conn,
+ session->master_conn->results, NULL, 0);
+ }
+ rspamd_http_message_free(msg);
+ REF_RELEASE(session);
+ }
+ else {
+ rspamd_http_connection_reset(session->client_conn);
+ rspamd_http_connection_write_message(session->client_conn,
+ msg,
+ NULL,
+ ctype,
+ session,
+ session->ctx->timeout / 10.0);
+ }
+}
+
+static gboolean
+rspamd_proxy_task_fin(void *ud)
+{
+ struct rspamd_task *task = ud;
+
+ msg_debug_task("finish task");
+
+ if (RSPAMD_TASK_IS_PROCESSED(task)) {
+ rspamd_proxy_scan_self_reply(task);
+ return TRUE;
+ }
+
+ if (!rspamd_task_process(task, RSPAMD_TASK_PROCESS_ALL)) {
+ rspamd_proxy_scan_self_reply(task);
+ return TRUE;
+ }
+
+ if (RSPAMD_TASK_IS_PROCESSED(task)) {
+ rspamd_proxy_scan_self_reply(task);
+ return TRUE;
+ }
+
+ /* One more iteration */
+ return FALSE;
+}
+
+static gboolean
+rspamd_proxy_self_scan(struct rspamd_proxy_session *session)
+{
+ struct rspamd_task *task;
+ struct rspamd_http_message *msg;
+ const gchar *data;
+ gsize len;
+
+ msg = session->client_message;
+ task = rspamd_task_new(session->worker, session->ctx->cfg,
+ session->pool, session->ctx->lang_det,
+ session->ctx->event_loop, FALSE);
+ task->flags |= RSPAMD_TASK_FLAG_MIME;
+
+ if (session->ctx->milter) {
+ task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_MILTER |
+ RSPAMD_TASK_PROTOCOL_FLAG_BODY_BLOCK;
+ }
+
+ task->sock = -1;
+
+ if (session->client_milter_conn) {
+ task->client_addr = rspamd_inet_address_copy(
+ session->client_milter_conn->addr, NULL);
+ }
+ else {
+ task->client_addr = rspamd_inet_address_copy(session->client_addr, NULL);
+ }
+
+ task->fin_arg = session;
+ task->resolver = session->ctx->resolver;
+ /* TODO: allow to disable autolearn in protocol */
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_AUTO;
+ task->s = rspamd_session_create(task->task_pool, rspamd_proxy_task_fin,
+ NULL, (event_finalizer_t) rspamd_task_free, task);
+ data = rspamd_http_message_get_body(msg, &len);
+
+ if (session->backend->settings_id) {
+ rspamd_http_message_remove_header(msg, "Settings-ID");
+ rspamd_http_message_add_header(msg, "Settings-ID",
+ session->backend->settings_id);
+ }
+
+ /* Process message */
+ if (!rspamd_protocol_handle_request(task, msg)) {
+ msg_err_task("cannot handle request: %e", task->err);
+ task->flags |= RSPAMD_TASK_FLAG_SKIP;
+ }
+ else {
+ if (task->cmd == CMD_PING) {
+ task->flags |= RSPAMD_TASK_FLAG_SKIP;
+ }
+ else {
+ if (!rspamd_task_load_message(task, msg, data, len)) {
+ msg_err_task("cannot load message: %e", task->err);
+ task->flags |= RSPAMD_TASK_FLAG_SKIP;
+ }
+ }
+ }
+
+ /* Set global timeout for the task */
+ if (session->ctx->default_upstream->timeout > 0.0) {
+ task->timeout_ev.data = task;
+ ev_timer_init(&task->timeout_ev, rspamd_task_timeout,
+ session->ctx->default_upstream->timeout,
+ session->ctx->default_upstream->timeout);
+ ev_timer_start(task->event_loop, &task->timeout_ev);
+ }
+ else if (session->ctx->has_self_scan) {
+ if (!isnan(session->ctx->task_timeout) && session->ctx->task_timeout > 0) {
+ task->timeout_ev.data = task;
+ ev_timer_init(&task->timeout_ev, rspamd_task_timeout,
+ session->ctx->cfg->task_timeout,
+ session->ctx->default_upstream->timeout);
+ ev_timer_start(task->event_loop, &task->timeout_ev);
+ }
+ }
+
+ session->master_conn->task = task;
+ rspamd_task_process(task, RSPAMD_TASK_PROCESS_ALL);
+
+ rspamd_session_pending(task->s);
+
+ return TRUE;
+}
+
+static gboolean
+proxy_send_master_message(struct rspamd_proxy_session *session)
+{
+ struct rspamd_http_message *msg;
+ struct rspamd_http_upstream *backend = NULL;
+ const rspamd_ftok_t *host;
+ GError *err = NULL;
+ gchar hostbuf[512];
+
+ host = rspamd_http_message_find_header(session->client_message, "Host");
+
+ if (host == NULL) {
+ backend = session->ctx->default_upstream;
+ }
+ else {
+ rspamd_strlcpy(hostbuf, host->begin, MIN(host->len + 1, sizeof(hostbuf)));
+ backend = g_hash_table_lookup(session->ctx->upstreams, hostbuf);
+
+ if (backend == NULL) {
+ backend = session->ctx->default_upstream;
+ }
+ }
+
+ if (backend == NULL) {
+ /* No backend */
+ msg_err_session("cannot find upstream for %s", host ? hostbuf : "default");
+ goto err;
+ }
+ else {
+ session->backend = backend;
+
+ if (backend->self_scan) {
+ return rspamd_proxy_self_scan(session);
+ }
+ retry:
+ if (session->ctx->max_retries &&
+ session->retries > session->ctx->max_retries) {
+ msg_err_session("cannot connect to upstream, maximum retries "
+ "has been reached: %d",
+ session->retries);
+ goto err;
+ }
+
+ /* Provide hash key if hashing based on source address is desired */
+ guint hash_len;
+ gpointer hash_key = rspamd_inet_address_get_hash_key(session->client_addr,
+ &hash_len);
+
+ if (session->ctx->max_retries > 1 &&
+ session->retries == session->ctx->max_retries) {
+
+ session->master_conn->up = rspamd_upstream_get_except(backend->u,
+ session->master_conn->up,
+ RSPAMD_UPSTREAM_ROUND_ROBIN,
+ hash_key, hash_len);
+ }
+ else {
+ session->master_conn->up = rspamd_upstream_get(backend->u,
+ RSPAMD_UPSTREAM_ROUND_ROBIN,
+ hash_key, hash_len);
+ }
+
+ session->master_conn->timeout = backend->timeout;
+
+ if (session->master_conn->up == NULL) {
+ msg_err_session("cannot select upstream for %s",
+ host ? hostbuf : "default");
+ goto err;
+ }
+
+ session->master_conn->backend_sock = rspamd_inet_address_connect(
+ rspamd_upstream_addr_next(session->master_conn->up),
+ SOCK_STREAM, TRUE);
+
+ if (session->master_conn->backend_sock == -1) {
+ msg_err_session("cannot connect upstream: %s(%s)",
+ host ? hostbuf : "default",
+ rspamd_inet_address_to_string_pretty(
+ rspamd_upstream_addr_cur(
+ session->master_conn->up)));
+ rspamd_upstream_fail(session->master_conn->up, TRUE,
+ strerror(errno));
+ session->retries++;
+ goto retry;
+ }
+
+ msg = rspamd_http_connection_copy_msg(session->client_message, &err);
+ if (msg == NULL) {
+ msg_err_session("cannot copy message to send it to the upstream: %e",
+ err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ goto err; /* No fallback here */
+ }
+
+ session->master_conn->backend_conn = rspamd_http_connection_new_client_socket(
+ session->ctx->http_ctx,
+ NULL,
+ proxy_backend_master_error_handler,
+ proxy_backend_master_finish_handler,
+ RSPAMD_HTTP_CLIENT_SIMPLE,
+ session->master_conn->backend_sock);
+ session->master_conn->flags &= ~RSPAMD_BACKEND_CLOSED;
+ session->master_conn->parser_from_ref = backend->parser_from_ref;
+ session->master_conn->parser_to_ref = backend->parser_to_ref;
+
+ if (backend->key) {
+ msg->peer_key = rspamd_pubkey_ref(backend->key);
+ }
+
+ if (backend->settings_id != NULL) {
+ rspamd_http_message_remove_header(msg, "Settings-ID");
+ rspamd_http_message_add_header(msg, "Settings-ID",
+ backend->settings_id);
+ }
+
+ if (backend->local ||
+ rspamd_inet_address_is_local(
+ rspamd_upstream_addr_cur(
+ session->master_conn->up))) {
+
+ if (session->fname) {
+ rspamd_http_message_add_header(msg, "File", session->fname);
+ }
+
+ msg->method = HTTP_GET;
+
+ rspamd_http_connection_write_message_shared(
+ session->master_conn->backend_conn,
+ msg, rspamd_upstream_name(session->master_conn->up),
+ NULL, session->master_conn,
+ session->master_conn->timeout);
+ }
+ else {
+ if (session->fname) {
+ msg->flags &= ~RSPAMD_HTTP_FLAG_SHMEM;
+ rspamd_http_message_set_body(msg,
+ session->map, session->map_len);
+ }
+
+ msg->method = HTTP_POST;
+
+ if (backend->compress) {
+ proxy_request_compress(msg);
+ if (session->client_milter_conn) {
+ rspamd_http_message_add_header(msg, "Content-Type",
+ "application/octet-stream");
+ }
+ }
+ else {
+ if (session->client_milter_conn) {
+ rspamd_http_message_add_header(msg, "Content-Type",
+ "text/plain");
+ }
+ }
+
+ rspamd_http_connection_write_message(
+ session->master_conn->backend_conn,
+ msg, rspamd_upstream_name(session->master_conn->up),
+ NULL, session->master_conn,
+ session->master_conn->timeout);
+ }
+ }
+
+ return TRUE;
+
+err:
+ if (session->client_milter_conn) {
+ rspamd_milter_send_action(session->client_milter_conn,
+ RSPAMD_MILTER_TEMPFAIL);
+ REF_RELEASE(session);
+ }
+ else {
+ rspamd_http_connection_steal_msg(session->client_conn);
+ rspamd_http_connection_reset(session->client_conn);
+ proxy_client_write_error(session, 404, "Backend not found");
+ }
+
+ return FALSE;
+}
+
+static void
+proxy_client_error_handler(struct rspamd_http_connection *conn, GError *err)
+{
+ struct rspamd_proxy_session *session = conn->ud;
+
+ msg_info_session("abnormally closing connection from: %s, error: %s",
+ rspamd_inet_address_to_string(session->client_addr), err->message);
+ /* Terminate session immediately */
+ proxy_backend_close_connection(session->master_conn);
+ REF_RELEASE(session);
+}
+
+static gint
+proxy_client_finish_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_proxy_session *session = conn->ud;
+
+ if (!session->master_conn) {
+ session->master_conn = rspamd_mempool_alloc0(session->pool,
+ sizeof(*session->master_conn));
+ session->master_conn->s = session;
+ session->master_conn->name = "master";
+
+ /* Reset spamc legacy */
+ if (msg->method >= HTTP_SYMBOLS) {
+ msg->method = HTTP_POST;
+
+ if (msg->flags & RSPAMD_HTTP_FLAG_SPAMC) {
+ session->legacy_support = LEGACY_SUPPORT_SPAMC;
+ msg_info_session("enabling legacy spamc mode for session");
+ }
+ else {
+ session->legacy_support = LEGACY_SUPPORT_RSPAMC;
+ msg_info_session("enabling legacy rspamc mode for session");
+ }
+ }
+
+ if (msg->url->len == 0) {
+ msg->url = rspamd_fstring_append(msg->url,
+ "/" MSG_CMD_CHECK_V2, strlen("/" MSG_CMD_CHECK_V2));
+ }
+
+ if (!proxy_check_file(msg, session)) {
+ goto err;
+ }
+
+ session->client_message = rspamd_http_connection_steal_msg(
+ session->client_conn);
+ session->shmem_ref = rspamd_http_message_shmem_ref(session->client_message);
+ rspamd_http_message_remove_header(msg, "Content-Length");
+ rspamd_http_message_remove_header(msg, "Transfer-Encoding");
+ rspamd_http_message_remove_header(msg, "Keep-Alive");
+ rspamd_http_message_remove_header(msg, "Connection");
+ rspamd_http_message_remove_header(msg, "Key");
+
+ proxy_open_mirror_connections(session);
+ rspamd_http_connection_reset(session->client_conn);
+
+ proxy_send_master_message(session);
+ }
+ else {
+ msg_info_session("finished master connection");
+ proxy_backend_close_connection(session->master_conn);
+ REF_RELEASE(session);
+ }
+
+ return 0;
+
+err:
+ rspamd_http_connection_steal_msg(session->client_conn);
+ rspamd_http_message_remove_header(msg, "Content-Length");
+ rspamd_http_message_remove_header(msg, "Key");
+ rspamd_http_message_remove_header(msg, "Transfer-Encoding");
+ rspamd_http_message_remove_header(msg, "Keep-Alive");
+ rspamd_http_message_remove_header(msg, "Connection");
+ rspamd_http_connection_reset(session->client_conn);
+ proxy_client_write_error(session, 404, "Backend not found");
+
+ return 0;
+}
+
+static void
+proxy_milter_finish_handler(gint fd,
+ struct rspamd_milter_session *rms,
+ void *ud)
+{
+ struct rspamd_proxy_session *session = ud;
+ struct rspamd_http_message *msg;
+
+ session->client_milter_conn = rms;
+
+ if (rms->message == NULL || rms->message->len == 0) {
+ msg_info_session("finished milter connection");
+ proxy_backend_close_connection(session->master_conn);
+ REF_RELEASE(session);
+ }
+ else {
+ if (!session->master_conn) {
+ session->master_conn = rspamd_mempool_alloc0(session->pool,
+ sizeof(*session->master_conn));
+ }
+
+ msg = rspamd_milter_to_http(rms);
+ session->master_conn->s = session;
+ session->master_conn->name = "master";
+ session->client_message = msg;
+
+ proxy_open_mirror_connections(session);
+ proxy_send_master_message(session);
+ }
+}
+
+static void
+proxy_milter_error_handler(gint fd,
+ struct rspamd_milter_session *rms, /* unused */
+ void *ud, GError *err)
+{
+ struct rspamd_proxy_session *session = ud;
+
+ if (err && err->code != 0) {
+ msg_info_session("abnormally closing milter connection from: %s, "
+ "error: %e",
+ rspamd_inet_address_to_string_pretty(session->client_addr),
+ err);
+ /* Terminate session immediately */
+ proxy_backend_close_connection(session->master_conn);
+ REF_RELEASE(session);
+ }
+ else {
+ msg_info_session("normally closing milter connection from: %s, "
+ "%e",
+ rspamd_inet_address_to_string_pretty(session->client_addr),
+ err);
+ /* Terminate session immediately */
+ proxy_backend_close_connection(session->master_conn);
+ REF_RELEASE(session);
+ }
+}
+
+static void
+proxy_accept_socket(EV_P_ ev_io *w, int revents)
+{
+ struct rspamd_worker *worker = (struct rspamd_worker *) w->data;
+ struct rspamd_proxy_ctx *ctx;
+ rspamd_inet_addr_t *addr = NULL;
+ struct rspamd_proxy_session *session;
+ gint nfd;
+
+ ctx = worker->ctx;
+
+ if ((nfd =
+ rspamd_accept_from_socket(w->fd, &addr,
+ rspamd_worker_throttle_accept_events, worker->accept_events)) == -1) {
+ msg_warn("accept failed: %s", strerror(errno));
+ return;
+ }
+ /* Check for EAGAIN */
+ if (nfd == 0) {
+ rspamd_inet_address_free(addr);
+ return;
+ }
+
+ session = g_malloc0(sizeof(*session));
+ REF_INIT_RETAIN(session, proxy_session_dtor);
+ session->client_sock = nfd;
+ session->client_addr = addr;
+ session->mirror_conns = g_ptr_array_sized_new(ctx->mirrors->len);
+
+ session->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "proxy", 0);
+ session->ctx = ctx;
+ session->worker = worker;
+
+ if (ctx->sessions_cache) {
+ rspamd_worker_session_cache_add(ctx->sessions_cache,
+ session->pool->tag.uid, &session->ref.refcount, session);
+ }
+
+ if (!ctx->milter) {
+ int http_opts = 0;
+
+ if (ctx->encrypted_only && !rspamd_inet_address_is_local(addr)) {
+ http_opts |= RSPAMD_HTTP_REQUIRE_ENCRYPTION;
+ }
+ session->client_conn = rspamd_http_connection_new_server(
+ ctx->http_ctx,
+ nfd,
+ NULL,
+ proxy_client_error_handler,
+ proxy_client_finish_handler,
+ http_opts);
+
+ if (ctx->key) {
+ rspamd_http_connection_set_key(session->client_conn, ctx->key);
+ }
+
+ msg_info_session("accepted http connection from %s port %d",
+ rspamd_inet_address_to_string(addr),
+ rspamd_inet_address_get_port(addr));
+
+ rspamd_http_connection_read_message_shared(session->client_conn,
+ session,
+ session->ctx->timeout);
+ }
+ else {
+ msg_info_session("accepted milter connection from %s port %d",
+ rspamd_inet_address_to_string(addr),
+ rspamd_inet_address_get_port(addr));
+
+#ifdef TCP_NODELAY
+
+#ifndef SOL_TCP
+#define SOL_TCP IPPROTO_TCP
+#endif
+
+ if (rspamd_inet_address_get_af(addr) != AF_UNIX) {
+ gint sopt = 1;
+
+ if (setsockopt(nfd, SOL_TCP, TCP_NODELAY, &sopt, sizeof(sopt)) ==
+ -1) {
+ msg_warn_session("cannot set TCP_NODELAY: %s",
+ strerror(errno));
+ }
+ }
+#endif
+
+ rspamd_milter_handle_socket(nfd, 0.0,
+ session->pool,
+ ctx->event_loop,
+ proxy_milter_finish_handler,
+ proxy_milter_error_handler,
+ session);
+ }
+}
+
+static void
+adjust_upstreams_limits(struct rspamd_proxy_ctx *ctx)
+{
+ struct rspamd_http_upstream *backend;
+ gpointer k, v;
+ GHashTableIter it;
+
+ /*
+ * We set error time equal to max_retries * backend_timeout and max_errors
+ * to max_retries - 1
+ *
+ * So if we failed to scan a message on a backend for some reasons, we
+ * will try to re-resolve it faster
+ */
+
+ g_hash_table_iter_init(&it, ctx->upstreams);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ backend = (struct rspamd_http_upstream *) v;
+
+ if (!backend->self_scan && backend->u) {
+ rspamd_upstreams_set_limits(backend->u,
+ NAN, NAN, ctx->max_retries * backend->timeout, NAN,
+ ctx->max_retries - 1, 0);
+ }
+ }
+}
+
+__attribute__((noreturn)) void
+start_rspamd_proxy(struct rspamd_worker *worker)
+{
+ struct rspamd_proxy_ctx *ctx = worker->ctx;
+ gboolean is_controller = FALSE;
+
+ g_assert(rspamd_worker_check_context(worker->ctx, rspamd_rspamd_proxy_magic));
+ ctx->cfg = worker->srv->cfg;
+ ctx->event_loop = rspamd_prepare_worker(worker, "rspamd_proxy",
+ proxy_accept_socket);
+
+ ctx->resolver = rspamd_dns_resolver_init(worker->srv->logger,
+ ctx->event_loop,
+ worker->srv->cfg);
+
+ rspamd_upstreams_library_config(worker->srv->cfg, ctx->cfg->ups_ctx,
+ ctx->event_loop, ctx->resolver->r);
+
+ ctx->http_ctx = rspamd_http_context_create(ctx->cfg, ctx->event_loop,
+ ctx->cfg->ups_ctx);
+ rspamd_mempool_add_destructor(ctx->cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_http_context_free,
+ ctx->http_ctx);
+
+ if (ctx->has_self_scan) {
+ /* Additional initialisation needed */
+ rspamd_worker_init_scanner(worker, ctx->event_loop, ctx->resolver,
+ &ctx->lang_det);
+ ctx->task_timeout = rspamd_worker_check_and_adjust_timeout(ctx->cfg, NAN);
+
+ is_controller = rspamd_worker_check_controller_presence(worker);
+ }
+ else {
+ worker->flags &= ~RSPAMD_WORKER_SCANNER;
+ }
+
+ if (worker->srv->cfg->enable_sessions_cache) {
+ ctx->sessions_cache = rspamd_worker_session_cache_new(worker,
+ ctx->event_loop);
+ }
+
+ ctx->milter_ctx.spam_header = ctx->spam_header;
+ ctx->milter_ctx.discard_on_reject = ctx->discard_on_reject;
+ ctx->milter_ctx.quarantine_on_reject = ctx->quarantine_on_reject;
+ ctx->milter_ctx.sessions_cache = ctx->sessions_cache;
+ ctx->milter_ctx.client_ca_name = ctx->client_ca_name;
+ ctx->milter_ctx.reject_message = ctx->reject_message;
+ ctx->milter_ctx.cfg = ctx->cfg;
+ rspamd_milter_init_library(&ctx->milter_ctx);
+
+ if (is_controller) {
+ rspamd_worker_init_controller(worker, NULL);
+ }
+ else {
+ if (ctx->has_self_scan) {
+ rspamd_map_watch(worker->srv->cfg, ctx->event_loop, ctx->resolver,
+ worker, RSPAMD_MAP_WATCH_SCANNER);
+ }
+ else {
+ rspamd_map_watch(worker->srv->cfg, ctx->event_loop, ctx->resolver,
+ worker, RSPAMD_MAP_WATCH_WORKER);
+ }
+ }
+
+ rspamd_lua_run_postloads(ctx->cfg->lua_state, ctx->cfg, ctx->event_loop,
+ worker);
+ adjust_upstreams_limits(ctx);
+
+ ev_loop(ctx->event_loop, 0);
+ rspamd_worker_block_signals();
+
+ if (ctx->has_self_scan) {
+ rspamd_stat_close();
+ }
+
+ if (is_controller) {
+ rspamd_controller_on_terminate(worker, NULL);
+ }
+
+ REF_RELEASE(ctx->cfg);
+ rspamd_log_close(worker->srv->logger);
+ rspamd_unset_crash_handler(worker->srv);
+
+ exit(EXIT_SUCCESS);
+}
diff --git a/src/worker.c b/src/worker.c
new file mode 100644
index 0000000..8f99ad5
--- /dev/null
+++ b/src/worker.c
@@ -0,0 +1,537 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Rspamd worker implementation
+ */
+
+#include "config.h"
+#include "libutil/util.h"
+#include "libserver/maps/map.h"
+#include "libutil/upstream.h"
+#include "libserver/protocol.h"
+#include "libserver/cfg_file.h"
+#include "libserver/url.h"
+#include "libserver/dns.h"
+#include "libmime/message.h"
+#include "rspamd.h"
+#include "libstat/stat_api.h"
+#include "libserver/worker_util.h"
+#include "libserver/rspamd_control.h"
+#include "worker_private.h"
+#include "libserver/http/http_private.h"
+#include "libserver/cfg_file_private.h"
+#include <math.h>
+#include "unix-std.h"
+
+#include "lua/lua_common.h"
+
+/* 60 seconds for worker's IO */
+#define DEFAULT_WORKER_IO_TIMEOUT 60.0
+
+gpointer init_worker(struct rspamd_config *cfg);
+void start_worker(struct rspamd_worker *worker);
+
+worker_t normal_worker = {
+ "normal", /* Name */
+ init_worker, /* Init function */
+ start_worker, /* Start function */
+ RSPAMD_WORKER_HAS_SOCKET | RSPAMD_WORKER_KILLABLE | RSPAMD_WORKER_SCANNER,
+ RSPAMD_WORKER_SOCKET_TCP, /* TCP socket */
+ RSPAMD_WORKER_VER /* Version info */
+};
+
+#define msg_err_ctx(...) rspamd_default_log_function(G_LOG_LEVEL_CRITICAL, \
+ "worker", ctx->cfg->cfg_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_warn_ctx(...) rspamd_default_log_function(G_LOG_LEVEL_WARNING, \
+ "worker", ctx->cfg->cfg_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+#define msg_info_ctx(...) rspamd_default_log_function(G_LOG_LEVEL_INFO, \
+ "worker", ctx->cfg->cfg_pool->tag.uid, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+struct rspamd_worker_session {
+ gint64 magic;
+ struct rspamd_task *task;
+ gint fd;
+ rspamd_inet_addr_t *addr;
+ struct rspamd_worker_ctx *ctx;
+ struct rspamd_http_connection *http_conn;
+ struct rspamd_worker *worker;
+};
+/*
+ * Reduce number of tasks proceeded
+ */
+static void
+reduce_tasks_count(gpointer arg)
+{
+ struct rspamd_worker *worker = arg;
+
+ worker->nconns--;
+
+ if (worker->state == rspamd_worker_wait_connections && worker->nconns == 0) {
+
+ worker->state = rspamd_worker_wait_final_scripts;
+ msg_info("performing finishing actions");
+
+ if (rspamd_worker_call_finish_handlers(worker)) {
+ worker->state = rspamd_worker_wait_final_scripts;
+ }
+ else {
+ worker->state = rspamd_worker_wanna_die;
+ }
+ }
+ else if (worker->state != rspamd_worker_state_running) {
+ worker->state = rspamd_worker_wait_connections;
+ }
+}
+
+static gint
+rspamd_worker_body_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *chunk, gsize len)
+{
+ struct rspamd_worker_session *session = (struct rspamd_worker_session *) conn->ud;
+ struct rspamd_task *task;
+ struct rspamd_worker_ctx *ctx;
+ const rspamd_ftok_t *hv_tok;
+ gboolean debug_mempool = FALSE;
+
+ ctx = session->ctx;
+
+ /* Check debug */
+ if ((hv_tok = rspamd_http_message_find_header(msg, "Memory")) != NULL) {
+ rspamd_ftok_t cmp;
+
+ RSPAMD_FTOK_ASSIGN(&cmp, "debug");
+
+ if (rspamd_ftok_cmp(hv_tok, &cmp) == 0) {
+ debug_mempool = TRUE;
+ }
+ }
+
+ task = rspamd_task_new(session->worker,
+ session->ctx->cfg, NULL, session->ctx->lang_det,
+ session->ctx->event_loop,
+ debug_mempool);
+ session->task = task;
+
+ msg_info_task("accepted connection from %s port %d, task ptr: %p",
+ rspamd_inet_address_to_string(session->addr),
+ rspamd_inet_address_get_port(session->addr),
+ task);
+
+ /* Copy some variables */
+ if (ctx->is_mime) {
+ task->flags |= RSPAMD_TASK_FLAG_MIME;
+ }
+ else {
+ task->flags &= ~RSPAMD_TASK_FLAG_MIME;
+ }
+
+ /* We actually transfer ownership from session to task here */
+ task->sock = session->fd;
+ task->client_addr = session->addr;
+ task->worker = session->worker;
+ task->http_conn = session->http_conn;
+
+ task->resolver = ctx->resolver;
+ /* TODO: allow to disable autolearn in protocol */
+ task->flags |= RSPAMD_TASK_FLAG_LEARN_AUTO;
+
+ session->worker->nconns++;
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) reduce_tasks_count,
+ session->worker);
+
+ /* Session memory is also now handled by task */
+ rspamd_mempool_add_destructor(task->task_pool,
+ (rspamd_mempool_destruct_t) g_free,
+ session);
+
+ /* Set up async session */
+ task->s = rspamd_session_create(task->task_pool, rspamd_task_fin,
+ NULL, (event_finalizer_t) rspamd_task_free, task);
+
+ if (!rspamd_protocol_handle_request(task, msg)) {
+ msg_err_task("cannot handle request: %e", task->err);
+ task->flags |= RSPAMD_TASK_FLAG_SKIP;
+ }
+ else {
+ if (task->cmd == CMD_PING) {
+ task->flags |= RSPAMD_TASK_FLAG_SKIP;
+ }
+ else {
+ if (!rspamd_task_load_message(task, msg, chunk, len)) {
+ msg_err_task("cannot load message: %e", task->err);
+ task->flags |= RSPAMD_TASK_FLAG_SKIP;
+ }
+ }
+ }
+
+ /* Set global timeout for the task */
+ if (!isnan(ctx->task_timeout) && ctx->task_timeout > 0.0) {
+ task->timeout_ev.data = task;
+ ev_timer_init(&task->timeout_ev, rspamd_task_timeout,
+ ctx->task_timeout,
+ ctx->task_timeout);
+ ev_set_priority(&task->timeout_ev, EV_MAXPRI);
+ ev_timer_start(task->event_loop, &task->timeout_ev);
+ }
+
+ /* Set socket guard */
+ task->guard_ev.data = task;
+ ev_io_init(&task->guard_ev,
+ rspamd_worker_guard_handler,
+ task->sock, EV_READ);
+ ev_io_start(task->event_loop, &task->guard_ev);
+
+ rspamd_task_process(task, RSPAMD_TASK_PROCESS_ALL);
+
+ return 0;
+}
+
+static void
+rspamd_worker_error_handler(struct rspamd_http_connection *conn, GError *err)
+{
+ struct rspamd_worker_session *session = (struct rspamd_worker_session *) conn->ud;
+ struct rspamd_task *task;
+ struct rspamd_http_message *msg;
+ rspamd_fstring_t *reply;
+
+ /*
+ * This function can be called with both struct rspamd_worker_session *
+ * and struct rspamd_task *
+ *
+ * The first case is when we read message and it is controlled by this code;
+ * the second case is when a reply is written and we do not control it normally,
+ * as it is managed by `rspamd_protocol_reply` in protocol.c
+ *
+ * Hence, we need to distinguish our arguments...
+ *
+ * The approach here is simple:
+ * - struct rspamd_worker_session starts with gint64 `magic` and we set it to
+ * MAX_INT64
+ * - struct rspamd_task starts with a pointer (or pointer + command on 32 bit system)
+ *
+ * The idea is simple: no sane pointer would reach MAX_INT64, so if this field
+ * is MAX_INT64 then it is our session, and if it is not then it is a task.
+ */
+
+ if (session->magic == G_MAXINT64) {
+ task = session->task;
+ }
+ else {
+ task = (struct rspamd_task *) conn->ud;
+ }
+
+
+ if (task) {
+ msg_info_task("abnormally closing connection from: %s, error: %e",
+ rspamd_inet_address_to_string_pretty(task->client_addr), err);
+
+ if (task->processed_stages & RSPAMD_TASK_STAGE_REPLIED) {
+ /* Terminate session immediately */
+ rspamd_session_destroy(task->s);
+ }
+ else {
+ task->processed_stages |= RSPAMD_TASK_STAGE_REPLIED;
+ msg = rspamd_http_new_message(HTTP_RESPONSE);
+
+ if (err) {
+ msg->status = rspamd_fstring_new_init(err->message,
+ strlen(err->message));
+ msg->code = err->code;
+ }
+ else {
+ msg->status = rspamd_fstring_new_init("Internal error",
+ strlen("Internal error"));
+ msg->code = 500;
+ }
+
+ msg->date = time(NULL);
+
+ reply = rspamd_fstring_sized_new(msg->status->len + 16);
+ rspamd_printf_fstring(&reply, "{\"error\":\"%V\"}", msg->status);
+ rspamd_http_message_set_body_from_fstring_steal(msg, reply);
+ rspamd_http_connection_reset(task->http_conn);
+ /* Use a shorter timeout for writing reply */
+ rspamd_http_connection_write_message(task->http_conn,
+ msg,
+ NULL,
+ "application/json",
+ task,
+ session->ctx->timeout / 10.0);
+ }
+ }
+ else {
+ /* If there was no task, then session is unmanaged */
+ msg_info("no data received from: %s, error: %e",
+ rspamd_inet_address_to_string_pretty(session->addr), err);
+ rspamd_http_connection_reset(session->http_conn);
+ rspamd_http_connection_unref(session->http_conn);
+ rspamd_inet_address_free(session->addr);
+ close(session->fd);
+ g_free(session);
+ }
+}
+
+static gint
+rspamd_worker_finish_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_worker_session *session = (struct rspamd_worker_session *) conn->ud;
+ struct rspamd_task *task;
+
+ /* Read the comment to rspamd_worker_error_handler */
+
+ if (session->magic == G_MAXINT64) {
+ task = session->task;
+ }
+ else {
+ task = (struct rspamd_task *) conn->ud;
+ }
+
+ if (task) {
+ if (task->processed_stages & RSPAMD_TASK_STAGE_REPLIED) {
+ /* We are done here */
+ msg_debug_task("normally closing connection from: %s",
+ rspamd_inet_address_to_string(task->client_addr));
+ rspamd_session_destroy(task->s);
+ }
+ else if (task->processed_stages & RSPAMD_TASK_STAGE_DONE) {
+ rspamd_session_pending(task->s);
+ }
+ }
+ else {
+ /* If there was no task, then session is unmanaged */
+ msg_info("no data received from: %s, closing connection",
+ rspamd_inet_address_to_string_pretty(session->addr));
+ rspamd_inet_address_free(session->addr);
+ rspamd_http_connection_reset(session->http_conn);
+ rspamd_http_connection_unref(session->http_conn);
+ close(session->fd);
+ g_free(session);
+ }
+
+ return 0;
+}
+
+/*
+ * Accept new connection and construct task
+ */
+static void
+accept_socket(EV_P_ ev_io *w, int revents)
+{
+ struct rspamd_worker *worker = (struct rspamd_worker *) w->data;
+ struct rspamd_worker_ctx *ctx;
+ struct rspamd_worker_session *session;
+ rspamd_inet_addr_t *addr = NULL;
+ gint nfd, http_opts = 0;
+
+ ctx = worker->ctx;
+
+ if (ctx->max_tasks != 0 && worker->nconns > ctx->max_tasks) {
+ msg_info_ctx("current tasks is now: %uD while maximum is: %uD",
+ worker->nconns,
+ ctx->max_tasks);
+ return;
+ }
+
+ if ((nfd =
+ rspamd_accept_from_socket(w->fd, &addr,
+ rspamd_worker_throttle_accept_events, worker->accept_events)) == -1) {
+ msg_warn_ctx("accept failed: %s", strerror(errno));
+ return;
+ }
+ /* Check for EAGAIN */
+ if (nfd == 0) {
+ rspamd_inet_address_free(addr);
+
+ return;
+ }
+
+ session = g_malloc0(sizeof(*session));
+ session->magic = G_MAXINT64;
+ session->addr = addr;
+ session->fd = nfd;
+ session->ctx = ctx;
+ session->worker = worker;
+
+ if (ctx->encrypted_only && !rspamd_inet_address_is_local(addr)) {
+ http_opts = RSPAMD_HTTP_REQUIRE_ENCRYPTION;
+ }
+
+ session->http_conn = rspamd_http_connection_new_server(
+ ctx->http_ctx,
+ nfd,
+ rspamd_worker_body_handler,
+ rspamd_worker_error_handler,
+ rspamd_worker_finish_handler,
+ http_opts);
+
+ worker->srv->stat->connections_count++;
+ rspamd_http_connection_set_max_size(session->http_conn,
+ ctx->cfg->max_message);
+
+ if (ctx->key) {
+ rspamd_http_connection_set_key(session->http_conn, ctx->key);
+ }
+
+ rspamd_http_connection_read_message(session->http_conn,
+ session,
+ ctx->timeout);
+}
+
+gpointer
+init_worker(struct rspamd_config *cfg)
+{
+ struct rspamd_worker_ctx *ctx;
+ GQuark type;
+
+ type = g_quark_try_string("normal");
+ ctx = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(struct rspamd_worker_ctx));
+
+ ctx->magic = rspamd_worker_magic;
+ ctx->is_mime = TRUE;
+ ctx->timeout = DEFAULT_WORKER_IO_TIMEOUT;
+ ctx->cfg = cfg;
+ ctx->task_timeout = NAN;
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "mime",
+ rspamd_rcl_parse_struct_boolean,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_worker_ctx, is_mime),
+ 0,
+ "Set to `false` if this worker is intended to work with non-MIME messages");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "encrypted_only",
+ rspamd_rcl_parse_struct_boolean,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_worker_ctx, encrypted_only),
+ 0,
+ "Allow only encrypted connections");
+
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "timeout",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_worker_ctx,
+ timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Protocol IO timeout");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "task_timeout",
+ rspamd_rcl_parse_struct_time,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_worker_ctx,
+ task_timeout),
+ RSPAMD_CL_FLAG_TIME_FLOAT,
+ "Maximum task processing time, default: 8.0 seconds");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "max_tasks",
+ rspamd_rcl_parse_struct_integer,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_worker_ctx,
+ max_tasks),
+ RSPAMD_CL_FLAG_INT_32,
+ "Maximum count of parallel tasks processed by a single worker process");
+
+ rspamd_rcl_register_worker_option(cfg,
+ type,
+ "keypair",
+ rspamd_rcl_parse_struct_keypair,
+ ctx,
+ G_STRUCT_OFFSET(struct rspamd_worker_ctx,
+ key),
+ 0,
+ "Encryption keypair");
+
+ return ctx;
+}
+
+/*
+ * Start worker process
+ */
+__attribute__((noreturn)) void
+start_worker(struct rspamd_worker *worker)
+{
+ struct rspamd_worker_ctx *ctx = worker->ctx;
+ gboolean is_controller = FALSE;
+
+ g_assert(rspamd_worker_check_context(worker->ctx, rspamd_worker_magic));
+ ctx->cfg = worker->srv->cfg;
+ ctx->event_loop = rspamd_prepare_worker(worker, "normal", accept_socket);
+ rspamd_symcache_start_refresh(worker->srv->cfg->cache, ctx->event_loop,
+ worker);
+
+ ctx->task_timeout = rspamd_worker_check_and_adjust_timeout(ctx->cfg, ctx->task_timeout);
+
+ ctx->resolver = rspamd_dns_resolver_init(worker->srv->logger,
+ ctx->event_loop,
+ worker->srv->cfg);
+ rspamd_upstreams_library_config(worker->srv->cfg, ctx->cfg->ups_ctx,
+ ctx->event_loop, ctx->resolver->r);
+
+ ctx->http_ctx = rspamd_http_context_create(ctx->cfg, ctx->event_loop,
+ ctx->cfg->ups_ctx);
+ rspamd_mempool_add_destructor(ctx->cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_http_context_free,
+ ctx->http_ctx);
+ rspamd_worker_init_scanner(worker, ctx->event_loop, ctx->resolver,
+ &ctx->lang_det);
+
+ is_controller = rspamd_worker_check_controller_presence(worker);
+
+ if (is_controller) {
+ rspamd_worker_init_controller(worker, NULL);
+ }
+ else {
+ rspamd_map_watch(worker->srv->cfg, ctx->event_loop, ctx->resolver,
+ worker, RSPAMD_MAP_WATCH_SCANNER);
+ }
+
+ rspamd_lua_run_postloads(ctx->cfg->lua_state, ctx->cfg, ctx->event_loop,
+ worker);
+
+ ev_loop(ctx->event_loop, 0);
+ rspamd_worker_block_signals();
+
+ if (is_controller) {
+ rspamd_controller_on_terminate(worker, NULL);
+ }
+
+ rspamd_stat_close();
+ REF_RELEASE(ctx->cfg);
+ rspamd_log_close(worker->srv->logger);
+ rspamd_unset_crash_handler(worker->srv);
+
+ exit(EXIT_SUCCESS);
+}
diff --git a/src/worker_private.h b/src/worker_private.h
new file mode 100644
index 0000000..697961d
--- /dev/null
+++ b/src/worker_private.h
@@ -0,0 +1,72 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_WORKER_PRIVATE_H
+#define RSPAMD_WORKER_PRIVATE_H
+
+#include "config.h"
+#include "libcryptobox/cryptobox.h"
+#include "libcryptobox/keypair.h"
+#include "libserver/task.h"
+#include "libserver/cfg_file.h"
+#include "libserver/rspamd_control.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static const guint64 rspamd_worker_magic = 0xb48abc69d601dc1dULL;
+
+struct rspamd_lang_detector;
+
+struct rspamd_worker_ctx {
+ guint64 magic;
+ /* Events base */
+ struct ev_loop *event_loop;
+ /* DNS resolver */
+ struct rspamd_dns_resolver *resolver;
+ /* Config */
+ struct rspamd_config *cfg;
+
+ ev_tstamp timeout;
+ /* Detect whether this worker is mime worker */
+ gboolean is_mime;
+ /* Allow encrypted requests only using network */
+ gboolean encrypted_only;
+ /* Limit of tasks */
+ guint32 max_tasks;
+ /* Maximum time for task processing */
+ ev_tstamp task_timeout;
+ /* Encryption key */
+ struct rspamd_cryptobox_keypair *key;
+ /* Keys cache */
+ struct rspamd_http_context *http_ctx;
+ /* Language detector */
+ struct rspamd_lang_detector *lang_det;
+};
+
+/*
+ * Init scanning routines
+ */
+void rspamd_worker_init_scanner(struct rspamd_worker *worker,
+ struct ev_loop *ev_base,
+ struct rspamd_dns_resolver *resolver,
+ struct rspamd_lang_detector **plang_det);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/test/.depends b/test/.depends
new file mode 100644
index 0000000..fc75544
--- /dev/null
+++ b/test/.depends
@@ -0,0 +1,6 @@
+../src/mem_pool.c
+../src/hash.c
+../src/url.c
+../src/util.c
+../src/memcached.c
+../src/statfile.c
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
new file mode 100644
index 0000000..08e9556
--- /dev/null
+++ b/test/CMakeLists.txt
@@ -0,0 +1,50 @@
+SET(TESTSRC rspamd_mem_pool_test.c
+ rspamd_statfile_test.c
+ rspamd_url_test.c
+ rspamd_dns_test.c
+ rspamd_dkim_test.c
+ rspamd_rrd_test.c
+ rspamd_radix_test.c
+ rspamd_shingles_test.c
+ rspamd_upstream_test.c
+ rspamd_lua_pcall_vs_resume_test.c
+ rspamd_lua_test.c
+ rspamd_cryptobox_test.c
+ rspamd_heap_test.c
+ rspamd_test_suite.c)
+
+ADD_EXECUTABLE(rspamd-test EXCLUDE_FROM_ALL ${TESTSRC})
+SET_TARGET_PROPERTIES(rspamd-test PROPERTIES COMPILE_FLAGS "-DRSPAMD_TEST")
+ADD_DEPENDENCIES(rspamd-test rspamd-server)
+SET_TARGET_PROPERTIES(rspamd-test PROPERTIES LINKER_LANGUAGE CXX)
+TARGET_LINK_LIBRARIES(rspamd-test rspamd-server)
+
+SET(CXXTESTSSRC rspamd_cxx_unit.cxx)
+
+ADD_EXECUTABLE(rspamd-test-cxx EXCLUDE_FROM_ALL ${CXXTESTSSRC})
+SET_TARGET_PROPERTIES(rspamd-test-cxx PROPERTIES LINKER_LANGUAGE CXX)
+ADD_DEPENDENCIES(rspamd-test-cxx rspamd-server)
+TARGET_LINK_LIBRARIES(rspamd-test-cxx PRIVATE rspamd-server)
+SET_TARGET_PROPERTIES(rspamd-test-cxx PROPERTIES LINKER_LANGUAGE CXX)
+
+IF(NOT "${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}")
+ # Also add dependencies for convenience
+ FILE(GLOB_RECURSE LUA_TESTS CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/lua/*.*")
+ ADD_CUSTOM_TARGET(units-dir COMMAND
+ ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/lua/unit"
+ )
+ ADD_DEPENDENCIES(rspamd-test units-dir)
+ FOREACH(_LF IN LISTS LUA_TESTS)
+ GET_FILENAME_COMPONENT(_NM "${_LF}" NAME)
+ IF("${_LF}" MATCHES "^.*/unit/.*$")
+ SET(_DS "${CMAKE_CURRENT_BINARY_DIR}/lua/unit/${_NM}")
+ ELSE()
+ SET(_DS "${CMAKE_CURRENT_BINARY_DIR}/lua/${_NM}")
+ ENDIF()
+ ADD_CUSTOM_TARGET("${_NM}" COMMAND
+ ${CMAKE_COMMAND} -E copy_if_different ${_LF} ${_DS}
+ SOURCES "${_LF}"
+ )
+ ADD_DEPENDENCIES(rspamd-test "${_NM}")
+ ENDFOREACH()
+ENDIF() \ No newline at end of file
diff --git a/test/coverage.md b/test/coverage.md
new file mode 100644
index 0000000..8d177f3
--- /dev/null
+++ b/test/coverage.md
@@ -0,0 +1,58 @@
+Coverage collection explained
+=============================
+
+Hi mate. In short, you don't wanna know this. Believe me, you don't. Please, close this file and forget about it.
+
+Surely? You still here?
+
+Please, stop it until it's too late.
+
+You were warned.
+
+
+Preamble
+--------
+RSPAMD is written mainly in two languages: C and Lua. Coverage for each of them is being collected using different
+tools and approaches and is sent into [coveralls.io](https://coveralls.io).
+Each approach is not quite compatible to other tools. This document describes how we crutch them to work together.
+
+
+C coverage
+----------
+In general, pretty boring. When you run `cmake` with "-DENABLE_COVERAGE=ON" flag, it adds "--coverage" flag to both
+CFLAGS and LDFLAGS. So that each run of generated binary will create `*.gcda` file containing coverage data.
+
+However, there are some moment to highlight:
+
+- RSPAMD is run under "nobody" user. Hence, directories and files should be writable for this user.
+- To make it possible, we explicitly run `umask 0000` in "build" and "functional" stages in .circleci/config.yml
+- After run, we persist coverage data in "coverage.${CIRCLE\_JOB}.dump" during this build flow, see `capture_coverage_data`,
+ to use it on the final stage.
+- we use `cpp-coveralls` because it is able to save data for coveralls without actually sending it. We send on our own
+ along with Lua-coverage.
+
+Lua coverage
+------------
+Lua coverage is collected for unit-tests and functional test suite.
+First part contains nothing interesting, just see `test/lua/tests.lua`.
+
+"Functional" part is completely unobvious.
+
+1. Coverage collecting is initiated and dumped in `test/functional/lua/test_coverage.lua` (there are a lot of comments inside).
+ This file should be included on the very early stage of test run. Usually it's included via config.
+2. Coverage is dumped into ${TMPDIR}/%{worker_name}.luacov.stats.out
+3. All worker coverage reports are merged into `lua_coverage_report.json` (see `collect_lua_coverage()`)
+4. finally, `lua_coverage_report.json` is persisted in build flow (see `functional` stage)
+
+Altogether
+----------
+
+Finally, we get all the reports:
+
+- `coverage.functional.dump`
+- `coverage.rspamd-test.dump`
+- `lua_coverage_report.json`
+- `unit_test_lua.json`
+
+and merge them and send the resulting report using `test/functional/util/merge_coveralls.py`. Also, this scripts maps installed
+paths into corresponding repository paths and removes unneeded files (i.e. test sources).
diff --git a/test/functional/cases/001_merged/100_general.robot b/test/functional/cases/001_merged/100_general.robot
new file mode 100644
index 0000000..16b9b5f
--- /dev/null
+++ b/test/functional/cases/001_merged/100_general.robot
@@ -0,0 +1,61 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${GTUBE} ${RSPAMD_TESTDIR}/messages/gtube.eml
+${SETTINGS_NOSYMBOLS} {symbols_enabled = []}
+
+*** Test Cases ***
+GTUBE
+ Scan File ${GTUBE}
+ ... Settings=${SETTINGS_NOSYMBOLS}
+ Expect Symbol GTUBE
+
+GTUBE - Encrypted
+ ${result} = Run Rspamc -p -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_NORMAL} --key ${RSPAMD_KEY_PUB1}
+ ... ${GTUBE} --header=Settings=${SETTINGS_NOSYMBOLS}
+ Check Rspamc ${result} GTUBE (
+
+GTUBE - Scan File feature
+ Scan File By Reference ${GTUBE}
+ ... Settings=${SETTINGS_NOSYMBOLS}
+ Expect Symbol GTUBE
+
+GTUBE - Scan File feature (encoded)
+ ${encoded} = Encode Filename ${GTUBE}
+ Scan File By Reference ${encoded}
+ ... Settings=${SETTINGS_NOSYMBOLS}
+ Expect Symbol GTUBE
+
+GTUBE - SPAMC
+ ${result} = Spamc ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_NORMAL} ${GTUBE}
+ Should Contain ${result} GTUBE
+
+GTUBE - RSPAMC
+ ${result} = Rspamc ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_NORMAL} ${GTUBE}
+ Should Contain ${result} GTUBE
+
+EMAILS DETECTION 1
+ Scan File ${RSPAMD_TESTDIR}/messages/emails1.eml
+ ... URL-Format=Extended Settings=${SETTINGS_NOSYMBOLS}
+ Expect Email jim@example.net
+ Expect Email bob@example.net
+ Expect Email rupert@example.net
+
+EMAILS DETECTION ZEROFONT
+ Scan File ${RSPAMD_TESTDIR}/messages/zerofont.eml
+ ... Settings={symbols_enabled = [MANY_INVISIBLE_PARTS, ZERO_FONT]}
+ Expect Symbol MANY_INVISIBLE_PARTS
+ Expect Symbol ZERO_FONT
+
+HTML ONLY - TRUE POSITIVE
+ Scan File ${RSPAMD_TESTDIR}/messages/zerofont.eml
+ ... Settings={symbols_enabled = [MIME_HTML_ONLY]}
+ Expect Symbol MIME_HTML_ONLY
+
+HTML ONLY - TRUE NEGATIVE
+ Scan File ${RSPAMD_TESTDIR}/messages/btc.eml
+ ... Settings={symbols_enabled = [MIME_HTML_ONLY]}
+ Do Not Expect Symbol MIME_HTML_ONLY
diff --git a/test/functional/cases/001_merged/101_lua.robot b/test/functional/cases/001_merged/101_lua.robot
new file mode 100644
index 0000000..d1a771c
--- /dev/null
+++ b/test/functional/cases/001_merged/101_lua.robot
@@ -0,0 +1,50 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RSPAMD_MAP_MAP} ${RSPAMD_TESTDIR}/configs/maps/map.list
+${RSPAMD_RADIX_MAP} ${RSPAMD_TESTDIR}/configs/maps/ip2.list
+${RSPAMD_REGEXP_MAP} ${RSPAMD_TESTDIR}/configs/maps/regexp.list
+
+*** Test Cases ***
+Recipient Parsing Sanity
+ Scan File ${MESSAGE} Rcpt=rcpt1@foobar,rcpt2@foobar,rcpt3@foobar,rcpt4@foobar
+ ... Settings={symbols_enabled = [TEST_RCPT]}
+ Expect Symbol With Exact Options TEST_RCPT rcpt1@foobar,rcpt2@foobar,rcpt3@foobar,rcpt4@foobar
+
+TLD parts
+ Scan File ${MESSAGE} Settings={symbols_enabled = [TEST_TLD]}
+ Expect Symbol With Exact Options TEST_TLD no worry
+
+Hashes
+ Scan File ${MESSAGE} Settings={symbols_enabled = [TEST_HASHES]}
+ Expect Symbol With Exact Options TEST_HASHES no worry
+
+Maps Key Values
+ Scan File ${MESSAGE} Settings={symbols_enabled = [RADIX_KV, REGEXP_KV, MAP_KV]}
+ Expect Symbol With Exact Options RADIX_KV no worry
+ Expect Symbol With Exact Options REGEXP_KV no worry
+ Expect Symbol With Exact Options MAP_KV no worry
+
+Option Order
+ Scan File ${MESSAGE} Settings={symbols_enabled = [OPTION_ORDER, TBL_OPTION_ORDER]}
+ Expect Symbol With Exact Options OPTION_ORDER one two three 4 5 a
+ Expect Symbol With Exact Options TBL_OPTION_ORDER one two three 4 5 a
+
+Remove Result
+ Scan File ${MESSAGE} Settings={symbols_enabled = [REMOVE_RESULT_EXPECTED, REMOVE_RESULT_UNEXPECTED]}
+ Expect Symbol REMOVE_RESULT_EXPECTED
+ Do Not Expect Symbol REMOVE_RESULT_UNEXPECTED
+
+Rule conditions
+ Scan File ${MESSAGE} Settings={symbols_enabled = [ANY_A]}
+ Expect Symbol With Option ANY_A hello3
+ Expect Symbol With Option ANY_A hello1
+ Expect Symbol With Option ANY_A hello2
+
+External Maps Simple
+ Scan File ${MESSAGE} Settings={symbols_enabled = [EXTERNAL_MAP]}
+ Expect Symbol With Exact Options EXTERNAL_MAP +hello map \ No newline at end of file
diff --git a/test/functional/cases/001_merged/102_multimap.robot b/test/functional/cases/001_merged/102_multimap.robot
new file mode 100644
index 0000000..135f5ce
--- /dev/null
+++ b/test/functional/cases/001_merged/102_multimap.robot
@@ -0,0 +1,436 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${FREEMAIL_CC} ${RSPAMD_TESTDIR}/messages/freemailcc.eml
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RCVD1} ${RSPAMD_TESTDIR}/messages/received1.eml
+${RCVD2} ${RSPAMD_TESTDIR}/messages/received2.eml
+${RCVD3} ${RSPAMD_TESTDIR}/messages/received3.eml
+${RCVD4} ${RSPAMD_TESTDIR}/messages/received4.eml
+${URL1} ${RSPAMD_TESTDIR}/messages/url1.eml
+${URL2} ${RSPAMD_TESTDIR}/messages/url2.eml
+${URL3} ${RSPAMD_TESTDIR}/messages/url3.eml
+${URL4} ${RSPAMD_TESTDIR}/messages/url4.eml
+${URL5} ${RSPAMD_TESTDIR}/messages/url5.eml
+${URL_ICS} ${RSPAMD_TESTDIR}/messages/ics.eml
+${UTF_MESSAGE} ${RSPAMD_TESTDIR}/messages/utf.eml
+
+*** Test Cases ***
+URL_ICS
+ Scan File ${URL_ICS}
+ ... Settings={symbols_enabled = []}
+ Expect URL test.com
+
+MAP - DNSBL HIT
+ Scan File ${MESSAGE} IP=127.0.0.2
+ ... Settings={symbols_enabled = [DNSBL_MAP]}
+ Expect Symbol DNSBL_MAP
+
+MAP - DNSBL MISS
+ Scan File ${MESSAGE} IP=127.0.0.1
+ ... Settings={symbols_enabled = [DNSBL_MAP]}
+ Do Not Expect Symbol DNSBL_MAP
+
+MAP - IP HIT
+ Scan File ${MESSAGE} IP=127.0.0.1
+ ... Settings={symbols_enabled = [IP_MAP]}
+ Expect Symbol IP_MAP
+
+MAP - IP MISS
+ Scan File ${MESSAGE} IP=127.0.0.2
+ ... Settings={symbols_enabled = [IP_MAP]}
+ Do Not Expect Symbol IP_MAP
+
+MAP - IP MASK
+ Scan File ${MESSAGE} IP=10.1.0.10
+ ... Settings={symbols_enabled = [IP_MAP]}
+ Expect Symbol IP_MAP
+
+MAP - IP MASK MISS
+ Scan File ${MESSAGE} IP=11.1.0.10
+ ... Settings={symbols_enabled = [IP_MAP]}
+ Do Not Expect Symbol IP_MAP
+
+MAP - IP V6
+ Scan File ${MESSAGE} IP=::1
+ ... Settings={symbols_enabled = [IP_MAP]}
+ Expect Symbol IP_MAP
+
+MAP - IP V6 MISS
+ Scan File ${MESSAGE} IP=fe80::1
+ ... Settings={symbols_enabled = [IP_MAP]}
+ Do Not Expect Symbol IP_MAP
+
+MAP - FROM
+ Scan File ${MESSAGE} From=user@example.com
+ ... Settings={symbols_enabled = [FROM_MAP]}
+ Expect Symbol FROM_MAP
+
+MAP - COMBINED IP MASK FROM
+ Scan File ${MESSAGE} IP=10.1.0.10 From=user@example.com
+ ... Settings={symbols_enabled = [COMBINED_MAP_AND, COMBINED_MAP_OR]}
+ Expect Symbol With Score COMBINED_MAP_AND 10
+ Do Not Expect Symbol COMBINED_MAP_OR
+ Expect Action no action
+
+MAP - COMBINED IP MASK ONLY
+ Scan File ${MESSAGE} IP=10.1.0.10
+ ... Settings={symbols_enabled = [COMBINED_MAP_AND, COMBINED_MAP_OR]}
+ Do Not Expect Symbol COMBINED_MAP_AND
+ Expect Symbol COMBINED_MAP_OR
+
+MAP - COMBINED FROM ONLY
+ Scan File ${MESSAGE} From=user@example.com
+ ... Settings={symbols_enabled = [COMBINED_MAP_AND, COMBINED_MAP_OR]}
+ Do Not Expect Symbol COMBINED_MAP_AND
+ Expect Symbol COMBINED_MAP_OR
+
+MAP - COMBINED MISS
+ Scan File ${MESSAGE} IP=11.1.0.10 From=user@other.com
+ ... Settings={symbols_enabled = [COMBINED_MAP_AND, COMBINED_MAP_OR]}
+ Do Not Expect Symbol COMBINED_MAP_AND
+ Do Not Expect Symbol COMBINED_MAP_OR
+
+MAP - FROM MISS
+ Scan File ${MESSAGE} From=user@other.com
+ ... Settings={symbols_enabled = [FROM_MAP]}
+ Do Not Expect Symbol FROM_MAP
+
+MAP - FROM REGEXP
+ Scan File ${MESSAGE} From=user123@test.com
+ ... Settings={symbols_enabled = [REGEXP_MAP]}
+ Expect Symbol REGEXP_MAP
+ Scan File ${MESSAGE} From=somebody@example.com
+ ... Settings={symbols_enabled = [REGEXP_MAP]}
+ Expect Symbol REGEXP_MAP
+
+MAP - FROM REGEXP MISS
+ Scan File ${MESSAGE} From=user@other.org
+ ... Settings={symbols_enabled = [REGEXP_MAP]}
+ Do Not Expect Symbol REGEXP_MAP
+
+MAP - RCPT DOMAIN HIT
+ Scan File ${MESSAGE} Rcpt=user@example.com
+ ... Settings={symbols_enabled = [RCPT_DOMAIN]}
+ Expect Symbol RCPT_DOMAIN
+
+MAP - RCPT DOMAIN MISS
+ Scan File ${MESSAGE} Rcpt=example.com@user
+ ... Settings={symbols_enabled = [RCPT_DOMAIN]}
+ Do Not Expect Symbol RCPT_DOMAIN
+
+MAP - RCPT USER HIT
+ Scan File ${MESSAGE} Rcpt=bob@example.com
+ ... Settings={symbols_enabled = [RCPT_USER]}
+ Expect Symbol RCPT_USER
+
+MAP - RCPT USER MISS
+ Scan File ${MESSAGE} From=example.com@bob
+ ... Settings={symbols_enabled = [RCPT_USER]}
+ Do Not Expect Symbol RCPT_USER
+
+MAP - DEPENDS HIT
+ Scan File ${MESSAGE} IP=88.99.142.95 From=user123@rspamd.com
+ ... Settings={symbols_enabled = [DEPS_MAP,REGEXP_MAP,FROM_MAP,SPF_CHECK]}
+ Expect Symbol DEPS_MAP
+
+MAP - DEPENDS MISS
+ Scan File ${MESSAGE} IP=1.2.3.4 From=user123@rspamd.com
+ ... Settings={symbols_enabled = [DEPS_MAP,REGEXP_MAP,FROM_MAP,SPF_CHECK]}
+ Do Not Expect Symbol DEPS_MAP
+
+MAP - MULSYM PLAIN
+ Scan File ${MESSAGE} Rcpt=user1@example.com
+ ... Settings={symbols_enabled = [RCPT_MAP, SYM1]}
+ Expect Symbol RCPT_MAP
+
+MAP - MULSYM SCORE
+ Scan File ${MESSAGE} Rcpt=user2@example.com
+ ... Settings={symbols_enabled = [RCPT_MAP, SYM1]}
+ Expect Symbol With Score RCPT_MAP 10.0
+
+MAP - MULSYM SYMBOL
+ Scan File ${MESSAGE} Rcpt=user3@example.com
+ ... Settings={symbols_enabled = [RCPT_MAP, SYM1]}
+ Expect Symbol With Score SYM1 1.0
+
+MAP - MULSYM SYMBOL MISS
+ Scan File ${MESSAGE} Rcpt=user4@example.com
+ ... Settings={symbols_enabled = [RCPT_MAP, SYM1]}
+ Expect Symbol With Score RCPT_MAP 1.0
+
+MAP - MULSYM SYMBOL + SCORE
+ Scan File ${MESSAGE} Rcpt=user5@example.com
+ ... Settings={symbols_enabled = [RCPT_MAP, SYM1]}
+ Expect Symbol With Score SYM1 -10.1
+
+MAP - MULSYM SYMBOL + SCORE + OPTS
+ Scan File ${MESSAGE} Rcpt=user6@example.com
+ ... Settings={symbols_enabled = [RCPT_MAP, SYM1]}
+ Expect Symbol With Score And Exact Options SYM1 -10.1 opt1 opt2
+
+MAP - UTF
+ Scan File ${UTF_MESSAGE}
+ ... Settings={symbols_enabled = [HEADER_MAP]}
+ Expect Symbol HEADER_MAP
+
+MAP - UTF MISS
+ Scan File ${MESSAGE}
+ ... Settings={symbols_enabled = [HEADER_MAP]}
+ Do Not Expect Symbol HEADER_MAP
+
+MAP - HOSTNAME
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=example.com
+ ... Settings={symbols_enabled = [HOSTNAME_MAP]}
+ Expect Symbol HOSTNAME_MAP
+
+MAP - HOSTNAME MISS
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=rspamd.com
+ ... Settings={symbols_enabled = [HOSTNAME_MAP]}
+ Do Not Expect Symbol HOSTNAME_MAP
+
+MAP - TOP
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=example.com.au
+ ... Settings={symbols_enabled = [HOSTNAME_TOP_MAP]}
+ Expect Symbol HOSTNAME_TOP_MAP
+
+MAP - TOP MISS
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=example.com.bg
+ ... Settings={symbols_enabled = [HOSTNAME_TOP_MAP]}
+ Do Not Expect Symbol HOSTNAME_TOP_MAP
+
+MAP - CDB - HOSTNAME
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=example.com
+ ... Settings={symbols_enabled = [CDB_HOSTNAME]}
+ Expect Symbol CDB_HOSTNAME
+
+MAP - CDB - HOSTNAME MISS
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=rspamd.com
+ ... Settings={symbols_enabled = [CDB_HOSTNAME]}
+ Do Not Expect Symbol CDB_HOSTNAME
+
+MAP - REDIS - HOSTNAME
+ Redis HSET hostname redistest.example.net ${EMPTY}
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=redistest.example.net
+ ... Settings={symbols_enabled = [REDIS_HOSTNAME]}
+ Expect Symbol REDIS_HOSTNAME
+
+MAP - REDIS - HOSTNAME MISS
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=rspamd.com
+ ... Settings={symbols_enabled = [REDIS_HOSTNAME]}
+ Do Not Expect Symbol REDIS_HOSTNAME
+
+MAP - REDIS - HOSTNAME - EXPANSION - HIT
+ Redis HSET 127.0.0.1.foo.com redistest.example.net ${EMPTY}
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=redistest.example.net Rcpt=bob@foo.com
+ ... Settings={symbols_enabled = [REDIS_HOSTNAME_EXPANSION]}
+ Expect Symbol REDIS_HOSTNAME_EXPANSION
+
+MAP - REDIS - HOSTNAME - EXPANSION - MISS
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=redistest.example.net Rcpt=bob@bar.com
+ ... Settings={symbols_enabled = [REDIS_HOSTNAME_EXPANSION]}
+ Do Not Expect Symbol REDIS_HOSTNAME_EXPANSION
+
+MAP - REDIS - IP
+ Redis HSET ipaddr 127.0.0.1 ${EMPTY}
+ Scan File ${MESSAGE} IP=127.0.0.1
+ ... Settings={symbols_enabled = [REDIS_IPADDR]}
+ Expect Symbol REDIS_IPADDR
+
+MAP - REDIS - IP - MISS
+ Scan File ${MESSAGE} IP=8.8.8.8
+ ... Settings={symbols_enabled = [REDIS_IPADDR]}
+ Do Not Expect Symbol REDIS_IPADDR
+
+MAP - REDIS - FROM
+ Redis HSET emailaddr from@rspamd.tk ${EMPTY}
+ Scan File ${MESSAGE} From=from@rspamd.tk
+ ... Settings={symbols_enabled = [REDIS_FROMADDR]}
+ Expect Symbol REDIS_FROMADDR
+
+MAP - REDIS - FROM MISS
+ Scan File ${MESSAGE} From=user@other.com
+ ... Settings={symbols_enabled = [REDIS_FROMADDR]}
+ Do Not Expect Symbol REDIS_FROMADDR
+
+MAP - REDIS - URL TLD - HIT
+ Redis HSET hostname example.com ${EMPTY}
+ Scan File ${URL1}
+ ... Settings={symbols_enabled = [REDIS_URL_TLD]}
+ Expect Symbol REDIS_URL_TLD
+
+MAP - REDIS - URL TLD - MISS
+ Scan File ${URL2}
+ ... Settings={symbols_enabled = [REDIS_URL_TLD]}
+ Do Not Expect Symbol REDIS_URL_TLD
+
+MAP - REDIS - URL RE FULL - HIT
+ Redis HSET fullurlre html ${EMPTY}
+ Scan File ${URL2}
+ ... Settings={symbols_enabled = [REDIS_URL_RE_FULL]}
+ Expect Symbol REDIS_URL_RE_FULL
+
+MAP - REDIS - URL RE FULL - MISS
+ Scan File ${URL1}
+ ... Settings={symbols_enabled = [REDIS_URL_RE_FULL]}
+ Do Not Expect Symbol REDIS_URL_RE_FULL
+
+MAP - REDIS - URL FULL - HIT
+ Redis HSET fullurl https://www.example.com/foo?a=b ${EMPTY}
+ Scan File ${URL1}
+ ... Settings={symbols_enabled = [REDIS_URL_FULL]}
+ Expect Symbol REDIS_URL_FULL
+
+MAP - REDIS - URL FULL - MISS
+ Scan File ${URL2}
+ ... Settings={symbols_enabled = [REDIS_URL_FULL]}
+ Do Not Expect Symbol REDIS_URL_FULL
+
+MAP - REDIS - URL PHISHED - HIT
+ Redis HSET phishedurl www.rspamd.com ${EMPTY}
+ Scan File ${URL3}
+ ... Settings={symbols_enabled = [REDIS_URL_PHISHED]}
+ Expect Symbol REDIS_URL_PHISHED
+
+MAP - REDIS - URL PHISHED - MISS
+ Scan File ${URL4}
+ ... Settings={symbols_enabled = [REDIS_URL_PHISHED]}
+ Do Not Expect Symbol REDIS_URL_PHISHED
+
+MAP - REDIS - URL PLAIN REGEX - HIT
+ Redis HSET urlre www ${EMPTY}
+ Scan File ${URL3}
+ ... Settings={symbols_enabled = [REDIS_URL_RE_PLAIN]}
+ Expect Symbol REDIS_URL_RE_PLAIN
+
+MAP - REDIS - URL PLAIN REGEX - MISS
+ Scan File ${URL4}
+ ... Settings={symbols_enabled = [REDIS_URL_RE_PLAIN]}
+ Do Not Expect Symbol REDIS_URL_RE_PLAIN
+
+MAP - REDIS - URL TLD REGEX - HIT
+ Redis HSET tldre net ${EMPTY}
+ Scan File ${URL5}
+ ... Settings={symbols_enabled = [REDIS_URL_RE_TLD]}
+ Expect Symbol REDIS_URL_RE_TLD
+
+MAP - REDIS - URL TLD REGEX - MISS
+ Scan File ${URL4}
+ ... Settings={symbols_enabled = [REDIS_URL_RE_TLD]}
+ Do Not Expect Symbol REDIS_URL_RE_TLD
+
+MAP - REDIS - URL NOFILTER - HIT
+ Redis HSET urlnofilter www.example.net ${EMPTY}
+ Scan File ${URL5}
+ ... Settings={symbols_enabled = [REDIS_URL_NOFILTER]}
+ Expect Symbol REDIS_URL_NOFILTER
+
+MAP - REDIS - URL NOFILTER - MISS
+ Scan File ${URL4}
+ ... Settings={symbols_enabled = [REDIS_URL_NOFILTER]}
+ Do Not Expect Symbol REDIS_URL_NOFILTER
+
+MAP - REDIS - ASN - HIT
+ Redis HSET asn 15169 ${EMPTY}
+ Scan File ${MESSAGE} IP=8.8.8.8
+ ... Settings={symbols_enabled = [REDIS_ASN, ASN_CHECK]}
+ Expect Symbol REDIS_ASN
+
+MAP - REDIS - ASN - MISS
+ Scan File ${MESSAGE} IP=46.228.47.114
+ ... Settings={symbols_enabled = [REDIS_ASN, ASN_CHECK]}
+ Do Not Expect Symbol REDIS_ASN
+
+MAP - REDIS - CC - HIT
+ Redis HSET cc US ${EMPTY}
+ Scan File ${MESSAGE} IP=8.8.8.8
+ ... Settings={symbols_enabled = [REDIS_COUNTRY, ASN_CHECK]}
+ Expect Symbol REDIS_COUNTRY
+
+MAP - REDIS - CC - MISS
+ Scan File ${MESSAGE} IP=46.228.47.114
+ ... Settings={symbols_enabled = [REDIS_COUNTRY, ASN_CHECK]}
+ Do Not Expect Symbol REDIS_COUNTRY
+
+MAP - REDIS - ASN FILTERED - HIT
+ Redis HSET asn 1 ${EMPTY}
+ Scan File ${MESSAGE} IP=8.8.8.8
+ ... Settings={symbols_enabled = [REDIS_ASN_FILTERED, ASN_CHECK]}
+ Expect Symbol REDIS_ASN_FILTERED
+
+MAP - REDIS - ASN FILTERED - MISS
+ Scan File ${MESSAGE} IP=46.228.47.114
+ ... Settings={symbols_enabled = [REDIS_ASN_FILTERED, ASN_CHECK]}
+ Do Not Expect Symbol REDIS_ASN_FILTERED
+
+MAP - RECEIVED - IP MINMAX POS - ONE
+ Scan File ${RCVD1}
+ ... Settings={symbols_enabled = [RCVD_TEST_01, RCVD_TEST02]}
+ Expect Symbol RCVD_TEST_01
+ Do Not Expect Symbol RCVD_TEST_02
+
+# Relies on parsing of shitty received
+#MAP - RECEIVED - IP MINMAX POS - TWO / RCVD_AUTHED_ONE HIT
+# Scan File ${RCVD2}
+# Expect Symbol RCVD_TEST_02
+# Do Not Expect Symbol RCVD_TEST_01
+# Expect Symbol RCVD_AUTHED_ONE
+
+MAP - RECEIVED - REDIS
+ Redis HSET RCVD_TEST 2a01:7c8:aab6:26d:5054:ff:fed1:1da2 ${EMPTY}
+ Scan File ${RCVD1}
+ ... Settings={symbols_enabled = [RCVD_TEST_REDIS_01]}
+ Expect Symbol RCVD_TEST_REDIS_01
+
+RCVD_AUTHED_ONE & RCVD_AUTHED_TWO - MISS
+ Scan File ${RCVD3}
+ ... Settings={symbols_enabled = [RCVD_AUTHED_ONE, RCVD_AUTHED_TWO]}
+ Do Not Expect Symbol RCVD_AUTHED_ONE
+ Do Not Expect Symbol RCVD_AUTHED_TWO
+
+RCVD_AUTHED_TWO HIT / RCVD_AUTHED_ONE MISS
+ Scan File ${RCVD4}
+ ... Settings={symbols_enabled = [RCVD_AUTHED_ONE, RCVD_AUTHED_TWO]}
+ Expect Symbol RCVD_AUTHED_TWO
+ Do Not Expect Symbol RCVD_AUTHED_ONE
+
+FREEMAIL_CC
+ Scan File ${FREEMAIL_CC}
+ ... Settings={symbols_enabled = [FREEMAIL_CC]}
+ Expect Symbol With Score And Exact Options FREEMAIL_CC 19.00 test.com test1.com test2.com test3.com test4.com test5.com test6.com test7.com test8.com test9.com test10.com test11.com test12.com test13.com test14.com
+
+MAP - MULTISYMBOL DISABLED
+ Scan File ${MESSAGE} Rcpt=user3@example.com
+ ... Settings={symbols_enabled = [RCPT_MAP_NOMULTISYM, SYM1]}
+ Expect Symbol With Exact Options RCPT_MAP_NOMULTISYM user3@example.com SYM1
+
+MAP - EXTERNAL
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=example.com.au
+ ... Settings={symbols_enabled = [EXTERNAL_MULTIMAP]}
+ Expect Symbol EXTERNAL_MULTIMAP
+
+MAP - EXTERNAL MISS
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=example.com.bg
+ ... Settings={symbols_enabled = [EXTERNAL_MULTIMAP]}
+ Do Not Expect Symbol EXTERNAL_MULTIMAP
+
+MAP - DYNAMIC SYMBOLS - SYM1
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=foo
+ ... Settings={symbols_enabled = [DYN_TEST1,DYN_TEST2,DYN_MULTIMAP]}
+ Expect Symbol DYN_TEST1
+ Do Not Expect Symbol DYN_TEST2
+
+MAP - DYNAMIC SYMBOLS - SYM2
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=bar
+ ... Settings={symbols_enabled = [DYN_TEST1,DYN_TEST2,DYN_MULTIMAP]}
+ Expect Symbol DYN_TEST2
+ Do Not Expect Symbol DYN_TEST1
+
+MAP - DYNAMIC SYMBOLS - MISS
+ Scan File ${MESSAGE} IP=127.0.0.1 Hostname=baz
+ ... Settings={symbols_enabled = [DYN_TEST1,DYN_TEST2,DYN_MULTIMAP]}
+ Do Not Expect Symbol DYN_TEST2
+ Do Not Expect Symbol DYN_TEST1
diff --git a/test/functional/cases/001_merged/104_get_from.robot b/test/functional/cases/001_merged/104_get_from.robot
new file mode 100644
index 0000000..f01119f
--- /dev/null
+++ b/test/functional/cases/001_merged/104_get_from.robot
@@ -0,0 +1,57 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${OPTIONS1} ,user@example.org,user,example.org
+${OPTIONS2} First Last,user@example.org,user,example.org
+${OPTIONS3} First M. Last,user@example.org,user,example.org
+${SETTINGS_GETFROM} {symbols_enabled = [${SYMBOL}]}
+${SYMBOL} GET_FROM
+
+*** Test Cases ***
+task:get_from('mime') - address only
+ Scan File ${RSPAMD_TESTDIR}/messages/from/from.eml
+ ... Settings=${SETTINGS_GETFROM}
+ Expect Symbol ${SYMBOL}
+
+task:get_from('mime') - comment
+ Scan File ${RSPAMD_TESTDIR}/messages/from/from_comment.eml
+ ... Settings=${SETTINGS_GETFROM}
+ Expect Symbol With Exact Options ${SYMBOL} ${OPTIONS1}
+
+task:get_from('mime') - display name
+ Scan File ${RSPAMD_TESTDIR}/messages/from/from_dn.eml
+ ... Settings=${SETTINGS_GETFROM}
+ Expect Symbol With Exact Options ${SYMBOL} ${OPTIONS2}
+
+task:get_from('mime') - display name Base64
+ Scan File ${RSPAMD_TESTDIR}/messages/from/from_dn_base64.eml
+ ... Settings=${SETTINGS_GETFROM}
+ Expect Symbol With Exact Options ${SYMBOL} Кириллица,user@example.org,user,example.org
+
+task:get_from('mime') - display name and comment
+ Scan File ${RSPAMD_TESTDIR}/messages/from/from_dn_comment.eml
+ ... Settings=${SETTINGS_GETFROM}
+ Expect Symbol With Exact Options ${SYMBOL} ${OPTIONS2}
+
+task:get_from('mime') - quoted display name
+ Scan File ${RSPAMD_TESTDIR}/messages/from/from_quoted_dn.eml
+ ... Settings=${SETTINGS_GETFROM}
+ Expect Symbol With Exact Options ${SYMBOL} ${OPTIONS3}
+
+task:get_from('mime') - quoted display name and comment
+ Scan File ${RSPAMD_TESTDIR}/messages/from/from_quoted_dn_comment.eml
+ ... Settings=${SETTINGS_GETFROM}
+ Expect Symbol With Exact Options ${SYMBOL} ${OPTIONS3}
+
+task:get_from('mime') - quoted in the middle of DN (outer spaces)
+ Scan File ${RSPAMD_TESTDIR}/messages/from/from_quoted_dn_middle.eml
+ ... Settings=${SETTINGS_GETFROM}
+ Expect Symbol With Exact Options ${SYMBOL} ${OPTIONS3}
+
+task:get_from('mime') - quoted in the middle of DN (inner spaces)
+ Scan File ${RSPAMD_TESTDIR}/messages/from/from_quoted_dn_middle_inner.eml
+ ... Settings=${SETTINGS_GETFROM}
+ Expect Symbol With Exact Options ${SYMBOL} ${OPTIONS3}
diff --git a/test/functional/cases/001_merged/105_mimetypes.robot b/test/functional/cases/001_merged/105_mimetypes.robot
new file mode 100644
index 0000000..e16150a
--- /dev/null
+++ b/test/functional/cases/001_merged/105_mimetypes.robot
@@ -0,0 +1,74 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${SETTINGS_MIMETYPES} {symbols_enabled = [MIME_TYPES_CALLBACK]}
+
+*** Test Cases ***
+Zip
+ Scan File ${RSPAMD_TESTDIR}/messages/zip.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Expect Symbol With Exact Options MIME_BAD_EXTENSION exe
+
+Zip Double Bad Extension
+ Scan File ${RSPAMD_TESTDIR}/messages/zip-doublebad.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Expect Symbol With Exact Options MIME_DOUBLE_BAD_EXTENSION .pdf.exe
+
+Next-to-last Double Bad Extension
+ Scan File ${RSPAMD_TESTDIR}/messages/next2last-doublebad.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Expect Symbol With Exact Options MIME_DOUBLE_BAD_EXTENSION .scr.xz
+
+Date is followed by Bad Extension
+ Scan File ${RSPAMD_TESTDIR}/messages/rar-date-bad-ext.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Expect Symbol With Exact Options MIME_BAD_EXTENSION scr
+ Do Not Expect Symbol MIME_DOUBLE_BAD_EXTENSION
+
+Dotted file name is followed by Bad Extension
+ Scan File ${RSPAMD_TESTDIR}/messages/bad_ext.dotted_file_name.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Expect Symbol With Exact Options MIME_BAD_EXTENSION exe
+ Do Not Expect Symbol MIME_DOUBLE_BAD_EXTENSION
+
+Dotted numbers in parentheses is followed by Bad Extension
+ Scan File ${RSPAMD_TESTDIR}/messages/next2last-digits_in_parens.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Expect Symbol With Exact Options MIME_BAD_EXTENSION msi
+ Do Not Expect Symbol MIME_DOUBLE_BAD_EXTENSION
+
+Dotted numbers in square brackets is followed by Bad Extension
+ Scan File ${RSPAMD_TESTDIR}/messages/next2last-digits_in_brackets.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Expect Symbol With Exact Options MIME_BAD_EXTENSION msi
+ Do Not Expect Symbol MIME_DOUBLE_BAD_EXTENSION
+
+Rar4
+ Scan File ${RSPAMD_TESTDIR}/messages/rar4.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Expect Symbol With Exact Options MIME_BAD_EXTENSION exe
+
+Cloaked Archive Extension
+ Scan File ${RSPAMD_TESTDIR}/messages/f.zip.gz.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Expect Symbol With Exact Options MIME_ARCHIVE_IN_ARCHIVE .zip.gz zip
+
+Multipart Archive Extension
+ Scan File ${RSPAMD_TESTDIR}/messages/f.zip.001.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Do Not Expect Symbol MIME_ARCHIVE_IN_ARCHIVE
+
+Exe file, but name in filename_whitelist
+ Scan File ${RSPAMD_TESTDIR}/messages/exe_attm.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Do Not Expect Symbol MIME_BAD_EXTENSION
+ Do Not Expect Symbol MIME_BAD_ATTACHMENT
+ Do Not Expect Symbol MIME_DOUBLE_BAD_EXTENSION
+
+Empty text part should not be treat as html
+ Scan File ${RSPAMD_TESTDIR}/messages/empty-plain-text.eml
+ ... Settings=${SETTINGS_MIMETYPES}
+ Do Not Expect Symbol FORGED_OUTLOOK_HTML
diff --git a/test/functional/cases/001_merged/106_mid.robot b/test/functional/cases/001_merged/106_mid.robot
new file mode 100644
index 0000000..c5510bb
--- /dev/null
+++ b/test/functional/cases/001_merged/106_mid.robot
@@ -0,0 +1,36 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${SETTINGS_MID} {symbols_enabled = [DKIM_CHECK,INVALID_MSGID,INVALID_MSGID_ALLOWED,KNOWN_NO_MID,KNOWN_MID,MISSING_MID,MISSING_MID_ALLOWED]}
+
+*** Test Cases ***
+MID - invalid Message-ID
+ Scan File ${RSPAMD_TESTDIR}/messages/fws_fp.eml
+ ... Settings=${SETTINGS_MID}
+ Expect Symbol With Score INVALID_MSGID 1.70
+ Do Not Expect Symbol MISSING_MID
+ Do Not Expect Symbol INVALID_MSGID_ALLOWED
+
+MID - invalid Message-ID allowed
+ Scan File ${RSPAMD_TESTDIR}/messages/invalid_mid_allowed.eml
+ ... Settings=${SETTINGS_MID}
+ Expect Symbol With Score INVALID_MSGID_ALLOWED 0.00
+ Do Not Expect Symbol MISSING_MID
+ Do Not Expect Symbol INVALID_MSGID
+
+MID - missing Message-ID
+ Scan File ${RSPAMD_TESTDIR}/messages/freemail.eml
+ ... Settings=${SETTINGS_MID}
+ Expect Symbol With Score MISSING_MID 2.50
+ Do Not Expect Symbol MISSING_MID_ALLOWED
+ Do Not Expect Symbol INVALID_MSGID
+
+MID - missing Message-ID allowed
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/onsubdomain_pass_relaxed.eml
+ ... Settings=${SETTINGS_MID}
+ Expect Symbol With Score MISSING_MID_ALLOWED 0.00
+ Do Not Expect Symbol MISSING_MID
+ Do Not Expect Symbol INVALID_MSGID
diff --git a/test/functional/cases/001_merged/114_phishing.robot b/test/functional/cases/001_merged/114_phishing.robot
new file mode 100644
index 0000000..bc7f398
--- /dev/null
+++ b/test/functional/cases/001_merged/114_phishing.robot
@@ -0,0 +1,26 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE1} ${RSPAMD_TESTDIR}/messages/phishing1.eml
+${MESSAGE2} ${RSPAMD_TESTDIR}/messages/phishing2.eml
+${MESSAGE3} ${RSPAMD_TESTDIR}/messages/phishing3.eml
+${SETTINGS_PHISHING} {symbols_enabled = [PHISHING,STRICT_PHISHING,STRICTER_PHISHING]}
+
+*** Test Cases ***
+TEST PHISHING
+ Scan File ${MESSAGE1}
+ ... Settings=${SETTINGS_PHISHING}
+ Expect Symbol PHISHING
+
+TEST PHISHING STRICT ONE
+ Scan File ${MESSAGE2}
+ ... Settings=${SETTINGS_PHISHING}
+ Expect Symbol STRICT_PHISHING
+
+TEST PHISHING STRICT TWO
+ Scan File ${MESSAGE3}
+ ... Settings=${SETTINGS_PHISHING}
+ Expect Symbol STRICTER_PHISHING
diff --git a/test/functional/cases/001_merged/115_dmarc.robot b/test/functional/cases/001_merged/115_dmarc.robot
new file mode 100644
index 0000000..e5ed670
--- /dev/null
+++ b/test/functional/cases/001_merged/115_dmarc.robot
@@ -0,0 +1,101 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${DMARC_SETTINGS} {symbols_enabled = [DMARC_CHECK, DKIM_CHECK, SPF_CHECK]}
+
+*** Test Cases ***
+DMARC NONE PASS DKIM
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/pass_none.eml
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_ALLOW
+
+DMARC NONE PASS SPF
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/fail_none.eml
+ ... IP=8.8.4.4 From=foo@spf.cacophony.za.org
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_ALLOW
+
+DMARC NONE FAIL
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/fail_none.eml
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_SOFTFAIL
+
+DMARC REJECT FAIL
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/fail_reject.eml
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_REJECT
+
+DMARC QUARANTINE FAIL
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/fail_quarantine.eml
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_QUARANTINE
+
+DMARC SP NONE FAIL
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/subdomain_fail_none.eml
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_SOFTFAIL
+
+DMARC SP REJECT FAIL
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/subdomain_fail_reject.eml
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_REJECT
+
+DMARC SP QUARANTINE FAIL
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/subdomain_fail_quarantine.eml
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_QUARANTINE
+
+DMARC SUBDOMAIN FAIL DKIM STRICT ALIGNMENT
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/onsubdomain_fail_alignment.eml
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_REJECT
+
+DMARC SUBDOMAIN PASS DKIM RELAXED ALIGNMENT
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/onsubdomain_pass_relaxed.eml
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_ALLOW
+
+DMARC SUBDOMAIN PASS SPF STRICT ALIGNMENT
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/onsubdomain_fail_alignment.eml
+ ... IP=37.48.67.26 From=foo@yo.mom.za.org
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_ALLOW
+
+DMARC SUBDOMAIN FAIL SPF STRICT ALIGNMENT
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/onsubdomain_fail_alignment.eml
+ ... IP=37.48.67.26 From=foo@mom.za.org
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_REJECT
+
+DMARC SUBDOMAIN PASS SPF RELAXED ALIGNMENT
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/onsubdomain_fail.eml
+ ... IP=37.48.67.26 From=foo@mom.za.org
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_ALLOW
+
+DMARC DNSFAIL
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/dmarc_tmpfail.eml
+ ... IP=37.48.67.26 From=foo@mom.za.org
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_DNSFAIL
+
+DMARC NA NXDOMAIN
+ Scan File ${RSPAMD_TESTDIR}/messages/utf.eml
+ ... IP=37.48.67.26 From=foo@mom.za.org
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_NA
+
+DMARC PCT ZERO REJECT
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/pct_none.eml
+ ... IP=37.48.67.26 From=foo@mom.za.org
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_QUARANTINE
+
+DMARC PCT ZERO SP QUARANTINE
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/pct_none1.eml
+ ... IP=37.48.67.26 From=foo@mom.za.org
+ ... Settings=${DMARC_SETTINGS}
+ Expect Symbol DMARC_POLICY_SOFTFAIL
diff --git a/test/functional/cases/001_merged/117_spf.robot b/test/functional/cases/001_merged/117_spf.robot
new file mode 100644
index 0000000..dda35f6
--- /dev/null
+++ b/test/functional/cases/001_merged/117_spf.robot
@@ -0,0 +1,157 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${SETTINGS_SPF} {symbols_enabled = [SPF_CHECK]}
+
+*** Test Cases ***
+SPF FAIL UNRESOLVEABLE INCLUDE
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=37.48.67.26 From=x@fail3.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_FAIL
+
+SPF DNSFAIL FAILED INCLUDE UNALIGNED
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@fail2.org.org.za
+ ... Settings={symbols_enabled = [SPF_CHECK,DKIM_CHECK,DMARC_CHECK]}
+ Expect Symbol R_SPF_DNSFAIL
+ Expect Symbol DMARC_POLICY_SOFTFAIL
+
+SPF ALLOW UNRESOLVEABLE INCLUDE
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@fail3.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_ALLOW
+
+SPF ALLOW FAILED INCLUDE
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.4.4 From=x@fail2.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_ALLOW
+
+SPF NA NA
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_NA
+
+SPF NA NOREC
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@co.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_NA
+
+SPF NA NXDOMAIN
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@zzzzaaaa
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_NA
+
+SPF PERMFAIL UNRESOLVEABLE REDIRECT
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@fail4.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_PERMFAIL
+
+SPF REDIRECT NO USEABLE ELEMENTS
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@fail10.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_PERMFAIL
+
+SPF DNSFAIL FAILED REDIRECT
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@fail1.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_DNSFAIL
+
+SPF PERMFAIL NO USEABLE ELEMENTS
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@fail5.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_PERMFAIL
+
+SPF FAIL
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@example.net
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_FAIL
+
+SPF FAIL UNRESOLVEABLE MX
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=1.2.3.4 From=x@fail6.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_FAIL
+
+SPF FAIL UNRESOLVEABLE A
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=1.2.3.4 From=x@fail7.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_FAIL
+
+SPF DNSFAIL FAILED A
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=1.2.3.4 From=x@fail8.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_DNSFAIL
+
+SPF DNSFAIL FAILED MX
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=1.2.3.4 From=x@fail9.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_DNSFAIL
+
+SPF DNSFAIL FAILED RECORD
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=1.2.3.4 From=x@www.dnssec-failed.org
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_DNSFAIL
+
+SPF PASS INCLUDE
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@pass1.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_ALLOW
+
+SPF PTRS
+ Scan File /dev/null
+ ... IP=88.99.142.95 From=foo@crazyspf.cacophony.za.org
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_ALLOW
+ Scan File /dev/null
+ ... IP=128.66.0.1 From=foo@crazyspf.cacophony.za.org
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_FAIL
+ Scan File /dev/null
+ ... IP=209.85.216.182 From=foo@crazyspf.cacophony.za.org
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_FAIL
+ #Scan File /dev/null
+ #... IP=98.138.91.166 From=foo@crazyspf.cacophony.za.org
+ #Expect Symbol R_SPF_ALLOW
+ #Scan File /dev/null
+ #... IP=98.138.91.167 From=foo@crazyspf.cacophony.za.org
+ #Expect Symbol R_SPF_ALLOW
+ #Scan File /dev/null
+ #... IP=98.138.91.168 From=foo@crazyspf.cacophony.za.org
+ #Expect Symbol R_SPF_ALLOW
+
+SPF PERMFAIL REDIRECT WITHOUT SPF
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim4.eml
+ ... IP=192.0.2.1 From=a@fail1.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_DNSFAIL
+
+SPF EXTERNAL RELAY
+ Scan File ${RSPAMD_TESTDIR}/messages/external_relay.eml
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol With Score And Exact Options R_SPF_ALLOW -0.2 +ip4:37.48.67.26
+
+SPF UPPERCASE
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=8.8.8.8 From=x@fail11.org.org.za
+ ... Settings=${SETTINGS_SPF}
+ Expect Symbol R_SPF_ALLOW
diff --git a/test/functional/cases/001_merged/160_antivirus.robot b/test/functional/cases/001_merged/160_antivirus.robot
new file mode 100644
index 0000000..0870ba6
--- /dev/null
+++ b/test/functional/cases/001_merged/160_antivirus.robot
@@ -0,0 +1,151 @@
+*** Settings ***
+Suite Teardown Antivirus Teardown
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE2} ${RSPAMD_TESTDIR}/messages/freemail.eml
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${SETTINGS_AVAST} {symbols_enabled = [AVAST_VIRUS]}
+${SETTINGS_CLAM} {symbols_enabled = [CLAM_VIRUS]}
+${SETTINGS_FPROT} {symbols_enabled = [FPROT_VIRUS, FPROT2_VIRUS_DUPLICATE_DEFAULT]}
+
+*** Test Cases ***
+CLAMAV MISS
+ Run Dummy Clam ${RSPAMD_PORT_CLAM}
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_CLAM}
+ Do Not Expect Symbol CLAM_VIRUS
+ Shutdown clamav
+
+CLAMAV HIT
+ Run Dummy Clam ${RSPAMD_PORT_CLAM} 1
+ Scan File ${MESSAGE2}
+ ... Settings=${SETTINGS_CLAM}
+ Expect Symbol CLAM_VIRUS
+ Do Not Expect Symbol CLAMAV_VIRUS_FAIL
+ Shutdown clamav
+
+CLAMAV CACHE HIT
+ Scan File ${MESSAGE2}
+ ... Settings=${SETTINGS_CLAM}
+ Expect Symbol CLAM_VIRUS
+ Do Not Expect Symbol CLAMAV_VIRUS_FAIL
+
+CLAMAV CACHE MISS
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_CLAM}
+ Do Not Expect Symbol CLAM_VIRUS
+ Do Not Expect Symbol CLAMAV_VIRUS_FAIL
+
+FPROT MISS
+ Run Dummy Fprot ${RSPAMD_PORT_FPROT}
+ Scan File ${MESSAGE2}
+ ... Settings=${SETTINGS_FPROT}
+ Do Not Expect Symbol FPROT_VIRUS
+ Do Not Expect Symbol FPROT_EICAR
+ Shutdown fport
+
+FPROT HIT - PATTERN
+ Run Dummy Fprot ${RSPAMD_PORT_FPROT} 1
+ Run Dummy Fprot ${RSPAMD_PORT_FPROT2_DUPLICATE} 1 /tmp/dummy_fprot_dupe.pid
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_FPROT}
+ Expect Symbol FPROT_EICAR
+ # Also check ordered pattern match
+ Expect Symbol FPROT2_VIRUS_DUPLICATE_PATTERN
+ Do Not Expect Symbol FPROT2_VIRUS_DUPLICATE_DEFAULT
+ Do Not Expect Symbol FPROT2_VIRUS_DUPLICATE_NOPE
+ Shutdown fport
+ Shutdown fport duplicate
+
+FPROT CACHE HIT
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_FPROT}
+ Expect Symbol FPROT_EICAR
+ Do Not Expect Symbol CLAMAV_VIRUS
+ # Also check ordered pattern match
+ Expect Symbol FPROT2_VIRUS_DUPLICATE_PATTERN
+ Do Not Expect Symbol FPROT2_VIRUS_DUPLICATE_DEFAULT
+
+FPROT CACHE MISS
+ Scan File ${MESSAGE2}
+ ... Settings=${SETTINGS_FPROT}
+ Do Not Expect Symbol FPROT_VIRUS
+
+AVAST MISS
+ Run Dummy Avast ${RSPAMD_PORT_AVAST}
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_AVAST}
+ Do Not Expect Symbol AVAST_VIRUS
+ Shutdown avast
+
+AVAST HIT
+ Run Dummy Avast ${RSPAMD_PORT_AVAST} 1
+ Scan File ${MESSAGE2}
+ ... Settings=${SETTINGS_AVAST}
+ Expect Symbol AVAST_VIRUS
+ Do Not Expect Symbol AVAST_VIRUS_FAIL
+ Shutdown avast
+
+AVAST CACHE HIT
+ Scan File ${MESSAGE2}
+ ... Settings=${SETTINGS_AVAST}
+ Expect Symbol AVAST_VIRUS
+ Do Not Expect Symbol AVAST_VIRUS_FAIL
+
+AVAST CACHE MISS
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_AVAST}
+ Do Not Expect Symbol AVAST_VIRUS
+ Do Not Expect Symbol AVAST_VIRUS_FAIL
+
+*** Keywords ***
+Antivirus Teardown
+ Shutdown clamav
+ Shutdown fport
+ Shutdown avast
+
+Shutdown clamav
+ ${clamav_pid} = Get File if exists /tmp/dummy_clamav.pid
+ Run Keyword if ${clamav_pid} Shutdown Process With Children ${clamav_pid}
+
+Shutdown fport
+ ${fport_pid} = Get File if exists /tmp/dummy_fprot.pid
+ Run Keyword if ${fport_pid} Shutdown Process With Children ${fport_pid}
+
+Shutdown fport duplicate
+ ${fport_pid} = Get File if exists /tmp/dummy_fprot_dupe.pid
+ Run Keyword if ${fport_pid} Shutdown Process With Children ${fport_pid}
+
+Shutdown avast
+ ${avast_pid} = Get File if exists /tmp/dummy_avast.pid
+ Run Keyword if ${avast_pid} Shutdown Process With Children ${avast_pid}
+
+Run Dummy
+ [Arguments] @{varargs}
+ ${process} = Start Process @{varargs}
+ ${pid} = Get From List ${varargs} -1
+ ${pass} = Run Keyword And Return Status Wait Until Created ${pid}
+ IF ${pass}
+ Return From Keyword
+ END
+ Wait For Process ${process}
+ ${res} = Get Process Result ${process}
+ Log To Console ${res.stdout}
+ Log To Console ${res.stderr}
+ Fail Dummy server failed to start
+
+Run Dummy Clam
+ [Arguments] ${port} ${found}= ${pid}=/tmp/dummy_clamav.pid
+ Run Dummy ${RSPAMD_TESTDIR}/util/dummy_clam.py ${port} ${found} ${pid}
+
+Run Dummy Fprot
+ [Arguments] ${port} ${found}= ${pid}=/tmp/dummy_fprot.pid
+ Run Dummy ${RSPAMD_TESTDIR}/util/dummy_fprot.py ${port} ${found} ${pid}
+
+Run Dummy Avast
+ [Arguments] ${port} ${found}= ${pid}=/tmp/dummy_avast.pid
+ Run Dummy ${RSPAMD_TESTDIR}/util/dummy_avast.py ${port} ${found} ${pid}
diff --git a/test/functional/cases/001_merged/240_redis.robot b/test/functional/cases/001_merged/240_redis.robot
new file mode 100644
index 0000000..3196dd0
--- /dev/null
+++ b/test/functional/cases/001_merged/240_redis.robot
@@ -0,0 +1,16 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${SETTINGS_REDIS} {symbols_enabled = [REDIS_TEST, SIMPLE_REDIS_ASYNC_TEST, SIMPLE_REDIS_ASYNC201809_TEST]}
+
+*** Test Cases ***
+Redis client
+ Redis SET test_key test value
+ Scan File ${MESSAGE}
+ Expect Symbol With Exact Options REDIS hello from lua on redis
+ Expect Symbol With Exact Options REDIS_ASYNC test value
+ Expect Symbol With Exact Options REDIS_ASYNC201809 test value
diff --git a/test/functional/cases/001_merged/250_dns.robot b/test/functional/cases/001_merged/250_dns.robot
new file mode 100644
index 0000000..d3e64cd
--- /dev/null
+++ b/test/functional/cases/001_merged/250_dns.robot
@@ -0,0 +1,21 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${SETTINGS_DNS} {symbols_enabled = [SIMPLE_DNS, SIMPLE_DNS_SYNC]}
+
+*** Test Cases ***
+Simple DNS request
+ Scan File ${MESSAGE} To-Resolve=example.com
+ ... Settings=${SETTINGS_DNS}
+ Expect Symbol With Exact Options DNS_SYNC 93.184.216.34
+ Expect Symbol With Exact Options DNS 93.184.216.34
+
+Faulty DNS request
+ Scan File ${MESSAGE} To-Resolve=not-resolvable.com
+ ... Settings=${SETTINGS_DNS}
+ Expect Symbol With Exact Options DNS_SYNC_ERROR requested record is not found
+ Expect Symbol With Exact Options DNS_ERROR requested record is not found
diff --git a/test/functional/cases/001_merged/270_selector.robot b/test/functional/cases/001_merged/270_selector.robot
new file mode 100644
index 0000000..fa3ab87
--- /dev/null
+++ b/test/functional/cases/001_merged/270_selector.robot
@@ -0,0 +1,19 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/subject1.eml
+
+*** Test Cases ***
+Newlines
+ Scan File ${MESSAGE} User=test@user.com Pass=all
+ ... Settings={symbols_enabled = [CONFIG_SELECTOR_RE_RCPT_SUBJECT, LUA_SELECTOR_RE]}
+ Expect Symbol CONFIG_SELECTOR_RE_RCPT_SUBJECT
+ Expect Symbol LUA_SELECTOR_RE
+
+Rspamd_text selector
+ Scan File ${MESSAGE}
+ ... Settings={symbols_enabled = [RSPAMD_TEXT_SELECTOR]}
+ Expect Symbol RSPAMD_TEXT_SELECTOR
diff --git a/test/functional/cases/001_merged/280_rules.robot b/test/functional/cases/001_merged/280_rules.robot
new file mode 100644
index 0000000..b7cbe81
--- /dev/null
+++ b/test/functional/cases/001_merged/280_rules.robot
@@ -0,0 +1,139 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE1} ${RSPAMD_TESTDIR}/messages/fws_fn.eml
+${MESSAGE2} ${RSPAMD_TESTDIR}/messages/fws_fp.eml
+${MESSAGE3} ${RSPAMD_TESTDIR}/messages/fws_tp.eml
+${MESSAGE4} ${RSPAMD_TESTDIR}/messages/broken_richtext.eml
+${MESSAGE5} ${RSPAMD_TESTDIR}/messages/badboundary.eml
+${MESSAGE6} ${RSPAMD_TESTDIR}/messages/pdf_encrypted.eml
+${MESSAGE7} ${RSPAMD_TESTDIR}/messages/pdf_js.eml
+${MESSAGE8} ${RSPAMD_TESTDIR}/messages/yand_forward.eml
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/newlines.eml
+
+*** Test Cases ***
+Broken MIME
+ Scan File ${MESSAGE3} Settings={symbols_enabled = [MISSING_SUBJECT]}
+ Expect Symbol MISSING_SUBJECT
+
+Issue 2584
+ Scan File ${MESSAGE1}
+ ... Settings={symbols_enabled = [BROKEN_CONTENT_TYPE, MISSING_SUBJECT, R_MISSING_CHARSET]}
+ Do Not Expect Symbol BROKEN_CONTENT_TYPE
+ Do Not Expect Symbol MISSING_SUBJECT
+ Do Not Expect Symbol R_MISSING_CHARSET
+
+Issue 2349
+ Scan File ${MESSAGE2}
+ ... Settings={symbols_enabled = [MULTIPLE_UNIQUE_HEADERS]}
+ Do Not Expect Symbol MULTIPLE_UNIQUE_HEADERS
+
+Broken Rich Text
+ Scan File ${MESSAGE4}
+ ... Settings={symbols_enabled = [BROKEN_CONTENT_TYPE]}
+ Expect Symbol BROKEN_CONTENT_TYPE
+
+Broken boundary
+ Scan File ${MESSAGE4}
+ ... Settings={symbols_enabled = [BROKEN_CONTENT_TYPE]}
+ Expect Symbol BROKEN_CONTENT_TYPE
+
+PDF encrypted
+ Scan File ${MESSAGE6}
+ ... Settings={symbols_enabled = [PDF_ENCRYPTED]}
+ Expect Symbol PDF_ENCRYPTED
+
+PDF javascript
+ Scan File ${MESSAGE7}
+ ... Settings={symbols_enabled = [PDF_JAVASCRIPT]}
+ Expect Symbol PDF_JAVASCRIPT
+
+BITCOIN ADDR
+ Scan File ${RSPAMD_TESTDIR}/messages/btc.eml
+ ... Settings={symbols_enabled = [BITCOIN_ADDR]}
+ Expect Symbol BITCOIN_ADDR
+
+BITCOIN ADDR 2
+ Scan File ${RSPAMD_TESTDIR}/messages/btc2.eml
+ ... Settings={symbols_enabled = [BITCOIN_ADDR]}
+ Expect Symbol BITCOIN_ADDR
+
+BITCOIN ADDR 3
+ Scan File ${RSPAMD_TESTDIR}/messages/btc3.eml
+ ... Settings={symbols_enabled = [BITCOIN_ADDR]}
+ Expect Symbol BITCOIN_ADDR
+
+BITCOIN ADDR 4
+ Scan File ${RSPAMD_TESTDIR}/messages/btc4.eml
+ ... Settings={symbols_enabled = [BITCOIN_ADDR]}
+ Expect Symbol With Exact Options BITCOIN_ADDR 1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2
+ ... bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq bitcoincash:qztslqhavnjcgth9zwu6dw0jjcfy4zahfy7vf0smwp
+
+RCVD_COUNT_ONE
+ Scan File ${RSPAMD_TESTDIR}/messages/btc.eml
+ ... Settings={symbols_enabled = [RCVD_COUNT_ONE]}
+ Expect Symbol RCVD_COUNT_ONE
+
+RCVD_COUNT_FIVE
+ Scan File ${RSPAMD_TESTDIR}/messages/yand_forward.eml
+ ... Settings={symbols_enabled = [RCVD_COUNT_ONE]}
+ Expect Symbol RCVD_COUNT_FIVE
+
+RCVD_COUNT_SEVEN
+ Scan File ${RSPAMD_TESTDIR}/messages/rcvd7.eml
+ ... Settings={symbols_enabled = [RCVD_COUNT_ONE]}
+ Expect Symbol RCVD_COUNT_SEVEN
+
+FROM_NEQ_ENVFROM
+ Scan File ${MESSAGE8} From=test@test.net
+ ... Settings={symbols_enabled = [FROM_NEQ_ENVFROM]}
+ Expect Symbol FROM_NEQ_ENVFROM
+
+PHISH_SENDER_A_1
+ Scan File ${RSPAMD_TESTDIR}/messages/phish_sender.eml
+ ... Settings={symbols_enabled = [MULTIPLE_FROM]}
+ Expect Symbol With Score And Exact Options MULTIPLE_FROM 8.0 <any@attack.com> <admin@legitimate.com>
+
+PHISH_SENDER_A_2
+ Scan File ${RSPAMD_TESTDIR}/messages/phish_sender.eml
+ ... Settings={symbols_enabled = [MULTIPLE_UNIQUE_HEADERS]}
+ Expect Symbol With Score And Exact Options MULTIPLE_UNIQUE_HEADERS 7.0 From
+
+PHISH_SENDER_B
+ Scan File ${RSPAMD_TESTDIR}/messages/phish_sender2.eml
+ ... Settings={symbols_enabled = [BROKEN_HEADERS]}
+ Expect Symbol BROKEN_HEADERS
+
+PHISH_SENDER_C
+ Scan File ${RSPAMD_TESTDIR}/messages/phish_sender3.eml
+ ... Settings={symbols_enabled = [BROKEN_HEADERS]}
+ Expect Symbol BROKEN_HEADERS
+
+PHISH_SENDER_D
+ Scan File ${RSPAMD_TESTDIR}/messages/phish_sender4.eml
+ ... Settings={symbols_enabled = [BROKEN_HEADERS]}
+ Expect Symbol BROKEN_HEADERS
+
+PHISH_SENDER_E
+ Scan File ${RSPAMD_TESTDIR}/messages/phish_sender5.eml
+ ... Settings={symbols_enabled = [MULTIPLE_FROM, DMARC_CHECK, DKIM_CHECK, SPF_CHECK]}
+ Expect Symbol MULTIPLE_FROM
+ Expect Symbol With Exact Options DMARC_NA Duplicate From header
+
+PHISH_SENDER_ROUTING_PART
+ Scan File ${RSPAMD_TESTDIR}/messages/phish_sender6.eml
+ ... Settings={symbols_enabled = [FROM_INVALID]}
+ Expect Symbol FROM_INVALID
+
+REPLYTO_ADDR_EQ_FROM
+ Scan File ${RSPAMD_TESTDIR}/messages/replyto_addr_eq_from.eml
+ ... Settings={symbols_enabled = [REPLYTO_ADDR_EQ_FROM]}
+ Expect Symbol REPLYTO_ADDR_EQ_FROM
+
+SUBJECT_HAS_CURRENCY
+ Scan File ${RSPAMD_TESTDIR}/messages/currency.eml
+ ... Settings={symbols_enabled = [SUBJECT_HAS_CURRENCY]}
+ Expect Symbol SUBJECT_HAS_CURRENCY
diff --git a/test/functional/cases/001_merged/281_fnames.robot b/test/functional/cases/001_merged/281_fnames.robot
new file mode 100644
index 0000000..bb600b1
--- /dev/null
+++ b/test/functional/cases/001_merged/281_fnames.robot
@@ -0,0 +1,13 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${SETTINGS_FNAME} {symbols_enabled = [TEST_FNAME]}
+
+*** Test Cases ***
+FILE NAMES
+ Scan File ${RSPAMD_TESTDIR}/messages/fname.eml Settings=${SETTINGS_FNAME}
+ Expect Symbol With Option TEST_FNAME [삼성ìƒëª…]2020.08.14 ë°ì¼ë¦¬ 경제뉴스.pdf
+ Expect Symbol With Option TEST_FNAME 01029_402110_10620_RGT06902_PRT180ML_20200803_101820.pdf
diff --git a/test/functional/cases/001_merged/290_greylist.robot b/test/functional/cases/001_merged/290_greylist.robot
new file mode 100644
index 0000000..05ce3d3
--- /dev/null
+++ b/test/functional/cases/001_merged/290_greylist.robot
@@ -0,0 +1,25 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${SETTINGS_GREYLIST} {symbols_enabled = [GREYLIST_CHECK, GREYLIST_SAVE], symbols = [FOUR_POINTS]}
+
+*** Test Cases ***
+GREYLIST NEW
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_GREYLIST}
+ Expect Symbol With Option GREYLIST greylisted
+
+GREYLIST EARLY
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_GREYLIST}
+ Expect Symbol With Option GREYLIST greylisted
+
+GREYLIST PASS
+ Sleep 4s Wait greylisting timeout
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_GREYLIST}
+ Expect Symbol With Option GREYLIST pass
diff --git a/test/functional/cases/001_merged/300_rbl.robot b/test/functional/cases/001_merged/300_rbl.robot
new file mode 100644
index 0000000..8b19fae
--- /dev/null
+++ b/test/functional/cases/001_merged/300_rbl.robot
@@ -0,0 +1,98 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+
+*** Test Cases ***
+RBL FROM MISS
+ Scan File ${MESSAGE} IP=1.2.3.4
+ ... Settings={symbols_enabled = [FAKE_RBL_UNKNOWN_CHECK]}
+ Do Not Expect Symbol FAKE_RBL_CODE_2
+
+RBL FROM HIT
+ Scan File ${MESSAGE} IP=4.3.2.1
+ ... Settings={symbols_enabled = [FAKE_RBL_UNKNOWN_CHECK]}
+ Expect Symbol FAKE_RBL_CODE_2
+
+RBL FROM MULTIPLE HIT
+ Scan File ${MESSAGE} IP=4.3.2.3
+ ... Settings={symbols_enabled = [FAKE_RBL_UNKNOWN_CHECK]}
+ Expect Symbol FAKE_RBL_CODE_2
+ Expect Symbol FAKE_RBL_CODE_3
+
+RBL FROM UNKNOWN HIT
+ Scan File ${MESSAGE} IP=4.3.2.2
+ ... Settings={symbols_enabled = [FAKE_RBL_UNKNOWN_CHECK]}
+ Expect Symbol FAKE_RBL_FAKE_RBL_UNKNOWN
+
+RBL RECEIVED HIT
+ Scan File ${MESSAGE} IP=8.8.8.8
+ ... Settings={symbols_enabled = [FAKE_RECEIVED_RBL_FAKE_RBL_UNKNOWN]}
+ Expect Symbol FAKE_RECEIVED_RBL_CODE_3
+
+RBL FROM HIT WL
+ Scan File ${MESSAGE} IP=4.3.2.4
+ ... Settings={symbols_enabled = [FAKE_RBL_UNKNOWN, FAKE_WL_RBL_UNKNOWN]}
+ Do Not Expect Symbol FAKE_RBL_CODE_2
+ Expect Symbol With Exact Options FAKE_WL_RBL_CODE_2 4.3.2.4:from
+
+EMAILBL Compose Map 1
+ Scan File ${RSPAMD_TESTDIR}/messages/url14.eml
+ ... Settings={symbols_enabled = [RSPAMD_EMAILBL]}
+ Expect Symbol With Exact Options RSPAMD_EMAILBL dirty.sanchez.com:email
+
+EMAILBL Compose Map 2
+ Scan File ${RSPAMD_TESTDIR}/messages/url15.eml
+ ... Settings={symbols_enabled = [RSPAMD_EMAILBL]}
+ Expect Symbol With Exact Options RSPAMD_EMAILBL very.dirty.sanchez.com:email
+
+EMAILBL Compose Map 3
+ Scan File ${RSPAMD_TESTDIR}/messages/url16.eml
+ ... Settings={symbols_enabled = [RSPAMD_EMAILBL]}
+ Expect Symbol With Exact Options RSPAMD_EMAILBL 41.black.sanchez.com:email
+
+CONTENT URLS
+ Scan File ${RSPAMD_TESTDIR}/messages/content_url.eml
+ ... Settings={symbols_enabled = [URIBL_CONTENTONLY, URIBL_NOCONTENT, URIBL_WITHCONTENT]}
+ Expect Symbol With Exact Options URIBL_NOCONTENT example.org:url
+ Expect Symbol With Option URIBL_WITHCONTENT example.com:url
+ Expect Symbol With Option URIBL_WITHCONTENT example.org:url
+ Expect Symbol With Option URIBL_WITHCONTENT 8.8.8.8:url
+ Expect Symbol With Exact Options URIBL_CONTENTONLY example.com:url
+
+SELECTORS
+ Scan File ${RSPAMD_TESTDIR}/messages/btc.eml From=user@example.com Helo=example.org
+ ... Settings={symbols_enabled = [RBL_SELECTOR_SINGLE, RBL_SELECTOR_MULTIPLE]}
+ Expect Symbol With Exact Options RBL_SELECTOR_SINGLE example.org:selector
+ Expect Symbol With Option RBL_SELECTOR_MULTIPLE example.com:sel_from
+ Expect Symbol With Option RBL_SELECTOR_MULTIPLE example.org:sel_helo
+
+SELECTORS COMBINED
+ Scan File ${RSPAMD_TESTDIR}/messages/btc.eml From=user@example.org Helo=example.org
+ ... Settings={symbols_enabled = [RBL_SELECTOR_MULTIPLE]}
+ Expect Symbol With Option RBL_SELECTOR_MULTIPLE example.org:sel_from
+ Expect Symbol With Option RBL_SELECTOR_MULTIPLE example.org:sel_helo
+
+NUMERIC URLS
+ Scan File ${RSPAMD_TESTDIR}/messages/numeric_urls.eml
+ ... Settings={symbols_enabled = [URIBL_NUMERIC]}
+ Expect Symbol With Exact Options URIBL_NUMERIC 4.3.2.1:url
+
+NUMERIC URLS WITH IMAGES
+ Scan File ${RSPAMD_TESTDIR}/messages/numeric_urls.eml
+ ... Settings={symbols_enabled = [URIBL_NUMERIC_IMAGES]}
+ Expect Symbol With Exact Options URIBL_NUMERIC_IMAGES 4.3.2.1:url 12.11.10.9:url
+
+NUMERIC URLS WITH CONTENT
+ Scan File ${RSPAMD_TESTDIR}/messages/numeric_urls.eml
+ ... Settings={symbols_enabled = [URIBL_NUMERIC_CONTENT]}
+ Expect Symbol With Exact Options URIBL_NUMERIC_CONTENT 4.3.2.1:url 8.7.6.5:url
+
+NUMERIC URLS WITH EVERYTHING
+ Scan File ${RSPAMD_TESTDIR}/messages/numeric_urls.eml
+ ... IP=127.0.0.1
+ ... Settings={symbols_enabled = [URIBL_NUMERIC_EVERYTHING]}
+ Expect Symbol With Exact Options URIBL_NUMERIC_EVERYTHING 12.11.10.9:url 4.3.2.1:url 8.7.6.5:url
diff --git a/test/functional/cases/001_merged/310_udp.robot b/test/functional/cases/001_merged/310_udp.robot
new file mode 100644
index 0000000..e94ed60
--- /dev/null
+++ b/test/functional/cases/001_merged/310_udp.robot
@@ -0,0 +1,40 @@
+*** Settings ***
+Test Setup UDP Setup
+Test Teardown UDP Teardown
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${SETTINGS_UDP} {symbols_enabled = [UDP_FAIL,UDP_SENDTO,UDP_SUCCESS]}
+
+*** Test Cases ***
+Simple UDP request
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_UDP}
+ Expect Symbol With Exact Options UDP_SUCCESS helloworld
+
+Sendonly UDP request
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_UDP}
+ Expect Symbol UDP_SENDTO
+
+Errored UDP request
+ Scan File ${MESSAGE}
+ ... Settings=${SETTINGS_UDP}
+ Expect Symbol With Exact Options UDP_FAIL read timeout
+
+*** Keywords ***
+UDP Setup
+ Run Dummy UDP
+
+UDP Teardown
+ ${udp_pid} = Get File /tmp/dummy_udp.pid
+ Shutdown Process With Children ${udp_pid}
+
+Run Dummy UDP
+ [Arguments]
+ ${result} = Start Process ${RSPAMD_TESTDIR}/util/dummy_udp.py 5005
+ Wait Until Created /tmp/dummy_udp.pid
diff --git a/test/functional/cases/001_merged/321_arc_check.robot b/test/functional/cases/001_merged/321_arc_check.robot
new file mode 100644
index 0000000..3f6488b
--- /dev/null
+++ b/test/functional/cases/001_merged/321_arc_check.robot
@@ -0,0 +1,19 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${SETTINGS_ARC} {symbols_enabled = [ARC_CALLBACK]}
+
+*** Test Cases ***
+ARC ALLOW CHECK
+ Scan File ${RSPAMD_TESTDIR}/messages/arcallow.eml
+ ... Settings=${SETTINGS_ARC}
+ Expect Symbol ARC_ALLOW
+
+ARC BAD CHECK
+ Scan File ${RSPAMD_TESTDIR}/messages/arcbad.eml
+ ... Settings=${SETTINGS_ARC}
+ Expect Symbol ARC_INVALID
+
diff --git a/test/functional/cases/001_merged/340_surbl.robot b/test/functional/cases/001_merged/340_surbl.robot
new file mode 100644
index 0000000..a1d0736
--- /dev/null
+++ b/test/functional/cases/001_merged/340_surbl.robot
@@ -0,0 +1,182 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${SETTINGS_SURBL} {groups_enabled = [rbl]}
+
+*** Test Cases ***
+SURBL resolve ip
+ Scan File ${RSPAMD_TESTDIR}/messages/url7.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol With Exact Options URIBL_SBL_CSS 8.8.8.9:example.ru:url
+ Expect Symbol With Exact Options URIBL_XBL 8.8.8.8:example.ru:url
+ Expect Symbol With Exact Options URIBL_PBL 8.8.8.8:example.ru:url
+
+SURBL Example.com domain
+ Scan File ${RSPAMD_TESTDIR}/messages/url4.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol With Exact Options RSPAMD_URIBL example.com:url
+ Expect Symbol With Exact Options DBL_SPAM example.com:url
+ Expect Symbol With Exact Options DBL_PHISH rspamd.tk:url
+ Do Not Expect Symbol URIBL_BLACK
+
+SURBL Example.net domain
+ Scan File ${RSPAMD_TESTDIR}/messages/url5.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol DBL_PHISH
+ Do Not Expect Symbol DBL_SPAM
+ Do Not Expect Symbol RSPAMD_URIBL
+ Do Not Expect Symbol URIBL_BLACK
+
+SURBL Example.org domain
+ Scan File ${RSPAMD_TESTDIR}/messages/url6.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol URIBL_BLACK
+ Do Not Expect Symbol DBL_SPAM
+ Do Not Expect Symbol RSPAMD_URIBL
+ Do Not Expect Symbol DBL_PHISH
+
+SURBL Example.ru domain
+ Scan File ${RSPAMD_TESTDIR}/messages/url7.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol URIBL_GREY
+ Expect Symbol URIBL_RED
+ Do Not Expect Symbol DBL_SPAM
+ Do Not Expect Symbol RSPAMD_URIBL
+ Do Not Expect Symbol DBL_PHISH
+ Do Not Expect Symbol URIBL_BLACK
+
+SURBL Example.ru ZEN domain
+ Scan File ${RSPAMD_TESTDIR}/messages/url7.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol URIBL_SBL_CSS
+ Expect Symbol URIBL_XBL
+ Expect Symbol URIBL_PBL
+ Do Not Expect Symbol URIBL_SBL
+ Do Not Expect Symbol DBL_SPAM
+ Do Not Expect Symbol RSPAMD_URIBL
+ Do Not Expect Symbol DBL_PHISH
+ Do Not Expect Symbol URIBL_BLACK
+
+SURBL Example.com domain image false
+ Scan File ${RSPAMD_TESTDIR}/messages/urlimage.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_URIBL_IMAGES
+ Do Not Expect Symbol DBL_SPAM
+ Do Not Expect Symbol RSPAMD_URIBL
+ Do Not Expect Symbol DBL_PHISH
+ Do Not Expect Symbol URIBL_BLACK
+
+SURBL @example.com mail html
+ Scan File ${RSPAMD_TESTDIR}/messages/mailadr.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_URIBL
+ Expect Symbol With Exact Options DBL_SPAM example.com:email
+ Do Not Expect Symbol RSPAMD_URIBL_IMAGES
+ Do Not Expect Symbol DBL_PHISH
+ Do Not Expect Symbol URIBL_BLACK
+
+SURBL @example.com mail text
+ Scan File ${RSPAMD_TESTDIR}/messages/mailadr2.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_URIBL
+ Expect Symbol With Exact Options DBL_SPAM example.com:email
+ Do Not Expect Symbol RSPAMD_URIBL_IMAGES
+ Do Not Expect Symbol DBL_PHISH
+ Do Not Expect Symbol URIBL_BLACK
+
+SURBL example.com not encoded url in subject
+ Scan File ${RSPAMD_TESTDIR}/messages/urlinsubject.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_URIBL
+ Expect Symbol DBL_SPAM
+ Do Not Expect Symbol DBL_PHISH
+ Do Not Expect Symbol URIBL_BLACK
+
+SURBL example.com encoded url in subject
+ Scan File ${RSPAMD_TESTDIR}/messages/urlinsubjectencoded.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_URIBL
+ Expect Symbol DBL_SPAM
+ Do Not Expect Symbol DBL_PHISH
+ Do Not Expect Symbol URIBL_BLACK
+
+WHITELIST
+ Scan File ${RSPAMD_TESTDIR}/messages/whitelist.eml
+ ... Settings=${SETTINGS_SURBL}
+ Do Not Expect Symbol RSPAMD_URIBL
+ Do Not Expect Symbol DBL_SPAM
+ Do Not Expect Symbol RSPAMD_URIBL_IMAGES
+
+EMAILBL full address & domain only
+ Scan File ${RSPAMD_TESTDIR}/messages/emailbltext.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_EMAILBL_FULL
+ Expect Symbol RSPAMD_EMAILBL_DOMAINONLY
+
+EMAILBL full subdomain address
+ Scan File ${RSPAMD_TESTDIR}/messages/emailbltext2.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_EMAILBL_FULL
+
+EMAILBL full subdomain address & domain only
+ Scan File ${RSPAMD_TESTDIR}/messages/emailbltext3.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol With Exact Options RSPAMD_EMAILBL_DOMAINONLY baddomain.com:email
+ Expect Symbol With Exact Options RSPAMD_EMAILBL_FULL user.subdomain.baddomain.com:email
+
+EMAILBL REPLY TO full address
+ Scan File ${RSPAMD_TESTDIR}/messages/replyto.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_EMAILBL_FULL
+ Do Not Expect Symbol RSPAMD_EMAILBL_DOMAINONLY
+
+EMAILBL REPLY TO domain only
+ Scan File ${RSPAMD_TESTDIR}/messages/replyto2.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_EMAILBL_DOMAINONLY
+ Do Not Expect Symbol RSPAMD_EMAILBL_FULL
+
+EMAILBL REPLY TO full subdomain address
+ Scan File ${RSPAMD_TESTDIR}/messages/replytosubdomain.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_EMAILBL_FULL
+ Do Not Expect Symbol RSPAMD_EMAILBL_DOMAINONLY
+
+SURBL IDN domain
+ Scan File ${RSPAMD_TESTDIR}/messages/url8.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_URIBL
+ Expect Symbol DBL_SPAM
+ Do Not Expect Symbol DBL_PHISH
+ Do Not Expect Symbol URIBL_BLACK
+
+SURBL IDN Punycode domain
+ Scan File ${RSPAMD_TESTDIR}/messages/url9.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_URIBL
+ Expect Symbol DBL_SPAM
+ Do Not Expect Symbol DBL_PHISH
+ Do Not Expect Symbol URIBL_BLACK
+
+SURBL html entity&shy
+ Scan File ${RSPAMD_TESTDIR}/messages/url10.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol RSPAMD_URIBL
+
+SURBL url compose map 1
+ Scan File ${RSPAMD_TESTDIR}/messages/url11.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol With Exact Options BAD_SUBDOMAIN clean.dirty.sanchez.com:url
+
+SURBL url compose map 2
+ Scan File ${RSPAMD_TESTDIR}/messages/url12.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol With Exact Options BAD_SUBDOMAIN 4.very.dirty.sanchez.com:url
+
+SURBL url compose map 3
+ Scan File ${RSPAMD_TESTDIR}/messages/url13.eml
+ ... Settings=${SETTINGS_SURBL}
+ Expect Symbol With Exact Options BAD_SUBDOMAIN 41.black.sanchez.com:url
diff --git a/test/functional/cases/001_merged/350_magic.robot b/test/functional/cases/001_merged/350_magic.robot
new file mode 100644
index 0000000..66a18f2
--- /dev/null
+++ b/test/functional/cases/001_merged/350_magic.robot
@@ -0,0 +1,67 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${SETTINGS_MAGIC} {symbols_enabled = [MAGIC_SYM]}
+
+*** Test Cases ***
+Magic detections bundle 1
+ Scan File ${RSPAMD_TESTDIR}/messages/gargantua.eml Settings=${SETTINGS_MAGIC}
+ Expect Symbols MAGIC_SYM_ZIP_2
+ ... MAGIC_SYM_RAR_3
+ ... MAGIC_SYM_EXE_4
+ ... MAGIC_SYM_ELF_5
+ ... MAGIC_SYM_LNK_6
+ ... MAGIC_SYM_CLASS_7
+ ... MAGIC_SYM_RTF_8
+ ... MAGIC_SYM_PDF_9
+ ... MAGIC_SYM_PS_10
+ ... MAGIC_SYM_CHM_11
+ ... MAGIC_SYM_DJVU_12
+ ... MAGIC_SYM_ARJ_13
+ ... MAGIC_SYM_CAB_14
+ ... MAGIC_SYM_ACE_15
+ ... MAGIC_SYM_TAR_16
+ ... MAGIC_SYM_BZ2_17
+ ... MAGIC_SYM_XZ_18
+ ... MAGIC_SYM_LZ4_19
+ ... MAGIC_SYM_ZST_20
+ ... MAGIC_SYM_DMG_21
+ ... MAGIC_SYM_ISO_22
+ ... MAGIC_SYM_ZOO_23
+ ... MAGIC_SYM_EPUB_24
+ ... MAGIC_SYM_XAR_25
+ ... MAGIC_SYM_PSD_26
+ ... MAGIC_SYM_PCX_27
+ ... MAGIC_SYM_TIFF_28
+ ... MAGIC_SYM_ICO_29
+ ... MAGIC_SYM_SWF_30
+ ... MAGIC_SYM_DOC_31
+ ... MAGIC_SYM_XLS_32
+ ... MAGIC_SYM_PPT_33
+ ... MAGIC_SYM_MSI_34
+ ... MAGIC_SYM_MSG_35
+ ... MAGIC_SYM_DOCX_36
+ ... MAGIC_SYM_XLSX_37
+ ... MAGIC_SYM_PPTX_38
+ ... MAGIC_SYM_ODT_39
+ ... MAGIC_SYM_ODS_40
+ ... MAGIC_SYM_ODP_41
+ ... MAGIC_SYM_7Z_42
+ ... MAGIC_SYM_VSD_43
+ ... MAGIC_SYM_PNG_44
+ ... MAGIC_SYM_JPG_45
+ ... MAGIC_SYM_GIF_46
+ ... MAGIC_SYM_BMP_47
+ ... MAGIC_SYM_TXT_48
+ ... MAGIC_SYM_HTML_49
+ ... MAGIC_SYM_CSV_50
+ ... MAGIC_SYM_DWG_51
+ ... MAGIC_SYM_JAR_52
+ ... MAGIC_SYM_APK_53
+ ... MAGIC_SYM_BAT_54
+ ... MAGIC_SYM_ICS_55
+ ... MAGIC_SYM_VCF_56
+ ... MAGIC_SYM_CSV_57
diff --git a/test/functional/cases/001_merged/__init__.robot b/test/functional/cases/001_merged/__init__.robot
new file mode 100644
index 0000000..7ac640d
--- /dev/null
+++ b/test/functional/cases/001_merged/__init__.robot
@@ -0,0 +1,28 @@
+*** Settings ***
+Suite Setup Multi Setup
+Suite Teardown Rspamd Redis Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/merged.conf
+${REDIS_SCOPE} Suite
+${RSPAMD_EXTERNAL_RELAY_ENABLED} false
+${RSPAMD_MAP_MAP} ${RSPAMD_TESTDIR}/configs/maps/map.list
+${RSPAMD_RADIX_MAP} ${RSPAMD_TESTDIR}/configs/maps/ip2.list
+${RSPAMD_REGEXP_MAP} ${RSPAMD_TESTDIR}/configs/maps/regexp.list
+${RSPAMD_SCOPE} Suite
+
+*** Keywords ***
+Multi Setup
+ Run Redis
+ Run Dummy Http
+ Run Dummy Https
+ Rspamd Setup
+
+Multi Teardown
+ Rspamd Teardown
+ Dummy Http Teardown
+ Dummy Https Teardown
+ Redis Teardown \ No newline at end of file
diff --git a/test/functional/cases/101_lua.robot b/test/functional/cases/101_lua.robot
new file mode 100644
index 0000000..c820d1b
--- /dev/null
+++ b/test/functional/cases/101_lua.robot
@@ -0,0 +1,35 @@
+*** Settings ***
+Test Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/lua_test.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RSPAMD_SCOPE} Test
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+Flags
+ [Setup] Lua Setup ${RSPAMD_TESTDIR}/lua/flags.lua
+ Scan File ${MESSAGE}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} stat
+ Should Contain ${result.stdout} Messages scanned: 0
+
+Dependencies
+ [Setup] Lua Setup ${RSPAMD_TESTDIR}/lua/deps.lua
+ Scan File ${MESSAGE}
+ Expect Symbol DEP10
+
+Pre and Post Filters
+ [Setup] Lua Setup ${RSPAMD_TESTDIR}/lua/prepostfilters.lua
+ Scan File ${MESSAGE}
+ Expect Symbol TEST_PRE
+ Expect Symbol TEST_POST
+
+*** Keywords ***
+Lua Setup
+ [Arguments] ${RSPAMD_LUA_SCRIPT}
+ Set Test Variable ${RSPAMD_LUA_SCRIPT}
+ Rspamd Setup
diff --git a/test/functional/cases/103_password.robot b/test/functional/cases/103_password.robot
new file mode 100644
index 0000000..2237a6a
--- /dev/null
+++ b/test/functional/cases/103_password.robot
@@ -0,0 +1,71 @@
+*** Settings ***
+Test Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/password.conf
+${CONTROLLER_ERRORS} False
+${RSPAMD_CATENA_PASSWORD} "$2$xu1581gidj5cyp4yjgo68qbj6jz1j8o3$j9yg4k58jy3fj8suijxx9d7pea6a6obtufq9kfenosyq8erm87ky"
+${RSPAMD_PBKDF_PASSWORD} "$1$rhzzahtm8a5homdhh7z4qiiy7j8pzp4u$k5toro136brshjjuy9t39r785td69qodmd39qzygxuyehn9tqauy"
+${RSPAMD_ENABLE_CATENA_PASSWORD} "$2$irotou88u89r1gj53pqnom96qo36hgkn$d4dt3466db7ccqx96k18yz9b1brx8hmk3b4w6erf4oqpmf9sag6y"
+${RSPAMD_SCOPE} Test
+
+*** Test Cases ***
+PASSWORD - PBKDF
+ [Setup] Password Setup ${RSPAMD_PBKDF_PASSWORD} ${RSPAMD_ENABLE_CATENA_PASSWORD}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -P nq1 stat
+ Check Rspamc ${result} Messages scanned:
+
+PASSWORD - PBKDF WRONG
+ [Setup] Password Setup ${RSPAMD_PBKDF_PASSWORD} ${RSPAMD_ENABLE_CATENA_PASSWORD}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -P q1q1 stat
+ Should Be Equal As Integers ${result.rc} 1
+
+PASSWORD - CATENA
+ [Setup] Password Setup ${RSPAMD_CATENA_PASSWORD} ${RSPAMD_ENABLE_CATENA_PASSWORD}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -P nq1 stat
+ Check Rspamc ${result} Messages scanned:
+
+PASSWORD - CATENA WRONG
+ [Setup] Password Setup ${RSPAMD_CATENA_PASSWORD} ${RSPAMD_ENABLE_CATENA_PASSWORD}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -P q stat
+ Should Be Equal As Integers ${result.rc} 1
+
+PASSWORD - ENABLE
+ [Setup] Password Setup ${RSPAMD_CATENA_PASSWORD} ${RSPAMD_ENABLE_CATENA_PASSWORD}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -P nq2 stat_reset
+ Check Rspamc ${result} Messages scanned:
+
+PASSWORD - ENABLE WITH NORMAL
+ [Setup] Password Setup ${RSPAMD_CATENA_PASSWORD} ${RSPAMD_ENABLE_CATENA_PASSWORD}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -P nq1 stat_reset
+ Should Be Equal As Integers ${result.rc} 1
+
+PASSWORD - ENABLE INCORRECT
+ [Setup] Password Setup ${RSPAMD_CATENA_PASSWORD} ${RSPAMD_ENABLE_CATENA_PASSWORD}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -P q2q2 stat_reset
+ Should Be Equal As Integers ${result.rc} 1
+
+PASSWORD - ENABLE EQUAL TO NORMAL PRIV COMMAND
+ [Setup] Password Setup ${RSPAMD_CATENA_PASSWORD} ${RSPAMD_CATENA_PASSWORD}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -P nq1 stat_reset
+ Check Rspamc ${result} Messages scanned:
+
+PASSWORD - ENABLE EQUAL TO NORMAL NON PRIV COMMAND
+ [Setup] Password Setup ${RSPAMD_CATENA_PASSWORD} ${RSPAMD_CATENA_PASSWORD}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -P nq1 stat
+ Check Rspamc ${result} Messages scanned:
+
+PASSWORD - ENABLE EQUAL TO NORMAL WRONG PASSWORD
+ [Setup] Password Setup ${RSPAMD_CATENA_PASSWORD} ${RSPAMD_CATENA_PASSWORD}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -P nq2 stat
+ Should Be Equal As Integers ${result.rc} 1
+
+*** Keywords ***
+Password Setup
+ [Arguments] ${RSPAMD_PASSWORD} ${RSPAMD_ENABLE_PASSWORD}
+ Set Test Variable ${RSPAMD_PASSWORD}
+ Set Test Variable ${RSPAMD_ENABLE_PASSWORD}
+ Rspamd Setup
diff --git a/test/functional/cases/108_settings.robot b/test/functional/cases/108_settings.robot
new file mode 100644
index 0000000..1b51343
--- /dev/null
+++ b/test/functional/cases/108_settings.robot
@@ -0,0 +1,268 @@
+*** Settings ***
+Suite Setup Settings Setup
+Suite Teardown Settings Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/settings.conf
+${HAM_MESSAGE} ${RSPAMD_TESTDIR}/messages/ham.eml
+${MESSAGE_7BIT} ${RSPAMD_TESTDIR}/messages/utf.eml
+${MESSAGE_ABSENT_MIME} ${RSPAMD_TESTDIR}/messages/ed25519.eml
+${MESSAGE_CUSTOM_HDR} ${RSPAMD_TESTDIR}/messages/empty-plain-text.eml
+${MESSAGE_PRIORITY} ${RSPAMD_TESTDIR}/messages/priority.eml
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RSPAMD_LUA_SCRIPT} ${RSPAMD_TESTDIR}/lua/settings.lua
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+${SPAM_MESSAGE} ${RSPAMD_TESTDIR}/messages/spam.eml
+
+*** Keywords ***
+Check Everything Disabled
+ Expect Action no action
+ Do Not Expect Symbol SIMPLE_VIRTUAL
+ Do Not Expect Symbol SIMPLE_PRE
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol BAYES_SPAM
+
+*** Test Cases ***
+NO SETTINGS SPAM
+ Scan File ${SPAM_MESSAGE}
+ Expect Symbol SIMPLE_TEST
+ Expect Symbol SIMPLE_VIRTUAL
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Expect Symbol SIMPLE_PRE
+ Expect Symbol SIMPLE_POST
+ Expect Symbol BAYES_SPAM
+
+NO SETTINGS HAM
+ Scan File ${HAM_MESSAGE}
+ Expect Symbol SIMPLE_TEST
+ Expect Symbol SIMPLE_PRE
+ Expect Symbol SIMPLE_POST
+ Expect Symbol BAYES_HAM
+
+EMPTY SYMBOLS ENABLED - STATIC
+ Scan File ${SPAM_MESSAGE} IP=5.5.5.5
+ Check Everything Disabled
+
+EMPTY GROUPS ENABLED - STATIC
+ Scan File ${SPAM_MESSAGE} IP=5.5.5.6
+ Check Everything Disabled
+
+EMPTY SYMBOLS ENABLED - SETTINGS-ID
+ Scan File ${SPAM_MESSAGE} Settings-Id=empty_symbols_enabled
+ Check Everything Disabled
+
+EMPTY GROUPS ENABLED - SETTINGS-ID
+ Scan File ${SPAM_MESSAGE} Settings-Id=empty_groups_enabled
+ Check Everything Disabled
+
+ENABLE SYMBOL - NORMAL
+ Scan File ${HAM_MESSAGE} Settings={symbols_enabled = ["SIMPLE_TEST"]}
+ Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_PRE
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol BAYES_HAM
+
+ENABLE SYMBOL - POSTFILTER
+ Scan File ${HAM_MESSAGE} Settings={symbols_enabled = ["SIMPLE_TEST", "SIMPLE_POST"]}
+ Expect Symbol SIMPLE_TEST
+ Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+ Do Not Expect Symbol BAYES_HAM
+
+ENABLE SYMBOL - PREFILTER
+ Scan File ${HAM_MESSAGE} Settings={symbols_enabled = ["SIMPLE_PRE"]}
+ Expect Symbol SIMPLE_PRE
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol BAYES_HAM
+
+ENABLE SYMBOL - CLASSIFIER
+ Scan File ${HAM_MESSAGE} Settings={symbols_enabled = ["BAYES_HAM", "BAYES_SPAM"]}
+ Expect Symbol BAYES_HAM
+ Do Not Expect Symbol SIMPLE_PRE
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_TEST
+
+DISABLE SYMBOL - NORMAL
+ Scan File ${MESSAGE} Settings={symbols_disabled = ["SIMPLE_TEST"]}
+ Do Not Expect Symbol SIMPLE_TEST
+ Expect Symbol SIMPLE_PRE
+ Expect Symbol SIMPLE_POST
+
+RESCORE SYMBOL - NORMAL
+ Scan File ${MESSAGE} Settings={SIMPLE_TEST = 3.33}
+ Expect Symbol With Score SIMPLE_TEST 3.33
+
+INJECT SYMBOL - NORMAL
+ Scan File ${MESSAGE} Settings={symbols = ["INJECTED_SYMBOL1", "INJECTED_SYMBOL2"]}
+ Expect Symbol INJECTED_SYMBOL1
+ Expect Symbol INJECTED_SYMBOL2
+
+RESCORE ACTION
+ Scan File ${MESSAGE} Settings={actions { reject = 1234.5; } }
+ Expect Required Score 1234.5
+
+DISABLE GROUP - NORMAL
+ Scan File ${MESSAGE} Settings={groups_disabled = ["b"]}
+ Do Not Expect Symbol SIMPLE_TEST
+ Expect Symbol SIMPLE_PRE
+ Expect Symbol SIMPLE_POST
+
+ENABLE GROUP - NORMAL
+ Scan File ${MESSAGE} Settings={groups_enabled = ["b"]}
+ Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_PRE
+ Do Not Expect Symbol SIMPLE_POST
+
+SETTINGS ID - NORMAL
+ Scan File ${MESSAGE} Settings-Id=id_test
+ Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_PRE
+ Do Not Expect Symbol SIMPLE_POST
+
+SETTINGS ID - PRE
+ Scan File ${MESSAGE} Settings-Id=id_pre
+ Expect Symbol SIMPLE_PRE
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_POST
+
+SETTINGS ID - VIRTUAL
+ Scan File ${MESSAGE} Settings-Id=id_virtual
+ Expect Symbol SIMPLE_VIRTUAL
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Do Not Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - VIRTUAL GROUP
+ Scan File ${MESSAGE} Settings-Id=id_virtual_group
+ Expect Symbol With Score SIMPLE_VIRTUAL 10
+ Expect Symbol With Score EXPLICIT_VIRTUAL 10
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Do Not Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - VIRTUAL FROM
+ Scan File ${MESSAGE} From=test2@example.com
+ Expect Symbol With Score SIMPLE_VIRTUAL 10
+ Expect Symbol With Score EXPLICIT_VIRTUAL 10
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Do Not Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - VIRTUAL USER
+ Scan File ${MESSAGE} User=test@example.com
+ Expect Symbol With Score SIMPLE_VIRTUAL 10
+ Expect Symbol With Score EXPLICIT_VIRTUAL 10
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Do Not Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - VIRTUAL HOSTNAME
+ Scan File ${MESSAGE} Hostname=example.com
+ Expect Symbol With Score SIMPLE_VIRTUAL 10
+ Expect Symbol With Score EXPLICIT_VIRTUAL 10
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Do Not Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - VIRTUAL SELECTOR
+ Scan File ${MESSAGE} Rcpt=test3@example.com
+ Expect Symbol With Score SIMPLE_VIRTUAL 10
+ Expect Symbol With Score EXPLICIT_VIRTUAL 10
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Do Not Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - ANGLED RECIPIENT
+ Scan File ${MESSAGE} Rcpt=<test3@example.com>
+ Expect Symbol With Score SIMPLE_VIRTUAL 10
+ Expect Symbol With Score EXPLICIT_VIRTUAL 10
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Do Not Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - VIRTUAL HEADER MATCH
+ Scan File ${MESSAGE_7BIT}
+ Expect Symbol With Score SIMPLE_VIRTUAL 10
+ Expect Symbol With Score EXPLICIT_VIRTUAL 10
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Do Not Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - VIRTUAL HEADER EXISTS
+ Scan File ${MESSAGE_CUSTOM_HDR}
+ Expect Symbol With Score SIMPLE_VIRTUAL 10
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Do Not Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - VIRTUAL HEADER ABSENT
+ Scan File ${MESSAGE_ABSENT_MIME}
+ Expect Symbol With Score SIMPLE_VIRTUAL 10
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Do Not Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - VIRTUAL REQUEST HEADER
+ Scan File ${MESSAGE} Test=passed
+ Expect Symbol With Score SIMPLE_VIRTUAL 10
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL1
+ Do Not Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - VIRTUAL DEP
+ Scan File ${MESSAGE} Settings-Id=id_virtual1
+ Expect Symbol EXPLICIT_VIRTUAL1
+ Expect Symbol DEP_VIRTUAL
+ Expect Symbol DEP_REAL
+ Do Not Expect Symbol SIMPLE_TEST
+ Do Not Expect Symbol SIMPLE_VIRTUAL
+ Do Not Expect Symbol SIMPLE_POST
+ Do Not Expect Symbol SIMPLE_PRE
+
+SETTINGS ID - EXTERNAL MAP
+ Scan File ${MESSAGE} Settings-Id=external
+ Expect Symbol EXTERNAL_SETTINGS
+
+PRIORITY
+ Scan File ${MESSAGE_PRIORITY} Settings-Id=id_virtual_group From=user@test.com
+ Expect Symbol PRIORITY_2
+
+
+*** Keywords ***
+Settings Setup
+ Copy File ${RSPAMD_TESTDIR}/data/bayes.spam.sqlite3 /tmp/bayes.spam.sqlite3
+ Copy File ${RSPAMD_TESTDIR}/data/bayes.ham.sqlite3 /tmp/bayes.ham.sqlite3
+ Run Dummy Http
+ Rspamd Setup
+
+Settings Teardown
+ Rspamd Teardown
+ Dummy Http Teardown
+ Remove Files /tmp/bayes.spam.sqlite3 /tmp/bayes.ham.sqlite3
diff --git a/test/functional/cases/109_composites.robot b/test/functional/cases/109_composites.robot
new file mode 100644
index 0000000..4d73cf4
--- /dev/null
+++ b/test/functional/cases/109_composites.robot
@@ -0,0 +1,78 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/composites.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RSPAMD_LUA_SCRIPT} ${RSPAMD_TESTDIR}/lua/composites.lua
+${RSPAMD_SCOPE} Suite
+
+*** Test Cases ***
+Composites - All in One
+ Scan File ${MESSAGE}
+ Expect Symbols With Scores EXPRESSIONS=5
+ ... EXPRESSIONS_B=0
+ ... POLICY_REMOVE_WEIGHT=5
+ ... POLICY_REMOVE_WEIGHT_B=0
+ ... POLICY_FORCE_REMOVE=5.00
+ ... POLICY_FORCE_REMOVE_A=1.00
+ ... POLICY_LEAVE=5.00
+ ... POLICY_LEAVE_B=1.00
+ ... DEFAULT_POLICY_REMOVE_WEIGHT=5.00
+ ... DEFAULT_POLICY_REMOVE_WEIGHT_A=0.00
+ ... DEFAULT_POLICY_REMOVE_WEIGHT_B=0.00
+ ... DEFAULT_POLICY_REMOVE_SYMBOL=5.00
+ ... DEFAULT_POLICY_LEAVE=5.00
+ ... DEFAULT_POLICY_LEAVE_A=1.00
+ ... DEFAULT_POLICY_LEAVE_B=1.00
+ ... SYMBOL_GROUPS=5.00
+ ... POSITIVE_A=-1.00
+ ... ANY_A=-1.00
+ ... NEGATIVE_B=1.00
+ Do Not Expect Symbols DEFAULT_POLICY_REMOVE_SYMBOL_A
+ ... DEFAULT_POLICY_REMOVE_SYMBOL_B
+ ... NEGATIVE_A
+ ... POLICY_REMOVE_WEIGHT_A
+ ... POLICY_FORCE_REMOVE_B
+ ... POLICY_LEAVE_A
+ Expect Score 50
+ Expect Required Score To Be Null
+
+Composites - Opts Plain
+ Scan File ${MESSAGE} opts=sym1
+ Expect Symbol With Score SYMOPTS1 5.00
+ Do Not Expect Symbol SYMOPTS2
+
+Composites - Opts RE Miss one
+ Scan File ${MESSAGE} opts=sym1,foo1
+ Expect Symbol With Score SYMOPTS1 5.00
+ Do Not Expect Symbol SYMOPTS2
+ Do Not Expect Symbol SYMOPTS3
+
+Composites - Opts RE Miss both
+ Scan File ${MESSAGE} opts=sym2
+ Do Not Expect Symbol SYMOPTS1
+ Do Not Expect Symbol SYMOPTS2
+ Do Not Expect Symbol SYMOPTS3
+
+Composites - Opts RE Hit
+ Scan File ${MESSAGE} opts=foo1,sym2
+ Expect Symbol With Score SYMOPTS2 6.00
+ Do Not Expect Symbol SYMOPTS1
+ Do Not Expect Symbol SYMOPTS3
+
+Composites - Opts RE Hit 2
+ Scan File ${MESSAGE} opts=sym2,foo/
+ Expect Symbol With Score SYMOPTS3 6.00
+ Do Not Expect Symbol SYMOPTS2
+ Do Not Expect Symbol SYMOPTS1
+
+Composites - Opts RE Hit 3
+ Scan File ${MESSAGE} opts=example.com->app.link
+ Expect Symbol With Score SYMOPTS4 6.00
+ Do Not Expect Symbol SYMOPTS2
+ Do Not Expect Symbol SYMOPTS1
diff --git a/test/functional/cases/110_statistics/090-peruser.robot b/test/functional/cases/110_statistics/090-peruser.robot
new file mode 100644
index 0000000..cd8f3ac
--- /dev/null
+++ b/test/functional/cases/110_statistics/090-peruser.robot
@@ -0,0 +1,15 @@
+*** Settings ***
+Suite Setup Rspamd Redis Setup
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Variables ***
+${RSPAMD_REDIS_SERVER} ${RSPAMD_REDIS_ADDR}:${RSPAMD_REDIS_PORT}
+${RSPAMD_STATS_PER_USER} true
+
+*** Test Cases ***
+Learn
+ Learn Test test@example.com
+
+Relearn
+ Relearn Test test@example.com
diff --git a/test/functional/cases/110_statistics/100-redis-keyed-siphash.robot b/test/functional/cases/110_statistics/100-redis-keyed-siphash.robot
new file mode 100644
index 0000000..d889502
--- /dev/null
+++ b/test/functional/cases/110_statistics/100-redis-keyed-siphash.robot
@@ -0,0 +1,16 @@
+*** Settings ***
+Suite Setup Rspamd Redis Setup
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Variables ***
+${RSPAMD_REDIS_SERVER} ${RSPAMD_REDIS_ADDR}:${RSPAMD_REDIS_PORT}
+${RSPAMD_STATS_HASH} siphash
+${RSPAMD_STATS_KEY} ${RSPAMD_KEY_PVT1}
+
+*** Test Cases ***
+Learn
+ Learn Test
+
+Relearn
+ Relearn Test
diff --git a/test/functional/cases/110_statistics/110-redis-keyed-xxhash.robot b/test/functional/cases/110_statistics/110-redis-keyed-xxhash.robot
new file mode 100644
index 0000000..928a9ea
--- /dev/null
+++ b/test/functional/cases/110_statistics/110-redis-keyed-xxhash.robot
@@ -0,0 +1,16 @@
+*** Settings ***
+Suite Setup Rspamd Redis Setup
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Variables ***
+${RSPAMD_REDIS_SERVER} ${RSPAMD_REDIS_ADDR}:${RSPAMD_REDIS_PORT}
+${RSPAMD_STATS_HASH} xxhash
+${RSPAMD_STATS_KEY} ${RSPAMD_KEY_PVT1}
+
+*** Test Cases ***
+Learn
+ Learn Test
+
+Relearn
+ Relearn Test
diff --git a/test/functional/cases/110_statistics/200-redis-plain-siphash.robot b/test/functional/cases/110_statistics/200-redis-plain-siphash.robot
new file mode 100644
index 0000000..790a63e
--- /dev/null
+++ b/test/functional/cases/110_statistics/200-redis-plain-siphash.robot
@@ -0,0 +1,15 @@
+*** Settings ***
+Suite Setup Rspamd Redis Setup
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Variables ***
+${RSPAMD_REDIS_SERVER} ${RSPAMD_REDIS_ADDR}:${RSPAMD_REDIS_PORT}
+${RSPAMD_STATS_HASH} siphash
+
+*** Test Cases ***
+Learn
+ Learn Test
+
+Relearn
+ Relearn Test
diff --git a/test/functional/cases/110_statistics/210-redis-plain-xxhash.robot b/test/functional/cases/110_statistics/210-redis-plain-xxhash.robot
new file mode 100644
index 0000000..55e7524
--- /dev/null
+++ b/test/functional/cases/110_statistics/210-redis-plain-xxhash.robot
@@ -0,0 +1,15 @@
+*** Settings ***
+Suite Setup Rspamd Redis Setup
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Variables ***
+${RSPAMD_REDIS_SERVER} ${RSPAMD_REDIS_ADDR}:${RSPAMD_REDIS_PORT}
+${RSPAMD_STATS_HASH} xxhash
+
+*** Test Cases ***
+Learn
+ Learn Test
+
+Relearn
+ Relearn Test
diff --git a/test/functional/cases/110_statistics/lib.robot b/test/functional/cases/110_statistics/lib.robot
new file mode 100644
index 0000000..7928063
--- /dev/null
+++ b/test/functional/cases/110_statistics/lib.robot
@@ -0,0 +1,69 @@
+*** Settings ***
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/stats.conf
+${MESSAGE_HAM} ${RSPAMD_TESTDIR}/messages/ham.eml
+${MESSAGE_SPAM} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_REDIS_SERVER} null
+${RSPAMD_SCOPE} Suite
+${RSPAMD_STATS_BACKEND} redis
+${RSPAMD_STATS_HASH} null
+${RSPAMD_STATS_KEY} null
+${RSPAMD_STATS_PER_USER} ${EMPTY}
+
+*** Keywords ***
+Broken Learn Test
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} learn_spam ${MESSAGE_SPAM}
+ Check Rspamc ${result} Unknown statistics error
+
+Empty Part Test
+ Set Test Variable ${MESSAGE} ${RSPAMD_TESTDIR}/messages/empty_part.eml
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} learn_spam ${MESSAGE}
+ Check Rspamc ${result}
+ Scan File ${MESSAGE}
+ Expect Symbol BAYES_SPAM
+
+Learn
+ [Arguments] ${user} ${class} ${message}
+ IF "${user}"
+ ${result} = Run Rspamc -d ${user} -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} learn_${class} ${message}
+ ELSE
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} learn_${class} ${message}
+ END
+ Check Rspamc ${result}
+
+Learn Test
+ [Arguments] ${user}=${EMPTY}
+ Set Suite Variable ${RSPAMD_STATS_LEARNTEST} 0
+ Set Test Variable ${kwargs} &{EMPTY}
+ IF "${user}"
+ Set To Dictionary ${kwargs} Deliver-To=${user}
+ END
+ Learn ${user} spam ${MESSAGE_SPAM}
+ Learn ${user} ham ${MESSAGE_HAM}
+ Scan File ${MESSAGE_SPAM} &{kwargs}
+ Expect Symbol BAYES_SPAM
+ Scan File ${MESSAGE_HAM} &{kwargs}
+ Expect Symbol BAYES_HAM
+ Set Suite Variable ${RSPAMD_STATS_LEARNTEST} 1
+
+Relearn Test
+ [Arguments] ${user}=${EMPTY}
+ IF ${RSPAMD_STATS_LEARNTEST} == 0
+ Fail "Learn test was not run"
+ END
+ Learn ${user} ham ${MESSAGE_SPAM}
+ Set Test Variable ${kwargs} &{EMPTY}
+ IF "${user}"
+ Set To Dictionary ${kwargs} Deliver-To=${user}
+ END
+ Scan File ${MESSAGE_SPAM} &{kwargs}
+ ${pass} = Run Keyword And Return Status Expect Symbol BAYES_HAM
+ IF ${pass}
+ Pass Execution What Me Worry
+ END
+ Do Not Expect Symbol BAYES_SPAM
diff --git a/test/functional/cases/116_dkim.robot b/test/functional/cases/116_dkim.robot
new file mode 100644
index 0000000..5c1005c
--- /dev/null
+++ b/test/functional/cases/116_dkim.robot
@@ -0,0 +1,59 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/dkim.conf
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+DKIM PERMFAIL NXDOMAIN
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim2.eml
+ ... IP=37.48.67.26
+ Expect Symbol R_DKIM_PERMFAIL
+
+DKIM PERMFAIL BAD RECORD
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim1.eml
+ ... IP=37.48.67.26
+ Expect Symbol R_DKIM_PERMFAIL
+
+DKIM TEMPFAIL SERVFAIL UNALIGNED
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim3.eml
+ ... IP=37.48.67.26
+ Expect Symbol R_DKIM_TEMPFAIL
+
+DKIM NA NOSIG
+ Scan File ${RSPAMD_TESTDIR}/messages/utf.eml
+ ... IP=37.48.67.26
+ Expect Symbol R_DKIM_NA
+
+DKIM Ignore Unknown Tags
+ Scan File ${RSPAMD_TESTDIR}/messages/dmarc/dkim_unknown_tags.eml
+ Expect Symbol R_DKIM_ALLOW
+
+DKIM Sign
+ Set Suite Variable ${RAN_SIGNTEST} 0
+ ${result} = Scan Message With Rspamc ${RSPAMD_TESTDIR}/messages/spam_message.eml --mime --header=dodkim=1
+ Check Rspamc ${result} DKIM-Signature
+ Set Suite Variable ${SIGNED_MESSAGE} ${RSPAMD_TMPDIR}/dkim_sign_test.eml
+ Create File ${SIGNED_MESSAGE} ${result.stdout}
+ Set Suite Variable ${RAN_SIGNTEST} 1
+
+DKIM Self Verify
+ IF ${RAN_SIGNTEST} == 0
+ Fail "Sign test was not run"
+ END
+ Scan File ${SIGNED_MESSAGE}
+ Expect Symbol R_DKIM_ALLOW
+
+DKIM Verify ED25519 PASS
+ Scan File ${RSPAMD_TESTDIR}/messages/ed25519.eml
+ Expect Symbol R_DKIM_ALLOW
+
+DKIM Verify ED25519 REJECT
+ Scan File ${RSPAMD_TESTDIR}/messages/ed25519-broken.eml
+ Expect Symbol R_DKIM_REJECT
diff --git a/test/functional/cases/120_fuzzy/encrypted.robot b/test/functional/cases/120_fuzzy/encrypted.robot
new file mode 100644
index 0000000..548ea8c
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/encrypted.robot
@@ -0,0 +1,14 @@
+*** Settings ***
+Suite Setup Fuzzy Setup Encrypted Siphash
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+ Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+ Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+ Fuzzy Multimessage Miss Test
diff --git a/test/functional/cases/120_fuzzy/fasthash-keyed.robot b/test/functional/cases/120_fuzzy/fasthash-keyed.robot
new file mode 100644
index 0000000..6add33d
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/fasthash-keyed.robot
@@ -0,0 +1,14 @@
+*** Settings ***
+Suite Setup Fuzzy Setup Keyed Fasthash
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+ Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+ Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+ Fuzzy Multimessage Miss Test
diff --git a/test/functional/cases/120_fuzzy/fasthash.robot b/test/functional/cases/120_fuzzy/fasthash.robot
new file mode 100644
index 0000000..c33ede1
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/fasthash.robot
@@ -0,0 +1,14 @@
+*** Settings ***
+Suite Setup Fuzzy Setup Plain Fasthash
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+ Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+ Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+ Fuzzy Multimessage Miss Test
diff --git a/test/functional/cases/120_fuzzy/general.robot b/test/functional/cases/120_fuzzy/general.robot
new file mode 100644
index 0000000..2978ea6
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/general.robot
@@ -0,0 +1,17 @@
+*** Settings ***
+Suite Setup Fuzzy Setup Plain Siphash
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+ Fuzzy Multimessage Add Test
+
+Fuzzy Delete
+ Fuzzy Multimessage Delete Test
+
+Fuzzy Overwrite
+ Fuzzy Multimessage Overwrite Test
+
+Fuzzy Skip Hash Test
+ Fuzzy Skip Hash Test Message \ No newline at end of file
diff --git a/test/functional/cases/120_fuzzy/lib.robot b/test/functional/cases/120_fuzzy/lib.robot
new file mode 100644
index 0000000..02e0a0a
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/lib.robot
@@ -0,0 +1,181 @@
+*** Settings ***
+Library OperatingSystem
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/fuzzy.conf
+${FLAG1_SYMBOL} R_TEST_FUZZY_DENIED
+${FLAG2_SYMBOL} R_TEST_FUZZY_WHITE
+${REDIS_SCOPE} Suite
+${RSPAMD_FLAG1_NUMBER} 50
+${RSPAMD_FLAG2_NUMBER} 51
+${RSPAMD_FUZZY_BACKEND} redis
+${RSPAMD_FUZZY_ENCRYPTED_ONLY} false
+${RSPAMD_FUZZY_ENCRYPTION_KEY} null
+${RSPAMD_FUZZY_INCLUDE} ${RSPAMD_TESTDIR}/configs/empty.conf
+${RSPAMD_FUZZY_KEY} null
+${RSPAMD_FUZZY_SHINGLES_KEY} null
+${RSPAMD_SCOPE} Suite
+${SETTINGS_FUZZY_CHECK} ${EMPTY}
+${SETTINGS_FUZZY_WORKER} ${EMPTY}
+@{MESSAGES_SKIP} ${RSPAMD_TESTDIR}/messages/priority.eml
+@{MESSAGES} ${RSPAMD_TESTDIR}/messages/spam_message.eml ${RSPAMD_TESTDIR}/messages/zip.eml
+@{RANDOM_MESSAGES} ${RSPAMD_TESTDIR}/messages/bad_message.eml ${RSPAMD_TESTDIR}/messages/zip-doublebad.eml
+
+*** Keywords ***
+Fuzzy Skip Add Test Base
+ [Arguments] ${message}
+ Set Suite Variable ${RSPAMD_FUZZY_ADD_${message}} 0
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -w 10 -f
+ ... ${RSPAMD_FLAG1_NUMBER} fuzzy_add ${message}
+ Check Rspamc ${result}
+ Sync Fuzzy Storage
+ Scan File ${message}
+ Expect Symbol R_TEST_FUZZY_DENIED
+ Create File ${RSPAMD_TMPDIR}/skip_hash.map.tmp 2d875d4737c59c4822fd01dadeba52a329de3933f766c6f167904c6a426bbfa7ea63a66bf807b25c5ee853baee58bfb18d3b423fcd13cfa7c3d77a840039a1ea
+ Move File ${RSPAMD_TMPDIR}/skip_hash.map.tmp ${RSPAMD_TMPDIR}/skip_hash.map
+ Sleep 1s Wait for reload
+ Scan File ${message}
+ Do Not Expect Symbol R_TEST_FUZZY_DENIED
+
+Fuzzy Add Test
+ [Arguments] ${message}
+ Set Suite Variable ${RSPAMD_FUZZY_ADD_${message}} 0
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -w 10 -f
+ ... ${RSPAMD_FLAG1_NUMBER} fuzzy_add ${message}
+ Check Rspamc ${result}
+ Sync Fuzzy Storage
+ Scan File ${message}
+ Expect Symbol ${FLAG1_SYMBOL}
+ Set Suite Variable ${RSPAMD_FUZZY_ADD_${message}} 1
+
+Fuzzy Delete Test
+ [Arguments] ${message}
+ IF ${RSPAMD_FUZZY_ADD_${message}} == 0
+ Fail "Fuzzy Add was not run"
+ END
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -f ${RSPAMD_FLAG1_NUMBER} fuzzy_del
+ ... ${message}
+ Check Rspamc ${result}
+ Sync Fuzzy Storage
+ Scan File ${message}
+ Do Not Expect Symbol ${FLAG1_SYMBOL}
+
+Fuzzy Fuzzy Test
+ [Arguments] ${message}
+ IF ${RSPAMD_FUZZY_ADD_${message}} != 1
+ Fail "Fuzzy Add was not run"
+ END
+ @{path_info} = Path Splitter ${message}
+ @{fuzzy_files} = List Files In Directory ${pathinfo}[0] pattern=${pathinfo}[1].fuzzy* absolute=1
+ FOR ${i} IN @{fuzzy_files}
+ Scan File ${i}
+ Expect Symbol ${FLAG1_SYMBOL}
+ END
+
+Fuzzy Miss Test
+ [Arguments] ${message}
+ Scan File ${message}
+ Do Not Expect Symbol ${FLAG1_SYMBOL}
+
+Fuzzy Overwrite Test
+ [Arguments] ${message}
+ ${flag_numbers} = Create List ${RSPAMD_FLAG1_NUMBER} ${RSPAMD_FLAG2_NUMBER}
+ FOR ${i} IN @{flag_numbers}
+ ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -w 10
+ ... -f ${i} fuzzy_add ${message}
+ Check Rspamc ${result}
+ END
+ Sync Fuzzy Storage
+ Scan File ${message}
+ Do Not Expect Symbol ${FLAG1_SYMBOL}
+ Expect Symbol ${FLAG2_SYMBOL}
+
+Fuzzy Setup Encrypted
+ [Arguments] ${algorithm}
+ Set Suite Variable ${RSPAMD_FUZZY_ALGORITHM} ${algorithm}
+ Set Suite Variable ${RSPAMD_FUZZY_ENCRYPTED_ONLY} true
+ Set Suite Variable ${RSPAMD_FUZZY_ENCRYPTION_KEY} ${RSPAMD_KEY_PUB1}
+ Set Suite Variable ${RSPAMD_FUZZY_INCLUDE} ${RSPAMD_TESTDIR}/configs/fuzzy-encryption-key.conf
+ Rspamd Redis Setup
+
+Fuzzy Setup Encrypted Keyed
+ [Arguments] ${algorithm}
+ Set Suite Variable ${RSPAMD_FUZZY_ALGORITHM} ${algorithm}
+ Set Suite Variable ${RSPAMD_FUZZY_ENCRYPTED_ONLY} true
+ Set Suite Variable ${RSPAMD_FUZZY_ENCRYPTION_KEY} ${RSPAMD_KEY_PUB1}
+
+ Set Suite Variable ${RSPAMD_FUZZY_KEY} mYN888sydwLTfE32g2hN
+ Set Suite Variable ${RSPAMD_FUZZY_SHINGLES_KEY} hXUCgul9yYY3Zlk1QIT2
+ Rspamd Redis Setup
+
+Fuzzy Setup Plain
+ [Arguments] ${algorithm}
+ Set Suite Variable ${RSPAMD_FUZZY_ALGORITHM} ${algorithm}
+ Rspamd Redis Setup
+
+Fuzzy Setup Keyed
+ [Arguments] ${algorithm}
+ Set Suite Variable ${RSPAMD_FUZZY_ALGORITHM} ${algorithm}
+ Set Suite Variable ${RSPAMD_FUZZY_KEY} mYN888sydwLTfE32g2hN
+ Set Suite Variable ${RSPAMD_FUZZY_SHINGLES_KEY} hXUCgul9yYY3Zlk1QIT2
+ Rspamd Redis Setup
+
+Fuzzy Setup Plain Fasthash
+ Fuzzy Setup Plain fasthash
+
+Fuzzy Setup Plain Mumhash
+ Fuzzy Setup Plain mumhash
+
+Fuzzy Setup Plain Siphash
+ Fuzzy Setup Plain siphash
+
+Fuzzy Setup Plain Xxhash
+ Fuzzy Setup Plain xxhash
+
+Fuzzy Setup Keyed Fasthash
+ Fuzzy Setup Keyed fasthash
+
+Fuzzy Setup Keyed Mumhash
+ Fuzzy Setup Keyed mumhash
+
+Fuzzy Setup Keyed Siphash
+ Fuzzy Setup Keyed siphash
+
+Fuzzy Setup Keyed Xxhash
+ Fuzzy Setup Keyed xxhash
+
+Fuzzy Setup Encrypted Siphash
+ Fuzzy Setup Encrypted siphash
+
+Fuzzy Skip Hash Test Message
+ FOR ${i} IN @{MESSAGES_SKIP}
+ Fuzzy Skip Add Test Base ${i}
+ END
+
+Fuzzy Multimessage Add Test
+ FOR ${i} IN @{MESSAGES}
+ Fuzzy Add Test ${i}
+ END
+
+Fuzzy Multimessage Fuzzy Test
+ FOR ${i} IN @{MESSAGES}
+ Fuzzy Fuzzy Test ${i}
+ END
+
+Fuzzy Multimessage Miss Test
+ FOR ${i} IN @{RANDOM_MESSAGES}
+ Fuzzy Miss Test ${i}
+ END
+
+Fuzzy Multimessage Delete Test
+ FOR ${i} IN @{MESSAGES}
+ Fuzzy Delete Test ${i}
+ END
+
+Fuzzy Multimessage Overwrite Test
+ FOR ${i} IN @{MESSAGES}
+ Fuzzy Overwrite Test ${i}
+ END
diff --git a/test/functional/cases/120_fuzzy/mumhash-keyed.robot b/test/functional/cases/120_fuzzy/mumhash-keyed.robot
new file mode 100644
index 0000000..934a4ee
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/mumhash-keyed.robot
@@ -0,0 +1,14 @@
+*** Settings ***
+Suite Setup Fuzzy Setup Keyed Mumhash
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+ Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+ Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+ Fuzzy Multimessage Miss Test
diff --git a/test/functional/cases/120_fuzzy/mumhash.robot b/test/functional/cases/120_fuzzy/mumhash.robot
new file mode 100644
index 0000000..78abe7f
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/mumhash.robot
@@ -0,0 +1,14 @@
+*** Settings ***
+Suite Setup Fuzzy Setup Plain Mumhash
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+ Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+ Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+ Fuzzy Multimessage Miss Test
diff --git a/test/functional/cases/120_fuzzy/siphash-keyed.robot b/test/functional/cases/120_fuzzy/siphash-keyed.robot
new file mode 100644
index 0000000..a11d559
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/siphash-keyed.robot
@@ -0,0 +1,14 @@
+*** Settings ***
+Suite Setup Fuzzy Setup Keyed Siphash
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+ Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+ Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+ Fuzzy Multimessage Miss Test
diff --git a/test/functional/cases/120_fuzzy/siphash.robot b/test/functional/cases/120_fuzzy/siphash.robot
new file mode 100644
index 0000000..b477f0c
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/siphash.robot
@@ -0,0 +1,14 @@
+*** Settings ***
+Suite Setup Fuzzy Setup Plain Siphash
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+ Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+ Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+ Fuzzy Multimessage Miss Test
diff --git a/test/functional/cases/120_fuzzy/xxhash-keyed.robot b/test/functional/cases/120_fuzzy/xxhash-keyed.robot
new file mode 100644
index 0000000..84df1df
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/xxhash-keyed.robot
@@ -0,0 +1,15 @@
+*** Settings ***
+Suite Setup Fuzzy Setup Keyed Xxhash
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+ Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+ [Tags] isbroken
+ Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+ Fuzzy Multimessage Miss Test
diff --git a/test/functional/cases/120_fuzzy/xxhash.robot b/test/functional/cases/120_fuzzy/xxhash.robot
new file mode 100644
index 0000000..f7b6fa2
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/xxhash.robot
@@ -0,0 +1,15 @@
+*** Settings ***
+Suite Setup Fuzzy Setup Plain Xxhash
+Suite Teardown Rspamd Redis Teardown
+Resource lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+ Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+ [Tags] isbroken
+ Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+ Fuzzy Multimessage Miss Test
diff --git a/test/functional/cases/121_json/100_preresult.robot b/test/functional/cases/121_json/100_preresult.robot
new file mode 100644
index 0000000..04aac06
--- /dev/null
+++ b/test/functional/cases/121_json/100_preresult.robot
@@ -0,0 +1,23 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource lib.robot
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/lua_test.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RSPAMD_LUA_SCRIPT} ${RSPAMD_TESTDIR}/lua/preresult.lua
+${RSPAMD_SCOPE} Suite
+
+*** Test Cases ***
+Stat
+ Stat Test
+
+History
+ History Test soft reject
+
+Scan
+ Scan Test
diff --git a/test/functional/cases/121_json/101_simple.robot b/test/functional/cases/121_json/101_simple.robot
new file mode 100644
index 0000000..98ff0ea
--- /dev/null
+++ b/test/functional/cases/121_json/101_simple.robot
@@ -0,0 +1,23 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource lib.robot
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/lua_test.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RSPAMD_LUA_SCRIPT} ${RSPAMD_TESTDIR}/lua/simple.lua
+${RSPAMD_SCOPE} Suite
+
+*** Test Cases ***
+Stat
+ Stat Test
+
+History
+ History Test SIMPLE_TEST
+
+Scan
+ Scan Test
diff --git a/test/functional/cases/121_json/lib.robot b/test/functional/cases/121_json/lib.robot
new file mode 100644
index 0000000..68bf566
--- /dev/null
+++ b/test/functional/cases/121_json/lib.robot
@@ -0,0 +1,22 @@
+*** Variables ***
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Keywords ***
+Stat Test
+ @{result} = HTTP GET ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_CONTROLLER} /stat
+ Check JSON ${result}[1]
+ Should Be Equal As Integers ${result}[0] 200
+
+History Test
+ [Arguments] ${rspamc_expected_result}
+ ${result} = Scan Message With Rspamc ${MESSAGE}
+ Check Rspamc ${result} ${rspamc_expected_result}
+ @{result} = HTTP GET ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_CONTROLLER} /history
+ Check JSON ${result}[1]
+ Should Be Equal As Integers ${result}[0] 200
+
+Scan Test
+ ${content} = Get File ${MESSAGE}
+ @{result} = HTTP POST ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_NORMAL} /check ${content}
+ Check JSON ${result}[1]
+ Should Be Equal As Integers ${result}[0] 200
diff --git a/test/functional/cases/123_whitelist.robot b/test/functional/cases/123_whitelist.robot
new file mode 100644
index 0000000..c5a672f
--- /dev/null
+++ b/test/functional/cases/123_whitelist.robot
@@ -0,0 +1,79 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/whitelist.conf
+${M_DKIM_RSPAMD_BAD} ${RSPAMD_TESTDIR}/messages/dmarc/bad_dkim_rspamd.eml
+${M_DKIM_RSPAMD_OK} ${RSPAMD_TESTDIR}/messages/dmarc/good_dkim_rspamd.eml
+${M_DMARC_BAD} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none.eml
+${M_DMARC_OK} ${RSPAMD_TESTDIR}/messages/dmarc/pass_none.eml
+${M_NO_DKIM_VALID_SPF} ${RSPAMD_TESTDIR}/messages/dmarc/no_dkim_valid_spf.eml
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+${UTF_MESSAGE} ${RSPAMD_TESTDIR}/messages/utf.eml
+
+*** Test Cases ***
+WHITELISTS
+ Scan File ${M_DMARC_OK} IP=8.8.4.4 From=foo@spf.cacophony.za.org
+ Expect Symbol With Score WHITELIST_DKIM -1
+ Expect Symbol With Score STRICT_DMARC -3
+ Expect Symbol With Score WHITELIST_SPF_DKIM -3
+ Expect Symbol With Score WHITELIST_DDS -3
+ Expect Symbol With Score WHITELIST_DMARC -2
+ Expect Symbol With Score WHITELIST_DMARC_DKIM -2
+ Expect Symbol With Score WHITELIST_SPF -1
+ Do Not Expect Symbol BLACKLIST_SPF
+ Do Not Expect Symbol BLACKLIST_DKIM
+ Do Not Expect Symbol BLACKLIST_DMARC
+
+BLACKLIST SHOULD FIRE IF ANY CONSTRAINT FAILED
+ Scan File ${M_DMARC_OK} IP=9.8.4.4 From=foo@spf.cacophony.za.org
+ Expect Symbol With Score BLACKLIST_DDS 3
+ Do Not Expect Symbol WHITELIST_DDS
+ Do Not Expect Symbol WHITELIST_SPF
+
+BLACKLISTS
+ Scan File ${M_DMARC_BAD} IP=9.8.4.4 From=foo@cacophony.za.org
+ Expect Symbol With Score BLACKLIST_SPF 3
+ Expect Symbol With Score BLACKLIST_SPF 3
+ Expect Symbol With Score STRICT_DMARC 3
+ Expect Symbol With Score BLACKLIST_DDS 3
+ Expect Symbol With Score BLACKLIST_DMARC 2
+ Do Not Expect Symbol WHITELIST_DDS
+ Do Not Expect Symbol WHITELIST_SPF
+ Do Not Expect Symbol WHITELIST_DKIM
+ Do Not Expect Symbol WHITELIST_DMARC
+ Do Not Expect Symbol WHITELIST_DMARC_DKIM
+
+WHITELIST_WL_ONLY - VALID SPF AND VALID DKIM
+ Scan File ${M_DKIM_RSPAMD_OK}
+ ... IP=88.99.142.95
+ Expect Symbol With Score WHITELIST_DKIM -2
+ Do Not Expect Symbol BLACKLIST_DKIM
+ Expect Symbol With Score R_SPF_ALLOW 1
+ Expect Symbol With Score R_DKIM_ALLOW 1
+ Expect Symbol With Score WHITELIST_SPF_DKIM -6
+
+BLACKLISTS_WL_ONLY - VALID SPF AND INVALID DKIM
+ Scan File ${M_DKIM_RSPAMD_BAD}
+ ... IP=88.99.142.95
+ Expect Symbol With Score R_DKIM_REJECT 1
+ Do Not Expect Symbol WHITELIST_DKIM
+ Do Not Expect Symbol BLACKLIST_DKIM
+ Expect Symbol With Score R_SPF_ALLOW 1
+ Expect Symbol With Score R_DKIM_REJECT 1
+ Do Not Expect Symbol WHITELIST_SPF_DKIM
+ Do Not Expect Symbol R_DKIM_ALLOW
+
+VALID SPF and NO DKIM
+ Scan File ${M_NO_DKIM_VALID_SPF}
+ ... IP=88.99.142.95
+ Expect Symbol With Score R_SPF_ALLOW 1
+ Expect Symbol With Score R_DKIM_NA 1
+ Do Not Expect Symbol R_DKIM_REJECT
+ Do Not Expect Symbol WHITELIST_SPF_DKIM
+ Do Not Expect Symbol R_DKIM_ALLOW
diff --git a/test/functional/cases/125_map_reload.robot b/test/functional/cases/125_map_reload.robot
new file mode 100644
index 0000000..4c05ef4
--- /dev/null
+++ b/test/functional/cases/125_map_reload.robot
@@ -0,0 +1,47 @@
+*** Settings ***
+Suite Setup Map Reload Setup
+Suite Teardown Map Reload Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/lua_test.conf
+${MAP_WATCH_INTERVAL} 0.5s
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+CHECK HIT AND MISS
+ Scan File ${MESSAGE}
+ Expect Symbol With Score And Exact Options MAP_SET_HIT_AND_MISS 1 example.com
+
+WRITE NEW MAP
+ ${TMP_FILE} = Make Temporary File
+ Copy File ${RSPAMD_TESTDIR}/configs/maps/domains.list.2 ${TMP_FILE}
+ Move File ${TMP_FILE} ${MAP_FILE}
+
+CHECK HIT AND MISS AFTER RELOAD
+ Sleep 1s Wait for map reload
+ Scan File ${MESSAGE}
+ Expect Symbol With Score And Exact Options MAP_SET_HIT_AND_MISS 1 rspamd.com
+
+*** Keywords ***
+Map Reload Setup
+ ${MAP1} = Get File ${RSPAMD_TESTDIR}/configs/maps/domains.list
+ ${MAP_FILE} = Make Temporary File
+ ${RSPAMD_LUA_SCRIPT} = Make Temporary File
+ Set Suite Variable ${RSPAMD_LUA_SCRIPT}
+ Set Suite Variable ${MAP_FILE}
+ Set Suite Variable ${MAP1}
+ ${lua} = Get File ${RSPAMD_TESTDIR}/lua/mapreload.lua
+ ${lua} = Replace Variables ${lua}
+ Create File ${RSPAMD_LUA_SCRIPT} ${lua}
+ Create File ${MAP_FILE} ${MAP1}
+ Rspamd Setup
+
+Map Reload Teardown
+ Remove File ${MAP_FILE}
+ Remove File ${RSPAMD_LUA_SCRIPT}
+ Rspamd Teardown
diff --git a/test/functional/cases/131_dkim_signing/001_simple.robot b/test/functional/cases/131_dkim_signing/001_simple.robot
new file mode 100644
index 0000000..3929d87
--- /dev/null
+++ b/test/functional/cases/131_dkim_signing/001_simple.robot
@@ -0,0 +1,35 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/dkim_signing/simple.conf
+${MESSAGE_FAIL} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none1.eml
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+TEST SIGNED
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org
+ Check Rspamc ${result} DKIM-Signature:
+ Should Contain ${result.stdout} DKIM_SIGNED
+
+TEST SIGNED HTTP HEADERS
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org --header=PerformDkimSign:yes --header=DkimDomain:cacophony.za.org --header=DkimSelector:dkim --header=DkimPrivateKey:MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANe3EETkiI1Exyrb+VzbMSt90K8MXJA0GcyNs6MFCs9JPaTh90Zu2l7ki7m5LTUx6350AR/3hcvwjSHCZjD6fvQ8/zfjN8kaLZ6DAaqtqSlpawIM+8glkuTEkIkpBED/OtDrba4Rd29iLFVuwQZXDtTjAAZKZPmtTZ5TXLrcCU6VAgMBAAECgYEA1BFvmBsIN8Gu/+6kNupya2xUNVM0yLu/xT5lpNV3LBO325oejAq8+d87kkl/LTW3a2jGFlQ0ICuLw+2mo24QUWRyv8if3oeBMlnLqHE+6wNjFVqo5sOjKzjO363xSXwXNUrBT7jDhnZcDN8w3/FecYKjifGTVtUs1SLsYwhlc8ECQQDuCRymLZQ/imPn5eFVIydwUzg8ptZlvoA7bfIxUL9BQRX33s59kLCilA0tTed8Dd+GnxsT93XOj1ApIfBwmTSlAkEA5/63PDsN7fH+WInqVD8nU07M9S8LcGDlPbVVBr2S2I78/iwrSDAYtbkU2vEbhFK/JuKNML2j8OkzV3v1QulfMQJBALDzhx+l/HHr3+8RPhx7QKNIyiKUaAdEwbDsP8IXY8YPq1QThu9jM1v4sX7/TdkzuvoppwiFykbe1NlvCH279p0CQCmTg4Ee0DtBcCSr6rvYaZLLf329RZ6JLuwlMCy6ErQOxBZFEiiovfTrS2qFZToMnkc4uLbwdY36LQJTq7unGTECQCCok8LzBeZtAw+TJofpOM3F2Rlm2qXiBVBeubhRedsiljG0hpvvLJBMppnQ6r27p5Jk39SmaTRkxEKrxPWWLNM=
+ Check Rspamc ${result} DKIM-Signature:
+ Should Contain ${result.stdout} DKIM_SIGNED
+
+TEST NOT SIGNED - USERNAME WRONG DOMAIN
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@example.tk
+ Check Rspamc ${result} DKIM-Signature: inverse=1
+ Should Not Contain ${result.stdout} DKIM_SIGNED
+
+TEST NOT SIGNED - USERNAME WRONG PUBKEY
+ ${result} = Scan Message With Rspamc ${MESSAGE_FAIL} -u bob@invalid.za.org
+ Check Rspamc ${result} DKIM-Signature: inverse=1
+ Should Not Contain ${result.stdout} DKIM_SIGNED
diff --git a/test/functional/cases/131_dkim_signing/002_redis.robot b/test/functional/cases/131_dkim_signing/002_redis.robot
new file mode 100644
index 0000000..6e8f888
--- /dev/null
+++ b/test/functional/cases/131_dkim_signing/002_redis.robot
@@ -0,0 +1,31 @@
+*** Settings ***
+Suite Setup DKIM Signing Setup
+Suite Teardown Rspamd Redis Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/dkim_signing/redis.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+TEST SIGNED
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org
+ Check Rspamc ${result} DKIM-Signature:
+ Should Contain ${result.stdout} DKIM_SIGNED
+
+TEST NOT SIGNED - USERNAME WRONG DOMAIN
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@example.tk
+ Check Rspamc ${result} DKIM-Signature: inverse=1
+ Should Not Contain ${result.stdout} DKIM_SIGNED
+
+*** Keywords ***
+DKIM Signing Setup
+ Rspamd Redis Setup
+ Redis HSET TEST_DKIM_SELECTORS cacophony.za.org dkim
+ ${key} = Get File ${RSPAMD_TESTDIR}/configs/dkim.key
+ Redis HSET TEST_DKIM_KEYS dkim.cacophony.za.org ${key}
diff --git a/test/functional/cases/131_dkim_signing/003_eddsa.robot b/test/functional/cases/131_dkim_signing/003_eddsa.robot
new file mode 100644
index 0000000..27d1c0c
--- /dev/null
+++ b/test/functional/cases/131_dkim_signing/003_eddsa.robot
@@ -0,0 +1,30 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/dkim_signing/eddsa.conf
+${MESSAGE_FAIL} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none1.eml
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+TEST SIGNED
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org
+ Check Rspamc ${result} DKIM-Signature:
+ Should Contain ${result.stdout} DKIM_SIGNED
+
+TEST NOT SIGNED - USERNAME WRONG DOMAIN
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@example.tk
+ Check Rspamc ${result} DKIM-Signature: inverse=1
+ Should Not Contain ${result.stdout} DKIM_SIGNED
+
+TEST NOT SIGNED - USERNAME WRONG PUBKEY
+ ${result} = Scan Message With Rspamc ${MESSAGE_FAIL} -u bob@invalid.za.org
+ Check Rspamc ${result} DKIM-Signature: inverse=1
+ Should Not Contain ${result.stdout} DKIM_SIGNED
diff --git a/test/functional/cases/131_dkim_signing/004_invalidate_key.robot b/test/functional/cases/131_dkim_signing/004_invalidate_key.robot
new file mode 100644
index 0000000..2218c34
--- /dev/null
+++ b/test/functional/cases/131_dkim_signing/004_invalidate_key.robot
@@ -0,0 +1,50 @@
+*** Settings ***
+Force Tags isbroken
+Suite Setup Key Invalidation Setup
+Suite Teardown Key Invalidation Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/dkim_signing/invalidate.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+TEST SIGNED
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org
+ Check Rspamc ${result} DKIM-Signature:
+ Should Contain ${result.stdout} DKIM_SIGNED
+
+TEST NOT SIGNED - MISSING KEY
+ [Setup] Delete Key
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org
+ Check Rspamc ${result} DKIM-Signature: inverse=1
+ Should Not Contain ${result.stdout} DKIM_SIGNED
+
+TEST NOT SIGNED - KEY NO LONGER MATCHES
+ [Setup] Move Key
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org
+ Check Rspamc ${result} DKIM-Signature: inverse=1
+ Should Not Contain ${result.stdout} DKIM_SIGNED
+
+*** Keywords ***
+Key Invalidation Setup
+ ${key_dir} Make Temporary Directory
+ Set Suite Variable ${RSPAMD_KEY_DIR} ${key_dir}
+ Copy File ${RSPAMD_TESTDIR}/configs/dkim-eddsa.key ${RSPAMD_KEY_DIR}/dkim-eddsa.key
+ Rspamd Setup
+
+Delete Key
+ Remove File ${RSPAMD_KEY_DIR}/dkim-eddsa.key
+
+Move Key
+ Copy File ${RSPAMD_TESTDIR}/configs/dkim.key ${RSPAMD_KEY_DIR}/dkim-eddsa.key
+ Set Modified Time ${RSPAMD_KEY_DIR}/dkim-eddsa.key NOW + 3s
+
+Key Invalidation Teardown
+ Cleanup Temporary Directory ${RSPAMD_KEY_DIR}
+ Rspamd Teardown
diff --git a/test/functional/cases/131_dkim_signing/005_multiple.robot b/test/functional/cases/131_dkim_signing/005_multiple.robot
new file mode 100644
index 0000000..067220d
--- /dev/null
+++ b/test/functional/cases/131_dkim_signing/005_multiple.robot
@@ -0,0 +1,20 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/dkim_signing/multiple.conf
+${MESSAGE_FAIL} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none1.eml
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+TEST DOUBLE SIGNED
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org
+ Check Rspamc ${result} (?s)DKIM-Signature.+DKIM-Signature re=1
+ Should Contain ${result.stdout} DKIM_SIGNED
diff --git a/test/functional/cases/131_dkim_signing/006_milter.robot b/test/functional/cases/131_dkim_signing/006_milter.robot
new file mode 100644
index 0000000..2b63d07
--- /dev/null
+++ b/test/functional/cases/131_dkim_signing/006_milter.robot
@@ -0,0 +1,31 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Test Tags miltertest
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/dkim_signing/milter.conf
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+SINGLE SIGNATURE
+ Milter Test dkim_one.lua
+
+MULTIPLE SIGNATURES
+ Milter Test dkim_many.lua
+
+*** Keywords ***
+Milter Test
+ [Arguments] ${mtlua}
+ Skip If not ${HAVE_MILTERTEST} msg=miltertest not installed
+ ${result} = Run Process miltertest -Dport\=${RSPAMD_PORT_PROXY} -Dhost\=${RSPAMD_LOCAL_ADDR} -s ${RSPAMD_TESTDIR}/lua/miltertest/${mtlua}
+ ... cwd=${RSPAMD_TESTDIR}/lua/miltertest
+ Should Match Regexp ${result.stderr} ^$
+ Log ${result.rc}
+ Log ${result.stdout}
+ Should Be Equal As Integers ${result.rc} 0 msg=${result.stdout} values=false
diff --git a/test/functional/cases/131_dkim_signing/007_sign_maps.robot b/test/functional/cases/131_dkim_signing/007_sign_maps.robot
new file mode 100644
index 0000000..f0523cd
--- /dev/null
+++ b/test/functional/cases/131_dkim_signing/007_sign_maps.robot
@@ -0,0 +1,25 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/dkim_signing/sign_maps.conf
+${MESSAGE_FAIL} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none1.eml
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+TEST SIGNED
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org
+ Check Rspamc ${result} DKIM-Signature:
+ Should Contain ${result.stdout} DKIM_SIGNED
+
+TEST NOT SIGNED - FROM WRONG DOMAIN
+ ${result} = Scan Message With Rspamc ${MESSAGE_FAIL} -u bob@cacophony.za.org
+ Check Rspamc ${result} DKIM-Signature: inverse=1
+ Should Not Contain ${result.stdout} DKIM_SIGNED
diff --git a/test/functional/cases/135_spamassassin.robot b/test/functional/cases/135_spamassassin.robot
new file mode 100644
index 0000000..f9691fd
--- /dev/null
+++ b/test/functional/cases/135_spamassassin.robot
@@ -0,0 +1,34 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/spamassassin.conf
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+FREEMAIL
+ Scan File ${RSPAMD_TESTDIR}/messages/freemail.eml
+ ... From=faked.asdfjisiwosp372@outlook.com
+ Expect Symbol FREEMAIL_FROM
+ Expect Symbol FREEMAIL_ENVFROM_END_DIGIT
+ Expect Symbol FREEMAIL_SUBJECT
+ Expect Symbol TEST_META4
+
+WLBL WHITELIST
+ Scan File ${RSPAMD_TESTDIR}/messages/bad_message.eml
+ Expect Symbol USER_IN_WHITELIST
+ Expect Symbol USER_IN_WHITELIST_TO
+ Do Not Expect Symbol USER_IN_BLACKLIST_TO
+ Do Not Expect Symbol USER_IN_BLACKLIST
+
+WLBL BLACKLIST
+ Scan File ${RSPAMD_TESTDIR}/messages/utf.eml
+ Expect Symbol USER_IN_BLACKLIST
+ Expect Symbol USER_IN_BLACKLIST_TO
+ Do Not Expect Symbol USER_IN_WHITELIST_TO
+ Do Not Expect Symbol USER_IN_WHITELIST
diff --git a/test/functional/cases/140_proxy.robot b/test/functional/cases/140_proxy.robot
new file mode 100644
index 0000000..d43cfa4
--- /dev/null
+++ b/test/functional/cases/140_proxy.robot
@@ -0,0 +1,50 @@
+*** Settings ***
+Suite Setup Proxy Setup
+Suite Teardown Proxy Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RSPAMD_LUA_SCRIPT} ${RSPAMD_TESTDIR}/lua/simple.lua
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+HTTP PROTOCOL
+ Set Test Variable ${RSPAMD_PORT_NORMAL} ${RSPAMD_PORT_PROXY}
+ Scan File ${MESSAGE}
+ Expect Symbol SIMPLE_TEST
+
+SPAMC
+ ${result} = Spamc ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_PROXY} ${MESSAGE}
+ Should Contain ${result} SPAMD/1.1 0 EX_OK
+
+RSPAMC Legacy Protocol
+ ${result} = Rspamc ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_PROXY} ${MESSAGE}
+ Should Contain ${result} RSPAMD/1.3 0 EX_OK
+
+*** Keywords ***
+Proxy Setup
+ # Run slave & copy variables
+ Set Suite Variable ${CONFIG} ${RSPAMD_TESTDIR}/configs/lua_test.conf
+ Rspamd Setup
+ Set Suite Variable ${SLAVE_PID} ${RSPAMD_PID}
+ Set Suite Variable ${SLAVE_TMPDIR} ${RSPAMD_TMPDIR}
+
+ # Run proxy & copy variables
+ Set Suite Variable ${CONFIG} ${RSPAMD_TESTDIR}/configs/proxy.conf
+ Rspamd Setup
+ Set Suite Variable ${PROXY_PID} ${RSPAMD_PID}
+ Set Suite Variable ${PROXY_TMPDIR} ${RSPAMD_TMPDIR}
+
+Proxy Teardown
+ # Restore variables & run normal teardown
+ Set Suite Variable ${RSPAMD_PID} ${PROXY_PID}
+ Set Suite Variable ${RSPAMD_TMPDIR} ${PROXY_TMPDIR}
+ Rspamd Teardown
+ # Do it again for slave
+ Set Suite Variable ${RSPAMD_PID} ${SLAVE_PID}
+ Set Suite Variable ${RSPAMD_TMPDIR} ${SLAVE_TMPDIR}
+ Rspamd Teardown
diff --git a/test/functional/cases/150_rspamadm.robot b/test/functional/cases/150_rspamadm.robot
new file mode 100644
index 0000000..ef3db98
--- /dev/null
+++ b/test/functional/cases/150_rspamadm.robot
@@ -0,0 +1,45 @@
+*** Settings ***
+Library Process
+Library ../lib/rspamd.py
+
+Suite Teardown Terminate All Processes kill=True
+
+*** Test Cases ***
+Config Test
+ ${result} = Run Process ${RSPAMADM} configtest
+ Should Match Regexp ${result.stderr} ^$
+ Should Match Regexp ${result.stdout} ^syntax OK$
+ Should Be Equal As Integers ${result.rc} 0
+
+Config Help
+ ${result} = Run Process ${RSPAMADM} confighelp
+ Should Match Regexp ${result.stderr} ^$
+ Should Be Equal As Integers ${result.rc} 0
+
+Simple interpreter
+ ${handle} = Start Process ${RSPAMADM} lua stdin=PIPE
+ ${result} = Write to stdin ${handle} 1+1
+ Should Be Equal As Strings ${result} 2\n
+
+Simple interpreter, two results
+ ${handle} = Start Process ${RSPAMADM} lua stdin=PIPE
+ ${result} = Write to stdin ${handle} 1+1, 2 * 5
+ Should Be Equal ${result} 2\n10\n
+
+Process message callback
+ ${handle} = Start Process ${RSPAMADM} lua stdin=PIPE
+ ${result} = Write to stdin ${handle} .load ${RSPAMD_TESTDIR}/lua/rspamadm/test_message_callback.lua\n.message message_callback ${RSPAMD_TESTDIR}/messages/empty_part.eml
+ Should Contain ${result} n parts = 2
+ Should Contain ${result} 1\n2\n4\n6
+
+Lua batch mode
+ ${result} = Run Process ${RSPAMADM} lua -b ${RSPAMD_TESTDIR}/lua/rspamadm/test_batch.lua
+ Should Be Equal ${result.stderr} hello world
+ Should Match Regexp ${result.stdout} ^$
+ Should Be Equal As Integers ${result.rc} 0
+
+Verbose mode
+ ${result} = Run Process ${RSPAMADM} -v lua ${RSPAMD_TESTDIR}/lua/rspamadm/test_verbose.lua
+ Should Match Regexp ${result.stderr} ^$
+ Should Match Regexp ${result.stdout} hello world\n
+ Should Be Equal As Integers ${result.rc} 0
diff --git a/test/functional/cases/151_rspamadm_async.robot b/test/functional/cases/151_rspamadm_async.robot
new file mode 100644
index 0000000..6aff67a
--- /dev/null
+++ b/test/functional/cases/151_rspamadm_async.robot
@@ -0,0 +1,54 @@
+*** Settings ***
+Test Setup Rspamadm test Setup
+Test Teardown Rspamadm test Teardown
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/plugins.conf
+${REDIS_SCOPE} Test
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+#Tcp client
+# ${result} = Run Process ${RSPAMADM} lua -b ${RSPAMD_TESTDIR}/lua/rspamadm/test_tcp_client.lua
+# Should Match Regexp ${result.stderr} ^$
+# Should Be Equal As Integers ${result.rc} 0
+# Should Be Equal ${result.stdout} hello post
+
+Redis client
+ ${result} = Run Process ${RSPAMADM} lua -b ${RSPAMD_TESTDIR}/lua/rspamadm/test_redis_client.lua
+ Should Match Regexp ${result.stderr} ^$
+ Should Be Equal As Integers ${result.rc} 0
+ Should Be Equal ${result.stdout} true\thello from lua on redis
+
+# Broken due to tmpdir override
+#DNS client
+# ${result} = Run Process ${RSPAMADM} --var\=CONFDIR\=${tmpdir} lua -b ${RSPAMD_TESTDIR}/lua/rspamadm/test_dns_client.lua
+# Log ${result.stdout}
+# Log ${result.stderr}
+# Should Be Equal As Integers ${result.rc} 0
+# Should Be Equal ${result.stdout} true\tk=ed25519; p=yi50DjK5O9pqbFpNHklsv9lqaS0ArSYu02qp1S0DW1Y=
+# Cleanup Temporary Directory ${tmpdir}
+
+*** Keywords ***
+
+Rspamadm test Setup
+ Run Dummy Http
+ Run Redis
+
+Rspamadm test Teardown
+ Dummy Http Teardown
+ Redis Teardown
+
+Prepare temp directory
+ [Arguments] ${CONFIG}
+ ${template} = Get File ${CONFIG}
+ ${tmpdir} = Make Temporary Directory
+ ${config} = Replace Variables ${template}
+ ${config} = Replace Variables ${config}
+ Log ${config}
+ Create File ${tmpdir}/rspamd.conf ${config}
+ [Return] ${tmpdir}
diff --git a/test/functional/cases/161_p0f.robot b/test/functional/cases/161_p0f.robot
new file mode 100644
index 0000000..9b525d4
--- /dev/null
+++ b/test/functional/cases/161_p0f.robot
@@ -0,0 +1,97 @@
+*** Settings ***
+Suite Setup p0f Setup
+Suite Teardown p0f Teardown
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/p0f.conf
+${MESSAGE2} ${RSPAMD_TESTDIR}/messages/freemail.eml
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+p0f MISS
+ Run Dummy p0f
+ Scan File ${MESSAGE} IP=1.1.1.1
+ Do Not Expect Symbol P0F_FAIL
+ Do Not Expect Symbol WINDOWS
+ Expect Symbol With Exact Options P0F Linux 3.11 and newer link=Ethernet or modem distance=10
+ Shutdown p0f
+
+p0f HIT
+ Run Dummy p0f ${RSPAMD_P0F_SOCKET} windows
+ Scan File ${MESSAGE} IP=1.1.1.2
+ Do Not Expect Symbol P0F_FAIL
+ Expect Symbol With Exact Options P0F link=Ethernet or modem distance=10
+ Expect Symbol WINDOWS
+ Shutdown p0f
+
+p0f MISS CACHE
+ Run Dummy p0f
+ Scan File ${MESSAGE} IP=1.1.1.3
+ Do Not Expect Symbol WINDOWS
+ Shutdown p0f
+ Scan File ${MESSAGE} IP=1.1.1.3
+ Do Not Expect Symbol WINDOWS
+ Do Not Expect Symbol P0F_FAIL
+
+p0f HIT CACHE
+ Run Dummy p0f ${RSPAMD_P0F_SOCKET} windows
+ Scan File ${MESSAGE} IP=1.1.1.4
+ Expect Symbol WINDOWS
+ Shutdown p0f
+ Scan File ${MESSAGE} IP=1.1.1.4
+ Expect Symbol WINDOWS
+ Do Not Expect Symbol P0F_FAIL
+
+p0f NO REDIS
+ Shutdown Process With Children ${REDIS_PID}
+ Run Dummy p0f
+ Scan File ${MESSAGE} IP=1.1.1.5
+ Expect Symbol With Exact Options P0F Linux 3.11 and newer link=Ethernet or modem distance=10
+ Do Not Expect Symbol P0F_FAIL
+ Shutdown p0f
+
+p0f NO MATCH
+ Run Dummy p0f ${RSPAMD_P0F_SOCKET} windows no_match
+ Scan File ${MESSAGE} IP=1.1.1.6
+ Do Not Expect Symbol P0F
+ Do Not Expect Symbol WINDOWS
+ Shutdown p0f
+
+p0f BAD QUERY
+ Run Dummy p0f ${RSPAMD_P0F_SOCKET} windows bad_query
+ Scan File ${MESSAGE} IP=1.1.1.7
+ Expect Symbol With Exact Options P0F_FAIL Malformed Query: /tmp/p0f.sock
+ Do Not Expect Symbol WINDOWS
+ Shutdown p0f
+
+p0f BAD RESPONSE
+ Run Dummy p0f ${RSPAMD_P0F_SOCKET} windows bad_response
+ Scan File ${MESSAGE} IP=1.1.1.8
+ Expect Symbol With Exact Options P0F_FAIL Error getting result: IO read error: connection terminated
+ Do Not Expect Symbol WINDOWS
+ Shutdown p0f
+
+*** Keywords ***
+p0f Setup
+ Rspamd Redis Setup
+
+p0f Teardown
+ Rspamd Redis Teardown
+ Shutdown p0f
+ Terminate All Processes kill=True
+
+Shutdown p0f
+ ${p0f_pid} = Get File if exists /tmp/dummy_p0f.pid
+ Run Keyword if ${p0f_pid} Shutdown Process With Children ${p0f_pid}
+
+Run Dummy p0f
+ [Arguments] ${socket}=${RSPAMD_P0F_SOCKET} ${os}=linux ${status}=ok
+ ${result} = Start Process ${RSPAMD_TESTDIR}/util/dummy_p0f.py ${socket} ${os} ${status}
+ Wait Until Created /tmp/dummy_p0f.pid
diff --git a/test/functional/cases/162_url_redirector.robot b/test/functional/cases/162_url_redirector.robot
new file mode 100644
index 0000000..857a349
--- /dev/null
+++ b/test/functional/cases/162_url_redirector.robot
@@ -0,0 +1,34 @@
+*** Settings ***
+Suite Setup Urlredirector Setup
+Suite Teardown Urlredirector Teardown
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/url_redirector.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/redir.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+${SETTINGS} {symbols_enabled=[URL_REDIRECTOR_CHECK]}
+
+*** Test Cases ***
+RESOLVE URLS
+ Scan File ${MESSAGE} Flags=ext_urls Settings=${SETTINGS}
+ Expect Extended URL http://127.0.0.1:18080/hello
+
+RESOLVE URLS CACHED
+ Scan File ${MESSAGE} Flags=ext_urls Settings=${SETTINGS}
+ Expect Extended URL http://127.0.0.1:18080/hello
+
+*** Keywords ***
+Urlredirector Setup
+ Run Dummy Http
+ Rspamd Redis Setup
+
+Urlredirector Teardown
+ Rspamd Redis Teardown
+ Dummy Http Teardown
+ Terminate All Processes kill=True
diff --git a/test/functional/cases/180_milter.robot b/test/functional/cases/180_milter.robot
new file mode 100644
index 0000000..d41751c
--- /dev/null
+++ b/test/functional/cases/180_milter.robot
@@ -0,0 +1,40 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Test Tags miltertest
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/milter.conf
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+ACCEPT
+ Milter Test mt1.lua
+
+REJECT
+ Milter Test mt2.lua
+
+REWRITE SUBJECT
+ Milter Test mt3.lua
+
+DEFER
+ Milter Test mt4.lua
+
+COMBINED TEST
+ Milter Test combined.lua
+
+*** Keywords ***
+Milter Test
+ [Arguments] ${mtlua}
+ Skip If not ${HAVE_MILTERTEST} msg=miltertest not installed
+ ${result} = Run Process miltertest -Dport\=${RSPAMD_PORT_PROXY} -Dhost\=${RSPAMD_LOCAL_ADDR} -s ${RSPAMD_TESTDIR}/lua/miltertest/${mtlua}
+ ... cwd=${RSPAMD_TESTDIR}/lua/miltertest
+ Should Match Regexp ${result.stderr} ^$
+ Log ${result.rc}
+ Log ${result.stdout}
+ Should Be Equal As Integers ${result.rc} 0 msg=${result.stdout} values=false
diff --git a/test/functional/cases/210_clickhouse/001_migration.robot b/test/functional/cases/210_clickhouse/001_migration.robot
new file mode 100644
index 0000000..2153c23
--- /dev/null
+++ b/test/functional/cases/210_clickhouse/001_migration.robot
@@ -0,0 +1,91 @@
+*** Settings ***
+Documentation Checks if rspamd is able to upgrade migration schema from v0 (very initial) to v2
+Test Setup Clickhouse Setup
+Test Teardown Clickhosue Teardown
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Library clickhouse.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/clickhouse.conf
+${RSPAMD_SCOPE} Suite
+${CLICKHOUSE_PORT} ${18123}
+
+*** Test Cases ***
+# Usually broken
+#Migration
+ #Initial schema
+ # Prepare rspamd
+ # Sleep 2 #TODO: replace this check with waiting until migration finishes
+ # Column should exist rspamd Symbols.Scores
+ # Column should exist rspamd Attachments.Digest
+ # Column should exist rspamd Symbols.Scores
+ # Schema version should be 3
+# Upload new schema ${RSPAMD_TESTDIR}/data/initial_schema/schema.sql
+# Insert data rspamd ${RSPAMD_TESTDIR}/data/initial_schema/data.rspamd.sql
+# Insert data rspamd_asn ${RSPAMD_TESTDIR}/data/initial_schema/data.rspamd_asn.sql
+# Insert data rspamd_urls ${RSPAMD_TESTDIR}/data/initial_schema/data.rspamd_urls.sql
+# Insert data rspamd_emails ${RSPAMD_TESTDIR}/data/initial_schema/data.rspamd_emails.sql
+# Insert data rspamd_symbols ${RSPAMD_TESTDIR}/data/initial_schema/data.rspamd_symbols.sql
+# Insert data rspamd_attachments ${RSPAMD_TESTDIR}/data/initial_schema/data.rspamd_attachments.sql
+# Prepare rspamd
+# Sleep 2 #TODO: replace this check with waiting until migration finishes
+# Column should exist rspamd Symbols.Scores
+# Column should exist rspamd Attachments.Digest
+# Column should exist rspamd Symbols.Scores
+# # Added in schema version 7
+# Column should exist rspamd Helo
+# Column should exist rspamd SMTPRecipients
+# # Added in schema version 8
+# Column should exist rspamd Groups.Scores
+# Schema version should be 8
+
+# Eventually broken
+#Retention
+# Upload new schema ${RSPAMD_TESTDIR}/data/schema_2/schema.sql
+# Insert data rspamd ${RSPAMD_TESTDIR}/data/schema_2/data.rspamd.sql
+# Assert rows count rspamd 56
+# Prepare rspamd
+# Sleep 2 #TODO: replace this check with waiting until migration finishes
+# Assert rows count rspamd 30
+
+*** Keywords ***
+Clickhouse Setup
+ ${RSPAMD_TMPDIR} = Make Temporary Directory
+ Set Suite Variable ${RSPAMD_TMPDIR}
+ Set Directory Ownership ${RSPAMD_TMPDIR} ${RSPAMD_USER} ${RSPAMD_GROUP}
+ ${template} = Get File ${RSPAMD_TESTDIR}/configs/clickhouse-config.xml
+ ${config} = Replace Variables ${template}
+ Create File ${RSPAMD_TMPDIR}/clickhouse-config.xml ${config}
+ Copy File ${RSPAMD_TESTDIR}/configs/clickhouse-users.xml ${RSPAMD_TMPDIR}/users.xml
+ Create Directory ${RSPAMD_TMPDIR}/clickhouse
+ Set Directory Ownership ${RSPAMD_TMPDIR}/clickhouse clickhouse clickhouse
+ ${result} = Run Process su -s /bin/sh clickhouse -c
+ ... clickhouse-server --daemon --config-file\=${RSPAMD_TMPDIR}/clickhouse-config.xml --pid-file\=${RSPAMD_TMPDIR}/clickhouse/clickhouse.pid
+ IF ${result.rc} != 0
+ Log ${result.stderr}
+ END
+ Should Be Equal As Integers ${result.rc} 0
+ Wait Until Keyword Succeeds 5 sec 50 ms TCP Connect localhost ${CLICKHOUSE_PORT}
+ Set Suite Variable ${RSPAMD_TMPDIR} ${RSPAMD_TMPDIR}
+
+Clickhosue Teardown
+ # Sleep 30
+ ${clickhouse_pid} = Get File ${RSPAMD_TMPDIR}/clickhouse/clickhouse.pid
+ Shutdown Process With Children ${clickhouse_pid}
+ Log File ${RSPAMD_TMPDIR}/clickhouse/clickhouse-server.err.log
+ Rspamd Teardown
+
+Prepare rspamd
+ &{d} = Run Rspamd CONFIG=${RSPAMD_TESTDIR}/configs/clickhouse.conf TMPDIR=${RSPAMD_TMPDIR}
+ ${keys} = Get Dictionary Keys ${d}
+ FOR ${i} IN @{keys}
+ IF '${RSPAMD_SCOPE}' == 'Suite'
+ Set Suite Variable ${${i}} ${d}[${i}]
+ ELSE IF '${RSPAMD_SCOPE}' == 'Test'
+ Set Test Variable ${${i}} ${d}[${i}]
+ ELSE
+ Fail 'RSPAMD_SCOPE must be Test or Suite'
+ END
+ END
diff --git a/test/functional/cases/210_clickhouse/clickhouse.py b/test/functional/cases/210_clickhouse/clickhouse.py
new file mode 100644
index 0000000..69430eb
--- /dev/null
+++ b/test/functional/cases/210_clickhouse/clickhouse.py
@@ -0,0 +1,78 @@
+import requests
+import json
+from robot.libraries.BuiltIn import BuiltIn
+from robot.api import logger
+
+__client = None
+
+
+class Client:
+ def __init__(self):
+ self.port = BuiltIn().get_variable_value('${CLICKHOUSE_PORT}', default=18123)
+
+ def get_query_string(self):
+ return "http://localhost:%d/?default_format=JSONEachRow" % (self.port)
+
+ def execute(self, sql):
+ r = requests.post(self.get_query_string(), sql)
+ logger.info("Client.execute: response: %s" % str(r))
+ if r.status_code != 200:
+ raise Exception("Clickhouse request failed: " + r.content)
+ return r
+
+ def query(self, sql):
+ r = self.execute(sql)
+
+ # logger.console("decoding " + r.content)
+ # [logger.console("decoding " + _) for _ in r.content.strip().split("\n")]
+ response = [json.loads(_) for _ in r.content.strip().split("\n")]
+ return response
+
+
+def client():
+ global __client
+ if __client is None:
+ __client = Client()
+ return __client
+
+
+def upload_new_schema(schema_file):
+ with open(schema_file, 'r') as content_file:
+ content = content_file.read()
+
+ queries = content.split(";")
+ for q in queries:
+ if q.strip() == "":
+ continue
+ client().execute(q) # throws exception on error
+
+
+def insert_data(table_name, filename):
+ with open(filename, 'r') as content_file:
+ content = content_file.read()
+
+ client().execute("insert into %s format Values %s;" % (table_name, content)) # throws exception on error
+
+
+def column_should_exist(table_name, column_name):
+ sql = "select hasColumnInTable('default', '%s', '%s') as is_exist" % (table_name, column_name)
+ r = client().query(sql)
+ logger.info("response: %s" % str(r))
+ if r[0]['is_exist'] != 1:
+ raise Exception("Failed asseting that column '%s' exists in table 'default'.'%s'" % (column_name, table_name))
+
+
+def schema_version_should_be(version):
+ sql = "select max(Version) as version from rspamd_version"
+ r = client().query(sql)
+ logger.info("response: %s" % str(r))
+ if r[0]['version'] != int(version):
+ raise Exception("Failed asseting that schema version is %d, %d schema version in ClickHouse" % (int(version), int(r[0]['version'])))
+
+
+def assert_rows_count(table_name, number):
+ sql = "select count(*) as cnt from %s" % table_name
+ r = client().query(sql)
+ logger.info("response: %s" % str(r))
+ if int(r[0]['cnt']) != int(number):
+ raise Exception("Failed asserting that table '%s' has %d rows (actual number: %d)" % (table_name, int(number), int(r[0]['cnt'])))
diff --git a/test/functional/cases/220_http.robot b/test/functional/cases/220_http.robot
new file mode 100644
index 0000000..b3c42a3
--- /dev/null
+++ b/test/functional/cases/220_http.robot
@@ -0,0 +1,74 @@
+*** Settings ***
+Test Setup Http Setup
+Test Teardown Http Teardown
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/lua_test.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RSPAMD_LUA_SCRIPT} ${RSPAMD_TESTDIR}/lua/http.lua
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+Simple HTTP request
+ Scan File ${MESSAGE} Url=/request Method=get
+ ... Settings={symbols_enabled = [SIMPLE_HTTP_TEST]}
+ Expect Symbols HTTP_DNS_200 HTTP_200 HTTP_CORO_DNS_200
+ Expect Symbol With Exact Options HTTP_CORO_200 hello world
+
+ Scan File ${MESSAGE} Url=/request Method=post
+ ... Settings={symbols_enabled = [SIMPLE_HTTP_TEST]}
+ Expect Symbols HTTP_DNS_200 HTTP_200 HTTP_CORO_DNS_200
+ Expect Symbol With Exact Options HTTP_CORO_200 hello post
+
+HTTP request 403
+ Scan File ${MESSAGE} Url=/error_403 Method=get
+ ... Settings={symbols_enabled = [SIMPLE_HTTP_TEST]}
+ Expect Symbols HTTP_DNS_403 HTTP_403 HTTP_CORO_DNS_403 method_get
+
+ Scan File ${MESSAGE} Url=/error_403 Method=post
+ ... Settings={symbols_enabled = [SIMPLE_HTTP_TEST]}
+ Expect Symbols HTTP_DNS_403 HTTP_403 HTTP_CORO_DNS_403 method_post
+
+HTTP timeout
+ Scan File ${MESSAGE} Url=/timeout Method=get
+ ... Settings={symbols_enabled = [SIMPLE_HTTP_TEST]}
+ Expect Symbols HTTP_ERROR HTTP_ERROR HTTP_CORO_DNS_ERROR HTTP_CORO_ERROR method_get
+ # FIXME: where is "IO timeout"
+
+ Scan File ${MESSAGE} Url=/timeout Method=post
+ ... Settings={symbols_enabled = [SIMPLE_HTTP_TEST]}
+ Expect Symbols HTTP_ERROR HTTP_ERROR HTTP_CORO_DNS_ERROR HTTP_CORO_ERROR method_post
+ # FIXME: where is "IO timeout"
+
+# Broken as dummy server is now not so dummy, not sure what is expected to be tested here
+#HTTP empty response
+# Scan File ${MESSAGE} Url=/empty Method=get
+# ... Settings={symbols_enabled = [SIMPLE_HTTP_TEST]}
+# Expect Symbols HTTP_ERROR HTTP_ERROR HTTP_CORO_DNS_ERROR HTTP_CORO_ERROR method_get
+# # FIXME: where is "IO read error: unexpected EOF"
+#
+# Scan File ${MESSAGE} Url=/empty Method=post
+# ... Settings={symbols_enabled = [SIMPLE_HTTP_TEST]}
+# Expect Symbols HTTP_ERROR HTTP_ERROR HTTP_CORO_DNS_ERROR HTTP_CORO_ERROR method_post
+# # FIXME: where is "IO read error: unexpected EOF"
+
+SSL Large HTTP request
+ Scan File ${MESSAGE}
+ ... Settings={symbols_enabled = [LARGE_HTTP_TEST]}
+ Expect Symbol HTTP_SSL_LARGE
+
+*** Keywords ***
+Http Setup
+ Run Dummy Http
+ Run Dummy Https
+ Rspamd Setup
+
+Http Teardown
+ Rspamd Teardown
+ Dummy Http Teardown
+ Dummy Https Teardown
diff --git a/test/functional/cases/230_tcp.robot b/test/functional/cases/230_tcp.robot
new file mode 100644
index 0000000..d53a7fc
--- /dev/null
+++ b/test/functional/cases/230_tcp.robot
@@ -0,0 +1,75 @@
+*** Settings ***
+Suite Setup Servers Setup
+Suite Teardown Servers Teardown
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/lua_test.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RSPAMD_LUA_SCRIPT} ${RSPAMD_TESTDIR}/lua/tcp.lua
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+Simple TCP request
+ Scan File ${MESSAGE}
+ ... Settings={symbols_enabled = [SIMPLE_TCP_ASYNC_TEST]}
+ Expect Symbol HTTP_ASYNC_RESPONSE
+ Expect Symbol HTTP_ASYNC_RESPONSE_2
+
+#SSL TCP request
+# Scan File ${MESSAGE}
+# ... Settings={symbols_enabled = [SIMPLE_TCP_ASYNC_SSL_TEST]}
+# Expect Symbol With Exact Options TCP_SSL_RESPONSE hello
+# Expect Symbol With Exact Options TCP_SSL_RESPONSE_2 hello
+
+#SSL Large TCP request
+# Scan File ${MESSAGE}
+# ... Settings={symbols_enabled = [LARGE_TCP_ASYNC_SSL_TEST]}
+# Expect Symbol TCP_SSL_LARGE
+# Expect Symbol TCP_SSL_LARGE_2
+
+Sync API TCP request
+ Scan File ${MESSAGE}
+ ... Settings={symbols_enabled = [SIMPLE_TCP_TEST]}
+ Expect Symbol HTTP_SYNC_RESPONSE
+ Should Contain ${SCAN_RESULT}[symbols][HTTP_SYNC_RESPONSE][options][0] hello world
+ Should Contain ${SCAN_RESULT}[symbols][HTTP_SYNC_RESPONSE_2][options][0] hello post
+
+Sync API TCP get request
+ #Check url /request get HTTP_SYNC_EOF_get hello world
+ Check url /request get HTTP_SYNC_CONTENT_get hello world
+
+# Broken due to dummy_https issues, disable for now
+#Sync API TCP post request
+# Check url /request post HTTP_SYNC_EOF_post hello post
+# Check url /content-length post HTTP_SYNC_CONTENT_post hello post
+
+*** Keywords ***
+Servers Setup
+ Run Dummy Http
+ Run Dummy Ssl
+ Rspamd Setup
+
+Servers Teardown
+ Dummy Http Teardown
+ Rspamd Teardown
+ Teardown Dummy Ssl
+
+Run Dummy Ssl
+ [Arguments]
+ ${result} = Start Process ${RSPAMD_TESTDIR}/util/dummy_ssl.py ${RSPAMD_TESTDIR}/util/server.pem
+ Wait Until Created /tmp/dummy_ssl.pid timeout=2 second
+
+Teardown Dummy Ssl
+ ${ssl_pid} = Get File /tmp/dummy_ssl.pid
+ Shutdown Process With Children ${ssl_pid}
+
+Check url
+ [Arguments] ${url} ${method} ${expect_symbol} @{expect_options}
+ Scan File ${MESSAGE} URL=${url} Method=${method}
+ ... Settings={symbols_enabled = [HTTP_TCP_TEST]}
+ Expect Symbol With Exact Options ${expect_symbol} @{expect_options}
diff --git a/test/functional/cases/231_tcp_down.robot b/test/functional/cases/231_tcp_down.robot
new file mode 100644
index 0000000..5d6c791
--- /dev/null
+++ b/test/functional/cases/231_tcp_down.robot
@@ -0,0 +1,28 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/lua_test.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${RSPAMD_LUA_SCRIPT} ${RSPAMD_TESTDIR}/lua/tcp.lua
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+
+*** Test Cases ***
+Sync API TCP get request when server is down
+ [Documentation] We don't create HTTP server here, that's why
+ ... all requests fail with "connection refused"
+ Check url /request get HTTP_ASYNC_RESPONSE Socket error detected: Connection refused
+ Check url /content-length get HTTP_SYNC_WRITE_ERROR Socket error detected: Connection refused
+
+*** Keywords ***
+Check url
+ [Arguments] ${url} ${method} ${expect_symbol} @{expect_options}
+ Scan File ${MESSAGE} URL=${url} Method=${method}
+ ... Settings={symbols_enabled = [SIMPLE_TCP_ASYNC_TEST, SIMPLE_TCP_TEST]}
+ Expect Symbol With Exact Options ${expect_symbol} @{expect_options}
diff --git a/test/functional/cases/241_redis_is_dead.robot b/test/functional/cases/241_redis_is_dead.robot
new file mode 100644
index 0000000..4c12c14
--- /dev/null
+++ b/test/functional/cases/241_redis_is_dead.robot
@@ -0,0 +1,26 @@
+*** Settings ***
+Documentation Test the case when trying to connect to nowhere
+... (i.e. redis is not running)
+Test Setup Rspamd Setup
+Test Teardown Rspamd Teardown
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/redis.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${REDIS_SCOPE} Test
+${RSPAMD_LUA_SCRIPT} ${RSPAMD_TESTDIR}/lua/redis.lua
+${RSPAMD_SCOPE} Test
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+
+*** Test Cases ***
+Dead Redis client
+ Scan File ${MESSAGE}
+ Expect Symbol With Exact Options REDIS_ERROR_3 Connection refused
+ Expect Symbol With Exact Options REDIS_ASYNC201809_ERROR Connection refused
+ Expect Symbol With Exact Options REDIS_ASYNC_ERROR Connection refused
diff --git a/test/functional/cases/260_regex.robot b/test/functional/cases/260_regex.robot
new file mode 100644
index 0000000..d5055b0
--- /dev/null
+++ b/test/functional/cases/260_regex.robot
@@ -0,0 +1,33 @@
+*** Settings ***
+Test Setup Rspamd Setup
+Test Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/regexp.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/newlines.eml
+${RSPAMD_SCOPE} Test
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+${UTF_MESSAGE} ${RSPAMD_TESTDIR}/messages/utf.eml
+
+
+*** Test Cases ***
+Newlines
+ Scan File ${MESSAGE}
+ Expect Symbol SA_BODY_WORD
+ Expect Symbol SA_BODY_WORD_WITH_SPACE
+ Do Not Expect Symbol SA_BODY_WORD_WITH_NEWLINE
+ Expect Symbol SA_BODY_WORD_WITH_SPACE_BOUNDARIES
+ Expect Symbol SA_BODY_WORD_WITH_SPACE_BOUNDARIES_2
+ Expect Symbol SA_BODY_WORD_WITH_SPACE_BOUNDARIES_3
+ Expect Symbol SA_BODY_WORD_WITH_SPACE_AND_DOT
+ Expect Symbol With Option FOUND_URL https://google.com/maps/
+ Expect Symbol With Option FOUND_URL https://www.google.com/search?q\=hello world&oq\=hello world&aqs\=chrome..69i57j0l5.3045j0j7&sourceid\=chrome&ie\=UTF-8
+ Expect Symbol With Option FOUND_URL https://github.com/google/sanitizers/wiki/AddressSanitizer
+
+Dynamic Config
+ Scan File ${MESSAGE}
+ Expect Symbol With Score SA_BODY_WORD 10
+ Expect Required Score 20
diff --git a/test/functional/cases/320_arc_signing/001_simple.robot b/test/functional/cases/320_arc_signing/001_simple.robot
new file mode 100644
index 0000000..10078c6
--- /dev/null
+++ b/test/functional/cases/320_arc_signing/001_simple.robot
@@ -0,0 +1,31 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/arc_signing/simple.conf
+${MESSAGE_FAIL} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none1.eml
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+TEST SIGNED
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org
+ Should Contain ${result.stdout} ARC_SIGNED
+
+TEST SIGNED HTTP HEADERS
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org --header=PerformArcSign:yes --header=DkimDomain:cacophony.za.org --header=DkimSelector:dkim --header=DkimPrivateKey:MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANe3EETkiI1Exyrb+VzbMSt90K8MXJA0GcyNs6MFCs9JPaTh90Zu2l7ki7m5LTUx6350AR/3hcvwjSHCZjD6fvQ8/zfjN8kaLZ6DAaqtqSlpawIM+8glkuTEkIkpBED/OtDrba4Rd29iLFVuwQZXDtTjAAZKZPmtTZ5TXLrcCU6VAgMBAAECgYEA1BFvmBsIN8Gu/+6kNupya2xUNVM0yLu/xT5lpNV3LBO325oejAq8+d87kkl/LTW3a2jGFlQ0ICuLw+2mo24QUWRyv8if3oeBMlnLqHE+6wNjFVqo5sOjKzjO363xSXwXNUrBT7jDhnZcDN8w3/FecYKjifGTVtUs1SLsYwhlc8ECQQDuCRymLZQ/imPn5eFVIydwUzg8ptZlvoA7bfIxUL9BQRX33s59kLCilA0tTed8Dd+GnxsT93XOj1ApIfBwmTSlAkEA5/63PDsN7fH+WInqVD8nU07M9S8LcGDlPbVVBr2S2I78/iwrSDAYtbkU2vEbhFK/JuKNML2j8OkzV3v1QulfMQJBALDzhx+l/HHr3+8RPhx7QKNIyiKUaAdEwbDsP8IXY8YPq1QThu9jM1v4sX7/TdkzuvoppwiFykbe1NlvCH279p0CQCmTg4Ee0DtBcCSr6rvYaZLLf329RZ6JLuwlMCy6ErQOxBZFEiiovfTrS2qFZToMnkc4uLbwdY36LQJTq7unGTECQCCok8LzBeZtAw+TJofpOM3F2Rlm2qXiBVBeubhRedsiljG0hpvvLJBMppnQ6r27p5Jk39SmaTRkxEKrxPWWLNM=
+ Should Contain ${result.stdout} ARC_SIGNED
+
+TEST NOT SIGNED - USERNAME WRONG DOMAIN
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@example.tk
+ Should Not Contain ${result.stdout} ARC_SIGNED
+
+TEST NOT SIGNED - USERNAME WRONG PUBKEY
+ ${result} = Scan Message With Rspamc ${MESSAGE_FAIL} -u bob@invalid.za.org
+ Should Not Contain ${result.stdout} ARC_SIGNED
diff --git a/test/functional/cases/320_arc_signing/002_redis.robot b/test/functional/cases/320_arc_signing/002_redis.robot
new file mode 100644
index 0000000..ca979ad
--- /dev/null
+++ b/test/functional/cases/320_arc_signing/002_redis.robot
@@ -0,0 +1,29 @@
+*** Settings ***
+Suite Setup ARC Signing Setup
+Suite Teardown Rspamd Redis Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/arc_signing/redis.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/dmarc/fail_none.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+TEST SIGNED
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@cacophony.za.org
+ Should Contain ${result.stdout} ARC_SIGNED
+
+TEST NOT SIGNED - USERNAME WRONG DOMAIN
+ ${result} = Scan Message With Rspamc ${MESSAGE} -u bob@example.tk
+ Should Not Contain ${result.stdout} ARC_SIGNED
+
+*** Keywords ***
+ARC Signing Setup
+ Rspamd Redis Setup
+ Redis HSET TEST_DKIM_SELECTORS cacophony.za.org arc
+ ${key} = Get File ${RSPAMD_TESTDIR}/configs/dkim.key
+ Redis HSET TEST_DKIM_KEYS arc.cacophony.za.org ${key}
diff --git a/test/functional/cases/330_neural/001_autotrain.robot b/test/functional/cases/330_neural/001_autotrain.robot
new file mode 100644
index 0000000..793fc75
--- /dev/null
+++ b/test/functional/cases/330_neural/001_autotrain.robot
@@ -0,0 +1,63 @@
+*** Settings ***
+Suite Setup Rspamd Redis Setup
+Suite Teardown Rspamd Redis Teardown
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/neural.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+Train
+ Sleep 2s Wait for redis mess
+ FOR ${INDEX} IN RANGE 4 14
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["SPAM_SYMBOL1", "SPAM_SYMBOL2", "SPAM_SYMBOL3", "SPAM_SYMBOL${INDEX}"]}
+ Expect Symbol SPAM_SYMBOL${INDEX}
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["HAM_SYMBOL1", "HAM_SYMBOL2", "HAM_SYMBOL3", "HAM_SYMBOL${INDEX}"]}
+ Expect Symbol HAM_SYMBOL${INDEX}
+ END
+
+Check Neural HAM
+ Sleep 5s Wait for neural to be loaded
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["HAM_SYMBOL1","HAM_SYMBOL2","HAM_SYMBOL3","HAM_SYMBOL8"];groups_enabled=["neural"];symbols_disabled = ["NEURAL_LEARN"]}
+ Expect Symbol NEURAL_HAM_SHORT
+ Do Not Expect Symbol NEURAL_SPAM_SHORT
+ #Expect Symbol NEURAL_HAM_SHORT_PCA
+ #Do Not Expect Symbol NEURAL_SPAM_SHORT_PCA
+
+Check Neural SPAM
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["SPAM_SYMBOL1","SPAM_SYMBOL2","SPAM_SYMBOL3","SPAM_SYMBOL8"];groups_enabled=["neural"];symbols_disabled = ["NEURAL_LEARN"]}
+ Expect Symbol NEURAL_SPAM_SHORT
+ Do Not Expect Symbol NEURAL_HAM_SHORT
+ #Expect Symbol NEURAL_SPAM_SHORT_PCA
+ #Do Not Expect Symbol NEURAL_HAM_SHORT_PCA
+
+
+Train INVERSE
+ FOR ${INDEX} IN RANGE 4 14
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["SPAM_SYMBOL1", "SPAM_SYMBOL2", "SPAM_SYMBOL3","SPAM_SYMBOL${INDEX}"]; SPAM_SYMBOL1 = -5; SPAM_SYMBOL2 = -5; SPAM_SYMBOL3 = -5; SPAM_SYMBOL${INDEX} = -5;}
+ Expect Symbol SPAM_SYMBOL${INDEX}
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["HAM_SYMBOL1", "HAM_SYMBOL2", "HAM_SYMBOL3","HAM_SYMBOL${INDEX}"]; HAM_SYMBOL1 = 5; HAM_SYMBOL2 = 5; HAM_SYMBOL3 = 5; HAM_SYMBOL${INDEX} = 5;}
+ Expect Symbol HAM_SYMBOL${INDEX}
+ END
+
+Check Neural HAM INVERSE
+ Sleep 5s Wait for neural to be loaded
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["HAM_SYMBOL1","HAM_SYMBOL2","HAM_SYMBOL3","HAM_SYMBOL5",];groups_enabled=["neural"]}
+ Expect Symbol NEURAL_SPAM_SHORT
+ #Expect Symbol NEURAL_SPAM_SHORT_PCA
+ Do Not Expect Symbol NEURAL_HAM_SHORT
+ #Do Not Expect Symbol NEURAL_HAM_SHORT_PCA
+
+Check Neural SPAM INVERSE
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["SPAM_SYMBOL1","SPAM_SYMBOL2","SPAM_SYMBOL3","SPAM_SYMBOL5"];groups_enabled=["neural"]}
+ Expect Symbol NEURAL_HAM_SHORT
+ #Expect Symbol NEURAL_HAM_SHORT_PCA
+ Do Not Expect Symbol NEURAL_SPAM_SHORT
+ #Do Not Expect Symbol NEURAL_SPAM_SHORT_PCA \ No newline at end of file
diff --git a/test/functional/cases/330_neural/002_manualtrain.robot b/test/functional/cases/330_neural/002_manualtrain.robot
new file mode 100644
index 0000000..bbc6c2e
--- /dev/null
+++ b/test/functional/cases/330_neural/002_manualtrain.robot
@@ -0,0 +1,72 @@
+*** Settings ***
+Suite Setup Rspamd Redis Setup
+Suite Teardown Rspamd Redis Teardown
+Library Process
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/neural_noauto.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+Collect training vectors & train manually
+ @{HAM_VEC}= Create List
+ @{SPAM_VEC}= Create List
+
+ FOR ${INDEX} IN RANGE 4 14
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["SPAM_SYMBOL1", "SPAM_SYMBOL2", "SPAM_SYMBOL3", "SPAM_SYMBOL${INDEX}", "SAVE_NN_ROW"]}
+ Expect Symbol SPAM_SYMBOL${INDEX}
+ # Save neural inputs for later
+ ${SPAM_ROW} = Get File ${SCAN_RESULT}[symbols][SAVE_NN_ROW][options][0]
+ Remove File ${SCAN_RESULT}[symbols][SAVE_NN_ROW][options][0]
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["HAM_SYMBOL1", "HAM_SYMBOL2", "HAM_SYMBOL3", "HAM_SYMBOL${INDEX}", "SAVE_NN_ROW"]}
+ Expect Symbol HAM_SYMBOL${INDEX}
+ # Save neural inputs for later
+ ${HAM_ROW} = Get File ${SCAN_RESULT}[symbols][SAVE_NN_ROW][options][0]
+ Remove File ${SCAN_RESULT}[symbols][SAVE_NN_ROW][options][0]
+ ${HAM_ROW} = Run ${RSPAMADM} lua -a ${HAM_ROW} ${RSPAMD_TESTDIR}/util/nn_unpack.lua
+ ${HAM_ROW} = Evaluate json.loads("${HAM_ROW}")
+ ${SPAM_ROW} = Run ${RSPAMADM} lua -a ${SPAM_ROW} ${RSPAMD_TESTDIR}/util/nn_unpack.lua
+ ${SPAM_ROW} = Evaluate json.loads("${SPAM_ROW}")
+ Append To List ${HAM_VEC} ${HAM_ROW}
+ Append To List ${SPAM_VEC} ${SPAM_ROW}
+ END
+
+ ${json1} = Evaluate json.dumps({"spam_vec": ${SPAM_VEC}, "ham_vec": ${HAM_VEC}, "rule": "SHORT"})
+ HTTP POST ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_CONTROLLER} /plugins/neural/learn ${json1}
+
+ # Save variables for use in inverse training
+ Set Suite Variable ${HAM_VEC}
+ Set Suite Variable ${SPAM_VEC}
+
+ Sleep 5s Wait for neural to be loaded
+
+Check Neural HAM
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["HAM_SYMBOL1","HAM_SYMBOL2","HAM_SYMBOL3","HAM_SYMBOL8"];groups_enabled=["neural"];symbols_disabled = ["NEURAL_LEARN"]}
+ Do Not Expect Symbol NEURAL_SPAM_SHORT
+ Expect Symbol NEURAL_HAM_SHORT
+
+Check Neural SPAM
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["SPAM_SYMBOL1","SPAM_SYMBOL2","SPAM_SYMBOL3","SPAM_SYMBOL8"];groups_enabled=["neural"];symbols_disabled = ["NEURAL_LEARN"]}
+ Do Not Expect Symbol NEURAL_HAM_SHORT
+ Expect Symbol NEURAL_SPAM_SHORT
+
+Train inverse
+ ${json2} = Evaluate json.dumps({"spam_vec": ${HAM_VEC}, "ham_vec": ${SPAM_VEC}, "rule": "SHORT"})
+ HTTP POST ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_CONTROLLER} /plugins/neural/learn ${json2}
+ Sleep 5s Wait for neural to be loaded
+
+Check Neural HAM - inverse
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["HAM_SYMBOL1","HAM_SYMBOL2","HAM_SYMBOL3","HAM_SYMBOL8"];groups_enabled=["neural"];symbols_disabled = ["NEURAL_LEARN"]}
+ Do Not Expect Symbol NEURAL_HAM_SHORT
+ Expect Symbol NEURAL_SPAM_SHORT
+
+Check Neural SPAM - inverse
+ Scan File ${MESSAGE} Settings={symbols_enabled = ["SPAM_SYMBOL1","SPAM_SYMBOL2","SPAM_SYMBOL3","SPAM_SYMBOL8"];groups_enabled=["neural"];symbols_disabled = ["NEURAL_LEARN"]}
+ Do Not Expect Symbol NEURAL_SPAM_SHORT
+ Expect Symbol NEURAL_HAM_SHORT \ No newline at end of file
diff --git a/test/functional/cases/360_force_actions.robot b/test/functional/cases/360_force_actions.robot
new file mode 100644
index 0000000..18edf9a
--- /dev/null
+++ b/test/functional/cases/360_force_actions.robot
@@ -0,0 +1,44 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/force_actions.conf
+${MESSAGE} ${RSPAMD_TESTDIR}/messages/url7.eml
+${RSPAMD_SCOPE} Suite
+${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+FORCE ACTIONS from reject to add header
+ Scan File ${MESSAGE} Settings-Id=id_reject
+ Expect Action add header
+ Expect Symbol FORCE_ACTION_FORCE_REJECT_TO_ADD_HEADER
+
+FORCE ACTIONS from reject to no action
+ Scan File ${MESSAGE} Settings-Id=id_reject_no_action
+ Expect Action no action
+ Expect Symbol FORCE_ACTION_FORCE_REJECT_TO_NO_ACTION
+
+FORCE ACTIONS from no action to reject
+ Scan File ${MESSAGE} Settings-Id=id_no_action
+ Expect Action reject
+ Expect Symbol FORCE_ACTION_FORCE_NO_ACTION_TO_REJECT
+
+FORCE ACTIONS from no action to add header
+ Scan File ${MESSAGE} Settings-Id=id_no_action_to_add_header
+ Expect Action add header
+ Expect Symbol FORCE_ACTION_FORCE_NO_ACTION_TO_ADD_HEADER
+
+FORCE ACTIONS from add header to no action
+ Scan File ${MESSAGE} Settings-Id=id_add_header
+ Expect Action no action
+ Expect Symbol FORCE_ACTION_FORCE_ADD_HEADER_TO_NO_ACTION
+
+FORCE ACTIONS from add header to reject
+ Scan File ${MESSAGE} Settings-Id=id_add_header_to_reject
+ Expect Action reject
+ Expect Symbol FORCE_ACTION_FORCE_ADD_HEADER_TO_REJECT
+
diff --git a/test/functional/cases/380_external_relay.robot b/test/functional/cases/380_external_relay.robot
new file mode 100644
index 0000000..459aaac
--- /dev/null
+++ b/test/functional/cases/380_external_relay.robot
@@ -0,0 +1,48 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Rspamd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/merged.conf
+${RSPAMD_EXTERNAL_RELAY_ENABLED} true
+${RSPAMD_SCOPE} Suite
+
+*** Test Cases ***
+EXTERNAL RELAY AUTHENTICATED
+ Scan File ${RSPAMD_TESTDIR}/messages/received5.eml
+ ... Settings={symbols_enabled [EXTERNAL_RELAY_TEST, EXTERNAL_RELAY_AUTHENTICATED]}
+ ... IP=8.8.8.8 User=user@example.net
+ Expect Symbol With Exact Options EXTERNAL_RELAY_TEST
+ ... IP=192.0.2.1 HOSTNAME=mail.example.org HELO=mail.example.org
+
+EXTERNAL RELAY COUNT
+ Scan File ${RSPAMD_TESTDIR}/messages/received4.eml
+ ... Settings={symbols_enabled [EXTERNAL_RELAY_TEST, EXTERNAL_RELAY_COUNT]}
+ ... IP=8.8.8.8
+ Expect Symbol With Exact Options EXTERNAL_RELAY_TEST
+ ... IP=151.18.193.131 HOSTNAME=ca-18-193-131.service.infuturo.it
+ ... HELO=ca-18-193-131.service.infuturo.it
+
+EXTERNAL RELAY HOSTNAME MAP
+ Scan File ${RSPAMD_TESTDIR}/messages/received6.eml
+ ... Settings={symbols_enabled [EXTERNAL_RELAY_TEST, EXTERNAL_RELAY_HOSTNAME_MAP]}
+ ... Hostname=lame.example.net IP=192.0.2.10
+ Expect Symbol With Exact Options EXTERNAL_RELAY_TEST
+ ... IP=192.0.2.1 HOSTNAME=mail.example.org HELO=mail.example.org
+
+EXTERNAL RELAY IP MAP
+ Scan File ${RSPAMD_TESTDIR}/messages/received7.eml
+ ... Settings={symbols_enabled [EXTERNAL_RELAY_TEST, EXTERNAL_RELAY_IP_MAP]}
+ ... IP=198.51.100.1
+ Expect Symbol With Exact Options EXTERNAL_RELAY_TEST
+ ... IP=4.31.198.44 HOSTNAME=foobar.example.org HELO=foobar.example.org
+
+EXTERNAL RELAY LOCAL
+ Scan File ${RSPAMD_TESTDIR}/messages/ham.eml
+ ... Settings={symbols_enabled [EXTERNAL_RELAY_TEST, EXTERNAL_RELAY_LOCAL]}
+ ... IP=127.0.0.1
+ Expect Symbol With Exact Options EXTERNAL_RELAY_TEST
+ ... IP=4.31.198.44 HOSTNAME=mail.ietf.org HELO=mail.ietf.org
diff --git a/test/functional/cases/400_known_senders.robot b/test/functional/cases/400_known_senders.robot
new file mode 100644
index 0000000..f258113
--- /dev/null
+++ b/test/functional/cases/400_known_senders.robot
@@ -0,0 +1,36 @@
+*** Settings ***
+Suite Setup Rspamd Redis Setup
+Suite Teardown Rspamd Redis Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/known_senders.conf
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+
+*** Test Cases ***
+UNKNOWN SENDER
+ Scan File ${RSPAMD_TESTDIR}/messages/spam_message.eml
+ ... Settings={symbols_enabled [KNOWN_SENDER]}
+ Do Not Expect Symbol KNOWN_SENDER
+ Expect Symbol UNKNOWN_SENDER
+
+UNKNOWN SENDER BECOMES KNOWN
+ Scan File ${RSPAMD_TESTDIR}/messages/spam_message.eml
+ ... Settings={symbols_enabled [KNOWN_SENDER]}
+ Expect Symbol KNOWN_SENDER
+ Do Not Expect Symbol UNKNOWN_SENDER
+
+UNKNOWN SENDER WRONG DOMAIN
+ Scan File ${RSPAMD_TESTDIR}/messages/empty_part.eml
+ ... Settings={symbols_enabled [KNOWN_SENDER]}
+ Do Not Expect Symbol KNOWN_SENDER
+ Do Not Expect Symbol UNKNOWN_SENDER
+
+UNKNOWN SENDER WRONG DOMAIN RESCAN
+ Scan File ${RSPAMD_TESTDIR}/messages/empty_part.eml
+ ... Settings={symbols_enabled [KNOWN_SENDER]}
+ Do Not Expect Symbol KNOWN_SENDER
+ Do Not Expect Symbol UNKNOWN_SENDER
diff --git a/test/functional/cases/410_logging/000_console/000_systemd_logger.robot b/test/functional/cases/410_logging/000_console/000_systemd_logger.robot
new file mode 100644
index 0000000..8817846
--- /dev/null
+++ b/test/functional/cases/410_logging/000_console/000_systemd_logger.robot
@@ -0,0 +1,24 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Systemd Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/loggingtest.conf
+${RSPAMD_LOGGINGTYPE} console
+${RSPAMD_JSON} false
+${RSPAMD_SYSTEMD} true
+${RSPAMD_SCOPE} Suite
+
+*** Test Cases ***
+EMPTY TEST
+ Pass Execution No worries
+
+*** Keywords ***
+Systemd Teardown
+ Touch ${RSPAMD_TMPDIR}/rspamd.log
+ Rspamd Teardown
+ ${log} = Get File ${EXECDIR}/robot-save/rspamd.stderr.last
+ Should Match Regexp ${log} \\n\\(main\\) lua; lua_cfg_transform\\.lua:\\d+: overriding actions from the legacy metric settings\\n
diff --git a/test/functional/cases/410_logging/000_console/001_timestamps.robot b/test/functional/cases/410_logging/000_console/001_timestamps.robot
new file mode 100644
index 0000000..bd8e2c3
--- /dev/null
+++ b/test/functional/cases/410_logging/000_console/001_timestamps.robot
@@ -0,0 +1,24 @@
+*** Settings ***
+Suite Setup Rspamd Setup
+Suite Teardown Console Timestamps Teardown
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/loggingtest.conf
+${RSPAMD_LOGGINGTYPE} console
+${RSPAMD_JSON} false
+${RSPAMD_SYSTEMD} false
+${RSPAMD_SCOPE} Suite
+
+*** Test Cases ***
+EMPTY TEST
+ Pass Execution No worries
+
+*** Keywords ***
+Console Timestamps Teardown
+ Touch ${RSPAMD_TMPDIR}/rspamd.log
+ Rspamd Teardown
+ ${log} = Get File ${EXECDIR}/robot-save/rspamd.stderr.last
+ Should Match Regexp ${log} \\n\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2} #\\d+\\(main\\) lua; lua_cfg_transform\\.lua:\\d+: overriding actions from the legacy metric settings\\n
diff --git a/test/functional/cases/410_logging/001_file/000_json.robot b/test/functional/cases/410_logging/001_file/000_json.robot
new file mode 100644
index 0000000..a2f04e8
--- /dev/null
+++ b/test/functional/cases/410_logging/001_file/000_json.robot
@@ -0,0 +1,17 @@
+*** Settings ***
+Test Setup Rspamd Setup
+Library ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${RSPAMD_TESTDIR}/configs/loggingtest.conf
+${RSPAMD_LOGGINGTYPE} file
+${RSPAMD_JSON} true
+${RSPAMD_SYSTEMD} true
+${RSPAMD_SCOPE} Test
+
+*** Test Cases ***
+JSON LOGS
+ Rspamd Teardown
+ Check JSON Log ${EXECDIR}/robot-save/rspamd.log.last
diff --git a/test/functional/cases/__init__.robot b/test/functional/cases/__init__.robot
new file mode 100644
index 0000000..51192d7
--- /dev/null
+++ b/test/functional/cases/__init__.robot
@@ -0,0 +1,19 @@
+*** Settings ***
+Suite Setup Export Global Variables
+Library ../lib/rspamd.py
+Variables ../lib/vars.py
+
+*** Keywords ***
+Export Global Variables
+ ${RSPAMD_TESTDIR} = Get Test Directory
+ ${TOPDIR} = Get Top Dir
+ ${RSPAMADM} = Get Rspamadm
+ ${RSPAMC} = Get Rspamc
+ ${RSPAMD} = Get Rspamd
+ ${RSPAMD_INSTALLROOT} = Get Install Root
+ Set Global Variable ${RSPAMD_INSTALLROOT}
+ Set Global Variable ${RSPAMADM}
+ Set Global Variable ${RSPAMC}
+ Set Global Variable ${RSPAMD}
+ Set Global Variable ${RSPAMD_TESTDIR}
+ Set Global Variable ${TOPDIR}
diff --git a/test/functional/configs/arc_signing/redis.conf b/test/functional/configs/arc_signing/redis.conf
new file mode 100644
index 0000000..0d049c3
--- /dev/null
+++ b/test/functional/configs/arc_signing/redis.conf
@@ -0,0 +1,10 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+arc {
+ use_redis = true;
+ key_prefix = "TEST_DKIM_KEYS";
+ selector_prefix = "TEST_DKIM_SELECTORS";
+}
+redis {
+ servers = "{= env.REDIS_ADDR =}:{= env.REDIS_PORT =}";
+}
diff --git a/test/functional/configs/arc_signing/simple.conf b/test/functional/configs/arc_signing/simple.conf
new file mode 100644
index 0000000..f559e61
--- /dev/null
+++ b/test/functional/configs/arc_signing/simple.conf
@@ -0,0 +1,10 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+arc {
+ path = "{= env.TESTDIR =}/configs/dkim.key";
+ check_pubkey = true;
+ allow_pubkey_mismatch = false;
+ selector = "dkim";
+ use_http_headers = true;
+ allow_headers_fallback = true;
+}
diff --git a/test/functional/configs/clickhouse-config.xml b/test/functional/configs/clickhouse-config.xml
new file mode 100644
index 0000000..3b5c914
--- /dev/null
+++ b/test/functional/configs/clickhouse-config.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0"?>
+<yandex>
+ <logger>
+ <!-- Possible levels: https://github.com/pocoproject/poco/blob/develop/Foundation/include/Poco/Logger.h#L105 -->
+ <level>debug</level>
+ <log>${RSPAMD_TMPDIR}/clickhouse/clickhouse-server.log</log>
+ <errorlog>${RSPAMD_TMPDIR}/clickhouse/clickhouse-server.err.log</errorlog>
+ <size>1000M</size>
+ <count>10</count>
+ <!-- <console>1</console> --> <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) -->
+ </logger>
+
+ <listen_host>127.0.0.1</listen_host>
+ <http_port>${CLICKHOUSE_PORT}</http_port>
+
+ <!-- Path to configuration file with users, access rights, profiles of settings, quotas. -->
+ <users_config>${RSPAMD_TMPDIR}/users.xml</users_config>
+
+ <!-- <listen_reuse_port>0</listen_reuse_port> -->
+
+ <!-- <listen_backlog>64</listen_backlog> -->
+
+ <max_connections>4096</max_connections>
+ <keep_alive_timeout>3</keep_alive_timeout>
+
+ <!-- Maximum number of concurrent queries. -->
+ <max_concurrent_queries>100</max_concurrent_queries>
+
+ <!-- Set limit on number of open files (default: maximum). This setting makes sense on Mac OS X because getrlimit() fails to retrieve
+ correct maximum value. -->
+ <max_open_files>256</max_open_files>
+
+ <!-- Approximate size of mark cache, used in tables of MergeTree family.
+ In bytes. Cache is single for server. Memory is allocated only on demand.
+ You should not lower this value.
+ -->
+ <mark_cache_size>5368709120</mark_cache_size>
+
+ <!-- Path to data directory, with trailing slash. -->
+ <path>${RSPAMD_TMPDIR}/clickhouse/</path>
+
+ <!-- Default profile of settings. -->
+ <default_profile>default</default_profile>
+
+ <!-- System profile of settings. This settings are used by internal processes (Buffer storage, Distibuted DDL worker and so on). -->
+ <!-- <system_profile>default</system_profile> -->
+
+ <!-- Default database. -->
+ <default_database>default</default_database>
+
+ <!-- Server time zone could be set here.
+
+ Time zone is used when converting between String and DateTime types,
+ when printing DateTime in text formats and parsing DateTime from text,
+ it is used in date and time related functions, if specific time zone was not passed as an argument.
+
+ Time zone is specified as identifier from IANA time zone database, like UTC or Africa/Abidjan.
+ If not specified, system time zone at server startup is used.
+
+ Please note, that server could display time zone alias instead of specified name.
+ Example: W-SU is an alias for Europe/Moscow and Zulu is an alias for UTC.
+ -->
+ <!-- <timezone>Europe/Moscow</timezone> -->
+
+ <!-- You can specify umask here (see "man umask"). Server will apply it on startup.
+ Number is always parsed as octal. Default umask is 027 (other users cannot read logs, data files, etc; group can only read).
+ -->
+ <!-- <umask>022</umask> -->
+
+</yandex>
diff --git a/test/functional/configs/clickhouse-users.xml b/test/functional/configs/clickhouse-users.xml
new file mode 100644
index 0000000..6f746ba
--- /dev/null
+++ b/test/functional/configs/clickhouse-users.xml
@@ -0,0 +1,109 @@
+<?xml version="1.0"?>
+<yandex>
+ <!-- Profiles of settings. -->
+ <profiles>
+ <!-- Default settings. -->
+ <default>
+ <!-- Maximum memory usage for processing single query, in bytes. -->
+ <max_memory_usage>10000000000</max_memory_usage>
+
+ <!-- Use cache of uncompressed blocks of data. Meaningful only for processing many of very short queries. -->
+ <use_uncompressed_cache>0</use_uncompressed_cache>
+
+ <!-- How to choose between replicas during distributed query processing.
+ random - choose random replica from set of replicas with minimum number of errors
+ nearest_hostname - from set of replicas with minimum number of errors, choose replica
+ with minumum number of different symbols between replica's hostname and local hostname
+ (Hamming distance).
+ in_order - first live replica is choosen in specified order.
+ -->
+ <load_balancing>random</load_balancing>
+ </default>
+
+ <!-- Profile that allows only read queries. -->
+ <readonly>
+ <readonly>1</readonly>
+ </readonly>
+ </profiles>
+
+ <!-- Users and ACL. -->
+ <users>
+ <!-- If user name was not specified, 'default' user is used. -->
+ <default>
+ <!-- Password could be specified in plaintext or in SHA256 (in hex format).
+
+ If you want to specify password in plaintext (not recommended), place it in 'password' element.
+ Example: <password>qwerty</password>.
+ Password could be empty.
+
+ If you want to specify SHA256, place it in 'password_sha256_hex' element.
+ Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
+
+ How to generate decent password:
+ Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
+ In first line will be password and in second - corresponding SHA256.
+ -->
+ <password></password>
+
+ <!-- List of networks with open access.
+
+ To open access from everywhere, specify:
+ <ip>::/0</ip>
+
+ To open access only from localhost, specify:
+ <ip>::1</ip>
+ <ip>127.0.0.1</ip>
+
+ Each element of list has one of the following forms:
+ <ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
+ 2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
+ <host> Hostname. Example: server01.yandex.ru.
+ To check access, DNS query is performed, and all received addresses compared to peer address.
+ <host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
+ To check access, DNS PTR query is performed for peer address and then regexp is applied.
+ Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
+ Strongly recommended that regexp is ends with $
+ All results of DNS requests are cached till server restart.
+ -->
+ <networks incl="networks" replace="replace">
+ <ip>::/0</ip>
+ </networks>
+
+ <!-- Settings profile for user. -->
+ <profile>default</profile>
+
+ <!-- Quota for user. -->
+ <quota>default</quota>
+ </default>
+
+ <!-- Example of user with readonly access. -->
+ <readonly>
+ <password></password>
+ <networks incl="networks" replace="replace">
+ <ip>::1</ip>
+ <ip>127.0.0.1</ip>
+ </networks>
+ <profile>readonly</profile>
+ <quota>default</quota>
+ </readonly>
+ </users>
+
+ <!-- Quotas. -->
+ <quotas>
+ <!-- Name of quota. -->
+ <default>
+ <!-- Limits for time interval. You could specify many intervals with different limits. -->
+ <interval>
+ <!-- Length of interval. -->
+ <duration>3600</duration>
+
+ <!-- No limits. Just calculate resource usage for time interval. -->
+ <queries>0</queries>
+ <errors>0</errors>
+ <result_rows>0</result_rows>
+ <read_rows>0</read_rows>
+ <execution_time>0</execution_time>
+ </interval>
+ </default>
+ </quotas>
+</yandex>
diff --git a/test/functional/configs/clickhouse.conf b/test/functional/configs/clickhouse.conf
new file mode 100644
index 0000000..f92f2e7
--- /dev/null
+++ b/test/functional/configs/clickhouse.conf
@@ -0,0 +1,63 @@
+options = {
+ filters = ["spf", "dkim", "regexp"]
+ pidfile = "${RSPAMD_TMPDIR}/rspamd.pid"
+ lua_path = "${INSTALLROOT}/share/rspamd/lib/?.lua"
+ dns {
+ nameserver = ["8.8.8.8", "8.8.4.4"];
+ retransmits = 10;
+ timeout = 2s;
+ fake_records = [{ # ed25519
+ name = "test._domainkey.example.com";
+ type = txt;
+ replies = ["k=ed25519; p=yi50DjK5O9pqbFpNHklsv9lqaS0ArSYu02qp1S0DW1Y="];
+ }];
+ }
+}
+clickhouse {
+ # Push update when 1000 records are collected (1000 if unset)
+ limit = 1;
+ # IP:port of Clickhouse server
+ server = "localhost:18123";
+ allow_local = true;
+ retention {
+ # disabled by default
+ enable = true;
+ # drop | detach, please refer to ClickHouse docs for details
+ # http://clickhouse-docs.readthedocs.io/en/latest/query_language/queries.html#manipulations-with-partitions-and-parts
+ method = "drop";
+ # how many month the data should be kept in ClickHouse
+ period_months = 3;
+ # how often run the cleanup process
+ run_every = "7d";
+ }
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "${RSPAMD_TMPDIR}/rspamd.log"
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ }
+ unknown_weight = 1
+}
+worker {
+ type = normal
+ bind_socket = ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_NORMAL}
+ count = 1
+ task_timeout = 60s;
+}
+worker {
+ type = controller
+ bind_socket = ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER}
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "${RSPAMD_TMPDIR}/stats.ucl"
+}
+lua = "${RSPAMD_TESTDIR}/lua/test_coverage.lua";
+modules {
+ path = "${RSPAMD_TESTDIR}/../../src/plugins/lua/"
+}
+lua = "${INSTALLROOT}/share/rspamd/rules/rspamd.lua"
diff --git a/test/functional/configs/composites.conf b/test/functional/configs/composites.conf
new file mode 100644
index 0000000..28b645e
--- /dev/null
+++ b/test/functional/configs/composites.conf
@@ -0,0 +1,87 @@
+options = {
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+}
+
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ task_timeout = 60s;
+}
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl"
+}
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
+lua = {= env.LUA_SCRIPT =};
+
+composites {
+ EXPRESSIONS {
+ expression = "(EXPRESSIONS_A | ~EXPRESSIONS_B) & !EXPRESSIONS_C";
+ score = 5.0;
+ }
+
+ POLICY_REMOVE_WEIGHT {
+ expression = "POLICY_REMOVE_WEIGHT_A and ~POLICY_REMOVE_WEIGHT_B";
+ score = 5.0;
+ }
+ POLICY_FORCE_REMOVE {
+ expression = "POLICY_FORCE_REMOVE_A & ^POLICY_FORCE_REMOVE_B";
+ score = 5.0;
+ }
+ POLICY_FORCE_REMOVE_LEAVE {
+ expression = "-POLICY_FORCE_REMOVE_A and -POLICY_FORCE_REMOVE_B";
+ score = 5.0;
+ }
+ POLICY_LEAVE {
+ expression = "POLICY_LEAVE_A & -POLICY_LEAVE_B";
+ score = 5.0;
+ }
+
+ DEFAULT_POLICY_REMOVE_WEIGHT {
+ expression = "DEFAULT_POLICY_REMOVE_WEIGHT_A and DEFAULT_POLICY_REMOVE_WEIGHT_B";
+ score = 5.0;
+ policy = "remove_weight";
+ }
+ DEFAULT_POLICY_REMOVE_SYMBOL {
+ expression = "DEFAULT_POLICY_REMOVE_SYMBOL_A & DEFAULT_POLICY_REMOVE_SYMBOL_B";
+ score = 5.0;
+ policy = "remove_symbol";
+ }
+ DEFAULT_POLICY_LEAVE {
+ expression = "DEFAULT_POLICY_LEAVE_A & DEFAULT_POLICY_LEAVE_B";
+ score = 5.0;
+ policy = "leave";
+ }
+
+ SYMBOL_GROUPS {
+ expression = "!g+:positive & g-:negative & -g:any";
+ score = 5.0;
+ }
+
+ SYMOPTS1 {
+ expression = "OPTS[sym1]";
+ score = 5.0;
+ }
+
+ SYMOPTS2 {
+ expression = 'OPTS[/foo[0-9]/,sym2]';
+ score = 6.0;
+ }
+ SYMOPTS3 {
+ expression = 'OPTS[sym2,/FoO\//i]';
+ score = 6.0;
+ }
+ SYMOPTS4 {
+ expression = 'POSITIVE_A & OPTS[/>app.link$/i] & EXPRESSIONS';
+ score = 6.0;
+ }
+}
diff --git a/test/functional/configs/dkim-eddsa.key b/test/functional/configs/dkim-eddsa.key
new file mode 100644
index 0000000..45282e1
--- /dev/null
+++ b/test/functional/configs/dkim-eddsa.key
@@ -0,0 +1 @@
+m5kGxtckRfsNe5EuYTe7bvkDjSh7LXaX3aXyIMPGLR0=
diff --git a/test/functional/configs/dkim.conf b/test/functional/configs/dkim.conf
new file mode 100644
index 0000000..50712d1
--- /dev/null
+++ b/test/functional/configs/dkim.conf
@@ -0,0 +1,66 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+options = {
+ filters = ["dkim"]
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+ dns {
+ retransmits = 10;
+ timeout = 2s;
+ }
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ }
+ unknown_weight = 1
+}
+
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ keypair {
+ pubkey = "{= env.KEY_PUB1 =}";
+ privkey = "{= env.KEY_PVT1 =}";
+ }
+ task_timeout = 60s;
+}
+
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl"
+}
+
+dkim {
+
+sign_condition =<<EOD
+return function(task)
+ local dodkim = task:get_request_header('dodkim')
+ if not dodkim then return end
+ return {
+ key = "{= env.TESTDIR =}/configs/dkim.key",
+ domain = "cacophony.za.org",
+ selector = "dkim"
+ }
+end
+EOD;
+
+ dkim_cache_size = 2k;
+ dkim_cache_expire = 1d;
+ time_jitter = 6h;
+ trusted_only = false;
+ skip_multi = false;
+}
+modules {
+ path = "{= env.TESTDIR =}/../../src/plugins/lua/"
+}
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
diff --git a/test/functional/configs/dkim.key b/test/functional/configs/dkim.key
new file mode 100644
index 0000000..3bc1fb8
--- /dev/null
+++ b/test/functional/configs/dkim.key
@@ -0,0 +1,16 @@
+-----BEGIN PRIVATE KEY-----
+MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANe3EETkiI1Exyrb
++VzbMSt90K8MXJA0GcyNs6MFCs9JPaTh90Zu2l7ki7m5LTUx6350AR/3hcvwjSHC
+ZjD6fvQ8/zfjN8kaLZ6DAaqtqSlpawIM+8glkuTEkIkpBED/OtDrba4Rd29iLFVu
+wQZXDtTjAAZKZPmtTZ5TXLrcCU6VAgMBAAECgYEA1BFvmBsIN8Gu/+6kNupya2xU
+NVM0yLu/xT5lpNV3LBO325oejAq8+d87kkl/LTW3a2jGFlQ0ICuLw+2mo24QUWRy
+v8if3oeBMlnLqHE+6wNjFVqo5sOjKzjO363xSXwXNUrBT7jDhnZcDN8w3/FecYKj
+ifGTVtUs1SLsYwhlc8ECQQDuCRymLZQ/imPn5eFVIydwUzg8ptZlvoA7bfIxUL9B
+QRX33s59kLCilA0tTed8Dd+GnxsT93XOj1ApIfBwmTSlAkEA5/63PDsN7fH+WInq
+VD8nU07M9S8LcGDlPbVVBr2S2I78/iwrSDAYtbkU2vEbhFK/JuKNML2j8OkzV3v1
+QulfMQJBALDzhx+l/HHr3+8RPhx7QKNIyiKUaAdEwbDsP8IXY8YPq1QThu9jM1v4
+sX7/TdkzuvoppwiFykbe1NlvCH279p0CQCmTg4Ee0DtBcCSr6rvYaZLLf329RZ6J
+LuwlMCy6ErQOxBZFEiiovfTrS2qFZToMnkc4uLbwdY36LQJTq7unGTECQCCok8Lz
+BeZtAw+TJofpOM3F2Rlm2qXiBVBeubhRedsiljG0hpvvLJBMppnQ6r27p5Jk39Sm
+aTRkxEKrxPWWLNM=
+-----END PRIVATE KEY----- \ No newline at end of file
diff --git a/test/functional/configs/dkim_signing/eddsa.conf b/test/functional/configs/dkim_signing/eddsa.conf
new file mode 100644
index 0000000..a8cdfaa
--- /dev/null
+++ b/test/functional/configs/dkim_signing/eddsa.conf
@@ -0,0 +1,8 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+dkim_signing {
+ path = "{= env.TESTDIR =}/configs/dkim-eddsa.key";
+ selector = "eddsa";
+ check_pubkey = true;
+ allow_pubkey_mismatch = false;
+}
diff --git a/test/functional/configs/dkim_signing/invalidate.conf b/test/functional/configs/dkim_signing/invalidate.conf
new file mode 100644
index 0000000..b7913cd
--- /dev/null
+++ b/test/functional/configs/dkim_signing/invalidate.conf
@@ -0,0 +1,8 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+dkim_signing {
+ path = "{= env.KEY_DIR =}/dkim-eddsa.key";
+ selector = "eddsa";
+ check_pubkey = true;
+ allow_pubkey_mismatch = false;
+}
diff --git a/test/functional/configs/dkim_signing/milter.conf b/test/functional/configs/dkim_signing/milter.conf
new file mode 100644
index 0000000..a373e16
--- /dev/null
+++ b/test/functional/configs/dkim_signing/milter.conf
@@ -0,0 +1,76 @@
+options = {
+ filters = ["dkim"]
+ url_tld = "{= env.URL_TLD =}"
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+ lua_path = "{= env.INSTALLROOT =}/share/rspamd/lib/?.lua"
+ dns {
+ nameserver = ["8.8.8.8", "8.8.4.4"];
+ retransmits = 10;
+ timeout = 2s;
+ }
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ }
+ unknown_weight = 1
+}
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ task_timeout = 60s;
+}
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl"
+}
+worker {
+ type = "rspamd_proxy";
+ count = 1;
+ timeout = 120;
+ upstream {
+ local {
+ hosts = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}";
+ default = true;
+ }
+ }
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_PROXY =}";
+ milter = true;
+}
+dkim_signing {
+ domain {
+ cacophony.za.org {
+ selectors = {
+ path: "{= env.TESTDIR =}/configs/dkim.key";
+ selector: "dkim";
+ }
+ selectors = {
+ path: "{= env.TESTDIR =}/configs/dkim-eddsa.key";
+ selector: "eddsa";
+ }
+ }
+ invalid.za.org {
+ selectors = [
+ { path: "{= env.TESTDIR =}/configs/dkim-eddsa.key";
+ selector: "eddsa"; }
+ ]
+ }
+ }
+ allow_pubkey_mismatch: true;
+}
+modules {
+ path = "{= env.TESTDIR =}/../../src/plugins/lua/dkim_signing.lua"
+}
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
+lua = "{= env.INSTALLROOT =}/share/rspamd/rules/rspamd.lua"
+lua = "{= env.TESTDIR =}/lua/params.lua"
diff --git a/test/functional/configs/dkim_signing/multiple.conf b/test/functional/configs/dkim_signing/multiple.conf
new file mode 100644
index 0000000..029c2b5
--- /dev/null
+++ b/test/functional/configs/dkim_signing/multiple.conf
@@ -0,0 +1,17 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+dkim_signing {
+ domain {
+ cacophony.za.org {
+ selectors = {
+ path: "{= env.TESTDIR =}/configs/dkim.key";
+ selector: "dkim";
+ }
+ selectors = {
+ path: "{= env.TESTDIR =}/configs/dkim-eddsa.key";
+ selector: "eddsa";
+ }
+ }
+ }
+ allow_pubkey_mismatch: false;
+}
diff --git a/test/functional/configs/dkim_signing/redis.conf b/test/functional/configs/dkim_signing/redis.conf
new file mode 100644
index 0000000..02dc2ea
--- /dev/null
+++ b/test/functional/configs/dkim_signing/redis.conf
@@ -0,0 +1,10 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+dkim_signing {
+ use_redis = true;
+ key_prefix = "TEST_DKIM_KEYS";
+ selector_prefix = "TEST_DKIM_SELECTORS";
+}
+redis {
+ servers = "{= env.REDIS_ADDR =}:{= env.REDIS_PORT =}";
+}
diff --git a/test/functional/configs/dkim_signing/sign_maps.conf b/test/functional/configs/dkim_signing/sign_maps.conf
new file mode 100644
index 0000000..6993f1f
--- /dev/null
+++ b/test/functional/configs/dkim_signing/sign_maps.conf
@@ -0,0 +1,11 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+dkim_signing {
+ signing_table = [
+ "*@cacophony.za.org cacophony.za.org",
+ ];
+
+ key_table = [
+ "cacophony.za.org %:eddsa:m5kGxtckRfsNe5EuYTe7bvkDjSh7LXaX3aXyIMPGLR0=",
+ ];
+}
diff --git a/test/functional/configs/dkim_signing/simple.conf b/test/functional/configs/dkim_signing/simple.conf
new file mode 100644
index 0000000..9b812ec
--- /dev/null
+++ b/test/functional/configs/dkim_signing/simple.conf
@@ -0,0 +1,9 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+dkim_signing {
+ path = "{= env.TESTDIR =}/configs/dkim.key";
+ check_pubkey = true;
+ allow_pubkey_mismatch = false;
+ use_http_headers = true;
+ allow_headers_fallback = true;
+}
diff --git a/test/functional/configs/dynamic.conf b/test/functional/configs/dynamic.conf
new file mode 100644
index 0000000..b766644
--- /dev/null
+++ b/test/functional/configs/dynamic.conf
@@ -0,0 +1,117 @@
+[
+ {
+ "metric": "default",
+ "actions": [
+ {
+ "name": "reject",
+ "value": 20.0
+ },
+ {
+ "name": "add header",
+ "value": 6.0
+ }
+ ],
+ "symbols": [
+ {
+ "name": "SA_BODY_WORD",
+ "value": 10.0
+ },
+ {
+ "name": "FORGED_RECIPIENTS",
+ "value": 0.0
+ },
+ {
+ "name": "PHISHING",
+ "value": 0.0
+ },
+ {
+ "name": "PRECEDENCE_BULK",
+ "value": 2.0
+ },
+ {
+ "name": "SPAM_FLAG",
+ "value": 6.0
+ },
+ {
+ "name": "BAYES_SPAM",
+ "value": 10.0
+ },
+ {
+ "name": "BAYES_HAM",
+ "value": -6.0
+ },
+ {
+ "name": "MISSING_TO",
+ "value": 3.0
+ },
+ {
+ "name": "FUZZY_DENIED",
+ "value": 0.0
+ },
+ {
+ "name": "DMARC_POLICY_QUARANTINE",
+ "value": 3.500000
+ },
+ {
+ "name": "DMARC_POLICY_SOFTFAIL",
+ "value": 2.0
+ },
+ {
+ "name": "DNSWL_BLOCKED",
+ "value": 1.0
+ },
+ {
+ "name": "RCVD_COUNT_TWO",
+ "value": 1.0
+ },
+ {
+ "name": "R_SPF_FAIL",
+ "value": 10.0
+ },
+ {
+ "name": "R_DKIM_ALLOW",
+ "value": -1.500000
+ },
+ {
+ "name": "FAKE_REPLY",
+ "value": 2.0
+ },
+ {
+ "name": "SUBJECT_ENDS_EXCLAIM",
+ "value": 2.0
+ },
+ {
+ "name": "FORGED_SENDER_MAILLIST",
+ "value": 1.0
+ },
+ {
+ "name": "RCVD_NO_TLS_LAST",
+ "value": 1.0
+ },
+ {
+ "name": "HFILTER_URL_ONLY",
+ "value": 4.200000
+ },
+ {
+ "name": "URI_COUNT_ODD",
+ "value": 2.0
+ },
+ {
+ "name": "FORGED_RECIPIENTS_MAILLIST",
+ "value": 1.0
+ },
+ {
+ "name": "SEM_URIBL_FRESH15",
+ "value": 4.0
+ },
+ {
+ "name": "FROM_NEQ_ENVFROM",
+ "value": 1.0
+ },
+ {
+ "name": "DMARC_POLICY_REJECT",
+ "value": 3.0
+ }
+ ]
+ }
+] \ No newline at end of file
diff --git a/test/functional/configs/empty.conf b/test/functional/configs/empty.conf
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/functional/configs/empty.conf
diff --git a/test/functional/configs/force_actions.conf b/test/functional/configs/force_actions.conf
new file mode 100644
index 0000000..aed3d05
--- /dev/null
+++ b/test/functional/configs/force_actions.conf
@@ -0,0 +1,88 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+force_actions {
+ rules {
+ FORCE_REJECT_TO_ADD_HEADER {
+ action = "add header";
+ expression = "UBER_REJECT";
+ require_action = ["reject"];
+ }
+ FORCE_REJECT_TO_NO_ACTION {
+ action = "no action";
+ expression = "UBER_REJECT2";
+ require_action = ["reject"];
+ }
+ FORCE_NO_ACTION_TO_REJECT {
+ action = "reject";
+ expression = "UBER_HAM";
+ require_action = ["no action"];
+ }
+ FORCE_NO_ACTION_TO_ADD_HEADER {
+ action = "add header";
+ expression = "UBER_HAM2";
+ require_action = ["no action"];
+ }
+ FORCE_ADD_HEADER_TO_NO_ACTION {
+ action = "no action";
+ expression = "UBER_ADD_HEADER";
+ require_action = ["add header"];
+ }
+ FORCE_ADD_HEADER_TO_REJECT {
+ action = "reject";
+ expression = "UBER_ADD_HEADER2";
+ require_action = ["add header"];
+ }
+ }
+}
+
+
+settings {
+ id_reject {
+ id = "id_reject";
+ apply {
+ symbols {
+ UBER_REJECT = 100500.0;
+ }
+ }
+ }
+ id_reject_no_action {
+ id = "id_reject_no_action";
+ apply {
+ symbols {
+ UBER_REJECT2 = 100500.0;
+ }
+ }
+ }
+ id_no_action {
+ id = "id_no_action";
+ apply {
+ symbols {
+ UBER_HAM = 1.0;
+ }
+ }
+ }
+ id_no_action_to_add_header {
+ id = "id_no_action_to_add_header";
+ apply {
+ symbols {
+ UBER_HAM2 = 1.0;
+ }
+ }
+ }
+ id_add_header {
+ id = "id_add_header";
+ apply {
+ symbols {
+ UBER_ADD_HEADER = 50500.0;
+ }
+ }
+ }
+ id_add_header_to_reject {
+ id = "id_add_header_to_reject";
+ apply {
+ symbols {
+ UBER_ADD_HEADER2 = 50500.0;
+ }
+ }
+ }
+}
diff --git a/test/functional/configs/fuzzy-encryption-key.conf b/test/functional/configs/fuzzy-encryption-key.conf
new file mode 100644
index 0000000..522081b
--- /dev/null
+++ b/test/functional/configs/fuzzy-encryption-key.conf
@@ -0,0 +1,2 @@
+# Setting this to null does not work out so it's hidden in an include
+encryption_key = {= env.FUZZY_ENCRYPTION_KEY =};
diff --git a/test/functional/configs/fuzzy.conf b/test/functional/configs/fuzzy.conf
new file mode 100644
index 0000000..8af1cfa
--- /dev/null
+++ b/test/functional/configs/fuzzy.conf
@@ -0,0 +1,95 @@
+redis {
+ servers = "{= env.REDIS_ADDR =}:{= env.REDIS_PORT =}";
+}
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
+options = {
+ filters = "fuzzy_check";
+ pidfile = "{= env.TMPDIR =}/rspamd.pid";
+ control_socket = "{= env.TMPDIR =}/rspamd.sock mode=0600";
+ url_tld = "{= env.TESTDIR =}/../lua/unit/test_tld.dat";
+ dns {
+ retransmits = 10;
+ timeout = 2s;
+ }
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ }
+ unknown_weight = 1
+ symbol {
+ weight = 10.0;
+ name = "{= env.FLAG1_SYMBOL =}";
+ }
+ symbol {
+ weight = -1.0;
+ name = "{= env.FLAG2_SYMBOL =}";
+ }
+}
+
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}";
+ count = 1
+ task_timeout = 60s;
+}
+
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}";
+ count = 1
+ secure_ip = ["{= env.LOCAL_ADDR =}"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl";
+}
+
+worker {
+ count = 1;
+ backend = "{= env.FUZZY_BACKEND =}";
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_FUZZY =}";
+ type = "fuzzy";
+ hashfile = "{= env.TMPDIR =}/fuzzy.db";
+ allow_update = ["{= env.LOCAL_ADDR =}"];
+ encrypted_only = {= env.FUZZY_ENCRYPTED_ONLY =};
+ keypair {
+ privkey = "{= env.KEY_PVT1 =}";
+ pubkey = "{= env.KEY_PUB1 =}";
+ }
+}
+
+fuzzy_check {
+ min_bytes = 100;
+ timeout = 1s;
+ retransmits = 10;
+
+ rule {
+ min_bytes = 0;
+ min_length = 0;
+ algorithm = "{= env.FUZZY_ALGORITHM =}";
+ servers = "{= env.LOCAL_ADDR =}:{= env.PORT_FUZZY =}";
+ symbol = "R_TEST_FUZZY";
+ max_score = 10.0;
+ mime_types = ["application/*"];
+ read_only = false;
+ skip_unknown = true;
+ skip_hashes = "{= env.TMPDIR =}/skip_hash.map";
+ fuzzy_key = {= env.FUZZY_KEY =};
+ fuzzy_shingles_key = {= env.FUZZY_SHINGLES_KEY =};
+.include "{= env.FUZZY_INCLUDE =}";
+ fuzzy_map = {
+ R_TEST_FUZZY_DENIED {
+ max_score = 10.0;
+ flag = {= env.FLAG1_NUMBER =};
+ }
+ R_TEST_FUZZY_WHITE {
+ max_score = 1.0;
+ flag = {= env.FLAG2_NUMBER =};
+ }
+ }
+ }
+}
diff --git a/test/functional/configs/known_senders-local.conf b/test/functional/configs/known_senders-local.conf
new file mode 100644
index 0000000..40522ae
--- /dev/null
+++ b/test/functional/configs/known_senders-local.conf
@@ -0,0 +1,4 @@
+known_senders {
+ enabled = true;
+ domains = "{= env.TESTDIR =}/configs/maps/known_senders_domains.map";
+}
diff --git a/test/functional/configs/known_senders.conf b/test/functional/configs/known_senders.conf
new file mode 100644
index 0000000..0880cea
--- /dev/null
+++ b/test/functional/configs/known_senders.conf
@@ -0,0 +1,7 @@
+.include "{= env.TESTDIR =}/../../conf/rspamd.conf"
+
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua"
+
+.include(priority=1,duplicate=merge) "{= env.TESTDIR =}/configs/known_senders-local.conf"
+.include(priority=1,duplicate=merge) "{= env.TESTDIR =}/configs/merged-local.conf"
+.include(priority=2,duplicate=replace) "{= env.TESTDIR =}/configs/merged-override.conf"
diff --git a/test/functional/configs/loggingtest-local.conf b/test/functional/configs/loggingtest-local.conf
new file mode 100644
index 0000000..7330d97
--- /dev/null
+++ b/test/functional/configs/loggingtest-local.conf
@@ -0,0 +1,5 @@
+logging {
+ type = "{= env.LOGGINGTYPE =}";
+ json = {= env.JSON =};
+ systemd = {= env.SYSTEMD =};
+}
diff --git a/test/functional/configs/loggingtest.conf b/test/functional/configs/loggingtest.conf
new file mode 100644
index 0000000..99026df
--- /dev/null
+++ b/test/functional/configs/loggingtest.conf
@@ -0,0 +1,3 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/trivial.conf"
+
+.include(priority=1,duplicate=merge) "{= env.TESTDIR =}/configs/loggingtest-local.conf"
diff --git a/test/functional/configs/lua_script.conf b/test/functional/configs/lua_script.conf
new file mode 100644
index 0000000..8798bff
--- /dev/null
+++ b/test/functional/configs/lua_script.conf
@@ -0,0 +1,25 @@
+options = {
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+}
+
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ task_timeout = 60s;
+}
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl"
+}
+
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
+lua = "{= env.LUA_SCRIPT =}";
diff --git a/test/functional/configs/lua_test.conf b/test/functional/configs/lua_test.conf
new file mode 100644
index 0000000..1d36536
--- /dev/null
+++ b/test/functional/configs/lua_test.conf
@@ -0,0 +1,52 @@
+options = {
+ filters = ["spf", "dkim", "regexp"]
+ url_tld = "{= env.URL_TLD =}"
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+ map_watch_interval = {= env.MAP_WATCH_INTERVAL =};
+ dns {
+ retransmits = 10;
+ timeout = 2s;
+ fake_records = [{
+ name = "example.com",
+ type = "a";
+ replies = ["93.184.216.34"];
+ }, {
+ name = "site.resolveme",
+ type = "a";
+ replies = ["127.0.0.1"];
+ }, {
+ name = "not-resolvable.com",
+ type = "a";
+ rcode = 'norec';
+ }]
+ }
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+ log_usec = true;
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ }
+ unknown_weight = 1
+}
+
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ task_timeout = 10s;
+}
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl"
+}
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
+lua = "{= env.LUA_SCRIPT =}";
diff --git a/test/functional/configs/maps/domains.cdb b/test/functional/configs/maps/domains.cdb
new file mode 100644
index 0000000..889268e
--- /dev/null
+++ b/test/functional/configs/maps/domains.cdb
Binary files differ
diff --git a/test/functional/configs/maps/domains.list b/test/functional/configs/maps/domains.list
new file mode 100644
index 0000000..04b0a9b
--- /dev/null
+++ b/test/functional/configs/maps/domains.list
@@ -0,0 +1,5 @@
+example.com
+cacophony.za.org both:1.0
+highsecure.ru bl:1.0
+rspamd.com wl:2.0
+#other.com \ No newline at end of file
diff --git a/test/functional/configs/maps/domains.list.2 b/test/functional/configs/maps/domains.list.2
new file mode 100644
index 0000000..15fb908
--- /dev/null
+++ b/test/functional/configs/maps/domains.list.2
@@ -0,0 +1,16 @@
+rspamd-test.com
+test.com
+test2.com
+test3.com
+test4.com
+test5.com
+test6.com
+test7.com
+test8.com
+test9.com
+test10.com
+test11.com
+test12.com
+test13.com
+test14.com
+#other.com
diff --git a/test/functional/configs/maps/dynamic_symbols.map b/test/functional/configs/maps/dynamic_symbols.map
new file mode 100644
index 0000000..ec92c74
--- /dev/null
+++ b/test/functional/configs/maps/dynamic_symbols.map
@@ -0,0 +1,2 @@
+foo DYN_TEST1:10:opt1,opt2
+bar DYN_TEST2:20:opt3,opt4
diff --git a/test/functional/configs/maps/external_relay.hostname_map b/test/functional/configs/maps/external_relay.hostname_map
new file mode 100644
index 0000000..fdb4fc0
--- /dev/null
+++ b/test/functional/configs/maps/external_relay.hostname_map
@@ -0,0 +1,3 @@
+cool.example.org direct
+lame.example.net
+
diff --git a/test/functional/configs/maps/external_relay.ip_map b/test/functional/configs/maps/external_relay.ip_map
new file mode 100644
index 0000000..f5b106f
--- /dev/null
+++ b/test/functional/configs/maps/external_relay.ip_map
@@ -0,0 +1,2 @@
+2001:db8::/32
+198.51.100.0/24
diff --git a/test/functional/configs/maps/external_relay.user_map b/test/functional/configs/maps/external_relay.user_map
new file mode 100644
index 0000000..bd04568
--- /dev/null
+++ b/test/functional/configs/maps/external_relay.user_map
@@ -0,0 +1,2 @@
+user@example.net
+
diff --git a/test/functional/configs/maps/external_relay_ip.list b/test/functional/configs/maps/external_relay_ip.list
new file mode 100644
index 0000000..3fc5c17
--- /dev/null
+++ b/test/functional/configs/maps/external_relay_ip.list
@@ -0,0 +1 @@
+192.168.1.1 \ No newline at end of file
diff --git a/test/functional/configs/maps/ip.list b/test/functional/configs/maps/ip.list
new file mode 100644
index 0000000..f35c9bc
--- /dev/null
+++ b/test/functional/configs/maps/ip.list
@@ -0,0 +1,4 @@
+127.0.0.1
+#127.0.0.2
+10.0.0.0/8
+[::1]/128 \ No newline at end of file
diff --git a/test/functional/configs/maps/ip2.list b/test/functional/configs/maps/ip2.list
new file mode 100644
index 0000000..7d99d28
--- /dev/null
+++ b/test/functional/configs/maps/ip2.list
@@ -0,0 +1,3 @@
+8.8.8.8 test one
+::1 another
+192.168.1.1
diff --git a/test/functional/configs/maps/known_senders_domains.map b/test/functional/configs/maps/known_senders_domains.map
new file mode 100644
index 0000000..8ddda0f
--- /dev/null
+++ b/test/functional/configs/maps/known_senders_domains.map
@@ -0,0 +1,2 @@
+outlook.com
+example.com
diff --git a/test/functional/configs/maps/map.list b/test/functional/configs/maps/map.list
new file mode 100644
index 0000000..8e68bab
--- /dev/null
+++ b/test/functional/configs/maps/map.list
@@ -0,0 +1,3 @@
+foo bar
+asdf.example.com value
+asdf
diff --git a/test/functional/configs/maps/mid.list b/test/functional/configs/maps/mid.list
new file mode 100644
index 0000000..b89c084
--- /dev/null
+++ b/test/functional/configs/maps/mid.list
@@ -0,0 +1,2 @@
+cacophony.za.org /^<[A-z0-9+]{18}>$/
+mom.za.org
diff --git a/test/functional/configs/maps/mime_types.wl b/test/functional/configs/maps/mime_types.wl
new file mode 100644
index 0000000..eca07bd
--- /dev/null
+++ b/test/functional/configs/maps/mime_types.wl
@@ -0,0 +1 @@
+/^hello_world\.exe$/
diff --git a/test/functional/configs/maps/multiple.list b/test/functional/configs/maps/multiple.list
new file mode 100644
index 0000000..3d4f32f
--- /dev/null
+++ b/test/functional/configs/maps/multiple.list
@@ -0,0 +1,12 @@
+#empty string
+user1@example.com
+#just score
+user2@example.com 10
+#just symbol
+user3@example.com SYM1
+#unknown symbol
+user4@example.com SYM2
+#symbol + score
+user5@example.com SYM1:-10.1
+#symbol + score + options
+user6@example.com SYM1:-10.1:opt1,opt2
diff --git a/test/functional/configs/maps/rcvd.list b/test/functional/configs/maps/rcvd.list
new file mode 100644
index 0000000..5c59711
--- /dev/null
+++ b/test/functional/configs/maps/rcvd.list
@@ -0,0 +1 @@
+2a01:7c8:aab6:26d:5054:ff:fed1:1da2
diff --git a/test/functional/configs/maps/rcvd2.list b/test/functional/configs/maps/rcvd2.list
new file mode 100644
index 0000000..97dba5b
--- /dev/null
+++ b/test/functional/configs/maps/rcvd2.list
@@ -0,0 +1,2 @@
+151.18.193.131
+
diff --git a/test/functional/configs/maps/redir.map b/test/functional/configs/maps/redir.map
new file mode 100644
index 0000000..5b7eb38
--- /dev/null
+++ b/test/functional/configs/maps/redir.map
@@ -0,0 +1,3 @@
+t.co
+bit.ly
+127.0.0.1
diff --git a/test/functional/configs/maps/regexp.list b/test/functional/configs/maps/regexp.list
new file mode 100644
index 0000000..ceb2bf5
--- /dev/null
+++ b/test/functional/configs/maps/regexp.list
@@ -0,0 +1,5 @@
+/^.*@example.com/i
+/^user.*@.*com/i
+/foo/ bar
+/asdf\.example\.com/ value
+/^asdf$/
diff --git a/test/functional/configs/maps/strict.phishing b/test/functional/configs/maps/strict.phishing
new file mode 100644
index 0000000..af2de57
--- /dev/null
+++ b/test/functional/configs/maps/strict.phishing
@@ -0,0 +1 @@
+myspace.com
diff --git a/test/functional/configs/maps/stricter.phishing b/test/functional/configs/maps/stricter.phishing
new file mode 100644
index 0000000..a1240ea
--- /dev/null
+++ b/test/functional/configs/maps/stricter.phishing
@@ -0,0 +1 @@
+bank.com
diff --git a/test/functional/configs/maps/top.list b/test/functional/configs/maps/top.list
new file mode 100644
index 0000000..d8a152e
--- /dev/null
+++ b/test/functional/configs/maps/top.list
@@ -0,0 +1,2 @@
+au
+#bg \ No newline at end of file
diff --git a/test/functional/configs/maps/url_compose_map.list b/test/functional/configs/maps/url_compose_map.list
new file mode 100644
index 0000000..808c455
--- /dev/null
+++ b/test/functional/configs/maps/url_compose_map.list
@@ -0,0 +1,3 @@
+*.dirty.sanchez.com
+!not.dirty.sanchez.com
+41.black.sanchez.com \ No newline at end of file
diff --git a/test/functional/configs/maps/url_compose_map_for_mails.list b/test/functional/configs/maps/url_compose_map_for_mails.list
new file mode 100644
index 0000000..1d54a3e
--- /dev/null
+++ b/test/functional/configs/maps/url_compose_map_for_mails.list
@@ -0,0 +1,3 @@
+*.dirty.sanchez.com
+!admin.dirty.sanchez.com
+41.black.sanchez.com \ No newline at end of file
diff --git a/test/functional/configs/maps/users.list b/test/functional/configs/maps/users.list
new file mode 100644
index 0000000..696fb6b
--- /dev/null
+++ b/test/functional/configs/maps/users.list
@@ -0,0 +1 @@
+bob
diff --git a/test/functional/configs/maps/utf.list b/test/functional/configs/maps/utf.list
new file mode 100644
index 0000000..4709308
--- /dev/null
+++ b/test/functional/configs/maps/utf.list
@@ -0,0 +1 @@
+/ваÑиÑуал/iu
diff --git a/test/functional/configs/merged-local.conf b/test/functional/configs/merged-local.conf
new file mode 100644
index 0000000..0708e3c
--- /dev/null
+++ b/test/functional/configs/merged-local.conf
@@ -0,0 +1,972 @@
+history_redis {
+ enabled = false;
+}
+
+neural {
+ enabled = false;
+}
+
+bayes_expiry {
+ enabled = false;
+}
+
+metric_exporter {
+ enabled = false;
+}
+
+emails {
+ "whitelist" = [
+ "rspamd-test.com"
+ ]
+ rules {
+ "RSPAMD_EMAILBL_FULL" {
+ dnsbl = "test5.uribl";
+ replyto = true;
+ }
+ "RSPAMD_EMAILBL_DOMAINONLY" {
+ dnsbl = "test6.uribl";
+ domain_only = true;
+ replyto = true;
+ }
+ }
+}
+
+external_relay {
+ enabled = {= env.EXTERNAL_RELAY_ENABLED =};
+
+ rules {
+ EXTERNAL_RELAY_AUTHENTICATED {
+ strategy = "authenticated";
+ user_map = "{= env.TESTDIR =}/configs/maps/external_relay.user_map";
+ }
+ EXTERNAL_RELAY_COUNT {
+ count = 4;
+ # `count` strategy always pops Received headers out, this will break other rules.
+ # So it should always be the last rule.
+ priority = 30;
+ strategy = "count";
+ }
+ EXTERNAL_RELAY_HOSTNAME_MAP {
+ hostname_map = "{= env.TESTDIR =}/configs/maps/external_relay.hostname_map";
+ strategy = "hostname_map";
+ }
+ EXTERNAL_RELAY_IP_MAP {
+ ip_map = "{= env.TESTDIR =}/configs/maps/external_relay.ip_map";
+ strategy = "ip_map";
+ }
+ EXTERNAL_RELAY_LOCAL {
+ strategy = "local";
+ }
+ }
+}
+
+greylist {
+ check_local = true;
+ timeout = 4;
+}
+
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+ log_usec = true;
+}
+
+mid = {
+ source = {
+ url = [
+ "https://maps.rspamd.com/rspamd/mid.inc.zst",
+ "fallback+file://{= env.TESTDIR =}/../../../conf/mid.inc",
+ "file://{= env.TESTDIR =}/configs/maps/mid.list"
+ ];
+ }
+}
+
+mime_types {
+ file = [
+ "https://maps.rspamd.com/rspamd/mime_types.inc.zst",
+ "fallback+file://{= env.TESTDIR =}/../../../conf/mime_types.inc"
+ ];
+ extension_map {
+ html = "text/html";
+ txt [
+ "message/disposition-notification",
+ "text/plain",
+ "text/rfc822-headers",
+ ]
+ pdf [
+ "application/octet-stream",
+ "application/pdf",
+ ]
+ }
+ filename_whitelist = "{= env.TESTDIR =}/configs/maps/mime_types.wl";
+}
+
+options = {
+ pidfile = "{= env.TMPDIR =}/rspamd.pid";
+ url_tld = "{= env.TESTDIR =}/../lua/unit/test_tld.dat";
+ dns {
+ nameserver = ["8.8.8.8", "8.8.4.4"];
+ retransmits = 10;
+ timeout = 2s;
+ fake_records = [{
+ # non-existent records requested by 350_surbl tests
+ name = "114.73.21.104.test4.uribl", type = "a", rcode = "norec"},
+ {name = "153.23.128.52.test4.uribl", type = "a", rcode = "norec"},
+ {name = "158.136.181.135.test4.uribl", type = "a", rcode = "norec"},
+ {name = "177.115.236.44.test4.uribl", type = "a", rcode = "norec"},
+ {name = "180.136.102.34.test4.uribl", type = "a", rcode = "norec"},
+ {name = "180.144.67.172.test4.uribl", type = "a", rcode = "norec"},
+ {name = "2.7.9.4.5.1.8.6.0.0.0.0.0.0.0.0.0.0.0.0.7.3.0.3.0.0.7.4.6.0.6.2.test4.uribl", type = "a", rcode = "norec"},
+ {name = "217.228.62.64.test4.uribl", type = "a", rcode = "norec"},
+ {name = "34.216.184.93.test4.uribl", type = "a", rcode = "norec"},
+ {name = "4.b.0.9.3.4.c.a.0.0.0.0.0.0.0.0.0.0.0.0.1.3.0.3.0.0.7.4.6.0.6.2.test4.uribl", type = "a", rcode = "norec"},
+ {name = "4eikheqjb4rb3y4oxgnfxek9wrwnyii4.test.uribl", type = "a", rcode = "norec"},
+ {name = "6.4.9.1.8.c.5.2.3.9.8.1.8.4.2.0.1.0.0.0.0.2.2.0.0.0.8.2.6.0.6.2.test4.uribl", type = "a", rcode = "norec"},
+ {name = "69so7b146n15x6qkr8rj9x8iqb7zfr1s.test.uribl", type = "a", rcode = "norec"},
+ {name = "6cqpxfrojdnzawwjmacjwtstzwehxnzb.test.uribl", type = "a", rcode = "norec"},
+ {name = "7.5.2.e.9.5.e.f.f.f.1.9.c.3.0.f.0.0.0.0.0.0.0.0.1.0.c.3.0.0.6.2.test4.uribl", type = "a", rcode = "norec"},
+ {name = "baddomain.com.test2.uribl", type = "a", rcode = "norec"},
+ {name = "baddomain.com.test3.uribl", type = "a", rcode = "norec"},
+ {name = "bbjo9td11ewijyjkq8rsn4j3kxhthz4m.test.uribl", type = "a", rcode = "norec"},
+ {name = "emailbl.com.test2.uribl", type = "a", rcode = "norec"},
+ {name = "emailbl.com.test3.uribl", type = "a", rcode = "norec"},
+ {name = "emailbl.com.test6.uribl", type = "a", rcode = "norec"},
+ {name = "example.com.test3.uribl", type = "a", rcode = "norec"},
+ {name = "example.com.test6.uribl", type = "a", rcode = "norec"},
+ {name = "example.com.test7.uribl", type = "a", rcode = "norec"},
+ {name = "example.net.test3.uribl", type = "a", rcode = "norec"},
+ {name = "example.net.test7.uribl", type = "a", rcode = "norec"},
+ {name = "example.org.test2.uribl", type = "a", rcode = "norec"},
+ {name = "example.org.test3.uribl", type = "a", rcode = "norec"},
+ {name = "example.org.test7.uribl", type = "a", rcode = "norec"},
+ {name = "example.ru.test2.uribl", type = "a", rcode = "norec"},
+ {name = "example.ru.test7.uribl", type = "a", rcode = "norec"},
+ {name = "gdhhir83i5pjk6s8i3e5afwa4md7uns7.test.uribl", type = "a", rcode = "norec"},
+ {name = "k8qo8m33z19cqejfncirs85rf4nr9h3u.test.uribl", type = "a", rcode = "norec"},
+ {name = "kr1adm7tnzuudiftdt1g4t6yg1rbt1ez.test.uribl", type = "a", rcode = "norec"},
+ {name = "rspamd.com.test2.uribl", type = "a", rcode = "norec"},
+ {name = "rspamd.com.test3.uribl", type = "a", rcode = "norec"},
+ {name = "rspamd.com.test7.uribl", type = "a", rcode = "norec"},
+ {name = "rspamd.tk.test3.uribl", type = "a", rcode = "norec"},
+ {name = "rspamd.tk.test7.uribl", type = "a", rcode = "norec"},
+ {name = "sanchez.com.test2.uribl", type = "a", rcode = "norec"},
+ {name = "sanchez.com.test3.uribl", type = "a", rcode = "norec"},
+ {name = "sanchez.com.test7.uribl", type = "a", rcode = "norec"},
+ {name = "subdomain.emailbl.com.test6.uribl", type = "a", rcode = "norec"},
+ {name = "testtest.com.test2.uribl", type = "a", rcode = "norec"},
+ {name = "testtest.com.test3.uribl", type = "a", rcode = "norec"},
+ {name = "testtest.com.test7.uribl", type = "a", rcode = "norec"},
+ {name = "user.baddomain.com.test5.uribl", type = "a", rcode = "norec"},
+ {name = "user.example.com.test5.uribl", type = "a", rcode = "norec"},
+ {name = "xn--80arbjktj.xn--p1ai.test3.uribl", type = "a", rcode = "norec"},
+ {name = "xn--80arbjktj.xn--p1ai.test7.uribl", type = "a", rcode = "norec"},
+ {name = "y84tis6xzaf41h4p5kzxiw6puixnm43k.test.uribl", type = "a", rcode = "norec"},
+ # other stuff is here too! :\
+ { # ed25519
+ name = "test._domainkey.example.com";
+ type = txt;
+ replies = ["k=ed25519; p=yi50DjK5O9pqbFpNHklsv9lqaS0ArSYu02qp1S0DW1Y="];
+ },
+ {
+ name = "brisbane._domainkey.football.example.com";
+ type = txt;
+ replies = ["v=DKIM1; k=ed25519; p=11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo="];
+ },
+ {
+ name = "test._domainkey.football.example.com";
+ type = txt;
+ replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDkHlOQoBTzWRiGs5V6NpP3idY6Wk08a5qhdR6wy5bdOKb2jLQiY/J16JYi0Qvx/byYzCNb3W91y3FutACDfzwQ/BC/e/8uBsCR+yz1Lxj+PL6lHvqMKrM3rG4hstT5QjvHO9PzoxZyVYLzBfO2EeC3Ip3G+2kryOTIKT+l/K4w3QIDAQAB"],
+ },
+ {
+ name = "dkim._domainkey.cacophony.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXtxBE5IiNRMcq2/lc2zErfdCvDFyQNBnMjbOjBQrPST2k4fdGbtpe5Iu5uS01Met+dAEf94XL8I0hwmYw+n70PP834zfJGi2egwGqrakpaWsCDPvIJZLkxJCJKQRA/zrQ622uEXdvYixVbsEGVw7U4wAGSmT5rU2eU1y63AlOlQIDAQAB"];
+ },
+ {
+ name = "eddsa._domainkey.cacophony.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=ed25519; p=+nU+aC33ICeS4zx8VUjFYCtxj0fRbHWQn2gP2hTkm9w="];
+ },
+ {
+ name = "dkim._domainkey.invalid.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDEEXmNGQq7PUrr9Mg4UakTFHgXBCy2DOztkrZm+0OrVWtiRzGluxBkbOWTBwuU3/Yw97yTphBMQxzWFN603/f/KPAQcF/Lc1l+6kmIBBxNXjjGuOK/3PYKZVntUdKmqcQBYfnHdzH2Tohbuyx1a7xqnv6VSChqQrZU4CwkeT3+eQIDAQAB"];
+ },
+ {
+ name = "eddsa._domainkey.invalid.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=ed25519; p=Wkkrp5DJTvknDMGWYv8vm3p3sZjiQp03LZo80RregY8="];
+ },
+ {
+ name = "dkim._domainkey.rspamd.com",
+ type = "txt";
+ replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCd/XhZBEGGAss48lEuMmwZv9lOFf6FTllBiQ3sPhdTpDdIPaW9TInW7iYnYD/bXHeVxYAyD/sKhYk6+qGBRu10rEi+iyPvLCIED+Boq0tEQosuKuV6Fjoomb+QhZY9KdjyZTjsrFPZ+wCkUY/30uTmpX2SwSqyxxlK0pUIsRgMAQIDAQAB"];
+ },
+ {
+ name = "_dmarc.rspamd.com",
+ type = "txt";
+ rcode = 'norec';
+ },
+ {
+ name = "dkim._domainkey.highsecure.ru",
+ type = "txt";
+ replies = ["p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDK4ZQYky30GH0Ak9OQ1fv3IdFNbpOtpa4S/PR20ZLgPXfd/FCA//ztUmu7kHlELI+/+4f8W+xX0oZlOc/cFxhopRjXZMlSsQqmWOZ40/GxWFBtcqafKu78FCqO7URqZUmMCM5Jlp4zt/yzH3dbYNG3i5PVlB5QtQnZvY+dvBL3dwIDAQAB"];
+ },
+ {
+ name = "_dmarc.cacophony.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=none; sp=reject"];
+ },
+ {
+ name = "_dmarc.my.mom.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=reject"];
+ },
+ {
+ name = "example.net",
+ type = "txt";
+ replies = ["v=spf1 -all"];
+ },
+ {
+ name = "fail4.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 redirect=asdfsfewewrredfs"];
+ },
+ {
+ name = "_dmarc.reject.cacophony.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=reject"];
+ },
+ {
+ name = "spf.cacophony.za.org",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.4.4 -all"];
+ },
+ {
+ name = "fail7.org.org.za",
+ type = "a";
+ rcode = 'norec';
+ },
+ {
+ name = "fail6.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 mx -all"];
+ },
+ {
+ name = "fail6.org.org.za",
+ type = "mx";
+ rcode = 'norec';
+ },
+ {
+ name = "fail7.org.org.za",
+ type = "aaaa";
+ rcode = 'norec';
+ },
+ {
+ name = "_dmarc.quarantine.cacophony.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=quarantine"];
+ },
+ {
+ name = "_dmarc.yo.mom.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=reject; aspf=s; adkim=s;"];
+ },
+ {
+ name = "yo.mom.za.org",
+ type = "txt";
+ replies = ["v=spf1 ip4:37.48.67.26 -all"];
+ },
+ {
+ name = "testdkim._domainkey.mom.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC3v4VPE1QMHUzsMRbC8VzXNq82mDjiv9Gi1NB/YYC+vIYZT+sE/Uxnr0Clk8C2jgzEr3jcxgQEWZfMtEEg/EfEJvh4SrXWv9c0gw1EEfxKxX9i+r8yBQtc/EWospWVDkhF2lAvQAK1lV1ZiU7psJ6fh1CI39uZyWdAktZzWLf0zQIDAQAB"];
+ },
+ {
+ name = "_dmarc.rspamd.tk",
+ type = "txt";
+ replies = ["bio=a263adeab8acdcdb8b89e127b67d696061fdfbee"];
+ },
+ {
+ name = "fail2.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.4.4 include:www.dnssec-failed.org -all"];
+ },
+ {
+ name = "fail3.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 include:total.barf -all"];
+ },
+ {
+ name = "mom.za.org",
+ type = "txt";
+ replies = ["v=spf1 ip4:37.48.67.26 -all"];
+ },
+ {
+ name = "testdkim._domainkey.asdf.rspamd.tk", # testdkim._domainkey.asdf.rspamd.tk is an alias for rspamd.tk
+ type = "txt";
+ replies = ["bio=a263adeab8acdcdb8b89e127b67d696061fdfbee"];
+ },
+ {
+ name = "testdkim._domainkey.rspamd.tk", # testdkim._domainkey.rspamd.tk is an alias for rspamd.tk
+ type = "txt";
+ replies = ["bio=a263adeab8acdcdb8b89e127b67d696061fdfbee"];
+ },
+ {
+ name = "pass1.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 include:pass2.org.org.za -all"];
+ },
+ {
+ name = "95.142.99.88.in-addr.arpa",
+ type = "ptr";
+ replies = ["mail.highsecure.ru"];
+ },
+ {
+ name = "mail.highsecure.ru",
+ type = "a";
+ replies = ["88.99.142.95"];
+ },
+ {
+ name = "mail.highsecure.ru",
+ type = "aaaa";
+ rcode = 'norec';
+ },
+ {
+ name = "1.0.66.128.in-addr.arpa",
+ type = "ptr";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "182.216.85.209.in-addr.arpa",
+ type = "ptr";
+ replies = ["mail-qt0-f182.google.com"];
+ },
+ {
+ name = "crazyspf.cacophony.za.org",
+ type = "txt";
+ replies = ["v=spf1 ptr:cacophony.za.org ptr:rspamd.com ptr:yahoo.com ptr:yahoo.net ptr:highsecure.ru -all"];
+ },
+ {
+ name = "pass2.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 -all"];
+ },
+ {
+ name = "_dmarc.yoni.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=reject; sp=none;"];
+ },
+ {
+ name = "fail10.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 redirect=fail5.org.org.za"];
+ },
+ {
+ name = "fail11.org.org.za",
+ type = "txt";
+ replies = ["v=sPF1 ip4:8.8.8.8 -all"];
+ },
+ {
+ name = "fail5.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 OMGBARF"];
+ },
+ {
+ name = "fail7.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 a -all"];
+ },
+ {
+ name = "trusted.com",
+ type = "txt";
+ replies = ["v=spf1 ip4:192.168.1.1"];
+ },
+ {
+ name = "external.com",
+ type = "txt";
+ replies = ["v=spf1 ip4:37.48.67.26"];
+ },
+ {
+ name = "co.za",
+ type = "txt";
+ rcode = 'norec';
+ },
+ {
+ name = "testdkim1._domainkey.yoni.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=rsa; p=BARF"];
+ },
+ {
+ name = "_dmarc.yoni.za.net",
+ type = "txt";
+ replies = ["v=DMARC1; p=none; sp=quarantine"];
+ },
+ {
+ name = "za",
+ type = "txt";
+ replies = ["Top-level domain for South Africa"];
+ },
+ {
+ name = "_dmarc.foo.yoni.za.org",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "_dmarc.foo.cacophony.za.org",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "_dmarc.foo.yoni.za.net",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "_dmarc.dnssec-failed.org",
+ type = "txt";
+ rcode = 'timeout';
+ },
+ {
+ name = "_dmarc.example.com",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "_dmarc.zero_pct.com",
+ type = "txt";
+ replies = ["v=DMARC1; p=reject; sp=quarantine; pct=0"];
+ },
+ {
+ name = "example.com",
+ type = "txt";
+ replies = ["$Id: example.com 4415 2015-08-24 20:12:23Z davids $", "v=spf1 -all"];
+ },
+ {
+ name = "example.com",
+ type = "a";
+ replies = ["93.184.216.34"];
+ },
+ {
+ name = "testdkim1._domainkey.dnssec-failed.org",
+ type = "txt";
+ rcode = 'timeout';
+ },
+ {
+ name = "total.barf",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "_dmarc.foo.cacophony.za.org",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "zzzzaaaa",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "asdfsfewewrredfs",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "95.142.99.88.asn.rspamd.com",
+ type = "txt";
+ replies = ["24940|88.99.0.0/16|DE|ripencc|"];
+ },
+ {
+ name = "2.a.d.1.1.d.e.f.f.f.0.0.4.5.0.5.d.6.2.0.6.b.a.a.8.c.7.0.1.0.a.2.asn6.rspamd.com",
+ type = "txt";
+ replies = ["20857|2a01:7c8::/32|NL|ripencc|"];
+ },
+ {
+ name = "2.a.d.1.1.d.e.f.f.f.0.0.4.5.0.5.d.6.2.0.6.b.a.a.8.c.7.0.1.0.a.2.rspamd.com",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "2.a.d.1.1.d.e.f.f.f.0.0.4.5.0.5.d.6.2.0.6.b.a.a.8.c.7.0.1.0.a.2.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "rspamd.com",
+ type = "txt";
+ replies = ["v=spf1 mx -all"];
+ },
+ {
+ name = "rspamd.com",
+ type = "mx";
+ replies = ["10 mail.highsecure.ru"];
+ },
+ {
+ name = "95.142.99.88.rspamd.com",
+ type = "a";
+ rcode = 'norec';
+ },
+ {
+ name = "95.142.99.88.rspamd.com",
+ type = "aaaa";
+ rcode = 'norec';
+ },
+ {
+ name = "2.0.0.127.rspamd.com",
+ type = "a";
+ replies = ["127.0.0.1"];
+ },
+ {
+ name = "8.8.8.8.asn.rspamd.com",
+ type = "txt";
+ replies = ["15169|8.8.8.0/24|US|arin|"];
+ },
+ {
+ name = "8.8.8.8.asn.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "10.0.1.10.asn.rspamd.com",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "10.0.1.10.asn.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "10.0.1.11.asn.rspamd.com",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "10.0.1.11.asn.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "test.com",
+ type = "txt";
+ replies = [""];
+ },
+ {
+ name = "other.com",
+ type = "txt";
+ rcode = 'norec';
+ },
+ {
+ name = "bob",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "mail.highsecure.ru",
+ type = "aaaa";
+ rcode = 'norec';
+ },
+ {
+ name = "mail.highsecure.ru",
+ type = "a";
+ replies = ["88.99.142.95"];
+ },
+ {
+ name = "4.3.2.1.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "4.3.2.1.asn.rspamd.com",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "1.0.0.127.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "114.47.228.46.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "114.47.228.46.asn.rspamd.com",
+ type = "txt";
+ replies = ["34010|46.228.40.0/21|GB|ripencc|"];
+ },
+ {
+ name = "10.0.1.10.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "10.0.1.11.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "other.org",
+ type = "txt";
+ rcode = 'norec';
+ },
+ {
+ name = "8.8.8.8.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "rspamd.tk",
+ type = "txt";
+ replies = ["bio=a263adeab8acdcdb8b89e127b67d696061fdfbee"];
+ },
+ {
+ name = "fail1.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 redirect=www.dnssec-failed.org"];
+ },
+ {
+ name = "www.dnssec-failed.org",
+ type = "txt";
+ rcode = 'timeout';
+ },
+ {
+ name = "www.dnssec-failed.org",
+ type = "mx";
+ rcode = 'timeout';
+ },
+ {
+ name = "www.dnssec-failed.org",
+ type = "a";
+ rcode = 'timeout';
+ },
+ {
+ name = "www.dnssec-failed.org",
+ type = "aaaa";
+ rcode = 'norec';
+ },
+ {
+ name = "cacophony.za.org",
+ type = "txt";
+ replies = ["v=spf1 redirect=asdfsfewewrredfs"];
+ },
+ {
+ name = "fail9.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 mx:www.dnssec-failed.org -all"];
+ },
+ {
+ name = "fail8.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 a:www.dnssec-failed.org -all"];
+ },
+ {
+ name = "1.2.3.4.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "2.2.3.4.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.10"];
+ },
+ {
+ name = "3.2.3.4.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.2", "127.0.0.3"];
+ },
+ {
+ name = "4.2.3.4.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "4.2.3.4.fake.wl";
+ type = "a";
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "4.3.2.1.fake.rbl";
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "131.193.18.151.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.3"];
+ },
+ # SURBL tests
+ {
+ name = "rciuosbadgpq6b5wt436nhgnwzmfh9w9.test.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ # testtest.com
+ name = "rcf1ecxtxrrpfncqzsdaiezjkf7f1rzz.test.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "jhcszdsmo3wuj5mp8t38kdisdmr3ib3q.test.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "g9ifs3q39oh5jwru94cj7ffaqd6rfyq6.test.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "xn--80arbjktj.xn--p1ai.test2.uribl";
+ type = a;
+ replies = ["127.0.1.2"];
+ },
+ {
+ name = "мойÑайт.рф.test2.uribl";
+ type = a;
+ replies = ["127.0.1.2"];
+ },
+ {
+ name = "user.emailbl.com.test5.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "user.subdomain.emailbl.com.test5.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "baddomain.com.test6.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "user.subdomain.baddomain.com.test5.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "example.com.test2.uribl";
+ type = a;
+ replies = ["127.0.1.2"];
+ },
+ {
+ name = "user.example.com.test2.uribl";
+ type = a;
+ replies = ["127.0.1.5"];
+ },
+ {
+ name = "example.net.test2.uribl";
+ type = a;
+ replies = ["127.0.1.4"];
+ },
+ {
+ name = "rspamd.tk.test2.uribl";
+ type = a;
+ replies = ["127.0.1.4"];
+ },
+ {
+ name = "example.org.test3.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "example.ru.test3.uribl";
+ type = a;
+ replies = ["127.0.0.12"];
+ },
+ {
+ name = "example.ru";
+ type = a;
+ replies = ["8.8.8.8", "8.8.8.9"];
+ },
+ {
+ name = "8.8.8.8.test4.uribl";
+ type = a;
+ replies = ["127.0.0.4", "127.0.0.11"];
+ },
+ {
+ name = "uppht14nj4fsoycu3huctg9d5psx9je4.test.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "rspamd-test.com.test2.uribl";
+ type = a;
+ replies = ["127.0.1.2"];
+ },
+ {
+ name = "9.8.8.8.test4.uribl";
+ type = a;
+ replies = ["127.0.0.3"];
+ },
+ {
+ name = "4.very.dirty.sanchez.com.test7.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "clean.dirty.sanchez.com.test7.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "not.dirty.sanchez.com.test7.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "41.black.sanchez.com.test7.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "black.sanchez.com.test7.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "dirty.sanchez.com.test8.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "very.dirty.sanchez.com.test8.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "41.black.sanchez.com.test8.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "black.sanchez.com.test8.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "example.com.test9.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "example.org.test9.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "4.3.2.1.test9.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "judo.za.org.test9.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "12.11.10.9.test9.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "8.7.6.5.test9.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "8.8.8.8.test9.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "site.resolveme",
+ type = "a";
+ replies = ["127.0.0.1"];
+ },
+ {
+ name = "not-resolvable.com",
+ type = "a";
+ rcode = 'norec';
+ },
+ # TODO: add IPv6 tests
+ ];
+ }
+}
+
+phishing {
+ symbol = "PHISHING";
+ strict_domains = {
+ STRICT_PHISHING = [
+ "{= env.TESTDIR =}/configs/maps/strict.phishing",
+ ];
+ STRICTER_PHISHING = [
+ "{= env.TESTDIR =}/configs/maps/stricter.phishing"
+ ]
+ }
+}
+
+redis {
+ servers = "{= env.REDIS_ADDR =}:{= env.REDIS_PORT =}";
+ expand_keys = true;
+}
+
+regexp {
+ CONFIG_SELECTOR_RE_RCPT_SUBJECT {
+ re = 'test=/test@user.com some subject/$',
+ score = 100500,
+ }
+}
+
+spf {
+ external_relay = [
+ "{= env.TESTDIR =}/configs/maps/external_relay_ip.list",
+ ];
+}
+
+symbols {
+ FOUR_POINTS = {
+ score = 4.0,
+ }
+ SYM1 = {
+ score = 1.0,
+ }
+}
+
+worker "controller" {
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}";
+ keypair {
+ pubkey = "{= env.KEY_PUB1 =}";
+ privkey = "{= env.KEY_PVT1 =}";
+ }
+}
+
+worker "normal" {
+ count = 1;
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}";
+ keypair {
+ pubkey = "{= env.KEY_PUB1 =}";
+ privkey = "{= env.KEY_PVT1 =}";
+ }
+}
diff --git a/test/functional/configs/merged-override.conf b/test/functional/configs/merged-override.conf
new file mode 100644
index 0000000..09ac005
--- /dev/null
+++ b/test/functional/configs/merged-override.conf
@@ -0,0 +1,435 @@
+antivirus {
+ clam {
+ attachments_only = false;
+ symbol = "CLAM_VIRUS";
+ type = "clamav";
+ servers = "127.0.0.1:{= env.PORT_CLAM =}";
+ }
+ fprot {
+ attachments_only = false;
+ symbol = "FPROT_VIRUS";
+ type = "fprot";
+ servers = "127.0.0.1:{= env.PORT_FPROT =}";
+ patterns {
+ FPROT_EICAR = "^EICAR_Test_File$";
+ }
+ }
+ fprot_duplicate {
+ prefix = "fp_dupe";
+ attachments_only = false;
+ symbol = "FPROT2_VIRUS_DUPLICATE_DEFAULT";
+ type = "fprot";
+ servers = "127.0.0.1:{= env.PORT_FPROT2_DUPLICATE =}";
+ patterns = [
+ {FPROT2_VIRUS_DUPLICATE_PATTERN = "^E"},
+ {FPROT2_VIRUS_DUPLICATE_NOPE1 = "^EI",
+ FPROT2_VIRUS_DUPLICATE_NOPE2 = "^EIC",
+ FPROT2_VIRUS_DUPLICATE_NOPE3 = "^EICA",
+ FPROT2_VIRUS_DUPLICATE_NOPE4 = "^EICAR",
+ FPROT2_VIRUS_DUPLICATE_NOPE5 = "^EICAR_"}
+ ];
+ }
+ avast {
+ attachments_only = false;
+ symbol = "AVAST_VIRUS";
+ type = "avast";
+ servers = "127.0.0.1:{= env.PORT_AVAST =}";
+ }
+}
+
+multimap {
+ DNSBL_MAP {
+ type = "dnsbl";
+ map = "rspamd.com";
+ }
+ IP_MAP {
+ type = "ip";
+ map = "{= env.TESTDIR =}/configs/maps/ip.list";
+ }
+ FROM_MAP {
+ type = "from";
+ filter = "email:domain";
+ map = "{= env.TESTDIR =}/configs/maps/domains.list";
+ }
+ FREEMAIL_CC {
+ type = "header";
+ header = "Cc";
+ filter = "email:domain";
+ map = "{= env.TESTDIR =}/configs/maps/domains.list.2";
+ score = 1.0;
+ }
+ REGEXP_MAP {
+ type = "from";
+ filter = "email:addr";
+ regexp = true;
+ map = "{= env.TESTDIR =}/configs/maps/regexp.list";
+ }
+ DEPS_MAP {
+ type = "from";
+ filter = "email:addr";
+ regexp = true;
+ map = "{= env.TESTDIR =}/configs/maps/regexp.list";
+ require_symbols = "(R_SPF_ALLOW|R_SPF_DNSFAIL) & REGEXP_MAP & !FROM_MAP";
+ }
+ RCPT_DOMAIN {
+ type = "rcpt";
+ filter = "email:domain";
+ map = "{= env.TESTDIR =}/configs/maps/domains.list";
+ }
+ RCPT_USER {
+ type = "rcpt";
+ filter = "email:user";
+ map = "{= env.TESTDIR =}/configs/maps/users.list";
+ }
+ RCPT_MAP {
+ type = "rcpt";
+ filter = "email:addr";
+ symbols = ["SYM1"];
+ map = "{= env.TESTDIR =}/configs/maps/multiple.list";
+ score = 1.0;
+ }
+ RCPT_MAP_NOMULTISYM {
+ type = "rcpt";
+ filter = "email:addr";
+ disable_multisymbol = true;
+ map = "{= env.TESTDIR =}/configs/maps/multiple.list";
+ score = 1.0;
+ }
+ HEADER_MAP {
+ type = "header";
+ header = "To";
+ filter = "email:name";
+ map = "{= env.TESTDIR =}/configs/maps/utf.list";
+ regexp = true;
+ }
+ HOSTNAME_MAP {
+ type = "hostname";
+ map = "{= env.TESTDIR =}/configs/maps/domains.list";
+ }
+ HOSTNAME_TOP_MAP {
+ type = "hostname";
+ filter = "top";
+ map = "{= env.TESTDIR =}/configs/maps/top.list";
+ }
+ CDB_HOSTNAME {
+ type = "hostname";
+ map = "cdb://{= env.TESTDIR =}/configs/maps/domains.cdb";
+ }
+ REDIS_HOSTNAME {
+ type = "hostname";
+ map = "redis://hostname";
+ }
+ REDIS_HOSTNAME_EXPANSION {
+ type = "hostname";
+ map = "redis://${ip}.${principal_recipient_domain}";
+ }
+ REDIS_IPADDR {
+ type = "ip";
+ map = "redis://ipaddr";
+ }
+ REDIS_FROMADDR {
+ type = "from";
+ filter = "email:addr";
+ map = "redis://emailaddr";
+ }
+ REDIS_URL_TLD {
+ type = "url";
+ map = "redis://hostname";
+ filter = "tld";
+ }
+ REDIS_URL_RE_FULL {
+ type = "url";
+ map = "redis://fullurlre";
+ filter = "full:regexp:/(html)$/";
+ }
+ REDIS_URL_FULL {
+ type = "url";
+ map = "redis://fullurl";
+ filter = "full";
+ }
+ REDIS_URL_PHISHED {
+ type = "url";
+ map = "redis://phishedurl";
+ filter = "is_phished";
+ }
+ REDIS_URL_RE_TLD {
+ type = "url";
+ map = "redis://tldre";
+ filter = "tld:regexp:/(net)$/";
+ }
+ REDIS_URL_RE_PLAIN {
+ type = "url";
+ map = "redis://urlre";
+ filter = "regexp:/^(www)/";
+ }
+ REDIS_URL_NOFILTER {
+ type = "url";
+ map = "redis://urlnofilter";
+ }
+ REDIS_COUNTRY {
+ type = "country";
+ map = "redis://cc";
+ }
+ REDIS_ASN {
+ type = "asn";
+ map = "redis://asn";
+ }
+ REDIS_ASN_FILTERED {
+ type = "mempool";
+ variable = "asn";
+ map = "redis://asn";
+ filter = "regexp:/^([0-9]).*/";
+ }
+ RCVD_TEST_01 {
+ type = "received";
+ max_pos = 1;
+ map = "{= env.TESTDIR =}/configs/maps/rcvd.list";
+ }
+ RCVD_TEST_02 {
+ type = "received";
+ min_pos = -1;
+ map = "{= env.TESTDIR =}/configs/maps/rcvd.list";
+ }
+ RCVD_TEST_REDIS_01 {
+ type = "received";
+ map = "redis://RCVD_TEST";
+ }
+ RCVD_AUTHED_ONE {
+ type = "received";
+ map = "{= env.TESTDIR =}/configs/maps/rcvd2.list";
+ flags = ["authenticated"];
+ nflags = ["ssl"];
+ }
+ RCVD_AUTHED_TWO {
+ type = "received";
+ map = "{= env.TESTDIR =}/configs/maps/rcvd2.list";
+ flags = ["authenticated", "ssl"];
+ }
+ COMBINED_MAP_AND {
+ type = "combined";
+ rules {
+ ip = {
+ type = "radix";
+ map = "{= env.TESTDIR =}/configs/maps/ip.list";
+ selector = "ip";
+ }
+ from {
+ map = "{= env.TESTDIR =}/configs/maps/domains.list";
+ selector = "from:domain";
+ }
+ }
+ expression = "from & ip";
+ score = 10;
+ action = "no action"
+ }
+ COMBINED_MAP_OR {
+ type = "combined";
+ rules {
+ ip = {
+ type = "radix";
+ map = "{= env.TESTDIR =}/configs/maps/ip.list";
+ selector = "ip";
+ }
+ from {
+ map = "{= env.TESTDIR =}/configs/maps/domains.list";
+ selector = "from:domain";
+ }
+ }
+ expression = "from || ip"
+ }
+
+ EXTERNAL_MULTIMAP {
+ type = "hostname";
+ filter = "top";
+ map = {
+ external = true;
+ backend = "http://127.0.0.1:18080/map-query",
+ method = "query",
+ }
+ }
+
+ DYN_MULTIMAP {
+ type = "hostname";
+ map = "{= env.TESTDIR =}/configs/maps/dynamic_symbols.map";
+ dynamic_symbols = true;
+ }
+}
+
+rbl {
+ rbls {
+ fake {
+ from = true;
+ ipv4 = true;
+ ipv6 = true;
+ rbl = "fake.rbl";
+ symbol = "FAKE_RBL_UNKNOWN";
+ received = true;
+ symbols_prefixes = {
+ received = 'FAKE_RECEIVED_RBL',
+ from = 'FAKE_RBL',
+ }
+ unknown = true;
+ returncodes_matcher = "regexp";
+ returncodes = {
+ "CODE_2" = '^127\.0\.0\.2$';
+ "CODE_3" = '^127\.0\.0\.3$';
+ }
+ }
+ fake_whitelist {
+ from = true;
+ ipv4 = true;
+ ipv6 = true;
+ received = true;
+ is_whitelist = true;
+ rbl = "fake.wl";
+ symbol = "FAKE_WL_RBL_UNKNOWN";
+ unknown = true;
+ #returncodes_matcher = "luapattern";
+ returncodes = {
+ "FAKE_WL_RBL_CODE_2" = "127%.0%.0%.2";
+ "FAKE_WL_RBL_CODE_3" = "127%.0%.0%.3";
+ }
+ }
+ RSPAMD_EMAILBL {
+ rbl = "test8.uribl";
+ url_compose_map = "{= env.TESTDIR =}/configs/maps/url_compose_map_for_mails.list";
+ ignore_defaults = true;
+ emails = true;
+ emails_domainonly = true
+ returncodes_matcher = "radix";
+ returncodes = {
+ RSPAMD_EMAILBL = "127.0.0.2/32";
+ }
+ }
+ URIBL_NUMERIC {
+ checks = ["numeric_urls"];
+ rbl = "test9.uribl";
+ }
+ URIBL_NUMERIC_IMAGES {
+ checks = ["numeric_urls"];
+ images = true;
+ rbl = "test9.uribl";
+ }
+ UNKNOWN_URIBL_NUMERIC_CONTENT {
+ checks = ["numeric_urls"];
+ content_urls = true;
+ rbl = "test9.uribl";
+ returncodes_matcher = "glob";
+ returncodes = {
+ URIBL_NUMERIC_CONTENT = "*.*.*.*";
+ }
+ }
+ URIBL_NUMERIC_EVERYTHING {
+ checks = ["numeric_urls"];
+ images = true;
+ content_urls = true;
+ rbl = "test9.uribl";
+ exclude_local = false;
+ }
+ URIBL_NOCONTENT {
+ rbl = "test9.uribl";
+ ignore_defaults = true;
+ urls = true;
+ }
+ URIBL_WITHCONTENT {
+ rbl = "test9.uribl";
+ ignore_defaults = true;
+ urls = true;
+ content_urls = true;
+ }
+ URIBL_CONTENTONLY {
+ rbl = "test9.uribl";
+ ignore_defaults = true;
+ content_urls = true;
+ no_ip = true;
+ }
+ RBL_SELECTOR_SINGLE {
+ rbl = "test9.uribl";
+ ignore_defaults = true;
+ selector = "helo()";
+ }
+ RBL_SELECTOR_MULTIPLE {
+ rbl = "test9.uribl";
+ ignore_defaults = true;
+ selector = {
+ sel_from = "from('smtp'):domain";
+ sel_helo = "helo()";
+ }
+ }
+ }
+}
+
+surbl {
+ "whitelist" = [
+ "rspamd-test.com"
+ ];
+ rules {
+ "RSPAMD_URIBL" {
+ suffix = "test.uribl";
+ check_dkim = true;
+ check_emails = true;
+ images = false;
+ process_script =<<EOD
+function(url, suffix)
+ local cr = require "rspamd_cryptobox_hash"
+ local h = cr.create(url):base32():sub(1, 32)
+ return string.format("%s.%s", h, suffix)
+end
+EOD;
+ }
+ "DBL" {
+ suffix = "test2.uribl";
+ no_ip = true;
+ check_emails = true;
+ check_dkim = true;
+ ips = {
+ # spam domain
+ DBL_SPAM = "127.0.1.2";
+ # phish domain
+ DBL_PHISH = "127.0.1.4";
+ }
+ }
+ "URIBL_MULTI" {
+ suffix = "test3.uribl";
+ check_dkim = true;
+ check_emails = true;
+ bits {
+ URIBL_BLOCKED = 1;
+ URIBL_BLACK = 2;
+ URIBL_GREY = 4;
+ URIBL_RED = 8;
+ }
+ }
+ "SPAMHAUS_ZEN_URIBL" {
+ suffix = "test4.uribl";
+ resolve_ip = true;
+ check_emails = true;
+ ips {
+ URIBL_SBL = "127.0.0.2";
+ URIBL_SBL_CSS = "127.0.0.3";
+ URIBL_XBL = ["127.0.0.4", "127.0.0.5", "127.0.0.6", "127.0.0.7"];
+ URIBL_PBL = ["127.0.0.10", "127.0.0.11"];
+ URIBL_DROP = "127.0.0.9";
+ }
+ }
+ "RSPAMD_URIBL_IMAGES" {
+ suffix = "test.uribl";
+ check_dkim = true;
+ check_emails = false;
+ images = true;
+ process_script =<<EOD
+ function(url, suffix)
+ local cr = require "rspamd_cryptobox_hash"
+ local h = cr.create(url):base32():sub(1, 32)
+ return string.format("%s.%s", h, suffix)
+end
+EOD;
+ }
+ "BAD_SUBDOMAIN" {
+ suffix = "test7.uribl";
+ url_compose_map = "{= env.TESTDIR =}/configs/maps/url_compose_map.list";
+ check_dkim = true;
+ check_emails = false;
+ }
+ }
+}
diff --git a/test/functional/configs/merged.conf b/test/functional/configs/merged.conf
new file mode 100644
index 0000000..0718d02
--- /dev/null
+++ b/test/functional/configs/merged.conf
@@ -0,0 +1,39 @@
+.include "{= env.TESTDIR =}/../../conf/rspamd.conf"
+
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua"
+
+# 101_lua
+lua = "{= env.TESTDIR =}/lua/conditions.lua"
+lua = "{= env.TESTDIR =}/lua/hashes.lua"
+lua = "{= env.TESTDIR =}/lua/maps_kv.lua"
+lua = "{= env.TESTDIR =}/lua/option_order.lua"
+lua = "{= env.TESTDIR =}/lua/recipients.lua"
+lua = "{= env.TESTDIR =}/lua/remove_result.lua"
+lua = "{= env.TESTDIR =}/lua/tlds.lua"
+
+# 104_get_from
+lua = "{= env.TESTDIR =}/lua/get_from.lua"
+
+# 240_redis
+lua = "{= env.TESTDIR =}/lua/redis.lua"
+
+# 250_dns
+lua = "{= env.TESTDIR =}/lua/dns.lua"
+
+# 270_selector
+lua = "{= env.TESTDIR =}/lua/selector_test.lua"
+
+# 281_fnames
+lua = "{= env.TESTDIR =}/lua/test_fname.lua"
+
+# 310_udp
+lua = "{= env.TESTDIR =}/lua/udp.lua"
+
+# 350_magic
+lua = "{= env.TESTDIR =}/lua/magic.lua"
+
+# 380_external_relay
+lua = "{= env.TESTDIR =}/lua/external_relay.lua"
+
+.include(priority=1,duplicate=merge) "{= env.TESTDIR =}/configs/merged-local.conf"
+.include(priority=2,duplicate=replace) "{= env.TESTDIR =}/configs/merged-override.conf"
diff --git a/test/functional/configs/milter.conf b/test/functional/configs/milter.conf
new file mode 100644
index 0000000..dc623c8
--- /dev/null
+++ b/test/functional/configs/milter.conf
@@ -0,0 +1,61 @@
+options = {
+ filters = ["spf", "dkim", "regexp"]
+ url_tld = "{= env.URL_TLD =}"
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+ lua_path = "{= env.INSTALLROOT =}/share/rspamd/lib/?.lua";
+ gtube_patterns = "all";
+ dns {
+ nameserver = ["8.8.8.8", "8.8.4.4"];
+ retransmits = 10;
+ timeout = 2s;
+ }
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ }
+ unknown_weight = 1
+}
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ task_timeout = 60s;
+}
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl"
+}
+worker {
+ type = "rspamd_proxy";
+ count = 1;
+ timeout = 120;
+ upstream {
+ local {
+ hosts = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}";
+ default = true;
+ }
+ }
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_PROXY =}";
+ milter = true;
+}
+modules {
+ path = "{= env.TESTDIR =}/../../src/plugins/lua/"
+}
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
+lua = "{= env.INSTALLROOT =}/share/rspamd/rules/rspamd.lua"
+lua = "{= env.TESTDIR =}/lua/params.lua"
+milter_headers {
+ extended_spam_headers = true;
+ skip_local = false;
+ skip_authenticated = false;
+}
diff --git a/test/functional/configs/neural.conf b/test/functional/configs/neural.conf
new file mode 100644
index 0000000..62ff856
--- /dev/null
+++ b/test/functional/configs/neural.conf
@@ -0,0 +1,83 @@
+options = {
+ url_tld = "{= env.URL_TLD =}"
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+ lua_path = "{= env.INSTALLROOT =}/share/rspamd/lib/?.lua"
+ filters = [];
+ explicit_modules = ["settings"];
+}
+
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+ log_usec = true;
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ add_header = 50500,
+ }
+ unknown_weight = 1
+}
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ task_timeout = 10s;
+}
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl"
+}
+
+modules {
+ path = "{= env.TESTDIR =}/../../src/plugins/lua/"
+}
+
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
+
+neural {
+ rules {
+ SHORT {
+ train {
+ learning_rate = 0.001;
+ max_usages = 2;
+ spam_score = 1;
+ ham_score = -1;
+ max_trains = 10;
+ max_iterations = 250;
+ }
+ symbol_spam = "NEURAL_SPAM_SHORT";
+ symbol_ham = "NEURAL_HAM_SHORT";
+ ann_expire = 86400;
+ watch_interval = 0.5;
+ }
+ SHORT_PCA {
+ train {
+ learning_rate = 0.001;
+ max_usages = 2;
+ spam_score = 1;
+ ham_score = -1;
+ max_trains = 10;
+ max_iterations = 250;
+ }
+ symbol_spam = "NEURAL_SPAM_SHORT_PCA";
+ symbol_ham = "NEURAL_HAM_SHORT_PCA";
+ ann_expire = 86400;
+ watch_interval = 0.5;
+ max_inputs = 10;
+ }
+ }
+ allow_local = true;
+
+}
+redis {
+ servers = "{= env.REDIS_ADDR =}:{= env.REDIS_PORT =}";
+ expand_keys = true;
+}
+
+lua = "{= env.TESTDIR =}/lua/neural.lua";
diff --git a/test/functional/configs/neural_noauto.conf b/test/functional/configs/neural_noauto.conf
new file mode 100644
index 0000000..e228da8
--- /dev/null
+++ b/test/functional/configs/neural_noauto.conf
@@ -0,0 +1,85 @@
+options = {
+ url_tld = "{= env.URL_TLD =}"
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+ lua_path = "{= env.INSTALLROOT =}/share/rspamd/lib/?.lua"
+ filters = [];
+ explicit_modules = ["settings"];
+}
+
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+ log_usec = true;
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ add_header = 50500,
+ }
+ unknown_weight = 1
+}
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ task_timeout = 10s;
+}
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl"
+}
+
+modules {
+ path = "{= env.TESTDIR =}/../../src/plugins/lua/"
+}
+
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
+
+neural {
+ rules {
+ SHORT {
+ train {
+ learning_rate = 0.001;
+ max_usages = 2;
+ spam_score = 1;
+ ham_score = -1;
+ max_trains = 10;
+ max_iterations = 250;
+ store_pool_only = true;
+ }
+ symbol_spam = "NEURAL_SPAM_SHORT";
+ symbol_ham = "NEURAL_HAM_SHORT";
+ ann_expire = 86400;
+ watch_interval = 0.5;
+ }
+ SHORT_PCA {
+ train {
+ learning_rate = 0.001;
+ max_usages = 2;
+ spam_score = 1;
+ ham_score = -1;
+ max_trains = 10;
+ max_iterations = 250;
+ store_pool_only = true;
+ }
+ symbol_spam = "NEURAL_SPAM_SHORT_PCA";
+ symbol_ham = "NEURAL_HAM_SHORT_PCA";
+ ann_expire = 86400;
+ watch_interval = 0.5;
+ max_inputs = 2;
+ }
+ }
+ allow_local = true;
+
+}
+redis {
+ servers = "{= env.REDIS_ADDR =}:{= env.REDIS_PORT =}";
+ expand_keys = true;
+}
+
+lua = "{= env.TESTDIR =}/lua/neural.lua";
diff --git a/test/functional/configs/nginx.conf b/test/functional/configs/nginx.conf
new file mode 100644
index 0000000..9a27f49
--- /dev/null
+++ b/test/functional/configs/nginx.conf
@@ -0,0 +1,20 @@
+events {
+}
+worker_processes 1;
+pid ${RSPAMD_TMPDIR}/nginx.pid;
+error_log ${RSPAMD_TMPDIR}/error.log;
+http {
+ default_type application/octet-stream;
+ sendfile on;
+
+ server {
+ # no need for root privileges
+ listen ${NGINX_ADDR}:${NGINX_PORT};
+ server_name localhost;
+
+ location / {
+ root ${RSPAMD_TMPDIR};
+ autoindex on;
+ }
+ }
+}
diff --git a/test/functional/configs/p0f.conf b/test/functional/configs/p0f.conf
new file mode 100644
index 0000000..cb492f9
--- /dev/null
+++ b/test/functional/configs/p0f.conf
@@ -0,0 +1,13 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+redis {
+ servers = "{= env.REDIS_ADDR =}:{= env.REDIS_PORT =}";
+}
+p0f {
+ socket = "{= env.P0F_SOCKET =}";
+ patterns {
+ WINDOWS = '^Windows.*';
+ ETHER = '^Ethernet.*';
+ DISTGE10 = '^distance:[0-9]{2}$';
+ }
+}
diff --git a/test/functional/configs/password.conf b/test/functional/configs/password.conf
new file mode 100644
index 0000000..27b88ee
--- /dev/null
+++ b/test/functional/configs/password.conf
@@ -0,0 +1,45 @@
+options = {
+ filters = ["spf", "dkim", "regexp"]
+ url_tld = "{= env.TESTDIR =}/../lua/unit/test_tld.dat"
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+ dns {
+ retransmits = 10;
+ timeout = 2s;
+ }
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ }
+ unknown_weight = 1
+}
+
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ keypair {
+ pubkey = "{= env.KEY_PUB1 =}";
+ privkey = "{= env.KEY_PVT1 =}";
+ }
+ task_timeout = 60s;
+}
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ keypair {
+ pubkey = "{= env.KEY_PUB1 =}";
+ privkey = "{= env.KEY_PVT1 =}";
+ }
+ password = {= env.PASSWORD =};
+ enable_password = {= env.ENABLE_PASSWORD =};
+ stats_path = "{= env.TMPDIR =}/stats.ucl";
+}
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
diff --git a/test/functional/configs/plugins.conf b/test/functional/configs/plugins.conf
new file mode 100644
index 0000000..22190a3
--- /dev/null
+++ b/test/functional/configs/plugins.conf
@@ -0,0 +1,770 @@
+options = {
+ filters = [ "dkim", "regexp"]
+ url_tld = "{= env.URL_TLD =}"
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+ lua_path = "{= env.INSTALLROOT =}/share/rspamd/lib/?.lua"
+ explicit_modules = ["settings", "bayes_expiry"];
+ dns {
+ nameserver = ["8.8.8.8", "8.8.4.4"];
+ retransmits = 10;
+ timeout = 2s;
+ fake_records = [{ # ed25519
+ name = "test._domainkey.example.com";
+ type = txt;
+ replies = ["k=ed25519; p=yi50DjK5O9pqbFpNHklsv9lqaS0ArSYu02qp1S0DW1Y="];
+ },
+ {
+ name = "brisbane._domainkey.football.example.com";
+ type = txt;
+ replies = ["v=DKIM1; k=ed25519; p=11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo="];
+ },
+ {
+ name = "test._domainkey.football.example.com";
+ type = txt;
+ replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDkHlOQoBTzWRiGs5V6NpP3idY6Wk08a5qhdR6wy5bdOKb2jLQiY/J16JYi0Qvx/byYzCNb3W91y3FutACDfzwQ/BC/e/8uBsCR+yz1Lxj+PL6lHvqMKrM3rG4hstT5QjvHO9PzoxZyVYLzBfO2EeC3Ip3G+2kryOTIKT+l/K4w3QIDAQAB"],
+ },
+ {
+ name = "dkim._domainkey.cacophony.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXtxBE5IiNRMcq2/lc2zErfdCvDFyQNBnMjbOjBQrPST2k4fdGbtpe5Iu5uS01Met+dAEf94XL8I0hwmYw+n70PP834zfJGi2egwGqrakpaWsCDPvIJZLkxJCJKQRA/zrQ622uEXdvYixVbsEGVw7U4wAGSmT5rU2eU1y63AlOlQIDAQAB"];
+ },
+ {
+ name = "eddsa._domainkey.cacophony.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=ed25519; p=+nU+aC33ICeS4zx8VUjFYCtxj0fRbHWQn2gP2hTkm9w="];
+ },
+ {
+ name = "dkim._domainkey.invalid.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDEEXmNGQq7PUrr9Mg4UakTFHgXBCy2DOztkrZm+0OrVWtiRzGluxBkbOWTBwuU3/Yw97yTphBMQxzWFN603/f/KPAQcF/Lc1l+6kmIBBxNXjjGuOK/3PYKZVntUdKmqcQBYfnHdzH2Tohbuyx1a7xqnv6VSChqQrZU4CwkeT3+eQIDAQAB"];
+ },
+ {
+ name = "eddsa._domainkey.invalid.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=ed25519; p=Wkkrp5DJTvknDMGWYv8vm3p3sZjiQp03LZo80RregY8="];
+ },
+ {
+ name = "dkim._domainkey.rspamd.com",
+ type = "txt";
+ replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCd/XhZBEGGAss48lEuMmwZv9lOFf6FTllBiQ3sPhdTpDdIPaW9TInW7iYnYD/bXHeVxYAyD/sKhYk6+qGBRu10rEi+iyPvLCIED+Boq0tEQosuKuV6Fjoomb+QhZY9KdjyZTjsrFPZ+wCkUY/30uTmpX2SwSqyxxlK0pUIsRgMAQIDAQAB"];
+ },
+ {
+ name = "_dmarc.rspamd.com",
+ type = "txt";
+ rcode = 'norec';
+ },
+ {
+ name = "dkim._domainkey.highsecure.ru",
+ type = "txt";
+ replies = ["p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDK4ZQYky30GH0Ak9OQ1fv3IdFNbpOtpa4S/PR20ZLgPXfd/FCA//ztUmu7kHlELI+/+4f8W+xX0oZlOc/cFxhopRjXZMlSsQqmWOZ40/GxWFBtcqafKu78FCqO7URqZUmMCM5Jlp4zt/yzH3dbYNG3i5PVlB5QtQnZvY+dvBL3dwIDAQAB"];
+ },
+ {
+ name = "_dmarc.cacophony.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=none; sp=reject"];
+ },
+ {
+ name = "_dmarc.my.mom.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=reject"];
+ },
+ {
+ name = "example.net",
+ type = "txt";
+ replies = ["v=spf1 -all"];
+ },
+ {
+ name = "fail4.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 redirect=asdfsfewewrredfs"];
+ },
+ {
+ name = "_dmarc.reject.cacophony.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=reject"];
+ },
+ {
+ name = "spf.cacophony.za.org",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.4.4 -all"];
+ },
+ {
+ name = "fail7.org.org.za",
+ type = "a";
+ rcode = 'norec';
+ },
+ {
+ name = "fail6.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 mx -all"];
+ },
+ {
+ name = "fail6.org.org.za",
+ type = "mx";
+ rcode = 'norec';
+ },
+ {
+ name = "fail7.org.org.za",
+ type = "aaaa";
+ rcode = 'norec';
+ },
+ {
+ name = "_dmarc.quarantine.cacophony.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=quarantine"];
+ },
+ {
+ name = "_dmarc.yo.mom.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=reject; aspf=s; adkim=s;"];
+ },
+ {
+ name = "yo.mom.za.org",
+ type = "txt";
+ replies = ["v=spf1 ip4:37.48.67.26 -all"];
+ },
+ {
+ name = "testdkim._domainkey.mom.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC3v4VPE1QMHUzsMRbC8VzXNq82mDjiv9Gi1NB/YYC+vIYZT+sE/Uxnr0Clk8C2jgzEr3jcxgQEWZfMtEEg/EfEJvh4SrXWv9c0gw1EEfxKxX9i+r8yBQtc/EWospWVDkhF2lAvQAK1lV1ZiU7psJ6fh1CI39uZyWdAktZzWLf0zQIDAQAB"];
+ },
+ {
+ name = "_dmarc.rspamd.tk",
+ type = "txt";
+ replies = ["bio=a263adeab8acdcdb8b89e127b67d696061fdfbee"];
+ },
+ # For unknown dkim tags
+ {
+ name = "18457.62be233b.k2206._domainkey.taugh.com";
+ type = "txt";
+ replies = ["v=DKIM1; h=sha256; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvLwxxLlZJ+uU3SctsQ2pjq6K0xyjhmvlIfWWGxRLxVpwyLPaNCUJNDnP0d0Fk+HQXub1T6R22T79L9yGQEZuHrD8MxchBKO++ywk7HOd1LvhweeKPUiXD03Dda54svQ2hnT7MQBFU92CWXoD0BRs9QPMyCC2QiZk0IwB1rK9sClOCjOdOH1mT1Oz8XObUqT3Nd6Oi7LSppyoMzYg4TEkmyiz0c34uiXOkqZwonf2V6+s/v/1/fz4dH6hgnn2cHLnjGmzmiKQgs8lJNMjhfI4sIzg26xNb4wCTVlggP6zDr7lxe9DZuTRcP5/tSI6ihDO/zc+7HmG83EIkqgqllI6IQIDAQAB ; n=Signing=20key=20at=20https://www.iecc.com/dkimkeys/k2206 ;"];
+ },
+ {
+ name = "fail2.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.4.4 include:www.dnssec-failed.org -all"];
+ },
+ {
+ name = "fail3.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 include:total.barf -all"];
+ },
+ {
+ name = "mom.za.org",
+ type = "txt";
+ replies = ["v=spf1 ip4:37.48.67.26 -all"];
+ },
+ {
+ name = "testdkim._domainkey.asdf.rspamd.tk", # testdkim._domainkey.asdf.rspamd.tk is an alias for rspamd.tk
+ type = "txt";
+ replies = ["bio=a263adeab8acdcdb8b89e127b67d696061fdfbee"];
+ },
+ {
+ name = "testdkim._domainkey.rspamd.tk", # testdkim._domainkey.rspamd.tk is an alias for rspamd.tk
+ type = "txt";
+ replies = ["bio=a263adeab8acdcdb8b89e127b67d696061fdfbee"];
+ },
+ {
+ name = "pass1.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 include:pass2.org.org.za -all"];
+ },
+ {
+ name = "95.142.99.88.in-addr.arpa",
+ type = "ptr";
+ replies = ["mail.highsecure.ru"];
+ },
+ {
+ name = "mail.highsecure.ru",
+ type = "a";
+ replies = ["88.99.142.95"];
+ },
+ {
+ name = "mail.highsecure.ru",
+ type = "aaaa";
+ rcode = 'norec';
+ },
+ {
+ name = "1.0.66.128.in-addr.arpa",
+ type = "ptr";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "182.216.85.209.in-addr.arpa",
+ type = "ptr";
+ replies = ["mail-qt0-f182.google.com"];
+ },
+ {
+ name = "crazyspf.cacophony.za.org",
+ type = "txt";
+ replies = ["v=spf1 ptr:cacophony.za.org ptr:rspamd.com ptr:yahoo.com ptr:yahoo.net ptr:highsecure.ru -all"];
+ },
+ {
+ name = "pass2.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 -all"];
+ },
+ {
+ name = "_dmarc.yoni.za.org",
+ type = "txt";
+ replies = ["v=DMARC1; p=reject; sp=none;"];
+ },
+ {
+ name = "fail10.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 redirect=fail5.org.org.za"];
+ },
+ {
+ name = "fail11.org.org.za",
+ type = "txt";
+ replies = ["v=sPF1 ip4:8.8.8.8 -all"];
+ },
+ {
+ name = "fail5.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 OMGBARF"];
+ },
+ {
+ name = "fail7.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 a -all"];
+ },
+ {
+ name = "trusted.com",
+ type = "txt";
+ replies = ["v=spf1 ip4:192.168.1.1"];
+ },
+ {
+ name = "external.com",
+ type = "txt";
+ replies = ["v=spf1 ip4:37.48.67.26"];
+ },
+ {
+ name = "co.za",
+ type = "txt";
+ rcode = 'norec';
+ },
+ {
+ name = "testdkim1._domainkey.yoni.za.org",
+ type = "txt";
+ replies = ["v=DKIM1; k=rsa; p=BARF"];
+ },
+ {
+ name = "_dmarc.yoni.za.net",
+ type = "txt";
+ replies = ["v=DMARC1; p=none; sp=quarantine"];
+ },
+ {
+ name = "za",
+ type = "txt";
+ replies = ["Top-level domain for South Africa"];
+ },
+ {
+ name = "_dmarc.foo.yoni.za.org",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "_dmarc.foo.cacophony.za.org",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "_dmarc.foo.yoni.za.net",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "_dmarc.dnssec-failed.org",
+ type = "txt";
+ rcode = 'timeout';
+ },
+ {
+ name = "_dmarc.example.com",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "_dmarc.zero_pct.com",
+ type = "txt";
+ replies = ["v=DMARC1; p=reject; sp=quarantine; pct=0"];
+ },
+ {
+ name = "example.com",
+ type = "txt";
+ replies = ["$Id: example.com 4415 2015-08-24 20:12:23Z davids $", "v=spf1 -all"];
+ },
+ {
+ name = "example.com",
+ type = "a";
+ replies = ["93.184.216.34"];
+ },
+ {
+ name = "testdkim1._domainkey.dnssec-failed.org",
+ type = "txt";
+ rcode = 'timeout';
+ },
+ {
+ name = "total.barf",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "_dmarc.foo.cacophony.za.org",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "zzzzaaaa",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "asdfsfewewrredfs",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "95.142.99.88.asn.rspamd.com",
+ type = "txt";
+ replies = ["24940|88.99.0.0/16|DE|ripencc|"];
+ },
+ {
+ name = "2.a.d.1.1.d.e.f.f.f.0.0.4.5.0.5.d.6.2.0.6.b.a.a.8.c.7.0.1.0.a.2.asn6.rspamd.com",
+ type = "txt";
+ replies = ["20857|2a01:7c8::/32|NL|ripencc|"];
+ },
+ {
+ name = "2.a.d.1.1.d.e.f.f.f.0.0.4.5.0.5.d.6.2.0.6.b.a.a.8.c.7.0.1.0.a.2.rspamd.com",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "2.a.d.1.1.d.e.f.f.f.0.0.4.5.0.5.d.6.2.0.6.b.a.a.8.c.7.0.1.0.a.2.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "rspamd.com",
+ type = "txt";
+ replies = ["v=spf1 mx -all"];
+ },
+ {
+ name = "rspamd.com",
+ type = "mx";
+ replies = ["10 mail.highsecure.ru"];
+ },
+ {
+ name = "95.142.99.88.rspamd.com",
+ type = "a";
+ rcode = 'norec';
+ },
+ {
+ name = "95.142.99.88.rspamd.com",
+ type = "aaaa";
+ rcode = 'norec';
+ },
+ {
+ name = "2.0.0.127.rspamd.com",
+ type = "a";
+ replies = ["127.0.0.1"];
+ },
+ {
+ name = "8.8.8.8.asn.rspamd.com",
+ type = "txt";
+ replies = ["15169|8.8.8.0/24|US|arin|"];
+ },
+ {
+ name = "8.8.8.8.asn.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "10.0.1.10.asn.rspamd.com",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "10.0.1.10.asn.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "10.0.1.11.asn.rspamd.com",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "10.0.1.11.asn.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "test.com",
+ type = "txt";
+ replies = [""];
+ },
+ {
+ name = "other.com",
+ type = "txt";
+ rcode = 'norec';
+ },
+ {
+ name = "bob",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "mail.highsecure.ru",
+ type = "aaaa";
+ rcode = 'norec';
+ },
+ {
+ name = "mail.highsecure.ru",
+ type = "a";
+ replies = ["88.99.142.95"];
+ },
+ {
+ name = "4.3.2.1.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "4.3.2.1.asn.rspamd.com",
+ type = "txt";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "1.0.0.127.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "114.47.228.46.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "114.47.228.46.asn.rspamd.com",
+ type = "txt";
+ replies = ["34010|46.228.40.0/21|GB|ripencc|"];
+ },
+ {
+ name = "10.0.1.10.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "10.0.1.11.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "other.org",
+ type = "txt";
+ rcode = 'norec';
+ },
+ {
+ name = "8.8.8.8.rspamd.com",
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "rspamd.tk",
+ type = "txt";
+ replies = ["bio=a263adeab8acdcdb8b89e127b67d696061fdfbee"];
+ },
+ {
+ name = "fail1.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 redirect=www.dnssec-failed.org"];
+ },
+ {
+ name = "www.dnssec-failed.org",
+ type = "txt";
+ rcode = 'timeout';
+ },
+ {
+ name = "www.dnssec-failed.org",
+ type = "mx";
+ rcode = 'timeout';
+ },
+ {
+ name = "www.dnssec-failed.org",
+ type = "a";
+ rcode = 'timeout';
+ },
+ {
+ name = "www.dnssec-failed.org",
+ type = "aaaa";
+ rcode = 'norec';
+ },
+ {
+ name = "cacophony.za.org",
+ type = "txt";
+ replies = ["v=spf1 redirect=asdfsfewewrredfs"];
+ },
+ {
+ name = "fail9.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 mx:www.dnssec-failed.org -all"];
+ },
+ {
+ name = "fail8.org.org.za",
+ type = "txt";
+ replies = ["v=spf1 ip4:8.8.8.8 a:www.dnssec-failed.org -all"];
+ },
+ {
+ name = "1.2.3.4.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "2.2.3.4.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.10"];
+ },
+ {
+ name = "3.2.3.4.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.2", "127.0.0.3"];
+ },
+ {
+ name = "4.2.3.4.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "4.2.3.4.fake.wl";
+ type = "a";
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "4.3.2.1.fake.rbl";
+ type = "a";
+ rcode = 'nxdomain';
+ },
+ {
+ name = "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "131.193.18.151.fake.rbl";
+ type = "a";
+ replies = ["127.0.0.3"];
+ },
+ # SURBL tests
+ {
+ name = "rciuosbadgpq6b5wt436nhgnwzmfh9w9.test.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ # testtest.com
+ name = "rcf1ecxtxrrpfncqzsdaiezjkf7f1rzz.test.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "jhcszdsmo3wuj5mp8t38kdisdmr3ib3q.test.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "g9ifs3q39oh5jwru94cj7ffaqd6rfyq6.test.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "xn--80arbjktj.xn--p1ai.test2.uribl";
+ type = a;
+ replies = ["127.0.1.2"];
+ },
+ {
+ name = "мойÑайт.рф.test2.uribl";
+ type = a;
+ replies = ["127.0.1.2"];
+ },
+ {
+ name = "user.emailbl.com.test5.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "user.subdomain.emailbl.com.test5.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "baddomain.com.test6.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "user.subdomain.baddomain.com.test5.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "example.com.test2.uribl";
+ type = a;
+ replies = ["127.0.1.2"];
+ },
+ {
+ name = "user.example.com.test2.uribl";
+ type = a;
+ replies = ["127.0.1.5"];
+ },
+ {
+ name = "example.net.test2.uribl";
+ type = a;
+ replies = ["127.0.1.4"];
+ },
+ {
+ name = "rspamd.tk.test2.uribl";
+ type = a;
+ replies = ["127.0.1.4"];
+ },
+ {
+ name = "example.org.test3.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "example.ru.test3.uribl";
+ type = a;
+ replies = ["127.0.0.12"];
+ },
+ {
+ name = "example.ru";
+ type = a;
+ replies = ["8.8.8.8", "8.8.8.9"];
+ },
+ {
+ name = "8.8.8.8.test4.uribl";
+ type = a;
+ replies = ["127.0.0.4", "127.0.0.11"];
+ },
+ {
+ name = "uppht14nj4fsoycu3huctg9d5psx9je4.test.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "rspamd-test.com.test2.uribl";
+ type = a;
+ replies = ["127.0.1.2"];
+ },
+ {
+ name = "9.8.8.8.test4.uribl";
+ type = a;
+ replies = ["127.0.0.3"];
+ },
+ {
+ name = "4.very.dirty.sanchez.com.test7.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "clean.dirty.sanchez.com.test7.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "not.dirty.sanchez.com.test7.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "41.black.sanchez.com.test7.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "black.sanchez.com.test7.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "dirty.sanchez.com.test8.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "very.dirty.sanchez.com.test8.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "41.black.sanchez.com.test8.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "black.sanchez.com.test8.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "example.com.test9.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "example.org.test9.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ {
+ name = "8.8.8.8.test9.uribl";
+ type = a;
+ replies = ["127.0.0.2"];
+ },
+ # TODO: add IPv6 tests
+ ];
+ }
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+ log_usec = true;
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ add_header = 50500,
+ }
+ unknown_weight = 1
+}
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ task_timeout = 10s;
+}
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl"
+}
+modules {
+ path = "{= env.TESTDIR =}/../../src/plugins/lua/"
+}
+spf {}
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
+lua = "{= env.INSTALLROOT =}/share/rspamd/rules/rspamd.lua";
diff --git a/test/functional/configs/proxy.conf b/test/functional/configs/proxy.conf
new file mode 100644
index 0000000..ff31eae
--- /dev/null
+++ b/test/functional/configs/proxy.conf
@@ -0,0 +1,26 @@
+options = {
+ filters = ["spf", "dkim", "regexp"]
+ url_tld = "{= env.URL_TLD =}"
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+ lua_path = "{= env.INSTALLROOT =}/share/rspamd/lib/?.lua"
+ dns {
+ nameserver = ["8.8.8.8", "8.8.4.4"];
+ retransmits = 10;
+ timeout = 2s;
+ }
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+}
+worker "rspamd_proxy" {
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_PROXY =}";
+ upstream {
+ name = "{= env.LOCAL_ADDR =}";
+ default = yes;
+ hosts = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}";
+ }
+ count = 1;
+}
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
diff --git a/test/functional/configs/redis-server.conf b/test/functional/configs/redis-server.conf
new file mode 100644
index 0000000..0943025
--- /dev/null
+++ b/test/functional/configs/redis-server.conf
@@ -0,0 +1,7 @@
+bind ${RSPAMD_REDIS_ADDR}
+daemonize yes
+loglevel debug
+logfile ${RSPAMD_TMPDIR}/redis.log
+pidfile ${RSPAMD_TMPDIR}/redis.pid
+port ${RSPAMD_REDIS_PORT}
+dir ${RSPAMD_TMPDIR}
diff --git a/test/functional/configs/redis.conf b/test/functional/configs/redis.conf
new file mode 100644
index 0000000..8b3f0c4
--- /dev/null
+++ b/test/functional/configs/redis.conf
@@ -0,0 +1,7 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+lua = "{= env.LUA_SCRIPT =}";
+redis {
+ servers = "{= env.REDIS_ADDR =}:{= env.REDIS_PORT =}";
+ expand_keys = true;
+}
diff --git a/test/functional/configs/regexp.conf b/test/functional/configs/regexp.conf
new file mode 100644
index 0000000..06f22c9
--- /dev/null
+++ b/test/functional/configs/regexp.conf
@@ -0,0 +1,64 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+regexp {
+ SA_BODY_WORD_BOUNDARY_ON_NEWLINE {
+ re = '/\bhello\s/{sa_body}',
+ score = 0.0,
+ description = 'Test if word boundary works correctly on sa_body regexes',
+ group = 'body',
+ }
+ SA_BODY_WORD {
+ re = '/hello/{sa_body}',
+ score = 0.0,
+ description = 'Test if regex works at all in sa_body',
+ group = 'body',
+ }
+ SA_BODY_WORD_WITH_NEWLINE {
+ re = '/helloworld/{sa_body}',
+ score = 0.0,
+ description = 'Test if regex matches word separated with newline (should not work!)',
+ group = 'body',
+ }
+ SA_BODY_WORD_WITH_SPACE {
+ re = '/hello world/{sa_body}',
+ score = 0.0,
+ description = 'Test if regex matches word separated with newline (should work - newline is replaced with space)',
+ group = 'body',
+ }
+ SA_BODY_WORD_WITH_SPACE_BOUNDARIES {
+ re = '/\bhello world\b/{sa_body}',
+ score = 0.0,
+ description = 'Test if regex matches word separated with newline (should work - newline is replaced with space)',
+ group = 'body',
+ }
+ SA_BODY_WORD_WITH_SPACE_BOUNDARIES_2 {
+ re = '/\shello\sworld\s/{sa_body}',
+ score = 0.0,
+ description = 'Test if regex matches word separated with newline (should work - newline is replaced with space)',
+ group = 'body',
+ }
+ SA_BODY_WORD_WITH_SPACE_BOUNDARIES_3 {
+ re = '/\shello\sworld\sthis\s/{sa_body}',
+ score = 0.0,
+ description = 'Test if regex matches word separated with newline (should work - newline is replaced with space)',
+ group = 'body',
+ }
+ SA_BODY_WORD_WITH_SPACE_AND_DOT {
+ re = '/\bword\.\sagain\b/{sa_body}',
+ score = 0.0,
+ description = 'Test if regex matches word separated with newline (should work - newline is replaced with space)',
+ group = 'body',
+ }
+}
+
+options {
+ dynamic_conf = "{= env.TESTDIR =}/configs/dynamic.conf";
+}
+dmarc { }
+spf { }
+dkim { }
+modules {
+ path = "{= env.TESTDIR =}/../../src/plugins/lua/"
+}
+
+lua = "{= env.TESTDIR =}/lua/regex_test.lua"
diff --git a/test/functional/configs/settings.conf b/test/functional/configs/settings.conf
new file mode 100644
index 0000000..506bde1
--- /dev/null
+++ b/test/functional/configs/settings.conf
@@ -0,0 +1,117 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+lua = "{= env.LUA_SCRIPT =}";
+
+settings {
+ id_test {
+ id = "id_test";
+ apply {
+ symbols_enabled = ["SIMPLE_TEST"];
+ }
+ }
+
+ id_pre { # implicit id
+ apply {
+ symbols_enabled = ["SIMPLE_PRE"];
+ }
+ }
+
+ id_virtual {
+ apply {
+ symbols_enabled = ["SIMPLE_VIRTUAL"];
+ }
+ }
+
+ id_virtual1 {
+ apply {
+ symbols {
+ EXPLICIT_VIRTUAL1 = 10.0
+ }
+ symbols_enabled = ["DEP_REAL"];
+ }
+ }
+
+ id_virtual_group {
+ user = "test@example.com";
+ from = "test2@example.com";
+ hostname = "example.com";
+ selector = "rcpts:addr.in(test3@example.com)";
+ header = {
+ "Content-Transfer-Encoding" = "7bit";
+ "Custom-Header" = true;
+ "Mime-Version" = false;
+ }
+ request_header = {
+ "Test" = "passed";
+ }
+
+ expression = 'user || from || hostname || selector:1 || header:mime_version || header:custom_header || header:content_transfer_encoding || request_header:test'
+ apply {
+ symbols_enabled {
+ SIMPLE_VIRTUAL = 10.0;
+ }
+ symbols {
+ EXPLICIT_VIRTUAL = 10.0
+ }
+ }
+ }
+
+ id_test_priority {
+ priority = high;
+ from = "user@test.com";
+ apply {
+ symbols_enabled {
+ PRIORITY = 10.0;
+ }
+ symbols {
+ PRIORITY_2 = 10.0
+ }
+ }
+ }
+
+ empty_symbols_enabled {
+ ip = "5.5.5.5";
+ apply {
+ symbols_enabled = [];
+ }
+ }
+
+ empty_groups_enabled {
+ selector = "ip.in(\"5.5.5.6\")";
+ apply {
+ groups_enabled = [];
+ }
+ }
+
+ external {
+ disabled = true
+ external_map = {
+ map = {
+ backend = "http://127.0.0.1:18080/settings";
+ external = true;
+ method = "body";
+ encode = "json";
+ }
+ selector = "id('from');from('mime')";
+ }
+ register_symbols = {
+ EXTERNAL_SETTINGS = { score = 1.0 }
+ }
+ }
+}
+
+classifier {
+ backend = "sqlite3";
+ statfile {
+ spam = true;
+ symbol = BAYES_SPAM;
+ path = "/tmp/bayes.spam.sqlite3";
+ }
+ statfile {
+ spam = false;
+ symbol = BAYES_HAM;
+ path = "/tmp/bayes.ham.sqlite3";
+ }
+ min_learns = 1;
+ min_token_hits = 1;
+}
diff --git a/test/functional/configs/spamassassin.conf b/test/functional/configs/spamassassin.conf
new file mode 100644
index 0000000..0b66951
--- /dev/null
+++ b/test/functional/configs/spamassassin.conf
@@ -0,0 +1,7 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+spamassassin {
+ rules = "{= env.TESTDIR =}/configs/spamassassin.rules"
+}
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
+lua = "{= env.TESTDIR =}/lua/simple.lua"
diff --git a/test/functional/configs/spamassassin.rules b/test/functional/configs/spamassassin.rules
new file mode 100644
index 0000000..614f85e
--- /dev/null
+++ b/test/functional/configs/spamassassin.rules
@@ -0,0 +1,97 @@
+# These rules are from SpamAssasin project! (but might be modified)
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ifplugin Mail::SpamAssassin::Plugin::FreeMail
+
+freemail_domains qq.com yahoo.com outlook.com
+freemail_domains lycosmail.com hotmail.com
+
+header FREEMAIL_FROM eval:check_freemail_from()
+describe FREEMAIL_FROM Sender email is commonly abused enduser mail provider
+score FREEMAIL_FROM 1.0
+
+header FREEMAIL_ENVFROM_END_DIGIT eval:check_freemail_header('EnvelopeFrom', '\d@')
+describe FREEMAIL_ENVFROM_END_DIGIT Envelope-from freemail username ends in digit
+score FREEMAIL_ENVFROM_END_DIGIT 1.0
+
+header FREEMAIL_SUBJECT eval:check_freemail_header('Subject')
+describe FREEMAIL_SUBJECT Subject contains freemail
+score FREEMAIL_SUBJECT 1.0
+
+endif
+
+ifplugin Mail::SpamAssassin::Plugin::WLBLEval
+
+header USER_IN_BLACKLIST eval:check_from_in_blacklist()
+describe USER_IN_BLACKLIST From: address is in the user's black-list
+tflags USER_IN_BLACKLIST userconf noautolearn
+score USER_IN_BLACKLIST 10.0
+
+header USER_IN_WHITELIST eval:check_from_in_whitelist()
+describe USER_IN_WHITELIST From: address is in the user's white-list
+tflags USER_IN_WHITELIST userconf nice noautolearn
+score USER_IN_WHITELIST -10.0
+
+header USER_IN_BLACKLIST_TO eval:check_to_in_blacklist()
+describe USER_IN_BLACKLIST_TO User is listed in 'blacklist_to'
+tflags USER_IN_BLACKLIST_TO userconf noautolearn
+score USER_IN_BLACKLIST_TO 10.0
+
+header USER_IN_WHITELIST_TO eval:check_to_in_whitelist()
+describe USER_IN_WHITELIST_TO User is listed in 'whitelist_to'
+tflags USER_IN_WHITELIST_TO userconf nice noautolearn
+score USER_IN_WHITELIST_TO -10.0
+
+# not implemented
+#header USER_IN_DEF_WHITELIST eval:check_from_in_default_whitelist()
+#describe USER_IN_DEF_WHITELIST From: address is in the default white-list
+#tflags USER_IN_DEF_WHITELIST userconf nice noautolearn
+#score USER_IN_DEF_WHITELIST -10.0
+
+# not implemented
+#header USER_IN_MORE_SPAM_TO eval:check_to_in_more_spam()
+#describe USER_IN_MORE_SPAM_TO User is listed in 'more_spam_to'
+#tflags USER_IN_MORE_SPAM_TO userconf nice noautolearn
+
+# not implemented
+#header USER_IN_ALL_SPAM_TO eval:check_to_in_all_spam()
+#describe USER_IN_ALL_SPAM_TO User is listed in 'all_spam_to'
+#tflags USER_IN_ALL_SPAM_TO userconf nice noautolearn
+
+blacklist_to xxx@example.com
+blacklist_from yy@example.com
+whitelist_from nasutkadqw@esumare.ru
+whitelist_to zelen@megafonkavkaz.ru
+# not implemented
+#whitelist_from example@example.net yy@example.com
+# not implemented
+#whitelist_from *@example.com
+# not implemented
+#whitelist_from_rcvd *@example.org
+# not implemented
+#def_whitelist_from rspamd.com
+endif
+
+# These rules are /not/ from SpamAssassin project
+
+header TEST_XFOO X-Foo =~ /.{1,50}/
+score TEST_XFOO 1
+header TEST_XBAR X-Bar =~ /.{1,50}/
+score TEST_XBAR 1
+meta TEST_META1 TEST_XFOO && TEST_XBAR
+meta TEST_META2 TEST_META1 && SIMPLE_TEST
+meta TEST_META3 TEST_META1 && TEST_META2
+meta TEST_META4 TEST_META3 && TEST_XBAR
diff --git a/test/functional/configs/stats.conf b/test/functional/configs/stats.conf
new file mode 100644
index 0000000..ba6a5fe
--- /dev/null
+++ b/test/functional/configs/stats.conf
@@ -0,0 +1,85 @@
+options = {
+ filters = ["spf", "dkim", "regexp"]
+ url_tld = "{= env.TESTDIR =}/../lua/unit/test_tld.dat"
+ pidfile = "{= env.TMPDIR =}/rspamd.pid"
+ dns {
+ retransmits = 10;
+ timeout = 2s;
+ fake_records = [{
+ name = "example.net";
+ type = txt;
+ replies = ["v=spf1 -all"];
+ }]
+ }
+}
+logging = {
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log"
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ }
+ unknown_weight = 1
+}
+
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ keypair {
+ pubkey = "{= env.KEY_PUB1 =}";
+ privkey = "{= env.KEY_PVT1 =}";
+ }
+ task_timeout = 60s;
+}
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ keypair {
+ pubkey = "{= env.KEY_PUB1 =}";
+ privkey = "{= env.KEY_PVT1 =}";
+ }
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl";
+}
+
+classifier {
+ languages_enabled = true;
+ tokenizer {
+ name = "osb";
+ hash = {= env.STATS_HASH =};
+ key = {= env.STATS_KEY =};
+ }
+ backend = "{= env.STATS_BACKEND =}";
+ statfile {
+ spam = true;
+ symbol = BAYES_SPAM;
+ size = 1M;
+ server = {= env.REDIS_SERVER =}
+ }
+ statfile {
+ spam = false;
+ symbol = BAYES_HAM;
+ size = 1M;
+ server = {= env.REDIS_SERVER =}
+ }
+
+ cache {
+ server = {= env.REDIS_SERVER =}
+ }
+
+ {% if env.STATS_PER_USER ~= '' %}
+ per_user = <<EOD
+return function(task)
+ return task:get_principal_recipient()
+end
+EOD;
+ {% endif %}
+}
+lua = "{= env.TESTDIR =}/lua/test_coverage.lua";
+
+settings {}
diff --git a/test/functional/configs/trivial.conf b/test/functional/configs/trivial.conf
new file mode 100644
index 0000000..97e3a30
--- /dev/null
+++ b/test/functional/configs/trivial.conf
@@ -0,0 +1,50 @@
+options = {
+ filters = ["spf", "dkim", "regexp"]
+ url_tld = "{= env.TESTDIR =}/../lua/unit/test_tld.dat"
+ pidfile = "{= env.TMPDIR =}/rspamd.pid";
+ lua_path = "{= env.INSTALLROOT =}/share/rspamd/lib/?.lua";
+ dns {
+ nameserver = ["8.8.8.8", "8.8.4.4"];
+ retransmits = 10;
+ timeout = 2s;
+ }
+}
+logging = {
+ log_urls = true;
+ type = "file",
+ level = "debug"
+ filename = "{= env.TMPDIR =}/rspamd.log";
+ log_usec = true;
+}
+metric = {
+ name = "default",
+ actions = {
+ reject = 100500,
+ }
+ unknown_weight = 1
+}
+
+worker {
+ type = normal
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_NORMAL =}"
+ count = 1
+ keypair {
+ pubkey = "{= env.KEY_PUB1 =}";
+ privkey = "{= env.KEY_PVT1 =}";
+ }
+ task_timeout = 10s;
+}
+
+worker {
+ type = controller
+ bind_socket = "{= env.LOCAL_ADDR =}:{= env.PORT_CONTROLLER =}"
+ count = 1
+ secure_ip = ["127.0.0.1", "::1"];
+ stats_path = "{= env.TMPDIR =}/stats.ucl"
+}
+
+modules {
+ path = "{= env.TESTDIR =}/../../src/plugins/lua/"
+}
+lua = "{= env.INSTALLROOT =}/share/rspamd/rules/rspamd.lua"
+
diff --git a/test/functional/configs/url_redirector.conf b/test/functional/configs/url_redirector.conf
new file mode 100644
index 0000000..e6009ba
--- /dev/null
+++ b/test/functional/configs/url_redirector.conf
@@ -0,0 +1,8 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+redis {
+ servers = "{= env.REDIS_ADDR =}:{= env.REDIS_PORT =}";
+}
+url_redirector {
+ redirector_hosts_map = "{= env.TESTDIR =}/configs/maps/redir.map";
+}
diff --git a/test/functional/configs/url_tags.conf b/test/functional/configs/url_tags.conf
new file mode 100644
index 0000000..9c82b84
--- /dev/null
+++ b/test/functional/configs/url_tags.conf
@@ -0,0 +1,9 @@
+url_tags {
+}
+redis {
+ servers = "${RSPAMD_REDIS_ADDR}:${RSPAMD_REDIS_PORT}";
+}
+
+options {
+ enable_experimental = true;
+}
diff --git a/test/functional/configs/whitelist.conf b/test/functional/configs/whitelist.conf
new file mode 100644
index 0000000..db264aa
--- /dev/null
+++ b/test/functional/configs/whitelist.conf
@@ -0,0 +1,83 @@
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+dmarc {}
+spf {}
+whitelist {
+
+ rules {
+
+ "WHITELIST_DDS" = {
+ valid_dkim = true;
+ valid_dmarc = true;
+ valid_spf = true;
+ domains = [
+ "{= env.TESTDIR =}/configs/maps/domains.list",
+ ];
+ score = -3.0;
+ inverse_symbol = "BLACKLIST_DDS";
+ }
+
+ "WHITELIST_DKIM" = {
+ valid_dkim = true;
+ domains = [
+ "{= env.TESTDIR =}/configs/maps/domains.list",
+ ];
+ description = "Mail comes from the whitelisted domain and has a valid DKIM signature";
+ score = -1.0;
+ inverse_symbol = "BLACKLIST_DKIM";
+ }
+
+ "WHITELIST_SPF" = {
+ valid_spf = true;
+ domains = [
+ "{= env.TESTDIR =}/configs/maps/domains.list",
+ ];
+ score = -1.0
+ inverse_multiplier = 3.0
+ description = "Mail comes from the whitelisted domain and has a valid SPF policy";
+ inverse_symbol = "BLACKLIST_SPF";
+ }
+
+ "WHITELIST_SPF_DKIM" = {
+ valid_spf = true;
+ valid_dkim = true;
+ domains = [
+ "{= env.TESTDIR =}/configs/maps/domains.list"
+ ];
+ score = -3.0;
+ description = "Mail comes from the whitelisted domain and has a valid SPF policy and valid DKIM signature";
+ }
+
+
+ "WHITELIST_DMARC_DKIM" = {
+ valid_dmarc = true;
+ valid_dkim = true;
+ domains = [
+ "{= env.TESTDIR =}/configs/maps/domains.list",
+ ];
+ score = -2.0;
+ description = "Mail comes from the whitelisted domain and has valid DMARC and DKIM policies";
+ inverse_symbol = "BLACKLIST_DMARC_DKIM";
+ }
+
+ "WHITELIST_DMARC" = {
+ valid_dmarc = true;
+ domains = [
+ "{= env.TESTDIR =}/configs/maps/domains.list",
+ ];
+ score = -2.0;
+ description = "Mail comes from the whitelisted domain and has valid DMARC policy";
+ inverse_symbol = "BLACKLIST_DMARC";
+ }
+
+ "STRICT_DMARC" = {
+ strict = true;
+ valid_dmarc = true;
+ domains = [
+ "{= env.TESTDIR =}/configs/maps/domains.list",
+ ];
+ score = -3.0;
+ }
+
+ }
+}
diff --git a/test/functional/data/bayes.ham.sqlite3 b/test/functional/data/bayes.ham.sqlite3
new file mode 100644
index 0000000..5f82ffa
--- /dev/null
+++ b/test/functional/data/bayes.ham.sqlite3
Binary files differ
diff --git a/test/functional/data/bayes.spam.sqlite3 b/test/functional/data/bayes.spam.sqlite3
new file mode 100644
index 0000000..64f8064
--- /dev/null
+++ b/test/functional/data/bayes.spam.sqlite3
Binary files differ
diff --git a/test/functional/data/initial_schema/data.rspamd.sql b/test/functional/data/initial_schema/data.rspamd.sql
new file mode 100644
index 0000000..50eca37
--- /dev/null
+++ b/test/functional/data/initial_schema/data.rspamd.sql
@@ -0,0 +1,56 @@
+(toDate(now()),'2018-08-21 11:24:36','xent.com','evergo.net','127.0.0.0',1.19,0,3248,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','johnhall','','','friends of rohit khare <fork.xent.com>','162bc579fb3145dc3c65669328fa39cb'),
+(toDate(now()),'2018-08-21 11:24:46','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2890,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','c772a14916e765a4593939a5991bb859'),
+(toDate(now()),'2018-08-21 11:24:46','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,975,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','83cfbad1dbcb7234843ff01b810ea2c0'),
+(toDate(now()),'2018-08-21 11:24:46','xent.com','cse.ucsc.edu','127.0.0.0',0.69,0,3225,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','elias','','','friends of rohit khare <fork.xent.com>','9fa1aa06c0afe91f1aad86a128c475f6'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.sourceforge.net','users.sourceforge.net','127.0.0.0',0.69,0,4198,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','spamassassin-commits-admin','yyyyason','','','<spamassassin-commits.example.sourceforge.net>','b8c4054a876b3e6593b0c7ea427045ba'),
+(toDate(now()),'2018-08-21 11:24:15','freshrpms.net','camperquake.de','127.0.0.0',1.69,0,3663,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','ralf','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','bb4fcb9417cb59838b2c83c4b4342ffa'),
+(toDate(now()),'2018-08-21 11:24:15','freshrpms.net','rpmforge.net','127.0.0.0',1.69,0,3934,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','matthias','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','2b1b347f25159ca1ea8d4a9ee5187ba8'),
+(toDate(now()),'2018-08-21 11:24:15','linux.ie','steorn.com','127.0.0.0',1.19,0,2934,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','ilug-admin','carlos.luna','','','irish linux users\' group <ilug.linux.ie>','c95d8ffd4f70a91328eeec15c452ab6b'),
+(toDate(now()),'2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1078,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','5a46062bde8cce224ea90aa134e10da1'),
+(toDate(now()),'2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2088,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','cf0d79f76e51a3e6c5bcfdfdac094685'),
+(toDate(now()),'2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2388,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','2ab3790853d8bf1a90623b6d788abeb0'),
+(toDate(now()),'2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1124,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','47362127e1870aa9aaf3eef8ae5c749b'),
+(toDate(now()),'2018-08-21 11:24:15','xent.com','mithral.com','127.0.0.0',0.69,0,3546,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','beberg','','','friends of rohit khare <fork.xent.com>','f4d74c809ba48b987fd6271baf82c12f'),
+(toDate(now()),'2018-08-21 11:24:20','baesystems.com','baesystems.com','127.0.0.0',0.9,0,3341,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','robert.chambers','robert.chambers','','','','835aaf36a75a56ce93dfa3badaeb92eb'),
+(toDate(now()),'2018-08-21 11:24:20','example.sourceforge.net','telus.net','127.0.0.0',1.69,0,3852,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','spamassassin-talk-admin','cfortune','','','talk about spamassassin <spamassassin-talk.example.sourceforge.net>','8e799460b63b7919b4c4587dadd7b063'),
+(toDate(now()),'2018-08-21 11:24:20','freshrpms.net','egwn.net','127.0.0.0',1.69,0,5074,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','matthias','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','3950ac78c11ec35d55cb56c265d9474a'),
+(toDate(now()),'2018-08-21 11:24:20','freshrpms.net','fluid.com','127.0.0.0',0.69,0,3712,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','tengel','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','61936d1c00246dc8410391c9e5f03f5a'),
+(toDate(now()),'2018-08-21 11:24:20','linux.ie','eircom.net','127.0.0.0',1.3,0,2763,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','ilug-admin','cout','','','irish linux users\' group <ilug.linux.ie>','d1e819de70503d5ebfa49b6708703942'),
+(toDate(now()),'2018-08-21 11:24:20','petting-zoo.net','','127.0.0.0',1.8900000000000001,0,2538,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','0xdeadbeef-request','','','','','c6046b370d90b6b13a4788557e77cf24'),
+(toDate(now()),'2018-08-21 11:24:20','returns.groups.yahoo.com','hotmail.com','127.0.0.0',0.89,0,4135,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-56028-1034088521-zzzz=example.com','skitster','','','','c22813bc4c29a3f5fa0dd1cbd598c7c8'),
+(toDate(now()),'2018-08-21 11:24:20','returns.groups.yahoo.com','srv0.ems.ed.ac.uk','127.0.0.0',1.3900000000000001,0,5651,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-53809-1031308404-zzzz=example.com','martin','','','','a10e11dcdc3d527ade0f5e85bc583f78'),
+(toDate(now()),'2018-08-21 11:24:20','returns.groups.yahoo.com','earthlink.net','127.0.0.0',1.3900000000000001,0,2885,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-60166-1038789202-yyyy=spamassassin.taint.org','billjac','','','','5a914e03141f3f7099e50be362c007c2'),
+(toDate(now()),'2018-08-21 11:24:20','returns.groups.yahoo.com','srv0.ems.ed.ac.uk','127.0.0.0',1.3900000000000001,0,5706,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-53809-1031308404-zzzz=spamassassin.taint.org','martin','','','','a10e11dcdc3d527ade0f5e85bc583f78'),
+(toDate(now()),'2018-08-21 11:24:20','returns.groups.yahoo.com','bestirishmusic.com','127.0.0.0',1.3900000000000001,0,3709,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-55915-1033992114-zzzz=spamassassin.taint.org','webmaster','','','','5e3597c0f14b241c915c161c462c4804'),
+(toDate(now()),'2018-08-21 11:24:20','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2168,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','472774888e942d317fecc91724b304f8'),
+(toDate(now()),'2018-08-21 11:24:20','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1125,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','54b07928d68e59fb370ea89f2efdaabd'),
+(toDate(now()),'2018-08-21 11:24:20','xent.com','ianbell.com','127.0.0.0',0.69,0,15012,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','fork','','','friends of rohit khare <fork.xent.com>','64216f07ed5995242c78fe27b537c30b'),
+(toDate(now()),'2018-08-21 11:24:20','xent.com','techmonkeys.net','127.0.0.0',0.69,0,4501,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','cdale','','','friends of rohit khare <fork.xent.com>','a96c963bc4d5297e3ebdf4f2ea27927f'),
+(toDate(now()),'2018-08-21 11:24:20','xent.com','hotmail.com','127.0.0.0',1.69,0,2606,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','fork_list','','','friends of rohit khare <fork.xent.com>','9d743f10d435a607f9179d9429eb0ea1'),
+(toDate(now()),'2018-08-21 11:24:20','xent.com','vertexdev.com','127.0.0.0',1.19,0,3224,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','jeff','','','friends of rohit khare <fork.xent.com>','5008d567ff90facc36d94be316d4e0fe'),
+(toDate(now()),'2018-08-21 11:24:20','xent.com','panix.com','127.0.0.0',2.69,0,2366,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','lgonze','','','friends of rohit khare <fork.xent.com>','154b831310873a800b92169b7e049a8f'),
+(toDate(now()),'2018-08-21 11:24:20','xent.com','magnesium.net','127.0.0.0',0.69,0,6039,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','bitbitch','','','friends of rohit khare <fork.xent.com>','0fcc0fdd0995b10de6f07585354e2941'),
+(toDate(now()),'2018-08-21 11:24:20','xent.com','canada.com','127.0.0.0',0.69,0,3685,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','garym','','','friends of rohit khare <fork.xent.com>','c0796f14caaf4e02ca683309021f6812'),
+(toDate(now()),'2018-08-21 11:24:25','freshrpms.net','egwn.net','127.0.0.0',1.69,0,3500,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','matthias','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','9848f3ace4d8b54ed52061cf0330d64f'),
+(toDate(now()),'2018-08-21 11:24:25','xent.com','endeavors.com','127.0.0.0',0.69,0,3561,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','gbolcer','','','friends of rohit khare <fork.xent.com>','a5109e184c129e5521438be5375e7804'),
+(toDate(now()),'2018-08-21 11:24:26','comcast.net','','undefined',6.9,0,4682,'unknown','unknown','unknown','unknown','unknown','unknown',0,'add header','tim.one','','','','','9367812a3c0f279aa3dbf691e9d9ec11'),
+(toDate(now()),'2018-08-21 11:24:26','example.sourceforge.net','pobox.com','127.0.0.0',1.19,0,4167,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','razor-users-admin','wstearns','','','<razor-users.example.sourceforge.net>','8dd6255d0616f90f16ec8cfbba496143'),
+(toDate(now()),'2018-08-21 11:24:26','xent.com','canada.com','127.0.0.0',0.69,0,3031,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','garym','','','friends of rohit khare <fork.xent.com>','e5ccbf749700f93ece4a41907d81593b'),
+(toDate(now()),'2018-08-21 11:24:36','example.com','example.com','127.0.0.0',0.9,0,2154,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','27e6cbfca0423d5c2aa6fa6731670699'),
+(toDate(now()),'2018-08-21 11:24:36','example.com','example.com','127.0.0.0',0.9,0,1630,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','b35cf34eb7186872f7c035a7bd450e92'),
+(toDate(now()),'2018-08-21 11:24:36','example.com','panasas.com','127.0.0.0',0.69,0,4910,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','exmh-users-admin','welch','','','discussion list for exmh users <exmh-users.example.com>','7987eec136f5391445df5bfbe64d4b33'),
+(toDate(now()),'2018-08-21 11:24:36','example.sourceforge.net','gmx.net','127.0.0.0',1.69,0,5667,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','spamassassin-talk-admin','msquadrat.nospamplease','','','talk about spamassassin <spamassassin-talk.example.sourceforge.net>','2c6645233690c3bd7677fc5a6b637000'),
+(toDate(now()),'2018-08-21 11:24:36','freshrpms.net','bellsouth.net','127.0.0.0',0.69,0,4817,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','lance_tt','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','7201979c9c64847048d2e20512a6b5d9'),
+(toDate(now()),'2018-08-21 11:24:36','freshrpms.net','ckloiber.com','127.0.0.0',0.69,0,3161,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','ckloiber','','','freshrpms rpm discussion list <rpm-zzzlist.freshrpms.net>','52fee32a1ec50eb0d0dab4dada87c497'),
+(toDate(now()),'2018-08-21 11:24:36','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1474,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','e3a16ef77e2ee48767c499104f67270b'),
+(toDate(now()),'2018-08-21 11:24:36','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1053,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','c3a8d98433975fefb64f3c059d6046f0'),
+(toDate(now()),'2018-08-21 11:24:36','xent.com','techmonkeys.net','127.0.0.0',0.69,0,2218,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','cdale','','','friends of rohit khare <fork.xent.com>','672fa037061f621085fbe997d236f374'),
+(toDate(now()),'2018-08-21 11:24:36','xent.com','','127.0.0.0',0.69,0,5136,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','','','','friends of rohit khare <fork.xent.com>','3b86bc8189eba5d6ed9b633ce2995c6d'),
+(toDate(now()),'2018-08-21 11:24:36','xent.com','permafrost.net','127.0.0.0',0.69,0,3388,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','owen','','','friends of rohit khare <fork.xent.com>','03190fd537aa97e9be7d4a75fed57d36'),
+(toDate(now()),'2018-08-21 11:24:36','xent.com','slack.net','127.0.0.0',0.69,0,2554,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','tomwhore','','','friends of rohit khare <fork.xent.com>','2a1f34828b8a6a9750a1a09bc305983d'),
+(toDate(now()),'2018-08-21 11:24:36','xent.com','permafrost.net','127.0.0.0',0.69,0,4069,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','owen','','','friends of rohit khare <fork.xent.com>','9167f411217b18af953884010508dc2f'),
+(toDate(now()),'2018-08-21 11:24:36','xent.com','magnesium.net','127.0.0.0',0.69,0,2547,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','bitbitch','','','friends of rohit khare <fork.xent.com>','8d5d3caf91661a5b36f857a3010c4a48'),
+(toDate(now()),'2018-08-21 11:24:36','xent.com','canada.com','127.0.0.0',0.69,0,2806,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','garym','','','friends of rohit khare <fork.xent.com>','236f08f521364e701ad8dc43afde4fc4'),
+(toDate(now()),'2018-08-21 11:24:46','example.com','example.com','127.0.0.0',0.9,0,1639,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','a8f132bbb47ea2b4310e550d9617023f'),
+(toDate(now()),'2018-08-21 11:24:46','xent.com','hotmail.com','127.0.0.0',1.69,0,3376,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','deafbox','','','friends of rohit khare <fork.xent.com>','8d7f99415bb42f744c9d06ccf8d5ccbf') \ No newline at end of file
diff --git a/test/functional/data/initial_schema/data.rspamd_asn.sql b/test/functional/data/initial_schema/data.rspamd_asn.sql
new file mode 100644
index 0000000..3ac9919
--- /dev/null
+++ b/test/functional/data/initial_schema/data.rspamd_asn.sql
@@ -0,0 +1,66 @@
+(toDate(now()),'43607711d98b62ce6f3d13013ce7241f','--','--','--'),
+(toDate(now()),'4ddf3d1d31558d038d9e7bba5a862494','--','--','--'),
+(toDate(now()),'8db807c0b535792ea4e2d9b7f69b113c','--','--','--'),
+(toDate(now()),'905721f001ce446b2ad23d6833716bcb','--','--','--'),
+(toDate(now()),'8d7f99415bb42f744c9d06ccf8d5ccbf','--','--','--'),
+(toDate(now()),'a8f132bbb47ea2b4310e550d9617023f','--','--','--'),
+(toDate(now()),'03190fd537aa97e9be7d4a75fed57d36','--','--','--'),
+(toDate(now()),'0fcc0fdd0995b10de6f07585354e2941','--','--','--'),
+(toDate(now()),'154b831310873a800b92169b7e049a8f','--','--','--'),
+(toDate(now()),'162bc579fb3145dc3c65669328fa39cb','--','--','--'),
+(toDate(now()),'236f08f521364e701ad8dc43afde4fc4','--','--','--'),
+(toDate(now()),'27e6cbfca0423d5c2aa6fa6731670699','--','--','--'),
+(toDate(now()),'28ebd9e7e8a4ea4d54f18c02305382a8','--','--','--'),
+(toDate(now()),'2a1f34828b8a6a9750a1a09bc305983d','--','--','--'),
+(toDate(now()),'2ab3790853d8bf1a90623b6d788abeb0','--','--','--'),
+(toDate(now()),'2b1b347f25159ca1ea8d4a9ee5187ba8','--','--','--'),
+(toDate(now()),'2c6645233690c3bd7677fc5a6b637000','--','--','--'),
+(toDate(now()),'3950ac78c11ec35d55cb56c265d9474a','--','--','--'),
+(toDate(now()),'3b86bc8189eba5d6ed9b633ce2995c6d','--','--','--'),
+(toDate(now()),'472774888e942d317fecc91724b304f8','--','--','--'),
+(toDate(now()),'47362127e1870aa9aaf3eef8ae5c749b','--','--','--'),
+(toDate(now()),'5008d567ff90facc36d94be316d4e0fe','--','--','--'),
+(toDate(now()),'51cd92041e96327f9fc502ee19990c04','--','--','--'),
+(toDate(now()),'52fee32a1ec50eb0d0dab4dada87c497','--','--','--'),
+(toDate(now()),'54b07928d68e59fb370ea89f2efdaabd','--','--','--'),
+(toDate(now()),'5a46062bde8cce224ea90aa134e10da1','--','--','--'),
+(toDate(now()),'5a914e03141f3f7099e50be362c007c2','--','--','--'),
+(toDate(now()),'5e3597c0f14b241c915c161c462c4804','--','--','--'),
+(toDate(now()),'61936d1c00246dc8410391c9e5f03f5a','--','--','--'),
+(toDate(now()),'64216f07ed5995242c78fe27b537c30b','--','--','--'),
+(toDate(now()),'672fa037061f621085fbe997d236f374','--','--','--'),
+(toDate(now()),'69afbde36dc807ef7190ad549f94e697','--','--','--'),
+(toDate(now()),'7201979c9c64847048d2e20512a6b5d9','--','--','--'),
+(toDate(now()),'7987eec136f5391445df5bfbe64d4b33','--','--','--'),
+(toDate(now()),'835aaf36a75a56ce93dfa3badaeb92eb','--','--','--'),
+(toDate(now()),'8d5d3caf91661a5b36f857a3010c4a48','--','--','--'),
+(toDate(now()),'8dd6255d0616f90f16ec8cfbba496143','--','--','--'),
+(toDate(now()),'8e799460b63b7919b4c4587dadd7b063','--','--','--'),
+(toDate(now()),'9367812a3c0f279aa3dbf691e9d9ec11','--','--','--'),
+(toDate(now()),'9848f3ace4d8b54ed52061cf0330d64f','--','--','--'),
+(toDate(now()),'9d743f10d435a607f9179d9429eb0ea1','--','--','--'),
+(toDate(now()),'a10e11dcdc3d527ade0f5e85bc583f78','--','--','--'),
+(toDate(now()),'a10e11dcdc3d527ade0f5e85bc583f78','--','--','--'),
+(toDate(now()),'a5109e184c129e5521438be5375e7804','--','--','--'),
+(toDate(now()),'a96c963bc4d5297e3ebdf4f2ea27927f','--','--','--'),
+(toDate(now()),'b35cf34eb7186872f7c035a7bd450e92','--','--','--'),
+(toDate(now()),'b3e5e01c86a8fd8a9bc5a66d033cc32e','--','--','--'),
+(toDate(now()),'b8c4054a876b3e6593b0c7ea427045ba','--','--','--'),
+(toDate(now()),'bb4fcb9417cb59838b2c83c4b4342ffa','--','--','--'),
+(toDate(now()),'c0796f14caaf4e02ca683309021f6812','--','--','--'),
+(toDate(now()),'c22813bc4c29a3f5fa0dd1cbd598c7c8','--','--','--'),
+(toDate(now()),'c4bb938afcc5d6af78c0ea66145bbc2d','--','--','--'),
+(toDate(now()),'c6046b370d90b6b13a4788557e77cf24','--','--','--'),
+(toDate(now()),'c772a14916e765a4593939a5991bb859','--','--','--'),
+(toDate(now()),'c95d8ffd4f70a91328eeec15c452ab6b','--','--','--'),
+(toDate(now()),'cd6dea756b83f81172d07a1e0f5b08a9','--','--','--'),
+(toDate(now()),'cf0d79f76e51a3e6c5bcfdfdac094685','--','--','--'),
+(toDate(now()),'d0bbb7fbc8d4e31471a25456fcfaff64','--','--','--'),
+(toDate(now()),'d10e5a118896958c447eaa343e3d2db8','--','--','--'),
+(toDate(now()),'d1e819de70503d5ebfa49b6708703942','--','--','--'),
+(toDate(now()),'d9ffc9704faf5057b1394c507153c277','--','--','--'),
+(toDate(now()),'e3a16ef77e2ee48767c499104f67270b','--','--','--'),
+(toDate(now()),'e5ccbf749700f93ece4a41907d81593b','--','--','--'),
+(toDate(now()),'f4d74c809ba48b987fd6271baf82c12f','--','--','--'),
+(toDate(now()),'faf1b3f92d8aa05c97fbc212f11afc54','--','--','--'),
+(toDate(now()),'feae0c04282d5e27705366d3211bc69e','--','--','--') \ No newline at end of file
diff --git a/test/functional/data/initial_schema/data.rspamd_attachments.sql b/test/functional/data/initial_schema/data.rspamd_attachments.sql
new file mode 100644
index 0000000..839ee09
--- /dev/null
+++ b/test/functional/data/initial_schema/data.rspamd_attachments.sql
@@ -0,0 +1,10 @@
+(toDate(now()), 'd1e819de70503d5ebfa49b6708703942', ['fsouoxtjvb.xls'], ['application/vnd.ms-excel'], [39424], ['d66d8dfcab696065']),
+(toDate(now()), '5e3597c0f14b241c915c161c462c4804', ['eybxf.gif','llfoe.gif'], ['image/gif','image/gif'], [6558,67], ['d3e6d83395091599','abfae96a0fc709e9']),
+(toDate(now()), '8dd6255d0616f90f16ec8cfbba496143', ['fake.xls'], ['application/vnd.ms-excel'], [361472], ['e0791f7961db0e00']),
+(toDate(now()), '8d7f99415bb42f744c9d06ccf8d5ccbf', ['dsn_status','header'], ['message/delivery-status','text/rfc822-headers'], [451,650], ['4e5127839ad4af96','ee1db81d572b606c']),
+(toDate(now()), '2b1b347f25159ca1ea8d4a9ee5187ba8', ['vdxcpjg.xls'], ['application/vnd.ms-excel'], [39424], ['163b098ba7aee8f9']),
+(toDate(now()), '2ab3790853d8bf1a90623b6d788abeb0', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']),
+(toDate(now()), '0fcc0fdd0995b10de6f07585354e2941', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']),
+(toDate(now()), '7987eec136f5391445df5bfbe64d4b33', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']),
+(toDate(now()), '2c6645233690c3bd7677fc5a6b637000', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']),
+(toDate(now()), '52fee32a1ec50eb0d0dab4dada87c497', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']) \ No newline at end of file
diff --git a/test/functional/data/initial_schema/data.rspamd_emails.sql b/test/functional/data/initial_schema/data.rspamd_emails.sql
new file mode 100644
index 0000000..90597ba
--- /dev/null
+++ b/test/functional/data/initial_schema/data.rspamd_emails.sql
@@ -0,0 +1,6 @@
+(toDate(now()), '2b1b347f25159ca1ea8d4a9ee5187ba8', ['fake1@mail.ru']),
+(toDate(now()), '2ab3790853d8bf1a90623b6d788abeb0', ['fake2@mail.ru']),
+(toDate(now()), '0fcc0fdd0995b10de6f07585354e2941', ['info_stop_mailer@yahoo.co.jp']),
+(toDate(now()), '7987eec136f5391445df5bfbe64d4b33', ['info_stop_mailer@yahoo.co.jp']),
+(toDate(now()), '2c6645233690c3bd7677fc5a6b637000', ['info_stop_mailer@yahoo.co.jp']),
+(toDate(now()), '52fee32a1ec50eb0d0dab4dada87c497', ['fake3@g-mail.com','fake4@g-mail.com']) \ No newline at end of file
diff --git a/test/functional/data/initial_schema/data.rspamd_symbols.sql b/test/functional/data/initial_schema/data.rspamd_symbols.sql
new file mode 100644
index 0000000..eb16d67
--- /dev/null
+++ b/test/functional/data/initial_schema/data.rspamd_symbols.sql
@@ -0,0 +1,10 @@
+(toDate(now()), 'd1e819de70503d5ebfa49b6708703942', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','missing_mime_version','to_match_envrcpt_all','to_dn_none','rcpt_count_one','many_invisible_parts','freemail_envrcpt','mime_base64_text','subj_all_caps','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','has_x_prio_one','asn','mid_rhs_match_from'], [0,0,0.01,0,0,2,0,0,0,0.05,0,0.1,3,0.2,0,0,0,0,0,0,0], ['','','','','fakexx@qq.com','','','','1','1','qq.com','','68','','1','qq.com','','','1','asn:4134, ipnet:49.64.0.0/11, country:cn','']),
+(toDate(now()), '5e3597c0f14b241c915c161c462c4804', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake@mailto.gq','','','asn:201725, ipnet:176.97.248.0/21, country:pl','']),
+(toDate(now()), '8dd6255d0616f90f16ec8cfbba496143', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','missing_mime_version','to_match_envrcpt_all','subject_needs_encoding','to_dn_none','from_needs_encoding','rcpt_count_one','many_invisible_parts','freemail_envrcpt','mime_base64_text','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','asn','has_x_prio_five','mid_rhs_match_from'], [0,0,0.01,0,0,2,0,1,0,1,0,0.05,0,0.1,0.2,0,0,0,0,0,0,0], ['','','','','fakeq3@qq.com','','','','','','1','1','qq.com','','','1','qq.com','','','asn:4134, ipnet:113.120.0.0/13, country:cn','5','']),
+(toDate(now()), '8d7f99415bb42f744c9d06ccf8d5ccbf', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake@mailto.gq','','','asn:28187, ipnet:189.89.208.0/22, country:br','']),
+(toDate(now()), '2b1b347f25159ca1ea8d4a9ee5187ba8', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake3@wowmail.cf','','','asn:52935, ipnet:191.5.108.0/23, country:br','']),
+(toDate(now()), '2ab3790853d8bf1a90623b6d788abeb0', ['arc_na','rcvd_via_smtp_auth','zetascan_uri','from_has_dn','missing_mime_version','to_match_envrcpt_all','mime_good','to_dn_none','freemail_envrcpt','rambler_uribl','has_x_prio_three','rcpt_count_seven','dbl_abuse','rcvd_count_one','freemail_to','fsl_single_url','rcvd_no_tls_last','invuri','asn','freemail_cc'], [0,0,0,0,2,0,-0.1,0,0,0,0,0,6.5,0,0,0.1,0,0,0,0], ['','','suportdesk.com.e93be9862ef26e5fccb23696d6707ffc.dblack.api.zetascan.com','','','','text/plain','','bellsouth.net,aol.com,gmx.de,hotmail.com,gmail.com,me.com','suportdesk.com','3','7','suportdesk.com.dbl.uri.fslupdate.com','1','bellsouth.net','','','suportdesk.com.invuri.uri.fslupdate.com','asn:10429, ipnet:189.56.0.0/15, country:br','aol.com']),
+(toDate(now()), '0fcc0fdd0995b10de6f07585354e2941', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','to_match_envrcpt_all','freemail_envrcpt','r_bad_cte_7bit','to_dn_none','subject_needs_encoding','rcpt_count_one','many_invisible_parts','has_x_prio_three','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0.01,0,0,0,0,3.5,0,1,0,0.05,0,0.2,0,0,0,0,0,0], ['','','','','fake5@qq.com','','qq.com','7bit','','','1','1','3','','1','qq.com','','','asn:4134, ipnet:113.120.0.0/13, country:cn','']),
+(toDate(now()), '7987eec136f5391445df5bfbe64d4b33', ['arc_na','rcvd_via_smtp_auth','zetascan_uri','from_has_dn','to_match_envrcpt_all','sem_uribl','to_dn_none','rambler_uribl','many_invisible_parts','mime_base64_text','has_x_prio_three','rcpt_count_twelve','mime_html_only','rcvd_count_one','uribl_black','rcvd_no_tls_last','from_eq_envfrom','mid_rhs_not_fqdn','invuri','asn','dbl_spam','uribl_sbl','forged_outlook_html'], [0,0,0,0,0,3.5,0,0,0.7,0.1,0,0,0.2,0,7.5,0,0,0.5,0,0,6.5,6.5,5], ['','','gdguojian.net.e93be9862ef26e5fccb23696d6707ffc.dblack.api.zetascan.com','','','gdguojian.net.uribl.spameatingmonkey.net','','gdguojian.net','8','','3','21','','1','gdguojian.net.multi.uri.fslupdate.com','','','','gdguojian.net.invuri.uri.fslupdate.com','asn:4134, ipnet:182.96.0.0/12, country:cn','gdguojian.net.dbl.uri.fslupdate.com','gdguojian.net','']),
+(toDate(now()), '2c6645233690c3bd7677fc5a6b637000', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','missing_mime_version','to_match_envrcpt_all','to_dn_none','rcpt_count_one','many_invisible_parts','freemail_envrcpt','subj_all_caps','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','asn','has_x_prio_five','mid_rhs_match_from'], [0,0,0.01,0,0,2,0,0,0,0.05,0,3,0.2,0,0,0,0,0,0,0], ['','','','','fake9@qq.com','','','','1','1','qq.com','74','','1','qq.com','','','asn:4134, ipnet:182.32.0.0/12, country:cn','5','']),
+(toDate(now()), '52fee32a1ec50eb0d0dab4dada87c497', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake10@mailto.gq','','','asn:262812, ipnet:200.66.120.0/23, country:br','']) \ No newline at end of file
diff --git a/test/functional/data/initial_schema/data.rspamd_urls.sql b/test/functional/data/initial_schema/data.rspamd_urls.sql
new file mode 100644
index 0000000..e9749a5
--- /dev/null
+++ b/test/functional/data/initial_schema/data.rspamd_urls.sql
@@ -0,0 +1,10 @@
+(toDate(now()), 'd1e819de70503d5ebfa49b6708703942', ['gdguojian.net'], ['http://gdguojian.net/']),
+(toDate(now()), '5e3597c0f14b241c915c161c462c4804', ['gdguojian.net'], ['http://gdguojian.net/']),
+(toDate(now()), '8dd6255d0616f90f16ec8cfbba496143', ['gdguojian.net'], ['http://gdguojian.net/']),
+(toDate(now()), '8d7f99415bb42f744c9d06ccf8d5ccbf', ['gdguojian.net'], ['http://gdguojian.net/']),
+(toDate(now()), '2b1b347f25159ca1ea8d4a9ee5187ba8', ['gdguojian.net'], ['http://gdguojian.net/']),
+(toDate(now()), '2ab3790853d8bf1a90623b6d788abeb0', ['gdguojian.net'], ['http://gdguojian.net/']),
+(toDate(now()), '0fcc0fdd0995b10de6f07585354e2941', ['gdguojian.net'], ['http://gdguojian.net/']),
+(toDate(now()), '7987eec136f5391445df5bfbe64d4b33', ['gdguojian.net'], ['http://gdguojian.net/']),
+(toDate(now()), '2c6645233690c3bd7677fc5a6b637000', ['gdguojian.net'], ['http://gdguojian.net/']),
+(toDate(now()), '52fee32a1ec50eb0d0dab4dada87c497', ['gdguojian.net'], ['http://gdguojian.net/']) \ No newline at end of file
diff --git a/test/functional/data/initial_schema/schema.sql b/test/functional/data/initial_schema/schema.sql
new file mode 100644
index 0000000..7fa8cb7
--- /dev/null
+++ b/test/functional/data/initial_schema/schema.sql
@@ -0,0 +1,70 @@
+CREATE TABLE IF NOT EXISTS rspamd
+(
+ Date Date,
+ TS DateTime,
+ From String,
+ MimeFrom String,
+ IP String,
+ Score Float64,
+ NRcpt UInt8,
+ Size UInt32,
+ IsWhitelist Enum8('blacklist' = 0, 'whitelist' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('blacklist' = 0, 'whitelist' = 1, 'unknown' = 2)),
+ IsBayes Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2)),
+ IsFuzzy Enum8('whitelist' = 0, 'deny' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('whitelist' = 0, 'deny' = 1, 'unknown' = 2)),
+ IsFann Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2)),
+ IsDkim Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2)),
+ IsDmarc Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2)),
+ NUrls Int32,
+ Action Enum8('reject' = 0, 'rewrite subject' = 1, 'add header' = 2, 'greylist' = 3, 'no action' = 4, 'soft reject' = 5) DEFAULT CAST('no action' AS Enum8('reject' = 0, 'rewrite subject' = 1, 'add header' = 2, 'greylist' = 3, 'no action' = 4, 'soft reject' = 5)),
+ FromUser String,
+ MimeUser String,
+ RcptUser String,
+ RcptDomain String,
+ ListId String,
+ Digest FixedString(32)
+) ENGINE = MergeTree(Date, (TS, From), 8192);
+
+
+CREATE TABLE IF NOT EXISTS rspamd_attachments (
+ Date Date,
+ Digest FixedString(32),
+ `Attachments.FileName` Array(String),
+ `Attachments.ContentType` Array(String),
+ `Attachments.Length` Array(UInt32),
+ `Attachments.Digest` Array(FixedString(16))
+) ENGINE = MergeTree(Date, Digest, 8192);
+
+
+CREATE TABLE IF NOT EXISTS rspamd_urls (
+ Date Date,
+ Digest FixedString(32),
+ `Urls.Tld` Array(String),
+ `Urls.Url` Array(String)
+) ENGINE = MergeTree(Date, Digest, 8192);
+
+
+CREATE TABLE IF NOT EXISTS rspamd_emails (
+ Date Date,
+ Digest FixedString(32),
+ Emails Array(String)
+) ENGINE = MergeTree(Date, Digest, 8192);
+
+
+CREATE TABLE IF NOT EXISTS rspamd_asn (
+ Date Date,
+ Digest FixedString(32),
+ ASN String,
+ Country FixedString(2),
+ IPNet String
+) ENGINE = MergeTree(Date, Digest, 8192);
+
+
+CREATE TABLE IF NOT EXISTS rspamd_symbols (
+ Date Date,
+ Digest FixedString(32),
+ `Symbols.Names` Array(String),
+ `Symbols.Scores` Array(Float64),
+ `Symbols.Options` Array(String)
+) ENGINE = MergeTree(Date, Digest, 8192);
+
+
diff --git a/test/functional/data/schema_2/data.rspamd.sql b/test/functional/data/schema_2/data.rspamd.sql
new file mode 100644
index 0000000..5bcfd24
--- /dev/null
+++ b/test/functional/data/schema_2/data.rspamd.sql
@@ -0,0 +1,56 @@
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+(toDate(now()),'2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
+('2017-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','',[],[],[],[],[],[],[],'','\0\0','',[],[],[],'b3e5e01c86a8fd8a9bc5a66d033cc32e'),
diff --git a/test/functional/data/schema_2/schema.sql b/test/functional/data/schema_2/schema.sql
new file mode 100644
index 0000000..a0df35a
--- /dev/null
+++ b/test/functional/data/schema_2/schema.sql
@@ -0,0 +1,43 @@
+CREATE TABLE rspamd
+(
+ Date Date,
+ TS DateTime,
+ From String,
+ MimeFrom String,
+ IP String,
+ Score Float64,
+ NRcpt UInt8,
+ Size UInt32,
+ IsWhitelist Enum8('blacklist' = 0, 'whitelist' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('blacklist' = 0, 'whitelist' = 1, 'unknown' = 2)),
+ IsBayes Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2)),
+ IsFuzzy Enum8('whitelist' = 0, 'deny' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('whitelist' = 0, 'deny' = 1, 'unknown' = 2)),
+ IsFann Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2)),
+ IsDkim Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2)),
+ IsDmarc Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2)),
+ NUrls Int32,
+ Action Enum8('reject' = 0, 'rewrite subject' = 1, 'add header' = 2, 'greylist' = 3, 'no action' = 4, 'soft reject' = 5) DEFAULT CAST('no action' AS Enum8('reject' = 0, 'rewrite subject' = 1, 'add header' = 2, 'greylist' = 3, 'no action' = 4, 'soft reject' = 5)),
+ FromUser String,
+ MimeUser String,
+ RcptUser String,
+ RcptDomain String,
+ ListId String,
+ Subject String,
+ `Attachments.FileName` Array(String),
+ `Attachments.ContentType` Array(String),
+ `Attachments.Length` Array(UInt32),
+ `Attachments.Digest` Array(FixedString(16)),
+ `Urls.Tld` Array(String),
+ `Urls.Url` Array(String),
+ Emails Array(String),
+ ASN String,
+ Country FixedString(2),
+ IPNet String,
+ `Symbols.Names` Array(String),
+ `Symbols.Scores` Array(Float64),
+ `Symbols.Options` Array(String),
+ Digest FixedString(32)
+) ENGINE = MergeTree(Date, (TS, From), 8192);
+
+CREATE TABLE rspamd_version ( Version UInt32) ENGINE = TinyLog;
+
+INSERT INTO rspamd_version (Version) Values (3); \ No newline at end of file
diff --git a/test/functional/lib/.gitignore b/test/functional/lib/.gitignore
new file mode 100644
index 0000000..0d20b64
--- /dev/null
+++ b/test/functional/lib/.gitignore
@@ -0,0 +1 @@
+*.pyc
diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py
new file mode 100644
index 0000000..3976ca9
--- /dev/null
+++ b/test/functional/lib/rspamd.py
@@ -0,0 +1,357 @@
+from urllib.request import urlopen
+import glob
+import grp
+import http.client
+import os
+import os.path
+import psutil
+import pwd
+import shutil
+import signal
+import socket
+import stat
+import sys
+import tempfile
+
+from robot.api import logger
+from robot.libraries.BuiltIn import BuiltIn
+import demjson
+
+def Check_JSON(j):
+ d = demjson.decode(j, strict=True)
+ logger.debug('got json %s' % d)
+ assert len(d) > 0
+ assert 'error' not in d
+ return d
+
+def check_json_log(fn):
+ line_count = 0
+ f = open(fn, 'r')
+ for l in f.readlines():
+ d = demjson.decode(l, strict=True)
+ assert len(d) > 0
+ line_count = line_count + 1
+ assert line_count > 0
+
+def cleanup_temporary_directory(directory):
+ shutil.rmtree(directory)
+
+def save_run_results(directory, filenames):
+ current_directory = os.getcwd()
+ suite_name = BuiltIn().get_variable_value("${SUITE_NAME}")
+ test_name = BuiltIn().get_variable_value("${TEST NAME}")
+ if os.path.exists(directory):
+ onlyfiles = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))]
+ logger.debug('%s content before cleanup: %s' % (directory, onlyfiles))
+ if test_name is None:
+ # this is suite-level tear down
+ destination_directory = "%s/robot-save/%s" % (current_directory, suite_name)
+ else:
+ destination_directory = "%s/robot-save/%s/%s" % (current_directory, suite_name, test_name)
+ if not os.path.isdir(destination_directory):
+ os.makedirs(destination_directory)
+ for file in filenames.split(' '):
+ source_file = "%s/%s" % (directory, file)
+ logger.debug('check if we can save %s' % source_file)
+ if os.path.isfile(source_file):
+ logger.debug('found %s, save it' % file)
+ shutil.copy(source_file, "%s/%s" % (destination_directory, file))
+ shutil.copy(source_file, "%s/robot-save/%s.last" % (current_directory, file))
+
+def encode_filename(filename):
+ return "".join(['%%%0X' % ord(b) for b in filename])
+
+def get_test_directory():
+ return os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "../../")
+
+def get_top_dir():
+ if os.environ.get('RSPAMD_TOPDIR'):
+ return os.environ['RSPAMD_TOPDIR']
+
+ return get_test_directory() + "/../../"
+
+def get_install_root():
+ if os.environ.get('RSPAMD_INSTALLROOT'):
+ return os.path.abspath(os.environ['RSPAMD_INSTALLROOT'])
+
+ return os.path.abspath("../install/")
+
+def get_rspamd():
+ if os.environ.get('RSPAMD'):
+ return os.environ['RSPAMD']
+ if os.environ.get('RSPAMD_INSTALLROOT'):
+ return os.environ['RSPAMD_INSTALLROOT'] + "/bin/rspamd"
+ dname = get_top_dir()
+ return dname + "/src/rspamd"
+
+def get_rspamc():
+ if os.environ.get('RSPAMC'):
+ return os.environ['RSPAMC']
+ if os.environ.get('RSPAMD_INSTALLROOT'):
+ return os.environ['RSPAMD_INSTALLROOT'] + "/bin/rspamc"
+ dname = get_top_dir()
+ return dname + "/src/client/rspamc"
+
+def get_rspamadm():
+ if os.environ.get('RSPAMADM'):
+ return os.environ['RSPAMADM']
+ if os.environ.get('RSPAMD_INSTALLROOT'):
+ return os.environ['RSPAMD_INSTALLROOT'] + "/bin/rspamadm"
+ dname = get_top_dir()
+ return dname + "/src/rspamadm/rspamadm"
+
+def HTTP(method, host, port, path, data=None, headers={}):
+ c = http.client.HTTPConnection("%s:%s" % (host, port))
+ c.request(method, path, data, headers)
+ r = c.getresponse()
+ t = r.read()
+ s = r.status
+ c.close()
+ return [s, t]
+
+def hard_link(src, dst):
+ os.link(src, dst)
+
+def make_temporary_directory():
+ """Creates and returns a unique temporary directory
+
+ Example:
+ | ${RSPAMD_TMPDIR} = | Make Temporary Directory |
+ """
+ dirname = tempfile.mkdtemp()
+ os.chmod(dirname, stat.S_IRUSR |
+ stat.S_IXUSR |
+ stat.S_IWUSR |
+ stat.S_IRGRP |
+ stat.S_IXGRP |
+ stat.S_IROTH |
+ stat.S_IXOTH)
+ return dirname
+
+def make_temporary_file():
+ return tempfile.mktemp()
+
+def path_splitter(path):
+ dirname = os.path.dirname(path)
+ basename = os.path.basename(path)
+ return [dirname, basename]
+
+def rspamc(addr, port, filename):
+ mboxgoo = b"From MAILER-DAEMON Fri May 13 19:17:40 2016\r\n"
+ goo = open(filename, 'rb').read()
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.connect((addr, port))
+ s.send(b"CHECK RSPAMC/1.0\r\nContent-length: ")
+ s.send(str(len(goo+mboxgoo)).encode('utf-8'))
+ s.send(b"\r\n\r\n")
+ s.send(mboxgoo)
+ s.send(goo)
+ r = s.recv(2048)
+ return r.decode('utf-8')
+
+def Scan_File(filename, **headers):
+ addr = BuiltIn().get_variable_value("${RSPAMD_LOCAL_ADDR}")
+ port = BuiltIn().get_variable_value("${RSPAMD_PORT_NORMAL}")
+ headers["Queue-Id"] = BuiltIn().get_variable_value("${TEST_NAME}")
+ c = http.client.HTTPConnection("%s:%s" % (addr, port))
+ c.request("POST", "/checkv2", open(filename, "rb"), headers)
+ r = c.getresponse()
+ assert r.status == 200
+ d = demjson.decode(r.read())
+ c.close()
+ BuiltIn().set_test_variable("${SCAN_RESULT}", d)
+ return
+
+def Send_SIGUSR1(pid):
+ pid = int(pid)
+ os.kill(pid, signal.SIGUSR1)
+
+def set_directory_ownership(path, username, groupname):
+ if os.getuid() == 0:
+ uid=pwd.getpwnam(username).pw_uid
+ gid=grp.getgrnam(groupname).gr_gid
+ os.chown(path, uid, gid)
+
+def spamc(addr, port, filename):
+ goo = open(filename, 'rb').read()
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.connect((addr, port))
+ s.send(b"SYMBOLS SPAMC/1.0\r\nContent-length: ")
+ s.send(str(len(goo)).encode('utf-8'))
+ s.send(b"\r\n\r\n")
+ s.send(goo)
+ s.shutdown(socket.SHUT_WR)
+ r = s.recv(2048)
+ return r.decode('utf-8')
+
+def TCP_Connect(addr, port):
+ """Attempts to open a TCP connection to specified address:port
+
+ Example:
+ | Wait Until Keyword Succeeds | 5s | 10ms | TCP Connect | localhost | 8080 |
+ """
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.settimeout(5) # seconds
+ s.connect((addr, port))
+ s.close()
+
+def ping_rspamd(addr, port):
+ return str(urlopen("http://%s:%s/ping" % (addr, port)).read())
+
+def redis_check(addr, port):
+ """Attempts to open a TCP connection to specified address:port
+
+ Example:
+ | Wait Until Keyword Succeeds | 5s | 10ms | TCP Connect | localhost | 8080 |
+ """
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.settimeout(1.0) # seconds
+ s.connect((addr, port))
+ if s.sendall(b"ECHO TEST\n"):
+ result = s.recv(128)
+ return result == b'TEST\n'
+ else:
+ return False
+
+def update_dictionary(a, b):
+ a.update(b)
+ return a
+
+
+TERM_TIMEOUT = 10 # wait after sending a SIGTERM signal
+KILL_WAIT = 20 # additional wait after sending a SIGKILL signal
+
+def shutdown_process(process):
+ # send SIGTERM
+ process.terminate()
+
+ try:
+ process.wait(TERM_TIMEOUT)
+ return
+ except psutil.TimeoutExpired:
+ logger.info( "PID {} is not terminated in {} seconds, sending SIGKILL...".format(process.pid, TERM_TIMEOUT))
+ try:
+ # send SIGKILL
+ process.kill()
+ except psutil.NoSuchProcess:
+ # process exited just before we tried to kill
+ return
+
+ try:
+ process.wait(KILL_WAIT)
+ except psutil.TimeoutExpired:
+ raise RuntimeError("Failed to shutdown process {} ({})".format(process.pid, process.name()))
+
+
+def shutdown_process_with_children(pid):
+ pid = int(pid)
+ try:
+ process = psutil.Process(pid=pid)
+ except psutil.NoSuchProcess:
+ return
+ children = process.children(recursive=True)
+ shutdown_process(process)
+ for child in children:
+ try:
+ child.kill()
+ except psutil.NoSuchProcess:
+ pass
+ psutil.wait_procs(children, timeout=KILL_WAIT)
+
+def write_to_stdin(process_handle, text):
+ if not isinstance(text, bytes):
+ text = bytes(text, 'utf-8')
+ lib = BuiltIn().get_library_instance('Process')
+ obj = lib.get_process_object()
+ obj.stdin.write(text + b"\n")
+ obj.stdin.flush()
+ obj.stdin.close()
+ out = obj.stdout.read(4096)
+ return out.decode('utf-8')
+
+def get_file_if_exists(file_path):
+ if os.path.exists(file_path):
+ with open(file_path, 'r') as myfile:
+ return myfile.read()
+ return None
+
+def _merge_luacov_stats(statsfile, coverage):
+ """
+ Reads a coverage stats file written by luacov and merges coverage data to
+ 'coverage' dict: { src_file: hits_list }
+
+ Format of the file defined in:
+ https://github.com/keplerproject/luacov/blob/master/src/luacov/stats.lua
+ """
+ with open(statsfile, 'r') as fh:
+ while True:
+ # max_line:filename
+ line = fh.readline().rstrip()
+ if not line:
+ break
+
+ max_line, src_file = line.split(':')
+ counts = [int(x) for x in fh.readline().split()]
+ assert len(counts) == int(max_line)
+
+ if src_file in coverage:
+ # enlarge list if needed: lenght of list in different luacov.stats.out files may differ
+ old_len = len(coverage[src_file])
+ new_len = len(counts)
+ if new_len > old_len:
+ coverage[src_file].extend([0] * (new_len - old_len))
+ # sum execution counts for each line
+ for l, exe_cnt in enumerate(counts):
+ coverage[src_file][l] += exe_cnt
+ else:
+ coverage[src_file] = counts
+
+
+def _dump_luacov_stats(statsfile, coverage):
+ """
+ Saves data to the luacov stats file. Existing file is overwritted if exists.
+ """
+ src_files = sorted(coverage)
+
+ with open(statsfile, 'w') as fh:
+ for src in src_files:
+ stats = " ".join(str(n) for n in coverage[src])
+ fh.write("%s:%s\n%s\n" % (len(coverage[src]), src, stats))
+
+
+# File used by luacov to collect coverage stats
+LUA_STATSFILE = "luacov.stats.out"
+
+
+def collect_lua_coverage():
+ """
+ Merges ${RSPAMD_TMPDIR}/*.luacov.stats.out into luacov.stats.out
+
+ Example:
+ | Collect Lua Coverage |
+ """
+ # decided not to do optional coverage so far
+ #if not 'ENABLE_LUA_COVERAGE' in os.environ['HOME']:
+ # logger.info("ENABLE_LUA_COVERAGE is not present in env, will not collect Lua coverage")
+ # return
+
+ tmp_dir = BuiltIn().get_variable_value("${RSPAMD_TMPDIR}")
+
+ coverage = {}
+ input_files = []
+
+ for f in glob.iglob("%s/*.luacov.stats.out" % tmp_dir):
+ _merge_luacov_stats(f, coverage)
+ input_files.append(f)
+
+ if input_files:
+ if os.path.isfile(LUA_STATSFILE):
+ _merge_luacov_stats(LUA_STATSFILE, coverage)
+ _dump_luacov_stats(LUA_STATSFILE, coverage)
+ logger.info("%s merged into %s" % (", ".join(input_files), LUA_STATSFILE))
+ else:
+ logger.info("no *.luacov.stats.out files found in %s" % tmp_dir)
+
+
+def file_exists(file):
+ return os.path.isfile(file)
diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot
new file mode 100644
index 0000000..696b5f9
--- /dev/null
+++ b/test/functional/lib/rspamd.robot
@@ -0,0 +1,398 @@
+*** Settings ***
+Library Collections
+Library OperatingSystem
+Library Process
+
+*** Keywords ***
+Check Controller Errors
+ @{result} = HTTP GET ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_CONTROLLER} /errors
+ Should Be Equal As Integers ${result}[0] 200
+ Log ${result}[1]
+
+Check Pidfile
+ [Arguments] ${pidfile} ${timeout}=1 min
+ Wait Until Created ${pidfile} timeout=${timeout}
+ ${size} = Get File Size ${pidfile}
+ Should Not Be Equal As Integers ${size} 0
+
+Check Rspamc
+ [Arguments] ${result} @{args} &{kwargs}
+ IF ${result.rc} != 0
+ Log ${result.stderr}
+ END
+ ${has_rc} = Evaluate 'rc' in $kwargs
+ ${inverse} = Evaluate 'inverse' in $kwargs
+ ${re} = Evaluate 're' in $kwargs
+ ${rc} = Set Variable If ${has_rc} == True ${kwargs}[rc] 0
+ FOR ${i} IN @{args}
+ IF ${re} == True
+ Check Rspamc Match Regexp ${result.stdout} ${i} ${inverse}
+ ELSE
+ Check Rspamc Match String ${result.stdout} ${i} ${inverse}
+ END
+ END
+ IF @{args} == @{EMPTY}
+ Check Rspamc Match Default ${result.stdout} ${inverse}
+ END
+ Should Be Equal As Integers ${result.rc} ${rc}
+
+Check Rspamc Match Default
+ [Arguments] ${subject} ${inverse}
+ IF ${inverse} == False
+ Should Contain ${subject} success = true
+ ELSE
+ Should Not Contain ${subject} success = true
+ END
+
+Check Rspamc Match Regexp
+ [Arguments] ${subject} ${re} ${inverse}
+ IF ${inverse} == False
+ Should Match Regexp ${subject} ${re}
+ ELSE
+ Should Not Match Regexp ${subject} ${re}
+ END
+
+Check Rspamc Match String
+ [Arguments] ${subject} ${str} ${inverse}
+ IF ${inverse} == False
+ Should Contain ${subject} ${str}
+ ELSE
+ Should Not Contain ${subject} ${str}
+ END
+
+Do Not Expect Symbol
+ [Arguments] ${symbol}
+ Dictionary Should Not Contain Key ${SCAN_RESULT}[symbols] ${symbol}
+ ... msg=Symbol ${symbol} was not expected to be found in result
+
+Do Not Expect Symbols
+ [Arguments] @{symbols}
+ FOR ${symbol} IN @{symbols}
+ Dictionary Should Not Contain Key ${SCAN_RESULT}[symbols] ${symbol}
+ ... msg=Symbol ${symbol} was not expected to be found in result
+ END
+
+Expect Action
+ [Arguments] ${action}
+ Should Be Equal ${SCAN_RESULT}[action] ${action}
+
+Expect Email
+ [Arguments] ${email}
+ List Should Contain Value ${SCAN_RESULT}[emails] ${email}
+
+Expect Required Score
+ [Arguments] ${required_score}
+ Should Be Equal As Numbers ${SCAN_RESULT}[required_score] ${required_score}
+
+Expect Required Score To Be Null
+ Should Be Equal ${SCAN_RESULT}[required_score] ${NONE}
+
+Expect Score
+ [Arguments] ${score}
+ Should Be Equal As Numbers ${SCAN_RESULT}[score] ${score}
+
+Expect Symbol
+ [Arguments] ${symbol}
+ Dictionary Should Contain Key ${SCAN_RESULT}[symbols] ${symbol}
+ ... msg=Symbol ${symbol} wasn't found in result
+
+Expect URL
+ [Arguments] ${url}
+ List Should Contain Value ${SCAN_RESULT}[urls] ${url}
+
+Expect Extended URL
+ [Arguments] ${url}
+ ${found_url} = Set Variable ${FALSE}
+ ${url_list} = Convert To List ${SCAN_RESULT}[urls]
+ FOR ${item} IN @{url_list}
+ ${d} = Convert To Dictionary ${item}
+ ${found_url} = Evaluate "${d}[url]" == "${url}"
+ Exit For Loop If ${found_url} == ${TRUE}
+ END
+ Should Be True ${found_url} msg="Expected URL was not found: ${url}"
+
+Expect Symbol With Exact Options
+ [Arguments] ${symbol} @{options}
+ Expect Symbol ${symbol}
+ ${have_options} = Convert To List ${SCAN_RESULT}[symbols][${symbol}][options]
+ Lists Should Be Equal ${have_options} ${options} ignore_order=True
+ ... msg="Symbol ${symbol} has options ${SCAN_RESULT}[symbols][${symbol}][options] but expected ${options}"
+
+Expect Symbol With Option
+ [Arguments] ${symbol} ${option}
+ Expect Symbol ${symbol}
+ ${have_options} = Convert To List ${SCAN_RESULT}[symbols][${symbol}][options]
+ Should Contain ${have_options} ${option}
+ ... msg="Options for symbol ${symbol} ${SCAN_RESULT}[symbols][${symbol}][options] doesn't contain ${option}"
+
+Expect Symbol With Score
+ [Arguments] ${symbol} ${score}
+ Dictionary Should Contain Key ${SCAN_RESULT}[symbols] ${symbol}
+ ... msg=Symbol ${symbol} wasn't found in result
+ Should Be Equal As Numbers ${SCAN_RESULT}[symbols][${symbol}][score] ${score}
+ ... msg="Symbol ${symbol} has score of ${SCAN_RESULT}[symbols][${symbol}][score] but expected ${score}"
+
+Expect Symbols
+ [Arguments] @{symbols}
+ FOR ${symbol} IN @{symbols}
+ Dictionary Should Contain Key ${SCAN_RESULT}[symbols] ${symbol}
+ ... msg=Symbol ${symbol} wasn't found in result
+ END
+
+Expect Symbols With Scores
+ [Arguments] &{symscores}
+ FOR ${key} ${value} IN &{symscores}
+ Dictionary Should Contain Key ${SCAN_RESULT}[symbols] ${key}
+ ... msg=Symbol ${key} wasn't found in result
+ Should Be Equal As Numbers ${SCAN_RESULT}[symbols][${key}][score] ${value}
+ ... msg="Symbol ${key} has score of ${SCAN_RESULT}[symbols][${key}][score] but expected ${value}"
+ END
+
+Expect Symbol With Score And Exact Options
+ [Arguments] ${symbol} ${score} @{options}
+ Expect Symbol With Exact Options ${symbol} @{options}
+ Expect Symbol With Score ${symbol} ${score}
+
+Export Rspamd Variables To Environment
+ &{all_vars} = Get Variables no_decoration=True
+ FOR ${k} ${v} IN &{all_vars}
+ IF '${k}'.startswith("RSPAMD_")
+ Set Environment Variable ${k} ${v}
+ END
+ END
+
+Export Scoped Variables
+ [Arguments] ${scope} &{vars}
+ IF '${scope}' == 'Test'
+ FOR ${k} ${v} IN &{vars}
+ Set Test Variable ${${k}} ${v}
+ END
+ ELSE IF '${scope}' == 'Suite'
+ FOR ${k} ${v} IN &{vars}
+ Set Suite Variable ${${k}} ${v}
+ END
+ ELSE IF '${scope}' == 'Global'
+ FOR ${k} ${v} IN &{vars}
+ Set Global Variable ${${k}} ${v}
+ END
+ ELSE
+ Fail message="Don't know what to do with scope: ${scope}"
+ END
+
+Log does not contain segfault record
+ ${log} = Get File ${RSPAMD_TMPDIR}/rspamd.log encoding_errors=ignore
+ Should not contain ${log} (Segmentation fault) msg=Segmentation fault detected
+
+Redis HSET
+ [Arguments] ${hash} ${key} ${value}
+ ${result} = Run Process redis-cli -h ${RSPAMD_REDIS_ADDR} -p ${RSPAMD_REDIS_PORT}
+ ... HSET ${hash} ${key} ${value}
+ IF ${result.rc} != 0
+ Log ${result.stderr}
+ END
+ Log ${result.stdout}
+ Should Be Equal As Integers ${result.rc} 0
+
+Redis SET
+ [Arguments] ${key} ${value}
+ ${result} = Run Process redis-cli -h ${RSPAMD_REDIS_ADDR} -p ${RSPAMD_REDIS_PORT}
+ ... SET ${key} ${value}
+ IF ${result.rc} != 0
+ Log ${result.stderr}
+ END
+ Log ${result.stdout}
+ Should Be Equal As Integers ${result.rc} 0
+
+Redis Teardown
+ ${redis_pid} = Get Variable Value ${REDIS_PID}
+ Shutdown Process With Children ${redis_pid}
+ Cleanup Temporary Directory ${REDIS_TMPDIR}
+
+Rspamd Setup
+ # Create and chown temporary directory
+ ${RSPAMD_TMPDIR} = Make Temporary Directory
+ Set Directory Ownership ${RSPAMD_TMPDIR} ${RSPAMD_USER} ${RSPAMD_GROUP}
+
+ # Export ${RSPAMD_TMPDIR} to appropriate scope according to ${RSPAMD_SCOPE}
+ Export Scoped Variables ${RSPAMD_SCOPE} RSPAMD_TMPDIR=${RSPAMD_TMPDIR}
+
+ Run Rspamd
+
+Rspamd Redis Setup
+ Run Redis
+ Rspamd Setup
+
+Rspamd Teardown
+ IF '${CONTROLLER_ERRORS}' == 'True'
+ Run Keyword And Warn On Failure Check Controller Errors
+ END
+ Shutdown Process With Children ${RSPAMD_PID}
+ Save Run Results ${RSPAMD_TMPDIR} configdump.stdout configdump.stderr rspamd.stderr rspamd.stdout rspamd.conf rspamd.log redis.log clickhouse-config.xml
+ Log does not contain segfault record
+ Collect Lua Coverage
+ Cleanup Temporary Directory ${RSPAMD_TMPDIR}
+
+Rspamd Redis Teardown
+ Rspamd Teardown
+ Redis Teardown
+
+Run Redis
+ ${RSPAMD_TMPDIR} = Make Temporary Directory
+ ${template} = Get File ${RSPAMD_TESTDIR}/configs/redis-server.conf
+ ${config} = Replace Variables ${template}
+ Create File ${RSPAMD_TMPDIR}/redis-server.conf ${config}
+ Log ${config}
+ ${result} = Run Process redis-server ${RSPAMD_TMPDIR}/redis-server.conf
+ IF ${result.rc} != 0
+ Log ${result.stderr}
+ END
+ Should Be Equal As Integers ${result.rc} 0
+ Wait Until Keyword Succeeds 5x 1 sec Check Pidfile ${RSPAMD_TMPDIR}/redis.pid timeout=0.5s
+ Wait Until Keyword Succeeds 5x 1 sec Redis Check ${RSPAMD_REDIS_ADDR} ${RSPAMD_REDIS_PORT}
+ ${REDIS_PID} = Get File ${RSPAMD_TMPDIR}/redis.pid
+ ${REDIS_PID} = Convert To Number ${REDIS_PID}
+ Export Scoped Variables ${REDIS_SCOPE} REDIS_PID=${REDIS_PID} REDIS_TMPDIR=${RSPAMD_TMPDIR}
+ ${redis_log} = Get File ${RSPAMD_TMPDIR}/redis.log
+ Log ${redis_log}
+
+Run Rspamd
+ Export Rspamd Variables To Environment
+
+ # Dump templated config or errors to log
+ ${result} = Run Process ${RSPAMADM}
+ ... --var\=TMPDIR\=${RSPAMD_TMPDIR}
+ ... --var\=DBDIR\=${RSPAMD_TMPDIR}
+ ... --var\=LOCAL_CONFDIR\=/non-existent
+ ... --var\=CONFDIR\=${RSPAMD_TESTDIR}/../../conf/
+ ... configdump -c ${CONFIG}
+ ... env:RSPAMD_LOCAL_CONFDIR=/non-existent
+ ... env:RSPAMD_TMPDIR=${RSPAMD_TMPDIR}
+ ... env:RSPAMD_CONFDIR=${RSPAMD_TESTDIR}/../../conf/
+ ... env:LD_LIBRARY_PATH=${RSPAMD_TESTDIR}/../../contrib/aho-corasick
+ ... env:RSPAMD_NO_CLEANUP=1
+ ... env:ASAN_OPTIONS=quarantine_size_mb=2048:malloc_context_size=20:fast_unwind_on_malloc=0:log_path=${RSPAMD_TMPDIR}/rspamd-asan
+ # We need to send output to files (or discard output) to avoid hanging Robot
+ ... stdout=${RSPAMD_TMPDIR}/configdump.stdout stderr=${RSPAMD_TMPDIR}/configdump.stderr
+ IF ${result.rc} == 0
+ ${configdump} = Get File ${RSPAMD_TMPDIR}/configdump.stdout encoding_errors=ignore
+ ELSE
+ ${configdump} = Get File ${RSPAMD_TMPDIR}/configdump.stderr encoding_errors=ignore
+ END
+ Log ${configdump}
+
+ # Fix directory ownership (maybe do this somewhere else)
+ Set Directory Ownership ${RSPAMD_TMPDIR} ${RSPAMD_USER} ${RSPAMD_GROUP}
+
+ # Run Rspamd
+ ${result} = Run Process ${RSPAMD} -u ${RSPAMD_USER} -g ${RSPAMD_GROUP}
+ ... -c ${CONFIG}
+ ... --var\=TMPDIR\=${RSPAMD_TMPDIR}
+ ... --var\=DBDIR\=${RSPAMD_TMPDIR}
+ ... --var\=LOCAL_CONFDIR\=/non-existent
+ ... --var\=CONFDIR\=${RSPAMD_TESTDIR}/../../conf/
+ ... env:RSPAMD_LOCAL_CONFDIR=/non-existent
+ ... env:RSPAMD_TMPDIR=${RSPAMD_TMPDIR}
+ ... env:RSPAMD_CONFDIR=${RSPAMD_TESTDIR}/../../conf/
+ ... env:LD_LIBRARY_PATH=${RSPAMD_TESTDIR}/../../contrib/aho-corasick
+ ... env:RSPAMD_NO_CLEANUP=1
+ ... env:ASAN_OPTIONS=quarantine_size_mb=2048:malloc_context_size=20:fast_unwind_on_malloc=0:log_path=${RSPAMD_TMPDIR}/rspamd-asan
+ ... stdout=${RSPAMD_TMPDIR}/rspamd.stdout stderr=${RSPAMD_TMPDIR}/rspamd.stderr
+
+ # Log stdout/stderr
+ ${rspamd_stdout} = Get File ${RSPAMD_TMPDIR}/rspamd.stdout encoding_errors=ignore
+ ${rspamd_stderror} = Get File ${RSPAMD_TMPDIR}/rspamd.stderr encoding_errors=ignore
+ Log ${rspamd_stdout}
+ Log ${rspamd_stderror}
+
+ # Abort if it failed
+ Should Be Equal As Integers ${result.rc} 0
+
+ # Wait for pid file to be written
+ Wait Until Keyword Succeeds 10x 1 sec Check Pidfile ${RSPAMD_TMPDIR}/rspamd.pid timeout=0.5s
+
+ # Confirm worker is reachable
+ Wait Until Keyword Succeeds 5x 1 sec Ping Rspamd ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_NORMAL}
+
+ # Read PID from PIDfile and export it to appropriate scope as ${RSPAMD_PID}
+ ${RSPAMD_PID} = Get File ${RSPAMD_TMPDIR}/rspamd.pid
+ Export Scoped Variables ${RSPAMD_SCOPE} RSPAMD_PID=${RSPAMD_PID}
+
+Run Nginx
+ ${template} = Get File ${RSPAMD_TESTDIR}/configs/nginx.conf
+ ${config} = Replace Variables ${template}
+ Create File ${RSPAMD_TMPDIR}/nginx.conf ${config}
+ Log ${config}
+ ${result} = Run Process nginx -c ${RSPAMD_TMPDIR}/nginx.conf
+ IF ${result.rc} != 0
+ Log ${result.stderr}
+ END
+ Should Be Equal As Integers ${result.rc} 0
+ Wait Until Keyword Succeeds 10x 1 sec Check Pidfile ${RSPAMD_TMPDIR}/nginx.pid timeout=0.5s
+ Wait Until Keyword Succeeds 5x 1 sec TCP Connect ${NGINX_ADDR} ${NGINX_PORT}
+ ${NGINX_PID} = Get File ${RSPAMD_TMPDIR}/nginx.pid
+ IF '${NGINX_SCOPE}' == 'Test'
+ Set Test Variable ${NGINX_PID}
+ ELSE IF '${NGINX_SCOPE}' == 'Suite'
+ Set Suite Variable ${NGINX_PID}
+ END
+ ${nginx_log} = Get File ${RSPAMD_TMPDIR}/nginx.log
+ Log ${nginx_log}
+
+Run Rspamc
+ [Arguments] @{args}
+ ${result} = Run Process ${RSPAMC} -t 60 --header Queue-ID\=${TEST NAME}
+ ... @{args} env:LD_LIBRARY_PATH=${RSPAMD_TESTDIR}/../../contrib/aho-corasick
+ Log ${result.stdout}
+ [Return] ${result}
+
+Scan File By Reference
+ [Arguments] ${filename} &{headers}
+ Set To Dictionary ${headers} File=${filename}
+ ${result} = Scan File /dev/null &{headers}
+ [Return] ${result}
+
+Scan Message With Rspamc
+ [Arguments] ${msg_file} @{vargs}
+ ${result} = Run Rspamc -p -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_NORMAL} @{vargs} ${msg_file}
+ [Return] ${result}
+
+Sync Fuzzy Storage
+ [Arguments] @{vargs}
+ ${len} = Get Length ${vargs}
+ IF $len == 0
+ ${result} = Run Process ${RSPAMADM} control -s
+ ... ${RSPAMD_TMPDIR}/rspamd.sock fuzzy_sync
+ ELSE
+ Run Process ${RSPAMADM} control -s ${vargs}[0]/rspamd.sock
+ ... fuzzy_sync
+ END
+ Log ${result.stdout}
+ Sleep 0.1s Try give fuzzy storage time to sync
+
+Run Dummy Http
+ ${fileExists} = File Exists /tmp/dummy_http.pid
+ IF ${fileExists} is True
+ ${http_pid} = Get File /tmp/dummy_http.pid
+ Shutdown Process With Children ${http_pid}
+ END
+ ${result} = Start Process ${RSPAMD_TESTDIR}/util/dummy_http.py -pf /tmp/dummy_http.pid
+ Wait Until Created /tmp/dummy_http.pid timeout=2 second
+
+Run Dummy Https
+ ${fileExists} = File Exists /tmp/dummy_https.pid
+ IF ${fileExists} is True
+ ${http_pid} = Get File /tmp/dummy_https.pid
+ Shutdown Process With Children ${http_pid}
+ END
+ ${result} = Start Process ${RSPAMD_TESTDIR}/util/dummy_http.py
+ ... -c ${RSPAMD_TESTDIR}/util/server.pem -k ${RSPAMD_TESTDIR}/util/server.pem
+ ... -pf /tmp/dummy_https.pid -p 18081
+ Wait Until Created /tmp/dummy_https.pid timeout=2 second
+
+Dummy Http Teardown
+ ${http_pid} = Get File /tmp/dummy_http.pid
+ Shutdown Process With Children ${http_pid}
+
+Dummy Https Teardown
+ ${https_pid} = Get File /tmp/dummy_https.pid
+ Shutdown Process With Children ${https_pid}
diff --git a/test/functional/lib/vars.py b/test/functional/lib/vars.py
new file mode 100644
index 0000000..0a12a81
--- /dev/null
+++ b/test/functional/lib/vars.py
@@ -0,0 +1,30 @@
+import shutil
+import socket
+
+CONTROLLER_ERRORS = True
+HAVE_MILTERTEST = shutil.which('miltertest') and True or False
+RSPAMD_EXTERNAL_RELAY_ENABLED = False
+RSPAMD_KEY_PVT1 = 'ekd3x36tfa5gd76t6pa8hqif3ott7n1siuux68exbkk7ukscte9y'
+RSPAMD_KEY_PUB1 = 'm8kneubpcjsb8sbsoj7jy7azj9fdd3xmj63txni86a8ye9ncomny'
+RSPAMD_LOCAL_ADDR = '127.0.0.1'
+RSPAMD_MAP_WATCH_INTERVAL = '1min'
+RSPAMD_PORT_CONTROLLER = 56790
+RSPAMD_PORT_CONTROLLER_SLAVE = 56793
+RSPAMD_PORT_FUZZY = 56791
+RSPAMD_PORT_FUZZY_SLAVE = 56792
+RSPAMD_PORT_NORMAL = 56789
+RSPAMD_PORT_NORMAL_SLAVE = 56794
+RSPAMD_PORT_PROXY = 56795
+RSPAMD_PORT_CLAM = 2100
+RSPAMD_PORT_FPROT = 2101
+RSPAMD_PORT_FPROT2_DUPLICATE = 2102
+RSPAMD_PORT_AVAST = 2103
+RSPAMD_P0F_SOCKET = '/tmp/p0f.sock'
+RSPAMD_REDIS_ADDR = '127.0.0.1'
+RSPAMD_REDIS_PORT = 56379
+RSPAMD_NGINX_ADDR = '127.0.0.1'
+RSPAMD_NGINX_PORT = 56380
+RSPAMD_GROUP = 'nogroup'
+RSPAMD_USER = 'nobody'
+SOCK_DGRAM = socket.SOCK_DGRAM
+SOCK_STREAM = socket.SOCK_STREAM
diff --git a/test/functional/lua/composites.lua b/test/functional/lua/composites.lua
new file mode 100644
index 0000000..648eda0
--- /dev/null
+++ b/test/functional/lua/composites.lua
@@ -0,0 +1,140 @@
+rspamd_config:register_symbol({
+ name = 'EXPRESSIONS_B',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+
+rspamd_config:register_symbol({
+ name = 'POLICY_REMOVE_WEIGHT_A',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'POLICY_REMOVE_WEIGHT_B',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'POLICY_FORCE_REMOVE_A',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'POLICY_FORCE_REMOVE_B',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'POLICY_LEAVE_A',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'POLICY_LEAVE_B',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+
+rspamd_config:register_symbol({
+ name = 'DEFAULT_POLICY_REMOVE_WEIGHT_A',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'DEFAULT_POLICY_REMOVE_WEIGHT_B',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'DEFAULT_POLICY_REMOVE_SYMBOL_A',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'DEFAULT_POLICY_REMOVE_SYMBOL_B',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'DEFAULT_POLICY_LEAVE_A',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'DEFAULT_POLICY_LEAVE_B',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+
+rspamd_config:register_symbol({
+ name = 'POSITIVE_A',
+ score = -1.0,
+ group = "positive",
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'NEGATIVE_A',
+ score = -1.0,
+ group = "negative",
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'NEGATIVE_B',
+ score = 1.0,
+ group = "negative",
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+rspamd_config:register_symbol({
+ name = 'ANY_A',
+ score = -1.0,
+ group = "any",
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+
+rspamd_config:register_symbol({
+ name = 'OPTS',
+ score = -1.0,
+ group = "any",
+ callback = function(task)
+ local lua_util = require "lua_util"
+ local woot = lua_util.str_split(tostring(task:get_request_header('opts') or ''), ',')
+
+ if woot and #woot > 0 and #woot[1] > 0 then
+ return true, 1.0, woot
+ end
+ end
+})
diff --git a/test/functional/lua/conditions.lua b/test/functional/lua/conditions.lua
new file mode 100644
index 0000000..2baa04b
--- /dev/null
+++ b/test/functional/lua/conditions.lua
@@ -0,0 +1,22 @@
+local logger = require 'rspamd_logger'
+
+rspamd_config:register_symbol({
+ name = 'ANY_A',
+ score = -1.0,
+ group = "any",
+ callback = function()
+ return true, 'hello3'
+ end
+})
+
+rspamd_config:add_condition('ANY_A', function(task)
+ logger.infox(task, 'hello from condition1')
+ task:insert_result('ANY_A', 1.0, 'hello1')
+ return true
+end)
+
+rspamd_config:add_condition('ANY_A', function(task)
+ logger.infox(task, 'hello from condition2')
+ task:insert_result('ANY_A', 1.0, 'hello2')
+ return true
+end)
diff --git a/test/functional/lua/deps.lua b/test/functional/lua/deps.lua
new file mode 100644
index 0000000..6171db6
--- /dev/null
+++ b/test/functional/lua/deps.lua
@@ -0,0 +1,30 @@
+local cb = function(task)
+ task:insert_result('TOP', 1.0)
+end
+
+local cb_dep1 = function(task)
+ if task:get_symbol('TOP') then
+ task:insert_result('DEP1', 1.0)
+ end
+end
+
+local cb_gen = function(num)
+ local cb_dep = function(task)
+ if task:get_symbol('DEP' .. tostring(num)) then
+ task:insert_result('DEP' .. tostring(num + 1), 1.0)
+ end
+ end
+
+ return cb_dep
+end
+
+local id = rspamd_config:register_callback_symbol(1.0, cb)
+rspamd_config:register_virtual_symbol('TOP', 1.0, id)
+
+rspamd_config:register_symbol('DEP1', 1.0, cb_dep1)
+rspamd_config:register_dependency('DEP1', 'TOP')
+
+for i = 2,10 do
+ rspamd_config:register_symbol('DEP' .. tostring(i), 1.0, cb_gen(i - 1))
+ rspamd_config:register_dependency('DEP' .. tostring(i), 'DEP' .. tostring(i - 1))
+end
diff --git a/test/functional/lua/dns.lua b/test/functional/lua/dns.lua
new file mode 100644
index 0000000..702b985
--- /dev/null
+++ b/test/functional/lua/dns.lua
@@ -0,0 +1,53 @@
+local rspamd_dns = require "rspamd_dns"
+local logger = require "rspamd_logger"
+
+local function dns_sync_symbol(task)
+ local to_resolve = tostring(task:get_request_header('to-resolve'))
+ local is_ok, results = rspamd_dns.request({
+ task = task,
+ type = 'a',
+ name = to_resolve ,
+ })
+
+ logger.errx(task, "is_ok=%1, results=%2, results[1]=%3", is_ok, results, results[1])
+
+ if not is_ok then
+ task:insert_result('DNS_SYNC_ERROR', 1.0, results)
+ else
+ task:insert_result('DNS_SYNC', 1.0, tostring(results[1]))
+ end
+end
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE_DNS_SYNC',
+ score = 1.0,
+ callback = dns_sync_symbol,
+ no_squeeze = true,
+ flags = 'coro',
+})
+
+
+-- Async request
+local function dns_symbol(task)
+ local function dns_cb(_, to_resolve, results, err)
+ logger.errx(task, "_=%1, to_resolve=%2, results=%3, err%4", _, to_resolve, results, err)
+ if err then
+ task:insert_result('DNS_ERROR', 1.0, err)
+ else
+ task:insert_result('DNS', 1.0, tostring(results[1]))
+ end
+ end
+ local to_resolve = tostring(task:get_request_header('to-resolve'))
+
+ task:get_resolver():resolve_a({
+ task = task,
+ name = to_resolve,
+ callback = dns_cb
+ })
+end
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE_DNS',
+ score = 1.0,
+ callback = dns_symbol,
+}) \ No newline at end of file
diff --git a/test/functional/lua/external_relay.lua b/test/functional/lua/external_relay.lua
new file mode 100644
index 0000000..6aa3a29
--- /dev/null
+++ b/test/functional/lua/external_relay.lua
@@ -0,0 +1,10 @@
+rspamd_config:register_symbol({
+ name = 'EXTERNAL_RELAY_TEST',
+ score = 0.0,
+ callback = function(task)
+ local from_ip = string.format('IP=%s', task:get_from_ip() or 'NIL')
+ local hostname = string.format('HOSTNAME=%s', task:get_hostname() or 'NIL')
+ local helo = string.format('HELO=%s', task:get_helo() or 'NIL')
+ return true, from_ip, hostname, helo
+ end
+})
diff --git a/test/functional/lua/flags.lua b/test/functional/lua/flags.lua
new file mode 100644
index 0000000..5d01ab1
--- /dev/null
+++ b/test/functional/lua/flags.lua
@@ -0,0 +1,35 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+]]--
+
+rspamd_config:register_post_filter(function(task)
+ task:set_flag('no_stat')
+ task:set_flag('no_log', false)
+ task:set_flag('no_log', true)
+ task:set_flag('skip', true)
+ task:set_flag('extended_urls')
+
+ task:insert_result('FLAGS_SYM', 1.0, table.concat(task:get_flags(), ','))
+end)
diff --git a/test/functional/lua/get_from.lua b/test/functional/lua/get_from.lua
new file mode 100644
index 0000000..9282866
--- /dev/null
+++ b/test/functional/lua/get_from.lua
@@ -0,0 +1,10 @@
+rspamd_config:register_symbol({
+ name = 'GET_FROM',
+ score = 1.0,
+ callback = function(task)
+ local a = task:get_from('mime')
+ if not a then return end
+ a = a[1]
+ return true, (a.name or '') .. ',' .. (a.addr or '') .. ',' .. (a.user or '') .. ',' .. (a.domain or '')
+ end
+})
diff --git a/test/functional/lua/hashes.lua b/test/functional/lua/hashes.lua
new file mode 100644
index 0000000..7a1728b
--- /dev/null
+++ b/test/functional/lua/hashes.lua
@@ -0,0 +1,74 @@
+rspamd_config:register_symbol({
+ name = 'TEST_HASHES',
+ score = 1.0,
+ callback = function()
+ local hash = require 'rspamd_cryptobox_hash'
+ local logger = require 'rspamd_logger'
+
+ local worry = {}
+ local test_data = {
+ {
+ ['str'] = 'asdf.qwerty.123',
+ ['hex'] = 'bf22dd95750034b9af93f0e4e5954aca3506bbcdbc051d91bd9af2d1a8fc294e848626b1c1751e58b44c4d3ea69dec5efa5a214dc59c77b1a9ca3bde3babac9d',
+ },
+ {
+ ['specific'] = 'md5',
+ ['str'] = 'asdf.qwerty.123',
+ ['hex'] = 'cf25ddc406c50de0c13de2b79d127646',
+ },
+ {
+ ['init'] = 'asdf.qwerty.123',
+ ['str'] = 'asdf.qwerty.123',
+ ['hex'] = 'bf22dd95750034b9af93f0e4e5954aca3506bbcdbc051d91bd9af2d1a8fc294e848626b1c1751e58b44c4d3ea69dec5efa5a214dc59c77b1a9ca3bde3babac9d',
+ ['reset'] = true,
+ },
+ {
+ ['init'] = 'asdf.qwerty.123',
+ ['str'] = 'asdf.qwerty.123',
+ ['hex'] = 'e445046aa21a705dcce1343795630f88bc0196a0070011fdce789d5a2a349a8f85349834ade555ca21439f65fdc4dbcf82dcff7fcc559ef11c508507515c1532',
+ },
+ {
+ ['init'] = 'asdf.qwerty.123',
+ ['specific'] = 'md5',
+ ['str'] = 'asdf.qwerty.123',
+ ['hex'] = '9ef941c4d050e43b1e665300f4fbe054',
+ },
+ {
+ ['init'] = 'asdf.qwerty.123',
+ ['specific'] = 'md5',
+ ['str'] = 'asdf.qwerty.123',
+ ['hex'] = 'cf25ddc406c50de0c13de2b79d127646',
+ ['reset'] = true,
+ },
+ {
+ ['init'] = 'hello',
+ ['specific'] = 'xxh3',
+ ['str'] = 'hello',
+ ['hex'] = 'c1156ae6cb7ff175',
+ }
+ }
+
+ for _, t in ipairs(test_data) do
+ local h
+ if not t['specific'] then
+ h = hash.create(t['init'])
+ else
+ h = hash.create_specific(t['specific'], t['init'])
+ end
+ if t['reset'] then
+ h:reset()
+ end
+ h:update(t['str'])
+ if h:hex() ~= t['hex'] then
+ t['error'] = 'sum mismatch: ' .. h:hex()
+ table.insert(worry, logger.slog('%1', t))
+ end
+ end
+
+ if (#worry == 0) then
+ return true, "no worry"
+ else
+ return true, table.concat(worry, ",")
+ end
+ end
+})
diff --git a/test/functional/lua/http.lua b/test/functional/lua/http.lua
new file mode 100644
index 0000000..c5b91ff
--- /dev/null
+++ b/test/functional/lua/http.lua
@@ -0,0 +1,169 @@
+local rspamd_http = require "rspamd_http"
+local rspamd_logger = require "rspamd_logger"
+
+local function http_symbol(task)
+
+ local url = tostring(task:get_request_header('url'))
+ local method = tostring(task:get_request_header('method'))
+ task:insert_result('method_' .. method, 1.0)
+
+ local function http_callback(err, code, body)
+ if err then
+ rspamd_logger.errx('http_callback error: ' .. err)
+ task:insert_result('HTTP_ERROR', 1.0, err)
+ else
+ task:insert_result('HTTP_' .. code, 1.0, body)
+ end
+ end
+
+ local function http_dns_callback(err, code, body)
+ if err then
+ rspamd_logger.errx('http_dns_callback error: ' .. err)
+ task:insert_result('HTTP_DNS_ERROR', 1.0, err)
+ else
+ task:insert_result('HTTP_DNS_' .. code, 1.0, body)
+ end
+ end
+
+ rspamd_logger.errx(task, 'do http request with callback')
+ rspamd_http.request({
+ url = 'http://127.0.0.1:18080' .. url,
+ task = task,
+ method = method,
+ callback = http_callback,
+ timeout = 1,
+ })
+
+ --[[ request to this address involved DNS resolver subsystem ]]
+ rspamd_logger.errx(task, 'do http request with callback + dns resolving')
+ rspamd_http.request({
+ url = 'http://site.resolveme:18080' .. url,
+ task = task,
+ method = method,
+ callback = http_dns_callback,
+ timeout = 1,
+ })
+
+ rspamd_logger.errx(task, 'rspamd_http.request[before]')
+
+ local err, response = rspamd_http.request({
+ url = 'http://127.0.0.1:18080' .. url,
+ task = task,
+ method = method,
+ timeout = 1,
+ })
+ rspamd_logger.errx(task, 'rspamd_http.request[done] err: %1 response:%2', err, response)
+
+ if not err then
+ task:insert_result('HTTP_CORO_' .. response.code, 1.0, response.content)
+ else
+ task:insert_result('HTTP_CORO_ERROR', 1.0, err)
+ end
+
+ rspamd_logger.errx(task, 'do http request after coroutine finished')
+ err, response = rspamd_http.request({
+ url = 'http://site.resolveme:18080' .. url,
+ task = task,
+ method = method,
+ timeout = 1,
+ })
+
+ if not err then
+ task:insert_result('HTTP_CORO_DNS_' .. response.code, 1.0, response.content)
+ else
+ task:insert_result('HTTP_CORO_DNS_ERROR', 1.0, err)
+ end
+end
+
+
+local function finish(task)
+ rspamd_logger.errx('function finish')
+ local err, response = rspamd_http.request({
+ url = 'http://site.resolveme:18080/timeout',
+ task = task,
+ method = 'get',
+ timeout = 1,
+ })
+ if err then
+ task:insert_result('HTTP_CORO_DNS_FINISH_ERROR', 1.0, err)
+ else
+ task:insert_result('HTTP_CORO_DNS_FINISH_' .. response.code, 1.0, response.content)
+ end
+end
+
+local function periodic(cfg, ev_base)
+ local err, response = rspamd_http.request({
+ url = 'http://site.resolveme:18080/request/periodic',
+ config = cfg,
+ })
+ if err then
+ rspamd_logger.errx('periodic err ' .. err)
+ else
+ rspamd_logger.errx('periodic success ' .. response.content)
+ end
+
+ return false
+end
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE_HTTP_TEST',
+ score = 1.0,
+ callback = http_symbol,
+ no_squeeze = true,
+ flags = 'coro'
+})
+
+local function http_large_symbol(task)
+ if task:get_queue_id() == 'SSL Large HTTP request' then
+ local data = {}
+ for i = 1,2 do
+ local st = {}
+ for j=1,300000 do
+ st[j] = 't'
+ end
+ data[i] = table.concat(st)
+ end
+ data[#data + 1] = '\n'
+
+ local function http_callback(err, code, body)
+ if err then
+ rspamd_logger.errx('http_callback error: ' .. err)
+ task:insert_result('HTTP_ERROR', 1.0, err)
+ else
+ task:insert_result('HTTP_SSL_LARGE', 1.0)
+ end
+ end
+ rspamd_http.request({
+ url = 'https://127.0.0.1:18081/',
+ task = task,
+ method = 'post',
+ callback = http_callback,
+ timeout = 10,
+ body = data,
+ no_ssl_verify = true,
+ })
+ end
+end
+rspamd_config:register_symbol({
+ name = 'LARGE_HTTP_TEST',
+ score = 1.0,
+ callback = http_large_symbol,
+ no_squeeze = true,
+ flags = 'coro'
+})
+
+rspamd_config:register_finish_script(finish)
+
+rspamd_config:add_on_load(function(cfg, ev_base, worker)
+ local err, response = rspamd_http.request({
+ url = 'http://site.resolveme:18080/request/add_on_load',
+ config = cfg,
+ })
+ if err then
+ rspamd_logger.errx('add_on_load err ' .. err)
+ else
+ rspamd_logger.errx('add_on_load success ' .. response.content)
+ end
+
+ rspamd_config:add_periodic(ev_base, 0, periodic, false)
+end)
diff --git a/test/functional/lua/magic.lua b/test/functional/lua/magic.lua
new file mode 100644
index 0000000..c0d5793
--- /dev/null
+++ b/test/functional/lua/magic.lua
@@ -0,0 +1,14 @@
+rspamd_config.MAGIC_SYM = {
+ callback = function(task)
+ local parts = task:get_parts()
+
+ for i,p in ipairs(parts) do
+ local ext = p:get_detected_ext()
+
+ if ext then
+ task:insert_result('MAGIC_SYM_' .. ext:upper() .. '_' .. tostring(i), 1.0)
+ end
+ end
+ end,
+ type = 'callback'
+} \ No newline at end of file
diff --git a/test/functional/lua/mapreload.lua b/test/functional/lua/mapreload.lua
new file mode 100644
index 0000000..ae20120
--- /dev/null
+++ b/test/functional/lua/mapreload.lua
@@ -0,0 +1,20 @@
+local test_map = rspamd_config:add_map ({
+ url = '${MAP_FILE}',
+ type = 'set',
+})
+
+rspamd_config:register_symbol({
+ name = 'MAP_SET_HIT_AND_MISS',
+ score = 1.0,
+ callback = function()
+ local has_example = test_map:get_key('example.com')
+ local has_rspamdtest = test_map:get_key('rspamd-test.com')
+ if has_example and not has_rspamdtest then
+ return true, 'example.com'
+ elseif has_rspamdtest and not has_example then
+ return true, 'rspamd.com'
+ else
+ return true, string.format('invalid: has_example=%s, has_rspamdtest=%s', has_example, has_rspamdtest)
+ end
+ end
+})
diff --git a/test/functional/lua/maps_kv.lua b/test/functional/lua/maps_kv.lua
new file mode 100644
index 0000000..b90f44d
--- /dev/null
+++ b/test/functional/lua/maps_kv.lua
@@ -0,0 +1,94 @@
+local rspamd_ip = require 'rspamd_ip'
+local rspamd_logger = require 'rspamd_logger'
+local lua_maps = require "lua_maps"
+
+local radix_map = rspamd_config:add_map ({
+ url = rspamd_env.RADIX_MAP,
+ type = 'radix',
+})
+
+local map_map = rspamd_config:add_map ({
+ url = rspamd_env.MAP_MAP,
+ type = 'map',
+})
+
+local regexp_map = rspamd_config:add_map ({
+ url = rspamd_env.REGEXP_MAP,
+ type = 'regexp',
+})
+
+rspamd_config:register_symbol({
+ name = 'RADIX_KV',
+ score = 1.0,
+ callback = function()
+ local sip = {'8.8.8.8', '::1', '192.168.1.1', '10.0.1.1'}
+ local expected = {'test one', 'another', '1', false}
+ for i = 1, #sip do
+ if (radix_map:get_key(rspamd_ip.from_string(sip[i])) ~= expected[i]) then
+ local rip = rspamd_ip.from_string(sip[i])
+ local val = radix_map:get_key(rip)
+ return true, rspamd_logger.slog('plain: get_key(%s) [%s] -> %s [%s] [expected %s]', rip, type(rip), val, type(val), expected[i])
+ end
+ if (radix_map:get_key(sip[i]) ~= expected[i]) then
+ local val = radix_map:get_key(sip[i])
+ return true, rspamd_logger.slog('string: get_key(%s) [%s] -> %s [%s] [expected %s]', sip[i], type(sip[i]), val, type(val), expected[i])
+ end
+ end
+ return true, 'no worry'
+ end
+})
+
+rspamd_config:register_symbol({
+ name = 'MAP_KV',
+ score = 1.0,
+ callback = function()
+ local str = {'foo', 'asdf.example.com', 'asdf', 'barf'}
+ local expected = {'bar', 'value', '', false}
+ for i = 1, #str do
+ if (map_map:get_key(str[i]) ~= expected[i]) then
+ local val = map_map:get_key(str[i])
+ return true, rspamd_logger.slog('get_key(%s) [%s] -> %s [%s] [expected %s]', str[i], type(str[i]), val, type(val), expected[i])
+ end
+ end
+ return true, 'no worry'
+ end,
+})
+
+rspamd_config:register_symbol({
+ name = 'REGEXP_KV',
+ score = 1.0,
+ callback = function()
+ local str = {'foo', 'asdf.example.com', 'asdf', 'barf'}
+ local expected = {'bar', 'value', '1', false}
+ for i = 1, #str do
+ if (regexp_map:get_key(str[i]) ~= expected[i]) then
+ local val = regexp_map:get_key(str[i])
+ return true, rspamd_logger.slog('get_key(%s) [%s] -> %s [%s] [expected %s]', str[i], type(str[i]), val, type(val), expected[i])
+ end
+ end
+ return true, 'no worry'
+ end,
+})
+
+local simple_ext_map = lua_maps.map_add_from_ucl({
+ external = true,
+ backend = "http://127.0.0.1:18080/map-simple",
+ method = "body",
+ encode = "json",
+}, '', 'external map')
+rspamd_config:register_symbol({
+ name = 'EXTERNAL_MAP',
+ score = 1.0,
+ callback = function(task)
+ local function cb(res, data, code)
+ if res then
+ task:insert_result('EXTERNAL_MAP', 1.0, string.format('+%s', data))
+ else
+ task:insert_result('EXTERNAL_MAP', 1.0, string.format('-%s:%s', code, data))
+ end
+ end
+ simple_ext_map:get_key({
+ key = "value",
+ }, cb, task)
+ end,
+})
diff --git a/test/functional/lua/miltertest/combined.lua b/test/functional/lua/miltertest/combined.lua
new file mode 100644
index 0000000..69fa2d6
--- /dev/null
+++ b/test/functional/lua/miltertest/combined.lua
@@ -0,0 +1,35 @@
+-- Combine tests
+
+dofile './lib.lua'
+dofile './data.lua'
+
+setup()
+
+local old_setup = setup
+local old_teardown = teardown
+
+local empty_function = function() end
+setup = empty_function
+teardown = empty_function
+
+local function shuffle(tbl)
+ local size = #tbl
+ for i = size, 1, -1 do
+ local rand = math.random(size)
+ tbl[i], tbl[rand] = tbl[rand], tbl[i]
+ end
+ return tbl
+end
+
+local files = {'mt1.lua','mt2.lua','mt3.lua','mt4.lua'}
+local num_files = #files
+for i = 1, num_files do
+ table.insert(files, files[i])
+end
+files = shuffle(files)
+
+for _, f in ipairs(files) do
+ dofile(f)
+end
+
+old_teardown()
diff --git a/test/functional/lua/miltertest/data.lua b/test/functional/lua/miltertest/data.lua
new file mode 100644
index 0000000..84b953f
--- /dev/null
+++ b/test/functional/lua/miltertest/data.lua
@@ -0,0 +1,26 @@
+innocuous_hdrs = {
+ ['Message-ID'] = '<20180202155326.Horde.GfEWpxCo_Dip2xJswIpQNgK@example.org>',
+ ['From'] = 'Andrew Lewis <nerf@example.org>',
+ ['To'] = 'nerf@example.org',
+ ['Subject'] = 'innocuous test message',
+ ['User-Agent'] = 'Horde Application Framework 5',
+ ['Content-Type'] = 'text/plain; charset=utf-8; format=flowed; DelSp=Yes',
+ ['MIME-Version'] = '1.0',
+ ['Content-Disposition'] = 'inline',
+ ['Date'] = 'Fri, 02 Feb 2018 15:53:26 +0200',
+}
+
+default_hdrs = {
+ ['Subject'] = 'spam message',
+}
+
+innocuous_msg = 'Hello Rupert'
+
+gtube = [[lo
+
+XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+
+thx]]
+
+gtube_add_header = string.gsub(gtube, "XJS", "YJS")
+gtube_rw_subject = string.gsub(gtube, "XJS", "ZJS")
diff --git a/test/functional/lua/miltertest/data_dkim.lua b/test/functional/lua/miltertest/data_dkim.lua
new file mode 100644
index 0000000..15adf15
--- /dev/null
+++ b/test/functional/lua/miltertest/data_dkim.lua
@@ -0,0 +1,23 @@
+multi_hdrs = {
+ ['Message-ID'] = '<a44q4StVFY04V4_4gOMYXjTgMDvmlSFzZxnoyJPHFwM@cacophony.za.org>',
+ ['From'] = 'Rspamd <foo@cacophony.za.org>',
+ ['To'] = 'nerf@example.org',
+ ['Subject'] = 'dkim test message',
+ ['User-Agent'] = 'Vi IMproved 8.1',
+ ['Content-Type'] = 'text/plain; charset=utf-8;',
+ ['MIME-Version'] = '1.0',
+ ['Date'] = 'Sat, 02 Feb 2019 10:34:54 +0000',
+}
+
+single_hdr = {
+ ['Message-ID'] = '<a44q4StVFY04V4_4gOMYXjTgMDvmlSFzZxnoyJPHFwM@cacophony.za.org>',
+ ['From'] = 'Rspamd <foo@invalid.za.org>',
+ ['To'] = 'nerf@example.org',
+ ['Subject'] = 'dkim test message',
+ ['User-Agent'] = 'Vi IMproved 8.1',
+ ['Content-Type'] = 'text/plain; charset=utf-8;',
+ ['MIME-Version'] = '1.0',
+ ['Date'] = 'Sat, 02 Feb 2019 10:34:54 +0000',
+}
+
+innocuous_msg = 'hello'
diff --git a/test/functional/lua/miltertest/dkim_many.lua b/test/functional/lua/miltertest/dkim_many.lua
new file mode 100644
index 0000000..70a3a1b
--- /dev/null
+++ b/test/functional/lua/miltertest/dkim_many.lua
@@ -0,0 +1,11 @@
+print('Check we get multiple dkim signatures')
+
+dofile './lib.lua'
+dofile './data_dkim.lua'
+
+setup()
+
+send_message(innocuous_msg, multi_hdrs, 'test-id', 'foo@cacophony.za.org', {'nerf@example.org'})
+check_headers(2)
+
+teardown()
diff --git a/test/functional/lua/miltertest/dkim_one.lua b/test/functional/lua/miltertest/dkim_one.lua
new file mode 100644
index 0000000..0c7def8
--- /dev/null
+++ b/test/functional/lua/miltertest/dkim_one.lua
@@ -0,0 +1,11 @@
+print('Check we get single dkim signature')
+
+dofile './lib.lua'
+dofile './data_dkim.lua'
+
+setup()
+
+send_message(innocuous_msg, single_hdr, 'test-id', 'foo@invalid.za.org', {'nerf@example.org'})
+check_headers(1)
+
+teardown()
diff --git a/test/functional/lua/miltertest/lib.lua b/test/functional/lua/miltertest/lib.lua
new file mode 100644
index 0000000..44dc76e
--- /dev/null
+++ b/test/functional/lua/miltertest/lib.lua
@@ -0,0 +1,119 @@
+function setup(c_ip, helo, hn)
+ if not c_ip then c_ip = "127.0.0.1" end
+ if not helo then helo = "it.is.i" end
+ if not hn then hn = "localhost" end
+ conn = mt.connect("inet:" .. port .. "@" .. host)
+ if conn == nil then
+ error "mt.connect() failed"
+ end
+ if mt.conninfo(conn, hn, c_ip) then
+ error "mt.conninfo() failed"
+ end
+ if mt.getreply(conn) ~= SMFIR_CONTINUE then
+ error "mt.conninfo() unexpected reply"
+ end
+ if mt.helo(conn, helo) then
+ error "mt.helo() failed"
+ end
+ if mt.getreply(conn) ~= SMFIR_CONTINUE then
+ error "mt.helo() unexpected reply"
+ end
+end
+
+function teardown()
+ if conn then
+ mt.disconnect(conn)
+ end
+ conn = nil
+end
+
+function send_message(body, hdrs, id, sender, rcpts)
+ mt.macro(conn, SMFIC_MAIL, "i", id or "test-id")
+ if mt.mailfrom(conn, sender or "sender@example.com") then
+ error "mt.mailfrom() failed"
+ end
+ if mt.getreply(conn) ~= SMFIR_CONTINUE then
+ error "mt.mailfrom() unexpected reply"
+ end
+ if not rcpts then
+ rcpts = {"rcpt@example.com"}
+ end
+ for _, r in ipairs(rcpts) do
+ mt.rcptto(conn, r)
+ end
+ if not hdrs then
+ hdrs = default_hdrs
+ end
+ if not hdrs['From'] then
+ hdrs['From'] = sender or "sender@example.com"
+ end
+ for k, v in pairs(hdrs) do
+ if mt.header(conn, k, v) then
+ error (string.format("mt.header(%s) failed", k))
+ end
+ end
+ if mt.eoh(conn) then
+ error "mt.eoh() failed"
+ end
+ if mt.getreply(conn) ~= SMFIR_CONTINUE then
+ error "mt.eoh() unexpected reply"
+ end
+ if mt.bodystring(conn, body .. "\r\n") then
+ error "mt.bodystring() failed"
+ end
+ if mt.getreply(conn) ~= SMFIR_CONTINUE then
+ error "mt.bodystring() unexpected reply"
+ end
+ if mt.eom(conn) then
+ error "mt.eom() failed"
+ end
+end
+
+function check_accept()
+ local rc = mt.getreply(conn)
+ if rc ~= SMFIR_ACCEPT then
+ error (string.format("mt.eom() unexpected reply: %s", rc))
+ end
+end
+
+function check_gtube(code, ecode, msg)
+ if not mt.eom_check(conn, MT_SMTPREPLY, code or '554', ecode or '5.7.1', msg or 'Gtube pattern') then
+ error "mt.eom_check() failed"
+ end
+ local rc = mt.getreply(conn)
+ if rc ~= SMFIR_REPLYCODE then
+ error (string.format("mt.eom() unexpected reply: %s", rc))
+ end
+end
+
+function check_defer(code, ecode, msg)
+ if not mt.eom_check(conn, MT_SMTPREPLY, code or '451', ecode or '4.7.1', msg or 'Try much later') then
+ error "mt.eom_check() failed"
+ end
+ local rc = mt.getreply(conn)
+ if rc ~= SMFIR_REPLYCODE then
+ error (string.format("mt.eom() unexpected reply: %s", rc))
+ end
+end
+
+function check_subject_rw(subj, tmpl)
+ if not subj then
+ subj = default_hdrs['Subject']
+ end
+ if not tmpl then
+ tmpl = "*** SPAM *** %s"
+ end
+ local new_subj = string.format(tmpl, subj)
+ if not mt.eom_check(conn, MT_HDRCHANGE, "Subject", new_subj) then
+ error "subject not rewritten"
+ end
+end
+
+function check_headers(count)
+ for i=0, count-1 do
+ local hdr = mt.getheader(conn, "DKIM-Signature", i)
+ if not hdr then
+ error (string.format("Signature %s not added", i))
+ end
+ end
+end
diff --git a/test/functional/lua/miltertest/mt1.lua b/test/functional/lua/miltertest/mt1.lua
new file mode 100644
index 0000000..019a852
--- /dev/null
+++ b/test/functional/lua/miltertest/mt1.lua
@@ -0,0 +1,11 @@
+print('Check we will accept a message')
+
+dofile './lib.lua'
+dofile './data.lua'
+
+setup()
+
+send_message(innocuous_msg, innocuous_hdrs, 'test-id', 'nerf@example.org', {'nerf@example.org'})
+check_accept()
+
+teardown()
diff --git a/test/functional/lua/miltertest/mt2.lua b/test/functional/lua/miltertest/mt2.lua
new file mode 100644
index 0000000..1c8fa83
--- /dev/null
+++ b/test/functional/lua/miltertest/mt2.lua
@@ -0,0 +1,11 @@
+print('Check we will reject a message')
+
+dofile './lib.lua'
+dofile './data.lua'
+
+setup()
+
+send_message(gtube)
+check_gtube()
+
+teardown()
diff --git a/test/functional/lua/miltertest/mt3.lua b/test/functional/lua/miltertest/mt3.lua
new file mode 100644
index 0000000..6b30126
--- /dev/null
+++ b/test/functional/lua/miltertest/mt3.lua
@@ -0,0 +1,12 @@
+print('Check we will rewrite subjects')
+
+dofile './lib.lua'
+dofile './data.lua'
+
+setup()
+
+send_message(gtube_rw_subject)
+check_accept()
+check_subject_rw()
+
+teardown()
diff --git a/test/functional/lua/miltertest/mt4.lua b/test/functional/lua/miltertest/mt4.lua
new file mode 100644
index 0000000..300cf69
--- /dev/null
+++ b/test/functional/lua/miltertest/mt4.lua
@@ -0,0 +1,11 @@
+print('Check we will defer messages')
+
+dofile './lib.lua'
+dofile './data.lua'
+
+setup()
+
+send_message(innocuous_msg, innocuous_hdrs, 'test-id', 'defer@example.org', {'nerf@example.org'})
+check_defer()
+
+teardown()
diff --git a/test/functional/lua/neural.lua b/test/functional/lua/neural.lua
new file mode 100644
index 0000000..5a09c50
--- /dev/null
+++ b/test/functional/lua/neural.lua
@@ -0,0 +1,64 @@
+local logger = require "rspamd_logger"
+
+for i = 1,14 do
+ rspamd_config:register_symbol({
+ name = 'SPAM_SYMBOL'..tostring(i),
+ score = 5.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+ })
+ rspamd_config:register_symbol({
+ name = 'HAM_SYMBOL'..tostring(i),
+ score = -3.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+ })
+end
+
+
+
+rspamd_config:register_symbol({
+ name = 'NEUTRAL_SYMBOL',
+ score = 1.0,
+ flags = 'explicit_disable',
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+
+rspamd_config.SAVE_NN_ROW = {
+ callback = function(task)
+ local fname = os.tmpname()
+ task:cache_set('nn_row_tmpfile', fname)
+ return true, 1.0, fname
+ end
+}
+
+rspamd_config.SAVE_NN_ROW_IDEMPOTENT = {
+ callback = function(task)
+ local function tohex(str)
+ return (str:gsub('.', function (c)
+ return string.format('%02X', string.byte(c))
+ end))
+ end
+ local fname = task:cache_get('nn_row_tmpfile')
+ if not fname then
+ return
+ end
+ local f, err = io.open(fname, 'w')
+ if not f then
+ logger.errx(task, err)
+ return
+ end
+ f:write(tohex(task:cache_get('SHORT_neural_vec_mpack') or ''))
+ f:close()
+ return
+ end,
+ type = 'idempotent',
+ flags = 'explicit_disable',
+ priority = 100,
+}
+
+dofile(rspamd_env.INSTALLROOT .. "/share/rspamd/rules/controller/init.lua")
diff --git a/test/functional/lua/option_order.lua b/test/functional/lua/option_order.lua
new file mode 100644
index 0000000..caff30f
--- /dev/null
+++ b/test/functional/lua/option_order.lua
@@ -0,0 +1,15 @@
+rspamd_config:register_symbol({
+ name = 'TBL_OPTION_ORDER',
+ score = 1.0,
+ callback = function()
+ return true, {'one', 'two', 'three', '4', '5', 'a'}
+ end
+})
+
+rspamd_config:register_symbol({
+ name = 'OPTION_ORDER',
+ score = 1.0,
+ callback = function()
+ return true, 'one', 'two', 'three', '4', '5', 'a'
+ end
+})
diff --git a/test/functional/lua/params.lua b/test/functional/lua/params.lua
new file mode 100644
index 0000000..2846527
--- /dev/null
+++ b/test/functional/lua/params.lua
@@ -0,0 +1,80 @@
+rspamd_config.TEST_RCPT = {
+ callback = function(task)
+ local l = {}
+ local rcpts = task:get_recipients(1)
+ if not rcpts then return end
+ for _, r in ipairs(rcpts) do
+ table.insert(l, r['addr'])
+ end
+ table.sort(l)
+ local t = table.concat(l, ",")
+ return true, t
+ end
+}
+
+rspamd_config.TEST_HELO = {
+ callback = function(task)
+ local helo = task:get_helo()
+ if not helo then return end
+ return true, helo
+ end
+}
+
+rspamd_config.TEST_HOSTNAME = {
+ callback = function(task)
+ local h = task:get_hostname()
+ if not h then return end
+ return true, h
+ end
+}
+
+rspamd_config.TEST_SMTP_FROM = {
+ callback = function(task)
+ local f = task:get_from('smtp')
+ if not (f and f[1] and f[1].addr) then return end
+ return true, f[1].addr
+ end
+}
+
+rspamd_config.TEST_MTA_TAG = {
+ callback = function(task)
+ local h = task:get_request_header('MTA-Tag')
+ if not h then return end
+ return true, tostring(h)
+ end
+}
+
+rspamd_config.TEST_USER = {
+ callback = function(task)
+ local u = task:get_user()
+ if not u then return end
+ return true, u
+ end
+}
+
+rspamd_config.TEST_QUEUEID = {
+ callback = function(task)
+ local q = task:get_queue_id()
+ if not q then return end
+ return true, q
+ end
+}
+
+rspamd_config.TEST_IPADDR = {
+ callback = function(task)
+ local i = task:get_from_ip()
+ if not (i and i:is_valid()) then return end
+ return true, tostring(i)
+ end
+}
+
+rspamd_config.FORCE_DEFER = {
+ callback = function(task)
+ local f = task:get_from('smtp')
+ if not (f and f[1] and f[1].addr) then return end
+ if f[1].addr == "defer@example.org" then
+ task:set_pre_result('soft reject', 'Try much later')
+ return true
+ end
+ end
+}
diff --git a/test/functional/lua/prepostfilters.lua b/test/functional/lua/prepostfilters.lua
new file mode 100644
index 0000000..c87c958
--- /dev/null
+++ b/test/functional/lua/prepostfilters.lua
@@ -0,0 +1,65 @@
+for i = 1,10 do
+ local name = string.format('DEP_TEST%d', i)
+ local dep_name = string.format('DEP_TEST%d', i - 1)
+ rspamd_config:register_symbol({
+ type = 'normal',
+ name = name,
+ callback = function(task)
+ local function dns_cb()
+ if i ~= 1 then
+ if task:has_symbol(dep_name) then
+ task:insert_result(name, 1.0)
+ end
+ else
+ task:insert_result(name, 1.0)
+ end
+ end
+ if task:has_symbol('TEST_PRE') then
+ local r = task:get_resolver()
+ r:resolve_a({task = task, name = 'example.com', callback = dns_cb})
+ end
+ end
+ })
+
+ if i ~= 1 then
+ rspamd_config:register_dependency(name, dep_name)
+ end
+
+ rspamd_config:set_metric_symbol({
+ name = name,
+ score = 1.0
+ })
+end
+
+
+rspamd_config:register_symbol({
+ type = 'postfilter',
+ name = 'TEST_POST',
+ callback = function(task)
+ for i = 1,10 do
+ local name = string.format('DEP_TEST%d', i)
+ if not task:has_symbol(name) then
+ return
+ end
+ end
+ if task:has_symbol('TEST_PRE') then
+ task:insert_result('TEST_POST', 1.0)
+ end
+ end
+})
+rspamd_config:set_metric_symbol({
+ name = 'TEST_POST',
+ score = 1.0
+})
+
+rspamd_config:register_symbol({
+ type = 'prefilter',
+ name = 'TEST_PRE',
+ callback = function(task)
+ task:insert_result('TEST_PRE', 1.0)
+ end
+})
+rspamd_config:set_metric_symbol({
+ name = 'TEST_PRE',
+ score = 1.0
+})
diff --git a/test/functional/lua/preresult.lua b/test/functional/lua/preresult.lua
new file mode 100644
index 0000000..72a73b1
--- /dev/null
+++ b/test/functional/lua/preresult.lua
@@ -0,0 +1,3 @@
+rspamd_config:register_post_filter(function(task)
+ task:set_pre_result('soft reject', 'Pre Result Set')
+end)
diff --git a/test/functional/lua/recipients.lua b/test/functional/lua/recipients.lua
new file mode 100644
index 0000000..f11c1a8
--- /dev/null
+++ b/test/functional/lua/recipients.lua
@@ -0,0 +1,14 @@
+rspamd_config:register_symbol({
+ name = 'TEST_RCPT',
+ score = 1.0,
+ callback = function(task)
+ local l = {}
+ local rcpts = task:get_recipients(1)
+ for _, r in ipairs(rcpts) do
+ table.insert(l, r['addr'])
+ end
+ table.sort(l)
+ local t = table.concat(l, ",")
+ return true, t
+ end
+})
diff --git a/test/functional/lua/redis.lua b/test/functional/lua/redis.lua
new file mode 100644
index 0000000..1a1eaf2
--- /dev/null
+++ b/test/functional/lua/redis.lua
@@ -0,0 +1,113 @@
+--[[[
+-- Just a test for Redis API
+--]]
+
+local logger = require "rspamd_logger"
+local redis_lua = require "lua_redis"
+
+local redis_params
+local N = 'redis_test'
+
+local lua_script = [[
+local f = function() end
+return "hello from lua on redis"
+]]
+
+local function redis_simple_async_symbol(task)
+ local function redis_cb(err, data)
+ if err then
+ task:insert_result('REDIS_ASYNC_ERROR', 1.0, err)
+ else
+ task:insert_result('REDIS_ASYNC', 1.0, data)
+ end
+ end
+
+ redis_lua.rspamd_redis_make_request(
+ task,
+ redis_params,
+ "test_key",
+ false,
+ redis_cb,
+ 'GET',
+ {'test_key'}
+ )
+end
+
+local function redis_simple_async_api201809(task)
+ local function redis_cb(err, data)
+ if err then
+ task:insert_result('REDIS_ASYNC201809_ERROR', 1.0, err)
+ else
+ task:insert_result('REDIS_ASYNC201809', 1.0, data)
+ end
+ end
+
+ local attrs = {
+ task = task,
+ callback = redis_cb
+ }
+ local request = {
+ 'GET',
+ 'test_key'
+ }
+ redis_lua.request(redis_params, attrs, request)
+end
+
+local function redis_symbol(task)
+
+ local attrs = {task = task}
+ local is_ok, connection = redis_lua.connect(redis_params, attrs)
+
+ logger.infox(task, "connect: %1, %2", is_ok, connection)
+
+ if not is_ok then
+ task:insert_result('REDIS_ERROR', 1.0, connection)
+ return
+ end
+
+ local err, data
+
+ is_ok, err = connection:add_cmd('EVAL', {lua_script, 0})
+ logger.infox(task, "add_cmd: %1, %2", is_ok, err)
+
+ if not is_ok then
+ task:insert_result('REDIS_ERROR_2', 1.0, err)
+ return
+ end
+
+ is_ok,data = connection:exec()
+
+ logger.infox(task, "exec: %1, %2", is_ok, data)
+
+ if not is_ok then
+ task:insert_result('REDIS_ERROR_3', 1.0, data)
+ return
+ end
+
+ task:insert_result('REDIS', 1.0, data)
+
+end
+
+redis_params = rspamd_parse_redis_server(N)
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE_REDIS_ASYNC_TEST',
+ score = 1.0,
+ callback = redis_simple_async_symbol,
+ no_squeeze = true
+})
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE_REDIS_ASYNC201809_TEST',
+ score = 1.0,
+ callback = redis_simple_async_api201809,
+ no_squeeze = true
+})
+
+rspamd_config:register_symbol({
+ name = 'REDIS_TEST',
+ score = 1.0,
+ callback = redis_symbol,
+ flags = 'coro',
+})
+-- ]]
diff --git a/test/functional/lua/regex_test.lua b/test/functional/lua/regex_test.lua
new file mode 100644
index 0000000..01f0e22
--- /dev/null
+++ b/test/functional/lua/regex_test.lua
@@ -0,0 +1,12 @@
+local function get_urls(task)
+ local urls = task:get_urls()
+ for _, u in ipairs(urls) do
+ task:insert_result('FOUND_URL', 1.0, tostring(u))
+ end
+end
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE',
+ score = 1.0,
+ callback = get_urls
+})
diff --git a/test/functional/lua/remove_result.lua b/test/functional/lua/remove_result.lua
new file mode 100644
index 0000000..106d7f4
--- /dev/null
+++ b/test/functional/lua/remove_result.lua
@@ -0,0 +1,26 @@
+local id = rspamd_config:register_symbol({
+ name = 'REMOVE_RESULT_CB',
+ callback = function(task)
+ task:insert_result('REMOVE_RESULT_UNEXPECTED', 1.0, 'ohno')
+ end,
+ type = 'callback',
+})
+
+rspamd_config:register_symbol({
+ name = 'REMOVE_RESULT_UNEXPECTED',
+ type = 'virtual',
+ score = 0.1,
+ group = 'remove_result_test',
+ parent = id,
+})
+
+rspamd_config:register_symbol({
+ name = 'REMOVE_RESULT_EXPECTED',
+ callback = function(task)
+ return task:remove_result('REMOVE_RESULT_UNEXPECTED') and true or false
+ end,
+ type = 'normal',
+ score = 0.1,
+})
+
+rspamd_config:register_dependency('REMOVE_RESULT_EXPECTED', 'REMOVE_RESULT_UNEXPECTED')
diff --git a/test/functional/lua/rspamadm/test_batch.lua b/test/functional/lua/rspamadm/test_batch.lua
new file mode 100644
index 0000000..dd50c9d
--- /dev/null
+++ b/test/functional/lua/rspamadm/test_batch.lua
@@ -0,0 +1,4 @@
+local rspamd_logger = require "rspamd_logger"
+
+rspamd_logger.info(rspamd_config, "nope")
+rspamd_logger.err(rspamd_config, "hello world")
diff --git a/test/functional/lua/rspamadm/test_dns_client.lua b/test/functional/lua/rspamadm/test_dns_client.lua
new file mode 100644
index 0000000..c54d594
--- /dev/null
+++ b/test/functional/lua/rspamadm/test_dns_client.lua
@@ -0,0 +1,30 @@
+local rspamd_dns = require "rspamd_dns"
+local logger = require "rspamd_logger"
+
+local config_path = rspamd_paths['CONFDIR'] .. '/rspamd.conf'
+local _r,err = rspamd_config:load_ucl(config_path)
+
+if not _r then
+ logger.errx('cannot parse %s: %s (r=%s)', config_path, err, _r)
+ os.exit(1)
+end
+
+_r,err = rspamd_config:parse_rcl({'logging', 'worker'})
+if not _r then
+ logger.errx('cannot process %s: %s (r=%s)', config_path, err, _r)
+ os.exit(1)
+end
+
+rspamd_config:init_subsystem('dns', rspamadm_ev_base)
+
+
+local is_ok, results = rspamd_dns.request({
+ config = rspamd_config,
+ session = rspamadm_session,
+
+ type = 'txt',
+ name = 'test._domainkey.example.com',
+ -- name = '_dmarc.google.com',
+ })
+
+print(is_ok, results[1])
diff --git a/test/functional/lua/rspamadm/test_message_callback.lua b/test/functional/lua/rspamadm/test_message_callback.lua
new file mode 100644
index 0000000..6be512a
--- /dev/null
+++ b/test/functional/lua/rspamadm/test_message_callback.lua
@@ -0,0 +1,5 @@
+function message_callback(task)
+ local parts = task:get_text_parts()
+ print("n parts = " .. tostring(#parts))
+ return 1,2,4,6
+end
diff --git a/test/functional/lua/rspamadm/test_redis_client.lua b/test/functional/lua/rspamadm/test_redis_client.lua
new file mode 100644
index 0000000..a7428a8
--- /dev/null
+++ b/test/functional/lua/rspamadm/test_redis_client.lua
@@ -0,0 +1,40 @@
+local logger = require "rspamd_logger"
+local redis = require "lua_redis"
+local upstream_list = require "rspamd_upstream_list"
+
+local upstreams_write = upstream_list.create('127.0.0.1', 56379)
+local upstreams_read = upstream_list.create('127.0.0.1', 56379)
+
+local is_ok, connection = redis.redis_connect_sync({
+ write_servers = upstreams_write,
+ read_servers = upstreams_read,
+-- config = rspamd_config,
+-- ev_base = rspamadm_ev_base,
+-- session = rspamadm_session,
+ timeout = 2
+})
+
+
+local lua_script = [[
+local f = function() end
+--for k = 1,100000000 do
+-- for i=1,100000000 do
+-- f()
+-- end
+--end
+return "hello from lua on redis"
+]]
+
+local a,b = connection:add_cmd('EVAL', {lua_script, 0})
+local is_ok,ver = connection:exec()
+
+print(is_ok, ver)
+
+--[[
+a,b = connection:add_cmd('EVAL', {lua_script, 0})
+print(a,b)
+
+is_ok,ver = connection:exec()
+
+print(is_ok, ver)
+]] \ No newline at end of file
diff --git a/test/functional/lua/rspamadm/test_tcp_client.lua b/test/functional/lua/rspamadm/test_tcp_client.lua
new file mode 100644
index 0000000..eb103db
--- /dev/null
+++ b/test/functional/lua/rspamadm/test_tcp_client.lua
@@ -0,0 +1,64 @@
+local logger = require "rspamd_logger"
+local tcp_sync = require "lua_tcp_sync"
+
+local is_ok, connection = tcp_sync.connect {
+ config = rspamd_config,
+ ev_base = rspamadm_ev_base,
+ session = rspamadm_session,
+ host = '127.0.0.1',
+ timeout = 20,
+ port = 18080,
+}
+if not is_ok then
+ logger.errx(rspamd_config, 'connect error: %1', connection)
+ return
+end
+local err
+is_ok, err = connection:write(string.format('POST /request HTTP/1.1\r\nConnection: close\r\n\r\n'))
+
+logger.info('write %1, %2', is_ok, err)
+if not is_ok then
+ logger.errx(rspamd_config, 'write error: %1', err)
+ return
+end
+
+local content_length, content
+
+while true do
+ local header_line
+ is_ok, header_line = connection:read_until("\r\n")
+ if not is_ok then
+ logger.errx(rspamd_config, 'failed to get header: %1', header_line)
+ return
+ end
+
+ if header_line == "" then
+ logger.info('headers done')
+ break
+ end
+
+ local value
+ local header = header_line:gsub("([%w-]+): (.*)",
+ function (h, v) value = v; return h:lower() end)
+
+ logger.info('parsed header: %1 -> "%2"', header, value)
+
+ if header == "content-length" then
+ content_length = tonumber(value)
+ end
+
+end
+
+if content_length then
+ is_ok, content = connection:read_bytes(content_length)
+ if is_ok then
+ end
+else
+ is_ok, content = connection:read_until_eof()
+ if is_ok then
+ end
+end
+logger.info('(is_ok: %1) content [%2 bytes] %3', is_ok, content_length, content)
+
+
+print(content)
diff --git a/test/functional/lua/rspamadm/test_verbose.lua b/test/functional/lua/rspamadm/test_verbose.lua
new file mode 100644
index 0000000..4470c63
--- /dev/null
+++ b/test/functional/lua/rspamadm/test_verbose.lua
@@ -0,0 +1,3 @@
+local rspamd_logger = require "rspamd_logger"
+
+rspamd_logger.info(rspamd_config, "hello world")
diff --git a/test/functional/lua/selector_test.lua b/test/functional/lua/selector_test.lua
new file mode 100644
index 0000000..dd52ee3
--- /dev/null
+++ b/test/functional/lua/selector_test.lua
@@ -0,0 +1,23 @@
+local lua_selectors = require 'lua_selectors'
+local rspamd_text = require 'rspamd_text'
+
+rspamd_config:register_re_selector('test', 'user.lower;header(Subject).lower', ' ')
+
+config['regexp']['LUA_SELECTOR_RE'] = {
+ re = 'test=/^test@user\\.com some subject$/{selector}',
+ score = 100500,
+}
+
+lua_selectors.register_extractor(rspamd_config, 'some_rspamd_text', {
+ get_value = function()
+ return {rspamd_text.fromstring('hello'), rspamd_text.fromstring('world')}, 'string_list'
+ end,
+ description = 'Return some rspamd_texts',
+})
+
+rspamd_config:register_re_selector('some_rspamd_text_re', 'some_rspamd_text', ' ')
+
+config['regexp']['RSPAMD_TEXT_SELECTOR'] = {
+ re = 'some_rspamd_text_re=/^hello$/{selector}',
+ score = 1,
+}
diff --git a/test/functional/lua/settings.lua b/test/functional/lua/settings.lua
new file mode 100644
index 0000000..384c68e
--- /dev/null
+++ b/test/functional/lua/settings.lua
@@ -0,0 +1,68 @@
+rspamd_config:register_symbol({
+ name = 'SIMPLE_PRE',
+ score = 1.0,
+ priority = 9, -- after settings
+ group = 'a',
+ type = 'prefilter',
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE_POST',
+ score = 1.0,
+ type = 'postfilter',
+ group = 'c',
+ callback = function()
+ return true, 'Fires always'
+ end
+})
+
+local id = rspamd_config:register_symbol({
+ name = 'SIMPLE_TEST',
+ score = 1.0,
+ group = 'b',
+ callback = function(task)
+ task:insert_result('SIMPLE_VIRTUAL', 1.0)
+ task:insert_result('SIMPLE_VIRTUAL1', 1.0)
+ return true, 'Fires always'
+ end
+})
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE_VIRTUAL',
+ type = 'virtual',
+ score = 1.0,
+ group = 'vg',
+ parent = id,
+})
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE_VIRTUAL1',
+ type = 'virtual',
+ forbidden_ids = 'id_virtual,id_virtual_group',
+ allowed_ids = 'id_virtual1',
+ score = 1.0,
+ group = 'vg',
+ parent = id,
+})
+
+id = rspamd_config:register_symbol({
+ name = 'DEP_REAL',
+ callback = function(task)
+ task:insert_result('DEP_VIRTUAL', 1.0)
+ return true
+ end,
+ score = 1.0,
+})
+
+rspamd_config:register_symbol({
+ name = 'DEP_VIRTUAL',
+ parent = id,
+ type = 'virtual',
+ allowed_ids = 'id_virtual1',
+ score = 1.0,
+})
+
+rspamd_config:register_dependency('DEP_VIRTUAL', 'EXPLICIT_VIRTUAL1') \ No newline at end of file
diff --git a/test/functional/lua/simple.lua b/test/functional/lua/simple.lua
new file mode 100644
index 0000000..22ecde8
--- /dev/null
+++ b/test/functional/lua/simple.lua
@@ -0,0 +1,7 @@
+rspamd_config:register_symbol({
+ name = 'SIMPLE_TEST',
+ score = 1.0,
+ callback = function()
+ return true, 'Fires always'
+ end
+})
diff --git a/test/functional/lua/tcp.lua b/test/functional/lua/tcp.lua
new file mode 100644
index 0000000..e5c765b
--- /dev/null
+++ b/test/functional/lua/tcp.lua
@@ -0,0 +1,279 @@
+--[[[
+-- Just a test for TCP API
+--]]
+
+local rspamd_tcp = require "rspamd_tcp"
+local logger = require "rspamd_logger"
+local tcp_sync = require "lua_tcp_sync"
+
+-- [[ old fashioned callback api ]]
+local function http_simple_tcp_async_symbol(task)
+ logger.errx(task, 'http_tcp_symbol: begin')
+ local function http_get_cb(err, data, conn)
+ logger.errx(task, 'http_get_cb: got reply: %s, error: %s, conn: %s', data, err, conn)
+ task:insert_result('HTTP_ASYNC_RESPONSE_2', 1.0, data)
+ end
+ local function http_read_post_cb(err, conn)
+ logger.errx(task, 'http_read_post_cb: write done: error: %s, conn: %s', err, conn)
+ conn:add_read(http_get_cb)
+ end
+ local function http_read_cb(err, data, conn)
+ logger.errx(task, 'http_read_cb: got reply: %s, error: %s, conn: %s', data, err, conn)
+ conn:add_write(http_read_post_cb, "POST /request2 HTTP/1.1\r\n\r\n")
+ task:insert_result('HTTP_ASYNC_RESPONSE', 1.0, data or err)
+ end
+ rspamd_tcp:request({
+ task = task,
+ callback = http_read_cb,
+ host = '127.0.0.1',
+ data = {'GET /request HTTP/1.1\r\nConnection: keep-alive\r\n\r\n'},
+ read = true,
+ port = 18080,
+ })
+end
+
+local function http_simple_tcp_ssl_symbol(task)
+ logger.errx(task, 'ssl_tcp_symbol: begin')
+ local function ssl_get_cb(err, data, conn)
+ logger.errx(task, 'ssl_get_cb: got reply: %s, error: %s, conn: %s', data, err, conn)
+ task:insert_result('TCP_SSL_RESPONSE_2', 1.0, tostring(data):gsub('%s', ''))
+ end
+ local function ssl_read_post_cb(err, conn)
+ logger.errx(task, 'ssl_read_post_cb: write done: error: %s, conn: %s', err, conn)
+ conn:add_read(ssl_get_cb)
+ end
+ local function ssl_read_cb(err, data, conn)
+ logger.errx(task, 'ssl_read_cb: got reply: %s, error: %s, conn: %s', data, err, conn)
+ conn:add_write(ssl_read_post_cb, "test2\n")
+ task:insert_result('TCP_SSL_RESPONSE', 1.0, tostring(data):gsub('%s', ''))
+ end
+ rspamd_tcp:request({
+ task = task,
+ callback = ssl_read_cb,
+ host = '127.0.0.1',
+ data = {'test\n'},
+ read = true,
+ ssl = true,
+ ssl_noverify = true,
+ port = 18081,
+ })
+end
+
+local function http_large_tcp_ssl_symbol(task)
+ local data = {}
+
+ local function ssl_get_cb(err, rep, conn)
+ logger.errx(task, 'ssl_get_cb: got reply: %s, error: %s, conn: %s', rep, err, conn)
+ task:insert_result('TCP_SSL_LARGE_2', 1.0)
+ end
+ local function ssl_read_post_cb(err, conn)
+ logger.errx(task, 'ssl_large_read_post_cb: write done: error: %s, conn: %s', err, conn)
+ conn:add_read(ssl_get_cb)
+ end
+ local function ssl_read_cb(err, rep, conn)
+ logger.errx(task, 'ssl_large_read_cb: got reply: %s, error: %s, conn: %s', rep, err, conn)
+ conn:add_write(ssl_read_post_cb, 'foo\n')
+ task:insert_result('TCP_SSL_LARGE', 1.0)
+ end
+
+ if task:get_queue_id() == 'SSL Large TCP request' then
+ logger.errx(task, 'ssl_large_tcp_symbol: begin')
+ for i = 1,2 do
+ local st = {}
+ for j=1,300000 do
+ st[j] = 't'
+ end
+ data[i] = table.concat(st)
+ end
+ data[#data + 1] = '\n'
+
+ rspamd_tcp:request({
+ task = task,
+ callback = ssl_read_cb,
+ host = '127.0.0.1',
+ data = data,
+ read = true,
+ ssl = true,
+ stop_pattern = '\n',
+ ssl_noverify = true,
+ port = 18081,
+ timeout = 20,
+ })
+ else
+ logger.errx(task, 'ssl_large_tcp_symbol: skip')
+ end
+end
+
+local function http_simple_tcp_symbol(task)
+ logger.errx(task, 'connect_sync, before')
+
+ local err
+ local is_ok, connection = tcp_sync.connect {
+ task = task,
+ host = '127.0.0.1',
+ timeout = 20,
+ port = 18080,
+ }
+
+ if not is_ok then
+ task:insert_result('HTTP_SYNC_WRITE_ERROR', 1.0, connection)
+ logger.errx(task, 'write error: %1', connection)
+ end
+
+ logger.errx(task, 'connect_sync %1, %2', is_ok, tostring(connection))
+
+ is_ok, err = connection:write('GET /request HTTP/1.1\r\nConnection: keep-alive\r\n\r\n')
+
+ logger.errx(task, 'write %1, %2', is_ok, err)
+ if not is_ok then
+ task:insert_result('HTTP_SYNC_WRITE_ERROR', 1.0, err)
+ logger.errx(task, 'write error: %1', err)
+ end
+
+ local data
+ local got_content = ''
+ repeat
+ is_ok, data = connection:read_once();
+ logger.errx(task, 'read_once: is_ok: %1, data: %2', is_ok, data)
+ if not is_ok then
+ task:insert_result('HTTP_SYNC_ERROR', 1.0, data)
+ return
+ else
+ got_content = got_content .. data
+ end
+ if got_content:find('hello') then
+ -- dummy_http.py responds with either hello world or hello post
+ break
+ end
+ until false
+
+ task:insert_result('HTTP_SYNC_RESPONSE', 1.0, got_content)
+
+ is_ok, err = connection:write("POST /request HTTP/1.1\r\n\r\n")
+ logger.errx(task, 'write[2] %1, %2', is_ok, err)
+
+ got_content = ''
+ repeat
+ is_ok, data = connection:read_once();
+ logger.errx(task, 'read_once[2]: is_ok %1, data: %2', is_ok, data)
+ if not is_ok then
+ task:insert_result('HTTP_SYNC_ERROR_2', 1.0, data)
+ return
+ else
+ got_content = got_content .. data
+ end
+ if got_content:find('hello') then
+ break
+ end
+ until false
+
+ task:insert_result('HTTP_SYNC_RESPONSE_2', 1.0, data)
+
+ connection:close()
+end
+
+local function http_tcp_symbol(task)
+ local url = tostring(task:get_request_header('url'))
+ local method = tostring(task:get_request_header('method'))
+
+ if url == 'nil' then
+ return
+ end
+
+ local err
+ local is_ok, connection = tcp_sync.connect {
+ task = task,
+ host = '127.0.0.1',
+ timeout = 20,
+ port = 18080,
+ }
+
+ logger.errx(task, 'connect_sync %1, %2', is_ok, tostring(connection))
+ if not is_ok then
+ logger.errx(task, 'connect error: %1', connection)
+ return
+ end
+
+ is_ok, err = connection:write(string.format('%s %s HTTP/1.1\r\nConnection: close\r\n\r\n', method:upper(), url))
+
+ logger.errx(task, 'write %1, %2', is_ok, err)
+ if not is_ok then
+ logger.errx(task, 'write error: %1', err)
+ return
+ end
+
+ local content_length, content
+
+ while true do
+ local header_line
+ is_ok, header_line = connection:read_until("\r\n")
+ if not is_ok then
+ logger.errx(task, 'failed to get header: %1', header_line)
+ return
+ end
+
+ if header_line == "" then
+ logger.errx(task, 'headers done')
+ break
+ end
+
+ local value
+ local header = header_line:gsub("([%w-]+): (.*)",
+ function (h, v) value = v; return h:lower() end)
+
+ logger.errx(task, 'parsed header: %1 -> "%2"', header, value)
+
+ if header == "content-length" then
+ content_length = tonumber(value)
+ end
+
+ end
+
+ if content_length then
+ is_ok, content = connection:read_bytes(content_length)
+ if is_ok then
+ task:insert_result('HTTP_SYNC_CONTENT_' .. method, 1.0, content)
+ end
+ else
+ is_ok, content = connection:read_until_eof()
+ if is_ok then
+ task:insert_result('HTTP_SYNC_EOF_' .. method, 1.0, content)
+ end
+ end
+ logger.errx(task, '(is_ok: %1) content [%2 bytes] %3', is_ok, content_length, content)
+end
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE_TCP_ASYNC_TEST',
+ score = 1.0,
+ callback = http_simple_tcp_async_symbol,
+ no_squeeze = true
+})
+rspamd_config:register_symbol({
+ name = 'SIMPLE_TCP_ASYNC_SSL_TEST',
+ score = 1.0,
+ callback = http_simple_tcp_ssl_symbol,
+ no_squeeze = true
+})
+rspamd_config:register_symbol({
+ name = 'LARGE_TCP_ASYNC_SSL_TEST',
+ score = 1.0,
+ callback = http_large_tcp_ssl_symbol,
+ no_squeeze = true
+})
+rspamd_config:register_symbol({
+ name = 'SIMPLE_TCP_TEST',
+ score = 1.0,
+ callback = http_simple_tcp_symbol,
+ no_squeeze = true,
+ flags = 'coro',
+})
+
+rspamd_config:register_symbol({
+ name = 'HTTP_TCP_TEST',
+ score = 1.0,
+ callback = http_tcp_symbol,
+ no_squeeze = true,
+ flags = 'coro',
+})
+-- ]]
diff --git a/test/functional/lua/test_coverage.lua b/test/functional/lua/test_coverage.lua
new file mode 100644
index 0000000..68f2545
--- /dev/null
+++ b/test/functional/lua/test_coverage.lua
@@ -0,0 +1,46 @@
+--[[
+-- This should be the very first file executed during a test
+-- otherwise coverage will be partly missed
+--]]
+local logger = require "rspamd_logger"
+local mempool = require "rspamd_mempool"
+local loaded, luacov = pcall(require, 'luacov.runner')
+if not loaded then
+ logger.errx('luacov is not loaded, will not collect coverage')
+ return
+end
+
+luacov.init()
+
+local pool = mempool.create()
+-- we don't need the pool, we need userdata to put __gc() on it
+-- __gc() is not called for tables, that't why there is such trick
+-- so, we are free to clean memory, let's do this :)
+pool:destroy()
+
+local woker_name
+
+rspamd_config:add_on_load(function(cfg, ev_base, worker)
+ woker_name = worker:get_name()
+ local stats_path = rspamd_paths["DBDIR"] .. '/' .. woker_name .. '.luacov.stats.out'
+ local config = luacov.load_config()
+ config.statsfile = stats_path
+end)
+
+-- use global variable to prevent the object from being GC'ed too early
+__GLOBAL_COVERAGE_WATCHDOG = {pool = pool}
+
+local mt = {
+ __gc = function()
+ --[[
+ -- We could've used finish_script but in that case some coverage would be missed:
+ -- pool destructors are executed after finish_scripts (when Lua state is terminated and that's
+ -- how we can collect coverage of cove executed there
+ --]]
+ if woker_name then
+ luacov.shutdown()
+ end
+ end
+}
+
+debug.setmetatable(__GLOBAL_COVERAGE_WATCHDOG.pool, mt)
diff --git a/test/functional/lua/test_fname.lua b/test/functional/lua/test_fname.lua
new file mode 100644
index 0000000..ffa7bb9
--- /dev/null
+++ b/test/functional/lua/test_fname.lua
@@ -0,0 +1,12 @@
+rspamd_config.TEST_FNAME = {
+ callback = function(task)
+ local r = task:get_parts()
+ local fnames = {}
+ for _,rh in ipairs(r) do
+ if rh:get_filename() then
+ table.insert(fnames, rh:get_filename())
+ end
+ end
+ return true,1.0,fnames
+ end
+} \ No newline at end of file
diff --git a/test/functional/lua/tlds.lua b/test/functional/lua/tlds.lua
new file mode 100644
index 0000000..24836b3
--- /dev/null
+++ b/test/functional/lua/tlds.lua
@@ -0,0 +1,58 @@
+rspamd_config:register_symbol({
+ name = 'TEST_TLD',
+ score = 1.0,
+ callback = function()
+ local prefixes = {
+ '',
+ 'example.'
+ }
+ local test_domains = {
+ 'example.ac',
+ 'example.b.br',
+ 'example.co',
+ 'example.com',
+ 'example.co.za',
+ 'example.in.net',
+ 'example.star.kawasaki.jp',
+ 'example.net',
+ 'example.net.in',
+ 'example.star.nom.br',
+ 'example.org',
+ 'example.org.ac',
+ 'example.ru.com',
+ 'example.za.net',
+ 'example.za.org',
+ 'org.org.za',
+ }
+ local worry = {}
+ local rspamd_mempool = require 'rspamd_mempool'
+ local rspamd_url = require 'rspamd_url'
+ local rspamd_util = require 'rspamd_util'
+ local pool = rspamd_mempool.create()
+ for _, d in ipairs(test_domains) do
+ (function()
+ for _, p in ipairs(prefixes) do
+ local test = rspamd_util.get_tld(p .. d)
+ if (test ~= d) then
+ local opt = string.format('util.get_tld:p=%s;d=%s;got=%s', p, d, test)
+ table.insert(worry, opt)
+ return
+ end
+ local u = rspamd_url.create(pool, p .. d)
+ assert(u, "cannot parse string: " .. p .. d)
+ test = u:get_tld()
+ if (test ~= d) then
+ local opt = string.format('url.create:p=%s;d=%s;got=%s', p, d, test)
+ table.insert(worry, opt)
+ return
+ end
+ end
+ end)()
+ end
+ if (#worry == 0) then
+ return true, 1.0, "no worry"
+ else
+ return true, 1.0, worry
+ end
+ end
+})
diff --git a/test/functional/lua/udp.lua b/test/functional/lua/udp.lua
new file mode 100644
index 0000000..0ed4b15
--- /dev/null
+++ b/test/functional/lua/udp.lua
@@ -0,0 +1,81 @@
+--[[[
+-- Just a test for UDP API
+--]]
+
+local rspamd_udp = require "rspamd_udp"
+local logger = require "rspamd_logger"
+
+-- [[ old fashioned callback api ]]
+local function simple_udp_async_symbol(task)
+ logger.errx(task, 'udp_symbol: begin')
+ local function udp_cb(success, data)
+ logger.errx(task, 'udp_cb: got reply: %s', data)
+
+ if success then
+ task:insert_result('UDP_SUCCESS', 1.0, data)
+ else
+ task:insert_result('UDP_FAIL', 1.0, data)
+ end
+ end
+ rspamd_udp:sendto({
+ task = task,
+ callback = udp_cb,
+ host = '127.0.0.1',
+ data = {'hello', 'world'},
+ port = 5005,
+ })
+end
+
+rspamd_config:register_symbol({
+ name = 'UDP_SUCCESS',
+ score = 0.0,
+ callback = simple_udp_async_symbol,
+})
+
+local function send_only_udp(task)
+ logger.errx(task, 'udp_symbol_sendonly: begin')
+ if rspamd_udp:sendto({
+ task = task,
+ host = '127.0.0.1',
+ data = {'hoho'},
+ port = 5005,
+ }) then
+
+ task:insert_result('UDP_SENDTO', 1.0)
+ end
+end
+
+rspamd_config:register_symbol({
+ name = 'UDP_SENDTO',
+ score = 0.0,
+ callback = send_only_udp,
+})
+
+local function udp_failed_cb(task)
+ logger.errx(task, 'udp_failed_cb: begin')
+ local function udp_cb(success, data)
+ logger.errx(task, 'udp_failed_cb: got reply: %s', data)
+
+ if success then
+ task:insert_result('UDP_SUCCESS', 1.0, data)
+ else
+ task:insert_result('UDP_FAIL', 1.0, data)
+ end
+ end
+ rspamd_udp:sendto({
+ task = task,
+ callback = udp_cb,
+ host = '127.0.0.1',
+ data = {'hello', 'world'},
+ port = 5006,
+ retransmits = 2,
+ timeout = 0.1,
+ })
+end
+
+rspamd_config:register_symbol({
+ name = 'UDP_FAIL',
+ score = 0.0,
+ callback = udp_failed_cb,
+})
+-- ]]
diff --git a/test/functional/messages/arcallow.eml b/test/functional/messages/arcallow.eml
new file mode 100644
index 0000000..f56ce93
--- /dev/null
+++ b/test/functional/messages/arcallow.eml
@@ -0,0 +1,107 @@
+Delivered-To: inboobswetrusts1@gmail.com
+Received: by 2002:a19:c1cd:0:0:0:0:0 with SMTP id r196csp1333505lff;
+ Fri, 25 Sep 2020 10:19:08 -0700 (PDT)
+X-Google-Smtp-Source: ABdhPJz0QI60pshL5/0ubV0rKvSL19K7Ot++feNpjP2IZZ2jdoqLDX7eTqyiU1gupOkugQz0J60N
+X-Received: by 2002:aa7:d585:: with SMTP id r5mr2446562edq.278.1601054347960;
+ Fri, 25 Sep 2020 10:19:07 -0700 (PDT)
+ARC-Seal: i=2; a=rsa-sha256; t=1601054347; cv=pass;
+ d=google.com; s=arc-20160816;
+ b=dJINoqknDRaxNQ6l0jhlQtXtC6RqApR1PlfeuL0IMshXmn7VogDqIytsjyptM4VSkL
+ r92KtpG/v+v/xthYUwOY7qwv+XBBi6Bb8o4ia2lhXUBaz8hgCe8FTPEfioqGKQJElD6l
+ fHDXymPRs16Utuq9KSgQ2d9LTOvTpUdrQ79keBWBVDR8wwW4fdphyqDITRcY7z6zh0Dc
+ +JDRGMCt/414oNjsm3U/qrUzZHrIkTn7BqjgM706zS9v3m+f1DwUsgZMFtXWPqt0kaCg
+ 0v+knXZlpdToNQ9GBZqEaGWY2Hvis9miIyfL1T3/9Ik1yV+iEcmG7LaAhoAE4fvSV2Fs
+ oNaA==
+ARC-Message-Signature: i=2; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+ h=mime-version:content-language:accept-language:message-id:date
+ :thread-index:thread-topic:subject:to:from:dkim-signature;
+ bh=Qt9TSIDN8JU9CiNmMN7BS30A3Jgd5CsJoCnAZ6lhuZk=;
+ b=qQAFoUqsdbpbsfWrfZnVpJRGnOx5v7qiA0v5r9+w/Hw6JXTaUxmG5wm9H+pn6JF2h7
+ LbLANFBar42PzVrfpdbPh3XOyAkauaZaCHUXrFQS/0OKUWxoUQP8CB9uaSjPpeGLM7Ge
+ JKDpPDFVX9KlSF8V281OjpVJA9D6Jy8YY7WjoYkRR9vDOtSh3Z15fpY0IdTg78U1adCC
+ D5eWd+1A8kAjJ0vvmBmhWMdBhP6YNF4ESZfTaDb43+cZZibW3Sx5diGYyYlLq8pgOCA7
+ b1PO8K4Zy+/4vhvJvkBAuqwK6bFbO58nAvLFIDl/OHe736jSz0O+F1cZb37oERX5iSuD
+ gd2Q==
+ARC-Authentication-Results: i=2; mx.google.com;
+ dkim=pass header.i=@outlook.com header.s=selector1 header.b=CjJhz37G;
+ arc=pass (i=1);
+ spf=pass (google.com: domain of testcpsf@outlook.com designates 40.92.74.29 as permitted sender) smtp.mailfrom=testcpsf@outlook.com;
+ dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=outlook.com
+Return-Path: <testcpsf@outlook.com>
+Received: from EUR04-DB3-obe.outbound.protection.outlook.com (mail-oln040092074029.outbound.protection.outlook.com. [40.92.74.29])
+ by mx.google.com with ESMTPS id v11si2241830edr.99.2020.09.25.10.19.07
+ for <inboobswetrusts1@gmail.com>
+ (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128);
+ Fri, 25 Sep 2020 10:19:07 -0700 (PDT)
+Received-SPF: pass (google.com: domain of testcpsf@outlook.com designates 40.92.74.29 as permitted sender) client-ip=40.92.74.29;
+Authentication-Results: mx.google.com;
+ dkim=pass header.i=@outlook.com header.s=selector1 header.b=CjJhz37G;
+ arc=pass (i=1);
+ spf=pass (google.com: domain of testcpsf@outlook.com designates 40.92.74.29 as permitted sender) smtp.mailfrom=testcpsf@outlook.com;
+ dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=outlook.com
+ARC-Seal: i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none; b=Mxai9H/O1SAT1aKkW3+8AoAlmTby1+b4QF6qulkeVGBfoGdJzws0UxMiFYkspN1Jc+qtUbNuNIyvn6vlG/Bkd3oWXrKyWyqKyqAZG1k1FfS38s5CYaV/Z3MbYccSPmrbbNEphRYeqaFoLrR0FlUEELOw96u0SiSErAh0uAmeEagyctTnSXiuHcbyugVoMlnZ4mT/EQhhFZ8mLph40eidSI+PL40oh2muClfGjJf+KFCZL67dIdk4zcz6OJNJljCk3LuI3L5Uf3FAmmBsrhp5Ru0bSnGZkwcZjdgOwsLPBQiF+djwHa5WYO0xt//+O/lG6sZGVhnvgotGfRPk3sh3zw==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com; s=arcselector9901; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=Qt9TSIDN8JU9CiNmMN7BS30A3Jgd5CsJoCnAZ6lhuZk=; b=heQfkNck7BGKvNrB7mn8r+lTbciW4KGlu2is5dNR2ieVWMk0oPdJ9/8akIGYb+kopp4p0JVIHseB9qf8iDzwZlPIIJ4NytoRakNAJtt2SX7nB8gzeYmS1btD/kFp2slF9LxBDh6gKFlYazqnOCjqR1wU4JGjXPaXqbC+qeKzlvOi2TWTY0B5dt/kksYlZfMBEPNHmEPuHQIDKTg7m64FFSgEGDFxjI099QwfY6Ia5UTtQHBIESRjp89lElV1yDywqW6dyVLbf8gs+DlI+HlR87ECcDExwPwpZvAhq0/SUjhq0Vk/UDCKOebwGBkVEMQo4tctBvXXkcN7yEZADyqNdw==
+ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=none; dmarc=none; dkim=none; arc=none
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=outlook.com; s=selector1; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=Qt9TSIDN8JU9CiNmMN7BS30A3Jgd5CsJoCnAZ6lhuZk=; b=CjJhz37GIhNre1eIVwKvJSEZHTzsMyuycnaUsFqIYHOTdOWzHZ7UKRSDvRcWjTZm7iaQFiEGspGeAa4KDSQn6gWGxTlMPmD/JyoUFJlgoW1NDW7YwjPvCmjZKri8zag1WCBxarFZhQO3SsIR9iU1VcnlKhAAUtNsMnL5huGl34AyhrHAzAtYuC+j8rO7oXD20+yfc/4I/Efk9N5sg9MW8o6a/1TSEsDsP7bSTW9v7BXutXZCgfGWSqEJDbt1n5nt6ZA8+5WqrEXd0dANOwvaewV2AwrQEQwZguofLhlNwpyYRnwT6fY3C4e2x/Ge0WdtbzIs9w1t7X5mZgf/1tDcPQ==
+Received: from VI1EUR04FT008.eop-eur04.prod.protection.outlook.com (2a01:111:e400:7e0e::52) by VI1EUR04HT055.eop-eur04.prod.protection.outlook.com (2a01:111:e400:7e0e::302) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.3412.21; Fri, 25 Sep 2020 17:19:07 +0000
+Received: from AM0PR09MB3876.eurprd09.prod.outlook.com (2a01:111:e400:7e0e::4d) by VI1EUR04FT008.mail.protection.outlook.com (2a01:111:e400:7e0e::406) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.3412.21 via Frontend Transport; Fri, 25 Sep 2020 17:19:07 +0000
+Received: from AM0PR09MB3876.eurprd09.prod.outlook.com ([fe80::d82c:67bf:ec39:a403]) by AM0PR09MB3876.eurprd09.prod.outlook.com ([fe80::d82c:67bf:ec39:a403%9]) with mapi id 15.20.3412.024; Fri, 25 Sep 2020 17:19:07 +0000
+From: sad sad <testcpsf@outlook.com>
+To: "inboobswetrusts1@gmail.com" <inboobswetrusts1@gmail.com>
+Subject: test
+Thread-Topic: test
+Thread-Index: AQHWk1/5sqfBiHPcPEmWqAb7mPK/fA==
+Date: Fri, 25 Sep 2020 17:19:07 +0000
+Message-ID: <AM0PR09MB38763650C46C6A6FC3C0DE77CC360@AM0PR09MB3876.eurprd09.prod.outlook.com>
+Accept-Language: ru-RU, en-US
+Content-Language: ru-RU
+X-MS-Has-Attach:
+X-MS-TNEF-Correlator:
+x-incomingtopheadermarker: OriginalChecksum:39E7B13889D2F181F64CFEA6EEA08284B8F1BAD1868D8D18F9E90BE5BCA959F7;UpperCasedChecksum:F3A5A0F5617A448E098AF439F6016D44C37772C4D39C6D9ED88578C46A95CB27;SizeAsReceived:6673;Count:41
+x-tmn: [ggDyUrCobrXP/L2vAfZhcD6i3aFoVJc/Pu2bqWM7VfA4p11xOL37Jw==]
+x-ms-publictraffictype: Email
+x-incomingheadercount: 41
+x-eopattributedmessage: 0
+x-ms-office365-filtering-correlation-id: 54f426e0-67f0-473e-1317-08d861771c15
+x-ms-traffictypediagnostic: VI1EUR04HT055:
+x-microsoft-antispam: BCL:0;
+x-microsoft-antispam-message-info: RIRxucAjNkFMbHAVHyCSGanQXCZOjwMDFoy5OQa8+ewDTDNgZUtGD8vmKmoHVNp41dotypaJu8+p3piVOHXPpA==
+x-ms-exchange-antispam-messagedata: x/DoDE6wUuMAedPLA8tUqGpR3rpo5Gvu464iZGrRRTO++YhFxsOEQs2GnhQMJVAocl8M5TbBedYSIAm7RIuCLSlzt7Bd8K7Z8Yo9riMU1T3WdNuYHE6Qko9u95uvWjPIKPAovKSbTaQH34egSIA0mRzwEfSbM+SL1PE5Ub7WpTukrR1ue0MJ+VW6skZOcSVD
+x-ms-exchange-transport-forked: True
+Content-Type: multipart/alternative; boundary="_000_AM0PR09MB38763650C46C6A6FC3C0DE77CC360AM0PR09MB3876eurp_"
+MIME-Version: 1.0
+X-OriginatorOrg: outlook.com
+X-MS-Exchange-CrossTenant-AuthAs: Anonymous
+X-MS-Exchange-CrossTenant-AuthSource: VI1EUR04FT008.eop-eur04.prod.protection.outlook.com
+X-MS-Exchange-CrossTenant-RMS-PersistedConsumerOrg: 00000000-0000-0000-0000-000000000000
+X-MS-Exchange-CrossTenant-Network-Message-Id: 54f426e0-67f0-473e-1317-08d861771c15
+X-MS-Exchange-CrossTenant-originalarrivaltime: 25 Sep 2020 17:19:07.1617 (UTC)
+X-MS-Exchange-CrossTenant-fromentityheader: Internet
+X-MS-Exchange-CrossTenant-id: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa
+X-MS-Exchange-CrossTenant-rms-persistedconsumerorg: 00000000-0000-0000-0000-000000000000
+X-MS-Exchange-Transport-CrossTenantHeadersStamped: VI1EUR04HT055
+
+--_000_AM0PR09MB38763650C46C6A6FC3C0DE77CC360AM0PR09MB3876eurp_
+Content-Type: text/plain; charset="koi8-r"
+Content-Transfer-Encoding: quoted-printable
+
+test
+
+--_000_AM0PR09MB38763650C46C6A6FC3C0DE77CC360AM0PR09MB3876eurp_
+Content-Type: text/html; charset="koi8-r"
+Content-Transfer-Encoding: quoted-printable
+
+<html>
+<head>
+<meta http-equiv=3D"Content-Type" content=3D"text/html; charset=3Dkoi8-r">
+<style type=3D"text/css" style=3D"display:none;"> P {margin-top:0;margin-bo=
+ttom:0;} </style>
+</head>
+<body dir=3D"ltr">
+<div style=3D"font-family: Calibri, Arial, Helvetica, sans-serif; font-size=
+: 12pt; color: rgb(0, 0, 0);">
+test</div>
+</body>
+</html>
+
+--_000_AM0PR09MB38763650C46C6A6FC3C0DE77CC360AM0PR09MB3876eurp_-- \ No newline at end of file
diff --git a/test/functional/messages/arcbad.eml b/test/functional/messages/arcbad.eml
new file mode 100644
index 0000000..8f48de1
--- /dev/null
+++ b/test/functional/messages/arcbad.eml
@@ -0,0 +1,194 @@
+Return-Path: <owner-freebsd-test@freebsd.org>
+X-Original-To: vsevolod@highsecure.ru
+Delivered-To: vsevolod@highsecure.ru
+Received: from mx2.freebsd.org (mx2.freebsd.org [96.47.72.81])
+ (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits))
+ (No client certificate requested)
+ by mail.highsecure.ru (Postfix) with ESMTPS id 7B5193003B9
+ for <vsevolod@highsecure.ru>; Mon, 7 Sep 2020 17:43:49 +0200 (CEST)
+Received: from mx1.freebsd.org (mx1.freebsd.org [96.47.72.80])
+ (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
+ key-exchange X25519 server-signature RSA-PSS (4096 bits)
+ client-signature RSA-PSS (4096 bits))
+ (Client CN "mx1.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK))
+ by mx2.freebsd.org (Postfix) with ESMTPS id 515996EBC0;
+ Mon, 7 Sep 2020 15:43:40 +0000 (UTC)
+ (envelope-from owner-freebsd-test@freebsd.org)
+Received: from mailman.nyi.freebsd.org (mailman.nyi.freebsd.org [IPv6:2610:1c1:1:606c::50:13])
+ by mx1.freebsd.org (Postfix) with ESMTP id 4BlXcr04ftz4JVG;
+ Mon, 7 Sep 2020 15:43:40 +0000 (UTC)
+ (envelope-from owner-freebsd-test@freebsd.org)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim;
+ t=1599493420; h=from:from:sender:sender:reply-to:subject:subject:date:date:
+ message-id:message-id:to:to:cc:mime-version:mime-version:
+ content-type:content-type:
+ content-transfer-encoding:content-transfer-encoding:list-id:list-help:
+ list-unsubscribe:list-subscribe:list-post;
+ bh=KEklvNYAgI4rodPALAhxWKF6FIuGje3BV2G0j7zhqSo=;
+ b=H7pb97c0QZD4Zc7JltI+DXgqONMBy3PRldROVhMw79FLN/Ii04XX8FjGyy14OHavd3qKTF
+ hgYZbAqC4SScjwRxqJ+Ex+TpNUCWCaGGGkXzNYJR19NLHQcUwYsDUkuWmbmsxwuNAlonmJ
+ 7XUiaSPe60csOlwwJGs1e7TbXMjv2NHo4Rama0wJa47eCCyj6WDRoq//qSUwmBvICK4eiA
+ E0v4dCcT68KOygTxXz7Bzg7uxf6fwMlqX87OMYynQeJygw536cUEGEzdKwZmFIHQpuakTK
+ qKNNRYYIvpu4npR+J2gqeygP0E2CD4sNHHX2bOhTjfJNkGFCb0sDN388QnMvWA==
+Delivered-To: freebsd-test@mailman.nyi.freebsd.org
+Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1])
+ by mailman.nyi.freebsd.org (Postfix) with ESMTP id 9974F3CD951
+ for <freebsd-test@mailman.nyi.freebsd.org>;
+ Mon, 7 Sep 2020 15:43:36 +0000 (UTC)
+ (envelope-from vstakhov@outlook.com)
+Received: from EUR04-HE1-obe.outbound.protection.outlook.com
+ (mail-he1eur04olkn0804.outbound.protection.outlook.com
+ [IPv6:2a01:111:f400:fe0d::804])
+ (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
+ (Client CN "mail.protection.outlook.com",
+ Issuer "GlobalSign Organization Validation CA - SHA256 - G3" (verified OK))
+ by mx1.freebsd.org (Postfix) with ESMTPS id 4BlXcl5FF6z4JJ6
+ for <freebsd-test@freebsd.org>; Mon, 7 Sep 2020 15:43:35 +0000 (UTC)
+ (envelope-from vstakhov@outlook.com)
+ARC-Seal: i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none;
+ b=B/A8ZiFYxOSeylXcjyZTt8XXFt7YgTbngdq4AxTQ36z9v/VUw+rMglqR2j93IHtdNzAkr5e6ZRYdmAUwbocvZijACTfi3SoRaj/rAfhIR/ualRfIgQTSetMBeSqcBO95Ai/KEY4NFtDvS4pR4rxydBYVuUZE6UaHSUYwlaEcWonjbb1DtnNOHMH6CT5loNMevNXlgrPTQNgje8PCT6jDhgahA84WWjahM+bywUFwJdSTOdv7pAD2YXFoZKrlHLZskx7CVV+5N8XEjAgyvil3q048bgDeMW98q35PSe7E3zDvoTKWMAmXYTbL13A1FBLttRxVvq6Q46v9aVZc6tpnPQ==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com;
+ s=arcselector9901;
+ h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;
+ bh=l2MxonmDV0Cxeb7j4O7trwBTQ/jKYy4cF6V7PG/mFuc=;
+ b=dNW7to2Stc2Kt3qw9UK393U0tTTNHoZJhTJCy0n81IevefWs6ZmttQ3plKF/IWxifhHbZgaFclKhNcuEfpYuzYtjZFzusX34zK8e/YWl1VcJMcESDfu0dDcURag4FqAvd1RIPORgKIfRUFuT+JNzEYW7msgkwei0QQn/T+AtMJ8kVx2WtpH+S0JmMN5j7alGUnzghJIyBHavbfDoxcOBX+oY1W5AOTh7APBfIH8WXFlNztbKcwCkY/ujN+p62diFoHaNno3ml2XbxLl1LBA78I1IzIpISdkFE9TM7wDxRjaEVYTMuOI6OlkBjgM5i4TCWx3tW0/Jb/SYsFvu/OIFcA==
+ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=none; dmarc=none;
+ dkim=none; arc=none
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=outlook.com;
+ s=selector1;
+ h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;
+ bh=l2MxonmDV0Cxeb7j4O7trwBTQ/jKYy4cF6V7PG/mFuc=;
+ b=LWxzK+W8PCp8/DGYEOXmewEaJYvM7hue+67rVa05r+fLVS+hMeIANCPNJnZFUe+2bxCew5n5tRULwy3qXhFCxqbk3oy+sUwP5Gb7+XTyNQoCwCjW4tcFRs663CzamXPQKcN10XclDf/nEsU52NfkwmvEBD59j3MHDN6vcyeM2r18UT5tW1dfG2bVrIVO9b8xyehJnUbc82tDUs9ok51LxbgH0aF9vmSp66vlVXhe33tM/f+gmxOIFr4PtMxtpzk3m83Ovp+SskixfCXdldYrP5X7G2jiqyy5tS+tBNs8VnInW6b3pjQNXYcaihcdLluPLfvw8Fx5g46QeHaG+87PZQ==
+Received: from HE1EUR04FT049.eop-eur04.prod.protection.outlook.com
+ (2a01:111:e400:7e0d::49) by
+ HE1EUR04HT035.eop-eur04.prod.protection.outlook.com (2a01:111:e400:7e0d::317)
+ with Microsoft SMTP Server (version=TLS1_2,
+ cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.3348.16; Mon, 7 Sep
+ 2020 15:43:33 +0000
+Received: from LNXP265MB1258.GBRP265.PROD.OUTLOOK.COM (2a01:111:e400:7e0d::4c)
+ by HE1EUR04FT049.mail.protection.outlook.com
+ (2a01:111:e400:7e0d::171) with Microsoft SMTP Server (version=TLS1_2,
+ cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.3348.16 via Frontend
+ Transport; Mon, 7 Sep 2020 15:43:33 +0000
+Received: from LNXP265MB1258.GBRP265.PROD.OUTLOOK.COM
+ ([fe80::1de8:374c:90f8:62f3]) by LNXP265MB1258.GBRP265.PROD.OUTLOOK.COM
+ ([fe80::1de8:374c:90f8:62f3%4]) with mapi id 15.20.3348.019; Mon, 7 Sep 2020
+ 15:43:33 +0000
+From: Vsevolod Stakhov <vstakhov@outlook.com>
+To: "freebsd-test@freebsd.org" <freebsd-test@freebsd.org>
+Subject: test MS arc
+Thread-Topic: test MS arc
+Thread-Index: AQHWhS2I5V1kL4IFmUeXtTRUjyf2sA==
+Date: Mon, 7 Sep 2020 15:43:32 +0000
+Message-ID: <LNXP265MB12586E8A30257B6D494149B7D6280@LNXP265MB1258.GBRP265.PROD.OUTLOOK.COM>
+Accept-Language: en-GB, en-US
+Content-Language: en-GB
+X-MS-Has-Attach:
+X-MS-TNEF-Correlator:
+x-incomingtopheadermarker: OriginalChecksum:CA84590A4F01BB4CFB6FE8854C7A3124D845DCA76ED1A8BE88B6B871F8571461;
+ UpperCasedChecksum:191CB9448CDB911635CB69E29CF981BEFC870595A54A21C77102C996928176AC;
+ SizeAsReceived:6597; Count:41
+x-tmn: [Mn3DYL5eFlmEW5EHFTqFAtynCPFwPWHn]
+x-ms-publictraffictype: Email
+x-incomingheadercount: 41
+x-eopattributedmessage: 0
+x-ms-office365-filtering-correlation-id: 7967397b-54bc-46a6-62e1-08d85344c6c2
+x-ms-traffictypediagnostic: HE1EUR04HT035:
+x-microsoft-antispam: BCL:0;
+x-microsoft-antispam-message-info: HhTia0vUVW11W6KU6atvHK8QEQqehAaE91dm5ZOqe3JQmq63kBs1gqPcgwnvTtHC5wlGJ7oJ/+r9GhkQBYIzBQ==
+x-ms-exchange-antispam-messagedata: xy4Pbb5JXejui+jybnwU7c+74dAsS9t9PWI6LRfNckV7jG+tPbE8MEQHwLvDNi5LJxHtR0IG3NbX+Oz+OPa+hEE3kmTjJ8kejutuWcS9RunnnasqnI2877iMYRakZEqFMKHRJ0t1Me8VLuekvfnldA==
+x-ms-exchange-transport-forked: True
+MIME-Version: 1.0
+X-OriginatorOrg: outlook.com
+X-MS-Exchange-CrossTenant-AuthAs: Anonymous
+X-MS-Exchange-CrossTenant-AuthSource: HE1EUR04FT049.eop-eur04.prod.protection.outlook.com
+X-MS-Exchange-CrossTenant-RMS-PersistedConsumerOrg: 00000000-0000-0000-0000-000000000000
+X-MS-Exchange-CrossTenant-Network-Message-Id: 7967397b-54bc-46a6-62e1-08d85344c6c2
+X-MS-Exchange-CrossTenant-rms-persistedconsumerorg: 00000000-0000-0000-0000-000000000000
+X-MS-Exchange-CrossTenant-originalarrivaltime: 07 Sep 2020 15:43:32.9335 (UTC)
+X-MS-Exchange-CrossTenant-fromentityheader: Internet
+X-MS-Exchange-CrossTenant-id: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa
+X-MS-Exchange-Transport-CrossTenantHeadersStamped: HE1EUR04HT035
+X-Content-Filtered-By: Mailman/MimeDel 2.1.33
+X-BeenThere: freebsd-test@freebsd.org
+X-Mailman-Version: 2.1.33
+Precedence: list
+List-Id: Test posting area <freebsd-test.freebsd.org>
+List-Unsubscribe: <https://lists.freebsd.org/mailman/options/freebsd-test>,
+ <mailto:freebsd-test-request@freebsd.org?subject=unsubscribe>
+List-Archive: <http://lists.freebsd.org/pipermail/freebsd-test/>
+List-Post: <mailto:freebsd-test@freebsd.org>
+List-Help: <mailto:freebsd-test-request@freebsd.org?subject=help>
+List-Subscribe: <https://lists.freebsd.org/mailman/listinfo/freebsd-test>,
+ <mailto:freebsd-test-request@freebsd.org?subject=subscribe>
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+Errors-To: owner-freebsd-test@freebsd.org
+Sender: owner-freebsd-test@freebsd.org
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org;
+ s=dkim; t=1599493420;
+ h=from:from:sender:sender:reply-to:subject:subject:date:date:
+ message-id:message-id:to:to:cc:mime-version:mime-version:
+ content-type:content-type:
+ content-transfer-encoding:content-transfer-encoding:list-id:list-help:
+ list-unsubscribe:list-subscribe:list-post:dkim-signature;
+ bh=KEklvNYAgI4rodPALAhxWKF6FIuGje3BV2G0j7zhqSo=;
+ b=fylre+bQ06chZbeFj+oG/gfGdHsDM32k57JIKZGJ7OXEiBBGFYABHPUt+bIiuKKqlMqmfQ
+ /50ejP5WvAueZD0eBFUqnLu5j/fr6n7g0JwyS6Eq8K1piB944eLqWtNIyhNG7KA8EwdXGx
+ WSmQaa+xeac+DP0YiSg/mEEZpikf9fhr+pdH+O0PdVCASQglsXdOrtXTWOBUszBZDxn2on
+ wpfEiC4cHOG5r82bEX/ftcjxtOfhLx0bQFvvxPw+5hGFm3KYH85kEOCV4nTW8OLJrVYBBj
+ 71WEG5kYQN5/ZnauVLOaFNkMHbQ4zNTl1nTQvsN1cmNaKr8ucIEgxnBi7NmYLw==
+ARC-Seal: i=1; s=dkim; d=freebsd.org; t=1599493420; a=rsa-sha256; cv=none;
+ b=pYPOUWIwvn3F/A+dZ8VAJ/5Uxfbo/OTsjc1LRbyFsEeLEAOxolNQQ3ysD1JDnryuLPHaKa
+ PaYUBEyl13QXRDkKW/pWc35AmgpEHEOWGyxl7u2h98QWfXJ8AHBJgJ5vH3e658etTDsFAM
+ xINyoITvp/eSikmy1At/c8dhm1zTk8+02hMOBSqecCCFy4Ca535ybQxeTZHWBHqtN7kUBk
+ NivNKbqdkCR3/889X04rT+LzYoMlzS/aGaLY/oKyi6zSp+Vh641krCWq//cYju1xKQnmMs
+ lfvZR8E35lZ+9+k9M6s90wripNmKNM1UwgKzuhCil8CQLEJcl6ujg3/3O+QFlQ==
+ARC-Authentication-Results: i=1;
+ mx1.freebsd.org;
+ dkim=pass header.d=outlook.com header.s=selector1 header.b=LWxzK+W8;
+ dmarc=pass (policy=none) header.from=outlook.com;
+ spf=pass (mx1.freebsd.org: domain of vstakhov@outlook.com designates 2a01:111:f400:fe0d::804 as permitted sender) smtp.mailfrom=vstakhov@outlook.com
+X-Rspamd-Queue-Id: 7B5193003B9
+X-Spamd-Bar: -
+Authentication-Results: mail.highsecure.ru;
+ dkim=pass header.d=freebsd.org header.s=dkim header.b=H7pb97c0;
+ dkim=fail (body hash did not verify) header.d=outlook.com header.s=selector1 header.b=LWxzK+W8;
+ dmarc=fail reason="SPF not aligned (relaxed), DKIM not aligned (relaxed)" header.from=outlook.com (policy=none);
+ spf=pass (mail.highsecure.ru: domain of owner-freebsd-test@freebsd.org designates 96.47.72.81 as permitted sender) smtp.mailfrom=owner-freebsd-test@freebsd.org
+X-Spamd-Result: default: False [-1.31 / 15.00];
+ FORGED_RECIPIENTS_FORWARDING(0.00)[];
+ FORWARDED(0.00)[freebsd-test@mailman.nyi.freebsd.org];
+ R_SPF_ALLOW(-0.50)[+ip4:96.47.72.81:c];
+ FREEMAIL_FROM(0.00)[outlook.com];
+ DKIM_MIXED(0.00)[];
+ R_DKIM_REJECT(0.00)[outlook.com:s=selector1];
+ ARC_INVALID(1.00)[invalid count of seals and signatures];
+ DKIM_TRACE(0.00)[freebsd.org:+,outlook.com:-];
+ MAILLIST(-0.10)[mailman];
+ FORGED_SENDER(0.00)[vstakhov@outlook.com,owner-freebsd-test@freebsd.org];
+ FORGED_RECIPIENTS_MAILLIST(0.00)[];
+ MIME_TRACE(0.00)[0:+];
+ RCVD_TLS_LAST(0.00)[];
+ FORGED_RECIPIENTS(0.00)[m:freebsd-test@freebsd.org,s:vsevolod@highsecure.ru];
+ FROM_NEQ_ENVFROM(0.00)[vstakhov@outlook.com,owner-freebsd-test@freebsd.org];
+ R_DKIM_ALLOW(-0.20)[freebsd.org:s=dkim];
+ RCVD_DKIM_ARC_DNSWL_HI(-1.00)[];
+ FROM_HAS_DN(0.00)[];
+ FORGED_SENDER_FORWARDING(0.00)[];
+ MIME_GOOD(-0.10)[text/plain];
+ SENDER_REP_HAM(0.00)[ip: 96.47.72.81(-0.65)];
+ HAS_LIST_UNSUB(-0.01)[];
+ RCPT_COUNT_ONE(0.00)[1];
+ NEURAL_HAM(-0.00)[-0.965];
+ TO_DN_EQ_ADDR_ALL(0.00)[];
+ RCVD_IN_DNSWL_HI(-0.50)[96.47.72.81:from];
+ FORGED_SENDER_MAILLIST(0.00)[];
+ RCVD_COUNT_SEVEN(0.00)[8];
+ DMARC_POLICY_SOFTFAIL(0.10)[outlook.com : SPF not aligned (relaxed), DKIM not aligned (relaxed),none]
+
+123
+_______________________________________________
+freebsd-test@freebsd.org mailing list
+https://lists.freebsd.org/mailman/listinfo/freebsd-test
+To unsubscribe, send any mail to "freebsd-test-unsubscribe@freebsd.org" \ No newline at end of file
diff --git a/test/functional/messages/bad_ext.dotted_file_name.eml b/test/functional/messages/bad_ext.dotted_file_name.eml
new file mode 100644
index 0000000..a125106
--- /dev/null
+++ b/test/functional/messages/bad_ext.dotted_file_name.eml
@@ -0,0 +1,9 @@
+Content-Type: multipart/mixed; boundary="------------D6BBFC1853527FEEDD26DC71"
+
+--------------D6BBFC1853527FEEDD26DC71
+Content-Type: application/x-msdownload; name="bad_ext.dotted_file_name.exe"
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="bad_ext.dotted_file_name.exe"
+
+MAo=
+--------------D6BBFC1853527FEEDD26DC71--
diff --git a/test/functional/messages/bad_message.eml b/test/functional/messages/bad_message.eml
new file mode 100644
index 0000000..9bf152b
--- /dev/null
+++ b/test/functional/messages/bad_message.eml
@@ -0,0 +1,4270 @@
+Return-Path: <nasutkadqw@esumare.ru>
+Received: from mx.mezonplus.ru
+ by mx.mezon.local (Dovecot) with LMTP id YVBbOi5Cm1WCLgEAPZa6/Q
+ ; Tue, 07 Jul 2015 06:06:22 +0300
+Received-SPF: pass (mx.mezonplus.ru: domain of esumare.ru designates 89.163.231.166 as permitted sender) client-ip=89.163.231.166; envelope-from=nasutkadqw@esumare.ru; helo=esumare.ru;
+Received: from esumare.ru (mail.esumare.ru [89.163.231.166])
+ by mx.mezonplus.ru (Postfix) with ESMTP id 4125139820
+ for <info@mezonplus.ru>; Tue, 7 Jul 2015 06:06:22 +0300 (MSK)
+Received: from esumare.ru (unknown [93.170.123.20])
+ by esumare.ru (Postfix) with ESMTPA id A5DA0C68C0;
+ Tue, 7 Jul 2015 02:06:44 +0300 (EEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=simple; d=esumare.ru;
+ s=key2; h=DomainKey-Signature:Message-ID:Reply-To:From:To:
+ Subject:Date:MIME-Version:Content-Type; bh=kszeuZuN940hkEU5OpoWo
+ 40lSaK5/8ucmGTNjBBbfWs=; b=Fv8IUO3ub1pe5hzI0LpPZtG+n6pnNcskNJbb2
+ NKJkrLPpS4JUCgYggEGgmfxZ7Hps6s+xyu8NWQC8OP4lWp/d7en9NoydngSn6IRH
+ N2sgdz5inrO8iIkhiPJyJlXAxZalwPRhzSRAfRRiCzpY3FO1lgMoOVFT9MEzVDyK
+ r7zyi4=
+DomainKey-Signature: a=rsa-sha1; q=dns; c=simple;
+ s=key1; d=esumare.ru;
+ h=Message-ID:Reply-To:From:To:Subject:Date:MIME-Version:Content-Type:X-Priority:X-MSMail-Priority:X-Mailer:X-MimeOLE;
+ b=FOhI04bi1DAoRYBLs8pLBKMmcjaXmSfMH9w8ex1bjOyuUL3NxcbQ61mzMT2zsqXDpYQa62Rlr7NOpf7SAQrTFZNzfHYe+wzaDRCLL4vNtj+kduviptuiGzacDM/+ze8qTQPnJ76CaJLqfhLol51O4C+aP0eofiL5RBwdWy90Sg0=;
+Message-ID: <588d01d0b859$92a6a7e0$10a887d3@nasutkadqw>
+Reply-To: "=?windows-1251?B?zeDx7vH7?=" <nasutkadqw@esumare.ru>
+From: "=?windows-1251?B?zeDx7vH7?=" <nasutkadqw@esumare.ru>
+To: <zelen@megafonkavkaz.ru>
+Subject: =?windows-1251?B?0ero5OroIOTuIDM2IO/w7vbl7fLu4iDt4CDt4PHu8fsh?=
+Date: Tue, 07 Jul 2015 02:06:44 +0300
+MIME-Version: 1.0
+Content-Type: multipart/related;
+ type="multipart/alternative";
+ boundary="----=_NextPart_000_0018_01D0B859.4A05D240"
+X-Priority: 3
+X-MSMail-Priority: Normal
+X-Mailer: Microsoft Windows Live Mail 14.0.8117.416
+X-MimeOLE: Produced By Microsoft MimeOLE V14.0.8117.416
+X-Spamd-Result: default: False [-0.70 / 15.00]
+ HFILTER_MID_NOT_FQDN(0.50)
+ R_SPF_ALLOW(-1.10)
+ R_DKIM_ALLOW(-1.10)
+ FORGED_RECIPIENTS(1.00)
+X-Spamd-Server: 127.0.0.1
+X-Spamd-Scan-Time: 0.22
+X-Spamd-Queue-ID: 4125139820
+
+This is a multi-part message in MIME format.
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: multipart/alternative;
+ boundary="----=_NextPart_000_0019_01D0B859.4A05D240"
+
+------=_NextPart_000_0019_01D0B859.4A05D240
+Content-Type: text/plain;
+ charset="windows-1251"
+Content-Transfer-Encoding: quoted-printable
+
+=0D=0A=0D=0A=0D=0A=0D=0A=0D=0A=0D=0A=0D=0A=0D=0A=0D=0A=0D=0A=0D=0A=
+=0D=0A=0D=0A
+------=_NextPart_000_0019_01D0B859.4A05D240
+Content-Type: text/html;
+ charset="windows-1251"
+Content-Transfer-Encoding: quoted-printable
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">=0D=
+=0A<HTML><HEAD>=0D=0A<META content=3D"text/html; charset=3Dwindow=
+s-1251" http-equiv=3DContent-Type>=0D=0A<META name=3DGENERATOR co=
+ntent=3D"MSHTML 8.00.6001.23588">=0D=0A<STYLE></STYLE>=0D=0A</HEA=
+D>=0D=0A<BODY bgColor=3D#ffffff>=0D=0A<DIV align=3Dcenter><FONT s=
+ize=3D2 face=3DArial><table border=3D0 cellpadding=3D0 cellspacin=
+g=3D0>=0D=0A<tr style=3D"padding:0;margin:0"><td style=3D"padding=
+:0;margin:0" height=3D125><img src=3D"cid:edbc301d0b859492df1d009=
+d1ee02c@nasutkadqw" border=3D0 vspace=3D0 hspace=3D0 width=3D172=20=
+height=3D125></td><td style=3D"padding:0;margin:0" height=3D125><=
+img src=3D"cid:6e1e801d0b859d92688d70de258af3@nasutkadqw" border=3D=
+0 vspace=3D0 hspace=3D0 width=3D110 height=3D125></td><td style=3D=
+"padding:0;margin:0" height=3D125><img src=3D"cid:4b33101d0b85969=
+28da1a0a6340bb5@nasutkadqw" border=3D0 vspace=3D0 hspace=3D0 widt=
+h=3D157 height=3D125></td><td style=3D"padding:0;margin:0" height=
+=3D125><img src=3D"cid:37b3701d0b859492b834102f6705c4@nasutkadqw"=
+ border=3D0 vspace=3D0 hspace=3D0 width=3D113 height=3D125></td><=
+td style=3D"padding:0;margin:0" height=3D125><img src=3D"cid:0b64=
+701d0b85909251f830b812fc8f@nasutkadqw" border=3D0 vspace=3D0 hspa=
+ce=3D0 width=3D90 height=3D125></td><td style=3D"padding:0;margin=
+:0" height=3D125><img src=3D"cid:b487701d0b859d928eb310d4888f47@n=
+asutkadqw" border=3D0 vspace=3D0 hspace=3D0 width=3D158 height=3D=
+125></td></tr>=0D=0A<tr style=3D"padding:0;margin:0"><td style=3D=
+"padding:0;margin:0" height=3D143><img src=3D"cid:7570b01d0b859d9=
+2805b30d0b08e13@nasutkadqw" border=3D0 vspace=3D0 hspace=3D0 widt=
+h=3D172 height=3D143></td><td style=3D"padding:0;margin:0" height=
+=3D143><img src=3D"cid:44a9f01d0b859992dc36d08890cb74@nasutkadqw"=
+ border=3D0 vspace=3D0 hspace=3D0 width=3D110 height=3D143></td><=
+td style=3D"padding:0;margin:0" height=3D143><img src=3D"cid:b72d=
+801d0b859f927cd8c0818aa475@nasutkadqw" border=3D0 vspace=3D0 hspa=
+ce=3D0 width=3D157 height=3D143></td><td style=3D"padding:0;margi=
+n:0" height=3D143><img src=3D"cid:8745601d0b859e929606101eb4ea59@=
+nasutkadqw" border=3D0 vspace=3D0 hspace=3D0 width=3D113 height=3D=
+143></td><td style=3D"padding:0;margin:0" height=3D143><img src=3D=
+"cid:cb29601d0b859e92919940c94fd8ed@nasutkadqw" border=3D0 vspace=
+=3D0 hspace=3D0 width=3D90 height=3D143></td><td style=3D"padding=
+:0;margin:0" height=3D143><img src=3D"cid:7713b01d0b85939271dc403=
+5be5a9d@nasutkadqw" border=3D0 vspace=3D0 hspace=3D0 width=3D158=20=
+height=3D143></td></tr>=0D=0A<tr style=3D"padding:0;margin:0"><td=
+ style=3D"padding:0;margin:0" height=3D119><img src=3D"cid:b8db60=
+1d0b859e92e6700073c717e5@nasutkadqw" border=3D0 vspace=3D0 hspace=
+=3D0 width=3D172 height=3D119></td><td style=3D"padding:0;margin:=
+0" height=3D119><img src=3D"cid:08ee401d0b859092973e90a6de4e3a@na=
+sutkadqw" border=3D0 vspace=3D0 hspace=3D0 width=3D110 height=3D1=
+19></td><td style=3D"padding:0;margin:0" height=3D119><img src=3D=
+"cid:f567501d0b859992e4e960155fa7b0@nasutkadqw" border=3D0 vspace=
+=3D0 hspace=3D0 width=3D157 height=3D119></td><td style=3D"paddin=
+g:0;margin:0" height=3D119><img src=3D"cid:a881401d0b859392a08d60=
+67ee3315@nasutkadqw" border=3D0 vspace=3D0 hspace=3D0 width=3D113=
+ height=3D119></td><td style=3D"padding:0;margin:0" height=3D119>=
+<img src=3D"cid:7255e01d0b859492655920a319957c@nasutkadqw" border=
+=3D0 vspace=3D0 hspace=3D0 width=3D90 height=3D119></td><td style=
+=3D"padding:0;margin:0" height=3D119><img src=3D"cid:56db101d0b85=
+9b92ce2d10da21d9d2@nasutkadqw" border=3D0 vspace=3D0 hspace=3D0 w=
+idth=3D158 height=3D119></td></tr>=0D=0A<tr style=3D"padding:0;ma=
+rgin:0"><td style=3D"padding:0;margin:0" height=3D295><img src=3D=
+"cid:c636301d0b859292bc52c015ce5bde@nasutkadqw" border=3D0 vspace=
+=3D0 hspace=3D0 width=3D172 height=3D295></td><td style=3D"paddin=
+g:0;margin:0" height=3D295><img src=3D"cid:d182101d0b8599926de4a0=
+f6ee2d19@nasutkadqw" border=3D0 vspace=3D0 hspace=3D0 width=3D110=
+ height=3D295></td><td style=3D"padding:0;margin:0" height=3D295>=
+<img src=3D"cid:d86de01d0b85929296f07066112547@nasutkadqw" border=
+=3D0 vspace=3D0 hspace=3D0 width=3D157 height=3D295></td><td styl=
+e=3D"padding:0;margin:0" height=3D295><img src=3D"cid:61a3001d0b8=
+59092a10290e3af2ce3@nasutkadqw" border=3D0 vspace=3D0 hspace=3D0=20=
+width=3D113 height=3D295></td><td style=3D"padding:0;margin:0" he=
+ight=3D295><img src=3D"cid:e137e01d0b8593928e8c00883660c1@nasutka=
+dqw" border=3D0 vspace=3D0 hspace=3D0 width=3D90 height=3D295></t=
+d><td style=3D"padding:0;margin:0" height=3D295><img src=3D"cid:f=
+9f9401d0b859e9261fdc071801d40@nasutkadqw" border=3D0 vspace=3D0 h=
+space=3D0 width=3D158 height=3D295></td></tr>=0D=0A</table></FONT=
+></DIV></BODY></HTML>=0D=0A
+
+------=_NextPart_000_0019_01D0B859.4A05D240--
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="fxnyw.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <edbc301d0b859492df1d009d1ee02c@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAfQCs
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++hv23/235PgjOfBPgloJvGckSyXd7Kokj01GGUGw8NKwIbDcKpBIOQB+ZHjD4keK/iBqMl/4k8Ra
+nrd07bi97dPLt9gCcKPYYA9KT4j+L7zx/wCP/EXiS/kaS61S/mu3LHON7khR6AAgAdAAB2rnK/f8
+pymhltCMYxXP1fW/+R+cYzGVMVUbb93oiT7RJ/z1f/vo0faJP+er/wDfRqOivfPOuSfaJP8Anq//
+AH0aPtEn/PV/++jUdFAXJPtEn/PV/wDvo0faJP8Anq//AH0ajooC5J9ok/56v/30aPtEn/PV/wDv
+o1HRQFyT7RJ/z1f/AL6NH2iT/nq//fRqOigLkn2iT/nq/wD30aPtEn/PV/8Avo1HRQFyT7RJ/wA9
+X/76NH2iT/nq/wD30ajooC5J9ok/56v/AN9Gj7RJ/wA9X/76NR0UBck+0Sf89X/76NH2iT/nq/8A
+30ajooC5J9ok/wCer/8AfRo+0Sf89X/76NR0UBckuP8AXy/7x/nUdSXH+vl/3j/Oo6BsKQkAEk4H
+XJpa7z4aeD4dVL6tfRiW2hk2W8LDKu4xlj6hcgY7n6VnOapx5mY1akaMHORzGmeE9Z1iIS2mnSvC
+ekr4RT9C2M/hV7/hXXiL/nwX/v8AJ/jXtZQscnJP9PSk8v2rz/rcr6I8F5lWbuor8TxX/hXPiL/n
+wX/v8n+NH/CufEX/AD4L/wB/k/xr2ryqTy/al9an2Qv7Rr9l+P8AmeLf8K58Rf8APgv/AH+T/Gj/
+AIVz4i/58F/7/J/jXtXle1Hle1P61PsL+0q3Zfj/AJniv/CufEX/AD4L/wB/k/xo/wCFc+Iv+fBf
++/yf417T5Qo8oUvrc+yH/aNfsvx/zPFv+Fc+Iv8AnwX/AL/J/jR/wrnxF/z4L/3+T/GvafKFHlCj
+61Psg/tKv2X4/wCZ4t/wrnxF/wA+C/8Af5P8aP8AhXPiL/nwX/v8n+Ne0+UKPKFH1ufZB/aNfsvx
+/wAzxb/hXPiL/nwX/v8AJ/jR/wAK58Rf8+C/9/k/xr2nyhSiAnoCfwprFTeyJeZ1o7pfj/meK/8A
+CufEX/Pgv/f5P8aP+Fc+Iv8AnwX/AL/J/jXtPlexo8oUfWp9kUsyrvVJf18zxb/hXPiL/nwX/v8A
+J/jR/wAK58Rf8+C/9/k/xr2nyhR5QpfW59kH9o1+y/H/ADPne4/18v8AvH+dR1Jcf6+X/eP86jr1
+T6VhXu3w7gC+B9IKjG6MucdyWNeE19AfDqPPgTRDnrAf/Qmrz8Y7QXqebjo81NLzNfy/rWFe3M6X
+1yizOqB8BQeAMdq6jy/euV1MY1S7HpJ/QV+RccV6tHA0nSm4vnWza+zLsfvfgNl+Exuf4uGLoxqR
+VFtKUVJJ+0pq6TT11CzuZnvbZGnkZWcAqTwevBroFi+YcE81zunDOp2g9ZP6V1ywncOvX0p8C16t
+fCVXVm5Pn6tv7K7h4/YDCYHOMLHCUo006N2oxjFN889bJLXzOQe6uBJIBPJjcw69OTxVjTJ5pr9E
+eV3QoxIY5BOBg1Tl/wBdL/10b/0I1b0Nd2qx/wC6/wDSvzXK8bipZvQjKrJr2kdOZ2+JeZ/T3FmR
+ZTS4Nx9Wng6UZrDTaapwTT9m9U7XT8zb8v61U1SQ29k5ViHYhFI6j3H4VreX71g+IpQbmKAEERrv
+I9Cen6Cv2riTHvAZXWqRdpNcq9ZaaeaV38j+G/DLh6PEHFmDwtWPNThL2k09Vy0/es12lLli/wDE
+Z/2y4/57yfnVnTLqVr1Elld0kBUBjnDdv61S2naTg4BxnsD2H6UgdomV1zuQhhjrkHpX4Lgs3xeE
+xNLETqyai02nJu6vqtX1V0f6E55wbk+c5Xisvo4WlCdSEoqSpwTjJr3XdK6admdPs+tGwAEk8YyS
+egq0gEqK68q4DD6EVmeIZjFbpbg8zElh/sjt+fFf0ZmOY08vwVTGS1UVdebei+9tfmf5ocNcNYni
+PO8PktP3J1JWk2vhSu5trvGKbt1atpcz7rWHkYrbHy4x0kIyze49BVNpZHOTK5PrvIpK2dO0KKa1
+jmuN5Z+QgbaFH9TX4NRlnHFGKlFVLta7tRivJK/4Jt7vqz/QTGUuDfCjKadWeGSi2opqEZ1akrXd
+5O19m3eUYraKV4xMqO9uIT8kzY/ut8wP1rYsLtL5DxslTG5Bz+I9RWZqdgdOuQgYujjehPXGeR+f
+eo7Kc2t5DKOgYK2O6ngj+td2VZzj8hzD6njJtwT5ZJu6XnHtbfTRr5NeBxbwRw94gcOvOclpRjXl
+BzpThFQc2vsTSSvdpx973oS62TT6ERZOBWZcaukcpWKJp1HBdSACfaruvzta2gjRsPMduR1C9z7e
+ma55QAMAYAr67ivibEZfXWCwL5ZKzk7J77RSaa21bt1Vup+N+EvhZlvEmXzzvPoOdKTcacFKUL8r
+tKbcXGXxXjFXS0k2n7tvFLj/AF8v+8f51HUlx/r5f94/zqOv3c/AWFfRHw2QHwDoRwf9Qf8A0Jq+
+d6+jvhmhPw+0E4625/8AQ2rzMfpTXqcmJV4L1N7yx71xurjGr3o9JP6Cu68s+lcNrIxrd+P+mp/k
+K/HOO/8AcKX+P/22R/RfgCrcQYv/AK8P/wBOUyorFGDKSrKcgjgg1L9vuv8An5l/76NFlbi7vre3
+LFBK+wsuCQMdf0rol8GQsyj7XOOcZ2LX5vlWU5pmFOU8BflTs7SUddO7XQ/pni3i7hThzEU6HEFu
+eUeZXpOp7t2t1GVtU9Pn1OXJJJJOTnP1NaHh3nWYh/sP/KqMgCSyKDkI7Lz3wTz+laPhkZ1uIf8A
+TN/5VzZMnHNcOnv7SP8A6Uj1ONpRqcI5lKGzw9W3p7NnVCMEjOQK4i+uPtd7PLzhnO0HsBwP5Zrs
+tZnNjpVzMOGK7FJP8R4/rXDABQFHQDAr7zjvGXlRwSe15v8AKP8A7cfz74AZJyQx2dzW7VKL8lac
+/vvT+5m1a6eZPDNzJsy7v5y+p28ce3WsersOu31vbpBG8QhRdgUxZ+X+vWqAAUAAYAGMV8LmeIwd
+enho4VNOEFGV0ldp3utXu2+3Q/f+FsuzrL8Tmc82lBwrV3Up8sm2otKKjK8Y2tGELWvrf59V4blF
+xpgQ8tAxjIHYdR+hrL8T/wDISRf7sQ/XNSeFLjy9ReA5xMhx6bhz+eCad4thMeoQuQcPFj8Qf/r1
+9ri8ZLHcKQd9YSjGX/brsvwcWfhmT5NTyHxerwatGvCpVp+tRc0rf9vKqvRGG4yjD2IrvI4QsaAD
+gKB+lcG+dpxycdK9BtmE9rBKpyrxqwI78VfAUoqpiV1tH/26/wCaMfpCU6ksNlc18KlVT9WqdvwT
+t6MpXmk29+0ZnVmKAhdrEDB/nUQ8PWA625OOfmcmnazrA0iSCPyhK0gLYLYIAI/rVJfF0eQDZyEk
+gfLIOpP/ANevrMbjuHqWLnDGKHtdL3g29lbXld9LdfI/IMjyHxIxWT0a+SzrLCNS5FCuoKyk+a0P
+aRa97m6K++zuVPFORe2684ERP5n/AOtWRW74wgMdxaSkYDIyHPqCDisKvyXieMo5xiOfuv8A0lW/
+A/sHwrnSqcF5c6OyjJfNVJ3/ABueJ3H+vl/3j/Oo6kuP9fL/ALx/nUdf12f52MK+lfheufh3oB/6
+dz/6G1fNVfTfwsXPw58Pn/p3P/obV5WYfw4+v6MwrK8Totlef68Ma7qA/wCmx/kK9H21zeoeDXv9
+QubkXqxiZ9+wx5I4HHXnpX5XxXl+KzLCU6WFhzNSu9UtLNdWu5+0eEXEWVcMZxiMVm9b2cJUnFPl
+lK8ueDtaMZPZPpY5rRBnW9P95h/I16GifOv1H86wLDwW9lf21yb1XEMm/YIyCeOnXjrXSqu1geuD
+mjhLL8VlmGqQxcOVuV1qnpZLo32H4w8R5VxRmeHxGT1vawhS5W+WUbS5pO3vRi9mttDy64/4+p/+
+urf+hGtHwqM69CP+mcn8q15fArySyP8AblG92fHldMkn196s6R4TfStRS6N2soVWXYEwTkda+Hy/
+h7NKGZUq9SjaKmm3zR2Ur9Hc/eeIvEnhTHcM4vL8PjOarUoThGPJVV5ODSV3BLfq3bzKPjW52i0t
+VPJJlYe3QD+ZrB0yxbU9QhtgxTeSWYDJVQOT711mreE5NV1CS5N4sYYKqoYydqj8ee5qXQ/DH9jX
+Uk7XCzsybFAXbt55Pv0Ar1MbkePzTO3XxFL9y5JXuvgj6O/vW9U30Pk8j494e4V4Fjl+XYq+NVOT
+S5J6Val3vKHI/ZuW7bTUNL6Gd/wg8f8Az/Tf9+xWJrWlHR7xYPMMqugdXYAE888fWvRtlZWveH/7
+bEBEwgeIt8xXdlSOntyM16+ccLYSWDl/Z1G1VNW1euuq95221+R8dwX4tZxSzql/rJjXLCNSUvcj
+7rteMv3cFJ+8kuukm7dVwltcmzuobgdYnD/4/pmu28Q6SdW08GAbpoz5sQz97P8AD+I/Ws7/AIQJ
+zwb9P+/X/wBeuk0+zezsYLeSUTPEoXzAuMgd64+HcmxlGhiMDmNK1Kol1i9dtLN67NO32T2fEjjb
+Jcbjstz/AIaxanisNJprkqK8Hqr80YpxvzRaTu1UfmzzEHOeCCCQQRgg9+OxrR0/X7zTIfJiZHiy
+SqSru2n2PYe1dfq3hi01ZzKS1vckczRY+b6jv9etYkngS6DHZeQMv+0jA185U4czrKq7ngby7Si0
+nbzTa+a1R+mYfxL4H4ty9YfP1GD0bp1YOSUl1jJRae7Sfuys3eKuYF3dzX1w09w++VgBnGAB6D0H
+tWh4a0xtR1FJCM29udznsW7L9e/sK17TwIAwN3eF17pApXPtk9PwrpLe0itIEhgjWKFBhVUYA9/c
++5r08n4WxlXFLF5nok72bvKT31tfS+93d7W1ufL8aeLGTYXKZ5Nwt70pR5FKMXCFOFre6motyS0j
+ZKMd7trlefrOl/2tYPCCBMCHjZugYdvoelefyK0MjxyKYpEO1kccg16ps+lQT6Za3j757aGZwMbn
+XJxX1PEHDazica9KahUWjvqmvO3Vd+2j6H5N4d+Js+C6FXAYyi6uHk+aKi0pQlona+jjK2qurPVb
+u/yPcf6+X/eP86jqS4/18v8AvH+dR1+0n46wr6g+FKZ+G3h44625/wDQ2r5fr6m+EyZ+Gfhw/wDT
+sf8A0Y9eTmX8Jev6Mzmro6XZ7CvU/BHwn8Maj8LZ/HHi7xDqOh2I1I6fClhbLMX+6AcFSSdxYccA
+LmvMnARCxOAoyT7V7d8Xoz4V+Afwr8MMRHPcpJq90nTOV3An8Zv0r47FTnenSg7OT6dkm2b4WnG1
+SpNXUV+LaS7GV4j+Enge3+EWp+OfD3ijWNThtrlbKCK9tEgSSYuqkEFQ2BknI9PrXmcHhHXbnRm1
+eHQdTm0lQSb+OzkaED13YwR7jj3r3670ixsfhd8DPB2qBY7PXtXTUdSjc7QyElwG9maVFrrbPVvi
+JqH7U8thG2o6X4Q0slXtfLKaf9iEPDdNrMzkYI5GMcAGvMhjalKMtea3M05O3uxdrKy3buelPBU6
+ko6Wvyq0VfVq93d7JHyh4b05dX1/TrQ2V9qcEsqma10lPMupIcjd5Y5wccZPAJya9C+I3ww0PR/A
+P/CW+H4df0mK11BdN1HRvEkW25hkZcpIpAGVOVHcHPqCK9CsLn/hGfg18QPFHw7tZP7R1PxNPZ/b
+NPhMk1pYiXC+WFBKrg7gQOPMDdhjVe91G3+H/wAJfDHj29nutX1rxHDfTRak+6eK0jkLxJKTz94x
+L83OWx2OLqY2pKpGcNEpWavq7K8tNtPvIp4OCpyjPVtXTtoruy131+48zT4I6J4Z0W2u/HWu6zpe
+ozQrdS6domkvefYoWHym4kCsqMcE4OMe/WvP9W0mx13xfc2PgfT9W1HTXI+xQSIZ7uRQo3OwUcAt
+k84wMA19UWXiL4jat+1Le2MrX1l4PsC4mt5Ydli1p5XEhYjDOznOQSRgjAANcv4Wm8O6H8GPGet6
+Hd61pNvqPia4t5dR8L2guL2C1Ev7lFHVIiuDleQJMjrkZUsbVj703zSko2s/dXM3pZK90l5+vU1q
+YKlL3Ye7FOV7rV8vq7WbfkfN1zoWpWWqnS7nTbu21TcE+wywOs5Y9AEIySfbOe1WJ/COvWunT6hP
+oOpwWFvIYpruWzkWKJgeQzEcEZA56V9T6Tbp4s+MPwosby011tU0LTrq+n1PxFBHFdXkIVViZgjH
+nex+9gg54yTVPwB4m8aX3j/4o6p40m1C28JWVndw3FlqAK2ifNiJIg3BPlgklc7t4ySSK2eZTtdR
+Wiu9d/eastOtrrtdXMVlsL2cnq7LTb3U9fS9n6HzBovhnWPEs0sOjaRfavLCN0i2Ns8xQerbQceu
+DzUNpo9/qF81ja6fd3N+pYNaQwM0y7fvAoBuBHfPTvX0vAtl4f8Agv8AC3TNG1Txfoq6nGt0bnwj
+YpO93eNtyk7noQxIweDtwThcVq2Pie1074wfEL4gNpN1YNoujWVjcQX8aRTG6lkALMEZgCUEfQ8j
+FU8xn73LC9r216qSir+r7bdSVl0PdUp22v6Nczt6LvufKUGkX11qBsILC6n1AFlNnFCzTAr94bAM
+gjHPp3qKe0ltJ5IJ4ngnibbJFIpV0YdQQeh9jzX2v4a8K2Pw1+N/iTWL0Rzav4jnub23UEN9i02J
+EaaQ88NJKVX6Aehr44u573xXrd7exQT6hqGp3Mt0sMCNJLKzsz4AHJOD+ldOFxv1qTsrRSTv67/J
+WaOfE4L6tFXd5NtW8la3zd0xLLwnrmp2yXNloWp3ts5IWe2s5JEbB5wyqQeRjim/8IxrP9o/2f8A
+2NqH9oeX5v2P7K/nbP7+zGdvv0r6S1G58R+D9d+CXw20jWL3RmNrFPqsVo4Vm3uHdX4PHySjt1Na
++k+ILM/Fz42+ONUnng0vRLGPR1uLVd8sYA+Yxju25cgeprleY1EnJRTVrrV3fvcq+/c6Vl1NtRcm
+nez0Vl7vM/u2PlXRfDOr+JZZotH0i+1eSEZlWytmm8sf7W0cH261RmtpYZpIpYGimjYo8UyFHRh1
+UqRkH2NfTHjWLW/Cvw8+F/h34STaxHpOrE3J1S1jxcXMzFcfaHUYU/MzMDgYTB4GK8s/bSvdNu/j
+S8dus1zd22nW8F9LZcK1wNxO7H8QUoPbAHauvDYyWIqqFtHzW7rldtV0ucuJwkcPSc76rlv2fMr6
+PW9j88rj/Xy/7x/nUdSXH+vl/wB4/wA6jr9KMWFfVnwjXPwx8OH/AKdj/wCjHr5Tr6z+ECk/C7w1
+/wBex/8ARj14+Z6Uo+v6MVro6ZoldSrDKtwQe4rQ1zXtU8TG3OsajcaobaL7PAbpg3lRcfIvTA4F
+V9n+cV3vg74Ka54v0SDWpNQ0nw7pN1KILO71q58kXchOAIlAJIyCAeM4OARzXy1SrTpJTqNL+un/
+AAC6dOdR8kFf+uv/AATi9a1/VfEotRq+pXOpi0i8i2FwwIgj4+VcAYHC+/ArW1D4m+MdV0Q6Pe+K
+dVudMZPLa2kn4dMcKzY3MMDGCTnvmqvivwlqfgjxBd6JrFuLfULYguqMGV1Iyrqe6n14PUEAg10P
+hT4Q6r4n0KPXLnVNH8MaHNIYre/1268hblwcERjqwyCMnA4OM4rKU6EYRnK1un/A/wCAXGFeU5Qj
+e/X/AIP/AATmvDPi/XvBU8s2gazd6O8wCyi2cBZMcDcpBBIz1xketU9b1bUPE2oy3+sX9zql9KAH
+ubqTe5A6D2A9BgDtXZv8GPEkfxKg8DOtqNYuIzNDOJSbd4trN5gbGdvysMYzniuP1TT20nU76xkl
+ime0nkt5JYWLIzIxUlT3GQeauE6U580LOTV79bdNfkKcKsI8s7pJ2t0ua+o/E3xjq+inSL7xTql3
+pjL5bW0k/Dr/AHWYDcwwOhJz3zVDwz4u17wVcSz6BrF3o8kyhZfszALIBnG5SCCRnqRketdJP8Hv
+EUQ8IIotJbrxTGZbC2E210QKrlpSwAUbWB4znpyeKm8W/BrVPCugXusxaxoniLT9PuFtdQbRbkzP
+ZykgBZFxwcso45yRxWKqYa3s1a0ullZ629N/xNHSxF+d3uuvVaX9di18NvinF4Wu/G2s67dajqvi
+TWNLNjZ3LDzCXIbl3yNgB2YA4ABwBXH6r468T69oVro+r+I9Q1LTLZV2W1zLlPl+6W7tjHVs4613
+9l+zV4rube1W5v8AQ9K1m8iM1roV/ebLyVcZ4UAgHrxzjByRg1z3gL4Y+JfFPiq5tLKxgtp9DmEu
+oy6sdlraMjE7Zv7w+U/KM5UE8Dmso1MJzTqxkna3ytordvl1NZQxTUaUk0nf531d/wDgmJ4Z+I/i
+fwtp81n4f8TX2m2UrF3gtZQUDHqRkHaTjquM9az49b1OLT7/AE9NSuhY6hKJ723MhK3MgIIeQnlm
+BAOSeor334o+FG1f4NX/AIj1O38K3Graffwx2Os+EV/dXsLuqPG6j+IMxGCTyqkYOc8ha/s0+K54
+reKbUNC0/XLmEz2+gXd6FvZFA9ACAevHIGDk8Gpp4vDSi6krRd7P1Vn26aavYqphMQpKEbyVtN9n
+p/S6nn8/jzxPPqFzqM/iPUJL66tfsM91JMC8lvknyiccLkk4GOT1rP0LWL/wzqlvqOkXkmnahbZ8
+m4gxujBXacZzwQSOexrsfAHgbW7m/wBb1hLPT1j8Ig3Gp2euKRG+A4aBlwcthGGOMHafSrPjLw+1
+18PNC8U6doOneGPDM91Jb2lkt1Jc39xKxbfJLK6gsi+WwUZwo7HNdHtaSl7JJWdl0ttdK3pt0t2O
+dUqrj7Rt3WvXuk39/wA7nInxh4hbxKfEJ1u9PiA5H9plwZwNu3AOOBtJGBwBVaLxXqr2Op6fHrNw
+1lqMxnv7ZZAVuZMgl5PU5APXqK0fB/gXWPiJriaJodobq8kQu5ZtiRRjGXd/4RkgdySRgGvf9X8G
+T+JPhp4xGuR+CNSOjWJnsNZ8KgLLbXEYJMMo9MKO+CGYEdKzr4ijQlGLSb07aK+nTvtt5GlHD1a8
+XJN9e+rtr/wfxPAdB+Inizwrpj6boviTUdLsHJb7PbyjapPUrkHaT1+XHPNc64cuzGQszkszOxLM
+xOSSSckn1Nel+FvgVrnirwtp3iBdV0PSLDUGdLYareGF5SrFTgbSDyDwOcVynxO8MP8ACHxGmheI
+NQsv7Qa3W522jM6qjFgASVHPynj0xW0K1D2jhTa5utt9NzKpRrKmpVE+XpfbU+Crj/Xy/wC8f51H
+Ulx/r5f94/zqOv0YxYV9cfB1M/CzwyfW2P8A6Mevkevr/wCDSA/Cnwwf+nVv/Rj14ua/wo+v6MqO
+p080TPC6pw5UgH0PY19DeKfDuk/GPw/8Ori08U6Po3hzRtO+z6hBd3Cx3FnIqoGKRngt8m3JwO/I
+NeB7BTGtInkDNGjOOjFckV8dWpOq4yjKzjez33Vjqo1FTUouN07eWzuWfESW7axfpYald6vZrIY7
+fUL7PnToOjnOSBnJAPbBwM4r6KttQ03xt4L+Hh0nTPA2rW+lW0dnqFn4pl8u4sWGwSGMZxg7WPIO
+flPIzXzfsFMks4p2BeNHPYsucVNbDqso+9Zx/wArdLFUa7pOWl0/879bn0v4O1bRLn9o7xp4jttf
+t9QtbLTRDYXOo3SRxPO+MwxMcAxpsYZGcbjyep5jxZpGhXPwz1z/AISfSvA+k+K7uVIdH/4RK53y
+mV2G55OThQzEknjbuJxxXiL2scqruVWA6AjOKalrHFu2oq9jgYzXPHBKMlJTenKv/Afn166HQ8Zz
+RcXHe/8A5N/l01PpHxSPBHjD4z22iatc2F/pWieHUt9Kjnu/Lsrq6yW2SSKcDCqnH14OADgeLNfv
+PC2i+HBqt74V0rTk1e2ubjwb4RiEoliilVjJLMD82NobBUAkL1PA8O+yx+X5e1dh/h28flSpaR2/
+EaqgP91cUQwMY8qcrpdP1XRX32v2aFLGOXM1Gzf9a97bb27pn0hrXhnSNS+OMXxOvvGehN4UtvJv
+4pUuw1zmOLakIjxnG4luOTkjGSax9N8SWPxY8A/EfRdP1Gy0DXtb1ttSSDVZhAt1afugqF/UrFgj
+nBPTBzXgotIhJ5gjQP8A3tvP50r26TLtdQ4znDDIoWB0jeesUktNlF3WnXz/AEG8Xq7Q0d29d29H
+r08j2288QaF8I/C3gzwcuoW2u3EGvQ65r8umkSwRKrhxGp/iIIQ46kRk4BIFdFf+GdIPxyk+KGqe
+MtDl8LwFL+2khuw905EXlpCIxzgEseCSc4wCTj5xWFYl2qAoHQAYFNFpF5nmBFDn+ILz+dN4Ldqb
+u003Za82/oJYvZOCsrNLXS23qe9aJqdr49+FvxUl0/UNL0bXPEetm4lt9XuxbiK0zHtJPP8AyzDk
+4B53DrXP/Hq80vT/AA78PfCOi6jBqtjpGntNJdWrhleVgqgnB4Jw7YPIDc15M1rHKQzqrFeVLLkj
+6UJAqAhQFB5OBjJq6eEUKimpaJ3t8uXfyWxE8S5wcGtWrX+d/wAWetfAHVrGLSPHvh59VttB1zXd
+PWHTtQvGEcYIWQMm/wDhPzg8ckZIyRWjpPh22+HPw11LwRba5o0/jHxpMluy2t2GtNOtkQ5aWUDj
+5QwyRks6gA4zXiphWQFGAZT1DDIpotIkRowihD1ULwfwpzwvNNy5tG02rdVt8tE7ChiOWCjy6pNX
+8nv899T27xjaaFceNvhF4ETUrDUNK8OJG9/drIpgWXKu53njOIz343gHk4rlfi34F1H40fE/xNr9
+hNpDWEV39gt5bvV4rfzEiRV3IrHlSc4YcE5x6154LWMJ5YVQn93bx+VH2GKQAsiHjAytFPDuk1KE
+9UmtV3d291uxVK8aqcZR0bT0fZWS67H/2Q==
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="xucmuso.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <7570b01d0b859d92805b30d0b08e13@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAjwCs
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++Lrj/Xy/7x/nUdSXH+vl/wB4/wA6jr+nj8oYV9//ALOvwI8b+Lfgd4O1bStKgubC5sy8UjXkcZYe
+Y/VScivgCv2s/YPGf2RPhmf+oc3/AKOkr43ijF1MHhITppXcra+jPYyzCwxc5RqN6LoeL/8ADMfx
+I/6Alt/4MIv8aP8AhmP4kf8AQDtv/BhF/jXs/wC0/wDtQj9nuXw/aWmiRa9qGqedI0Ml15AhiTaN
+2QrZyzYA9jUP7MP7U/8Aw0Fe6/YXehw6BqGmJFNHDHdGcTxOWUtkouNrKAf94V+Vf6zVPa+x93m9
+H69z9IXAOMeV/wBs+zf1f+bmj/Ny35fi38vPY8e/4Zj+JH/QEtv/AAYRf40f8Mx/Ej/oB23/AIMI
+v8a9t/ag/aQu/wBnm08OTW2gQ66dXmniKy3Rg8ry1Vs8K2c7qsfDj9pew8S/AS9+J3iSxj8P2Nm9
+wsttFN5xPlvsVVJVcs7YAGOpFP8A1lqqo6T5bpXej/O/mZR4FxUsDSzKNOTpVJcsWnFtyu1bl+Ld
+NbHhP/DMXxI/6Alt/wCDCL/Gl/4Zi+JH/QDtv/BhF/jXpf7OX7VPif8AaB8W3VjH4HttI0Swi829
+1M6g0vlls+XEq+WMu3U84AUnuAeS8d/t6al4O+KmseEY/BVrdxWGqjTRdtqLIz/Oq7tvlnH3umax
+fFU1BVHaz02kepDw0zKpjamXwp3qwipSXPT0T2u72v5XvZpmD/wzH8SP+gHbf+DCL/Gj/hmP4kf9
+AO2/8GEX+Ne4/tP/ALRd3+z1p3h66ttBh106rPLCUluTB5exA2chWznNUPA37T134v8A2dfE3xNk
+8PQWlxo5ugumLdF1l8kKeZNgIzn+7Wr4mqxm6bSulfZ7feedT4DxdbBUsxhBulUkoRfNHWTbilbd
+ap6tWPHf+GY/iR/0A7b/AMGEX+NH/DMfxIP/ADA7b/wYRf416R+zZ+19f/Hrx9eeHbnwrb6JHb6c
+999oivTOWIkRAu0xr13k5z2rM/aB/bcn+DPxHvvC+n+GLPX1s7aKWa4OoGJkkcFjGVEbDIG09f4h
+UPimap+19222z/zO6PhtmUsxeVKk/bKPM1zQ+HTW+3XucV/wzH8SP+gJbf8Agwi/xo/4Zj+JH/QD
+tv8AwYRf419CeLP2kNC+H3wb8PeOvEMDwza1Z281rpNoRJLLNLEJPLQnAwoJy5wABnqQD833v/BT
+G8iuG8rwBaxQnlBcaxh8e+IsUqvFMqLSm4/c/wDMMt8Nc0zaMp4SjKUYtxbcoJXW9r2vburouf8A
+DMfxI/6Adt/4MIv8aP8AhmP4kf8AQDtv/BhF/jWh4A/4KB6p418eeGvDz+BrS0j1jUYLE3KakzmM
+SOF3BfKGcZzjNdN8fv20tV+CfxNvvCsfg611WGC2huY7uTUGhLq6kn5fLOMEEdaX+tcuT2mlr22f
++Z0S8L82jjI4B0n7SUXJLnh8KaTd723e17nE/wDDMfxI/wCgJbf+DCL/ABo/4Zj+JH/QDtv/AAYR
+f419a/Crxnc/EP4ceHfE13YJpk+rWcd2bRJPMEYcZA3EDPBHavD/ANpT9sS8+AnxBs/Ddv4XtdZS
+fTo777RNfmAgtJIm3aEbI+Trnv7VvPiStTpqrNRs/J9fmeHgeCa+ZY6WW4WLlVje65or4dHq7L8T
+zr/hmP4kf9AO2/8ABhF/jR/wzH8R/wDoBWp/7iEX+NUW/wCClWrIMt8P7JRnHOrMP/aNfVfwE+Ks
+vxo+FmkeLptOi0qS+aZTaRzeaqeXK8f3iFznbnp3rOjxROvLkp8rfo/8z0c38OcwyLDrFY+m4wb5
+b88Hq03sm3smfhFcf6+X/eP86jqS4/18v+8f51HX9En4swr9rP2Dv+TRPhl/2Dm/9HSV+Kdfsf8A
+sk+LLbwL+wj4P8Q3hAttL0Ge7cN/FseVtv1OAB7mvz/jRqOBg3/N+jPqOHqc62JdKmrylZJebeh8
+9fGxD+0R+2mnhSOWVtOtp00QvDy0UUKNLcuOwIcuOf7orkv2WfEtz8Jf2m9L0/Uc2v2i7uPDl/G3
+y4ZnKpn6TRp+dc58HPhR8QPjlrut6j4UuEg1W1b7TfX8t69owkuGclVdAWycPkeg561jfFX4d+LP
+g542/s3xE6L4iCR6nFdQXDTCUlyUfzGALMHQ5yOor+anOd1ieV/Fe/T0/A/0EpYPCeynw39Yi7UV
+Dk+0nZ3nv1UovbdJn1h/wUsGNL+Hf/X5ec/9skr5N1j4mahqPwq8O+A4VNtpWl3dzqE/zZF5cSuT
+GzDsI1JAHqxPYY+jf24vGcHxD+Fnwa8SWxUx6ok90VXorNBGWX8GyPwrifAP7Ndt42/ZU8SeO7Lf
+d+K4LqSe0iXICW1sSJYQP4mdd7Zx1VAMYOejFRlWxM/Z9r/Kyf8AkfPcMV8JlPDeBeZqzjVlFX6T
+dSpG79E5Xb233sfXn7GFv4Yh/Z/8OyeGUK+cHfUzMQZmvgcT78e4AX/YCV8EfG//AJOb8XDP/M0r
+/wCjY69Z/YB+Lo8K/EC78GXk+3SvEi+dZlmwqXqLnA9PMjGPrGg715D8ebkWn7R3je6ZS4g8RNOV
+U8sFdGIH5EU8RVjVwlNro7fcv6ZjkGVVss4rzKFRuSqQ54t7tSkvyd4/I+oP+ClQ/wCJD8P/APr/
+ALr/ANErWH8Fuf8Agnz8ST/t6n/6CleZ/tR/tOab+0Lp3hy2sNAvtGbSrmWdnu5Y3EgdAuBtPXit
+b4H/ABRtZ/2bPif8MxYTrfR6Lqeum/Lr5RT90vlheu75uvtVyrU54qpOL0cWl9yOShk+PwfC2Cwt
+eladOtGUldaR9pJ30dtmtjlv2UviNafCTxJ428W3YVxYeGpFt4GP+vuZLiIQxevzN1PYAntXlHjW
+DXJLr+2PEJd9S8QW7az50o+adJXfEuMcBirY/wBkCun+BvwnuvjX8S9J8LQmSOxkP2nUZ4+PJtEI
+8w/7zZCKfVx71337dun22kfHB9PsoUt7K00CygggQYWONRIFUD0AAAFcDU5Yfmfwp2Xq9z76FfC4
+fiJ4enrWqw5pP+WELKK/7ek5Sfklpsz0X9qbwNrWufs3/BnxFp9rNeabomkRLfpCpYwrLbQ7ZSB/
+CChBPbcPeuC/Zt/aS8JfBbwxqGl694G/t67ubtrpdUtxC0jIVVRG3mYwF2nGDj5umeT9TeKvi+vw
+W/Y/8L6xA0f9s3Og2NjpcTgHdcvbrtYg8FUAZznjCY714X+wl8BNG+IMXiPxV4s0i21zS4XGnWUG
+oRCSN5uHml2nuMooPu9enUhL61D2L95rXy0/r+mfmeAx9B8NYtZzSf1elVko8snGU25ttaW2ct72
+avpeNzwj4O3CXv7Qvgi5jTy0uPFEEyIQPlVrjcF444BAwOK9h/4KP6T9i+LejX8a4+3aDsJ9Xjmk
+A/SRa8i+EsaQftG+D4o0EcUfi2KNEUcKouSFA9gMCvpf9tv4neD9I+Jul6P4p+HI8YSadpi3lteD
+XJrEoJJG3IyRoQwzCpyT3rjppPBzTdtV3/RM+uzGtVo8VYCpQpOp+5ndJxva/wDelFb26n198PtI
+/sDwF4b0zG37Hpttb7fTZEq/0rnvi98PfC/iLwvr+rar4c0rU9Tt9KuFhvLuzjlljCxuyhXYEgAk
+ng9TXT+C9ffxT4P0LWngW1bUrGC8MCtuEfmRq+0HAzjdjPtVf4in/i33ifn/AJhd1/6KavrJRjKn
+bpY/lOjVr0sx5+Zxnz62f97VXR+aP7FGgaZ4r+Oug6frWn2urWMunXTvbXsKzRswjBBKsCCRXvn7
+T/x71T9njx3pnhHwVbW2i6Kmlx3Qs7O2jjiWR5ptxCgYGcA8d68R/YJOP2h/Dv8A2C7v/wBFCvtj
+4z/swaB8afFVvrmqXb21xBZpZhUiDZVXdwc5/wCmhH4V87g4Tng37L4r+nRH9A8WY7L8JxZTWbpy
+w/svh1a5uaVnbXXzsfiRcf6+X/eP86jqS4/18v8AvH+dR1/Y5/CDCv2U/Y88JaR45/Yk+H+h69ZJ
+qOlXmmYntZCQsgFw7AHBB6gH8K/Guv0C/Z1+JHizQ/gZ4MstO8Q3llZxWRWOCIJtUea/AypNfG8U
+4R4zCQppr4uvoz3MoxjwNZ1o3TVrNaNNO6afQ++fAHwp8JfCyzu7XwpodtokF3IJZ0t8/vGAwCck
+9BVDx98DvAnxR1C2vvFfhu01u7tojBFNcbtyITu28EcZJPPqa+Wv+Fv+Ov8Aoa9Q/wDIf/xFH/C3
+/HX/AENeof8AkP8A+Ir8v/sCTjyNxt26fkfXx4lxEcQ8XGpNVX9rmfN23vfbTc+l9R/Zx+GureGd
+I8PXnhOzuNF0hpXsLJ2cpbmVt0m35s/Meea6nwZ4B8PfDzQF0Tw5pcOlaUrvILWEkpuY5Y8k9a+P
+/wDhb/jr/oa9Q/8AIf8A8RR/wt/x1/0Neof+Q/8A4ihZBOL5k4p/P/IKvE2IxFP2NapOUb3s5Nq7
+bbdm7Xbbd97t9z6H0/8AZR+EulalbahZeCNPtb22nS6gnhaRWjlVg6suG4wwB9Km179l34V+KNbv
+tX1XwXp99qV9M09xcSl90kh5LHDY5r5y/wCFv+Ov+hr1D/yH/wDEUf8AC3/HX/Q16h/5D/8Aiaj/
+AFd0t7v3f8A6/wDXHMef2n1irzWtfnle3a/Nse/f8MgfBv8A6EHTPzk/+Kr5++AX7M3iHwt8ffGk
+OveFHh+H17Z6lpkLySoYZ7aSeMxR4DlwDGo688U9vjB45UZPiy/H18v/AOJpn/C4vG3/AENt9+cf
+/wATWUuGHOUZJxVuy39dD18L4h5jh8PXw85yqKrHl9+UpOPnH3tH5n1J4D+Avw/+GOsS6r4W8MWe
+jahLAbZ7i3LbmjJVivLEYyin8BUXjf8AZ7+HXxI11tZ8TeFbPV9TaJYTczl9xRc7RwwHGTXzD/wu
+Txv/ANDdffnH/wDE0f8AC5PG/wD0N19+cf8A8TXR/q9Ll5Pdt2t/wDwf9asX7f617ap7S1ubmfNb
+te97eR9ReJ/gJ4A8Z6Voema34atdQ0/RIPs2nW0ruEto9qrtUBh2RRk5PFdJ4O8E6F8PvD1tofh3
+TYdJ0m3LGK1g+6pZizHkkkkknJr45/4XJ43/AOhuvvzj/wDiaP8Ahcnjf/obr784/wD4mqWQVFLm
+Tjfvr/kY1OJK1aisPUnOUE7qLk2r662va+r1833PpDTf2YPhZpGv22t2fgywg1W2uVvIbpWfckwb
+eHHzYyG5rQ8dfAD4e/EzWhq/ifwvZ6xqQhW3FxOXDCMEkLww6Fj+dfL/APwuTxv/ANDdffnH/wDE
+0f8AC5PG/wD0N19+cf8A8TUf6uyty+7b0/4B0Pi3Guqqzr1OdKyfO7pdk73t5H2vpOlWmh6XZ6bY
+QLbWNnClvbwJ92ONFCqo9gABTtQ0+31SxuLK7jE9rcRtDLE3R0YEMp9iCa+Jv+FyeN/+huvvzj/+
+Jo/4XJ43/wChuvvzj/8Aia2/sOta3Mvx/wAjzP7Yg5c7TvvfrfufTfgv9nb4cfDrXoda8N+FLLSd
+VhRo0uoC+4Kwww5YjkV6MMCvh7/hcnjf/obr784//iaP+FyeN/8Aobr784//AImohkFSmrQcUvn/
+AJG+J4gqY2ftcTKc5bXk7u3a7bPy+uP9fL/vH+dR1Jcf6+X/AHj/ADqOv6APzphX3J8Bkz8GPCBx
+/wAujf8Ao16+G6+6/gGufgr4P/68z/6Nevns6/gR9f0Z14bdna7PajZ7VY2/Sjb9K+Pud1ivs9qN
+ntVjb9KNv0ouFitt9q2vCfhaXxTqZt1cwW0YEk82MlFzgAf7RPT6VnbK9Q+FFuiaDeyADe91hj7B
+Rj+Zrnr1HTpuS3N6NNTmkza0/wAH6JpcQSDTYWwOZJlEjt7kn+lW/wCw9N/6B1n/AN+F/wAKv7RR
+trwnOTd2z11GK2RQ/sPTf+gdZ/8Afhf8KP7D03/oHWf/AH4X/Cr+2jbS5n3HyrsUP7D03/oHWf8A
+34X/AAo/sPTf+gdZ/wDfhf8ACr+2jbRzPuHKuxQ/sPTf+gdZ/wDfhf8ACj+w9N/6B1n/AN+F/wAK
+v7aNvPejmfcOVdih/Yem/wDQOs/+/C/4Uf2Hp3/QOs/+/C/4Vs/2Vef8+lx/36b/AAo/sm8/59Z/
++/bf4UufzDkXYxv7D07/AKB1n/34X/Cj+w9O/wCgdZ/9+F/wrZ/sm8/59Z/+/bf4Uf2Tef8APrP/
+AN+2/wAKOfzDk8j8Tbj/AF8v+8f51HUlx/r5f94/zqOv3g+HYV94/ABf+LJ+Dv8ArzP/AKNevg6v
+vf8AZ9XPwQ8Gn/pzb/0a9fOZ3/Aj/i/RnbhN2dvso2VZ2GjZXxtz0bFbZRsqzso2Gi4WK2yvUfhY
+uPD1zx/y9N/6CK832GvTfheMeHrn/r6P/oIrjxT/AHR04dfvDrMe1GPan4oxXj3PVsMx7UY9qfij
+FFwsMx7UY9qfijFFwsMx7V47+1Z8aB8EvhLeXdlKqeJ9ZL6bo6nBZHK/vbjHpGhz/vMlezKqkks6
+RIAWeSQ4VFAyWY9gACSfSvyl/ag+NTfHL4s32rWkjHw3pynTtFibgC3ViTNj+9K+XPsVHaveybA/
+XsSuZe5HV/ovn+Vzhxlb2FO63ex5yvi3xCqhf+Ej1o47nUpyT/4/S/8ACX+If+hj1r/wZT//ABdZ
+PJICo0jsQqRoMs7E4CgdySQAPU19j6P/AMEsPivqmk2d7Pr3hfTJbiFJXsrqW4MsBYAmNysRXcM4
+OCRkcE1+kYrF4PApPEyUb7XPnqNLE4i7p3Z8nf8ACX+If+hj1r/wZT//ABdH/CX+If8AoY9a/wDB
+lP8A/F19e/8ADqD4qf8AQ2eD/wDv5df/ABmj/h1B8VP+hs8H/wDfy6/+M1wf21lP/P2P3f8AAOn6
+jjez+8+Mbj/Xy/7x/nUdSXH+vl/3j/Oo6+iPJYV99/s9Jn4HeDP+vJv/AEbJXwJX6Bfs7rn4F+Cj
+j/lyb/0bJXzeev8AcQ/xfozvwfxSO72e1Gz2qxs9qNntXxNz1bFfZ7UbParGz2o2e1O4WK+z2r0v
+4Yr/AMU/c/8AXy3/AKCK882V6R8NFxoNx/18n/0EVyYp/uzoofGdTj60Y+tSbfrRt+teMemR4+tG
+PrUm360bfrQBHj60bak2/WqWt63p3hbQ9S1vWLkWekaZbPeXk7fwRIMnHqTwAO5YDvTSbdkB82ft
+4/Gr/hX/AMOIvBelXHl+IPFcbLcNG2GttNBxIfYysPLH+yHr85FAGABgDoBXYfFz4naj8ZPiPrfj
+DU1MUuoy/wCj22ci1tlG2GFfZUAz6kk96yvBXgzWPiN4w0bwt4ft/tWtaxdJaWqH7oZs5diOiIoZ
+2PZVNfsWWYOOW4RRm7PeT8/8lsfIYqq8VWtH0R9U/wDBNz9nT/hZ/wATX8f61a+Z4Z8JzqbVZAdl
+1qeAyfUQqVc/7bR9cGv1h2iuI+C3wn0f4IfDLQfBmiKTZ6XAEe4YYe4mJLSzP/tO5Zj6Zx2FdxX4
+vnOZPNMXKr9laRXl/wAHc+5wWGWFoqHXqGKMVkeLPFWmeCPDWqa/rN2ljpOmW0l3dXMn3Y40Usx9
++B07nivLv2fP2qfCPx8+G9t4oiurbw5defLa3ek6jdoJrWVDnBJ27gUaNwQMYfHUGvLhhq1Sk60I
+NxTSb83/AMMdTqRjJRb1PxGuP9fL/vH+dR1Jcf6+X/eP86jr+lj8sYV+g/7Oi5+BPgr3sm/9GyV+
+fFfoX+zkn/Fh/BP/AF4t/wCjXr5rPv8Ad4f4v0Z6OB+KR6BtHpRtHpU/l/Sjy/pXw9z17EG0elG0
+elT+X9KPL+lFwsQbR6V6N8NRjQrj/r5b+QrgPL+lei/DdcaHcf8AXyf5CuTEv92dFBe+dNRUm2jb
+XknoWI6Kk20baAsRE4r4n/4KI/GsQxWHwq0mf5n8vUtfaM9B962tj/6NYf8AXMV9Z/FL4j6Z8H/h
+5rnjLVgJbXS4d0NtnBurhjiCAe7vj6KGPavx58S+JNT8ZeItU1/Wrk3mr6pcveXc7fxyu2T9AOAB
+2AAr7DhzAe3rPEzXuw2/xf8AA39bHk5jiPZU/Zx3l+Rndfev0o/4Jgfs6f2Pod18XdctMX+qxtZ6
+DHKvzRWmcS3AB6GVlAU8fImej18V/sy/Aq7/AGivjBpHhKPzItIX/TNZu04MFkhG8A9mkOI192z0
+Br9xtK0mz0PS7PTdPto7Ows4Ut7e2hXakUaKFVVHYAAAD2rr4szT2NJYGk/elrLyXb5/l6kZPhOZ
+/WJ9Ni1jFLRXn/x2+MWl/Aj4V694z1bEiWEP+j2u7DXVyx2wwr7s5A46DJ6A1+UU6c60404K7bsv
+Vn10pKCcnsj4o/4Ki/tDbmsfg/olyfm8vUPELxtj5chre1PPcgSsPRY/7xr86bjT4rl97IS2MHEh
+T+VbXibxNqnjXxJq3iHXLk3mtarcyXl5OejyOckAdlHCgdgAO1ZhAPUA/Wv6EyvAQy3CQw8d1u+7
+6v8Ay8j85xmLliKzqJ+noS3H+vl/3j/Oo6kuP9fL/vH+dR16xwMK/RL9m9c/ATwOcf8ALi3/AKNk
+r87a/Rr9mmPPwB8DH/pxb/0dJXy+fu2Hh/i/RnpYDWcvQ7/ZRs9qt+XR5dfC3PasVNntRs9qt+XR
+5dFwsVNntXoXw5T/AIkdxx/y8H+QriPLrvvh8mNEn/6+D/6CK5sQ/wB2bUV750W2jbU2w0bDXkno
+WIdtAQkgAEk4AAFTbDWR4vTXz4P18eEzAPFX9nzf2SbnPli62Hy8475+7/tbc8U0rtK4ban58f8A
+BQH43Dxv8QoPAWk3HmaD4Vkb7Y0bfJc6kRiT6iFT5Y/2i5r5RkYRoWbOAM8Dk/hUkgmE0ouvN+1h
+3+0faM+b5u479+ed27OffNev/sgr4Pb9pTwIvjjP9jG+HkBseSb7/l187P8Ayz83b7btmflzX7ZS
+pQyvBctNXUE3pu+r+8+Lm3i8R7ztd29D9Lv2Dv2cj8A/g/FcavbeT4z8SbL/AFXcPnt1wfJtfby1
+JyP77ye1fS1NA5p1fz/isTUxleeIqv3pO/8AXofolKnGjBQjshM1+TH/AAUh/aG/4Wl8VE8DaPde
+Z4Z8Iysk5jPyXWpEYkPuIlJjH+0Zfav02+MI8V/8Kr8V/wDCDeQfF/8AZs/9l/aPu+ftO3H+1/dz
+xuxnjNfgZlwWEvm+cHYS+fnzN+Tv3553bt2c85zX3XB+Bp1a08XN3cNEvXr+i+Z4Oc15U6Spx+0G
+MUfiK6X4b+HNF8XePNF0PxD4gPhXStRnFq+sm389LZ24jZ1LL8hfapbI2hsngGvub/h0VP2+KgP1
+0ID/ANr1+h43NsHl8lDFT5W9tG/xSsfMUMFXxMeakr/NH58XH+vl/wB4/wA6jqS4/wBfL/vH+dR1
+65xMK/SP9mZM/s++BD/04N/6Okr83K/S39mBM/s8eAj/ANODf+jpK+V4idsND/F+jPVy5XnL0PQv
+K9qPK9qt+WKPLFfA3PdsVPK9qPK9qt+WKPLFFwsVPK9q7vwFGRo0/H/Lc/8AoIrj/LFdx4FjA0if
+j/luf5CubEO8DakveNzZ7UbPap9ntRs9q82522INntTljbcNmd2eNvXNS7BnpWN4o8AePfEguLOx
+l0vRtLOULSXMjT3APUsUX5F/2VOT3IrGrVVNXeppTpubtsfnP+398KtP8KfFQ+MNBMb6b4hc/wBp
+RQD93a6kB+8GRx+9Ub8DowcV8sugkRlOcH04I+lfr/4t/Ysm8d+DtV8P6rr1jBBdwYie2sWYwzD5
+o3BLjAVh0HYnOc1+SPiPQb3wj4k1fQdVRItU0m9m0+7jjfcqzROUfaw6jKkg+lfqnC2aSx+FdGr8
+dPT1j0fy2fy7ny+cYRYaqqlJ6P8ABn7B/sJftGH4+/B6KLVrkS+MfDuyw1YN96cY/c3WP+mig5/2
+1kHAAr6Tr8MP2YPjxdfs6fGHSfFal30aT/QtatU586yYjeQO7xkCRfXYR/Ea/cqC4S5hjliO+ORQ
+6nGMg9K+B4jyv+zsXzwXuT1Xk+q+X5H0OW4r6zRV91oyQLj3r8n/APgpJ+zp/wAKy+JifEDRbUp4
+a8WTH7WsY+S11LBL/QTKGcf7ayeoFfrDXD/Gn4T6R8bvhnr3gzW1xaanAUS4VcvbTD5o5k/2kcKw
+9cY6E1wZNmTyvFxrfZekl5f8Dc3xuGWKouHXofgi6rIjRsAysMFT0Ir9Af2cf+Cmfh/4ffCjS/DX
+xKttZ1LXdKza2+oWMAm+02igeU0jFh+8HKH12Bs5Y18KeK/C2o+B/FmteGdXWNdW0a9msLsQtuTz
+I22sVPdTwR7HnmssNgdcZ9q/bcdgMNmtKMK6vHdNO34nwuHxVXAzfKf/2Q==
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="mpkueonpo.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <b8db601d0b859e92e6700073c717e5@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAdwCs
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++Lrj/Xy/7x/nUdSXH+vl/wB4/wA6jr+nj8oYV+mn7LsRb9nXwAcdbBv/AEfJX5l1+n37LEe79nH4
+fH/qHt/6Okr5LiN2w0P8X6M9jLPjl6Ho3kmk8r3q75VL5Vfn1z6CxR8r3o8o1d8r3NHk/Si4WKXl
+Gu28ERkaRMP+m5/kK5byq7HwZHjS5uP+Wx/kK5679w1pr3jY20bam2UbK8651EOzJGQcZzXTP4zw
+AEtG4/vOBXP7KNlRKMZ7lRk47Gy/jG5OdlvCo9yTX5uf8FI/gkdJ8U2XxV0q1WOw1tlsdbSFcLFe
+quIpiOwlRQpP96PuWr9CtlYXjzwJpPxN8E634S16PfpGsWrWs5A+aInlJV/2kYK491r1Mqxn9m4m
+NeK02fmuv+a80c2JpfWaTpyPw8IDAggEEYIPev1X/wCCa37RZ+JPw4l+H2t3Rl8TeE4UW2eVsvda
+aSFib3aI4iY+nlnksa/Mj4g+A9W+F/jjXPCWuxeXq2j3LWsxAO2QDlJV9VdCrg+jVpfCD4qav8FP
+iXoHjXRcveaVPukt92FurduJoGPo6E49GCnqor9WzfAQzfBOENXvF+fT5Pb8T5PBYiWCr2ltsz99
+KTFYHgPxzpHxJ8G6N4o0C5F5o+rWyXdtL0JRhnDDswOQR2II7V0FfgcouEnGSs0foSaauj82P+Co
+P7On9mala/GDQ7Ui1ujHYeIUjXhZOEt7o/XiJjxyIvc18A4B68/j/wDWr+gvxj4P0rx94V1fw5rl
+qt9pGq2slpdW7/xxuCDg9jzkEcg4Ir8JPjv8MLz4AfFHW/BWv3DCWwl3Wl2w2i9tW5hnH+8vB9GV
+h2r9f4UzT61Q+p1H70NvOP8AwNvSx8dm+DcZqtTW+/qcncf6+X/eP86jqS4/18v+8f51HX3p80wr
+9R/2U4y37Nvw9OP+Yc3/AKPlr8uK/VD9k5M/s0/Ds+unN/6Plr5DiZ2wsP8AF+jPaypXqS9P1PTP
+KPpR5R9Kt+WfSjyj6V+c8x9HYp+XR5XsKueV7GjyvY0cwWKnlH0Fdb4Pjxpko/6bH+QrnfK9q6rw
+nHt06Xg/60/yrCs/cNKa941NntRs9ql284qQ2sgGTGyj1IwPzrgudJW2e1Gz2qUhBwZYgfTzV/xq
+RbaR1ykbOP8AY+b+VSpxeiY2mtyts9qNnFTGMq20qQfQik2+xq7k2PiD/gpH8Ehruh2HxL0m3B1H
+RoUs9aWNfmlsy2IpjjqYnO0n+649K/PQ9a/cvxNYWepiay1C2jvNPu7Zra5tZR8s0Tgq6H6gmvx1
++PvwguvgZ8VdY8JzM09jGRc6XdOObmykyYm+oAKN6Mhr9L4VzL20JYGq9Y6x811Xyf4PyPms3wvL
+bER67+vc+sv+CYP7Rn/CP6/d/CPXLrFhqjyX2gPI3EVzgtPbD2cAyKP7yydSwr9Mc1/PHpup3uia
+nZanpl3JYalYzx3VpdwnDwzIwZHX3DAGv0N0j/grdaQ6VZx6p8Nb641JYUW6mtdSiSF5QAHZFZSQ
+pOSAckZxz1rz+IeHq+IxP1nBQvzfErpa99e/5+p1ZbmVONL2deVrbeh+iFcn4q+Ffg7xxqEd94h8
+M6VrN7HEIEnvrVJXWMEsFBYZxlmOPc18Uf8AD3bRf+iXat/4NYP/AImm/wDD3TRv+iYar/4NYP8A
+4mvmIcPZxTfNCk0/KS/zPVlmGDkrSmmfnHcf6+X/AHj/ADqOpLj/AF8v+8f51HX7sfnjCv1Z/ZKj
+J/Zk+HJ/6hr/APo+Wvymr9Yv2RY8/swfDc9f+Ja//o+WvjeKHbC0/wDF+jPcyn+JL0/U9Q8o+1Hl
+H2q55dHl1+a3PprFPyj7UeUfSrnl1kS6leaprD6F4bs11XWUx9oeVitpYA9Gncd8dI1+Y+1cmJxl
+LCQ56r8kt232S3b/AOH2R1YfC1cVPkpLzfRJd23ol5v03aRJf3VtpVm93fXMNlaJw087BUB9M+vs
+OaueGtW8Ra1ZmPwz4bea3kfcNW1xms7bGOqR482Qe+FHvXW+E/g7p2k3cWq65M3iXX05W7u0HlW5
+z0gi+7GPflvevQNua+eq1sbjd37KHZWc36vVL0SflI9iEMHhPhXtZ93dQXotJP1bS7wPNYPhl4h1
+VQ2u+Nr2JW62ugQJZRD23nc5+uRVqP4D+DTzeafc6rJ3fUb+ecn8GfH6V6Bilrl/szCT1qw53/fv
+P/0q5ssyxcdKc+RdoJQX/ktjiB8EfAQGP+ES0r6m3BP51BL8CPA7EmDRBYP2exuZrcj/AL4cV31F
+N5ZgJaOhD/wFf5CWZ45bV5/+BS/zPOJvhJqGnKDoPjXWrLHS31MpqEB9sSDdj6OKzLm88ZeF+db8
+Oxa7aDlr/wANMTKB6tayHd/3wxr1ocUhFCwKpa4apKm/J3j/AOAyuvuSfmP686umJpxmvNWf/gUb
+P7215HF+B9d8M+NLN59PubXUriFiJY3XbND6LJGwDIfYivAP+Chv7OifFj4OnxFodip8U+EUkvbe
+OCMB7m0IzcQYHXgeYo/vJgfeNfQ3i74YaP4tuU1AiXStdhH7jWdNfybqP2LD76/7Lgr7VzsfjfWf
+BFxHo/jryWt7hxDZ+KLdPLtZ2P3UuE/5YSH15Q9iOlb4fNsVlNeOIrJXjqpL4X5SW8b7bted7Imp
+l9DH03DCtu+8H8XrFrSVvRS/u2uz8Il1C1ZQwuYiD0O8Ufbrb/n5i/77FfvS3w58JoxU+FNDBzzn
+TIP/AIil/wCFd+E/+hV0P/wWQf8AxFfon/ESqC/5hX/4Ev8A5E+N/wBWpdKh+C32+2/5+Iv++xR9
+vtv+fiL/AL7FfvP/AMK68Kf9Crof/gtg/wDiaP8AhXfhP/oVdD/8FkH/AMRS/wCIlUP+gV/+BL/5
+Ef8Aq1L/AJ+H4R3H+vl/3j/Oo6kuP9fL/vH+dR1+znxDCv1s/Y/jz+y58NT/ANQ1v/R8tfknX66f
+sdx5/ZY+GZ/6hj/+lEtfFcVO2Ep/4v0Z72T/AMSfp+p6t5I9KPJHpVvyawvFF/exPYaLom1vEWru
+0VoWGVtkA/e3Lj+6g6DuxA71+UYnEwwtJ1amy+9vol5t6LzZ9jh8PPE1VSp7vvslu2/JLV+RVlTU
+vGGvS+G/D05tGgx/ausqMixU/wDLKPsZ2H/fA5POAfX/AAr4S0zwXosOlaRbC2tI8secvI5+87se
+WYnkk1F4M8H2Hgbw9baTp4YxRZaSaQ5kuJTy8sh7sx5J/AYAFbcjiNCx7V4FKnUqT+s4nWb+6K/l
+X6v7T8rJevWrQjD6thtKa++T/mf/ALatoru224b66FtCSWwa4LxJ8T9I8NXEUep6va6e02fLW4lC
+l8dcD+tP8beJxY2szk/LGpdsckAdeK+Ff2gfido2t+IbTUnXU5bC2t2txeQW5RC5fJUF8FsHjjiu
+fMMVVw1K+Ghz1G0lHq7u225nhqUKs7VZcsbNt9rH3v4X8eaf4ltVudP1CC/tmYr50DhlyOorsopB
+KgIIOa/Or9nX4zWfhPT722On6smk3d6siai0Yl8uR1ACyKhLKpwMMARnrivsLwz8RBfWtvIjMEkX
+Khxtb8RWuCxX1mhCdSym1rG+qadmrXurefkTXo+xqSjHVdH3uerUVi6f4gS5OGIz3zWyrBwCOc16
+BzC0UUUAFVdR0y11exnsr23iu7O4QxywToHSRT1BB4Iq1RSaUlZ7DTcWmnZnkG2f4N39tYX08t14
+FuZFhsr+di8mjyE4SCZjyYCeEc8ocKeMGu+ZSrEFcEcGtbUtKtNY0+4sb6CO7s7iNopoZV3LIhGC
+pHoRXnHgp7rwj4ifwJqlw86JE1xoN/Octc2gIDQOe8kOQPdCp9a+OxWCeEmlT+CTsvJ9vR/Z7P3f
+5UfQqqsdTlUf8SOsv7y/m9V9ruve6SZ2JXHajFaP9mv/AHl/KkNnGpw06A/596lYLEP7H5HB7eHc
+/nvuP9fL/vH+dR1Jcf6+X/eP86jr+0j8VYV+v37G0e79lT4Yn/qGP/6US1+QNfsP+xim79lH4Ycf
+8wx//SiWviOLNMJT/wAX6M+gyb+JP0/U9aZUiRpJHEcSAs8jdFUDJJ+gFU/g3pb6wb/xxeRlbjWQ
+ItPjcc2+noT5QHpvP7w/7y+lZfxIjl1HStP8OW7Mlx4ivE04un3lg5e4YfSNWH4169a2cVlbxQQR
+rFDEgjjRBgKoGAB7CvwvES+tY1Q+zS1/7elt/wCAx1/7eXY/RqS+rYNz+1Vdv+3I7/8AgUtP+3Gu
+rJqzNeuxaWZJOMitOue8YgmyHbg12nAfA37b/wC0Nc+GdXsvBmjjfdywi/1GUuyqkZJEUeV5JOCx
+GemK+RNW+IOoawVjv7a1cR5xHJvO325bivbPjzZXGr/taav/AMJHoa2dpNYJFpnnEOt1DGCsdwMH
+GSS3B6VxvgvQtN1nQLK5ntba8vSJfNkxCSSjspJCjHYDit8XxlheFuShKg5NxjJtNK/M5eT0XK0z
+z1lNTM3Kp7Syu1bXpb87nE6f8SNR0lBHarbQpjbtXIwP++quL8X/ABCTiFohgZAXcTxyf4q9Hh+G
+agFlltgDyoMA4HX0rnvid4MttC8GiYiCSU30QSUR7WUYbK8Ada6cB4j5bmmNpYGhG86slFay3fXW
+Edt3qjkr5DWw9KVWb0ir9P0k99tj3j9mD9r6KDSb7RvGl/Lb3ViUlsrkq8vnQtwU/vblOOCeh719
+ueAPiTZeNNCg1fRbuPVNLclPtMByquPvI391h3B5Ffjr4B8F2XjLWr6GbxhZeEXtLZZYpb2FpYrr
+LlWXapz8oAIr60+Hn7O2m6LoWn6hoHxJ8YaN4gOZT4g8OX4jtJye32Q7omUdPm+Yjqa93O8LhKDl
+KnzKo3tb3bNdHb9RZdWr1LRlZxS7+9f7z9ErbVo51549xV5HVxlSCK+ILf4x/Gv4PQNc+K9Ks/jB
+4QtVJn17wvALHWrWNQcvNZsfLmA4z5RB56V9B/B745eGfjD4cj17wlrcGtaYX8qUx5WW2kxkxTxN
+honH91gPbI5r5I9w9doqC3ulnXrk1PQAVxnxS8J3Hifw4J9MIj8QaXKuoaXMeonTPyE/3XUsjD0b
+2rs6THOc1hXoxxFKVKezX9W81ujehWnh6sasN1/Vn5PZrqtDjfCd/b+N/Den65Y2losF7CH2Tbme
+Nujo3oysGU+61rmwZPlMWlLjsYf/AK9fNvxO8KXGg/EnWrG1vJbTT7uI6rDELoxRqJDiYgZxkSAn
+js9chc+How6ltWjdioJJvScH06VyYTFSqUV7T41dP1Ts/k7XXk0duKwihVfsvgdnH0aul6rZ+aZ+
+Xdx/r5f94/zqOpLj/Xy/7x/nUdf1gfijCv2P/YqTd+yd8MP+wY//AKUS1+OFfsz+xA4H7JXwxBAP
+/Etfr/13kr4HjKfs8FTf9/8A9tkfRZJ/Fn6fqdzp0H9r/G2BSC0WhaMZR6Ca5k2g/wDfuI/nXqde
+c/Dr/S/iJ8Rbwj7t3aWa+wjt1OPzkNejV+F5e+eNSs95Tl+D5V+EUfo+O92VOmtowh+MVJ/jJhWV
+4itRc2J9a1aw/EGprDbsuR0r1DzT87v227OPwj8Yvh/r92JE0+4068spLhE3bZFfcAPVgG3fga8M
+8N+ObTw/o6W0utXmpytM8p3xELtbnGcH3J+tfeHxyTwn4s0qTQPFkdpd2szCeO3uM71ccLIhX5kY
+diPp0r4u+M/ws8HeFvDkVz4WvZDfm8VXjkv2l/dFW3cEDHbmscXlWSZtGE8zU+aC5fccbNJya3V0
+1zP10IjiMdh+anhXHlk0/eWt7JbrpoLP8YtHz5du88KCMr5hgLsTgYIGeOa5Dxx43h8U+HhpsMlx
+PJ9ojmDzrhVCA7hnvnPFYfwOi8OeIvGPiGz8Xz/6BBpaXNkJLp41M3mENkqQT8o6Z6Dium+C8nw5
+8WeJvFVl4nh06ytNM+zCFpNSlhjkLlhI8e58hDgcNkjjmoy3h7hXKsTTxmFp1faQalFtxeq2va1/
+vMK9bNMRTlSqThZ6OyZ5RHdXmm6ws9rIqFYtrB1yME8fQ96spq2rpKZ7bxNrOlytyU0+5McAPsnI
++vrXkN34juLDxJrciQpfRvdyrEZpXwkauwQLgjjGKifxxffdWygQg5+Uyf419LmWbVse5Qu1TvdR
+8+5zYXA08NaVrzta/wCh6X4n8b+ONIt3vV8aX14CFhKu7B3GeF8sHa/5Bq+x/wBnnw1428b/AAq8
+O/FjRNZg0f42SCY/bXVUsvE9lG+yK21BFwrllXas5AdeNx4BH5xt4nvpJd4t40mPAlAbcvuDng17
+Z8A/2u/F3wr1fTtIt4otW8KLcMZtDXbFuJXDNFI2TGxI3kcKWzwM14Z6Z+x/wH+OWl/GbweNXtba
+bRtWsp20/W9AvDi50m+T/WW8g9M8q3RlOeOQPYYJVmjDAg1+T1r+2P4Y0P4x6N8T9B0XxFpcWoqu
+leOrSW0WSG5skGIb7fGxHnW7EDJGWQ49a/S74ceNdO8Y6Dp+r6VdfbNMv4Vntrjy2j8yM8htrAMM
++4HagDt6KKKAPGP2hrBrafwtrUTPEY7qTTpnjKhjHMmQMtxjdGv515tdXMisgw3CAZN5ECffha9a
+/absWvPgzrkiMyy2piuI3T7ysJFGRnjPPfivz6vPGfiaGbyxr+tSBVUbxq1iuflHb9Pwr5pz9jjK
+1O2/LL704/8Ath9TRp/WMHRnf4eaP3NS/wDbz45uP9fL/vH+dR1Jcf6+X/eP86jr+wz+f2Ffsj+x
+O+P2TfhiP+oa3/o+Wvxur9iP2LX2/sofDEf9Qxv/AEfLX5xxy7YCl/jX/pMj6HJf4s/T9T1b4UnH
+iT4iKfv/ANuAkextocV6PXmPw8lFn8TvHdmThrpLHUUHqGiaNj+cYr06vxPLHfDtdpTX3TkfpGYa
+1Yy7wpv/AMkiZeo6wtqjAda8C+OnxZn8KeHJpbTB1K6njsLFWGQZ5W2rx3wAWx7V1Xx81mXRPh34
+lvYlv5DDaMdmlkC6ILAHys8bsH0r4M8Q/Ea1tPEPg++k0vxjDcWV+J7eTxKVe1d9rL8w2jnB4967
+6k+XQ89K50GjfD7xr8WtWjsLWa5a+1nef7TnkYkIj7XkPTKr0471Pa/8E6PHdtrmnXX/AAl9nFaX
+l59ku55bFvMggdG3ShXbDt/CAf4mWuZ0v9ozxLpeuWF5pOoOdSiZdPs7e1hzuMh4jRcY5OM+9dfr
+/wAc/jD4y0270LVdIuNSs7xCkllcS20O/DdRhw2VYDkHggVjHklq1dm3vpWTscn44/4Jk+JdI8az
+2HhjxIfEmiwxQvLNMqR3Vv5hIKPGDgnHzAgcgnIrxj9rz9me9+Ccfgu88P276nfajZ3WnX9vZRM0
+ztE7AzPGgJUdBnp0r3fQ/jD8QPglottHdWl5oumX2ohHu5bxbua4uCMgO+9mOAOAegHSvnr9sn4t
++IP+Fp2moWGrTxyx2Uxme2uSvDzZIypGRyM49K3UleyRjKLS3Plu3g8SaYRbPZX0IUco9m+evHVc
+1P5mukYa3vyeePsrf/E1Lq/xH8RXs7M2uakqP8wjGoSKo9cDd61jv4w1mQgnVr9sdc6hKf8A2atL
+XIuyWWDxHLkLZ6j14ItX6f8AfNdL4Y+G+s3fhi61cMbDyZTE0F6TE8xJGPLU/e4Yk9BXIN4l1Ryc
+6ndnHreSn/2ati38T3t7oRgu7mSbypxtZ2JwmzG3J5IyM0xGs0uoeFsXUs8mwNtuLWObHnR/xowH
+VWXI/Gv2A/Yg8dXsHhy/+Hus3j32p+EBamxu5Wy17o13CJ9Pmz32xt5RIzygFfi+bn7U8hKlv3Z4
+HJxX6z+BYbjwN4o/ZH8VtuQeLvA48G6mM4BMVulzZEjuQwkGfwpIo/Qa2kE0CsD1FS1meH5jNp6k
+9RWnTA83/aJYf8KV8VAnGbZR26mRce358V+b0mhzTyuyR6hYIMAQGz04Y+UcjLdzzX6C/tZ6mbD4
+LapGFaR7uWKBY0TzGbncQF43cKeK+DYdLnjeVLjw2+pOrACaLw5AFA2rgY39q+Wru+PqNdIwXzvN
+/qj67BrlwEL9ZTf4QX6Hxpcf6+X/AHj/ADqOiiv7HP56YV+v/wCxpJj9lT4ZD/qGv/6US0UV+a8e
+f8i+l/jX/pMj38m/iT9P1PR4rn+x/i94ZvR/qtXs7nSJfd0/fxn8hIPxr18HNFFfiGW6Trx6cy/G
+EW/xu/mfpGI1w+Gk93B/hOaX4JI5vX9Cj1GKVJEV0cEMrdCK+e/jN8HdM8X+HbnTL2AfZQyToyMQ
+8bIwYbSTxnGD9aKK9mSTR5585aN+yYNN1y2v9P8AF91FZ29zHdw6Vcw703A7iCwPHPQgdhXo9x8M
+0W+a6mjtXd8CMtCreWxPzDoD1ycg80UVEY8oczluc94y+A1l4u0b+yNR1N7WK1vY9Q8yCIHeeVCg
+cYOGwTXxR+1X8OrXwl8QPDttcSl9HXTpRbyclmIkwRjqOc9aKKzu/aDvpY+b/EkVpa6xJFa7lt1V
+QVBKknvWTcskhBijaFP7pctmiiuki5GjhNweESN0DMxyP1re8NadLqFvIiwiYbywO7b2waKKTA7T
+w94LvMG5lhtNL02WeKwn1S4dpUtvObG4omXbjJ+UHpX6x/GvW/Deval+ybpfg69N/pVr4n2Wlx5L
+w7obSwcMNrqCPlZaKKY0fXnhK58yyI9xW/RRQM+S/wBvbxbbWGhaHo91KkFtI5nmM0DTxtuOxFKK
+VJ/j5zxXyt4aWTUoLybQNV0m2sPtBHlvorZ3bFPeQ9io/CiivjVK9avJ7ub/AASS/BH30KajhKKX
+8v5tv9T/2Q==
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="p.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <c636301d0b859292bc52c015ce5bde@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgBJwCs
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++Lrj/Xy/7x/nUdSXH+vl/wB4/wA6jr+nj8oYV+u/7HUm39lj4Zj/AKhj/wDpRLX5EV+tv7H8uP2X
+PhqP+oa3/pRLX5jx87ZfS/xr/wBJke9k7tUn6fqekePobmTw+NR09PM1TRp49VtFHV2iOXT/AIEh
+cV7FomtWniLR7LVLGQS2d7Ck8Ug7qwyP515qtwUYMOo9e/8An+tV/hNq6+E/EN74JuG22U2/UdDZ
+unlE5mtx7xucgf3W9BX4RRrfV8UpP4alovykvh+9Nr1UV1P0XDP6zhZUftU7yXnF25l8mlK3ZzfQ
+9fKg9Rmub8Q6BHdo37sEfSuk3DNDANwRkV9OcZ4FrXw/vdOkzpKRLEwIeGfOB6bSAce4rnb/AEPx
+dBE729tp07RKzRQmR1LMB0BC8E9K+mJLC3l+9GDVZtBs2OfLpWA+PtNv/idfvq1h4j8NaT4e0xoF
+a2uIrnz3mkLcxsq/dIAzuz1r83v2pPiTp2pw+HtAlsbzTfFHhy61K11DTbwh2tw1zvjHmdGJU1+3
+/ijwpbTJlV7V+Mn7WP7JHxc8T/tOfEHVvDvw513V9GvNUa4s722tt0M8ZVcFWyAQTmkopAW/2Gfg
+F8Ov2mT4zXxdpd5c6hpDWnkCxvWtl8lw4ZnCj5mLKMk+tfXUn/BOP4I6LErJ4MuboNwTc6rcMeeO
+CGFZ3/BP3wB8TPCXj7xAni/4J6T8MdCvNJULqWm2rwvczxyLsik3zyZyGc8KOR1r7s1rTvNsPlG1
+gOGUcj6VQH4M3fwG8GfDn436zpXxB1rUtM+HunvdS2N7HZt9o1kQvhbeIjKo7HK7j/d7ZFfYv7L/
+AOyb4B1rwLaeKNSjGqWmuPJeadpdlqDzWumWzMcWpmGDPKnR24AbIA4zXmf7cHwu1j4N/EN9T0/e
+PCOvzS3Mct+PtVs17K2+4ikDDbCS2SgGAQT6Yryr4AftN+OfgZoOqXegNot94bubo3D6Bqc6rtfA
+zLDHkSAMMf6vcDjkZpIR94+If2R9M0621IeB9duPD8Op7BfaBrVumsaHf7furLbSjemOgaNwwzkc
+18ueCvhPrt/+0TryfCVNA+HPxB8D2zM2kw6g9/pWtahv23CWRkH7qPyWjV0bLIX2kjrXsupfttaj
+4++HOhR6F4b1nwJf6wPs+reN9Q0mefTdFG395JaKis87kZ2FgApIzzW94MHgfxBrfwi8KfByDVNY
+tfCesvql/wCI7iymiSOF4XW5M08iL5s90zqPLXPQZwFFMZ9Sfss/HO1+N/w0h15bFtE1mzuZNL1z
+RZGy2nX8RxLF/unO5c9j6ivekcMqnnmvh/8AZS1y11j9qr9phfD7rL4dF5pjSSQMDC2oqjpcFMcZ
+ODn3Br6B/aL+KS/Dj4bzx2zhtZ1GJobeMSKjBcYd8sQBwcDJHLD0rkxWIjhaMq0tbdO72SXm3ZI6
+sLh5YutGjDr17Lq35Jas+RP2lviVa+O/iXrUsGtLbwWLxWltDHqsVsJ1ViGLK6t905cdPv1wViml
+XLXT6h4uuNEmMxxbQeI4guMDnAj9cj8BUVxcajbyahEJ76/ga4h26ik+m4jGM7QSCcYOOf7ldJ4c
+1PW4rW5EGh3niKP7Q3+mS3Wn57Db932z+NfN4eMqdNKT97Vvzbd38rt28j7GuoTl7i0VkvRJJfOy
+18z4YuP9fL/vH+dR1Jcf6+X/AHj/ADqOv7NP5tYV+sf7IkuP2YPhuM/8w1v/AEfLX5OV+q/7JUu3
+9mX4cj/qHP8A+j5a/LfEJ2y6j/18X/pMj3Mp/iS9P1PZ/NHtWV4k0h9dsYhbXP2HVLOZbrT74DmC
+dehPqpGVYdwTU/nCjz+K/AJqNWLhLZ/1/wAMfWUa88PUjVpu0lt/XVPZrqtGeg/DP4gx+O9JlW4h
++wa7YN5Gpae3WCX1HqjdVboR9DXZ7hXztqVlew6rba9oNwtl4htF8tXk/wBTdxdTbzAdVPY9VPIr
+1X4d/E6w8e200PlvpmuWgAvNKuD+9hPqP7yHsw4Nepgce21hsQ/f6P8AmX/yXdfNabe7UhDEQeJw
+y937Uf5G/wD21v4X8nrv2tFJuH4Ute+cJDcW63Iw1Zdx4aiuOp49K2qKAMy10OK2xg1dltUki2YF
+TUUAec/EH4aaT410HUNG1rTLbVtHvk8u5sbtN8cq+47EdmHI7V8O6/8AsK/EL4Q6tc6h8EPF9n/Z
+EsvnN4T8VWsVzCGB4CSSKVIA6H5G96/SZkVxgjIqncaRb3HVcUAfm7qnxs/aZ8AxfZtS+AA1qWMA
+G50eaXyG/wB1ULDHsPWsTUNa/bI/aFjbRNJ8E23wh0W7Hlz6pdSNDMsbDDATSMzjIJ4jQN7iv0tP
+hiIN8krKPYmszxLfaH4E0mXVNYu/JgT5VAG6SVuyIvVifT+mazqVIUoudR2S3bNKdOdWap0023sl
+uzxb9nX4G+Ff2NPgnJpr6gbwrIb/AFjWZE2yX92wwAik5AAG1FJ6ZJ6mvmb4wfEjV/ij4j1TVLn7
+dZ2vkKltapHazRyw7/3YTzGyQPvZxzgnvXWfGj4rav8AFnUHR9Lvl8PRpcQw6Wth9oTgbRI/zjJz
+yT6cDjOfHL/wpLEmyXQI7mT7PbpA6eGkK2y5yFP7zrj5frXzE6ssdUVWatCPwrq/7zXT+6uibb1d
+o/YYfDrAU3TTTnL4nvb+6u+usn1aSWivKxdANLcyxWOoWlv9rjH9l/Y9P3DCdeT9Rj1evf8A4R/s
+u+K/Fng2LWV8TXGhW17K81vZz2sIcRnGCQiY65+oAPeuH+BXwYh+JfxOay1bTLWX7Fefbb2+Okpb
+FolGMIwY9XxyB39q/RaG3S2hjihURRRqFREGAoHQAdqqlh/rtSUVJxhHRtNpuXa61slv3bt0acYn
+F/UYRtFOctdUmku9n1b2vslfqrfzx3H+vl/3j/Oo6kuP9fL/ALx/nUdf2Ofzuwr9Tf2T5Mfs1fDs
+f9Q5/wD0fLX5ZV+oX7Ksu39m/wCHo9NPf/0fLX5R4jO2W0f+vi/9Jke1lbtUl6fqew+bR5tUfNpf
+OPrX8+859Fcu+bWXrGjQ6tNb3aTzafqtrza6naNtnh9s/wAS+qnj6VP5x9aTzqzmo1I8s1df1/V0
+b0cRUw9RVaUrSX9W809mno1o9Do/D3xwvvDipaeOLTNuDsTX9PjLQP8A9dk6xn1PIr1/SNc0/X7K
+O8028gvrWQZWW3kDqfxFfPomxn0PBB5BHvWMvhy3sbxr3Rrq68O37HJn0uTy1Y/7Uf3W/KuqhmGK
+w3uv95HzdpfftL52feTPZji8JiP4q9nLuleL9VvHztzLtFI+qc0Z4r56074mfEDRFCPLpPiWEDg3
+Ctaz/iRlT+Vb1t+0Lewgf2h4H1SM92spo51/mDXqwzvCtfvFKD84v84px/E3WFc9aVSEl5Sin/4D
+Jxl/5Kez0V5IP2jtIA+bw34kVv7v2If/ABVQT/tDmQEWPgvW5z2Nx5UC/qxrV51gOlS/om/yRf1H
+EL4kl6tJfe2kew7hz7U15UiRndgiKMlmOABXg9/8YPHOqgrZabpHh6M9JLmZruVR/urtX9aj8M+A
+7n4n3F7/AMJP4l1LVjAqOtu+1LTJJyDAPlYcfxZrFZt7eap4Wm5N9Ze6vx97/wAlJVKhB2rVlftH
+338mvc++aO41D4trrF3LpngewPivUUbZJdRt5enWp/6a3HQkf3E3E+1fPf7UfhXWrG38Oz32uDVf
+Et5fL5lxJZPJZ2yeW7eXFCjDYmQMuxJY4z14+mrXwVrGn20VtZeKZbO1iG2OCHTrZURewACjAr5Q
+/bP0zVNL1fQW13XpfEmkzzkLpz6Nn7Mwhb51e3Ic5JHDcdcZNZ4nCzlFVMRPmlfRLSK9Frd+bb8r
+HsYLFwhU9nhockWndvWT9Xsl5JJd7nz1pOjGC1lk063srK4SxmMry+HpwHiyMooMnUg4/OmNbWU9
+pfCG1jW1Vbb7XG/h+6DSMW6r+86bsn6EVk3a6elikmqWSy/6I3keTpuoL8+7gsAen3/zFb+k2C61
+4sttMntojqcs1skRht72NAu0dGc7c4MeM+hrlrTVGlKrJaRTf3K57VKLq1YwT1bS+9n3Z+yN8ObL
+wP8ADP7bbWsds+rzNcKqKVAhBIj4JJGeTyf4q9zqlo2lR6Jo9lp0IAitYEhQAYGFUD+lXa+hwVB4
+bDwpS3S1829W/m22fHY7EfWsTOstm9PJLRL5KyP53rj/AF8v+8f51HUlx/r5f94/zqOv6sPxphX6
+b/sty4/Z08ADP/MPb/0fJX5kV+ln7MMgX9nnwEPSwb/0dJX5J4ku2WUf+vi/9Jmevlvxy9D1zzvc
+Unn+9UvOHrSeaPav515z6C5e8/3o86qPnUed70cwXL3n/Sjz/pVHz6TzqXMFy95/vR5wqj51HnUc
+wrl8XJHc0hnzjk1R86jzafOF0XvOP+TXpvwT+c60/oYl/RjXknnV6x8C2za60/YyxD8lP+Nd2Al/
+tEX6/kdOH1qo9Vr47/beuJT4g8LR27xRXCPcMvnSXCKyi2O/mEZztJxnjIGOa+wd/tXxZ+3PcWkP
+i3w6t/cSWkTRXDRul9NaF5BGgUZjU7hkgHPSvcxc+aMUu/6M+vy1Xru/ZnzfpjCXT7oabdQu4sv9
+IWbUdQAQF1xtyM5yF/M16N8ANOiv/jBo8NpdJe6UdRgJZtTuLiTco3YMcgAA4bnr0ryy61rSxp8s
+V5q5srhLaIW6xeI7hRLluSxMf90k/WvY/wBmfV0vvjjpouLiNL5L1VNnFqxu0C+U43bSo2nIA79c
+e9eRitaXK+rivvlFM+qoq03K20ZP7otr8T9EDyaKKK+0Pz4/neuP9fL/ALx/nUdSXH+vl/3j/Oo6
+/p4/KGFfpD+zPLt/Z+8CD/pwb/0dJX5vV+iv7NsuPgH4HHpYt/6Okr8f8TXbK6H/AF8X/pEz1Mvd
+py9D1fz6TzaoebR5tfzdznt8xf8AOo86qHm+9J53vRzhzGh5tJ5p9qoecfWjzj60c4cxf80+1J59
+UfOPrR53vRzhzF7z6POqj53vR53vS5w5i/51ec/Hzx5rHgLwDHqWiSFbl7+GF+GYbCGycD6Dmu38
+7/Oa8y/aBtjqng3T7Xkq+oo7DcBkKjdz9frXVhZ/voW7nbgbTxNOL7nkth+0B491GItFrrWBXGBJ
+DKRLnuCWHT2qsdb1bWtVvdYl13U57q8Vo7l5Z5TEhChWEcbZ2YA4x+tclNPp1lZahZNYRJMC6F5r
+hCxxwDxggcZp2gxL4nS6YS+VHaSKGhDAgkruBGH5FfUSnOStJux+gQhGm7xSTOpm1vUEsbqJdUuJ
+PMRVSaSUSYKtkBcrlT2PtXc/s9+KtV0r4u+HTqd9FeQPepM0u8GQNlVIJwDjDHjnmvJdRH2OIRRz
+Wb712lC5D4A9A5NXvDviXVNM1/T7prCNIY5Y4zOqybwOmcex5/CvPxcW8PPkWqV16rVfikejhalq
+0FN+63Z+j0f4H7HZpaxfB3iCPxV4V0nV4mV1vLaOUlegYgbh+ByPwrar66nUjVgqkHdNXXzPjJwl
+Sm4TVmnZ/I/neuP9fL/vH+dR1Jcf6+X/AHj/ADqOv6iPyVhX6Ffs5y7fgR4JH/Tk3/o56/PWvv79
+nmXHwN8Fj/pyb/0a9fjfii7ZVQ/6+r/0iZ6GCdpSPUPO9zR559ao+d/nNHne/wCtfzNzHrcxe+0H
+1NHne9UfNPtSedRzBzF/zvejzveqHne9Hne9HMHMXvOo8/3qj5/0pPP/ABo5g5i/5/vSeb71S873
+/Wk8/wCtHMHMXvOryn9oq4uv+Eb0BbObyZG1Bw/73y9y+SePU/hXpHnV43+05O40Dw9EPMj8y6mY
+SocMu1F5B9ea7MHK+Iiv62PTyt3xlNev5M8ilsrqCOR57ie3KkAyJdfMwyMEZGSfbNPh0y8ur2Uw
+3F3bskQAklkClySe6g5/HtxT/sUWsaVZQzW0EMjhXW5uLpnUMBljz0J/rVaTT08NWjO1zbSXAmTd
+DbrklGHAGCGH1r627P0iyLMnhy5tp2urmWa8JCqsSTKuzHU4IGeakvdStdIhV7xLmIT7lTI3/XgH
+361l/aJriyuFtJIkurdVmE8ZJcgNypVmK8jjOOfaq0o1S8igNwkcip90PbRyBAeSQDyDj0qkupLP
+0f8A2GPipb+M/h3LoRuxcXWlNlPmyTGThsfRuf8AgYr6cr8o/gf8S4vgv4/sdZhv4Bbs4S6gAjiE
+iHg7sHjjgHp0P8Nfqbomt2fiHSLTU7CYXFndRrLFIvdSP59iOxrbLKipp4N/Z1j/AIXt/wCA/D6c
+r6nJmlJzksWvtaS/xLf/AMC+L15l0P57bj/Xy/7x/nUdSXH+vl/3j/Oo6/rw/C2FfeX7P8u34JeD
+RnpZt/6Nevg2vuf4Cy7fgv4QH/Tm3/o16/F/FR2ymh/19X/pEzswrs2el+cPWk86qPnUnn+9fzDz
+Hoc5f86jzqoef70ef70cwc5ea52gnaz+ycmoW1FQ5XypyQM8RHB/Gq/n+poE+PpVxnFbq5rCrCPx
+Rv8ANkratGrAeVLyM/dIqB9bEbKpQbjzhmPP6U/7SfU/nSC4Prj8a6I16K/5dfizvp43Cw+LDp/9
+vSIm1uUEAeUh7Z649MGoJNZuDkefGmewAH5VbMwOCQCfemsY3OWjjbHTKg10RxlFf8ul/XyPRp5t
+gof8wkfvT/OJnPqd1Jki6kPY7SP6V578QrFNRuoJFaOW6RGBSYlxnjBYZ46dsV6g8cEmN0ETY5GV
+FVbjR9MvG3T6dbStnOSnP5iuunmVKLvyW9LHoR4gwsV7tC3pb/JHgl74YjuWQva2uAmGG1sBvYdM
+VT0nwydFlZre1jh3k7nWdmJGcjqMgDsO1e/v4W0N12/2XAg/2Ny/yNQS+DNDlGPsPl+8UrKf5mup
+ZtQ6p/h/mbLiLDPeMvw/zPnweEJoYlKS75eSQERCWznlxycZ61eFheRBW3PK44GHGAP73vXt0ngH
+Q3XAiuUPqLgk/qKgk+HWjuPklu4m/veYG/QitFmmHff7jWOf4J/zL5f8E8DvvC51GWaaeOUzum1p
+CVYsoGAoXGOle2/C39o7xn8L/B1roGm3F/JZwksi3MUcsiZA+Uk59OxxznvViT4Zaey/Lf3St6lE
+I/Kq5+Fdk+C+p3BP/XBP8ayq4zB10lObVtmnKL++LT+V9fkdlDiHCUm+WW/Rw5l9zTX9M+Hbj/Xy
+/wC8f51HUlx/r5f94/zqOv7fPxdhX238DJdvwc8JD0tG/wDRr18SV9ofBKXb8IvCoz0tW/8ARj1+
+KeK3/Ioof9fV/wCkTNqT5bnonn0efVDz/ejz/ev5dub85f8APo8+qHm0ef70XDnL/n0nnVQ86jzT
+70XDnL/nUedWf53vR53vRcOc0POo86s/zvejzvei4e0L/n0efVDzvejzvei4e0L3nUvnD1qh53vR
+5tFw5y/5/wBKTzveqHn/AEo8/wClFxe0L/ne9IZS38RFUfP+lJ53vRcPaHwFcf6+X/eP86jqS4/1
+8v8AvH+dR1/oacbCvpr9m3xlDqfg+Tw/JIFv9KdpEjJ5e3ds7gP9lsg+mR618y1b0nV73QdTt9R0
+26ksr63bdFPEcMp/qD0IPBGa+R4pyCPEmWywTlyyTUovopK9r+TTadtr31tYaZ91/aP85o8+vn3Q
+v2nmWBE1zQzNOAAZ9NlCK/uUbgH6HHtWv/w09oX/AEA9V/7+Rf41/LVbgPiSjNw+qOVuqlBp+nvf
+mk/IV5Htfne9J5w9a8V/4ae0L/oB6r/38i/xpP8Ahp3Qf+gHqv8A38i/xrH/AFI4k/6ApffD/wCS
+FeR7Z53vR5x9a8U/4ae0L/oB6r/38i/xo/4ae0L/AKAeq/8AfyL/ABo/1I4k/wCgKX3w/wDkgvI9
+q8/3P5UedXin/DTmg/8AQD1X/v5F/jSf8NOaD/0A9V/7+Rf40f6kcSf9AUvvj/8AJBeR7Z51J5te
+Kf8ADTmhf9ATVf8Av5F/jR/w05oX/QE1X/v5F/jR/qRxJ/0By++P/wAkK8j2vzqPOrxT/hpzQf8A
+oB6r/wB/Iv8AGj/hpzQf+gHqv/fyL/Gj/UjiT/oCl98P/kgvI9r86jzT714p/wANOaD/ANAPVf8A
+v5F/jR/w05oP/QD1X/v5F/jR/qRxH/0BS++P/wAkF5Htfnj/ACaTz/8AOa8V/wCGm9B/6Aeq/wDf
+yL/Gj/hpvQf+gHqv/fyL/Gj/AFI4k/6ApffD/wCSC8j2rz/ejz/evFf+Gm9B/wCgHqv/AH8i/wAa
+P+GmtA/6Aeq/9/Iv8aP9SOJP+gKX3w/+SC8j56uP9fL/ALx/nUdSXH+vl/3j/Oo6/tkphijFT2EC
+3V7BCxwsjqhI9Ca9L/4QfR+9sxPc+Ywz+tfJZ5xNgsgnThioybndrlSe1t7tdz9I4T4CzTjKnWq5
+fOEVTaT521du7VuWMu2t7HltJXqn/CEaN/z7P/39b/Gj/hCNG/59n/7+t/jXzP8AxEbKP+fdT7o/
+/Jn3v/ED+JP+f1D/AMCn/wDKzyz86Pzr1P8A4QjRv+fZ/wDv63+NH/CEaN/z7P8A9/W/xo/4iNlH
+/Pup90f/AJMP+IH8Sf8AP6h/4FP/AOVnln50fnXqf/CEaN/z7P8A9/W/xo/4QjRv+fZ/+/rf40f8
+RGyj/n3U+6P/AMmH/ED+JP8An9Q/8Cn/APKzyz86Pzr1P/hCNG/59n/7+t/jR/whGjf8+z/9/W/x
+o/4iNlH/AD7qfdH/AOTD/iB/En/P6h/4FP8A+Vnln50fnXqf/CEaN/z7P/39b/Gj/hCNG/59n/7+
+t/jR/wARGyj/AJ91Puj/APJh/wAQP4k/5/UP/Ap//Kzyz86Pzr1P/hCNG/59n/7+t/jR/wAIRo3/
+AD7P/wB/W/xo/wCIjZR/z7qfdH/5MP8AiB/En/P6h/4FP/5WeWfnR+dep/8ACEaN/wA+z/8Af1v8
+aP8AhCNG/wCfZ/8Av63+NH/ERso/591Puj/8mH/ED+JP+f1D/wACn/8AKzyz86Pzr1P/AIQjRv8A
+n2f/AL+t/jR/whGjf8+z/wDf1v8AGj/iI2Uf8+6n3R/+TD/iB/En/P6h/wCBT/8AlZ5Z+dH516n/
+AMIRo3/Ps/8A39b/ABo/4QjRv+fZ/wDv63+NH/ERso/591Puj/8AJh/xA/iT/n9Q/wDAp/8Ays8w
+uP8AXy/7x/nUdSXH+vl/3j/Oo6/Uz+emW9HONVs/+uyfzr2Xua8Z0j/kLWf/AF2T+dezdzX4F4l/
+7zhf8MvzR/Y/gP8A7hj/APHD/wBJkFFFFfjJ/UYUUmaSSRIhl2CD1Y4oAdRUcdxFK2EkRz6KwNOZ
+1T7zAD1JpBZjqKj8+P8Avr+YpfOj/vr+YphYfSUvevcv2X/2Y7n4+6leX+oXc2l+EdNkEVxcW4Hn
+XU2NxhiJ4XCkFnOcblAGckaU6cqslCCu2eZmWZYXKcLPGYyfLCO7/JJdW30PDM/40Zr9RLH9kz4I
+6aYtHPhPS7i8Me/Zd3LyXLr3c5fcfqK+ff2qP2MtI8AeEr3xp4Gae2sdPAk1DRZpDKiwkgGWFmyw
+25BKkkFckYIwe+rl1alBz0dux8BlniNk+ZYuGD5Z03NpRckkm3tqpO1+mlu7R8fUmaM12nwe+E+s
+fGvx7Z+F9HZbdpEM93fSKWSzt1I3SsOpOSqqO7MOgyR50YuclGO7P0nE4ijhKM8RXkowirtvoluc
+XRmv0x8O/sbfBvwJpVvFrGlQaxdSERm/125JaZz2VdwQZ7BRXP8Axc/YM8E+JNEuZ/BFv/wimvxx
+loEjmd7Odh0SRGJ2g8DcmCOuD0r1HlldRvo32vqfllLxOyOpiFSlGcYt2U3Fcvro27fK/dH53Zz2
+NBOP/wBdd78G/gzrfxl+I0fhK0H9mTQ731O6mTeLCONtjkr/ABPuwqr3PcAEj730D9hn4RaNpcNr
+eaFNrdyo/eX1/ezebIe5IR1UfQAVz4fB1cSuaG3mfQ8QcZ5Vw5UjQxTlKo1flgk2l0bu0lfprfra
+1m/xguP9fL/vH+dR1Jcf6+X/AHj/ADqOv7VP80GWtI/5C1n/ANdk/nXs3c14zpH/ACFrP/rsn869
+m7mvwLxL/wB5wv8Ahl+aP7H8B/8AcMf/AI4f+kyCiiivxk/qMveHtAv/ABXr+maHpUIuNU1K4S1t
+Y2OAZHOASewHJJ7AGv0p+G37L/w2+A/g6XVddsrLWtQs7ZrnUte1eBZQoVdzlEORGgwcBRnHUk18
+P/snX1npv7R/gSW+2iF7qWBCx4EzwSJH+JYgfU1+hP7UtvPefs8fEFLZWeX+yJnwvXaoy3/joNe9
+l9OCpTrNXa/yv+J+C+IWZYyWZ4PJaVR06VXlcnHRvmm42v2ile213r0tg+GtK+C/7U3g27udO8P6
+bqVhFM1tK5sPsl1byAAjDAK65VgQQcHPsRXz14e+FfgH9mX4+a3p3xJuLDUPBmoaT5+hXWt2f2jc
+3nKGRgEI81BkFgBlSp4JIHV/8E1oJD4V8d3anday6jbxo45BZYctz9HWsj/gpTqNo938PbBWU6hH
+9tuXUfeWIiJQT9WHH+6a1m1PCxxbiuZeWj1t/wAE8HL6NfCcTYjhWliKjw1RNfF70fc57p2smmuV
+u2q31sfQ/gf4efBX4k+HYdd8N+EPDGqaTM7pHcx6TGoZlYqwwyA8EHtXAfFjUP2dvAP9u+HdS0vw
+rpPiWKybZbf2OPMV3iJjwyxkAnIPXvWv+wcP+Mb9FIx/x+3v/pQ9fIX7b/8Aycz4l7/6JY/+iBWt
+et7PCwqqCvK3TujiyLKJ4/ibFZTVxdX2dHns1PV8k1FXumtnrZI8Gtg628QfO8IobPXOK9K+Hv7S
+HxC+D+i/2d4Z1xLbSI3ef7BcWcU0e5juY5I3cn/arzrHSvor4R/sSa58YvBGl+J4PF2kWOk6nGxW
+NbeWaeLDFHRuVAYFSCMmvnqEKspWo7+Wh/QmeYrKMLhubOuX2TaXvR5lfV7Wetr2PV/ht+yN4p8Y
+fEfw18XfEfje3kub2e31+RLG1dJzuVXWAOWwseCEIH8ORjmvTP21fjFpHgH4T6t4adnl1/xLavZW
+duI2KBH+WSVnxtAVd2BnJOMDGSPnzRfjl8evhl8T9J+Hk9t/bEWmXcOmQ6eNHCrd2isI0kWZRkKY
+xuD54x83Q19r/GTwbpHjz4Z+I9H1u3jnspLKVw0gGYZFQssqnsykAg+1fQ0FCdGpDDpqXW/fr1/r
+sfzrnU8Vg85y/FZ1ONagrOmqTStBNOOnKtFdafaSspLp+Po4HHQV1fw7+Kniz4Talc6h4S1c6RdX
+KLHO3kRzLKqkkKwdT3JPGOtchayNLbQuxyzIGJ+o5r2D4C/s3ax+0EmrvpGv6XpX9lyRx3EV6kjy
+kOpKuqrwVO1hnI5Br5yipzklS38j+ls1r4HC4SpUzPl9irc3MuaO6SurO+tum56boPwh+IP7cvh2
+08YeIPGOl2kenSy6TFbHTmwm0qzyhVcLubcOeOFA4xX2Zr3izQ/2e/hJaXXiDUru70/RLGG0N08b
+TXF06RhVyBn53K9SQATyQOa+J/GE/wAYv2LRaeFvD+speeG78m/i1CHRBLG1y5xLEc7iCNqkAnkE
+Y719y/CrWtY8bfCvw/qXi3SE03WNQsVe/wBPkiIUMw5BRuQGHO08jOD0r6PBcqlOKTVW2rev6n80
+cYe1lSw2Jc4Ty7nfsoU7RkovVrWCaejW0rPdXPy48K/HLxn4F8V6/wCIvDGqDQ7zXbp7u7iWGOeN
+90jyKhDqeFMjcjGc17/4O+Hnxe/a68P2/jmT4mJ4fMTPpYtLKOa3U+U5JdljcKWJc8+mB0ArnoP2
+PLn4lfFn4jab4T13SdE0jQdYNsLO4SSSSFJEEihVXjYCXQZP8BFbWoeHv2if2Ypl8G+AYp/E3h5l
++3rfWOiLKnmyE+YpJDEEMvQnoQe9eXShUhrWUnDXZ9b+q8z9QzXH5djnCOTVKNLGtRd6kUmoOKdn
+Jxkr2cbK7020PziuP9fL/vH+dR1Jcf6+X/eP86jr+zz/ADxZa0j/AJC1n/12T+dezdzXjWkf8hWz
+/wCuyf8AoVeyngmvwLxL/wB5wv8Ahl+aP7I8B/8AcMf/AI4f+kyCikzRmvxk/qEdFJJbzRTQyyQT
+xOskc0TbXjdTlWU9iCAQa+5fhN/wUE0afQYNM+JGn3VtqUcYifU7C38+3uxjG541+ZGPdQCuScYH
+A+GM0ZrpoYiph5c1NnzWecO5dxDSjSx8L8u0k7SXez8+qd15XP0G1L9ub4SeAvD72ngvR7m/ZSzw
+6fp2m/2fbbz1LMyqFB7kKx9q+Ivid8Sdc+LfjS98T6/Mr31wAiQxZEVtEudkUYPIUZPuSSTya5aj
+NVXxVXEJKey6LY5ck4UyzIKk6+Fi5VJbzm+aVu3Reul31Psn9mH9rjwF8HvhBpvhjX/7W/tS3uLm
+WT7JYmWPEkzOuGzz8pFeBftJ/EbSPiz8ZdY8U6F9o/su7gtoo/tUXlSZjjCtlc8c15mKKU8TUqUl
+Slsi8Dwzl+XZnWzahze1q83Nd3XvNSdlbTVdxa9v/Zr/AGpNU+AVxdadc2T634TvZfPlsY3Cz20p
+ABkhJ4O4AbkOASAQRznw/NJ/npWNOpOlJTg7NHtZjl2FzbDSwmMhzwluvyaa1TXdH6VWn7e/wiub
+USyajqtrJjPkTaVMZM/8BUjP0NeG/tEftzL4+8NX3hbwLYXmn6dfxtBeavfgRzPEeGSGME7dwyC7
+HOCcDPI+SKOtd9TMcRUi4N29D4XL/DvIcuxMcVCMpyi7pSldJrbRJXt53QKoUAAYA4A9K7b4PfF3
+Xfgl42h8R6EUmbYYLyxnYiK7gJBKMRyCCMqwHB7EEg8Tmk9K86MpQalHRo/RMTh6OMozw+IipQkr
+NPZr+v8ANH6O+G/+Cgfww1WySTVhq/h+8x+8t57Fp1Df7LxbgR74H0rm/iV/wUO8M2OlTweBdLvd
+Z1d0IiutRgNtaQnszAne+P7oAz6ivgeivSeZYlx5br1tr/XyPzOl4acPUq6rOM5Ja8rl7v4JSt/2
+96nonwo+PHij4TfEe68ZW0/9r3mos7avb3bFU1AO5ZixA+VwxLKwHy9MFSRX2jpP/BQf4Z3thFLf
+22uaZdkYktmsfO2H2dCQR78fSvzqpOe1YUMZWw6tB6eZ7ud8G5Pn9SNbFQamla8XytpbJ6NadNLp
+aXseK3H+vl/3j/Oo6kuP9fL/ALx/nUdf2sf5msvaCgl13TIznD3USHHoXAr6Xl8DWCSuomucBiBl
+h/hXzV4d/wCRj0j/AK/If/Ri19bT/wDHxJ/vmvk86yzBY+cJYqkptJ2v02B8XZ9w1+6yfGToxnrJ
+Rtq1te6e12c5/wAITYf89rj8x/hR/wAITYf89rj8x/hW/RXzv+rmUf8AQNH8f8yf+Iqccf8AQ1q/
+ev8AIwP+EJsP+e1x+Y/wo/4Qiw/57XH5j/Cvob4afCXQ/i/8KNdTQ4Zbf4k6SykLJdt5F1GzZV/L
+b5QCu5Tjoyg96vfEv9n63h1jXbTwNGs1v4TsI5NclvLp3luLh4zIFhTB52L90EZaQDtXmvLchjVd
+GWHSa3uvSz32d1bue4uPPEKVCOJhmlSUZJNWa13uvh3jyty7LrqfNf8AwhFh/wA9rj/vof4Uf8IT
+Yf8APa4/Mf4V7H/wojxSPEVhoEkmkw6xcWgvri1lvgp0+E7cG5JGFYlgAq7j1PTmreqfs8eI9Lvf
+D8Y1Xw/e6fr0xtrDV7S/L2ck+1mWMvsyC21guAQSMZBxW39k5ArL2MNdev8AXQ5V4heIjTf9oVtH
+bp5eXmvTqeI/8ITYf89rj/vpf8KP+EIsP+etx+Y/wr2mX9nfxxDpl9eGwhL2mprpLWglbz5J2dEV
+kG3BjPmK27I+XJPSmXfwE8S2sHiyWO80e9XwvGH1MWt2z7G2FzEnyANIAOV45IHXoLKcge1KH4+X
+n5oH4heIi3zCt+Hm+3k/uPGf+EIsP+e1x/30v+FH/CE2H/Pa4/76X/Cvff8AhlzxqmoWNhcXWhWN
+7ewCe3t7vUDHI/qgXZksO4GcetVZv2a/GMXiZdDWfRLi6SF7m7mhv8w2Ea7ctcEqChO75RgkhWPR
+TiFlfDz/AOXUO/X/ADLfH/iNFXePr726b9tjwz/hCbD/AJ63H5j/AAo/4Qmw/wCe1x/30v8AhXpn
+j74Yaz8Ohps1/JY6lpWpxmSw1bSrj7Ra3IGCdrYHODnpyMkZwa6Lwj+zt4q8a+GNK8QWN5odrYao
+7RWgv70xSySBmXYF2HLHYxABzgVrLJ8hjTVWVGCi+uv+fkYw8RfEKdWVCOY1nNataXtprt5r7zxH
+/hCbD/ntcfmP8KP+EJsB/wAtrj8x/hXuFj+zj4tvn8QgXWiWyaDeCyv5ru+MUcbFEcuGK42hZFyT
+g8HipB+zP42/4Ta68KkaYmoQWI1JZ3umEE0G/ZuRtuchuCCBjg9CKz/srh//AJ9Q79fLz8zb/iIH
+iNp/t9fV26b6+Xkzwv8A4Qmw/wCe1x+Y/wAKP+EJsP8AntcfmP8ACvT/ABB8KfEHhjwHovi/UY4I
+dJ1d1jto/Mb7QNysylk28AhCeCeorS8B/A3xB8Q9Mg1Cyv8AQ9OgupWgs49SvxFNdyLnKxxgMc/K
+x5wSFJxjmtJZNkUYe0dGFr2vrv8AeYR8R/EGdRUY5jW5mk7abPZ7dbnj3/CEWH/Pa4/Mf4Uf8ITp
+46zzj/gQ/wAK9i0P4GeJNV1DWrW9utG8OR6Pc/Y7y71m/WGFZsA7VIBLAh0OcAfOO9cx8S/BusfC
+PxL/AGH4gtSLxoVuI5LJvNiljYsAwJweqsMEDpVQyTI6k/Zwowb3/rUmp4keIFKn7apmNZRva+m/
+3eTPiW4/18v+8f51HUlx/r5f94/zqOv1Y5WaHh3/AJGPSP8Ar8h/9GLX1tP/AMfEn++a+SfDv/Ix
+6R/1+Q/+jFr62n/4+JP98142P+KPzPkM9+On6P8AQjooorzT5g9e+A3xH8HfCZ7vxDqSa5deLAst
+vb2ViQtpPAwUqJCeh3gnJyBwcE1q/CH9oWPwJp3xH1DVZJpPFXiGZr2yeO2MsC3Gx9vmcjChmUY9
+F/CvDfyox+VeZVwFCs5upd81r69ndL0ue1QzfFYZU1RslC9tOslZt93b7j6Ui+OXwv1X4l6Z421n
+RrqTVp9JWz1BH08Sx210uCJkyTvON0e4DO1V4xmoPGvxt8CfELwp4U0DWtX8R3sOl6st9f3txpqR
+zXcaCUhVEJVU3MyAAAbV9xz85n3pMVisroKUZJu8dtdt7W08/wAjqef4txlBxi1LWWm7dm72et7f
+n0PqyX9tKMaFqUkUdx/bH9riWwtDaARHThIg8t5MkCRoxIc9mYDtWVoXx9+Hvge6+Il5oNvf3X/C
+ROt/ZafdacVijvBG25XJYjY0m189stwcDPzR2o/L0qVk+EinGKaT3131TXTpb8zR8SZhKUZzabjs
+2trpp9eqevTRaaH0H4x+P/h3xT8RvhZ4jdr+ZfDayNqkj2ex2kZFBaNc/NlgfoPWn+Ef2jtD8M/F
+zx/rDW16fDfioxMt3Dbqbq2eOLYrmNuGXLMMHPQHBya+eelB4Naf2ZhuT2dna1t+nNzfmYf29jva
++2TXNzc23Xl5O+zjv56nrnxw+LOnePdI0LRdJ1HWNYgsH86e/wBUgitlkkClR5cMaqF4Y5OABwAO
+9dH4K+O3hfw/8N/hxoV4t+b/AMPax9vvDHaF08vNx9xs/Mf3ycfX0r5//wD1UvX3NaPL6EqMaLvZ
+O/zd128zGOcYuOJnitOaSSfomn3v9lbtn1xpPiXwz4++Ffxu1i9mv4fDWqamZHaCJRdrF9mt1JWN
+jjIIPHcD1qjB+1L4NT4rPq8tvqQ8OWuhf2Tbt9l3zTuZVdmZc5VdqqOeSc9K+VCBnPP0z/n/ACKX
+PTrXIsoovm522nt5KyXzem/4Hoy4kxVoezik1u97u8pfJXlt+PQ9s+MHx2074p/Cjw/o7RTQeIbS
+/N1dQrb7LWKPEoVEbuFV0UcchTVP4E+OPAfw5vLXXdQu9a0zxNG7R3flWUd5a3ltyQifLuibOzLZ
+z8pGcNgePijp29q7FgaKoPDxuotvZ9+m2x5n9rYl4qOMmk5pJJtPps999Oh9EeHPjj8PdNPji7t9
+M1PwrrOraibuy1ays4ru78nCZQGXcsbMVkODkDzMg5Arhv2kfiRonxg8badrOivcw20GmR2bpewF
+XEiyyufXIw45+teYdu/0oye1KjgKNGt7eN+bzd+iXVX6FV83xOJw7wtRR5W76K2t2+jt17a6X2Pk
+C4/18v8AvH+dR1Jcf6+X/eP86jr74+7ZoeHf+Rj0j/r8h/8ARi19bT/8fEn++a+SfDv/ACMekf8A
+X5D/AOjFr62n/wCPiT/fNeNj/ij8z5DPfjp+j/QjooorzT5g9++Cf7P2j/Fv4Tarqj3L6f4gTUzZ
+w3zzN5MMY8okmLozbXfAPUlelath8BPCM3jX4gfbNO1Ky8PeCbRWexN+WuNTcxNN5xk/gjKrhVXH
+OSSMFa8V0z4la9pHgHUvBlpPbxaFqE/2mceV+/EmUYMkmflwY1PSull/aO8dTeJl15rvTv7RNp9g
+uFWyHk3kOSwSZN2GKkttIwRvYdGIr56phse51HCpo721att+auvLdH2FDG5RGlSjUo+9FLmdk7v3
+vybi/wC9az2R22rfALw/rerfCrUvDgvLLQ/Gbo91pdzceZNbx+SZ3McnX7iuvOcHaR1p+r/APwp4
+q1f4ZS+Dbi8sNA8Uy3UdxNcTmd1EKGQFC2cMyxyAdhwe1c34f/ad8RWHiG81rVbe3v7iLSJNN0a0
+toVhtNNdip8wR55HyqDg5IGMgVHf/tK+INX8Dy6Nd21raapbXtveaTqGk2yWyWJjfc42ZOS3zDjq
+GYHIJBx9jmScUpbab33vr58t46vs97nS8TkkoSbhvrblt8PLonf3ee0lZae8trHWeMfgp4J1Dwj8
+QLvwrbajo+qeCbtraaS9vDNFfBI0d8g525DEDGMED1rrvFH7N3w40m81OwSy1RWt9EbVlms9WNxe
+5DlcLZnLOvAO/G0n5eteEeNPj34x8eaFPo+o3Fha2FzIs16mm2Yga8dcYMzZO77q8DGcDsMVtXX7
+VPxCuppLhbjRre+kgNt9vg0xRcJGTnarljgZ5weM84qXhMycY2qd/tPtHrbXVPva5azDJOebdHov
+sR7yvpfTRxV762v6914X+A3gjTdJ+HNj4ng1LVtd8bxu63dldmCKyxb+dhUH3gAQuTnJBPTimaB+
+zboHh3RfF2s+JJ49bj0vVW06zgn1VdLgMQZB5k03ZyH+70yuP4uPM/CX7QfjXwVoNppFhd2Nzb2G
+4WM2oWazz2YYEERuSMDBI5zgcdOKo+EvjV4s8HJq0cF3bara6tcG7vbTWbcXUU05OTLgkYY4XOOD
+tHHArV4XMLytU693r73TT3fdstDnjj8mSp3oapP7K0fKlrd+/wC9d3a2+4n+O/gDR/hr4zFl4d1I
+appF1bC5t98odoDuKtGzjqMgMD1weema9O8X/Bv4b/DtNO8Ma8viO48R3ukvqP8AbmlxyXSxyKyq
+QLdM/ICc8jGAAWBOa8O8b+Nda+I+uS6x4hvPt17JGIRtQJHHGM4REHCqCWPqSck812Nr+0l8QLTw
+vHoaarAY44PskeoPbA3qQ8DYJc+gA3YzwDnPNdVShjHSpRU9V8Wtr9ne2tuqtqcNDF5bGviJype7
+L4PdvZdVbm0v0aenQ7Sz+AGhTfA661KQ3X/CxYdBHiE2wmbylhZnaNfL6cpGV9d3NdBov7PfgLT7
+nwN4Y1uPU9U1/wAWWM95/atreGGK2MUSyEIg4YfPgZDdMnrXkNl8e/GGn+L77xLDNp4v7yxj02WF
+rPNsLdPuosW7jHPf+I8c1PoH7RfjjwzoNppNnd2EiWMbQ2N3dWSy3NnGwxsjkJ+6AABkHhQOcVyV
+MLmMk0qm7vu1ZtbXttF2aXU9Cljsmg4uVHZJfCndKS1avbmnG6vumem+Ff2ffCNr4TvbnX421PUr
+PxJLoj3D6x/ZsUiCcRrIMnbv2kHYOWbgVWPwG8G6FqXxN1vUk1i78MeEYx5GkySmC4ncQCVyz4DG
+P5gEbgEEkkgDPmfhj48+LvCWgHRrZ9Mv7I3b37HVrEXUj3Dv5jSsxYZbdyDjIPenWf7QHje18Uar
+rz6jbXtzq0KW99aXdqr2k8SAhF8rjGAW5BydxznNP6tmHNN+00e2r7rbTTS602uCx+TclJex1W/u
+r+VrV397Vp2e9vv1vj78J9H+HsHhPW/Dz3EWj+I7UzpY3kvmSWzhEfh+pUq/fOCOuDXzdq/ja8h1
+vUbOCTR7GOzlEP8AxNZjHLKdit5ijP3DuwD3xXrXjv4ia/8AErVodR167SeSCLybeC3jEUFunB2o
+gzjJAySSTgc8CuWmsrW6cPPbW0zgY3TQI5x9WBOK9KnSxEaEYzn7y3e/fS/3a9TxamKwP1upVhRv
+Tey0Vnpd21WuunS58jXH+vl/3j/Oo6kuP9fL/vH+dR196fdM0PDv/Ix6R/1+Q/8Aoxa+tp/+PiT/
+AHzXyT4d/wCRj0j/AK/If/Ri19bT/wDHxJ/vmvGx/wAUfmfIZ78dP0f6EdFFFeafMHWaN8L9f8Qe
+B77xbYxW82kWVytnIvmn7Q0rMigLGBzzIvf1qP4kfDnW/hRq8OmeIVtlu5bb7WgtJvNXy9xXk4HO
+VPFekfCT46aT8M/hFrWjeTLceJ5r9ruwR7cvbqSI1WRm6AqQzY9VHrXPftKfEbRvit4tstR8PvdS
+WsGmCzdryEwuZA7HOD1GCOfrXj0q2Lli3TnC1O71t0srfrr1+R9FXw2Xxy9VqVS9a0fdv1bd/wAL
+adPmXdO/Zb8darDp0kD6Grahbi7toZdS2yvEQp3BNmeN65xnGRWR4O+Afi3xtpj6jarpunWH2t7G
+3uNTvBCLudWKFIeCW+ZSAeMkHGcGvYdH/aT8E2HiPwJqEz6l5WiaFNpt0RYsSJmFuAF5+ZT5T8j0
+HrXL+GfjL4M1PwV4c0HxTJqelv4Y1watZT2Np563kaSyOisByjYkwc+nB5NeesTmXK24dvsvTWXS
++u0fvPXeByTnio1ej+2tXaFru2m8v/AbHnvhv4H+MfFHjPV/Ctvp8VprekoJLyO+n8uONSQFIcA7
+g2cgjqKh/wCFP+JD4Z8Ua4EtDb+GbmW11S1E5NzE8ZG4hNvzKQcg55APpXpet/tI6fdajq2vaXDe
+WWt6nrmnySQMnCaZanhS4OC7kuSg4w2MnGT6L8MNesfFPxW8a+MdGtzJ8NdXs1XWZ9R2wLb3kKA5
+MbHLKUPJAx8x9KdXG42jF1KkEkkvv05lfu7vl9Aw+V5ZiZqjRquTba32i+blla32bR5vJ9LHz/ef
+APxbYWWvXMv9m7dCsY9R1CIXZ8yGN42kVcbfv7UJx7jnmrMX7OHjSfxLpmgqmmHUdR099TgH2w7D
+ArIp3Nt4bMi8AHvzUlr+0Trun+JfHOqW2n6bq9h4qlYzWGrxM8flqCkQ4IJHl4UqeCK9SH7Svgpv
+ij4e8RM2oR6dY6BPpk6x2DKVmeSJlCpn7uEbntwO/GlWtmVPaCejeivryrTf+a/qY4fC5HW3quNp
+Jau2nM9Vp/Lb0Z5BpPwC8W6v4d0/XEbSbLTb6d4IJb+/EGWVnXJyvAYxtt9cj1q1r/7Nvjjw7daf
+aTQaXd6hfzLBb2NlfCSd85+cqVGI1Cks/QYrpPix8ddF+JPwig0DypYdXXWFu1tPs223gtUkfy0V
+uhIi2Z9ya1vFf7R3h5/jp4Y8a6RaXeoaXYaU+mXkc0Hkz7XdiTGG67QQcEgHkZ5oVfMnryL7Wlu3
+w6369/IbwmSRVnVb+DW6+1fmfLZ/Da9tdzzHxp8FvEfgbRH1i5l0vVdLhuPsd1c6Pei5FpPnHlyj
+AKnJAPoSM4yKzfh58NNc+KGpXljoS2vm2cKzzPez+TGqs21RuweSe1eqfFb44eH9f8Ea5ouiavrO
+rzazcmQx3Gnw2VtaQl1cqwVA0rjbgN1ORluOa3wq1vwj4J+EMt5rGu3un6hretwtJHp9utxMkdnI
+JEQpnKo5Byx9eK1WJxccLz1I++3ZaPy1a301+455YDL549U6VS9JRcpe8t1fRS1Wr5e9rvscLovw
+U8Wa94w1zw3b2ttFeaGcald3E/l2lsuMgmQjkEZIwM4BPABq7P8As++M4PGGl+HPIsJbnVbeS60+
++ju91ncxooZisgHUAg4x0IPQ17N4r+MHgrTtf16Rrq7l8O/EXREkuLmziV59PnWNoBviBz8yEEg8
+gr3B4xtO/aI8I6L4t+HdpaJqUnhbwlp81s2oTWuLi6laARLiIchQASTxycYwOeZYzMJrmhT+z2e6
+j3782lu33nc8tyik+SpW15rfEvhc9NLbez97m76eR5V41+B/ifwD4a/t/U5NKn0wXIszJp9757CU
+krtxtA4KkHngitj4Ufs2+IPi/wCG5tc07VrPTbWO6e1WO4jLM5UKS3HQZYjH+zWl8Qfi74b8S/By
+48Lac12dTk8Qz6oBLalEML3Esi/NnG7a68euRWd8J/2ktf8AhB4am0PTtKs9RtXunule4kKshcKC
+v0ypP/Aq6XLMKmFl7NJVObS6tp8zhjSyaljoqtJui4JuzbtLteOuh+c9x/r5f94/zpldp8Z/h9e/
+Cr4q+KfCl9E0U2mX8sSFhjfCTmKQezRlGHswri6/RadSNWEakXdNJo+snFwk4y3Re0GRIdf0uSRx
+HGl3CzOxwAA4ySa+uZhmZyGVlLEghgQfpXxxxj60AsP43HbhyK56+H9u072seNj8v+uuL57W8r/q
+j7E2+6/99Cjb7r/30K+O8t/z0k/77NGW/wCekn/fZrl+of3vw/4J5X9hf9Pfw/4J9ibevI/76FG3
+nOVz9RXx3lv+ekn/AH2aMt/z0k/77NH1D+9+H/BD+wv+nv4f8E+xNp9V/MUbT6r/AN9CvjvLf89J
+P++zRlv+ekn/AH2aPqH978P+CH9hf9Pfw/4J9ibT6j/voUEMUZPMYRty0Yl+Rj2yucH8a+O8t/z0
+k/77NGW/56Sf99mj6h/e/D/gj/sL/p7+H/BPsTafUf8AfQo2n1X8xXx3lv8AnpJ/32aMt/z0k/77
+NH1D+9+H/BF/YX/T38P+CfYm33H/AH0KNvuP++hXx3lv+ekn/fZoy3/PST/vs0fUP734f8EP7C/6
+e/h/wT7E2+4/76FJ5YDlhtDEYJyOR6V8eZb/AJ6Sf99mjLf89JP++zR9Q/vfgH9g/wDT38P+CfYY
+iCliAoLHJwRzS7eO3p94V8d5b/npJ/32aMt/z0k/77NH1D+9+Af2F/09/D/gn2Jt9x/30KNv+7+J
+Br47y3/PST/vs0Zb/npJ/wB9mj6h/e/D/gh/YP8A09/D/gn7H/tffsYaX+0np8Orabcw6J43so/K
+gvpVJhuowSRFNjnAJOHAJGSMEYA/LL4q/APxt8F9ZfTPFWlx2U4PytFdRTJIOzDaxOD7gH2oor85
+4XzXEqqsFJ3h0vuvR9j9ezXB0nB10rS/M4T7DP8A3P1FH2Gf+5+ooor9W52fI8iD7DP/AHP1FH2G
+f+5+oooo52HIg+wz/wBz9RR9hn/ufqKKKOdhyIPsM/8Ac/UUfYZ/7n6iiijnYciD7DP/AHP1FH2G
+f+5+oooo52HIg+wz/wBz9RR9hn/ufqKKKOdhyIPsM/8Ac/UUfYZ/7n6iiijnYciD7DP/AHP1FH2G
+f+5+oooo52HIg+wz/wBz9RR9hn/ufqKKKOdhyIPsM/8Ac/UUfYZ/7n6iiijnYciP/9k=
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="oziqunega.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <6e1e801d0b859d92688d70de258af3@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAfQBu
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++LvtEn/PV/8Avo0faJP+er/99Go6K/p4/J7kn2iT/nq//fRo+0Sf89X/AO+jUdFAXJPtEn/PV/8A
+vo0faJP+er/99Go6KAuSfaJP+er/APfRo+0Sf89X/wC+jUdFAXJPtEn/AD1f/vo0faJP+er/APfR
+qOigLkn2iT/nq/8A30aPtEn/AD1f/vo1HRQFyT7RJ/z1f/vo0faJP+er/wDfRqOigLnR/wDCufEX
+/Pgv/f5P8aP+Fc+Iv+fBf+/yf417T5QpTAeuG/KvLWKm9kfMvNKq0fL/AF8zxU/DrxCP+XBf+/yf
+40n/AArvxD/z4L/3+T/GvavLFHlCl9bn2Q/7Sr9l+P8AmeK/8K78Q/8APgv/AH+T/Gj/AIV34h/5
+8F/7/J/jXtXlD2o8oUfW59kH9pV+y/H/ADPFf+Fd+If+fBf+/wAn+NH/AArvxD/z4L/3+T/GvaxB
+noM0nk4/+vT+tTXQlZpVbt7t/wCvM8V/4V34h/58F/7/ACf40f8ACu/EP/Pgv/f5P8a9q8oUeUKX
+1qfZFf2lX7L8f8zxX/hXfiH/AJ8F/wC/yf40f8K78Q/8+C/9/k/xr2ryhR5Qp/Wp9g/tKv2X4/5n
+iv8AwrvxD/z4L/3+T/Gj/hXfiH/nwX/v8n+Ne1eUKPKFL63Psg/tKv2X4/5mVLqFxOeX8tT/AAR8
+D8fWoBJIG3eY4P8AvGnW8D3M8cMY+Z225PQeprePhu2KbVeQSY4kZs8+4r+YsHgM54h58TCo5cvW
+Umrve0d7fglpqf6S53xBwT4cuhltfDRp+0V+WnSi7Rvbnns2rp6+9KVnozHg1S4gI3Hz07q/X8DW
+1byR3cIlibcp9Rgg+h9651laNijjDKdpHXkVc0a4+z36rn93N8jcZGex/wA+texw3xLisNioYPGT
+cqcny+9q4t6LV62vo09t1bW/xnif4X5TmmU1c6ySjGliKUXO0ElCpBK8vdjpzWu4ySvJ+7K9047X
+lVk3mrEuyWuFQcecRyfoPT3rQ16c21kI1O15m2Z9B3/wrnhwAAPpXu8X8Q18LUWX4OXK7XlJb67J
+PdaatrXVJNa3+A8GfDbL82wz4izqkqsOZxpQlrF8ukpyW0tbxjF3ScZNp+7ZzTSSfelkY+7Gnx3d
+xEfkmYf7LfMD+BrS0nRo7q2FxOWKv9xVOOOmc1V1bThp0ybGLQyDK7uqkdq+HqZbnGDwcc2c2ouz
+upPmSezfk9Ot9dbH7zhuJ+C86zmrwhGhCdSHNG0qUfZtwT5ox84pPeKXuvlb0ve0+9W+yjDy5lGS
+AeGHqP8ACjVmaGyLIxR96/MvWsaGY208cynDIwP4dx+Vb/iBR/Zm4dDIhH0zxX3eAz2tmeRYtVZf
+vacXqtG007PTro727X6n8/8AEPh/geFuP8olg6a+qYqrG0H7yi4zipw1veNpRavfdrZIwvtVx/z8
+P+dbWmbpbJGdmdsnLMeetYHeum0OJm0yIgcZb+deLwTisRXzKcalSUlyPRtv7Ue591465TluB4Yo
+1cLh6dOXt4K8YRi7ezq6XSTtotPJDjH+FHl+9XPJYdv0pPLr9uba3P4QjFS+FmD4bjD6qCf4Y3P8
+q6kxiuW8NyiPWYVY4Eisg9yRkfyrsRGWIHr2r854IlF5W1Hfnlf7on9MeOtOceK4TqbOjC3opVL/
+AI3OF1ddmrXgHTzP6CqqsVljYcYdT+oqxqUon1O8kU5UynH4cf0pllAbq+toRyXkUfrn+lfjuK/f
+5lU9j9qo7fOTsf2llVsDwzhvrmip4eHPfpy0lzX9LO5q+LCReW6dB5bNj/gVYbHCk/jXQ+MYSJ7S
+bsVZM++Qf5VzxGQR68V6vFal/bOIv1a/9JjY+T8I5U3wTlvJsoyT9fazv+J29lCqWVsB08pf5U28
+0yDUFjE6swjOV2sRyeKm0pxcaZaSDo0Q5+nH9Kg1nVf7HSAiHzmlYgqWxgAdf5V+31a2Dp5ZGri7
+ex5Y3urq2ltLO+tj+DsLgc6r8U1MLlHMsb7Wqlyy5JKS5+f3m48uild3Wl0Q/wDCO6fggwEgjHLE
+8VD4ljCaPgDAEiAD0Haoj4vQAlrJzjn5ZBVrxQN2h7ipXMkbYPbJzXzlfFZTiMrxn9mKKag+bljy
+9Ha/uq/U/TMDlXGGXcV5K+KpVGpV48ntKqqK6lHmtac7bxvtfQ5Ojc3ZnUeisQKPWun0LSLO70qK
+Wa1jlkLNl2HJANfkuUZXWzbEPD0JqLUb3d7WTS6J9z+xONOK8Fwdl0Mwx9KVSEpqFoqLd3GUr+84
+q1ovrfXbc5lZZEOVlkB/3zV61169tQRlbodhMcY/4F/SrnibS7fThbSW8YhEhZCinjgZyKxBgdTt
+reu8w4fxsqEKzUo21i9HdXWj02fVHBl8OHPEXJKWPrYOM6NS+k4pTi4ycXrF3WqesZar5oFJVlZW
+KspyrDqD7Vry+K9QmtzHujRiMGVF+Y/TsD71s3vgaCRi1ncG2z/yzcb0H0PUfSqQ8CXmcG7twPXa
+1evDJeIctc6WGTSlu4SVn+KfzaTPi6/HPhxxPCjis0lBzp6xVWlJyjtdaRlF7K6TlFtLsc3kKPQA
+etdR4R0ZwTqEylcqVgU9cd2/w/Gr2n+C7W1cSXLm8cchWGIwfXHf8a6Aqc19Lw9wtVwleOMx1k4/
+DHfXu2tNOiV9db6WPzHxJ8WsLnGBqZLw+24VNKlVpxvHrGCdpe9tKUkvduknzXWRrelHVdPeFDtm
+B3xE/wB4dj9en5VwJBVmVlZHU7WRhyp9DXqhSs3VfDtrrB3yAxXGMefHwfoexr1OJeHXmrWJwzSq
+JWaeikumvRr7ujasfJeGHiVDhGM8szSLlhZvmUlq6cnvp1i7JtLVNXSd2ji9O1u70tGjhZTETu8u
+RdwB9R6VBe30+pXHnXD72xgBRgKPQVvS+BLoMfLvYWXtvRgaltfAZ3A3V7le6wLgn8T/AEr4P+xe
+Ia1OOCnGXs10clyr/wAm6f8ADH9B/wCvHhxgcTUzujVp/WJrWUaU/aS7r4E7vq21fqzE0PTW1XUU
+TB8mIiSZh0AHQfU/410vjBcaKx6fvk49Oa2rTT4dPt1gto1hiH8I7+5Pc1W1vSDq9gbYSiE71feV
+3dD6V99hsgll+T4jC0/eq1Iu/RXtZJXtovO27emy/njM/EOnxHxpl+bYlOlhMNUjyp3bUeZOU5KN
+/elZXUb2SSV2m35zWlZ+Ib3T7dIIWhESkkb49xyT65rY/wCEDf8A5/k/79n/ABo/4QN/+f5f+/Z/
+xr8/w2Q8QYOftMNTcZNWupxTt2+I/ovM/EHw6zqgsNmeIhWgmpKM6NWSuk1ezp72bXzZz+oapc6m
+6NcOrbBhVVdoH4VTlkWNQWIGT3rrl8B8/PfHH+xFz+prZ0zw7Z6VkwxebKww0sx3Ej+QHsK6qXCm
+b4+s6mNfLfeUmpN/JN3+bWh5OL8XODeHcDHD5JD2qj8NOnB0oq7u7uUUlu37sZNvda3PYfAvwdtt
+c8Lx+JfEmqX2kaTdSNFp1vpenve3l6V+/IsaqdsYOfmI5wemRnmvGtn4Rt77T7PwZPrGqAI0d3Pq
+cHlyTTlsKkcIUFcDOeOT9DX0P8VdT8d6Te/DHw58PTfWujS6dbNDcabDuiuHG0YlfBARU+YqSAQx
+PXFbqQ6Yfj/8T9Z0K0t9Q8TaR4fia2t41Vs3pV/MZR/fwIUPfkg9TX6RHH1E3Wm7pptJPRa8q5tO
+t73u/Tov42lgab/cwVmmk21q9OZ8uvltZep8kaz4V1rw2ITrGi6hpInH7pr21eISfQsACfbrWx4a
+8K6Nf6XLN4j1LUPC3nyxnTtTfTZLm0u48kTRKFGTL02EHaeQa9a07VfE95+zN8RdS+IF5f3sV08a
+aN/bKlZjddzErAMo34wOg2tjjNdP4q8A3+reN/gp4FtrC4m0jRLCLUb+ZY28mPaV3EvjbnKEAZyd
+4966Z49q8J2TTd2np7sVJ2uu7Stbuc0MCtJw1TSsmtdZW1t5a3v2PmfxLpaaVr97bQ2WpWFork20
+GrxeXd+SfuNIuAAWxnA4+tVF0m8bTm1AWVwdPV/Ka88pvJDn+Avjbu5HFdx8c9fPin4w+K7/AMwS
+RJeG0hYHjy4QE4/EMfxr1GW28HaH+zl4D8O+MH12JPEFy+pJDoaoZWkZ/l37v4f3iY98eldUsVKn
+SpNxu5W03e1362scscLGpUqpSSUb69N7L77nztc6Ve2cFrNcWNzBDdjNtJJC4W4HrGSPn6jpnqKu
+ax4R13w9BBNq2halpUNxgRSXto8SOcdAzDGfavqbxj4cuNW+N+jaNourSaJbeAfDS3PnJapdzru+
+UCOJgVaQoijOOO3JFY3jvVUl/Zq1m9uLzxVdQ+I9Yt4IpfGGxZ4lMilpYogB5Ue1WYDjpnAGK5I5
+lKbp2j8Vr73956eW2v8AVzqllsYKpeT929traLXz30/qx84aX4V1vXbOe80zRNR1Kzt8+bcWlpJL
+GmOuWUYyPTNVNN0u81u9js9NsrjULuQZS3tImlkIHU4UZ/Gvqr4ip8RdM+K2geFfhrb3NhomiaXF
+PaRR5i0+4HzbzM+NrfwjBOSTnvmua8P6tren/ALxn4s8OxL/AMJlqviB7fVb3RYdz28QcAmJVztX
+HzAjP+tLe9XHMJSgppL3rW12vtzdu/4EywEIzcG37t76b235f68z591bRr/Qb17PVdPutLu0Xc0F
+7E0Thf72Gwce/SrH/CK63/Yn9s/2JqX9jj/mIG0k8jHrvxjHv0r6UvdHfX/DPwR0b4hyPd+I77WW
+kZNQ/wCPk2eJGEUx64Y+Spz1JAOTmqHinxR8Wx8RPH+paUrW3hrRFktXs9XUJpq2oUYKI+FkcqC2
+QejYPUCiOYTn7sUr66t2TtLl006sHgIQ96TdtNEtVdc2uvRHzQE46UbPapo4gEG0bV6gEYx+FO8v
+6V7Vzx+U6zRfiT4w8N6R/ZWk+KNS0/TRkLbQyjbGCckISCV69iKxtJ1fUdA1SPUtM1C60/UYyxW8
+t5CJct94k/xZ75zmvRdN/Z98Q3dvpb6hq3h7w9d6oiPZadq2oGO6mDY24jCk5OQMfh1rkPGng+88
+CeJrzQdQmtri+tAnmvaOWjBZQwUEgHIBGeK8+nVw85ONO13vtr/melOnXhFSney212/yK3inxl4g
+8bSxP4h1q71gw5ES3UgKR5GDtUAKCfXGfetGL4q+N4bexgi8XatHDYrstkScARjbtHbng4G7OBXr
+vw68IQaR8B4vEy/D238d67qGqukFvLbmR0tw2zdkKSFHlsfqwrL+L/hnQ4PhhoWuTeELfwF40u77
+yV0O2kBe4gyy72j4x0UgkAgkDPzVxrE0JVFR9mrJuP2d/S97edjqeHrRg63O7tX+1t012v5XPDZg
+Z3keRvNeQlnZurljkk/Uk/nWnf8AifWtUk0yS91a6u30tVSwaVgfsqqVKhOOMbV/IV6Kv7N3ido5
+IBqegN4gitvtb+HUvw18I/XAGM/jjPfvXP8Agf4T6x440e41pbvTdA0CB/KfVtbuBBCZM/cXrkg8
+HsDxknOOz61h5Lm5k0v17evlucv1avF8vK7v9P8ALz2MKDxx4ktvEreIo9fvl19l2PqXmZldcAbW
+4wwwAMEdhSaz428SeJbKSz1fX7/VbOWb7S9vdy+Yhl6BsdjjjA45ruvDnwg1HQ/jj4Z8J+JLWCeK
+5nW4YxN5kF3bKrMSpOMglMEEA/nXK/EuXTbj4ieIzo1tb2ekR3rwWsNqmyMJHhCQPdlY/jUwqUZ1
+EoRT0TTstr2Vhzp1YU25NrWzV3vbUg/4WN4uPh0aD/wk+p/2KE8r7H5/y+XjGzdjdtxxtzjHHSqf
+hjxdr3gqeWbw/rF3ozyqFlFqwCyAdNynIOPXGRms7aaNpro9nTs48qs99FqYc87qXM7rzZPrOtan
+4i1R9S1XUrrUdSbbm7uJS0g2nK7SPu46jbjFafiD4h+K/FmnR6frfiTUdUsEIYW1xINjEdCwAG7H
++1n1rF20bPaq5IaPlWm2m3p2J5p66vXfXf17kOyjZU2z2o2e1aXJsfWn/CU+F7z4q+BtH1VNC8Sa
+3o1iZL/xhOViSB0UmNISDtLbmzgnjOeuQPmzx7rv/CR+M/EutKpYXd9PLGBzlAxVMeo2qtZn2WNo
+/L2rs7Lt4FO8sDgGvKw+Ejh5cyd9Lfjd+notD06+JliI8rVtb/hb+nufT+qaXqv/AArfwHong34i
+aL4ZOm2X+nmTU1jeWRkU7eA3AYuT0rnfH2seGdVtvhx4U8VeI7PxXq9rfr/bPiG0YBYrch/3TSr2
+ZmjU85wpY4PNfPxsIGyTEh+qinrCqLtUYHoKwhgFFpufVvZJ3d+u73N541yTSjvZbtqyt026H1Xp
+V5pPgKbxzNar4F0qG106b+xLbRCjX86YJV5pSRySFGwdwTzxnz6z0C0+KnwN+Hnh/T/EWkaWdDnZ
+dYt9UufKZSQwMwU8OfmZh2O/rwa8TFnFGABGgCncAFAwaV7WOZvnRXK9CygmiGB5PeU/eune3ZNf
+q3vuEsZze64e7Zq1+7T/AEXyPVPA1taaB8TNb1Hw94shv7Lw/p0wtNS12UqJ2eNlxAAfm24OABg+
+nIJ8it0JgQncWIyxf7xJ5Ofxq01ukmNwDbemR0pSnPWvQp0+RuTd7pL7r/5nDOfMlG1rXf3/APDJ
+Ffy/ajy/arHl+9Hl+9bGNiv5ftR5ftVjy/ejZz1oCxX2f5FSW1lc305hs7O5vpwpcw2sDSuFyBuI
+XnGSBn3FFw3lRsRydwQZ9zj/ACK+/Pg/8KNH+GHh2OGyH2nULpFkvNRkUCWdsdP9lRnhRwPckk+b
+j8dHAwUmrt7I9DBYJ4ybSdkt2f/Z
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="sgaj.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <44a9f01d0b859992dc36d08890cb74@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAjwBu
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+9v8A+GY/iR/0A7b/AMGEX+NJ/wAMxfEj/oCW3/gwi/xrvfjr+3Xovwz8RXnh3w1o48T6tYuYr25k
+n8m0tpB1j3AEuw/iAAA6ZzkDg/A//BSX7TqcUfizwlDBpjuFe+0a5MzQD+80TAbgOpw2cdieK9CX
+FjjPkbjf0f8Ame/h/CzOcVhFjKWHlytXS5oqTXlF669NLvoH/DMfxI/6Adt/4MIv8aP+GY/iR/0A
+7b/wYRf419p6Lrdj4j0ez1XS7mK+068iWe3uYTuSWNhkMD7ivkH4kf8ABQa48H+PvEGgaV4OtdWs
+tLvHskvn1Ex+cyHa52iMgDcGA5PSuirxLVoRUp8tn5N/kzxMp4GxWeVp4fAwlKUFdpuMba215ra3
+6b7mV/wzF8SP+gHbf+DCL/Gk/wCGYviR/wBAO2/8GEX+NfWfwj+IkHxW+G3h/wAWW0Atl1S1Ez24
+bf5MgJWRM4GdrKwzgdK+arb9vbU7j4sR+Dv+EJtVhfX/AOxftn9otuC/aPJ8zb5XXvjNKfE1Wmoy
+ko2lto/8zTA8BYzMatehh4NyoX505RVrNp72vs9rmF/wzH8SP+gHbf8Agwi/xpP+GY/iR/0BLb/w
+YRf419xqc9cV5f8AtHfGWf4E/Dr/AISe20mPWpPtkNr9lknMI+ckZ3BW6Y6YrepxBiKUHOSjZeT/
+AMzxcFw1HMMRTwmHu5zaSV0rt+bVj5s/4Zi+JP8A0BLb/wAGEX+NH/DMXxJ/6Alt/wCDCL/GvbP2
+cf2mj8bvDPinWtW0i28M2ugyKJZPtZlTy/LMjOzFV2hQP84rifhj+2jrfxl+KyeFvCvgeGXSnleU
+6pdXzK0NirYNxJGIzgsMBUz951GeCRzf60TtFq3vbaP/ADPof+Id45VMTSlTaeHSdRuUEo3V1rs3
+bom2cT/wzF8Sf+gJbf8Agwi/xoP7MfxIH/MFth9dQi/xr3b9oX9qrw78BBbWElpJrviW6j82HSrd
+wmyPOPMlc52KSCBwSSDgdTXztpn/AAUq1s6mBf8AgjTZrMN88VnqLidRnsWTBI9wPqKmrxXKjPkn
+y39H/mdWWeGWa5thljMJRk4PZuUY39E7ffse4fAr9mxvDV4mv+L4YZtWifNnp6uJIrbH/LRiOGf0
+7L7noz4z6J8YfH2sfZvD9g2iaBaufJMOpRxT3LYx5jkHIHJwn4nnGPVfhN8W/Dvxo8IQeIvDdy0t
+qzGKa3mXbNbSjBaORcnDDIPcEEEEg12eTXDicTPHv2s5Xvtba3keMsDLKpyws4OM4uzT3v5n5Kfs
+veD7L4gfHjwbo+sxLfWMk8l5cxz/ADLOYonlw+fvBnAJB685617v/wAFE/A2i+HtS8D65pmn2+n3
+l8Lmzufs0SoJVjCPGSBwSu5xnrhsdhXgPhO91z9mP45abPrOnP8A2r4cumW5tM7ftNuysjPGx4Ku
+jblPTOM967L9rL9o3T/2gdb8PJounXtlo+kpL5QvlVZ555toICIW4AQKOSSWPtn4aE6cMJUpy+O/
+z6flqf2VjMLmGL4pwOY4V3wipyu0/d1U+l9ea8LadPLT279kX4sP4L/ZQ8b6jeyboPCdzdGzVz8o
+DxJLHEB7yyED/fr5X8M/DG98S/Bjx18QJ2mkk0O+s4iwOVlMrZunb1I8yI+2TXqnxP0O/wDgb+yd
+4a8GaopsvEPjHV31nUrVjh4LeIKVjceoxb5z0O4dq+lf2dvg3HP+x9H4avoRFP4q0+4vJw4wVe5U
+mNv+Ap5X5V0xpSxDjQf2Y/i9v0Pla+aYbh+lic6oNWxWJik11hB++15Nqav/AHkzj/8AgnP45/tD
+wR4l8IzyZm0m9F7bqTyYJwcgewkR/wDvuvj3xDrj+FvjlrOuRQpcy6X4puL1IZCQsjR3bMFJHIBx
+jjoDXe/sX+Mpvh/+0Lo1pelrdNXWXQ7tG6iY8oMf9dYwv/AjXEanrUPhj4/6hrF3C9xa6Z4tmvJo
+EALyJHeM7KAeCSARzxXNOr7TDU1/K2vysfU4LLVgeIcyqwhzRrU4TS7t8ykr+bTfzPofTP8AgpVr
+v2pTdeBtNntwQXjtdRdZMd8bkIz9a7T9rH4naP8AGD9kWw8UaG0n2K71a0BimXbLBIrsHjcdmUjH
+oeCMgg14H+1T+0R4b+O3/CPHQ/Dtxoh0tpnnvL5YlkkV1UBBsY/KCu7k9cYrbvvCWp+GP2A0m1GG
+S3GseKYdTtYpBtPkMVRGwem7yyw9QwPeuh4ipJVaTnzx5XrY+a/1ey3Cyy3MoYT6rXdaEeTncrq7
+7vslJNW7M8b0j4l3+gfCzxB4KsQ1tDruoRXd/dI/MkEce0QYx0L/ADMc8gAdCa+0/wDgnbbeGj8M
+tbuLBD/wlP8AaBj1d5CC+wDNuFx0j2E4/wBrzK8B/Zd/Zz0/44+EviHe3Mx/tSythZaPGSQkN06e
+Ys7EdfuqgHQBmPXGML9k74qzfBv402S6oXstL1V/7G1aGXIED78RuwPQxy/KT/ddqxw0nQq06tT4
+XdLy1/DX8GezxPh8PneXZjluXNqvScZzS+0+VNLzTirL+9FHIfG/xBd+Ovjb4xvrmVjNda1LZRsT
+kxxJL5Eaj/dVR/k19sftefBzwp4b/ZeuRpej2djL4a+yyWM8MKpKP3qRuCwGTuV2LZPJ5PIr5T/a
+4+FWpfC74za3cmF4dI1y8fVNLvAv7suzb5I89N6SE/L/AHSp716B8fP2zrH4wfBW38J2ujXlhq96
+0LavNOUFvGI2DsIiGy+5lHUDAznmrpyhS9vGt8T/AOD/AMBnPjcPi80eRYrJ23h4NOVnZKKUN1fW
+yUo26O63ZP8A8E6fEN1ZfFfxJogkb7Df6QLp4wePNhlRVb67ZWH5V+hf1x+NfFH/AATx+FGoaeNc
++IOoW7W9rf2407TPMUgzRB98soz/AAllRQe+1j0xX2qx9Ote5lsZRw0ebrc/FPEXEYfEcRVnQadl
+FSa/mS1+7Z+aOS+IHwf8F/FS3ii8V+HbLWvJ4iknTEsY7hZFIZR7A1geCP2Zfhh8OtWi1TQfB9ja
+6lEcx3czPcSRn1QyM20+4wa+bf8Ahcnjf/obr784/wD4mj/hcnjf/obr784//ia+vfD0nLmbjfvb
+/gH5bT4qxdKg8LTrVFTf2VJqP3XsfU3j/wCBngP4palb3/ivw3a63d28PkRS3DOCiZLbRhgOpNdp
+Y2Fvpllb2lrEsFtbxrFFEnCoijCqPYAAV8S/8Lk8b/8AQ3X35x//ABNH/C5PG/8A0N19+cf/AMTV
+LIaibkpRu/67HLU4gnWpQo1JScIfCm7pX7K9l8j6Sb9mT4Xt4mPiH/hDrFda+2f2h9tRnVxcb9/m
+DDYB3c+lV9U/ZT+E2t6pe6lfeCNPub68me4uJnL7pJHYszH5upJJr51/4XJ43/6G6+/OP/4mj/hc
+fjf/AKG2+/OP/wCJrL/Vx/3fu/4B6EeMMwi044iqmlb45bdt9vI+jNJ/ZX+EuiX8V5aeAtHFxEwZ
+GmhMwUjkHa5Izn2rsvG/w98OfEfQf7F8SaVDq2liRZvss2Qodc7T8pHTJ/OvkzSfiZ8Rdf1K307T
+PEepX19cNtigi8vLHuc7eAOpJ4Ar6LstVl+Dvghr/wAZ+I7jXNVm52sV+aTHEUCgDgf3j9TgVxYr
+LaeAp++469F2+41pZ9jsxrRqyq1HKO0nJtp+TvdfI3fh98J/CPwstr2DwpodtokN5IslwlvuxIyg
+gE5J6A1zWv8A7LXwp8Uazf6tqngrTrvUb+Vp7m4YurSSNyzHDDknnjvzXhXiD47eLdT1K4vItbk0
+qFz+7s7QoY4lHQZIJJ9T+grj9U+Pnja3ViPGF6p7YMX/AMTXzrr4eS5HC6Xkj2oY3MaVWWIhiJqc
+t5KUk36u938z7a1/wJ4e8V+GxoGtaRa6to4RUFpexiVAFGFI3ZIIHRuvvXnOm/se/BzSdSS+g8C2
+LzIwdVuJZZowf+ubuVP0Ir49uP2ifiLLKVi8c6iq+oMP/wART4fj58Sn4PjvU8+pMP8A8RXLXx+F
+jJc8Lv0Wn3m+ExGZ4enKGGxM4RlulKST9Unqfo/DBHbQpFDGsUUahURFAVQBgAAdB7U4gn61+feg
+/Fz4o67qEFjZeL9WvLyc4jhTysn1JOzgDuTwK+hdCtPEtvokcOs+NNXu9RZhJLPbzJGqnH3F+TlR
+nqeSa5a3EWFw9lOLu/T/ADOKOWVaj3RX/sPTv+gdZ/8Afhf8KP7D07/oHWf/AH4X/Ctn+ybz/n1n
+/wC/bf4Uf2Tef8+s/wD37b/Cv1D2nmfE8nkY39h6d/0DrP8A78L/AIUf2Hp3/QOs/wDvwv8AhWz/
+AGTef8+s/wD37b/Ck/sq8/59J/8Av23+FHtH3Dk8jH/sTTv+gdZ/9+F/wqaz8L2mo3cdtbaTazTy
+HCoIF49zxwPete10DUL24SGKzlLucZdCFX3JxwK9Bs9PtfAmkl1ie8vpRgmNSWkb0Hoormr4pUY3
+b19Telh3UdrFLS/D2h/DPTmu/s1v/aUy7C8ESq8h67F44X/PpXC+IrxfEN6bzUoYLiRRiJXQOsa+
+i5/X1q3rE+p6ndvdXcM7yngARnag9AOwrmb9rlc5gm/FDXytau67vJn0FOkqSskVLyz03Bzp9n+M
+Cf4Vh3OjaXdkg6VYsPe2T/CrLSvcOwOVUHByMGrEEJbgCuGpNR0judMYt6vYzI/C+kDAGj6ePpap
+/hWppnguwv7pLe30WxeV+g+zJgD1PHArX0jRZ9Tulgt0DydSW4VB6n2r1nwn4Ys9NhkhXc0hALzd
+C57/AIe1ee7SkoRXvM3cuVXexieGPAuk+GoWNvY2i3cgxLcRwKpYf3RgdPat37HD/wA8Y/8AvgV0
+A0aD1k/Ol/seD1k/MVm8sryd5JErFRWibP5/f+Ev8Q/9DHrX/gyn/wDi6P8AhL/EP/Qx61/4Mp//
+AIuvr3/h1B8VP+hs8H/9/Lr/AOM15t8fP2FPiJ+zz4I/4S3WbzR9c0WOdILqTRmmZrTedqSSB0X5
+CxVcjoWXPWv6wpZrlleoqVOpFyeiX9I/JZ4TGQi5STsjwv8A4S/xD/0Metf+DKf/AOLo/wCEv8Q/
+9DHrX/gyn/8Ai6yqs6ZbW97qdlbXl/FpVpPOkUt/OjPHaozAGV1X5iqg5OOcA4r2HCCV+VfccCnU
+k7Jv7y/H418Sw7vL8T67FuGDs1Sdc/k9EnjTxLK+6TxNrkj4xufVJ2OPxevr6D/glH8RrmJJYvHP
+hKWGRQ6SItyVZTyCCF5GO9Sf8OnPiT/0O3hT/v3c/wDxFeB/beUf8/Y/d/wD1Fgcd0T+8+Oj4u8Q
+H/mYtY/8GM3/AMVTG8U664O7XtWbPrqE3/xVfY7f8EnfiUBx418KE/7lyP8A2SvjbxFoGo+E/EGq
+aFq9q9hq2mXMlnd2sn3o5UOGHuO4I6gg13YTGYHHNrDSUmt9DnrUcVh0nUuvmff/AOxd8S9R+Mfg
+g+HXDX/inw4iQzFnAe5tDkQzEk8lcGNj14U96+rNL+F3iCTaJ4be1UnkvMGI/Bf8a/IT4C/GXUvg
+F8WND8a6aJJo7OTyr+zjOPtdm+BNF6ZwAy56OiGv2f0z406X4g0ew1XRrd9Q03ULaO7tLlZAFljd
+cqeM+uCOxBHavxjinIoYHGe2pr93Uu12T6r9V62Wx9xlWYzxFBRfxR0Z0WjeDo9HtBBE6gdXfbln
+Pqa2LTTVtJC4kZmxjniuEf4l6lJnyrCGIdi5Zv8ACmab441e61W0SaaEW7zKroiD7pOPevjaeDow
+lzqOvzPWlVnJWbPib/gqVqvirwb8R/A2q6J4j1vRbDUtKuLV4tP1Ga3iaWGUNuKowG7bN1xnAHpX
+xR/wt3x8f+Z88U/+Du6/+OV+iv8AwVj8L/b/AIP+D/ECjLaVrvkMcdEnhcflujT9K/L+v6B4a9lX
+yym5RTautuz/AMrH57msp08TJRk0j+iXFZnibw7pvi7w/qOiazZR6hpOo272t3ayjKyxOpVlOOeQ
+T0rSUk9aXGa/EE3FqS0aPumro/Cn9pH4D6l+zp8V9S8JXhkuNNI+1aRqEnP2qzYkIxP99D8j/wC0
+uejDPmGM5B6Eciv2l/bR/Zsh/aN+E8trYxxp4w0Yte6HcuQuZcfPbsT/AASqNp5wGCN/DX4uSwzW
+s0sFxDLa3MLtFNBMpWSKRSVZGB5DKwII9RX7xkGarM8KnP8AiR0l+j+f53PgMywf1WrePwvY/Un/
+AIJnftFnx74Bm+Gut3fma/4WhU6e8rZe603IVBnuYSRGf9kxdeTX2yOe9fgN8KviZrHwc+Iug+NN
+CbOo6RcCXyC2FuYj8ssDf7LoSvsSD1ANfut8OPH2j/FLwLoni3QLj7TpGr2qXVu/8QB6owHRlYFW
+HYqR2r884oyv6nifrNNe5U/CXVfPdfPsfS5Xi/rFLkk/eidGRketfnL/AMFQv2c/s8tp8YNDtfkb
+y9P8RJGPolvdH6EiJj7x+hNfo3WT4r8MaX418NapoGtWiX2k6nbSWl1byfdkidSrD8j16jqK+eyv
+Hzy3FQxEdluu66r+utj0cTQjiaTpyP58O9fdf/BOn43tcxXXwm1OKG8uE8zUPD7XVwyALkNcWwx1
+xzKvt5lfKXx2+Dmp/AP4q654K1MyTLZSCWxvHGPtlm5JhlHbJA2t6OjDtXJeG/Eep+D/ABDpevaJ
+dGz1jTLlLuzuF/glQ5GR3B6EdCCRX7XmWDp51l8qdOXxK8Zdn0fp0fWzZ8Lha0sBik5LbRo/ciLS
+NWbkW+jWffAt2lYH6satpourDBfXpIRnO21t0jH06Vg/A74uaX8c/hdofjLS1EKX0WLq03Za0uU4
+mhP+6wOD3Ug967sjFfyjiauLoVZUazalFtNdmtGfrFP2c4qcFozxf/goD4ZXxT+yT44IQyT6fDBq
+cWB0MMyOx/74D1+Mxr98/iN4cHjv4R+JtDMe86ro11ZhG5+Z4WUfqRX4D2wYW8YcYcKAw9D3r+h+
+Cq7ng6lN9Gn96/4B+d57TtUjI/Xr/gn7+0+3xu+HDeGfEF35vjjwzGkVxJK3z6hafdiuueS3Gx/9
+oAnG8Cvq8HIr8CvhH8VNb+CfxF0Xxp4ff/iYabLl7dmxHdwNgSwP/suvGexCsOVFfuZ8LviTonxe
+8BaL4v8AD1x9o0nVYBNFu4eNujxuOzowZWHYqa+Z4lyj6hiPb0l+7n+D6r9V8+x62WYz6zS5ZfFH
++rnUtX5j/wDBTD9mX/hFfEH/AAtzw7aY0nVZkh8QQwr8tvdHCx3WOyycI57OFPO8mv04IzWT4t8K
+aV438M6p4e1uyj1DSNTt3tLq1lGVkjcEMPY88HqDgjkV4eVZjPLMVHER1WzXddf+B5nfisPHFUnT
+kfz4190f8Exf2i/+ET8V3Pwo1y62aVrcj3eiPK3yw3gXMsAz0EqjeB/fRsZL18vftCfA7Vf2efir
+qng3UjJcWsf+k6XqDj/j9smJ8uTP94YKOOzqexGfPrK9utMvrW+sLmWyv7WZLi1u4DiSGVGDI6ns
+VYA59q/cMXhqOcYJwTvGaun59H/n9x8HQqVMBiLvpuf0PAk5yMUEZrx39lH4+2v7RXwc0rxL+7h1
+uD/QdZtE/wCWF4gG/A/uuCJF/wBlwOoNex1/P9ejPD1ZUaqtKLsz9EhONSKnHZnyT/wUV/ZyPxf+
+FQ8V6JZ+d4u8JpJcxpEuXvLI8zwe5AAkQeqkD75r8jlcSorqwZXXKtngg9K/omOa/Gj9uv8AZ2Hw
+C+Mk9xpdv5Xg/wATNJqGmBFwlvLnM9sPTazb1H9xwP4TX6Vwjml08vqvzj+q/VfM+YznCXX1iC9T
+f/4J7ftBD4VfFJvB+sXPleF/FsqQhpGwlpqH3YZPYSD92x9fLJ6HP6rEYODwR1Br+fxlLDGSD1DK
+cEH1B7YPf1FfsJ+xX+0H/wAL7+DtvJqdx5vi3QSmnawGPzSkL+6ucf8ATVBk/wC2rjtXzHiDknI1
+m1FaOyn69Jfo/O3c7+H8dzL6rN6rb/I+mNMG3T7cA8bBX4I/F/wwfBPxd8c+H8bRpuu3tuo/2BMx
+X/x0iv3v09cafbjpiMfyr8av+CgfhgeGP2tPGYjQrDqcdpqacYyZIFVz/wB9xtXqcE1eXEVKXeKf
+3Nf5mOeQ5qKn2f5nzwDivrz/AIJ2ftN/8Kg+IJ8CeILvyvB/ii4VYJZWwlhqLYVG9kl+VG9GCHgb
+ifUP+HRU3/RVcf8AcBH/AMkU2T/gkPI8bI/xVyjDBH9hY/8Abivq8dnWSY/Dyw9Wto/7stOz26Hk
+4bA47DVFUjH8V/mfo0DmgjNfHvxb/aU8cfsT+E/BujeK/Dx+KFpNbmzTxjHefYGmnjziOeEpJtkM
+YU7t/wA+1zgEEV5h/wAPeE/6JRL/AOD5f/jFfm9HIMfiYe1w0OeD2aa1+9pr0aufT1Mfh6L5asrP
+tZn0P+27+zNH+0T8KZP7LgT/AITXQd97o0rYBmOP3lqx/uygADPAcIegOfxmZXjZ45Y3hlRikkUq
+lXRgSGVgeQQQQR6g1+iJ/wCCu6k/8kolz/2Hl/8AjFfGXx/+JuhfGP4naj4y0Tws/hBtVAl1Cw+2
+C5SW65DzqQibS42lhg/MC2csa/ReG8PmWBjLC4unaG6d4u3daN6Pf7+581mlTDYi1WjL3vnr+B6B
++w/+0Qf2f/jLbnU7jyvB3iIx6fq+44S3bd+4uj/1zZiGP9x2PO0Y/Z5GDDIORX87UoQxv5m3y9p3
+bumMc59sV+237FEnjGX9mbwQ3jf/AJCpsx9mL5882Wf9GM2f+WnlbM+2M/NmvG4wwFOPJjYu0n7r
+Xfs/ls/kduS4iUoui9lse4kZryP9qX4DWf7RPwe1bwtIY4NWUfbNIvZB/wAe14gJjY/7LZKN/su3
+fFeuU1+3evzijWnh6sa1J2lF3R9LOCqRcZbM/nj1HTrzRtSvNO1K0ksNSsp3tbq0mGHhmRiro3uG
+BH4e9epfsvfHaf8AZ6+LuneJJGkbQLkfYNbt058yzc8uB3aM4dfow/ir0n/gpAvg9f2mtQ/4Rgk6
+sbKE+IfLx5H23Hybcf8ALTyvL39vu/xbq+XCQoJJwPU1/QMFTzjL0q8Pdqx1Xqv6afoz86lzYHFf
+u38LP6GtIuoL/SrO5tZ47m2mhSSKeE5SRCoKsp9CDkfWvzH/AOCsnho2HxZ8EeIAgjj1DRpbN5Om
+54Jtw5+k9fT3/BPTV/Edr+zj4Z0zxZcxi4dZLjQraQt9pGk7wITJntnds/6Z+XXn3/BWL4e3fjH4
+T+C9QsY991Ya20Hy9dksDs36wrX5Pkijl2dqhzJpOUb/ACf62Psscvb4PnkrXsz7Y0HW9P8AE2i2
+Gr6VdxX+mX8CXNrdQNlJonUMrqfQgg1eKg1+cv8AwS//AGjriO5m+DmtySTQ+XLf+H5iC3lKPmnt
+SeyjJkUnjlxn7or9Ggc5rxczy+eW4qWHnstn3XR/11OzDV44mkqkTh/jX8I9D+OXw11rwbr6H7Hq
+EWI7hFzJazLzHPH/ALSNgj15B4JFfhp8RPh/rfwq8c614Q8R24t9Z0m4MEwUHZKOqSx56o6kOp9D
+6g1/QGQGGDXxb/wUo/Zut/H/AMPH+JejxRxeJPCts73vIX7bpy5aRCT1aL5pF56b15LLj6LhfNvq
+Vf6rVfuTf3S6ffs/kebmmD+sU+ePxR/I/K2jPf8AlSAhgCDnjOabNKIYmkILBRnA6n2r9nPhErux
+77+xZ+z0f2hPjRZWeoW5l8JaHs1LW2IOyVA37q2z/wBNXU5/2Ekr9qUUIoUAADgADFeHfscfs+xf
+s8/BbTdHuUibxJqJGo61cR4O65dR+7DDqsahYxzg7S38Rr3PGK/B+IMz/tLGPkfuQ0j+r+f5WP0T
+L8KsLRSe71YhOK8k/aj+PFn+zv8AB/V/FUvlzaoR9j0mzkP/AB83kgIjU/7IwXbp8qNXrZOCK/G7
+9vf4/T/Gz433um2jyR+GPCUs2l2MLgr5twrbbm4K9iWXYv8Asxg8bjWeQ5Z/aeMUJL3I6y9O3z/K
+5WPxSwtFy6vY+dNS1O91vVL3UtSu5L/Ur6eS6u7uY5knmdizux9SxJrsPg78P4/iH41gs74Oui2o
++06gyHkxA8Rr7u3yj2ye1cMz7VJxn/Gvvn4M/Cm3+GHgezsJ0R9ZucXmozKAf3zAYRT/AHUXAGOp
+ye9fp/EmarKsFam7TnpHy7v5dPNo+YyfCfW8T7Sorxjq/Py+Z7FqPx90qLwHHaWOn3SX09skF1Cm
+14LbymGGRT13qA3y/cwBX1D4l8KaV8T/AA3Yxagoms3aO8j+XgkoQDz7Oa+Ohbx7SoXjvg4/lX2B
+8JNRbUvhp4cuGyW+yJEf+AZT/wBlr8HoS5JXjoz9BxCU4Wa0/wAz/9k=
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="vvwujjimpiwifa.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <08ee401d0b859092973e90a6de4e3a@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAdwBu
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++PvCvinVvA/ifSvEegXjWGtaVcpd2dyv8EingEd1IJVlPVWYd6/cX9nX446T+0L8KdJ8X6YBBNMP
+I1Cw3bmsrtQPNiPsCcqe6sp71+E2f/1V9C/sSftKN+zt8V411S4KeCfEDJaawpyVtmziK7A/2M4b
+HWNj1KrX7hxHlP8AaOG9pSX7yG3muq/y8/U+EyrGfV6ns5v3Zfgz9nwc1Fd20N7bS29xEk9vKhjk
+ikUMrqRgqwPBBHUGnwypNGskbrJG4DK6nIIPQg04jNfh+qPuz8Uv2yf2bJv2bfixNZWUUn/CG6yX
+vNDnbJEaA5ktST/FESAOpKMh67sdX/wT6/Z5/wCF0fGNNf1a187wn4RkivLhZFyl1eZ3W8BB4IGP
+NYc/dQHh6/Sz9pr4Dad+0T8JdV8KXZjt9Sx9q0nUJB/x53qA+VJxztOSrAdUZh1xT/2Z/gdZfs+f
+B3Q/CFuY5r6KP7Tql3F0ur2TBmkBwDjICrnoiKO1fos+J5Tyn2Tf79+635fzettPXU+dWVxWM9p9
+jf5nqa0E4paq6nqFrpGn3N9fXEdpZWsTzzzzOFSONRuZmJ4AABJPtX50k3oj6HY+ev26P2j/APhn
+/wCDsw0q5EXjHxAX0/Rwp+aE7f3tz9IlIIP99ox3r8aETaAoJb/aY5LH1J7knv8AWvWv2o/j1dft
+GfGLVfFO6RNDi/0HRLaTI8qzQkq5HZpCTI3puVf4RXnXhbwzqfjbxPpPh7Rrc3er6pcpaWsPYu5x
+knsoGST2UE1+75Fl0cpwV6ukn70n28vkvxufBZhiXjK/JDVLRHq/7LnwsHjbxe3iDUYPN0PQnVwj
+rlbi76xx+4X77D2Ud6+zXcsxZyWZiSWPc+tdD4K/Zb0/4e6DYeGYNfuJILKaGCWaGBVM8sqlpZeS
+eSRx7Y9K6mz+BehnUIYJ77UZ1a9mtmPmKhKpHuB4HBJr8Rz3PIZri5V0/cS91f3d7+r3/DofeZfg
+/qdBU0vee/r/AMA8zMqKCTIgA754r6b/AGcNT+2/DnyGcMbS8miAU5wCQ4/9Dri9I+B/hIx2Es1p
+dXDTRXTuJLp8ZjlCJ0x2Ndp8D9JsPDz6vp1jbi3R1guWUMW3MVKk8n2WvKpte0ce363/AMjrnK8b
+n5DftRfAW5/Zz+MOq+Fdsj6HL/puiXL5JlsnJCqT3eMgxt67Qf4hXkxAIIIBBHIIyDX1h+1j+2b4
+S/aj8E2OmyfD/UtC8QaXc/aNN1d7+GVYg2BNE6hclHUDgY+ZEPbB+UK/pbLZ4meFh9cjy1Fo9tbd
+dL7/AJ3Py3GRpRrN0XeL1P0//wCCaf7Th8aeFT8K/EV20mvaDBv0eed/mu9PXA8vJ6vDkL6lChx8
+rGvuUNmv59PBfjHV/h54v0fxRoF2bHWtJuVurWccgMOCrD+JGUlGB6qxFff0H/BXW0WGMS/Cu7aX
+aN5j1qPbuxzjMXTPSvz7PeG8RUxTr4GF4y1aulZ/NrR7/wBI+kwGaUvYqNeVmvxP0NIzQBivz2/4
+e7WH/RKb/wD8Hcf/AMao/wCHu1h/0Sm//wDB3H/8ar5z/VrNf+fP4x/zPR/tPB/8/F+J+hJOBXwZ
+/wAFPv2i/wDhH/DNt8JtDutupa3ELrW5Im5hscnZDkdDMynI/uIw6ODWQ3/BXWxI4+Fd8D2zrcf/
+AMZr4D8feONX+JnjfXPFuvT/AGnWNYumurhxnaucBI1yThEQKijPAUV9HkXDmJpYtV8dDlUdUrp3
+fTZvbf1t5nnY/M6TouFCV2zC9+mK++/+Ce/wGOi+G774oazbhb7VIfsuhRyD5orQyBZrjB6GUjap
+/uqx6NXyl+zh8Ebn4+/FKx8NgyQaLApvtZu4/vQ2aEbgD/fckRr7tntX7ADT7fT7SOzsraOzsra2
+t4ILeEYSKNWCoijsAAB+FHG+d/V6ccsov3p2cvKN1p/29+XqLIMBzyeKmtFt62eoXUWdWkbHW9tz
++SGi3i/4mkR/6f7hv/HAKtyR5v2PJzdxHJ9kNJEuL6I/9PMzfoK/CV8P/bv/ALaz777Xz/UsWEfl
+2unjptt7zn6ziuetPEEXg2V9QI2STu1sxfgAKkTAf+PGuus4j5Fo5U+WsUwLdhmcf4GuK+L3w88Q
++LdOaLSbUmUarJcZMiLmMxIoPLDuP0r3Y/x5ei/OR5kv4a+f6H4f/b7b/n4i/wC+x/jR9vtv+fiL
+/vsV+8x+HfhP/oVdD/8ABZB/8RR/wrvwp/0K2hf+CyD/AOIr9H/4iVQ/6BX/AOBr/wCRPmv9WZf8
+/D8GRfW3a4iz/wBdB/jR9tt2PE8R/wCBiv3l/wCFeeFP+hV0L/wWQf8AxFct418KeFbUW9pB4Z0J
+ZD88gGmQZA7D7n1/Krh4j0Zuywr/APAl/wDIifDUlvU/A/ET7Vb/APPaP/voUfbLf/nvF/32K/Yy
+Twl4fwf+Kc0TOP8AoF2//wARWtH4M8PSCNP+Eb0MfJk/8Sq3/wDiK9Glx3Tq3thnp/eX/wAiZT4e
+5LfvPw/4J+Ln222H/LxEP+BilS8hkdVjkWWRiFSOM7mck8KAOST0A9cV+5XhTwL4XutO1cSeFtDa
+WLy9rf2XAAM5/wBitv8A4QjwtZzRXNv4Y0RHLJLG6adCrRMhHIITrkcfnWFfxAo0JOLwz2v8S8/7
+vey+ZVPhxzV/afgeO/safs2R/BH4S29vq0Qj8W67KlxrZGC0RKHy7bd6RKcHHBZnPpj3aLSYJ4le
+TcWkR92D12cqatyXItLtSMPE8n2nPfnoPyJoUm2uxbtzEuUDY5O/qf5V+MY3HVMbiJ4mvK8m9fm4
+tW8klp2SsfbUKUaFNU6askv0f+ZXOmwGEzmPMhhE5Of+WgOAfypbuwt4bWZ1jUOBFIGHZm+8R9cV
+ZgR2na1kDbCDAMDkL2Ofrz+NEEM148kMqOFlwAcYCleVrjj71o/9u/OzX3arU2bS1+f4mlqcSw2A
+jjQKplX5VHH3sn9a+dvj5pUM2keW0swDa9dyj963UpHnoentX0hcWrSWYjJMjgg7j3IPWvObv4fR
++L/EWp2HiPR2utBErXlrM77Cs3CEAg5wV5Of7tfQQi/aSl3t+p5cneCR2G2jbir5htl+/eRL9WH+
+NQXBsRBLm/ib5G4DqT0NfPLL8S/s/iju9vHuYuleINI16WSPTNX0/UpIwDIlndJMyAnAJCk4H1rz
+PxLqtpZ395d3l5b2kDzlRNdTLGhOcAAscZ46exr53/4JyR20XjrxwHAjH9nQgbB/02f0+grtv25t
+Ptk+Fs8sCfuDrlrswMLghweD+P8Ak1WHo1FRliEtLP8AA++x/D1LCcS08g9s2pOC5rK/vJPa/n3P
+Q4rqC9t1ntp4rmBwSk0EgdG7ZDDIP4Vq3V3a2L2z3t/a6ZbOuzz7u4SEFhg7QWIBOOcDPFeffs4a
+fbv8CvBbNGSXtHJx0z50leKftKXGq/GH40aH8OdGt0ll0yIxJbyN8rXTp5shf0KoiL7V60Zyw1BV
+WruVrLzZ52A4ejmed1srlV5KdF1HObW0aba5rX7266Xv0PvXwde6TdeHYbuxjbUIbxAz3NtGZI58
+cZDDgjryK2xcRBURdLuGVTgAxAAZ69TXyL+wD8ULa98Daz4M1TUJbObRZ/tdmhfafs0zfMoGP4Jd
+2f8AroK1f21vj8/gbwhYeHfCOsXKazrW9rm9gnbzba1TG7Yf4XdiFDDkAPjnBrb6xRlh/rM49P6X
+3mNThXGR4gfD1KV581k3ouW3NzNa/Z1aV+x9C6z8U/CHh++Nlqet6Lpt4p2/Z73UreJx7EF8j8a3
+rHWW1S1ivLG3t7q1lG6O4iukeNx6hlyCPpXwT4E/ZT+FVz4Rt7nxp41vT4ovoRM8OlYEdq7DdsO6
+NjKwz8xJ5Oab+zudd+BvxhvdLF2Nb+G9xI0Mty2EhlGMxXMULHKyA/Kyr1BPXArOniKqlH2lKyfb
+W3qeji+H8klQrvAZjzVaKbanFQjO2/I29X2Wt3tpqffdxqNzBC8sq2dtFGCzyTXPCr6k7a5WL4w+
+F7m++yR+NPCZuM4MK6pG7k+mNw5r4Y/aO+LF38f/AIrw+B9Auhpvg61uVgAEZVZ5QMy3MyDBcJyq
+qfTPBbizqH7O3wdfQWtdP1fxYuuhSsd/cQW5tmk55aEAEJnsDnHc03i6tSUlQhdLS7drvyNI8MZX
+l+HoVM8xkqVStFSUYw5uWL2lJ3/BK/TU/QKTVZUI8zUbOMYz8kROc9MfNTReyyttTVSxHXyLQt/j
+Xwb+yx+0br/wj12/+HviSK41jToTJHZwCXL2c8f3o0Lc+U4BIX+EjgDcaxP2rv2nfE/jnxp/YeiX
+eo+HtH0ghXg026aOae4K/O7umDtUEKF6Z3E9RhvMIKh7Vxd72t1uFLgHHV86eU06seXk9oqn2XB6
+KVr9Xpa/fVrU/Q1reGI5a50yDHH+oX+rVTu9RtLWGVn17T4fkbGFjXt/vV8pT+EIzO63N1brclkC
+Ri5Zg2fXA/3fzNOtvBlm84SOaJrsu42hZGTocdvXP6V2OqujPzxYdo+bv2ffg/afGHWtds7zxkfB
+a6fCs4udobz90jLt++nAwD1PXpWt8ZPgJZfC7wqdVt/iAviqb7ZHbfY41O0h92ZAd5zjb6d66b4E
+fB3xD4e1bWp9dslsYprQeQ5HnhirFmGF6HHrVn4q+D7/AMVeExZaNZia6SeKZozF5PyruydzH3Xi
+vn6WEi8K5OHv69/lof0XmHGOIp8U06FDFx+puULtcjjZpc152utb3fNoesfAXxfpvhj9nDQtVupC
+8Gj6XPd3KhTtGySQhc9Mk4XA/vV8ofDj4rW/hLxrrXivWPtMus6gk3ltbFco8z7pCSxBxjCjHv0r
+3S4+H3jjVf2WPD3grRNMjlv5LyRtUSS7jiEcSSF1QEkBtzFc4/u1N4K+DbeHNAi0OfSbW/1W2t/P
+1GYxJII5WG5l8wggKBhRjJODiuyVOvWlSjFWUIp3a0vZaeqPm6WZZNlmHzPEYlqtLF1px5YTSl7J
+Sk+a6u1GTdtveXLbRnmPw4+Mvhzwh+1LB4x8NRzaf4Q1K9EF1a3e0NBDcBVm3BSRtWb94Oegr0f/
+AIKP2ctp8S/BuoOhFo2lSRq2OCyThmA99rCvXP2sv2XJPib4J8Pz+BdB0231+xnKywW6xWa3FvIm
+H3NgAlWVCM9t2OtZXxQ+GOsfEb4L+CNA8fR/2P4x02zlVbuO6jlWOaMBEkYg/vFlULuAIIJPpXJO
+hWjCpQfW0k+l+q8j2aXEWUSxmW59B8vKpUakHJSqKNmoTfWSWrlK2t7asr6Z4Z03URp0lhKW86Hz
+RNHYSGNgV3KQfTbx9aydG13wfc+OoPCC6zLaeJo7gQm2ewKEuQHCqScEqOoOMV5l4e+Hn7Q3h7R4
+tH8PzRXukwAx27wanbSLbKTn5DI4aNc5IyOO1en/ALPf7MWl+C9buvFHxM8UadN4ikDiCwttREjQ
+O+fMlllzl5TnHHABPJzx1xrV6koxjBru3t8v0PicTw/k2X4fEV8Tj4VFZ+yjSkpTlLpzqz5Vbdeu
+uiT8J+F2nLpn7Tl7o+pLcW8815e2UYAVZBK5yg+Y4GRz1+lfT8XhtJESBdOv8Nc4NxJLCpJ+7jjt
+3xXK/tFfs1+FPHF5Dr3gPxRp+neIUCrcW19cssV3txskWUAmOUYAzyDxnGMnyWfTvjzfaZ/wjkv2
+eK1diH1JZLdJGyu0k3K/Nyo6gZP1rCn7XC81NwbTd1bXc9vMqWA4uWHzCljaVGcacYVI1ZcrTjfW
+P8yd3+Gt20snS4YNb/a4uRpgN3Yw6tO27zwPMjiiKu3mDjkjr+VcV8Qza+EPjL4vjuv31tHfTxoR
+IWzuZGU7h14719dfs4/BzwH8GtNudQ8S38PiPxNeRiJ/stnK9tZxA5MURKguSQCznGcAAAA5zv2h
+vgp4F+L2tWeteG9XPhXVljEF35mmSSQXMaj5G2rgiQdM9x16CspYSvKjz297mvb1/rufQYTjDJsN
+naoSm/q0aEaKqWerg782ibs9VtvZ7anZSTQMZFS7M9uZUD3BvgCvHsvpn8qrl1AhScgWZkkPmjUJ
+S7dB2X12/nXy5deOPEcupvLJr+qabcm/jC6MdetkVgBnBAXj5gFx/tGobTxxqpu7R4vFE15eGS5Z
+rKbxSQkYAOPur/dLH/gIrrdddj8YWFa3Z9U2+wwx/aY90RRioSW5kYsTwDx61x1zYyHPm2ZJ2AoP
+s0xAGe4+lfPln4x1ZY3Nh4ijvU+wyfaGn8VXTbEJG5hheoBXBHPNePar4r1eK81byfEN0wyom363
+fSBWL8BTxkbuM9cV00KqldHNXw7ikz9OfA9rcR+GLaKSIidpXG2OJlyc8YU85xj8qyvF2uauZl0n
+T447GKWVEllKFmcFgCTjpxXGfsfa/qNv8I7a+s/B3iTXUa3lNv4gur1Ps80bSPnyPMk8z5cEFnUH
+I44rwXxdq+sP40s8anfLuvrdQiXL7SvnqvTOCCP0Neqpcy0PClG03c/UJry1hVVNxEi4wBvA4FeR
+/GO4hu9Z00xKl9GlvKp8uz+1bHONvI4X6HrXeaX8MPC2iWn2Wy0W3hg3tJs+ZvmY5Y8k9TXyD+2R
+aJ4N+I+nx+Ho7XTDfaLdT3v/ABP5NO3jeqtKUU/OVHRu2cd65MTJRgn5nTg4c9W3kfQPwc8MRXF5
+q1xqFp5xkhgEfn2QhCgF+AO/PP5Vw/xU8J/2V4yvZLeArb3DebHsXoc4YD8Rn8axv2CrpbxfGbLq
+aakwj08zBdYk1DyZSJ9yZf7gwF6defStT9pXRF1Xwjr1689xbXttfRfZ7m3mZHjUy4ZRg9COo9hW
+SrKMYX6uw69JqrNdjzPWVvk1BkWS6RASCFdgB9ahsFvX1S0jaS4dTMik7mIPzD9K+Pdbg1EeK9dk
+PiPWh5d42yP7fIAQOwwelb3w1ku9V+IXhU3Wt6lEp1ux/dfbZCsg+0JkPk8g9MfWuq8dTynJ3PrP
+x3YXP/CV6ikazeUhGNpOMe1YFtDdW7lkjlyRgjmvHf2k7O6H7Rvjmy06XWSv9oZ8u21CWOKJSoJC
+qCAoznivLdd065s7iJBrWtB3Uuy/2pNxzjj5qPaJA5XbR6rLYXlrevG99qb51LL60o01MALjrk/7
+/rUFrM7xWPmXuo6XGqXLC+a+09DcNk4XhfXCf8CrZW0ad96eHZYrJdQf/iUPo9oHPyjnl/TC/wDA
+TWeljqloloWs7zUIza3Gy3i07TlFseRvPzeuHr55yaPv1FPdopPIsliTc6xd6cq2TGN0121Xz23c
+KdqdD83/AHwK43XPDkGo+KNSudWvobaALbeTZHxEFe6yi5JZU4G3DH1zj3r0W3utW0+wmc2Wpago
+ssBCNMjFtlh83Gcc8f8AAqW+vb4m7jGpXU6Frcm/+26cpg74ztPrt/4BVRqShexm6UZtJ20Pu/8A
+Z0Zk/Zk0MvDFbMtjd5ht5vOjTEsvCuQCwGOuK+JNUjSb4gabldytqVpnI/6brX298Cw0X7L+i77y
+W/b+ybhvtcsiSPNlpDvLJ8pJ68cc18backUnxR8OrcqGSTVbLcoOcgzjpXvYb4H8vyPka38SXqz9
+Mm7V8V/tsWk938SdNNvPeW/k+H53f7LpCXnnJ5o3R72+6Tn7vfBNfaTNgYr4Y/bivNNT4uadHqI0
+4MugM1ubzV57VjIZjgeXGMMuN3zdc8VyY5v2at3/AEZ35bZ19ex3f7Bz+ZZ+MsW+oQCL7BCzX+nR
+2fnOElJkUJ94HcOT6Cut/aGt1Hw511yOHvrfB/7b1y37BzCXSfF8hSwVt1giGwu5bgGEQExljJ0b
+DHpXafHDxXrXhXwlf33h/UDpWox6hCn2jyI5wY2chl2uCPxrnt7tFef6jxNvbVbf1ofmXrGnXd94
+j1iaCCeQfbJRlImPf6VufC/Rby4+JnguMafdIBrthvY2sgVQLhOclfbrXc3/AO1v8bbbXb2Kb4iu
+LOKeRBDBo1kjBQSB83lHJ6Vr/C79o74qeJ/ix4FsLz4mapcWd3r9pBd2MlpapHNE0qhoyViBwRkc
+EV6ErM8J2luUf2kU1uf47eOf7M0fVblhqLDzLewlkRxtUcELg9xXnmn+FPHN6+T4Y8QwhUwrjRLh
+gQO3CV6J+0L8dPip4c+NXjbS9H+Jev6dpVlqcsUFpB5GyFM5CKTGTgZwMkmsLwl8Xvi1rVm1xc/F
+TxTLuJwiXUaEfkgqHGL1M5tJ2M21k0q919YhLpr+IFvZmkvRogC5w2Rgy+oc/wDAqdpN7pF7GP7M
+itrWSDTp5LuRdEg/fKOSADJxlGC/hRRXhx95an6PUXItB+1Z9O1J7C4ezhgtYxcRpo9p+8y4X+//
+AHiD+Fdd8L/hnrPxf1O9PhDT3t7TzojMk1rYRRnaOSeSeuTwP4zRRXFUqyVSlTWnPKz+57dPwZVk
+qVWo9XCN1963Puv4b/Dvxh4X+HVt4R1C70O3sbe1azils4ZHkWNs9QSqhhu7DHFczafsY+FBfxXt
+7ret3V3EVKukscQBUkggKnYmiivq44eMF8Uvva/KyPiXiHOT91L5J/ndnX6P8ANP0HSrKws/FPie
+SKzTZFJe6k1xIRkn5ncFmOT1J9q838W/sYPqs+oXemfEHV4bq7SQf8TWCO+WJ3cMWjLjcgHICg4A
+J4ooonhoVVyyb/8AApf5lwxVSj70Er/4Y/5G18EPA2u/s66Zr0fizVH8R2V7cQvBqNuiKIEWMJsM
+fDYyCeN3Ws/48XsGp/C2e+tZRcWs+p24WUKQG+dj0YA/pRRXiOpOGYQwjd4pX1338rafK/merVpx
+qYF4y1pOXK7bbb+v4eR+dl1Cg8X3gR9zrcSPskXKA7iTn1roPgsTc/tCfDmR9qk+JLM7Y1wv+s9K
+KK9mOjsfKJ3dzY+Pphb45/EXzEMs0+sTxRjoBgjJNeaafBcSXUsUNw8MUYwCpxzmiipi9CXuz//Z
+
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="haecyeiizizimo.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <d182101d0b8599926de4a0f6ee2d19@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgBJwBu
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+55762iS2im1grp3n3LLqI8TKJJOO+I/UKP8AgRrc8E+DfEPxMvrLTtDsLm8uPsP7o2XiKRl5c7Wl
+YRgBc7wSey13/wAHPhl4l+MmuRJa3V5p2mxNKbyWU2s0FspbAC7E5duSAD1UE8V94eAvh9ovw50R
+NM0W1ECcGWdwDLO+PvuwHJ/QdhXzNONTEycaWiW8v0Xd9+ie92mj6+vVpYSKdTWT2X6vsvxfSyaZ
+4h8Lv2LNA0Ka11Xxbd3mvakkSKLBr6VrOMgdSMgyHp97jjpX0XYaba6VapbWVtFaWyDCwwIEVfoB
+VkDFLmvco4anR1Su+71f3/otOyPmq2Jq19JPTstF93679xMUoFRXNwlrA0shwijJr5S/aD/bn0r4
+PawulWFi/iDUc/vYLW4VBB6BmOefYdK6jlPrL8KQ18lfC79r28+JFg1zcf2b4eIGRDd6gpY/yrX8
+FftiWOqePv8AhGNVjhsmditvei9jmhmbsvy/dJ96APp0L71w3jf4M+G/G9hNBPbPp8srrIZ7B/Ky
+6klWZR8rYJzyPx5rtLO8S+t1ljPBHQ9qmIzWNWjCsrTV7fh5p7p+hrTqzpX5Ha+/n6rqj8yvjl+y
+N4w+EtzqOs2TjxD4dfe7XlvDiaAMxP71BnHX7w4rzT9nfTnvPjv8OmU5EOvWrkd8Bic/pX6/SRLL
+GyOoZGG0qwyCPQ18x/ET9l7S/C3xW8MfE3wxE1laaVqCXWraZbRFwYRndJEg54JyVGeM4HavOn7T
+Ce9J80O/Vevdee6631a0VCGKaVNcs+3R+nZ+Wz6W0T+Mvjldwv8AtCeNlK7W/tq5RWxkE7uPxJrz
+kM1raMHAiczvvDcHI6V758RPCngLVfiZ4j14fHPwPp6z6nNM1vPDK7xs5ztcg4yOOlchqXgn4cPK
+BcftB+BICM8R2lwQff73610Ri7JxPNnG0mmj9Q/AfgXSfh14Ys9C0W2W3s7YHO0AGRz952x1JP8A
+QdBXQgYoUEZzS11UqcKMFTpqyWx21Kk6s3Um7t6sK5rxt480nwDpz3+r3kNnbLgeZM2Bk9B9TXS9
+K+Uv27bh18FQx7iEN/b5GevDVqZmj40/bc+G9pAtpP4ktbUyNgSvHJs46jO2vy2+KHirTL/x3q00
+F4k8VxdyTJLGSwdWYkHPuKufFvEd7p6DjdGzEnr1rym9YNqC5wBtAzt4702rAdLcazYHzYpJkDJw
+wYGqlhq9nDcxsJlUKwKgZHOcjpz1FU/Cug2et6pNHfT3MFtEo3CyRGmYliBjedtUtb0mHSNeazjn
+aeABJY5JE2vhl3AMB/EO+KQH78/B27kvfAGkTSq6O9rCxSQYZSYxwfeu1rhPgjI03w20OR9297K3
+Zi4w2fKXqK7ugApCM0tFAHw/+2z+zq2nafdeM/C9oiWszh9TsoUHDHjeuB0Pp6/XA+FxC7xgpGrp
+/DkZ4+h/zzX7e6rplrrOnXNjewrcWdzG0UsT9HUjBFflF+0Z8MW+C3xE1DTWEwsJZDLazKgIZGyR
++P8AUNXiKP1SsqK+Cfw+TWrj6NXce1pLblR14iCxdF1/txtzea2UvVOyl3vF78zP1oooor2zkCvk
+39vAZ8HW59NQt+PwavrLrXzL+2n4U1bxL4Pji0jTrnU5xfQMYbVN7hQGy2PQf1prcD8wPjCxW+0r
+HBMLAn/gVeY3TlWxuK/Op4GecV9BfEr4HfETW9Q00WXgnWp8I6FxanC85+Y9q8V8ceCPEPgbXTpm
+vaXNpN4ux3imxuCkcHj15pvcCLwAn2jUNSgE8EM0sIEKXDLGJCHyVDMQAcc5JFZ/ikrJ4qdY5kkM
+ccUTNCwkG4IAwBHBweMjikbTY/3+ACkjErv54rT8B/DnUvHHi3StG0e3a9vLy5SNLVGCllzluSQP
+ug/lUlXufup8DIzF8MdAQmRitjbjMxy5/dL973rvq5/wFpg0jwvY2wXYEiRAvoAoGP0roKCQoooo
+AQjNeCftWfBM/FPStGurS2E9/aTGNgByYyrEfkc/99GvfKQ8Vw43CRx1CVCUnG/VaNNa3T7nbg8S
+8JXjWUVK26eqaelmuwtFFFdxxBUNxZw3X+tjV/rU1FAGPqek2Vvp9xItsgITqBX5Aftma3f+KfjJ
+emfRre0g0zdYou8t9oTPyyMRjBBPAHFfsL4juktNGuXc7QUwM1+PH7TkS6l8U/EEXQTXDxZ9jJt4
+ppXA8X0rw+GSaC80fU9RlCjZPaIQqn3UD/PNenfsz3uq/DH4oaNr83hK9vLaab7FHLcI0SRl2CM6
+sRgsM4/OvQte+Cms6f8AbH8MXkL209pGttFNMY5YpxgHL9GXGffmtW88G3tifhq/iHVrjUtS0vWL
+WKFFlb7KimQFkC9CQSTkjn8KqwH6maDL5ulwHrtG3P04rQrO8PlG0i32EEY7fWtGoAKKKKACkNLS
+Hnpn8KAFooooAKKKKAOB+Nqas3gS/wD7EdE1MxMLcyfd39s+nfmvyw+IHgrxuPErX+qeDb7UHhuE
+lUeSdtyA4IBZScqx9K/Wzx6AfDU+a+ePEcgafSVJIxa27ZHbk1S1Efn94v8Aid8VdZ8Q6jc2ehal
+oEUk3/IOtbNmitiONiZHA9iTVHRdQ+MniDV7JoNH1jWLqxf7clt9j4xGcsx6YA4ya+32tgbC/mXO
++S7mbg+rGl+FCk+Lrvdk/wDEmvUOfQquafKM+hv2c9Z8Ra/8L9Ev/FWnjStduLUSXliP+WEmTlep
+7AH8a9Rrn/Akap4YtNoAyOcd66CoAKKKKACmvnjAz+OKdUNzdQ2iB55UhQnAaRgoz6c0ntqGvQmo
+pAc0tMAooooA5/x5/wAi1cV86a9GTd6Z/wBetvX0J8RJNvh11BI3NjPpXzddeI4byazV7aeFgkMC
+71A3FeC30Jq4iMeNGGlzAdTNKf8Ax41L8NIxD4su+MZ0i65+oWkspkubB0UNuWWQHAyT857U/wAI
+Xp0jxaWmtpkiubKezR3AGJGAKgj6A1Qz6s8D/wDIs2n0rdrD8E8eGrUd8GtysgCiignFACE469K+
+W/2zfi9a+FxougDUWsnLm7nkiieQg7SqLhORwXP/AHz6ivorxl4tsPBHhq/1rUpBHa2kZcgnBc/w
+qPcnA/8ArV+aHxH8ea9418QT63caN4ojku5pW+06PqFqkVwN2FIDEnCKAo56Yr5zNmsSvqK2lrK2
+6j0V+8n+CkfTZNB0JvHNL3dI32cvnuor8XE/Qf4KfGTSfjJ4Th1KzZYNQjVVvLInDQv64znaecfl
+2r0POa/LXwx4k1/4OeN2vNHujFcQEN/0zuoT0LL3BxgjsR9DX3t8F/2gtA+LtgkUbrp2vouZ9NlY
+Bie7Rn+Jf1H6nbBY7SNOu9/hl0l2T7S8uu66xXn1sPTrw+tYPWDV2u3murj+MdpdG/VKDSZ5oBz0
+6V7p5J5l8XfiLomgaQ0F9f21o+d/7+QZCg43FeuPwr4mH7T2mP8AEy20iXSpNc0y/vbe0065MitJ
+FukCqDkdATnjnFfUPx5/ZR034wa5a67Lf39nqFmmyJrR1CsOcblI56mvkLxT+wr8SX8ZNJo76fBa
+q6SR3oufLYODkSbcEhgQOlUnYR6R4P8AEOna3Z+KdR0/RLy5k0K41KG4twoGZ4ZWQq7KcgsRlRjO
+3Jqt+z/401PxN4Kj8Ta94c0/w3eT3smnQSyXBitWjOCJ0EpG5gMrxnqTVHwl/wAE9fH2na0dfj+I
+8+k6zO/nTXemxSGSRickuWYByec5HOa7jxF/wT91T4hXcF14z+Imt+IZoE8uFXhiijiHosY+VfwF
+PmGfUfwz8XadrOhW8VrPHKRwPLkD/wAjXb18+/s7fsnad8ANV1S+0/VNQv31GOOKRL11KIqMSCoH
+QnNfQOcde9QwFJxUN1dQ2ltLcTypDBEheSSRgqooGSSTwAPWqXiHxHpnhXSptT1e9hsLCFd0k07b
+VA/qfYV+fn7S/wC1lrHxSvD4c8KW93YeE1kxcXijbLdgc54OVXOCMfzHHmYvGKh+7p+9N9O3nJ9F
++L6Js9HCYOWI9+ekFu/0S6v8F1sWv2oP2ndF+IPiNtCGoWSeFLSOYGG9ilZbqQoRHIdnbcMcngZ4
+5Ir5yur3TpNG0kJ/wgwtB5/2QCSeIEeZ+8OP98Vct/H95ps7xz3GpXlusWAGXzCjAnPDjPNTad8V
+o9au/Li3zW8UXySy20Ploc/MgUqcHJyfXmvHp0nFOUneT1b7v/Lol0SSPpPrCXLCKtFaJdl+F31b
+6s9W8WaHNr9rC1ncw2eo27HypriIyRMjfeR1BBI7gg8EVhWvhbxRp08V1aanokd3E25JY47mJlI7
+ghiQa6rz8UGf3P518RTxtalT9kmuXs0n+Z+J4bN8Zg4qNCpZLVbaenU9l+FX7T3i/wAOwJZeOLex
+1+3XAS+0x2W4UcfeDgBz+RPfNfR3g74ueFPGyKNM1iA3JAzaXB8qdT6bGwT+Ga+DPPz3xSMyPjco
+JHIJ7H1Fehhs7xeGdrqUez/R7/ffysepTz9zf+1U0/OPuv5r4X8lF92fpJRgE9Bn3Ffn7onxL8V+
+G0WPTfEeo20SnIiMxkT/AL5fIrrrb9pz4g2yhW1KyuAO81iuf/HSK+gp8TUWv3lOSflZr8WvyPRj
+mmAnr7Rx/wAUX/7bzH2qTzRn3r4wk/ak8fuPlutNjPqtln+bVh6r8evH2sZE3iSe3U9Vso0hH5gZ
+/WrlxLhkvdpyf/gP+f6DlmWBjr7W/pGV/wDyZRX4n25q+u6boFsbnU7620+AAnzLmVYxx9TXzr8R
+v27/AAF4eaax8N39pruqKSm+a4WC3VvcsQzd+g7da+d77UJ9TuDPfXM97P1826kaVvzYnFV5Vt5y
+DLbW82Onmwq2PzFeVV4irVXZR5Y+T1+9q34X8znWf4WlK8KLn/idv/JVf/0p+hzvxL+Keo/GK+E3
+iLxPb3EakmKytL9IoYh/sqrfqcmuTi8OwyMjW17LtVdoSKdXUjtngnt616FNomj3Egkl0bTJZB/G
+9jET+e3NUbvwN4XvpvOuPDmlyy9N/wBmVT+gFYwzahBWVNr53u+7b1b83qdj4sjN3nSfya08krJJ
+eRx//CHz75JEnufnbefkBwfbjge1Vk8H3kJK/bmMW5mCC0VMEkn+EjJ56966/wD4Vn4PEvmr4dtU
+f1jkmQfkHApr/Dbw60hdIb+3z/Bb6pcov5b63WcYfrF/cv8AM0jxRhutOX4f5m752e4o873FeK/8
+NNaB/wBAPVf+/kX+NH/DTegf9APVf+/kX+NdH+pHEn/QFL74f/JH5feR7T53uKPO9xXi3/DTeg/9
+APVf+/kX+NH/AA03oP8A0A9V/wC/kX+NH+pHEn/QFL74f/JCvI9p873FHne4rxb/AIab0H/oB6r/
+AN/Iv8aP+GmtA/6Aeq/9/Iv8aP8AUjiT/oCl98P/AJILyPaPOx3o8+vF/wDhpvQP+gHqv/fyL/Gj
+/hpvQP8AoB6r/wB/Iv8AGj/UjiT/AKApffD/AOSD3j2jz6PPrxf/AIab0D/oB6r/AN/Iv8aP+Gmt
+A/6Aeq/9/Iv8aP8AUjiT/oCl98P/AJIPePaPP98Unn/7Qrxj/hprQP8AoB6r/wB/Iv8AGj/hpvQP
++gHqv/fyL/Gj/UjiT/oCl98P/kg949n8/wD2qPPx3rxc/tNaAf8AmB6r/wB/Iv8AGgftNaAP+YHq
+v/fyL/Gj/UjiT/oCl98P/khe8fO/50fnXqf/AAhGjf8APs//AH9b/Gj/AIQjRv8An2f/AL+t/jX9
+Df8AERso/wCfdT7o/wDyZ/RP/ED+JP8An9Q/8Cn/APKzyz86Pzr1P/hCNG/59n/7+t/jR/whGjf8
++z/9/W/xo/4iNlH/AD7qfdH/AOTD/iB/En/P6h/4FP8A+Vnln50fnXqf/CEaN/z7P/39b/Gj/hCN
+G/59n/7+t/jR/wARGyj/AJ91Puj/APJh/wAQP4k/5/UP/Ap//Kzyz86Pzr1P/hCNG/59n/7+t/jR
+/wAIRo3/AD7P/wB/W/xo/wCIjZR/z7qfdH/5MP8AiB/En/P6h/4FP/5WeWfnR+dep/8ACEaN/wA+
+z/8Af1v8aP8AhCNG/wCfZ/8Av63+NH/ERso/591Puj/8mH/ED+JP+f1D/wACn/8AKzyz86Pzr1P/
+AIQjRv8An2f/AL+t/jR/whGjf8+z/wDf1v8AGj/iI2Uf8+6n3R/+TD/iB/En/P6h/wCBT/8AlZ5Z
++dH516n/AMIRo3/Ps/8A39b/ABo/4QjRv+fZ/wDv63+NH/ERso/591Puj/8AJh/xA/iT/n9Q/wDA
+p/8Ays3M0Zr9RG/ZM+CPiOxuLaz8JaWfKJiebT7l1lib3dHyGHvXxL+1D+znJ+z/AOJrE2V5JqPh
+jVt5sZrjHnQumN8MhGA3BDB8Akbu4yf5/r4Grh4c8rNeR/UORcdZXn2K+pU1KnVeymkr2V3azett
+bO2h4rmj9fpX11+yt+xpYfEHw5beM/HYnk0q8+fTdGikaLz4s8TTMCG2t1VVI+XBJ5wPoLUv2RPg
+n4tsL3T7Hw9Y2dxbN5Elxo92yXFq/oxDHDd8OD7irpZdWqwU1ZX2uceZ+ImT5ZjJYOSnNwdpOKTS
+e1rtq9no7K3mz8w6WvQfjt8GtS+BfxAuPDt9Mb20kjF1p+obdv2mAnGSOzqQVYewI4Irz0150ouE
+nGSs0fouExVHHUIYnDy5oTSafdP+vVdQzRmvtL9jb9nb4efFb4RT634r8OR6vqa6rc2wne4mjPlq
+E2rhHUcZPavdf+GLPgsSR/whMIP/AF+3P/x2vTpZbVqwVSLVn6/5H5rmXiPlOVYyrga9Ko5U207K
+Nrrteaf4H5cZ/Giv0N+I/wDwT88Da3pFw/g+a78L6woLQCS4e4tHbHCujlmAJx8ynI64PSvz+1jS
+L3w9rOoaTqdu1nqNhcPa3NuxyY5EYqwz3GRwe4wa5MRhamGaVRb9j6jIOKMt4khOWBk+aO8ZK0lf
+Z7tNeab87aFSiiiuU+tCiiigD9Iv2V/2TL79nrXta1a+8SRam+oWqWi2VlA0MICtu8x9zHc/YegL
+cnPHiH7fHxX0n4g+KdA+H+js802k3pOpXDxsixzS4iWJcgZIVmYsMjlQCTkDvf2Qf2gvin8UfiHq
+Oj+LLY6josNo8kt6dMNobKYMoRCQAGLZb5Tz8uRwDWn/AMFD/DOmL8NNE8YfZ411/SNVt47edQBJ
+JExJMRPddyhgOxBx1NfR1OSWAf1dWit7+Wr6n8y4GeLwnG1J57JVcRJJRlBrlTceWLa5VfTS2ltJ
+arf6Ze3h8J+Emis4wltpliUhjxgBY48KPyUV8Bf8E7PEN2/xq16Oad5P7Y0aS7udzf6yZZ0YO3qf
+3rjP+0a++LHU7Xx34HgvrGQSWes6eJYZAcgpLHkH8mr4Y/4J7eAtY0r4xeJ7rUdNurRdE019Mnaa
+IoFuGmT5ORydsTHjtj1rpxKbxFBx21/r7jwuG5U6fD+dxxOk7Qvfe95f+3W+Z6N/wUg8OxT/AA/8
+Ja9tH2ix1Y2m/HPlzRMSM/70SGvgjvX3n/wUh8TwQeCfCHh0OpubzU2vygPIjhjZcn23Sr+vpXxr
+8Lvh1qXxa8eaZ4S0i6tbPUb5ZXjnvC3lKI42dt20E9FI4HXFeRmC5sU4w3dvvP2Pw/rfVeF6dfFS
+5YRc5XfSKbbfpfmPvD/gnlz8A7n/ALDt3/KOsD9r39nX4hfGD4laNqvhLyF0+20tbSVpdSa2xL50
+jfdA5G1l5HP5V61+y38G9X+Bnwzm8Oa3eWV9ePqM94JdPLmMK4QAfMoOflNdR48k+JUeqW3/AAhd
+v4Vm00w/vzr1xcxzCXd0Tyo2G3bjk857V7aoKeEjSqp9Ntz8Nq53PDcU4nMssnB80pWc/hafzXyL
+/wAL/Dmr+D/h34e0XXdUbWtXsLKOC6vyWYzOBgnLcn0yeTjJ5Nfl5+0lrlh4j+P3jvUNMdJbJ9Q8
+pJYzlZGjjSN2B6EF0bkda7r4m/tufEjx/pl5osJsPCtlIXt7htIDm4lAJVlErHKA46qAcd6+fFUI
+oVRhQMADtXi43FwrRjSp7Lqz9n4I4Tx2S4nEZlmUoqpVTXLHZXkpNu2m6VktEh1FFFeSfr4UUUUA
+fpRfft7fCO0tDLDf6rfSYz9ng0qYOfbLhVH518fftJftM6n+0DqVlbpYtovhfTnMtrYO4eWWUgr5
+spHy5CkgKM7QzcnNeL+vH6UZrvr46vXjyTenkfn+S8DZNkWIWLw8ZSqLZzd7eiSSv5tPyPpv9l/9
+sZvg7oyeE/Fdndan4YidmsruzAe4sQxy0ZQkb49xJGDlckfMMAfQXiH/AIKAfC/TNLebTG1bXL0r
+mOzhsXhy3YM8m1VHvz9DX5xUn+cdqqlj69KHInoYZlwBkWaYx46tCUZSd5KMrKT6tqzd31s1fffU
+7H4s/FXW/jN43u/E+usiTyKIbazhJMVpApJWJSevJJJ7kk4HSuX0/UrzRryO80+8udPvIs+Xc2kr
+RSpkYOHUgjIJHB71XorgcpSlzN6n3tHDUMNQjhqMFGEVZLol2/rfqfYv7Mv7Yfhr4ZfDiXSPHGpe
+INV1ptQmuFnaKS8xC23YPMZs9jx2zXn/AO1F+0onxQ8a6RqPgTXvEOk6Zbaebe4iWaWy3TeYzbtq
+tg8Ec+1fPXQ8cD0xSV1SxdaVJUW9F958ph+EMqw+ZzzaEX7SV7ptOOu9o2+7sLnJJzkk5Oe57mlp
+BxRmuM+0FopM0ZoAWikzRmgDtP8AhCLD/ntcf99D/Cj/AIQiw/57XH5j/Cvb2/Zy8Zi40aMLppg1
+OxfURem6K29rbqqsXncr8nDrgDOeewJqHwh8A9e8a2MN7a6z4esrW7na309r/UPLfUGVmH7lApbD
+FSV3AEgZxiv6IeU5Ao83soW+f+fk/uP8p14g+IrkoLMK1/l5eXmvvPFv+EIsP+etx+Y/wo/4Qiw/
+57XH5j/CvZ9E+APinVtV1Wwup9G0KTTbtbGaTVtQWJXnYKUSMAFm3B0wcAHcAOcgNtf2f/GU/jDW
+PDs0FjYy6PAt1qGo3d0FsreBgSshkx0IVjjGRsbOMU/7IyBNr2UNFf5aefmiV4h+IjSf9oVtXbpv
+r5eT+5njX/CE2H/Pa4/Mf4Uf8ITYf89rj8x/hXpPj/4a618N7qxTVPsl1ZahCbiw1PTp/PtbuPjJ
+R8DsVOMdGBGRXK1008gyarFThh4tP1/zOOr4n8d0JunVzSqpLo2v8jA/4Qmw/wCe1x+Y/wAKP+EJ
+sP8AntcfmP8ACt+ir/1cyj/oGj+P+Zl/xFTjj/oa1fvX+Rgf8ITYf89rj8x/hR/whNh/z2uPzH+F
+b9FH+rmUf9A0fx/zD/iKnHH/AENav3r/ACMD/hCbD/ntcfmP8KP+EJsP+e1x+Y/wrfoo/wBXMo/6
+Bo/j/mH/ABFTjj/oa1fvX+Rgf8ITYf8APa4/Mf4Uf8ITYf8APa4/Mf4Vv0Uf6uZR/wBA0fx/zD/i
+KnHH/Q1q/ev8j6rtf2sfDENh4X0C4s7q98Mro76drsM1kTIX8qNE8sbvnU4kVh6MD7VjfD342/Dj
+4d+F9I07S31rTbjT9QMl1cWelRNPq0AZtvmyPkqCChYAhvkKrwa+V/EPi3SvCi27arcNbJcEiNhE
+zglevTpwak1nxNpvh/7CL+4MRvZBHbqiNIZGOMdOnUcn1rR5dg4xlDmaWl1dedm/vt9x85HOM1nK
+nVVPmbTUXyvZWuo69OW7te2t9HY+jV+Jvw61D4x6z46nvNf0vVGu4JtOvY7COeJoRFGskUkDAsrk
+q6hwf4gy4IrodK/ag8L2HxB8aXMGnajpmheI4Yv+JjFCk1zHdIjIZjA24FSpXC8jMfKncQPka28d
+aFda6dFivGOp+a0HkmFlBcZyu7GOxpB480E69/YovS2pGf7MIhC23zPTd0olg8FONpTurKK95dLa
+euidnf01FDMM3pSvCjZ3dR2g9U7pyflZtXVvXQ90+OnxW0/4hxaDp2lXuq6naaYrPJf6pHHB50pU
+L+7hRVCLjPp1AA4yfKqzNP8AEum6rq17plrc+bfWWftEWxh5eDjqRg8+lae07d2Dj1xXrYanTpUl
+TpbK/wCev4nzuOq161d1cUrSdt1bSytv0ts+u4lFKAWOAMn0FBGK6ThEooooAKKKKACiiigDznxu
+W8c/DvWLtrdYpdNv5TFDz5ipE2x94PRmUk4HGMVieDL+Tx34z8JrMGeLQrDfMXHDSKeD+PyflXr8
+kEUsU0bRpsnVllAGN4Iwc+pxxk81ieFfBGkeC3uH0qKZHnVVdp5TIcKcgD0Gf6V49TB1J14Tvdac
+3nZ3R9dh84oUsDWoKDUrv2fVR51yyV/S7Wmrep41fpLp2u6t4gg/12ma8oIzxhmYgfiVx+NS+HrS
+QeN/Cd5IWS41W4e8Ld13yOFI9xXrT/DzRJLTWbZo7lodXlE91/pBzvByChx8vJNSN4E0aTU9I1Dy
+pxcaTEkVoEmIVVXONwxzyTz3riWW1eZSut09+vNr07fie7LiXCulKnyu7i43stvZ2S3/AJ236Wfk
+eb+BdLls/iD4iJ1O6kbSBJcScjdqIRj8kvseDxWbZeIvFOv6bd67Beawb+O4TZNbSRrYQqRnY6sR
+zzx7DnrXq7fD7RD4oj8QLDNHqSTC4+SYiJpMY3FMHOfTvVC7+EHhe8vXne1uYkkbe9rDcskLHOfu
+emc8D1pvAV1Dlg7JOXV9dns9V/wSI5/gJ1XVrRbbjBawi7cvxRtzK6lve67NWSOa1fxDrPivxV4a
+8PtezaBHeW0Ul0bMgOZHVicHOCBt4AOOTWxoWv6joek20F1rNh4nd9SWxEls7PMiMrcN0yUK7jno
+ob0Fb/iDwFofiaG1jurQwm0QRQPZv5TRoOiDHBXPQevNXdB8MaX4YsFs9PtVjiDFy0v7x3YjBYse
+c44+nFdkMNXjVcnLfrd+Wltu+p4tfMsBUwkKUKdrbx5YpXvK8ufWWqaVvK12M8K60/iLw9Z6lJAL
+eScNuVDlGIYjeh6lGxlSexrVoVVRQqqqIoACqMAAcAADtS4xXqxTjFKTuz5WrKE6kpU48qbdlvZd
+F52EopcGkqzMKKKKAPo/xN8IvhDo2tat4Jl13UND8T6fYJc/25ql0FtJJWGQhU/KTypKgDg8EkGn
+eF/2Y9N8Y+E/hvqdot5brqcf2rxBdC4LqsQTO2JT0Z3wFIBwCTzgVB4s+K/we8Q6vqnjK48Palrn
+im/sktxoepW4+xxSqAA5boOAAWBPHQAk0/QP2nrTwh4N+GulaVJdO+kvHHr8DWuFkgCEMInPXazb
+gBjO0dK+Of8AaPso+y5+frzbX5Xe3le1r6Xt5n6Sv7G+sT9v7Pk6cm9udct/Plve3vWvfoYPw7+G
+/gj4jw/E7UbSw1O20vQLfzNJia7cynbHLlpR1JZowdv8P3ab8SPgXY/Dj9n/AEjxLfR3H/CYyzQJ
+ex+eTFH5m5jF5fQMq7AfcH1rqvCPx3+H/gPxr8QdZ0tb+ez8QLDdWtnJp7oFulEhkRvRGYqwP+23
+oMyfDnx34c+Lvhfwt4G8S3t5J4gk1mXW9RlKBIXZHln2+acjbyi4644473OrjKdT2rUlTTi2ne9u
+XVffv3djOlh8trUfq6cHWkppNWtzc/utebTXKukUzP8AFn7PfhnR/hPq93YyXsvjnQ9LtNR1GNpy
+Yv3g3OBH0HyrJj0wKTwZ+zRp/j74c/D3VtO+1QX+rz+ZrN4ZyYobZfM3bEPCuxVFXHQnPSu28NfG
+H4X618Ttcnj1PX57/wAXiLSJ4Ly0CWiqAY1wcZUYLcknqa5zUPjzpnwi0Dw74L8M3N7PceGdYNtq
+KzQAi4skkk3KsvQk7kOQOSO3fmVbMZL2UObnunqns4u69FLbodksNk0JfWKvJ7JJxtFp6qacX/ic
+N+u5y/gL4ffDj4n+MvGPhfTrXU9IvbeOU6FPc3zkzNGCshkRu4fDbP7pPTBrmvi58NtL+FnhXwdY
+3Ec48cX9u17qubgtDAgyAip0zu43f9M2Peuqbx58Kh8b7rx0l74htoN0V/bwWNoY2+2DcswcHqjr
+sPUA7nzxXl/xR8eT/E/x3qniG4hNvHdFY7e2ZsmGBBhFPv1J92Nexho4meIi25KnZNqV/ita1+vd
++aPncbPBU8HOKUHWcnFONrct781lon0XXlZ7H4B/Z88PP4E8Mat4h0jxHr2o+IQZtuhsVTTLcqSk
+jgfeJG3rkksQFwDXLaJ8JfDuo+B/FeoI+o31zY3F/Db37P8AZmtDCqfZYprZgCZLh3KYODkACuq8
+BftCeHYvCXgyy8Sap4g0e/8ACkmVi0lN8GqxhCsayge3VWwMg888ecXPxJ07UPiN4t8fPaNba5OT
+LoNgYxJHFcECNbmZvu7okG8DBy7D+7k89JY+VSpztrXTe3xaJeVtW1oktU9TrryymFGi6Si7qz2u
+vdTcn5p3ST1bfutaM57x5oOn+EtStNBty0+sWMGzWboSboWvCcvBEBxthGEZgeW3Z6VzRr2fVPiX
+4K8TeFPCeiau2qS6fbzWjX1qlsFbT0it3SdoZQcyNPKyuxHvnmvK/FH9jnxHqR8PC4Gg+d/oIu8+
+b5WB97POc7utezhqk5LlqRaa6vb+vLa2ztY+ax1ClCXtKM4uLtot1p+Pru3uk7oy6KKK7jyh23py
+o/EUYOQcj8xXx3lv+ekn/fZoy3/PST/vs16v1D+9+B9V/YP/AE8/D/gn2Jt4/h/76FNeISLtfay+
+hINfHuW/56Sf99mjLf8APST/AL7NH1D+/wDh/wAEP7B/6e/h/wAE+wzGGG1tpUjBBIwRQEwoA2gA
+YAyOK+PMt/z0k/77NGW/56Sf99mj6h/e/D/gh/YP/T38P+CfYmPcf99Cjb7r+Yr47y3/AD0k/wC+
+zRlv+ekn/fZo+of3vw/4If2D/wBPfw/4J9ibT6j/AL6FG0+o/wC+hXx3lv8AnpJ/32aMt/z0k/77
+NH1D+9+H/BH/AGF/09/D/gn2JtPqv5ijb7r+Yr47y3/PST/vs0Zb/npJ/wB9mj6h/e/D/gi/sL/p
+7+H/AAT7E2+6/wDfQo2+6/8AfQr47y3/AD0k/wC+zRlv+ekn/fZo+of3vw/4If2F/wBPfw/4JP8A
+YZ/7n6ij7DP/AHP1FFFehzs+w5EH2Gf+5+oo+wz/ANz9RRRRzsORB9hn/ufqKPsM/wDc/UUUUc7D
+kQfYZ/7n6ij7DP8A3P1FFFHOw5EH2Gf+5+oo+wz/ANz9RRRRzsORB9hn/ufqKPsM/wDc/UUUUc7D
+kQfYZ/7n6ij7DP8A3P1FFFHOw5Ef/9k=
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="vyhimdapv.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <4b33101d0b8596928da1a0a6340bb5@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAfQCd
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++LvtEn/PV/8Avo0faJP+er/99Go6K/p4/J7kn2iT/nq//fRo+0Sf89X/AO+jUdFAXJPtEn/PV/8A
+vo0faJP+er/99Go6KAuSfaJP+er/APfRo+0Sf89X/wC+jUdFAXJPtEn/AD1f/vo0faJP+er/APfR
+qOigLkn2iT/nq/8A30aPtEn/AD1f/vo1HRQFyT7RJ/z1f/vo0faJP+er/wDfRqOigLkn2iT/AJ6v
+/wB9Gj7RJ/z1f/vo1HRQFyT7RJ/z1f8A76NH2iT/AJ6v/wB9Go6KAuSfaJP+er/99Gj7RJ/z1f8A
+76NR0UBc6L/hXfiH/nwX/v8AJ/jR/wAK68Q/8+C/9/1/xr2ryhR5Qry/rU+yPmv7Sr9l+P8AmeK/
+8K68Q/8APgv/AH/X/Gj/AIV14h/58F/7/L/jXtXlD2o8oe1L61Psg/tKv2X4/wCZ4r/wrvxD/wA+
+C/8Af5P8aP8AhXfiH/nwX/v8n+Ne1eUKwvGHiSPwppiXHlefcTP5cMROFJxklj6AVUcTOTUUlcuG
+PxNSSjGKu/67nmX/AArvxD/z4L/3+T/Gj/hXfiH/AJ8F/wC/yf41e/4WrrfaKyH/AGyP+NH/AAtX
+W/8AnnZf9+T/AI103r9kd98d2j+JR/4V34h/58F/7/J/jR/wrvxD/wA+C/8Af5P8avf8LV1v/nnZ
+f9+T/jR/wtXW/wDnnZf9+T/jTvX7IObHdo/iUf8AhXfiH/nwX/v8n+NH/Cu/EP8Az4L/AN/k/wAa
+vf8AC1db/wCedl/35P8AjR/wtXW/+edl/wB+T/jRev2Qc2O7R/Eo/wDCu/EP/Pgv/f5P8aP+Fd+I
+f+fBf+/yf41e/wCFq63/AM87L/vyf8aP+Fq63/zzsv8Avyf8aL1+yDmx3aP4lH/hXfiH/nwX/v8A
+J/jR/wAK78Q/8+C/9/k/xq9/wtXW/wDnnZf9+T/jR/wtXW/+edl/35P+NF6/ZBzY7tH8Sj/wrvxD
+/wA+C/8Af5P8aP8AhXfiH/nwX/v8n+NXv+Fq63/zzsv+/R/xo/4Wrrf/ADzsv+/R/wAaL1+yC+O7
+R/Eo/wDCu/EP/Pgv/f5P8aP+Fd+If+fBf+/yf41e/wCFq63/AM87L/v0f8aP+Fq63/zzsv8Av0f8
+aL1+yC+O7R/E9lMeFYjqAf5Vzi3VxtH+kSdB1Ndc8fyP/un+VcYo+QfSvwfjvE1qM8N7Kbjfn2bX
+8vY/qnwAyvA42lmbxlCFTldG3PGMrXVW9uZO17K5q6NJJNNOJJWcBFIDHpz2rU8v3rN8OLuurn/r
+mv8AM1u+VX1fCVWdXKKUqknJ3lq3d/E+rPyPxgwdDCcZ4ujhqcYQSpWUUopXpQb0SS1epU8uvOfj
+Qu2z0b3ml/8AQRXqPlfSvNPjgm2x0T/rtL/6CtfcYZ/vYn5HhYWrRf8AWx5TRRSMssmEt0MlxIQk
+SAZLOThQPqSBXun0aV3ZC0V+rHhX9kP4IXklz8MLvwVYS+O9L8KWV7f62Xcv9onEsXm8N98SQs/T
+GGWvlT9iP4KeDNXX4i+L/jBpsF74T8JiLSJIb1X8sX7zBXbCkZK4Rcf9Nvy+ZpZ/h6tKrVjGXucu
+ml5c3w216nsSyypGUYuS1v8AK29z5Sor7u+MP7Jfw78Kftt+AfDN/FH4Z+GXimxadLWKcwRveQqy
+tarKxyvmt5B4IJ3lRgkV9H+HP2OvhZqPjzxLo+p/AHStH8OacludL8QNqfnjVC6kyDyQ2+LYcL85
+O7rWFbiXCUacKnLJqUebotL2tq1dp7pX7msMoqzk4uSVnbqfkFRV/wAdwRaX4m8U29mgt4LTUr2G
+3jTpGiTSKijPoAo/Cv1B1/8A4J+fD/4k/Dv4cf8ACOaHY+E57iSyvte1ezL/AGma0+zM0sUWSQHk
+kaP5iPlGTyQAfTx+bUMuVOVe9p317WV9f+AclDAVMRzqD1iflfRX3dF+zh4K+Mn7ZuqfC7TvA0Hw
+88GeCrRr3URaO4vtcB8oRkylm2xPvUjHzbQ5JDMNvqXhv4Mfs+/Gv4m/E34P2Pwsi8M3fg2KBI/E
+OnTmO4laRfmdSOQUbA/eFw3JI7VwVeIaFJJunJ+6pO1vdi2km9db3WivudUcpnK/vrey31Z+X9Ff
+bf7Kv7MPgrR9e+LepfF+ztdd8OeE9bi8J2kk6MIJrxp1jMqhSCCfMtgB280+lX/iP+xBoeofty6P
+4E0uE+HPh/rGk/26YbBjuijgAjngiZskF5PLOedolJHQCuiWe4SNedF3tGLle2jsk2l1vZroYrLK
+zgppq7drfqfCmaWv1J074L/s+eMPjr4t+A0Xwmt9NutC0SHUv+Eht7hlndpBHwr58wMomQhmYgnc
+Mcc/Hdn+xD4v8V/F34keCvD1493F4MvYbdrzCqZ451eSEsMgBtijIHcntilhs9w1fm9onTslL3ra
+xlaz0b7r7x1csqQtytS1tp3RvW08V/Y+fECFZG+VuqnHIPuK4lfuD6V0vhMs9lfoT8qtkD0ynP8A
+KuaX7g+n9K/n7ibGyzHBYDFT3lGd/VOKf3tXP7L8LsipcNZ3xBlVB3hTnR5b6vllGpOKb6tKSTfV
+q5ueFhuurr/rmv8AM10RQVg+EBuu7z2jX/0I102w+9fovCD/AOEal6y/9KZ/NfjJG/GuMflS/wDT
+UCt5Y/yK8w+OybbHQv8ArtN/6CtesbD715Z8fARp+g9f9fN/6CtfeYV/vo/10PxuhG1RM8fr2X9j
+rT/Cup/tKeDE8ayWUXh6CSa8kk1G5FvAs0ULSQFnJA4lVCATgnAOc4rxqkZQ6lWAIPYjNe3iKTr0
+Z0k7cyaut1fqe3RqeyqRna9mfpFZ/t7fB7QP2gtd12z8C63/AG9qlzH4fvPFA1CNrWa1jmEaTBDL
+tWIYD7lUHBJ7mvNf23vjr4H0zw14t+Dvwy05LaDWNcj8R6tr1hepcWV7NLieQIVdju81YcjhV8sg
+AcY+KMDGMDGMY9qRVVBhQAvoBj9K8GjkGFoV4Vot+6lo22rq3K/l0Wx6k81qzg4WV35fefcvjD9v
+T4b/ABQl+Guo+MvAHiLUNZ8GXiakjwy2phuLkQlG3Bmzs8zZKOnMS1Q+Ev8AwURj8G/Fv4oeMvE2
+i65rVt4quLU6Zptrdps023gWREjKuwXcVZMsnVg2e1fFVFa/2DgPZulyPlatu9Ffm07a9iP7UxF1
+K60/ytqXPFd6niLXNevoVMEepXt1dRxyclFlkd1VsdwGwcccGvsH4pf8FC28R/DnwDovgzStb8Oa
+34X1LTr+S6uLlBb3sdtGytA4jbc6OSuQeMDnnFfGVFehiMBh8U6brRvybfl8zlp4urS5uX7R9pfE
+L/goFo9/8ZPC3xQ8FeD9R0vxJYWkmkaza6nPF9n1bTXYOIi0ZLLJHIA6PjjJyCOK3z/wUK+HHg7V
+vFvjLwH8K9RtfiJ4qSIajdardoLQyRLtRiEkYlRnJCKhY4yQeR8HUV5zyDAOMY8rslb4nqr3s9dV
+c6v7UxF27r7j37x5+1CviX9nLR/h3pdlqFr4kk15vEviHxDdPGF1G9aWSdmVUOcec0ZGQMLEo5Ne
+sfFn/gohZ+Mdc+Hni7wz4Vv9K8beE7mQvJfzRNZXlpPDsu7ZthLYcrGytj5TGDjNfFNFdEsnwUmm
+4bOT3f2tJX8mtLdFsZrMa6vr2/A++W/4KIfDLSPFetfEXQvhPqsXxO1nT49Ou7q6vkFsY4/uBmDH
+IBVMlYwxCLXsn7Fvxo8B6d8I5vF3i/xboGlePPF+qXmpa81zqCRSyyiZ0iHlM2Y0WIRqijA2gHkk
+mvyipjRRscmNWJ74zXm4jhvCVaLo0243td3bdltHXou3kjpp5tWjLmkk/l36n1P4c017HRpGlXZL
+OGlZT1UbcKD74riE5RfpXqrxl1Yd2BGfqOtcongJ1XH25Djj/Vn/ABr8n4hyDE1aGFw2Xw5o0lJP
+WK35ddWt7N6H9BeG/iHlmEx2b5lxFiFSqYqdOSSjOS932isuWMtIpxir62tvqQeCV3Xl9/1yX/0I
+11nl1naF4cbRZ7iQ3CziVAuFTGMHPrWvsr6zh3CV8BltPD4iPLNOV1dPeTa2utj8e8Ss3wOf8T4n
+Mctqe0pSVO0rSV+WnGL0kk9GmtiHy68o/aCXbp+gf9d5v/QVr13y68l/aHXbp/h/3nm/9BWvscI/
+38f66H5pSjaaPF6ZNJ5UTuFLFQSFHc+lP6V6/wDsifDXTPi5+0b4L8M6y8yabJPJeyiDGZPs8ZnE
+ZyCNrGMKfYnHNfQV60cPSnWntFNv5HqUabq1IwXVn1z4a/4JwfDbVfDaaNda94iX4mxeHLfVruzW
+6jW2hnmWRUyhiyE82J127s4XqM185fsW/s06J+0Brniu48c395oXhfw7axLdXNncJbMLyWQqkZd1
+YAAI+RjqUr7X0v8AaK+Bvh39qPxPr7/EfU5vE+sC38Jy6LJpk32OCWCfy0CSCH/nq0nzFiv7w9sV
+4f8AtgeI/An7PPwx+Ifwj8C6ncy+NfFHiaPxDqsE1uyC0hldblUSXYFZB5USKoJbDnPevzvCY3Mq
+jlhZOfNU5Wm4tcqv77V1sr2XyPq6uHw0UqqtaN1vv2OJ+IH7DNno37Yfhr4T6TrN3Y+F9f046pb6
+nfMs9yscSObiMEBVaTdH8vHAkyQcc+3eHP8Agnn8EvEXjrxJ4Sin+JFvfeH0ga4vb1o4LO581dw8
+iUwYkx/Ft6Gs74qftKfAX45eIPg54p1rxrcaJq/hG/XUtQgh0u88x1aEM9usiR9PPjhBIOCofnnn
+V+Ev/BRjwzcfFv4oX3jjxRcad4MlntYvCdmNOklAgjWRZZT5cZcNIQjkP03YGMVnWq53VoRcFNSh
+CzVmry5rXWnvNx1a0SWo4QwMZtO2r016W/A/N7xhaR6Br/iGxtizQ6dfXdtCZTlisUropb1OFGeO
+Tmv0IvP2Cvg14d8H+BdW1U/EfUbjxK9nbbdCKXKwTTxh98irATHEDnLHOOK/PnxxdRaz4j8T3dk3
+mW97qF5PbsRjckk0jISD0yGHX8a/SvVv2vvgx4q8A/D/AElfi/4k8FXmgPZXF3/YNhdo135MW17a
+U+UQ0ZPUcg7a+gzqWNjGh9V5uvNypt7K17J9TzsDHDuVT2lt9LnhXiz9gjTrj9qXTfhh4L8Yve6P
+Jpbazqt1deXNd6RCknlmNggCs7koEBCkZJIIHPo9t+wv8CfiD4n8bfDnwX4q8T2PxA8JwwG7u75x
+NbF5V3LlCih1BwGCFSC3Wsbxb+3R8N/DH7V+keP/AAP4emuNBn0qbSPFGoQ2gtp9QV5UkjmSNsF3
+hMf3nwWV2XsprtdH/ag/Z1+EXxA+Inxa8PeKNZ8UeJvGMUG/w7FYSRmB41wVDPGoXewBYsx24OM9
+K8OrVzlwh8afIuWyWs+bXnsu197aedz0YQwKcvh3116eR82fs6fsu6H4q8S/FZPixeX+g6B8PYzb
+6nPpMyqwu/NZSAzI25NsbEYGTvXn12v2ov2Kovhh8bvh34H8AzX13beM0NvbT61MJPKuUkAlLOiL
+8ixsjkBc8N1yK9wsb74cD9jq91f4neLtQ8J3Hxu1WbWdRvNNtGnkWYuH+zQhY32xIkIUFgThm5yR
+XX/HP4veDvG/wJ0D42+AtXuPEt58KtctjC9xbvbi7kcwwXEE3mIGUPFcA7lGA23qM1o8zx31xVI8
+3K24Ws+Tmsknf/r5frsT9Uw6pcsrXtffW27/AAODl/YT+A3hrx54c+FOu+KvFd18R9e02a/tr23d
+YoMRhskJsKIDskIVixIQ89M/PWt/sGfE5fiz4w8E+GorLXV8PC1nOozzfZluLe5EhgYLzhv3Uisv
+QFeMgivqK9/al/Zw8Y/Frwp8btR8Wa1pfijw/pM2nR+G5NNlZ38wP12oysy+bIAVfacjkY58Gv8A
+/goz4v0r4yeOfGPhrQLMab4gSytbex1KT95bwWolEbMVyC7mZ2YDheBk4ybwU86blyqTfKr+0Vlz
+82tttOXtp+BnXjgklzWWulu1uvzNTZ7UeX7VY8v6UeX9K7bnxPKV/L9qPL9qn8v6UeX9KVw5Sv5f
+tXkX7Ra7dN8Pf9d5v/QVr2Xy68e/aQXbpvh33uJ//QFrtwb/ANoj8/yLgvePD6ltby4sJ1uLS5ns
+7hM7ZraVopF4xwykEcccetRUyWQQxPI3CqCx+gr6q19DoV7qxIZHMhkLsZS2/wAwsd27Od2euc9+
+tS3d7c6jO095cz3lw3DTXMrSyNjplmJNfZOh/wDBM3Xda8BWutDx5YQ67NosesHw8dOLTR+ZGWSJ
+m83uylN23GQ3pXn37MH7GV9+058PPEHiex8W2vhyXSrtrMWV5ZmVJHEKSgvIHGxcvtJ2tjBPPSvG
+/tjL3CdX2itBpN2el9unXy0PR+o4m6jy767nzjRX1H8V/wBhibwP8CY/ip4U+IGmfEHQbeIT6gdP
+t9kaxh9kklvIHbzFRshgQDhSeoxV3wp+wWlt4B8P+Kvin8StL+F6+IpIodK066thPM0kozEkpaRA
+rkfMVGdoPzEEHB/bGB9n7RVNLuOzvdbq1r/gH9nYnm5eXpfy+8+T6K+lb39hXxZoX7R3h/4T6zrN
+pZx6/bXN3pviSC3aWCeOGNnceUWBWQYAK7uN4IJBFdJ4d/4J56j4h+OXi74ar47tYLnw7ptlqL6i
+dLZlnFxvwgj80bdu3ruOc9Kcs4wEFzOqrcvNs37t7X276W3BZfiW7KPW3Q+RqK+hfhf+yC3j3xN4
+3t9Y+IOg+DvDfhTVLrSrjWNRdRLdPAx8x4oGkUhAoDFmYDnA3ENjb+Ln7EkPgj4PRfFXwh8Q7Px9
+4FVo3uru3sfJmjgaURPNEvmMJdjdUyp4NXLNcHGqqLnq2ktHa72V7Wu+17kxwFeUXNR0Xmj5mmvr
+q4tobea7uJraDPkwSzM0cWeuxScL+A5oS/uorWW0ju7iO0lIaS2SZhFIw6FkB2sfcjivqfWf2A9R
+gvvhteaB44svE/g3xtdC1j8SWunOq2ReJpIWeLeSVkCFc5Xa2A2CRm63/BOjxAvxd1nwg3jGyg0H
+RdFi1fUvFV1YtFbwNI0gW32GTltsbOW3gKvJ5IBx/tnLra1Fs3s+js+m9+m/kafUMV28t12/Kx8i
+UV9Q/C79ia3+MXgHxr4z8N/EWCbQPDt7d2kE8ujup1FbeBJTMo83KK+/CggnABI5xXV/Bf8A4Js6
+n8ZPhX4X8bw/EC10uHXbGO9WybSmlMO7Pyl/NG7HrgUVc6wFBSdSpazSektG1e2xMcuxM/hj+KL2
+yjZU2z2o2e1eBc8qxDso2VNs9qNntRcLEJTivGv2ll26Z4b/AOvif/0Ba9r2e1eL/tNjGl+Gv+vi
+f/0Ba7cE74iPz/IpLU8Gr0j9nH4T/wDC8Pjb4U8GPdR2Vve3BnuZJELBoIQZpUAHdkRlHuwrzjvW
+34J8ca98N/E1p4i8MapLo2uWgcQXsKqzRh0KPgMCOVJHI719NiI1J0Zxou0mnZ9nbQ3oyjGpGU1d
+J6n7D+H9R0bRv2n/ABZ4mu/ib4QfTNS0qy8PWfh1NRiW9tp4JXO1lL9TJNMNoGeVHY1zf7MPwqtv
+g9H8ffCEt9H/AGaniCa+juUjO2C2ubOOZEK9zGjYPrivyLl1S7m8RPrzzs2tvenUWvyB5pufM8wy
+5/vb/m+te5+Ev25vi54Qj8RbNW07WLrX5RNqF7q1gss0hEKwqAUZAFCIABt9T3r4Kvw3iY0pQo1V
+LmUU76axas+vS59NDNaUpXmrWb/FH014l8U+CP2c/wDgnpP8PYPHuh+Ndf1ixubbTxo86yLcm5lZ
+94QElI0R8sWxypHUgV64v7QekftGfDDw5qHgL4h+DvBevW0kb6tp3i60iuJLVdmJEWJ3QgqeVcZV
+gOozx+Q9vaxWqhYownAGcDJ+p70skEUxHmRpIR03AGvVqcN0qi5pVG58zldpNXla65drafecizaU
+ZaR921vuP0svvjn4d1n9tb4U2lx8UbHxbpvhyLV5LzWZILaysLOSazKLEtwhCysdvPO0HYASSQvt
+fhX9qPQ7/wDaQ8d+HL/xZ4Uh8G6fpGn3Gm6l9shQ3E8m/wA5PPMm19u0fKOVzX41NGjJsKgp/dPT
+8qabeIptMaFRyAQMA/SorcMYetFRc9o8q0X83NfSy6tbCp5tKF/d3d932SP0q/ZWl+D91rXxo1aa
++8Ij4ov4p1d7DUfEZikjS2MjG0lgLEBoSxJYxEE85ONldB8ZPilpvi/9hvxb4dvviZ4X8eePXhay
+nk0q4gtUurgXi/JDESuUQYUOBhghbPNflrJBHKqrJGrqOgZcgfSlMEZfJjQnGMlRmtp8PRniFXdV
+tqUZK6Tty9L72fYSzVqDhybpr7z9Qvhl8V/Cn7KmhfBj4QaH4z0PxDeaxqM114l1n+0EnsrCIxvJ
+KEcOFiDTFVjB7K5K7mzXf+Kv2lfh/wCOPjJ4n+D3irW/D974C17wwk1vqsV9GIGmZpUubaScOVDl
+PLdOhG1upK1+PwgjVWVUUIeoCjBpPJjEewRrsPJXHFYT4YoVZurOo+d3u7K/M3fm+W1uxSzeUVyq
+On6dj9RP2Rdb8Pfs5/Bb4peGLzxx4ZuNU0vXNQn0ud9Ut2XUYTaQtbTKof5t+ACoPDK69RXtX7Pn
+7S/hfxf8FPButeLPGXhTSvEl9p0c1/YrfwWwglPVfKZ8pj0NfigLaEEfukGOh2ihraF2LGNCT1JX
+NRiuF6WLlOdSq+aTTbsu1u/XcdPN3SSjGGiP2T+FH7J1hJ4dnu/G8MkmpX0WIbOGUr9gU8htw6y/
+oOnPNeW/Fj9nXxD8MvOv7VZNe8PJljewR/vrdev76Mc4H99ePULVH4/fG/xR4i8f3kNpqNxolnoF
+/Lb2UNhMyHzUO0zOwxvY5OARtAJGDkk+0fsp/tF678W7vWPD3iC0t5L7SIVk/tSA+WbhSQPmjxgN
+7ggH+6K/KaefYhV2779Ony6r+rn0c8mw7oqNtuvX/g/1Y+T4mSeJJImWSNxlXQ5Vh7Edad5ftX1p
+8ev2avD99o+s+LPD0o8MaxaW8t7cR28Aks77YhYiSDKhWO3/AFiFW553dK+QPC+sjxP4astXEBtf
+tIf9zv37SDj72B/KvtsDmVPHXjFWkldry20f/DM+TxuXVMGlJu8W7J+fmv6Rb2V4l+1Eyx6X4Z3E
+Lm4n+8cfwLXueACeOhplxZwXGBPbw3AXkCaJXA+mQcV71Cr7CqqjV7HlKx8I+fF/z0X/AL6FHnxf
+89F/76FfdP8AZOn/APQOsf8AwFj/APiaP7J0/wD6B1j/AOAsf/xNe1/a6/k/H/gBZHwt58X/AD0X
+/voUefF/z0X/AL6FfdP9k6f/ANA6x/8AAWP/AOJo/snT/wDoHWP/AICx/wDxNH9rr+T8f+AFkfC3
+nxf89F/76FHnxf8APRP++hX3T/ZVh/0DrH/wEj/+Jpy6Rp4J3adZFVDEgWsYJwM9dvfFCzZN25Px
+/wCAFkfCynzFyp3DpxSgVt+I/HEvxBvJr650+0t3u9QW1iijTEdtbqxxGirtGTtyzdWyelXde02S
+1stZmadQk+pxaSscECR+XEoz8pwSOFxgY65JNfEVOP6EJuMcO2uj5rX+XKz7enwrVlBSnVSdtVa9
+vndHLZABORgdc0zz4/764+orb0jxlJceK4YYrcRxw3kUFpJKVklt34+YttAdSeqMuD2x1r7D0SKy
+1nQ7DUH0vT4nuYEleNLSPaGI5xxwMg8e9evlnF1LMZSh7FxaSfxXuvuR4+Z5PLLoxqc/NFu21rP8
+fvPiLz4v+ei/99Cjz4v+ei/99Cvug6VYD/mHWP8A4CR//E0n9l2H/QOsf/ASP/4mvoP7XX8n4/8A
+APAsj4Y8+L/nov8A30KPPi/56L/30K+5/wCy7D/oHWP/AICR/wDxNH9l2H/QOsf/AAEj/wDiaP7X
+X8n4/wDACyP/2Q==
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="aiuuwycywezze.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <b72d801d0b859f927cd8c0818aa475@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAjwCd
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+9P1L9lH4qS5SDw7ahB0/4mcI/rXPj9kz44aRqUGp6No0enanbHdBe2utRRSxn2YHp7Hg96+mv2i/
+2xtF+Buqf8I/Y6a3iHxMIhNNbiYQwWaMMoZZME7iOQgBOOTjIz5B4P8A+Ck9xLqsa+J/B8EelM4W
+W50i6Z5YB3Yxuo3Y7gEH+VeJUq4SnPklPX+vI+/wPB3EGYYNY3D4e8Grq7im13SbT9O/S57t8CfE
+fxqhWHSPij4QtGCriPxDpmoQMW9BPAGHP+1Hx/sjk1U+L/7JGheN7i71zwndDwV4qmJklmtoQ9jf
+v/09W2QrE/8APRCsnP3j0r2/RdXs/EGlWWp6fcJd2F7Alxb3ERyskbqGVh7EEGrrDANezS5qbUoS
+d1s+v3o+DqRvenOPk01+aZ8Hab+zb8VpmuIdQ8MWNvPA/lie21WOS3uQP+WkW7Dqp/uuoYe/Wr3/
+AAzF8SP+gJa/+DCL/GrHi7/gorc6J4u1rTNN8FWuo2FlezWtveSam0ZuFjcp5m0RHAOM9elfUHwR
++KEXxk+GWieLIrRLF75HE1osnmeRKjsjruwM8qe3QivVw/EtSo/Zwak13T/4COzN/D/G5Rh443G0
+XTpzaStKL1aula7a0T39D5Y/4Zi+JP8A0BLX/wAGEX+NH/DMXxJ/6Alr/wCDCL/Gt74w/t66l8Lv
+iN4k8MQ+CbXUYtInEIun1Fo2kzGrZ2+Wcfe9a9o/aB+OVz8E/hdZ+LYNFi1eWe6t7Y2klwYgPNBO
+d209MelWuKKjU2re7vo/8xVfDzHUZYWM6bvif4fvQ97RP5aNb2Pnn/hmL4k/9AS1/wDBhF/jR/wz
+F8Sf+gJbf+DCL/GvZ/2af2nZfjxpviu+1LRLXw1b6CYS8gvDMpV0dmZiUXaFCda8o8af8FJNMsNT
+mi8L+EJNX02Ntq6jqF39lEwzjcsYRiFPbdg4IyBUy4qlGCqNxs9tH0+ZvQ8Nc0xGNq4ClRk6lO3N
+70LLmV173w3a2SdzPP7MfxJx/wAgW1/8GEX+Ncd8WP2WvjdfeDJYPCei2n9pSTIssbajAGeD5t6q
+XO0H7vXtnrXRR/8ABS7VJl3R+ALF1HUrq7HH/kKvrj4JfEWX4tfC/QPFs1immPqkLSm0jl8wR4kZ
+cbsDP3c9Kz/1jlj4SoRtqtdGtPvHmPAGY8NRp43HU+WPMkvehL3rNrSLb6M/Je0/4J6/tOJbadEv
+hmxt/sUjSKv/AAkVuiksMEgR4w3q2c88Yquf+CaX7Q0yvHN4X0wRvN5/Gr25KnJOAWZjjnnjJ75r
+9qdopSK8pU4LZHA69SWjZ+Kms/8ABM79oW3vZLmz8OaffZUeaja1bASHH8BO3HQcEY9K9q+A37Kn
+7Q/hmKXRPFfhS0/sgIZbO6/tu2le2fvFgMSUbr3wR1wa/S/xDr+n+FdDvtY1a7jsdNsYWuLm5l4W
+ONRkk456DoOTXxP4v/4KTzxaw8fhrwfbtpisRHPrF40cswzjd5aKQg+pJ/lQsfDLKsaydn6XuvRd
+D3ct4YzPimnUo4OlzxVrttJJ9NW1r5av5E//AAzH8ST/AMwS2/8ABhF/jR/wzF8Sf+gJa/8Agwi/
+xr2H9nH9rrSfjvqE+hXelt4e8TwwmdbUy+bDcxggM0T4ByuQSjAHBBGRnFD9o39svSvglrR8OaVp
+a+IvEiRrLcxvP5NvZqwyokYAksRztA6EEkZr2/8AWiape293l9H/AJnkLw9zCWZf2SqMvbJXteNr
+fzc3w2872vpvoeW/8MxfEn/oCWv/AIMIv8aP+GYviT/0BLX/AMGEX+Nb3wc/4KC2vi/xTZaJ4x0C
+30FL6ZbeDVLK4MkEcrHCrKrAFASQN2SASMgDmvYvjr+1J4Q/Z9v9JsvEUV7c3WpRyTRxWEQcoikD
+LZIxkkgf7ppQ4pnOm6qcbLvdfqXivDvMsHjYZfUoTdWabik4u6W9mtNOut0cn8dfgj8MvBMOt/GD
+UfDE+raxpUi6rPbf2hIsV5KGUDej7kxnacBcfKBjtXxN8a/i1a/tFfEXTNTsNF0vwehhTThJc3aC
+MndnzbiYKFVVzjOOB39Pdfid+1b8Q/h18aNQ8K+OtI0rUfCdvdt52lw6eM6hp758uVWkYhiAQSOB
+uQqSK8Z/af8Aif4B+JvijStQ8DeHf7As7W0eK8uJLVLX7UxYFB5S8fINw3Hk7sdAK/PcZUpzUlTf
+Kr6q1m33uf0zwdl2YYSpQlj4SrOUH7OsqjlCEHFWhytpa9LJ7q2zt+lHwh8JJ4B+GPhbw7HepqI0
+7Tobf7XGcpMQoy6/7JOSPbFQfG7x0Phv8JPFfiTdslsNPleA/wDTYjbEPxdlFct+yTpWr6J+zt4J
+tNbSaK9W0ZlhnBEkcLSO0KnPpGUGO3TtXz5+3v8AHa3vIbr4V2lndQ3tvdWt7fXjlRDLDsMixrg5
+J3FCcgAbDXt1K6o4VTWl1ovO2iPxfLckq5vxPLBX9rGNWTnJWs4qfvS+fS19zwb4efCoa7+zR8Uv
+GEtus93pd1YJaXBGWAjffdbT/tLMuf8Adr6Q/wCCcHjT7T4e8XeEZZPnsbtNSt0PUxzLtfHsHi/8
+f968s+Fn7S/gbwJ+z1e/DjUfD2t3l3qVveJfXcCQ+S0s+4bly4OFUoOR/DXmf7NvxlT4BfESPxJf
+W9zf6c1hLZXtvabfMkU7WQgMQOHQHk9CeteBRqU8PUpTi9LWf4/5/gfvWcZdmOe5dmuErUmn7RSp
+Xt7yiopW10vyve3xkn7VsXnftE/EJN23dfoM+n7iKus+N/7XVx8b/h3a+EZfC8OjRwXMFyLuO+Mx
+PlgjGzYOufXj3rjv2rJTL8fPHk6EqJriGZexAe1hYfjgivpn9sz4W+DvB3wA0zVtD8L6To+pSajZ
+I91ZWiRSlWViw3AZwcc0uWo/buLsle/nqzoliMvowyKnjKLnUkoqnJNrklyU7tq6v03vscb+xb4Y
+vPGnwh+OWgae4iv9StIrS3Odo3vbzqoJ7Ak4+hrwb4V+JoPgx8VbLUfFXhQ6r/ZDSwXeiXiBJIpC
+hUMFcbdyE5GRjngg4NfUP/BNmeO20v4lSyusUST2TM7nCqBHLkkntwfyryHW9Vm/a/8A2rrKGDcd
+DuLoW0G0bSml25Lu5PYyfMef+eqjsBVyivYUJRfvXaS+Zx0MXJZ7nWFxMP8AZeSMpyu01+6irJrX
+WN3urW0ZzX7SHxf0P40+MdK1jQNBl8PWtnYG0lt5kiUu/mF937skdCBzzX3x+xtx+zL4DP8A05yf
++jpK+PP26vAnh34e/E/w7YeGdFstDs5tGM0sFjCIkeTz2G4gdTgYr7C/Y3OP2ZfAg/6c5P8A0fJX
+ZglKOMqKb1t/kfG8Z1cLW4PwNTBxcaTmuVSd2ly1N3rc9I8WfETwz4DFqfEniDTdCF0WEH9oXKQ+
+aVALbdxGcZGcetJ4T+I3hfx410vhzxDpmum12+eNOu0mMW7O3dtJxnBxn0rzH9pb9miL9otfDqy+
+IpNAGjtOwMdmtwZfNCA9WXbjy/fOaZ+zT+zHH+zrL4haPxHJr39sCDPmWa2/leV5no7Zz5ntjFev
+z1/bcvJ7ne/l/mfkyweR/wBjfWHipfW/+ffK+X4rfFa3w+9vvoep+PPAeifEvwteeHfEVob/AEe8
+2+dbiV4t+1gy/MhB4ZQevavjr4n+OvAP7HWp6t4C8P8Aw/i1qXXbAX0l3qt8JFO/fEsTh0Z2jXyy
+doP8R7nNfSn7SHivxp4H+Fmoa/4GtrW81TT3We4huoGmzajPmsiqy5ZQQ3XorcZxXyf4H/bL8KeI
+vDWqr8XvCdt4o1xg6Wl3ZaVA6zwEcQMWOY9rFvmzjBz1HPFjalNTULqM7fE1fTsfYcH5dj62Elie
+SWIwqlaVGFRxfPpaTirLTTrr6LTC/wCCf3w/Gr/FseI31azh/sG0mRdPacfa7hpU8veI+vlKGOXz
+97aPevPPBEi/Ej9q7S5tYUXiap4uaa4ilG5ZFWZmVCP7uEUY6YGKt/siaZqmp/tHeE20SKWIWk81
+1dCNiwgs/LcMrt6HcqDP3iRU3j7Rrv8AZw/amW+vbKWSx0/XBrdmq8G6snlLHYTwWAZ0P+0uK8KL
+/cU3b3VLX8P00P3Gun/bmOoxqJ1qmHXs1onFJzVr9byalfz7JM6r/goT4ds9K+N9nNaQxwPqmhxz
+z+WoXdIsskYc++0KM/7I9K7X40/s++Nf2n/BPwo8Y+H5LWa7fw3DDfrfXHlHfhXDg4O7cXfPpget
+eQftIfElf2mPjTp8nhCxvJlltIdI0y3nj2TTSFmZmK5O0BpDkk8BSTxX6X+APC0fgfwN4f8ADsTC
+RNKsILPeo4fy4wpb8SM/jXoUaNPGVay+w7H59nWb4zhXKcomrLFQUtJa2i42aa36xXrG3RlPx98K
+fCHxRsorTxX4fstciiJMRuY/3kRPXY4wy59iOgrkfC/7KHwm8H6rDqWmeCrAXsLb4pblpLnY3YgS
+MwBHY4zWfYeEvG114WS50vxzqV/qKTSq8d6IlEqhsKoIX5WA4z0PtXxV8Rfin+0ZF478X2ej6r4v
+jW2uZorSzjtYkT5RhdheMBhnnOSD2NenCrQxFOGIjC/NZrTXuj8Op5hmOFhLB0sTOMFe8VOSj56J
+pfgfp8VwMivij4t/s6+JviL+2JBrV14Ylv8AwFPLZi9u3ljEUkUcHzqRvD43ALwK+W9En/bY1hVM
+/j7UdIUgEnUtSs4yPX5VjY/hXuXwPuPj34O1XULzx/8AFlvEttPaiG3sLfa3kS71PmbvKUH5Qw49
+a6auHWKSjNNJO50ZLneIyCrVr4WzlOEoa30UraqzWqtofU//AAyD8HP+hB0wfjJ/8VXz5+1r+x//
+AKN4eb4S+B0BYXMepRWEqoCCEMRbzH7Hf09ai8Y/GjxxY+M9NsYPF19BDJp01w8YMXzFZo1B5XPc
+j8arN8afHgXP/CY6hn6xf/EV42Oq4WlKVCULPTVJep7mTcSZ1gcTTx0cRKpy392cpuLumtVfpe68
+7M+lrj9mT4c+MRBq3ijwbZ6h4gubeH7bc3DP5jyLGqHO1sZG0Dj0rufGfw28NfETw/FofiPSIdW0
+mKRJUtZt21WUEKeCDwCR+NfFMnxu8ejp4z1AfjF/8RVWb44/EAA48b6iPxi/+IqVmeFSaVN676LX
+1PHqVsxqzhOWIleHw+9L3f8AD/LstuyPsfw78A/AHhHQdd0XR/DNtp+l65EIdRtoWcLcoFZdrHdn
+GHYcEdTS+A/gP8P/AIYavNqvhbwvZaNqM0Jt3uINxYxlgxX5icAlQePQV8TTfHT4jL93x3qQ/GH/
+AOIqpJ8fviMgwfHmpZ+sP/xFL+1MKrP2T020WgSqZlUU4yxMmp/FeUve6e930SWp9zePvgX4C+KO
+q2+peKvDNprV9BD9nimuC2Vj3Ftoww7kn8a6Twp4T0jwN4es9D0Kxj03SbNSsFrFnbGCxY4yc9ST
++NfHXwp/bT1XwzNHp/jjdr2mZx/acCqLuD/fQYWQfQBvY19OanJpXx28BM/hPxtfaZHMwaHWvDly
+izwuP4WV1YfVHX8utephsRh67c6SXN17nm4irjVSjhq9WTpR2V24r0Wy3fQ9AxwaTH+RX5oftBWH
+7U/wDe4u7r4ka/4h8Jjdt8RaXBDiCPPW5i8omJgCPm5TOfmHSvBj+2D8YYkZz8VtZt4nTAlU28oR
+M4AUsg8yZ+3GBnpW0sSouzi7mccFKa5oyTR+1OMivIvEf7JXwj8VanLqGoeCLD7XM2+R7VpLcOT1
+JWNlUk9+K/KuT9sL40SI/mfFbWbYqAkro0Ei2y9k+SH95cN6A4FKf2xfjRCZ1b4ratYmNQs3nNbS
+LZRnpvZly87dlAwuelZTrUqitON/WzO/CPH4CTnhK8qbe7jJx++1v6+Z+w/gL4XeE/hhp8lj4V0C
+y0O3lIMv2WPDykdC7nLNj3JpvxB+FfhL4q6dFY+LNCtNbt4WLxfaFIeInqUdSGXOOcEZ4r8fX/a+
++NUpYf8AC1NagcRcCV4CLSL/AJ6zlYeZG/hQHuKns/2vvjPC0Sx/FHV0l8vdbwXxthiMj5rq4Zly
+q8ZVMCh16XJycunbSxKWN+sfWlXftd+a75r9773+Z+snw7/Z/wDh98KbyS88L+GLPTb+RSjXhLzT
+7T1USSFmUHuAQD3r0Aqp+9g/Wvxcj/a3+Mk5tfs/xR1+8Vsi0iLQh7xu8r7IPkhHbnJ/GrFl+1x8
+co4mTTvijdXKoxWW+1EWwSaT+JYg2PlXpnufzIsTSprljGyDEUcZjavtcTVc5vrJuT+9/wBfgfsV
+4fiWLTtoVUXzH4AAHWqviOy0DVIQmqtZbk+7JKyB0+hNFx4L067neWcXM2452NO2wfRQcU+HwfpF
+sMx2MSn12An8zRTVajCMYw2SW55EuScm29zzXUdP0SxmKwvY3sZ6NFCpI9jxjP41myPYK2ItJilH
+rsRPw5Fev3/h61vLQwNCsqHohAAH09DXJz/DK5+0Yt7qMQ9vOBLD2461Lr4rm5eT+vUcYUXuzzG/
+0Kx1LVYr6fS9PRordrdQIVZsFwx5KjjjpUT6JpY4/s2z4/6d1/wr1hfhUHA87U2HqIoQP1JNZmrf
+DIaaDLE817AOTg4dfqB1/Csnh6taXNNamyqwgrRZ5i+i6X/0DLL/AMB0/wAKrvo+k99Msc/9e6f4
+V3qaPYgEpbrIeg3ZPNd7afDzTEgiLrIkpUbgmAAcc44rT6jbdon6z2ueAjw7p0v3dGtX/wB20U/0
+ph8FQTHK+H7QD1a1jH9K+ij4B00jAlucezj/AAph+HemN1kuv+/g/wAK0WCp9X+BLxE+h86/8K4t
+3OW0jS4x/tQIf5LWl4a0KbwVftfaLLb6TcOAJfsduFWUDoHXo348+hr3R/htpRH+sux/20H+FQSf
+DHR8FnmuwoGSTKOB+VWsLTi7pakuvN6Nlfw38SrHW4xZaukdpcONjb+YZc8Y56Z9D+dcL48/ZM8G
+62z6j4c0TRtG1JjvMX2CM20p9cBfkPuvoOKwrtGmmk+zo5t3crErAFmXJ2j3JGK9Y+G+geINGtyd
+TviLRx+7sJP3jR/Vv4f90Z/Ct7KWkkQpODvFnx34i+E9h4Svo7LV/Cem6fcRn9yZbCJk/wBqSJiN
+rE+vX1rNh8IeGQFEHh/Rl8vMiKLKNhB6yuQgDOfSvtX4ifEj4c+FEFv438ReG9O2kOtvrN3ArD0I
+Rzn8hXCWvxw/Z18UyrZReLPAV0zMCIpLi2QE9vvYFbRy+vUXPCDa72Zf12C0k9fU+aD4L8NybQ3h
+vSZDI3nJHcWcJMh/57zZz+ANOTwd4e2Hy/DmmSpOwJVbCMNeOOjNhB8g7fSvq/WPgR4O8X2H23Q5
+jp63H7wXGmT77ef0yMkFfZSK8G8beBNX8B6k1trKgLKuf7SgVjFKnaKLc3yt6qeee/JrhlTcN0dU
+KqnszipvCHh52lkuNE0t84jnnFjFuc9oY93btxx1/GKbwJ4SnKR3XhDSL14lCrbwWMJjtl7KNsZG
+T1Nbal45o1UR29wFzGgZFW1jx95sA/Mf89KdFveFDDfPY23JR2aQvOf4nJxWdkbXv/X9f15n57f8
+Lc8ff9D54p/8Hd1/8cqO4+LXj94JFXx74pDFSARrdzwcf79cxQK/qn2FL+Rfcj8VVaqnfmf3n7yf
+BLxkPFvwL8FeJ2d7p7zQbW6mYnLPJ5Cl8knk7gevpU8vxQVv+PfTpHHZnkAH6A15D/wTu8RJ4n/Z
+H8JW7nc+mNd6XJzziOd9o/74ZK9Gi0+7GUh8Ofdyu68vj274UV/N+aezwWLq0pu1pSS+TP0/DXrU
+oyXZFyf4jarKuIrS3i9zlj/MVny+MdeujtF4see0Uag/1q7HpOsyfdtdGs/fyWlP5k1aTRdab/Wa
+68APVbO2SMfhxXhSzCgu7OxUJeRi+HtNuNZ8SWzXUhlw/nSlgedvIB/HFfNf/BUj41al4M8OeDfB
+nh/WbvSdU1S5k1O8m0+5eCdbaEbEXchB2vJJn38o19i+FPDZ0m7vL2S7mvJp1VPMnPzDHJ/M4/Kv
+x5/bj+Jv/C0/2nPF95DMZdN0eRdCssNlQtvkSEexmaU/lX3/AAjho4/GxrOPuxV9fuX+fyPBzes8
+PQcU9XoeZ/8AC3fH3/Q+eKf/AAd3X/xyj/hbnj7/AKHzxSfprd1/8crlKMZr9u9hS/kX3I+C9tU/
+mf3noXg/xf8AEvx94u0LwxpvjzxV/aGtX0Onwka3dfK0jhd3+s6KCWPsK/a3xOU8D/DyDTbWWWUx
+W8WnwSXEhklYBQu52PLHaCSTyTX5nf8ABMP4X/8ACafH678U3MJfT/CVg0yMVyv2u4BjjH1EYmP4
+Cv0c+IYbxH4r0rQ4ydikGTHYt1P4ID+dfj3GuKiq8cNSSXKuneX+St959xklKTp+0qO9/wAkS/C/
+wuxt11u9QGRxi0j242L03/U9vb618Eftkf8ABQXWvEWu6j4K+FmqvpHh+0dra98SWjYuL6RTh1tn
+/wCWcQII8xfmc/dIXlvpf/goT8aJPg3+z5NpejT/AGLXPE8n9i2TRNteGDYTcSKe2IxtB6hpFPav
+x+RFjQIoCqoAAHYV6HCeS06sPruJXNZ2intdbvz7Ltr5HLnGPlTfsaWje4sw+0XMlzMWnuZWLSTz
+MXkc9yzHJJ9yaRkVxhlDD0YZp1FfrGx8Y227tnoHwX+Pfjj4Aa9FqXgzWZbOAOGuNInZnsLsd1kh
+zgcZG9cMM8Gv18+EfxO8K/tf/BWHWbWAWzTMYLyxl2yTaXfIBkA4wSu4MrY+ZGUkYJFfiHX2B/wT
+A+Js/hL4/XXhJ5SNM8V2Ei+UT8ourdTJG/18vzl9/l9K+J4mymlisLPFQjapBXv3S3T+WvyPocqx
+06dVUpPR/gz3vWNOm8O6lf6Rfs6y2U/k3ClyDdy9QcKPukYYex9qqSsyysZLFtRuiB5nlhisQ/hU
+En0r64+Ivw30HxZuvb+L7FeRQsf7Vt2CSQqB1YnhlA5w3pXyibUSzzpoN1FcadFIyx308D7roZ/1
+m1jxn9evevwiS5XY/SacvaI/MWiiiv6sPxQ/Tr/gkx4la9+FHjfQGbI03XFuVHos8Cf+zRN+tfbj
+aczSMwKgEkivzQ/4JMeJDafFPx7oDSbUv9HgvlTP3mhmKH9JxX3H8VPjNf8AgDxD/ZkVhFIj26Tx
+zOCSckgjqO61/P8AxZgqcszqKa0dn96V/wAT9Hyqq/qsXfyPThpo7v8AkKDpi5B8xlHfgc189XPx
+w8XXkeYRb26sOCFX/A12nwU8X674m1nWF1e8FxFFBE0aD+Elmz/KvlVl1FK/s/v/AOHPU9s3pzG7
+8dPiLD8Gfgv4u8XO3zaRp0s0AYZ3zkbYV990jIPxr8Gt0smXncy3DsXlkY5LuTlm/EknPvX6Z/8A
+BV74nDTPAXhPwDbSgT61etqV4itgi2t8bAR6NK6H/tka/M3Nft/B+CWGwLrWtzv8FovxufD53X56
+yp9vzYUf55orc8B+Cbz4leOfD3hKwz9r1zUIdPRh/AJGAZz7Km5j7CvuZSUIucnZI+fhFzkorqfq
+1/wTb+Ga/Dz9muz127iEOoeKbiTWZWdcMtvjZbqfby0D/wDbQ17J4B3a74x1HV5AeFZkz23nC/8A
+jq/rWx4kgtPBXw/ttF0yP7NawwRadaxL/BEqhQB9EXFM+FdmIdGu7nGDPNgfRQB/PNfzNmGKeOx8
+qsurb/yXyR+q4ekqGH5V00PzS/4KjePn8S/tBab4ajkY2vhrSUDR5youLhvMc/Xy1hFfHnevRv2j
+/GI+IH7QPxE19ZDNBda3cRwOT1iiPkx/+OxCvOq/ojK8P9VwNGj2ir+r1f4n5pjantcROXmFFITi
+tjxL4Wv/AAnLpcWoR+W+paZbavAB/wA8J13xk+5Fek5JNRb1ZyKLabXQyK9M/Zh8QDwv+0f8MdSZ
+/Lij1+1ikb0WVvJP6SV5nU+nam+h6nYanESsljcw3akdjHIrj/0Gsq9P21KdN/aTX3qxpRlyVIy7
+M/bL9qWylk8K6TdC6ltrWK7MN0I5vLEkTofkbsQWVeOa+cp2srhIn1O6FumCLeAys5RB6hRgE/09
+q+pvjzHB4g+D1xqMMZuYojbX8KogYsN64wD7NXy612unOwu7e7urx/mkVGSPyh2XBz+Xb8a/l2qr
+SP2Cg7wPzPopBzRX9UH4yfSf/BOzxH/wj37WvhaN5PLi1S0vdPb0JMJlQf8AfUQr9Df2oNDefW/D
+1+rhVaCWA5HUqysP5mvyd/Z+8Sr4O+PPw51p22RWmv2fmHOMRvKI3/8AHXav2d+Oui/2n4d0+cHa
+be7ALHsrqy/zxX5FxlRaxlOpH7Ubfc3/AJo+1yeXNhpR7M8A0mya4mhPmnacrIAOBgZr3L4F6AbK
+PV9SJYpKyW8ee+0Et+rY/A15jZ2a6Zp4QDewG5iByxr1bx/4vh+An7Puu+JboKJNE0mS62kf6y5K
+/In/AAKVlX8a+NjCc+WktZSaR7KtG8nsj8qP27viafih+074rmimMum6EU0Gz5yuIM+cR7GZ5fyF
+eA0rSz3Ekk11IZrqZ2lmlPJeRiWdvfLZNJX9DYWhHC0IUI7RSX3H5ziKjrVZVH1DrX2T/wAEuvhf
+/wAJb8cdV8YXERex8K2BELkcfa7nKL+IiWU/8CWvjXOOScD1r9g/+Cc/wx/4Vz+zPpOo3UHk6n4n
+lfXJ964YROAtuPp5KI3/AAM187xNjPqmXTSes/d+/f8AA9PKaPtcSpPaOp734u0m31sRwSsyNCCy
+SJ1Vj1+tZHi7W0+E3wU8Q63NKgbRtIur5n6BnSNnH5nArccm6mJ6mR+n414D/wAFIfF48Kfsp+Ib
+RH2za5dWukoc84eQO+P+2cTj8a/AcipyzDMFF7Skl8m/8j9CxslQoeiPx5iaR4leY7pn+eRj3c8s
+fzJp1H8qK/qo/I27u4ySGS6TyIQWmmIijA7sx2qPzIr60/4KM/DlPh38QvhxbwriIeDbTTSR03Wj
+GP8A9BdK8T/Zx8Jjx1+0F8OdDkj8yC5122eZPWKJvOf/AMdjNfb/APwVv8OGbwt8N/ECxjNrqF1p
+8kmO0sQdR+cB/Ovl8bi/Z5thKHdTv81p+KPZw9DmwVWo/I/Nuo54/Ogkj/vqV/MU/NLX1B4y0dz9
+ufhpqZ+I/wCyB4bvYyrzXvhOA/OSR5q24BJx/toa8++CPwxsPH9pq9xOlrPBbSxxC4ZC6vKVLSYJ
+643JnA61f/4J360nin9kHwxZudx09r3S5MHnCzyY/wDHXWvc/hl4Dh+G3hC10OK6N60TM8lx5YTz
+GY5ztBOOMDr2r+bcyoKljqtN7KUl+J+q4Ws/YRa3aR+AtFNjkEsauvRgCPxp1f0kflT0GvcyWO26
+iJE1uwmjI6hlO4fqBX753kqfEH4XWWoQSQxjUbK3v45J22xqGVZMk9hg1+BrKHUqeQRgiv2o/ZW1
+T/hYv7Gfgo3GLhpvDx06RSc5aIPBg/8AfuvznjOn+5o1l0bX3q/6H0+Rz96cPma+heBbW616yhk8
+QaVeMkgne0tZPMkdUOSOvAzjnFfPX/BVv4mjR/hv4Y8CW02LjX74312itz9mtsEAj0aV4z/2zNfQ
+H7Nvh6O106+1A2ghaLFnHJnO7oz/AKlR+FfmX+3x8Tf+Fm/tP+JzDKZdO8PBNBteeAYsmcj6zPIP
++AivmOGcN9bzGM5aqmub59Pxd/kermVX2OFdt5aHz1RRRX7YfBHSfDTwFc/FP4i+GfB1puE2uahD
+ZFwM+XGzfvX/AOAxh2/Cv3pNnbeH9BtdOsYltrS3iS1t4U6RxqNqqPYKAK/Mb/glj8MP+El+MWv+
+NrmLdaeGLH7NbMen2q5yMj/dhVx/20Ffpf4ovvsb2xaOSSNn8seWM4Y85OenAr8T47xsqlVYanry
+r8Zf8Cx99kGHUaftJdf0F01fMvYh2GW/Kvgv/grb4x+T4ceEo5DlpLrV54weMKqwxf8Aocv5V9r/
+AA48eaX4z1rX7fTPNlTSZEgkuiB5MrMCf3ZzzjGDnHNflt/wUh8YnxV+1Tq9kkolt9A0600xMHhX
+KmeT9ZgD9K8ngTBt42M5L4by/Cy/M7c/qunRlHvZfqfMNFFGcV/QR+aH1Z/wTL8JnxF+1Hb6iVDx
+aDo93ekkcB5NkC/jiV/yr7O/4KY+Gm179lbVrtE3No2pWOoZ9B5oiY/lKa8X/wCCR/hMbPiT4okj
+IZpbTSoX9lV5ZB/4/H+VfX/7UvhceN/2b/iRpKYd59Cu3jA5zJHGZF/Hcgr8izjF8vEFOfSm4L8m
+/wA2fc4KjbL2u6bPwuJ5NFMikE0Ucg6MoYfQin1+vHw7VnY/UD/gk14j+1/B7xloZbJ03XvPUeiz
+QRn+cbfrX1r4/wDi14X+Fv2D/hJdVh0z7d5n2fzjjfs278fTev51+f3/AAST8RNB44+JGg5+S60+
+z1AL7xySRsf/ACKteff8FRvHzePfj/a+F7a6dbLwnp6QuI2yPtM+JZfyTyB+FfkmKylY/P6tBu0W
+uZv/ALdX6s+1pYv2GXxqddvxPlK/0uTQtRvdLmUrNYXEtm4PYxuyH/0Goa9V/av8OL4U/aa+JunR
+qqRDW5rpFXoFnCz4/wDIleVV+p0Kvt6MKv8AMk/vVz4+tHkqSj2YV+sn/BLbxH/a/wCzVNprybpN
+H1y7ttn91H2TL+H701+TVfol/wAEj/EGF+J2gsWwsljqEY7Dcssbf+gJXzPFNL2mVzl/K0/xt+p6
+2Tz5cUl3TPsjxp4is/2e/gZ4k1+4dHTRLC5vskYEsxLMif8AAnZFH1FfhRNdXF9PNdXkpnvLiRp5
+5WOS8jks7H6sTX6g/wDBVn4iy6B8JPDfg22Z0k8S6kZrgg4Bt7UK5U/WR4T/AMBNfl5XBwfhPY4O
+WIa1m/wWn53OjOq3NVjSXRfmFISFBJ4A5NLXU/CrwG3xS+KHhLwcr+WNc1OGylfP3Yi2ZT+EYf8A
+GvupzjTg6ktlr9x89CDqSUV1P1f/AGB/h5F8H/2VdI1LUYzbXmtJL4jviy4ZUkUGIevECxcepNcH
+8PvjBDaeNNe1DxNa3t5peqpLOyLG7C3cvuH7o/LymEJA4AB9a+ifjjqEPhf4aLpNlGtvFeNHp8UU
+YwqQgZKjHQbE2/jXzTMEjjZyvCgnqelfzNmGKlisTOvPeTb+/wDyR+u4On7Kj7NbaL7j6E/Zghiu
+Ph/d64lkNOXVr+a4SAnIjjU+Wq5PUDa3PvX40fF3xgfiF8WvG3iYncura1d3UZ/6ZmUiP8NgSv2T
++IWtr8If2T/EOqxErLpfhmeaMp1EzQnaf++2Ffh5BH5MMcZ5KKFJ9SBX6ZwTh1GFav6Jfm/0PkeI
+a8qkoqT1ev6ElH44oqO4l8m3lkxnYhbHrxX6gfHJXdj9fP8Agmh4W/4Rj9lXTNQkUI2t6heamT32
+eZ5KH/vmEfhXvPgK4j8XeBJ4roeZFcSXUEgPdGduP++WrlfhVoMfwz/Zc8L6VEOdP8NW8bY6GVoV
+LH/vtia1PgdOf7G1G0/hhnV1HoGUf/Emv5rzLE+3zKpUX2nJ/jp+CP1PD0lDDKPax+Fmp6TJoGq3
++lSrtl0+6ms2U+scjJ/7LVevVf2rvDy+Fv2mfidp6KqRf25Ncoq9AswWb/2p+leVV/RmHq+2owq/
+zJP71c/Mq8eSrKPZn07/AME5fGsHgn9pMz3k4trC68P6gty5PCpEi3BJ+ghY18+eOvGl18RfG/iH
+xZeki61zUJ9Rcc/KJHLKv/AVKr+FZ+napeaRPJNZXMlrNJDLbtJEcMY5EKSLn0ZWZT7E1VrCGFjD
+FTxXWSS+6/53X3GssQ3RjR7Ns//Z
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="tizomeeegsi.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <f567501d0b859992e4e960155fa7b0@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAdwCd
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+5H/gpf4YbQP2qL6+KbY9b0izvVIHDMgaBv8A0UtfLFfoD/wVv8MeVrvw08SqM+dBe6ZI3+6Y5U/n
+JX5+1+/ZBW9vllCXZW+52/Q/O8zhyYqa76hX2N/wSx8SjSP2iNa0lmwusaBKFHYyQzRuP/HWevjo
+da9e/ZI+JenfCH9obwr4r1ac22k2iXkd1IOfke1lAH1L7APfFdWbUHiMBWpJXbi7eq1X4mOBqKli
+ISe1z0T/AIKP/E0fED9pa/0u3mEmn+FLSPSo9p489v3s5+uWRD/1zr5dq5reu33inXNT1zVHMmp6
+pdzX1056mWVy7/q2PwFU66MFhlg8NTw8fspL59X83qZ4qr7atKp3YV9kf8EufhgfFnxy1bxfcw77
+Hwrp5WFjyv2u53Iv5RLN/wB9LXxsTtGScAc5NfsF/wAE7vhqvww/Zh0zVb2Lyb/xJJJr1yWXDrE4
+AgB/7YojfVjXg8TYz6rl04p6z91fPf8AC6PRymj7XEKT2jqXv2jPGFve+MbXRxdIF02DdIhP/LWT
+B/MKF/76rzfSoY9a1aw0+KWN5Lu4jgChuu5wD+ma971LwvoWveIdWvbvRbK5ee1jnLyxAsWO7kn1
+wFH4Cq3gP4c+HYvFmh3cejW0dxBYi7WRFIIlBAD9evJr+dI4mNWpZJ62/E/T+V04eh55/wAFLPFo
+8J/srajpcLCN9e1C00lVzz5e/wA1wP8AgEJB+tfkSa/QX/grb4w83Vvhx4TjbHlR3erzLnrnbDF/
+7Wr8+a/ojhWh7HLIy/nbf6fofmOcVOfE8vZC1seC/DMvjTxt4c8OwqXk1fVLWwCjuJZlQ/oTWPXv
+f7B/hMeMP2sfAcLqWi06WfVZDjgeTCxT/wAiNHX0mLrfV8NUrfyxb+5HmYan7WtCHdn64fFeVNN8
+DG0gURxzSRW6ovQKDnA9sLXOfBa6KavqVsT8ssCyAZ7qxH/s1bPxa06+1caXbWlu0yRs8zkEAA4A
+HU+5rH+Hnh7U9H8T2888CxQNG8b/ALwE8jI4+oFfyvUrRWJSb10/r8T9chB+xeh+a/8AwUo8NnQf
+2rtWu/L2R6zpVlfKQMBiFaFj/wCQRXy9X3v/AMFbfDJg8Z/DjxGB8t1Y3mnO3vG8cij8pHr4Ir+l
+ciq+2yyhLsrfdp+h+W5lDkxU18wooor3TzD9XP8Agqd4X/tb9nTT9XSMtNouu20xYfwxyq8LfhmR
+Pyr8o8d6/cX9pD4e3vxR/Zh8ZeGpIT/a91ojSRxD5ybqJRKigjqTJGo49a/DeKQTRJIBjcAcHqOK
++E4PrqpgZUusZfg1f87n0ed03GtGfdfkPpKWivuz5wKKKKAOl+GXgG6+KnxG8MeDrPd52uahFZs6
+jPlxMcyv9FjDt+Ffu5rNna6D4LbTbGFba1gtRaQQpwI41TaAPYKMV+df/BK34NvrfjjxB8Tb6D/Q
+tFibSdMdhw11IoM7r/uRlU/7bMO1folrl/bT61Z6e7qzyIW8puhG4Bv8K/E+OMx566w9N6QsvnJq
+/wBysfoGRYblp88vta/cc6kYS7vOP+XSJf0bitDwBBFNcfaY5Uka3s47V1U52sTuIP5CsO58TWx0
+1biGAxXVzdvaxbhncqsoJb0HzHArqvAmnQ6NpWoyBVghe9uHBJ4CKxUc+gC1+X4Kk/a8z6W/9JPq
+a0lyW/rc/Jb/AIKI+Mh4v/at8SwpL5tvolra6TH6KVj82Qf99zMPwr5trofiP4tfx98RvFnieQ7m
+1jV7u+B/2XlYp/47trnq/q/A0Pq2FpUf5YpfO2p+QYqp7WtOfdhX3H/wSd8JnUfiz438SPFuj0vR
+4rJJCOFkuJd5x77YD+Br4br9Sf8AglF4TGmfBTxR4gZSsms668akjrHBEiD/AMfaSvE4mrexyuou
+srL73r+Fzvymnz4qLfTU+u9cfzL8jsqgf1/rVazPl3MLZ6OKsaoM30x9T/QVWAwPpzX8o1qn7+U/
+P9T9bppeyUfI+Uv+Crnhkaj8CvDmtquZdJ1+JSQOkc0UiH/x7y6/LPvX7Rft6+HR4o/ZI8fqI98t
+naRalHjqDDNHKT/3yrV+Lpr+n+EK3tMucf5ZP8bP9T8szuHLiFLugoopCa+4Pnj96fhZrSf2fB4f
+wSdP0uynjkyTvSWPPJ9QVP51+Sn7an7PN58APjLqQt7Rk8H+IJ5L/RblFPlpuO6W1z2aNicDuhU9
+jj9PfBHhnxzp3j3+17rT47aynjeCdDMmwRqD5KqoJOFIUD0BNdr40+G+ifGn4fN4f8d+HoLqzvUV
+riwkk3mCUdGjkXBVlOcOuCPxr8AyPN5ZViOdq8JaSX6rzR+kY3CRxlPkejWx+CFFff3xM/4JOavb
+300/w+8aWtzYsS0dh4kjZJYx2Xz4gQ/1KD+teVH/AIJl/HNbnyxZ+HCmceaNWO3648vP6V+v0s+y
+ytHmVZL10f4nxk8sxUHbkv6HytXW/Cn4VeJPjV480/wh4Vszc6rdnLysD5NnDnDzzH+FF/NjhRkk
+CvsTwB/wSc8TX11FL438b6dpVnwXttBhe4nb28yUKqn32t9K+8Pgv8AfBP7P3hptG8F6OtjHKQ91
+eSt5l1eOBgNLKeWxk4HCrk4Arx8y4qwmHpuOEfPPp2Xm319F953YTJ6k5KVbRfiS/Cr4Z6P8BPhT
+pHhDQV3Wmk2rKJZRh7qbBeSV/wDad9zH0zjoBXIx382par4OurmTzbiZyzOR1JnP/wCquy8TaV4n
+1y1v444La3BmEcAWf5jb4+bJxwWIXj04rA07wX4l0/SryL7FA91EQ1hIJlLQM52y7ScY+XkZ6HkV
++F4ipVxFZ1Kl23d3763f3n31JQpwsmc9dPs0rSD2/tK5bH0kSuj/AGl/Fi/C/wDZv+IOsRHbLa6P
+crA3/TaUFI//AB+RaZ4W8Ea3aXBt9R0yN7WN1uYDJKriOYEDdweQVJyO5UVo/tCfBK2/aD+Gd34L
+vdavdBsbyeGae4sVRpHWNw4Q7gRgsqn8K68rjThiITr6RvG/otzLFtyg4w31PwhhjEMKRjkIoWn1
++mn/AA6R8G/9FB8Sf+A9t/8AEUf8OkfB3/RQfEn/AH4tv/iK/dP9acr/AJ3/AOAs+AeT4pu9l95+
+ZTEKpJ6AZNftj+w74Sbwb+yr8O7OSLyp7nT/AO0ZcjBLXDtNk/hIPyrwCb/gkb4MlidD8QvEgDqV
+P7i27/8AAK+4/Dmh23hjw/pmj2eRaadaxWkIbrsjQKv6AV8hxLnWFzGhTo4WTdnd6NdLLf1PZyzA
+VcLOU6vVGfqIzey/X+gqFVzV68hZ7uQhScnsKYtq56Ix/CvwGpGTqSsurPuozSgjO8e+Gv8AhOvh
+d4k8PFVc6rpN1YAN0zJEyD9WFfgJHFLboIZ1KXEWYpUbqrrwwPuCK/ob09XjVkdGVTyCRXwt+0b/
+AMEypfiJ8QNV8WeAPEtjobavO13eaRqsDtAlwxzJJFImSodvmKFTgliCAcD9s4Mzmhg6cqWKfKpW
+1ae60f3/AKHxmcYKeItKlq0fmhnFRuLp222tjc3rAZcW8ZcoD0zgHGcH8j6V9uad/wAEn/iXNcoL
+/wAZeFbO3z80lutzO4HspRAf++q+2v2a/wBknwj+zd4Uu9OsmbxBq+oyLLqOsahAu+coCEVEGRGi
+7mwoJOWYkkmvvsdxRgMNTvRl7SXZX/F2PCw2U1pyvUVkdhq/xZ8H+H9RNhqnxA0yyvA2Gt5LiBJF
+PoQeQfrW/DewajBHPa3moXtvIA6T2pRo3X1VlGCOOxr85PD37PXh8+G4312DV7rXrnl5bSVUityQ
+D91lJkIzyWIyc10n7POqfEr4EeLNSsrCzN94TmZo5YbqTbb7/wCC4ij3ZV+MMF4OeegI/n6OKrqU
+faU7J9ne3rofv+KyDI5YavLLsx5qtFXanHkU0t/Zty1fZat6W01PvVPGek6dfRaXNfImoyFRHaXV
+3Ctwxb7oCFwxJ7cVsi9n4/0Gce25P/iq/OPxvq194s/bT8DapeRW9pf3d3pDhYkJWLk4yCeSMZ61
++iEWnauoYyawpPYLaqAP1rehX9tKaatyux4Wd5JDKMPgq0Z83t6aqPS1r9PMludQkFuzS2d3AijL
+SB4xtHrndxWLpPiDSdcEi6XqtxqYTG9rK7juPLyOM7CcZwcZrN+Jtv4is/Amv3NrqttP5Wn3LyW1
+zaACQeU3AYN8p/A18Q/sH+JtX8Jr40bRLOwmaW2tZJPtknllmQSbVU5Gc7j+VKrifZVoUmtJX/Ar
+L8jjjsoxmZ87ToOCUbfFzytvfS2+x+gKsUUJ5+rKRzuaENn/AMdNUY/FmkXGpnTYPE8bamhYPp4e
+J7gEckGMDdwOTx715rF8aPHTKS3hjTZFAyTHd42/Xk18jfD/AMTajYftt6vrsGkRz6q17qBNgsgA
+BaEggP7DJ/GlWxapuCir8zS9C8lyD+1aWMqTm4+wpSqLS9+XpurX76n6F3etQabay3V3rv2O1jXc
+811AIkjA7sxAAHuag0zxZY6xAZNO8UafqMSuUaa2VZUDddpKtgHBHFeC/Hf4ta/r3wQ8b2N54Pls
+4JtMlje7FwGSPOBnGOcZ6V5f+xj8U2+GHwi8RyHw/cajYLqk13Nc27hVjCwR7gwx1AQenWiWKSrq
+l0avceGyD6xklbNVN88KkYKNr35ra3v57WPs7VPFVpodn9s1DxDpVhaA4M99iFM+m5nAqhoXxP0H
+xPd/ZtH8XeGtWuD0hs7xJJP++Vcmvznjutb/AGsviRf+IPFk2pf8I9aHcbfTYxMbOJifLt4Fb5A5
+AyzkZOCcE4FdD8VP2a/DWgaPc3vgy18V2Gt6aouGg1ZY5UlQKHJSVArRuAdw6jjHFcix1WadSnTv
+HzerPq6nCOUYCtDLsyx7hiZWuow5oQctlKV9d1role7stT9E/wC1bpUJNxphIBJPmkAfX0rKsfiL
+pN/cw29vr3h67uZm2xwW+pozufRQM5NfNX7Mv7Uj+NPh7d6J4qtbvWvEGnqYFvoIBI11A0ZMbSnj
+5xgqT3wD1Jr4X8L3UnhO80rxRaxRiXSdRguAQv8AEjeaPzCsMd6K2YqEYTgrqX4WLyfw+qY7E43B
+Yyr7OpQcUrK6k5puLu7aOyt117n7IX+tzafaz3VzcaZZW0K73luLkhUX1Y4AFZNh48sdUn+z2nib
+w7c3G0v5VtdCZ9o6nAfPGa+ff24/iHaW/wCz/a2thYtCnime3SOUog3W4AuG6EkZCKPxr5p/ZL1+
+3+GHxuuL65sDd+RpdzC0UACtuYxHPI9iPxrSpjOTERoxjdO2vqeZlvB8cZkFfOa1VwlDncY23ULX
+d76e87bM/RO6+IeiadezWt/428P2dzEQHt5Z4kdDjPKtJkVEvxP8Lb8nx/oBGPurdwY/9Dr85viJ
+FpPxa/aq1RtTtLq203Vb1BLFbFPtCKtso+VmBGSUB5HQn611Lfs9fDmTAis/FXzNgGS4t+/r+6qY
+4uvUcuSmmk2t+3yOnE8M5Fl1HDSx+OnGdanCpZU+ZJSXfm73R+g8PjDSbnS21KLWoJ9LAJOoQ7Wt
+xg4OZOVGCMdetY8nj7wu+G/4S6Fs/wDPPUrYD/0KvkbxF4kj8A/szeKfh5pOj3D6PHazgXl5KGlU
+yS7iSFUL1JA+leFfCj4O6L480C5v72O+WSK5aAGyaJVIAU87lPPzVc8VXjONKMFzNX3/AOAcmB4d
+ymvgMRmmJxcoUadTkTVO7kmk02ubS99uh9brb6YWtZoU0ppZZFljDX8jLsIVoyOOc5/lXPHxz4On
+8R6j4blvtHi8TQXJie1eGdlc4LOqN0YjI9OhrzPwfo37RfhDQxoPhi1g1LSYWaG1vo4ba5Nv/wBc
+pZBvjHOQGHy+1a/wZ+BWr+BNVvPEPiOHUJfFBaSOMWjJIsBfPmvJI/3pH56cAZ5545PbVZSilBrv
+dfkehVyHKcBhq9avj4VbL91GlJOUm9nNWfKu69dbpc3DfGPw3L4w/aX8P+H4b1NOuNWh0yzS9t42
+AgLhgHVcg8emR9a9d1L/AIJ9eM7O0km0n4qT3N8gLRRXCXEKsw6DeszFfrg1k/Er9nv4reKvijpP
+jnwTp0dx9ktbN7S+vb6BZEuIQQSyMcEhuORg4rY1Tw1+2PrdhNY3Oo2sEE6lHe1ubGCQA9cOq7l+
+orl9lDnqOrSlJtuzSZ97TzTEvLsvp5bmWHoqFKKnGpKLfMkr6OLei0tdFX9l343eKPFvgn4l+AvE
+19PqlzpWhXV3Z3l6fOmiVQ0UsLt1kAYqVJ55IzjGPPf2Lf7OFv4y/tFLVs2lt5H2m2aXEvzbT8vQ
+DnP4V738DP2Vr/4L/DXx1qGtXcNx4s1rSpbOOOxmxHaxbWKxiRgNzM5BZjxwB6k537D3wp8W/CS1
+8VS+J/DTrDqYtUtTBNDPkxeYsgIDHHLD61dKjWc6Kne6T+S6XPJzbNMmeDzpYCcUqjocqTS55Jrn
+cF1XV2XnsdRYw2DrcLCdAYGMedi3lT5dw9/72K+OdV8FR/ED9o7xJ4eTU4NEin1C6YXsYYxpsTdg
+c5wcYr9D/wDhd3g9JpYZLKaNlZo3HkRnBBwR97nkV8teCfDNxo/7XeqfEG/s4D4QuLq9lRhLE77Z
+YtseYs5647cV0YrDyk4LdX1PmODc6pZasfXdRU5+xlyXa1lo0kno3dbWd+x534y/Zcn8JeBdY8Rr
+44h1G00+J3+zKWAnZQCUA3dcMO1X/gfp41L9nzx7DCI2vpWulUNetCwxbqRiPo+QD1619W/GjxN4
+b8c/CHxX4e0GyU6tqFhLFaRtbLGplYYB3dB0HPtXnn7I2iab8MfAOraZ45062TUptVa5hUILrMJi
+jUfMoIHIfg1l9T5K6jBaNPXWx7D4uq5hw9Vnj6ylWjWg4x91ScVyv4UldXvd2PHf2PLuC40XxVZq
+pe9Wa3vAv2/7MTEUK5x/Fhhg+m4etfQXir7NoWla5efabuTTLOxlle5/tZGG3ycnIPbJ2478V4Z8
+TfgFP4T8by+K/g/rsAgaRpU0mUmCa0LHLxoXXZLETn5GwQMDnGa5/VvCfxe+MEttpHiObSPDelh1
+3+YsdpA5B4ZkhDNK3oOmcdOozh7ajD2Tpttdtn8zozPB5TxBmP8AbFPMKVOjU5XOM5WqRskmlHq9
+NPPa6SbtfsUaNc39/wCLZhLeWFobGK3kurSVI/KfEj/Nu68Dt615/wDDHwlJ4w+DnxT8uOQyaVb2
+mrKqAeWfJZ94bPOdjPjHpzX3P4I+Fnw1+DvwyOjmZNTvgrm41G5hdZLq6dSM7RwBn5VXoAByTknx
+X9jD4U6loOr+KdI8U2ltbrrentbIBIJZGjKOsgBU4AwwznrxjvUvCTh7OlJdH8rnpf62YSvPNczo
+zS9/DuCbs5KnJXst3dK77J6njms+N5vjXafA/wAFeZLcyaPB9hu4x13+cACCeD/o8K8n+8a3Ph7a
+xXv7UnjWP7PJboragUtBKsbJhkAUseOB6V1vwZ/ZV8ZfCzxPfeIfGml2Nvpem6fLFaSNcrcLNO7L
+GhKIdwAQu3bsKr+D/h/rmhfH/wAVeINZs7RdPu2vBFMF8+Mu7KVxEDuAwD16d6ypU6l4Smne6+5K
+x6Wb5tlbjjcHgq0XSVCfLaSac6lRzko93totUeZ+L/CCeMf2kr/w39u/shL29EX2suJfJIgDZ3Ag
+HO3HUda63/hkqdo45G8bKIWbZvxkDpu6PzjNYXjzRfE1t8ZtU8VeGfsTtHdCazugI1jyIghPlPkY
++8OaqX+v/F3UuLrVN6AlhGskKIpPUgKBiqjSXNN1KcndvZP/ADQq2bYueEwccqzWhRjGjTjKM5wv
+zpa7xk1pZWutU9D13xr4ci8Kfs9eJdGSbTr5LDS1ge981/NmxL8jhOik5PFeLfCvwx4+1zQbmfwp
+4htdI09blkkhnuxCxk2rlsFG7Ec5/Cuhj1jxhd/DjW9G1HUZpLvUI3hkty6GORNylQTjA6HvXUfs
+/wDimH4b+FL7TNaudSsJ5b1rhBp4iljZSqjOTyDlf5V0Tw8qlSFk0uX0t5HyuFzunlOTY6MqlGtX
+de9nacZ3UbzjHS6vezSSR9jfs2FX+HcrqpTdqE7bSu0jhe1eZa4EN5qxb+y2t21GQszvIzeZ83Xj
+0zVL/gmjdXF3+zrfSXN1cXjjxHqCLJdStK4UFABliTivm3xLqXiyTUPEFzB4i8bxwHXrmBLS3tIl
+wNzsHUs3KgfKDXq4majy3/rY/Hsvg5ptPt+p95/DLxRcQ+HLLTrXw7fSWtvETHeRKqW8wLn/AFeT
+nv3rszrup/Lt0Cc59ZkGK4/9mm8uNR+BXg+6ujeNcyWe5/7Qx5+d7ffxxn6V6LJI/wBthQHEZViR
+6mupSSijikvfZxPjjxFc/wBhXNpf6DcQ2dzbyCS6fbNDDgceYqnOCSOlXvhOEHgfT9nkBMyEfZkK
+R/6xuinkVn/H+ae1+DXi6W1F+bgWLhP7KKLdAkgZjL/KGHbNYv7LLXb/AAQ0A30upzXJa43PrLI1
+3/r3x5hT5ScY6dsVlzfv+Xy/U35P9m5v71vwPmu68P3o1XVD9mDF7ucqQfWRqltPDGoW1vIxtlLE
+HjjA96+UNY1nxHdeINbMepamsf8AaN3gi8kAA898Y+apz4y11rFNPk1HUzIW3B/tcgLY7Dmo9o09
+EefI+wvhn4Sv9S8U21rJEIgYbhxI4yMiM8frXNHw1fyqgWB9q9854Neffsq65rNz8Y9KS81DUGSK
+w1L9zPO7AkWrEcE8kda8x0rV9f1/TY5odbvzOsQZv9KdUAA6daPa7WXcUnZXPpTUNLvkhFtFaT5X
+AeQL146A1tfDTw1qE+uSyyxyRpbCKc+bnDASoMD35/Svj221XxBeOsn9o6oYVcBsXkg4zz/FXtHw
+Pn1BPEOuRtqt5Kg05CEa4dgv+mW2OSeeMj86v2ybStqEN0foP8Qprm10ATWlpJqFxHcRMttGwUv8
+3TJ/zxXl3gjV30LxBHf6wb+x0+GGQy3F7PG0QJAABwN2enSui/aUGsSfDy3i8PXlpYa3JqdqtpPf
+RPLCr7sncqc/dDV8tfsm3lzqHxw0iC71TRL+3NjdkQ21ndxythVwf3p2YHfP4Vy16nLXjG3Y9/DU
+lPCzn2v08j6z+JPivTL3w1NY2l68t9cf6mKDKSPggttZhgYHP0zXhHxB1PU9EsrzT7f+0JXvBsNy
+boMREGBJAXux4+ma9Q/bCn+w/AnWpVuILMhov388cjIn7xeSI/m/Kvya+NHiG71PxP4gl0HxOI7W
+2b/R4o57q0CQIgPRjncST168UnUi6trdiaNJvD8y8z65mhuQFXyLlFA6PDVGWCc/8sTg9whFfm3c
++N/Eu3MfiLWcZyvlaxKdw/FqyL7x54pVGH/CS68MkY36hLjH/fVd9zjcT9MJba4lY/uJdvYBTUYs
+Lg/8sZR9FNfmOPHXigj/AJGfWv8AwYzf/FUo8c+KP+hn1r/wYy//ABVO6FyM/dj/AIJ9aJe6D8Ap
+7fUbae0u217UJHiuIWicZl7qwBHSvi7xG2gzanr0s0Ph9dROuXSGK58T3AHkb2O8gfdbfkY7Cvtr
+9gbXdZ8S/s9x6lr+p3msapPrOo+bd30m+VsXDKMn6CvjrU7qU3eowQ2upC9bWLib7VH4XiP7suVE
+YJHzfNls9815mN+x8/0PSyeyU/l+p+gP7LjtJ8APBJdbdG+wD5bSQyRD5m+6x5I9zXp2P9KX/cP9
+K8Z/Zw+IHhu2+EHhfSrrxLpx1ezsB9stp54oriI7yD5kQP7s54xgYNejSfEPwtFdqH8RaaJCNoX7
+Unf8a9JJ8sfkeRP45fM5z9pGbyfgZ4ycpYSKLBt66nK0VsVyN29l5AxnpWN+yQYH/Z+8KPbLYrAy
+TlBpkzzW2PtEn+rd/mYfXvmj48+PtDvvhn4g0nS/EVs+uXNk8ltaadJDc3cqIw3mOFid4xkHjHNX
+/wBmPzT8DfC5n+0+a0czN9st1t5QTPJw0afKp9hXMv8Ae3/h/U7H/uP/AG/+h+euv/BTx1PfXz2n
+w68X+a13PIJX012Ukyscr/skH9ap3Xwj+Ic1pH9s+Hvireo2qE0qXAx0PA4rlo/iL48h1O/EXxG8
+bRoLyfaq+IbnAHmtgAbunYYq4fi58QY3Oz4l+Od3bfr85A9sE0LkZ46tfQ9h/Z08HeMNA+LWk3Gu
++H9Z061XT9SiEt7YyxRmR7YqihmUAsx4A6k8CvNNL8BeLdPgi0+Lwn4ktBKi+dIdKn2px0zsr0j9
+nTxf4z1/4n2EOteOvEWvaaNJ1K7NnqeovcxiWOENFIFYcOjfMp7EV5tb/FP4nPppuE+K3jWO5fay
+QyaipULj3U81DcbLV9SZ6PUiv/htrikpBoOtIiA5uprCcA/QbK7b9mjQNU03xD4oa/0+/tUNhAqS
+XdrJEpP2y36FgB+Fczpvx9+ImmWsqXvxQ8Xy3TH5M3MTKo+pjr0H9nb4n+O/Ger+J7TxV401jxLp
+q21tJBaai8RSNxeQYYbI1OfxrWEY6WYQa5tD7R/aMtE1D4exWkuoXWkRT6jbI+oWd0lu9uN2d29+
+McAfjXz1+zFDcf8AC5NLeSfXnU2dySl7qdrcQg7R1WP5s+mOPWvcf2sLvw9ZfCkTeK4Ibrw8upW3
+2uGe1e4DruO3CIQc7tvPsa+Zf2RZfDx+PGmPYjw8t4NOuyWsdHubWbbtXOHkO0D19RXn4r/eofL8
+z63Bp/UKjt3/ACPpX9r0SH4JasIheM++PAsJUinPzj7rP8o/Htmvy/8AiJ9pvPFmvXVzZ6xYSm7k
+kks5Xt7ggkrmM+uPav02/bDu7U/A/VRJJZPGZIs/a4muIv8AWL96NDubnHTpwelflz8SWt4fG2vL
+q40T7ct5KEa2t57ZTNlN2BnpWNS/tH8isJb6svVnzH4s0s6Vrd1aMGGxy8RuLXYfLOSOnTvXLXyq
+AoXbjccBcjj8a9N+KFjbpPZ3tuYsSKYZ/JvGfEgyQPmGQMcV5rfjO0ktksTtLA7a9alLngmeZVhy
+TcSoq5FOC8CnouV/ClVSRWhKR+9n/BPJGj/Zg0oNnd/aepEknJP+lyc18gXnhXxRcRXVuPD1y9s2
+rXFyszeI3U7mO0ggdgoBAxxRRXFi/s/12Ly2ThCVvL9T7u/Z1+E/g/Sfhj4c1GDwnpNrqt3p0X2u
+6+zpJPORzmSUjc5J5JPU16l/wjOibwn9kWGSCf8Aj2T/AAoor0U7RSR50nzSbZ5f+0n8N9C1X4T6
+5dweG9On1m1tttjOgFrNGTIpKpOi7owSOcde9aP7Lunz6X8BvCVtc232SeOCXfD9pa42nzpDjzGw
+W+poorkX+9P0Oxyf1RL+9+h+UdtdI2rXRkJgxcy/MuWx+8Y5xW1FpEuuTt9kfzWVcuX+UH3xRRSi
+tbHjz2uenfsrsg+LT267xL/wj+r5LHgfuAMCvKJ9EaOxhntWL20EUaSFjht2Bk/nRRUS05fmRLZM
+fHBDLAkBUhZVLJIeWLf0r2H9lxUl1LWZB982lurHsT9ut6KK0g7yRdPc+3f2odZvPDnw1g1Sx0+5
+1We11S2f7FbXEcBlBYqQWf5cDOce1fKP7PvhI/Eb4j2/hvxVpniSfw9c2Nw1xBf6tCYZGXYVz5DC
+Tr6HHrRRXLiEniY38j6nCzccFO3me2ftXfCDwzovww/tfRtDmh1yxlVbS4sLoxzoHKq+Gdtpyo53
+Z4zjmvzr+In9paF4x12zuJNauHN5LAJ7qe3lbIK/OcnqfaiisqutV38isK2sOvVni/xFvUvfDd0J
+LiTbbOhIlt48s6lsnK+xrx27kR2wCpcH5iExmiiu3DfAzmxHxr0GR8ginJ0ooroMUf/Z
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="p.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <d86de01d0b85929296f07066112547@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgBJwCd
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++vP+CeuR+ytoEnzYe81B13HJx9rlxXxBeCwk0oiRfCLaWdUuHV38Qz58448zp/s7TjsTX3v+xRpV
+hoX7NGgWum67ZeJbFJb8xarpyOsFwv2qX5lD8/nXwzJa6tFokNwdT1V7Rr2YJap4UTKsCpZ8ejAg
+A99prxsbdOK/roe7lC9yV/L9T9I/2f4be3+CngiO0MZtRpNv5RiYshTYMbSeSMdz1ruycXC/7p5r
+ivgWyt8GvBLAMobSLYhXTy2H7scFR90j07V2pQ+erfw7SK9LpH5Hiy+Jnl37Uy2L/AbxWmpNYrat
+DGGOpTtBb581Nu915UbgOntUf7LAt4v2e/B4tfsQtxayFP7Nnaa3x5r/AHHblh7mp/2nn2/BDxMm
+LjdJHGita2i3UikyKAwiPDY689ue1J+zQrQ/AHwmJZJWIs3LSXNsLZ/9Y/LRLwh9hXMv97l/hO5/
+7iv8X6H5WQQ6fcalfcPEGnk8nPf5z/Wui8M3S6eLmMqyHb+8lbgcdlrsNG8P/CKxvJm/4X94VuC8
+rkpJpFyNvzHgc8df0qS80L4UXdwWi+P3hVFBxsOnXOM/nUxhLex4T5mrXLX7Mbmf4r3x8poEtvDW
+sSrjnOYVGc+vNedaXfwad4b05lDzXc0andu+QLgZJHevoH4G+HvBWl+L9buND+K+heML0+HLy3ls
+7G1kia2hfYslyzMceXGOWHU9smvL9D8CfD2fR4tOsfj54MlDouZI9PuNzLgY5z0rSVN2WhTVo2PP
+2vLZb6WZZFWJ+mOieuBXt37Mkdq2rauloxZEtLZSWBGf9Og5xSeGP2SYvHtq9z4Q+IGl+JkjPls1
+lpdwYvM9DIxCAfifoa+gvgb+xrq/w9e/vta8VwteX8cSPBYWoKRBJFkAVn77kXPy1zxmoy9xOTXb
+/PRX8rnXTw89JTtFef8Alq/nY9X/AGjLrULLwPY3GkWiX+qR6pAbe1kvvsglYhwRv7/KWO3vivnP
+9lS0uIfjNp5dcR/YLn5h4pa/yMJjMJPP+92r6z1L4aafr5j/ALbur/WUjlE6RXFwUjWQZwwWMKAR
+k1m6X8BPAGh3CT6d4WsrC4RSiz2u6OQA9QGUg4OPWsqtHE1aqqxikl0ctfwi/wAz3aOIw1HDyoyk
+23fVLT8ZL8jjP2x42l+B+pBSQ3nRc/bPsf8AGOs38H9enevzC8X2AtfEmux6fczXlobiRBcx6+Js
+xgrh9xPJOOtfrp4o+Cvhfxhosukapb3lzpspBktXv52jYg5GVZyODg/hXgvj7/gnR4K8QLcy+HtT
+udAuJgd0csEdxCx46qQCBx2NY1aeJU3N07+jT/PlNcNWwsaSp89nfqmvyufmHq1nfX9tLBs1B1ki
+MIiF3FL5ikNhvevnnUIpIgqSiQOjFWLgckfT8a/QX4s/sG/E34aLc6jBpuk+JtOgGEutKtGMqRgH
+O6MHOee1fBnjeyGm+JtStPLEBjuGKwmJ4nQHB5VsEcmtcHWjOUqaeu9no/udmGLpWUai1Xdar71o
+YkQ4NOUcUkQ605TgV6LOFH78/sSr9l/ZB8GvjDm0u5COnP2iavhCM2iabaXKW6y3jXMnmW58VsMI
+CpjxzxuJb8q+6f2KWb/hi/wUzkljpt2xLDP/AC8TnpXwU02kNo1nFE2kx6ikkjSzHwlNh8lfKA54
+2kNn61xYy3NG5vlSapyt5d+z7H3t8BfDnxlg8E6c+peKfCx0aWwtm0q0s7GR5LaLZnZLKT+8YKVG
+4dSCe9emDRviPkbvEWjY74sGP9a0fhJbtbfC7wfG5UuukWgYomxSfJXJC9hnt2rrmHFemtrHjSd5
+NnzV+0vpnxD0/wAEahdXWs6RrXhD7EItR05lbTrgytKu2WO4DEKqjHynrXe/s3ny/wBnjwmwjESj
+TmbYtz9pA+Zz/rT9/wCveq37WIc/ArxBiS2jTMBlN3YteR7PNXIMQ5b+nXtVn4ABIf2c/CxjaFox
+pJYNBbG2jx8x+WI8oP8AZ7VyR/3qXod0n/sUdPtfoflXp+q21rCxI8yZpHyp6Bdx/Wli09LmN5oJ
+WyMsYyc4H1rFsB5sJAjyWYktjpz+lbelw3s17bWdjay3l7duIILWIEvM7HAQAdzUN8sd7WPGiuaS
+SWrPcf2UJLR/HPiXy42jlk8L38Aj3Z3u2xVVR/EzEgADk17v+z7+xTDpFrY678QiJLvykMXh+B8Q
+QYAx5zD/AFjeqj5R713f7MX7MVh8FdFTWtcjhv8AxteIGnnA3R2an/ljD9P4n6k+1ereJvFVvo9u
+JJmYmRvLihiG6SVz0VRkc8jk4AHUilToPEWdRe70Xf1/y7b72XfeOHWj95bvovTz8/u7vchksNIt
+I7W1iitreFdscMCBEQegA4FVrrxRbwoSP515pqHiPWby5SG2tRJLvPnQWg81YVAOVknI2h+gG0Y6
+81Ej61G0jX2nB0ZBt3am52nJyOCOfpivVVNQSSOX2nM3udLqfxe0TTr2e0n1qxtbuAgSwTXCq8ZI
+yAQTwcc4pIfi1o5hE51zTfIwG8z7ZHjB6H71fm1+0LPaTT6xDd6VNplwmrsk00kjOWbjG7cScYwS
+QTwa8MudM0+0LzxvGjqCzxiYfI2D8w9VI7D61oqbavc5HirO1j9x9L1+LUEV45opkbGGikVxyMjo
+TWx94dODX5tf8Erb2Jf+FjXKy5icWK/LKzA5Mhzgk4P+FfovFrNqyL8/as5LldjrhLnjcvYx04r5
+1/aZ/YZ+HP7SemzS32nR6L4lVD9n1qwQRyq2ON+OGH1r6EjvYZfuvmpjyOD1rmrUKddJTW2z6r0e
+6OqlXqUHeD9ez9Vsz+eH9ov9lzxr+zH4sfS/Etm02mStiy1iBT5FwOwP91uOnft0xXkA6DGT9Oa/
+pK+LHwk8N/GfwXfeGvFFhFf2F0hXMigtGT/Ep7Hp+X0r8Mf2pP2SPFf7OHxJm0OGxvtZ0W53T6de
+WsRlZos/dfHcZHPf6g15yqyw01SxL0e0tr+T6KXXs1dq2x7EYxxUOegrNbx3t5rq1+K633P2K/Zc
+0a78JfsleGNM1Bo1u7XS7lS0UgwR5kpU56AlSCfQk+lfCUg8TN4W02zksfGUUcRnYXf9tW+ZQ+3d
+k552Y/U1+gGhhdJ8MeOvDjhRHo+o3ZhjcZVbeeE3CDHoPMcY9Fr84LiXSpPCmlLdP4KOmulwbbNt
+OgHK+cT6c459q85YiWIhBzXvJWfqtJfimejQw8MNOpCn8N01/havHr2aP1g+Gkf2f4d+FYizuV0q
+1UtI25j+5Xknua6Yng14l8Hvjno+ueCtJtrPw54riTTrG1t2kn0G4gik/dDDRFwN6ccMueMV33/C
+yrDODpmuA++lzf8AxNfTJN2aPkJaSaZx/wC1XBdz/BPWfsMupxXaSQPGNHuEguXIlX5VdyFGec57
+ZrY+D9pcTfAjw1bH7Q142jJGRfTLJN5hjxiR14LZPJHfNeSftY/E/Q9Y+HN/4dv9JvLJ5RBdR3ni
+LSriPSwyyjajzKMByVOEzk4r079mExH4D+ETB9nELWzFfsYIhx5jf6vPO30zzXHH/e5LyPSlpgY3
+/mf5HxP4f/Yg+I2mWEyz/wDCOvetlFj/ALV+VPUn5Oa9w/ZA/Zal8B67qXi7xQbO+1a3d7PTPssn
+nQxAcSSqxAyxOUz22t61xPgr9oL4yfEL4oL4VsPGXgRhPrMlv5Flo9y11HZxyuJX3MdnmKijk8Z9
+e/3JpWmW+i6bb2Nomy2t4xHGvfAGOT3J7n1NYw5MRPlg7xjq/Xov1fy6NmU8M8HHmmrSlt6dX+i+
+fVIzPEmrQ6ZaTzXEwht7eNpZZD/Cigkn8q800dJ/GyNPdRzWcl2g8+MjElrDuzHbrkcFsB3bHP5V
+qfFK8SeCz0+UsYNR1GC2nCqWPkBt8hwOTwgB9jW/4QgW7kNzsCmd2lbbg98Dnv8AKB+Ve0vdjc8t
++/Pl6JamjBokNhaIBGIolHyxoMKPoK8l+L/xz8C/DDSrm413WVhaGaO2a0s4nublpXOERYkBJJJx
+7d69u11jDZhlHKmvz4/aY17WPhxq6+FWvXl8P+KrmS6soBcFpZbuW6V33QkEJFDgHzQQWaRFXlSR
+k9EaSbSOV/am+JGleJkvILS5vbWJorW0jnezj3wTLcO90qKxwZRGUGTxkFcg5FbWlftEfAi+i0/T
+7fTIfEGvtEkD6bB4bBnMqxjcpBXGSQwwM8mrPwY/Yn8B/tI/Cbwv4jh1S30a+ht30/WrCztxOHvo
+p5VeeQM/Eki7GORzwe9emeHv+CaPg3wTLe6tbazdz6hbWryaZJbQfZZLW7AJjm3K3OD/AA8daFDl
+vd/gZObl8MfxPNNc/aRg+E3iCXTvDXgnSdCsrrQrfxC0TW4tLq4ifKnKKAG8tgy8ehx1qvL+3d4i
+iiV49L0xgXCjcWAAPfO6vDfijNeeOPivo+l3up/a9dj0qytbV7h8skC2+7yiOyH5uf7xyckk1xEc
+yKLWSKyvZIgrOjJGMSK3T3BHuOv519HRyyE4xlKpbmSa0b9Vo911Pm6uaVYykoQejaeqXzV+j6H3
+b8Gf24J/F3xG0Lwlq+madZvqb7Fu1umjaM4YqNpJBJYBQM5yRxX2fpWrJdRAHg988YP0r8ONN0tr
+3UbRoo7yC5WcSFxGFVo1IOWbPbaGB4wVHNfoD+y78WfGOlfEa3+Hfi/WI/EdrfaU+qaTqjEm4TY2
+Wikc8uNpOCckFepB4zxGVygpzpyTUdeq076/kdWFzRTlCnVi05adHr20/M+2uorA8T+AvD/jNrdt
+c0i11RrcMImuEyUBxnH1wPyratm8yBG9RUtfOVKVOtFwqxUk+jV0fTUqtShNVKUnGS6p2f4HxX+1
+t8S/G/wv+KDQ+EojJY6/axrqnl6et1IkaRSKrpuYBTyRk5Hr0r5evLfV7Lw7YzNeeMJ4Winb7MNL
+tnMQRsEN6b8Z/wAa+mv25n2eOtNeK00+7lXSn8xb+/a0VYyJQ7Bl6sFJ4/GvlS6ttJj0mzlsrPRp
+NQaKZp4Y/FUoCsrHygAeuVwcV8pSdp1IrZTl+Lv+bPs/Z3pU5vrCP4RsvwR+s3w/Zn8CeG2YuWbT
+bYkyDa2fKXqOxroOpIrD8Dgp4L8PgrtI0+3GAcgfu14z3+tbY+8a+oT95I+JfU8U/bBZx8ENQjjG
+oGSa7tY1bTLNbqZD5qnIjbgjjk9s18b+E/2ofjr4M0VtF0XwzcS+H9FnitYprvwy5uJopJSA5xMF
+JAyTgYr7I/bAmaL4J33+hx30b3lsskcmpHTwq+YPm84cjBA479K+IrXTdDutE1W5vbLT7bVYHt1s
+IJPG8rLMrMwmZj22AIQO+6vIxNeVHENx6o+lwOFjiMKubpLy8j6P/Yy8PXmq+MPFXiLUJ9XnFudk
+Mes2MNq0cszM7tGI8nbtwPmPfv1r63cYRvpXg37FuiwaZ8IXuYYIYDe6jPIRBfNeoQuEBWZuWHy5
+9OcV72RkGuvLY2w/N1k2/wAbL8Ejzs0lfFSito2X+f4tnkHje9jsfFvheSd/LTzr4qxHG/7MAo9s
+7sfjXY+A0VLG0CgKBbxgKvAHyDpXKeOjHZ694dvJnEMEV9LBJI5woE1u6jP4jFdL8O5Q+nWZACqY
+VUAdOPlwP++a9d/Cjw4352mdRrKeZYSCvzh/4KGQ/wBk+PvhV4gljItrQT20jj1F1FLj8iTX6SXr
+IIWDnAx3r5l/ai+FGl/FbwpFbT2E+qX2mXA1CysraZYZLqQcG3DtwokHyE9s8c4rMuavFnzF4f8A
+2PPi58W7TSvFek+JNF+G+nNYQQWNlaG6FxcpGWCXFysWAJHB3HJPUdK6+H9gj4sR3Vrqmp/Gpw1i
+plcW6XpEqr8xB3Sgc4IPsa8+k+Mvxj+JGu6fdtf674b8LwtPHPNo9vFNY2LINpiSOJxK5BUJiRsg
+nPArzFPjX8SJPGk/h3xB4k1uLSR56XV5Id7wxBCyyCNZgGONp27uMng4wfSeGaT5qiv2Wv47Hkqv
+FvSm7d3ZfgY/xjhutZ+Puu+Jp5YbOW6s7QstkjbrfZByEA7YI/rXK6P4kuGtiI1vNkgBlQwErJxx
+u9fr1rvbDVdI8W/GW0ZLjUYvDmrXENlcyNIba5ltmRY5NxTO3dgnA7YFfUfhL9k/4Ja1FJ/Zlne3
+4hIVimrzjaSMgcgc4Hb0r6LFOOXUqNOUXZxTenVpX/FHgYSMsxqVqikrqVl6K9vzPjG61e9vrFrE
+ebb285VDGE2IxyMZPce2e1fR37D/AIH1LUP2gLTVY5ZL3TNJ0y5a4ui2VjeRPLjjB56ls49jXt+m
+/sXfCOHWLW+l8O387W7FltbrVZnt2JGMsmRu4z3719HfDnwboHgTRYNH8P6PZ6HpsZLra2ke1dx5
+LHuSTzkk1xTzeCw86FGPx6dNvvZ3Usmn9Yp16svg1sr79OiO9sl2WsY9BU1NTG0Y5GOKdXzJ9UfC
+n7XmpXeo/F69tLK6mg+yaYfNKaUL1XjWPc8fP3SfM47mvnS9uLa5sLW3tkigu445hcTy+EseY5JK
+HjphcDPbrXa/F/Vbvx38UfEerGO7iieOWe1kttbjtTIc5jj8snIyMDJ644rjrlNdl0/T4JrDxHaw
+RQzKlzFrsTB1ZmZm5bnByPwr43CfvKftV9tykvSUm1+DR99X/dNUn9hKL23jFJ/imfq54WjMPhnR
+0IwVs4VIxjHyLWqOMn2rxb4b/szeFPBmhBNN1LxU8d4kczC/8Q3VwyHYBhWZiQMHoMD2rqYvgvpS
+Oca14gOP4TqkmK+ojbm8z4R7nJftgts+DFyzPaiAXtt5wvNPN6hXf08odTnGD2xXxfZ3fhxdI1G3
+kj0qTVp5IDZXEPguQCCNSxmUjPO4MuD2wa+lv2nPgvZ+HvDUni3Rr3xfc67vt7HyLPWnZWh3HdiK
+VvLB7lgN3FfOli/ia28O6rYLo3jSWG7ntpHu21aBGtmj3kKGzgByxBGRnaK8LHJuuz67LJKOGST6
++nY+1v2VAg+CejeXsKmW4OY7T7Ip/et0h/5Z/wC7XrvWvEf2Qrq4k+Ei2l3FdwXVnfzxvHf3KXE4
+DEOC8ikhiQ35V7cOtevl/wDu0F2Vvu0PnswX+1VH3d/v1PMPiVIlnbw3T/KtrqNpcEnoAJQpJ9sO
+aveArmKC0iWNk8uN5I1ZCCpG8njHGOQfTmqfxWgkudGv4otu9ngxvOBxOh/pVHS7n+zpr2PakIN9
+OdqYwAWGPp+Feldctjy7P2nyD40/GHSfhT4Tvtf1eVhbQcLEn35XPRV9+lfCeuft0+Ltc1Jm0jR9
+Ot42fMQuEJKjsCc9fqa6b9rr4n3viTw/rMMehPPo1sIo11HcTGqyy+WZHUgEMrqACODkc18v/DyK
+yl8baPZ6rbC4sp5RDLDLkDcwIBPoQSCK+z4ewmDxdGdWrDma7+XZHxfEGNxmGqwp0J8qfX/gian4
+l1VreT+y7WHw6v8AaL6wZtGidSt2w2vN85dcsPlORggYqDWvE97rrRnXrCx1S9UfvJL602szdSxj
+Uqqk8HhQOlfUMnwW8KSo6TwahKrrhle+mH4fe/IciuHsv2XrCbVL4alr+qPZeYTbG3TbMVzx5jtk
+OcegHIzX0sPqEJOcKdtNrea9T52f1+pBU6lS+u9/L5eR4S3j2bwld2es2dvYade2UyzxXkFmzywk
+H76EMcMOMZ4rrb/45ePNa2vdeJLuUkbg4VQ3POeBxn/GvVNI/Z00TSNJ1kavNLqrvH+4uXj8prUA
+nLLhiGYjHJH4VZ0r9mfwpDYzf2lJqd9Oy/uJ/PEIj4OCFThucH5vyrWWIw/tJScVJaW0189Xf8kc
+8cNX5IwUnF6310fbTT82eITfFPxdOcy+KtTLf3TcNXvn7MH7YGs+EvFNj4f8Zao+peGr6VYFvbhs
+yWEjHCvu6lCcBs9M57Yrc8MfDrw54W0SCwewjv3jZn+0XkcckpJOeTjp6cVlfEjwV4e1/wAK6hbL
+p1tYzJA8kN1DEqNGwBOcr2OMEe9TXlg8ZF0J0tH1srrzRrh44vBzVeNXVdO/kfpRpN0Z4yrY3KcG
+tHFeG/sl+PJviB8EvB2r3MhkvGshaXLE9ZYSYifxCqfxr3EnBr8mq05UakqUt4tr7tD9ZpVFVhGp
+HaST+/U/IbV20SLVLyTX9Q8KXV5LYBoHbRbhG3simDccdlznNZuqjRpdP04X03gh7B7WT7OUhnic
+RF3zjjg792M+1dncf2nYtd2wXxle+dpwCTE2c4gLqjbu/KfdrK1GHU7GDTZbnUvE93DLbmTyJdIt
+5FiUs6+WxB45Bb8a+SjpZLRH38mmrt6+v/AP1c0LK6Lp4wOLeMcf7oq7GcSP64FVdMQxafaoTnbC
+i59flFSW1wJLi4QZATaOfpXs03eoj4fo2eHftqQ6ZcfBuNdUOk+SNUtjGNauXgty+WwNyc7sbsDo
+a+MtPi04+EtXWD/hXR0D7ZbfbjLeXEg88CQwDjnOPMPH419rfthyY+E0SLLPbyyanbqktvpgv2U/
+Mf8AVHjGARu7Zr5P0mw14eGNSvovEOv29hDeQQy2cXhOKNppGVykm1uCECsCT039s142Ob+sP5fk
+fZZUl9UV+/n5eR7V+w/4p0y31PxJ4as7rw6Vnjj1GG28PtJsUr8khYSDJJynIz07V9cA18E/DHxT
+qvhLxDpeorb+L9Vmt78SOsmhQwqYDGY5FPl/Ntwxfjuo4r7xtriK7t4poXEsMqB0dTkMpGQR+ddu
+V1bxlSfR3+T/AODf8O55ec0eWqqq2lp81p+VvxOH+J3h2XxBoWoWkEz2s7+XJHNHnchSVJOMeuwj
+8a8x8U3epW15eR6PpN5qt1c3M0saxOoUKVyrbmYFgMAEDoc19B3VoLgf7VcL4o8IkzQXMTywSwuW
+V7d9h5GG9sEE54r3Va6ufNSjJ6xep8A+K73UIPA3xLi1eyWOz1XQPKtI47lZQG3xvHIAP+mkbE9c
+Zr528OaDFrsOsTy6pHYS6fo76pCJD81y6SIvkqcghyHLDHPy19TeKoF0v4eeP7fW9PWHUNLsb6JT
+dYZyiFhCynsp8xePUV8q6HpFtJpsc93E07Pbusa7iuyQcK/qcV9PwbVqctWnN6xa2222V/JHxvFE
+acI0p20fNa/a7Ov0v4++MtLso7YX8Vysa7VkuYg8mOwJ4zT5P2hPGkmR9utk+lutc7b6DbFQTEWz
+3YmrK6LbKOIE/Hmv01ww1/gX3H5ssRibaSf3kuofGTxdqlvLBPq5WKVSrrFGqkg9RmqsnxR8Uynj
+WrlUAAVEICgYwBVkaTCFP7mP/vmrC2KKQFVRj0X2pc1GO0F9wuavLeT+8yH8e+KLjrq9+3ptOP5C
+oJPEfiW4gkjkv9QkikUq6u5IYHqMV0Qtx0z+tSLbAHgc9KXt6a2iilCtLqfdn/BPV5f+FC2CurKF
+1a9CBhj5dyn+ea+tD1rwz9lHwnL4X+EXhy3mj8uaaN711IxgytuH6Y/OvczzX41jqiq4qrOOzk/z
+P3DAwdPC0oS3UV+R+PcF3o0V1qv9i2fhe6szY+XOyatNGRGwQytyRgCTAFPWC0jfTv7K02yvL6SE
+O0UPiRwPtJZhsCk4xjb7cmsWXxf4gs2KXC2F59rLRmK3sbd2dB8xVsKDjABoi8QLKdNlj0rSLfUl
+fzMy6VHGykdADnbn6V8WkfoDraM/TjwN45+K95pDt4l+FdloV3E/lx21n4jiuleMKMMG2DHPGDXS
+2fi3xS8zEeAbiKaQAF31KDZx0yRk/pXw/H+278XY3jBm8OzMQeJdP2qxA6cSg19SfsmfG7xJ8b9F
+8S3viG00qzbTL2OyiTTRICzeUHctuJ4ywAI44r06VSMppJHzVShOEXJ2sY/x7HxN8dw/8IxH8Nnu
+NClkt5YtT0zxIsMxmwSVddgKRqQAWJOc8Cqfgf8AY3tLuyS58Y3+qQXuQUtNM1yeVEXuruyjJz/d
+AH86+m0kSRQysGUjIPtThgd8VpLDUqlR1Ju5rDH1qVJUqXu+avf8zB8HeAtB+H2mmx0DTYtPtydz
+7Ms8h9XdiWY/U1U0Rh4X1ltDkG2wuS8+mt2X+KSD/gJyyj+6SP4K6C5sftDlhd3EOccRuAB+YrO1
+vwuutaa9tLfXEZUh4Z1274ZFOVdTjqD+YyDwTV1abspUl70dvNdV8/zSfQwp1btxqu6lv69H8vyb
+XU2gQcVBe2wuYivesjw/rs0tw+laqqwazAu4hBiO5j6CWP26ZXqpODngneBBPSt6VWNWPNH/AIby
+fmYVKcqUuWX/AA/mvI+X/wBpv4RJrPhHXNStLeW4lWIteWKDMdzD1kO3GdwHPX+Gvz7ltLZYoY7U
+BLWMYTJzhfc/1r9mL/TxcjcvDjkHvmvkj9oL9i/RvGFlf6h4PtRoPiWQmUQQyGO0vWPVCnIjduzL
+gZ6jByPpcox9LAuUZqyk91+v+Z8tnWXVcfGMqb1j0f36f5HwNB4qiku4oFtW2M4j3l/fANdHsjJO
+WAx1zxXB3mh3OkatPbTRFLyxuGimtpWJeJ0bBR8d/lIOPTNfpf8ACD4AfCHx34V0zxHofhHRNRs7
+yFZPMffMYpMDfE25jhkbIIPp7ivs8xxzwMIVJRupdn+fr0Picuy5Y+c6cZ2cej/Tc+CJJbSPHmTx
+oCcZZwKJNR0+CEyvOixDAL54Hpz0r9U9K+A3hDRpjNZeFNDtJWUIXSwiyVHQcg1oax8MtK1HSbnT
+7rStOutPuIzHNaS2sZilU9VZcYIr5yXEF37tP8f+AfSR4bsveqfh/wAE/GGXxDfylmW+lZQxwEHU
+A+1fXn7O37PF38SJdG8R63Glt4XmRLlEDgyXo/uqB91Mggk89RitX4wf8E/bo+IYLn4dJaW+k3GR
+c6dqN2V+xtnIMDMPmjI42sSV7EivfP2U/hJ43+EPh+bw94nn0i90VHaewNlKzXEDu2ZEb5drIT8w
+Ocgk9QeOvHZnCrg1UoVUpvdbO3l5/ocmX5XOjjHTxFJuC2e6v5+R73o9itnbrtQIqqFVVGAABgAf
+hx+FWLrUrSw2/arqG2352+dIEzjrjJ57fnVDxR4p07wfok+palMYbaLCqqKWklc8LGijlnY8BRyT
+XmE/w6tfiVO+s/EDS1vLmTiy0gzssemQ9QhKn5pW+9I3TIVRwvP5bjsc8PanRSlPzdkl3bs99krX
+b8k7fq2Ewcaq9rXbjT2uldt9km0tN27pJebSf5zXPgkXrwG6kEohLEEKUbJGOoORxn86hT4fiN1e
+GYJgsTvJbqMAjnqK7c/D3SwymLUdftQDnbBq0uD9d2aWbwNGzboPEGvW4/um5jk/9CjNfMLNsM+6
++X/BPIjxNgnvGS+S/wAzjo/B11EVaJrUORhnZTnHYD+de8fss/FbTf2e9N8UQahpNxqM+sXUNyjW
+DLxtj2kMWIxz0xnivND4KvFZfJ8V6gijqs1nbzE/iVFNfwnrSyL5XilGjHVZtJjJ/NXH8q2hmuGi
++ZTs/R/5Gjz/AC6rHlnJpej/AEPpc/tH/Dm/uZJ7rwbrVq8rM7vBd9yck4WUCr8X7QnwukGPL8Ya
+f2zHPKcflKa+Vz4d8QpJ+71bR5YvWexlR/8Ax2TFRnRvFIkwH8OyR/3t1yrflgj9a2WaUJf8vfz/
+AMhrNcqltUt8n/kfXcHx++G0wAXxZ4ysgOB5hlIHvyGrSg+N3w/kULF8UdctjnAN1CTz/wACh5r4
+uNj4oWQg6Tpcyf8APWPU2TP4NHTUj8RmRlk8LoijpImsQMG/AgGtVj6L2qR+/wDzNY47LJbVl9/+
+aPti7+JfgvXIooz8YIvMicSQzyW0KyxP/eVjGMccHPBBIIrufDHxx8NXV7b6ZfeJ9GvLmVtlvfWd
+yvl3B7Blz+6c+hyD2Pavzoa61dGZX8Kaocd4pLeQH6fOM0xdSuJlfzfCviOPs2/TgQfphzmrWLgp
+c9OpFP1Vn6q/3dfO2h1xxWAnHklWi1/iV16f5beV9T9aAQQCDketV7uzW6XB5PSvzm+HP7V3iv4b
+IttA2sX+lQ8Gw1jS7h1jUdkkCllHsCR7V7/4Q/4KDfDjV4F/t5NR8PyjAeRrGeaAHv8AMIwR+K16
+tLMKctKmj8tV96/VI55UYPWjUjJeqv8Adf8AK57dc/Cfw3c6pPqMvh3R5r+dt01zLYxNJI3TLEry
+fc10uk6NbaND5VrbQWkWS3l28axrk9ThQBn3rjfC37Qfw18aQCbRvG+i3iEZx9rWNh9QxBH411ae
+MNAeMOuuacyH+IXcZH55ruWMpVFpUT+ZksJUi7qm0/Q16D0rlNS+LPgnSFJvPFui2+P4Wv4t35bs
+1wniX9rHwFoqulhdXWv3AHyx6dASpP8AvtgfzrGeNw0NHNN9lq/uV3+B0RwWJntTdu7Vl97sj2Bo
+Im5KA4rmPEnjjT/D94ul2VtLrXiCVd0Wk2ODJjs0jH5Yk5HzuR7ZPFfKfj79rXxl4lWW10OzHhmz
+bK+amZLkj/fIwp+g/GsPwX+0z4q8C6VHYWml6Ncrgedc3EDie6fu8rhvnc9ya8+pjalX3aK5V3e/
+yj+srW/lZ3wwMKXvVXzP+VPT5y/SN7/zI+qtN8OXd3q0ev8Aia4i1DXIsi1t4P8Aj000HqIQRlpD
+0MrDJ6AKOK6B0YY3KB6buK+KvEX7Svj/AMQmQJqsejwt/wAs9MgWMj/gZy1eeahr2qarctPe6nf3
+k7dZJ7p2P8/5V5io8t7dfvfm3/XbZI6pOdRpydraJLZLslpb+m9bl77R70n2j3rxn/hpvQP+gHqv
+/fyL/Gj/AIab0D/oB6r/AN/Iv8a5P9SOJP8AoCl98P8A5I/n73j2b7R70eeK8Z/4ab0D/oB6r/38
+i/xo/wCGm9A/6Aeq/wDfyL/Gj/UjiT/oCl98P/kg949m88DvSG4rxn/hprQP+gFqv/fyL/Gj/hpr
+QP8AoBar/wB/Iv8AGj/UjiP/AKApffD/AOSD3j2bzx+NHn/WvGv+GmtA/wCgHqv/AH8i/wAaT/hp
+rQP+gFqv/fyL/Gj/AFI4k/6ApffD/wCSD3j2bzx7mjz68Z/4aa0D/oBar/38i/xo/wCGmtA/6AWq
+/wDfyL/Gj/UjiT/oCl98P/kg949o+1MOjMMdOaedRnP/AC2kx/vGvFP+GmtA/wCgFqv/AH8i/wAa
+P+GmtA/6AWq/9/Iv8aX+o/Ef/QFL74f/ACQrSZ7HKYp8+ZGkmeu9Qc/pUD2VlJEY3tIGjPVDEuPy
+xXkn/DTegf8AQD1X/v5F/jR/w03oH/QD1X/v5F/jT/1I4k/6A5ffD/5IqLqQ+HQ9Ln8IeHLqJopv
+D+mSRN1U2aDP4gZqq/w/8KtB5I8P2aRf3Yt8ePoVYV57/wANNaB/0AtV/wC/kX+NH/DTWgf9ALVf
++/kX+NbR4O4pgrRws1/29H/5I0VavF3Umvmd8/w88NmNUisZ7Tb0a1v7iNh+O+nN4G04IFh1HXbX
+Bzug1WTJ+u7dXAf8NN6B/wBAPVf+/kX+NH/DTegf9APVf+/kX+NarhTi1f8AMPU/8Cj/APJHRHHY
+2Pw1Zf8AgT/zO/n8H71UW/iXxDasOrG7jlLfXfGaQ+Fr4Y8vxXqaDvvt7eQn8dgrgf8AhpvQP+gH
+qv8A38i/xo/4ab0D/oB6r/38i/xrVcMcXL/mGn98P/kjoWa5jHatL7/8z52/Oj869T/4QjRv+fZ/
++/rf40h8EaN/z7P/AN/W/wAa/dv+IjZR/wA+6n3R/wDkz90/4gfxJ/z+of8AgU//AJWeWkY9aPzr
+ovGmi22jXsK2qsiSR7ipOcHJrna/Qsvx1LM8JTxlC/LNXV997a/cfiud5RiMhzGtlmLadSm7Pl1W
+yejaT2fZB+dH50UV6J4YfnR+dFFAB+dH50UUAH50fnRRQAfnR+dFFAB+dH50UUAH50fnRRQAfnR+
+dFFAHuFIelLSHpX8OH+tZwPxJ/4/rT/rl/7Ma46ux+JP/H9af9cv/ZjXHV/WXCH/ACIsL6P/ANKk
+f5xeJn/JX4//ABR/9IiNeRY1LMwVB1Y9BTYriKckRypIRyQrA4rp/hx47vPhj490PxXY2sGoTaXc
+iZrK6jWSK6iwVlhYMCPnQsuccEgjkCv0+/ar+Efhb9o/9k1PF/w70izkv7a1j8R6O+nWiRzXKKhM
+tuQgzuaMuuzn94ijqK9TH5osvr0qdWHuTdua+z81/wAHv2PhcLgliqcpRl70eh+TL3MMb7HmRW/u
+lgDUrfLnPGPWv1A+I+gaH+yH+wTY2sulabL40vLCPT4rq4s43mOo3eXmfcRn90GlI9olHHSvzT8G
+xKvi7w1FtDx/2pZIVbkMvnxjBz1BHr6mtMvzJZhCpVhC0YtpO/xW67aE4jB/V5Qg5ay/Axo7mGVt
+scqSN1wrAmke7gico80auOqlhmv1n/4KP+AdOt/2abptB8M241A6vYjdpemqZtnmZb/Vruxgc1u/
+sU+A9If9j/wfNq/hmz/tRbS9Mh1LTVE+Rcz43iRd3QDr2xXhPian9RWN9lvLltfXa99v0PQ/sh+2
+dLm0te9j8gW+QEk4A6moo7qCV9qTRu3orAmvav2LfhmvxY/aR8EaRcQC4020uG1e/SRdyNDbjeFc
+HghpPLTB4O6v1F/aJ+C3hb4s/Abx9onh/SdI/tiO0lFtLYWsQlivYMSpHuUZUllVSPRj6135lntL
+LsVTw0435rXd/hTdjmw2XPE0pVL2tt52PxVxzUJu4BnM0fBwTuGBX0f+wh+zrp37R/xWmTxBHJJ4
+S0G1S/1C2UlPtbuxWG3JHKqdrs2OSExxuyPvvxf+0L8HfhJ8aPD3wPPg6BH1EW9rI9lpsAsbN5zt
+t4pF4J38ZIUgBlJ6nBj87WFxDwtGk6k0uZ2drL8bjw2Xe1p+1qT5Vex+PY5GRyPaivsr/go9+zT4
+e+DniXQPF/hCyh0nRvEMslreaVbqFht7pV3iSJOiK67sqBgMgI+9ivAf2bvgvN+0B8ZtB8FLcPY2
+VzvudQu4x88NpEN0hTg4ZvlRSRgFwe1elhsyoYnB/Xk7Qs2/K25x1cHUp4j6vuzy9riJHKNIocDJ
+UnkClSVJUDo6up6FTkV+vvxe+J/wZ/YM8NeGNHh8DI6akXWOz0i0he48mPb5s80khBcjeo+ZizE+
+xI4j9tX9lPwV8TPgrf8AxU8BaZa6X4gsdNGtLNpsIhi1Wz8vzGEiDA3+WS6vjdkbTkHjwKHEsKk6
+ftaLhTqO0ZNrXW2q6fez1J5Q4xlyzTkt0flsbuASeWZoxJnG3cM56U6SeKEgSSpGSM4dgDX6/aX4
+S8Py/wDBP+LUv7C0w3p+HRm+1Czi8zf/AGeTv3bc7uh3dc815N/wSl8NaRr3wr8avqemWN/JHraK
+kl3bRysoNtESAWBIGece5rT/AFih9WrYn2X8OSja+93a+xCyu9SFPm+JXPk6kPSlpD0r+TD/AFMO
+B+JP/H9af9cv/ZjXHV2PxJ/4/rT/AK5f+zGuOr+suEP+RFhfR/8ApUj/ADi8TP8Akr8f/ij/AOkR
+D9PrX6U/8Es/GXjKz0vxN4B1vw7rNv4dtMappmpXtlLDBA8jAS2waQDO4kSqBn/loT1FfmnLH5sT
+ITgMMZr7k1v/AIKt+M7jw22n6H4I0nSNR8nyU1G6vpLvyjtxvEWxdxHUBmIz1z36s/wuJx2GWGw9
+JS5t23bltaz8+p8VllWlQm6lSdrdO5k/8FQfi9/wmfxl0zwPZTiTTfCVtvuFQ5BvpwrNnHdIvLHs
+ZHFfJXg/nxp4b/7C1l/6UJVLVdUvdd1S91PUrubUNSvp3urq7uG3STSuSzux7kkn/PRdIvzpOs6b
+qAjExsruG6EROA5jkV9ue2duM+9epgsHHBYOOFh0X3vq/mzkrYj2+J9tLa/4H7iftM/HuD9m/wCG
+EnjK50SXX4kvYLM2cFwsLEytt3bmBHHpirPwZ+LsXx3+Bmm+OYNKl0WPVra5ZbCaZZWi2SSRcsAA
+clM/jX5o/tJft76l+0f8MZvBt34JtdAhkvILv7ZFqTTsDE24LtMa9emc960Pgj/wUO1P4J/B3RvA
+EHgS01m302KaIX0upvC0vmSySElBEcYMhHB5xX5n/qxiVgYtUv3/AD6+8vht623+Z9V/atH2z9/3
+Ldup6v8A8EkfhuVtPHHxBniOZXj0KykI/hTE0+PqzQj/AIBX2L8DfhLc/Cb/AITXz/FMnib/AISP
+xBc+ID5lqsH2R58bol2scqNox0r8u/BP7auofDz9mi7+EOheErazN1ZXVrL4hF+3nM9wzmWbyhHg
+NhyB83GBXC/sy/tC6p+zL8Qp/E+m6cutxXOnvp9xp1xdNEkiFkdW3YbBVk446EjvXr5hkePzCpia
+7ly8zXLHR8yjtrf3f89zkoZhhsPGnTWvd9rn3x+xh4Ws/g5+1J+0N8PAi28ktxZ61pkefvWLmVgB
+6hDcIv6V88ftlfArxr48/bim03S9H1FofFR082OqpayPbRRiGOKWVpFGFERjZmGQcAHuK85+Lf7Y
+/iD4hfGnw58UvDmkReBfFWjWf2Ey21ybpLuPezBZVZF3Lh3Ur3BHIKg17zYf8Fb9ai0Ly774Z2U+
+tBcfaLbV2jtmb18sxMwHtuP1q44LNcLiVj6VNSnOCjJNrRpJX7NaJ790EsRg61P2E5WSd1oeA/tP
+/si6/wDsxafoV74g8X6f4hbWbmS3t4LdJlmxGm55DvJG0ZUHBzl1rrf+CZ3iSy0D9qO3tbx0jk1f
+Rruxtmc4BmBjl2/UrE/5V4r8cPjp4r/aD8bv4m8WXMTTpH5FnY2qlbaxhzny41JJOTyWYkscdgAO
+H03UrzRdSs9R067nsNQs5kuba7tnKSwSo25HRh0YEAg19T9UxGJy6WGxc06kk02lom9vu69zxniK
+VLFqrSXupn33/wAFYfh/4j1jxP4A1/S9Hv8AVrA2dzpZ+wWzzlLlpEeNCEBILjOPUqa+qvFd1F8I
+P2JrtPEWLd9H8ELYXMUhGftH2MQiL6tIVUe5FfHvgD/gq/4r0XRY7Xxf4IsvE2oRJtGpafemxMxA
++88RjdQT32kD0A6V4Z+0t+2P40/aYFtp+pQ23h/wtayiaLQ7CRnWWQfdknkODIVzwMBQecE818dT
+yfMsRDD4LEQUadJt8107q99Fuvme5LG4Wnz1oSblJbH1R+yn+2z8LJ/2ftL+GnxUvF0O407Sv7Fn
+N5BI9nqNpsMa4dAdpMZ2srY5GQTmvob9kG++C03h3xNZfBCw+zaBZagkd7cqs4jubgxKdytMS7YX
+aMnA9K/FnPrzX0f+y1+2lf8A7LvhjW9Gs/CNv4jXVL4XzTzX7W5jIjVNgAjbI+XOeOtdua8Oc1Kr
+PBOTlNp8vMlG99XZ226XZz4TNbzSrpJJb21MCkPSjNB5r+XD/UI4H4k/8f1p/wBcv/ZjXHV2PxJ5
+vrT/AK5f+zGuOr+suEP+RFhfR/8ApUj/ADj8TP8Akr8f/ij/AOkRCiitnSvBPiTXrMXel+G9Z1S0
+LFRcWOnTTxEjqNyKRkema+vlJRV5Ox+ZxjKbtFXMaiuk/wCFY+Nf+hK8S/8Agmuf/iKq6l4G8T6L
+ZS3upeGNb02yiAMl1e6ZPDFGCcDc7IAOSByepFQqtNuykvvLdKot4v7jFpK6SP4aeM5Y1kj8G+I5
+I3AZXTR7kqwPQghORS/8Kx8a/wDQleJf/BNc/wDxFL2tP+ZfeP2NT+V/cc1S1sat4L8R6BafatW8
+OaxpNruCeff6fNBHuPRdzqBk4PFUNO0u91m9js9OsrnUb2TPl2tnA00r4GThFBJwASeOgq1OLXMn
+oQ4ST5WtStSfpXQ3Hw58X2cEs9x4Q8Q28ESl5JpdIuERFAyWYlMAAckmue6gEcg85FEZRl8LuEoS
+j8SsLRRRVEB/npSUtFABQOKTNLQB9R/8ITYf89rj8x/hR/whNh/z2uPzH+Fb9KK/Mv8AVzKP+gaP
+4/5i/wCIqccf9DWr96/yPAPjbpUWja9pkMDO6S2hkYyHJBDkcV53XqH7Qf8AyM2jf9eLf+jDXl9f
+e5fQpYbC06NGPLFLRLpqyv7Sxmcf7dj6rqVZ6yk929tfkkgr9Gf2Ev2sfhR8Gf2fLPw74z8YQ6Jr
+EWpXs7WklrcSFY3lLI2UjYYIOetfnNUV1xbyfSs8yy6lmlD2FZtK6em+nqn3O7B4qWFqc0Ve/c/o
+mt7qO7t4pojvilUOjDjIIyDXwn+27+158I/iT+zR8QPCHhvxnBqfiO6jjhhsUtbhC7x3UbONzRhe
+Ajnr24zX3B4eX/in9M5/5dYuf+Aiv58PF3/IU8RH/p8u/wD0a9fl3C+WUcbiZVajadJxatbu99PL
+pY+vzHFSoU4qK+LQ/oA8AEnwL4bZsknTbYnPX/VLXiGuf8FCPgb4e1vUtIvvFNzFfafcy2lxGNJu
+mCSxuUcZEeDhlPI4Ne3/AA/H/FB+G/T+zLb/ANFLXkmsfsKfArX9Xv8AVL/4f2txqF/cSXVzM17d
+AySyMWdsCXAyWJwOK+dwzwKqz+vKdunLb8bnfNVnCPsbX87/AKHyp+3f+1z8L/jp8D4vDfg3Xp9R
+1hdYtbtoJNPngHlJv3HdIgXjI4zmsL/glL8NxrnxQ8VeN7iHfb6DYpp1q7dri4O5yPcRx4+kleXf
+t+/Cfwl8F/jjY6D4M0ePQdIl0OC8e2jleQNK006s+XZj0RRjOOK+9v8Agn94Bh+FP7Kei6lqAFpc
+a2JvEV7JKMFI5BmLJ9BAkR/OvvcbPD5fkChhG7VXpzWvrvtpsrfM8CgqlfHt1re522Ppa+srbVbC
+4s7uFZ7W4jaGWJxlXRhhlPsQT+dfgJ8TfA83wt+Inirwldna2hajPZB2GA0SMfLf8Yyh+hr9Uf8A
+gnr8eJ/jZ4T+Icl9M8l7beKbu+jWRs7LO7YzQKM9Av7xQP8AZr5M/wCCp/w0Phn44WPii2hItfFe
+lYdsfKbu3xGw+pjaH8jXPw1z5bmdXLq27X4rX8mzbM4xxOGjWj0f4bHimnfsn/GbWNPtb6x+Gmu3
+VldRLPBPGkW2SNgGVhl+hBBry68tJrC8uLS5iaG5t5Xhmib7yOrFWU+4II/Cv1u+GH7eXwM8O/Db
+wlpOoeOkt7+x0i0tbmI6ddt5ciQorjIiI4II4Pavyg8W30Op+L/EF9bP5ttd6ndXML4I3o8zsrYP
+IyCD+NfX5Vj8bjKlWOKo8ijs7NX37/oeFjcNh6MIypTu3vqZfevRv2cvh7pXxX+Ofg7wfrhuV0jW
+Lt4Lg2cnlyhRDI42tg45Qdumay/gt4S0/wAe/GDwT4Y1cyrpOsatb2V2YJPLkETthtrdj71+qnwx
+/YE+EPwu+IGieLNBudafWtJmaa2W41bzYyxRkOV288ManOc4o5dCVKV1OUXZpbPZa37lYDAzxMlU
+0smfCX7df7OHhL9mzxj4T0nwjJqUltqthPdXB1O689tySKq7TtGBhjXzN26gfXiv20/aC/ZN+Hv7
+ROuaPqfjOfUobvTLeS3txY3/ANnBRmDHIwc8gc1+ZH7dnwS8Lfs4/E7RNA8FS3UmnXujpfTHUbkX
+T+aZpUOGwMDai8VwZBndPG06eEm26tndvbfv6HVmOXyhN1YWUex6LSikpRXUfg54Z+0H/wAjNo3/
+AF4t/wCjDXl9eoftB/8AIzaN/wBeLf8Aow15fX0WG/gxP0TLv90p+n6sKiuv+PeT6VLTZEEiFSSA
+eMiulHpLc/oX8OnGgaYDwRaxf+giv58PF4I1TxF/1+Xf/o16+2LP/gq/47s7SG3TwB4c2RIIx/pt
+x0AwO1fEmqSf2rdX8zgRm7llmYL/AAl2LED6bq+G4ayrFZbOs8TG3Na2qe1/8z6TMsXRrqmqbvZn
+9Afw/wDm8B+Gx/1DLb/0Utfnt45/4Ji/E3xR458Sa3a+P9EtrTU9Uur2GB5LzdHHLMzqpxxkBgOO
+OOK5nRv+CqvjrRNIsdPi8BeHZIrSBIEdr2cFgqhQTx6Crh/4K0+PiMf8IB4c/wDA64/wr5/B5Tnm
+XVJzw0Uubu4s9GrjMFXio1JbHE/Ev/gnZ45+Hh8Lf2z4q0nWo/EGv2Xh9I7MTmZTOxBfMgwFVFdj
+6Yr7u/ba8ZW3wb/ZJ8T2+mEWL3VnF4e05I/4POxDgf7sW8/8Br421b/gqP4t1y40ya++HHhq5k02
+6+22pe9uP3cwR4w4GOoWRx+NeWftLftneKv2nfD+i6NrOi6doFhpl216E06eSTz5DGUXfvAxtDPj
+H96vUeXZrj8RhpY+K5YO71XdPZeljl+tYTD06nsHq9juP+CYPj4eD/2ipPD0j7LLxPpUlqqFsL58
+H76P8dnnAfWvrT/gp58Ov+Eu/ZrufEVvAJdR8JXkeqKR94wN+6mUe2HDH/rnX5afD7xrffDbx74e
+8W6YqSahol9FfQRSEiOQoeY2xztYEqcc4Y19UeNv+CnPi3x94N1zwxqvw+8Ovpms2M2n3IW9nz5c
+sZRscdcNxXXmWVYuWa0swwkbpW5tUtnZ/fHQxwmMo/VZUKzOz8Ff8Eq4vGHg3QdePxPuLX+1LC3v
+jbjRFcReZGr7Q3nDON2M4FfCWvab/YniDVdM83zvsF7PaeaRjzPLkZN2O2ducds19e+Df+CoXjjw
+X4P0Pw9B4I0G8g0mwgsI7ie8nDyrFGqB2AHBIXJ9zXyBrWptret6nqUkaxSX13NePGhyqNJIzlR7
+AtivWyqOaRq1f7Qd4/Z+Hu+3y3ODGvCezh9XWvXcp4yMZx717R+xWyRftXfDOSSTYi6hIxaR8KP9
+Gm9TivF6QgMMHkV7mIpfWKM6N7cyav2urHn0ansqkZ9mfcv/AAViuLe8+Jfw8eCdJlGkXYJhkzg+
+dH6dO9fDagKMYLfVjSIgjGFGPxpa5svwn1DCww3NzcvXbrfa7NcViHiarqWtc+wqUUlKK8U/Hjwz
+9oP/AJGbRv8Arxb/ANGGvL69Q/aD/wCRm0b/AK8W/wDRhry+vosN/BifomXf7pT9P1YUUUV0HohR
+RRQAUUUUAFFFFABRRRQAUUUUAFFFH5fjQAUV6R4R/Zz+IXjz4aan4+0DQl1Hwvp0ksVxNHcp5++P
+buVIc73PzAAKDk9K9M0f/gnZ8c9W0u2vn8P6XpgnQOttqOrRxToDz86qGCn2zkd68+rmOCo3VStF
+Wdt1v2OyODrzV4wZ11KKSlFeKfjx4Z+0H/yM2jf9eLf+jDXl9eoftB/8jNo3/Xi3/ow15fX0WG/g
+xP0TLv8AdKfp+rCiiiug9EKKKKACiiigAooooAKKKKACiiigAooooA/TX9gjxDc+Ef2HPGviCy2i
+80q71m/g3ruHmR26OuR3GQK+ZP2cf2oPipaXXieW48catqkl2LW4kOozm4CyN524oHyEBwOFwPlH
+HFekfswfH74e+A/2LPHvgrX/ABTZ6Z4o1FdY+yabKshkl863CxYIUj5mGBzXyt8G9f07w5Jqp1S7
+SyE0NssfmA/MV83cOAem4fnXw1DAqdTHSrUr3mrXW6v0v09D6ipiOSFFRlbTufRe33X/AL6FAU+o
+/MV8d5b/AJ6Sf99mjLf89JP++z/jX0n1D+9+H/BPyz+wv+nv4f8ABPTv2gpY38V6VGsiO8did6qw
+O3MhIz6ZFeY0dyc5J6k8k0V6NOHs4KHY+jw9H6vSjSvewUUUVodAUUUUAFFFFABRRRQAUUUUAFFF
+FABRRRQAZ5zR/wACK/SiimBP9hn/ALn6ij7DP/c/UUUVjzs15EH2Gf8AufqKPsM/9z9RRRRzsORB
+9hn/ALn6ij7DP/c/UUUUc7DkQfYZ/wC5+oo+wz/3P1FFFHOw5EH2Gf8AufqKPsM/9z9RRRRzsORB
+9hn/ALn6ij7DP/c/UUUUc7DkQfYZ/wC5+oo+wz/3P1FFFHOw5EH2Gf8AufqKPsM/9z9RRRRzsORB
+9hn/ALn6ij7DP/c/UUUUc7DkQfYZ/wC5+oo+wz/3P1FFFHOw5Ef/2Q==
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="bmoihymooxwdej.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <37b3701d0b859492b834102f6705c4@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAfQBx
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++LvtEn/PV/8Avo0faJP+er/99Go6K/p4/J7kn2iT/nq//fRo+0Sf89X/AO+jUdFAXJPtEn/PV/8A
+vo0faJP+er/99Go6KAuSfaJP+er/APfRo+0Sf89X/wC+jUdFAXJPtEn/AD1f/vo0faJP+er/APfR
+qOigLkn2iT/nq/8A30aPtEn/AD1f/vo1HRQFyT7RJ/z1f/vo0faJP+er/wDfRqOigLkn2iT/AJ6v
+/wB9Gio6KAudF/wrvxD/AM+C/wDf5P8AGj/hXfiH/nwX/v8AJ/jV7/haut/887L/AL9H/Gj/AIWr
+rf8Azzsv+/R/xrmvX7I8y+O7R/Eo/wDCu/EP/Pgv/f8AT/Gj/hXfiH/nwX/v+v8AjV7/AIWrrf8A
+zzsv+/J/xo/4Wrrf/POy/wC/J/xovX7IObHdo/j/AJlH/hXfiH/nwX/v+v8AjR/wrvxD/wA+C/8A
+f9f8avf8LV1v/nnZf9+T/jR/wtXW/wDnnZf9+T/jRev5BzY7tH8f8yj/AMK78Q/8+C/9/wBf8aP+
+Fd+If+fBf+/6/wCNXv8Ahaut/wDPOy/78n/Gj/haut/887L/AL8n/Gi9fyDmx3aP4/5lH/hXfiH/
+AJ8F/wC/6/40f8K78Q/8+C/9/wBf8avf8LV1v/nnZf8Afk/40f8AC1db/wCedl/35P8AjRev5BzY
+7tH8Sj/wrvxD/wA+C/8Af9f8aP8AhXfiH/nwX/v+v+NXv+Fq63/zzsv+/J/xo/4Wrrf/ADzsv+/J
+/wAaL1/IObHdo/iUf+Fd+If+fBf+/wCv+NL/AMK78Q/8+C/9/l/xq7/wtXW/+edl/wB+T/jR/wAL
+V1v/AJ52X/fk/wCNF6/kHNju0fxKX/CvPEP/AD4L/wB/l/xoq7/wtXW/+edl/wB+T/jRRev5BzY7
+tH8f8zj6K/Ujxj+yr8EvFnhX4weFvB3gS00/xh4PsVgi1WN3aQ3b2S3MTD5uvzKrZHUmvF/gX8Cv
+h94t/wCCfXjD4hap4UsrzxhZ2GszW2rS7/OieIOYSMEDKYGOO3PfPz9PiHD1KXtOSS96MbaX99Xi
+99mj695VUUuXmWzfXofD9JkV91ftM/AD4d+Bv2J/AXjTRPC9ppPiXUk0Q3mrWyu87+dCGmO0tgli
+c46E4r1v4bfs9fDgad4PtLf9nR4tC1N/Iv8AXPiHqVvb6opI+RlgDSNI78nYDFjHA7BT4hw8aPtl
+B2vJbxXw77v7rasUcqm5crktk9n1Py7or6B/bt+Dfhn4CfHd9E8LxnT9CvtNg1KOzmlLrbO7yIyI
+zEtt/dhgCTjcQDjivn1WDqGUgqRkEHOa+gwuIhi6EK9PaSurnmVqMqFR05dHYWiv0A/ZI+Afwom+
+EPw4vfiT4QtPEPiH4ha1eQ6XNdFwYbeOGV0GAw+Ui2JHqZh2rzu1/ZT069/4KF3fwyXTVXwZbXP9
+tyWa7vKXTfJWUQ5znaZGWLr615Ec7wzq1aTTXs03fSzUXZ2176Hf/ZtTlhJP4ml6X1PkOivun9s7
+9n74XW3wS8FfE74U6XaaD4dn1WO11G9skkKvaTMYxOysT/q5E9s769+tP2OfhFpuseBtK0D4Naf4
+58JanbSSX/jafXA5twse6KRk3ZuPNPePCrnIAFck+I8NCjGs4S1clbRNOO6d2lfst32ubRympKbi
+pKytr3ufkxRXvf7cnw98OfC39o/V/DnhTSYND0SDT7KWOytt2xXdCXPJJyTXffsc/Cn4fXHwn8cf
+Er4n+GofEmiWurafoWmwXDlQJpZY45HXBGTuuYc56BGr1Z5lThg4YzlbUuWy6vmtZduvc4o4KUsQ
+6F9r3fofI1FfYv7X37Ong74XftReBNH0rw1qqeCfEdrGzaH4Wi8y8mmjkdZYrcO33nDQ5ORgFj2r
+6k8H/sx/DHxFr/8Awjmp/s9aP4Y8OTad9ptb/U9WibV5JQVDKYInd1ADHMnnHkAY5zXmVuIcPRoU
+6/K2pq+8VZbPdq78lc64ZVUnOUHK1nbZn5LUV+tH/Drj4Kf3vE//AIOT/wDE0Vw/64Zb/e+7/gmv
+9i4jujzv4Jft4fB5Pi3qB0vwLrPhW/8AHWpxnV9d1PUY2tvMUMscsgaUiNQDj5Qo556VyVj+118L
+PgZpHxE+Dr+Cb7xV4Mm1nUUi/si9iazmsrg7jGjmQMRh2GQfofT4HIDAhuQexoACKFGAo6AV3/6u
+4PmbvKztpzPeOzve+i0XYy/tatZWSur9O5+l1t+1h8KPjb8C/E+jw+HYvCsnw/sLbVPDel+I76MJ
+cXNrG5tVhCybpNjRIpU5yHHXNcL41/4KHfDTx7rPgPxbqvwo1jUPF/hm58+2FzqSpbWJk2CaSMK2
+JnCr8nmIuDg5WvgpkViCyglemR0p1FPh3BQk2+Zq7a956XVpK97u/UUs1rNWSXnofoB8Of2kvC37
+Qv7alh4ibwJcXWk3PhSTSrq31wWskWnrDObiS+cnKLGqZUnIbcwA4PPxl8U/EOh+OPjR4n1rQoBa
++GdV115bOIRiFVtGlAB2j7gKDdjsD61yEN3PaLN5NxNbiWMxy+TKU8yM/eRsEZU45B4r0f4a/sz/
+ABS+L+nyX/hLwRqOp6cuAL2bbaQSZ/55yTMgk6c7SQMjPWuulgsNlknWc+WPKoq70Vr936Gc8RVx
+UVFQu73dkfcPxv8A2o/gR8KfHXgXw9aeDbrxhH4CtYrjQL/w7qqNaWJddnlgiYCRgka537vvc9TU
+3xw/bC+Gvhrwtp/xJ8K6Mb74iePPC0ukrc2uoRNPoyqvmRJdxhzsKyzE8LljFjOAMfmhNamxuJrZ
+lRWhkeJxEwZdykq2CpwwyOoJBqIRqrMQoBbqQOTXBDhvCr2blKT5d9X719e+ib103Z0TzWq+ZcqV
+9tNj6p+G/wC1v4U0r9km4+B3jbw3rurWckU9tHqWkywKYommM0RAkP30c5GRj5RXQXv7emlR/HD4
+beIdG8Naxo3w58E6TPp8HheC6RXmkkhaIOQGEZVF8oAMTjaxHJr44or0ZZNg5SnKUfi5m9Xa8laT
+S6No5VmWISSutLdOx6t+1P8AGix/aA+MmpeNtM0y60i1urK2tltL10aUNEpBYlCRg5HevqzVvjH8
+DPhV+yx8KfBGr+GZPG1rq1rFrepadoOrBZLTUFWOR5LgiVWVmlkbCZwNhGMKK/P2mqioSVUDPJwB
+zVVsqo1qdGjzNQp7JNrZWWu+gqeOnCc6lk3I/TP4j/to+BfFHw98DfGrSdGgfxb4X8QyWMXh/Ub6
+NdQWznUw3RREc8Omxg5BAKj3zxE3/BQz4XWHxoHxF0z4V61Nr97pg0vUNWu75BOkCkMkUEXmNGAW
+ALNlT8o4OTXwJsXfu2jf0z3/ADpa8+nw5goR5ZXa1S95r3W78uj1SevqdEs2rt3ikuu3U/TT/h7X
+4O/6J74k/wC/1t/8XRX5l0Vl/qrlf8j/APAmP+2cV3X3H6E6r+w5+z/8Nb/w54H8e+P9atfHeu2U
+1zDqX2hLW0UxDLsFZDHGuc7RIxLbSM5Ncx+zp+wf4J+Jh+Jlpr3iW+12fwvq32HT9T8L3kQtb6Fr
+dJY3+7JlzvwyhvlII5xmu68e/tAfspftC6j4Z8ffEK51NdZ0Kymtz4Tu7OaRJ/MwdkipGyy7HyUZ
+XUZwT0wMH9k/9rj4PfCyz+KPnRn4dafrOsfadD0S1s57gQ2y2yRKxZA4DsyFmBOAzHHArwlVzb6r
+Nr2ntNL3jpfm+zbW3LvZW8z1+TB+0jflt018jitY/Yk0bwp+xPe/FLxDa+JtG+Itnp73M+jXsqxQ
+xyi5MahoTHuAMeDjd/FnjpVj4LfsyfA3xV4J0C8vNS8c/EzXtSuIra9/4QvTrhLLTHcfMXcxgeXG
+eGkLnJGQvatXxB+1/wCGfG/7A2oeBPE3i+91j4q32mvBcJd2szPNL9rLqrTBPL/1YUZz0HXNej+L
+v2yfgj4u+EvgyCx8Z+J/Af8AYlxaXE3hHwxbSQ3FwkQH+htIFCeRkDLK4DBfetZ1s25JQmp3dSWs
+b2UbK3Rvlve1mvNkRhg+ZSi46RW//D7nHwfsJeGPh3+2j8PfDbXtx4h8F6hZXeumw1QK8ge12gQy
+EALJGXkhbkZIDKcjr7H8XPhj4b/aR+L/AI3+GUfxc8daN4l0vToLx9CsZFi0W1t3VAi+SqL5x+dC
+258/vAAQAQPIfjt+3F4HT9on4VfEPwTezeKNM0O01Cw1mzW1ltpfIuPK5j81VDMNpYDOMx4JG7Nd
+dZ/tT/s4eC/ir4s+Nul+Kdb1fxZ4i0qGwl8Nx6fKrJ5YToGjVVZvKiBLPtGCR1rz6kM0qKliKsZu
+fJaPu7T5/tJrS8ertfudEZYWKlTg1a+uvS3Q8K+Ef7HnhL4n6J8R/Asup3+i/HfwbJcQLZm8Q6Zf
+hSfJnjQoH8p/lRvmym5Wx8wFeheJP+CaltcfCvwDd+G4/EFr451mfTotbi1K5ikttKSSIveSNGFU
+nyyGCgMcnaOc5rkfhh+1B4G+H2hfFf4tXV7He/Hnxk1ydN0eKzlaHSYmbbBD5zJ5bYxHJId3zCNF
+4I59J8c/8FCNE0z4JfDhvC/i691r4gaTcaTNr9m9nJEdRjSLbfRPI8YT5yW5U/ewRwK9LETzr26+
+r81uZbp25uX3v+3L7N6dtDljHA8vvW27rv8AmeSap+yV4B8fftLx/CD4WarrZXQ0ln8WeI9WuEuE
+tghVTBbxhEzIGcKS2RubuFavWZ/+Cenwv8Y6l4q8I+Fz8QfDvifQok8vxBr9qW0q/kYf8s2KBZVB
+4bZtIzxmszxX+138JPAv7S2m/F7wLfT63Br+nnRvF+jxWEsE6oCjw3sRkVVeRSgRkDfMoGOea6rx
+J+1b8D7LVPEnjKP4wfEPxJJqkYe18GafdXdpHZSYGRHhI9mSo++5VdzYzmuerWzdqn7NVEuVdG3z
+397m0+69lY1jDB+8nyvXv0PJPgB+w1oXxL+BPjvxB4hGvr4+8O6jqulJpelXSeS9zap8kewxlmJk
+44IzxWd8X/2MvD3wt/Y60j4lXa+IdP8AHskGnG/0vUZVSCCaeREmQw7Ay43HALZGBmuy/Zy/bI8F
++Av2ffiNp+t+IrvQ/HmuaprGq2MSw3F3KktxGDATcbCGfeB8znORk9ay/jf+1n4R+KX7D+jeCbnx
+Pd6z8Szb6W2ox3NpMGe4iljactKVCHG1uQxz+NdiqZx9eSkpez9or6P4evly/Mw5cF7J2te3f+tT
+pviZ/wAE5vCGi+OvhXoXhS88Q3UPiLUZf7amvbpH+z2EUHmSujLENjklUUnI3Oo9axfGn/BP3wjD
++1H4T+GmheINWsNCvvD9xr2o3V9NHPdMIp1i8uA7FUE716g4AY88CvVPi9/wUL+Hd5F4G07wb4im
+mtpNdsj4gvm064ia10uNxJOFDICxkKIhCZO0tweK4X4jftK/Bj4hfth/Dz4hTeMru18L+FNFnZp4
+rC7R570ynyodoQPt2uzscbSECnO7FcGHrZ3yp1ede7P7Lbvra6tv0S8rm84YHmsuXddT1L/h1J8I
+/wDoOeMf/BhB/wDGKKo/8PVvh1/0L2sfkP8A4mivNtxR2n+B0WyzyPy2or6m+Ef7B9/8WIfiNJD4
+2ttN/wCEM1y70STdprzfazAuTKMSDaGz935iPWvmvwdoj+LvFnh7QjIdPm1i+trJZJoyfK86RY95
+XgsBuz747V+qUcbh68pxpyu4Wvo9Lq/5dj4+eFq01FyW+xm0V9KfGj9hrxP8Lfif4E8B6LrUPjHX
+PFyzm22WhtEtxEV3vIS7/IFYsT1AQjByK72X/gnHZ3Gs33g7SvjLoWofFCwsVv7jwy1kURI2xglx
+IXAO5OSh4dSV5FcUs5wEYRm6mkldaPa9rvTRX6uxusuxLbXLt6HxdRV/xBoGo+FNf1LRNYtHsNW0
+25ktLy1kIJilRtrKSOCMjgjgjmneGdHPiPxNoujLMLdtSvoLETMu4RGWRY95HfG7OPavY5o8vPfQ
+4OSXNyW1M6ivuW//AOCX66b4o0zw5dfGLSYdc1OGaezsH0dxLcRxbfMZR5/IXeufrXGeB/2Arzxn
+8c/iD8NT46gs5/CFvZTvqQ0xnW7+0x7wBH5g2bemcnNeJHPcunGU41dIq70ltdK+3d2PQeW4mLSc
+d9N1/mfJtFfYGo/8E6r+b4XeIfEfhb4jaL4x8QeHpriDUdF0yEmJZYPmkt1m3k+cFx8rIOSBx1rl
+/gn+xJd/ET4VH4neNfGun/DXwGyebb319CJZZot2xZWDOixozEBcks2c4wVzp/bOA9m6ntNE7bO9
+3skrXd+llqL+zsTzKPKfM9FfSPxq/Ye8TfBvxb4LtX12013wl4r1S20qz8SWsBVbeWZgF86Lceqk
+upViG2kccVv/ALRH/BP7Vf2f/hTrfjj/AITW08TQ6PJAt1YQae0Dokjqm4sZGxgupxjkGqjm+Bk6
+aVVfvPh311t20172F/Z+IXN7u258n0V9geKf+Cc2reEvgvq3j+68eWbS6Zoh1i50Yaa29GEHmmAy
+ebwcHbnb6HFcT48/Y3vvA37NOi/GE+KYtRt9Tt7G4XRYdOcSR/aSuB5m85K7uTtGcUqecYGrbkqX
+vLlWj+Ltt/wAll+IjvHpfpsfOtFP8mb/AJ4S/wDfhv8A4mivYuu5wcp+of7H3xK8M6C3x/EvjPw9
+ol7qHjnU59Nn1DUIUjlVlHlzKGYeZHnByMg4OKqfGr4meEZ/hF8P9J+IHjzwd45+K8PiXS57bVfD
+skQS226jG0kxKn91GtuHVmbaCe3p+YDW8UmN0SNgYAK5wKVYY0UhUVVPUAcGvkXw5SeJ+sOo909l
+fRWtfez6rqe8s1kqfJy/1/mfqP8AtD/tKeCPh/8AtYfB3xaNc0/XPD0GmappupXekXKXhsVmaALK
+wjLcAqCR127yAcYr0HxR8RrSbxfqXjaw+O3gHQfAE1gjwPb2drcaismxQQZmkO9Tjdt27ui44yfx
+3SJIshEVM9lAFILaISeYIk8zrv281i+F6PJTjGo04rlbsndXb2d0nruV/a8ru8dHru0faHgT4L+B
+f2uLj4x/ETWvHOpRaxYXlxd20Ua2tnJd2cNrHtu5bcglFZhgkYGOvzE4+UPhhdpH4/8ABd1culvE
+msWE00jsFSNRPGzMxPQDnJPQA1f8P/EvVfC3gHxZ4U0yG0t7fxQYE1PUPLJu3t4iWFqr5wsTMdzj
+GW6E44rkjg5B5Br6DDYWrS9rCc7wdlFaaJK3Tz/roefWr058k4x95as/Xjx/8V/A91+198I9Vh8Z
+eH5tMtNB12K4vY9VgaGB3+y7Fdw2FLYbAJBODjoaofs6a1p3iL9uP9oTUtI1C11TTp9P0Uw3llOs
+0MmISDtdSQcEEHB4IIr8kxbRBWXykCnkjHWvUfgR+0T4x/Zwv9Yu/BZ0yObVYoobn+0LQzjbGWK7
+QGXacuc+vHpXzFbhpwwsqdCpeThyK+i+Pnv+h6sM2jOonUjZXv8AhY/QfwD4m8AfsrfDT4za/ffE
+TQ/ET6/4i1PU7TTtPlC3PmtmNLQw5L+aHUq3AA6nABNee+DfHPhD9qv9he1+EH/CYaN4J8b6bYWd
+hJba1MIYmNtIhWVckb43VB93JVicjjn8+tf1ebxL4m1bXr1IRqWqXc19cvCgRTJK5d9o5wuWOB2F
+Z8sKTLiRFcDswyK7o8PRt7SVV+15oy5rLRxVkrbNGLzX3rRj7tmj9KP2mfj94Cs7H4MfCXw/4jsP
+EV1pPiLQ5tU1W2mVrWygtJIxl5QSgZiASATtVW3Y4z6Br/jr4e/Ff4hfG74fa/460ODwxrmmaPcW
+t6dSg8lm2yJMsbl9pZTDCSAcjfX5MKiqmxVUJ024GPypv2eIqF8tNo5AK8D8Kn/VukoRjCo01fWy
+vdyUr/hb0J/tWTbvDTt5Wt+p+pnij47+G/HH7P8A+0utx4p0X7ZcajrFjpNq2oQiW6t4rOGCIxJu
+y6uY2Klchs8Vv+AfiYB+yR8NdJ8EfFXwR4Q8YW2k6as8mv3kMqxRrGvmxvEXBD9ucEYI46j8lfJj
+3K3lrlRgEqOKabWFyWaKNmPJJUVMuGqTh7NT05ubZPZWtZjjm0k7uPS343P3i/4Xj8PP+ij+DP8A
+wbW3/wAdor8HfssH/PKP/vgf4UV5n+plL/n8/wDwFHR/bcv5PzHefF/z0X/voUefF/z0X/voV9z/
+ANl2H/QOsf8AwEj/APiaP7LsP+gdY/8AgJH/APE19N/a6/k/H/gHzdkfDHnxf89F/wC+hR58X/PR
+f++hX3P/AGXYf9A6x/8AASP/AOJo/suw/wCgdY/+Akf/AMTR/a6/k/H/AIAWR8MefF/z0X/voUef
+F/z0X/voV9z/ANl2H/QOsf8AwEj/APiaP7LsP+gdY/8AgJH/APE0f2uv5Px/4AWR8MefF/z0X/vo
+UefF/wA9F/76Ffc/9l2H/QOsf/ASP/4mj+y7D/oHWP8A4CR//E0f2uv5Px/4AWR8MefF/wA9F/76
+FHnxf89F/wC+hX3P/Zdh/wBA6x/8BI//AImj+y7D/oHWP/gJH/8AE0f2uv5Px/4AWR8MefF/z0X/
+AL6FHnxf89F/76Ffc/8AZdh/0DrH/wABI/8A4mj+y7D/AKB1j/4CR/8AxNH9rr+T8f8AgBZHwx58
+X/PRf++hR58X/PRf++hX3P8A2XYf9A6x/wDASP8A+Jo/suw/6B1j/wCAkf8A8TR/a6/k/H/gBZHw
+x58X/PRf++hRX3P/AGXYf9A6x/8AASP/AOJoo/tdfyfj/wAALI//2Q==
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="vy.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <8745601d0b859e929606101eb4ea59@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAjwBx
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+9u/4Zj+JP/QEtf8AwYRf40f8Mx/En/oCWv8A4MIv8a9D+Fv7ZV/4t8beGNH8UeCT4X03xdG8vh/U
+FvRP5wUkBZF2jG4rgEdyvGCGr0f45fHKT4W3fhjQdE0ZPEnjHxLeC10/SjP5I2D780jBW2ovHOO5
+PRTXrx4nqSpuorWXk7+Wl766WOivwFi8NjIYCpB+0mm170GrK/M+Ze6lHlfNd6Wdz51/4Zj+JP8A
+0BLX/wAGEX+NH/DMfxJ/6Alr/wCDCL/GvuGAOYYzKEEu0b9n3c45xntVXV9XstB0y81HULiKzsbO
+F557iY7UjjUEsxPYACur+3cTvaP4/wCZ82skoSkoxcm32t/kfFH/AAzH8Sf+gJa/+DCL/Gj/AIZj
++JP/AEBLX/wYRf40eN/+Ckv2fVJU8J+EoLnSo22pfaxctE047Msar8oPUbjnHUCvTP2ev219I+Me
+vxeGta0lfDfiG4UmzKT+dbXhAJKoxClXwCQpHODg54rz4cWOc/Zxcb+j/wAz7nF+F2b4LBvHV6El
+BK796LaXdpa6de3W2p5n/wAMx/En/oCWv/gwi/xo/wCGY/iT/wBAS1/8GEX+Ndf8ff24NQ+C/wAT
+dY8KweDrbVotPhhlF3JqDRF/MiD42iM9Mkde1elfHr9pWw+CPw/0jWHsI9T13WAn2HSfP8sONqtI
+7Pg4RARzjkso75Gv+tFT39vd30f+Zwvw8x6+q2pt/WdadpR1Vk7v+VJNN81rL0PBv+GY/iT/ANAS
+1/8ABhF/jR/wzH8Sf+gJa/8Agwi/xr3T4R/HjxH8Q/hBr/j/AFTwnbaHY21tPcabbi8aRr1YkZmc
+koNqFl2qcEnBOMYzwP7Pn7a+ofG74k6f4WuPCFtpEd1azXP2qO/aYrsUNjaY1659aX+tFT3Fp722
+j/zEvD3GuOKnGN1hv4lpwfLZXfrazva9npucT/wzH8Sf+gJa/wDgwi/xo/4Zj+JP/QEtf/BhF/jX
+pP7SP7YkvwK8d2nhuw8O2niCR7Fbu4d74wtbszsFRgEbqq7uexr2f4QeP5Pih8MvD3iuWxTTZNVt
+hObVZPMEeSRjdgZ6elaQ4lq1KjpR5brfR/5nFiuCcRgsBSzOvFqjV+F80ddG9lqtE90fJ/8AwzH8
+Sf8AoCWv/gwi/wAar3f7N3xNtgvl+GY7olsER6jANo9TuYV0HxK/4KC3Hgv4heIfD2m+DrXVrPSr
+x7Jbx9SMZmaPhztEZAwwYdT0zX0D+z78Yo/jn8NbPxOLJNNumnmtrmzSXzRDJG+MBsDOVKt0/irO
+HE9WvJ0qbV/R9PXQ7cf4f47KcHDMMZSapStZ80X8SurparTvbtufJ/8Awz/8Vv8AoQpf/BvZ/wDx
+dFffP40Vp/bGN/n/AAj/AJHzv9mYP+T8Zf5n59eDfBPjPVfEnwpu/FvhS+8EeG/hXZPPqusaqAsU
+wifzP3WPv52IPlz1Y56Z9E+Dfi7RNS8W6t8fviZq1n4bj1t30vwnaarOsZt7BCQXQHks2TkjpmQ9
+HFb37RmqXnxw+J2h/AzQbh4rA7NV8W3kB5gtEIZIM/3nJQ/Vo+26vPPifougWn7aOj6N4ptrODwZ
+YeE2i0u0vwBaRRrBKAFDccMGx3yq9wK+AcfYSvB8yUkrvbm26dIr8X5H9LwxH9sUOXEx9lOpSnPl
+hrJUVJzaXM371abv1SpxStaR9t6VrFnrenQahp13Bf2FwgkhubaQSRyIejKw4I+lfOv/AAUD8Q3G
+jfAM2ds7Iur6rbWU+P4ovmlYfQmJR75rO/4JzS6g/wAAZ0ut/wBhj1i4Wx8z/nntjLBfbzDJ+O6u
+6/bB+GGofFb4IanYaRA13q2nzxanaWyD5p2izuRfVijOAO5wO9enUnLEYJzitWj8/wAFg8PkHF1P
+C4ialTpVUuZ6LR6N9raX7WZ87/8ABOzwHoviHUfGuvapp9vqF3Y/Z7K1+0xiQRK6u8jAMCAWwgz1
+wD6nPU6z+wTqa/FmfxT4c8Uad4d02LVk1SwsYrF3a2Kur7Vw6gDcDgDgA4HpXhX7KP7SVl+z5qfi
+GLV9MvNR0jVljZ/sQUz288QYAFHKjBDYPIIKjg10HhH48fFn44/H2Gw8M+ItU0fTtU1ATnTYSksW
+n2Kkb2clCBhB14BdgB1FeNSq4Z0KdOcXKV9lv+n9eh+xZtl3EcM7x+PwlaNKg6a96dnFxUVdJWlZ
+pqV3bv8Aza8t+3BGJv2lfFkWdoa0slJHOP8AR17fjXJeP/ihD8XPilp+u+LIrm28NRPa2TWNpJuk
+tdOQqJEjb+8Rvcn1Y46DHW/txSmH9pTxbIACUtLI4/7d1rc/aX/ZqtPhD8PfAfiHRjJd21xbrZ6x
+cuD+9unXzEmIz8qt86Y6AKg5JzXLWhKVSs47J3f3u36n1eU4rA0MBlFHEaVatJRg+37uLlrsm7RS
+e99Fuz708d2un2nwU8RQaSkMelxeHrlLRLYDyliFswQJjjbtxjHavza/ZJ8Y2Xw9+Jo8UaiwFjpH
+h++u5ELhWkxGgWNc9WYlVA9TX0r+zF8XP+E1/Ze8a+FL+436v4X0e6t03t80lk0Ehgb/AIDhoz6B
+F9a+NPhL8O9S+K/jLQPCemMYZ9QZRLOOkECjdLKf91QSB3O0d67MXW9pKjVpLXovO60+8+M4UyhZ
+fhc5y3MZ2inaUu8HGT5vnF3HeO7/AMQeKNVHjPxFzd+KpZ76KTP3lSTyiFHZFI2L7J7V+ivwL8Ww
+eA/2NfD/AIjuSvlaX4fkuyGPDFA5C/iQB+NfLf7efhvT/B3xB8DaHpVuttpmn+GUtbeIfwos7gZP
+c8ZJ6k5qT4k/GyHRf2U/BPwst7W5GpapollfT3nyiD7M00jGPru3kxAdMYJ5rOjP6pWqcz1S+96f
+qd+c4Z8WZTlkaFO0J1U7LTlprmX4Rtt12OE+EnwzuPiH8I/jN4puoPtWoaXYQ3FvNIMsLnzTczsv
+vsUg+z17l/wTd8bBL7xl4RkkyJVi1i2XPH/PKXH/AJBrh/2e/wBprwT8IfhLqXhDWPD+talc6tPc
+S39xYrCY3WVRGqgu4PEajqOueteR/AH4pJ8D/idpPimSCe+06zimt7q2t8CSaB4yBjJA3BhG2Ccf
+LWVKpToTo1Iv1/r5/getmuX5hneDzbA1aLSbi6O3vcsVpHXS7h5fGfrzk+lFfNP/AA3b4R/6F7XP
+++IP/jlFfUfW8P8Azo/mn/VHPP8AoGl+H+Z7h4W+GvhnwXrGt6roukQWOp63N9o1G7Us0lzJljli
+xPdmOBgc1U+Inwe8F/FmG0i8XeHrTW1tCTbtOCHjz1CupDAHAyM4OB6V+RsX7YfxeaO28v4ra3fQ
+b2S22PapNqM5HJAWFtsC+vXH4Uv/AA1p8YbeKXzfixrZt4pf9MvY1iYPKfu20AZFHXILe3sawdek
+48jhp20/r/gnHFY6FdYmNdqovtcz5lZW3320328j9k9A8P6b4W0e00nR7KDTtNtIxHBa2yBI41HY
+AfnV/Ga/F2b9r/4z3E1xBJ8V9St7uRPOucTWaQ6dDjgEiJiZCOp/HHIFSQ/tdfGdpbe4tPiLr0zT
+r5enWcyozz+txMDEoCDsvHp1zi1ioJKyOSeCqzk5Tldve97+fz/Vn6n+Nf2Y/hf8QtYl1XXPB9jc
+6lMd0t1CzwSSt6uY2Xcfc5rpPAPwq8I/CyxltPCnh+y0OGUgym3j+eQjpvc5Zsdsk9TX5ED9sH4v
+QW5eb4t61Nptk+JrqNrJJby4PHlRDY3yg5H4HsKcv7V3xrllltx8UtVjvph9oupAyPDpkHXbnyQp
+Ygdzx1xyKzVWhGXMoa97I9CrUzOvQWFq4mUqa2i5ScfKy29Pv2P1Y8Zfs5fDb4g+ILrXPEPhKy1X
+VrlUWa6mLhnCqFXowHCgD8K6rxN4D8P+MvC0nhvW9Lg1LQ5USNrKYHYQhBT3GCoIIOeK/H5v2xfj
+IoGo/wDCzdbWFh9n02wlS0ja7bvK+5SQOhPHAwOp4af2rvi3zaSfGHWAtiPO1PURPC6g9RCgjgx3
+55znCg96FWo6+5vvovx7kyeYT9mpV5P2fwXlL3bWty/y7aWtZJeR+rPhr9m/4a+Drm9n0XwlZ6fJ
+e2cun3BhaTEtvJjzI2BbBBwPyp/gn9nf4c/DrXoda8N+FLLSdUijaNLmAvuCsMMOWPUV+VbftefG
+xdsqfEbXINQ1NfK07T5ILfbBF086QuoJYZznGCSB2NVn/a/+KwkdZfi5rX9m6UuLq8F3abrqboY0
+CQnIB+XjOWJ5wKlVaCtaG3kv63NqlfNaimqmKm1NWlectV2euqs+uy9T9Z/H3wL8B/FDVbbUvFXh
+q01q+t4fs8U1wX3LHuLbRhhxlifxr5Z+N37L+u+I/wBoTwdBoXhJ3+HNlbadp87xzIIYbZJnaaPB
+cORtY56n5uDXB/sV+Kfjh8fvHZv9W+I+rL4Q0x1n1FI1Uq3eK0R2jGWYDc7DJCg8jctey/Gb9oTV
+9T8WvZ+E9ak03SdP3Qme2KE3cucM2SD8gI2j1OT0Nc+Lnh3T56kba9LXdv0PRyfPMzyatajU50ot
+JScnGPN1SurPtY9Y/wCGQPg4ef8AhAtMz7mT/wCKrwr9rH9j61PhvRH+E/geJdQ+0yR30djIFZoj
+GdpbzHAwGA6etY9v8RPil4isdXudN8ZaksOmWzXMzgR4OBkIPk+8Rk/QV5f/AMNF/E6PUGT/AIWB
+qZQ4ZcmH/wCIrlWIweL5qUYWfklc3wGf55leJp4x4mVTlfwzlNxeltVfX/M0v+Gbvi1/0J97/wCB
+Fv8A/F0U3/hoT4lf9D5qf5w//EUVHsqHeX3o+n/4iHmv/Pil90//AJM6o+E9FRpHOi6bHKibZrmO
+yXbbR/8APJOByRioV8FeHEktlTwvokcqD/RIHsbfFuveRsqcN/n66pjVFhZLf92GJtreROZm7yOW
+PT/PU0rFI1mje4ITcPtcyMN0jZwIkCgk88YH0r2bI/L7vq/6/r+tjLTwn4b+ysqaBpstk0m4xizU
+m9mznJwgyoPP6+lOl8H6C005m0PSVuGUfbLl9PhAhjH/ACyUvyOP5+pr3XwF+z1qWumPUvE0jaTb
+suILCNi88aHsxPyox78E/Su11PQfg58IrFE8QT+HNGUfvPM1+8i8yUj+L962SfoK3p4edV8sY3fp
+c554mMN2fKEfh7w4n2byNA0eN1+SwtFtYf3frIyonXH+HrTpvBXhu4jmSbQNMmskcyXdw2nrJ9qk
+yDgluDz+Zx6V9KW37QX7O+qXP2WPxp4FaVx5YBubdAR6BjgfrW3cfBf4feOdKhvfD5to4AS0F5pE
+6zW5b12glGA9sda2q4GtRV6kGvVNfmZwxkJ6J/1/X+R8qDwhokNwkqeHdHi1OddsO2ytk+zRHuDt
+JB/mcntSDwl4ae28v+wtMk0q1ff5f2QN9om55xtAJz0P1Nd745+HWtfD/UPst63mRXpY/wBsRbYo
+WTugGMq2B09BkZ5rl/tcUiJc7VltYT5dpb5dvMb+9jgdvwA964uWztY61K6umZD+BtENwzHw3pH9
+q3QG9306DNvFzgBnz2yB9Se9aPhj4ZaP4r1ew0TSPD+keQHxFGttEELD70rBU+6ozj/Eipmt5Yi1
+uFc3Uw33UvkhSoPRRk8E8DHpX0n8NfCsXwp8HS6tqUbDV7tAq2zFcxjqsQwMA92+n+zTST1eyJlN
+w9f6/pljU7LSvhj4Rg8J6BbxW7SqWuGiQKTu4d2x/E+Mew+grjtD8HafqmoQ2sWk2TkjhPs64z0G
+eOgGSalurqbULya5uX8yeZi8j+/oPQAYro1juvCnw28R+IYD5Wom2ItnYf6sdA35nP5VzwlGdVOS
+0Wy9DGSlGD11Z1C+CdK8M2S2tpYW6W0uWkCxKA7kYYn6jtXz/wCIvBGkaVq9xbnR9PzDIVU/ZU+7
+1B6ehFepfBDxfqXirwXf2OsXD32o6TOqi7kOXmicZQse7D5gT7Csn4p6YUvLe8UfLMhjY/7S9P0P
+6VwuMKOKbitJa/1+JvHmlTs90eb/APCP6T/0CLD/AMBk/wAKKseY3rRXq8vmjl5pdjiRDMk8q5Ed
+0y5nlaJEW2TrgFiecf49TXvH7P3wyja1tvFWpW7rHydIs5sERp/z8MAMb3/h/urz1Y15B4G8JDxp
+4m03RkDjTZrjdPMbcZnC/M5JY5xgH8Tn0rpP+Cinxxk+DfwPi8OaFP8AYde8Vs2mW7wfK1tZqoNx
+IuOh2ssY9DKCOlenl2DnjsRChT3k7end/JGWMrrD03Jng37YP/BRLVr/AFrUfBXwl1AadplpI1ve
++KoMNNcyA4ZLQ8hUByDLyWI+TAG4/BV9PNql/LfX081/fSsWku7uVppnPqzsSx/E1GiCNVVVCqow
+FHQClr+icBl2Hy2kqVCNu76v1f8AS7H5dicZVxM3KT07CFVYEFQR712Hwp+LvjD4H+Iotb8E63No
+10GDS2ysWtLpR1SaH7rqfXG4dQQRmuQorvqQhVi4VFdPdPY5adSdOXNB2Z+2/wAE/ir4e/bB+BSa
+o9v9kkuN1lqenq+XsL1ACQrfiro3dWXI6ivB9btrnQtXvLfUVQ3llO9lBbSTswyD9/Ax1GG+hArx
+f/glV8QZND+MfiXwfLMRZa/pf22KLqPtFu4GR6Exytn12L6V9q/EP4X3Hin402xheeO1vLJJbmaM
+qot1RirkHruYbQP/AK1fz5xBlyy/GypU/h3Xo/8AJ6H6dleKeIoKUvn8jF+Afw1i1G5bxLqiwvYW
+rl4mdDiaYctJlv4U9hjI6/LXU+LfET+JtV81cizhylunqO749Tx+H410PjfUY9OsIPD2lxiKyhRU
+mKHoo6Jn9Sf/AK9chb2MtzPFDCpaWVtqgDv/APWr5HET5P3Z60PffOXfDOg/2xfgyA/ZYSGk/wBr
+0X8a9G1Hw9F4p8Lapo8sjQRXcZi8yMAlMjggH044qvpelxaTYx20WDt5dv7zdzTbuG8S4S70+7MF
+xGNphlJMEy+jDsfRhz9RXk0cRFYhSlstC5xcoaHn3wi8A654B8U+IbPVy1zDPYq8V9GuIZdrtwAP
+ukbuQeefSuj8c6b/AGj4euQozJDiZfw6j8s12On+JINWjntpYzZaikZaSzmI3f7yn+Jf9ofp0rOn
+jDKVYZVhgg9x3rXHtRnCUNVb9SqDcuZSPnzyx6L+dFem/wDCsbX+8PzorP6xSNuR9zmf2Y/Dq3uq
+6r4ieKOOG3iFjaRruPl7sNIxLfxECPOOnSvgD/goz8Sv+FgftN6pp8Ewk07wtaxaPEFPHnY82c/X
+c6of+uftX6aeD76L4SfAm48Q64xQWWnz61fK7/dGxpNgPsoC/hX4c6zr174r1zUtc1KQyajqt1Lf
+3TsckyyuXb9WNftnBWD5q1TEy+yrL1f+SX4nwufYi0VTXX9CpRRWtovhi+17TNe1C1jzaaJZre3s
+pHCI0yQoPqzyKB9D6V+uOSirs+IUXJ2Rk0UUUyT3f9hXWn0L9rb4dSq20XFzcWTe6yW0ox+YH5V+
+xHjrxUnhXSPMQb9QuMxWyKAWJxywB67Qc479O9fit+ybP9n/AGofhW+cf8T+FP8AvpWX+tft1r3h
+bS/EywpqdnHeRQksiyZ+UngkY71+Q8ZxtjKcl/L+r/zPuMkf7hp9zxXStYN1PGl5cRoZZNrag3Ee
+49TIDyjex4zXsNl4U0+yZbmzQLKYggfOQw/vfU+orAv7H4daLqD299caJZXsahXgub1EkAIyNys2
+TkYxmrukeLvAuhWv2ax8R6Jb2+dwjGpxkL9MvwPYV+fVMNOvDllFtejPoFUjF3TNaSFkbDLtPvUL
+IRUbfETwg64bxRohHvqEP/xVRnx54LPXxLoZ/wC4jF/8VXhSyavze5t5p/5HTHFQW426sYLt4Hmh
+WR4HEkTHqjeoPanSrkUn/Cc+Cj/zMmh/+DGL/wCKpD438En/AJmTQ/8AwYxf/FVP9j4rb/P/ACNV
+i6SIdnt+tFS/8Jr4I/6GPQv/AAYxf/FUVP8AYuJ7fn/kafXqf9f8OfNn/BTj4jp4L/Z1Xw1ayLFd
++Kr6LThGvGLaP97MR6DCIh/66V+TBOetfXn/AAU9+JP/AAmH7Qlr4ZglL2PhTTkgZQ2V+1T4llP1
+EYgH4GvkOv6Y4bwn1XLad1rP3n89vwSPy7Na3tcS0to6BX1h8NPhrJof/BO/4xeNp4tsviPULG1t
+yy9bW1vIhkH0MzS/98CvlGO2uL2WK2tIzNeXDrDBEoyXkdgqKB7sQPxr9Z/2n/htb/Cf/gnTq/hC
+2CldF0jT4HZekkouoDK//AnLN+NPOsZ7GeGw63nUj9ykn+dh5fR541avaL/I/JaiiivpzxT0f9mq
+5Fr+0b8LZW4VfE1gCfrMo/rX7v8AevwJ+DFz9i+M3w+nzjy/EenNn/t5jr99MnNflHGitXoy8n+Z
+9nkbvSmvM/EL9t26g179rT4mXEkMcpj1GO13MgPEVvEmP/HTXiH9n2v/AD7Q/wDfsV6N+0Fq41/4
++/ErUVOVuPEd+VOc/KJ2UfoorgOa/TMFD2eFpQ7RivwR8xiakpVptPqyD+z7X/n2h/79ij+z7X/n
+2h/79irGOPT8KK7Dm55dyv8A2fa/8+0P/fsUf2fa/wDPtD/37FWMc9KSi4c0u5B/Z9r/AM+0P/fs
+UVPRQHNLubvjvxndfEfxx4i8WXuftWu6hPqDqf4RI5Kp/wABXav/AAEVh9aBx06Un+RUQgoRUI7L
+QJyc5OT6n0J+wV8Mj8Tv2nfDCzRGTTvD4fXbvjI/c4EIP1meP/vk1+j/AO38u/8AZA+JA9LW3P5X
+UJrxX/glJ8Mf7H+G/inx5cxbbjXr4WFo7Lz9mtsgkH0aV5B/2zFe3/t6Jv8A2QviUP8ApxjP/kxE
+a/JM2xn1nP6UE9KcoR/FN/jp8j7bB0PZYCV95Jv8D8WKKKK/XT4Y3PAVx9k+IHhObp5etWL5+lxH
+X9A7kKCSdo9ScV/PPosv2fXtJmzjy7+2f8pVP9K/oL1pc2DZ7Mp/WvyjjuTpKjVXRS/Cx9nkC51O
+Hmjxm+/Yl+A+pX11e3Xw60me6upnnmlaWXLyMxZmPz9ySahX9hf4AscD4a6OT6CSX/4uvUMD0FXN
+HQfbgQOik1+W0eIsdOcaaqSV/wC9I+tnl9GKcrL7kfmR/wAFIvgz8Pfgrd/D+w8DeF7Pw5cait7P
+etalyZUTyVjB3Mehdq+MK+3f+Csmped8Z/BFhuyLbQJJivoZLgj/ANpV8RV/RWQynLLaMqkm5NNt
+t3erZ+Z5nyrFSUVZHVfCXw7D4w+K/gjQLmEXFrqmuWVnPC3SSN50DqfYruH41+v0/wCw3+z/AG4B
+l+G+jRgnALySjJ/7+V+W37HumHV/2p/hdABnZrKXBHtHHJJ/7LX7G/FDX9O0WDT4tX0ptS0y5dxL
+LGMvbEAbXA6nrjgg/WvjOL8biMPXpQoVHH3b6Nrr5eh72S0YTpSc1fU8z/4Yq/Z3/wCifaB/4Eyf
+/HKK1fN+GX/QXvv+/Mn/AMbor87/ALUzL/n7L/wOR9L9Vw3b8EfiVQsM9y6QWsRmupmWGGNRkvIx
+2qo+pIFFe+fsKfDiP4l/tQeEba4RXsdGZ9duFbHzeRtMYwev754vwBr+jcViI4WhOvLaKb+4/L8P
+TdarGmup+rnw68EQfAz4B6D4Yt9ofRtJjtmZTw9wV/eOP96Rmb8a4v8AbbQn9jX4hr1I0eM5PtJH
+XpXxNvT9ntbIdGzM49QOB+p/SvO/20V3/sdfEYf9QPP/AI8lfz/gqkqmPpVJ7uaf3yR+k1YqNGUV
+0X6H4q0UUV/RJ+XieYYWSQdUdW/Jga/oTu5vtWhxzdfMRH/PBr+ee8bbaTMOyE/pX9BOjz/avAel
+zf37CB/zRTX5Xx7G+HpPyn+SPseHXacvVFYHitDQxm5lPon9aza1dBXP2g/Qfzr8GwSviIf10P0C
+vpTZ+T3/AAU31U6h+1Tc22ciw0OygA9NxlkP/oYr5Tr3/wDb41b+1/2uviA56W72loPolpFn9Sa+
+f6/rfKYcmX0I/wB2P5Jn47jpc2JqPzZ9I/8ABOzTRqP7XfhFmXctpaX91z2It2QH85BX6x+LfC0X
+irxRpC3LH7JbQyyyRj/lr8yYX6cc1+Zn/BLTSRqH7SmpXZx/oHhy4kH1eeBP8a/VKYf8VFB6fZJP
+/Q0r8r4xqXx9l0il+Lf6n1uTK2GXqzC/s2L/AJ42/wD3wP8ACit/yR6n8hRX5ryvufU8yP/Z
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="lomuahoee.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <a881401d0b859392a08d6067ee3315@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAdwBx
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++Kq/R3/gk38MvI0fxr8Q7iLD3kyaJYyHr5UWJJz9C7xj/tlX5wSMUjZlUs2OFXqx7Ae9fuj+zl8N
+x8EP2ffCHhmWILeWGnpJejGC13KfMmz/ANtHYfQCv2bi7GLD4D2Kes3+C1f6fefEZLQ9pX9o+n5s
+f4u1FdS1u8YOCkQ8lec8L1/8eJrnP2xl3/sffEYf9QBj+QU17jHY28WSkESk9cIBmvHv2y49/wCy
+t8UV9NCuD+S1+UYCSeLo/wCKP5o+xrL93L0Z+H9LSDoKWv6OPy0r6h/x4XHtGx/Q1+/vgSf7X8I/
+DM+c+bo1nJn6woa/AW7G60mHqjD9K/en4Ozfa/gF4FmznzPDenvn620Zr8y47V8JTf8Ai/JH13D7
+tUkvQ0t1bWgD9zM3+3j9P/r1iDoK39CGLIn1c1+B5er4hejP0LFaUz8Pv2rdR/tX9pz4pXO7eP8A
+hILiEEekeI//AGSvLK6L4laqde+JfjPUi243muX8+713XEhFc7X9g4aHs6FOHZJfcj8WxEuarJ+Z
+90/8EltMMvxR+IWo7ciDRra33enmTs2P/IYr9K5dg123JkUSGCRVj5yw3Jk/QcfnXwL/AMEi9NA0
+n4paiV5e7sLVW9ljlYj/AMiCvvl2J1e2P+zKP/Qa/DeK6nNmtRei/wDJUfe5VFrCQZLt+v5Gipto
+9P1or4ux7XMj8RP2Pvhl/wALa/aR8E6JNF52nW12NWvgwyvkW2JMH2aQRp/wOv2r8RyH7GIR1Ygn
+6A18E/8ABJv4YmPT/G3xDuYsNcSpoVi5/uR4lnI+rNEP+2Zr7s1iXzZ5T2TCj8DzX2XHWYe0xXsY
+vSNo/N6v/L5Hj5Fh+SipPrr/AJHSV49+18nmfsv/ABSX/qX7v9Iya9gHSvJ/2r4/M/Zo+KK/9S7f
+H/yC1eTgdMVS/wAS/NHZW/hy9Gfhev3RS01DlFPsKdX9Jn5aMlG6Nx6giv3R/ZquTf8A7MXw4lzk
+t4YsRn6W6j+lfhgRlTX7dfseXBvP2TPho/X/AIp+FP8AvlSv9K/OeN43wVN+b/Jn1OQP99L5fmd0
+CcVsC9XSPDNzeucJBBJOc9goJ/pWNyBzwao/GDUxoHwI8bagW2fZfDt9MG9CLdyP1r8FyaHtMSo9
+7L72j9Dx8uWlc/Bpbg3Za4c5ad2lY+7Esf50DpUVonl2kKHqqKP0qav7A8j8Vluz9Sv+CT+kfZvg
+Z4p1Irg3viOVAfUR28C/zJr7EifzNTtDnkmf+Yr5q/4JlaUdN/ZS0qfbj7dql/dZ9R57Jn/yH+le
+233jeXw3qA+1aJcXWnKpZb+0IkdC2cgpxx06H8K/nDiGsnm9Ryf2pfhZH6dl9N/VIpdkd5tPoKK4
+r/hcvhf/AJ/Ln/wBm/8AiaK8e0Ds5ZdjB/Zl+F5+CXwB8H+Fpo1j1CzsFlvgAObqUmWbJ74d2APo
+BXW3KnyXPc8muiuYHnQKMAZyc1Ul0lpYmXeoJHXFefmjxGNxLqcret2/Nu7NMM4UYcpqV5f+0+nm
+/s5/E9f+pb1A/wDku9eoA8Cud+IPg+D4g+BvEPhi6uJbS21nT59PkuIQC8SyxshZc5GRuyM19Dh5
+qnWhOWyaf4nHVi5QlFdUz+feL/VJ/uin1+mS/wDBI/waiKv/AAsHxIcAD/j3tv8A4inf8OkfB3/R
+QfEn/fi2/wDiK/af9acq/nf/AICz4d5Piuy+8/MvtX7U/sLut/8Asg/DVZAHU6Y0RB7gSyLj9K8K
+H/BJHwd/0UHxJj/rhbf/ABFfV3we+Etr8Efg7pPgSw1G51W00m2miivLtVWWQPI8nIQAcF8cdhXy
+XEmc4LMsLCnh5XkpX2a0sz2crwNbC1JOps0bF14He2TOkalLZADC21yPPgwBwAGO5R/utXm/7Zuo
+y6D+yP8AEmV2VZv7Ea2Yx9N0hWM4z2+avL/BXx68XeHls7d9Vi1uAukQgvIXc8kKAJR8355r6E+O
+/wAIrb47fCzWvA97ql1o9nqnlCW6slRpVCSpJgBgRyUAOexr4DL/AGFHF06zVkpJv0TTPo8Qqkqb
+he+jsfgxjHFL071+mf8Aw6S8Gn/moPiT/vxbf/EUH/gkj4OII/4WD4k/CC2/+Ir9s/1pyv8Anf8A
+4Cz4P+x8V2X3nun7Cun/ANk/shfDlNuwyadJckf9dJpJP/Zq9MXlQehIqPwh4U0/4NfCLSfDkN1P
+cab4e0uOzFzKm6V0jQLvZUHU4ycDvVfSdYsNdthPpt7b38Xd7eQPj6gcj8a/njiCr7fFyqx2k5P7
+2fpWXRVOlyvpZFn7LF/zxi/74FFP49aK+Y5j1tBNP8UaHqtwtvZa41/M3Ajt9ShZj9Ar5rX+zhhz
+Z374GPnusf8As9flp8RfgjZeCfD513TZbqI20sYZbnYG+ZsK0bpghgea+0P2RPiD42+LvwwW41PX
+lil0uf8As/7W1qJZroKOGkZj97GATjk5J5r62jipzqujVhyu197ntZvw5g8PlkM4yzEutSc/ZyvB
+wala+13fT81vrb3o6aN3/IJLEDAL3Wfz5NM1TxDa+FNOSbU5dO0axXEazX18sUe7su5hjPB79qq+
+L9Uk8G+E9Y1+91G7ms9Ks5r6aKFIld0jQuQCRjOF7mvzU/aN+LHjf45eH7PxPq+l3Gj+ARcPb6Nb
+TNlZp/LJMhJAMrbQRuChF5VeScvGYyOFjteXb/Mz4V4Vr8SYhJzVOinZydr3auoxV7uT+5K7fZ/o
+P8QNV1E+Ery803VJomWNbiK4tJeGTrlSOqkHgjqK+fbb4leNE+zyS+LNSeJvm2RxOWIyR/SvffCe
+nxXnwl8L2rfJHLoFpESB0BtlAP4V8+6l4dl8JeJ108zT30trOq+YbkIrnIxx75HFKlN7M+YxFJUa
+k4LXlbX3ET/GLWCkpb4h3asj7DGJ41YHnsWzxj9afD8UNe1e6KWvjjVJMpny4ryEkkL0xnuR+tfK
+/hPwXY+OfiVrOnXpMUfmXc2+ORVbcsnA3EHPU/lW18SfgvpPgjw8+qWuoFbiOWJVt5ZELShj/AQA
+cgc/SuGOLrODqqC5V59j9Xr8IZLh8bRyurjpRr1VGy9lde/tqpd/TzPdPKuIFVtt+kiPvDtIiMGH
+p75rcm+IHi+3lnEfibWLiNVPztqKgBePm6cfWvN/hnqk2s/DrSbvUvsb3ALxiWdWaSTacK7e5XHP
+fFdRcvZSJqP2NrNYxayn/j2cnZt5H516cJc8FNdVc/Mcbg5YLF1cHUacqcpRbW14uzsaEfxc8QSv
+AkPjaWS5diBD/bKsxOflAA659K77w9+094g0HzF14aZqtrFwXMvkSrzg5fG0/iBzX58aI8mjNY61
+Aoj+x3kR3hejD5/5A17r8dNe8jwQIVkaYatcIEZoAgaNf3hIPfPyfnXm0cc50p1Grcv9fmfpWccB
+LAZpg8voVnONeTi5ctuXlactLvaL5t1sz7p8JftJ+AfFssVsmuW1hfuM/ZbuVVJ9cOCVP51p6xof
+gLxbrzW7XGnL4jGSH069WG+GBuz+7YMcA55yMV+ZfwX06TT/AIjXNs0DmaG0lWSOJBuOChKkfpW9
+4V1fUPDn7UFzqml+Za6pDPcGMlFLrmDBBDDGMMR9PStI42U4Qco7yscWM4LoYTG4rDUsQ5Kjh3XT
+5fitst9F5n6Jf8KjX/ocfFf/AIMl/wDjdFeF/wDDQXxB9LT/AMBF/wDiqK9H2dP+VfcflnNL+c+e
+NQ+BvxF8XyWkPijxhZmxQK6SSTSXQiGPlIjRBk46Z6V9efso6LpvgTw/rGgWl1m1tpY/LM0Rill+
+XLyMDySWPXp2rnNPvLueK2v7E6pqdrcW5kjb7TEqMrKeePT6V1XwIhmXx7rE0/nJJNYAqk1wJSqi
+RcHj15riw1GFKo5Ld9z7LO+IcbnGGhhq3LGlB3UYRUY372XXV9er7ns93d6bfwTWtyY7iCVTHJFJ
+HvR1PBUjGCDXyB/wUneH/hXPg2OABI11CZVRVKqFFu2AB6dK+08cV81ftwfBjxb8aPCHhzT/AAlp
+8V/c2d7LNOstykIVGiKg5cgHk1rjoOeHmoq7/wCCLgrFUcHn+FrYiahCLd23ZL3WtW9D03wDiT4Y
+eDjyM6LZdRj/AJYJXmPjC10+X4gyRxJYC9N0gkeaxeVvMONpJzjoRXsXg7RrnRvBXhzTL6MRXtnp
+drbTxq24LIkSqwyODgg81554r0/W7jxtKlvpOtm0W5UfbLQhYypwSw46Ln9K51dSPExEoyrVWno3
+L8z8/fDHgT/hYXxS1/Rl1KXSQk97P9otbVpfuzEbQikEA5Hfium+IP7NWreDtAvdXj1afWobGAXc
+qXFjJHiHAJOWYjgEZB9xmujt/wBnX46+HPG+s654a8O3NlPcXFyI7v7ZArvDJIW5BbuME1pax8Bf
+2mPGFi+marbzSafcgedFJqsCo464fa2SPb2rxo0fcalTk3r0Z/S+Izmr9doVcHmuHhh4qHNGUoOW
+nxfZb1Wi95a9i18JvF93468GNPIJoLixmFlLHYQRrFjYCjKvRfl4I9q7XUILu6TUXEeo2y/Zpn2F
+o1HCfdx71v8Awz/Zf1vwR4KTSrjw/DqGoyTNPdXTX/lLIxAGFAbgKBgeuM+1b9/8CPFeom+c6Rp8
+ZktpRGHuyf3mzCLkV9Dh3P2cVUWp/O+dSwM80r1MvsqLk+X08r62bu12VkfC3hLRG1j4WeMpEjZ/
+sElpd5DgKAu4NkHr8pPSrsGpN8Q9X+H2glmc2qpbXBmmO1mEmS2Two8tFFe+/CH9kP4kaFpPiKy1
+3QtMgW+jWOMzXySnGxwxGwnGMjrXO/Cr9lf4g+CvGIvvEemWWmQ29u8cMlzcJOkrsQuQFJ/hyc14
+lPDVZKlFRdno9O0r6n71juJsrjVzOt7eDnRfPRfMvelOgqdo662a1S2bucT8N0Wb46eIvljA3XZR
+ZJ8KD5i4G7v2+tZHiHXIvCnxz8RaldQJcfZpp8W6sWWSUxAKuf8AePP0r1vwH+z9410f4ra1rF7b
+WFlpk/2gw3hmSVWLOCuI1yy9M8jisHxh+zB448V/FHUrmRbOPRb+8DSauLhBtjIXc4iB3Z4OBjJP
+51tKlXVFOMHzc7e34+h5mHzPJamc1vreJh7CWDjTb5lr8PNFNP4rX0XveR5B/wALa8Vf9BO1/wC+
+f/r0V9Rf8MjeFv8AnnB/37eij+z8b/z8/F/5Ef688D/9Cv8A8pUv8yr8I7++ttTm0gW9sqTxySxS
+3kDP5ZCkuox/eXPGOte3fBW4Nl47uBZWEF7LJp7CVrSIwLGu8FCxbrk5HFfhl4Q+I/irS/FWiXi+
+K9ehMN9CTJDqMm9RvAJXLYzgkc8V+tf7El9rI+Pev6ZqNx4juoIfD8syTa9cRyFy13EMDYSPlC8e
+m5vWvZulVVup+BNN0JN9LH2wt/qrKf8AiVIrds3Q/otNa41hl4sLdWzyWuSRj/vmtcDijFdfMeYc
+Pdaz4mtLmRpvD0N0gOV+y3fzkdshhW8fE2mWqILzUbSyn2gvBNcoGjOOQee1Z3iKZ4b+TEjKNo6M
+Rivz/wD2hfDV9F8b9S1ZLvxdew3V4lx/Z2mvCtsPL8tTGNzbv3m3PTueDXiSxUY1ZRaPWp4R1Yp8
+3Q/Q2Txr4fhQs2t6cFGAW+1IceneqZ+JvhQSCP8A4SCwLkkALMDyOSK/LT+wL6W21K8Y/EGFop49
+lq+o2kbSCR2BUAt/yzx+taEOl6hrGsafFdR+MNDtWtFhaebX7aP5UibbMef4yAuR3atVil2NP7P/
+ALx+mT/F7wdGOdftm4z8m5v5CmS/GDwhE5RtZXIOMCCU/wAlr8yI7fXE0WRWstez5gl3TeLY0YPs
+ICZXgDHOParMtnc6RrF/FpT32rxGzkjzc+NXBMTRAytgdCh3YPbbVrFJ/ZB5db7X5f5n6US/Gjwj
+EisNRnkDZA2WU5yeO2z3FVJfjZ4RnjkRnvZ0ClmQ6dKQR+K1+ZdxfxWumW0xli+1maQTxS+PLkqk
+YC7G4PJJL/kK0L3WpJNb1uTXLvR9NvjHNIkI8XXm1bvK7EYKxAU5bpxxR9bXYpZb/e/r7z781Px7
+4MuTutrXWbVmOA0Wnvs/I1zt34+0+Gd0itdQnVckSC3CBh68tXxGurzyaXbRjWPDTWyPIIyddvpe
+cqXBO7J7cnOM0t5qttNf64mi6t4Xmsfss0jC7uL2ZltFZc9W6jK9OetUsb5CeWu2/wCB9pf8LFs/
++gdff98R/wDxdFfn79qtP+g54U/761H/AOLoqvrsew/7Pqd/zPhu2C/arTdt2+fHnccDG4dfav2C
+/YSn0+f9oHXzpSaMLAeG5AraXNJI3mfa4t27efTb0r8f7cbbq2PPE8Z4GT94dBX67fsa+KY9I+P+
+q3Gu6te6fbXfh6W3trfWrWGyDyJcROxTbjcduTj0UntT/wCXsPmQ/wCBU+X6n6EDpS1zMvxG8LwR
+l317Two9J1P8qij+KfhN22pr1m7eisT/AErr1vseOQeKj/xMJOeir9elfBP7QzeF4PjbfNjw7J4j
+e8gMialqdxHJ5oEf2f5F+UZG39M9a+v/ABD8cvh7datPbjxhpcc64Ro5ZCpBHBGSMV8gfHPxVHrv
+xakHh2TVZEkngUavpWjQXlrOX2eXIs5GWEXQnPGD0xXy1eLVeTa6n1eCknBJdjyy9TwsRq63Nv4J
+XWFnQxK1/dOmTI32jdg47DHvmkku9Ai1vRh4jXwJHpYsYiogguXk+ymN/s4GD13Yz3wDXRRajr1s
+NbtprrxdfzPKix3qaFahkaOVt7Alekn4dKuRSeINF1bQr24XxhrsZtI7kWpsrTLB0cCFhtPKHnGO
+MDgURTe56XMkmv1/4BxGnXlhb6NIsF14IZDcL5mzQbiQCXYecHn1ro7TW9C1PW5bfw7/AGDbs1tI
+cyeFZTmNYB9oXcc9cOQB1BFOWLxRHo/mh/GlyRKIypa0jONp/eYA6Ag/nV6a48SeJtdbzrfxnoMb
+WpJIv7aJAY4AFAHB/elRz0O/uK1SMpO7vf8AH/gGBdalpsmlRLEdIS/hlmaedPBcjB4tq7FAxxgh
++e+fap7/AFS70rxHrUGsar/aN75MtuJrPwXlhdMqlJskdF5/OrbN4it/D0ds1j4ydEupJ1nPiCBC
+7GNR5WQcEDaDt65YnFXpo9V0HWNZtrK38Ra1DJZzW32hvE8YXypEXfLgN8pQ8Z7YOaLP+rj5klb/
+AOR/yMFLzVU0qCWDXdd8vzHjEdv4SjQ7sLkgMMc5H5V0EfiHU9f1nVDpR8WaRbraTXixroMKoYkK
+5gGFGS2eAeDiuX1J7mGyjkW01SVmeRZfO8WKWjTaMMOfc/lTNYv77W9e1efxFpculTmCacRnxYyj
+7QFXy49pY7d3+TVJP+rmbcX2/A6D7R4o/wCfzxr/AOCO2/8AiaK8/wAQ/wDPtbf+Fi/+NFTyy7/m
+VePb8j4cjUme32jcTPGAM453DvX69/sReC0tfj9qV5q+jQ22pr4dlMSPeNerEn2iEB0Lj5WO51OO
+3HeiivZt+8j8/wBD5+TaozS8v1PvpNPtUA228S/RBT1tYozlI0U+oUCiiuw8g4Pxl4e0m7vpftGl
+WNxlQWMtsjE568kV8LfHP4fajH8TP7I0bwda2nhKCVFtmstakslCSsjTsIEyFIYtj19DRRXy1XWv
+JPuz6bDzdOnGUd7HFf8ACvfEMceuWlt4XtXsp5o0lll8RXG4pHKxhP3eM9xjv2qaz+HGt6R4h0a4
+0fw/p0+oi3jkZJ9ZuCFumV1kQZUZUL0OaKK3UVoayxE7N/59vUx7b4YXKaOzr4f0OGRSFXzNTu3D
+w7Tkk7M5yFqxfeCNRTxAr+JPDnhaIfZF2yRXF07HFviDIA6cJkfXNFFXypdDRVZt2v8AmZdr4L1v
+XdAaDT/D3g9rOG8Yqqi4ytyUVSfmxngL7V7v4J/Yi8Ra/cT6i1j4d0XS7yBoFjlhMzmF0AdWIds5
+OeeMZHpRRXi4epUxma4jAuXLCmotctru66tp/hY78XNYTLaOMiuac21reys+iTS++56ZZ/8ABOvw
+ZcQBdSuYWyMNHa2hUEfVnP8AKty7/wCCfvw31G7kvLo3NxeSkNJM6ISxxj09AKKK+jWU4XrzP/t+
+f/yR8283xl9Gl/25D/5Erf8ADvD4bej/APfhKKKKX9j4TtL/AMDn/wDJB/bGN/mX/gEP/kT/2Q==
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="avylevfbyujwws.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <61a3001d0b859092a10290e3af2ce3@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgBJwBx
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+/NVQPOhzj/WoOTgfeGfwr9fP2GbLT7X9orWng/sYXr+F5A40m7kmAj+1Q43b+hzur8hAT50OOD5q
+YP8AwIc1+wH7Et1Ov7Q+pW93LqlzP/wjUrifUdMS0Gz7RANileuDz+Nec/4kPme0/wCBU+R99jOO
+aWm54o3CutNHinIeK/8Aj+k/3Vr4S/aZ0+2u/i239pjwzHpylVWXUdVlguDG3lmf5F4yO34V91eL
+zm7kx/dWvhz9pc61YfFdrqHV9YGnSeS66fY6El2iBBH5qmQ/89OPyNfL1n+/l6s+swf8OPoeL3Gi
+6RDZa8LE+Bf7J3w/anfV7gsIhO32YtxknPp+NSaQmhW/iHQh4dk8BtrXlLxJfzsPtZDhwuRgrtIr
+dhv77UotfntLrxNa2kRjlNifC0PzK8+EVQfvbMjr0AzTHudSu9W0aOxvvEmiSiOK3a5bw1bpmbLZ
+uCeCMggcenFDmoJzm7Ja3PTUef3I3bfr/kM+Gnw7sPiPaavbaBafDu5fS7dLid3up44YbZiVLuzc
+fe2ivbPhj+yNqPxUK6n4p0jRNL0hkSKKa2s3WWaJFCoUBYEjaBhmAGOcGvWf2cv2WV8NWcOv+NpX
+1fWpQrR2tzCkaRKDuTzI0AUkHkIc7Tycnp9NgcVtQwlTGJTqNwh22lLzb3ivJe8+rjsebiswhhJO
+FK05994x8l0k/N+6uilucJ8Pfgh4M+GFpFFoOiW0MsYAF1JGrS/UHHH/AAHFd3tz6Uo6UV9BRw9L
+DQ5KMVFeR8zWr1cTP2laTk+71ILy8t9Ns5rq7nitbS3jaWaedwkcaKMszMcAAAZJPSud8B/FLwj8
+ULS7ufCfiPTvEENpIIrhrCcSeUxGRuA5AIBIPQgZGa4D9qSEal4c8EaPeIJ9D1fxlpNjqts4zHcW
+zSlvKkB4KPIsSsDwwJB4OKfYwW+o/tcXs+mQRKui+Dks9VuIcDdLcXYktYXA6lEgncf3RMMcNz0H
+Oey4PpRXC/8AC49D/wCeq/8AfYooC6P5ydM0q91zVdP03TrObUNRvLqK3trO3UtJPK7hUjQDksxw
+BX7L/Bz4O6r8J/j74T0bTPEcFprd94ZvL/xDZTSvfG0BeBY4YSxxzIc7mxkRtgc183fsefBCP9mP
+4PXP7SHjnw/Lqnim7WOy8C+GnXDma5YRQStn7jTM4AJ+5FubkuAPoT4CxXKft+eObqfUoNTS+0i5
+uPOit5Fy/mWyNtkYnciFGjUdlXHvXDJKMot7nqOpKUJxjt/X+Z9ff2D4m/6GpP8AwWx//FU0+HPF
+JOf+EuUD0GmR/wCNdSpyKWtozuro82x4X4r8OfFJdWuvK8Z6GbUt+6E2kneE7BsHBNfF3xv0vU9N
++JCS+Kb/AE/XfGB8lrs6Zr7aZbEKVNqFtiAQSpwxz8xHvX6MeLRi5YjrtFfBX7VA0ZvjCYxFpSaw
+sVsZprzw9JesxIXyCZV4+XB47cV87WletJPufUYFNRTXbz/Q8wvJIbu51y41C0gi1NZt0VqvjCTM
+kzzYljxnjbk8A9QBX2L+yZ8EV+waX4016waC4gtUtNOs5bqS5AVGYiVmfqQXIHbI3DotfP8A8Cfh
+avxP+K91pV9baDfx21w0l9dWuhNZy28iSbnlR24YtyoOOrg1+ktraRWVtDb28awwQoI441GAqgYA
+A9hSw1BYuvaXwU7X85bpekVZ+ba/lZ0Y3EvCUEo6TqLz0js/nLVeST/mRLg47U4cCiivqj44KKKK
+YHh37Y+rS6V8G4/sNl/aesS6/oy6fZRzrDLNOt/DIFjYgjcVjf8ADJ7VwvwZ+Ntp4Y8BeK/F2t6L
+dP4l1vxBqeoajYW1zFdSRGIxRwWySLxIVtjbKAvGQ3SuX/b0n1XX/FfhfQ9L1K802HQ7U+JL6awk
+EcqrJPFYR4c/cx9onbcOQAcVw3xC8OaL8MNJttV1SXWYtMuLi2s7qx0xfl3qgTzRJ8zsSsaZU4GV
+zkUE9TwD/hdHxL/6JL4z/wDAU/4UV73/AML3+H3/AD6eKv8Avp//AImip+ZmdR8aPCmuaL8TY/D+
+uapd61Y3MulSaBJOpEel2Wn3S3jSZHEk8syxQsWAO1BzyK0fgjJc3X7YenXU0d7AkngRwsNxcI8a
+/wCmEkgDnPqcc19E/HzwcPEnhBNQi81brRriO9Jgx5klusiNPEP95Fz9VFfJHwZ8UaVof7WnhK+l
+tfPv9R8AlEg0m0mnnEbXDSKZFA+RflPzHvgdxXztNOjVeFl9n4f8L2+as11dkm9WfTVf31F4qP2t
+Jf4lv/4Emn6tpaI/QJDxT65BfiLZKMtpWuKOmTpU3/xNSL8RbBmA/s7Wl9zpk3/xNepTjLlPCYzx
+e+26P+6tfFn7S2p68vxJa2stK8RvZJBAyX2l6jDbwuzqu4BXxzHgH8a+l/GPxn8OLqctubfXHuIg
+EeOLRrhiD9duK+Mv2gdfi8f/ABGsZ0stHi0y5EcNrZ+JbC4t9UeRAvnYjOPlICbGPBHSvAxT9lUn
+UqbK7+4+py+Lq8lOO70Prv8AZJ8DNoHhTUtYurq61G7vrqSCG7v3EkrQRsR94cEFt3TjgV74Ogrn
+fh7oEfhnwLoGlxLsW2sokIPrtBb9Sa6Kvcy2i6GEhGXxNXf+KWr/ABZ4mY1liMXUnH4b2XotF+CC
+iiivTPOCiiigD4g/aV137V8UPilC+CYNL8O6OPUI11JcuPzK12/xrhj+weGxuIc+IotzAAlvkm6+
+1eb/ABTtV1vx38eNVIyba+0KLPuttEf/AGavR/jdzZ+HyuML4gQg56HZLR0A0v3X/PX/AMd/+tRX
+Jf6b/wA/rflRRqB9gywrOjpIodHBVlIyCPQ18T/Ca2uNB/bjOgzpPv03wxd2cM3lARm2jnjEce/q
+Thlbn1FfbdfNmo6T9j/bo0m8VPluvDd2SQ+0bv3Izt7/AHP0rxMfHkr0Kq7uL9HFy/OKPYwcubD4
+ik+yl800vykz6OU8daeD71GOgp4p0pOx5bOW8WTyJOyKxA2DgGvjb46aPqN58fdEmOr6s2mXJtYG
+01LONrMEKAzPMfmBbcOndMdK+x/GCYug3qg/rXx78Zf7Gtf2hdGu2MH9tL9iBDaqyusQwVItfutk
+7xuznjNfOZm/3WIv/LL8j7DJ0vbULd1+aPvAAAADgCiiivvT4sKKKKACo5yRby46hDj8qkrL8UeJ
+NP8AB/hvVdd1Wb7Ppem2st5dTYJ2RRqWc4HJ4B4oA+J9WvpL0ftJBolU2PifS7ckZ5AtbU857/Su
+8+Mh8y10ArgBfECE/wDfMtfnF8K/2g4dF0/9oLUtUv5Le58W2Y1TS7a6dj9pvI9QEwizziQwvj6I
+R2Ar9A/Gvia28ZeAvBviO1Rkg1O7t9RhQnlfMiZsH1xux+FLoI18Tf31/Kiq+U/vL+dFaDPruvDP
+EuwftdeESAu8+HbzPyZP3x37V7nXlEvh2y8afGDxSbxJNumadY2kc0ErRSxO7SSuFdSCMrsz7Yrw
+8ya/cp9Z/lGT/Q9XA6Rry7Qf4yiv1PUc04GuNX4Y2Kfc1fX4x6Lq03/xVH/Ct4x93xF4jT/uKOf5
+5rKMkupwWLPjDJnjCgklOg+tfDX7UevaTonxnso5VhttZjtbSVppNFaeR4mBWNVnB+Qhg3OOM817
+940+EWv6x4juY5viPr8Wl+cFhtI23Mi8ZDPkbsnnpXw74lsvEPh+41jSbbSvGN1b22sXjJqMerRO
+0xJ24+YnjChtvbd0ry8TShiOeHNfmuvk1Y+oy+cqDhN/Zs/xufrH4a1Zdb8O6XqCtuF1axzZBz95
+QTWnXj37L/iUav8ADaLSpHY3ejSG0cPjfsPzRlsEjOCRwf4a9hHSvosBXeJwtOrLdrX1WjXyd0fP
+46gsPiqlJbJ6ej1T+aaYUUUV6BwhXln7UV+tl+z/AOPQSVebRrqMEDOMxnn8q9Trh/jHo8+teCbu
+CCBbklTvidQyspHIIPUHoR70Afzw3I3QzDnJJGTznk/n1r9XfCk4u/2ZfhBLj5ja2P8A6KYf0r5h
+8TfsiaLoN54r1RriYafPE0NppX2fH9nyyyovmq+fm8sMxQEdxnpX1/4u8O6f4H+HfhHw5ppcWGj3
+dvYWxkOWKRqyhmPqcZPuaXQnqae2P/Zoql5sn/PQUUWKPtEkAn071518ID/a1hr3iU/MNe1ae5gY
+9TbxkQw/gVj3f8Cq78XNbudP8LHStNcrrevTLpdjg8o0md8vsI4w75/2R610eh6PbeHtGsdMsk8u
+0soEt4V9FRQB/KvncZUVTFxh0gr/ADlovuje/wDiR68F7HBSk96jsv8ADHV/JytbzizQHQUoOKQd
+KWiLPNPP9cP/ABPJOD/r+PzFfnH4+062l1bxKsUfgp9PXxFfMHnuZopFu+rh+4baFyRxnNfo54jP
+la3IRj/Wg+noa/Pb4jnW11jxNLbeIdctrRtdu7dbQeHo7hY3GW8wY+8hBChjycV5n22fXYb4E/JH
+pv7KnxD0n4Z/E+40exutGi8L6kRFGmn6k9y+TgiV1flQGPJzwHPpX6ABuK/JtvEDajrGrTaPqV5p
+MMNlLOYZvB5ysKhA8e5OuSc4GO9fd/7LHx1tPin4STTLm5dtb01RExuYmhkuI1AAfY3zBh0Oeeh5
+zXbga/1au6M/hqO68pdV/wBvbrz5u6Msyw31igq8PigrS849H/27s/Ll6Jnu1FJnApRyK+pPkQpG
+UOpVgGB4IPelooA8H/ao8IaZb/B7X7i1so4bphG3mIOSRNGf6V5V8YryNNP0IHAaXV856/wsa95/
+aOvbez+HN2104S16SuQSANwA4HJ5I6V8r/GHUrhJPBdsyNsk1OYvIoyqbYSQCfc0PYlm79vT+6v5
+UVk/aD/dWioux3PqLwpct468VzeNZgf7Mijey0CJxjMBP726x2MpUBf9hAf4jXoryFY4yP4mUViR
+bV2oiLHGqhURFwqgDAAHoB2rU8zdDD7SL/OvjaV4u8ndt3b7t/1Zdkkuh7OMn7SScVaKVkuy7fq3
+1bb6l0HFKDzUckgReTzUAudxroU9bHAot6nDeKJNuvzc8iRcfkK/MXxN4i8K+LfEOtXsb3trLqF/
+cyEw69NEERnORt2cNu7dMV+mPiw51qYjhvMH8hX5TSaBBdavqkkM7FBdTAIWUMvznkgjJ/GsYpOT
+ufSUZyhBcvZHSXXjWy8QapdX7XuqWZmJZRZ+ICVBGFBw68KQDkdenWuj8LfE6TwbqehX+m6hrJvt
+MecxSy6jBKJg5D4feAcDGBg8gkHg153ZaL5d9FMjRSmJw5SUR59zx/KrmqeJ7fTbk2TWJE20MwVY
+2UAjjPvWk6UKsXCSun/X39U909TaGIqU2pRdmv6/LR90fqV8DPj1onxn0KKS3uIItYjX/SLNXByR
+1ZOeR7dV79ifUweP8K/EvwR8XU8DfETUtRtbi80mSK1t3iECYAYMSThTwevPX8K/QP8AZ/8A27PD
+3xAtI7LxHI1neINpvjCURvd1/h+o/IV2UsfLCWhjX7vSfT/t/s/73wv+63Y86tgI4lupg1r1h1/7
+d7r+78S/vLU+sqKq6dqlnq9nHd2N1DeWsg3JPBIHRh6gjirOa+hUlJXT0PnmnF2krM8Y/amS6X4e
+STQXD2yq67pI4xIQNwzlTwRjr7V8u3eqwa1pOnoZWlMF3lSk4VS4yD8h5yOcCvv3VtJtNcsZLS8i
+Wa3kGGRq8hvf2V/DDaxb6lZs8E0E32iJWJKrJgjOOnQmmSeBf2ppf/Pa5/8AAdf/AIqivpf/AIUh
+a/8APSz/APAb/wCvRRYVh2v/ABK8K+Eif7X8Q6dZOP8Alk1wHkP0Vcn9K4XVv2uPBOmFUsotV1hl
+YHdBbiJMD/acjP5V8aqiISVUAnqQBk/X1p6xs5wilj6AV8j7Jn0zpQlpI+wx+2R4Ml/1una3F/27
+o2Pyep4v2vPh6SNz6xET13aeTj8QTXxDqXiPSdJcJeajBFMTgQIxllJ9PLTc36UkN7qeojOneHb5
+kPSfVHWyjI9QGy7D/gINTNQo61JJersctephMMv3s1H1f9M+rPGn7U/hT7ZJLpFjqesOxDDdGLZB
+9S/J/AV8YS+FJDc3EquoSW4kmCsCSA7Z2k+2etd7aaJeG3Q3k9ktxnLi2EjRgdgC2CTT30KTPyTQ
+n13AiuJYzCJv95+D/wAjjjnuWpcqq/g/8jzxvDVyqMkXleW3LKU6n196pN4Mmfh4LYg9cRDJ9Mnv
+Xpz6LOv3Xik992MfnTW0e5GABG3/AG0wB79K3WNw/SojVZxl8tVWX9ep4/pPw2ksvFGpXsuj2k9l
+PawxxN5vz+YpO7cp6D0IP1ro7DwpcWl9bSLYReRHcQztbvKojXy2DEDGck884/KvRoNEUYM8+490
+hGB+Z5qaTRrVgNjSx/8AA8/zFc880w97Xb9EefLiPL6cuVNvzS0/GzPWYPjl4WsL+e/0W18TeEr2
+Vy7NprwvDIc5zJCTsb34Brq9G/bXm0yVYtTsV1q3/wCfmKBrOb8Vy6/kRXz02jwEYWaRD6nDf0pk
+mixMDid93qUFclPF4Wk70Kkqf+G9v/AWnH52udb4py+tpiHzr+9Ft/8AgS975c1j7G0r9tDwFeIP
+taahYy91MSuB+Ib+lbI/ay+G+zcdWnHsbZs18MSeHFdeLjLejJgfzqo/hLI/1kOfQIcV3Rzaa0WJ
+Xzhr+HKvwIWcZBPV3Xo5L84y/M+7P+Gu/ht/0Err/wABWor4M/4RSf1t/wBf8KK0/taf/QTH/wAA
+f/yRf9qZB3f/AIE//kBg8O6wqHHi2V29ZNKtzx74IzTh4NtbrDarqGo60w6xzz+TAf8AtlFtH5k1
+5/8A8NN6B/0A9V/7+Rf40f8ADTegf9APVf8Av5F/jXbLhji6as8NP5OC/KSPyqpmuY1Y8kq0retv
+yPWtMs7PRY/L06zt7BPS2iEZP1IGT+NWDOc9c145/wANNaB/0AtV/wC/kX+NL/w01oH/AEAtV/7+
+Rf41wPgniWTu8HL74/8AyR5b527s9iM5PXmjzzXjv/DTWgf9ALVf+/kX+NH/AA01oH/QC1X/AL+R
+f40v9SOJP+gKX3x/+SFaR7F5/rR59eO/8NNaB/0AtV/7+Rf40f8ADTWgf9ALVf8Av5F/jR/qRxJ/
+0BS++P8A8kFpHsPn0edXj3/DTWgf9ALVf+/kX+NH/DTWgf8AQC1X/v5F/jR/qRxJ/wBAUvvj/wDJ
+BaR7D51HnGvHv+GmtA/6AWq/9/Iv8aP+GmtA/wCgFqv/AH8i/wAaP9SOJP8AoCl98P8A5ILSPYfO
+NHnGvHv+Gm9A/wCgHqv/AH8i/wAaP+Gm9A/6Aeq/9/Iv8aP9SOJP+gKX3w/+SC0j2DzTRXj/APw0
+3oH/AEA9V/7+Rf40Uv8AUjiT/oDl98P/AJILSPnb86Pzoor+2jQPzo/OiigA/Oj86KKAD86Pzooo
+APzo/OiigA/Oj86KKAD86PzoooAPzooooAijuYZgxjlRwoyxVgcCmfb7YdbiIf8AAxX6Xf8ABSfw
+zo+ha38CRpukWFgJ/E+2UW1skYkXfBw2ANw5PB9a+l/jz4r+H37P3gdPE+seABrVo15FZC30TRoZ
+5t7hsNtO35RtOTnuK+JfEt6VCpCg26t7K/Z27dT6NZQuecXPSNunc/D2O7gmYLHNG7HoFYEmkS7g
+lcIk0bueiqwJNfdH7U/7YHwv+J3wO8R+FfD/AMOdb8M67qXkfZb/AFDQ7e0RCk8cj/OGLAlUI+Uf
+xc8V9OftneA9IX9jrxfNpPhmz/tU2FkYjp2mqZ9xuIM7Aibumc47Zrqlns6Toxr4dwdSXKrvb4dd
+vP8AAyWWwlz8k78qvt6n4+SXUMLbXmjRvRmANLLcRQAGSVEyMjcwGRX6x/8ABN/wFp1z+zZCdf8A
+DNu1+NYvhnVNOUTFd42/6xd2OeK4j/gmr4Y0fXdU+OI1HSLC/EHicrCLq1jk8obpvlXIOBwOBxWd
+XiKFJYl+zv7Fpb73du2hccq5vZ3l8XkfmsrK6BlYMp5BBzmlr3X9qj4ZeJl/aK+JM+m+DNb/ALHG
+ryGCWy0ec2/lhEGUKJtx15FeFYIZlIKujbWVhtKnuCD0Psa+nw9eOIpRqR6pO3a541ajKjNxfTqF
+QNe26OVaeMMDjBcZzVq3sbnVLqCxskMt5dypbW8ajlpXYIg/EsPzr9xfBvgj4efDtPBfw0l0bSbj
+XE0IvAXsImaeO1EEUsjMRksWlU575bNePm+cRypQvBzcr6J20W7O3BYF4zmfNZI/DZSGAIOQeQR3
+psk0cO3e6pu6BjjNe0/tVfCq48DftTeMPCOlWa/8THVo59KtYhtQreFXiRQOgDyMnHQLX6J6L8J/
+hV+wV8B7rxTrGjQ65q9jDEL/AFZrZJr3ULtyFWOEv/q0LthUBACjJyckxjM7pYWjRnCDnKrblivO
+3X59i6GXSq1JxlKyjuz8g4po5iRG6uR1welOyAM54r9e4fA3wj/4KBfA4eIrXw8uh6jMZraHUlto
+4tR0u7Q4ILocSKDtO0kqysOhPHzJ+wT+xzY+N/HXjDWfiNpsV/ZeDNUk0RNIlXdbXOoRn968g/jR
+Bs2oeCZMn7oB5qfEVD2FapXg4TpaSi9Xd6Kz66+hrLKp+0hGErqXU+Gfttv/AM94v++x/jRX7+f8
+Kh8B/wDQleHf/BZB/wDE0V4n+ulH/nw/vX+R2f2I/wCf8D5D/wCCngJ134Bf9jR/7Pb19NftF/F7
+Wfgp4BTxDoXgnUfH1817Da/2VphcShH3Zk+SNzhdoz8vfrX5eftH/tpX/wC0be+Bri78I2+hHwrq
+f9pxpFftP9pOUOw5jXaP3Y5Gete2f8Pcdb/6JhYf+Dt//jFcFTIsc8NhaboqTp83MuZLeV1rf8jt
+jmGH9pUana9rO3keZ/ts/HXxr+0L4V8Pzav8IPEXgLTvD1xLPJfahFM8L+aqxgMzQoqc46nkkCv0
+k+L3xai+BXwH1Dx1NpcutRaPZ2ztYwzCJ5d7xx8OwIGN4PTtX5yftAf8FDtV+Pvwk13wJceBrXQ4
+NWEIe+i1Rpmj8uZJRhDEoOTHjk980fGr/godqfxn+Des/D6fwJaaRb6lBDAb+LU2laMRyRuCEMQz
+kx45Peuytk2JxVPC0ZYdQhCbulK/utxd73vr73W+hzwx1GlKpP2l20rO1tdfL0P0T/Zl+PkH7SPw
+xXxjb6JN4fja9nsvsc86zMDEwG7coA5r5t/4JgHGq/HjP/Q0/wDs09fNv7NX7eupfs3fDOPwbaeC
+rXXolvJ7z7ZNqLQHMrZ27BG3T1zz7Vkfs3ftp3/7ON143ms/CNvr3/CU6odTkWW/aD7McudgxG24
+fOeTjpWUsgxVKljKNCn7s3Hk1Wylfq+3c1WY0JSpTnLVXvo+x93eDv2qfHniD9rjVvhZdfDt08J2
+tzdQp4jijuF2JHFvSR2ZPLKs2E4I5cYz3+aP+CrfgPQNA8f+CfEunQQ2ms65bXcWpLCoU3AhMRjm
+YDq37xk3HkgKP4RWhc/8FbvELwOIPhlpyTY+RpdZkZQfcCEZ/OvkT41/G3xT8fvHU3irxZcRPeGI
+W9taWqlLezgBJEUakk4ySSxJJJ5OMAdeUZPi8PjoYmVJUoxjZpSvzOzV9/n20OfG46hUw8qcZczb
+7bHo/wCwn8K7z4kftK+D55LC5l0PRp31a6vPIYwB4F3RIXxt3ea0R25zgV9XfEr4ka7D/wAFNPBL
+waPq03h3SrNPDk11FYzNbFruN5HcyBduA724JzgbOehqn/wT5+INt8Mf2NviP4lnMMz6Nqeo6ilp
+JMIzMY7OBlTuRuK7c4PtmuUT/grjrjRqT8L7AEjJA1tz/wC0KzxixuPzCvKjQU4wi6fxJWv1162N
+MO6GGw0FOdm3zfcdj/wUD8If8Il8f/gp8WvsE8+k2GoQW+tTwQM6W8VvcpOkspAwqhHn+Y8DaK9X
+/wCCjXhHVvG/7Lmqf2FaTanLYX9pqUsNohkkaBH+dlVcltobdx2UntXMw/tKr+0t+w/8WvEV3ptr
+4dv4dO1TTX05L3z9wW2DBgSqn5lkxjHr1r5T/Z8/4KMeN/gv4WsPDGt6RB440CxjWGya4umt723i
+AwsXm7WDqo4XcuQABkgDHnYXBY+rClKEE6mFlblb3V1LfbTbfb0OmrXw1NyUn7tRXufYP/BMvwnq
+3hP9mZX1iwudNfVNZu9Qt4bqJonaBhHGr7WAIDeWSMjkEHvV/wDYW+Iei+PV+NUuk3KTq3xA1K+X
+YR89vMI/Jl+jCN8HvtNfHHx7/wCCkfjb4v8Ahi98NeH9Fg8DaPfRmG7uIbprm9ljIw0aybUWNSOC
+QpbHQivA/gd8cfFf7PPjOPxH4RuYo5TGLe7sLlS1rewg5Ecigg8H7rAgrzjgkH0HkGLxlLE169oV
+KjTUb3tZ31fnt+LOX+0cPRnThDWMd2ftr/wgt3/0Em/Wivz7/wCHtni7/onGi/8Ag1m/+N0V87/q
+9nP/AD7X/gUT0v7RwX835nwhRgVsaV4L8R69afa9K8OaxqtpuKfaLDT5p49w6jcikZHcZq5/wrHx
+r/0JXiX/AME1z/8AEV+zurBOzkvvPhFSqPVRf3HNUtbt94A8V6XaTXd94U16xtIV3S3F1pc8UUa+
+rOyAKPcmsKqjKMtYu5MoSh8SsFHbpR+lbMXgnxJNpX9qR+G9afSvKM/29dOmNv5QGTJ5m3btxzuz
+jHNDlGPxOwlGUvhVzFpals7S41K9hs7K2nvryY4itrWJpZZD/sqoJP4Cuv1X4I/EjQtNOo6l8PPF
+Vjp4G43M+jXCoB6k7OB9amVWnBpSkk35lRpzkrxTZxLRqzBiPmHQ07g01HWRNyMGXrkHilYhBzxk
+gAdyT0A9/wDGtSbN6CGNWYMR8w6GnV2kPwQ+JNxpH9qxfDvxXJpm3d9rXRbgx7fXOzp71xZBV3Rg
+UeNijowIZGHUEHkEdwelZQqQqfBJP0dy5U5x+JWCitnSvBPiXXrRbvS/DWtapaMxUXFjp008ZYHB
+G5FIyD71b/4Vj41/6ErxL/4Jrn/4ih1aadnJfeCo1GrqL+45uiuk/wCFYeNf+hK8S/8Agmuf/iKK
+Xtaf8y+8fsav8r+4/Q39hf8AaW+Efwi/Zx0LQfE/jvS9H103V5c3NjcF/MiL3DlQ2FIyVCng96+l
+/CH7XPwf8feJtP8AD3hzx1p2s63fu0dtZWiyPJIQpZv4OAFBJJ4FfhuSFBJOAOpJ7V+rf/BNv9nK
+H4b/AAxT4g6xaAeKvFkCywGVfntNNJ3Qxj0MnEreoMY/gr804hyfA4SFTHVJy55vRaWu9e2yPq8u
+x1au1RjFWitXqdx/wUG+IGi+Cv2YfFmnandeXqHiOA6TplqnMk878nj+6qqzMegA9SAfxsr6U/4K
+C/GOX4r/ALRWq6fBOZNC8JbtGs4wcoZwQbqTHTJkATPpEtfNVfUcOYB4HAR5vin7z8rrRfdb5nj5
+riFXr2W0dD2X9kj4ED9of436R4au1Y+HrVG1LWmUkZtY2A8rI6ea5VM9cM5HSvsj/gqP8Zv+ED+H
+3h74WeHytgNXhN1fw2gEYj0+DCxwADgLI4AwMDbEy9DVb/gkl4Vhj8LfEXxK0YNzc6hb6akhHKxx
+ReYQPq036Cvmn/goxr0vif8Aau8X25lYxaZaWemwbuiAQCVsf8DmavKlL+0eIVSn8FBXS89Nfva+
+474pYTL1JbzZ+in7Gv7Neg/AX4V6Rciyhm8YatZxXeraq6Ayl3UN5CMeVjjztCjGcbjyaqfBz9u3
+4efG34uXvw/0SHVbe8UTHT7+7hVbfUfJz5nl4YsvALDeBuVSeOldt+zF8ZtF+Ofwa8P65pVzHJdx
+WkVpqdnuHmWl2iASRuvbkEqT95SD3rzL4OfsJfD79n/4vy+PbTXtRnmeSWDSNOv5Y0hs2nyGRSAG
+lbaSihjwpPBPI+CnKjUq4p5k5e2+z667+W3la578VOMaaoW5OvoeB/8ABTT9mfRvDVjZfFjwvYxa
+a092tlr1rbIFilaTiK6CgYV9w2Of4t6HGQSej/4Jo/sv6RH4Sh+LviSxiv8AV7+V10CK4QMllbox
+Q3Cg/wDLSRg2GxkIowRuNe8/8FA443/Y7+JTShf3dlDIhYdHW5iKn65Ar0T9nnSLfQfgN8OrC0QR
+wQ+HrAKF9fs6En8SSfxr0J5tiHkcaPM787jf+6knb8bemhgsJTWN9rbpf5nAa1+218O9C/aDtPhF
+MdSk1ua5jsH1GGBTZQXcgBjgZ927cdygkKQGZQSOceGf8FMvgv4O1XwNJ8Q9Mk03T/GukzRJexxS
+xxy6jau4Qh06vIjMrK3XaGHIxj2Ky/Y2+CMvjTX5/Eun2Hi3xtrt7cavdvqtzm4AlkL4ihDjy41y
+ACBngZJr5m/bf/YD8LfDvwRe/Ef4b2kmmQaURJquiPK00fkEhWngZyWRkyCy5KlSSMFcNtlcsBTx
+9D6vUnCWid1pJv53SeyuicUq0qE1OKkvy/A7/wDYk/ad+EPwm/Zs8L+HvEnjzS9I12N7ue7spy++
+JpLmRwDhSPulT1719JeC/wBrP4R/EXxPY+HfDPjfT9b1u9LCCytFkd32qWY/c4AAJJOB+Yr8M2YK
+GZiAAMknsB1NfrR/wTl/Zyi+Fvwrj8b6vahfFviyBbjMi/PaWB+aCEehYYlf1LKD9wV6XEOT4LBw
+qY2pOTnNuy0td69tl/W5x5dja1dqjGKtFas+vt49KKyP+Eo0X/oLWH/f9P8AGivzb2c+zPpLn4O/
+CLwF/wALS+K3g/wed3la1qsFpOy9VgLbpT+EavX7u+JNTt/BPgrVNQigSO10jT5bhIUG1VSKMsFA
+7DC4r8Kfg38ULv4LfE3Q/G1hptrq97pDSvDZ3jskTs8TxZJXkYDkj3FfTXi7/gqJ428ZeEta8P3P
+gbw/b22q2U9hLNFeTl0SWNkLKCMZAYmv1/iHKsbmeIpexjenHfVLVvX8Ej47LcVQw1GXO7SZ8Zvq
+E+rTS6hdO0l1eSNczOx5Z3Jdj+JY0lMij8qNEBJCgDJ70+vvNtj5yTu2z9Lv+CSXiCKfwD8QtD3D
+7RaavDebM87JYAoP5wsK+Vv+Cgeiy6H+1x443qSt6tnfRE8bla2jU4/4EjD8KP2E/jhb/A/4/adN
+qlwLfw54hjGjajK5wkBZwYJj2AWTCknosjHtX0X/AMFXvhFNI3hP4nWUJeGFf7D1RkH3FZjJbOcd
+txlQn1dB3r4KMfqHETlP4a0dPXTT71+KPpX/ALTlqUd4f1+pS/ZM/YYude8GeHPiX4e+NWt+Hm1u
+0WZ4/DFuLd05w8EkjuwkKOHQ7kxlTgV3XwV/Zh+Otz+0bpniX4v+LpPEfhbwnPcXGkNLqHnLeTMj
+xQyLbqAIiEcsxYZyABkEmvi39nb9rrx5+zU91a6BJa6t4eu5fOn0LUwxh8wgAyROp3RMQBnGQcDI
+zzX0Hr//AAVp8W3WmNFo/wAPdJ06/ZcC6vNRkuI0PqI1RCfpuFYY/A53KvVVPlnGaaUmopxT+5p2
+06m2HxeBVON24tdNT17/AIKm/Faz8P8Awa0/wDDcK2r+J7uOWW3ByUsoGEjuw7AyCJRnr839016p
++wf8VbX4o/s0+E9twsmq6Dbromow5+aKSBQiEj0eLy3B/wBr2NfkB8QPiF4i+Kfi6+8T+KtUl1jW
+7wjzLiQBVRR92ONBwiLnhRwMnqSTW58Gfjn4y+APilte8GaoLKeZVju7OdPMtL2MEkLLHkZxk4YE
+MuTg8kHerwzfKo4SMl7RPmv0be69LafJMxjmsVi3Ua916f8ABPsiX9jH4wP+3X/wnmIj4f8A+ElG
+uf8ACUfbk3Cz3bvs3l58zd5f7jZjZjvivrL9tLxnpngj9mD4g3OpSIv27SptLtomIBmnuFMUaKO5
+y2eOgUnoK+PIP+CtviYaX5c3w20qTUduPPTVZFhJ9dhjJx7bvxr5b+PP7SXjn9o7XLe/8XX0S2dm
+WNjo9ghjs7QngsFJJdyON7EnHAwOK4aeT5nj8TQnj4xhGkktLXaXo3+iXRHTPHYXD0pqi3JyMP4H
+fD1fin8Y/BHg+UM1rq2qQwXW3r9nU75uP+uaOPxr9z/HOup4H+H/AIg1mKNETSNLuLxIwMKBFEzg
+Y9Plr8NPgj8WL34G/E/R/HGnaba6vfaYJhFaXjskTGSJoySV5yAxI96+j/HP/BTzxr498Fa/4ZvP
+A+gW1rrFhPp8s8N5OXjWWNkLKCMEgMSAfQV38QZVjczxVF0o3pxWuqWrev4JHJl2LoYajLmfvM+O
+v+FweOP+gxdf9/DRVL+z1/56v+lFfdezpfyr7ked9Yj2/Mt9e9JXstj+x58Yr/4kXHgWPwVcpr1v
+Ct1LJJKi2SwMSFlFxnYVJVgACWJUjbkGsv4z/sz/ABG+AAspvGehpa6feP5MGo2VwtzbNLgnyy45
+V8AkBgMgHGcGuSOOws5xpRqxcnqldXZzPCV4xc3B2R5fRXpmhfs2/EbxR8K5PiLpHh/+0fCiSNEJ
+7a4R7h3EohKpADvY+YQMAV3d7/wT++O1j4ZOtP4PhlAi846bBqMT3u3GT+6BwW/2QxPbGameYYOm
+7TrRTvbVrdbr5dRxwdeSvGD2ufO7qsisrgMrDBB7j0r1/VP2uPizrvw5k8B6r4qXVvC0lkNPks77
+T7eZ3hAAUNKU3lhhSHzuyAc5FeU2GnXmq6jbafY2dxeajczLbQWUETNPLKTgRqgGS2eMYzX0Mn/B
+PH48SaGNSHhSyB8vzP7PbVYBdAYzjbnbu9t2c8VGMqYGm4fXHFPePNbfur/mVQhiWn7FPzsfOHT1
+x70v+elajeFtXg8Ujw1c6fPY6/8AbV05tPvF8mWO4ZwgjcNjadzDrxznpXr8v7EPxph8dWfhFvB4
+bWLm0N8HjvYmtoYA5TdLMDtUlgcLksccA10VMXh6Fva1FG6vq0tF1MYYerU+GLfQ8MoPFerfGb9l
+z4lfAK0tb7xhoKW+lXMogTU7C5W5txKekbMOUY4ONwGcHGay/hB+z/4++PN/d23gjQH1RLMqLq9l
+mWC2gJGVVpHIG4jnaMnHOMUljMNKj9YVRcne6t943hqyqeycXzdjzzGfelr3z4g/sK/Gj4a+HbzX
+dS8MQX+mWcZmuX0e+S6khjAJZzHwxUAEkqDgc9K8s+Gfws8V/GTxKmgeDNFn13UynnOsLKkcMWce
+ZJIxCouSBknknAzSp43C1qbrU6sXFbu6svXsOWFrQkoSg7s5aivo/Xv+CeHx20LSpb8+F7HUhEpd
+rbTtUilnI9FRtu4+wJJ7Zr5xlje3keKWOSKaNzG8ToVkVwcFSuMhgeMYzniqw+Lw+LTeHqKVuzTJ
+q4erRsqkWriUV3H/AAof4of9E08X/wDgkuP/AIiij61h/wDn4vvRPsKv8rP0C/4Kg/FbxT8PvCng
+jQvDWuX2gw61eXct7Pp0xgmdIVTZGJFIZVLSZOCM7QKzPjJ4g1Hx9/wS40rX/EF0+q6xNY6ZNLe3
+HMkkq3ip5jHuxA5PU5OeteU/8FH/AI6eAfjYPh+fA/ia08R/2Y9+bwWquPJEgh2E7lHXa3T0NWvF
+Hx5+H9//AME5dL+HNv4ps5fHEFjZRPooWTzg6XiyOudu3IXJ69jX55hMDUp4LASVJqaq3l7uqXM9
+Xpe1ktz6yrXUq2Ii5acumvket/sweNdQ+HH/AATg1nxRpLKmqaTb6zc2kjqGCSiZwjFTwcEg4PpX
+D/8ABMj4veNvGvxX8ZaL4k8V6t4isZNIGp7NVumuClys6KXQtnZlXIIXAOBxxXNfDX47/D/Q/wDg
+nt4k+Ht/4ps7Xxpd2mqRwaO6yea7SzFowCF2/MMEc9xXEf8ABPH4t+EPgz8XfEeseNddtvD2m3Oh
+G0huLkMVeXz422jaCc4Un8K2q4CU8LmMnSblKbt7urV01bT8iI10qmHipaW11PcP2T/A+lXf/BQD
+456nLaxmTw/d3baegX/UyXFyVkdR67d4H++a6fxL4N+J7ftE3PjeP9o7wfpulW2qbU8Lz6qyQRWC
+PtNpLBu2F9gIYkZ385BAx81+Cf2pdN+D37afjz4gWDvrngnxDqV3DdvZDLy2kjq8c8QbGWRlBwcZ
+BYcZr17xD4F/Y58c/FV/itefEmxjtrq5/tS/8MzECC6uTyxkhaLzgGbloxwTnsSKyxOHr08QqtWM
+uWVOKVoc+tleOuzv+ZvRqU503GDV1J9bfMxP27ofC+qftPfCLxP4b1PTNSuNUubS31B9Muo5yzw3
+kPlM5RjhikhXJ6hAP4a7/wD4KgfGHxb4FTwT4a8N67faBaaqLu8v5tNnaCafy2jWNDIpDBQWYkAj
+JxnpXyH8VPHvw01/9oXw9q3w68LWngrwPpuo2ReWKAwm72XCvLdPFzsUDIVRztBJGTgep/8ABRn4
+0+B/jT4n8C3XgjxHa+IoNPtLyO6e2Vx5LPJGyA7lHUA9PQ13UcBKOIwEKkG4xjK/MttLpPdJrSxy
+1MQvZYhwkk21se6a/wCKdU+JX/BKy+1zxLeSazrD6QfOvbo7pZXgvwscjHu+FXLdSRk9a8c/ZK03
+9ofxZ8F9V0DwBqGg+Cvh7JJcbvFGpxm3uElJzPJBKmXZh9wyMMKBgMCvF7Svj58Prf8A4Jv3Pw2k
+8U2a+OZNOnhXRSr+cXa+MirnbtyU561vfAL44/CT4l/shSfBH4heKB4DuoLZ7FrqWTyluI/OM0U0
+UhBQndgPG3XB7GuP2VbDYavGNC6dd7xvyx/mUetunQ3UoVKtNuevJ3td9mz6J/Yw8MyeE9N8U6Df
+fGmL4x3sU0D3MMNwbmLSNyuNiyNI7HzMZIJwCnAGTXyn/wAE6vjX4F+GGs/ELwn4o1WDwvca9eI2
+narcyCGFljMkfkGU8Rsu8Mm7g5PcAH0z9l34wfs2/su3Wu+E9G8fXGpteCO8v/FeoW5SzupEyiW0
+IRf4FYtwCPmPzE8D55/Ze8e/BHwb4u8XeG/iv4c0bxBpV7f3H9l+MHs2ukjiZmTYwxuWNlw6SKu5
+WZs44Iyp4SdSONVSnOUZ8jVoqLdr6pWS36b231LlVUZUbSimrp63/E+s9L/ZT+J/w7vb/wAT/CL4
++X2s/b4XQWfjZjqdrKCQVfzlYrvB6Ps7nqDivnb9lr4ea/4l/b41JPitZR/8JhoyXOuXtu8MaxzX
+qiMQzKqAIVxKJVKgAkK3XNeg/BjRf2bP2XvGknjfR/j5e6tZQQzLbeHobjzEcOpULLFEmZmAPG4L
+zgnpXgOrftg3v/DYM3xr0rSGSxDrZLpEzhZJ9NEQiKORkLKwHmDqFYKOQDnfCUsZX+s0qfvKVNpT
+cOSV+kel9DPETowdOU3a0tr3Vu5+yX2P/pvL/wB/KK+N/wDh6b8Hv+fLxr/4Lof/AI9RXwv9g5n/
+AM+H+B7X17D/AM6Pyq/Sj+dFFf0EfmYUZ4oooAM/nR3oopgGaOvtRRSAKP8A9VFFMA4/GiiikAmB
+6UUtFMBd59/zNFJRSAn+wz/3P1FH2Gf+5+ooorLnZryIPsM/9z9RR9hn/ufqKKKOdhyIPsM/9z9R
+R9hn/ufqKKKOdhyIPsM/9z9RR9hn/ufqKKKOdhyIPsM/9z9RR9hn/ufqKKKOdhyIPsM/9z9RR9hn
+/ufqKKKOdhyIPsM/9z9RR9hn/ufqKKKOdhyIPsM/9z9RRRRRzsORH//Z
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="cabiomuefog.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <0b64701d0b85909251f830b812fc8f@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAfQBa
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++LvtEn/PV/8Avo0faJP+er/99Go6K/p4/J7kn2iT/nq//fRo+0Sf89X/AO+jUdFAXJPtEn/PV/8A
+vo0faJP+er/99Go6KAuSfaJP+er/APfRo+0Sf89X/wC+jUdFAXJPtEn/AD1f/vo0faJP+er/APfR
+qOigLkn2iT/nq/8A30aPtEn/AD1f/vo1HRQFzo/+FeeIf+fBf+/y/wCNH/CvPEP/AD4L/wB/l/xq
+7/wtXW/+edl/35P+NH/C1db/AOedl/35P+Nc16/keZzY7tH8f8yj/wAK88Q/8+C/9/k/xo/4V54h
+/wCfBf8Av8n+NXv+Fq63/wA87L/vyf8AGj/haut/887L/vyf8aL1/IObHdo/j/mUf+FeeIf+fBf+
+/wAn+NH/AArzxD/z4L/3+T/Gr3/C1db/AOedl/35P+NH/C1db/552X/fk/40Xr+Qc2O7R/H/ADKP
+/CvPEP8Az4L/AN/k/wAaP+FeeIf+fBf+/wAn+NXv+Fq63/zzsv8Avyf8aP8Ahaut/wDPOy/78n/G
+i9fyDmx3aP4/5lH/AIV54h/58F/7/J/jR/wrzxD/AM+C/wDf5P8AGr3/AAtXW/8AnnZf9+T/AI0f
+8LV1v/nnZf8Afk/40Xr+Qc2O7R/H/Mo/8K88Q/8APgv/AH+T/Gj/AIV54h/58F/7/J/jV7/haut/
+887L/vyf8aP+Fq63/wA87L/vyf8AGi9fyDmx3aP4/wCZx9FfffgH9lnwj4o/YV8S67p3gmLxF8TL
+VNYtLLUbWOR72aeC8miiZUVuXCouAB26HPOn+1B+zP8ADr4c/C74O32meB7LRda1bxTounasyiQS
+SxSo3nwyAtwGI5xg5HUV4C4gwzr+wUXfmcenTrvsfXvKqqjzOS2T+8/PGiv1b8WfsJ/Di7+Pnghd
+M8FadpXgSx069udbgAcQ6jcOY4rSAkvncCZZOP7gBzkV8Tft6/D7wr8Iv2hbjw/4V0q18PaIulWk
+62kDEJ5rmTc3zE8nav5VrgM9w+YVY0aSd3Hm1tpraz136+jRniMtqYeDqNp2dj59opFYMoIIIPQg
+5Br9DP2U/wBn34TP8MfhPF8RPBdr4i8V/EW41C5srm6LAwW8MbyopAYDb5canp1krvzDMKeXUlVn
+Fyv0W+ibe7WyTZyYXCyxUnFO1v10Pz0or688GfsqaZq3/BQDX/hzeaYB4I0W5n1q4smJ8sacY1kg
+hJznbvmiTJOcK3Oc11X7Z37PHwv034cfDD4ifDewtPDPg3V9Tgs9U1CzDlfslztMdyVYnGzY47ff
+A9K43nWG9vSoWf7xJp9FdNpPzdtEdKy2q4SndaO1j4Zor9b7X9jj4SWPirwnouk/BTTvE/g6/sJJ
+7vxrLrXmNA4TMWYy2+bzeDvQhRu6Yr4A/bW8BeH/AIY/tK+J/DfhbS4dF0O0gsmgsbfdsRnt0ZyM
+knkkk81nl+eUMxr+wpRadubW217dG2n5MeJy6eFp+0lK+tup4fRX398CvhB8J7L9ha4+LHiv4bad
+4v1vSrbULuf7RM8L3QiuJFRS4JC/KFGQp4HQ19R6N+xL8BNX0exvv+FX6RF9qgjn8vfKdu5QcZ38
+4zXFieJ8NhZyhUpy92TjdW3W/W/U6KWT1KsVJTWqv1Pyi034/wDxA8FT+JoPA/jPW/C+kaxf3V99
+jgnA2GVywbodr42gsmCdvXpXv/xz/bp0P4w+B/hzoieG9bt7zwxr2mazd3d7cQv9rW2VhIqkMSXc
+nIZsDrmvjyivdq5Zha1SNWUPejs1o9ra23+ZwQx9eCcU9Gfd/wAQv+ClWm+MviL8OtVtfC2s2fhj
+w5fXGoalpkk0HnX8zW7w2+0hioWMyO5DHk7e4BrQ+EH7Q/hX47/tl614rvPA0s2g3nhVYLv+2lt5
+otIitWaWW8mJyqx4ZUBBzuYevHwBUsN3cWyTJBcTQJMhimWKUoJU7q4B+ZeAcHIPpXmyyDBxpOFF
+cr5XFO72bv33v+i2OqOaVnK9TVXudZ4q1fQvHfxr1XVLS2Wz8K614laaK3cCARWMt190hcCMeW3Q
+Y2j0xX3z8a/2tPgd8MPin4T0+28F33i+4+H9kiaBqvh/VEaztFkhVGjjxNtciNVU7g3p1r81TznP
+TvSIixqFQBR6AYFdeKyqjjJU/aSfLBNWTa3SWrXlded9TCjjpUFLkirt3P0p/aK/a0+HHhfwtN4t
+8GaSt94/+JXhT7DLqVpfRvJpKrH+7juow52OhuG4UZJiwSQox4F4U/a48HXH7JFv8DvHHhjX9Sto
+ozbf2ppMtuGWJbnzodvmnIZRhM46D3r5SVFQkhQCTk4GDmnVz0MiwtGkqbu2pKSd3e6vy/JLS2xr
+UzOtKfMrJWtb8z7Ln/b+sf8AhoXwf4vsvDur6Z8PfDGhy6TZ+FrS5RGMjrt8wqG8shQI1AOcBARX
+gn7TXxfsvjz8a9c8c6dp11pNpqMVtGtpeMrSIYoVjJJUkYJXPXpXl9FduGyvC4SoqtGNpKPLu9r3
+++/U5q2NrV4OnN6XufUfhX9r7Q/D37GmrfBSXw5qk2rX1lfWo1VJohbK08rurFSd+FDqCACSQfWv
+oPRv+CrnhDStHsbI+AfETm2gjhLC5tudqgZ+97V+bNFcdfIMvxLbqwbvJy3e7tfr5G9PNMTTSUWt
+FbbsffmpfsSfB7VfgT468e+HW+IenXWgWmoNBa+JNtqzz20TMCYmhDNGSBzkZGehFfAKMWVSepGa
+/QDwv+3Vofi79kvxzoHxG8VSzfEjWrTV4Le0XTnKKJkcW0QeOPywo3KuWPA6mvKfiv4H+Den/sTe
+AfF/hjSxZ+PNTubXT7i9la4WS6nhT/T9iSNsZFcEF0G3OADk4rz8txOLws5UsdGTcp8sXutt7u2m
+j1sdmLoUayUqDSsrv7/meH/A74bP8YfjD4Q8GKZUh1fUFiuZYMeZHbKC8zrkHBEaMQSCM4r6k/aU
+/Yr+HHgD4B678QPhzrGvarNomqmyvRqV0ksIWO6NrcAARqcpJ3yRhT1zmqv/AATV0LwvoOp/EX4t
+eJ72W0h8EaeqBlUvFHFMjtNKVVSzMFh2gL2duDkY+gfgLrfwR+Kfw28W/Anw38QdY8VS+JItT1Ga
+4v8AT5ILiMXDBpXjZoVTKyShwDk5PQ4rkzXMsRRxt6PNyUnHmsm076yu/wDDy/ebYLCUpUEp25p3
+t+h4f+yN+wr4H+M3wl0nxR471jWNO1PXrq6/smz029ig821iO3fteNizEq7ZHAXbx3PKfsyfsP2f
+xW+I/wATdF8ZazfafY+BdQOmz2ukhRdXkhaTY4LKwVCkYIwpLFuCMc9d4i/aw8B+Dvj98CbbwdqU
+9x8Nvh7p82k313JZyxuWlj+zSsYygZyoijbIXks2M11zftL/AAK8GfHn4h/GbQPF1zqes6v4fitL
+HQY9Ou4EnvUVgzSEoE+YR2yhmPGZT3rCrXzf97KKn+8jePut8r57KPleOrb2NY08GuVO3uuz1303
++85Tx3+xn8JoP2X/ABT8VfCrePbC606znmtbHxQVt5BJHJs/ewmFW2kg45GRg18h/CT4Z6p8ZPiV
+4f8ABWjPHDf6xceULiVSyQRqrPLKw7hUV2xxkgDIzmvsS/8A20vD3jv9hvXPCHjTxdPqvxX1PTru
+CaJtOdVeVp3aJfMjjEQHl7BnI9+a+SvgT8Vrn4H/ABd8NeOLW0+3/wBlTsZ7QEKZ4JEaOVFJ6Ntc
+kE8ZA969jLXmMcPiVWu6iclC/Wy0s2ldX6nDi1hXVpcluV72/U+yfEn7FX7OmheIL34cXfxI1fQf
+iDa6Qmptqmq3ccdqAzbASrqsR558sMG2nOeMjj/hD+wroHxG/Zx8VeKRdaprXj/TbrVNPsLfQ76J
+tPvbi3kaOApuQ7o3ZVO7cAVbPFd58SvjH+x/8SfFV38VPE7al4s8QT6Umn/8Ipd2E2CyHKP5ZQIs
+oBKbzLsAJ781ifAP9svwL8LP2WvFeh2WpJ4S8dTz6xe6Jo1tYTXFvYyTO7WkaOUKMq5QfOe3NeCq
+mbfVk6ftOfmhfmWl9ea1teXvpY9OUcHz+9y2s9n/AF/mea/Gf9l/wH8BIfhv4M13UNc1r4qeKbmz
++3y2MyxaVY20lykcrRho8uwyUQFsk5dgowp+rD/wSo+ERJxrHjAD/sJQf/GK8T+Kn7UPwu+Pvwc+
+GGqeJ9ak0v4r+E9RsNRmt/sEzLK0c0YvEEip5YWRE80AHhkReDmvqL/h4z8A/wDocpv/AAU3f/xq
+uXG186VKmqSqc95c1k2r3VrWXw22+e5pRhgeZ83LbS2v9an441sa74w1rxPp+h2Gq6jLe2GhWhsd
+LtWAWO0hLbiqKAOSQCWOWOBknAr1/wDZR/ZUuf2qrzxLBZeKIPDn9iJbuzS2ZufO80yDjDrt2+Wf
+XOfY10vwm/Ydvvit4B8deKYfGVtpsXhTVdR0uS2fTmlNybRQxkDeYNofPTBI96++rZlgqM5RqzSl
+C19HpzbdOq7HzNPCYiULwWkvPsfNUN/dW0E0EF3cQQTgCaGKZkSUDpvUHDD2OaS0v7rTpxPZ3VxZ
+TgECa1maJwD1G5SDj2719Baz+x1eaP8Ast6T8aj4st5bXULeyuBoosWDp9onSLb5u/BK792dozgj
+0pv7Vf7HWofsuab4ZupvE0XiqXXbqWzht7exa3ZHRQwxmRtxYkDHFEMzwVWapRneUnKNrPVxSbW3
+RWKeDxMFzNbJPfv8z55JJJJJJJzk85NFfW/xp/4J1+JPgx8G9T8eXHiu11aTS4Ibi90mGwZGRWZV
+k2y+Yd3l7ic4GQpPHSuY+D37GkvjX4Uy/FLx5420/wCGfgAH/R7+8g8+a4Xf5Yk27lCIznavJZv7
+oBBMRzjAzo/WI1Lxvy7Pfsla7fohf2fiOf2fLrufN9FfRvxY/Yn1/wCF/i7wDbReIrHxB4O8a6la
+6bpvimyhYRxPOy7DJGGOcoS6lWIYK2CK6zXv+CeeoaD8dPCnw0bx5azXGv6XeammojS2VYBblQUM
+fmfMW3ddwxg8Hij+18Dyxl7TRpvZ7R36aW7PUP7PxDbSjt6dT5Gor6u8IfsBah4t+O/j34ZJ43tr
+W48J2lldSal/ZrMt0LlN4Aj8wbNvTOTmqngD9hS+8feBPiJ4mi8a21nH4N1bVdJktzpzubs2QyZA
+3mDYH9MHHqaHnGBjq6n8r2f2tY9Oof2fiP5e/VdNz5cor6If9jq+X9k+H44r4ohltZLGO9GgpYM0
+nzziHb5ofqC2c7f8a+djlSQYnBHUEEH8scV34fFUcVzexlflbi99Gt1qc1XD1KFudb6n61fB/wCL
+3wgT9qH4sar4d8R+HtK0m50rSYrq9a8itrW/vke6814SxCuVRoVZlyCeuep5D9jjxn4RT4N/GLQ9
+U8Z6BoN1rHi/XRbNqOowxFo5VRUmVWcFkOcgjg4ODX5hPDHIAGRWA6ZXpSPbxyHLxI2BgblBwK+X
+lw3TlGcVVfvcm6u/cVl957CzZ6Xhtf8AE/V/X/BPgvWP2OdG+C8fxj8FQ6np9tYwHVm1KBoX+z3E
+cpIj80HkJjr1NXP2jda+HvxY+NPwISTxn4audE0PV7zWtRnGr2zRR+TCrQq/z4+eXYMHqA3pX5J/
+YoP+eMX/AHyKPssABAhjx/ujBqYcNuE+f27veb2W81Zv8rehUs25lbk7fcnc/UX4C/tH+FPjZr37
+RHhvxp4l0/TfDus6nJHpb6neRwLLYSW5sz5e9gD8tuj4HeXPerPwL+Nuj+K/2arT4ZaV428KeFPi
+V4SiTRWXxEsdzZXH2WQRieMMyiaKVEyGQkqzdDgZ/LN40lADqr46ZAwKR4I5UCPGroOgK5ArSpw1
+Qnzcs2leLSsmk4q3Xe/UmGbzjbmj3v8AM/RX9oL4wWEXib4QeDb34s6V42vrfxXo9/qcOl6dbW+n
+aakMylpWnRj5YycLGWOF3FiABn6C8T/tS6JY/tJ+CPDll4t8KTeDb/Rr+51DUvtsLmC4Rk8pPPEm
+1N2Sdp5bHtX41LEqJsVAExjaAAKaLeIRlBEgUkErt4NTU4ZoVYRjKfwqS2S1l1srLTp+YRzacW2o
+7tdX0P1V+EfxN8G6f+3N8ddYufFug2+kX2laMlpqEmpwrb3DLD84jkLbXKnggE4NbPg39pnSvH/w
+X+M0fibxP4WsNQttT17S9Kt4LuK3a6s0iK28gUuTIXyfnXhu1fkibeMoEMSFByARwKUwxsQTGhIA
+AJGSKVThijUak6julBLT+RJfjYazaSVuXv8Aifqh+zx8SIbH9hbwRoXhb4l+D/Bvj2LTY0ik8QXs
+LLakXJZ1lhZt2Sm4AEcEg19B6f8AHn4X29hbRap8R/BNxqSRKt1NFqlsiSSgAOyqZDgFskDJxX4V
+vbRSMWaJGY8klQTR9niH/LNP++RXPX4Uo4icpuq1zSctEr69L9jSOcyikuTZC+fF/wA9F/76FHnx
+f89F/wC+hX3P/Zdh/wBA6x/8BI//AImj+y7D/oHWP/gJH/8AE17n9rr+T8f+AfPWR8MefF/z0X/v
+oUefF/z0X/voV9z/ANl2H/QOsf8AwEj/APiaP7LsP+gdY/8AgJH/APE0f2uv5Px/4AWR8MefF/z0
+X/voUefF/wA9F/76FfdH9k2H/QOsf/ASP/4mk/suw/6B1j/4Cx//ABNH9rr+T8f+AFkfDHnxf89F
+/wC+hR58X/PRf++hX3P/AGXYf9A6x/8AAWP/AOJo/suw/wCgdY/+Asf/AMTR/a6/k/H/AIAWR8Me
+fF/z0X/voUefF/z0X/voV9z/ANl2H/QOsf8AwFj/APiaP7LsP+gdY/8AgLH/APE0f2uv5Px/4AWR
+8MefF/z0X/voU37Qn/PRPzFfdP8AZen8f8S2x/8AAWP/AOJq4vhzTyoP2Cx6f8+cf+FaQzN1HaNP
+8f8AgHPWr0cOk6rsn/XRH//Z
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="atqayyjym.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <cb29601d0b859e92919940c94fd8ed@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAjwBa
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+9vk/Zj+IyMAukWkgKqcrfxgAkAlecHjkE47ccUn/AAzH8Sf+gJa/+DCL/Guk1/8Abt1HRfjJeeB1
+8GW0sNvrw0b7adRYMwMwi8zZ5fXnOM/jXoP7Tf7V9j+z9LpWnWmmQ6/r16puJLR7nyFtrYZHmOwV
+jlmGFGOdrHtz6i4pmoSk2vd0ej/r7j1JeHGZLEUMN7F89dOUEpRfupXbb6JJr4mvvPGv+GY/iT/0
+BLX/AMGEX+NH/DMfxJ/6Alr/AODCL/Gvqn4M+NNf+IXw/wBO8ReItAi8NXOojz4NPWcyukB+4zkq
+uGYfNtxwCM85A8Q+P/7bUvwc+Jd14T07wtb659jt4pLi5lvjDskkBbYFEbdE2nOf4q0nxLVp01Um
+kk/J/lc4MDwNicyx1TLsJFzqwvdKUbKzSfvfDu7aPXocL/wzJ8Sv+gJbf+DCL/Gj/hmP4k/9AS1/
+8GEX+NfSH7Ofxuj+PXw8HiE6fHpV5Fdy2d1ZJN5ojdcFcMQMgoyHp3ryj4//ALaOqfBL4mXnhWPw
+da6rDDbQ3KXcl+0JdXBz8vlnGCpHWlPierCmqsrcr8n/AJmmF4BxmMzCpldGDdane65orZpPV6Pd
+bPXoea+IfgL458K6Je6vqmlW9vp9nGZZ5VvY3KqOpCg5NcDtNfYniXxpc/Ef9lK88T3dgmmTatoo
+uzaJIZBGGwQNxAzxjtXyFt+lfV5Xjp46i6s7b9PRM+BzfLv7MxLwz+KN09U9U2nqtOhBtNG01Pt+
+lDbI1Z3YIigszHooAyTXscx4vLfY5/xX4ii8LaWbhgstzITHbwk43t6n2HevHJtY1S4mklfVLre7
+Fm2yEDJ5OAOlXfFXiJ/E+tSXmSLZf3dtGf4YweD9T941k7a/C8+zqpmOJapSapx0jZ2v5/Pp2VvM
+/wBA/DzgahwvlaljKcZYqqk58yT5e0Ffbl+01vK+rSjb0r4p6snh/wDac8W6rJCbqPTvF0l40Ctt
+MojuA+zPbO3Ge2c1c8FeLtM+KX7S2ka/8SCbnTdY1dXuYgcxRk/LbQtn/lgjCNSD1UHPU5ofF7R2
+139pbxhpKzi2fUfFclmJ9u4R+bOq7sd8Zzj2rr/2wPgDZfBTxdpC6JDInhnVrFUiZ2LMtzEAsuW/
+vMCsn1LYAArwZRnedRK8Yy1+/wDr7z6enWwMo4TA1JONevQcYyW6SjFuz6PXmXfl8kfpzd3MNlay
+3M8gigiQyPI5wqqASSfYDNflVofhi/8A2kvG3xY8VIJg1rp17r8KxjO6QOBbQH2Matx32V9G6t+0
+u/iX9hzUtSnuseKmVfC90c/M1w+FMvr88BMmfXcO1fPfwb/Zv+JnxM8LS694MuYNN0qWZ7Ji+py2
+jTeXwflRTuQEkcnru969PGVViZ04QV1a9l5/5H5fwdlcuHcLj8VjasaE1NUlKW3uvmlba6kmrelz
+1L/gnN48Ww8c+I/CksuLfWLNNRtgx6yxHa4HuUcE/wDXOsr/AIKQaWLP4r6HqCLt+26CYy3ctHM+
+P0kFeRfD/Vr/AOAnx90efVALS68Pax9i1FI2yvlE+VNtbjK7HLA9xivpb9t34neDtJ+I+k6P4p+H
+Q8YtYaYL22vF12aw2CSRgyFY0IYfuUOSe9c0ZqeClSk7OL8/0T8z6XE4Wpg+NKGZ4Sm6kK9JtqLj
+rypLTmlGL05Hv1ue++NNH/sD9lqbSwuwWfh6C3Kgf3YkU/yr488v2r7M+IOvt4s/Zsvdaa3W0bUt
+CivDbq24R+ZGj7QcDON2M8dK+OfL+tfsGQtfVnba/wCiP4s4h53jpOp8Wt/W7uQ+X7VxXxX1r+zd
+AjsImIn1BirY7RL978zgfnXdiPNeHfE3VDqXjK8QHMVmFtU9Mry3/jx/SseJsa8Jl0lB+9P3V89/
+wTXzPuPCvIo5zxLSnVV6dBOq/WLSgv8AwNp+aizl8UtFFfhx/fx+tV9+zR8MdR8UyeI7nwfZTa5J
+d/b3vWL7zPv37/vYzu59K6bx78NPDHxR0mHTfFWjW2t2MMwuI4bgHCSAEBgQQRwxHXvXwpo/x7+I
+M3EvjjUXOO5i/wDiK9N8J/Fnxdr3w+8VR/8ACT3baxpEsGpxzgJ5j2pPlzIcLjap2t0zzX2+GlRx
+FVYdRS53bVafM/gPE43MKcY4mdecnSXu+9K8V/dd9PlY9pH7JnwjW0ktB4IsBaySLK8AeTYzqCFY
+jfgkBmAPoTXoPhHwZongLw9aaF4f0+LStJtd3k2sGdqbmLN1JPLMTz618cf8Lj8b/wDQ3X35x/8A
+xNH/AAuTxv8A9DdffnH/APE19VDh+UHeDin5f8MfPYnijE4yHs8TVqTje9pSbV9r2b3tofSviv8A
+Zp+GPjjX7zW9c8H2GoareEG4upC4aQhQoJww5wAPwqXxb+zp8N/Hd5a3ev8AhSz1S5trWOyhlmZ9
+yQpnYgIYdNx56818y/8AC5PG/wD0N19+cf8A8TR/wuTxv/0N19+cf/xNT/q63e/Lr5f8A1jxbjYc
+nLXqLkVo++/dW1lrorJaLsfTXxd0i00L4Ga7plhAltYWeli3t4E+7HGgVVUewAA/CvjLYa6nVPif
+4q1vT57HUPEt3eWVwpSWCQptde4OFBrmtyf31/OvpMvwksHTcJNO7vp8j5LHYpYyr7TW/n6kYYQh
+pW+7Gpc/QDNfL89w17czXLHc00jSk/7xJr6U8Q3C23h7VZQ65W0lIwf9g18zRDZEg9AP5V8LxnUf
+NQp/4n+S/Q/qHwMwsVRzDFPdunH7lNv80Pooor82P6kP0BstD0pCD/ZViPpbJ/hXZ+CrPTbHxBAh
+sbaK3vUayn8uFV3JIMYPHTdtrk7Q5AOa24WLICp2uOQR2I6V9j8ElJbo/wA+/iXKy3N4csLOaWCT
+TrQSQsY2zAvUEj09qb/Ymm/9A2z/AO/C/wCFdL4j23V5BqKYEeoW6XGB2fG1x+Y/WsrbX3FOq6kF
+JPc+SnTUJONtjP8A7E03/oG2f/fhf8KP7E03/oG2f/fhf8K0NtG2tOaXcjlXYz/7E03/AKBtn/34
+X/Cj+xNN/wCgbZ/9+F/wrQ20baOaXcOVdjjviPoFjJ8OfFixafapKdJuSrLCoIIjJyDivzwgbdDG
+R3UV+nN/p/8AamnX1kf+Xu2lt/rvjZf6ivzHSF7ZfIkBWSEmJgeu5Tg/qDXwnEibqUpPs/zX+Z/U
+3gxVj9Wx1DqpQf3qS/8AbR9FFFfGn9HH6BwExuV9K2bGTIwetZF5E9ndNG4wysUb2IODVyzk+YGv
+r0+aKZ/n5azsd1YP9v8ACLLyZNNuePaKX/7IVT21L4LkWTVnsZCBHqFu9sc9A2NyH8x+tIARwwww
+4IPY96+my6pzUeV9DwsbDlqc3cj20bakxRivVuefYj20bakxRii4WGxuY5FdThlII/Cvz2+OfhX/
+AIQ74u+KdOVCls92by3z0MUw8xSP++iPwr9C8V8x/tp+BjNa6F4ytot3kn+y75lHRSS0Ln8d6/lX
+zefUPa4X2i3i7/LZ/wCfyP2PwrzWOXZ99WqO0a8XD/t5e9H77OK85HyxTd49/wAFpc0oZ8DAJHbm
+vzpH9mH6VfEjSxZ67LKowlwomXA79G/UfrXP2bggHNe4apqvw41oRi91nw/ceXnbv1CLjPX+P2p2
+jeFPAGurK2kwaVqSxECQ2dwJQhPQHaxxnmvtaWDrUqdqi/M/z3liKcpe6eUWl49rJDcRHEsLLIv+
+8pBH8q7HxBEg1SSeIEQXardRkj+Fxn+ea53xZbWmneLNRtLKFbe2gZUWNOg+UE/qa27KUX/hOyk4
+82wme0f12N8yE/qK7cBPkrOD6nNi489NT7FXFLir2m6Ne6u2LS3aVc4Mh4Qfif6V01l8N5GAN3eh
+D/cgXP6n/CvfnVhD4meTGnKWyOLxRjnrXpEfgHR7dd0gllAGS0kpA/TFUrmTwHpx23V/pEDDtNfK
+D+rVisTGTtFN/Iv2Elu7HCYrJ8WeFbHxt4Y1Tw/qYzY6lA0Ej4yYyeVce6sFb8K9Ps7zwBqF1Fa2
+uo6Hc3MrbY4Yb6N3dvQANkmoPHGi6fpNlam0tUgkklwWXqQAf/rU3VjU/dzi9e5pTVXDzjXpTtKL
+TTW6ad015p6n5MeJvDWo+DfEWpaFq0Xlajp07W8w7NjkOPUMuGHsazOa+0v2p/gq/jrRF8V6JbmX
+xDpUG24t4ly17arzwO7x8keq5HYV8WpcRMikTpgjP3gK/MsdhJYGs6ctuj7r/gdT+7uEeI6XFGWw
+xcNKi0qR/ll/8jLePlpe6Z4T/Z9r/wA+0P8A37Ffp5/wSQ0GOz+F3jzUkiWP7VrqQblXGRFbRnH/
+AJE/WvzKr9af+CXGnGy/ZgNwRgXuvX0+R3AKR/8AtOv6i4sny5ZJd5Jfr+h/nNk/NLE3b2R2Piaf
+z/GOsuuW33jooAyTg7QAPfFeneA/A09hY3D6qoKXYQmxIzt2nKsx/ve1V/h74L+y3F14i1WPbdTz
+Sy28UvHkIWYhzn+Ig/gPfNfn1+2p+3tqHxD1HUPAvw11KWw8IwlrfUNds3KTaqejRwuOUg6gsOZO
+cfL978hyjKK+aV+WjpbVvov+D2R9pjMZTw1P3/8Ahz6c+P8A/wAFDvh58Fbi50Lw/EfHHiaDMb2e
+mSqlnauP4ZbjBUEHqiBmHQgV8M/Ej/goT8a/iFLKlt4gh8Hae5+W08PQBHUe877pCfcFa+bURYkC
+ooRRwAowBTsd/T36V+04Hh3AYKK9znl3lr+Gy/rU+ExGaV6ztF8q8jX8QeM/Efi2d5td8R6zrUrn
+LNqGozT5/BmIrBNlbsctBGx9SoNT45or6SKUFaOiPLc5t3bPcv2EPDdvq/7XPw7jW3j/ANHuLm7J
+CAY8u1lb+eK/X74jvk6fH6b2/kK/LX/gmhpZ1D9q3T7gDIsdFvp/pkRx/wDs5r9QfiG+dTtVPRYC
+fzb/AOtX5LxRLmzSEe0F+bPscsv9Tbb3ZyoYxsGU7WByD3z/AI1yt3+zL4I1e6mvrn4b2k1xdO08
+kg3JuZjknaGAGSeg6V6/4R8LABNQvUO88wwsPuj+8ff0Hb8ePOtV/bk+A2i6peafefEvR47u0meC
+ZFEjhXVirAMqEMMg8gkHtXyU6TxcuSnR9pb+7zWPoMHisTgG50K8qTl/LJxv62auVR+xL+zu23b8
+PdCbd0xPJz/5Er1r4c+AfC3w08KW3h/wdpttpOg2zyNFa2jFkVnYs5ySTksSTzXlGs+A7vR7u5+z
+WrXVmgMiyJFvVkwDllHKt9K7+21bTvhJ8KNS8SayRa2lhZS6tfNkFsKhcqOm47QFHcn3rz1j8Rj7
+Up1JS12bb12LlQp0PeSSPk3/AIKX/tNzeEtEi+E/hm8aHV9Zt/P1y5hfDW1i2QsAI6NNhs9CI1P9
+8GvzLACgAAAAYAA4FdD8QPHmq/FLxzr3i/W3L6prV293MuciIHhIl/2UQIg9Atc9X9AZTl0Mswka
+K+LeT7v/AIGy8j84x2KeKrOXRbA7rGhZmCqOST0r6q/Z6/4J4ePvjRY2uua/N/wgfhidRJDLeQGS
+/ukIyGjgJGxSMYaQg4OQpHNd7/wTj/ZPsfiHeN8UvF9kt5oem3Jh0PT7hcx3VyhxJcuDwyRt8qjk
+Fwx/hGf0g17UtSgYwadYyyNj5rjZkD2UdzXyuecR1MPVeDwVuZbyfTyXmut7+h6+AyyEoKtX67I+
+bfCX/BMz4JeG7dDqlhq3iecD55dT1ORFY9zsh8sAV10f7Cn7PcAx/wAK70k/9dLqdj+sprt7jStZ
+u3Lz215Mx7vz+lQ/8I5qH/QNm/74FfDSx+OqO88VL/wJ/wCaPdjTox+GkvuG/DT9nL4U/B3X59b8
+GeFdM0DVZrc2kl1bSMWaIsrFPmYjGUU/hXVT6Wmu+LGlciSztY0zjkM/JA/qfwrlj4cvwCf7Mm+m
+wVpePvG2nfAn4O694s1NQbbRbGS8liBwZpf4IwfVnKoPqK4ZurVqL33OctFd3f5s3TjbWNktT5P/
+AOCj37WVx4LsX+FPhG9a313UYBJrt/A2HsrRx8sCkfdklHJP8KdOXUj8yVjVFCqoVQMAAdBWr4n8
+Uap438Tat4j1y4N3rWrXUl7eTH+KVzkgeijhVA6AADpWZx/kV+45VltPLMNGjHfdvu/8u3kfCY3F
+yxVVy6dD+gmOOZru5SCRY7gxjbI67gDgckV8v/8ABT/xs/hn9ms6LBLsm8Sarbae6jq0KFp5Pw/d
+KP8AgVfU9rzq8v8Au/4V8Af8Fc9WYp8LdKGdpk1C7b0JCwov/obfnX4VwjRVbMacX0m39yv+h99m
+0/Z4eUl2R+dx5prhjGwQhXI4J6A06iv6NPzA/QL4Vf8ABSzwP8J/hv4b8H6b8ONeNnotjFZqy3du
+PMZV+eQ89Wbcx9ya6v8A4e4eFv8AonHiD/wLt/8A4qvzSo/AV8tPhnLKknOcG29X7z3fzPajm+Ki
+kk19x+lv/D3Dwt/0TjxB/wCBdv8A/FUf8PcPC3/ROPEH/gXb/wDxVfml+Ao/AVP+q+Vf8+3/AOBP
+/Mr+2MV3X3H7f/sv/tLWP7T3hDVvEOneH73QLbT9QOn+XfSxyNKwjRywKEjA3gV88f8ABWD4hyaT
+8NvCPgq3kKtrmote3SqeTBbKCqkehlkjP1Suu/4Jcaf9k/ZhknOM3niC+mJ9cbI//adfMv8AwVW1
+lr79oPw5p2W8uw8OIwB6bpbiUk/ki/lXxeW4KjHiF0qStCm20vRf5nvYqvP6hzy3Z8Z0DOOlFG4j
+jNfr+vQ+EZ//2Q==
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="aziujd.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <7255e01d0b859492655920a319957c@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAdwBa
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+/Tu141iUf7P9BX52/wDBXGB18TfC+Y/6trXUUH+8HgJ/pX6IwHbrLe6f0FfE/wDwVn8Ky3/wx8De
+I0GY9K1mS0lI/hW4hOPw3QqPxFerwdUVPM43/mkvvi0cmcRcsNJLsj8yKvaHod/4m1vT9H0q1kvt
+U1CdLW0tY8bppXOEQZIGSSOpqjWl4Y8R3Hg7xRoniC1UvdaRf2+oRqOrGKRZMfjtxX9DT5uV8u/Q
+/NYJOSUtj1X/AIYv+On/AETDWf8Avu3/APjlH/DF/wAdP+iYaz/33b//AByv2f0fxja+KfBNl4m8
+PgavZ6hZJfWSo4UzxugdQD0BIP58VzOifGSDV9XtLF9LmtPPl8ku8oJjb0Ix1zgY96/I5cZ4yD5Z
+Uop/9vf5n2kckoSV02fkN/wxf8dP+iYaz/33b/8Axyj/AIYv+On/AETDWf8Avu3/APjlfs63i4Iz
+K1m4ZSVI3jgj8KT/AIS9f+fR/wDvsf4Vt/rbj/8An1H8f8zP+yMN/MzyX9hv4d618LP2afC+heIt
+Ll0fXFlu7i7spypeJnuZCu4qSPubTwe9fCH/AAVGtZLf9pqzlcHZceHLVk9CBNOpx+Ir9TrHxMl5
+dxQm3aLecbiwPPavz7/4K1+CJY9Y+HnjONGaF47nRp2A4VgRNECff99+VceQYqVTOva1VZ1Ob73r
++ljozCkvqXJDaNvwPz7o2nsOKKOfQ1+yHwunU/oIvSU1C5KnBELEH321xH7SXwp/4Xh8BfFPhJAv
+9oX1j5tjI2Pku4yJYTnsN6qCfQmuutdTttekF5YSrdWtzAWikTo429q6C3BjtYVYbSFUc+uOlfyp
+ltaWHqyqwesZ3XqrM/XcTFTiovqj+eFo5YmeOeJ7eeNjHLDIMNG4JDKw7EEEEeopK+1f+CkH7Ltx
+4A8ZXPxT8PWZbwtrs4OsRwrkWF83HmkDpHMcEnoJM5xvWviroea/pzAY2nmGHjiKWz/B9V8v+Dsf
+luKw8sNVdOR9i/sQ/tyJ8DLWPwP45M9x4HaUvY6jCpll0lnOXRkHLwFiWwoJQlsAg/L+jmi3Pw9+
+M9jHrnh7VdO1tG2yC/0e7UuCDkbth4IPZhkV+D1S2F3caVeC7sLmfT7sdLizmaGT/vpSDXzWa8L4
+fMKjr05cknvpdN97aanq4TN6mHioTV0vvP6CLnwzBczyS+bIhc7iFwRk1D/wiUP/AD8S/kK/DjTf
+2h/ito8Qisvib4ut4h0QaxOwH/fTGtO0/aa+NN9eWlpH8VfFu+4njgXGpNnLOFH86+efCGLjtXjb
+0Z6KzihJ25Gftbe6AmnWz3Uc8hkhw6g4wTnivPf2q/g4v7Q3wC17w5aKn9rtEt/pLvwEvIvmjBPY
+NzGT6Oa9T1pDBojRlmkICIXbq3I5NZ/h6/8AIc2shwjnKE9j3H418LRrVKUo16b96Luvke9OMZXg
+9mfz/tFLDJJFPDJb3ETNHLBKpV4nU4ZGHYgggj2pAxA7flX3z/wUZ/ZEuNI1XUPi94OsWn026Pm+
+JdPt0y1tJ3vUUdUbH70Dofn6FyPgZfmUEEEHkEHNfvuXY+lmWHjiKPXddn1X9bn55i8LLC1HCXyP
+3pv7A3WnSWkFzNpxYAJNaNsePBzwRUGjXHiS0maz1G+t9T04plbspsuEYY2g44b64BrRyaWv49hW
+qU4uCejP2uVOMmm1qadxBp/ivR7vSNYtLe/s7uJre5tLmMPFcRsMMrKeCCDgg1+bX7TX/BNXX/CF
+7ea/8JYZPEPh5iZG8OSSA31n6iFmP76MdlJ3joN9fomCQc557EVuaPey3ReOUh9gyH6Gvs8h4jxW
+XVFGD36PZ+vn5o8PH5bSrxbktPy9D+fTULK60jUZtP1C1n07UIDtls7yJoZoz6MjAMPxFQd6/e7x
+18LvAHxdsxbeLfDWj+IkAwp1G1RpE/3XI3L+BFeE+JP+CZXwQ16V5LPTtZ8P7udularJsH0WXzBX
+7Fh+McLJWxFOUX5Wa/R/gfE1MkmtaUkz8iDxXW/CDSD4g+L/AIC0wLv+1+INPh2+oNxHn+tfole/
+8ElvADuTa+N/FNsmeFkFtJj8fKFbHw2/4JjeE/hv8Q/DXi238ba9f3Oh38V/FbXENuI5XQ5VWKqD
+jPpXZW4py2VKShN3s7aPexlSyjExmpNKyfc+uvEbf6Cq+sgrndn+PFb/AIiOY4F9WJ/Ssbb7V+Q0
+vhPsJas1tO1JLuL7Nd4ZiCuXGVkHofevEdQ/4J9/ADU7+5vJvh1ZrLcStK4gu7qKMMxJO1EkCqOe
+FUADoABXq5QHtUourlRgTOAOOtbUqtfDtvD1HC+9m1+Qmoy+NXOV1vT/ABT4Fs5L2TWNK1zSoiNx
+1d1sJ1ycKPOH7tiSQPmC5zVC0+LOglEOpu+hBiVE92yvaEjqFuYy0Rx7sMd8V8WftT/tGax8X2sf
+Ck9tDZ6dpEjS3qWzl0urkDAJHZYwW4yRuYnsK9m/Z1vtNn/Yc8Z6aJ7V7qOz1lnsyyl1BRyvyfTB
+FfDTjQrV5U4RtZPVd15H6/PhOvg8mw2Z4qpaVacY8ttozvZt33sr2ts+59Cr478MHp4n0M56f8TO
+Dn/x+ul8IaxYatLO9hf2d+iqoZrS5SUKcnrtJxX56fsn/sreGv2gvCmv3+r6jqGm3WnXUdtD9gEW
+1g0SuWYOhycnsRWd8RPAWsfsW/Gnw9caDr/28yIl5E0a+Q88HmbJLedASGDDIB/EYIzXHRjOioYm
+Ufdv3+R71fg/LcRja2S4XGt4qKbUZU2k2kpW5k2tv+GP0J1XXtI0SVE1PVbDTnlyY1vbqOFnAPJA
+YgnFVE8e+GU+74r0ZfYapCP/AGavif8Ab48TWnibX/B0iWF5Yz21ldefZalbGKSItJGQOeCOvKki
+vULH/gnb8P7uwtrj+3de/exLJlWt8ZK54/d19O8biJ1p0qME1G27tuj5Cnw3k+HynCZlmmMnTdfm
+so0+b4JWevMvL7z6a0nxTY620q6XrlrqTRYMi2V2kxTPTO0nGcHH0pLXxfYX2oPp9t4gs7m/QsHt
+IbxHmUr94FAdwxznjivPPgX+zZ4e+AF9q82h6hqN62rLCkwvjGQgjZiu3Yo67znNfM/7Pumvb/tz
+eLozEFeC41h5MDpmQYP/AI/+taTxVSn7JTgrydnrsefhchy/MP7QqYLEylTw9Pni3Gzlpqmr6Wel
+9T7okklmI8yRnx0yelM8v2qXb9aNv1r1j4Ei8v2o2e1S7frRt+tFwPy30rwV/Y/wl13Xb5ovt+o2
+itbRykmVIC6neOwZ+v0x610PgSNW+B+rSb4FKC7DZjJkzg7cN+de3fEj4P69rngXWLCwvob29mgW
+OC0ESwq2GXC7jgKAo7+lc54L+BfiXTPg7qWkXs/2TWpVu/L02ORHjcuPkBkBwN36V8osLKjVUYRd
+lB6+d/zP3WvxJTzfK5V8XViqksXCXLdXjBQSVlvyx2b2vdvVni/wc1n4ladpt9B4D8RS6Jay3Cfa
+Y0lRA8pT5W+ZG7YGfavQ/hr8Mv8AhJfijaeJPjF4nubyOCaOVldJbprtozlI5JAMRxKQMgDnkcZr
+ofgf8CPEHhfQtWg8Ru+jTzXMckMdpKkwdQhBO5Twc44969EX4ZQwrKp1O/kEgAOSBjBzU4bAxdKM
+qqd+zbt9wuJuNsRSzPFUcslTUJWSqQinNpxV/f1vrpdbW01VzgP+Ci2t2Os638P7/TrmG+tmsb0p
+LA4ZT88PcV0Ft/wTfa50+GeH4nagjSRB0Q6edoyMgcTdK4D44fATXPEl3o7eGrb+0o44JUujdXKR
+FWLgqBnGcjNbdv40/aqtII4I9ZsykahF3ixZsDpklcn8ayqQh9YqSrU5STtayfbXqj28uzTEx4ew
+FDKsyo4ecPac6qSinrN8ujjK2l303W57N8E/hf4m/Zi0jVtN1Cxu/GOmX92t2dR0lt0kGECkGFju
+I4zkE1wnwDGnX/7afxT1OCVVilsy9utwphkLTNCWG1sEHKtxXb/s1ePPiq/inVY/ixrNmdMe1QWA
+H2ZB52/nmIA/d/vcUvhv4W/EvxX8e9fu/Hapq3wzeS8bSVe6gYxq0gNuU8vEi4XOOeM12qzVLkg0
+lLa22++u33nxcq1anXzR4nF0qlSpSs5xmrS1i7QtFKUrR5eVcu+/f6DaFkOGBU+4puyuTb4ZeI/D
+PzeFPFk7W46aXr6m7gx3Cyf6xfzNVpPiJqnhs7fF/hS902McHUtKze2nXgnHzp+Ir21VX2tD8r9m
+/su52uw0bKp+H/EujeLLfz9G1S11NB1FvIGZfqvUfiK0SACQSBWqaexnys8Bk+IdrvjRdNvZGcAg
+Dyxg+h+bg1WPxIttrkabcZGMK0qAtXwKYXvdX0i38N+OdLiv5FhjeKeK8kH2tnI+Rs/d5QDI4wa5
+W68Q6zpFvew32v7LncghuLayuJEQq58zo3dcAcflXmRxUZbM9iWBnDdfmfo9J8TIVKbtNdQ2Dua5
+Xpnr09v0qpJ8TEkMpTTo2SMZZmuegJwD9zvX52x+M9aCWEx8XXstnuDbYdNlO5FfLr17jP50Dxd4
+muzqElh4v1+3gjVpdiadIP3ZkAVAc8kbh+VX7Tp/kZfV1/Vz9BZfiarhQtjCXOQVMzfLzx/D+NV5
+/iBIkkqPY26PHlcbpG+b0+7X56XHjjxabW1ij8UeMTLD5nmSJZuvm7myvf8AhAxU1x4n8ZaXqF/b
+3Ov+Nr9gklvvjDABmGBIuT261POUqH9an33J45upIi62VuV3bSAkp5xmp9G+KPiLTLxF0iWS03Hh
+BBI0Z9cqSRjIPavzzGpeLLqzaQ67482+Z5YVrkoTxncOenallvfFGs39vH/afjW0jdEiMv8AaJVR
+tUDf174zn3qebyD2KP1I8MftRarbbI/EHh+a+jIGLjToGjcDuSjEg/gRXsvhP4m+HvGgjXT7xo7l
+13fZbuJoJhzj7rAZ/DNfh2154uS0ukS78YuJfLJdtZYGPa2Rj5uMniqF3ceKPs9qz6h4imkYOzxy
+6+37oq2Bn5u45p+1a6EvDp9T9y/Evwj8LeJrj7ZPpwstRX7uoac5trgc5++mM/jmubb4Y64jFY/i
+Z4lRAcKpSBiB2GdnP1r8pvhn+1r8aPhFqTf2br13Np+4tJpXiTUlv7VBgnaokPmRj/dcV9QaV+3t
+8aNS0uzvP+FTeG2+0QpLuXWZlB3KDwNhx16ZP1Nc9fGYTDpSxNRU0+7tc1o4HGYhuOHg5tdle35n
+Aeb4n13U9GhgufHWkybYbN7kaRbxKXMhBmY4GCA4zz0XrWfDeeJ7a31m1cfEW+ecpH9pEMCGDy5d
+xZWx/wAtPu89Qetc2bq3vV0C31Sx0+LTI1W3SebxXIWNv5zF2HPzFdz9u3tS211aWkeu21ta+H5t
+OkCC4ml8RykiFZswseeMtt7d68rldz6RTjZL/Lv6li/+Hst2mn3raX46mujIXIW7t4WLLJwrDGDu
+wOMd65rUPhp4pv8AUdUYaZ4gt1jSW6WC51eKF8+YB5OCQCfmPH+zWxt0qBNMaxtvCbTq5Lxvr0z7
+ZPMBixk9+tWLix0u91fXv+EhsvA0d/5dw4jGqTF/t3mD5WAJG3mTI6cCtoVZw6mMqUJ62/r7zy/W
+tL1m3srW2vNJ1y2Fu0wjM2tqhl3FSQTnBwQBxnGarHRLvSdZvksNMvtRg8qWFXfxApDRNH874B4w
+MnPtXqV1Z2Enhext/s3w9n0oTXIskmkllCzHZ5o556+X+tZ+seAfDl14pvV8Iap4RtIpI3+zxyRz
+yAR+TmYHcuRwJO5zxXRHERb97T+vQ55YeaVo6/16nlkfh9ZLPc+kxq4kACTa8Dldp56+vFWZ7P7V
+e6dFqmlaVbWgihiaRtXORbgYVxzzwDV2/wBAg0+0JS48LSYfelxa2MsiFNuDnK5BzjtWa9zape6U
++pX/AIdls/KgcRppzufsuThfujBwGGDzXTFxktLfgcsoTT1TX3lT+zbQ22oxx2egNbMqG4Y6qzcC
+QbCcHj5iKzdS0TRvsFvJHF4d84iT7SH1CRtgyNmMeq5zW5bzWgTU2sdS0iC3SPfOBpD/ADReYNo5
+XnDFTUN3q+mvpltt1azW5jMpnZNGJEikgoAMdgGocV/VhJy/q5WsfCaeIPGtnpl7D4ba6vbtILiK
+CR3lO4ZIUDgnbz1r9i/C37N3h638MaRFPGyTx2cKyKFHDBACPzr80f2ctEk8TftF+HNJu9Ui1K4S
+4SSUpp3lByQmDvI7K3b0Nfsv0ryKmBw+Y4uUMVBTjCKsnrrJu/4RidUsbXwGGjLDTcZTbvbtFK34
+uR8j65/wTy8KXUFv9gvYEa2XZClxZZUJuLFSQ3QktnjvXi3xD/YY8e+C7W/utDvNButOkAEyvZxg
+bAwYLzg8EDHXpRRWuKwEcNRlUoVJRcU38TktP8fN+FjHCZlUxFeNKvCMk2l8Ki9X/d5fxufPXiaV
+/BL2lnqms6RZ34dkMsOieZHISw2AY54HGSBUsnjePStY1hdZ8X2FzdGKe3SVPDisY7osCJs4GcYb
+8+lFFeZleLnjsBSxVVLmktbbdu59BmWDhgsbVw1Ntxjte1+nkWG8d27eH7S4b4gQpZPczW8MS+GE
+O2UIhdgO3BXnviuisfGul+IPGLR6F40bSDcBjbovhyNfK2W5MgyD3CSf99Y96KK7nNqSRzeyjyuX
+r2/yM6x+I2kXGlt5fja/eSQpLFMnh+BCqAHcv3u+V6+lQ65JoZvvDuo6z4v1W+hubS31JIotJhRz
+bmRv3O7dwSEYc5HzUUUKpJaop4eDai+vp/kcpfaZYak+uz6J4z18wWds9/JbXVlCjGLzVURgqxDE
+GReo/hrkLzxlpy2NvY/2r4gFxbmWR3WKAGTdtwOvbBx9aKK66deblys48RhKcE2u/wCh9AfsaeQv
+7UOj2H9p6tqElvHHKftwTy8MquMYJORkD8K/WiiitcHricR6x/8ASUeTjv4FD0f/AKUz/9k=
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="uloarfifitb.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <e137e01d0b8593928e8c00883660c1@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgBJwBa
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+/Tvw94p0jxZb3c+jajb6nBa3U1jPJbOHWOeJikkZI/iVgQRWoAQK+SP2HLRfC1zLpm5g3ibwjo3j
+F48nabm4kukuZMHuxWEnH9a+uKBLU89+IvwH8GfE+ykh1nRrdpn/AOXmFAr56gnjDc/3ga+NvjF+
+x54j8F6vd6vp2rWt3oNxFKHurjTkl+ySN92R1VdygHOTyOe1foXTGTepVgGBGCCOD7V5FbLqbk6t
+D3JvdrZ/4o7P10l2kj2KGZVYJU63vw7Pdf4Xq16ax7xZ+ZGmfs5eL761jih+JXhqGDJeMDTFwrMo
+G8ZXqQB+Qr0OD9mLxZcao95pfxS8NaLGVURrFp6I0beWFcg4/iO4/wDAua7P9q79jCw8X2tx4m8H
+QnT9VgUyS2dqNquBydox077R06r6H4Rt9BktLuW1u3u4biJtrxtIOD9cdPQ1hSlGpJ0aseWotWru
+zXeL6r8U9H0v01p1acVVpS5qb0vyrR9pLo/wa1T3t9uaV+yj4nt7O5huvi7oBkYJ5DxabAPKwTuI
+GR1BHT0q3H+yx4gN1Ytd/GnTJYbZ4z5J0+Da0auGKYLcAjI49elfH2n+G4JgCZrpvrIp4/75raXw
+fbvGNzTk+pZc/wDoNdPsIL7L+9nK8bWfVfcv8j6hvf2UNRlu7yWP442NlDcPIfJjsbcAIzEhP9YM
+gDHHtmpj+xVJ4y0CPTU+Ldvq01pcPLLcw6XBK4V1UCNtsmQPkYjJ/iNfJ1z4ItmGD55+rr/8TX1d
+/wAE8NDg0O9+ICRbgZvsLENjsJR2A9aqNGDe1vmyHjK1t19y/wAjxz4I+EB8Of2mdDt/7R8UahFD
+cpC8+qWapZfMVAKSAlsjOAD7+lfpjmvzA1bQ7nwb8YbjUrCz1W7hi1F7t7ibWonhhZpWdx5Oedh4
+2/eGa/TXTb+PVdOtb2ABoLmJJo2z1VgCD+Rrz8tqP61WjLqov80/usvvPTzWEfq9Gcejkvyf43f3
+HzP8FkTTPiz8MrZEkzb/AAyTRrqSSMoPtEDWUipg87gsjkgjvX1FXyf4W8F3WiatZ/FPS/E8qprt
+h9qi0qazVpULRBmV5pHzsGwAfKp4Ga+ro3Lxo2MblBIPavoz5hDqKKKBjQpA618X/tm/s7pDG/jr
+w5ZAuG/021jIUbifvDsAxP4N7McfaVVNT0u21jT7mxvYlntLmNopYm6MpGCD+BrhxWHdaKlTdpx1
+i/Ps/J7NdvOzO3C4hUZONRXhLSS8u681un38ro/JDQb25YLjTSQf+nlBXc6e93Kg/wCJST/2+R16
+L4g+EXhzwN431XSNWgVJIp98U8lyYlnif5kcDOMkdQO4Nd54X8C/DyUJ5v2fHvqeP/ZqKGMp1qSq
+crV912a0a+TTXyJxGHnQqum3e2z7p6p/NNP5ngt1bXhTP9kY/wC3yOvoL9iD7TF4l8XxzWn2VWtr
+ZgfOV9x3Sdh0rhPidp+i6H4yvLDR54F09IoXjUXSyYLJlvmyc816R+x40Q8ZeJ1jljdjYQEqjgn/
+AFjc8V16NKSOV3Wh8rfFSz0TTPi34hTTLHwfPOupM8v23UpVlW+aVi6su8bTu28DjnrXead+0zea
+Tp9tYv4ms7d7aJYTEt6CEKgDaPpjFcV8b9V0C2+N3iyK21HwvZXUGqMk8d14feWdrvzCWdpMEMfm
+Tnp14rhdSm0GPUbpbx/BD3aysJmPhtiS+TuOdvPOa+FxmX08ao3qyg431i7PW2j+4/RMHjKmFT/d
+KaklpJN7dV959/6bIp/Z68KM+Qx8P3ZLZ54gOSfXpX0HZkPaQMOQY1I/KvnSx3n9njwajcu2g3i9
+Mf8ALBhX0NpQxpVmPSBP/QRX3vQ/OS1RRRQMKKKKAPAP2pvCkLRaF4n8iORreX7Bcl0DZjflDyOM
+OCP+B1xXhFLIhN1laH62yH+lfQXxg0P/AISL4Z+I7LGXNm80Z7h4/nXH4qK+TPBiazPDDJH4jVVd
+QwB0qI4Hp96vAS9li6tNbSSmvV3jL/0lP1bPWqv2uFpVXvFuD9FaUf8A0pr0SIPjDaWZ+IU22ytE
+DWducLboB9089K9B/ZHtbe18d6+YbeGEvpqZaOMKTiX1HbmvK/irDqaeOh5mrJPIbKDMhsVTPDdg
+1enfskpdp8Q9WM91HNGdK+6sPlnPnJznJ/Kvdj/DXoeU92fPf7QviJpPjf4pt7bUvEGlra37W8kd
+poccsTuDu8wP1bIYDcf7tcNqGpapb39zFJrviiWRJWVpB4eQ7iCQT97vXqX7R9prGqfGrxFs0rxZ
+HBb3Jihn0/UFiiuFyH3KpHQFivHpXnt/4b8RQ31xHFbeNJY1kZVkGpp8wB4PTvXy0lZvQ+zjJckb
+b28j7QRWf4FeCxlS66JeHk/9MCP619CaXn+y7PI58lP/AEEV4M5RPhB4LVFKxnR73AYc/wCpr3yw
+4sLX/rkv8hX1fQ+LJ6KKKACiiigCG7thd2c8DcrLGyH6EYr4S8Ia/HppW2bT9VlMJMe+3sWkQ4OO
+COtfeEkoiieRjhVUsTXxH4IR5Y1dVYq7s4wOxYn+teJidMdT/wAE/wD0qnb9T1KX+41P8cP/AEmo
+cr8Vtdgn8X20v2XUIQ1jECs9nIjcFucYr0r9kzWYJ/ibdQRiYPJpch+eF1GBIh6kAVxPxjiZfFmm
+5GCdPTr/AL7V237LkjJ8UkTJw+nT5z/vJXsQ1pI8yW54h+1Zot3e/HHxC9z4W069hjkIsbqXXFtZ
+J0IVnyhb+F2K9BXAS/DvTBK/kaPYNDk7CPEq4K9v4vSvTf2vYvL+OOsC8bwGYGUGwXXEf7UBsQy7
+io6eYeCfWuIjt/h+0alpfAe4gZxJcAZ+mK+alZt27n2EOf2cHd7dLn2lqbeT8K/BSO2CdGvSc9/3
+Ir36w/48LbHTyl/kK+aviXqk2meB/h6kPyGTSNSU4PBxbL29/wBK+j9EnFxpFnJt2kxKMfhivqOh
+8aXaKKKBhRRRQBzfxG1pPD/gLxDqLnb9nsZmU/7RUhf1Ir468H+C7GeOEzfavNIBYx3kyDPfgNgV
+77+1P4jls/BdnoVntkvdWuR+6d9qtFEQ7bjgkAtsHTvXkXhKLxAoj2aNYSHuP7RKfzjrwW/aY2pN
+bQSj8370vw5D15L2eDpw6zbl8l7q/HnOQ+L3hS1sPEOkCKW/VZLEn5r2Rjw5HUkkD2rq/wBmGwSy
++L9kwubxy1hcqEmuGdP4T0P0rP8AjcusJrWgvc6LDC5sXwsepI4/1nXJQVb/AGbbu6Pxm0hJLDyE
+e2uQXFykm3EeRwOea9uDfslfseVL4mcF+2e15pfxm1GW58T2djYXFvG1pZ3Hh83nk7Y4/NbzNp+8
+zKcCvPI/FXgvYu/VNNL4G4jwu3J/KvUf2y47zTPjRd3iah4qEF3bwotvpFuskERSNcsCT1bcM8fw
+157F8WI4YkjaHxs7IoUt/Z8fJHfrXzbupy9T66nrRp6dP66H0T8bNSSDT/g1a55u7HVY9p/68wf5
+ivqnw7/yA7L/AHP618c/E2zu9St/glqkyMv9nanewCJY2fz7Zrd979ONqjPv2ya+xPDDh/D9gwzh
+oww+h5r6jofGrdmnRRRSKCmlwucnAAyTShga8V/aB+IzWVofB+kTEapfR5vZozza2x6jPZ5OQPQZ
+PpXHisQsNT57XeyXdvZf1srt6I68Lh3ianJey3b6JLd/1u7JatHj3j/4jaX48+Jd/ftqtpFZWn+g
+2SyzKuY1OWfn+8xJ+gWu58IX+iKI861pi56br2IfzasLwf4XiKRq1tEUUYCtGGAA+tev+H/C9hHG
+vmadYvx/Haxn+a15VGn7Gnyyd5O7b7t6v5X27Ky6HTiKvt6rlFWirJLslovnbfu7vqeM/tC3mnya
+n4ce31CznC2cqHyrhGx+8B7GsT9nq6iX4y6GvmIS0dygw4Of3Tf4V1P7UmjafbXHhpk0+zQtHOCV
+tkGeUx2rz34AWllb/Grwy8dnBFJ5kyhkiCkZhf0Fe1TadFW8zhktXczf21vCL3fxgk1K0tvEN9d3
+FpDDJFpGqR26Roi5VthGckswz0+WvO7LVtXs7K3gHhTxpKIo1QSf2qh3YGM5C85r0H9ufw9o158W
+7e8l0rTNS1hrKBGjvNdNg4twHKsq5GfnLjNeSabFfW2nWsVv4D0toI4kSNh4lY5UAAHO7nivD5nz
+yXmfSxgnQptq+i30/U+j9f8AHGlDwvo2jve21rdR5jtYJZt8sROcbFGQCc55r6x8F+YfCul+aSX8
+hcl+p9z71wGhfs2eENN1xNZu9Pt73UlcSLO0fIYdDzx+lerxqkMaogARRgAdhX0Z8n6j6QsBVPVt
+asNCsJb7UbuGxs4hl57iQIi/UmvDvHfx51fXbaW08A2DyRE7G1i6Ih3Dv9nR/vHH8TAD0BrgxGMp
+YdqD1m9ord/5LzdkurO/D4OriE5r3YLeT0S/zfkrt9Ezsfiz8XYvBSnSNIWO/wDE86ZS3Jylqp/5
+azeg7herfTmvBNH8B6lqeoXN/Pr+pfa7uTzp5SkTbnPVvmQn9eBxVzw7omqwM0knhi+nnmkLz3D6
+hBLLK56s7FgSa9J0Y3NouH8Ka18ozmFIZAfpiSvMjGrOp7ev8XRLaK627t9X8lZb9lWrThT+r4fS
+HVveT7vsl0j8229o9A8C6tBEnl+KLtGH8Umn27/ptFdnbeHfEUOwr4sGwDlW0eHn8mFR2niYWyIX
+8NeI0z2OnBsfXa5qy/ja3UMX0jXoAvUvpUuP0BrZ8+9vwORJPS54N+1Fa63aDw211rVrfZNwEJ0/
+ytv3M5xIc15v8CLi9X42eEN81o8Zu3DBY3VsGJ+nzEZrvP2sPGemXlj4Z8t7qJhLcZE9lPHxtX1Q
+V8yx/E268JaxpuueH72FNT0+6E6faIWaNlCkEMDtyCG9a9GnNRocz6X/ADIcOafLHqe1/t3jRE+J
+mlLcReGW12SxjZG1yKQubUF+FZTgDzO3Xk814tpzXH9n2v2e1+H4t/KXywkk+3bgYxk9MVm/FT9p
+HxJ8Ur+11LV5NAlntLcoslpYqXKBjkfMHO3Jztz1AOOKxYPiXfiCMRa9oSRhRtUW0YwMcDHk8V4e
+vM2up9DGXLSjTe6P0L1H9s/wHD8lkLy8l7eYEhT/AL6Zv6Vi3v7Td54hBj03WvC3h2Jukkt59snH
+0GFQH65r49/4RWYLkPCx/ujI/WlTwxMR8yw59N2f6V5bxqq/xMU2uytFfelzf+THiwzfKIfweRPv
+K7f3S9374n1SkOj+I76O91jxZa+I7xTmOW/1CNkjP+xECET8BXdaTo2nzFWS/sZAehW6jP8A7NXw
+/H4bdRua2h98suami0ll6WYGDgHCj+tbUq+EpJqlKKvvqrv1b1b83qOpmOFxLTqYmLtt7y08kr2S
+8lZH6IaVoMYCmNo5F9UcN/I10MOn+SuBGTj/AGa/NWOO6XIjS5jx/dcqP0NSrqurQMyxX+pxsOoS
+5lGPyNdKxFOW0l96EqmGl8NaL+a/zP0n8uRf+WT4/wB2q0270Yf0r87Lfxt4osnAt/EWswsBgBby
+b/GrS/Ffx1bMFXxnrUbdg10zH/x7NWpKWx1RjF/DJM96/bEmkj0LwyTIw/0ycdT/AM8wa+Ndd19t
+Nw8sct0hlVCof7u443c9hXceLfFvifxxawW2u+KNTv4oHMkIkmUFGIwSPl9K4dvB14t0Jk1+4kjw
+QYLiNGU++5cGvShWgqDpvfUn6vU9qprY891m3vINSlmaeGGFRLPGIpBgqCeOB97HY1HBJq9zBHMl
+o7JIodW+0JyCM16Dd+CZ73J+2IJQjRpIsecA+oJ9ajh+HFpBDHGLO3IRQvAYdB6bq4b9zuaZ6Z5x
+pPNOP/r14/8A8NN6B/0A9V/7+Rf40f8ADTegf9APVf8Av5F/jXk/6kcSf9AUvvh/8kfzzaR7B5pz
+2o81sV4//wANN6B/0A9V/wC/kX+NH/DTegf9APVf+/kX+NH+pHEn/QFL74//ACQWkewGUmjzW45/
+WvH/APhpvQP+gHqv/fyL/Gj/AIab0D/oB6r/AN/Iv8aX+o/Ef/QFL74f/JByyPYfPf8AvEfjR57+
+vPrmvHv+Gm9A/wCgHqv/AH8i/wAaP+Gm9A/6Aeq/9/Iv8aP9R+I/+gKX3w/+SFyPsevs5c5YBj6k
+A01gr4LRox91BryL/hpvQP8AoB6r/wB/Iv8AGj/hpvQP+gHqv/fyL/GqXBPEq2wcvvj/APJFLnWx
+628Ucow8aMM5+6BR5UX/ADyT8q8k/wCGm9A/6Aeq/wDfyL/Gj/hpvQP+gHqv/fyL/GtFwdxQtsLP
+/wACj/8AJG6r4iOim/vZ87fnR+dFFf2oQH50fnRRQAfnR+dFFAB+dH50UUAH50fnRRQAfnR+dFFA
+ES3ULsoWVCW6AHr9PWlluIoCBJKiEjIDMBkV+v8A8Lvjp8F/2vfFHi74cJ4LivLfS4XkjbU9Ph8i
+9tlkELywlSWTDsuM7Ww6sO+PHPhf4g+GH7Gnxc8XfB/xPo154hl1nxDa3Ph+8OnQ3fl2l1HFHFHL
+I7BgUkDqeDkLuxkkV8dDP6j56UsNJVYpPlutU+t7dL9j6F5XBWkqicXpfzPzf/tG1/5+Yf8AvsUp
+vrYHBuIgfQuK/bv9oH4ifCv9mzwtp2v+KvCENxZX16thEul6RbzSCQo7gkMV4wh79ccV5D+w98Xv
+Bfx7g8a+G7/w9pMuoaVqVzd6et3psAluNInmdoN6gY3RBvKYDgDy+u6sIcS1J4WWMWGfJG2vN8u3
+RlvKqcaipOpqz8p5JEhXdIwRemWOBUZvbcKGM8YU9DvGDX6dfskfsZwfDT9pP4q3WuaWl94d0QrZ
+eHft0HmRzQ3OJ94LDDNFGEhJ9S9dn+yR4g8MfHL4n/HPXIdD0q40O01u10zSo/sURjFtDCyb1G3/
+AJaNvkz/ALY9K3xHElKn7SVKnzxhGLbvb4rWW29n+DM6eUuXKpys22vuPyQ+323/AD8Rf99ipFuI
+nRnWVGRfvMGBA/Gv1/8ACX7RXw18XftLal8GYPhrHb6tYz3UDanLZWv2ZjBHvYgD5sEcDivKf2tP
+DGj6b+3J+zlY2uk2FtZXMx8+2htkWOX9+B86gYbj1FKjxDKdZUauHcG4uavJO6Sb7dbFTyqKhzxq
+X1sfmtFcRTkiOVJCBkhWBppu4BJs86Pfnbt3DOfSv0x/4Kl+Bba1+Gngn/hHPDSLO2tP5p0jTcyb
+fs8mN3lrnGSOvGcV6laeCNFi/YFF5ceHNPh1VPh2zvLNp6JOso08/M2VDB885POeetH+skHhqOJ9
+l/Ely2vt57EvKmqk4c2kVe9j8fZLmGFyrzRow/hZsGpNwHev1o/4J3+END1r9kbw5d32h6bf3Zmv
+1M1zZxySNi5kwCxUk+nPYAV+WVx8K/HouJdvgfxPjccf8SW69f8ArnXqYPNqeKxFeg1y+ydrt77/
+AOXmctfL5UoQnF35lfb0PYPgV+xn8Rfi9458SeFxHd+CJtBR1vNS1SznSFpRIEWFWUrvLDLggkbV
+z3FZ2tfCK9+BP7U/hXwTqes2mvajY63pEs91ZBxGpknjdYzv53BSpP8Avivorxb/AMFZvEWoaE9v
+4b+H1louryRlft2oaibqOJiMbliWNNxHUZbHrXxdY+OtTX4j2fjXVZpNc1mPV4tZuZLp8NdzJKsp
+DED5QxUDgcDoOK58I80xE6k8XFQjy2UVZu/e66eV+pvWlg6SjGi23e9z9J/+Csf/ACQ7wl6f8JLH
+/wCk1xXwP+zZ4u8YfDX4veHPF/hDQNX8QzWV2ttdWel2ck/2qCXCy25KqQGZTuXPRlU9s16P+1H+
+29qH7T/gnS/Dl54OtvDqWGorqK3UOoNcMxEckezaY1AB8zOc9VFVP2WP2z9R/Zc8Oa/pNl4Ut/Ea
+6teremWe/a38oiNY9oURtkfLnPHpXDl+CxeCyh4aVFSm7+62rNPz22Na1ehWxqq89oq2vmfoz+21
+8Zj8HP2cPEOqWsrWuuavGNI0xTw6zzqQW4PBjjEj/VK+ff8AgkSixeDviUi/dXU7NR9PINfGn7RX
+7TPi/wDaY8S2mpeJPs9jp+nqyafo9ju+z227G9yW5eRsAFjjgYAAznrv2Vf2xb79lnSfEdjZ+Fbf
+xINauorlpJr5rYxbE2BQBG+c9c8V5qyDEUMmnhYJOrNpvXs1pd9l+Ldjp/tKlPGqo37iTR7J8Fj/
+AMbXfFZ/6iGr/wDpMK9N/bBOf28/2af+u/8A7cCvjTwd+0/d+D/2oNU+NEfhyC5vL+e6mOjNeMsa
+efGEIEuwk4xnO3nPauj+K/7aV/8AFb43/Dr4kz+EbbTLjwY5eLTkv2lW7+cPhnMYKcjsDXVUyvFy
+xlOso+6qPI9V8XLJW37vfYUMbh1SlFy1cr/I/SH9rD9qO2/ZZ8MaHrN14buPEiapetZiK3ulgMZE
+bPuJZTn7uMVZ8f8AjVPiT+xx4l8Vx2j6emt+CbrUFtZHDtCJbJn2FhgEjOM1+af7VP7Z9/8AtS+G
+ND0a88JW/htdLvWvVnhv2uDITG0e3BjXA+bOeeldZbf8FD9Ut/gGvwuHgO0NsPDv/CPf2n/ab7iv
+2fyPO8vy8Z/i27sZ4zXhw4axFPD0JRpfvVK8veXw3062+7U6pZpSlOceb3baaPc+xf8AgmzIY/2P
+fDLKu5luL8gDv/pMtfOkv/BTT41JK6j4JREAkAm11Hn/AMh157+zt/wUE1P9nj4UaV4GtPA1prsN
+hJPIL6XUmgaQyStIcoImAxux17V6T/w9w1s/80wsv/B5J/8AGK6p5Rio4zEVp4RVYzk2rzSsrvz6
+/oKOPoeyhFVbNLsfAlFbOpeCfEui26T6l4b1nTYHdYlmvdOmhRnbhUDMoBY9h1NN1rwZ4i8NW4uN
+Y8O6xo9uZBEJ9Q0+a3jLnou51AzwePav0pVIO1mtT5B0preLMikpaKszCkqewsbrVb6CxsbW4v76
+c7YrW0iaaWQ+iooJP4Cum8SfCLx74M00aj4g8D+I9D07G77XqGlTRQgepcrgD64qJVIRajKSTfma
+KnOSulocnSHmgEEZByDyCOhqazs7jUr2Gzsrae9vJjtitrWJpZZD6Kigk/gKt6ashJt2SIaWu01T
+4IfEnQ9MOo6j8PPFdjYBdxuZtGuAgHqTs4H1riUdZF3KwYdMg1EKkKivCSfoVKnOHxKw7tRT7eCa
+7uora3hlubqZtsVvBG0kkh9FVQSfwFdwnwB+KcqK6fDLxg6MMqw0O4wR6/cpTq06fxyS9WONOcle
+KP0I/wCCrHxA0Sy+EGi+CHuifEOralDqEVrF95LaDO+Vz/CNzKq9yc4+6cdh+zT4l079tP8AY/m8
+N+N2bUtQijfQdXuHbMxmjCtBdgnnzCpil3f3wa/NP9ov4v3Hx0+NPijxjLI72VzcG30xGz+6soiU
+hAHbIy5/2pG9a+rv+CSXiaaHxv8AEbw7uzb3NhaakFPZ45HjJH1Ei/8AfIr86xmUvA5HF3/eU2p3
+7NtXS9PzR9XRxnt8c4fZeh8T+O/BOp/Dbxtr3hTWVC6rot7JZTsoO1yp+WRf9l1KsPZhWA7+WjMA
+XIHCqMk+gHua+vv+CovhGDw/+0nZatAoU6/ocFxNx1lhd4Sff5BF+VfMvw1s4dR+Jfg20uMNbz67
+YRyBuhU3MYNfb4LF/WcFDFPrG79ev4nz9egqeJdJbXP1+/Y5/Zh0b9nb4ZWM91Zwv421O2S51rU5
+UHmI5Ab7OjH7sUeduB1ILHk8avwI/ay+H/7TGseKtC8NC8M2jtiSPUrdUS9t2ZkE0Q3HdGWUjDAH
+lcjmvSvibPJa/DXxZNESJY9Ju3QjsRC5Ffl9/wAEnbmSL4/6lGpJSbwrKH/Ce3I/UmvyChh/7Vwu
+MzDESbnGzX9draLsfaTq/VqlKhBaPQyf+Chv7OOnfAj4mWOueGLJLTwr4pSWWGwhGI7S8jI82KMf
+wo4dXUdFPmAYAAH3H+yt8Jfhx+zZ8GpNaGp6Pf61Fp51DxJ4lt50uGG1N8iK6E4hjAIVV4O3OCxN
+cJ/wVc02Cb4AeHb1gDcWviWBI2x0DwXAYfoPyr87f2dfBej+Nvjt4G8NavbGTRtY1RLW+ghkaDzo
+irEqWQgjlR39RX1dCnVzrJYOtWcVC/NZXclHbqun3vVnkznDB41qEE3K1vI/XH9n79sz4e/tJeIt
+V0TwsdVtdTsIPtfkataCAzwbghkjwzZALKCDhhuHFfJP/BUv4G+HfBy+H/iToVhFpl9ql4+natBa
+oES6fymkjm2DjzP3bqzfxBlz05+1fg3+zF8NPgPf3+oeCPDkem399GLe4vXuZbiVowdwjDSM21c4
+OBjJAz0FfCX/AAVE8Q/EXUPGegadr2gJo/gG0klbR7y2n89b+4K4d5WAHluEztiI6Fzubnb4WSuj
+LOY/2e3GnbVSau1bVW6669bbnfjVJYOXt1d+R9LfsS/Bj4f/AAQ+ENt4ka/0bUvFt1Yf2jrmtxzx
+zG0TZvaBGBPlxRqcHH3iCx7AYU//AAVa+DEU8iRWPjC6iViEnh0lNkgzwy7pQcHqMgH1Ar80vgt4
+Y0zxP8Y/A+h6jCW0zVtbs7G9jhkMJkhlmVJFLqQRkEjOa/Vv/h3R+z//ANCrOPYazdf/AB2vQzPB
+ZfgsS6maTnUlU1XKkrLtq/u7GGEr1sRTSw0VFR01/wCGPx1RBGiovCqMCvun/gkto803xU8f6qEJ
+t7bRre1Z+weScuB+URr4XJABJ4wMnPYV+sP/AATw+Htt8Ev2Z7/xx4mkTSD4hL69dT3J2C30+OPE
+BbPQeWGl+ktfU8TV1Ry2cOs2opfO/wCSPFymm54nney1Pm3/AIKsa3BqPx+8M6dE4eTTvDymYD+F
+pbiQgH8EB/EV8bWV/caVfWl/aHF3Zzx3UB/6aRuGX9QK7b47fFSf43/GDxV43mR4YtVus2kMgG6K
+0QCOBD7+WoJ92auEPI6V62WYV4XA0qE90lf13f4nHi63tMTKpHufvd4I8X6P8dfhHp+u6bMJNI8R
+6ZnKHJj8xCsiH/aRtykdipFfKH7AP7HXjf8AZ7+IfjDXfGcFlbxfYho+mG1uFm+1R+aHefA+4pEc
+YCthslsgYGfiv9nT9rvx5+zTJc2ugyW2reHbuTzrjQtT3GHzD1kiZTuiY8ZxkNgEqTzXvPi3/grB
+421TR3tvDvgjSNAv3Tb9uurx7zyiR95Y9iAkdtxI9Qa/P55DmmE9thMGk6VW2rdmkn9/k9HdH0sc
+xwlVQq1XaUTtP+CofxX8PyeIvhz8PtQkmu7C11BNd8QwWMgWaO1/1SKrc7ZCjTsOM/Kp6GvQNV/4
+Ju/B7xD8Lb9/AwuF1rUtPE2i+IrvU57iON2UPDKAGClGGM4U/KxxX5b+IfEOqeLde1DXNb1CfVtZ
+1CUz3d9dNukmc9yfQAAADgAADgV7h+z7+298Rv2eNKTQtNe08ReFoyTDo+rhsWuTkiCVTuRSSTtI
+ZQScAZr3K2TY7C4KjSy+s1OGrV7KTbv+G2ujW5wwzDD1a83Xjo9vI+qf2E/2RPi/8D/i5qHiDxbc
+2+jeH/sElpLp1vqRu11GRmBjbaPlUJhjubDfNgDBNeif8FOdW0TTv2b4IdYjNw9xr9h9ntomCzSb
+JN83lk5w3krKM443D1rwjUf+Ct3iSWxKWHw10y3vMcS3OqySxg/7qxKT/wB9Cvkb4zfHTxn8ffFC
+a74z1X7dPArR2dnbp5VrZISCyxR5OM4GWJLHAyeBXBh8pzLG5lDHY9KHLbbrbbZv5ts6KmOwtDDu
+jQfNfv8A8E/RnSv+Cd/wM8Z/C2TUfBwv7i41nSzPo+v3WpzTeQ8keYZtgYKcEglSPUcGvkWb/gnD
++0DDM8Y0HSboIxXz08QKFkx/EAwBwevIz61y37Pf7ZXxD/ZxtW0zRJrXWvDLO0v9h6sGMULscs0L
+qQ0WSSSOVJycZJr6IX/grjrm0bvhhYFscka0+M/9+a2+r8QYGpONGSrRb0cnqvvat59CFWy6vFOa
+5X5X/Q+BVIVkJRXCsG2uMqcHOCOhHqK9c+Iv7WvxY+K/g+bwr4m8VfavDszRmWwtLC3tEkCEFUYx
+IpKAgHbnHA9K7LTf+CePx41PR11BfC1haF1DrY3erQx3IB6AqMqp9iw968G8W+Etb8A+Ib3QvEul
+XWhazZECeyvk2SICMhvQqRyGBKkcg19dGtgMfUXJKFSUNVs2vNdvVHhunisNDVOKZk/Xn8KSvevh
+1+wx8Z/if4ctde0rwvDY6XdxiW2l1i9S1edCMq6xnLhSCCCwGRz0rivjB+z38QPgNeWkHjbw/Jpk
+V4SttewyrcW07AZKrIhIDAc7WwcZOOK1hj8JUq+xhVi5dk1czlhK8Ie0lBpeh53ir+h+HtX8UTND
+omkajrMqnDJp1nJcFT6HYpwfrX0D+y38A/CviLwv4h+LnxXuJLT4W+GH8o2keRJq90MfuVwQxQFk
+TaMF3YLkANXvPgb41ftGfHHS5D8BPAGgfDj4d2jNBZyTW8CLIAccM42Mwxg+XGVByNxrzcXmvspS
+hRimo6SlKXLFPtezbfklp1O2hgFOKlUb12SV36nwb4g8G+IvCUYk13w7rGiRH/lrqWnzW6f99OoH
+61kDmv0V8M/EL9sfQ77xDD4w8HaN4z8PaKpOr2esraWsU8BjMhaGdSFkXaOu1wDwQDxXgn7Ufwm8
+D6j8PfC/xy+EtudO8D+JZvsmo6IRj+yr7n5VUE7FLK6sg+UMFK/K+BOGzZzqxo1lH3tFKEuaN2rp
+PZptbXWo62AUabqU29N01Z/I+ZO47fWlr1j4N/ssfE349WM+oeD9AWbSIZDCdT1C5W1tnkHVEZuX
+I77QQOhINbtp+w78a73xhqXhiPwcq6tYQR3Tma+hS3lidiqvFKW2ycqQQDleMgZFenPMMHTlKE60
+U47ptaepxRwleUVJQdn5HhVH4/pXovwn/Z88d/HDWta0nwZpMOpX2i4+3RTXsduIsuycFyA3zI3T
+0rKPwX+IpJKfDnxZcJ/DNBo07xuOzKwXBB6g963+s0Od0/aK6tdXWl9vvIWGrNXUXZnrnxk/bO+L
+cHxy8Sa3pPi/UdGs9G1S4t7DQ4ZMWKwQSlBHJFjbJv2fMzc5JxjAx9If8FJdF0vxJ8Gvhh8T5dOh
+XWIr60ie3OAZ7WeEztbse6h0GM9Nzepqj4m8LfsZ/E/x3N8Sr/4hpZ/bJxf6j4fN21vDdTcFjJA0
+fmqWOd6oQGOfU58Y/bN/a+0/47+LPDum+GNNWfwJ4ZuhdxwajE0SatMMA74gQUgCAxqOGw7E4OAP
+hqFNYnFYV4TDyp+zT5248vS3Lfq7/nfufTVJOjSqe2qJ8z06n2N4xs/A37dXhjQbrwV8YdY8JXtm
+pf8As3RNQEEqO4UlLq03K5dMbQQcYzgkHNfO/wC2d8Pv2g/AvwYh0/xT4v0/4gfDXTWgW4vobARa
+hE6MBDNdF9zkkkL5iv3+brmptc0r9kj9orwv4dvdP8RW3wM8R6chElrDAltIN2CyOWXbLtYfLKrA
+4zng4En7RX7Snw58Ifswj4JfD/xXqHxIvLq2Fjda/fyNMIYDL5shaZgA7HGxETIVT1G0A8GDo1qG
+Io0qEJNKWsZ017ivrJT/ACszprThOlOVSSV1unv8ih8ZvCMl98Hf2Sfgnp87Wen+KGg1C/ePgPJL
+5ZZz6kG5mb649Ku/8FI/iVe+BtQ8H/B7wfdTeHPCmm6Sl3c2emSNB5mWMdvEzKQSiLGWxnDM4JyQ
+MQQ3WsfF79lb4R/EvwRANY8e/Be/WHUdIQbpZreLYchRlmBjjifgcgy4yVxV340+P/2Xf2rZNC+I
+HiTx3rXhDW7OzFpf6FaWrNdXMSsXEA/dspYFnUSIejduMdVG8MTSnVg5xpyqcyS5mpuTabW+qasz
+GacqUuSSTko2b00S1S+Z2Oj/ABL8Q6P/AMEsr3W9e1K4utTvdPudIsLm6kLzSQzXRt4sseWwhYAn
+nCivC9FtJfDP/BLnxhc6nmC317xXGdESQcMVkhUunqMwTdP7hr3rx98ff2W/jB8BPDvhfWPEuoeG
+fDWlPa3KeFbGCWO+VbdWVLV9qOCpzyyvzwdwPNfPn7R+veNf2jfh1beLfDXglvCP7P3g2E2mkR3M
+0NsrL8sRm2FsyE5WNRGGAywBJLGjAQm5uNSm6SlW525LlWj92Me7b7eYVmrc6lz2jbTX1bPpP9sX
+xbq37PP7IPw/8PeAryXw7HeNZ6S19p58uWOAWzSybHHKtIy8uDk5bnmn/wDBML4v+LPiL4Y8Z6F4
+n1i91+HQbq0ksbzUJDNNGkyyboWkbJYAxhhkkjce2AOM8CftG/C/4w/s86X8Nf2g4NR8L3VlDBHD
+qd5Z3EMN6Ilxb3UE6odkuzgg8NzjIbFe1fsP678Jc+MvCfwdsb6bQdFmtZ7/AMRaiWMuqXMwkAxv
+AbYiRAAkKOeF4JPjYuDw+V18PWoP2ildztprJa83W+1vP1Oqk3PEwqRn7trcvXY8H/4JcoH+Lvxj
+Uj5SIgR/2+T15D8VP+Chfxp0P4n+MNN0zxHDaabZ6xeW9rb/AGGJvKiSd1RckZOFAGT6V1X7Afxr
+8D/Br4n/ABQvvGviS08PWupOkdpLchyJmW7mZgNqnoCvX1FfH3xRkXXfib4u1LT2F3YXmr3lxb3C
+fdljeZ2VhnBwQQefWvsMPl9PE5riJ4mlzR5YWbV1sr2bPOddww8FGVnd9fMg6nnrRmiivsD5QDg+
+5ooooA7n4OfGzxf8BfF6+I/B2pCyu2URXNrOnmW15EDny5o8jcBzgghlJOCMmvoW+/ai/Z/+K032
+/wCJnwHe01+Xm51PwvchBM3djtaJsn/a3H3r5BorzMRl2HxM/ayTjP8Ami3F/emr/M7aWMq0Y8is
+12aufXMHxt/ZO8FyLeeHfgdrXiPUYyGiXxBdBoQw6ZEk0i/+OGvJ/wBoT9qbxj+0XLZ2esLa6L4V
+08g2HhrSwVtYSF2qzngyOFyASAFydqjJrx7P4UVFHLMPRqKs7ymtnKTk16Xdl8iqmNqzjyKyXkrH
+2n8Kv2+/DbfBu0+Gnxh8DXHjHRrS2jsVubQxyi5t48eUJopCuHQKvzq3O0HAPNW9Z/bz+H/w4+Hu
+reGfgT8NJPCNxqYcPqV8scSwuy7DNsRnaWRR93cwAwOo4PxDnmiuR5DgHNy5XZu7jzPlb72vY3WZ
+4hR5bq9rXtr941FCIFBJxxk8n/8AXTqKK+hueST/AGGf+5+oo+wz/wBz9RRRWPOzXkQfYZ/7n6ij
+7DP/AHP1FFFHOw5EH2Gf+5+oo+wz/wBz9RRRRzsORB9hn/ufqKPsM/8Ac/UUUUc7DkQfYZ/7n6ij
+7DP/AHP1FFFHOw5EH2Gf+5+oo+wz/wBz9RRRRzsORH//2Q==
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="xo.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <b487701d0b859d928eb310d4888f47@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAfQCe
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
++LvtEn/PV/8Avo0faJP+er/99Go6K/p4/J7kn2iT/nq//fRo+0Sf89X/AO+jUdFAXJPtEn/PV/8A
+vo0faJP+er/99Go6KAuSfaJP+er/APfRo+0Sf89X/wC+jUdFAXJPtEn/AD1f/vo0faJP+er/APfR
+qOigLkn2iT/nq/8A30aPtEn/AD1f/vo1HRQFyT7RJ/z1f/vo0faJP+er/wDfRqOigLkn2iT/AJ6v
+/wB9Gj7RJ/z1f/vo1HRQFyT7RJ/z1f8A76NH2iT/AJ6v/wB9VHRSHc9L+Ff7R/xF+DWpw3XhnxRf
+W8CEF9OuJWmtJQOoeFjtPpkYPoR1r9cf2Wf2l9H/AGl/ALarbxJp2u2DLBqumB93kSEEq6E8mN8M
+VJ7qw525P4h16/8AszfHnWP2f/GOp6xpTg/bbBrSSKQZRv3iOGI6ZG0gH/aPrXyee5JSzCg50opV
+Vs+/kz2Mvx88NUUZv3H/AFocF/wrzxD/AM+C/wDf5P8AGj/hXniH/nwX/v8AJ/jV7/haut/887L/
+AL8n/Gj/AIWrrf8Azzsv+/J/xr6S9fyPk+bHdo/j/mUf+FeeIf8AnwX/AL/J/jR/wrzxD/z4L/3+
+T/Gr3/C1db/552X/AH5P+NH/AAtXW/8AnnZf9+T/AI0Xr+Qr47tH8f8AMo/8K88Q/wDPgv8A3+T/
+ABo/4V54h/58F/7/ACf41e/4Wrrf/POy/wC/J/xo/wCFq63/AM87L/vyf8aL1/IL47tH8f8AMo/8
+K88Q/wDPgv8A3+T/ABo/4V54h/58F/7/ACf41e/4Wrrf/POy/wC/J/xo/wCFq63/AM87L/vyf8aL
+1/IL47tH8f8AMo/8K88Q/wDPgv8A3+T/ABo/4V54h/58F/7/ACf41e/4Wrrf/POy/wC/J/xo/wCF
+q63/AM87L/vyf8aL1/IL47tH8f8AMo/8K88Q/wDPgv8A3+T/ABo/4V54h/58F/7/ACf41e/4Wrrf
+/POy/wC/J/xo/wCFq63/AM87L/vyf8aL1/IL47tH8f8AMon4eeIf+fBf+/yf40n/AArzxD/z4L/3
+/T/Gr/8AwtXW/wDnnZf9+T/jR/wtXW/+edl/35P+NF6/ZDvju0fx/wAygPh54hz/AMeC/wDf5P8A
+GqWpeFtX0eEy3enyxwjrKo3oPxBNdh4V+IGq634hsbC4S1EE7FWMcZDfdJ4OfavRwjDoeowR/jWM
+69Sm7SSOWpjMTh5qNWKfofOYxjjp1pa7j4leEItHePVLGMRWk77JokHyxyHkEegPp61w1dcJqpHm
+R69KrGtBTj1Fq3pv/Hw3+7/hVSrem/8AHw3+7/hV9DdFSiv0V1z9ijwB4y/bYk8HWGkp4e8B6N4X
+ttZvtM02R0N5M88sapvyWUNtyxUg4jAGC2R0/hX4IfAH9oXxd8V/hjpvwyi8Hah4InisYvEGnS7J
+5WcOPNXH9142G2TcGBBPXA+TnxJhoxU1CTXKpPb3U2kr667ra57iyio205K97LzPzDor7h/Y7/Zk
+8DIPiDq3xh06y1nTtM8TxeDNN+0lhDJeifyndApB+d5IVGeBhvQ1qf8ADKng3w9/wURtPA1/4Ztr
+n4e61os2r2OkylzCm2Hayg5z8ssbsBnjePauqee4aFarSs3yRcr6Wdkm0td7MzjllVxjK61dvQ+C
+qK/VzwH+wf8ADqf41fEa+1rwXpx8F/6HaeHdGl3hQUgRru4X5s4MsgTOSMo3qK4H9lj9mn4Y+PG+
+N7a14A03xBLoHjPUdP0m1mkeMRW8aqYrdW3YVc8ZOcZ5zXH/AKzYRwnOMZNRUW9vtdN+nU1WUVLp
+OS1v36H5w0V93ftY/Bb4W+CP2X9O8VXHw7j+FPxKvpEWy0rTL5tRiD+dh45Zo8wuhi3ODwc7QMng
++k/Cb9n/AOHQ8MeDEtf2eZrjS9TljttT8QfEbULe0v8Aa+3bLHDukZ2bJIjAi4GB1raXEFBUFX9n
+KzbW8fs72fNZrtZu7IWVz9o4c62T2fX5H5kUV9I/t/8AwS8J/AH4zafp/hKA6XouraSmoGxklLR2
+0vmyRsIyxJCkIDtJODnHBAHzYrrIoZWDKeQQcivewmJhjKEMRT+GSvqeZXoSw83CXQdRRRXUc4UU
+UUAdF8Ohu8caSP8Abb/0Bq9y8rrxXh/w2G7x3o4/6aN/6A1e9eVk9K8nGO1Reh4ePhzVE/I5H4kQ
+K3gjUyyj5QjD2IcYrw+veviTGF8C6scY+RP/AENa8FrpwbvTfqdeAXLSa8/0QVb03/j4b/d/wqpV
+vTf+Phv93/Cu3oekj6u8T/t83CftRWnxY8KeH7m2019Fi0TU9D1SZA15CsryEq6bgjAspU88hgRh
+jXZy/wDBQn4feBm8aeIfhp8L7/TPHvi90uNSvdXulNr56qVSQqrsWA3MdqBNxJJIr4TorwJZFgZq
+KcHZJLd6pO6UtdbeZ6izPEK7utfL8j3/AMe/tQx638BfAfw/8PWWo2WqaNrA8Ra1rWoPGTqWo73m
+aRdhztM8rP8ANggKg7V7j4q/4KI+CvEvxe+H/j5fA2v2t94Zgv7WaIT2zG4huYlXYDu/hdEYE44L
+dzXwfRWlTJsFUS5o7c3V/b+L7/w6CWZYhX17fgfdukf8FKNOg+PPiDxzqXhfWbnQbjRrfSNJ0uGa
+ES2u2RpZ3ky20l3I5U/dRQa534H/ALdHhP4WxfFO31fwdrWrWnjXxJe6ysVtcQqYbe4UL5LksDvA
+Byy8cjGK+NKKy/sHAcrgoNJqK3f2div7TxF+bTr07n1x8QP20vBifs7/APCovhj8OLzw5o2+No5N
+cvVuUgC3K3BIXc7OzSL/ABMAMn0Fdz49/wCChvw5+Iv/AAg2va78KdU1Txf4Xvkv7OOXUgllaz5Q
+PKhV/wB6wCkoHThgDxya+DqT9af9hYF2vF3TbvzSu3Kyd3e+qQf2niLt6apLbtsffvhX9p3w1+0j
++2h4I1iLwBe3to2g3GhTWGq/ZpY7cGYTteNklRHGiybujZIAznB+QPj54o0Hxr8bPG+ueFoY4PDd
+5qbnT1hiEUZhVVQOqDACsULDjo2Tya4aG5ntS5gnmt2dDGzQyNGWU/eUkfwnuD1qLGOBwPQDFdGE
+yylgqvPSbsoqKV/O/wDX9Wzr42WIp8slre4tFFFeueaFFFFAHT/DEbvH+ij/AKaN/wCgNX0F5VfP
+/wALRn4haIP+mrf+gNX0Rsrxce7VF6Hm4qN5r0OR+Jke3wFrB/2E/wDRi18+V9E/FBcfD/WT/sJ/
+6MWvnUV04H+G/X/I3wytB+otW9N/4+G/3f8ACqlW9N/4+G/3f8K9HodiP0Kl/wCCf/wX1m58c6VZ
+a5488J3nhjy1m1zxF5UemMXQsHjkeJVmjXHzYZceoyDXgP7NHwQ+Evj7whqes+Mdc8TeKPEVsZDH
+4N8DafPLciNX2qzOIyGaQAuF3KAhBJznH1K37YHwD03VfH+raz8QvEXxE0nxII9ng3UtJmnsrREX
+Bjt4pY1QBzydxA4/GuC/Z+/am+Emkfsp3vw7vfFOqfCPX1N0DfaRayS3R8y4aRJYJkRtz+WVjJba
+w28YAU1+a0q+axw81NVG24a2s0rPm+y5NX3su1n3+wlDCOrHl5dn19LdbHJ/tQfsc/D/AOBfg7wd
+8RrCTxSfB97qFrba14ev5FXUoYZkZt0TlcpKmMGN8gnjIrs9N/4Jz+CfEvxF8H3/AIb1rXPEHwh8
+Q6PNdHUrW6i+0W10oVoiZPLA8uRWYAbMq0ZBxkVR/aI/aQ+Dvxx/Z88FfD7TPGesaZ9n1PTft93r
+llPNf21nCGSWeR1UrLNt+bCsdzNXT+H/ANuX4Y+Cfi54J8MeFdWn0H4KeGvD89u8sVhNi7vXKCJW
+j8syEIqs2/A3PI5OcAl+1zh4aPJz+0XPe6fw/Z6fFf4bb9dA5cHzvm5badevX5HD+Jv2Gfh98J0+
+KHjD4j6n4k0r4c6FcRWnh6G0mj/tHUWMaFmJMZVg8z+VGMLyGLEDBrnNA/Y+8IeLv2MLj4t6ZH4h
+/wCExuo7htO0aK7SaDzftrW8EJHlhnONgJyMkk4A4r1i3/be+GPxA8SfGLwT8RtaudR+F2veW2g3
+7WEzOkbwRpPCEVC6FJV8yNmXg7jnhazvhH+2Z8PPgL+yW/g7SvEg8QeOtFh1GPSk/sy4SG8la5me
+1mJdNqgq0blWII5HWn7bOI0o3jN1Oan0fK48uq20d/jv1FyYJy0ceWz++/r9xyf7S37Cvg/4EfCP
+wjrtvqOs3PiS+1rS9J1ES3SSWoM/E7RL5asPmB2kk4HXNdj49/4JueE9D+Lvwz0PQYfGGoeEtafU
+B4h1Lz0l+wLFAHtj5giAj3yEr8wOcYHNc7+0L+1f8PviV+zp8NvDVn4nuNW8WaXquiX2r+bYzo2Y
+ADcyF2QKx3FjhSc54zXpnxF/b+8A3/xp+FN/4d8d38XgqxfUz4nhi0+4SOUPbgWu9GjDPiUHG3p1
+PFZKpnapwtzc372+jtt7vTv8PnYfLgdb2+z1/r5nmWg/sKfD3VP2u/E/wsl1HxCvh3S/C9vrUE6X
+kYujPJMqMGfysFMHoFBz3r52/aE/Zm8ZfB7xV4ymtPBviIeA9KvJY7LXLu3MsbW4xtkkkUYwf7xC
+ivq3w7+2D8KdP/bX8W/ESfxHKnhLUPCNtpVvff2dcEtcpOrsnl7N4wo6kAe9amu/tofCXwh4V+Kk
+th8QPFHxOv8Axb9pex8PapZzC10/zI3VbeIyogjg+cBsk8KMAnO7roYvN6FaDlTlNOMLpp7u3Nra
+yfe5FSjgqkWk0tX1R4v+0P8Asi+C/hP+yv4R+JWj3utTeINXGmG4hvbpHth9ph3ybUEYIwenzHHv
+XyF0r7f+Of7S3wt8dfsmfDL4f22vSX+qaLLoQ1e1+w3EarBbRql1hyoBwA33SSe2a8+/bp8CfCrw
+FrPgCH4X6UmjJqukPq17a75zJ5MpQ2ryJMxZCQJcKcHA5A4r2MqxmIhbD4yMnOcp2bWiSta/6aHn
+YzD05R9rRaskrnzDRRRX1R4R1fwpGfiLof8A11f/ANFtX0fsz2r5y+Eoz8SNC/66v/6LavpTZ7V4
+OYfxV6fqcddXkjjfiqmPh3rZx/yzT/0YtfOFfSvxYXHw51zj/lmn/oxa+aq68v8A4T9f8jWirRCr
+em/8fDf7v+FVKt6b/wAfDf7v+Fen0OhFSivob9lD9j27/aq0rxHf2Pi2Dw6mi3MNuyS2RufP8xC4
+YESLtxjHem/AT9jzUvjD4O1zxzr/AIosfAXw90eSaKbW76HzWn8okStGm5QEUjBYnlsgA4NeZUzP
+B0XUjOpZwsno95bLbVvyud0MDXmouMd9vkfPdFfQPx+/ZDv/AIP+BdH8f+H/ABTZfED4eaoUWLWr
+GLyXhMn+raRNzDYxyoZTw3BAyK+fe9dOGxVHF0/aUJXW3zXRp6p+pz1qFShLkqKzFooorqMAoooo
+AKKKKAFRijo4ALIwYZGRkHI4PWtfxh4x1v4geJr/AMReJNTn1nXL9la5vrnG+QqoRRwAAAqqAAAA
+BWPRU8seZStr/X+RSk1HlvoFFFFUSdd8Ihn4laCP+mr/APotq+milfM/wfGfiboH/XV//RbV9QbD
+6V89mT/ex9P1Maiuzivi2uPhvrv/AFzT/wBGLXzJX1B8X1I+Gmu8f8s0/wDRiV8v12Zb/Cfr+iLg
+rRCrem/8fDf7v+FVKt6b/wAfDf7v+Fer0NEfq5+zp8XfhNZfGj45ahoPiXw7o3hy/wBQ01reR7uG
+0gvLhbUi5lhVyu5S+AWAwzBjznJ88/Yx/aE8M6v+zdd/Co+KNF8G+ONKN9Dp8/iKNHtLpZLiSaOY
+K5VZBmTayZzxuxgivzYkhSXG9FbHTK5xSvEsy7HRXX+6QCK+Qlw1RnGcXUfvclnvZwi4r1vfU91Z
+rJOLUdr/AIu59/8A7YnxVOh/Ac+BtT+LWh+MvEmqiJLnQfC+k2yWcIWZZWcyKxaFV2ADJ3M2PlAL
+Y+AgCTxkn3rodC8B3N4iu6rZWx55XBPuB/U122meFdN0wApAJpB/y0m+Y/l0FfP1+Jso4apywtOf
+tql22oJJX82rRXbq11R+pZL4acQ8VOOKnBYei9pVL3a7xj8T8r8sX0Z5tZaJfahzb20kin+ILx+d
+bEHw+1SUDf5UOeoeTn9M16T0HtSD5iQMGvhcX4j5lVdsNShBed5P77pfgft+XeB+RYeKeOr1KsvK
+0I/daUv/ACY4NfhtPj57yIH2BNB+G0wBIvYyf90iu6SWOUsEcOV+8FIOPrTg6szKCCy9RnJFeL/r
+3n17qsv/AACH/wAifUf8Qi4QSs8LL/wZU/8Akv0POpvh5qUZyjwSjttbn9QKyL3w5qNgCZrSQKOr
+AZUfiOK9dozg162F8Rc0pNe3pwmvRxf3ptf+SnzmYeCPD2Ii3g6tSlL1U4/c0m//AAI8QII60V69
+qHh7T9TDGa2TzD/y0QbW/MdfxrjtY8AXFpuksm+0x/3Dw/5d6/Scp46yvMmqdZujN/zfD8pbffyn
+4XxH4RZ/kkZV8KliaS6wT50vOG//AIC5edjkqKdJG0TsrAhgeQeDTa/RNz8Qaadmdl8Gxn4n+Hx/
+01f/ANFtX1N5ftXy38GBn4peHv8Ars//AKLavqsp7V85mbtWj6fqRJHDfGJMfDHXj/0zT/0YlfLP
+rX1Z8ZVx8L9fOP8Almn/AKMSvlM9a7csd6MvX9EUtgq3pv8Ax8N/u/4VUq3pv/Hw3+7/AIV63QaI
+rC2l1S5WC2XzpG/hTmvR/D3g+30lVluFWe79+VT6Z612/ibV/wC19Wn2WtvZ29vI8EUNvEqYwcFi
+QBkmtn4QeDLT4ifFTwv4Yv557ex1W8EE8tqQsoTYzHaSCATtxnBxmv5w4l4zxGayeCwN4Utm+svn
+paPl169l/bnAvhvgeG8Gs7zuKqV1Hnt8UaaS5tFtKaW71SekduZ8j/nmu4/4U7rsnwaHxJt7iyud
+HbURpv2SN2a4icts3ycbVG7aMZJw6mvWf2yP2bfD/wAC49G1zwxdXMem6pc/ZW0m5JlEDhc70lJ3
+YOPutnknBA4rX/Y112Ob4f8Axe8P6np1rrOj2VlHr8dlfKXhM8QcgMvcFoIm6jlK/OY4flrvD1NH
+Z/ldM/VMVxGq+TU87y33qfPHmTVm48yjKOuikm97200ep5T+0l8O/Dfwg12z8G6MdQuvEOnWazaz
+q11KDHcySxhkWKMfcC889TuwckZrqf2rPBmheHvCvwj1LQdJs9Jj1rw6Zrk2UQjE02yBi7EdW/eH
+nr1rtP2i/AOifEr4CWPx9traTQvEeoRQS6nYJM1xBd7nEI5blCuAQRxgYIPWvUtO/Zytf2kP2fPg
+4+oa9c6K2j6OuGtYEkMoeONcHceMeWPXrXV9Xc3UhTW6Tj6X8/Lc+UXEdLB4fL8dja0l7KpUp1r3
+b9p7N30jdNKVuW10k1a1rLwb9p/TtPT4M/AfXbKytrI3mivHcPbQrGZHENuxZsD5jkMeeeT61peO
+fgzc+IPh78BfBPhfQbNvGOr6fJqeoXwhVHSBgjCS4kA3eWpmPXPKhV5IFeqfta/AvTfDX7Lnh6wT
+U7u5fwhNDa2k8qoDMs8qQtvAHZXyMY5UVb/bb+JV/wDBLwT4f0HwrDFp+pa3Yyac+uRjFzBaW6Kf
+KjI5Ut5h+bPy84GSCNatJQdSVXRJR2+W3q1Y8nLc6q42nl2Fyt803VxFua6WnM4uXVqMKqlbdtW3
+PFf2ofgN4H+Cnw68Fz6BeXGq65qVy8FzqrXW+G5WONjMyxglVO8qBg8AEHJ5rg9B/Zc+KPinwVb+
+KtK8LPdaVcx+dbx/aI0uZojyJEiYglT1HcjBAIIrsf2zlj0LQ/hF4dgTGnad4SSVEU4yzhFY/U7O
+vua9Zj8Aan8fv2cfBvjZvFN/4V8TeFNPlitp9K3LDIsQADFN4KyEIvzKwHXjGAMJUYVK04Rj8KWi
+su19X/TPfo5zjctybB4mtXT9rUmpVKilK15T5FyxaavZJ2do9Ez4p1HT7vRtSn0/UbO407UIDiW0
+u4mimjP+0jAEVD0r6s/bl1W8sdE+F3hi8n/tS4j0v7ddaxcopurqTasY3MBkDLOxAOCSM9M18ya9
+ojaDNpiPcC4+36Zb6kCI9nliUE+X1OcY+9xn0FcNal7Ko6d72Pu8lzX+1cFSxc4qDnzWSbd+VtXv
+ZWva9nqlpucrrvhm11qMsyiK4x8sqj+frXmus6VPodwYrpQn9188N7ivYmOB+lT2OqPot4t4kMM5
+jHzRTxq6SL3Ugg/mK+84b4xxGSNUMS3Oh26x84+XeO3az3/M+PPDPB8U05YzARVLGdHtGo+0/N9J
+7/zXW3nnwSkjf4reHVDqT5r8A/8ATN6+tNg4plrZ2WyC4isLSB3jEqmO3RWXI6AgA96sgdPev2vE
+4pYuaqRVtD+B5QcHyyVmjhPjSuPhX4h/65R/+jFr5Nr63+NYx8KfEX/XKP8A9GJXyRXuZX/Bl6/o
+iHogq3pv/Hw3+7/hVSrem/8AHw3+7/hXsdCUf//Z
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="mvzfeaehfsmikr.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <7713b01d0b85939271dc4035be5a9d@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAjwCe
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+8YAAGAMAdq6P4a69/wAIr8S/CGs7tosNXtJmJ/uiVQ3/AI6TXO02VC8ZCttbqreh7H86/O0+Vpro
+f6HVqUa9OVKe0k0/mrM/RD4n+MdH+DPiLxxD8SvEVt4n8G+IH8/SvAot/tt8Cyp5smWI8qPeJCAT
+t7qQQQfn/R/iT8O/2dfizb+J/hvqd74u8OavpksF94emQxyWe7a0cRmkGGw64KkFlGclsivMfjh8
+X7j43+NIPENzpcOkvDYxWCxxyGR5Qm4+ZI2ACxLNjAGBgc4zXn9ehWxTlO8Nk9Hrf/hvLY/Ocl4S
+VHBKOPnLmqQ5asE4qD91RV1FfErX501JvVu2h3Phn41+LfAuheIfD/hjUzpXhvWpZWk0uaNLlIEc
+kFIzIpC5UhWIHzYB6jNZNr8S/GVlpNtpVr4w1600u2QRQWdvqUsUMSf3VVWGBXOYpa4eeVkr7H3i
+wGEUpT9lG8ndvlV20rXb3btpd6nb23xk8RR/DjxH4Lu5Tq2n67cQXM17qM8s1zA0TKyiIs2BkoMk
+g965HWNV1DxDIsmraje6rIkXkJJfXLzukY52KWJIHJ4qqBilpOTkkm9jSjhKGHlKVKCi5O7t3slf
+7klp2PUNN+P2oX/xA0nxV460HTfiDLpdmljaWl6ot44URiyuqoCjOCx5dSPbOCPUPhh+0H46+Nf7
+QPhmPW/Gq+DfD0UzXDadYXAtLVooxuFud3MrOdq/Mem4qBgCvl7tikeNZF2sodf7pGf0reGIqRa1
+0vf19TxMXw7l+JpyUaUYz5HCLtdQTvrGL91PW+iV+p9C+INQtf2qP2pNVvtRvktPA+kq8lzdTSbE
+h0i1PzkEngyuScDkCXOMCuP/AGi/jWvxr8bQXOn2aab4X0iH7Do1mIhGwhGMyMAOC2FwnRFCgc7i
+eTt/iPqlv8Mb3wGttp7aLd36ai1w1sDdJIAMgSejbV7ZAXbnBIrl8UTrOSav8Wr9e3oiMBk0MNXp
+1ZK0aEfZ0le6UbK83t789n2S3d3daQ/p6UtJ1rmPqTuvhz4tNvLHot7JmCQ4tZWP+rb/AJ5n2Pb0
+PFellMHGPwr56Ybgckj3HBr2rwL4jPiXRA0zA39sRFcf7XHyv+I/UV+s8KZzKtH+z671ivdfkvs/
+Lp5X7H8deMHBMMFU/wBY8BC0Ju1VLpJ7T8lN6S/v2ermzC+N67fhN4j/AOuUf/o1K+Q6+wvjkv8A
+xaTxJ/1yj/8ARqV8eV+35T/Bl6/oj+Xpq1hat6b/AMfDf7v+FVKt6b/x8N/u/wCFe10M0eyV0XgT
+4eeJfidrv9jeFtHn1i/Ch5FiwscKE43SO2FQZ9Tk9ga51jhSeeOeK/SX9jxvAvgH4KaXFbeItFl1
+u9h/tLWDFeRmZZWXd5brncPLTCY7bSe5r+KcLQWIqcsnZdT/AE24sz+pw9l/1ihS9pUk7RVnZebt
+0XbS7a8z52i/4J5/E6TT/tD6r4ahuCM/ZGuJy2fQuI8V4h8SfhT4s+EWsx6b4r0eTTJZgWt7gMJL
+e4A6+XIvDEZGV4IzyK+iLz/go74l/wCEva8tvDWmt4OWbAtJC/22S3z9/wAzdtDkDIXaQM4ycZr6
+z+PfgbSfix8FNfsryIMjWD39lcFcNBOkZeKRe4OeD6qWHevQ+q4bEQk8M3ePfqfnsuKuJMgxeHjx
+DSg6VZ293eO19n9m6une/SR+S+c9vwr0z4Sfs4+PPjXbveeHNNii0hHMZ1bUZfJtiwOCEOC0mOhK
+ggEYzXI/DPwjN8S/HHhjw1ExhbWbyK3kkTrHGfmkce4QOR71+pnxU8baT+zl8FrzVbDTYxZaNbxW
+un6ZEdiO5KxxR57DJBJ64BNcmDw0a6lVqO0Yn1nGHE+JyWrQy7Laaniazsk9kr8q0utW9FrZWdz4
+h8VfsEfFLw5p0l3Zvo3iMxqWa20+4eOY4/uiRFDH2yM187TwTWlzPb3EEttcwSNFNBOhSSJ1OGR1
+PKsDwQea/QT9lL9r/WvjJ40vfCvizT9OtL+S3e8sLjTVdEcIR5kTKzMcgNkMDyFavO/+Ch/wutdG
+1zQPHlhbrC2qOdN1IouA8qruhkPq21XUnuFX0rathqUqP1jDt2W6f9f0jysk4nzahnKyHiCEVUmr
+wlHZ6N27NOzSas01Zp30+cvhH8HPFHxt8RyaP4YtY3MCCS7vrpiltaIThS7AE5ODhVBJwewJr3bX
+f+CdXjiw0p7jTPEujaveou77E0Mlv5h/urISwz6ZAHuK9p/4J6aLbWPwNvNQSNTdX+sXDTSfxER7
+Y0Un0AU4/wB4+teS/sQ/E/xJ4m/aH8VQarq15f22sWl3dzQ3E7OiSx3CbGVScLhXKgDAxgY4FXSw
+1FRp+0Tbn57HBmfEudVsVmMstqQhSwVrpx5nN6316fDK1rdFu7nybqml3mhand6dqVpLYajZzNBc
+2s67XikU4ZSPX/6x7ius+D3wp1X40+PbPwvpMqWsksUlxcXsqF47WFBy7AEE5YqoGerD3r1f9vzR
+7bSP2gjdQKqNqOj21zORxl1aSPJ/4Cij/gNe3fsjeEtP+AvwC1v4o+Jk+zXGp2x1A7xh47JAfIjU
+H+KQncPXzEHauanhb4h0pP3Y7vyR9NmXFMqPDtLM6Ef39dRUI7+/Lsuqjq9d7JPc+SPjl8IB8EPG
+MPhqXxFbeINR+zC5ufstu0QttxOxGyzZZgC3bAx6ivPa1fF3ivUfHnivV/Eertu1LVLl7qYA5Cbv
+uoP9lFAQey1lVwzcXJuCsuh9zgYYmnhacMZPnqpLmdkk5dbJWVr6Ly8xDXReANd/sHxPbPI221uv
+9Gn9MMflY/RsfrXOnmkddykA4PYjtW2GxE8JWhiKe8Wmv689mZZpltDOMDWy7Er3KsXF+V1uvNPV
+dmkz1r47RlPhJ4mB4IijGP8AtqlfGwr65+J+qf25+z5qd+T88lpCJPZxKit+oJ/Gvkev65yOrGth
+faw2k7r0aTR/l1mGEq4HE1MJXVp05SjL1i2n+KCrem/8fDf7v+FVKt6b/wAfDf7v+FfQdDzkey1E
+1tEzbmiQt6lRmpabC/2osIFefAJbyUL7R1ydo44r+HD/AFru1qfX/wCy/wDsaaF8SfB3h3x54n1O
+7ubO7kkmGhRxqkUqxzMiiR+WKtsyVGM5xmvWP22vjlqPw88ET+FtI0TUUm1yE2suutbMtlbQuCHR
+ZMYMrLkBeMZz2APw5oHxt+IXha10620Xxrq+m2enIEtLOGcfZ4lByFMeNrLnswOa/TX4Y63Z/tB/
+AbRdQ8Rabb3Ft4g07bf2bLmJmyUk2jqBuUkc5HHORmvosJKFalKjR92VtX3P5z4tw+OynNaGc51J
+YjDKo1GCfLyXd4q1km7L/t5xtJ2sfBn7Dmnx3P7Sfh3coK2tlezICOhEWwfkHP519Rf8FEJzD8Br
+NFOBNr1ojD1AWRv5qK+eP2VNHHw+/bMfw1JMZfsMuq6UkrnmTy1JQn3KoDX0P/wURt2l+A1k6jIi
+160c+wKyr/NhU4dWwFVPu/yR1Z/UVbjnLKt7xlGm16OU7Hyb+xxdNZ/tMeC2U8yG6hPuDbSf4V9h
+ft/WKXX7O91MwybXVLKZT6HzNn8nNfHv7G9o17+0x4LVefKN1Of91baTn9RX2D+39fpafs8XULHm
+61SyhQHuRJv/AJIaWF/3Crfz/JGnFX/JcZXyb2p/d7Sf6HkX7Bfx40LwlpupeAfEV/DpRuLxr7TL
+q6cJFIzqokhLnhWyoZc9dx7jn2H4YfADwR+zf4x8UfEG48WxLY30cq263zRxRWMDyCR1D7v3hyFA
+OAcDGCa/NNkV1KsAynqDyDTTBGFG4ZROgc5C/QHgfhXHSxvs4xjKCbjs77H1+acE/XsViK+FxcqM
+MRb2sVFSUrdm2uW/z3fR2PpPVjH+2f8AtaounJN/wiwWNHlcFWGm25y8h7r5ruVXI48xc969A/4K
+C/FWKFdG+GOjukNvAkeoanHDgKqrxbQYHQcGQj/Zjrrf2R/COn/Ab4A658UPEyG3utUtjqL7xh47
+JAfIjUH+KQktx18xB1FeTfsrfDeH9pv4weLvG/jmBdSsrSZbqawckxTXMxPlRt/ejijjA29D8mcj
+IPVyzdNU18dV3fkv6/U+UWIwUMwnj5p/UcrioQW/NV0jfs2nZX2vyy0TZ8rC8gxkSpjpndxmp+39
+a/Syw+PfgLWPjjc/Bk+EoPs0e+0S6eCE2kk6ReY8PlbeAFDDPcjGMYNfJn7ZHwR0z4M/Eiyk0CE2
+nh/XYHuYLTJK20yMBLGmeiHcjAdssBwAK4q2E9nTdSEuZJ2fSzPtsn4u/tHGwy/F4WVCpUhzwvJS
+Uo6volZ2TdrdHez38GpKWkrgP0M6bVLw3H7OvjK2Jz9lmTA9A8kbfzBr5qr6CupM/B34iRdvItZM
+f9tgK+fe1f1JwNUdXJKbfRtfc7L8Ef50+KOFjhOLsdCO0pRl85wjJ/i2FW9N/wCPhv8Ad/wqpVvT
+f+Phv93/AAr77oflaPYpo/NidMkbgRkdq/VD9mTx54f+LvwX05oYLRLyK0Gm6zYxIqFJgm19wGPl
+kHzA9w3qDX5ZVveCPiB4k+GuurrPhbWLjR9QA2O8JBSZc52SIcq655ww+lfxVhMT9WnzNXT3P9MO
+L+GnxLgo0ac+SrB3i3e3mnbWz01V2mvU+mNY/wCCcfiePxS9vo3ijS18MNL+6ubtJDeQRE8KYwNr
+so4B3KDjPHSvrya78L/s2fB2Bbm5Np4f8O2KwxmVh5s7AYCgcbpHbsOpbsK+I7f/AIKEfFGGx8iS
+w8N3E+MC7e0mVs+pUS4rxn4l/GDxh8YNRiu/FmtSakICWt7RFEVtbk8ZSNeAcfxHLe9d8cVhsMnL
+DxfM+/Q+BrcK8T8RVKNDP68VQpu/u/FLz0SV2tLu1rtpd3eFvile6H8Z7H4i3MTPeDWm1e6giOSy
+SOTLGp7/ACOwH0Ffpx8UPBmj/tG/Bi80qy1OJ7HWbaO507VIfnRHBDxSAdxkAEcHBI4Nfkp69ea9
+I+FH7RHjz4LI9t4a1ZDpbuZG0rUI/Pttx6soyGQnvtIB7iuXCYqNLmhVV4y3Pq+LOFK+bSw2Oyua
+p4ihblvs0ndLrZxa00s7tM+wP2Vf2QNX+DXja88VeKdT069v0t3tLG30wu0aByN8rM6qdxCgBQOA
+x5rzj/god8U7XW9d0HwHp9ws50pzqOp7GyEmZNsMZ/2gjOxHo6+tcL4p/bz+KniTTpLO1fR/DvmK
+Va50y1dp8H+60jMFPuBmvnuaaW5uJri4lluLmaRpZZ5nLySuxyzMx5JJ6k1rXxNJUfq+HTs92zzc
+k4XzatnKz3iCpF1IK0Ix2WjV+ySu2kru7u9tW4r0H4A/CqX40fFbRvDRR/7NLfa9TkXPyWkZBcZ7
+FyVjHu+a89J4r2n9nn9pKP8AZ7tdYNt4Ri13VNTkQy38195BSJBhYlURtgBizE55J9q8+ioOovau
+0ep+iZ1LHxy+r/ZkOau1aOqVm9L3bS93dd2rHtH/AAUG+KscSaL8MdIdYreJY9Q1SOHAVVHFtBgc
+AZBkI7bYz3q3/wAE2dXtxa+P9KLqt559rdqh6tGUdMgegK4/EetfHfjPxXqHjzxdrPiTVpfN1HVL
+p7mYjO1M8Ki/7KKFQeyirvw5+I2v/CjxbbeJPDV2LTUoVMbB13xTxNjdFIvG5DgH1BAIwRXWsX/t
+f1iS0/TY+LqcINcKvIqMkqjSbb2c7qTu+za5b9rdj6M8MfD/AF5P+ChF4z6bdJbw6xc6w9yYm8oW
+zwMUffjGCXVOD1yOtbP/AAUl1iCbXfAGkKwN3bwXl5IoPKo5iRc/Uo//AHzVR/8AgpH4iOl+XH4G
+0waltx57ahIYc+vl7N2Pbd+NfLvjjxzrfxJ8VX3iLxFfG/1W7IDuBtSNB92ONf4VXsPck5JJrSrW
+owozp0nfmd9rWPPyjJM4xOc4XMs1pRpLDUvZxSkpObs482mytJuz8lrqzDpD0paQ8CvJP180riM/
+8Ki+I79ha2i/j54NfP1fSlzZGP8AZ4+IV2wx5ssEan12Omf1avmqv6h4Eg4ZJTT7yf3s/wA8PFat
+GvxfjHHpyL/wGnBP8U0LVvTf+Phv93/CqlW9N/4+G/3f8K/Qeh+SI+mPhF8Jdc+NPjSHw7oXlwuI
+zPdXs4JitIQQC7AcsSSAqjqT1ABNe5r+zx8B4fER8IT/ABfvX8W+b9lLIsSwCfOPLz5ZQNnjaZM5
+4zmtz/gn0FOh/FMWW3+2zDbCDHD7fKm2bf8Agf618cWySGOKNlc3BYKVIO8ybuR/vbv1r+LbQo0o
+TcVJyvvfp00P9JZ1MdnWb4zBUsVKhDDqmlyKN5SnHm5pcyd0tuVWv3PX/Gf7O+qfDn43eG/Aeu3n
+m2OuXkEVrq9nHt863kkCMwRshZF7oSeoPIIrX+In7M0uiftCaT8L/C+oXGpS6haw3P23UEUeQreY
+ZHYIACqrHnsSTjvWH4Z8MeM/C/7QHw/i8cWesWmpza5aOj6zI8jTKJVyyOzMGHTO0mvuzU5vC7/H
+zxLZ6TKunfFu48NoLK+1KPzbVbbe2PLUMMsGALg4JAGMgMK6KGHp1lLS3vLfe1tvXsfO51xFmOT1
+KHLVVa9CbbilyOfMoqo7X92O8rO29kunxt4h+CHw7tPi14X+H2geONT13VbrVjpmtTC2REsyF/5Z
+tt2lgwIIy2MHuK4r4yfCdvh58Z77wFoD3mvTI9rDZiZVE9xLNGrBMKAPvNjPpWr8IPDmreEf2qvC
+mia9Gya3Y+IljvN7bt8h3sXDdw+7eD3DCvrzxF4N0f4ZfFz4i/HbxiimzsYILbRLfcDJI/2dEeRR
+2d3JiT2Lk8YIinQjXhKSjy2lr5K2u/8AVzqxue18hxtKjKu8QpULxXu/vKsqiULcqWjTdrbQ72uf
+O/x5/ZNs/gh8JNP8TSa/d6lrb3dvZ3VsUQWqvIrF9mBuwpXAyeepHNR/CX9nHwV4u+CTfETxh4x1
+HwxYxXcttO0MUbwxhZRGh5VmJJI/OvQ/j/4z1P4i/sSeFPE+sSJJqWq61FcyiJdqIPNuAqKPRVCg
+fTPesXUYm8Mf8E5rGOf5JNb1FHRW4JWS8Lj/AMdjzWsqVFVW4x91Q5knc83D5pnFXK6NHEYhqvPF
++xlKKjflV1LlvG1lZtXXqRWP7Knwv8V+BPFHiTwd8R9U8QpoVpNPIEgjVBKkRkVGzGDg47V518E/
+gHpnxU+Dnjjxneavfafd+HkleG1tUjMU2y2Eo3FhnrxwRxXq/wCx5x+zp8av+uNwf/JBql/YqOnL
++zV8VW1hLl9HBlN6tkcTtB9iXeIz2fbnHPWnTpUqrpvlS5lJ9baBis1zPLqGY01iJTdGtRjFtR5u
+WdnJaRSd722PJvhP8BdB1/4W3HxJ+Ifim48IeE/tH2WzWzg8ye7fO0lchuN2VUBSTtYnAFeY6L4X
+fxf49s/Dnh5nu/7R1L7FYTTptZomkISRx2wnzn0wa+n/ANqrQG8QfAT4fa/4Dmhk+E+kwR7LKJGE
+8DMDEs0rEndtO6NhwVd2JLZyKH/BP74bnV/Gms+Obm3MttosJsrFcffupVy5B9Vj+X/trXO8OnWh
+h4rtd9+rfp2PcpcRVKOT43Pq9VtqUoxptJKm0+WEGrJ87bTndtWemiuYv7SP7I1r8FPBFt4l0PWr
+/XLWO7W21BbxIx5KuMJICgGBvwpz03rXzYPSv0W+E/wv8f694e+Kfhz4paTBa6V4supr+3khvEuP
+s8kw2tGAp42bImU9MqfWvz21zQ77wvrupaLqcZj1HTbmSzuFP/PRG2k/Q9R7GpxlFQcakYuKfR9G
+v8zq4Nzmrjo18Bi8RGtVotPni01KMknpy6e7K8X8inSMQqkngDk0tavhLw+/izxNp2lJkLcSjzWx
+nZEvLsfbAP5iuGnTlVmqcFdt2Xq9D9AxOJo4OhUxWIlaEE5SfZRV2/uR3XjfRzov7KmqxOuyae2S
+6kXuDJOhAP4Yr5A7193/ALRkap8EPFaouxFghVUx91RNGAPyr4P71/XHDlGOHwKox2jZfdFI/wAv
+M9x9TNMfWx9XSVWUpvy5pN2+V7C1b03/AI+G/wB3/CqlW9N/4+G/3f8ACvp+h4KPpb4SfFjXfgx4
+yg8RaA0TybDBc2VxnybuAkExtjkHIBDDoR3GQfeE/ad+DA8S/wDCZN8GZ18YiT7R5iyxeT5/XzM7
+tu7dzv8AL3Z5618qMdoJY7cdc9q9B+GHwC8e/GApL4Z0CaXTicHVb0/Z7MeuHb7+PRA1fxPRq1Y/
+u6av5Wv80f6d51lGUVm8fmEvZ6cspKcqfNH+WTUo8y8nr0Ro+Jvj5q/jz42aD8QPENuHi0i9t5oN
+Ks3+WG3ik3+UjN1ZuSWPU+gAA1fil+0VJ4w+PWjfEzw7YXOjXGlxW6R293IrlzGz7wSnG10kKke5
+r3bwV/wTfiCpL4v8ZTSycbrTQ4BGo/7ayBmP/fIr1Gw/Yj+C3hmASajpEt8g4M2rapLj9GUfpXpQ
+weNqpp6Xd9d79Nrn55i+LuDcFUg6UZVPZwdNKEXy8j3j7zinfq9b66u58qfEP9o7wp4w+Nfgv4j2
+HhXVNOv9GlU6hBJPCwvY0B8raQeHXcwyeq7Rxisf9pP9pO5+Pl5pVra2E+jeG9OBlSxnlV3muTkG
+V9vGAp2qOcZY96+xz8Af2cISUbSfCqkcENqfP45lzVnTv2c/2e9ZvEtNP8P+Gr66cErBbX3mOwAy
+SFEhNdcstx0oyUmrS1e/+R85heNeE8JWw9anharlQi4wbafKm33m+7SbvZaLZHyR4a+MWkePPhr8
+Kvg5d6BcTfZfENgLu5mkU291F9obem0Hf8wkx26HmvZ/2hfjd8NPBuvp8M/Enw2uPEWmeHo7d7OG
+KZI7eIND8mxcg/Kjbec9680/bk+Gfg74Uax4LsvB+hW+gS3cV1cXL2jOGfaYlTkscYJbGPWvmOWW
+S4cvNLJPIeC8rl2I+p5rzKlaph3KjKzeivZbJbbH32X5FlvEUKGb0eeFGXtJqHNKMvaSnrPmjPTZ
+qy6PfofdX7Ofxq+GvjLxFdfDrwt8OJ/DNjr1vPJeq86PDMixYcMAxPKkrwR1rxbwr8ctH+DXh/4w
+eAF8PXc66zqmo21pLayosNpGVaBEIY5IXA9civn6OaWCQSQzSwSDOJIZCje/IOaaSWYszMzE5LMc
+k+uT3+prF4yo4xWzV9dNn0tY9mjwbgaVetKUpSpVORuLlJvnhJtScnJt9Fby8z3D4CftHWXwu+Hn
+iHwN4o0G68T+GtUB8uC2lRWhEibJ1+fs3ysMHhs+vCt+0ZY+H/2eI/hp4O0zVdGvJZvMvNcmuEWS
+UGXfIQUOQxwicfwg14bjmisliKiiop7K3yfQ9erw5ltbESxM6espxqNXfK5xVlLlva+uumr1d2d3
+8O/jN4n8AeOdE8Q/21q2pw2Fysk9jcahK6XMJ+WSPDMRkqTgkcHB7Vb+P3xI0P4ufEabxVomjXeh
+teW8aXsF26P5kyDaJF2dMoEBz3UV5yeaMYrL2kuT2d9L3+Z6KyvCRxscwjC1VRcLrS8W72a2euqv
+sB6V7x8BfBzabpE3iG6j23OoL5dqGHK24PLf8DI/JfevOPhf8P5PH2vbJldNFtCHvZhxu7rEv+03
+6DJr6hWFUVFREjjVQqRoMBQBgAewAx+FfdcMZa5z+vVVorqPm+r+W3r6H4D4ucWRo4f/AFdwkvfn
+Z1bfZjvGHrJ2k/7qS1UzzH9o5f8Aixviz/rjF/6OSvgvvX31+0jHj4F+LT6Qw/8Ao6Ovgav6DyP/
+AHeX+L9EfxvjPiQVb03/AI+G/wB3/CqlW9N/4+G/3f8ACvoehwo/XT4X/sIeCvBGuXeueJZz4lhi
+lM1hp98ALa0iHI80dJmHq2F4+7nmua+Ov/BSfwB8LXm0LwJZr461m1BgMlnKItLtivG3zgD5mPSM
+Edtwr5B/az/bi8R/tCXt3oOgS3Xhz4dKxRbJWKXOpqDw9yRyEPUQjgcbsngfMQAUKAAFXgAdBX5b
+knBlGjBVcYrX15V/7c9/ktu/Q/Rc/wCMMfm1VOvWdSS0u9l6Jaertq9We+/En9uv41fEuaUS+MJf
+DVg5O2w8Np9kRQe3m8yt+L14Xqup32vTtPquoXuqTMcmS+uZJ2J+rk1Xor9IoYWhhly0KaivJJHw
+VTEVqzvOTZX+wWv/AD7Rf98CvrL/AIJg+H4Lz9qmK7S3jU6foV7PuVACpZoowfycj8TXyrX3B/wS
+a00zfGTx1f4yLfQYoM+hkuN3/tL9K83PZ8mWV3/dt9+n6nZlzlLFQV+p3H/BRe/8/wCMHhmzDZ+z
+6GZCPQvO/wD8RXyvX0D+3ff/AGz9o7UYc5+yaXZw/TId/wD2evn6v5Exj5sRN+Z/pNwhS9jw/gof
+3E//AAL3v1CiiiuQ+wCikNIzBRkkAdck8CgBTxXReBvAmpeP9Y+xWA8q3jw11fOMx26ep9WPZep+
+ldB8O/gvq3jfy7298zR9CPzfaXTE049IkP8A6GePrX0jomgaf4Z0qHTdKtVs7KL7sanJY92Y/wAT
+HuTX1uUZDUxjVbELlp/jL07Lz+7uvw/jfxJwuRQngcrkqmK2b3jT830cl0j0fx7csq3h3w5YeFNG
+t9K0yExWkHOWOXkc9Xc92P8AgB0rR2ewqXZjtRt9q/V4RjTioQVktEvI/jKvWq4mrKvXk5Tk2227
+tt6tt92zy/8AaVXHwI8X8f8ALGH/ANHR18AGv0D/AGmFx8BvF/8A1xh/9HR1+fnWvu8i/wB3l/i/
+RHhY34kFW9N/4+G/3f8ACqlW9N/4+G/3f8K+j6Hnoqdf8cU1nWNNzHCjGT/SlZlRWZiFUAkknGB6
+1+lX7AH7FNno+laZ8U/H+nLc61dKLnQtHukylhCeUuZFPWZx8yg8RqQfv/d8rM8yo5XQderr2XVv
++t3/AMMd2Dwk8XPkjt1Z4N8Af+Cc3xA+Ltnba14mnHgDw7OA8X2yDzNQuEPRlgyBGD6yEHkHaa+y
+fCH/AATT+CHhe3Q6ppWpeKZwAHm1jUpArHudkRjUfTFe+674xZHeDT8ZHytcEZGfRfX61ytzPNdu
+XnmkmY93Yn9OlflGJzvMsbJy9o6ceijp+O7+/wCR9fTwmGw6tGN33Zw19+yJ+zJpNwLa98H+F7Sf
+tFcX7I5/Bpc12fwu+HHwY+ClzqM/ge28PeHZ9RVI7p7W+UmVU3bAdznpubp61ma14R0LxND5Os6F
+pmrxf3b6zjl/IsMj8DXnmufso/CnXmLyeE0sHPfTbqSEf985I/SvnsTVzOonBVeeL6SlL/go+pwE
+cjaX1x1Kb7whCS/GUH+Z8rftXa3F4i/aL8a3tvPHc24nggjmiYOjKlvGOCOCM56eleT5zX27P+xH
+8O2P+j3fiCzQcBFvEcD80psP7EfgBGBl1HxDcD+6bqNP5JXxk8mxk5uTS1d9/wDgH9MYDxK4ZwOD
+o4WM6jVOMY/BZ+6ku7XTufEnPpSI3mzrDGDLMxwsUYLufoBya+/dK/ZL+FulOHbQLnUnH/QRv5ZV
+/IFRXo/hvwb4f8Hoseg6DpujAd7K1RH/AO+8bj+da08gry/iTS9Lv/I4Mb4v5ZTi/qeGqTf97lgv
+vTm/wPz38NfBHxh4kZWbTTo9qTzcapmH8k++fpgV7R4K+BPh3wm8d1eA6/qSEMJrtAIYz6pFyPxb
+J9q9d8QIX17UmYlma4ckk5J5qh5XtX2+ByLB4S03Hnl3lrb0W35vzPxDiDxHz7PYyoKoqNJ6ctO6
+bXnK7k/NJxi+sSu2XbcxLH1NJtFWPKpfKHvX0lz8qUbaIrbRSbBVryh70eUKVw5Tyj9ptcfAPxh/
+1xh/9Hx1+eo6V+iP7T8YX4AeMTj/AJYw/wDo+OvzuFfeZA74af8Ai/RHjY5WmvQWrem/8fDf7v8A
+hVSrem/8fDf7v+FfS9DzUevfsc/B2H44ftC+GtAv4PtOh2ZbV9UjYZWSCAqRG3s8jRofZmr9l/GO
+qNa2qWcJ2vOCXK8YTp+v+Nfnv/wSQ0KK48Y/ErWioM1tY2Nkjdwsjyu35+Wv5V94eKGMuuT88IFU
+fln+tfj3E1d18z9k/hppaebSd/xX3H2+WQVPCKS3kzECYx2HsKXaKl2n1o2n1r5653WIttG33NS7
+T60bT60XCxFtH+RSbB71NtPrRtPrRcLEOwf5FKF5FS7T60qqdw570XCx5Jrif8TvUP8Aru/86peW
+PQ1sa1D/AMTq/wD+u7fzqn5R9q9mL0R5jWrKflj3pPL+tXfKPtR5R9qq4rFLy/rR5f1q75R9qPJo
+5gseP/tSR4/Z98Zn/phD/wCj46/OYV+kf7VMW39nfxqfSCH/ANHx1+bvSv0Dh53w0/8AF+iPCzBW
+nH0Crem/8fDf7v8AhVSrem/8fDf7v+FfT9Dy0f/Z
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="jotoxyonsu.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <56db101d0b859b92ce2d10da21d9d2@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgAdwCe
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+6X/gkhr8UHi/4laIxHn3NlY3yL6rG8qN+RlX86+8fEsDJrU57OFcce2P6Gvxq/Zb+Nf/AAz/APG/
+w/4uuDIdHUtY6tHGCSbOXAdsDqUYJJjv5eO9ftfqENt4p0i01LTZ4ruKWJZ7e4gcNHPE4DKVYcEE
+EEEda+94nw8sPmP1hr3aiWvmla35P5nz+V1FVwvIt4nI7TRsNTlCjFWG1l4IPBBoxXzR6BBtNG0/
+5FT4pNtK4EO0/wCRRtP+RU+2k2+1AEO0/wCRSqvzD61Lt9qULyOKAPLdYT/icX3/AF2b+dVNntWv
+q8WdXveP+Wzfzqp5VevF6I4GtSns9qTy/arvlUeVVXFYpeX7Uvl1c8qjyqLhY8c/atTH7OnjY/8A
+TCD/ANHx1+atfpn+1nHt/Zw8cH/phB/6URV+ZlfonDn+6z/xfoj57MtKkfQKt6b/AMfDf7v+FVKt
+6b/x8N/u/wCFfU9DyUVBntz6V9a/sc/t3X/wAt4fCPi6G61zwDu/0V7f57rScnJCA/6yHJzs6rzt
+z90/JPalJ5PvXFjMHQx9F0MRG8X+HmvM6cPiKmGnz02fvZ4H+IfgX41aPHqvhXXtP8QWzLkyWM48
+2P2kTh0Ps4B9q3G8KW7H5biYD8D/AEr+fywvLnSb5b2wurjT7xfu3NnM0Mo/4EpB/WvQrH9pb4v6
+bbrBbfFLxdHCowqHVpXx9CxJr8+q8HVYy/2evp2a/wAv8kfSQzqnJfvIan7gHwnCB/x8y/ktU9U0
+GLTrRphNI7AgBSBjk1+QnwY/aF+MHi74y+AtDufif4pubXUdesbaaGTUWKyRtOu9SO4K5BFfsb4m
+bFki/wB6QfyJr5XNMsr5TVhTq1FLm10PWw2Jp4uDlBWscrto2VNto2151zaxDspQvNS7aAvNO47H
+nGqx/wDE1vP+urfzqr5Q9K1NTjzqd4f+mrVW8v6V6cXojia1KnlD0o8oelW/L+lHlj0FVcVip5Q9
+KPLFW/KHpR5Y9KLhY8Y/a5TH7Nfjk+lvB/6URV+YdfqJ+18mP2ZvHZ/6d4P/AEoir8u6/R+GnfCz
+/wAX6I+bzT+JH0Crem/8fDf7v+FVKt6b/wAfDf7v+FfWdDx0fW37R3/BOLxn8Mbi71n4fR3Pjfwp
+kstknzapZr2UoP8AXqP7yfNxypxmvj+VWguJbeVHguImKyQTIUkjPoynlT7Gv6CbXXGUBbhC4/vo
+OfxFcl8RPgT8MvjSufFvhPSNeuAMLdTwBLlR6CVcOPwNflWB4trUIqnjYc6/mW/zWz/A+zxGUU6r
+5qL5X26H4RDk4or9YvEX/BLP4Pas8j6bd+JtAZjkJa6iJo1/CZHP61ycn/BJLwUWyvj/AMSqvo0N
+sT/6BX00OK8skruTXqn+lzyXk2JT0s/mfFv7Gml/2z+1X8MYQu4R6r9pI/65QySZ/NRX7TeJeUt1
+/wBon9K+ZvgV/wAE7PDHwL+Kej+ObHxdrWsXmmLOsdpewwLExkiaMklVB4DkjHfFfS/iKRRPArMB
+8hPJ96+C4gzGhmWMp1MO7xUbbW1u/wDgH0OX4aphaDhU3uY232o2+1BuYR1kFH2mA/8ALQV8/c7r
+Bt9qAvPSnLLG3Rwfxp4AJ4INFwOD1KPOpXXH/LQ1W8r/AGa1NQi/0+5/66Gq/le1einojla1Kflf
+7NHlf7NXPK9hR5XsKdxWKfl+1Hl+1XPK9hR5XtRcLHiP7YaY/Zi8enGP9Hg/9KIq/LKv1W/bJj2/
+svePz/07W/8A6UxV+VNfpfDDvhJ/4n+UT5nNV+9j6fqFW9N/4+G/3f8ACqlW9N/4+G/3f8K+v6Hi
+I/e/Z+dIY8nOBn1qbbUVxOIQeRmv5wufp4vnvAMiZ1+jGqd14hmgHFxJ/wB9Vj6prPlZAPPtXA3X
+im617VJNL0Gyn13UUOJIrYgRQf8AXWU/Kn05PtXLiMTRw0VOtK19F1bfZJat+STZ00MNWxMnCjG9
+tX2S7tvRLzbSO6vPG9zEflu5PQYauZ1vx6IR5t/epGBwGuZlT8BkitPSPghq2rbZfE2vvaq3XT9E
+zGAPRp2BdvwC12uifCLwd4fYPaeH7N5u9xdJ9olPvvk3H9a814vFVf4FFRXebs//AAGN/wAXF+R3
+/VsLS/jVeZ9oK6/8Clb8IyXmeKt8TtNlbEN6Lk+ltHJLj/vlTSn4hwINzpeov95rGcD/ANAr6Wih
+jgQJGixoOioMAflT6T/tF6+1h/4BL/5YF8v/AOfU/wDwOP8A8rPmy0+KOlO4QapbLITjZJJ5bfk+
+K6qy8XblVt2UPRgwIP4167qOjWGrxmO+sra9jIwUuIVkH5EVw+rfAbwles02n2s/h66PPnaRMYBn
+3TlD/wB80/bZhT+KMJ+l4P7nzp/eg9ngKnwynD1tNfNrka+5+hxza9HPez/MDmQ96vwzxyjqM1zf
+iP4SeLvC7SXNjIniizHzMIVEF4B67M7H/wCAlSfSsDRvF6SSSRlnSWFtksUqlJIm/uuhwVP1H516
+uGzKjXkqTvCf8stH8t1Lz5W7dbHDiMvq0YOrFqcP5ou6+ezj5cyV+h6UIgeRR5VZWm62kwA3CtyF
+1mUEEZr1OY8yxB5VHlVc8o+lJ5R9KdwseG/tnRbf2WviCfS2t/8A0pir8n6/Wn9tOMr+yt8Qz/07
+W/8A6UxV+S9fqHCzvg6n+J/lE+WzfSrH0/UKt6b/AMfDf7v+FVKt6b/x8N/u/wCFfY9DwkfvrO3l
+ISa5TxDrsdhbzTzzx28MSl5JZG2qijuSe1betXYijYlgoAJLMcAAdSfQVwvgzwz/AMLZ1RNc1KM/
+8Ifay7tPsnXjUpVP/HxID1iU/cU8NjceMA/y9isV7DlhTXNOWy/Nvsl1folq0fr+Fwyrc06j5acd
+367Jd5PovJt6JlXw14Q1X4rFb27e50Lwi/MYUGO81JexHeKI9sfMw9Ac1ofGr48fDv8AY+8DWFzr
+ltNpuly+ZHZ2mmWbSb5FAODtHykllG5upPNezhcHOa+Df+Cnnwo+I3xz8P8Ah/TfB2lafeeH9Lka
+W7nmvEt55p3YqIlLkAImyNzjO8tgEbDnmoYZU5e1qvmm95P8kvsryXzu9TavinViqNNclNbRX5t/
+al5v0SSslF8Jv2s9a8ZabqHxK0/x9d+KYdJEeqeKfAq6H5FppujSOVLWc3liSSe2UGRmLkSqjjA+
+Qj7ysryDUbSC6tZo7m2nRZYpomDJIjDKspHBBGDmvzk/4Jr/AAU+IfwP1PWW8ZeHNPk0fVof7Ovb
+19SjuHtlQboQiIWDI5dkZSMjCtnGQfq74E3L/CfxXqfwW1J3Nrp0Lap4QuZST9p0ZnANsGPV7SRh
+ERnPlNbnucd7aldo4Foe6UUgNLUjCgjNFFACY964zx98KNG8exedOrWGrxriDVLUBZk9A3Z09Vb8
+MHmu0oIyKwrUKeIhyVY3X9ars10a1RvRr1MPP2lJ2f8AWj7p9U9H1Pk7U7bWvh5ra6Xrsao8mfs1
+5FnyLtR1KHs3qh5HbIrsNC8QCZVO8HPpXs/i3wjpnjXQrjStVg861lGQw4eNx0kRv4WHY/zHFfLu
+raZqnw28USaJqj+bx5lpe42rdQ/3vZxwGXseehBqcPiamGqRw+IlzKWkZPe/8svPs/tbP3vi3rUK
+eKpyr4ePLKOsora380fL+ZfZ3Xu3Ufa7ScTr1zVnyTXE+HNeWdF+au6tJVuIxg5Ne6pHi8p4f+2z
+Ft/ZS+Ih/wCna3/9Koq/I7tX69ftvR7f2TviMfS2t/8A0qir8hewr9V4Td8HP/E/yifJZx/Fj6fq
+FW9N/wCPhv8Ad/wqpVvTf+Phv93/AAr7ToeCj9l5tA8VeK/GZ8B6trdrrGj+Sl5qWoWtuYLlbbcd
+tvJg7d0pHUc7Qxr362tYbOCKCCNYYYlCJGi7VVQMAAdgBXD/AAe0G5svDkmt6mqjXPEEv9pXhXkR
+hgBHEp67UQKoz7nvXe1/J2Bi6qeLqfFPbyj9lfq/7zfkfteOmoNYSm/dhvbrL7T+9WX91LrcK8t+
+Nw/4tpcL/wBPcWP+/pr1KvlP9oPxh8VYdP8AFllpPgm31TR7O5tmsZGbZ9oUyqCUffh2+YkjAxtP
+J4r02rpnks7z4GKT4K1zaMA3y8Dv8iVp/tKeG7uTwjY+ONDgM3inwLdf27YCP788KqVvLXPcTW5l
+XH98RnqoryD9l/xb8VrjQntPFXw/s7CxvNTIm1SxvCY7eMIm0mMkliSOSDhScHpX12QHBDAEHg57
+1MNAW1ijoes2XiPRtP1bTrhbvT7+3juradPuyROoZGHsQQav14l+zrct4GvvFXwkvSUm8KXP2nRh
+ITm40S5d3tWUn73lMJrY46eQv94V7aD7YrRqzGgooopDCiiigAIyK4n4sfDqL4h+FJbNCsWqWx8+
+wuSOY5QOhP8Adb7pHofYV21B6VhWowr03Sns/wCtPNdH0ZtRrTw9SNWnuv617p7NdUfGfhXXJoJj
+BcRtb3MLtFNC5+aN1JDKfocivZPDOsCZVGa4T9obwx/wifjy2162TZZa0pEwUYC3KDk+25MH6qaj
+8H+IP9X81Y4TGzlTcKyvOD5ZebXX/t5NSt0vboXjaMKVVSpL3Jrmj5J6W/7dacfO1+o79t7En7I3
+xGcf8+tv/wClUNfj/wBq/W39sbUxdfshfEZd2c2tt/6VQ1+SXav27g2p7XA1H/ff/pMT8/zn+NH0
+/UKt6b/x8N/u/wCFVKt6b/x8N/u/4V950Pn0f0MIoUAKAqjoB0p1FFfzCfq4V8j/ALTvhHUtafxK
+0Vx4na1nuLQ7bG8lit7VUULvjVVxncFbq3zA8EEivriuC+KEXmeCrhOcC5jI9v3lD2ZLPmb4D+D9
+Yk13T9Tv77xS0Sa8l6PMuJEtrg+QIWMyBArKeGI+UbsYHr9q+x79jXlvw0V4vCN8D3vV/H7tep9K
+iLuNHlHxx+HWuavJo/jjwMIV+IXhjzGsoJ5PKh1a0fH2jTp36BJQqlXIxHKkb9AwPT/Cz4n6R8Wf
+CcWt6UtxayLI1tfaZfR+VeabdJxLbXER5jlQnkHqMMCVYE9h1ryf4n/CzWf7dPj34c3VtpHjyKJY
+rq1uyVsNft0+7b3gUEhlBPlzqC8ZOPmQsh23VmB6xRXm/wAIPjno3xZXUNO+y3Xhzxlo5EeteFNX
+Ajv9Oc9CQDiWFuqTRlkcEYOcgekZqRhRRRQAUdaKKAPNf2hfDX/CRfCvV2RC9zp4W/hx1BjOW/8A
+HN4/GvmLwjq/3Dur7e1SxTU9Nu7OQAx3ETwsD6MCD/Ovz68OzmyvJLY5BhkMR/A4rwqv7rHNL7cL
+/ODs/vU18kd9T95gIv8Aknb5TV19zg/vOn/ap1Pzv2U/iCm7O61t/wD0pir8wAcgV+jH7S96Zf2Y
+fHaZ620A/wDJiKvzkiOUFfufA0v9hqr++/8A0mJ+dZyv3sX5fqSVb03/AI+G/wB3/CqlW9N/4+G/
+3f8ACv0fofPI/oaooor+Yj9XCuK+IS7vCV2PSZT/AOPmiik9mTLoYnw7fHhW5XqPty/+y16ge/ei
+ioiNCE7c5+tGc0UVYHmfxi+BGl/FdtP1i21C78JeOtHDHRfF2kbReWLHrGwYbZ4GPDwSAowJ6Ehh
+5v8AD39ryz0bxjqvw1+K0tppHxA0aKKVr7SIZptO1WCTHlzxAKXhY9GicfKfusw6FFaxV07kt2Po
+2wv4NTs4bq1fzYJl3o+CMj1weasZwOeKKKhlJ3EBBA96WiikMK/OzUW+zeONcjXgLfTgf9/Goorw
+8Z/vlL/DP84Hev8AkX1f8cPyqGb+0XOX/Zs8brnrbQf+lEdfnzbniiiv2zgX/cqn+N/+kxPzvOP4
+kfQmq3pv/Hw3+7/hRRX6b0PnUf/Z
+
+------=_NextPart_000_0018_01D0B859.4A05D240
+Content-Type: image/jpeg;
+ name="faixeevavzeky.jpeg"
+Content-Transfer-Encoding: base64
+Content-ID: <f9f9401d0b859e9261fdc071801d40@nasutkadqw>
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA7Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcg
+SlBFRyB2NjIpLCBxdWFsaXR5ID0gOTAK/9sAQwADAgIDAgIDAwMDBAMDBAUIBQUEBAUKBwcGCAwK
+DAwLCgsLDQ4SEA0OEQ4LCxAWEBETFBUVFQwPFxgWFBgSFBUU/9sAQwEDBAQFBAUJBQUJFA0LDRQU
+FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU/8AAEQgBJwCe
+AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMF
+BQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkq
+NDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi
+o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/E
+AB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMR
+BAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
+R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKz
+tLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A
+/VCikyOlLQAVxnjrDeFrwd/O/kxrs64fx1Lt8PTrx81yw5+pqZbEvoYnw/48P3Sd/tsXH1xWz8Qf
+jX4V+HF5BpmoXst/4ju0L2fhzSIGvNTugAeY7ePLbeD87YQd2FcRZ6odK8E+I5knNq0EX2hZwwUx
+7QTvz2xjOfavHPB3hzw74i8IalDpN6bDTvExa81HW9CvZIH1L+87X6N5rjsV3gckAdqKavcic+Rb
+Gh8RP2kPFereJx4W+3T+EtUm+54K8D20Wv8Ai6VCcFp5ObPTl2srbpC+BzvFfn38b/2xfGXhTxvf
++HPBGja94G8W6bffY9Q13V/F15rGqTzREh43Jl+ylcjBVY2Xj5TX65fBXw/4E+GXhyPw74Q8N2Hg
++NsN5EMe03T4wJJJW+eVz6yEuR61/Px8atL8UeF/iz4s0zxvp9xpviVdUnnvI51IdJJHLllJ+8jb
+gykHBBDA810wV9GF1JXibvxI/aL+KfxIvJYvF3xB8Q6jdglZbSfU3S0B7qI4yqIQeCMY968zmuPs
+67bmxDM3Kszsc++c1uaHqCjWbLULnT7DXJoJFlKT/Ms4H8M0W5C4PfBB9699sfj58P8A+zYxP+zj
+8PnvcfvJ/O1KNCfaLzWx/wB9n6itkpfZQrx6ngfg34ueL/h/eRT+GfE+ueG3Q7gdJ1Oa3/QNjH1F
+fWPwl/4KlfHf4S6jZw+LLm1+IWhuBJ9m1VEjuWi45iuYgDk/3nDj2r5v+Jni3TvGuqW19ZeAvDfg
+yCCMxx22jrNBbvznfIJp5HkboByo9jXAXOqTPOZTJ9oumG0SAfKo6AKPboOw7Cpcb/ENP+U/po+G
+vjuw+J/w+8O+L9Mjnh0/XLCHUIIrldsqLIgYKwHGRnBx6V0teHfsR+F/Efgz9lL4aaP4sga01y20
+pfMtpAVkhjZmaKNweQ6xsgI7EEV7ieK5XuaIRiACScAetfm5c3v2zxbqtwDkS3cr5+rk/wBa/QL4
+ia8vhnwJr2pudn2ezlZT/tlSFH/fRFfnXogaS6LHks2c189i5c2Ogl9mDv8A9vSjb/0lnfJ8mXu/
+2pq3/bsZX/8AS0S/tCyZ/Zx8aj/p2g/9Hx18CW9ffX7QiEfs5eNf+vaH/wBHx18CwCv27gPXBVX/
+AH3/AOkxPzvOH78fQnq3pv8Ax8N/u/4VUq3pv/Hw3+7/AIV+n9D55H9AfhLxFb+LPDWmaxasGhvY
+EmGOxI5H4HI/CtevkP8AYY+N9pr9rqXge51O3vbyxfzbZ7eFoYjuXcY0VsbeBu2j0evrsNmv5RwG
+IeIoJ1PjWkvVfo915NH7RmGGWGruMPgesfR/qtn5pi15r4/vA2mTxDJMd2+8Y6cjFelHmvJviHNJ
+pul6tflWlRbvaIuBnkcg12z2PNORWO6v9G1G0s0E895E0CRKBl252qM8cnjniuc8B6JJ4emvYLuz
+aylOwyWL2ot1XHOSg+U9PvAc46mu0+HFxHrVzY3cSyRKl4qrFwejckn8a9g13wxYeIkUXcX75ARH
+PGdsifQ+nscinSdkZVabmtDygXCyoridZHkUskk3Kg5yvToucD1968+/aD/Zw8B/tQ6JFb+PtIkt
+9dtIjHZeI9MKrfWgPIBbGJosnO1xxk4wTmvVNd8H3PhVzdQiS7sRgvJGg3r2LFRxn1xwR6VSi1GK
+aMOkisiMd+3ICbcA5HYjI4P4VumcHvU5H5a+Pf8AglB8VdCvpx4RbQviDpCEus9vdCzugMkASRSM
+pVjjOAzD09K8pn/YO+OthJJbt8JfEjMeA0HlSKOezc/zr9q9E1w6fr1j5JG6eYRMuMbkY9B/OvWx
+0rT2jXQ7Kb9otT8MfAX/AAS2+OvjGeAz+EtP8K2sh2vd69fpuQHqTGhduPZK+/f2Uv8AgmF4B+Ad
+9ZeJvFEieOvG1swlguLiLZY2Mg5DQwknc4IyHfJBwVCkV9pUmKzlNs3UbABilPSgnFQXl5DY2k1z
+cSLDBChkkkc4VVAyST6YBrJtRV2Wk27I8C/bE8bDR/BVnoEMmLrU5RJIo6iJCDz7F9v5Gvlnw3bE
+uuK2vi/49k+KPxDvtUG4WSN5NpG38MS5C8dieWI9WPpUvhjTSShxXyFOr7aU8S/tvT/CtI/frL/t
+42zKSpyjhY/8u1Z/4nrL7tI+fLcw/wBoiAp+zh42Yjpbwf8Ao+Ovz9iXAr9Gf2lrXyv2a/G5/wCn
+eD/0ojr86VGFr988Pvey6rL/AKeP/wBJifnubO9SPoLVvTf+Phv93/CqlW9N/wCPhv8Ad/wr9R6H
+ho/Rr9jPSX8ear8VNIbV9TOsW40nUtN1W9sBavazosyqyqPvLkYYd1JHevtL4d+N28W6fcW9/bjT
+vEWmSfZtV03OTDLjIZfWNx8yN0IPqDj4t/Ye8Z23gj4m/ECLxZc3nhuG+07TpIZPFOoIN8itMpSN
+mwDwQcdeelfVnjk6TqOo23iLw34l0nTvFlihjSSa6Tyb2HOTb3AByUJyQeqHkd8/yG4zouOJpK+i
+Ul3S2a/vLp3WnZr94nOnWlPC1nZXbjL+VvdO32X17PXun6nuBr8ef2tf24fi/wDAz9oz4l+E9IvN
+O1Hwl/aava2Ws2aXSQkwxllQ5BUbiTtzxmv1W+HvxL0zx/aSiHNlq1qdl7pczgy279Oo4dD2deCP
+fgfnt+3N/wAEzviD8YPizrPjn4f6npGo22szJcXOl6lMbaeCURqh2Pgo6HYDyVIJI5r2aNWliIKp
+B3i/6+TXVbpniVqNTDzdOqrNf180+jWj3R4b+zZ+298WvjH+0P8ADDwle6hp+keHbnxFbSXlno1m
+tstwobJWRslmBx0z1FftX+gPavyx/Yw/4JZ/ET4afGLw5488f6xpOj2mg3QvYdL02Y3NxcyqCFVm
+2hETJySCxOMYGcj9T62slsYiFQQfyri9X+HI1PVfMF4ItOkkLz22zkgrgqpBGM9c9a7WigmUVLcx
+NH8G6RodwLi0s1W52BPOdi74HTkk+prbxiiigaSWwUE4oPFJuBFAxGIxXy7+1R8ZhIkvgnRJg8hO
+NSnQ9CD/AKkfjy30C+uOi+OP7QyaRFceH/Cc6z6ocx3GoRHKW3YrGehk689F+tfMun6K88xklDSS
+Mcszckn1/nXymNxkcZehSf7v7T/m/ury/mf/AG6tb29CU/7Lj7SX8Z/Cv5L/AGn5/wAi3+09FHmq
+6ForMwyPfmvS9A0oxqvFVNG0YRgfJ+ldhp9qIscYrhqVrnyrmecftSw7P2avHH/XvB/6URV+a/av
+0s/auk/4xx8bqP8An3g/9KI6/NOv6A8OHfK63/Xx/wDpMD5rM3epH0Crem/8fDf7v+FVKt6b/wAf
+Df7v+Ffq3Q8dH3v+xt8KvC3xD+LXi/Q/Gfw+0e8trbSbPUIvtd8uohZhNIoZRuIQgcZHXFffX/Cr
+fCWSf+Ee0/J/6YCviX9gJbeH9oLxatpJ4aeObwvA8g0CRmO5bogbw3+8a/QOv5RpfBGx+zYy/t5X
+OD1z4N+H7+OKfS4T4b1i25tdV0rEc0J9D2dD3RgQR+dZ0XxE1rwE32fx5p+bJPlTxLpMTPauPWeM
+ZeBvU8pnoR0r0wjIpGXIIOCDwQa56mGfO61CXJN79U/8S0v6pp9L20NKWMtBUcRHngtujX+GXT0a
+cetr6lXStYsdcsYr3TryC/tJRlJ7aRZEb6MOKthga+J/2iNYvfhb8dV/4RS9v9HS40qK7k03RLIK
+kr+ZIpkZyQgLbQCCDnFVNB/bg8UaDAh8RW2lvEzbVF3MqzY9/L4J/Csv7Sq0ZOGIpPTrHVfc7S+S
+T9Tu/sqFaKqYaqtektH96vF+t16H3JRmvl/Qv249O1O286fwhqdvCOty7eVCeOoaRVyPpmrY/bm8
+LTOY7fRNRunHOYnQp/30OP1rVZthevN/4BP/AORMP7HxnaP/AIHD/wCSPpTNGa+S9a/bxtcmLRtA
+ju7sKWNutw1xIv8AvLGuF/Fq4m1/aZ8ffEbxlo+i3Rk0TTL66EMttprrHdsmDkJgkg5A6sOM1nLN
+oP8Ag05S+XKv/J+V/cmaxyeqta1SMV68z/8AJbr72j7D8afEvw34Cg3azqUcMxGUtI/nnk/3UHP4
+8D3r5w+Ivx28QePFlsNKjl0DRX4YK/8ApU6/7bDhFP8AdXn1NVvF3gq10LxZqEEFo9ltKEpcTefN
+yoOWkJO4nOetUI9KjXqRXzuLxmIxEnTrPljtyxvr6y0b9LRXR3PCqZpRwrccFF8y+3K1/wDt2Oqj
+6tyfVOLORsfDyrjCbR7CujsNFVMfLWrHbxx1YDqvSuP2tlZHzkqsptyk7t9+o61tkiHSriyheBxV
+Lzfejzvf9ahzuZ8x5x+1VKW/Z38aj/phD/6Pjr83etfov+1HJu/Z98ZjPWCH/wBHx1+c4r+kPDR3
+ymt/18f/AKTA8LMXecfQWrem/wDHw3+7/hVSrem/8fDf7v8AhX630PKR+hf7BOoxSftI61FBqmna
+gG8JEyCz0v7G2Vu48MflGRyR+NfocGBr8vv2bfHHiDwV8dBqul6P4x+IM6+HriH+xxbQW0zqbiI7
+1eSRUITuN2fn4FfoFb/EnXJLaCWX4fa9AZY1cxl4GaMkZ2tiTqM4NfyhQi5U00ftGOXLXf8AX+R6
+BmkPSuGPxI1KIgzeCPECJ3ZI4pCPwD0lz8VltI98vhPxQFJAG3TS5JJwBhWNdPJI8+58q/txWUK/
+FXSZLlLua3n0dAytfLa2qETScsRlyxyOnpXzta3yJN5GlOsE/l/KvhrS/OuFXod11MSB1/nXsf7Y
+HiG81vx5ouqeKPCVl4Y26e8dp/bF+tzJJAJSQzQRfKrEn7pZv6V4rc3N/qOn75FuZNPGCJr+Qadp
++PQIMF+/QGvHqq9RtH1uGdqMb9v67kl1FLJerJdeRDdkbXN/O+p3YA5A2DKIT7YqK5mgSWOO5kub
+2Y52LeSdT/swRfyJpEtCIGaSd5rdV3MLVfsNkBjqZGG9x24AqW0jItXktI1gtAo3tbD7Jbkn+9M/
+7yT8KysludPM3sRyyXTWyWzBbOAD5LOZmREz0xaQDfJn0dgPeus+Euy0+K/hI8eZFqMQCSDYy5yP
+lhQnYDk/fY4rm1htrQiJ3O13QGG2VoI5R9f9bJ39M10fw+upLbx54XjjVLNRqMTNbQR7SQZB95F+
+7gd3OfbNRN+67DUb7n1n8YW8v4i6jg/eigcf9+x/hXG+dXVfGo+V8QJuwaztyPptIrhvO96+Zxcr
+Yip6v8z8hru1Wa82XzNSecPWqHnUvn1ycxhzF7zh60ed71R8+jz6OYOY8+/acl3fAPxiM/8ALGH/
+ANHx1+eY6V+gP7S0ob4D+Lx/0xh/9HR1+fw6V/THhg75RW/6+v8A9JgeNjnea9Bat6b/AMfDf7v+
+FVKt6b/x8N/u/wCFfr/Q85H3r+x3p9xZftYaL5kHim0hfQNROzWrpZYWffCeAoGGA49eK/SIDivz
+W/ZOtVh/a48KSP4bXRZH0zVh5ketfawzEISNmTjvX6VDp+FfyXTf7uL9fzP2nGq1d/IKD/nNJRWq
+ZwHxT+3e8dp8R/BM6tBFdTafPErx2X2m7YCVDiIEEKOeuM81813Nq9vIZ75ha3TuMPqDG+1Bs8fL
+CDhPxIxmvqv9uie7TxP4IiszfySS2t7vttOQK8wDRcNN1jUH3Gc18o2VusEUsEUqQ7X82az0dxtB
+6YmvG4zxyE3GvPrN+0f9dD6vBJOhF/1uPuTFb3cyGJ7i+O0K93i8uyfVYV/dx9O+aW4aae4d7iVx
+dKgXCuJ5io6hnP7uHPoD+FVYLtXglGmxxLaA7bkwMYLMEdTJcMfMmI9M8+lZ76tYQ3KafLJ/aEqs
+GjgSPyrSI4JASMcyevzfjisLHW5KO5oxXLSCR7RB5ZJ82WOUhVbH8Vwfmb/dQCqkfiB9H1SyNsj3
+F5HKjxvEnlw28qsCr+X/ABtnHLn6iua1PWptRJ1G4+0ww2MpR0UqRGAf7o6cBuR9BUD20Ooi4Fkn
+2WO6UTpd5INvnkE4Ocnjjrmqt3OaVZvSJ9MfDPxrr3jbQbvUvEupXGqaot9LAbi5cMwRcbVGFUAc
+nAAxXW+fXi3ww8VQ+FtBntb3e0ks/mA7DyNgBJweM4zXbx/ELTZhxNEns+4V85icHXnVlOK0bPg8
+XlGKqV5zpRXK3pqkdl53vQZuOtc3B4rs5c4uIG/3ZQP51oW9+tyrMuMA4yr7v5VwVcNWox55rQ8r
+EZdisLTdWrG0V5r/ADNPzqPN96o+fR5/v+tcfMeVzHC/tIS7vgZ4tHrDD/6Ojr4Kr7q/aJl3fBHx
+UPWGLv8A9Nkr4Vr+n/C13yet/wBfX/6RA8zFu8kFW9N/4+G/3f8ACqlW9N/4+G/3f8K/Y+hwo+3v
+2WZrCH9r7wUxj8MR3LRavCf7HnlMiy+SSw2scdm/Kv033gHGa/HG28f6n4O17TvG+jS6Pba5pksi
+w3+mWMU9wHnUxsxjZVDNhmJB9SRmvXNG/b/+JUMUUNxqWmag0agPdXGiNG0x9SEfAPsOK/kaElyJ
+Nn7ji6UqtXmjsfpnuFLmvzdsf+CjPj7y4jc2nhrc/wDyzezuUOc4xwx5ro4f+ChfjpbWY/8ACNeH
+7iRVyuBdJk+4x+lWpR7/AJ/5HF9Wq9F+KPRf2/rW4uLfwSIrdrqEveCQS3Pk2qYWIhpv7wHYex4r
+49vr+2SBXnZtamgBMUSRGGxVsZwkY5YD+82F+tXPG/xg8d/F7xLZXvizWor+E3csVlaWqCK0tvlH
+EcXJZiD99nJOPTiuH1e3gj1c3Qla+hKpG4aUnzDnLgYwoACk49hXLO0pto9uhKVKkoPdGrPqmra3
+pV284EEtuqtEBGpVPlBURr90dfvceuKxZpr2O3gl02RLrVY5G3WvlhtuFwGGOxBJ9yvJ5pyWDz6p
+Dq1tI8Wmu4eKGbLNIduAwXuQOMnoO1bOnaT9nDF8Rl1Ado+HkGSQregGTwOtToh6t6lKCyN1clir
+smMGIsDHIcclyOMA5OMc1tWtjDb4KhS46MFAA9gPSrEcaooVQFUdAFGP8+9TKvfkmp3GIuc5zmp0
+zxyaRY8+lSCMj2+lNAKgJAxj8K9B+HJePSrxmUhJJwVY9GwuDj2zXN+HPDR1dhcXOUsFOODgzH+6
+D/d9T+Vd8kixoqIqxxqAqoowFA6AV89mmLhyvDx1fXysfFZ7mVLkeEhrLS/lbW3r37epoeeKPPFU
+PPo86vmbnw/Ocb+0FLu+C3igf9MYv/RyV8QV9p/HuTPwc8TD1ii/9GpXxZX9SeFTvk1f/r6//SIH
+JXd2gq3pv/Hw3+7/AIVUq3pv/Hw3+7/hX7N0OZHtOn3HlJfpf2cqLcFT5cTq/mYHVsMOQcUzTdEt
+J4fNjm84NkYErAqe6jK/pyK6v/hXUSMGjhRBnPyljn65Joi8EXkMcQLQSFFKnhuecj6V/Htz9/t3
+MOLR/wB4zQXISRcMA9wrZYMDknbntjitPXfE0mmRxxtbNcSXW9IliuhkEDkkHqOauf8ACK6jD8kU
+NssJJJHQ59en/wCuoP8AhCr0hs2lkR2wFz+BI4p7hsYF7p9v4kTyLGH7DLADFczY8tUfbyRyeRjt
+irWm6Oxsbe2WQzJbD5HChYW/Dn5Tntya6i28KzYBuFi+9uEMf3Aexb+83f0+taC6MwwMAAdAOlLU
+ehh29kIcMTvlIAL7cYH91R/CPYfrVpIicdTWwukkdaeumAUrMLmUsB9KkSE+lag04DvThYgd6aTC
+5mrH3I4ra0Hw+L/bcXIK2nVU7yn/AOJ/nU1ho6SsJJ1zEOVQ/wAf19q3fP8Aw4x+FfPZhmPs70aL
+16vt5Lz/ACPh85z5UW8NhX73WXbyXn3fT12viUKAFAVQMADgAegHajzveqHn0edXydz879oX/Oo8
+6s/zvelE3I5ouHtDkvjpLu+EXiQf9Mo//RqV8cV9efG6Xd8J/EQ/6ZR/+jUr5Cr+p/CnXJq//X1/
++kQJb5lcWrem/wDHw3+7/hVSrem/8fDf7v8AhX7R0JR9rtbwswJiXI6ckf1pGto2xhSgHZGIryf/
+AIab0D/oB6r/AN/Iv8aP+Gm9A/6Aeq/9/Iv8a/i9cI8VrbDT/wDAo/8AyR0RxuNj8NaX/gT/AMz1
+lrdGGMuvOflfFNe1UrhWdD65Dfzryj/hpvQP+gHqv/fyL/Gj/hpvQP8AoB6r/wB/Iv8AGtFwrxat
+sNP/AMCj/wDJGyzPMI7Vpf8AgT/zPVja5HEjA+pANKbUBTiT5vUqK8o/4ab0D/oB6r/38i/xo/4a
+a0D/AKAeq/8AfyL/ABrVcM8XL/mGl98P8zeOc5nHas/wf5o9WW2OOXBbH904/nSC1bblim70ANeV
+f8NNaB/0AtV/7+Rf40D9prQB/wAwPVf+/kX+NaLhzi5f8wsv/JP8zeOf5ov+Xn4R/wAj1RbeUj5t
+in2JP9KWO3LH94Nqg8KGzn615T/w01oH/QD1X/v5F/jS/wDDTWgZ/wCQHqv/AH8i/wAac+HOLpxc
+XhJa/wCC/wD6UOpn+ZVabpudr9Ukn96PXi5Pejca8h/4ab0D/oB6r/38i/xo/wCGm9A/6Aeq/wDf
+yL/GvJ/1I4k/6ApffD/5I+c5ZHr240bj615D/wANN6B/0A9V/wC/kX+NH/DTegf9APVf+/kX+NH+
+pHEn/QFL74f/ACQcsj17J9aAx69fSvIf+Gm9A/6Aeq/9/Iv8ax9f/aaeSBk0PQzbzkcXOoyCTYfU
+IvB/E1vQ4D4krVFD6o436ylFJevvX+678gUZG1+0b4rh0/wxb+Ho3DXuoyJNKgPMcCHIJ/3mxgeg
+Jr5zq1qeqXmtahcX+oXMl5e3Db5Z5TlnP+AGAAOABiq1f1NwvkMOHMshgVLmldyk+jk7Xt5JJJeS
+uzW1lYKt6b/x8N/u/wCFVKt6b/x8N/u/4V9X0Gip+dH50UUxB+dH50UUAH50fnRRQAfnR+dFFAB+
+dH50UUAH50fnRRQAfnR+dFFABRRRQAUUUUgCrem/8fDf7v8AhVSrem/8fDf7v+FHQaKlQyXcELFX
+mRGHZmANbWteE9e8Moj6zoOraMjnCPqNhNbqx9AXUDPtX6pf8E5fCGg63+yvotzqGiabf3P9oX6m
+a5s45HYC4fALFST/AJHavEzXNIZZhlieXnTaWj9euvY9LCYKWJqOnLTQ/JSW5igIEkqISMgMwGRT
+Pt9t/wA/EX/fYr9N/wDglt4X0bX/AIY+PX1PSLDUpIvE8qo93axysq+TFwCwOB7dOa6Xxv8AtpfB
+7wN4y17w1d/CDxFd3Wj301hNcWfhm3eCRo3KsyNuGVJHBIGRXmVs+qRxVTC0cO5uG9ml+aO2GWQd
+KNSVS1/I/KpGWRQyMGU9CDwfxqOa5ht22yypG2M4ZgDXofx88daL8TvjJ4q8VeHdPk0rQ9TuI5LW
+ymhSF4lWFEIKISqksrHAJ6+tfe3/AASy8L6Nr/wU8VS6lpGn6jLH4jlVZLu1jlZR9ngOAWBOM9vr
+616mPzL+z8EsXUp66Xjfa/nbocOHwixFd0Yy011PzHeeOKMO8iqh6MWAB+lC3ETRGQSoYwcFwwwD
+9a+9/wBirQdM1f8Abs+M9le6bZXdlENVMVtNbI0UeNSjA2KRhcDgY7V1PxK8OaRb/wDBUf4d6XHp
+NhHpkmjDfZLaxiFiYL3kpjaTwOSM8D0FcdTO40686Dp/DT5738r22/E6I5dzU1U5t5W/E/Nv7fbf
+8/EX/fYp6XUMgYpNGwUZYhhwPU+1ftD+0R8X/hp+zdLoEet/Da61w6yJzCdB0KC5EXlbM+ZnbjPm
+DHXOD6V8t+Of2i/h18ffjJ8DdC8M+BNQ8NXFt4xtbi8TV9IgtVuIGDJtwrEsMnoRjj1rlwmfVsXB
+VY4VqFm+a6tpfy8rG9XLKdJ8rqa+h8BxXEU7bY5UkI5IVs4FNN3AJNhnjDg42lhnNfpz/wAFRvAt
+ra/Cbwh/wjvhlBcNr373+yNN3Sbfs033vLTO3OPbOK9L8JeBtGT9gq2urvw5YQ6rH8P5HeWewRJ1
+lFix3ElQwfIzk85pf6yQ+q0sV7L45ctr7eewPKWqkoc2yvex+Pkl1DC+ySaNG9GYA1L2HvX6w/8A
+BNrwfoWt/sn6Bdajomm39yb2+UzXNnHJIwFw+AWIJOK/MDV/h54u0o3t1eeEfEFlarNKWnn0m4RF
+G88ligAGK9XB5rDF4mvh3HldN21e+/p2OPEYCVGnCcXfm8tjmpZo4QDI6xg8DccU03MIiEhlTyyc
+B9wwT9a+4v8AglRoml+IviJ8QU1HT7PU4U0q0eNbqFJlU+dIMjcDjj+Vem6DYaNp3/BU/wAR6A2j
+ac2l3/hpIUs2tIzCrrbwTblTbgN8jcgZ5Nc2IzuNDE1sN7O/s4c9777abeZtSy72lKFRy+J2PzSi
+mjnBMciyAcEqc0+vsb/gqMul6d8b/C+i6VY2mnxWeg+fNHZ26RAvLO4+baBn5Yh19a+Oa9fA4r67
+hoYjltzK9jzsTRWHqumnewVb03/j4b/d/wAKqVb03/j4b/d/wru6HMj9rfgT4zvP2ovgF9s+IvgV
+dEXV2ubK70W9ify7iFWKiRVkUOFYdM8gjIPQ1yX/AATy0630X9m+PT7Sc3NpZ69q9tDOTkyRpeyq
+rZ75Civj34p/8FQPiB478L3ui+H/AA/p/gw3sTQzalDdPdXSIww3lEqioxBIDYJHUYIBGJ+zl+37
+qP7OnwqsPBFl4GtNbgtJ55hey6k0LN5sjPgoI26bsdecV+TT4fzCWFqxhTUeeaagpLRJS6t26rrc
++2WZYb2kby2W9j6T/wCCUP8AyS34hD/qaJP/AETHW547/bo8f+EfHHiLQbL9nrxTrdnpeoT2UGqW
+7T+XdojlVlXFqRtYDIwT1618b/sv/ttah+zH4Z17RrPwhbeIU1bU31JpptQa3MRKKmzAjYEDbnPH
+WvaP+HuOtn/mmFh/4On/APjFdeNybF1MwrYh4ZVYy2vPl6Ls0/vMaOOoKhCCq8rXlf8AQ+DtZ0u/
+0TV72y1SxudM1COUmazvYWimhLHdtZWAIOCDyOlfp5/wSbOPgh4u/wCxlk/9JoK/Ov40fEyb4y/F
+TxH42uNPTSZtZmjmayjmMqw7YkjwHIGchAeg617H+y5+29qH7MPgrVfDtn4OtvESX+pNqLXM2oNb
+shMcabMCN8geXnOf4jX0+dYTE5hlvsqcP3j5W1dad9XpoePga1LD4vnlL3ddT6C/Yt8GeItC/bl+
+MWqan4e1bTtLul1X7Pf3djLFbzZ1GNl2SMoVsrkjB5AJrofilompx/8ABUT4a6w2m3iaQ+mLbJqL
+W7C3aX7NfHyxJjaXxztzn2rz+T/grlrixuR8L7AkAkD+2n5/8gV7H+05+0kmg/BD4L/Fyx0611K4
+/tez1Q6L9t2gNPpt0pQuAT8hlPO3qOQM18hWo5jHGKdeiourB017yevLvp+p7sKmGdG1Od1FqW3m
+j1L9pv8AaK8S/Aefw4nh74Y6z8RV1QXBnbSjIPsfl+Xt37IZPvbzjOPunrXw34q+Jfir41ftqfBX
+xl4h+HWtfD6CPVNP0mGDVY5SszJcSSkq7xoCcSH5QOgzXYj/AIK4a5j/AJJhY/8Ag6f/AOMV5t8X
+f+Cgup/FvxB8PNUm8D2ult4O16PXYok1NpRdMiMoiYmNdoO7O7BPHSunLMpxmDXLPCpScZJz59dU
++l7dlsZYjGUKuqraaaW/U/QL9q39p22/Zc8JaLrlz4cuPEialf8A2EQW1ysBjPlvJuJZSD9wjHvV
+vxJ49T4n/sha/wCL0sn02LW/Bl3qC2kjh2hElm7bSwABxnrivzR/am/bW1D9qLwho+g3nhC28OJp
+t/8Ab1uIb9rkyHynj2YMa4Hz5zz0FdRo3/BQ/U9H+AkPwvXwJaSWsfh86B/af9puGKmAw+b5flEZ
+wd23djPeuSPDOIhhqMo0v3ql73vL4enW33amjzSlKpOLl7ttNOp9ff8ABMc4/ZE8P/8AX/ff+lDV
+Y/ZC/am8eftA+J/FGmeLPh3J4UstMgSW31FI7hIpWMjL5R81FyxA3DHYHPUV8Wfs4ft+6n+zn8Kt
+P8D2vgi116GzmnmF9LqTQM5kkLkFBEwGM469q9C1j/grX4subCSPTPh3pFleMCEnu9SlnRD6lFjQ
+t/30KrGZFja2IxMlQUvaSvGTklyq7d7X6+ZVPMMPGEP3lrLVW3Poj4K+BNB8AftzfGK18PW0Nja3
+/h3S9Sns7cBY4biWWbzNqj7u4qHI9XPrXgPxN13UvC3/AAVP/tvTNE1TxC9haW8tzp+jQCa6e3bT
+zG7IhZQ2DIuea8O+B37afib4RfE/xr491fS08b6/4rjiS8lurs2wjMbMV2BUYBQpChcAKqACpbb9
+s6/tv2qrv43jwlbve3Gnf2edFN+wRR5SR7/O8vOcIDjb3r1KWUY2jiKs5x506PJe6952irb36PVn
+LPHYepCKUre9fbYrft4+JNS8Z/tE32u33h3W/DFtd6ZaLY2PiC2FvdNCgZWcoGbCmTzAOe1fPdeu
+ftPftD3X7TPxBsPFV3oUPh6W001NNFrBdG4DhZZJN5YquD+8xjHb3ryOvssvpzo4SlTqR5Wklbe1
+vPqfO4ucaleU4u6b3Crem/8AHw3+7/hVSrem/wDHw3+7/hXodDlRUpK6jxX8K/G/gS1+1eJfBfiH
+w/a/8/GpaXNBEPq7Lgfia5cHIBHIPQ+tRCcai5oO68ipQlD4lYXijAopGIQZY45wM9z2A9/arJSv
+sLRXbab8C/iXrGnDULD4c+LLyxI3C4i0W4KEeo+TkVx99YXelX0tjf2lxp99DxJa3cLQyp/vIwBH
+4is41ac3aEk36lypziryViHp0600RorFgoBPWtnSPBviLxDam60nw7rGrWoYxm4sNPmnjDDqu5FI
+zyOPerv/AArDxr/0JXiX/wAE1z/8RQ6sIuzkvvKVKo1dRf3HNY5o71v3nw98W6daS3V54S1+ztYV
+Ly3FxpVxHHGo6lmKAAe5rBqozjPWLuRKEofErBSGlrW8M+ENf8bX7WXhzQdU8QXifeg0uzkuWX/e
+2A4/GnKSiuaTshRi5O0VcyO1GK6Pxb8NfGPgCJZfFHhHXfDkLHAm1XTZrePPpvZQv61ztKM4zXNB
+3XkOUJQdpKwUVq+GvCWveNL1rPw7oWqeILtPvQ6XZSXLL/vBFOPxrS8V/Cvxv4DtftPibwZ4g8PW
+ve41LTJoYh9XK7R+dS6tNS5HJX7X1KVKo1zKLscxRR17iitDIKt6b/x8N/u/4VUq3pv/AB8N/u/4
+UdBo/cX4E/tBeC/2nfBmoar4aW5a1t52sL/T9VthHLE5QHa6ZZWVlYEEEggn0Ir8wP2/fgjoXwP+
+O623hmCOw0PXtPXVYtOiGI7STzGSVIx2QlQ4HQbiBgACv1Z+Fvwb8IfA3wnPovgTQodKspJGumiW
+Z3e4mKgbnkcsxJCqMknAHFflz+0X4N+Mf7Q/7W1x4b13w3b6L4tnsv8AiT6Q96DaJp0Qdw0dyQBJ
+kmQs2B8+5dq7cD8j4cqUoZjVqUJ8tFJu0mr2/wCBvfotL6n2+ZwlPDxjKN5t9D5br9Hf+CYf7O/h
+6+8J3PxX1uyh1PWpL6ay0hbhA6WEURCvKgPHmO+4buoVAARls/P/APw7a+PX/QC0X/wcx/4V9Ef8
+ErfiTrs/h3xH4BuPD99PoGmXst1b+IIgPs0EzkebaOSRltw3rtB4c7tvy5+nz/GwxOWVHgqqdmua
+zWz/AM/x1PKy3DTo4he3ha+1z1f4y/8ABQvwP8EfjLP4A1vQdcn+xCA6hq9vHH5Vv5qB1YIWDyKF
+YFmUeoGSK6X9s74HeGPjT8DfEWpXlrbrrui6ZPqek60qASwtFGZdm/vG4BVlORhs4yAR6J8SPgF8
+O/i/e6feeMvCGmeIL2wdXt7m6h/eptbcFLrgsmeShJU85BzXkn7f9v8AEmf4AaxB4BgtJtKeCQeI
+lQt9u+wBQXFuPukFd2/POzO0E1+dYSeHliMKsJenUT95t6N6ar11080j6arGap1Pae9HorHi37AP
+7Snwn+EX7O1jo/ivxvpmia3PqV5ey2NyX8yMPJhM4UjlVB69DX1J4W/a/wDg5438S6b4f0Dx5p2r
+a1qMvk2llarK8krYJOBs6AAkk8ADJIr8OVZSAVI2kcEdMV+pf/BM/wDZ0g8E/Dz/AIWdrNoD4j8T
+xY0/zV+a007PyYz0aYgSE/3fLHGDn6ziHJ8FhoVMdVnLnm9Fpa7+WyPHy7G1a0lRjFJI9c/br+Im
+ieAv2ZfGcWq3RhutdspdG063j5kuLiZCoUD0A3Mx7Kp74B/FgcAegGP8/wCe1fUv/BRr4xyfEz9o
+O80C3nL6H4NQ6bDGD8rXbANdSY9QdkX/AGyPqa53wT+wb8ZPiJ4R0jxPoWjaTcaPq1sl3aSzaqkb
+tE4ypZSPlOO3avayKlRyfL4zxVRRdTXV23Wi+7U4MxdTG4hwoxvy9jzL4K/D21+KvxW8N+FNQ1aH
+QdMv7nF7qM86QiC3RS8hVn4DsFKrn+Jlr9fvFnxJ+E37GXwc065t7WLTvDCzLZWFhoMSzy3cxBJ2
+/MN7YVmZ3bsSTnGfyR+Nv7O3jT4BXek2PjnT7K1l1aOWW1S1u1uQyxlQ27A4wXXGf6V7L/wT3+An
+gb47+LPGemeNtNk1Kw0yxtri0hivZrdYneR1Zv3bLkkKo/D3pZ3hsPjqEcdUrN0IauMbPm1to7rX
+p5a9ysvqTw83h1C1R9X0/M/TL4Q/F3wZ+078Mm17Q4W1HQruSWxu9P1a1AdJFxvhmjOVPDKeCQQw
+5r8wvjZ+y1pekftt6d8KfD0smmeHvEVza3UAQ7msLeUO0yJn+75UxTPQFQc4r9T/AAP8NNB+C3w/
+fw/4B8PW1nZ2qyz2+nLOyC4mbkl5W3NuYgZdskDHYAV+SHxV+JvxO8Oftkf8Jp4q8Pm0+IGm6tbS
+W/h6EmWN4R8kNvCygmRJIyVDqOWdjgHIHzHDnO8ViXgpcsOV8qk1e/2W15dX8up62YKPsqarK7ut
+j9SfFmreCP2NvgNqGq6f4fez8LeHoEI0/SYgZZWZ1jUszEZZmZd0jn1JNYn7M/7VvhP9q/Rtei07
+SbzSr7TNkd/pWqLHIGilDbGVlJV0bawIODkHIwRn1nRbpPHHgy0n1jQJtOj1O0Bu9F1iON5Ig6/N
+DMoLIx5IIyQaxPhr8EvA3wbGsnwR4YsPDratKtxeCzUqJXUEKOSdqjJwq4UZOBya+SVXDujUjXjJ
+173Ur6ed/wAdT1eWfNHka5O1j8pv2/8A4J6D8EPjwtr4Yt49P0XXdOXVU02EYjtJTI8ciRj+FCUD
+hegLMBgACvm2veP23IfiQn7QWry/E6K2j1WWFP7NOnFjZGwVmEYgLfNgMX3bhneW7EV4PX7tlfP9
+Ro+0nzPlWq1v8+p+fY631ifKrK4Vb03/AI+G/wB3/CqlW9N/4+G/3f8ACvT6HEj9bf8Agn98AviL
+8BPBnia08fXiRpqV5FNYaLFe/alsgqESOWHygyEr8q5H7sHOTxx37cPx30r4B/tD/B3xSulDX9Z0
+rTtXa406K4EEjW1wsUURLlWwN6yEcc7Wrx/xZ/wVj8aalp8kHh7wNo2iXLqVF3fXkl55ZI+8qBYw
+SPckV8ZeNfG2vfEfxTqHiTxPqk+ta7fuHuLy4Iy2BhVVRgKijhVUAAcAV+c4HI8ZicdPG5lFRUk0
+4p73XL0eit53v959XiMwoUqCpYZttWs+1nfqffn/AA94tzwfhRc/+D5P/jFfQ3/BP/8Astv2T/BE
+mlmItKtzLemMDd9ra5kMwfuWDHHPYDtivxjr1r4A/tR+Pf2b9QuX8K3sFxpV5IJbvRNSUyWkz4A3
+gAho3wANykZAGQcCu7MuGaM8JKll65ZXT1badk9Nb23ObC5rL2qliXdWt6bf5H1R4w8E/th+Cv2k
+vEOseFJtS1jTdT1Zp7V3vYZNHktC+Io5YJHHk7Y8KdqhsgkFicn9Ddd1iy8P+GdQ1TXJIbbT7O0k
+uL2Rj+7SNELSE56jAPWvzpi/4K36+LLbL8MtOa7x/rE1hxHn/dMJOPxrwD4/ftrfEj9obTX0bVri
+10Hwu7BpNE0gMsdxggjzpGJaQAgHbwuQCVOBXh1MjzHMp0o4ilCko6Nq12tOzevbZHpLMMLh4ycJ
+uTfQ8l+HXg0/E34ieGvCtoDBHr+rQ2KDoYoZZQD9NsZb8q/e6c2PgrwnIYIFttM0mxJSCPhY4oo+
+FHoAq4r8HfhP8Rbn4SfEnw/4zs9Pt9VvNFuGuYbS7dlikcxsnzFeRjfu47gV9UeIf+Cp/jjxJoGp
+6TP4E8PRQX9rLaySJeTllV0Kkjjrg17HEeV43Mq1JUI3hFa6pat+fkjgy3F0MPTm5u0mfGep6zc+
+JtTv9YvJGkvNTuZb2Z2OSzyuZGJ/FjX258HP+CmcHwm+FfhbwafhvPqp0TT4rE3i6wsQnKLjds8k
+4z6ZNfDEEQghjiBLKihcnqcCnY/L0r6vGZdhcfBUsRC8Vtq1+Vjx6OLq4ecp0nufcf8Awur4fft8
+/tD+CNC8feHNR8J6ZHp93ZWHkauD5987xSIrOI1wGSORQvdioHNekftI/wDBNLS5PBmmTfBawFj4
+gs52+2WuoapMTqMLADAkkZlV0YAgcKQWyc4r81kd4pEkjd45EZXSSNijKwOVYEcggjII6GvsL4X/
+APBT74l+CdIt9M8SaTpvjmOBQiX9zI1pesAMDzHUMjn/AGtgJ75NfOY3LMfhJUp5RO0IfYbduvd6
+3v1foeth8Zh68ZRxi1fW3+R9o/sJ/BPx38CvhFd6L481FZru51F7q00yO6N0unQlVHliQ/3mVn2r
+lRu45Jri/ifqPhS+/wCCknwmtL17WTVbLw1eYEm0hLhvMa2VvR9guGUHn5gR1FfPvjb/AIKuePda
+06a28M+D9G8NXEi7RfXVw99JF7qm1Fz/AL2R7V8b6p4u13W/FU3ie/1q+ufEs10L5tXaYi5E4IKy
+K4xtKkDbjAXAAAAxXlYPIMdXxFbFY1qEpxkrLvJW6dF63Z11syw9KEKdH3rNfgfsL+3N4H+J/j34
+NRWHwsu7mHVodQiub21sLz7Jc3VsqsdkUu5cEP5bFdw3BcZ7HM/Ybk+O6eGNctfjRBOtvBJEujT6
+o8Tag4w3miUxk7lB2bWf5uWzkAV8j/Dz/gqb8RvDOl29j4n8O6R4waFdovxK1jcSD1cKrIT7hV+l
+aPjD/grB471bT5rfw34M0Xw9PIu0Xt5cyXrx+6ptjXP1yPY1539iZrHCvAexg43up3V1873/AAvb
+Q6f7Qwjqe253tsbn/BW7WtLn8RfDPSImR9btra+u5wANyW8hiWPPszxyY/3Gr4CrW8XeL9b8feJb
+/wAQ+JNUuNa1y/ffc3102XfAwAMYCqoAAVQAAMAVk1+jZXgnl+Dp4Zu7j1827/qfKYyusTXlVSsm
+FW9N/wCPhv8Ad/wqpVvTf+Phv93/AAr1OhxoqYor37X/ANg743+GvB03iS88IxPawQG5nsra/imv
+Y4wMkmJT8xA6qpY9eKj8C/sM/Gn4ieEbfxJpfhaK1026jEtomqXqWs9yhGVZY25AYHI37c8HpXm/
+2ngeT2nt42va/Mtzt+pYi/LyM8ForQ1zw7qvhnX7zQ9W065sNas5za3GnzRnzkmzjZtGck5GMZzk
+YzkV7/4f/wCCeXx21/SYtQHhey0xJUDrbanqccVwARxuQbtp9mIPrW1fGYbDRUq1RRT2u1r6GVPD
+VqrahBux838DJJAHrVu60jUbGCOe702+s4JMbJrm1kiR8jIwzAA8c8Gt74s/CTxZ8G9bm0Dxvos+
+h6g0BmQOVljni5HmRSKSrr24OQeCBX1R+2D4y+LOvfs7eBbHxv4A0zwx4Yiu7N7DVrPW1vJLp1tJ
+FjDRD7m5CzZPQjFYVcaozoqlaUaj35ktPJfa+WxvDC3hUdS6cfL8z45fSNRis1vJNNvY7JgCt29r
+IsLA9CJCNpz255qpX6MfFuR/+HVPhIFjg2Gjjr2+0r/gK/Oipy3HPHwqScbcs3He97W1DGYZYaUY
+xd7pMKK6v4Z/Cjxd8Y/Eg0HwZoVxrupBPMlWIqkVumcb5ZGIVFzxycnsDXq3jf8AYK+N/gLw9ca1
+e+FYNSs7eMyzpo98l1PGgGS3lDDMB6IGPtXTVxuFo1FSq1YqT6NpMxhha1SHPCDaPn6g1a0XSb7x
+LqllpmkWdxqupX0gitbOzjMss7noqqOSf5c+hr6Hi/4J2/HiXRxqH/CLafExTeLF9XhFz0zjGduf
+bdTr4zDYWyr1FG+12kKnhq1ZNwi3Y+bfelrS1zwzq/hjxFc6DrOm3Ok63azrbz2F4hjlikJAAYHs
+cgg9CDkEivY779h342af4s0vw3L4N8zVdRhkuIjb30MkEUUZUM80obbGMsAATljnAOKdTF4ejZ1K
+iV1dXaV0t2hQw9Wo2oxbtueFY/zikFez/ET9jn4vfDHUdHs9U8IS6g2r3Is7KXRJReRyTkEiJiuD
+G2FY5cAYUnPBrR8ffsOfGf4a+ELnxLrHheGbTLSIz3i6bfR3U9rGBlneNeSqgEkruwAT0BNYrMcE
++W1aPvbe8tTV4PEK/uPTyPCKKTIOMHIPelr0DiCrem/8fDf7v+FVKt6b/wAfDf7v+FHQaP0O/wCC
+XvxP8X+Otd+Iun+JvE2q+I4Y47K9jbVLp53jmd5FcoWJ2hgFyBx8o4rhfgj+0N4+8af8FAxbah4o
+1KXRL7WNT0s6KbhvsUdrEswiRYvugqYlbdjcWySeTXO/8E5fjT4I+CvifxzdeOPEdr4dg1C0s47V
+7pXPnMkkjOBtU9AR19RXnPwF+IPhzwh+2VYeMdY1eCx8Lxa9qd42pyBjGIZBceW+ACcNvTHH8Qr4
+OrgIvF4+So6OmuX3dL8utvO9ttbn1EMQ/Y4f39ebXXz6nqP7ZD+I9L/b90688D6cmoeMTbaZNptu
+1uswluvLdVYoxwdoUHcxwuwNkbc16h48+H/xmvPEOkeMPi/+0V4f+EP2dImh0nQr11VQrZLCEuqS
+O5zuJ8wHp0GK84+If7T/AIK0L9vvRfino98vifwhbaZDp9zd2CMWjDwyxSOisASybw2McgkDk11v
+7Qdn+y38TviQPi9rPxWuNUYwQef4W0YebNqDQACONFZQ8IYKoZflGcnK5JrkarQjhac6TS9mlzez
+55X/AJdU1H5r9TrXI5VZRnf3tr2XqXf+CuqRyeEfhnchB5puNRVZMYJQwxEj6EhTj2rS/wCCh3/J
+ofwp/wCv3T//AE3vXm//AAUN+OngD9oH4a+BLvwb4mtNQ1WwuJpLrRdrrcwJPbjggrt+RkCMA3Uj
+Gatfto/Hv4e/E/8AZr+Hvhrwt4qs9a13TLuze7sYFkDwqlm8bkllAwHwOvcVGX4avCll8ZQacZzv
+o9Lt7meKqQbrtSWqXU7f4t8/8EqvCH/Xjo//AKUrX5019s/Ef48fD/W/+Cevhv4eWPimzufGlrZ6
+ZHNoyrJ5qNHOrSKSV2/KuSeexrzzwhZ/s+t+yRq82tzaePjWLe8Nmjz3In80SN9nART5Wdm3GeOe
+fb2sqqTwNCq6lKT5q0krJ7O3vdPd8zixlNYmpFRklaK6n0b+yvdN8GP+Ceni/wCIHh+KJfE1zFqW
+om6ZAzebE5gg3A/eWMKGCnjOfU18tfCL9un4rfCDUNVvZ9en8bR6hHg23ie7lnigm3A+emCCpxlS
+oIUgjjivS/2Kv2rPBvgfwHrfwm+KaGLwfqjTtb6hJG8kKLOu2e3nC5ZVY5ZXA4LNnGAa9N+H2o/s
+nfslya74s0DxtJ481e8tGtbbTfPXUJRCWVjDGixqo3FVDSSHOFAz1z5k4rC18VTxWGdWVSV42V01
+0XNra34HXFurTpSpVVFRWv8AwxR/4Ji6Pp3xC+JPxU+Jl5Y2FtrHnQxW1vYQ7Lez+1NJLOYFJOwM
+VCjByBuGea+d/E37b3xe/wCFxX/i238W39ta22pSCHw6r404WySlRbtDjByi4Ln5sknIOK2f2cf2
+x5PhJ8evFXjDWtIEfhnxhMW1PTNIjUCww5MDwIMBvKBKkcFgWPJHPul74N/Yx8QePpPiXN8QYYYJ
+rr+05/DZumS2kud28s1sYvOALZYxA7ST0xwdasFhMfVrYzDucZxio2jz2srOPld9fn1Jg3WoQhQq
+JNN36X8yT/gqH4Q0ue3+FfjmG1W21i41FNMnfGHmt2VZkVvUxsGx/vt612n/AAVA+Knir4eeCPBm
+l+GdbvdBTXL+4+3XGnzNDPJHDEhWMSKdyqWfJ2kZwOetfJn7ZH7VkP7SPxD0FdIgnsvBfh64DWhu
+02TXMrOnm3Dr1VQqhUU5IUEnBbA9J/4KQfHv4f8Axs0v4fxeBvE9p4jk026vZLxbVXHkq8cQQnco
+6lT09K5MLl2IhPLqeJptqPtG7q6jfWKfb/PQ0rYmm44iVKVn7v8AwbHvOmfGTxfo3/BNGPx+mtXE
+/i9NAMS6tOd8283Rt1lJP3nVSDuOeRk5rC/4JmePvEfxF+H3xEsPFWuah4khsb+NYG1a4a5dFmgc
+yJuck7SVzjOMk+teUP8AH34et/wTbX4ajxTZ/wDCc/2asH9i7X83f9v83bnbt+583Wov+CdHx5+H
+/wAGPDnxAtfG3iiz8Oz6ne20lol0shMqrC6sRtU9CwFc1bL5rLsby0Xzuq+X3dbXVrabb7aG0cQn
+iaKc9OTXXqfEnlrC0iKMKkjIo9AGIFFLIQ0spByGkdgfUFiR/Okr9SR8XLdhVvTf+Phv93/CqlW9
+N/4+G/3f8KOgkVKDzRRTEGTR0NFFABxg8UfyoooAM0E5oopAA7elA4oopgFHfPf1oooAO4NJS0UA
+HY0UUUAH5/jRRRSAKt6b/r2/3f6iqlfQH7HX7N93+0X4x12z3/ZNM0ywEs14ykoszSKI4zjuyiUj
+/cNc2KxFPCUZVqrtFbm1GnKtNQgtWeDfYZ/7n6ij7DP/AHP1FFFac7FyIPsM/wDc/UUfYZ/7n6ii
+ijnYciD7DP8A3P1FH2Gf+5+oooo52HIg+wz/ANz9RR9hn/ufqKKKOdhyIPsM/wDc/UUfYZ/7n6ii
+ijnYciD7DP8A3P1FH2Gf+5+oooo52HIg+wz/ANz9RR9hn/ufqKKKOdhyIPsM/wDc/UUfYZ/7n6ii
+ijnYciD7DP8A3P1FH2Gf+5+oooo52HIj3P4EfsXfEP483Ec2mw2elaCGAn1a8uUKxg+kSsXZsZwM
+AepFfrL8B/gR4d/Z88A23hnw7G0g3edeX0wAlvJyAGkbHToAFHAAA9SSivxbiHNcTiq8sNN2hF7L
+r69z7jLcHSo01VivefU//9k=
+------=_NextPart_000_0018_01D0B859.4A05D240--
+
diff --git a/test/functional/messages/badboundary.eml b/test/functional/messages/badboundary.eml
new file mode 100644
index 0000000..a298ee9
--- /dev/null
+++ b/test/functional/messages/badboundary.eml
@@ -0,0 +1,17 @@
+Content-Type: multipart/mixed; boundary="===============3209040583106423479=="
+MIME-Version: 1.0
+From: test@example.org
+To: test@example.org
+Date: Thu, 26 Sep 2019 15:52:32 -0000
+Subject: Test message
+Message-ID: <156951315267.43830.14912869070271436609@example.org>
+
+--===============3209040583106423479==
+Content-Type: image/gif
+MIME-Version: 1.0
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="Transparent.gif"
+
+R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7
+
+--===============3209040583106423479== \ No newline at end of file
diff --git a/test/functional/messages/broken_richtext.eml b/test/functional/messages/broken_richtext.eml
new file mode 100644
index 0000000..e4786c1
--- /dev/null
+++ b/test/functional/messages/broken_richtext.eml
@@ -0,0 +1,27 @@
+From: user@example.com
+Message-ID: <XXX@yyy>
+MIME-Version: 1.0
+To: user@example.com
+Subject: Hi
+Content-Type: multipart/mixed; boundary=
+ "xxx"
+
+
+--xxx
+Content-Type: text/plain
+Content-Transfer-Encoding: 7bit
+
+Hi
+
+--xxx
+Content-Type: text/richtext
+Content-Description: eicar.zip
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="eicar.zip"
+
+UEsDBAoAAAAAAOCYuCg8z1FoRAAAAEQAAAAJAAAAZWljYXIuY29tWDVPIVAlQEFQ
+WzRcUFpYNTQoUF4pN0NDKTd9JEVJQ0FSLVNUQU5EQVJELUFOVElWSVJVUy1URVNU
+LUZJTEUhJEgrSCpQSwECFAAKAAAAAADgmLgoPM9RaEQAAABEAAAACQAAAAAAAAAB
+ACAA/4EAAAAAZWljYXIuY29tUEsFBgAAAAABAAEANwAAAGsAAAAAAA==
+
+--xxx--
diff --git a/test/functional/messages/btc.eml b/test/functional/messages/btc.eml
new file mode 100644
index 0000000..840a5cb
--- /dev/null
+++ b/test/functional/messages/btc.eml
@@ -0,0 +1,22 @@
+Received: from localhost by mail-lf1-f54.gulugulu.com with SMTP id j14so14439709lfg.9
+ for <test@test.ru>; Mon, 27 Apr 2020 09:54:21 -0700 (PDT)
+From: test@test.com
+Date: Mon, 27 Apr 2020 19:54:10 +0300
+Message-ID: <CA+1S=h4aGimA6vSBJF=t1F+5z-Mua5+Cimf+NU_NDWJk8ZNOcw@mail.gmail.com>
+Subject: Fwd:
+To: <test@test.ru>
+Content-Type: multipart/alternative; boundary="00000000000004de7805a4489190"
+
+--0000000000004bee6805a4484c02
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+1EHbEvWFkR3oY93EKym1bweaWUW2PKe6v9
+
+--0000000000004bee6805a4484c02
+Content-Type: text/html; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+1EHbEvWFkR3oY93EKym1bweaWUW2PKe6v9
+
+--0000000000004bee6805a4484c02-- \ No newline at end of file
diff --git a/test/functional/messages/btc2.eml b/test/functional/messages/btc2.eml
new file mode 100644
index 0000000..bbb9e82
--- /dev/null
+++ b/test/functional/messages/btc2.eml
@@ -0,0 +1,22 @@
+Received: by mail-lf1-f54.gulugulu.com with SMTP id j14so14439709lfg.9
+ for <test@test.ru>; Mon, 27 Apr 2020 09:54:21 -0700 (PDT)
+From: test@test.com
+Date: Mon, 27 Apr 2020 19:54:10 +0300
+Message-ID: <CA+1S=h4aGimA6vSBJF=t1F+5z-Mua5+Cimf+NU_NDWJk8ZNOcw@mail.gmail.com>
+Subject: Fwd:
+To: <test@test.ru>
+Content-Type: multipart/alternative; boundary="00000000000004de7805a4489190"
+
+--0000000000004bee6805a4484c02
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+qq5llzv84punzrx2vlv33m5d5jm3tnr8aqqz3kwel3 - bech32
+
+--0000000000004bee6805a4484c02
+Content-Type: text/html; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+qq5llzv84punzrx2vlv33m5d5jm3tnr8aqqz3kwel3 - bech32
+
+--0000000000004bee6805a4484c02-- \ No newline at end of file
diff --git a/test/functional/messages/btc3.eml b/test/functional/messages/btc3.eml
new file mode 100644
index 0000000..d55c9d6
--- /dev/null
+++ b/test/functional/messages/btc3.eml
@@ -0,0 +1,22 @@
+Received: by mail-lf1-f54.gulugulu.com with SMTP id j14so14439709lfg.9
+ for <test@test.ru>; Mon, 27 Apr 2020 09:54:21 -0700 (PDT)
+From: test@test.com
+Date: Mon, 27 Apr 2020 19:54:10 +0300
+Message-ID: <CA+1S=h4aGimA6vSBJF=t1F+5z-Mua5+Cimf+NU_NDWJk8ZNOcw@mail.gmail.com>
+Subject: Fwd:
+To: <test@test.ru>
+Content-Type: multipart/alternative; boundary="00000000000004de7805a4489190"
+
+--0000000000004bee6805a4484c02
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+bitcoincash:qztslqhavnjcgth9zwu6dw0jjcfy4zahfy7vf0smwp
+
+--0000000000004bee6805a4484c02
+Content-Type: text/html; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+bitcoincash:qztslqhavnjcgth9zwu6dw0jjcfy4zahfy7vf0smwp
+
+--0000000000004bee6805a4484c02--
diff --git a/test/functional/messages/btc4.eml b/test/functional/messages/btc4.eml
new file mode 100644
index 0000000..523ce59
--- /dev/null
+++ b/test/functional/messages/btc4.eml
@@ -0,0 +1,26 @@
+Received: by mail-lf1-f54.gulugulu.com with SMTP id j14so14439709lfg.9
+ for <test@test.ru>; Mon, 27 Apr 2020 09:54:21 -0700 (PDT)
+From: test@test.com
+Date: Mon, 27 Apr 2020 19:54:10 +0300
+Message-ID: <CA+1S=h4aGimA6vSBJF=t1F+5z-Mua5+Cimf+NU_NDWJk8ZNOcw@mail.gmail.com>
+Subject: Fwd:
+To: <test@test.ru>
+Content-Type: multipart/alternative; boundary="00000000000004de7805a4489190"
+
+--0000000000004bee6805a4484c02
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq
+bitcoincash:qztslqhavnjcgth9zwu6dw0jjcfy4zahfy7vf0smwp
+1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2
+
+--0000000000004bee6805a4484c02
+Content-Type: text/html; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq
+bitcoincash:qztslqhavnjcgth9zwu6dw0jjcfy4zahfy7vf0smwp
+1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2
+
+--0000000000004bee6805a4484c02--
diff --git a/test/functional/messages/content_url.eml b/test/functional/messages/content_url.eml
new file mode 100644
index 0000000..c559d2f
--- /dev/null
+++ b/test/functional/messages/content_url.eml
@@ -0,0 +1,202 @@
+Subject: lo
+Content-Type: multipart/mixed; boundary="=_BFgoT7oi_t2JDLtlkNV7Zzc"
+MIME-Version: 1.0
+
+This message is in MIME format.
+
+--=_BFgoT7oi_t2JDLtlkNV7Zzc
+Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes
+Content-Disposition: inline
+
+
+lo http://www.example.org bye
+
+--=_BFgoT7oi_t2JDLtlkNV7Zzc
+Content-Type: application/pdf; name=spam3.pdf
+Content-Disposition: attachment; size=10322; filename=spam3.pdf
+Content-Transfer-Encoding: base64
+
+JVBERi0xLjUKJcOkw7zDtsOfCjIgMCBvYmoKPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURl
+Y29kZT4+CnN0cmVhbQp4nI2RTUsDMRCG7/kVcxY2zpvNJ4RAq13BW3XBQ/HkF4gV7KV/39lsrV1w
+qYTMMExmeN43rEF79UVMrNlEcslpExxFCx09aPeiHi7oc3whZ/emlr1yXkcKodUpJuqf6bIDwVD/
+usmMYjKb0oTMbXnsb9WqV+ufee0YyR+2yLyZ22LZDXtq8BxK4zLHoaghleYkept5wcuCzFd8PaTV
+ONbVsdpYgIdkpHXChEoFmpA8bRXrQPsJ8d2NYmogbm2FEfK8Fh90/yvuIAvJajunqx0BOwEEKr85
+Z5NBEpz/+FRND0eXhgAj0o/3L+3T9WfFe0h7VruJmNEu0tHCwsl/9O8jxJq+AQdrhoQKZW5kc3Ry
+ZWFtCmVuZG9iagoKMyAwIG9iagoyNjQKZW5kb2JqCgo3IDAgb2JqCjw8L0xlbmd0aCA4IDAgUi9G
+aWx0ZXIvRmxhdGVEZWNvZGUvTGVuZ3RoMSAxMjYxNj4+CnN0cmVhbQp4nOV6a3hb1bXgXufoadmW
+ji3JUhRJR1GcxPFDtuU8DEl8ktiy85ZfwUpILNmSLYNtKZLiNEAaU8KjDikGcml4fIOnk8vQXmaQ
+SaApl1vcDtDpcLnkzg20lHKbuYR27pSUlKZMhxJ51t46suU0pd/MN//myOectddrr73W2mvvLTmV
+OBghhWSc8ETqHwnFu9evrSeE/D0hUNI/lhLX7zLdjPBFQrj/OhAfHHniu7deJURxlhD12cHhwwPf
+/eUHKUIKo4QYz0UjofBU2UMeQlwi6lgdRUQgc1iN7R5sL42OpL7ya6ViI7bvwnbHcKw/ROon67B9
+Hts1I6GvxFcqtnOELMEmEUdDI5ElTao2bKM+3Vg8lkyFydJZQqpPUXo8EYlvf6LvdWyfI4SfRBzg
+h16FCKpom+MVSpVaoy3QFRYV6w1CSanRRP4/upQniIm0KdcTPYmz54KLf45YyeOEzH5MW/PPzPbZ
+z/9fWqHJvk6RZ8hZcoK8R/bJBB/xkyFyEDH51w/IPyKWXn6yh3yHTPwZtc+Rc0jP8gXJQ3QkN7z8
+5JvkDPnRgl78ZITciba8SN6DOvJjTJUY+RQ05G7yOmr9FHE7bqSKK8bHAAMH8rDvkye542Qrdwkb
+j1MK5+EM5DXyFOxHzSkc54m5Ea/7E6X3kyP47CRRMoYwu5Trv/gZ0c7+Dkd1hGwlXyMbyXCexCvw
+NF+A8esiT6NPf8BwnhxR3cbfxr3EcdcexcbDZBDvEODYuRP8xj/jof/ji+8mRVDBlxPtjahcA9Fn
+PufqZ6/yS0kB6Z69ksPNbpv9HR/KjCp6FYuV6xVvflkfqocVIyhNZj/K3JkJK3cqn8FoPUuI1Lp3
+T6Cnu6uzo92/a+eO7du2bmlr9bU0b960UWrasH7dzTc1rl2zelVdraemumrF8mXlS91LXE6LUTDo
+i4t0BVqNWqVU8ByQKjENwZY0Xy4KvpC7xR1qq64SWyzR5uqqFrcvmBZDYhpfimXutjaGcofSYlBM
+L8NXKA8dTEvIOXAdp5TllOY4wSCuI+toF24x/VazWzwHe9p7ED7R7A6I6csM3sFgxTLWKMKGy4US
+zCpqrdiS9o1FJ1qCaCNM6wo2uzdHCqqryHSBDkEdQukV7vg0rNgADOBWtNw0zRFNEe0WR9oSCqf9
+7T0tzTaXK1BdtSVd7G5mJLKZqUyrNqfVTKU4RE0nx8XpqpmJB88ZSF+wsjDsDodu7UnzIZSd4Fsm
+Ju5PC5XpCndzuuKOSxYceSRd5W5uSVdSrds65vrZNt8lpJXlBrc48XuCw3Ff/nghJiRjVOWG3xMK
+prnNaejocdHL5kNfT0z43KJvIjgROjc73ucWDe6J6cLCiXgLupv4e1DFudmXj9vSvgcDaUMwCjcF
+5KH7OralS9v39qS5cp8YDSEG/5rcrrU2lzDH4/9zZIJuQeegh10u6obj5yTSh430eHtPti2SPtsL
+RPJUBtJckFJmchRTN6WM5yhz4kE3xnZbZ89EWlG+JexuQY8fD6XH+zC7bqOBcRvSxZ/ZXO6JEkFs
+9AQYr4hWbQkPiWnlMnQSSuULYN5QkQkDaxR/ln1dtmEHy4QSsdGNaqieFndLUP4bi1pQgYiObqvM
+JkJXT1pqRkAKyRFrma71oEQoiAEbambBTHvc8bTRvWkuutSslqHOHiYii6WNm9Mk2C9LpT0tbF6J
+LRPB5qwJVJe7ved7xDt7cbpBtJ3xkgYSaKbM5s2YZctaJnrCA2ln0BbGeTcg9thcaSmAEQ64eyIB
+mnbooYqLNpYcAZYrXT3bOt3b2vf0rJUNyRKoOkV5y3Vq3D22rBpMwLSmXCP2cDY+gIwGRIg+BNyb
+1uEzrS7X4G1AhzMsTdxN68QesJEcN5qRrhBbIs0yH20vUKqk6bS5LadNRZuoZ3ObzRVwZa/qKg7J
+otwxSmioU9tyJCxTSNBgfm5uYyjqSwtNerHHHXEH3FExLfl76Nioe5iXZWcwn8ux6lrQynMWuom4
+kJxrUGemfZW2fOemW1l7rtl2HXlLjixOaNzbOieocreskKDlW9KEprC0VrCxWkAntBtrr2jAKc0m
+9MS0JNHJHL2JKnFvCU+4O3vWMW6sJ0dsd9C+Ssg22Na1qboKS9umaTc80D4twQOde3q+Z8B94QNd
+PS9wwG0ObgpML0Vaz/dEXDQYlqNYiqQNkTaopg5saBi/7XsSIeOMqmAI1u4/B4ThNDkckP5zXBZn
+yHa0jHUkEQ4piixFynErEKfJ4sYZjl3ThLpMKlBKGkkrFXJFnG0aKOoFxLyM+1gtkDOFUAS2aZTq
+YOhzMD6tlWxZjnHkkLIWPtA933X3np4zhbg629gTO9pEL0wXSxSDjctKiximiXJXIDoRDNDJRswY
+GvyDNLg3YJjcG9AQVWG6wB3ZlNa5N1F8E8U3ZfEqildjioIZUHwcY+9PA82AvT0unJLioh/bJgyX
+aaQCWFQmDB9Vo8fwHKGI0j0orJV+xql1OsEAhcWFvYFiXqvrDRA1FPNqtVbLBwPakrMCnBbgpADH
+BEgJMCDAbgF8AiwTwCyASoCrAvxSgAsCvCHASwI8I8AYY+vKsb0rwGsC5OuZY2gWoF4AEAUwCkAE
+aERll5gyZAwL0JAjcFcEuCjAeQFmBIgLIAlQK1A5Qx4+LcAUo/oZwz75OpC7ehPytR+xvfvmrwPz
+V2L+Ik2VAvFaPL3791USC2sIJdAoNHrrakEtuOrXlHp516o1wvJVLu7Q26B5J7xhY8kfP3ruOW6d
+UrPoj1EjlGWOrrIp/s2iVXQrRnbOfsy/wL+OOykzeVm6W1DqiJKUWTTF/oDGwBn9Ac4sWoBY4KIF
+/BaotYDBAldY87wFZiyQtsCUBSYtMG6BuAWCFpAskBW5+WmG8jNULcMaGCFffopJZsXwue/6Qe/f
+t+86rzAvsMHjyBsFr0DHbnAtWbaqYbW33qxuWOZeojIZzd761fwLmbYLP/3pz9/92dmv3nfPwUN3
+HxuH9zNC5re/+eJ//u6nP3z54od/9xrdTAIx4q71IzwP2dELh4nRaC0qLtZatQ6nfZE/YCdGbJRZ
+/YHCMlMpxymVQkdAaZhywkUnzDjB4ATihEZsTDoh7oSgE/xOkJxQ6wTRCU5GRtJ4joqk80wy7YSp
+PHx+7OeDzlxwIBt6wWtp8lLA6833gOwFo3oDoAtMQpmJJkGDe4laBjEfwKdu+27THXclMrcfeWb/
+PUcz4UMPQj3/WbSmYt037r/2mLW62srtf85+rZRCSs5C56Vx9mOuWnE3ZkertLyguFhdyvNlFkWh
+rtAf0Kp1evSb0B4g5qdZJjRZwGOho8glapPXm8vRksb6emqkEgMluFc1gdfkNbkFGqc1pmKAncHe
+O49Emn7yk5trb+p0HzMmBrlHq5e/807XtaMbNxk2WpwsX12Z7Xwa87WMuMi9UrtDrygpKbMUlBUs
+cZeVGEv8AaOtSPQHisx2m9rWHlCoDTzxB3i95IZxNxA3NNa64aIbZlg76AYpD25yw7z792dHQWR/
+W5ryJ5zsdC+OpxRdW5Z1u5FzL1lutgNzOdA8VAs4vFO3g4ZbeWLL2dd/8uaBAdXpjHSICx85enBn
+4LYv+AFr9ZqlVZ//j08yn5vbKjIWj8fC75z5W9c1QaDj/dbsh/Ao/wOiwxGnpcOFhJSoVBarSf/k
+XpOB1z65ly+9YIXXrHDWCqetcMwKKSuErSBZodYKS61gtILCikWMMY3nyF1WaLbC+TxJ0QqcwQrE
+ClesMGWFSSvErRC0gp8pm8vM3vzkXDA582YleoaGutyocuempZAHw6Mt3oYWn7fe56v3+loavC23
+1rW11SHInfP6fF5s0DlJMPcalT7CkyVSCQ9KBacgpUQJ66aUEFTCgQP7sc8mFo662jI3eMNh+Emf
+UbkHZbnZj5Xf55/Def07yVGg1BuVRpOZ0xQUtXFFRUZ9gVKt9AcEtb5Ypzs3+wfpHiToeCAKc5cZ
+ms2w1AwGMyjMcNUMZ80wZYaTZjhmhhSj1jKG0wwZNgOKEDO62AyXWEMyA4csItOBlHEzBM3gZ4Qs
+/rwZXmVakRQ3Qy9DLij81Lu98y5euBYQnFZ0YnlzpQBXBG9+OVS6deAGTEyT2gZeWxZS2Hb+94+2
+Zv42Bq8+9YsPu/7lnSdgIGrkhq+d5O/AGW+7di8XufZN7m6ETawm3o+5d5i8g7lnkXREpSosyiYc
+yQV5YYThcEsDjarXJ0fTh/nrx/XFh/PVRBaTE9IeK4B+kcakN9kdVpyXeqvTyhXyVmthSYnZHygx
+FCrbA4XmGQekHTDlgEkHjDsg7oCgA/wOIA7YgC/JAbUOEB1gcMAVxodMOd/s/9N8JI3ZBVN2UK5Y
+mowOnLWrafHBMQg4BlEwAS4droZloFh/dHD1ydrav979/pv/8CoMZb4ZjcEjt8J7JROP+0t0a501
+H4Pys08zAx3w1LOnzzxO/VWCa8ivMOds5IIUUelKigqVi0qNVoXNZjXxpUpBXaQrWGy3S6Fwm8Ju
+tHNL7Q32ZnvYfsx+2v6a/YL9kl1L8UsRSVFnEXnJftVe0KhAHGU7iVjVUsZNCUr7udmZM3ZXG31L
+K/VlbZIdOGKvtXNa3mosKdIX+QPKwkWlCq1Zr7epFDqtDhd0rYnMLyHekrLGLAzoot59lZW08h1I
+oLPK5laWrMPKaVXTZYubNpdc7KUF/leZ3Ufey9yd+Y8jsCpzJQbfOfLi+buhYzjzB1hVXV1dBjsy
+06bqagOcgodpsmU+BQNNtMx3MjW0zrVhnhzAOmcj5WREahI05eUKsbDQquCXLytfUrCkPWAxCcJi
+TBnBKWDKYHXUFJjVCn9AbSImf4AYxpdD73KQlgMC++Y3CbR+lzTK8SeNcgHPjY3mgbxeLsdMFho2
+QBOsopVbD+5Vq0FdjGlCkwT+8YmHD2YypYnp326ZOnWidWu4c8nabwG5577eh5r76/kffPVr1+61
+Vu9PgGX/nRt5xaOhWz0H33JnHArl/tG000Lzo5IQlRXnQh28IM0KharFi11kxYrqalch762vq/EH
+6vQrXIuFwurKan/Aqa80WVUqrdbYEdAalmNq8eUdAd4w5oXdXljthaVeMHtB5YXPvHDJCxe88IYX
+TnvhMS/0eQH8Xmj2Qi3jM3pB4YXolRzjWS+kvCB5oYGRkXbVC+97YcYLaabjmBfCXllFlseQYzvv
+hde88DdemGRst3vhZi+IuT7WZjuY8kLQC125PoxM8hKTPOmFcexeqsyj25jsJWYAl2YMcdY99qr3
+gkae07033BXnr0EHDtyAITEvnsc0v52a21bliqm8s8rPf0wTtsSzhd4Kc9urYk49v9tadt12a9uz
+UstB+463m68cznQ/OLWopaXJJJzIbDre3d1zz4nM7kOHoJQPVt7U0Fi5KfNreQPW8xwuUorVG3PN
+zoD9mpWCvGitzu5VK3DCGHCvqiXfkuJKXYFWhXsxQpQ8Lmm86YIOXtPBWR2c1sFJHRzTQUoHYR0s
+1YFRBwodLlSMY1IHXFwHQR34dSDpYEYHaR1MsaZBB0QHV1gT+fLZbrhK5ccgt/Nj5RarxrxDYv82
+Y52aAp8vb39Jzx87cU6YSVBah6cPs9KMpw897rQ1BrORN7YHeDOeGDbknyCusLND9uCA+Oct0Es3
+nXMW7Z+zYcGiWI51XqRlvr5McC9fhjDbe/I7657bk1nzr+/dP7WmsjOVufrv/uaR4calFfDbX19z
+Zj5/xpOJXnjRRf3ehfuRf0JbV5CA1OBSGxcV4faiYmWRiy8rc/gDtjIDj/VVzZvHV0J8JQRXgn8l
+iCvh+ZXQuxJ2rYSc32j9ze4rsSI1zi9LbAtPF9Tlq7xlaNuqBg/UcGx1LTNRk9m5pszBc/80/R98
+366trtv2lR8+HojcWv/tycEnPStXJdq7d+x8dA/uYzUPTtpLfnVP8zN3NNhdzf2+ux5yvjXi8Tc3
+7lxUX7N5N/2SPnv2U3ag75eQGjIo3WyvENxIKBYUujKVTuWp5YurFlUtSgaqqkSjcVkyYFRrxWRA
+a5VqgdTCxVqYqoVgLWAzmxMLltwFo5NXk3o2QrUDq6qKBWNNDWR3D3z+sS0f/vrtexNH3v2XTNf4
+yC0jb868/NbxI/cfS4197d6jFcMjtw2GR+JD/Gejj69YeWb8+zPw1Xufr1hxKvbMSy9+8dbU5Nf/
+/bfvO87XTnztq984fvhOGkMspAr6u5eVvCbtKFarVXpiMJhAVaTRqEz8IptkC9q4KRsQm4iw3zZj
+u2hTrTfY0jbOYKtFRNB23nbFpiIIxm2TiJ9BhFrD2+haHNjXxt47urPvVY3sLS2prGuz4gbHSgxF
+GlMprld6UqxW8LpSkwp4oAeUuZSdW5DRkVCJVy9dj+mdvyTjmkZzWptbgLNHjvzy0wN398H2g5mr
+0DOQObo7k7kznDl66DjUwevwtA2X5Mxvrv2mDGci/NX9mU/npiSuxYTlxQnc99nJuNReolbbSZm9
+zOFcpPUHFplVJSVGI98eMBr0hXQgEjvA0gPwJDsKX8ydbLMn3uxxdyYHi07Iq9SJvGPWfD2eP2Bh
+urBcYOeo1XSwglG9OnfWWracy4zfd3NqUdfBibuuHf86eFThUzNv/bd3dr+9E66cO2sqvFZm+Kmi
+xlKdSa+e3PmvH1/L/K9lTlZDrRkf/wfcqy2GD+laXGoptDisXAkUKUp0dkupUFiMYzMV64kaKxBZ
+nHLAgAN2O4BzwKcOuOSACw54wwEvOeDrDkjlNqibHFDvgKUOwK2lygHRzxzwrgN+6IAXHHDaAY84
+4BjjRl1dDvAx7iUOKHGAwgFXHfBLxv+aA8464Jkcf8IBfQ7ocECzAxocsMwB5hz/nB1nF+q/jj9r
+zdrPGPecNSdz2qVb8u1Zyuyhe23cW190wPmcQSdz2sNsrFntVxkDl92xz23XdzlA75hbt69buA8k
+5r70utFXPNcv4wu/ALsBf7ai1s+d1FklxUnjnS/9pUvkiroB1izYuUIx96Od2z0u55qGni1rMk8E
+4ezJzGePwv6+zMMbg6mMr+TNYNm6sVN8nJ2QYtyjdNP6xd6/ntyaXYvZfNEmf6zq1a/7PefM/pb9
+n5vP/8P8L5UZn8qqpL/wauZQKKd2ZVrILfmYBVepqhEr1YfkZgUhO7ELrFrEyDUSI75d+P4W3kT5
+I8Ih7X7+BPEjvgTfbShXibQKhHfydtJF5WnVw4qXvV4BCT9/zxm5B/kCfgX/X/iPFX1KokyoHlN9
+oe5Qv6xp0LyhXVtwq2yVmdTjWsF+uCUG4iGI5/4T/wbiKNUBo3O2754bByDnbhnmiJoMyDCP+/sR
+GVYgzwMyrCTF5JQMq4iePCPDanIHeVGGNcQIHhnWkmLYLMMFMArtMqwji7lX5/7Lo4Z7X4aLyCpe
+K8PFZBG/gVqvoL9OP8f3yDAQUaGQYY4UK5bKME9WK+plWIE8URlWksWKB2RYRRyK0zKsJlcVP5Rh
+DVmhfEmGtWSx8p9luID7ufJzGdaRtZp3ZbiQ3KotluEicpv2NhkuJg3aC81Dg0OpoTsiYTEcSoXE
+/lj8cGJoMJoSV/RXiPW1dbViayw2OBwRN8cS8VgilBqKjdYUbL6erV7sQBVtoVSVuGW0v2b7UF8k
+yyt2RhJDAx2RwYPDocTGZH9kNBxJiNXi9RzXt3dHEknaqK+pra1ZNU+9nnkoKYbEVCIUjoyEEreL
+sYGFhoiJyOBQMhVJIHJoVOyu6awR/aFUZDQlhkbDYtec4K6BgaH+CEP2RxKpEDLHUlE09baDiaFk
+eKif9pasmRtBnjs6U5GxiLgjlEpFkrHRTaEk9oWWdQ2NxpJV4qHoUH9UPBRKiuFIcmhwFIl9h8WF
+MiJSQziW0dHYGKoci1Sh3QOJSDI6NDooJumQZWkxFQ2l6KBHIqnEUH9oePgwxmwkjlJ9GKRDQ6ko
+djwSSYo7I4fEjthIaPQ7NVlT0DcD6FRxaCSeiI0xG6uT/YlIZBQ7C4VDfUPDQynUFg0lQv3oMXTb
+UH+SeQQdIcZDo9UtBxOxeAQtvaV1+zwjGpj1ZjI2PIY9U+7RSCRMe0SzxyLDKIQdD8dit9PxDMQS
+aGg4Fa3Os3wgNppC0ZgYCodx4OitWP/BERondHMqZ1yoPxFDWnw4lEItI8maaCoVv8njOXToUE1I
+Dk0/RqYGNXu+jJY6HI/I8UhQLSPD2zH8ozR0B1l86SA6t2wXd8XRPz40TpQZqsRcatbV1MldoBuH
+4qlkTXJouCaWGPTs8m0nzWSIDOKdwvsOEiFhIuIdwnYIoX4SI3FymCQYVxSxIp4A+vEUJmJlrCV1
+eIukFbliSB9GeZFsRjiBUvQZYnpjZBT32AWM8uXa6hHqkK1oY9JVCG1B+X7UsB3l+pCar1cknQwz
+hHWWSg6Sg2hHCDEbSRKlIsgTZhwiqcb7L+n4S/TdDErOUerRrlr81JBVN5T9S5qHUJfIfJ1iFGrr
+CLP/dsTFUO7LPCIiX4TFL4mUCGuFmVaquxs5OhmXn0lSX6RYb6OMq+sGPe7CHgdQvp/FMsfZz3TT
+nMhqjiEclb16G3o8wSwIM7nc2JLY85/G4MbZ0cmsG2N97mB42k4y2iZsJ+VxZX3WxayIIZb64hBa
+QvuNMjjE/Blm0jTLRmXJPsw78Uv7EWXZkByXUdbHmGwllamS/T3AnknW7yj2ITL7slFe2LfI/BRi
+Xs9GegSpKcbbj/hh/ByW59kIeiXbV588kw6xeRmVRzzC9Iq4o4kghWZFjMVt1LWExXjeK9m8GZAz
+VWSycYRjbBQ5P1az2NCRRJilFAqxud+HEsOs76xtUZYdIRbbiBzrFBtBzl9heaTU6jjDVJMWlhd0
+xkdkn96ClWL7DTVmPZifmzQmw8zeZJ7uUWZteG6MWW9TrmG5p+yIh1lFun0uPgMs37IeDTNt1X/G
+5wPMNym51xizKIyfbMSzuRVD2YMsHtn5lM3m1J94LsT8G5Pl4qwupWRbRtj8iLIMjJObcG/pQevo
+p4blYf6s6ZfnTI1ss+f/Wo7aFWcezJ8fiTlbRtDG7fLsH52bdQfz5m8uEp1Yg7azehGX88cne068
+TgOdNddXzTrsr+66UWSzcQjbKWZPkvmyho1hEOm7sIftbB/NrlkX2nSDa1rr39gHEQIQhUFSikfz
+INkJvaQbNpL1IOFbQtomfG/GNn3XwHoyjnzrEb8B2+sQfzMWTyc+m/DehfdDeCvwznLUIocH3x65
+XY3tKpR4G5/AboptQix9b8V2G75b5bcP8S34bpHbW7CNbxIENW7Em9jzVVBIZ+DiNXj7GojX4Ogf
+wf9HGP908lPut1cqnM9fefUKt+uT3k+e/4Sv/QT0n4CGXDZc9l8OXo5fnrqsKtB/DIXk1yB8eHGt
+8xfrP+j+5/U/7yYf4Mg+qP3A/8H4B+kPlB8A3/1z3uw0zIgztTPxmfGZ8zMXZ67MaMa/P/l97u9e
+8Tj1rzhf4Zxndp05eoYPPgv6Z53Pcv4ng09yk0+B/innU56n+Ccer3E+3upwfvOx5c6Lj115jKNf
+/DxWJPhegV2wnaxHH+48w886n99ogh04LD0+nXh78N6Fdwzvh/DGcw+yO/H2wHZpLd/7V6B7xPZI
+5SN3PnL8EWX8vvH7Ju/jx++dvJd7fuzVMS7pr3DGRiudo60rnVavpVvt5btV2A39umlLX/kKX7BX
+cvYi0949tc49rRXOUm9JtxIHrEBGPe/km/hdfIx/iH+VV2s6/A5nO94X/Vf8nOTXFvr0u5y7PLv4
+c7MXpcg2F2rbGt86vpXf4qtwtrWudepbna2e1rdbf9H6SauqtxWexj/f875Xfbzkq/D4JJ/D5Vvc
+Zus2e03dAui7DV59NwcYaC/p9uhn9Zxe36s/qqdfexFu3AxKOAeT012dlZXbzqlnO7alNf69aXgg
+Xd5Jn1L7nrTqgTTp3rO3ZxrgG4F7T5wgm+zb0vWdPemgPbAtHUZAosA4Agb7tJlsCiSTqUp2QWUl
+wgfxSSoPViJyfzKLJXN0UpmEJNaoJBOCSsqQbQM+KykNEVQOUHp/ktAHJVZmhah0UlbHhLMPBlj2
+/28b/Cp6CmVuZHN0cmVhbQplbmRvYmoKCjggMCBvYmoKNzU0NgplbmRvYmoKCjkgMCBvYmoKPDwv
+VHlwZS9Gb250RGVzY3JpcHRvci9Gb250TmFtZS9CQUFBQUErTGliZXJhdGlvblNlcmlmCi9GbGFn
+cyA0Ci9Gb250QkJveFstNTQzIC0zMDMgMTI3NyA5ODFdL0l0YWxpY0FuZ2xlIDAKL0FzY2VudCA4
+OTEKL0Rlc2NlbnQgLTIxNgovQ2FwSGVpZ2h0IDk4MQovU3RlbVYgODAKL0ZvbnRGaWxlMiA3IDAg
+Ugo+PgplbmRvYmoKCjEwIDAgb2JqCjw8L0xlbmd0aCAzMTgvRmlsdGVyL0ZsYXRlRGVjb2RlPj4K
+c3RyZWFtCnicXZLLboMwEEX3fIWX6SLilYREQkgJCRKLPlTSDyD2kFoqxjJkwd/XM0NbqQusM+O5
+o6trwrI+10ZP4ZsbZAOT6LRRDsbh4SSIG9y1CeJEKC2npaJT9q0NQq9t5nGCvjbdkOdB+O7vxsnN
+YnVUww2egvDVKXDa3MXqo2x83Tys/YIezCSioCiEgs7veW7tS9tDSKp1rfy1nua1l/wNXGcLIqE6
+ZityUDDaVoJrzR2CPIoKkVdVEYBR/+6SRXLr5Gfr/GjsR6NoUxaeE+JdhZwSJxHyhvt75C1xtkHe
+MdNMRpwekfespT0HnsmQj9y/IJ945xa55Bnaf+Z+jHxhPiNXzOgzjphTZPafEbP/FPfE7H93Ql78
+J8iL/wOFs6SAMeE7/sQv5MM5Hz09NmWOaWsDv/+DHSyq6PsGy4GcewplbmRzdHJlYW0KZW5kb2Jq
+CgoxMSAwIG9iago8PC9UeXBlL0ZvbnQvU3VidHlwZS9UcnVlVHlwZS9CYXNlRm9udC9CQUFBQUEr
+TGliZXJhdGlvblNlcmlmCi9GaXJzdENoYXIgMAovTGFzdENoYXIgMjEKL1dpZHRoc1s3NzcgNjEw
+IDUwMCAyNTAgNTAwIDI3NyA1MDAgMjc3IDI3NyA3MjIgMjUwIDQ0MyA1MDAgNDQzIDc3NyAyNzcK
+NDQzIDM4OSA1MDAgNTAwIDUwMCA1MDAgXQovRm9udERlc2NyaXB0b3IgOSAwIFIKL1RvVW5pY29k
+ZSAxMCAwIFIKPj4KZW5kb2JqCgoxMiAwIG9iago8PC9GMSAxMSAwIFIKPj4KZW5kb2JqCgoxMyAw
+IG9iago8PC9Gb250IDEyIDAgUgovUHJvY1NldFsvUERGL1RleHRdCj4+CmVuZG9iagoKMSAwIG9i
+ago8PC9UeXBlL1BhZ2UvUGFyZW50IDYgMCBSL1Jlc291cmNlcyAxMyAwIFIvTWVkaWFCb3hbMCAw
+IDU5NS4zMDM5MzcwMDc4NzQgODQxLjg4OTc2Mzc3OTUyOF0vQW5ub3RzWwo0IDAgUiA1IDAgUiBd
+Ci9Hcm91cDw8L1MvVHJhbnNwYXJlbmN5L0NTL0RldmljZVJHQi9JIHRydWU+Pi9Db250ZW50cyAy
+IDAgUj4+CmVuZG9iagoKNiAwIG9iago8PC9UeXBlL1BhZ2VzCi9SZXNvdXJjZXMgMTMgMCBSCi9N
+ZWRpYUJveFsgMCAwIDU5NSA4NDEgXQovS2lkc1sgMSAwIFIgXQovQ291bnQgMT4+CmVuZG9iagoK
+NCAwIG9iago8PC9UeXBlL0Fubm90L1N1YnR5cGUvTGluay9Cb3JkZXJbMCAwIDBdL1JlY3RbNzMu
+MDQzIDc3MS4zODkgMTk0LjMwNyA3ODUuMTg5XS9BPDwvVHlwZS9BY3Rpb24vUy9VUkkvVVJJKGh0
+dHA6Ly93d3cuZXhhbXBsZS5jb20vKT4+Cj4+CmVuZG9iagoKNSAwIG9iago8PC9UeXBlL0Fubm90
+L1N1YnR5cGUvTGluay9Cb3JkZXJbMCAwIDBdL1JlY3RbMjE5LjU5MyA3NzEuMzg5IDI4MS4zMDcg
+Nzg1LjE4OV0vQTw8L1R5cGUvQWN0aW9uL1MvVVJJL1VSSShodHRwOi8vOC44LjguOC8pPj4KPj4K
+ZW5kb2JqCgoxNCAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgNiAwIFIKL09wZW5BY3Rpb25b
+MSAwIFIgL1hZWiBudWxsIG51bGwgMF0KL0xhbmcoZW4tWkEpCj4+CmVuZG9iagoKMTUgMCBvYmoK
+PDwvQ3JlYXRvcjxGRUZGMDA1NzAwNzIwMDY5MDA3NDAwNjUwMDcyPgovUHJvZHVjZXI8RkVGRjAw
+NEMwMDY5MDA2MjAwNzIwMDY1MDA0RjAwNjYwMDY2MDA2OTAwNjMwMDY1MDAyMDAwMzYwMDJFMDAz
+MT4KL0NyZWF0aW9uRGF0ZShEOjIwMjAxMDAyMTI1MjU0KzAyJzAwJyk+PgplbmRvYmoKCnhyZWYK
+MCAxNgowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDg5NDMgMDAwMDAgbiAKMDAwMDAwMDAxOSAw
+MDAwMCBuIAowMDAwMDAwMzU0IDAwMDAwIG4gCjAwMDAwMDkyMzQgMDAwMDAgbiAKMDAwMDAwOTM4
+NCAwMDAwMCBuIAowMDAwMDA5MTM1IDAwMDAwIG4gCjAwMDAwMDAzNzQgMDAwMDAgbiAKMDAwMDAw
+ODAwNSAwMDAwMCBuIAowMDAwMDA4MDI2IDAwMDAwIG4gCjAwMDAwMDgyMjEgMDAwMDAgbiAKMDAw
+MDAwODYwOSAwMDAwMCBuIAowMDAwMDA4ODU1IDAwMDAwIG4gCjAwMDAwMDg4ODggMDAwMDAgbiAK
+MDAwMDAwOTUyNyAwMDAwMCBuIAowMDAwMDA5NjI0IDAwMDAwIG4gCnRyYWlsZXIKPDwvU2l6ZSAx
+Ni9Sb290IDE0IDAgUgovSW5mbyAxNSAwIFIKL0lEIFsgPDY4MUNCQ0JEMzlFNEVCM0RBNUE2RjQw
+NkM0OEE3NkQzPgo8NjgxQ0JDQkQzOUU0RUIzREE1QTZGNDA2QzQ4QTc2RDM+IF0KL0RvY0NoZWNr
+c3VtIC9DMjBCNTdGMTc5QTgxMTNDMjAwNEVFNDk0MTM2ODMwRAo+PgpzdGFydHhyZWYKOTc5OQol
+JUVPRgo=
+--=_BFgoT7oi_t2JDLtlkNV7Zzc--
+
diff --git a/test/functional/messages/currency.eml b/test/functional/messages/currency.eml
new file mode 100644
index 0000000..d00ea21
--- /dev/null
+++ b/test/functional/messages/currency.eml
@@ -0,0 +1,5 @@
+From: <>
+Subject: £600
+Content-Type: text/plain
+
+test
diff --git a/test/functional/messages/dmarc/bad_dkim1.eml b/test/functional/messages/dmarc/bad_dkim1.eml
new file mode 100644
index 0000000..4183686
--- /dev/null
+++ b/test/functional/messages/dmarc/bad_dkim1.eml
@@ -0,0 +1,17 @@
+Date: Tue, 09 Aug 2016 10:01:27 +0200
+Message-ID: <20160809100127@rspamd.tk>
+From: Rspamd <foo@cacophony.za.org>
+To: foo@rspamd.tk
+Subject: hello
+Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes
+MIME-Version: 1.0
+Content-Disposition: inline
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yoni.za.org; s=testdkim1;
+ t=1470729879; h=from:subject:date:message-id:to:mime-version:content-type;
+ bh=7HkRgYnNru3SR2EWfgWU8yhM0MOH6ZZrPoEIgNIh8wc=;
+ b=kTIV4jcgv9sWFh2JFrS/+PcNxiloituqjmHHqeJOTfa+/9C+Er8BjnMysTJyYVq36Gnv0OZDgLr3Yy4YP5Lzbt1M9ZdN5cJqO7yn1N7wyaGfkt++b09rIYBy5Dkk7OWyP3cDThqDzv8C9heSvqBSEsirFsbt3Wx2g/hWiJlnjew=
+
+
+hello
+
+
diff --git a/test/functional/messages/dmarc/bad_dkim2.eml b/test/functional/messages/dmarc/bad_dkim2.eml
new file mode 100644
index 0000000..fde6cbb
--- /dev/null
+++ b/test/functional/messages/dmarc/bad_dkim2.eml
@@ -0,0 +1,17 @@
+Date: Tue, 09 Aug 2016 10:01:27 +0200
+Message-ID: <20160809100127@rspamd.tk>
+From: Rspamd <foo@rspamd.tk>
+To: foo@rspamd.tk
+Subject: hello
+Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes
+MIME-Version: 1.0
+Content-Disposition: inline
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=asdf.rspamd.tk; s=testdkim;
+ t=1470729879; h=from:subject:date:message-id:to:mime-version:content-type;
+ bh=7HkRgYnNru3SR2EWfgWU8yhM0MOH6ZZrPoEIgNIh8wc=;
+ b=kTIV4jcgv9sWFh2JFrS/+PcNxiloituqjmHHqeJOTfa+/9C+Er8BjnMysTJyYVq36Gnv0OZDgLr3Yy4YP5Lzbt1M9ZdN5cJqO7yn1N7wyaGfkt++b09rIYBy5Dkk7OWyP3cDThqDzv8C9heSvqBSEsirFsbt3Wx2g/hWiJlnjew=
+
+
+hello
+
+
diff --git a/test/functional/messages/dmarc/bad_dkim3.eml b/test/functional/messages/dmarc/bad_dkim3.eml
new file mode 100644
index 0000000..d178a48
--- /dev/null
+++ b/test/functional/messages/dmarc/bad_dkim3.eml
@@ -0,0 +1,17 @@
+Date: Tue, 09 Aug 2016 10:01:27 +0200
+Message-ID: <20160809100127@rspamd.tk>
+From: Rspamd <foo@cacophony.za.org>
+To: foo@rspamd.tk
+Subject: hello
+Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes
+MIME-Version: 1.0
+Content-Disposition: inline
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=dnssec-failed.org; s=testdkim1;
+ t=1470729879; h=from:subject:date:message-id:to:mime-version:content-type;
+ bh=7HkRgYnNru3SR2EWfgWU8yhM0MOH6ZZrPoEIgNIh8wc=;
+ b=kTIV4jcgv9sWFh2JFrS/+PcNxiloituqjmHHqeJOTfa+/9C+Er8BjnMysTJyYVq36Gnv0OZDgLr3Yy4YP5Lzbt1M9ZdN5cJqO7yn1N7wyaGfkt++b09rIYBy5Dkk7OWyP3cDThqDzv8C9heSvqBSEsirFsbt3Wx2g/hWiJlnjew=
+
+
+hello
+
+
diff --git a/test/functional/messages/dmarc/bad_dkim4.eml b/test/functional/messages/dmarc/bad_dkim4.eml
new file mode 100644
index 0000000..6d57aaf
--- /dev/null
+++ b/test/functional/messages/dmarc/bad_dkim4.eml
@@ -0,0 +1,17 @@
+Date: Tue, 09 Aug 2016 10:01:27 +0200
+Message-ID: <20160809100127@rspamd.tk>
+From: Rspamd <a@fail1.org.org.za>
+To: foo@rspamd.tk
+Subject: hello
+Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes
+MIME-Version: 1.0
+Content-Disposition: inline
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yoni.za.org; s=testdkim1;
+ t=1470729879; h=from:subject:date:message-id:to:mime-version:content-type;
+ bh=7HkRgYnNru3SR2EWfgWU8yhM0MOH6ZZrPoEIgNIh8wc=;
+ b=kTIV4jcgv9sWFh2JFrS/+PcNxiloituqjmHHqeJOTfa+/9C+Er8BjnMysTJyYVq36Gnv0OZDgLr3Yy4YP5Lzbt1M9ZdN5cJqO7yn1N7wyaGfkt++b09rIYBy5Dkk7OWyP3cDThqDzv8C9heSvqBSEsirFsbt3Wx2g/hWiJlnjew=
+
+
+hello
+
+
diff --git a/test/functional/messages/dmarc/bad_dkim_highsecure.eml b/test/functional/messages/dmarc/bad_dkim_highsecure.eml
new file mode 100644
index 0000000..4a89598
--- /dev/null
+++ b/test/functional/messages/dmarc/bad_dkim_highsecure.eml
@@ -0,0 +1,61 @@
+Delivered-To: vstakhov@gmail.com
+Received: by 2002:a6b:e610:0:0:0:0:0 with SMTP id g16-v6csp2167100ioh;
+ Mon, 1 Oct 2018 07:01:38 -0700 (PDT)
+X-Google-Smtp-Source: ACcGV600DIuvoFwRUFUQRw3uefS2vbyPNMVmB4EVMRmpQN5W/q6VFysTqCZxRezyjaLn+UH6TEhI
+X-Received: by 2002:a1c:4887:: with SMTP id v129-v6mr8698698wma.139.1538402498770;
+ Mon, 01 Oct 2018 07:01:38 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1538402498; cv=none;
+ d=google.com; s=arc-20160816;
+ b=uKHnpS+DExFKAe2EsUhcF9Nx9aYDFuqxV8cUYpabBGOXa3cO33wzlwEINeoG0GqESe
+ nacEwvHeHYplUkAM2SQz+AdSTjzk5TSUzqdzyNox6KU3OripFgApCaFGXVn7rQqU0YYh
+ OcB/0aBpU61ix2buEETdAV+umKd+qKzos5V4WxJmulVWES9pWi44SdXJK9yb+/savl49
+ Gipc0kSdiAkB7r1qMt13tm5oZ+66sCQxCK9IfvOj0QIf5sALgwHsTCSq5yfyCZdtFwjn
+ CZPduL1EwfFIUx+6axy1MEyImDsLhJv4pDKpFfko4X59hoVFGi63YU89YmsO6guTWOZ0
+ jwmw==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+ h=content-transfer-encoding:content-language:mime-version:user-agent
+ :date:message-id:autocrypt:openpgp:subject:from:to:dkim-signature;
+ bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=oGaANMvTT16cEY+U7T08L9TiRD7bE5KwtJa2Fl3p+2813bomvchlWQdCh65CMcxm3+
+ felXzope3YyDKJmf8waAEG/jMK6VnTz+2hTkUm/p08LLoOv6u6OoRWsI+h8cGgnS3ylG
+ xidRereLZxqaC6PC/U5OSSJ63UgOu6S//hL+o1WYdKhGoQyj2jOD3DxnCY7z/qq048Al
+ SNqycpynWXdoZjwrhB6HtydlO3VQSdPL5qaHNFLKCCTvaQ89Dlk5R/01rWeRKxnSM8bI
+ w4LBKGDUrpGYE/A3AvS+ByZu7ASGeg8cEL2GPfNlvnmy9sAQj0pU1Q/xmHyP1NPxUnra
+ xCpg==
+ARC-Authentication-Results: i=1; mx.google.com;
+ dkim=pass header.i=@highsecure.ru header.s=dkim header.b="oVxOH/NU";
+ spf=pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@highsecure.ru
+Return-Path: <vsevolod@highsecure.ru>
+Received: from mail.highsecure.ru (mail.highsecure.ru. [88.99.142.95])
+ by mx.google.com with ESMTPS id z129-v6si8080952wmg.157.2018.10.01.07.01.38
+ for <vstakhov@gmail.com>
+ (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+ Mon, 01 Oct 2018 07:01:38 -0700 (PDT)
+Received-SPF: pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) client-ip=88.99.142.95;
+Authentication-Results: mx.google.com;
+ dkim=pass header.i=@highsecure.ru header.s=dkim header.b="oVxOH/NU";
+ spf=pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@highsecure.ru
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=highsecure.ru;
+ s=dkim; t=1538402490; h=from:from:sender:reply-to:subject:subject:date:date:
+ message-id:message-id:to:to:cc:mime-version:mime-version:
+ content-type:content-type:
+ content-transfer-encoding:content-transfer-encoding:in-reply-to:
+ references; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=oVxOH/NUK6jcI0tcG73U5qZr2kj4WWZqzOCi3n08KHHW3w/nDFs/V1GfJ5VJy8O0Si+Qtl
+ yx7E+loc2sAXpzlis3myUL8AdACQlyzuxujwY3kj4/7ZmdMaN7/x9mXfnxHfFy0MkbdINq
+ e6mVhumtfbbWA+rJBcMvxQBB+IDgDks=
+To: vstakhov@gmail.com
+From: Vsevolod Stakhov <vsevolod@highsecure.ru>
+Subject: test
+Message-ID: <5ab16dfe-f4e5-3b01-6ee2-9fccb68fc1fc@highsecure.ru>
+Date: Mon, 1 Oct 2018 15:01:29 +0100
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:52.0)
+ Gecko/20100101 Thunderbird/52.9.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Language: en-US
+Content-Transfer-Encoding: 7bit
+Authentication-Results: mail.highsecure.ru;
+ auth=pass smtp.auth=vsevolod@highsecure.ru smtp.mailfrom=vsevolod@highsecure.ru
+
+Malicious content. \ No newline at end of file
diff --git a/test/functional/messages/dmarc/bad_dkim_rspamd.eml b/test/functional/messages/dmarc/bad_dkim_rspamd.eml
new file mode 100644
index 0000000..2281128
--- /dev/null
+++ b/test/functional/messages/dmarc/bad_dkim_rspamd.eml
@@ -0,0 +1,61 @@
+Delivered-To: vstakhov@gmail.com
+Received: by 2002:a6b:e610:0:0:0:0:0 with SMTP id g16-v6csp2201135ioh;
+ Mon, 1 Oct 2018 07:27:55 -0700 (PDT)
+X-Google-Smtp-Source: ACcGV6270qkcKzPIvBSxvaIpBZbNAdj6Qp7qqenTBQi6YaUXdBFD2+2ZYmZaw2WM/SxZP2kYTIz2
+X-Received: by 2002:adf:93e6:: with SMTP id 93-v6mr1901854wrp.81.1538404075350;
+ Mon, 01 Oct 2018 07:27:55 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1538404075; cv=none;
+ d=google.com; s=arc-20160816;
+ b=RaCtnaDmStierMmg+OOhEkzaXxQVAcFO/Rc/ey+6INIQJx+lKVO+dWT0qNA7cZcwUm
+ my6bQE0AZNf45s3bVmQeECtvfe2yS7zVSRx1HFTJJ+iiNR9iSvC8j5PUz1VShRez9Csm
+ 4tqy1ic5t0t9NoOL24f82ju5gTbpl0cc7aH9sMn8gr4DwBxnvuJu4+EdP1QcDKE9qTVa
+ QpjOOOpnkmA46PypufkX+ENaq+bfNDpgbAppKfz2rmutF49jouF8XkrB9Z2ZRWPHE4YA
+ gHJ78GT/4NPlFNo95Ik/nDdnUI6gHkTmiSS6aDJh1W5MiXbkuLT8DSa4Htc43nIr2/m6
+ uQ7g==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+ h=content-transfer-encoding:content-language:mime-version:user-agent
+ :date:message-id:autocrypt:openpgp:to:subject:from:dkim-signature;
+ bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=utFz5J7XQOwR6hdXC0uF+PX17r57baD9GXm5v+3ztobSCFAI0ex+psgbX2gBo6izq3
+ Vo/QjJ9SeJEYhTsLR7jZ3o5meWXJZJqRH073eZlisUGOnDJkJQ5aN/4DY0L5btqLYhwI
+ mJ7c3g4Uh9zFNK8eSIDLdLAIPXNXWRT3SvoS4Ck9ok7fivfZzNfKIPUXbQFIql4+vIAj
+ t1v47QwIrTU+ojwBfaaDjtQEnOB2t8c7RNXys+LQFawG6QZGmG8PCrkVZTU+1v23qbUb
+ M7kDhvSISDchgSrHFwSIniXnnqZe6MRm24xlfW5yebFgmjzMCZQLiyA+WuMIUVxDJpKO
+ V/6g==
+ARC-Authentication-Results: i=1; mx.google.com;
+ dkim=pass header.i=@rspamd.com header.s=dkim header.b=iuniqXuF;
+ spf=pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@rspamd.com
+Return-Path: <vsevolod@rspamd.com>
+Received: from mail.highsecure.ru (mail.highsecure.ru. [88.99.142.95])
+ by mx.google.com with ESMTPS id u13-v6si8362844wmd.167.2018.10.01.07.27.54
+ for <vstakhov@gmail.com>
+ (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+ Mon, 01 Oct 2018 07:27:55 -0700 (PDT)
+Received-SPF: pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) client-ip=88.99.142.95;
+Authentication-Results: mx.google.com;
+ dkim=pass header.i=@rspamd.com header.s=dkim header.b=iuniqXuF;
+ spf=pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@rspamd.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=rspamd.com; s=dkim;
+ t=1538404074; h=from:from:sender:reply-to:subject:subject:date:date:
+ message-id:message-id:to:to:cc:mime-version:mime-version:
+ content-type:content-type:
+ content-transfer-encoding:content-transfer-encoding:in-reply-to:
+ references; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=iuniqXuFCfL63FjU4vXZQEjBRMXkRMwF3kAv+5heeVbgsAvpd+t4I/6w6CU+GVX8xLmsuW
+ jz+Ycmobj0O7eaw+93gfJ3rYnWNA4/WV2J3vAPSpqgBloYpoktEUOQcNm0VsWiCt6WNAq2
+ Ad2CNaQOOmvDMC5lBfp+YVJITiMcYoU=
+From: Vsevolod Stakhov <vsevolod@rspamd.com>
+Subject: test
+To: vstakhov@gmail.com
+Message-ID: <6f4415bf-ff61-f0f5-b60c-ba71a56b9e48@rspamd.com>
+Date: Mon, 1 Oct 2018 15:27:53 +0100
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:52.0)
+ Gecko/20100101 Thunderbird/52.9.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Language: en-US
+Content-Transfer-Encoding: 7bit
+Authentication-Results: mail.highsecure.ru;
+ auth=pass smtp.auth=vsevolod@highsecure.ru smtp.mailfrom=vsevolod@rspamd.com
+
+Evil content \ No newline at end of file
diff --git a/test/functional/messages/dmarc/dkim_unknown_tags.eml b/test/functional/messages/dmarc/dkim_unknown_tags.eml
new file mode 100644
index 0000000..15a2f3d
--- /dev/null
+++ b/test/functional/messages/dmarc/dkim_unknown_tags.eml
@@ -0,0 +1,19 @@
+Received: (qmail 99419 invoked from network); 30 Jun 2022 22:27:07 -0000
+DKIM-Signature: v=1; a=rsa-sha256; c=simple; d=taugh.com; h=date:message-id:from:to:content-type:content-transfer-encoding:mime-version:subject:references:in-reply-to; s=18457.62be233b.k2206; olt=johnl-iecc.com@submit.iecc.com; bh=Pb6s/Xlf4u1eDlYyO0NCaMRMrCg6xDNkK5byz8RDY1s=; b=XpuXx66miTUH5iEhGqjLUHKozhLjgs2wfGX2msE/3A7L7/IsnSIM7VAjzWS3tYJL6qgmoVqmGcAwtzy0WfXGGaF/n5h46mLxU1f9aLiXvVPAaN5m0w39B+bG6EE6X2Tp267cq+DQJSufUwvoaZX8gLPQ0VbTIMBJ/vV1QQ2JjphaAyggKYR6rcVTMs0A1gFO9S55ksERzW32WlmjOEmcvuzDzrmLAfhvA+jHekrC6yWLo7pmmYyt9/vJVfXsiq8xOacCxkSKWOkzm138OK+1G3Z+FrCZ8OtgWIgHCT6GevgFvZKdeCFkdxvbtMadGT/okOYBIe70tEFO/cUs2tj8tg==
+Date: 30 Jun 2022 18:27:04 -0400
+Message-Id: <9128D54C-D180-4133-A4DE-A98EDA971B73@taugh.com>
+From: "John Levine" <johnl@taugh.com>
+To: "Vsevolod Stakhov" <vsevolod@rspamd.com>
+Content-Type: text/plain;
+ charset=us-ascii
+Content-Transfer-Encoding: 7bit
+Mime-Version: 1.0 (Mac OS X Mail 16.0 \(3696.100.31\))
+Subject: DKIM tags test
+References: <20220630000404.4460844A5EFD@ary.qy>
+ <a02a67c8-5336-e3c9-97ca-71e56c3566c1@rspamd.com>
+ <E4088547-0AD0-499C-8E48-B94482524484@taugh.com>
+ <35781b14-a61c-6e56-2265-6a5161a2b2cd@rspamd.com>
+In-Reply-To: <35781b14-a61c-6e56-2265-6a5161a2b2cd@rspamd.com>
+
+test message
+
diff --git a/test/functional/messages/dmarc/dmarc_tmpfail.eml b/test/functional/messages/dmarc/dmarc_tmpfail.eml
new file mode 100644
index 0000000..1c8db5b
--- /dev/null
+++ b/test/functional/messages/dmarc/dmarc_tmpfail.eml
@@ -0,0 +1,17 @@
+Date: Tue, 09 Aug 2016 10:01:27 +0200
+Message-ID: <20160809100127@rspamd.tk>
+From: Rspamd <foo@dnssec-failed.org>
+To: foo@rspamd.tk
+Subject: hello
+Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes
+MIME-Version: 1.0
+Content-Disposition: inline
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=rspamd.tk; s=testdkim;
+ t=1470729879; h=from:subject:date:message-id:to:mime-version:content-type;
+ bh=7HkRgYnNru3SR2EWfgWU8yhM0MOH6ZZrPoEIgNIh8wc=;
+ b=kTIV4jcgv9sWFh2JFrS/+PcNxiloituqjmHHqeJOTfa+/9C+Er8BjnMysTJyYVq36Gnv0OZDgLr3Yy4YP5Lzbt1M9ZdN5cJqO7yn1N7wyaGfkt++b09rIYBy5Dkk7OWyP3cDThqDzv8C9heSvqBSEsirFsbt3Wx2g/hWiJlnjew=
+
+
+hello
+
+
diff --git a/test/functional/messages/dmarc/fail_none.eml b/test/functional/messages/dmarc/fail_none.eml
new file mode 100644
index 0000000..159f1f5
--- /dev/null
+++ b/test/functional/messages/dmarc/fail_none.eml
@@ -0,0 +1,3 @@
+From: Rspamd <foo@caCophony.za.org>
+
+hello
diff --git a/test/functional/messages/dmarc/fail_none1.eml b/test/functional/messages/dmarc/fail_none1.eml
new file mode 100644
index 0000000..1579bb7
--- /dev/null
+++ b/test/functional/messages/dmarc/fail_none1.eml
@@ -0,0 +1,3 @@
+From: Rspamd <foo@invalid.za.org>
+
+hello
diff --git a/test/functional/messages/dmarc/fail_quarantine.eml b/test/functional/messages/dmarc/fail_quarantine.eml
new file mode 100644
index 0000000..2d4e462
--- /dev/null
+++ b/test/functional/messages/dmarc/fail_quarantine.eml
@@ -0,0 +1,3 @@
+From: Rspamd <foo@quarantine.cacophony.za.org>
+
+hello
diff --git a/test/functional/messages/dmarc/fail_reject.eml b/test/functional/messages/dmarc/fail_reject.eml
new file mode 100644
index 0000000..e21248c
--- /dev/null
+++ b/test/functional/messages/dmarc/fail_reject.eml
@@ -0,0 +1,3 @@
+From: Rspamd <foo@reject.cacophony.za.org>
+
+hello
diff --git a/test/functional/messages/dmarc/good_dkim_highsecure.eml b/test/functional/messages/dmarc/good_dkim_highsecure.eml
new file mode 100644
index 0000000..450e7a3
--- /dev/null
+++ b/test/functional/messages/dmarc/good_dkim_highsecure.eml
@@ -0,0 +1,60 @@
+Delivered-To: vstakhov@gmail.com
+Received: by 2002:a6b:e610:0:0:0:0:0 with SMTP id g16-v6csp2167100ioh;
+ Mon, 1 Oct 2018 07:01:38 -0700 (PDT)
+X-Google-Smtp-Source: ACcGV600DIuvoFwRUFUQRw3uefS2vbyPNMVmB4EVMRmpQN5W/q6VFysTqCZxRezyjaLn+UH6TEhI
+X-Received: by 2002:a1c:4887:: with SMTP id v129-v6mr8698698wma.139.1538402498770;
+ Mon, 01 Oct 2018 07:01:38 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1538402498; cv=none;
+ d=google.com; s=arc-20160816;
+ b=uKHnpS+DExFKAe2EsUhcF9Nx9aYDFuqxV8cUYpabBGOXa3cO33wzlwEINeoG0GqESe
+ nacEwvHeHYplUkAM2SQz+AdSTjzk5TSUzqdzyNox6KU3OripFgApCaFGXVn7rQqU0YYh
+ OcB/0aBpU61ix2buEETdAV+umKd+qKzos5V4WxJmulVWES9pWi44SdXJK9yb+/savl49
+ Gipc0kSdiAkB7r1qMt13tm5oZ+66sCQxCK9IfvOj0QIf5sALgwHsTCSq5yfyCZdtFwjn
+ CZPduL1EwfFIUx+6axy1MEyImDsLhJv4pDKpFfko4X59hoVFGi63YU89YmsO6guTWOZ0
+ jwmw==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+ h=content-transfer-encoding:content-language:mime-version:user-agent
+ :date:message-id:autocrypt:openpgp:subject:from:to:dkim-signature;
+ bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=oGaANMvTT16cEY+U7T08L9TiRD7bE5KwtJa2Fl3p+2813bomvchlWQdCh65CMcxm3+
+ felXzope3YyDKJmf8waAEG/jMK6VnTz+2hTkUm/p08LLoOv6u6OoRWsI+h8cGgnS3ylG
+ xidRereLZxqaC6PC/U5OSSJ63UgOu6S//hL+o1WYdKhGoQyj2jOD3DxnCY7z/qq048Al
+ SNqycpynWXdoZjwrhB6HtydlO3VQSdPL5qaHNFLKCCTvaQ89Dlk5R/01rWeRKxnSM8bI
+ w4LBKGDUrpGYE/A3AvS+ByZu7ASGeg8cEL2GPfNlvnmy9sAQj0pU1Q/xmHyP1NPxUnra
+ xCpg==
+ARC-Authentication-Results: i=1; mx.google.com;
+ dkim=pass header.i=@highsecure.ru header.s=dkim header.b="oVxOH/NU";
+ spf=pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@highsecure.ru
+Return-Path: <vsevolod@highsecure.ru>
+Received: from mail.highsecure.ru (mail.highsecure.ru. [88.99.142.95])
+ by mx.google.com with ESMTPS id z129-v6si8080952wmg.157.2018.10.01.07.01.38
+ for <vstakhov@gmail.com>
+ (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+ Mon, 01 Oct 2018 07:01:38 -0700 (PDT)
+Received-SPF: pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) client-ip=88.99.142.95;
+Authentication-Results: mx.google.com;
+ dkim=pass header.i=@highsecure.ru header.s=dkim header.b="oVxOH/NU";
+ spf=pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@highsecure.ru
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=highsecure.ru;
+ s=dkim; t=1538402490; h=from:from:sender:reply-to:subject:subject:date:date:
+ message-id:message-id:to:to:cc:mime-version:mime-version:
+ content-type:content-type:
+ content-transfer-encoding:content-transfer-encoding:in-reply-to:
+ references; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=oVxOH/NUK6jcI0tcG73U5qZr2kj4WWZqzOCi3n08KHHW3w/nDFs/V1GfJ5VJy8O0Si+Qtl
+ yx7E+loc2sAXpzlis3myUL8AdACQlyzuxujwY3kj4/7ZmdMaN7/x9mXfnxHfFy0MkbdINq
+ e6mVhumtfbbWA+rJBcMvxQBB+IDgDks=
+To: vstakhov@gmail.com
+From: Vsevolod Stakhov <vsevolod@highsecure.ru>
+Subject: test
+Message-ID: <5ab16dfe-f4e5-3b01-6ee2-9fccb68fc1fc@highsecure.ru>
+Date: Mon, 1 Oct 2018 15:01:29 +0100
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:52.0)
+ Gecko/20100101 Thunderbird/52.9.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Language: en-US
+Content-Transfer-Encoding: 7bit
+Authentication-Results: mail.highsecure.ru;
+ auth=pass smtp.auth=vsevolod@highsecure.ru smtp.mailfrom=vsevolod@highsecure.ru
+
diff --git a/test/functional/messages/dmarc/good_dkim_rspamd.eml b/test/functional/messages/dmarc/good_dkim_rspamd.eml
new file mode 100644
index 0000000..13fe5b0
--- /dev/null
+++ b/test/functional/messages/dmarc/good_dkim_rspamd.eml
@@ -0,0 +1,60 @@
+Delivered-To: vstakhov@gmail.com
+Received: by 2002:a6b:e610:0:0:0:0:0 with SMTP id g16-v6csp2201135ioh;
+ Mon, 1 Oct 2018 07:27:55 -0700 (PDT)
+X-Google-Smtp-Source: ACcGV6270qkcKzPIvBSxvaIpBZbNAdj6Qp7qqenTBQi6YaUXdBFD2+2ZYmZaw2WM/SxZP2kYTIz2
+X-Received: by 2002:adf:93e6:: with SMTP id 93-v6mr1901854wrp.81.1538404075350;
+ Mon, 01 Oct 2018 07:27:55 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1538404075; cv=none;
+ d=google.com; s=arc-20160816;
+ b=RaCtnaDmStierMmg+OOhEkzaXxQVAcFO/Rc/ey+6INIQJx+lKVO+dWT0qNA7cZcwUm
+ my6bQE0AZNf45s3bVmQeECtvfe2yS7zVSRx1HFTJJ+iiNR9iSvC8j5PUz1VShRez9Csm
+ 4tqy1ic5t0t9NoOL24f82ju5gTbpl0cc7aH9sMn8gr4DwBxnvuJu4+EdP1QcDKE9qTVa
+ QpjOOOpnkmA46PypufkX+ENaq+bfNDpgbAppKfz2rmutF49jouF8XkrB9Z2ZRWPHE4YA
+ gHJ78GT/4NPlFNo95Ik/nDdnUI6gHkTmiSS6aDJh1W5MiXbkuLT8DSa4Htc43nIr2/m6
+ uQ7g==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+ h=content-transfer-encoding:content-language:mime-version:user-agent
+ :date:message-id:autocrypt:openpgp:to:subject:from:dkim-signature;
+ bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=utFz5J7XQOwR6hdXC0uF+PX17r57baD9GXm5v+3ztobSCFAI0ex+psgbX2gBo6izq3
+ Vo/QjJ9SeJEYhTsLR7jZ3o5meWXJZJqRH073eZlisUGOnDJkJQ5aN/4DY0L5btqLYhwI
+ mJ7c3g4Uh9zFNK8eSIDLdLAIPXNXWRT3SvoS4Ck9ok7fivfZzNfKIPUXbQFIql4+vIAj
+ t1v47QwIrTU+ojwBfaaDjtQEnOB2t8c7RNXys+LQFawG6QZGmG8PCrkVZTU+1v23qbUb
+ M7kDhvSISDchgSrHFwSIniXnnqZe6MRm24xlfW5yebFgmjzMCZQLiyA+WuMIUVxDJpKO
+ V/6g==
+ARC-Authentication-Results: i=1; mx.google.com;
+ dkim=pass header.i=@rspamd.com header.s=dkim header.b=iuniqXuF;
+ spf=pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@rspamd.com
+Return-Path: <vsevolod@rspamd.com>
+Received: from mail.highsecure.ru (mail.highsecure.ru. [88.99.142.95])
+ by mx.google.com with ESMTPS id u13-v6si8362844wmd.167.2018.10.01.07.27.54
+ for <vstakhov@gmail.com>
+ (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+ Mon, 01 Oct 2018 07:27:55 -0700 (PDT)
+Received-SPF: pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) client-ip=88.99.142.95;
+Authentication-Results: mx.google.com;
+ dkim=pass header.i=@rspamd.com header.s=dkim header.b=iuniqXuF;
+ spf=pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@rspamd.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=rspamd.com; s=dkim;
+ t=1538404074; h=from:from:sender:reply-to:subject:subject:date:date:
+ message-id:message-id:to:to:cc:mime-version:mime-version:
+ content-type:content-type:
+ content-transfer-encoding:content-transfer-encoding:in-reply-to:
+ references; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=iuniqXuFCfL63FjU4vXZQEjBRMXkRMwF3kAv+5heeVbgsAvpd+t4I/6w6CU+GVX8xLmsuW
+ jz+Ycmobj0O7eaw+93gfJ3rYnWNA4/WV2J3vAPSpqgBloYpoktEUOQcNm0VsWiCt6WNAq2
+ Ad2CNaQOOmvDMC5lBfp+YVJITiMcYoU=
+From: Vsevolod Stakhov <vsevolod@rspamd.com>
+Subject: test
+To: vstakhov@gmail.com
+Message-ID: <6f4415bf-ff61-f0f5-b60c-ba71a56b9e48@rspamd.com>
+Date: Mon, 1 Oct 2018 15:27:53 +0100
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:52.0)
+ Gecko/20100101 Thunderbird/52.9.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Language: en-US
+Content-Transfer-Encoding: 7bit
+Authentication-Results: mail.highsecure.ru;
+ auth=pass smtp.auth=vsevolod@highsecure.ru smtp.mailfrom=vsevolod@rspamd.com
+
diff --git a/test/functional/messages/dmarc/no_dkim_valid_spf.eml b/test/functional/messages/dmarc/no_dkim_valid_spf.eml
new file mode 100644
index 0000000..2ed5a9d
--- /dev/null
+++ b/test/functional/messages/dmarc/no_dkim_valid_spf.eml
@@ -0,0 +1,48 @@
+Delivered-To: vstakhov@gmail.com
+Received: by 2002:a6b:e610:0:0:0:0:0 with SMTP id g16-v6csp2201135ioh;
+ Mon, 1 Oct 2018 07:27:55 -0700 (PDT)
+X-Google-Smtp-Source: ACcGV6270qkcKzPIvBSxvaIpBZbNAdj6Qp7qqenTBQi6YaUXdBFD2+2ZYmZaw2WM/SxZP2kYTIz2
+X-Received: by 2002:adf:93e6:: with SMTP id 93-v6mr1901854wrp.81.1538404075350;
+ Mon, 01 Oct 2018 07:27:55 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1538404075; cv=none;
+ d=google.com; s=arc-20160816;
+ b=RaCtnaDmStierMmg+OOhEkzaXxQVAcFO/Rc/ey+6INIQJx+lKVO+dWT0qNA7cZcwUm
+ my6bQE0AZNf45s3bVmQeECtvfe2yS7zVSRx1HFTJJ+iiNR9iSvC8j5PUz1VShRez9Csm
+ 4tqy1ic5t0t9NoOL24f82ju5gTbpl0cc7aH9sMn8gr4DwBxnvuJu4+EdP1QcDKE9qTVa
+ QpjOOOpnkmA46PypufkX+ENaq+bfNDpgbAppKfz2rmutF49jouF8XkrB9Z2ZRWPHE4YA
+ gHJ78GT/4NPlFNo95Ik/nDdnUI6gHkTmiSS6aDJh1W5MiXbkuLT8DSa4Htc43nIr2/m6
+ uQ7g==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+ h=content-transfer-encoding:content-language:mime-version:user-agent
+ :date:message-id:autocrypt:openpgp:to:subject:from:dkim-signature;
+ bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=utFz5J7XQOwR6hdXC0uF+PX17r57baD9GXm5v+3ztobSCFAI0ex+psgbX2gBo6izq3
+ Vo/QjJ9SeJEYhTsLR7jZ3o5meWXJZJqRH073eZlisUGOnDJkJQ5aN/4DY0L5btqLYhwI
+ mJ7c3g4Uh9zFNK8eSIDLdLAIPXNXWRT3SvoS4Ck9ok7fivfZzNfKIPUXbQFIql4+vIAj
+ t1v47QwIrTU+ojwBfaaDjtQEnOB2t8c7RNXys+LQFawG6QZGmG8PCrkVZTU+1v23qbUb
+ M7kDhvSISDchgSrHFwSIniXnnqZe6MRm24xlfW5yebFgmjzMCZQLiyA+WuMIUVxDJpKO
+ V/6g==
+ARC-Authentication-Results: i=1; mx.google.com;
+ dkim=pass header.i=@rspamd.com header.s=dkim header.b=iuniqXuF;
+ spf=pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@rspamd.com
+Return-Path: <vsevolod@rspamd.com>
+Received: from mail.highsecure.ru (mail.highsecure.ru. [88.99.142.95])
+ by mx.google.com with ESMTPS id u13-v6si8362844wmd.167.2018.10.01.07.27.54
+ for <vstakhov@gmail.com>
+ (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+ Mon, 01 Oct 2018 07:27:55 -0700 (PDT)
+Received-SPF: pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) client-ip=88.99.142.95;
+From: Vsevolod Stakhov <vsevolod@rspamd.com>
+Subject: test
+To: vstakhov@gmail.com
+Message-ID: <6f4415bf-ff61-f0f5-b60c-ba71a56b9e48@rspamd.com>
+Date: Mon, 1 Oct 2018 15:27:53 +0100
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:52.0)
+ Gecko/20100101 Thunderbird/52.9.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Language: en-US
+Content-Transfer-Encoding: 7bit
+Authentication-Results: mail.highsecure.ru;
+ auth=pass smtp.auth=vsevolod@highsecure.ru smtp.mailfrom=vsevolod@rspamd.com
+
diff --git a/test/functional/messages/dmarc/onsubdomain_fail.eml b/test/functional/messages/dmarc/onsubdomain_fail.eml
new file mode 100644
index 0000000..51e1535
--- /dev/null
+++ b/test/functional/messages/dmarc/onsubdomain_fail.eml
@@ -0,0 +1,3 @@
+From: <koos@my.mom.za.org>
+
+hello
diff --git a/test/functional/messages/dmarc/onsubdomain_fail_alignment.eml b/test/functional/messages/dmarc/onsubdomain_fail_alignment.eml
new file mode 100644
index 0000000..d144337
--- /dev/null
+++ b/test/functional/messages/dmarc/onsubdomain_fail_alignment.eml
@@ -0,0 +1,11 @@
+From: <koos@yo.mom.za.org>
+Subject: yo mom
+Content-type: text/plain
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=mom.za.org; s=testdkim;
+ t=1471961268; h=from:subject:content-type;
+ bh=BIB4FUu24DrAHudZNxS2O9+aq+jb0j8fPGz27CAMcyM=; b=LNrJ+02t42FSzzN0La6BSI4zN5m/pOLqjSSNHoLTfz7ewDrH9Fc27VJ4W1c+X18C40htFp
+ RuGsjH1XyirwLAd+gDCB12ehuO2uIGWtKyi1EWwivqYs6ACzRvcOMvNM3m/YfJyqZRBWDC
+ C6wcJsJ2gbRZIGVbn4VdTVm6Kbhmj/M=
+
+yo mom deserves respect
+
diff --git a/test/functional/messages/dmarc/onsubdomain_pass_relaxed.eml b/test/functional/messages/dmarc/onsubdomain_pass_relaxed.eml
new file mode 100644
index 0000000..714c560
--- /dev/null
+++ b/test/functional/messages/dmarc/onsubdomain_pass_relaxed.eml
@@ -0,0 +1,11 @@
+From: <koos@my.mom.za.org>
+Subject: my mom
+Content-type: text/plain
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=mom.za.org; s=testdkim;
+ t=1471961323; h=from:subject:content-type;
+ bh=7nTyaR2hZHHEh0TrHDN2UloE2SBtWOqNvwm8wB4BI+M=; b=S/CEbLcr3duA/2rwNAcX03T1KYz4YXI8sH+OQ1YX1+y4Pxi4MjF3cqstLX4jKyrjbBHSAJ
+ mC9QT9vC+4R8syDXZnJw6gChLsR2NymCPp3nbbukmgsNustzyIu7wg4amwZ1yx/m4Ihwvv
+ NfSaAZcJS4tAgtOHmFzkt2zPD4y9+OU=
+
+my mom deserves respect
+
diff --git a/test/functional/messages/dmarc/pass_none.eml b/test/functional/messages/dmarc/pass_none.eml
new file mode 100644
index 0000000..f7fa380
--- /dev/null
+++ b/test/functional/messages/dmarc/pass_none.eml
@@ -0,0 +1,18 @@
+Date: Tue, 09 Aug 2016 10:01:27 +0200
+Message-ID: <20160809100127@rspamd.tk>
+From: Rspamd <foo@cacophony.za.org>
+To: foo@rspamd.tk
+Subject: hello
+Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes
+MIME-Version: 1.0
+Content-Disposition: inline
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=cacophony.za.org;
+ s=dkim; t=1502912195; h=from:subject:date:message-id:to:mime-version:content-type;
+ bh=zS7KNTV0HyeorkDDGwxB1AV6enuRKzO5rthkhdHIRnY=;
+ b=GEGigFZw2oSlVM+lZcMsnq6XHsoIQ3HsehPjbfn9nZjpLpuZ5Yzc/Oki8eiyDg+Dw+203N
+ lRott5+RToquJiGTM0uhxG7bcLu72HmT/JvGO/kNSML9w4GuzBp8OinZLhZqmaxz4RlkLx
+ 41feLYvmQnWWfNhssQNp/5ifi4+AXxM=
+
+hello
+
+
diff --git a/test/functional/messages/dmarc/pct_none.eml b/test/functional/messages/dmarc/pct_none.eml
new file mode 100644
index 0000000..50fe331
--- /dev/null
+++ b/test/functional/messages/dmarc/pct_none.eml
@@ -0,0 +1,17 @@
+Date: Tue, 09 Aug 2016 10:01:27 +0200
+Message-ID: <20160809100128@rspamd.tk>
+From: Rspamd <foo@zero_pct.com>
+To: foo@rspamd.tk
+Subject: hello
+Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes
+MIME-Version: 1.0
+Content-Disposition: inline
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=rspamd.tk; s=testdkim;
+ t=1470729879; h=from:subject:date:message-id:to:mime-version:content-type;
+ bh=7HkRgYnNru3SR2EWfgWU8yhM0MOH6ZZrPoEIgNIh8wc=;
+ b=kTIV4jcgv9sWFh2JFrS/+PcNxiloituqjmHHqeJOTfa+/9C+Er8BjnMysTJyYVq36Gnv0OZDgLr3Yy4YP5Lzbt1M9ZdN5cJqO7yn1N7wyaGfkt++b09rIYBy5Dkk7OWyP3cDThqDzv8C9heSvqBSEsirFsbt3Wx2g/hWiJlnjew=
+
+
+hello
+
+
diff --git a/test/functional/messages/dmarc/pct_none1.eml b/test/functional/messages/dmarc/pct_none1.eml
new file mode 100644
index 0000000..a0cd9fd
--- /dev/null
+++ b/test/functional/messages/dmarc/pct_none1.eml
@@ -0,0 +1,17 @@
+Date: Tue, 09 Aug 2016 10:01:27 +0200
+Message-ID: <20160809100128@rspamd.tk>
+From: Rspamd <foo@subdomain.zero_pct.com>
+To: foo@rspamd.tk
+Subject: hello
+Content-Type: text/plain; charset=utf-8; format=flowed; DelSp=Yes
+MIME-Version: 1.0
+Content-Disposition: inline
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=rspamd.tk; s=testdkim;
+ t=1470729879; h=from:subject:date:message-id:to:mime-version:content-type;
+ bh=7HkRgYnNru3SR2EWfgWU8yhM0MOH6ZZrPoEIgNIh8wc=;
+ b=kTIV4jcgv9sWFh2JFrS/+PcNxiloituqjmHHqeJOTfa+/9C+Er8BjnMysTJyYVq36Gnv0OZDgLr3Yy4YP5Lzbt1M9ZdN5cJqO7yn1N7wyaGfkt++b09rIYBy5Dkk7OWyP3cDThqDzv8C9heSvqBSEsirFsbt3Wx2g/hWiJlnjew=
+
+
+hello
+
+
diff --git a/test/functional/messages/dmarc/subdomain_fail_none.eml b/test/functional/messages/dmarc/subdomain_fail_none.eml
new file mode 100644
index 0000000..7dc0d64
--- /dev/null
+++ b/test/functional/messages/dmarc/subdomain_fail_none.eml
@@ -0,0 +1,3 @@
+From: Rspamd <foo@foo.yoni.za.org>
+
+hello
diff --git a/test/functional/messages/dmarc/subdomain_fail_quarantine.eml b/test/functional/messages/dmarc/subdomain_fail_quarantine.eml
new file mode 100644
index 0000000..366ba79
--- /dev/null
+++ b/test/functional/messages/dmarc/subdomain_fail_quarantine.eml
@@ -0,0 +1,3 @@
+From: Rspamd <foo@foo.yoni.za.net>
+
+hello
diff --git a/test/functional/messages/dmarc/subdomain_fail_reject.eml b/test/functional/messages/dmarc/subdomain_fail_reject.eml
new file mode 100644
index 0000000..697cf57
--- /dev/null
+++ b/test/functional/messages/dmarc/subdomain_fail_reject.eml
@@ -0,0 +1,3 @@
+From: Rspamd <foo@foo.cacophony.za.org>
+
+hello
diff --git a/test/functional/messages/ed25519-broken.eml b/test/functional/messages/ed25519-broken.eml
new file mode 100644
index 0000000..85426fb
--- /dev/null
+++ b/test/functional/messages/ed25519-broken.eml
@@ -0,0 +1,26 @@
+DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed;
+ d=football.example.com; i=@football.example.com;
+ q=dns/txt; s=brisbane; t=1528637909; h=from : to :
+ subject : date : message-id : from : subject : date;
+ bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=;
+ b=/gCrinpcQOoIfuHNQIbq4pgh9kyIK3AQUdt9OdqQehSwhEIug4D11Bus
+ Fa3bT3FY5OsU7ZbnKELq+eXdp1Q1Dw==
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+ d=football.example.com; i=@football.example.com;
+ q=dns/txt; s=test; t=1528637909; h=from : to : subject :
+ date : message-id : from : subject : date;
+ bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=;
+ b=F45dVWDfMbQDGHJFlXUNB2HKfbCeLRyhDXgFpEL8GwpsRe0IeIixNTe3
+ DhCVlUrSjV4BwcVcOF6+FF3Zo9Rpo1tFOeS9mPYQTnGdaSGsgeefOsk2Jz
+ dA+L10TeYt9BgDfQNZtKdN1WO//KgIqXP7OdEFE4LjFYNcUxZQ4FADY+8=
+From: Joe SixPack <joe@evil.example.com>
+To: Suzie Q <suzie@shopping.example.net>
+Subject: Is dinner ready?
+Date: Fri, 11 Jul 2003 21:00:37 -0700 (PDT)
+Message-ID: <20030712040037.46341.5F8J@football.example.com>
+
+Hi.
+
+We lost the game. Are you hungry yet?
+
+Joe.
diff --git a/test/functional/messages/ed25519.eml b/test/functional/messages/ed25519.eml
new file mode 100644
index 0000000..a3397f2
--- /dev/null
+++ b/test/functional/messages/ed25519.eml
@@ -0,0 +1,26 @@
+DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed;
+ d=football.example.com; i=@football.example.com;
+ q=dns/txt; s=brisbane; t=1528637909; h=from : to :
+ subject : date : message-id : from : subject : date;
+ bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=;
+ b=/gCrinpcQOoIfuHNQIbq4pgh9kyIK3AQUdt9OdqQehSwhEIug4D11Bus
+ Fa3bT3FY5OsU7ZbnKELq+eXdp1Q1Dw==
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+ d=football.example.com; i=@football.example.com;
+ q=dns/txt; s=test; t=1528637909; h=from : to : subject :
+ date : message-id : from : subject : date;
+ bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=;
+ b=F45dVWDfMbQDGHJFlXUNB2HKfbCeLRyhDXgFpEL8GwpsRe0IeIixNTe3
+ DhCVlUrSjV4BwcVcOF6+FF3Zo9Rpo1tFOeS9mPYQTnGdaSGsgeefOsk2Jz
+ dA+L10TeYt9BgDfQNZtKdN1WO//KgIqXP7OdEFE4LjFYNcUxZQ4FADY+8=
+From: Joe SixPack <joe@football.example.com>
+To: Suzie Q <suzie@shopping.example.net>
+Subject: Is dinner ready?
+Date: Fri, 11 Jul 2003 21:00:37 -0700 (PDT)
+Message-ID: <20030712040037.46341.5F8J@football.example.com>
+
+Hi.
+
+We lost the game. Are you hungry yet?
+
+Joe.
diff --git a/test/functional/messages/emailbltext.eml b/test/functional/messages/emailbltext.eml
new file mode 100644
index 0000000..6ae877d
--- /dev/null
+++ b/test/functional/messages/emailbltext.eml
@@ -0,0 +1,4 @@
+Content-Type: text/plain
+
+user@emailbl.com
+user@baddomain.com \ No newline at end of file
diff --git a/test/functional/messages/emailbltext2.eml b/test/functional/messages/emailbltext2.eml
new file mode 100644
index 0000000..164288b
--- /dev/null
+++ b/test/functional/messages/emailbltext2.eml
@@ -0,0 +1,3 @@
+Content-Type: text/plain
+
+user@subdomain.emailbl.com \ No newline at end of file
diff --git a/test/functional/messages/emailbltext3.eml b/test/functional/messages/emailbltext3.eml
new file mode 100644
index 0000000..414a0f5
--- /dev/null
+++ b/test/functional/messages/emailbltext3.eml
@@ -0,0 +1,4 @@
+Content-Type: text/plain
+
+user@baddomain.com
+user@subdomain.baddomain.com \ No newline at end of file
diff --git a/test/functional/messages/emails1.eml b/test/functional/messages/emails1.eml
new file mode 100644
index 0000000..8d0c19c
--- /dev/null
+++ b/test/functional/messages/emails1.eml
@@ -0,0 +1,8 @@
+Content-type: text/plain
+
+mailme
+bob@example.net
+mailme*
+jim@example.net
+mailme rupert@example.net
+.net
diff --git a/test/functional/messages/empty-plain-text.eml b/test/functional/messages/empty-plain-text.eml
new file mode 100644
index 0000000..1deeaf1
--- /dev/null
+++ b/test/functional/messages/empty-plain-text.eml
@@ -0,0 +1,15 @@
+Return-Path: test@test.com
+From: TEST <test@test.com>
+MIME-Version: 1.0
+X-Priority: 1 (Highest)
+X-MSMail-Priority: High
+X-Mailer: Microsoft Outlook 16.0
+Importance: High
+Date: Mon, 23 Jul 2018 16:24:13 +0200
+Message-ID: <d9946a191e0c97733a86424c48e65eca@test.com>
+Subject: Test Subject
+To: Me <me@me.me>
+Content-Type: text/plain; charset="UTF-8"
+Custom-Header: foo
+
+
diff --git a/test/functional/messages/empty_part.eml b/test/functional/messages/empty_part.eml
new file mode 100644
index 0000000..7c3e6dc
--- /dev/null
+++ b/test/functional/messages/empty_part.eml
@@ -0,0 +1,22 @@
+Return-Path: <example@example.net>
+Date: 29 Apr 2015 04:56:53 -0000
+MIME-Version: 1.0
+From: example@example.net
+Subject: This is the subject
+Content-Type: multipart/alternative;
+ boundary="=_1ad00c4b74ff9921bb562f02b6e6df7a"
+Message-ID: <mid1234@example.net>
+To: example@example.net
+
+--=_1ad00c4b74ff9921bb562f02b6e6df7a
+Content-Type: text/plain; charset="ISO-8859-1"
+Content-Transfer-Encoding: 7bit
+
+
+--=_1ad00c4b74ff9921bb562f02b6e6df7a
+Content-Type: text/html; charset="ISO-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+<b>HTML part only. The previous part makes the controller process crash</b>
+
+--=_1ad00c4b74ff9921bb562f02b6e6df7a--
diff --git a/test/functional/messages/exe_attm.eml b/test/functional/messages/exe_attm.eml
new file mode 100644
index 0000000..2ab0e94
--- /dev/null
+++ b/test/functional/messages/exe_attm.eml
@@ -0,0 +1,22 @@
+Return-Path: <root@srv.example.com>
+To: test@example.com
+From: root@srv.example.com
+Subject: test Sat, 26 Jan 2019 12:04:58 +0100
+Message-Id: <20190126120458.015328@srv.example.com>
+Date: Sat, 26 Jan 2019 12:04:58 +0100
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="----=_MIME_BOUNDARY_000_15328"
+
+------=_MIME_BOUNDARY_000_15328
+Content-Type: text/plain
+
+This is a test mailing
+------=_MIME_BOUNDARY_000_15328
+Content-Type: application/octet-stream; name="hello_world.exe"
+Content-Description: hello_world.exe
+Content-Disposition: attachment; filename="hello_world.exe"
+Content-Transfer-Encoding: BASE64
+
+f0VMRgIBAQAAAAAAAAAAAAMAPgABAAAAUBAAAAAAAABAAAAAAAAAAGA5AAAAAAAAAAAAAEAAOAAL
+
+------=_MIME_BOUNDARY_000_15328--
diff --git a/test/functional/messages/external_relay.eml b/test/functional/messages/external_relay.eml
new file mode 100644
index 0000000..14ac8ae
--- /dev/null
+++ b/test/functional/messages/external_relay.eml
@@ -0,0 +1,16 @@
+Return-path: root@external.com
+Received: from trusted.com (trusted.com [192.168.1.1]) by
+ example.com with LMTP id MJX+NoRd5F2caAAAzslS3g for <test@example.com>;
+ Thu, 5 Dec 2019 18:22:18 +0300
+Received: from external.com (external.com [37.48.67.26]) by
+ trusted.com (Postfix) with ESMTP id C018DA00021;
+ Thu, 5 Dec 2019 18:22:18 +0300
+To: test@example.com
+From: root@external.com
+Subject: test Sat, 26 Jan 2019 12:04:58 +0100
+Message-Id: <20190126120458.015328@srv.example.com>
+Date: Sat, 26 Jan 2019 12:04:58 +0100
+MIME-Version: 1.0
+Content-Type: multipart/mixed
+
+dsadas
diff --git a/test/functional/messages/f.zip.001.eml b/test/functional/messages/f.zip.001.eml
new file mode 100644
index 0000000..f5d5ceb
--- /dev/null
+++ b/test/functional/messages/f.zip.001.eml
@@ -0,0 +1,11 @@
+Content-Type: multipart/mixed; boundary="------------CB783608BCB68F7F91E9F2A1"
+
+--------------CB783608BCB68F7F91E9F2A1
+Content-Type: application/octet-stream; name="f.zip.001"
+Content-Disposition: attachment; filename="f.zip.001"
+Content-Transfer-Encoding: base64
+
+UEsDBAoAAAAAAGVXsUwAAAAAAAAAAAAAAAAFAAAAZi50eHRQSwECPwAKAAAAAABlV7FMAAAA
+AAAAAAAAAAAABQAkAAAAAAAAACAgAAAAAAAAZi50eHQKACAAAAAAAAEAGAC4V3vvtO3TAbhX
+e++07dMBuFd777Tt0wFQSwUGAAAAAAEAAQBXAAAAIwAAAAAA
+--------------CB783608BCB68F7F91E9F2A1--
diff --git a/test/functional/messages/f.zip.gz.eml b/test/functional/messages/f.zip.gz.eml
new file mode 100644
index 0000000..afb1444
--- /dev/null
+++ b/test/functional/messages/f.zip.gz.eml
@@ -0,0 +1,11 @@
+Content-Type: multipart/mixed; boundary="------------283A5F1FAE533F3FB6BF6ED3"
+
+--------------283A5F1FAE533F3FB6BF6ED3
+Content-Type: application/octet-stream; name="f.zip.gz"
+Content-Disposition: attachment; filename="f.zip.gz"
+Content-Transfer-Encoding: base64
+
+H4sICPY5/VoEAGYuemlwAHVLuQmAQBCcU8/gKjA80NgWLGANNtsKtIILBKsSDOzDOsQWXE8F
+EZxl54EZpjRzONHJ1OIFq9/XYQhMJmnwV6pu573SM3G4kkGBRcZ93lbzVSabx46eqJZxcQAU
+cyqqkAAAAA==
+--------------283A5F1FAE533F3FB6BF6ED3--
diff --git a/test/functional/messages/fname.eml b/test/functional/messages/fname.eml
new file mode 100644
index 0000000..045940c
--- /dev/null
+++ b/test/functional/messages/fname.eml
@@ -0,0 +1,29 @@
+Received: by mail-lf1-f54.gulugulu.com with SMTP id j14so14439709lfg.9
+ for <test@test.ru>; Mon, 27 Apr 2020 09:54:21 -0700 (PDT)
+From: test@test.com
+Date: Mon, 27 Apr 2020 19:54:10 +0300
+Message-ID: <CA+1S=h4aGimA6vSBJF=t1F+5z-Mua5+Cimf+NU_NDWJk8ZNOcw@mail.gmail.com>
+Subject: Fwd:
+To: <test@test.ru>
+Content-Type: multipart/mixed; boundary="00000000000004de7805a4489190"
+
+--00000000000004de7805a4489190
+Content-Type: application/octet-stream
+Content-Disposition: attachment;
+ filename="=?UTF-8?B?W+yCvOyEseyDneuqhV0yMDIwLjA4Lg==?=
+ =?UTF-8?B?MTQg642w7J2866asIOqyveygnOuJtOyKpC5wZGY=?="
+Content-Transfer-Encoding: base64
+
+yCvOy
+
+--00000000000004de7805a4489190
+Content-Type: application/pdf;
+ name==?utf-8?B?MDEwMjlfNDAyMTEwXzEwNjIwX1JHVDA2OTAyX1BSVDE4ME1MXzIwMjAwODAzXzEwMTgyMC5wZGY=?=
+Content-Disposition: attachment;
+ filename==?utf-8?B?MDEwMjlfNDAyMTEwXzEwNjIwX1JHVDA2OTAyX1BSVDE4ME1MXzIwMjAwODAzXzEwMTgyMC5wZGY=?=
+Content-Transfer-Encoding: base64
+
+yCvOy
+
+
+--00000000000004de7805a4489190-- \ No newline at end of file
diff --git a/test/functional/messages/freemail.eml b/test/functional/messages/freemail.eml
new file mode 100644
index 0000000..ab1b90e
--- /dev/null
+++ b/test/functional/messages/freemail.eml
@@ -0,0 +1,9 @@
+From: <faked.asdfjisiwosp372@outlook.com>
+Subject: Reply me at faked.asdfjisiwosp372@hotmail.com
+Date: Sun, 31 Jul 2016 11:40:08 +0100
+X-Foo: 1
+X-Bar: 1
+Content-type: text/plain
+MIME-Version: 1.0
+
+hello
diff --git a/test/functional/messages/freemailcc.eml b/test/functional/messages/freemailcc.eml
new file mode 100644
index 0000000..6926c6e
--- /dev/null
+++ b/test/functional/messages/freemailcc.eml
@@ -0,0 +1,10 @@
+Content-Type: multipart/mixed; boundary="===============3209040583106423479=="
+MIME-Version: 1.0
+From: test@example.org
+To: test@example.org
+CC: <testp@test.com>, <testp1@test1.com>, <testp2@test2.com>,<testp3@test3.com>,<testp4@test4.com>,<testp5@test5.com>,<testp6@test6.com>,<testp7@test7.com>,<testp8@test8.com>,<testp9@test9.com>,<testp10@test10.com>,<testp11@test11.com>,<testp12@test12.com>,<testp13@test13.com>,<testp14@test14.com>,<testp15@test.com>,<testp16@test.com>,<testp17@test.com>,<testp18@test.com>,<test19@test.com>
+Date: Thu, 26 Sep 2019 15:52:32 -0000
+Subject: Test message
+Message-ID: <156951315267.43830.14912869070271436609@example.org>
+
+skdjaskdjasd \ No newline at end of file
diff --git a/test/functional/messages/from/from.eml b/test/functional/messages/from/from.eml
new file mode 100644
index 0000000..20c2b1b
--- /dev/null
+++ b/test/functional/messages/from/from.eml
@@ -0,0 +1,2 @@
+From: user@example.org
+
diff --git a/test/functional/messages/from/from_comment.eml b/test/functional/messages/from/from_comment.eml
new file mode 100644
index 0000000..235ccf1
--- /dev/null
+++ b/test/functional/messages/from/from_comment.eml
@@ -0,0 +1,2 @@
+From: user@example.org (Comment text)
+
diff --git a/test/functional/messages/from/from_dn.eml b/test/functional/messages/from/from_dn.eml
new file mode 100644
index 0000000..fd714c2
--- /dev/null
+++ b/test/functional/messages/from/from_dn.eml
@@ -0,0 +1,2 @@
+From: First Last <user@example.org>
+
diff --git a/test/functional/messages/from/from_dn_base64.eml b/test/functional/messages/from/from_dn_base64.eml
new file mode 100644
index 0000000..e4fdbf2
--- /dev/null
+++ b/test/functional/messages/from/from_dn_base64.eml
@@ -0,0 +1,2 @@
+From: =?UTF-8?B?0JrQuNGA0LjQu9C70LjRhtCw?= <user@example.org>
+
diff --git a/test/functional/messages/from/from_dn_comment.eml b/test/functional/messages/from/from_dn_comment.eml
new file mode 100644
index 0000000..5f8b339
--- /dev/null
+++ b/test/functional/messages/from/from_dn_comment.eml
@@ -0,0 +1,2 @@
+From: First Last <user@example.org> (Comment text)
+
diff --git a/test/functional/messages/from/from_quoted_dn.eml b/test/functional/messages/from/from_quoted_dn.eml
new file mode 100644
index 0000000..8e0d01e
--- /dev/null
+++ b/test/functional/messages/from/from_quoted_dn.eml
@@ -0,0 +1,2 @@
+From: "First M. Last" <user@example.org>
+
diff --git a/test/functional/messages/from/from_quoted_dn_comment.eml b/test/functional/messages/from/from_quoted_dn_comment.eml
new file mode 100644
index 0000000..eddba68
--- /dev/null
+++ b/test/functional/messages/from/from_quoted_dn_comment.eml
@@ -0,0 +1,2 @@
+From: "First M. Last" <user@example.org> (Comment text)
+
diff --git a/test/functional/messages/from/from_quoted_dn_middle.eml b/test/functional/messages/from/from_quoted_dn_middle.eml
new file mode 100644
index 0000000..98ddac4
--- /dev/null
+++ b/test/functional/messages/from/from_quoted_dn_middle.eml
@@ -0,0 +1,2 @@
+From: First "M." Last <user@example.org>
+
diff --git a/test/functional/messages/from/from_quoted_dn_middle_inner.eml b/test/functional/messages/from/from_quoted_dn_middle_inner.eml
new file mode 100644
index 0000000..4e7f363
--- /dev/null
+++ b/test/functional/messages/from/from_quoted_dn_middle_inner.eml
@@ -0,0 +1,2 @@
+From: First" M. "Last <user@example.org>
+
diff --git a/test/functional/messages/fws_fn.eml b/test/functional/messages/fws_fn.eml
new file mode 100644
index 0000000..67601be
--- /dev/null
+++ b/test/functional/messages/fws_fn.eml
@@ -0,0 +1,30 @@
+From: <admin@aaa.example.com>
+To: <ragamuffin@bbb.example.com>
+Subject: Test content type
+Date: Tue, 17 Jul 2018 18:06:01 +0200 (CEST)
+Message-ID: <20180717160601.B25511E37E7@aaa.example.com>
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="sd8f70as98f70as98f70as98f70as98df70as98df7as90f87as90d8f7"
+
+--sd8f70as98f70as98f70as98f70as98df70as98df7as90f87as90d8f7
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/html; charset=utf-8
+Content-Disposition: inline
+
+<html>
+<body style=3D"font-family:arial">
+Hello world
+</body>
+</html>
+
+--sd8f70as98f70as98f70as98f70as98df70as98df7as90f87as90d8f7
+Content-ID: <qrcode>
+Content-Transfer-Encoding: base64
+Content-Type: image/jpeg
+Content-Disposition: inline
+
+alskfhaslkfjhlaskfjhlaskjfhlaskjfhlaksjfhklasjfhklasdjfhlask
+lksjdfhalskjfhklasjfhlaskfhlaskfhklasfhklasdjfhlkasdjfhklasd
+
+--sd8f70as98f70as98f70as98f70as98df70as98df7as90f87as90d8f7--
+
diff --git a/test/functional/messages/fws_fp.eml b/test/functional/messages/fws_fp.eml
new file mode 100644
index 0000000..4c80969
--- /dev/null
+++ b/test/functional/messages/fws_fp.eml
@@ -0,0 +1,56 @@
+From: Sender <sender@example.com>
+Mime-Version: 1.0 (1.0)
+Date: Fri, 5 Oct 2018 19:56:40 -0400
+Message-Id: <E7015E63-E006-4E30-9313-851CB7F2424E>
+To: Receiver <receiver@example.com>
+X-Spam_report: Spam detection software, running on the system "www.kamailio.org",
+ has NOT identified this incoming email as spam. The original
+ message has been attached to this so you can view it or label
+ similar future email. If you have any questions, see
+ the administrator of that system for details.
+
+ Content preview: Here an old one from 5 years ago. The principles remain
+ the same. https://www.fredposner.com/1457/kamailio-behind-nat/ -- fred [...]
+
+
+ Content analysis details: (-1.9 points, 5.5 required)
+
+ pts rule name description
+ ---- ---------------------- --------------------------------------------------
+ -1.9 BAYES_00 BODY: Bayes spam probability is 0 to 1%
+ [score: 0.0000]
+ 0.0 HTML_MESSAGE BODY: HTML included in message
+ 0.0 MIME_QP_LONG_LINE RAW: Quoted-printable line longer than 76 chars
+List-Id: yYY
+Subject: XXX
+
+--===============1364639178==
+Content-Type: multipart/alternative;
+ boundary=Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461
+Content-Transfer-Encoding: 7bit
+
+
+--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461
+Content-Type: text/plain;
+ charset=utf-8
+Content-Transfer-Encoding: quoted-printable
+
+xxx
+
+--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461
+Content-Type: text/html;
+ charset=utf-8
+Content-Transfer-Encoding: quoted-printable
+
+yyy
+--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461--
+
+
+--===============1364639178==
+Content-Type: text/plain; charset="utf-8"
+MIME-Version: 1.0
+Content-Transfer-Encoding: base64
+Content-Disposition: inline
+
+zzzz
+--===============1364639178==--
diff --git a/test/functional/messages/fws_tp.eml b/test/functional/messages/fws_tp.eml
new file mode 100644
index 0000000..4c80969
--- /dev/null
+++ b/test/functional/messages/fws_tp.eml
@@ -0,0 +1,56 @@
+From: Sender <sender@example.com>
+Mime-Version: 1.0 (1.0)
+Date: Fri, 5 Oct 2018 19:56:40 -0400
+Message-Id: <E7015E63-E006-4E30-9313-851CB7F2424E>
+To: Receiver <receiver@example.com>
+X-Spam_report: Spam detection software, running on the system "www.kamailio.org",
+ has NOT identified this incoming email as spam. The original
+ message has been attached to this so you can view it or label
+ similar future email. If you have any questions, see
+ the administrator of that system for details.
+
+ Content preview: Here an old one from 5 years ago. The principles remain
+ the same. https://www.fredposner.com/1457/kamailio-behind-nat/ -- fred [...]
+
+
+ Content analysis details: (-1.9 points, 5.5 required)
+
+ pts rule name description
+ ---- ---------------------- --------------------------------------------------
+ -1.9 BAYES_00 BODY: Bayes spam probability is 0 to 1%
+ [score: 0.0000]
+ 0.0 HTML_MESSAGE BODY: HTML included in message
+ 0.0 MIME_QP_LONG_LINE RAW: Quoted-printable line longer than 76 chars
+List-Id: yYY
+Subject: XXX
+
+--===============1364639178==
+Content-Type: multipart/alternative;
+ boundary=Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461
+Content-Transfer-Encoding: 7bit
+
+
+--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461
+Content-Type: text/plain;
+ charset=utf-8
+Content-Transfer-Encoding: quoted-printable
+
+xxx
+
+--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461
+Content-Type: text/html;
+ charset=utf-8
+Content-Transfer-Encoding: quoted-printable
+
+yyy
+--Apple-Mail-C9403573-708A-4D69-BF6C-F2730C368461--
+
+
+--===============1364639178==
+Content-Type: text/plain; charset="utf-8"
+MIME-Version: 1.0
+Content-Transfer-Encoding: base64
+Content-Disposition: inline
+
+zzzz
+--===============1364639178==--
diff --git a/test/functional/messages/gargantua.eml b/test/functional/messages/gargantua.eml
new file mode 100644
index 0000000..c6edc50
--- /dev/null
+++ b/test/functional/messages/gargantua.eml
@@ -0,0 +1,23505 @@
+From: <user@example.com>
+To: undisclosed-recipients:;
+Content-Type: multipart/mixed; boundary=XXX
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: zip
+
+UEsDBBQDAAAAADxGUzwAAAAAAAAAAAAAAAACAAAAdC9QSwMECgMAAAAAHUZTPIiw
+JDICAAAAAgAAAAcAAAB0L3QudHh0NDJQSwECPwMUAwAAAAA8RlM8AAAAAAAAAAAA
+AAAAAgAAAAAAAAAAABCA6EEAAAAAdC9QSwECPwMKAwAAAAAdRlM8iLAkMgIAAAAC
+AAAABwAAAAAAAAAAACCAoIEgAAAAdC90LnR4dFBLBQYAAAAAAgACAGUAAABHAAAA
+AAA=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: rar
+
+UmFyIRoHAM+QcwAADQAAAAAAAABs53QggCcACwAAAAIAAAADiLAkMh0+UzwdMwcA
+oIEAAHRcdC50eHQRC/TEnu9/Np/2WmMfdOCAIQAAAAAAAAAAAAMAAAAAPD5TPBQw
+AQDoQQAAdMQ9ewBABwA=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: exe
+
+TVqQAAMAAAAEAAAA//8AALgAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAEAAA4fug4AtAnNIbgBTM0hVGhpcyBwcm9ncmFtIGNhbm5v
+dCBiZSBydW4gaW4gRE9TIG1vZGUuDQ0KJAAAAAAAAABONUIfClQsTApULEwKVCxM
+Ayy/TAZULEwUBr9MCFQsTDEKLU0OVCxMMQovTQhULEwxCilNFFQsTDEKKE0GVCxM
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: elf
+
+f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAA8A5AAAAAAABAAAAAAAAAAAgiAAAAAAAA
+AAAAAEAAOAAJAEAAHQAcAAYAAAAFAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAA
++AEAAAAAAAD4AQAAAAAAAAgAAAAAAAAAAwAAAAQAAAA4AgAAAAAAADgCQAAAAAAA
+OAJAAAAAAAAcAAAAAAAAABwAAAAAAAAAAQAAAAAAAAABAAAABQAAAAAAAAAAAAAA
+AABAAAAAAAAAAEAAAAAAAJwbAAAAAAAAnBsAAAAAAAAAACAAAAAAAAEAAAAGAAAA
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: lnk
+
+TAAAAAEUAgAAAAAAwAAAAAAAAEb7QAgAICAAAGgu1hzcM9MBaC7WHNwz0wF/qu8c
+3DPTAQDYIAAAAAAAAQAAAAAAAAAAAAAAAAAAAMkBFAAfUOBP0CDqOmkQotgIACsw
+MJ0ZAC9DOlwAAAAAAAAAAAAAAAAAAAAAAAAAXAAxAAAAAAB8SiliEABERVZFTE9+
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: class
+
+yv66vgAAADQBCQcAZgcAZwcAaAoAQQBpCQBqAGsHAGwKAAYAaQgAbQoABgBuCgBv
+AHAIAHEKAAYAcgoAcwB0CgA9AHUKAD0AdgkAPQB3CgB4AHkKAAEAegoAAQB7CgAB
+AHwKAH0AfgkAagB/CgB9AIAKAH0AgQoAAQCCBwCDCACECgAaAIUKAAEAhgMAAgAE
+CgABAIcDAAIAAwgAiAoAAQCJBwCKCACLCgAjAIUSAAAAkAoAAQCRCgCSAJMKAJIA
+lAoAAQCVCgABAJYKAAEAlwoAmACZCgCaAJsKAJgAnAoAAQCdCgCSAJ4HAJ8KADIA
+oAoAAQChCgABAKIKAAEAowoApAClCgACAKYKAAEApwoAAgCoCgABAKkKAAEAqgcA
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: rtf
+
+e1xydGYxXGFuc2lcYW5zaWNwZzEyNTFcY29jb2FydGYxNjcxXGNvY29hc3VicnRm
+MjAwCntcZm9udHRibFxmMFxmbmlsXGZjaGFyc2V0MCBIZWx2ZXRpY2FOZXVlO30K
+e1xjb2xvcnRibDtccmVkMjU1XGdyZWVuMjU1XGJsdWUyNTU7XHJlZDBcZ3JlZW4w
+XGJsdWUwO30Ke1wqXGV4cGFuZGVkY29sb3J0Ymw7O1xjc3NyZ2JcYzBcYzBcYzA7
+fQpccGFwZXJ3MTE5MDVccGFwZXJoMTY4MzdcbWFyZ2wxMTMzXG1hcmdyMTEzM1xt
+YXJnYjExMzNcbWFyZ3QxMTMzClxkZWZ0YWI3MjAKXHBhcmRccGFyZGVmdGFiNzIw
+XHBhcnRpZ2h0ZW5mYWN0b3IwCgpcZjBcZnMyMiBcY2YyIFxleHBuZDBcZXhwbmR0
+dzBca2VybmluZzAKXHVwMCBcbm9zdXBlcnN1YiBcdWxub25lIHRlc3R9
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: pdf
+
+JVBERi0xLjMKJcTl8uXrp/Og0MTGCjQgMCBvYmoKPDwgL0xlbmd0aCA1IDAgUiAv
+RmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeAFVjs0KwkAMhO99ijnqwXTT
+7l+uig9QWPABSotIK9Tt+2O2VlDmkEyY8M2CDguMyomjJiJapih4DbjhifqSGX0G
+b8q95vbArA/+a6ZiqpMhNiLRWUwa/LN3jBvJtWSjsMB58tJIAxsjtYYDQssUTCjs
+6pddyhkoe9ESZWU4TQePfsY5gffrZ6QZdUqlbxpxWIe8Hqv0wDUpvnsD+yAz+Qpl
+bmRzdHJlYW0KZW5kb2JqCjUgMCBvYmoKMTYwCmVuZG9iagoyIDAgb2JqCjw8IC9U
+eXBlIC9QYWdlIC9QYXJlbnQgMyAwIFIgL1Jlc291cmNlcyA2IDAgUiAvQ29udGVu
+dHMgNCAwIFIgL01lZGlhQm94IFswIDAgNTk1LjI4IDg0MS44OV0KPj4KZW5kb2Jq
+CjYgMCBvYmoKPDwgL1Byb2NTZXQgWyAvUERGIC9UZXh0IF0gL0NvbG9yU3BhY2Ug
+PDwgL0NzMSA3IDAgUiA+PiAvRm9udCA8PCAvVFQxIDggMCBSCj4+ID4+CmVuZG9i
+agoxMCAwIG9iago8PCAvTGVuZ3RoIDExIDAgUiAvTiAzIC9BbHRlcm5hdGUgL0Rl
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: ps
+
+JSFQUy1BZG9iZS0zLjAKJSVCb3VuZGluZ0JveDogMCAwIDU5NiA4NDIKJSVIaVJl
+c0JvdW5kaW5nQm94OiAwIDAgNTk2LjAwIDg0Mi4wMAolJUNyZWF0b3I6IEdQTCBH
+aG9zdHNjcmlwdCA5MjMgKHBzMndyaXRlKQolJUxhbmd1YWdlTGV2ZWw6IDIKJSVD
+cmVhdGlvbkRhdGU6IEQ6MjAxOTA5MTAxNTUxNDAtMDQnMDAnCiUlUGFnZXM6IDEK
+JSVFbmRDb21tZW50cwolJUJlZ2luUHJvbG9nCi9EU0NfT1BERlJFQUQgdHJ1ZSBk
+ZWYKL1NldFBhZ2VTaXplIHRydWUgZGVmCi9FUFMyV3JpdGUgZmFsc2UgZGVmCmN1
+cnJlbnRkaWN0L0RTQ19PUERGUkVBRCBrbm93bnsKY3VycmVudGRpY3QvRFNDX09Q
+REZSRUFEIGdldAp9ewpmYWxzZQp9aWZlbHNlCjEwIGRpY3QgYmVnaW4KL0RTQ19P
+UERGUkVBRCBleGNoIGRlZgovdGhpcyBjdXJyZW50ZGljdCBkZWYKL3kgNzIwIGRl
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: chm
+
+SVRTRgMAAABgAAAAAQAAAP0XYzoZBAAAEP0BfKp70BGeDACgySLm7BH9AXyqe9AR
+ngwAoMki5uxgAAAAAAAAABgAAAAAAAAAeAAAAAAAAABUUAAAAAAAAMxQAAAAAAAA
+/gEAAAAAAAAcLhEAAAAAAAAAAAAAAAAASVRTUAEAAABUAAAACgAAAAAQAAACAAAA
+AgAAAAQAAAAAAAAAAwAAAP////8FAAAACQQAAGqSAl0uIdARnfkAoMki5uxUAAAA
+////////////////UE1HTEkAAAAAAAAA/////wEAAAABLwAAAAgvI0lEWEhEUgGB
+6swxoAAILyNJVEJJVFMAAAAJLyNTVFJJTkdTAYHsjyvAFggvI1NZU1RFTQCIZqEk
+CC8jVE9QSUNTAYHq7DGtUAgvI1VSTFNUUgGB67wh0woILyNVUkxUQkwBgeuaAaIg
+CS8jV0lORE9XUwGB2d9LgUwLLyRGSWZ0aU1haW4Bgdn2OpDVdwkvJE9CSklOU1QB
+gdnhH5UbFS8kV1dBc3NvY2lhdGl2ZUxpbmtzLwAAAB0vJFdXQXNzb2NpYXRpdmVM
+aW5rcy9Qcm9wZXJ0eQGB2eEbBBEvJFdXS2V5d29yZExpbmtzLwAAABkvJFdXS2V5
+d29yZExpbmtzL1Byb3BlcnR5AYHZ4RcEEy9hdHRyaWJ1dGVzL2NvbW1vbi8AAAAd
+L2F0dHJpYnV0ZXMvY29tbW9uL2NsYXNzLmh0bWwBgZyFXdFxGy9hdHRyaWJ1dGVz
+L2NvbW1vbi9kaXIuaHRtbAGBnNdOzWAaL2F0dHJpYnV0ZXMvY29tbW9uL2lkLmh0
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: djvu
+
+QVQmVEZPUk0AAAGbREpWTURJUk0AAAAxgQACAAAASgAAAND//9u/ih/u3ePF0n9P
+OwFAIkp1Fcic8jvB6ayx8f/hmzcHYxXSvgBGT1JNAAAAfkRKVklBTlR6AAAAcv//
+f/qCnEyUkAXNM/uS8nlcL5SEMEad4fYMQbEUR44QP061lwFuQ8u1dD8yNXuqCRCT
+QBGF0b4QCavU2EBXjz83iubJS/TeFRz40flTspDH1dKH3U/KWn+rAWdY48jn4WlC
+di99wPFHJzy97Q2f7q3mb0ZPUk0AAADPREpWVUlORk8AAAAKCbANtBgALAEWAUlO
+Q0wAAAAPc2hhcmVkX2Fubm8uaWZmAFNqYnoAAABhgA5O8V17OKD69sKvjqrxziCP
+g/qwr2IIM+L1Jv3MriWMAXQwujsmDr7TVB210C2b2DYi8zGnvUc+StN4zrGSybZN
+CTFU6uPtEsEy3VpiTyTUdQ+kEfD2DbSiKEJ2p+IpPwBUWFR6AAAAL///wYhkQEiG
+UlIQfW3qIrwtNSJbr3okngkEERJiw8wUjiMI78B5f+3uflgmfDST
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: arj
+
+YOopACILAQIQAAKAAaa+TwGmvk8AAAAAAAAAAAAAAAAAAAAAAAB0LmFyagAAAhNE
+hgAAYOo3AC4LAQIAAAABWjR+SwIAAAACAAAAiLAkMgIAoBEAAAAAAAATlr5P7ZW+
+TwAAAAB0L3QudHh0AADRIsLVAAA0MmDqAAA=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: cab
+
+TVNDRgAAAABOAAAAAAAAACwAAAAAAAAAAwEBAAEAAADSBAAARAAAAAEAAAACAAAA
+AAAAAAAAUzwdPiAAdFx0LnR4dAAwNAIAAgACADQy
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: ace
+
+u0UxAAAAkCoqQUNFKioUFAIAIW8EQaY44CgAAAAAFipVTlJFR0lTVEVSRUQgVkVS
+U0lPTir1uiAAAQEAAAAAAAAAAACSbQRBEAAAAP////8CAwoAVEUBAHRdMiYAAQGA
+AgAAAAIAAADabQRBIAAAAHdP280AAwoAVEUHAHRcdC50eHQ0Mg==
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: tar
+
+dC8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAADAwMDA3NTAAMDAwMTc1MAAwMDAxNzUwADAwMDAwMDAwMDAwADExMzM3NDMy
+MjI0ADAxMDYyNwAgNQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAB1c3RhciAgAGNhbHZpbgAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAY2FsdmluAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: bz2
+
+QlpoOTFBWSZTWaak/tgAAKX7hMGAAQFAAf+AACFoJR9AAAIACCAAkQ0hNNNNHqaM
+gMm0EkIJNqNMjIMTJ7Odr7YzJpAYVQkhvFBZQWgmGAwkDjJzUc6qLBc1AWelRQDD
+B5ZxrQnCsZ5No0D0mMSJS5xOzIp0vrcSCEfQ7jmhwGobAxwE+pug/sEAMwRAXF3J
+FOFCQpqT+2A=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: xz
+
+/Td6WFoAAATm1rRGAgAhARYAAAB0L+WjAQABNDIAAADkcm+njl2JkQABGgLcLqV+
+H7bzfQEAAAAABFla
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: lz4
+
+bW96THo0MACJCQAA8g97ImFwcC1zeXN0ZW0tZGVmYXVsdHMiOnsiYWRkb24KAPAF
+dXNoZWxwZXJAbW96aWxsYS5vcmcjAP8nZW5hYmxlZCI6dHJ1ZSwibGFzdE1vZGlm
+aWVkVGltZSI6MTUyMDcwMDc5ODAwMCwicGF0aCI6UAAD9hIueHBpIiwidmVyc2lv
+biI6IjIuMCIsImJvb3RzdHJhcHBoAPQQZGVwZW5kZW5jaWVzIjpbXSwicnVuSW5T
+YWZlTW9kZY8A8gRoYXNFbWJlZGRlZFdlYkV4dGVuWwD/DmZhbHNlfSwiZmlyZWZv
+eEBnZXRwb2NrZXQuY29t2AAoD1AAAg3YAF8xLjAuNdoATL9zY3JlZW5zaG90c7QB
+NA9SAAQN3gBPMjUuMLkBTp93ZWJjb21wYXTdADQPUAACDdsAPzEuMbcBTOVmb2xs
+b3dvbnNlYXJjaN0AD5YCLA9VAAYN4gBfMC45LjbkAEz/BHNoaWVsZC1yZWNpcGUt
+Y2xpZW7HATUPWwANDfAAHzhiBE31AGFjdGl2aXR5LXN0cmVhbdIBD0AFLA9WAAcN
+4wD/CDIwMTguMDIuMTcuMDAyNi0xNzNlMjc5gQRNz2Zvcm1hdXRvZmlsbPYANA9T
+AAUN8wAfMYAETq9vbmJvYXJkaW5n3AA0D1EAAw/aAF4VfaMH8gAvQXBwbGljYXRp
+b25zL0YpB/EcLmFwcC9Db250ZW50cy9SZXNvdXJjZXMvYnJvd3Nlci9mZWF0dXJl
+cyJ9LFUIYGdsb2JhbFEHBkwI8Bd7OTcyY2U0YzYtN2UwOC00NDc0LWEyODUtMzIw
+ODE5OGNlNmZkfTQAD10IJA9hABMNWwEhNTk2At90eXBlIjoidGhlbWUiEAEtFGVz
+CPIBcyIsImNoZWNrU3RhcnR1cCUJA14BgCI6dHJ1ZX19
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: zst
+
+KLUv/WQ/IdW4AEq9bVQW8AA+/P//+QHAUCRmi9/xBp6BZ5CMvj8FQgU9BdRkBFC0
+xLwukdsjh9HhvefUZhZ5bLj6iPJ0Fm5oswmFYBwpZ/LqfGKbkKSYYQFLLtoyJoP2
+J0VKU6fpdCZyEmnK5BuYGuGqIPWTsyHOLrs60NRulJht1DUwK8FPBwKcbCNOEtME
+FYtxFVbza1hdTTNM7dryh8JuInt77Z7xI9rMIUt17I7zVNtQS3MajXk15MGR39XI
+QceRq0PsDtE8BNXYAmRIj2I76cutG5eBtK3RAZK1LW7eOO9xYEsDe4hzNE4iuKRl
+xeBs2uKJEwg2HdieYDpsHIDH5UxHSRseB/nJbjjKqaCrGaFmlRy0bFCjor+oyVXH
+y5qArw3WkBqjj+CnYfPNsS6nieWmSvxG1Jqocqq2adwDag6s8TuI7d2GRzyWoBua
+SBs48CqOxzvkeGXnckUXhZkKMk4BdKPoaxfZRXOeUlZ9UKz//RkTzt1pF1z86Zte
+vI+5CRTmPWGMryCH2/YWwLHaC0xAcm0AwpTuejNVxolC3YwEGJFwoNoAJ1+cG7IA
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: dmg
+
+eAHt3WuMHVd9APC5d73eh58xMQQI8Sax88TLOlnbSRyIH4lJ20CtJER84FHHj9TU
+sanj0IRE1I1oeBTRCLW0olEVtUGNVDUC1FZBonGgtKoqSimiEa4QHyKKEAIJVULy
+h0ju/z8zd+/d3bvrtX3xruF3rDMzd/bMzJnfnHmcMw8XhUCAwK+qwJ3XF4uOFY1i
+w9joWBqceOw9L5947Phz1fDxndFfH3FJUayMXn/ERWuKonEiJolexKK4M+Lmcqg1
+pv7R2bv3A4d//2/f/x+vVuNyXhkn5lLNqfrjWXSfiWkyFsMRm3U/f59daGyN6Z7J
+aS+O+JW6n7/PLrStcqWXRixX/uxmZioCBAgQIECAAAECZy9w6tSpn3WZesWpUx1j
+b4tU08NLHSl+UYPnWC/4RWXLfAkQIECAAAECBAgQIECAwIUpEBXtvm45jxtVeU8t
+41D+/ZnsVPfYqqELtHvsAs23bBMgQIAAAQIECBAgQIAAgTMRWHGyOFk8Uiw/k2mk
+JUCAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECA
+AAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAEC
+BAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQ
+IECAAAECBAgQIECAAAECF7pAoyj6Yh2iV8Zma3ikaAzH8GDExfW46JXD2b9gw0sX
+bM5lnAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQItAXWnCxOFo8Uy1tj
+TjWKvOc/Xv6+uO7uKQ4XDxajxe7ig/HvYLEvhn+3eDh+H4l4qDhaHIjuvki9spyi
+6tw8NnbTjVs27t04vn/Tng1b7tm9f/eRA1t2btq+c/vGHRvX7xjbNLZ+fPOm29dv
+37FhW3Ru3rh9286xG7fdsfnFevmXnNPyXxfL35TL37hp49j+Lffue+ToHXsPHN3y
+/Y48GiRAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAr8sAj8pPt2+///L
+slLWgwABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBA
+gAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAAB
+AgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQI
+ECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBA
+gAABAgQIECBAgAABAgQIEDPZc9V6zaXsTJzJ2vt8NfHaqpcmjeraqB4x773cA18T
+sTr3nk4pU2erXJX6zM6DOW0eZ1r7+tmcGWJygQABAgTKSp/iX9hx2HWdQ80Nfi+e
+6vTKiqa/9bzZ+kLbZw7XzPzxH4V7ra5vXWF76dPYgsYn5zv/HDGjRb2C2RE5A6zb
+JB757s6Kui+4RNR2bqurvWlCt5ycAzlFZyNf2p6R/U2asr32mshTxz1+y7Rp23hx
++YufZ03xeLa496h5dpc+2OewvzT74m73a+dMPVeZ+bUWeEIJ+OfU/LZP/dsmpuZY
+i4AWAS0CWgS0CGgR0CKgRUCLgBYBLQKPOQL3v/93qn/j9UXLmumjD+3bl7/Qf+mw
+3MUB3ktXBC9duWTpm8PnBB4YlepiHp7c9sPm+T1bTqqTsVL+4UC+CH19wnis+2Pu
+luZOi4AWAS0CWgS0CGgR0CKgRUCLgBYBLQJaBLQIaBF4jBH4L+Elq2B4AfPQZmBp
+YGBgMDTQMwBSDDerY67drN4/D8Le7waktYGYm4FBAEixAjGLIgMD4zUGRiAThBkY
+QGqMwSyYCJSDTIVk5devjj/7ByIGMguE4aZATIJIkkEuAOoBYQYuIGaC0iA+eYDR
+AahvAUivCBDvgdIgPnkAEVYgT/MAMdjz5BlGXV0AlpQS9Tw/eG1sIHZlcnNpb249
+IjEuMCIgZW5jb2Rpbmc9IlVURi04Ij8+CjwhRE9DVFlQRSBwbGlzdCBQVUJMSUMg
+Ii0vL0FwcGxlLy9EVEQgUExJU1QgMS4wLy9FTiIgImh0dHA6Ly93d3cuYXBwbGUu
+Y29tL0RURHMvUHJvcGVydHlMaXN0LTEuMC5kdGQiPgo8cGxpc3QgdmVyc2lvbj0i
+MS4wIj4KPGRpY3Q+Cgk8a2V5PnJlc291cmNlLWZvcms8L2tleT4KCTxkaWN0PgoJ
+CTxrZXk+YmxreDwva2V5PgoJCTxhcnJheT4KCQkJPGRpY3Q+CgkJCQk8a2V5PkF0
+dHJpYnV0ZXM8L2tleT4KCQkJCTxzdHJpbmc+MHgwMDUwPC9zdHJpbmc+CgkJCQk8
+a2V5PkNGTmFtZTwva2V5PgoJCQkJPHN0cmluZz53aG9sZSBkaXNrIChBcHBsZV9I
+RlMgOiAwKTwvc3RyaW5nPgoJCQkJPGtleT5EYXRhPC9rZXk+CgkJCQk8ZGF0YT4K
+CQkJCWJXbHphQUFBQUFFQUFBQUFBQUFBQUFBQUFBQUFBQ2dBQUFBQUFBQUFBQUFB
+CgkJCQlBQWdJLy8vLy9nQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QQoJCQkJQUFJQUFBQWdrU3lpSWdBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUEKCQkJCUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBCgkJCQlBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQQoJCQkJQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUEKCQkJCUFBQUFBQUFHZ0FBQUJRQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+Z0FBQUFBCgkJCQlBQUFBQUFBQUFBQUFBQUFoWjRBQUFBVUFBQUFBQUFBQUFBQUFD
+QUFBQUFBQQoJCQkJQUFBRzBBQUFBQUFBQUNGbkFBQUFBQUFFNWt3QUFBQUNBQUFB
+QUFBQUFBQUEKCQkJCUFBN1FBQUFBQUFBQUdTNEFBQUFBQUFVSHN3QUFBQUFBQUFB
+QWdBQUFCUUFBCgkJCQlBQUFBQUFBQUFBQW4vZ0FBQUFBQUFBQUJBQUFBQUFBRkI3
+TUFBQUFBQUFBQQoJCQkJaVFBQUFBSUFBQUFBQUFBQUFBQUFKLzhBQUFBQUFBQUFB
+UUFBQUFBQUJRZzgKCQkJCUFBQUFBQUFBQUFELy8vLy9BQUFBQUFBQUFBQUFBQ2dB
+QUFBQUFBQUFBQUFBCgkJCQlBQUFBQUFVSVBBQUFBQUFBQUFBQQoJCQkJPC9kYXRh
+PgoJCQkJPGtleT5JRDwva2V5PgoJCQkJPHN0cmluZz4wPC9zdHJpbmc+CgkJCQk8
+a2V5Pk5hbWU8L2tleT4KCQkJCTxzdHJpbmc+d2hvbGUgZGlzayAoQXBwbGVfSEZT
+IDogMCk8L3N0cmluZz4KCQkJPC9kaWN0PgoJCTwvYXJyYXk+CgkJPGtleT5wbHN0
+PC9rZXk+CgkJPGFycmF5PgoJCQk8ZGljdD4KCQkJCTxrZXk+QXR0cmlidXRlczwv
+a2V5PgoJCQkJPHN0cmluZz4weDAwNTA8L3N0cmluZz4KCQkJCTxrZXk+RGF0YTwv
+a2V5PgoJCQkJPGRhdGE+CgkJCQlBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQQoJCQkJQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUEKCQkJCUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBCgkJCQlBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQQoJCQkJQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUEKCQkJCUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBCgkJCQlBQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQQoJCQkJQUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUEKCQkJCUFBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBCgkJCQlBQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQQoJCQkJQUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUEKCQkJCUFBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBCgkJCQlBQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQQoJCQkJQUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUEKCQkJCUFBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBCgkJCQlBQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFFQUFRQUFBQUFBQUFBQQoJCQkJQUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUEKCQkJCUFBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBCgkJCQlBQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQQoJCQkJQUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUEKCQkJCUFBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBCgkJCQlBQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQQoJCQkJQUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUEKCQkJCUFBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBCgkJCQlBQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQQoJCQkJQUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUEKCQkJCUFB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBCgkJCQlB
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQQoJCQkJ
+QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUEKCQkJ
+CUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBCgkJ
+CQlBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQQoJ
+CQkJQUFBQUFBQUFBQUFBCgkJCQk8L2RhdGE+CgkJCQk8a2V5PklEPC9rZXk+CgkJ
+CQk8c3RyaW5nPjA8L3N0cmluZz4KCQkJCTxrZXk+TmFtZTwva2V5PgoJCQkJPHN0
+cmluZz48L3N0cmluZz4KCQkJPC9kaWN0PgoJCTwvYXJyYXk+Cgk8L2RpY3Q+Cjwv
+ZGljdD4KPC9wbGlzdD4K+t4MwAAABqUAAAABAAEAAgAAABT63gsBAAAGkXM4Y2gB
+AAAA7wUAAEIAAAAwggXrMIIC/TCCAqSgAwIBAgIIclUd+WQ+lGgwCgYIKoZIzj0E
+AwIwcjEmMCQGA1UEAwwdQXBwbGUgU3lzdGVtIEludGVncmF0aW9uIENBIDQxJjAk
+BgNVBAsMHUFwcGxlIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MRMwEQYDVQQKDApB
+cHBsZSBJbmMuMQswCQYDVQQGEwJVUzAeFw0xOTA1MjEyMDEyMDFaFw0yMDA2MTky
+MDEyMDFaMEQxIDAeBgNVBAMMF1NvZnR3YXJlIFRpY2tldCBTaWduaW5nMRMwEQYD
+VQQKDApBcHBsZSBJbmMuMQswCQYDVQQGEwJVUzBZMBMGByqGSM49AgEGCCqGSM49
+AwEHA0IABHDNNnCJB+6LbFScw9wbegIxwh1bMlawKii6/eTMpMMzTLYivmYc5CXX
+OkBcDPDOl8wqG6vyr7ebVtco/v8nbNCjggFQMIIBTDAMBgNVHRMBAf8EAjAAMB8G
+A1UdIwQYMBaAFHpHujiKFSRIIkbNvo8aJHs0AyppMEEGCCsGAQUFBwEBBDUwMzAx
+BggrBgEFBQcwAYYlaHR0cDovL29jc3AuYXBwbGUuY29tL29jc3AwMy1hc2ljYTQw
+MjCBlgYDVR0gBIGOMIGLMIGIBgkqhkiG92NkBQEwezB5BggrBgEFBQcCAjBtDGtU
+aGlzIGNlcnRpZmljYXRlIGlzIHRvIGJlIHVzZWQgZXhjbHVzaXZlbHkgZm9yIGZ1
+bmN0aW9ucyBpbnRlcm5hbCB0byBBcHBsZSBQcm9kdWN0cyBhbmQvb3IgQXBwbGUg
+cHJvY2Vzc2VzLjAdBgNVHQ4EFgQUWJiwxGAi+C7YS4L8z8TpGrQSnIcwDgYDVR0P
+AQH/BAQDAgeAMBAGCiqGSIb3Y2QGAR4EAgUAMAoGCCqGSM49BAMCA0cAMEQCIBt/
+SCuYFRdMcmVzN499YKtXIVf6QVp5KI/CGtxWZYCvAiBZsJVnQD1zWPF5UisCOF9Q
+SkOkDd7pxyLtnEhPVhKyRDCCAuYwggJtoAMCAQICCDMN7vi/TGguMAoGCCqGSM49
+BAMDMGcxGzAZBgNVBAMMEkFwcGxlIFJvb3QgQ0EgLSBHMzEmMCQGA1UECwwdQXBw
+bGUgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxEzARBgNVBAoMCkFwcGxlIEluYy4x
+CzAJBgNVBAYTAlVTMB4XDTE3MDIyMjIyMjMyMloXDTMyMDIxODAwMDAwMFowcjEm
+MCQGA1UEAwwdQXBwbGUgU3lzdGVtIEludGVncmF0aW9uIENBIDQxJjAkBgNVBAsM
+HUFwcGxlIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MRMwEQYDVQQKDApBcHBsZSBJ
+bmMuMQswCQYDVQQGEwJVUzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABAZrpFZv
+fZ8n0c42jpIbVs1UNmRKyZRomfrJIH7i9VgP3OJq6xlHLy7vO6QBtAETRHxaJq2g
+nCkliuXmBm9PfFqjgfcwgfQwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBS7
+sN6hWDOImqSKmd6+veuv2sskqzBGBggrBgEFBQcBAQQ6MDgwNgYIKwYBBQUHMAGG
+Kmh0dHA6Ly9vY3NwLmFwcGxlLmNvbS9vY3NwMDMtYXBwbGVyb290Y2FnMzA3BgNV
+HR8EMDAuMCygKqAohiZodHRwOi8vY3JsLmFwcGxlLmNvbS9hcHBsZXJvb3RjYWcz
+LmNybDAdBgNVHQ4EFgQUeke6OIoVJEgiRs2+jxokezQDKmkwDgYDVR0PAQH/BAQD
+AgEGMBAGCiqGSIb3Y2QGAhEEAgUAMAoGCCqGSM49BAMDA2cAMGQCMBUMqY7Gr5Zp
+a6ef3VzUA1lsrlLUYMaLduC3xaLxCXzgmuNrseN8McQneqeOif2rdwIwYTMg8Sn/
++YcyrinIZD12e1Gk0gIvdr5gIpHx1Tp13LTixiqW/sYJ3EpP1STw/MqyZzh0awIA
+FAACAAAAAAAAAFpCWF0AAAAAAjQ0+XHEyXkEcFCJNTddNrK15GvbAnRTzD4dld2d
+luLerQWZXMriK6BOMEUCICmIlvFhKGNu6s04jAB2PxeVnM97S6tcOJWUghS/DDmG
+AiEAour1G4njQVp+4F//Y8APG3H05WO+FdLRUtjjkqnaCG4Aa29seQAAAAQAAAIA
+AAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAUIPAAAAAAAAAAAAAAAAAAAAAAAAAAB
+AAAAAXKqDF7XDkhTmJoyQtvBB3QAAAACAAAAIEEF4VkAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAFCDwAAAAAAAALvQAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAUT+QAAAAAAAAalAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAIAAAAgXL551gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC
+AAAAAAAAKAAAAAAAAAAAAAAAAAA=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: iso
+
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABQ0QwMDEBAEFQUExFIElO
+Qy4sIFRZUEU6IDAwMDIgICAgICAgICAgXwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAABoAAAAAAAAaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAABAAABAQAAAQAICAAKAAAAAAAAChQAAAAAAAAAAAAAEwAAAAAiABUA
+AAAAAAAVAAgAAAAACAB3CQsTNTAAAgAAAQAAAQEAICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAyMDE5MDkxMTE5NTYzODAwADIw
+MTkwOTExMTk1NjM4MDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAJDRDAwMQEAAEEAUABQAEwARQAgAEkATgBDAC4ALAAg
+AFQAWQBQAEUEHQQ+BDIEMARPACAEPwQwBD8EOgQwAAAAAAAAAAAAAAAAAAAAAAAA
+GgAAAAAAABolL0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAEBAAAB
+AAgIAAoAAAAAAAAKFwAAAAAAAAAAAAAWAAAAACIAGAAAAAAAABgACAAAAAAIAHcJ
+CxM1MAACAAABAAABAQAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgAAAgACAAIAAgACAAIAAg
+ACAAIAAgACAAIAAgACAAIAAgACAAIAAAIAAgACAAIAAgACAAIAAgACAAIAAgACAA
+IAAgACAAIAAgACAAADIwMTkwOTExMTk1NjM4MDAAMjAxOTA5MTExOTU2MzgwMAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+/0NEMDAxAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAABUAAQAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAEAFQAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+4gAVAAAAAAAAFQAIAAAAAAgAdwkLEzUwAAIAAAEAAAEBAFNQBwG+7wBFUrkBCkle
+AElFRUVfUDEyODJUSEUgSUVFRSBQMTI4MiBQUk9UT0NPTCBQUk9WSURFUyBTVVBQ
+T1JUIEZPUiBQT1NJWCBGSUxFIFNZU1RFTSBTRU1BTlRJQ1MuUExFQVNFIENPTlRB
+Q1QgVEhFIElFRUUgU1RBTkRBUkRTIERFUEFSVE1FTlQsIFBJU0NBVEFXQVksIE5K
+LCBVU0EgRk9SIFRIRSBQMTI4MiBTUEVDSUZJQ0FUSU9OLiIAFQAAAAAAABUACAAA
+AAAIAHcJCxM1MAACAAABAAABAQGEABkAAAAAAAAZPAAAAAAAADx3CQoUCSIAAAAA
+AQAAAQdUVFhULlhaQUEOAgAAAAAAAAAAAABQWCwBpIEAAAAAgaQBAAAAAAAAAfUB
+AAAAAAH1FAAAAAAAABQCAAAAAAAAAlRGIQEPdwkKFAkiAHcJChQJIgB3CQsTMwIA
+dwkLEzUwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAABgAAQAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAEAGAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+4gAYAAAAAAAAGAAIAAAAAAgAdwkLEzUwAAIAAAEAAAEBAFNQBwG+7wBFUrkBCkle
+AElFRUVfUDEyODJUSEUgSUVFRSBQMTI4MiBQUk9UT0NPTCBQUk9WSURFUyBTVVBQ
+T1JUIEZPUiBQT1NJWCBGSUxFIFNZU1RFTSBTRU1BTlRJQ1MuUExFQVNFIENPTlRB
+Q1QgVEhFIElFRUUgU1RBTkRBUkRTIERFUEFSVE1FTlQsIFBJU0NBVEFXQVksIE5K
+LCBVU0EgRk9SIFRIRSBQMTI4MiBTUEVDSUZJQ0FUSU9OLiIAGAAAAAAAABgACAAA
+AAAIAHcJCxM1MAACAAABAAABAQGOABkAAAAAAAAZPAAAAAAAADx3CQoUCSIAAAAA
+AQAAARAAdAAuAHQAeAB0AC4AeAB6AEFBDgIAAAAAAAAAAAAAUFgsAaSBAAAAAIGk
+AQAAAAAAAAH1AQAAAAAB9RQAAAAAAAAUAgAAAAAAAAJURiEBD3cJChQJIgB3CQoU
+CSIAdwkLEzMCAHcJCxM1MAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD9N3pYWgAABObWtEYCACEB
+FgAAAHQv5aMBAAE0MgAAAORyb6eOXYmRAAEaAtwupX4ftvN9AQAAAAAEWVoAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: zoo
+
+Wk9PIDIuMTAgQXJjaGl2ZS4aAADcp8T9KgAAANb///8CAAEAAAAAAAAD3KfE/QIA
+cwAAAHEAAACSQbRdlxUCAAAAAgAAAAEAAAAAAAAAAAB0LnR4dADvMP9/AAAACgD8
+VIoAAAAApAFAAAAAQCkjKAA0MtynxP0CALwAAAC6AAAAVkJSdpcVAgAAAAIAAAAB
+AAAAAAAAAAAAdDIudHh0ADD/fwAAAAoA/EgFAAAAAKABQAAAAEApIygANDLcp8T9
+AgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAD8gw==
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: epub
+
+UEsDBBQAAAAAAOK6K09vYassFAAAABQAAAAIAAAAbWltZXR5cGVhcHBsaWNhdGlv
+bi9lcHViK3ppcFBLAwQUAAAACADiuitPnfSYClYBAADCAQAAEwAUAE9QUy9jaGFw
+dGVyLTEueGh0bWwBABAAAAAAAAAAAAAAAAAAAAAAADVQy27bMBC85ysWe2oPNO08
+4MgRFVhWAhTooYf6XNDUNmJMkwK5iWygf9EfCXrtT+iTSjnNbWd3Zgcz5f3x4OCV
+YrLBK1zM5gjkTWitf1K4/f4obvG+uig7zrRMXTk9HeILTsgnhR1zv5JyGIbZcDUL
+8UkuiqKQx0mBVdmRbquSLTuqFjD+Hv+Ob+Of8Q0W8AuYElP/sivlO6F01u8hklOY
++OQodUSM0EX6qdCkJHch7Gd5QOBTTwqZjiwnLKsymWh7hhSNwuf/1Ofz5UCswXQ6
+JuKPUB9rrw/5z8O3Wmy3XxoEEzyTz7Srzc1jcftQiHrT1OJ6vmlEXSznYn25XDfN
+zfq6WV5OX+R7wl1oT9DaqNBxRAjeBd0qrPP6R/BfM/j0OdfR2lcwTqdc3KRAOOdU
+OHSWSaReG1r1kcQQdX8HuQ4Su0h6v0ocreG7s+OknIxzw9XFP1BLAwQUAAAACADi
+uitP5hoW8IABAAAfAgAADQAUAE9QUy90b2MueGh0bWwBABAAAAAAAAAAAAAAAAAA
+AAAAAFWQwU7jMBCG7zyF5bvjtIBKghPUNiBx2wN9ADcZiNXUtuyhaZ+DO8+AWCEh
+rfYd/Ehr0/awl9HY882v+X9xt98OZAfOK6MrOslySkC3plP6paKrpwd2Q+/qC9Fj
+xCJaDjIN3CtNL+1LsK/rivaItuR8HMdMdfY5M+6FT/N8xo31J/I/aLz8QSZFUfB9
+0qa16EF2tUCFA9ThPfwOf8JH+Iz1K/wN3+FL8ONMDEpviIOhoh4PA/geACnpHTxX
+tPWer43ZZLGhBA8WKoqwR57evBZbQEnaXjoPePZ3/tZyG+n7Xwu2Wj02lLRGI+iI
+XS6vH4qb+4Itls2CXeXLhi2KWc7m09m8aa7nV81smlT40cLadAfSKVfRAV001qkd
+aQfpYwRa7lgaU/JzekXHXiEwb2ULpXXARiftLYkOga0dyE3p0akWb6NM3CUp7fLk
+yrSUqO7Y1MIMKZhayHMQvbQIjk2yU77h7RToB5kILuO1Cedpj0fpWOOdsabzkpW4
+VF/8A1BLAwQUAAAACADiuitPxTEwa6UAAADqAAAAFgAUAE1FVEEtSU5GL2NvbnRh
+aW5lci54bWwBABAAAAAAAAAAAAAAAAAAAAAAAF2OwQ6CMBBE73wF2auB6s00FG5e
+NVE/oJQFG9vdpi1G/17iAY23OcybN0339K58YEyWScGu3kKJZHiwNCm4Xg7VHrq2
+aAxT1pYw/nUXmpKCOZJknWySpD0mmY3kgDSwmT1Slp+aXEegbSJzHq3D9I3lODtX
+BZ1vCo6ns8DQ1xxGKD0OVlf5FVCBDsFZo/NyQTD2IS2AuesJN4sDRNuIn2WxGtvi
+DVBLAwQUAAAACADiuitPTf3QgnIAAACVAAAALQAUAE1FVEEtSU5GL2NvbS5hcHBs
+ZS5pYm9va3MuZGlzcGxheS1vcHRpb25zLnhtbAEAEAAAAAAAAAAAAAAAAAAAAAAA
+s7GvyM1RKEstKs7Mz7NVMtQzUFJIzUvOT8nMS7dVCg1x07VQsrfjsknJLC7ISayM
+zy8oASostrMB8krS8otyFfISc1NtlbSU7GwgclCB4oLU5My0zNQU3bT8vJJiJbuS
+otJUG32IGjsbfZh+IBPdbC4AUEsDBBQAAAAIAOK6K09uOyWS9wAAAL4CAAAQAAAA
+T1BTL2Nzcy9ib29rLmNzc41QS27CMBBdx6fwhk0kN0FIrRROY5whGWE8kT1JaVHv
+jh0KLUoa6pX9fn4zRS57N2BAhlpadAeZF0JX4+0sZDyGLPlKokNGbbfiS4gil1PL
+DVlyHakPIGkA/8vYju8lWwAL5vE3bRgH+NulzyJjOLGqwZDXjOQq6chBYrtIHrVv
+MGJld0pQ6HcRjEUYjbZKW2wiqdblaiuyPTlWAT+hkq9vq6u8m5FvNvPqlx3VH7Fr
+9o41t5Vcl+UV710NPg6VBpnWvbOjljyC45FS1HUU4j6eBKTF3v2BPR6AW099087K
+k1R9Cx7rqefmnyaTmOXm/8i+zTGJvgBQSwMEFAAAAAgA4rorT8ZW1i8eAAAAHAAA
+AA4AAABPUFMvanMvYm9vay5qc0srzUsuyczPU3DKT6mMz8/zyU9M0dBUqOaq5eIC
+AFBLAwQUAAAACADiuitP4mU1iU0BAABVAgAACwAUAE9QUy9lcGIubmN4AQAQAAAA
+AAAAAAAAAAAAAAAAAABtkduOgjAQhu99iqb3UDytYgCDsF7twWTdByh0IiTQkjIo
+7tMvRY2s61VnOt/8M3/rrduyIEfQda6kT8e2QwnIVIlcHnz6vd9aS7oORp5M2zs1
+cZy5Naaka5W1TzPEasXY6XSyBc/rs630gf1M3eULMyTrehkNvAy4CLwSkBPJS/Cp
+wGTV5IKSVEkEiT6dRvOtu3x1rU0Ub6yZE8XWxl04VjhZhHE8D2fxYkLZHxGomsRK
+NXBUeqC04weoya5JirzOOi+k5OnnFzkag+zfFgIqzAbd4ycMKuSFkY1UI3EAPxMs
+eWvQj6ZMQD+y7PISQqX7HAsIPIQWA4QajRmP9anH7vUuChvMlL6iQ+RWkPz4zqv+
+3KlcIsmFT7ukMon5rKrg508tQBt7PffGEygeJe/316VJrVOfphmvELQ1ttsMy6K3
+cRt1Cfvp5q+D0S9QSwMEFAAAAAgA4rorT2ftW/C3AgAA9AUAAAsAFABPUFMvZXBi
+Lm9wZgEAEAAAAAAAAAAAAAAAAAAAAAAArVTdbtMwFL7nKSxziRIn3aauUZNpXZnE
+FZMYN9w5zmnrLT/Gdrr2ObjnGSbQJCTEO+SROHHSNoMNJIRUVbH9ne+c73zHnp5t
+ipysQRtZlTEN/YASKEWVyXIZ0/fXl94pPUteTBUXt3wJBNGliWRaVbcmpitrVcTY
+uhI8rXOut760dQnG50rl4IuqYDpbsA4+gHmwsVC2KY2HKRkldSk/1uDJDEorFxJ0
+TGcY9Cajh+KO2uKUhoXcxLTjjMj/K8FJ22u6u7vzZaYWfqWXbBQEY1apBU2mBVie
+ccv7TmRiH6FqnTt0JhjkUKASw0I/3FFHSPBX+kxEVtocEgvGgqrTKdtvtYeiKq2W
+aW0rTWQW08GaJlfokCFXdZpLs0IDScHF23dkjfoczQDcCSFtM7FZMX05JMImVwq0
+3cZUVzlQYsQK9cS04FpEGnKOKEOT9FZNWcvjSsOuQDIKwokXTLwwdBndXnt4MNaV
+3XubHF2cXE5OX0+82cV85h0HF3NvNhkH3vlofD6fn5wfz8cjR3SId3Q5L5c1qk10
+7Y73607Wof5MWNCFiQqcaIzGlIcKr0dHEf6Ckw87Fb8E9zNmFAgX7C2wSajb6hr+
+HNLPLE3CHY7tBgdDeCkXaG8ylRYK1w9bCUpW6Ib79DcrW+SUFJBJ7tmtwta34ywF
+t8jK3PGrTQvpM8vWw5KvKRuQihVXqN4Ld9T7jX9MYISWSJA9ymLsNgezArD7NMaw
+tg0+fjzOYfHSsXZ3SHDD17xj3hHc9PE3T4UP4EOWUmx24aBS3y2fledlNkWEU8ha
+b/aOGIX3gaAHHWOXAFkxx+MWUpIjkuM7tQWnh7nQZLqscVaTKaJB40uKZF3trcPu
+Hse0+dx8bb43980X/H9ofjTfmoff7WdPkGADDiyfeo578qzDbV19Qax/wpMXPwFQ
+SwECPgAUAAAAAADiuitPb2GrLBQAAAAUAAAACAAAAAAAAAAAAAAAAAAAAAAAbWlt
+ZXR5cGVQSwECPgAUAAAACADiuitPnfSYClYBAADCAQAAEwAAAAAAAAAAAAAAAAA6
+AAAAT1BTL2NoYXB0ZXItMS54aHRtbFBLAQI+ABQAAAAIAOK6K0/mGhbwgAEAAB8C
+AAANAAAAAAAAAAAAAAAAANUBAABPUFMvdG9jLnhodG1sUEsBAj4AFAAAAAgA4ror
+T8UxMGulAAAA6gAAABYAAAAAAAAAAAAAAAAAlAMAAE1FVEEtSU5GL2NvbnRhaW5l
+ci54bWxQSwECPgAUAAAACADiuitPTf3QgnIAAACVAAAALQAAAAAAAAAAAAAAAACB
+BAAATUVUQS1JTkYvY29tLmFwcGxlLmlib29rcy5kaXNwbGF5LW9wdGlvbnMueG1s
+UEsBAj4AFAAAAAgA4rorT247JZL3AAAAvgIAABAAAAAAAAAAAAAAAAAAUgUAAE9Q
+Uy9jc3MvYm9vay5jc3NQSwECPgAUAAAACADiuitPxlbWLx4AAAAcAAAADgAAAAAA
+AAAAAAAAAAB3BgAAT1BTL2pzL2Jvb2suanNQSwECPgAUAAAACADiuitP4mU1iU0B
+AABVAgAACwAAAAAAAAAAAAAAAADBBgAAT1BTL2VwYi5uY3hQSwECPgAUAAAACADi
+uitPZ+1b8LcCAAD0BQAACwAAAAAAAAAAAAAAAABLCAAAT1BTL2VwYi5vcGZQSwUG
+AAAAAAkACQA9AgAAPwsAAAAA
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: pkg
+
+eGFyIQAcAAEAAAAAAAAdIAAAAAAAALiMAAAAAXjalFfZsqLIFn03wn84UffRqBYQ
+HDpOVUcm86QyiOAbgzKLCJjg19/knD7VXXV7qPtgmOSwM3Pvtdfa+fpbXxYvj/O9
+Savrl0/kL8Snl/M1rKL0Gn/5dLCFz+tPv32dvPb+/evk5bWtQvz38hrez36LV3xu
+0/L8lSJI5jOx+UySNkH/Sq1/JZav8++nvC1KzmHedOVL0w7F+cunJvHJT+PIy2t1
+uTTn9ivxOv+99dbbpM/R+Ov8rTGamH/YePtq0vjqt939/GHRtMD3Bqm/tMgs/zD5
+8qqeB/l6qV6wI67Nl09J295+nc8RQr+gxS/VPZ5TBEHMic0cT4jwlv953+Ll1WWI
+Dee3/vvn+zd7vrfpJQ399vxVl2XhnLEsFNUYIBmCWJbdDbfhEGiUmkac4SlqdZKT
+R7gFBq9BA6Czzbc624iAPPAAodwUHWo6OR37Ryg6g8wXvMxvi/Bq3k5lkXmueQso
+OjZch/DFzeC7JqNbHlKAxzmGoXGQhKEEm5MFuZOrENOJfzzdPEog/OOmk3mBjES8
+e1kQeF9bh/zbvmyMVMOFKBAPsXVkMi3jNR3kb2Mw0VnH0dF0IiXhVrdlpGdyr9vx
+sM14dHzrC7/vy1hYZryvw1hk49JSqtJep32Ob6yzWaROJzu+oQ52vDjkVb/jAAXj
+rQOBrmuEWUSl0wQLWIQpVEwWKsF1S3jHvjm58hKfv/WuyiOkikQWtg/sp6u58Fyl
+0MaZpd7ppo5443dfgGphZYm2LXh6SyqqbgEkxW9jKgcjGByVIVhsb54F7ekkoE5E
+RAnDyUJ/64M45lMdECJr1aIlBwvO4CEwDgDQMsQxHsfx7UCFI29wktgolquAZVdQ
+F+9yA/XOViBzkGYbZLbukqEAv994xAytDk6lg34A8mN/36vuuTPOFnubTnj5lIKt
+DQoaCgAAAA1JSERSAAAAwAAAAC4IBgAAAH6SFvUAAAAZdEVYdFNvZnR3YXJlAEFk
+b2JlIEltYWdlUmVhZHlxyWU8AAAMeklEQVR42uxd/VXjSBJvePv/eiNAEwEmAkQE
+mAgsIrC5u/9tArhnHIHlCLAjsIkATwSniWC1Ecyp2V9DUfRHdUue2Xmo3tMbBqRS
+d31XdXXr5Pv376qHHj4tpCrA3fTf8+Ya9RTs4VeW/d8ShX/Q/DNprqq5Nj0pe/hV
+4TTxOW35tRIMG2UY9mTs4bMpwDX5edyTsYdfARpjnXODfaLjoJOTk9jw50/yq3rx
+8N8/Ohjci1dpcJUdTXbR/A==
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: psd
+
+OEJQUwABAAAAAAAAAAMAAAAcAAAAFwAIAAMAAAAAAAAAHDhCSU0D7QAAAAAAEAAA
+AAEAAQABAAAAAQABAAEAAAfcAAAH1AABAAAAAAAAAAAAAAAcAAAAFwADAAAAAAKG
+AAEAAAKGAAIAAAKGOEJJTW5vcm3/AAEAAAAADAAAAAAAAAAAAkwxAAAA/f7////+
++//////+//35+/7////////++ff6/f/////+/Pv9/Pr7/v/////////8///////y
+/////v79+fn+/////////ff9+eLj38Pr9v7///37+////////////fzhvsbOt8Tc
+9f7+/f79/////////////+/Oy9LIwt74//7+//7/////////9/7//+bLyNHf8///
+///++////////////P7/7sjC3/H9///+///6///////////8///40b/K//////39
+/Pz7/f79/P3+//7///7m2eP//f78+/v6+vv8/f38/f7+//////v2/v78+/j39/f4
++vv9/fz9/v///fr/////+/j28/Lx8/X4+vz8/P3+/P789v3++/j49fHu6+zv8PX4
++/z9/f76/fz4/f749ffy7ejn6Ovu8/b6/P39/vr9/fr+/Pb38+/r5eXl5uvx9Pn8
+/f3+/f79+/759fjy7unl4uTn6/D0+fv9/f7+/v///vv6+PPv6ubj5urt8/X4/P3+
+/f/////+/Pr48/Ds6Ofp7O/19vn8/v79//////79+/n18u7r6uzv8fb3+vz+/v3/
+/////v37+/f08e/u8PLz+Pn7/f7+/f/////+/fz8+Pf18/P09fb6+/3+/v79////
+/v7+/f36+fj39/j4+f39/f7///7+///+/v7+/vz7+/v6+/v7//////////////7+
+/v7//Pz9/Pz8/Pz////////////////////+/v7+/v7+/v//////////////////
+//7+/v7+/v7+/////////////////////v7+/v7+/v7/////////////////////
+//////////////////8AAP3+/f3+/fr+/f/9/v39/f///////////vn1+fz+/v7+
+/fv7+/z+///////////9+v7+/v797////v3+/f39//////////z2/Pbf39u/6fT+
+/v79/Pz///////////353brAyLHB2fP+/v7+/f///////////v/ryMXMwrzY9v7+
+/f/+//////////T7/vvgxcDJ1+3////+//v//////////vn7/urBudbo9v7//f7/
++v/////////++/v988y4w/n6+/v8/Pz++/3+/fz9/v79+/364dTe+vn6+vn7+/v7
+/P39/P3+/v37/f338Pj4+Pb29/j4+fr7/f38/f7///v4/f36+vb29PLz9fT2+Pr8
+/Pz9/v7+/Pb9/vn29vPx7+/y8/T2+Pv8/f3+/P/8+P/++PP18u/s6+7x8vX2+vz9
+/f78///8//z29/Px7enp6+/x8/T5/P39/v////3++fX49PDr5+fp7e/y9Pn7/f3+
+/v7///77+vjz7+rm5ejs7/P1+Pz9/v3//////vz6+PPw7Ojn6ezv9fb5/P7+/f//
+///+/fv59fLu6+rs7/H29/r8/v79//////79+/v39PHv7vDy8/j5+/3+/v3/////
+/v38/Pj39fPz9PX2+vv9/v7+/f////7+/v39+vn49/f4+Pn9/f3+///+/v///v7+
+/v78+/v7+vv7+//////////////+/v7+//z8/fz8/Pz8////////////////////
+/v7+/v7+/v7////////////////////+/v7+/v7+/v////////////////////7+
+/v7+/v7+////////////////////////////////////////AAD////+/Pn1+fr9
+/v////7////////////79vf4+fn3+fn5+/z+/////////////vv8+vn19Ob29/n5
+/P3////////////68vfv1tTQs9zo9vn6+/7+//////////f18NKusrqjssrn9Pb5
+/P3/////////9fXz4Ly3vrKuyur19vj7/P/////////r8vXw1Lm1vMzh8/X4+fr5
+//////////fy9Pfht7DN3+739/j5+/j/////////+vf4+u/Isr329vj4+vr8/fv9
+/v38/f78+/r8+d7R2/f4+fv6/f39+/z9/fz9/v7+/P7++PL6+vn6+fn8/Pv6+/39
+/P3+///8+f7+/v76+ff39/j4+Pr6/Pz8/f77/Pr2/f78+fn28/Hw8vb1+Pj7/P39
+/vn8+vb8/vr2+PLu6+ru8fP09vr8/f3+9/r69/r69Pfz8Ozo6Ons7/L0+fz9/f78
++/z4/Pfz9vHt6ubj5evu8fT5+/39/v78//3++/r48+/q5uTn6+7z9fj8/f79////
+//78+vjz8Ozo5+ns7/X2+fz+/v3//////v37+fXy7uvq7O/x9vf6/P7+/f/////+
+/fv79/Tx7+7w8vP4+fv9/v79//////79/Pz49/Xz8/T19vr7/f7+/v3////+/v79
+/fr5+Pf3+Pj5/f39/v///v7///7+/v7+/Pv7+/r7+/v//////////////v7+/v/8
+/P38/Pz8/P////////////////////7+/v7+/v7+/////////////////////v7+
+/v7+/v7////////////////////+/v7+/v7+/v//////////////////////////
+/////////////wAAAAAAAP3+/////vv//////v/9+fv+/////////vn3+v3/////
+/vz7/fz6+/7//////////P//////8v////7+/fn5/v////////33/fni49/D6/b+
+///9+/v///////////384b7GzrfE3PX+/v3+/f/////////////vzsvSyMLe+P/+
+/v/+//////////f+///my8jR3/P//////vv///////////z+/+7Iwt/x/f///v//
++v///////////P//+NG/yv/////9/fz8+/3+/fz9/v/+///+5tnj//3+/Pv7+vr7
+/P39/P3+/v/////79v7+/Pv49/f3+Pr7/f38/f7///36//////v49vPy8fP1+Pr8
+/Pz9/vz+/Pb9/vv4+PXx7uvs7/D1+Pv8/f3++v38+P3++PX38u3o5+jr7vP2+vz9
+/f76/f36/vz29/Pv6+Xl5ebr8fT5/P39/v3+/fv++fX48u7p5eLk5+vw9Pn7/f3+
+/v7///77+vjz7+rm4+bq7fP1+Pz9/v3//////vz6+PPw7Ojn6ezv9fb5/P7+/f//
+///+/fv59fLu6+rs7/H29/r8/v79//////79+/v39PHv7vDy8/j5+/3+/v3/////
+/v38/Pj39fPz9PX2+vv9/v7+/f////7+/v39+vn49/f4+Pn9/f3+///+/v///v7+
+/v78+/v7+vv7+//////////////+/v7+//z8/fz8/Pz8////////////////////
+/v7+/v7+/v7////////////////////+/v7+/v7+/v////////////////////7+
+/v7+/v7+/////////////////////////////////////////f79/f79+v79//3+
+/f39///////////++fX5/P7+/v79+/v7/P7///////////36/v7+/v3v///+/f79
+/f3//////////Pb89t/f27/p9P7+/v38/P///////////fndusDIscHZ8/7+/v79
+///////////+/+vIxczCvNj2/v79//7/////////9Pv+++DFwMnX7f////7/+///
+///////++fv+6sG51uj2/v/9/v/6//////////77+/3zzLjD+fr7+/z8/P77/f79
+/P3+/v37/frh1N76+fr6+fv7+/v8/f38/f7+/fv9/ffw+Pj49vb3+Pj5+vv9/fz9
+/v//+/j9/fr69vb08vP19Pb4+vz8/P3+/v789v3++fb28/Hv7/Lz9Pb4+/z9/f78
+//z4//748/Xy7+zr7vHy9fb6/P39/vz///z//Pb38/Ht6enr7/Hz9Pn8/f3+////
+/f759fj08Ovn5+nt7/L0+fv9/f7+/v///vv6+PPv6ubl6Ozv8/X4/P3+/f/////+
+/Pr48/Ds6Ofp7O/19vn8/v79//////79+/n18u7r6uzv8fb3+vz+/v3//////v37
++/f08e/u8PLz+Pn7/f7+/f/////+/fz8+Pf18/P09fb6+/3+/v79/////v7+/f36
++fj39/j4+f39/f7///7+///+/v7+/vz7+/v6+/v7//////////////7+/v7//Pz9
+/Pz8/Pz////////////////////+/v7+/v7+/v////////////////////7+/v7+
+/v7+/////////////////////v7+/v7+/v7/////////////////////////////
+///////////////+/Pn1+fr9/v////7////////////79vf4+fn3+fn5+/z+////
+/////////vv8+vn19Ob29/n5/P3////////////68vfv1tTQs9zo9vn6+/7+////
+//////f18NKusrqjssrn9Pb5/P3/////////9fXz4Ly3vrKuyur19vj7/P//////
+///r8vXw1Lm1vMzh8/X4+fr5//////////fy9Pfht7DN3+739/j5+/j/////////
++vf4+u/Isr329vj4+vr8/fv9/v38/f78+/r8+d7R2/f4+fv6/f39+/z9/fz9/v7+
+/P7++PL6+vn6+fn8/Pv6+/39/P3+///8+f7+/v76+ff39/j4+Pr6/Pz8/f77/Pr2
+/f78+fn28/Hw8vb1+Pj7/P39/vn8+vb8/vr2+PLu6+ru8fP09vr8/f3+9/r69/r6
+9Pfz8Ozo6Ons7/L0+fz9/f78+/z4/Pfz9vHt6ubj5evu8fT5+/39/v78//3++/r4
+8+/q5uTn6+7z9fj8/f79//////78+vjz8Ozo5+ns7/X2+fz+/v3//////v37+fXy
+7uvq7O/x9vf6/P7+/f/////+/fv79/Tx7+7w8vP4+fv9/v79//////79/Pz49/Xz
+8/T19vr7/f7+/v3////+/v79/fr5+Pf3+Pj5/f39/v///v7///7+/v7+/Pv7+/r7
++/v//////////////v7+/v/8/P38/Pz8/P////////////////////7+/v7+/v7+
+/////////////////////v7+/v7+/v7////////////////////+/v7+/v7+/v//
+/////////////////////////////////////w==
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: pcx
+
+CgUBCAAAAAAWABsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAADFwABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB/cH+w//B/sH7xP/B/sH/
+wf3B+cH7wf7G/8H9wf7C/cH+wf3B+sH+wf3B/8H9wf7D/cj/w//B/sH8wfnB9cH5
+wfrB/cH+w//B/sj/wf7B+cH3wfrB/cT/wf7B/MH7wf3B/MH6wfvB/sb/wf7B+cH1
+wfnB/MT+wf3D+8H8wf7I/8H/wfvB9sH3wfjC+cH3w/nB+8H8wf7J/8H/wfzF/8Hy
+w//C/sH9wvnB/sb/wf3B+sT+wf3B78L/wf7B/cH+w/3H/8H+wfvB/MH6wfnB9cH0
+webB9sH3wvnB/MH9yf/B/cH3wf3B+cHiwePB38HDwevB9sH+wv/B/cL7x//B/MH2
+wfzB9sLfwdu/wenB9MP+wf3C/Mf/wfrB8sH3we/B1sHUwdCzwdzB6MH2wfnB+sH7
+wv7H/8H/wf3B/MHhvsHGwc63wcTB3MH1wv7B/cH+wf3H/8H/wf3B+cHdusHAwcix
+wcHB2cHzxP7B/cf/wffB9cHwwdKusrqjssHKwefB9MH2wfnB/MH9x//D/8Hvwc7B
+y8HSwcjBwsHewfjB/8L+wf/B/sf/wf/B/sH/wevByMHFwczBwrzB2MH2wv7B/cH/
+wf7H/8L1wfPB4Ly3vrKuwcrB6sH1wfbB+MH7wfzH/8H3wf7C/8HmwcvByMHRwd/B
+88T/wf7B+8f/wfTB+8H+wfvB4MHFwcDBycHXwe3D/8H+wf/B+8f/wevB8sH1wfDB
+1Lm1vMHMweHB88H1wfjB+cH6wfnH/8H/wfzB/sH/we7ByMHCwd/B8cH9wv/B/sL/
+wfrH/8H+wfnB+8H+werBwbnB1sHowfbB/sH/wf3B/sH/wfrH/8H3wfLB9MH3weG3
+sMHNwd/B7sL3wfjB+cH7wfjH/8H/wfzC/8H4wdG/wcrE/8L9wvzB+8H9wf7B/cH8
+wf3B/sH+wvvB/cHzwcy4wcPB+cH6wvvD/MH+wfvB/cH+wf3B/MH9wf7B+sH3wfjB
++sHvwciyvcL2wvjC+sH8wf3B+8H9wf7B/cH8wf3B/sH/wf7C/8H+webB2cHjwf/B
+/cH+wfzC+8L6wfvB/ML9wfzB/cH+wf7B/cH7wf3B+sHhwdTB3sH6wfnC+sH5xPvB
+/ML9wfzB/cH+wfzB+8H6wfzB+cHewdHB28H3wfjB+cH7wfrD/cH7wfzC/cH8wf3B
+/sH+xP/B+8H2wv7B/MH7wfjD98H4wfrB+8L9wfzB/cH+wf7B/cH7wv3B98Hww/jC
+9sH3wvjB+cH6wfvC/cH8wf3B/sL+wfzC/sH4wfLC+sH5wfrC+cL8wfvB+sH7wv3B
+/MH9wf7C/8H9wfrE/8H7wfjB9sHzwfLB8cHzwfXB+MH6w/zB/cH+wv/B+8H4wv3C
++sL2wfTB8sHzwfXB9MH2wfjB+sP8wf3B/sL/wfzB+cT+wfrB+cP3w/jC+sP8wf3B
+/sH8wf7B/MH2wf3B/sH7wvjB9cHxwe7B68Hswe/B8MH1wfjB+8H8wv3B/sL+wfzB
+9sH9wf7B+cL2wfPB8cLvwfLB88H0wfbB+MH7wfzC/cH+wfvB/MH6wfbB/cH+wfzC
++cH2wfPB8cHwwfLB9sH1wvjB+8H8wv3B/sH6wf3B/MH4wf3B/sH4wfXB98Hywe3B
+6MHnwejB68HuwfPB9sH6wfzC/cH+wfzB/8H8wfjB/8H+wfjB88H1wfLB78HswevB
+7sHxwfLB9cH2wfrB/ML9wf7B+cH8wfrB9sH8wf7B+sH2wfjB8sHuwevB6sHuwfHB
+88H0wfbB+sH8wv3B/sH6wv3B+sH+wfzB9sH3wfPB78Hrw+XB5sHrwfHB9MH5wfzC
+/cH+wfzC/8H8wf/B/MH2wffB88Hxwe3C6cHrwe/B8cHzwfTB+cH8wv3B/sH3wvrB
+98L6wfTB98HzwfDB7MLowenB7MHvwfLB9MH5wfzC/cH+wf3B/sH9wfvB/sH5wfXB
++MHywe7B6cHlweLB5MHnwevB8MH0wfnB+8L9wf7D/8H9wf7B+cH1wfjB9MHwwevC
+58Hpwe3B78HywfTB+cH7wv3B/sH8wfvB/MH4wfzB98HzwfbB8cHtwerB5sHjweXB
+68HuwfHB9MH5wfvC/cH+wv7C/8H+wfvB+sH4wfPB78HqwebB48HmwerB7cHzwfXB
++MH8wf3B/sH9wv7C/8H+wfvB+sH4wfPB78HqwebB5cHowezB78HzwfXB+MH8wf3B
+/sH9wf7B/MH/wf3B/sH7wfrB+MHzwe/B6sHmweTB58Hrwe7B88H1wfjB/MH9wf7B
+/cT/wf7B/MH6wfjB88HwwezB6MHnwenB7MHvwfXB9sH5wfzC/sH9xP/B/sH8wfrB
++MHzwfDB7MHowefB6cHswe/B9cH2wfnB/ML+wf3E/8H+wfzB+sH4wfPB8MHswejB
+58HpwezB78H1wfbB+cH8wv7B/cT/wf7B/cH7wfnB9cHywe7B68HqwezB78HxwfbB
+98H6wfzC/sH9xP/B/sH9wfvB+cH1wfLB7sHrwerB7MHvwfHB9sH3wfrB/ML+wf3E
+/8H+wf3B+8H5wfXB8sHuwevB6sHswe/B8cH2wffB+sH8wv7B/cT/wf7B/cL7wffB
+9MHxwe/B7sHwwfLB88H4wfnB+8H9wv7B/cT/wf7B/cL7wffB9MHxwe/B7sHwwfLB
+88H4wfnB+8H9wv7B/cT/wf7B/cL7wffB9MHxwe/B7sHwwfLB88H4wfnB+8H9wv7B
+/cT/wf7B/cL8wfjB98H1wvPB9MH1wfbB+sH7wf3D/sH9xP/B/sH9wvzB+MH3wfXC
+88H0wfXB9sH6wfvB/cP+wf3E/8H+wf3C/MH4wffB9cLzwfTB9cH2wfrB+8H9w/7B
+/cP/w/7C/cH6wfnB+ML3wvjB+cP9wf7C/8H+w//D/sL9wfrB+cH4wvfC+MH5w/3B
+/sL/wf7D/8P+wv3B+sH5wfjC98L4wfnD/cH+wv/B/sH+wv/F/sH8w/vB+sP7x//B
+/sL/xf7B/MP7wfrD+8f/wf7C/8X+wfzD+8H6w/vH/8P/xP7B/8L8wf3F/Mf/w//E
+/sH/wvzB/cX8x//D/8T+wf/C/MH9xfzH/8j/yP7H/8j/yP7H/8j/yP7H/8j/yP7H
+/8j/yP7H/8j/yP7H/8j/yP7H/8j/yP7H/8j/yP7H/9f/1//X/w==
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: tiff
+
+SUkqAJQHAAD9/f/+/v///f///f7//vz+/fn7+vX//vn//fr///3//f7+/v///f/9
+/f/5/f77///+///////////////////////////+/v/5+fv39fb6+ff9/Pj//vn/
+/vn//vf//vn+/fn8+/n7+/v9+/z8/P76/v/7///+////////////////////////
+/////f78+vv//vz//vr//vn//vX//fTy7+b///b///f//vn+/fn+/vz9/f35/f/5
+/f/+///////////////////////////9/Pr39vL9/Pf59u/i39bj39Tf29DDv7Pr
+6dz29Oj+/vb//vn//vr9/fv7/P77/P7///////////////////////////////f9
+/fX8+fDh3dK+uq7GwLLOyLq3saPEwbLc2cr18+f+/vT+/vb9/vn+/vz9/f3/////
+//////////////////////////X//vX///Pv6+DOyLzLxbfSzL7IwrLCvK7e2Mr4
+9ur//vX+/vb+/fj///v+/vz////////////////////////////39Ov++/L//vX/
++/Dm4NTLxbnIwLXRybzf18zz7eH///P///X///j//vn+//r7+/n/////////////
+/////////////////vf8+fL++/T//vfu6uHIwbfCubDf1s3x6N/99u7//vf///f+
+/fj//vn///v6+vj//////////////////////////////vr8+/f/+/j//fr48+/R
+zMi/uLLKw73/+fb/+vb/+/j/+/j9/Pr9/Pr8/Pz8/v37+/v9/f3+/v79/f38/Pz9
+/f3+/v7//vz+/fv/+/r//fz++vnm4d7Z1NHj3tv/+vf9+fj++vn8+vv7+fr7+/36
++/36+/37+/v8/Pz9/f39/f38/Pz9/f3+/v7+/v7//f7/+/z//f7//f779/j28PL+
++Pr++Pr8+Pn79vr49vn39/n3+Pz3+Pz4+fv6+vr7+/v9/f39/f38/Pz9/f3+/v7/
+///////9+/z6+Pn//f7//f7/+v7/+v779vr49vn29Pfz8vfy8/fx9fjz9Pj19vj4
++Pr6+vr8/Pz8/Pz8/Pz9/f3+/v78/vv+/vz8/Pr29vb9/f3+/v77+fz49vn49vn1
+8/bx8fPu7/Hr7/Ds8vLv8/bw9PX19vj4+Pj7+/v8/Pz9/f39/f3+/v76/Pn9//z8
+/Pr4+Pb9//z+/v74+Pr18/b39fjy8vLt7+7o7Ovn6+ro7u7r8fHu8vPz9fT29vb6
++vr8/Pz9/f39/f3+/v76/Pf9//r9//r6/Pf+//r8/Pr29vT39/fz8/Pv8fDr7ezl
+6ejl6ejl6+nm7+zr8e/x8/L09PT5+fn8/Pz9/f39/f3+/v79//z+//v9//z7/fj+
+/vz5+ff19fP4+Pby9PHu8O3p6+rl5+bi5+Pk6eXn7evr7+7w8vH09PT5+fn7+/v9
+/f39/f3+/v7+/v7+/vz///////3+/v77+/v6+vr4+Pjz8/Pv7+/q6urm5ubj5eTm
+6Ofq7Ovt7+7z8/P19fX4+Pj8/Pz9/f3+/v79/f3////////////////+/v78/Pz6
++vr4+Pjz8/Pw8PDs7Ozo6Ojn5+fp6ens7Ozv7+/19fX29vb5+fn8/Pz+/v7+/v79
+/f3////////////////+/v79/f37+/v5+fn19fXy8vLu7u7r6+vq6urs7Ozv7+/x
+8fH29vb39/f6+vr8/Pz+/v7+/v79/f3////////////////+/v79/f37+/v7+/v3
+9/f09PTx8fHv7+/u7u7w8PDy8vLz8/P4+Pj5+fn7+/v9/f3+/v7+/v79/f3/////
+///////////+/v79/f38/Pz8/Pz4+Pj39/f19fXz8/Pz8/P09PT19fX29vb6+vr7
++/v9/f3+/v7+/v7+/v79/f3////////////+/v7+/v7+/v79/f39/f36+vr5+fn4
++Pj39/f39/f4+Pj4+Pj5+fn9/f39/f39/f3+/v7////////+/v7+/v7////////+
+/v7+/v7+/v7+/v7+/v78/Pz7+/v7+/v7+/v6+vr7+/v7+/v7+/v/////////////
+///////////////////////////+/v7+/v7+/v7+/v7////8/Pz8/Pz9/f38/Pz8
+/Pz8/Pz8/Pz8/Pz/////////////////////////////////////////////////
+///////////+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7/////////////////////
+///////////////////////////////////////+/v7+/v7+/v7+/v7+/v7+/v7+
+/v7+/v7/////////////////////////////////////////////////////////
+///+/v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7/////////////////////////////
+////////////////////////////////////////////////////////////////
+//////////////////////////8QAAABAwABAAAAFwAAAAEBAwABAAAAHAAAAAIB
+AwADAAAAWggAAAMBAwABAAAAAQAAAAYBAwABAAAAAgAAAAoBAwABAAAAAQAAAA4B
+AgA7AAAAoAgAABEBBAABAAAACAAAABIBAwABAAAAAQAAABUBAwABAAAAAwAAABYB
+AwABAAAAHAAAABcBBAABAAAAjAcAABwBAwABAAAAAQAAACkBAwACAAAAAAABAD4B
+BQACAAAAkAgAAD8BBQAGAAAAYAgAAAAAAAAIAAgACAAACtej/////4DhelT/////
+AM3MTP////8AmpmZ/////4BmZib/////8ChcD/////+AGw1Q/////wBYOVT/////
+Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcgSlBFRyB2NjIpLCBxdWFs
+aXR5ID0gMTAwCgA=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: ico
+
+AAABAAEAFxwAAAEAIACoCgAAFgAAACgAAAAXAAAAOAAAAAEAIAAAAAAAEAoAAAAA
+AAAAAAAAAAAAAAAAAAD/////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////v7+//7+
+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//////////////////////////////
+///////////////////////////////////////////////////+/v7//v7+//7+
+/v/+/v7//v7+//7+/v/+/v7//v7+////////////////////////////////////
+//////////////////////////////////////////////7+/v/+/v7//v7+//7+
+/v/+/v7//v7+//7+/v/+/v7/////////////////////////////////////////
+//////////////7+/v/+/v7//v7+//7+/v///////Pz8//z8/P/9/f3//Pz8//z8
+/P/8/Pz//Pz8//z8/P///////////////////////////////////////v7+////
+/////////v7+//7+/v/+/v7//v7+//7+/v/8/Pz/+/v7//v7+//7+/v/+vr6//v7
++//7+/v/+/v7////////////////////////////////////////////////////
+///+/v7//v7+//7+/v/9/f3//f39//r6+v/5+fn/+Pj4//f39//39/f/+Pj4//j4
++P/5+fn//f39//39/f/9/f3//v7+/////////////v7+////////////////////
+///+/v7//f39//z8/P/8/Pz/+Pj4//f39//19fX/8/Pz//Pz8//09PT/9fX1//b2
+9v/6+vr/+/v7//39/f/+/v7//v7+//7+/v/9/f3///////////////////////7+
+/v/9/f3/+/v7//v7+//39/f/9PT0//Hx8f/v7+//7u7u//Dw8P/y8vL/8/Pz//j4
++P/5+fn/+/v7//39/f/+/v7//v7+//39/f///////////////////////v7+//39
+/f/7+/v/+fn5//X19f/y8vL/7u7u/+vr6//q6ur/7Ozs/+/v7//x8fH/9vb2//f3
+9//6+vr//Pz8//7+/v/+/v7//f39///////////////////////+/v7//Pz8//r6
++v/4+Pj/8/Pz//Dw8P/s7Oz/6Ojo/+fn5//p6en/7Ozs/+/v7//19fX/9vb2//n5
++f/8/Pz//v7+//7+/v/9/f3//v7+//z+/v///////f////7+/v/7+/v/+vr6//j4
++P/z8/P/7+/v/+rq6v/m5ub/5OXj/+fo5v/r7Or/7u/t//Pz8//19fX/+Pj4//z8
+/P/9/f3//v7+//39/f/8//3/+//+//z//f/4/fv//P7+//f5+f/z9fX/9vj4//H0
+8v/t8O7/6uvp/+bn5f/j5+L/5enk/+vt5//u7+v/8fLw//T09P/5+fn/+/v7//39
+/f/9/f3//v7+//f8+v/6//3/+v/9//f8+v/6//7/+vz8//T29v/39/f/8/Pz//Dx
+7//s7ev/6Onl/+jp5f/p6+X/7O/m/+/x6//y8/H/9PT0//n5+f/8/Pz//f39//39
+/f/+/v7/+fz6//z//f/6/Pz/9vj4//z//f/+/v7/+vj4//bz9f/49ff/8vLy/+7v
+7f/r7Oj/6uvn/+7u6P/x8ev/8/Lu//T18//29vb/+vr6//z8/P/9/f3//f39//7+
+/v/7/vz//P7+//r8/P/29vb//f39//7+/v/8+fv/+fb4//n2+P/28/X/8/Hx//Hv
+7v/w7+v/8vLs//bz7//19PD/+Pb1//j4+P/7+/v//Pz8//39/f/9/f3//v7+////
+/////////Pv9//n4+v/+/f///v3///76///++v//+vb7//n2+P/39Pb/9/Lz//fz
+8v/49fH/+PTz//j29f/6+Pj/+vr6//z8/P/8/Pz//Pz8//39/f/+/v7//v7+//79
+///8+////v3///79///49/v/8vD2//r4/v/6+P7/+fj8//r2+//59vj/+ff3//z4
+9//8+Pf/+/n4//r6+v/7+/v//f39//39/f/8/Pz//f39//7+/v/8/v//+/3+//r7
+///8/f//+fr+/97h5v/R1Nn/297j//f6///4+f3/+fr+//v6/P/6+fv//fv7//37
++v/9+/r/+/v7//z8/P/9/f3//f39//z8/P/9/f3//v7+//r+///3+/z/+Pv///r9
+///v8/j/yMzR/7K4v/+9w8r/9vn///b6///4+///+Pv///r8/f/6/P3//Pz8//3+
+/P/7+/v//f39//7+/v/9/f3//Pz8//39/f/+/v7/9/7///L5/P/0+/7/9/7//+Hq
+7v+3wcj/sLnC/83W3//f6PH/7vb9//f+///3////+P3+//n+///7////+Pr6////
+///////////////////////////////////r9Pf/8vv+//X+///w+///1ODm/7nF
+y/+1wMj/vMnR/8zX3//h7fP/8/////X////4////+f7///r//v/5+/v/////////
+//////////////////////////////X////1/v//8////+Dr7/+8yM7/t8XL/77M
+0v+ywsj/rrzC/8rY3v/q9vj/9f7///b+/v/4/f7/+/////z+/v//////////////
+////////////////////////9/////X9/f/w+fz/0t3h/666vv+ywMb/usjO/6Ox
+t/+ywcT/ytnc/+fz9f/0/v7/9v7+//n+/f/8/v7//f39////////////////////
+///////////////////6/P3/8vb3//f8/f/v9vn/1t/i/9Tf4//Q29//s7/D/9zp
+6//o9Pb/9v7+//n+///6/v//+/39//78+//+/Pv/////////////////////////
+//////////////79///7+vz//P7///r+///5/v//9f7///T9///m7/L/9v////f/
+///5/v//+f3+//z+/v/9/f3///35///9+f////7/////////////////////////
+//////////7+//v5+f/29ff/9/n6//j8/f/5/v//+f7///f+///5/v//+f3+//n7
+/P/7+/v//Pv9//78/P///vr////7/////v//////////////////////////////
+/////f3///7+///9///+/f///P7///n9/v/1+vv/+f7///r9///9/////v3////+
+/v///f////39//79+f////v////+/////////////////////////////////wAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAA=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: swf
+
+RldTBcgDAACAAAHCAAACfGgADAMAIggBAIAAAcIAAAJ8aAEA/////wARDCfRwgfK
+fG+kfA+bBzAAzwECAAgBAAAAIAAAAAAABgDGCQMAAQAAAH8FyQIAAAUA/9j/2wBD
+AAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcp
+LDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIy
+MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/xAAfAAABBQEB
+AQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0BAgMA
+BBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1
+Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOU
+lZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm
+5+jp6vHy8/T19vf4+fr/xAAfAQADAQEBAQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/
+xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGhscEJ
+IzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNk
+ZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4
+ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2f/Y/+AAEEpG
+SUYAAQEBCewJ7AAA/8AAEQgAHAAXAwEiAAIRAQMRAf/aAAwDAQACEQMRAD8A9s1O
+/nslQxRowfIDMTw3YYq5bSPNbpJInlsRkrnOKS4giuI9syBlBBAPrUZuMPtqUne9
+xtqxcopiHcAaKoQ5hkYqqbfL7qt0YoAaq7RiinUUAf/ZKQgGAFgARUABUgABQQUA
+/ADBCyABgttkAAAAAAAAEQwnkRVyqk5LrctXANIJBwABAIoGBgEABgAgAAAAAAAA
+0gkEAAEAigYGAAAHACA2EkzmAACKBgYCAAAAIAAAAACKBgYDAAMAIAAAAACKBgYE
+AAQAIAAAAACKBgYKAAIAIAAAAAACAwcAQAACBwoAQAACBwIAAgcDAAIHBAAGA4EC
+AAAAAEAAAAA=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: doc
+
+0M8R4KGxGuEAAAAAAAAAAAAAAAAAAAAAPgADAP7/CQAGAAAAAAAAAAAAAAABAAAA
+GAAAAAEAAAAAEAAAGwAAAAEAAAD/////AAAAABwAAAD/////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+///////////////////////////////////////////spcEAcWAJBAAAABLBAAAA
+AAAAAAAAAAAABAAAEAYAAA4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABYA
+ABAAAAAAAAAAAAAAAgAAAAAAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKMAAAAAAMgE
+AAAAAAAAyAQAAMgEAAAAAAAAyAQAAAAAAADIBAAAAAAAAMgEAAAAAAAAyAQAABQA
+AADcBAAAAAAAANwEAAAAAAAA3AQAAAAAAADcBAAAAAAAANwEAAA4AAAAFAUAABQA
+AAAoBQAADAAAADQFAAAAAAAABwcAAMoBAADRCAAAAAAAANEIAAAAAAAA0QgAAAAA
+AADRCAAAAAAAANEIAAAAAAAA0QgAAAAAAADRCAAAAAAAANEIAAAAAAAA0QgAAAAA
+AADRCAAAAAAAANEIAAAAAAAA0QgAAAAAAADRCAAAAAAAANEIAAAAAAAA0QgAAAAA
+AADRCAAAaAIAADkLAAAAAAAA8gYAABUAAAAHBwAAAAAAAAcHAAAAAAAABwcAAAAA
+AAAHBwAAAAAAAAcHAAAAAAAABwcAAAAAAAAHBwAAAAAAAAcHAAAAAAAABwcAAAAA
+AAAHBwAAAAAAAAcHAAAAAAAABwcAAAAAAAAHBwAAAAAAAAcHAAAAAAAABwcAAAAA
+AAAHBwAAAAAAAEoFAACiAQAANAUAABYAAABKBQAAAAAAAEoFAAAAAAAASgUAAAAA
+AABKBQAAAAAAAEoFAAAAAAAASgUAAAAAAABKBQAAAAAAAEoFAAAAAAAASgUAAAAA
+AABKBQAAAAAAAEoFAAAAAAAASgUAAAAAAABKBQAAAAAAAEoFAAAAAAAASgUAAAAA
+AABKBQAAAAAAAEoFAAAAAAAASgUAAAAAAABKBQAAAAAAAEoFAAAAAAAASgUAAAAA
+AADsBgAAAAAAAOwGAAAAAAAASgUAAAAAAABKBQAAAAAAAEoFAAAAAAAASgUAAAAA
+AABKBQAAAAAAAEoFAAAAAAAASgUAAAAAAABKBQAAAAAAAEoFAAAAAAAASgUAAAAA
+AABKBQAAAAAAAEoFAAAAAAAASgUAAAAAAABKBQAAAAAAAEoFAAAAAAAASgUAAAAA
+AADsBgAABgAAAPIGAAAAAAAA8gYAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AADyBgAAAAAAAPIGAAAAAAAAOQsAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AADyBgAAAAAAAPIGAAAAAAAA8gYAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AADyBgAAAAAAAPIGAAAAAAAA8gYAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AADyBgAAAAAAAPIGAAAAAAAA8gYAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AADyBgAAAAAAAPIGAAAAAAAA8gYAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AADyBgAAAAAAAPIGAAAAAAAA8gYAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AADyBgAAAAAAAPIGAAAAAAAA8gYAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AADyBgAAAAAAAPIGAAAAAAAA8gYAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AADyBgAAAAAAAPIGAAAAAAAA8gYAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AADyBgAAAAAAAPIGAAAAAAAA8gYAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AADyBgAAAAAAAPIGAAAAAAAA8gYAAAAAAADyBgAAAAAAAPIGAAAAAAAA8gYAAAAA
+AAACAAwBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAxAA0ADQANAA0ADQANAA0A
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAIYCEQASAAEAnAAPAAQAAAAAAAAAAAAEAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA
+AQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAEAAAADx/wIA
+QAAAAAAAAAAAAAAABgBOAG8AcgBtAGEAbAAAAAIAAAAYAENKGABzSAkEbUgJBHRI
+CQRfSAEEYUoYAAAAAAAAAAAAAAAAAAAAAAAAAEQAQQDy/6EARAAAAQAAAAAAAAAA
+FgBEAGUAZgBhAHUAbAB0ACAAUABhAHIAYQBnAHIAYQBwAGgAIABGAG8AbgB0AAAA
+AABkAGkA8/+zAGQAAAEAAAAAAAAAAAwAVABhAGIAbABlACAATgBvAHIAbQBhAGwA
+AAAuAGH2AwAAF/YDAAA01gYAAQEDAAA01gYAAQIDbAA01gYAAQQDAAA01gYAAQgD
+bAACAAsAAAAoAGsA9P/BACgAAAEAAAAAAAAAAAcATgBvACAATABpAHMAdAAAAAIA
+AAAAAAAALgBVAPL/8f8uAAAAAAAAAAAAAAAJAEgAeQBwAGUAcgBsAGkAbgBrAAAA
+AwA+KgEA4AD+D/H/AgHgAAAAAAAAAAAAAAAOAB4EQQQpAAkwAh0wASgyACOQbgQk
+kG4EF7DFAhiwUgMfsIIuILDGQSGwbgQisG4EAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAGAAACBgAABAYAAAYGAAAIBgAACgYAAA4GAAD9uHNy
+LXIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAIg1CAA2CAA4CAA6CAA7CAA8CAA3CAA5CABYCABUCABTKgBcCABd
+CABaCABDShQAT0oAAFBKAwBRSgAAXkoAAECIAABwaAAAAP9SSGQAc0j//21I//90
+SP//X0j//0gqAD4qAEtIAABFSAAAd2gAAAD/YUoUAAwqAGVoAAAAAHLKCAAAAP8A
+AAAAAAAAiDUIADYIADgIADoIADsIADwIADcIADkIAFgIAFQIAFMqAFwIAF0IAFoI
+AENKFABPSgAAUEoDAFFKAABeSgAAQIgAAHBoAAAA/1JIZABzSP//bUj//3RI//9f
+SP//SCoAPioAS0gAAEVIAAB3aAAAAP9hShQADCoAZWgAAAAAcsoIAAAA/wAAAAAA
+iDUIADYIADgIADoIADsIADwIADcIADkIAFgIAFQIAFMqAFwIAF0IAFoIAENKFABP
+SgAAUEoDAFFKAABeSgAAQIgAAHBoAAAA/1JIZABzSP//bUj//3RI//9fSP//SCoA
+PioAS0gAAEVIAAB3aAAAAP9hShQADCoAZWgAAAAAcsoIAAAA/wAAAAAAA1oIAAAG
+DgYAABAGAAC7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACINQgANggAOAgA
+OggAOwgAPAgANwgAOQgAWAgAVAgAUyoAXAgAXQgAWggAQ0oUAE9KAABQSgMAUUoA
+AF5KAABAiAAAcGgAAAD/UkhkAHNI//9tSP//dEj//19I//9IKgA+KgBLSAAARUgA
+AHdoAAAA/2FKFAAMKgBlaAAAAAByyggAAAD/AAAAAAEABgAABAYAAAYGAAAKBgAA
+DgYAABAGAAD8AAAAAAAAAAAAAAAA+gAAAAAAAAAAAAAAAPoAAAAAAAAAAAAAAAD6
+AAAAAAAAAAAAAAAA/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAABAAADEABBJAAABT0EPgQyBD0EPgQ5BCAAQgQ1BDoEQQRCBAAAVAAQAAUk
+AAYkAAckABOkAAAUpAAAEmTwAAEAKiQAMSQBQCYJQSQADoQAAA+EAAARhAAAXYQA
+AF6EAABghAAAAyQAYSQALUQAAE3GCgAAAP8AAAD/AABWADUIADYIADgIADoIADsI
+ADcIAFMqAFwIAF0IAENKFgBPSgQAUEoDAFFKBABeSgMAQIgAAHBoAAAAAHNICQRt
+SAkENCoASCoAPioAS0gAAEVIAABhShYAAAAAAAgAAAAAAAAIAAAAAP////8AAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAACAAAABAAAAAQAAAAEAAAA
+BwAAAAAGAAAOBgAAEgYAAAUAAAAGAAAAAAYAABIGAAAHAAAA//8BAAAABwBVAG4A
+awBuAG8AdwBuAA8AAPD4AAAAAAAG8CAAAAABCAAAAwAAAAIAAAACAAAAAQAAAAEA
+AAACAAAAAQAAAOMBC/C0AAAAgQDgjAEAggDgjAEAgwDgjAEAhADgjAEAhQAAAAAA
+hwABAAAAiAAAAAAAiQAAAAAAiwAAAAAAvwACAAMAgAEAAAAAgQEAov8AggEAAAEA
+vwERABEAwAEAAAAAwQEAAAEAywE4YwAAzAEAAAQAzQEAAAAAzgEAAAAA0AEAAAAA
+0QEAAAAA0gEBAAAA0wEBAAAA1AEBAAAA1QEBAAAA1gEBAAAA1wECAAAA/wEYABgA
+PwIAAAIAIwAi8QwAAACMAAEAAACNANSUAAAADwAC8EgAAAAQAAjwCAAAAAEAAAAA
+BAAADwAD8DAAAAAPAATwKAAAAAEACfAQAAAAAAAAAAAAAAAAAAAAAAAAAAIACvAI
+AAAAAAQAAAUAAAABDwAC8EgAAAAgAAjwCAAAAAEAAAAACAAADwAD8DAAAAAPAATw
+KAAAAAEACfAQAAAAAAAAAAAAAAAAAAAAAAAAAAIACvAIAAAAAAgAAAUAAAD//wAA
+AAACEAAAAAAAAAAIAAAAYAAABgAAAAAFAAAAaRaQAQAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAFQAaQBtAGUAcwAgAE4AZQB3ACAAUgBvAG0AYQBu
+AAAAAABUAGkAbQBlAHMAIABOAGUAdwAgAFIAbwBtAGEAbgAAAEUWkAECAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABTAHkAbQBiAG8AbAAAAAAAUwB5
+AG0AYgBvAGwAAABBJpABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAQQByAGkAYQBsAAAAAABBAHIAaQBhAGwAAABtFJABAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAQQByAGkAYQBsACAAVQBuAGkAYwBvAGQAZQAg
+AE0AUwAAAAAAQQByAGkAYQBsACAAVQBuAGkAYwBvAGQAZQAgAE0AUwAAAGUUkAEA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIAGUAbAB2AGUAdABp
+AGMAYQAgAE4AZQB1AGUAAAAAAEgAZQBsAHYAZQB0AGkAYwBhACAATgBlAHUAZQAA
+ACIABAAACIgQAADQAgAAaAEAAAAAOW1ZqTltWakAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAEAIMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAhAwAAAAAAAJAABwAd
+ABkgHSApABUwXQB9AAkwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAYIBwgKAAUMFsAewAIMAowDDAOMBAwhSkYMBYwqwAdMDX+N/45
+/jv+Pf4//kH+Q/5H/ln+W/5d/mL/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAEgAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAACIxEAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAHAAAAAAASgAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABSAG8AbwB0ACAARQBuAHQA
+cgB5AAAAAADYYyAJiX8AACAAAAAAAAAAAAAAAEM4NUL/////////////////////
+FgAFAP//////////AQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIA+1d6xnQGAkfa4
+XGrVARkAAABAAQAAAAAAADEAVABhAGIAbABlAAAA////////////////////////
+//////////////////////////////////////////8OAAIBAwAAAAIAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAACAkfa4XGrVAYCR9rhcatUBBAAAAAAQAAAAAAAA
+VwBvAHIAZABEAG8AYwB1AG0AZQBuAHQAAAAgCYl/AAAgAAAAAAAAAAAAAAAAAAAA
+/////////////////////xoAAgD/////BAAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAICR9rhcatUBgJH2uFxq1QEAAAAAABAAAAAAAABEAGEAdABhAAAA////////
+////////////////////////////////////////////////////////////////
+CgACAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAgJH2uFxq1QGAkfa4
+XGrVARAAAAAAEAAAAAAAAP7/AAAFAAIAAAAAAAAAAAAAAAAAAAAAAAIAAAAC1c3V
+nC4bEJOXCAArLPmuRAAAAAXVzdWcLhsQk5cIACss+a58AAAAOAAAAAMAAAABAAAA
+IAAAABAAAAAoAAAACwAAADAAAAACAAAAAAAAAAsAAAAAAAAACwAAAAAAAAAUAAAA
+AQAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAP7/AAAFAAIAAAAAAAAAAAAAAAAAAAAAAAEAAADghZ/y
++U9oEKuRCAArJ7PZMAAAABgAAAABAAAAAQAAABAAAAACAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+BQBTAHUAbQBtAGEAcgB5AEkAbgBmAG8AcgBtAGEAdABpAG8AbgAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACgAAgD/////BQAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAICR9rhcatUBgJH2uFxq1QEDAAAASAAAAAAAAAAFAEQAbwBjAHUAbQBlAG4A
+dABTAHUAbQBtAGEAcgB5AEkAbgBmAG8AcgBtAGEAdABpAG8AbgAAAAAAAAAAAAAA
+OAACAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAgJH2uFxq1QGAkfa4
+XGrVAQAAAACQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAgD7V3rGdAQCAPtXesZ0B/////wAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAD///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAACAPtXesZ0BAIA+1d6xnQH/////AAAAAAAAAAABAAAAAgAAAP7///8EAAAA
+/v//////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+/////////////////////wEAAAACAAAAAwAAAAYAAAAFAAAACgAAAAcAAAAIAAAA
+CQAAAP7///8LAAAADAAAAA0AAAAOAAAADwAAAP7///8RAAAAEgAAABMAAAAUAAAA
+FQAAABYAAAAXAAAA/v///xoAAAD+/////v////7////9////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: xls
+
+0M8R4KGxGuEAAAAAAAAAAAAAAAAAAAAAOwADAP7/CQAGAAAAAAAAAAAAAAABAAAA
+BQAAAAAAAAAAEAAAAAAAAAEAAAD+////AAAAAAYAAAD/////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+//////////////////////////////////////////8BAAAAAgAAAAMAAAD+////
+BQAAAAYAAAAHAAAACAAAAAkAAAAKAAAACwAAAAwAAAANAAAADgAAAA8AAAAQAAAA
+EQAAABIAAAATAAAAFAAAABUAAAD+////FwAAABgAAAAZAAAA/v//////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+//////////////////////7/AAAGAQIAAAAAAAAAAAAAAAAAAAAAAAEAAADghZ/y
++U9oEKuRCAArJ7PZMAAAALgAAAAHAAAAAQAAAEAAAAACAAAASAAAAAQAAABoAAAA
+CAAAAIAAAAAMAAAAmAAAAA0AAACkAAAAEwAAALAAAAACAAAA5AQAAB4AAAAYAAAA
+VW50aXRsZWQgU3ByZWFkc2hlZXQAAAAAHgAAABAAAABVbmtub3duIENyZWF0b3IA
+HgAAABAAAABVbmtub3duIENyZWF0b3IAQAAAAAAR5fd1atUBQAAAAAAR5fd1atUB
+AwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJCBAAAAYFALsNzAfRAAEA
+BgQAAEIAAgCwBD0AEgAAAAAAvCVyFTgAAAAAAAEAWAIiAAIAAAAxAB4A3AAAAAgA
+kAEAAAAAAAAHAUMAYQBsAGkAYgByAGkA4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE
+4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE
+4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE
+4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE
+4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE
+4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE
+4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE
+4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE4AAUAAAAAAD1/yAAAMAAAAAAAAAAAAkE
+4AAUAAAAAAABACAAAMAAAAAAAAAAAAkEkwIEAACAAP+SAOIAOAAAAAAA////AP8A
+AAAA/wAAAAD/AP//AAD/AP8AAP//AIAAAAAAgAAAAACAAICAAACAAIAAAICAAMDA
+wACAgIAAmZn/AJkzZgD//8wAzP//AGYAZgD/gIAAAGbMAMzM/wAAAIAA/wD/AP//
+AAAA//8AgACAAIAAAAAAgIAAAAD/AADM/wDM//8AzP/MAP//mQCZzP8A/5nMAMyZ
+/wD/zJkAM2b/ADPMzACZzAAA/8wAAP+ZAAD/ZgAAZmaZAJaWlgAAM2YAM5lmAAAz
+AAAzMwAAmTMAAJkzZgAzM5kAMzMzAIUAGgAUAwAAAAAJAVcAbwByAGsAcwBoAGUA
+ZQB0AMEBCADBAQAAZ+YBAK4BBAABAAEEFwAIAAEAAAAAAAAA/AAIAAAAAAAAAAAA
+CgAAAAkIEAAABhAAuw3MB9EAAQAGBAAAKgACAAAAKwACAAAAggACAAEAgAAIAAAA
+AAAAAAAAgQACAMEEFAADAAAAARUAAwAAAAGDAAIAAACEAAIAAAAmAAgAZmZmZmZm
+5j8nAAgAZmZmZmZm5j8oAAgAAAAAAAAA6D8pAAgAAAAAAAAA6D+hACIAAQBkAAEA
+AQABAAIAWAJYAjMzMzMzM9M/MzMzMzMz0z8BAFUAAgAIAH0ADAAAAAAAJAEPAAAA
+AAAAAg4AAAAAAAMAAAABAAIAAAAFAggAAAAAAA8AAAABAgYAAQAAAA8APgISALYG
+AAAAAEAAAAAAAGQAAAAAAIsIEACLCAAAAAAAAAAAAABkAAAAHQAPAAMAAAAAAAAB
+AAAAAAAAAGcIFwBnCAAAAAAAAAAAAAACAAH//////38AAAoAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD+/wAABgECAAAAAAAAAAAA
+AAAAAAAAAAABAAAAAtXN1ZwuGxCTlwgAKyz5rjAAAAC8AAAACAAAAAEAAABIAAAA
+FwAAAFAAAAALAAAAWAAAABAAAABgAAAAEwAAAGgAAAAWAAAAcAAAAA0AAAB4AAAA
+DAAAAI4AAAACAAAA5AQAAAMAAAAAAAwACwAAAAAAAAALAAAAAAAAAAsAAAAAAAAA
+CwAAAAAAAAAeEAAAAQAAAAoAAABXb3Jrc2hlZXQADBAAAAIAAAAeAAAAEwAAAEZl
+dWlsbGVzIGRlIGNhbGN1bAADAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+UgBvAG8AdAAgAEUAbgB0AHIAeQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAABYABQD//////////wEAAAAACQIAAAAAAMAAAAAAAABG
+AAAAAAAR5fd1atUBABHl93Vq1QEBAAAAgAYAAAAAAAAFAFMAdQBtAG0AYQByAHkA
+SQBuAGYAbwByAG0AYQB0AGkAbwBuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KAACAAIAAAADAAAA/////wAJAgAAAAAAwAAAAAAAAEYAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAADoAAAAAAAAAFcAbwByAGsAYgBvAG8AawAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASAAIA////////////////
+AAkCAAAAAADAAAAAAAAARgAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAFYEAAAAAAAA
+BQBEAG8AYwB1AG0AZQBuAHQAUwB1AG0AbQBhAHIAeQBJAG4AZgBvAHIAbQBhAHQA
+aQBvAG4AAAAAAAAAAAAAADgAAgD///////////////8ACQIAAAAAAMAAAAAAAABG
+AAAAAAAAAAAAAAAAAAAAAAAAAAAWAAAA7AAAAAAAAAD+////AgAAAAMAAAAEAAAA
+/v////7////9////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+/////////////////////w==
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: ppt
+
+0M8R4KGxGuEAAAAAAAAAAAAAAAAAAAAAPgADAP7/CQAGAAAAAAAAAAAAAAADAAAA
+AQAAAAAAAAAAEAAAAgAAAAEAAAD+////AAAAAAAAAAAFAAAABgAAAP//////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+///////////////////////////////////////////9////JQAAAP7///8EAAAA
+FQAAAP3////9////CAAAAAkAAAAKAAAACwAAAAwAAAANAAAADgAAAA8AAAAQAAAA
+EQAAABIAAAATAAAAFAAAAAMAAAAWAAAAFwAAABgAAAAZAAAAGgAAABsAAAAcAAAA
+HQAAAB4AAAAfAAAAIAAAACEAAAAiAAAA/v///yQAAAD+/////v//////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+/////////////////////1IAbwBvAHQAIABFAG4AdAByAHkAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWAAUA//////////8BAAAA
+EI2BZJtPzxGG6gCqALkp6AAAAAAAAAAAAAAAADAmSu5catUBIwAAAIADAAAAAAAA
+UABvAHcAZQByAFAAbwBpAG4AdAAgAEQAbwBjAHUAbQBlAG4AdAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACgAAgECAAAAAwAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAHAAAACDsAAAAAAAAFAFMAdQBtAG0AYQByAHkA
+SQBuAGYAbwByAG0AYQB0AGkAbwBuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KAACAQQAAAD//////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAADYAAAAAAAAAAUARABvAGMAdQBtAGUAbgB0AFMAdQBtAG0AYQByAHkA
+SQBuAGYAbwByAG0AYQB0AGkAbwBuAAAAAAAAAAAAAAA4AAIB////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAANgBAAAAAAAA
+AQAAAAIAAAADAAAA/v///wUAAAAGAAAABwAAAAgAAAAJAAAACgAAAAsAAAD+////
+DQAAAP7/////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+///////////////////////////////////////////bG+vbT8cGCrsIhKTv96k9
+/q6L+eGU5yAWmqoGw+JDP8to4XY9v3+CyYWkpyUIW3hwhqN727VfvFDRozzNMRul
+SLYwlRIPiNlPvFKuQmTRyRDSSkXbNGIkf6eRcV/XH5ieGeA2TNP1FlLXN2Cuj6jJ
+/7PDMMyeT8F/ryzlRQRuN5RMaeRioagv41O9kKhlqtQe0LW4+db9AQAA//8DAFBL
+AwQUAAYACAAAACEAa3mWFoMAAACKAAAAHAAAAHRoZW1lL3RoZW1lL3RoZW1lTWFu
+YWdlci54bWwMzE0KwyAQQOF9oXeQ2TdjuyhFYrLLrrv2AEOcGkHHoNKf29fl44M3
+zt8U1ZtLDVksnAcNimXNLoi38Hwspxuo2kgcxSxs4ccV5ul4GMm0jRPfSchzUX0j
+1ZCFrbXdINa1K9Uh7yzdXrkkaj2LR1fo0/cp4kXrKyYKAjj9AQAA//8DAFBLAwQU
+AAYACAAAACEAadq2p4MGAABiGwAAFgAAAHRoZW1lL3RoZW1lL3RoZW1lMS54bWzs
+WU9v2zYUvw/YdyB0b2MnthsHdYrYsZutTRvEboceaYmWWFOiQNJJfRva44ABw7ph
+lwG77TBsK9ACu3SfJluHrQP6FfZISrYYKUjSBtuwxYdEIn98/9/jI3X9xqOYoQMi
+JOVJx6tfrXmIJD4PaBJ2vHujwZV1D0mFkwAznpCONyfSu7H5/nvX8YaKSEwQrE/k
+Bu54kVLpxsqK9GEYy6s8JQnMTbiIsYJXEa4EAh8C3ZitrNZqrZUY08RDCY6B7N3J
+hPoEjTRJbzMn3mfwmiipB3wmhpo0cVYYbDCta4Scyx4T6ACzjgd8An44Io+UhxiW
+CiY6Xs38vJXN6yt4I1vE1AlrC+sG5petyxYE01XDU4TjBdP6oNG+tr2gbwBMlXH9
+fr/Xry/oGQD2fdDUylKk2Ris17s5zQLIPpZp92rNWsPFF+ivlWRud7vdZjuTxRI1
+IPvYKOHXa63G1qqDNyCLb5bwje5Wr9dy8AZk8a0SfnCt3Wq4eAOKGE2mJbR26GCQ
+UV9AJpztVMLXAb5ey+BLFETDIro0iwlP1EmxFuOHXAwAoIEMK5ogNU/JBPsQxT3M
+6FhQzQBvEFyYsUO+LA1pXkj6gqaq432YYsiIJb03L79/8/I5evPy2dHjF0ePfzp6
+8uTo8Y+WlrNwBydhceHrbz/78+uP0R/Pv3n99ItqvCzif/3hk19+/rwaCBm0lOjV
+l89+e/Hs1Vef/v7d0wr4lsDjInxEYyLRHXKI9nkMuhnDuJKTsTjfilGEaXHFVhJK
+nGDNpYJ+X0UO+s4cM1yB6xLXgvcFVJAq4M3ZQ0fgYSRmKnO5o9mtKHaAu5yzLheV
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+/////////////////////w8A6AMbCwAAAQDpAygAAACcEgAATxoAAOAQAACAFgAA
+AQAAAAIAAAAAAAAAAAAAAAEABgAAAAABDwDyA6wBAAAvAMgPDAAAADAA0g8EAAAA
+AQAAAA8A1QfkAAAAAAC3D0QAAABDAGEAbABpAGIAcgBpAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEIhAAtw9EAAAA
+QQByAGkAYQBsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAABAAgALcPRAAAAFQAaQBtAGUAcwAgAE4AZQB3ACAA
+UgBvAG0AYQBuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQS
+AACkDwgAAACAAEAAAAAAAAAApQ8MAAAAAAAACC4AAAACAAAAAACpDwoAAAAHAAAA
+AgAJBAAAQACjD24AAAAFAP/9PwAAACIgAQBkAAAAAP8AAGQAAAAAAAAAAABAAgAA
+AAAHAAAA///vAAAAAAD//wAA//8SAAAAAAEAAAAFAAAgASABAAAAAAAFAABAAkAC
+AAAAAAAFAABgA2ADAAAAAAAFAACABIAEAAAAAA8ACwR0AAAADwAA8GwAAAAAAAbw
+IAAAAAIIAAADAAAAAwAAAAIAAAACAAAAAwAAAAEAAAACAAAAYwAL8CQAAACBAQQA
+AAiDAQAAAAi/ARAAEADAAQEAAAj/AQgACAABAgIAAAhAAB7xEAAAAAQAAAgBAAAI
+AgAACPcAABAfAPAPHAAAAAAA8wMUAAAAAgAAAAAAAAAAAAAAAAAAgAAAAAAPANAH
+cQAAAA8AiBNpAAAADwCKEykAAAAAALoPEAAAAF8AXwBfAFAAUABUADEAMgAAAIsT
+CQAAAAAAJQQBAAAAAQ8AihMwAAAAAAC6DxAAAABfAF8AXwBQAFAAVAAxADAAAACL
+ExAAAAAAAA0ECAAAAADAAAAAwAAAPwDZDwwAAAAAANoPBAAAAA0AAgBPANkPDAAA
+AAAA2g8EAAAADQACAA8A8A8cAAAAAADzAxQAAAADAAAABAAAAAAAAAAAAQAAAAAA
+AAAAKATCBwAAUEsDBBQABgAIAAAAIQBWrgfD9wAAAKkBAAATAAgCW0NvbnRlbnRf
+VHlwZXNdLnhtbCCiBAIooAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAB8kM1OwzAQhO9IvIPlK4odOCCEmvTAzxGQKA+wOJvEqv/k3VbN2+OkRUKocLLW
+uzPzaVbrg3dij5lsDI28VrUUGEzsbBga+bF5ru6kIIbQgYsBGzkhyXV7ebHaTAlJ
+FHWgRo7M6V5rMiN6IBUThrLpY/bAZcyDTmC2MKC+qetbbWJgDFzx7CHb1SP2sHMs
+ng7l+0iS0ZEUD8fDOauRkJKzBriQ6n3ofqVUpwRVlMsNjTbRVcGQ+mzCvPk74KR7
+LdVk26F4g8wv4AuGZvh0+M6TQ1L/m5yhjH1vDXbR7HxpQKWMVN4F2Dv1w/qbXC9F
+t18AAAD//wMAUEsDBBQABgAIAAAAIQDt5AxLuwAAACYBAAALAAgCX3JlbHMvLnJl
+bHMgogQCKKAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAhI/NCsIw
+EITvgu8Q9m5TPYhI015E8Kr1AdZ0+4NpEpJV7NubYwuCx9lhvtkpqs9oxJtCHJxV
+sM1yEGS1awbbKbjX580BRGS0DRpnScFEEapyvSquZJBTKPaDjyJRbFTQM/ujlFH3
+NGLMnCebnNaFETnJ0EmP+okdyV2e72WYM6BcMMWlURAuzRZEPfnU/J/t2nbQdHL6
+NZLlHxWS8WHoxpNJK0SNoSNWMDtm6VuQZSEX68ovAAAA//8DAFBLAwQUAAYACAAA
+ACEA2P2Nj6wAAAC2AAAADwAAAHRhYmxlU3R5bGVzLnhtbAzMSQ6CMBhA4b2Jd2j+
+fS1DUSQUwiArd+oBKpQh6UBooxLj3WX58pIvzT9KopdY7GQ0A//gARK6Nd2kBwaP
+e4NjQNZx3XFptGCwCgt5tt+lPHFPeXOrFFfr0KZom3AGo3NzQohtR6G4PZhZ6O31
+ZlHcbbkMpFv4e9OVJIHnHYnikwbUiZ7BN6qCIKK0wKfL5YhpSANcejTGcVTW1bmp
+/SosfkCyPwAAAP//AwBQSwECLQAUAAYACAAAACEAVq4Hw/cAAACpAQAAEwAAAAAA
+AAAAAAAAAAAAAAAAW0NvbnRlbnRfVHlwZXNdLnhtbFBLAQItABQABgAIAAAAIQDt
+5AxLuwAAACYBAAALAAAAAAAAAAAAAAAAADADAABfcmVscy8ucmVsc1BLAQItABQA
+BgAIAAAAIQDY/Y2PrAAAALYAAAAPAAAAAAAAAAAAAAAAABwGAAB0YWJsZVN0eWxl
+cy54bWxQSwUGAAAAAAMAAwC3AAAA9QYAAAAAAADqAwAAAAAPAPgDLSEAAAIA7wMY
+AAAAAQAAAAECBwkIAAAAAAAAAAAAAAACAAAAYADwByAAAAD///8AAAAAAO7s4QAf
+SX0AT4G9AMBQTQAAAP8AgACAAAAAow8+AAAAAQD//T8AAAAiIAEAZAAAAAD/AABa
+AAAAAAAAAAAAQAIAAAAABwAAAP//7wAAAAAA//8AAP//LAAAAAABAAAQAKMPdgAA
+AAUA//0/AAMAIiABAGQAAAAA/wAAWgCw/wAAkAAAAEACAAAAAAcAAAD//+8AAAAA
+AP//AAD//xwAAAAAAQAAACUAANj/sAEgAQAAAgAYAAAFAADQAkACAAACABQAAAUA
+APADYAMAAAIAEgAABQAAEAWABAAAAAAgAKMPbgAAAAUA//0/AAAAIiABAGQAAAAA
+/wAAZAAeAAAAAAAAAEACAAAAAAcAAAD//+8AAAAAAP//AAD//wwAAAAAAQAAAAUA
+ACABIAEAAAAAAAUAAEACQAIAAAAAAAUAAGADYAMAAAAAAAUAAIAEgAQAAAAAQACj
+D24AAAAFAP/9PwAAACIgAQBkAAAAAP8AAGQAAAAAAAAAAABAAgAAAAAHAAAA///v
+AAAAAAD//wAA//8SAAAAAAEAAAAFAAAgASABAAAAAAAFAABAAkACAAAAAAAFAABg
+A2ADAAAAAAAFAACABIAEAAAAAFAAow9SAAAABQAAAAEJAAACAAEAAAAAAAAAAQAB
+CQAAAgABACABAAAAAAIAAQkAAAIAAQBAAgAAAAADAAEJAAACAAEAYAMAAAAABAAB
+CQAAAgABAIAEAAAAAGAAow8MAAAAAQAAAAAAAAAAAAAAcACjDz4AAAAFAAAAAAAA
+AAAAAgAYAAEAAAAAAAAAAgAUAAIAAAAAAAAAAgASAAMAAAAAAAAAAgAQAAQAAAAA
+AAAAAgAQAIAAow8+AAAABQAAAAAAAAAAAAIAFAABAAAAAAAAAAIAEgACAAAAAAAA
+AAIAEAADAAAAAAAAAAIADgAEAAAAAAAAAAIADgAAACMEYwYAAFBLAwQUAAYACAAA
+ACEAke/N8vsAAAC7AQAAEwAAAFtDb250ZW50X1R5cGVzXS54bWx8kMlOwzAQhu9I
+vIPlK4qdckAIJemB5cZyKA8wciaJhTd53Kp9eyZpkSpUOI1m/f75m/XeO7HDTDaG
+Vq5ULQUGE3sbxlZ+bl6qeymoQOjBxYCtPCDJdXd91WwOCUnwdqBWTqWkB63JTOiB
+VEwYuDPE7KFwmkedwHzBiPq2ru+0iaFgKFWZb8iuecIBtq6I5z2Xj0oyOpLi8Tg4
+s1oJKTlroLBSvQv9L0p1IijeXGZosoluWIbUFwlz52/Aae+drcm2R/EBubyBZxm6
+z6TJcfEVqLBz58lK/X/2gu44DNZgH83WsycqZSSOywveqTPQzy96sb77BgAA//8D
+AFBLAwQUAAYACAAAACEAjuoq+r4AAAA4AQAACwAAAF9yZWxzLy5yZWxzhI/BCsIw
+EETvgv8Q9m7TehCRpr2I0IMX0Q9Ykm0bbJOQjaJ/b44WBI/DMG9m6vY1T+JJka13
+CqqiBEFOe2PdoOB2PW32IDihMzh5RwrexNA261V9oQlTDvFoA4tMcaxgTCkcpGQ9
+0oxc+EAuO72PM6Ys4yAD6jsOJLdluZPxmwHNgik6oyB2pgJxfYfc/J/t+95qOnr9
+mMmlHxWSJ2vojJwoZizGgZICE/nbWIiqyPtBNrVc/G0+AAAA//8DAFBLAwQUAAYA
+CAAAACEAOJj1rTIDAADBHgAAIQAAAGRycy9zbGlkZU1hc3RlcnMvc2xpZGVNYXN0
+ZXIxLnhtbOyZX2/aMBDA3yftO0R+nVj+kJAQEaq1E0/VhEb7AUxiSFbHiWzTlT3t
+s+yj7ZPs7JiSds1UpE0iU+CBENvH+X53F98xu3goqXVPuCgqliD3vYMswtIqK9g2
+Qbc3i1GELCExyzCtGEnQngh0MX/7ZlbH8mEl95QIC0QwEeME5VLWsW2LNCclFu+r
+mjAY21S8xBK+8q2dcfwVRJfU9hxnYpe4YMis569ZX202RUo+VumuJEw2QjihWIL6
+Ii9qcZBWv0ZazYkAMXr1E5XmanuFpETvcD7DMb2nbr3kFqZbsBNFVkY2N3i9+pag
+qev7DtiNS5ogZT98zS75HVgTWUozZr7CUI7ZFra/3LFUqnEtma3qVF2IOl2m0rrH
+IGbqwAvZ85kNP92acEk2z6cepsFyM7refQJWsBjHoOVn0FqAmo2Sd4QrzmB9/eOi
+okW2KCjVUhU3ckV5o4N8cI0GT2bpLVlyX5MNTsEj3pVfRlSqmTgm+NkAwc1AKp4N
+pMLIbjRsNtrYGK6fWr+O11W2/w1Fifl1gjwvmijrFywDlAkaHW6cDykpGoO6R6Zt
+WIuKyZZ1PvACg3/VmFUCzOt4zqUzcXz4PLx9GC1kmi9wWdB9gsZwI80xFwT2r90B
+x+vdFdzRtxP08/uPBkPLHbxIWe1fuAPrcgc26nAHNvqjO+jo81T0NcgnUaCV7wHy
+4DGKz4C4zlJnTlxhVpkE8u34SNx1/bEKnj5E+Vkh10Y7c+SKs0Hut5BDVldPqQH5
+aXnd7UNeV5wN8uCI3HOCUOeoIbGf9CjvBXLF2SCftJAHrt+X49s5JfZeIFecDfKw
+hXwaNtoPUf7/RbnibJBHR+Rj31MF7fAsP7FG60WUK84G+bSFPIomw/Ht9LK8F8gV
+Z92lafVl6riSOeGPXRpoNSwbxzBNBwq9rwQRNrpdHVs/Zsqhu9bU95Ao/n7zptX6
+6IWNVbfRhFWr9eEHoY6qwT4vNwoOvdjBPh1V9Th09Xl7MNDLNagbeZGuoAcDdVRs
+ut8+pGh4ZHXUN6E/HnK07uB2VANgHd3mGUKs4+w8CcIhSWsPejxptg+X6u9B89/z
+/BcAAAD//wMAUEsBAi0AFAAGAAgAAAAhAJHvzfL7AAAAuwEAABMAAAAAAAAAAAAA
+AAAAAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECLQAUAAYACAAAACEAjuoq+r4A
+AAA4AQAACwAAAAAAAAAAAAAAAAAsAQAAX3JlbHMvLnJlbHNQSwECLQAUAAYACAAA
+ACEAOJj1rTIDAADBHgAAIQAAAAAAAAAAAAAAAAATAgAAZHJzL3NsaWRlTWFzdGVy
+cy9zbGlkZU1hc3RlcjEueG1sUEsFBgAAAAADAAMAyQAAAIQFAAAAAA8ADATFBQAA
+DwAC8L0FAAAQAAjwCAAAAAEAAAABCAAADwAD8DAAAAAPAATwKAAAAAEACfAQAAAA
+AAAAAAAAAAAAAAAAAAAAAAIACvAIAAAAAAgAAAUAAAAPAATwbQUAABIACvAIAAAA
+AQgAAAAMAABjAAvwJAAAAIEBAAAACIMBBQAACL8BEAAQAP8BAAAIAAQDCQAAAD8D
+AQABABMAIvEpBQAAqcMjBQAAUEsDBBQABgAIAAAAIQDw94q7/QAAAOIBAAATAAAA
+W0NvbnRlbnRfVHlwZXNdLnhtbJSRzUrEMBDH74LvEOYqbaoHEWm6B6tHFV0fYEim
+bdg2CZlYd9/edD8u4goeZ+b/8SOpV9tpFDNFtt4puC4rEOS0N9b1Cj7WT8UdCE7o
+DI7ekYIdMayay4t6vQvEIrsdKxhSCvdSsh5oQi59IJcvnY8TpjzGXgbUG+xJ3lTV
+rdTeJXKpSEsGNHVLHX6OSTxu8/pAEmlkEA8H4dKlAEMYrcaUSeXszI+W4thQZude
+w4MNfJUxQP7asFzOFxx9L/lpojUkXjGmZ5wyhjSRJQ8YKGvKv1MWzIkL33VWU9lG
+fl98J6hz4cZ/uUjzf7PbbHuj+ZQu9z/UfAMAAP//AwBQSwMEFAAGAAgAAAAhADHd
+X2HSAAAAjwEAAAsAAABfcmVscy8ucmVsc6SQwWrDMAyG74O9g9G9cdpDGaNOb4Ve
+Swe7CltJTGPLWCZt376mMFhGbzvqF/o+8e/2tzCpmbJ4jgbWTQuKomXn42Dg63xY
+fYCSgtHhxJEM3Elg372/7U40YalHMvokqlKiGBhLSZ9aix0poDScKNZNzzlgqWMe
+dEJ7wYH0pm23Ov9mQLdgqqMzkI9uA+p8T9X8hx28zSzcl8Zy0Nz33r6iasfXeKK5
+UjAPVAy4LM8w09zU50C/9q7/6ZURE31X/kL8TKv1x6wXNXYPAAAA//8DAFBLAwQU
+AAYACAAAACEAa0u0o74AAAARAQAAEAAAAGRycy9zaGFwZXhtbC54bWyMj01uAjEM
+RvdIvUPkfUmmC4RGk2GBxAGq9gCeiQkjEidKAp3envAjJHbsbH/Se/66zeydOFPK
+U2ANzVKBIB6Dmdhq+P3Zfa5B5IJs0AUmDf+UYdN/LLrYDjgebQonNqJCOLdRw6GU
+2EqZxwN5zMsQiWu2D8ljqWuyMibKxAVLFXonv5RaSY8TQ39F2m/ai8nM9RWlmnrD
+9sairUsPC75jMQn/aoUXgTij0zDYBmTfyYfsPj2b9BcAAAD//wMAUEsDBBQABgAI
+AAAAIQBLc0FM1gAAAP8AAAAPAAAAZHJzL2Rvd25yZXYueG1sTI9BS8NAEIXvgv9h
+GcGb3ShYJM2mFEUED4qNoMdpdrqJZmdDdpvG/nqnXurlMcMb3nyvWE6+UyMNsQ1s
+4HqWgSKug23ZGXivHq/uQMWEbLELTAZ+KMKyPD8rMLdhz280rpNTEsIxRwNNSn2u
+dawb8hhnoScWbxsGj0nWwWk74F7CfadvsmyuPbYsHxrs6b6h+nu98wZeHubjwd8e
+XPX59brSU/WRPbsnYy4vptUCVKIpnY7hiC/oUArTJuzYRtUZkCLpT8WTeXNUXRb6
+P3f5CwAA//8DAFBLAQItABQABgAIAAAAIQDw94q7/QAAAOIBAAATAAAAAAAAAAAA
+AAAAAAAAAABbQ29udGVudF9UeXBlc10ueG1sUEsBAi0AFAAGAAgAAAAhADHdX2HS
+AAAAjwEAAAsAAAAAAAAAAAAAAAAALgEAAF9yZWxzLy5yZWxzUEsBAi0AFAAGAAgA
+AAAhAGtLtKO+AAAAEQEAABAAAAAAAAAAAAAAAAAAKQIAAGRycy9zaGFwZXhtbC54
+bWxQSwECLQAUAAYACAAAACEAS3NBTNYAAAD/AAAADwAAAAAAAAAAAAAAAAAVAwAA
+ZHJzL2Rvd25yZXYueG1sUEsFBgAAAAAEAAQA9QAAABgEAAAAABAA8AcgAAAA////
+AAAAAADu7OEAH0l9AE+BvQDAUE0AAAD/AIAAgAAAAA4E+wsAAFBLAwQUAAYACAAA
+ACEA6d4Pv/8AAAAcAgAAEwAAAFtDb250ZW50X1R5cGVzXS54bWyskctOwzAQRfdI
+/IPlLUqcskAIJemCx47HonzAyJkkFsnYsqdV+/dM0lRCqCAWbCzZM/eeO+NyvR8H
+tcOYnKdKr/JCKyTrG0ddpd83T9mtVomBGhg8YaUPmPS6vrwoN4eASYmaUqV75nBn
+TLI9jpByH5Ck0vo4Ass1diaA/YAOzXVR3BjriZE448lD1+UDtrAdWD3u5fmYJOKQ
+tLo/Nk6sSkMIg7PAktTsqPlGyRZCLsq5J/UupCuJoc1ZwlT5GbDoXmU10TWo3iDy
+C4wSw7AMiV/PZyAZLea/O56J7NvWWWy83Y6yjnw2XsxOwf8UYPU/6BPTzH9bfwIA
+AP//AwBQSwMEFAAGAAgAAAAhAKXWp+fAAAAANgEAAAsAAABfcmVscy8ucmVsc4SP
+z2rDMAyH74W9g9F9UdLDGCV2L6WQQy+jfQDhKH9oIhtWuKV5Fcw8miVhNXMxK+L2
+MT6o4t3DiePf/iyF0kmrSPYi4oi5x3CicEgSopCe41NCKuz1gFLHrrvUF1zyiUIP
+KOpiWmmSER070bRctENj8Mu8SkDwt2Ob3fuoy1mV1tvkwEVCVmBWIfyIMMeMN/FM
+4biK5AjHrGjw21hFVUIO58Iv4vpSgadDwjjqB0TKqjV3BehbcPotqB7Vbt9l89hF
+CkWnVTRvY86LyG0+7UU4TquwQ5pERewHcgohitEeV1XwXe5miH4HP+DkRHffp8Rx
+9+nV4B4NHZGWAaJnZkL7Eqq1U4RjmlxW5DNX5C1BK1Ni51gdPgl3vPr2uAjov7/4
+buNZskcg3ss70GXtvay93n++9p6Uz2etuMsiC/VX9zm2QTbtcnxitzyhjA3VnJHb
+0jTMEjaMYACDep05KZLF6SmN4DEr8A4uFNisQYKrj6iKhhFOodmue5pIKDPSoUQp
+l3DIM8OVtDUeGnZlj4hNfXiw9UBitcsDO7ymh/MzwoKM2XZCcxDNGa1pAmdltnYt
+Iwpqvw2zuhbqzNzqRjRT6hxuC5XBh2XVYHBhTehEEPQvYOUWnNU1azikYEYCbXe7
+CeduMV64SBfJCAck85HWu+yjunFSHivmVgBip8JH+sB3itUK3Nqa7DtwO4uTiuwa
+J7DLvfcuXsojeOklnbfH0pElxeRkCTrseO3matNDPk473gTOt/AYp+B1qZs/zEK4
+JPKVsGF/ajKbLF96s50r5iZBHa4srN1LCjt1IBVSbWMZ2dAwU1kIsERzsvKvNsGs
+F6WAjfS3kGJtHYLhH5MC7Oi6lkwmxFdFZxdGtO3sa1ZK+UwRMYyCQzRmM7GPwf06
+VEGfgEq4pjAVQb/AnZq2tplyi3OWdMWbLIOz45ilEc7KrU7RPJMt3OTxQgbzVhAP
+dKuU3Sh3flVMyl+QKsUw/p+povcTuDJYC7QHfLjSFRjpfO14XKiIQxVKI+oPBDQO
+pnZAtMC9LExDUMHFsvkvyIH+b3PO0jBpDSc/tU9DJCjsRyoShOxBWTLRdwqxerZ3
+WZIsI2QiqiCuTK3YY3JA2EjXwJbe2z0UQaibapKVAYM7Hn/ue5ZB41A3OcV8c2rI
+Yu+1OfB3dz42mUEptw6bhia3/0LEil3VrjfL8723qIieWLZZjTwrgFlhK2hnaf+W
+Ipxzq7UVq6TxajMXDrxY1hgGFw1RChc/SP+B/Y8Kn9mvFHpDHfF9qK0IPjpkQiEI
+6ystXe48BBUyHx1nhTO3sCZmjZs1T9pu+XZ9wb2u7YNK+p+51z2nuRftmetdJxsv
+1Ny5jV1zZ6M2eyvMDf49nqgwNMmPM8Y55iNX8TsUHz8Ed2/Djf+MKWlMCl+ZBIYG
+dGiyAUqA5WiWbv4FAAD//wMAUEsDBBQABgAIAAAAIQAN0ZCftgAAABsBAAAnAAAA
+dGhlbWUvdGhlbWUvX3JlbHMvdGhlbWVNYW5hZ2VyLnhtbC5yZWxzhI9NCsIwFIT3
+gncIb2/TuhCRJt2I0K3UA4TkNQ02PyRR7O0NriwILodhvplpu5edyRNjMt4xaKoa
+CDrplXGawW247I5AUhZOidk7ZLBggo5vN+0VZ5FLKE0mJFIoLjGYcg4nSpOc0IpU
++YCuOKOPVuQio6ZByLvQSPd1faDxmwF8xSS9YhB71QAZllCa/7P9OBqJZy8fFl3+
+UUFz2YUFKKLGzOAjm6pMBMpburrE3wAAAP//AwBQSwECLQAUAAYACAAAACEA6d4P
+v/8AAAAcAgAAEwAAAAAAAAAAAAAAAAAAAAAAW0NvbnRlbnRfVHlwZXNdLnhtbFBL
+AQItABQABgAIAAAAIQCl1qfnwAAAADYBAAALAAAAAAAAAAAAAAAAADABAABfcmVs
+cy8ucmVsc1BLAQItABQABgAIAAAAIQBreZYWgwAAAIoAAAAcAAAAAAAAAAAAAAAA
+ABkCAAB0aGVtZS90aGVtZS90aGVtZU1hbmFnZXIueG1sUEsBAi0AFAAGAAgAAAAh
+AGnatqeDBgAAYhsAABYAAAAAAAAAAAAAAAAA1gIAAHRoZW1lL3RoZW1lL3RoZW1l
+MS54bWxQSwECLQAUAAYACAAAACEADdGQn7YAAAAbAQAAJwAAAAAAAAAAAAAAAACN
+CQAAdGhlbWUvdGhlbWUvX3JlbHMvdGhlbWVNYW5hZ2VyLnhtbC5yZWxzUEsFBgAA
+AAAFAAUAXQEAAIgKAAAAAAAADwQ6AQAAPD94bWwgdmVyc2lvbj0iMS4wIiBlbmNv
+ZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9InllcyI/Pg0KPGE6Y2xyTWFwIHhtbG5z
+OmE9Imh0dHA6Ly9zY2hlbWFzLm9wZW54bWxmb3JtYXRzLm9yZy9kcmF3aW5nbWwv
+MjAwNi9tYWluIiBiZzE9Imx0MSIgdHgxPSJkazEiIGJnMj0ibHQyIiB0eDI9ImRr
+MiIgYWNjZW50MT0iYWNjZW50MSIgYWNjZW50Mj0iYWNjZW50MiIgYWNjZW50Mz0i
+YWNjZW50MyIgYWNjZW50ND0iYWNjZW50NCIgYWNjZW50NT0iYWNjZW50NSIgYWNj
+ZW50Nj0iYWNjZW50NiIgaGxpbms9ImhsaW5rIiBmb2xIbGluaz0iZm9sSGxpbmsi
+Lz4QAB4EYgQAAFBLAwQUAAYACAAAACEATY7z/P0AAAC7AQAAEwAAAFtDb250ZW50
+X1R5cGVzXS54bWx8kMtOxDAMRfdI/EOULWrSYYEQajsLHisELIYPsBK3jchLcTqa
+/j1pZ5AADawsP67PtZvtwVm2x0Qm+JZvRM0ZehW08UPL33dP1S1nlMFrsMFjy2ck
+vu0uL5rdHJFYUXtq+ZhzvJOS1IgOSISIvnT6kBzkkqZBRlAfMKC8rusbqYLP6HOV
+lx28ax6wh8lm9ngo5aOThJY4uz8OLqyWQ4zWKMjFqdx7/YtSnQiiKNcZGk2kq2KD
+y7OEpfM34KR7La9JRiN7g5RfwBUbUieSZEvxGeYw5R/JRvy/9ozv0PdGoQ5qcuUn
+IiakEtcTnBXfQF+3yPX13ScAAAD//wMAUEsDBBQABgAIAAAAIQBw8DjcvgAAADgB
+AAALAAAAX3JlbHMvLnJlbHOEj8EKwjAQRO+C/xD2btN6EJGmvYggeBL9gCXZtsE2
+Cdko9u/N0YLgcRjmzUzdvqdRvCiy9U5BVZQgyGlvrOsV3G+nzR4EJ3QGR+9IwUwM
+bbNe1VcaMeUQDzawyBTHCoaUwkFK1gNNyIUP5LLT+ThhyjL2MqB+YE9yW5Y7Gb8Z
+0CyY4mwUxLOpQNzmkJv/s33XWU1Hr58TufSjQvJoDV1w9s+UsRh7SgpM5G9jIaoi
+7wfZ1HLxt/kAAAD//wMAUEsDBBQABgAIAAAAIQBFHMLSLwEAAEkCAAAhAAAAZHJz
+L3NsaWRlTGF5b3V0cy9zbGlkZUxheW91dDEueG1sjFK7bsMwDNwL9B8E7Y3SDkUh
+xM7Q19I2AdJ+ACHTsQG9QCmu8/elnbgvZMgiUeTdiSdqseydFR1SaoMv5PVsLgV6
+E6rWbwv58f50dSdFyuArsMFjIfeY5LK8vFhEnWz1Avuwy4I1fNJQyCbnqJVKpkEH
+aRYieq7VgRxkPtJWVQSfrO2supnPb5WD1ssjn87hh7puDT4Es3Po80GE0ELm/lPT
+xjSpxXPUImFimZH9p6WS/ZmNrYY9xXdCHCLfPVPcxDWN5bduTaKt+NWk8OD4caQ6
+Fo6w8egZxoH6R99OSqD7mly5AM3eRF9IHsF+WJkEGvsszCFpfrKmWZ3AmubxBFpN
+F3AH35dyPNnicHJqLL1CXHXsDzQPMCPdj6nIIzt4+AUZNKYvUH4BAAD//wMAUEsB
+Ai0AFAAGAAgAAAAhAE2O8/z9AAAAuwEAABMAAAAAAAAAAAAAAAAAAAAAAFtDb250
+ZW50X1R5cGVzXS54bWxQSwECLQAUAAYACAAAACEAcPA43L4AAAA4AQAACwAAAAAA
+AAAAAAAAAAAuAQAAX3JlbHMvLnJlbHNQSwECLQAUAAYACAAAACEARRzC0i8BAABJ
+AgAAIQAAAAAAAAAAAAAAAAAVAgAAZHJzL3NsaWRlTGF5b3V0cy9zbGlkZUxheW91
+dDEueG1sUEsFBgAAAAADAAMAyQAAAIMDAAAAAAAAHAQEAAAAAQAAACAAug8YAAAA
+TwBmAGYAaQBjAGUAIABUAGgAZQBtAGUADwDuA2wOAAACAO8DGAAAABAAAAAAAAAA
+AAAAAAAAAIAAAAAAAQAAAA8ADAT2CAAADwAC8O4IAAAgAAjwCAAAAAIAAAACBAAA
+DwAD8JgIAAAPAATwKAAAAAEACfAQAAAAAAAAAAAAAAAAAAAAAAAAAAIACvAIAAAA
+AAQAAAUAAAAPAATwYAgAABIACvAIAAAAAgQAAAAKAADDAAvwYAAAAH8AAAAEAIAA
+QO/T44EAAAAAAIIAAAAAAIMAAAAAAIQAAAAAAIUAAgAAAL8AAAAGAL8BAAAQAP8B
+AAAIAIDDGAAAAL8DAAACAFIAZQBjAHQAYQBuAGcAbABlACAAMQAAABMAIvEjBwAA
+qcMdBwAAUEsDBBQABgAIAAAAIQDw94q7/QAAAOIBAAATAAAAW0NvbnRlbnRfVHlw
+ZXNdLnhtbJSRzUrEMBDH74LvEOYqbaoHEWm6B6tHFV0fYEimbdg2CZlYd9/edD8u
+4goeZ+b/8SOpV9tpFDNFtt4puC4rEOS0N9b1Cj7WT8UdCE7oDI7ekYIdMayay4t6
+vQvEIrsdKxhSCvdSsh5oQi59IJcvnY8TpjzGXgbUG+xJ3lTVrdTeJXKpSEsGNHVL
+HX6OSTxu8/pAEmlkEA8H4dKlAEMYrcaUSeXszI+W4thQZudew4MNfJUxQP7asFzO
+Fxx9L/lpojUkXjGmZ5wyhjSRJQ8YKGvKv1MWzIkL33VWU9lGfl98J6hz4cZ/uUjz
+f7PbbHuj+ZQu9z/UfAMAAP//AwBQSwMEFAAGAAgAAAAhADHdX2HSAAAAjwEAAAsA
+AABfcmVscy8ucmVsc6SQwWrDMAyG74O9g9G9cdpDGaNOb4VeSwe7CltJTGPLWCZt
+376mMFhGbzvqF/o+8e/2tzCpmbJ4jgbWTQuKomXn42Dg63xYfYCSgtHhxJEM3Elg
+372/7U40YalHMvokqlKiGBhLSZ9aix0poDScKNZNzzlgqWMedEJ7wYH0pm23Ov9m
+QLdgqqMzkI9uA+p8T9X8hx28zSzcl8Zy0Nz33r6iasfXeKK5UjAPVAy4LM8w09zU
+50C/9q7/6ZURE31X/kL8TKv1x6wXNXYPAAAA//8DAFBLAwQUAAYACAAAACEA8ZFh
+SLgCAAD4BQAAEAAAAGRycy9zaGFwZXhtbC54bWykVF1P2zAUfZ+0/2D5dWJpaGEQ
+kSKYxDYJoaotP8BJnJLVuY5sp7T8+h3bCTAepgny4NzkHl8fn/txcblvFdtJYxtN
+OU+/TjiTVOqqoU3O79c3R2ecWSeoEkqTzPlBWn45//zpostsx7CZbNbl/MG5LksS
+Wz7IVtivupMEX61NKxw+zSbpjLSSnHA4qFXJ8WRymrSiIT5HKNqtuoXxVnm3WxjW
+VDk/5oxEiyOXsgSBjZIs5cmA8fBgE+AwktcxbIglsn1t2oGi+B+KlRGPuPcbdiLT
+dc32Of+Wnk+PIccB5nR2cnrm2YhM7h0rvXuanoBzCXc6TSenM+9OIguP64x1P6T+
+MCPmA+XcQBVoJzKxu7UuHjUeEfSIKnSZ21/r6uCRBd4QNybt3YqwRyOQcUI5cKZ+
+kc05isaNhhmNYjD8yaSveqfrZuAZiXiHsm7lDkjtB0mFTIz1+O6reUaoHdYKcxtu
+BWMZDLVT4d1QhSIOplAbdIzirJL1WhSrp5yfp7PZBGIYF9FS3NK12aIgOKs1uauw
+RUAKKIdOoMGNLQ+ocNTeoqcS4dOQWUWrrvScbFcuSsd2AmHTiX+G2nqNuJb1iHU2
+YkcY9r94r2r3D9zgLfrvyqz3QdaiXz09mze4xvPHHWogQJwoYhGKDGosY0sGLf1R
+aGws0HXbt02rfzdRUtw455KO7lcYMVAvnZxAiCLqHNZ+LDPrTLPFKCC9ChZnW2n8
+vMIYQcu91CNuGnaSnzyqeZI/w2chrFSNn1+Ak14YrWtve2KK/Er6plEqtlH8Y7Vq
+Kv/Tu8Ngk9AkCuv2YRShu/9CybpGV45K9Lc0KNX7MIMd8s7coZO1KEFo3bTSsjv5
+yJa6FTTMFPEK8aWlIymio7RvHKUdSgH6eqJuzvzQQYqx4g/WLoyDcQxgUtpu/gcA
+AP//AwBQSwMEFAAGAAgAAAAhAORiipHWAAAA+QAAAA8AAABkcnMvZG93bnJldi54
+bWxEj1FrwjAUhd8H/odwhb3NtDJkdkZxg9Eh+FCVPV+ba9OtSbrkTuu/X9jDfDyc
+w3f4FqvBduJMIbbeKcgnGQhytdetaxQc9m8PTyAio9PYeUcKrhRhtRzdLbDQ/uIq
+Ou+4EQniYoEKDHNfSBlrQxbjxPfkUnfywSKnGBqpA14S3HZymmUzabF16cFgT6+G
+6q/dj1VAL2U1/1xHs5Uhj/l2M5s/lt9K3Y+H9TMIpoFv46P54E31X/6h3rWCKYhT
+eT2GVlcYmYKC5JZMkyXI5S8AAAD//wMAUEsBAi0AFAAGAAgAAAAhAPD3irv9AAAA
+4gEAABMAAAAAAAAAAAAAAAAAAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECLQAU
+AAYACAAAACEAMd1fYdIAAACPAQAACwAAAAAAAAAAAAAAAAAuAQAAX3JlbHMvLnJl
+bHNQSwECLQAUAAYACAAAACEA8ZFhSLgCAAD4BQAAEAAAAAAAAAAAAAAAAAApAgAA
+ZHJzL3NoYXBleG1sLnhtbFBLAQItABQABgAIAAAAIQDkYoqR1gAAAPkAAAAPAAAA
+AAAAAAAAAAAAAA8FAABkcnMvZG93bnJldi54bWxQSwUGAAAAAAQABAD1AAAAEgYA
+AAAAAAAQ8AgAAADPAcUB8wEhAg8AEfBCAAAADwCIEzoAAAAPAIoTMgAAAAAAug8O
+AAAAXwBfAF8AUABQAFQAOQAAAIsTFAAAAAAArA8MAAAAAAAAAAAAAAAAAAAADwAN
+8FsAAAAAAJ8PBAAAAAQAAAAAAKgPAQAAADEAAKEPGAAAAAIAAAAAAAAAAAACAAAA
+AABDAAIAAgAKAAAAqg8KAAAAAgAAAAEAAAAAAAAApg8MAAAA8AAAANQB0ALwAxAF
+DwAE8DYAAAASAArwCAAAAAEEAAAADAAAUwAL8B4AAACDAQUAAAi/ARAAEAD/AQAA
+CAAEAwkAAAA/AwEAAQAQAPAHIAAAAP///wAAAAAA7uzhAB9JfQBPgb0AwFBNAAAA
+/wCAAIAAAAAOBKUDAABQSwMEFAAGAAgAAAAhAFIZUwT0AAAAqAEAABMAAABbQ29u
+dGVudF9UeXBlc10ueG1sfJDLTsQwDEX3SPxD5C1qU1gghJrOgseOx2L4ACt124g2
+iRLPaObvcTtFQmhgE8mx773HrjeHaVR7StkFb+C6rECRt6F1vjfwsX0u7kBlRt/i
+GDwZOFKGTXN5UW+PkbIStc8GBuZ4r3W2A02YyxDJS6cLaUKWMvU6ov3EnvRNVd1q
+GzyT54JnD2jqR+pwN7J6Osj3iSTRmEE9nAbnLAMY4+gsspDqvW9/pRRrQinKZSYP
+LuYrwQB9NmHu/B2w6t7kNMm1pN4x8StOgqFZlqSf7wt6WS2V/zueQQ5d5yy1we4m
+OUe5GK9m3+B6uXPzBQAA//8DAFBLAwQUAAYACAAAACEApdan58AAAAA2AQAACwAA
+AF9yZWxzLy5yZWxzhI/PasMwDIfvhb2D0X1R0sMYJXYvpZBDL6N9AOEof2giG9sb
+69tPxwYKuwiEpO/3qT3+rov54ZTnIBaaqgbD4kM/y2jhdj2/f4LJhaSnJQhbeHCG
+o3vbtV+8UNGjPM0xG6VItjCVEg+I2U+8Uq5CZNHJENJKRds0YiR/p5FxX9cfmJ4Z
+4DZM0/UWUtc3YK6PqMn/s8MwzJ5PwX+vLOVFBG43lExp5GKhqC/jU72QqGWq1B7Q
+tbj51v0BAAD//wMAUEsDBBQABgAIAAAAIQBreZYWgwAAAIoAAAAcAAAAdGhlbWUv
+dGhlbWUvdGhlbWVNYW5hZ2VyLnhtbAzMTQrDIBBA4X2hd5DZN2O7KEVissuuu/YA
+Q5waQceg0p/b1+XjgzfO3xTVm0sNWSycBw2KZc0uiLfwfCynG6jaSBzFLGzhxxXm
+6XgYybSNE99JyHNRfSPVkIWttd0g1rUr1SHvLN1euSRqPYtHV+jT9yniResrJgoC
+OP0BAAD//wMAUEsBAi0AFAAGAAgAAAAhAFIZUwT0AAAAqAEAABMAAAAAAAAAAAAA
+AAAAAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECLQAUAAYACAAAACEApdan58AA
+AAA2AQAACwAAAAAAAAAAAAAAAAAlAQAAX3JlbHMvLnJlbHNQSwECLQAUAAYACAAA
+ACEAa3mWFoMAAACKAAAAHAAAAAAAAAAAAAAAAAAOAgAAdGhlbWUvdGhlbWUvdGhl
+bWVNYW5hZ2VyLnhtbFBLBQYAAAAAAwADAMQAAADLAgAAAAAAAA8EYQEAADw/eG1s
+IHZlcnNpb249IjEuMCIgZW5jb2Rpbmc9IlVURi04IiBzdGFuZGFsb25lPSJ5ZXMi
+Pz4NCjxhOmNsck1hcE92ciB4bWxuczphPSJodHRwOi8vc2NoZW1hcy5vcGVueG1s
+Zm9ybWF0cy5vcmcvZHJhd2luZ21sLzIwMDYvbWFpbiI+PGE6b3ZlcnJpZGVDbHJN
+YXBwaW5nIGJnMT0ibHQxIiB0eDE9ImRrMSIgYmcyPSJsdDIiIHR4Mj0iZGsyIiBh
+Y2NlbnQxPSJhY2NlbnQxIiBhY2NlbnQyPSJhY2NlbnQyIiBhY2NlbnQzPSJhY2Nl
+bnQzIiBhY2NlbnQ0PSJhY2NlbnQ0IiBhY2NlbnQ1PSJhY2NlbnQ1IiBhY2NlbnQ2
+PSJhY2NlbnQ2IiBobGluaz0iaGxpbmsiIGZvbEhsaW5rPSJmb2xIbGluayIvPjwv
+YTpjbHJNYXBPdnI+AAAiBAgAAAABAAAAAQAAAAAAchcQAAAAAQAwAAAAAAAjCwAA
+WCwAAAAA9Q8cAAAAAAAAAPESAAMAAAAAzDoAAAEAAAADAAAAAQDO3gAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+/v8AAAoAAgAAAAAAAAAAAAAAAAAAAAAAAQAAAOCFn/L5T2gQq5EIACsns9kwAAAA
+qAAAAAcAAAABAAAAQAAAAAIAAABIAAAACAAAAGgAAAAJAAAAfAAAAAoAAACIAAAA
+DQAAAJQAAAAPAAAAoAAAAAIAAADkBAAAHgAAABgAAABQb3dlclBvaW50IFByZXNl
+bnRhdGlvbgAeAAAADAAAAEpvbk1NeCAyMDAwAB4AAAAEAAAAMQAAAEAAAAAAAAAA
+AAAAAEAAAAAQ2EnuXGrVAQMAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAP7/AAAKAAIAAAAAAAAAAAAAAAAAAAAAAAEAAAAC1c3V
+nC4bEJOXCAArLPmuMAAAAKgBAAAPAAAAAQAAAIAAAAADAAAAiAAAAAQAAACYAAAA
+BgAAAKAAAAAHAAAAqAAAAAgAAACwAAAACQAAALgAAAAKAAAAwAAAABcAAADIAAAA
+CwAAANAAAAAQAAAA2AAAABMAAADgAAAAFgAAAOgAAAANAAAA8AAAAAwAAABPAQAA
+AgAAAOQEAAAeAAAACAAAAEN1c3RvbQAAAwAAAAg7AAADAAAAAQAAAAMAAAABAAAA
+AwAAAAAAAAADAAAAAAAAAAMAAAAAAAAAAwAAAAAAEAALAAAAAAAAAAsAAAAAAAAA
+CwAAAAAAAAALAAAAAAAAAB4QAAAFAAAACAAAAENhbGlicmkABgAAAEFyaWFsABAA
+AABUaW1lcyBOZXcgUm9tYW4ADQAAAE9mZmljZSBUaGVtZQAYAAAAUG93ZXJQb2lu
+dCBQcmVzZW50YXRpb24ADBAAAAYAAAAeAAAACwAAAEZvbnRzIFVzZWQAAwAAAAMA
+AAAeAAAABgAAAFRoZW1lAAMAAAABAAAAHgAAAA0AAABTbGlkZSBUaXRsZXMAAwAA
+AAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAD2DyMAAAAUAAAAX8CR4+Q6AAALAPQDAwAP8Epvbk1NeCAyMDAwCAAAAEoAbwBu
+AE0ATQB4ACAAMgAwADAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAEMAdQByAHIAZQBuAHQAIABVAHMAZQByAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaAAIA////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAEEAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAD///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: msi
+
+0M8R4KGxGuEAAAAAAAAAAAAAAAAAAAAAPgAEAP7/DAAGAAAAAAAAAAUAAAANAAAA
+AQAAAAAAAAAAEAAAAgAAAAEAAAD+////AAAAAAAAAAAABAAAAAgAAAAMAAAAEAAA
+ABQAAAAYAAAAHAAAACAAAAAkAAAAKAAAACwAANwvAAD/////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+//////////////////////////////////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAP3////HLAAA/v///wUAAAADAAAABgAAAAcAAAAIAAAA
+CQAAAAoAAAALAAAADAAAAA0AAAAOAAAADwAAABAAAAARAAAAEgAAABMAAAAUAAAA
+FQAAABYAAAAXAAAAGAAAABkAAAAaAAAAGwAAABwAAAAdAAAAHgAAAB8AAAAgAAAA
+IQAAACIAAAAjAAAAJAAAACUAAAAmAAAAJwAAACgAAAApAAAAKgAAACsAAAAsAAAA
+LQAAAC4AAAAvAAAAMAAAADEAAAAyAAAAMwAAADQAAAA1AAAANgAAADcAAAA4AAAA
+OQAAADoAAAA7AAAAPAAAAD0AAAA+AAAAPwAAAEAAAABBAAAAQgAAAEMAAABEAAAA
+RQAAAEYAAABHAAAASAAAAEkAAABKAAAASwAAAEwAAABNAAAATgAAAE8AAABQAAAA
+UQAAAFIAAABTAAAAVAAAAFUAAABWAAAAVwAAAFgAAABZAAAAWgAAAFsAAABcAAAA
+XQAAAF4AAABfAAAAYAAAAGEAAABiAAAAYwAAAGQAAABlAAAAZgAAAGcAAABoAAAA
+aQAAAGoAAABrAAAAbAAAAG0AAABuAAAAbwAAAHAAAABxAAAAcgAAAHMAAAB0AAAA
+dQAAAHYAAAB3AAAAeAAAAHkAAAB6AAAAewAAAHwAAAB9AAAAfgAAAH8AAACAAAAA
+gQAAAIIAAACDAAAAhAAAAIUAAACGAAAAhwAAAIgAAACJAAAAigAAAIsAAACMAAAA
+jQAAAI4AAACPAAAAkAAAAJEAAACSAAAAkwAAAJQAAACVAAAAlgAAAJcAAACYAAAA
+mQAAAJoAAACbAAAAnAAAAJ0AAACeAAAAnwAAAKAAAAChAAAAogAAAKMAAACkAAAA
+pQAAAKYAAACnAAAAqAAAAKkAAACqAAAAqwAAAKwAAACtAAAArgAAAK8AAACwAAAA
+sQAAALIAAACzAAAAtAAAALUAAAC2AAAAtwAAALgAAAC5AAAAugAAALsAAAC8AAAA
+vQAAAL4AAAC/AAAAwAAAAMEAAADCAAAAwwAAAMQAAADFAAAAxgAAAMcAAADIAAAA
+yQAAAMoAAADLAAAAzAAAAM0AAADOAAAAzwAAANAAAADRAAAA0gAAANMAAADUAAAA
+1QAAANYAAADXAAAA2AAAANkAAADaAAAA2wAAANwAAADdAAAA3gAAAN8AAADgAAAA
+4QAAAOIAAADjAAAA5AAAAOUAAADmAAAA5wAAAOgAAADpAAAA6gAAAOsAAADsAAAA
+7QAAAO4AAADvAAAA8AAAAPEAAADyAAAA8wAAAPQAAAD1AAAA9gAAAPcAAAD4AAAA
++QAAAPoAAAD7AAAA/AAAAP0AAAD+AAAA/wAAAAABAAABAQAAAgEAAAMBAAAEAQAA
+BQEAAAYBAAAHAQAACAEAAAkBAAAKAQAACwEAAAwBAAANAQAADgEAAA8BAAAQAQAA
+EQEAABIBAAATAQAAFAEAABUBAAAWAQAAFwEAABgBAAAZAQAAGgEAABsBAAAcAQAA
+HQEAAB4BAAAfAQAAIAEAACEBAAAiAQAAIwEAACQBAAAlAQAAJgEAACcBAAAoAQAA
+KQEAACoBAAArAQAALAEAAC0BAAAuAQAALwEAADABAAAxAQAAMgEAADMBAAA0AQAA
+NQEAADYBAAA3AQAAOAEAADkBAAA6AQAAOwEAADwBAAA9AQAAPgEAAD8BAABAAQAA
+QQEAAEIBAABDAQAARAEAAEUBAABGAQAARwEAAEgBAABJAQAASgEAAEsBAABMAQAA
+TQEAAE4BAABPAQAAUAEAAFEBAABSAQAAUwEAAFQBAABVAQAAVgEAAFcBAABYAQAA
+WQEAAFoBAABbAQAAXAEAAF0BAABeAQAAXwEAAGABAABhAQAAYgEAAGMBAABkAQAA
+ZQEAAGYBAABnAQAAaAEAAGkBAABqAQAAawEAAGwBAABtAQAAbgEAAG8BAABwAQAA
+cQEAAHIBAABzAQAAdAEAAHUBAAB2AQAAdwEAAHgBAAB5AQAAegEAAHsBAAB8AQAA
+fQEAAH4BAAB/AQAAgAEAAIEBAACCAQAAgwEAAIQBAACFAQAAhgEAAIcBAACIAQAA
+iQEAAIoBAACLAQAAjAEAAI0BAACOAQAAjwEAAJABAACRAQAAkgEAAJMBAACUAQAA
+lQEAAJYBAACXAQAAmAEAAJkBAACaAQAAmwEAAJwBAACdAQAAngEAAJ8BAACgAQAA
+oQEAAKIBAACjAQAApAEAAKUBAACmAQAApwEAAKgBAACpAQAAqgEAAKsBAACsAQAA
+rQEAAK4BAACvAQAAsAEAALEBAACyAQAAswEAALQBAAC1AQAAtgEAALcBAAC4AQAA
+uQEAALoBAAC7AQAAvAEAAL0BAAC+AQAAvwEAAMABAADBAQAAwgEAAMMBAADEAQAA
+xQEAAMYBAADHAQAAyAEAAMkBAADKAQAAywEAAMwBAADNAQAAzgEAAM8BAADQAQAA
+0QEAANIBAADTAQAA1AEAANUBAADWAQAA1wEAANgBAADZAQAA2gEAANsBAADcAQAA
+3QEAAN4BAADfAQAA4AEAAOEBAADiAQAA4wEAAOQBAADlAQAA5gEAAOcBAADoAQAA
+6QEAAOoBAADrAQAA7AEAAO0BAADuAQAA7wEAAPABAADxAQAA8gEAAPMBAAD0AQAA
+9QEAAPYBAAD3AQAA+AEAAPkBAAD6AQAA+wEAAPwBAAD9AQAA/gEAAP8BAAAAAgAA
+AQIAAAICAAADAgAABAIAAAUCAAAGAgAABwIAAAgCAAAJAgAACgIAAAsCAAAMAgAA
+DQIAAA4CAAAPAgAAEAIAABECAAASAgAAEwIAABQCAAAVAgAAFgIAABcCAAAYAgAA
+GQIAABoCAAAbAgAAHAIAAB0CAAAeAgAAHwIAACACAAAhAgAAIgIAACMCAAAkAgAA
+JQIAACYCAAAnAgAAKAIAACkCAAAqAgAAKwIAACwCAAAtAgAALgIAAC8CAAAwAgAA
+MQIAADICAAAzAgAANAIAADUCAAA2AgAANwIAADgCAAA5AgAAOgIAADsCAAA8AgAA
+PQIAAD4CAAA/AgAAQAIAAEECAABCAgAAQwIAAEQCAABFAgAARgIAAEcCAABIAgAA
+SQIAAEoCAABLAgAATAIAAE0CAABOAgAATwIAAFACAABRAgAAUgIAAFMCAABUAgAA
+VQIAAFYCAABXAgAAWAIAAFkCAABaAgAAWwIAAFwCAABdAgAAXgIAAF8CAABgAgAA
+YQIAAGICAABjAgAAZAIAAGUCAABmAgAAZwIAAGgCAABpAgAAagIAAGsCAABsAgAA
+bQIAAG4CAABvAgAAcAIAAHECAAByAgAAcwIAAHQCAAB1AgAAdgIAAHcCAAB4AgAA
+eQIAAHoCAAB7AgAAfAIAAH0CAAB+AgAAfwIAAIACAACBAgAAggIAAIMCAACEAgAA
+hQIAAIYCAACHAgAAiAIAAIkCAACKAgAAiwIAAIwCAACNAgAAjgIAAI8CAACQAgAA
+kQIAAJICAACTAgAAlAIAAJUCAACWAgAAlwIAAJgCAACZAgAAmgIAAJsCAACcAgAA
+nQIAAJ4CAACfAgAAoAIAAKECAACiAgAAowIAAKQCAAClAgAApgIAAKcCAACoAgAA
+qQIAAKoCAACrAgAArAIAAK0CAACuAgAArwIAALACAACxAgAAsgIAALMCAAC0AgAA
+tQIAALYCAAC3AgAAuAIAALkCAAC6AgAAuwIAALwCAAC9AgAAvgIAAL8CAADAAgAA
+wQIAAMICAADDAgAAxAIAAMUCAADGAgAAxwIAAMgCAADJAgAAygIAAMsCAADMAgAA
+zQIAAM4CAADPAgAA0AIAANECAADSAgAA0wIAANQCAADVAgAA1gIAANcCAADYAgAA
+2QIAANoCAADbAgAA3AIAAN0CAADeAgAA3wIAAOACAADhAgAA4gIAAOMCAADkAgAA
+5QIAAOYCAADnAgAA6AIAAOkCAADqAgAA6wIAAOwCAADtAgAA7gIAAO8CAADwAgAA
+8QIAAPICAADzAgAA9AIAAPUCAAD2AgAA9wIAAPgCAAD5AgAA+gIAAPsCAAD8AgAA
+/QIAAP4CAAD/AgAAAAMAAAEDAAACAwAAAwMAAAQDAAAFAwAABgMAAAcDAAAIAwAA
+CQMAAAoDAAALAwAADAMAAA0DAAAOAwAADwMAABADAAARAwAAEgMAABMDAAAUAwAA
+FQMAABYDAAAXAwAAGAMAABkDAAAaAwAAGwMAABwDAAAdAwAAHgMAAB8DAAAgAwAA
+IQMAACIDAAAjAwAAJAMAACUDAAAmAwAAJwMAACgDAAApAwAAKgMAACsDAAAsAwAA
+LQMAAC4DAAAvAwAAMAMAADEDAAAyAwAAMwMAADQDAAA1AwAANgMAADcDAAA4AwAA
+OQMAADoDAAA7AwAAPAMAAD0DAAA+AwAAPwMAAEADAABBAwAAQgMAAEMDAABEAwAA
+RQMAAEYDAABHAwAASAMAAEkDAABKAwAASwMAAEwDAABNAwAATgMAAE8DAABQAwAA
+UQMAAFIDAABTAwAAVAMAAFUDAABWAwAAVwMAAFgDAABZAwAAWgMAAFsDAABcAwAA
+XQMAAF4DAABfAwAAYAMAAGEDAABiAwAAYwMAAGQDAABlAwAAZgMAAGcDAABoAwAA
+aQMAAGoDAABrAwAAbAMAAG0DAABuAwAAbwMAAHADAABxAwAAcgMAAHMDAAB0AwAA
+dQMAAHYDAAB3AwAAeAMAAHkDAAB6AwAAewMAAHwDAAB9AwAAfgMAAH8DAACAAwAA
+gQMAAIIDAACDAwAAhAMAAIUDAACGAwAAhwMAAIgDAACJAwAAigMAAIsDAACMAwAA
+jQMAAI4DAACPAwAAkAMAAJEDAACSAwAAkwMAAJQDAACVAwAAlgMAAJcDAACYAwAA
+mQMAAJoDAACbAwAAnAMAAJ0DAACeAwAAnwMAAKADAAChAwAAogMAAKMDAACkAwAA
+pQMAAKYDAACnAwAAqAMAAKkDAACqAwAAqwMAAKwDAACtAwAArgMAAK8DAACwAwAA
+sQMAALIDAACzAwAAtAMAALUDAAC2AwAAtwMAALgDAAC5AwAAugMAALsDAAC8AwAA
+vQMAAL4DAAC/AwAAwAMAAMEDAADCAwAAwwMAAMQDAADFAwAAxgMAAMcDAADIAwAA
+yQMAAMoDAADLAwAAzAMAAM0DAADOAwAAzwMAANADAADRAwAA0gMAANMDAADUAwAA
+1QMAANYDAADXAwAA2AMAANkDAADaAwAA2wMAANwDAADdAwAA3gMAAN8DAADgAwAA
+4QMAAOIDAADjAwAA5AMAAOUDAADmAwAA5wMAAOgDAADpAwAA6gMAAOsDAADsAwAA
+7QMAAO4DAADvAwAA8AMAAPEDAADyAwAA8wMAAPQDAAD1AwAA9gMAAPcDAAD4AwAA
++QMAAPoDAAD7AwAA/AMAAP0DAAD+AwAA/wMAAAEEAABSAG8AbwB0ACAARQBuAHQA
+cgB5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+FgAFAP//////////BgAAAIQQDAAAAAAAwAAAAAAAAEYAAAAAAAAAAAAAAABwIVps
+AnPRAborAAAA0AAAAAAAABxB8EQvQr5BZEEAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAIBGwAAACsAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAPvDugIAAAAA
+BQBTAHUAbQBtAGEAcgB5AEkAbgBmAG8AcgBtAGEAdABpAG8AbgAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACgAAgD///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAADQAAAAAAIAAAAAAABASP8/5EPsQeRFrEQxSAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+EAACAS0AAAAcAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAALsrAAA4FgAAAAAAAEBIikE3Q3JEHUL7RQAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAAIBKQAAACwAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAKQBAAAAAAAA
+QEjKQTBDsTs7QiZGN0IcQjRGaEQmQgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAABgAAgEKAAAADAAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAPAAAAMAAAAAAAAABASMpBMEOxPxI/KEU4QrFB
+KEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+FAACARQAAAAFAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAABAAAAAqAAAAAAAAAEBIykH5Rc5GqEH4RSg/KEU4QrFBKEgAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAIB/////zUAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEQAAADAAAAAAAAAA
+QEjKRDM/KEG1QStIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAA4AAgH///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAASAAAANAAAAAAAAABASAtDMUE1RwAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+CgACAf///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAADsAAAAgAAAAAAAAAAtDMUE1R748sUU3Qe9DEULvRGhFAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWAAIBFQAAABEAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAvisAAAAeAgAAAAAA
+C0MxQTVHPkDsRow6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAA4AAgH///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAADgKwAAAG4CAAAAAAALQzFBNUc+QOxGnjz/OvBE
+/zpkRDFCNUgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+GgACAQcAAAAOAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAgsAACITwEAAAAAAAtDMUE1Rz5A7EaePP868ER/OyxBr0QqSAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaAAIB////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHSwAAJhjCQAAAAAA
+C0MxQTVHPkDsRp48vzymRL87u0EvQTBIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAABoAAgANAAAAkgAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAC0LAAA5k0AAAAAAAALQzFBNUc+QOxGnjy/PKZE
+vzxxQjJIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+GAACAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAALksAADmTQAAAAAAAAtDMUE1Rz5A7EaePP868ET/PahGAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWAAIB////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAviwAANZXAAAAAAAA
+C0MxQTVHPkDsRp48/zrwRL8/M0gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAABYAAgAQAAAAIwAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAATAAAA9gkAAAAAAABASMxCqEHuOvJGAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+DAACAf////8xAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAADwAAAAMAAAAAAAAAEBIjETwRHJEaEQ3SAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAAIBGgAAACEAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPQAAACABAAAAAAAA
+QEiMRPFFtUQvSAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAwAAgAZAAAAAwAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAADELAAAZBkAAAAAAABASIxE8UW1RC87ckQnQzdD
+ckQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+FAACAf////8lAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAEIAAAAoAQAAAAAAAEBIjETxRbVErzs5QvFFAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAIBKgAAAP//////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARwAAAEwFAAAAAAAA
+QEhMRShBN0KPRO9BaEUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAABAAAgAWAAAAGAAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAABdAAAAFAAAAAAAAABASAxG9kUyRIpBN0NyRAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+EAACASgAAAD//////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAF4AAAAEAgAAAAAAAEBIDUPkQ7JCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAIBOgAAAAEAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZwAAACYCAAAAAAAA
+QEgNQzVC5kVyRTxIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAA4AAgEyAAAA//////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAABwAAAAeAAAAAAAAABASE5FtUQ1SAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+CgACAf///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAHIAAACMAgAAAAAAAEBITkZoRLc95EQzQ7FCAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAIBFwAAAC8AAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAfQAAACgAAAAAAAAA
+QEgPQuRFeEUoSAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAwAAgD///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAB+AAAAQAAAAAAAAABASA9C5EV4RSg7MkSzRDFC
+8UU2SAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+FgACAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAH8AAABgAAAAAAAAAEBID0MvQgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAIBIAAAAE4AAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgQAAAMgAAAAAAAAA
+AQAAAAIAAAADAAAABAAAAAUAAAAGAAAABwAAALgAAAAJAAAACgAAAAsAAAAMAAAA
+DQAAAA4AAAD+/////v////7////+/////v///xQAAAAVAAAAFgAAABcAAAAYAAAA
+GQAAABoAAAAbAAAAHAAAAB0AAAAeAAAAHwAAACAAAAAhAAAAIgAAACMAAAAkAAAA
+JQAAACYAAAAnAAAAKAAAACkAAAAqAAAAKwAAACwAAAAtAAAALgAAAC8AAAAwAAAA
+MQAAADIAAAAzAAAANAAAADUAAAA2AAAANwAAADgAAAA5AAAAOgAAAP7////+////
+/v///z4AAAA/AAAAQAAAAEEAAAD+////QwAAAEQAAABFAAAARgAAAP7///9IAAAA
+SQAAAEoAAABLAAAATAAAAE0AAABOAAAATwAAAFAAAABRAAAAUgAAAFMAAABUAAAA
+VQAAAFYAAABXAAAAWAAAAFkAAABaAAAAWwAAAFwAAAD+/////v///18AAABgAAAA
+YQAAAGIAAABjAAAAZAAAAGUAAABmAAAA/v///2gAAABpAAAAagAAAGsAAABsAAAA
+bQAAAG4AAABvAAAA/v///3EAAAD+////cwAAAHQAAAB1AAAAdgAAAHcAAAB4AAAA
+eQAAAHoAAAB7AAAAfAAAAP7////+/////v///4AAAAD+////ggAAAIMAAACEAAAA
+/v////7///+HAAAAiAAAAIkAAACKAAAAiwAAAIwAAAD+////jgAAAI8AAACQAAAA
+/v////7////+/////v////7///+WAAAA/v///5gAAACZAAAA/v////7///+cAAAA
+nQAAAJ4AAACfAAAAoAAAAKEAAACiAAAAowAAAKQAAAClAAAApgAAAKcAAAD+////
+qQAAAKoAAAD+/////v////7////+/////v///7AAAAD+/////v///7MAAAC0AAAA
+tQAAAP7////+/////v///7kAAAC6AAAAuwAAALwAAAC9AAAAvgAAAL8AAADAAAAA
+wQAAAMIAAADDAAAAxAAAAMUAAADGAAAAxwAAAMgAAADJAAAAygAAAMsAAADMAAAA
+zQAAAP7////PAAAA/v///9EAAADSAAAA0wAAANQAAADVAAAA1gAAANcAAAD+////
+2QAAANoAAADbAAAA3AAAAN0AAADeAAAA3wAAAOAAAADhAAAA/v////7////+////
+/v////7////nAAAA6AAAAOkAAADqAAAA6wAAAOwAAADtAAAA7gAAAO8AAADwAAAA
+8QAAAPIAAADzAAAA9AAAAPUAAAD2AAAA9wAAAPgAAAD5AAAA+gAAAPsAAAD8AAAA
+/QAAAP4AAAD/AAAAAAEAAAEBAAACAQAAAwEAAAQBAAAFAQAABgEAAAcBAAAIAQAA
+CQEAAAoBAAALAQAADAEAAA0BAAAOAQAADwEAAP7////+////EgEAABMBAAAUAQAA
+FQEAABYBAAAXAQAAGAEAABkBAAD+/////v///xwBAAAdAQAAHgEAAB8BAAAgAQAA
+IQEAACIBAAAjAQAAJAEAACUBAAAmAQAAJwEAACgBAAApAQAAKgEAACsBAAAsAQAA
+LQEAAC4BAAAvAQAAMAEAADEBAAD+/////v////7////+////NgEAADcBAAA4AQAA
+OQEAADoBAAA7AQAAPAEAAD0BAAA+AQAAPwEAAEABAABBAQAAQgEAAEMBAAD+////
+RQEAAEYBAAD+/////v///0kBAABKAQAASwEAAEwBAAD+////TgEAAE8BAABQAQAA
+UQEAAFIBAABTAQAAVAEAAFUBAABWAQAA/v////7////+/////v////7///9cAQAA
+XQEAAF4BAABfAQAAYAEAAGEBAABiAQAAYwEAAGQBAABlAQAAZgEAAGcBAABoAQAA
+aQEAAGoBAABrAQAAbAEAAG0BAABuAQAAbwEAAHABAABxAQAAcgEAAHMBAAB0AQAA
+dQEAAHYBAAB3AQAAeAEAAHkBAAB6AQAAewEAAHwBAAB9AQAAfgEAAH8BAACAAQAA
+gQEAAIIBAACDAQAA/v///4UBAACGAQAAhwEAAIgBAACJAQAAigEAAIsBAACMAQAA
+/v////7///+PAQAAkAEAAJEBAACSAQAAkwEAAJQBAACVAQAAlgEAAJcBAACYAQAA
+mQEAAJoBAACbAQAAnAEAAJ0BAACeAQAAnwEAAKABAAChAQAAogEAAKMBAAD+////
+/v///6YBAACnAQAAqAEAAKkBAACqAQAAqwEAAKwBAACtAQAArgEAAK8BAACwAQAA
+sQEAALIBAACzAQAA/v///7UBAAC2AQAA/v////7///+5AQAAugEAALsBAAC8AQAA
+/v///74BAAC/AQAAwAEAAMEBAADCAQAAwwEAAMQBAADFAQAAxgEAAP7////+////
+/v////7////+////zAEAAM0BAADOAQAAzwEAANABAADRAQAA0gEAANMBAADUAQAA
+1QEAANYBAADXAQAA2AEAANkBAADaAQAA2wEAANwBAADdAQAA3gEAAN8BAADgAQAA
+4QEAAOIBAADjAQAA5AEAAOUBAADmAQAA5wEAAOgBAADpAQAA6gEAAOsBAADsAQAA
+7QEAAO4BAADvAQAA8AEAAPEBAADyAQAA8wEAAP7////1AQAA9gEAAPcBAAD4AQAA
++QEAAPoBAAD7AQAA/AEAAP7////+/////wEAAAACAAABAgAAAgIAAAMCAAAEAgAA
+BQIAAAYCAAAHAgAACAIAAAkCAAAKAgAACwIAAAwCAAANAgAADgIAAA8CAAAQAgAA
+EQIAABICAAATAgAA/v////7///8WAgAAFwIAABgCAAAZAgAAGgIAABsCAAAcAgAA
+HQIAAB4CAAAfAgAAIAIAACECAAAiAgAAIwIAAP7///8lAgAAJgIAAP7////+////
+KQIAACoCAAArAgAALAIAAP7///8uAgAALwIAADACAAAxAgAAMgIAADMCAAA0AgAA
+NQIAADYCAAD+/////v////7////+/////v///zwCAAA9AgAAPgIAAD8CAABAAgAA
+QQIAAEICAABDAgAARAIAAEUCAABGAgAARwIAAEgCAABJAgAASgIAAEsCAABMAgAA
+TQIAAE4CAABPAgAAUAIAAFECAABSAgAAUwIAAFQCAABVAgAAVgIAAFcCAABYAgAA
+WQIAAFoCAABbAgAAXAIAAF0CAABeAgAAXwIAAGACAABhAgAAYgIAAGMCAAD+////
+ZQIAAGYCAABnAgAAaAIAAGkCAABqAgAAawIAAGwCAAD+/////v///28CAABwAgAA
+cQIAAHICAABzAgAAdAIAAHUCAAB2AgAAdwIAAHgCAAB5AgAAegIAAHsCAAB8AgAA
+fQIAAH4CAAB/AgAAgAIAAIECAACCAgAAgwIAAP7////+////hgIAAIcCAACIAgAA
+iQIAAIoCAACLAgAAjAIAAI0CAACOAgAAjwIAAJACAACRAgAAkgIAAJMCAAD+////
+lQIAAJYCAAD+/////v///5kCAACaAgAAmwIAAJwCAAD+////ngIAAJ8CAACgAgAA
+oQIAAKICAACjAgAApAIAAKUCAACmAgAA/v////7////+/////v////7///+sAgAA
+rQIAAK4CAACvAgAAsAIAALECAACyAgAAswIAALQCAAC1AgAAtgIAALcCAAC4AgAA
+uQIAALoCAAC7AgAAvAIAAL0CAAC+AgAAvwIAAMACAADBAgAAwgIAAMMCAADEAgAA
+xQIAAMYCAADHAgAAyAIAAMkCAADKAgAAywIAAMwCAADNAgAAzgIAAM8CAADQAgAA
+0QIAANICAADTAgAA1AIAAP7////WAgAA1wIAANgCAADZAgAA2gIAANsCAADcAgAA
+3QIAAP7////+////4AIAAOECAADiAgAA4wIAAOQCAADlAgAA5gIAAOcCAADoAgAA
+6QIAAOoCAADrAgAA7AIAAO0CAADuAgAA7wIAAPACAADxAgAA8gIAAPMCAAD0AgAA
+9QIAAP7////+////+AIAAPkCAAD6AgAA+wIAAPwCAAD9AgAA/gIAAP8CAAAAAwAA
+AQMAAAIDAAADAwAABAMAAAUDAAD+////BwMAAAgDAAD+/////v///wsDAAAMAwAA
+DQMAAA4DAAD+////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+/////////////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAASgXMBEUExQNvAwYDoQLGAQcBhwA+ADwAHgACAAwAVQBuAGMAbwBt
+AHAAcgBlAHMAcwBlAGQAAAAMAE0AUwBDAG8AbQBwAHIAZQBzAHMAZQBkAAAAewA3
+AEYAQwAyADgAOQA0ADAALQA5AEQAMwAxAC0AMQAxAEQAMACvwAYAAAAAAAYAAABM
+WlhDAgAAAAIAAAACAAAAAQAAAAAAAAACAAAADgAAAAgAAAAoAAAAr8AGAAAAAACu
+2wUAAAAAAACAAAAAAAAAAAAAAAAAAACISwAAAAAAAEBeAAAAAAAAyrkAAAAAAADq
+NAEAAAAAAPq0AQAAAAAA+jQCAAAAAAAKtQIAAAAAAAo1AwAAAAAAGrUDAAAAAAAa
+NQQAAAAAAGy1BAAAAAAAijMFAAAAAADAsQUAAAAAAAMAAAAKAAQAVSjJTQkAFgBI
+SEEgVmVyc2lvbiA0Ljc0Ljg3MDIABAAkABkEAAAAAAAAAAAAAAAAAAAAAAAAiO8P
+igkPzAEAAAAAAAAAAAIADQBhZGxheWVycy5odG0ABgAJAGFkbGF5ZXJzAAUABQBt
+YWluAAwABAAAAAAADQAAEFQjU037/Zk/AQAAAAUAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABNU0NGAAAAAPvDugIAAAAA
+LAAAAAAAAAADAQEACgAAAAAAAABNAQAAdgUAACH+BQAAAAAAAAClRGNAIABER2lz
+TGF5ZXJDaG0A6DwcACH+BQAAAF1IOaggAERHaXNMYXllckRsbADcEAAACTsiAAAA
+pURiQCAAREdpc0xheWVyU2duAB+ZAQDlSyIAAAClRGNAIABER2lzTm90ZXNDaG0A
+6MQTAATlIwAAAF1IOaggAERHaXNOb3Rlc0RsbAD5DQAA7Kk3AAAAXUg3qCAAREdp
+c05vdGVzU2duAOgqAwLltzcAAABdSDmoIABncnltRVhFAGEMAADN4joCAABdSDqo
+IABNYXJrSW5zdGFsbADoFkYALu86AgAAXUg4qCAAVHJheU5vdGlmeUVYRQDokDkA
+FgaBAgAAXUg4qCAAVXBkYXRlU2VydmljZUVYRQCISY6EAIAAgElUU0YDAAAAYAAA
+AAEAAABWTxGKGQQAABD9AXyqe9ARngwAoMki5uwR/QF8qnvQEZ4MAKDJIubsYAAA
+AAAAAAAYAAAAAAAAAHgAAAAAAAAAVBAAAAAAAADMEAAAAAAAAP4BAAAAAAAAIf4F
+AAAAAAAAAAAAAAAAAElUU1ABAAAAVAAAAAoAAAAAEAAAAgAAAAEAAAD/////AAAA
+AAAAAAD/////AQAAAAkEAABqkgJdLiHQEZ35AKDJIubsVAAAAP//////////////
+/1BNR0xACQAAAAAAAP//////////AS8AAAAILyNJRFhIRFIBmt4PoAAILyNJVEJJ
+VFMAAAAJLyNTVFJJTkdTAZuABYEqCC8jU1lTVEVNAIIeoQkILyNUT1BJQ1MBmv4P
+UAgvI1VSTFNUUgGa/xtqCC8jVVJMVEJMAZr+XzwJLyNXSU5ET1dTAZrFXIMQCy8k
+RklmdGlNYWluAQAACS8kT0JKSU5TVAGayHSVGxUvJFdXQXNzb2NpYXRpdmVMaW5r
+cy8AAAAdLyRXV0Fzc29jaWF0aXZlTGlua3MvUHJvcGVydHkBmshwBBEvJFdXS2V5
+d29yZExpbmtzLwAAABkvJFdXS2V5d29yZExpbmtzL1Byb3BlcnR5AZrIbAQML19f
+ZHVtbXkuZ2lmAaF8KyUvYWRkaXRpb25hbF9tb2R1bGVzX2xheWVyc19hZGRyZXMu
+cG5nAYTdJvkLJS9hZGRpdGlvbmFsX21vZHVsZXNfbGF5ZXJzX2JvbG90by5wbmcB
+hdYxzGwlL2FkZGl0aW9uYWxfbW9kdWxlc19sYXllcnNfZmlsdGVyLnBuZwGGox3i
+ZCMvYWRkaXRpb25hbF9tb2R1bGVzX2xheWVyc19saW5lLnBuZwGHhgHPKiUvYWRk
+aXRpb25hbF9tb2R1bGVzX2xheWVyc19wZXJpb2QucG5nAYfVK6lNIi9hZGRpdGlv
+bmFsX21vZHVsZXNfbGF5ZXJzX3BpYy5wbmcBh/54jpoKJC9hZGRpdGlvbmFsX21v
+ZHVsZXNfbGF5ZXJzX3BvaW50LnBuZwGWmQL7MyYvYWRkaXRpb25hbF9tb2R1bGVz
+X2xheWVyc19wb2x5Z29uLnBuZwGXlDWBtwQlL2FkZGl0aW9uYWxfbW9kdWxlc19s
+YXllcnNfdXBkYXRlLnBuZwGYyznPfw0vYWRsYXllcnMuaHRtAYGsSOpOCy9hdXRo
+b3IuaHRtAYKXFphuDy9hdXRob3JfcnVzLmh0bQGCsASjFQ4vYXV0aG9yX3VrLmh0
+bQGC0xmjKgwvY29udGFjdC5odG0BgvZDgeZjDi9mb290ZXJfYmcuZ2lmAaInhGsS
+L2Zvb3Rlcl9idXR0b24uZ2lmAacSmSgVL2Zvb3Rlcl9zZXBhcmF0b3IuZ2lmAcA6
+hlQLL2dyZWVuYi5naWYBxw4yDC9ncmVlbmJsLmdpZgHHQIElDC9ncmVlbmJyLmdp
+ZgHIZYElCy9ncmVlbmwuZ2lmAcoKMgwvZ3JlZW5sdC5naWYByjyBJQsvZ3JlZW5y
+LmdpZgHLYTIML2dyZWVucnQuZ2lmAcwTgSULL2dyZWVudC5naWYBzTgyDi9oZWFk
+ZXJfYmcuZ2lmAc1qiHsQL2hlYWRlcl9sb2dvLmdpZgHWZbV+Gi9ob21lX2hlYWRl
+cl9ib3R0b21fYmcuZ2lmAYGMY4oZHy9ob21lX2hlYWRlcl9tZW51X3NlcGFyYXRv
+ci5naWYBgZZ8QyMvaG9tZV9oZWFkZXJfbW9uaXRvcl9yZWZsZWN0aW9uLmdpZgGB
+lz+ObRcvaG9tZV9oZWFkZXJfdG9wX2JnLmdpZgGBpiyEewkvbGlzdC5naWYBgasn
+SAkvbG9nby5wbmcBmZs4rUwaL21lbnVfc2Vjb25kX3NlbGVjdF9iZy5naWYBgatv
+LhAvb3Blbl9sYXllcnMucG5nAZnJBPxYCy9zcGFjZXIuZ2lmAYGsHSsKL3N0eWxl
+LmNzcwEAoXwUOjpEYXRhU3BhY2UvTmFtZUxpc3QAADwoOjpEYXRhU3BhY2UvU3Rv
+cmFnZS9NU0NvbXByZXNzZWQvQ29udGVudACjJ5e3Liw6OkRhdGFTcGFjZS9TdG9y
+YWdlL01TQ29tcHJlc3NlZC9Db250cm9sRGF0YQBqHCk6OkRhdGFTcGFjZS9TdG9y
+YWdlL01TQ29tcHJlc3NlZC9TcGFuSW5mbwBiCC86OkRhdGFTcGFjZS9TdG9yYWdl
+L01TQ29tcHJlc3NlZC9UcmFuc2Zvcm0vTGlzdAA8Jl86OkRhdGFTcGFjZS9TdG9y
+YWdlL01TQ29tcHJlc3NlZC9UcmFuc2Zvcm0vezdGQzI4OTQwLTlEMzEtMTFEMC05
+QjI3LTAwQTBDOTFFOUM3Q30vSW5zdGFuY2VEYXRhLwAAAGk6OkRhdGFTcGFjZS9T
+dG9yYWdlL01TQ29tcHJlc3NlZC9UcmFuc2Zvcm0vezdGQzI4OTQwLTlEMzEtMTFE
+MC05QjI3LTAwQTBDOTFFOUM3Q30vSW5zdGFuY2VEYXRhL1Jlc2V0VGFibGUAgQaB
+GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+DwAEAAAAAAAFEFBlAABSAFAUAAA/ADv37Y6QEIiDpIwRFlFXIQnUS4XaKqVRSdi1
+ilJIKliVUlEWnOyIaC1hLdbaorUgrjNyRUQAYAgA+4BZMix7fra33UbwgcZ/eKyA
+dhFDFVBFhfVaRzyLltRA9OqZLVlF9nkmHMms33dpwerWt8qkLGU6zLQQIeZCp5qG
+MKsF6linHiYKhKBggJgIIuM/AAgAAAgBwNAVAbd+2W5ld2zGQIDAGSQgAmAmPGcK
+nFl4pnEm44vvC+T63lkyLVjnvNbh9TIccKKb69vNeR5jcb50y8ARoVit3l6WbqIz
+yzVLbBtL6L9W+vUlD4ZPfWBbLeCvsf/tbyIVvs++lm+ux/4kNnDG++MNohcIs4kf
+EPn1R77n4V/8LQUlhQnyk4Hhdn+et1teXV9jXqDQIKJhHi/e559DfbY/yjLfxrpy
+vMXu37PWod/fNP/E8Z9i0sVdHRb+QJi7Flawlu/Kr+RazynGI75AM/iRP2vN8QEP
+3n9bY/xaRNS6jTJCLyqQ3oDP8kD+DG9ov+SGori4sor/QV7A/EPxq+ikEyRz/aAb
+00whISSiY/5/8JFPOAu1gi/9VoygT7dG/dBYJdcW/hhlPB9ELf3V/pj645+xjBbx
+kH9df35C4CdhfNEn/Ff0n3Zff/tevfhCxNq9X8n6cJEH4rLHX+gxGez6d7P/izBH
+QX3/w0bUzYPbP/pD3zDbF1H/9b+Pv1D/W38X0kiaO1+t6dc980ILUWTNNB+ivh//
+MR0hId0j9tcgRJ3+zv9M/XxQQdLf+WNNuu6aDz6IPGz+7Edtq1DNrf/nt9sH7H/+
+gff+iAZ7PyozVCkbswHky10CUf32uwMm3+Mnyg/kPVwkzWv+DlPp839tQnjmvY/9
+FuD+lvc1gP18IVGOD/yJ+1h9/wNfcW+kie4g+gftft79Pf+GX6U/+f7fUHs/vhDo
+fFH41vTZNy/gjef9vey6+ITu+4mVWQj74L/9f4vpjv/6zf2Ov0Txa+jXhIuxb3c/
+8/ml+nD/iY//Ce8UAPb2H/P4ZP8YeaRI7v2flAIr/Q+1erK93vKtI22xA76/d2/b
+/ZbECMew741qlWS//tcPjm77pJmjn2Irfvj/JtuM8/6q+VpfIt5/MaF3qRj4Yfyd
+7IJ4bN9nflc1K2r0Fl//KmezPpmlLyI+/juxykaaym+0xP8hiGYunyRCkk+VoqaP
+2yiHfP4CH5lwkm1ECgLID/+zg9FN/8m/PIX7//71RBzbftX78djASPK3y+dP/DiX
++tc/5H/1JU79+iLzO32y2vXFtsh22/4tZ+nN6f8IkRz8RZdPX2ifyqzP/4CbGT/G
+N6+digy5XQjf9Gr/N3dK+e1v5Tt3gyjuoiFp/7q/jHfnre79Xr8+tInX9W/9Jf89
+tGq9cfv05K/3BeDZ/XODFdjfs/+nHz6kBE7Rf/66jO5rvjf+QYMm9hk/tfE/iLda
+mbre/EgbQD8f0M78ODLfj3oCgAvCfo/F8d9+Xhfm/BB27LZ/GbZE9Xvx+L76WvbX
+/0f/APTMz0UJ9WU/UgpwP+qf9rnfhXT/rf+q791mGYO+//819HAx0Hyv4f9/bBis
+Lvk55PP685+pb/wlgxDWB/A3Z7yDYkP4EKIsMJdvP7eq+Kl+DzpfzlzHm4gv//pX
+/8D402/VWeDp/xoLzsIPMzj34MQjfYgv+mF0aOVvZn3fSO9X0KK1xtDx359+G0+2
+Rd9YfbC+0X+tKpAfbS13b+/1MX5DX5v//4v/EsfVRfn0WxZS0Hzms4PJDM1pHvky
+xaSG5jYvjEsfS3T4VL5/BJ3oPf5N+4pIU8wsrvyt16AM+8trH2wktWL/7CPE0PNH
+Mv/ov+Qvt98jcPn+6p/7F/z/vwf+zborKm6hdh01YCEeGKBwHQBg+DkPCMAPCJ2d
+h/4Z/sD3ddokYGb+gbM+Wt6SymvLO/LjXuFcYVzT4U4weat//vfaO4lby7pRujO3
+N9U0VD73vCt7G9oLP5xHOTNvfWVjWVV7dVHkQfny7ys9Ksfzg3WBbb7j4fKrmjuq
+6jvr2nvK6ku76ky6JR+t/YX9/bX3je6OAt8Qz68ornD+sEUFB83bvmHcf8/Enzya
+dz9R6MW4fsNHT1Pe/FeeFW4orvCFPo8/+BevpA/FMcJzu2mBqo2cuzccboYLPnYW
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: msg
+
+0M8R4KGxGuEAAAAAAAAAAAAAAAAAAAAAIQADAP7/CQAGAAAAAAAAAAAAAAABAAAA
+GAAAAAAAAAAAEAAAFgAAAAIAAAD+////AAAAACYAAAD/////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+//////////////////////////////////////////8IIAYAAAAAAMAAAAAAAABG
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+DoUAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAA6FAAAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABJAFAATQAuAE4AbwB0AGUA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+UgBlADoAIABbAHIAcwBwAGEAbQBkAC8AcgBzAHAAYQBtAGQAXQAgAFsAVABlAHMA
+dABdACAATQBBAEcASQBDACAAUgBPAEIATwBUACAAKAAjADMAMAAzADQAKQAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABTTVRQOk5PVElGSUNBVElP
+TlNAR0lUSFVCLkNPTQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+VgBzAGUAdgBvAGwAbwBkACAAUwB0AGEAawBoAG8AdgAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAG4AbwB0AGkAZgBpAGMAYQB0AGkAbwBuAHMAQABnAGkA
+dABoAHUAYgAuAGMAbwBtAAAAAAAAAAAAAAAAAAAAAABSAGUAOgAgAFsAcgBzAHAA
+YQBtAGQALwByAHMAcABhAG0AZABdACAAWwBUAGUAcwB0AF0AIABNAEEARwBJAEMA
+IABSAE8AQgBPAFQAIAAoACMAMwAwADMANAApAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAE0ASQBNAEUALQBWAGUAcgBzAGkAbwBuADoAIAAxAC4A
+MAANAAoARABhAHQAZQA6ACAAVABoAHUALAAgADEAMgAgAFMAZQBwACAAMgAwADEA
+OQAgADEAMwA6ADIAOAA6ADAAMgAgACsAMAAzADAAMAANAAoARgByAG8AbQA6ACAA
+IgBWAHMAZQB2AG8AbABvAGQAIABTAHQAYQBrAGgAbwB2ACIAIAA8AG4AbwB0AGkA
+ZgBpAGMAYQB0AGkAbwBuAHMAQABnAGkAdABoAHUAYgAuAGMAbwBtAD4ADQAKAFMA
+dQBiAGoAZQBjAHQAOgAgAFIAZQA6ACAAWwByAHMAcABhAG0AZAAvAHIAcwBwAGEA
+bQBkAF0AIABbAFQAZQBzAHQAXQAgAE0AQQBHAEkAQwAgAFIATwBCAE8AVAAgACgA
+IwAzADAAMwA0ACkADQAKAFQAaAByAGUAYQBkAC0AVABvAHAAaQBjADoAIABSAGUA
+OgAgAFsAcgBzAHAAYQBtAGQALwByAHMAcABhAG0AZABdACAAWwBUAGUAcwB0AF0A
+IABNAEEARwBJAEMAIABSAE8AQgBPAFQAIAAoACMAMwAwADMANAApAA0ACgBUAG8A
+OgAgAHIAcwBwAGEAbQBkAC8AcgBzAHAAYQBtAGQAIAA8AHIAcwBwAGEAbQBkAEAA
+bgBvAHIAZQBwAGwAeQAuAGcAaQB0AGgAdQBiAC4AYwBvAG0APgANAAoAQwBjADoA
+IABrAG8AcgBnAG8AdABoADEAIAA8AHYAbABhAGQAaQBzAGwAYQB2AC4AcwB0AGEA
+awBoAG8AdgBAAGcAbQBhAGkAbAAuAGMAbwBtAD4ALAAgAEEAdQB0AGgAbwByACAA
+PABhAHUAdABoAG8AcgBAAG4AbwByAGUAcABsAHkALgBnAGkAdABoAHUAYgAuAGMA
+bwBtAD4ADQAKAEMAbwBuAHQAZQBuAHQALQBUAHIAYQBuAHMAZgBlAHIALQBFAG4A
+YwBvAGQAaQBuAGcAOgAgAHEAdQBvAHQAZQBkAC0AcAByAGkAbgB0AGEAYgBsAGUA
+DQAKAEMAbwBuAHQAZQBuAHQALQBUAHkAcABlADoAIAB0AGUAeAB0AC8AaAB0AG0A
+bAA7ACAAYwBoAGEAcgBzAGUAdAA9ACIAdQB0AGYALQA4ACIADQAKAA0ACgAAAAAA
+AAAAAIErH6S+oxAZnW4A3QEPVAIAAAGAbgBvAHQAaQBmAGkAYwBhAHQAaQBvAG4A
+cwBAAGcAaQB0AGgAdQBiAC4AYwBvAG0AAABTAE0AVABQAAAAbgBvAHQAaQBmAGkA
+YwBhAHQAaQBvAG4AcwBAAGcAaQB0AGgAdQBiAC4AYwBvAG0AAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+VgBzAGUAdgBvAGwAbwBkACAAUwB0AGEAawBoAG8AdgAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAFNNVFA6Tk9USUZJQ0FUSU9OU0BHSVRIVUIuQ09NAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABTAE0AVABQAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+bgBvAHQAaQBmAGkAYwBhAHQAaQBvAG4AcwBAAGcAaQB0AGgAdQBiAC4AYwBvAG0A
+AAAAAAAAAAAAAAAAAAAAAGsAbwByAGcAbwB0AGgAMQA7AEEAdQB0AGgAbwByAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAByAHMAcABhAG0AZAAvAHIA
+cwBwAGEAbQBkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+UgBlADoAIABbAHIAcwBwAGEAbQBkAC8AcgBzAHAAYQBtAGQAXQAgAFsAVABlAHMA
+dABdACAATQBBAEcASQBDACAAUgBPAEIATwBUACAAKAAjADMAMAAzADQAKQAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABNAGUAcgBnAGUAZAAgACMA
+MwAwADMANAA8AGgAdAB0AHAAcwA6AC8ALwBnAGkAdABoAHUAYgAuAGMAbwBtAC8A
+cgBzAHAAYQBtAGQALwByAHMAcABhAG0AZAAvAHAAdQBsAGwALwAzADAAMwA0AD4A
+IABpAG4AdABvACAAbQBhAHMAdABlAHIALgANAAoADQAKABQgDQAKAFkAbwB1ACAA
+YQByAGUAIAByAGUAYwBlAGkAdgBpAG4AZwAgAHQAaABpAHMAIABiAGUAYwBhAHUA
+cwBlACAAeQBvAHUAIABhAHUAdABoAG8AcgBlAGQAIAB0AGgAZQAgAHQAaAByAGUA
+YQBkAC4ADQAKAFIAZQBwAGwAeQAgAHQAbwAgAHQAaABpAHMAIABlAG0AYQBpAGwA
+IABkAGkAcgBlAGMAdABsAHkALAAgAHYAaQBlAHcAIABpAHQAIABvAG4AIABHAGkA
+dABIAHUAYgA8AGgAdAB0AHAAcwA6AC8ALwBnAGkAdABoAHUAYgAuAGMAbwBtAC8A
+cgBzAHAAYQBtAGQALwByAHMAcABhAG0AZAAvAHAAdQBsAGwALwAzADAAMwA0AD8A
+ZQBtAGEAaQBsAF8AcwBvAHUAcgBjAGUAPQBuAG8AdABpAGYAaQBjAGEAdABpAG8A
+bgBzACYAZQBtAGEAaQBsAF8AdABvAGsAZQBuAD0AQQBNADcAMwAyAEIAVwBDAEUA
+VQBTAEQAVwBTADIAQgA3ADUASABUADUASwBUAFEASgBJAEsAVABEAEEANQBDAE4A
+RgBTAE0ANABJAFYAWQBSAEQAQgAyAFkAWQAzAFAATgBWAFcAVwBLADMAVABVAEwA
+NQAyAEgAUwA0AEQARgBXAFoARQBYAEcANAAzAFYATQBWAEMAWABNAFoATABPAE8A
+UgBIAEcANgA1AEQASgBNAFoAVQBXAEcAWQBMAFUATgBGAFgAVwA1AEsAVABEAE4A
+NQBXAFcAMgBaAEwATwBPAFIAUABXAFMAWgBHAE8AVABTAFgAQwBKAFEAUQAjAGUA
+dgBlAG4AdAAtADIANgAyADgANgA1ADgAMwA3ADAAPgAsACAAbwByACAAbQB1AHQA
+ZQAgAHQAaABlACAAdABoAHIAZQBhAGQAPABoAHQAdABwAHMAOgAvAC8AZwBpAHQA
+aAB1AGIALgBjAG8AbQAvAG4AbwB0AGkAZgBpAGMAYQB0AGkAbwBuAHMALwB1AG4A
+cwB1AGIAcwBjAHIAaQBiAGUALQBhAHUAdABoAC8AQQBNADcAMwAyAEIAUQBPAEEA
+RQBLADIAVABKAFMAWQBQAEkAUwBIAE0AUQBMAFEASgBJAEsAVABEAEEATgBDAE4A
+RgBTAE0ANABJAFYAWQBSAEQAQgBRAD4ALgANAAoAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAJUKAACVCgAATUVMQQAAAAB7XHJ0ZjFcYW5zaVxhbnNp
+Y3BnMTI1MVxmcm9taHRtbDEgXGZiaWRpcyBcZGVmZjB7XGZvbnR0YmwNCntcZjBc
+ZnN3aXNzIEFyaWFsO319DQp7XGNvbG9ydGJsXHJlZDBcZ3JlZW4wXGJsdWUwO30N
+Clx1YzFccGFyZFxwbGFpblxkZWZ0YWIzNjAgXGYwXGZzMjQNCntcKlxodG1sdGFn
+NjQgPHA+fVxodG1scnRmIHtcaHRtbHJ0ZjAgTWVyZ2VkIA0Ke1wqXGh0bWx0YWc4
+NCA8YSBjbGFzcz0iaXNzdWUtbGluayBqcy1pc3N1ZS1saW5rIiBkYXRhLWVycm9y
+LXRleHQ9IkZhaWxlZCB0byBsb2FkIGlzc3VlIHRpdGxlIiBkYXRhLWlkPSI0OTI0
+NTA4NTQiIGRhdGEtcGVybWlzc2lvbi10ZXh0PSJJc3N1ZSB0aXRsZSBpcyBwcml2
+YXRlIiBkYXRhLXVybD0iaHR0cHM6Ly9naXRodWIuY29tL3JzcGFtZC9yc3BhbWQv
+aXNzdWVzLzMwMzQiIGRhdGEtaG92ZXJjYXJkLXR5cGU9InB1bGxfcmVxdWVzdCIg
+ZGF0YS1ob3ZlcmNhcmQtdXJsPSIvcnNwYW1kL3JzcGFtZC9wdWxsLzMwMzQvaG92
+ZXJjYXJkIiBocmVmPSJodHRwczovL2dpdGh1Yi5jb20vcnNwYW1kL3JzcGFtZC9w
+dWxsLzMwMzQiPn0jMzAzNA0Ke1wqXGh0bWx0YWc5MiA8L2E+fSBpbnRvIG1hc3Rl
+ci5caHRtbHJ0ZlxwYXIgfVxodG1scnRmMCANClxodG1scnRmIFxwYXINClxodG1s
+cnRmMCANCntcKlxodG1sdGFnNzIgPC9wPn0NCntcKlxodG1sdGFnMCBccGFyIH0N
+CntcKlxodG1sdGFnNjQgPHAgc3R5bGU9ImZvbnQtc2l6ZTpzbWFsbDstd2Via2l0
+LXRleHQtc2l6ZS1hZGp1c3Q6bm9uZTtjb2xvcjojNjY2OyI+fVxodG1scnRmIHtc
+aHRtbHJ0ZjAgDQp7XCpcaHRtbHRhZzg0ICZtZGFzaDt9XGh0bWxydGYgXCc5N1xo
+dG1scnRmMCANCntcKlxodG1sdGFnMTE2IDxiciAvPn1caHRtbHJ0ZiBcbGluZQ0K
+XGh0bWxydGYwIFlvdSBhcmUgcmVjZWl2aW5nIHRoaXMgYmVjYXVzZSB5b3UgYXV0
+aG9yZWQgdGhlIHRocmVhZC4NCntcKlxodG1sdGFnMTE2IDxiciAvPn1caHRtbHJ0
+ZiBcbGluZQ0KXGh0bWxydGYwIFJlcGx5IHRvIHRoaXMgZW1haWwgZGlyZWN0bHks
+IA0Ke1wqXGh0bWx0YWc4NCA8YSBocmVmPSJodHRwczovL2dpdGh1Yi5jb20vcnNw
+YW1kL3JzcGFtZC9wdWxsLzMwMzQ/ZW1haWxfc291cmNlPW5vdGlmaWNhdGlvbnMm
+YW1wO2VtYWlsX3Rva2VuPUFNNzMyQldDRVVTRFdTMkI3NUhUNUtUUUpJS1REQTVD
+TkZTTTRJVllSREIyWVkzUE5WV1dLM1RVTDUySFM0REZXWkVYRzQzVk1WQ1hNWkxP
+T1JIRzY1REpNWlVXR1lMVU5GWFc1S1RETjVXVzJaTE9PUlBXU1pHT1RTWENKUVEj
+ZXZlbnQtMjYyODY1ODM3MCI+fXZpZXcgaXQgb24gR2l0SHViDQp7XCpcaHRtbHRh
+ZzkyIDwvYT59LCBvciANCntcKlxodG1sdGFnODQgPGEgaHJlZj0iaHR0cHM6Ly9n
+aXRodWIuY29tL25vdGlmaWNhdGlvbnMvdW5zdWJzY3JpYmUtYXV0aC9BTTczMkJR
+T0FFSzJUSlNZUElTSE1RTFFKSUtUREFOQ05GU000SVZZUkRCUSI+fW11dGUgdGhl
+IHRocmVhZA0Ke1wqXGh0bWx0YWc5MiA8L2E+fS4NCntcKlxodG1sdGFnODQgPGlt
+ZyBzcmM9Imh0dHBzOi8vZ2l0aHViLmNvbS9ub3RpZmljYXRpb25zL2JlYWNvbi9B
+TTczMkJWWVhaQlAyQ1RYUVlFM0FVM1FKSUtUREE1Q05GU000SVZZUkRCMllZM1BO
+VldXSzNUVUw1MkhTNERGV1pFWEc0M1ZNVkNYTVpMT09SSEc2NURKTVpVV0dZTFVO
+RlhXNUtURE41V1cyWkxPT1JQV1NaR09UU1hDSlFRLmdpZiIgaGVpZ2h0PSIxIiB3
+aWR0aD0iMSIgYWx0PSIiIC8+fVxodG1scnRmXHBhciB9XGh0bWxydGYwIA0KXGh0
+bWxydGYgXHBhcg0KXGh0bWxydGYwIA0Ke1wqXGh0bWx0YWc3MiA8L3A+fQ0Ke1wq
+XGh0bWx0YWcwIFxwYXIgfQ0Ke1wqXGh0bWx0YWcyNDAgPHNjcmlwdCB0eXBlPSJh
+cHBsaWNhdGlvbi9sZCtqc29uIj59Ww0KIFx7DQogIkBjb250ZXh0IjogImh0dHA6
+Ly9zY2hlbWEub3JnIiwNCiAiQHR5cGUiOiAiRW1haWxNZXNzYWdlIiwNCiAicG90
+ZW50aWFsQWN0aW9uIjogXHsNCiAiQHR5cGUiOiAiVmlld0FjdGlvbiIsDQogInRh
+cmdldCI6ICJodHRwczovL2dpdGh1Yi5jb20vcnNwYW1kL3JzcGFtZC9wdWxsLzMw
+MzQ/ZW1haWxfc291cmNlPW5vdGlmaWNhdGlvbnNcXHUwMDI2ZW1haWxfdG9rZW49
+QU03MzJCV0NFVVNEV1MyQjc1SFQ1S1RRSklLVERBNUNORlNNNElWWVJEQjJZWTNQ
+TlZXV0szVFVMNTJIUzRERldaRVhHNDNWTVZDWE1aTE9PUkhHNjVESk1aVVdHWUxV
+TkZYVzVLVERONVdXMlpMT09SUFdTWkdPVFNYQ0pRUSNldmVudC0yNjI4NjU4Mzcw
+IiwNCiAidXJsIjogImh0dHBzOi8vZ2l0aHViLmNvbS9yc3BhbWQvcnNwYW1kL3B1
+bGwvMzAzND9lbWFpbF9zb3VyY2U9bm90aWZpY2F0aW9uc1xcdTAwMjZlbWFpbF90
+b2tlbj1BTTczMkJXQ0VVU0RXUzJCNzVIVDVLVFFKSUtUREE1Q05GU000SVZZUkRC
+MllZM1BOVldXSzNUVUw1MkhTNERGV1pFWEc0M1ZNVkNYTVpMT09SSEc2NURKTVpV
+V0dZTFVORlhXNUtURE41V1cyWkxPT1JQV1NaR09UU1hDSlFRI2V2ZW50LTI2Mjg2
+NTgzNzAiLA0KICJuYW1lIjogIlZpZXcgUHVsbCBSZXF1ZXN0Ig0KIFx9LA0KICJk
+ZXNjcmlwdGlvbiI6ICJWaWV3IHRoaXMgUHVsbCBSZXF1ZXN0IG9uIEdpdEh1YiIs
+DQogInB1Ymxpc2hlciI6IFx7DQogIkB0eXBlIjogIk9yZ2FuaXphdGlvbiIsDQog
+Im5hbWUiOiAiR2l0SHViIiwNCiAidXJsIjogImh0dHBzOi8vZ2l0aHViLmNvbSIN
+CiBcfQ0KIFx9DQogXQ0Ke1wqXGh0bWx0YWcyNDggPC9zY3JpcHQ+fX0AAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8cD5NZXJnZWQgPGEgY2xh
+c3M9Imlzc3VlLWxpbmsganMtaXNzdWUtbGluayIgZGF0YS1lcnJvci10ZXh0PSJG
+YWlsZWQgdG8gbG9hZCBpc3N1ZSB0aXRsZSIgZGF0YS1pZD0iNDkyNDUwODU0IiBk
+YXRhLXBlcm1pc3Npb24tdGV4dD0iSXNzdWUgdGl0bGUgaXMgcHJpdmF0ZSIgZGF0
+YS11cmw9Imh0dHBzOi8vZ2l0aHViLmNvbS9yc3BhbWQvcnNwYW1kL2lzc3Vlcy8z
+MDM0IiBkYXRhLWhvdmVyY2FyZC10eXBlPSJwdWxsX3JlcXVlc3QiIGRhdGEtaG92
+ZXJjYXJkLXVybD0iL3JzcGFtZC9yc3BhbWQvcHVsbC8zMDM0L2hvdmVyY2FyZCIg
+aHJlZj0iaHR0cHM6Ly9naXRodWIuY29tL3JzcGFtZC9yc3BhbWQvcHVsbC8zMDM0
+Ij4jMzAzNDwvYT4gaW50byBtYXN0ZXIuPC9wPg0KDQo8cCBzdHlsZT0iZm9udC1z
+aXplOnNtYWxsOy13ZWJraXQtdGV4dC1zaXplLWFkanVzdDpub25lO2NvbG9yOiM2
+NjY7Ij4mbWRhc2g7PGJyIC8+WW91IGFyZSByZWNlaXZpbmcgdGhpcyBiZWNhdXNl
+IHlvdSBhdXRob3JlZCB0aGUgdGhyZWFkLjxiciAvPlJlcGx5IHRvIHRoaXMgZW1h
+aWwgZGlyZWN0bHksIDxhIGhyZWY9Imh0dHBzOi8vZ2l0aHViLmNvbS9yc3BhbWQv
+cnNwYW1kL3B1bGwvMzAzND9lbWFpbF9zb3VyY2U9bm90aWZpY2F0aW9ucyZhbXA7
+ZW1haWxfdG9rZW49QU03MzJCV0NFVVNEV1MyQjc1SFQ1S1RRSklLVERBNUNORlNN
+NElWWVJEQjJZWTNQTlZXV0szVFVMNTJIUzRERldaRVhHNDNWTVZDWE1aTE9PUkhH
+NjVESk1aVVdHWUxVTkZYVzVLVERONVdXMlpMT09SUFdTWkdPVFNYQ0pRUSNldmVu
+dC0yNjI4NjU4MzcwIj52aWV3IGl0IG9uIEdpdEh1YjwvYT4sIG9yIDxhIGhyZWY9
+Imh0dHBzOi8vZ2l0aHViLmNvbS9ub3RpZmljYXRpb25zL3Vuc3Vic2NyaWJlLWF1
+dGgvQU03MzJCUU9BRUsyVEpTWVBJU0hNUUxRSklLVERBTkNORlNNNElWWVJEQlEi
+Pm11dGUgdGhlIHRocmVhZDwvYT4uPGltZyBzcmM9Imh0dHBzOi8vZ2l0aHViLmNv
+bS9ub3RpZmljYXRpb25zL2JlYWNvbi9BTTczMkJWWVhaQlAyQ1RYUVlFM0FVM1FK
+SUtUREE1Q05GU000SVZZUkRCMllZM1BOVldXSzNUVUw1MkhTNERGV1pFWEc0M1ZN
+VkNYTVpMT09SSEc2NURKTVpVV0dZTFVORlhXNUtURE41V1cyWkxPT1JQV1NaR09U
+U1hDSlFRLmdpZiIgaGVpZ2h0PSIxIiB3aWR0aD0iMSIgYWx0PSIiIC8+PC9wPg0K
+PHNjcmlwdCB0eXBlPSJhcHBsaWNhdGlvbi9sZCtqc29uIj5bDQp7DQoiQGNvbnRl
+eHQiOiAiaHR0cDovL3NjaGVtYS5vcmciLA0KIkB0eXBlIjogIkVtYWlsTWVzc2Fn
+ZSIsDQoicG90ZW50aWFsQWN0aW9uIjogew0KIkB0eXBlIjogIlZpZXdBY3Rpb24i
+LA0KInRhcmdldCI6ICJodHRwczovL2dpdGh1Yi5jb20vcnNwYW1kL3JzcGFtZC9w
+dWxsLzMwMzQ/ZW1haWxfc291cmNlPW5vdGlmaWNhdGlvbnNcdTAwMjZlbWFpbF90
+b2tlbj1BTTczMkJXQ0VVU0RXUzJCNzVIVDVLVFFKSUtUREE1Q05GU000SVZZUkRC
+MllZM1BOVldXSzNUVUw1MkhTNERGV1pFWEc0M1ZNVkNYTVpMT09SSEc2NURKTVpV
+V0dZTFVORlhXNUtURE41V1cyWkxPT1JQV1NaR09UU1hDSlFRI2V2ZW50LTI2Mjg2
+NTgzNzAiLA0KInVybCI6ICJodHRwczovL2dpdGh1Yi5jb20vcnNwYW1kL3JzcGFt
+ZC9wdWxsLzMwMzQ/ZW1haWxfc291cmNlPW5vdGlmaWNhdGlvbnNcdTAwMjZlbWFp
+bF90b2tlbj1BTTczMkJXQ0VVU0RXUzJCNzVIVDVLVFFKSUtUREE1Q05GU000SVZZ
+UkRCMllZM1BOVldXSzNUVUw1MkhTNERGV1pFWEc0M1ZNVkNYTVpMT09SSEc2NURK
+TVpVV0dZTFVORlhXNUtURE41V1cyWkxPT1JQV1NaR09UU1hDSlFRI2V2ZW50LTI2
+Mjg2NTgzNzAiLA0KIm5hbWUiOiAiVmlldyBQdWxsIFJlcXVlc3QiDQp9LA0KImRl
+c2NyaXB0aW9uIjogIlZpZXcgdGhpcyBQdWxsIFJlcXVlc3Qgb24gR2l0SHViIiwN
+CiJwdWJsaXNoZXIiOiB7DQoiQHR5cGUiOiAiT3JnYW5pemF0aW9uIiwNCiJuYW1l
+IjogIkdpdEh1YiIsDQoidXJsIjogImh0dHBzOi8vZ2l0aHViLmNvbSINCn0NCn0N
+Cl08L3NjcmlwdD4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAIqIJ9YAAAAAAKXa42s31wgAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAA
+AwAAAAAAAAAAAAAAAAAAAEAABzAGAAAAy5Wb+mVq1QFAAAgwBgAAAMuVm/platUB
+QAA5AAYAAAAApWPBVGnVAQMA9w8GAAAAAAAAAAMAAAADAPQPBgAAAAIAAAADAAAA
+AwDePwYAAADp/QAAAwAAAAMAFwAGAAAAAQAAAAMAAAALABsOBgAAAAAAAAADAAAA
+AwANNAYAAAB5DgUAAwAAAB8AGgAGAAAAEgAAAAMAAAAfAB4MBgAAAAoAAAADAAAA
+HwA9AAYAAAACAAAAAwAAAB8AHQ4GAAAAXgAAAAMAAAAfADcABgAAAF4AAAADAAAA
+HwAAEAYAAAAgBAAAAwAAAB8ABA4GAAAAHAAAAAMAAAALAACABgAAAAAAAAAAAAAA
+AwAHDgYAAAABAAAAAwAAAEAABg4GAAAAAKVjwVRp1QEfAHAABgAAAF4AAAADAAAA
+AgEdDAYAAAAeAAAAAwDqBwIBGQwGAAAAhgAAAAMA6gcfABoMBgAAACIAAAADAAAA
+HwAfDAYAAAAyAAAAAwAAAAIBOwAGAAAAHgAAAAMA6gcfAEIABgAAACIAAAADAAAA
+HwBlAAYAAAAyAAAAAwAAAB8AAw4GAAAAIAAAAAMAAAAfAAIOBgAAAAIAAAADAAAA
+HwB9AAYAAAB+AwAAAwAAAAIBExAGAAAAywcAAAMA6gcLAB8OBgAAAAEAAAADAAAA
+AgEJEAYAAACZCgAAAwDqBwMANgAGAAAAAAAAAAMAAAACAQswBgAAABAAAAADAOoH
+HwBkAAYAAAACAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAjwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAACBKx+kvqMQGZ1uAN0BD1QCAAABgHIAcwBwAGEA
+bQBkAC8AcgBzAHAAYQBtAGQAAABTAE0AVABQAAAAcgBzAHAAYQBtAGQAQABuAG8A
+cgBlAHAAbAB5AC4AZwBpAHQAaAB1AGIALgBjAG8AbQAAAAAAAAAAAAAAAAAAAAAA
+cgBzAHAAYQBtAGQALwByAHMAcABhAG0AZAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAFMATQBUAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAByAHMAcABhAG0AZABAAG4A
+bwByAGUAcABsAHkALgBnAGkAdABoAHUAYgAuAGMAbwBtAAAAAAAAAAAAAAAAAAAA
+U01UUDpSU1BBTURATk9SRVBMWS5HSVRIVUIuQ09NAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgH2DwYAAAAEAAAAAwDqBwIB/w8GAAAA
+cgAAAAMA6gcfAAEwBgAAABwAAAADAAAAHwACMAYAAAAKAAAAAwAAAB8AAzAGAAAA
+NAAAAAMAAAACAQswBgAAAB8AAAADAOoHAwAVDAYAAAABAAAAAwAAAAMAADAGAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAMAAI8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgSsfpL6jEBmdbgDd
+AQ9UAgAAAYBrAG8AcgBnAG8AdABoADEAAABTAE0AVABQAAAAdgBsAGEAZABpAHMA
+bABhAHYALgBzAHQAYQBrAGgAbwB2AEAAZwBtAGEAaQBsAC4AYwBvAG0AAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAGsAbwByAGcAbwB0AGgAMQAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABTAE0AVABQAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+dgBsAGEAZABpAHMAbABhAHYALgBzAHQAYQBrAGgAbwB2AEAAZwBtAGEAaQBsAC4A
+YwBvAG0AAAAAAAAAAAAAAFNNVFA6VkxBRElTTEFWLlNUQUtIT1ZAR01BSUwuQ09N
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIB9g8GAAAA
+BAAAAAMA6gcCAf8PBgAAAGwAAAADAOoHHwABMAYAAAASAAAAAwAAAB8AAjAGAAAA
+CgAAAAMAAAAfAAMwBgAAADgAAAADAAAAAgELMAYAAAAhAAAAAwDqBwMAFQwGAAAA
+AgAAAAMAAAADAAAwBgAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAACPAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAIErH6S+oxAZnW4A3QEPVAIAAAGAQQB1AHQAaABvAHIAAABTAE0AVABQAAAA
+YQB1AHQAaABvAHIAQABuAG8AcgBlAHAAbAB5AC4AZwBpAHQAaAB1AGIALgBjAG8A
+bQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBAHUAdABoAG8AcgAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+UwBNAFQAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAGEAdQB0AGgAbwByAEAAbgBvAHIAZQBwAGwAeQAuAGcA
+aQB0AGgAdQBiAC4AYwBvAG0AAAAAAAAAAAAAAAAAAABTTVRQOkFVVEhPUkBOT1JF
+UExZLkdJVEhVQi5DT00AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAACAfYPBgAAAAQAAAADAOoHAgH/DwYAAABkAAAAAwDqBx8AATAGAAAA
+DgAAAAMAAAAfAAIwBgAAAAoAAAADAAAAHwADMAYAAAA0AAAAAwAAAAIBCzAGAAAA
+HwAAAAMA6gcDABUMBgAAAAIAAAADAAAAAwAAMAYAAAACAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAP7////+/////v////7///8FAAAA/v////7////+////
+/v///woAAAD+////DAAAAA0AAAAOAAAADwAAABAAAAARAAAAEgAAABMAAAAUAAAA
+FQAAABYAAAAXAAAAGAAAAP7///8aAAAAGwAAAP7////+/////v////7////+////
+/v////7///8jAAAA/v///yUAAAAmAAAAJwAAACgAAAApAAAAKgAAACsAAAAsAAAA
+LQAAAC4AAAAvAAAAMAAAADEAAAAyAAAAMwAAADQAAAD+////NgAAADcAAAA4AAAA
+OQAAADoAAAA7AAAAPAAAAD0AAAA+AAAAPwAAAEAAAABBAAAAQgAAAEMAAABEAAAA
+RQAAAEYAAABHAAAASAAAAEkAAABKAAAASwAAAEwAAABNAAAATgAAAE8AAABQAAAA
+UQAAAFIAAABTAAAAVAAAAFUAAABWAAAAVwAAAFgAAABZAAAAWgAAAFsAAABcAAAA
+XQAAAF4AAABfAAAA/v///2EAAABiAAAAYwAAAGQAAABlAAAAZgAAAGcAAABoAAAA
+aQAAAGoAAABrAAAAbAAAAG0AAABuAAAAbwAAAHAAAABxAAAAcgAAAHMAAAB0AAAA
+dQAAAHYAAAB3AAAAeAAAAHkAAAB6AAAAewAAAHwAAAB9AAAAfgAAAH8AAAD+////
+/v///4IAAACDAAAAhAAAAIUAAACGAAAAhwAAAIgAAACJAAAAigAAAP7////+////
+jQAAAP7////+/////v////7////+////kwAAAJQAAAD+/////v///5cAAAD+////
+/v////7////+/////v///50AAACeAAAA/v////7///+hAAAA/v////7////+////
+/v////7///+nAAAAqAAAAP7///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABSAG8AbwB0ACAARQBuAHQA
+cgB5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+FgAFAf//////////AQAAAAsNAgAAAAAAwAAAAAAAAEYAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAABAKgAAAAAAAF8AXwBuAGEAbQBlAGkAZABfAHYAZQByAHMAaQBvAG4A
+MQAuADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAEB/////wYAAAACAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMAAwADAAMgAwADEAMAAyAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////AwAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAwADAAMAAzADAAMQAwADIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8EAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAEAAAAIAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADAAMAAwADQA
+MAAxADAAMgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////wUAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/v///wAAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMQAwADEAMgAwADEAMAAyAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAACAAAACAAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAwADAAMQBBADAAMAAxAEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8HAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAMAAAAQAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADAAMAAzADcA
+MAAwADEARgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////wgAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAFwAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMAAwADMAQgAwADEAMAAyAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////CQAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAGAAAAHgAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAwADAAMwBEADAAMAAxAEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8KAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAP7///8AAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADAAMAA0ADIA
+MAAwADEARgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////wsAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABwAAACAAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMAAwADYANAAwADAAMQBGAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////DAAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAD+////AAAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAwADAANgA1ADAAMAAxAEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8NAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAgAAAAwAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADAAMAA3ADAA
+MAAwADEARgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////w4AAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACQAAAFwAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMAAwADcARAAwADAAMQBGAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////DwAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAALAAAAfAMAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAwAEMAMQA5ADAAMQAwADIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8QAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAABkAAACGAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADAAQwAxAEEA
+MAAwADEARgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////xEAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHAAAACAAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMABDADEARAAwADEAMAAyAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////EgAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAdAAAAHgAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAwAEMAMQBFADAAMAAxAEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8TAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAB4AAAAIAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADAAQwAxAEYA
+MAAwADEARgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////xQAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHwAAADAAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMABFADAAMgAwADAAMQBGAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////FQAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAD+////AAAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAwAEUAMAAzADAAMAAxAEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8WAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAACAAAAAeAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADAARQAwADQA
+MAAwADEARgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////xcAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIQAAABoAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMABFADEARAAwADAAMQBGAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////GAAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAiAAAAXAAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAxADAAMAAwADAAMAAxAEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8ZAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAACQAAAAeBAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADEAMAAwADkA
+MAAxADAAMgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////xoAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANQAAAJkKAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMQAwADEAMwAwADEAMAAyAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////GwAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAABgAAAAywcAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAzADAAMABCADAAMQAwADIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8cAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAIAAAAAQAAAAAAAAAF8AXwBwAHIAbwBwAGUAcgB0AGkAZQBzAF8AdgBlAHIA
+cwBpAG8AbgAxAC4AMAAAAAAAAAAAAAAAAAAAAAAAAAAwAAIB/////x0AAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgQAAAGACAAAAAAAA
+XwBfAHIAZQBjAGkAcABfAHYAZQByAHMAaQBvAG4AMQAuADAAXwAjADAAMAAwADAA
+MAAwADAAMAAAAAAAAAAAADoAAQH/////JQAAAB4AAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAwAEYARgA2ADAAMQAwADIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8fAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAIsAAAAEAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADAARgBGAEYA
+MAAxADAAMgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////yAAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAjAAAAHIAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMwAwADAAMQAwADAAMQBGAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////IQAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAACOAAAAGgAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAzADAAMAAyADAAMAAxAEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8iAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAI8AAAAIAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADMAMAAwADMA
+MAAwADEARgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////yMAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAADIAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMwAwADAAQgAwADEAMAAyAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////JAAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAACRAAAAHwAAAAAAAABfAF8AcAByAG8AcABlAHIA
+dABpAGUAcwBfAHYAZQByAHMAaQBvAG4AMQAuADAAAAAAAAAAAAAAAAAAAAAAAAAA
+MAACAf///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAJIAAACIAAAAAAAAAF8AXwByAGUAYwBpAHAAXwB2AGUAcgBzAGkAbwBuADEA
+LgAwAF8AIwAwADAAMAAwADAAMAAwADEAAAAAAAAAAAA6AAEB/////y0AAAAmAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMABGAEYANgAwADEAMAAyAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////JwAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAACVAAAABAAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAwAEYARgBGADAAMQAwADIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8oAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAJYAAABsAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADMAMAAwADEA
+MAAwADEARgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////ykAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAmAAAABAAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMwAwADAAMgAwADAAMQBGAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////KgAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAACZAAAACAAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAzADAAMAAzADAAMAAxAEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8rAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAJoAAAA2AAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADMAMAAwAEIA
+MAAxADAAMgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////ywAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAmwAAACEAAAAAAAAA
+XwBfAHAAcgBvAHAAZQByAHQAaQBlAHMAXwB2AGUAcgBzAGkAbwBuADEALgAwAAAA
+AAAAAAAAAAAAAAAAAAAAADAAAgH///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAACcAAAAiAAAAAAAAABfAF8AcgBlAGMAaQBwAF8A
+dgBlAHIAcwBpAG8AbgAxAC4AMABfACMAMAAwADAAMAAwADAAMAAyAAAAAAAAAAAA
+OgABAf//////////LgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADAARgBGADYA
+MAAxADAAMgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////y8AAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAnwAAAAQAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMABGAEYARgAwADEAMAAyAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////MAAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAZAAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAzADAAMAAxADAAMAAxAEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////8xAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAKIAAAAMAAAAAAAAAF8AXwBzAHUAYgBzAHQAZwAxAC4AMABfADMAMAAwADIA
+MAAwADEARgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqAAIB/////zIAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAowAAAAgAAAAAAAAA
+XwBfAHMAdQBiAHMAdABnADEALgAwAF8AMwAwADAAMwAwADAAMQBGAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAACoAAgH/////MwAAAP////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAACkAAAAMgAAAAAAAABfAF8AcwB1AGIAcwB0AGcA
+MQAuADAAXwAzADAAMABCADAAMQAwADIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+KgACAf////80AAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAKUAAAAfAAAAAAAAAF8AXwBwAHIAbwBwAGUAcgB0AGkAZQBzAF8AdgBlAHIA
+cwBpAG8AbgAxAC4AMAAAAAAAAAAAAAAAAAAAAAAAAAAwAAIB////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAApgAAAIgAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAH///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAf///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AQAAAAIAAAADAAAABAAAAAUAAAAGAAAABwAAAAgAAAAJAAAACgAAAAsAAAAMAAAA
+DQAAAA4AAAAPAAAAEAAAABEAAAASAAAAEwAAABQAAAAVAAAA/v///xcAAAD+////
+GQAAABoAAAAbAAAAHAAAAB0AAAAeAAAAHwAAACAAAAAhAAAAIgAAACMAAAAkAAAA
+JQAAAP7////9////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+//////////////////////////////////////////8=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: docx
+
+UEsDBBQAAAAIAM+rLU9LVVb02AAAAD0CAAALABQAX3JlbHMvLnJlbHMBABAAAAAA
+AAAAAAAAAAAAAAAAAK2STUsDQQyG7/0VQ+7dbCuISHd7EaE3kfoDwkx2d2jng0zU
++u8dRNGFUgQ95s2bh+eQzfYUjuaFpfgUO1g1LRiONjkfxw6e9vfLG9j2i80jH0lr
+pUw+F1NvYulgUs23iMVOHKg0KXOsmyFJIK2jjJjJHmhkXLftNcpPBvQzptm5DmTn
+VmD2b5n/xsbASo6U0CbhZZZ6Leq5VDjJyNqBS/ahxuWj0VQy4Hmh9e+F0jB4y3fJ
+PgeOes6LT8rRsbusRDlfMrr6T6N541vmNYlD9xl/2eDsC/rFO1BLAwQUAAAACADP
+qy1PBCHWFboAAAAbAQAAEQAUAGRvY1Byb3BzL2NvcmUueG1sAQAQAAAAAAAAAAAA
+AAAAAAAAAABtjk1rhEAQRO/+Cpm7tm4gBFn1llMWAklgr0Pb0WGdD6Y7GX9+JrKY
+S45FvXrUedzsWn5TZONdr9q6USU59JNxc68+3p+rJzUOxRlDhz7Sa/SBohjiMu8c
+dxh6tYiEDoBxIau5zoTL5aePVkuOcYag8aZnglPTPIIl0ZMWDb/CKhxGdVdOeCjD
+V1x3wYRAK1lywtDWLfyxQtHyv4O9OciNzUGllOr0sHP5UQvXy8vbfr4yjkU7JAVD
+8QNQSwMEFAAAAAgAz6stT/eOlC+MAAAA1wAAABAAFABkb2NQcm9wcy9hcHAueG1s
+AQAQAAAAAAAAAAAAAAAAAAAAAACdzs0KwjAQBOB7nyLk3qZ6ECn9uRTPHqr3kmza
+gNkNyVrq2xsRfACPwzAf0w67f4gNYnKEnTxUtRSAmozDpZO36VKe5dAX7TVSgMgO
+ksgDTJ1cmUOjVNIr+DlVucbcWIp+5hzjoshap2Ek/fSArI51fVKwM6ABU4YfKL9i
+s/G/qCH9+Zfu0ytkT/XFG1BLAwQUAAAACADPqy1P4yOQggQCAAD8BAAAEQAUAHdv
+cmQvZG9jdW1lbnQueG1sAQAQAAAAAAAAAAAAAAAAAAAAAACllN9u0zAUxu/3FJHv
+2yTQjS1qOiFNoF0gIcoewE2cxML2sY7dhnK3PQgPgcQFSOMZ0jfCjpOGCQlV48b2
+sf39jv35z/L6sxTRjqHhoHKSzhMSMVVAyVWdk7uPb2aX5Hp1tmyzEoqtZMpGTqBM
+hjlprNVZHJuiYZKaOWim3FgFKKl1IdYxVBUv2M2gjF8kyUWMTFDrkpmGa0MGWqtP
+wZVIW7cuKQKpBSw1QsGMcb03YfBIPAX4lDByJeXqiEmTnGxRZQNlJnmBYKCyswJk
+FjaYec6kWPyV+iiaO9HgSp/cJUyTviXF5IV5DmDax7qhmk20+v9obxG2eqTJ4hRb
+JcVPW+0N0u6oN1xwu++tHTG7f3m6m6yAE7wP1XGBz72WbrAhkSyy21oB0o1gOXFH
+SVbu6m+g3Pta98V77Ku13QsWtdmOipx0Xw/33WP3q/vWlz+iw0P3vft5uD88RK9J
+vFrGR2EohrYVIyHxs9pMUFWPXUzN7taDeBDYVeojG/oC1vcbVtgwo2G0ZPiBVQzd
+O/YLtHvt9lKyim6FJRFmvMwJ3paLkLECsKcJzoNA1+svbpZ7YGl6lbj/wiV17YvL
+Rd8G5M7UnGhAi5TbUfSOomeD9rqXCz8Ved3YKdyAtSCnWLDqj9GwsZy8Sq58GJad
+k8vzwbkNL/kTM+PJlng8w3j6x1ZnvwFQSwMEFAAAAAgAz6stT/NWBZ7BBAAAGhUA
+AA8AFAB3b3JkL3N0eWxlcy54bWwBABAAAAAAAAAAAAAAAAAAAAAAAO1YzW7jNhC+
+71MIuieyjaBojXUWjosgAdI0yE/vFEVbrClSIGkr2dumx32DHvoQBQq0XWyfwX6j
+DilKlkVbMRboYlFsgEScb4ajmeH8iHn95jFjwZJIRQUfhf3jXhgQjkVC+WwUPtyf
+H30bvjl99boYKv3EiApAnKthMQpTrfNhFCmckgypY5ETDrypkBnSQMpZVAiZ5FJg
+ohRoy1g06PW+iTJEeejUZPgQPRmS80V+hEWWI01jyqh+sroqNUX/xNOTUSyFElN9
+DPsiMZ1STKxFsLPfs6uMhUGGh5czLiSKGRmFoCg8BV8Tgb8nU7RgWhlS3khHOso+
+zgXXKiiGSGFKR+E9zSA816QIbkWGwMViiNVOOB1ztXsDQUqPFUWjcCwpYsEDp3AU
+JPjhLozMO2OQWSI2CnuOnqgWQtu0J4FR3oZUhhib7MC1pHPSApOdqFhoRnkbJVks
+lGdilkvKdQtdIk5V2rZVMCErDC20cHblCENGtYSLiu73HDInkreEcqGohlRve/q2
+AgY1solchS0qgAtO7AEb+5qWxUltLqfMeUakHjM6q98ZI0VMsEo2Q3wGi8glVtRO
+t7xNlZ6R/Jo8toNo4CvQ3A55jmbkTBI0PyNQWO1TmkqUkRtjOOI4FfJK4LlhAlC4
+Z+qeyzsIPanAJvFoXRyFjEy1oZ8qoduFqSzyiLDFl2P7jlEIVT2DTlAeHU1EMYF6
+koLVp1iy+CIrXaZsyVqGA+8y2cKixga1yHMJvccE5HqRxdDj2mE5S6ykFrl3asYP
+D4yF1iLzYEln6Q5hogtCuI+jdo5EG1NUWjuEGUGynWVATinbrgbn5xiQi6c8Jdyr
+47peYnv+7mhKwmxzAg5HU01kc+2LmPSFujipifKUN0ZRbvwwQXQ7bIzcekql0ldW
+hbPwZ1yZbPOn2VKuNsf+XRUsVyjN0oi8rm0HFmzVTzm8KEcSzSTKU2NAUoqZJAuc
+4GUyCq/N0GF2AnAoieq1Di5TrlF0TRxMiRrDodFPTnb0k5NN6VcY4UcPd9szoIZi
+msC8QPLobhxutwpruu8sTsFbDCfX4awLVXBTBSYwA81zfo+YF4z9cgeaWRsGOUwk
+nPzcs2XDaca67srmK4ORwyKkzcDviM694Qd7EmKL6UXC51bJoWNWmgyLS1sgVX8t
+jUoekbN+IymrvXij5M5YaRt2uc/W060own0CDHXzrYKJYJ0Kuvgx4kn/Jxh0nRKD
+FyX6F0K+7dbRKcHJhLD9ZvKik626d6t69/684nbQQCJ2tpngiiq9o8+U+I5G02CU
+ybTfgq1GV79z9evqn9UH+P24fl79uX5e/7L6sH7vmbBHzLOoS+6/+ETp+ED4hIkJ
+xa/cs9pqx5ORhQ9EGDS9QfU5Ucm+PGVNlz5s8DYH7BcwSfdeaS4IWxJNMYJbyoJU
+NxrvarK50vgb/m83mq0bSc/+dN1JPukG0vHFsH0DOeB60T8ZaijCH0tnSoCLcygI
+mwC72AYwAiWlBKNJg5SzeMIgArBuxSBqiUfbyl6eyXs612/rd6uP0Gp+t3//CqDj
+/LH6e/1u/RyM/QbWKe33sZfFv/x29rU1fW1Nn6k1DXa0pkH3P0ealhzwr5Dt+1D0
+mVtStVKnr/4FUEsDBBQAAAAIAM+rLU/0I42L7wAAALYCAAASABQAd29yZC9mb250
+VGFibGUueG1sAQAQAAAAAAAAAAAAAAAAAAAAAAC1kc9OwzAMh+97iih3lrIDQtXS
+iQviAge2PYCXuqul/KnirGFvT9ZtJxBCGtwS/774k+Pl6sNZMWJkCl7L+3klBXoT
+WvJ7Lbeb57tHuWpmy1x3wScWhfZcZy37lIZaKTY9OuB5GNCXrAvRQSrXuFc5xHaI
+wSBzaeasWlTVg3JAXjaXfiLXHhxquSGHLN4wi/fg4AyYHiLjiRnBallVUk3vwJE9
+XqtxwqdgoGT6a32ESLCzeIrUWfZFuj66XbDfuhZ/7XoqiP39WJyJ+RaV2HoqW0Tx
+ur79M1vs4GDTT9IXtCMmMlCWeMD/UV4O3Mw+AVBLAwQUAAAACADPqy1P/8xHDjkG
+AACKVwAAFQAUAHdvcmQvdGhlbWUvdGhlbWUxLnhtbAEAEAAAAAAAAAAAAAAAAAAA
+AAAA7Vzbbts4EH3vVwh6b2XZkmMHTYpc1ugC2dZIsthnWqIs1RSlJek0ydfvkLpb
+cpI2TZsuxgbsIXnEIYeHw8vAfv/hNmXWDRUyyfiR7b4b2RblQRYmfH1k/329eDuz
+Pxy/eU8OVUxTagGay0NyZMdK5YeOIwPIJvJdllMOZVEmUqIgKdZOKMhXqCVlzng0
+mjopSbhdPi+e8nwWRUlAz7Ngm1KuikoEZURBS2Wc5NK2OEnpkf3ZAO3jqpF/MKqf
+kDojYOIqMC3vYcONq7+kWK/OmLBuCDuyR+ZlO8fvnRrAVB+3MK8SVwLCzbiHOznQ
+77q+cVFfH+dP9LuuzwBIEEAv+rq9xcw9PS+xLVAh9us+G/kjr4tv1T/p4eenp6f+
+vIOfNHivh5+Npt7JuIP3Grzfb//pydnZtIP3G/y0b+uD+dTr4g0oZgnfDI5gPTI1
+JMrYx0H4YtGCNyinxZziea728SglXzKxAIAZXKAnt9RdTiMSAO4jZTdUJQGxPtEt
+1XrIISWPAAL5IMDZ0Zkm/Oc3oNHptM1jjJXutVWUMHal7hi9kKa1MmNJuIBMkzAP
+1UOTxyCW6jq4tSBGtkSm/klUfBWTHNS4RsNallWvpZVnEghh763beIyEq3IOVlMf
+0ET9lYVF9qTtEupqTGot24omuoKnKpscPE+ZWwCfqM31h7X5D2pzWtaEaWERvSS4
+03Gh2pIBYTTUdi8qqIblBYfIHbXGKCYhHchu9c8dz+H1w63pf1MjfoyRRz0jO/3Z
+xHg3ZX0Fb+6PfdsKSH5kR+AbQExzqE/ytW0RtoY1P1BFBx+fizs9ng+zyh15+6ze
+UZELqc6JjIunTFG1APKm/WPf03b4MR1wvrcVk5n7C1vh7A4tjSIaqD05TRLKikoG
+S58J1olsq6i4isOvlrytZ4Al7xp5c6u5a23uzFdhqZW94xagZMW24pKEtaXDRCoY
+/KKSMIENo+ZBPUkH9mxmC8HymJTTbtaadQXcyHWTW301vdntuTNk5tV68YpXMO+7
+PFCjyPsWL+j5Q15wPn9eE57iiVvqxsM9Hvv+U31uTlRs6Q+YwYkIWLNPuc4uYfSt
+2uFbQMi3s0IUdeYK2jxrdU5X9bOWw9no5TcRLWNP9hh7NHoZY/sDtvYfNrXTn6JO
+a0NqUr0zYrb6ArrPYbu7ZUWOzCFVCEvRn+V7T4Id2J716zGnUZ7ovmGVIIci2/Kw
+vWq1XbruxaQsrfqzysK7pdB0NN5X5sEigcoviFRLIoimpr4UUJ/hI2IZ9CMrJduK
+M3E/lK/xcK6HUtv6KnR/5b9bIqhtsT+5obWlKkFUwqoS+DY9y5hRDK0xYrkUCMVM
+Uq8fPID6j2xYf7e5SNaxqmdLfrJV2SIp17Cif2YUZLN2hTRaQq9TIi5MdSBcGiHh
+Idi8tUQxWHRodE1WV/fg01zPKxtiIPq0c2JgBHRC9/SR64Kfio0pjmELl/D1csuD
+unmwfOdB0c5gGfR2s04XcVqRL1gqWS5yFR/apSeRegBXlq62QK3rW6eQr+5rUZ/e
+6sSnjFMjKrKqaAMWuARzrYqBIpLC/rRYrQ2bOTwCpqsIVHwrkWyoLrsyEuTcFw4S
+Rqx6ZLNNkzT7UjzJ9Y0PS+7px8ZW+pNnmvVtTu+fg51bmw6sOw+2Orvs8u45+Yyk
+K5GQgQNyp6RzMu6UyLt0uMipLFmJy5KZN8zNX5iOLtLxd6BjlyIlMUqOjJEjyJEh
+jowbjkyQI8iRIY5MGo54yBHkyBBHvIYjPnIEOTLEEb/hyBQ5ghwZ4si04cgBcgQ5
+MsSRg4YjM+QIcmSII7OGI3PkCHJkiCPzvJJb17uyEhi/pJGVhLelgYsQw25eoayH
+BDPXecYAZTvrqGAdHGB8J0pQG+GVXP7/30K0ryWSAR5mMq+iGZ5/4M6riEZZsmqX
+PDOywTMd2Yh+eWQDXeVv6Cox2oAUwWgDcgSjDcgRjDYgRzDagBzBaANyBKMNyBGM
+NiBHkCMYbUCOvOZoQx1kULePRRvc8UE/2tBCpYmiwmJJWv7SZ2TjTwbwJwPoFfAn
+A7hI4SU+cgQv8ZEjyBG8xEeO4CU+cgQv8ZEjeImPHMFLfOQIXuIjR/AS//sv8cu7
+e6f/X0PV/xEdv/kPUEsDBBQAAAAIAM+rLU+bs5whXQEAAIMDAAAQABQAd29yZC9o
+ZWFkZXIxLnhtbAEAEAAAAAAAAAAAAAAAAAAAAAAApZNLTsMwEIb3nCLyvk2KEEJW
+024qEDukwgFcx00s4ofGTkK3ZckNOAlSBWdIboTzFkJCUVn4OTPf2L/Hy/WLSL2c
+geFKhmgxD5DHJFURl3GInh5vZzdovbpYFjiJwHO+0mAIUWKtxr5vaMIEMXOlmXS2
+vQJBrFtC7Kv9nlO2UTQTTFr/MgiufWApsS6PSbg2qKMVegouAlK4I4m0JRUKIg2K
+MmPc7qY1DsQpwJ+EnisIlwNmEYQoA4k7ykxwCsqovZ1RJXB7QVxzxoirX6mHoLkL
+6lRpkruEi6CZiXTUwpwDGO+xTYhmIy3+H+0OVKZ7mqBTZBUEnjNdC6TdU+94yu2h
+kbbH5H9pmo9SqAnat8NwwHPL0hkT5AmK72OpgOxSFiL3lGjlql433QM0w9YeUuYV
+OCdpiMr38qs8ufZZHcuP6li9lqfqDfm1545HvPcL6i1/gEC3anr3p1YX31BLAwQU
+AAAACADPqy1PnKm4sF4BAACDAwAAEAAUAHdvcmQvZm9vdGVyMS54bWwBABAAAAAA
+AAAAAAAAAAAAAAAAAKWTS07DMBCG95wi8r5NihBCVtNuKhA7pMIBXMdJLOKHxk5C
+t2XJDTgJUgVnSG6Ek+YhhISisvBzZr6xf4+X6xeReQUDw5UM0WIeII9JqiIukxA9
+Pd7ObtB6dbEscWzBc77SYAhRaq3Gvm9oygQxc6WZdLZYgSDWLSHxVRxzyjaK5oJJ
+618GwbUPLCPW5TEp1wZ1tFJPwUVASnckkZ1IpYJIg6LMGLe7ORkH4hTgT0LPFYTL
+AbMIQpSDxB1lJjgFZVRsZ1QJfLogbjhjxNWv1EPQ3AV1qrTJXcJF0M5ENmphzgGM
+99imRLORlvyPdgcq1z1N0CmyCgLPuW4E0u6pdzzjdt9K22OKvzQtRinUBO1Pw3DA
+c8vSGVPkCYrvE6mA7DIWIveUaOWqXrfdA7TD1u4z5pW4IFmIqvfqqzq69lkfqo/6
+UL9Wx/oN+Y3njke89wuaLX+AQLdqe/enVhffUEsDBBQAAAAIAM+rLU/FEYUUggIA
+AOEEAAARABQAd29yZC9zZXR0aW5ncy54bWwBABAAAAAAAAAAAAAAAAAAAAAAAI1U
+y24TMRTd9ysir2DRTpo+QFEnVauq6oJCpRQ2iIUz40yseGzL9iRECMlVhXgIIRaA
+WkE2LNiwRsyAKiS+oPmFzAfML+B5uKGNBF2N77mv43OvZ2PzcUhqAyQkZtQFy0t1
+UEPUYz6mgQvuH+4u3gabrYWNYVMipQwoayaBymbouaCnFG86jvR6KIRyiXFEjbPL
+RAiVMUXghFD0I77osZBDhTuYYDVyGvX6OqjKMBdEgjarEosh9gSTrKvylCbrdrGH
+qo/NGPwrYxASGze8Dr8hEz4XzENSmruFpOBmWGMKWubOA4yGNfOBxAVcYKqAk8Mh
+FoKJfSgCTKX110tfx1Q0au6wu0y1IxMXUX8PQYNdI3CXMTUX6GPJCRxtQ68fFFHt
+HuSocAk0wPngHpQ8S7XNFIExjBwhokpWpmG6g0hllBKoYsa2C+rCiKhD2Gkrxi2F
+W43KDSPF9ka8h6iZI6NXKKIBolvUv+dXV50XhfV3GfEPcg1N1ytuyu5girYFgn25
+1S0VIDAnN9GT48nR5Oj8x3l8ngCbN9UnU/3pRqrfPXyS6hepfpXq16l+k+q30y/P
+Un2S6g+/v6Z6nMXfsvh7FidZ/DOLz7L4V5YcZclxljzPktMs+Zgl4+zs8zyNbWQ0
+Qv/ncTrV45upfv/oaapflmXKZZ+d2uXDMTkUhsgFlx7DPvNRXi4SeG5fL1Z7yaQ4
+5SsoNvai//Ja3tK51JOIdp6P9iHnZdtOsOwCgoOeKoavjOWbTSmMTtCofI3C1yh9
+hQE9zyyQia4OM6xhsb/iViy2MsNWLbY6w9YstjbD1i22nmO9EUeCYNo3ithjubWE
+sCHy92b+OajSw/6sWgt/AFBLAwQUAAAACADPqy1P+S7/uO4AAACXAwAAHAAUAHdv
+cmQvX3JlbHMvZG9jdW1lbnQueG1sLnJlbHMBABAAAAAAAAAAAAAAAAAAAAAAAK2T
+z4rCMBDG7z5FmLtNq6ssi6kXEbwu9QGy6fQPtklIxmV9e4NFrSBlDzl+k8z3+zJD
+Ntu/vmO/6HxrtIAsSYGhVqZsdS3gWOznn7DNZ5tv7CSFK75prWehR3sBDZH94tyr
+BnvpE2NRh5PKuF5SkK7mVqqTrJEv0nTN3dgD8hdPdigFuEOZASsuFv/jbaqqVbgz
+6tyjpjcI7pEovMMHT+lqJAH3ShK8gL+PsIgZoTKaCvnT4TPDozQVYhl1DnTpcDyF
+m57Cf8TENyhLdE/8oLMp/iruDgyN+YOe5K9j8in0jvZ/k0PxkYG//K98dgVQSwME
+FAAAAAgAz6stT1hc8cqnAQAAvwcAABMAFABbQ29udGVudF9UeXBlc10ueG1sAQAQ
+AAAAAAAAAAAAAAAAAAAAAAC1lctOwzAQRff9iihb1LiwQAi1ZcFjCSzKB7jxJDHE
+D9lDCX/POKFdVE7Squ0mUWZ87h2PJ/L8oVF1sgHnpdGL9DqbpQno3Aipy0X6sXqZ
+3qUPy8l89WvBJ7RW+0VaIdp7xnxegeI+MxY0ZQrjFEf6dCWzPP/iJbCb2eyW5UYj
+aJxi0EiX8yco+HeNyXND4c6X8DR57NYFq0XKra1lzpHSLGRZlHNQ+wFwo8VeddP/
+yjIi2zW+ktZf9Tt8Wij3HKQKW/u0ZQ9idZwI8TixVjZKhHicKGURJUI8TmAPgb2E
+FcVAb0M2zimzGeAoK6GH3AyOQeQ0TVHIHITJvxUhGfFPjv/I3kY3tW9OcvDWARe+
+AkBVZ+07WL3RH+SkgOSdO3zlinQZMe/OWE/z7yBrjt3adlADPbUkBA4l7EZ10JGk
+jzfc2ymErgkQB3r/GCfYDj7VPKiRbw7e02FSp3cZxaUercMDInH+/HVslUdLKMh0
+xdc1nL+GnfR4H/C3hkt0odUdta/oVwF3fX7/TviAMzB4Ef9OeNQf6X6E7nl6Ea3M
+1pK19/Fy8gdQSwECPgAUAAAACADPqy1PS1VW9NgAAAA9AgAACwAAAAAAAAAAAAAA
+AAAAAAAAX3JlbHMvLnJlbHNQSwECPgAUAAAACADPqy1PBCHWFboAAAAbAQAAEQAA
+AAAAAAAAAAAAAAAVAQAAZG9jUHJvcHMvY29yZS54bWxQSwECPgAUAAAACADPqy1P
+946UL4wAAADXAAAAEAAAAAAAAAAAAAAAAAASAgAAZG9jUHJvcHMvYXBwLnhtbFBL
+AQI+ABQAAAAIAM+rLU/jI5CCBAIAAPwEAAARAAAAAAAAAAAAAAAAAOACAAB3b3Jk
+L2RvY3VtZW50LnhtbFBLAQI+ABQAAAAIAM+rLU/zVgWewQQAABoVAAAPAAAAAAAA
+AAAAAAAAACcFAAB3b3JkL3N0eWxlcy54bWxQSwECPgAUAAAACADPqy1P9CONi+8A
+AAC2AgAAEgAAAAAAAAAAAAAAAAApCgAAd29yZC9mb250VGFibGUueG1sUEsBAj4A
+FAAAAAgAz6stT//MRw45BgAAilcAABUAAAAAAAAAAAAAAAAAXAsAAHdvcmQvdGhl
+bWUvdGhlbWUxLnhtbFBLAQI+ABQAAAAIAM+rLU+bs5whXQEAAIMDAAAQAAAAAAAA
+AAAAAAAAANwRAAB3b3JkL2hlYWRlcjEueG1sUEsBAj4AFAAAAAgAz6stT5ypuLBe
+AQAAgwMAABAAAAAAAAAAAAAAAAAAexMAAHdvcmQvZm9vdGVyMS54bWxQSwECPgAU
+AAAACADPqy1PxRGFFIICAADhBAAAEQAAAAAAAAAAAAAAAAAbFQAAd29yZC9zZXR0
+aW5ncy54bWxQSwECPgAUAAAACADPqy1P+S7/uO4AAACXAwAAHAAAAAAAAAAAAAAA
+AADgFwAAd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVsc1BLAQI+ABQAAAAIAM+r
+LU9YXPHKpwEAAL8HAAATAAAAAAAAAAAAAAAAABwZAABbQ29udGVudF9UeXBlc10u
+eG1sUEsFBgAAAAAMAAwA+wIAAAgbAAAAAA==
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: xlsx
+
+UEsDBBQAAgAIAEqZLU9HkkSyWAEAAPAEAAATAAAAW0NvbnRlbnRfVHlwZXNdLnht
+bK2UTU7DMBCF95wi8hYlblkghJp2QWEJlSgHMPakserYlmf6d3smaQsIiUDVbmJF
+9nvf+Hns0WTbuGwNCW3wpRgWA5GB18FYvyjF2/wpvxMZkvJGueChFDtAMRlfjea7
+CJix2GMpaqJ4LyXqGhqFRYjgeaYKqVHEv2kho9JLtQB5MxjcSh08gaecWg8xHr0w
+P1kD2UwlelYNY+TWSWI32H+HBfuJ7GEvbNmlUDE6qxVx4XLtzQ9qHqrKajBBrxqW
+FJ3NdesifwUi7Rzg2SiMCZTBGoAaV+xNj+QpVGrlKHvcsvs+8wQOT+MdwixY2a3B
+2sY+Qv+GftetzwyC9dOkNtxKPaFvQlq+h7C8dOztWDTK+r5DZ/EshYiSUWcXAG1y
+Bkwe2RIS2a9j72XrkOB0+LEJWvU/iYe0u2hQdsPwwrF/+v911WqVwLxS4ua4+I37
+7n2sQ3YP1vgDUEsDBBQAAgAIAEqZLU8Xtjc46QAAAEsCAAALAAAAX3JlbHMvLnJl
+bHOtks1qwzAMgO97CqN7o7SFMUadXsqgtzKyB9Bs5YcklrG9LX37eYexBbrSw46W
+pU+fhHb7eRrVO4fYi9OwLkpQ7IzY3rUaXuqn1QOomMhZGsWxhjNH2Fd3u2ceKeWa
+2PU+qgxxUUOXkn9EjKbjiWIhnl3+aSRMlPIztOjJDNQybsryHsNvBlQLpjpaDeFo
+t6Dqs+db2NI0veGDmLeJXbrQAnlO7CzblQ+5PqQ+D6NqCi0nDVbMKYcjkvdFRgNe
+NtrcbvT3tDhxIkuJ0Ejg6z5fGdeE1v+5omXGj8084oeE4VVk+HbBxQ1Un1BLAwQU
+AAIACABKmS1PhCSxVukAAAC5AgAAGgAAAHhsL19yZWxzL3dvcmtib29rLnhtbC5y
+ZWxzrZLBasMwEETv/Qqx91p2WkopkXMpgVxb9wOEtLZMbEloN23991UbSBwIoQef
+xKzYmcdI6833OIhPTNQHr6AqShDoTbC97xR8NNv7ZxDE2ls9BI8KJiTY1HfrNxw0
+5x1yfSSRTTwpcMzxRUoyDkdNRYjo800b0qg5y9TJqM1edyhXZfkk09wD6gtPsbMK
+0s5WIJop4n+8Q9v2Bl+DOYzo+UqEJJ6GzC8anTpkBUddZB+Q1+NXS8Zz3sVz+p88
+DqtbDA+LVuB0QvvOKT/wvIn5+BbM45IwXyHtySHyGeQ0+kXNx6kZefHj6h9QSwME
+FAACAAgASpktT9fNBOqGAQAAOwMAABAAAABkb2NQcm9wcy9hcHAueG1snVPBbtsw
+DL3vKwzdGznBMBSBrGJIN/TQYgGStmdOpmOhtiSIrJHs6yc7sOu0PdWnx8eHpyeK
+VjfHtsk6jGS9K8RykYsMnfGldYdCPO5/X12LjBhcCY13WIgTkrjR39Q2+oCRLVKW
+HBwVomYOaynJ1NgCLVLbpU7lYwucyniQvqqswVtvXlt0LFd5/kPikdGVWF6FyVCc
+Hdcdf9W09KbPR0/7U0h+Wv0MobEGOF1SP1gTPfmKs19Hg42S86ZKRjs0r9HySedK
+zku1M9DgJhnrChpCJd8IdYfQz2wLNpJWHa87NOxjRvZfmtpKZH+BsI9TiA6iBcfi
+LDsXA24CcdTPPr5Qjcik5EQOcK6dY/tdLwdBApdCOQVJ+DLi3nKD9KfaQuRPEi/n
+iYcM4rOMHyKOh72z3/g2gDvNhr/xMfg4TF3Jsa0ewMEBe4MJ3Vv3Qo9h72+BcZz8
+Jal2NUQs02NNLzMR6i5dITa9flODO2A5aj42+j15Ov8Lerla5Okb1mPklHxbe/0f
+UEsDBBQAAgAIAEqZLU+9f3E6cgEAABEDAAARAAAAZG9jUHJvcHMvY29yZS54bWyl
+ksFOwzAQRO98ReQrSu2kEoIoDRIgTlRCogiuxl5a08S27C0hf4/jtGlLe+MUTWb8
+vDtyefvT1Mk3OK+MnpFswkgCWhip9HJGXheP6TVJPHIteW00zEgHntxWF6WwhTAO
+np2x4FCBTwJI+0LYGVkh2oJSL1bQcD8JCR3MT+MajkG6JbVcrPkSaM7YFW0AueTI
+aQ9M7UgkW6QUI9JuXB0BUlCooQGNnmaTjO6zCK7xZw9E5yDZKOwsnI3uzDH949UY
+bNt20k5jNMyf0ff500tcNVW6r0oAqUopCuGAo3HVq15r0+rkftAlPfD6HmvucR4a
+/1Qg77rT+Gmk3K45YEAmYbxiWGbnvE3vHxaPpMpZdpOymzSbLvK8yFiRs0s2LRjr
+xziC7KnN9qb/YXeU2AUqrCGsFr8yebHhTulXABjrGOw+KMELpyyG51hF6+hH0H7z
+8QUCB3MUoaM1dK1x0lexsb3qX2rYb2lcN1h/1NErrn4BUEsDBBQAAgAIAEqZLU9z
+kXtZswUAAKYbAAATAAAAeGwvdGhlbWUvdGhlbWUxLnhtbO1ZT2/bNhS/71MQurey
+bMl1gjpF7Njr1qYNErdDj7RES6wpUSDppL4N7XHAgGHdsMuA3XYYthVogV26T5Ot
+w9YB/Qp7+mOLiuk2aVNsQ+uDLZK/95/v8VG+fOVezNAhEZLypGs5FxsWIonPA5qE
+XevWaHihYyGpcBJgxhPSteZEWle2PriMN1VEYoKAPJGbuGtFSqWbti19mMbyIk9J
+AmsTLmKsYChCOxD4CNjGzG42Gm07xjSxUIJj4HpzMqE+QaOMpbW1YD5g8JUomU34
+TBz4uUSdIscGUyf7kXPZZwIdYta1QE7Aj0bknrIQw1LBQtdq5B/L3rpsL4mYWkOr
+0Q3zT0lXEgTTZk4nwvGS0Bm6G5d2lvybBf9V3GAw6A+cJb8cgH0fLHVWsO6w4/QW
+PDVQ8bjKu9/wGm4dr/FvreA3er2et1HDtyq8u4LvNNrudrOGdyu8t6p/b7vfb9fw
+XoVvr+CHlzbabh2fgyJGk+kKOovnMjJLyISzq0Z4B+CdxQaoULa2uwr6RK3bazG+
+y8UQAHlwsaIJUvOUTLAPuD6Ox4LiTADeJFhbKaZ8uTKVyULSFzRVXevjFENGVJAX
+T3988fQxevH00fH9J8f3fzl+8OD4/s8Gwqs4CXXC599/8fe3n6K/Hn/3/OFXZrzU
+8b//9Nlvv35pBiod+OzrR388efTsm8///OGhAb4t8FiHj2hMJLpBjtA+j8E2gwAy
+FmejGEWY1ihwBEgDcKCiGvDGHDMTrkfqzrstoACYgB/O7tZ0PYjETFED8FoU14C7
+nLMeF0ZzrmWydHNmSWgWLmY6bh/jQ5Ps/onQDmYp7GRqYtmPSE3NPQbRxiFJiELZ
+Gp8SYiC7Q2nNr7vUF1zyiUJ3KOphanTJiI6VmegqjSEuc2wOdc03u7dRjzMT+x1y
+WEdCQmBmYklYzY0f4pnCsVFjHDMdeR2ryKTkwVz4NYdLBZEOCeNoEBApTTQ3xbym
+7jUMlcgY9l02j+tIoejUhLyOOdeRO3zaj3CcGnWmSaRjP5JT2KIY7XFlVILXMyQb
+QxxwsjbctylRZ0vrWzSMzBskW5kJU0oQXs/HOZtgkpT1vVapY5q8rGwzCnX7fdle
+wLfhEGOnKNbrcP/DEr2DZ8kegax4X6HfV+h3sUKvy+Xzr8tVKbb1XjtnE69tvCeU
+sQM1Z+S6zIu4BPOCIUzmg5xo2eenETyW4mq4UOD8GQmuPqEqOohwCmKcXEIoS9ah
+RCmXcLuw1vLOr6gUbM7nvMW9EtBY7fKgmG7p980lm3wUSl1QK2NwWmGtS28mzCmA
+p5TmeGZp3kul2Zo3IW8Qzl4mOO1mIRo2CmYkyPxeMFiE5dxDJCMckDJGjtEQp3VK
+t3Ve7TVN2kbrzaSdJki6OHeNOO8cotRYiZK9mo4sqY/QEWjlNT0L+TjtWhPoueAx
+ToGfzEoVZmHStXxVmvLKZD5psHlbOo21BtdEpEKqHSyjgipfWryOSSr9m56b+eF8
+DLBfV4tWx/kXtbBPhpZMJsRXa2aqYbnGZ4qIgyg4QmM2E/sY9HaL3RVQCUdFczEQ
+kKFuufHqmV9mwcnXPmV2YJZGuKxJHS32BTx/XuqQjzT17DW6v6YprXM0xXt3Tcl2
+LjS4rSC/ekEbIDDK9mjX4kJFHKpQGlF/KKBxyGWBXgjSIlMJsewddqYrOazqVsGj
+KHJhpPZpiASFSqciQcieKu18BTOnqZ+vC0ZlnVmqK9Pid0wOCRtl2dvO7LdQtKgm
+pSNy3Mmg2absGofD/3Dn4zZepz2oBLln6UVcrehrR8HGm6lwxqO2aba46Z36qE3h
+moKyLyjcVPis6m9HfB+ij5YdJYKNeKFTpt9ycgw6dzTjMlZvt42qQtBpvP3mU3N2
+a42zG42342zP4Gvv5a62V1PU1i4y+Wjlzyw+vguyd+B+NGNKFu+d7sGltL/4GwL4
+2BXp1j9QSwMEFAACAAgASpktT3gfEeh/AAAAkAAAABQAAAB4bC9zaGFyZWRTdHJp
+bmdzLnhtbA3LQQ7CIBBA0b2nILO3oAtjTGkXJp5AD0DKWEjKQJnB6O1l+fPyx/mb
+NvXByjGThdNgQCEt2UdaLbyej+MVFIsj77ZMaOGHDPN0GJlF9ZXYQhApN615CZgc
+D7kgdXnnmpz0rKvmUtF5DoiSNn025qKTiwSqUdwb3nMjsWBAT39QSwMEFAACAAgA
+SpktT+JfnN75AQAAaAQAAA0AAAB4bC9zdHlsZXMueG1shZTLbp0wEIb3fQrL+4bD
+kVK1ERA1kZC6aFQ1VOrWwABWfEH2cAp5+tqYw0WJFDbM/B4+Dz8ekvtRCnIBY7lW
+KY1vTpSAqnTNVZvSP0X++SslFpmqmdAKUjqBpffZp8TiJOC5A0DiCHe2Z5Vb7Q1Y
+MBegXlQ2pR1ifxdFtupAMnuje1BupdFGMnSpaSPrnmG19SQpovPp9CWSjCuaJWqQ
+uURLKj0oTOmJRlnSaLUpMQ1ClpTkwsRSwnexRcNfYCcMIVbuXeb115DHsc8qLbQh
+pi1Tmuen+fKyYnJBPDLBS8O9GIWN55t1bXAh1r7ONAhZ0jNEMCp3CVniYuphayAK
+dR9Ut4ZN8fnWY9vHfZPh8qDyuLB1H+2o637zzbVdalO7b783NEj+wWXR+QJCPPvv
+/bc5lI4NCd/oR+3tJd6La+j4SxgwIfHYPS2w32LH5i3yuNX7fML6Xky5DuclZE+D
+LMHk85Hb1TjAlj3MjC3/LnirJARMlrBrSjpt+KvD+9PQggLDBPXjg7zyUqkRtaQE
+YcTfGhnOU+Wo/wzrCyemtGHCuvmwneHqpdA5XzVvztgsDq3mzFYd/FlV4k9mSp/8
+q7k2VtPKgQvk6h3LHbMem+NIISvdJIddamjYILBYpZRu8U+o+SC/0WvVL37RuFTN
+8VY6j1O0/SOy/1BLAwQUAAIACABKmS1PXO7tLqwBAAD1AgAADwAAAHhsL3dvcmti
+b29rLnhtbI2SS08bQQyA7/0Vo7mXXSpAbZQNUqFpkfpAJaXnyayXtZiXxt4E+PX1
+TJKWcmIvHr/tzzs/f/BObSATxtDp46NWKwg29hjuOv1rtXz7XitiE3rjYoBOPwLp
+88Wb+Tbm+3WM90ryZ5SMFV/KQJA3oIsxUKdH5jRrGrIjeENHMUEQzxCzNyxqvmtI
+ckxPIwB717xr27PGGwz7CrP8mhpxGNDCZbSTh8C7IhmcYVmJRkykF/MBHdzutlQm
+pe/Gy7wPTitniD/1yNB3+kTUuIX/DHlKHyd0RTltT3Wz+Lv5dVbCCXalViPS772j
+BBV5i7Clf/FFVcYybmBl1p0W0mbiuETHkC8Nw+ccp1S5c54E4oCZ+KagqcEeA3p8
+KnMNxpEE0Bi3X2LGpxjYuBubo3OH5OKrudKLnhuFAqN9Gc5m/bMA6/RZK702SLhG
+h/zY6fp2ULZqnq1Vb3aQKlQKhUDVSysRVzLrsTCcoTzyVX9SixwyexgwQF8Akthl
+JluYiih5H+qnq/5NOHe60NrpF9EnB/VEUn6YnLsQ64/wNZq+spLfw8Jyby+W0vdw
+iMUfUEsDBBQAAgAIAEqZLU/qp5H3KQMAAHcHAAAYAAAAeGwvd29ya3NoZWV0cy9z
+aGVldDEueG1spVVNc9s2EL33V3B4jyg5dptoJGXcpKoz40wyttOcIWApogKxLLCU
+rP76LgCSoqwePJODJOwD9u3XA7T48FybbA/Oa7TLfDaZ5hlYiUrb7TL//rR+8y7P
+PAmrhEELy/wIPv+w+mVxQLfzFQBlTDD3jZC82Tjw4PaQB9D6ZV4RNfOi8LKCWvgJ
+NmB5p0RXC2LTbQvPPkJFptoUV9Ppr0UttO0Y5u41HFiWWsInlG0NlhKJAyOIa/KV
+bnzP9qxexaecOHD9fT6jFD+lnYFvdn3BV2vp0GNJE4l1l9plle+L92d11vI1idXC
+7drmDRM3XNxGG03HmGOe1XL+eWvRiY3hQXBiQo6yFPIn8mSm1SJi39xqgS0ZbeGb
+y3xbc0LH38HggaWT98CD3lYUgGK1KAY/pXk4QWWZg3KZ387mt/FEPPCXhoMfrTMS
+m0cwIAlUpA5y2yDuwuZnhlimvsLDn06re86GpUauhQQ+4OEjmjsuhGUdvJk4coXg
+gn/28BGMCTmwwz9dOqdsQ4zxuk9sHSfBhSsoRWuI49xBV+v15DrPutbcwx7MQ+jJ
+9BzjrALGgSQaH7+zWttYYC2eU6FaUbXMryZX725mNzzYDXhaa+oLlK0nrH+kU13N
+dAwzj8RFYk5iFSRWC4eHzKXxNCLcydk8dEQG8Das9qvZotgHV/7w6b705N7ND6nr
+X7SXeSmM59C4+ZtxP9heghVO4wlJ8g39vsDQtLV9iXLfTpC2/JxcHEzo/xy8Ozbg
+uNu704biwRO8ZEjoGUNSyD3KHajzbD26U8GiJVxrQ+AGqNF7pKdw716yfbfmko+H
+1Dht6WsTn6dse5Jw5zwgj9CPPTiJLXwRbsulZgZK3plOfsszlwQY14RNXN2wapBY
+Jr1VxbsQrLehz0i90fFyoLbJGsHte9T/QlQLOs2PaXxDl3mn+DBgEbQ2m7K0S01P
+ONyAzu6UGa9TCruO8TKlyxIcU35V6o892NMw+o21dp5GUuJIPzRV/K73ShdGb23A
+ukZ0OD9KSqX7zuIdrXmZoie4X/NdtMPxsRHWg8PYKENqg8uZFY3B6dwaN4DN4T9z
+9R9QSwMEFAACAAgASpktT81LUiJ4AAAAjQAAACMAAAB4bC93b3Jrc2hlZXRzL19y
+ZWxzL3NoZWV0MS54bWwucmVsc02MMQ4CIRAAe19BtvdAC2PMcdf5AKMP2HArEGEh
+LDH6eyktJ5OZef3kpN7UJBa2cJgMKGJXtsjewuN+3Z9BSUfeMBUmC18SWJfdfKOE
+fTQSYhU1JiwWQu/1orW4QBllKpV4mGdpGfvA5nVF90JP+mjMSbf/B+jlB1BLAQI/
+AxQAAgAIAEqZLU9HkkSyWAEAAPAEAAATAAAAAAAAAAAAAAC2gQAAAABbQ29udGVu
+dF9UeXBlc10ueG1sUEsBAj8DFAACAAgASpktTxe2NzjpAAAASwIAAAsAAAAAAAAA
+AAAAALaBiQEAAF9yZWxzLy5yZWxzUEsBAj8DFAACAAgASpktT4QksVbpAAAAuQIA
+ABoAAAAAAAAAAAAAALaBmwIAAHhsL19yZWxzL3dvcmtib29rLnhtbC5yZWxzUEsB
+Aj8DFAACAAgASpktT9fNBOqGAQAAOwMAABAAAAAAAAAAAAAAALaBvAMAAGRvY1By
+b3BzL2FwcC54bWxQSwECPwMUAAIACABKmS1PvX9xOnIBAAARAwAAEQAAAAAAAAAA
+AAAAtoFwBQAAZG9jUHJvcHMvY29yZS54bWxQSwECPwMUAAIACABKmS1Pc5F7WbMF
+AACmGwAAEwAAAAAAAAAAAAAAtoERBwAAeGwvdGhlbWUvdGhlbWUxLnhtbFBLAQI/
+AxQAAgAIAEqZLU94HxHofwAAAJAAAAAUAAAAAAAAAAAAAAC2gfUMAAB4bC9zaGFy
+ZWRTdHJpbmdzLnhtbFBLAQI/AxQAAgAIAEqZLU/iX5ze+QEAAGgEAAANAAAAAAAA
+AAAAAAC2gaYNAAB4bC9zdHlsZXMueG1sUEsBAj8DFAACAAgASpktT1zu7S6sAQAA
+9QIAAA8AAAAAAAAAAAAAALaByg8AAHhsL3dvcmtib29rLnhtbFBLAQI/AxQAAgAI
+AEqZLU/qp5H3KQMAAHcHAAAYAAAAAAAAAAAAAAC2gaMRAAB4bC93b3Jrc2hlZXRz
+L3NoZWV0MS54bWxQSwECPwMUAAIACABKmS1PzUtSIngAAACNAAAAIwAAAAAAAAAA
+AAAAtoECFQAAeGwvd29ya3NoZWV0cy9fcmVscy9zaGVldDEueG1sLnJlbHNQSwUG
+AAAAAAsACwDRAgAAuxUAAAAA
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: pptx
+
+UEsDBBQAAggAAAAAAAAAAAAAAAAAAAAAAAAGAA8AX3JlbHMvdXALAAE3iv4tX3Jl
+bHMvUEsDBBQAAAAIAAAAAABJT3tuuwAAADEBAAALABQAX3JlbHMvLnJlbHN1cBAA
+AXU1KDlfcmVscy8ucmVsc43POw7CMAwG4B2JO0TeaVoGhFCTLgipKyoHiFL3Idok
+igOiZ2PgSFyBiIkiBkbbvz/Lz/sjL27jwK7oqbdGQJakwNBoW/emFXCqDqstMArK
+1GqwBgVMSFDI5SI/4qBCXKKud8SiYkhAF4LbcU66w1FRYh2aOGmsH1WIpW+5U/qs
+WuTrNN1w/2mAnJmsrAX4ss6AVZPDf2zbNL3GvdWXEU34ceIrEWXlWwwCnAvceaTY
+fKeTKAOXOZ99KV9QSwMEFAACCAAAAAAAAAAAAAAAAAAAAAAAAAQADQBwcHQvdXAJ
+AAF7lgj+cHB0L1BLAwQUAAAACABPmS1PKjgJcBIBAAAGAgAAFAAdAHBwdC9wcmVz
+ZW50YXRpb24ueG1sdXAZAAFoJ+GNcHB0L3ByZXNlbnRhdGlvbi54bWyNkU1OwzAQ
+hfdI3MGaPbWTJmkSxekGISHBCjiAlThNpPhHHgMtV2PBkbgCDqlEikDqzvPzPb3x
++3z/qLZ7NZIX6XAwmkO0YkCkbkw76B2Hp8ebqxwIeqFbMRotORwkwra+vKhsaZ1E
+qb3wASVBRmNpOfTe25JSbHqpBK6MlTrMOuOU8KF0O7rk1EhjxjKqxKDhKCLOEWmd
+eA0e/+LdObzpuqGR16Z5VsHLLOLk+G0K+8Ei1OFEHNt7gV662/YO/a8OGVoOcZRs
+knydJeGfXDl1wiQCWlf0H/z0PYuk2YJOf+jl7sMbafYcNmmaFkUMpDmEvFhWxOs8
+mQhbauMlHteyPM0ZY/NaESXJVNApOHqaXP0FUEsDBBQAAggAAAAAAAAAAAAAAAAA
+AAAAAAAKABMAcHB0L19yZWxzL3VwDwABsNOzaHBwdC9fcmVscy9QSwMEFAAAAAgA
+AAAAAHaaIb4AAQAATAMAAB8AKABwcHQvX3JlbHMvcHJlc2VudGF0aW9uLnhtbC5y
+ZWxzdXAkAAGj10+IcHB0L19yZWxzL3ByZXNlbnRhdGlvbi54bWwucmVsc62TzUrE
+MBDH74LvEOZu064fiGy6l2VhD4Lo+gAxnbbBNAmZKPbZPPhIvoLBj7WVpXjocf7J
+/PIjmby/vi1XL51hzxhIOyugyHJgaJWrtG0E3O82J5fAKEpbSeMsCuiRYFUeHy1v
+0ciYmqjVnliiWBLQxuivOCfVYicpcx5tWqld6GRMZWi4l+pRNsgXeX7Bw5AB5YjJ
+tpWAsK0KYLve43/Yrq61wrVTTx3aeOAITkZXeC0pYkhYGRqMAgbhaEeW8MAPWy3m
+tIqpF399PsuvcErhdE4FH5BugkuvsNfYR1MSZ7Peg3wweBd7gwONQTglcj77mPwZ
+kO+0+JHgow9QfgBQSwMEFAACCAAAAAAAAAAAAAAAAAAAAAAAABEAGgBwcHQvc2xp
+ZGVMYXlvdXRzL3VwFgABga2MbnBwdC9zbGlkZUxheW91dHMvUEsDBBQAAAAIAE+Z
+LU/tqrDnAgEAANoBAAAgACkAcHB0L3NsaWRlTGF5b3V0cy9zbGlkZUxheW91dC54
+bWx1cCUAATo408hwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0LnhtbI1RS07D
+MBDdI3GHyHvqwAKhqGkXINjwqdRygJE9SSz5p7Eb6NlYcCSuwDSJCEIsuvJ45n3m
+8/XxuVy/O1v0SMkEX4vLRSkK9Cpo49tavO7uL25EkTJ4DTZ4rMUBk1ivzs+WsUpW
+P8Ih7HPBGj5VsRZdzrGSMqkOHaRFiOi51gRykPlLrYyECX2GzH7OyquyvJYOjBeT
+CJwiogneuMH/+HQKPzSNUXgX1N5xL6MIoR2aSp2JSax4PrW1+vimuCPEY+T7B4rb
+uKGh/NxvqDCatyYKD46XI+RUmGByJA2B/ENvfyBytpCzq7L0BPGlZzBUPEdGuh1S
+kScfWTOE7yF/HWT1DVBLAwQUAAIIAAAAAAAAAAAAAAAAAAAAAAAAFwAgAHBwdC9z
+bGlkZUxheW91dHMvX3JlbHMvdXAcAAHE+WOVcHB0L3NsaWRlTGF5b3V0cy9fcmVs
+cy9QSwMEFAAAAAgAAAAAAKAhdRjAAAAAOQEAACsANABwcHQvc2xpZGVMYXlvdXRz
+L19yZWxzL3NsaWRlTGF5b3V0LnhtbC5yZWxzdXAwAAE+lGIYcHB0L3NsaWRlTGF5
+b3V0cy9fcmVscy9zbGlkZUxheW91dC54bWwucmVsc43PO2rEMBAG4D6QO4jpI9kp
+whIsu1kCLrYJ3gMM0tgWa0tCoyzx2VLkSLlCVNqQIuW8vp/5+fpuus91EXdK7ILX
+UMsKBHkTrPOThuvw9nQCwRm9xSV40rARQ9c+PjTvtGAuRzy7yKIonjXMOcdXpdjM
+tCLLEMmXyRjSirmUaVIRzQ0nUs9V9aLS3oD2YIreaki9rUEMW6T/2GEcnaFzMB8r
++fxHhOLFWbogZ0qFxTRR1iDlvn9YkiUBVNuow7ftL1BLAwQUAAIIAAAAAAAAAAAA
+AAAAAAAAAAAAEQAaAHBwdC9zbGlkZU1hc3RlcnMvdXAWAAHHAxlAcHB0L3NsaWRl
+TWFzdGVycy9QSwMEFAAAAAgAT5ktT/dPgcpeAQAAvAIAACAAKQBwcHQvc2xpZGVN
+YXN0ZXJzL3NsaWRlTWFzdGVyLnhtbHVwJQABziVRynBwdC9zbGlkZU1hc3RlcnMv
+c2xpZGVNYXN0ZXIueG1sjZLLTgIxFIb3Jr5D072UGYYRCQMbo5KAIQEfoEw7l9Bp
+m7YgPJsLH8lX8MyloMYFq/79zrV/+vXxOZkdK4EO3NhSyQQHvT5GXKaKlTJP8Nvm
+6W6EkXVUMiqU5Ak+cYtn09ubiR5bwZbUOm4Q9JB2rBNcOKfHhNi04BW1PaW5hFim
+TEUdXE1OtOGWS0cdzKsECfv9mFS0lLhrQq9pwgx9hwX/qzfX1KssK1P+qNJ9Bbu0
+TQwXzVK2KLXFU3hfuhasPq3eGM5rJQ/PRq/1yjTh18PKoJKBaxhJWoE5mHSBLo20
+RY0gf8rzcwq5jCCXqakwS6rRNg8SLByMcEdQbAdqm4c1C2sW1gwUTVN4CmR0wpPQ
+k3POwJOBJ5EnkSdDT4aexJ7EGBWilDvwuT4wypR4aYFXrQ3wPRb0pPZuzhbW/SGN
+b2EQ3UejQRw9YGTGNTFzFuDOkt/l8OPIjy83/QZQSwMEFAACCAAAAAAAAAAAAAAA
+AAAAAAAAABcAIABwcHQvc2xpZGVNYXN0ZXJzL19yZWxzL3VwHAABPks4O3BwdC9z
+bGlkZU1hc3RlcnMvX3JlbHMvUEsDBBQAAAAIAAAAAABECgwe1AAAAL8BAAArADQA
+cHB0L3NsaWRlTWFzdGVycy9fcmVscy9zbGlkZU1hc3Rlci54bWwucmVsc3VwMAAB
+iaIT0XBwdC9zbGlkZU1hc3RlcnMvX3JlbHMvc2xpZGVNYXN0ZXIueG1sLnJlbHOt
+kM9KAzEQxu+C7xDmbrLtQUSa7UWEQk9SH2BIZndDd5OQmRb32Tz4SL6CQRG60IMH
+LwPz5/t9H/P5/rHZvk2jOlPhkKKFlW5AUXTJh9hbeD083z2AYsHocUyRLMzEsG1v
+bzYvNKJUEQ8hs6qUyBYGkfxoDLuBJmSdMsW66VKZUGpbepPRHbEns26ae1MuGdAu
+mGrnLZSdX4E6zJn+wk5dFxw9JXeaKMoVC8Nj8LTHOZ2kYrH0JBa0vpwvjnR1AHM9
+2Po/g0nV0iLS9+Sn/qYwi5+3X1BLAwQUAAIIAAAAAAAAAAAAAAAAAAAAAAAACgAT
+AHBwdC90aGVtZS91cA8AAYZ20jJwcHQvdGhlbWUvUEsDBBQAAAAIAAAAAABp2ran
+rwUAAGIbAAATABwAcHB0L3RoZW1lL3RoZW1lLnhtbHVwGAAB5ja48HBwdC90aGVt
+ZS90aGVtZS54bWztWUuP00Ycv1fqdxj5Do4TJ2RXZNEmm0CBhdVuoOI4sSf2kLHH
+mpnsklsFx0qVqtKql0q99VC1RQKpF/pptqVqqcRX6N+PxOPNpJuFRUWCHJJ5/P7v
+h2ecy1ceRAwdEiEpjzuWc7FmIRJ73Kdx0LHuDAcX2haSCsc+ZjwmHWtGpHVl6+OP
+LuNNFZKIIKCP5SbuWKFSyaZtSw+WsbzIExLD3piLCCuYisD2BT4CvhGz67Vay44w
+jS0U4wjY3h6PqUfQMGVpbc2Z9xl8xUqmCx4TB14mUafIsP7ESX/kTPaYQIeYdSyQ
+4/OjIXmgLMSwVLDRsWrZx7K3LtsLIqZW0Gp0g+xT0BUE/qSe0YlgtCB0Bu7GpZ0F
+/3rOfxnX7/d7fWfBLwNgzwNLnSWsO2g73TlPDZQPl3n3as2aW8Vr/BtL+I1ut9vc
+qOAbJd5dwrdrLXe7XsG7Jb65rH93u9drVfDNEt9awg8ubbTcKj4DhYzGkyV0Gs9F
+ZBaQMWfXjPA2wNvzBChRtpZdOX2sVuVahO9zMQBAFlysaIzULCFj7AGuhxkdCZoK
+wJsEazv5kieXllJZSHqCJqpjXU8wVEQJefX8p1fPn6JXz58cP3x2/PDX40ePjh/+
+YiC8huNAJ3z5w5f/fPcZ+vvp9y8ff23GSx3/x8+f//7bV2ag0oEvvnny57MnL779
+4q8fHxvg2wKPdPiQRkSiW+QI7fMIbDMIICNxNophiKlOsR0HEsc4pTGg+yqsoG/N
+MMMGXJdUPXhXQBcwAa9O71cUPgjFVFED8EYYVYC7nLMuF0abbqSydC9M48AsXEx1
+3D7GhybZvRPx7U8TSGdqYtkLSUXNPQYhxwGJiULpHp8QYiC7R2nFr7vUE1zysUL3
+KOpianTJkI6UmegajSAuM2yOd8U3u3dRlzMT+x1yWEVCVWBmYklYxY1X8VThyKgx
+jpiOvIlVaFLyYCa8isOlgkgHhHHU94mUJprbYlZR9wZ0D3PYd9ksqiKFohMT8ibm
+XEfu8EkvxFFi1JnGoY79RE4gRTHa48qoBK9WSDqHOOB4ZbjvUqLOVtt3aBCaEyTd
+mYqidVeacETjDx157Y68LaixJE724VW4k923x4VP3/3mu4On8R6BfP/Qez/03vex
+966q53U7btlkbf2AnPGLVp6Wx5SxAzVj5KbM2rMEpf0BLGaTjGhxOE9CGBbiKrhA
+4GyMBFefUhUehDgBMU4mIZAF60CihEu4ElgreWf3SgrGZ2vN+WUQ0Fjtcj9fbuiX
+xAWbbBZIXVAjZbCusMalNxPm5MA1pTlNs7Tmf0qzNW9CNSCcvgJwWvVcNGQMZsRP
+/Z4zmIfl3EMkQ+yTIkaO0RCnsabb2qd7TZO20XgzaesESRfnrhDXPIco1ZaiZC+X
+I4urM3QEWjXrTQt5OOlYYzhNwTBKgJ9MGxBmQdyxPFWYcmoxnzTYnJZObaXBFRGJ
+kGoHyzCnyrbm71DiUv960039cD4G2K+rRaPt/I9a2CdDS8Zj4qkVK+W02ONTRcRB
+6B+hEZuKfQx6u3l2+VTCM6M+nwioULdIvGrlF1Vw8l1NUR2YJSEuelJbi30Oz8YL
+HbKZpp69QvfXNKVxjqY0319T0syFY2vDzy5VcAwQGKU52rG4UCGHLpSE1BsIODhk
+skAvBGWRqoRY+uY51ZUcln0r55E3uSBU+zRAgkKnU6EgZE8Vdp7CzKnrz9c5o6LP
+LNSVSf47IoeEDdPqbaX2Wyicd5PCERnuZNBsU3WNgsE7fPJxa69zPCgFuWc5i7ha
+09ceBRtvpsIZH7V1s8X15tqP2gQuHyj9gsZNhcfK8+2Q70P0EVsohSATL7SK+itX
+R4Uac3kps7d7kCqD0K69/eOn5u7GCnfXam/J3fNh1d3OKe62lwvV1q4z2Wzpfyg+
+ug/yd+C6NGVK5u+VHsCFszf/BwH42CXp1r9QSwMEFAAAAAgAT5ktTz2SW/mvAAAA
+NgEAABEAGgBwcHQvcHJlc1Byb3BzLnhtbHVwFgABDT1E8XBwdC9wcmVzUHJvcHMu
+eG1sjY85DsIwEEV7JO4QTU8cKBCKsjSImgIOYDmTxFI8tjxmOxsFR+IKmKVAiCLl
+aPSe3r9fb0V9NkNyRM/aUgnzNIMESdlGU1fCfreZrSDhIKmRgyUs4YIMdTWdFC53
+HhkpyBDRrU+iiDh3JfQhuFwIVj0ayal1SPHXWm9kiKfvxDdpBrHIsqUwUhN8JHKM
+pPHyFCv/8X4Mb9tWK1xbdTCx5S3xOLyiuNeO4TlT/O6sHlBLAwQUAAAACABPmS1P
+dOsUorIAAADKAAAAEwAcAHBwdC90YWJsZVN0eWxlcy54bWx1cBgAAa5yN3JwcHQv
+dGFibGVTdHlsZXMueG1sVY1LDoIwFAD3Jt6heftawKJIKEQRV+7UA1Qon6QfQxuV
+GE/mwiN5BXHpcjLJzOf1TrK7kugqetsZzcCfeYCELk3V6YbB6bjDESDruK64NFow
+GISFLJ1OEh67szy4QYq9dWisaBtzBq1zl5gQW7ZCcTszF6FHV5tecTdi35Cq57ex
+riQJPG9BFO80oErUDB5hHgQhpWu8LIoFpnMa4I1HIxyFm22+2m39fL5+wu9O/vbp
+F1BLAwQUAAIIAAAAAAAAAAAAAAAAAAAAAAAACwAUAHBwdC9zbGlkZXMvdXAQAAH8
+6vqNcHB0L3NsaWRlcy9QSwMEFAAAAAgAT5ktT5ABerdEAgAAuQQAABUAHgBwcHQv
+c2xpZGVzL3NsaWRlMS54bWx1cBoAAUrF3JxwcHQvc2xpZGVzL3NsaWRlMS54bWyN
+VEtu2zAQ3RfoHQjuE/1sNzUsBW0KpwXaNKidA1ASJROhSIJkFLlXy6JH6hU6JMWk
+SLKIF+LTm5k3Hw399+HP5nwaOBqpNkyKEmenKUZUNLJloi/xzX57coaRsUS0hEtB
+S3ykBp9X799t1NrwFkG0MGtV4oO1ap0kpjnQgZhTqagAWyf1QCy86j5RmhoqLLGQ
+aeBJnqarZCBM4FmEvEWk1eQeSnstXr8lXnYda+gX2dwNUEsQ0ZT7osyBKYMr6KzZ
+8daddR+e17rakLWRnLVbxrl/0X19wTUaCS/x1v9wUm2SZ26062hjvxvrbFHKg3mG
+aq8pdVnEeKnVTjk7FHA1XmvEWvgiGAkywOCdujfMbkkI8iB5Ft5HlzlHcH0pnr8Q
+f1U5BhoVBjF1enAnDBNNJf6QfSxyWJMjwGKxXJ05NWh9sqhx5iJbQqIGzFmRpavF
+PKeoorSxl1QOyIESa5gXdjwZ57E9ufh6zFyNnT7L9ug8azihI/5NmBLDAtsIdAR1
+BPeawLIKWGWfQ8hPd1Z2bM4ThHxR/uHGJFpYFBcbKgnm8HA5ibsnVJzc7OCe/IYW
+02Xqpd1OCWSPinakgRHv2UANuqL36JcciJiHoIOerTL3ZgMXMvlm5ybhO/rOH0HY
+muRpVRuufxD1c/RyEi60Zi298KSCC4PqPisxt7BOdgLU3gKq+9xxueNyxwEiTQP9
+gscMIpNH5tGniEwRmUVkFpFZRmYZmVVkVhgdOBO3cGvdgVEn+ddARITDAv7XWuL/
+dap/UEsDBBQAAggAAAAAAAAAAAAAAAAAAAAAAAARABoAcHB0L3NsaWRlcy9fcmVs
+cy91cBYAATvy/C1wcHQvc2xpZGVzL19yZWxzL1BLAwQUAAAACAAAAAAAjeKkdsEA
+AAA5AQAAIAApAHBwdC9zbGlkZXMvX3JlbHMvc2xpZGUxLnhtbC5yZWxzdXAlAAFU
+mY+vcHB0L3NsaWRlcy9fcmVscy9zbGlkZTEueG1sLnJlbHONz82KwjAQB/C74DuE
+uZvUPSwiTXtZFgRPog8wJNM22CYhE2X7bB72kfYVNkcLHjzO1+/P/D1+6/ZnGsWd
+ErvgNWxlBYK8Cdb5XsPl/L3ZgeCM3uIYPGmYiaFt1qv6RCPmcsSDiyyK4lnDkHPc
+K8VmoAlZhki+TLqQJsylTL2KaK7Yk/qoqk+Vng1oFqY4WA3pYLcgznOkd+zQdc7Q
+VzC3iXx+EaF4dJaOOIdbLiymnrIGKZ/7iyVZEkA1tVp82/wDUEsDBBQAAAAIAAAA
+AAD9t2d/XwEAAD4FAAATABwAW0NvbnRlbnRfVHlwZXNdLnhtbHVwGAABnyAXNFtD
+b250ZW50X1R5cGVzXS54bWy1lEtuwjAQhveVeofI24oYuqiqisCij1UfSKUHcJNJ
+cOuXPAaRs3XRI/UKnSQgBRQKUmETaR7/P5+dSX6+vofjpVbRAjxKaxI2iPssApPa
+TJoiYW/Th941izAIkwllDSSsBGTj0fnZcFo6wIjUBhM2C8HdcI7pDLTA2DowVMmt
+1yJQ6AvuRPopCuCX/f4VT60JYEIvVB5sNLyDXMxViO6XlG5IPChk0W3TWM1KmHBO
+yVQEqvOFybam9FYTYlLWPTiTDi+ogfHOCVVl94Ddug8HxZZQ6upodaFb40y3pMqT
+4oVegJcZRBPhw7PQVOfOBe48IClqovhv3o4LsXkuU8hsOtckidtmWm2EsRbSrK9q
+FwwqSj4JDLQs7eDYYC3rg5AeRWnnAdvBSZAa631Igb4AaJ7/pqhd9g2sUCfeOjzF
+gtTGe48s3hW8hlLB0Rla1gftwmoLBidZgDUBr399o19QSwECFAAUAAIIAAAAAAAA
+AAAAAAAAAAAAAAAABgAPAAAAAAAAACAAAAAAAAAAX3JlbHMvdXALAAE3iv4tX3Jl
+bHMvUEsBAhQAFAAAAAgAAAAAAElPe267AAAAMQEAAAsAFAAAAAAAAAAgAAAAMwAA
+AF9yZWxzLy5yZWxzdXAQAAF1NSg5X3JlbHMvLnJlbHNQSwECFAAUAAIIAAAAAAAA
+AAAAAAAAAAAAAAAABAANAAAAAAAAACAAAAArAQAAcHB0L3VwCQABe5YI/nBwdC9Q
+SwECFAAUAAAACABPmS1PKjgJcBIBAAAGAgAAFAAdAAAAAAAAACAAAABaAQAAcHB0
+L3ByZXNlbnRhdGlvbi54bWx1cBkAAWgn4Y1wcHQvcHJlc2VudGF0aW9uLnhtbFBL
+AQIUABQAAggAAAAAAAAAAAAAAAAAAAAAAAAKABMAAAAAAAAAIAAAALsCAABwcHQv
+X3JlbHMvdXAPAAGw07NocHB0L19yZWxzL1BLAQIUABQAAAAIAAAAAAB2miG+AAEA
+AEwDAAAfACgAAAAAAAAAIAAAAPYCAABwcHQvX3JlbHMvcHJlc2VudGF0aW9uLnht
+bC5yZWxzdXAkAAGj10+IcHB0L19yZWxzL3ByZXNlbnRhdGlvbi54bWwucmVsc1BL
+AQIUABQAAggAAAAAAAAAAAAAAAAAAAAAAAARABoAAAAAAAAAIAAAAFsEAABwcHQv
+c2xpZGVMYXlvdXRzL3VwFgABga2MbnBwdC9zbGlkZUxheW91dHMvUEsBAhQAFAAA
+AAgAT5ktT+2qsOcCAQAA2gEAACAAKQAAAAAAAAAgAAAApAQAAHBwdC9zbGlkZUxh
+eW91dHMvc2xpZGVMYXlvdXQueG1sdXAlAAE6ONPIcHB0L3NsaWRlTGF5b3V0cy9z
+bGlkZUxheW91dC54bWxQSwECFAAUAAIIAAAAAAAAAAAAAAAAAAAAAAAAFwAgAAAA
+AAAAACAAAAANBgAAcHB0L3NsaWRlTGF5b3V0cy9fcmVscy91cBwAAcT5Y5VwcHQv
+c2xpZGVMYXlvdXRzL19yZWxzL1BLAQIUABQAAAAIAAAAAACgIXUYwAAAADkBAAAr
+ADQAAAAAAAAAIAAAAGIGAABwcHQvc2xpZGVMYXlvdXRzL19yZWxzL3NsaWRlTGF5
+b3V0LnhtbC5yZWxzdXAwAAE+lGIYcHB0L3NsaWRlTGF5b3V0cy9fcmVscy9zbGlk
+ZUxheW91dC54bWwucmVsc1BLAQIUABQAAggAAAAAAAAAAAAAAAAAAAAAAAARABoA
+AAAAAAAAIAAAAJ8HAABwcHQvc2xpZGVNYXN0ZXJzL3VwFgABxwMZQHBwdC9zbGlk
+ZU1hc3RlcnMvUEsBAhQAFAAAAAgAT5ktT/dPgcpeAQAAvAIAACAAKQAAAAAAAAAg
+AAAA6AcAAHBwdC9zbGlkZU1hc3RlcnMvc2xpZGVNYXN0ZXIueG1sdXAlAAHOJVHK
+cHB0L3NsaWRlTWFzdGVycy9zbGlkZU1hc3Rlci54bWxQSwECFAAUAAIIAAAAAAAA
+AAAAAAAAAAAAAAAAFwAgAAAAAAAAACAAAACtCQAAcHB0L3NsaWRlTWFzdGVycy9f
+cmVscy91cBwAAT5LODtwcHQvc2xpZGVNYXN0ZXJzL19yZWxzL1BLAQIUABQAAAAI
+AAAAAABECgwe1AAAAL8BAAArADQAAAAAAAAAIAAAAAIKAABwcHQvc2xpZGVNYXN0
+ZXJzL19yZWxzL3NsaWRlTWFzdGVyLnhtbC5yZWxzdXAwAAGJohPRcHB0L3NsaWRl
+TWFzdGVycy9fcmVscy9zbGlkZU1hc3Rlci54bWwucmVsc1BLAQIUABQAAggAAAAA
+AAAAAAAAAAAAAAAAAAAKABMAAAAAAAAAIAAAAFMLAABwcHQvdGhlbWUvdXAPAAGG
+dtIycHB0L3RoZW1lL1BLAQIUABQAAAAIAAAAAABp2ranrwUAAGIbAAATABwAAAAA
+AAAAIAAAAI4LAABwcHQvdGhlbWUvdGhlbWUueG1sdXAYAAHmNrjwcHB0L3RoZW1l
+L3RoZW1lLnhtbFBLAQIUABQAAAAIAE+ZLU89klv5rwAAADYBAAARABoAAAAAAAAA
+IAAAAIoRAABwcHQvcHJlc1Byb3BzLnhtbHVwFgABDT1E8XBwdC9wcmVzUHJvcHMu
+eG1sUEsBAhQAFAAAAAgAT5ktT3TrFKKyAAAAygAAABMAHAAAAAAAAAAgAAAAghIA
+AHBwdC90YWJsZVN0eWxlcy54bWx1cBgAAa5yN3JwcHQvdGFibGVTdHlsZXMueG1s
+UEsBAhQAFAACCAAAAAAAAAAAAAAAAAAAAAAAAAsAFAAAAAAAAAAgAAAAgRMAAHBw
+dC9zbGlkZXMvdXAQAAH86vqNcHB0L3NsaWRlcy9QSwECFAAUAAAACABPmS1PkAF6
+t0QCAAC5BAAAFQAeAAAAAAAAACAAAAC+EwAAcHB0L3NsaWRlcy9zbGlkZTEueG1s
+dXAaAAFKxdyccHB0L3NsaWRlcy9zbGlkZTEueG1sUEsBAhQAFAACCAAAAAAAAAAA
+AAAAAAAAAAAAABEAGgAAAAAAAAAgAAAAUxYAAHBwdC9zbGlkZXMvX3JlbHMvdXAW
+AAE78vwtcHB0L3NsaWRlcy9fcmVscy9QSwECFAAUAAAACAAAAAAAjeKkdsEAAAA5
+AQAAIAApAAAAAAAAACAAAACcFgAAcHB0L3NsaWRlcy9fcmVscy9zbGlkZTEueG1s
+LnJlbHN1cCUAAVSZj69wcHQvc2xpZGVzL19yZWxzL3NsaWRlMS54bWwucmVsc1BL
+AQIUABQAAAAIAAAAAAD9t2d/XwEAAD4FAAATABwAAAAAAAAAIAAAAMQXAABbQ29u
+dGVudF9UeXBlc10ueG1sdXAYAAGfIBc0W0NvbnRlbnRfVHlwZXNdLnhtbFBLBQYA
+AAAAFwAXAKsIAABwGQAAAAA=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: odt
+
+UEsDBBQAAAgAADeaLU9exjIMJwAAACcAAAAIAAAAbWltZXR5cGVhcHBsaWNhdGlv
+bi92bmQub2FzaXMub3BlbmRvY3VtZW50LnRleHRQSwMEFAAACAAAN5otTwd8pfJI
+AQAASAEAABgAAABUaHVtYm5haWxzL3RodW1ibmFpbC5wbmeJUE5HDQoaCgAAAA1J
+SERSAAAAxgAAAQAIAwAAAN+D+XIAAAAYUExURZOQjqekoamqrPb4+vr28v///gAA
+AP////uvYSUAAADrSURBVHja7dPJEQAgAAIxvPpv2Rr8YraDDENORcHAwMDAwMDA
+wMDAwMD4k7FnxxqjY40s38DAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA
+wMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA
+wMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA
+wMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA
+wMDAwHjqAuCI3h4P12H6AAAAAElFTkSuQmCCUEsDBBQACAgIADeaLU8AAAAAAAAA
+AAAAAAALAAAAY29udGVudC54bWylV01v2zgQvfdXCFpgbwqtuC1iNXYPLQrsNikW
+dRbolaEomVhKVEnKsv99h6RFU47laOuLbHHeG84XZ6j7j7uKR1sqFRP1Mk5vZnFE
+ayJyVpfL+N+nL8ld/HH15l4UBSM0ywVpK1rrhIhaw28E7FplTrqMW1lnAiumshpX
+VGWaZKKhdc/KQnRm93IrSu/5ZLoFh2xNd3oq2WAHXPw8fWcLDtm5xN1UssFCUEN6
+IaaSd4onhYCoVw3W7MSKHWf1f8t4o3WTIdR13U03vxGyROlisUBW6g0mHte0kltU
+ThDl1GymUHqToh5bUY2n2mewoUl1Wz1TOTk0WOMXWVXbcnJFbMuR0JANlpNrw4KH
+6Z3n09M7z0NuhfVmJCd36BGE9vH4cKwFWU3dy2AHoSKSNZPddOiQL4TwphqCO6DW
+3NvZ7C1y7wG6uwjvJNNUBnByEU4wJz7iojoXNMClCBAJ3Zoy9YVvAqFGCLfIiT1Y
+5aOqfzw+rMmGVvgIZq+DE1YrjetjZKRJwqin75CkjZDaB6aY3jAhW7feto2u+Phx
+N9IeWso8PwsFc+YIjj4cvGTLaPdHPOjklwticVIQti2+RrGgsG9eJKQzZDD+GEOJ
+HJu8LP0cKkRbgxMwuw4BpLuGSmZEmFtaNtAQVj0Xv6HyMLsCDYOGzijvO4Z36awa
+IZJKQQHBQRFNFrCH80FWu2nqzOEQeXGq8aRREKXm+lw9PH1HRpaY8QgD4LBTcC24
+jVf9HcD1D4X8QgF3gaTAhCY5JVyt7l0v98uRezd2L+MvktI1rlUaR9C2e1DF+P4o
+i6OBBiNMSlqDv9BJVMeUitHlXR4YjB+bxGgNtOLMZn/iRqgPp0C3etEAKSpcDxAN
+0wSa/RZLZqv8fxjnvH3dNsBNMM3F5grTggz8XnL2StPqGgu+CS2su9Gnv79G60/R
+d1q2HMvRKI0RpoRrkrVorMwP67jVUBGakcTq8fVvnwPf/kn9XgcXGixxKXGz6QWw
+YK7X9iVxrDXMmBzLPO4Vm0OaNHDKqdSMqsj37AyOK7QfGFKzt+/fLeJA4jdKTjDG
+wcDeo7sv3ToInkW+9y/GltW9vVUr+rOFTwcfnpeLkV3KmWo43iei1XA1pQmHoQ5D
+CpqUFTun/+K8VdpVv7HxKmVPfSqv0wJ/r1by2X0F2LCPR61xlLAKoHZW6YHTHLPk
+4o8GqUEjH2urX1BLBwgErBeMhgMAAO0NAABQSwMEFAAICAgAN5otTwAAAAAAAAAA
+AAAAAAoAAABzdHlsZXMueG1s7Vpbj9s2Fn7fX2Go6L7RtnwZXzZOHwIU290kC3TS
+54KWaIsbShRIypf8+h6SokTZkkedyRqFsXmYwDwXHn7nwkNS7346pWxwIEJSnm2C
+cDgOBiSLeEyz/Sb47cvPaBn89P5v7/huRyOyjnlUpCRTSKozI3IAwplcW+ImKES2
+5lhSuc5wSuRaRWuek8wJrX3utZnKjhhlfcUNsy+tyEn1Fda8DVm87T+zYfalY4GP
+fYU1L2Dqi+94X+GTZGjHUcTTHCt6YcWJ0ezrJkiUytej0fF4HB6nQy72o3C1Wo0M
+tTI4qvjyQjDDFUcjwoieTI7CYThyvClRuK99mtc3KSvSLRG9ocEKX3lVHva9I+Kw
+74AmSrDoHRuGueneadzfvdPYl02xSjp8shx9AqL58+ljHQsi7TuX5m1AFQma916m
+5fblOeeVqVrAJqgxdzIez0b2t8d9vMl+FFQR4bFHN9kjzKIKcZ62gQZ84Qg4EDno
+MHXcQi+6U/N8JEjOhaoM2fUvUIDOpEqvRKWsO7001bHuRRy3soI50xGkGgQ6OlBy
+/CFoVM7bDlhdOMCUoZdEDJNfp24KhOOR5qnSBlxSF1Wxr8r+jhcZLAK2ihJAcsqJ
+oJqEmRFbNzT4Ucb4K1SWe4WnoVFAKWEuQ6sltarhHKUS0QwCk+drT7pRLaScqjb3
+ffl1pGlI7x5QH8t5vE1zErx3O+SOw+64wxFBMYmYfP/OVrZqeGB/ayM3wc+CkGec
+yTAYQBFzTCll55oWDBoaNBHtSQaLg7ySRyplMLo9y0cKxdhAPHgGsV3LZH/HOZf/
+uGS0ozcNEDzFWYMjpyqC0nfAgpoY/BPG2dW+bBvw9TDNYvMG0zwPvM45Z6lI+hYL
+PnPFzXIHH/7178Hzh8GvZF8wLDpR6hLoA1cva0ddYV6O277QrSomO1ywslt0mkuD
+9wLnCY0Cx1v+RjlkKBGKQnepFymV4F8J7O+MQ0Pxw3T2NMezYKDLFWQwYxVlMVnt
+IojtHV8fQRXiuTLZmXGkf5ciMsExPyKwVhKFTptgPAzDZUizVvr5mq6gSUDQUxEk
+cxxBR4cSLug3ruuM5Q5nt7gPem1RCy9sQ731XvG2aS3hZrCcI1UJsn3yDjPpxV2O
+BTbIN3A3JM2PcKG4ngSCkcaEW1bM8gS7CYwdW0Ew9KDgLBopR9GNgDYu5TGIM4HU
+thFeNIuJ3n31ecJfjTPS2Qh7GUQQz6WOv26zK3Zt99VqCkkAhkw710xeho0SBWkY
+1VUyIawMXdJvQA8nuTJjDGf7Au9hiGRmIIL9TAkInN+eK4SIgn0HfSUiM6trnRNB
+W4KzmylfC2gjnEA4Hs7zCnNnj6N+SxylNMwRPny+NkE30YycOsqbmbRisQhcTFpR
+E3o5bUX65XNQ+7FRIPpUjcrxwc0IBkdwAUGayU0wsTWBguurX8k5T0hmHIwYjmNw
+jzHUlAtGU1qtrWcW5EUWqcIq1OUGIABQwN0vp4kLbxRTqBeZnmQ8nK0m8zqJm5mU
+A9R1Bv8/3P/q4e4HHHHF7TIKBUkxzZA+hLpQvA7VvJDJBcsbcsmeEbwqy4gfSPa6
+Y8uFzg4debC7QBgxnEsd12+dGAl+vJgcRi6S+CshOVJ8T1Si7xN0Hr40sT+hje5n
+yKoYizjorCXOeQxLKBE6o+r8utb3T4JjL7M71cFAdVWG2k3JdPL6DF9g4PfJ+Pct
+j89tZr1U9VIsoOoAZLlpBZ6eTBWpCVuulD5kj4fj5dTQ7KkqgpCH/wvMLnfhyg2m
+ichME4HZEZ/lSzWoo8CYRHP15X/f5F+WslmdyK+rRqUJpcgb+26npqv7dvR6OV11
+sXVdt0tcaUovJmdvxdxpccXRbXNdVrXVdUK/kMhtyQGbZs7w2UufgU9+S3K+Ou86
+U261mPRNOdMvJITuE6V3n/GP/WH6CH3Ea9Z/o/YwrbJHtjcictIrIsPvGJLuNqQf
+Th+wOSJ+x1ABYAT+kzXa1eE31WjzrmFv3c1RSjYpJpjcpfy4u253dH12TEtA/wvn
+UTi193a6Ra2ktgjfKSj6dXaexRW9tLl3VP0CB9rTd4wpavTdjKnv5P2/eh6bxfBC
+2buCK+D/YynBBSMjB8JKdguHHoBlVT1YkSL9pIKh0FY464JXil4C7ZO4pKYxB8+Z
+nCTusgNvgQyBs8/0FXSb2guWUrcZ3EGXzY8kRtuzLb3QHgfe5NWJ0c2va8W0LCLG
+q9TcqmwCVI+XxYWRnXLsNcKtywVqF5I9MZ48GMazDoxn7RjP7oHx9MEwnndgPG/H
+eH4PjGcPhvFTB8ZP7Rg/3QPj+YNhvOjAeNGO8eIeGD89GMbLDoyX7Rgv74Hx4sEw
+XnVgvGrHeHUPjJcPhXHYinDYhm94D3RXj4XusAPfYTvCw7tgHI4fDORJB8iTdpAn
+bwS5SfKRz7giEs6S2Y7ui/IyuSKg8lC941zp321OCMu12sf7A2aFfpErB52g9BZv
+3uN8GXvW1g92Wp/70kmvt7+FJIu7DKTtBjr1GpHagrZpOm8K7GcP5uVg9eS9P7bB
+U2qpYdCuLWk0i4T5pFS3dd5XI0Zb/bGIfs4BnXD0dwR3ybEHd+MzuLdxwv+Up2HQ
+wnRxhWUoRxrrLzCX1bnBjFYXqmHn4krtAJ5CXFD9AVnpZi6UwFQF13d4i+Xi4ghY
+3+Fd06qW74oirHU1yRpz80MKG48oxadqdfoCuv6wqGSQJHfqLDTj4Xi88HBwz89o
+SwAOI2CYpqtlCxPe6UfeVp46JTeB5IxWN1o4/m8hlY0LGy12XEBel0ZN5j/WT6n2
+w56x+Rf4H3+0ed4tOCFYP1iaHyMfBW/wWlEdo9dBWRJSLCsd1WzloNZ0883Rt9kL
+Zi83LtSP2r+4f/8HUEsHCOUI1/BqCAAAsS8AAFBLAwQUAAgICAA3mi1PAAAAAAAA
+AAAAAAAACAAAAG1ldGEueG1sjZNPj5swEMXv/RSI3SvYJiRZLGClHnpaqZWaSnuL
+HHuWeAs2MmZJv33Nv5QlOfTIm9/4vRmb9PlSld4HmEZqlfkkxL4HimshVZH5vw7f
+gif/Of+S6rc3yYEKzdsKlA0qsMxzraqhYynzW6OoZo1sqGIVNNRyqmtQcwtd0nQw
+GpVLKdXvzD9bW1OEuq4Lu02oTYFIkiRoqM6o4Feubk05UIIjKKF3aBAJCZrZPuH/
+hurZZSSt9dWox8fQg12EcYzG75kujBDlvQEcu0EuIbMs+JDQPfjeNP5i4ZGfz9vt
+Y+TpEIYbYNYRgWuGPMIkCXASkM0hiijZU4zDbUwiZ7DdpehORyo4vd8a78NthON9
+jLdJimZsdAUhrbv4QLRmOCv/cYj3PyeDm+LnHv6Hl9DkZEVP8shen09j3RGNldwb
+dMtOJQRct8pmvruFQZQVK25EfXoHbtdqvSDJVTOsMKw+rwudNmKt8bOjuQWzLii3
+0O4sLTQ1487jlkPTbAUocIvRJn+RJwPfhytF25CEuzB6fJGqvRxfn3bHXewtgGNt
+dD8RIrjCj19bWYogmlb478QUfXoh6N7fmP8FUEsHCJgoIvu5AQAAywMAAFBLAwQU
+AAgICAA3mi1PAAAAAAAAAAAAAAAADAAAAHNldHRpbmdzLnhtbL1aUXPaOBB+v1+R
+4T0lSVsuZZJ0gJSWlgQG02aub8JesA5Z65HkAP++K9mkKcFXiq17IsH2rrz7fbvf
+Sly9Xyfi5BGU5iivG+evzhonIEOMuFxcN75O+6eXjfc3f13hfM5DaEcYZglIc6rB
+GLpFn9DjUrfzy9eNTMk2Ms11W7IEdNuEbUxBbh9rP7+77Zzl36wFl8vrRmxM2m42
+V6vVq9XrV6gWzfN379413dXtrSHKOV8c6iq/+7krRHxyZB/IF+OcXZydvWnm/zdO
+ikU+C81F42Ybh+3r31wVDvKPU24gsbE5Kb62S7tukMv2I4fVU9Qa+5779ZlvdH9H
+AZti2theMZuUrgiUi8bN2VXzpYnDzQ5hbnzYfeCRifcafv3m79ZlNeOfgC/i/cs+
+b12evT7OehDjagIRYQx6MZML0DseZogCmGzcGJXBcT4GsqtwpeEOIyizPmdCH2z+
+NGHpKZcRrCF6Gaz9AHPPEDXU5rCQD6KdpWqjuA22xfLF8aksxd7l5VkF+JUx5eLt
+2fmxVjWfCaifK85s7cx2VielFLEEbFWy3UVjMCnl35FR/o6YTMnULthiVKZSPRqy
+DWamhyJL5C6nC+tHA2NrvYu4rI3UL+PSZ6FBVbL2Y8ky0AEICA1EfUVfHLH0PV8+
+Ly5ll4t6tf8G6o2Hd9P8i0wxQ735T9rqmCqYGSvUKQXAQ9KcfVswxsxLI8nXT7a7
+LFwuFGZyt0jX5aSHlEwUvt6hIyUal7499KnGfWf/o2JpzENfy3dl1meO+2x9/5Kc
+RQduHgyUFFRfYRKAyXa7TW1wn8LajAULIUYRwW61qsNNJ03FZswUW9is3jG17KNK
+mJnifZbMwMXE08tNwE4AUMqyCvanbDYi4/Q6Cy492P9AoYn6RONSjFYwHmTKVR+b
+/AfKSpAwITz4+VnnbP5tvG4VW7lqX7+znuBpCtGYhyZT5dSu4KEvkBnDSEzdY+IS
+78PLVzmjcWnp3GwJ4sOPQ1iw0XTBF84KkswFrjxV2qkNk68yHlDkBXzGmRcOWs71
+WKrHoELSXtSOWi0ftEAhyAt8SFKz6YEQlor1B+yOmbjLNNhJvCP4Qtp9nPrddKJc
+947mcxKOPvIy0EM2A3FbbEX5ak2gKimEUWZspIfU4MQ/HESkfTbTwUKigj5X2gzJ
+64BGAWkG0qfLr2nEDFj5M4UkFfS3J5J3SfwsPRZAmupx9QXAh4JzJTyC6JZRVyLy
+VYJUnmRase7IiKIil5qEms13j4kwE07v+8i0hpGIClLcAa23XPhXwbCkTkQjvyKV
+G6Qs9APbgD3CR4Ez9lRAKIBLT9DSE7BZeYQp5pSsnyITzXfVM+XJ7l+0Wm8v31aZ
+g4s5fjId+qqxruX955xXdawh7CpLPk+16RMndkur0n28QORUuStRBSE6JjBMeYBR
+37aoTmYwL+v1O8iLiM+GtCdc9b9GLyYXIdXBHiYpDTH29Kr27VVbo77lJ2Mj2ROo
+fWyA2jcxPjNeNAxb+OzJoa20blO35Ngj5ov4lCKKIrN97Fht+AWU7GjO5DiTNGd6
+7Il2kwTUaPYv1cgxam49eUM2wcHwkAnP2jrIZsbiuy82JDJCQjFEHfefP8IWXJ2i
+Y64dgDyMi+Trw5rQKJkYAov81IZMKUL5VuvZzwAzFR6v+DqCBnPHI5pxe0yGULoJ
+VIWnaAjCFs4+iWJ1qj9Z54L0u72rSt0rf4vthlzqaaiSa/ITwndQSHj9DVSridOO
+O0nJdSk5fdoA1gM55NqHoCl+B1AAjsrlF9jsuiHitN50uWRqc9hGPKbPkBXEyqr5
+p0HYA8lRkuL+CYSRpA5A5d9HuMjRBHE3E9XlfWA2AvQ93sKcZcJHpu1MleuK8vOn
+I0+H3d4VwTcwmG7bbv1pvsV7NJ8zbfh8Y4GkH7iJ75jMmOjazV8PMXMuJ0Cd3VKx
+Y4yy4z0pZrv34UMA/tquSNAmTEZ7xKwD3JFSdogsmlAVQyleUL2umPVYag8UbPUn
+LuqRtBPl/xawameIoPYeHv5hFbQnKx39NJtslduDolsVqTenGo8pUaU/JWi++Kle
+s+xHjDc/AFBLBwi2X6qCEgYAAAYpAABQSwMEFAAICAgAN5otTwAAAAAAAAAAAAAA
+ACcAAABDb25maWd1cmF0aW9uczIvYWNjZWxlcmF0b3IvY3VycmVudC54bWwDAFBL
+BwgAAAAAAgAAAAAAAABQSwMEFAAACAAAN5otTwAAAAAAAAAAAAAAABoAAABDb25m
+aWd1cmF0aW9uczIvdG9vbHBhbmVsL1BLAwQUAAAIAAA3mi1PAAAAAAAAAAAAAAAA
+GAAAAENvbmZpZ3VyYXRpb25zMi9mbG9hdGVyL1BLAwQUAAAIAAA3mi1PAAAAAAAA
+AAAAAAAAGAAAAENvbmZpZ3VyYXRpb25zMi9tZW51YmFyL1BLAwQUAAAIAAA3mi1P
+AAAAAAAAAAAAAAAAHwAAAENvbmZpZ3VyYXRpb25zMi9pbWFnZXMvQml0bWFwcy9Q
+SwMEFAAACAAAN5otTwAAAAAAAAAAAAAAABoAAABDb25maWd1cmF0aW9uczIvcG9w
+dXBtZW51L1BLAwQUAAAIAAA3mi1PAAAAAAAAAAAAAAAAHAAAAENvbmZpZ3VyYXRp
+b25zMi9wcm9ncmVzc2Jhci9QSwMEFAAACAAAN5otTwAAAAAAAAAAAAAAABgAAABD
+b25maWd1cmF0aW9uczIvdG9vbGJhci9QSwMEFAAACAAAN5otTwAAAAAAAAAAAAAA
+ABoAAABDb25maWd1cmF0aW9uczIvc3RhdHVzYmFyL1BLAwQUAAgICAA3mi1PAAAA
+AAAAAAAAAAAADAAAAG1hbmlmZXN0LnJkZs2TzW6DMBCE7zyFZc7YQC8FBXIoyrlq
+n8A1hlgFL/KaEt6+jpNWUaSq6p/U465GM9+OtJvtYRzIi7KowVQ0Yyklykhotekr
+OrsuuaXbOtrYtisfmh3xaoOlnyq6d24qOV+WhS03DGzPs6IoeJrzPE+8IsHVOHFI
+DMa0jggJHo1CafXkfBo5zuIJZldRdOugkHn3ID2L3TqpoLIKYbZSvYe2IJGBQI0J
+TMqEdIMcuk5LxTOW81E5waHt4sdgvdODojxg8CuOz9jeiAym5V7gvbDuXIPffJVo
+eu5jenXTxfHfI5RgnDLuT+q7O3n/5/4uz/8Z4q+0dkRsQM6jZ/qQ57TyH1VHr1BL
+Bwi092jSBQEAAIMDAABQSwMEFAAICAgAN5otTwAAAAAAAAAAAAAAABUAAABNRVRB
+LUlORi9tYW5pZmVzdC54bWytlMFuwyAMhu99iojrFNh6mlCTHirtCboHYMRJkcBE
+YKr27Ueituk0dWq03Gxj/v+TsdhsT84WRwjReKzYG39lBaD2jcGuYp/7j/KdbevV
+xik0LUSS16DI9zDe0oqlgNKraKJE5SBK0tL3gI3XyQGS/NkvR6dbdgewZvWqmPxa
+Y6HM98N56m6TtWWv6FAx8UhkKjtojCrp3EPFVN9boxXlNnHEho/A/J6TE5yIiTkM
++0NyX6iMjYKuIe+xe8BgnOpADOezXLRHGvjyHB8ID+RiOJ6lG+lsIS4u64DU8qxA
+lBdzedqdx9Z0KYybEddCaQ0WcuqD0CmEvwf/P68nVzUmHBB4MlzfK8x8lkuNh6Z9
+wjh3vVyGuRG//oD6G1BLBwj1ByOHHAEAAD4EAABQSwECFAAUAAAIAAA3mi1PXsYy
+DCcAAAAnAAAACAAAAAAAAAAAAAAAAAAAAAAAbWltZXR5cGVQSwECFAAUAAAIAAA3
+mi1PB3yl8kgBAABIAQAAGAAAAAAAAAAAAAAAAABNAAAAVGh1bWJuYWlscy90aHVt
+Ym5haWwucG5nUEsBAhQAFAAICAgAN5otTwSsF4yGAwAA7Q0AAAsAAAAAAAAAAAAA
+AAAAywEAAGNvbnRlbnQueG1sUEsBAhQAFAAICAgAN5otT+UI1/BqCAAAsS8AAAoA
+AAAAAAAAAAAAAAAAigUAAHN0eWxlcy54bWxQSwECFAAUAAgICAA3mi1PmCgi+7kB
+AADLAwAACAAAAAAAAAAAAAAAAAAsDgAAbWV0YS54bWxQSwECFAAUAAgICAA3mi1P
+tl+qghIGAAAGKQAADAAAAAAAAAAAAAAAAAAbEAAAc2V0dGluZ3MueG1sUEsBAhQA
+FAAICAgAN5otTwAAAAACAAAAAAAAACcAAAAAAAAAAAAAAAAAZxYAAENvbmZpZ3Vy
+YXRpb25zMi9hY2NlbGVyYXRvci9jdXJyZW50LnhtbFBLAQIUABQAAAgAADeaLU8A
+AAAAAAAAAAAAAAAaAAAAAAAAAAAAAAAAAL4WAABDb25maWd1cmF0aW9uczIvdG9v
+bHBhbmVsL1BLAQIUABQAAAgAADeaLU8AAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAA
+APYWAABDb25maWd1cmF0aW9uczIvZmxvYXRlci9QSwECFAAUAAAIAAA3mi1PAAAA
+AAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAsFwAAQ29uZmlndXJhdGlvbnMyL21lbnVi
+YXIvUEsBAhQAFAAACAAAN5otTwAAAAAAAAAAAAAAAB8AAAAAAAAAAAAAAAAAYhcA
+AENvbmZpZ3VyYXRpb25zMi9pbWFnZXMvQml0bWFwcy9QSwECFAAUAAAIAAA3mi1P
+AAAAAAAAAAAAAAAAGgAAAAAAAAAAAAAAAACfFwAAQ29uZmlndXJhdGlvbnMyL3Bv
+cHVwbWVudS9QSwECFAAUAAAIAAA3mi1PAAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAA
+AADXFwAAQ29uZmlndXJhdGlvbnMyL3Byb2dyZXNzYmFyL1BLAQIUABQAAAgAADea
+LU8AAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAABEYAABDb25maWd1cmF0aW9uczIv
+dG9vbGJhci9QSwECFAAUAAAIAAA3mi1PAAAAAAAAAAAAAAAAGgAAAAAAAAAAAAAA
+AABHGAAAQ29uZmlndXJhdGlvbnMyL3N0YXR1c2Jhci9QSwECFAAUAAgICAA3mi1P
+tPdo0gUBAACDAwAADAAAAAAAAAAAAAAAAAB/GAAAbWFuaWZlc3QucmRmUEsBAhQA
+FAAICAgAN5otT/UHI4ccAQAAPgQAABUAAAAAAAAAAAAAAAAAvhkAAE1FVEEtSU5G
+L21hbmlmZXN0LnhtbFBLBQYAAAAAEQARAHAEAAAdGwAAAAA=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: ods
+
+UEsDBBQAAAgAAFeaLU97lqNONwAAADcAAAAIAAAAbWltZXR5cGVhcHBsaWNhdGlv
+bi92bmQub2FzaXMub3BlbmRvY3VtZW50LnNwcmVhZHNoZWV0LXRlbXBsYXRlUEsD
+BBQAAAgAAFeaLU8WMzGJVwEAAFcBAAAYAAAAVGh1bWJuYWlscy90aHVtYm5haWwu
+cG5niVBORw0KGgoAAAANSUhEUgAAAL0AAAD/CAMAAACTmSdlAAAAJ1BMVEUuNThM
+YG5YaHFvaWN5bWF7hIzczsPj29D57Nz08Or+/v4AAAD///9P8aN8AAAA60lEQVR4
+2u3UyQ3AIBAEweUwNuQfMDH4hVZUR1CP0cTKXNyin71k1r8183I++lP60eLxOfT0
+9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0
+9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0
+9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0
+9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0v9vFNlq+E3U+RAAA
+AABJRU5ErkJgglBLAwQUAAgICABXmi1PAAAAAAAAAAAAAAAADAAAAHNldHRpbmdz
+LnhtbO1Z23LaMBB971cwficGcmnjCXQIbUp6zWCSXt6EvYAmstYjyTj06yvbuENc
+uzFgddqZPiVY0tllfXT2SFy8fAhYawVCUuR9q3vUsVrAPfQpX/St2+lV+4X1cvDs
+Audz6oHjoxcFwFVbglJ6imzp5Vw62XDfigR3kEgqHU4CkI7yHAyB58uc7dlOGix7
+8sAov+9bS6VCx7bjOD6Kj49QLOzu+fm5nY7mUz3kc7qoGyqbvR0KEX8GShZkyaTB
+ep3OiZ19tlqbJLdK07MGeR3yrz+42ATI/rSpgiCpTWvzOEmtb+mQzopC/LNqVtm6
+x2vuqKQzBkMBZIqhlQ+qdagHKVfWoHNh/wqyE/B7mCszyJ+pr5Zl0L3e6fOD0cdA
+F8vSzE9Ou3XR2wEJ25T78AB+MRLE5a8oXaPJJdZ18oX42i8kKZXQ798aJGzYLdME
+tJDnlOh6PJXo4yXuEkB1a9BvFAmJ4gYlVZr9XxpkyWPkrw0ij1HQ78gVYW7IqPqA
+PhTLv0RxAL9BKOqZQi9knxeoyf25nb8B/KGn6ApS9Anhi4ry9PYDz/NtWLNy2EmV
+ohyI26x256iXqBQGDQJ/QwymGqVRRiegd4RFRdQ00W5n3xqQBSTa+lv0sz3B3SXG
+bwQtyvYMkQHh1kCJCMqRSx5ud4uq4VTW626tVPArOkqm7AdLjyeQsRkRlQ282zs7
+/c+xQzmWBLjUTub+RkDiBar4NidMVhCuTphvIDDNX+5O6Dr4H1GZgt5vFz6FnKCO
+kKEoQDNM9pDm9rE2qGcNvFcDVRkTqVOPAj7BeAzE1wcTI0FSJdFSYwD9Wn6KlD7M
+gbsOZsikC8We20iQlPJj3dFZ0tW1PL7miXJW8umAXXYtXU7CKU6IVFBkVRMBMmD9
+nbLjj7EIE5CaXJVmv9t7vqfcFeFLHf+h8G408+mKysr0GwIvT35f6mTwwwcq3TX3
+lgI5/V7N0sa9x+bwWz5Bgqp/uZE9iARJXvAutxzX8tXmvsZdEmFkh75H4k+0XCJn
+awPwQ8a07GsPpt7ibES4B6x5SbsNfaLgSmgDBUHI9P8mZPMdCD6UlPCbiHsqIiXH
+wyYq9jqYgX+l7WZlg/n79fhfVUvT7b0pRXsqkmmvZc6BGrTNf8RdGT9WkBXcZXfu
+n/iIoTRheUweA8wbtrTbgPioP5RfBtg1uiLl91lXqb7IPP4LHdkfOQQZtX3DSOGI
+MC8y08Y35NB7PyrePM6IhLOTS8qJWNfhyDAM2fpWgnhFFGk+05F2fMTTuY4wCAXI
+pNA73w9VWlf7l1/q7KrfMAc/AFBLBwjjEYf1zwMAAAUdAABQSwMEFAAICAgAV5ot
+TwAAAAAAAAAAAAAAAAsAAABjb250ZW50LnhtbLVXS5PiNhC+51e4nKrchIDZpGYc
+YA+b2kMyk8POJrVXIbeNsrLlkmQM/z4t+YHMYNaVx4UZur/u/tTqh9i8PxUyOoI2
+QpXbeLVYxhGUXKWizLfxH58/ksf4/e67jcoywSFJFa8LKC3hqrT4N0Lr0iStdhvX
+ukwUM8IkJSvAJJYnqoKyt0pCdOJjtRJjz3K2uQeH1hZOdq6xw45s2X5+ZA8OrVPN
+mrnGDotJDc0zNdf4ZCTJFGa9qJgVVyxOUpRft/HB2iqhtGmaRfOwUDqnq6enJ+q1
+A2E+4KpaS49KOQUJLpihq8WK9tgCLJvLz2FDSmVd7EHPTg2z7M2tVhoMQvC4rjDn
+OQptRvV1zGdX1zGfSDM/MD27zjx4XCoP6fxSeUhD24LZw8T9PtIXVPqPl+dLXeli
+biyHHaWKa1HNPmaLDu2VUgNVZ9A2u6e7Xi7f0fZ7gG7uwhstLOgAzu/COZN8yLgq
+biUNcSuKCAJHV/JDE7lEmAmDNW3VA9ikk66/vDy/8gMU7AIW3wYTURrLyktmtLuE
+yZP+SDVUStshMdn84Yu3tR64HWwhp0eH0/bQXKfpTSjSeaA4RrCJyVFA8/1ott6v
+hyfqQeE8vWuwWlKHGVoSr/sy/HU+7KdM1WXajoE2GXCqQAunYtKbJSMPYQVL9Q9c
+djst8DAa9AJk3/3DkW66UYoUBosBi15VSWA93hu6OM1z5wpdpdm1x6um58Y82Ft3
++/kTdTri1iYuhi5S8FxYx7v+bdDOAkMHQYZvBJIxDiQFLs1u0874QRy13x3vbfws
+cGH49EavrMROw0HcQwshz9v4B1Yp8/MVrhXG0ci1w5McSkwEjgvTCGNGiEpYjgP1
+yLTw1UfvU/sF/mJ/1vdpBZg5lM7GQvFvOH3UABOEAtX/yeB3ZZU/b/Th19+i1w/R
+J8hryfRkiqYM/rN80am66+SstgpXqeDE+xkK0n+OzsbVagjWncHPKXwZyLoo494y
+FJIK+wu0FWCiTCV7Dewr2QO2Gjp0oXuPHbwRqdvqP71bLFeV9fwDOtPc9BQ3rZor
+YigJWbUqJzyAyA/Y8Kv14tEFv0+4NkBUZUXBJAmtra5hPm/LbvPuhQU+AUGTiuVA
++r7LWC3t1aGCA7Xv8VSYSrJzx6fz5t4N+NomhUrRk9TE7t9SpZOF0Sn2Kj1f5hu+
+LllqDgB2t2lDu+2BJeymETFgXcSe1cVnJsqUSLYHic+LjEnjk9ai/Gdn0R761fnH
+TLUyz4dcEjiy6wrvBtTXb5edNomEg5QkxPTZHXNxF3zDoSu6q9job9gHTNZA7LlC
+ZCYVs/FIg3UWR/2ifYtFv05R7VYb2v23odehrkTIcizpyTm6KcEliL8E3I4ywVwY
+XSAdXTGd+H27+xtQSwcIe22YDA8EAAAgDwAAUEsDBBQACAgIAFeaLU8AAAAAAAAA
+AAAAAAAIAAAAbWV0YS54bWyNk81u2zAQhO99CoHNVSIpWYpEyArQQ08BWqAukJsh
+kxuHKUUaJBW5b1/qz3VSH3rU7De7wyVVP5w7Fb2BddLoLaIJQRFoboTUxy36ufsa
+l+ih+VSb52fJgQnD+w60jzvwbRSs2rG5tEW91cy0Tjqm2w4c85yZE+jVwq5pNg2a
+lbOS+tcWvXh/YhgPw5AMWWLsEdOqqvBUXVHBL9ypt2qiBMegYJzgME0oXtkx4f+G
+GtnrSMaYy6ARn0NP41JCNnj+XumjFULdOkBgMxwStr6N3yQMn1G0HP9q4Slq1u2O
+MZp6CsMttD4QcTBDkxJaxaSKabZLU0ZLlt4nZXGf52FDpMY3HLXg7LZ1UyRFleck
+y8pgXbF5Kgjpw8XHordTr+b7jlY/lgH/FN97+G+uwDX0A73IM3t5Ps6HFs5LHk26
+bw8KYm567cNK0CxyUOqjZg6vwP2qEoSXxkfQEFIZ2zzKg4Vv0z5xntCkSNK7R6n7
+8/6pLPbFJroC9idrxn6Yko7cfemlEnG65P/bscbvrgff+hWaP1BLBwh+MOVvlAEA
+AEgDAABQSwMEFAAICAgAV5otTwAAAAAAAAAAAAAAAAoAAABzdHlsZXMueG1s3Vnb
+jtQ4EH3fr4gyKwTSZpLuGXbpZrrnAYT2AggNsO+exEl7ceLIdrp7+Pot27E79wk3
+sQsgIPap8vGpcrmSubo+5tTbYy4IKzb+4jzyPVzELCFFtvHfv3sRPPGvtz9dsTQl
+MV4nLK5yXMhAyDuKhQfGhVibyY1f8WLNkCBiXaAci7WM16zEhTVaN9FrvZQZ0c7m
+mmtw01rio5xrrLAtW3Q7f2UNblonHB3mGissaNo0T9lc46OgQcqCmOUlkqTD4khJ
+8WHj76Qs12F4OBzODxfnjGfhYrVahXrWEY4drqw41agkDjHFajERLs4XocXmWKK5
+/BS2Samo8lvMZ0uDJOpFteRYAAS2q/JynqOmTSu/9tns7NpnIzLHO8Rn55kGt1Pl
+IpmfKhdJ0zZHcjcS3yfhK5jUf716ecorns9dS2FbUsWclLO3adBNe8aYo6oMzGHX
+dJdRdBma5wb6MAk/cCIxb8DjSXiMaOwUZ/mQaIBbhIAI8F6lvEVztelRz49DjkvG
+pSOSzi92oM7SHdWdzOn4UVWzFprxJBmEAp2LEI4tHJpgT/DhrFXLpvVfhRrUrF+T
+BosoVBh3BEDeU7HlmbsOUlYViTl2Rgx8LDEnagpRbbZueWhmDGWf4bK+QxoeWoWV
+YGpPm9vSoBvGglwEpIAkY+W6Yd06+UJcyKFQvLsJ1VygbhWom/U6jct06W/tzZky
+uDVTFOMgwTEV2ytT8dywZ54VyY3/kkD51Jv33qICkhTKkoXmhN5t/AeoZOJpB2cG
+fa/lWuGDDBewazg84kCEaCFKImMoL3vEic6NcJrac/wP+ruaptXAzKF0JyTOv4TT
+C47xCKHG1Ldk8JpJpvfrPfvzL+/tM+8GZxVFfFSiMYOvplc4lnf1uGng7K4SnKKK
+1m2d9VwT1iUjiDGlvoWXiKOMo3IXlHBuMJcEekEzBWjwwsogIUKiQrWFF7+W8iSg
+Oix9M01zJPlTtqaoyCqUwSwu9EAM5UFyYPf+rd91EUBVRkU3VTXG+rGQjzs7Uzu0
+E89e992qvoDi47RjB9qRrms39cdrHaEB6bdXpnWqO6hWPOpMi/wOyKufclLoSpaB
+XUIyIgWUIL3QgE/nI644h4b/bmipRXT5JrK72DMKQVHdsuQV9gcc3OW3jFo27YhZ
+bCNq258dsY6Lkf1B/pIc0aCkkM2wN7hVTeVXGx+YnJTFzmacVaV+19G7aqjVFmau
+Xv5YmuucpQya4rM0jeCXWqv2qbDbwK2sH39UeY06OSrd4SgSYpr8PaIVfvjoQSaf
+blzaobKktdhBKy0nQmUs+wF6bo6af3+Bu6dI2SoxUfZ79bs2+cLyb92MXQJ23l0F
+41Vs6mqcBbKUHHiUlEO0bqhGmMZjdoPFdMg8dyW57xJBO9wTR1JzrD8/EOgISWz9
+aTC0oJjDCzS2GMEoSUYgB5Ko9zRUSTaCqAuAXlT/33ccDphkO2gy4XwmnyrO8rPU
+aQurXye6p+x0XvtL/46R+kb0dQLjjAb6CSUhBCYrAsEqrvqJlBytc3gtw0iJCRWo
+APlSREWjzAz2KSD5yenGj7Hq/sf7E5cl5COsvVC9zFjmfFkwa0UXnyWpDcccSTkz
+n0gCuEMU/VXUJxl22sT6USV3DsZx0O4fS9X6UHTHKtna06syX/gDoD4p9aYPGwhy
+loAd5YG8PcVkB7s7dS6tsZQxCF8nYOpS2tVBWMJrmA1ajngGUxSnMBG1B7mBd0Zv
+mZTqQ8Jv59HKdLHhOKWay3egCQ33AMc2n7AXhRnRW/5I0VMzt4wn6tPk8vwShjxd
+0L2zSP/SgBIl5uN3dP54YY1Q/EG1MkVia/hZHKnfTp0GAhqlDPcSpbfT/0Em/Tf1
+uj+rw9FqVU/kSDgX4tST6kHlaaprbJ6DgSJn6G+v9E8ayvpfscPYoLfX19dXYXew
+Hik7InQir6Jor2sioBW/6114Rhu3+hu1l/pB0Tb98nZh12uM9ShYVy3RJymEPR3v
+k/am/qo6oeyyp6y9+TN1hSlCnyi299DgJJG0CTHPj3pCtFZqDekT1FkdGik82lNB
+p+ZAgX7dgYMVLVZBtAoWF/42ikL9J4pqFgq4/cWzhIF9FK31H0d6KH3a/L5bTnlh
+E6jfTrerVRNoxr5R7oXDpz0c/lnm9l9QSwcIEwUhmUoGAAALHQAAUEsDBBQACAgI
+AFeaLU8AAAAAAAAAAAAAAAAnAAAAQ29uZmlndXJhdGlvbnMyL2FjY2VsZXJhdG9y
+L2N1cnJlbnQueG1sAwBQSwcIAAAAAAIAAAAAAAAAUEsDBBQAAAgAAFeaLU8AAAAA
+AAAAAAAAAAAaAAAAQ29uZmlndXJhdGlvbnMyL3Rvb2xwYW5lbC9QSwMEFAAACAAA
+V5otTwAAAAAAAAAAAAAAABgAAABDb25maWd1cmF0aW9uczIvZmxvYXRlci9QSwME
+FAAACAAAV5otTwAAAAAAAAAAAAAAABgAAABDb25maWd1cmF0aW9uczIvbWVudWJh
+ci9QSwMEFAAACAAAV5otTwAAAAAAAAAAAAAAAB8AAABDb25maWd1cmF0aW9uczIv
+aW1hZ2VzL0JpdG1hcHMvUEsDBBQAAAgAAFeaLU8AAAAAAAAAAAAAAAAaAAAAQ29u
+ZmlndXJhdGlvbnMyL3BvcHVwbWVudS9QSwMEFAAACAAAV5otTwAAAAAAAAAAAAAA
+ABwAAABDb25maWd1cmF0aW9uczIvcHJvZ3Jlc3NiYXIvUEsDBBQAAAgAAFeaLU8A
+AAAAAAAAAAAAAAAYAAAAQ29uZmlndXJhdGlvbnMyL3Rvb2xiYXIvUEsDBBQAAAgA
+AFeaLU8AAAAAAAAAAAAAAAAaAAAAQ29uZmlndXJhdGlvbnMyL3N0YXR1c2Jhci9Q
+SwMEFAAICAgAV5otTwAAAAAAAAAAAAAAAAwAAABtYW5pZmVzdC5yZGbNk81ugzAQ
+hO88hWXO2EAvBQVyKMq5ap/ANYZYBS/ymhLevo6TVlGkquqf1OOuRjPfjrSb7WEc
+yIuyqMFUNGMpJcpIaLXpKzq7Lrml2zra2LYrH5od8WqDpZ8qunduKjlfloUtNwxs
+z7OiKHia8zxPvCLB1ThxSAzGtI4ICR6NQmn15HwaOc7iCWZXUXTroJB59yA9i906
+qaCyCmG2Ur2HtiCRgUCNCUzKhHSDHLpOS8UzlvNROcGh7eLHYL3Tg6I8YPArjs/Y
+3ogMpuVe4L2w7lyD33yVaHruY3p108Xx3yOUYJwy7k/quzt5/+f+Ls//GeKvtHZE
+bEDOo2f6kOe08h9VR69QSwcItPdo0gUBAACDAwAAUEsDBBQACAgIAFeaLU8AAAAA
+AAAAAAAAAAAVAAAATUVUQS1JTkYvbWFuaWZlc3QueG1srVTLasMwELznK4yuxVKb
+UxFxcgj0C9IPUOW1I5BWQrsKyd/XDnmVkhLT3PalmWF20WK1D77aQSYXsRFv8lVU
+gDa2DvtGfG4+6nexWs4WwaDrgFifg2p4h3RJG1Ey6mjIkUYTgDRbHRNgG20JgKx/
+zusj0yW7ETAXy1l15euch3p4nw/X6a54XyfD20aoeyDXcoDWmZoPCRphUvLOGh7G
+1A5beRQsb3VKShlMS1sArhlC8oZBqCmaNtsSvtA4T4rPoUzY39HkgulBjf1JLATM
+w5JIDsbeQWbYsxrbk4BtRB6NeDZuADZPByU+eHi+B+uInetLPl4KzZWxFjwMaczK
+lpz/9ud/XA+eLhUcJcjipL1FmLiUU03mtnuAeJh6OZm5UL/+hOU3UEsHCJlLVasm
+AQAATgQAAFBLAQIUABQAAAgAAFeaLU97lqNONwAAADcAAAAIAAAAAAAAAAAAAAAA
+AAAAAABtaW1ldHlwZVBLAQIUABQAAAgAAFeaLU8WMzGJVwEAAFcBAAAYAAAAAAAA
+AAAAAAAAAF0AAABUaHVtYm5haWxzL3RodW1ibmFpbC5wbmdQSwECFAAUAAgICABX
+mi1P4xGH9c8DAAAFHQAADAAAAAAAAAAAAAAAAADqAQAAc2V0dGluZ3MueG1sUEsB
+AhQAFAAICAgAV5otT3ttmAwPBAAAIA8AAAsAAAAAAAAAAAAAAAAA8wUAAGNvbnRl
+bnQueG1sUEsBAhQAFAAICAgAV5otT34w5W+UAQAASAMAAAgAAAAAAAAAAAAAAAAA
+OwoAAG1ldGEueG1sUEsBAhQAFAAICAgAV5otTxMFIZlKBgAACx0AAAoAAAAAAAAA
+AAAAAAAABQwAAHN0eWxlcy54bWxQSwECFAAUAAgICABXmi1PAAAAAAIAAAAAAAAA
+JwAAAAAAAAAAAAAAAACHEgAAQ29uZmlndXJhdGlvbnMyL2FjY2VsZXJhdG9yL2N1
+cnJlbnQueG1sUEsBAhQAFAAACAAAV5otTwAAAAAAAAAAAAAAABoAAAAAAAAAAAAA
+AAAA3hIAAENvbmZpZ3VyYXRpb25zMi90b29scGFuZWwvUEsBAhQAFAAACAAAV5ot
+TwAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAFhMAAENvbmZpZ3VyYXRpb25zMi9m
+bG9hdGVyL1BLAQIUABQAAAgAAFeaLU8AAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAA
+AEwTAABDb25maWd1cmF0aW9uczIvbWVudWJhci9QSwECFAAUAAAIAABXmi1PAAAA
+AAAAAAAAAAAAHwAAAAAAAAAAAAAAAACCEwAAQ29uZmlndXJhdGlvbnMyL2ltYWdl
+cy9CaXRtYXBzL1BLAQIUABQAAAgAAFeaLU8AAAAAAAAAAAAAAAAaAAAAAAAAAAAA
+AAAAAL8TAABDb25maWd1cmF0aW9uczIvcG9wdXBtZW51L1BLAQIUABQAAAgAAFea
+LU8AAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAPcTAABDb25maWd1cmF0aW9uczIv
+cHJvZ3Jlc3NiYXIvUEsBAhQAFAAACAAAV5otTwAAAAAAAAAAAAAAABgAAAAAAAAA
+AAAAAAAAMRQAAENvbmZpZ3VyYXRpb25zMi90b29sYmFyL1BLAQIUABQAAAgAAFea
+LU8AAAAAAAAAAAAAAAAaAAAAAAAAAAAAAAAAAGcUAABDb25maWd1cmF0aW9uczIv
+c3RhdHVzYmFyL1BLAQIUABQACAgIAFeaLU+092jSBQEAAIMDAAAMAAAAAAAAAAAA
+AAAAAJ8UAABtYW5pZmVzdC5yZGZQSwECFAAUAAgICABXmi1PmUtVqyYBAABOBAAA
+FQAAAAAAAAAAAAAAAADeFQAATUVUQS1JTkYvbWFuaWZlc3QueG1sUEsFBgAAAAAR
+ABEAcAQAAEcXAAAAAA==
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: odp
+
+UEsDBBQAAAgAAHWaLU8zJqyoLwAAAC8AAAAIAAAAbWltZXR5cGVhcHBsaWNhdGlv
+bi92bmQub2FzaXMub3BlbmRvY3VtZW50LnByZXNlbnRhdGlvblBLAwQUAAAIAAB1
+mi1PRjHGWMUHAADFBwAAGAAAAFRodW1ibmFpbHMvdGh1bWJuYWlsLnBuZ4lQTkcN
+ChoKAAAADUlIRFIAAAEAAAAAwAgDAAAA3DNNVAAAAv1QTFRFAgMMCgUFCQULDw4O
+AQQQAAgXCg0TCRQdEwsMExAPGRMKFRUWFRUbGxYUGxQZFBsjHRwiGSc6HTA7JBYM
+IRoVKx8VJiMfLyUcOiccOjAeJCEnJSovLSYjJCM1IyQ4JyszJig9KS0wJzA6KzM+
+MykjMi8sPyslNzItNDIxMzY6Ozs8IzpLKzVCKDVNKDpSPj9AN0JMMEBWN0ZZNExe
+PktbNktgSTcdQywjQzs0Rj84Tz4yUT0uSkM+VEU4WEIyX0g3W0o8YUk4QkNDQkVI
+SEdLTEpKQktSTU9SS1JbWlBEV1dXW1VRXllTWlpaRlVmVlthVml8ZFRDZVZKa1ZD
+allFcF1LYV9pbWZedWRLdWNSe2ZSfmtWfGpaZGNjYGNpaWdla21pZXF/bHN6em1i
+d3Jse3JmfnNqcXF2dHh/e3t8XHKKYnOFYneLaXeFa32OZXqSdn6IeYONdouggmlU
+hnFeiXFbg3dshHluinVhjHtmjn1shH13ln9tgYF/i4N8lIJymId1mohxnYt6oI14
+pZF9hIWFhoeIjIiFiIuOgoyXgJCfjZOWiZKcl5KNnJeTmpqahJaoiZ60kZykkZ6p
+n52gjqCzi6G5n6WtlKW2k6e4nq69qZqKs56JraSbtKGPuKeVoKOopaquv7GjtLKw
+mKzDm7XGpbfMprnKqbXBrL/XqsHPucLLtMXTucfRvcrVvszZv9Dbs8bgvMzjutTn
+xbGdyrSdwbSlxLaoxruvyLWiz7ykyruqwru0y76y0r6my8e828av18az0si92su1
+39G8wsLCzMvKxc3Vzc/R1M7I287B3NDD29XN1NLS1tnc3trUxNTixNrqzNjizNvq
+0dvj097q29/j0+Pt2+Di2+fv1ez62ebx3unz2+7659nF5NrN7N/G6tzM5N3V6t/S
+7uTO6+HU6+Tb8ObZ8+jb4+Pk5uns6ubj7Onl7u7u5Ozz6+7x5fT87PDy6vT68uzk
+9/Dm9vHr+vLl+vPo/fnv9PT08fb69fn8+fby+/n1/v7+mnLngAAABINJREFUeNrt
+3HtQVGUYx3EoJxxzsguOYWYXtRTsOpTMqEk7s0o5rFAyJoljpRVpulmOWZqOo1OW
+FaiMFyJq8NJNycrNyulGalHWYEwQXdgYYUA5JGsbHU/nmc7ZDZbF3fSPZpz1/b4D
+u+95993D7md+73ue8w9xoniLAwAAAAAAAAAAAAAAAAAAAAAAAAA4bU1THKAi7ie1
+ASrVAtAPfNluPx/Z961hPbXuO1zRSyUAT+IVw/puEpnWL23gldVS2nvE8BlKAWTP
+FVl0geE9q1zMWev9F80Xc7JSAFbm9y5N8DX1efgHe/+LrxLZpRTA4nMdt6cl+GR3
+2tmXPNK+q9fPloJKAL/GbxV5OeGovQuWJM752P7uH6oE8Gnc92KOSzj69WrrYMFN
+jfFbRB5XCeCP/tcvGbMy/p6vEjNWzkwskpx++VMcStUBDU8v2S97XvE1vZD/zH6R
+jpL89a3rfNwMAQAAAAAAAAAAAAAAAAAAAADA6QDQRbeaBH4D3VBHQgcndrumdRsN
+e6t0TYsyO2w0UleP1D3VDxX+faIDaMGX9YBEcGZnRw+NduuGvxQ2euI5wib0nK13
+/1ORuhLW7Xmak3S7vVXXtegAuhKh1wEAIOoeoAQAewAAigOwBAAAQO3LIAkAAAAA
+AACAUpgEAAAApTAJAAAA9gASAAClMAkgASSAyyAJAAAA9gASAAAAAFAKcxlkCZAA
+EgAAAACoWQdoAPyvzTN8gnPEDkmq+64tcOxNCY6b0yNMXlBkP46qEWnIzOwzPnNt
+6KUV78TmEvjt/IMiH2WZSXXFb0tpnrv9UIo0uo9bX6iv2yjJe7DannTsybxVx+Xz
+Ge6pRdacNTaA1W5+V/xP5a02KleJ/4FPBqTWxCTAiqzgc1LdXVs8qbXLZ3tTWse8
+aQ3UD24rTW3xDLEsZFlBS05Bx6Dt3wzY8NnIak/vLoA7C1qmzZf0tx4rMMdujc0E
+LJ4bDLwNcHehdXbvhWM32yPNgyVno8hVVkDkyPMznXO+uEYkd+3yeSIXdwL8ed5z
+Ly68QRquzjBk3M7YvBkqTbUfDwQSkGuvcO85EwOhaL5cJm8Wc1SV1R+9pnbpvIpr
+RR4tXBYO8GpZ2V5pHOoy5JadsZmAjqHP/q494Qok4CWX8UGWN8VM32ADXGpYy6P+
+Mp/IX4Nq/OkPNQ/80Z9U9N6NRmVoCYzeJntek+xtuYVy6+sxWgc0TUlOdvsk/ZdF
+b5gLHRkHD7mkcfxha1Hc4dLud06wtwMpdkwvSy4vvm7SrE1/3+e497bgPxTLrpL6
+ic5Jte/PlmPOtuLkcm6GqAQBAAAAAAAAIKZKYRIAQMwBUAoDAAB7AAkAAABKYRIA
+AJdBEgAAAAAAAAAAAADAzRClMEuABJAAAAAAgDqABADAEiABAFAKkwAAAADgDAbQ
+dPsn1LSTdYMH//5qonUOaZHmaxHOovU4paZHm9R59lP7WF3zNb3nB/uPQkjFBgAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGd4+wf7joof
+9714yQAAAABJRU5ErkJgglBLAwQUAAgICAB1mi1PAAAAAAAAAAAAAAAACAAAAG1l
+dGEueG1sjVNdb5swFH3fr0BeX8EfhAwsoFIf9lRplZZJe4uI7aZuwUa2Kem/HxjI
+yBpNefS959xzfD/y+1NTB+/CWKlVAXCEQCAU01yqYwF+7b6HKbgvv+T6+VkyQblm
+XSOUCxvhqmCgKkunVAE6o6iurLRUVY2w1DGqW6EWCl2jqReaIqdaqrcCvDjXUgj7
+vo/6ONLmCHGWZdBnFyhnZ1zbmdqjOIOiFqOChTjCcMGODm81NWLXlloj7JCtnG/K
+bTXWnHUtrfXZ9EibGuCtE4Q2cHovaNvI+lbFERsy3bSD5qG+6GmlZHNrmRH7yfXR
+cF5fG8rgOYZD1ytXhe9S9F/BxRr8/7fZ+bfzGqwWj4By2bJxHGXuh8KM8N7CQVCU
+BOEsRFmI4x0hFKc0IVGSpijDKYlzeIWRc0avUTO6IdEmTnBMkm8ohwtsUhVcuuEA
+Qt4ZX6t82iXo5yzwKXnJYR+sFrbE/6Dn8IQ9n5Ed98U6yQIf14dXwdww0065ApAE
+wJlwFEoMatqUj/JgxA/fJ5hEONpG5O5Rqu60/51u99tNsALsW6PHghCjBt09dLLm
+IZl9/a2Yw4u2w2unXv4BUEsHCKlTnKy5AQAAKAQAAFBLAwQUAAgICAB1mi1PAAAA
+AAAAAAAAAAAAJwAAAENvbmZpZ3VyYXRpb25zMi9hY2NlbGVyYXRvci9jdXJyZW50
+LnhtbAMAUEsHCAAAAAACAAAAAAAAAFBLAwQUAAAIAAB1mi1PAAAAAAAAAAAAAAAA
+GgAAAENvbmZpZ3VyYXRpb25zMi90b29scGFuZWwvUEsDBBQAAAgAAHWaLU8AAAAA
+AAAAAAAAAAAYAAAAQ29uZmlndXJhdGlvbnMyL2Zsb2F0ZXIvUEsDBBQAAAgAAHWa
+LU8AAAAAAAAAAAAAAAAYAAAAQ29uZmlndXJhdGlvbnMyL21lbnViYXIvUEsDBBQA
+AAgAAHWaLU8AAAAAAAAAAAAAAAAfAAAAQ29uZmlndXJhdGlvbnMyL2ltYWdlcy9C
+aXRtYXBzL1BLAwQUAAAIAAB1mi1PAAAAAAAAAAAAAAAAGgAAAENvbmZpZ3VyYXRp
+b25zMi9wb3B1cG1lbnUvUEsDBBQAAAgAAHWaLU8AAAAAAAAAAAAAAAAcAAAAQ29u
+ZmlndXJhdGlvbnMyL3Byb2dyZXNzYmFyL1BLAwQUAAAIAAB1mi1PAAAAAAAAAAAA
+AAAAGAAAAENvbmZpZ3VyYXRpb25zMi90b29sYmFyL1BLAwQUAAAIAAB1mi1PAAAA
+AAAAAAAAAAAAGgAAAENvbmZpZ3VyYXRpb25zMi9zdGF0dXNiYXIvUEsDBBQACAgI
+AHWaLU8AAAAAAAAAAAAAAAAKAAAAc3R5bGVzLnhtbO1dS5PjthG+51eo5LJvpPjS
+g/LOuhy7XBVnx3F517m6IBKS6KUIFkiNRnvKP8ghh/y//JLgQZCg+BApcSSNhusq
+7wpogN0fuhtAowm+++554w+eII48FDwMdVUbDmDgINcLVg/D3z/9pMyG373/yzu0
+XHoOnLvI2W5gECtRvPdhNCCNg2jOKx+GWxzMEYi8aB6ADYzmsTNHIQxEo7lMPWeP
+4iWss6bNGbHcOobPcdPGlDbXFiyaP5kRy61dDHZNG1NagqncfImaNn6OfGWJFAdt
+QhB7B1w8+17w+WG4juNwPhrtdjt1Z6oIr0a6bdsjVpsy7KR04Rb7jMp1RtCH9GHR
+SFf1kaDdwBg05Y/SyiwF280C4sbQgBgURjXEMCIkRFyql806ktvk9Otp1Vi7nlYV
+MDtrgBvrGSPOq4rpNlcV05XbbkC8rhjf2eiRVLL/PX7I9Apvmj6L0uagcrAXNhaT
+U8vtEUIpq7QBN3bGrqFp1oj/lqh3teQ77MUQS+ROLbkDfCdFHG3KQCN0+ohQKPCJ
+qnwq98bzG0tNaCuUBAReY+gpbUFVMQW/UsLxCMMQ4TgFZNnc6ZKnGKnLWMcbv9pl
+0FpBusKuW0pK2DFHxH0Q41WePLj7apibDeoVwT5QBOZajzVhRLLvrW2gayNKk5ov
+UY1sosCrdCpbom3g8nHgAMLnEGKPVgGfNZvnepBHy0cndJnMf1IPuUnBg77wFKlI
+pd0gpGwixQuIgaBwLrXOea0oMuOy4fv024jWKXRGJD4/eY60EDCG78Wsv0Rkxl8C
+ByoudPzo/TvurdPiAf9NmXwYfvCI62fCDz6CgBgYcamClBjO/mH4DQhR9O0BHS8c
+DnJdU3plBQMiNTF8jDYgyFGEXuwQ1/gEsMd0Y9ScNdLlshFvlPASzP0I/wT/3NZj
+JtE0YCnaRzHcnMPTTxjCCoakqpfk4BcUIybv4Ief/z74+MPgN7ja+gBXQlTVoDO8
+RlVGkZTzlfH7d9T5kKkbf4Z4wP7NBfoeY7Tj3FOv+Vf0/DDUBtrA0AamxsuJ+T/q
+pMxXdFq4NrQvGUwuXIKtnyzABasJAisMwrXnDAVt8lsJiYOAOPbIgp32H8UYfYZk
+BvMRWaN9ZVqTMbCGnMml5/tpzdSwlw6xkiWa70hXCgr5aixACv2dMRUCDNjDco9i
+VdS/KGAboygEdI/guRBxUuCHayDQDreBE2+ZybG+yWh4m5C6e15PVrFQWWAIyEqX
+8O85saihSwSysFY2yCXd+1iJF7kh9AIX0vmQ7lpYL5QRtrtZAj+CKVhkdiGgojCi
+Y1wtVkpO5SpIu42gQpb5Ltop7OEJkjHewhxTVf6IIM3qI+8LqTesMGZlPghWW7Ai
+RTBgBQ6ZYWJMhvz3j4V+FbIYAMGhQ8loaN+Chj8hgTh5iKj7shY1ydNExQ+/FJ9J
+l0M+fD7y1JSq9Llp7do7fHJa9bdfhtn45GxBDIxsGBzmKAZkOsbusL25MJvg9kL6
+Qb7nDmUT2nkuXZprzmZ41LK4KyC8kp1B2lA1aNNivQPptC5UNEdAVLmmOa0ta0zt
+OpWg2tJZDVVtQExNWSPsfUF0tUGM1VuRof9zG8Xecs90MAQujRIoxGYoK7oxpsxI
+FQsUx3QRXlbnw2XMBDiswN5qLdXwAVgDl20UPNelBiAVKsTrRjBWnvNo5Cv3pZVC
+9plG/yNawIIDvhflneuBDslkPtlD+JxYWWx9H8YDXknLyRpqyH/yKoXuCR+G//vv
+v1OFkzqRdI612XiB4oMFqUnHekIEGFW5H+E4hG5/JCP4cb9ZIH/YxDXl/Y41/ppZ
+WZ2kLXAwzsCBzRrKApJ9KhQg3B9CZmcI6VzP7w4hq0OEZneJ0LgzhAzVukuEJp0h
+ZN4lPtPu8LlTPz3rDCHrTv203SFC9+mnda0ziMavy1FL1cmObVS18aoNLJBnk93O
+igrMdxHJFiIpFDuIfCnfpeTL0g1KUszCA2vI2+ua9jUrZch5LIDASY9iysJFlDpA
+eAP8rJcYk20xP54JUAAbIS09Cm1jymK6s5OqGOvxGqPtaq0k56nyI4pE8T48oKmI
+TLCd/KG6vFwM90C19FkSA+FlQrI8rmKDWJB4S4YN86iPjInobpeMtegvsT8Yk/2z
+QrbTAYsTlUZvRDCkJkBZQCFpcmagUnRTFa4U9RmiFcEfDq1cycw1qc1jIsFVQ5AP
+BdWFiBsRCYFT4kqRU4pqoVOSCrHT+mrBK0iYpsFNuAZcZQs2haHvwWWJeqKnMu0s
+I0hcQuYepDDYkfAXWvwJnXjnxWSi4bHn0iiYCMQCnOZ3KIXYx7mBMlUfV8TKNPan
+JBaWhsxromjT+igaN+DKIJrJfXpbNLnTeUk4E6/25EUeU+kXiXudIDmZh2j08GVE
+P6YbPJLJLKUx6/9grP9haH9Q9unfAaL/p33RvwlXWSG1NyGD60WhD/aK3M2A9jEI
+0IC2HpCm9N9yoxe0LO4gTsaB+pTXweUCuftrclq/wEuWJhMykbSWiobQlx6sPI24
+nHRVi2t+ZJcP+LcWky+Yb1fIFjuI/AZAbKsawuHF/lUdQxNFtqxWikxF0q8hU+Xx
+lcansgtPm82tiC9Emg6H0X44jBefi9PVkpady+XGYLl0HNu+rTFoYuT8KYeBAlWv
+CBWkFSXje1q0gA+62WoyWUPgXnvJ0Qj6BE3LqILT0Jtj1M4wBEZXcVXXAykfMBER
+jgXy3fbQvZBbuVnorJJYkxcTI3fORXQDQbTFl7DX4ua7ky12WaZJabuK/BRpbIVL
+Jw59G3hxsj9vOkZGGx/wY5IhtADO5xWmabmHIyC/M/BSqy1eWhpWPEMSvh+PzhKo
+SarP5VJwLgJggGJ4Hmid7jz4SWmzvYfS8EynP39Iuzv7/MHQbuT8gbTJbD8bIiI2
+nzXu8YSCgy9Xspnx3k8oKsS+2xMK4ZoT/1JYsHfonXkBzUdXiCXt0tPefK4s8ZQo
+sf9ojb3gM/1Nio8kiRbkuFCy6GGSZG16kn1jWQFnppH+6z+npACObwOh6a2mkRr1
+yTez16pD5Wmkp+jQzSRwdaxD3aWRvrLspCukkU7uNE2yu0TS6Z0mAXaXSGrfJT7d
+pZHq2q046tvNI9X11+Wqu08kTcLI44ooch90uWzQxTT6oMvVgi4c/DcXdKkQ+60E
+XQpHfbmgS/X5VEnMo43Htc7wuEmAtpDOKwcQSyuzQNvslMOkRFSzG8SMloiZ5yNm
+1SFWWpl/cf9UxKxuEDNbIlZ1NtwCsUJAtjZIXQzmnorYuBvErJaIVSW/vALEJt0g
+Nn47iE27QWzydhCbdYPY9O0gZneD2OwNIBZtF6VJw50fgqW5H0+0pZNlfmw81/Vh
+w7OulN3+YpT+YpT+YpQbOdHqL0bpL0bpL0a5vfOs/mKUV3OufrMnWv3FKPeWenCd
+i1GY4F28wNSffKXd9Sdf/clXf/L1Sk6+XlOYpY+x9DGWPsbSx1huFKE+xtLHWPoY
+Sx9j6WMst41QH2PpIMZSEjjpYyQXjJFYZdd09DGSy8RIrEJ+4JuIkVSIfScxEjno
+oYT0W0w+2BPXkgtFfP9B+2TQLK/cZ1ZDn0wKa+QToxrkKvj1JA/DNSBebEvBe1rN
+aVaMqo1n4hLYPd0STS1T/E6mD11Tx+OJKBTvaCuaapjJZa3n8aCTyUu3rszEIRBk
+3aob1wbiGkwcAkF4munXBuI0Jkb19tTS3vRPWmtzE0HN5sZmmKqdjbmQylTH2vQU
+ZKX8tarhnZmzRjzoJkG2IbCjg88vJj/pTQ/0K7uOIirE2q4c9F8ftWEJTV2WYXWO
+YcsDOfY8AYiujm25XGBiTFWb62kCiEecO8Je9q1s+oFgDDw59bBMBSsB0G8DgFmp
+9Hqt6D6x6sgBIayXvRhmf3TDwq0jyRfbmZqlkMiFheMAaTHIJ/UFwi49Vj35KmTC
+V+HFnFZ85UxU3GZNr/fL7kUvJVkiFB8hYY8SX3pPth+lhC6IyR7DY5eVtrqy63GF
+r3d5Y+kVp/TPsTtkqHoT3U6dmGra05wTOy719e5dPCZ15WkWN/sXxyYsakTLVPLi
+BXq3oieWZbfE4tyX9m4Xi/Z6ce7reJfG4tqW9GvRkES8a3gsmiVfl1oRJyivzD7C
+0uo9wsdfi5ouMesjGturGqZGY9LqhtjLiFxU6ML4NIlQwsC9RemK74K+0IC+YoyK
+b3+epAHtbp+/pIDFlzWvqATXhqkuDejxw8UuE3wFZzl98k+f/NMn/9wMQn3yT5/8
+0yf/9Mk/10WoT/7pk3+aJ//UL7aNfrHdL7b7xXa/2L49hPrFdr/Y7hfb/WK7X2zf
+NkL9YlvKtK9MxkoqNiCK2ScHeSmL5/tgT4toj9lvHurny3Se00MZKq+XvvnXgEYc
++laSOgQHjPwaiuQbjzT1llGNDuXggCephwoXOp8yU5XfpuRScNPD7MN6mr0mzqb3
+6Qk3SyJKmGZt4vV2swiA5w8yDsuwkDP0bNWw0/xIceo8Udmhc5pEKWX5kdWFOeaa
+edkH65pqzcyrPHmmjk39Yk8mewB7fDXED59+YdgLjz/EfomJ7pfYAkspSxMwcpkg
+LMugDZ/mxC6krKqmPZEY1SQe2b9z5u74IIr453YhFibK+Fqg52SyCZMpIceofpgX
+zLtInXMovE/W2SiD5RyIrM4h0g3VMJrAlOYRtkHKPEQq7aUrsIxr6ZMxUcc8m7sM
+LZ7BeZZS8S4ugNPLK1U9VlIqa1vd4rVZ+/ff+PG3/J/frOJvBXQSQUM0R2WrBbGG
+SBZMtNuy60LqVgh6+QpBH+YGMIeSTHpweUrzYTPGqm4Xxo2+ZpAbN1V+CUNTZ+a0
+atASDqreTkiWsAfDOarW2aMiZ5+nPFdqop361K4U21Jt/i5KmdgJF5cQnGfbduLc
+iPkZZtFoLWtWiYJuqzqb0rubDAoe7qzJ4ETcJm1wm6mz6fgYbrY6nk4aI9d+Yhh3
+NzGciFmrCaKRrhmaqrUA7RZniByf7LPmtRvD8nV5945eN1V7MivgT/YHU9nTm+os
+v1/Ujdr5+WBlf5Tv5DvvLfieqsZUL3HV46khK46qj21Zb0zRrIzzhIuL+OqrLURv
+c2NzIlRve4NTB5rZb3S6wqvf8JRseIrzWboLkvY7WTj5IGosil3kbDfpWz7R+/8D
+UEsHCPbR4UERDwAA47gAAFBLAwQUAAgICAB1mi1PAAAAAAAAAAAAAAAACwAAAGNv
+bnRlbnQueG1s3VrNjts2EL73KQwF7U2SJdnxT9cbFCl6aDdFkE2LXmmJkplQokBS
+a7sv0WPfr0/SIfVjSra8ctYtWu9hFyK/IWe+GQ5niL17s0vp6AlzQVi2sjxnbI1w
+FrKIZMnK+uXjD/bcenP/1R2LYxLiZcTCIsWZtEOWSfg7AulMLMvZlVXwbMmQIGKZ
+oRSLpQyXLMdZLbU00Uu9Vzki5J4OFtdgU1rinRwqrLAtWbQevrMGm9IRR9uhwgoL
+pJriMRsqvBPUjhmwnuZIko4WO0qyzytrI2W+dN3tdutsA4fxxPUWi4WrZxuFwwaX
+F5xqVBS6mGK1mXA9x3NrbIolGqqfwpoqZUW6xnwwNUiiI6/mHAuAgLkqMIctZMq0
+4uspGRxdT0kPzeEG8cFxpsHtUAmi4aESRKZsiuSmx79z9x1M6l/vHg5xxdOheyls
+i6qQk3ywmSXalGeMNaoqgfKwa3X98Xjilt8GensWvuVEYm7Aw7PwENGwYZylp0gD
+nOcCwsZPKuSbQ6SIED0CvltON2AR9S7927uHx3CDU3QAk+fBNsmERNmBGZESOtgL
+gO0JWpSRwaGgsEdHh6tg6GV86nKcMy4bB8XDLwHYxW842siU9qcwNVtDEx5FJ6Gg
+TuBCOoNkYj8RvH1ltW6n84G56ASmTvXPiWiQeRecFfDGrsI06QRC9XBx8aS5W2NW
+ZFHph5JAvMsxJ2oKUS22bK1geouyL1iyuo+NFVqXFMG0zlyNSSeXYcxOBQQyHFiW
+Lw3p9p3H092w5dQhZVHcXbGTsEIhAnkqHj5+cNWcra58uNSqnYxSx7fu67qmzGPC
+bQZiqG/sGIXYjnBIxf1deT81w6PyW+m9sh4IXHaa3tEjyiBLwCVSQ+Fo7lfWNyhn
+4tsOrhy0Rq2lFd5OcAZEQKrjLEVZC5ETGcJl8IQ40dHnDlcNlowH6aaA/4Zy3+NP
+6NfiPGcGZoBKYi8kTl+i0w8c4x6FjKl/UoOfmWTa3tHbH38aPb4dfcBJQRHvpahP
+4Gp8uX2HohpHhWTq5ghtvU5zWvTvlm1R7jWbVTZUdbGdowRbtaQ5aOdw+DGXBItR
+q8Bbo/BzwlVqg3wviC7kJS8gI/fB2PoTDqU4C4+IyCnaQ7nNpCphz0C0dnWpGyMq
++oCQe7EtSVrvqDg1KDrHl389vmptNhhFz1j2nzA+4UfBknCUb0jY2F19myaXEzAg
+wdUQ4eT3C7bMj7c0barncsRVE6w/7DqXxaigEgyVFJ/TL2bLlGTgBJJsQL/AmY5f
+h+klKh6FxIUqimL9rJYqoODypqqupAw8/CrWP1ZXf893vNniMgOCFxqQgW/FtbSf
+zvxLtH9/HCCII60DaFRWT5dodLSxfqGgRFSmt29zzzIBFFoYWjG0LijFclROqnEw
+zio/yylbtaQr668//2iYMxYxVNUyiiSK1jCzJZHqPsdOFaWlqAJ1orp1Mz1KxB/3
+6ZrRmq5CYFgL2oCtrZEVD2WGqcXVaV1Zk+nXmpdzll7Ag/8CHkSurrs1hqoT1yTc
+HkPB1RiCqvomGZpckaH5TTI0vRpDvjO5SYZeX42h4Cb5mV2PnxvN0/OrMTS50Ty9
+uCJDt5mnvfHVKJr+vxK1MV3V2m7vA0Y1sWbRvvkwu5T7O13Rqya4rO2r3ga+oezW
+I2bnot899GiKBHTWVfdsdjWd7tn8KNHQS7NCVkLfPXgfx1alRcxhrC1ubq6bWg2E
+JVS3Xq5UvidV3vKnjrdQbZwe7DanenCnapdJ/bVXHp4HM/Xd2jmkSAhwU9Vh6n21
+89dsV4Vpfo9EVLkjBy90MO7BqKEG+pWBehFz6v2Xmt70tadtnzgLb9Jne9Ngd1xK
+4fBsGD08vnTocTu2t6R103sqsnzLCEZbbop0nSFCj6HJgDjwAmfxen5ExtiZzMxA
+CJy5Z7ABXPm6JjnoUb8Meacp0q9X7lD3Bi9wrzdz/Jl3wr1l419bBCZMF4ZJQEQl
+dkr98gXihe51j/1bIxQ9Hf8LLCXJks6bXspUmmzeM8t3OOPJtp2z3FZac3v+qeP+
+b1BLBwhTBnGq+AUAABUiAABQSwMEFAAICAgAdZotTwAAAAAAAAAAAAAAAAwAAABz
+ZXR0aW5ncy54bWzdWl1z2jgUfd9fkfF0Z9oHYiBpEphAB0homNKEDaS73TdhX0Ab
+WfJIcgj76/fKGIaCocS2dnY2D/mwpXOOpKv7Fa4/vQbs5AWkooI3nMpp2TkB7gmf
+8mnDeRp1S1fOp+Yv12IyoR7UfeFFAXBdUqA1DlEnOJ2r+vJ1w4kkrwuiqKpzEoCq
+a68uQuCrafXN0fWYbPnklVH+3HBmWod1153P56fzs1Mhp26lVqu58dvV0FCCQiii
+Y8HHEW7O2aT1BJ/Q6bEoy9Gb84UQa9FmwnJhsfBquXzuLv9ejVYBZcdymbElTwQh
+ah6zH/aKcBocC2PG7qx6qeqw9tpae3JYGyZSdZore1iZQfM62ZzljxLVEBgbOUke
+G40NBynrLxTma+tx0ub9OOcbVWYDWhLISITO6qVehPiScu00S5Xq1cW1uwv0JvA+
+THQq+nn5PDf479TXszT0aq18cZkb/g7odJaqvnp2eVY7Fr8UkLBEuQ+v4G9zwTz9
+sOI5aG9ycYximPf8LZlKS7QEp2nsopJtJz5L6vdUsh9b8GMhGBDuNCeEKciD35WC
+b+9xEeg9NeQkHAnDsg9eyygn+oBM4SuRU8qVPRLzvU85WKR4GP8Fnu5KfGibZCDw
+Cu1dSq4TH7BI3RHuM1AtNicLi8ZrDBd/uZFkOsSLxmwczi2jAeVEw0CwRbxtfXyg
+W4YuzStVPpbLWVezy2XjiJLz6JMFhr1tfKLg4ryNIuTCabpHf101smkZoIPU5L+i
+pi+8Z/B/LsX9OdS9aGn0/mMbB3gvOoKJXYn5jf0xYiBthhvjqr9gGN4OlDMh8e5k
+vDhDYOjSwDfghQL3VGwLX4Vvx3vdiAi3ucOo9zyCV33r071BOI/jjwk6M8Kn8CiW
+hULxNENGfVADkI9inn4IGVNNsyspJxA724zn+v/Pu6/s5t3nGTfe5IEdQaSC/eo/
+5gc/oD0Xehfzv0PC8yEfVJ0d2qR9seg/7rF6lkQLaZHjBovtOIdJZ8lRBa05vv8L
+6/huZx09FeeNhuaWm7Rnb1WUI64YdAvp6Z9CBA88JcYeETxSHm4W1/teJ/V6+gAF
++vjOzPJBJOPg95YWTSsM2eJJgbwhmhQfNdtU40JHxhSeHvt72gfv3mNlqz+4akYk
+uCFhKB9+PWu/ex+hsA8Jsas0Vl1E+qdKjLOpwRxBEsykZEcEpq1ommIjlFJoUhXn
+roWv2Msm5oaoWeFa/IxaYEIiplHOUKdnJZXqZcZNvw3G4HdFUQXl/m5aV8gx9X3g
+a1vK31vrY+YapeT2yQEBz2qIkeHfg/o0zJhCEUnJTjMtQT2icGzDlHLTaMqMcMv9
+g/PzOuOfB1LiUwQr/F5Ns52IaUaJSKs7IenfeAUIK96P3xHtFe9IZlmzjC8geUuh
+HQ4i7unoYMmXr9ln+jdtIZ5xMdYJ2sR7tlEfb3Ic7ITnJsEkwk4nYYkuydxYmi31
+XaoPpX/5F5DcVIsEcWgyi7DSxo857oXej55/CQ+RZrvevUACszv3B/4FkZ9hRBnY
+NaQRtaLfhFWMroX7+axyBPEfgfiCs+0spqheLZbaY5CY1AUktTORsdYeYG44lSSc
+DaMgsBWdYkv4LSKM6u3tydM8jGFB9jAdMp9JwESnTxa7XmtlDUzMS1hKCRaZZeai
+TLmVx2eFCcYQdLRdW7z1fxtD8gLflh+ZeOAdJpSNu7buJ6cHhaQGzdijfQp9jMQY
+64MRBCE7EJXf1tOIexLuzudH3H2fMGr+A1BLBwgiJiQnHgUAAKMkAABQSwMEFAAI
+CAgAdZotTwAAAAAAAAAAAAAAABUAAABNRVRBLUlORi9tYW5pZmVzdC54bWytU8Fu
+wyAMvfcrIu6BracJJe1h0r6g+wBGnBQJDAJTtX8/EqltpilTs+2EjZ/fexi52Z+d
+rU4Qk/HYsmf+xCpA7TuDQ8veD2/1C9vvNo1TaHpIJK9BVfow3dKW5YjSq2SSROUg
+SdLSB8DO6+wASX7Fy0npls0MbNluU931emOhLv3xckf32do6KDq2TCyR3K8ddEbV
+dAnQMhWCNVpRgYkTdnwyzOc+eYiQyjlhmFjj5XDM7gOVsUnQNeQBhwUvxqkBxFhf
+peKAFC/DXGAlOJMYy6tIXz32ZshxenXaCqU1WCipj0LnGMfBLGv+TevBr0oZRws8
+G67nDOvEE10spH+fX3FEP8/od7wJiMomPmq4Ed/2dPcJUEsHCD41jEAVAQAA4gMA
+AFBLAQIUABQAAAgAAHWaLU8zJqyoLwAAAC8AAAAIAAAAAAAAAAAAAAAAAAAAAABt
+aW1ldHlwZVBLAQIUABQAAAgAAHWaLU9GMcZYxQcAAMUHAAAYAAAAAAAAAAAAAAAA
+AFUAAABUaHVtYm5haWxzL3RodW1ibmFpbC5wbmdQSwECFAAUAAgICAB1mi1PqVOc
+rLkBAAAoBAAACAAAAAAAAAAAAAAAAABQCAAAbWV0YS54bWxQSwECFAAUAAgICAB1
+mi1PAAAAAAIAAAAAAAAAJwAAAAAAAAAAAAAAAAA/CgAAQ29uZmlndXJhdGlvbnMy
+L2FjY2VsZXJhdG9yL2N1cnJlbnQueG1sUEsBAhQAFAAACAAAdZotTwAAAAAAAAAA
+AAAAABoAAAAAAAAAAAAAAAAAlgoAAENvbmZpZ3VyYXRpb25zMi90b29scGFuZWwv
+UEsBAhQAFAAACAAAdZotTwAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAzgoAAENv
+bmZpZ3VyYXRpb25zMi9mbG9hdGVyL1BLAQIUABQAAAgAAHWaLU8AAAAAAAAAAAAA
+AAAYAAAAAAAAAAAAAAAAAAQLAABDb25maWd1cmF0aW9uczIvbWVudWJhci9QSwEC
+FAAUAAAIAAB1mi1PAAAAAAAAAAAAAAAAHwAAAAAAAAAAAAAAAAA6CwAAQ29uZmln
+dXJhdGlvbnMyL2ltYWdlcy9CaXRtYXBzL1BLAQIUABQAAAgAAHWaLU8AAAAAAAAA
+AAAAAAAaAAAAAAAAAAAAAAAAAHcLAABDb25maWd1cmF0aW9uczIvcG9wdXBtZW51
+L1BLAQIUABQAAAgAAHWaLU8AAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAK8LAABD
+b25maWd1cmF0aW9uczIvcHJvZ3Jlc3NiYXIvUEsBAhQAFAAACAAAdZotTwAAAAAA
+AAAAAAAAABgAAAAAAAAAAAAAAAAA6QsAAENvbmZpZ3VyYXRpb25zMi90b29sYmFy
+L1BLAQIUABQAAAgAAHWaLU8AAAAAAAAAAAAAAAAaAAAAAAAAAAAAAAAAAB8MAABD
+b25maWd1cmF0aW9uczIvc3RhdHVzYmFyL1BLAQIUABQACAgIAHWaLU/20eFBEQ8A
+AOO4AAAKAAAAAAAAAAAAAAAAAFcMAABzdHlsZXMueG1sUEsBAhQAFAAICAgAdZot
+T1MGcar4BQAAFSIAAAsAAAAAAAAAAAAAAAAAoBsAAGNvbnRlbnQueG1sUEsBAhQA
+FAAICAgAdZotTyImJCceBQAAoyQAAAwAAAAAAAAAAAAAAAAA0SEAAHNldHRpbmdz
+LnhtbFBLAQIUABQACAgIAHWaLU8+NYxAFQEAAOIDAAAVAAAAAAAAAAAAAAAAACkn
+AABNRVRBLUlORi9tYW5pZmVzdC54bWxQSwUGAAAAABAAEAA2BAAAgSgAAAAA
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: 7z
+
+N3q8ryccAAReYyLCBQAAAAAAAABSAAAAAAAAAAwZK5YBAAAxAAEEBgABCQUABwsB
+AAEhIQEADAEACAoBt+/cgwAABQEZDAAAAAAAAAAAAAAAABENADEALgB0AHgAdAAA
+ABQKAQA9zwXiaWrVARUGAQAgAAAAAAA=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: vsd
+
+0M8R4KGxGuEAAAAAAAAAAAAAAAAAAAAAPgADAP7/CQAGAAAAAAAAAAAAAAABAAAA
+AgAAAAAAAAAAEAAABQAAAAEAAAD+////AAAAAAMAAAD/////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+///////////////////////////////////////////9/////v//////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+/////////////////////1IAbwBvAHQAIABFAG4AdAByAHkAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWAAUA////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/v///wAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAD///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+UgBvAG8AdAAgAEUAbgB0AHIAeQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAABYABQD//////////wIAAAAUGgIAAAAAAMAAAAAAAABG
+AAAAAAAAAAAAAAAAwL0Ei3dPzwEGAAAAgAIAAAAAAABWAGkAcwBpAG8ARABvAGMA
+dQBtAGUAbgB0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+HAACAf////8EAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAACQAAAAcXAAAAAAAAAUAUwB1AG0AbQBhAHIAeQBJAG4AZgBvAHIAbQBhAHQA
+aQBvAG4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAIBAQAAAAMAAAD/////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAACw2AAAAAAAA
+BQBEAG8AYwB1AG0AZQBuAHQAUwB1AG0AbQBhAHIAeQBJAG4AZgBvAHIAbQBhAHQA
+aQBvAG4AAAAAAAAAAAAAADgAAgH///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAEAIAAAAAAAD//////////wQAAAD9////
+/v////7///8HAAAA/v///wkAAAAKAAAACwAAAAwAAAANAAAADgAAAA8AAAAQAAAA
+EQAAABIAAAATAAAAFAAAABUAAAAWAAAAFwAAABgAAAAZAAAAGgAAABsAAAAcAAAA
+HQAAAB4AAAAfAAAAIAAAACEAAAAiAAAAIwAAAP7///8lAAAAJgAAACcAAAAoAAAA
+KQAAACoAAAArAAAALAAAAC0AAAAuAAAALwAAADAAAAAxAAAAMgAAADMAAAA0AAAA
+NQAAADYAAAA3AAAAOAAAADkAAAA6AAAAOwAAADwAAAA9AAAAPgAAAD8AAABAAAAA
+QQAAAEIAAABDAAAARAAAAEUAAABGAAAARwAAAEgAAABJAAAASgAAAEsAAABMAAAA
+TQAAAE4AAABPAAAAUAAAAFEAAABSAAAA/v//////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+/////////////////////1YAaQBzAGkAbwBJAG4AZgBvAHIAbQBhAHQAaQBvAG4A
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAiAAIA////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABwAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAD///////////////8AAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////////////////
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+/v///wIAAAADAAAABAAAAAUAAAAGAAAABwAAAAgAAAAJAAAA/v//////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////
+///////////////////////////////////////////+/wAABAACAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+/v8AAAYBAgAAAAAAAAAAAAAAAAAAAAAAAgAAAALVzdWcLhsQk5cIACss+a5EAAAA
+AAAEAAAApXTr8DTv9OrxA+vwAhbk9yFA6PMmDgTp8gIB+gIFC+vwzJ/MA76vVwAA
+DuvwQS0CRC9+wgPMNwAKOwLq8QEB3PQAAAEAAAADAAAAAAAldOvwNO/06vEC3P/d
+/vT+8f71C+vw9H3CA18XWAAACuvwQS0Cn/ylwAMhNwD+8UEC6fIBEAQUAAAASgEA
+AAAAAAAAAAAAAAAAAA8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAQAAAAPR00APWVwAAQQAAAFIAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAACx10AMjWAAANgAA
+AFIAAAAAAAVI6/A83P8EDxYPKA4FSOvwPNz/BA8WDygOFAAAAEoBAAAAAAAAAAAA
+AAAAAAAPAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAEUAAABkddADh1kAAA0AAABSAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEUAAADsb9ADlFkAAA0AAABSAAAAAAB7
+2gLr8AEAAM7z8E7g+wEAA+vwCgUE6/B9CevwdwEBACLc/+A0D0YPWA/5/uT3FBoA
+8TCGAWgP6PMolvxi/0mVYORAKGg6e2JfqwLlDIpcqwLOpw5gTdMaAeL5EG2pywoB
+6fIV5/QW6/Dkr0DaA4Dr8FTr8EL7ABfr8CRi0APU6uvwIOvwUAsSPH/QowP4FRYv
+H0EfGuvwdPojEBzz8EYKAADS7EIfdh8AHevwpHrQvwPZOQAAguvwUvsAJIMBe9AD
+WzqnAAAcKx/p8iHr8NzSjxB7pRAYEykNEXfQnwOiOwAA2xCZECf66/Bc1xC7TAAA
+aOoZEjHr8MzXECdNAHMAo+vwZB8AANjr8Hu0duoQUgAA0PPw11AAMuvwRB8gOlZ3
+AACA8/BUAD/r8H18HyBZWAAAKici/UTr8Jx10AOhWVpIJf9VEXHQ6QZgCCIAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+
+--XXX
+Content-Type: image/unknown
+Content-Transfer-Encoding: base64
+X-Real-Type: png
+
+iVBORw0KGgoAAAANSUhEUgAAABQAAAASCAYAAABb0P4QAAAMSWlDQ1BJQ0MgUHJv
+ZmlsZQAASImVVwdYU8kWnltSSWiBCEgJvYlSpEsJoUUQkCrYCEkgocSQEETsLssq
+uHYRARu6KqLoWgBZK+paF8HuWh6KqCjrYsGGypsUWNf93nvfO9839/45c85/Sube
+OwOATg1PKs1FdQHIkxTI4iNCWJNS01ikLoADGtADFsCFx5dL2XFx0QDK0P3v8vYG
+QJT3qy5Krn/O/1fREwjlfACQOIgzBHJ+HsQHAcBL+FJZAQBEH6i3nlkgVeIpEBvI
+YIIQS5U4S41LlDhDjStVNonxHIh3A0Cm8XiyLAC0m6GeVcjPgjzatyB2lQjEEgB0
+yBAH8kU8AcSREI/Ky5uhxNAOOGR8xZP1N86MYU4eL2sYq2tRCTlULJfm8mb9n+34
+35KXqxiKYQcHTSSLjFfWDPt2K2dGlBLTIO6VZMTEQqwP8XuxQGUPMUoVKSKT1Pao
+KV/OgT0DTIhdBbzQKIhNIQ6X5MZEa/QZmeJwLsRwhaBF4gJuosZ3sVAelqDhrJHN
+iI8dwpkyDlvj28CTqeIq7U8rcpLYGv5bIiF3iP9NsSgxRZ0zRi0UJ8dArA0xU56T
+EKW2wWyKRZyYIRuZIl6Zvw3EfkJJRIiaH5uWKQuP19jL8uRD9WKLRWJujAZXFYgS
+IzU8u/k8Vf5GEDcLJeykIR6hfFL0UC0CYWiYunasXShJ0tSLdUoLQuI1vq+kuXEa
+e5wqzI1Q6q0gNpUXJmh88cACuCDV/HiMtCAuUZ0nnpHNGx+nzgcvAtGAA0IBCyjg
+yAAzQDYQt/U29cJf6plwwAMykAWEwEWjGfJIUc1I4DUBFIM/IBIC+bBfiGpWCAqh
+/vOwVn11AZmq2UKVRw54DHEeiAK58LdC5SUZjpYMHkGN+B/R+TDXXDiUc//UsaEm
+WqNRDPGydIYsiWHEUGIkMZzoiJvggbg/Hg2vwXC44z6471C2f9kTHhM6CA8J1wmd
+hNvTxYtk39TDAhNAJ4wQrqk54+uacTvI6omH4AGQH3LjTNwEuOBjYSQ2HgRje0It
+R5O5svpvuf9Ww1dd19hRXCkoZQQlmOLwrae2k7bnMIuyp193SJ1rxnBfOcMz38bn
+fNVpAbxHfWuJLcYOYGexk9h57AjWBFjYcawZu4QdVeLhVfRItYqGosWr8smBPOJ/
+xONpYio7KXetd+1x/aSeKxAWKd+PgDNDOksmzhIVsNjwzS9kcSX80aNY7q5uvgAo
+vyPq19Rrpur7gDAv/KXLPwGAbxlUZv2l41kDcPgxAIy3f+msX8HHYwUAR9v5Clmh
+WocrLwRABTrwiTIG5sAaOMB63IEX8AfBIAyMB7EgEaSCabDLIrieZWAmmAMWglJQ
+DlaAtaAKbAJbwU6wB+wHTeAIOAl+BRdBO7gO7sDV0w2egz7wFgwgCEJC6AgDMUYs
+EFvEGXFHfJBAJAyJRuKRVCQdyUIkiAKZg3yHlCOrkCpkC1KH/IwcRk4i55EO5Dby
+AOlBXiEfUQyloQaoGWqHjkF9UDYahSaiU9EsNB8tRkvQZWglWovuRhvRk+hF9Dra
+iT5H+zGAaWFMzBJzwXwwDhaLpWGZmAybh5VhFVgt1oC1wP/5KtaJ9WIfcCLOwFm4
+C1zBkXgSzsfz8Xn4UrwK34k34qfxq/gDvA//QqATTAnOBD8ClzCJkEWYSSglVBC2
+Ew4RzsCnqZvwlkgkMon2RG/4NKYSs4mziUuJG4h7iSeIHcQuYj+JRDImOZMCSLEk
+HqmAVEpaT9pNOk66QuomvSdrkS3I7uRwchpZQl5EriDvIh8jXyE/IQ9QdCm2FD9K
+LEVAmUVZTtlGaaFcpnRTBqh6VHtqADWRmk1dSK2kNlDPUO9SX2tpaVlp+WpN1BJr
+LdCq1NqndU7rgdYHmj7NicahTaEpaMtoO2gnaLdpr+l0uh09mJ5GL6Avo9fRT9Hv
+099rM7RHa3O1Bdrztau1G7WvaL/QoejY6rB1pukU61ToHNC5rNOrS9G10+Xo8nTn
+6VbrHta9qduvx9Bz04vVy9NbqrdL77zeU32Svp1+mL5Av0R/q/4p/S4GxrBmcBh8
+xneMbYwzjG4DooG9Adcg26DcYI9Bm0Gfob7hWMNkwyLDasOjhp1MjGnH5DJzmcuZ
++5k3mB9HmI1gjxCOWDKiYcSVEe+MRhoFGwmNyoz2Gl03+mjMMg4zzjFeadxkfM8E
+N3EymWgy02SjyRmT3pEGI/1H8keWjdw/8ndT1NTJNN50tulW00um/WbmZhFmUrP1
+ZqfMes2Z5sHm2eZrzI+Z91gwLAItxBZrLI5bPGMZstisXFYl6zSrz9LUMtJSYbnF
+ss1ywMreKslqkdVeq3vWVGsf60zrNdat1n02FjYTbObY1Nv8bkux9bEV2a6zPWv7
+zs7eLsXuB7smu6f2RvZc+2L7evu7DnSHIId8h1qHa45ERx/HHMcNju1OqJOnk8ip
+2umyM+rs5Sx23uDcMYowyneUZFTtqJsuNBe2S6FLvcuD0czR0aMXjW4a/WKMzZi0
+MSvHnB3zxdXTNdd1m+sdN3238W6L3FrcXrk7ufPdq92vedA9wj3mezR7vBzrPFY4
+duPYW54MzwmeP3i2en728vaSeTV49XjbeKd713jf9DHwifNZ6nPOl+Ab4jvf94jv
+Bz8vvwK//X5/+rv45/jv8n86zn6ccNy2cV0BVgG8gC0BnYGswPTAzYGdQZZBvKDa
+oIfB1sGC4O3BT9iO7Gz2bvaLENcQWcihkHccP85czolQLDQitCy0LUw/LCmsKux+
+uFV4Vnh9eF+EZ8TsiBORhMioyJWRN7lmXD63jts33nv83PGno2hRCVFVUQ+jnaJl
+0S0T0AnjJ6yecDfGNkYS0xQLYrmxq2PvxdnH5cf9MpE4MW5i9cTH8W7xc+LPJjAS
+pifsSnibGJK4PPFOkkOSIqk1WSd5SnJd8ruU0JRVKZ2TxkyaO+liqkmqOLU5jZSW
+nLY9rX9y2OS1k7uneE4pnXJjqv3Uoqnnp5lMy512dLrOdN70A+mE9JT0XemfeLG8
+Wl5/BjejJqOPz+Gv4z8XBAvWCHqEAcJVwieZAZmrMp9mBWStzuoRBYkqRL1ijrhK
+/DI7MntT9ruc2JwdOYO5Kbl788h56XmHJfqSHMnpGeYzimZ0SJ2lpdLOfL/8tfl9
+sijZdjkinypvLjCAG/ZLCgfF94oHhYGF1YXvZybPPFCkVyQpujTLadaSWU+Kw4t/
+mo3P5s9unWM5Z+GcB3PZc7fMQ+ZlzGudbz2/ZH73gogFOxdSF+Ys/G2R66JVi958
+l/JdS4lZyYKSru8jvq8v1S6Vld78wf+HTYvxxeLFbUs8lqxf8qVMUHah3LW8ovzT
+Uv7SCz+6/Vj54+CyzGVty72Wb1xBXCFZcWNl0Mqdq/RWFa/qWj1hdeMa1pqyNW/W
+Tl97vmJsxaZ11HWKdZ2V0ZXN623Wr1j/qUpUdb06pHpvjWnNkpp3GwQbrmwM3tiw
+yWxT+aaPm8Wbb22J2NJYa1dbsZW4tXDr423J287+5PNT3XaT7eXbP++Q7OjcGb/z
+dJ13Xd0u013L69F6RX3P7im72/eE7mlucGnYspe5t3wf2KfY9+zn9J9v7I/a33rA
+50DDQduDNYcYh8oakcZZjX1NoqbO5tTmjsPjD7e2+Lcc+mX0LzuOWB6pPmp4dPkx
+6rGSY4PHi4/3n5Ce6D2ZdbKrdXrrnVOTTl07PfF025moM+d+Df/11Fn22ePnAs4d
+Oe93/vAFnwtNF70uNl7yvHToN8/fDrV5tTVe9r7c3O7b3tIxruPYlaArJ6+GXv31
+Gvfaxesx1ztuJN24dXPKzc5bgltPb+fefvl74e8DdxbcJdwtu6d7r+K+6f3afzn+
+a2+nV+fRB6EPLj1MeHini9/1/JH80afuksf0xxVPLJ7UPXV/eqQnvKf92eRn3c+l
+zwd6S//Q+6PmhcOLg38G/3mpb1Jf90vZy8FXS18bv97xZuyb1v64/vtv894OvCt7
+b/x+5wefD2c/pnx8MjDzE+lT5WfHzy1for7cHcwbHJTyZDzVVgCDA83MBODVDgDo
+qXDv0A4AdbL6nKcSRH02VSHwn7D6LKgSLwB2BAOQtACAaLhH2QiHLcQ0eFdu1ROD
+AerhMTw0Is/0cFdz0eCJh/B+cPC1GQCkFgA+ywYHBzYMDn7eBpO9DcCJfPX5UilE
+eDbYPEaJ2rufUMA38m9eX39+ZKan3gAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAAC1J
+REFUOBFj1NLW/c9ARcBERbPARo0aSHmIjobhaBiSEQKjyYaMQEPTQvUwBAD4XQGl
+r/6/wAAAAABJRU5ErkJggg==
+
+--XXX
+Content-Type: image/unknown
+Content-Transfer-Encoding: base64
+X-Real-Type: jpg
+
+/9j/4AAQSkZJRgABAQAAAQABAAD//gA8Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1
+c2luZyBJSkcgSlBFRyB2NjIpLCBxdWFsaXR5ID0gMTAwCv/bAEMAAwICAgICAwIC
+AgMDAwMEBgQEBAQECAYGBQYJCAoKCQgJCQoMDwwKCw4LCQkNEQ0ODxAQERAKDBIT
+EhATDxAQEP/bAEMBAwMDBAMECAQECBALCQsQEBAQEBAQEBAQEBAQEBAQEBAQEBAQ
+EBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEP/AABEIABwAFwMBEQACEQEDEQH/
+xAAZAAEAAgMAAAAAAAAAAAAAAAAFAQMEBgn/xAAgEAABBQACAgMAAAAAAAAAAAAC
+AAMEBRIBBhMUIkJS/8QAGAEBAQEBAQAAAAAAAAAAAAAAAAIDAQT/xAAbEQEAAgMB
+AQAAAAAAAAAAAAAAAQIDETESIf/aAAwDAQACEQMRAD8A6P8Aeu2XHWG2DroEd4ZW
+wFx0i+Lv1HK8mTLaeNa122GlmzLGrYnTofqPOhomtaytMWSZ6i9dfCa3SKuaqtt4
+3r2kMH2QMXBEv0p1WeqrfTFO4xI8PCuPMIvcrGc8oCa46udHY5QCOU25HmQMMsct
+N44QXoIzwglB/9k=
+
+--XXX
+Content-Type: image/unknown
+Content-Transfer-Encoding: base64
+X-Real-Type: gif
+
+R0lGODlhFwAcAPfIALexo766rr+4ssK8rsK5sMO/s8TBssbAssjCssjAtcjBt8vF
+t8vFucrDvc7Ius7IvNHJvNLMvtHMyN/XzN/WzdzZyt7YytnU0d/b0OHd0uPf1OLf
+1uPe2+bg1Obh3uvp3PHo3+Ln4+Pl5OXn5ubm5ufn5+Tp5ebo5+/r4O7q4eXp6OXr
+6efr6uft6+bv7Ojo6Onp6enr6urq6uvr6+js6+rs6+vt7Oju7uvv7uzs7O3v7u7u
+7u/v7/Pt4fLv5uvx7+7w7fXz5/b06Pf06/j26vjz7/n27/327uvv8O7v8evx8e/x
+8Ozy8u7y8+/z9vDw8PHx8fDy8fHx8/Hz8vLy8vPz8/bw8vL08fX18/f28vPy9/Lz
+9/Xz9vD09fP19PT09PX19fb29Pf19vb09/b29vf39/z58P/78Pz58v778v//8/v6
+9fj49vn59/r59/779P/59vz79//69v/69/r89/399f/99P7+9P/+9f//9f389/7+
+9v//9v/+9///9/H1+PP0+Pf1+PX2+Pf3+fj2+fv3+Pv2+vf4/Pj4+Pn5+fr4+fr6
++Pv7+fj4+vj5+/n5+/v5+vr6+vv7+/35+Pz4+fz7+f/7+P76+f74+vz6+//7+vv9
++Pr8+f38+P79+P79+f3++f//+P/++fz8+v38+v39+//9+v79+/3/+vz++//++v7/
++v7/+///+/v5/Pr7/fv7/f37/P/7/P/6/vn9/vn9//v8/vr+//v///z8/P39/f/9
+/P3//Pz+/f7+/P/+/P///fz8/v39///9/v/9//7+/v7+//7//////wAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAACH5BAAAAAAAIf46Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1
+c2luZyBJSkcgSlBFRyB2NjIpLCBxdWFsaXR5ID0gMTAwCgAsAAAAABcAHAAACP4A
+hRkjNuzXpzWiTAEbNlAYLVzHkEmcaOyRGDedRInqI+pTJUmxgt2COHHisEy/VInC
+Y8cHHz8dfemqVStiSWSksugxskEDhgIfhOwRpaqULVs3JfqpYyZDgAMOABioEOTO
+HlAykyLLg0cNigcLIiAYYIEInj2eWvnSOiQNnjMdGCSAMKGHmjyhRK1ipLUPmjd9
+UiggQAHEkT5+PIlqtUirqjiWTBWRIKABHDmWLJEilauXJF3FdOUC/euUpl2XPFzg
+MGfSpUyQYL16JWm0aNDFhskaNqyQFUyYKBkiNOjQIUeRPt8uJjGWIt6zZg0fo2XL
+H0CCGkXKxR10Kl+jyPmAdkWIEBcpSZAwcdJFEKLaukBz4jWKDa9ijbgEoqKDBosb
+SjThBRnbxVcMHaigQscqo4RRRhVL2KCCCiu48MMUXyRiWzG8sMLLJr60gQUbVwAR
+wwghmNACDlFkqFwxxawFTDGSRIJIFTzIQIIIJ9SgQxVgIDJaaCUVk4uNVTyRwwsl
+wJADD2CQoSGMuhSpiySJgEHFDjPI8CQUZJSxHZVWSiJJGV9AwcMOT1BRBSKJfEbm
+RKFxh0gZYFRRxRdRJocbkRLBGJoukSRyZxmIwBkfaIEyh4ygMOZiZo2TakWnoMh0
+x92mlmoFKaSdJvUppqEWOaqjpaZ6U0AAOw==
+
+--XXX
+Content-Type: image/unknown
+Content-Transfer-Encoding: base64
+X-Real-Type: bmp
+
+Qk1qCAAAAAAAAIoAAAB8AAAAFwAAABwAAAABABgAAAAAAOAHAAAAAAAAAAAAAAAA
+AAAAAAAAAAD/AAD/AAD/AAAAAAAA/0JHUnOAwvUoYLgeFSCF6wFAMzMTgGZmJkBm
+ZgagmZkJPArXAyRcjzIAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAA////////
+////////////////////////////////////////////////////////////////
+////////////////////AAAA/////////////////////////////////v7+/v7+
+/v7+/v7+/v7+/v7+/v7+/v7+////////////////////////////AAAA////////
+/////////////////////////v7+/v7+/v7+/v7+/v7+/v7+/v7+/v7+////////
+////////////////////AAAA/////////////////////////////////v7+/v7+
+/v7+/v7+/v7+/v7+/v7+/v7+////////////////////////////AAAA////////
+/////v7+/v7+/v7+/v7+/////Pz8/Pz8/f39/Pz8/Pz8/Pz8/Pz8/Pz8////////
+////////////////////AAAA/v7+/////////v7+/v7+/v7+/v7+/v7+/Pz8+/v7
++/v7+/v7+vr6+/v7+/v7+/v7////////////////////////////AAAA////////
+/////v7+/v7+/v7+/f39/f39+vr6+fn5+Pj49/f39/f3+Pj4+Pj4+fn5/f39/f39
+/f39/v7+/////////v7+AAAA/////////////////v7+/f39/Pz8/Pz8+Pj49/f3
+9fX18/Pz8/Pz9PT09fX19vb2+vr6+/v7/f39/v7+/v7+/v7+/f39AAAA////////
+/////////v7+/f39+/v7+/v79/f39PT08fHx7+/v7u7u8PDw8vLy8/Pz+Pj4+fn5
++/v7/f39/v7+/v7+/f39AAAA/////////////////v7+/f39+/v7+fn59fX18vLy
+7u7u6+vr6urq7Ozs7+/v8fHx9vb29/f3+vr6/Pz8/v7+/v7+/f39AAAA////////
+/////////v7+/Pz8+vr6+Pj48/Pz8PDw7Ozs6Ojo5+fn6enp7Ozs7+/v9fX19vb2
++fn5/Pz8/v7+/v7+/f39AAAA/v7+/P7+/////f///v7++/v7+vr6+Pj48/Pz7+/v
+6urq5ubm5OXj5+jm6+zq7u/t8/Pz9fX1+Pj4/Pz8/f39/v7+/f39AAAA/P/9+//+
+/P/9+P37/P7+9/n58/X19vj48fTy7fDu6uvp5ufl4+fi5enk6+3n7u/r8fLw9PT0
++fn5+/v7/f39/f39/v7+AAAA9/z6+v/9+v/99/z6+v/++vz89Pb29/f38/Pz8PHv
+7O3r6Onl6Onl6evl7O/m7/Hr8vPx9PT0+fn5/Pz8/f39/f39/v7+AAAA+fz6/P/9
++vz89vj4/P/9/v7++vj49vP1+PX38vLy7u/t6+zo6uvn7u7o8fHr8/Lu9PXz9vb2
++vr6/Pz8/f39/f39/v7+AAAA+/78/P7++vz89vb2/f39/v7+/Pn7+fb4+fb49vP1
+8/Hx8e/u8O/r8vLs9vPv9fTw+Pb1+Pj4+/v7/Pz8/f39/f39/v7+AAAA////////
+/Pv9+fj6/v3//v3//vr//vr/+vb7+fb49/T29/Lz9/Py+PXx+PTz+Pb1+vj4+vr6
+/Pz8/Pz8/Pz8/f39/v7+AAAA/v7+/v3//Pv//v3//v3/+Pf78vD2+vj++vj++fj8
++vb7+fb4+ff3/Pj3/Pj3+/n4+vr6+/v7/f39/f39/Pz8/f39/v7+AAAA/P7/+/3+
++vv//P3/+fr+3uHm0dTZ297j9/r/+Pn9+fr++/r8+vn7/fv7/fv6/fv6+/v7/Pz8
+/f39/f39/Pz8/f39/v7+AAAA+v7/9/v8+Pv/+v3/7/P4yMzRsri/vcPK9vn/9vr/
++Pv/+Pv/+vz9+vz9/Pz8/f78+/v7/f39/v7+/f39/Pz8/f39/v7+AAAA9/7/8vn8
+9Pv+9/7/4erut8HIsLnCzdbf3+jx7vb99/7/9///+P3++f7/+///+Pr6////////
+////////////////////AAAA6/T38vv+9f7/8Pv/1ODmucXLtcDIvMnRzNff4e3z
+8///9f//+P//+f7/+v/++fv7////////////////////////////AAAA9f//9f7/
+8///4OvvvMjOt8XLvszSssLIrrzCytje6vb49f7/9v7++P3++////P7+////////
+////////////////////AAAA9///9f398Pn80t3hrrq+ssDGusjOo7G3ssHEytnc
+5/P19P7+9v7++f79/P7+/f39////////////////////////////AAAA+vz98vb3
+9/z97/b51t/i1N/j0Nvfs7/D3Onr6PT29v7++f7/+v7/+/39/vz7/vz7////////
+////////////////////AAAA/v3/+/r8/P7/+v7/+f7/9f7/9P3/5u/y9v//9///
++f7/+f3+/P7+/f39//35//35///+////////////////////////AAAA//7++/n5
+9vX39/n6+Pz9+f7/+f7/9/7/+f7/+f3++fv8+/v7/Pv9/vz8//76///7///+////
+////////////////////AAAA//39//7+//3//v3//P7/+f3+9fr7+f7/+v3//f//
+/v3///7+//3///39/v35///7///+////////////////////////AAAA
+
+--XXX
+Content-Type: text/plain
+Content-Transfer-Encoding: base64
+X-Real-Type: txt
+
+RWggYmllbiwgbW9uIHByaW5jZS4gR8OqbmVzIGV0IEx1Y3F1ZXMgbmUgc29udCBw
+bHVzIHF1ZSBkZXMgYXBhbmFnZXMsCmRlcyDQv9C+0LzQtdGB0YLRjNGPLCBkZSBs
+YSBmYW1pbGxlIEJ1b25hcGFydGUuIE5vbiwgamUgdm91cyBwcsOpdmllbnMsIHF1
+ZSBzaQp2b3VzIG5lIG1lIGRpdGVzIHBhcywgcXVlIG5vdXMgYXZvbnMgbGEgZ3Vl
+cnJlLCBzaSB2b3VzIHZvdXMgcGVybWV0dGV6CmVuY29yZSBkZSBwYWxsaWVyIHRv
+dXRlcyBsZXMgaW5mYW1pZXMsIHRvdXRlcyBsZXMgYXRyb2NpdMOpcyBkZSBjZXQK
+QW50aWNocmlzdCAobWEgcGFyb2xlLCBq4oCZeSBjcm9pcykg4oCUIGplIG5lIHZv
+dXMgY29ubmFpcyBwbHVzLCB2b3VzIG7igJnDqnRlcwpwbHVzIG1vbiBhbWksIHZv
+dXMgbuKAmcOqdGVzIHBsdXMg0LzQvtC5INCy0LXRgNC90YvQuSDRgNCw0LEsIGNv
+bW1lIHZvdXMgZGl0ZXMuWzFdINCd0YMsCtC30LTRgNCw0LLRgdGC0LLRg9C50YLQ
+tSwg0LfQtNGA0LDQstGB0YLQstGD0LnRgtC1LiBKZSB2b2lzIHF1ZSBqZSB2b3Vz
+IGZhaXMgcGV1cixbMl0g0YHQsNC00LjRgtC10YHRjCDQuArRgNCw0YHRgdC60LDQ
+t9GL0LLQsNC50YLQtS4K
+
+--XXX
+Content-Type: text/unknown
+Content-Transfer-Encoding: base64
+X-Real-Type: html
+
+PCFET0NUWVBFIGh0bWwgUFVCTElDICItLy9XM0MvL0RURCBIVE1MIDQuMDEvL0VO
+IiAiaHR0cDovL3d3dy53My5vcmcvVFIvaHRtbDQvc3RyaWN0LmR0ZCI+CjxodG1s
+Pgo8aGVhZD4KICA8bWV0YSBodHRwLWVxdWl2PSJDb250ZW50LVR5cGUiIGNvbnRl
+bnQ9InRleHQvaHRtbDsgY2hhcnNldD11dGYtOCI+CiAgPG1ldGEgaHR0cC1lcXVp
+dj0iQ29udGVudC1TdHlsZS1UeXBlIiBjb250ZW50PSJ0ZXh0L2NzcyI+CiAgPHRp
+dGxlPjwvdGl0bGU+CiAgPG1ldGEgbmFtZT0iR2VuZXJhdG9yIiBjb250ZW50PSJD
+b2NvYSBIVE1MIFdyaXRlciI+CiAgPG1ldGEgbmFtZT0iQ29jb2FWZXJzaW9uIiBj
+b250ZW50PSIxNjcxLjIiPgogIDxzdHlsZSB0eXBlPSJ0ZXh0L2NzcyI+CiAgICBw
+LnAxIHttYXJnaW46IDAuMHB4IDAuMHB4IDAuMHB4IDAuMHB4OyBmb250OiAxMi4w
+cHggSGVsdmV0aWNhfQogIDwvc3R5bGU+CjwvaGVhZD4KPGJvZHk+CjxwIGNsYXNz
+PSJwMSI+MTwvcD4KPC9ib2R5Pgo8L2h0bWw+Cg==
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: csv
+
+dGVzdCwxMjUsNzc3LDc3NwoxLDIsMyw0Cg==
+
+--XXX
+Content-Type: image/unknown
+Content-Transfer-Encoding: base64
+X-Real-Type: dwg
+
+QUMxMDE1AAAAAAAPAdwAAAAbGR0ABgAAAADvRgAAVQIAAAFESQAAWwIAAAJs8Q4A
+12oAAAMAAAAAAAAAAATXXA8ABAAAAAVhAAAAewAAAHN2laBOKJmCGuVeQeBfnTpN
+AP93ARsAGQACAAAAx8IOAAIAAAAAAAAAFwAGABcABgAFAJMIBQEoCgAAAQAAAAAA
+AAAAAAAAAAAAAQAAAQABAPxzJQCxYBIDCXQlAG5hZAOJMAAAAAAAAAAAAgAAAAAA
+AAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAB8lbQfUNigonVfKP51EECvvRQAA
+AgEDAQAAUAAAAAJTAQAAjEUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+Af4MAaZMAeANAYxOAa8NAf8CAcVBAYoDAZFGAYzXQAHcPQFgAVsB/3sBWQFZjhoA
+AgHQ1HshziiTn79TJEAJEjyqAXQAAAAQ1wPAEFDMTAxNQAAAAAADwEGxkdAEGADv
+RgAAFUCAAABERJAAAWwIAAAIbPEOADXagAAA6BKBVhXtDgIAMIkBAQEBAgIBAwMB
+BAUBBQYBBgcBBwgBCAkBCQoBCgsBCwwBDBcBDQ3Hc17groGU8vihK4TeMddsYECs
+27/27cNV/gAAAQA=
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: jar
+
+UEsDBAoAAAAAAP22VDwAAAAAAAAAAAAAAAAJAAAATUVUQS1JTkYvUEsDBAoAAAAA
+AP22VDwVa8g6KAAAACgAAAAUAAAATUVUQS1JTkYvTUFOSUZFU1QuTUZNYW5pZmVz
+dC1WZXJzaW9uOiAxLjAKQ3JlYXRlZC1CeTogMC45OAoKUEsDBAoAAAAAADw+UzwA
+AAAAAAAAAAAAAAACAAAAdC9QSwMECgAAAAgAHT5TPDEphtEFAAAAAwAAAAcAAAB0
+L3QudHh0MzHiAgBQSwECCgAKAAAAAAD9tlQ8AAAAAAAAAAAAAAAACQAAAAAAAAAA
+AAAAAAAAAAAATUVUQS1JTkYvUEsBAgoACgAAAAAA/bZUPBVryDooAAAAKAAAABQA
+AAAAAAAAAAAAAAAAJwAAAE1FVEEtSU5GL01BTklGRVNULk1GUEsBAgoACgAAAAAA
+PD5TPAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAgQAAAHQvUEsBAgoACgAAAAgA
+HT5TPDEphtEFAAAAAwAAAAcAAAAAAAAAAAAAAAAAoQAAAHQvdC50eHRQSwUGAAAA
+AAQABADeAAAAywAAAAAA
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: apk
+
+UEsDBBQACAAIADeUXDgyD/By+AAAAFkBAAAUAAQATUVUQS1JTkYvTUFOSUZFU1Qu
+TUb+ygAAbcxdT4JQHIDxeza+A5c1Ji+1qNi6ANN4USlxit64w+GvHD28dA4g8Omz
+1la5bp/t+U1RTnbAq8ESGCdFbkq6oonCkAGqIBnY3WcwFG17Y0hXVllSkNwcK9ei
+IAozlIEpWXnCCpJMvyGlzagohI6lD57J/hxM6XFMnGiRwRJwPI5vIec9Cmnkbkrt
+6cdhwFWKuqKu1BQoLbYIV6QhVfePaHu9ipzYX6gB0yYn39uoh3Ymr/ps9FcsaoaB
+K4hxfEEc09VJNqj3kOzQwUX+sYmH9dv9et27vwhMEednIIH24pfR/I6+hnNsNQGL
+5CDl7WgS2fr7y/7r/wBQSwcIMg/wcvgAAABZAQAAUEsDBBQACAAIADeUXDhy3Kc0
+DwEAAI4BAAAQAAAATUVUQS1JTkYvQ0VSVC5TRm3MW2uDMACG4XvB/5DLDbEHcC0K
+u7B24NCWqT1Qb0YaoybGpE10rf76lh2gLbv7+OB9ElJw2LQSmxssFRHcAePBSNc8
+iWGDM3PWfR/gyeWZFCQDyTVwD9WzriW+OzbnpMCqMReQk/w6HBBhvu+NlIbKnx38
+7d71F7syPOQEq1dd07UlrLEDfrW/bHCu2R3ogLYnRzIJakqLLuZCRFP6kvrvwWZ1
+unEkVkMGO9E2wxIzJj4hasgXabp/xFMpjI+O0h2N/eWkhktRWLblScN7uxdFKxFW
+AygVeiC2bJ6Owm1q9NKtz+tVkLa9bZdrkYobAjGo1BXI8Pmhj+gJRW6VW2SDqqq1
+jvm0ImGc8OynvwBQSwcIctynNA8BAACOAQAAUEsDBBQACAAIADeUXDhqGTzXgwQA
+ALIGAAARAAAATUVUQS1JTkYvQ0VSVC5SU0EzaGJbx8ap1ebR9p2XkZ1pQRPbfIMm
+ttlMjIyG3AacbKzafMxMUqwMBtwIRYwLmljWGDSxrDBoYp6wgJmJkYmJk2Fy3pp9
+7J8Y7xvwwhUysgL1NU4Bm8McysImzBQabChsIAjicAhzOSfmZKblF+VlJhqKGYiA
+BNmFeX3zS/NKEjPzFMIyU8sNBQz4QOJcwuyOeSlF+ZkpMBFuDBFmhIiSgQLcDZyM
+YsKJEHEHKK2XnJ9rICfOa2BhYGRkaWBobGxiFiXOa2xqYG5oDuUOVmc3MSogBzAj
+KwNzEyMvA1Ccg6mJkZHh2mRJlnvHuFU2vj2e9ODmDVU7u7M5r2Uf/WfrWfFiz4qz
+2Zcr8pZzLc95xrebf6Zp5P+bduUrnZ+/sPVO2fHk36LLz8I+PkpfIb1/k8HWikMs
+zvt8iiR2uH1VFG37YLHQr/PQPov6vn3n+08dviX32ODkq8kXlh++4/U5QIk3gIPd
+6EHDdPF3WazGkc+WTXmjs/lTy4IlaSeqptywNuS0SjfX008ROpCX6/QxQuL+P+bG
+MzxXXG7l3D28QyVC0oNxs2qKsP/+ezNOapR73P6anhXC2xhyYvcp9p2P3EONBY9k
+z/qef+/Nmb7M5ycWXeirUFOYYj+zqNaG5V/RTNmZ92evs1iwSbx+sWx01r+X8kyM
+zIsb/xg0/jSQBYauLB+LGIuIRyRDmK26jts6wTbWJSWSazhP9QgaNJ4EySuzNB40
+aNzXgFXNwsZZSxqnD9KU0ISU43hAXhFmYTVgZmT8j5b/mEGpomr9tJ7XAUccWQMl
+LtxavZ4xuqssSn256KJDW/yVRET/3zrHzBm1f4nud3aOohyFzKeH8+6uY2HYp+mq
+w+F9qG7bwro1c/dJ6Cj4bQwW/BJ6Q2Vb2O0nd5QcJuqWtv3rmCrLuOLf1nVRTgnB
+sT9MDFlVlNx6zB7paMX9nHJN9rpB1pOT36ZKLz6oLyspcveYvFTSLd0f6t+Y/y1N
+sNYJ4d1bwzjHbNemWU6FB8XvB9ncPvq5scrzwftlPHvrS8TrqvwlbZ2/KLEXpeX5
++DY/3BvVxlHz2V/3jeIj11M52hvEnjUnNwRcOmLwbvmhLBnPy2VcEavrpZrOTO72
+MLRwVtnLwvhLSMwqIJwvw9ewifEUMIscAxZqBo2LBmmMIpehSOUuam5naWJkmGyU
+O3G7ahi7kk96zqI/MhtSQo/ceH39V7Hm1MjjTtfuPT4SMfN10MPHs2wSYxsETUX4
+Zz2ZbFt+4MZDpRxZn2c8CzieLWB03MpiUcV3hXupq9bt3+t+Htf9rfvF8Afb2k9P
+/W+4yFWt5TQ1cuVdLni4ufSu46lvx091vJr2hLP4160F6X+VufdPrzB5Oc3y/HHj
+z3XsSgHS4ofNc+ZlM6fbzdlawTV7l3WP8Jk/thfXMb3STq3X4nuTO2eybizzsTkF
+84Ou6AqdPSjZNFuqQ8e9hVfpyL4PM6331ifxmCqLV9kLP1wgOOnFTDnV6P2tJvZ6
+K9etiy/n/NP6qlWromjq8d2Clx5k9vB3lZcuAQBQSwcIahk814MEAACyBgAAUEsD
+BBQACAAIADeUXDhS6J1KbwIAAFgGAAATAAAAQW5kcm9pZE1hbmlmZXN0LnhtbIWU
+zU4UQRSFT08DoswgIn/CqEPCykAP+Bc0bggxkQSNkUhcOmAjo8MwGVrUxIULly58
+AFeufBKfwJVP4DO4QL+qrsaiZyZU50xXnXvvufdWVU+oQT0bkAKV9T2UpvR/fPDm
+0+AmeAIOwSfwBXwDf8ARGAmkOfAK/AC/wBGYKkj3wGPQBJ/BV/AT/AZDKMZq60B1
+7aupVX5fwHRaHqmmPWsp8a7DbOD5WpsnvKhECZ5tvYRPevj0q4HPFmyDVR9cpn2G
+WZOYNp513tI17aKTqKW7qvIcaBsmxr/GPMr5R1j3sVXhW2SuYonxq3boymZrwW/j
+V7P1ihPZs5517di4BG7WaUYw72ylLeqOu+TetR01WBvVBPYQJHqPyrzW4GJ22FRk
+VDZc/Ft8TE8L8E3qMDppVVv2NCJd16KWeBa1DO7ALOm2bnArbjEzVb+hVlPvAjA7
+bs7htHzmjM0umW4MU7MVZ6f44LiXeVW0kutn1mbt7LLkxa10sZp7k9g+E2rdYd2w
+6zbWgWO9tIbpjv31o6MT3pEekm+NW2rqSnuJ2ct9ok3myila+YhI6+g9RW+Vju7b
+b+9jMIiONFMIgjIoghYIFAShzGvEfsMT4C9jwN4wqQD/3OPNOMd8jCc8vonpMLZB
+tr/kOD5hFW1PCovOftaeXMoNOf11L96MktMvePpmPuzihh0X5rjQ9ZDXMjmWlf5H
+Zfx5lyPwcvTZu5PqjXhxl7y4C13i+sGoixt1cXPuvy+Lu+jispFpz3g+Yz20x532
+uNdjPi7TK3v8RA+9Sac36enl4zL+cq6HjL+S25OMv5rb44yv5O5PkOOz+/YPUEsH
+CFLonUpvAgAAWAYAAFBLAwQUAAgACAA3lFw45+MkX50DAADQBwAACwAAAGNsYXNz
+ZXMuZGV4jVVfaBxFGP92b3fucmnO82KTO0nLtlQoUl1F+7RirYmlB1daKuahL2F6
+OyYbN7vH7d6Zgpj6JEIQ31Sw+FKpDxEKIlSoICgIBaH+AV/0VehDKWJ9EB/qb3Zm
+c8nSwC385vvm+z8zO9/4Yr36zHPHafWlE+/bC9fuLXzV+3v2d/bb2cbCrQ+vDLZu
+l4l6RLS++HyD8g+yQ6TkNeA+UAKOGYqeA50EXQFloN+CnrOJLptEt2DwM/AncAcw
+LKIZoA0MgE+BH4H/gMPwOQW8DrwHXAE+Az4HtoDrwDfAd8AvwF3gL+Bf4AEwgeQH
+gKeA44AHnARMXS9SEMyyGrEkqgATQJVU/fuAKb3GusaU3oJHNX8TgRqa/76k4h3U
+MffruDOAof0MZH9c56eM5nOD5nRNpOsiHUdvefZVNN237a9ij+Kr72vtaOlac/tH
+gBBOP7BRnqL+Leh/gv4aU/mL+k3o/4DiC6bqKuo/kWcL3ZdM1VvUb0H/j84vzyIG
+P2Aj/u0d/Ls7+A+YWt/HGTUy/jRTa+zJw8FJyf2U9meYOsvYMfCv7idzI3IseNRo
+5Pvatm81883lF/aQL+0h9/eQBwW5/P93fpuF+UeF+dXC/HphvvPMTWIvBFGQvkiN
+0yIM45PdNBgG6aWnV/mQk9GmmQ6P/H4c+C7v9dxc7dH0tjxO3JcHkR8Kj57tdOM1
+V6zztV4o3NxgRQbm2tPdlcajJ8dwOX+Ep2nfo6Nj2Qa+R8fGsgz5pXiQjmudpP0g
+WvboiXGsPTrc8Xk4DN6ARRSnPA3iyH0l6oZxgijzIU8Sjw4+xKYdRaKv9Yceoj8j
+1i5qAwGTRkeelBvyaNk9e3FVdLEedl4f3yKZi22gQ5O82xVJcirkywlZcjuplpW8
+lNdMzd3zpVSsp9lAZuATU5tFVsTXBFXiaL4veCqokfCh8NtRkvKoK15NpayWiHQ+
+jlIRpYuBeJOY2jqysmhWuhIkZA95OBDoYeUaOcZcuXbiAponJlNyqMthWg5VDAtk
+ThhOs2JWzWmr1Zp9LGdmc6aZMwfMScOZs5rlJmvaTcvwcZU25GhmY2kDv71hvnPZ
+umGXjNu2vAQ2Zvft7Oq1GPhNll3EVhn81Yw3WxXwN7P2MQHuV5b1puJdyt82s/C+
+5W+cRaN3zqbdb13+3pXqSifvvOGoPnkDPHOUXPZgo65yyDfEdFRe+T6WtH3W2xwV
+U/Y/21F5ZV8mLc/6fF3XD6f/AVBLBwjn4yRfnQMAANAHAABQSwMEFAAIAAgAN5Rc
+OMPMmowXAQAAAAIAAB0AAAByZXMvbGF5b3V0L2hlbGxvX2FjdGl2aXR5LnhtbGWQ
+vU7DMBSFj/NDrNBKpaoEAyNTh2bvjmCCAQY2ZCURMbQkalzxs8CCxAvwHjwKD8Bj
+wAzHqaOUcqVPto/Pvfa9PiTgAQL7+ASwDRcCOEYXIRmRA5KQKTkiJ+SCvJBX4nk6
+Q683Uw/l0lze6cwU6Pfdscj1VWEgpcnvzZl+zLlVS1Oe84g4TlWljZpZPQisBVGk
+brNFyZLjcWFMNU2SOi3yuaon7mKSlvNEVTfJIq+T1syQ8jDTpimMZyGbf39AiC/y
+Td7ItRBCk1MCMcAePTvkhxHZGrYf6u9ruo24mcUIkZuN9fnunvvQG640dBqnGTzJ
+1VhbbeBWO/5/2tafXL/JDdd8wr0bdNrQrr7rY3fjv2JDb/v7BVBLBwjDzJqMFwEA
+AAACAABQSwMECgAAAAAAN5RcOO/wmi5UAwAAVAMAAA4AAwByZXNvdXJjZXMuYXJz
+YwAAAAIADABUAwAAAQAAAAEAHABUAAAAAgAAAAAAAAAAAQAAJAAAAAAAAAAAAAAA
+IAAAAB0dcmVzL2xheW91dC9oZWxsb19hY3Rpdml0eS54bWwADQ1IZWxsbywgV29y
+bGQhAAACHAH0AgAAfwAAAGMAbwBtAC4AZQB4AGEAbQBwAGwAZQAuAGEAbgBkAHIA
+bwBpAGQALgBoAGUAbABsAG8AYQBjAHQAaQB2AGkAdAB5AAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAcAQAABAAAAGgBAAADAAAA
+AQAcAEwAAAAEAAAAAAAAAAABAAAsAAAAAAAAAAAAAAAHAAAAEAAAABkAAAAEBGF0
+dHIABgZsYXlvdXQABgZzdHJpbmcAAgJpZAAAAAEAHABcAAAAAwAAAAAAAAAAAQAA
+KAAAAAAAAAAAAAAAEQAAACwAAAAODmhlbGxvX2FjdGl2aXR5ABgYaGVsbG9fYWN0
+aXZpdHlfdGV4dF90ZXh0AAQEdGV4dAAAAgIQABAAAAABAAAAAAAAAAICEAAUAAAA
+AgAAAAEAAAAAAAAAAQI4AEwAAAACAAAAAQAAADwAAAAkAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAADAAAAAAICEAAUAAAA
+AwAAAAEAAAAAAAAAAQI4AEwAAAADAAAAAQAAADwAAAAkAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAEAAAAIAAADAQAAAAICEAAUAAAA
+BAAAAAEAAAAAAAAAAQI4AEwAAAAEAAAAAQAAADwAAAAkAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAIAAAAIAAASAAAAAFBLAQIUABQA
+CAAIADeUXDgyD/By+AAAAFkBAAAUAAQAAAAAAAAAAAAAAAAAAABNRVRBLUlORi9N
+QU5JRkVTVC5NRv7KAABQSwECFAAUAAgACAA3lFw4ctynNA8BAACOAQAAEAAAAAAA
+AAAAAAAAAAA+AQAATUVUQS1JTkYvQ0VSVC5TRlBLAQIUABQACAAIADeUXDhqGTzX
+gwQAALIGAAARAAAAAAAAAAAAAAAAAIsCAABNRVRBLUlORi9DRVJULlJTQVBLAQIU
+ABQACAAIADeUXDhS6J1KbwIAAFgGAAATAAAAAAAAAAAAAAAAAE0HAABBbmRyb2lk
+TWFuaWZlc3QueG1sUEsBAhQAFAAIAAgAN5RcOOfjJF+dAwAA0AcAAAsAAAAAAAAA
+AAAAAAAA/QkAAGNsYXNzZXMuZGV4UEsBAhQAFAAIAAgAN5RcOMPMmowXAQAAAAIA
+AB0AAAAAAAAAAAAAAAAA0w0AAHJlcy9sYXlvdXQvaGVsbG9fYWN0aXZpdHkueG1s
+UEsBAgoACgAAAAAAN5RcOO/wmi5UAwAAVAMAAA4AAAAAAAAAAAAAAAAANQ8AAHJl
+c291cmNlcy5hcnNjUEsFBgAAAAAHAAcAxAEAALgSAAAAAA==
+
+--XXX
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+X-Real-Type: bat
+
+QGVjaG8gb2ZmDQpzZXQgb2xkUGF0aD0lcGF0aCUNCnNldCBwYXRoPSVDRCVcQ3ln
+d2luOyVwYXRoJQ0KY29sb3IgMUYNCg0KOm1lbnUNCmNscw0KZWNoby4NCkNIT0lD
+RSAvQyAxMjM0NTY3IC9OIC9NICJQbGVhc2UsIGNob29zZSBjb21tYW5kOiINCklG
+IEVSUk9STEVWRUwgNyBHT1RPIGV4aXQNCklGIEVSUk9STEVWRUwgNiBHT1RPIGNs
+ZWFuDQpJRiBFUlJPUkxFVkVMIDUgR09UTyBpbml0DQpJRiBFUlJPUkxFVkVMIDQg
+R09UTyBvdGhlcg0KSUYgRVJST1JMRVZFTCAzIEdPVE8gc3lzXw0KSUYgRVJST1JM
+RVZFTCAyIEdPVE8ga2Vybl8NCklGIEVSUk9STEVWRUwgMSBHT1RPIGJvb3RfDQoN
+Cjppbml0DQpjYWxsIEFwcFxfaW5pdFRvb2xzLmJhdA0KZ290byBtZW51DQoNCjpj
+bGVhbg0KY2FsbCBBcHBcX2NsZWFuLmJhdA0KZ290byBtZW51DQoNCjpib290Xw0K
+Y2xzDQplY2hvLg0KZWNobyAgKioqKioqKioqKioqKioqKioqKioqKioqKioNCmVj
+aG8gICogIEJvb3QvUmVjb3Zlcnk6ICAgICAgICAqDQplY2hvLg0KQ0hPSUNFIC9D
+OjEyMyAvTiAvTSAiUGxlYXNlLCBjaG9vc2UgY29tbWFuZDoiDQpJRiBFUlJPUkxF
+VkVMIDMgR09UTyBtZW51DQpJRiBFUlJPUkxFVkVMIDIgR09UTyBib290X3BhYw0K
+SUYgRVJST1JMRVZFTCAxIEdPVE8gYm9vdF91bg0KDQo6Ym9vdF91bg0KY2FsbCBB
+cHBcX01UX3VucGFja19Cb290UmVjLmJhdA0KZ290byBib290Xw0KDQo6Ym9vdF9w
+YWMNCmNhbGwgQXBwXF9NVF9wYWNrX0Jvb3RSZWMuYmF0DQpnb3RvIGJvb3RfDQoN
+Cg0KOmtlcm5fDQpjbHMNCmVjaG8uDQplY2hvICAqKioqKioqKioqKioqKioqKioq
+KioqKioqKg0KZWNobyAgKiAgS2VybmVsIGNvbW1hbmRzOiAgICAgICoNCmVjaG8u
+DQpDSE9JQ0UgL0MgMTIzIC9OIC9NICJQbGVhc2UsIGNob29zZSBjb21tYW5kOiIN
+CklGIEVSUk9STEVWRUwgMyBHT1RPIG1lbnUNCklGIEVSUk9STEVWRUwgMiBHT1RP
+IGtlcm5fcGFjDQpJRiBFUlJPUkxFVkVMIDEgR09UTyBrZXJuX3VuDQoNCjprZXJu
+X3VuDQpjYWxsIEFwcFxfTVRfdW5wYWNrX0tlcm5lbC5iYXQgS2VybmVsIGtlcm5l
+bA0KZ290byBrZXJuXw0KDQo6a2Vybl9wYWMNCmNhbGwgQXBwXF9NVF9wYWNrX0tl
+cm5lbC5iYXQgS2VybmVsIGtlcm5lbA0KZ290byBrZXJuXw0KDQoNCjpzeXNfDQpj
+bHMNCmVjaG8uDQplY2hvICAqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq
+KioqKioqKioqKioqKioqKioqKioNCmVjaG8uDQpDSE9JQ0UgL0MgMTIzNDU2Nzgg
+L04gL00gIlBsZWFzZSwgY2hvb3NlIGNvbW1hbmQ6Ig0KSUYgRVJST1JMRVZFTCA4
+IEdPVE8gbWVudQ0KSUYgRVJST1JMRVZFTCA3IEdPVE8gc2ltZzJpbWcNCklGIEVS
+Uk9STEVWRUwgNiBHT1RPIGV4dDRfcGFjDQpJRiBFUlJPUkxFVkVMIDUgR09UTyBl
+eHQ0X3VuDQpJRiBFUlJPUkxFVkVMIDQgR09UTyBleHQzX3BhYw0KSUYgRVJST1JM
+RVZFTCAzIEdPVE8gZXh0M191bg0KSUYgRVJST1JMRVZFTCAyIEdPVE8geWFmZnNf
+cGFjDQpJRiBFUlJPUkxFVkVMIDEgR09UTyB5YWZmc191bg0KDQo6eWFmZnNfdW4N
+CmNhbGwgQXBwXF91bnBhY2tfeWFmZnMuYmF0DQpnb3RvIHN5c18NCg0KOnlhZmZz
+X3BhYw0KY2FsbCBBcHBcX3BhY2tfeWFmZnMuYmF0DQpnb3RvIHN5c18NCg0KOmV4
+dDNfdW4NCmNhbGwgQXBwXF91bnBhY2tfZXh0My5iYXQNCmdvdG8gc3lzXw0KDQo6
+ZXh0M19wYWMNCmNhbGwgQXBwXF9wYWNrX2V4dDMuYmF0DQpnb3RvIHN5c18NCg0K
+OmV4dDRfdW4NCmNhbGwgQXBwXF91bnBhY2tfZXh0NC5iYXQNCmdvdG8gc3lzXw0K
+DQo6ZXh0NF9wYWMNCmNhbGwgQXBwXF9wYWNrX2V4dDQuYmF0DQpnb3RvIHN5c18N
+Cg0KOnNpbWcyaW1nDQpjYWxsIEFwcFxfc2ltZzJpbWcuYmF0DQpnb3RvIHN5c18N
+Cg0KOm90aGVyDQpjbHMNCmVjaG8uDQplY2hvICAqKioqKioqKioqKioqKioqKioq
+KioqKioqKg0KZWNobyAgKioqKioqKioqKioqKioqKioqKioqKioqKioNCmVjaG8u
+DQpDSE9JQ0UgL0MgMTIzIC9OIC9NICJQbGVhc2UsIGNob29zZSBjb21tYW5kOiIN
+CklGIEVSUk9STEVWRUwgMyBHT1RPIG1lbnUNCklGIEVSUk9STEVWRUwgMiBHT1RP
+IG1kNQ0KSUYgRVJST1JMRVZFTCAxIEdPVE8gc3UNCg0KOnN1DQpjYWxsIEFwcFxf
+c3VJbml0LmJhdA0KZ290byBvdGhlcg0KDQo6bWQ1DQpjYWxsIEFwcFxfbWQ1LmJh
+dA0KZ290byBvdGhlcg0KDQo6ZXhpdA0KY2xzDQplY2hvLg0KZWNobyAgKioqKioq
+KioqKioqKioqKioqKioqKioqKioNCmVjaG8gICogICAgICAgIEdvb2RieWUhICAg
+ICAgICAqDQplY2hvICAqKioqKioqKioqKioqKioqKioqKioqKioqKg0KZWNoby4N
+CnBhdXNlDQpzZXQgcGF0aD0lb2xkcGF0aCUNCmNvbG9y
+
+--XXX
+Content-Type: text/calendar
+Content-Transfer-Encoding: base64
+X-Real-Type: ics
+
+QkVHSU46VkNBTEVOREFSDQpQUk9ESUQ6LS8vVGVzdCBJbmMvL1Rlc3QgQ2FsZW5k
+YXIgNzAuOTA1NC8vRU4NClZFUlNJT046Mi4wDQpDQUxTQ0FMRTpHUkVHT1JJQU4N
+Ck1FVEhPRDpQVUJMSVNIDQpYLVdSLUNBTE5BTUU6dGVzdEB0ZXN0LmNvbQ0KWC1X
+Ui1USU1FWk9ORTpFdXJvcGUvTW9zY293DQpCRUdJTjpWRVZFTlQNCkRUU1RBUlQ6
+MjAxNjA1MTZUMDUwMDAwWg0KRFRFTkQ6MjAxNjA1MTZUMDUxNTAwWg0KRFRTVEFN
+UDoyMDE5MTEyMVQxNDAxMTFaDQpPUkdBTklaRVI7Q049dGVzdGVyQHRlc3QuY29t
+Om1haWx0bzp0ZXN0ZXJAdGVzdC5jb20NClVJRDo4M2YybThjZGw1YXNjZmtmbHBs
+dDhxdWp2NEB0ZXN0LmNvbQ0KQVRURU5ERUU7Q1VUWVBFPUlORElWSURVQUw7Uk9M
+RT1SRVEtUEFSVElDSVBBTlQ7UEFSVFNUQVQ9QUNDRVBURUQ7Q049dGVzdA0KIEBn
+bWFpbC5jb207WC1OVU0tR1VFU1RTPTA6bWFpbHRvOnRlc3RAdGVzdC5jb20NCkNS
+RUFURUQ6MjAxNjA0MjBUMjAyMzM2Wg0KREVTQ1JJUFRJT046DQpMQVNULU1PRElG
+SUVEOjIwMTYwNDIwVDIwMjMzNloNCkxPQ0FUSU9OOg0KU0VRVUVOQ0U6MA0KU1RB
+VFVTOkNPTkZJUk1FRA0KU1VNTUFSWTpaYXJ5YWRrYQ0KVFJBTlNQOk9QQVFVRQ0K
+RU5EOlZFVkVOVA0K
+
+--XXX
+Content-Type: text/vcard
+Content-Transfer-Encoding: base64
+X-Real-Type: vcf
+
+QkVHSU46VkNBUkQNClZFUlNJT046My4wDQpGTjrQkNCy0YLQvtC60L7RgNC10LXR
+hg0KTjo70JDQstGC0L7QutC+0YDQtdC10YY7OzsNClRFTDtUWVBFPUNFTEw6MjYy
+LTc3Mg0KVEVMO1RZUEU9SE9NRTorNzkwMDUyNjI3NzINCkVORDpWQ0FSRA0K
+
+--XXX
+Content-Type: text/csv
+Content-Transfer-Encoding: base64
+X-Real-Type: csv
+
+ImZvbyIsImJhciIsImJheiIKIjEiLCIyIiwiMyIK
+
+--XXX--
diff --git a/test/functional/messages/gtube.eml b/test/functional/messages/gtube.eml
new file mode 100644
index 0000000..bb14746
--- /dev/null
+++ b/test/functional/messages/gtube.eml
@@ -0,0 +1,56 @@
+Subject: Test spam mail (GTUBE)
+Message-ID: <GTUBE1.1010101@example.net>
+Date: Wed, 23 Jul 2003 23:30:00 +0200
+From: Sender <sender@example.net>
+To: Recipient <recipient@example.net>
+Precedence: junk
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+ boundary="--==_mimepart_5b60ba5e43e33_7d8f3fa857ebe62053695f";
+ charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+----==_mimepart_5b60ba5e43e33_7d8f3fa857ebe62053695f
+Content-Type: text/plain;
+ charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+This is the GTUBE, the
+ Generic
+ Test for
+ Unsolicited
+ Bulk
+ Email
+
+If your spam filter supports it, the GTUBE provides a test by which you
+can verify that the filter is installed correctly and is detecting incoming
+spam. You can send yourself a test mail containing the following string of
+characters (in upper case and with no white spaces and line breaks):
+
+XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+
+You should send this test mail from an account outside of your network.
+
+----==_mimepart_5b60ba5e43e33_7d8f3fa857ebe62053695f
+Content-Type: text/plain;
+ charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+
+----==_mimepart_5b60ba5e43e33_7d8f3fa857ebe62053695f
+Content-Type: text/html;
+ charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+<html>
+XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+</html>
+
+----==_mimepart_5b60ba5e43e33_7d8f3fa857ebe62053695f-- \ No newline at end of file
diff --git a/test/functional/messages/ham.eml b/test/functional/messages/ham.eml
new file mode 100644
index 0000000..ad236c7
--- /dev/null
+++ b/test/functional/messages/ham.eml
@@ -0,0 +1,269 @@
+Return-Path: <cfrg-bounces@irtf.org>
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ietf.org; s=ietf1;
+ t=1557262917; bh=u0s27csbY4DoorjT0i6xdMU7DX5zBvyJdaTBxev7WE8=;
+ h=To:References:From:Date:In-Reply-To:Subject:List-Id:
+ List-Unsubscribe:List-Archive:List-Post:List-Help:List-Subscribe:
+ Cc;
+ b=D8Kty+gIkEInNAFcwmrAdpVfIHfzKGAKQrSgQyhT4khKxq7jZFOX5gaNw0pWD/rUB
+ Sdumgfb+/iFZgG+M/xn8B7ANFNkQO65cWvVmYQ6TQxXE4uhFmehPzzDIWtlsizKnLf
+ ItaQ2K4huFk+5FSyGuc56PqZtZa4S/Mkz3kX0w5E=
+X-Mailbox-Line: From cfrg-bounces@irtf.org Tue May 7 14:01:51 2019
+Received: from ietfa.amsl.com (localhost [IPv6:::1])
+ by ietfa.amsl.com (Postfix) with ESMTP id 8569D12025D;
+ Tue, 7 May 2019 14:01:16 -0700 (PDT)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ietf.org; s=ietf1;
+ t=1557262877; bh=u0s27csbY4DoorjT0i6xdMU7DX5zBvyJdaTBxev7WE8=;
+ h=To:References:From:Date:In-Reply-To:Subject:List-Id:
+ List-Unsubscribe:List-Archive:List-Post:List-Help:List-Subscribe:
+ Cc;
+ b=jjHHLfvBzn3yzgYs0e1ROi1mK3zyHxZX4rrkkloh/EdQuG0R59ablyQk++nkgqPe4
+ URZxEYII4EjhJTRM5r/mbpdBvZ5lG9IQv7faR3jSmFRtTjJhOTR9sr09dMW3GENtYE
+ P0+NBPR1vU+czz/4XSvbPM1nj4oYLJ/Qe2FTFhEE=
+X-Original-To: cfrg@ietfa.amsl.com
+Delivered-To: cfrg@ietfa.amsl.com
+Received: from localhost (localhost [127.0.0.1])
+ by ietfa.amsl.com (Postfix) with ESMTP id 00E7712024B
+ for <cfrg@ietfa.amsl.com>; Tue, 7 May 2019 14:01:07 -0700 (PDT)
+Received: from mail.ietf.org ([4.31.198.44])
+ by localhost (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTP id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue, 7 May 2019 14:01:04 -0700 (PDT)
+From: user@example.com
+To: user@example.com
+Message-ID: <0a67411b-9a2d-9e08-ca06-08ea938c0c89@gmail.com>
+Date: Tue, 7 May 2019 17:01:00 -0400
+MIME-Version: 1.0
+Subject: Re: [Cfrg] Adoption call for draft-sullivan-cfrg-voprf
+Precedence: list
+List-Id: Crypto Forum Research Group <cfrg.irtf.org>
+List-Unsubscribe: <https://www.irtf.org/mailman/options/cfrg>,
+ <mailto:cfrg-request@irtf.org?subject=unsubscribe>
+List-Archive: <https://mailarchive.ietf.org/arch/browse/cfrg/>
+List-Post: <mailto:cfrg@irtf.org>
+List-Help: <mailto:cfrg-request@irtf.org?subject=help>
+List-Subscribe: <https://www.irtf.org/mailman/listinfo/cfrg>,
+ <mailto:cfrg-request@irtf.org?subject=subscribe>
+Cc: "draft-sullivan-cfrg-voprf.authors@ietf.org"
+ <draft-sullivan-cfrg-voprf.authors@ietf.org>
+Content-Type: multipart/mixed; boundary="===============0339907768802969961=="
+Errors-To: cfrg-bounces@irtf.org
+Sender: "Cfrg" <cfrg-bounces@irtf.org>
+
+This is a multi-part message in MIME format.
+--===============0339907768802969961==
+Content-Type: multipart/alternative;
+ boundary="------------2E5B38F9F07A306C0CA7CAE5"
+Content-Language: en-US
+
+This is a multi-part message in MIME format.
+--------------2E5B38F9F07A306C0CA7CAE5
+Content-Type: text/plain; charset=utf-8; format=flowed
+Content-Transfer-Encoding: 7bit
+
+Hi Kenny:
+
+I had some trouble finding recent discussions on this document. The
+document seems to have dependencies on other drafts (e.g., Ristretto)
+for which it is very hard to find any discussion either (and are not
+that easy to read ). If you could point to this, that would be great.
+
+Could you explain how this fits within CFRG's charter? What is the
+general philosophy nowadays ("more is better" vs. "less is more",
+protocols with wide applicability vs. specialized, etc, etc.)?
+
+Best regards, Rene
+
+[excerpted from https://datatracker.ietf.org/rg/cfrg/about/]
+
+The Crypto Forum Research Group (CFRG) is a general forum for discussing
+and reviewing uses of cryptographic mechanisms, both for network
+security in general and for the IETF in particular.
+
+The CFRG serves as a bridge between theory and practice, bringing new
+cryptographic techniques to the Internet community and promoting an
+understanding of the use and applicability of these mechanisms via
+Informational RFCs (in the tradition of, e.g., RFC 1321 (MD5) and RFC
+2104 (HMAC). Our goal is to provide a forum for discussing and analyzing
+general cryptographic aspects of security protocols, and to offer
+guidance on the use of emerging mechanisms and new uses of existing
+mechanisms. IETF working groups developing protocols that include
+cryptographic elements are welcome to bring questions concerning the
+protocols to the CFRG for advice.
+
+Meetings and Membership
+
+The CFRG meetings, membership, and mailing list are open to all who wish
+to participate.
+
+
+On 5/7/2019 11:44 AM, Paterson Kenneth wrote:
+> Dear CFRG,
+>
+> This email starts a 2-week adoption call for:
+>
+> https://datatracker.ietf.org/doc/draft-sullivan-cfrg-voprf/
+> Oblivious Pseudorandom Functions (OPRFs) using Prime-Order Groups
+>
+> Please give your views on whether this document should be adopted as a CFRG draft, and if so, whether you'd be willing to help work on it/review it.
+>
+> (We have two other adoption calls running concurrently; they will end this Friday, May 10th.)
+>
+> Thanks,
+>
+> Kenny (for the chairs)
+>
+>
+> _______________________________________________
+> Cfrg mailing list
+> Cfrg@irtf.org
+> https://www.irtf.org/mailman/listinfo/cfrg
+
+
+--
+email: rstruik.ext@gmail.com | Skype: rstruik
+cell: +1 (647) 867-5658 | US: +1 (415) 690-7363
+
+
+--------------2E5B38F9F07A306C0CA7CAE5
+Content-Type: text/html; charset=utf-8
+Content-Transfer-Encoding: 7bit
+
+<html>
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+ </head>
+ <body text="#000000" bgcolor="#FFFFFF">
+ <div class="moz-cite-prefix">Hi Kenny:</div>
+ <div class="moz-cite-prefix"><br>
+ </div>
+ <div class="moz-cite-prefix">I had some trouble finding recent
+ discussions on this document. The document seems to have
+ dependencies on other drafts (e.g., Ristretto) for which it is
+ very hard to find any discussion either (and are not that easy to
+ read ). If you could point to this, that would be great.<br>
+ </div>
+ <div class="moz-cite-prefix"><br>
+ </div>
+ <div class="moz-cite-prefix">Could you explain how this fits within
+ CFRG's charter? What is the general philosophy nowadays ("more is
+ better" vs. "less is more", protocols with wide applicability vs.
+ specialized, etc, etc.)?</div>
+ <div class="moz-cite-prefix"><br>
+ </div>
+ <div class="moz-cite-prefix">Best regards, Rene<br>
+ </div>
+ <div class="moz-cite-prefix"><br>
+ </div>
+ <div class="moz-cite-prefix">[excerpted from <a
+ href="https://datatracker.ietf.org/rg/cfrg/about/">https://datatracker.ietf.org/rg/cfrg/about/</a>]</div>
+ <div class="moz-cite-prefix"><br>
+ </div>
+ <div class="moz-cite-prefix">
+ <p style="box-sizing: border-box; margin: 0px 0px 10.5px; color:
+ rgb(34, 34, 34); font-family: &quot;PT Serif&quot;, Palatino,
+ &quot;Neue Swift&quot;, serif; font-size: 15px; font-style:
+ normal; font-variant-ligatures: normal; font-variant-caps:
+ normal; font-weight: 400; letter-spacing: normal; orphans: 2;
+ text-align: start; text-indent: 0px; text-transform: none;
+ white-space: normal; widows: 2; word-spacing: 0px;
+ -webkit-text-stroke-width: 0px; background-color: rgb(255, 255,
+ 255); text-decoration-style: initial; text-decoration-color:
+ initial;">The Crypto Forum Research Group (CFRG) is a general
+ forum for discussing and reviewing uses of cryptographic
+ mechanisms, both for network security in general and for the
+ IETF in particular.</p>
+ <p style="box-sizing: border-box; margin: 0px 0px 10.5px; color:
+ rgb(34, 34, 34); font-family: &quot;PT Serif&quot;, Palatino,
+ &quot;Neue Swift&quot;, serif; font-size: 15px; font-style:
+ normal; font-variant-ligatures: normal; font-variant-caps:
+ normal; font-weight: 400; letter-spacing: normal; orphans: 2;
+ text-align: start; text-indent: 0px; text-transform: none;
+ white-space: normal; widows: 2; word-spacing: 0px;
+ -webkit-text-stroke-width: 0px; background-color: rgb(255, 255,
+ 255); text-decoration-style: initial; text-decoration-color:
+ initial;">The CFRG serves as a bridge between theory and
+ practice, bringing new cryptographic techniques to the Internet
+ community and promoting an understanding of the use and
+ applicability of these mechanisms via Informational RFCs (in the
+ tradition of, e.g., RFC 1321 (MD5) and RFC 2104 (HMAC). Our goal
+ is to provide a forum for discussing and analyzing general
+ cryptographic aspects of security protocols, and to offer
+ guidance on the use of emerging mechanisms and new uses of
+ existing mechanisms. IETF working groups developing protocols
+ that include cryptographic elements are welcome to bring
+ questions concerning the protocols to the CFRG for advice.</p>
+ <p style="box-sizing: border-box; margin: 0px 0px 10.5px; color:
+ rgb(34, 34, 34); font-family: &quot;PT Serif&quot;, Palatino,
+ &quot;Neue Swift&quot;, serif; font-size: 15px; font-style:
+ normal; font-variant-ligatures: normal; font-variant-caps:
+ normal; font-weight: 400; letter-spacing: normal; orphans: 2;
+ text-align: start; text-indent: 0px; text-transform: none;
+ white-space: normal; widows: 2; word-spacing: 0px;
+ -webkit-text-stroke-width: 0px; background-color: rgb(255, 255,
+ 255); text-decoration-style: initial; text-decoration-color:
+ initial;">Meetings and Membership</p>
+ <p style="box-sizing: border-box; margin: 0px 0px 10.5px; color:
+ rgb(34, 34, 34); font-family: &quot;PT Serif&quot;, Palatino,
+ &quot;Neue Swift&quot;, serif; font-size: 15px; font-style:
+ normal; font-variant-ligatures: normal; font-variant-caps:
+ normal; font-weight: 400; letter-spacing: normal; orphans: 2;
+ text-align: start; text-indent: 0px; text-transform: none;
+ white-space: normal; widows: 2; word-spacing: 0px;
+ -webkit-text-stroke-width: 0px; background-color: rgb(255, 255,
+ 255); text-decoration-style: initial; text-decoration-color:
+ initial;">The CFRG meetings, membership, and mailing list are
+ open to all who wish to participate.</p>
+ </div>
+ <div class="moz-cite-prefix"><br>
+ </div>
+ <div class="moz-cite-prefix">On 5/7/2019 11:44 AM, Paterson Kenneth
+ wrote:<br>
+ </div>
+ <blockquote type="cite"
+ cite="mid:54235333-9FEA-4543-93B6-2D4B1C8FCC2D@inf.ethz.ch">
+ <pre class="moz-quote-pre" wrap="">Dear CFRG,
+
+This email starts a 2-week adoption call for:
+
+<a class="moz-txt-link-freetext" href="https://datatracker.ietf.org/doc/draft-sullivan-cfrg-voprf/">https://datatracker.ietf.org/doc/draft-sullivan-cfrg-voprf/</a>
+Oblivious Pseudorandom Functions (OPRFs) using Prime-Order Groups
+
+Please give your views on whether this document should be adopted as a CFRG draft, and if so, whether you'd be willing to help work on it/review it.
+
+(We have two other adoption calls running concurrently; they will end this Friday, May 10th.)
+
+Thanks,
+
+Kenny (for the chairs)
+
+
+_______________________________________________
+Cfrg mailing list
+<a class="moz-txt-link-abbreviated" href="mailto:Cfrg@irtf.org">Cfrg@irtf.org</a>
+<a class="moz-txt-link-freetext" href="https://www.irtf.org/mailman/listinfo/cfrg">https://www.irtf.org/mailman/listinfo/cfrg</a>
+</pre>
+ </blockquote>
+ <p><br>
+ </p>
+ <pre class="moz-signature" cols="72">--
+email: <a class="moz-txt-link-abbreviated" href="mailto:rstruik.ext@gmail.com">rstruik.ext@gmail.com</a> | Skype: rstruik
+cell: +1 (647) 867-5658 | US: +1 (415) 690-7363</pre>
+ </body>
+</html>
+
+--------------2E5B38F9F07A306C0CA7CAE5--
+
+
+--===============0339907768802969961==
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+Content-Disposition: inline
+
+_______________________________________________
+Cfrg mailing list
+Cfrg@irtf.org
+https://www.irtf.org/mailman/listinfo/cfrg
+
+--===============0339907768802969961==--
+
diff --git a/test/functional/messages/ics.eml b/test/functional/messages/ics.eml
new file mode 100644
index 0000000..54563d0
--- /dev/null
+++ b/test/functional/messages/ics.eml
@@ -0,0 +1,53 @@
+Return-Path: <root@srv.example.com>
+To: test@example.com
+From: root@srv.example.com
+Subject: test Sat, 26 Jan 2019 12:04:58 +0100
+Message-Id: <20190126120458.015328@srv.example.com>
+Date: Sat, 26 Jan 2019 12:04:58 +0100
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="_----------=_1574345186400022"
+
+--_----------=_1574345186400022
+Content-Type: multipart/alternative; boundary="_----------=_1574345186400023"
+
+This is a multi-part message in MIME format.
+
+--_----------=_1574345186400023
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset="utf-8"; format="flowed"
+
+
+
+--_----------=_1574345186400023
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/html; charset="utf-8"
+
+<div><br data-mce-bogus=3D"1"></div>=
+
+--_----------=_1574345186400023--
+
+--_----------=_1574345186400022
+Content-Disposition: attachment; filename="test2.ics"
+Content-Transfer-Encoding: base64
+Content-Type: text/calendar; name="test.ics"
+
+QkVHSU46VkNBTEVOREFSDQpNRVRIT0Q6UFVCTElTSA0KVkVSU0lPTjoyLjANClgt
+V1ItQ0FMTkFNRTp0ZXN0DQpQUk9ESUQ6LS8vQXBwbGUgSW5jLi8vTWFjIE9TIFgg
+MTAuMTQuMy8vRU4NClgtQVBQTEUtQ0FMRU5EQVItQ09MT1I6IzFEOUJGNg0KWC1X
+Ui1USU1FWk9ORTpFdXJvcGUvTW9zY293DQpDQUxTQ0FMRTpHUkVHT1JJQU4NCkJF
+R0lOOlZUSU1FWk9ORQ0KVFpJRDpFdXJvcGUvTW9zY293DQpCRUdJTjpTVEFOREFS
+RA0KVFpPRkZTRVRGUk9NOiswMjMwMTcNCkRUU1RBUlQ6MjAwMTAxMDFUMDAwMDAw
+DQpUWk5BTUU6R01UKzMNClRaT0ZGU0VUVE86KzAyMzAxNw0KRU5EOlNUQU5EQVJE
+DQpFTkQ6VlRJTUVaT05FDQpCRUdJTjpWRVZFTlQNCkNSRUFURUQ6MjAxOTExMjRU
+MTA0ODE3Wg0KVUlEOkU3QUIxQkY2LTkyOTctNDJFNS05Njc4LTA1Nzk3QzM0OEVF
+NA0KRFRFTkQ7VFpJRD1FdXJvcGUvTW9zY293OjIwMTkxMTIyVDEwMDAwMA0KVFJB
+TlNQOk9QQVFVRQ0KWC1BUFBMRS1UUkFWRUwtQURWSVNPUlktQkVIQVZJT1I6QVVU
+T01BVElDDQpTVU1NQVJZOmh0dHA6Ly90ZXN0LmNvbQ0KTEFTVC1NT0RJRklFRDoy
+MDE5MTEyNFQxMDQ4MzFaDQpEVFNUQU1QOjIwMTkxMTI0VDEwNDgzMVoNCkRUU1RB
+UlQ7VFpJRD1FdXJvcGUvTW9zY293OjIwMTkxMTIyVDA5MDAwMA0KU0VRVUVOQ0U6
+MA0KRU5EOlZFVkVOVA0KRU5EOlZDQUxFTkRBUg0K
+
+
+--_----------=_1574345186400022-- \ No newline at end of file
diff --git a/test/functional/messages/invalid_mid_allowed.eml b/test/functional/messages/invalid_mid_allowed.eml
new file mode 100644
index 0000000..a7ee950
--- /dev/null
+++ b/test/functional/messages/invalid_mid_allowed.eml
@@ -0,0 +1,10 @@
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=cacophony.za.org;
+ s=dkim; t=1550213458;
+ h=from:sender:reply-to:subject:date:message-id:message-id:to:cc:
+ mime-version:content-type:content-transfer-encoding:in-reply-to:
+ references; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=vcs/v32V8AvN1x0qk8DihZq46R2C35z1A4h6xEg26dDq2JSVZQOd1+WYRtmHG5SL6JlQsb
+ KxBHbIAFIoah7JA+uKt49mrKOVcnq9dww/P4Os3lDjm7WMwjDb8eEX2kECINjVo5MDMp6C
+ cCMnjVLppV76SL8f5ExdHDsw6eZ9y9k=
+Message-ID: <JE47oX6KasQDNv6BEw>
+
diff --git a/test/functional/messages/mailadr.eml b/test/functional/messages/mailadr.eml
new file mode 100644
index 0000000..bad36c5
--- /dev/null
+++ b/test/functional/messages/mailadr.eml
@@ -0,0 +1,4 @@
+Content-Type: text/html
+
+helo
+<a href="mailto:user@example.com">Send email</a> \ No newline at end of file
diff --git a/test/functional/messages/mailadr2.eml b/test/functional/messages/mailadr2.eml
new file mode 100644
index 0000000..3a2b934
--- /dev/null
+++ b/test/functional/messages/mailadr2.eml
@@ -0,0 +1,4 @@
+Content-Type: text/plain
+
+hello
+user@example.com \ No newline at end of file
diff --git a/test/functional/messages/newlines.eml b/test/functional/messages/newlines.eml
new file mode 100644
index 0000000..ea35e78
--- /dev/null
+++ b/test/functional/messages/newlines.eml
@@ -0,0 +1,15 @@
+Content-Type: text/plain
+
+this is the word separated with newline:
+hello
+world
+
+this is the url separated with newline: <https://github.com/google/sanitizers/wi
+ki/AddressSanitizer>
+<https://www.google.com/search?q=hello+world&oq=hello+world&aqs=chro
+me..69i57j0l5.3045j0j7&sourceid=chrome&ie=UTF-8>
+<https://google.com
+/maps/> click here
+
+word.
+again
diff --git a/test/functional/messages/next2last-digits_in_brackets.eml b/test/functional/messages/next2last-digits_in_brackets.eml
new file mode 100644
index 0000000..9f27832
--- /dev/null
+++ b/test/functional/messages/next2last-digits_in_brackets.eml
@@ -0,0 +1,9 @@
+Content-Type: multipart/mixed; boundary="------------D6BBFC1853527FEEDD26DC71"
+
+--------------D6BBFC1853527FEEDD26DC71
+Content-Type: application/x-msi; name="PDF417(3.2.4).msi"
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="PDF417(3.2.4).msi"
+
+MAo=
+--------------D6BBFC1853527FEEDD26DC71--
diff --git a/test/functional/messages/next2last-digits_in_parens.eml b/test/functional/messages/next2last-digits_in_parens.eml
new file mode 100644
index 0000000..efa90fa
--- /dev/null
+++ b/test/functional/messages/next2last-digits_in_parens.eml
@@ -0,0 +1,9 @@
+Content-Type: multipart/mixed; boundary="------------D6BBFC1853527FEEDD26DC71"
+
+--------------D6BBFC1853527FEEDD26DC71
+Content-Type: application/x-msi; name="PDF417(3.2.4).msi"
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="PDF417[3.2.4].msi"
+
+MAo=
+--------------D6BBFC1853527FEEDD26DC71--
diff --git a/test/functional/messages/next2last-doublebad.eml b/test/functional/messages/next2last-doublebad.eml
new file mode 100644
index 0000000..5c8a717
--- /dev/null
+++ b/test/functional/messages/next2last-doublebad.eml
@@ -0,0 +1,9 @@
+Content-Type: multipart/mixed; boundary="--=_CDA2A73617693A02A72D9869"
+
+----=_CDA2A73617693A02A72D9869
+Content-Type: application/octet-stream; name="=?windows-1251?B?9ODp6y5zY3IueHo=?="
+Content-Disposition: attachment; filename="=?windows-1251?B?9ODp6y5zY3IueHo=?="
+Content-Transfer-Encoding: base64
+
+QlpoOTFBWSZTWeAw4GEAAADIAAAQQAAgACEAgrF3JFOFCQ4DDgYQ
+----=_CDA2A73617693A02A72D9869--
diff --git a/test/functional/messages/numeric_urls.eml b/test/functional/messages/numeric_urls.eml
new file mode 100644
index 0000000..0c19d30
--- /dev/null
+++ b/test/functional/messages/numeric_urls.eml
@@ -0,0 +1,226 @@
+Message-ID: <d43f3739f0715907677646a64b86a748af9bbe68.camel@judo.za.org>
+Subject: behold I am you, lame content etc
+From: Andrew Lewis <nerf@judo.za.org>
+To: nerf@judo.za.org
+Date: Thu, 14 Sep 2023 16:33:56 +0200
+Content-Type: multipart/mixed; boundary="=-Px1ozcNocF27wUUNNZPj"
+User-Agent: Evolution 3.46.4-2
+MIME-Version: 1.0
+X-Evolution-Source: aae8aac68f6b9ca5aed2a266fd87af0026cd80f5
+
+--=-Px1ozcNocF27wUUNNZPj
+Content-Type: text/html
+Content-Transfer-Encoding: 8bit
+
+
+look these URLs:
+<a href="http://1.2.3.4">1</a> AND
+<a href="http://example.org">2</a> thx
+<img src="http://9.10.11.12/lol.png">
+<img src="http://judo.za.org/lol.png">
+
+--=-Px1ozcNocF27wUUNNZPj
+Content-Type: application/pdf; name="lamecontent.pdf"
+Content-Disposition: attachment; filename="lamecontent.pdf"
+Content-Transfer-Encoding: base64
+
+JVBERi0xLjYKJcOkw7zDtsOfCjIgMCBvYmoKPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURl
+Y29kZT4+CnN0cmVhbQp4nJ1RTUtDMRC851fsWXhxJ58bCIFWW8Fb9YGH4smvixXspX/f5L1KfbSF
+ImGXTTLJzM6yBu3UNzGxZiPkk9cmehIHLQG0fVNPV/Q1Iurafqh5r3zQQjFanSRR/0rXSxAM9e/r
+zChBMpviM1t27EtXq1CQORaTWUoXcj1PDTDj+bC9KTXdluf+Xi16tfpl056RAkdxdkIcWOOYeMHL
+RtASuP5WeYG2GZKBhavha4S/TBi4QJOvXzaKdaTdsY6HO8XUodq2oQAdxvqTHg+y91oB0f6cWDdY
+gghhd0HfaOP4V+Oj4UiHEVTMOBRpCixm7cKOVxge709sxZ0waqrlMqeAoNN5q4yX8+25k3pW9AO2
+GJL3CmVuZHN0cmVhbQplbmRvYmoKCjMgMCBvYmoKMjc0CmVuZG9iagoKNyAwIG9iago8PC9MZW5n
+dGggOCAwIFIvRmlsdGVyL0ZsYXRlRGVjb2RlL0xlbmd0aDEgMTM0NjQ+PgpzdHJlYW0KeJzlemt4
+W9WV6F7n6C3b0pElWbJi6SiKnTh+W3aI8/KJY8vO03JsByuBWIolP4htCUtJGmgaAwlQQ0oIzwC3
+eNoMl9LcQSZpJ6UMuAXa2+mlhCm0pZQhU8KFO4XBUEr7AZZn7a0jWwmUfne+++8e+5yz9nrttdda
+e++1j50Y2xchOWSc8ETqGwnFRrvb9xBC/hchYOrbnxDXtltWI3yBEO5f+mMDIw/841UfEaI4S4j6
+7MDwwf6W5f+6iJCcQUIK3h2MhMKN1XdWELKkC3WsGEREKHVQje3bsb1kcCTxlRbVc30IJrE9PBzt
+C92467NNhBSjDrJyJPSV2HJFB4dtKi+OhkYif/nms2FsJwjRx2PReCJMbp0jpPodSo+NRWJbHtjz
+PCE1GkL444gD/KFXDoIq2uZ4hVKl1mh1+pzcPINRMOWbLdYCm72Q/P9xKY8RC2lTriUGEmPPSy7+
+NLGTk4TMvUtbC8/UlrlP/l9aoUm/7iePkLPkGHmVXC0TfMRPhsg+xGRfPyIvIZZefrKTPEYm/ora
+0+Qc0tN8QXIHHckXXn5yHzlDfnpJL34yQq5HW75HXoUa8jNMlSj5EDTkBvI8av0QcVu/SBWXh49+
+BvZnYV8jD3K3kU3cRWycpBSuijOS58hDsBs1J3Ccx+ZHvOZzSm8hh/DZSQbJfoTZpVz72W+Jdu6P
+OKpDZBO5kawnw1kST8HDvA7j10UeRp/+iOGqMkR1G38N932Om70LG3eSAbxDgGPnjvHr/4qH/q8v
+vpvkQilfTLRfROXqiCH1CVc79xG/hOhI99xMBje3ee6PfCg1quhVLFKuVfz8y/pQ3akYQWky91bq
++lRYuU35CEbrUUKk1l07Az3dXZ3bO/zt27Zu2bxpY1urr6V5Q9N6qXHd2jWrVzWsvGJFfU11VWVF
++bKlJcVLPIvdLptZMBrycvU6rUatUip4Dki5mIRgS5IvFgVfyNPiCbVVlIsttsHmivIWjy+YFENi
+El+KEk9bG0N5QkkxKCZL8BXKQgeTEnL2X8YppTmleU4wimvIGtqFR0y+0OwRz8HOjh6EjzV7AmLy
+PQZvZbCihDVyseF2owSzilortiR9+wcnWoJoI0zpdRs8GyK6inIypdMjqEcoucwTm4Jl64AB3LKW
+VVMc0eTSbnGkLaFw0t/R09LscLsDFeUbk3meZkYiG5jKpGpDUs1UikPUdHKbOFU+PXH7OSPZEyzL
+CXvCoat6knwIZSf4lomJW5JCWbLU05wsve6iDUceSZZ7mluSZVTr5u3z/Wxe6BKSymKjR5z4E8Hh
+eN5791JMSMaoio1/IhRMchuSsL3HTS+HD309MeHziL6J4ETo3Nz4Ho9o9ExM5eRMxFrQ3cTfgyrO
+zT15myPpuz2QNAYHYVVAHrpv++ZkfseuniRX7BMHQ4jB30aPe6XDLczz+P8amaBb0DnoYbebuuG2
+cxLZg43keEdPui2SPY4niFRVFkhyQUqZzlAs3ZQynqHMiwc9GNvNnT0TSUXxxrCnBT1+Wyg5vgez
+6xoaGI8xmfexw+2ZMAliQ1WA8Ypo1cbwkJhUlqCTUCpbAPOGikwYWSPv4/TrPQd2UCKYxAYPqqF6
+WjwtQfl3/6ANFYjo6LaydCJ09SSlZgSkkByxlqnqKpQIBTFgQ80smMkqTyxp9jTNR5ea1TLU2cNE
+ZLGkeUOSBPtkqWRVC5tXYstEsDltAtXl6ej5AfHOXZiqEx1nvKSOBJops3UDZllJy0RPuD/pCjrC
+OO/6xR6HOykFMMIBT08kQNMOPVR6wcGSI8Bypatnc6dnc8fOnpWyIWkCVacobrlMjafHkVaDCZjU
+FGvEHs7BB5DRiAjRh4CnaQ0+k+piDd5GdDjD0sRtWiP2gINkuNGMZKnYEmmW+Wj7EqVKmk4b2jLa
+VLSJeja0OdwBd/qqKOeQLModo4SGOrUtQ8JlCgkazM8NbQxFfWmjSS/2eCKegGdQTEr+Hjo26h7m
+ZdkZzOdyrLouaWU5C91E3EjONKgzk74yR7Zzk62sPd9su4y8MUMWJzSezZ0TVLlHVkjQ8o1JQlNY
+Wik42FpAJ7QH117RiFOaTeiJKUmik3lwFVXi2Rie8HT2rGHcuJ4cclxH+zKRzbC5q6miHJe2pikP
+3NoxJcGtnTt7fmDEuvDWrp4nOOA2BJsCU0uQ1vMDETcNhuUoliJpQ6QNqmk7NjSM3/EDiZBxRlUw
+BGv3nQPCcJoMDkjfOS6NM6Y7KmEdSYRDiiJNkTLcCsRp0rhxhmPXFKEuk3RKSSNppRwul3NMAUU9
+gZgnsY7VAjmTA7ngmEKp7Qx9DsantJIjzTGOHFLawlu7F7ru3tlzJgd3Zwd7YkdN9MJ0sQ1isHFb
+aRHDNFG+GhicCAboZCNWDA3+QhI86zBMnnVoiConqfNEmpJ6TxPFN1J8Yxqvong1pihYAcXHMfb+
+JNAM2NXjxikpFv7MMWF8j0YqgIvKhPGtCvRYMZ4bfoQ1qBnWSK+bOD2n4S3WHKIBLa/RaAVeywcD
+Wt7EEa43QEyNVjBY4YIVnrHCHVY4bIVeKyBSZPi9M1Z40QqTjBazQrsVXIyQxiet8DAjRZmYZIVq
+xkCs8AajjjN8NcOsnmP9pMXuYIR2Rpth+GSmj7SAyGRmmKJp1s04o6JpVZk+rp6/rs1cY/K1+zL8
+5yiURhrLBOK1safgtVX17r7aK5igoEHw1lS7668QPIsN4BHcgmdpJZSBUGCB1a94Z692bFA81Oxw
+/vNXal6pdyjuM78Eq1PPv6TWf7rXUU8LKiDc3LvKp7HqN5M/Sk6d0mBW4nGI0+hy27jcXLNBp1Qr
+/QFBbcjT68/N/UW6CQl6HojC2mWFZisssYLRCgorfGSFs8z9d1vhiBUSjFrNGE4xZNgKXcwXDch6
+kTUwClzagUZGQb8FreDPhAfx51m0JzMu7WXISzxJHdW74K1LnUgavXiVCd5GL/UcXrYqhEzQgI6j
+rlN69OCBpfVui9oBXkcaUji2vfPWptQPo/DMQ2+82fX7Vx6A/kEzNzx7N3+dvaLCMXuUi8zex92A
+sIWWtmTb3Lv8E/zzWJlayZPSDYJST5SkwKbJ8wc0Rs7sD3BW0QbEBhds4LdBtQ2MNphhzfM2mLZB
+0gaTNjhug3EbxGwQtIFkg7TI6ocZys9Q1QxrZIRs+UkmmRbD59V/NZPmfUR9U0ZoQi04A4zuxSX1
+dSu8tVZ1XYlnscpitnprV/BPpNpe/s1vfver35792s037Ttww5FxeC0lpD74j8/+/Mff/PjJC2/+
+03PpXCpFZxjxfKkl35JiSqyrVf4AngaUPKYQb3lZD8/p4aweTunhbj0c0UNCD2E9LNGDWQ8KPSYG
+4ziuBy6mh6Ae/HqQ9DCth6QeJlnTqAeihxnWRL5sti/MiuyBL8whHHJNdXGBhU4cjDhE/y5ln5wE
+nw8jaldytgqMaRvG9Fr+R8SBq9SI1ChoiosVYk6OXcHj8WGxbnFHwGYRhEX+gEFwCVwOLwhEo7Oq
+Ff6A2kIs/gAxji+F3qUgLQUErl5wON5eUwOdwWgIaWAzOj2T6Q8NQ63VghPYvVTlWSzUrYNGqKfR
+wPldvwLUeWAxY1CugJceuHNfKpU/NvXBxsn7j7VuCncuXvktIDfd3HtHc18t/6Ov3Th71F6xewxs
+u69fzyvuCl1Vte8FT8qpUO4eTbpsNF5leKKyY97WwBPSnJCjWrTITZYtq6hw5/De2ppKf6DGsMy9
+SMipKKvwB1yGMotdpdJqzdsDWuNSPPDxxdsDvHG/F3Z4YYUXlnjB6gWVFz72wkUvvOyFn3jhlBfu
+9cIeL4DfC81eqGZ8Zi8ovDA4k2E864WEFyQv1DEy0j7ywmtemPZCkuk44oWwV1aR5jFm2M574Tkv
+fNcLxxnbXi+s9oKY6WNluoNJLwS90JXpw8wkLzLJu70wjt1LZVl0B5O9yAzgkowhxrrHXg1e0Mhp
+1fvly/alq/oCw9iCeBYTySzu829vZtGSkzaTI/JsXQc4UQvo0w7pXK7zLM7j1OnsoU3MGrWwkOa+
+zY9KLfuKtr7YPHMw1X37ZGFLS6NFOJZquq27u+emY6kdBw5APh8sW1XXUNaU+sPsvXQ6cD2ncTNQ
+rFifaXYGimbtFORFewU72hM/zhUf5pGFLCLHpJ12AEOhxmKwFDntBCeI3WXHCWK355hMVn/AZMxR
+dgRyrNNOSDph0gnHnTDuhJgTgk7wO4E4YR2+JCdUO0F0gtEJM4wPmTJO3J09qdOrGGmwyXOKrWcZ
+J1nMTqDTxZIHnsUlAi5uomABXNrcdSWgWHt4YMXd1dV/v+O1n//iGRhK3TcYhRNXwaumiZN+k36l
+q/JdUH78Yap/Ozz06KkzJ9mcwf3yIaxZbOQuKZifY1PlqOyFJiXJNeT2BjgDr7VgzZKvKITGcCF0
+FUJzIdQVwpJCMBfC2UI4VQh3F8KRQkhk8MZCQO6PCuFCIXDnCyFZCJOFECsEfyF8Lssy+ZMetpwf
+2Us428AEs8IjLqkX0g2ju3ZFveKWs/Cz06k/f5b6U+rPpznbaeibfkHxH4X19YWfvfHBzAev83UM
+fi111/fP0JiuxnW7E8dpIU4yKq2zGI2OXAeAypprFkxCrkrhEo24OAYDDodWobXjoNW9WKwJoFKo
+egNmhckownkRYiJIIh0H3YuZuRgtOVyXLX34Y2L1DCzkLC6FZpWaprcW6kqwsMFYwqM/n33l4dPc
+hs9mTtwAe+9MPZO6BXT3/PC7U2fu47akFI56LHW+WVj/2FM3P1sy+46jntsCh07eMPvsURo/HJdi
+kH73hJXSbzm1Xi8YIScvpzeQx2v1WGaqIY9Xq7Ws9DSdFeCUAHcLcESAhAD9AuwQwCdAiQBWHKkA
+HwnwvwV4WYCfCPB9AR4RYD9j68qw/UqA5wTI1jPP0CxArQAgCmAWgAi4/QlwkSlDxrAAdRkCNyPA
+BQHOCzAtQEwASYBqgcoZs/BJASYZ1c8YPrcA9WYvPr1XL1xftECxzYpulmxKEXlJwjQTGmh81IK7
+9op8Ly8HiTvwImheCa9bb/r0rdOnuTVKTeGng2YoSB1OB4LVBl1z73K/xDViGQlIdW61uTAXS87S
+5bluvqDA6Q84Coy8HvdO3jq+HGLLIbgc/MtBXA6PL4fe5dC+HDKm0jxK5z4mUcPCdKeGmXHLLFla
+7y3AuqW+rgoqOVbOFFg8S+V6psDJc7+c+h++71RX1Gz+yo9PBiJX1X7n+MCDVcvrxzq6t267a2ej
+BzS3Hy8yvX1T8yPX1RW5m/t8X73D9cJIlb+5YVthbeWGHWzN+9bcm3AX1gd6UkCS0sEcQkwqlc1u
+MTy4y2LktQ/u4vNftsNzdjhrh1N2OGKHhB3CdpDsUG2HJXYw20Fhx6AzpvEMucsOzXY4nyUp2oEz
+2oHYYcYOk3Y4boeYHYJ28DNl81HszQ7hJQVfVqXHPIW1D3NUutQTsmC4q8Vb1+Lz1vp8tV5fS523
+5aqatrYaBLlzXp/Piw0aSzNu/m9hnVeE1e5BYjbbc/PytHat01VU6A8UETM2Cuz+QE6BJZ/jlEph
+e0BpnHTBBRdMu8DoAuKCBmwcd0HMBUEX+F0guaDaBaILXIyMpPEMFUnnmWTSBZNZ+Oz8XUhcNvKs
+zVReI7MrXXmpNKvZDjq/W162Warb/rHxuq+OpfYeemT3TYdT4QO3Qy3/8WBl6Zpv3CLvhbtPF83m
+z9eM6BfM8QrFDXgKaJWW6vLy1Pk8X2BT5OhzsA5W6w3oN6EjQKwPs4q/0QZVNnlpTFem6Q2f2mlq
+qK1l5xOMjOCpbwSvxWvxCLQep1sZbAv2Xn8o0vjrX6+uXtXpOWIeG+Duqlj6yitds4fXNxnX21ws
+R92pLXwS51wBcZOjUofToDCZCmy6At1iT4HJbPIHzI5c0R/ItRY51I6OgEJt5HG/5g2SB8Y9QDzQ
+UO2BCx6YZu2gB6QsGOfJgvt3p0eR2ZNsjdmLRqa0xfHko2sL0m43c57FS61F6dIF0rUKDu/+vaDh
+lh/bePb5X//82n7VqZR0gAsfOrxvW+Caz/h+e8UVS8o/+ff3U59Y20pTtqoqG79t+ofuWay+6d/5
+0P8NSh/hyWLJxINSwSlIPlHCmkklBJVYPdCNs5GZVFNd4AFvOAy/3mNW7kTZA1jDJHFfcJIxqaXI
+ZtMKPO/Q8i6xAMv5eMBAXASLGFow2xftD2AhnBcPxHTjOs6gA51OZcddjohwQYRJEYKZTY+d3L1V
+V199eZFyycaXLvnpNFzKewtYvVJfV4n1ihpP9Ba3sALeeufNj996+h59U0X59Su/eY90+/5dVwa5
+R2efGOaNTz/7z6+WQeFTNuc9U189vMbGnTyZ6rKxNfcWXKMOkldwjbJJeqJS5eSmFyaSWQwuXQng
+YEsdnf1enzzrfZhDPeiXZ5VbiJ0sIV+RWi2LF6uKcomKFJcgKBaIiYChwFWAriko4HU6UzxA8ox5
+Im6mfF6eTs0XxgO8XSoBUgIXSmCyBIIlgM35rxo0Y+bPQgvO8V5WxlErOaHOJC/m1Dkcz2ymvqK5
+YzTxz/7yp79//2tv3nPiziaPO1Zx4tuab5+aejK15+DwQOzrD0zA5EtvQhA2/QZWnfK8/XWTI/VW
+ambHT2ff+G/fO3WiNvVi6jXmM1zbnsc80JAOqUqpUoGaB16rU9oUnIbT7A/cQfMMfzmVpAOigws6
+mNRBUAfYvCTibECN8jjShU0+jSd4wP2zJn7d7GM3cJtnzygIrN36qVFxCrOWfUdQbsf1YzGpJAPS
+6qJSwYP25AkKfYFKr6qq5vPKC8vRp+XlotlcEg+Y1VoxHtDapWog1XChGiarIVgN2Mz28Hy9lbVj
+mtLZV8ucq3YCzT8R6+MrKiGdDXz2J4Bs+Ot7d40d+tXvU13jI1eO/Hz6yRduO3TLkcT+G48eLh0e
+uWYgPBIb4j8ePbls+Znxp6fha0cfL112f/SR73/vsxcmj3/9v3/n5tv46okbv/aN2w5eTx1ZheP7
+N1yjFpFp6RDJz7fpc3LUNnWRcxHuIIsM+diw2vwBndViokdOIz1ynnLCRSc85wQs7hVOaMDG3U5I
+OCHshC4nNDuhzglLnOBgZDxlcNlnDDxZnHfC/PFjHp+9n/T+l3aUS/eTy05fzVv/YVV6P+no3okb
+yjXXXgs5fLC8YX476enslfcTdrTiyCbMh7dxn83HnXZcajcr9MRuNyqMTle+0R/ItxhwXzEQ9SKs
+m4x2TEmuoCPAWXH7bM3eUQnbPMcZJg0EGV4e7MKJInP8JLbLPg2lawaWHyacgCVr6TGDDg3YhiSU
+eETul9felzr825eHo6pvQnMi9ZeUa/zItTsDY6nPfDvh3/4MUOA++pGt4pMf2Cvghad/uJR7W2Br
+FI3q2/xpPEq8LEVUelNujrIw32xXOBx2C5+vFNS5et2ioiIpFG5TFJmLuCVFdUXNReGiI0Wnip4r
+ernoYpGW4pcgkqLOIvJi0UdFugYF4ijb3YhVLWHclKAsOjc3fabI3Ubf0nJDQZtUBBwpqi7itLzd
+bMJznD+gzCnMV2itBoNDpdBrsS7ltBayEHg6edIw4ILVe3VZGd0Brx1Dj2Wf0ZnXaPz16U1Om/nQ
+yF5a4N9O7Tj0auqG1D+MQH1qJgqPHfre+Rtg+3DqL1BfUVFRAFtTU5aKCiPcD3fSD4+pD8FIPzqm
+HktVUt/R9WIbzh0rCUprBKXeqrQW2DQGrLw0RquZN3cEeCuxwbrsL4cz7Jth+oMh4h+3QS8tQuar
+qN3z38ku+VBajOdnujxg+AVaTIvpWoTfVnN6Z+qK//PqLZNXlHUmUh99+7snhhuWlMIHf5h1pT55
+pCo1+PL33GzBJJz9ZNe/736117DmT8SV/v+P/9l8/hcLf91PbVHZlfS/IjSY++kL5dTuVAu5cp4J
+LvuXAIeqAU90b5Ji5U8JbvdkGxXlGkgpf4y0Ia0M336klSl3kNXIt5ovIl1I/xbymvFtxrebozoI
+OYCyt+C7R5F+b8O7Cu9NeJtQzzbW448hDO/AO9w2bpq/qNip+IVylfJj1W7V79SD6qRmmcanSWr+
+RbtJe6uO0x3Xc/pJ/cfMagepxTU+/cXEiOveVQg8y/8EcZTqhNH5se2YHycg5w4Z5oia9Mswj9pG
+ZFiBPLfKsJLkkvtlWEUM5O9lWE2uI2dlWEPMUCnDWpIHTTKsg1Hwy7CeLOKenv/PqUrutzKcS+p5
+jQznkUJ+LbVeQf/j4zR/pQxjHaTgZZgjeQqPDPNkhaJGhhXIMyDDSlKouEWGVcSp+DsZVpOPFM/I
+sIYsU56RYS1ZpHxNhnXc75R/lmE9Wan5pQznkKu0ehnOJddoM33lkTrtS81DA0OJoesiYTEcSoTE
+vmjs4NjQwGBCXNZXKtZW11SLrdHowHBE3BAdi0XHQomh6GilbsPlbLXidlTRFkqUixtH+yq3DO2J
+pHnFzsjYUP/2yMC+4dDY+nhfZDQcGRMrxMs5Lm/viIzFaaO2sqayfoF4Oe9QXAyJibFQODISGtsr
+RvsvtUMciwwMxRORMUQOjYrdlZ2Voj+UiIwmxNBoWOyaF2zv7x/qizBkX2QsEULmaGIQLb1m39hQ
+PDzUR3uLV84PIMsbnYnI/oi4NZRIROLR0aZQHPtCy7qGRqPxcvHA4FDfoHggFBfDkfjQwCgS9xwU
+L5URkRrCsYyORvejyv2RcrS7fywSHxwaHRDjdMiytJgYDCXooEciibGhvtDw8EEM2UgMpfZgjA4M
+JQax45FIXNwWOSBuj46ERh+rTJuCvulHn4pDI7Gx6H5mY0W8bywSGcXOQuHQnqHhoQRqGwyNhfrQ
+Y+i2ob448wg6QoyFRita9o1FYxG09MrWLQuMaGDam/Ho8H7smXKPRiJh2iOavT8yjELY8XA0upeO
+pz86hoaGE4MVWZb3R0cTKBoVQ+EwDhy9Fe3bN0LjhG5OZIwL9Y1FkRYbDiVQy0i8cjCRiK2qqjpw
+4EBlSA5NH0amEjVXfRktcTAWkeMxRrWMDG/B8I/S0O1j8aWD6Ny4RWyPoX98aJwoM5SLmcysqayR
+u0A3DsUS8cr40HBldGygqt23hTSTITKAdwLv60iEhImIdwjbIYT6SJTEyEEyxrgGESuSZYgtxXct
+qSY1eIukFbmiSB9GeZFsQHgMpegzxPRGySiWxjpG+XJttQhtl61oY9LlCG1E+T7UsAXl9iA1W69I
+OhlmCJdZKjlA9qEdIcSsJ3GUiiBPmHGIpALvv6Xjb9F3MCg+T6lFu2rwrv9Cyb+ldwg1iczTCUah
+lo4w6/ciLopyX+YPEfkiLHpxpERYK8y0Ut3dyNHJuPxMknoiwXobZVxdX9BjO/bYj/J9LJIZzj6m
+m2ZEWnMU4UHZp9egv8eYBWEmlxlbHHv+fAS+ODc6mXX7WZ9bGZ6244zWhO24PK60z7qYFVHEUl8c
+QEtov4MMDjF/hpk0zbFRWXIPZp34pf2IsmxIjsso62O/bCWVKZf93c+ecdbvKPYhMvvSUb60b5H5
+KcS8no70CFITjLcP8cP4c1CeZSPolXRfe+R5dIDNykF5xCNMr4jlTAQpNCuiLG6j7sUsxgteSedN
+v5ynIpONIRxlo8j4sYLFho4kwiylUIjN/D0oMcz6Tts2yLIjxGIbkWOdYCPI+Cssj5RaHWOYCtLC
+8oLO94js0ytxndjyhRrTHszOTRqTYWZvPEv3KLM2PD/GtLcp17DcU3rEw2w92jsfn36Wb2mPhpm2
+ir/i837mm4Tca5RZFMafdMTTuRVF2X0sHun5lM7mxOc8F2L+jcpyMbYqJWRbRtj8GGQZGCOrsLCs
+QuvoTyXLw+xZ0yfPmUrZ5qr/shy1K8Y8mD0/xuZtGUEbt8izf3R+1u3Lmr+ZSHTiGrSFrRcxOX98
+sufEyzTQWXP5mlnD1sxLR5HOxiFsJ5g9cebLSjaGAaS3Yw9biFyLk7mjaNIXXFNa//o9ECEAgzCA
+R3A8NpNt0Eu6YT1ZCxK+JaQ14XsDtum7EtaSceRbi/h12F6D+NW4drrw2Yh3O9534K3AO81RjRxV
++K6S2xXYLkeJF/EJ7KbYRsTS9yZst+G7VX77EN+C7xa5vRHb+CZBUGMR3siez4BCOgMXZuHFWRBn
+4fCn4P8Uxj88/iH3wUyp6/GZZ2a49vd733/8fb76fTC8DxrynvE9/3vB92LvTb6n0hnehRzyBxDe
+vLDS9cba17v/de3vusnrOLLXq1/3vz7+evJ15evAd/+Ot7qM0+J09XRsenz6/PSF6ZlpzfjTx5/m
+/umpKpfhKddTnOtM+5nDZ/jgo2B41PUo538w+CB3/CEwPOR6qOoh/oGTla6TrU7XffcudV24d+Ze
+jh7S780VfE9BO2wha9GH287wc67H11tgKw7LgE8X3lV4t+MdxfsOvPHMg+wuvKtgi7SS770H9Ccc
+J8pOXH/ithPK2M3jNx+/mR8/evwo9/j+Z/ZzcX+pKzpa5hptXe6ye23dai/frcJu6KeBjXuKl/mC
+vZKrF5l27ax27WwtdeV7Td1KHLACGQ28i2/k2/kofwf/DK/WbPc7XR14X/DP+DnJr83xGdpd7VXt
+/Lm5C1Jksxu1bYptGt/Eb/SVutpaV7oMra7WqtYXW99ofb9V1dsKD+Ov73HfMz5e8pVW+SSf0+1b
+1Obotnot3QIYuo1eQzcHGGgv6a4yzBk4g6HXcNjAG0gj4catoIRzcHyqq7OsbPM59dz2zUmNf1cS
+bk0Wd9Kn1LEzqbo1Sbp37uqZAvhG4OixY6SpaHOytrMnGSwKbE6GEZAoMI6AsWjKSpoC8XiijF1Q
+VobwPnySsn1liNwdT2PJPJ2UxSGOS1ScCUEZZUi3AZ9llIYIKgcovTtO6IMSy9JCVDouq2PC6QcD
+bLv/E/0U414KZW5kc3RyZWFtCmVuZG9iagoKOCAwIG9iago4MTY1CmVuZG9iagoKOSAwIG9iago8
+PC9UeXBlL0ZvbnREZXNjcmlwdG9yL0ZvbnROYW1lL0JBQUFBQStMaWJlcmF0aW9uU2VyaWYKL0Zs
+YWdzIDQKL0ZvbnRCQm94Wy01NDMgLTMwMyAxMjc3IDk4MV0vSXRhbGljQW5nbGUgMAovQXNjZW50
+IDAKL0Rlc2NlbnQgMAovQ2FwSGVpZ2h0IDk4MQovU3RlbVYgODAKL0ZvbnRGaWxlMiA3IDAgUgo+
+PgplbmRvYmoKCjEwIDAgb2JqCjw8L0xlbmd0aCAzNDEvRmlsdGVyL0ZsYXRlRGVjb2RlPj4Kc3Ry
+ZWFtCnicXZLNboMwDIDvPEWO3aGCJC1sEkLqaJE47EdjewBKTIc0QhTogbdfbHebtAPoM7Gdjzhx
+WR9rOyzxq5+6BhbRD9Z4mKer70Cc4TLYSCphhm65RfTuxtZFcaht1nmBsbb9lOdR/BbW5sWvYnMw
+0xnuovjFG/CDvYjNR9mEuLk69wUj2EUkUVEIA33o89S653aEmKq2tQnLw7JuQ8lfwvvqQCiKJat0
+k4HZtR341l4gypOkEHlVFRFY829NpVxy7rvP1odUGVKTZL8rAiviLEPWxGmFvCNWCfKev5fIKbNE
+zpiPyPfMe+QH7k98YFbIj8Q76lPyvhr5SKwPyCfuc49ccQ56yoQZfST7K/SU7K9xL8n+6oTM/jpF
+Zn+N/yjZX2N/efOn/Js/7cX+GeWwf6rpYG8niEeMd+BndKK7eh/GRheF5oWTGiz83iU3Oayi5xsJ
+U6gUCmVuZHN0cmVhbQplbmRvYmoKCjExIDAgb2JqCjw8L1R5cGUvRm9udC9TdWJ0eXBlL1RydWVU
+eXBlL0Jhc2VGb250L0JBQUFBQStMaWJlcmF0aW9uU2VyaWYKL0ZpcnN0Q2hhciAwCi9MYXN0Q2hh
+ciAyNgovV2lkdGhzWzAgNjEwIDcyMiA1MDAgMjUwIDI3NyA0NDMgNzc3IDQ0MyA3MjIgNjY2IDYx
+MCAzODkgMjc3IDUwMCAyNzcKNTAwIDI3NyA1MDAgMjUwIDUwMCA1MDAgNTAwIDUwMCA1MDAgNTAw
+IDQ0MyBdCi9Gb250RGVzY3JpcHRvciA5IDAgUgovVG9Vbmljb2RlIDEwIDAgUgo+PgplbmRvYmoK
+CjEyIDAgb2JqCjw8L0YxIDExIDAgUgo+PgplbmRvYmoKCjEzIDAgb2JqCjw8L0ZvbnQgMTIgMCBS
+Ci9Qcm9jU2V0Wy9QREYvVGV4dF0KPj4KZW5kb2JqCgoxIDAgb2JqCjw8L1R5cGUvUGFnZS9QYXJl
+bnQgNiAwIFIvUmVzb3VyY2VzIDEzIDAgUi9NZWRpYUJveFswIDAgNTk1LjMwMzkzNzAwNzg3NCA4
+NDEuODg5NzYzNzc5NTI4XS9Bbm5vdHNbCjQgMCBSIDUgMCBSIF0KL1RhYnMoUykKL0NvbnRlbnRz
+IDIgMCBSPj4KZW5kb2JqCgo2IDAgb2JqCjw8L1R5cGUvUGFnZXMKL1Jlc291cmNlcyAxMyAwIFIK
+L01lZGlhQm94WyAwIDAgNTk1LjMwMzkzNzAwNzg3NCA4NDEuODg5NzYzNzc5NTI4IF0KL0tpZHNb
+IDEgMCBSIF0KL0NvdW50IDE+PgplbmRvYmoKCjQgMCBvYmoKPDwvVHlwZS9Bbm5vdC9TdWJ0eXBl
+L0xpbmsvQm9yZGVyWzAgMCAwXS9SZWN0WzU2LjY5MyA3NTcuNTg5IDExOC40MDcgNzcxLjM4OV0v
+Q29udGVudHM8RkVGRjAwNjgwMDc0MDA3NDAwNzAwMDNBMDAyRjAwMkYwMDM1MDAyRTAwMzYwMDJF
+MDAzNzAwMkUwMDM4Pi9BPDwvVHlwZS9BY3Rpb24vUy9VUkkvVVJJKGh0dHA6Ly81LjYuNy44Lyk+
+Pgo+PgplbmRvYmoKCjUgMCBvYmoKPDwvVHlwZS9Bbm5vdC9TdWJ0eXBlL0xpbmsvQm9yZGVyWzAg
+MCAwXS9SZWN0WzE0MS42OTMgNzU3LjU4OSAyNTguNzA3IDc3MS4zODldL0NvbnRlbnRzPEZFRkYw
+MDY4MDA3NDAwNzQwMDcwMDAzQTAwMkYwMDJGMDA2NTAwNzgwMDYxMDA2RDAwNzAwMDZDMDA2NTAw
+MkUwMDYzMDA2RjAwNkQwMDJGMDA2MzAwNkYwMDZGMDA2Qz4vQTw8L1R5cGUvQWN0aW9uL1MvVVJJ
+L1VSSShodHRwOi8vZXhhbXBsZS5jb20vY29vbCk+Pgo+PgplbmRvYmoKCjE0IDAgb2JqCjw8L1R5
+cGUvQ2F0YWxvZy9QYWdlcyA2IDAgUgovT3BlbkFjdGlvblsxIDAgUiAvWFlaIG51bGwgbnVsbCAw
+XQovTGFuZyhlbi1aQSkKPj4KZW5kb2JqCgoxNSAwIG9iago8PC9DcmVhdG9yPEZFRkYwMDU3MDA3
+MjAwNjkwMDc0MDA2NTAwNzI+Ci9Qcm9kdWNlcjxGRUZGMDA0QzAwNjkwMDYyMDA3MjAwNjUwMDRG
+MDA2NjAwNjYwMDY5MDA2MzAwNjUwMDIwMDAzNzAwMkUwMDM0PgovQ3JlYXRpb25EYXRlKEQ6MjAy
+MzA5MTQxNjMxNTgrMDInMDAnKT4+CmVuZG9iagoKeHJlZgowIDE2CjAwMDAwMDAwMDAgNjU1MzUg
+ZiAKMDAwMDAwOTYwOCAwMDAwMCBuIAowMDAwMDAwMDE5IDAwMDAwIG4gCjAwMDAwMDAzNjQgMDAw
+MDAgbiAKMDAwMDAwOTg4OSAwMDAwMCBuIAowMDAwMDEwMTAyIDAwMDAwIG4gCjAwMDAwMDk3NjQg
+MDAwMDAgbiAKMDAwMDAwMDM4NCAwMDAwMCBuIAowMDAwMDA4NjM0IDAwMDAwIG4gCjAwMDAwMDg2
+NTUgMDAwMDAgbiAKMDAwMDAwODg0NSAwMDAwMCBuIAowMDAwMDA5MjU2IDAwMDAwIG4gCjAwMDAw
+MDk1MjAgMDAwMDAgbiAKMDAwMDAwOTU1MyAwMDAwMCBuIAowMDAwMDEwMzYwIDAwMDAwIG4gCjAw
+MDAwMTA0NTcgMDAwMDAgbiAKdHJhaWxlcgo8PC9TaXplIDE2L1Jvb3QgMTQgMCBSCi9JbmZvIDE1
+IDAgUgovSUQgWyA8NjJEQzZBMUZGNjdGMDM3NzdENTRBQzA5QTdCRjZCN0M+Cjw2MkRDNkExRkY2
+N0YwMzc3N0Q1NEFDMDlBN0JGNkI3Qz4gXQovRG9jQ2hlY2tzdW0gLzI4MjNGODg2OUFCRDZFMzU4
+NEVCMjg3QURDRkVEQ0RGCj4+CnN0YXJ0eHJlZgoxMDYzMgolJUVPRgo=
+
+
+--=-Px1ozcNocF27wUUNNZPj--
+
diff --git a/test/functional/messages/pdf_encrypted.eml b/test/functional/messages/pdf_encrypted.eml
new file mode 100644
index 0000000..190903f
--- /dev/null
+++ b/test/functional/messages/pdf_encrypted.eml
@@ -0,0 +1,692 @@
+Return-Path: <root@srv.example.com>
+To: test@example.com
+From: root@srv.example.com
+Subject: test Sat, 26 Jan 2019 12:04:58 +0100
+Message-Id: <20190126120458.015328@srv.example.com>
+Date: Sat, 26 Jan 2019 12:04:58 +0100
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="_----------=_1574345186400022"
+
+--_----------=_1574345186400022
+Content-Type: multipart/alternative; boundary="_----------=_1574345186400023"
+
+This is a multi-part message in MIME format.
+
+--_----------=_1574345186400023
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset="utf-8"; format="flowed"
+
+
+--_----------=_1574345186400023
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/html; charset="utf-8"
+
+<div><br data-mce-bogus=3D"1"></div>=
+
+--_----------=_1574345186400023--
+
+--_----------=_1574345186400022
+Content-Disposition: attachment; filename="990777.pdf"
+Content-Transfer-Encoding: base64
+Content-Type: application/pdf; name="990777.pdf"
+
+JVBERi0xLjUKJeLjz9MKMSAwIG9iaiAKPDwKL0xhbmcgKMdWjlC/KQovUGFnZXMgMiAwIFIKL1R5
+cGUgL0NhdGFsb2cKPj4KZW5kb2JqIAoyIDAgb2JqIAo8PAovS2lkcyBbMyAwIFJdCi9Db3VudCAx
+Ci9UeXBlIC9QYWdlcwo+PgplbmRvYmogCjMgMCBvYmogCjw8Ci9Hcm91cCAKPDwKL0NTIC9EZXZp
+Y2VSR0IKL1R5cGUgL0dyb3VwCi9TIC9UcmFuc3BhcmVuY3kKPj4KL1BhcmVudCAyIDAgUgovUmVz
+b3VyY2VzIAo8PAovWE9iamVjdCAKPDwKL1gwIDQgMCBSCj4+Ci9Gb250IAo8PAovRjIgNSAwIFIK
+L0YxIDYgMCBSCj4+Cj4+Ci9NZWRpYUJveCBbMCAwIDYxMiA3OTJdCi9Db250ZW50cyA3IDAgUgov
+VHlwZSAvUGFnZQo+PgplbmRvYmogCjcgMCBvYmogCjw8Ci9GaWx0ZXIgL0ZsYXRlRGVjb2RlCi9M
+ZW5ndGggMjk2NAo+PgpzdHJlYW0KSv0zY+7m39IMcD92XE9hHmKDGw5NINuRdGhau3oGWLi54qga
+NuDlMWNdgnTziYVZIRKcIh8AhJV+ySjqg5SSeAFmg3POIzBWnjMBd10mpLmvtmmsnjaJTCfzv1MP
+UmUs45NDAtnAKSeH47ARczIiXyEdo/tlL3QQicDb7sQcvvr6DedaZZ3+Vu1H8B4/HloiJjFT7sxa
+okkAt60kPCtNsQoQUlpVt99KXhnIBM8u1E0voCK2xLbUKGTU5zMprCTeXhfclPj4yFi7X9m4IJRW
+rXoFTR/SYNyA+t0LNN9Vx4a1oMatRenyZ5NvGQHktNiYrKGxK7SKcKtGqwwl/39CNbHsWGNqcKd3
+bpC8P8ZiN6MgTI8BUFJdv19x3xF57zmt0W91GbKRBR67xaGTrr7GI18eoca/6FPbQcPM+H1JiQ5O
+ZLk35I6XIlD3prYHgXC9dij2YcCob8l6NkPIeVwd1LV0RlV+EnaDyjinPfpjr4YH/pPAuOAh47br
+tSl1xdH6gWlV++jJCgTmHhsPwPEuOEteYl1Zr3vaVVal0LAqJYul8j8OxB4dKDcEsepAtFnAvi5t
+1Kf/FxBqBSqIodzxquUqYRvEPqGdpt11O1BrAwzmA8nLSzo/pkPljRSEmImdhO7842PpJ4yK6gQp
+Shh2sx358J5q0YQTZAXL9L5qZiH9fM+jy26KiZ5sNHpuqAxqm6KWxO+OfW5eJFL4EJq5WZ4irkMv
+Cc58SO9fY8tT1HMrMVpMmaOMb3JfzBfaOOrSaBS0vBTFDPbJh+PTu0TfoBk5Bxqe1ydTEFrdtNJk
+pohHw/0UgcNaOXeQM2e8Qe7WegWVy/4d7JNVr8KqHxM03TMbEBBom0NpDonW1eRAbq4b8DQPZUOW
+1U9/BJPuW3u1C+fDO57qTH0ynANuc+kygxoZN8DBJv1h9wGBaVrYt5qOLKQdXZBXCzTgXGqNoGEH
+UHuxxBD90YVYg2PfCFd8RRGeyiMOlljEVe2WUmHctvT1RzG29VRqlFpjQBKDuCP1sIOOmawj/NBH
+Oe8ewZ8aXDRa35L7mb+H4nVBtRenV4UnEXT5HPEAz2YA9GxakL4fwTVkcEsO/q49LXrrIX7Sl5Ja
+yBqtGgQAlljureXq3LorO9HgCE9rtX4OdD7/Qul+LxfIQCAYQVqVQtGwO9JbBp/LA/wgE3qKLBAD
+v40uNvR7j9LRfRkQGK2q3yKO5VoK0DR5YiHMHvylpvt2+nvGZ3+iScjJXivAmNPHye4+K2rzuECf
+O7fLRed0E7gkHXaaGAiTaAsS9EAxFjmHc88yKWt8PmuXfdG5b6JqvizDgGX0bTHbLdpeMTZG1cGH
+9tB4K4jPIyktyQp/1GU61mstggxPCt33v9/KmHuwk27ON0FsrXE7FbAkCi8JoNtBJ8+pQdJ6Wlu7
+xhvoSEshHqsRsudANCR8YHCfI1dsZxd+sqgb5EA/9dfKuTELp6KFWaVmMR8idgaMbTpglNe/+bPi
+vkH+Yb+g1ZRkJjLcuUiMaM+SdCX6VDgJg5sRWihfXEqZQ+/Ld/NishGgY6nTW9Y49eXWsYWh2pgL
+/t6G4s/EpYuLYZy2hQkZfRNU0TFMH8oYoXSS7cnZCDI9MLmTZzowUS7YqvOn/xBbXma5Dhq6hQtF
+s1X2CF+3vTTAQYjVMyf+eWIm52LHlzmNg3Sj4LmGMy2IwMODZ7HDzBLBSNMqSk2SIeJagh3xTOvR
+8uZgK7GPXX/PJjTOgR3IkYG8SPuIMMV2HTXMYsWH1L5sSRnhwkW+7cxqzy2eV/fGbnY34GEuHN/H
+bs+ZX+TO+WsYIbhalN+0ZriqoAn1U3//ScOP4Kr1sw3bXcvRDXN2m3cAuAEkpoaGKULybLpfm1Av
+C8pLekhtS+sRX6+yHI23Asmh2s5INJLglOZigeQEBMxjAq2osjr8RUuwdHc4G9Az++ibTUr4uqJ0
+JP84KWoWAkrbtqhS7laN435YwWwavOECA4IZ74fx+ofm3mP1pGvMiO7k0eLkGskJnzXMQf8VfHSo
+kt4aS/62n4wuppjBgPaZv1vGYhK3Wl2QJp3x8u+aRDNZ6IWqxD5L4TAT1ovuyAjoQBG1lHhZJ78s
+IFCkb5jIPeoaookRKmPgVLbT7h062HJdCHJfeBZ1MfTY16wOFEURl/KapT/Ydadqk433sJCtgbWy
+H3A3DfrW+VtkkZNjZlggHOo4fTq59q3pkSlfYl0s8S3hVVOZwOZew8GAxuTeJavggrsJMfPMLy2+
+Ps4q1fcJZFV+4hjiKt1SN5X6zKwXWKoxINjOzZBRjTn1+xrhq5fFwJ1dPatlPNdtWz0VGnVKWe6T
+WZJxytspffXANXIxbtaaUeOwaVuLIiT5ypZ3maspUufsMH6jooSdvrt3CweWqDLzVeQbKrFNBECx
+Au4IXV8V752NeUzpSE4nLqKgwnBPp7UCNXYIzu7NYbwT7PP1cAbqTL5vSVOjBpeeMgw6tVddNt3h
+XMiAQ03ndKFnUuBqhdt6B6rYJdl9SzZ31x0sA7cdEKRzdod2VbUCiQmjC5FQ6PWYY5pu3mhgNaaS
+rUQCUOlt+UisQIDoN+Z1ee2oWFkpkcnhRD7QlaueAaB9nNFh7AFseftBifYglpxOART6/jYcqrfa
+CCv9PPgK0w0T/DXxENejkSW6mP6LyfdbJ8+P/NezhTAcIXhzKMSkgetsTb6NJCwpoCqCqqyDbsyV
+Qyoc1/fjHM/UEQTA2XEWby6S9IOnJe8JeL713t4NaYpD442lQwZXXWH1B/UgE021dAAxx5LF0puv
++DD+mztSQw4Y0rMPaDtUfTX/NRXAbxF7QpLA0dB0pTtGbSItqgwDFEKL8NU4pzw/QuwzLEC8MQY4
+odqHVWR/d8GUBkJEPNV+6K6DCkCw5nbaNnDw+JFy4LlUSi+Re5d0YYAc+2UWUmyOXkk2LrhGtDtk
+kLmWY+nXWuQXi14Y70H3hjfbfvMJVVi+RKv/CV888BmKk8QIkkELIvh6m+YBWSaaBtTH/o6xnW2Z
+Ur0Hxp4dBKbgH+nw2atxcHJ4X1BshfTYRjAKHV6cS1cbpwPHxW+8cgfLgdvACI/SK8nQ1zEB5FqG
+sPIAbQspTnRkZogO1TdioGDOrWrnsPqk96DVlPcFL3HcgRsT20V4xOogFtqsFpxR5gDFNCYx3qwR
+550iIRZ6J3Yalh1Uy0bTbR3oo1Fny/xGqEhHRNgwzm/TvVJHW9l/Y6fjlZTmFhOqvGqzx/OBvFtp
+W0Trt0/dUe35j9weoLHuLBhBCbVlqgUTIZx9C0EOSWJ30Qg4eqhS7HLPAJh3qWTGkqeIh4xS1wd3
+G7hUfPd24EWF7kjcYQ5eBeeBAkPFqgNpt/WO4wMNiSbmw5oXN/ayJZnIW2+zDfzfF8+PWpoFZlRz
+qShf79fOW8asFml8Yqs02N6i5E4qLri5vhL6PLk/BqGEnFPBuF6cpkcHDEOvsYYLTcPmeMNjdacf
+Xb8XcH/qxOKN6d0G9ODvYzMoqyg6LneIPAeBi2u1uSf1J69HnwfK4FdawJVaG87icGO7QrnNJxY0
+JV+HCF3geF19qXedKSI97EDUgc8KHxQnrZpc/3KA+smM5SjRks6C/rglrpaNxSzYjAXLc5rIaCCG
+/ds1ZVGjRVLvGmP6Cm7xIPzB5VF8gymtYmmnHPN44GxWQyOnyvukDJyt4sYB0kHfedTkKeGYrELA
+qEk+KlhH0vqDRUzsebIzQEfYpimaSV8iz45djF/RD4xRd34VDh5SSaN6t60+YH03r65cdXVV042w
+GrhtJmeM3a83i9C94cPxldhQ8OtZmnoP+LfFLbyEZEHq1AHRQ+S49Co9NJhUzlZ+3sMqKKwuP7Hj
+Y2bjOmLq4UVXzLTn+fVDlNh6k2JTVDK87NzERpZyVzFnQMv4KRml6ObfiMHMWKyzWcLYrb5MYrtD
+MFgwpuqglZ8Fw+FD9wZ/+BzLv/zZCmVuZHN0cmVhbSAKZW5kb2JqIAo2IDAgb2JqIAo8PAovTGFz
+dENoYXIgMTIyCi9CYXNlRm9udCAvQkFQUlBUK0NhbGlicmkKL1N1YnR5cGUgL1RydWVUeXBlCi9X
+aWR0aHMgOCAwIFIKL0ZvbnREZXNjcmlwdG9yIDkgMCBSCi9FbmNvZGluZyAvV2luQW5zaUVuY29k
+aW5nCi9UeXBlIC9Gb250Ci9GaXJzdENoYXIgMzIKPj4KZW5kb2JqIAo5IDAgb2JqIAo8PAovRm9u
+dE5hbWUgL0JBUFJQVCtDYWxpYnJpCi9TdGVtViA1MgovRm9udEZpbGUyIDEwIDAgUgovQXNjZW50
+IDc1MAovRmxhZ3MgMzIKL0ZvbnRXZWlnaHQgNDAwCi9YSGVpZ2h0IDI1MAovRGVzY2VudCAtMjUw
+Ci9BdmdXaWR0aCA1MjEKL0l0YWxpY0FuZ2xlIDAKL01heFdpZHRoIDE3NDMKL0ZvbnRCQm94IFst
+NTAzIC0yNTAgMTI0MCA3NTBdCi9UeXBlIC9Gb250RGVzY3JpcHRvcgovQ2FwSGVpZ2h0IDc1MAo+
+PgplbmRvYmogCjEwIDAgb2JqIAo8PAovRmlsdGVyIC9GbGF0ZURlY29kZQovTGVuZ3RoIDE5MDcz
+Ci9MZW5ndGgxIDM4OTM2Cj4+CnN0cmVhbQrYusqSy3UuFRDwkgedfQ4hAJh8HzhH5nOlUX6RoQw0
+Ucw8rnKp03J52nsi+HEmsYIYLDTyUesFVgXSUcwuB7vc45ylYtKTJVedyIj1e98w58x6whw4lHVC
+z+ff6wbuQVCBq236rlL/+1GFPkyx33wMgbVcn0PRMmCe3PJvPrxu/8neNhh3o27WVhJPCJIUbjP1
+DEWrE4L3z/Qpixl4ipe8VZOmh3Ph/9TJ5s9JYmcd3V/NA79ZFmmgofkN3Ai1razQYXoNXFScQZbM
+Vcwi44/qttcTkr/+hIWHtiBc1UXI0RyH3MWE6/nL1DYrLF4jPcnrLJMRSicbEAejIIhFxcbywDtF
+NYIaSdDdJ3NZXRS3xdo84iNhJWTtyxSLLDB4XZbVrWij8iV7Xk+mGoRYF+3kCguRnGVDu/3ea+oc
+Z64z6KZpI5WsSSiKYCQ2qMRWJSSNPtNa4jxaFwREig0hHlf6oZ5j7eYrMXvvbykws1AfNAWOi9Sn
+nAf9CoFjI/btOyakHUVOZdeRkgA23yPAhq1G9lSVMIlJgb7hKJNmVoK7BoG2Dbp3esjoANahOzZw
+xsvqjhlQGLthE69FFfD54oOtKZ1kd3FAHi33568mnHB1CBDM2mZe3xqVeCse2uRxrO1CeStW/bqo
+bk6JFI60oyISb+J5yS+Agk3qmp4/jw2o9Pt9O7b7kLIwgv/x8SpKIG4OW97h5g8enG69L8dPCF1D
+hyf/BpiNUZNl8tTlNDOQ63NOTc1vvuFTrtbifIletzbrPxM137DOnGNLHjA+oRaW7DLnhPh3KdBv
+JBITIFFsRY85KBeN1GF/FD6b4blMJXyfAhRHixju0d4PcppPmDwSnMTXCrXJmJJ0h2lziRYuVDur
+o//PKrekvbD6qmXjvFq1xsBJ+3hiNyxzRy0gfy4dGyRuvf+6bDWsvlvPkqMX8y/yYlfg6FVn3vR/
+Ou9jT5omWLVbNLbdwLeh+4p3M04FBsYSy3aL8FKkB06Lsa04AcJPCIfxWnZvDpsyF2HkHzI3mxCW
+cbKRUvBSub/6ib8diw/oB5SKEclS+DMgfsmThg13Co4n7w3pbueZCHoi7aNZipIQs8gR6WQKCdXV
+ftgkFBdy+YFl8f6Ht9dcccHHL52EYATg9wq9CyW8GKNMMY4a/4MF5mjUjibVEqcHQgY/A4qAVLn7
+ZqGSVxfAmycq5VD1cM/7Xqg0kwbMzw9W9yypzatGNGXCqnKzOX4qUxWtruQPnKnAm8BfuJ0Jxisx
+y7t/Fjp3TyP+U8qIzdi+jsW1YhHrDJSfXgr7hAfhdynoxgt7yX+zG/X+H0VdvIHuscSuPclWCUg9
+TJy/KnB9BzYNjYUfrBdZYrNzkW2XFt7JocfEJv+Mf/0x+Qr6NFOrwmWih26tiUiHCC07+7NXyIp3
+QUR4vYnBvwvHUsuJOXEGp8pPBLyNZp+1cMdoP2CyarLplSkPLHVFY77nhhFdjvE27oH6s56NPxs/
+JUsM1T85LbDITk1EuGG4GHqyur6DJQ1GaU2cAhGpH+zqUAFJdCAmWaEH6MVKQJKx57E2oQ9lsR9j
+zCjBStkSyKVfDHteDM6lmCJfUKf/wcIE1kwn2BscALMvhs//r+mX5HtZRh7HU+hO2PiKhyqilWGK
+5TXzws6f42s9tRjRt8DbyTDmRO4vNeiJwZp3MQCopEX4AqalTPSUAWInT53YfkRpXad2HmsRl/V3
+CkQRMR2wTBv4z/AZCEHte0fhww7m1B738wShKECh7SE67yJ/iMnYOvJ2BgB8KFf7LJM7p0V8NnAu
+OTHyhaRavbuZto1YfKei8J21+dOASUsqoWEEbg0+i78qcvLUeKaHKC7/9kkQlqklV9g5/930xGLI
+9Tz1XvK/iIe8etddEJXxzYVq9cQ9EGSp29mKmWj9PnGoeMiORt7vfMeKEgDNwUfH+xcX0wgCiJfL
+4r1RcP5CNw2s3sgwj53sRO0n0aeosJCYlchoIa5Min3WxVw7JAmSrgcCy7ghKH7lXBKXbOQ3oICT
+w1poPrWOkkl4p1C5JMYAe2SKgIfU4zChWRfWpF2d6r1DSCiv/Tj6FbxItmht+Vgwsts5+hq6U91+
+4vYb9Wo3pJe9N9PfhyP1KMNmBHgF3/tVhaVIf1yLhmkHFabqD0uUOIYcMNnPNIQl0WKk/T2uoBn2
+2nTznWdZWSFzG74huW6X7aDhQyCWSiPbWh/qrY6YcPaHeaIh4VoD2PR+H+UlOZT8h7ct+LKcLXwm
+C5BPm1cSeqcMaDouCNhNx4+PcQmbK4+w/rlI3SiObeBxpbaVdP/zz+W7JxIpzg57/xFkgUVJXp77
+IEChIu5Y9z1w/Q8SOaeMhH6jbRprK/VNo0WGHXq/AjqLE0x9//MgzgzOw15h/EsyxhEvAPCP6EVE
+JPdoikRhEXT+dfeUl29OT+rusVtrP4LPdq3CXbJ8rKUQt/R8mHE8rHn/4HysfmUAx+T50kJgp6D2
+kGaNRDngKm4hr2nF2ZFzW5Jkfk9MgpmfUJ/mgvvsUJ8EVHofV0NvdocsncrvkN4qlUACG7GknkHa
+5ZueSsPQ/WPCwbd6SAxyj9DPHUa4YYplgGvJGGrqPf68tr/NJ+elrBkxObDRn5qWocJVeV4rDyV0
+k9IZA4rjO2114cTufbxQkmgNt+cdPW1qK04NFZsddo/5nN9hCdMJteJTM9w3JyI+Nxs8FtFRzi7m
+Zv2yxgpZmRRqOdColg0jqHfo+qbrFQB7kJdMe0S5BY0Xdbew4vd2N2lucY0yKwjDyGRIr9Rk6q+P
+yU/t3QWjJ9H64StzyK87WsAkxV9z0IJhFkeH/a2PoKpxKCy9WknrSe5bKR/7cWr0iE5B4gvpj0zp
+Vikkdo3TR0najttgnaZTcpk+ryWBRjtrVL6glgkkyVkTDS2bU+7wyhR0zFo1Jx9UtwY+Fu4M4S8b
+7xM3s5C25C4PwmmE3tzz72YTPj5WT5XNblBjvjd470ZxTpMqiOWGKGXvP4NaZ8/OZBzI3P2cCOwc
+Fqi+UAEF8YHg+ZpxnqWFsKPJ0tQgIIWK/ySiJTd3eM80MiGNMZbQ0ob91MlrOjrRWTm+HUhWP87s
+8mFy0a7SdIau1Mm9iQ+l3bfaD92n+KXAnzF9leNAOIAaqJBcODqCH/G3fZyNTDVY80tsHY7UfOuO
+XKQk+LXybYKNj+xVOb/H+NV6tLZaB/4yJQv56xvrkGEQTe+AXgp+Nhb/YjFKkofltmllI5YT48S/
+aHJbRv+b84NRkLIueTk5Pz8n4dsc0J0sguZ8ZZpltuopyXcCiKOXWJ9KBwAPN4FhAaVBmLO/EivQ
++ZS1GcqGxXHARcKMa0+enJI2Vz59IKC5ySAWENro/3UBYEFc7+a7xPZV4i8PD/qgQ1vRE3hclXQc
+vwUQTAsa8FnrgGKTJ4x6km6RhsqrtB+7XxgJabEA3UuM0wJHP8XdErDo+xcH8DkeA+rtNQyUKdQW
+CBBr7ZMVNncDsmMSRX4Jw+d9ojBvUZo6uuKnZfJzQrHpAze2Zyyti3IzlhCJen8fU4vJc3XkY8sY
+YF0Da3FIhBAVWF93NOyLdq3Stkqt828nDG/xsozIEIi43++DinqhjyzL4qMjIfJ1d1JFaMKFD0a9
+04+NUr6vTWVuPJbjAN2Yc1U0jQkozgyl4B3z5cn6/UTPcMMOTafGhuT7HeqEeUkqM8GQZtGWk7mI
+QRShxEv/E7ooulLkb1uRMu3vr/AK0yk/dzIPyryy7jF3AaVXNWR3AszjJUi3DLT8ptCBI6+dKDLK
+qrh8HZqSy+JQjBRQJcJth1EHBF8mF/oS3pzexDli/h4Ihd9xulTn0gz/aP1MDQzHNP+xOuvur+9B
+65m61N+ScoqWyqnPRUfNoSrvRxqH6dwSi1GIaSVhKVOYQUZRBfSuEDNnFc7rZnKDT1FNVATTD+rw
+W57SuGXD834D4wABjVWNBnLtsGrN1NjMVQ4mMPTgdONVsLx/i1CYebQ07ZWhW4GhUqNtPdl4KZzp
+/JvGrUjGVOOus4kORRUbqqyge/kE5ybKxL6LLay0iKVNRMgxiOlhSdnELMJjva9CUYq1rx4F8z0u
+vhrKyqRr00FuBvUO40xBTScq0ycukLI78w4TNCpT4+FmgyUfRTsikxmxU/5gHRegEz8gz/P5DYCV
+ItZwBP18YgaBReuL5IyFgx6IUF6fTeZ4QLwFIn/5LlxNPc4i1p9cZUD8nOtEmwzs2/Ih+yKzIjgz
+2xcLyx4XqyUDl2CAbUnpXwLbRmM3xeAio7j1lep4K0+V0+MBC0S0g97KZlBYRSutW85C2EhcZEGi
+nQmsORhJH/Lz52zyIp+GWKkyAeBzWYxfm+fxzATeF0rpoorePIrPWVcvtAWxD3le60LKNn/H8+BB
+6BTBUFFq4+yK1EFLZ5voH8hqFT+VHXzIhxGXWmJHU7YPrMz8ZXiN7UNYGv1OrLDOJX0bEnVuo8aV
+O/bjGJrSw1CfXy/NvHeijsiuNAWewyEQSMoaBG52gucbmWi8H+RbT+Th77cm6NlGR68xkaDP88Eg
+UIMnB1UiPH95uLFDtXFBAz8l/yvgQ/aDjEQ+98lMITZHWFtk4T2ACQOjLY+o19CzOSW0OyHEb1W3
+IEfxSwRtQgyoUPt7fLnnqJF8o72kqQGk9xKSIP3qW+4DSZVnwhvi2RRz2hMpVlwSlTf0ExypiE9V
+Z073LY5U+O3zy/d5al03X9PV6U3FfElkmOwSzZkAaQ2cN0Ds0V53AetW6RpVKCPRxuIzStV0Qexv
+oFonPAkEMtUqJMaqz5YzNyYPAIPAsrACTrksqGM3Qly2wwW/K+0DDy437yzP1jKOyTT7B68K5RMK
+0jrb8V+vwhfTCPbhqSMcCBuA9ir/DHIVpvY+mryLHJ1ZEA8YeF5fEb6NATms38ZM6cJ4u1nWu/M5
+Y9Vak03L2MgbNvIAvLljLlCoT4UJ9jLDyG78c1BVXJqHdwXd3xOj4Rddlm08YjMGaZrk7wQ/XUJM
+ihAT33QDojB0SpWbXvzWkCLn4zBMMOGV779ZrGU656a/2ElnuqWwTHIYOuESJfMfwmEjhFbJJ4EP
+AS0RQUVh/fdbGuNY4RgSNCRXLDbOK18AlW0X7SpOPK2B1ajgvVMPHzx2deOZCMg2vi0fIKv1DcDi
+VoBtu28Stk5J0iABGcpVhdhvpVkRNkzOrrH1ZVhG4WqmxI6JLqf55lcVl9kIwcTcSKN12iWSBvRQ
+UNMihlm/WJUsi7Bo6YGYsWacrHG8PgArAXPJ60ELq7NQaoIYZ5c4SAYC8io9IwcTz04rwEtbKNT/
+8XAxVMN5LZ9xCQxv7etHKcfuCh+w8zswG8YbMSl2a4TqiwFB5hXfbNhQWULRVO7javp7RAyX2I0h
+yc/xABGyT3XoDe8hIqKyWihbi1ywb5Fuh2hGZxpAwvk2U7QLb4q6lqUBWHsCyQUbEYZqVErCvIy3
+CkJjCGM/BJjtCxf475bAZOeZAL4bSNAlqLpkPAD/WvAq3V2YUTHaBcSl6F8FC2GgZSyRSptNLnKc
+4Ce0q7+qdYCacma/MLPWyTgIx6IbXdKta0cu67jUDProzXGskrSpi2BrMik4WI91rG76rthRiZ4v
+7H7PWVYzQuGZ4szGINxx2i6VwSR/MW44m4VK+sLtx68vUd98DZ3LJyqMvAxAE8/WXPAzJB7dQJQh
+PX6+h0L1LPXosEO72j+seE0quMydbgDyzEGuP6YTCSAwd1Zoqi8AUAph+aeFxPAhCszeZHzRbQYA
+vkLR2jopZoT6FmXtXhz97ldTdlad9vYvtSYHKCS6N5/GDzYxkR/chqJMuT/n4dfL029FcvRxZ17I
+6XqKDEUZ0AVihDj30FeK0GT3r9VAgHR4CIEf+X1hbGFtyYoMIVZGq0iinL4AM8UFK9Hje+xPSuPb
+ugMuX+hc5Fvow2NO4qbAXTguFQFcVqJdIcKXPYJR8oSUv6I+g75ywmLZnlx8BbE2fGRx3Crqqo0a
+QuDzIyiVo/X7y8hkxWREv8NnxMqi2+GTwgL/zBYitqBsegURtdSG75e79nc15L6GGSG1vhOr5haf
+ZFp35Jq1AeYn0PNRsNsOZQzGNS4jOmVvBE8DgKPCIngcus7UAyYURYzOwprOkTvTj3nT+GFv0EvG
+UpKMt0JfsCJNpd/k74kHB8pwuRLSp7K6dHsVVBpWsFuGayL1pN09XJFvV8NobbrqLuHcAqvot6Jw
+lDrCMlvaXc9pzYKrjejCho9ITdKeQ8yuAS3kdpGJw+VjyQ8o6wAeaN0XPfYkGXYbHNSruFxkvtJY
+npqqYZhS9mFUlAl/ebp+P1H9rO3d0ya0abZ+7mTCEgXcgljzpPnZqqrn+qGeWjNd67d46BeQu2f8
+YIwsjEpYQBqMtBSacYd/e2W8J6VHJEGoq+Ai3fRIb05TqjdxP7ToQNdMIvWZyBiAfz9SMwtquNLf
+usqwUqUM+Ntj7KRjtfGoH9IXdaZDdcH8apw95OT4ukgWkTYBvsQldU6n9GbhoaYip3feFe+JoLiJ
+mbM/X06PPpW/DU5lKm8jAC50iMvUhQxrTsON85yVFP4iQhgGFHBafw/phPhcE4Tn4lTiTTo55H31
+zqsj3zkzQeLOD1Py9zQJ9FNfbRqHFjJPSnpgIac2hIjLaps0sLx+h+MFCwtoSldv7+ea5nu1m2gd
+4DcSGLzULLkjDbK/0nffowT4tQcb4p0I6EfrGep1SbeYuyckdCl5jVc4DGoeW6gz5Kt3CHY7fdvy
+/tPX44NMbbKuuFBr83ody8Bu9AlaZZ6Z1Ejc00URF7nxlESa7yEvTvL36LUXwrMVipFbjedkKBb5
+gU1DDV616Ab0mt3obHCQ275uWIeXv0ZDbfjsTsuVFJCBdO7AMeTIvCeWxsaD4P7zjPm/UoZijI8L
+QlTSt8/oZm+JYSmza+co+n1FyrkkB6jRqAFLgj5ifUusPmSLZnNJTXKy+C5ON+8phL0ySUh8x5xt
+YFtXoGmvv5mzxKaib880f6GFNArKsVlIhzJBtWBK0tgqGTSCDB03YZ68bngSNxxhQhB+filXmW2i
+OBXkxfLtUEJJxZaFrYggWUWBStrajHOr4aLo6/1/CwtpmAkr8f4jNbmLSngNiD3B7vrgGSktPuea
+24FSwdPVvKZp5uyVxsxBSwMzr9+hEIPgWGmXZfQ18DksydUDdvoqIZ353foKGJQsWXrjIxNGUIi6
++j9iBK70+FI9kFITG/4CvoX9eTUekyUM9sZAdtvf7Ty/pvNMIoNzLQC7NxkGwPTC6pQkCfBLFiTS
+GtxkZ05JDnaXEtqTO3VGxWjj97EAW/PMc/jot1fwg37MSZJK57JHTHCie2dW0p9l0IEBmzIzqb7T
+4WWplG1bwG9FBDLT/FCwt1aV4h1jTgSDTFlWCm4TcItzEJ6cC3r05MSMEm8uMRo4cikLIRjt7gbh
+x+KFh1k/gePLF2uXzwdin7UXGZz4n+v6ltbN+xelHEY7tWCULTojHetXFPqGzLC9nbPoTiSIjMiP
+VVMmityBRj6K6GbX3ZWorGXtTI474S+jO/RDmRPecx0GGfMMmi+VUNsgDTLMs1UYeTsKHJGcZl0X
+IwnbUgCJYD8wYAWSQyoaeywzHjfBwy4J2eMso/wqL0RfrH3kymoLUeoRi/O7ufN6rAA/DX3iaT7Z
+0ixFrrVnmS1seg3AZkGyA9vC31qQlEaK0P1URJnadoGi7RIHZdEL9heHc/aoWXjj7WZTsWz5x7tJ
+565k1XTzjU5Zr703Lxg8/JPIsxVYeqDkQEfbwEU+UAOrEGIGH1EjAYNTjk0zc7VzFXP1rNV0MpGp
+eegKIDKPrAKNGAVtC3Y13pNZvDPv05gpuIabcv2XNezCkC9111yCoxxNUFVfLsIzPJ/EPkkeGRU1
+W97fulLcHPBDlT2ol2HRCJ8PGsKxoMeuRZXQ1zSdS6SdvsOZYF8DRxsbYZK1ug4KUMTq5lOi/EOf
+u1q8YRsJxLap/Ah2PUs27OhB3UBR93RfAb9GuNPoCzFXR74NXpgU8QeG1/BlvSanNc60qSPQurog
+whRv7TaC03Vqh0VXbn1zpte7jAIYx8K5wEloVOm4veRUGayouz6FHjA4CBfTaoRkZ1+BFWC8bPeb
+OE7P2SXcgBdoEnhhyP6tB15fxw1atZ5YLmj52VHCbM0fGLzPoZ/yCqXwRM5cgjr55XjK3Na5nWFg
+R4j2B0m0CqSjYDVSEyQYulfFWodDHxFjeQGRl3EbhttkLTJYAnKWVXBfnqGBI8G7h8Fmz0c342i7
+uKJJOVRLqJCYxpDKjsmEo70buc1gKXQTHhK2N7ms2fGmPp0pxXR3xj2+vyfaHBvm3wqD+/K8FzX9
+gBCgNqxzqKNJhWKLzVCyMSkp+AUFIu9qm0vbPYx8LVeEmhMg7redbxV36shEbSQNtStmym9pM+XE
+nIWdqo+ALNicz/KCkWsWm0Xp8V+AVlQ95Pto874T1d1FqB69WYMkdwwyB+0VmaOoqDz8YS8k0ckh
+3vY0vD54iIEN2jfTlGmiLOUrpPtagofdGtDx6VKzAKF0iGywhGJSgsaqwilmAVKEQ5oHFI8v68bp
+dhCcMQeRs4J+iXsQlXNcqy+2wOu7ywqEUXtCeDOyg4fMp9rTPQ/n3vgQKlJQTrANUylYJiWOFfYT
+AuYJsLuegleR2qi7OjUq/n08n5pKOSXI8iJU0n9277HVyAgEumIxFb8h76wWrDMxjy+xuzAZxZU8
+Jf8DlyoxK2EfuLdj46F0y7G6Vqy6XVfcjo6q+JYhcHsdv51DtsNccYtQ0GHlnk4ppz44yhHtAfcT
+dEs/HCvRVrOUknr7khIF4Mu9HJfJrav+KhFf2mfq3dHOSCq5QVLNQEfDTKadh/1aaIeFFpFEjrp5
+bJ9tZYp8XT6PNwsJsp83IURJBrboPOfBMPU/m84InciG/3DDM7UERoSt/krkqDQLBDa7IFALeKU4
+2jZhX3utdAp7AQZTDDnTVve/KJ0QHBEwXDvhy/Zt+jXbJjotaN53aOGSB9C/RYJUJK/L6AnrGMIy
+pby/oFTEins3DNxyObUrywgcOQi0J6qutpHozdL3k9/YHSAqve0IoHBE5kKMFt4oByG6ha1B9gym
+8XsSdfrZRjbzXipLH4VZp5fq2SOG+PDHQYpwUHsFsel/IMqQAGDuLcl/2HfVn2UyhTJh+cRXY6J8
+R2x32I0i9WvKRNE+XOMKrpOh1K/zyanKEYgQ3eFsDm+VGykJK3jgaEqFhy8v/jgcwlntEQtefZsP
+Pur/4io1iL36UTsTqQAWPdemjpHo7GSRZtsfqIKSy3DXJS6pvrnMmyBtZg4tJDPDRetNxpwUqYDx
+7ls0eUhnR6K17htxbRme9cdn/AS8nbzcrtyqlOyIQBsxculdFFXFBWNcmuTMhryV5Kgpo19SzlWS
+CcrFYLG3bZqSdmJIDqsO643oJyDI6szuSS2HLBEktZ0Pq7KC6mrno3qz/sSP0glMSVBWHVf0gG1q
+f+OENXURBJ7EhEuP7AdqaFop/ajTjyAfoK3+Bei7/7UPoAM/jAEHzQx0uVrxVnZO+53GGHGKzJ09
+uoFu3BML6RnoFjqlN/4cdhEcw1tmjzMLrb+WaW0JKHurRU5+3g5OaqTAHiSA974xaO1FUpEAhW/l
+m3RJqkvoHijtyF+5F5nkxW5BTVsjoCcTww4JmGZatQWukHqLEvYlTlnr8JiYMaFke5X2nAjWfvAu
+MR/ftIoM61oZ+PGOgoJ5QhpTVwLe0aygpGghslUMAnO/qiAk4RIXT1+GW3N1PtJ8oSgIbdLdWfGy
+Ri9JjPoGFyMjEZxcXHGUp470+0AxxFIjW8k1Ttx0/WEHwaUFzYzSgbO50K4lJ6xGKcdI/CvmBrdQ
+m0+HSvQREfrszOpZvZB0qLGvf802NlmIBMrcYa0s2yuBTsy+Oq+S0UMk4EKxeVoF0n7QxAfT2Lxk
+SjXKQ3CVDeDAIpALNH3sA0TlgNa2GVeFe+IfH4haBxdeqZn/eTOVmyVFKcgt9oQKiy/YAO17Pzvz
+wAskbuCNzGVC+QTGVxmkJLnooSp/WNEoLfs7QFVYdCifAvvfJ36cnQIKgHYlG37+n32zP/pvInlR
+K653hClUeZaIk7eyEj1BWVp/4kmLbiWJCjGpxv/YpLUnCbX06MRS3yvQZ9dUhAt+HsLwZjRM7JRa
+Y9OPjSHPBgxXPPFx00G0Kd+fAUN1LegrFXlXOxJiBn21dNETPdRejuXzKqDpif6PBVcNwDZPpzWc
+OBdpHdsg0DgHF3ele948GrEDQZSNMjRL31KFZdZx+jrJFU0kM1eDnoPC6EU95u9Ed7Sr8C6dHgUh
++pyzR36iCEqIYKJQdHSpKzVq/HPPvthTgcZEXTE7x8YZRVSHxhlHecoNRbs6V7TlAR9FkIf3vB9P
+c/owQSV3DXfJA6gQbe/+eo1iIIPi0j/hdb0bR+Ix47R7p0KYPPG8n1BkLL1+eFvO18M22jzLTifP
+yZ+pQ4T1UmOR+4qW2+0QhHdvXOR2LHEQ7IuvrAlP/kcPDyNLP/IQ7Obawi/4YUi7Ki/22ZXr4+kA
+NwmxzgSzLgbbilBdaR7J3I3E1bs2ZspTFr60ahXVwo2lQtxHHZ3avkm8UBfX2IrkeGthNt7LkeDV
+MYo1mlxt2oXHChhXSvDxMziuFMUgNf2hEo1HjRm8oOlggkV2Oytjdh9B4Iyy4dJd9CrD5RvVa0/E
+POA/90DDfs/cJzgWygsbhDumkdrHOM08VQRUP5rVPVBoFLtzDy1oM9JF38/D8HXbDwhfBeHbMa2u
+6UQXaJXB5YrPajJaKPqfMuL6F3biT803ow5lvRfoEarlPfy1VE88lBTiErD6E+jfNKTqo2cEDqF1
+X8OcZvpnUsa+e3k8b/+JYt8ffLU9H9tos4IaS/f95rvhrpEiyo7fuf3kOVB9GHVeoUWce+RUe1ET
+ESJhDjk5INRXNUBSLFJk4P44v1WjpCq9QFhX8rBsf03UA8r/L3oOz/n0BhFLPhHs2jDdbIoVnp5A
+z5cb6z6R68aqprPTWF4lNt7SGkT8ZoGQTJl+7DHq+CG/CVmdH8rO77Zp5RxVpz4jkKPSB6M9oXse
+nMWFlZDfeAmuojxFlG0+F8IcmF2xWwpshQ5TY9y1/tSVHmuBnEVH/yiRgMlP4XS+x4mnrJIiSu3g
+emiJ3DXHudMi/Nuht0niIMyypCY93EWvkItDxymuzlNNrN9kftY9+S9TEVsSGw4AMFx9TAAL6kYh
+qQT38SZCjtGWl0qepDXV0fAtQ451i3Tvb4t1+K6Vv5hCeTsxjNXbiSnXiVVwriPW8q9xe5HXbomM
+5j0cJiCzQOrVPbpJi4hFs/9CpBtNbkoaqCyCh/RUH37SanxICqBhGyj07I9srhiI/f061Ke9EIwv
+/W2POtNnY3gpjug9YLy7Vwml7MZnsdtKC58P+WW6dDgpeLznN4bxZBGLPSj10w0HCAco8zDfZdLs
+wepxBN8EWD2YO8R9Xzd83NshecFGKe4POOymV29sJ9ZEFb1DKo1FeXxjYCq39vcBU66m+A1s44uJ
+9PNUSpCx01SVhYJfHOSfis9EZdtP0oXWX/+JKUNuGdvIgboGFLjR/liekwSmJODq+hDkYVcWyJvF
+GnNDbtBEowEOPuhw7pNtJx7eCa2xVhXpqknWgnvhlCiNvq9nfKqx2J7pd+sLwNe6l+dfdyNN56Od
+vP2/jDNULjcpPM2oYnPab0VTXB+pHZ4JatNLrpPOiTmC0DSO3XsncppF7doqg+jIU9PdXAMzmTsr
+NzC4KOITQhvX2rYQF4oarQPzT6luK+mb46zADKvWfNLEyltPOypFZuaKyP8RJcrWdZg2UwJvX517
+2Qwb9TOu2cpaMcu11LWwPJ8B57IDeh/+ZFq8eBmKy4vVPfzHg0ODb7d+Mq+RNAvu4s9iTGTRP9mw
+c0jXBmVaIawQCsWmB5H9D7Jdp6a5YftjqAc7ORLhkXaNHJg8ApYRxbST03IsmyoY5Mslln7PXxaE
+VjhLw7cAkaSCVK3dvCeUzEaehBssXZMYu4F88yxad0jZhtGPPj8uIgI0zjHtIEJ1HVwuIVX5wbCF
+fOgoV6/xsKZjOD1Rkg/sRmkF92AsBZch9hTw+1zO7mxqeb5OoGy04ktRNyuKRY8c7bR+wyosR4NT
+Xod9/7Hdly29rQGJ5Ski7MLzD9LJPRtuOntmyhJu2nWr0luwnIePxWcyQyiuojiuXxhjkydWMqUl
+rJmu5AqnWoFDq052OHmi8yAi5Zdk06E0+DYkM+LyzFCx0XC/uFr3o3La62FmSwnMuYBHmCDEQc99
+dyfCx1/t2LKnBdT0b966q9W14oQrzMxzGjGKFXFdgVTAz68YUou4Cq+YZPb5GU5GgMOg3OfIV0Vh
+rISlI+NBc8/uoniPmhh83TW3h189u5hGVNv+kJwnXzKRrnq6lgW0myzlPmXBPFAHNAnMv5Ubr8S/
+EiIi1QcuU721xGbhrzNJy8n0oNOMDOqN+mlewZT7bQO2Ix1JXlPc0iFxfJmLNBGOC36BYsoDyfK/
+71IhxxajUVpH2Ew8PUrtbm6y6SAP/IKCftrDKQPloOMPmmSwEM5Cx1NQ0g3W4KIhapL8bLxngCsg
+CfYosIVEy8W8O00kw68az5t3e2jsKQbgzCyvwRDsawYuXHzY87P8CBhgCaelRlmLP5345r8fW6Io
+cITGS/KiYo4U+vK+iXj6zCGDkWJbcvSYyBitJEe12I8C+tIJbCgdaPV+1HjiUs02UlumY75ZFLXU
+ovdS5pi3ZWxmc0ZaVbdG3/l8Mq+1N2+uHqdlf0rI5pJkFXKhyBJPGSw7u6yFEH9E94oUiA+HRKxw
+NzaIvBp5dao9NJA2FMJa1x293pPBGNn9IQAm23Xc0BIZMB86x2fsO9ZaXf4IQkp2LJyzPYt4lvC+
+6nUwx+Ek1uS23ucVB0LM81LCs3qCsxXdsYhQR2i13ifUOXWhUpWkhAT5iUFR83f28pn+dRN0JSZu
+O6P0NtPkbSJi1MVyv36qVRg/yU1p/7gITBsu6D0g1oYwr5U3GHylemPWMMH5rFQfJSwOSOZfyY5a
+GJ9PBZ9GPgUrfLFsytJgN4iqoGM0XsHBRGydzfDZyjKXlDKwcnDvKPdIM+iGclOadYSMg172UORs
+RRVKi3XM30V/2w0b9cmyjUX1+FgbFAi42OlmD1C0rBZ/zPur66lXKq67aKbGjPsDmdAoGtfYy3yx
+N7ep5M2jJ5NNjJjeNf9gJ6ov4YekATgceIs1Ku6ueftgXZ3wr3feJKKCfgO28EINEXUb5F6SzoqF
+MtNYZpM3QbFDm4VdJKvryKUS5cD3AlRMRmvPBQNstw6JTt3WuiazrFdLtFtpXgtXbB0yz7nWCwhH
+jHAE6XFfUu5m8szs095vws37PwcVlJugp7NpTFpVMkvb7G4i6KgJVtmeTAd87+wJjkt+if3hwfxg
+m9fGf1DovySpeG34EOxTDiY6RZ/FLpJpvg8IsZqxvwKoNCe7kBRRqIPTJvbItT7gBJAhmvBSmBSL
+2HTXA2Z4M4pv+hWFFocg9fCs/zzo5F4HnKG0+RG3dUkz4gs2ui/pkOREVY5ion4YjnMspWxW1yh2
+sgn3iJa99x8swKnzDlmX6fnz7B5A4GZdUVOE9SI8c5F5lzsLgMIZvG4PiNRazNtm5X1G3kLypE7p
+UJhdhPjb/iQPBHxoDWR0sf5CtyzDna5SJZGbjI5FLISO+Q7RW/awlff8auI+jhEHLiy0PrFs2l4y
+jKUrWQmLiQ7/F4DIjuXm0Tw/0I/UB9m3JqIs11qJbjYiJK5oP9bhfmBdHQFEy1VtGaUZMZ6SqXAC
+NbbNogllHDIlMFSIJVc14NR1axz89+UW6oGno2eaABwSAFTWtvB1I0WXqqtX1UMJ50tlYTqk/Nrj
+YoAOn9Eh1oTrx0JG738p7CLGTZXZuI0JehdtLxLwYDG9zRgOWYy2EVSAhA5bAKoQEwcREjuBrSTa
+Rb4AUMixnRRIpXcoD5ok46Zn2Uth+LrYKVe65DH1J6e6GeqQ7594F8euKJRBirQA8STpJ7MzO2bC
+4qKmitkiv2qnUodzCJOJxIlQFj7xBfI0OoXi/DzaaPLfWxlNIrgqghmDmMYvt4inCqmH8zZintd/
+7n546Gk3V4+5q0ErqfPVsXPeK3v4v58YpGXOEAoLLnVrtg535if7o5mhtQ/bnmyJJc9NS+4Vuy1A
+qZ+6dn+eowCxusTb76nOQ7SVNLcvUxxFiQStL0IFe+uF+QONyTUJs83E6tQqQ4i9LlNdtG08zeag
+KVM2i39IlgN7FxZJel5rcxiWztazHwY+lp7EhFX881x4wGsZ2UZeDSRw06Vpp6xq+Wk+TMEIZUNF
+k51SVw1H1rPcHUepjDcjPwyLP/eQFscQW9w2vvv9CoamL/cFwD9rirT2gVuc77rvSK4h3sudhLNY
+fy9MhbgtN4BwNwBuY4ktawW2qH6h3Q5RtCgxD5w2/SbIrd3+5CV9nu82hkzgwYsGIq3kJQVLE4je
+4JNw7YQ3zp543UCjT/djuwBXbssdSCCC5qXEqP5kSRfzwBA0faxE4qCO38pahe8xQ8hgoHL3OijV
+yD4SQG+C1SwSeccOwXlCrRIoa3DWVYi2sjnlvT2AnBbACwwNa3lJd+h2+xDhJS+gJvJ3/cwd5WGQ
+14Wk//zV+yBMwBrlCyGWBHjmaXmFJw2Np0g39YiKM1W3fblN3gAe78BIR5MOQiOvDmu5ucqsvK2u
+ma+8jpo+I+UN+9fg6b4bxqQc3S+0bjB45O9ZGga58W95PJ0GkSHyPFMhPtnG0/sgudMHeYnIW7bu
+27pG1HQi0HLPEaCkIZMz1Aj+p2rAXMPfUkb+cqiYejq2jTViaOiiYPAwD2AMjmdTb5L6u3/nnqLF
+BgE1zw3HmdGSpTIVZo/k36oAlExs7UvEpGyN5BLtcCXkYdGMjf+PATFMt62C1Tt0/6DHy35rpU8X
+dJszaWuTetTWmOxKE8X+5s/y2tE9okGVhe0uzYHHWiylyTsytuIOs3X4Qp1GO+J0ZY/loI71pccu
+pJPWvYrGUQ1TGb6JCWsjpgHOT3xVg592f2hfJnpz89un/9XUuUEYEDOYq75pveVI5GUE02eHfsht
+0nx/V2hFe1JI9I404E8y5fsxP5pmXcUSpb4EvadrmvWOOOKfBBgkme9W49Np3fZckBPmeNO7iYq/
+5bPKh8U2qEc4PTlwGI1ajuHW4205Ef9jgAhfJ0SBf+/0i3nJqdUFfSKh+oVk+Z1UHZiExrQZRJWm
+a8Mg8SPImXOeii4FjqLQrG7c+Zqefx553yJSBE9Z3RuESrkAysUeeoG48WiFCKYql7IO+lu2Hw7z
+9KX+r27pcV4nwtUTCVDzzcuBMlQo0b1Axp88Nz4swOOi5Whu561YynUrjo/ZXbfAPd9dS50m/mD0
+5vnj3ZX11oD/GYEPyiAVYd7TMZNgg1Fv8mDjoNAnpRQBVQrYTktVpjHStD+yBK6IJDTo4pFF4Hcq
+22KTuqd1TjS7N2VPfeHxgIxPGSKz8iz6IyW8xJvRVzXc/xsY8i3ieucfrD6VYw3cCKN0jeMWgJyG
+flpkObA1I7lgiOPplKPV+hx+KYFATAWxIBE1+Yer24koXNBaedIrgiI7X27lYqd/w4SmNbAzzHnh
+1Q3unC3zrDwlZQHMTKDB4anZUyvbcl6/33tFDqPksKT2VYGhAAdZKliB+oGYVDluOrpfxektUX/o
+ezKDIkl+Nyi9sxq9MtjRkQ2qMXYKcc7J1cc6Q9QbVoyCyyqB8gL1TbkXRo8aHBtw3ynRRqPAmNNv
+ktj+iSPBTlZyDTw9rHbLKJ3GHPPuzIN6mzJhn2bhMg1cg3JCBxUUl6oAOrTcDaY1R0qdJ84hwdxp
+OgI4FW50qD9MVmtIvd1ERowuRHVRsszDvfHW8UtRBfg13FgboTJkNntv1ovmBAFr2O6bt0h7n7Z5
+3Bbe9nh3f1Tp89AfSlNFjvCp+RPTH1snIex/PCx/m783+TA+vexv5rVdnsMVD1DlTxIoTx+n7QND
+aoVdpwYYDVg+tZIICPmRAK0aLoZYHio+cwksiuAtZiIMGis428z6ATvKKy7B4mAkvH5XgVJRjfPV
+TXQZn4npFnJ0L9sP28o4ICNwmNK7qdSXZfdrhafElH6dkJTlcUXS/NaA1fZ7kEEkBXgXxZTXEY33
+xneWrgD0e1q+CwKECh26vgpIpX/OIyIR1H2O0om22X4BNbiVpziWuiZ7/ZINS9b+0RPk9iQ1UfIP
+yZYxtF4ZLd1E0dO/gokBF3S9CSlNl+RsxTt9Wy5Nnw7L57BdankXHW57VMaW7SWezpJqIby4hwKe
+l2REwuBGZxqs+XbuvxMr/fHKUvTGjtSwl/H2NIQnfYlaOe3Ap/hIqK9uXWcuZrdC+NPj0QSjsjwJ
+VuQxnm399CQgXaw+3/0kHSY4dLYdsdtFHMT8F9D83OA8sJ7LP8c6yRN5UIjeJ+MQGwumDkncTGad
+fpaWLGy+rdWvwqb+9mDVUzy7ZU6eMU9tcSSv4p3/vz6T2N5JpUFp+La8RwcETEtUfiVhe63gIuH2
+vBSCciCxLv8VVACo44ObstvrKVv58Rkgn1PU3b83hI4mmzsHpDYMMSW3A3wQ/yvsHSh8X6dXwZ0S
+TVpvyvYs90+Uze/WsFtHXrb6bBMNSz7+TJvxuOY63o0gU6adjVyp1SX4/L60bMl2VkWL0mW5faii
+nfaDGrbc3NLwnWSWdXAzWguEzO+KlnK4JiiyHy3vzxyrvJRgAY3pa9LPigweSP9opW8zQGK9CoZs
+3cAuQxCcbdcie8mIJYkUVLcm4cOa7R/crdskb13iT6KyD6b8hO7Ev1ooR+r2YIDhyMzGH6yKoedK
+6K9kqUimHgU+mrrA8vK5AwqE4SBfhePKd33VGDSAzxyqS/rfUqkSWfAjUi2nQHfeMPaoe9du1JtZ
+CaXKEo4t8KanV1oPH3zNgkqV6iuO29CQEhC4iizkSCRFyY8lUdOglfAKlLGfs2Cp5OwMMr62UhPc
+GtbEsftzX4KsAfdW57RrnAtTMAfkXostsjOYbaPjnnTlEhQKIXrJF4DV15rf99p7a+APNufKSRSK
+Px3b+nwqnGMAD9n4yFCocGbxjRI+HoyxE5wNFHVnEI88/gqlNRNpRBZNXZLKARgJsNGmeEX3QT1Q
+2nPVVd0SwWB4az7/2cEbSAHkyipdaa0nPV3kWR1U8BJOqkoWaMqHJ8WvFRfHImYAfq097eNANQCB
+iu5/XYX92S42pyRKfSfdt0uWPInupfc4FbOXts5JlPPWJJJy10ZIWmqFP4W3MerPe/xwiBnf1xUT
+BmZvJVobNAw0a0TeU5ZtbIjbFLQULCfcKu+nubjrYQFQqkfGIeDaTHOj/VEzuVRYUmCqw1fWkm/T
+30JQz+Wnjf/Qkyn/pzEkS3ApHE6JTgUr7aKX+51VpArN8lTxqqvwaXvt8HEOUZ3RGXEGfJcNIkEw
+M8cWGbNfLF90178nB8JNoZdaIT+5y3Lg/Wh4rcfuwaHF7D/7ZCMRHHcR9KTO29Ba6vUCDR4OO2X+
+0ttspKorfpO19yHawntLKa8oCnsaUPx1o05c5A9KSMpv6cAqxjuzjdLkrj97EtBnikIIW4CzR6P9
+Vw6OMumdy6d2VsAu93kDB4lZI+aeoLYOn9fNtSJdleN5KaLL2vgdsI2L2YPHlnFUU8ag+9zbWiZA
+DLOmO99Ac/WDzZn0eH8tePAjz6Snr6h04n7bT4LclbfpQFJMz/7L/xbGSaEGON3VSzgm+kSRTtX2
+CTlXT0zjrfc3d0I3PDDqCJm4IfkDqsdIsYIwl2Y8sp12kBJp1aPO/oTUQ9XLQDULYf1dmWjbrGIP
+gg4VZrB6cPho7pUV2Xf0VlqisX6lVquplEFluIRKlhhtFcxgjRwB78YtgsdTawE9x5mzprDwpGEV
+TiBp+MczAkJbX2FSizBH01RYDbYHr45mub8b59XVS9xOePXw/2rIfUI0/9pgDXyv+YOBQ1sS08Al
+q8hCsfMwiFwqA3qc5tM0qP3iWxm8meyDWzu9mFEplglR9v8BAKxnxV7tpJeFcME+41ZFp+jyaI4M
+vdqVQnw5XtTulQfKDRMEgfgvXFPzn5avkwioWB1spDRd5Oq2Am9jZidFZRw6fuQqfkOlfiGWP7QT
+fcyRXQx1x0JtdMp1rfC+K8jCWIU4eG+mnVZecUSSfsY2Au8vuIyG1YVy3bVFgqKOoTWttbxV8obY
+P2fB/YyQ2F7Rj/fHKdoARgud5vzc137vFOv3V694Eh0EdTKPe/vliFt0kT3WMsl+qmkeNAVTuxa9
+Rd1V/luMP66VWo7DT9LQ/8v4rCECh5ASNKa6MYjTeiuchmqsbFOEMDXouJlMV6JZtmj898O7D10O
+Ds9BcvNCdWERfReQ+6ZBgZiLK+RMhxsbKsRmbvchKYqN9FDd5dhN6L2iwlIBoPR5qY8cGk5C+Ah4
+fh6iTIxqE3eILzR4dn+icD2EBlgIMUs/20IIdsXPlw7rAkHPps+yY9x8xynlVaAhNHS5uqxh90Zp
+nPtfE5MtERYvTaZYTRUmViw7DXvbAFZeD1nyEdFYybZ7R5REJL2lrvwfDb4CUiD5kPF6U2Swdy/p
+9VkhQ7BdGcHB487nBdcsXdDPbupUgBXoKXYof+CJ5xrcFJ/rpo6ZDyht7SNruRzbRzG2VEQh7oDq
+W032XiNmVgkTJBsDfQykPUpn1+65HZDzuA8cbEr6XfWa1bu2sWa0nYcbDr/EZ8zi7z3MIBPxLzTZ
+/J7SxKs4uPi3eip7JLWI6svfFWEpQ+ydtZxQjboxn2llIT99sBLSP5Ri8l84iLrA5Faf+S9LRjie
+eAeFz1YQy6XKDkw0/Job8CUZtceUtAa+daK2c92o/hqdxWEb3bppkS/smy+O03l5cqoiXkLxY4DL
+C/L+lcEB1bUxnwCtVMRV1GZ2DwnhmUtso4d3tcT1xPNaI/nJAqivaXdjf+tjhvATCuGQwbGbuTGT
+TGYiC3B9MMBDIVq0d16gtF3HYGA1KJyEgvFOKPgGUHrcSQK6hZ5f9vKImFFTaibqVsFtMhftWq/l
+/6IBVXTcc7HYzm+Cy/KaRAuhbE92SxvvIm8ywyLInMtTQBgo3SCt2M7qAMcA4UPCNh7nd/y2BOQT
+nUXuPWOjqpW97DRCJZtvkuLxEeeQxtgn5VWwu/2U+m8kP9d45Dxn+T26roqHFdK4YOP5iTRZUrRw
+3uCuJEnoWzVhmdgUbQ4xLutF+Rd4WrQUbAAZTJBLygGZ93D23SlLuoqzcZdp/Lw7+YUzAr4FvMLC
+fu6XKaJv3ShdZ6Rj5fzEpxmzKlJE6wk7a9Loz2qf8vFGcQvFCP4iVd4zRXV9XSIrNxuJYsriNR3i
+CEpUIxCJt7wt1tioqf1grtf3cldor3LttNOR7l//gCqE7e9OfBxUGa6omM+5ZNR/XVh1jZM3vdZc
+4cUTyCnAKcU2Og5Gwe0qSQCm3Qlq4rip9HzypbpKBJ3XjS5BEL4T8ztn724hUYGHAq+LHvK6cD6I
+Gf8WllcuiqbnuotRxXlHmGPpFwk7gvqDAPiBg0NmRFYqMBdV3hgheghaLQ9VIm7l9Za18Fd4yCi7
+5fKbskS4D2HNuHpxUAOMijuheDG+y6kJsWFLzu5YAIY4VzxIkKRUPKU0234urC2RUWUZxl0zNgeH
+F/zy48HtUXoMEoaQRcnUWmklJl1U+EtZ2x/veVZzRGG+xt8K13sqFGq6HkWoZAVn7fwQg170Ngel
+TRYHMlpDjAGGVEbduDGibQSePsPherf1wOyF3Tu0Q1Beh8ypDhNZRQSG7XnfKhzPNiZXrPDYc1zm
+MHq7uQ4iv7GdkOJCk7RcEB27eW0Iuu6lXmqUPLFfBYITjiJiT9Dlrt999XFEorSOYIVG4MdFlZI3
+frGQGyl4wiQVFRiOVhWcrHMpzNzKiQuqtLsDpJ4VpNAmbxJSRsFfuWR9g8Wc7mTXWH/u1/BWpdyq
+m1oP7IWH3M0gNXdrXSrFW7BGaA4XBUBMBWCY5VI4FGcXYDWuGU8qoTRZEUIkvF4pfc8lQlXweu5B
+mBeSg4yyyqgPJDblQ1kbWdw1Az4OHr9N+mSJHwk2ZFkUg+SKPt1/WWMW2kstwQ8rJyzS3rojRQCF
+PeKNNw1A5B6XwJ/5EMWkQEvW3177FlvCC+FzJeNOYhdp6kRIMFQ45qj1ryAIbqNBtQjn6hVAhvzB
+wZwZDcKlwGWy9/YScbdxtsPWpf3+PozHmgdnmUctpOuaIoe8hNEd/M97PAfuBbxhzP6VtbOy0Rhj
+8LqeCm1cPavxxb+hppKXcOQ51yx08kqJoxQU37KtHD87P7uwk6HZVvdwypvd8goNT0QyNECNhJ91
+fbP4MWX4089KohzrWMwxCVTJqWH+VbVfQ0G08iVjKfT7sNS9ZD2AnWuXDTRU7bNxLznMMwmsemcR
+/yHrkSqkFkZIV947Cmp5+/gkefwS5359y9GsV4M7JPcnZ3sQggD8n2UIeJ/60sJ+NVpaXp4PmTOg
+AJOkmW7NSbzDnax7P4hWf0FQgfF8yLT/nmChQh71va6VspBXgdRoF7uHl7xMyloEffF8i/yPYEme
+2Oeo98j9yuDBgLfyZ39kV+bfUZJUo77bVBoHCgKMzzDzoOYQhh7fCflN/vgSTGsjj0VQruqFa9GQ
+bHOhcdccNjYmFd4EPRVVRkn5PL67gvoO1+Z6ZGfS3ZA7M4XEv/Y8BZTUbsvCOmY0Ac/oeky2BH4a
+N9Ses0U8zZg60gK999anwCtX7DROW3euQA1Sx8Ur4Avso64kaWziyiQNMakovdr5mZ5+Ys8E5NPN
+ECtaYYC4eXqQD8kIeCttwrVImVoJQ8BueHJDlrIPXYA8uIHedl/ZWp037TO8xTelPQCLrBEOAp7e
+gmNx8+G7BhpWWAhWW+NoI6XGMabL0NCWVJbK52GQCWkx0Ky5EYIbie1e+OzyfZ2jdZOWWqP6ENIG
+m7rzTf5yaVR/s8PFHD5HNi0PeUX1h+MbA4I22RG1ErCLo06wxzN9j8FC6H3nLPqEx2JxPiKcX6qZ
+p7WLG+UwtrREz5KxRsDfKUtP2ugbuhyNufd+dOdokjxr8DVufv/0AaiQHTDGHVU8YPaDSLyPQFMB
+Rs+JZUzwQSaP84dILAx7nDWD6Wx8Cjj+I56f4mmiCm6yMEbY1CcNCAuJxaXuoNdelmbvhQATdi56
+yDczmOLrRXx45xUZU/FO6FMc2ffg/jxuBeP325XgchjTlSHEKOP+8eeexZTpCN5xUy+q9inX3CA6
+EQme3wV1NS4k79Kbc3I/98L+H/XJ8Ry4FWYlxTFTbBoYyzsRL2Kqg3M7htaqUPv4RCvO8bVpK8Fg
+Fo7z8DGs8YXuEOVNi3YqcGYeA9uYGgpEAyJMETKxthHKjYeeP/1mxhQSSGDZf+6vjtdtarvudLVp
+SSuVOOty/fxdlHkDjTHuxc7PDvWcKpezPYM19ijPCT0ZeUEU1cNDlWZMmwoJWNucbUDMHn4GdpuJ
+Yr3I1SjeO2I7gv8PsriE2NvjPvasy+bbz5JzjxPIsB4HnFN8M4SPqwAcYBqHYMJLIQ8tgE6BbtH9
+MSev4vQeq/sEG3DrNqwuZWJhrkoPxy7hTSThixtsnRzb8rQ0P76BL+5Wf2WOHwc1DOdUPIfujtUH
+YpsCruS3fcfdCnerGJ2SuElNGM3Z2IH3wtQhtT1J9OlLOv5MIYyIFuEZtOW3OeCP3NnLLyFy4Sjj
+FgacdjJW/LL+mBjawkpxl2R0d+EviMZ0nPr61zeMaLf6j5HxrozkKZRMYkr8TxYU9YzxGwAHnQFq
+ZfGXSfmvhx+7yF5yNmN3bjc0rHiFp/ZbeZ01WuOmJMn47B4sr4gBuARzNe22aUzfkaWgTqG4JxI5
+KE2Mv9v11HllgRhJLd3QqXst08l36vm0WlexZ/W+yhMUIn1TK4OC0CCsmdPgfpgYEUf4560yTtTL
+P9gWsjfHnc+HBkz3Vgm+XFjP3y6hjFGGlnEuTcu6U+I4Xz3VJdZvZl2ciraCMTwOzfkKdSGREz/e
+9YvIaqRAPt49MFlVXn4GTaH+MGAP176lwYoBfdVHvV5HQIe6TMtkGWIDTZ1Cen3UVZy9wk4PeSop
+6ARrakuM7Eg80JeLGpXNjDDSKTUv9eTLrdVFpQSJxDHTaQGpD0xBOlwgBKV2O0jMUHbTp3O+qkf4
+rRpdb9HU5QNSuwojwmqt4PeHt67Bxz6hnvP5lmJbDKxsjTiYsscJbAmrcTZxApW8KDLb0F3zkFms
+ZJSDi9tARaZtcksRtM74qBTHJXCisOB21ZJiNtuLsA/kD8XNjZv+GqBHkWI8pBPE+EQnsjRpa6yi
+nT/rPSkvRENl6EYNAo0Ew1fhUBnVpgj8IJZ9IfhKIAiCtLpz+uSnSGB2zXi78TsKwnaGWg/FwnXy
+ZzeAPNGEBSYsZ6fxvJJVz5ysB4ga03kk3SRwEgjKBU9WjvN+/vN7dsFgMPf6QaBNF1/D0sohcbDI
+dFGVHTfjqXxdhafdDICsvEkty3WIif255qKQsBRxFT/J5x1AQOMXAzZ+ZOU2G1/kkhwmXDUdDMIf
+SFb1Wd4zu30DpOlzUlZVmDgArfYybGAG6y6P61abPIauLwDVA8QQqD3zS7z43u27gcpWMhD1gUO4
+y4ZrBbVzqZehCdSs7pZ5qpT/tt4Zt2Q0kQ6IRdQ/T95e7j3a9Ud60Iqgpvo4LGf3tjFrG/eAawg/
+tKS82GBUuoapYSdP9C5byea0FeyudPqxFqqDCPVovxC5sPhlVpFl5gkzMGQrx6NFBbTmQ0LkIvT8
+JoLh5pcm0vefDCNl5P3zYTcjGhgj6xb8Ghnqi7XKbvERJrRI0JtJR+hDa2iz3Fn76U9JG+rkvTeO
+RPFZ/XlMy1eRssPwUw2z9Lfy0dCDu7STNFT16qv4v782/1XA0hVWjyOhGgAvjeLvX1uN2edi/o38
+rWC0yOpBIQaHyovaPf/jm9IsinvvfG/BPPw7okPu4ohn+ucBmX0yjH+JeuoKY/6lFH9UvFuIKktF
+OM/Gh1y5ycySO3eZuuFVrnLReMi9AirsYfdya3dBkwz7ADi//V8h0Xi9TsrVRhvM9BEGuT0YXyQ9
+32/N5ZWozKPr4bEuX1CnZFvHezv70I+zGX9R0EoapV6TxjuaudZdqHJwJ3bQlvO6IB1feOdnrJhS
+uwaM4U8WKMg5MtdmrpzIMkhCObSp6qYrbSIHVWV0KgQ8kWVgSwhxKwax1D2KJg1V4DuCXlI+bYP6
+P64lfGOEgEQ0kjYNzLia2NUxv1gWSt4TAL74veiGLJZ/EOqwHl/ZWFsb6V+y6tAknyb+EoMME+lU
+fTIZfaix08yNXcUTscMLQzIxfEYFfm7yGLgjs4xiFqtYSh7fJfHgVj51ziK39m3VMcc3O88LBM0M
+ACE3/1CFDxHHqC91syLC7SAl2jHCMRFCKEW9HSuMM6f1gYOUgTkr+yz+Tdw1q57GA52WjTlcMvzN
+NbhJ+k13WrfaMWN2/+U5PwzxI0Ezirwca8V9bKczXdwHe9ct4BlBJitxfAIdQ0ik1cMp3YeWbHob
+DCst0Hm/4G1Xw18jfkmjRb6JgEC4fHlpLcULXGNaSdSiD+AZvzCVGadOwHr+/d6KNaTK3l6TPHXi
+/CqTZm/VsFt727rjiTtvkDczCvdpS3dOr/E8fWWURThZ4wBGXnxZ8Z5AhRMMxSOkMgYFb4QtrbM2
+V2vPyf1WqzcpQpJXm5pBj/YS7vuY7cM2NPrV+NcQlFrAJro9qKfBZXRNU0ESESgCiUZcR463CDmj
+v1RuMszf5P4Bos0mevBzFA1RreBFZf8gy/x6EA1X1JwEzsyRkqA86g3wbyyrQvIKKg7RONd+oh6v
+17z+6Ob+vllcXQOYnYPmkktJGgNMFYTmc0bFFZ/OvWI5e/OPy9PmPyQBYGDnywawJdhEXk2rCB4B
+3UMrAk+uyvzZCI7EDDicqW9Ao4UdrriF1wGdW5sUk4pUDpJ3g/n2XdKV34G4/Cs4frJbPg2ieLGQ
+VFGGLEhzzZjnOt7eoLrUGKLkpGRI05Pl0qUAog7+Hlh1pSm1HDEXJXStYVUsQtPIsxWOn1Qgcsa8
+znQ2Sc/4ZgGrvX29AjUTd5EIGT9nndmRC2jqG3Q2Zvtx04cRsYMQuHwujhbF0/jamR3YCOJNQsF5
+ue3LT6fjzq2em4HyO0L71YZ4SgYHeD1+eSmSSFI1MBolriT5nQ/PUHz9DmHMPOv4p4CljTPZBqMZ
+amF0FEbbqTW95tG+PvV2/RKkYEbH6T4Mhd3sorR1iFvO8DJFDa8twdSg7v16f6TMstXu5CwfGljo
+//it9eJA5DnXhDnUiQGhd+WUnmKnA03zwKWt1p7WNwnvrktVnA0BKb4LOdaQixxx8dZ4LO1FOVJB
+RTkGbm2WCfXQB54kX1Uj4pOEAcA0+HIbspGq7MrXWS4QUxpxQgX7bSzbvSNKTNxyBYgZwLjoPQ50
+nVucd4ydce/SBRm4bRQ6KAbiH+vPqPoD7t7cseyb6pwObz4uh713c9pnli9T7AXr1uTlmy247Arc
+j0gOwGcvvGVMp49URjRp2jMSSBgqqZ71rkp0LfYh6k3MMQzQllnm/Nn8I5gkKiH4JnAtUKL1WN3y
+xtrH23LQd2UuwVx1gzFfYrAiphMHh4rI0+cGwAGgV9MQBKIrShryWeBOL43SANrydwOh4fXR6q95
+SxChd0KK1zpOFy9nJyZfU+Lzec2+xtaCU7xxV6K4HJSd1VfFNI+l4BNDeAJjI2PcXslanMKfJzJQ
+n6okQFmhDEoLv9ak5w5SXhwvfYGpBUHmJ5AqFOhv9NEdv1N8gk716gU6Z8f/1PnH+n0sTicqWUeb
+fpDTrWpyVkm7/7uiJ/6uck4GAAIKmGdLQLlIMBFj/6TvJsVCrnOff+F04PzIUrwmYaCpQmJS4UkR
+436FVxtWJO82cSaBlOr5LdRYy4Q6OcnPDhx9r8FMwc0W+bjbQ8m77cuWfh7HuTGcmrxTYBrrkAcO
+4drCCrzxLccZ/cs/+0iW6zXnaA3wrm+A4qJJnfliLalX/9cweNfLMSQ9T2zaLHotE72dflv71bDP
+ZGw9eLRfGHRja1rbjDZrsDbC1Ii5wS45G5V93fcjpO1bucaCKNi3OoMeOSslpWr1bTtI4Jt2W1Gr
++Bcb/siZ/038hOojoNyB+ssfO1MzLuetoA5dWUWceKJH6qi5XwVvmUlHNaYhSezByMS1CwEnQGu0
+ExMrvZIvcn8QZe6fh0qfvL00XNqKZWra0zs1BbZiJZnisygy5f/JbnrXF423kWpjWFJdad7bXq5m
+PkVwHonXUbOb0reaBNThwyErvcEMCWxmtFDFL9Gko24UfJwYjnT2bhZw/B0kwmn68zsH0Qhp76Hd
+y0I1PJkHVOb4xNo7IHKHKyWeH9dQi+6Itur9CWUstzyH23ma7TrHAXOsr7aigGBv9ac9sy7VfYYo
+j1lbTQplbmRzdHJlYW0gCmVuZG9iaiAKOCAwIG9iaiBbMjI2IDMyNiA0MDEgMCA1MDcgMCAwIDIy
+MSAzMDMgMzAzIDAgNDk4IDI1MCAzMDYgMjUyIDAgNTA3IDUwNyA1MDcgNTA3IDUwNyA1MDcgMCAw
+IDUwNyAwIDI2OCAwIDAgMCAwIDQ2MyAwIDAgNTQ0IDUzMyA2MTUgNDg4IDQ1OSA2MzEgNjIzIDI1
+MiAwIDUyMCA0MjAgODU1IDY0NiAwIDUxNyA2NzMgNTQzIDQ1OSA0ODcgMCA1NjcgODkwIDAgNDg3
+IDQ2OCAwIDAgMCAwIDAgMCA0NzkgNTI1IDQyMyA1MjUgNDk4IDMwNSA0NzEgNTI1IDIzMCAyMzkg
+NDU1IDIzMCA3OTkgNTI1IDUyNyA1MjUgNTI1IDM0OSAzOTEgMzM1IDUyNSA0NTIgNzE1IDQzMyA0
+NTMgMzk1XQplbmRvYmogCjUgMCBvYmogCjw8Ci9MYXN0Q2hhciAxMjEKL0Jhc2VGb250IC9BSkNL
+UEorQ2FsaWJyaSxCb2xkCi9TdWJ0eXBlIC9UcnVlVHlwZQovV2lkdGhzIDExIDAgUgovRm9udERl
+c2NyaXB0b3IgMTIgMCBSCi9FbmNvZGluZyAvV2luQW5zaUVuY29kaW5nCi9UeXBlIC9Gb250Ci9G
+aXJzdENoYXIgMzIKPj4KZW5kb2JqIAoxMiAwIG9iaiAKPDwKL0ZvbnROYW1lIC9BSkNLUEorQ2Fs
+aWJyaSxCb2xkCi9TdGVtViA1MwovRm9udEZpbGUyIDEzIDAgUgovQXNjZW50IDc1MAovRmxhZ3Mg
+MzIKL0ZvbnRXZWlnaHQgNzAwCi9YSGVpZ2h0IDI1MAovRGVzY2VudCAtMjUwCi9BdmdXaWR0aCA1
+MzYKL0l0YWxpY0FuZ2xlIDAKL01heFdpZHRoIDE3NTkKL0ZvbnRCQm94IFstNTE5IC0yNTAgMTI0
+MCA3NTBdCi9UeXBlIC9Gb250RGVzY3JpcHRvcgovQ2FwSGVpZ2h0IDc1MAo+PgplbmRvYmogCjEz
+IDAgb2JqIAo8PAovRmlsdGVyIC9GbGF0ZURlY29kZQovTGVuZ3RoIDExMDUzCi9MZW5ndGgxIDI1
+MzA0Cj4+CnN0cmVhbQrC9s36b9MLlCRTrcwvwoHtDSMwrggVXnGkk2ubKPINmAD+iv2g56GePOvD
+ZD22cbU9A9RDS6CpYyAXceIzuQBQvmSzzD37tr1Lg8mdljB7EQBZg7JhAo+a8On5j8K4SWns2BFy
+iKKzDJYTM0MCDgwte2QQu81IozNXxb0Qb5tpWjH97ZK+8kFbarUhJu3P4gErg9K2UrZZzGODk2J4
+pwDHFS/1lSlxohae/WBfCDFFDEp0MXZZUxm2PvqbKWtkA3vCaM8/kA48gF8m6iDbEX27cfYCA7kb
+vtTuE2Zb63HwfXfFEJA8sKPGxlFkQadfzwKtp1Bt+WIdmM3BR7VmUZkACkkMs6A1S6sxPjOKIjQO
+DvREElgA6JnEw6mGseKeRgRmiYGHaqK73ykfhKYyhlEinhhzbIT7gs2/ou5fwsS3E18BxyI+jUQl
+d7AVJl99rTsZSfRjJyG3oCanuCOgy7QQLF6DUsAzKLlyYe+m14Kp4UuOpmAwPFyVrTAzBreYjyMQ
+1mkZEfiu1/9V73PrV2G7KbPvs7zuLe091OTZzzWtTJ5dSSCuqafPHd2XHMhcahDc1uP6X8IraURm
+6kVSXyt64H/8TlGUDP3aU/1LoKRonc4rXM/uE0JT+rGqlXdXr9Gs/f4U6dxAMcRdns65QUyI6FW6
+BgG4JyyuA1MvEpTJ4zT/Uji9u6zRTMcHpNBa4lXfdhRQTwB7Lg9XYWwdkexNSfoIb+APYUXbs+4H
+iXaAYbV2J8203IwCxT6j3wnzDJJOpKT01rsENQA52UHWM7cEAgoQm13PmdEQHDTQh61psgi5oK/q
+UrTRjosR0TCt8xFXz0zAO5ePMX/ZRpgn1FgF9EAnSmzBm7Q4/MOf30YdKDzM+q2q9k9ZKkEHZHEY
+HtNsWXraxHEOaxskDm1PpssdHVbq+bKIDlFHZMAdJOAd8A6tNEuaBeHmz3vYa5Gi5PwQMpelsQ6g
+F1BwVYb+0awinWRxxpHQdaSWmM+nt9Gd2rbItw08ERTlYo0MJe9TBvhB4jOCMsVT2UebmIBy6744
+bVt2y/FQPcXPt3dSxQ4T/m74aNAvsDuc/wmN9Zp5Hly7cjlzf3JrAv4eJD3vE6fRt5TtEZyFDNkj
+h2UP9vW+ZEoRdw/5S3o85wgXFdvTmAUKQPG2eDuV8/T+Y5bwucTDQH2+PCpnjw7cpz5Mz2L1WSID
++4A9yeuhDdrPykVx6czuBDXbmpozg37sZ4o4rYKE4Te4/1A9lvQ/3JBJ9EMZo1d1f2VhisZYf9XE
+f/rnLAFnquREyoTwmSRQd/G128PyqTpNVU9ruIlI2783iXYLx2tzl1x5PS0F1WhM2vVlCqfzSzHk
+koFW4P+ciUv1rqXF2Hoa8WrxMgcqMOgyvKF7PJwNr7AL6cnG69f9PEcxwNpobnrg+p8W8GdMRGRR
+pFZYC2kdzz3tyjhUDFQOQFnYEZe/f1TVrmJcOBZBMXibNYPkDmGBpvSJNcbiQkIA7zbcgw9gOHVN
+Y03qBymmCAbQWSYIvQ5pp8FZ2Tk98wKsMTqKjc1ji+ZUTfeGF+cRPMiweQev3iNabuEbP2LMUPvb
+eAC2RITndRkObSHYX2Y5G8jixBgZDQsmW1HiO3/nhAtNIZkEELYihhHPsPIVupi6Pp4R75ShuuY2
+v7PpNDeZL2mDHCDBwGrVII2J//05PCTe3bEGWncQ66eXCzR8h5DRXciVm0Roit7L3YaVagjF1OJI
+Jf5oZ/0ZgD9vPbmoVaDt16/GTlFmUSjT+DWTYVLhGoVLr1KCSuzhE68Lo+jPXmTGcFGrefBX4PuK
+r0WIKwYTK0mfDxvfLDQhMGDb2IzJRbXaBsGIq1Eu1yDF6xLT1x3K+TRSQyXLml/XVhHZiJqPtuBE
+lHH6jFJSZYZLPi0RxaWA43FwZ9UZkX1EKQRIJAhQYtvNjSQQXMMiad/WtUNElfvD7AMv7FQ+7Pvc
+QC+Sj3IdXAD6N9mHgjd3MLm45rr/Rxsrx2vl4KT6xK/4/ZT8+KQRNjhiKh/dXdGFtYZPRrRHJoOQ
+yozAj4T1ao8LiNWsSpBRhbyQvs32qcgjwXDcBR0QHQXWSSL2OWQ9OJ5tCxYDpkURaX6/aZorIJCS
+/+A32f1fCzTunWrs7GU7wAtKZDi/fEVVSljR2758fKnRrJ0G+DKVPODSrtZdVF6Ee3vDrJVnzeXa
+1g0RvTD3nAOTQ3VuZ5sPo+J0H+vn2UTNlNkKjR0XsZk3r3w/KLVjZMMW90n8TVePhL/lpFHydcKZ
+lrs3liXShXqls5stwuIj2m1Prv3TPI8BBjNP7hDOFYkbaxGwG7b/NleKx+qLEYgMKQP1iX58EfPw
+ri2By1tz0+0cD2jYikTjUmzuzhNeNwkvi68KUiLc+euS8P9WE08qQFE4MNb/GFqjmdd0bWoG5th0
+6vgbs6K5KCd8YYA6PCGVtyLFdhSEHAZCzlK3QrE4lU2HYwjCim2kCnyBUclr658h+m3ajorJB6oA
+Gd+Vn2t11FF6i8dl21+Qi4WgrQ3ZM9pLqL+vA2sboTSsUhOyHV5QfiWc63E7CZf6D23oLQdPgnw6
+z6EebtAAFOrxfe8q8+TptXJ/wbBFc1uuVtWcSdmJ/vJfIFddL/8C7kgKXhprFQzVpM/a0p0x0Eg4
+eH/NvHKAE9atK0sOFk0vhSeFRKQhePlM5LKEPgsJGqUsMLbRk3uryiItIjFo8I4gRZTtH//DHH2m
+s9RvpAbRGt0m89yoSS/Dbpdf7iVpB8Qckf4EvEf6D4MXPN2c1mg25aPVD/YkioIh9jwJ0ujMY22D
+yRzFf7GQ89OoFJRd2x8D/VZrWeIiEKnGysxE6/lvw2mAsioIUc8WtiFH+k1QG3D6ycdVZ7e0rbXi
+7tyNPtzJTFmVdaC1uQULV63GcdaOLqpEkgDKLAnH8gjIDsVwJrhTw/MK4uHOcN0+ax9OlEi/xO9U
+xaXTkOgAO1lQv10eigP3MCdT/qEEJ6Ylpg5wZ8XHQ9d6K69eDDCQldDvqCQwxFRFAhP95CtQm6pT
+zgyJ+D/A5pv9iVnBjNMOZUFdzZTfJhTTHbzhbrD52wyGn+HmP6U9qcUvZUD8B99sWKwNgeyGKtqD
+R9wyGhG8NH2g7mjDCM7DZ/hnh0Hj5+9gy0rGH5fuyBnY6NLLmUTLV59YU6jYXVHek7pXFQ8hCykw
+SaD74Lq9RoxoJRuEZyBS16QrW0u2wGq3WAIJVbPMdiYKuujHYcFj+wN9UkzzwHTTP1q0fKxeqz47
+wVhypcqUT87mSaGAGvnz7n36hZnhSSWMnMu7nSXTuVsoz6CLQuffwvWOJo5WdP5h0ypvF3uPsj2g
+K+2+pK0tTCMwBuba5JXHXUqNUe5USY2mFHkH8s6FLbJ4OAs9QSysfdp4vJIrEwd4/zKx2eT+2+Nj
+Ioq+DM1faMjWESTnJW4BPztEzTr4q4ikx1qt8HOSXoDPWbwSkhb/KzEGwgI9hfxqPYRk3Wf6oh44
+vDSoRvxiy9nww2CqzJBEuUYajgyXY03DK/TwVvJoA0ayBCP1f11qupFOKIJjbHQbzG60+8wgh9At
+Nzv3fFeZCn2Tgwd0oKuyNyAgk3BwKQC7fDYk6Wj3RTR2DsBN5OLFKslu+KLWGKBwz9FAZAqUAuy4
+R9ez+BaURu6Ct94wXslFeQoj4SKZ9auvbkE4pTi0uYm/dhDi70rmEUYieJ1IBQwnq8FbsqpSL0GB
+UhGakE9yGvtPMBhvaO1P8WucB52Os/yXqfrqML3C3tW6JKuAh7jvXkNr7ZPpeU7ERlQFIG53dl1U
+EHIumywsJmFeWQqEAAX9rU7mpxm4ptbOIhhIOF1M4i6kGww2nJGo01yT097WC+WAjsOqi7PMK5rl
+fnM5qV1KbQbqKVyDyQhwBBrZ2xIwzk2z3Cutj+Cedk05nOsOljp4y6VuudHSP7WBSYYTqB0tgSnX
+pO0csf3u7M8qmumslyqr8f/rLEWrRZJ/8PCnNr+gBsnGImwhldF9+wTviXQla1kv9imBHNP1E0/S
+OR6yCnB/TNlrwIDAoAaRk6XEmI4MA9wN/x7C0NYz+88jgvbh3Gqrn1hMucThcEcDBywalx16TUBc
+XYd0R3EJyqKtkuUhU6dfSVn0BNPJvLUbuBQsmX0/tntio3MKHo3eFZgDoH8LI0brnYEIwtaZBK1B
+sBoSQ/CgRYHbRTFTlX4o/GpurRQAL8FWdMSaR0dDjWE62lbxQTX+teUm6VUrZn6FoTyWsrbDmH99
+jqkn+SWzDP3T1Xi6hofIwBI01WZ+DccPgS9tiaAYkrgtxte7faDOePoGbdUN2QPe0VaAKP772rrv
+OUFK0TTgxQlSQ6aZQFJeGEORx1u9p4h3Qnxn3ZXIEF7T/2lTEiCAQaDo0Ni8Q+LAEwUcy0ZyBaSw
+kvpecsOeAeP0vXbSJij4rn9idU2tM+xeIPzFETL/gz7NfSp36XMY2SEoSOL/xUGin9T6QriCDdJr
+dx/MUwxmlj+2WLwEAK05q/gupDaSoUi40S5tRC0v6GM5A1h9+LBiQi7v1ZH/0PRhdkYQCTEU15lA
+aUnQkNcOCRw3lP4nmXwTNI3el4t4+YziObeovzAQV2x+ebEZqzlMXTLLyvo5oNkPosopIcpyfHrR
+MZC8xWKWI9goJDI2FN9XpqZ4ZNYjPmljLxYPS+d/yxyWNuEs7TLNreqHxS6PyY2ESKxQgbiKhSTZ
+b38KVHsqOfusshdv15XlAEl2YdeQQMcCgkzpXRJ0KdXzJjFyV+1w5J/e+jLlejel+b26uO396LoQ
+NOSd+28Q1/9ryJCn25/f9b65onCFXQdon8aSkX/UWjbX5+ciHbaokm0prFuaNIVQF8n6tNtN9+Sq
+a4r7Yhe89t9f+Cxu8O5OlvzZ+FVRFEoccoVADj0K6kfWGSF9LE5TPa0jXIQgaq+Vu/x3h2BwOYZk
+pRQr9hOSaI6FECc1qNXywtu2MewL2kQwEUO9aWKiII+4ybPC+1XabGMHcuReRDDb9mXa5m81sDod
+lsgXWLdUfi1o1aS+gGrneH9T6dISRm9vdlpv3NJcXTGc+vY8Rfc5CIT3jtXJHL3doOEw665mUt2y
+qpZBrWxbnuFRg2NUqhUjx8Frv8yGf/8yBI9gebF1mdcERIs1S/sR3PZcFbx1fblyM650IJUIIUvm
+PAMVgX7eYRPzyX46+cUExzqFOLAEYM+6xe9Rk5HX3ldpau1Dz92bPCkH8E4CyiaP7PWFEC/BZZu6
+npU0tjFrU664+6uG6qgjC61mDz0iT9SmjvXrqbfL1AFYolDI2YmwjKtl2CO73FjgpFO0OO9MN/qv
+SOdfEo5/io/v9+dnlpbC7K2T64gsDidV+VvYVKLOdZs1oc68u4QJcM/blKLjIJ0f1FoM38TZp8gC
+mQcoTBg6SNh4zriRTw766W+inNagUgOrW0xYyihPPdGRzb/JGmljqEFOaY5/P4QPaQixLSQOg30T
+hFOq+81oejBgovW4CuzKoBpp1OigC5CyPGTjPdvpM4HpmuqRoxntu9b1ipQfMQWd4UH24po+fYP7
+5R2HpNG/tdG+Ubx1pJ8o2g4SCyd3b3U9vsuGhcj93D041vrMFIP19f1a1N8QIIQ3tpKn38EDsB2s
+fwTp+mHy42nVqdNuOuxn5SCTnr12gMO/F2FlUC/F3ZFFMtkUZm+vcR7Camtp2Pr1/pv9mrFxd7I8
+d607AuxaGpVhp6kCPy++JqfjjVShtkHCAusuRdc6pCZw222CVNprgOuSGRMY47W/le2h/Hvm8vF9
+gjdoeC24c4EwB/MRYWa6LJCdpLxAvv3IH8W1X3HKBz8AFLFCat6+gHMpjXt1M7Ec++lvnf31eXU7
+4q1WHGfj/HVt6J4fzr7ns4L4i7ksqZMRENMD4E1jDroqhRXQmcsdSGMxiwdBZ45YWtUd6nNd3xwg
+9JPIxSSYHsBx03IVtwNoxqWeR18t9RgnEmb9q83nxarDp0JENJEMKNXPmgu97//zvyAOrPvzrm9i
+FIl5+vT67oyHTQVzmxu5Iz7/Q4WGfzeHm8zOkQ7froz/fBGPEV38WkEwNzJV7ErRjE7A4T6ZfpM2
+i6qWxVPOYhsoFjshglDgeJm0sXYliZJqa2QTBj5FlUGoiN51RdYKpUtNTme0+Wx8qAOER07GrmOs
+gNiypmJrlrwW0nnRIJJxiTolElys1t/37o+zM2RpoyqYWenZkehC+goI+0xUsPKzS518d5t1Gmsf
+UZwjMPED0EotzMkmWGTsU5HRGdyHOj/cf0DysBGullp/iOipozxGwjOXfiZB7lItlT5IZsk/CaPc
+C3aRxaOuLngfne0PGLDpbSNUvboWkj1qGgW9BF15nEnUFUD38nkkiK1GbsWgeGdefDEGOTdYPMzK
+47GFgFL67VBgxNICckSZNJmLP+7VoCN+QKZjhmmGbF11lk+WTSR8VhyVkAghRqTLQp6HOavOYbay
+jP8Bam13rzQWSv6pSVmaZhmHMNs63jKaMg6loTMtBqh+THaS0f1Ebvr4wxTqiNQmqCHpeM02n7UO
+EZOx5Pi42zu4u8VJ/h3Wl68mmhdRfoIfo6JRbTYIACXQORkFJl068/pZJfV4k4mystiKv4MyyNBC
+U2AqCLGqmQL2BlzxU/YJflfEZfp1lNUL9HbVbF3HnK/0K7DRor7WwTEpbRvkyjW6MoT2XIqcjDXn
+uS/rnmWdnL20LBZc9G0wq2eZ4NyBgjseuV0rSaBIUDWJMWm4nZIcGjDmG3OvYKqEG6uoI1HrM3oc
+Xno1jKWX6IT4XvIV2M8sgkt+1UWmSuU5VNKFDBCgq8jxuPhr84KtEI1Bfi/NYNkULS2Z06J9DUhl
+jm+EYPxT12dbszkf0yYjOMOK6uEf/jgG+oIy/MmD4IXkOwHAMYVdmaao9tuEF1w6TNpNronZd92T
+DgKdh07p1z4fC2evDOEMrl1dR5q0HXOGZVrSh3ibwpNVyYOyNuWM0sSHPx/c+t7KHmofvtlCmwv+
+8GkKePTg6PNJ7EpjqXc0BnMsl9J3P+bOGgZr6+k3r5HT21qSopwYn9EfH0YIQ8NpWTGXcLqWcpcM
+bQKvNgN1TJy6EQBKWjR+j8T6oDb7y/rDhbs/8arGMPFBqjJZRGIhW2lCjnjmdf+KSkWLARFA1si5
+FefuEbLl+E4rlLeJ07fNcQqML5+3HkLjOy6n7hKbfIaAyhKqm04vPlT0QKrP0qLWO9yVXoH5nBB0
+A078/83+9WrvaKBJstduDel+jmaVkwwGMRBCybJEZxHNcr9MLQ9bEAfJHjucQW2+QaJHuq95JQ0P
+gCoQkR1H8tsWK4FdtnHBUV/vSBJeUvxXTFqoW1TgkY6Yzdblau5chUN3Jqr1cMDAw4cEispBUIwV
+wT5/RYMZhyT4zKXAbWlchcVzBZCjbd26+2ZcVuboi2LQh95+pwPV0m5V+oY+J63VmHCXtgxV8mZ1
+MCNbLVQU7ZHB133whOIZY5sid3n742NIlwUf+MA1FRru/03vMTc/ZDs2o/uRbuFIqOZ8NIy9lTiO
+IdfK/+sj6WgNGANRnR8YMRn6ytzMMYfJ2AwEtkSrv2zqzeo2gXfQrQ3Ct/4cacaNjqm48XnmTFMS
+0pfypZG4E/tJOgPw7gZdG9v2p1/5NyQwskBTrZGyzztQ36RsZCLDkqKu7NWLc0+B+lUsy2v4R9uU
+L8PyNJwOb7rqjsHiOtxUho+kZU3EukNlJx6krUMM3Uw5GZbQgjov1FFlHJAOYFzW1ZRvVXKVgbBW
+qY78H5wqQHmWnSO73TYwjDR7XBbgBNR/rnfK3grucrHkzBCBoF0M+JB829AqugTbltE4x/Hc2Z+a
+ODEA35AZzwgZwgJkWy9A84Fb7b80EIc7jPigKZR8WeSlmpYD0nbvdA50fKCLOopWUadJLupJPq+z
+CbEnmHjL/VmLgPUPBTHqrvNAtEbpziW+g9RRqBc9jq4Hw8kQfJhqcwxdUX52OgukfDc8AFRstALG
+JlBBl2p8yOPMTTAWpRlhKfXkEYZXO8JbukOx+IbYNZOUWfGXkAt2NAYPpQKgqsP+A5yGhjkJWDno
+6+7gpgZjJCC0xhKtWnyS8BJ7MckgA4sNsFbQ0atis6nbuTE7Y0BZiCZHJnpyxlqhz7OYdFDupaKN
+P6tynwyQQMSyweG6nG/DBsXn8qI1azSLCAuqzfyVDMEeJWbaEg4etJj1cH584YUi8gmi/gILEzon
+QsD/hAhl63SF3lF0kJ18veMa1ipg50DIhlNQfitg5BJzjJpEV78RGSVFUScfso9sgmSR5KYuO7PE
+qYjutjb+53jTP5THvRkpZJftG7kFh1/p0zi/nd0m6KGitky3Sv+BGvV9ct5fUGgGnSsUHPntX3EC
+youA3zQJA+zJH+3TDrRWQlkDUcoxXCBo0614jnRJJDD7BYIsmOLHSkZFnUStfOMoHG0IH9xA+64B
+u6MZsswNL/2RKg60+Xbyd9WXAtofOJuzlA4aS+pm3jZmodbfIo/Y5q1tUhEU92ISKCrLX4m/hERb
+dFTXHO1mRTPuPpvyp6nNOwsl2Q+LAmyO2t1vDH5ynyCLkL3SADk1oSmDj+ssLXFCrVtl08AAaWJp
+MauRq8Eyi/eFofTRKHCPDswPFCyjvpZb3hfdlrlWR3Iy92oJfosuxOZTTkwQgu6qfrzhcZsPN4YB
+KC3evYvUe8ERAEbAUxJkaw+CYIZQUiUDAYJiRonOHjWn1fmPfEPIBAR4hRC/HZIrFHjC8m8O+fXu
+Kt4V3qCtESup/iyzv2IzcGyxBpBtaOK+DVc40kMeLqZ6PeWAwiGZgykABkxWRUCb75zFXfIip0Vl
+G56puawuIQ8yfDWvz7v62mJxa6MpKG5nD+YRJE1C7MNDB6XYdK+AaYkRbpD8yC3E1Ar9eRr20/mL
+YKwXiXRLQIwcW4AX/fYa3ti9LgMm+C8r5yhSDWKMXnhhLCwtXcknTcDPWVcOOresXyILTmzHVn08
+meRtBOSD0kU2/2CxKMYp6Hzjo/gtoQb9ii9wmcSFUEJPIzkF3tIGlfD1Sf46U3I323udRMizRq2f
+0iqkPTmJ/NQJUJpdWcVWdGPO8tYgYTX2aoMreaZSRVPtZyPR68OdfhHOpqxV4J6PLXa+PKXV+T8B
+t7GhazY2h5gQhZvbCs1unDHfW2NmlE//33+QvAqom8V4eCQwUKScDji7V45G4z/U3p+qsB1W+9q4
+2nk9CBzLDBwazl6izdh1QrpmFK+A9pNVujwf64yzNeAGc01xu8PQnKVyPuuwVf7SepphyARY2hLl
+80rXkA+hW5vjIEPGHv5E0IGlzZIId25WC3bBytuLaILjkQ053juMxvyOD3J8XCyzs4d/5QaKRNxH
+6d01ewYY79KLJyhqhoqXQ0zDqdJkyr9Tsl2NZmiP6MNG1nBoUYHMhIcbTBFYWXQcXBWs/MRHlrh5
+d9hmmgXke6VHS5XaFJBn73oAew6nuoAQqrCVK1gwYhHfHTeH9iV1srF0am8qnzv9Ct+tafgocjOz
+6VXiL/wtPnc/RPnDeiunRaImUOhSCHnszNnU4EjEoB8wf5zwRgiXWMoHbc6+rQmV7rrrXJ70F5bz
+z6scYwwolaGpR1YfYNyd56aLVuuffyTaWrSJxgXRuhrxLXh2rSdvXz+tuGd9Nt+ns3z6RLtBUNmu
+Dy2320kbcIA97cuywM5E59QRcS+mlLk5b5Rms6NuapMCo5/08ruZAjH63u0XuARIqb/ZgKW9+IEb
+PU9psUF+e5d9uxSPB40A80vNgOj01SSKR8iSXGvHhl2cpPxQ3YYMVxsKe2iv6TdLZlFQLon0aW8a
+HmsND0Zs00ugnQXyC/DnYjWtkJ0cF0G7IZgmxLcR3ScagarXUC7asowALOSztznAs62iJ/aeAvfL
+W18SgmkxHApjCBPSUWLufSb4VF6/Vz+NKx1BXYWvZWxEx7dswrc5C59tllKijnsSdrg2tG0Pc8wI
+j6QqDVuSfTPgyLop+pmWrOFu7LwkI1Gw0dwWLCitkfIO+2ZYwJRBmeJb7zSm7CE/mKdHH4lMJOO+
+zjZ5VorMbD5zGxTjUnRFvQv+yMrY3YZ6MxzqLSuqTxowJR6XrdBM36ykzNoXffW9F9aBMJTqSlP1
+8bBVhSKKbPFobdvyh98KlEnUljBkTwg3AMZlPQQzlVJGaIRLHF/HgQ7LcF3v3OxVY58WToaCsb9W
+5tahHL1Sah8lGElIilfsRnRdko7coEu2gfRa3KgaahfUwfvu2M0XZIAv9XUtAKengbLZeJVB/yeu
+Y+bzabQuQDYArNUq5KNCt0/e+vteyydM5vLvATJaSIrx2GfK437ZFaBQmB+R+G0Aa9ofSUjkoLmS
+R0zo8R6VPHJpZpqodQgcHRA6RH2W9slDVbbdwuxFowscvZkyg0s8XevJm95iv/q56pJcGv62lrDl
+lCsiec3/9iuFz7GxwOmrty27Y0qyBS2LS9M/+IeoG38vIZNmSko7gXXrwIoWMN2hApN9a99bEPzK
+Ph/Yxbtl2fg3cf4jjcpB/MWn/HFrC2RrYttdGVaeHGppTNijHQQoSHtd3Tto0o2o84a6tif1874v
+QPyKQiYA1EDhGbJXcw+8IFq8U1npt0rH+pTD9S3w+3Y6vgmQg+1sLBl9IRJ1CI7dAZtSOocbQRjU
+BEkBAgDSAFPXQpJL9wN4CJu7YRx3dzbmYEVmxiXBkJtvs/WsdgG1fqUV+hN0XCzCgf6wXQ/i72zM
+hwTcH9y6WkG5SjzFZYTYeZpnXD+H+kIZX7X/8VX23d37MoJtrrFVBrVRVKbE34S/08Mu2P8dWZoy
+cgWG+LJMVTIloYDwtvak1MOLsCdjBht4556ZB3LPxnZNnUiA0FHWhtr4Wjr8ipgrWOSOXOjYymmE
+1Co11V8LAz/znaGGha+GmY0gDeP5cOugmzyQrj9kR8D7osdTIpYFvNJPK9zIFcgP1hPoNfhVCRJA
+H/PERcgkXtTRID94sLUhh98wxK9GdcgArwUTnradBN+1kLq8kJue5L+1Zry82C417FGOcMHc/Tqv
+YxdO0Fmw0umiFGIXPhR0q2ArBOigPtQt+aZooydGH7wQ1r1MqQoq7h9lAYAP5IUM1T92PPx6Hzj+
+jAhDgCKu9kNhHWdrRztzLX10B4aB62mPu/cHSvdW6ge1kfWO34HKk+VnGWsS354ujNNt1iJ/WgjM
+M5Mg5NNUCd66R/u3hXoprhQvErO7ZwCtIDZIbu8csQ6Hjp+1VrraFV/n+1iZBHS3UAmeTSTAJrms
+h8zKaK/c7y57cWDRKIj9mbso2pCRYw0DadkXgJRl6hzu7bbEar73ykHclBPBfz8RbtapNmtyhlNk
+pocTelxpTWltOD5BpD9o0W9gB2Ibp8OZv85h3lYUudWhMr38onwwuMdmIdzUyjO13PlSC7v91kcQ
+NIa534DJ6uS0uCuhDKXouQrMdH+mrK9U8G09HZFjdldFG79sVOp6NX93Sc1zPl0k7wab3pm+PD6p
+j5aZNjAlPDJUgAaGFFyvChhNlOs+mWZ0e9kW+iw1+qj4Ud7PhU5qK0s+SPn3kAbeOHxT6TQwixy8
+iMbluTZA4JdPRtRpdasV3AmIAw9n87/XRCZUY6XU62uFo22ZrhhA5pPdlt8Fjl+jlhzdC/+MMXB3
+7K7G5Cbpnaw6GlNrQgRwFvNgD+x5Mkh1FCyh7NDuLEzFJE4bayD9s2yvzCuYQ4Tdmnns+hncSLBh
+Le8Abx/L8Qdi2R/hU/kK92kSja7mC4qfYQJ1Uf+yP14U49O/1ilAP/F7wePceJBEdl4ColCx5ujY
+fzy0ZWMFPv7yv/zIMEfFnu8Fetkqe/D9BBbwmVbu2YxxSk8xLqaHYSNDAxL4mNxc8tnW9+fNG0EV
+SF0pkKAqwsKwQG8fdAnlzDaqznmkmRhSVGlYNhFeIcbQJ6O/TXKGY9EESwzSXoSGXaFqY5VTT4fU
+dMapXDjlMwtyaeXnn4BCv/3UUhs5XjJz5SWJ+WyRUy6IrVszdiw7BnaAEXy4VSRyBGpo4DLluoGL
+7npEpB5D6aiOOyeX8Y7gq+27hIrhJWq4oht6W/jGoAHaSaviZWshW590DH+1ZJ6pVIf1v6U1e8mX
+y9dx2BrUrY9FZXMLPPwJJefXVjseDhoQGyZ6h4FznG2VEBxaHwJ29x+Bv6NCmY+ou5wCN9ZYYKM0
+56vfwBywuDy1YjT1Pgi2MZnPv9wxoUTqQOtiYClMq2zw7V6gOLaQrPKEeiXowJYaWv8Jr8epS26k
+W/d3uFNHNTEI4ml/5XjBRRjc+HnkThYQ3O4vKasClQHCtSf3nksGHRus9CJLKDsib73TRpZe+ERL
+wiFsHfdQPBEUmJVzxy9UEcLFtykhG0wpyYbRbvHKZEnNV7yzuf9VDjVnzgBCNRkEkjA5Er7yYuJ6
+nsz/xwP9QGvsmZ0+0VvF3uPvohLvHcg6hGlZyMgdSmWIapqrjOGKY6RN8p7cLTiCKb+zYKbGg3j0
+bw3bQU8UDdnoqIWZZ3/7ahKdsYI1X9qori+pJqwjULNYIX3epl5N79MJHNV3j0Ci315TBulj17oh
+juNJhQ/L1OA8PW0VRJtkVd/kCMzSaCkBoxhJEt24R3hHcQun7FkwF+osb2XLAO1TZUCU+XqwtjtE
+mD8jguVPtESvWgZ2mYJo0RByBCC4ndwuD4vr00P462JaxoYN4IEEjnOVkQn6kespR1yAMXWwS+t7
+cAqhb8ajdnHUiySiCCZTqHWXELzGdd5m3jvkXW9eAjKY0ewV5G19kMpdrU7BvXRXY62LGiLQdN1L
+FmZ0uiST3Q3oWFF3FYKqwBaXXrO5um5iQFxwvJQH5F7Tqri+OeUb5oBThtHnq51axS2kbBzVIxKe
+fGBlGXEYywcioVg3OeTbGqPMTwDc5P8URpNZI0J0h+Yc+6dzcf5DtPci6sFz3BF0mD0uY3pPE1wi
+dY2yQw0kNGshto/z1LnpdFbxImrxNE01gmv9kjNLjL/a4krVVigChGtYlJEPQhlz6P6Qgc+K5GOm
+zjLNZtyAS7Pif907JOpZpMU7Ye98pxBlHQwwrEKvF9InzfwSN6v17ZXgaA5SMTjsTgYml/tHDle2
+chIFEIk1ZBr+gDMN5ZjbnhhVhdPa1YwLjiIlcURkblcnSwFw3eC3myuw9Y2WUg2cW0nDLUnGVtaM
+zFD2OpwhToblsBbuiAsGUNUOEZQL2YpENso1t5pLj4z6y6I53vmtPJM/SRWT/t31AxiL7Dice5B9
+NXLLjH3eYw58AcnpoLUtNJ/dXznO5k7qTJn8EMU6hlvIwrDaO8DlNEhvdFsSWHBYn2El/zip9LY/
+nKfrgO80IQbfjWIhGoy+ctZnjQ90B7Unf1wcCn/nSCFl+xDtS9s3Ks9lszWLzit5ADsGe9Bll6cX
+QEBd0C7iAj5D8SG9x9izhMudyj4Qd5pd/uwu4s4HPiWI8qwAK5aT7zZNahlLIQSMH6iyBRSEBskq
+IxkKUiPuecS1jEumfeL5F/CwmYf2lgN7TYbih1vQUoIwqpOT0H/sxFGDTXLroTRGHR2VVHIpJvgn
+l++lgbcJkW5AddVtu5fqw9PufdZdAEovQeEUhiECnu6Qxf95tJnJtkzFEh86HXSWepsfozIUz+Ut
+sHZVxQr01mpEGMsa+zJHcvN/S6BD8sFZ6ZKgL+Glgm8AlZqb+yohkGEiPhmKFKGHs65+bY9ciRhN
+dmLOCRZdgZi8EYIjw25X6aLfAB1cPwh64ciQ1sGqAt/yd2EemNV7cq9+JFhmJnb8CLp68NPxTWx1
+28mupoBAeyn7b5zyyLra0FcwGausEfvwBIdibqcmZsRJerj7BhZi4thM6/DPRquemfMJEvnFLy13
+3rKblgGr5d07sbwtgwCQ1OfjbEmLc8MlAJQ+iT1Vh0wBa4yISbXXLUKrG//9TBRtMk2cKzKCz0jV
+ec8+VH7NOv5cZIGKYZG0jAYUnt16Je/zdoemWix3dHq1ZU7h/ttTek9XygKnh5V6Y+COycNEJewG
+wz2lXJvD8NnsgwtNg0N9gmNeR4XzR9MdoxSXulskNrgIiV/YMemtpoFpQHnmNQ098PBbNwXnfWCi
+OHxH8xHtcTuLcamIBYqaPqUgsw0TB3lu22YWDJmDtzA29fRQo/xeGztSPeARjNhB+9mzv5Dg6xIT
+4gGWIoBa+ReS5dCCTNalEVEdafw/3BxF3ATXfF4dbABKetlRNjHOCu7DK3oL+DBTirTVTXW3wy97
+NzwaNA09TEL8ezyfbnU9lDbtTJA38rjH4ikhHlWjxwWx33oQkDKNr4OWgy2V28Wpxa6u8wka4nL7
+OAxdzt851TYDXtCO8E+yA4me14THakjeskiCsf3i7YvxirI8JLATVNHlc8LV1FRwAr6V2E1VXP18
+v/BlbeRK/o60CjvqDhhx7jb29oFxsJilIXuiMRebTlGYNAQcGGssr4KbwCkm9zNLLnF8x6gUgu5h
+ZOX49xjM0V9UVuw5dg7ZEY1mTJNzbUud0kSZu3W7ZHVimk7y7e1Cby6W/6eMSPFHQjY6a1GwhHFU
+wMSWxpDzcltBEutaWgfS5o9GFa2ihRAEv3RJhJ6tmmUelT0g8rYZdTRhSbwRSLSVn3aIeAbFMb9r
+khbgNwVx9l076i5OC9ilLA26FzcG+NyvMUpcHnkdpIGpvOdNGh7DsnkwaLGfxv5GW4cayKF5VgiV
+wirDzCcy7njqhGddt/JgOSxQgQ/KSNc4T/P7vLgQMG6bk9p9TqqEHI/mUlih8+ROYjoV/hyb3lJ7
+tuvoP97m693UCmVuZHN0cmVhbSAKZW5kb2JqIAoxMSAwIG9iaiBbMjI2IDMyNiAwIDAgMCAwIDAg
+MCAzMTIgMzEyIDAgMCAyNTggMCAyNjcgMCAwIDAgMCAwIDUwNyAwIDAgMCA1MDcgMCAwIDAgMCAw
+IDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAyNjcgMCAwIDAgMCAwIDAgMCAwIDAgMCA0OTUgMCAwIDAg
+MCA1MjAgMCAwIDAgMCAwIDAgMCA0OTQgNTM3IDQxOCA1MzcgNTAzIDMxNiA0NzQgNTM3IDI0NiAw
+IDAgMjQ2IDgxMyA1MzcgNTM4IDUzNyAwIDM1NSAzOTkgMzQ3IDUzNyA0NzMgNzQ1IDAgNDc0XQpl
+bmRvYmogCjQgMCBvYmogCjw8Ci9XaWR0aCAxMTQKL0JpdHNQZXJDb21wb25lbnQgOAovSW50ZXJw
+b2xhdGUgZmFsc2UKL0hlaWdodCAxMTUKL1N1YnR5cGUgL0ltYWdlCi9GaWx0ZXIgL0ZsYXRlRGVj
+b2RlCi9MZW5ndGggMTE3NwovVHlwZSAvWE9iamVjdAovQ29sb3JTcGFjZSAvRGV2aWNlUkdCCj4+
+CnN0cmVhbQp1br5FLjda1eD0zjtyAvE+/lTlB0Dp6bsMLbxjDhVJe5KGBEh2bGZM+87+10OBJx7v
+20LR6d6ro3zRvq5rM1pmCLOQmGGS5rcffyhdOnzsbeYQ9e1Lp+wonfzbbxGEm6K7V+6AClW9aGey
+ixjx6jkwBWZkg6IuzRpFyXgNj6iE22WuH2YuS1jPnRTdbYPy+8YCEJwt4CpBYS4uJlYqnY79bfCa
+BU96Hs1emDXJjFp/3IHK61M1fNOO7eeb5Gze8+kUqE8MQioCRm+Gy7veXfjf22cuh8LH9HP0Qbeq
+YTPQX/7yEo7qdylXQNQMq2Cvy8Ms+UyzNs3Rbq5Zc3EgjZKSM/2urW2M7PWLvrdvuVd8LzH1+yaB
+tsDbQKfLkj+TL+bxNkWFy/R+8wKJTaSd7vUVbjF2x7dGzhqTyrq3Yc3lTy//+UKgFwzjdPKPSU11
+4LybpTwR6PeHg9pWlEhra+m0/ovLPV/QAyPhF/0b3c7MT5n1CC78iQwvgm9QZH8+7duTnGW7rw9u
+cdWJGeS4S6n6x/kldfl+8GvsqzsHizts/C6DYnkwH2JYSXnIMcne4Q3C7z5gJxDLnO/l/rfrzkIO
+wzJn172IiNpcT/astSp1G35a7W21Mw/dWMQYy5L/GMhDdoPxnLRAhhAm6ajFThd92qj25EafimYw
+owUth7IFZv/3akjnQ5A8RMQDOavphwtIlCXGo6Rjtz/omr67M4AML0XmOQDFRf6iikfVWkrsSzw3
+BEB6a2FgOmYPWFE/kFqmOlBsiyP26PMyir7/tBxtKjgzwkrzMfUitUTlT0PD9DEwcBud0mTwtr7k
+gzTrFXRFrB60z6GNFZIOf2dCBpa/VXPvcyfK6jm1MM04UWlPGHpoRYCmmUr56foDzVOmXNMj0OJF
+85MZn/1Vfm3e28yC10NzaUiZO8imcfC6x89oiahlVXmOK2INa9XtMMoKLE/XApcOf1Q+VoEzSJEm
+xYEx1/bNyKHKSiYKlr/LFS7X5N09B11lztkX7nVdknVLCj82r9eNcrZ8zo5lB/UEzg/C6wmgxFgJ
+pHGQhY17ZL0d0wrHfl82+kkoW5m4eWrneNIz+MZkIyKoTt1XLUuhnAoXe1GPIhQqEjhJ4fRL71Il
+RSC2+SN+ZuGuN+j5+esV+iNJiXsw8D5HwpKlybC5R6bQG9m2CFFGiZDUck4lucnLtgfBXmIIimzx
+I7j65osM9FH4FB5qk8K9yqhldW90jtQgsWw9k0Y14ykBALpjxf0pi/jkks33zTrHU9BzDWyhice+
+eXpPHkdswQ9SuJOSXUr7oWqcY+6BnsW4KuzkTZeHgsofPnpyQ+ebRDqbpgvkdu3s1kwCfb0MNTT5
+4EuZBeHkL3jhcgrKwHPQiFLftxCaXMG9XPX8RwbjSZGnzHj+lfW3XHDE4V/zAA3sxCmku67IjEYI
+adyy6R3s52pXEmmbqHcXZlH46jfdvQG3GRJsR50oHRCXyKM+XrRIRLfYXhMgiDjvsyY7UJ1oQOEy
+b3lsxK0Sm7NwfjtWsIFWLUlQiu3LXTKrc60OIy0VkUQkOIn8a2pEOb5hFhIVCmVuZHN0cmVhbSAK
+ZW5kb2JqIAoxNCAwIG9iaiAKPDwKL1IgMwovUCAtMzkwNAovTyAoEl+d7xqkNcRcYiGTMAQ9qsP5
+2Q5mlp8QaZAWL5ISodlEKQovRmlsdGVyIC9TdGFuZGFyZAovTGVuZ3RoIDEyOAovViAyCi9VICi0
+adZcdFxy+EWzWcIvpVL+80kAAAAAAAAAAAAAAAAAAAAAKQo+PgplbmRvYmogCjE1IDAgb2JqIAo8
+PAovQ3JlYXRvciA8NmI0ZTRkOGQxNjRlZTZmN2QxMmQ5OTlhMzM4YjBhNjdmY2JhMjk4N2ZiNDhh
+ZjMwMWZmNjM0MmM1OGU3NWQ3MmY2OTQwMWFhNTg5MGFmZWYwODU4PgovUHJvZHVjZXIgKFt+fb0m
+ftbH4R2pqgO7OlcpCi9Nb2REYXRlICgvdH+9J3bXx+AVqK0BuT5epikKL0NyZWF0aW9uRGF0ZSAo
+L3R/vSd218fgFamjArk7UtGKEaDLeIgpCj4+CmVuZG9iaiB4cmVmCjAgMTYKMDAwMDAwMDAwMCA2
+NTUzNSBmIAowMDAwMDAwMDE1IDAwMDAwIG4gCjAwMDAwMDAwODAgMDAwMDAgbiAKMDAwMDAwMDEz
+OSAwMDAwMCBuIAowMDAwMDM1MTc2IDAwMDAwIG4gCjAwMDAwMjMzMjkgMDAwMDAgbiAKMDAwMDAw
+MzQwNCAwMDAwMCBuIAowMDAwMDAwMzY0IDAwMDAwIG4gCjAwMDAwMjI5OTUgMDAwMDAgbiAKMDAw
+MDAwMzU3NSAwMDAwMCBuIAowMDAwMDAzODI5IDAwMDAwIG4gCjAwMDAwMzQ5MTMgMDAwMDAgbiAK
+MDAwMDAyMzUwNyAwMDAwMCBuIAowMDAwMDIzNzY3IDAwMDAwIG4gCjAwMDAwMzY1NDUgMDAwMDAg
+biAKMDAwMDAzNjY5NyAwMDAwMCBuIAp0cmFpbGVyCgo8PAovRW5jcnlwdCAxNCAwIFIKL0luZm8g
+MTUgMCBSCi9Sb290IDEgMCBSCi9TaXplIDE2Ci9JRCBbPDg5MGFhMzdhMmRjMGE2YzljNzhmODY0
+NzhlNDNkMzg5PjwyNWU3ZTAwZTc5ZWFiZjM2Y2ZjZTdjNTM2YjgwNGJmOT5dCj4+CnN0YXJ0eHJl
+ZgozNjkxNAolJUVPRgo=
+
diff --git a/test/functional/messages/pdf_js.eml b/test/functional/messages/pdf_js.eml
new file mode 100644
index 0000000..0eb7ba1
--- /dev/null
+++ b/test/functional/messages/pdf_js.eml
@@ -0,0 +1,1800 @@
+Return-Path: <root@srv.example.com>
+To: test@example.com
+From: root@srv.example.com
+Subject: test Sat, 26 Jan 2019 12:04:58 +0100
+Message-Id: <20190126120458.015328@srv.example.com>
+Date: Sat, 26 Jan 2019 12:04:58 +0100
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="_----------=_1574345186400022"
+
+--_----------=_1574345186400022
+Content-Type: multipart/alternative; boundary="_----------=_1574345186400023"
+
+This is a multi-part message in MIME format.
+
+--_----------=_1574345186400023
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset="utf-8"; format="flowed"
+
+
+--_----------=_1574345186400023
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/html; charset="utf-8"
+
+<div><br data-mce-bogus=3D"1"></div>=
+
+--_----------=_1574345186400023--
+
+--_----------=_1574345186400022
+Content-Disposition: attachment; filename="DynamicEmail_AcroForm.pdf"
+Content-Transfer-Encoding: base64
+Content-Type: application/pdf; name="DynamicEmail_AcroForm.pdf"
+
+JVBERi0xLjUNJeLjz9MNCjEgMCBvYmo8PC9OdW1zWzAgMiAwIFJdPj4NZW5kb2Jq
+DTIgMCBvYmo8PC9TL0Q+Pg1lbmRvYmoNMyAwIG9iajw8L0NvdW50IDEvS2lkc1s5
+IDAgUl0vVHlwZS9QYWdlcz4+DWVuZG9iag01IDAgb2JqPDwvTW9kRGF0ZShEOjIw
+MDQxMDI4MTgxNjQ1KzEwJzAwJykvQ3JlYXRpb25EYXRlKEQ6MjAwNDEwMjgxODEz
+MzgrMTAnMDAnKS9UaXRsZShQbGFuZXQgUERGIEphdmFTY3JpcHQgTGVhcm5pbmcg
+Q2VudGVyIEV4YW1wbGUgIzIpL0NyZWF0b3IoUFNjcmlwdDUuZGxsIFZlcnNpb24g
+NS4yLjIpL0F1dGhvcihDaHJpcyBEYWhsLCBBUlRTIFBERiBHbG9iYWwgU2Vydmlj
+ZXMpL1Byb2R1Y2VyKEFjcm9iYXQgRGlzdGlsbGVyIDYuMC4xIFwoV2luZG93c1wp
+KT4+DWVuZG9iag03IDAgb2JqPDwvUGFnZXMgMyAwIFIvVHlwZS9DYXRhbG9nL05h
+bWVzIDM4IDAgUi9QYWdlTGFiZWxzIDEgMCBSL0Fjcm9Gb3JtIDMyIDAgUi9NZXRh
+ZGF0YSA0MyAwIFIvRklDTDpFbmZvY3VzIDI5IDAgUj4+DWVuZG9iag05IDAgb2Jq
+PDwvQW5ub3RzIDM3IDAgUi9Db250ZW50c1sxMiAwIFIgMTMgMCBSIDE0IDAgUiAx
+NSAwIFIgMTYgMCBSIDE3IDAgUiAyMSAwIFIgMjIgMCBSXS9UeXBlL1BhZ2UvUGFy
+ZW50IDMgMCBSL1JvdGF0ZSAwL01lZGlhQm94WzAgMCA2MTIgNzkyXS9Dcm9wQm94
+WzAgMCA2MTIgNzkyXS9UcmltQm94WzAgMCAzODYuMzI3MDI2IDc4Ljg3NjAwN10v
+UmVzb3VyY2VzIDEwIDAgUj4+DWVuZG9iag0xMCAwIG9iajw8L0NvbG9yU3BhY2U8
+PC9DczggMjAgMCBSL0NzNiAxMSAwIFI+Pi9Gb250PDwvRjEgMTggMCBSL0YyIDE5
+IDAgUj4+L1Byb2NTZXRbL1BERi9UZXh0XS9FeHRHU3RhdGU8PC9HUzEgMjcgMCBS
+L0dTMiAyOCAwIFI+Pj4+DWVuZG9iag0xMSAwIG9ialsvSUNDQmFzZWQgMjMgMCBS
+XQ1lbmRvYmoNMTIgMCBvYmo8PC9MZW5ndGggNzc2OS9GaWx0ZXIvRmxhdGVEZWNv
+ZGU+PnN0cmVhbQ0KSIlsV0uOJLsN3Pcp6gQ5+pGS1m/hvX0CozD2DNDtB888+Pwm
+GaHMVHVjMGiQRaYofoKh/759+/v393/+9fN/3//48/3PXz8/vv/16+fz8evn4+3b
+H7/18fz9SP7vaI/fz/+8ffvbP/Lj37/f8iEPMymlHXn2R7e/pffHh2nq0eowTT50
+lMe7acoh/VWTSzZNParM0NhvWjebdIxUXjRJzqOgmClvmnyUOU1jRyoVfZbNxH5K
+40Uze707WViqm8l+zfe3f5luHr217erjmGMPedhF92t1u6huV+9HSX2z0aM22TRy
+yKxbRHq0Ji+a0bZr9KPW/abDMlheNJLH3cnusKd0v2ZcvcpRxlb12g4p2yVqtYu+
+avai13J0yZtJPsZ81eT9EqaJEl+KctRxv4MpxtguYUfLVj8POGm5O7WjSdtM9lvi
+5rqV2HrU+mI+vBH7aI9f392oWcyzhdG0j39AUyc0uTxcThCHTohNIdf6eIZDSmic
+PtUs7IRGWWbIA8m1/+GQzHClu4dBXwURyIoT1AKBgyAEtS+42JZ/RJiO2uspwyG3
+dlnUabeT60STR8Lv3ofmYRoVJGtOWEjXyHmem9hKo8PSqHVlfCBD7hZMyBNFG3Md
+0XVGU6WcEURgTT2ygYUHnUqD3Nu6hsKiNGS2RmZNnvCQGCxr01JXattk4yJVc42w
+RxXl5u/Wh8+3u6YZQL1vGolTTW49ZG3CDsBwdJ45K37v1pURA4LuA9WRfoqwb102
+g6q4VVf4W+9CLqv+OSEi9Q7zcg58QQWJVCTOJ4epnv3UvN80NiCWKmhGJyo1pv/E
+bISVB3Csmh5hVMWs1rrST1kK+7ieMpOr82aRrXOvwXVZJxAjl1UO7IlqV/QCFg6X
+J7dscmvwKOf4EYduM/1ZMwNCftx0Bknn4FMxLCi4JQFajSGcbKysEd2UmHiT9ey/
+gDPTsP9wQZMrarumZKTlIdFdPjeTw4256Q1JBdbZnM3l0bJAQ4s6IUvvm+zJZuGo
+qSxtK4iq7EeWthxEOqcfFirYV7nr435Nl+ExZDOYpfADOHEyAluQCzWVMQhwNVdk
+pqY1qWd5noHVcigny3L7AUXOS/EeCqFB8VNNrJgTqTnk1pUywjZNpkeDRw0YrIEs
+LpeRTxkeeepmkVPfzkiZw5jQoO2Y+GQbfrF2jELAEcidcNLypAM0V//aNNSvNecK
+PL0+a8zW0J6ajrMdsu6xYNpM5nwmLjcLnjOfBq6bSubEUu7wyHImqAIFUHpLYSon
+8iClhJ1OhyqLNsChVTrwSEmXDA8pc7eQvstj/V7ooYSmMnCGnuxqPq7WQvKer80W
+hKKNo2YSiooG7JG3pXG5kzuob1GThbt/kYVul4NmBp0w4qdrUDaxzUkHaAwAM5q8
+kxl+1lylX16fNd6f0Bjh7flLzeVlFFP6l5rr9OX1WWMYNj9pHpBwSy0UyYYF+DOC
+AkQaRoMFaaPTD5dLPvEMDqXiE9XZiMs8oXQJuRbiT1selfDjZMTEpkS0CQchPLkM
+B4DNZaELwBJi6rVvEDcOgmRpuObgNikKcShD7LSfRN3XZBt3TJWaoeuBhkM7Vx1T
+qZ3cketxkKd5iyL3bVHBiovfmzqAVmxNJ/rYbx+hKVlOjcupcuna3yK+CfmVHo0u
+ifzPQjc8dJnHBskxkQQVA7pkf/GJX9Tt03jVcFM6fwoZy7h5ev3AShnA65oxr7Vn
+cmcBakUMo7ECU+kx2VhFec20ajpDzimfNX9GYvDatK4YYbAetdleVJHJ1TTa6SCl
+bRZaSLS91QX5wpF0WGtS4uf1cvTq3UXw2ZumNXx/8DxnA/H9wWEDZ3ZNu8YvIlLy
+CUYkZD2a6dBG5/ji0i3RoNWQ6yIoSofCtGpHTJmjohNpTTyxl9UOnI0+UOrOOviT
+ISqbsGRTWg5C3ph9uqKfsNmicJfYZj4bjpw60WGx8szuAs8cua8TBjST7UaEjdeM
+h6igz+k8YSiw00lwNFcwB8NS1i63ed85nriQa0HeW8cJlZVUrae8aq03i0Lu4V+Y
+IWflieCO4m3DM0XDwml0yDGl3gOFMS8PWYwoBSi5z+ivGlKi6IdCgmoyD6my3ifr
+o4XvE2WghU8oFQZOnuLvVXjkvJYezkjrkdSQzClcQrraevDp1wf7HtKEvZKV9bm6
+WkgOl0XrXHQdcp24Ra9rMmtGR+gEWpROuc5Nlr48Tk1WwgVfMwqPWilXWWd0nOrP
+04iK71mih5CN1rpmDdNbo9KYbsoJwzmYuDKWxyzUxEXLyWBtO0DmGT7FLN+ARVZ4
+tMJxDKAFtvi4+nMLHpr5cqoosCoRYOgm881y01R+c8H76lwZJE5g5nEqkM5bA73N
+R1/AksM+yJmvUN6jYJl2RoGb5njIRmYa92CfCyvn2p7Eyty2vXjfnNimamSb2THb
+j5vGIg78U9JwXzsS8nolpozGUfsuzunSwyLzbrHFRYJgXlvcNU4Z7hYySVX93WJy
+m/xCb/Soc5GNEhZlZaPNkEHCLH95nQGNYV8NeiKGPDG4nzUngzy9vtIk7nqLJRE7
+A50tWjL+PBDLwqkilbFoh0fNuN8g0gEbxaj6jp7KLl4WakPJSAPGrArsclBxLxM3
+hOTHva4tVs5e5+fbj3BZhMGx/+OmWdOltvv4/GOdJtcMX5yuSXIDR+EGdyhEbjT3
+U4aHtLZZYJOYnFh7guuqYy2DJyBXRRY811vlHchkqzz4NSpWar7Np8TL6mS/fgYg
+odQVZOUAg1sJxxX8OS7BISCf9ovn3WKQs5cxkDry58ZEKB9GJZJ5L0ev8/FanhhX
+NTKyCPII8qvJrnBp3jfNlKXJpLu+PFzkPA5bRy4nJd/NmLZpRxJ7ghCbXCmXHvLg
+tHVSHNOUi2O73Pk40OikaWtoYcI6QwkBGQYid7Catkv6Daxck9frIiwGWzrHznO5
+CyB0EhAHX7heE1g0EmCJIbnkKcsDGlsewe1GsIZYUANfkNVseMKYRrn08JQYkaLg
+ZjJCXsSMj4nBjeZcDLlNjeCBe2dCReEJ00AjEzzg0OQCi0hkuqACHr3OzWIsj4oz
+Jh1SNK93QOkn6kWHFKJgRcegpSzInukBjU/SiCZbrflZs1rzx83P9p0l+t7ANs+Y
+VteQQKs/Zj0aWZhS0K/keL1c8RNDYnTsgoteNfZvOUVmqMhmIIvTdUWOKctyqINI
+mODgqQykDERfsrXSqsFSdFaNrwrQ0EmG73CwTpBCThFkysoqcsOoee3htgZirgcf
+LWavF8p5VtYTclTmKWc+MpnJPABa1bfoS3UAOc24DtpPNbaEaWSOU/MODbm81zBk
+bqtgUCY3Qcf1hP6x3hvouNgCLhd0WLAbkwu/2PESVN+25Wbhm49xBaqZXPE7WYZp
+RkfbO8UNucGi6SaWwNdNg+Xvgc+vFN6KORR+SPlScyGyLd2FdS+ay6twZj9rrsOX
+11can/T/s10tSZbjIPAqdYIOSwghnadjImZRvZn7LwbIxLZe9RIMFn+SkxPJCbrx
+DNtRuFM4TdTlOr0eeX144zKwGbVAOT0zcc1908zE0kOib3a+10XmMrGZ5uqDBr/j
+sy5+nqBRYLHYB8XfJfedZTjz6KsY+vnaIrl+tl7mJfzfPyFkV14BtSL+JOd6Yd4+
+vYsoIJbk5A6JKynejumKoWVZJfsXV6Uk0bCTcAfMmN3owJh07o7XIHq0bdCr7oSJ
+EvP1sImuBRJYWpFsO+iNO+DmcNLkHzD9RofGMs5Pym8etGYjbY5E5waKRRs+bHTa
+JXRCOiqmxfCMCNQGaqBnwVmaFKOIcNbDnUFuz7qxgBvP+vmdWRA2VkmMpVxI+MGU
+9m5F55QNAUCc3G3QJicbcSlMDvn2INWRWyRUiBNfHA79EUPe6UUMrdcGLTX0y27b
++KsumGFCLBqxDLtXoV8qzAKrUTBOK83K1ZWOc3UBBgWncRIoQiNCjYkneL0agh/B
+FnkEIhmV/42CNT6pymzNwUBcENBdq6u/6KfkHo5RYxZSpYYZyl5W1dDatXoUFaGD
+jQAnVIpkQZzfx6D6hTBp52brVQ+6CeXw/zmJzw1Rs3ux3fXDTTbwR+J5MegHOsP6
+pfzaZAwI0J8qlULvCsh0Fxw43y9OwOhenIbpYRtG1M064fXmoJGyQchpeNSMgHcr
+neSwsgrjFCIsFo9Orh5hXDeHZq/qGZ0afEPqDyzQ9wRNBGC+OnhPbbfnz8FRjTIf
+3LSejvxr7GBYvics9b15MRSZQJ8+Rsuu+IMQRoMODXksSYnuGXr9oLvL43gicrle
+RvRf8jKxqHAB0sKesK9HVzD1778DZEB+cRnL+Hqs8wmWkS/7QcPjy14CFYD4gbxC
+JBlKKCCIYUP7eoLMvv1Iw++A085rOsnrTA04khA+6QFPEhQGTU9fhm7GAgKX1onT
+QTeicZa+mzzJyTqOzVIQCLFZWnCbCgTTeU0G2emovWMHGgpqp4ReHJsBakyInm9A
+b3GgMYG5IZwW2mgL+Vo1ucuLXtNe8c9+MQyGimwGq4STN6AU/uEgCRKcxE3ai0Yy
+qEHOit1tBcY6cHG+QTSwS6HGECLbZXBMddCUFy0vpIp+wyYxDkaFxhDeAL1sUl4N
+bSMbyn+2BS9mzerVKn1NDwnjMK0/rLE+3thtHlZsLgxYGTXFBTJmFeE9kMfXU5QR
+iPEqW4/UqrJFYUcsv96dsHKZn52BQba5jYEh/7w4IZX4+eZ4JnLsOE0Ykdsyaazb
+vpGz4EAjTpqkBYUzdIFme2gTaozF0puQGOzZ2UCy2mO6Q0FKQHdK9FXnKozqNwX5
+VjiF3y+zQ//uaYFGoEa9uzzo28IJslxYlF/HjeMMpPQn54ltcSK2Lf96XYS2mbOg
+Oas2kmw+f9crtk4vecXWsFGf0DqDniO0Tl/tCa35aGpHaJ3DNxEac2Q2X6GzgHbv
+2JpXoBwC2io3MHFw+s0+qCGFInOPOy00MSlcuk9si/OEcv4af2Xch+Ct85MTK4wM
+IGjE3s0u1L/g9xzcT1qhtGxJ32ANZq/1wPrMlsyvN8xfPhDmISElwfzitsIOhIYa
+OE0hMSe25sU/WEeFLB4GXnhrUoKlLZ1/QL824ylRzSZXGcFmG28jvT3XOtxweFur
+mxKr8Vrp+MNuda2Ewoojk9eKV1fS/GUL3Od0U8a6GTUC2OIIk5ToXJE74IXT0tCw
+q9Ubwsq1aNCg52t9vWipJ8B4iuYcf98xEtNEu+vGR2K0FrEb9WLm2iendY48zeKK
+b1MOGUezV//gXPouUWdsHo7FaTlVc8xPMmz3QyQA9/rgbKJPKsXKPprhw024vhM8
+vF33IXhONee0frrlfcK2Kte90TL7j8z0wtGDo976clg089A4OWscbsScOD31ltP+
+wdHai5OMfYb0dDNdF/X9cWRdho+DwwkRtuKbcyZd/D7UY+aL3z77k9NOJ5yz98Ho
+xG7lgzPWOpwQ4fn3cLxzeAFSyQG2jkPk9BKezyPFV7SXN14Uoq3x9d8/ITRaApu8
+Lfznf8DhaRZQN+gL5AoEHCQH9MJgC07dOz64+vAXBukYdE4vBDfgaij4cbcr3JYC
+VglR0LUTZFNB2d4BeEecepRPC73968DKQy44AEyUkM2VzxedxiZsWYeu4ZypCNbe
+kFADpvKh9CYHNt2LM70q8wfEgRaYOuiNpK1dTxjHVkCDNMIWIYKmG1fHem44JsKN
+SRQ+EFkZQtAADR3Y2IINEKHktRAjNHNRLRxWZbr5HafYizN8QH0fHM1XnR5EVENZ
+ARzPfHMLLxavyrSB58lCdggqg4T8MD0EZPICmtDvhCXWK/+N4GlGhUU6F5GKIpA8
+0hRYKkK97eZ8vzjeIB4qcBb3cR8M/z2zYVZbwmWqNEMmelWkwk9aO+tYbprBnfsl
+0RLiV5sGPTe3Z690FIjbcYCMzuaK4PaDHgMa/W4/zqFXT//k7Bwh/754PpLuxidj
+uVFQq0tlLWVnY2WtrKaLgXd63vW3AIQX62/yyFuC3FaXrKs0NKsr+mazudE3NhBU
+zDrvs10agzfapIRs0Gp20BFsJo4cYWpHr0vyeLKPUsAl1LMV0uw6FG1+vd0MGhpL
+D4Hd627Ei5sW+IKsqTlpg2KuNl6vclWn3unJm26ow3AC/pYZcwaPDEXr+u6lQI9X
+nRT0iUpLehjxt8Bs5zRqDGgIEb1mU8ReaTcNjVZ3CCUaL7h6o+6twC6hMX5t/HKs
+cGz4OcWBo6CN42QA/92cp34DK/6dc6/AW+snx2V92pNzY087bEG3Oc3+vLjc3Hj2
+/LXg7tUbO5a08dbQO0BCBLwR0n71e/IgpBw7RgXRgg1QGIT5nU/q9dDQ0L5PCbWT
+XvW9U2NyNPWFN+aNrvbXU1oI3u/PYktAMfzGaQQUggK0jFtxgjZihxlbdMQBaQdY
+8MtMwNkJJxz4zWqUgxx7UwEcH4ANRW5Ehj85T+pL6ycn6hOcxfPpJ+fRcoip9lfO
+83pp/eT4DNs/OF+g4OXsJImGFfNnJQTIMKwBCcLGgB9B93bPMyh0wS8k0EjQfKGb
+Ji2d82eUhnD8BBhxckxOtA0F5XgKGgoYNo/ErAF2wSYTO0ac36/tfjNJbpM+Qa5J
+E43ym1P3M9iOHS8hZ8060PCocdUxlNOIHbkeF3FalChiPwoKChx/F3UOWvU1fVHH
+v/1JTm96c4K+hEs3jlqNTci/WBa6XsR/brrPw6D5bIIcJwlQ0aBFx8Wn4WjIX+uT
+w00Z+ClpLOMR4Y0HhTQGb3DWftae08YEiMCGNZiBPamxWVh90s2rcrqTble7c/47
+A4Nr06tipUAdtc0vqoxkFc00Kmgfh8TsBNpR6op44Ukq1JrU/FyXY2TvTQLPvjhj
+4P+L7wUayP8vNhswc3DG035p0SSeoEVK1DMbFcYyti+cHhcFhiQtBVAmFTrDOg02
+NbbK3AjrxRetVzmwN2wh1cY8xMmQmb2wZK+rFJS4sUV3ZT1hs2XiHnLsdhccMfVF
+hULljdUFnLma1QsLnM1y44TNayZMnIDP1/3CmpidAYKzuBI5+Cxl7trY750TgUta
+OuI+DC8IMzmn3HTler4kOrFH/GEn3SZfBHbUKBu+qTMlAkYnnV0aNdBpc2loIaLr
+f7arJTl0XQXO7yqyglOWEEJaz6lXdQfJ5O5/8IBubDnJEIwsxKdpEpTizLLvGlKi
+rIdOguoyLxGt/aR+2rmfTDrauUJNpePkKbGv4kRrNfRwx1VL0kAwt3IIzSrrxdXP
+Fuse0ob9JCuzXVWtJIdlMYyDziDLxitMqjOloSLmBlp0oyz7JavViVvTJuGC28zE
+CRHKonWH4dZYT9Mr7rNEDyUbFaleQ/dKZhrdTflCcy4Grq86sTs1+dB+M1ifDpB5
+R3Qx07dg0SZOjM52TKAFtkS7xrqFE7NxcxIkeE4iwJovmTvLoRH+s+C9KlcXiROY
+ed4KpIvSQG1z6UtYCtgHOYsRynd0DFOjF3hpy0U2IzM4B20XVu6ansTKNl5z8Zyc
+mKbTyTaj47Zfh8Y9TvybpOExdjTl2hKvhsKZ/l/cY2pp0fi2nOKqSTCfKR6aoAyn
+hW5S1dhbXB6bf7DBE7KLbPS06BWNsVMGCfP4tboDGsc+SXqijjzZuD81N4O8T/2m
+uTjr3ZeL2Jno7N6S8bcFXwqnugp9mYYT0vC+RaQDNqpT9Td6TlZxWUxvSnqaMOZZ
+YJWDikeaOCG0fZx5HTly3nn++8+/eaQIQ2D/16Gp7po++7j+MU+bY4YbZ2guPcBR
+OcEDChGb2eyWcULHeFlgkrh8MfcE18qj9MUbEKuuBc9yZD6ATF+ZB79Gxrq0oz81
+N6ub/cYdgIQu5aSwgcGtlO0K/pyPYBOQT8fD29tikbP3tRA68ufBQEwuRj2DeabD
+ZH98T0+263QyUgR5Jfmdlz/h0Xy+NFtL00h3Y3iEyH5cPo5Cvib5bkO3bb+S2JOE
+2GWh3C3lxW4zUhzX9Idjh2xcDmZW0vYxVJhQd0xCQIOB6glW22eJHWAVmlbbRVos
+lnTLmReyKSB0ExAXN9zICSwGCbBmkzzy1joBjQ+P5HYrWUMOqIU/aBUbVhjXTA49
+rBIrQ5TcTFfKRcy4TCxOtOBiiO01CB54dyNUdN6wHTQawQMHhj5gkYG8HqjACZP9
+slh1QnDH5oErizcqoNuNelkhnSgoqBiUlDtpjSegiU5aWWRVmj81VZr/Hud83nmg
+zwL2fka3hoYEesYyG95oYUpHvZLjWX/8J4Zk6/gDi14N1m+/RUao68tAi9PZRIwp
+ax2QRSS8cCBCmUiZiF6yl1LloBTGrHGrAA3dZPgBB3WDdnKKJFOeVtUDo/Yzh0c1
+xK6Fjxbb5EG5iEqtkEsYp9a4ZDKSbQG0JKbot+wAcoZzHZTfnDklXKN73ZpPaMjl
+I4cpc1olg3J5KCrOLtSP195CxeUUCLmjwpLduNz5R8MmOGPa9sMiJh/9SlRzWfCd
+LMM1y1D2QXFTHrAY8yX2xNeXBsM/HN+/KaIUWyrikv6r5kFkH7qFdd80z6nOnv2p
+eS6vU79potPfmkhOyI1r2I7CnUI0UbfrfPXI7cMbl4HNqAXL6ZmJa+5bZiaWviz6
+Zud7XWQuk5tpjj6c4Hd81sXPEzIKLAb7oPlZcp9ZhjOXvoqhr68tkutr62Vewv/9
+L4zsyi2gRsRXaq6D8/bpXUQDsRQnZ0hsSXF3oCtAy7JK9h+OSkmhYSZhD5iB3ejA
+QDp/jtcgerRtyKv2hIkS8/Gwya4FFhhakWx7yRt7wK0h0uQfgH6j48Qy4iftNxda
+s5E+R6JzAsWgjTdsdNolfIR0VEwL8IwI1ARqkGfRWboUUEQ66+HOILdn3FjQjWf8
+/M0sCBurLMZSDiT8YEo7W9E15UMQEBd3G/TJxUZeCpfDvj1MdeQUiSPkiYeGoD8C
+5F1e5NB6bchSoF9+28ZfdcENE3LRiGX4vYr98sAsshoF47LSrRxd+XCOLtCg0DQi
+gSI0IjwxcQW3V0PwI9gij0Eko/K/UbDGK1WZrTkYiAsGumt09UN+Su7RGE/MYqo8
+YYayl1U1tHaNHkVF6GAj4BEqJbIg3t/H4PELYdLOydarHnSTyuH/c5KfG6Jm92C7
+64eTbOCP5PNiOB/sDOOX9mtTMWDA91SpFHtXUKa74KD5PDRBo3tpGtDDNpyonXXi
+1ZtAI+WDUNNwqRkJ71Y+kmBlFcYpZFgsHp0cPcK4boJmr+oZnSd4h9QfWKAngiYD
+MB8d3Ke2+/P10qhGmQ9OWk9H/jVmMDzfE5763LwYikygo4/Rsyv+IKTRkOOEPJ6k
+RfcMHT/o/uTxuiJyuQ4n+h85XCwpngBrYU/Yx3NWgPr330EyYL84jGV8PN45gmXk
+y3/IePFlh0EFIH4gR4gkQ4kDCGL40D6eILNvv6Xhb9Bp1zWd1HWmBhpJCp/ywEuS
+FIbMlx6ObsYCBpfWitMhN7Jxlr67PKnJOo7JUhQIsVladJsHSKZzmwyx86F2xg4y
+Dqi9LfQibAapMSF7vgm9xYLGBOaEcFnooy3kaxVy1yt6ob3in/1iGAwV2QxeCZE3
+qBT+4SQJFkTiJu2QkQyeoGbF7LYiYx28OO8gG9h1oGAIke0yCFMdMu1F6xVSRb/h
+kxiBUXFiCHeAXj4pt4a2kQ3lP9vCK2Zh9WqVvqYvCyOY1h/WWN/u2G2+vNgcGPAy
+aooDZMwqwhuQx8dTlBGIcZStR2pV2aKwI5YfZyesHObvzgCQbU5jcMivQxNWyZ9v
+jWciYcdl0oiclilj3PaNnIUGJ2KlSVlQOEMXZLaHNuGJsVh6ExaDPTsbRFZ7oDsO
+SBnoTou+al2FU/2WYN+Kp/D7ZfY6f/e04ESwRr27POTbwwmxnrBov147jiuQ0p+a
+J7alidi2/Ot1kdpmzkImVm0k2Rx/1xFbl5ccsTVM1Ce0ruDLEVqXr/aE1hya2iu0
+ruGdCI05M5tH6Cyo3Rlb8wqUl4G2yg1cHES/2QdPSLHInOMuC11MCZvuE9vSPKGc
+f8avinsRvM/81MQIowIMGrF3t4v1L7x7Ds4nrVBatqRPsAa313pofWZL5sdJ85cD
+wnxZSFkwv9itMANxQg2aprCYE1Pz4h+so0IWFwMvvDVpwdKWzj+gX5txlahmk6uc
+YLON00lvz7Vez3B6W6ObFqtxW+n4w261rcSBFUsmtxWvrpT5yxa8z+WmjHUznghi
+iyVM0qJzRO6gFy5LQ8OuVncIK9eiQUOex/g6ZKkroHiK5g1/nwGJw5McU0+mc88Z
+kKjeWrEVUfOZmlQQWdULKwIecfNdoHmQpMRYBYbP7P1893zF2yV2CWyT3dlN/r4V
+NdyxkIbGR3c4qu66PTdGFeROtEFSl+vHI8djjcObFpuLmgh22lu2HCd///n4vwAD
+ABb+jmYNCmVuZHN0cmVhbQ1lbmRvYmoNMTMgMCBvYmo8PC9MZW5ndGggODc0My9G
+aWx0ZXIvRmxhdGVEZWNvZGU+PnN0cmVhbQ0KSIlsV7uOXbsN7ecrzg9kX4mUSKl2
+ESBAAgQuUgSpnBcuxhdIXOT3s8jF/RjbGAxmFjcpiW9Sdjtc18tHP9aQlyw/pCnw
+PNQ2sIDeL/zlTXwdq9nN4fOYAeXQMRO2tV849BhDQ8D20b0I8/UOwjpcJEWa9ZfY
+PFYXnqgO3HGUXhhHTD90jptjjqP1nifIxqVTj7EG7wAfJIYfHnoU5T0pfW8+tOPd
+Q4/l+9ZDN/AHTRXv0vngkGMuv1UF7t2eqorxjltXmYePfesqkna+NOnrUP+gaw/d
+Hrr2Bt3ooH4h9WPguGAvgh69BXdJy8Zb7+MJ44U4BnzndzxvLTwHqniDilAA/gI2
+3EYj4OHhrKBID5VA0a0ps0IGZnLLEw1evOxYOHyhx/Z5c8ATKrx06X7dvsIVapD4
+51t/xc+3L7+9hQqjhYb9EFjuK3WwcVFExmEWHND9A4aREB2hdVFGO3Zy1Al4GX5f
+1x2F6YU9+4MDvh52n3Di+46LUq84Tzhf+Z0eoWV74ecYpedGXIZ8RFsPPUHpkReg
+iPOVO3wbWHbizKPCX94udz84et4JvCN6/NiyE4+e8QBKuAYEZEkgs57QbT0w4rpp
+8Z8UVUkO7/TdxOsDL0aLTSkBXmmHb5qh+aKzC4vzgF0RDUNtT9O2MpyLEm+mzJ4z
+cRc+CoZalOjIS8CRJQhwjcTLKCBS/CRATc00BcXyTtjBecTsZSjCp2++vP07RcYe
+mXnLtOLS5SK83wQoAGs/CDh9eobhGBlT3SP5GpJRE8+dpgBFJTg6tVLJQiBt0nD9
+guXK3h8MULezcnQLO8ID4ombUwLZrpUWES1RDPie3TWxGfEqZxYFCvSRKobM1A8U
+xMqaSZGtee0U4tFH4qwPYYRe7xYhwbKEOQQKzwqxOvCMKYT59AcHDLyIp1uaqkVJ
+iivFyuNqfKa40L8tOR7/L9x/xtNJmZbNAzcsYxObwlSULDGIVassoIAuL6VOQlup
+tC22o+40iiyG5PZVpl0VtL5WGn9WTE+hc9SJpZ3O6PmmMOhMjlapOYY/8IQ7rSRO
+ipBBqm66Elsm70C0awVI28aQ3tQiW7ke5hVhMRwUptozmuqDYzFHbLMeNeEBPrVc
+iXt5QzurZN3oI7E1nrDtrHHZwaLOemc5MOLuzJO2WIelHgWnN1K0klvz2QN1wl93
+JqNL9FPimdvvHzsSevmc7PJQ7msSNKtoEt6TIDUEpRbA3YoB9gzcajrZPZOg5q+k
+hG+AzQrrSDwqAANTQnQ8OJC1a9UdO7ErT1iatgZl1DPdKIGGz8APOyA59uoMcva0
+i3KVtAjPOX9GifudlJL6kRJ/NYcIfGtVa5o83hK1MKpdvLXxVFnn6zl94YsI9SPD
+VKP6TRKblH2asZpG9U17kX+VOau0MazC4FUKcybpVy0OHdJjfmv5JX3cx/zAIeKF
+e+LsGYHNS2K04jByjNJoT0ZFxlXOGKfEI9IiDMXiIQzN6IqIPusVXKSIwSJMmKg8
+FoZelQ6p6UTVdKa5xeyMWK9mKDESXjh6vpQAKTstD+/NyMqfUm6fn1I/oShm1JTC
+0y1GyZ9QLinT7D4/ody3n1I/oWCsQO5/pBjilZh6jvB4Yg4mwh0ClHMuzUoTWDkk
+bNSkwJytfHrxryqvhjsDszJhiIjpy859AbjxBsSMUyKGKZMaXGaWrsC+5MIU2K0/
+OGC4SQlNJLOOG9lVQIlFJClRFIDn4oA4RsJz3htLSoC2JeVpbQSUWFGs+VUDAuus
+KtE7H8WVxN3q1atVhI7S8xHD38d0jMaxeNqwq5N8TUrPObB6y4LDCPGbsFvV7JGB
+vpD5rVfvCA5MU511vsXWCNw6E0EuCVIQ+6xToPTdv6NI9kRExeSpo9PE4e3AczAu
+lkudak4nuZNjGeMg6PnyptfwGRIdY0ONnysZosZdXgT2XX7OWgTTtLpTorIAq1Ag
+94AwpqwLU2I/FoWlOVVccbUibojV0oeLVudE7Mnh2+5gf+DVTomTsktAOa7ETJEH
+Mrn66sU/Y9iNBDS+QRsTMtp14NZqvpHSITpbEkprq1V1xiAZVjB6ajKjI4Q6j4hR
+Lwy5lldNoKHPGlGug2sGj1ybznbhatlP5w9ObTEw0Nk9q6vn4zKkqk2Lrw94aT+D
+Ts+hmhxc2lDMYriMgJssts5pNgJ5s3G0tpMjBpdslhRYm81tcLgMNZzdzTb1ntWr
+MoOB97ndcvNI23GZ6NE9T1sHRl1I58wb031rPjliiqwTlNj5uSkFZurLlhvuQ/hX
+C1mxfgD7Yjs0PyX8HKss9zZQrPfvKMyVGKR4ysig2Pm/1N4QmyRPzDyD6WGxeONW
+mp5pgDcT6qDlB0ajUY1mUcnNwWZkUGMIrT1jsoXDLFtJiUE206DVdlOZN2qlskpN
+hV3swQHT1/A0nWFttdz4mQdjkEGNAlpXSobgjSszT0KNioFrO/HYjuLAzdHR3EvC
+amCdyyocaOWxmavkH+3UQRfHVTXmLsf4lUNRBsuYlRW9LNvqThFaUoUndKOl56Sn
+6wq4btYKJZOOVqamLWIa2q8ShkI0ydEy0aISVTWIGfSBo0KFxEnBWakGcHaQaJ8j
+71jVfvu4QrRxt1iDQT3StKMMB86cOmIX2aU4Fz+EzqTiU6trLWVIsf3GVVVX267G
+t2j751b2XeeMbvofrDQtWgkm9dyj+m5Ztn+XG8p+/fcfb395/fb2y6dv9vr0+QXe
+veCv7/9+/vSnt/b6w6u9fn1F6Xj97zVef3z99W/t9fe3X37/WV7/+vYmG0EStaPu
++npTsA7i5e9vn9/+jHNanJsr1SmKtfOYneaZyOuvSenK9U5hHu2hHZu4oFoEbsr5
+ts0odig6CIQyIM5QaL60BgNwBpayT2BINBhhrZujoXa3ugMZqA1DZHWyGA9CIrpW
+tXR80RaVia1mo/XceFXluCmo4q9LPpYTBOt1Q2whrSd/vSGWhB4SK+f5wEtmapEj
+QmHq3bJEnxw4S3riGFDTUs4Tc0MKwsydS3LWC7wjNmOTCaVg+uG8MuvGTYiO5fBj
+UKRWvZvSna0lGkfgVq0milFcspTjvGbfT4+q1SaXDJFbWfbL4bUYRgqSvxVDrGih
+p58rw04401Awh58BIVKdyTxNtwtLBUjkwolDIh47bo62USgnT4g3As90X0+1w107
+99R8hCRDo1li4A3nRWFNFZtUyM1BitC72W4RHdtn4lwmowf0XQJtcwrAvJwcrerj
+Gv4Bq49TIilRAJVnrip33fioitmudipRU2GsSIH3qoWlM+aW6YVpqD36gyMMcmJP
+3GtyjS5czqgKWkGqpwB9OdoqeGVzq4lrRIDhUTFH/IChjVlFIGvIjxRWlXdURO3R
+PlioG7Ikqgg8DrdFC0U5BMv/+a5yJEt2HOb/U9QFZkK7xPOUM0b1/d0hllxeV8Q3
+wZSUEhcQbNPib2eMNsjotqREzeLsbXDWoExO8gFuS2yvGDRw8BK55/t7a4yWRqV8
+b8vc3f3G2FFudueKGu7+PqEiRq8/1Hzlc4GKDjyfC95YT9B6WZBx7evaHy4Fn54/
+bSp5/x5aMiXUfT1m9n4eYIwH4InttQIu8AGXi/wHLocLLbmyZB4X5/3m+Po7CN//
+/I9e1dS7OSn8+bAs7up37sfe9Hssq/A+fM2wfumKi3WkVruy5vTaueWWAk5rzdNR
+cCgEbmEc145qSdZm0E0YDonBD3fchbEjactK8lox7CikFHDzHVrZ3gFhzb/OytCE
+GaKglhKfbW18xpVb05XRtWItrZgoxsQzVEtNLQKWJu9HbK4YZIT085ofGJHQjmFW
+wZCrE8Q607eaHrkaRRZvMVVURUfuLgo4s+kZHkV3uX4RU5wxp1xTzrYqs6uuqtzd
+rhpVxNVZt9UPzYFgKTw72o21I+Z4rcDkGK8TMuTn8x+ZFPO8bpFJ4zPn0AnrItdz
+V4pvdabODBdvQedj0i67ymxSWJ7L8bqyPn3NDPisCyi/3tHtFSH0hT8vy0rqzWzu
+jf2BF01UqYEoYKk3YeEgBzWULugd8TJDoeG2i6GEcc9sj8yzZ0Uzy0WfxFQq4F7n
+8lEPh2Wjwo6nuGQ9VsPhi6gCpafSUnfc/J0s39JNrIdfltGpDX/+uc/5bQEhpRdk
+mWs8CqZhthSD9abbV6ug4brKUexoxWLoQ9MNKBCcnR5rs91YPi21vlYgIaVyUPWI
+SSlixSO+Ssto5gf8447iYWf9O87kyfzvpFZOW/rhz4cl4OfEY/iUrtiqdDIelZFJ
+S/HcFaG3bfM12Ae497ixvFHnfq3InVtMVlk6iPU2T03HEoqE/zjy+BBZR3kHX4Ll
++xW0TNkWDtooqraxlxPoEhJBvPalO3zEqd6gOw4t71GVoWfc2Dm958eKS/mQhNJP
+9biYOXXRt5CZKN+vx/OpvvLkv2OjWk3N35TBuypem6upXCDJE8+prB+oi8TDZaHS
+w3FanzMFv9e4alfrUXFSE1y/OCywaNDKcL2h4g4QfOKxJVhCQhS0UTw+7c4VZbnY
+sSFpep4LYn028NOe7/k2z1bsUripLzDIo/iT33QgUzHmTju+9A98whtkWKTizh+p
+n+ylPxR7umiCgDe6NCVVE3WhamfGID5b6mGv40e0IR2Fy3WSuFQKxj1I6FCR1H55
+aYUKqZVDxzcm2FE7R6i7ZZgGvrelU7a+LAvNF9cpas8xhIdLr4iIEG5meZIESSFd
+cImGECYRPbID0av9tSLD1z1mjEM8qnC181I4h8oZ4yQDelytEJQIpB562ra3e9m3
+5eexgFHjsmhoy5pB60MEumTlbIPXaPq8SvctdpMORSAYAc9f2y9fPT4oN70347Xi
+qKA5duKX8KL+USV+MHzxnfsEv59pz7QXTOKZw8tlufvPq55/W1DhP2TpywbXu+iP
+RReaobYN5SsymclkVV/q4GMjxC1xrvTbRzUQS0Uq9kpcVdZ96/sZVyLUkCpgY1hs
+kKKrzQD467RzsvyWTqCuB65mu10/cFnj2mG5s5dXhEl8ttcv79kPZ9kz7Lz413pk
+ml5Zv16yDe+b+7UiC6J7PirCvWlHPxdhzjLdOLjg6hPzuE6f4JClR+Xk9G+T46yW
+Ip2iImM6M8v2begj2OlIq+j3AwrCuaOZY6AXiidBn8CzhhUQYTmCs1dvkCVDdNgT
+09IstR+Lo04lNzCvSMRG1y12c4Q6a3cW8166A210YoLQCiQi3unPgzGe8E28v8/u
+5WiSibfDMY7Xh70NlpyNEkEpUYlbU0qswnClpW9bMqGBZ9lOqk28qgJ81H/SsqtS
+t0B+AC9lNtrLC3bV8csyty6xjjIZyoe/dGaf069LUQugHkPP6OoftW4+O4oKGppV
+715FK9rUihbVaoBujqkCHw7D0v6xFIXu3dDZwKUKL2dPUCfDcqBWxuE0xvYjKDrI
+zlKOs6cd9QEIBODaJfMmhMSFwY6S7C9LVK0YKMYsD7oNyUXZnAWiCWmAIDstc+va
+uwQxNMidHcZybIP4vFcgNIN4o9XPnmQjvEQAadnH/1iIdrZ6LYAWA1wYJhM3Z+xl
+yUxnK0qDtfuNq3sPB1j8oUuqb76ia7iBH9aVQIfNrKYQ7rx15xhS1VPznWf0Gzsd
+MG2+VmC0IO6qo/sEeT8t1d6u2gAuImyHrj1zEUO9K4W2nokWwYwqwie0YRb3z728
+YRSNQctJ1qfePWt84C6R9LJUcClO6MqpyH/zH6ERxk0HlxoaBXYVP2jYCMld8AfR
+kAhHhvgEjkkTuSU905dKVYUINXPxhfggLV38cMLjynQGFSmm4oRIin7JFWbQlL6Z
+GF4TT49MmpAmtEqo1opyLI4nMRTbjRcioA0yoIU6J0VhldXd6S5QO+y6Ugt1h9qc
+UOdpL3+3G3aphfyVkE8eRAfCC29hnro3C7nrTCjamdwyNBlszXFzUfio++Id6L7i
+zRkPhLBdXm+/dK0GIUs48njxdkY65OeU0uGJN3ifM1xWTB7MVw/+5otijGfFyqIJ
+VeqAZFnJuqFfhER4WnZReowU2cAxJaDhthsmbUqq3pZFaYLqT1aztHoss8qCbCbe
+biDMsEPFRD/ZkRh41SWDxZmt3bPpxoyVT61VCbCGeE9qXPPt4IrhEwYa9USj9rgq
+5Q93LQsbuB/hr9fUtIhH1BvLnfcZXvF5QlKJo69GmoZRL31m/7dw8xY+/j6je0cc
+XWItR0jdnY00YbN8O33Y/8MCJJoCNu2pwASUeLVjUdm9g1xqy8/LMijxZFl7ydIU
+d3gVuDkP+lHqN/umeVrL4pi6R9EBBfMhXyrxVfrlmxmqtmAuJmHvKoyGDmd26z8R
+E9zN+kPNbrp/UaaoPD/L9xtzwpX/IJ708x9ZuvKZCjrxMYYy4ZlDI0yt/umYKkrm
+auK6zafUW6Huaay86ud8rCin+gRl//GJJa5sX55azxyqj9p0SRf6IK3oGd+/HvZD
++sJujZErXf2HlsUMluVHlupyJiGc/1rTgEYTdQ9oSzItLa34zND6YsbaUIYrCXyd
+G2MHBNXnimEHL+iPxBLQuAlfvzBwLYfkC3DrZWMvQon4vJs62m25R8Q1nlHg08JJ
+VS/3rt8WzKyZqrJcnQ9+u68CAt6Eyr8kDpNfqlcKtlST0KSJWxWuaK6Js/NIGZlf
+0x0hadSbdviAUbr9aYJe2xEo87p3ZQSaxdZkJaaf/Mu5rphJCT0r5rxW6IRlFQq7
+duzLN1M7dnTrt/N1ZxaxN7xTjem3lzqHHZzptwclxissG2zWPgKVwua6vsKykwFX
++7BcvTxnsPTmD0/mLPBa8/l33SgogaHoRibpf/Ln0P4Vo93B2IkVFMe+D+68MbS+
+ryzD60/e89tyb8p/n5Q87ZYOefDpVJEXWSUcIe4KqNHExfRZJsN4koJFpwffk16r
+fNmRiyf59FjxkBHSMKc2BKKeuBf1w84fJJ26OW11zAMqdNdFZSY+Q24G9z9YA5d2
+XBbeCSQkukgFyT8MsdaR/MAdVD+Nb64cnVheQ1j773rCK0GuqJ/QqyXys1rCXrJ+
+gXe/6VZITEnuzRURVjwt+T3jMPYNvxmY0+trQUrIEF7n/2xXSZJkOxHc1ynqAoDm
+4QSsMRYcoA2DRfU3A+5vhg+hl8rq3qVHSk+hUAzuWbj3ePDtIzBJc6QWZnFdSMPp
+I0kpiSMffIAhg6KmAmwadxnKnsGOl/bn7f43ir+fYsM0XaYHXrCyXVwxspQKoApu
+kHv2WN5bCNM+tKDOmGllxJUNV4kNKRob2xRjZPYFzEHLZB7jwY5iauNaUXFpd84U
+6V2773jijqezoZIK42W36gOpNPMbPsl2LIgnmSg/oDE6/pwNaygCUm6fMJzgbQw5
+nfZ8kTXgcZM5JkK//0cQVg0y14TLjP9Nv2Fp0+SrdafOCKrVSZKAZ+wYcQZTx/Rt
+LD918hFTFYerFG9Y4yRPKd6wuxbU5BJNXl+DN+c71Qi/XrBJU9pwWJuvOIKZNHE1
+nVZPkznnp2grOXddaZegccOpZ4kCnPdJthE0bngH+bVwBKXkIG2muAx0CxbX9C4r
+51djfGuc1GX/+SgZ4xC2zjrYn3knBfhPrAAwPnTyf3z+8QF6gB5w67dNofTSbwuE
+bF36bVGu+nKh32DJ49JvC9nZIueUtQc/Cu6xWMLxA+0l4QBnf5Nw8KHOl4SDh6Vc
+Eg5XqKXfEm6DaZdLwm3W0SXhgPv0iSHhdos3DAkHXKv7MgfDAx8J91gewQYv16zf
+LfuScMC7XxIOFxETekm4TaJ8STjgmi8Jx8fZbxKOzxr1JQkHvOYl4RCcnN4kHMPV
+LgG2qSnaq6aB16vEHc2crqJndNu1HzIn/rWAg2HuaCo1oh8uSMABc1BdAg6WOi4B
+x/cpn4+AAxxuQ6HfYFjp0m/bhODRb3zN9KbfYCmlfF767bE8+o27uhmx9Bu/mi79
+BjyHUz/0Gx0rl34DbsFpqN900Tf9htDseek3hqp/PvKNocxv8m1Tu5bPR77hcfZ+
+ybf36pV8O7n/km+05Eu+8ZvZKyTfgFMx5bB8YwbUS74xQzzZpM2YUuNNvSHp5rjU
+G9N0fD7ijQVbjUO8MfOPCxRvqpRLvBGPN/H27Vriixs0P9pm9C6U5xG7jA5kU0Q3
+8d6guyWodXIRk1W06NxYgPSJ965FcIzIkGm3j4Udfylnglbchl1i+pD4bE4hl8eu
+xnm4+lZUMJxYrqdJ5gHc+ohi6MIjJmQ3bYVlDVva3Lqnp3y3IAI+XSPqZQbP5Xge
+WrCy6600OoVA7vrgHwptTOmzYFSfUDiTN9/NB0xHEk97EwngPNcz1m88I7Nflt22
+V5zAqdrwxcC55fDJcxUBy3Yi+m1hdwVsMXeooLyhLJNzfC6uKdhacRS6P9DWiDiN
+7DOlG4FbUIm+vKOET6PFS8zsEya59h7KeM6JVJ1yaTbhGq+Nd8+2jN6dZMs8NXGy
+X7jPfJIuLGtowexRrdUb1opi3TmcSsW8cjQ7VbKrc83pBJtWpmnuc42YsCc/UmjX
+Vn3vVvejbR2pVcz4x3CClFAAUgh6m3prCL7vvjQE8MihIaYzZB+Zol7WUlIO0gmU
+JeFRqxw0xHuYBpS+YoMsHNp4ti9ZVmrfLHKMFmQJ8SjGKHThNkMolMeNYraywU+I
+8/Q36TBx6IAUacfmleOFlm86rB1K8s1bEJxi2QpLOQ0zRe77k7VEteUHOrF7bm8L
+ynAOlGmcokPneGHmun1IyoGJLLK0oJy88TwPfAyjuqes0CI9O0V2CoUW3AWHJvfs
+2oadKl7h8YmdGvLrEGcliVdk5xDZmEphO245NqS+Ik7McVnydGR1JJwZjuwyS1ua
+M3ybouLAbVDyem2pvEmCofRQg5hylRvy04V25WNnsX51AHSpG6cxYoctrEuvyHNE
+JzROQZey2I5ONSs4+TSW5cHALFEGhgBp4r/KuGZi4u6La+agHc4WVedLX6gb15h8
+fgrHKebi29ykXPjbB5IYcgpkhNncORUThwnC+L8ff3z85a9/z5//+t8HKBJrsOPR
+UfU/H4wWlcmhDsbVcIXNG3QiCNNPUez82eGT4iBWt4hH6p8kiVv/ooeLYe4AWJnl
+fvzHeNZnG8v3+ibeG1l6TkyPJ+n4yDXH/y9+uYCw/ILNQ29c0BTEnIe+OQl6FZgj
+nJxFuHfdIEFHADXs4H1GOQhri5jI69+NeJ+dII1L/zGdFKaKPOKhcIkMEy8rhy4w
+RAwOfm7z9lqkTBoWCh4L5OeDoS6xEDswKvr0pXAYkh7XBhpoomQdAuJ84hxa2VCJ
+ZBzVccfQIt1AFwmktRwW598l+sWdaCBAueo/ykys5SDZDob+rUWBQ3t7gZWnlwbe
+Vf+SKfMq6EdE8Bkop+61ma/TRTrYvJqjiMBsj2IgipAfDMKGalTCVF67aSMpF9kb
+n2qoS/z4CLYHXDGbOZr972RMAoGEFi8N2MGouBHfA1roBaQN6IB4qNybPeBw4sMV
+upezXpHsnc6yaowUW5ZY/IlbVyVAzwoe1QS/0rZjsJK+WjEU2COWEEcGO2vXZ7Mz
+NDCdp5YVG5P3BYXGxg0zUEMHI/EaAjueJW/92acyY2PIdTMrTpXaD5L3ba/rX/JW
+7kQu8N4RsNwj41gI00UznUUkVutCJYISMGcvHXIhZS3l+MYj7RIuFLxdH9I8dIi5
+MKjLGOvF32PEylmMfaR/FyX82kKmLsxMKAbgpnyfWYCJzSGHNsQMEjffpnHA7Bck
+Alupt6oQS4SJ6GttvStwY2JuVydSeuwHNSWU1xo3LSUBBkhDb526muY6tTHxA7g3
+51Cpp9lO8aanEfP51/OffrBlDd+wijUrfTnB2KKz/s1NmyYL860DcfaI+Gl1g48/
+P3IquFF7ta2c8OE9bwOkUD07iJsU7WWooju8fbeB3fkd13J94t2Jr49/yw3SY7YF
+BP2n3WjtGL60YjXla8HbfX3fwoaLykU7H9ftjoHPRkaac3J6/sYQ18WW7oCwyG1Y
+LIVfDOgxQxjHlf47fM54NnwzPCF8d1wRebmOr/oqJaqu+5isEnsde29wODhZdMxA
+4vAbrBn1QV+keYR9x8/TZ2Qkx+7L0KkS8EFW3Nf3E3woaiGtcxXKiImWx4GKFf/9
+J5eUgv+UDxPM++fL8GQMDB7D7ItEyyOOQycXsBfPzSIyTEOpoj4c5oQtacRnRIqw
+j3qgl3OmPf83xxGfK4Z5iFpUNSIaNC8wJMGLBJd8mynfcEtzPoauxiyoHMX81reL
+krqXGq7s5qbTp1wdHqtUiIS1zHhRr64xALfCz/X1MehiS+2tjeKLDTVbMgq7VpqW
+zxauVTVSsu8HTk3tuIkNFdWq3d0jwldJS3/O1M9VSLOQTE2uMDzdwkVPUtaB5yrt
+/r9kb95+wfj4Ei1jBuzRnKnTCaGjh0H36Borx2IZlrqOs2kOUzMtJ0kl6CuO8vTf
+40kmlzumuxx9zXwlkzmsGYGuvdfrf6asu2c1HEUw7xIxXZ7Bu/Oi3fyaddZvWJrz
+4zGk7GTbS9NtpSq46PuIAU/DnJp3z+nVw3D6yZrfO/ccy8fwOGz++qpKxrW8mzqP
+uTnncWapr7Vp37OZW6ntBZu4lpeHoSen1/ZYzNWHzaQZsOKqpqica93JmcqRKAp7
+K89o1KvMcv3LmUBUnXtkpYRPQiwLF2gtZYCqjKKmfn7rRj/UoCDh3qbkMTyNsEKQ
+lP4bAzK0ujbJJ/JvDEONxVtYp78aUO/lbcsvhmdwvHuqBlyLtdIkq/p548XAEjJ0
+ZOVNcDTdLIn40TCzbkbJQbj+z3aVJIkRg7Cv5AnesOE9ueb/5yAJd/dUzVEYY7ND
+07ScyQQYLS4Eexq7f85zoWrzvT01/73S01WaX/X61B5x//bA83AL71XHumz2EZ5T
+pF92Pn44sj1fy/p1zvt1QWlK1d5zKn5vl2Fe6Y/h9PpjVv3ta/W/aOCg+PkE0SVk
++XXFDPRZvxAe9/6UIfdqCfyILcIrJfP02K8E9E0RuHlm5KU9AddgZFo2I0C7oziL
+VhJOmgQCaagsEBq4Hek2UT7mhWDPArP793zcouWE7M4Y+nex22CbQkEA3KauNQRr
+zbN+2ZeqVHBZAmFOPoc2zucGayCKFT+jGpi5UvebdoC+pWzUeUg1/aVHab7VMet0
+hXi7EU5jp4+9i3to7/Uus/LplNF0m17IW2sVOwmvt3+4Vt7GuKfiNDjCXcKiy/6R
+gI0T4wseQSfhMNOFnIN4lx+dew82E/SqhDABYD+Es8eFYl9+vue7zhkl2LQ0ZT3S
+ncL7MR67aarKWPjAefrlFoFTFYTrNtY//mXw1Ncu9jE44nmIvS/T1EXUagYr4Yeh
+jxlsOM1gh3AcWWVtVtU1drFPdTIkOE0cqrr2RVHZUISMMwxpuOzquZgm8Zbbpyfz
+L13JI7M1U3KYFJkK75p0kmBq4ff8KBt8yi5RuRKKoVwvQ8mPv+ZGu5mKaHCAS/Ni
+yS6cqihzQvGaGUEfpayl8F6yqqtq9PD62mn82jC51KTJKMVWGxeWy7Z9z7vVbSuf
+6bRdD6s7dpOP4lCT9kVlUILMMgxokBvMQqwqDCSTc2+c7cOcnBU4OwS9zNuFxjWn
+Rpomg8zGuEGBo/kU8dFOcdsggWUxkbYUC/9CvFTcIkj0Yqb7Fm9l+Q67/5hbE4sy
+9Szti6XjjH6hlOy+3vO0kGsFK1t60/hDU//5L8AApYeXyQ0KZW5kc3RyZWFtDWVu
+ZG9iag0xNCAwIG9iajw8L0xlbmd0aCA4Mjc5L0ZpbHRlci9GbGF0ZURlY29kZT4+
+c3RyZWFtDQpIiWxXTdIlqwqc31WcFVSoCOp6OuLFHXTvf/ogE7VO329WiZbyl4BV
+xmPVPiqPqnyqwy4S0FoHbAo4Wv38+ud//9TeHukVO4p+/kDQZOGEPj4BayMsHbDU
+FrCP4SfUXp+5FIK+PgGHTUAxQOP9Abm9mbzXSwHU0K+XZypO59nlGQW6zTGwas1t
+609d6w3N9nYI7GlVP78hKONHPBawK+N+0OGmti1Y9oPAnjE7BO6u9hPeV5wf/hL0
+Z3hcvgTd7wgYDuruFvpD1oCJc7u3W4egDKzrClSMaIy+YQbjCgLysDIrQ2fKoyZ2
+e2h5WhWGtmkFHPY5ieGauYW//s6U30gefRrDN90df66gP7PRfH16gw4K+/QxHipj
+As6utL7jDntKp0rIJoNbwoIlgD0Ncsjtw534Wl9CC9XR8GiP7drY7dlfJy6HPxxO
+3KUy3nBI39spmMu4HqrrU3J7DdUVPuT2AtxmgyqztYBdJzU3/GyTvtdnRWq7vdrg
+CO0zoAhhj8PMVWu5vRnzTirW68Lfy+aFoeHengJrC+sSQRjP6hmEj07fIrl5DglB
+75U+dy46VFNAU92QPp/d7rqbLdy+vdhw1mzpFVjigkYn6sJd0hmCMbG9Nc3tEPhf
+SR//IfI//mv4QaJ8DBQwBMGp7HCWmsqtqEYuoNdnhRtWo2WWbmh7N5S767Dcf07L
+ywQcqjtmHZdb75/jN1etyBtKW2+/uoMyI2pY4iEEvSJ1UYz61mUshFx2/ggQ907F
+r9J3JosweeAEY2KOudKnM6DaDgFY4UmAw6aX3IU81hf0tG/cfgTSBcfNQYq1lQFE
+fR8nwBJO8wqOIuIBGKzv9Kh2S0Qjo/Te1SLsFT2LQctesBPTCtZ1rSQFm0MlfNUe
+9DL19iH91ctcMKSdXqbRIOT0Moda9NXLXCBznl7mEKTLXuWwRvrcXrY8M8btZcur
+k91ettyBenvZcr7p7WUO67q97MDdy1Jw+0jwTX4WZDdzdaq+m9dCRvxXcLpXOKzK
+j4J9y/nlL8HtZ0fAfrYYxd3QHK53P1vP0nn6md8HjmXDgsfnq6FFRGy817MboqNF
+APdhM7ebttPRIvxsL+hoOz1OR/vOF3Q09YFlosl1j8ufK7gu8jYo80dB8WygQFvd
+dA1oEyNBlPiAU7Bq5KcL1hgsDGGjuPIoBBM2yCPFNoztrlr7WtdOWCagGbVTy+2D
+IS/cvdBGMA6GHxU3az2bC0rJYvsG93B6qTy98/8ylbpxtbaZ/7eF26txe+UwFIR1
+VJhKdW27az2rDRoHiuYdTos0ck3YvF3QUbVmpU9xc1QI/owQeInrPXdLlpYFcnzH
+laEeD9g6hYH2QjY5G38C5BQdrT/6bM7AjWnpDlysY9HdNbt7REYAG//uLGShZm7H
+eswIqBs6BVFYLIqroM9EFRicvNrAulZwwKJgbxgtvuX2FFgeZ8aQRSOJ0zk5CCeP
+uF0RBlsdylhnfqFERiz7hjRV13ivr8EMWhV+an3trvULXlRmZxlctyCsgpcvGJdw
+ewqKZ9BvalfbXwKprDdUNyakqDa2ACN1w1Xry9VO+TqwvlgB0L0iMto3pHWI3F23
+keWGcJW2p1GqG5mOyEyYI8ZOSQSgdVu2s+v3P/++LeuLzck4TujJ0Ugz7BhCV884
+PsjIJLOZnpUdGekszVwXFoKGyASR5ob0zLR5191RxodK6+lIjhQrd4uCb0v1c5JG
+QdK/bEHvNX+ojPbikwuaZv8LUKQeQlnBwHYJ5YLspMEnR5KvxuCLRZMYbz6tvRt0
+8lap89JpPlPfbJpof4dNkz1+synhZdMRWJ6mYpdNkx3jsskvl0smV4xOJ1kWp9FD
+Jov+ddfDDV0OmdxJtb24ZPkYTC45VM5x4NKFyaUr2NRZrrR8CyrzmVxa+7FGLvnA
+MsebS+HmernkrZx1CFyxGIfsxaUdtbuufR0uOcy3anLJBYtMBZciXUhskGmn06bT
+d3qRUMe8JFS4s+qLUHHnkkOogGUcQoWC7DhJqIgOX10glEOU8U2YCOabTz739Peq
+cgggncKX/UWnxSq26cSs2Wz6MoRsiiC8BlmLAWqeQdZhYcvAIGue7Nl/OMi6YPZ+
+BlmLkfkOqkam3EHWBaDzXvfGrO0Msg6TrDzbO7PWM8g61KFnkL0wB9ktOAOkC8rX
+ZPsScJB1deqor7k19Ov6g+DMYOEvWz8K9i3nl78EZ5C9AgyyAcc6g6zDmObuJGvx
+iJEzyYZTZZ1JNVzOMTgn2QjJFSBkHPgxyUZAa39NssavPcmGOaWeSTbT4wyyX+mC
+4cYimeoO6R8IJItohMlRZQFWAcyXjmUt0Z0R5ikZMNNvjPmGc9TcnoK1Go9rrBSL
+p2OEDeX36TLh6ejG0HVy3snLMHMScvsQe69P5Xbh6YueoyqGaheqzPBzsJt1A63k
+QM7eFwuioqxJnoB95k0eM+eDbcVHWOKz6qAX8eJzRsR7Fma3gKr1eFGxPpm34YmB
+/41+koH/p/L2zv+XbcPRLnxwHuknxpRUO1C2cq+QMwsW73PrnRZ/rqC7O8iCqFtw
+piLL134xCQLtNa9z3qEDhhd9PRNRQLlxdmRM0sIGMqK/2V2PIYCzmQLVuekVm73b
+NTaAYETAxQYQtfvCQWuvYMZEEbCz+eX2xueCV9fcXhWDZos+7pqthtkKA4dDNbZW
+tnUXlML6jV67+K40DJ9wWkFx0eSfu5XD1JCK9bbw97J5YczKe3sKDJ1yecRQ71bP
+GMTd08MsuR0PxOmFqdLpkeETOgcceiB1D/rsZTdcuLz92HB455DhAtjigkY/2sJd
+0hmFOXFYa5rbIRhe/JBA8UPkqNfmoL7DTjga/2/RagZaSMYBkF5fFWetRrtGuqHV
+NASq3XXY7T+n3ag5fpNuu4s3x/Bq75/jNVesyBtKjitHUDIjGkezERU2UlfxOO1b
+l8mXrez8yTmQm5diUIsWQSu7ZPrACaMj2cZc9OnC8KwsQqORF/G0idOalzKObfqC
+nvltb0+BdLJqjTOH4bKCijZOgDsLYrwWQMLBPkCfmiaglZKTARdr51jQsxg0Plvb
+zsvBJqFrJSs48FfCV+3BVDPiFcG+sfBGcEFZ/AWm+FtVsjovwDGySdGtPlUUlOc+
+Fetd9OjnEJwnjO1CR+91PzZPa0A9tdWRuytrfxT7gGVxGg5SXmil5/bC4TwnjhEn
+yQ8CWvElUKSFq4dqKYXaLc4HNePmARU+XZXGNtqOFhu2d9swfVPlvT72w7YDTr56
+ymQ91DNwRAt2WFnJm8nnBKqD5b9u5K5xSu7+RyB7+L7B/o9gQhAJMYIgoDDSYcMJ
+i4OZxrLYAauiMpXUyetpQ+WK12FAqViv0ekdxuswIbcry+Zet5ju799R+r9OH1+X
+j/WlWsJxNhNbz2X8q/o6eiBruRtXDwyQRzHXf4yrOCG3w7C7DrPv33DL+/R02779
+/2xXS5JluQrbSq6gwsZf1lPRET3Imrz9Dx6S8Dm+WT3KFPb1wRiEKB+eldfvf7m7
+Stz3LMk0vA+7WCF/G96H/TiDD7uDy/u6BpgwwIEzwAQcKl0OMAF73dcAE4aWR0Jn
+BIR0PANKQBLmM8Cs6M/zMQD2sp8BJmAm3srNNYmJ8YzpS4MnVdULc4A5hmdwWNEq
+1/xvgwYYuKMXzXlluUj5L8MTVATM5n8azleen/wwPAPMa+AAAygC4QATcJd2DTAw
+rGd+WRhv5zOfIOL7Gl8C5zR0lls2HYg1vN+6ppedeiqnFzz+WM/0cpLjGV8+s0UJ
+FJeYpLPdmJe7azrDIwIUy86DA8PoObDK2/hPXE95F5Cip7GyAMvaB2K7Pdu5blS3
+OG4DRr/uXPXCCgvDaIwkphzAptBM9OsDg1HFzK9h5nEs9yndgdNNSkLSE183FuD0
+TmdSa24WjnGuSair9nGtI2Bdv64MVK31yJbfDCO+g68vrQ/pYOjbC1bNga8BQ9U3
+vcMs8mlwdbkqd6vGsT6dMGNj/oZa1VZ1na0xl+IlvIc5oW7Hl3vXh9AUWpll/ex2
+fcw2L2OWOkmQGRU3qnm1K8O+QYbv5bqLtWKxDBlY2fjk5o7VFO0JVRi9z3a6txTc
+dtz3LGKtW+G68XHwlgcpNqvdq8P4sGY9I8mGWzw3c9iMT6MjnawJx5iEHzeh+NrR
+Ica4mD4MqygAmFICZvBYMjEASB/MJef2r9l4VaXtjrSlZOUzBpr8elvKG4eb8AZK
+J5CorBQjanlPDSxh6Js5OxjF0JFdHWjaDTFMaTsNGKcq1/vkMIU64Om1arZquT0b
+5Nq6mA+NIHJ8ahITJwYeGklytTVNL7py7Ro4qp4gIjbVWxmEWFgajcC3Accm59vs
+uZ2GTZb81gNIcxg9xwNUjl4rP7c1mPWevjVtH3VyfSQc2r78IF3bWn2XI2ibcqez
+TIP7ByFkX2z34HubGq6+gGwySD007gVLP7tpCJaJG/PHXamKjroxKJA0tqT1dmXa
+oF7nExXm1sY8BM/FpgprnDbrOIbv1/AIjs9UZr/wSmZoqSQTdibWN/FKLohvAbpo
+x5AT/nCYUiawra1+ZVzuyVmYUgJiwEyo7S5dr/WocpGQQNu8TOMUBINa4YCuDDRF
+SpBUDwS71tydhmZcHoNlNkaerTivunI3hVgEXrdam7/1qVsgrxLK7Vbmvc6EAaFM
+QmZ7wOeWZWp7RjBPq0TgEH5Yt4xliXXUJh5AwfnbAEXNaojzh1TzZRBtDO9pmG3d
+hrjvqsfA8Dgpsmd4nItVDOZ4EqUlw9M16w70gwsVScpjiIdCr+GPJZ4qj96aW/ey
+86oSoGPKk5oSuSphhidSJLv3a5XyHtrVFfeWYnqeUPauuVWBX53iBXT8kfekeY8i
+6+ui+TDM9BU0H9A1dpLnfSoJHp4PQ2YgOC/Qri/PA4528bwvCsBD9AFt2MP0Advs
+F9OHIZUymT7gkLQm078wmf4Ykunx6zEfpsfpEtXJ9GEgDSXTh7OeGSbXp8YiMX3g
+Uea1Sp0hpg9U+0X0rpo4RB9wzfUQfcCx+kX0x/AQPX4gcibR8wnsIXp8bdeL6OHM
+y/Nw9KV5x4BQL55HyM3e9QjZWg/PB4QsvHg+/qv74Xl41F6ef+Dh+TQcnkcHrA/P
+x+Fe98XzYWCqJc/DNY0U5Hm4rho6RB/Nt6yb6NPwEP1nMovoo7mkVo9C/UPD1nSH
+STEQ43OqBTMQJ4MpQRsGqHAYfHB95mi49g13VsNjcDetu/Sp5+maguo6p++h1jU6
+16k1oH7Hl5UYW6VXjaIMBlPxnfVm2h6qA7BXKeXc3F2uRMYCji71Zu2GJPcLt6nd
+XfwEUteXYjUaF1MWBoxmYUCiKsiEk4WHa3P7GE9Y0NVifXOeZSQWfz8Vp935+822
+FhDzmVEv6ffeeBeo1OtN498bthPV68mRBVbiBdpLcgef7mIFJbhvA3TzPAbAwQJs
+m2BJeBVyQ2h0sh+KJD4IiEEPVRJ5fMHCzvgYokqi+z6/jioZ4z0dnKjT+W0IdX2b
+fgVJN/iVfgv+5sUwaaTh+zWckvgMBcaYZwtodzI6cWppx6Dw7P7seJyY1BOANqr6
+aDrcC/vqWI3Lfac+9xciHPVsl6FF63h+vTjHPoev54b58UUJ9LhGJn48J7rCQfz9
+86pKjeCssk9AQjqOaCYdmVe//vcPdtTIQ9EMJr8/NKg44vvVyOogkPAK0Jqg8fXC
+UIq0J96vRjue+hrKMOAwOxDb67Od61GWztcaW9A2T/eiUEeqL1GiIRjO6kZ3HvOF
+I+rEcnsaQGr6tdovgofTJ2U55gY4E1+ffEgMPYAudkfE4Wpv9UD57uVaT1WMX1fC
+MYZOt4zMVvsuS+uuxB12oUoefnExtEi6Vv2HoTESq8pXjp+dPAaYgTF/4ywqrroL
+hUjUhrmeZdYDtZ3P9q77pqQvPD1uivG1U0/I2966nmVzfU761rqgq4+RHX/kF0vy
+vV73kSln6pSqYmBj4SOfAWuXvDZlmSeHrHPdLWFVtTwlu4wvEw9p/UDtrmu86/Hu
+S3RgCu2uTJPimSUzhT746yQNCGt+/bjJb9bTOBo0lOMfYshD9KKOX4zwckkbN0IF
+Z1SlTdRmlsyEc53HJNkDZsF06nIYzP1ez9MKkAqJX1YKh1NNpQ3qDjily53llzDn
+vgtj3uOPu7QxeigON8mZfjyvhZEpfAUMl4SV20FNDKOpqyBMkJWI1uCbw8AiwNAg
+yMiKG+IvM2AWOz83Su1p9euG3T/gHp/b99b2vpgTxXR4W+RNiQu8kXpWqDYu18yg
+zpt4OUj3Xlav1dG0WkxRIKp23qt29pbS9QKuEdabHgxtEMm153mwQZLppsLIqEfz
+yajW2qVI82vGVF7Pm4wksZrZdFNc3GTvm+LGr17f5XCjMipr6EG8+Pk0dsetShXl
+4XBFi3T7osHqyd0Si0Hl3zeurCsETFpydUETtJE/N2lfdARCqb+h3c3WgdoNoXet
+dzVBnwmHKe/PdpQvvsa6wRWZ2iV/PSfprrbcTYxi6nmV0aeGUNPx8mYMucqZEkW5
+TyTU5ro+plUVMDLhru9I2GyYubyqtB9LEum+xR79pPkeIhdtH91FAFkVDzP9JhFn
+EeIRPMkqq7Jmwg2SbOxwTzpiK9gkr+CCwsWVCdQ564AjtNzUdhDdC7ZTOIlD2GhZ
+j6BM7Ts71imcMfPF88ubN/Mq3tt6AcCk0WnvOm6h9Qxj3U8yZcE77yXuGsoXJGPG
+7Y2SSH6R1hDLqP0/ryH8DsM3DU1Zw5a6OIHhCKGlMCXpY3RUtlMdQHo3ebMITe0W
+ENtDthe710eus//G2Ghq13n6Dif1tTW4vvSixg7zwLbq2S4DFSpO94v2w5luov2Z
+2xWb7dot3UFFtaX2AuXRIXPdlV2bYcAIgZ6wFJW22CJ6vnlMEpN6bpDdluZDtN4b
++TxBpCGYgZohThNNdzbq9UtSL0O4NLpOTrp8gMnf7qFbNFHdHn4i7u1en0tE2RST
+Ld5crsz2eBkTc34B1VSK1KEe+c/FPDsxBAYTx3+J0ks+jw+1pq6QLldj2unYFONj
+jKCfuoctpUITgwPmY+17uagLcjTLlXJi78aI1aG32WoV5Ua5kyCYqjUeWsUgk6W2
+xaXsLPJgOG/bMl9mZQNs+/9sl0mWJCkMRK+SJ6gHiPE89XpXuen7L1pmJoaozlWE
+Ae4OQsNXyNIlt++uSYooaCbdNoUPo9ek5eTpS72kD1RhMWvE0qWBmeYr8alYrQG9
+2ZQ5Z9firHTTV9sbaVmdj0K0v1w2d47fedtN+3LZEDOgTYr4z3NzmmzIOm3MEvRd
+5fVOpvhINsw/pQToBWSWfFHPv+dyRGFAInbZH8Z0WZsdxnRZVj17dUnaOWcpTsT3
+qK56yocxXTLjH8b0ARU/IqarnWlBcVtuxLya9YzPjpNr8O72Iia+bRcxXa50ERPn
+6ONBTBiptgcxOXARk/IiJqRgLBjzGQBjUpbDmI8UY94BMiZkaocxeUPzMqbrme0w
+Jq5I/SNuHDdkW2lxiZZGs0mNKxkT96NsFJAJM40LmTDjLAcyYeT5QuYeOJC57R6Q
+idfVdiATn2v1gUxupxzIxF7VjZEiXbb8QiYM8VCmm6nUeSjTZa0PZLrudiHT5Shz
+Q+ZWBzL3wIbMqwmZkHYhE7LNBzIxMC5kupyirRVbm+1Ap5bPNd/5pe5qhXctxU1Q
+ZjHCzKZMyDUOZbrMgT8Wq6kPZWK9NkfKxOvbpUzsxcZDmThLPpTpqt10BCP3/IZ4
++WXW3vlSL2a6ZAE7mIkBvZuYCU+fhzI/UhMpc8fhocwbmEGZ8P5UN2W6ssOYSEf2
+MiaiXnTelVGmWImMeaUY82oyJp7VFchRWaUPY2JASZuMCUe+CIlNhW/kvbqK1fd8
+17lXZNFI8YGYiHc7hIkgsEuYnxZShsd7lYaXjBYDHihTDuGFbfykzeNaeqj5bBXH
+BaJxvjMW2i/R0ojDe9osATGL07ZERFYpWwR1GLbJ2fd8p638YfomUom+HIHVcVnY
+mmaxCjEMetrSg1iQsgd08j8cSKP9MAAuz58DY+iVsDIy3jBuHoDQ1K9q80M4hyYT
+smeepayso0/bMkyT2zufulgJxbwAdosSuSLXDa3lpFyXfWXVga99SY0p+/e5tHu0
+j2v/Q09YBEfEl/vwNwfaVAlmBnWOGVGRK6VFdQ/Pc2gyZQLG5/qV8w1155rR3lQw
+FYp33moV91TKLKcfsRgkBmqi3YHRSrf0oSN7JPMY6LTnHw7UnH8eGLLHJMI0ByQ3
+sgY6Et3/Bm4cuL16/nFgf+U88tdAZc78GKi18NgEfvRMMspQ9Oa57TDV9+Q0ZFRT
+ZukjLN62jBu5A7yxSESscX6f+2Uzlkdh0PW2oNbRv45zgKhKLH68hQ5kyd9PJM9e
+ab/fge7xCVkV3Am9qsupnsI6fdQ8BavAd/8yZBvtNEPmuavXp1nyAfLknW/KWxN5
+0lCSlcbKXr6Mn+uAevO6pzzU8qIMOLFGY5htOIHtIZWY+JeXhOBusXQ08R3i1uVM
+bCfRZj4SVtFyDng8A/H5tJqJpH11OXNV+4AvC1YLujzuK+sSn32jHORYznOhLsoM
+a3OzhVVUvnsYpZUdw7LoPDG5LX5idt/Ino/7MpL6uU9shbl733d48/aGSvP95S0s
+4D60FrNW9iN+3wGQedUH2WdNuYfJfU3ejt3JvXPj9BAEIbAsmFlSB7ea3/m29GiW
+lfRZ6/vKkrihTV0p4o/bGldq23EHGkD/c552h8kl3q6qUea+hKXMXtuXtlaV9+1u
+XVLnFHVrOk69Hw6r3JfTZvvTx8Da2Ie9iQMG/jk18PsOnCxn/nCyHwcCCKyR1zYQ
+uBx2eMDVVJkKILBOMN1AYCiWt+C7rPnwgZYPdSGaH+qdgghcFsFJEIEPVOXkmG7K
+IUSCLQ8S7IFTJv1rK/UfBg4S3AEigSEb5YME2K3NBwl8oJd+kADHi8tl6ugvAYRx
+6rjz/toDBLDrsAcIDB3jOECAa1B+BRDsWzpEsAfu0T6unhm9OpGkrJCt8IY9gELM
+h6pnMWUbZHhXPfOdbnMoEwLIcSs6EnUHiLLqDwUhSNXcQmGxx5DdWW87Ai0GVU9F
+sKA3I2T43bY0H4A90L5VR2LmvNnoE65VFLnW6TsL7EHuLlNSNp7ah2c1BW9G/+Ny
+JFJk6+3Kzl5Ly2PAuG+XnVfW0au5XIn+t8aKrSTgrbstSh42g0PDDbVxtKzDd1Bi
+NW/IaQjhi2Pi1JNdFI0wCqT1FlbhXuYv8JgbFBTgqrnH0d5XyuDofM482pVKKVVg
+0cmiqLtsg9O2ND87P0U/dTfJnTvLamH3wGBzI7+ZOluT3+CjrjqCqKrWuRwq9b43
+Jiq3RNXeUdQhUQn2USR18DnHnfc2NPFbMAfNVigtXC2zo8SthNGVvMo4yuNu7iuQ
+rihIfHKon2x6c15qGev2yyVqGna8GJ4X2y62ZdxA8Bnn6y46ueZ7BZWUHVew6Mhm
+cQWLb2ct/wxd5vY6GAFEA2M0u72X2MCrmqtmCiJ0vC5tcjXqy2+uLqYegtHtmXdc
+WKjIg0diuft8bu+8KYRbklTTC/tpdYq9AWsq8ifnF28wJM4e7hS69sKXpZ53A8B3
+q7ZPFfuKPhGXhm6NGyOtZS6OAuLmEmPDSsyTrZLZqSPi65ck8wEphJIdYU/lPN7F
+vDnmWWI7g/DK2c5yNWpTy3F+5my9vauBzDlWI8Ex5WvvAQraWCkhtDRF71F55ikX
+L0kGQzFmpdiXFWUycZu+rnNXK+4uDdpkzL08qfTUojIQJveoiRuoKuJ9xOdUBce5
+kKlKNXO4kvo18hQ8T0Ax674SlJcz7/lPJD9apSwi/SYQ9IFgBqNN5yYkFoNQfgO9
+x+rAxqLMNC9GNq0vCsqhj6EfhBRgYEAQvIb21hbdfMXeei1banm0TXt+BDP3kH3I
+7ffyKUpl0Ex6CWQ8vNQdWqwVvA+rcZDZWUE788Ok9RGATRvt6hOauj/sXKW96lP2
+oD5sXNsb3Z7a83rnYQ8+XHiDqyuV5O2NS5lFiWaKsk5M3LTERiBiEFewIlEpJnO4
+G3LTUt6V+0wmwsnE5Zkoq4Sf3DLUMsU0sgRyB3P2kZGyjy5J04OPZv1XW5BO1KDA
+8DIVrSnxICvrmFn2h4wEOsY7b2InBZnHtngMjqTlXf2S0taQq8ARdybZJlJ6X9uX
+wG/JI9xjyi3U/ez//oMVzcFHddOv9vvVuABK3IvfFYqby45M5leb6eo+AJ7CgKdK
+ysX5jFzlcsF/JLHcq2Eadx5Q1u7Thfn3ebs3nzbv112Oevd25DirpXuNaQuceF7u
++XHu5fy4D+TYSxobR87WJeOkONqdHwIIPR2GuW8/htPXj1m1t9fqdG+MjPU0WXvg
+9FQ8T/9h4ND65ztI6yyP+X0tAi6/D8WA2gwNZCs/DuwPn0f+GkDQfWpD6EMOE8Iu
+SZFbXwwCRD1Jb6IuQUkukxyFPpTKitUzEzMzqi2CQwBsKGPw/szlptrNJE6nqkPz
+VRiKNFTIB2tLLUfX+MzPxLe3tCRnFz73WL7ksrj3Ap7Q62qXzEu0rRS5B+DBspQP
+UCJ+8bQmkYXw6karrdTiW3PSYVfW/Cx0uQWqwMbH2FLL25jvfFU4zK6nrQVP92On
+cj207rYDzegj85h7uX34AKjSfhzYfnQe+Wvg9demXbpZvBX4fgd6kUV7moIhWWwJ
+Gqzr0P7yySABJBX+Mohmw3Lf/hxbYjnCd77zASUT2Zi9hRglXA9dFp/vFW+PCuiU
+kRdlzTyftRnLGx/PzG5ebJFguv4uEiJ+tHIGYaJXcrkKWctGfyWMouUccA8p5UtP
+syKNZPF2yjr3ttn0qYXitoa63flsGzUnx/Io7KPr66lFS2AyShMk9DBhr6zc7DFg
+UVE8IWBb/EDCvpE9H/flL0/53qcx9hUwvG/fOvfyn6CpAZS0DBXQUgukHAW1X8Ft
+d0OgK3MRAqCWvwnEQiNwDQpuq4Oaq+Aq0xjSTgQ5z9gMUoeCZS0sINUtqI8JTv1m
+MC40+ZiZIsubG0IrYEMwF9LRMzYDOU4BIMAAFyx3Dw0KZW5kc3RyZWFtDWVuZG9i
+ag0xNSAwIG9iajw8L0xlbmd0aCA4MjA3L0ZpbHRlci9GbGF0ZURlY29kZT4+c3Ry
+ZWFtDQpIiWxXTZIuIQrcv1P0CSpEwZ/zvIiJWXTffzuQiZbfm15VJKCl/CRYbT5S
+55fpY7N/VYe1ScCp44X2iMnX3z+XQK1Bj9X2DKkOx7NqDbhcGubDFf3L+iNq0Gsf
+AStWj0dG3zDM+zOnvvr+9Hat7o+2e3M3q/r+3M3qdbQDefT/+IL1zL6+rD261tfP
+K+jP8N9+/6m9PFb7r4JiIwWjYElT95fDOXBi0+lQniLQ9l7jjC6QbthwDOhbbQFn
+U0BdtiHNu3zoRyMs3H0aN7Oe5mtMCEqY10cKYK+EVWFuss3nXPSg4TK+/8T+Rbh/
+nwI4DdCoFTo8jjsZj07zxmj5lQPVMRLRuKx1tOVZSlsxuk0QnNV5tPL0gaNNoVvx
+a3t64WpEwdzpmuYQMJLf/4b2O6Lt119yR3sL3uC2x3jk/xM0vycFO7ryFWgVwG5x
+CX1KxrbyEuoJyXCsBb2WK5hRYusOtrpP7NWbr+Z2rQJqQzDMpWHuzmjQp3o0/Ku0
+8UK3GTOtITgeckH5BZsnh3xgJqqjOIwHaDScfXmNRfiK5Nnx81oHtJ1FUpcANl40
+4PaL3PoyssoXvaqZOGABF0yaR94F7Av6Ur9OiDxRhqQ1BO/FPuLOVOigs2AjQyYc
+PBHIoJ0WMHI04Ioym14V9PzwXw8IZlx3IInBfQ0wOI+I1q1+qDVK+l2s85/Nzcb7
+c4ervUc7cBxr4q6p5uL4nM35oTl+zs85mn/GOCcnojUu9qpx7b043fJuftzGnx+n
+8mi3z//++S/t17wLMgVv/cVW9ovgDe7HHgjuKE8zxHv0Ftu6IDLJBT0o2mFp4Arr
+cY1FEnQopOjlNS7Ysgv0Nitg64AqB9JcZL76ebYrhCN6UvybPp1+A55uLuh14EJr
+thd6gbS0Tqy9Qm0dsPfBzdXYnGuaL3ZA8rcfbQEKzBed57wycZbwVAOrmoL/IQCv
+WtcvQnJCHA5wkYTrXq9g7e71eUNdH5D58QrmpHldbDaVuwvLWiStg1tAAzz8Ulij
+lXmM6kYMQjDyq21GLSjC3WDsPMnL7iYSdVHGYAw2n4ygDThhzG1uHEu0MunS707F
+GYbFrtxH/k7BjUN3QjXD4aLH4awDeoxUkX9lbMiL17qO3qFx6BnGmAz2lPg3zVdl
+iwqnxsDB+WS+yEPAtroFnl+YXl4swS4BK2AQMqASsuWEoKN0VhDCyJbu0Ggu2jek
+ucx+66M9BOwJhzD1t3kTxe+idAJO/j1X64RWWloDeyE1zbu0jlLq0RBje54GbBRn
+qca6nPsuhVRCUphjnSoOL+u6qtwFnXW89da4tgJhjA0K0bRW7m2d1q1PkkAWxktQ
+oMJdiYpM/blLUzLnPE0izt54V5ISpvSZHKYVypEss7yXwlmploEhPqbLC7ZdPIlr
+oboKWIXJ2oowBTbdtckgGrc2jhFLyI6dMViyzQfZc+vnoj7dWMrY+ZRFL7gXCAyl
+MJiP6bjXSxjhRxRl45iicJwXfeekGLQxgk4wHvQWO3iNz3pTmLErbcYz8goZMWDM
+mwlpXl9teP8QXEC04jPDuqDlGJTmwko1nCxhNNWa5ilQJA3CE32/CRCGvRgi0ngF
+SXh/QKH7JOw1Y9FcKq/R24Z/6ZX2qp2pCjZri3BGMvkIMmgd77DKVh3/HqzLgesc
+GFNtT/MUSIyesZo0b+gKYYd0WrryLBq15fcbnSFRZMjsvEkd7K9sgeFl1nlk0Pef
+c9dwF0Pa2QR15XZM2FksfycGDhRRHId1pY3pcaUPSzGGe/CxuQUzyoxDv/v0mwK+
+qhbvZ2xj0S4uOLXn/3NCk57mwijHwyeOE87wIamMfdxgvcE3Q1xnMA0WEzIqlYj5
+mHNravHkcsgkkIY/9T53znAajPcfIE/SZNywvOac8efWT15b9u7oUppkH2NFTvk8
+i/KxWGUffGyYhce3ZU0/lJYNnF6KJyU203RL41OrKb2cjyGtDOMVMxDDdGpZ50X4
+8wo4DH0jbeMt85tAOzqKL9FxyoaCMeYvgqg3YO9BzX7D+Y93wT+CM9F+Hvw7UvI9
+uu/Kq4DUfFw3lkTMffcx7gUYimfdZcUXz8aKJvoNgVpnVTqvO+zoH9VjHmhyLmBN
+zub8jCILngiIOdOhNsCo7IQ0D4a89Kvh3zUGv6m8TbwSEWsXtAKat3gBBVzYbkQj
+3zAepLLNKYi2P+PIZHDj5sXoGRl5lkkCwMUaagGcnRcpumFelBMw9RWjS0D3Fdwy
+OTjmRatXLc3pw5G7CT1acvrVNO5k8Mw2p6xhvwji6TchcMeW9SmopCJbui36+hSs
+0rYAvuL0qOmdzlE1BgD6ctFdE+4J0uR7QG5UOBRtgQfKf8rFbM8xisbmHPcm2wri
+KpyKFM7Ol1JUfJx8SSJ6vtZ+aXNWiy4efu8cxtiBwpWDA5jR8cphrjdm75v7JAcn
+ucmONVlRXkr5ukCcBw/jbCML0Oi1ZqDGOTBQhyBa+QyyhlOFaBXbMKy9z4q8+ulO
+J7cNwiWgiLXQH+eC/5Cy8fPF7mqYIQ+MdmppngKLx06s7iw+3N2foxyyJyeBmY+Z
+HE0dnmaLe+jde8MtfJ1S3x++bKcCdOHsVhiE6KzkjDqhFw6s3fSFnjJ8Db6Cqdy7
+dWVmcvVgqpe6XR7PgGheK0MSs5fPQ4VOrByH7FyzC4qpkBHdENuNzhBNbqbwKbsv
+WrTlz/jkwnTtCC254yH2T/JgZgjHcBIMP/1QoJNBq3QcQxgPm0ANsNpIVyR918K7
+r5pNo37AeqzBpnhOxnGKsQjzsD15/NyF3C2TV48eGSP1ovncgG4rbb1ad2sjz3cm
+UxJ7O5kdM150kUwmYx/ovd1w8cm1BcbXZkChk4wxbD3JZh+msjI66GViUo66UAYt
+Bq6EWZT009brwGQ/Kt3UlJM/p/PI7YL1XeiIeGPE32Pe/owoSGP5g3FenOG4cgIC
+Zyzhwyg5w6FVvTjDBb3XwxkOB5MKpOBwXZSxqqfKeLXRjfVQhsMlF2Msb1t5Vfw6
+20QyxoaHMV4BGCNW93oYYwW5josxXGCkTzCGQ8n5JDgh7tHmxRnhFjIO9cVn035Y
+w2G4+2WNVR4+6UAajoTvWpDGhoc0XgFIIxb3wxmORrGLMyJiHK7BGREST7TNGX6R
+Wu3ijPCyyMsZcVPOyyANXz65G0gj4lnaRRqRAPayhkOZ9bDGR/aANPy4iw+TJI0Q
+qB7SCNj6YQ3AizQCVz2kASiHNC5Yj7Uc0ojT8JVk+6xMgSSNuEuthzTCc4wSaCFi
+zoksacM9VUj91Ff27qSNSOYhF21ERnHi08woW4c1DkrSSJycgaV6OCO27vXijPi1
+yOGMOBgpBpyAopOLMyKIVW698gUAzggvtHFxRuR2qYczwovDDmd8BpScYXT7eYUs
+T64l15C3FnliC3xC4iC+5/9WPOdVrheCLwF1uPuMAmc+HZ+CaveKj2PgTREHIf01
+d9APdtUhW/ANi8Hxs3qq/I/tKkmy7MZh+zpFncAhUtR0HocjepG18f0XTRDU8NO5
+yQxQ+k8SJ4Bf33+CscJHsr9qfHdFCgPnXYfXlKt476LRS7pXJOCYKdMg8mDohW/1
+IRewLipCT3xA6XVDbIe8rnfdc6FE62k2A0Zb80NCb8IgNKzF7SWzcD4I3YubS2g0
+iNWvuFoZ+oPBQr9/GLr31bhtUuEimhxq2sqnag+aVuN6ZQwxOQG2tlE6Zo13edac
+v2a6lTysI7aj8OL3Jnip72Nxh18YI3+pWm6uHy9tLM//GG4G7ih/x4g686CzX/i1
+Y748Br+RSvwGfbDxzvCW67ucAYmWcHJYjIb38D1R4v2Q5/RWxG5wyiHk9hQ1ex2t
+JBhtBRzG7VNz+9xKlV+flDUYdh9Yh+3tNFgE2qGl4M6vT3bSvXtwKOrC5dbJaIs/
+rhRYY+Xuyt1jcbeOxTbN3dlIZ1aAe5ENTAr9pmyOFkl14By5OzCUtsRyHRz+Ii86
+JbL/i+kVhtliCJqVfpCQIKu2fEfdMN857F1fg+uCyneS7UyqzhC5uovDpucMUBCK
+Z0HrAZmB7LMbtxC6X4GN2qoYf62T79LFsyQF+Q7BolrKd4yROq0zOZj/mof5y0p9
+163w13EWAtbow51rytlPGte1Rp8omTwblu2mMHhMMv5G1h5dmEtkq7IzMxqmM4rF
+wwpHJJv5zklo26fGuQHSFjDGGFcEUtOpkQ0lM9NnQdbciuRySK016ouQ1bmbBskI
+pSqMVILwj9Y7yv62Zmv3pGEEjkiki+2KxPDKtHe95JwZ5Y5WRE05W2ZmHxw9tce6
+rdjemPVP7/k72tEi3/VoAH/CYHYMX6+h+rPcICUigcFwJl7jv/g0SVEoqx9wHnD2
+f2Kwh3zg7nevIqRWYyELxaKFuPH3O0ayoUP5EwGV5ILUArQpG3J7G+1d7wmNv2bD
+0rF3T1KRLp6NsQKHeyjyWQ5WFEtgMjqfIdQ84OCR3ybkucIk6CtPQgcN/p58xrhJ
+gkeSrjNJ3EV9yl0vnD5rtA5AoRgYbW8vEnmwIqHXVrw1pMiBczK/P3IkWMwft5Ze
+tXYMN8pOtM1+MHirGpIGZWfzB/0GNA5yDRXtsFFVDBa8UIDggxF41Gb0qhVP9j4v
+fUNsZ20/6zXXC6Fx5sQEwu2N3Wjm9t54OpqPwzFJAGPs7YVqmG/xzy82q8Wf6wr9
+N+MtmDwaO5vmz0XCsCo/XzS2o/HhZXUm2g8fz2qjdJ6dXqs8qc/tJuUshD4CKMph
+qP8+MfATKCu34QiWz8Ay1t6wWpg8xf4ELkynPvFWb2BscA2t1+Fo7DhUXG5ok1IU
+pOGwrnmUqVAzX+UqnRJir7vLK5ulBYL2jwamuVv5tbiKI4nyLf0D1lH3bhrA44CV
+Wmh2HhUX9WDUta9C0giWwM3JOKAQ94Ks4IxO2Qo35cQz5DdhZWAsYSQI1Ore3phB
+e3uEDRX5oKZnNzNE+DVhepZKWIy0Lpo+X5XMu2MSlaWYLRAQaxtyd239XU9BIlPC
+Lasf2qZbRo+XxbAAt/V1ExLSiX4hI8GwohDBSF9haKtTwzCmk91lZH4UigYIz7yd
+CCtXePvZj5hyONl3Umy5L+q4Yszh0OwadHOwrZ+drXD5Y8N1zM7ljZae7O3CG4c0
+cHz4eg1IpYCT9BMJuXzQIp+c30PUh54WrnfyUeHpdhB3G4edvdw4iSmEIyDFunJc
+gWEl2XF7z4+3GXDk19e+y8gJNF/Cuu3Q6rgZGaUZLxq9GmzFnxaSl7Vxvbpr1vWQ
+2FvTLrdsvOtNSG6dvzahZKk7gmrULFkZfbHdMKC3G/3963+/TiV6CNxLf349pQnR
++xUpFEMh86VxXIJkA7TBkW/pqQXOINmdZny6TXkhtAd3pyHK0H+rDH7LfjFzmNq7
+M5yieRWOamVxe++6Ibczs/byZOaI8CqLrUxs95NSOAdOdhC5mfDhodB9WjyXxu7z
+LqL8aGeF7rTw7z+xQV1vVcg6T6k/F/un0CQAUeYITg04IIHnngTdMFesG9qfVu9U
+8etGFIIkEDZXd7o9qxDx8dMWMLIan265e6AX4ugR6+BoXAxcumHcO3cTz1zFb93X
+TjLn25jYeu6OswcTe18Mg0699ybk9njWXs9H49fzOmVEg6RTwmkj/j0unqjnD49H
+ZsPS5VFZ23BEFd6j9QfDIfTPbwShKzh1XkJ3bGMdQlcIMjuE7lBMH0JXb/wsyGiZ
+mtooi9thm/Mpfm2RIs96kXEYXb0Q20PojjvHnbiKxbi3Cf3CJPRrCEJ3ONQOoTtc
+4+Fz3ISqK/gcF29187l28srh8zDMw+eA69B5IHnoPAx66Dxg3XT+INLANQSdA1La
+Bp0jHlUeOkdEGGXZEWmXz5Uq8vI5Hmb6rjebh88dVlsPn7tBbB0+h9daO3wOn/b+
+8DkM86Fz7J/10DmcTBk/Mjus1YfOFS6bh85x+VEPXfvTlF9POld8x+76oHBOOndY
+OUIknbuhNTl0ruiph80T3SgQHy4/OKgciNQQVA649KFyNwwOfEHlgDYOVwOuA7l9
+mr7rMwfTqP0UbIfLYZh2OrhC4Y3D5coufLl8G5LL43PjkDku0+chczxFD5crKFAP
+l2+X7nL9rs/dgCQ964zj5nKHs/TL5QifHCpH40mNMRns24fY8LIID5XfqkwqR4LE
+PMPUmXKoXDFszYfKUQaU98HlSLzRD5dfmFx+DZVpPMo6XA442sPlOI3hDC6PGjzk
+jVsWSrPkcpSs9nddRQ6Zo74p7JLMYWCySXZko1s0ff54Kei8igvJ9jDFNswYmNxx
+Vf8qo/5gwKE1DO4K+uub4XDJ5ynBJRXDINs1z/W8HsZEBOB26H2gDx5xDGdG4nn8
+qmcShZ15gAFHChQNF7qBKpHLCAhpZBIaOWmVyGo3SIpfHbFeCnOh9Qt9E3PlGnp+
+TjgRlan8OptYZcvE6dGFumvJuBmnrWkjYBfbkA8tc9x19wMHqo6icDjq4sd5F3ep
+8HAQT8Wsx8mx6QuFneIaineAL16Og+hjUIpcsI3DqZFccBmuWwqTa72ORm7yNcbO
+VJS7h7UNuTvCttcxPE4yNmEMfxYUQ190YSMDNyFdmrDo5u+dTV6isj1xkusLDeK+
+zVbLfLNqNDBH3SDJvHR21WggK8+Tkb6tOzYawVJhEk6GUiM2PRpoQr62tvWuZ56g
+bdOVkSdlZZrMYFIQ6k4a/GUKPi+JIrbiXztK6k+8vzHPxmTNYp7Ux+A/qRL+7YUG
+8Ya13h3iBNs+f4J6y1O+vh8bVW0t1NktazfoGlnWDqJzZF2bN2CrT2G7oYuewjYL
+5b4r16HUp67N2eAaAPWUtUFQ61PWbmhJiyhrh1ZPVSc6RX1wz281SouoaXyafsma
+9oOFSiSK2mGjI6No/daljqeoDUkq7/qc7RS1of3oU9RusJR4g+ttjFPUF2ZRX0PW
+MG6n7ZvB7BQ13Ma6jKI2eHM9Rb29nFXtcLZ1qhYxyaplVe+Y3fWQu1nVeKz0p6qR
+EXaKGslCZomi3sm0q/ozuaKs7+uyrHFCG09Zw1DbKWuHIzWrzrzffMraQOvjlLVR
+N+2yjWDqU9Z+/ph615F+p6rDl/1WNVw96inrnTdZ159PYV07X5P58JiCZPWEdNnc
+3SH//oMtrcR0gem04v2J/RUtnu94UC5WlJTDNehw5HITUgD0QWSbG0LEGSW/w8Yx
+6f9sV0uSbSkI3Mpbgl/E9bzoWdWk9z9okgT1VtfgRtxEjyK/BFQrQPa5QNy96SAu
+N/bY8CZhE1/t4pFvgsHGd6K5BFTfvtDQX7jHzO2sgtUMAtg5lXV7kJ/efJCaLAcm
+KCPobrmua+3jd4Oj+epK1QfLN/qlLxesPlPgT1OOA40vE6Z4GXFZH/45SM1V5cdo
+xg8EZ2g+hYKOEoGvQV42c/AlFaXJ6Lzkw4sF30Tjz4eY9w3tonQJgoowX9Le9Y6G
+xD7e9GBVv1k5nZqgiG9XFEILCN2+fQkhypBB8db6CNQHSQaUFwBrB/300nzVCzYu
+Q6U0uOWEkzeT06LKVUVVNjhpYxmaMB6+5103qyE67bhOuJrDKNMm2Nvftiee2p0f
+THUMfAe1wfBIXCr3Fnevlu4nb1Z4tBg8WZdXgXszyXvRYZPerixMWGeeDwbyZsVR
+zzG7anogylqpiZLAF1VpjNuG+SEhsjy3h2Bi/sNxm0nYeZlyjNKRaTALe+c5IydJ
+2owG6TtQxI7sZ7Vtok7nYnYCPLGwOaKhtKOyrM1Rsf/5LERevTBfLXm6EmQ66X/g
+e4xXbMQxKAEOTp5kshml2QSe/gb58Vr6Ql01t1OwPX9H0pbPTX4ZC9nK09dimz+H
+r6uwu4zLtibCbvBrvcvwFXd7WFtJapzxauwebEy2V6qZfZO2/kJpuZu4C3d3dZKr
+4EC/yuxqMSUjthckZHPru+LqUNyF6OcW4JxpFilcV9Y5WKL790I7LfF19Vpkx/F7
+45j4Xqe/paywE31a935hT6s+LveO0NJV93wZKgSXojZflBS1jZsuQ21a5zLUthBe
+l6G2KSyXgrYrfilKihH5PusGRzsMZWiyewmGMsFif+IMBUjNnKEuDIZKQTAUYOmH
+oXA6J4hgKBN09h7OUAaLjstQm0XmMNTOohMMhf39CPxlKoei8DLRh6JMIGMfioJy
+4zBUoENQBzs/4dO5kp9w8n7oCc8g4Sj31vLQj/mjv+yEV8i7LCzDZKcdRyc52Zgi
+csnJQqHUS05quS0vOYXgktMmT5OcNie4JCe7S8dLTttz5pCTaYp6lORjkTT7Q04w
+w1mFvaYeapLq7e6lJhP0Q0wGgv2dmS4MaroC5yZ8q+OQk8Gm/SEnqR76SU64u/dD
+TqaZMkqCnLAul51w3DjkhLukP+QEXVQOORmcDFknp4SHnK7AyQlfl3rICZoWecgJ
+ytXDTZ6M47CPGXwkisgJtuAqaOZwkzmz95ebNpvNJCerKTovOX0UIWcnsRIWZdjs
+ZHVJ8CgGI9OtsfL9xOZyuwgCs2ibr8COJ5v21ULw3OHlUIT12GYA5bUhEH8t4Bre
++8HcFhGDKgh9aQnalQHffH31JyjVvFTfkFVLPr3rmxNGfm02rPs9fftIGHdbInMQ
+pV4BqTc3h6Dq/nO+tUCQfs8eXhy43e/u5KnUjM5NxdP1+a5YjUfnp2GUe3SYLK8+
+FqZiHwb/i6ERB7YQNffBEfQlfgMbMN42p9PN1J66xZAzuXeP02bkQwjpART/Zz1U
+5cdWv8p8D99szuNus9l41DoQaodNh4fOjO3R2ZfnbHua7gwGRtbCZJuamff10ZyQ
+79yj3XV+dr6GWfZ7+mLmx+Vh0NTsw+CefstSg20qRtvvKzjZZAKt/ReBOX54tV9W
+45b8IuChXz9v8QRcYEB/x/RrEyoqEpBworBS1Je9pjsllOrUZ4JaPKEKGB6Q/FPR
+2RnspSXk9sHtuT64Pb+e8uN0787v7bLm1e3AdXYTy4hlfox4PYfblZrbB8kTvelR
+zVRe66pOGC/l9lyv3J5fu2Hu6cdwvD2tStUek3sCQqCv74ktj5WeXuzx/ie4jn1P
+cL+q9SxNWPHds4r2i4xgcwXgqJyLrD8CbJw+Bh9sgpiTMJwYKrNnI9uXcVOsdu8d
+TDCHvutNlewzHFZWLNrSUlFJg25Lg5UkiR76QpGzvXryNFP4ywV9yu+CRXOYOoUN
+22gp0PqL4JgU9irzV0Hecj75IRjeJH8Iht0CKFEahUbBzOGd+gqjrRGksHxdl28v
+QgexkS9yHDL3u4yaCYi+32Bnn88+E+4dFHS6d3aWBpTODI4zRXxGCwOoW0R8BBCG
+nCeAOtv1DCCMSPoGEPqDG0A2X7UbQAanvgFkAp9Lcr1aq3gDyKCPYRFABr0fjwAy
+GGb3ALowAigFx3GK9lR+FzCATJ263gCCfrP+IrjxYvZa81dB3nI++SE4AXQFHkCA
+egPIYJ9vAJngdBWLRh1yI8RM3t8AMoe0/S7v9gSQubO9AWTerk8A2WPKugEUwXED
+6CNaGEDjZR8foYyO0VXaE//9x7dYH8ot6NG/XSAzKgWAzwEWBFscNpLXbHyQEcNi
+W4SBRtEl8T6wi6LAt4TYLme7rxsZi6+j2QBsrHCbY50aO9OaDbOXwbXdQIJWJeHM
+CecKxI+b3gdZS4yxAoezgNsrQpfZfV0wYEK15s2lemph7twJ+VJdz7pZYgi/rg69
+suN06mJmZQ9dFtd396FF0PZeWO0rbg9BsR70y7UbFmEfgslihKYWUBgMQ7bD5XHV
+9oehzf+Vr2mNTSBGEYOzJ+LbFktTLu9Y97OtV1jKnmaHsqN0ukV9Xfh1H4TKFojD
+3meAfYFs7+PGnhFzDWQ2k0xx5/Ydq9PatbJbawyzzfFx9fRN2cxqLnNMaO4aeHIk
+5G40Z2fdTMWWtTUadqkzf9lhSek+nW60PBk1ppjH4MdLvI3cVrfGaQy+r+B0jdsy
+9IXsxEMAOJbDbgYEXKOyYfMyvS0k2PfsgnWLITbFKLAPrCw5KRDv3/Ex67CYeXH4
+FjbFcfiw8HYB2BzQq3zYBqrRzrXX2L6YFAVJgdOnJ6BaHgJivppJAobHoLHM/oCT
+sK6LoMCO3SEY1rgA2o8RzZtIhCqp+NpeO6TRap3MCCfA4io74d+fTvKg3M2bWjy9
+0W/dhryagvDb7ikAZBQivHBjZTmZTGmoUDyQZHK9c7vOfqH5qaUpQ9AjCno77Oen
+q7fSMntsLyTD5duRXRyNmqPZA/y9L3P89fOlH49H6ZwRtC44g4ydufaHYLCjpID6
+e00oVH+yr+onsFbZpFVlVKt/jAnhgTs3q57IsE3ba6ONhDyak9vYx9DI53lcX6Tn
+U9xpWhP+/flUp8mNcPekrqbc9yuQhjMkq7vXR4MqjAOh76y8lsasgikWm064Gtst
+dUtPiO2W2r2/63OSWao4XDx9tdy+mWTihrZO3uNo1u2os23q02nNBChFyFFMdAYF
+TZ/438UiuvaMrYuGnV4brPkdnr19yQthFG53wXCi5Neen6tQrRWEpDu2S3UBWjDq
+xVbnUdtSKCoJX2VBJrRB9Al8sroYfYKERebc2ejSnnfySXufxjb9kevhLfRJ9XoT
+qjCewtvR8mUsWBSu+udHrPjsuDF6iossJb+vALVs+IWRKd7mGOycC3rURYyobJ6n
+L/ve4nmvVudqwogdre+6NCahk4VxbY+j02dldF5Nn9YssutCKh5eiAo96dMySHa1
+xemaLExlpHtk1jHpByWxzn51J+Q79a7Go/PbMMo9O4yWdx8bj/j6MTl4dxRrpqv7
+UZ13IVju9WUWAIpxRyytAEdzApgeUxA0IWNYWQAsjDDz8ig2C5WRELttnhn6rrcI
+UMtkwAJatavn8u3FUt2/33YH4OIUgR7ygVJGbF+sCJxj/hPIQGjFiyYAqn8NUQVM
+DUzBLjAxgyQEA4j7TKFNOHCVABIwg+RGU4hqCzNwUFlaGoH9bmhgCOMmcykABBgA
+uPSI8A0KZW5kc3RyZWFtDWVuZG9iag0xNiAwIG9iajw8L0xlbmd0aCA4MjM0L0Zp
+bHRlci9GbGF0ZURlY29kZT4+c3RyZWFtDQpIiWxXOZJjOw70+xS6wChILFzO0/En
+xqh2/v2NARIg+FRdlpQgHhesCWn0prFfyu+96SUGpY+XyruRAKow4Nqv379cMFxd
+3l1jfW6s02DA3QGla6pDMN5zyevLBPzW2X8Q8Fv2hiA/+UGwIPivieQ9qXT+XMHd
+Vt+8+o+CpjMFOvAJy3g5nLQcqizANcXhGISX2Id9Y8M5Xw77hPpiAWQpGOoy5nNd
+V6y3BThbbKYj1RcTBC3U98BlB7ww3wDaj/J0p+h7k+Ippsc4rPXYXVkBlwIK4+xu
+D4zvmXF6H6FO8bW9Dw+Lm/RwuJ2iUqtmpR17dQUcQ3GTMVJb/WuzXg+j4mhTa/E1
+fGA/IqkOwfX1h2PD1+stZqQU/ae9h061zexI7a9///nl5pl732j43y/pprYrHjxA
+9psJR9NiP8k09kfcmaCZZx93sYNdYJ+Ya7+wxwonZkx9noLbdgv9hdsuRmR2T43l
+AosaA03UgW4zTzetHpCm28MEuiAQXVhniSuZURz2Rge6umWnHVzrZDGJJ+ly2N9r
+YHU3JKMJVJHbRBPr0rrDoeNCNUdTqqdg5HaaQUWxuSBoeK28S5dImC24izKC1tX8
+6i0C3mG8VBY/15dfzb/usFNnid3jLl5RcPiMZZWNmyk9YSdJ7RQ0gufsNm3PT0GP
+Mjd73Jb6RN2y4uYwLUP7GlpR9frE+uIeRXKHX+SgeBvcdpdRaAyOgNOjyPaSnbdt
+UTSJFl5DgrO8olRA2ZP6scQNsC8Ee71OtiLmbJFWCCK2TdCgMTmsPSnqR0SZRvGY
+fFwTlY56LEdlJXjGHSkHhmmmPtbNUoryQRSG5agXbac2eZR4UdFXBY2iDn57yG/k
+03xrQxCvSOQjEHsiip8JULjNQMP3mO8t8A1Pv/2yEzQaVMTGMrvuY36H/pk7ZzMg
+6ltAV9+WsXzXtxkjAkGBeM7YfKS2dnzuZQ1w4SyF2QvOaBJXsFAODAoSrqW6WGfy
+sNad6hxhTUjBbfEON0axsKuPTNeVL6WWHY9gmCVwK3PA2ZBxXrB/h2FHtCDuWJeN
+r7c3iQOn+e+op2AgyP13OdySXnip2T6taP8suEwg0sPo5jx12yugR0PCeGj3l591
+ezfH+jEjYa+sDdtegmWhsOLaWGc/ixoKlkEibH4EExzlC98vb2L2/h1mHV42PazC
+h4J3rdbzNDJnuWCFDzo22xQP22kF6vlwWPmuK2HvlQ/nFUepHpcJBEPkVWbzkOcn
+5Cw0JYiabBYYGhQlDNGiRMu5edt6iiquHvQndHt0eSdDoRzkqIUDtiDSpoVrWBBx
+qMGNTECMSGXfzCBHGOtF3tiPcmC2vHLUvRadWrlB8rxblnOdAnlPcx7l1p/R8cKc
+S0aisIfO/VhlRucVyUqQO2X3NKe0Hc14Z0Kgu84e8FF4UIuIoyuxsRnUWRM0xifq
+TYyMqFJAJ3EGV4vbTfQ8E3hq4kIb6zqCgcBqhPhNGOpOVWu9W2kacX8BHEElYu+O
+munXdXrqsEeV87ddOEapnxqPJDA/gCz8JJjoIX6dHmlisZACD4O/BMWP3GAf7P8K
+zin1yTeBN+bxKRA7xWEnPiUXMJ65jolJUZF7C4/wytY70+LrwPTImM/1OaKTe/l2
+B2p06rZSfSu28ziA+6O3dM+3Ex52taion/HylSG014OqluCayHwk40dBDi7krW/X
+4GJwBwnF4ELWLoLc5+BC3i6kBheDIlyDicHR5TG4kHMEeq6vaO8YXAzuqY/BhUa0
+7BxcyPv+qMGFvLroY3Rx9bYfo4tv2KhGFz+uBhe/SuPH4OKXbVqDi0HN9Q3E+4wx
++e4I8VxtUdAwuLjRokbl4GICNP4cXBw2rsHl+KAGlyOoYeHTseFrq5ptPUYBEzi7
+jVHAAAbEHAVoxxBWo4AJWps1CpB3911Un7KR1ChggqOOdW9mUqOAQQrSmKOA+WHP
+VaOAwUVSo8CBNQpcAUYBfE01C/juYz9mAT99zJoFDG69XN+uKnc0iLvvcGusb5Sc
+MwsYVJ2PWcAEKyIUw4CbefYaBi7MYeAKkvv77fr+JuBVw4DBwbuGAYMrK87+MHUO
+AwYpigI6mHtmVEMLdXjuru/g85gG/LH9MQx4WGQRcW7uIRMFCsPACakzDHyGGIaB
++7ocBvwE7Y9hwAW0axggJ1tc04Dfb9NjHCBvslTjgMGRBB/eCdZ15wFaqIa1br6f
+UvMAjHnHATf1mDUOnLjJceDzKWjBbLkTHT5zijnMjZwykB0fOWWwh3Uyp9gK+lyV
+UwZH5ZAjjg6aKcW9tLFuLXOMSimDPbpzphS34NeZUgbH1kqpAyulrgApha+lUsp3
+j4E3U8pP51Up5XejmzJ+98aPlDLBDC/EOvu8cjKK/UXrkVFu1RjqkFEGV9A9ZNSF
+mVFXkAlkt+H9iWMEQz4Z0t4rnwz6E24+HTtnPjEFT8t8MWgHPvIpvXaXl85KJ7sa
+yH3lk4dEG5VPHi6R28inE04nnz7DC/lUb8t0clPm1BXp5IJKJj+/ayWT3y5pdCST
+++UQoFhHGc9kMUjRwjKZTJAUPtbNUqKVTLBkv9nkhuZV2XRCJrPp4yGRTNaW93rw
+WRNIUkxEjQfIKD5rkMZ68FkTdO3FZw020uKrbK10P+isYaXnKukuNmuwBxePna09
+R3KBzRpcQePAZi9MNnsExSJNwDR/FgSbtQNlrgd59esp/yAoIubmmuNHwTmlPvkm
+KDZ7BWCzDtcsNsuecfKgsyaY2R6czrpJZRRddYPHKJV01h3Cz2WvRIfNujfPXiu1
+pY9is/6abGvOZk9wFJv9jBYwHJ5BmsKnfyAYWUPdT4Z61F9lQA0KM7KOZH1nfOQw
+R5g51xOu2VM9Bd6g8DWjwLbNeVjYbp7dx0YpIBWsz5gjKA8D7wzo6tbVhe+6tTHN
+5zMgSfb81GaOu6yJ5eg7i/iBgoAX5JGqMRV2VCA7x5ANFkPONfwZhELk11wNcPhI
+izc7Uj0mUS9GPlhG0LoVFJ+PsJF3F40iCRt45/fTNL9Hm7CqNdNG4c/Is4J8LPpw
+NyJArCIFi8155giMFgwkiVj99iJyBWbSiSThiSQRC2Ov6Fdj4Dl/C4pti8Lufwum
+bcrPU4yxj5330MFPjRTcPT7eguepdeX+pBsmGO1QeAM8L90w2OVJ4cXYWb8U3uC4
+dEKS6BXdkFXaWDce2bTohsG+n3RDLDLWpRsGJ18Gf2DRjSsA3cDXXHTDd18PtmFY
+orKBbRhcQ4pN2NVZnmzDBKtfNuKGWJfAGwRxKrphgilcdMONvC7duDDpxhUkvfDb
+8TcczAt8wy2nRTcMpWGSbhw7J92wu/cHn3C3zCfdSK/d5RWsDnTDX/pkGx4RctmG
+QQ1mA7Zxoumwjc/oAtuopyXb8APGk7wjyIpuGGp6ubsHXHvSDRNM7UU3DGrQEdAJ
+9yM/ubsJ2rrc3d2+LneHJR90w+08L3lHzFzy/vES0A3teGfmlyWSNU/7zmtRf/37
+D1T4JPowg/25gioFbkvf5C+BBVvjFEg8Eh4yY2+E5vRLeozEk+dG6JpgJVFmd5EF
+FUek6wakNg909f+zXSVZkus47Cp1gnwaSek8f1t1/20TAGUrsmNnUIMpjiDy51pH
+6WLiePwcsOeoqt2hg+hbce32rUnV+g3rmLmdgpH1JE63rY7WdHnJfthSl71YiybX
+wa/6Uz/i2O7tqi98+sf67EneKiEJUict0vbadHuF4VB4VNyYsx+O+g+hexwx2Dj/
+yREzeb1Mj6OkI40w80pdEuqpnoFrwi37TTu4pcwrK1HX571sSqzajHDxR73qKZY5
+R39tIlC4B4YHuqcHUjCWTF6Ksg0Ugvcmvx4nGhQtFQQHSiUNTYgikVBP3JibzvpQ
++hsZFg1QmF5ZsqHz0uVaHpbJdwV9KSeML+Ord4ErqI5mLj0Ca/CHn//JhuA/DF1T
+cQqe4lnEO5d99zfw909b7U6MKC213etZK1Y1wq1x1uU/KxodkCBxe8Cu6RaEEHCm
+MnPldnPeDmIMuKi6PpeGHPG4EGynCydqmNWfwjd3kNoH1aF7hcMvwaJ0lFHFbObV
+aiTrKO0q/i22UakM4XUpHYnSa27vIn5u0rTOTJmeJuH2aWlAlPScrmRPf8alY+9n
+mjr+OOvprU56cpwJVRSo6eucM04kRAoymT8iRckcfFd9Bm769woiZsfQ/9REyU4C
+svYMctxUTwPb5Or2TO2mt/hBGTrzY3nlHIKUDbMV/bjbiYXWmn4ts/Y2pZi/UIqn
+FyQYU07l6YiX2vJ2Ow1UyqzCsEdRpm5W35SV6m9G58vOsl59zqZR3rvTaPnv18Yj
+D18mZ8c0zDb0+hK7tvBoo5udaaPQ4qSxCU1un4opQ3FXkKzJ9ZYMFHQkYMmYymyI
+qaHNdz2Y+xADRQEK2NS+fXpuL2oomF2QS1vz07ALQSNlmsoHCu9fHp5rfRFE8kVM
+fgjAgwFtKA6KlHPV9SpSGYKV2Tj11qKnc1jE05UfW8XV9MhrfSavA9WBYVs9qa3t
+K+e/qfXtSie4Id0UhaKqBqXgfVy8aqwvgmf4eF39f4J1phGbmlmfYesI3mtNqfRF
+UMJpErgqcaebLG5gW5l0ucc+XmjqyiGoW4qjKgXsM9nSIJzKh6U6EwIb8153Mf9V
+dPtKn2d8YthVW0DTD1hZ9a0JNc+eYHn5VuruxsER92nsKFXXe82ZZhJa5c+r0g/a
+asqppu0j17delnm/z8N7e1ZNTcVEZQyDjjLbpFq8oJLKsd0FNFUVKzpNJ0QTGiO3
+U/A6+8OzcnaUlvnh7BS8vt28/pugR2WUwNU2JzhewC3ya0gRj9qXvm18RQhacuPN
+5dFfXwe01i5fO7rletdr1EDGgoPNBEwWPhtzIgQmqpzLLmKMMSZRmEvcM/FjnsC1
+9i+CaPK1fgoYpwER73BQFGSqqrCspabuq4upNdfbRLU49+DlrV5zEQwzx71elTQN
+XAdm9QwddoUQrNlP5NELCsv25/FRRIrX3EzBGwsfjmcseDzL/AyK/yiYQ7X4D0Bv
+7AIcGwKSw3bmJrSP1mgq6/FIQEumjzkgIEnrMyd4f7ZzvbM94bolWNUUdpFfo+72
+HHmhTJTppcIPenbgPKT2FRivw2mRUBBu3N4UJGulMqNNsdDB9ZUsE80+dG/7IL3U
+FXRaBq9fOlwJh8pFV8sIgalMF9d6aoZ296DaRm4WLm0x4nowLv8lUFPwKlWHJpiB
+OdRzYILGr5nVwKqeUmt7JpqA40F6mSVpyuWlZlh4eU4accfYqWxLpooIRLjodB+C
+DCfORP/9Dq+/4H/v60bco4irituTgvHPJN5dti6lHbJDBcXvi/fjGmZM1WoOC8qn
+UHONO9/iXf3Kt/5mqyxrKjxlpyWnGN5mWcuQmewyv15CWrWCqQ9lmJHcehCzOBoM
+LNouH7fjte0SxBEMujkpQAA2dB8JwUZ9u49wBNJf/v7+LRN7hep4aHQuWvnFC08B
+dCBMCkCG1o2mydQLARteCDAVBeRUFF0UibuQon4gtodVa3vXI1reswG83XeDx433
+33DRq9dB/uwlRO+/Ts553ezkoNpdCauUoE5OJz06C+qJfNO7zhef02mR9+5jMP36
+saYUu43NKQeSNa8eewRPS+Vj6hfBU64/75BXw2hL01ak/L9XEOoqEpK4gFn/AbIp
+2l6ccOwcBFkDQ9AzbrS7rncwDJgkPAfDZc9xroP5rac5LMweKt9Vl0fzcNWlweWm
+OdGikDzwDIaXwLq2d/GLAn5xLrczqvHnWwVYuoyleo2+CdXVRgH10GrzXu/JODDs
+wC6u1Zq7VevZ8QP5qofM0OLqK6Zh5QhA2Vr6wLOKTN1u6jwYzfCvZIr9uEDXtWLp
+A9WjpXeIN7V2nm3rWo9ni9dWdMhlx4ZbzCjMthdLqzMvo9Go06CQXPD1wVKHXW1y
+fbP0T2t5ObtKT5uauAwGF/3cy3ymJKiuKQpQD23V7/Xu6jMuI8221LMeHwgPGWJ1
+xVJa6UoDVt4dNdEYylZY8EKgAXaiYQeaSguMAxsEVHGvYhCCXnM+3Vyvaz+RHrAo
+jU4mBLNa482ETUJDXjQIm3dRKG2ujSm6GE47QlrdGVz4hWZnOwVoaJxFVtT9Xb8L
+XIUi1BksdegCEnj1L4Kn2MBc4xs+/3gO/BKAg9inYIzGd/iUS0wmWVX+W+dZazE8
+apE/VA2KCVWRimLHH22Pe71vbQ+3033nrpXbp37W5V1zBabLe4yM0Ky33H2FCuvq
+jpniLUf/boEhFQJOF+kASQooB3RjsO5o2SPpYARzQAy5p9rsoJ7zKkaBxR3PqioV
+xrsNFqpSo61oYipzKFQb8e8aXDdhF//vc+V28gCj1QFBvoxfPkWpSFUgWINUZoIH
+AapQdLcbZmU4ApEUnWah8NLzdnoLRVjbzShoPrk++5L7LrU766a281mRMKa/FyWM
+Hg2bqIlZms+KneSVNeebqzL2k6rHF2c5PYU76uNIaEIqfvycUXyiYDCgfkUJ+3zc
+V7KGThK+IxCNBqyL6c0ZJ2DfjMWegd7CAeLXk8sYL2Ao5BTeknYbLZ8+2se6ufJg
+VhlKv84mCUOqgC+ZtSqnOMIcKM21OwWYynSYj/Xa8vJ5WHi6IWdTkAeoJiI9+6u5
+oN7Jl531fPY5nGZ5L0+znZ8/Vh55+jK6yv9kz3r51hG8FQ4UoX0VlOhfEjBqI50G
+7BU5qRlwogUF3CIJiCfo6D+aNMgSAjWZY7EGOShVIm2e9rFsyVfKIsy0zxAHpRAD
+YbsK7ixCYk2QbR5V4GznzKa+/5cCl/lL1fXWWWTKmoSzy/hp7VBWVLSatvdc33qY
+dKk7N6OBn9V4QTqSAWxMYXIhqRY6d7pK9c306yirRafpgig7Y+R2Ch4O/OlX1er1
+g0A8g80DOT8ACXK4CLinX8PH3qzgZzgJWOt6BpeArfVrsAlBb36vj3jGdXqs9Xk7
+OM/5eaB9qfZAfzYLc4Z4znLAOFc/88f5dc4nR7GcXo7iz3RzHvau89nndJrlvf0x
+m/5+bCrNLoOr6MXxO9UI37yKh8z1RfA69ToPl84S+W+q/z1uBCYfQAcIecDSNWkg
+DUr9ETmbFYkI7EODSrRjwPmSM8BxdwQIko9pvcRtVdRQEDSfv3ZuD25hXHf8NSCG
+J7QMlKADQS5yd+KBrsPD67A7Xq76vkbL7bsxX5CoVG0TVm6vdEfAtqRL2KlzfQ4k
+ugTMp4nKBcg6MKkbkDhMaef46CI19c8Nx/6ADIpLsJa2tz3FgXR7PZmfu2vJOiHd
+typgG0TeDpIPZrYLrXaVlFaarGAqR82OlUhkou3JBe7U639sV0eSHbsR3P9TzAXE
+aHjgGFrpAAzthoqQ7r9QmqrufuTfzJssmC6TZXAigGP5kbNz+3Ap7vXIS+F2FJ+I
+wnE5nCs+Z01XTz614fGtBJ/iddht2L5OQhte/YLzevWwMz0TAXpEGzN3Hz89m5za
+csC79oMQgmlTQsBBSpbcuNBhhH41XZzeCbthHXl8mqxrab1cHneGtxfPQoTeHs+e
+XPer5sxAMbXt3N2KX3hKHM5h/ngc7h58Sovdwkij1sOUZt0mn2K83sqI8lSlGmrE
+lymeEaMiiJOZw3DyqO8cx9TnLM71Ee89ZWXFb1SQHtu7R8oxvb0tl4DIi7s4sf7d
+acgonKhXkZclGMcSpVmLKUIGKWg7ylevWltBoIKhyE9AL5flYZrZ9sCWmRO4Xl6u
+DoKZ2q7m+GepaxlQXz2K419cGVWVDKOQunLm+j5eDyde102myHjNsq5dSIN6zMXw
+2uOjn6ryTEhFga+GX48Apg67DSnpVyMfNIT7UrmsHAoAjynlh9y4MIH5mVGUS5xb
+mj85Bbtn7TJmbI/qm+vHsJIqF7q436AtMhVjlh85g/kO2KzbYg49kGXO2yUYbl08
+Xf2i2su3D9WsEY0CU8FSbZ/iO3pt0SS0mqEoAZiqC0/Nyd8SiKoW6LaiMmOCw7Sj
+SXZePZQra7rGhXIuqI5sQLhz7TTdgsZreHpruhhhStHiukaa4jGn2xBtPSpoy5E0
+TEM+1vm0W27BQJdnnC3OiABHuzeHHcDlKWZNw+GhZ+4S24fHt4af4BMv3j+62TS0
+2seOjwmdeZNJo94giag5sxRw2KN9l4Ters6d6/DYFuzNcAzBcmp4dHteO8PsOQwU
+nTXekIzz9hBcJbYXtxjmNy8TmiWZui61nPvjinVdDldzrP1Y0vJp7qa+W95kl/fh
+7WffjCzH6Fj8LFzcDrjEzMpRI2FXh47tFnic2nrSqQ34Y9OFeN+GNtfaoYYW/kGh
+Nh/6qoHs83a9V4unvuboHhf8mwsrxjpSRpVFcMqDH6VI1alcP2zmwQW/bjw9awBx
+BuPELDC2orOnPgaBkpnEg7wUt8rgBqAXgzqlOrtyGb1qtucwBgSnTF7eMmX4aRjo
+t5/VCmitvTkE6uR5tntyzrsRjStV0bfZ6srXrZkdHHqn99OqWAyT82S45Lk5HJZf
+Tu9arbev1VJ5nd+NuOXXCzeSqXBic7G1XtcxxXZLzYZ7IokPGDP4RZ6HGUa2+Zr1
+Wa7O3DyMUSCq/s7tHJ/z20D70SsRtY69ysORey8n8evipZYcNNiqdot9I9VC3Pej
+tZFt7Pu9WsfrLD2y33fTYef+tJ2Zar1dTer/969zfU080jD+9laUqf9oSBzk+//+
+/de/vv6DC5nA450eIUgiskffJIXz1geFMfowkZPiKB70UyYAu9x8JwhK13zlD3KX
+81Oenp5mnttZNk9+mz1rPnoFfBLkFpiJcTZpGnc/NI5vJ81Ds8iBUPzOkLArVsPo
+PBpOea4Ol+Wnbw9bsQ+HO0dw4fhIkltgNoKird9MxZvS408SGdqNh+es5/3JgjDl
+yRK0h7GedRjOnpOnUT/7+25ya92f5svqpdcNM01CkISMw0FWX/1QOb6cVA+9MhFC
+7ydPaNd8r9dRn9N0ysft4bP4djg0FftwuLsEOnqr7zwIQfINc4Cp3wXG+mAqmut8
+MfmAL8+MAbhmfeVBRQ0dT54ATjbJOA14xuvyippgz+EPgAbDUCvhnQaPQGzLs8HF
+uPqman45qJx6Bc9D7TsLwqpYDZMzC8IlTxaEwzILbv86Cz7c7SzAhaO+s+AWmG2Y
+ONrNREwjfpQmUaFcf2XB+dFfJA9L7iSAoZfLgdZpt3XVYcDexutyCMa5WwWdth69
+HhhZkIIgWx42E/Pum6j57SByapY0D82fNDgenZ712p/Tcst5pwGdVu40CI9mGnx4
+nGnwz79KR/fCpIafgtb7CwI2rE0BRyrCjsGrN32YsOJqwLU5J5Z+6baO5xbGaMKN
+dgO4934gk7nFdgk4tB6tn0ME6utufgrz4PDd+DbqcPcLhXCztfFRw7N4mVJzWqXd
+0BgmGH8L8xnZt34I+SjhZcd3XUK1xqcGUoWa+Etdal1j3gg2tJ1qHZl4vHfAVDrA
+aCFQ8M++Rmw+dGdTqsndiKe8azjxwfD+z9/D8Y0YFRwdl+xYsyhC4G0pKfh+BEOv
+iw8BXwiCGN4B29yGIFnn6D/1UQhQuCEoeGYWmdJp0dlHkBSWvTW2D4d8Fa8vuIlS
+o4PoEYUDcOuQuxA7ora13DAWEU4MUwG5Gz5f41n3fM/DohPCs4e/XGL76XL+UYzB
+e9lZ63ohsp+bjR+3gQr77wQ4sT4xmUM4tiwZ09eXIzfV2kKZdRTq0JX9jpmiSIel
+hvYLZ/R7vesdSriG4KzafmI7E4wfu+RG/E5/u1EVfHLIL/3Y6bimys1TaYaq3RSi
+XYVY6GRWbB6C5eyvm27jR5wMtHfNzQ8bTVD8b0eUbYKGADfCBjqSb0+FaU+GCVXM
+xnB8IAyCjWmFDtQ1ZfrS+jalICzjUiswwmZgpfm9Ouq8jwLOaTbOFrv3dfzprfW9
+nMuYHm4Iz7SS2y0oTCTCpsxpsX3RrXCVCxY/180f69KLfLcQHCk+dPjsGWb2o1rY
+5OqDHuliWAlRyjHgAu4qQ1XYGkvWVbfWF/mFCqaSdcMoJDc+qsu4rYjrtTsC5aiE
+9JOqTNXO1bY9XoZKKQoFNWezDWg7O3j3Wp/V682G7qhPK0MkUyCo9CKeJlsUmvwY
+YFsq68O1OAVscpX04QHSm03Qfj0u9CSVoiBd+HTy18bW10qEuF/uGrG7hiPKDtND
+Oa0/ppeg6lyC16ixfSD29GsvX7fjoNobzDrfbkV/Xo7wsh3XMpsaHMCycSV92nYT
+Sv5Ut6SwA6MdYU8qn2r2LHlROYX6CePtQxF3jxNOVWKgDDXHQKCNF7hKbp0uHVrq
+09lVV4RO+XI9oaW7UICQMfI9DY6iBljaehc9ePdV846oTtiH3Vd0eKery9HyPkHh
+4xJYDF9Vh9MMtWMb4H04gELEge+jwkMwr/UWwPl75hELrlHeAmR3cbecZmPLbpmC
+aOLPkQ89vjHt8tBYKg3nsmosjiUFVq3RVyAPPPz9+xFV2slEqO4KnJkpuOx9UKNM
+BMP+Q2En2rW6o6gkQLC8e/WpdZYpQjJiBr8Mvf1atwA9HInhyaYWwTndoI7Ji8Qo
+Ck9ZXesqEWxJ8w3P7LFdgqlTNBfVY5e/Fyx5iAo11YXdTgiKh7g/BNdWeOmw9ie+
+v5EH/hAwGdpbgJHDHi492wlRvbank/Rw9fTSL+/ul4bhNg2Hp5c2c/vc/b2uks3Z
+Zzt+edmI7We6LcINiv30bMRqktyAbu4Xn2RJ/pBcbFhIuF9vwam+cg9VJA3zE1Po
+WVmBeGVF8ZO7kJdEZ+S4jmqp+fiZ3iGYc7/X+SQgLFS3q16rm6lWQtAufft0Lffo
+fkx2wDmLNRmxeyuQ9DzRsd7MzDIRva7DlydWCEqLKWZrvSxl8JTnbkivxHYL5JSh
+tysrlC725LtT6RNOX1V67OoI7/Vo7fHb22WVpzebrAwMk8ulyrPD2yx8/R4J6c/V
+nnwNfz/56nDkcoSKd5wnlBwYS7BDoU4qBxGYn/vrN6L8ZAXDDerKFB1w5+v/AgwA
+63GGYQ0KZW5kc3RyZWFtDWVuZG9iag0xNyAwIG9iajw8L0xlbmd0aCA4MTg1L0Zp
+bHRlci9GbGF0ZURlY29kZT4+c3RyZWFtDQpIiXxXS5IkuQrc9ynyBGkSQh/OUzZj
+b9G1mfsvHrijkHK6bFZhDgRC/FWHvNuUl/Z31fqqDnXqS/W9FtBYAcbQ19evwGvN
+IKhOZ7d3bT1gawaoMjcM8faeddx8G8q/Q7m+66zU3iHucnPx7Aa+rrRMb6hqKZ6E
+2V/82V463qVYKm8BRUbaYmMGQVVeNG0F7P1YDkRhXCy5eev9K51yNKfL9smPQ2e/
+Ea3+2+VdsPu93QIbr/IefYpDN8+p//wFkfleVf5TZPqpYlvk2wn1PcfGvx/sV3BN
+xE3sB8J8l1WvP/7Ebc4PPMYLIKLnlm5oPaBZC684YYWD17u66YAjkDQiI7PRh36b
+UlsQIjMcVQXqZoDNPetwrJ7S6s51whzkj1EDhtMChkkJKW7tsJvHibAMwqXQlYZ7
+7AeVL4pL5E2cRdgWbJltiyeheq78BkEMP9RJ/ZJ+MP5foVyK5O9lwRrZxgn5lY6w
++UBeZc5x80cD9JQF7B3KSm+PoxAkU3pGGfFIzYPCmV+/LsJOAPdkJO8fhJND+csf
+hJ2Gf+OKs887U5Nw1HolV/2BMN6taxK6IZuHxkXVo1EDLu2AVlCg1tEaZocHQqFX
+ZkCpcENF9nUPmWxI8V4++EMISwecSmXp1e4JhlwvheJmaBaGTB+uBT5adYtPVqII
+feT6YW2rVN/ZTxryrTNk3myk5++NnUxRaH6X5HtPCohsJfyCp2yNw/dm2iiO/PJm
+a2xeo6V4HxCvlY7F6R0OfeLgBmtNcRBOl/kILuNt777Q12tjvO3dBn4yC6WGUeNw
+WRy5vBcACWbMXGivobeDG003oABVm4lCdm7ZYE5WasPomkwGt0IKb+q502FWFwVf
+DcPKejvQvb8sxZNg1DYUbo4hA+WKGO+O5GcjX80qLBmCTonJFVYX3ZBX7DHoDt86
+tBksN2+S0D1pifuz8OxBNkxxw9RuqFJTPAlN2JAmWs8HAeXgRVJpbYveGzU1ANUw
+tLOWZu4AEfhAJhzpMmG65AYgM6+GeB3+6Pi7QffCaMcGMNLYWhaD0nGZhgVh6Hhd
+ibTK3J44mfX71//uy03/fMMAEyWBGeqEAn+WRmevmtXI/BojA7tDUzn5K9nhtIAI
+jXdWsw3pnKXr8N1Xyj1CarqSbSyLc+J+fnbPhOXegAz8uAk2heWzhd7LSnJClg4q
+afl4qPOppBWLybxKyQnax1NLDjGKWC7LZ3D0lV1LjrcwuUV1F9MqPu7GVUxOUFlP
+MTkUbo0opg2fYjoEozZlhqOYQvldS3E0I4RiCrtKe4rF4Vr1Kia/VQyJh+8TkhMJ
+xbRi2vermpwgqz3VtBrqZlfTgVlNh5DF49aaLx2fBD9pV1NYv+ypJocy26mm7eUs
+J4eztKdclkDJKacdssNXFgjKKQI+rmpasaj2p5oiW/gzyulkU5bTZ3qhnM7tspzC
+nVcxxYm5ZTc6u1t7iinMY1tsj/VWn2JyWIs8xRKRnP0qJieMq9jCVUwTFBM8edWS
+41rnU0wna0qm4HURVpNikVW3wvOF1WQRCxJ+UyJGhN85huFSVnNsgB1wKQpmFcbS
+Q85g9PCFT+JG96I2E3omtZXSIISBA/zCNWHrFqwkwmL0o7mxoEK87gxrYGsNgVwT
+K2sWjMIzgLzFxMIpI83qWFYGas/z1jiauMw4YTS4cRSahXnuVuoFYynZ4iDEG43q
+MGwUi3xAVIv3kAxS3wNbYrGHCw0OLnTwFREGKdozJKLRfR+CG+zxjOuZ66g/EHzh
+spoEW1zYIoxWuJBNZF9AWZMeQBidoNzvxPMp4KjgS+yWDldZG4Z45OE6/IoajNVV
+gKyjp0++JE1wc6xz7gCHjXtsgS0Jw70zxRuHb6h5/vaCjsV0a482PNOWMdikeTgs
+YweH4Xb6e9xL9OJqpIWjiLPDVvJNulK6Kv5edEIp0GxhR3gcVhV2YEKG7Pe/Y8i+
+QhGvVY/I96+jYY6+Q4asGpbHjdwDMoADO8Uo2zjuID2v0rjpdfgsbrY2pPSsH3wT
+tNzI+3BamcKuKDu+gh5sg3xhiKrpDXU94iTMkXzBFLcYydCOfIgtncZgf/I21haM
+mewCKmm71Q0p3nKbT74sqNPCqNWV/XbHOF5QWzsKAV6PIH7EAMVmDYl0XmabsPC8
+8riYp9pYPxA8wXUTlDuNxUCzCBLfdkRzcdGtmFAWXiUB3nSvTjQnaQLIyg4U0v2d
+lQbuYFb4v0T04zJqdp+wcY3ZwO6dfo65vqGHhYN1E56HTJxt+gMhbJCb4GakrTzA
+dQG2yVdc5mhnRcXDa/GmDJVOXrw8iH5Z+aqb6bedFnacGmG1FFfGWZT8xqBUdLCM
+Uew0NcUl30l5t4/Ax6tNimdiee4fAI9WD5dPxH/+gojvQhXDoEYz986wXNAXAL94
+Snha5JD2dP8GoSsfr56CAZVOifdkwMbVPN6oXxAX7nrFiyhg5WoXj4wL6rAUT0LM
+Q/zNMSmFsMcV/EWCt24QwuRIZonTJ3crzM1Ao69nqEpZ7AvJdSUVqqOVBByx5C3k
+HKVtQDqyQYrxneCGDbuhzgHxhyAtxdU4sKjdsAYtaal8dXpNBtiz6s6VgL3Z9jnF
+Y/WM5JipjUtZhRP9rMmFN33uTzBDtkToAoq7Cw1u3ZBd4yFEZVNdPuAUfy8MEKyU
+lsZovkl6Bb9wH5hl0ekyNgzxmMpy+AN95/k7alxu7X4pbef06Lo07aCOdh7CGytC
+FHsYy7sxWWqszuGWmaob98k4C0dzcMtiZl6J/uUjDiSFH/2kzP3J4e01hF+W5Mzj
+1YwjcFTZV6d4DOaArfNlMCc9VeqGFEfH2Py1nx3amRTS+xmZiAOn2hzCpOHuah7M
+B0aUJcWToJ18FbTPeElA+8hnQU3xWjOpaPsyvhJK2s4Vs5Rte+NU3PxYxfA3fWvW
+d+PNukUKjsYkSD9rBuZyOyOx0Daj3Lw6v2/CWLx864PVSSjc8Mx2zpYsZlQnqxht
+JRuFrA3zNqY3Hz3s/F36vLX79Yqd0+Nyk7a1G+rQ5/JzD0H8XbHMlsYCKoMddLU0
+RiqaxxRjFjESo7PlDXIDZhZ1OXy/uU7+zThX/r2yb4XneHry068i9fUvv39hIPgG
+wg4fvej7ECa6kFRfWHrl62QCNkWbrR1p64SYFE4Y0Rmrr1Nt7KbsqLfzDvLUdW59
+uIKm8Pzqbw6RW3Xjg3Ef3XxF02NYQhpOcRA6+tnzt8/tcZQrvE1TcHgszOv1WJZt
+bRv+tL19r83PW++/0ytHe3qNh2+XpmWfLkdB1Fz9nyhsQseyFHAWYQ0xCotXe4yr
+uVA3GC+adz1XAaLwkv5w/d79/CncOh7F/tihS3ms+0wvoxKeCDyEeHo+f0efnEd5
+1NHY2YBUKpW5Ars8/OvOnLqvKKoXt8j5s6BQL8UFr9fn4Me5NOvD1ygCz5VVPtyf
+hJNrdheBb2s670x1/91FoJhRO82jD/arCNhF/893lSRp0uqwq/QJOhKMGc7ztlX3
+3z5LMglfdcW/ypAZ0niU92o0KBUrHQWFsPtqlJN5fo1y4kexhMcFKdhJkKczCfLy
+E6b58x3GqdkO8lT8JEG+a6/nq/fptMq5Pa2WP0+TnvS8TK4kiPuq315IwY439Eo7
+0dhEPd5gDe32FGnUPpkH4zzf8maBs4Ce1dRUR4NSPOu6GZNMPz/uyvatVsLjg1eg
+eMvTOxrz8hOt+fMdzalZRvobPJkHeNXJknhyvY7CInZfDYtdGfgaWIp92FuJEN3m
++c8Boj6R4/VMpBB08ZoCEhVwjS5Thtdr+WsiMVQ/YJdHDcw3YP6uB18mbIKDza9G
+hRjJS6K1BqxNFG8tQnM2P+81t6s3opAAedWg6+uG8IZ2p8ARvjjMxBrUO361RAr2
+1YwmkDinppnhJWgWnzU30jP9KfeyFfITTH+AxbLwSZNnl5dVZEIG9qA3Ab2zAngv
+ud31uxb184uCMVgcHW6mC9RXu4xchiajVlI7k2AW47qPXZ+AxuinWsEQ5a1lsJlm
+rhFsEnA2wmbkQjV6cMkeDk1MJSFestoNre3tVR5bo3KdhulUCLeTSWmGlC59aSDI
+aCAqU4E21O49jWqbj0PwdQSK3K+fofzF6A52bjwElvV9C8YsPNRE0OKDB4ZcA41T
+iQj0yV6LyIISoR1LYsNm3wNDa0SuipS28LBle1cj5FfVTUR18J4+5cIYdjqXZytc
+d2daYWK6oKlqHEFH2cDpRjZBn3Z5OP5YtiZrye5DanfNAmCUgWrfKF+4xrVaqqgN
+CmnAx5ihVTeHfVybq6w3xY1ZV2FMzXP5SLyOaYN0gO3Bqf1XQXFlQpTLPn4I0ACj
+JqySAq8fgq5ZQgLaZopgyzYTMQNzy5KDOdw0jALWwaY2+YIXmllupwAP7X90uioe
+5IchHlf6et3KEJ1dujxtvv0Wqnu5+jECavq9XtFZApp88dQM7239KQ5L1SOYbKl7
+C16Rz2ZQV5jCdn6AU7EyRC0PLdQNLMri6v/VL+ztyplRR5AZZXFYky4yKlCd882o
+gG3YlVEh6D53SpkzATJpLOxV7aRUYLN6rfpYO6UCjd6ulArBautNKYvQUo4wpQ7M
+lDoCphRO284o3P30K6MMQVXejApYMu71xOHlZBReOP1a7aW8GWVv+CqjYJ8kH1X2
+ME2LzCgYs40rpbCu8SwzKJ7h4pv/CDKDDPTEfwj0uswgmL38ELi/GQXTqH4go+L+
+uq6ECtwffxPK1PR2Qh2YCbUFmVA4/aaTvcmW6USXzjedoNY1w0Ftm1c6IZiGnXUn
+5d7pBMuXK5tgytp2NiGI9zgJ+BH1zCabjLTDbrcgqXzAWQ/RtxUXtmsQCAErWQ4K
+AWdpuy+3GEBLPV078HzWuxrDkyYtHg3Y1rquDsFY/v464Gr1VWzDl99uQTL5fVo0
+f1/+TgH75zklbM3SyFvx44R813ZCvjpPb6u8t2+r6efbpKnZp8k5Y+C+p95eSEGS
+ecD5Mv1A3eYZBALnUzknhPJP3TPEfsmeMAJzWnlXZx33yaXKP7cHSC/zv7BZPUpt
+eHsgBSTy+3TS/H35Owbsn+eYsDXTDHFCR4WNj9pYL57vURikXTenufLHx7hU69PW
+TIJWY7ipVzc4guwGIVjPy6/C78+hV4Hq8KsZhKD13QsC9EOgkJca8NQLUBVeugVk
+s+xegE+/6RXU8UOv+D306sDsBUfAXoDT9tIrXP7MqxmEoE57mwFCu7/lPhAC5G0G
+eOEc12orh14FNDu9oL1smr0gYPHDrmBKv1oBlu1mVy0L2i+CrPxISp8/BM9NpmD1
+D7oFQ/thVzDNeNkVDCqimc0AifscdhXQu7/N4MBsBluQzYCnD7vC7eNmV/Tq2w2g
+2Sn/1LxddQjRNPxeZlnLZgDLF7u6AaxZX26FIK6HW31GvRIhepUsZ/GCbwqeyi7Z
+eANyiKbpjEAwRa5Ok/e6SMJgshIaU76a37Dl01/BHE3XacirD2Dc4qwPrcuwkdAZ
+G7Vwva/TTeYOnOw1GDvbtTqXGAZ7zfqr6bG5MjZqQlPtWZ3Lzvox+7pQGz03C1fL
+vaZ+mTc3U2RVy90mQa/aTmoaBuWTV1iqbINr+86WkdfJ/oUWjJ85c3ilwaPysTEM
+5laUxdbUaOYNa9m6DNWiMfO2yTrW8nQVEy3ZK2bUgKXoKbQhmeqOvTDWcxMV+MNO
+j4SBmp3ToKLtvh0W8fN3WCRH2guG7nWHVgoaXZSsKV5uCp3euwwzcvvQAFjY6MD2
+ZNZZ//yIc7ZgiJAcTl7wfQScLHjEVPRH1+sQxCjO0aby9dreXNunnmssp9Gppm0o
+25rbve56fWMtjzg1lf60bbhCbXv0yvVs1KvbgXB03dslaJ7rKGDRCVrX7Y1PHavs
+XFkZVkoWE0F5HulWVD0B9dQpJq31wWrA07ItEgW3z20ZK1zvprRNQzd65sPuckWo
+y4xCdn5fuE+9fcI2yE5B+GH8XWsHrWcqm95qrpKSRaJslC/ZFUdOY/naZ2PosHnf
+HS+RJvr1YKvlKbth6+19eFaK3L5IUx5TBLlmpDl3eesqDaPKMFNe6Jl9j1YBM4LU
+Z/d6jmCj6t1JMmdWLZjNVJq1rpfUWv58WpydwMO5mFpqNNUGJ7hjpomWEWEIgFbP
+1j4JHzUW5G78zKP3PYxPw88C9tXEK+eBEQHDcjsF8c+IS54Wn0BHxOVO3l9UrPFz
+aTvgNCjW5WJUaweBaacPBPYxE38RF6fLYT2cnvQJqi2uMhklf9QU+F3/MWn1eD8Q
+YTxzNwWh/NL2VpjRlko+asSKPPz34eBWfVLrBxS+qnZ6ZwVP6//vpzu+6KHBso64
+EGk9ggL6CNj48Ipe7QhG9kdTM/AMZiR747p2N511JUKrubdrb66Ose6TqPAfNy9x
+9f3ntflZvaFoxysYnN/e07j1un3scrL/Hh9d3rM4gLttxQW1W6PAXs4EHpdJ7qur
+du9f06BHscvgrFQQ9SRcfftAIfRocOBr/sU68PXzBjl2st5cl6bgXBJd3/svgs5K
+JIF42kSpCbQEFzzQnxjMkqXRYSGwhyUVJQawTWUcCnbAPjbC7hLn/CyXOJy3LUIv
+F/8JPIormQeXZ2EjNuTFhtE75K8teA0UgtLLLwIk0LwFShHCp4lBSfslNm9r5mMn
+4ifqQm1cH5pAHLUJb+9zw22bcq/n+OIuUz2Vz2nsELA0UXWaPccKw9OPk4rmii04
+wfDheQZDx8RJUrViiPimwAcjaGJkDNi64FMIrYmDDRaOEGDsggCtP2BR2Rmofh1T
+Xt8Q22PQW36vw02cWgth0Tiy1sjtT5I6mj4GDBcHYhy9cKkhbkEYL3jKFwWcjX4T
+DHnbRONBp1YKyLN+EWQywGRefxGcv+SRfwTRaordguBbje/unZOSd1llsNC32dIK
+5JoheLR7VVG73tLkY8N0STbHXDfNh4b5EHBf5rm9Lc2PYBHw/2waJ+3PGx8hrSu3
+XwGjGIpCvFzUmwUlBM8Y24+1q97By8sIexWsUjiJSzwT1LYPddCA0KeTOhBgL/LM
+3sXI68lV5k40QRmuPoo2pwmofON6VctcYK0bYppZuT0FS7eZgsnQu3H5LoGeu5cy
+da1CPY1xVJg2UZ9EzQD1xiQVWh+aJ+IwNUcLECMZUmWCI3B+wDJIJsewBf5yYNPs
+cgRWSUKg29N/CFBTEYBSFiyNdL8TVtrF123jCIGit8iIhjkK/kAPE9TL6K69Hs78
+P9vVjh1bDgLzWYVX8I6EPqD1vDNZdzL7DwaqkK7admSDaF3EpyiU5YarfWBmufWZ
+5ias9cCwKJWB27E6PKVk3AU+a+sVQ/F5nC5CViyGSgWb2hUV0Q30gA8NECsy0sNM
+7E4Nme2oPCa8E48jkXLjtf835T5vk2DPuEohIpOoRqAHh8XIeh3J8ViB10vAi+dC
+8T7DeSsetPHRO9YvCv8ouJuoD40OF6cXVYjplPlUDrFl3TLdrugLtYr55uIcZx6H
+aGLXeHbF6vM592md4zvYsovV6N1AD7oC25krCs17Ry6WUAR7nmDkNBdWfvx5QVGZ
+y1Z5f6mdERZ4tzjuYyLSO6sckVF+Lmqee0Lwtlq3mG/nkrPPhRO1V94e3RrOzJbm
+hb+vNfK3+PVIo36dRAzsq3+fzDzc4iO7AE8Nym1Xwrfi5NcVVscvisPGXLEyikHH
+1Mdof+iYi630i465IoKz6ZiL4e9JaPue8M6pvM879s/Nx1ycc12EzBWm/RAyF5fO
+Q8i2eAjZVpwYuQKs+YfiELKtSELmIihuErJwlwwxCZkrSrNDyOJ982nweL3chAzR
+uY9b9v9grKrNh4+5XMbDyCIRbRxG9iQqGdlWPG/7yD4LQgGFjniF9RAbBmbcisnh
+YsvpOSFVzb2PDjEtmKXhvqdFD91ySS8y5mIvH6eVU9sKxVJlfzfMY5UFlBd82lGL
+c0HslgbHwqPQsXBuxPngari8YbutbaUvEl8b2eaTK8JAreEZHeaDlCiCJDgP/SsV
+GHA25esWraaIRrW1zu85Uavl+eKAnR8iF7etiAjRfBhLjN/qhix3zqxIkbEoFjMm
+HD+j8ceJISHy6ZZssTFlkwjZ12IkeiLczpoQvxt8nbsCBUmMAMCZYm1njQoVllyG
+fqINcB0Bd838HKfNyctsi/Ce9TQPi1EiyMNyIjBV7/OgeBgGkQWf9YRyI1N1hRQj
+7TGcN47epvqInoVE31QETVY85Sg6gM7FSqai/FxrSVzO71mDAtwwTOkQ07tO8xBp
+3rXf56OiL30FpMjlQ7JG7M9Mno32sU1GW/5aSYu6WJqnorSaj5kDs2EBV/3+xj4f
+dLbXxuYcO3ZJhAeDVefVyx7q9tHrzjDZzTxX8tOW3amg3nF7rbvgF9e+SfPcJE9/
+HIj6G5xs92NkYqNWNmjPsvOaQMOJbVzCGKisYQXvLllG3godt62glFGTDHXQpEuc
+JHCPYhSej0zE4q8Dya8yiHIbzCu/TYIqlagHhkYxAfU6jlfwOMMo61QUrXvuXpWB
+Ck6Jity4cqIEvmeVOwfXqTcUtQDALVrInOAsikGfXDRjGvlBV+jkRInMuDibnTS7
+OJJvsAxcUVlVLAMnKMo0g40tBwnZe1s8Zzk9g8MVHbMck/C6DuJwxJX9nIqz93oZ
+rv6rrCgKd6dEmXk1OshQUUlNfiiS/ER81k/5fGL/4IciVp92K7ycGGBuI5Pxkslt
+xXZ4E0N6oTHrPtZKBLv3LWYyHgWS1Suzb0zlvgst7K5jVrsi5ki8hGtY1/Z1KiMm
+6krzq1ReWT0r0u3d7jPifSvq4JUrIMyxokf1iNvNEBuL2xU18DqQsONcCs67QAJz
+h0TjXu/TUfr908jpx9Wz2/3paf1x7IjheJqTZK68Hb9Wot++3WHKti+DnFZ5OTxT
+NPZxnCKt8a7nvHJK6RWU5+4M2v52hnR79hFygF+o7DDW96N4ilQImD8Uhwh+3sHk
+tj8DqNh9bXtf8pLww3lrbrESje9YzD6YA/PFYvBgtgaLcEmVHD+mhTkMzy2FsX+E
+Du9jLRQD9CxYEp8jE+bOg8hYgt8Y2S52uAmxW6MnI63nyKWL5krHJeDZgg3g14EE
+NF+jk7oZztfC4JjoiiNGVNKcCkQl4XvkxYBqte21ciIHpsGtbGbTx22Hcg7o/awY
+cnSb8xdPjiRwGme0PWJJ7FFhA9TvwDDD/aBwZmMfZ6oa2OlJZQdt/PvPSfUGKdZB
+oK59fVYJy9HRejWqFgrnKCpa0S9eGEqIb9/TciaIuXMF8zJ4Z4hrMExt8SlcAhoJ
+oitG0/tcjSDXjYFaeftOV12LH2c6gy3BtX6LnfvVo1CmFEy1lJVXY5hIVmV8GhXf
+0ajhGCfDePyGxFfiWXmab94/zZg8V2fI9qdPQHV8fYs3p7pjx9Pe70fxAIBtAPum
+8Kq0lorJxrBYC1xkVy3gpzGu0ZIsqeWty/tiNLvYhB3cKsTNy7OkcvDn+SqA5c7l
+McQ2+Gne7ooovXAu9loXJ3eK5tvuET0hxNutOOjmilLsF4U3uf+5FB5hTOXl7wRm
+iNJ/Yy7bsvQ/1plIn9B8DCRsoKYX6FeKGZ1u93kpXMHGZGi1c+OCtVPzvXINHE/h
+WhKPfxJVp6U5FA+Wf2QfWL4aQkXG/IZclWji7rtUkty7O0u4sQXlgz+uUFqrw0SI
+Yzwcz8VmdqGLK8oaz3n16yavo3g+HMbVu4SO4dsVqxS2mlsaommdCqwN8ePk6QGR
+cbUAiyqxwRUrxFYH/IxHeJfA1h/RYDvItCJCkK2j/CGjq2zKF8WcCjVFlFxA5v45
+WN6qluf4GhaWS+SM3wpPotJcZg7TLyaH9VA0rYuyIBZ9Xyy2xnToFhj9Ue0+bRyV
+PbA9gtAGS6vtEHWUYuuMv7JQJXM3Bvqu2DZPhXKtQszhjGUKFm5bM79WQV1K25XU
+hDhQspLIsoC+8RJysNp3TsSp6jlv4NRx3WAOlI1gc5sb6f/MoC7Gqak+oudgtjTP
+2TAc5163IorMFyvv0kb6XClyeco9LhSsvli+IO61juaV5iHSvE65z6VwdZspcq5J
+VkhAPsk72iY4PcX8NblDJ1k4cmlA8zDv3KQAqMEquGcNulpLZUuOHQojIRgMlY2r
+g/2vjLvDnRDcp6NQRE82H0PEklp3qfPuoEMQeycCZGccYAJ9yEbkjvq+GrNnxQU8
+oa3FsoYmGrMSuxrZWMkScuqNHC5RFiSjrEtucTZN61TE/IUIrtAWfyyqdwH4x8jw
+hPjSjfmvhMXJ5Uvqtg74vM5NeZ4oupL+6S7msrjLVcZJdjFuQNlBwshfAz2VoO8D
+0BcWz7gXu9fLf//CZJJzCEb1GwqALxUvWrSQFS4q+EcPcBoQe5LOnAu+z1RjTXac
+KwmrIVwpeuISmVLhYBJTD78W8nReXrAgiKQxKb9GTU0OSF/uECtHrwAhH5usXj/2
+yUn5hd+2AVlm+tUAK3MZxKAJ2DWYGLfjtJ4l/Iqv0M1+if4M3eaNhBP7QfxamCeB
+OVJgul+sDGC3DKAhvIXPuNIRSWwl4LVC5bXzfhTurNf8yxWxibZfFB6DVVOBVUAx
+00PEGhKQKy42kvt4fSQxFJXhEE9OiK3jXJzPh0hWJ2D3rcRu8JxGZxCQTSBOQyB1
+bOtFVuvPbsUrVBhYeJJiBFZhnYqO/Wv/uGNOn7sDNTXvxre9GvPbRDSV9HpLfCP4
+5DmtRskTjIi0pE8Ga/lDPmMD4VOiFbwQ1mwgf5pCZr5e3xP4cjw7NhN9//7nukTn
+2BnLFWjlJxeeOnsmkOvSLNu/tc4eCf85KAbC5q97RD6+j/t4jrEhL8QYAVHMZYcK
+C2wAhzKsAjytq99iN9n1QIVO3o5fD3jI2yep6sjb5zASd8N5J63vwl+LHDFrs677
+vLBRe5kIzKrkxZlmjxQjzdujERj4zOSVB7ab+9kOZr4fhRPwyXz+L3MERwEgwADp
+LnxbDQplbmRzdHJlYW0NZW5kb2JqDTE4IDAgb2JqPDwvVHlwZS9Gb250L0VuY29k
+aW5nL1dpbkFuc2lFbmNvZGluZy9CYXNlRm9udC9IZWx2ZXRpY2EtQm9sZC9GaXJz
+dENoYXIgMzIvTGFzdENoYXIgMTIwL1N1YnR5cGUvVHlwZTEvRm9udERlc2NyaXB0
+b3IgMjUgMCBSL1dpZHRoc1syNzggMzMzIDQ3NCA1NTYgNTU2IDg4OSA3MjIgMjM4
+IDMzMyAzMzMgMzg5IDU4NCAyNzggMzMzIDI3OCAyNzggNTU2IDU1NiA1NTYgNTU2
+IDU1NiA1NTYgNTU2IDU1NiA1NTYgNTU2IDMzMyAzMzMgNTg0IDU4NCA1ODQgNjEx
+IDk3NSA3MjIgNzIyIDcyMiA3MjIgNjY3IDYxMSA3NzggNzIyIDI3OCA1NTYgNzIy
+IDYxMSA4MzMgNzIyIDc3OCA2NjcgNzc4IDcyMiA2NjcgNjExIDcyMiA2NjcgOTQ0
+IDY2NyA2NjcgNjExIDMzMyAyNzggMzMzIDU4NCA1NTYgMzMzIDU1NiA2MTEgNTU2
+IDYxMSA1NTYgMzMzIDYxMSA2MTEgMjc4IDI3OCA1NTYgMjc4IDg4OSA2MTEgNjEx
+IDYxMSA2MTEgMzg5IDU1NiAzMzMgNjExIDU1NiA3NzggNTU2XT4+DWVuZG9iag0x
+OSAwIG9iajw8L1R5cGUvRm9udC9FbmNvZGluZy9XaW5BbnNpRW5jb2RpbmcvQmFz
+ZUZvbnQvSGVsdmV0aWNhL0ZpcnN0Q2hhciAzMi9MYXN0Q2hhciAxMTgvU3VidHlw
+ZS9UeXBlMS9Gb250RGVzY3JpcHRvciAyNiAwIFIvV2lkdGhzWzI3OCAyNzggMzU1
+IDU1NiA1NTYgODg5IDY2NyAxOTEgMzMzIDMzMyAzODkgNTg0IDI3OCAzMzMgMjc4
+IDI3OCA1NTYgNTU2IDU1NiA1NTYgNTU2IDU1NiA1NTYgNTU2IDU1NiA1NTYgMjc4
+IDI3OCA1ODQgNTg0IDU4NCA1NTYgMTAxNSA2NjcgNjY3IDcyMiA3MjIgNjY3IDYx
+MSA3NzggNzIyIDI3OCA1MDAgNjY3IDU1NiA4MzMgNzIyIDc3OCA2NjcgNzc4IDcy
+MiA2NjcgNjExIDcyMiA2NjcgOTQ0IDY2NyA2NjcgNjExIDI3OCAyNzggMjc4IDQ2
+OSA1NTYgMzMzIDU1NiA1NTYgNTAwIDU1NiA1NTYgMjc4IDU1NiA1NTYgMjIyIDIy
+MiA1MDAgMjIyIDgzMyA1NTYgNTU2IDU1NiA1NTYgMzMzIDUwMCAyNzggNTU2IDUw
+MF0+Pg1lbmRvYmoNMjAgMCBvYmpbL1NlcGFyYXRpb24vQWNyb0Zha2VHcmF5IDEx
+IDAgUiAyNCAwIFJdDWVuZG9iag0yMSAwIG9iajw8L0xlbmd0aCA4NDEwL0ZpbHRl
+ci9GbGF0ZURlY29kZT4+c3RyZWFtDQpIiWxXS7IkKQ7c9yneCdIAid952qZtFq82
+c//FyN0FEa+qV5kuFIA+SHIr88vHZ3r9+v7LyvzUMSDY1b8AzfaXz08R6g2g1vH1
+N5XH5mrdWt7YbH6atYDrU90OhPr69Nre63MPfi20t+HgtbX5/tS1IRjTvgDbWoDm
++4H9s9tMdQqMm37ztBZLfwr6Z672FoxPyevqgBr6hJXHlZWXL8b7eV2yNRwEOGX6
+7HZguiZ2fa376IRtP24d/JF6q/y+udZ5dq0TgGZGfKj4tijE9rL5n7+s9k8pcVDc
+Pg769Qhi57LxUQ0Py6+/CcIvldueT/4Q3IOrf/6Ej3r7jDr/RfAcyQ9+g7ry9+82
+yKxIhsHg+DKatT/WuEfv4bCAPuk/HAQ4FL3RO9wWgun8fuzO9TUZvYksD7gVesBQ
+b+Vju9/1gNO4XZ+dcG+Zo1Rt9dMGTy9hMKBt2j9Gf8PmLdUp8M/iKkIewG1o7+6A
+1XdeZYSF4ewVRwC2yF74vsmO5olkpvV1V+NJSXetSshDWzHqrsjkAUFvnastjAu4
+e31gD4e3VKcgktK57K660fXx3HzZ1kZepBpLAV4z44FSEF8V+dMQnvVZ/di4F7D1
+iQyAlZP6ZWR8jJViDkVvLJYdH8dsV6WpOqwNJviKB/tb6vz913+pX9yZoC2TqURF
+QsAymUozwqINENaAvXseV3D9EBRdpxSWIj9fJ6zzqEtQ44cQyRbRz8sq/OOG0Ivy
+Ycn0sDGQaesVNgjJb7XVs0qvctWHshT3wc43qWFPCGZm0piM/x4PiiQtNyYSeNR8
+QjnIc2ugVVO1bdXiMblaCx9X8XxrCgdgvsWy3ut96O3IO95ce1lqn0JfM5h0/Wrp
+yxvIv1EomoX72qlN8W4Lij1sjKz+33+ogncLnyL0F0Qis1Y15Inchsxu43ipz024
+ej4ahSA6o/HZ+MRrjjpe+Hy9G6ENO1DqQ4/urK86tZ3gHlxF4YA6mhC/XwVBQoui
++t77gU9huQLfTeuMGRIJeyup9Z4DT1dn6Vr2xRJXTLA2RaXPVE8Bnt43BW7rCGjY
+MhUEwaV+t0vNqxUfGiTyapajgr2hz3EskWAg7PgaVRG7Tu0OO0NarjGNDcO03AvL
+C3KZMWnrwGOLv9elXVdGWHvXrquEScPVjZzrHTNPlqaAtll69lypTsHCu1U+9cG7
+jT2pP3X2zPRCbQbcdvOJl11WFYfds1IynfqrbIbVKptc1I2xVVNyeCNEOsuluAGs
+9sX1ZSzZ29obqsY9GPMPIN8R5hLupFzZ9SjjdSD45+h3u4wTlSnuK9Vd7dJMew9j
+L6lTX0898T3vVZbmiyH17SrHefGEbRx1CvBmlFp0Ury4pq97yf6a2q2wYy6Wucgs
+VFRjiaLD/cJMnfljvRRBRmvIaQHnzmQYQx24jS/VFqor0bIMsXBZNOasg2pKISga
+GzaGa4uP6ISNFA+01JPW5kkhGKySK84D6l3L0RcB2R8Fpc1h9qzHfLZVdFcjdM3F
+TTU4BLUzvt21XlQLFm+WMHxVd6qngI0GX6uZY5zj7oNZi0Fa6tBD4uI5xt2q7ZPV
+sERZq5yHV/QauYqpibosQgFn41uzoZvcSbIg5wNWTIFRVVBuH1j0+I4gwoQiGNvt
+mX1uEY62VG5H3qWpHZOhGMLOVlBH2pFDcztmViYiJv1vBqGqVaABI0aTfQeVgJtZ
+19DQMmRbU7rH7IGz62AEZ171lT0cckJkeqXLPBPKFsO62OngPFEYDL+EjSW3LHvD
+mjlzBd6lbiroE/QN91mqg8VPoBiKktatoqqa1nX3A2WdFX+vVw2IZEDhuo1uMvkj
+X67KCXBVZeDUXcZPaFc9BTXX+cjRnJSQeza1rpqXqY31qW89Jqv0ba/nbdUDpT7l
+x7Oera+b/FREBPuu6ZiajdPlZxQu1k5l9CtqqgxZVS73CcHel/rYvPvRV9Ht5nhR
+nxC4qCWpT8Bul+sADlmT1MfQal/r6zAh1vKALGiX+oRgt3Wpj8VApuJN6vPAnFCO
+gNQH3+bACuqDvet+UR+cjTgk9YmrbkwsIjcwa+3X85oa2nMV3p2X+gS04S/yE4Ky
+6iU/hrK6L/k58JKfIxD5iY+r2CV7eEDv80V+EDB0hSQ/ph56yA/MmONV0VCt6ov8
+QMHmnTAQIe1G8oPdNAol+UH8yzrkB7lhfsnPz9xRXRgYih/yA1ftcckPoGZIkh/C
++iI/EKgokvwQ2iU/Lyjy8whIfnB4t0N+cNlqL/KDIE6/5Ae5vPrhN8hcXw/7CTzX
+eK1iAjvsx5be8GU/IWhjXfYDr/shPwku97mY1Idf9kt+sHP1h/3gYFVish/c6qE7
+em72Yj+I37T3emt+6Q8ckGxnnETdvR36g3C2cenPj1CyVniNfjAO/fn1CMKWSKZv
+vsBR7V8FGLtC4E0GMkGFve4/sbiBBOyTfwjuGeeTPwS65/fvF/9Goh6Rtk1bOjO9
+d3ZQLzFbr/fBPz75pkeMQ+nLI9FI57xtGjVBFA0TE2BfS4nB6QT/5B7DdBq/RXMY
+Ww3yWqkyJrMBeirnqPrGKbWrXurr4WolerzwuiarhTrEpOLjLn2/YRsj1VPA3sLu
+T4dM3j0OW1zdZ3PO4VFIljZvYgQViZomJpSlc+33+ijj9mAO0GrodaR60+G7yo/F
+9x36OFCzZfchx6QAVVBphEmOG/Y+Mw7MujG2jsuh0GseN6S+qnF9JZRpz1Qoy3vx
+1+oSMcLkhCLOjZ0NFliDB28xFC8wK39D86Oc8cTIwG9zUkB3RJjZJFQqSWGGZrTM
+hcnXWpeiVYoSLx0KLriP4PsRPI/kRy4zvXs5+YZZ5xcFVSMsTA9UzrAB2/dnZ3OQ
+U3cQAPaW0hqX5yZs1t/QNU4/gjVduxlj0NDY4qymuuGqXSFwkdcZ4xbg6qfxWa8c
+IW7IAs96F4NB9KaQbUJLLtuZqiHoW6mK+h2Qzz7uNfYb+hypnoJmUh+LsOTuPefi
+ZqnuS05rUjeNSA6rO+jMPB5P9cIkaPNsxywwWQKmSN5WU3upIM+YpgFz0FgPAEGQ
+qghBumSK1np+5yayoW6Ia7UuIlvpXvKHJKrwNtLkEtkIBkN51iON9n6+jrTZr80R
++NXu4QFHm7raC6K+nLRKAakN4G6XqyDpWjtUVuq7m1y2eXhVy2xgoz8znCMMRKuf
+hv3rETgJAhNvdPV6GYdAchZox3ipe5f6prU2p1xlFyoze/mxPmW8YxwIuLYGjfRs
+Y/Hm4U3p4k1XswciyO3ETQLvWi+aJtAosPsuyhaxB5zumU96Rd2m2k3effqBUi8a
+VbVeWDn4tVw7TIFZxzE+NPyZ0iD97BmYl9sVCVxXT009+hEMBi/uv00vU3C5iNze
+eb+RD9lkblfPwQnK2nlgmqOOpvWi8nW+DnO2v3eP+cDnc3rhC+bd7A0xlKT1ErT0
+jjoYKihdV0TM1v/Zro4jy24keKcVY8EEtLCH1+Ze1v+ITQH1uBN9+J0FPAAls2pX
+t1E9Nqb5y09bc5fzLzmmW92OK/NdbmNNfdZ72I1jVSzYbUwX5vbrsWpSJf1YXY1f
+LeoRHh4onnA2EeRV+cUDQDLcIQII+kMEgM1uExFcuIjgCkQEPC7nSwRlN5CbCNii
+90sESI3aT60HH87+EAFbq/iszlouE6AopP4yAaj38gDUy/3ywIGbB47APNDc620e
+aJ4oLw+03x76TANte9Q0gJTq46UB/BfKpQGeFi8P4JQwXx6A63O/PLBmxs0EB24u
+WILNBmu+3WyAu3N72aApcE49Z0K3W+2ret+HDVC9XjaAv+63iJwZ37OzcvvcnX/X
+NA4ZbHjI4ApEBoQzHzJg4KXykEFdfeImA4Sx51mTwSfMXYIg8sC6yWAJNhlAmdYv
+GaC9cvneZFDcDWwyQGNpdV3tYarykgEn23e1pXm5oKptfLiAc9ylAs58+VLBgpcK
+jsBUwI/LpQJ0sV8q4MK8VMAqPW6px8N7e6mAg0y663T5uFQAs3iQ2VRAs41LBcfK
+ZbnlMbr9wOf2lwqOwFSA97s6uxjjJtfmXaxx0y3luCeMW+irZpKHCKDMrjh2Wn+J
+AMrM9p4NB5Vx76b76iWCAzcRHEFatik1Xiage9PLBKxn+TIB3zZuqa+aBx4mwIg4
+47teR7tUwMSYLxVUB+SmgmNVU8HH6qYCzFxDMRqKHXEEHT8/EvTmhoiRADMVzx2c
+S+pQxSrOEz5g/M5O967V6qmlFCHRmJC3zlnO6vydnCw1CZXVhQ3bYbrQF3ERYQ+K
+wFDLC3Mca/sStDi0Xt1Uqh5OJzmCO5a1Oxb3lN3PHsllxCqWmBdaGpb2rKbuVaUh
+7GFaSFvHGHRVSj45rLPCMqar/lLSMUj+arJ943xW/yiIqD8/uqDF/hXUpoBiIKwd
+/SOYqiFLIMu6F+m2zXSjUposiRuTjcWLCUtSWzeowYU557VdAvgJQeOvVzzYD3NR
+z+oN6FaxR/NTIsvMZhao/fIONHunFPihenNekdiTx92yjTkWq8nUfWhmcf37hL0y
+oaHnMlOtTLiClQkQtJsJQB4LlAgtuaM7iQABc25lQuPYd2K9JWfxzgTgMetZzXsI
+YCYAZTPmygQIaswnEwBbqCcTLlyZcAXKBMBiRpU3M9wznkyAIDyZgIexzqxYB8qx
+3kygimU8q9HjhDKBBnFwp61jCONkAqw311lhGTO+qcD1N+7ZoNY/Clbc06ZxfgXl
+kwjc0ecrgKGdjUoEGtZ9dbdpxjcRHLQnEVD2U7uJcOBOhCXYicCv80kEnp7Hkwjy
+6s0E+mHOE+t4egvxyQUq9/Ro9EQ7qUBT0xMnFWjd0XcqMIyfVPjGvVNheLLjeVmp
+0MxSXc/90Q6G20ewhsUMG1KAklvbs6OjdIXyFYwxn096VJv17Egeo64gwrtzP8xn
+cEa5Alw7og5lV+WHzVVT1o6Pcj/Ut8PJYdob6go7nSyTIC3A7tYeBgObEbIToDWT
+fAdBSX4VaJUwVl+A/O0MgLEQN7MZGmd1DUP44RwAWF0mOOVwd3ZSoRYgWwjd1896
+EVqCuDd76Ay5//Knnumqb1LiI7rdt/Dq6JEVvYPeVfJpYwB7HxtayRxum9M5imh7
+aDQRR4jhvsRvaUqEoqlIBh4qQpOl4sKc69q9BGna0ZwK/4hH/eAQu9+uTONACZCH
+Mo0Nry1em2vvzHaI62eRSeGtg5aerr57WWFc3GZT7eb2J6/d0zzSghWLLuQaQG9c
+jW2UJ85+2AhLEfXGI4p1GExWbSRVKT7IjlNDSWhLq3B2DgSecU4wxlCv2zk/KApq
+XG4ddUMbp4X6rpf6+Trnz+mwdHxux+tL9dscQ48yKiUd8VJF2pwL/7mCU8n7EFX9
+vwA1B9RhQXGgZroAbYBHgygH4Ts2pF1N/9/aPVegygcT8e0WuFRBzYSG3I5Osaaz
+PlClsrYH0vpgOyPzeaIDVl/ZHA6AY+mfHggXjrVbuHhsAmxNZYVdMWF26DFm/JTp
+jobVmpAjxipU1GuVim6GkSHGu56rSyEbAcDgmrQqCcYZFxolJPqfmm5J+3hJE9r2
+AseVKcfBC2G15La7NNPAt33CUaqut4ljmHr2EfkHqHi3hhRDKx5Lu+voz4tbnmSX
+lemXzLy2N29naMuD6u3jGA8qnogO5hBzPkXazn20Ob3tl8iITUnmhykUmZPn4YZW
+s8/0rrdZXfTSsopOx5i3PRZcFKPNonLg/umGfRxt7X48QKIKv/D3u/z679//+Sup
+b/+FvqJA7X/+SiT5X6A/BuYPIFq7XF4BDAQTrA+8A4784AIKx75SLIh4eH8F4fd8
+T/g8geWMn3Sohn4hh65XZTLpwjwiqdKDhONcdzwfUEk+tLf6arYEV5WuovUIOIXN
+LSDsyFdAVq+EXzZT+ImBRY8CZjW2kZoJKzqgRu/UF3JW83YJMEIgdwThbcC9HT0F
+YY5rNzvc5vqky4MsMKtfBinQEN1QwEawqcAQTRQzoDbXt+jJAOtcR7PcNrJl0TK5
+mLZDzOohqE2NHMTdbLhe9zI4y1cQMJs1kvKy6pQnYuFTUC9KsWOy4Hp4Hn4LJyit
+l160LlBjE2A+LejNHGne9Wi9ZCIXW8G8toe6BNHr1rOGaeiQIVzb4yeyPxHkwOzq
+LfnMOBVUXfPTEsgYa73ZEREVnUq2tPyUEEs8pno9Ta0PhNQDp6rtEeCY4YBMWTAn
+R4H0R/6ndbqiL9nsWaFZm3XVsw0fXS34uQLr8fNvTZVRaJtZ9W9GbQFOyeRcxcqs
+fxCcFGPnwPD4P8H9BKle8x8Exy/fd/hpKC4WDaTYP1dQRUU/ErBfo09pebZYSesV
+BibMQ0VL6QTYijaTqwjHFGQdAMtF9JtG3IyU7fNZZZtINLtg01pR307MlOVFxcvD
+3zbk5gNH2GcvwaTHxb5K7ACPE3ZFcNCApsuGIpxNDCFpCTCxnJGd48rzvLSszvTG
+6GPz7YQYlYrQUrv+/G2LDgUnmkMts1wwoUO7ECHpbLqCmrPWK9ow1inMA/IGpgpV
+ov2WFFVryWVyQBMP1WJFUyobWtMa2rvuul0D9UYS9qyz8tDuYQZiiZ1ab1mQGUM4
+o2I05rK2S8D0KoodKDGkSxpJHyRDR8fYuVTGdlpfqrcVEUIt2ClpHujdoZa7DvN3
+p1C3UxYKZUdm8mW5alnRBgVzf2FaXjgCdvP6OiphSSY6PMuHM/a1fRQl/Uh+eTBF
+ibH49NxdvLaiHKQoyFYtVxe36OijXtC+7+icTDnsmvZCMuGlYi+UFW8pLy+0qeq/
+do9azHiGszl01UIl9efmEcXydHojE+q4sCr6vH0Jyj6uKv1DdQA1ki1L046I1MSC
+rElycHcBcjlY7cs2YS17DeZn4AEYpabPalopwqkPeKVb0Hdy+6eecQRC5MFp/am+
+W3BK6cjm2ivATfF0c3AV3Fndz0RBd3bBYQvcmkiCMUA4nLGFWbIh2b2u7UvAtk5f
+ux6wG9DhwcUkeDva7CqPVLQmhMWVjC0Un97dJhD+LV1SHVvwcwWHBb72EDvTAH5D
+ht/+0aVh9C0gTM2cVqJg7orX6sTnoxSgLQ0t16IAZU910TSVb8FQM6Vvx3CNMGSt
+YWKOfXaqLhK+OozNcX734dFH+8OBW2BFfv6tqjhwwLTjpectuPGA22v8CuYoN0Cm
+SIswGioDGS9WGZPlUAAlNhkzaNxj4WQSAdagyjazXA7BcLVpYHhUPWSXII0PlJJo
+PjglIRAFw9z/Y7tasiRZddhWagn8bGA9fd6sanL3P3iWZCLI6h5lChxg/JWHpEfy
+YZj/wKgGs6d4LkBzfCxU4bqAVW13+kpVtkYQ1NhA7JOmqhiQsRrJ5iOly+7KxU07
+LGWxd5nFNiPRpqVZOjPXKgIFteQJ1Mfmgn9+u0lxi3pLd49B9nIWjA1cjpo5Q+VC
+GF3EAAvUqZF1OEhIwAnPRYGp50HKpgrGv8HKRKz7vuGxbS5MNbjz9eQE+Ry+SKf1
+/uztHeY8quk1j+qCf36/lqG7W/j+rm1n4YnUHS8r+x8LT4rsHmlr/1hAH5r5STKY
+Xwthh9buT/5aeKrOp6ZQPuaaKKt0VnEo30uLIYcBgdEAsCuaPDoUYBGdNUZbL/WI
+o18CdvRgNJBeIm67n+YCaMPevZqqwC07otQVlWz9gcG8cO1e3NZDFvpLIHZ240gL
+6UUqGVW7Q1hzAcKpCoJJIZz6TOms7g39MmBTM77+bz+i6hKTWR8nbfq++0g11HKy
+ZCCUxNTifDyxlYcGwjSsIIcG0lb+se/TxAONcE8FUZd4E3GL0+MYQJ3WhhPton64
+e0pzAW6mrIhY5DgPchHC7brINZetVMs9W2d6dGnEKZKuGagk+XilqxTaeXRT63D5
+dQ+Kl+1pIwwLJGmd+6g7bGr+wtcBzwIrBb/uqpRTpzc6flnqUrfmoqkYy2lmdKle
+mh2ol9ax7v2mKjW23q0qNKaltFXWXTQKmqlzrEIZYh4Y70J9zMQQ52scQXn9rP9Y
+CApmgwuZi38tKDmVr4PsN1M4TsCcO0Br6td//6NE9A0NVugiP/eCxyeAe0zNRoFC
+nUV/HKOEmGj1dOwHJRLN2G0QemsHQhwxVd/9xaR7vl6KLZwuceSGLi8IxYjuuaWa
+3xCqS3ycCdO+nq8tqLzn6aw9u7ZUxpuY7ppf0k3kttmruyDE821nP1+Or/21zFAb
+euyGy/vXY1VjRP6y+p9ojDhguYI1SOfPuyAiD7gVQW5p6KZ4W6ldY/ypmIgr4fZW
+37cISnpZe/fR3+z9Oi3xHh75qRLKu5n7j15CUjtlRWZNJ/cqj9Wvx/2YG44wGiuY
+dNvUc4mV1up3+ABK7b7t3m+131+X2e/T02Tn9sfCUu3D4Jg7ekVFoIs9euXPu3Ba
+cVhB5PSvBQyCngtdKTKj2ACyNcXLiqCrnhktG3gN01gaOtVomCpw6D6ArfiBEG/x
+eX/3I+PnFG3fhJ3Sg7Qd2AcbEmg74Fzs+LN+wDqPOBfCy0hOfK30gQkAwYQ5ph1d
+1p6nCwMODXVMj3jYnPVKHxpi3vtNzHDILKW/xBF2n/PMoEBj82rb9vXLS0ye4wWj
+a3/kBaUfKDVgdvUeAfU4JcI/31I5B2EBoQHlEyJ54aR+kF6OOfLZRqvV4VVOG12d
+QX0mFmy9uQ8fqgutByAG13GwK0S1i9LEAD4nbwX4UaSJvJam4FlqCOXVWkhvnMq1
+s+2dwVHWSJPw4zI8xcFZsVDWlwyqocRkwZxZFynYLw+w+VSPTxgCc7O1VHDyJnaI
+MzyC3RRAuCEsMp7ZAkca5Uj6nPtjXQFkJ4AyvtjW3u24tXLf+Nr4VfS61xTv4pCr
+Lu73Qh8XPu+BPVPpWfCu48ahEoOQmjoLUSpTaZ7G241PgFvL5sNbunXI6863OHwv
+oJqZVmq7n6Z3hEkV5tcN+IwXijW+C5uKBBwZMIJVxammcNEUUY9DVNoa49pOwLTH
+P2OMe79l8RYqPEojLyyyTIPXSovZVCjLnC6DzJHSYzH4LKvq4PNRZk32V9zPjIxq
+dlhiaka4q/S+OEnAvS5KEo8ebu+2MyDwMe6Zaoxx7dDJ8VjLucq43z1J2QVf8+dC
+6BPW+L4Xepv6QG280ddv236+H5UlubE94FpVZN3GAUNQ0jb6vW9kQ63oW1dZQs+U
+tGcVa5KeumlIsSx5K2VXzlf5Cu9LBXlJEd9vuZ6sslc1R0jYW+zTqCdTFZtXJof5
+Pvc9G4spPazosnqSZ1QNmejyqDo2VWT09VWE1Cwy/VBALOtS5mOtnIAQI+L+e21F
+0GpiPAptkw/nqifcunjnUmVC0+FcUW84NDK9C73p9FGPz4m0WeeRPk6t483CSK8l
+8Wn1QImvtu/9XfW1Cq6KWO2nlGAA5GUqNVWNrKXhLzuJIC310Wue6J/zRIseVado
+QoNxz0IostEYsDA1Ks3wU289+hEvtVAFsGsKYywETOY6QJ8Crqn0ARphk6zSFB5y
+/bNrg5/2PQnd1aBEdxpCQiPZ0P6y7NL7hqucw3MBoUGoUbOAUeAwteayR4q7SkF1
+nQ4ujYaA+gXNVeO7Onk8jAUPsY7jWlhrqDw6IfkIeK/M0sjxHOXDuI9QQj0s/sJg
+22WneC6A9gBa3BKQDD4gWfcOmniUQQEByUW7gg/ApTbtCd0b8lFQTx1r3PvTJY6h
+raERVB6m8I+F3smpu2/ugzkHbL4IF7wSymgGPAuL8f/NBTM+piHCGzgboSIkohG6
+B1zHbzPf7tpH6Ab0Ir9EHU4k6YL0OtudVAsfI0ECnrPKOOFZKD66cZ8hF0NPnzds
+6YdnoZYU39SFFAunl61ZYKb4alV9TcG/5eSuAG1VrcnOQ5uzNXnXy7qmgQhDPlud
+3uaJz104OvQtL6Duo5cPeWGozVV1VHjJGd8pvTqjdVXBPURRKgtiizQwTVqMZleK
+xw9Y7YEaICWeC+MclyXOFEAm3jf3iYg2nm4D/5pKkvJK1f64Z/V+9sLa2VaE8FKc
+0jJFtrGtKIFc/W4qGa+SxjrYYwQZGobC+KhyW3V2ZAbu6D5VDXIQjuq6TLdFeLv0
+HHhxZOISma2N0Es9EOILBP/Zngxl8uZK2Hqqt9OcW4NbYwyDjYi6jgv6E5TPAmJb
+X79DI05vGipNQQmSwVo0ma8RrRe7Ck2t1otexUtLX+/+FhNO5hZwij/B839o2OKu
+qSSeGrCJRzvo7QtLmynOhbe5RK7osX8vRFv9WJgl9dcoODDltsUSi6Yx0vJtqx2j
+teN5GisrTRt+835gvnb6ux9FbbEZy5YoiWr1TxjMKr4HHoUgapkH9esJMtCNlq+9
+ou4bXIavYV6CFCkQ62DnsIjHby2Ayxz3hoacRBZvXJG5W/NAzfcuuQu9gJG4RfHL
+8e48UOJd4me/rnl/Xfzj9DC2vZdPFgBqlvB9ykmz7fPQjZ93IUSd7u2NI9ffCxHF
+nripqIZCAXqOFUMwQ9k39YsF+mPSCIBrK7aRd71Hc28HQjz65Nr3ftP+2oI6yo6w
+qeCuFHa1dDTwQJkH7p7SY9JPvkm5Y6GTbsyVV2l64jugB9mECpQ01bCVehdW643G
+Ea/adRx4Xt3vfcvibjKZRlsXbYJBKxfGlg3LFrEZ+riIfqpWHywPfv926Te9HAH+
+8P8far/VyI5PoxPsT7z6UBmQcZy94JLAwPaxEC1mjnPJ969LpcbirBrdeJuCLRc2
+xwLA2slMSOECttFFY9giYwFkDkQnbgIk0YicR94F9OEHSnyVee9v5Pb5eh/W1J2M
+r2fHwO3GfXZ/JPeLpLqkcwFTz/Nx6OD2Hh7ZLOpwLl8a0I5q8SDGT6ouKPHkUGe/
+iyQNNJhjGJw+U5yGWyTgl51XaV+/zM4BDR+oXZ60z4XXnfGgOf+x8MbZxxl08IjA
+1+QYA8YPsaZSA0MP9H+qq+BIYhiE/a+Kq+AGgo1JPdt/D2tJOLf7FDgMAVsg6Efq
+yiAcrTq1pGzD1Td5yG+Rj0gd/lefGnZjaKjHbXvGLmnUizBuHge/4Pgee5oDzvb0
+VEnomAOTIk3NbcOyYDALka5QVM/r6NBLe+DahMLETAMaW+KGq8aBSvzO+vdfm8oF
+8QYGhISeppgGZdJSWphwG+aif91EsAJ59mkaQFTsEs7nFFu7eqDkzBUtJgnV5mmB
+VhMruU2pOuvgusSC+pWML//o4KgyChEizXlaZs24LOO9mNhgYg/0rK4qDZN6VN+6
+dqwOvSQ1q0Njc8NW4bosmYRXdOJVB75+ft8CDAALM4tvDQplbmRzdHJlYW0NZW5k
+b2JqDTIyIDAgb2JqPDwvTGVuZ3RoIDg1NDcvRmlsdGVyL0ZsYXRlRGVjb2RlPj5z
+dHJlYW0NCkiJbFdNsiapCp3XKu4KbogiynoqouMN6u5/2nAOZvp1vVkeIJV/cGj/
+lr6/bH73Mb5GwL4G4OyA2hVw+9fvX0mYqydh9AX+GhNwK6A3T6gmJQ6Cfpt+/fn1
+T1Dse8fJQdnTvn5AsA7CavaVcGxJOF0BpRtgCOWJ83sviKv6V0JwVTrAkINSNs4c
+L1fDNB61JOD4dl1QpEwb3yYLtrYFvk54wvSCFtwjXoTV/Yt/h6fWt+jk6emYFUKr
+dOnxFYRlG8rYtoTeFZr3LgfSzKX75YcbDMetSScpzlKhLqHEwmltd7A9YxqelX3D
+1leJO2PuLhGW1GbS1r8Ja30SViv1FVDdAHdnGhztW4MrZU9aF2onhGdnRGAcWMbu
+efPdkUV0ZbhckHK9UsCepHSaZwPWqMjXk2BxiR3xK+P+/PofrbF0yIxY/jCppoIg
+A+YGIcN3ghsKRs4E3LgxcmI63N2lDJgMiOgAXxtyobcT3H1guUf3yw/N/Po7snZ9
+nB7Otuv2UH5v6lbwseU3SmxRm/xNYV4R0kOwLpNQEC7J+yPJJhN/pLH7W+hPiQzN
++3fIMR5oETuST95o7ajUeUdzx+n95u/N46DujpRC8GTROv9uE6d3KOPxG6O99g2H
+jRIvgg6KN2PHcYq3BmUyW6mMK5S1RtuW43ZDvwrdDeLpcYpPsNemrrMhUXejtE6c
+lcn+G37cDb5vTj/qRhbp2Dfc00schIjzcvAtLQnqWIBb0QJs9NKlMQ92px+Hz7cj
+bOh4dYzNqjn88JOT3zrgYL/Z1Y88ihDHbSF/TYgv1JxH+4K4zVniIETnEEUOxQ8C
+7dvg/7PanzivG05Tn7CxgfV+4sQKmZM5Mu3AyqEqmeJbx3Edl2XY4CrZJ+XUOytI
+KlAoqLb2C0N3P64qwt51Omt9ClVj6U8/CSp5eBB6mSbwhFaCalsn6BQ3zoUxeNwe
+wrkx6FjF6Y0JOlt8oTd68gOKsM+mH1+Y6U3xIohSGXf2ti0VRUDz4/bsaDnzMFwd
+VfdMxI32dg3MjS5y8Tk+q+xReAGXVjp71lqcLYb8TZUxXpndVxNCX9pZxF/Rr8LQ
+6Eo7RmR4LgRHODJSanPmX4Qo47g62qVEZichvWy3RIQwxulFCC0iz7PlNgMhbfw4
+NAIm//nFI/DUi9giAV4c8yWyJadOW6WWxVR8BS67sON4ix3FL0vjyugY75XOKF4E
+iSyQy9Ig+LzV9o4Cvgjje0QJvZZ6bBjabwlFzX78ouav4oElL3lw7IKxRryWhhbi
+lys+DKOpkea+XlPzTv3QO5wV+X8Roo5SidfUhYy6JJhYFyHaQx+3qY4B9khoa1HR
++vmLR3t8NY8aj7XxxRHktOw1Nco/Q/II3IalqbjD4C2NTvlzEzLjE84GZ1nUIuDG
+AcuypyXBFvJsx/BPuBYu3Px5+0EpHAkh+nAjDmLvr4Lefh0tsZ/392qJ7Xy8ij0w
+Fad4EbJJX3/38XG6zHN63d7Sha9q4ZlL8TbKyjKruGUzf318co5+fMarH49SsQ+P
+/0YQoml0/DCiJn9AWBMp2TVNjzW8Dd6wAXVCXJrhwqwahjW6bkLR/Wgbr5W1L1t6
++HNeXJ3GswSw1729hKU025Ru2hmDdcONcf0QYhWc5MsA7MbDB9pBrgaUVgfBhIcv
+NDlUQRrRwdy7l5GeqY1JDRSNO3E6ClAB1y6Imhdf52cSuljx0f26jg+4/BFnd1wU
+z21H+fRAdGJvy/dJmyVu2bCD4FQ9B06u6NlyA+Y7syDFhcqQHwES59902uoo4nVC
+MHGZZQtKHw5oNumy3K7xvDzhUrTg1ummdDl6iliHPFpQN+NFjTNjWP3tgtmlrdJo
+gK8Z00yyNQ6kGcvt5StGaR6Hco4c4twTo3gk/IZdDp/mwk+71rxht5IGjtQyWvIQ
+jrijBC3bbkATtopzV+6byFOqZoYsnqVadcWEFM8N4+Y7W+Ak3GOdxkPxXEzyNnBr
+0JqmJpn60LIz1geP7mWGD4j3zr83FZc8JBWlWVLFmpeysBVs1bd0FTvbW9jx1/aL
+29zenpBUtpC2T4qzaVU9cDvoSLKPfvQ7HpdVfeknrQaFabf6qGrUCcLczPhshVnp
+cGDkVTbI2ChUKtE2a7t35vQa4DfMhYIKnShehOXFRz/DrMuzbDH053CfFd0FXdpi
+7IW12SszRE7v1HXz1RCBOaqU1ziJVKW+kRvsWVkD4NuqRnK8xM4eHukcFU7HFeEd
+8tWy/iLkgtaLMNAtMidbLuyoYUHRGUdRPinY6KKpSK1HAr4viGv29Za/rQNTPHvB
+B39swizlgLOW2Sklnk+WvM4pjjEVUAg3G4Ic6ZnjK1QWJv/C+xZ9dPC2gR6ezTNV
+GdX3zu/oP8Zu2/JRBj8s5FhY1veBtHzNmz25CpUb1HDTtJIdaDYq5NbNvdHFCEA8
+XoaXtNQEdfaij6hij5JYfgdinwvgDwhtcOHACHeqm79swP1sEHmF43GTBBRgvHBs
+PaUcm/boV6HHY87k4cYeiWswmvDqyzp2OiXfkFBL4YfNnV2ZtQdyPpc4Z09ox4Bt
+Dra/8GLhhyoZ+nhGDTuEzNO/CE9+p6fup8pDeC6pP/6LZ24PH7hlVwgo9Ow2OlrY
+99aW8lY3Dg7kdLx0uUObEVaBmz2R4E59+IuvC46piNt7GMWdHXqlEhl1ZQtYX1dS
+bE61zyw5iTM7t8Oo25+XEOaJlX/yDZqnZOcJ6I1XAMbiSiSIeOBe+quBrbwRAzLg
+mZvSS3zNffM5QizXiEDOVXFx0ZTY75i3MC+XQyqWI+aFzgX+ECamGU6btQ0eVVBm
+yqmQqjqnhlNcBfvgzOIMOIjMSrpxxqze4ZWt4GdmwmdsLQxTYGOOSa4+6WP2g6F6
+wzlPnIqwcwrk3+hrOcrzpsktSuTRpN691Ht07EF4vAU00wMpvtVffri0syWnwukz
+Z8vdu5w4jOWSo1863RG3VwjMwZWtJQ5CmBLif0BQXxwnDNIQWLYyLfI6ZVP1Udr5
+4rJa6bN7PwUPY9YDKT7ZlA+f6ygsTE9MMtvxVN8YCGtSGuuGcaS+cPrxVBEUtRWH
+G1TrCEpo2nFYayfzXbFGeOWPG+E+uYydI9fPcizKDEts+nUhnWyUXzfgLNU7d8Jo
+QsYwwMRIbe8vjNR/wlAEcwZ5KepGs3vkZaz5fBiULo2vMOHtTZ5NDhl0tf/0w7SL
+2/lUbOXTtsZ5zzGVd71OlS1pNV6dN3x2H6wsEitPNtKOxvHzEt6GXT//H4L4LMJQ
+1EBHZHP81NPAAa3xf6eO0U9qLcmxHHD7VSHRCWtHqQrKlrBvvnflUpKhifxgyvrC
+SiTGaZ0V4OBbPf32gyIL+Kh5cEca2Bn0u19HK9YiauJ0Hzbvo1hFpvR+4pZm6cNN
+mznwj0sIKyMyVRg25H44tAbHpnhngvSlJQ4Cg/Yv21WOZUmOw/w+RZ6gnjZqOU+9
+GavSmfsbQwBkKH51OpkflEKiuIJ//vbiH1D43OMv9JXvf16n7DrSbUEPkNTQoQe5
+lQ42RH3DVv7LJGBhRFeQdfYML+6EMlZSb66byr1/HbabitN90stbNKzKx4vlve/2
+hit6Rwim6FV+6/WdVTLOdm/u3D6CHW5FUFO32G/NBSM+18c63z1FhtIuaBZplyF6
+qdISVs2S++EGpZxzjrHlTnEAZyVN3mcpc84Swxor33lSXrECBqQK4Iub7TejbOco
+GzG4fz1MctTlDLffoHJyL5jRDX7eb/SjyfSbGw/M1AlBZlZ8nWkXp9+0jNszbaVZ
+5nTofXMez7rL/uLab8GAQfq7oMBgEtiXrKkCsQVp7cVO+/ua33mlMXla8XJsPwie
+CeB67F8CuFD59ojA/r6vACUmP4oBaUmv0ava5ZTTa3TiHmq2IgE6tMOyFaUMeRlI
+SAabbbyXY3hiSDo9F4+3k747i7tJjh1G/tTyggY9tTswrZwfe6ZinMrDvRkGa9Td
+YJFanqqZhiIfegvplUX8XMvxaPDTdY2CDEyj0GhotEoNU6NlQ/l0AXOtNY1Y8px7
+pTlLWePlyhTk/OHOP+MH/LS//OAHQTj6CjgsAKKQwFCdEDpvTwQ8CRBR7bMYKCvg
+4kRlINMNjIsQVtT2iRbhGbN0Ngm9T4Wwa6vqUJvs57fefxjaaC+OyCM9nyB1OECS
+BLUbPPK1TjccMXrAo8NOHr70fVnavlERcJmjTqYG1LU7BP5Qnwv+8PNT9fKl41FR
+AbdOW5PbLe4i03VYpYqMhJkBap9E2ttPey83o8WHEON1o1iEhTbvgeEyXrb/Wxcu
+doTf/7wE6XzYcP4geOInvvgbZwD+V5bprDfDffn9FhgN63At9c5KWNUQliYvF7TB
+Or8R/g67btj6eMxE2mya8mI1+3B8qovuydv6++bgNKlXQOgd2yUgfblft/4+vFoe
+HncXtNurGIPtUVut7D4qVuPF+WlY5B4dFsubw56p18veqhMxkLkd/Lvvt6AVJbX3
+kynSjhRxWEXhaoEG3uiiQ2hUapqxyPiM65w0Qn2fcfb7cfZ8jVWMIXoNipzDqout
+KgaHpjT/h9LQUB056jRMYxdGo04B2Oz50tdq1LBBnu76z9TFalWj/pJmTyOG3rZe
+fRrPqvO9vsT8Bx4LozStFm1fv4qG21VpwiYSgL+O+ta9VmNz1/hofmd4oIlKMTeX
+yo7fWXX36dFdd3pgi8aF/fm7hf3FHwG1t/Zz151QTubsavLANJ5k9YSNhrZ31EeH
+wX0KanHC64FHgJjFx4MOqpjreDgj44RNwWbp3sVSNxnU7IFEoGtC4QB131jdYvQU
+wiRrqxVbmLQqqBdDBRRJgfT1VwZkUpz2jEvfVzBj/PAqeQjZ4h0OsY4TWYgmFUwM
+NxwxzwwW79ZrvoPpkELkei/OcwUrUe+qs2qELmCvWuKnDlfnXaSKDjlUQhf6oFfv
+SlszgL/SYRWBMsS3wyYy20/s7hoBKkpSR8s2GALPvSj8ewVWtd41VMbuRvJU9ojN
+MYVpscyory+tPdWbxSP5Ki9cU0awEWVthU2i7KVNqii8yqIbtNanrIS9n6IT7ojV
+8NWQXdKX0GSHL4uGEjS3Gxlt2tdfgfIbdNj3TxQxiEiHr4Cpg/swxngWkMc4Y6li
+kbOmduKNayhY+pKhWFn8LeoOgBE79ghgiFaCGBrhKHF6D0OxbvntW7FDVUZEmhAU
+Dx8wFMgI8kuPFMQqTq4qUq3FbjSMHP+olyrN2FdvQenNd9312l5fh1We09NqcXva
+NHT7tDlTuDvbqS/6G/hhKx0sa/0gmGSQEmi2qmgXvfMkZN04hM0iK+k2F5DGLs0K
+Di36R9P2pRo/RH67K9baXffAbszpzrTzsN9LrWjGdhusIA3u7Egb3l6mvl5HI9La
+sf0cKlMwcXV4W8cxTb0qT7WfIt09f7tsgVMdLikj3Tz5W3/r7lVwnbvuTpoyLaqW
+wzU1drbYHSPXQbfqSwQFbam9YRN9SMEzpvj3o/wkwNA2PwUcPhx2Y2QZmi20FQ2d
+YUu0vPG0PDwvIjMf31vCMM58Lw/FIQmewwaHobju2F1N1RbudUjCBbYgvx7l0FAX
+S8F9nAeCeuBfgvvaR0AuAVh5g01FBkcWvNYiLs+pem1jIO76ek4Xob2v7c/7Yr2F
+cUxRXft6P9ezQlVAz8UgtO9zn5TK54bgvu6dpRwJ3P8bjNb1aEpcj6v5CPDNYUfB
+cDZWCOz0t+DjDB47vHotcQ+vL98UYBGFHKo6bJ11saMUOqxlZ2P7zTvIKJyzYMAC
+rCQ2sG2iyQErdktgfG5xfiQfkLJ492pL2ckUgXKFGdbYp7w31palCdCsJ8T2puc9
+pSsFqqUSiBG+sAoEyoGjIaeSTjo0FW4TB4BgyG04BVAE4hNVsqVH4KYoWifDd0OR
+uzcxRU+bVWN7HSzV+dbTTJywE86h5ijijMdv+q0crY9DN5UwDeddh7PF9tJ5eimy
+9J6ML+y6aKkSXMFcOrxWDZ8zdGlcjeh1AZuqCziCVTcvU3U0bafPB1ibHtpEKzt3
+uV1WjIREORWp7MGFGKc8W6tctGFFbxufcETEPILaZWVq5lJkMx1MuEqezgbgrmoy
+OrtREyNx2OyBsgtUzWWQraGPRwQzWrRnwohgX6RGbkxjMkx0dFet1jcErdL2EEzk
+Br5GMnhdQt10uLcMU3boUobomcmjTcy+TbnUNutnr6n6ED/qDK+S7EmkzmF4ODY3
+HuoOH1EDxKTmC8EDsbnHyIDf08RlLSxELc7oqUVRmFZVE1MSlFDqVYtIGnEgaQOT
+5/st6K3GO+jveDQpiVs4bYTe5oISmh857ByTe0mSCCOn7GN96Sq2KsTxnDrdMng6
+q2k9KhdNDuvlA46R1SkErGn4miyBXBKHs+2TsEROdbmfXd/ze3SlgWCxljAytu27
+7g8fU1/r4VZMp6cf+uxKI1kxjDqjOryMLj/AVpbc4FsFZbSHLOB+0ZU1Bdc4qqRZ
+UVgzwFuVOiY20NcHrE8BCkEpCsBZWb22jTi9vgh51iuvrV3Vr7AHGUpBqi4o0w7t
+znUyrvvxsqDM6Ym9TZfL9EdPK2W+Yar+CPQ0fN3uwys7zbWLXy4+v6K0Xiv/ZXSy
+d4bTQwzcLGwxHpgeDf/7D3d47iXlb/RUCJzuccz03WI2k1XLG7sUOKaoNhzKngvG
+gncuEXSafsoxgtguyvOsLzWY9YtagliL+zftBjUW4WaVXxqdnBKzQyyVX+jSYvsJ
+Eokc2WpuUxOEwzq5GsV5e9Fiu65wF+BhbWFyPkjz6EvACsSPGQQtttep/hkxs8lJ
+WNio6FHDm/ult5fLZqH3FHvtU2bgQDCeVzf1z2pplaLqqzum6s5gFKfFiWRwOiRW
+w1uD4fd4E5ooeMPbrvf8esWGqvRHqCjPnY+cJ7W+3wL1PJPxjSwHkLedWVM3jSRr
+KFiayUx8t79kzITavvu860v8dQSbwJCnw3vGziGf4gvdxIUeGhFpAbMkhMA1Gjp7
+iOh1Zu1ShUNWZqSVTfq/+ExotpMoPpoLSnO+7K4XDVorjLTfZ4fN8u6waGr2YXEl
+uRP5Qru1SOEQHDKMPxRMxOe/BIu9SIK1TWmMo44ag2ciShmoivLy0HEuqEeTa+9c
+76oBTGqHLF5P0lvVe3O98hwadROKCJ9tsftosAXjMEw2tEZDOU1oNB12PwJW/Mpa
+gJhA78XRjRE35oyzO/PA83VR060QKwgSA4/pCfXOoojkultFDSX4QTCwSz6PR5c4
+FvLEYYu7tPvlJWVPeMG1tXRcVAamrF/HyuWWkLL/Z7taduQ6bui+v6KBLOwsZNeD
+9eDWlr0wDNhI5gcGg0kgo3usSHKSzzd5DqvunUkgjBpk1a1i8XF4mGi7l3BEIQfR
+YpQKDgNQtxiOKHI32fleV7JuxxDzSybJ1lyWo0pdA4SLwo6KUeoQR13boehkZvga
+MazCqGRhN5flV20sAKHfR9r1AtM1n8qnBU081rNU9rnwS6mE/RFh0EQFKsadCrED
+LsLpJqYc209RwJTbPGGAx2blHXJhJAqIhIUUvZ9cuhmLUkadcalroEleFSbyYrKM
+ah1koTU3s89y1bkksd6HQBNnOeWMKzoNm9zdAKv8OcQZIQ+FtaWoHcySDudxuHCk
+JL6aKYkzZM+0tCjEge2VraOBoDzRSZUNVBxCICOoGYTIxRhwJ0Vu1rG+Zk4Ub30t
+hkETPbQncejeDoUMbq/ESNAhxIe0KLXYXhJOH8qApWBGqJRKt1Lky3tkVKwLqcFA
+zzdHEOvLclPvJGVCNw4yNoakk3TUuQI2AJ2p0Evh9RGMwacFARZ25grePzAI8TJp
+AGoOQpVFPOgNz7ygT7JiUk/L0SntNKGXJnHZr/bdlhhk2QqnWrHnQN52FkuP3ZCd
+G/ApW7G2E2t7V4oS+LO+7pGrBcuokAbDGicr2fhgijnO65VHg9q4SCRrkRyWwiRE
+fIVwlO1COxo6mkWvxeZQ1KLxDKFlpfD7SmNyy7StkDFFwTbUOapb4FSdu3zNN+1U
+27z0WOw1H7Bg5neCSJqxu7FBrILoJIEi19eIhOaxys+9LwFSUFi9GhzeLsQl4eDC
+rEZgLKEIWo0AXyVHptWGci/oD5aJIyitHqK9dpdMKIZyvZLCdo0KSpnRX6fLVEaU
+JdUTxUxsGcQ1FwNCZz2vq3K9Bv6muXIpyp0gSuSSlS+kEK88BcrUrFHNN3NRfz0Y
+tfENObu/675l84fJN8idIZmO7iYCO+wSiDYARG1k+muSMgm/MrHyxT0XiC1vkds5
+K+z1GRkOeJzo0356tBzdTNq9q4u2TGTaFu15azdDnRNPm4BoK/SwhW1AIjOmzSPk
+9MrtFehiwWgQy0RfaMGxbNqZZKjOR0wcA+JEMMxtjUyhxO5GaAZLci9P4qfIWWxt
+HR6KiUywryeYp1eOXzUBliXSyEwh+gIw3FQ58VB7qOYzT51maj+vayOaOjcyr+VC
+bJ4z3IipwOG1YV3wcYkYdO7OHAmXouO9NyiEih5RKlX3UOAxGiQxWsM4XSSJCQTu
+1MOt/hZdYgSt1/N6JeOaQEUPGilWWjEuMZI0bs8jb850iD6ZRDZTIV5ofrrCtoKo
+TBK1DsbL7XMAiTQyaOLXEiWyOYGIeBsKx4Ke29vpmQY2xG7mjgQoQMvtRKraGYYx
+mNxaDtFhesUhFB3gqCRFQnd5UEi4pi5TKkm8l7ObGgNDOJ1tKi2f10CV8GF0g8bE
+10UTV3JOMkE0Si8MSECyV+gDyOqGsIINPnPdocjsTKBp3ahjAiZkkGcnHmSSRGBT
+gHCFOSaWJHu06T446Wn0MYUEiY31VGg96I0x9oLDEF8ThU+Z4BM+R6HVgGVtKfj5
+Uji541mSgnZ0Hp2E82AJS1JiVU9a4vW3OI6Js/cTBbKHkVBxuRgAk185TvTg132l
+Wq9MNbscE0Ylz3Ia2g/RTM8a20PRi88Kbq0CK86Kwb6awm/M/IRqzPb0yabxxs0j
+8zV9zj2keVT0PMQdUeO6xZQjX6fUOX4OqbF7kkNMR2JPGGYIJr+VUBzxnt5m2M1J
+xnqeVbIi5+wK+NFkcAxX4G7jncy6RvrbSgn7wrtz2T8IYplJmHjcxI/HMi+R3kGp
+H+uSCHKRk6W1BVLcjgc6xOp1J47Z1piVx1tYUt7+52IB90MxMObeoBh+xf8oHLRL
+KJRjDuaRRgsM9eFiJ7Y4rzSmvqVFtAzAp2FYsHsvrk7qFiK3Y5I81rXHMOCnO4mh
+cY0lbmmb2ILQCzqmCLRDijJQCnntztAb9QAR9vPobbAov43edprqpiBDUfi0zamN
+Kzp38yur8saXpbbEeHmZ5/XMqSn8wJJnk3efEhEE9si6Grm/QmDmVY3tg/zGw3h7
+G9cbQj3oOkDzHW+tpAorsOaWpGfFBHKij9A9yrnv2KGLbhyflL74/u3NpTBjJOLn
+BGW4nxXiKWticbiyxuJJ5aIn/gRbfsL22rC9e6MxUQTrI3N780ynyO3gFsf68K5y
+fD29qZ1Pd1g63a5FzraF6LZzeyhAXI6vMTgdp6dlS1yeEg+Hac7QymE6RW4XebUe
+D19fh2P24eG3dXd4dVl28jqmJ/+A/SxKfyl2MPGc/n8UO81en8H4CnY74hoJvJ8V
+nrs3KAqpe3ZvDPkmZj7Aho09yqZMFmcKsEFTYLcNbZwKnGCYJEvwvYb2uqdNk4RF
+ATpgYua1jQPGyFa37A1ol9moHShX8TMPccjaDkUPL8XXhg0+A6zTDQT7skXaIBJh
+OVwJXu12p3ri3f6swLVYZ7ra8wadMgL1uN1eWPoG0RGcxMQCqRbeTFBcCh8hRkQg
+swd1DAN+GpEqS9wWRHiuEBQ2HQYg+leJCJDCu8jNgOS1bg/PCzNdCo7eyCZMUdn+
+qleKiUTI5Jke0hGBJaMh+JeIT0YRWRdOSAUNj/rFypE3w87o/J1ZJK2GRKN7DJdc
+HSSO3rNdRJH7qNmW95nRA5li+VyY8dc36Y8eO6yA2zjRVlMM2azVpBh8mabB0zZr
+HWPNrmCtI1p/sFIT26gn1jrcef1Yt6bEEmGGWA+qbbNWE1MZm7UOiVEJrPWQgrUu
+RbBWfFw2a/Wj2UyDtfrVuW3W6oaRGIDumSibw9LuNWpiebBZhjdM7GymwVpNMWfZ
+rHUYDJZNWkPanHXLwVDNstbnGwWTCZTVLZ9zU1YTUX6bsh4+BmU1sUZueZ6MTsq2
+KesRMq5bQKtszuoB59wRnNUUMmVzVs+WPDZnXdm0Oevr9AJnXe9bnNWdeWKsfuPQ
+zVhNxJwYjNXNk3JirG6+bsJq0qCrCWhOqdqJsA7Hx3ZejySZkY4zUJOEdXiQ+ias
+yJm+Ceurh7CYrBVzHvCx8H4oLFamuEHRlZOYe9FEJIq5F6JTGBZzZgkoc8aigdTR
+Ncd1hFbNAFkit49azuuTMyuqzETlcDMSvDGTZQqvcyKQFghNcJItKmFoKTy2cVoB
+zSs9bGE9+hRLWwQvF+XuihmzoR+ZVDF1td5jcwo0RZQn4bGxpE0crK9UomIacyJ3
+FhROtbCInMVGunooJjLKQlBRy4m1OSs5eN6WRPOi3UVPrVHtWz23Ro1OwnX3aMps
+jYU+RAu0bjDDh1WoANcyVpWIIxGBzu2Z+bcUnn/N08cUwnktQlTYn0BC/DZi3NAa
+xgG17OGRPUPph3gakooit7cY32JdyPonkt9iFn02LU/FeDdaeApTH4rnJDZdngqF
+ADrtdCFcIyhKcuQwvDJfkWwa6aMCsq9zZTLyJwDX/ZqhGErHoGCtXmv4tSG7Gk03
+xSDE1s4wAMwaoWaJlvk7DKHoyiAP9mPx6vKYsOjL1NjOjpvBGsxWXUMGE6ifSKFy
+dNyrpdajm1ousvpzuNyStWeS08FMHrzZ4/4afRyQvnu4fPtjvubrwz8uBgjJ/tlP
+Se4XM70n/8L8+3C/fPv9Z8vQz7bj89PLJV0fnvy//1y+/vX2+PL85frr+x+vPz3+
++/HvT58+fPxy/fn58dPLh5d/Xr9/fvny/On614ffLu+MP1kBvsvmxuvD+8vXP/z3
+8f7x9nz9S7m+u371tz9ezif8/nJ9//vTH3f7/vrLx+eXr/yIHx4u/7rk64frpfSK
+aNmfFaReHXosyT89X/4ku+p5EISB6M6vuA1HaK1BBhODLiZiDCQOxgFpCX4EEgv6
+932lRCAOTdrm+vJ67+7aO1Fl2UYJ2CZRDKI7zO645wd2tKfzxSM5gRDMvDV8ISY7
+T0c7R+sl9uclxhHwCCrho1KY3zS8NNtkzZSnz2zKC49bnpx3a8RdzxXwI9S5byKa
+TVFV2IN2ZILOOCD0IUhsU+AFvuseCoGxjQ/pNqS0VGNn5rVU1Golqahf1JQ3TU2m
+H5RnFV1Vpw6aQc+Kg9Jh1CnqtpKEAfnctXxnVY7zqzHsapDIQKB7WyJJRxiDrXZ/
+V6CvAAMAvjaLtA0KZW5kc3RyZWFtDWVuZG9iag0yMyAwIG9iajw8L0xlbmd0aCAy
+NTc1L0ZpbHRlci9GbGF0ZURlY29kZS9OIDMvQWx0ZXJuYXRlL0RldmljZVJHQj4+
+c3RyZWFtDQpIiZyWeVRTdxbHf2/JnpCVsMNjDVuAsAaQNWxhkR0EUQhJCAESQkjY
+BUFEBRRFRISqlTLWbXRGT0WdLq5jrQ7WferSA/Uw6ug4tBbXjp0XOEedTmem0+8f
+7/c593fv793fvfed8wCgJ6WqtdUwCwCN1qDPSozFFhUUYqQJAAMKIAIRADJ5rS4t
+OyEH4JLGS7Ba3An8i55eB5BpvSJMysAw8P+JLdfpDQBAGTgHKJS1cpw7ca6qN+hM
+9hmceaWVJoZRE+vxBHG2NLFqnr3nfOY52sQKjVaBsylnnUKjMPFpnFfXGZU4I6k4
+d9WplfU4X8XZpcqoUeP83BSrUcpqAUDpJrtBKS/H2Q9nuj4nS4LzAgDIdNU7XPoO
+G5QNBtOlJNW6Rr1aVW7A3OUemCg0VIwlKeurlAaDMEMmr5TpFZikWqOTaRsBmL/z
+nDim2mJ4kYNFocHBQn8f0TuF+q+bv1Cm3s7Tk8y5nkH8C29tP+dXPQqAeBavzfq3
+ttItAIyvBMDy5luby/sAMPG+Hb74zn34pnkpNxh0Yb6+9fX1Pmql3MdU0Df6nw6/
+QO+8z8d03JvyYHHKMpmxyoCZ6iavrqo26rFanUyuxIQ/HeJfHfjzeXhnKcuUeqUW
+j8jDp0ytVeHt1irUBnW1FlNr/1MTf2XYTzQ/17i4Y68Br9gHsC7yAPK3CwDl0gBS
+tA3fgd70LZWSBzLwNd/h3vzczwn691PhPtOjVq2ai5Nk5WByo75ufs/0WQICoAIm
+4AErYA+cgTsQAn8QAsJBNIgHySAd5IACsBTIQTnQAD2oBy2gHXSBHrAebALDYDsY
+A7vBfnAQjIOPwQnwR3AefAmugVtgEkyDh2AGPAWvIAgiQQyIC1lBDpAr5AX5Q2Io
+EoqHUqEsqAAqgVSQFjJCLdAKqAfqh4ahHdBu6PfQUegEdA66BH0FTUEPoO+glzAC
+02EebAe7wb6wGI6BU+AceAmsgmvgJrgTXgcPwaPwPvgwfAI+D1+DJ+GH8CwCEBrC
+RxwRISJGJEg6UoiUIXqkFelGBpFRZD9yDDmLXEEmkUfIC5SIclEMFaLhaBKai8rR
+GrQV7UWH0V3oYfQ0egWdQmfQ1wQGwZbgRQgjSAmLCCpCPaGLMEjYSfiIcIZwjTBN
+eEokEvlEATGEmEQsIFYQm4m9xK3EA8TjxEvEu8RZEolkRfIiRZDSSTKSgdRF2kLa
+R/qMdJk0TXpOppEdyP7kBHIhWUvuIA+S95A/JV8m3yO/orAorpQwSjpFQWmk9FHG
+KMcoFynTlFdUNlVAjaDmUCuo7dQh6n7qGept6hMajeZEC6Vl0tS05bQh2u9on9Om
+aC/oHLonXUIvohvp6+gf0o/Tv6I/YTAYboxoRiHDwFjH2M04xfia8dyMa+ZjJjVT
+mLWZjZgdNrts9phJYboyY5hLmU3MQeYh5kXmIxaF5caSsGSsVtYI6yjrBmuWzWWL
+2OlsDbuXvYd9jn2fQ+K4ceI5Ck4n5wPOKc5dLsJ15kq4cu4K7hj3DHeaR+QJeFJe
+Ba+H91veBG/GnGMeaJ5n3mA+Yv6J+SQf4bvxpfwqfh//IP86/6WFnUWMhdJijcV+
+i8sWzyxtLKMtlZbdlgcsr1m+tMKs4q0qrTZYjVvdsUatPa0zreutt1mfsX5kw7MJ
+t5HbdNsctLlpC9t62mbZNtt+YHvBdtbO3i7RTme3xe6U3SN7vn20fYX9gP2n9g8c
+uA6RDmqHAYfPHP6KmWMxWBU2hJ3GZhxtHZMcjY47HCccXzkJnHKdOpwOON1xpjqL
+ncucB5xPOs+4OLikubS47HW56UpxFbuWu252Pev6zE3glu+2ym3c7b7AUiAVNAn2
+Cm67M9yj3GvcR92vehA9xB6VHls9vvSEPYM8yz1HPC96wV7BXmqvrV6XvAneod5a
+71HvG0K6MEZYJ9wrnPLh+6T6dPiM+zz2dfEt9N3ge9b3tV+QX5XfmN8tEUeULOoQ
+HRN95+/pL/cf8b8awAhICGgLOBLwbaBXoDJwW+Cfg7hBaUGrgk4G/SM4JFgfvD/4
+QYhLSEnIeyE3xDxxhrhX/HkoITQ2tC3049AXYcFhhrCDYX8PF4ZXhu8Jv79AsEC5
+YGzB3QinCFnEjojJSCyyJPL9yMkoxyhZ1GjUN9HO0YrondH3YjxiKmL2xTyO9YvV
+x34U+0wSJlkmOR6HxCXGdcdNxHPic+OH479OcEpQJexNmEkMSmxOPJ5ESEpJ2pB0
+Q2onlUt3S2eSQ5KXJZ9OoadkpwynfJPqmapPPZYGpyWnbUy7vdB1oXbheDpIl6Zv
+TL+TIcioyfhDJjEzI3Mk8y9ZoqyWrLPZ3Ozi7D3ZT3Nic/pybuW65xpzT+Yx84ry
+duc9y4/L78+fXOS7aNmi8wXWBeqCI4WkwrzCnYWzi+MXb1o8XRRU1FV0fYlgScOS
+c0utl1Yt/aSYWSwrPlRCKMkv2VPygyxdNiqbLZWWvlc6I5fIN8sfKqIVA4oHyghl
+v/JeWURZf9l9VYRqo+pBeVT5YPkjtUQ9rP62Iqlie8WzyvTKDyt/rMqvOqAha0o0
+R7UcbaX2dLV9dUP1JZ2Xrks3WRNWs6lmRp+i31kL1S6pPWLg4T9TF4zuxpXGqbrI
+upG65/V59Yca2A3ahguNno1rGu81JTT9phltljefbHFsaW+ZWhazbEcr1FraerLN
+ua2zbXp54vJd7dT2yvY/dfh19Hd8vyJ/xbFOu87lnXdXJq7c22XWpe+6sSp81fbV
+6Gr16ok1AWu2rHndrej+osevZ7Dnh1557xdrRWuH1v64rmzdRF9w37b1xPXa9dc3
+RG3Y1c/ub+q/uzFt4+EBbKB74PtNxZvODQYObt9M3WzcPDmU+k8ApAFb/pi4mSSZ
+kJn8mmia1ZtCm6+cHJyJnPedZJ3SnkCerp8dn4uf+qBpoNihR6G2oiailqMGo3aj
+5qRWpMelOKWpphqmi6b9p26n4KhSqMSpN6mpqhyqj6sCq3Wr6axcrNCtRK24ri2u
+oa8Wr4uwALB1sOqxYLHWskuywrM4s660JbSctRO1irYBtnm28Ldot+C4WbjRuUq5
+wro7urW7LrunvCG8m70VvY++Cr6Evv+/er/1wHDA7MFnwePCX8Lbw1jD1MRRxM7F
+S8XIxkbGw8dBx7/IPci8yTrJuco4yrfLNsu2zDXMtc01zbXONs62zzfPuNA50LrR
+PNG+0j/SwdNE08bUSdTL1U7V0dZV1tjXXNfg2GTY6Nls2fHadtr724DcBdyK3RDd
+lt4c3qLfKd+v4DbgveFE4cziU+Lb42Pj6+Rz5PzlhOYN5pbnH+ep6DLovOlG6dDq
+W+rl63Dr++yG7RHtnO4o7rTvQO/M8Fjw5fFy8f/yjPMZ86f0NPTC9VD13vZt9vv3
+ivgZ+Kj5OPnH+lf65/t3/Af8mP0p/br+S/7c/23//wIMAPeE8/sKDQplbmRzdHJl
+YW0NZW5kb2JqDTI0IDAgb2JqPDwvU2l6ZVsyNTVdL0xlbmd0aCA3ODAvRmlsdGVy
+L0ZsYXRlRGVjb2RlL0RlY29kZVswIDEgMCAxIDAgMV0vUmFuZ2VbMCAxIDAgMSAw
+IDFdL0JpdHNQZXJTYW1wbGUgOC9Eb21haW5bMCAxXS9FbmNvZGVbMCAyNTRdL0Z1
+bmN0aW9uVHlwZSAwPj5zdHJlYW0NCkiJAP0CAv0jHyAkICElIiImIyQnJCUoJSYq
+JygrKCksKSotKiwuLC0wLS4xLzAyMDEzMTM0MzQ2NDU3NTc4Nzg5ODo6OTs8Ozw9
+PD4+PT8/PkBAQEFBQUJCQkNDQ0REREVFRUZGRkdHRkhIR0lISEpJSUtKSkxLS01M
+TE5NTU9OTlBPT1FQUFJRUVNSUlRTU1VTVFZUVVdVVlhWV1lXWFpYWVtZWlxaW11b
+XF5bXF9cXWBdXmBeX2FfYGJgYWNgYWRhYmViY2ZjZGZkZWdkZmhlZ2lmZ2pnaGto
+aWtpamxpa21qa25rbG9sbXBtbnBtb3FucHJvcHNwcXRwcnRxc3VydHZzdHd0dXh0
+dnh1d3l2d3p3eHt3eXt4enx5en16e356fH97fX98fYB8foF9f4J+gIJ/gIN/gYSA
+goWBg4WCg4aChIeDhYiEhoiFhomFh4qGiIuHiYuIioyIio2Ji46KjI6LjY+LjZCM
+jpGNj5GOkJKOkJOPkZSQkpWRk5WRk5aSlJeTlZiUlpmVl5qWmJuXmZuXmZyYmp2Z
+m56anJ6anZ+bnaCcnqGdn6GeoKKeoKOfoaSgoqWho6WhpKaipKejpaikpqmlp6ml
+qKqmqKunqayoqq2pq62prK6qrK+rrbCsrrCtr7GtsLKusLOvsbSwsrSxs7WytLay
+tLeztbi0tri1t7m2uLq2uLu3uby4ury5u726vL67vb+7vcC8vsC9v8G+wMK/wcPA
+wcTAwsTBw8XCxMbDxcfExsjFxsjFx8nGyMrHycvIyszJy8zKy83LzM7Lzc/MztDN
+z9DOz9HP0NLP0dPQ0tPR09TS09XT1NbU1dfU1tfV19jW19nX2NrY2dvZ2tvZ29za
+293b3N7c3d7d3t/e3+De3+Hf4OHg4eLh4uPi4+Tj5OXj5OXk5ebl5ufm5+jn6Ojo
+6Onp6erp6uvq6+zr7Ozs7e3t7e7t7u/u7/Dv8PDw8fHx8fLy8vPz8/Pz9PT09fX1
+9vb29vf39/j4+Pn5+fr6+/v8/Pz9/f3///8CDAADYLy/Cg0KZW5kc3RyZWFtDWVu
+ZG9iag0yNSAwIG9iajw8L1R5cGUvRm9udERlc2NyaXB0b3IvRm9udEJCb3hbLTE3
+MCAtMjI4IDEwMDMgOTYyXS9Gb250TmFtZS9IZWx2ZXRpY2EtQm9sZC9GbGFncyAy
+NjIxNzYvU3RlbVYgMTQwL1N0ZW1IIDE0MC9DYXBIZWlnaHQgNzE4L1hIZWlnaHQg
+NTMyL0FzY2VudCA3MTgvRGVzY2VudCAtMjA3L0l0YWxpY0FuZ2xlIDA+Pg1lbmRv
+YmoNMjYgMCBvYmo8PC9UeXBlL0ZvbnREZXNjcmlwdG9yL0ZvbnRCQm94Wy0xNjYg
+LTIyNSAxMDAwIDkzMV0vRm9udE5hbWUvSGVsdmV0aWNhL0ZsYWdzIDMyL1N0ZW1W
+IDg4L1N0ZW1IIDg4L0NhcEhlaWdodCA3MTgvWEhlaWdodCA1MjMvQXNjZW50IDcx
+OC9EZXNjZW50IC0yMDcvSXRhbGljQW5nbGUgMD4+DWVuZG9iag0yNyAwIG9iajw8
+L1R5cGUvRXh0R1N0YXRlL1NBIGZhbHNlL09QIGZhbHNlL1NNIDAuMDIvb3AgZmFs
+c2UvT1BNIDE+Pg1lbmRvYmoNMjggMCBvYmo8PC9UeXBlL0V4dEdTdGF0ZS9TQSB0
+cnVlL09QIGZhbHNlL1NNIDAuMDIvb3AgZmFsc2UvT1BNIDE+Pg1lbmRvYmoNMjkg
+MCBvYmo8PC9QaXRTdG9wIDMwIDAgUj4+DWVuZG9iag0zMCAwIG9iajw8L0NDIDMx
+IDAgUj4+DWVuZG9iag0zMSAwIG9iajw8Pj4NZW5kb2JqDTMyIDAgb2JqPDwvRmll
+bGRzWzM2IDAgUl0vRFI8PC9FbmNvZGluZzw8L1BERkRvY0VuY29kaW5nIDM1IDAg
+Uj4+L0ZvbnQ8PC9IZWx2IDM0IDAgUi9aYURiIDMzIDAgUj4+Pj4vREEoL0hlbHYg
+MCBUZiAwIGcgKT4+DWVuZG9iag0zMyAwIG9iajw8L1R5cGUvRm9udC9OYW1lL1ph
+RGIvQmFzZUZvbnQvWmFwZkRpbmdiYXRzL1N1YnR5cGUvVHlwZTE+Pg1lbmRvYmoN
+MzQgMCBvYmo8PC9UeXBlL0ZvbnQvTmFtZS9IZWx2L0VuY29kaW5nIDM1IDAgUi9C
+YXNlRm9udC9IZWx2ZXRpY2EvU3VidHlwZS9UeXBlMT4+DWVuZG9iag0zNSAwIG9i
+ajw8L1R5cGUvRW5jb2RpbmcvRGlmZmVyZW5jZXNbMjQvYnJldmUvY2Fyb24vY2ly
+Y3VtZmxleC9kb3RhY2NlbnQvaHVuZ2FydW1sYXV0L29nb25lay9yaW5nL3RpbGRl
+IDM5L3F1b3Rlc2luZ2xlIDk2L2dyYXZlIDEyOC9idWxsZXQvZGFnZ2VyL2RhZ2dl
+cmRibC9lbGxpcHNpcy9lbWRhc2gvZW5kYXNoL2Zsb3Jpbi9mcmFjdGlvbi9ndWls
+c2luZ2xsZWZ0L2d1aWxzaW5nbHJpZ2h0L21pbnVzL3BlcnRob3VzYW5kL3F1b3Rl
+ZGJsYmFzZS9xdW90ZWRibGxlZnQvcXVvdGVkYmxyaWdodC9xdW90ZWxlZnQvcXVv
+dGVyaWdodC9xdW90ZXNpbmdsYmFzZS90cmFkZW1hcmsvZmkvZmwvTHNsYXNoL09F
+L1NjYXJvbi9ZZGllcmVzaXMvWmNhcm9uL2RvdGxlc3NpL2xzbGFzaC9vZS9zY2Fy
+b24vemNhcm9uIDE2MC9FdXJvIDE2NC9jdXJyZW5jeSAxNjYvYnJva2VuYmFyIDE2
+OC9kaWVyZXNpcy9jb3B5cmlnaHQvb3JkZmVtaW5pbmUgMTcyL2xvZ2ljYWxub3Qv
+Lm5vdGRlZi9yZWdpc3RlcmVkL21hY3Jvbi9kZWdyZWUvcGx1c21pbnVzL3R3b3N1
+cGVyaW9yL3RocmVlc3VwZXJpb3IvYWN1dGUvbXUgMTgzL3BlcmlvZGNlbnRlcmVk
+L2NlZGlsbGEvb25lc3VwZXJpb3Ivb3JkbWFzY3VsaW5lIDE4OC9vbmVxdWFydGVy
+L29uZWhhbGYvdGhyZWVxdWFydGVycyAxOTIvQWdyYXZlL0FhY3V0ZS9BY2lyY3Vt
+ZmxleC9BdGlsZGUvQWRpZXJlc2lzL0FyaW5nL0FFL0NjZWRpbGxhL0VncmF2ZS9F
+YWN1dGUvRWNpcmN1bWZsZXgvRWRpZXJlc2lzL0lncmF2ZS9JYWN1dGUvSWNpcmN1
+bWZsZXgvSWRpZXJlc2lzL0V0aC9OdGlsZGUvT2dyYXZlL09hY3V0ZS9PY2lyY3Vt
+ZmxleC9PdGlsZGUvT2RpZXJlc2lzL211bHRpcGx5L09zbGFzaC9VZ3JhdmUvVWFj
+dXRlL1VjaXJjdW1mbGV4L1VkaWVyZXNpcy9ZYWN1dGUvVGhvcm4vZ2VybWFuZGJs
+cy9hZ3JhdmUvYWFjdXRlL2FjaXJjdW1mbGV4L2F0aWxkZS9hZGllcmVzaXMvYXJp
+bmcvYWUvY2NlZGlsbGEvZWdyYXZlL2VhY3V0ZS9lY2lyY3VtZmxleC9lZGllcmVz
+aXMvaWdyYXZlL2lhY3V0ZS9pY2lyY3VtZmxleC9pZGllcmVzaXMvZXRoL250aWxk
+ZS9vZ3JhdmUvb2FjdXRlL29jaXJjdW1mbGV4L290aWxkZS9vZGllcmVzaXMvZGl2
+aWRlL29zbGFzaC91Z3JhdmUvdWFjdXRlL3VjaXJjdW1mbGV4L3VkaWVyZXNpcy95
+YWN1dGUvdGhvcm4veWRpZXJlc2lzXT4+DWVuZG9iag0zNiAwIG9iajw8L0YgNC9U
+eXBlL0Fubm90L1JlY3RbMjYwLjI1MTAwNyA1MDIuNDk4OTAxIDM2OS43NTE0MDQg
+NTQwLjc0OTAyM10vRlQvVHgvU3VidHlwZS9XaWRnZXQvUCA5IDAgUi9UKHRvZGF5
+c0RhdGUpL1EgMS9EQSgvSGVsdiAxMiBUZiAwIGcpL01LPDw+Pj4+DWVuZG9iag0z
+NyAwIG9ialszNiAwIFJdDWVuZG9iag0zOCAwIG9iajw8L0phdmFTY3JpcHQgMzkg
+MCBSPj4NZW5kb2JqDTM5IDAgb2JqPDwvTmFtZXNbKGRvY09wZW5lZCk0MCAwIFJd
+Pj4NZW5kb2JqDTQwIDAgb2JqPDwvUy9KYXZhU2NyaXB0L0pTIDQyIDAgUj4+DWVu
+ZG9iag00MSAwIG9iaiAzMTkNZW5kb2JqDTQyIDAgb2JqPDwvTGVuZ3RoIDQxIDAg
+Ui9GaWx0ZXJbL0ZsYXRlRGVjb2RlXT4+c3RyZWFtDQpIiXRSQWrDMBA82+A/bHWy
+odgPCDkESq99wzraxCqKZKRVQij5Q5/clVxDklKBDxrN7IxGHgY4YzA4WgL2ENkH
+gstEPFEA7ffpRI5hwggjkQM/kyMNaAOhvoIP4Dw3tYyAcbeAH4WyaeqmPiS3Z+Nd
+HrTAbdfUX/moMof2UQEvW1AcEinhVEKqqmF4jPCZIt/nyJxsrWELji7whkxtt1nh
+mPdylNjYfg7GsW7V6TRoPVxlqVfQXckJT0t8IzHoLHf+8peA89yjpcCt2o0+ca7O
+uCjAIhIrDwdDVme9WiLxZGJ/JH7PeKvYa7yWiKrrz2hTjlr2JVO5vYhLkqemZHgu
+CqK4iqOn6L75VxKSAzyicU39pFrrzWFu8pGN9F/T6wM/li2qW84m7D1aC/KT3D8t
+rO/d1Heo2P0IMACaArkEDQplbmRzdHJlYW0NZW5kb2JqDTQzIDAgb2JqPDwvTGVu
+Z3RoIDM0MjcvVHlwZS9NZXRhZGF0YS9TdWJ0eXBlL1hNTD4+c3RyZWFtDQo8P3hw
+YWNrZXQgYmVnaW49J++7vycgaWQ9J1c1TTBNcENlaGlIenJlU3pOVGN6a2M5ZCc/
+Pgo8P2Fkb2JlLXhhcC1maWx0ZXJzIGVzYz0iQ1JMRiI/Pg0KPHg6eG1wbWV0YSB4
+bWxuczp4PSdhZG9iZTpuczptZXRhLycgeDp4bXB0az0nWE1QIHRvb2xraXQgMi45
+LjEtMTQsIGZyYW1ld29yayAxLjYnPg0KPHJkZjpSREYgeG1sbnM6cmRmPSdodHRw
+Oi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjJyB4bWxuczpp
+WD0naHR0cDovL25zLmFkb2JlLmNvbS9pWC8xLjAvJz4NCjxyZGY6RGVzY3JpcHRp
+b24gcmRmOmFib3V0PSd1dWlkOjI4NWVhMTI5LWZlYmUtNDgyYS1iYzFhLTVkYTZj
+ZWIwNTg2NicgeG1sbnM6cGRmPSdodHRwOi8vbnMuYWRvYmUuY29tL3BkZi8xLjMv
+JyBwZGY6UHJvZHVjZXI9J0Fjcm9iYXQgRGlzdGlsbGVyIDYuMC4xIChXaW5kb3dz
+KSc+PC9yZGY6RGVzY3JpcHRpb24+DQo8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91
+dD0ndXVpZDoyODVlYTEyOS1mZWJlLTQ4MmEtYmMxYS01ZGE2Y2ViMDU4NjYnIHht
+bG5zOnhhcD0naHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wLycgeGFwOk1vZGlm
+eURhdGU9JzIwMDQtMTAtMjhUMTg6MTY6NDUrMTA6MDAnIHhhcDpDcmVhdGVEYXRl
+PScyMDA0LTEwLTI4VDE4OjEzOjM4KzEwOjAwJyB4YXA6TWV0YWRhdGFEYXRlPScy
+MDA0LTEwLTI4VDE4OjE2OjQ1KzEwOjAwJyB4YXA6Q3JlYXRvclRvb2w9J1BTY3Jp
+cHQ1LmRsbCBWZXJzaW9uIDUuMi4yJz48L3JkZjpEZXNjcmlwdGlvbj4NCjxyZGY6
+RGVzY3JpcHRpb24gcmRmOmFib3V0PSd1dWlkOjI4NWVhMTI5LWZlYmUtNDgyYS1i
+YzFhLTVkYTZjZWIwNTg2NicgeG1sbnM6eGFwTU09J2h0dHA6Ly9ucy5hZG9iZS5j
+b20veGFwLzEuMC9tbS8nIHhhcE1NOkRvY3VtZW50SUQ9J3V1aWQ6MmNhMjBjZmEt
+M2FlZi00MTU1LThkOWMtYTM4ZWJjNDM3MjVmJy8+DQo8cmRmOkRlc2NyaXB0aW9u
+IHJkZjphYm91dD0ndXVpZDoyODVlYTEyOS1mZWJlLTQ4MmEtYmMxYS01ZGE2Y2Vi
+MDU4NjYnIHhtbG5zOmRjPSdodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4x
+LycgZGM6Zm9ybWF0PSdhcHBsaWNhdGlvbi9wZGYnPjxkYzp0aXRsZT48cmRmOkFs
+dD48cmRmOmxpIHhtbDpsYW5nPSd4LWRlZmF1bHQnPlBsYW5ldCBQREYgSmF2YVNj
+cmlwdCBMZWFybmluZyBDZW50ZXIgRXhhbXBsZSAjMjwvcmRmOmxpPjwvcmRmOkFs
+dD48L2RjOnRpdGxlPjxkYzpjcmVhdG9yPjxyZGY6U2VxPjxyZGY6bGk+Q2hyaXMg
+RGFobCwgQVJUUyBQREYgR2xvYmFsIFNlcnZpY2VzPC9yZGY6bGk+PC9yZGY6U2Vx
+PjwvZGM6Y3JlYXRvcj48L3JkZjpEZXNjcmlwdGlvbj4NCjwvcmRmOlJERj4NCjwv
+eDp4bXBtZXRhPg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAog
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgIAo8P3hw
+YWNrZXQgZW5kPSd3Jz8+DQplbmRzdHJlYW0NZW5kb2JqDXhyZWYNCjAgNDQNCjAw
+MDAwMDAwMDQgNjU1MzUgZg0KMDAwMDAwMDAxNiAwMDAwMCBuDQowMDAwMDAwMDQ5
+IDAwMDAwIG4NCjAwMDAwMDAwNzIgMDAwMDAgbg0KMDAwMDAwMDAwNiAwMDAwMSBm
+DQowMDAwMDAwMTIyIDAwMDAwIG4NCjAwMDAwMDAwMDggMDAwMDEgZg0KMDAwMDAw
+MDM5NSAwMDAwMCBuDQowMDAwMDAwMDAwIDAwMDAxIGYNCjAwMDAwMDA1MjEgMDAw
+MDAgbg0KMDAwMDAwMDc0NiAwMDAwMCBuDQowMDAwMDAwODg3IDAwMDAwIG4NCjAw
+MDAwMDA5MjEgMDAwMDAgbg0KMDAwMDAwODc2MCAwMDAwMCBuDQowMDAwMDE3NTcz
+IDAwMDAwIG4NCjAwMDAwMjU5MjIgMDAwMDAgbg0KMDAwMDAzNDE5OSAwMDAwMCBu
+DQowMDAwMDQyNTAzIDAwMDAwIG4NCjAwMDAwNTA3NTggMDAwMDAgbg0KMDAwMDA1
+MTI2MyAwMDAwMCBuDQowMDAwMDUxNzU2IDAwMDAwIG4NCjAwMDAwNTE4MTIgMDAw
+MDAgbg0KMDAwMDA2MDI5MiAwMDAwMCBuDQowMDAwMDY4OTA5IDAwMDAwIG4NCjAw
+MDAwNzE1NzggMDAwMDAgbg0KMDAwMDA3MjUzMyAwMDAwMCBuDQowMDAwMDcyNzIz
+IDAwMDAwIG4NCjAwMDAwNzI5MDIgMDAwMDAgbg0KMDAwMDA3Mjk3OCAwMDAwMCBu
+DQowMDAwMDczMDUzIDAwMDAwIG4NCjAwMDAwNzMwODggMDAwMDAgbg0KMDAwMDA3
+MzExOCAwMDAwMCBuDQowMDAwMDczMTM4IDAwMDAwIG4NCjAwMDAwNzMyNjggMDAw
+MDAgbg0KMDAwMDA3MzM0NCAwMDAwMCBuDQowMDAwMDczNDMzIDAwMDAwIG4NCjAw
+MDAwNzQ2MjQgMDAwMDAgbg0KMDAwMDA3NDc4MyAwMDAwMCBuDQowMDAwMDc0ODA3
+IDAwMDAwIG4NCjAwMDAwNzQ4NDUgMDAwMDAgbg0KMDAwMDA3NDg5MCAwMDAwMCBu
+DQowMDAwMDc0OTMzIDAwMDAwIG4NCjAwMDAwNzQ5NTMgMDAwMDAgbg0KMDAwMDA3
+NTM0NiAwMDAwMCBuDQp0cmFpbGVyDQo8PC9TaXplIDQ0L1Jvb3QgNyAwIFIvSW5m
+byA1IDAgUi9JRFs8NTQ4YTEwMTkxOWUxNmI1ZmRhOTMyZTcyZmFjNDYyYjc+PDBm
+MzAxYmVmZTJkMzA2NGU5OWZkODRjZWUxNjAyOGM4Pl0+Pg0Kc3RhcnR4cmVmDQo3
+ODg1MA0KJSVFT0YNCjUgMCBvYmo8PC9Nb2REYXRlKEQ6MjAwNDEwMjgxODE3NDYr
+MTAnMDAnKS9DcmVhdGlvbkRhdGUoRDoyMDA0MTAyODE4MTMzOCsxMCcwMCcpL1Rp
+dGxlKFBsYW5ldCBQREYgSmF2YVNjcmlwdCBMZWFybmluZyBDZW50ZXIgRXhhbXBs
+ZSAjMikvQ3JlYXRvcihQU2NyaXB0NS5kbGwgVmVyc2lvbiA1LjIuMikvQXV0aG9y
+KENocmlzIERhaGwsIEFSVFMgUERGIEdsb2JhbCBTZXJ2aWNlcykvUHJvZHVjZXIo
+QWNyb2JhdCBEaXN0aWxsZXIgNi4wLjEgXChXaW5kb3dzXCkpPj4NZW5kb2JqDTcg
+MCBvYmo8PC9QYWdlcyAzIDAgUi9UeXBlL0NhdGFsb2cvT3BlbkFjdGlvbiA0NiAw
+IFIvTmFtZXMgMzggMCBSL1BhZ2VMYWJlbHMgMSAwIFIvQWNyb0Zvcm0gMzIgMCBS
+L01ldGFkYXRhIDQ3IDAgUi9GSUNMOkVuZm9jdXMgMjkgMCBSPj4NZW5kb2JqDTM2
+IDAgb2JqPDwvRiA0L1R5cGUvQW5ub3QvUmVjdFsyNjAuMjUxMDA3IDUwMi40OTg5
+MDEgMzY5Ljc1MTQwNCA1NDAuNzQ5MDIzXS9GVC9UeC9TdWJ0eXBlL1dpZGdldC9Q
+IDkgMCBSL1QodG9kYXlzRGF0ZSkvVigxMC8yOC8yMDA0KS9BUDw8L04gNDUgMCBS
+Pj4vUSAxL0RBKC9IZWx2IDEyIFRmIDAgZykvTUs8PD4+Pj4NZW5kb2JqDTQ0IDAg
+b2JqIG51bGwNZW5kb2JqDTQ1IDAgb2JqPDwvTGVuZ3RoIDExNS9UeXBlL1hPYmpl
+Y3QvQkJveFswLjAgMC4wIDEwOS41MDAzOTcgMzguMjUwMTIyXS9SZXNvdXJjZXM8
+PC9Gb250PDwvSGVsdiAzNCAwIFI+Pi9Qcm9jU2V0Wy9QREYvVGV4dF0+Pi9TdWJ0
+eXBlL0Zvcm0vRm9ybVR5cGUgMS9NYXRyaXhbMS4wIDAuMCAwLjAgMS4wIDAuMCAw
+LjBdPj5zdHJlYW0NCi9UeCBCTUMgCnEKMSAxIDEwNy41MDA0IDM2LjI1MDEgcmUK
+VwpuCkJUCi9IZWx2IDEyIFRmCjIgMTQuNjczIFRkCjEzLjg3MjEgVEwKMjIuNzI2
+MiAwIFRkCigxMC8yOC8yMDA0KSBUagpFVApRCkVNQwoNCmVuZHN0cmVhbQ1lbmRv
+YmoNNDYgMCBvYmo8PC9EWzkgMCBSL1hZWiAtMzI3NjggLTMyNzY4IDEuMF0vUy9H
+b1RvPj4NZW5kb2JqDTQ3IDAgb2JqPDwvTGVuZ3RoIDM0MjcvVHlwZS9NZXRhZGF0
+YS9TdWJ0eXBlL1hNTD4+c3RyZWFtDQo8P3hwYWNrZXQgYmVnaW49J++7vycgaWQ9
+J1c1TTBNcENlaGlIenJlU3pOVGN6a2M5ZCc/Pgo8P2Fkb2JlLXhhcC1maWx0ZXJz
+IGVzYz0iQ1JMRiI/Pg0KPHg6eG1wbWV0YSB4bWxuczp4PSdhZG9iZTpuczptZXRh
+LycgeDp4bXB0az0nWE1QIHRvb2xraXQgMi45LjEtMTQsIGZyYW1ld29yayAxLjYn
+Pg0KPHJkZjpSREYgeG1sbnM6cmRmPSdodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAy
+LzIyLXJkZi1zeW50YXgtbnMjJyB4bWxuczppWD0naHR0cDovL25zLmFkb2JlLmNv
+bS9pWC8xLjAvJz4NCjxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSd1dWlkOjY5
+YmJhMDBhLTdjYzUtNGEwMS04MjNmLTUxMzM0Y2EzZmEwNicgeG1sbnM6cGRmPSdo
+dHRwOi8vbnMuYWRvYmUuY29tL3BkZi8xLjMvJyBwZGY6UHJvZHVjZXI9J0Fjcm9i
+YXQgRGlzdGlsbGVyIDYuMC4xIChXaW5kb3dzKSc+PC9yZGY6RGVzY3JpcHRpb24+
+DQo8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0ndXVpZDo2OWJiYTAwYS03Y2M1
+LTRhMDEtODIzZi01MTMzNGNhM2ZhMDYnIHhtbG5zOnhhcD0naHR0cDovL25zLmFk
+b2JlLmNvbS94YXAvMS4wLycgeGFwOk1vZGlmeURhdGU9JzIwMDQtMTAtMjhUMTg6
+MTc6NDYrMTA6MDAnIHhhcDpDcmVhdGVEYXRlPScyMDA0LTEwLTI4VDE4OjEzOjM4
+KzEwOjAwJyB4YXA6TWV0YWRhdGFEYXRlPScyMDA0LTEwLTI4VDE4OjE3OjQ2KzEw
+OjAwJyB4YXA6Q3JlYXRvclRvb2w9J1BTY3JpcHQ1LmRsbCBWZXJzaW9uIDUuMi4y
+Jz48L3JkZjpEZXNjcmlwdGlvbj4NCjxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0
+PSd1dWlkOjY5YmJhMDBhLTdjYzUtNGEwMS04MjNmLTUxMzM0Y2EzZmEwNicgeG1s
+bnM6eGFwTU09J2h0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8nIHhhcE1N
+OkRvY3VtZW50SUQ9J3V1aWQ6MmNhMjBjZmEtM2FlZi00MTU1LThkOWMtYTM4ZWJj
+NDM3MjVmJy8+DQo8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0ndXVpZDo2OWJi
+YTAwYS03Y2M1LTRhMDEtODIzZi01MTMzNGNhM2ZhMDYnIHhtbG5zOmRjPSdodHRw
+Oi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLycgZGM6Zm9ybWF0PSdhcHBsaWNh
+dGlvbi9wZGYnPjxkYzp0aXRsZT48cmRmOkFsdD48cmRmOmxpIHhtbDpsYW5nPSd4
+LWRlZmF1bHQnPlBsYW5ldCBQREYgSmF2YVNjcmlwdCBMZWFybmluZyBDZW50ZXIg
+RXhhbXBsZSAjMjwvcmRmOmxpPjwvcmRmOkFsdD48L2RjOnRpdGxlPjxkYzpjcmVh
+dG9yPjxyZGY6U2VxPjxyZGY6bGk+Q2hyaXMgRGFobCwgQVJUUyBQREYgR2xvYmFs
+IFNlcnZpY2VzPC9yZGY6bGk+PC9yZGY6U2VxPjwvZGM6Y3JlYXRvcj48L3JkZjpE
+ZXNjcmlwdGlvbj4NCjwvcmRmOlJERj4NCjwveDp4bXBtZXRhPg0KICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAg
+ICAgICAgICAgICAgICAgICAgICAgIAo8P3hwYWNrZXQgZW5kPSd3Jz8+DQplbmRz
+dHJlYW0NZW5kb2JqDXhyZWYNCjUgMQ0KMDAwMDA3OTg4NSAwMDAwMCBuDQo3IDEN
+CjAwMDAwODAxNTggMDAwMDAgbg0KMzYgMQ0KMDAwMDA4MDMwMiAwMDAwMCBuDQo0
+NCA0DQowMDAwMDgwNDkxIDAwMDAwIG4NCjAwMDAwODA1MTIgMDAwMDAgbg0KMDAw
+MDA4MDgzNSAwMDAwMCBuDQowMDAwMDgwODkzIDAwMDAwIG4NCnRyYWlsZXINCjw8
+L1NpemUgNDgvUm9vdCA3IDAgUi9JbmZvIDUgMCBSL0lEWzw1NDhhMTAxOTE5ZTE2
+YjVmZGE5MzJlNzJmYWM0NjJiNz48ZTVlN2QzZmFiM2MyYzA0MDhmOTZmZDU1ZmE2
+NzgzZWI+XS9QcmV2IDc4ODUwID4+DQpzdGFydHhyZWYNCjg0Mzk3DQolJUVPRg0K
+
diff --git a/test/functional/messages/phish_sender.eml b/test/functional/messages/phish_sender.eml
new file mode 100644
index 0000000..5f5dabb
--- /dev/null
+++ b/test/functional/messages/phish_sender.eml
@@ -0,0 +1,15 @@
+Received: by mail-lf1-f54.gulugulu.com with SMTP id j14so14439709lfg.9
+ for <test@test.ru>; Mon, 27 Apr 2020 09:54:21 -0700 (PDT)
+From: <any@attack.com>
+From: <admin@legitimate.com>
+Date: Mon, 27 Apr 2020 19:54:10 +0300
+Message-ID: <CA+1S=h4aGimA6vSBJF=t1F+5z-Mua5+Cimf+NU_NDWJk8ZNOcw@mail.gmail.com>
+Subject: Fwd:
+To: <test@test.ru>
+Content-Type: multipart/alternative; boundary="00000000000004de7805a4489190"
+
+--0000000000004bee6805a4484c02
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+sadsad
diff --git a/test/functional/messages/phish_sender2.eml b/test/functional/messages/phish_sender2.eml
new file mode 100644
index 0000000..1cb9688
--- /dev/null
+++ b/test/functional/messages/phish_sender2.eml
@@ -0,0 +1,16 @@
+Received: by mail-lf1-f54.gulugulu.com with SMTP id j14so14439709lfg.9
+ for <test@test.ru>; Mon, 27 Apr 2020 09:54:21 -0700 (PDT)
+From
+ : <any@attack.com>
+From: <admin@legitimate.com>
+Date: Mon, 27 Apr 2020 19:54:10 +0300
+Message-ID: <CA+1S=h4aGimA6vSBJF=t1F+5z-Mua5+Cimf+NU_NDWJk8ZNOcw@mail.gmail.com>
+Subject: Fwd:
+To: <test@test.ru>
+Content-Type: multipart/alternative; boundary="00000000000004de7805a4489190"
+
+--0000000000004bee6805a4484c02
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+sadsad \ No newline at end of file
diff --git a/test/functional/messages/phish_sender3.eml b/test/functional/messages/phish_sender3.eml
new file mode 100644
index 0000000..13b1d79
--- /dev/null
+++ b/test/functional/messages/phish_sender3.eml
@@ -0,0 +1,15 @@
+Received: by mail-lf1-f54.gulugulu.com with SMTP id j14so14439709lfg.9
+ for <test@test.ru>; Mon, 27 Apr 2020 09:54:21 -0700 (PDT)
+From: <any@attack.com>
+From : <admin@legitimate.com>
+Date: Mon, 27 Apr 2020 19:54:10 +0300
+Message-ID: <CA+1S=h4aGimA6vSBJF=t1F+5z-Mua5+Cimf+NU_NDWJk8ZNOcw@mail.gmail.com>
+Subject: Fwd:
+To: <test@test.ru>
+Content-Type: multipart/alternative; boundary="00000000000004de7805a4489190"
+
+--0000000000004bee6805a4484c02
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+sadsad
diff --git a/test/functional/messages/phish_sender4.eml b/test/functional/messages/phish_sender4.eml
new file mode 100644
index 0000000..64830fe
--- /dev/null
+++ b/test/functional/messages/phish_sender4.eml
@@ -0,0 +1,16 @@
+Received: by mail-lf1-f54.gulugulu.com with SMTP id j14so14439709lfg.9
+ for <test@test.ru>; Mon, 27 Apr 2020 09:54:21 -0700 (PDT)
+From
+ : <any@attack.com>
+Sender: <admin@legitimate.com>
+Date: Mon, 27 Apr 2020 19:54:10 +0300
+Message-ID: <CA+1S=h4aGimA6vSBJF=t1F+5z-Mua5+Cimf+NU_NDWJk8ZNOcw@mail.gmail.com>
+Subject: Fwd:
+To: <test@test.ru>
+Content-Type: multipart/alternative; boundary="00000000000004de7805a4489190"
+
+--0000000000004bee6805a4484c02
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+sadsad
diff --git a/test/functional/messages/phish_sender5.eml b/test/functional/messages/phish_sender5.eml
new file mode 100644
index 0000000..cf7bfdb
--- /dev/null
+++ b/test/functional/messages/phish_sender5.eml
@@ -0,0 +1,15 @@
+Received: by mail-lf1-f54.gulugulu.com with SMTP id j14so14439709lfg.9
+ for <test@test.ru>; Mon, 27 Apr 2020 09:54:21 -0700 (PDT)
+From: <any@attack.com>, <any2@attack.com>
+Sender: <admin@legitimate.com>
+Date: Mon, 27 Apr 2020 19:54:10 +0300
+Message-ID: <CA+1S=h4aGimA6vSBJF=t1F+5z-Mua5+Cimf+NU_NDWJk8ZNOcw@mail.gmail.com>
+Subject: Fwd:
+To: <test@test.ru>
+Content-Type: multipart/alternative; boundary="00000000000004de7805a4489190"
+
+--0000000000004bee6805a4484c02
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+sadsad \ No newline at end of file
diff --git a/test/functional/messages/phish_sender6.eml b/test/functional/messages/phish_sender6.eml
new file mode 100644
index 0000000..d50c434
--- /dev/null
+++ b/test/functional/messages/phish_sender6.eml
@@ -0,0 +1,14 @@
+Received: by mail-lf1-f54.gulugulu.com with SMTP id j14so14439709lfg.9
+ for <test@test.ru>; Mon, 27 Apr 2020 09:54:21 -0700 (PDT)
+From: <@test.com, @test.ru:test@attack.com>
+Date: Mon, 27 Apr 2020 19:54:10 +0300
+Message-ID: <CA+1S=h4aGimA6vSBJF=t1F+5z-Mua5+Cimf+NU_NDWJk8ZNOcw@mail.gmail.com>
+Subject: Fwd:
+To: <test@test.ru>
+Content-Type: multipart/alternative; boundary="00000000000004de7805a4489190"
+
+--0000000000004bee6805a4484c02
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+sadsad
diff --git a/test/functional/messages/phishing1.eml b/test/functional/messages/phishing1.eml
new file mode 100644
index 0000000..d168068
--- /dev/null
+++ b/test/functional/messages/phishing1.eml
@@ -0,0 +1,3 @@
+Content-type: text/html
+
+lol <a href="http://www.example.net">http://www.cnn.com</a>
diff --git a/test/functional/messages/phishing2.eml b/test/functional/messages/phishing2.eml
new file mode 100644
index 0000000..575dd0b
--- /dev/null
+++ b/test/functional/messages/phishing2.eml
@@ -0,0 +1,3 @@
+Content-type: text/html
+
+lol <a href="http://www.example.net">http://www.myspace.com</a>
diff --git a/test/functional/messages/phishing3.eml b/test/functional/messages/phishing3.eml
new file mode 100644
index 0000000..8ec9af5
--- /dev/null
+++ b/test/functional/messages/phishing3.eml
@@ -0,0 +1,3 @@
+Content-type: text/html
+
+lol <a href="http://www.example.net">http://www.bank.com</a>
diff --git a/test/functional/messages/priority.eml b/test/functional/messages/priority.eml
new file mode 100644
index 0000000..2a533ef
--- /dev/null
+++ b/test/functional/messages/priority.eml
@@ -0,0 +1,5 @@
+From: user@test.com
+To: undisclosed-recipients;;
+
+Content-Transfer-Encoding: base64
+asdasdlsadklsad \ No newline at end of file
diff --git a/test/functional/messages/rar-date-bad-ext.eml b/test/functional/messages/rar-date-bad-ext.eml
new file mode 100644
index 0000000..309efec
--- /dev/null
+++ b/test/functional/messages/rar-date-bad-ext.eml
@@ -0,0 +1,10 @@
+Content-Type: multipart/mixed; boundary="--=_CDA2A73617693A02A72D9869"
+
+----=_CDA2A73617693A02A72D9869
+Content-Type: application/octet-stream; name="06.12.2017.gz"
+Content-Disposition: attachment; filename="06.12.2017.gz"
+Content-Transfer-Encoding: base64
+
+UmFyIRoHAM+QcwAADQAAAAAAAAATSHQgkDMAAgAAAAIAAAACEs1KfkazhksdMA4AICAAADA2
+LjEyLjIwMTcuc2NyALB/OjwwCsQ9ewBABwA=
+----=_CDA2A73617693A02A72D9869--
diff --git a/test/functional/messages/rar4.eml b/test/functional/messages/rar4.eml
new file mode 100644
index 0000000..61f6836
--- /dev/null
+++ b/test/functional/messages/rar4.eml
@@ -0,0 +1,10 @@
+Content-Type: multipart/mixed; boundary="=_hqO3MQBWZkrB1Zd_nWFL3XM"
+
+--=_hqO3MQBWZkrB1Zd_nWFL3XM
+Content-Type: application/x-rar-compressed; name=f.rar
+Content-Disposition: attachment; size=68; filename=f.rar
+Content-Transfer-Encoding: base64
+
+UmFyIRoHAM+QcwAADQAAAAAAAADSkHQggCgAAQAAAAEAAAADkwbXMiqO6UgdMAgAtIEAAGZha2Uu
+ZXhlCsQ9ewBABwA=
+--=_hqO3MQBWZkrB1Zd_nWFL3XM--
diff --git a/test/functional/messages/rcvd7.eml b/test/functional/messages/rcvd7.eml
new file mode 100644
index 0000000..2da00de
--- /dev/null
+++ b/test/functional/messages/rcvd7.eml
@@ -0,0 +1,34 @@
+Return-Path: <test@test.ru>
+Received: from mx7.mail.testler.ru ([] verified)
+ by mail2006.testler.ru (testlerMail 6.1 SMTP 6.1a7)
+ with ESMTP id 60578244 for test@test.com; Mon, 27 Apr 2020 19:45:05 +0300
+Received: from forward106p.mail.test.net (forward106p.mail.test.net [])
+ by mx7.mail.testler.ru (Postfix) with ESMTP id 9B38C6E043A
+ for <test@test.com>; Mon, 27 Apr 2020 19:45:05 +0300 (MSK)
+Received: from forward101p.mail.test.net (forward106p.mail.test.net [])
+ by mx7.mail.testler.ru (Postfix) with ESMTP id 9B38C6E043A
+ for <test@test.com>; Mon, 27 Apr 2020 19:45:05 +0300 (MSK)
+Received: from forward100p.mail.test.net (forward106p.mail.test.net [])
+ by mx7.mail.testler.ru (Postfix) with ESMTP id 9B38C6E043A
+ for <test@test.com>; Mon, 27 Apr 2020 19:45:05 +0300 (MSK)
+Received: from forward106p.mail.test.net (forward106p.mail.test.net [])
+ by resmtp2.mail.testler.ru (resmtp/testler) with ESMTP id j51GAaUB;
+ Mon, 27 Apr 2020 16:45:05 +0000
+Received: from forward102q.mail.test.net (forward102q.mail.test.net [])
+ by forward106p.mail.test.net (test) with ESMTP id 71D441C81C0C
+ for <test@test.com>; Mon, 27 Apr 2020 19:45:05 +0300 (MSK)
+Received: from mxback10q.mail.test.net (mxback10q.mail.test.net [])
+ by forward102q.mail.test.net (test) with ESMTP id 6E2CA7F20010
+ for <test@test.com>; Mon, 27 Apr 2020 19:45:05 +0300 (MSK)
+From: test@test.ru
+Envelope-From: test@test.ru
+To: <test@test.ru>,
+ <test@test.com>
+In-Reply-To: <878111588005719@mail.test.ru>
+Subject: Fwd: 123
+MIME-Version: 1.0
+X-Mailer: test [ http://test.ru ] 5.0
+Date: Mon, 27 Apr 2020 19:45:05 +0300
+Message-Id: <906211588005891@mail.test.ru>
+Content-Type: text/html; charset=utf-8
+Content-Transfer-Encoding: quoted-printable \ No newline at end of file
diff --git a/test/functional/messages/received1.eml b/test/functional/messages/received1.eml
new file mode 100644
index 0000000..6cc5529
--- /dev/null
+++ b/test/functional/messages/received1.eml
@@ -0,0 +1,16 @@
+Received: from server.chat-met-vreemden.nl (unknown [IPv6:2a01:7c8:aab6:26d:5054:ff:fed1:1da2])
+ (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
+ (Client did not present a certificate)
+ by mx1.freebsd.org (Postfix) with ESMTPS id CF0171862
+ for <test@example.com>; Mon, 6 Jul 2015 09:01:20 +0000 (UTC)
+ (envelope-from upwest201diana@outlook.com)
+Received: from ca-18-193-131.service.infuturo.it ([151.18.193.131] helo=User)
+ by server.chat-met-vreemden.nl with esmtpa (Exim 4.76)
+ (envelope-from <upwest201diana@outlook.com>)
+ id 1ZC1sl-0006b4-TU; Mon, 06 Jul 2015 10:36:08 +0200
+Received: apparently
+Received: or is it
+MIME-Version: 1.0
+Content-Type: text/plain
+
+Hello
diff --git a/test/functional/messages/received2.eml b/test/functional/messages/received2.eml
new file mode 100644
index 0000000..b5587e8
--- /dev/null
+++ b/test/functional/messages/received2.eml
@@ -0,0 +1,16 @@
+Received: apparently
+Received: or is it
+Received: from server.chat-met-vreemden.nl (unknown [IPv6:2a01:7c8:aab6:26d:5054:ff:fed1:1da2])
+ (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
+ (Client did not present a certificate)
+ by mx1.freebsd.org (Postfix) with ESMTPS id CF0171862
+ for <test@example.com>; Mon, 6 Jul 2015 09:01:20 +0000 (UTC)
+ (envelope-from upwest201diana@outlook.com)
+Received: from ca-18-193-131.service.infuturo.it ([151.18.193.131] helo=User)
+ by server.chat-met-vreemden.nl with esmtpa (Exim 4.76)
+ (envelope-from <upwest201diana@outlook.com>)
+ id 1ZC1sl-0006b4-TU; Mon, 06 Jul 2015 10:36:08 +0200
+MIME-Version: 1.0
+Content-Type: text/plain
+
+Hello
diff --git a/test/functional/messages/received3.eml b/test/functional/messages/received3.eml
new file mode 100644
index 0000000..9fea8ee
--- /dev/null
+++ b/test/functional/messages/received3.eml
@@ -0,0 +1,16 @@
+Received: apparently
+Received: or is it
+Received: from server.chat-met-vreemden.nl (unknown [IPv6:2a01:7c8:aab6:26d:5054:ff:fed1:1da2])
+ (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
+ (Client did not present a certificate)
+ by mx1.freebsd.org (Postfix) with ESMTPS id CF0171862
+ for <test@example.com>; Mon, 6 Jul 2015 09:01:20 +0000 (UTC)
+ (envelope-from upwest201diana@outlook.com)
+Received: from ca-18-193-131.service.infuturo.it ([151.18.193.131] helo=User)
+ by server.chat-met-vreemden.nl with esmtp (Exim 4.76)
+ (envelope-from <upwest201diana@outlook.com>)
+ id 1ZC1sl-0006b4-TU; Mon, 06 Jul 2015 10:36:08 +0200
+MIME-Version: 1.0
+Content-Type: text/plain
+
+Hello
diff --git a/test/functional/messages/received4.eml b/test/functional/messages/received4.eml
new file mode 100644
index 0000000..d34ebb1
--- /dev/null
+++ b/test/functional/messages/received4.eml
@@ -0,0 +1,16 @@
+Received: apparently
+Received: or is it
+Received: from server.chat-met-vreemden.nl (unknown [IPv6:2a01:7c8:aab6:26d:5054:ff:fed1:1da2])
+ (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
+ (Client did not present a certificate)
+ by mx1.freebsd.org (Postfix) with ESMTPS id CF0171862
+ for <test@example.com>; Mon, 6 Jul 2015 09:01:20 +0000 (UTC)
+ (envelope-from upwest201diana@outlook.com)
+Received: from ca-18-193-131.service.infuturo.it ([151.18.193.131] helo=User)
+ by server.chat-met-vreemden.nl with esmtpsa (Exim 4.76)
+ (envelope-from <upwest201diana@outlook.com>)
+ id 1ZC1sl-0006b4-TU; Mon, 06 Jul 2015 10:36:08 +0200
+MIME-Version: 1.0
+Content-Type: text/plain
+
+Hello
diff --git a/test/functional/messages/received5.eml b/test/functional/messages/received5.eml
new file mode 100644
index 0000000..84d89e9
--- /dev/null
+++ b/test/functional/messages/received5.eml
@@ -0,0 +1,13 @@
+Received: from localhost (localhost [127.0.0.1])
+ by ietfa.amsl.com (Postfix) with ESMTPA id 00E7712024B
+ for <cfrg@ietfa.amsl.com>; Tue, 7 May 2019 14:01:07 -0700 (PDT)
+Received: from mail.ietf.org ([4.31.198.44])
+ by localhost (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTPA id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue, 7 May 2019 14:01:04 -0700 (PDT)
+Received: from mail.example.org ([192.0.2.1])
+ by localhost (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTP id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue, 7 May 2019 14:01:04 -0700 (PDT)
+
+aa
diff --git a/test/functional/messages/received6.eml b/test/functional/messages/received6.eml
new file mode 100644
index 0000000..38dd801
--- /dev/null
+++ b/test/functional/messages/received6.eml
@@ -0,0 +1,17 @@
+Received: from localhost (localhost [127.0.0.1])
+ by ietfa.amsl.com (Postfix) with ESMTPA id 00E7712024B
+ for <cfrg@ietfa.amsl.com>; Tue, 7 May 2019 14:01:07 -0700 (PDT)
+Received: from cool.example.org ([4.31.198.44])
+ by lame.example.net (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTPA id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue, 7 May 2019 14:01:04 -0700 (PDT)
+Received: from mail.example.org ([192.0.3.1])
+ by localhost (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTP id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue, 7 May 2019 14:01:04 -0700 (PDT)
+Received: from mail.example.org ([192.0.2.1])
+ by cool.example.org (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTP id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue, 7 May 2019 14:01:04 -0700 (PDT)
+
+aa
diff --git a/test/functional/messages/received7.eml b/test/functional/messages/received7.eml
new file mode 100644
index 0000000..fa946ec
--- /dev/null
+++ b/test/functional/messages/received7.eml
@@ -0,0 +1,17 @@
+Received: from localhost (localhost [2001:db8:114::514])
+ by ietfa.amsl.com (Postfix) with ESMTP id 00E7712024B
+ for <cfrg@ietfa.amsl.com>; Tue, 7 May 2019 14:01:07 -0700 (PDT)
+Received: from mail.example.org ([198.51.100.233])
+ by lame.example.net (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTP id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue, 7 May 2019 14:01:04 -0700 (PDT)
+Received: from foobar.example.org ([4.31.198.44])
+ by localhost (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTP id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue, 7 May 2019 14:01:04 -0700 (PDT)
+Received: from mail.example.org ([198.51.100.2])
+ by foobar.example.org (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTP id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue, 7 May 2019 14:01:04 -0700 (PDT)
+
+haha
diff --git a/test/functional/messages/redir.eml b/test/functional/messages/redir.eml
new file mode 100644
index 0000000..c521802
--- /dev/null
+++ b/test/functional/messages/redir.eml
@@ -0,0 +1,6 @@
+Content-type: text/plain
+
+bla http://127.0.0.1:18080/redirect2
+
+lol http://127.0.0.1:18080/redirect3
+
diff --git a/test/functional/messages/replyto.eml b/test/functional/messages/replyto.eml
new file mode 100644
index 0000000..234bc59
--- /dev/null
+++ b/test/functional/messages/replyto.eml
@@ -0,0 +1,5 @@
+Reply-to: user@emailbl.com
+From: xxx@abrakadabra.com
+Content-Type: text/plain
+
+salkdmaslkd \ No newline at end of file
diff --git a/test/functional/messages/replyto2.eml b/test/functional/messages/replyto2.eml
new file mode 100644
index 0000000..2813f03
--- /dev/null
+++ b/test/functional/messages/replyto2.eml
@@ -0,0 +1,5 @@
+Reply-to: user@baddomain.com
+From: xxx@abrakadabra.com
+Content-Type: text/plain
+
+salkdmaslkd \ No newline at end of file
diff --git a/test/functional/messages/replyto_addr_eq_from.eml b/test/functional/messages/replyto_addr_eq_from.eml
new file mode 100644
index 0000000..8d61078
--- /dev/null
+++ b/test/functional/messages/replyto_addr_eq_from.eml
@@ -0,0 +1,5 @@
+From: "OMG" <kewl.example.20210428@gmail.com>
+Reply-To: "GMO" <kewl.example.20210428@gmail.com>
+Content-type: text/plain
+
+hi
diff --git a/test/functional/messages/replytosubdomain.eml b/test/functional/messages/replytosubdomain.eml
new file mode 100644
index 0000000..a87ec9a
--- /dev/null
+++ b/test/functional/messages/replytosubdomain.eml
@@ -0,0 +1,5 @@
+Reply-to: user@subdomain.emailbl.com
+From: xxx@abrakadabra.com
+Content-Type: text/plain
+
+salkdmaslkd \ No newline at end of file
diff --git a/test/functional/messages/spam.eml b/test/functional/messages/spam.eml
new file mode 100644
index 0000000..6f63fd8
--- /dev/null
+++ b/test/functional/messages/spam.eml
@@ -0,0 +1,760 @@
+Return-Path: <info@jbshvnthwigaf.ru>
+Message-ID: <07FDB7CD95416D1EC2C78E3490CA97E8@jbshvnthwigaf.ru>
+Reply-To: =?windows-1251?B?wO3k8OXpIM/u8uDv7uI=?= <info@jbshvnthwigaf.ru>
+From: =?windows-1251?B?wO3k8OXpIM/u8uDv7uI=?= <info@jbshvnthwigaf.ru>
+To: <user@example.com>
+Subject: =?windows-1251?B?wPPk6O797fbo6uvu7+Xk6Ogg5Ov/IOTl8uXp?=
+ =?windows-1251?B?IC0g4uXr6Oru6+Xv7eD/IOru6+vl6vbo/yDi?=
+ =?windows-1251?B?IO7y6+j37e7sIOrg9+Xx8uLlLiAwOF8wMV8y?=
+ =?windows-1251?B?MDE5IDAyXzEwIDI1MTU2NA==?=
+Date: Mon, 10 Jun 2019 13:39:44 +0300
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="fe880034ac1aaf7f71abb1871107"
+DKIM-Signature: v=1; a=rsa-sha256; d=jbshvnthwigaf.ru; s=mail;
+ c=relaxed/relaxed; t=1560163184;
+ h=message-id:from:to:subject:date:mime-version;
+ bh=pa9P4zmq9pSY6zcn2X53VjhlouJIQeovi/VTn25PfPM=;
+ b=oDfHYiH3QqBt6vvfmKEd5yByW5d+BGRNf7D4ogIi6RAdqFWPuxsOarFuwhhgmu
+ Z0MIjiuqidEpjfS+5oAzIiTRZ/H923UDjtm7PSoHgAcKXh0GMkE2jJEWkMa5y9YQ
+ ZJsnotERFY30Fp7kt+8/snOhweIlOpVWa2pgzrK3+ZTOs=
+
+This is a multi-part message in MIME format.
+
+--fe880034ac1aaf7f71abb1871107
+Content-Type: multipart/alternative; boundary="f5da00c32c2b4701679ab35c9255"
+
+
+--f5da00c32c2b4701679ab35c9255
+Content-Type: text/plain; charset="windows-1251"
+Content-Transfer-Encoding: quoted-printable
+
+=C4=C5=D2=D1=CA=C8=C5 =C0=D3=C4=C8=CE=DD=CD=D6=C8=CA=CB=CE=CF=C5=C4=C8=C8
+
+=CF=EE=E7=ED=E0=E2=E0=F2=E5=EB=FC=ED=E0=FF =F1=E5=F0=E8=FF =E4=EB=FF =E4=E5=
+=F2=E5=E9. =CC=F3=E7=FB=EA=E0=EB=FC=ED=FB=E9 =F1=EF=E5=EA=F2=E0=EA=EB=FC =
+=E2 =EB=E8=F6=E0=F5, =F1 =EF=E5=F1=E5=ED=EA=E0=EC=E8 =E8 =F1=F2=E8=F5=EE=F2=
+=E2=EE=F0=E5=ED=E8=FF=EC=E8, =E2 =EB=B8=E3=EA=EE=E9 =E8 =E4=EE=F1=F2=F3=EF=
+=ED=EE=E9 =F4=EE=F0=EC=E5 =E7=ED=E0=EA=EE=EC=E8=F2 =E4=E5=F2=E5=E9 =F1 =F3=
+=E2=EB=E5=EA=E0=F2=E5=EB=FC=ED=FB=EC =EC=E8=F0=EE=EC =ED=E0=F3=F7=ED=FB=F5=
+ =E8 =EA=F3=EB=FC=F2=F3=F0=ED=FB=F5 =E7=ED=E0=ED=E8=E9. =C2=E0=F8 =F0=E5=E1=
+=E5=ED=EE=EA =E1=E5=E7 =F2=F0=F3=E4=E0, =F0=E0=E7=E2=EB=E5=EA=E0=FF=F1=FC=
+, =F3=F1=E2=EE=E8=F2 =EC=ED=EE=E6=E5=F1=F2=E2=EE =F2=E5=F0=EC=E8=ED=EE=E2=
+, =EF=EE=ED=FF=F2=E8=E9 =E8 =E7=ED=E0=ED=E8=E9. =C4=E0=ED=ED=E0=FF =EA=EE=
+=EB=EB=E5=EA=F6=E8=FF =EF=F0=EE=F1=F2=EE =ED=E0=F5=EE=E4=EA=E0 =E4=EB=FF =
+=F0=EE=E4=E8=F2=E5=EB=E5=E9, =E4=E5=F2=E8 =E2 =E2=EE=F1=F2=EE=F0=E3=E5 =E7=
+=E0=F3=F7=E8=E2=E0=FE=F2 =E2=F1=E5 =ED=E0=E8=E7=F3=F1=F2=FC =EF=EE =F1=EE=
+=E1=F1=F2=E2=E5=ED=ED=EE=EC=F3 =E6=E5=EB=E0=ED=E8=FE, =EF=F0=E5=EA=F0=E0=F1=
+=ED=E0=FF =E0=EB=FC=F2=E5=F0=ED=E0=F2=E8=E2=E0 =EC=F3=EB=FC=F2=F4=E8=EB=FC=
+=EC=E0=EC, =E0 =F2=E0=EA=E6=E5 =E2=EE=E7=EC=EE=E6=ED=EE=F1=F2=FC =EE=F2=E2=
+=EB=E5=F7=FC =E4=E5=F2=E5=E9 =EE=F2 =EF=F0=EE=F1=EC=EE=F2=F0=E0 =F2=E5=EB=
+=E5=E2=E8=E7=EE=F0=E0 =E8 =E8=E3=F0 =ED=E0 =EA=EE=EC=EF=FC=FE=F2=E5=F0=E5=
+. =DD=F2=EE =EE=E3=F0=EE=EC=ED=E0=FF =EF=EE=EC=EE=F9=FC =E2 =EF=EE=E7=ED=E0=
+=ED=E8=E8 =F0=E5=E1=E5=ED=EA=EE=EC =EC=E8=F0=E0, =EF=F0=E8=F7=E5=EC =E8=ED=
+=F2=E5=F0=E5=F1=ED=E0=FF =E8 =F0=EE=E4=E8=F2=E5=EB=FF=EC =F2=EE=E6=E5. =C0=
+ =E3=EB=E0=E2=ED=EE=E5 - =C2=FB =EC=EE=E6=E5=F2=E5 =E1=FB=F2=FC =F1=EF=EE=
+=EA=EE=E9=ED=FB =E7=E0 =EA=E0=F7=E5=F1=F2=E2=EE =E8 =E4=EE=F1=F2=F3=EF=ED=
+=EE=F1=F2=FC =EC=E0=F2=E5=F0=E8=E0=EB=E0, =EA=EE=EB=EB=E5=EA=F6=E8=FF =F1=
+=EE=E7=E4=E0=ED=E0 =EF=F0=EE=F4=E5=F1=F1=E8=EE=ED=E0=EB=FC=ED=EE, =F2=E2=EE=
+=F0=F7=E5=F1=EA=E8 =E8 =FD=EC=EE=F6=E8=EE=ED=E0=EB=FC=ED=EE =EF=EE-=E4=EE=
+=E1=F0=EE=EC=F3. =CA=E0=F7=E5=F1=F2=E2=EE =E7=E0=EF=E8=F1=E8, =F0=E0=E1=EE=
+=F2=E0 =E0=F0=F2=E8=F1=F2=EE=E2 =ED=E0 =E2=FB=F1=EE=EA=EE=EC =F3=F0=EE=E2=
+=ED=E5, =EF=EE=FD=F2=EE=EC=F3 =F0=E5=E1=E5=ED=EE=EA =ED=E5 =EF=E5=F0=E5=E3=
+=F0=F3=E6=E0=E5=F2=F1=FF =E2=F2=EE=F0=EE=F1=F2=E5=EF=E5=ED=ED=EE=E9, =E0 =
+=E7=E0=F7=E0=F1=F2=F3=FE =E8 =ED=E5=ED=F3=E6=ED=EE=E9 =E2 =FE=ED=EE=EC =E2=
+=EE=E7=F0=E0=F1=F2=E5 =E8=ED=F4=EE=F0=EC=E0=F6=E8=E5=E9. =CA=EE=EB=EB=E5=EA=
+=F6=E8=FF =F0=E0=F1=F1=F7=E8=F2=E0=ED=E0 =ED=E0 =E2=EE=E7=F0=E0=F1=F2 =EE=
+=F2 3 =E4=EE 10 =EB=E5=F2. =CF=EE=F1=EB=E5 =EF=F0=EE=F1=EB=F3=F8=E8=E2=E0=
+=ED=E8=FF =EA=E0=E6=E4=EE=E3=EE =E4=E8=F1=EA=E0 =F3 =C2=E0=F8=E5=E3=EE =F0=
+=E5=E1=B8=ED=EA=E0 =EE=E1=FF=E7=E0=F2=E5=EB=FC=ED=EE =EF=EE=FF=E2=FF=F2=F1=
+=FF =ED=EE=E2=FB=E5 =F3=E2=EB=E5=F7=E5=ED=E8=FF =E8 =E8=ED=F2=E5=F0=E5=F1=
+=FB. =CD=E0=F8=E0 =EA=EE=EB=EB=E5=EA=F6=E8=FF =EF=EE=F1=EB=F3=E6=E8=F2 =EF=
+=F0=E5=EA=F0=E0=F1=ED=FB=EC =EF=EE=E4=E0=F0=EA=EE=EC, =ED=E5 =EB=E8=F8=E0=
+=E9=F2=E5 =F1=E5=E1=FF =E8 =F1=E2=EE=E8=F5 =E1=EB=E8=E7=EA=E8=F5 =F3=E4=EE=
+=E2=EE=EB=FC=F1=F2=E2=E8=FF !
+
+! =D1=EF=E8=F1=EE=EA =E7=E0=EF=E8=F1=E5=E9 =E2=EE=F8=E5=E4=F8=E8=F5 =E2 =EA=
+=EE=EB=EB=E5=EA=F6=E8=FE =E2=FB =EC=EE=E6=E5=F2=E5 =F3=E2=E8=E4=E5=F2=FC =
+=E2 =EF=F0=E8=EA=F0=E5=EF=EB=B8=ED=ED=EE=EC =EA =EF=E8=F1=FC=EC=F3 =F4=E0=
+=E9=EB=E5 !
+
+=C4=F0=E5=E2=ED=E8=E9 =C5=E3=E8=EF=E5=F2; =C4=F0=E5=E2=ED=E8=E9 =D0=E8=EC=
+; =D3=E4=E8=E2=E8=F2=E5=EB=FC=ED=FB=E5 =ED=E0=F1=E5=EA=EE=EC=FB=E5; =D6=E0=
+=F0=F1=F2=E2=EE =F0=E0=F1=F2=E5=ED=E8=E9; =C4=E8=EA=E8=E5 =E6=E8=E2=EE=F2=
+=ED=FB=E5; =C4=EE=EC=E0=F8=ED=E8=E5 =E6=E8=E2=EE=F2=ED=FB=E5; =C6=E8=E2=EE=
+=F2=ED=FB=E5 =D1=E8=E1=E8=F0=E8; =C6=E8=E2=EE=F2=ED=FB=E5 =DE=E6=ED=EE=E9=
+ =C0=EC=E5=F0=E8=EA=E8; =C0=E2=F2=EE=EC=EE=E1=E8=EB=E8 - =F2=F0=E0=ED=F1=EF=
+=EE=F0=F2; =D1=E0=EC=EE=EB=B8=F2=FB - =E0=E2=E8=E0=F6=E8=FF; =CF=EE=EB=B8=
+=F2=FB =E2 =EA=EE=F1=EC=EE=F1; =D3=E2=EB=E5=EA=E0=F2=E5=EB=FC=ED=E0=FF =E0=
+=F1=F2=F0=EE=ED=EE=EC=E8=FF; =CA=EE=F0=E0=E1=EB=E8 - =EC=EE=F0=E5=EF=EB=E0=
+=E2=E0=ED=E8=E5; =CE=E1=E8=F2=E0=F2=E5=EB=E8 =EE=EA=E5=E0=ED=EE=E2; =C6=E8=
+=E2=EE=F2=ED=FB=E5 =C0=E2=F1=F2=F0=E0=EB=E8=E8; =C6=E8=E2=EE=F2=ED=FB=E5 =
+=C0=F4=F0=E8=EA=E8; =CB=E5=F1=ED=FB=E5 =E8 =F3=E4=E8=E2=E8=F2=E5=EB=FC=ED=
+=FB=E5 =EF=F2=E8=F6=FB; =D3=E4=E8=E2=E8=F2=E5=EB=FC=ED=FB=E5 =F0=FB=E1=FB=
+; =CB=E5=F1=E0 =D0=EE=F1=F1=E8=E8; =C4=E8=ED=EE=E7=E0=E2=F0=FB =E8 =EF=F0=
+=E5=F1=EC=FB=EA=E0=FE=F9=E8=E5=F1=FF; =C2=E5=EB=E8=EA=E0=FF =CE=F2=E5=F7=E5=
+=F1=F2=E2=E5=ED=ED=E0=FF; =D0=EE=F1=F1=E8=FF =E2 1812; =C2=E5=EB=E8=EA=E8=
+=E5 =EF=F3=F2=E5=F8=E5=F1=F2=E2=E8=FF; =CF=EB=E0=ED=E5=F2=E0 =C7=E5=EC=EB=
+=FF; =CD=E0=F0=EE=E4=ED=FB=E5 =E8=ED=F1=F2=F0=F3=EC=E5=ED=F2=FB; =D1=E8=EC=
+=F4=EE=ED=E8=F7=E5=F1=EA=E8=E9 =EE=F0=EA=E5=F1=F2=F0; =CA=E0=EA =F1=E5=E1=
+=FF =E2=E5=F1=F2=E8; =CC=FB =F1=EE=F7=E8=ED=FF=E5=EC =F1=F2=E8=F5=E8; =C3=
+=EE=F0=EE=E4=F1=EA=E8=E5 =EF=F2=E8=F6=FB; =CF=F3=F2=E5=F8=E5=F1=F2=E2=E8=E5=
+ =F0=E0=F1=F2=E5=ED=E8=E9; =C4=F0=E5=E2=ED=FF=FF =C3=F0=E5=F6=E8=FF; =C7=ED=
+=E0=EA=EE=EC=F1=F2=E2=EE =F1 =F2=E5=E0=F2=F0=EE=EC; =CA=E0=EA =E6=E8=EB=E8=
+ =ED=E0 =D0=F3=F1=E8; =CC=EE=F1=EA=EE=E2=F1=EA=E8=E9 =CA=F0=E5=EC=EB=FC; =
+=CA=E0=EA =F3=F1=F2=F0=EE=E5=ED =F7=E5=EB=EE=E2=E5=EA; =CA=E0=EB=E5=ED=E4=
+=E0=F0=FC =EF=F0=E8=F0=EE=E4=FB; =C7=E0=EC=E5=F7=E0=F2=E5=EB=FC=ED=FB=E5 =
+=EF=F0=EE=F4=E5=F1=F1=E8=E8; =CD=E0=F8=E0 =F1=F2=F0=E0=ED=E0 =D0=EE=F1=F1=
+=E8=FF; =CF=EE=EB=E5=E7=ED=FB=E5 =E8=F1=EA=EE=EF=E0=E5=EC=FB=E5; =CF=F3=F2=
+=E5=F8=E5=F1=F2=E2=E8=E5 =E2 =CA=E0=EC=E5=ED=ED=FB=E9 =E2=E5=EA.
+
+=C7=E0=EF=E8=F1=E8 =E2 =F4=EE=F0=EC=E0=F2=E5 MP3, =E1=E8=F2=F0=E5=E9=F2 =3D=
+ 320 kb\s. =C7=E0=EF=E8=F1=E0=ED=E0 =ED=E0 =E2=ED=E5=F8=ED=E8=E9 USB =ED=E0=
+=EA=EE=EF=E8=F2=E5=EB=FC (=F4=EB=E5=F8=EA=E0). =CF=F0=EE=E1=EB=E5=EC =F1 =
+=E2=EE=F1=EF=F0=EE=E8=E7=E2=E5=E4=E5=ED=E8=E5=EC =ED=E5 =E2=EE=E7=ED=E8=EA=
+=ED=E5=F2, =EC=EE=E6=ED=EE =F1=EB=F3=F8=E0=F2=FC =ED=E0 =EA=EE=EC=EF=FC=FE=
+=F2=E5=F0=E5, =EF=EB=E0=ED=F8=E5=F2=E5, =F1=EC=E0=F0=F2=F4=EE=ED=E5, =F2=E5=
+=EB=E5=E2=E8=E7=EE=F0=E5 =E8 =F2.=E4. =C7=E0=EF=E8=F1=FC =ED=E0 =E2=ED=E5=
+=F8=ED=E8=E9 USB =ED=E0=EA=EE=EF=E8=F2=E5=EB=FC =E8=EC=E5=E5=F2 =F0=FF=E4=
+ =EF=F0=E5=E8=EC=F3=F9=E5=F1=F2=E2 =E2 =F1=F0=E0=E2=ED=E5=ED=E8=E8 =F1 =EE=
+=E1=FB=F7=ED=FB=EC=E8 CD =E4=E8=F1=EA=E0=EC=E8, USB =ED=E0=EA=EE=EF=E8=F2=
+=E5=EB=FC =E3=EE=F0=E0=E7=E4=EE =EB=E5=E3=F7=E5, =E7=E0=ED=E8=EC=E0=E5=F2=
+ =EC=E5=ED=FC=F8=E5 =EC=E5=F1=F2=E0, =EE=E1=EB=E0=E4=E0=E5=F2 =E2=FB=F1=EE=
+=EA=EE=E9 =ED=E0=E4=B8=E6=ED=EE=F1=F2=FC=FE =F1=EE=F5=F0=E0=ED=ED=EE=F1=F2=
+=E8 =E7=E0=EF=E8=F1=E5=E9, =E0 =FD=F2=EE =E7=ED=E0=F7=E8=F2, =F7=F2=EE =ED=
+=E0=F8=E0 =EA=EE=EB=EB=E5=EA=F6=E8=FF =E1=F3=E4=E5=F2 =F0=E0=E4=EE=E2=E0=F2=
+=FC =C2=E0=F1 =EC=ED=EE=E3=EE =EB=E5=F2. =CC=FB =E3=E0=F0=E0=ED=F2=E8=F0=F3=
+=E5=EC =EE=F2=EB=E8=F7=ED=EE=E5 =EA=E0=F7=E5=F1=F2=E2=EE =E2=F1=E5=F5 =E7=
+=E0=EF=E8=F1=E5=E9. =CD=E0 =F1=E0=EC=EE=EC =ED=EE=F1=E8=F2=E5=EB=E5 =F1=EE=
+=E7=E4=E0=ED=E0 =EF=F0=EE=E4=F3=EC=E0=ED=ED=E0=FF =F1=F2=F0=F3=EA=F2=F3=F0=
+=E0, =E2=F1=E5 =E7=E0=EF=E8=F1=E8 =F0=E0=E7=ED=E5=F1=E5=ED=FB =EF=EE =EA=E0=
+=F2=E0=EB=EE=E3=E0=EC, =E8=EC=E5=FE=F2=F1=FF =EF=EB=E5=E9=EB=E8=F1=F2=FB,=
+ =EF=F0=EE=EF=E8=F1=E0=ED=FB =F2=E5=E3=E8, =E0 =F2=E0=EA=E6=E5 =EF=EE=EB=ED=
+=FB=E9 =F1=EF=E8=F1=EE=EA =E2=EE=F8=E5=E4=F8=E8=F5 =E7=E0=EF=E8=F1=E5=E9,=
+ =EF=EE=FD=F2=EE=EC=F3 =EF=F0=EE=E1=EB=E5=EC =F1 =EF=EE=E8=F1=EA=EE=EC =E8=
+ =ED=E0=E2=E8=E3=E0=F6=E8=E5=E9 =ED=E5 =E2=EE=E7=ED=E8=EA=ED=E5=F2.
+
+=D1=F2=EE=E8=EC=EE=F1=F2=FC =EA=EE=EB=EB=E5=EA=F6=E8=E8 =ED=E0 =E2=ED=E5=F8=
+=ED=E5=EC USB =ED=E0=EA=EE=EF=E8=F2=E5=EB=E5 =97 6500 (=D8=E5=F1=F2=FC =D2=
+=FB=F1=FF=F7 =CF=FF=F2=FC=F1=EE=F2) =D0=F3=E1=EB=E5=E9.
+=CF=F0=EE=E4=E0=FE=F2=F1=FF =F2=EE=EB=FC=EA=EE =E2=EC=E5=F1=F2=E5. =C4=EE=
+=F1=F2=E0=E2=EA=E0 =E2=EA=EB=FE=F7=E5=ED=E0 =E2 =F1=F2=EE=E8=EC=EE=F1=F2=FC=
+.
+
+=C4=EE=F1=F2=E0=E2=EA=E0 =F2=EE=EB=FC=EA=EE =EF=EE=F7=F2=EE=E9 =EF=EE =E2=
+=F1=E5=E9 =D0=EE=F1=F1=E8=E8, =F1=F0=EE=EA=E8 7-14 =F1=F3=F2=EE=EA =F1 =EC=
+=EE=EC=E5=ED=F2=E0 =EE=F2=EF=F0=E0=E2=EA=E8. =CE=EF=EB=E0=F2=E0 =E2 =EC=EE=
+=EC=E5=ED=F2 =EF=EE=EB=F3=F7=E5=ED=E8=FF =E7=E0=EA=E0=E7=E0 =ED=E0 =EF=EE=
+=F7=F2=E5 =ED=E0=EB=EE=E6=E5=ED=ED=FB=EC =EF=EB=E0=F2=E5=E6=EE=EC. =D3 =ED=
+=E0=F1 =ED=E5=F2 =EA=F3=F0=FC=E5=F0=F1=EA=EE=E9 =E4=EE=F1=F2=E0=E2=EA=E8 =
+=97 =F2=EE=EB=FC=EA=EE =EF=EE=F7=F2=EE=E9, =E2 =F2=EE=EC =F7=E8=F1=EB=E5 =
+=E8 =EF=EE =CC=EE=F1=EA=E2=E5.
+
+=C4=EB=FF =EE=F4=EE=F0=EC=EB=E5=ED=E8=FF =E7=E0=EA=E0=E7=E0 =EF=F0=EE=F1=FC=
+=E1=E0 =ED=E5 =E7=E0=E1=FB=E2=E0=F2=FC =F3=EA=E0=E7=FB=E2=E0=F2=FC:
+=A0--- =C2=E0=F8 =EF=EE=F7=F2=EE=E2=FB=E9 =E8=ED=E4=E5=EA=F1 (=EF=E8=F8=E8=
+=F2=E5 =EF=F0=E0=E2=E8=EB=FC=ED=FB=E9 =E8=ED=E4=E5=EA=F1 =97 =FD=F2=EE =F3=
+=F1=EA=EE=F0=E8=F2 =E4=EE=F1=F2=E0=E2=EA=F3);
+=A0--- =C2=E0=F8 =E3=EE=F0=EE=E4 =E8 =F2=EE=F7=ED=FB=E9 =E0=E4=F0=E5=F1 (=
+=ED=E0=E7=E2=E0=ED=E8=E5 =F3=EB=E8=F6=FB, =ED=EE=EC=E5=F0 =E4=EE=EC=E0 =E8=
+ =ED=EE=EC=E5=F0 =EA=E2=E0=F0=F2=E8=F0=FB);
+=A0--- =D4.=C8.=CE. =EF=EE=EB=F3=F7=E0=F2=E5=EB=FF =E8 =CE=C1=DF=C7=C0=D2=
+=C5=CB=DC=CD=CE =ED=EE=EC=E5=F0 =EA=EE=ED=F2=E0=EA=F2=ED=EE=E3=EE =F2=E5=EB=
+=E5=F4=EE=ED=E0 (=EB=F3=F7=F8=E5 =F1=EE=F2=EE=E2=FB=E9);
+=C7=E0=EA=E0=E7=FB\=E2=EE=EF=F0=EE=F1=FB =ED=E0=EF=F0=E0=E2=EB=FF=E9=F2=E5=
+ =EF=EE =E0=E4=F0=E5=F1=F3: audioencyclopedia@cwhflash.ru
+
+=CC=FB =EE=F7=E5=ED=FC =EE=F2=E2=E5=F2=F1=F2=E2=E5=ED=ED=EE =EE=F2=ED=EE=F1=
+=E8=EC=F1=FF =EA =EA=E0=F7=E5=F1=F2=E2=F3 =ED=E0=F8=E5=E3=EE =F2=EE=E2=E0=
+=F0=E0, =EF=EE=FD=F2=EE=EC=F3 =EF=E5=F0=E5=E4 =EE=F2=EF=F0=E0=E2=EA=EE=E9=
+ =E2=F1=B8 =E4=EE=EF=EE=EB=ED=E8=F2=E5=EB=FC=ED=EE =EF=F0=EE=E2=E5=F0=FF=E5=
+=F2=F1=FF, =EA=E0=EA =F1=EB=E5=E4=F1=F2=E2=E8=E5 =EE=F2=EF=F0=E0=E2=EA=E0=
+ =E1=F0=E0=EA=EE=E2=E0=ED=ED=EE=E9 =EF=F0=EE=E4=F3=EA=F6=E8=E8 =F1=E2=E5=E4=
+=E5=ED=E0 =EA =ED=F3=EB=FE. =D2=EE=E2=E0=F0 =F3=EF=E0=EA=EE=E2=FB=E2=E0=E5=
+=F2=F1=FF =E2 =F1=EF=E5=F6=E8=E0=EB=FC=ED=FB=E9 =F3=E4=E0=F0=EE=F1=F2=EE=E9=
+=EA=E8=E9 =EC=E0=F2=E5=F0=E8=E0=EB, =F7=F2=EE =E2 =E7=ED=E0=F7=E8=F2=E5=EB=
+=FC=ED=EE=E9 =F1=F2=E5=EF=E5=ED=E8 =F3=EC=E5=ED=FC=F8=E0=E5=F2 =F0=E8=F1=EA=
+ =EF=EE=E2=F0=E5=E6=E4=E5=ED=E8=FF =EF=F0=E8 =F2=F0=E0=ED=F1=EF=EE=F0=F2=E8=
+=F0=EE=E2=EA=E5. =C5=F1=EB=E8 =E2=E4=F0=F3=E3 =F1 =EF=EE=EB=F3=F7=E5=ED=ED=
+=FB=EC =F2=EE=E2=E0=F0=EE=EC =E2=EE=E7=ED=E8=EA=ED=F3=F2 =EF=F0=EE=E1=EB=E5=
+=EC=FB, =F2=EE =E2=F1=E5 =ED=E0=F8=E8 =EF=EE=EA=F3=EF=E0=F2=E5=EB=E8 =E2=F1=
+=E5=E3=E4=E0 =EC=EE=E3=F3=F2 =F0=E0=F1=F1=F7=E8=F2=FB=E2=E0=F2=FC =ED=E0 =
+=EA=E2=E0=EB=E8=F4=E8=F6=E8=F0=EE=E2=E0=ED=ED=F3=FE =F2=E5=F5=ED=E8=F7=E5=
+=F1=EA=F3=FE =EF=EE=E4=E4=E5=F0=E6=EA=F3. =CC=FB =ED=E8=EA=EE=E3=E4=E0 =ED=
+=E5 =EE=F2=EA=E0=E7=FB=E2=E0=E5=EC=F1=FF =EE=F2 =E3=E0=F0=E0=ED=F2=E8=E9=ED=
+=FB=F5 =EE=E1=FF=E7=E0=F2=E5=EB=FC=F1=F2=E2, =E2 =F1=EB=F3=F7=E0=E5 =EF=F0=
+=EE=E1=EB=E5=EC=FB =C2=FB =EC=EE=E6=E5=F2=E5 =F0=E0=F1=F1=F7=E8=F2=FB=E2=E0=
+=F2=FC =ED=E0 =E7=E0=EC=E5=ED=F3, =EF=EE=F7=F2=EE=E2=FB=E5 =F0=E0=F1=F5=EE=
+=E4=FB =EC=FB =E1=E5=F0=B8=EC =ED=E0 =F1=E5=E1=FF.
+
+=CF=EE =E2=E0=F8=E5=EC=F3 =E6=E5=EB=E0=ED=E8=FE, =E4=E0=ED=ED=E0=FF =EA=EE=
+=EB=EB=E5=EA=F6=E8=FF =EC=EE=E6=E5=F2 =E1=FB=F2=FC =E7=E0=EF=E8=F1=E0=ED=E0=
+ =ED=E0 CD =E4=E8=F1=EA=E8. =C4=EB=FF =E7=E0=EF=E8=F1=E8 =E8=F1=EF=EE=EB=FC=
+=E7=F3=FE=F2=F1=FF =ED=E0=E4=B8=E6=ED=FB=E5 CD =E4=E8=F1=EA=E8 =F1=EE =F1=
+=EF=E5=F6=E8=E0=EB=FC=ED=FB=EC =EF=EE=EA=F0=FB=F2=E8=E5=EC, =EA=EE=F2=EE=F0=
+=EE=E5 =EF=EE=E2=FB=F8=E0=E5=F2 =F3=F1=F2=EE=E9=F7=E8=E2=EE=F1=F2=FC =E4=E8=
+=F1=EA=E0 =EA =EC=E5=F5=E0=ED=E8=F7=E5=F1=EA=E8=EC =EF=EE=E2=F0=E5=E6=E4=E5=
+=ED=E8=FF=EC, =F2=E0=EA=E8=EC =EA=E0=EA =F2=F0=E5=F9=E8=ED=FB =E8 =F6=E0=F0=
+=E0=EF=E8=ED=FB, =E0 =FD=F2=EE =E7=ED=E0=F7=E8=F2, =F7=F2=EE =ED=E0=F8=E0=
+ =EA=EE=EB=EB=E5=EA=F6=E8=FF =E1=F3=E4=E5=F2 =F0=E0=E4=EE=E2=E0=F2=FC =C2=
+=E0=F1 =EC=ED=EE=E3=EE =EB=E5=F2. =CA=EE=EB=EB=E5=EA=F6=E8=FF =F3=EF=E0=EA=
+=EE=E2=E0=ED=E0 =E2 =EF=EB=E0=F1=F2=E8=EA=EE=E2=FB=E5 =E1=EE=EA=F1=FB (sl=
+im-dvd), =E8=EC=E5=E5=F2 =EA=F0=E0=F1=E8=E2=FB=E5 =E8 =EF=F0=EE=E4=F3=EC=E0=
+=ED=ED=FB=E5 =EE=E1=EB=EE=E6=EA=E8, =F1 =EE=E1=F0=E0=F2=ED=EE=E9 =F1=F2=EE=
+=F0=EE=ED=FB =EA=EE=F2=EE=F0=FB=F5 =F3=EA=E0=E7=E0=ED =F1=EF=E8=F1=EE=EA =
+=E2=EE=F8=E5=E4=F8=E8=F5 =ED=E0 =EA=E0=E6=E4=FB=E9 =E4=E8=F1=EA =E7=E0=EF=
+=E8=F1=E5=E9 =E8 =E4=F0=F3=E3=E0=FF =EF=EE=EB=E5=E7=ED=E0=FF =E8=ED=F4=EE=
+=F0=EC=E0=F6=E8=FF, =EF=EE=FD=F2=EE=EC=F3 =EF=F0=EE=E1=EB=E5=EC =F1 =EF=EE=
+=E8=F1=EA=EE=EC =E8 =ED=E0=E2=E8=E3=E0=F6=E8=E5=E9 =ED=E5 =E2=EE=E7=ED=E8=
+=EA=ED=E5=F2. =C5=F1=EB=E8 =F5=EE=F2=E8=F2=E5 =EF=F0=E8=EE=E1=F0=E5=F1=F2=
+=E8 =EA=EE=EB=EB=E5=EA=F6=E8=FE, =E7=E0=EF=E8=F1=E0=ED=ED=F3=FE =ED=E0 CD=
+ =E4=E8=F1=EA=E0=F5, =F2=EE =E2 =FD=F2=EE=EC =F1=EB=F3=F7=E0=E5 =EF=F0=EE=
+=F1=FC=E1=E0 =F1=EE=EE=E1=F9=E8=F2=FC =ED=E0=EC =EE=E1 =FD=F2=EE=EC =E2 =F1=
+=E2=EE=E5=E9 =E7=E0=FF=E2=EA=E5, =F6=E5=ED=E0 =EF=F0=E5=E6=ED=FF=FF, =EA=E0=
+=EA =F3 =E2=E5=F0=F1=E8=E8 =ED=E0 =E2=ED=E5=F8=ED=E5=EC USB =ED=E0=EA=EE=EF=
+=E8=F2=E5=EB=E5 (=F4=EB=E5=F8=EA=E0) =97 6500 (=D8=E5=F1=F2=FC =D2=FB=F1=FF=
+=F7 =CF=FF=F2=FC=F1=EE=F2) =D0=F3=E1=EB=E5=E9.
+
+=C5=F1=EB=E8 =E2=FB =ED=E5 =F5=EE=F2=E8=F2=E5 =E1=EE=EB=FC=F8=E5 =EF=EE=EB=
+=F3=F7=E0=F2=FC =EE=F2 =ED=E0=F1 =EF=E8=F1=FC=EC=E0, =EE=F2=EF=F0=E0=E2=FC=
+=F2=E5 =ED=E0=EC =EF=E8=F1=FC=EC=EE =F1 =F2=E5=EC=EE=E9 =93deletemail=94 =
+=E8 =C2=E0=F8 =E0=E4=F0=E5=F1 =ED=E0=E2=F1=E5=E3=E4=E0 =E1=F3=E4=E5=F2 =F3=
+=E4=E0=EB=E5=ED =E0=E2=F2=EE=EC=E0=F2=E8=F7=E5=F1=EA=E8.
+
+08_01_2019 02_10 251564
+
+vsevolod@highsecure.ru
+
+--f5da00c32c2b4701679ab35c9255
+Content-Type: text/html; charset="windows-1251"
+Content-Transfer-Encoding: quoted-printable
+
+<HTML><HEAD><TITLE>=C4=C5=D2=D1=CA=C8=C5 =C0=D3=C4=C8=CE=DD=CD=D6=C8=CA=CB=
+=CE=CF=C5=C4=C8=C8</TITLE>
+<META http-equiv=3D"Content-Type" content=3D"text/html; charset=3Dwindows=
+-1251">
+</HEAD>
+<BODY>
+<TABLE border=3D1>
+ <TBODY>
+ <TR>
+ <TD>
+ <P align=3Dcenter><B><FONT color=3D#ff0000 size=3D5 face=3DArial>=C4=
+=C5=D2=D1=CA=C8=C5=20
+ =C0=D3=C4=C8=CE=DD=CD=D6=C8=CA=CB=CE=CF=C5=C4=C8=C8</FONT></B></P><=
+/TD></TR>
+ <TR>
+ <TD>
+ <P align=3Djustify><B><FONT color=3D#008000 face=3DArial>=CF=EE=E7=ED=
+=E0=E2=E0=F2=E5=EB=FC=ED=E0=FF =F1=E5=F0=E8=FF=20
+ =E4=EB=FF =E4=E5=F2=E5=E9. =CC=F3=E7=FB=EA=E0=EB=FC=ED=FB=E9 =F1=EF=
+=E5=EA=F2=E0=EA=EB=FC =E2 =EB=E8=F6=E0=F5, =F1 =EF=E5=F1=E5=ED=EA=E0=EC=E8=
+ =E8 =F1=F2=E8=F5=EE=F2=E2=EE=F0=E5=ED=E8=FF=EC=E8, =E2=20
+ =EB=B8=E3=EA=EE=E9 =E8 =E4=EE=F1=F2=F3=EF=ED=EE=E9 =F4=EE=F0=EC=E5 =
+=E7=ED=E0=EA=EE=EC=E8=F2 =E4=E5=F2=E5=E9 =F1 =F3=E2=EB=E5=EA=E0=F2=E5=EB=FC=
+=ED=FB=EC =EC=E8=F0=EE=EC =ED=E0=F3=F7=ED=FB=F5 =E8=20
+ =EA=F3=EB=FC=F2=F3=F0=ED=FB=F5 =E7=ED=E0=ED=E8=E9. =C2=E0=F8 =F0=E5=
+=E1=E5=ED=EE=EA =E1=E5=E7 =F2=F0=F3=E4=E0, =F0=E0=E7=E2=EB=E5=EA=E0=FF=F1=
+=FC, =F3=F1=E2=EE=E8=F2 =EC=ED=EE=E6=E5=F1=F2=E2=EE=20
+ =F2=E5=F0=EC=E8=ED=EE=E2, =EF=EE=ED=FF=F2=E8=E9 =E8 =E7=ED=E0=ED=E8=
+=E9. =C4=E0=ED=ED=E0=FF =EA=EE=EB=EB=E5=EA=F6=E8=FF =EF=F0=EE=F1=F2=EE =ED=
+=E0=F5=EE=E4=EA=E0 =E4=EB=FF =F0=EE=E4=E8=F2=E5=EB=E5=E9,=20
+ =E4=E5=F2=E8 =E2 =E2=EE=F1=F2=EE=F0=E3=E5 =E7=E0=F3=F7=E8=E2=E0=FE=F2=
+ =E2=F1=E5 =ED=E0=E8=E7=F3=F1=F2=FC =EF=EE =F1=EE=E1=F1=F2=E2=E5=ED=ED=EE=
+=EC=F3 =E6=E5=EB=E0=ED=E8=FE, =EF=F0=E5=EA=F0=E0=F1=ED=E0=FF=20
+ =E0=EB=FC=F2=E5=F0=ED=E0=F2=E8=E2=E0 =EC=F3=EB=FC=F2=F4=E8=EB=FC=EC=
+=E0=EC, =E0 =F2=E0=EA=E6=E5 =E2=EE=E7=EC=EE=E6=ED=EE=F1=F2=FC =EE=F2=E2=EB=
+=E5=F7=FC =E4=E5=F2=E5=E9 =EE=F2 =EF=F0=EE=F1=EC=EE=F2=F0=E0=20
+ =F2=E5=EB=E5=E2=E8=E7=EE=F0=E0 =E8 =E8=E3=F0 =ED=E0 =EA=EE=EC=EF=FC=
+=FE=F2=E5=F0=E5. =DD=F2=EE =EE=E3=F0=EE=EC=ED=E0=FF =EF=EE=EC=EE=F9=FC =E2=
+ =EF=EE=E7=ED=E0=ED=E8=E8 =F0=E5=E1=E5=ED=EA=EE=EC=20
+ =EC=E8=F0=E0, =EF=F0=E8=F7=E5=EC =E8=ED=F2=E5=F0=E5=F1=ED=E0=FF =E8=
+ =F0=EE=E4=E8=F2=E5=EB=FF=EC =F2=EE=E6=E5. =C0 =E3=EB=E0=E2=ED=EE=E5 - =C2=
+=FB =EC=EE=E6=E5=F2=E5 =E1=FB=F2=FC=20
+ =F1=EF=EE=EA=EE=E9=ED=FB =E7=E0 =EA=E0=F7=E5=F1=F2=E2=EE =E8 =E4=EE=
+=F1=F2=F3=EF=ED=EE=F1=F2=FC =EC=E0=F2=E5=F0=E8=E0=EB=E0, =EA=EE=EB=EB=E5=EA=
+=F6=E8=FF =F1=EE=E7=E4=E0=ED=E0=20
+ =EF=F0=EE=F4=E5=F1=F1=E8=EE=ED=E0=EB=FC=ED=EE, =F2=E2=EE=F0=F7=E5=F1=
+=EA=E8 =E8 =FD=EC=EE=F6=E8=EE=ED=E0=EB=FC=ED=EE =EF=EE-=E4=EE=E1=F0=EE=EC=
+=F3. =CA=E0=F7=E5=F1=F2=E2=EE =E7=E0=EF=E8=F1=E8,=20
+ =F0=E0=E1=EE=F2=E0 =E0=F0=F2=E8=F1=F2=EE=E2 =ED=E0 =E2=FB=F1=EE=EA=EE=
+=EC =F3=F0=EE=E2=ED=E5, =EF=EE=FD=F2=EE=EC=F3 =F0=E5=E1=E5=ED=EE=EA =ED=E5=
+ =EF=E5=F0=E5=E3=F0=F3=E6=E0=E5=F2=F1=FF=20
+ =E2=F2=EE=F0=EE=F1=F2=E5=EF=E5=ED=ED=EE=E9, =E0 =E7=E0=F7=E0=F1=F2=F3=
+=FE =E8 =ED=E5=ED=F3=E6=ED=EE=E9 =E2 =FE=ED=EE=EC =E2=EE=E7=F0=E0=F1=F2=E5=
+ =E8=ED=F4=EE=F0=EC=E0=F6=E8=E5=E9.=20
+ =CA=EE=EB=EB=E5=EA=F6=E8=FF =F0=E0=F1=F1=F7=E8=F2=E0=ED=E0 =ED=E0 =E2=
+=EE=E7=F0=E0=F1=F2 =EE=F2 3 =E4=EE 10 =EB=E5=F2. =CF=EE=F1=EB=E5 =EF=F0=EE=
+=F1=EB=F3=F8=E8=E2=E0=ED=E8=FF=20
+ =EA=E0=E6=E4=EE=E3=EE =E4=E8=F1=EA=E0 =F3 =C2=E0=F8=E5=E3=EE =F0=E5=
+=E1=B8=ED=EA=E0 =EE=E1=FF=E7=E0=F2=E5=EB=FC=ED=EE =EF=EE=FF=E2=FF=F2=F1=FF=
+ =ED=EE=E2=FB=E5 =F3=E2=EB=E5=F7=E5=ED=E8=FF =E8=20
+ =E8=ED=F2=E5=F0=E5=F1=FB. =CD=E0=F8=E0 =EA=EE=EB=EB=E5=EA=F6=E8=FF =
+=EF=EE=F1=EB=F3=E6=E8=F2 =EF=F0=E5=EA=F0=E0=F1=ED=FB=EC =EF=EE=E4=E0=F0=EA=
+=EE=EC, =ED=E5 =EB=E8=F8=E0=E9=F2=E5 =F1=E5=E1=FF =E8=20
+ =F1=E2=EE=E8=F5 =E1=EB=E8=E7=EA=E8=F5 =F3=E4=EE=E2=EE=EB=FC=F1=F2=E2=
+=E8=FF !</FONT></B></P></TD></TR>
+ <TR>
+ <TD>
+ <P align=3Dcenter><B><FONT color=3D#ff00ff face=3DArial>! =D1=EF=E8=
+=F1=EE=EA =E7=E0=EF=E8=F1=E5=E9=20
+ =E2=EE=F8=E5=E4=F8=E8=F5 =E2 =EA=EE=EB=EB=E5=EA=F6=E8=FE =E2=FB =EC=
+=EE=E6=E5=F2=E5 =F3=E2=E8=E4=E5=F2=FC =E2 =EF=F0=E8=EA=F0=E5=EF=EB=B8=ED=ED=
+=EE=EC =EA =EF=E8=F1=FC=EC=F3 =F4=E0=E9=EB=E5=20
+ !</FONT></B></P></TD></TR>
+ <TR>
+ <TD>
+ <P align=3Djustify><B><FONT face=3DArial>=C4=F0=E5=E2=ED=E8=E9 =C5=E3=
+=E8=EF=E5=F2; =C4=F0=E5=E2=ED=E8=E9 =D0=E8=EC;=20
+ =D3=E4=E8=E2=E8=F2=E5=EB=FC=ED=FB=E5 =ED=E0=F1=E5=EA=EE=EC=FB=E5; =D6=
+=E0=F0=F1=F2=E2=EE =F0=E0=F1=F2=E5=ED=E8=E9; =C4=E8=EA=E8=E5 =E6=E8=E2=EE=
+=F2=ED=FB=E5; =C4=EE=EC=E0=F8=ED=E8=E5=20
+ =E6=E8=E2=EE=F2=ED=FB=E5; =C6=E8=E2=EE=F2=ED=FB=E5 =D1=E8=E1=E8=F0=E8=
+; =C6=E8=E2=EE=F2=ED=FB=E5 =DE=E6=ED=EE=E9 =C0=EC=E5=F0=E8=EA=E8; =C0=E2=F2=
+=EE=EC=EE=E1=E8=EB=E8 - =F2=F0=E0=ED=F1=EF=EE=F0=F2;=20
+ =D1=E0=EC=EE=EB=B8=F2=FB - =E0=E2=E8=E0=F6=E8=FF; =CF=EE=EB=B8=F2=FB=
+ =E2 =EA=EE=F1=EC=EE=F1; =D3=E2=EB=E5=EA=E0=F2=E5=EB=FC=ED=E0=FF =E0=F1=F2=
+=F0=EE=ED=EE=EC=E8=FF; =CA=EE=F0=E0=E1=EB=E8 -=20
+ =EC=EE=F0=E5=EF=EB=E0=E2=E0=ED=E8=E5; =CE=E1=E8=F2=E0=F2=E5=EB=E8 =EE=
+=EA=E5=E0=ED=EE=E2; =C6=E8=E2=EE=F2=ED=FB=E5 =C0=E2=F1=F2=F0=E0=EB=E8=E8;=
+ =C6=E8=E2=EE=F2=ED=FB=E5 =C0=F4=F0=E8=EA=E8;=20
+ =CB=E5=F1=ED=FB=E5 =E8 =F3=E4=E8=E2=E8=F2=E5=EB=FC=ED=FB=E5 =EF=F2=E8=
+=F6=FB; =D3=E4=E8=E2=E8=F2=E5=EB=FC=ED=FB=E5 =F0=FB=E1=FB; =CB=E5=F1=E0 =D0=
+=EE=F1=F1=E8=E8; =C4=E8=ED=EE=E7=E0=E2=F0=FB =E8=20
+ =EF=F0=E5=F1=EC=FB=EA=E0=FE=F9=E8=E5=F1=FF; =C2=E5=EB=E8=EA=E0=FF =CE=
+=F2=E5=F7=E5=F1=F2=E2=E5=ED=ED=E0=FF; =D0=EE=F1=F1=E8=FF =E2 1812; =C2=E5=
+=EB=E8=EA=E8=E5 =EF=F3=F2=E5=F8=E5=F1=F2=E2=E8=FF;=20
+ =CF=EB=E0=ED=E5=F2=E0 =C7=E5=EC=EB=FF; =CD=E0=F0=EE=E4=ED=FB=E5 =E8=
+=ED=F1=F2=F0=F3=EC=E5=ED=F2=FB; =D1=E8=EC=F4=EE=ED=E8=F7=E5=F1=EA=E8=E9 =EE=
+=F0=EA=E5=F1=F2=F0; =CA=E0=EA =F1=E5=E1=FF=20
+ =E2=E5=F1=F2=E8; =CC=FB =F1=EE=F7=E8=ED=FF=E5=EC =F1=F2=E8=F5=E8; =C3=
+=EE=F0=EE=E4=F1=EA=E8=E5 =EF=F2=E8=F6=FB; =CF=F3=F2=E5=F8=E5=F1=F2=E2=E8=E5=
+ =F0=E0=F1=F2=E5=ED=E8=E9; =C4=F0=E5=E2=ED=FF=FF=20
+ =C3=F0=E5=F6=E8=FF; =C7=ED=E0=EA=EE=EC=F1=F2=E2=EE =F1 =F2=E5=E0=F2=
+=F0=EE=EC; =CA=E0=EA =E6=E8=EB=E8 =ED=E0 =D0=F3=F1=E8; =CC=EE=F1=EA=EE=E2=
+=F1=EA=E8=E9 =CA=F0=E5=EC=EB=FC; =CA=E0=EA=20
+ =F3=F1=F2=F0=EE=E5=ED =F7=E5=EB=EE=E2=E5=EA; =CA=E0=EB=E5=ED=E4=E0=F0=
+=FC =EF=F0=E8=F0=EE=E4=FB; =C7=E0=EC=E5=F7=E0=F2=E5=EB=FC=ED=FB=E5 =EF=F0=
+=EE=F4=E5=F1=F1=E8=E8; =CD=E0=F8=E0 =F1=F2=F0=E0=ED=E0=20
+ =D0=EE=F1=F1=E8=FF; =CF=EE=EB=E5=E7=ED=FB=E5 =E8=F1=EA=EE=EF=E0=E5=EC=
+=FB=E5; =CF=F3=F2=E5=F8=E5=F1=F2=E2=E8=E5 =E2 =CA=E0=EC=E5=ED=ED=FB=E9=20
+ =E2=E5=EA.</FONT></B></P></TD></TR>
+ <TR>
+ <TD>
+ <P align=3Djustify><SPAN lang=3Dru><B><FONT color=3D#0000ff face=3D=
+Arial>=C7=E0=EF=E8=F1=E8=20
+ </FONT></B></SPAN><B><FONT color=3D#0000ff face=3DArial>=E2 =F4=EE=F0=
+=EC=E0=F2=E5 MP3, =E1=E8=F2=F0=E5=E9=F2=20
+ =3D 320 kb\s<SPAN lang=3Den-us>. </SPAN>=C7=E0=EF=E8=F1=E0=ED=E0 =ED=
+=E0 =E2=ED=E5=F8=ED=E8=E9 USB =ED=E0=EA=EE=EF=E8=F2=E5=EB=FC=20
+ (=F4=EB=E5=F8=EA=E0). =CF=F0=EE=E1=EB=E5=EC =F1 =E2=EE=F1=EF=F0=EE=E8=
+=E7=E2=E5=E4=E5=ED=E8=E5=EC =ED=E5 =E2=EE=E7=ED=E8=EA=ED=E5=F2, =EC=EE=E6=
+=ED=EE =F1=EB=F3=F8=E0=F2=FC =ED=E0=20
+ =EA=EE=EC=EF=FC=FE=F2=E5=F0=E5, =EF=EB=E0=ED=F8=E5=F2=E5, =F1=EC=E0=
+=F0=F2=F4=EE=ED=E5, =F2=E5=EB=E5=E2=E8=E7=EE=F0=E5 =E8 =F2.=E4. =C7=E0=EF=
+=E8=F1=FC =ED=E0 =E2=ED=E5=F8=ED=E8=E9 USB=20
+ =ED=E0=EA=EE=EF=E8=F2=E5=EB=FC =E8=EC=E5=E5=F2 =F0=FF=E4 =EF=F0=E5=E8=
+=EC=F3=F9=E5=F1=F2=E2 =E2 =F1=F0=E0=E2=ED=E5=ED=E8=E8 =F1 =EE=E1=FB=F7=ED=
+=FB=EC=E8 CD =E4=E8=F1=EA=E0=EC=E8, USB=20
+ =ED=E0=EA=EE=EF=E8=F2=E5=EB=FC =E3=EE=F0=E0=E7=E4=EE =EB=E5=E3=F7=E5=
+, =E7=E0=ED=E8=EC=E0=E5=F2 =EC=E5=ED=FC=F8=E5 =EC=E5=F1=F2=E0, =EE=E1=EB=E0=
+=E4=E0=E5=F2 =E2=FB=F1=EE=EA=EE=E9=20
+ =ED=E0=E4=B8=E6=ED=EE=F1=F2=FC=FE =F1=EE=F5=F0=E0=ED=ED=EE=F1=F2=E8=
+ =E7=E0=EF=E8=F1=E5=E9, =E0 =FD=F2=EE =E7=ED=E0=F7=E8=F2, =F7=F2=EE =ED=E0=
+=F8=E0 =EA=EE=EB=EB=E5=EA=F6=E8=FF =E1=F3=E4=E5=F2=20
+ =F0=E0=E4=EE=E2=E0=F2=FC =C2=E0=F1 =EC=ED=EE=E3=EE =EB=E5=F2. =CC=FB=
+ =E3=E0=F0=E0=ED=F2=E8=F0=F3=E5=EC =EE=F2=EB=E8=F7=ED=EE=E5 =EA=E0=F7=E5=F1=
+=F2=E2=EE =E2=F1=E5=F5 =E7=E0=EF=E8=F1=E5=E9. =CD=E0=20
+ =F1=E0=EC=EE=EC =ED=EE=F1=E8=F2=E5=EB=E5 =F1=EE=E7=E4=E0=ED=E0 =EF=F0=
+=EE=E4=F3=EC=E0=ED=ED=E0=FF =F1=F2=F0=F3=EA=F2=F3=F0=E0, =E2=F1=E5 =E7=E0=
+=EF=E8=F1=E8 =F0=E0=E7=ED=E5=F1=E5=ED=FB =EF=EE=20
+ =EA=E0=F2=E0=EB=EE=E3=E0=EC, =E8=EC=E5=FE=F2=F1=FF =EF=EB=E5=E9=EB=E8=
+=F1=F2=FB, =EF=F0=EE=EF=E8=F1=E0=ED=FB =F2=E5=E3=E8, =E0 =F2=E0=EA=E6=E5 =
+=EF=EE=EB=ED=FB=E9 =F1=EF=E8=F1=EE=EA=20
+ =E2=EE=F8=E5=E4=F8=E8=F5 =E7=E0=EF=E8=F1=E5=E9, =EF=EE=FD=F2=EE=EC=F3=
+ =EF=F0=EE=E1=EB=E5=EC =F1 =EF=EE=E8=F1=EA=EE=EC =E8 =ED=E0=E2=E8=E3=E0=F6=
+=E8=E5=E9 =ED=E5=20
+ =E2=EE=E7=ED=E8=EA=ED=E5=F2.</FONT></B></P></TD></TR>
+ <TR>
+ <TD>
+ <P align=3Dcenter><B><FONT color=3D#ff0000 face=3DArial>=D1=F2=EE=E8=
+=EC=EE=F1=F2=FC =EA=EE=EB=EB=E5=EA=F6=E8=E8 =ED=E0=20
+ =E2=ED=E5=F8=ED=E5=EC USB =ED=E0=EA=EE=EF=E8=F2=E5=EB=E5 =97 6500 (=
+=D8=E5=F1=F2=FC =D2=FB=F1=FF=F7 =CF=FF=F2=FC=F1=EE=F2) =D0=F3=E1=EB=E5=E9=
+. <BR>=CF=F0=EE=E4=E0=FE=F2=F1=FF=20
+ =F2=EE=EB=FC=EA=EE =E2=EC=E5=F1=F2=E5. =C4=EE=F1=F2=E0=E2=EA=E0 </F=
+ONT></B><FONT color=3D#ff0000=20
+ face=3DArial><B>=E2=EA=EB=FE=F7=E5=ED=E0 </B></FONT><B><FONT color=3D=
+#ff0000 face=3DArial>=E2=20
+ =F1=F2=EE=E8=EC=EE=F1=F2=FC.</FONT></B></P></TD></TR>
+ <TR>
+ <TD>
+ <P align=3Djustify><B><SPAN style=3D'FONT-FAMILY: "Arial",sans-seri=
+f'>=C4=EE=F1=F2=E0=E2=EA=E0=20
+ =F2=EE=EB=FC=EA=EE =EF=EE=F7=F2=EE=E9 =EF=EE =E2=F1=E5=E9 =D0=EE=F1=
+=F1=E8=E8, =F1=F0=EE=EA=E8 7-14 =F1=F3=F2=EE=EA =F1 =EC=EE=EC=E5=ED=F2=E0=
+ =EE=F2=EF=F0=E0=E2=EA=E8. =CE=EF=EB=E0=F2=E0=20
+ =E2 =EC=EE=EC=E5=ED=F2 =EF=EE=EB=F3=F7=E5=ED=E8=FF =E7=E0=EA=E0=E7=E0=
+ =ED=E0 =EF=EE=F7=F2=E5 </SPAN><SPAN=20
+ style=3D"FONT-FAMILY: Arial,sans-serif">=ED=E0=EB=EE=E6=E5=ED=ED=FB=
+=EC =EF=EB=E0=F2=E5=E6=EE=EC</SPAN><SPAN=20
+ style=3D'FONT-FAMILY: "Arial",sans-serif'>. =D3 =ED=E0=F1 =ED=E5=F2=
+ =EA=F3=F0=FC=E5=F0=F1=EA=EE=E9 =E4=EE=F1=F2=E0=E2=EA=E8 =97=20
+ =F2=EE=EB=FC=EA=EE =EF=EE=F7=F2=EE=E9, =E2 =F2=EE=EC =F7=E8=F1=EB=E5=
+ =E8 =EF=EE =CC=EE=F1=EA=E2=E5.</SPAN></B></P>
+ <P align=3Djustify><FONT face=3DArial><STRONG><FONT color=3D#008000=
+>=C4=EB=FF=20
+ =EE=F4=EE=F0=EC=EB=E5=ED=E8=FF =E7=E0=EA=E0=E7=E0 =EF=F0=EE=F1=FC=E1=
+=E0 =ED=E5 =E7=E0=E1=FB=E2=E0=F2=FC=20
+ =F3=EA=E0=E7=FB=E2=E0=F2=FC:</FONT><BR>&nbsp;<BR>&nbsp;--- =C2=E0=F8=
+ =EF=EE=F7=F2=EE=E2=FB=E9 =E8=ED=E4=E5=EA=F1 (=EF=E8=F8=E8=F2=E5=20
+ =EF=F0=E0=E2=E8=EB=FC=ED=FB=E9 =E8=ED=E4=E5=EA=F1 =97 =FD=F2=EE =F3=
+=F1=EA=EE=F0=E8=F2 =E4=EE=F1=F2=E0=E2=EA=F3);<BR>&nbsp;--- =C2=E0=F8 =E3=EE=
+=F0=EE=E4 =E8 =F2=EE=F7=ED=FB=E9=20
+ =E0=E4=F0=E5=F1 (=ED=E0=E7=E2=E0=ED=E8=E5 =F3=EB=E8=F6=FB, =ED=EE=EC=
+=E5=F0 =E4=EE=EC=E0 =E8 =ED=EE=EC=E5=F0 =EA=E2=E0=F0=F2=E8=F0=FB);<BR>&nb=
+sp;--- =D4.=C8.=CE.=20
+ =EF=EE=EB=F3=F7=E0=F2=E5=EB=FF =E8 =CE=C1=DF=C7=C0=D2=C5=CB=DC=CD=CE=
+ =ED=EE=EC=E5=F0 =EA=EE=ED=F2=E0=EA=F2=ED=EE=E3=EE =F2=E5=EB=E5=F4=EE=ED=E0=
+ (=EB=F3=F7=F8=E5=20
+ =F1=EE=F2=EE=E2=FB=E9);<BR>&nbsp;<BR>=C7=E0=EA=E0=E7=FB\=E2=EE=EF=F0=
+=EE=F1=FB =ED=E0=EF=F0=E0=E2=EB=FF=E9=F2=E5 =EF=EE =E0=E4=F0=E5=F1=F3:</S=
+TRONG>=20
+ <B><A=20
+ href=3D"mailto:audioencyclopedia@cwhflash.ru">audioencyclopedia@cwh=
+flash.ru</A></B></FONT></P></TD></TR>
+ <TR>
+ <TD>
+ <P align=3Djustify><B><SPAN=20
+ style=3D'FONT-FAMILY: "Arial",sans-serif; COLOR: fuchsia'>=CC=FB =EE=
+=F7=E5=ED=FC=20
+ =EE=F2=E2=E5=F2=F1=F2=E2=E5=ED=ED=EE =EE=F2=ED=EE=F1=E8=EC=F1=FF =EA=
+ =EA=E0=F7=E5=F1=F2=E2=F3 =ED=E0=F8=E5=E3=EE =F2=EE=E2=E0=F0=E0, =EF=EE=FD=
+=F2=EE=EC=F3 =EF=E5=F0=E5=E4 =EE=F2=EF=F0=E0=E2=EA=EE=E9=20
+ =E2=F1=B8 =E4=EE=EF=EE=EB=ED=E8=F2=E5=EB=FC=ED=EE =EF=F0=EE=E2=E5=F0=
+=FF=E5=F2=F1=FF, =EA=E0=EA =F1=EB=E5=E4=F1=F2=E2=E8=E5 =EE=F2=EF=F0=E0=E2=
+=EA=E0 =E1=F0=E0=EA=EE=E2=E0=ED=ED=EE=E9=20
+ =EF=F0=EE=E4=F3=EA=F6=E8=E8 =F1=E2=E5=E4=E5=ED=E0 =EA =ED=F3=EB=FE.=
+ =D2=EE=E2=E0=F0 =F3=EF=E0=EA=EE=E2=FB=E2=E0=E5=F2=F1=FF =E2 =F1=EF=E5=F6=
+=E8=E0=EB=FC=ED=FB=E9 =F3=E4=E0=F0=EE=F1=F2=EE=E9=EA=E8=E9=20
+ =EC=E0=F2=E5=F0=E8=E0=EB, =F7=F2=EE =E2 =E7=ED=E0=F7=E8=F2=E5=EB=FC=
+=ED=EE=E9 =F1=F2=E5=EF=E5=ED=E8 =F3=EC=E5=ED=FC=F8=E0=E5=F2 =F0=E8=F1=EA =
+=EF=EE=E2=F0=E5=E6=E4=E5=ED=E8=FF =EF=F0=E8=20
+ =F2=F0=E0=ED=F1=EF=EE=F0=F2=E8=F0=EE=E2=EA=E5. =C5=F1=EB=E8 =E2=E4=F0=
+=F3=E3 =F1 =EF=EE=EB=F3=F7=E5=ED=ED=FB=EC =F2=EE=E2=E0=F0=EE=EC =E2=EE=E7=
+=ED=E8=EA=ED=F3=F2 =EF=F0=EE=E1=EB=E5=EC=FB, =F2=EE=20
+ =E2=F1=E5 =ED=E0=F8=E8 =EF=EE=EA=F3=EF=E0=F2=E5=EB=E8 =E2=F1=E5=E3=E4=
+=E0 =EC=EE=E3=F3=F2 =F0=E0=F1=F1=F7=E8=F2=FB=E2=E0=F2=FC =ED=E0 =EA=E2=E0=
+=EB=E8=F4=E8=F6=E8=F0=EE=E2=E0=ED=ED=F3=FE=20
+ =F2=E5=F5=ED=E8=F7=E5=F1=EA=F3=FE =EF=EE=E4=E4=E5=F0=E6=EA=F3. =CC=FB=
+ =ED=E8=EA=EE=E3=E4=E0 =ED=E5 =EE=F2=EA=E0=E7=FB=E2=E0=E5=EC=F1=FF =EE=F2=
+ =E3=E0=F0=E0=ED=F2=E8=E9=ED=FB=F5=20
+ =EE=E1=FF=E7=E0=F2=E5=EB=FC=F1=F2=E2, =E2 =F1=EB=F3=F7=E0=E5 =EF=F0=
+=EE=E1=EB=E5=EC=FB =C2=FB =EC=EE=E6=E5=F2=E5 =F0=E0=F1=F1=F7=E8=F2=FB=E2=E0=
+=F2=FC =ED=E0 =E7=E0=EC=E5=ED=F3, =EF=EE=F7=F2=EE=E2=FB=E5=20
+ =F0=E0=F1=F5=EE=E4=FB =EC=FB =E1=E5=F0=B8=EC =ED=E0 =F1=E5=E1=FF.</=
+SPAN></B></P></TD></TR>
+ <TR>
+ <TD>
+ <P align=3Djustify><SPAN style=3D"COLOR: #008000"><B><SPAN=20
+ style=3D"FONT-FAMILY: Arial,sans-serif">=CF=EE =E2=E0=F8=E5=EC=F3 =E6=
+=E5=EB=E0=ED=E8=FE, =E4=E0=ED=ED=E0=FF =EA=EE=EB=EB=E5=EA=F6=E8=FF=20
+ =EC=EE=E6=E5=F2 =E1=FB=F2=FC =E7=E0=EF=E8=F1=E0=ED=E0 =ED=E0 CD =E4=
+=E8=F1=EA=E8. =C4=EB=FF =E7=E0=EF=E8=F1=E8 =E8=F1=EF=EE=EB=FC=E7=F3=FE=F2=
+=F1=FF =ED=E0=E4=B8=E6=ED=FB=E5 CD =E4=E8=F1=EA=E8=20
+ =F1=EE =F1=EF=E5=F6=E8=E0=EB=FC=ED=FB=EC =EF=EE=EA=F0=FB=F2=E8=E5=EC=
+, =EA=EE=F2=EE=F0=EE=E5 =EF=EE=E2=FB=F8=E0=E5=F2 =F3=F1=F2=EE=E9=F7=E8=E2=
+=EE=F1=F2=FC =E4=E8=F1=EA=E0 =EA=20
+ =EC=E5=F5=E0=ED=E8=F7=E5=F1=EA=E8=EC =EF=EE=E2=F0=E5=E6=E4=E5=ED=E8=
+=FF=EC, =F2=E0=EA=E8=EC =EA=E0=EA =F2=F0=E5=F9=E8=ED=FB =E8 =F6=E0=F0=E0=EF=
+=E8=ED=FB, =E0 =FD=F2=EE =E7=ED=E0=F7=E8=F2, =F7=F2=EE=20
+ =ED=E0=F8=E0 =EA=EE=EB=EB=E5=EA=F6=E8=FF =E1=F3=E4=E5=F2 =F0=E0=E4=EE=
+=E2=E0=F2=FC =C2=E0=F1 =EC=ED=EE=E3=EE =EB=E5=F2. =CA=EE=EB=EB=E5=EA=F6=E8=
+=FF =F3=EF=E0=EA=EE=E2=E0=ED=E0 =E2=20
+ =EF=EB=E0=F1=F2=E8=EA=EE=E2=FB=E5 =E1=EE=EA=F1=FB (slim-dvd), =E8=EC=
+=E5=E5=F2 =EA=F0=E0=F1=E8=E2=FB=E5 =E8 =EF=F0=EE=E4=F3=EC=E0=ED=ED=FB=E5 =
+=EE=E1=EB=EE=E6=EA=E8, =F1=20
+ =EE=E1=F0=E0=F2=ED=EE=E9 =F1=F2=EE=F0=EE=ED=FB =EA=EE=F2=EE=F0=FB=F5=
+ =F3=EA=E0=E7=E0=ED =F1=EF=E8=F1=EE=EA =E2=EE=F8=E5=E4=F8=E8=F5 =ED=E0 =EA=
+=E0=E6=E4=FB=E9 =E4=E8=F1=EA <SPAN=20
+ lang=3Dru>=E7=E0=EF=E8=F1=E5=E9</SPAN> =E8 =E4=F0=F3=E3=E0=FF =EF=EE=
+=EB=E5=E7=ED=E0=FF =E8=ED=F4=EE=F0=EC=E0=F6=E8=FF, =EF=EE=FD=F2=EE=EC=F3 =
+=EF=F0=EE=E1=EB=E5=EC =F1=20
+ =EF=EE=E8=F1=EA=EE=EC =E8 =ED=E0=E2=E8=E3=E0=F6=E8=E5=E9 =ED=E5 =E2=
+=EE=E7=ED=E8=EA=ED=E5=F2. =C5=F1=EB=E8 =F5=EE=F2=E8=F2=E5 =EF=F0=E8=EE=E1=
+=F0=E5=F1=F2=E8 =EA=EE=EB=EB=E5=EA=F6=E8=FE,=20
+ =E7=E0=EF=E8=F1=E0=ED=ED=F3=FE =ED=E0 CD =E4=E8=F1=EA=E0=F5, =F2=EE=
+ =E2 =FD=F2=EE=EC =F1=EB=F3=F7=E0=E5 =EF=F0=EE=F1=FC=E1=E0 =F1=EE=EE=E1=F9=
+=E8=F2=FC =ED=E0=EC =EE=E1 =FD=F2=EE=EC =E2=20
+ =F1=E2=EE=E5=E9 =E7=E0=FF=E2=EA=E5, =F6=E5=ED=E0 =EF=F0=E5=E6=ED=FF=
+=FF, =EA=E0=EA =F3 =E2=E5=F0=F1=E8=E8 =ED=E0 =E2=ED=E5=F8=ED=E5=EC USB =ED=
+=E0=EA=EE=EF=E8=F2=E5=EB=E5=20
+ (=F4=EB=E5=F8=EA=E0) =97 <SPAN lang=3Den-us>65</SPAN>00 (=D8=E5=F1=F2=
+=FC =D2=FB=F1=FF=F7 =CF=FF=F2=FC=F1=EE=F2)=20
+ =D0=F3=E1=EB=E5=E9.</SPAN></B></SPAN></P></TD></TR>
+ <TR>
+ <TD>
+ <P align=3Dcenter><B><FONT color=3D#ff0000 face=3DArial>=C5=F1=EB=E8=
+ =E2=FB =ED=E5 =F5=EE=F2=E8=F2=E5 =E1=EE=EB=FC=F8=E5=20
+ =EF=EE=EB=F3=F7=E0=F2=FC =EE=F2 =ED=E0=F1 =EF=E8=F1=FC=EC=E0, =EE=F2=
+=EF=F0=E0=E2=FC=F2=E5 =ED=E0=EC =EF=E8=F1=FC=EC=EE =F1 =F2=E5=EC=EE=E9 =93=
+deletemail=94 =E8 =C2=E0=F8=20
+ =E0=E4=F0=E5=F1 =ED=E0=E2=F1=E5=E3=E4=E0 =E1=F3=E4=E5=F2 =F3=E4=E0=EB=
+=E5=ED =E0=E2=F2=EE=EC=E0=F2=E8=F7=E5=F1=EA=E8.</FONT></B></P></TD></TR>
+ <TR>
+ <TD>
+ <P align=3Dcenter><FONT face=3DArial>08_01_2019 02_10=20
+ 251564</FONT></P></TD></TR>
+ <TR>
+ <TD>
+ <P align=3Dcenter><FONT face=3DArial><A=20
+ href=3D"mailto:[%25=FEmail%25%25]">vsevolod@highsecure.ru</A></FONT=
+></P></TD></TR></TBODY></TABLE></BODY></HTML>
+
+--f5da00c32c2b4701679ab35c9255--
+
+--fe880034ac1aaf7f71abb1871107
+Content-Type: text/plain; name="=?windows-1251?B?0e/o8e7qLcDz5Oju/e326Orr7u/l5OjoLnR4?=
+ =?windows-1251?B?dA==?="
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="=?windows-1251?B?0e/o8e7qLcDz5Oju/e326Orr7u/l5OjoLnR4?=
+ =?windows-1251?B?dA==?="
+
+xPDl4u3o6SDF4+jv5fINCg0KLSDN4CDh5fDl4/Mg4uXr6Oru6SDw5eroDQotIM/o8ODs6OTgINXl
+7u/x4A0KLSDH4OPg5O737fvpINH06O3q8Q0KLSDK4Oog8fLw7ujr6CDv6PDg7Ojk+w0KLSDK8u4g
+8uDq7ukg9ODw4O7tDQotIMIg5PDl4u3l7CDM5ez06PHlDQotIM3u4vvpIMPu5CDiIMTw5eLt5ewg
+xePo7/LlDQotINHl5+7t+yDk8OXi7eXl4+jv5fLx6u7j7iDj7uTgDQotIM3g+OXx8uLo5SDj6Orx
+7vHu4g0KLSDP6/viuOwg7+4gzejr8y4gy+7y7vENCi0gx+Ds5ffg8uXr/O376SDv4O/o8PPxDQot
+IMIg+Oru6+UuIMjl8O7j6+j0+w0KLSDH4CDx8u7r7uwuIMTu7OD47ejlIO/o8u7s9vsNCi0g0eDk
+IOgg7uPu8O7kDQotIMLg5u3g/yDw4OHu8uAg7+jx9u7iDQotIMTg7OH7IOgg6uDt4Ov7LiDX6O3u
+4u3o6ugNCi0g0eXn7u0g7+D17vL7IOgg7+7x5eLu4g0KDQrE8OXi7ejpINDo7A0KDQotIMvl4+Xt
+5OAg7uEg7vHt7uLg7ejoINDo7OAgDQotIMPu8O7kIO3gIPHl7Ogg9e7r7OD1DQotIMLu6e37IPEg
+yuDw9ODj5e3u7A0KLSDT6+j24CDQ6OzgIOLw5ey47SDQ5fHv8+Hr6OroDQotIMTw5eLt5fDo7PHq
+4P8g+Oru6+ANCi0gxO7s8/EuIMDy8OjpDQotINfy7iDoIOrg6iDl6+gg4iDE8OXi7eXsINDo7OUN
+Ci0g0u7j4A0KLSDU7vDz7C4g0Ojs8eru5SDP8ODi7g0KLSDD8ODm5ODt6O0g0Ojs4A0KLSDC7vHx
+8uDt6OUg0e/g8PLg6uANCi0gw+DpIN7r6Okg1uXn4PD8DQotIMvl4+ju7eXw+w0KLSDK4Ovl7eTg
+8PwNCi0g0OjsIOLw5ey47SDI7O/l8OjoLiDB5fLu7Q0KLSDS8Ojz7PTg6/zt4P8g4PDq4A0KLSDK
+7uvo5+XpDQotIMDq4uXk8+ouINDo7PHq6OUg4eDt6A0KDQrT5Oji6PLl6/zt++Ug7eDx5eru7Pvl
+DQoNCi0gzOXk7u3u8e3g/yDv9+Xr4A0KLSDO8eANCi0gyuDqIO/35ev7IOTl6+D+8iDs5eQNCi0g
+2Ozl6/wNCi0gweDh7vfq4A0KLSDP8OXi8OD55e3o5SDj8/Hl7ej2+y4NCi0gzvLq8+TgIOHl8OXy
+8f8g+OXr6g0KLSDB7ub86CDq7vDu4uroIOgg8uvoLiDM8/Dg4uXpDQotIMzz8ODi5ent6OouIMbz
+6i3u6+Xt/A0KLSDR8vDl6u7n4A0KLSDK7uzg8PsNCi0gzPP16A0KLSDP4PPy6O3gDQotINHg8ODt
+9+ANCi0gyvPn7eX36OoNCg0K1uDw8fLi7iDw4PHy5e3o6Q0KDQotIMz7IOIg1uDw8fLi5SDw4PHy
+5e3o6Q0KLSDC4Obt4P8g7vHu4eXt7e7x8vwNCi0gwu7k7vDu8evoLerw7vjq6A0KLSDM7vUg6CDv
+4O/u8O7y7ejqDQotINPk6OLo8uXr/O375SDV4u7p7fvlDQotINHl7OXp8fLi7iDR7vHt7uL75Q0K
+LSDW4uXy6u7i++Ug8ODx8uXt6P8NCi0gy+jx8iwg6u7w5e38LCDx8uXh5ev8DQotIMLg5u3g/yD3
+4PHy/CDw4PHy5e3o/yCWIPbi5fLu6g0KLSDD5OUg8ODx8uXt6P8g7/D/9/PyIOfg7+Dx+w0KLSDC
+INLz7eTw5Q0KLSDP8/Hy++3/IMDy4Org7OANCi0g1/LuIOfgIPfz5O4/IQ0KLSDE7ubk5eLu6SDS
+8O7v6Pfl8ero6SDr5fEgDQotINHy5e/8LiDR5ezl6fHy4u4gx+vg6ugNCi0g0O7n7uL76SDx4OQN
+Cg0KxOjq6OUg5uji7vLt++UNCg0KLSDO4eXn/P/t+w0KLSDL5eINCi0g0ujj8Cwg8e3l5u376SDh
+4PDxDQotIMvl7u/g8OQsIOPl7+Dw5Cwg8eXs5enx8uLuIOru+OD3/Oj1DQotINHr7u0NCi0gweXj
+5ezu8g0KLSDC7uvqDQotIMzl5OLl5PwNCi0gz+Xr6Org7Q0KLSDR8vDg8/ENCi0gze7x7vDu4w0K
+DQrE7uzg+O3o5SDm6OLu8u375Q0KDQotIM/l8SDP7uvq4O0NCi0gyu7w7uLgIMfu8Pzq4A0KLSDK
+7vjq4CDM8/Dq4A0KLSDK7u38IMLl8uXw7uoNCi0gz/Lo9+jpIOTi7vAuIMrz8PssIPPy6ugsIOPz
+8egNCi0gzuL2+ywg4eDw4O37DQotIMHu8O7iIMHu8Pzq4CDoIPHi6O38/yDV4OLw7u38/w0KLSDK
+7ufgIMzg+OrgIOgg6u7n6+Xt7uog1OXk/OrgDQoNCsbo4u7y7fvlINHo4ejw6A0KDQotIMHl6+rg
+DQotIMzl5OLl5PwNCi0gyu7x8+v/DQotINHl8PvlIObz8ODi6+gsIPHy5fD16A0KLSDB4Onq4Ov8
+8erg/yDt5fDv4A0KLSDD7vDt7vHy4OkNCi0g0e7h7uv8DQotIMHz8PPt5PPqDQotIMrl5PDu4urg
+DQotIMrg4eDw4+ANCi0gzODw4OsNCi0g0O7x7uzg9eANCi0gzO746uANCi0g0eXi5fDt++kg7uvl
+7fwNCi0gz+j58/XgDQoNCsbo4u7y7fvlIN7m7e7pIMDs5fDo6ugNCg0KLSDe5u3g/yDA7OXw6Org
+DQotIMzg4+Xr6+Dt7uL7IO/o7ePi6O37DQotIMDt5PHq6Okg6u7t5O7wDQotIMLo8erg+OANCi0g
+2Ojt+Ojr6+ANCi0gw/Pg7eDq7g0KLSDN4O3k8w0KLSDM8/Dg4vzl5A0KLSDM7vDx6uD/IPHi6O3q
+4A0KLSDA4/Py6A0KLSDB8O7t5e3u8eX2DQotIMjj8PPt6ugNCi0g0OXi8+37DQotIN/j8+DwDQot
+IMDt4Oru7eTgDQotIM/o8ODt/OgNCi0g3evl6vLw6Pfl8ero6SDz4+7w/A0KLSDK7uvo4fDoDQot
+IMv/4/P46uAg5PDl4u7r4OcNCi0gwu7k7vHi6O3q4A0KLSDS4O/o8A0KLSDW5e/q7vXi7vHy++kg
+5Ojq7uHw4OcNCi0gy+Xt6OLl9g0KDQrA4vLu7O7h6OvoIOgg8vDg7fHv7vDyDQoNCi0gyu7r5fHu
+IC0g4uDm7eXp+OXlIOjn7uHw5fLl7ejlDQotIMrw5fHy/P/t8erg/yDy5evl4+ANCi0gyu7w7uvl
+4vHq4P8g6uDw5fLgDQotIMTi6OPg8uXr6CDi7fPy8OXt7eXj7iDx4+7w4O3o/w0KLSDP5fDi++Ug
+4OLy7uzu4ejr6A0KLSDA4vLu7O7h6OvoINTu8OTgLiDK7u3i5enl8A0KLSDD7u3u9+375SDh7uvo
+5PshDQotINLw4O3x7+7w8iDiIPHu4vDl7OXt7e7sIOPu8O7k5Q0KDQrR4Ozu67jy+yDoIODi6OD2
+6P8NCg0KLSDM5ffy4CDuIO/u67jy5Q0KLSDC7ufk8/jt++kg5+zl6Q0KLSDP7uT6uOzt4P8g8ejr
+4A0KLSDP5fDi++kg8+/w4OLr/+Xs++kg7+7ruPINCi0gzS4gxS4gxvPq7uLx6ujpDQotIM3g9+Dr
+7iD98Psg4OLo4Pbo6A0KLSDP5fLr/yDN5fHy5fDu4uANCi0gwvvx+OjpIO/o6+7y4OYuIMDxDQot
+IMLu5e3t4P8g6CDj8ODm5ODt8erg/yDg4ujg9uj/DQotIMDi6ODq7u3x8vDz6vLu8PsNCi0g0vDz
+5O3u5SDo8e/78uDt6OUNCi0g0eDs7uu48iDI6y0yDQotIMAuIM0uINLz7+7r5eINCi0g0OXg6vLo
+4u376SDk4ujj4PLl6/wNCi0gwP3w7u/u8PINCi0gyvLuIPLw8+To8vH/IOIg4OLo4Pbo6A0KDQrP
+7uu48vsg4iDq7vHs7vENCg0KLSDU5enl8OLl8OoNCi0g1/LuIPLg6u7lIOru8ezu7eDi8ujq4A0K
+LSDQ4Orl8u376SDk4ujj4PLl6/wg6CDy7u/r6OLuDQotIM/l8OLg/yDq7vHs6Pfl8erg/yDx6u7w
+7vHy/A0KLSDKLiDdLiDW6O7r6u7i8ero6Q0KLSDK7vHs7uTw7uwuINDg6uXy4A0KLSDQ4Orl8uAt
+7e7x6PLl6/wgq9Hv8/Lt6Oq7IA0KLSDP5fDi++kg6PHq8/Hx8uLl7e376SDx7/Py7ejqIMfl7Ovo
+DQotIMHl6+rgIOgg0fLw5evq4A0KLSDP7uu48iDe8Oj/IMPg4+Dw6O3gDQotIMru8ODh6/wgq8Lu
+8fLu6rsuINHq4PTg7eTwDQotIM/l8OL76SD35evu4uXqIOIg7vLq8Pvy7uwg6u7x7O7x5Q0KLSDO
+8OHo8uDr/O3g/yDx8uDt9uj/DQotIM3gIO7w4ejy4Ov87e7pIPHy4O326Oggq8zo8LsNCi0g0ODh
+7vLgIPHv8/Lt6Oru4g0KDQrT4uvl6uDy5ev87eD/IODx8vDu7e7s6P8NCg0KLSDR7/Py7ejqIMfl
+7OvoIC0gy/Pt4A0KLSDP7uLl8PXt7vHy/CDL8+37DQotINHu6+3l9+3g/yDx6PHy5ezgDQotIMzl
+8Orz8OjpLiDC5e3l8OAuIMzg8PENCi0g3u/o8uXwDQotINHg8vPw7Q0KLSDT8ODtLiDN5e/y8+0u
+IM/r8/Lu7Q0KLSDR7uvt9uUNCi0gyuDq6OUg4fvi4P7yIOfi5efk+w0KLSDR7ufi5efk6P8uIMHu
+6/z44P8gzOXk4uXk6PbgIOggzODr4P8gzOXk4uXk6PbgDQotIMzl8uXu8Ojy+w0KLSDK7uzl8vsN
+Ci0gzOvl9+376SDv8/L8LiDN4PjgIOPg6+Dq8ujq4A0KDQrK7vDg4evoIOgg7O7w5e/r4OLg7ejl
+DQoNCi0gz+735ezzIOru8ODh6+gg4iDi7uTlIO3lIPLu7fPyDQotIM/l8OL75SDs7vDl7+vg4uDy
+5evoDQotIMTw5eLt5ePw5ffl8erg/yDy8Ojl8OANCi0gyu7w7Pfo6S4gyu7w7O7i7uUg4uXx6+4N
+Ci0g0u7w4+7i++kg4+Dr6O7tDQotIMIg6uD+8uUg6uDv6PLg7eANCi0gyvPh8OjqLiDS8P7sDQot
+IM/o8ODy+yENCi0g0O7m5OXt6OUg0O7x8ejp8eru4+4g9Ovu8uANCi0gwe7y6Oog9uDw/yDP5fLw
+4A0KLSDK6+jv5fAg0uDp7+jtDQotIM/l8OL76SDv4PDu9e7kDQotIMvl5O7q7usgyvDg8ejtDQot
+IM7q5eDt8ero6SDr4Ont5fANCg0KzuHo8uDy5evoIO7q5eDt7uINCg0KLSDM7vDx6ujlIPfg6ero
+DQotIMDr/OHg8vDu8SDoIPTw5ePg8g0KLSDP8uj36Okg4eDn4PANCi0gyujyLiDO6uXg7fHq6OUg
+4uXr6Org7fsNCi0gzO7w8ero5SDq7vLo6ugNCi0gy+Xm4ej55SDy/uvl7ejpLiDB5ev76SDs5eTi
+5eT8DQotIMzu8OboLiDP7uTi7uTt7uUg7+Dx8uHo+eUNCi0gyuDqIPD74fsgIvDg5+Pu4uDw6OLg
+/vIiLiDCIOfg8O7x6//1IOzu8PHq7ukg6uDv8/Hy+w0KLSDK7vDg6+v7LiDK8u4g7+7x8vDu6Osg
+8Oj0Pw0KLSDd6+Xq8vDo9+Xx6ujpIPHq4PINCi0gwOrz6/sNCi0gxuXs9/Pm6O3gDQotIMvl8vP3
+6OUg8Pvh+w0KLSDM5eTz5/sNCi0gyuDq6OUg8Pvh+yDx4Oz75S3x4Oz75T8NCi0gxOXr/PTo7fsu
+IMLl8eXr7uUg6uDy4O3o5S4g0eLl8Pfu6g0KDQrG6OLu8u375SDA4vHy8ODr6OgNCg0KLSDM4PLl
+8OjqIMDi8fLw4Ovo/w0KLSDP6+D55e3u8e3g/yD/+eXw6PbgDQotIMrl7ePz8PMNCi0gwuDr6+Dh
+6A0KLSDC7uzh4PINCi0gz+7x8fPsDQotIMrz8erz8Q0KLSDd4urg6+jv8u7i++Ug6+Xx4C4gyu7g
+6+ANCi0g3ezzDQotIMLu6+3o8fL75SDv7u/z4+Dp9+jq6A0KLSDF9ejk7eANCi0g0/Lq7u3u8Q0K
+LSDR8+z34PL76SDi7uvqLiDS4Ozg7ejp8ero6SDk/P/i7usNCi0g2ODr4Pjt6OoNCi0g0ODp8ero
+5SDv8uj2+w0KLSDD6+Dn9+Dy++Ug6vPw+w0KLSDE6O3j7g0KLSDK8O7r6OroIOgg6+jx+w0KDQrG
+6OLu8u375SDA9PDo6ugNCg0KLSDE5ev88uAgzejr4A0KLSDI9e3l4uzu7SDo6+gg9ODw4O7t7uLg
+/yDq8Pvx4A0KLSDU5e3l6g0KLSDC5fDh6/7kDQotIMbo8OD0DQotIMD08Ojq4O3x6ujpIPHy8ODz
+8Q0KLSDR6+7tDQotIMHl4+Xs7vINCi0gze7x7vDu4w0KLSDL5eIuIMv84ujt++kg7/Dg6eQNCi0g
+wO3y6Ovu7+Ag4+3zLiDH5eHw4A0KLSDB7vDu5ODi7vft6OoNCi0g7ufl8O4gwujq8u7w6P8NCi0g
+yuDw6+jq7uL76SDh5ePl7O7yDQotIM3o6/zx6ujpIOrw7uru5OjrDQotINjo7O/g7eflDQotIMPu
+8Ojr6+ANCg0Ky+Xx7fvlIOgg8+To4ujy5ev87fvlIO/y6Pb7DQoNCi0gyvLuIPLg6ujlICLu8O3o
+8u7r7uPoIg0KLSDK8+rz+OrgLiDH4PD/7ergLiDC5fDy6Pjl6ergLg0KLSDP7u/u6+fl7fwuIMrr
+6O3y8/UuIMTw7ufkLg0KLSDK8ODv6OLt6OouIMbg4u7w7u3u6i4gxP/y5esuDQotIMji7uvj4C4g
+z+Xt7vfq4C4gw+vz9eDw/A0KLSDK7vDu6+XqLiDQ5fft7ukg8eLl8Pfu6i4g0e7r7uLl6SAuDQot
+IM/u7/Pj4Ogg4PDgLiDD4uDy5ezg6/zx6ujpIOri5efg6y4NCi0gy+jw7vXi7vHyLiDK8+rg4eDw
+8PsuDQotINjr5ezu7e7x7fvpIOrg5/Pg8C4g0ODp8ero5SDv8uj2+y4NCi0gx+7r7vLu6SD04Ofg
+7S4g0/Lq6C3s4O3k4PDo7eroLg0KLSDM5eTu8+rg5/fo6i4gzuH55fHy4uXt7fvlIPLq4PfoLg0K
+LSDA9PDo6uDt8ero5SDv6O3j4ujt+y4NCi0gyOzv5fDg8u7w8ero5SDv6O3j4ujt+w0KLSDP8uj3
+6Okg4eDn4PAuIM7r8/joLiDB4Orr4O37Lg0KLSDP7uv/8O3g/yDq8OD36uAg0P/h9+jqLiDR7uL7
+LiDU6Ovo7S4NCi0gy+7i9+jlIO/y6Pb7Lg0KLSDK6+Dx8SDv8uj2+w0KDQrT5Oji6PLl6/zt++Ug
+8Pvh+w0KDQotIMro8u7i4P8g4Orz6+AuIMrg6iDw++H7IOT7+ODyDQotIM/r4O3q8u7tLiDB7uru
+4uD/IOvo7ej/DQotIMzg7fLgDQotINXw//nl4vvlIOgg6u7x8u375SDw++H7DQotIMvl8vP36OUg
+8Pvh+w0KLSDL8+3gLfD74eANCi0gxujy5evoIOPr8+Ho7Q0KLSDO4ejy4PLl6+gg6u7w4Ovr7uLu
+4+4g8Oj04A0KLSDK4Ozh4OvgDQotIMHw++fj8+0g6CDv7uvn8+0NCi0gz/Du8u7v8uXwDQotINPk
+6OLo8uXr/O376SDq7u325fDyDQotINHu7A0KLSDB5evz4+AuINDz8fHq6Okg7vG48vANCi0gwevg
+4+7w7uTt++kg6+7x7vH8DQotIMrg8O8uIMfu6+7y4P8g8Pvh6uAgDQotIMDpIOTgIOrg8ODx/CEN
+Cg0Ky+Xx4CDQ7vHx6OgNCg0KLSDE8OXi7eXw8/Hx6u7lIO/u8eXr5e3o5Q0KLSDP7uTx5fft7i3u
+4+3l4u7lIOfl7Ovl5OXr6OUNCi0gz+7k4PDq6CDL5fHgLeHg8v746ugNCi0gy+XxLefg+ejy7ejq
+DQotIMvl8eAg4fvi4P7yIPDg5+375Q0KLSDF6/zt6OogDQotIMTg8Psg5evu4u7j7iDr5fHgDQot
+IMHl8Ljn7uL76SDr5fENCi0gweXwuOfgIC0g8+To4ujy5ev87e7lIOTl8OXi7g0KLSDR7vHt7uL7
+6SDh7vANCi0gyuDqIOvl8SDw5erg7CDv7uzu4+Dl8g0KLSDI4uAuIM/u6ezl7e376SDr5fENCi0g
+y+Xx7ufg+ejy7fvlIO/u6+7x+w0KLSDE8+Hw4OLgDQotIMfg7+7i5eTt6OroLiDL5fHu4u7k8fLi
+7g0KLSDL5fHu7+7x4OTq4A0KLSDS4Onj4CDoIOW4IO7h6PLg8uXr6A0KLSDL5fEgLSDt4PjlIOHu
+4+Dy8fLi7g0KDQrE6O3u5+Di8Psg6CDk8PPj6OUg7/Dl8ez76uD++ejl8f8NCg0KLSDP8OXx7Pvq
+4P756OXx/w0KLSDR8uXj7ufg4vAsIPLw6Pbl8ODy7u/xLCDy6PDg7e3u5+Di8A0KLSDB8O7t8u7n
+4OLwDQotIM/y5fDu5+Di8CDoIOj18uju5+Di8A0KLSDP7vfl7PMg6PH35efr6CDk6O3u5+Di8PsN
+Ci0g0evu7e7i4P8g9+Xw5e/g9eANCi0gx+XruO3g/yDs7vDx6uD/IPfl8OXv4PXgDQotIMPg8vLl
+8Oj/DQotIMru7O7k8ero6SDi4PDg7Q0KLSDR8uXt7e7pIOPl6uru7Q0KLSDV4Ozl6+Xu7Q0KLSDI
+7eTo6fHq4P8g6u7h8OANCi0gx+Dj4OTu9+375SDn7OXoDQotIN/o9+3g/yDn7OX/DQotIMrw7uru
+5Ojr+w0KLSDM6PHx6PHo7/Hq6Okg4Ovr6OPg8u7wDQoNCsLl6+jq4P8gzvLl9+Xx8uLl7e3g/w0K
+DQotIDIyIOj+7f8gMTk0MSDj7uTgLiDB8OXx8vHq4P8g6vDl7+7x8vwNCi0g3eLg6vPg9uj/LiDD
+ys4NCi0gwevu6uDk4CDL5e3o7ePw4OTgDQotIM/g8ODkIDct4+4g7e7/4fD/IDE5NDEg4+7k4A0K
+LSDM7vHq7uLx6uD/IOHo8uLgDQotIMTu8O7j4CDm6Oft6A0KLSDP4PDy6Ofg7fsgDQotIMIg0fLg
+6+jt4/Dg5OUNCi0g0fLg6+jt4/Dg5PHq4P8g4ejy4uAgDQotIMPl8O7o9+Xx6ujpIPL76w0KLSDK
+4PL++OAuINjy8/Ds7uLo6iDI6y0yDQotINLg7eog0i0zNA0KLSDK8/Dx6uD/IOTz4+AuIM/w7vXu
+8O7i8eru5SDx8ODm5e3o5Q0KLSDCIOfl7Ov/7erlDQotIM3gIMHl8Ovo7SEgz+7h5eTgIQ0KLSDP
+4PDg5CDP7uHl5PsNCg0K0O7x8ej/IOIgMTgxMg0KDQotIM3g9+Dr7iDi7unt+w0KLSDQ7vHx6Onx
+6uD/IODw7Oj/DQotINDu5OAg4u7p8eoNCi0gz+X17vLgLCDq4OLg6+Xw6P8sIO7h7ucNCi0g1ejy
+8O7x8vwgzeDv7uvl7u3gDQotINHw4Obl7ejlIO/w6CDK8ODx7e7sDQotINHs7uvl7fHq7uUg8fDg
+5uXt6OUNCi0gzOj14OjrIMjr6+Dw6O7t7uLo9yDK8/Lz5+7iDQotIM/g8PLo5+Dt+w0KLSDB7vDu
+5Ojt8eru5SDv7uvlDQotIM/l8OXkIOHu5ewNCi0gwe7w7uTo7fHq7uUg8fDg5uXt6OUuINHu4uXy
+IOIg1Ojr//UNCi0gzeDv7uvl7u0g4iDM7vHq4uUNCi0g0uDw8/Lo7fHq6Okg6+Dj5fD8LiDO7+7r
+9+Xt6OUNCi0gzODr7v/w7vHr4OLl9uru5SDx8ODm5e3o5Q0KLSDR8ODm5e3o5SDv8Oggwv/n/Ozl
+DQotIMHl4/Hy4u4g9PDg7fbz5+7iLiDP7uHl5OANCg0KwuXr6Oro5SDv8/Ll+OXx8uLo/w0KDQot
+INTo7ejq6Onx6ujlIOzu8OXv6+Di4PLl6+gNCi0gz+Xw4vvlIOPl7uPw4PTo9+Xx6ujlIOrg8PL7
+DQotINjl6+ru4vvpIO/z8vwuIMzg8OruIM/u6+4NCi0gzvLq8Pvy6OUgyu7r8+zh4A0KLSDC4PHq
+7iDk4CDD4OzgDQotIM/r4OLg7ejlIMzg4+Xr6+Dt4A0KLSDd6vHv5eTo9uj/IMHl8Ojt4+ANCi0g
+yvPqLiDO8urw+/Lo5SDA4vHy8ODr6OgNCi0gweXr6+jt8ePg8+fl7SDoIMvg5+Dw5eINCg0Kz+vg
+7eXy4CDH5ezr/w0KDQotINfy7iDy4Oru5SD46PDu8uAg6CDk7uvj7vLgLiDD6+7h8/ENCi0gw+7w
++w0KLSDO5+Xw4A0KLSDC+/Hu6u4g4iDj7vDg9Q0KLSDK8PPj7uLu8O7yIOLu5Psg4iDv8Ojw7uTl
+DQotIMzl8PLi7uUg7O7w5Q0KLSDP8Ojr6OL7IOgg7vLr6OL7DQotINHg8OPg8fHu4u4g7O7w5Q0K
+LSDA6fHh5fDjDQotIM/u4/Dz5uDl7PH/IQ0KLSDX8u4g8yDt4Pjl6SDv6+Dt5fL7IOIg7fPy8Ojo
+DQotIMLz6+rg7Q0KLSDD5enn5fANCi0gwuXx5evu5SDq8+/g7ejlDQoNCs3g8O7k7fvlIOjt8fLw
+8+zl7fL7DQoNCi0gw/Px6+gNCi0g0vDoIOPw8+/v+yDs8+f76uDr/O379SDo7fHy8PPs5e3y7uIN
+Ci0gxO7s8OANCi0gyvLuIPLg6ujlIPHq7uzu8O716A0KLSDB4Ovg6+Dp6uANCi0gw/Pk7uoNCi0g
+0vDl+e7y6uAuIMHz4eXtDQotIMvu5uroLiDQ8+Hl6/wNCi0gyu7r7uru6/z36OroLiDB8+Hl7ffo
+6ugNCi0gxPP17uL75SDo7fHy8PPs5e3y+y4g0eLo8fLz6/zq4A0KLSDR4ujw5ev8DQotIMbg6+Xp
+6uANCi0g0O7m7uoNCi0gw+Dw7O7t6OrgDQotIMHg/+0NCi0gyuDqIOfi8/fo8iDu8Orl8fLwDQoN
+CtHo7PTu7ej35fHq6Okg7vDq5fHy8A0KDQotIM7w6uXx8vDu4uD/IP/s4C4gxOjw6Obl8A0KLSDK
+7u3y8ODh4PENCi0gwuju6+7t9+Xr/A0KLSDA6/zyDQotINHq8Ojv6uANCi0gwPD04A0KLSDU6+Xp
+8uAt7+jq6u7r7iwg4e7r/Pjg/yD06+Xp8uAuDQotIMPu4e7pDQotIMrr4PDt5fINCi0g1ODj7vIN
+Ci0g0vDz4eAsIPLw7uzh7u0NCi0g0vDz4eANCi0gwuDr8u7w7eANCi0g0uDsLfLg7Cwg8uDw5evq
+6Cwg8vDl8+Pu6/zt6OosIOHz4eXtLiANCi0gyu7r7uru6/z36OroLCDq7uvu6u7r4Cwg6+jy4OLw
++y4NCi0g1/LuIPMg8O7/6/8g4iDt8/Lw6OguDQotINHo7PTu7ej35fHq6Okg7vDq5fHy8A0KDQrK
+4Oog8eXh/yDi5fHy6A0KDQotINfy7iDs+yDv7uTg8OjsPyANCi0g1/LuIPLg6u7lIOLl5uvo4u7x
+8vwNCi0g1+jx8u7y4CDi7urw8+Mg7eDxDQotIM3gIPPr6PblLiDR4uXy7vTu8CDoIOfl4fDgDQot
+IM/u5Ofl7O376SDv5fDl9e7kDQotINHr7uLuIKvo5+Lo7ejy5bsNCi0gyuDqIO3lIO/u8uXw//L8
+8f8NCi0gwiDy5eDy8OUgDQotIMIg5/Do8uXr/O3u7CDn4OvlIA0KLSDN5fPk4OL46Onx/yDw4Ofj
+7uLu8CANCi0g0e/u8Ojy/CDy7ublIO3z5u3uIPPs5fL8IQ0KLSDA8OPz7OXt8ujw7uLg7e376SDx
+7+7wDQotIMIg7uH55fHy4uXt7e7sIPLw4O3x7+7w8uUNCi0g0vDoIO/w4OLo6+Ag4+7x8v8NCi0g
+wiDj7vHy//UNCi0gx+Ag8fLu6+7sDQotIMTr/yD35ePuIO3z5u37IO/w4OLo6+ANCi0gzvfl7fwg
+wuDm7e7lIO/w4OLo6+4NCg0KzPsg8e736O3/5ewg8fLo9egNCg0KLSDT5ODw5e3o5S4g0evu4+gN
+Ci0g0Oj07OANCi0g0Ojy7A0KLSDR8uj17vLi7vDt++kg8ODn7OXwLiDR8u7v4A0KLSDf7OENCi0g
+1e7w5ekuINHy8O70+w0KLSDP8/jq6O3x6uD/IPHy8O704A0KLSDD5ern4Ozl8vAuIMHl6/vpIPHy
+6PUNCi0gzOXy4PTu8OANCi0gzuvo9uXy4u7w5e3o5Q0KLSDH4vPq7u/u5PDg5uDt6OUNCg0Kw+7w
+7uTx6ujlIO/y6Pb7DQoNCi0gwu7w7uHl6Q0KLSDD7uvz4egNCi0g0ejt6PbgDQotINHt5ePo8PwN
+Ci0gx//h6+jqDQotIMzz9e7r7uLq4C3v5fHy8PP46uANCi0gzuLx/+3q4A0KLSDH5evl7fP46uAN
+Ci0g0eDk7uLg/yDj7vDo9eLu8fLq4A0KLSDX5ffl4uj24A0KLSDQ5fft++Ug9+Dp6ugNCi0gwu7w
+7u3gDQotIMvg8fLu9+rgDQoNCs/z8uX45fHy4ujlIPDg8fLl7ejpDQoNCi0gyuDw8u705ev8DQot
+IM/u7Ojk7vAuIM/l8OX2DQotIMvz6iDoIPfl8e3u6g0KLSDM7vDq7uL8LiDX8u4g8uDq7uUg8eXr
+5er26P8NCi0gyuDv8/Hy4A0KLSDW4PD8LcPu8O71DQotIMrg6uDuLiDY7uru6+DkDQotINHg9eDw
+7fvpIPLw7vHy7ejqLiDO4/Pw5fYNCi0g0eLl6uvgDQotINfg6e376SDq8/HyLiDX4OkNCi0gy+js
+7u0uINbo8vDz8fsuIMDv5ev88ejtDQotIMrg6iDw4PHy5fIg8OjxDQotIM/45e3o9uAuINDu5vwN
+Cg0KxPDl4u3//yDD8OX26P8NCg0KLSDP5fDi7uUg5+3g6u7s8fLi7iDxIN3r6+Dk7ukNCi0gzeAg
+wOrw7u/u6+UNCi0gzODx8uXw4CDE8OXi7eXpIMPw5fbo6A0KLSDP4PD05e3u7Q0KLSDE8OXi7eXj
+8OX35fHq6OUg4uDn+w0KLSDR8uDk6O7tLiDO6+js7+jp8ero5SDI4/D7DQotINHv4PDy4A0KLSDA
+4+7w4CANCi0gwiDy5eDy8OUNCi0gw+7s5fANCi0gzvLl9iDo8fLu8OjoIJYgw+Xw7uTu8g0KLSDR
+7urw4PIuINfy7iDy4Oru5SD06Ovu8e706P8NCi0g0eDk+yDA6uDk5ezo6C4gz+vg8u7tIOggwPDo
+8fLu8uXr/A0KLSDD6O/v7urw4PIgDQotIMDw9ejs5eQg6CDl4+4g5+Dq7u0NCg0Kx+3g6u7s8fLi
+7iDxIPLl4PLw7uwNCg0KLSDX8u4g8uDq7uUg8uXg8vAuIM7v5fDgLiDB4Ovl8g0KLSDK4Oro5SDh
+++Lg/vIg8uXg8vD7DQotIMru7OXk6P8sIPLw4OPl5Oj/IOgg7OXr7uTw4OzgDQotINH25e3gIOgg
+5+DrDQotINTu6eUuINLl4PLw4Ov87fvlIO/w4OLo6+ANCi0g0uXg8vDg6/zt++kg5+DrDQotIM/u
+5O3o7ODl7PH/IO3gIPH25e3zDQotIMDs7+vz4A0KLSDP8O705fHx6P8t4Ory5fANCi0gw/Do7OXw
+7fvlDQotINji5ent++kg9uX1LiDS5eDy8ODr/O376SD18+Tu5u3o6g0KLSDB8/Lg9O7w6P87INDl
+5ujx8eXwDQotIM7j7u38LCDi7uTgIOgg7OXk7fvlIPLw8+H7DQoNCsrg6iDm6OvoIO3gINDz8egN
+Cg0KLSDE8OXi7eXw8/Hx6uD/IOTl8OXi7f8NCi0gyvPw7eD/IOjn4eANCi0g0PPx8erg/yDv5ff8
+DQotIM3gIO7j7vDu5OUuIMzl5CDoIPLw4OL/7e7pIPfg6Q0KLSDL/O3/7eD/IPLq4O38LiDL8/fo
+7eANCi0gzPPm8erg/yDw4OHu8uANCi0gzuTl5uTgOiDx7vDu9+j24Cwg8eDw4PTg7Q0KLSDN4CDk
+4u7w5S4gwiDq6+Xy6C4gz+vz4y4g0eDt6A0KLSDQ8/Hx6uD/IOHg7f8NCi0gwe7w8u3o6uguIMvl
+8SAtIOru8Ozo6+X2DQotIM3u4uPu8O7kLiDP7vHg5O3o6i4gwuX35TsNCi0g0u7w4y4gzOXw+yDk
+6+jt+y4gweXw5fHy4A0KLSDK8OXs6/wuIM/w6PHy4O38LiDK7vDg4evoDQotINHy4PD75SDn7eDq
+7uz75Q0KDQrM7vHq7uLx6ujpIMrw5ezr/A0KDQotINfy7iDy4Oru5SDK8OXs6/wNCi0gz+Xw4u7l
+IOft4Oru7PHy4u4NCi0gwe7w7uLo9uro6SD17uvsIOIg5PDl4u3u8fLoDQotIM/l8OL76SDK8OXs
+6/wuIMHw5eLl7ffg8vvlIPHy5e37DQotIMHl6+7q4Ozl7e376SDK8OXs6/wuIMTs6PLw6OkgxO7t
+8eru6Q0KLSDN7uL76SDK8OXs6/wuIMjy4Ov8/+3x6ujlIODw9ejy5ery7vD7DQotINLu8OPu4uD/
+IO/r7vng5PwNCi0g0eru7O7w7vjo7eANCi0gz+735ezzIO/r7vng5PwgLSDK8ODx7eD/DQotIMrw
+5ezruOLx6ujlIPHy5e37IOgg4eD47egNCi0gz/Du5O7r5uDl7CDn7eDq7uzo8vzx/yDxIOHg+O3/
+7OgNCi0gzeDh4PLt4P8g4eD47f8NCi0g0e7h7vDt4P8g7+vu+eDk/A0KLSDW4PD8Leru6+7q7usg
+6CDW4PD8Le/z+OrgDQotIMrw4PHt4P8g7+vu+eDk/CDiIO3g+Ogg5O3oDQotIM3l5+Dh++Lg5ez7
+6SDv4PDg5A0KLSDK8/Dg7fL7IOH8/vIuINHv4PHx6uD/IOHg+O3/DQoNCsrg6iDz8fLw7uXtIPfl
+6+7i5eoNCg0KLSDR5fDk9uUuIM/z6/zxDQotIMDw8uXw6Ogg6CDi5e37DQotIMT79eDt6OUNCi0g
+x+D35ewg7eDsIOro8evu8O7kDQotIMfz4fsNCi0gz+j55eLg8OXt6OUNCi0gxOv/IPfl4+4g7fPm
+7eAg4u7k4A0KLSDI5yD35ePuIPHu8fLu//Ig7/Du5PPq8vsNCi0gzPv49vsg6CDx6uXr5fINCi0g
+yu7m4A0KLSDQ5fbl7/Lu8PsNCi0gw+7r7uLt7ukg7O7n4w0KLSDH8OXt6OUNCi0g0evz9Q0KDQrK
+4Ovl7eTg8Pwg7/Do8O7k+w0KDQotINHl7fL/4fD8LiDP5fDi++kg5uXr8vvpIOvo8fINCi0gzury
+/+Hw/C4gxvPw4OLr6A0KLSDN7v/h8PwuINHi6PDo8fLl6+gNCi0gxOXq4OHw/C4gz/Do9e7kIOfo
+7PsNCi0g3+3i4PD8LiDN7uL76SDj7uQuINTl4vDg6/wuIMzl8uXr6A0KLSDM4PDyLiDD8OD36CDv
+8Ojr5fLl6+gNCi0gwO/w5ev8LiDP7uvu4u7k/OUNCi0gzODpLiDR7uvu4uXpDQotIMj+7fwuINHu
+6+325SDi7vDu8g0KLSDI/uv8LiDM4Orz+OrgIOvl8uANCi0gwOLj8/HyLiDP7vDgIPPw7ubg/w0K
+DQrH4Ozl9+Dy5ev87fvlIO/w7vTl8fHo6A0KDQotIMru7OHg6e3l8Cwg8vDg6vLu8Ojx8g0KLSDN
+4CDs7uvu9+3u6SD05fDs5Q0KLSDP5eTg4+7jDQotIMLw4Pfl4e375SDx7+X26ODr/O3u8fLoDQot
+IM/u5uDw7fvlLiDe8Ojx8vsNCi0gz/Du9OXx8ejoIOLu5e3t+/UNCi0g0vDg7fHv7vDy7fvlIO/w
+7vTl8fHo6A0KLSDA4ujg6u7t8fLw8+ry7vD7DQotIMrg6uD/IPDg4e7y4CDzIOvl8e3o6uANCi0g
+wiDq4PTlLiDP7uLg8A0KLSDR8vDu6PLl6+guIMDw9ejy5ery7vANCi0gz/Du7Pv46+Xt7e7x8vwu
+IMPl7uvu4+gNCi0gwOLy7uzg8uj35fHq4P8g6+jt6P8NCi0gz/Du4/Dg7Ozo8fL7DQotINLi7vD3
+5fHq6OUg7/Du9OXx8ejoDQotIMPk5SDw4OHu8uDl8iDk/+T/IMrz5/8NCi0g0/e47fvlDQoNCs3g
++OAg8fLw4O3gINDu8fHo/w0KDQotIMPk5SDs+yDu6uDn4Ovo8fwNCi0gw+Xu4/Dg9Oj/INDu8fHo
+6A0KLSDN4PjoIOHu4+Dy8fLi4A0KLSDE7Ojy8OjpIMTu7fHq7ukNCi0gyuDw4Ozn6O0uIMvl8u7v
+6PHoDQotIMTw5eLt//8g0PPx/C4gyOPuDQotIM/u6+ru4u7k9vsNCi0gzeAg8fLw8+Pg9SDF8Ozg
+6uANCi0gze7i7uUg4+7x8+Tg8PHy4u4gLSDQ7vHx6P8NCi0gwiDv7uzu8PHq7uwg8eXr5Q0KLSDB
+++vo7fsg6CDx6uDn6uguIMzg8fLl8OANCi0gz7jy8CDP5fDi++kNCi0gz/P46ujtDQotINDz8fHq
+4P8g6+jy5fDg8vPw4A0KLSDS8OXy/P/q7uLx6uD/IOPg6+Xw5f8NCi0gMTIg4O/w5ev/IDE5NjEt
+4+4g4+7k4A0KLSDMw9MuINDu8fHo6fHq4P8g7eDz6uANCi0g0fLu6+j24CDQ7uTo7fsgliDM7vHq
+4uANCg0Kz+7r5eft++Ug6PHq7u/g5ez75Q0KDQotIM/l8ffg7fvpIOrg8Pzl8A0KLSDD6+jt4CDo
+IPHu6/wNCi0gzeAg8e7r/+3u6SDi4PDt6PblDQotIMfu6+7y7g0KLSDM5fHy7vDu5uTl7ej/IOfu
+6+7y4A0KLSDA6+zg5+376SD07u3kDQotINfl7CDn4O3o7ODl8vH/IO3g8+rgIOPl7uvu4+j/DQot
+IMPl7uvu4+gNCi0g0+Pu6/zt4P8g+OD18uANCi0gyuDqIO/u/+Lo6/H/IPPj7uv8DQotIM3l9PL/
+7e7pIO/w7uz78eXrDQotIM/w6PDu5O376SDj4OcNCi0gyvPw8erg/yDs4OPt6PLt4P8g4O3u7ODr
+6P8NCi0g1/LuIPLg6u7lIP3r5ezl7fL7DQotINfz4/PtIOgg8fLg6/wNCi0g0PPk+yD24uXy7fv1
+IOzl8uDr6+7iDQotIMrg6iDv7uvl5+375SDo8eru7+Dl7PvlIO/u7O7j4P7yIOL78ODx8ujy/CDz
+8O7m4OkNCg0Kz/Py5fjl8fLi6OUg4iDK4Ozl7e376SDi5eoNCg0KLSDE4uDk9uDy/CDy+/H/9yDr
+5fIg7eDn4OQNCi0gwiDm6Ovo+eUNCi0gyPHy7vDo/yDQ/e4gy/P3+OXj7iDO9e7y7ejq4A0KLSDN
+4CDq7uPuIO717vLo6+jx/CDiIMrg7OXt7e7sIOLl6uUNCi0gz+7x5evl7ejlLiDCIOPu8fL/9SDz
+INHy4PDu4+4gzODx8uXw4A0KLSDR5erw5fL7IOzg8fLl8PHy4uANCi0g1/LuIOXr6CDk8OXi7ejl
+IOv+5OgNCi0gzuTl5uTgDQotIM/u9+Xs8yDs4Ozu7fIgliDr8/f44P8g5O7h+/fgDQotINXz5O7m
+7ejqIMDw8vMNCi0g0uDt5fYg0fvt7uLl6SDB8/Du4+4gzOXk4uXk/w0KLSDO9e7y4CENCi0gwu7v
+8O7x+yDoIO7y4uXy+w0KLSDN4CDv8ODn5O3o6uUuIMzz5/vq4O3y+w0KLSDT8u736uAgzODr+/jq
+6CDd6egNCi0gwPD15e7r7uPo
+
+--fe880034ac1aaf7f71abb1871107--
diff --git a/test/functional/messages/spam_message.eml b/test/functional/messages/spam_message.eml
new file mode 100644
index 0000000..38168cf
--- /dev/null
+++ b/test/functional/messages/spam_message.eml
@@ -0,0 +1,59 @@
+Received: from server.chat-met-vreemden.nl (unknown [IPv6:2a01:7c8:aab6:26d:5054:ff:fed1:1da2])
+ (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
+ (Client did not present a certificate)
+ by mx1.freebsd.org (Postfix) with ESMTPS id CF0171862
+ for <test@example.com>; Mon, 6 Jul 2015 09:01:20 +0000 (UTC)
+ (envelope-from upwest201diana@outlook.com)
+Received: from ca-18-193-131.service.infuturo.it ([151.18.193.131] helo=User)
+ by server.chat-met-vreemden.nl with esmtpa (Exim 4.76)
+ (envelope-from <upwest201diana@outlook.com>)
+ id 1ZC1sl-0006b4-TU; Mon, 06 Jul 2015 10:36:08 +0200
+Reply-To: <h_ada15@aol.co.uk>
+From: "Un Approved"<upwest201diana@outlook.com>
+Subject: 06.07.2015
+Date: Mon, 6 Jul 2015 10:35:56 +0200
+MIME-Version: 1.0
+Content-Type: text/plain;
+ charset="Windows-1251"
+Content-Transfer-Encoding: 7bit
+X-Priority: 3
+X-MSMail-Priority: Normal
+X-Mailer: Microsoft Outlook Express 6.00.2600.0000
+X-MimeOLE: Produced By Microsoft MimeOLE V6.00.2600.0000
+
+Dear Western Union Beneficiary,
+
+
+UNITED NATIONS PAYMENT NOTIFICATION.
+
+We wish to inform you that the United Nations (UN) has authorized us to remit to you a total amount of $920,000.00, (Nine Hundred and Twenty Thousand United States Dollars).
+
+Your Cash prize was paid out to us by the United Nations, and they have successfully succeeded in depositing your whole funds with us here at Western Union London United Kingdom.
+
+They have now ordered us to take full responsibility in the transfer process of your funds and thus commence the immediate remittance of your funds to you.
+
+Be duly informed that because of our Western Union transfer policy, your funds will be paid to you via our Western Union Daily Transfer limit of $4,600.00 USD. This means that you will Continuously receive a daily amount of $4,600.00 USD, and this amount Can be collected from any of our numerous Western Union outlets in your current location.
+
+To begin the claim process of your daily payment as stated above, kindly furnish us with the following;
+
+Full Name:
+Address:
+Phone Number:
+
+Upon the receipt of the above mentioned details, your first transaction will be activated and we shall then Proceed to provide you with the Money Transfer Control Number (MTCN) for the First installment and we will continue to email you others after 12 hours of Receiving each payment.
+
+For more information on your payment status;
+
+Contact Person
+Henry Adams
+Email: h_ada15@aol.co.uk
+
+OR call our 24 hours Helpline @ +442032903681, for any inquiries on the above message.
+
+
+Yours truly,
+
+Diana Mckay
+For: Western Union London, United Kingdom.
+WESTERN UNION... Over 380,000 Outlets Worldwide
+
diff --git a/test/functional/messages/spam_message.eml.fuzzy1 b/test/functional/messages/spam_message.eml.fuzzy1
new file mode 100644
index 0000000..986d524
--- /dev/null
+++ b/test/functional/messages/spam_message.eml.fuzzy1
@@ -0,0 +1,34 @@
+MIME-Version: 1.0
+Content-Type: text/plain;
+ charset="Windows-1251"
+Content-Transfer-Encoding: 7bit
+
+Your Cash prize was paid out to us by the United Nations, and they have successfully succeeded in depositing your whole funds with us here at Western Union London United Kingdom.
+
+They have now ordered us to take full responsibility in the transfer process of your funds and thus commence the immediate remittance of your funds to you.
+
+Be duly informed that because of our Western Union transfer policy, your funds will be paid to you via our Western Union Daily Transfer limit of $4,600.00 USD. This means that you will Continuously receive a daily amount of $4,600.00 USD, and this amount Can be collected from any of our numerous Western Union outlets in your current location.
+
+To begin the claim process of your daily payment as stated above, kindly furnish us with the following;
+
+Full Name:
+Address:
+Phone Number:
+
+Upon the receipt of the above mentioned details, your first transaction will be activated and we shall then Proceed to provide you with the Money Transfer Control Number (MTCN) for the First installment and we will continue to email you others after 12 hours of Receiving each payment.
+
+For more information on your payment status;
+
+Contact Person
+Henry Adams
+Email: h_ada15@aol.co.uk
+
+OR call our 24 hours Helpline @ +442032903681, for any inquiries on the above message.
+
+
+Yours truly,
+
+Diana Mckay
+For: Western Union London, United Kingdom.
+WESTERN UNION... Over 380,000 Outlets Worldwide
+
diff --git a/test/functional/messages/subject1.eml b/test/functional/messages/subject1.eml
new file mode 100644
index 0000000..473bf98
--- /dev/null
+++ b/test/functional/messages/subject1.eml
@@ -0,0 +1,3 @@
+Subject: Some subject
+
+Hello
diff --git a/test/functional/messages/url1.eml b/test/functional/messages/url1.eml
new file mode 100644
index 0000000..42361ae
--- /dev/null
+++ b/test/functional/messages/url1.eml
@@ -0,0 +1,3 @@
+Content-Type: text/plain
+
+hello https://www.example.com/foo?a=b hello
diff --git a/test/functional/messages/url10.eml b/test/functional/messages/url10.eml
new file mode 100644
index 0000000..525d4bc
--- /dev/null
+++ b/test/functional/messages/url10.eml
@@ -0,0 +1,6 @@
+Content-Type: text/html
+Content-Transfer-Encoding: 8bit
+From: user@example.com
+To: undisclosed-recipients;;
+
+<a href="https:/\test&shy;test.&shy;com/redirect?url=https%3A%2F%2Fexample%2Ecom&amp;urlhash=rH0t#100xp@example.com">click here!</a> \ No newline at end of file
diff --git a/test/functional/messages/url11.eml b/test/functional/messages/url11.eml
new file mode 100644
index 0000000..82ddbf7
--- /dev/null
+++ b/test/functional/messages/url11.eml
@@ -0,0 +1,4 @@
+Content-Type: text/plain
+
+http://clean.dirty.sanchez.com
+http://not.dirty.sanchez.com \ No newline at end of file
diff --git a/test/functional/messages/url12.eml b/test/functional/messages/url12.eml
new file mode 100644
index 0000000..75e8ecc
--- /dev/null
+++ b/test/functional/messages/url12.eml
@@ -0,0 +1,4 @@
+Content-Type: text/plain
+
+http://4.very.dirty.sanchez.com
+http://not.dirty.sanchez.com \ No newline at end of file
diff --git a/test/functional/messages/url13.eml b/test/functional/messages/url13.eml
new file mode 100644
index 0000000..f73c965
--- /dev/null
+++ b/test/functional/messages/url13.eml
@@ -0,0 +1,4 @@
+Content-Type: text/plain
+
+http://41.black.sanchez.com
+http://black.sanchez.com \ No newline at end of file
diff --git a/test/functional/messages/url14.eml b/test/functional/messages/url14.eml
new file mode 100644
index 0000000..32632da
--- /dev/null
+++ b/test/functional/messages/url14.eml
@@ -0,0 +1,4 @@
+Content-Type: text/plain
+
+user@dirty.sanchez.com
+user@admin.dirty.sanchez.com \ No newline at end of file
diff --git a/test/functional/messages/url15.eml b/test/functional/messages/url15.eml
new file mode 100644
index 0000000..1240979
--- /dev/null
+++ b/test/functional/messages/url15.eml
@@ -0,0 +1,3 @@
+Content-Type: text/plain
+
+4@very.dirty.sanchez.com \ No newline at end of file
diff --git a/test/functional/messages/url16.eml b/test/functional/messages/url16.eml
new file mode 100644
index 0000000..0dbe807
--- /dev/null
+++ b/test/functional/messages/url16.eml
@@ -0,0 +1,5 @@
+Content-Type: text/plain
+
+test@admin.dirty.sanchez.com
+admin@41.black.sanchez.com
+admin@black.sanchez.com \ No newline at end of file
diff --git a/test/functional/messages/url2.eml b/test/functional/messages/url2.eml
new file mode 100644
index 0000000..bfa7253
--- /dev/null
+++ b/test/functional/messages/url2.eml
@@ -0,0 +1,3 @@
+Content-Type: text/plain
+
+hello https://www.rspamd.com/doc/modules/multimap.html hello
diff --git a/test/functional/messages/url3.eml b/test/functional/messages/url3.eml
new file mode 100644
index 0000000..eabfd98
--- /dev/null
+++ b/test/functional/messages/url3.eml
@@ -0,0 +1,3 @@
+Content-Type: text/html
+
+hello <a href="https://www.rspamd.com/doc/modules/multimap.html">https://www.bank.com</a> hello
diff --git a/test/functional/messages/url4.eml b/test/functional/messages/url4.eml
new file mode 100644
index 0000000..eba4f87
--- /dev/null
+++ b/test/functional/messages/url4.eml
@@ -0,0 +1,4 @@
+Content-Type: text/html
+
+hello <a href="https://example.com/">https://rspamd.tk</a> hello
+hello <a href="https://rspamd.com/doc/modules/multimap.html">https://rspamd.com</a> hello
diff --git a/test/functional/messages/url5.eml b/test/functional/messages/url5.eml
new file mode 100644
index 0000000..13dd651
--- /dev/null
+++ b/test/functional/messages/url5.eml
@@ -0,0 +1,3 @@
+Content-Type: text/html
+
+hello <a href = "https://www.example.net/?">hello</a>
diff --git a/test/functional/messages/url6.eml b/test/functional/messages/url6.eml
new file mode 100644
index 0000000..7bc0b5b
--- /dev/null
+++ b/test/functional/messages/url6.eml
@@ -0,0 +1,3 @@
+Content-Type: text/html
+
+hello <a href = "https://www.example.org/?">hello</a>
diff --git a/test/functional/messages/url7.eml b/test/functional/messages/url7.eml
new file mode 100644
index 0000000..63d1c53
--- /dev/null
+++ b/test/functional/messages/url7.eml
@@ -0,0 +1,3 @@
+Content-Type: text/html
+
+hello <a href = "https://example.ru/?">hello</a>
diff --git a/test/functional/messages/url8.eml b/test/functional/messages/url8.eml
new file mode 100644
index 0000000..7d1358d
--- /dev/null
+++ b/test/functional/messages/url8.eml
@@ -0,0 +1,3 @@
+Content-Type: text/plain
+
+http://мойÑайт.рф \ No newline at end of file
diff --git a/test/functional/messages/url9.eml b/test/functional/messages/url9.eml
new file mode 100644
index 0000000..b6091f2
--- /dev/null
+++ b/test/functional/messages/url9.eml
@@ -0,0 +1,3 @@
+Content-Type: text/plain
+
+http://xn--80arbjktj.xn--p1ai \ No newline at end of file
diff --git a/test/functional/messages/urlimage.eml b/test/functional/messages/urlimage.eml
new file mode 100644
index 0000000..6ae7088
--- /dev/null
+++ b/test/functional/messages/urlimage.eml
@@ -0,0 +1,5 @@
+Content-Type: text/html
+
+helo
+<body text="#000000" bgcolor="#FFFFFF">
+ <p><img src="https://example.com" alt=""moz-do-not-send="false" width="326" height="326" border="0"></a></p>
diff --git a/test/functional/messages/urlinsubject.eml b/test/functional/messages/urlinsubject.eml
new file mode 100644
index 0000000..4f8a5a5
--- /dev/null
+++ b/test/functional/messages/urlinsubject.eml
@@ -0,0 +1,7 @@
+Content-Type: text/html; charset="utf-8"
+Content-Transfer-Encoding: quoted-printable
+Subject: http://example.com
+Date: Tue, 4 Sep 2019 23:25:01 +0000
+
+
+hello \ No newline at end of file
diff --git a/test/functional/messages/urlinsubjectencoded.eml b/test/functional/messages/urlinsubjectencoded.eml
new file mode 100644
index 0000000..3562b23
--- /dev/null
+++ b/test/functional/messages/urlinsubjectencoded.eml
@@ -0,0 +1,9 @@
+Content-Type: text/html; charset="utf-8"
+Content-Transfer-Encoding: quoted-printable
+Subject: =?utf-8?b?0J/QntCU0KLQktCV0KDQlNCY0KLQlSDQktCr0J/Qm9CQ0KLQoyAt?=
+ =?utf-8?q?_https=3A//example=2Ecom/nT6a--Check_out_our_New_=F0=9F=97=BA?=
+ =?utf-8?b?77iPIEdlb2dyYXBoeSBUaGVtZWQgU2V0IPCfj5TvuI8g?=
+Date: Tue, 4 Sep 2019 23:25:01 +0000
+
+
+hello \ No newline at end of file
diff --git a/test/functional/messages/utf.eml b/test/functional/messages/utf.eml
new file mode 100644
index 0000000..93513cc
--- /dev/null
+++ b/test/functional/messages/utf.eml
@@ -0,0 +1,15 @@
+To: =?UTF-8?B?0JLQsNGB0LjRgdGD0LDQu9C40Lkg0JXQstC70LDQvNC/0LjQtdCy0LjRhw==?=
+ <xxx@example.com>
+From: ZZZ <yy@example.com>
+Subject: Hello
+Message-ID: <f13bcdad-273b-9a44-6209-cb1631c01dca@example.com>
+Date: Sun, 31 Jul 2016 11:40:08 +0100
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:45.0)
+ Gecko/20100101 Thunderbird/45.2.0
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+
+
+--
+ZZZ
diff --git a/test/functional/messages/whitelist.eml b/test/functional/messages/whitelist.eml
new file mode 100644
index 0000000..aa19512
--- /dev/null
+++ b/test/functional/messages/whitelist.eml
@@ -0,0 +1,3 @@
+Content-Type: text/plain
+
+http://rspamd-test.com
diff --git a/test/functional/messages/yand_forward.eml b/test/functional/messages/yand_forward.eml
new file mode 100644
index 0000000..8ea2636
--- /dev/null
+++ b/test/functional/messages/yand_forward.eml
@@ -0,0 +1,31 @@
+Return-Path: <test@test.net>
+Received: from mx7.test.ru ([0.0.0.0] verified)
+ by mail.test.ru (test mail)
+ with ESMTP id 60578244
+ (envelope-from <test@test.net>)
+ for test@test.ru; Mon, 27 Apr 2020 19:45:05 +0300
+Received: from forward106p.test.eu (forward106p.test.eu [])
+ by mail.test.ru (Postfix) with ESMTP id 9B38C6E043A
+ (envelope-from <test@test.net>)
+ for <test@test.ru>; Mon, 27 Apr 2020 19:45:05 +0300 (MSK)
+Received: from forward106p.test.eu (forward106p.test.eu [])
+ by test.ru (test) with ESMTP id j51GAaUB;
+ Mon, 27 Apr 2020 16:45:05 +0000
+Received: from forward102q.test.eu (forward102q.test.eu [])
+ by forward106p.test.eu (test) with ESMTP id 71D441C81C0C
+ (envelope-from <test@test.net>)
+ for <test@test.ru>; Mon, 27 Apr 2020 19:45:05 +0300 (MSK)
+Received: from mxback10q.test.eu (mxback10q.test.eu [])
+ by forward102q.test.eu (test) with ESMTP id 6E2CA7F20010
+ (envelope-from <test@test.net>)
+ for <test@test.ru>; Mon, 27 Apr 2020 19:45:05 +0300 (MSK)
+From: test@test.eu
+Envelope-From: test@test.net
+To: <test@test.ru>,
+ <test@test.com>
+In-Reply-To: <test@test.eu>
+Subject: Fwd: 123
+MIME-Version: 1.0
+Date: Mon, 27 Apr 2020 19:45:05 +0300
+
+lsakdlsa;dk \ No newline at end of file
diff --git a/test/functional/messages/zerofont.eml b/test/functional/messages/zerofont.eml
new file mode 100644
index 0000000..06af493
--- /dev/null
+++ b/test/functional/messages/zerofont.eml
@@ -0,0 +1,20 @@
+From: foobar@example.com
+Content-Type: text/html
+
+<!doctype html>
+<html lang="en-US" dir="ltr">
+ <head>
+ <meta charset="utf-8">
+ <meta http-equiv="Content-Security-Policy-Report-Only" content="script-src 'unsafe-inline'; img-src http: https: data: blob:; style-src 'unsafe-inline'; child-src 'none'; object-src 'none'; report-uri https://tiles.services.mozilla.com/v4/links/activity-stream/csp">
+ <title>New Tab</title>
+ <link rel="icon" type="image/png" href="chrome://branding/content/icon32.png"/>
+ <link rel="stylesheet" href="chrome://browser/content/contentSearchUI.css" />
+ <link rel="stylesheet" href="resource://activity-stream/css/activity-stream.css" />
+ </head>
+ <body class="activity-stream">
+ <div>fi<span style="FONT-SIZE: 0px">le </span>sh<span style="FONT-SIZE: 0px">aring </span></div>
+ <a href="https://example.com">test url</a>
+
+<div>foo<div>bar</div>ttt<div>ololo</div>baz</div>
+ </body>
+</html>
diff --git a/test/functional/messages/zip-doublebad.eml b/test/functional/messages/zip-doublebad.eml
new file mode 100644
index 0000000..b8fbb41
--- /dev/null
+++ b/test/functional/messages/zip-doublebad.eml
@@ -0,0 +1,12 @@
+Content-Type: multipart/mixed; boundary="=__i0GhaBNmijobzbiTaqMvfI"
+
+--=__i0GhaBNmijobzbiTaqMvfI
+Content-Type: application/zip; name=f.zip
+Content-Disposition: attachment; size=174; filename=f.zip
+Content-Transfer-Encoding: base64
+
+UEsDBAoAAAAAAJNg6kgAAAAAAAAAAAAAAAAMABwAZmFrZS5wZGYuZXhlVVQJAAO1HYJXtR2CV3V4
+CwABBOgDAAAE6AMAAFBLAQIeAwoAAAAAAJNg6kgAAAAAAAAAAAAAAAAMABgAAAAAAAAAAAC0gQAA
+AABmYWtlLnBkZi5leGVVVAUAA7Udgld1eAsAAQToAwAABOgDAABQSwUGAAAAAAEAAQBSAAAARgAA
+AAAA
+--=__i0GhaBNmijobzbiTaqMvfI--
diff --git a/test/functional/messages/zip.eml b/test/functional/messages/zip.eml
new file mode 100644
index 0000000..dd6bc0b
--- /dev/null
+++ b/test/functional/messages/zip.eml
@@ -0,0 +1,11 @@
+Content-Type: multipart/mixed; boundary="=_MlaYox31rMNP821ZlG2h4Xe"
+
+--=_MlaYox31rMNP821ZlG2h4Xe
+Content-Type: application/zip; name=f.zip
+Content-Disposition: attachment; size=166; filename=f.zip
+Content-Transfer-Encoding: base64
+
+UEsDBAoAAAAAAINe6kgAAAAAAAAAAAAAAAAIABwAZmFrZS5leGVVVAkAA8YaglfGGoJXdXgLAAEE
+6AMAAAToAwAAUEsBAh4DCgAAAAAAg17qSAAAAAAAAAAAAAAAAAgAGAAAAAAAAAAAALSBAAAAAGZh
+a2UuZXhlVVQFAAPGGoJXdXgLAAEE6AMAAAToAwAAUEsFBgAAAAABAAEATgAAAEIAAAAAAA==
+--=_MlaYox31rMNP821ZlG2h4Xe--
diff --git a/test/functional/util/dummy_avast.py b/test/functional/util/dummy_avast.py
new file mode 100755
index 0000000..a4a6c62
--- /dev/null
+++ b/test/functional/util/dummy_avast.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+PID = "/tmp/dummy_avast.pid"
+
+import os
+import socket
+import socketserver
+import sys
+
+import dummy_killer
+
+class MyTCPHandler(socketserver.BaseRequestHandler):
+
+ def handle(self):
+ self.request.sendall(b"220 DAEMON\r\n")
+ self.data = self.request.recv(1024).strip()
+ self.request.sendall(b"210 SCAN DATA\r\n")
+ if self.server.foundvirus:
+ self.request.sendall(b"SCAN /some/path/malware/xpaj/00908235ee9e267fa2f4c83fb4304c63af976cbc\t[L]0.0\t0 Eicar\\ [Heur]\r\n")
+ else:
+ self.request.sendall(b"SCAN /some/path/malware/xpaj/00908235ee9e267fa2f4c83fb4304c63af976cbc\t[+]\r\n")
+ self.request.sendall(b"200 SCAN OK\r\n")
+ self.request.close()
+
+if __name__ == "__main__":
+ HOST = "localhost"
+
+ alen = len(sys.argv)
+ if alen > 1:
+ port = int(sys.argv[1])
+ if alen >= 3:
+ foundvirus = bool(sys.argv[2])
+ else:
+ foundvirus = False
+ else:
+ port = 3310
+ foundvirus = False
+
+ server = socketserver.TCPServer((HOST, port), MyTCPHandler, bind_and_activate=False)
+ server.allow_reuse_address = True
+ server.foundvirus = foundvirus
+ server.server_bind()
+ server.server_activate()
+
+ dummy_killer.setup_killer(server)
+ dummy_killer.write_pid(PID)
+
+ try:
+ server.handle_request()
+ except socket.error:
+ print("Socket closed")
+
+ server.server_close()
diff --git a/test/functional/util/dummy_clam.py b/test/functional/util/dummy_clam.py
new file mode 100755
index 0000000..1b614f5
--- /dev/null
+++ b/test/functional/util/dummy_clam.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+PID = "/tmp/dummy_clamav.pid"
+
+import os
+import socket
+import socketserver
+import sys
+
+import dummy_killer
+
+class MyTCPHandler(socketserver.BaseRequestHandler):
+
+ def handle(self):
+ self.data = self.request.recv(1024).strip()
+ if self.server.foundvirus:
+ self.request.sendall(b"stream: Eicar-Test-Signature FOUND\0")
+ else:
+ self.request.sendall(b"stream: OK\0")
+ self.request.close()
+
+if __name__ == "__main__":
+ HOST = "localhost"
+
+ alen = len(sys.argv)
+ if alen > 1:
+ port = int(sys.argv[1])
+ if alen >= 3:
+ foundvirus = bool(sys.argv[2])
+ else:
+ foundvirus = False
+ else:
+ port = 3310
+ foundvirus = False
+
+ server = socketserver.TCPServer((HOST, port), MyTCPHandler, bind_and_activate=False)
+ server.allow_reuse_address = True
+ server.foundvirus = foundvirus
+ server.server_bind()
+ server.server_activate()
+
+ dummy_killer.setup_killer(server)
+ dummy_killer.write_pid(PID)
+
+ try:
+ server.handle_request()
+ except socket.error:
+ print("Socket closed")
+
+ server.server_close()
diff --git a/test/functional/util/dummy_fprot.py b/test/functional/util/dummy_fprot.py
new file mode 100755
index 0000000..31ae2c4
--- /dev/null
+++ b/test/functional/util/dummy_fprot.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+import os
+import signal
+import socket
+import socketserver
+import sys
+
+import dummy_killer
+
+PID = "/tmp/dummy_fprot.pid"
+
+class MyTCPHandler(socketserver.BaseRequestHandler):
+
+ def handle(self):
+ self.data = self.request.recv(1024).strip()
+ if self.server.foundvirus:
+ self.request.sendall(b"1 <infected: EICAR_Test_File> FOO->bar\n")
+ else:
+ self.request.sendall(b"0 <clean> FOO\n")
+ self.request.close()
+
+if __name__ == "__main__":
+
+ HOST = "localhost"
+
+ alen = len(sys.argv)
+ if alen > 1:
+ port = int(sys.argv[1])
+ if alen >= 4:
+ PID = sys.argv[3]
+ foundvirus = bool(sys.argv[2])
+ elif alen >= 3:
+ foundvirus = bool(sys.argv[2])
+ else:
+ foundvirus = False
+ else:
+ port = 10200
+ foundvirus = False
+
+ server = socketserver.TCPServer((HOST, port), MyTCPHandler, bind_and_activate=False)
+ server.allow_reuse_address = True
+ server.foundvirus = foundvirus
+ server.server_bind()
+ server.server_activate()
+
+ dummy_killer.setup_killer(server)
+ dummy_killer.write_pid(PID)
+
+ try:
+ server.handle_request()
+ except socket.error:
+ print("Socket closed")
+ server.server_close()
diff --git a/test/functional/util/dummy_http.py b/test/functional/util/dummy_http.py
new file mode 100755
index 0000000..c1abf7e
--- /dev/null
+++ b/test/functional/util/dummy_http.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+
+import asyncio
+import dummy_killer
+import tornado.ioloop
+import tornado.web
+import tornado.httpserver
+import ssl
+import argparse
+import os
+
+class MainHandler(tornado.web.RequestHandler):
+ @tornado.gen.coroutine
+ def get(self, path):
+ if path == '/empty':
+ # Return an empty reply
+ self.set_header("Content-Type", "text/plain")
+ self.write("")
+ elif path == '/error_403':
+ # Return a 403 HTTP error
+ raise tornado.web.HTTPError(403)
+ elif path == '/timeout':
+ # Wait for 4 seconds before returning an empty reply
+ yield tornado.gen.sleep(4)
+ self.set_header("Content-Type", "text/plain")
+ self.write("")
+ elif path == '/request':
+ # Return a string 'hello world'
+ self.set_header("Content-Type", "text/plain")
+ self.write("hello world")
+ elif path == '/map-simple':
+ # Return a string 'hello map'
+ self.set_header("Content-Type", "text/plain")
+ self.write("hello map")
+ elif path == '/map-query':
+ # Parse the 'key' argument from the HTTP request
+ key = self.get_query_argument("key", default=None)
+ if key == 'au':
+ # Return a string 'hit' if 'key' is equal to 'au'
+ self.set_header("Content-Type", "text/plain")
+ self.write("1.0")
+ else:
+ # Return a 404 HTTP error if 'key' is not equal to 'au'
+ raise tornado.web.HTTPError(404)
+ elif path == '/settings':
+ self.set_header("Content-Type", "application/json")
+ self.write("{\"actions\": { \"reject\": 1.0}, \"symbols\": { \"EXTERNAL_SETTINGS\": 1.0 }}")
+ else:
+ raise tornado.web.HTTPError(404)
+
+ @tornado.gen.coroutine
+ def post(self, path):
+ if path == '/empty':
+ # Return an empty reply
+ self.set_header("Content-Type", "text/plain")
+ self.write("")
+ elif path == '/error_403':
+ # Return a 403 HTTP error
+ raise tornado.web.HTTPError(403)
+ elif path == '/request':
+ # Return a string 'hello post'
+ self.set_header("Content-Type", "text/plain")
+ self.write("hello post")
+ elif path == '/timeout':
+ # Wait for 4 seconds before returning an empty reply
+ yield tornado.gen.sleep(4)
+ self.set_header("Content-Type", "text/plain")
+ self.write("")
+ elif path == '/map-simple':
+ # Return a string 'hello map'
+ self.set_header("Content-Type", "text/plain")
+ self.write("hello map")
+ elif path == '/map-query':
+ # Parse the 'key' argument from the HTTP request
+ key = self.get_query_argument("key", default="")
+ if key == 'au':
+ # Return a string 'hit' if 'key' is equal to 'au'
+ self.set_header("Content-Type", "text/plain")
+ self.write("hit")
+ else:
+ # Return a 404 HTTP error if 'key' is not equal to 'au'
+ raise tornado.web.HTTPError(404)
+ elif path == '/settings':
+ self.set_header("Content-Type", "application/json")
+ self.write("{\"actions\": { \"reject\": 1.0}, \"symbols\": { \"EXTERNAL_SETTINGS\": 1.0 }}")
+ else:
+ raise tornado.web.HTTPError(404)
+
+ def head(self, path):
+ self.set_header("Content-Type", "text/plain")
+ if path == "/redirect1":
+ # Send an HTTP redirect to the bind address of the server
+ self.redirect(f"{self.request.protocol}://{self.request.host}/hello")
+ elif path == "/redirect2":
+ # Send an HTTP redirect to the bind address of the server
+ self.redirect(f"{self.request.protocol}://{self.request.host}/redirect1")
+ elif self.path == "/redirect3":
+ # Send an HTTP redirect to the bind address of the server
+ self.redirect(f"{self.request.protocol}://{self.request.host}/redirect4")
+ elif self.path == "/redirect4":
+ # Send an HTTP redirect to the bind address of the server
+ self.redirect(f"{self.request.protocol}://{self.request.host}/redirect3")
+ else:
+ self.send_response(200)
+ self.set_header("Content-Type", "text/plain")
+
+def make_app():
+ return tornado.web.Application([
+ (r"(/[^/]+)", MainHandler),
+ ])
+
+async def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--bind", "-b", default="localhost", help="bind address")
+ parser.add_argument("--port", "-p", type=int, default=18080, help="bind port")
+ parser.add_argument("--keyfile", "-k", help="server private key file")
+ parser.add_argument("--certfile", "-c", help="server certificate file")
+ parser.add_argument("--pidfile", "-pf", help="path to the PID file")
+ args = parser.parse_args()
+
+ # Create the Tornado application
+ app = make_app()
+
+ # If keyfile and certfile are provided, create an HTTPS server.
+ # Otherwise, create an HTTP server.
+ if args.keyfile and args.certfile:
+ ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+ ssl_ctx.load_cert_chain(args.certfile, args.keyfile)
+ server = tornado.httpserver.HTTPServer(app, ssl_options=ssl_ctx)
+ else:
+ server = tornado.httpserver.HTTPServer(app)
+
+ # Write the PID to the specified PID file, if provided
+ if args.pidfile:
+ dummy_killer.write_pid(args.pidfile)
+
+ # Start the server
+ server.bind(args.port, args.bind)
+ server.start(1)
+
+ await asyncio.Event().wait()
+
+if __name__ == "__main__":
+ loop = asyncio.get_event_loop()
+ loop.run_until_complete(main())
diff --git a/test/functional/util/dummy_killer.py b/test/functional/util/dummy_killer.py
new file mode 100644
index 0000000..0a052fb
--- /dev/null
+++ b/test/functional/util/dummy_killer.py
@@ -0,0 +1,30 @@
+import signal
+import os
+import atexit
+import tempfile
+
+def setup_killer(server, method = None):
+ def default_method():
+ server.server_close()
+
+ if method is None:
+ method = default_method
+
+ def alarm_handler(signum, frame):
+ method()
+
+ signal.signal(signal.SIGALRM, alarm_handler)
+ signal.signal(signal.SIGTERM, alarm_handler)
+ signal.alarm(120)
+
+
+def write_pid(path):
+ with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
+ f.write(str(os.getpid()))
+ f.close()
+ os.rename(f.name, path)
+
+ def cleanup():
+ os.remove(path)
+
+ atexit.register(cleanup)
diff --git a/test/functional/util/dummy_p0f.py b/test/functional/util/dummy_p0f.py
new file mode 100755
index 0000000..1d86ba0
--- /dev/null
+++ b/test/functional/util/dummy_p0f.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+PID = "/tmp/dummy_p0f.pid"
+
+import os
+import sys
+import struct
+import socket
+import socketserver
+
+import dummy_killer
+
+class MyStreamHandler(socketserver.BaseRequestHandler):
+
+ def handle(self):
+ S = {
+ 'bad_query' : 0x0,
+ 'ok' : 0x10,
+ 'no_match' : 0x20
+ }
+
+ OS = {
+ 'windows' : (b'Windows', b'7 or 8'),
+ 'linux' : (b'Linux', b'3.11 and newer')
+ }
+
+ self.data = self.request.recv(21).strip()
+
+ if self.server.p0f_status == 'bad_response':
+ response = 0
+ else:
+ response = struct.pack(
+ "IbIIIIIIIhbb32s32s32s32s32s32s",
+ 0x50304602, # magic
+ S[self.server.p0f_status], # status
+ 1568493408, # first_seen
+ 1568493408, # last_seen
+ 1, # total_conn
+ 1, # uptime_min
+ 4, # up_mod_days
+ 1568493408, # last_nat
+ 1568493408, # last_chg
+ 10, # distance
+ 0, # bad_sw
+ 0, # os_match_q
+ OS[self.server.p0f_os][0], # os_name
+ OS[self.server.p0f_os][1], # os_flavor
+ b'', # http_name
+ b'', # http_flavor
+ b'Ethernet or modem', # link_type
+ b'' # language
+ )
+
+ self.request.sendall(response)
+ self.request.close()
+
+def cleanup(SOCK):
+ if os.path.exists(SOCK):
+ try:
+ os.unlink(SOCK)
+ except OSError:
+ print("Could not unlink socket: " + SOCK)
+
+if __name__ == "__main__":
+ SOCK = '/tmp/p0f.sock'
+ p0f_status = 'ok'
+ p0f_os = 'linux'
+
+ os.umask(0000)
+
+ alen = len(sys.argv)
+ if alen > 1:
+ SOCK = sys.argv[1]
+ if alen >= 4:
+ p0f_os = sys.argv[2]
+ p0f_status = sys.argv[3]
+ elif alen >= 3:
+ p0f_os = sys.argv[2]
+
+ cleanup(SOCK)
+
+ server = socketserver.UnixStreamServer(SOCK, MyStreamHandler, bind_and_activate=False)
+ server.allow_reuse_address = True
+ server.p0f_status = p0f_status
+ server.p0f_os = p0f_os
+ server.server_bind()
+ server.server_activate()
+
+ dummy_killer.setup_killer(server)
+ dummy_killer.write_pid(PID)
+
+ try:
+ server.handle_request()
+ except socket.error:
+ print("Socket closed")
+
+ server.server_close()
+ cleanup(SOCK)
diff --git a/test/functional/util/dummy_ssl.py b/test/functional/util/dummy_ssl.py
new file mode 100755
index 0000000..44b1782
--- /dev/null
+++ b/test/functional/util/dummy_ssl.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+import os
+import socket
+import ssl
+import sys
+import time
+
+import dummy_killer
+import socketserver
+
+PORT = 14433
+HOST_NAME = '127.0.0.1'
+
+PID = "/tmp/dummy_ssl.pid"
+
+class SSLTCPHandler(socketserver.StreamRequestHandler):
+ def handle(self):
+ time.sleep(0.5)
+ data = self.request.recv(6000000)
+ while data:
+ print("{} wrote:".format(self.client_address[0]))
+ print(data)
+ time.sleep(0.1)
+ self.request.sendall(b'hello\n')
+ time.sleep(0.1)
+ data = self.request.recv(6000000)
+
+class SSL_TCP_Server(socketserver.ThreadingMixIn, socketserver.TCPServer):
+ def __init__(self,
+ server_address,
+ RequestHandlerClass,
+ certfile,
+ keyfile,
+ bind_and_activate=True):
+ self.allow_reuse_address = True
+ super().__init__(server_address, RequestHandlerClass, False)
+ self.timeout = 1
+ ctx = ssl.create_default_context()
+ ctx.load_cert_chain(certfile=certfile)
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+ self.socket = ctx.wrap_socket(self.socket, server_side=True)
+ if (bind_and_activate):
+ self.server_bind()
+ self.server_activate()
+
+ def run(self):
+ dummy_killer.write_pid(PID)
+ try:
+ self.serve_forever()
+ except KeyboardInterrupt:
+ print("Interrupt")
+ except socket.error as e:
+ print("Socket closed {}".format(e))
+ finally:
+ self.server_close()
+
+ def stop(self):
+ self.keep_running = False
+ self.server_close()
+
+if __name__ == '__main__':
+ server = SSL_TCP_Server((HOST_NAME, PORT), SSLTCPHandler, sys.argv[1], sys.argv[1])
+ dummy_killer.setup_killer(server, server.stop)
+ server.run()
diff --git a/test/functional/util/dummy_udp.py b/test/functional/util/dummy_udp.py
new file mode 100755
index 0000000..f05d6d6
--- /dev/null
+++ b/test/functional/util/dummy_udp.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+import socket
+import sys
+
+import dummy_killer
+
+UDP_IP = "127.0.0.1"
+PID = "/tmp/dummy_udp.pid"
+
+if __name__ == "__main__":
+ alen = len(sys.argv)
+ if alen > 1:
+ port = int(sys.argv[1])
+ else:
+ port = 5005
+ sock = socket.socket(socket.AF_INET, # Internet
+ socket.SOCK_DGRAM) # UDP
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ sock.bind((UDP_IP, port))
+ dummy_killer.write_pid(PID)
+
+ while True:
+ data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
+ print("received message:", data)
+ sock.sendto(data, addr)
diff --git a/test/functional/util/nn_unpack.lua b/test/functional/util/nn_unpack.lua
new file mode 100644
index 0000000..fee98d5
--- /dev/null
+++ b/test/functional/util/nn_unpack.lua
@@ -0,0 +1,16 @@
+local ucl = require "ucl"
+
+local function unhex(str)
+ return (str:gsub('..', function (cc)
+ return string.char(tonumber(cc, 16))
+ end))
+end
+
+local parser = ucl.parser()
+local ok, err = parser:parse_string(unhex(arg[1]), 'msgpack')
+if not ok then
+ io.stderr:write(err)
+ os.exit(1)
+end
+
+print(ucl.to_format(parser:get_object(), 'json-compact'))
diff --git a/test/functional/util/server.pem b/test/functional/util/server.pem
new file mode 100644
index 0000000..b5ec4c3
--- /dev/null
+++ b/test/functional/util/server.pem
@@ -0,0 +1,46 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA1nROqQC9tDLdnC2o1246g1T/oDq5Szbktof7TPHm/wDmWG47
+pwD22UkyEXthlq9kT9BZEgN6E2XF9UeA/7tZw7S4nbDUY/gWoGu7KlGLIIb5YXBW
+CzSHAwlJtW/Pg5Muk7vJldHOufBJXP99PjJVzZZuYvv9+JJfSWHEeuY+RysWnTX5
+saq4cwDG8fCKqKEwy8PKatr8eJFL1/cPnC/0SAB77mIEdYHV5hBDUoNTpPWQrB2s
+SgTqLa2pYTCWGUAvu4TiZKx00f6Z7soLa8RXddhEj6Qu5zLl+t5Llc8iSr7MgWTZ
+0PD/lChOUg7H9STM4KiQ/qtBvSyGB0Cz71aWdQIDAQABAoIBAQDVAuG+7IWBn6SP
+pKq4kTx9hsgCF1JFa1jyFU9/tieD0xj5bUTDNSn72cBprOvaCIzS2lgelGWFLuna
+IBP579XRlohp2WKdiaav4VIfTq+qt2atai+NIbeZRHh6R30Pa/ovs68fqS51cj7s
+qLl2NfkUI/+xQiuZ44nSEdJfYKMre8d/sF2zWg5sO5DaJWYsKdAqclRR4q8ZrEQY
+6AZ8hsAlsyczpvU1A7Y/XWnyg7jr2towxeAbey13phFdukrZnRq2pRaprvwA1KqC
+AtO0jx7L3MP3eamwQaNyaBD/PE+wnGpbPr36oU1ewhUTSZUlx3aYzaUDPrqMWwMk
+7cb/gnsBAoGBAO9OQCnVE4D8r3DCRtmFfgqasWIakEs591+lzVSnHrLDsiaPk/5Q
+Rv6FAyxU8T78SJNaYgCH3XW0NHeyyNi3Z1TQ9HDZLuiAhrVptIWFSvjCGIE4ijCG
+3JQLkKPs2d/5/fXcZbgNhcm0lLYtP3SOsVB2p75dER8BjVVSsl14UfUJAoGBAOVq
+PNSI55/jgwaA/zvX+fsawY6V20C42KpDq7LywAk/JCf5+pHZX4szClj3CS3shdP3
+B7WRqNHfLHTF7ID+JYpkahJI/Zq6a5wR4i/zXLIQCtSIaR0zZcDBu9GZvcwQlnLR
+wvkkOlHXlmkmq94FV/i/b2rmbK7RTuc2TGk5+j0NAoGAGmZpkbPeCPbXa/si0dB2
+TTkvpIEFtibY8YZbFqGxM0t/ld11GDNHAcEuzm84hhhS8V6hPSm/9sJAn4vruGzT
+S3oZ3XE4SZIUSmM09R31XWgcR/Uy2ZOnNfXoqQzyJFFyAPOljR6AyfXQCiEHxRYQ
+3a2ZZ9jgkKkdLHKJFuK1N/ECgYAcdoDTkaTDJpQEA48nGpWuPNSU3yzTq9tdzIWJ
+7yo6O3Y963rWC5UaDRwUi5m88+JquPRg55B9cWXvmvrLyjxYHjs2x42HW+er9mAM
+uPHgObNOSRpZgB34u1CVIbD1l31DA5lgFcmSi9/ibeTW5+zRNNca+Tm0us1CTG9Q
+gtv0JQKBgQCdqEhX8h10N4qQQ/E0ejX5VSR6JE/EfbFO2gaL62YOhSRKaddZvG7R
+npQeNY2yS1TIoU/LT5pC5ytQ6hwjWFLo98MHY6B0rUWbUl8Uzsz+mcKbVSZnEIWj
+9CGVOzPlLELRJdTqVUNPdJu7NjzNKoizZjAU1SynHFu7jy98e730QA==
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIDGDCCAgACCQCoEpIa1RdPlzANBgkqhkiG9w0BAQsFADBOMQswCQYDVQQGEwJh
+YTELMAkGA1UECAwCYWExCzAJBgNVBAcMAmFhMQswCQYDVQQKDAJhYTELMAkGA1UE
+CwwCYWExCzAJBgNVBAMMAmFhMB4XDTE5MDUyOTIxMTMzNFoXDTE5MDYyODIxMTMz
+NFowTjELMAkGA1UEBhMCYWExCzAJBgNVBAgMAmFhMQswCQYDVQQHDAJhYTELMAkG
+A1UECgwCYWExCzAJBgNVBAsMAmFhMQswCQYDVQQDDAJhYTCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBANZ0TqkAvbQy3ZwtqNduOoNU/6A6uUs25LaH+0zx
+5v8A5lhuO6cA9tlJMhF7YZavZE/QWRIDehNlxfVHgP+7WcO0uJ2w1GP4FqBruypR
+iyCG+WFwVgs0hwMJSbVvz4OTLpO7yZXRzrnwSVz/fT4yVc2WbmL7/fiSX0lhxHrm
+PkcrFp01+bGquHMAxvHwiqihMMvDymra/HiRS9f3D5wv9EgAe+5iBHWB1eYQQ1KD
+U6T1kKwdrEoE6i2tqWEwlhlAL7uE4mSsdNH+me7KC2vEV3XYRI+kLucy5freS5XP
+Ikq+zIFk2dDw/5QoTlIOx/UkzOCokP6rQb0shgdAs+9WlnUCAwEAATANBgkqhkiG
+9w0BAQsFAAOCAQEAF0TDCTa239+aQikOsqPjhZL9+6W/2z84uzskpTs449P9sJ9a
+l6NcS0RnMmrhkUS/XdesFUOqabH2hMrj7rr8ezH6Y9GMT84ncvUaZAhFy1k5wwyB
+E2hUJ6rv66VNWQ4I4Uxc9xoiqkAk4waFXhFiQZfVJ91RwLzXMJPsgAV9Wspg/jk8
+dNP1MW0rGrUx8CodkT370chI7DYHFTVudeb+MLUIF1RHQ4p/ATvzWIex3sIpkllv
+BjUz5pvTiSy7PZGIHZdZh5n4JfFjrUJHGlWRulPhX5sw8XPn1bMRDr/9EKJDcsWS
+rx5ZUoYjrLdTA2XJMcQ2AYrIZLyyD5+ihN841g==
+-----END CERTIFICATE-----
diff --git a/test/lua/compat_env.lua b/test/lua/compat_env.lua
new file mode 100644
index 0000000..2ecd4b6
--- /dev/null
+++ b/test/lua/compat_env.lua
@@ -0,0 +1,391 @@
+--[[
+
+ compat_env v$(_VERSION) - Lua 5.1/5.2 environment compatibility functions
+
+SYNOPSIS
+
+ -- Get load/loadfile compatibility functions only if using 5.1.
+ local CL = pcall(load, '') and _G or require 'compat_env'
+ local load = CL.load
+ local loadfile = CL.loadfile
+
+ -- The following now works in both Lua 5.1 and 5.2:
+ assert(load('return 2*pi', nil, 't', {pi=math.pi}))()
+ assert(loadfile('ex.lua', 't', {print=print}))()
+
+ -- Get getfenv/setfenv compatibility functions only if using 5.2.
+ local getfenv = _G.getfenv or require 'compat_env'.getfenv
+ local setfenv = _G.setfenv or require 'compat_env'.setfenv
+ local function f() return x end
+ setfenv(f, {x=2})
+ print(x, getfenv(f).x) --> 2, 2
+
+DESCRIPTION
+
+ This module provides Lua 5.1/5.2 environment related compatibility functions.
+ This includes implementations of Lua 5.2 style `load` and `loadfile`
+ for use in Lua 5.1. It also includes Lua 5.1 style `getfenv` and `setfenv`
+ for use in Lua 5.2.
+
+API
+
+ local CL = require 'compat_env'
+
+ CL.load (ld [, source [, mode [, env] ] ]) --> f [, err]
+
+ This behaves the same as the Lua 5.2 `load` in both
+ Lua 5.1 and 5.2.
+ http://www.lua.org/manual/5.2/manual.html#pdf-load
+
+ CL.loadfile ([filename [, mode [, env] ] ]) --> f [, err]
+
+ This behaves the same as the Lua 5.2 `loadfile` in both
+ Lua 5.1 and 5.2.
+ http://www.lua.org/manual/5.2/manual.html#pdf-loadfile
+
+ CL.getfenv ([f]) --> t
+
+ This is identical to the Lua 5.1 `getfenv` in Lua 5.1.
+ This behaves similar to the Lua 5.1 `getfenv` in Lua 5.2.
+ When a global environment is to be returned, or when `f` is a
+ C function, this returns `_G` since Lua 5.2 doesn't have
+ (thread) global and C function environments. This will also
+ return `_G` if the Lua function `f` lacks an `_ENV`
+ upvalue, but it will raise an error if uncertain due to lack of
+ debug info. It is not normally considered good design to use
+ this function; when possible, use `load` or `loadfile` instead.
+ http://www.lua.org/manual/5.1/manual.html#pdf-getfenv
+
+ CL.setfenv (f, t)
+
+ This is identical to the Lua 5.1 `setfenv` in Lua 5.1.
+ This behaves similar to the Lua 5.1 `setfenv` in Lua 5.2.
+ This will do nothing if `f` is a Lua function that
+ lacks an `_ENV` upvalue, but it will raise an error if uncertain
+ due to lack of debug info. See also Design Notes below.
+ It is not normally considered good design to use
+ this function; when possible, use `load` or `loadfile` instead.
+ http://www.lua.org/manual/5.1/manual.html#pdf-setfenv
+
+DESIGN NOTES
+
+ This module intends to provide robust and fairly complete reimplementations
+ of the environment related Lua 5.1 and Lua 5.2 functions.
+ No effort is made, however, to simulate rare or difficult to simulate features,
+ such as thread environments, although this is liable to change in the future.
+ Such 5.1 capabilities are discouraged and ideally
+ removed from 5.1 code, thereby allowing your code to work in both 5.1 and 5.2.
+
+ In Lua 5.2, a `setfenv(f, {})`, where `f` lacks any upvalues, will be silently
+ ignored since there is no `_ENV` in this function to write to, and the
+ environment will have no effect inside the function anyway. However,
+ this does mean that `getfenv(setfenv(f, t))` does not necessarily equal `t`,
+ which is incompatible with 5.1 code (a possible workaround would be [1]).
+ If `setfenv(f, {})` has an upvalue but no debug info, then this will raise
+ an error to prevent inadvertently executing potentially untrusted code in the
+ global environment.
+
+ It is not normally considered good design to use `setfenv` and `getfenv`
+ (one reason they were removed in 5.2). When possible, consider replacing
+ these with `load` or `loadfile`, which are more restrictive and have native
+ implementations in 5.2.
+
+ This module might be merged into a more general Lua 5.1/5.2 compatibility
+ library (e.g. a full reimplementation of Lua 5.2 `_G`). However,
+ `load/loadfile/getfenv/setfenv` perhaps are among the more cumbersome
+ functions not to have.
+
+INSTALLATION
+
+ Download compat_env.lua:
+
+ wget https://raw.github.com/gist/1654007/compat_env.lua
+
+ Copy compat_env.lua into your LUA_PATH.
+
+ Alternately, unpack, test, and install into LuaRocks:
+
+ wget https://raw.github.com/gist/1422205/sourceunpack.lua
+ lua sourceunpack.lua compat_env.lua
+ (cd out && luarocks make)
+
+Related work
+
+ http://lua-users.org/wiki/LuaVersionCompatibility
+ https://github.com/stevedonovan/Penlight/blob/master/lua/pl/utils.lua
+ - penlight implementations of getfenv/setfenv
+ http://lua-users.org/lists/lua-l/2010-06/msg00313.html
+ - initial getfenv/setfenv implementation
+
+References
+
+ [1] http://lua-users.org/lists/lua-l/2010-06/msg00315.html
+
+Copyright
+
+(c) 2012 David Manura. Licensed under the same terms as Lua 5.1/5.2 (MIT license).
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+--]]---------------------------------------------------------------------
+
+local M = {_TYPE='module', _NAME='compat_env', _VERSION='0.2.20120124'}
+
+local function check_chunk_type(s, mode)
+ local nmode = mode or 'bt'
+ local is_binary = s and #s > 0 and s:byte(1) == 27
+ if is_binary and not nmode:match'b' then
+ return nil, ("attempt to load a binary chunk (mode is '%s')"):format(mode)
+ elseif not is_binary and not nmode:match't' then
+ return nil, ("attempt to load a text chunk (mode is '%s')"):format(mode)
+ end
+ return true
+end
+
+local IS_52_LOAD = pcall(load, '')
+if IS_52_LOAD then
+ M.load = _G.load
+ M.loadfile = _G.loadfile
+else
+ -- 5.2 style `load` implemented in 5.1
+ function M.load(ld, source, mode, env)
+ local f
+ if type(ld) == 'string' then
+ local s = ld
+ local ok, err = check_chunk_type(s, mode); if not ok then return ok, err end
+ local err; f, err = loadstring(s, source); if not f then return f, err end
+ elseif type(ld) == 'function' then
+ local ld2 = ld
+ if (mode or 'bt') ~= 'bt' then
+ local first = ld()
+ local ok, err = check_chunk_type(first, mode); if not ok then return ok, err end
+ ld2 = function()
+ if first then
+ local chunk=first; first=nil; return chunk
+ else return ld() end
+ end
+ end
+ local err; f, err = load(ld2, source); if not f then return f, err end
+ else
+ error(("bad argument #1 to 'load' (function expected, got %s)"):format(type(ld)), 2)
+ end
+ if env then setfenv(f, env) end
+ return f
+ end
+
+ -- 5.2 style `loadfile` implemented in 5.1
+ function M.loadfile(filename, mode, env)
+ if (mode or 'bt') ~= 'bt' then
+ local ioerr
+ local fh, err = io.open(filename, 'rb'); if not fh then return fh, err end
+ local function ld() local chunk; chunk,ioerr = fh:read(4096); return chunk end
+ local f, err = M.load(ld, filename and '@'..filename, mode, env)
+ fh:close()
+ if not f then return f, err end
+ if ioerr then return nil, ioerr end
+ return f
+ else
+ local f, err = loadfile(filename); if not f then return f, err end
+ if env then setfenv(f, env) end
+ return f
+ end
+ end
+end
+
+if _G.setfenv then -- Lua 5.1
+ M.setfenv = _G.setfenv
+ M.getfenv = _G.getfenv
+else -- >= Lua 5.2
+ -- helper function for `getfenv`/`setfenv`
+ local function envlookup(f)
+ local name, val
+ local up = 0
+ local unknown
+ repeat
+ up=up+1; name, val = debug.getupvalue(f, up)
+ if name == '' then unknown = true end
+ until name == '_ENV' or name == nil
+ if name ~= '_ENV' then
+ up = nil
+ if unknown then error("upvalues not readable in Lua 5.2 when debug info missing", 3) end
+ end
+ return (name == '_ENV') and up, val, unknown
+ end
+
+ -- helper function for `getfenv`/`setfenv`
+ local function envhelper(f, name)
+ if type(f) == 'number' then
+ if f < 0 then
+ error(("bad argument #1 to '%s' (level must be non-negative)"):format(name), 3)
+ elseif f < 1 then
+ error("thread environments unsupported in Lua 5.2", 3) --[*]
+ end
+ f = debug.getinfo(f+2, 'f').func
+ elseif type(f) ~= 'function' then
+ error(("bad argument #1 to '%s' (number expected, got %s)"):format(type(name, f)), 2)
+ end
+ return f
+ end
+ -- [*] might simulate with table keyed by coroutine.running()
+
+ -- 5.1 style `setfenv` implemented in 5.2
+ function M.setfenv(f, t)
+ local f = envhelper(f, 'setfenv')
+ local up, val, unknown = envlookup(f)
+ if up then
+ debug.upvaluejoin(f, up, function() return up end, 1) -- unique upvalue [*]
+ debug.setupvalue(f, up, t)
+ else
+ local what = debug.getinfo(f, 'S').what
+ if what ~= 'Lua' and what ~= 'main' then -- not Lua func
+ error("'setfenv' cannot change environment of given object", 2)
+ end -- else ignore no _ENV upvalue (warning: incompatible with 5.1)
+ end
+ -- added in https://gist.github.com/2255007
+ return f
+ end
+ -- [*] http://lua-users.org/lists/lua-l/2010-06/msg00313.html
+
+ -- 5.1 style `getfenv` implemented in 5.2
+ function M.getfenv(f)
+ if f == 0 or f == nil then return _G end -- simulated behavior
+ local f = envhelper(f, 'setfenv')
+ local up, val = envlookup(f)
+ if not up then return _G end -- simulated behavior [**]
+ return val
+ end
+ -- [**] possible reasons: no _ENV upvalue, C function
+end
+
+
+return M
+
+--[[ FILE rockspec.in
+
+package = 'compat_env'
+version = '$(_VERSION)-1'
+source = {
+ url = 'https://raw.github.com/gist/1654007/$(GITID)/compat_env.lua',
+ --url = 'https://raw.github.com/gist/1654007/compat_env.lua', -- latest raw
+ --url = 'https://gist.github.com/gists/1654007/download',
+ md5 = '$(MD5)'
+}
+description = {
+ summary = 'Lua 5.1/5.2 environment compatibility functions',
+ detailed = [=[
+ Provides Lua 5.1/5.2 environment related compatibility functions.
+ This includes implementations of Lua 5.2 style `load` and `loadfile`
+ for use in Lua 5.1. It also includes Lua 5.1 style `getfenv` and `setfenv`
+ for use in Lua 5.2.
+ ]=],
+ license = 'MIT/X11',
+ homepage = 'https://gist.github.com/1654007',
+ maintainer = 'David Manura'
+}
+dependencies = {} -- Lua 5.1 or 5.2
+build = {
+ type = 'builtin',
+ modules = {
+ ['compat_env'] = 'compat_env.lua'
+ }
+}
+
+--]]---------------------------------------------------------------------
+
+--[[ FILE test.lua
+
+-- test.lua - test suite for compat_env module.
+
+local CL = require 'compat_env'
+local load = CL.load
+local loadfile = CL.loadfile
+local setfenv = CL.setfenv
+local getfenv = CL.getfenv
+
+local function checkeq(a, b, e)
+ if a ~= b then error(
+ 'not equal ['..tostring(a)..'] ['..tostring(b)..'] ['..tostring(e)..']')
+ end
+end
+local function checkerr(pat, ok, err)
+ assert(not ok, 'checkerr')
+ assert(type(err) == 'string' and err:match(pat), err)
+end
+
+-- test `load`
+checkeq(load('return 2')(), 2)
+checkerr('expected near', load'return 2 2')
+checkerr('text chunk', load('return 2', nil, 'b'))
+checkerr('text chunk', load('', nil, 'b'))
+checkerr('binary chunk', load('\027', nil, 't'))
+checkeq(load('return 2*x',nil,'bt',{x=5})(), 10)
+checkeq(debug.getinfo(load('')).source, '')
+checkeq(debug.getinfo(load('', 'foo')).source, 'foo')
+
+-- test `loadfile`
+local fh = assert(io.open('tmp.lua', 'wb'))
+fh:write('return (...) or x')
+fh:close()
+checkeq(loadfile('tmp.lua')(2), 2)
+checkeq(loadfile('tmp.lua', 't')(2), 2)
+checkerr('text chunk', loadfile('tmp.lua', 'b'))
+checkeq(loadfile('tmp.lua', nil, {x=3})(), 3)
+checkeq(debug.getinfo(loadfile('tmp.lua')).source, '@tmp.lua')
+checkeq(debug.getinfo(loadfile('tmp.lua', 't', {})).source, '@tmp.lua')
+os.remove'tmp.lua'
+
+-- test `setfenv`/`getfenv`
+x = 5
+local a,b=true; local function f(c) if a then return x,b,c end end
+setfenv(f, {x=3})
+checkeq(f(), 3)
+checkeq(getfenv(f).x, 3)
+checkerr('cannot change', pcall(setfenv, string.len, {})) -- C function
+checkeq(getfenv(string.len), _G) -- C function
+local function g()
+ setfenv(1, {x=4})
+ checkeq(getfenv(1).x, 4)
+ return x
+end
+checkeq(g(), 4) -- numeric level
+if _G._VERSION ~= 'Lua 5.1' then
+ checkerr('unsupported', pcall(setfenv, 0, {}))
+end
+checkeq(getfenv(0), _G)
+checkeq(getfenv(), _G) -- no arg
+checkeq(x, 5) -- main unaltered
+setfenv(function()end, {}) -- no upvalues, ignore
+checkeq(getfenv(function()end), _G) -- no upvaluse
+if _G._VERSION ~= 'Lua 5.1' then
+ checkeq(getfenv(setfenv(function()end, {})), _G) -- warning: incompatible with 5.1
+end
+x = nil
+
+print 'OK'
+
+--]]---------------------------------------------------------------------
+
+--[[ FILE CHANGES.txt
+0.2.20120124
+ Renamed module to compat_env (from compat_load)
+ Add getfenv/setfenv functions
+
+0.1.20120121
+ Initial public release
+--]]
diff --git a/test/lua/pcall_test.lua b/test/lua/pcall_test.lua
new file mode 100644
index 0000000..26f3ae4
--- /dev/null
+++ b/test/lua/pcall_test.lua
@@ -0,0 +1,45 @@
+--[[ https://en.wikipedia.org/wiki/Normal_distribution ]]
+
+-- The Box–Muller method
+local function gaussian(mean, variance)
+ local U = math.random()
+ local V = math.random()
+ return math.sqrt(-2.0 * variance * math.log(U)) *
+ math.cos(2.0 * math.pi * V) + mean
+end
+
+local function mean(t)
+ local sum = 0
+ local count = #t
+ for i = 1, count do
+ sum = sum + t[i]
+ end
+ return sum / count
+end
+
+local function std(t, mean)
+ local squares = 0.0
+ for i = 1, #t do
+ local deviation = math.abs(mean - t[i])
+ squares = squares + deviation * deviation
+ end
+ local variance = squares / #t
+ return math.sqrt(variance)
+end
+
+local function do_the_call()
+ local t = {}
+ local mu = 34.0
+ local sigma = 10.0
+
+ for i = 1, 5 do
+ table.insert(t, gaussian(mu, sigma))
+ end
+
+ return string.format("Got mean: %1.5f, mu: %1.5f\nstd deviance:%1.5f, expected: %1.5f",
+ mean(t), mu,
+ std(t, mu), math.sqrt(sigma))
+end
+
+math.randomseed(os.time())
+return do_the_call
diff --git a/test/lua/rspamd_assertions.lua b/test/lua/rspamd_assertions.lua
new file mode 100644
index 0000000..d0590fd
--- /dev/null
+++ b/test/lua/rspamd_assertions.lua
@@ -0,0 +1,138 @@
+local logger = require "rspamd_logger"
+local telescope = require "telescope"
+local util = require 'lua_util'
+
+local function rspamd_assert_equals(tbl)
+ return tbl.expect == tbl.actual
+end
+
+local function rspamd_assert_equals_msg(_, tbl)
+ return logger.slog(
+ "Failed asserting that \n (actual) : %1 \n equals to\n (expected) : %2",
+ tbl.actual, tbl.expect
+ )
+end
+
+local function rspamd_assert_table_equals(tbl)
+ return util.table_cmp(tbl.expect, tbl.actual)
+end
+
+local function rspamd_assert_table_equals_sorted(tbl)
+ local expect = util.deepcopy(tbl.expect)
+ local actual = util.deepcopy(tbl.actual)
+ util.deepsort(expect)
+ util.deepsort(actual)
+ return util.table_cmp(expect, actual)
+end
+
+local function table_keys_sorted(t)
+ local keys = {}
+
+ for k,_ in pairs(t) do
+ table.insert(keys, k)
+ end
+ table.sort(keys)
+ return keys;
+end
+
+local function format_line(level, key, v_expect, v_actual)
+ local prefix
+ if v_expect == v_actual then
+ prefix = string.rep(' ', level * 2 + 1)
+ return logger.slog("%s[%s] = %s", prefix, key, v_expect)
+ else
+ prefix = string.rep(' ', level * 2)
+ local ret = {}
+ if v_expect then
+ ret[#ret + 1] = logger.slog("-%s[%s] = %s: %s", prefix, key,
+ type(v_expect), v_expect)
+ end
+ if v_actual then
+ ret[#ret + 1] = logger.slog("+%s[%s] = %s: %s", prefix,
+ (key), type(v_actual), (v_actual))
+ end
+ return table.concat(ret, "\n")
+ end
+end
+
+local function format_table_begin(level, key)
+ local prefix = string.rep(' ', level * 2 + 1)
+ return string.format("%s[%s] = {", prefix, tostring(key))
+end
+
+local function format_table_end(level)
+ local prefix = string.rep(' ', level * 2 + 1)
+ return string.format("%s}", prefix)
+end
+
+local function rspamd_assert_table_diff_msg(_, tbl)
+ local avoid_loops = {}
+ local msg = rspamd_assert_equals_msg(_, tbl)
+
+ local diff = {}
+ local function recurse(expect, actual, level)
+ if avoid_loops[actual] then
+ return
+ end
+ avoid_loops[actual] = true
+
+ local keys_expect = table_keys_sorted(expect)
+ local keys_actual = table_keys_sorted(actual)
+
+ local i_k_expect, i_v_expect = next(keys_expect)
+ local i_k_actual, i_v_actual = next(keys_actual)
+
+ while i_k_expect and i_k_actual do
+ local v_expect = expect[i_v_expect]
+ local v_actual = actual[i_v_actual]
+
+ if i_v_expect == i_v_actual then
+ -- table keys are the same: compare values
+ if type(v_expect) == 'table' and type(v_actual) == 'table' then
+ if util.table_cmp(v_expect, v_actual) then
+ -- we use the same value for 'actual' and 'expect' as soon as they're equal and don't bother us
+ diff[#diff + 1] = format_line(level, i_v_expect, v_expect, v_expect)
+ else
+ diff[#diff + 1] = format_table_begin(level, i_v_expect)
+ recurse(v_expect, v_actual, level + 1)
+ diff[#diff + 1] = format_table_end(level)
+ end
+ else
+ diff[#diff + 1] = format_line(level, i_v_expect, v_expect, v_actual)
+ end
+
+ i_k_expect, i_v_expect = next(keys_expect, i_k_expect)
+ i_k_actual, i_v_actual = next(keys_actual, i_k_actual)
+ elseif tostring(v_actual) > tostring(v_expect) then
+ diff[#diff + 1] = format_line(level, i_v_expect, v_expect, nil)
+ i_k_expect, i_v_expect = next(keys_expect, i_k_expect)
+ else
+ diff[#diff + 1] = format_line(level, i_v_actual, nil, v_actual)
+ i_k_actual, i_v_actual = next(keys_actual, i_k_actual)
+ end
+
+ end
+
+ while i_k_expect do
+ local v_expect = expect[i_v_expect]
+ diff[#diff + 1] = format_line(level, i_v_expect, v_expect, nil)
+ i_k_expect, i_v_expect = next(keys_expect, i_k_expect)
+ end
+
+ while i_k_actual do
+ local v_actual = actual[i_v_actual]
+ diff[#diff + 1] = format_line(level, i_v_actual, nil, v_actual)
+ i_k_actual, i_v_actual = next(keys_actual, i_k_actual)
+ end
+ end
+ recurse(tbl.expect, tbl.actual, 0)
+
+ return string.format("%s\n===== diff (-expect, +actual) ======\n%s", msg, table.concat(diff, "\n"))
+end
+
+telescope.make_assertion("rspamd_eq", rspamd_assert_equals_msg, rspamd_assert_equals)
+-- telescope.make_assertion("rspamd_table_eq", rspamd_assert_equals_msg, rspamd_assert_table_equals)
+telescope.make_assertion("rspamd_table_eq", rspamd_assert_table_diff_msg, rspamd_assert_table_equals)
+telescope.make_assertion("rspamd_table_eq_sorted", rspamd_assert_table_diff_msg,
+ rspamd_assert_table_equals_sorted)
+
diff --git a/test/lua/rspamd_test_helper.lua b/test/lua/rspamd_test_helper.lua
new file mode 100644
index 0000000..5373448
--- /dev/null
+++ b/test/lua/rspamd_test_helper.lua
@@ -0,0 +1,45 @@
+local ffi = require "ffi"
+local cfg = rspamd_config
+
+ffi.cdef[[
+void rspamd_url_init (const char *tld_file);
+]]
+local exports = {}
+
+local function default_tld_file()
+ local test_dir = string.gsub(debug.getinfo(1).source, "^@(.+/)[^/]+$", "%1")
+ return string.format('%s/unit/%s', test_dir, "test_tld.dat")
+end
+
+function exports.init_url_parser(file)
+ ffi.C.rspamd_url_init(file or default_tld_file())
+end
+
+function exports.default_config()
+ local tld_file = default_tld_file()
+
+ local config = {
+ options = {
+ filters = {'spf', 'dkim', 'regexp'},
+ url_tld = tld_file,
+ dns = {
+ nameserver = {'8.8.8.8'}
+ },
+ },
+ logging = {
+ type = 'console',
+ level = 'debug'
+ },
+ metric = {
+ name = 'default',
+ actions = {
+ reject = 100500,
+ },
+ unknown_weight = 1
+ }
+ }
+
+ return config
+end
+
+return exports \ No newline at end of file
diff --git a/test/lua/telescope.lua b/test/lua/telescope.lua
new file mode 100644
index 0000000..e2d9e65
--- /dev/null
+++ b/test/lua/telescope.lua
@@ -0,0 +1,621 @@
+--[[
+The MIT License
+
+Copyright (c) 2009-2012 [Norman Clarke](mailto:norman@njclarke.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+]]--
+
+--- Telescope is a test library for Lua that allows for flexible, declarative
+-- tests. The documentation produced here is intended largely for developers
+-- working on Telescope. For information on using Telescope, please visit the
+-- project homepage at: <a href="http://github.com/norman/telescope">http://github.com/norman/telescope#readme</a>.
+-- @release 0.6
+-- @class module
+-- @module 'telescope'
+local _M = {}
+
+local compat_env = require 'compat_env'
+
+local getfenv = _G.getfenv or compat_env.getfenv
+local setfenv = _G.setfenv or compat_env.setfenv
+
+
+local _VERSION = "0.6.0"
+
+--- The status codes that can be returned by an invoked test. These should not be overridden.
+-- @name status_codes
+-- @class table
+-- @field err - This is returned when an invoked test results in an error
+-- rather than a passed or failed assertion.
+-- @field fail - This is returned when an invoked test contains one or more failing assertions.
+-- @field pass - This is returned when all of a test's assertions pass.
+-- @field pending - This is returned when a test does not have a corresponding function.
+-- @field unassertive - This is returned when an invoked test does not produce
+-- errors, but does not contain any assertions.
+local status_codes = {
+ err = 2,
+ fail = 4,
+ pass = 8,
+ pending = 16,
+ unassertive = 32
+}
+
+--- Labels used to show the various <tt>status_codes</tt> as a single character.
+-- These can be overridden if you wish.
+-- @name status_labels
+-- @class table
+-- @see status_codes
+-- @field status_codes.err 'E'
+-- @field status_codes.fail 'F'
+-- @field status_codes.pass 'P'
+-- @field status_codes.pending '?'
+-- @field status_codes.unassertive 'U'
+
+local status_labels = {
+ [status_codes.err] = 'E',
+ [status_codes.fail] = 'F',
+ [status_codes.pass] = 'P',
+ [status_codes.pending] = '?',
+ [status_codes.unassertive] = 'U'
+}
+
+--- The default names for context blocks. It defaults to "context", "spec" and
+-- "describe."
+-- @name context_aliases
+-- @class table
+local context_aliases = {"context", "describe", "spec"}
+--- The default names for test blocks. It defaults to "test," "it", "expect",
+-- "they" and "should."
+-- @name test_aliases
+-- @class table
+local test_aliases = {"test", "it", "expect", "should", "they"}
+
+--- The default names for "before" blocks. It defaults to "before" and "setup."
+-- The function in the before block will be run before each sibling test function
+-- or context.
+-- @name before_aliases
+-- @class table
+local before_aliases = {"before", "setup"}
+
+--- The default names for "after" blocks. It defaults to "after" and "teardown."
+-- The function in the after block will be run after each sibling test function
+-- or context.
+-- @name after_aliases
+-- @class table
+local after_aliases = {"after", "teardown"}
+
+-- Prefix to place before all assertion messages. Used by make_assertion().
+local assertion_message_prefix = "Assert failed: expected "
+
+--- The default assertions.
+-- These are the assertions built into telescope. You can override them or
+-- create your own custom assertions using <tt>make_assertion</tt>.
+-- <ul>
+-- <tt><li>assert_blank(a)</tt> - true if a is nil, or the empty string</li>
+-- <tt><li>assert_empty(a)</tt> - true if a is an empty table</li>
+-- <tt><li>assert_equal(a, b)</tt> - true if a == b</li>
+-- <tt><li>assert_error(f)</tt> - true if function f produces an error</li>
+-- <tt><li>assert_false(a)</tt> - true if a is false</li>
+-- <tt><li>assert_greater_than(a, b)</tt> - true if a > b</li>
+-- <tt><li>assert_gte(a, b)</tt> - true if a >= b</li>
+-- <tt><li>assert_less_than(a, b)</tt> - true if a < b</li>
+-- <tt><li>assert_lte(a, b)</tt> - true if a <= b</li>
+-- <tt><li>assert_match(a, b)</tt> - true if b is a string that matches pattern a</li>
+-- <tt><li>assert_nil(a)</tt> - true if a is nil</li>
+-- <tt><li>assert_true(a)</tt> - true if a is true</li>
+-- <tt><li>assert_type(a, b)</tt> - true if a is of type b</li>
+-- <tt><li>assert_not_blank(a)</tt> - true if a is not nil and a is not the empty string</li>
+-- <tt><li>assert_not_empty(a)</tt> - true if a is a table, and a is not empty</li>
+-- <tt><li>assert_not_equal(a, b)</tt> - true if a ~= b</li>
+-- <tt><li>assert_not_error(f)</tt> - true if function f does not produce an error</li>
+-- <tt><li>assert_not_false(a)</tt> - true if a is not false</li>
+-- <tt><li>assert_not_greater_than(a, b)</tt> - true if not (a > b)</li>
+-- <tt><li>assert_not_gte(a, b)</tt> - true if not (a >= b)</li>
+-- <tt><li>assert_not_less_than(a, b)</tt> - true if not (a < b)</li>
+-- <tt><li>assert_not_lte(a, b)</tt> - true if not (a <= b)</li>
+-- <tt><li>assert_not_match(a, b)</tt> - true if the string b does not match the pattern a</li>
+-- <tt><li>assert_not_nil(a)</tt> - true if a is not nil</li>
+-- <tt><li>assert_not_true(a)</tt> - true if a is not true</li>
+-- <tt><li>assert_not_type(a, b)</tt> - true if a is not of type b</li>
+-- </ul>
+-- @see make_assertion
+-- @name assertions
+-- @class table
+local assertions = {}
+
+--- Create a custom assertion.
+-- This creates an assertion along with a corresponding negative assertion. It
+-- is used internally by telescope to create the default assertions.
+-- @param name The base name of the assertion.
+-- <p>
+-- The name will be used as the basis of the positive and negative assertions;
+-- i.e., the name <tt>equal</tt> would be used to create the assertions
+-- <tt>assert_equal</tt> and <tt>assert_not_equal</tt>.
+-- </p>
+-- @param message The base message that will be shown.
+-- <p>
+-- The assertion message is what is shown when the assertion fails. It will be
+-- prefixed with the string in <tt>telescope.assertion_message_prefix</tt>.
+-- The variables passed to <tt>telescope.make_assertion</tt> are interpolated
+-- in the message string using <tt>string.format</tt>. When creating the
+-- inverse assertion, the message is reused, with <tt>" to be "</tt> replaced
+-- by <tt>" not to be "</tt>. Hence a recommended format is something like:
+-- <tt>"%s to be similar to %s"</tt>.
+-- </p>
+-- @param func The assertion function itself.
+-- <p>
+-- The assertion function can have any number of arguments.
+-- </p>
+-- @usage <tt>make_assertion("equal", "%s to be equal to %s", function(a, b)
+-- return a == b end)</tt>
+-- @function make_assertion
+local function make_assertion(name, message, func)
+ local num_vars = 0
+ -- if the last vararg ends up nil, we'll need to pad the table with nils so
+ -- that string.format gets the number of args it expects
+ local format_message
+ if type(message) == "function" then
+ format_message = message
+ else
+ for _, _ in message:gmatch("%%s") do num_vars = num_vars + 1 end
+ format_message = function(message, ...)
+ local a = {}
+ local args = {...}
+ local nargs = select('#', ...)
+ if nargs > num_vars then
+ local userErrorMessage = args[num_vars+1]
+ if type(userErrorMessage) == "string" then
+ return(assertion_message_prefix .. userErrorMessage)
+ else
+ error(string.format('assert_%s expected %d arguments but got %d', name, num_vars, #args))
+ end
+ end
+ for i = 1, nargs do a[i] = tostring(args[i]) end
+ for i = nargs+1, num_vars do a[i] = 'nil' end
+ return (assertion_message_prefix .. message):format(unpack(a))
+ end
+ end
+
+ assertions["assert_" .. name] = function(...)
+ if assertion_callback then assertion_callback(...) end
+ if not func(...) then
+ error({format_message(message, ...), debug.traceback()})
+ end
+ end
+end
+
+--- (local) Return a table with table t's values as keys and keys as values.
+-- @param t The table.
+local function invert_table(t)
+ local t2 = {}
+ for k, v in pairs(t) do t2[v] = k end
+ return t2
+end
+
+-- (local) Truncate a string "s" to length "len", optionally followed by the
+-- string given in "after" if truncated; for example, truncate_string("hello
+-- world", 3, "...")
+-- @param s The string to truncate.
+-- @param len The desired length.
+-- @param after A string to append to s, if it is truncated.
+local function truncate_string(s, len, after)
+ if #s <= len then
+ return s
+ else
+ local s = s:sub(1, len):gsub("%s*$", '')
+ if after then return s .. after else return s end
+ end
+end
+
+--- (local) Filter a table's values by function. This function iterates over a
+-- table , returning only the table entries that, when passed into function f,
+-- yield a truthy value.
+-- @param t The table over which to iterate.
+-- @param f The filter function.
+local function filter(t, f)
+ local a, b
+ return function()
+ repeat a, b = next(t, a)
+ if not b then return end
+ if f(a, b) then return a, b end
+ until not b
+ end
+end
+
+--- (local) Finds the value in the contexts table indexed with i, and returns a table
+-- of i's ancestor contexts.
+-- @param i The index in the <tt>contexts</tt> table to get ancestors for.
+-- @param contexts The table in which to find the ancestors.
+local function ancestors(i, contexts)
+ if i == 0 then return end
+ local a = {}
+ local function func(j)
+ if contexts[j].parent == 0 then return nil end
+ table.insert(a, contexts[j].parent)
+ func(contexts[j].parent)
+ end
+ func(i)
+ return a
+end
+
+make_assertion("blank", "'%s' to be blank", function(a) return a == '' or a == nil end)
+make_assertion("empty", "'%s' to be an empty table", function(a) return not next(a) end)
+make_assertion("equal", "'%s' to be equal to '%s'", function(a, b) return a == b end)
+make_assertion("error", "result to be an error", function(f) return not pcall(f) end)
+make_assertion("false", "'%s' to be false", function(a) return a == false end)
+make_assertion("greater_than", "'%s' to be greater than '%s'", function(a, b) return a > b end)
+make_assertion("gte", "'%s' to be greater than or equal to '%s'", function(a, b) return a >= b end)
+make_assertion("less_than", "'%s' to be less than '%s'", function(a, b) return a < b end)
+make_assertion("lte", "'%s' to be less than or equal to '%s'", function(a, b) return a <= b end)
+make_assertion("match", "'%s' to be a match for %s", function(a, b) return (tostring(b)):match(a) end)
+make_assertion("nil", "'%s' to be nil", function(a) return a == nil end)
+make_assertion("true", "'%s' to be true", function(a) return a == true end)
+make_assertion("type", "'%s' to be a %s", function(a, b) return type(a) == b end)
+
+make_assertion("not_blank", "'%s' not to be blank", function(a) return a ~= '' and a ~= nil end)
+make_assertion("not_empty", "'%s' not to be an empty table", function(a) return not not next(a) end)
+make_assertion("not_equal", "'%s' not to be equal to '%s'", function(a, b) return a ~= b end)
+make_assertion("not_error", "result not to be an error", function(f) return not not pcall(f) end)
+make_assertion("not_match", "'%s' not to be a match for %s", function(a, b) return not (tostring(b)):match(a) end)
+make_assertion("not_nil", "'%s' not to be nil", function(a) return a ~= nil end)
+make_assertion("not_type", "'%s' not to be a %s", function(a, b) return type(a) ~= b end)
+
+--- Build a contexts table from the test file or function given in <tt>target</tt>.
+-- If the optional <tt>contexts</tt> table argument is provided, then the
+-- resulting contexts will be added to it.
+-- <p>
+-- The resulting contexts table's structure is as follows:
+-- </p>
+-- <code>
+-- {
+-- {parent = 0, name = "this is a context", context = true},
+-- {parent = 1, name = "this is a nested context", context = true},
+-- {parent = 2, name = "this is a test", test = function},
+-- {parent = 2, name = "this is another test", test = function},
+-- {parent = 0, name = "this is test outside any context", test = function},
+-- }
+-- </code>
+-- @param contexts A optional table in which to collect the resulting contexts
+-- and function.
+-- @function load_contexts
+local function load_contexts(target, contexts)
+ local env = {}
+ local current_index = 0
+ local context_table = contexts or {}
+
+ local function context_block(name, func)
+ table.insert(context_table, {parent = current_index, name = name, context = true})
+ local previous_index = current_index
+ current_index = #context_table
+ func()
+ current_index = previous_index
+ end
+
+ local function test_block(name, func)
+ local test_table = {name = name, parent = current_index, test = func or true}
+ if current_index ~= 0 then
+ test_table.context_name = context_table[current_index].name
+ else
+ test_table.context_name = 'top level'
+ end
+ table.insert(context_table, test_table)
+ end
+
+ local function before_block(func)
+ context_table[current_index].before = func
+ end
+
+ local function after_block(func)
+ context_table[current_index].after = func
+ end
+
+ for _, v in ipairs(after_aliases) do env[v] = after_block end
+ for _, v in ipairs(before_aliases) do env[v] = before_block end
+ for _, v in ipairs(context_aliases) do env[v] = context_block end
+ for _, v in ipairs(test_aliases) do env[v] = test_block end
+
+ -- Set these functions in the module's meta table to allow accessing
+ -- telescope's test and context functions without env tricks. This will
+ -- however add tests to a context table used inside the module, so multiple
+ -- test files will add tests to the same top-level context, which may or may
+ -- not be desired.
+ setmetatable(_M, {__index = env})
+
+ setmetatable(env, {__index = _G})
+
+ local func, err = type(target) == 'string' and assert(loadfile(target)) or target
+ if err then error(err) end
+ setfenv(func, env)()
+ return context_table
+end
+
+-- in-place table reverse.
+function table.reverse(t)
+ local len = #t+1
+ for i=1, (len-1)/2 do
+ t[i], t[len-i] = t[len-i], t[i]
+ end
+end
+
+--- Run all tests.
+-- This function will exectute each function in the contexts table.
+-- @param contexts The contexts created by <tt>load_contexts</tt>.
+-- @param callbacks A table of callback functions to be invoked before or after
+-- various test states.
+-- <p>
+-- There is a callback for each test <tt>status_code</tt>, and callbacks to run
+-- before or after each test invocation regardless of outcome.
+-- </p>
+-- <ul>
+-- <li>after - will be invoked after each test</li>
+-- <li>before - will be invoked before each test</li>
+-- <li>err - will be invoked after each test which results in an error</li>
+-- <li>fail - will be invoked after each failing test</li>
+-- <li>pass - will be invoked after each passing test</li>
+-- <li>pending - will be invoked after each pending test</li>
+-- <li>unassertive - will be invoked after each test which doesn't assert
+-- anything</li>
+-- </ul>
+-- <p>
+-- Callbacks can be used, for example, to drop into a debugger upon a failed
+-- assertion or error, for profiling, or updating a GUI progress meter.
+-- </p>
+-- @param test_filter A function to filter tests that match only conditions that you specify.
+-- <p>
+-- For example, the folling would allow you to run only tests whose name matches a pattern:
+-- </p>
+-- <p>
+-- <code>
+-- function(t) return t.name:match("%s* lexer") end
+-- </code>
+-- </p>
+-- @return A table of result tables. Each result table has the following
+-- fields:
+-- <ul>
+-- <li>assertions_invoked - the number of assertions the test invoked</li>
+-- <li>context - the name of the context</li>
+-- <li>message - a table with an error message and stack trace</li>
+-- <li>name - the name of the test</li>
+-- <li>status_code - the resulting status code</li>
+-- <li>status_label - the label for the status_code</li>
+-- </ul>
+-- @see load_contexts
+-- @see status_codes
+-- @function run
+local function run(contexts, callbacks, test_filter)
+
+ local results = {}
+ local status_names = invert_table(status_codes)
+ local test_filter = test_filter or function(a) return a end
+
+ -- Setup a new environment suitable for running a new test
+ local function newEnv()
+ local env = {}
+
+ -- Make sure globals are accessible in the new environment
+ setmetatable(env, {__index = _G})
+
+ -- Setup all the assert functions in the new environment
+ for k, v in pairs(assertions) do
+ setfenv(v, env)
+ env[k] = v
+ end
+
+ return env
+ end
+
+ local env = newEnv()
+
+ local function invoke_callback(name, test)
+ if not callbacks then return end
+ if type(callbacks[name]) == "table" then
+ for _, c in ipairs(callbacks[name]) do c(test) end
+ elseif callbacks[name] then
+ callbacks[name](test)
+ end
+ end
+
+ local function invoke_test(func)
+ local assertions_invoked = 0
+ env.assertion_callback = function()
+ assertions_invoked = assertions_invoked + 1
+ end
+ setfenv(func, env)
+ local result, message = xpcall(func, debug.traceback)
+ if result and assertions_invoked > 0 then
+ return status_codes.pass, assertions_invoked, nil
+ elseif result then
+ return status_codes.unassertive, 0, nil
+ elseif type(message) == "table" then
+ return status_codes.fail, assertions_invoked, message
+ else
+ return status_codes.err, assertions_invoked, {message, debug.traceback()}
+ end
+ end
+
+ for i, v in filter(contexts, function(i, v) return v.test and test_filter(v) end) do
+ env = newEnv() -- Setup a new environment for this test
+
+ local ancestors = ancestors(i, contexts)
+ local context_name = 'Top level'
+ if contexts[i].parent ~= 0 then
+ context_name = contexts[contexts[i].parent].name
+ end
+ local result = {
+ assertions_invoked = 0,
+ name = contexts[i].name,
+ context = context_name,
+ test = i
+ }
+ table.sort(ancestors)
+ -- this "before" is the test callback passed into the runner
+ invoke_callback("before", result)
+
+ -- run all the "before" blocks/functions
+ for _, a in ipairs(ancestors) do
+ if contexts[a].before then
+ setfenv(contexts[a].before, env)
+ contexts[a].before()
+ end
+ end
+
+ -- check if it's a function because pending tests will just have "true"
+ if type(v.test) == "function" then
+ result.status_code, result.assertions_invoked, result.message = invoke_test(v.test)
+ invoke_callback(status_names[result.status_code], result)
+ else
+ result.status_code = status_codes.pending
+ invoke_callback("pending", result)
+ end
+ result.status_label = status_labels[result.status_code]
+
+ -- Run all the "after" blocks/functions
+ table.reverse(ancestors)
+ for _, a in ipairs(ancestors) do
+ if contexts[a].after then
+ setfenv(contexts[a].after, env)
+ contexts[a].after()
+ end
+ end
+
+ invoke_callback("after", result)
+ results[i] = result
+ end
+
+ return results
+
+end
+
+--- Return a detailed report for each context, with the status of each test.
+-- @param contexts The contexts returned by <tt>load_contexts</tt>.
+-- @param results The results returned by <tt>run</tt>.
+-- @function test_report
+local function test_report(contexts, results)
+
+ local buffer = {}
+ local leading_space = " "
+ local level = 0
+ local line_char = "-"
+ local previous_level = 0
+ local status_format_len = 3
+ local status_format = "[%s]"
+ local width = 72
+ local context_name_format = "%-" .. width - status_format_len .. "s"
+ local function_name_format = "%-" .. width - status_format_len .. "s"
+
+ local function space()
+ return leading_space:rep(level - 1)
+ end
+
+ local function add_divider()
+ table.insert(buffer, line_char:rep(width))
+ end
+ add_divider()
+ for i, item in ipairs(contexts) do
+ local ancestors = ancestors(i, contexts)
+ previous_level = level or 0
+ level = #ancestors
+ -- the 4 here is the length of "..." plus one space of padding
+ local name = truncate_string(item.name, width - status_format_len - 4 - #ancestors, '...')
+ if previous_level ~= level and level == 0 then add_divider() end
+ if item.context then
+ table.insert(buffer, context_name_format:format(space() .. name .. ':'))
+ elseif results[i] then
+ table.insert(buffer, function_name_format:format(space() .. name) ..
+ status_format:format(results[i].status_label))
+ end
+ end
+ add_divider()
+ return table.concat(buffer, "\n")
+
+end
+
+--- Return a table of stack traces for tests which produced a failure or an error.
+-- @param contexts The contexts returned by <tt>load_contexts</tt>.
+-- @param results The results returned by <tt>run</tt>.
+-- @function error_report
+local function error_report(contexts, results)
+ local buffer = {}
+ for _, r in filter(results, function(i, r) return r.message end) do
+ local name = contexts[r.test].name
+ table.insert(buffer, name .. ":\n" .. r.message[1] .. "\n" .. r.message[2])
+ end
+ if #buffer > 0 then return table.concat(buffer, "\n") end
+end
+
+--- Get a one-line report and a summary table with the status counts. The
+-- counts given are: total tests, assertions, passed tests, failed tests,
+-- pending tests, and tests which didn't assert anything.
+-- @return A report that can be printed
+-- @return A table with the various counts. Its fields are:
+-- <tt>assertions</tt>, <tt>errors</tt>, <tt>failed</tt>, <tt>passed</tt>,
+-- <tt>pending</tt>, <tt>tests</tt>, <tt>unassertive</tt>.
+-- @param contexts The contexts returned by <tt>load_contexts</tt>.
+-- @param results The results returned by <tt>run</tt>.
+-- @function summary_report
+local function summary_report(contexts, results)
+ local r = {
+ assertions = 0,
+ errors = 0,
+ failed = 0,
+ passed = 0,
+ pending = 0,
+ tests = 0,
+ unassertive = 0
+ }
+ for _, v in pairs(results) do
+ r.tests = r.tests + 1
+ r.assertions = r.assertions + v.assertions_invoked
+ if v.status_code == status_codes.err then r.errors = r.errors + 1
+ elseif v.status_code == status_codes.fail then r.failed = r.failed + 1
+ elseif v.status_code == status_codes.pass then r.passed = r.passed + 1
+ elseif v.status_code == status_codes.pending then r.pending = r.pending + 1
+ elseif v.status_code == status_codes.unassertive then r.unassertive = r.unassertive + 1
+ end
+ end
+ local buffer = {}
+ for _, k in ipairs({"tests", "passed", "assertions", "failed", "errors", "unassertive", "pending"}) do
+ local number = r[k]
+ local label = k
+ if number == 1 then
+ label = label:gsub("s$", "")
+ end
+ table.insert(buffer, ("%d %s"):format(number, label))
+ end
+ return table.concat(buffer, " "), r
+end
+
+_M.after_aliases = after_aliases
+_M.make_assertion = make_assertion
+_M.assertion_message_prefix = assertion_message_prefix
+_M.before_aliases = before_aliases
+_M.context_aliases = context_aliases
+_M.error_report = error_report
+_M.load_contexts = load_contexts
+_M.run = run
+_M.test_report = test_report
+_M.status_codes = status_codes
+_M.status_labels = status_labels
+_M.summary_report = summary_report
+_M.test_aliases = test_aliases
+_M.version = _VERSION
+_M._VERSION = _VERSION
+
+return _M
diff --git a/test/lua/tests.lua b/test/lua/tests.lua
new file mode 100644
index 0000000..82c5469
--- /dev/null
+++ b/test/lua/tests.lua
@@ -0,0 +1,51 @@
+-- Run all unit tests in 'unit' directory
+
+local telescope = require "telescope"
+require "rspamd_assertions"
+local loaded, luacov = pcall(require, 'luacov.runner')
+if not loaded then
+ luacov = {
+ init = function() end,
+ shutdown = function() end,
+ run_report = function() end
+ }
+end
+luacov.init()
+
+local contexts = {}
+
+for _,t in ipairs(tests_list) do
+ telescope.load_contexts(t, contexts)
+end
+local function test_filter(test)
+ return test.name:match(test_pattern)
+end
+if not test_pattern then
+ test_filter = function(_) return true end
+end
+
+local buffer = {}
+local results = telescope.run(contexts, callbacks, test_filter)
+local summary, data = telescope.summary_report(contexts, results)
+
+table.insert(buffer, telescope.test_report(contexts, results))
+table.insert(buffer, summary)
+
+local report = telescope.error_report(contexts, results)
+
+if report then
+ table.insert(buffer, "")
+ table.insert(buffer, report)
+end
+
+if #buffer > 0 then print(table.concat(buffer, "\n")) end
+
+for _, v in pairs(results) do
+ if v.status_code == telescope.status_codes.err or
+ v.status_code == telescope.status_codes.fail then
+ os.exit(1)
+ end
+end
+
+luacov:shutdown()
+luacov:run_report()
diff --git a/test/lua/unit/addr.lua b/test/lua/unit/addr.lua
new file mode 100644
index 0000000..6da72d3
--- /dev/null
+++ b/test/lua/unit/addr.lua
@@ -0,0 +1,46 @@
+-- inet addr tests
+
+context("Inet addr check functions", function()
+ local ffi = require("ffi")
+
+ ffi.cdef[[
+ typedef struct rspamd_inet_addr_s rspamd_inet_addr_t;
+ bool rspamd_parse_inet_address (rspamd_inet_addr_t **target,
+ const char *src, size_t len);
+ void rspamd_inet_address_free (rspamd_inet_addr_t *addr);
+ ]]
+
+ local cases = {
+ {'192.168.1.1', true},
+ {'2a01:4f8:190:43b5::99', true},
+ {'256.1.1.1', false},
+ {'/tmp/socket', true},
+ {'./socket', true},
+ {'[fe80::f919:8b26:ff93:3092%5]', true},
+ {'[fe80::f919:8b26:ff93:3092]', true},
+ {'IPv6:::1', true},
+ {'IPv6:[::1]', true},
+ {'IPv6[:::1]', false},
+ {'[::]', true},
+ {'[1::]', true},
+ {'[000:01:02:003:004:5:6:007]', true}, -- leading zeros
+ {'[A:b:c:DE:fF:0:1:aC]', true}, -- mixed case
+ {'[::192.168.0.1]', true}, -- embedded ipv4
+ {'[1:2:192.168.0.1:5:6]', false}, -- poor octets
+ {'[::ffff:192.1.2]', false}, -- ipv4 without last octet (maybe should be true?)
+ {'[0:0::0:0:8]', true}, -- bogus zeros
+ {'[::192.168.0.0.1]', false}, -- invalid mapping
+ }
+
+ for i,c in ipairs(cases) do
+ test("Create inet addr from string " .. c[1] .. '; expect ' .. tostring(c[2]), function()
+ local ip = ffi.new("rspamd_inet_addr_t* [1]");
+ local res = ffi.C.rspamd_parse_inet_address(ip, c[1], #c[1])
+ assert_equal(res, c[2], "Expect " .. tostring(c[2]) .. " while parsing " .. c[1])
+ if res then
+ ffi.C.rspamd_inet_address_free(ip[0])
+ end
+ end)
+
+ end
+end) \ No newline at end of file
diff --git a/test/lua/unit/base32.lua b/test/lua/unit/base32.lua
new file mode 100644
index 0000000..eb582f5
--- /dev/null
+++ b/test/lua/unit/base32.lua
@@ -0,0 +1,55 @@
+-- Test zbase32 encoding/decoding
+
+context("Base32 encodning", function()
+ local ffi = require("ffi")
+ ffi.cdef[[
+ void ottery_rand_bytes(void *buf, size_t n);
+ unsigned ottery_rand_unsigned(void);
+ unsigned char* rspamd_decode_base32 (const char *in, size_t inlen, size_t *outlen, int how);
+ char * rspamd_encode_base32 (const unsigned char *in, size_t inlen, int how);
+ void g_free(void *ptr);
+ int memcmp(const void *a1, const void *a2, size_t len);
+ ]]
+
+ local function random_buf(max_size)
+ local l = ffi.C.ottery_rand_unsigned() % max_size + 1
+ local buf = ffi.new("unsigned char[?]", l)
+ ffi.C.ottery_rand_bytes(buf, l)
+
+ return buf, l
+ end
+
+ test("Base32 encode test", function()
+ local cases = {
+ {'test123', 'wm3g84fg13cy'},
+ {'hello', 'em3ags7p'}
+ }
+
+ for _,c in ipairs(cases) do
+ local b = ffi.C.rspamd_encode_base32(c[1], #c[1], 0)
+ local s = ffi.string(b)
+ ffi.C.g_free(b)
+ assert_equal(s, c[2], s .. " not equal " .. c[2])
+ end
+ end)
+
+ if os.getenv("RSPAMD_LUA_EXPENSIVE_TESTS") then
+ test("Base32 fuzz test: zbase32", function()
+ for i = 1,1000 do
+ local b, l = random_buf(4096)
+ local how = math.floor(math.random(3) - 1)
+ local ben = ffi.C.rspamd_encode_base32(b, l, how)
+ local bs = ffi.string(ben)
+ local nl = ffi.new("size_t [1]")
+ local nb = ffi.C.rspamd_decode_base32(bs, #bs, nl, how)
+
+ assert_equal(tonumber(nl[0]), l,
+ string.format("invalid size reported: %d reported vs %d expected", tonumber(nl[0]), l))
+ local cmp = ffi.C.memcmp(b, nb, l)
+ ffi.C.g_free(ben)
+ ffi.C.g_free(nb)
+ assert_equal(cmp, 0, "fuzz test failed for length: " .. tostring(l))
+ end
+ end)
+ end
+end) \ No newline at end of file
diff --git a/test/lua/unit/base64.lua b/test/lua/unit/base64.lua
new file mode 100644
index 0000000..02948e2
--- /dev/null
+++ b/test/lua/unit/base64.lua
@@ -0,0 +1,194 @@
+context("Base64 encoding", function()
+ local ffi = require("ffi")
+ local util = require("rspamd_util")
+ local logger = require "rspamd_logger"
+ ffi.cdef[[
+ void rspamd_cryptobox_init (void);
+ void ottery_rand_bytes(void *buf, size_t n);
+ unsigned ottery_rand_unsigned(void);
+ unsigned char* g_base64_decode (const char *in, size_t *outlen);
+ char * rspamd_encode_base64 (const unsigned char *in, size_t inlen,
+ size_t str_len, size_t *outlen);
+ void g_free(void *ptr);
+ int memcmp(const void *a1, const void *a2, size_t len);
+ double base64_test (bool generic, size_t niters, size_t len, size_t str_len);
+ double rspamd_get_ticks (int);
+ ]]
+
+ ffi.C.rspamd_cryptobox_init()
+
+ local function random_buf(max_size)
+ local l = ffi.C.ottery_rand_unsigned() % max_size + 1
+ local buf = ffi.new("unsigned char[?]", l)
+ ffi.C.ottery_rand_bytes(buf, l)
+
+ return buf, l
+ end
+
+ local function random_safe_buf(max_size)
+ local l = ffi.C.ottery_rand_unsigned() % max_size + 1
+ local buf = ffi.new("unsigned char[?]", l)
+
+ for i = 0,l-1 do
+ buf[i] = ffi.C.ottery_rand_unsigned() % 20 + string.byte('A')
+ end
+
+ buf[l - 1] = 0;
+
+ return buf, l
+ end
+
+ test("Base64 encode test", function()
+ local cases = {
+ {"", ""},
+ {"f", "Zg=="},
+ {"fo", "Zm8="},
+ {"foo", "Zm9v"},
+ {"foob", "Zm9vYg=="},
+ {"fooba", "Zm9vYmE="},
+ {"foobar", "Zm9vYmFy"},
+ }
+
+ local nl = ffi.new("size_t [1]")
+ for _,c in ipairs(cases) do
+ local b = ffi.C.rspamd_encode_base64(c[1], #c[1], 0, nl)
+ local s = ffi.string(b)
+ ffi.C.g_free(b)
+ assert_equal(s, c[2], s .. " not equal " .. c[2])
+ end
+ end)
+
+ test("Base64 decode test", function()
+ local cases = {
+ {"", ""},
+ {"f", "Zg=="},
+ {"fo", "Zm8="},
+ {"foo", "Zm9v"},
+ {"foob", "Zm9vYg=="},
+ {"fooba", "Zm9vYmE="},
+ {"foobar", "Zm9vYmFy"},
+ }
+
+ for _,c in ipairs(cases) do
+ local b = tostring(util.decode_base64(c[2]))
+ assert_equal(b, c[1], b .. " not equal " .. c[1])
+ end
+ end)
+
+ test("Base64 line split encode test", function()
+ local text = [[
+Man is distinguished, not only by his reason, but by this singular passion from
+other animals, which is a lust of the mind, that by a perseverance of delight
+in the continued and indefatigable generation of knowledge, exceeds the short
+vehemence of any carnal pleasure.]]
+ local b64 = "TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0aGlz\r\nIHNpbmd1bGFyIHBhc3Npb24gZnJvbQpvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1c3Qgb2Yg\r\ndGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodAppbiB0aGUgY29udGlu\r\ndWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdlLCBleGNlZWRzIHRo\r\nZSBzaG9ydAp2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4="
+ local nl = ffi.new("size_t [1]")
+ local b = ffi.C.rspamd_encode_base64(text, #text, 76, nl)
+ local cmp = ffi.C.memcmp(b, b64, nl[0])
+ ffi.C.g_free(b)
+ assert_equal(cmp, 0)
+ end)
+
+ if os.getenv("RSPAMD_LUA_EXPENSIVE_TESTS") then
+ test("Base64 fuzz test", function()
+ for i = 1,1000 do
+ local b, l = random_safe_buf(4096)
+ local lim = ffi.C.ottery_rand_unsigned() % 64 + 10
+ local orig = ffi.string(b)
+ local ben = util.encode_base64(orig, lim)
+ local dec = util.decode_base64(ben)
+ assert_equal(orig, tostring(dec), "fuzz test failed for length: " .. #orig)
+ end
+ end)
+ test("Base64 fuzz test (ffi)", function()
+ for i = 1,1000 do
+ local b, l = random_buf(4096)
+ local nl = ffi.new("size_t [1]")
+ local lim = ffi.C.ottery_rand_unsigned() % 64 + 10
+ local ben = ffi.C.rspamd_encode_base64(b, l, lim, nl)
+ local bs = ffi.string(ben)
+ local ol = ffi.new("size_t [1]")
+ local nb = ffi.C.g_base64_decode(ben, ol)
+
+ local cmp = ffi.C.memcmp(b, nb, l)
+ ffi.C.g_free(ben)
+ ffi.C.g_free(nb)
+ assert_equal(cmp, 0, "fuzz test failed for length: " .. tostring(l))
+ end
+ end)
+
+ local speed_iters = 10000
+
+ local function perform_base64_speed_test(chunk, is_reference, line_len)
+ local ticks = ffi.C.base64_test(is_reference, speed_iters, chunk, line_len)
+ local what = 'Optimized'
+ if is_reference then
+ what = 'Reference'
+ end
+ logger.messagex("%s base64 %s chunk (%s line len): %s ticks per iter, %s ticks per byte",
+ what, chunk, line_len,
+ ticks / speed_iters, ticks / speed_iters / chunk)
+
+ return 1
+ end
+ test("Base64 test reference vectors 78", function()
+ local res = perform_base64_speed_test(78, true, 0)
+ assert_not_equal(res, 0)
+ end)
+ test("Base64 test optimized vectors 78", function()
+ local res = perform_base64_speed_test(78, false, 0)
+ assert_not_equal(res, 0)
+ end)
+
+ test("Base64 test reference vectors 512", function()
+ local res = perform_base64_speed_test(512, true, 0)
+ assert_not_equal(res, 0)
+ end)
+ test("Base64 test optimized vectors 512", function()
+ local res = perform_base64_speed_test(512, false, 0)
+ assert_not_equal(res, 0)
+ end)
+ test("Base64 test reference vectors 512 (78 line len)", function()
+ local res = perform_base64_speed_test(512, true, 78)
+ assert_not_equal(res, 0)
+ end)
+ test("Base64 test optimized vectors 512 (78 line len)", function()
+ local res = perform_base64_speed_test(512, false, 78)
+ assert_not_equal(res, 0)
+ end)
+
+ test("Base64 test reference vectors 1K", function()
+ local res = perform_base64_speed_test(1024, true, 0)
+ assert_not_equal(res, 0)
+ end)
+ test("Base64 test optimized vectors 1K", function()
+ local res = perform_base64_speed_test(1024, false, 0)
+ assert_not_equal(res, 0)
+ end)
+ test("Base64 test reference vectors 1K (78 line len)", function()
+ local res = perform_base64_speed_test(1024, true, 78)
+ assert_not_equal(res, 0)
+ end)
+ test("Base64 test optimized vectors 1K (78 line len)", function()
+ local res = perform_base64_speed_test(1024, false, 78)
+ assert_not_equal(res, 0)
+ end)
+
+ test("Base64 test reference vectors 10K", function()
+ local res = perform_base64_speed_test(10 * 1024, true, 0)
+ assert_not_equal(res, 0)
+ end)
+ test("Base64 test optimized vectors 10K", function()
+ local res = perform_base64_speed_test(10 * 1024, false, 0)
+ assert_not_equal(res, 0)
+ end)
+ test("Base64 test reference vectors 10K (78 line len)", function()
+ local res = perform_base64_speed_test(10 * 1024, true, 78)
+ assert_not_equal(res, 0)
+ end)
+ test("Base64 test optimized vectors 10K (78 line len)", function()
+ local res = perform_base64_speed_test(10 * 1024, false, 78)
+ assert_not_equal(res, 0)
+ end)
+ end
+end)
diff --git a/test/lua/unit/compression.lua b/test/lua/unit/compression.lua
new file mode 100644
index 0000000..d5c682c
--- /dev/null
+++ b/test/lua/unit/compression.lua
@@ -0,0 +1,58 @@
+-- Compression unit tests
+
+context("Rspamd compression", function()
+ local rspamd_zstd = require "rspamd_zstd"
+ local rspamd_text = require "rspamd_text"
+
+ test("Compressed can be decompressed", function()
+ local str = 'test'
+ local cctx = rspamd_zstd.compress_ctx()
+ local dctx = rspamd_zstd.decompress_ctx()
+ assert_rspamd_eq({actual = dctx:stream(cctx:stream(str, 'end')),
+ expect = rspamd_text.fromstring(str)})
+ end)
+ test("Compressed concatenation can be decompressed", function()
+ local str = 'test'
+ local cctx = rspamd_zstd.compress_ctx()
+ local dctx = rspamd_zstd.decompress_ctx()
+ assert_rspamd_eq({actual = dctx:stream(cctx:stream(str) .. cctx:stream(str, 'end')),
+ expect = rspamd_text.fromstring(str .. str)})
+ end)
+
+ if os.getenv("RSPAMD_LUA_EXPENSIVE_TESTS") then
+ local sizes = {10, 100, 1000, 10000}
+ for _,sz in ipairs(sizes) do
+ test("Compressed fuzz size: " .. tostring(sz), function()
+ for _=1,1000 do
+ local rnd = rspamd_text.randombytes(sz)
+ local cctx = rspamd_zstd.compress_ctx()
+ local dctx = rspamd_zstd.decompress_ctx()
+ assert_rspamd_eq({actual = dctx:stream(cctx:stream(rnd, 'end')),
+ expect = rnd})
+ end
+ end)
+ end
+ end
+
+ test("Compressed chunks", function()
+ local cctx = rspamd_zstd.compress_ctx()
+ local tin = {}
+ local tout = {}
+ for i=1,1000 do
+ local rnd = rspamd_text.randombytes(i)
+ tin[#tin + 1] = rnd
+ end
+ for i=1,1000 do
+ local o
+ if i == 1000 then
+ o = cctx:stream(tin[i], 'end')
+ else
+ o = cctx:stream(tin[i])
+ end
+ tout[#tout + 1] = o
+ end
+ local dctx = rspamd_zstd.decompress_ctx()
+ assert_rspamd_eq({actual = dctx:stream(rspamd_text.fromtable(tout)),
+ expect = rspamd_text.fromtable(tin)})
+ end)
+end) \ No newline at end of file
diff --git a/test/lua/unit/expressions.lua b/test/lua/unit/expressions.lua
new file mode 100644
index 0000000..3d05685
--- /dev/null
+++ b/test/lua/unit/expressions.lua
@@ -0,0 +1,111 @@
+-- Expressions unit tests
+
+context("Rspamd expressions", function()
+ local rspamd_expression = require "rspamd_expression"
+ local rspamd_mempool = require "rspamd_mempool"
+ local rspamd_regexp = require "rspamd_regexp"
+ local split_re = rspamd_regexp.create('/\\s+|\\)|\\(/')
+
+ local function parse_func(str)
+ -- extract token till the first space character
+ local token = str
+ local t = split_re:split(str)
+ if t then
+ token = t[1]
+ end
+ -- Return token name
+ return token
+ end
+
+ local atoms = {
+ A = 1.0,
+ B = 0,
+ C = 1,
+ D = 0,
+ E = 1,
+ F = 0,
+ G = 0,
+ H = 0,
+ I = 0,
+ J = 0,
+ K = 0,
+ }
+ local function process_func(token, input)
+
+ --print(token)
+ local t = input[token]
+
+ return t
+ end
+
+ local pool = rspamd_mempool.create()
+
+ local cases = {
+ {'A & (!B | C)', '(A) (B) ! (C) | &'},
+ {'A & B | !C', '(C) ! (A) (B) & |'},
+ {'A & (B | !C)', '(A) (B) (C) ! | &'},
+ {'A & B &', nil},
+ -- Unbalanced braces
+ {'(((A))', nil},
+ -- Balanced braces
+ {'(((A)))', '(A)'},
+ -- Plus and comparison operators
+ {'A + B + C + D > 2', '(A) (B) (C) (D) +(4) 2 >'},
+ -- Plus and logic operators
+ {'((A + B + C + D) > 2) & D', '(D) (A) (B) (C) (D) +(4) 2 > &'},
+ -- Associativity
+ {'A | B | C & D & E', '(A) (B) (C) (D) (E) &(3) |(3)'},
+ -- More associativity
+ {'1 | 0 & 0 | 0', '(1) (0) (0) (0) & |(3)'},
+ {'(A) & (B) & ((C) | (D) | (E) | (F))', '(A) (B) (C) (D) (E) (F) |(4) &(3)' },
+ -- Extra space
+ {'A & B | ! C', '(C) ! (A) (B) & |'},
+ -- False minus
+ {'A + B + -C', '(A) (B) (-C) +(3)'},
+ }
+ for _,c in ipairs(cases) do
+ test("Expression creation function: " .. c[1], function()
+ local expr,err = rspamd_expression.create(c[1],
+ {parse_func, process_func}, pool)
+
+ if not c[2] then
+ assert_nil(expr, "Should not be able to parse " .. c[1])
+ else
+ assert_not_nil(expr, "Cannot parse " .. c[1] .. '; error: ' .. (err or 'wut??'))
+ assert_equal(expr:to_string(), c[2], string.format("Evaluated expr to '%s', expected: '%s'",
+ expr:to_string(), c[2]))
+ end
+ end)
+ end
+ -- Expression is destroyed when the corresponding pool is destroyed
+ cases = {
+ {'(E) && ((B + B + B + B) >= 1)', 0},
+ {'A & B | !C', 0},
+ {'A & (!B | C)', 1},
+ {'A + B + C + D + E + F >= 2', 1},
+ {'((A + B + C + D) > 1) & F', 0},
+ {'(A + B + C + D) > 1 && F || E', 1},
+ {'(A + B + C + D) > 100 && F || !E', 0},
+ {'F && ((A + B + C + D) > 1)', 0},
+ {'(E) && ((B + B + B + B) >= 1)', 0},
+ {'!!C', 1},
+ {'(B) & (D) & ((G) | (H) | (I) | (A))', 0},
+ {'A & C & (!D || !C || !E)', 1},
+ {'A & C & !(D || C || E)', 0},
+ {'A + B + C', 2},
+ {'A * 2.0 + B + C', 3},
+ {'A * 2.0 + B - C', 1},
+ {'A / 2.0 + B - C', -0.5},
+ }
+ for _,c in ipairs(cases) do
+ test("Expression process function: " .. c[1], function()
+ local expr,err = rspamd_expression.create(c[1],
+ {parse_func, process_func}, pool)
+
+ assert_not_nil(expr, "Cannot parse " .. c[1] .. '; error: ' .. (err or 'wut??'))
+ res = expr:process(atoms)
+ assert_equal(res, c[2], string.format("Processed expr '%s'{%s} returned '%d', expected: '%d'",
+ expr:to_string(), c[1], res, c[2]))
+ end)
+ end
+end)
diff --git a/test/lua/unit/folding.lua b/test/lua/unit/folding.lua
new file mode 100644
index 0000000..8a92384
--- /dev/null
+++ b/test/lua/unit/folding.lua
@@ -0,0 +1,66 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+]]--
+
+context("Headers folding unit test", function()
+ local util = require("rspamd_util")
+ -- {header, value}, "expected result"
+ local cases = {
+ {{"test", "test"}, "test"},
+ {{"test1", "_abc _def _ghi _fdjhfd _fhdjkfh _dkhkjd _fdjkf _dshfdks _fhdjfdkhfk _dshfds _fdsjk _fdkhfdks _fdsjf _dkf"},
+ "_abc _def _ghi _fdjhfd _fhdjkfh _dkhkjd _fdjkf _dshfdks\r\n\t_fhdjfdkhfk _dshfds _fdsjk _fdkhfdks _fdsjf _dkf"
+ },
+ {{"Test1", "_abc _def _ghi _fdjhfd _fhdjkfh _dkhaaaaaaaaaaakjdfdjkfdshfdksfhdjfdkhfkdshfdsfdsjkfdkhfdksfdsjf _dkf"},
+ "_abc _def _ghi _fdjhfd _fhdjkfh\r\n\t_dkhaaaaaaaaaaakjdfdjkfdshfdksfhdjfdkhfkdshfdsfdsjkfdkhfdksfdsjf\r\n\t_dkf"
+ },
+ {{"Content-Type", "multipart/mixed; boundary=\"---- =_NextPart_000_01BDBF1F.DA8F77EE\"hhhhhhhhhhhhhhhhhhhhhhhhh fjsdhfkjsd fhdjsfhkj"},
+ "multipart/mixed; boundary=\"---- =_NextPart_000_01BDBF1F.DA8F77EE\"hhhhhhhhhhhhhhhhhhhhhhhhh\r\n\tfjsdhfkjsd fhdjsfhkj"
+ },
+ {{"Content-Type", "multipart/mixed; boundary=\"---- =_NextPart_000_01BDBF1F.DA8F77EE\"hkjhgkfhgfhgf\"hfkjdhf fhjf fghjghf fdshjfhdsj\" hgjhgfjk"},
+ "multipart/mixed; boundary=\"---- =_NextPart_000_01BDBF1F.DA8F77EE\"hkjhgkfhgfhgf\"hfkjdhf fhjf fghjghf fdshjfhdsj\" hgjhgfjk"
+ },
+ {{"Content-Type", "Content-Type: multipart/mixed; boundary=\"---- =_NextPart_000_01BDBF1F.DA8F77EE\" abc def ghfdgfdsgj fdshfgfsdgfdsg hfsdgjfsdg fgsfgjsg"},
+ "Content-Type: multipart/mixed; boundary=\"---- =_NextPart_000_01BDBF1F.DA8F77EE\" abc\r\n\tdef ghfdgfdsgj fdshfgfsdgfdsg hfsdgjfsdg fgsfgjsg"
+ },
+ {{"X-Spam-Symbols", "Returnpath_BL2,HFILTER_FROM_BOUNCE,R_PARTS_DIFFER,R_IP_PBL,R_ONE_RCPT,R_googleredir,R_TO_SEEMS_AUTO,R_SPF_NEUTRAL,R_PRIORITY_3,RBL_SPAMHAUS_PBL,HFILTER_MID_NOT_FQDN,MISSING_CTE,R_HAS_URL,RBL_SPAMHAUS_CSS,RBL_SPAMHAUS_XBL,BAYES_SPAM,RECEIVED_RBL10", ','},
+ "Returnpath_BL2,\r\n\tHFILTER_FROM_BOUNCE,\r\n\tR_PARTS_DIFFER,\r\n\tR_IP_PBL,\r\n\tR_ONE_RCPT,\r\n\tR_googleredir,\r\n\tR_TO_SEEMS_AUTO,\r\n\tR_SPF_NEUTRAL,\r\n\tR_PRIORITY_3,\r\n\tRBL_SPAMHAUS_PBL,\r\n\tHFILTER_MID_NOT_FQDN,\r\n\tMISSING_CTE,\r\n\tR_HAS_URL,\r\n\tRBL_SPAMHAUS_CSS,\r\n\tRBL_SPAMHAUS_XBL,\r\n\tBAYES_SPAM,\r\n\tRECEIVED_RBL10"
+ },
+ }
+ local function escape_spaces(str)
+ str = string.gsub(str, '[\r\n]+', '<NL>')
+ str = string.gsub(str, '[ ]', '<SP>')
+ str = string.gsub(str, '[\t]', '<TB>')
+
+ return str
+ end
+ for i,c in ipairs(cases) do
+ test("Headers folding: " .. i, function()
+ local fv = util.fold_header(c[1][1], c[1][2], 'crlf', c[1][3])
+ assert_not_nil(fv)
+ assert_equal(fv, c[2], string.format("'%s' doesn't match with '%s'",
+ escape_spaces(c[2]), escape_spaces(fv)))
+ end)
+ end
+end)
diff --git a/test/lua/unit/fpconv.lua b/test/lua/unit/fpconv.lua
new file mode 100644
index 0000000..e64626f
--- /dev/null
+++ b/test/lua/unit/fpconv.lua
@@ -0,0 +1,97 @@
+-- fpconv tests
+
+context("Fpconv printf functions", function()
+ local ffi = require("ffi")
+ local niter_fuzz = 100000
+ local function small_double()
+ return math.random()
+ end
+ local function large_double()
+ return math.random() * math.random(2^52)
+ end
+ local function huge_double()
+ return math.random(2^52) * math.random(2^52)
+ end
+ local function tiny_double()
+ return math.random() / math.random(2^52)
+ end
+ ffi.cdef[[
+int snprintf(char *str, size_t size, const char *format, ...);
+long rspamd_snprintf(char *str, size_t size, const char *format, ...);
+long rspamd_printf(const char *format, ...);
+]]
+ local benchmarks = {
+ {'tiny fixed', small_double, '%f'},
+ {'small fixed', tiny_double, '%f'},
+ {'large fixed', large_double, '%.3f'},
+ {'huge fixed', huge_double, '%.3f'},
+ {'tiny scientific', small_double, '%g'},
+ {'small scientific', tiny_double, '%g'},
+ {'large scientific', large_double, '%g'},
+ {'huge scientific', huge_double, '%g'},
+ }
+
+ local generic = {
+ {0, '%f', '0'},
+ {0, '%.1f', '0.0'},
+ {0, '%.2f', '0.00'},
+ {0, '%.32f', '0.000000000000000000000000000'}, -- max
+ {0, '%.150f', '0.000000000000000000000000000'}, -- too large
+ {1/3, '%f', '0.3333333333333333'},
+ {1/3, '%.1f', '0.3'},
+ {1/3, '%.2f', '0.33'},
+ {-1/3, '%.32f', '-0.333333333333333300000000000'},
+ {-1/3, '%.150f', '-0.333333333333333300000000000'},
+ {-3.6817595395344857e-68, '%f', '-3.6817595395344857e-68'},
+ {3.5844466002796428e+298, '%f', '3.5844466002796428e+298'},
+ {9223372036854775808, '%f', '9223372036854776000'}, -- 2^63 with precision lost
+ {2^50 + 0.2, '%f', '1125899906842624.3'}, -- 2^50 with precision lost
+ {2^50 + 0.2, '%.2f', '1125899906842624.30'}, -- 2^50 with precision lost
+ {-3.6817595395344857e-68, '%.3f', '-0.000'}, -- not enough precision
+ {3.5844466002796428e+298, '%.3f', '3.5844466002796428e+298'},
+ {9223372036854775808, '%.3f', '9223372036854776000.000'}, -- 2^63 with precision lost
+ {math.huge, '%f', 'inf'},
+ {-math.huge, '%f', '-inf'},
+ {0.0/0.0, '%f', 'nan'},
+ {math.huge, '%.1f', 'inf'},
+ {-math.huge, '%.2f', '-inf'},
+ {0.0/0.0, '%.3f', 'nan'},
+ {math.huge, '%g', 'inf'},
+ {-math.huge, '%g', '-inf'},
+ {0.0/0.0, '%g', 'nan'},
+ }
+
+ local buf = ffi.new("char[64]")
+ local buf2 = ffi.new("char[64]")
+
+ for i,c in ipairs(generic) do
+ test("Generic fp test fmt: " .. c[2] .. '; ' .. tostring(c[1]), function()
+ ffi.C.rspamd_snprintf(buf, 64, c[2], c[1])
+ local sbuf = ffi.string(buf)
+ assert_equal(sbuf, c[3], c[3] .. " but test returned " .. sbuf)
+ end)
+ end
+ if os.getenv("RSPAMD_LUA_EXPENSIVE_TESTS") then
+ for _,c in ipairs(benchmarks) do
+ test("Fuzz fp test " .. c[1], function()
+ for _=1,niter_fuzz do
+ local sign = 1
+ if math.random() > 0.5 then
+ sign = -1
+ end
+ local d = c[2]() * sign
+ ffi.C.snprintf(buf, 64, c[3], d)
+ ffi.C.rspamd_snprintf(buf2, 64, c[3], d)
+
+ local sbuf = ffi.string(buf)
+ local sbuf2 = ffi.string(buf2)
+
+ assert_less_than(math.abs(d - tonumber(sbuf2))/math.abs(d),
+ 0.00001,
+ string.format('rspamd emitted: %s, libc emitted: %s, original number: %g',
+ sbuf2, sbuf, d))
+ end
+ end)
+ end
+ end
+end) \ No newline at end of file
diff --git a/test/lua/unit/html.lua b/test/lua/unit/html.lua
new file mode 100644
index 0000000..81c52ec
--- /dev/null
+++ b/test/lua/unit/html.lua
@@ -0,0 +1,113 @@
+context("HTML processing", function()
+ local rspamd_util = require("rspamd_util")
+ local logger = require("rspamd_logger")
+ local cases = {
+ -- Entities
+ {[[<html><body>.&#102;&#105;&#114;&#101;&#98;&#97;&#115;&#101;&#97;&#112;&#112;.&#99;&#111;&#109;</body></html>]],
+ [[.firebaseapp.com]]},
+ {[[
+<?xml version="1.0" encoding="iso-8859-1"?>
+ <!DOCTYPE html
+ PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <title>
+ Wikibooks
+ </title>
+ </head>
+ <body>
+ <p>
+ Hello, world!
+
+ </p>
+ </body>
+ </html>]], 'Hello, world!\n'},
+ {[[
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>title</title>
+ <link rel="stylesheet" href="style.css">
+ <script src="script.js"></script>
+ <style><!--
+- -a -a -a -- --- -
+ --></head>
+ <body>
+ <!-- page content -->
+ Hello, world!
+ </body>
+</html>
+ ]], 'Hello, world!'},
+ {[[
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>title</title>
+ <link rel="stylesheet" href="style.css">
+ <script src="script.js"></script>
+ </head>
+ <body>
+ <!-- page content -->
+ Hello, world!<br>test</br><br>content</hr>more content<br>
+ <div>
+ content inside div
+ </div>
+ </body>
+</html>
+ ]], 'Hello, world!\ntest\ncontentmore content\ncontent inside div\n'},
+ {[[
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>title</title>
+ <link rel="stylesheet" href="style.css">
+ <script src="script.js"></script>
+ </head>
+ <body>
+ <!-- tabular content -->
+ <table>
+ content
+ </table>
+ <table>
+ <tr>
+ <th>heada</th>
+ <th>headb</th>
+ </tr>
+ <tr>
+ <td>data1</td>
+ <td>data2</td>
+ </tr>
+ </table>
+
+ </body>
+</html>
+ ]], 'content\nheada headb\ndata1 data2\n'},
+ {[[
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>title</title>
+ <link rel="stylesheet" href="style.css">
+ <script src="script.js"></script>
+ </head>
+ <body>
+ <!-- escape content -->
+ a&nbsp;b a &gt; b a &lt; b a &amp; b &apos;a &quot;a&quot;
+ </body>
+</html>
+ ]], 'a b a > b a < b a & b \'a "a"'},
+ }
+
+ for i,c in ipairs(cases) do
+ test("Extract text from HTML " .. tostring(i), function()
+ local t = rspamd_util.parse_html(c[1])
+
+ assert_not_nil(t)
+ assert_equal(c[2], tostring(t), string.format("'%s' doesn't match with '%s'",
+ c[2], t))
+
+ end)
+ end
+end)
diff --git a/test/lua/unit/kann.lua b/test/lua/unit/kann.lua
new file mode 100644
index 0000000..4f8185b
--- /dev/null
+++ b/test/lua/unit/kann.lua
@@ -0,0 +1,46 @@
+-- Simple kann test (xor function vs 2 layer MLP)
+
+context("Kann test", function()
+ local kann = require "rspamd_kann"
+ local k
+ local inputs = {
+ {0, 0},
+ {0, 1},
+ {1, 0},
+ {1, 1}
+ }
+
+ local outputs = {
+ {0},
+ {1},
+ {1},
+ {0}
+ }
+
+ local t = kann.layer.input(2)
+ t = kann.transform.relu(t)
+ t = kann.transform.tanh(kann.layer.dense(t, 2));
+ t = kann.layer.cost(t, 1, kann.cost.mse)
+ k = kann.new.kann(t)
+
+ local iters = 500
+ local niter = k:train1(inputs, outputs, {
+ lr = 0.01,
+ max_epoch = iters,
+ mini_size = 80,
+ })
+
+ local ser = k:save()
+ k = kann.load(ser)
+
+ for i,inp in ipairs(inputs) do
+ test(string.format("Check XOR MLP %s ^ %s == %s", inp[1], inp[2], outputs[i][1]),
+ function()
+ local res = math.floor(k:apply1(inp)[1] + 0.5)
+ assert_equal(outputs[i][1], res,
+ tostring(outputs[i][1]) .. " but test returned " .. tostring(res))
+ end)
+ end
+
+
+end) \ No newline at end of file
diff --git a/test/lua/unit/logger.lua b/test/lua/unit/logger.lua
new file mode 100644
index 0000000..dc01207
--- /dev/null
+++ b/test/lua/unit/logger.lua
@@ -0,0 +1,27 @@
+context("Logger unit tests", function()
+ test("Logger functions", function()
+ local log = require "rspamd_logger"
+
+ local cases = {
+ {'string', 'string'},
+ {'%1', 'string', 'string'},
+ {'%1', '1.1', 1.1},
+ {'%1', '1', 1},
+ {'%1', 'true', true},
+ {'%1', '{[1] = 1, [2] = test}', {1, 'test'}},
+ {'%1', '{[1] = 1, [2] = 2.1, [k2] = test}', {1, 2.1, k2='test'}},
+ {'%s', 'true', true},
+ }
+
+ for _,c in ipairs(cases) do
+ local s
+ if c[3] then
+ s = log.slog(c[1], c[3])
+ else
+ s = log.slog(c[1])
+ end
+ assert_equal(s, c[2], string.format("'%s' doesn't match with '%s'",
+ c[2], s))
+ end
+ end)
+end) \ No newline at end of file
diff --git a/test/lua/unit/lua_mime.message_to_ucl.lua b/test/lua/unit/lua_mime.message_to_ucl.lua
new file mode 100644
index 0000000..e88fb72
--- /dev/null
+++ b/test/lua/unit/lua_mime.message_to_ucl.lua
@@ -0,0 +1,330 @@
+
+--[=========[ ******************* message ******************* ]=========]
+local cases = {
+ { message = [[
+Received: from mail0.mindspring.com (unknown [1.1.1.1])
+ (using TLSv1.2 with cipher ECDHE-ECDSA-AES256-GCM-SHA384 (256/256 bits))
+ (No client certificate requested)
+ by mail.example.com (Postfix) with ESMTPS id 88A0C6B332
+ for <example@example.com>; Wed, 24 Nov 2021 19:05:43 +0000 (GMT)
+From: <>
+To: <nobody@example.com>
+Subject: test
+Content-Type: multipart/alternative;
+ boundary="_000_6be055295eab48a5af7ad4022f33e2d0_"
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 7bit
+
+Hello world
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/html; charset="utf-8"
+
+<html><body>
+lol
+</html>
+]],
+ expected = [[{
+ "parts": [
+ {
+ "type": "multipart/alternative",
+ "multipart_boundary": "_000_6be055295eab48a5af7ad4022f33e2d0_",
+ "size": 0,
+ "headers": []
+ },
+ {
+ "content": "Hello world\n\n\n",
+ "size": 14,
+ "type": "text/plain",
+ "boundary": "_000_6be055295eab48a5af7ad4022f33e2d0_",
+ "detected_type": "text/plain",
+ "headers": [
+ {
+ "order": 0,
+ "raw": "Content-Type: text/plain; charset=\"utf-8\"\n",
+ "empty_separator": false,
+ "value": "text/plain; charset=\"utf-8\"",
+ "separator": " ",
+ "decoded": "text/plain; charset=\"utf-8\"",
+ "name": "Content-Type",
+ "tab_separated": false
+ },
+ {
+ "order": 1,
+ "raw": "Content-Transfer-Encoding: 7bit\n",
+ "empty_separator": false,
+ "value": "7bit",
+ "separator": " ",
+ "decoded": "7bit",
+ "name": "Content-Transfer-Encoding",
+ "tab_separated": false
+ }
+ ]
+ },
+ {
+ "content": "<html><body>\nlol\n</html>\n",
+ "size": 25,
+ "type": "text/html",
+ "boundary": "_000_6be055295eab48a5af7ad4022f33e2d0_",
+ "detected_type": "text/html",
+ "headers": [
+ {
+ "order": 0,
+ "raw": "Content-Type: text/html; charset=\"utf-8\"\n",
+ "empty_separator": false,
+ "value": "text/html; charset=\"utf-8\"",
+ "separator": " ",
+ "decoded": "text/html; charset=\"utf-8\"",
+ "name": "Content-Type",
+ "tab_separated": false
+ }
+ ]
+ }
+ ],
+ "newlines": "lf",
+ "digest": "043cf1a314d0a1af95951d6aec932faf",
+ "envelope": {
+ "recipients_smtp": [
+ {
+ "addr": "test1@example.com",
+ "raw": "<test1@example.com>",
+ "flags": {
+ "valid": true
+ },
+ "user": "test1",
+ "name": "Test1",
+ "domain": "example.com"
+ },
+ {
+ "addr": "test2@example.com",
+ "raw": "<test2@example.com>",
+ "flags": {
+ "valid": true
+ },
+ "user": "test2",
+ "name": "Test2",
+ "domain": "example.com"
+ }
+ ],
+ "from_smtp": {
+ "addr": "test@example.com",
+ "raw": "<test@example.com>",
+ "flags": {
+ "valid": true
+ },
+ "user": "test",
+ "name": "Test",
+ "domain": "example.com"
+ },
+ "helo": "hello mail",
+ "from_ip": "198.172.22.91"
+ },
+ "size": 666,
+ "headers": [
+ {
+ "order": 0,
+ "raw": "Received: from mail0.mindspring.com (unknown [1.1.1.1])\n\t(using TLSv1.2 with cipher ECDHE-ECDSA-AES256-GCM-SHA384 (256/256 bits))\n\t(No client certificate requested)\n\tby mail.example.com (Postfix) with ESMTPS id 88A0C6B332\n\tfor <example@example.com>; Wed, 24 Nov 2021 19:05:43 +0000 (GMT)\n",
+ "empty_separator": false,
+ "value": "from mail0.mindspring.com (unknown [1.1.1.1]) (using TLSv1.2 with cipher ECDHE-ECDSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.example.com (Postfix) with ESMTPS id 88A0C6B332 for <example@example.com>; Wed, 24 Nov 2021 19:05:43 +0000 (GMT)",
+ "separator": " ",
+ "decoded": "from mail0.mindspring.com (unknown [1.1.1.1]) (using TLSv1.2 with cipher ECDHE-ECDSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.example.com (Postfix) with ESMTPS id 88A0C6B332 for <example@example.com>; Wed, 24 Nov 2021 19:05:43 +0000 (GMT)",
+ "name": "Received",
+ "tab_separated": false
+ },
+ {
+ "order": 1,
+ "raw": "From: <>\n",
+ "empty_separator": false,
+ "value": "<>",
+ "separator": " ",
+ "decoded": "<>",
+ "name": "From",
+ "tab_separated": false
+ },
+ {
+ "order": 2,
+ "raw": "To: <nobody@example.com>\n",
+ "empty_separator": false,
+ "value": "<nobody@example.com>",
+ "separator": " ",
+ "decoded": "<nobody@example.com>",
+ "name": "To",
+ "tab_separated": false
+ },
+ {
+ "order": 3,
+ "raw": "Subject: test\n",
+ "empty_separator": false,
+ "value": "test",
+ "separator": " ",
+ "decoded": "test",
+ "name": "Subject",
+ "tab_separated": false
+ },
+ {
+ "order": 4,
+ "raw": "Content-Type: multipart/alternative;\n boundary=\"_000_6be055295eab48a5af7ad4022f33e2d0_\"\n",
+ "empty_separator": false,
+ "value": "multipart/alternative; boundary=\"_000_6be055295eab48a5af7ad4022f33e2d0_\"",
+ "separator": " ",
+ "decoded": "multipart/alternative; boundary=\"_000_6be055295eab48a5af7ad4022f33e2d0_\"",
+ "name": "Content-Type",
+ "tab_separated": false
+ }
+ ]
+}]]
+ },
+ {
+ message = [[
+From: <>
+Content-Type: multipart/mixed; boundary="-"
+
+123
+---
+321
+
+--
+WBR yours
+--
+a
+-----------------
+b
+------
+c
+-
+d
+--------
+f
+-----
+]],
+ expected = [[
+{
+ "parts": [
+ {
+ "type": "multipart/mixed",
+ "multipart_boundary": "-",
+ "size": 0,
+ "headers": []
+ },
+ {
+ "content": "--\nWBR yours\n--\na\n-----------------\nb\n------\nc\n-\nd\n--------\nf\n",
+ "size": 62,
+ "type": "text/plain",
+ "boundary": "-",
+ "detected_type": "text/plain",
+ "headers": []
+ }
+ ],
+ "newlines": "lf",
+ "digest": "1a680eb7563f32a2fbf67cf45e90f045",
+ "envelope": {
+ "recipients_smtp": [
+ {
+ "addr": "test1@example.com",
+ "raw": "<test1@example.com>",
+ "flags": {
+ "valid": true
+ },
+ "user": "test1",
+ "name": "Test1",
+ "domain": "example.com"
+ },
+ {
+ "addr": "test2@example.com",
+ "raw": "<test2@example.com>",
+ "flags": {
+ "valid": true
+ },
+ "user": "test2",
+ "name": "Test2",
+ "domain": "example.com"
+ }
+ ],
+ "from_smtp": {
+ "addr": "test@example.com",
+ "raw": "<test@example.com>",
+ "flags": {
+ "valid": true
+ },
+ "user": "test",
+ "name": "Test",
+ "domain": "example.com"
+ },
+ "helo": "hello mail",
+ "from_ip": "198.172.22.91"
+ },
+ "size": 135,
+ "headers": [
+ {
+ "order": 0,
+ "raw": "From: <>\n",
+ "empty_separator": false,
+ "value": "<>",
+ "separator": " ",
+ "decoded": "<>",
+ "name": "From",
+ "tab_separated": false
+ },
+ {
+ "order": 1,
+ "raw": "Content-Type: multipart/mixed; boundary=\"-\"\n",
+ "empty_separator": false,
+ "value": "multipart/mixed; boundary=\"-\"",
+ "separator": " ",
+ "decoded": "multipart/mixed; boundary=\"-\"",
+ "name": "Content-Type",
+ "tab_separated": false
+ }
+ ]
+}]]
+ }
+}
+
+context("Task piecewise split", function()
+ local rspamd_task = require "rspamd_task"
+ local rspamd_util = require "rspamd_util"
+ local rspamd_test_helper = require "rspamd_test_helper"
+ local lua_mime = require "lua_mime"
+ local ucl = require "ucl"
+ local rspamd_parsers = require "rspamd_parsers"
+
+ rspamd_test_helper.init_url_parser()
+ local cfg = rspamd_util.config_from_ucl(rspamd_test_helper.default_config(),
+ "INIT_URL,INIT_LIBS,INIT_SYMCACHE,INIT_VALIDATE,INIT_PRELOAD_MAPS")
+
+ for i,case in ipairs(cases) do
+ test("Simple message split case " .. tostring(i), function()
+ local res,task = rspamd_task.load_from_string(case.message, cfg)
+
+ if not res or not task then
+ assert_true(false, "failed to load message")
+ end
+
+ task:set_from('smtp', rspamd_parsers.parse_mail_address("Test <test@example.com>")[1])
+ task:set_recipients('smtp', {
+ rspamd_parsers.parse_mail_address("Test1 <test1@example.com>")[1],
+ rspamd_parsers.parse_mail_address("Test2 <test2@example.com>")[1]
+ }, 'rewrite')
+ task:set_from_ip("198.172.22.91")
+ task:set_user("cool user name")
+ task:set_helo("hello mail")
+ task:process_message()
+ local parser = ucl.parser()
+ local res = parser:parse_string(case.expected)
+ assert_true(res)
+ local expected = parser:get_object()
+ local ucl_object = lua_mime.message_to_ucl(task, true)
+ local schema = lua_mime.message_to_ucl_schema()
+ assert_true(schema(ucl_object))
+ assert_rspamd_table_eq({
+ actual = ucl_object,
+ expect = expected
+ })
+ task:destroy()
+ end)
+ end
+
+end) \ No newline at end of file
diff --git a/test/lua/unit/lua_util.extract_specific_urls.lua b/test/lua/unit/lua_util.extract_specific_urls.lua
new file mode 100644
index 0000000..a7e2f9f
--- /dev/null
+++ b/test/lua/unit/lua_util.extract_specific_urls.lua
@@ -0,0 +1,345 @@
+
+local msg, msg_img
+local logger = require "rspamd_logger"
+local rspamd_util = require "rspamd_util"
+local rspamd_task = require "rspamd_task"
+local util = require 'lua_util'
+local mpool = require "rspamd_mempool"
+local fun = require "fun"
+local url = require "rspamd_url"
+
+--[=========[ ******************* message ******************* ]=========]
+msg = [[
+From: <>
+To: <nobody@example.com>
+Subject: test
+Content-Type: multipart/alternative;
+ boundary="_000_6be055295eab48a5af7ad4022f33e2d0_"
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: base64
+
+Hello world
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/html; charset="utf-8"
+
+<html><body>
+<a href="http://example.net">http://example.net</a>
+<a href="http://example1.net">http://example1.net</a>
+<a href="http://example2.net">http://example2.net</a>
+<a href="http://example3.net">http://example3.net</a>
+<a href="http://example4.net">http://example4.net</a>
+<a href="http://domain1.com">http://domain1.com</a>
+<a href="http://domain2.com">http://domain2.com</a>
+<a href="http://domain3.com">http://domain3.com</a>
+<a href="http://domain4.com">http://domain4.com</a>
+<a href="http://domain5.com">http://domain5.com</a>
+<a href="http://domain.com">http://example.net/</a>
+<img src="http://example5.org">hahaha</img>
+</html>
+]]
+msg_img = [[
+From: <>
+To: <nobody@example.com>
+Subject: test
+Content-Type: multipart/alternative;
+ boundary="_000_6be055295eab48a5af7ad4022f33e2d0_"
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: base64
+
+Hello world
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/html; charset="utf-8"
+
+<html><body>
+<a href="http://example.net">http://example.net</a>
+<a href="http://domain.com">http://example.net</a>
+<img src="http://example5.org">hahaha</img>
+</html>
+]]
+
+local function prepare_actual_result(actual)
+ return fun.totable(fun.map(
+ function(u) return u:get_raw():gsub('^%w+://', '') end,
+ actual
+ ))
+end
+
+context("Lua util - extract_specific_urls plain", function()
+ local test_helper = require "rspamd_test_helper"
+
+ test_helper.init_url_parser()
+
+ local task_object = {
+ urls = {},
+ cache_set = function(self, ...) end,
+ cache_get = function(self, ...) end,
+ get_urls = function(self, need_emails) return self.urls end
+ }
+
+ local url_list = {
+ "google.com",
+ "mail.com",
+ "bizz.com",
+ "bing.com",
+ "example.com",
+ "gov.co.net",
+ "tesco.co.net",
+ "domain1.co.net",
+ "domain2.co.net",
+ "domain3.co.net",
+ "domain4.co.net",
+ "abc.org",
+ "icq.org",
+ "meet.org",
+ "domain1.org",
+ "domain2.org",
+ "domain3.org",
+ "test.com",
+ }
+
+ local cases = {
+ {expect = url_list, filter = nil, limit = 9999, need_emails = true, prefix = 'p'},
+ {expect = {}, filter = (function() return false end), limit = 9999, need_emails = true, prefix = 'p'},
+ {expect = {"domain4.co.net", "test.com", "domain3.org"}, filter = nil, limit = 3, need_emails = true, prefix = 'p'},
+ {
+ expect = {"gov.co.net", "tesco.co.net", "domain1.co.net", "domain2.co.net", "domain3.co.net", "domain4.co.net"},
+ filter = (function(s) return s:get_host():sub(-4) == ".net" end),
+ limit = 9999,
+ need_emails = true,
+ prefix = 'p'
+ },
+ {
+ input = {"a.google.com", "b.google.com", "c.google.com", "a.net", "bb.net", "a.bb.net", "b.bb.net"},
+ expect = {"a.bb.net", "b.google.com", "a.net", "bb.net", "a.google.com"},
+ filter = nil,
+ limit = 9999,
+ esld_limit = 2,
+ need_emails = true,
+ prefix = 'p'
+ },
+ {
+ input = {"abc@a.google.com", "b.google.com", "c.google.com", "a.net", "bb.net", "a.bb.net", "b.bb.net"},
+ expect = {"abc@a.google.com", "a.bb.net", "b.google.com", "a.net", "bb.net"},
+ filter = nil,
+ limit = 9999,
+ esld_limit = 2,
+ need_emails = true,
+ prefix = 'p'
+ }
+ }
+
+ local pool = mpool.create()
+
+ local function prepare_url_list(list)
+ return fun.totable(fun.map(
+ function (u) return url.create(pool, u) end,
+ list or url_list
+ ))
+ end
+
+ for i,c in ipairs(cases) do
+ test("extract_specific_urls, backward compatibility case #" .. i, function()
+ task_object.urls = prepare_url_list(c.input)
+ if (c.esld_limit) then
+ -- not awailable in deprecated version
+ return
+ end
+ local actual = util.extract_specific_urls(task_object, c.limit, c.need_emails, c.filter, c.prefix)
+
+ local actual_result = prepare_actual_result(actual)
+
+ --[[
+ local s = logger.slog("%1 =?= %2", c.expect, actual_result)
+ print(s) --]]
+
+ assert_rspamd_table_eq_sorted({actual = actual_result, expect = c.expect})
+ end)
+
+ test("extract_specific_urls " .. i, function()
+ task_object.urls = prepare_url_list(c.input)
+
+ local actual = util.extract_specific_urls({
+ task = task_object,
+ limit = c.limit,
+ esld_limit = c.esld_limit,
+ need_emails = c.need_emails,
+ filter = c.filter,
+ prefix = c.prefix,
+ })
+
+ local actual_result = prepare_actual_result(actual)
+
+ --[[
+ local s = logger.slog("case[%1] %2 =?= %3", i, c.expect, actual_result)
+ print(s) --]]
+
+ assert_rspamd_table_eq_sorted({actual = actual_result, expect = c.expect})
+ end)
+ end
+
+ test("extract_specific_urls, another case", function()
+ task_object.urls = prepare_url_list {"abc.net", "abc.com", "abc.net", "abc.za.org"}
+ local actual = util.extract_specific_urls(task_object, 3, true)
+
+ local actual_result = prepare_actual_result(actual)
+ --[[
+ local s = logger.slog("%1 =?= %2", c.expect, actual_result)
+ print(s) --]]
+
+ local expect = {"abc.com", "abc.net", "abc.za.org"}
+ assert_rspamd_table_eq_sorted({actual = actual_result, expect = expect})
+ end)
+end)
+
+context("Lua util - extract_specific_urls message", function()
+
+--[[ ******************* kinda functional *************************************** ]]
+
+ local test_helper = require "rspamd_test_helper"
+ local cfg = rspamd_util.config_from_ucl(test_helper.default_config(),
+ "INIT_URL,INIT_LIBS,INIT_SYMCACHE,INIT_VALIDATE,INIT_PRELOAD_MAPS")
+ local res,task = rspamd_task.load_from_string(msg, cfg)
+
+ if not res then
+ assert(false, "failed to load message")
+ end
+
+ if not task:process_message() then
+ assert(false, "failed to process message")
+ end
+
+ test("extract_specific_urls - from email 1 limit", function()
+ local actual = util.extract_specific_urls({
+ task = task,
+ limit = 1,
+ esld_limit = 1,
+ })
+
+ local actual_result = prepare_actual_result(actual)
+
+ --[[
+ local s = logger.slog("case[%1] %2 =?= %3", i, expect, actual_result)
+ print(s) --]]
+
+ assert_rspamd_table_eq_sorted({actual = actual_result, expect = {"domain.com"}})
+
+ end)
+ test("extract_specific_urls - from email 2 limit", function()
+ local actual = util.extract_specific_urls({
+ task = task,
+ limit = 2,
+ esld_limit = 1,
+ })
+
+ local actual_result = prepare_actual_result(actual)
+
+ --[[
+ local s = logger.slog("case[%1] %2 =?= %3", i, expect, actual_result)
+ print(s) --]]
+
+ assert_rspamd_table_eq_sorted({actual = actual_result, expect = {"domain.com", "example.net"}})
+
+ end)
+
+ res,task = rspamd_task.load_from_string(msg_img, rspamd_config)
+
+ if not res then
+ assert_true(false, "failed to load message")
+ end
+
+ if not task:process_message() then
+ assert_true(false, "failed to process message")
+ end
+ test("extract_specific_urls - from email image 1 limit", function()
+ local actual = util.extract_specific_urls({
+ task = task,
+ limit = 1,
+ esld_limit = 1,
+ need_images = false,
+ })
+
+ local actual_result = prepare_actual_result(actual)
+
+ --[[
+ local s = logger.slog("case[%1] %2 =?= %3", i, expect, actual_result)
+ print(s) --]]
+
+ assert_rspamd_table_eq_sorted({actual = actual_result, expect = {"domain.com"}})
+
+ end)
+ test("extract_specific_urls - from email image 2 limit", function()
+ local actual = util.extract_specific_urls({
+ task = task,
+ limit = 2,
+ esld_limit = 1,
+ need_images = false,
+ })
+
+ local actual_result = prepare_actual_result(actual)
+
+ --[[
+ local s = logger.slog("case[%1] %2 =?= %3", i, expect, actual_result)
+ print(s) --]]
+
+ assert_rspamd_table_eq_sorted({actual = actual_result, expect = {"domain.com", "example.net"}})
+
+ end)
+ test("extract_specific_urls - from email image 3 limit, no images", function()
+ local actual = util.extract_specific_urls({
+ task = task,
+ limit = 3,
+ esld_limit = 1,
+ need_images = false,
+ })
+
+ local actual_result = prepare_actual_result(actual)
+
+ --[[
+ local s = logger.slog("case[%1] %2 =?= %3", i, expect, actual_result)
+ print(s) --]]
+
+ assert_rspamd_table_eq_sorted({actual = actual_result, expect = {"domain.com", "example.net"}})
+ end)
+ test("extract_specific_urls - from email image 3 limit, has images", function()
+ local actual = util.extract_specific_urls({
+ task = task,
+ limit = 3,
+ esld_limit = 1,
+ need_images = true,
+ })
+
+ local actual_result = prepare_actual_result(actual)
+
+ --[[
+ local s = logger.slog("case[%1] %2 =?= %3", i, expect, actual_result)
+ print(s) --]]
+
+ assert_rspamd_table_eq_sorted({actual = actual_result,
+ expect = {"domain.com", "example.net", "example5.org"}})
+ end)
+ test("extract_specific_urls - from email image 2 limit, has images", function()
+ local actual = util.extract_specific_urls({
+ task = task,
+ limit = 2,
+ esld_limit = 1,
+ need_images = true,
+ })
+
+ local actual_result = prepare_actual_result(actual)
+
+ --[[
+ local s = logger.slog("case[%1] %2 =?= %3", i, expect, actual_result)
+ print(s) --]]
+
+ assert_rspamd_table_eq_sorted({actual = actual_result,
+ expect = {"domain.com", "example.net"}})
+ end)
+end)
diff --git a/test/lua/unit/lua_util.misc.lua b/test/lua/unit/lua_util.misc.lua
new file mode 100644
index 0000000..bab44a3
--- /dev/null
+++ b/test/lua/unit/lua_util.misc.lua
@@ -0,0 +1,61 @@
+local util = require 'lua_util'
+
+context("Lua util - callback_from_string", function()
+ local cases = {
+ {'return function', 'return function(a, b) return a + b end'},
+ {'function', 'function(a, b) return a + b end'},
+ {'plain ops', 'local c = select(1, ...)\nreturn c + select(2, ...)'},
+ }
+ local fail_cases = {
+ nil,
+ '',
+ 'return function(a, b) ( end',
+ 'function(a, b) ( end',
+ 'return a + b'
+ }
+
+ for _,c in ipairs(cases) do
+ test('Success case: ' .. c[1], function()
+ local ret,f = util.callback_from_string(c[2])
+ assert_true(ret, f)
+ assert_equal(f(2, 2), 4)
+ end)
+ end
+ for i,c in ipairs(fail_cases) do
+ test('Failure case: ' .. tostring(i), function()
+ local ret,f = util.callback_from_string(c)
+ assert_false(ret)
+ end)
+ end
+end)
+
+context("Lua util - str_endswith", function()
+ local ending = {
+ {'a', 'a'},
+ {'ab', 'b'},
+ {'ab', 'ab'},
+ {'abc', 'bc'},
+ {'any', ''},
+ }
+ local not_ending = {
+ {'a', 'b'},
+ {'', 'a'},
+ {'ab', 'a'},
+ {'ab', 'ba'},
+ {'ab', 'lab'},
+ {'abc', 'ab'},
+ {'abcd', 'bc'},
+ {'a', 'A'},
+ {'aB', 'b'},
+ }
+ for _, c in ipairs(ending) do
+ test(string.format('True case: str_endswith("%s", "%s")', c[1], c[2]), function()
+ assert_true(util.str_endswith(c[1], c[2]))
+ end)
+ end
+ for _, c in ipairs(not_ending) do
+ test(string.format('False case: str_endswith("%s", "%s")', c[1], c[2]), function()
+ assert_false(util.str_endswith(c[1], c[2]))
+ end)
+ end
+end)
diff --git a/test/lua/unit/mempool.lua b/test/lua/unit/mempool.lua
new file mode 100644
index 0000000..fefd3d2
--- /dev/null
+++ b/test/lua/unit/mempool.lua
@@ -0,0 +1,47 @@
+context("Memory pool unit tests", function()
+ test("Mempool variables", function()
+ local mempool = require "rspamd_mempool"
+
+ local pool = mempool.create()
+
+ assert_not_nil(pool)
+
+ -- string
+ pool:set_variable('a', 'bcd')
+ local var = pool:get_variable('a')
+ assert_equal(var, 'bcd')
+
+ -- integer
+ pool:set_variable('a', 1)
+ var = pool:get_variable('a', 'double')
+ assert_equal(var, 1)
+
+ -- float
+ pool:set_variable('a', 1.01)
+ var = pool:get_variable('a', 'double')
+ assert_equal(var, 1.01)
+
+ -- boolean
+ pool:set_variable('a', false)
+ var = pool:get_variable('a', 'bool')
+ assert_equal(var, false)
+
+ -- multiple
+ pool:set_variable('a', 'bcd', 1, 1.01, false)
+ local v1, v2, v3, v4 = pool:get_variable('a', 'string,double,double,bool')
+ assert_equal(v1, 'bcd')
+ assert_equal(v2, 1)
+ assert_equal(v3, 1.01)
+ assert_equal(v4, false)
+
+ local t = {1,2,3,4,5}
+ pool:set_variable('a', t)
+ local bucket = pool:get_variable('a', 'bucket')
+ assert_rspamd_table_eq({
+ expect = t,
+ actual = bucket
+ })
+
+ pool:destroy()
+ end)
+end) \ No newline at end of file
diff --git a/test/lua/unit/quoted_printable.lua b/test/lua/unit/quoted_printable.lua
new file mode 100644
index 0000000..99a21a1
--- /dev/null
+++ b/test/lua/unit/quoted_printable.lua
@@ -0,0 +1,164 @@
+context("Quoted-Printable encoding", function()
+ local rspamd_util = require "rspamd_util"
+ -- These test cases are derived from https://github.com/mathiasbynens/quoted-printable
+ local cases = {
+ {
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=',
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=3D',
+ 'Exactly 73 chars of which the last one is `=`'
+ },
+ {
+ 'If you believe that truth=beauty, then surely mathematics is the most beautiful branch of philosophy.',
+ 'If you believe that truth=3Dbeauty, then surely mathematics is the most bea=\r\nutiful branch of philosophy.',
+ 'Equals sign'
+ },
+ {
+ 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Typi non habent claritatem insitam; est usus legentis in iis qui facit eorum claritatem. Investigationes demonstraverunt lectores legere me lius quod ii legunt saepius. Claritas est etiam processus dynamicus, qui sequitur mutationem consuetudium lectorum. Mirum est notare quam littera gothica, quam nunc putamus parum claram, anteposuerit litterarum formas humanitatis per seacula quarta decima et quinta decima. Eodem modo typi, qui nunc nobis videntur parum clari, fiant sollemnes in futurum.',
+ 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy =\r\nnibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wi=\r\nsi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lo=\r\nbortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure d=\r\nolor in hendrerit in vulputate velit esse molestie consequat, vel illum dol=\r\nore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio digni=\r\nssim qui blandit praesent luptatum zzril delenit augue duis dolore te feuga=\r\nit nulla facilisi. Nam liber tempor cum soluta nobis eleifend option congue=\r\n nihil imperdiet doming id quod mazim placerat facer possim assum. Typi non=\r\n habent claritatem insitam; est usus legentis in iis qui facit eorum clarit=\r\natem. Investigationes demonstraverunt lectores legere me lius quod ii legun=\r\nt saepius. Claritas est etiam processus dynamicus, qui sequitur mutationem =\r\nconsuetudium lectorum. Mirum est notare quam littera gothica, quam nunc put=\r\namus parum claram, anteposuerit litterarum formas humanitatis per seacula q=\r\nuarta decima et quinta decima. Eodem modo typi, qui nunc nobis videntur par=\r\num clari, fiant sollemnes in futurum.',
+ '76-char line limit',
+ },
+ {
+ 'foo ',
+ 'foo=20',
+ 'Trailing space'
+ },
+ {
+ 'foo\t',
+ 'foo=09',
+ 'Trailing tab'
+ },
+
+ {
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=',
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=\r\n=3D',
+ 'Exactly 74 chars of which the last one is `=`'
+ },
+ {
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=',
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=\r\n=3D',
+ 'Exactly 75 chars of which the last one is `=`'
+ },
+ {
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=',
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=\r\n=3D',
+ 'Exactly 76 chars of which the last one is `=`',
+ },
+ {
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=',
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=\r\nx=3D',
+ 'Exactly 77 chars of which the last one is `=`'
+ },
+ {
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ',
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=20',
+ 'Exactly 73 chars of which the last one is a space'
+ },
+ {
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ',
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=20',
+ 'Exactly 74 chars of which the last one is a space'
+ },
+ {
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ',
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx =\r\n',
+ 'Exactly 75 chars of which the last one is a space'
+ },
+ {
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ',
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=\r\n=20',
+ 'Exactly 76 chars of which the last one is a space'
+ },
+ {
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ',
+ 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=\r\nx=20',
+ 'Exactly 77 chars of which the last one is a space'
+ },
+ {
+ 'fdafadsf\r\n-- • Test\r\n',
+ 'fdafadsf\r\n-- =E2=80=A2 Test\r\n',
+ 'Newlines',
+ },
+ }
+ for _,c in ipairs(cases) do
+ test("QP sanity test case: " .. c[3], function()
+ local res = {
+ expect = c[1],
+ actual = tostring(rspamd_util.decode_qp((rspamd_util.encode_qp(c[1], 76))))
+ }
+ assert_rspamd_eq(res)
+ end)
+ test("QP encoding test case: " .. c[3], function()
+ local res = {
+ expect = c[2],
+ actual = tostring(rspamd_util.encode_qp(c[1], 76))
+ }
+ assert_rspamd_eq(res)
+ end)
+ end
+ -- Decode issues
+ cases = {
+ {
+ 'Mailscape External Mail Flow Outbound Test=',
+ 'Mailscape External Mail Flow Outbound Test=',
+ 'asan found'
+ },
+ {
+ 'foo=\n\nbar',
+ 'foo\nbar',
+ 'Soft newline followed by hard newline (LF)',
+ },
+ {
+ 'foo=\r\n\r\nbar',
+ 'foo\r\nbar',
+ 'Soft newline followed by hard newline (CRLF)',
+ },
+ {
+ '=gB',
+ '=gB',
+ 'Second character is okay, the first character is garbage'
+ },
+ {
+ '=bG',
+ '=bG',
+ 'First character okay, the second character is rubbish'
+ }
+ }
+
+ for _,c in ipairs(cases) do
+ test("QP decoding test case: " .. c[3], function()
+ local res = {
+ expect = c[2],
+ actual = tostring(rspamd_util.decode_qp(c[1]))
+ }
+ assert_rspamd_eq(res)
+ end)
+ end
+
+
+ if os.getenv("RSPAMD_LUA_EXPENSIVE_TESTS") then
+ -- Fuzz testing
+ local charset = {}
+ for i = 0, 255 do table.insert(charset, string.char(i)) end
+
+ local function random_string(length)
+
+ if length > 0 then
+ return random_string(length - 1) .. charset[math.random(1, #charset)]
+ else
+ return ""
+ end
+ end
+ for _,l in ipairs({10, 100, 1000, 10000}) do
+ test("QP fuzz test max length " .. tostring(l), function()
+ for _=1,100 do
+ local inp = random_string(math.random() * l + 1)
+ local res = {
+ expect = inp,
+ actual = tostring(rspamd_util.decode_qp((rspamd_util.encode_qp(inp, 0))))
+ }
+ assert_rspamd_eq(res)
+ end
+ end)
+ end
+ end
+end)
diff --git a/test/lua/unit/regxep.lua b/test/lua/unit/regxep.lua
new file mode 100644
index 0000000..a27e7b3
--- /dev/null
+++ b/test/lua/unit/regxep.lua
@@ -0,0 +1,90 @@
+context("Regexp unit tests", function()
+ local re = require("rspamd_regexp")
+
+ test("Regexp creation", function()
+ assert_not_nil(re.create_cached('/test$/m'))
+ assert_not_nil(re.create_cached('^test$', 'm'))
+ assert_not_nil(re.create_cached('m,test,m'))
+ assert_not_nil(re.create_cached('m|test|m'))
+ end)
+ test("Regexp match", function()
+ local cases = {
+ {'/ТеÑÑ‚/iu', 'теÑÑ‚', true},
+ {'/test$/m', '123test', true},
+ {'/^test$/m', '123test', false},
+ {'m,test,', 'test', true},
+ {'m,test,', 'test123', false},
+ {'m{https?://[^/?\\s]+?:\\d+(?<!:80)(?<!:443)(?<!:8080)(?:/|\\s|$)}', '', false},
+ {'/test/i', 'TeSt123', true},
+ -- Raw regexp
+ {'/\\S<[-\\w\\.]+\\@[-\\w\\.]+>/r', 'some<example@example.com>', true},
+ -- Cyrillic utf8 letter
+ {'/\\S<[-\\w\\.]+\\@[-\\w\\.]+>/r', 'some<example@exаmple.com>', false},
+ }
+
+ for _,c in ipairs(cases) do
+ local r = re.create_cached(c[1])
+ assert_not_nil(r, "cannot parse " .. c[1])
+ local res = r:match(c[2])
+
+ assert_equal(res, c[3], string.format("'%s' doesn't match with '%s'",
+ c[2], c[1]))
+ end
+ end)
+
+ test("Regexp capture", function()
+ local cases = {
+ {'Body=(\\S+)(?: Fuz1=(\\S+))?(?: Fuz2=(\\S+))?',
+ 'mc-filter4 1120; Body=1 Fuz1=2 Fuz2=3',
+ {'Body=1 Fuz1=2 Fuz2=3', '1', '2', '3'}},
+ {'Body=(\\S+)(?: Fuz1=(\\S+))?(?: Fuz2=(\\S+))?',
+ 'mc-filter4 1120; Body=1 Fuz1=2', {'Body=1 Fuz1=2', '1', '2'}},
+ {'Body=(\\S+)(?: Fuz1=(\\S+))?(?: Fuz2=(\\S+))?',
+ 'mc-filter4 1120; Body=1 Fuz1=2 mc-filter4 1120; Body=1 Fuz1=2 Fuz2=3',
+ {'Body=1 Fuz1=2', '1', '2'}, {'Body=1 Fuz1=2 Fuz2=3', '1', '2', '3'}},
+ }
+ for _,c in ipairs(cases) do
+ local r = re.create_cached(c[1])
+ assert_not_nil(r, "cannot parse " .. c[1])
+ local res = r:search(c[2], false, true)
+
+ assert_not_nil(res, "cannot find pattern")
+
+ for k = 3, table.maxn(c) do
+ for n,m in ipairs(c[k]) do
+ assert_equal(res[k - 2][n], c[k][n], string.format("'%s' doesn't match with '%s'",
+ c[k][n], res[k - 2][n]))
+ end
+ end
+ end
+ end)
+
+ test("Regexp split", function()
+ local cases = {
+ {'\\s', 'one', {'one'}}, -- one arg
+ {'\\s', 'one two', {'one', 'two'}}, -- trivial
+ {'/,/i', '1,2', {'1', '2'}}, -- trivial
+ {'\\s', 'one two', {'one', 'two'}}, -- multiple delimiters
+ {'\\s', ' one two ', {'one', 'two'}}, -- multiple delimiters
+ {'\\s', ' one ', {'one'}}, -- multiple delimiters
+ {'[:,]', ',,,:::one,two,,', {'one', 'two'}}, -- multiple delimiters
+ {'[\\|\\s]', '16265 | 1.1.1.0/22 | TR | ripencc | 2014-02-28',
+ {'16265', '1.1.1.0/22', 'TR', 'ripencc', '2014-02-28'}}, -- practical
+ {'|', '16265 | 1.1.1.0/22 | TR | ripencc | 2014-02-28', {}} -- bad re
+ }
+
+ for _,c in ipairs(cases) do
+ local r = re.create_cached(c[1])
+ assert_not_nil(r, "cannot parse " .. c[1])
+
+ local res = r:split(c[2])
+ assert_not_nil(res, "cannot split " .. c[2])
+
+ for i,r in ipairs(c[3]) do
+ assert_equal(res[i], r)
+ end
+ end
+ end)
+
+ end
+) \ No newline at end of file
diff --git a/test/lua/unit/rfc2047.lua b/test/lua/unit/rfc2047.lua
new file mode 100644
index 0000000..658f202
--- /dev/null
+++ b/test/lua/unit/rfc2047.lua
@@ -0,0 +1,92 @@
+--[[
+Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+]]--
+
+context("RFC2047 decoding", function()
+ local ffi = require("ffi")
+
+ ffi.cdef[[
+ const char * rspamd_mime_header_decode (void *pool, const char *in, size_t inlen);
+ void * rspamd_mempool_new_ (size_t sz, const char *name, int flags, const char *strloc);
+ void rspamd_mempool_delete (void *pool);
+ ]]
+
+ test("Decode rfc2047 tokens", function()
+ -- Test -> expected
+ local cases = {
+ {"=?US-ASCII*EN?Q?Keith_Moore?= <moore@cs.utk.edu>", "Keith Moore <moore@cs.utk.edu>"},
+ {[[=?windows-1251?Q?=C2=FB_=F1=EC=EE=E6=E5=F2=E5_=F5=E0=F0?=
+ =?windows-1251?Q?=E0=EA=F2=E5=F0=E8=E7=EE=E2=E0=F2=FC=F1?=
+ =?windows-1251?Q?=FF_=E7=EE=F0=EA=E8=EC_=E7=F0=E5=ED=E8?=
+ =?windows-1251?Q?=E5=EC?=]], "Ð’Ñ‹ Ñможете характеризоватьÑÑ Ð·Ð¾Ñ€ÐºÐ¸Ð¼ зрением"},
+ {'v=1; a=rsa-sha256; c=relaxed/relaxed; d=yoni.za.org; s=testdkim1;',
+ 'v=1; a=rsa-sha256; c=relaxed/relaxed; d=yoni.za.org; s=testdkim1;'},
+ {"=?windows-1251?B?xO7q8+zl7fIuc2NyLnV1ZQ==?=", "Документ.scr.uue"},
+ {"=?UTF-8?Q?=20wie=20ist=20es=20Ihnen=20ergangen?.pdf?=", " wie ist es Ihnen ergangen?.pdf"}, -- ? inside
+ {"=?UTF-8?Q?=20wie=20ist=20es=20Ihnen=20ergangen??=", " wie ist es Ihnen ergangen?"}, -- ending ? inside
+ }
+
+ local pool = ffi.C.rspamd_mempool_new_(4096, "lua", 0, "rfc2047.lua:49")
+
+ for _,c in ipairs(cases) do
+ local res = ffi.C.rspamd_mime_header_decode(pool, c[1], #c[1])
+ res = ffi.string(res)
+ assert_not_nil(res, "cannot decode " .. c[1])
+ assert_rspamd_eq({actual = res, expect = c[2]})
+
+ end
+
+ ffi.C.rspamd_mempool_delete(pool)
+ end)
+ if os.getenv("RSPAMD_LUA_EXPENSIVE_TESTS") then
+ test("Fuzz test for rfc2047 tokens", function()
+ local util = require("rspamd_util")
+ local pool = ffi.C.rspamd_mempool_new_(4096, "lua", 0, "rfc2047.lua:63")
+ local str = "ТеÑÑ‚ ТеÑÑ‚ ТеÑÑ‚ ТеÑÑ‚ ТеÑÑ‚"
+
+ for _ = 0,1000 do
+ local r1 = math.random()
+ local r2 = math.random()
+ local sl1 = #str / 2.0 * r1
+ local sl2 = #str / 2.0 * r2
+
+ local s1 = tostring(util.encode_base64(string.sub(str, 1, sl1)))
+ local s2 = tostring(util.encode_base64(string.sub(str, sl1 + 1, sl2)))
+ local s3 = tostring(util.encode_base64(string.sub(str, sl2 + 1)))
+
+ if #s1 > 0 and #s2 > 0 and #s3 > 0 then
+ local s = string.format('=?UTF-8?B?%s?= =?UTF-8?B?%s?= =?UTF-8?B?%s?=',
+ s1, s2, s3)
+ local res = ffi.C.rspamd_mime_header_decode(pool, s, #s)
+ res = ffi.string(res)
+ assert_not_nil(res, "cannot decode " .. s)
+ assert_rspamd_eq({actual = res, expect = str})
+ end
+ end
+
+ ffi.C.rspamd_mempool_delete(pool)
+ end)
+ end
+end)
diff --git a/test/lua/unit/rsa.lua b/test/lua/unit/rsa.lua
new file mode 100644
index 0000000..c67a36a
--- /dev/null
+++ b/test/lua/unit/rsa.lua
@@ -0,0 +1,50 @@
+-- Test rsa signing
+
+context("RSA signature verification test", function()
+ local rsa_privkey = require "rspamd_rsa_privkey"
+ local rsa_pubkey = require "rspamd_rsa_pubkey"
+ local rsa_signature = require "rspamd_rsa_signature"
+ local rsa = require "rspamd_rsa"
+ local hash = require "rspamd_cryptobox_hash"
+ local pubkey = 'testkey.pub'
+ local privkey = 'testkey.sec'
+ local data = 'test.data'
+ local signature = 'test.sig'
+ local test_dir = string.gsub(debug.getinfo(1).source, "^@(.+/)[^/]+$", "%1")
+ local rsa_key, rsa_sig
+
+ test("RSA sign", function()
+ -- Signing test
+ rsa_key = rsa_privkey.load_file(string.format('%s/%s', test_dir, privkey))
+ assert_not_nil(rsa_key)
+
+ local h = hash.create_specific('sha256')
+ local d = io.open(string.format('%s/%s', test_dir, data), "rb"):read "*a"
+ h:update(d)
+ local sig = rsa.sign_memory(rsa_key, h:bin())
+ assert_not_nil(sig)
+ sig:save(string.format('%s/%s', test_dir, signature), true)
+ end)
+
+ test("RSA verify", function()
+ -- Verifying test
+ local h = hash.create_specific('sha256')
+ local d = io.open(string.format('%s/%s', test_dir, data), "rb"):read "*a"
+ h:update(d)
+ rsa_key = rsa_pubkey.load(string.format('%s/%s', test_dir, pubkey))
+ assert_not_nil(rsa_key)
+ rsa_sig = rsa_signature.load(string.format('%s/%s', test_dir, signature))
+ assert_not_nil(rsa_sig)
+ assert_true(rsa.verify_memory(rsa_key, rsa_sig, h:bin()))
+ end)
+
+ test("RSA keypair + sign + verify", function()
+ local sk, pk = rsa.keypair()
+ local sig = rsa.sign_memory(sk, "test")
+ assert_true(rsa.verify_memory(pk, sig, "test"))
+ assert_false(rsa.verify_memory(pk, sig, "test1"))
+ -- Overwrite
+ sk, pk = rsa.keypair()
+ assert_false(rsa.verify_memory(pk, sig, "test"))
+ end)
+end)
diff --git a/test/lua/unit/rspamd_resolver.lua b/test/lua/unit/rspamd_resolver.lua
new file mode 100644
index 0000000..e987ff0
--- /dev/null
+++ b/test/lua/unit/rspamd_resolver.lua
@@ -0,0 +1,31 @@
+-- Rspamd resolver Lua tests
+
+context("Check punycoding UTF-8 URL", function()
+ local rspamd_resolver = require "rspamd_resolver"
+ local rspamd_util = require "rspamd_util"
+
+ local resolver = rspamd_resolver.init(rspamd_util.create_event_base(), rspamd_config)
+
+ local cases = {
+ -- https://unicode.org/reports/tr46/#Deviations
+ ['faß.de'] = 'fass.de', -- IDNA2008 result: xn--fa-hia.de
+ ['βόλος.com'] = 'xn--nxasmq6b.com', -- IDNA2008 result: xn--nxasmm1c.com
+ ['نامه‌ای.com'] = 'xn--mgba3gch31f.com', -- IDNA2008 result: xn--mgba3gch31f060k.com
+ ['à·à·Šâ€à¶»à·“.com'] = 'xn--10cl1a0b.com', -- IDNA2008 result: xn--10cl1a0b660p.com
+
+ -- https://unicode.org/reports/tr46/#Table_Example_Processing
+ ['日本語。JP'] = 'xn--wgv71a119e.jp', -- Fullwidth characters are remapped, including 。
+ --['u¨.com'] = 'xn--tda.com', -- Normalize changes u + umlaut to ü
+ ['☕.us'] = 'xn--53h.us', -- Post-Unicode 3.2 characters are allowed
+
+ -- Other
+ ['example.рф'] = 'example.xn--p1ai',
+ }
+
+ for k, v in pairs(cases) do
+ test(string.format("punycode %s -> %s", k, v), function()
+ local res = resolver:idna_convert_utf8(k)
+ assert_equal(res, v)
+ end)
+ end
+end)
diff --git a/test/lua/unit/rspamd_text.lua b/test/lua/unit/rspamd_text.lua
new file mode 100644
index 0000000..d643d9e
--- /dev/null
+++ b/test/lua/unit/rspamd_text.lua
@@ -0,0 +1,79 @@
+context("Rspamd_text:byte() test", function()
+ local rspamd_text = require "rspamd_text"
+
+ local str = 'OMG'
+ local txt = rspamd_text.fromstring(str)
+ local fmt = 'case rspamd_text:byte(%s,%s)'
+ local cases = {
+ {'1', 'nil'},
+ {'nil', '1'},
+ }
+
+ for start = -4, 4 do
+ for stop = -4, 4 do
+ table.insert(cases, {tostring(start), tostring(stop)})
+ end
+ end
+
+ for _, case in ipairs(cases) do
+ local name = string.format(fmt, case[1], case[2])
+ test(name, function()
+ local txt_bytes = {txt:byte(tonumber(case[1]), tonumber(case[2]))}
+ local str_bytes = {str:byte(tonumber(case[1]), tonumber(case[2]))}
+ assert_rspamd_table_eq({
+ expect = str_bytes,
+ actual = txt_bytes
+ })
+ end)
+ end
+end)
+
+context("Rspamd_text:find() test", function()
+ local rspamd_text = require "rspamd_text"
+
+ local cases = {
+ {{'foobarfoo', 'f'}, {1, 1}},
+ {{'foobarfoo', 'foo'}, {1, 3}},
+ {{'foobarfoo', 'bar'}, {4, 6}},
+ {{'foobarfoo', 'baz'}, nil},
+ {{'foobarfoo', 'rfoo'}, {6, 9}},
+ {{'foo', 'bar'}, nil},
+ {{'x', 'xxxx'}, nil},
+ {{'', ''}, {1, 0}},
+ {{'', '_'}, nil},
+ {{'x', ''}, {1, 0}},
+ }
+
+ for _, case in ipairs(cases) do
+ local name = string.format('case rspamd_text:find(%s,%s)', case[1][1], case[1][2])
+ test(name, function()
+ local t = rspamd_text.fromstring(case[1][1])
+ local s,e = t:find(case[1][2])
+
+ if case[2] then
+ assert_rspamd_table_eq({
+ expect = case[2],
+ actual = {s, e}
+ })
+ else
+ assert_nil(s)
+ end
+ local ss,ee = string.find(case[1][1], case[1][2], 1, true)
+ assert_rspamd_table_eq({
+ expect = { ss, ee },
+ actual = { s, e }
+ })
+ end)
+ -- Compare with vanila lua
+ name = string.format('case lua string vs rspamd_text:find(%s,%s)', case[1][1], case[1][2])
+ test(name, function()
+ local t = rspamd_text.fromstring(case[1][1])
+ local s,e = t:find(case[1][2])
+ local ss,ee = string.find(case[1][1], case[1][2], 1, true)
+ assert_rspamd_table_eq({
+ expect = { ss, ee },
+ actual = { s, e }
+ })
+ end)
+ end
+end)
diff --git a/test/lua/unit/rspamd_util.lua b/test/lua/unit/rspamd_util.lua
new file mode 100644
index 0000000..56f13d6
--- /dev/null
+++ b/test/lua/unit/rspamd_util.lua
@@ -0,0 +1,136 @@
+context("Rspamd util for lua - check generic functions", function()
+ local util = require 'rspamd_util'
+
+
+ local cases = {
+ {
+ input = "test1",
+ result = false,
+ mixed_script = false,
+ range_start = 0x0000,
+ range_end = 0x017f
+ },
+ {
+ input = "test test xxx",
+ result = false,
+ mixed_script = false,
+ range_start = 0x0000,
+ range_end = 0x017f
+ },
+ {
+ input = "ÐбЫрвÐлг",
+ result = true,
+ mixed_script = false,
+ range_start = 0x0000,
+ range_end = 0x017f
+ },
+ {
+ input = "ÐбЫрвÐлг example",
+ result = true,
+ mixed_script = true,
+ range_start = 0x0000,
+ range_end = 0x017f
+ },
+ {
+ input = "example ąłśćżłóę",
+ result = false,
+ mixed_script = false,
+ range_start = 0x0000,
+ range_end = 0x017f
+ },
+ {
+ input = "ąłśćżłóę ÐбЫрвÐлг",
+ result = true,
+ mixed_script = true,
+ range_start = 0x0000,
+ range_end = 0x017f
+ },
+ }
+
+ for i,c in ipairs(cases) do
+ test("is_utf_outside_range, test case #" .. i, function()
+ local actual = util.is_utf_outside_range(c.input, c.range_start, c.range_end)
+
+ assert_equal(c.result, actual)
+ end)
+ end
+
+ test("is_utf_outside_range, check cache", function ()
+ cache_size = 20
+ for i = 1,cache_size do
+ local res = util.is_utf_outside_range("a", 0x0000, 0x0000+i)
+ end
+ end)
+
+ test("is_utf_outside_range, check empty string", function ()
+ assert_error(util.is_utf_outside_range)
+ end)
+
+ test("get_string_stats, test case", function()
+ local res = util.get_string_stats("this is test 99")
+ assert_equal(res["letters"], 10)
+ assert_equal(res["digits"], 2)
+ end)
+
+ for i,c in ipairs(cases) do
+ test("is_utf_mixed_script, test case #" .. i, function()
+ local actual = util.is_utf_mixed_script(c.input)
+
+ assert_equal(c.mixed_script, actual)
+ end)
+ end
+
+ test("is_utf_mixed_script, invalid utf str should return errror", function()
+ assert_error(util.is_utf_mixed_script,'\200\213\202')
+ end)
+
+ test("is_utf_mixed_script, empty str should return errror", function()
+ assert_error(util.is_utf_mixed_script,'\200\213\202')
+ end)
+end)
+
+context("Rspamd string utility", function()
+ local ffi = require 'ffi'
+
+ ffi.cdef[[
+char ** rspamd_string_len_split (const char *in, size_t len,
+ const char *spill, int max_elts, void *pool);
+ void g_strfreev (char **str_array);
+]]
+ local NULL = ffi.new 'void*'
+ local cases = {
+ {'', ';,', {}},
+ {'', '', {}},
+ {'a', ';,', {'a'}},
+ {'a', '', {'a'}},
+ {'a;b', ';', {'a', 'b'}},
+ {'a;;b', ';', {'a', 'b'}},
+ {';a;;b;', ';', {'a', 'b'}},
+ {'ab', ';', {'ab'}},
+ {'a,;b', ',', {'a', ';b'}},
+ {'a,;b', ';,', {'a', 'b'}},
+ {',a,;b', ';,', {'a', 'b'}},
+ {',,;', ';,', {}},
+ {',,;a', ';,', {'a'}},
+ {'a,,;', ';,', {'a'}},
+ }
+
+ for i,case in ipairs(cases) do
+ test("rspamd_string_len_split: case " .. tostring(i), function()
+ local ret = ffi.C.rspamd_string_len_split(case[1], #case[1],
+ case[2], -1, NULL)
+ local actual = {}
+
+ while ret[#actual] ~= NULL do
+ actual[#actual + 1] = ffi.string(ret[#actual])
+ end
+
+ assert_rspamd_table_eq({
+ expect = case[3],
+ actual = actual
+ })
+
+ ffi.C.g_strfreev(ret)
+ end)
+ end
+end) \ No newline at end of file
diff --git a/test/lua/unit/selectors.combined.lua b/test/lua/unit/selectors.combined.lua
new file mode 100644
index 0000000..2c1aa08
--- /dev/null
+++ b/test/lua/unit/selectors.combined.lua
@@ -0,0 +1,130 @@
+local msg
+context("Selectors test", function()
+ local rspamd_task = require "rspamd_task"
+ local logger = require "rspamd_logger"
+ local lua_selectors = require "lua_selectors"
+ local test_helper = require "rspamd_test_helper"
+ local cfg = rspamd_config
+ local task
+
+ test_helper.init_url_parser()
+
+ before(function()
+ local res
+ res,task = rspamd_task.load_from_string(msg, cfg)
+ task:set_from_ip("198.172.22.91")
+ task:set_user("cool user name")
+ task:set_helo("hello mail")
+ task:set_request_header("hdr1", "value1")
+ task:process_message()
+ task:get_mempool():set_variable("int_var", 1)
+ task:get_mempool():set_variable("str_var", "str 1")
+ if not res then
+ assert_true(false, "failed to load message")
+ end
+ end)
+
+ local function check_selector(selector_string)
+ local sels = lua_selectors.parse_selector(cfg, selector_string)
+ local elts = lua_selectors.process_selectors(task, sels)
+ local res = lua_selectors.combine_selectors(task, elts, ':')
+ return res
+ end
+
+ local cases = {
+ ["rcpts + weekend"] = {
+ selector = "rcpts:addr.take_n(5).lower;time('message', '!%w').in(6, 7).id('weekends')",
+ expect = {
+ "nobody@example.com:weekends",
+ "no-one@example.com:weekends"}},
+
+ ["weekend + rcpts"] = {
+ selector = "time('message', '!%w').in(6, 7).id('weekends');rcpts:addr.take_n(5).lower",
+ expect = {
+ "weekends:nobody@example.com",
+ "weekends:no-one@example.com"}},
+
+ ["id(rcpt) + rcpts + weekend"] = {
+ selector = "id('rcpt');rcpts:addr.take_n(5).lower;time('message', '!%w').in(6, 7).id('weekends')",
+ expect = {
+ "rcpt:nobody@example.com:weekends",
+ "rcpt:no-one@example.com:weekends"}},
+
+ ["id(rcpt) + id(2) rcpts + weekend"] = {
+ selector = "id('rcpt'); id(2); rcpts:addr.take_n(5).lower; time('message', '!%w').in(6, 7).id('weekends')",
+ expect = {
+ "rcpt:2:nobody@example.com:weekends",
+ "rcpt:2:no-one@example.com:weekends"}
+ },
+
+ -- There are two rcpts but only one url in the message
+ -- resulting table size is the size of the smallest table
+ ["id(rcpt) + id(2) + rcpts and urls + weekend"] = {
+ selector = "id('rcpt'); id(2); rcpts:addr.take_n(5).lower; id('urls'); urls:get_host; time('message', '!%w').in(6, 7).id('weekends')",
+ expect = { "rcpt:2:nobody@example.com:urls:example.net:weekends"}
+ },
+ ["url + apply_methods"] = {
+ selector = "urls.apply_methods('get_host', 'get_path').join_tables('/')",
+ expect = {"example.net/path"}
+ },
+ }
+
+ for case_name, case in pairs(cases) do
+ test("case " .. case_name, function()
+ local elts = check_selector(case.selector)
+ assert_not_nil(elts)
+ assert_rspamd_table_eq({actual = elts, expect = case.expect})
+ end)
+ end
+end)
+
+
+--[=========[ ******************* message ******************* ]=========]
+msg = [[
+Received: from ca-18-193-131.service.infuturo.it ([151.18.193.131] helo=User)
+ by server.chat-met-vreemden.nl with esmtpa (Exim 4.76)
+ (envelope-from <upwest201diana@outlook.com>)
+ id 1ZC1sl-0006b4-TU; Mon, 06 Jul 2015 10:36:08 +0200
+From: <whoknows@nowhere.com>
+To: <nobody@example.com>, <no-one@example.com>
+Date: Sat, 22 Sep 2018 14:36:51 +0100 (BST)
+subject: Second, lower-cased header subject
+Subject: Test subject
+Content-Type: multipart/alternative;
+ boundary="_000_6be055295eab48a5af7ad4022f33e2d0_"
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: base64
+
+Hello world
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/html; charset="utf-8"
+
+<html><body>
+<a href="http://example.net/path?query">http://example.net/path?query</a>
+<a href="mailto:test@example.net">mail me</a>
+</html>
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: application/zip; name=f.zip
+Content-Disposition: attachment; size=166; filename=f.zip
+Content-Transfer-Encoding: base64
+
+UEsDBAoAAAAAAINe6kgAAAAAAAAAAAAAAAAIABwAZmFrZS5leGVVVAkAA8YaglfGGoJXdXgLAAEE
+6AMAAAToAwAAUEsBAh4DCgAAAAAAg17qSAAAAAAAAAAAAAAAAAgAGAAAAAAAAAAAALSBAAAAAGZh
+a2UuZXhlVVQFAAPGGoJXdXgLAAEE6AMAAAToAwAAUEsFBgAAAAABAAEATgAAAEIAAAAAAA==
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: application/zip; name=f.zip
+Content-Disposition: attachment; size=166; filename=f2.zip
+Content-Transfer-Encoding: base64
+
+UEsDBAoAAAAAAINe6kgAAAAAAAAAAAAAAAAIABwAZmFrZS5leGVVVAkAA8YaglfGGoJXdXgLAAEE
+6AMAAAToAwAAUEsBAh4DCgAAAAAAg17qSAAAAAAAAAAAAAAAAAgAGAAAAAAAAAAAALSBAAAAAGZh
+a2UuZXhlVVQFAAPGGoJXdXgLAAEE6AMAAAToAwAAUEsFBgAAAAABAAEATgAAAEIAAAAAAA==
+]]
diff --git a/test/lua/unit/selectors.custom.lua b/test/lua/unit/selectors.custom.lua
new file mode 100644
index 0000000..cf82fe6
--- /dev/null
+++ b/test/lua/unit/selectors.custom.lua
@@ -0,0 +1,81 @@
+local msg
+context("Selectors test", function()
+ local rspamd_task = require "rspamd_task"
+ local logger = require "rspamd_logger"
+ local lua_selectors = require "lua_selectors"
+ local test_helper = require "rspamd_test_helper"
+ local cfg = rspamd_config
+ local task
+
+ test_helper.init_url_parser()
+
+ before(function()
+ local res
+ res,task = rspamd_task.load_from_string(msg, cfg)
+ if not res then
+ assert_true(false, "failed to load message")
+ end
+ end)
+
+ local function check_selector(selector_string)
+ local sels = lua_selectors.parse_selector(cfg, selector_string)
+ local elts = lua_selectors.process_selectors(task, sels)
+ return elts
+ end
+
+ test("custom selector", function()
+ lua_selectors.register_extractor(rspamd_config, "get_something", {
+ get_value = function(task, args) -- mandatory field
+ return 'simple value','string' -- result + type
+ end,
+ description = 'Sample extractor' -- optional
+ })
+
+ local elts = check_selector('get_something')
+ assert_not_nil(elts)
+ assert_rspamd_table_eq({actual = elts, expect = {'simple value'}})
+ end)
+
+ test("custom transform", function()
+ lua_selectors.register_extractor(rspamd_config, "get_something", {
+ get_value = function(task, args) -- mandatory field
+ return 'simple value','string' -- result + type
+ end,
+ description = 'Sample extractor' -- optional
+ })
+
+ lua_selectors.register_transform(rspamd_config, "append_string", {
+ types = {['string'] = true}, -- accepted types
+ process = function(input, type, args)
+ return input .. table.concat(args or {}),'string' -- result + type
+ end,
+ map_type = 'string', -- can be used in map like invocation, always return 'string' type
+ description = 'Adds all arguments to the input string'
+ })
+
+ local elts = check_selector('get_something.append_string(" and a simple tail")')
+ assert_not_nil(elts)
+ assert_rspamd_table_eq({actual = elts, expect = {'simple value and a simple tail'}})
+
+ local elts = check_selector('get_something.append_string(" and", " a", " simple", " nail")')
+ assert_not_nil(elts)
+ assert_rspamd_table_eq({actual = elts, expect = {'simple value and a simple nail'}})
+ end)
+end)
+
+
+--[=========[ ******************* message ******************* ]=========]
+msg = [[
+From: <whoknows@nowhere.com>
+To: <nobody@example.com>, <no-one@example.com>
+Date: Wed, 19 Sep 2018 14:36:51 +0100 (BST)
+Subject: Test subject
+Content-Type: multipart/alternative;
+ boundary="_000_6be055295eab48a5af7ad4022f33e2d0_"
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: base64
+
+Hello world
+]]
diff --git a/test/lua/unit/selectors.lua b/test/lua/unit/selectors.lua
new file mode 100644
index 0000000..6362e5c
--- /dev/null
+++ b/test/lua/unit/selectors.lua
@@ -0,0 +1,472 @@
+local msg
+context("Selectors test", function()
+ local rspamd_task = require "rspamd_task"
+ local logger = require "rspamd_logger"
+ local lua_selectors = require "lua_selectors"
+ local lua_maps = require "lua_maps"
+ local test_helper = require "rspamd_test_helper"
+ local lua_util = require "lua_util"
+ local cfg = rspamd_config
+ local task
+
+ test_helper.init_url_parser()
+
+ lua_selectors.maps.test_map = lua_maps.map_add_from_ucl({
+ 'key value',
+ 'key1 value1',
+ 'key3 value1',
+ }, 'hash', 'test selectors maps')
+
+ before(function()
+ local res
+ res,task = rspamd_task.load_from_string(msg, cfg)
+ task:set_from_ip("198.172.22.91")
+ task:set_user("cool user name")
+ task:set_helo("hello mail")
+ task:set_request_header("hdr1", "value1")
+ task:process_message()
+ task:get_mempool():set_variable("int_var", 1)
+ task:get_mempool():set_variable("str_var", "str 1")
+ task:cache_set('cachevar1', 'hello\x00world')
+ task:cache_set('cachevar2', {'hello', 'world'})
+ if not res then
+ assert_true(false, "failed to load message")
+ end
+ end)
+
+ local function check_selector_plain(selector_string)
+ local sels = lua_selectors.create_selector_closure_fn(nil, cfg, selector_string, nil,
+ function(_, res, _) return res end)
+ local elts = sels(task)
+ return elts
+ end
+
+ local function check_selector_kv(selector_string)
+ local sels = lua_selectors.create_selector_closure_fn(nil, cfg, selector_string, nil,
+ lua_selectors.kv_table_from_pairs)
+ local elts = sels(task)
+ return elts
+ end
+
+ local cases_plain = {
+ ["ip"] = {
+ selector = "ip",
+ expect = {"198.172.22.91"}
+ },
+
+ ["header Subject"] = {
+ selector = "header(Subject)",
+ expect = {"Second, lower-cased header subject"}
+ },
+
+ ["header Subject lower"] = {
+ selector = "header(Subject).lower",
+ expect = {"second, lower-cased header subject"}
+ },
+
+ ["header Subject lower_utf8"] = {
+ selector = "header(Subject).lower_utf8",
+ expect = {"second, lower-cased header subject"}
+ },
+
+ ["header full Subject lower"] = {
+ selector = "header(Subject, 'full').lower",
+ expect = {{"second, lower-cased header subject", "test subject"}}
+ },
+
+ ["header full strong Subject"] = {
+ selector = "header(Subject, 'full,strong')",
+ expect = {{"Test subject"}}
+ },
+
+ ["header full strong lower-cased Subject"] = {
+ selector = "header(subject, 'full,strong')",
+ expect = {{"Second, lower-cased header subject"}}
+ },
+
+ ["digest"] = {
+ selector = "digest",
+ expect = {"1ac109c58a7d0f5f532100ac14e9f4d9"}
+ },
+
+ ["user"] = {
+ selector = "user",
+ expect = {"cool user name"}
+ },
+
+ ["from"] = {
+ selector = "from",
+ expect = {"whoknows@nowhere.com"}
+ },
+
+ ["rcpts"] = {
+ selector = "rcpts",
+ expect = {{"nobody@example.com", "no-one@example.com"}}
+ },
+
+ ["1st rcpts"] = {
+ selector = "rcpts.nth(1)",
+ expect = {"nobody@example.com"}
+ },
+
+ ["lower rcpts"] = {
+ selector = "rcpts.lower.first",
+ expect = {"nobody@example.com"}
+ },
+
+ ["first rcpts"] = {
+ selector = "rcpts.first",
+ expect = {"nobody@example.com"}
+ },
+
+ ["first addr rcpts"] = {
+ selector = "rcpts:addr.first",
+ expect = {"nobody@example.com"}
+ },
+
+ ["rcpts_uniq_domains"] = {
+ selector = "rcpts:domain.uniq",
+ expect = {{"example.com"}}
+ },
+
+ ["rcpts_sorted"] = {
+ selector = "rcpts:addr.sort",
+ expect = {{"nobody@example.com", "no-one@example.com"}}
+ },
+
+ ["to"] = {
+ selector = "to",
+ expect = {"nobody@example.com"}},
+
+ ["attachments"] = {
+ selector = "attachments",
+ expect = {{"ce112d07c52ae649f9646f3d0b5aaab5d4834836d771c032d1a75059d31fed84f38e00c0b205918f6d354934c2055d33d19d045f783a62561f467728ebcf0160",
+ "ce112d07c52ae649f9646f3d0b5aaab5d4834836d771c032d1a75059d31fed84f38e00c0b205918f6d354934c2055d33d19d045f783a62561f467728ebcf0160"
+ }}
+ },
+
+ ["attachments blake2 base32"] = {
+ selector = "attachments('base32', 'blake2')",
+ expect = {{"qqr41dwakt3uwhucxmxsypjiifi8er3gzqhyc3r48fw1ij9dp8b8x8nyyscmoe6tpmp1r4eafezguezurazo87ecs48cw5bfm9udyob",
+ "qqr41dwakt3uwhucxmxsypjiifi8er3gzqhyc3r48fw1ij9dp8b8x8nyyscmoe6tpmp1r4eafezguezurazo87ecs48cw5bfm9udyob"
+ }}
+ },
+
+ ["attachments blake2 base64"] = {
+ selector = "attachments('base64', 'blake2')",
+ expect = {{"zhEtB8Uq5kn5ZG89C1qqtdSDSDbXccAy0adQWdMf7YTzjgDAsgWRj201STTCBV0z0Z0EX3g6YlYfRnco688BYA==",
+ "zhEtB8Uq5kn5ZG89C1qqtdSDSDbXccAy0adQWdMf7YTzjgDAsgWRj201STTCBV0z0Z0EX3g6YlYfRnco688BYA=="
+ }}
+ },
+
+ ["attachments blake2 rfc base32"] = {
+ selector = "attachments('rbase32', 'blake2')",
+ expect = {{"ZYIS2B6FFLTET6LEN46QWWVKWXKIGSBW25Y4AMWRU5IFTUY75WCPHDQAYCZALEMPNU2USNGCAVOTHUM5ARPXQOTCKYPUM5ZI5PHQCYA",
+ "ZYIS2B6FFLTET6LEN46QWWVKWXKIGSBW25Y4AMWRU5IFTUY75WCPHDQAYCZALEMPNU2USNGCAVOTHUM5ARPXQOTCKYPUM5ZI5PHQCYA"
+ }}
+ },
+
+ ["attachments md5 rfc base32"] = {
+ selector = "attachments('rbase32', 'md5')",
+ expect = {{"LYXF2IMILRFFO4LLTDTM66MKEA",
+ "LYXF2IMILRFFO4LLTDTM66MKEA"
+ }}
+ },
+
+ ["attachments id"] = {
+ selector = "attachments.id",
+ expect = {""}},
+
+ ["files"] = {
+ selector = "files",
+ expect = {{"f.zip", "f2.zip"}}},
+
+ ["helo"] = {
+ selector = "helo",
+ expect = {"hello mail"}},
+
+ ["received ip"] = {
+ selector = "received:by_hostname.filter_string_nils",
+ expect = {{"server1.chat-met-vreemden.nl", "server2.chat-met-vreemden.nl"}}},
+
+ ["received by hostname last"] = {
+ selector = "received:by_hostname.filter_string_nils.last",
+ expect = {"server2.chat-met-vreemden.nl"}
+ },
+
+ ["received by hostname first"] = {
+ selector = "received:by_hostname.filter_string_nils.first",
+ expect = {"server1.chat-met-vreemden.nl"}
+ },
+
+ ["urls"] = {
+ selector = "urls",
+ expect = {{"http://subdomain.example.net"}}},
+
+ ["emails"] = {
+ selector = "emails",
+ expect = {{"test@example.net"}}},
+
+ ["specific_urls"] = {
+ selector = "specific_urls({limit = 1})",
+ expect = {{"http://subdomain.example.net"}}},
+
+ ["specific_urls + emails"] = {
+ selector = "specific_urls({need_emails = true, limit = 2})",
+ expect = {{"test@example.net", "http://subdomain.example.net"}}},
+
+ -- Broken test as order depends on the hash function internally
+ --["specific_urls + emails limit"] = {
+ -- selector = "specific_urls({need_emails = true, limit = 1})",
+ -- expect = {{"test@example.net"}}},
+
+ ["pool_var str, default type"] = {
+ selector = [[pool_var("str_var")]],
+ expect = {"str 1"}},
+
+ ["pool_var str"] = {
+ selector = [[pool_var("str_var", 'string')]],
+ expect = {"str 1"}},
+
+ ["pool_var double"] = {
+ selector = [[pool_var("int_var", 'double')]],
+ expect = {"1"}},
+
+ ["time"] = {
+ selector = "time",
+ expect = {"1537364211"}},
+
+-- ["request_header"] = {
+-- selector = "request_header(hdr1)",
+-- expect = {"value1"}},
+
+ ["get_host"] = {
+ selector = "urls:get_host",
+ expect = {{"subdomain.example.net"}}},
+
+ ["get_tld_method"] = {
+ selector = "urls:get_tld",
+ expect = {{"example.net"}}},
+ ["get_tld_transform"] = {
+ selector = "urls:get_host.get_tld",
+ expect = {{"example.net"}}},
+
+ ["transformation regexp"] = {
+ selector = "urls:get_tld.regexp('\\.([\\w]+)$')",
+ expect = {{{".net", "net"}}}},
+
+ ["transformation id"] = {
+ selector = "urls:get_tld.id",
+ expect = {''}},
+
+ ["transformation id arg"] = {
+ selector = "urls:get_tld.id('1')",
+ expect = {'1'}},
+
+ ["transformation id args"] = {
+ selector = "urls:get_tld.id('1', '2', '3')",
+ expect = {{'1', '2', '3'}}},
+
+ ["transformation in"] = {
+ selector = "time(message, '!%w').in(2,3,4)",
+ expect = {'3'}},
+
+ ["transformation in id"] = {
+ selector = "time(message, '!%w').in(2,3,4).id",
+ expect = {''}},
+
+ ["transformation not in"] = {
+ selector = "time(message, '!%w').not_in(1,6,7)",
+ expect = {'3'}},
+
+ ["transformation in not id"] = {
+ selector = "time(message, '!%w').not_in(1,6,7).id",
+ expect = {''}},
+
+ ["transformation in not id 1"] = {
+ selector = "time(message, '!%w').not_in(1,6,7).id(1)",
+ expect = {'1'}},
+
+ ["transformation take"] = {
+ selector = "rcpts.take_n(1).lower",
+ expect = {{'nobody@example.com'}}},
+
+ ["transformation take 2"] = {
+ selector = "rcpts.take_n(2).lower",
+ expect = {{'nobody@example.com', 'no-one@example.com'}}},
+
+ ["transformation take 3"] = {
+ selector = "rcpts.take_n(3).lower",
+ expect = {{'nobody@example.com', 'no-one@example.com'}}},
+
+ ["transformation nth"] = {
+ selector = "rcpts.nth(1).lower",
+ expect = {'nobody@example.com'}},
+
+ ["transformation nth 2"] = {
+ selector = "rcpts.nth(2).lower",
+ expect = {'no-one@example.com'}},
+
+ ["transformation last"] = {
+ selector = "rcpts.last.lower",
+ expect = {'no-one@example.com'}},
+
+ ["transformation substring"] = {
+ selector = "header(Subject, strong).substring(6)",
+ expect = {'subject'}},
+
+ ["transformation substring 2"] = {
+ selector = "header(Subject, strong).substring(6, 7)",
+ expect = {'su'}},
+
+ ["transformation substring -4"] = {
+ selector = "header(Subject, strong).substring(-4)",
+ expect = {'ject'}
+ },
+ ["map filter"] = {
+ selector = "id('key').filter_map(test_map)",
+ expect = {'key'}
+ },
+ ["map except"] = {
+ selector = "list('key', 'key1', 'key2', 'key3', 'key4').except_map(test_map)",
+ expect = {{'key2', 'key4'}}
+ },
+ ["map apply"] = {
+ selector = "id('key').apply_map(test_map)",
+ expect = {'value'}
+ },
+ ["map filter list"] = {
+ selector = "list('key', 'key1', 'key2').filter_map(test_map)",
+ expect = {{'key', 'key1'}}
+ },
+ ["map apply list"] = {
+ selector = "list('key', 'key1', 'key2', 'key3').apply_map(test_map)",
+ expect = {{'value', 'value1', 'value1'}}
+ },
+ ["map apply list uniq"] = {
+ selector = "list('key', 'key1', 'key2', 'key3').apply_map(test_map).uniq",
+ expect = {{'value1', 'value'}}
+ },
+ ["words"] = {
+ selector = "words('norm')",
+ expect = {{'hello', 'world', 'mail', 'me'}}
+ },
+ ["words_full"] = {
+ selector = "words('full'):2",
+ expect = {{'hello', 'world', '', 'mail', 'me'}}
+ },
+ ["header X-Test first"] = {
+ selector = "header(X-Test, full).first",
+ expect = {"1"}
+ },
+ ["header X-Test last"] = {
+ selector = "header(X-Test, full).last",
+ expect = {"3"}
+ },
+ ["header lower digest substring"] = {
+ selector = "header('Subject').lower.digest('hex').substring(1, 16)",
+ expect = {"736ad5f50fc95d73"}
+ },
+ ["header gsub"] = {
+ selector = "header('Subject'):gsub('a', 'b')",
+ expect = {"Second, lower-cbsed hebder subject"}
+ },
+ ["header regexp first"] = {
+ selector = "header('Subject').regexp('.*').first",
+ expect = {"Second, lower-cased header subject"}
+ },
+
+ ["task cache string"] = {
+ selector = "task_cache('cachevar1')",
+ expect = {"hello\x00world"}
+ },
+ ["task cache table"] = {
+ selector = "task_cache('cachevar2')",
+ expect = {{"hello", "world"}}
+ },
+ }
+
+ for case_name, case in lua_util.spairs(cases_plain) do
+ test("plain case " .. case_name, function()
+ local elts = check_selector_plain(case.selector)
+ assert_not_nil(elts)
+ assert_rspamd_table_eq_sorted({actual = elts, expect = case.expect})
+ end)
+ end
+
+ local cases_kv = {
+ ["ip"] = {
+ selector = "id('ip');ip",
+ expect = { ip = "198.172.22.91" }
+ },
+ ["ip+words"] = {
+ selector = "id('ip');ip;id('words');words('full'):2",
+ expect = { ip = "198.172.22.91", words = {'hello', 'world', '', 'mail', 'me'} }
+ },
+ }
+ for case_name, case in lua_util.spairs(cases_kv) do
+ test("kv case " .. case_name, function()
+ local elts = check_selector_kv(case.selector)
+ assert_not_nil(elts)
+ assert_rspamd_table_eq_sorted({actual = elts, expect = case.expect})
+ end)
+ end
+end)
+
+
+--[=========[ ******************* message ******************* ]=========]
+msg = [[
+Received: from ca-18-193-131.service1.infuturo.it ([151.18.193.131] helo=User)
+ by server1.chat-met-vreemden.nl with esmtpa (Exim 4.76)
+ (envelope-from <upwest201diana@outlook.com>)
+ id 1ZC1sl-0006b4-TU; Mon, 06 Jul 2015 10:36:08 +0200
+Received: from ca-18-193-131.service2.infuturo.it ([151.18.193.132] helo=User)
+ by server2.chat-met-vreemden.nl with esmtpa (Exim 4.76)
+ (envelope-from <upwest201diana@outlook.com>)
+ id 1ZC1sl-0006b4-TU; Mon, 06 Jul 2015 10:36:08 +0200
+From: <whoknows@nowhere.com>
+To: <nobody@example.com>, <no-one@example.com>
+Date: Wed, 19 Sep 2018 14:36:51 +0100 (BST)
+subject: Second, lower-cased header subject
+Subject: Test subject
+X-Test: 1
+X-Test: 2
+X-Test: 3
+Content-Type: multipart/alternative;
+ boundary="_000_6be055295eab48a5af7ad4022f33e2d0_"
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 7bit
+
+Hello world
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/html; charset="utf-8"
+
+<html><body>
+<a href="http://subdomain.example.net">http://subdomain.example.net</a>
+<a href="mailto:test@example.net">mail me</a>
+</html>
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: application/zip; name=f.zip
+Content-Disposition: attachment; size=166; filename=f.zip
+Content-Transfer-Encoding: base64
+
+UEsDBAoAAAAAAINe6kgAAAAAAAAAAAAAAAAIABwAZmFrZS5leGVVVAkAA8YaglfGGoJXdXgLAAEE
+6AMAAAToAwAAUEsBAh4DCgAAAAAAg17qSAAAAAAAAAAAAAAAAAgAGAAAAAAAAAAAALSBAAAAAGZh
+a2UuZXhlVVQFAAPGGoJXdXgLAAEE6AMAAAToAwAAUEsFBgAAAAABAAEATgAAAEIAAAAAAA==
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: application/zip; name=f.zip
+Content-Disposition: attachment; size=166; filename=f2.zip
+Content-Transfer-Encoding: base64
+
+UEsDBAoAAAAAAINe6kgAAAAAAAAAAAAAAAAIABwAZmFrZS5leGVVVAkAA8YaglfGGoJXdXgLAAEE
+6AMAAAToAwAAUEsBAh4DCgAAAAAAg17qSAAAAAAAAAAAAAAAAAgAGAAAAAAAAAAAALSBAAAAAGZh
+a2UuZXhlVVQFAAPGGoJXdXgLAAEE6AMAAAToAwAAUEsFBgAAAAABAAEATgAAAEIAAAAAAA==
+]]
diff --git a/test/lua/unit/selectors.negative.lua b/test/lua/unit/selectors.negative.lua
new file mode 100644
index 0000000..4262400
--- /dev/null
+++ b/test/lua/unit/selectors.negative.lua
@@ -0,0 +1,113 @@
+local msg
+context("Selectors test", function()
+ local rspamd_task = require "rspamd_task"
+ local logger = require "rspamd_logger"
+ local lua_selectors = require "lua_selectors"
+ local ffi = require "ffi"
+ local cfg = rspamd_config
+
+ local task
+
+ ffi.cdef[[
+ void rspamd_url_init (const char *tld_file);
+ ]]
+
+ local test_dir = string.gsub(debug.getinfo(1).source, "^@(.+/)[^/]+$", "%1")
+
+ ffi.C.rspamd_url_init(string.format('%s/%s', test_dir, "test_tld.dat"))
+
+ before(function()
+ local res
+ res,task = rspamd_task.load_from_string(msg, cfg)
+ task:set_from_ip("198.172.22.91")
+ task:set_user("cool user name")
+ task:set_helo("hello mail")
+ task:set_request_header("hdr1", "value1")
+ task:process_message()
+ task:get_mempool():set_variable("int_var", 1)
+ task:get_mempool():set_variable("str_var", "str 1")
+ if not res then
+ assert_true(false, "failed to load message")
+ end
+ end)
+
+ local function check_selector(selector_string)
+ local sels = lua_selectors.parse_selector(cfg, selector_string)
+ local elts = lua_selectors.process_selectors(task, sels)
+ return elts
+ end
+
+ -- Selectors which should not be parse
+ local cases = {
+ ["random string"] = {
+ selector = "'xxx'"},
+
+ ["random nonsense"] = {
+ selector = "13 / sd 42 x"},
+
+ ["unknown selector"] = {
+ selector = "unknownselector"},
+
+ ["unknown transformation"] = {
+ selector = "urls.somethingnew"},
+ }
+
+ for case_name, case in pairs(cases) do
+ test("case " .. case_name, function()
+ local sels = lua_selectors.parse_selector(cfg, case.selector)
+ print(logger.slog("%1", sels))
+ assert_nil(sels)
+ end)
+ end
+end)
+
+
+--[=========[ ******************* message ******************* ]=========]
+msg = [[
+Received: from ca-18-193-131.service.infuturo.it ([151.18.193.131] helo=User)
+ by server.chat-met-vreemden.nl with esmtpa (Exim 4.76)
+ (envelope-from <upwest201diana@outlook.com>)
+ id 1ZC1sl-0006b4-TU; Mon, 06 Jul 2015 10:36:08 +0200
+From: <whoknows@nowhere.com>
+To: <nobody@example.com>, <no-one@example.com>
+Date: Wed, 19 Sep 2018 14:36:51 +0100 (BST)
+subject: Second, lower-cased header subject
+Subject: Test subject
+Content-Type: multipart/alternative;
+ boundary="_000_6be055295eab48a5af7ad4022f33e2d0_"
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: base64
+
+Hello world
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: text/html; charset="utf-8"
+
+<html><body>
+<a href="http://example.net">http://example.net</a>
+<a href="mailto:test@example.net">mail me</a>
+</html>
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: application/zip; name=f.zip
+Content-Disposition: attachment; size=166; filename=f.zip
+Content-Transfer-Encoding: base64
+
+UEsDBAoAAAAAAINe6kgAAAAAAAAAAAAAAAAIABwAZmFrZS5leGVVVAkAA8YaglfGGoJXdXgLAAEE
+6AMAAAToAwAAUEsBAh4DCgAAAAAAg17qSAAAAAAAAAAAAAAAAAgAGAAAAAAAAAAAALSBAAAAAGZh
+a2UuZXhlVVQFAAPGGoJXdXgLAAEE6AMAAAToAwAAUEsFBgAAAAABAAEATgAAAEIAAAAAAA==
+
+
+--_000_6be055295eab48a5af7ad4022f33e2d0_
+Content-Type: application/zip; name=f.zip
+Content-Disposition: attachment; size=166; filename=f2.zip
+Content-Transfer-Encoding: base64
+
+UEsDBAoAAAAAAINe6kgAAAAAAAAAAAAAAAAIABwAZmFrZS5leGVVVAkAA8YaglfGGoJXdXgLAAEE
+6AMAAAToAwAAUEsBAh4DCgAAAAAAg17qSAAAAAAAAAAAAAAAAAgAGAAAAAAAAAAAALSBAAAAAGZh
+a2UuZXhlVVQFAAPGGoJXdXgLAAEE6AMAAAToAwAAUEsFBgAAAAABAAEATgAAAEIAAAAAAA==
+]]
diff --git a/test/lua/unit/smtp_addr.lua b/test/lua/unit/smtp_addr.lua
new file mode 100644
index 0000000..2cb7755
--- /dev/null
+++ b/test/lua/unit/smtp_addr.lua
@@ -0,0 +1,110 @@
+-- SMTP address parser tests
+
+context("SMTP address check functions", function()
+ local logger = require("rspamd_logger")
+ local ffi = require("ffi")
+ local util = require("rspamd_util")
+ local fun = require "fun"
+ ffi.cdef[[
+ struct rspamd_email_address {
+ const char *raw;
+ const char *addr;
+ const char *user;
+ const char *domain;
+ const char *name;
+
+ unsigned raw_len;
+ unsigned addr_len;
+ unsigned domain_len;
+ uint16_t user_len;
+ unsigned char flags;
+ };
+ struct rspamd_email_address * rspamd_email_address_from_smtp (const char *str, unsigned len);
+ void rspamd_email_address_free (struct rspamd_email_address *addr);
+ ]]
+
+ local cases_valid = {
+ {'<>', {addr = ''}},
+ {'<a@example.com>', {user = 'a', domain = 'example.com', addr = 'a@example.com'}},
+ {'<a-b@example.com>', {user = 'a-b', domain = 'example.com', addr = 'a-b@example.com'}},
+ {'<a-b@ex-ample.com>', {user = 'a-b', domain = 'ex-ample.com', addr = 'a-b@ex-ample.com'}},
+ {'1367=dec2a6ce-81bd-4fa9-ad02-ec5956466c04=9=1655370@example.220-volt.ru',
+ {user = '1367=dec2a6ce-81bd-4fa9-ad02-ec5956466c04=9=1655370',
+ domain = 'example.220-volt.ru',
+ addr = '1367=dec2a6ce-81bd-4fa9-ad02-ec5956466c04=9=1655370@example.220-volt.ru'}},
+ {'notification+kjdm---m7wwd@facebookmail.com', {user = 'notification+kjdm---m7wwd'}},
+ {'a@example.com', {user = 'a', domain = 'example.com', addr = 'a@example.com'}},
+ {'a+b@example.com', {user = 'a+b', domain = 'example.com', addr = 'a+b@example.com'}},
+ {'"a"@example.com', {user = 'a', domain = 'example.com', addr = 'a@example.com'}},
+ {'"a+b"@example.com', {user = 'a+b', domain = 'example.com', addr = 'a+b@example.com'}},
+ {'"<>"@example.com', {user = '<>', domain = 'example.com', addr = '<>@example.com'}},
+ {'<"<>"@example.com>', {user = '<>', domain = 'example.com', addr = '<>@example.com'}},
+ {'"\\""@example.com', {user = '"', domain = 'example.com', addr = '"@example.com'}},
+ {'"\\"abc"@example.com', {user = '"abc', domain = 'example.com', addr = '"abc@example.com'}},
+ {'<@domain1,@domain2,@domain3:abc@example.com>',
+ {user = 'abc', domain = 'example.com', addr = 'abc@example.com'}},
+
+ }
+
+
+ fun.each(function(case)
+ test("Parse valid smtp addr: " .. case[1], function()
+ local st = ffi.C.rspamd_email_address_from_smtp(case[1], #case[1])
+
+ assert_not_nil(st, "should be able to parse " .. case[1])
+
+ fun.each(function(k, ex)
+ if k == 'user' then
+ local str = ffi.string(st.user, st.user_len)
+ assert_equal(str, ex)
+ elseif k == 'domain' then
+ local str = ffi.string(st.domain, st.domain_len)
+ assert_equal(str, ex)
+ elseif k == 'addr' then
+ local str = ffi.string(st.addr, st.addr_len)
+ assert_equal(str, ex)
+ end
+ end, case[2])
+ ffi.C.rspamd_email_address_free(st)
+ end)
+ end, cases_valid)
+
+ local cases_invalid = {
+ 'a',
+ 'a"b"@example.com',
+ 'a"@example.com',
+ '"a@example.com',
+ '<a@example.com',
+ 'a@example.com>',
+ '<a@.example.com>',
+ '<a@example.com>>',
+ '<a@example.com><>',
+ }
+
+ fun.each(function(case)
+ test("Parse invalid smtp addr: " .. case, function()
+ local st = ffi.C.rspamd_email_address_from_smtp(case, #case)
+
+ assert_nil(st, "should not be able to parse " .. case)
+ end)
+ end, cases_invalid)
+
+ if os.getenv("RSPAMD_LUA_EXPENSIVE_TESTS") then
+ test("Speed test", function()
+ local case = '<@domain1,@domain2,@domain3:abc%d@example.com>'
+ local niter = 100000
+ local total = 0
+
+ for i = 1,niter do
+ local ncase = string.format(case, i)
+ local t1 = util.get_ticks()
+ local st = ffi.C.rspamd_email_address_from_smtp(ncase, #ncase)
+ local t2 = util.get_ticks()
+ ffi.C.rspamd_email_address_free(st)
+ total = total + t2 - t1
+ end
+
+ print(string.format('Spend %f seconds in processing addrs', total))
+ end)
+ end
+end)
diff --git a/test/lua/unit/smtp_date.lua b/test/lua/unit/smtp_date.lua
new file mode 100644
index 0000000..aa8fbce
--- /dev/null
+++ b/test/lua/unit/smtp_date.lua
@@ -0,0 +1,58 @@
+context("SMTP date functions", function()
+ local rspamd_util = require "rspamd_util"
+
+ local cases = {
+ { 'Mon, 05 Oct 2020 19:05:57 -0000', 1601924757 },
+ -- space instead of leading zero
+ { 'Mon, 5 Oct 2020 19:05:57 -0000', 1601924757 },
+ -- no padding
+ { 'Mon, 5 Oct 2020 19:05:57 -0000', 1601924757 },
+ -- no weekday
+ { '5 Oct 2020 19:05:57 -0000', 1601924757 },
+ -- different TZ offsets
+ { 'Tue, 22 Sep 2020 00:03:14 -0800', 1600761794 },
+ { 'Fri, 02 Oct 2020 20:00:40 +0100', 1601665240 },
+ { 'Mon, 5 Oct 2020 15:48:32 +0530', 1601893112 },
+ { 'Mon, 05 Oct 2020 10:30:36 +1200', 1601850636 },
+ -- extra comment
+ { 'Thu, 18 May 2006 16:08:11 +0400 (MSD)', 1147954091 },
+ { 'Thu, 18 May 2006 16:08:11 +0400', 1147954091 },
+ -- obs_zone
+ { 'Sat, 26 Sep 2020 17:36:21 GMT', 1601141781 },
+ { 'Sat, 26 Sep 2020 17:36:21 UT', 1601141781 },
+ { 'Sat, 26 Sep 2020 17:36:21 +0000', 1601141781 },
+ { 'Wed, 30 Sep 2020 20:32:31 EDT', 1601512351 },
+ { 'Wed, 30 Sep 2020 20:32:31 -0400', 1601512351 },
+ { 'Wed, 30 Sep 2020 17:32:31 PDT', 1601512351 },
+ { 'Wed, 30 Sep 2020 17:32:31 -0700', 1601512351 },
+ -- 2 digit year < 50
+ { 'Mon, 05 Oct 20 06:35:38 GMT', 1601879738 },
+ { 'Mon, 05 Oct 2020 06:35:38 GMT', 1601879738 },
+ -- 2 digit year >= 50
+ { '26 Aug 76 14:30 EDT', 209932200 },
+ { '26 Aug 1976 14:30 EDT', 209932200 },
+ -- Year 2038 problem (broken on 32-bit systems, see #4754)
+ --{ 'Tue, 19 Jan 2038 03:14:07 GMT', 2 ^ 31 - 1 },
+ --{ 'Tue, 19 Jan 2038 03:14:09 GMT', 2 ^ 31 + 1 },
+ -- double space before TZ
+ { 'Sat, 29 Aug 2020 08:25:15 +0700', 1598664315 },
+ -- XXX timestamp corresponding to Sat Dec 30 00:00:00 GMT 1899 returned on error
+ --{'Sat, Dec 30 1899 00:00:00 GMT', -2209161600},
+ -- Invalid format
+ { 'Mon Oct 5 20:29:23 BST 2020', nil },
+ -- Wrong date
+ { '32 Jan 2020 00:00 GMT', nil },
+ -- Wrong time
+ { '1 Jan 2020 25:00 GMT', nil }
+ }
+
+ for _, case in ipairs(cases) do
+ test("Parse date: " .. case[1], function()
+ local timestamp = rspamd_util.parse_smtp_date(case[1])
+ assert_rspamd_eq({
+ expect = case[2],
+ actual = timestamp
+ })
+ end)
+ end
+end) \ No newline at end of file
diff --git a/test/lua/unit/sqlite3.lua b/test/lua/unit/sqlite3.lua
new file mode 100644
index 0000000..c431258
--- /dev/null
+++ b/test/lua/unit/sqlite3.lua
@@ -0,0 +1,50 @@
+context("Sqlite3 API", function()
+ local sqlite3 = require "rspamd_sqlite3"
+ local tmpdir = os.getenv("TMPDIR") or "/tmp"
+
+ test("Sqlite3 open", function()
+ os.remove(tmpdir .. '/rspamd_unit_test_sqlite3.sqlite')
+ local db = sqlite3.open(tmpdir .. '/rspamd_unit_test_sqlite3.sqlite')
+ assert_not_nil(db, "should be able to create sqlite3 db")
+ db = sqlite3.open('/non/existent/path/rspamd_unit_test_sqlite3.sqlite')
+ assert_nil(db, "should not be able to create sqlite3 db")
+ os.remove(tmpdir .. '/rspamd_unit_test_sqlite3.sqlite')
+ end)
+
+ test("Sqlite3 query", function()
+ os.remove(tmpdir .. '/rspamd_unit_test_sqlite3-1.sqlite')
+ local db = sqlite3.open(tmpdir .. '/rspamd_unit_test_sqlite3-1.sqlite')
+ assert_not_nil(db, "should be able to create sqlite3 db")
+
+ local ret = db:sql([[
+ CREATE TABLE x (id INT, value TEXT);
+ ]])
+ assert_true(ret, "should be able to create table")
+ local ret = db:sql([[
+ INSERT INTO x VALUES (?1, ?2);
+ ]], 1, 'test')
+ assert_true(ret, "should be able to insert row")
+ os.remove(tmpdir .. '/rspamd_unit_test_sqlite3-1.sqlite')
+ end)
+
+ test("Sqlite3 rows", function()
+ os.remove(tmpdir .. '/rspamd_unit_test_sqlite3-2.sqlite')
+ local db = sqlite3.open(tmpdir .. '/rspamd_unit_test_sqlite3-2.sqlite')
+ assert_not_nil(db, "should be able to create sqlite3 db")
+
+ local ret = db:sql([[
+ CREATE TABLE x (id INT, value TEXT);
+ ]])
+ assert_true(ret, "should be able to create table")
+ local ret = db:sql([[
+ INSERT INTO x VALUES (?1, ?2);
+ ]], 1, 'test')
+ assert_true(ret, "should be able to insert row")
+
+ for row in db:rows([[SELECT * FROM x;]]) do
+ assert_equal(row.id, '1')
+ assert_equal(row.value, 'test')
+ end
+ os.remove(tmpdir .. '/rspamd_unit_test_sqlite3-2.sqlite')
+ end)
+end) \ No newline at end of file
diff --git a/test/lua/unit/task.lua b/test/lua/unit/task.lua
new file mode 100644
index 0000000..0739a2b
--- /dev/null
+++ b/test/lua/unit/task.lua
@@ -0,0 +1,162 @@
+context("Task processing", function()
+ local fun = require("fun")
+ local rspamd_task = require("rspamd_task")
+
+ test("Process a simple task", function()
+ --local cfg = rspamd_util.config_from_ucl(config)
+ --assert_not_nil(cfg)
+
+ local msg = [[
+From: <>
+To: <nobody@example.com>
+Subject: test
+Content-Type: text/plain
+
+Test.
+]]
+ local res,task = rspamd_task.load_from_string(msg)
+ assert_true(res, "failed to load message")
+ task:process_message()
+ task:destroy()
+ end)
+
+ local hdrs = [[
+From: <>
+To: <nobody@example.com>
+Subject: test
+]]
+ local mpart = [[
+Content-Type: multipart/mixed; boundary=XXX
+]]
+ local body = [[
+Content-Type: text/html
+Content-Transfer-Encoding: quoted-printable
+
+<html>
+<body>
+=0DAttached is your new documents.
+<br>
+<a href=3D"http://evil.com/Information/">http:=
+//example.com/privacy/XXX/YYY_April_25_2019.doc</a>
+<br>
+<br>
+<br>
+Thank you,
+<br>
+<b>Haloclaims.co</b>
+</body></html>
+]]
+ test("Process mime nesting: simple", function()
+ local msg = hdrs .. body
+ local res,task = rspamd_task.load_from_string(msg)
+ assert_true(res, "failed to load message")
+ task:process_message()
+ assert_rspamd_table_eq_sorted({actual = fun.totable(fun.map(function(u)
+ return u:get_host()
+ end, task:get_urls())), expect = {
+ 'evil.com', 'example.com'
+ }})
+ task:destroy()
+ end)
+ test("Process mime nesting: multipart", function()
+ local msg = table.concat{
+ hdrs, mpart, '\n', '--XXX\n', body, '\n--XXX--\n'
+ }
+ local res,task = rspamd_task.load_from_string(msg)
+ assert_true(res, "failed to load message")
+ task:process_message()
+ assert_rspamd_table_eq_sorted({
+ actual = fun.totable(fun.map(function(u)
+ return u:get_host()
+ end, task:get_urls())),
+
+ expect = {
+ 'evil.com', 'example.com'
+ }})
+ task:destroy()
+ end)
+ test("Process mime nesting: multipart, broken", function()
+ local msg = table.concat{
+ hdrs, mpart, '\n', '--XXX\n', 'garbadge\n', '\n--XXX--\n', '--XXX\n', body
+ }
+ local res,task = rspamd_task.load_from_string(msg)
+ assert_true(res, "failed to load message")
+ task:process_message()
+ assert_rspamd_table_eq_sorted({
+ actual = fun.totable(fun.map(function(u)
+ return u:get_host()
+ end, task:get_urls())),
+
+ expect = {
+ 'evil.com', 'example.com'
+ }})
+
+ task:destroy()
+ end)
+ test("Process mime nesting: message", function()
+ local msg = table.concat{
+ hdrs, 'Content-Type: message/rfc822\n', '\n', hdrs, body
+ }
+ local res,task = rspamd_task.load_from_string(msg)
+ assert_true(res, "failed to load message")
+ task:process_message()
+ assert_rspamd_table_eq_sorted({
+ actual = fun.totable(fun.map(function(u)
+ return u:get_host()
+ end, task:get_urls())),
+
+ expect = {
+ 'evil.com', 'example.com'
+ }})
+
+ task:destroy()
+ end)
+ test("Process mime nesting: message in multipart", function()
+ local msg = table.concat{
+ hdrs, mpart, '\n',
+ '--XXX\n',
+ 'Content-Type: message/rfc822\n', '\n', hdrs, body ,
+ '\n--XXX--\n',
+ }
+
+ local res,task = rspamd_task.load_from_string(msg)
+ assert_true(res, "failed to load message")
+ task:process_message()
+ assert_rspamd_table_eq_sorted({
+ actual = fun.totable(fun.map(function(u)
+ return u:get_host()
+ end, task:get_urls())),
+
+ expect = {
+ 'evil.com', 'example.com'
+ }})
+
+ task:destroy()
+ end)
+ test("Process mime nesting: multipart message in multipart", function()
+ local msg = table.concat{
+ hdrs, mpart, '\n',
+ '--XXX\n',
+ 'Content-Type: message/rfc822\n', '\n', hdrs, mpart, '\n',
+
+ '--XXX\n',
+ body ,
+ '\n--XXX--\n',
+
+ '\n--XXX--\n',
+ }
+ local res,task = rspamd_task.load_from_string(msg)
+ assert_true(res, "failed to load message")
+ task:process_message()
+ assert_rspamd_table_eq_sorted({
+ actual = fun.totable(fun.map(function(u)
+ return u:get_host()
+ end, task:get_urls())),
+
+ expect = {
+ 'evil.com', 'example.com'
+ }})
+
+ task:destroy()
+ end)
+end) \ No newline at end of file
diff --git a/test/lua/unit/test.data b/test/lua/unit/test.data
new file mode 100644
index 0000000..696972e
--- /dev/null
+++ b/test/lua/unit/test.data
@@ -0,0 +1,10 @@
+RLvXs8ZWOYXidwy4RSErSJFAGiRhimvMhHNIOzbxkkDC1IQz03tf9jvglA45PXAb
+AyYIMAlMn1DrRCwsGKV/u8EEjkO34ujwirJ6ytbiZkjTnANBhGtZdjMCfsEUIY9a
+y35d3CeKZF9KaRdlWRDJdfBbZE9mn4rSUQ1X0+HweUZ3AmMHwWLa9SB+ii7ysEEl
++6QLqHczu7K0Ji3LVKI+NzPJOWmWWCHjJyhs8HsuHpUrJ3iSeLxfW0TD8x6eZ52C
+EWC0BbR32vtquw8r5O+yR6hbBUJj8mTqTs3yAaTEs8Q+7y5uFuGsv+0NrmEOASyT
+NvGaxODKLO1A/8kXXsko3I3hZOoi+9GG/eAncMRWtdwllE/KqZfp9uzi5aYh1MMb
+px4SFqH5FQfvveZwGgEl9+BCkRQIptqv1fMlWouy35n5AeHkfflyyA4wC6iwgJAL
+R5R95Y8y2UPWoRkB+HFvoEryCNrkdC1QmW07n5shHO9NzNk34tQIzfjvYwcPi2yy
+3e/YNr3jyKOs86jTK6z9M/4htai/OxuF34rGS9pau/NINrDOpCNNy4zDgsQkvm5l
+H4CzhH5tNvYaog==
diff --git a/test/lua/unit/test_tld.dat b/test/lua/unit/test_tld.dat
new file mode 100644
index 0000000..da12d76
--- /dev/null
+++ b/test/lua/unit/test_tld.dat
@@ -0,0 +1,20 @@
+com
+org
+net
+рф
+za.org
+xn--p1ai
+ac
+b.br
+co
+co.za
+in.net
+star.kawasaki.jp
+net.in
+star.nom.br
+org.ac
+ru.com
+za.net
+za.org
+org.za
+tk
diff --git a/test/lua/unit/testkey.pub b/test/lua/unit/testkey.pub
new file mode 100644
index 0000000..6407aa0
--- /dev/null
+++ b/test/lua/unit/testkey.pub
@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxUYMGsqMbNZl4vw65Afi
+vuC5rXDzbvP8zqj96L8t9M/2bV7Df1k4Pit+TKBczhs3HolQStq46AmrhoyNbLJx
+yaA8g+8ETXOhHzQUR74ud/xQaPqx02E02kbR3LnQTp/wdrJARMAB8CsPm8X2wrpF
+CRus+DMdDGWQXV3RFc0FbeYFMehn46k3+5dB96Y3Wh4cK3/aS2zpR2ddynN6vBaW
+sSTNfadGbUtIodZgl50ecdyVeExmL/H9HWhcafcNJVUeI0jd79Px90puB1auK6fu
+MVinDv2zJL3HIbz3qUTRAlVHdmphf/UoRq0hkZmnbTR0v9eC0FDwJV/XKspicJbv
+1QIDAQAB
+-----END PUBLIC KEY-----
diff --git a/test/lua/unit/testkey.sec b/test/lua/unit/testkey.sec
new file mode 100644
index 0000000..4a0325b
--- /dev/null
+++ b/test/lua/unit/testkey.sec
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAxUYMGsqMbNZl4vw65AfivuC5rXDzbvP8zqj96L8t9M/2bV7D
+f1k4Pit+TKBczhs3HolQStq46AmrhoyNbLJxyaA8g+8ETXOhHzQUR74ud/xQaPqx
+02E02kbR3LnQTp/wdrJARMAB8CsPm8X2wrpFCRus+DMdDGWQXV3RFc0FbeYFMehn
+46k3+5dB96Y3Wh4cK3/aS2zpR2ddynN6vBaWsSTNfadGbUtIodZgl50ecdyVeExm
+L/H9HWhcafcNJVUeI0jd79Px90puB1auK6fuMVinDv2zJL3HIbz3qUTRAlVHdmph
+f/UoRq0hkZmnbTR0v9eC0FDwJV/XKspicJbv1QIDAQABAoIBAEmUQteLTK0bmoz6
+/wwmVNBVCWxDgMiVgGmkZm/1PrLdDlDk044gPPYTStxRw8usIvbkyGnjAqypTqy0
+p9svA3nspiWfdL9erW3yAs5vhO2D0ooVV1Y8H3Z6i7QEKknpJctf2NDLvO1TYlL7
+l3ox96XaCL3acq85AouQfnffLHM8e0sCj5zc2gDIAz2Vjh5eTR0qIPHNxVL1xeD0
+KGnhTz4WveHraoa3ARzB+fDskilLSdCHrvn8SjaotlFwcSIHVc6ymutBxC/wFIu9
+0O6YamR6b8J1smkVyi+UcIGrXfeLndm3t5jLhmhMJC9D350XhFRLKfoGSTweD1r7
+yGFgDtECgYEA/PRzBeZIcydKQDuFsrItMwrtno0xA9WOJMbKx1QXggX907lJqyZI
+CjeP8GLO7YgXxibHuH8HUC9dy8K33GyZ9YjLkg3f3DZwSUIHwJhKvTbYHDs53oeZ
+4Py5HrDRXYuIZJMpWUCJYEmRsiG1aNL1rTI1V6BrqjUuvu6BXAjKunMCgYEAx6YA
+Ly5i6s7Vcp2/Sm1p+6YhOr0IVjWeJYxJzUYhQKk/EEw/pjsOuVPqH2ikGXnnIOIn
+jveNwfhxEFEL+MU1vEAVOHegDlzP4AdDUSXYV61UN+oJPGRqtByROyR6/s6hbhRr
+HpzOhzndg61N3XzN80pG56UwhLVs1nRjIVRlspcCgYAIkY9D5+UUYYRWYK3Ku9Zk
+ID6kXEwIwTcrb8B2uBaDxQgwH9qq/YT7M56gmfhAe5eykqW5TjRFNxWKTXJE+TjN
+5HBg5i9rGjz7fk0c7Qu7FRyE+EyhiR5hTK5Ip9yvuoZIQePorhL5PAS/b/zhLCQ+
+VbEQ4tJC1cJbnWCsaW/UmQKBgQCH8Xe0qMBAJKBg0BseAcylxuRfi7HuicnqxVDH
+jtY7okLHxTOd7B7FgOctheIfWstPr87B4bzL9HCAbL9bIGXLjlMkxQfeX8JISInE
+6qEaanKrNBgf4Dxr8fvOKrP5ZxeyzgJ2sM1MdNFpxQr9IutVmyEWwHt+Ec9PY6bQ
+Xhh1dwKBgBAGqEz10PnWmJWYylP1wgxNSltS3kGQWP/vqsL7xo4NV0TOYbsu8Iun
+MvLD3C5sSqTD7ycOiweglIFLaZCWtKEp+01WiEsceQ6G9mZ131Zb1uzdp+fmykyi
+IL7R1kM99vpYh0JMj9l8AyNeHG2MKxriOdcDiAOolVxCjYBlnsD0
+-----END RSA PRIVATE KEY-----
diff --git a/test/lua/unit/tokenizer.lua b/test/lua/unit/tokenizer.lua
new file mode 100644
index 0000000..fbf7ee3
--- /dev/null
+++ b/test/lua/unit/tokenizer.lua
@@ -0,0 +1,81 @@
+context("Text tokenization test", function()
+ local util = require "rspamd_util"
+ local logger = require "rspamd_logger"
+
+ local cases = {
+ {"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer mattis, nibh",
+ {"Lorem", "ipsum", "dolor", "sit", "amet", "consectetur", "adipiscing", "elit",
+ "Integer", "mattis", "nibh"
+ }
+ },
+ {"Õ€Õ¥Õ¿Õ¡Ö„Ö€Ö„Ö€Õ¾Õ¸Õ²Õ¶Õ¥Ö€Õ« Õ°Õ¡Õ´Õ¡Ö€ Õ¸Õ¿Õ¸Ö€Ö‡ Õ¶Õ¥Ö€Õ¯Õ¡ÕµÕ¡ÖÕ¾Õ¡Õ®",
+ {"Õ€Õ¥Õ¿Õ¡Ö„Ö€Ö„Ö€Õ¾Õ¸Õ²Õ¶Õ¥Ö€Õ«", "Õ°Õ¡Õ´Õ¡Ö€", "Õ¸Õ¿Õ¸Ö€Ö‡", "Õ¶Õ¥Ö€Õ¯Õ¡ÕµÕ¡ÖÕ¾Õ¡Õ®"}
+ },
+ {"", {}},
+ {",,,,,", {}},
+ {"word,,,,,word ", {"word", "word"}},
+ {"word", {"word"}},
+ {",,,,word,,,", {"word"}}
+ }
+
+ for i,c in ipairs(cases) do
+ test("Tokenize simple " .. i, function()
+ local w = util.tokenize_text(c[1])
+ if #c[2] == 0 then
+ assert_equal(#w, 0, "must not have tokens " .. c[1])
+ else
+ assert_not_nil(w, "must tokenize " .. c[1])
+
+ for i,wrd in ipairs(w) do
+ assert_equal(wrd, c[2][i])
+ end
+ end
+ end)
+ end
+
+ cases = {
+ {"word https://example.com/path word",
+ {{5, 24}},
+ {"word", "!!EX!!", "word"}
+ },
+ {"Õ°Õ¡Õ´Õ¡Ö€ https://example.com/path Õ°Õ¡Õ´Õ¡Ö€",
+ {{11, 24}},
+ {"Õ°Õ¡Õ´Õ¡Ö€", "!!EX!!", "Õ°Õ¡Õ´Õ¡Ö€"}
+ },
+ {"word https://example.com/path https://example.com/path word",
+ {{5, 24}, {30, 24}},
+ {"word", "!!EX!!", "!!EX!!", "word"}
+ },
+ {"word https://example.com/path https://example.com/path",
+ {{5, 24}, {30, 24}},
+ {"word", "!!EX!!", "!!EX!!"}
+ },
+ {"https://example.com/path https://example.com/path word",
+ {{0, 24}, {25, 24}},
+ {"!!EX!!", "!!EX!!", "word"}
+ },
+ {"https://example.com/path https://example.com/path",
+ {{0, 24}, {25, 24}},
+ {"!!EX!!", "!!EX!!"}
+ },
+ {",,,,https://example.com/path https://example.com/path ",
+ {{4, 24}, {29, 24}},
+ {"!!EX!!", "!!EX!!"}
+ },
+ }
+
+ for i,c in ipairs(cases) do
+ test("Tokenize with exceptions " .. i, function()
+ local w = util.tokenize_text(c[1], c[2])
+ if #c[3] == 0 then
+ assert_equal(#w, 0, "must not have tokens " .. c[1])
+ else
+ assert_not_nil(w, "must tokenize " .. c[1])
+ for i,wrd in ipairs(w) do
+ assert_equal(wrd, c[3][i])
+ end
+ end
+ end)
+ end
+
+end) \ No newline at end of file
diff --git a/test/lua/unit/trie.lua b/test/lua/unit/trie.lua
new file mode 100644
index 0000000..9532eae
--- /dev/null
+++ b/test/lua/unit/trie.lua
@@ -0,0 +1,81 @@
+-- Trie search tests
+
+context("Trie search functions", function()
+ local t = require "rspamd_trie"
+ local logger = require "rspamd_logger"
+ local patterns = {
+ 'test',
+ 'est',
+ 'he',
+ 'she',
+ 'str\1ing'
+ }
+
+ local trie = t.create(patterns)
+
+ local cases = {
+ {'test', true, {{4, 1}, {4, 2}}},
+ {'she test test', true, {{3, 4}, {3, 3}, {8, 1}, {8, 2}, {13, 1}, {13, 2}}},
+ {'non-existent', false},
+ {'str\1ing test', true, {{7, 5}, {12, 1}, {12, 2}}},
+ }
+
+ local function cmp_tables(t1, t2)
+ if t1[2] ~= t2[2] then
+ return t1[2] < t2[2]
+ else
+ return t1[1] < t2[1]
+ end
+ end
+
+ for i,c in ipairs(cases) do
+ test("Trie search " .. i, function()
+ local res = {}
+ local function cb(idx, pos)
+ table.insert(res, {pos, idx})
+
+ return 0
+ end
+
+ ret = trie:match(c[1], cb)
+
+ assert_equal(c[2], ret, tostring(c[2]) .. ' while matching ' .. c[1])
+
+ if ret then
+ table.sort(c[3], cmp_tables)
+ table.sort(res, cmp_tables)
+ assert_rspamd_table_eq({
+ expect = c[3],
+ actual = res
+ })
+ end
+ end)
+ end
+
+ for i,c in ipairs(cases) do
+ test("Trie search, table version " .. i, function()
+ local match = {}
+
+ match = trie:match(c[1])
+
+ assert_equal(c[2], #match > 0, tostring(c[2]) .. ' while matching ' .. c[1])
+
+ if match and #match > 0 then
+ local res = {}
+ -- Convert to something that this test expects
+ for pat,hits in pairs(match) do
+ for _,pos in ipairs(hits) do
+ table.insert(res, {pos, pat})
+ end
+ end
+ table.sort(c[3], cmp_tables)
+ table.sort(res, cmp_tables)
+ assert_rspamd_table_eq({
+ expect = c[3],
+ actual = res
+ })
+ end
+ end)
+ end
+
+end)
diff --git a/test/lua/unit/url.lua b/test/lua/unit/url.lua
new file mode 100644
index 0000000..52b88d2
--- /dev/null
+++ b/test/lua/unit/url.lua
@@ -0,0 +1,253 @@
+-- URL parser tests
+
+context("URL check functions", function()
+ local mpool = require("rspamd_mempool")
+ local lua_urls_compose = require "lua_urls_compose"
+ local url = require("rspamd_url")
+ local lua_util = require("lua_util")
+ local logger = require("rspamd_logger")
+ local test_helper = require("rspamd_test_helper")
+ local ffi = require("ffi")
+
+ ffi.cdef [[
+ void rspamd_normalize_path_inplace(char *path, size_t len, size_t *nlen);
+ ]]
+
+ test_helper.init_url_parser()
+
+ local pool = mpool.create()
+
+ local cases = {
+ { "test.com", { "test.com", nil } },
+ { " test.com", { "test.com", nil } },
+ { "<test.com> text", { "test.com", nil } },
+ { "test.com. text", { "test.com", nil } },
+ { "mailto:A.User@example.com text", { "example.com", "A.User" } },
+ { "http://ТеÑÑ‚.Рф:18 text", { "теÑÑ‚.рф", nil } },
+ { "http://user:password@теÑÑ‚2.РФ:18 text", { "теÑÑ‚2.рф", "user" } },
+ { "somebody@example.com", { "example.com", "somebody" } },
+ { "https://127.0.0.1/abc text", { "127.0.0.1", nil } },
+ { "https:\\\\127.0.0.1/abc text", { "127.0.0.1", nil } },
+ { "https:\\\\127.0.0.1", { "127.0.0.1", nil } },
+ { "https://127.0.0.1 text", { "127.0.0.1", nil } },
+ { "https://[::1]:1", { "::1", nil } },
+ { "https://user:password@[::1]:1", { "::1", nil } },
+ { "https://user:password@[::1]", { "::1", nil } },
+ { "https://user:password@[::1]/1", { "::1", nil } },
+ }
+
+ for i, c in ipairs(cases) do
+ local res = url.create(pool, c[1])
+
+ test("Extract urls from text" .. i, function()
+ assert_not_nil(res, "cannot parse " .. c[1])
+ local t = res:to_table()
+ --local s = logger.slog("%1 -> %2", c[1], t)
+ --print(s)
+ assert_not_nil(t, "cannot convert to table " .. c[1])
+ assert_equal(c[2][1], t['host'],
+ logger.slog('expected host "%s", but got "%s" in url %s => %s',
+ c[2][1], t['host'], c[1], t))
+
+ if c[2][2] then
+ assert_equal(c[2][1], t['host'],
+ logger.slog('expected user "%s", but got "%s" in url %s => %s',
+ c[2][1], t['host'], c[1], t))
+ end
+ end)
+ end
+
+ cases = {
+ { [[http://example.net/path/]], true, {
+ host = 'example.net', path = 'path/'
+ } },
+ { 'http://example.net/hello%20world.php?arg=x#fragment', true, {
+ host = 'example.net', fragment = 'fragment', query = 'arg=x',
+ path = 'hello world.php',
+ } },
+ { 'http://example.net/?arg=%23#fragment', true, {
+ host = 'example.net', fragment = 'fragment', query = 'arg=#',
+ } },
+ { "http:/\\[::eeee:192.168.0.1]/#test", true, {
+ host = '::eeee:c0a8:1', fragment = 'test'
+ } },
+ { "http:/\\[::eeee:192.168.0.1]#test", true, {
+ host = '::eeee:c0a8:1', fragment = 'test'
+ } },
+ { "http:/\\[::eeee:192.168.0.1]?test", true, {
+ host = '::eeee:c0a8:1', query = 'test'
+ } },
+ { "http:\\\\%30%78%63%30%2e%30%32%35%30.01", true, { --0xc0.0250.01
+ host = '192.168.0.1',
+ } },
+ { "http:/\\www.google.com/foo?bar=baz#", true, {
+ host = 'www.google.com', path = 'foo', query = 'bar=baz', tld = 'google.com'
+ } },
+ { "http://[www.google.com]/", true, {
+ host = 'www.google.com',
+ } },
+ { "<test.com", true, {
+ host = 'test.com', tld = 'test.com',
+ } },
+ { "test.com>", false },
+ { ",test.com text", false },
+ { "ht\ttp:@www.google.com:80/;p?#", false },
+ { "http://user:pass@/", false },
+ { "http://foo:-80/", false },
+ { "http:////////user:@google.com:99?foo", true, {
+ host = 'google.com', user = 'user', port = 99, query = 'foo'
+ } },
+ { "http://%25DOMAIN:foobar@foodomain.com/", true, {
+ host = 'foodomain.com', user = '%25DOMAIN'
+ } },
+ { "http://0.0xFFFFFF", true, {
+ host = '0.255.255.255'
+ } },
+ --{"http:/\\030052000001", true, {
+ -- host = '192.168.0.1'
+ --}},
+ { "http:\\/0xc0.052000001", true, {
+ host = '192.168.0.1'
+ } },
+ { "http://192.168.0.1.?foo", true, {
+ host = '192.168.0.1', query = 'foo',
+ } },
+ { "http://twitter.com#test", true, {
+ host = 'twitter.com', fragment = 'test'
+ } },
+ { "http:www.twitter.com#test", true, {
+ host = 'www.twitter.com', fragment = 'test'
+ } },
+ { "http://example。com#test", true, {
+ host = 'example.com', fragment = 'test'
+ } },
+ { "http://hoho.example。com#test", true, {
+ host = 'hoho.example.com', fragment = 'test'
+ } },
+ { "http://hoho。example。com#test", true, {
+ host = 'hoho.example.com', fragment = 'test'
+ } },
+ { "http://hoho.example。com#test", true, {
+ host = 'hoho.example.com', fragment = 'test'
+ } },
+ { "http://hehe。example。com#test", true, {
+ host = 'hehe.example.com', fragment = 'test'
+ } },
+ { "http:////$%^&****((@example.org//#f@f", true, {
+ user = '$%^&****((', host = 'example.org', fragment = 'f@f'
+ } },
+ { "http://@@example.com", true, {
+ user = "@", host = "example.com"
+ } },
+ { "https://example.com\\_Resources\\ClientImages\\UserData?ol\\o#ololo\\", true, {
+ host = "example.com", path = "_Resources\\ClientImages\\UserData",
+ query = "ol\\o", fragment = "ololo\\",
+ } },
+ {
+ "http://0x3f8f29a4/pro/au.html", true, {
+ host = "63.143.41.164",
+ path = "pro/au.html",
+ } },
+ {
+ "http://localhost", true, {
+ host = "localhost",
+ tld = "localhost",
+ } },
+ {
+ "http://localhost.", true, {
+ host = "localhost.",
+ tld = "localhost",
+ } },
+ }
+
+ -- Some cases from https://code.google.com/p/google-url/source/browse/trunk/src/url_canon_unittest.cc
+ for i, c in ipairs(cases) do
+ local res = url.create(pool, c[1])
+
+ test("Parse url: " .. c[1], function()
+ if c[2] then
+ assert_not_nil(res, "we are able to parse url: " .. c[1])
+
+ local uf = res:to_table()
+
+ for k, v in pairs(c[3]) do
+ assert_not_nil(uf[k], k .. ' is missing in url, must be ' .. v)
+ assert_equal(uf[k], v, logger.slog('expected " %s ", for %s, but got " %s " in url %s => %s',
+ v, k, uf[k], c[1], uf))
+ end
+ for k, v in pairs(uf) do
+ if k ~= 'url' and k ~= 'protocol' and k ~= 'tld' then
+ assert_not_nil(c[3][k], k .. ' should be absent but it is ' .. v .. ' in: ' .. c[1])
+ end
+ end
+ else
+ assert_nil(res, "should not parse " .. c[1] .. ' parsed to: ' .. tostring(res))
+ end
+ end)
+ end
+
+ cases = {
+ { "/././foo", "/foo" },
+ { "/a/b/c/./../../g", "/a/g" },
+ { "/./.foo", "/.foo" },
+ { "/foo/.", "/foo/" },
+ { "/foo/./", "/foo/" },
+ { "/foo/bar/..", "/foo" },
+ { "/foo/bar/../", "/foo/" },
+ { "/foo/..bar", "/foo/..bar" },
+ { "/foo/bar/../ton", "/foo/ton" },
+ { "/foo/bar/../ton/../../a", "/a" },
+ { "/foo/../../..", "/" },
+ { "/foo/../../../ton", "/ton" },
+ { "////../..", "/" },
+ { "./", "" },
+ { "/./", "/" },
+ { "/./././././././", "/" },
+ { "/", "/" },
+ { "/a/b", "/a/b" },
+ { "/a/b/", "/a/b/" },
+ { "..", "/" },
+ { "/../", "/" },
+ { "../", "/" },
+ { "///foo", "/foo" },
+ }
+
+ for i, v in ipairs(cases) do
+ test(string.format("Normalize paths '%s'", v[1]), function()
+ local buf = ffi.new("uint8_t[?]", #v[1])
+ local sizbuf = ffi.new("size_t[1]")
+ ffi.copy(buf, v[1], #v[1])
+ ffi.C.rspamd_normalize_path_inplace(buf, #v[1], sizbuf)
+ local res = ffi.string(buf, tonumber(sizbuf[0]))
+ assert_equal(v[2], res, 'expected ' .. v[2] .. ' but got ' .. res .. ' in path ' .. v[1])
+ end)
+ end
+
+ cases = {
+ { 'example.com', 'example.com' },
+ { 'baz.example.com', 'baz.example.com' },
+ { '3.baz.example.com', 'baz.example.com' },
+ { 'bar.example.com', 'example.com' },
+ { 'foo.example.com', 'foo.example.com' },
+ { '3.foo.example.com', '3.foo.example.com' },
+ { 'foo.com', 'foo.com' },
+ { 'bar.foo.com', 'foo.com' },
+ }
+
+ local excl_rules1 = {
+ 'example.com',
+ '*.foo.example.com',
+ '!bar.example.com'
+ }
+
+ local comp_rules = lua_urls_compose.inject_composition_rules(rspamd_config, excl_rules1)
+
+ for _, v in ipairs(cases) do
+ test("URL composition " .. v[1], function()
+ local u = url.create(pool, v[1])
+ assert_not_nil(u, "we are able to parse url:" .. v[1])
+ local res = comp_rules:process_url(nil, u:get_tld(), u:get_host())
+ assert_equal(v[2], res, 'expected ' .. v[2] .. ' but got ' .. res .. ' in url ' .. v[1])
+ end)
+ end
+end)
diff --git a/test/lua/unit/utf.lua b/test/lua/unit/utf.lua
new file mode 100644
index 0000000..dbdab7f
--- /dev/null
+++ b/test/lua/unit/utf.lua
@@ -0,0 +1,207 @@
+-- Test utf routines
+
+context("UTF8 check functions", function()
+ local ffi = require("ffi")
+ ffi.cdef[[
+ unsigned int rspamd_str_lc_utf8 (char *str, unsigned int size);
+ unsigned int rspamd_str_lc (char *str, unsigned int size);
+ void rspamd_fast_utf8_library_init (unsigned flags);
+ void ottery_rand_bytes(void *buf, size_t n);
+ double rspamd_get_ticks(int allow);
+ size_t rspamd_fast_utf8_validate (const unsigned char *data, size_t len);
+ size_t rspamd_fast_utf8_validate_ref (const unsigned char *data, size_t len);
+ size_t rspamd_fast_utf8_validate_sse41 (const unsigned char *data, size_t len);
+ size_t rspamd_fast_utf8_validate_avx2 (const unsigned char *data, size_t len);
+ char * rspamd_str_make_utf_valid (const char *src, size_t slen, size_t *dstlen, void *);
+ ]]
+
+ local cases = {
+ {"ÐбЫрвÐлг", "абырвалг"},
+ {"ÐAБBвc", "аaбbвc"},
+ --{"STRASSE", "straße"}, XXX: NYI
+ {"KEÇİ", "keçi"},
+ }
+
+ for i,c in ipairs(cases) do
+ test("UTF lowercase " .. tostring(i), function()
+ local buf = ffi.new("char[?]", #c[1] + 1)
+ ffi.copy(buf, c[1])
+ local nlen = ffi.C.rspamd_str_lc_utf8(buf, #c[1])
+ local s = ffi.string(buf, nlen)
+ assert_equal(s, c[2])
+ end)
+ end
+
+ cases = {
+ {"AbCdEf", "abcdef"},
+ {"A", "a"},
+ {"AaAa", "aaaa"},
+ {"AaAaAaAa", "aaaaaaaa"}
+ }
+
+ for i,c in ipairs(cases) do
+ test("ASCII lowercase " .. tostring(i), function()
+ local buf = ffi.new("char[?]", #c[1] + 1)
+ ffi.copy(buf, c[1])
+ ffi.C.rspamd_str_lc(buf, #c[1])
+ local s = ffi.string(buf)
+ assert_equal(s, c[2])
+ end)
+ end
+
+ cases = {
+ {'теÑÑ‚', 'теÑÑ‚'},
+ {'\200\213\202', '���'},
+ {'теÑÑ‚\200\213\202test', 'теÑ�test'},
+ {'\200\213\202test', '���test'},
+ {'\200\213\202test\200\213\202', '���test���'},
+ {'теÑÑ‚\200\213\202test\200\213\202', 'теÑ�test���'},
+ {'теÑÑ‚\200\213\202test\200\213\202теÑÑ‚', 'теÑ�test���теÑÑ‚'},
+ }
+
+ local NULL = ffi.new 'void*'
+ for i,c in ipairs(cases) do
+ test("Unicode make valid " .. tostring(i), function()
+ local buf = ffi.new("char[?]", #c[1] + 1)
+ ffi.copy(buf, c[1])
+
+ local s = ffi.string(ffi.C.rspamd_str_make_utf_valid(buf, #c[1], NULL, NULL))
+ local function to_hex(s)
+ return (s:gsub('.', function (c)
+ return string.format('%02X', string.byte(c))
+ end))
+ end
+ print(to_hex(s))
+ print(to_hex(c[2]))
+ assert_equal(s, c[2])
+ end)
+ end
+
+ -- Enable sse and avx2
+ ffi.C.rspamd_fast_utf8_library_init(3)
+ local valid_cases = {
+ "a",
+ "\xc3\xb1",
+ "\xe2\x82\xa1",
+ "\xf0\x90\x8c\xbc",
+ "안녕하세요, 세ìƒ"
+ }
+ for i,c in ipairs(valid_cases) do
+ test("Unicode validate success: " .. tostring(i), function()
+ local buf = ffi.new("char[?]", #c + 1)
+ ffi.copy(buf, c)
+
+ local ret = ffi.C.rspamd_fast_utf8_validate(buf, #c)
+ assert_equal(ret, 0)
+ end)
+ end
+ local invalid_cases = {
+ "\xc3\x28",
+ "\xa0\xa1",
+ "\xe2\x28\xa1",
+ "\xe2\x82\x28",
+ "\xf0\x28\x8c\xbc",
+ "\xf0\x90\x28\xbc",
+ "\xf0\x28\x8c\x28",
+ "\xc0\x9f",
+ "\xf5\xff\xff\xff",
+ "\xed\xa0\x81",
+ "\xf8\x90\x80\x80\x80",
+ "123456789012345\xed",
+ "123456789012345\xf1",
+ "123456789012345\xc2",
+ "\xC2\x7F"
+ }
+ for i,c in ipairs(invalid_cases) do
+ test("Unicode validate fail: " .. tostring(i), function()
+ local buf = ffi.new("char[?]", #c + 1)
+ ffi.copy(buf, c)
+
+ local ret = ffi.C.rspamd_fast_utf8_validate(buf, #c)
+ assert_not_equal(ret, 0)
+ end)
+ end
+
+ if os.getenv("RSPAMD_LUA_EXPENSIVE_TESTS") then
+ local speed_iters = 10000
+ local function test_size(buflen, is_valid, impl)
+ local logger = require "rspamd_logger"
+ local test_str
+ if is_valid then
+ test_str = table.concat(valid_cases)
+ else
+ test_str = table.concat(valid_cases) .. table.concat(invalid_cases)
+ end
+
+ local buf = ffi.new("char[?]", buflen)
+ if #test_str < buflen then
+ local t = {}
+ local len = #test_str
+ while len < buflen do
+ t[#t + 1] = test_str
+ len = len + #test_str
+ end
+ test_str = table.concat(t)
+ end
+ ffi.copy(buf, test_str:sub(1, buflen))
+
+ local tm = 0
+
+ for _=1,speed_iters do
+ if impl == 'ref' then
+ local t1 = ffi.C.rspamd_get_ticks(1)
+ ffi.C.rspamd_fast_utf8_validate_ref(buf, buflen)
+ local t2 = ffi.C.rspamd_get_ticks(1)
+ tm = tm + (t2 - t1)
+ elseif impl == 'sse' then
+ local t1 = ffi.C.rspamd_get_ticks(1)
+ ffi.C.rspamd_fast_utf8_validate_sse41(buf, buflen)
+ local t2 = ffi.C.rspamd_get_ticks(1)
+ tm = tm + (t2 - t1)
+ else
+ local t1 = ffi.C.rspamd_get_ticks(1)
+ ffi.C.rspamd_fast_utf8_validate_avx2(buf, buflen)
+ local t2 = ffi.C.rspamd_get_ticks(1)
+ tm = tm + (t2 - t1)
+ end
+ end
+
+ logger.messagex("%s utf8 %s check (valid = %s): %s ticks per iter, %s ticks per byte",
+ impl, buflen, is_valid,
+ tm / speed_iters, tm / speed_iters / buflen)
+
+ return 0
+ end
+
+ for _,sz in ipairs({78, 512, 65535}) do
+ test(string.format("Utf8 test %s %d buffer, %s", 'ref', sz, 'valid'), function()
+ local res = test_size(sz, true, 'ref')
+ assert_equal(res, 0)
+ end)
+ test(string.format("Utf8 test %s %d buffer, %s", 'ref', sz, 'invalid'), function()
+ local res = test_size(sz, false, 'ref')
+ assert_equal(res, 0)
+ end)
+
+ if jit.arch == 'x64' then
+ test(string.format("Utf8 test %s %d buffer, %s", 'sse', sz, 'valid'), function()
+ local res = test_size(sz, true, 'sse')
+ assert_equal(res, 0)
+ end)
+ test(string.format("Utf8 test %s %d buffer, %s", 'sse', sz, 'invalid'), function()
+ local res = test_size(sz, false, 'sse')
+ assert_equal(res, 0)
+ end)
+ test(string.format("Utf8 test %s %d buffer, %s", 'avx2', sz, 'valid'), function()
+ local res = test_size(sz, true, 'avx2')
+ assert_equal(res, 0)
+ end)
+ test(string.format("Utf8 test %s %d buffer, %s", 'avx2', sz, 'invalid'), function()
+ local res = test_size(sz, false, 'avx2')
+ assert_equal(res, 0)
+ end)
+ end
+ end
+ end
+
+end) \ No newline at end of file
diff --git a/test/rspamd_cryptobox_test.c b/test/rspamd_cryptobox_test.c
new file mode 100644
index 0000000..e3d8504
--- /dev/null
+++ b/test/rspamd_cryptobox_test.c
@@ -0,0 +1,347 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "shingles.h"
+#include "fstring.h"
+#include "ottery.h"
+#include "cryptobox.h"
+#include "unix-std.h"
+
+static const int mapping_size = 64 * 8192 + 1;
+static const int max_seg = 32;
+static const int random_fuzz_cnt = 10000;
+enum rspamd_cryptobox_mode mode = RSPAMD_CRYPTOBOX_MODE_25519;
+
+static void *
+create_mapping(int mapping_len, guchar **beg, guchar **end)
+{
+ void *map;
+ int psize = getpagesize();
+
+ map = mmap(NULL, mapping_len + psize * 3, PROT_READ | PROT_WRITE,
+ MAP_ANON | MAP_SHARED, -1, 0);
+ g_assert(map != 0);
+ memset(map, 0, mapping_len + psize * 3);
+ mprotect(map, psize, PROT_NONE);
+ /* Misalign pointer */
+ *beg = ((guchar *) map) + psize + 1;
+ *end = *beg + mapping_len;
+ mprotect(*beg + mapping_len - 1 + psize, psize, PROT_NONE);
+
+ return map;
+}
+
+static void
+check_result(const rspamd_nm_t key, const rspamd_nonce_t nonce,
+ const rspamd_mac_t mac, guchar *begin, guchar *end)
+{
+ guint64 *t = (guint64 *) begin;
+
+ g_assert(rspamd_cryptobox_decrypt_nm_inplace(begin, end - begin, nonce, key,
+ mac, mode));
+
+ while (t < (guint64 *) end) {
+ g_assert(*t == 0);
+ t++;
+ }
+}
+
+static int
+create_random_split(struct rspamd_cryptobox_segment *seg, int mseg,
+ guchar *begin, guchar *end)
+{
+ gsize remain = end - begin;
+ gint used = 0;
+
+ while (remain > 0 && used < mseg - 1) {
+ seg->data = begin;
+ seg->len = ottery_rand_range(remain - 1) + 1;
+
+ begin += seg->len;
+ remain -= seg->len;
+ used++;
+ seg++;
+ }
+
+ if (remain > 0) {
+ seg->data = begin;
+ seg->len = remain;
+ used++;
+ }
+
+ return used;
+}
+
+static int
+create_realistic_split(struct rspamd_cryptobox_segment *seg, int mseg,
+ guchar *begin, guchar *end)
+{
+ gsize remain = end - begin;
+ gint used = 0;
+ static const int small_seg = 512, medium_seg = 2048;
+
+ while (remain > 0 && used < mseg - 1) {
+ seg->data = begin;
+
+ if (ottery_rand_uint32() % 2 == 0) {
+ seg->len = ottery_rand_range(small_seg) + 1;
+ }
+ else {
+ seg->len = ottery_rand_range(medium_seg) +
+ small_seg;
+ }
+ if (seg->len > remain) {
+ seg->len = remain;
+ }
+
+ begin += seg->len;
+ remain -= seg->len;
+ used++;
+ seg++;
+ }
+
+ if (remain > 0) {
+ seg->data = begin;
+ seg->len = remain;
+ used++;
+ }
+
+ return used;
+}
+
+static int
+create_constrained_split(struct rspamd_cryptobox_segment *seg, int mseg,
+ int constraint,
+ guchar *begin, guchar *end)
+{
+ gsize remain = end - begin;
+ gint used = 0;
+
+ while (remain > 0 && used < mseg - 1) {
+ seg->data = begin;
+ seg->len = constraint;
+ if (seg->len > remain) {
+ seg->len = remain;
+ }
+ begin += seg->len;
+ remain -= seg->len;
+ used++;
+ seg++;
+ }
+
+ if (remain > 0) {
+ seg->data = begin;
+ seg->len = remain;
+ used++;
+ }
+
+ return used;
+}
+
+void rspamd_cryptobox_test_func(void)
+{
+ void *map;
+ guchar *begin, *end;
+ rspamd_nm_t key;
+ rspamd_nonce_t nonce;
+ rspamd_mac_t mac;
+ struct rspamd_cryptobox_segment *seg;
+ double t1, t2;
+ gint i, cnt, ms;
+ gboolean checked_openssl = FALSE;
+
+ map = create_mapping(mapping_size, &begin, &end);
+
+ ottery_rand_bytes(key, sizeof(key));
+ ottery_rand_bytes(nonce, sizeof(nonce));
+
+ memset(mac, 0, sizeof(mac));
+ seg = g_slice_alloc0(sizeof(*seg) * max_seg * 10);
+
+ /* Test baseline */
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encrypt_nm_inplace(begin, end - begin, nonce, key, mac,
+ mode);
+ t2 = rspamd_get_ticks(TRUE);
+ check_result(key, nonce, mac, begin, end);
+
+ msg_info("baseline encryption: %.0f", t2 - t1);
+
+ mode = RSPAMD_CRYPTOBOX_MODE_NIST;
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encrypt_nm_inplace(begin,
+ end - begin,
+ nonce,
+ key,
+ mac,
+ mode);
+ t2 = rspamd_get_ticks(TRUE);
+ check_result(key, nonce, mac, begin, end);
+
+ msg_info("openssl baseline encryption: %.0f", t2 - t1);
+ mode = RSPAMD_CRYPTOBOX_MODE_25519;
+
+start:
+ /* A single chunk as vector */
+ seg[0].data = begin;
+ seg[0].len = end - begin;
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, 1, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ msg_info("bulk encryption: %.0f", t2 - t1);
+
+ /* Two chunks as vector */
+ seg[0].data = begin;
+ seg[0].len = (end - begin) / 2;
+ seg[1].data = begin + seg[0].len;
+ seg[1].len = (end - begin) - seg[0].len;
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, 2, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ msg_info("2 equal chunks encryption: %.0f", t2 - t1);
+
+ seg[0].data = begin;
+ seg[0].len = 1;
+ seg[1].data = begin + seg[0].len;
+ seg[1].len = (end - begin) - seg[0].len;
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, 2, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ msg_info("small and large chunks encryption: %.0f", t2 - t1);
+
+ seg[0].data = begin;
+ seg[0].len = (end - begin) - 3;
+ seg[1].data = begin + seg[0].len;
+ seg[1].len = (end - begin) - seg[0].len;
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, 2, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ msg_info("large and small chunks encryption: %.0f", t2 - t1);
+
+ /* Random two chunks as vector */
+ seg[0].data = begin;
+ seg[0].len = ottery_rand_range(end - begin - 1) + 1;
+ seg[1].data = begin + seg[0].len;
+ seg[1].len = (end - begin) - seg[0].len;
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, 2, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ msg_info("random 2 chunks encryption: %.0f", t2 - t1);
+
+ /* 3 specific chunks */
+ seg[0].data = begin;
+ seg[0].len = 2;
+ seg[1].data = begin + seg[0].len;
+ seg[1].len = 2049;
+ seg[2].data = begin + seg[0].len + seg[1].len;
+ seg[2].len = (end - begin) - seg[0].len - seg[1].len;
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, 3, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ msg_info("small, medium and large chunks encryption: %.0f", t2 - t1);
+
+ cnt = create_random_split(seg, max_seg, begin, end);
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, cnt, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ msg_info("random split of %d chunks encryption: %.0f", cnt, t2 - t1);
+
+ cnt = create_realistic_split(seg, max_seg, begin, end);
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, cnt, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ msg_info("realistic split of %d chunks encryption: %.0f", cnt, t2 - t1);
+
+ cnt = create_constrained_split(seg, max_seg + 1, 32, begin, end);
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, cnt, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ msg_info("constrained split of %d chunks encryption: %.0f", cnt, t2 - t1);
+
+ for (i = 0; i < random_fuzz_cnt; i++) {
+ ms = ottery_rand_range(i % max_seg * 2) + 1;
+ cnt = create_random_split(seg, ms, begin, end);
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, cnt, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ if (i % 1000 == 0) {
+ msg_info("random fuzz iterations: %d", i);
+ }
+ }
+ for (i = 0; i < random_fuzz_cnt; i++) {
+ ms = ottery_rand_range(i % max_seg * 2) + 1;
+ cnt = create_realistic_split(seg, ms, begin, end);
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, cnt, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ if (i % 1000 == 0) {
+ msg_info("realistic fuzz iterations: %d", i);
+ }
+ }
+ for (i = 0; i < random_fuzz_cnt; i++) {
+ ms = ottery_rand_range(i % max_seg * 10) + 1;
+ cnt = create_constrained_split(seg, ms, i, begin, end);
+ t1 = rspamd_get_ticks(TRUE);
+ rspamd_cryptobox_encryptv_nm_inplace(seg, cnt, nonce, key, mac, mode);
+ t2 = rspamd_get_ticks(TRUE);
+
+ check_result(key, nonce, mac, begin, end);
+
+ if (i % 1000 == 0) {
+ msg_info("constrained fuzz iterations: %d", i);
+ }
+ }
+
+ if (!checked_openssl) {
+ checked_openssl = TRUE;
+ mode = RSPAMD_CRYPTOBOX_MODE_NIST;
+ goto start;
+ }
+}
diff --git a/test/rspamd_cxx_local_ptr.hxx b/test/rspamd_cxx_local_ptr.hxx
new file mode 100644
index 0000000..2343a74
--- /dev/null
+++ b/test/rspamd_cxx_local_ptr.hxx
@@ -0,0 +1,344 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_RSPAMD_CXX_LOCAL_PTR_HXX
+#define RSPAMD_RSPAMD_CXX_LOCAL_PTR_HXX
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+#include "libutil/cxx/local_shared_ptr.hxx"
+
+namespace test_internal {
+struct deleter_test {
+ bool *pv;
+
+ deleter_test(bool &v)
+ {
+ v = false;
+ pv = &v;
+ }
+
+ ~deleter_test()
+ {
+ *pv = true;
+ }
+};
+}// namespace test_internal
+
+namespace std {
+template<>
+struct hash<test_internal::deleter_test> {
+ inline auto operator()(const test_internal::deleter_test &) const noexcept -> auto
+ {
+ return 42;
+ }
+};
+}// namespace std
+
+TEST_SUITE("local_ptr")
+{
+ using namespace test_internal;
+
+ TEST_CASE("shared_ptr from nullptr")
+ {
+ rspamd::local_shared_ptr<int const> pi(static_cast<int *>(nullptr));
+ CHECK((!!pi ? false : true));
+ CHECK(!pi);
+ CHECK(pi.get() == nullptr);
+ CHECK(pi.use_count() == 1);
+ CHECK(pi.unique());
+ }
+ TEST_CASE("shared_ptr from ptr")
+ {
+ int *p = new int(7);
+ rspamd::local_shared_ptr<int> pi(p);
+ CHECK((pi ? true : false));
+ CHECK(!!pi);
+ CHECK(pi.get() == p);
+ CHECK(pi.use_count() == 1);
+ CHECK(pi.unique());
+ CHECK(*pi == 7);
+ }
+
+ TEST_CASE("shared_ptr copy")
+ {
+ rspamd::local_shared_ptr<int> pi;
+
+ rspamd::local_shared_ptr<int> pi2(pi);
+ CHECK(pi2 == pi);
+ CHECK((pi2 ? false : true));
+ CHECK(!pi2);
+ CHECK(pi2.get() == nullptr);
+ CHECK(pi2.use_count() == pi.use_count());
+
+ rspamd::local_shared_ptr<int> pi3(pi);
+ CHECK(pi3 == pi);
+ CHECK((pi3 ? false : true));
+ CHECK(!pi3);
+ CHECK(pi3.get() == nullptr);
+ CHECK(pi3.use_count() == pi.use_count());
+
+ rspamd::local_shared_ptr<int> pi4(pi3);
+ CHECK(pi4 == pi3);
+ CHECK((pi4 ? false : true));
+ CHECK(!pi4);
+ CHECK(pi4.get() == nullptr);
+ CHECK(pi4.use_count() == pi3.use_count());
+
+ int *p = new int(7);
+ rspamd::local_shared_ptr<int> pi5(p);
+
+ rspamd::local_shared_ptr<int> pi6(pi5);
+ CHECK(pi5 == pi6);
+ CHECK((pi6 ? true : false));
+ CHECK(!!pi6);
+ CHECK(pi6.get() == p);
+ CHECK(pi6.use_count() == 2);
+ CHECK(!pi6.unique());
+ CHECK(*pi6 == 7);
+ CHECK(pi6.use_count() == pi6.use_count());
+ CHECK(!(pi5 < pi6 || pi5 < pi6));// shared ownership test
+
+ auto pi7 = pi6;
+ CHECK(pi5 == pi7);
+ CHECK((pi7 ? true : false));
+ CHECK(!!pi7);
+ CHECK(pi7.get() == p);
+ CHECK(pi7.use_count() == 3);
+ CHECK(!pi7.unique());
+ CHECK(*pi7 == 7);
+ CHECK(pi7.use_count() == pi7.use_count());
+ CHECK(!(pi5 < pi7 || pi5 < pi7));// shared ownership test
+ }
+
+ TEST_CASE("shared_ptr move")
+ {
+ rspamd::local_shared_ptr<int> pi(new int);
+
+ rspamd::local_shared_ptr<int> pi2(std::move(pi));
+ CHECK(!(pi2 == pi));
+ CHECK((!pi2 ? false : true));
+ CHECK(!pi);
+ CHECK(pi.get() == nullptr);
+ CHECK(pi2.get() != nullptr);
+ CHECK(pi.use_count() != pi2.use_count());
+
+ std::swap(pi, pi2);
+ CHECK(!(pi2 == pi));
+ CHECK((!pi ? false : true));
+ CHECK(!pi2);
+ CHECK(pi.get() != nullptr);
+ CHECK(pi2.get() == nullptr);
+ CHECK(pi.use_count() != pi2.use_count());
+ }
+
+ TEST_CASE("shared_ptr dtor")
+ {
+ bool t;
+
+ {
+ rspamd::local_shared_ptr<deleter_test> pi(new deleter_test{t});
+
+ CHECK((!pi ? false : true));
+ CHECK(!!pi);
+ CHECK(pi.get() != nullptr);
+ CHECK(pi.use_count() == 1);
+ CHECK(pi.unique());
+ CHECK(t == false);
+ }
+
+ CHECK(t == true);
+
+ {
+ rspamd::local_shared_ptr<deleter_test> pi(new deleter_test{t});
+
+ CHECK((!pi ? false : true));
+ CHECK(!!pi);
+ CHECK(pi.get() != nullptr);
+ CHECK(pi.use_count() == 1);
+ CHECK(pi.unique());
+ CHECK(t == false);
+
+ rspamd::local_shared_ptr<deleter_test> pi2(pi);
+ CHECK(pi2 == pi);
+ CHECK(pi.use_count() == 2);
+ pi.reset();
+ CHECK(!(pi2 == pi));
+ CHECK(pi2.use_count() == 1);
+ CHECK(t == false);
+
+ pi = pi2;
+ CHECK(pi2 == pi);
+ CHECK(pi.use_count() == 2);
+ CHECK(t == false);
+ }
+
+ CHECK(t == true);
+ }
+
+ TEST_CASE("make_shared dtor")
+ {
+ bool t;
+
+ {
+ auto pi = rspamd::local_make_shared<deleter_test>(t);
+
+ CHECK((!pi ? false : true));
+ CHECK(!!pi);
+ CHECK(pi.get() != nullptr);
+ CHECK(pi.use_count() == 1);
+ CHECK(pi.unique());
+ CHECK(t == false);
+
+ rspamd::local_shared_ptr<deleter_test> pi2(pi);
+ CHECK(pi2 == pi);
+ CHECK(pi.use_count() == 2);
+ pi.reset();
+ CHECK(!(pi2 == pi));
+ CHECK(pi2.use_count() == 1);
+ CHECK(t == false);
+
+ pi = pi2;
+ CHECK(pi2 == pi);
+ CHECK(pi.use_count() == 2);
+ CHECK(t == false);
+ }
+
+ CHECK(t == true);
+ }
+
+ TEST_CASE("shared_ptr dtor")
+ {
+ bool t;
+
+ {
+ rspamd::local_shared_ptr<deleter_test> pi(new deleter_test{t});
+
+ CHECK((!pi ? false : true));
+ CHECK(!!pi);
+ CHECK(pi.get() != nullptr);
+ CHECK(pi.use_count() == 1);
+ CHECK(pi.unique());
+ CHECK(t == false);
+ }
+
+ CHECK(t == true);
+
+ {
+ rspamd::local_shared_ptr<deleter_test> pi(new deleter_test{t});
+
+ CHECK((!pi ? false : true));
+ CHECK(!!pi);
+ CHECK(pi.get() != nullptr);
+ CHECK(pi.use_count() == 1);
+ CHECK(pi.unique());
+ CHECK(t == false);
+
+ rspamd::local_shared_ptr<deleter_test> pi2(pi);
+ CHECK(pi2 == pi);
+ CHECK(pi.use_count() == 2);
+ pi.reset();
+ CHECK(!(pi2 == pi));
+ CHECK(pi2.use_count() == 1);
+ CHECK(t == false);
+
+ pi = pi2;
+ CHECK(pi2 == pi);
+ CHECK(pi.use_count() == 2);
+ CHECK(t == false);
+ }
+
+ CHECK(t == true);
+ }
+
+ TEST_CASE("weak_ptr")
+ {
+ bool t;
+
+ {
+ rspamd::local_shared_ptr<deleter_test> pi(new deleter_test{t});
+
+ CHECK((!pi ? false : true));
+ CHECK(!!pi);
+ CHECK(pi.get() != nullptr);
+ CHECK(pi.use_count() == 1);
+ CHECK(pi.unique());
+ CHECK(t == false);
+
+ rspamd::local_weak_ptr<deleter_test> wp(pi);
+ CHECK(wp.lock().get() != nullptr);
+ CHECK(pi.use_count() == 1);
+ CHECK(wp.use_count() == 1);
+ pi.reset();
+ CHECK(pi.use_count() == 0);
+ CHECK(wp.use_count() == 0);
+ }
+
+ CHECK(t == true);
+
+ rspamd::local_weak_ptr<deleter_test> wp;
+ {
+ rspamd::local_shared_ptr<deleter_test> pi(new deleter_test{t});
+ wp = pi;
+ CHECK(!wp.expired());
+ CHECK(wp.lock().get() != nullptr);
+ }
+
+ CHECK(t == true);
+ CHECK(wp.expired());
+ }
+
+ TEST_CASE("std::swap")
+ {
+ bool t;
+
+ {
+ rspamd::local_shared_ptr<deleter_test> pi(new deleter_test{t});
+ CHECK(pi.use_count() == 1);
+ CHECK(pi.unique());
+ CHECK(t == false);
+
+ rspamd::local_shared_ptr<deleter_test> pi1;
+ CHECK(pi1.get() == nullptr);
+ CHECK(pi1.use_count() == 0);
+ std::swap(pi1, pi);
+ CHECK(pi.use_count() == 0);
+ CHECK(pi.get() == nullptr);
+ CHECK(pi1.get() != nullptr);
+ std::swap(pi, pi1);
+ CHECK(pi.use_count() != 0);
+ CHECK(pi.get() != nullptr);
+ CHECK(pi1.get() == nullptr);
+ }
+
+ CHECK(t == true);
+ }
+
+ TEST_CASE("std::hash")
+ {
+ bool v;
+ deleter_test dt(v);
+ CHECK(std::hash<deleter_test>()(dt) == 42);
+ auto pi = rspamd::local_make_shared<deleter_test>(v);
+ rspamd::local_shared_ptr<deleter_test> pi1;
+ CHECK(std::hash<decltype(pi)>()(pi) == 42);
+ // No hash for nullptr, different from std::smart_pointers!
+ CHECK_THROWS(std::hash<decltype(pi)>()(pi1));
+ }
+}
+
+#endif//RSPAMD_RSPAMD_CXX_LOCAL_PTR_HXX
diff --git a/test/rspamd_cxx_unit.cxx b/test/rspamd_cxx_unit.cxx
new file mode 100644
index 0000000..3dde89d
--- /dev/null
+++ b/test/rspamd_cxx_unit.cxx
@@ -0,0 +1,86 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "rspamd.h"
+#include <memory>
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+#include "rspamd_cxx_unit_utils.hxx"
+#include "rspamd_cxx_local_ptr.hxx"
+#include "rspamd_cxx_unit_dkim.hxx"
+
+static gboolean verbose = false;
+static const GOptionEntry entries[] =
+ {
+ {"verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose,
+ "Enable verbose logging", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+
+int main(int argc, char **argv)
+{
+ struct rspamd_main *rspamd_main;
+ rspamd_mempool_t *pool;
+ struct rspamd_config *cfg;
+ GOptionContext *options_context;
+
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), NULL, 0);
+ rspamd_main = (struct rspamd_main *) rspamd_mempool_alloc0(pool, sizeof(*rspamd_main));
+ rspamd_main->server_pool = pool;
+ cfg = rspamd_config_new(RSPAMD_CONFIG_INIT_DEFAULT);
+ cfg->libs_ctx = rspamd_init_libs();
+ rspamd_main->cfg = cfg;
+ cfg->cfg_pool = pool;
+
+ options_context = g_option_context_new("- run rspamd cxx test");
+ g_option_context_add_main_entries(options_context, entries, NULL);
+ g_option_context_set_ignore_unknown_options(options_context, true);
+ g_option_context_set_help_enabled(options_context, false);
+
+ GError *error = NULL;
+
+ if (!g_option_context_parse(options_context, &argc, &argv, &error)) {
+ fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_option_context_free(options_context);
+ exit(1);
+ }
+
+ if (verbose) {
+ rspamd_main->logger = rspamd_log_open_emergency(rspamd_main->server_pool,
+ RSPAMD_LOG_FLAG_USEC | RSPAMD_LOG_FLAG_ENFORCED | RSPAMD_LOG_FLAG_RSPAMADM);
+
+ rspamd_log_set_log_level(rspamd_main->logger, G_LOG_LEVEL_DEBUG);
+ }
+ else {
+ rspamd_main->logger = rspamd_log_open_emergency(rspamd_main->server_pool,
+ RSPAMD_LOG_FLAG_RSPAMADM);
+ rspamd_log_set_log_level(rspamd_main->logger, G_LOG_LEVEL_MESSAGE);
+ }
+
+ doctest::Context context(argc, argv);
+ int res = context.run();
+
+ if (context.shouldExit()) {
+ return res;
+ }
+
+ rspamd_mempool_delete(pool);
+
+ return res;
+} \ No newline at end of file
diff --git a/test/rspamd_cxx_unit_dkim.hxx b/test/rspamd_cxx_unit_dkim.hxx
new file mode 100644
index 0000000..3f6b71a
--- /dev/null
+++ b/test/rspamd_cxx_unit_dkim.hxx
@@ -0,0 +1,173 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Detached unit tests for the dkim utils */
+
+#ifndef RSPAMD_RSPAMD_CXX_UNIT_DKIM_HXX
+#define RSPAMD_RSPAMD_CXX_UNIT_DKIM_HXX
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+#include "fmt/core.h"
+#include "libserver/dkim.h"
+
+#include <vector>
+#include <utility>
+#include <string>
+#include <tuple>
+
+TEST_SUITE("rspamd_dkim")
+{
+
+ TEST_CASE("rspamd_dkim_parse_key")
+ {
+ struct test_case {
+ std::string input;
+ bool is_valid;
+ std::string expected_id;
+ };
+ std::vector<test_case> cases{
+ {"p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCA"
+ "QEA5CeQZpoPbsS8lG41UI1rxTtOSqPrfgZzhrZsk0t9dIbFTvaoql/FLuYcbdUARc"
+ "5zuyXsDj1eSprOgcPT9PY9RoSUsY8i/jnD49DHXtMfXoBk0J6epNzbZqqWU+"
+ "TG02HwWNy/kf1h+OlAGQKJLgakivZ3nMMnUIPHkUjwvhbkaMXCI046XoqsEQ7KW"
+ "VKRoF3cK1cFXLo+bgO3sEJgGtvwzPodG0CqVu+gjehrjwdLnPhqyEspfI1IbL"
+ "lnNaq5pWei/B8pG6teV+y3t4yay5ZGktALJjlylKHVo2USkVYQTFQ9Ji25m2jupdCd"
+ "kn1FTuYNqh0Nzg3KPQHNVp7mlE7lfwIDAQAB",
+ true, "e40cc5c40ee29cb4f21d95c7f0dc9989"},
+ // Spaces before p
+ {" p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCA"
+ "QEA5CeQZpoPbsS8lG41UI1rxTtOSqPrfgZzhrZsk0t9dIbFTvaoql/FLuYcbdUARc"
+ "5zuyXsDj1eSprOgcPT9PY9RoSUsY8i/jnD49DHXtMfXoBk0J6epNzbZqqWU+"
+ "TG02HwWNy/kf1h+OlAGQKJLgakivZ3nMMnUIPHkUjwvhbkaMXCI046XoqsEQ7KW"
+ "VKRoF3cK1cFXLo+bgO3sEJgGtvwzPodG0CqVu+gjehrjwdLnPhqyEspfI1IbL"
+ "lnNaq5pWei/B8pG6teV+y3t4yay5ZGktALJjlylKHVo2USkVYQTFQ9Ji25m2jupdCd"
+ "kn1FTuYNqh0Nzg3KPQHNVp7mlE7lfwIDAQAB",
+ true, "e40cc5c40ee29cb4f21d95c7f0dc9989"},
+ // Spaces and bogus semicolon before p
+ {"; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCA"
+ "QEA5CeQZpoPbsS8lG41UI1rxTtOSqPrfgZzhrZsk0t9dIbFTvaoql/FLuYcbdUARc"
+ "5zuyXsDj1eSprOgcPT9PY9RoSUsY8i/jnD49DHXtMfXoBk0J6epNzbZqqWU+"
+ "TG02HwWNy/kf1h+OlAGQKJLgakivZ3nMMnUIPHkUjwvhbkaMXCI046XoqsEQ7KW"
+ "VKRoF3cK1cFXLo+bgO3sEJgGtvwzPodG0CqVu+gjehrjwdLnPhqyEspfI1IbL"
+ "lnNaq5pWei/B8pG6teV+y3t4yay5ZGktALJjlylKHVo2USkVYQTFQ9Ji25m2jupdCd"
+ "kn1FTuYNqh0Nzg3KPQHNVp7mlE7lfwIDAQAB",
+ true, "e40cc5c40ee29cb4f21d95c7f0dc9989"},
+ // Spaces after p
+ {"k=rsa; t=s; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCA"
+ "QEA5CeQZpoPbsS8lG41UI1rxTtOSqPrfgZzhrZsk0t9dIbFTvaoql/FLuYcbdUARc"
+ "5zuyXsDj1eSprOgcPT9PY9RoSUsY8i/jnD49DHXtMfXoBk0J6epNzbZqqWU+"
+ "TG02HwWNy/kf1h+OlAGQKJLgakivZ3nMMnUIPHkUjwvhbkaMXCI046XoqsEQ7KW"
+ "VKRoF3cK1cFXLo+bgO3sEJgGtvwzPodG0CqVu+gjehrjwdLnPhqyEspfI1IbL"
+ "lnNaq5pWei/B8pG6teV+y3t4yay5ZGktALJjlylKHVo2USkVYQTFQ9Ji25m2jupdCd"
+ "kn1FTuYNqh0Nzg3KPQHNVp7mlE7lfwIDAQAB ",
+ true, "e40cc5c40ee29cb4f21d95c7f0dc9989"},
+ // ; and spaces
+ {"k=rsa; t=s; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCA"
+ "QEA5CeQZpoPbsS8lG41UI1rxTtOSqPrfgZzhrZsk0t9dIbFTvaoql/FLuYcbdUARc"
+ "5zuyXsDj1eSprOgcPT9PY9RoSUsY8i/jnD49DHXtMfXoBk0J6epNzbZqqWU+"
+ "TG02HwWNy/kf1h+OlAGQKJLgakivZ3nMMnUIPHkUjwvhbkaMXCI046XoqsEQ7KW"
+ "VKRoF3cK1cFXLo+bgO3sEJgGtvwzPodG0CqVu+gjehrjwdLnPhqyEspfI1IbL"
+ "lnNaq5pWei/B8pG6teV+y3t4yay5ZGktALJjlylKHVo2USkVYQTFQ9Ji25m2jupdCd"
+ "kn1FTuYNqh0Nzg3KPQHNVp7mlE7lfwIDAQAB ; ",
+ true, "e40cc5c40ee29cb4f21d95c7f0dc9989"},
+ // ; and spaces around '=' sign
+ {"k=rsa; t=s; p = MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCA"
+ "QEA5CeQZpoPbsS8lG41UI1rxTtOSqPrfgZzhrZsk0t9dIbFTvaoql/FLuYcbdUARc"
+ "5zuyXsDj1eSprOgcPT9PY9RoSUsY8i/jnD49DHXtMfXoBk0J6epNzbZqqWU+"
+ "TG02HwWNy/kf1h+OlAGQKJLgakivZ3nMMnUIPHkUjwvhbkaMXCI046XoqsEQ7KW"
+ "VKRoF3cK1cFXLo+bgO3sEJgGtvwzPodG0CqVu+gjehrjwdLnPhqyEspfI1IbL"
+ "lnNaq5pWei/B8pG6teV+y3t4yay5ZGktALJjlylKHVo2USkVYQTFQ9Ji25m2jupdCd"
+ "kn1FTuYNqh0Nzg3KPQHNVp7mlE7lfwIDAQAB;",
+ true, "e40cc5c40ee29cb4f21d95c7f0dc9989"},
+ // ; and spaces around '=' sign + bad stuff
+ {"ololo k=rsa; t=s; p = MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCA"
+ "QEA5CeQZpoPbsS8lG41UI1rxTtOSqPrfgZzhrZsk0t9dIbFTvaoql/FLuYcbdUARc"
+ "5zuyXsDj1eSprOgcPT9PY9RoSUsY8i/jnD49DHXtMfXoBk0J6epNzbZqqWU+"
+ "TG02HwWNy/kf1h+OlAGQKJLgakivZ3nMMnUIPHkUjwvhbkaMXCI046XoqsEQ7KW"
+ "VKRoF3cK1cFXLo+bgO3sEJgGtvwzPodG0CqVu+gjehrjwdLnPhqyEspfI1IbL"
+ "lnNaq5pWei/B8pG6teV+y3t4yay5ZGktALJjlylKHVo2USkVYQTFQ9Ji25m2jupdCd"
+ "kn1FTuYNqh0Nzg3KPQHNVp7mlE7lfwIDAQAB;",
+ true, "e40cc5c40ee29cb4f21d95c7f0dc9989"},
+ // ; and spaces around '=' sign + bad stuff
+ {"ololo=trololo; k=rsa; t=s; "
+ "p = MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCA"
+ "QEA5CeQZpoPbsS8lG41UI1rxTtOSqPrfgZzhrZsk0t9dIbFTvaoql/FLuYcbdUARc"
+ "5zuyXsDj1eSprOgcPT9PY9RoSUsY8i/jnD49DHXtMfXoBk0J6epNzbZqqWU+"
+ "TG02HwWNy/kf1h+OlAGQKJLgakivZ3nMMnUIPHkUjwvhbkaMXCI046XoqsEQ7KW"
+ "VKRoF3cK1cFXLo+bgO3sEJgGtvwzPodG0CqVu+gjehrjwdLnPhqyEspfI1IbL"
+ "lnNaq5pWei/B8pG6teV+y3t4yay5ZGktALJjlylKHVo2USkVYQTFQ9Ji25m2jupdCd"
+ "kn1FTuYNqh0Nzg3KPQHNVp7mlE7lfwIDAQAB ",
+ true, "e40cc5c40ee29cb4f21d95c7f0dc9989"},
+ // Spaces within base64
+ {"p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCA"
+ " QEA5CeQZpoPbsS8lG41UI1rxTtOSqPrfgZzhrZsk0t9dIbFTvaoql/FLuYcbdUARc"
+ " 5zuyXsDj1eSprOgcPT9PY9RoSUsY8i/ jnD49DHXtMfXoBk0J6epNzbZqqWU+"
+ " TG02HwWNy/kf1h+OlAGQKJLgakivZ3nMMnUIPHkUjwvhbkaMXCI046XoqsEQ7KW"
+ " VKRoF3cK1cFXLo+bgO3sEJgGtvwzPodG0CqVu+gjehrjwdLnPhqyEspfI1IbL"
+ " lnNaq5pWei/B8pG6teV+y3t4yay5ZGktALJjlylKHVo2USkVYQTFQ9Ji25m2jupdCd"
+ " kn1FTuYNqh0Nzg3KPQHNVp7mlE7lfwIDAQAB",
+ true, "e40cc5c40ee29cb4f21d95c7f0dc9989"},
+ // Invalid RSA
+ {"ololo=trololo; k=rsa; t=s; "
+ "p = BADMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCA"
+ "QEA5CeQZpoPbsS8lG41UI1rxTtOSqPrfgZzhrZsk0t9dIbFTvaoql/FLuYcbdUARc"
+ "5zuyXsDj1eSprOgcPT9PY9RoSUsY8i/jnD49DHXtMfXoBk0J6epNzbZqqWU+"
+ "TG02HwWNy/kf1h+OlAGQKJLgakivZ3nMMnUIPHkUjwvhbkaMXCI046XoqsEQ7KW"
+ "VKRoF3cK1cFXLo+bgO3sEJgGtvwzPodG0CqVu+gjehrjwdLnPhqyEspfI1IbL"
+ "lnNaq5pWei/B8pG6teV+y3t4yay5ZGktALJjlylKHVo2USkVYQTFQ9Ji25m2jupdCd"
+ "kn1FTuYNqh0Nzg3KPQHNVp7mlE7lfwIDAQAB ",
+ false, ""},
+ // Invalid RSA for eddsa
+ {"ololo=trololo; k=ed25519; t=s; "
+ "p = MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCA"
+ "QEA5CeQZpoPbsS8lG41UI1rxTtOSqPrfgZzhrZsk0t9dIbFTvaoql/FLuYcbdUARc"
+ "5zuyXsDj1eSprOgcPT9PY9RoSUsY8i/jnD49DHXtMfXoBk0J6epNzbZqqWU+"
+ "TG02HwWNy/kf1h+OlAGQKJLgakivZ3nMMnUIPHkUjwvhbkaMXCI046XoqsEQ7KW"
+ "VKRoF3cK1cFXLo+bgO3sEJgGtvwzPodG0CqVu+gjehrjwdLnPhqyEspfI1IbL"
+ "lnNaq5pWei/B8pG6teV+y3t4yay5ZGktALJjlylKHVo2USkVYQTFQ9Ji25m2jupdCd"
+ "kn1FTuYNqh0Nzg3KPQHNVp7mlE7lfwIDAQAB ",
+ false, ""},
+ };
+
+ auto cur_test_idx = 0;
+ for (auto &&c: cases) {
+ SUBCASE(fmt::format("process DKIM record {}: {}", cur_test_idx++, c.input).c_str())
+ {
+ GError *err = nullptr;
+ gsize klen = c.input.size();
+ auto *key = rspamd_dkim_parse_key(c.input.c_str(), &klen, &err);
+ if (c.is_valid) {
+ REQUIRE_MESSAGE(key != nullptr, (err ? err->message : "unknown error"));
+ char hexbuf[RSPAMD_DKIM_KEY_ID_LEN * 2 + 1];
+ auto *id = rspamd_dkim_key_id(key);
+ REQUIRE(id != nullptr);
+
+ auto hexlen = rspamd_encode_hex_buf(id, RSPAMD_DKIM_KEY_ID_LEN, hexbuf,
+ sizeof(hexbuf));
+ CHECK(hexlen > 0);
+ CHECK(std::string{hexbuf, (std::size_t) hexlen} == c.expected_id);
+ rspamd_dkim_key_free(key);
+ }
+ else {
+ CHECK(key == nullptr);
+ }
+ }
+ }
+ }
+}
+
+#endif
diff --git a/test/rspamd_cxx_unit_utils.hxx b/test/rspamd_cxx_unit_utils.hxx
new file mode 100644
index 0000000..126253f
--- /dev/null
+++ b/test/rspamd_cxx_unit_utils.hxx
@@ -0,0 +1,209 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Detached unit tests for the utils */
+
+#ifndef RSPAMD_RSPAMD_CXX_UNIT_UTILS_HXX
+#define RSPAMD_RSPAMD_CXX_UNIT_UTILS_HXX
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+#include "libmime/mime_headers.h"
+#include "contrib/libottery/ottery.h"
+#include "libcryptobox/cryptobox.h"
+
+#include <vector>
+#include <utility>
+#include <string>
+
+extern "C" long rspamd_http_parse_keepalive_timeout(const rspamd_ftok_t *tok);
+
+TEST_SUITE("rspamd_utils")
+{
+
+ TEST_CASE("rspamd_strip_smtp_comments_inplace")
+ {
+ std::vector<std::pair<std::string, std::string>> cases{
+ {"abc", "abc"},
+ {"abc(foo)", "abc"},
+ {"abc(foo()", "abc"},
+ {"abc(foo))", "abc)"},
+ {"abc(foo(bar))", "abc"},
+ {"(bar)abc(foo)", "abc"},
+ {"ab(ololo)c(foo)", "abc"},
+ {"ab(olo\\)lo)c(foo)", "abc"},
+ {"ab(trol\\\1lo)c(foo)", "abc"},
+ {"\\ab(trol\\\1lo)c(foo)", "abc"},
+ {"", ""},
+ {"<test_id@example.net> (added by postmaster@example.net)", "<test_id@example.net> "}};
+
+ for (const auto &c: cases) {
+ SUBCASE(("strip comments in " + c.first).c_str())
+ {
+ auto *cpy = new char[c.first.size()];
+ memcpy(cpy, c.first.data(), c.first.size());
+ auto nlen = rspamd_strip_smtp_comments_inplace(cpy, c.first.size());
+ CHECK(std::string{cpy, nlen} == c.second);
+ delete[] cpy;
+ }
+ }
+ }
+
+ TEST_CASE("rspamd_http_parse_keepalive_timeout")
+ {
+ std::vector<std::pair<std::string, long>> cases{
+ {"timeout=5, max=1000", 5},
+ {"max=1000, timeout=5", 5},
+ {"max=1000, timeout=", -1},
+ {"max=1000, timeout=0", 0},
+ {"max=1000, timeout=-5", -1},
+ {"timeout=5", 5},
+ {" timeout=5; ", 5},
+ {"timeout = 5", 5},
+ };
+
+ for (const auto &c: cases) {
+ SUBCASE(("parse http keepalive header " + c.first).c_str())
+ {
+ rspamd_ftok_t t;
+ t.begin = c.first.data();
+ t.len = c.first.size();
+ auto res = rspamd_http_parse_keepalive_timeout(&t);
+ CHECK(res == c.second);
+ }
+ }
+ }
+
+ TEST_CASE("rspamd_fstring_gzip tests")
+ {
+ rspamd_fstring_t *fstr;
+
+ // Test empty data compression
+ SUBCASE("Empty data")
+ {
+ fstr = rspamd_fstring_new_init("", 0);
+ gboolean result = rspamd_fstring_gzip(&fstr);
+ CHECK(result == TRUE);
+ CHECK(fstr->len == 20);
+ result = rspamd_fstring_gunzip(&fstr);
+ CHECK(result == TRUE);
+ CHECK(fstr->len == 0);
+ rspamd_fstring_free(fstr);
+ }
+
+ SUBCASE("Non empty data")
+ {
+ fstr = RSPAMD_FSTRING_LIT("helohelo");
+ gboolean result = rspamd_fstring_gzip(&fstr);
+ CHECK(result == TRUE);
+ CHECK(fstr->len == 26);
+ result = rspamd_fstring_gunzip(&fstr);
+ CHECK(result == TRUE);
+ CHECK(memcmp(fstr->str, "helohelo", fstr->len) == 0);
+ CHECK(fstr->len == sizeof("helohelo") - 1);
+ rspamd_fstring_free(fstr);
+ }
+
+ SUBCASE("Some real compression")
+ {
+ fstr = rspamd_fstring_sized_new(sizeof("helohelo") * 1024);
+ for (int i = 0; i < 1024; i++) {
+ fstr = rspamd_fstring_append(fstr, "helohelo", sizeof("helohelo") - 1);
+ }
+ gboolean result = rspamd_fstring_gzip(&fstr);
+ CHECK(result == TRUE);
+ CHECK(fstr->len == 49);
+ result = rspamd_fstring_gunzip(&fstr);
+ CHECK(result == TRUE);
+ CHECK(memcmp(fstr->str, "helohelo", sizeof("helohelo") - 1) == 0);
+ CHECK(fstr->len == (sizeof("helohelo") - 1) * 1024);
+ rspamd_fstring_free(fstr);
+ }
+
+ SUBCASE("Random data compression")
+ {
+ rspamd_cryptobox_fast_hash_state_t hst;
+ rspamd_cryptobox_fast_hash_init(&hst, 0);
+ fstr = rspamd_fstring_sized_new(30 * 1024 * 1024);
+ for (int i = 0; i < 30 * 1024; i++) {
+ char tmp[1024];
+ ottery_rand_bytes(tmp, sizeof(tmp));
+ fstr = rspamd_fstring_append(fstr, tmp, sizeof(tmp));
+ rspamd_cryptobox_fast_hash_update(&hst, tmp, sizeof(tmp));
+ }
+ auto crc = rspamd_cryptobox_fast_hash(fstr->str, fstr->len, 0);
+ CHECK(crc == rspamd_cryptobox_fast_hash_final(&hst));
+ gboolean result = rspamd_fstring_gzip(&fstr);
+ CHECK(result == TRUE);
+ // Assuming there are no miracles
+ CHECK(fstr->len >= 30 * 1024 * 1024);
+ result = rspamd_fstring_gunzip(&fstr);
+ CHECK(result == TRUE);
+ CHECK(fstr->len == 30 * 1024 * 1024);
+ auto final_crc = rspamd_cryptobox_fast_hash(fstr->str, fstr->len, 0);
+ CHECK(crc == final_crc);
+ rspamd_fstring_free(fstr);
+ }
+ }
+
+ TEST_CASE("rspamd_message_header_unfold_inplace")
+ {
+ std::vector<std::pair<std::string, std::string>> cases{
+ {"abc", "abc"},
+ {"abc\r\n def", "abc def"},
+ {"abc\r\n\tdef", "abc def"},
+ {"abc\r\n\tdef\r\n\tghi", "abc def ghi"},
+ {"abc\r\n\tdef\r\n\tghi\r\n", "abc def ghi"},
+ {"abc\r\n\tdef\r\n\tghi\r\n\t", "abc def ghi"},
+ {"abc\r\n\tdef\r\n\tghi\r\n\tjkl", "abc def ghi jkl"},
+ {"abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n", "abc def ghi jkl"},
+ {"abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\t", "abc def ghi jkl"},
+ {"abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\tmno", "abc def ghi jkl mno"},
+ {"abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\tmno\r\n", "abc def ghi jkl mno"},
+ {"abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\tmno\r\n\t", "abc def ghi jkl mno"},
+ {"abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\tmno\r\n\tpqr", "abc def ghi jkl mno pqr"},
+ {"abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\tmno\r\n\tpqr\r\n", "abc def ghi jkl mno pqr"},
+ {"abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\tmno\r\n\tpqr\r\n\t", "abc def ghi jkl mno pqr"},
+ {"abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\tmno\r\n\tpqr\r\n\tstu", "abc def ghi jkl mno pqr stu"},
+ // Newline at the end
+ {
+ "abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\tmno\r\n\tpqr\r\n\tstu\r\n", "abc def ghi jkl mno pqr stu"},
+ // Spaces at the end
+ {
+ "abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\tmno\r\n\tpqr\r\n\tstu\r\n\t", "abc def ghi jkl mno pqr stu"},
+ // Multiple spaces at the end
+ {
+ "abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\tmno\r\n\tpqr\r\n\tstu\r\n\t ", "abc def ghi jkl mno pqr stu"},
+ // Multiple spaces in middle
+ {
+ "abc\r\n\tdef\r\n\tghi\r\n\tjkl\r\n\tmno\r\n\tpqr\r\n\tstu \r\n\t a", "abc def ghi jkl mno pqr stu a"},
+ };
+
+ for (const auto &c: cases) {
+ SUBCASE(("unfold header " + c.second).c_str())
+ {
+ auto *cpy = new char[c.first.size()];
+ memcpy(cpy, c.first.data(), c.first.size());
+ auto nlen = rspamd_message_header_unfold_inplace(cpy, c.first.size());
+ CHECK(std::string{cpy, nlen} == c.second);
+ delete[] cpy;
+ }
+ }
+ }
+}
+
+#endif
diff --git a/test/rspamd_dkim_test.c b/test/rspamd_dkim_test.c
new file mode 100644
index 0000000..57e4d5a
--- /dev/null
+++ b/test/rspamd_dkim_test.c
@@ -0,0 +1,86 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "tests.h"
+#include "rspamd.h"
+#include "dkim.h"
+
+static const gchar test_dkim_sig[] = "v=1; a=rsa-sha256; c=relaxed/relaxed; "
+ "d=highsecure.ru; s=dkim; t=1410516996; "
+ "bh=guFoWYHWVzFRqVyAQebnvPcdm7bUQo7pRHt/uIHD7gs=; "
+ "h=Message-ID:Date:From:MIME-Version:To:Subject:Content-Type:Content-Transfer-Encoding; "
+ "b=PCiECkOaPFb99DW+gApgfmdlTUo6XN6YXjnj52Cxoz2FoA857B0ZHFgeQe4JAKHuhW"
+ "oq3BLHap0GcMTTpSOgfQOKa8Df35Ns11JoOFjdBQ8GpM99kOrJP+vZcT8b7AMfthYm0Kwy"
+ "D9TjlkpScuoY5LjsWVnijh9dSNVLFqLatzg=;";
+
+extern struct ev_loop *event_loop;
+#if 0
+static void
+test_key_handler (rspamd_dkim_key_t *key, gsize keylen, rspamd_dkim_context_t *ctx, gpointer ud, GError *err)
+{
+ struct rspamd_async_session *s = ud;
+ g_assert (key != NULL);
+
+ rspamd_session_destroy (s);
+}
+
+static gboolean
+session_fin (gpointer unused)
+{
+ struct timeval tv;
+
+ tv.tv_sec = 0;
+ tv.tv_usec = 0;
+ event_loopexit (&tv);
+
+ return TRUE;
+}
+#endif
+void rspamd_dkim_test_func(void)
+{
+#if 0
+ rspamd_dkim_context_t *ctx;
+ rspamd_dkim_key_t *key;
+ rspamd_mempool_t *pool;
+ struct rspamd_dns_resolver *resolver;
+ struct rspamd_config *cfg;
+ GError *err = NULL;
+ struct rspamd_async_session *s;
+
+ cfg = (struct rspamd_config *)g_malloc (sizeof (struct rspamd_config));
+ bzero (cfg, sizeof (struct rspamd_config));
+ cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
+ cfg->dns_retransmits = 2;
+ cfg->dns_timeout = 0.5;
+
+ pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
+
+ resolver = dns_resolver_init (NULL, base, cfg);
+
+ g_assert (resolver != NULL);
+
+ ctx = rspamd_create_dkim_context (test_dkim_sig, pool, 0, &err);
+
+ g_assert (ctx != NULL);
+
+ /* Key part */
+ s = rspamd_session_create (pool, session_fin, NULL, NULL, NULL);
+
+ g_assert (rspamd_get_dkim_key (ctx, resolver, s, test_key_handler, s));
+
+ event_base_loop (base, 0);
+#endif
+}
diff --git a/test/rspamd_dns_test.c b/test/rspamd_dns_test.c
new file mode 100644
index 0000000..d779193
--- /dev/null
+++ b/test/rspamd_dns_test.c
@@ -0,0 +1,103 @@
+
+#include "config.h"
+#include "tests.h"
+#include "dns.h"
+#include "logger.h"
+#include "rspamd.h"
+#include "async_session.h"
+#include "cfg_file.h"
+
+static guint requests = 0;
+extern struct ev_loop *event_loop;
+struct rspamd_dns_resolver *resolver;
+
+gboolean
+session_fin(gpointer unused)
+{
+ ev_break(event_loop, EVBREAK_ALL);
+
+ return TRUE;
+}
+
+static void
+test_dns_cb(struct rdns_reply *reply, gpointer arg)
+{
+ struct rdns_reply_entry *cur;
+ const struct rdns_request_name *name =
+ rdns_request_get_name(reply->request, NULL);
+
+ msg_debug("got reply with code %s for request %s",
+ rdns_strerror(reply->code), name->name);
+ if (reply->code == RDNS_RC_NOERROR) {
+ cur = reply->entries;
+ while (cur) {
+ switch (cur->type) {
+ case RDNS_REQUEST_A:
+ msg_debug("got ip: %s", inet_ntoa(cur->content.a.addr));
+ break;
+ case RDNS_REQUEST_PTR:
+ msg_debug("got name %s", cur->content.ptr.name);
+ break;
+ case RDNS_REQUEST_TXT:
+ msg_debug("got txt %s", cur->content.txt.data);
+ break;
+ case RDNS_REQUEST_SPF:
+ msg_debug("got spf %s", cur->content.txt.data);
+ break;
+ case RDNS_REQUEST_SRV:
+ msg_debug("got srv pri: %d, weight: %d, port: %d, target: %s", cur->content.srv.weight,
+ cur->content.srv.priority, cur->content.srv.port, cur->content.srv.target);
+ break;
+ case RDNS_REQUEST_MX:
+ msg_debug("got mx %s:%d", cur->content.mx.name, cur->content.mx.priority);
+ break;
+ }
+ cur = cur->next;
+ }
+ }
+ if (--requests == 0) {
+ session_fin(NULL);
+ }
+}
+
+void rspamd_dns_test_func(void)
+{
+
+ struct rspamd_config *cfg;
+ rspamd_mempool_t *pool;
+ struct rspamd_async_session *s;
+
+ cfg = (struct rspamd_config *) g_malloc(sizeof(struct rspamd_config));
+ bzero(cfg, sizeof(struct rspamd_config));
+ cfg->cfg_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), NULL, 0);
+ cfg->dns_retransmits = 2;
+ cfg->dns_timeout = 0.5;
+
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), NULL, 0);
+
+ s = rspamd_session_create(pool, session_fin, NULL, NULL, NULL);
+
+ resolver = rspamd_dns_resolver_init(NULL, event_loop, cfg);
+
+ requests++;
+ g_assert(rspamd_dns_resolver_request(resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_A, "google.com"));
+ requests++;
+ g_assert(rspamd_dns_resolver_request(resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_PTR, "81.19.70.3"));
+ requests++;
+ g_assert(rspamd_dns_resolver_request(resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_MX, "rambler.ru"));
+ requests++;
+ g_assert(rspamd_dns_resolver_request(resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_TXT, "rambler.ru"));
+ requests++;
+ g_assert(rspamd_dns_resolver_request(resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_TXT, "google.com"));
+ requests++;
+ g_assert(rspamd_dns_resolver_request(resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_SPF, "rambler.ru"));
+ requests++;
+ g_assert(rspamd_dns_resolver_request(resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_SRV,
+ "_xmpp-server._tcp.jabber.org"));
+ requests++;
+ g_assert(rspamd_dns_resolver_request(resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_TXT, "non-existent.arpa"));
+
+ g_assert(resolver != NULL);
+
+ ev_run(event_loop, 0);
+}
diff --git a/test/rspamd_heap_test.c b/test/rspamd_heap_test.c
new file mode 100644
index 0000000..dcc7bbc
--- /dev/null
+++ b/test/rspamd_heap_test.c
@@ -0,0 +1,182 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "rspamd.h"
+#include "heap.h"
+#include "ottery.h"
+
+static const guint niter = 100500;
+static const guint nrem = 100;
+
+static inline struct rspamd_min_heap_elt *
+new_elt(guint pri)
+{
+ struct rspamd_min_heap_elt *elt;
+
+ elt = g_slice_alloc0(sizeof(*elt));
+ elt->pri = pri;
+
+ return elt;
+}
+
+static gdouble
+heap_nelts_test(guint nelts)
+{
+ struct rspamd_min_heap *heap;
+ struct rspamd_min_heap_elt *elts;
+ gdouble t1, t2;
+ guint i;
+
+ heap = rspamd_min_heap_create(nelts);
+ /* Preallocate all elts */
+ elts = g_slice_alloc(sizeof(*elts) * nelts);
+
+ for (i = 0; i < nelts; i++) {
+ elts[i].pri = ottery_rand_uint32() % G_MAXINT32 + 1;
+ elts[i].idx = 0;
+ }
+
+ t1 = rspamd_get_virtual_ticks();
+ for (i = 0; i < nelts; i++) {
+ rspamd_min_heap_push(heap, &elts[i]);
+ }
+
+ for (i = 0; i < nelts; i++) {
+ (void) rspamd_min_heap_pop(heap);
+ }
+ t2 = rspamd_get_virtual_ticks();
+
+ g_slice_free1(sizeof(*elts) * nelts, elts);
+ rspamd_min_heap_destroy(heap);
+
+ return (t2 - t1);
+}
+
+void rspamd_heap_test_func(void)
+{
+ struct rspamd_min_heap *heap;
+ struct rspamd_min_heap_elt *elt, *telt;
+ guint i;
+ guint prev;
+ gdouble t[16];
+
+ /* Push + update */
+ heap = rspamd_min_heap_create(32);
+ elt = new_elt(2);
+ elt->data = GINT_TO_POINTER(1);
+ rspamd_min_heap_push(heap, elt);
+ elt = new_elt(3);
+ elt->data = GINT_TO_POINTER(2);
+ rspamd_min_heap_push(heap, elt);
+ elt = new_elt(4);
+ elt->data = GINT_TO_POINTER(3);
+ rspamd_min_heap_push(heap, elt);
+
+ rspamd_min_heap_update_elt(heap, elt, 0);
+ elt = rspamd_min_heap_pop(heap);
+ g_assert(elt->data == GINT_TO_POINTER(3));
+
+ rspamd_min_heap_destroy(heap);
+
+ /* Push + remove */
+ heap = rspamd_min_heap_create(32);
+ elt = new_elt(2);
+ elt->data = GINT_TO_POINTER(1);
+ rspamd_min_heap_push(heap, elt);
+ rspamd_min_heap_remove_elt(heap, elt);
+ elt = new_elt(3);
+ elt->data = GINT_TO_POINTER(2);
+ rspamd_min_heap_push(heap, elt);
+ elt = rspamd_min_heap_pop(heap);
+ g_assert(elt->data == GINT_TO_POINTER(2));
+ elt = rspamd_min_heap_pop(heap);
+ g_assert(elt == NULL);
+
+ /* Push + push + remove + pop */
+ elt = new_elt(2);
+ elt->data = GINT_TO_POINTER(1);
+ rspamd_min_heap_push(heap, elt);
+ telt = elt;
+ elt = new_elt(3);
+ elt->data = GINT_TO_POINTER(2);
+ rspamd_min_heap_push(heap, elt);
+ rspamd_min_heap_remove_elt(heap, telt);
+ elt = rspamd_min_heap_pop(heap);
+ g_assert(elt->data == GINT_TO_POINTER(2));
+ rspamd_min_heap_destroy(heap);
+
+ /* Bulk test */
+ heap = rspamd_min_heap_create(32);
+
+ for (i = 100; i > 0; i--) {
+ elt = new_elt(i - 1);
+ rspamd_min_heap_push(heap, elt);
+ }
+
+ for (i = 0; i < 100; i++) {
+ elt = rspamd_min_heap_pop(heap);
+ g_assert(elt->pri == i);
+ }
+
+ rspamd_min_heap_destroy(heap);
+
+ /* Fuzz test */
+ heap = rspamd_min_heap_create(128);
+
+ /* Add */
+ for (i = 0; i < niter; i++) {
+ elt = new_elt(ottery_rand_uint32() % G_MAXINT32 + 1);
+ rspamd_min_heap_push(heap, elt);
+ }
+
+ /* Remove */
+ for (i = 0; i < nrem; i++) {
+ elt = rspamd_min_heap_index(heap, ottery_rand_uint32() % niter);
+ rspamd_min_heap_remove_elt(heap, elt);
+ }
+
+ /* Update */
+ for (i = 0; i < niter / 10; i++) {
+ elt = rspamd_min_heap_index(heap, ottery_rand_uint32() % (niter - nrem));
+ rspamd_min_heap_update_elt(heap, elt,
+ ottery_rand_uint32() % G_MAXINT32 + 1);
+ }
+
+ prev = 0;
+
+ /* Pop and check invariant */
+ for (i = 0; i < niter - nrem; i++) {
+ elt = rspamd_min_heap_pop(heap);
+
+ if (prev != 0) {
+ g_assert(elt->pri >= prev);
+ }
+
+ prev = elt->pri;
+ }
+
+ rspamd_min_heap_destroy(heap);
+
+ /* Complexity test (should be O(n * logn) */
+ for (i = 1; i <= G_N_ELEMENTS(t); i++) {
+ t[i - 1] = heap_nelts_test(0x1 << (i + 4));
+ }
+
+ for (i = 1; i <= G_N_ELEMENTS(t); i++) {
+ rspamd_printf("Elements: %d, time: %.4f\n", 0x1 << (i + 4), t[i - 1]);
+ }
+}
diff --git a/test/rspamd_lua_pcall_vs_resume_test.c b/test/rspamd_lua_pcall_vs_resume_test.c
new file mode 100644
index 0000000..0502378
--- /dev/null
+++ b/test/rspamd_lua_pcall_vs_resume_test.c
@@ -0,0 +1,160 @@
+/*-
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "rspamd.h"
+#include "util.h"
+#include "lua/lua_common.h"
+#include "lua/lua_thread_pool.h"
+#include "unix-std.h"
+
+static const char *lua_src_name = "lua/pcall_test.lua";
+extern gchar *argv0_dirname;
+
+extern struct rspamd_main *rspamd_main;
+
+const int N = 20000;
+
+
+static gdouble
+test_pcall(lua_State *L, gint function_call)
+{
+ gdouble t1, t2;
+ gint i;
+ t1 = rspamd_get_virtual_ticks();
+
+ for (i = 0; i < N; i++) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, function_call);
+ lua_pcall(L, 0, 1, 0);
+ lua_pop(L, 1);
+ }
+
+ t2 = rspamd_get_virtual_ticks();
+
+ return t2 - t1;
+}
+
+static gdouble
+test_resume(lua_State *L, gint function_call)
+{
+ gdouble t1, t2;
+ gint i;
+ t1 = rspamd_get_virtual_ticks();
+
+ for (i = 0; i < N; i++) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, function_call);
+#if LUA_VERSION_NUM < 502
+ lua_resume(L, 0);
+#elif LUA_VERSION_NUM >= 504
+ lua_resume(L, NULL, 0, NULL);
+#else
+ lua_resume(L, NULL, 0);
+#endif
+ lua_pop(L, 1);
+ }
+
+ t2 = rspamd_get_virtual_ticks();
+
+ return t2 - t1;
+}
+
+static gdouble
+test_resume_get_thread(gint function_call)
+{
+ gdouble t1, t2;
+ gint i;
+ struct thread_entry *ent;
+
+ t1 = rspamd_get_virtual_ticks();
+
+ for (i = 0; i < N; i++) {
+ ent = lua_thread_pool_get_for_config(rspamd_main->cfg);
+
+ lua_rawgeti(ent->lua_state, LUA_REGISTRYINDEX, function_call);
+#if LUA_VERSION_NUM < 502
+ lua_resume(ent->lua_state, 0);
+#elif LUA_VERSION_NUM >= 504
+ lua_resume(ent->lua_state, NULL, 0, NULL);
+#else
+ lua_resume(ent->lua_state, NULL, 0);
+#endif
+ lua_pop(ent->lua_state, 1);
+
+ lua_thread_pool_return(rspamd_main->cfg->lua_thread_pool, ent);
+ }
+
+ t2 = rspamd_get_virtual_ticks();
+
+ return t2 - t1;
+}
+
+static gdouble
+test_resume_get_new_thread(gint function_call)
+{
+ gdouble t1, t2;
+ gint i;
+ struct thread_entry *ent;
+
+ t1 = rspamd_get_virtual_ticks();
+
+ for (i = 0; i < N; i++) {
+ ent = lua_thread_pool_get_for_task(rspamd_main->cfg->lua_thread_pool);
+
+ lua_rawgeti(ent->lua_state, LUA_REGISTRYINDEX, function_call);
+#if LUA_VERSION_NUM < 502
+ lua_resume(ent->lua_state, 0);
+#elif LUA_VERSION_NUM >= 504
+ lua_resume(ent->lua_state, NULL, 0, NULL);
+#else
+ lua_resume(ent->lua_state, NULL, 0);
+#endif
+ lua_pop(ent->lua_state, 1);
+
+ /* lua_thread_pool_return (rspamd_main->cfg->lua_thread_pool, ent); */
+ }
+
+ t2 = rspamd_get_virtual_ticks();
+
+ return t2 - t1;
+}
+
+void rspamd_lua_lua_pcall_vs_resume_test_func(void)
+{
+ lua_State *L = rspamd_main->cfg->lua_state;
+ gchar *lua_src;
+ gdouble t1, reference;
+
+ lua_src = g_build_filename(argv0_dirname, lua_src_name, NULL);
+ if (luaL_dofile(L, lua_src) != 0) {
+ msg_err("failed to load test file: %s ", lua_tostring(L, -1));
+ g_assert(0);
+ }
+ g_free(lua_src);
+
+ gint function_call = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ msg_info("calling");
+
+ reference = t1 = test_pcall(L, function_call);
+ msg_notice("pcall stat: ts: %1.5f, avg:%1.5f, slow=%1.2f", t1, t1 / (gdouble) N, t1 / reference);
+
+ t1 = test_resume(L, function_call);
+ msg_notice("resume stat: ts: %1.5f, avg:%1.5f, slow=%1.2f", t1, t1 / (gdouble) N, t1 / reference);
+
+ t1 = test_resume_get_thread(function_call);
+ msg_notice("resume+get thread stat: ts: %1.5f, avg:%1.5f, slow=%1.2f", t1, t1 / (gdouble) N, t1 / reference);
+
+ t1 = test_resume_get_new_thread(function_call);
+ msg_notice("resume+get [new] thread stat: ts: %1.5f, avg:%1.5f, slow=%1.2f", t1, t1 / (gdouble) N, t1 / reference);
+}
diff --git a/test/rspamd_lua_test.c b/test/rspamd_lua_test.c
new file mode 100644
index 0000000..87cc9b7
--- /dev/null
+++ b/test/rspamd_lua_test.c
@@ -0,0 +1,151 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "util.h"
+#include "lua/lua_common.h"
+#include "unix-std.h"
+
+#ifdef HAVE_GLOB_H
+#include <glob.h>
+#endif
+
+static const char *lua_src_name = "lua/tests.lua";
+extern gchar *lua_test;
+extern gchar *lua_test_case;
+extern gchar *argv0_dirname;
+extern struct rspamd_main *rspamd_main;
+
+static int
+traceback(lua_State *L)
+{
+ if (!lua_isstring(L, 1)) {
+ return 1;
+ }
+
+ lua_getglobal(L, "debug");
+
+ if (!lua_istable(L, -1)) {
+ lua_pop(L, 1);
+ return 1;
+ }
+
+ lua_getfield(L, -1, "traceback");
+
+ if (!lua_isfunction(L, -1)) {
+ lua_pop(L, 2);
+ return 1;
+ }
+ lua_pushvalue(L, 1);
+ lua_pushinteger(L, 2);
+ lua_call(L, 2, 1);
+
+ return 1;
+}
+
+_Noreturn void
+rspamd_lua_test_func(void)
+{
+ lua_State *L = (lua_State *) rspamd_main->cfg->lua_state;
+ gchar *lua_src, *rp, rp_buf[PATH_MAX], path_buf[PATH_MAX], *tmp, *dir, *pattern;
+ const gchar *old_path;
+ glob_t globbuf;
+ gint i, len;
+
+ rspamd_lua_set_env(L, NULL, NULL, NULL);
+ rspamd_lua_set_globals(rspamd_main->cfg, L);
+ rspamd_lua_start_gc(rspamd_main->cfg);
+
+ if (lua_test_case) {
+ lua_pushstring(L, lua_test_case);
+ lua_setglobal(L, "test_pattern");
+ }
+
+ rspamd_printf("Starting lua tests\n");
+
+ lua_src = g_build_filename(argv0_dirname, lua_src_name, NULL);
+ if ((rp = realpath(lua_src, rp_buf)) == NULL) {
+ msg_err("cannot find path %s: %s",
+ lua_src, strerror(errno));
+ g_assert(0);
+ }
+ g_free(lua_src);
+
+ tmp = g_strdup(rp);
+ dir = dirname(tmp);
+ /* Set lua path */
+ lua_getglobal(L, "package");
+ lua_getfield(L, -1, "path");
+ old_path = luaL_checkstring(L, -1);
+
+ rspamd_snprintf(path_buf, sizeof(path_buf), "%s;%s/?.lua;%s/unit/?.lua",
+ old_path, dir, dir);
+ lua_pop(L, 1);
+ lua_pushstring(L, path_buf);
+ lua_setfield(L, -2, "path");
+ lua_pop(L, 1);
+
+ lua_newtable(L);
+
+ globbuf.gl_offs = 0;
+ len = strlen(dir) + sizeof("/unit/") + sizeof("*.lua");
+ pattern = g_malloc(len);
+ rspamd_snprintf(pattern, len, "%s/unit/%s", dir, "*.lua");
+
+ gint lua_test_len = 0;
+ gint inserted_file = 1;
+ gint path_start;
+ if (lua_test) {
+ lua_test_len = strlen(lua_test);
+ }
+ if (glob(pattern, GLOB_DOOFFS, NULL, &globbuf) == 0) {
+ for (i = 0; i < (gint) globbuf.gl_pathc; i++) {
+ if (lua_test) {
+ path_start = strlen(globbuf.gl_pathv[i]) - lua_test_len;
+ if (path_start < 0 ||
+ strncmp(globbuf.gl_pathv[i] + path_start, lua_test, lua_test_len) != 0) {
+ continue;
+ }
+ }
+
+ lua_pushinteger(L, inserted_file);
+ lua_pushstring(L, globbuf.gl_pathv[i]);
+ lua_settable(L, -3);
+
+ inserted_file++;
+ }
+ globfree(&globbuf);
+ g_free(pattern);
+ }
+ else {
+ msg_err("pattern %s doesn't match: %s", pattern,
+ strerror(errno));
+ g_assert(0);
+ }
+
+ lua_setglobal(L, "tests_list");
+ rspamd_lua_set_path(L, NULL, NULL);
+
+ lua_pushcfunction(L, traceback);
+ luaL_loadfile(L, rp);
+
+ if (lua_pcall(L, 0, 0, lua_gettop(L) - 1) != 0) {
+ msg_err("run test failed: %s", lua_tostring(L, -1));
+ g_assert(0);
+ }
+
+ exit(EXIT_SUCCESS);
+}
diff --git a/test/rspamd_mem_pool_test.c b/test/rspamd_mem_pool_test.c
new file mode 100644
index 0000000..626cc07
--- /dev/null
+++ b/test/rspamd_mem_pool_test.c
@@ -0,0 +1,35 @@
+#include "config.h"
+#include "mem_pool.h"
+#include "tests.h"
+#include "unix-std.h"
+#include <math.h>
+
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+
+#define TEST_BUF "test buffer"
+#define TEST2_BUF "test buffertest buffer"
+
+void rspamd_mem_pool_test_func(void)
+{
+ rspamd_mempool_t *pool;
+ rspamd_mempool_stat_t st;
+ char *tmp, *tmp2, *tmp3;
+
+ pool = rspamd_mempool_new(sizeof(TEST_BUF), NULL, 0);
+ tmp = rspamd_mempool_alloc(pool, sizeof(TEST_BUF));
+ tmp2 = rspamd_mempool_alloc(pool, sizeof(TEST_BUF) * 2);
+ tmp3 = rspamd_mempool_alloc_shared(pool, sizeof(TEST_BUF));
+
+ snprintf(tmp, sizeof(TEST_BUF), "%s", TEST_BUF);
+ snprintf(tmp2, sizeof(TEST_BUF) * 2, "%s", TEST2_BUF);
+ snprintf(tmp3, sizeof(TEST_BUF), "%s", TEST_BUF);
+
+ g_assert(strncmp(tmp, TEST_BUF, sizeof(TEST_BUF)) == 0);
+ g_assert(strncmp(tmp2, TEST2_BUF, sizeof(TEST2_BUF)) == 0);
+ g_assert(strncmp(tmp3, TEST_BUF, sizeof(TEST_BUF)) == 0);
+
+ rspamd_mempool_delete(pool);
+ rspamd_mempool_stat(&st);
+}
diff --git a/test/rspamd_radix_test.c b/test/rspamd_radix_test.c
new file mode 100644
index 0000000..8d31dc1
--- /dev/null
+++ b/test/rspamd_radix_test.c
@@ -0,0 +1,376 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "radix.h"
+#include "ottery.h"
+#include "btrie.h"
+
+const gsize max_elts = 500 * 1024;
+const gint lookup_cycles = 1 * 1024;
+const gint lookup_divisor = 10;
+
+const uint masks[] = {
+ 8,
+ 16,
+ 24,
+ 32,
+ 27,
+ 29,
+ 19,
+ 13,
+ 22};
+
+struct _tv {
+ const char *ip;
+ const char *nip;
+ const char *m;
+ guint32 mask;
+ guint8 *addr;
+ guint8 *naddr;
+ gsize len;
+} test_vec[] = {
+ {"192.168.1.1", "192.168.1.2", "32", 0, 0, 0, 0},
+ {"192.168.1.0", "192.168.2.1", "24", 0, 0, 0, 0},
+ {"192.0.0.0", "193.167.2.1", "8", 0, 0, 0, 0},
+ {"172.0.0.0", "171.16.1.0", "8", 0, 0, 0, 0},
+ {"172.16.0.1", "127.0.0.1", "16", 0, 0, 0, 0},
+ {"172.17.1.0", "10.0.0.1", "27", 0, 0, 0, 0},
+ {"172.17.1.1", "0.0.0.1", "32", 0, 0, 0, 0},
+
+ /* Some bad data known to cause problem in the past */
+ {"191.245.170.246", NULL, "19", 0, 0, 0, 0},
+ {"227.88.150.170", NULL, "23", 0, 0, 0, 0},
+ {"105.225.182.92", NULL, "24", 0, 0, 0, 0},
+ {"223.167.155.240", NULL, "29", 0, 0, 0, 0},
+ {"125.241.220.172", NULL, "2", 0, 0, 0, 0},
+
+ /* Mask = 0 */
+ {"143.105.181.13", NULL, "8", 0, 0, 0, 0},
+ {"113.241.233.86", NULL, "26", 0, 0, 0, 0},
+ {"185.187.122.222", NULL, "8", 0, 0, 0, 0},
+ {"109.206.26.202", NULL, "12", 0, 0, 0, 0},
+ {"130.244.233.150", NULL, "0", 0, 0, 0, 0},
+
+ /* Close ip addresses */
+ {"1.2.3.1", NULL, "32", 0, 0, 0, 0},
+ {"1.2.3.2", NULL, "32", 0, 0, 0, 0},
+ {"1.2.3.3", NULL, "32", 0, 0, 0, 0},
+ {"1.2.3.4", NULL, "32", 0, 0, 0, 0},
+
+ {NULL, NULL, NULL, 0, 0, 0, 0}};
+
+static void
+rspamd_radix_test_vec(void)
+{
+ radix_compressed_t *tree = radix_create_compressed(NULL);
+ struct _tv *t = &test_vec[0];
+ struct in_addr ina;
+ struct in6_addr in6a;
+ gulong i, val;
+
+ while (t->ip != NULL) {
+ t->addr = g_malloc(sizeof(in6a));
+ t->naddr = g_malloc(sizeof(in6a));
+ if (inet_pton(AF_INET, t->ip, &ina) == 1) {
+ memcpy(t->addr, &ina, sizeof(ina));
+ t->len = sizeof(ina);
+ }
+ else if (inet_pton(AF_INET6, t->ip, &in6a) == 1) {
+ memcpy(t->addr, &in6a, sizeof(in6a));
+ t->len = sizeof(in6a);
+ }
+ else {
+ g_assert(0);
+ }
+ if (t->nip) {
+ if (inet_pton(AF_INET, t->nip, &ina) == 1) {
+ memcpy(t->naddr, &ina, sizeof(ina));
+ }
+ else if (inet_pton(AF_INET6, t->nip, &in6a) == 1) {
+ memcpy(t->naddr, &in6a, sizeof(in6a));
+ }
+ else {
+ g_assert(0);
+ }
+ }
+
+ t->mask = t->len * NBBY - strtoul(t->m, NULL, 10);
+ t++;
+ }
+ t = &test_vec[0];
+
+ i = 0;
+ while (t->ip != NULL) {
+ radix_insert_compressed(tree, t->addr, t->len, t->mask, ++i);
+ t++;
+ }
+
+ i = 0;
+ t = &test_vec[0];
+ while (t->ip != NULL) {
+ val = radix_find_compressed(tree, t->addr, t->len);
+ g_assert(val == ++i);
+ /* g_assert (val != RADIX_NO_VALUE); */
+ if (t->nip != NULL) {
+ val = radix_find_compressed(tree, t->naddr, t->len);
+ g_assert(val != i);
+ }
+ t++;
+ }
+
+ radix_destroy_compressed(tree);
+}
+
+static void
+rspamd_btrie_test_vec(void)
+{
+ rspamd_mempool_t *pool;
+ struct btrie *tree;
+ struct _tv *t = &test_vec[0];
+ struct in_addr ina;
+ struct in6_addr in6a;
+ gsize i;
+ gpointer val;
+
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), "btrie", 0);
+ tree = btrie_init(pool);
+
+ while (t->ip != NULL) {
+ t->addr = g_malloc(sizeof(in6a));
+ t->naddr = g_malloc(sizeof(in6a));
+ if (inet_pton(AF_INET, t->ip, &ina) == 1) {
+ memcpy(t->addr, &ina, sizeof(ina));
+ t->len = sizeof(ina);
+ }
+ else if (inet_pton(AF_INET6, t->ip, &in6a) == 1) {
+ memcpy(t->addr, &in6a, sizeof(in6a));
+ t->len = sizeof(in6a);
+ }
+ else {
+ g_assert(0);
+ }
+ if (t->nip) {
+ if (inet_pton(AF_INET, t->nip, &ina) == 1) {
+ memcpy(t->naddr, &ina, sizeof(ina));
+ }
+ else if (inet_pton(AF_INET6, t->nip, &in6a) == 1) {
+ memcpy(t->naddr, &in6a, sizeof(in6a));
+ }
+ else {
+ g_assert(0);
+ }
+ }
+
+ t->mask = strtoul(t->m, NULL, 10);
+ t++;
+ }
+ t = &test_vec[0];
+
+ i = 0;
+ while (t->ip != NULL) {
+ g_assert(btrie_add_prefix(tree, t->addr, t->mask,
+ GSIZE_TO_POINTER(++i)) == BTRIE_OKAY);
+ t++;
+ }
+
+ i = 0;
+ t = &test_vec[0];
+ while (t->ip != NULL) {
+ val = btrie_lookup(tree, t->addr, t->len * NBBY);
+ i++;
+
+ g_assert(GPOINTER_TO_SIZE(val) == i);
+ if (t->nip != NULL) {
+ val = btrie_lookup(tree, t->naddr, t->len * NBBY);
+ g_assert(GPOINTER_TO_SIZE(val) != i);
+ }
+ t++;
+ }
+}
+
+void rspamd_radix_test_func(void)
+{
+ struct btrie *btrie;
+ rspamd_mempool_t *pool;
+ struct {
+ guint32 addr;
+ guint32 mask;
+ guint8 addr6[16];
+ guint32 mask6;
+ guint8 addr64[16];
+ } *addrs;
+ gsize nelts, i, check;
+ gint lc;
+ gboolean all_good = TRUE;
+ gdouble ts1, ts2;
+ double diff;
+
+ /* Test suite for the compressed trie */
+
+ rspamd_btrie_test_vec();
+ rspamd_radix_test_vec();
+ rspamd_random_seed_fast();
+
+ nelts = max_elts;
+ /* First of all we generate many elements and push them to the array */
+ addrs = g_malloc(nelts * sizeof(addrs[0]));
+
+ for (i = 0; i < nelts; i++) {
+ addrs[i].addr = ottery_rand_uint32();
+ memset(addrs[i].addr64, 0, 10);
+ memcpy(addrs[i].addr64 + 12, &addrs[i].addr, 4);
+ addrs[i].mask = masks[ottery_rand_range(G_N_ELEMENTS(masks) - 1)];
+ ottery_rand_bytes(addrs[i].addr6, sizeof(addrs[i].addr6));
+ addrs[i].mask6 = ottery_rand_range(128 - 16) + 16;
+ }
+
+ pool = rspamd_mempool_new(65536, "btrie6", 0);
+ btrie = btrie_init(pool);
+ msg_notice("btrie performance ipv6 only (%z elts)", nelts);
+
+ ts1 = rspamd_get_ticks(TRUE);
+ for (i = 0; i < nelts; i++) {
+ btrie_add_prefix(btrie, addrs[i].addr6,
+ addrs[i].mask6, GSIZE_TO_POINTER(i + 1));
+ }
+ ts2 = rspamd_get_ticks(TRUE);
+ diff = (ts2 - ts1);
+
+ msg_notice("Added %hz elements in %.0f ticks (%.2f ticks per element)",
+ nelts, diff, diff / (double) nelts);
+
+ ts1 = rspamd_get_ticks(TRUE);
+ for (lc = 0; lc < lookup_cycles && all_good; lc++) {
+ for (i = 0; i < nelts / lookup_divisor; i++) {
+ check = rspamd_random_uint64_fast() % nelts;
+
+ if (btrie_lookup(btrie, addrs[check].addr6, sizeof(addrs[check].addr6) * 8) == NULL) {
+ char ipbuf[INET6_ADDRSTRLEN + 1];
+
+ all_good = FALSE;
+
+ inet_ntop(AF_INET6, addrs[check].addr6, ipbuf, sizeof(ipbuf));
+ msg_notice("BAD btrie: {\"%s\", NULL, \"%ud\", 0, 0, 0, 0},",
+ ipbuf,
+ addrs[check].mask6);
+ all_good = FALSE;
+ }
+ }
+ }
+ g_assert(all_good);
+ ts2 = rspamd_get_ticks(TRUE);
+ diff = (ts2 - ts1);
+
+ msg_notice("Checked %hz elements in %.0f ticks (%.2f ticks per lookup)",
+ nelts * lookup_cycles / lookup_divisor, diff,
+ diff / ((gdouble) nelts * lookup_cycles / lookup_divisor));
+ rspamd_mempool_delete(pool);
+
+ /*
+ * IPv4 part
+ */
+ pool = rspamd_mempool_new(65536, "btrie4", 0);
+ btrie = btrie_init(pool);
+ msg_notice("btrie performance ipv4 only (%z elts)", nelts);
+
+ ts1 = rspamd_get_ticks(TRUE);
+ for (i = 0; i < nelts; i++) {
+ btrie_add_prefix(btrie, (guchar *) &addrs[i].addr,
+ addrs[i].mask, GSIZE_TO_POINTER(i + 1));
+ }
+ ts2 = rspamd_get_ticks(TRUE);
+ diff = (ts2 - ts1);
+
+ msg_notice("Added %hz elements in %.0f ticks (%.2f ticks per element)",
+ nelts, diff, diff / (double) nelts);
+
+ ts1 = rspamd_get_ticks(TRUE);
+ for (lc = 0; lc < lookup_cycles && all_good; lc++) {
+ for (i = 0; i < nelts / lookup_divisor; i++) {
+ check = rspamd_random_uint64_fast() % nelts;
+
+ if (btrie_lookup(btrie, (guchar *) &addrs[check].addr, sizeof(addrs[check].addr) * 8) == NULL) {
+ char ipbuf[INET6_ADDRSTRLEN + 1];
+
+ all_good = FALSE;
+
+ inet_ntop(AF_INET, (guchar *) &addrs[check].addr, ipbuf, sizeof(ipbuf));
+ msg_notice("BAD btrie: {\"%s\", NULL, \"%ud\", 0, 0, 0, 0},",
+ ipbuf,
+ addrs[check].mask);
+ all_good = FALSE;
+ }
+ }
+ }
+ g_assert(all_good);
+ ts2 = rspamd_get_ticks(TRUE);
+ diff = (ts2 - ts1);
+
+ msg_notice("Checked %hz elements in %.0f ticks (%.2f ticks per lookup)",
+ nelts * lookup_cycles / lookup_divisor, diff,
+ diff / ((gdouble) nelts * lookup_cycles / lookup_divisor));
+ rspamd_mempool_delete(pool);
+
+ /*
+ * IPv4 -> IPv6 mapped
+ */
+ pool = rspamd_mempool_new(65536, "btrie4map", 0);
+ btrie = btrie_init(pool);
+ msg_notice("btrie performance ipv4 + ipv6map (%z elts)", nelts);
+
+ ts1 = rspamd_get_ticks(TRUE);
+ for (i = 0; i < nelts; i++) {
+
+ btrie_add_prefix(btrie, addrs[i].addr64,
+ addrs[i].mask + 96, GSIZE_TO_POINTER(i + 1));
+ }
+ ts2 = rspamd_get_ticks(TRUE);
+ diff = (ts2 - ts1);
+
+ msg_notice("Added %hz elements in %.0f ticks (%.2f ticks per element)",
+ nelts, diff, diff / (double) nelts);
+
+ ts1 = rspamd_get_ticks(TRUE);
+ for (lc = 0; lc < lookup_cycles && all_good; lc++) {
+ for (i = 0; i < nelts / lookup_divisor; i++) {
+ check = rspamd_random_uint64_fast() % nelts;
+
+ if (btrie_lookup(btrie, addrs[check].addr64,
+ sizeof(addrs[check].addr64) * 8) == NULL) {
+ char ipbuf[INET6_ADDRSTRLEN + 1];
+
+ all_good = FALSE;
+
+ inet_ntop(AF_INET, (guchar *) &addrs[check].addr, ipbuf, sizeof(ipbuf));
+ msg_notice("BAD btrie: {\"%s\", NULL, \"%ud\", 0, 0, 0, 0},",
+ ipbuf,
+ addrs[check].mask);
+ all_good = FALSE;
+ }
+ }
+ }
+ g_assert(all_good);
+ ts2 = rspamd_get_ticks(TRUE);
+ diff = (ts2 - ts1);
+
+ msg_notice("Checked %hz elements in %.0f ticks (%.2f ticks per lookup)",
+ nelts * lookup_cycles / lookup_divisor, diff,
+ diff / ((gdouble) nelts * lookup_cycles / lookup_divisor));
+ rspamd_mempool_delete(pool);
+
+ g_free(addrs);
+}
diff --git a/test/rspamd_rrd_test.c b/test/rspamd_rrd_test.c
new file mode 100644
index 0000000..2860cd8
--- /dev/null
+++ b/test/rspamd_rrd_test.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "tests.h"
+#include "rrd.h"
+#include "rspamd.h"
+#include "ottery.h"
+#include <unistd.h>
+
+const int rows_cnt = 20;
+const int pdp_per_cdp = 60;
+
+void rspamd_rrd_test_func(void)
+{
+ gchar tmpfile[PATH_MAX];
+ struct rrd_rra_def rra[4];
+ struct rrd_ds_def ds[2];
+ GArray ar;
+ GError *err = NULL;
+ struct rspamd_rrd_file *rrd;
+ gdouble ticks;
+ gint i;
+ gdouble t[2], cnt = 0.0;
+
+ rspamd_snprintf(tmpfile, sizeof(tmpfile), "/tmp/rspamd_rrd.rrd");
+ unlink(tmpfile);
+
+ /* Create sample rrd */
+ ticks = rspamd_get_calendar_ticks();
+ g_assert((rrd = rspamd_rrd_create(tmpfile, 2, 4, 1, ticks, &err)) != NULL);
+ /* Add RRA */
+ rrd_make_default_rra("AVERAGE", pdp_per_cdp, rows_cnt, &rra[0]);
+ rrd_make_default_rra("AVERAGE", pdp_per_cdp / 2, rows_cnt, &rra[1]);
+ rrd_make_default_rra("AVERAGE", pdp_per_cdp / 4, rows_cnt, &rra[2]);
+ rrd_make_default_rra("AVERAGE", pdp_per_cdp / 10, rows_cnt, &rra[3]);
+ ar.data = (char *) rra;
+ ar.len = sizeof(rra);
+ g_assert(rspamd_rrd_add_rra(rrd, &ar, &err));
+ /* Add DS */
+ rrd_make_default_ds("test", "COUNTER", 1, &ds[0]);
+ rrd_make_default_ds("test1", "COUNTER", 1, &ds[1]);
+ ar.data = (char *) ds;
+ ar.len = sizeof(ds);
+ g_assert(rspamd_rrd_add_ds(rrd, &ar, &err));
+ /* Finalize */
+ g_assert(rspamd_rrd_finalize(rrd, &err));
+ /* Close */
+ rspamd_rrd_close(rrd);
+
+ /* Reopen */
+ g_assert((rrd = rspamd_rrd_open(tmpfile, &err)) != NULL);
+ /* Add some points */
+ for (i = 0; i < pdp_per_cdp * rows_cnt / 2; i++) {
+ t[0] = i;
+ t[1] = cnt++;
+ ar.data = (char *) t;
+ ar.len = sizeof(t);
+ ticks += 1.0;
+ g_assert(rspamd_rrd_add_record(rrd, &ar, ticks, &err));
+ }
+
+ /* Add some more points */
+ for (i = 0; i < pdp_per_cdp * rows_cnt / 4; i++) {
+ t[0] = i + rspamd_time_jitter(1.0, 0.0);
+ t[1] = cnt++;
+ ar.data = (char *) t;
+ ar.len = sizeof(t);
+ ticks += 1.0;
+ g_assert(rspamd_rrd_add_record(rrd, &ar, ticks, &err));
+ }
+
+ /* Add undefined interval */
+ ticks += 200;
+
+ /* Add some more points */
+ for (i = 0; i < pdp_per_cdp * rows_cnt / 8; i++) {
+ t[0] = i;
+ t[1] = cnt++;
+ ar.data = (char *) t;
+ ar.len = sizeof(t);
+ ticks += 1.0;
+ g_assert(rspamd_rrd_add_record(rrd, &ar, ticks, &err));
+ }
+
+ /* Finish */
+ rspamd_rrd_close(rrd);
+ /* unlink (tmpfile); */
+}
diff --git a/test/rspamd_shingles_test.c b/test/rspamd_shingles_test.c
new file mode 100644
index 0000000..307634e
--- /dev/null
+++ b/test/rspamd_shingles_test.c
@@ -0,0 +1,339 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "shingles.h"
+#include "ottery.h"
+#include <math.h>
+
+static const gchar *
+algorithm_to_string(enum rspamd_shingle_alg alg)
+{
+ const gchar *ret = "unknown";
+
+ switch (alg) {
+ case RSPAMD_SHINGLES_OLD:
+ ret = "siphash";
+ break;
+ case RSPAMD_SHINGLES_XXHASH:
+ ret = "xxhash";
+ break;
+ case RSPAMD_SHINGLES_MUMHASH:
+ ret = "mumhash";
+ break;
+ case RSPAMD_SHINGLES_FAST:
+ ret = "fasthash";
+ break;
+ }
+
+ return ret;
+}
+
+static void
+generate_random_string(char *begin, size_t len)
+{
+ gsize i;
+
+ for (i = 0; i < len; i++) {
+ begin[i] = ottery_rand_range('z' - 'a') + 'a';
+ }
+}
+
+static GArray *
+generate_fuzzy_words(gsize cnt, gsize max_len)
+{
+ GArray *res;
+ gsize i, wlen;
+ rspamd_ftok_t w;
+ char *t;
+
+ res = g_array_sized_new(FALSE, FALSE, sizeof(rspamd_ftok_t), cnt);
+
+ for (i = 0; i < cnt; i++) {
+ wlen = ottery_rand_range(max_len) + 1;
+ /* wlen = max_len; */
+
+ w.len = wlen;
+ t = g_malloc(wlen);
+ generate_random_string(t, wlen);
+ w.begin = t;
+ g_array_append_val(res, w);
+ }
+
+ return res;
+}
+
+static void
+permute_vector(GArray *in, gdouble prob)
+{
+ gsize i, total = 0;
+ rspamd_ftok_t *w;
+
+ for (i = 0; i < in->len; i++) {
+ if (ottery_rand_unsigned() <= G_MAXUINT * prob) {
+ w = &g_array_index(in, rspamd_ftok_t, i);
+ generate_random_string((gchar *) w->begin, w->len);
+ total++;
+ }
+ }
+ msg_debug("generated %z permutations of %ud words", total, in->len);
+}
+
+static void
+free_fuzzy_words(GArray *ar)
+{
+ gsize i;
+ rspamd_ftok_t *w;
+
+ for (i = 0; i < ar->len; i++) {
+ w = &g_array_index(ar, rspamd_ftok_t, i);
+ g_free((gpointer) w->begin);
+ }
+}
+
+static void
+test_case(gsize cnt, gsize max_len, gdouble perm_factor,
+ enum rspamd_shingle_alg alg)
+{
+ GArray *input;
+ struct rspamd_shingle *sgl, *sgl_permuted;
+ gdouble res;
+ guchar key[16];
+ gdouble ts1, ts2;
+
+ ottery_rand_bytes(key, sizeof(key));
+ input = generate_fuzzy_words(cnt, max_len);
+ ts1 = rspamd_get_virtual_ticks();
+ sgl = rspamd_shingles_from_text(input, key, NULL,
+ rspamd_shingles_default_filter, NULL, alg);
+ ts2 = rspamd_get_virtual_ticks();
+ permute_vector(input, perm_factor);
+ sgl_permuted = rspamd_shingles_from_text(input, key, NULL,
+ rspamd_shingles_default_filter, NULL, alg);
+
+ res = rspamd_shingles_compare(sgl, sgl_permuted);
+
+ msg_info("%s (%z words of %z max len, %.2f perm factor):"
+ " percentage of common shingles: %.3f, generate time: %.4f sec",
+ algorithm_to_string(alg), cnt, max_len, perm_factor, res, ts2 - ts1);
+ //g_assert_cmpfloat (fabs ((1.0 - res) - sqrt (perm_factor)), <=, 0.25);
+
+ free_fuzzy_words(input);
+ g_free(sgl);
+ g_free(sgl_permuted);
+}
+
+static const guint64 expected_old[RSPAMD_SHINGLE_SIZE] = {
+ 0x2a97e024235cedc5,
+ 0x46238acbcc55e9e0,
+ 0x2378ff151af075b3,
+ 0xde1f29a95cad109,
+ 0x5d3bbbdb5db5d19f,
+ 0x4d75a0ec52af10a6,
+ 0x215ecd6372e755b5,
+ 0x7b52295758295350,
+ 0x17387d1beddc7f62,
+ 0x26264ca879ffcada,
+ 0x49d4a65ec0ab9914,
+ 0xa2763e6995350cf,
+ 0x3f4570231449c13f,
+ 0x3309f857a0e54ee5,
+ 0x24e4c5b561b0fce3,
+ 0x1f153e3b275bfd1b,
+ 0x4d067dbc97c3fd78,
+ 0x9ffa2d076fa4f8bc,
+ 0x3d8907f84b9ffc6c,
+ 0x1cfd664c5262d256,
+ 0xcdd7e744b699c15,
+ 0x5544a2bbe05124f7,
+ 0x5a4029b5d6a06f7,
+ 0xd5adfbdc756c0e4,
+ 0xa504b23d9689a67e,
+ 0x15d945f7007de115,
+ 0xbf676c0522a2c51d,
+ 0x1c8d8163ad4b0f93,
+ 0xa2c4ba20799344d7,
+ 0x27c6f13c02134388,
+ 0xa1d443d31fd5a3,
+ 0x99fbca9f8563080,
+};
+
+static const guint64 expected_xxhash[RSPAMD_SHINGLE_SIZE] = {
+ 0x33b134be11a705a,
+ 0x36e2ea657aa36903,
+ 0x6547b57f7470ce9d,
+ 0x8253eb6d2f8f158e,
+ 0x1cc99e3cf22388f,
+ 0x2396da27ea36ffe8,
+ 0x1b457d208ad3d96c,
+ 0x2d6ac733d7a2c107,
+ 0x17849cbed75cc4d1,
+ 0x4dd94e772330e804,
+ 0x39f592fa32014ed4,
+ 0xa2f6229ad356461,
+ 0x6dc825879a057b37,
+ 0x886b12cef4338b05,
+ 0x8b23af68c186518a,
+ 0x16932b40339aaf02,
+ 0x412090c6bb0b719c,
+ 0x4d4a88cbdf1935f3,
+ 0x233bcbddb5f67a7,
+ 0x474719442a33dcca,
+ 0x2da7ec30563e622,
+ 0x7ab90086960e1ad2,
+ 0x3ea2b45582539f75,
+ 0x108cd9287d95a6c5,
+ 0x69ba7c67c115597,
+ 0x10880860eb75e982,
+ 0x16f3d90e6ab995a6,
+ 0x5f24ea09379b9f5c,
+ 0x3c2dc04088e8fe54,
+ 0x340b8cf1c6f1227,
+ 0x193bc348ed2e9ce7,
+ 0x68454ef43da9c748,
+};
+
+static const guint64 expected_mumhash[RSPAMD_SHINGLE_SIZE] = {
+ 0x38d35473b80a7fc3,
+ 0x1300531adc2d16a1,
+ 0x26883bc89f78f4bd,
+ 0x57de365ef6d1a62,
+ 0x773603185fcbb20a,
+ 0x39c6cbd7ebbeaa88,
+ 0x676c7445ad167e70,
+ 0x432315d1ecc4c0b1,
+ 0x1380b95756dbb078,
+ 0x9ee12832fa53b90e,
+ 0x72970be210f0dd0b,
+ 0x62909bd520f5956,
+ 0x66196965a45eb32a,
+ 0x2466a9ca5436620e,
+ 0x157b828b10e10f6e,
+ 0x429bb673a523a7e5,
+ 0x51a6ace94f320f88,
+ 0x23f53a30bd7d7147,
+ 0xbee557664d3bc34c,
+ 0x65730c88cd212a9,
+ 0x87e72c0cd05fd0e,
+ 0x417a744669baeb3d,
+ 0x78e26f7917829324,
+ 0x439777dcfc25fdf4,
+ 0x582eac6ff013f00b,
+ 0x1e40aa90e367f4af,
+ 0x301d14a28d6c23a2,
+ 0x34140ecb21b6c69,
+ 0x390a091c8b4c31b9,
+ 0x2e35fecf9fff0ae7,
+ 0x94322e1a5cf31f1b,
+ 0x33cb9190905e049a,
+};
+
+static const guint64 expected_fasthash[RSPAMD_SHINGLE_SIZE] = {
+ 0x3843a716f94828a6,
+ 0x13fd5386dda3b28d,
+ 0x71cb09de527c40a,
+ 0x5d6f59ffd839c62,
+ 0x7ce3633acd568476,
+ 0x9014298cbd00167,
+ 0x6708ec29eedb5350,
+ 0x2882931ff2c5c410,
+ 0x1839d8b947b12571,
+ 0x58f7bc3829173302,
+ 0x4dac8103da51abc4,
+ 0x6c5cbcc6fb1de28,
+ 0x31fefcef9bafb755,
+ 0x6f2d1a0b1feca401,
+ 0x3e71f3718e520b06,
+ 0x42f6ba11164ab231,
+ 0x21164d010bd76f4a,
+ 0x4c597ccc7b60f620,
+ 0x2cf1ca3383b77574,
+ 0x54ff9c01660b8add,
+ 0x2ca344758f40380d,
+ 0x1b962321bd37d0f2,
+ 0x9323bb99c32bc418,
+ 0x375659d0eef2b8f2,
+ 0x1dbd23a1030084b7,
+ 0x83cb978dee06aa0a,
+ 0x42c97be5b27a7763,
+ 0x3b6d6b7270ed765,
+ 0x125c12fdba584aed,
+ 0x1c826397afe58763,
+ 0x8bdbe2d43f3eda96,
+ 0x954cda70edf6591f,
+};
+
+void rspamd_shingles_test_func(void)
+{
+ enum rspamd_shingle_alg alg = RSPAMD_SHINGLES_OLD;
+ struct rspamd_shingle *sgl;
+ guchar key[16];
+ GArray *input;
+ rspamd_ftok_t tok;
+ int i;
+
+ memset(key, 0, sizeof(key));
+ input = g_array_sized_new(FALSE, FALSE, sizeof(rspamd_ftok_t), 5);
+
+ for (i = 0; i < 5; i++) {
+ gchar *b = g_alloca(8);
+ memset(b, 0, 8);
+ memcpy(b + 1, "test", 4);
+ b[0] = 'a' + i;
+ tok.begin = b;
+ tok.len = 5 + ((i + 1) % 4);
+ g_array_append_val(input, tok);
+ }
+
+ sgl = rspamd_shingles_from_text(input, key, NULL,
+ rspamd_shingles_default_filter, NULL, RSPAMD_SHINGLES_OLD);
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ g_assert(sgl->hashes[i] == expected_old[i]);
+ }
+ g_free(sgl);
+
+ sgl = rspamd_shingles_from_text(input, key, NULL,
+ rspamd_shingles_default_filter, NULL, RSPAMD_SHINGLES_XXHASH);
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ g_assert(sgl->hashes[i] == expected_xxhash[i]);
+ }
+ g_free(sgl);
+
+ sgl = rspamd_shingles_from_text(input, key, NULL,
+ rspamd_shingles_default_filter, NULL, RSPAMD_SHINGLES_MUMHASH);
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ g_assert(sgl->hashes[i] == expected_mumhash[i]);
+ }
+ g_free(sgl);
+
+ sgl = rspamd_shingles_from_text(input, key, NULL,
+ rspamd_shingles_default_filter, NULL, RSPAMD_SHINGLES_FAST);
+ for (i = 0; i < RSPAMD_SHINGLE_SIZE; i++) {
+ g_assert(sgl->hashes[i] == expected_fasthash[i]);
+ }
+ g_free(sgl);
+
+ for (alg = RSPAMD_SHINGLES_OLD; alg <= RSPAMD_SHINGLES_FAST; alg++) {
+ test_case(200, 10, 0.1, alg);
+ test_case(500, 20, 0.01, alg);
+ test_case(5000, 20, 0.01, alg);
+ test_case(5000, 15, 0, alg);
+ test_case(5000, 30, 1.0, alg);
+ test_case(50000, 30, 0.02, alg);
+ test_case(50000, 5, 0.02, alg);
+ test_case(50000, 16, 0.02, alg);
+ }
+}
diff --git a/test/rspamd_statfile_test.c b/test/rspamd_statfile_test.c
new file mode 100644
index 0000000..0a3837d
--- /dev/null
+++ b/test/rspamd_statfile_test.c
@@ -0,0 +1,47 @@
+#include "config.h"
+#include "rspamd.h"
+#include "tests.h"
+#include "ottery.h"
+
+#define TEST_FILENAME "/tmp/rspamd_test.stat"
+#define HASHES_NUM 256
+
+void rspamd_statfile_test_func(void)
+{
+ /*
+ * XXX: broken, old, need to be rewritten
+ */
+#if 0
+ statfile_pool_t *pool;
+ rspamd_mempool_t *p;
+ stat_file_t *st;
+ uint32_t random_hashes[HASHES_NUM], i, v;
+ time_t now = time (NULL);
+
+ p = rspamd_mempool_new (rspamd_mempool_suggest_size ());
+ umask (S_IWGRP | S_IWOTH);
+ pool = statfile_pool_new (p, TRUE);
+
+ for (i = 0; i < HASHES_NUM; i ++) {
+ random_hashes[i] = ottery_rand_uint32 ();
+ }
+
+ /* Create new file */
+ g_assert (rspamd_mmaped_file_create (pool, TEST_FILENAME, 65535) != -1);
+ g_assert ((st = rspamd_mmaped_file_open (pool, TEST_FILENAME, 65535, FALSE)) != NULL);
+
+ /* Get and set random blocks */
+ rspamd_mmaped_file_lock_file (pool, st);
+ for (i = 0; i < HASHES_NUM; i ++) {
+ rspamd_mmaped_file_set_block (pool, st, random_hashes[i], random_hashes[i], now, 1.0);
+ }
+ rspamd_mmaped_file_unlock_file (pool, st);
+
+ for (i = 0; i < HASHES_NUM; i ++) {
+ v = rspamd_mmaped_file_get_block (pool, st, random_hashes[i], random_hashes[i], now);
+ g_assert(v == 1.0);
+ }
+
+ rspamd_mmaped_file_destroy (pool);
+#endif
+}
diff --git a/test/rspamd_test_suite.c b/test/rspamd_test_suite.c
new file mode 100644
index 0000000..b3d0391
--- /dev/null
+++ b/test/rspamd_test_suite.c
@@ -0,0 +1,95 @@
+#include "config.h"
+#include "rspamd.h"
+#include "libstat/stat_api.h"
+#include "lua/lua_common.h"
+#include "tests.h"
+#include "contrib/libev/ev.h"
+
+struct rspamd_main *rspamd_main = NULL;
+struct ev_loop *event_loop = NULL;
+worker_t *workers[] = {NULL};
+
+gchar *lua_test = NULL;
+gchar *lua_test_case = NULL;
+gboolean verbose = FALSE;
+gchar *argv0_dirname = NULL;
+
+static GOptionEntry entries[] =
+ {
+ {"test", 't', 0, G_OPTION_ARG_STRING, &lua_test,
+ "Lua test to run (i.e. selectors.lua)", NULL},
+ {"test-case", 'c', 0, G_OPTION_ARG_STRING, &lua_test_case,
+ "Lua test to run, lua pattern i.e. \"case .* rcpts\"", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+int main(int argc, char **argv)
+{
+ struct rspamd_config *cfg;
+ GOptionContext *context;
+ GError *error = NULL;
+
+ rspamd_main = (struct rspamd_main *) g_malloc(sizeof(struct rspamd_main));
+ memset(rspamd_main, 0, sizeof(struct rspamd_main));
+ rspamd_main->server_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), NULL, 0);
+ cfg = rspamd_config_new(RSPAMD_CONFIG_INIT_DEFAULT);
+ cfg->libs_ctx = rspamd_init_libs();
+ /* More aggressive GC, workaround for 'not enough memory' test failures */
+ cfg->lua_gc_step *= 2;
+ rspamd_main->cfg = cfg;
+ cfg->cfg_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), NULL, 0);
+
+ g_test_init(&argc, &argv, NULL);
+
+ argv0_dirname = g_path_get_dirname(argv[0]);
+
+ context = g_option_context_new("- run rspamd test");
+ g_option_context_add_main_entries(context, entries, NULL);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_option_context_free(context);
+ exit(EXIT_FAILURE);
+ }
+
+ /* Setup logger */
+ if (verbose || g_test_verbose()) {
+ rspamd_main->logger = rspamd_log_open_emergency(rspamd_main->server_pool,
+ RSPAMD_LOG_FLAG_USEC | RSPAMD_LOG_FLAG_ENFORCED | RSPAMD_LOG_FLAG_RSPAMADM);
+
+ rspamd_log_set_log_level(rspamd_main->logger, G_LOG_LEVEL_DEBUG);
+ }
+ else {
+ rspamd_main->logger = rspamd_log_open_emergency(rspamd_main->server_pool,
+ RSPAMD_LOG_FLAG_RSPAMADM);
+ rspamd_log_set_log_level(rspamd_main->logger, G_LOG_LEVEL_MESSAGE);
+ }
+
+ rspamd_lua_set_path((lua_State *) cfg->lua_state, NULL, NULL);
+ event_loop = ev_default_loop(EVFLAG_SIGNALFD | EVBACKEND_ALL);
+ rspamd_stat_init(cfg, event_loop);
+ rspamd_url_init(NULL);
+
+ g_log_set_default_handler(rspamd_glib_log_function, rspamd_main->logger);
+
+ g_test_add_func("/rspamd/mem_pool", rspamd_mem_pool_test_func);
+ g_test_add_func("/rspamd/radix", rspamd_radix_test_func);
+ g_test_add_func("/rspamd/dns", rspamd_dns_test_func);
+ g_test_add_func("/rspamd/dkim", rspamd_dkim_test_func);
+ g_test_add_func("/rspamd/rrd", rspamd_rrd_test_func);
+ g_test_add_func("/rspamd/upstream", rspamd_upstream_test_func);
+ g_test_add_func("/rspamd/shingles", rspamd_shingles_test_func);
+ g_test_add_func("/rspamd/lua", rspamd_lua_test_func);
+ g_test_add_func("/rspamd/cryptobox", rspamd_cryptobox_test_func);
+ g_test_add_func("/rspamd/heap", rspamd_heap_test_func);
+ g_test_add_func("/rspamd/lua_pcall", rspamd_lua_lua_pcall_vs_resume_test_func);
+
+#if 0
+ g_test_add_func ("/rspamd/http", rspamd_http_test_func);
+ g_test_add_func ("/rspamd/url", rspamd_url_test_func);
+ g_test_add_func ("/rspamd/statfile", rspamd_statfile_test_func);
+ g_test_add_func ("/rspamd/aio", rspamd_async_test_func);
+#endif
+ g_test_run();
+
+ return 0;
+}
diff --git a/test/rspamd_upstream_test.c b/test/rspamd_upstream_test.c
new file mode 100644
index 0000000..1f4e27d
--- /dev/null
+++ b/test/rspamd_upstream_test.c
@@ -0,0 +1,182 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "ottery.h"
+#include "contrib/libev/ev.h"
+
+#include <math.h>
+
+const char *test_upstream_list = "microsoft.com:443:1,google.com:80:2,kernel.org:443:3";
+const char *new_upstream_list = "freebsd.org:80";
+char test_key[32];
+extern struct ev_loop *event_loop;
+
+static void
+rspamd_upstream_test_method(struct upstream_list *ls,
+ enum rspamd_upstream_rotation rot, const gchar *expected)
+{
+ struct upstream *up;
+
+ if (rot != RSPAMD_UPSTREAM_HASHED) {
+ up = rspamd_upstream_get(ls, rot, NULL, 0);
+ g_assert(up != NULL);
+ g_assert(strcmp(rspamd_upstream_name(up), expected) == 0);
+ }
+ else {
+ up = rspamd_upstream_get(ls, RSPAMD_UPSTREAM_HASHED, test_key,
+ sizeof(test_key));
+ g_assert(up != NULL);
+ g_assert(strcmp(rspamd_upstream_name(up), expected) == 0);
+ }
+}
+
+static void
+rspamd_upstream_timeout_handler(EV_P_ ev_timer *w, int revents)
+{
+ struct rspamd_dns_resolver *resolver = (struct rspamd_dns_resolver *) w->data;
+
+ rdns_resolver_release(resolver->r);
+}
+
+void rspamd_upstream_test_func(void)
+{
+ struct upstream_list *ls, *nls;
+ struct upstream *up, *upn;
+ struct rspamd_dns_resolver *resolver;
+ struct rspamd_config *cfg;
+ gint i, success = 0;
+ const gint assumptions = 100500;
+ gdouble p;
+ static ev_timer ev;
+ rspamd_inet_addr_t *addr, *next_addr, *paddr;
+
+ cfg = rspamd_config_new(RSPAMD_CONFIG_INIT_SKIP_LUA);
+ cfg->dns_retransmits = 2;
+ cfg->dns_timeout = 0.5;
+ cfg->upstream_max_errors = 1;
+ cfg->upstream_revive_time = 0.5;
+ cfg->upstream_error_time = 2;
+
+ resolver = rspamd_dns_resolver_init(NULL, event_loop, cfg);
+ rspamd_upstreams_library_config(cfg, cfg->ups_ctx, event_loop, resolver->r);
+
+ /*
+ * Test v4/v6 priorities
+ */
+ nls = rspamd_upstreams_create(cfg->ups_ctx);
+ g_assert(rspamd_upstreams_add_upstream(nls, "127.0.0.1", 0,
+ RSPAMD_UPSTREAM_PARSE_DEFAULT,
+ NULL));
+ up = rspamd_upstream_get(nls, RSPAMD_UPSTREAM_RANDOM, NULL, 0);
+ rspamd_parse_inet_address(&paddr, "127.0.0.2", strlen("127.0.0.2"),
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT);
+ g_assert(rspamd_upstream_add_addr(up, paddr));
+ rspamd_parse_inet_address(&paddr, "::1", strlen("::1"),
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT);
+ g_assert(rspamd_upstream_add_addr(up, paddr));
+ /* Rewind to start */
+ addr = rspamd_upstream_addr_next(up);
+ addr = rspamd_upstream_addr_next(up);
+ /* cur should be zero here */
+ addr = rspamd_upstream_addr_next(up);
+ next_addr = rspamd_upstream_addr_next(up);
+ g_assert(rspamd_inet_address_get_af(addr) == AF_INET);
+ g_assert(rspamd_inet_address_get_af(next_addr) == AF_INET);
+ next_addr = rspamd_upstream_addr_next(up);
+ g_assert(rspamd_inet_address_get_af(next_addr) == AF_INET6);
+ next_addr = rspamd_upstream_addr_next(up);
+ g_assert(rspamd_inet_address_get_af(next_addr) == AF_INET);
+ next_addr = rspamd_upstream_addr_next(up);
+ g_assert(rspamd_inet_address_get_af(next_addr) == AF_INET);
+ next_addr = rspamd_upstream_addr_next(up);
+ g_assert(rspamd_inet_address_get_af(next_addr) == AF_INET6);
+ /* Test errors with IPv6 */
+ rspamd_upstream_fail(up, TRUE, NULL);
+ /* Now we should have merely IPv4 addresses in rotation */
+ addr = rspamd_upstream_addr_next(up);
+ for (i = 0; i < 256; i++) {
+ next_addr = rspamd_upstream_addr_next(up);
+ g_assert(rspamd_inet_address_get_af(addr) == AF_INET);
+ g_assert(rspamd_inet_address_get_af(next_addr) == AF_INET);
+ g_assert(rspamd_inet_address_compare(addr, next_addr, FALSE) != 0);
+ addr = next_addr;
+ }
+ rspamd_upstreams_destroy(nls);
+
+ ls = rspamd_upstreams_create(cfg->ups_ctx);
+ g_assert(rspamd_upstreams_parse_line(ls, test_upstream_list, 443, NULL));
+ g_assert(rspamd_upstreams_count(ls) == 3);
+
+ /* Test master-slave rotation */
+ rspamd_upstream_test_method(ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org");
+ rspamd_upstream_test_method(ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org");
+
+ /* Test round-robin rotation */
+ rspamd_upstream_test_method(ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
+ rspamd_upstream_test_method(ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
+ rspamd_upstream_test_method(ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com");
+ rspamd_upstream_test_method(ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
+ rspamd_upstream_test_method(ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com");
+ rspamd_upstream_test_method(ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "microsoft.com");
+
+ /* Test stable hashing */
+ nls = rspamd_upstreams_create(cfg->ups_ctx);
+ g_assert(rspamd_upstreams_parse_line(nls, test_upstream_list, 443, NULL));
+ g_assert(rspamd_upstreams_parse_line(nls, new_upstream_list, 443, NULL));
+ for (i = 0; i < assumptions; i++) {
+ ottery_rand_bytes(test_key, sizeof(test_key));
+ up = rspamd_upstream_get(ls, RSPAMD_UPSTREAM_HASHED, test_key,
+ sizeof(test_key));
+ upn = rspamd_upstream_get(nls, RSPAMD_UPSTREAM_HASHED, test_key,
+ sizeof(test_key));
+
+ if (strcmp(rspamd_upstream_name(up), rspamd_upstream_name(upn)) == 0) {
+ success++;
+ }
+ }
+
+ p = 1.0 - fabs(3.0 / 4.0 - (gdouble) success / (gdouble) assumptions);
+ /*
+ * P value is calculated as following:
+ * when we add/remove M upstreams from the list, the probability of hash
+ * miss should be close to the relation N / (N + M), where N is the size of
+ * the previous upstreams list.
+ */
+ msg_debug("p value for hash consistency: %.6f", p);
+ g_assert(p > 0.9);
+
+ rspamd_upstreams_destroy(nls);
+
+
+ /* Upstream fail test */
+ ev.data = resolver;
+ ev_timer_init(&ev, rspamd_upstream_timeout_handler, 2.0, 0.0);
+
+ up = rspamd_upstream_get(ls, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0);
+ for (i = 0; i < 100; i++) {
+ rspamd_upstream_fail(up, TRUE, NULL);
+ }
+ g_assert(rspamd_upstreams_alive(ls) == 2);
+
+ ev_timer_start(event_loop, &ev);
+
+ ev_run(event_loop, 0);
+ g_assert(rspamd_upstreams_alive(ls) == 3);
+
+ rspamd_upstreams_destroy(ls);
+ REF_RELEASE(cfg);
+}
diff --git a/test/rspamd_url_test.c b/test/rspamd_url_test.c
new file mode 100644
index 0000000..c06f8d0
--- /dev/null
+++ b/test/rspamd_url_test.c
@@ -0,0 +1,60 @@
+#include "config.h"
+#include "rspamd.h"
+#include "cfg_file.h"
+#include "url.h"
+#include "tests.h"
+
+const char *test_text =
+ "www.schemeless.ru\n"
+ "www.schemeless.rus\n"
+ " as ftp.schemeless.ru dasd \n"
+ "ftp12.schemeless.ru\n"
+ "ftpsearch.schemeless.ru\n"
+ "schemeless.ru\n"
+ "www.schemeless.microsoft\n"
+ "1.2.3.4\n"
+ "1.2.3.4/a\n"
+ "1.2.3\n"
+ "1.2.3.4.5\n"
+ "www.schemeless.ru,\n"
+ "www.schemeless.ru.\n"
+ "http://www.schemed.ru.\n"
+ "http://www.schemed.ru.\n"
+ "http://www.bolinfest.com/targetalert/'\n"
+ "http://www.bolinfest.com/targetalert/'';\n"
+ "https://www.schemed.ru.\n"
+ "ufps://www.schemed.ru.\n"
+ "http://ported.ru:8080\n"
+ "http://ported.ru:8080\n"
+ "http://1.2.3.4\n"
+ "http://1.2.3.4:80\n"
+ "1.2.3.4:80\n"
+ "www.a9.com\n"
+ "www.a-9.com\n"
+ "http://www.schemed.ru/a.txt:\n"
+ "http://www.schemed.ru/a.txt'\n"
+ "http://www.schemed.ru/a.txt\"\n"
+ "http://www.schemed.ru/a.txt>\n"
+ "http://www.schemed.ru/a=3&b=4\n"
+ "http://spam.ru/bad=user@domain.com\n"
+ "http://spam.ru/bad=user@domain.com\n"
+ "http://spam.ru user@domain.com\n"
+ "http://a.foto.radikal.ru/0604/de7793c6ca62.jpg\n"
+ "http://a.foto.radikal.ru/0604/de7793c6ca62.jpg\n"
+ "schemeless.gz\n"
+ "schemeless.jp\n"
+ "schemeless.ua\n"
+ "schemeless.gz/a\n"
+ "mysql.so\n"
+ "http://mysql.so\n"
+ "3com.com\n"
+ "lj-user.livejournal.com\n"
+ "http://lj-user.livejournal.com\n"
+ "http://vsem.ru?action;\n";
+const char *test_html = "<some_tag>This is test file with <a href=\"http://microsoft.com\">http://TesT.com/././?%45%46%20 url</a></some_tag>";
+
+/* Function for using in glib test suite */
+void rspamd_url_test_func(void)
+{
+ /* XXX: maybe write test for this */
+}
diff --git a/test/test.cfg b/test/test.cfg
new file mode 100644
index 0000000..3300a82
--- /dev/null
+++ b/test/test.cfg
@@ -0,0 +1,16 @@
+section2 {
+ param = "value";
+ param2 = value // comment
+ array = [ 100500, 10s, 50mb ] # array
+ /*
+ * This should be commented
+ *
+ * test123 = test
+ */
+}
+/*
+section3 {
+ key = value
+}
+/* Nested comment */
+*/
diff --git a/test/test.cfg.sig b/test/test.cfg.sig
new file mode 100644
index 0000000..de849d5
--- /dev/null
+++ b/test/test.cfg.sig
@@ -0,0 +1 @@
+’éBúDü æ%jŸ7¦Ë|æ"ìõïr{ªv>ú:óLƒ¹ˆÉWáíÄ(dL4¹{ñlÝÑOEéŠåçÈÓÛ5ߥ]&S°¤­/¶Pš·•MÕÉ”l,‰@ä%.Ïû@Ì­®V< ^%8‡9_M:ørl«¯¯F!2(ìƒí1 \ No newline at end of file
diff --git a/test/tests.h b/test/tests.h
new file mode 100644
index 0000000..be37bbc
--- /dev/null
+++ b/test/tests.h
@@ -0,0 +1,51 @@
+#ifndef RSPAMD_TESTS_H
+#define RSPAMD_TESTS_H
+
+/*
+ * Here are described test functions for rspamd test suite
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* URL parser test */
+void rspamd_url_test_func(void);
+
+/* Memory pools */
+void rspamd_mem_pool_test_func(void);
+
+/* Stat file */
+void rspamd_statfile_test_func(void);
+
+/* Radix test */
+void rspamd_radix_test_func(void);
+
+/* DNS resolving */
+void rspamd_dns_test_func(void);
+
+/* DKIM test */
+void rspamd_dkim_test_func(void);
+
+/* RRD test */
+void rspamd_rrd_test_func(void);
+
+void rspamd_upstream_test_func(void);
+
+void rspamd_shingles_test_func(void);
+
+void rspamd_http_test_func(void);
+
+void rspamd_lua_test_func(void);
+
+void rspamd_cryptobox_test_func(void);
+
+void rspamd_heap_test_func(void);
+
+void rspamd_lua_lua_pcall_vs_resume_test_func(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/test/tools/dump_coveralls.py b/test/tools/dump_coveralls.py
new file mode 100755
index 0000000..a96dc92
--- /dev/null
+++ b/test/tools/dump_coveralls.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+# Small tool to dump JSON payload for coveralls.io API
+
+import json
+from operator import itemgetter
+import os
+import sys
+
+
+def warn(*args, **kwargs):
+ print(*args, file=sys.stderr, **kwargs)
+
+
+def dump_file(json_file):
+ """Dumps coveralls.io API payload stored in json_file
+ Returns: 0 if successful, 1 otherwise
+ """
+ try:
+ with open(json_file, encoding='utf8') as f:
+ data = json.load(f)
+ except OSError as err:
+ warn(err)
+ return os.EX_DATAERR
+ except json.decoder.JSONDecodeError:
+ warn("{}: json parsing error".format(json_file))
+ return 1
+
+ if 'source_files' not in data:
+ warn("{}: no source_files, not a coveralls.io payload?".format(json_file))
+ return 1
+
+ print("{} ({} source files)".format(json_file, len(data['source_files'])))
+
+ for src_file in sorted(data['source_files'], key=itemgetter('name')):
+ covered_lines = not_skipped_lines = 0
+ for cnt in src_file['coverage']:
+ if cnt is None:
+ continue
+ not_skipped_lines += 1
+ if cnt > 0:
+ covered_lines += 1
+ if not_skipped_lines > 0:
+ coverage = "{:.0%}".format(covered_lines / not_skipped_lines)
+ else:
+ coverage = 'N/A'
+
+ print("\t{:>3} {}".format(coverage, src_file['name']))
+
+ return 0
+
+
+def main():
+ if (len(sys.argv) < 2):
+ warn("usage: {} file.json ...".format(sys.argv[0]))
+ return os.EX_USAGE
+
+ exit_status = 0
+ for f in sys.argv[1:]:
+ exit_status += dump_file(f)
+
+ return exit_status
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/test/tools/gcov_coveralls.py b/test/tools/gcov_coveralls.py
new file mode 100755
index 0000000..dcbd93f
--- /dev/null
+++ b/test/tools/gcov_coveralls.py
@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""
+Script to save coverage info for C source files in JSON for coveralls.io
+
+When C code compiled with --coverage flag, for each object files *.gcno is
+generated, it contains information to reconstruct the basic block graphs and
+assign source line numbers to blocks
+
+When binary executed *.gcda file is written on exit, with same base name as
+corresponding *.gcno file. It contains some summary information, counters, e.t.c.
+
+gcov(1) utility can be used to get information from *.gcda file and write text
+reports to *.gocov file (one file for each source file from which object was compiled).
+
+The script finds *.gcno files, uses gcov to generate *.gcov files, parses them
+and accumulates statistics for all source files.
+
+This script was written with quite a few assumptions:
+
+ * Code was build using absolute path to source directory (and absolute path
+ stored in object file debug symbols).
+
+ * Current directory is writable and there is no useful *.gcov files in it
+ (because they will be deleted).
+
+ * Object file has same base name as *.gcno file (e. g. foo.c.gcno and foo.c.o).
+ This is the case for cmake builds, but probably not for other build systems
+
+ * Source file names contain only ASCII characters.
+"""
+
+import argparse
+from collections import defaultdict
+from glob import glob
+import hashlib
+import json
+import os
+from os.path import isabs, join, normpath, relpath
+import os.path
+import subprocess
+import sys
+
+
+def warn(*args, **kwargs):
+ print(*args, file=sys.stderr, **kwargs)
+
+
+def parse_gcov_file(gcov_file):
+ """Parses the content of .gcov file written by gcov -i
+
+ Returns:
+ str: Source file name
+ dict: coverage info { line_number: hits }
+ """
+ count = {}
+ with open(gcov_file) as fh:
+ for line in fh:
+ tag, value = line.split(':')
+ if tag == 'file':
+ src_file = value.rstrip()
+ elif tag == 'lcount':
+ line_num, exec_count = value.split(',')
+ count[int(line_num)] = int(exec_count)
+
+ return src_file, count
+
+
+def run_gcov(filename, coverage, args):
+ """ * run gcov on given file
+ * parse generated .gcov files and update coverage structure
+ * store source file md5 (if not yet stored)
+ * delete .gcov files
+ """
+ if args.verbose:
+ warn("calling:", 'gcov', '-i', filename)
+ stdout = None
+ else:
+ # gcov is noisy and don't have quit flag so redirect stdout to /dev/null
+ stdout = subprocess.DEVNULL
+
+ subprocess.check_call(['gcov', '-i', filename], stdout=stdout)
+
+ for gcov_file in glob('*.gcov'):
+ if args.verbose:
+ warn('parsing', gcov_file)
+ src_file, count = parse_gcov_file(gcov_file)
+ os.remove(gcov_file)
+
+ if src_file not in coverage:
+ coverage[src_file] = defaultdict(int, count)
+ else:
+ # sum execution counts
+ for line, exe_cnt in count.items():
+ coverage[src_file][line] += exe_cnt
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description='Save gcov coverage results in JSON file for coveralls.io.')
+ parser.add_argument(
+ '-v',
+ '--verbose',
+ action="store_true",
+ help='Display additional information and gcov command output.')
+ parser.add_argument(
+ '-e',
+ '--exclude',
+ action='append',
+ metavar='DIR',
+ help=
+ ("Don't look for .gcno/.gcda files in this directories (repeat option to skip several directories). "
+ "Path is relative to the directory where script was started, e. g. '.git'"))
+ parser.add_argument(
+ '-p',
+ '--prefix',
+ action='append',
+ help=
+ ("Strip this prefix from absolute path to source file. "
+ "If this option is provided, then only files with given prefixes in absolute path "
+ "will be added to coverage (option can be repeated)."))
+ parser.add_argument(
+ '--out',
+ type=argparse.FileType('w'),
+ required=True,
+ metavar='FILE',
+ help='Save JSON payload to this file')
+ args = parser.parse_args()
+
+ # ensure that there is no unrelated .gcov files in current directory
+ for gcov_file in glob('*.gcov'):
+ os.remove(gcov_file)
+ warn("Warning: {} deleted".format(gcov_file))
+
+ # dict { src_file_name: {line1: exec_count1, line2: exec_count2, ...} }
+ coverage = {}
+
+ # find . -name '*.gcno' (respecting args.exclude)
+ for root, dirs, files in os.walk('.'):
+ for f in files:
+ # Usually gcov called with a source file as an argument, but this
+ # name used only to find .gcno and .gcda files. To find source
+ # file information from debug symbols is used. So we can call gcov
+ # on .gcno file.
+ if f.endswith('.gcno'):
+ run_gcov(join(root, f), coverage, args)
+
+ # don't look into excluded dirs
+ for subdir in dirs:
+ # path relative to start dir
+ path = normpath(join(root, subdir))
+ if path in args.exclude:
+ if args.verbose:
+ warn('directory "{}" excluded'.format(path))
+ dirs.remove(subdir)
+
+ # prepare JSON pyload for coveralls.io API
+ # https://docs.coveralls.io/api-introduction
+ coveralls_data = {'source_files': []}
+
+ for src_file in coverage:
+ # filter by prefix and save path with stripped prefix
+ src_file_rel = src_file
+ if args.prefix and isabs(src_file):
+ for prefix in args.prefix:
+ if src_file.startswith(prefix):
+ src_file_rel = relpath(src_file, start=prefix)
+ break
+ else:
+ # skip file outside given prefixes
+ # it can be e. g. library include file
+ if args.verbose:
+ warn('file "{}" is not matched by prefix, skipping'.format(src_file))
+ continue
+
+ try:
+ with open(src_file, mode='rb') as fh:
+ line_count = sum(1 for _ in fh)
+ fh.seek(0)
+ md5 = hashlib.md5(fh.read()).hexdigest()
+ except OSError as err:
+ # skip files for which source file is not available
+ warn(err, 'not adding to coverage')
+ continue
+
+ coverage_array = [None] * line_count
+
+ for line_num, exe_cnt in coverage[src_file].items():
+ # item at index 0 representing the coverage for line 1 of the source code
+ assert 1 <= line_num <= line_count
+ coverage_array[line_num - 1] = exe_cnt
+
+ coveralls_data['source_files'].append({
+ 'name': src_file_rel,
+ 'coverage': coverage_array,
+ 'source_digest': md5
+ })
+
+ args.out.write(json.dumps(coveralls_data))
+
+ if args.verbose:
+ warn('Coverage for {} source files was written'.format(
+ len(coveralls_data['source_files'])))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/test/tools/http_put.py b/test/tools/http_put.py
new file mode 100755
index 0000000..8ede68e
--- /dev/null
+++ b/test/tools/http_put.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""
+Small script to upload file using HTTP PUT
+"""
+
+import argparse
+import os
+import sys
+
+import requests
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description='Upload a file using HTTP PUT method',
+ epilog=(
+ "To use HTTP Auth set HTTP_PUT_AUTH environment variable to user:password\n"
+ "Example: %(prog)s file1 file2 https://example.com/dir/"))
+ parser.add_argument(
+ "file", type=argparse.FileType('rb'), nargs='+', help="File to upload")
+ parser.add_argument(
+ "dir_url", help="Remote URL (path to a directory, must include a trailing /)")
+ args = parser.parse_args()
+
+ if not args.dir_url.endswith('/'):
+ parser.error("URL must end with /")
+
+ http_auth = os.getenv('HTTP_PUT_AUTH')
+ if http_auth:
+ user, password = http_auth.split(':')
+ auth = (user, password)
+ else:
+ auth = None
+
+ exit_code = 0
+
+ for fh in args.file:
+ try:
+ r = requests.put(args.dir_url + fh.name, data=fh, auth=auth, timeout=(45, 90))
+ r.raise_for_status()
+ print("{} uploaded to {}".format(fh.name, r.url))
+ except (requests.exceptions.ConnectionError,
+ requests.exceptions.HTTPError) as err:
+ print(err, file=sys.stderr)
+ exit_code = 1
+
+ return exit_code
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/test/tools/merge_coveralls.py b/test/tools/merge_coveralls.py
new file mode 100755
index 0000000..1d294cc
--- /dev/null
+++ b/test/tools/merge_coveralls.py
@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+
+import argparse
+import json
+import os
+import sys
+import codecs
+
+import requests
+
+# Python 2/3 compatibility
+if sys.version_info.major > 2:
+ xrange = range
+
+# install path to repository mapping
+# if path mapped to None, it means that the file should be ignored (i.e. test file/helper)
+# first matched path counts.
+# terminating slash should be added for directories
+path_mapping = [
+ ("${install-dir}/share/rspamd/lib/fun.lua", None),
+ ("${install-dir}/share/rspamd/lib/", "lualib/"),
+ ("${install-dir}/share/rspamd/rules/" , "rules/"),
+ ("${install-dir}/share/rspamd/lib/torch/" , None),
+ ("${build-dir}/CMakeFiles/", None),
+ ("${build-dir}/contrib/", None),
+ ("${build-dir}/test", None),
+ ("${project-root}/test/lua/", None),
+ ("${project-root}/test/", None),
+ ("${project-root}/clang-plugin/", None),
+ ("${project-root}/CMakeFiles/", None),
+ ("${project-root}/contrib/", None),
+ ("${project-root}/", ""),
+ ("contrib/", None),
+ ("CMakeFiles/", None),
+]
+
+parser = argparse.ArgumentParser(description='')
+parser.add_argument('--input', nargs='+', help='input files')
+parser.add_argument('--output', help='output file)')
+parser.add_argument('--root', default="/rspamd/src/github.com/rspamd/rspamd", help='repository root)')
+parser.add_argument('--install-dir', default="/rspamd/install", help='install root)')
+parser.add_argument('--build-dir', default="/rspamd/build", help='build root)')
+parser.add_argument('--token', help='If present, the file will be uploaded to coveralls)')
+parser.add_argument('--parallel', action='store_true', help='If present, this is a parallel build)')
+parser.add_argument('--parallel-close', action='store_true', help='If present, close parallel build and exit)')
+
+
+def merge_coverage_vectors(c1, c2):
+ assert(len(c1) == len(c2))
+
+ for i in range(0, len(c1)):
+ if c1[i] is None and c2[i] is None:
+ pass
+ elif type(c1[i]) is int and c2[i] is None:
+ pass
+ elif c1[i] is None and type(c2[i]) is int:
+ c1[i] = c2[i]
+ elif type(c1[i]) is int and type(c2[i]) is int:
+ c1[i] += c2[i]
+ else:
+ raise RuntimeError("bad element types at %d: %s, %s", i, type(c1[i]), type(c1[i]))
+
+ return c1
+
+
+def normalize_name(name):
+ name = os.path.normpath(name)
+ if not os.path.isabs(name):
+ name = os.path.abspath(repository_root + "/" + name)
+ for k in path_mapping:
+ if name.startswith(k[0]):
+ if k[1] is None:
+ return None
+ else:
+ name = k[1] + name[len(k[0]):]
+ break
+ return name
+
+def merge(files, j1):
+ for sf in j1['source_files']:
+ name = normalize_name(sf['name'])
+ if name is None:
+ continue
+ if name in files:
+ files[name]['coverage'] = merge_coverage_vectors(files[name]['coverage'], sf['coverage'])
+ else:
+ sf['name'] = name
+ files[name] = sf
+
+ return files
+
+def prepare_path_mapping():
+ for i in range(0, len(path_mapping)):
+ new_key = path_mapping[i][0].replace("${install-dir}", install_dir)
+ new_key = new_key.replace("${project-root}", repository_root)
+ new_key = new_key.replace("${build-dir}", build_dir)
+
+ path_mapping[i] = (new_key, path_mapping[i][1])
+
+def close_parallel_build():
+ j = {'payload':{'status': 'done'}}
+ j['payload']['build_num'] = os.getenv('DRONE_BUILD_NUMBER')
+ query_str = {'repo_token': args.token}
+ try:
+ r = requests.post('https://coveralls.io/webhook', params=query_str, json=j)
+ r.raise_for_status()
+ except requests.exceptions.RequestException as e:
+ print("Failed to send data to coveralls: %s" % e)
+ sys.exit()
+
+ try:
+ response = r.json()
+ if 'url' in response:
+ print("[coveralls] URL %s" % response['url'])
+ if 'error' in response:
+ print("[coveralls] ERROR: %s" % response['error'])
+ except json.decoder.JSONDecodeError:
+ print("Bad response: '%s'" % r.text)
+
+if __name__ == '__main__':
+ args = parser.parse_args()
+
+ if args.parallel_close:
+ close_parallel_build()
+ sys.exit(0)
+
+ if not args.input:
+ print("error: the following arguments are required: --input")
+ sys.exit(1)
+
+ repository_root = os.path.abspath(os.path.expanduser(args.root))
+ install_dir = os.path.normpath(os.path.expanduser(args.install_dir))
+ build_dir = os.path.normpath(os.path.expanduser(args.build_dir))
+
+ prepare_path_mapping()
+
+ with codecs.open(args.input[0], 'r', encoding='utf-8') as fh:
+ j1 = json.load(fh)
+
+ files = merge({}, j1)
+ for i in range(1, len(args.input)):
+ with codecs.open(args.input[i], 'r', encoding='utf-8') as fh:
+ j2 = json.load(fh)
+
+ files = merge(files, j2)
+
+ if 'git' not in j1 and 'git' in j2:
+ j1['git'] = j2['git']
+ if 'service_name' not in j1 and 'service_name' in j2:
+ j1['service_name'] = j2['service_name']
+ if 'service_job_id' not in j1 and 'service_job_id' in j2:
+ j1['service_job_id'] = j2['service_job_id']
+
+ if args.parallel:
+ j1['parallel'] = True
+
+ if os.getenv('CIRCLECI'):
+ j1['service_name'] = 'circleci'
+ j1['service_job_id'] = os.getenv('CIRCLE_BUILD_NUM')
+ elif os.getenv('DRONE') == 'true':
+ j1['service_name'] = 'drone'
+ j1['service_branch'] = os.getenv('DRONE_COMMIT_BRANCH')
+ j1['service_build_url'] = os.getenv('DRONE_BUILD_LINK')
+ j1['service_number'] = os.getenv('DRONE_BUILD_NUMBER')
+ j1['commit_sha'] = os.getenv('DRONE_COMMIT_SHA')
+ if os.getenv('DRONE_BUILD_EVENT') == 'pull_request':
+ j1['service_pull_request'] = os.getenv('DRONE_PULL_REQUEST')
+ # git data can be filled by cpp-coveralls, but in our layout it can't find repo
+ # so we can override git info witout merging
+ j1['git'] = {
+ 'head': {
+ 'id': j1['commit_sha'],
+ 'author_email': os.getenv('DRONE_COMMIT_AUTHOR_EMAIL'),
+ 'message': os.getenv('DRONE_COMMIT_MESSAGE')
+ },
+ 'branch': j1['service_branch'],
+ 'remotes': [{
+ 'name': 'origin',
+ 'url': os.getenv('DRONE_GIT_HTTP_URL')
+ }]
+ }
+
+
+ j1['source_files'] = list(files.values())
+
+ if args.output:
+ with open(args.output, 'w') as f:
+ f.write(json.dumps(j1))
+
+ if args.token:
+ j1['repo_token'] = args.token
+ try:
+ r = requests.post('https://coveralls.io/api/v1/jobs', files={"json_file": json.dumps(j1)})
+ r.raise_for_status()
+ except requests.exceptions.RequestException as e:
+ print("Failed to send data to coveralls: %s" % e)
+ sys.exit()
+
+ try:
+ response = r.json()
+ print("[coveralls] %s" % response['message'])
+ if 'url' in response:
+ print("[coveralls] Uploaded to %s" % response['url'])
+ except json.decoder.JSONDecodeError:
+ print("Bad response: '%s'" % r.text)
diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
new file mode 100644
index 0000000..f2c6c54
--- /dev/null
+++ b/utils/CMakeLists.txt
@@ -0,0 +1,22 @@
+SET(UTILSERVERSRC rspamd_http_server.c)
+SET(UTILBENCHSRC rspamd_http_bench.c)
+SET(BASE64SRC base64.c)
+SET(MIMESRC mime_tool.c)
+
+MACRO(ADD_UTIL NAME)
+ ADD_EXECUTABLE("${NAME}" "${ARGN}")
+ SET_TARGET_PROPERTIES("${NAME}" PROPERTIES LINKER_LANGUAGE CXX)
+ TARGET_LINK_LIBRARIES("${NAME}" rspamd-server)
+ IF (ENABLE_SNOWBALL MATCHES "ON")
+ TARGET_LINK_LIBRARIES("${NAME}" stemmer)
+ ENDIF()
+ TARGET_LINK_LIBRARIES("${NAME}" rspamd-hiredis)
+ TARGET_LINK_LIBRARIES("${NAME}" ${RSPAMD_REQUIRED_LIBRARIES})
+ENDMACRO()
+
+IF (ENABLE_UTILS MATCHES "ON")
+ ADD_UTIL(rspamd-http-server ${UTILSERVERSRC})
+ ADD_UTIL(rspamd-http-bench ${UTILBENCHSRC})
+ ADD_UTIL(rspamd-base64 ${BASE64SRC})
+ ADD_UTIL(rspamd-mime-tool ${MIMESRC})
+ENDIF()
diff --git a/utils/asn.pl b/utils/asn.pl
new file mode 100644
index 0000000..4d54bad
--- /dev/null
+++ b/utils/asn.pl
@@ -0,0 +1,331 @@
+#!/usr/bin/env perl
+#
+
+use warnings;
+use strict;
+use autodie;
+
+use File::Basename;
+use File::Fetch;
+use Getopt::Long;
+use Pod::Usage;
+
+use FindBin;
+use lib "$FindBin::Bin/extlib/lib/perl5";
+
+use URI;
+
+my %config = (
+ asn_sources => [
+ 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest',
+ 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest',
+ 'http://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest',
+ 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest',
+ 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
+ ],
+ bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz']
+);
+
+my $download_asn = 0;
+my $download_bgp = 0;
+my $download_target = "./";
+my $help = 0;
+my $man = 0;
+my $v4 = 1;
+my $v6 = 1;
+my $parse = 1;
+my $v4_zone = "asn.rspamd.com";
+my $v6_zone = "asn6.rspamd.com";
+my $v4_file = "asn.zone";
+my $v6_file = "asn6.zone";
+my $ns_servers = [ "asn-ns.rspamd.com", "asn-ns2.rspamd.com" ];
+my $unknown_placeholder = "--";
+
+GetOptions(
+ "download-asn" => \$download_asn,
+ "download-bgp" => \$download_bgp,
+ "4!" => \$v4,
+ "6!" => \$v6,
+ "parse!" => \$parse,
+ "target=s" => \$download_target,
+ "zone-v4=s" => \$v4_zone,
+ "zone-v6=s" => \$v6_zone,
+ "file-v4=s" => \$v4_file,
+ "file-v6=s" => \$v6_file,
+ "ns-server=s@" => \$ns_servers,
+ "help|?" => \$help,
+ "man" => \$man,
+ "unknown-placeholder" => \$unknown_placeholder,
+) or
+ pod2usage(2);
+
+pod2usage(1) if $help;
+pod2usage(-exitval => 0, -verbose => 2) if $man;
+
+if ($download_asn) {
+ foreach my $u (@{ $config{'asn_sources'} }) {
+ download_file($u);
+ }
+}
+
+if ($download_bgp) {
+ foreach my $u (@{ $config{'bgp_sources'} }) {
+ download_file($u);
+ }
+}
+
+if (!$parse) {
+ exit 0;
+}
+
+# Prefix to ASN map
+my $networks = { 4 => {}, 6 => {} };
+
+foreach my $u (@{ $config{'bgp_sources'} }) {
+ my $parsed = URI->new($u);
+ my $fname = $download_target . '/' . basename($parsed->path);
+
+ use constant {
+ F_MARKER => 0,
+ F_TIMESTAMP => 1,
+ F_PEER_IP => 3,
+ F_PEER_AS => 4,
+ F_PREFIX => 5,
+ F_AS_PATH => 6,
+ F_ORIGIN => 7,
+ };
+
+ open(my $bgpd, '-|', "bgpdump -v -M $fname") or die "can't start bgpdump: $!";
+
+ while (<$bgpd>) {
+ chomp;
+ my @e = split /\|/;
+ if ($e[F_MARKER] ne 'TABLE_DUMP2') {
+ warn "bad line: $_\n";
+ next;
+ }
+
+ my $origin_as;
+ my $prefix = $e[F_PREFIX];
+ my $ip_ver = 6;
+
+ if ($prefix =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/) {
+ $ip_ver = 4;
+ }
+
+ if ($e[F_AS_PATH]) {
+
+ # not empty AS_PATH
+ my @as_path = split /\s/, $e[F_AS_PATH];
+ $origin_as = pop @as_path;
+
+ if (substr($origin_as, 0, 1) eq '{') {
+
+ # route is aggregated
+ if ($origin_as =~ /^{(\d+)}$/) {
+
+ # single AS aggregated, just remove { } around
+ $origin_as = $1;
+ } else {
+
+ # use previous AS from AS_PATH
+ $origin_as = pop @as_path;
+ }
+ }
+
+ # strip bogus AS
+ while (is_bougus_asn($origin_as)) {
+ $origin_as = pop @as_path;
+ last if scalar @as_path == 0;
+ }
+ }
+
+ # empty AS_PATH or all AS_PATH elements was stripped as bogus - use
+ # PEER_AS as origin AS
+ $origin_as //= $e[F_PEER_AS];
+
+ $networks->{$ip_ver}{$prefix} = int($origin_as);
+ }
+}
+
+# Remove default routes
+delete $networks->{4}{'0.0.0.0/0'};
+delete $networks->{6}{'::/0'};
+
+# Now roughly detect countries
+my $as_info = {};
+
+# RIR statistics exchange format
+# https://www.apnic.net/publications/media-library/documents/resource-guidelines/rir-statistics-exchange-format
+# https://www.arin.net/knowledge/statistics/nro_extended_stats_format.pdf
+# first 7 fields for this two formats are same
+use constant {
+ F_REGISTRY => 0, # {afrinic,apnic,arin,iana,lacnic,ripencc}
+ F_CC => 1, # ISO 3166 2-letter country code
+ F_TYPE => 2, # {asn,ipv4,ipv6}
+ F_START => 3,
+ F_VALUE => 4,
+ F_DATE => 5,
+ F_STATUS => 6,
+};
+
+foreach my $u (@{ $config{'asn_sources'} }) {
+ my $parsed = URI->new($u);
+ my $fname = $download_target . '/' . basename($parsed->path);
+ open(my $fh, "<", $fname) or die "Cannot open $fname: $!";
+
+ while (<$fh>) {
+ next if /^\#/;
+ chomp;
+ my @elts = split /\|/;
+
+ if ($elts[F_TYPE] eq 'asn' && $elts[F_START] ne '*') {
+ my $as_start = int($elts[F_START]);
+ my $as_end = $as_start + int($elts[F_VALUE]) - 1;
+
+ for my $as ($as_start .. $as_end) {
+ $as_info->{$as}{'country'} = $elts[F_CC];
+ $as_info->{$as}{'rir'} = $elts[F_REGISTRY];
+ }
+ }
+ }
+}
+
+# Write zone files
+my $ns_list = join ' ', @{$ns_servers};
+my $zone_header = << "EOH";
+\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300
+\$NS 43200 $ns_list
+EOH
+
+if ($v4) {
+ # create temp file in the same dir so we can be sure that mv is atomic
+ my $out_dir = dirname($v4_file);
+ my $out_file = basename($v4_file);
+ my $temp_file = "$out_dir/.$out_file.tmp";
+ open my $v4_fh, '>', $temp_file;
+ print $v4_fh $zone_header;
+
+ while (my ($net, $asn) = each %{ $networks->{4} }) {
+ my $country = $as_info->{$asn}{'country'} || $unknown_placeholder;
+ my $rir = $as_info->{$asn}{'rir'} || $unknown_placeholder;
+
+ # "8.8.8.0/24 15169|8.8.8.0/24|US|arin|" for 8.8.8.8
+ printf $v4_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
+ }
+
+ close $v4_fh;
+ rename $temp_file, $v4_file;
+}
+
+if ($v6) {
+ my $out_dir = dirname($v6_file);
+ my $out_file = basename($v6_file);
+ my $temp_file = "$out_dir/.$out_file.tmp";
+ open my $v6_fh, '>', $temp_file;
+ print $v6_fh $zone_header;
+
+ while (my ($net, $asn) = each %{ $networks->{6} }) {
+ my $country = $as_info->{$asn}{'country'} || $unknown_placeholder;
+ my $rir = $as_info->{$asn}{'rir'} || $unknown_placeholder;
+
+ # "2606:4700:4700::/48 13335|2606:4700:4700::/48|US|arin|" for 2606:4700:4700::1111
+ printf $v6_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
+ }
+
+ close $v6_fh;
+ rename $temp_file, $v6_file;
+}
+
+exit 0;
+
+########################################################################
+
+sub download_file {
+ my ($url) = @_;
+
+ local $File::Fetch::WARN = 0;
+ local $File::Fetch::TIMEOUT = 180; # connectivity to ftp.lacnic.net is bad
+
+ my $ff = File::Fetch->new(uri => $url);
+ my $where = $ff->fetch(to => $download_target) or
+ die "$url: ", $ff->error;
+
+ return $where;
+}
+
+# Returns true if AS number is bogus
+# e. g. a private AS.
+# List of allocated and reserved AS:
+# https://www.iana.org/assignments/as-numbers/as-numbers.txt
+sub is_bougus_asn {
+ my $as = shift;
+
+ # 64496-64511 Reserved for use in documentation and sample code
+ # 64512-65534 Designated for private use
+ # 65535 Reserved
+ # 65536-65551 Reserved for use in documentation and sample code
+ # 65552-131071 Reserved
+ return 1 if $as >= 64496 && $as <= 131071;
+
+ # Reserved (RFC6996, RFC7300, RFC7607)
+ return 1 if $as == 0 || $as >= 4200000000;
+
+ return 0;
+}
+
+__END__
+
+=head1 NAME
+
+asn.pl - download and parse ASN data for Rspamd
+
+=head1 SYNOPSIS
+
+asn.pl [options]
+
+ Options:
+ --download-asn Download ASN data from RIRs
+ --download-bgp Download BGP full view dump from RIPE RIS
+ --target Where to download files (default: current dir)
+ --zone-v4 IPv4 zone (default: asn.rspamd.com)
+ --zone-v6 IPv6 zone (default: asn6.rspamd.com)
+ --file-v4 IPv4 zone file (default: ./asn.zone)
+ --file-v6 IPv6 zone (default: ./asn6.zone)
+ --unknown-placeholder Placeholder for unknown elements (default: --)
+ --help Brief help message
+ --man Full documentation
+
+=head1 OPTIONS
+
+=over 8
+
+=item B<--download-asn>
+
+Download ASN data from RIR.
+
+=item B<--download-bgp>
+
+Download GeoIP data from Ripe
+
+=item B<--target>
+
+Specifies where to download files.
+
+=item B<--help>
+
+Print a brief help message and exits.
+
+=item B<--man>
+
+Prints the manual page and exits.
+
+=back
+
+=head1 DESCRIPTION
+
+B<asn.pl> is intended to download ASN data and GeoIP data and create a rbldnsd zone.
+
+=cut
+
+# vim: et:ts=4:sw=4
diff --git a/utils/base64.c b/utils/base64.c
new file mode 100644
index 0000000..d1202db
--- /dev/null
+++ b/utils/base64.c
@@ -0,0 +1,89 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "printf.h"
+#include "util.h"
+#include "cryptobox.h"
+#include "unix-std.h"
+
+static gdouble total_time = 0;
+
+
+static void
+rspamd_process_file(const gchar *fname, gint decode)
+{
+ gint fd;
+ gpointer map;
+ struct stat st;
+ guint8 *dest;
+ gsize destlen;
+
+ fd = open(fname, O_RDONLY);
+
+ if (fd == -1) {
+ rspamd_fprintf(stderr, "cannot open %s: %s", fname, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ if (fstat(fd, &st) == -1) {
+ rspamd_fprintf(stderr, "cannot stat %s: %s", fname, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ map = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
+ close(fd);
+
+ if (map == MAP_FAILED) {
+ rspamd_fprintf(stderr, "cannot mmap %s: %s", fname, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ if (decode) {
+ destlen = st.st_size / 4 * 3 + 10;
+ dest = g_malloc(destlen);
+ rspamd_cryptobox_base64_decode(map, st.st_size, dest, &destlen);
+ }
+ else {
+ dest = rspamd_encode_base64(map, st.st_size, 80, &destlen);
+ }
+
+ rspamd_printf("%*s", (gint) destlen, dest);
+ g_free(dest);
+
+ munmap(map, st.st_size);
+}
+
+int main(int argc, char **argv)
+{
+ gint i, start = 1, decode = 0;
+
+ if (argc > 2 && *argv[1] == '-') {
+ start = 2;
+
+ if (argv[1][1] == 'd') {
+ decode = 1;
+ }
+ }
+
+ for (i = start; i < argc; i++) {
+ if (argv[i]) {
+ rspamd_process_file(argv[i], decode);
+ }
+ }
+
+ return 0;
+}
diff --git a/utils/cgp_rspamd.pl b/utils/cgp_rspamd.pl
new file mode 100644
index 0000000..6898d26
--- /dev/null
+++ b/utils/cgp_rspamd.pl
@@ -0,0 +1,357 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+use JSON::XS;
+use AnyEvent;
+use AnyEvent::HTTP;
+use AnyEvent::IO;
+use EV;
+use Pod::Usage;
+use Getopt::Long;
+use File::stat;
+
+my $rspamd_host = "localhost:11333";
+my $man = 0;
+my $help = 0;
+my $local = 0;
+my $header = "X-Spam: yes";
+my $max_size = 10 * 1024 * 1024; # 10 MB
+my $request_timeout = 15; # 15 seconds by default
+my $reject_message = "Spam message rejected";
+
+GetOptions(
+ "host=s" => \$rspamd_host,
+ "header=s" => \$header,
+ "reject-message=s" => \$reject_message,
+ "max-size=i" => \$max_size,
+ "timeout=f" => \$request_timeout,
+ "help|?" => \$help,
+ "man" => \$man
+) or pod2usage(2);
+
+pod2usage(1) if $help;
+pod2usage( -exitval => 0, -verbose => 2 ) if $man;
+
+my $main_domain = cgp_main_domain();
+my $scanned = 0;
+
+# Turn off bufferization as required by CGP
+$| = 1;
+
+sub cgp_main_domain {
+ if ( open( my $fh, 'Settings/Main.settings' ) ) {
+ while (<$fh>) {
+ if (/^\s+DomainName\s+=\s+([^;]+);/) {
+ return $1;
+ }
+ }
+ }
+}
+
+sub cgp_string {
+ my ($in) = @_;
+
+ $in =~ s/\"/\\"/g;
+ $in =~ s/\n/\\n/gms;
+ $in =~ s/\r/\\r/mgs;
+ $in =~ s/\t/ /g;
+
+ return "\"$in\"";
+}
+
+sub rspamd_scan {
+ my ( $tag, $file ) = @_;
+
+ my $http_callback = sub {
+ my ( $body, $hdr ) = @_;
+
+ if ( $hdr && $hdr->{Status} =~ /^2/ ) {
+ my $js = eval('decode_json($body)');
+ $scanned++;
+
+ if ( !$js ) {
+ print "* Rspamd: Bad response for $file: invalid JSON: parse error\n";
+ print "$tag FAILURE\n";
+ }
+ else {
+ my $def = $js;
+ my $headers = "";
+
+ if ( !$def ) {
+ print "* Rspamd: Bad response for $file: invalid JSON: default is missing\n";
+ print "$tag FAILURE\n";
+ }
+ else {
+ my $action = $def->{'action'};
+ my $id = $js->{'message-id'};
+
+ my $symbols = "";
+ while ( my ( $k, $s ) = each( %{ $def->{'symbols'} } ) ) {
+ $symbols .= sprintf "%s(%.2f);", $k, $s->{'score'};
+ }
+
+ printf
+ "* Rspamd: Scanned %s; id: <%s>; Score: %.2f / %.2f; Symbols: [%s]\n",
+ $file, $id, $def->{'score'}, $def->{'required_score'}, $symbols;
+
+ if ( $js->{'dkim-signature'} ) {
+ $headers .= "DKIM-Signature: " . $js->{'dkim-signature'};
+ }
+
+ if ( $js->{'milter'} ) {
+ my $block = $js->{'milter'};
+
+ if ( $block->{'add_headers'} ) {
+ while ( my ( $h, $v ) = each( %{ $block->{'add_headers'} } ) ) {
+ if ( ref($v) eq 'HASH' ) {
+ if ( $headers eq "" ) {
+ $headers .= "$h: $v->{value}";
+ }
+ else {
+ $headers .= "\\e$h: $v->{value}";
+ }
+ }
+ else {
+ if ( $headers eq "" ) {
+ $headers .= "$h: $v";
+ }
+ else {
+ $headers .= "\\e$h: $v";
+ }
+ }
+ }
+ }
+ }
+
+ if ( $action eq 'reject' ) {
+ print "$tag DISCARD\n";
+ return;
+ }
+ elsif ( $action eq 'add header' || $action eq 'rewrite subject' ) {
+ if ( $headers eq "" ) {
+ $headers .= "$header";
+ }
+ else {
+ $headers .= "\\e$header";
+ }
+ }
+ elsif ( $action eq 'soft reject' ) {
+ print "$tag REJECTED Try again later\n";
+ return;
+ }
+
+ if ( $headers eq "" ) {
+ print "$tag OK\n";
+ }
+ else {
+ print "$tag ADDHEADER " . cgp_string($headers) . " OK\n";
+ }
+ }
+ }
+ }
+ else {
+ if ($hdr) {
+ print "* Rspamd: Bad response for $file: HTTP error: $hdr->{Status} $hdr->{Reason}\n";
+ }
+ else {
+ print "* Rspamd: Bad response for $file: IO error: $!\n";
+ }
+ print "$tag FAILURE\n";
+ }
+ };
+
+ if ($local) {
+
+ # Use file scan
+ # XXX: not implemented now due to CGP queue format
+ http_get(
+ "http://$rspamd_host/symbols?file=$file",
+ timeout => $request_timeout,
+ $http_callback
+ );
+ }
+ else {
+ my $sb = stat($file);
+
+ if ( !$sb || $sb->size > $max_size ) {
+ if ($sb) {
+ print "* File $file is too large: " . $sb->size . "\n$tag FAILURE\n";
+
+ }
+ else {
+ print "* Cannot stat $file: $!\n$tag FAILURE\n";
+ }
+ return;
+ }
+ aio_load(
+ $file,
+ sub {
+ my ($data) = @_;
+
+ if ( !$data ) {
+ print "* Cannot open $file: $!\n$tag FAILURE\n";
+ return;
+ }
+
+ # Parse CGP format
+ $data =~ s/^((?:[^\n]*\n)*?)\n(.*)$/$2/ms;
+ my @envelope = split /\n/, $1;
+ chomp(@envelope);
+ my $from;
+ my @rcpts;
+ my $ip;
+ my $user;
+
+ foreach my $elt (@envelope) {
+ if ( $elt =~ /^P\s[^<]*(<[^>]*>).*$/ ) {
+ $from = $1;
+ }
+ elsif ( $elt =~ /^R\s[^<]*(<[^>]*>).*$/ ) {
+ push @rcpts, $1;
+ }
+ elsif ( $elt =~ /^S (?:<([^>]+)> )?(?:SMTP|HTTPU?|AIRSYNC|XIMSS) \[([0-9a-f.:]+)\]/ ) {
+ if ($1) {
+ $user = $1;
+ }
+ if ($2) {
+ $ip = $2;
+ }
+ }
+ elsif ( $elt =~ /^S (?:<([^>]+)> )?(?:DSN|GROUP|LIST|PBX|PIPE|RULE) \[0\.0\.0\.0\]/ ) {
+ if ($1) {
+ $user = $1;
+ }
+ $ip = '127.2.4.7';
+ }
+ }
+
+ my $headers = {};
+ if ( $file =~ /\/([^\/.]+)\.msg$/ ) {
+ $headers->{'Queue-ID'} = $1;
+ }
+ if ($from) {
+ $headers->{From} = $from;
+ }
+ if ( scalar(@rcpts) > 0 ) {
+
+ # XXX: Anyevent cannot parse headers with multiple values
+ $headers->{Rcpt} = join( ',', @rcpts );
+ }
+ if ($ip) {
+ $headers->{IP} = $ip;
+ }
+ if ($user) {
+ $headers->{User} = $user;
+ }
+ if ($main_domain) {
+ $headers->{'MTA-Tag'} = $main_domain;
+ }
+
+ http_post(
+ "http://$rspamd_host/checkv2", $data,
+ timeout => $request_timeout,
+ headers => $headers,
+ $http_callback
+ );
+ }
+ );
+ }
+}
+
+# Show informational message
+print "* Rspamd CGP filter has been started\n";
+
+my $w = AnyEvent->io(
+ fh => \*STDIN,
+ poll => 'r',
+ cb => sub {
+ chomp( my $input = <STDIN> );
+
+ if ( $input =~ /^(\d+)\s+(\S+)(\s+(\S+)\s*)?$/ ) {
+ my $tag = $1;
+ my $cmd = $2;
+
+ if ( $cmd eq "INTF" ) {
+ print "$input\n";
+ }
+ elsif ( $cmd eq "FILE" && $4 ) {
+ my $file = $4;
+ print "* Scanning file $file\n";
+ rspamd_scan $tag, $file;
+ }
+ elsif ( $cmd eq "QUIT" ) {
+ print "* Terminating after scanning of $scanned files\n";
+ print "$tag OK\n";
+ exit 0;
+ }
+ else {
+ print "* Unknown command $cmd\n";
+ print "$tag FAILURE\n";
+ }
+ }
+ }
+);
+
+EV::run;
+
+__END__
+
+=head1 NAME
+
+cgp_rspamd - implements Rspamd filter for CommunigatePro MTA
+
+=head1 SYNOPSIS
+
+cgp_rspamd [options]
+
+ Options:
+ --host=hostport Rspamd host to connect (localhost:11333 by default)
+ --header Add specific header for a spam message ("X-Spam: yes" by default)
+ --reject-message Rejection message for spam mail ("Spam message rejected" by default)
+ --timeout Timeout to read response from Rspamd (15 seconds by default)
+ --max-size Maximum size of message to scan (10 megabytes by default)
+ --help brief help message
+ --man full documentation
+
+=head1 OPTIONS
+
+=over 8
+
+=item B<--host>
+
+Specifies Rspamd host to use for scanning
+
+=item B<--header>
+
+Specifies the header that should be added when Rspamd action is B<add header> or B<rewrite subject>.
+
+=item B<--reject-message>
+
+Specifies the rejection message for spam.
+
+=item B<--timeout>
+
+Sets timeout in seconds for waiting Rspamd reply for a message.
+
+=item B<--max-size>
+
+Define the maximum messages size to be processed by Rspamd in bytes.
+
+=item B<--help>
+
+Print a brief help message and exits.
+
+=item B<--man>
+
+Prints the manual page and exits.
+
+=back
+
+=head1 DESCRIPTION
+
+B<cgp_rspamd> is intended to scan messages processed with B<CommunigatePro> MTA on some Rspamd scanner. It reads
+standard input and parses CGP helpers protocol. On scan requests, this filter can query Rspamd to process a message.
+B<cgp_rspamd> can tell CGP to add header or reject SPAM messages depending on Rspamd scan result.
+
+=cut
diff --git a/utils/classifier_test.pl b/utils/classifier_test.pl
new file mode 100644
index 0000000..238417f
--- /dev/null
+++ b/utils/classifier_test.pl
@@ -0,0 +1,539 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+use Pod::Usage;
+use Getopt::Long;
+use Time::HiRes qw(gettimeofday tv_interval);
+use JSON::XS;
+use String::ShellQuote;
+use FileHandle;
+use IPC::Open2;
+use Data::Dumper;
+
+my $spam_dir;
+my $ham_dir;
+my $parallel = 1;
+my $classifier = "bayes";
+my $spam_symbol = "BAYES_SPAM";
+my $ham_symbol = "BAYES_HAM";
+my $timeout = 10;
+my $rspamc = $ENV{'RSPAMC'} || "rspamc";
+my $bogofilter = $ENV{'BOGOFILTER'} || "bogofilter";
+my $dspam = $ENV{'DSPAM'} || "dspam";
+my $train_fraction = 0.5;
+my $use_bogofilter = 0;
+my $use_dspam = 0;
+my $check_only = 0;
+my $rspamc_prob_trigger = 95;
+my $man;
+my $help;
+
+GetOptions(
+ "spam|s=s" => \$spam_dir,
+ "ham|h=s" => \$ham_dir,
+ "spam-symbol=s" => \$spam_symbol,
+ "ham-symbol=s" => \$ham_symbol,
+ "classifier|c=s" => \$classifier,
+ "timeout|t=f" => \$timeout,
+ "parallel|p=i" => \$parallel,
+ "train-fraction|t=f" => \$train_fraction,
+ "bogofilter|b" => \$use_bogofilter,
+ "dspam|d" => \$use_dspam,
+ "check-only" => \$check_only,
+ "help|?" => \$help,
+ "man" => \$man
+) or pod2usage(2);
+
+pod2usage(1) if $help;
+pod2usage( -exitval => 0, -verbose => 2 ) if $man;
+
+sub read_dir_files {
+ my ( $dir, $target ) = @_;
+ opendir( my $dh, $dir ) or die "cannot open dir $dir: $!";
+ while ( my $file = readdir $dh ) {
+ if ( -f "$dir/$file" ) {
+ push @{$target}, "$dir/$file";
+ }
+ }
+}
+
+sub shuffle_array {
+ my ($ar) = @_;
+
+ for ( my $i = 0 ; $i < scalar @{$ar} ; $i++ ) {
+ if ( $i > 1 ) {
+ my $sel = int( rand( $i - 1 ) );
+ ( @{$ar}[$i], @{$ar}[$sel] ) = ( @{$ar}[$sel], @{$ar}[$i] );
+ }
+ }
+}
+
+sub learn_rspamc {
+ my ( $files, $spam ) = @_;
+ my $processed = 0;
+
+ my $cmd = $spam ? "learn_spam" : "learn_ham";
+ my $args_quoted = shell_quote @{$files};
+ open( my $p, "$rspamc -t $timeout -c $classifier --compact -j -n $parallel $cmd $args_quoted |" )
+ or die "cannot spawn $rspamc: $!";
+
+ while (<$p>) {
+ my $res = eval('decode_json($_)');
+ if ( $res && $res->{'success'} ) {
+ $processed++;
+ }
+ }
+
+ return $processed;
+}
+
+sub learn_bogofilter {
+ my ( $files, $spam ) = @_;
+ my $processed = 0;
+
+ foreach my $f ( @{$files} ) {
+ my $args_quoted = shell_quote $f;
+ my $fl = $spam ? "-s" : "-n";
+ `$bogofilter -I $args_quoted $fl`;
+ if ( $? == 0 ) {
+ $processed++;
+ }
+ }
+
+ return $processed;
+}
+
+sub learn_dspam {
+ my ( $files, $spam ) = @_;
+ my $processed = 0;
+
+ foreach my $f ( @{$files} ) {
+ my $args_quoted = shell_quote $f;
+ my $fl = $spam ? "--class=spam" : "--class=innocent";
+ open( my $p, "|$dspam --user nobody --source=corpus --stdout --mode=toe $fl" )
+ or die "cannot run $dspam: $!";
+
+ open( my $inp, "< $f" );
+ while (<$inp>) {
+ print $p $_;
+ }
+ }
+
+ return $processed;
+}
+
+sub learn_samples {
+ my ( $ar_ham, $ar_spam ) = @_;
+ my $len;
+ my $processed = 0;
+ my $total = 0;
+ my $learn_func;
+
+ my @files_spam;
+ my @files_ham;
+
+ if ($use_dspam) {
+ $learn_func = \&learn_dspam;
+ }
+ elsif ($use_bogofilter) {
+ $learn_func = \&learn_bogofilter;
+ }
+ else {
+ $learn_func = \&learn_rspamc;
+ }
+
+ $len = int( scalar @{$ar_ham} * $train_fraction );
+ my @cur_vec;
+
+ # Shuffle spam and ham samples
+ for ( my $i = 0 ; $i < $len ; $i++ ) {
+ if ( $i > 0 && ( $i % $parallel == 0 || $i == $len - 1 ) ) {
+ push @cur_vec, @{$ar_ham}[$i];
+ push @files_ham, [@cur_vec];
+ @cur_vec = ();
+ $total++;
+ }
+ else {
+ push @cur_vec, @{$ar_ham}[$i];
+ }
+ }
+
+ $len = int( scalar @{$ar_spam} * $train_fraction );
+ @cur_vec = ();
+ for ( my $i = 0 ; $i < $len ; $i++ ) {
+ if ( $i > 0 && ( $i % $parallel == 0 || $i == $len - 1 ) ) {
+ push @cur_vec, @{$ar_spam}[$i];
+ push @files_spam, [@cur_vec];
+ @cur_vec = ();
+ $total++;
+ }
+ else {
+ push @cur_vec, @{$ar_spam}[$i];
+ }
+ }
+
+ for ( my $i = 0 ; $i < $total ; $i++ ) {
+ my $args;
+ my $spam;
+
+ if ( $i % 2 == 0 ) {
+ $args = pop @files_spam;
+
+ if ( !$args ) {
+ $args = pop @files_ham;
+ $spam = 0;
+ }
+ else {
+ $spam = 1;
+ }
+ }
+ else {
+ $args = pop @files_ham;
+ if ( !$args ) {
+ $args = pop @files_spam;
+ $spam = 1;
+ }
+ else {
+ $spam = 0;
+ }
+ }
+
+ my $r = $learn_func->( $args, $spam );
+ if ($r) {
+ $processed += $r;
+ }
+ }
+
+ return $processed;
+}
+
+sub check_rspamc {
+ my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_;
+
+ my $args_quoted = shell_quote @{$files};
+ my $processed = 0;
+
+ open(
+ my $p,
+"$rspamc -t $timeout -n $parallel --header=\"Settings: {symbols_enabled=[BAYES_SPAM]}\" --compact -j $args_quoted |"
+ ) or die "cannot spawn $rspamc: $!";
+
+ while (<$p>) {
+ my $res = eval('decode_json($_)');
+ if ( $res && $res->{'default'} ) {
+ $processed++;
+
+ if ($spam) {
+ if ( $res->{'default'}->{$ham_symbol} ) {
+ my $m = $res->{'default'}->{$ham_symbol}->{'options'}->[0];
+ if ( $m && $m =~ /^(\d+(?:\.\d+)?)%$/ ) {
+ my $percentage = int($1);
+ if ( $percentage >= $rspamc_prob_trigger ) {
+ $$fp_cnt++;
+ }
+ }
+ else {
+ $$fp_cnt++;
+ }
+ }
+ elsif ( !$res->{'default'}->{$spam_symbol} ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
+ else {
+ if ( $res->{'default'}->{$spam_symbol} ) {
+ my $m = $res->{'default'}->{$spam_symbol}->{'options'}->[0];
+ if ( $m && $m =~ /^(\d+(?:\.\d+)?)%$/ ) {
+
+ my $percentage = int($1);
+ if ( $percentage >= $rspamc_prob_trigger ) {
+ $$fp_cnt++;
+ }
+ }
+ else {
+ $$fp_cnt++;
+ }
+ }
+ elsif ( !$res->{'default'}->{$ham_symbol} ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
+ }
+ }
+
+ return $processed;
+}
+
+sub check_bogofilter {
+ my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_;
+ my $processed = 0;
+
+ foreach my $f ( @{$files} ) {
+ my $args_quoted = shell_quote $f;
+
+ open( my $p, "$bogofilter -t -I $args_quoted |" )
+ or die "cannot spawn $bogofilter: $!";
+
+ while (<$p>) {
+ if ( $_ =~ /^([SHU])\s+.*$/ ) {
+ $processed++;
+
+ if ($spam) {
+ if ( $1 eq 'H' ) {
+ $$fp_cnt++;
+ }
+ elsif ( $1 eq 'U' ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
+ else {
+ if ( $1 eq 'S' ) {
+ $$fp_cnt++;
+ }
+ elsif ( $1 eq 'U' ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
+ }
+ }
+ }
+
+ return $processed;
+}
+
+sub check_dspam {
+ my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_;
+ my $processed = 0;
+
+ foreach my $f ( @{$files} ) {
+ my $args_quoted = shell_quote $f;
+
+ my $pid = open2( *Reader, *Writer, "$dspam --user nobody --classify --stdout --mode=notrain" );
+ open( my $inp, "< $f" );
+ while (<$inp>) {
+ print Writer $_;
+ }
+ close Writer;
+
+ while (<Reader>) {
+ if ( $_ =~ qr(^X-DSPAM-Result: nobody; result="([^"]+)"; class="[^"]+"; probability=(\d+(?:\.\d+)?).*$) ) {
+ $processed++;
+ my $percentage = int( $2 * 100.0 );
+
+ if ($spam) {
+ if ( $1 eq 'Innocent' ) {
+ if ( $percentage <= ( 100 - $rspamc_prob_trigger ) ) {
+ $$fp_cnt++;
+ }
+ }
+ elsif ( $1 ne 'Spam' ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
+ else {
+ if ( $1 eq 'Spam' ) {
+ if ( $percentage >= $rspamc_prob_trigger ) {
+ $$fp_cnt++;
+ }
+ }
+ elsif ( $1 ne 'Innocent' ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
+ }
+ }
+ close Reader;
+ waitpid( $pid, 0 );
+ }
+
+ return $processed;
+}
+
+sub cross_validate {
+ my ($hr) = @_;
+ my $args = "";
+ my $processed = 0;
+ my $fp_spam = 0;
+ my $fn_spam = 0;
+ my $fp_ham = 0;
+ my $fn_ham = 0;
+ my $total_spam = 0;
+ my $total_ham = 0;
+ my $detected_spam = 0;
+ my $detected_ham = 0;
+ my $i = 0;
+ my $len = scalar keys %{$hr};
+ my @files_spam;
+ my @files_ham;
+ my @cur_spam;
+ my @cur_ham;
+ my $check_func;
+
+ if ($use_dspam) {
+ $check_func = \&check_dspam;
+ }
+ elsif ($use_bogofilter) {
+ $check_func = \&check_bogofilter;
+ }
+ else {
+ $check_func = \&check_rspamc;
+ }
+
+ while ( my ( $fn, $spam ) = each( %{$hr} ) ) {
+ if ($spam) {
+ if ( scalar @cur_spam >= $parallel || $i == $len - 1 ) {
+ push @cur_spam, $fn;
+ push @files_spam, [@cur_spam];
+ @cur_spam = ();
+ }
+ else {
+ push @cur_spam, $fn;
+ }
+ }
+ else {
+ if ( scalar @cur_ham >= $parallel || $i == $len - 1 ) {
+ push @cur_ham, $fn;
+ push @files_ham, [@cur_ham];
+ @cur_ham = ();
+ }
+ else {
+ push @cur_ham, $fn;
+ }
+ }
+ }
+
+ shuffle_array( \@files_spam );
+
+ foreach my $fn (@files_spam) {
+ my $r = $check_func->( $fn, 1, \$fp_ham, \$fn_spam, \$detected_spam );
+ $total_spam += $r;
+ $processed += $r;
+ }
+
+ shuffle_array( \@files_ham );
+
+ foreach my $fn (@files_ham) {
+ my $r = $check_func->( $fn, 0, \$fp_spam, \$fn_ham, \$detected_ham );
+ $total_ham += $r;
+ $processed += $r;
+ }
+
+ printf "Scanned %d messages
+%d spam messages (%d detected)
+%d ham messages (%d detected)\n", $processed, $total_spam, $detected_spam, $total_ham, $detected_ham;
+
+ printf "\nHam FP rate: %.2f%% (%d messages)
+Ham FN rate: %.2f%% (%d messages)\n", $fp_ham / $total_ham * 100.0, $fp_ham, $fn_ham / $total_ham * 100.0, $fn_ham;
+
+ printf "\nSpam FP rate: %.2f%% (%d messages)
+Spam FN rate: %.2f%% (%d messages)\n",
+ $fp_spam / $total_spam * 100.0, $fp_spam,
+ $fn_spam / $total_spam * 100.0, $fn_spam;
+}
+
+if ( !$spam_dir || !$ham_dir ) {
+ die "spam or/and ham directories are not specified";
+}
+
+my @spam_samples;
+my @ham_samples;
+
+read_dir_files( $spam_dir, \@spam_samples );
+read_dir_files( $ham_dir, \@ham_samples );
+shuffle_array( \@spam_samples );
+shuffle_array( \@ham_samples );
+
+if ( !$check_only ) {
+ my $learned = 0;
+ my $t0 = [gettimeofday];
+ $learned = learn_samples( \@ham_samples, \@spam_samples );
+ my $t1 = [gettimeofday];
+
+ printf "Learned classifier, %d items processed, %.2f seconds elapsed\n", $learned, tv_interval( $t0, $t1 );
+}
+
+my %validation_set;
+my $len = int( scalar @spam_samples * $train_fraction );
+for ( my $i = $len ; $i < scalar @spam_samples ; $i++ ) {
+ $validation_set{ $spam_samples[$i] } = 1;
+}
+
+$len = int( scalar @ham_samples * $train_fraction );
+for ( my $i = $len ; $i < scalar @spam_samples ; $i++ ) {
+ $validation_set{ $ham_samples[$i] } = 0;
+}
+
+cross_validate( \%validation_set );
+
+__END__
+
+=head1 NAME
+
+classifier_test.pl - test various parameters for a classifier
+
+=head1 SYNOPSIS
+
+classifier_test.pl [options]
+
+ Options:
+ --spam Directory with spam files
+ --ham Directory with ham files
+ --spam-symbol Symbol for spam (default: BAYES_SPAM)
+ --ham-symbol Symbol for ham (default: BAYES_HAM)
+ --classifier Classifier to test (default: bayes)
+ --timeout Timeout for rspamc (default: 10)
+ --parallel Parallel execution (default: 1)
+ --help Brief help message
+ --man Full documentation
+
+=head1 OPTIONS
+
+=over 8
+
+=item B<--spam>
+
+Directory with spam files.
+
+=item B<--ham>
+
+Directory with ham files.
+
+=item B<--classifier>
+
+Specifies classifier name to test.
+
+=item B<--help>
+
+Print a brief help message and exits.
+
+=item B<--man>
+
+Prints the manual page and exits.
+
+=back
+
+=head1 DESCRIPTION
+
+B<classifier_test.pl> is intended to test Rspamd classifier for false positives, false negatives and other parameters.
+It uses half of the corpus for training and half for cross-validation.
+
+=cut
diff --git a/utils/fann_train.pl b/utils/fann_train.pl
new file mode 100755
index 0000000..2ce422e
--- /dev/null
+++ b/utils/fann_train.pl
@@ -0,0 +1,247 @@
+#!/usr/bin/env perl
+
+# This script is a very simple prototype to learn fann from rspamd logs
+# For now, it is intended for internal use only
+
+use strict;
+use warnings FATAL => 'all';
+use AI::FANN qw(:all);
+use Getopt::Std;
+
+my %sym_idx; # Symbols by index
+my %sym_names; # Symbols by name
+my $num = 1; # Number of symbols
+my @spam;
+my @ham;
+my $max_samples = -1;
+my $split = 1;
+my $preprocessed = 0; # output is in format <score>:<0|1>:<SYM1,...SYMN>
+my $score_spam = 12;
+my $score_ham = -6;
+
+sub process {
+ my ( $input, $spam, $ham ) = @_;
+ my $samples = 0;
+
+ while (<$input>) {
+ if ( !$preprocessed ) {
+ if (/^.*rspamd_task_write_log.*: \[(-?\d+\.?\d*)\/(\d+\.?\d*)\]\s*\[(.+)\].*$/) {
+ if ( $1 > $score_spam ) {
+ $_ = "$1:1: $3";
+ }
+ elsif ( $1 < $score_ham ) {
+ $_ = "$1:0: $3\n";
+ }
+ else {
+ # Out of boundary
+ next;
+ }
+ }
+ else {
+ # Not our log message
+ next;
+ }
+ }
+
+ $_ =~ /^(-?\d+\.?\d*):([01]):\s*(\S.*)$/;
+
+ my $is_spam = 0;
+
+ if ( $2 == 1 ) {
+ $is_spam = 1;
+ }
+
+ my @ar = split /,/, $3;
+ my %sample;
+
+ foreach my $sym (@ar) {
+ chomp $sym;
+ if ( !$sym_idx{$sym} ) {
+ $sym_idx{$sym} = $num;
+ $sym_names{$num} = $sym;
+ $num++;
+ }
+
+ $sample{ $sym_idx{$sym} } = 1;
+ }
+
+ if ($is_spam) {
+ push @{$spam}, \%sample;
+ }
+ else {
+ push @{$ham}, \%sample;
+ }
+
+ $samples++;
+ if ( $max_samples > 0 && $samples > $max_samples ) {
+ return;
+ }
+ }
+}
+
+# Shuffle array
+sub fisher_yates_shuffle {
+ my $array = shift;
+ my $i = @$array;
+
+ while ( --$i ) {
+ my $j = int rand( $i + 1 );
+ @$array[ $i, $j ] = @$array[ $j, $i ];
+ }
+}
+
+# Train network
+sub train {
+ my ( $ann, $sample, $result ) = @_;
+
+ my @row;
+
+ for ( my $i = 1 ; $i < $num ; $i++ ) {
+ if ( $sample->{$i} ) {
+ push @row, 1;
+ }
+ else {
+ push @row, 0;
+ }
+ }
+
+ #print "@row -> @{$result}\n";
+
+ $ann->train( \@row, \@{$result} );
+}
+
+sub test {
+ my ( $ann, $sample ) = @_;
+
+ my @row;
+
+ for ( my $i = 1 ; $i < $num ; $i++ ) {
+ if ( $sample->{$i} ) {
+ push @row, 1;
+ }
+ else {
+ push @row, 0;
+ }
+ }
+
+ my $ret = $ann->run( \@row );
+
+ return $ret;
+}
+
+my %opts;
+getopts( 'o:i:s:n:t:hpS:H:', \%opts );
+
+if ( $opts{'h'} ) {
+ print "$0 [-i input] [-o output] [-s scores] [-n max_samples] [-S spam_score] [-H ham_score] [-ph]\n";
+ exit;
+}
+
+my $input = *STDIN;
+
+if ( $opts{'i'} ) {
+ open( $input, '<', $opts{'i'} ) or die "cannot open $opts{i}";
+}
+
+if ( $opts{'n'} ) {
+ $max_samples = $opts{'n'};
+}
+
+if ( $opts{'t'} ) {
+
+ # Test split
+ $split = $opts{'t'};
+}
+if ( $opts{'p'} ) {
+ $preprocessed = 1;
+}
+
+if ( $opts{'H'} ) {
+ $score_ham = $opts{'H'};
+}
+
+if ( $opts{'S'} ) {
+ $score_spam = $opts{'S'};
+}
+
+# ham_prob, spam_prob
+my @spam_out = (1);
+my @ham_out = (0);
+
+process( $input, \@spam, \@ham );
+fisher_yates_shuffle( \@spam );
+fisher_yates_shuffle( \@ham );
+
+my $nspam = int( scalar(@spam) / $split );
+my $nham = int( scalar(@ham) / $split );
+
+my $ann = AI::FANN->new_standard( $num - 1, ( $num + 2 ) / 2, 1 );
+
+my @train_data;
+
+# Train ANN
+for ( my $i = 0 ; $i < $nham ; $i++ ) {
+ push @train_data, [ $ham[$i], \@ham_out ];
+}
+
+for ( my $i = 0 ; $i < $nspam ; $i++ ) {
+ push @train_data, [ $spam[$i], \@spam_out ];
+}
+
+fisher_yates_shuffle( \@train_data );
+
+foreach my $train_row (@train_data) {
+ train( $ann, @{$train_row}[0], @{$train_row}[1] );
+}
+
+print "Trained $nspam SPAM and $nham HAM samples\n";
+
+# Now run fann
+if ( $split > 1 ) {
+ my $sample = 0.0;
+ my $correct = 0.0;
+ for ( my $i = $nham ; $i < $nham * $split ; $i++ ) {
+ my $ret = test( $ann, $ham[$i] );
+
+ #print "@{$ret}\n";
+ if ( @{$ret}[0] < 0.5 ) {
+ $correct++;
+ }
+ $sample++;
+ }
+
+ print "Tested $sample HAM samples, correct matched: $correct, rate: " . ( $correct / $sample ) . "\n";
+
+ $sample = 0.0;
+ $correct = 0.0;
+
+ for ( my $i = $nspam ; $i < $nspam * $split ; $i++ ) {
+ my $ret = test( $ann, $spam[$i] );
+
+ #print "@{$ret}\n";
+ if ( @{$ret}[0] > 0.5 ) {
+ $correct++;
+ }
+ $sample++;
+ }
+
+ print "Tested $sample SPAM samples, correct matched: $correct, rate: " . ( $correct / $sample ) . "\n";
+}
+
+if ( $opts{'o'} ) {
+ $ann->save( $opts{'o'} ) or die "cannot save ann into $opts{o}";
+}
+
+if ( $opts{'s'} ) {
+ open( my $scores, '>', $opts{'s'} ) or die "cannot open score file $opts{'s'}";
+ print $scores "{";
+ for ( my $i = 1 ; $i < $num ; $i++ ) {
+ my $n = $i - 1;
+ if ( $i != $num - 1 ) {
+ print $scores "\"$sym_names{$i}\":$n,";
+ }
+ else {
+ print $scores "\"$sym_names{$i}\":$n}\n";
+ }
+ }
+}
diff --git a/utils/rspamd_http_bench.c b/utils/rspamd_http_bench.c
new file mode 100644
index 0000000..232fc8a
--- /dev/null
+++ b/utils/rspamd_http_bench.c
@@ -0,0 +1,411 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "util.h"
+#include "libutil/http.h"
+#include "libutil/http_private.h"
+#include "ottery.h"
+#include "cryptobox.h"
+#include "unix-std.h"
+#include <math.h>
+#include <netinet/tcp.h>
+
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+
+static guint port = 43000;
+static gchar *host = "127.0.0.1";
+static gchar *server_key = NULL;
+static guint cache_size = 10;
+static guint nworkers = 1;
+static gboolean openssl_mode = FALSE;
+static guint file_size = 500;
+static guint pconns = 100;
+static gdouble test_time = 10.0;
+static gchar *latencies_file = NULL;
+static gboolean csv_output = FALSE;
+
+/* Dynamic vars */
+static rspamd_inet_addr_t *addr;
+static guint32 workers_left = 0;
+static guint32 *conns_done = NULL;
+static const guint store_latencies = 1000;
+static guint32 conns_pending = 0;
+
+static GOptionEntry entries[] = {
+ {"port", 'p', 0, G_OPTION_ARG_INT, &port,
+ "Port number (default: 43000)", NULL},
+ {"cache", 'c', 0, G_OPTION_ARG_INT, &cache_size,
+ "Keys cache size (default: 10)", NULL},
+ {"workers", 'n', 0, G_OPTION_ARG_INT, &nworkers,
+ "Number of workers to start (default: 1)", NULL},
+ {"size", 's', 0, G_OPTION_ARG_INT, &file_size,
+ "Size of payload to transfer (default: 500)", NULL},
+ {"conns", 'C', 0, G_OPTION_ARG_INT, &pconns,
+ "Number of parallel connections (default: 100)", NULL},
+ {"time", 't', 0, G_OPTION_ARG_DOUBLE, &test_time,
+ "Time to run tests (default: 10.0 sec)", NULL},
+ {"openssl", 'o', 0, G_OPTION_ARG_NONE, &openssl_mode,
+ "Use openssl crypto", NULL},
+ {"host", 'h', 0, G_OPTION_ARG_STRING, &host,
+ "Connect to the specified host (default: localhost)", NULL},
+ {"key", 'k', 0, G_OPTION_ARG_STRING, &server_key,
+ "Use the specified key (base32 encoded)", NULL},
+ {"latency", 'l', 0, G_OPTION_ARG_FILENAME, &latencies_file,
+ "Write latencies to the specified file", NULL},
+ {"csv", 0, 0, G_OPTION_ARG_NONE, &csv_output,
+ "Output CSV", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+struct lat_elt {
+ gdouble lat;
+ guchar checked;
+};
+
+static struct lat_elt *latencies;
+
+static gint
+rspamd_client_body(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg,
+ const gchar *chunk, gsize len)
+{
+ g_assert(chunk[0] == '\0');
+
+ return 0;
+}
+
+struct client_cbdata {
+ struct lat_elt *lat;
+ guint32 *wconns;
+ gdouble ts;
+ struct ev_loop *ev_base;
+};
+
+static void
+rspamd_client_err(struct rspamd_http_connection *conn, GError *err)
+{
+ msg_info("abnormally closing connection from: error: %s",
+ err->message);
+
+ g_assert(0);
+ close(conn->fd);
+ rspamd_http_connection_unref(conn);
+}
+
+static gint
+rspamd_client_finish(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct client_cbdata *cb = conn->ud;
+
+ cb->lat->lat = rspamd_get_ticks() - cb->ts;
+ cb->lat->checked = TRUE;
+ (*cb->wconns)++;
+ conns_pending--;
+ close(conn->fd);
+ rspamd_http_connection_unref(conn);
+ g_free(cb);
+
+ if (conns_pending == 0) {
+ event_base_loopexit(cb->ev_base, NULL);
+ }
+
+ return 0;
+}
+
+static void
+rspamd_http_client_func(struct ev_loop *ev_base, struct lat_elt *latency,
+ guint32 *wconns,
+ struct rspamd_cryptobox_pubkey *peer_key,
+ struct rspamd_cryptobox_keypair *client_key,
+ struct rspamd_keypair_cache *c)
+{
+ struct rspamd_http_message *msg;
+ struct rspamd_http_connection *conn;
+ gchar urlbuf[PATH_MAX];
+ struct client_cbdata *cb;
+ gint fd, flags;
+
+ fd = rspamd_inet_address_connect(addr, SOCK_STREAM, TRUE);
+ g_assert(fd != -1);
+ flags = 1;
+ (void) setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof(flags));
+ conn = rspamd_http_connection_new(rspamd_client_body,
+ rspamd_client_err,
+ rspamd_client_finish,
+ RSPAMD_HTTP_CLIENT_SIMPLE,
+ RSPAMD_HTTP_CLIENT,
+ c,
+ NULL);
+ rspamd_snprintf(urlbuf, sizeof(urlbuf), "http://%s/%d", host, file_size);
+ msg = rspamd_http_message_from_url(urlbuf);
+
+ g_assert(conn != NULL && msg != NULL);
+
+ if (peer_key != NULL) {
+ g_assert(client_key != NULL);
+ rspamd_http_connection_set_key(conn, client_key);
+ msg->peer_key = rspamd_pubkey_ref(peer_key);
+ }
+
+ cb = g_malloc(sizeof(*cb));
+ cb->ts = rspamd_get_ticks();
+ cb->lat = latency;
+ cb->ev_base = ev_base;
+ cb->wconns = wconns;
+ latency->checked = FALSE;
+ rspamd_http_connection_write_message(conn, msg, NULL, NULL, cb,
+ fd, NULL, ev_base);
+}
+
+static void
+rspamd_worker_func(struct lat_elt *plat, guint32 *wconns)
+{
+ guint i, j;
+ struct ev_loop *ev_base;
+ struct itimerval itv;
+ struct rspamd_keypair_cache *c = NULL;
+ struct rspamd_cryptobox_keypair *client_key = NULL;
+ struct rspamd_cryptobox_pubkey *peer_key = NULL;
+
+ if (server_key) {
+ peer_key = rspamd_pubkey_from_base32(server_key, 0, RSPAMD_KEYPAIR_KEX,
+ openssl_mode ? RSPAMD_CRYPTOBOX_MODE_NIST : RSPAMD_CRYPTOBOX_MODE_25519);
+ g_assert(peer_key != NULL);
+ client_key = rspamd_keypair_new(RSPAMD_KEYPAIR_KEX,
+ openssl_mode ? RSPAMD_CRYPTOBOX_MODE_NIST : RSPAMD_CRYPTOBOX_MODE_25519);
+
+ if (cache_size > 0) {
+ c = rspamd_keypair_cache_new(cache_size);
+ }
+ }
+
+ memset(&itv, 0, sizeof(itv));
+ double_to_tv(test_time, &itv.it_value);
+
+ ev_base = event_init();
+ g_assert(setitimer(ITIMER_REAL, &itv, NULL) != -1);
+
+ for (i = 0;; i = (i + 1) % store_latencies) {
+ for (j = 0; j < pconns; j++) {
+ rspamd_http_client_func(ev_base, &plat[i * pconns + j],
+ wconns, peer_key, client_key, c);
+ }
+
+ conns_pending = pconns;
+
+ event_base_loop(ev_base, 0);
+ }
+}
+
+static int
+cmpd(const void *p1, const void *p2)
+{
+ const struct lat_elt *d1 = p1, *d2 = p2;
+
+ return (d1->lat) - (d2->lat);
+}
+
+double
+rspamd_http_calculate_mean(struct lat_elt *lats, double *std)
+{
+ guint i, cnt, checked = 0;
+ gdouble mean = 0., dev = 0.;
+
+ cnt = store_latencies * pconns;
+ qsort(lats, cnt, sizeof(*lats), cmpd);
+
+ for (i = 0; i < cnt; i++) {
+ if (lats[i].checked) {
+ mean += lats[i].lat;
+ checked++;
+ }
+ }
+
+ g_assert(checked > 0);
+ mean /= checked;
+
+ for (i = 0; i < cnt; i++) {
+ if (lats[i].checked) {
+ dev += pow((lats[i].lat - mean), 2);
+ }
+ }
+
+ dev /= checked;
+
+ *std = sqrt(dev);
+ return mean;
+}
+
+static void
+rspamd_http_start_workers(pid_t *sfd)
+{
+ guint i;
+ for (i = 0; i < nworkers; i++) {
+ sfd[i] = fork();
+ g_assert(sfd[i] != -1);
+
+ if (sfd[i] == 0) {
+ gperf_profiler_init(NULL, "http-bench");
+ rspamd_worker_func(&latencies[i * pconns * store_latencies],
+ &conns_done[i]);
+ gperf_profiler_stop();
+ exit(EXIT_SUCCESS);
+ }
+
+ workers_left++;
+ }
+}
+
+static void
+rspamd_http_stop_workers(pid_t *sfd)
+{
+ guint i;
+ gint res;
+
+ for (i = 0; i < nworkers; i++) {
+ kill(sfd[i], SIGTERM);
+ wait(&res);
+ }
+}
+
+static void
+rspamd_http_bench_term(int fd, short what, void *arg)
+{
+ pid_t *sfd = arg;
+
+ rspamd_http_stop_workers(sfd);
+ event_loopexit(NULL);
+}
+
+static void
+rspamd_http_bench_cld(int fd, short what, void *arg)
+{
+ gint res;
+
+ while (waitpid(-1, &res, WNOHANG) > 0) {
+ if (--workers_left == 0) {
+ event_loopexit(NULL);
+ }
+ }
+}
+
+
+int main(int argc, char **argv)
+{
+ GOptionContext *context;
+ GError *error = NULL;
+ pid_t *sfd;
+ struct ev_loop *ev_base;
+ rspamd_mempool_t *pool = rspamd_mempool_new(8192, "http-bench");
+ struct event term_ev, int_ev, cld_ev;
+ guint64 total_done;
+ FILE *lat_file;
+ gdouble mean, std;
+ guint i;
+
+ rspamd_init_libs();
+
+ context = g_option_context_new(
+ "rspamd-http-bench - test server for benchmarks");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd test HTTP benchmark " RVERSION
+ "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, NULL);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ rspamd_fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ exit(EXIT_FAILURE);
+ }
+
+ rspamd_parse_inet_address(&addr, host, 0);
+ g_assert(addr != NULL);
+ rspamd_inet_address_set_port(addr, port);
+
+ latencies = rspamd_mempool_alloc_shared(pool,
+ nworkers * pconns * store_latencies * sizeof(*latencies));
+ sfd = g_malloc(sizeof(*sfd) * nworkers);
+ conns_done = rspamd_mempool_alloc_shared(pool, sizeof(guint32) * nworkers);
+ memset(conns_done, 0, sizeof(guint32) * nworkers);
+
+ rspamd_http_start_workers(sfd);
+
+ ev_base = event_init();
+
+ event_set(&term_ev, SIGTERM, EV_SIGNAL, rspamd_http_bench_term, sfd);
+ event_base_set(ev_base, &term_ev);
+ event_add(&term_ev, NULL);
+ event_set(&int_ev, SIGINT, EV_SIGNAL, rspamd_http_bench_term, sfd);
+ event_base_set(ev_base, &int_ev);
+ event_add(&int_ev, NULL);
+ event_set(&cld_ev, SIGCHLD, EV_SIGNAL | EV_PERSIST,
+ rspamd_http_bench_cld, NULL);
+ event_base_set(ev_base, &cld_ev);
+ event_add(&cld_ev, NULL);
+
+ event_base_loop(ev_base, 0);
+
+ total_done = 0;
+ for (i = 0; i < nworkers; i++) {
+ total_done += conns_done[i];
+ }
+
+ mean = rspamd_http_calculate_mean(latencies, &std);
+
+ if (!csv_output) {
+ rspamd_printf(
+ "Made %L connections of size %d in %.6fs, %.6f cps, %.6f MB/sec\n",
+ total_done,
+ file_size,
+ test_time,
+ total_done / test_time,
+ total_done * file_size / test_time / (1024.0 * 1024.0));
+ rspamd_printf("Latency: %.6f ms mean, %.6f dev\n",
+ mean * 1000.0, std * 1000.0);
+ }
+ else {
+ /* size,connections,time,mean,stddev,conns,workers */
+ rspamd_printf("%ud,%L,%.1f,%.6f,%.6f,%ud,%ud\n",
+ file_size,
+ total_done,
+ test_time,
+ mean * 1000.0,
+ std * 1000.0,
+ pconns,
+ nworkers);
+ }
+
+ if (latencies_file) {
+ lat_file = fopen(latencies_file, "w");
+
+ if (lat_file) {
+ for (i = 0; i < store_latencies * pconns; i++) {
+ if (latencies[i].checked) {
+ rspamd_fprintf(lat_file, "%.6f\n", latencies[i].lat);
+ }
+ }
+
+ fclose(lat_file);
+ }
+ }
+
+ rspamd_mempool_delete(pool);
+
+ return 0;
+}
diff --git a/utils/rspamd_http_server.c b/utils/rspamd_http_server.c
new file mode 100644
index 0000000..ecd1d38
--- /dev/null
+++ b/utils/rspamd_http_server.c
@@ -0,0 +1,300 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rspamd.h"
+#include "util.h"
+#include "libutil/fstring.h"
+#include "libutil/http.h"
+#include "libutil/http_private.h"
+#include "ottery.h"
+#include "cryptobox.h"
+#include "keypair.h"
+#include "unix-std.h"
+#include <math.h>
+
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+
+static guint port = 43000;
+static guint cache_size = 10;
+static guint nworkers = 1;
+static gboolean openssl_mode = FALSE;
+static GHashTable *maps = NULL;
+static gchar *key = NULL;
+static struct rspamd_keypair_cache *c;
+static struct rspamd_cryptobox_keypair *server_key;
+static struct timeval io_tv = {
+ .tv_sec = 20,
+ .tv_usec = 0};
+
+static GOptionEntry entries[] = {
+ {"port", 'p', 0, G_OPTION_ARG_INT, &port,
+ "Port number (default: 43000)", NULL},
+ {"cache", 'c', 0, G_OPTION_ARG_INT, &cache_size,
+ "Keys cache size (default: 10)", NULL},
+ {"workers", 'n', 0, G_OPTION_ARG_INT, &nworkers,
+ "Number of workers to start (default: 1)", NULL},
+ {"openssl", 'o', 0, G_OPTION_ARG_NONE, &openssl_mode,
+ "Use openssl crypto", NULL},
+ {"key", 'k', 0, G_OPTION_ARG_STRING, &key,
+ "Use static keypair instead of new one (base32 encoded sk || pk)", NULL},
+ {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
+
+struct rspamd_http_server_session {
+ struct rspamd_http_connection *conn;
+ struct ev_loop *ev_base;
+ guint req_size;
+ gboolean reply;
+ gint fd;
+};
+
+static void
+rspamd_server_error(struct rspamd_http_connection *conn,
+ GError *err)
+{
+ struct rspamd_http_server_session *session = conn->ud;
+
+ rspamd_fprintf(stderr, "http error occurred: %s\n", err->message);
+ rspamd_http_connection_unref(conn);
+ close(session->fd);
+ g_slice_free1(sizeof(*session), session);
+}
+
+static int
+rspamd_server_finish(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct rspamd_http_server_session *session = conn->ud;
+ struct rspamd_http_message *reply;
+ gulong size;
+ const gchar *url_str;
+ guint url_len;
+ rspamd_fstring_t *body;
+
+ if (!session->reply) {
+ session->reply = TRUE;
+ reply = rspamd_http_new_message(HTTP_RESPONSE);
+ url_str = msg->url->str;
+ url_len = msg->url->len;
+
+ if (url_str[0] == '/') {
+ url_str++;
+ url_len--;
+ }
+
+ if (rspamd_strtoul(url_str, url_len, &size)) {
+ session->req_size = size;
+
+ reply->code = 200;
+ reply->status = rspamd_fstring_new_init("OK", 2);
+ body = rspamd_fstring_sized_new(size);
+ body->len = size;
+ memset(body->str, 0, size);
+ rspamd_http_message_set_body_from_fstring_steal(msg, body);
+ }
+ else {
+ reply->code = 404;
+ reply->status = rspamd_fstring_new_init("Not found", 9);
+ }
+
+ rspamd_http_connection_reset(conn);
+ rspamd_http_connection_write_message(conn, reply, NULL,
+ "application/octet-stream", session, session->fd,
+ &io_tv, session->ev_base);
+ }
+ else {
+ /* Destroy session */
+ rspamd_http_connection_unref(conn);
+ close(session->fd);
+ g_slice_free1(sizeof(*session), session);
+ }
+
+ return 0;
+}
+
+static void
+rspamd_server_accept(gint fd, short what, void *arg)
+{
+ struct ev_loop *ev_base = arg;
+ struct rspamd_http_server_session *session;
+ rspamd_inet_addr_t *addr;
+ gint nfd;
+
+ do {
+ if ((nfd =
+ rspamd_accept_from_socket(fd, &addr, NULL)) == -1) {
+ rspamd_fprintf(stderr, "accept failed: %s", strerror(errno));
+ return;
+ }
+ /* Check for EAGAIN */
+ if (nfd == 0) {
+ rspamd_inet_address_free(addr);
+ return;
+ }
+
+ rspamd_inet_address_free(addr);
+ session = g_slice_alloc(sizeof(*session));
+ session->conn = rspamd_http_connection_new(NULL,
+ rspamd_server_error,
+ rspamd_server_finish,
+ 0,
+ RSPAMD_HTTP_SERVER,
+ c,
+ NULL);
+ rspamd_http_connection_set_key(session->conn, server_key);
+ rspamd_http_connection_read_message(session->conn,
+ session,
+ nfd,
+ &io_tv,
+ ev_base);
+ session->reply = FALSE;
+ session->fd = nfd;
+ session->ev_base = ev_base;
+ } while (nfd > 0);
+}
+
+static void
+rspamd_http_term_handler(gint fd, short what, void *arg)
+{
+ struct ev_loop *ev_base = arg;
+ struct timeval tv = {0, 0};
+
+ event_base_loopexit(ev_base, &tv);
+}
+
+static void
+rspamd_http_server_func(gint fd, rspamd_inet_addr_t *addr)
+{
+ struct ev_loop *ev_base = event_init();
+ struct event accept_ev, term_ev;
+
+ event_set(&accept_ev, fd, EV_READ | EV_PERSIST, rspamd_server_accept, ev_base);
+ event_base_set(ev_base, &accept_ev);
+ event_add(&accept_ev, NULL);
+
+ evsignal_set(&term_ev, SIGTERM, rspamd_http_term_handler, ev_base);
+ event_base_set(ev_base, &term_ev);
+ event_add(&term_ev, NULL);
+
+ event_base_loop(ev_base, 0);
+}
+
+static void
+rspamd_http_start_servers(pid_t *sfd, rspamd_inet_addr_t *addr)
+{
+ guint i;
+ gint fd;
+
+ fd = rspamd_inet_address_listen(addr, SOCK_STREAM, TRUE);
+ g_assert(fd != -1);
+
+ for (i = 0; i < nworkers; i++) {
+ sfd[i] = fork();
+ g_assert(sfd[i] != -1);
+
+ if (sfd[i] == 0) {
+ rspamd_http_server_func(fd, addr);
+ exit(EXIT_SUCCESS);
+ }
+ }
+
+ close(fd);
+}
+
+static void
+rspamd_http_stop_servers(pid_t *sfd)
+{
+ guint i;
+ gint res;
+
+ for (i = 0; i < nworkers; i++) {
+ kill(sfd[i], SIGTERM);
+ wait(&res);
+ }
+}
+
+static void
+rspamd_http_server_term(int fd, short what, void *arg)
+{
+ pid_t *sfd = arg;
+
+ rspamd_http_stop_servers(sfd);
+ event_loopexit(NULL);
+}
+
+int main(int argc, gchar **argv)
+{
+ GOptionContext *context;
+ GError *error = NULL;
+ struct ev_loop *ev_base;
+ GString *b32_key;
+ pid_t *sfd;
+ rspamd_inet_addr_t *addr;
+ struct event term_ev, int_ev;
+ struct in_addr ina = {INADDR_ANY};
+
+ rspamd_init_libs();
+
+ context = g_option_context_new(
+ "rspamd-http-server - test server for benchmarks");
+ g_option_context_set_summary(context,
+ "Summary:\n Rspamd test HTTP server " RVERSION
+ "\n Release id: " RID);
+ g_option_context_add_main_entries(context, entries, NULL);
+
+ if (!g_option_context_parse(context, &argc, &argv, &error)) {
+ rspamd_fprintf(stderr, "option parsing failed: %s\n", error->message);
+ g_error_free(error);
+ exit(EXIT_FAILURE);
+ }
+
+ maps = g_hash_table_new(g_int_hash, g_int_equal);
+
+ if (key == NULL) {
+ server_key = rspamd_keypair_new(RSPAMD_KEYPAIR_KEX,
+ openssl_mode ? RSPAMD_CRYPTOBOX_MODE_NIST : RSPAMD_CRYPTOBOX_MODE_25519);
+ b32_key = rspamd_keypair_print(server_key,
+ RSPAMD_KEYPAIR_PUBKEY | RSPAMD_KEYPAIR_BASE32);
+ rspamd_printf("key: %v\n", b32_key);
+ }
+ else {
+ /* TODO: add key loading */
+ }
+
+ if (cache_size > 0) {
+ c = rspamd_keypair_cache_new(cache_size);
+ }
+
+ sfd = g_alloca(sizeof(*sfd) * nworkers);
+ addr = rspamd_inet_address_new(AF_INET, &ina);
+ rspamd_inet_address_set_port(addr, port);
+ rspamd_http_start_servers(sfd, addr);
+
+ /* Just wait for workers */
+ ev_base = event_init();
+
+ event_set(&term_ev, SIGTERM, EV_SIGNAL, rspamd_http_server_term, sfd);
+ event_base_set(ev_base, &term_ev);
+ event_add(&term_ev, NULL);
+ event_set(&int_ev, SIGINT, EV_SIGNAL, rspamd_http_server_term, sfd);
+ event_base_set(ev_base, &int_ev);
+ event_add(&int_ev, NULL);
+
+ event_base_loop(ev_base, 0);
+
+ return 0;
+}
diff --git a/utils/rspamd_stats.pl b/utils/rspamd_stats.pl
new file mode 100755
index 0000000..9c5f2ac
--- /dev/null
+++ b/utils/rspamd_stats.pl
@@ -0,0 +1,1018 @@
+#!/usr/bin/env perl
+
+use 5.010;
+use Data::Dumper;
+use Getopt::Long;
+use Pod::Usage;
+use Time::Local;
+use IO::Handle;
+use warnings;
+use strict;
+
+my @symbols_search;
+my @symbols_exclude;
+my @symbols_bidirectional;
+my @symbols_groups;
+my @symbols_ignored;
+my %symbols_mult;
+my %groups;
+my $reject_score = 15.0;
+my $junk_score = 6.0;
+my $diff_alpha = 0.1;
+my $correlations = 0;
+my $nrelated = 10;
+my $log_file = "";
+my $search_pattern = "";
+my $startTime = "";
+my $endTime;
+my $num_logs;
+my $exclude_logs = 0;
+my $man = 0;
+my $json = 0;
+my $help = 0;
+
+# Associate file extensions with decompressors
+my %decompressor = (
+ 'bz2' => 'bzip2 -cd',
+ 'gz' => 'gzip -cd',
+ 'xz' => 'xz -cd',
+ 'zst' => 'zstd -cd',
+);
+
+GetOptions(
+ "reject-score|r=f" => \$reject_score,
+ "junk-score|j=f" => \$junk_score,
+ "symbol|s=s@" => \@symbols_search,
+ "symbol-bidir|S=s@" => \@symbols_bidirectional,
+ "exclude|X=s@" => \@symbols_exclude,
+ "ignore=s@" => \@symbols_ignored,
+ "group|g=s@" => \@symbols_groups,
+ "log|l=s" => \$log_file,
+ "mult=s" => \%symbols_mult,
+ "alpha-score|alpha|a=f" => \$diff_alpha,
+ "correlations|c" => \$correlations,
+ "nrelated=i" => \$nrelated,
+ "search-pattern=s" => \$search_pattern,
+ "start=s" => \$startTime,
+ "end=s" => \$endTime,
+ "num-logs|n=i" => \$num_logs,
+ "exclude-logs|x=i" => \$exclude_logs,
+ "json|j" => \$json,
+ "help|?" => \$help,
+ "man" => \$man
+) or pod2usage(2);
+
+pod2usage(1) if $help;
+pod2usage( -exitval => 0, -verbose => 2 ) if $man;
+
+# Global vars
+my $total = 0;
+my $total_spam = 0;
+my $total_junk = 0;
+my $junk_symbols = 0;
+my $spam_symbols = 0;
+my $ham_symbols = 0;
+my $ham_spam_change = 0;
+my $ham_junk_change = 0;
+my %sym_res;
+my $rspamd_log;
+my $enabled = 0;
+my $log_file_num = 1;
+my $spinner_update_time = 0;
+
+my %action;
+my %timeStamp;
+my %scanTime = (
+ max => 0,
+ total => 0,
+);
+my %bidir_match;
+
+foreach ( $startTime, $endTime ) { $_ = &normalized_time($_) }
+
+# Convert bidirectional symbols
+foreach my $s (@symbols_bidirectional) {
+ $bidir_match{$s} = {
+ spam => "${s}_SPAM",
+ ham => "${s}_HAM",
+ };
+ push @symbols_search, $s unless grep /^$s$/, @symbols_search;
+}
+
+# Deal with groups
+my $group_id = 0;
+foreach my $g (@symbols_groups) {
+ my @symbols = split /,/, $g;
+ my $group_name = "group$group_id";
+
+ foreach my $s (@symbols) {
+ $groups{$s} = $group_name;
+ push @symbols_search, $s unless grep /^$s$/, @symbols_search;
+ }
+}
+
+@symbols_search = '.*'
+ unless @symbols_search;
+
+if ( $log_file eq '-' || $log_file eq '' ) {
+ $rspamd_log = \*STDIN;
+ &ProcessLog();
+}
+elsif ( -d "$log_file" ) {
+ my $log_dir = "$log_file";
+
+ my @logs = &GetLogfilesList($log_dir);
+
+ # Process logs
+ foreach (@logs) {
+ my $ext = (/[^.]+\.?([^.]*?)$/)[0];
+ my $dc = $decompressor{$ext} || 'cat';
+
+ open( $rspamd_log, "-|", "$dc $log_dir/$_" )
+ or die "cannot execute $dc $log_dir/$_ : $!";
+
+ printf { interactive(*STDERR) } "\033[J Parsing log files: [%d/%d] %s\033[G", $log_file_num++, scalar @logs,
+ $_;
+ $spinner_update_time = 0; # Force spinner update
+ &spinner;
+
+ &ProcessLog;
+
+ close($rspamd_log)
+ or warn "cannot close $dc $log_dir/$_: $!";
+ }
+ print { interactive(*STDERR) } "\033[J\033[G"; # Progress indicator clean-up
+}
+else {
+ my $ext = ( $log_file =~ /[^.]+\.?([^.]*?)$/ )[0];
+ my $dc = $decompressor{$ext} || 'cat';
+ open( $rspamd_log, "-|", "$dc $log_file" )
+ or die "cannot execute $dc $log_file : $!";
+ $spinner_update_time = 0; # Force spinner update
+ &spinner;
+ &ProcessLog();
+}
+
+my $total_ham = $total - ( $total_spam + $total_junk );
+
+if ($json) {
+ print "{";
+ &Summary();
+ print '"symbols":{';
+ &SymbolsStat();
+ print "}}\n";
+}
+else {
+ &SymbolsStat();
+ &Summary();
+}
+
+exit;
+
+sub IsIgnored {
+ my ($sym) = @_;
+
+ foreach my $ex (@symbols_ignored) {
+ if ( $sym =~ /^$ex$/ ) {
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+sub GenRelated {
+ my ( $htb, $target_sym ) = @_;
+
+ my @result;
+ my $i = 0;
+ foreach my $sym ( sort { $htb->{$b} <=> $htb->{$a} } keys %{$htb} ) {
+ if ( $sym ne $target_sym ) {
+ my @elt = ( $sym, $htb->{$sym} );
+ push @result, \@elt;
+ $i++;
+ }
+
+ last if $i > $nrelated;
+ }
+
+ return \@result;
+}
+
+sub StringifyRelated {
+ my ( $ar, $total ) = @_;
+ return
+ join( "\n", ( map { sprintf "\t%s(%s: %.1f%%)", $_->[0], $_->[1], $_->[1] / ( $total * 1.0 ) * 100.0 } @{$ar} ) );
+}
+
+sub SymbolsStat {
+ if ( $total > 0 ) {
+ my $has_comma = 0;
+ while ( my ( $s, $r ) = each(%sym_res) ) {
+ if ( $r->{hits} > 0 ) {
+ my $th = $r->{hits};
+ my $sh = $r->{spam_hits};
+ my $jh = $r->{junk_hits};
+ my $hh = $r->{hits} - $sh - $jh;
+ my ( $htp, $stp, $jtp );
+ $htp = $hh * 100.0 / $total_ham if $total_ham != 0;
+ $stp = $sh * 100.0 / $total_spam if $total_spam != 0;
+ $jtp = $jh * 100.0 / $total_junk if $total_junk != 0;
+
+ if ($json) {
+ if ($has_comma) {
+ print ",";
+ }
+ else {
+ $has_comma = 1;
+ }
+ print "\"$s\":{";
+ JsonObjectElt( "avg_weight", $r->{'weight'}, "%.4f" );
+ print ",";
+ JsonObjectElt( "hits", $th, "%d" );
+ print ",";
+ JsonObjectElt( "hits_percentage", $th / $total, "%.4f" );
+ print ",";
+ JsonObjectElt( "spam_hits", $sh, "%d" );
+ print ",";
+ JsonObjectElt( "spam_to_total", $sh / $th, "%.4f" );
+ print ",";
+ JsonObjectElt( "spam_percentage", $stp / 100.0 || 0, "%.4f" );
+ print ",";
+ JsonObjectElt( "ham_hits", $hh, "%d" );
+ print ",";
+ JsonObjectElt( "ham_to_total", $hh / $th, "%.4f" );
+ print ",";
+ JsonObjectElt( "ham_percentage", $htp / 100.0 || 0, "%.4f" );
+ print ",";
+ JsonObjectElt( "junk_hits", $jh, "%d" );
+ print ",";
+ JsonObjectElt( "junk_to_total", $jh / $th, "%.4f" );
+ print ",";
+ JsonObjectElt( "junk_percentage", $jtp / 100.0 || 0, "%.4f" );
+ }
+ else {
+ printf "%s avg. weight %.3f, hits %d(%.3f%%):
+ Ham %7.3f%%, %6d/%-6d (%7.3f%%)
+ Spam %7.3f%%, %6d/%-6d (%7.3f%%)
+ Junk %7.3f%%, %6d/%-6d (%7.3f%%)
+", $s, $r->{weight} / $r->{hits}, $th, ( $th / $total * 100 ),
+ ( $hh / $th * 100 ), $hh, $total_ham, ( $htp or 0 ),
+ ( $sh / $th * 100 ), $sh, $total_spam, ( $stp or 0 ),
+ ( $jh / $th * 100 ), $jh, $total_junk, ( $jtp or 0 );
+ }
+ my ( $schp, $jchp );
+ $schp = $r->{spam_change} / $total_spam * 100.0 if $total_spam;
+ $jchp = $r->{junk_change} / $total_junk * 100.0 if $total_junk;
+
+ if ( $r->{weight} != 0 ) {
+ if ( !$json ) {
+ if ( $r->{weight} > 0 ) {
+ printf "
+Spam changes (ham/junk -> spam): %6d/%-6d (%7.3f%%)
+Spam changes / total spam hits: %6d/%-6d (%7.3f%%)
+Junk changes (ham -> junk): %6d/%-6d (%7.3f%%)
+Junk changes / total junk hits: %6d/%-6d (%7.3f%%)
+",
+ $r->{spam_change}, $th, ( $r->{spam_change} / $th * 100 ),
+ $r->{spam_change}, $total_spam, ( $schp or 0 ),
+ $r->{junk_change}, $th, ( $r->{junk_change} / $th * 100 ),
+ $r->{junk_change}, $total_junk, ( $jchp or 0 );
+ }
+ else {
+ printf "
+Spam changes (spam -> junk/ham): %6d/%-6d (%7.3f%%)
+Spam changes / total spam hits : %6d/%-6d (%7.3f%%)
+Junk changes (junk -> ham) : %6d/%-6d (%7.3f%%)
+Junk changes / total junk hits : %6d/%-6d (%7.3f%%)
+",
+ $r->{spam_change}, $th, ( $r->{spam_change} / $th * 100 ),
+ $r->{spam_change}, $total_spam, ( $schp or 0 ),
+ $r->{junk_change}, $th, ( $r->{junk_change} / $th * 100 ),
+ $r->{junk_change}, $total_junk, ( $jchp or 0 );
+ }
+ }
+ else {
+ print ",";
+ JsonObjectElt( "spam_change", $r->{spam_change}, "%.4f" );
+ print ",";
+ JsonObjectElt( "junk_change", $r->{junk_change}, "%.4f" );
+ }
+ }
+
+ if ($correlations) {
+
+ my $spam_related = GenRelated( $r->{symbols_met_spam}, $s );
+ my $junk_related = GenRelated( $r->{symbols_met_junk}, $s );
+ my $ham_related = GenRelated( $r->{symbols_met_ham}, $s );
+
+ if ( !$json ) {
+ print "Correlations report:\n";
+
+ while ( my ( $cs, $hits ) = each %{ $r->{corr} } ) {
+ my $corr_prob = $r->{'hits'} / $total;
+ my $merged_hits = 0;
+ if ( $r->{symbols_met_spam}->{$cs} ) {
+ $merged_hits += $r->{symbols_met_spam}->{$cs};
+ }
+ if ( $r->{symbols_met_junk}->{$cs} ) {
+ $merged_hits += $r->{symbols_met_junk}->{$cs};
+ }
+ if ( $r->{symbols_met_ham}->{$cs} ) {
+ $merged_hits += $r->{symbols_met_ham}->{$cs};
+ }
+
+ if ( $merged_hits > 0 ) {
+ printf "Probability of %s when %s fires: %.3f\n", $cs, $s,
+ ( ( $merged_hits / $total ) / $corr_prob );
+ }
+ }
+
+ print "Related symbols report:\n";
+ printf "Top related in spam:\n %s\n", StringifyRelated( $spam_related, $r->{spam_hits} );
+ printf "Top related in junk:\n %s\n", StringifyRelated( $junk_related, $r->{junk_hits} );
+ printf "Top related in ham:\n %s\n",
+ StringifyRelated( $ham_related, $r->{hits} - $r->{spam_hits} - $r->{junk_hits} );
+ }
+ else {
+ print ",";
+ print "\"correllations\":{";
+
+ my $has_comma_ = 0;
+ while ( my ( $cs, $hits ) = each %{ $r->{corr} } ) {
+ if ($has_comma_) {
+ print ",";
+ }
+ else {
+ $has_comma_ = 1;
+ }
+ my $corr_prob = $hits / $total;
+ my $sym_prob = $r->{hits} / $total;
+ JsonObjectElt( $cs, ( $corr_prob / $sym_prob ), "%.4f" );
+ }
+
+ print "}";
+ }
+ }
+
+ print "}" if $json;
+ }
+ else {
+ print "Symbol $s has not been met\n" if !$json;
+ }
+
+ print '-' x 80 . "\n" if !$json;
+ }
+ }
+}
+
+sub Summary() {
+ if ( !$json ) {
+ print "
+=== Summary ", '=' x 68, "
+Messages scanned: $total";
+ printf " [ %s / %s ]
+", $timeStamp{'start'}, $timeStamp{'end'}
+ if defined $timeStamp{'start'};
+ say '';
+ printf "%11s: %6.2f%%, %d\n", $_, 100 * $action{$_} / $total, $action{$_} for sort keys %action;
+ say '';
+ printf "scan time min/avg/max = %.2f/%.2f/%.2f s
+", $scanTime{'min'} / 1000, ($total) ? $scanTime{'total'} / $total / 1000 : undef, $scanTime{'max'} / 1000
+ if exists $scanTime{'min'};
+ say '=' x 80;
+ }
+ else {
+ JsonObjectElt( "total", $total, "%d" );
+ print ",";
+
+ if ( defined $timeStamp{'start'} ) {
+ JsonObjectElt( "start", $timeStamp{'start'} );
+ print ",";
+ }
+
+ if ( defined $timeStamp{'end'} ) {
+ JsonObjectElt( "end", $timeStamp{'end'} );
+ print ",";
+ }
+
+ print "\"actions\":{";
+
+ my $has_comma = 0;
+ foreach my $a ( sort keys %action ) {
+ if ($has_comma) {
+ print ",";
+ }
+ else {
+ $has_comma = 1;
+ }
+ JsonObjectElt( $a, $action{$a}, "%d" );
+ }
+ print "},";
+ }
+}
+
+sub ProcessRelated {
+ my ( $symbols, $target, $source ) = @_;
+
+ foreach my $s ( @{$symbols} ) {
+ $s =~ /^([^\(]+)(\(([^\)]+)\))?/;
+ my $sym_name = $1;
+ my $sym_score = 0;
+
+ if ( $groups{$sym_name} ) {
+ $sym_name = $groups{$sym_name};
+ }
+
+ next if ( $source eq $sym_name );
+
+ next if IsIgnored($sym_name);
+
+ if ($2) {
+ $sym_score = $3 * ($symbols_mult{$sym_name} or 1.0);
+
+ if ( abs($sym_score) < $diff_alpha ) {
+ next;
+ }
+
+ my $bm = $bidir_match{$sym_name};
+ if ($bm) {
+ if ( $sym_score >= 0 ) {
+ $sym_name = $bm->{'spam'};
+ }
+ else {
+ $sym_name = $bm->{'ham'};
+ }
+ }
+ }
+
+ if ( exists( $target->{$sym_name} ) ) {
+ $target->{$sym_name}++;
+ }
+ else {
+ $target->{$sym_name} = 1;
+ }
+ }
+}
+
+sub ProcessLog {
+ my ( $ts_format, @line ) = &log_time_format($rspamd_log);
+
+ while () {
+ last if eof $rspamd_log;
+ $_ = (@line) ? shift @line : <$rspamd_log>;
+
+ if ( !$enabled && ( $search_pattern eq "" || /$search_pattern/ ) ) {
+ $enabled = 1;
+ }
+
+ next if !$enabled;
+
+ if (/^.*rspamd_task_write_log.*$/) {
+ &spinner;
+ my $ts;
+ if ( $ts_format eq 'syslog' ) {
+ $ts = syslog2iso( join ' ', ( split /\s+/ )[ 0 .. 2 ] );
+ }
+ elsif ( $ts_format eq 'syslog5424' ) {
+ /^([0-9-]+)T([0-9:]+)/;
+ $ts = "$1 $2";
+ }
+ else {
+ $ts = join ' ', ( split /\s+/ )[ 0 .. 1 ];
+ }
+
+ next if ( $ts lt $startTime );
+ next if ( defined $endTime && $ts gt $endTime );
+
+ if ( $_ !~
+ /\(([^()]+)\): \[(NaN|-?\d+(?:\.\d+)?)\/(-?\d+(?:\.\d+)?)\]\s+\[([^\]]+)\].+? time: (\d+\.\d+)ms/ )
+ {
+ #print "BAD: $_\n";
+ next;
+ }
+
+ my @symbols = split /(?:\{[^}]*\})?(?:$|,)/, $4;
+ my $scan_time = $5;
+ my $act = $1;
+ my $score = $2 * 1.0;
+ my $skip = 0;
+
+ foreach my $ex (@symbols_exclude) {
+ my @found = grep { /^$ex/ } @symbols;
+
+ if ( scalar(@found) > 0 ) {
+ $skip = 1;
+ last;
+ }
+ }
+
+ next if ( $skip != 0 );
+
+ if ( defined( $timeStamp{'end'} ) ) {
+ $timeStamp{'end'} = $ts if ( $ts gt $timeStamp{'end'} );
+ }
+ else {
+ $timeStamp{'end'} = $ts;
+ }
+
+ if ( defined( $timeStamp{'start'} ) ) {
+ $timeStamp{'start'} = $ts if ( $ts lt $timeStamp{'start'} );
+ }
+ else {
+ $timeStamp{'start'} = $ts;
+ }
+
+ $scanTime{'min'} = $scan_time if ( !exists $scanTime{'min'} || $scanTime{'min'} > $scan_time );
+ $scanTime{'max'} = $scan_time if ( $scanTime{'max'} < $scan_time );
+ $scanTime{'total'} += $scan_time;
+
+ $action{$act}++;
+ $total++;
+
+ if ( $score >= $reject_score ) {
+ $total_spam++;
+ }
+ elsif ( $score >= $junk_score ) {
+ $total_junk++;
+ }
+
+ my @sym_names;
+
+ foreach my $s (@symbols_search) {
+ my @selected = grep /$s/, @symbols;
+
+ if ( scalar(@selected) > 0 ) {
+
+ foreach my $sym (@selected) {
+ $sym =~ /^([^\(]+)(\(([^\)]+)\))?/;
+ my $sym_name = $1;
+ my $sym_score = 0;
+ my $orig_name = $sym_name;
+
+ if ($2) {
+ $sym_score = $3 * ($symbols_mult{$sym_name} or 1.0);
+
+ if ( abs($sym_score) < $diff_alpha ) {
+ next;
+ }
+
+ my $bm = $bidir_match{$sym_name};
+ if ($bm) {
+ if ( $sym_score >= 0 ) {
+ $sym_name = $bm->{'spam'};
+ }
+ else {
+ $sym_name = $bm->{'ham'};
+ }
+ }
+ }
+
+ next if $orig_name !~ /^$s/;
+
+ if ( $groups{$s} ) {
+
+ # Replace with group
+ $sym_name = $groups{$s};
+ }
+
+ push @sym_names, $sym_name;
+
+ if ( !$sym_res{$sym_name} ) {
+ $sym_res{$sym_name} = {
+ hits => 0,
+ spam_hits => 0,
+ junk_hits => 0,
+ spam_change => 0,
+ junk_change => 0,
+ weight => 0,
+ corr => {},
+ symbols_met_spam => {},
+ symbols_met_ham => {},
+ symbols_met_junk => {},
+ };
+ }
+
+ my $r = $sym_res{$sym_name};
+
+ $r->{hits}++;
+ $r->{weight} += $sym_score;
+ my $is_spam = 0;
+ my $is_junk = 0;
+
+ if ( $score >= $reject_score ) {
+ $is_spam = 1;
+ $r->{spam_hits}++;
+ if ($correlations) {
+ ProcessRelated( \@symbols, $r->{symbols_met_spam}, $sym_name );
+ }
+ }
+ elsif ( $score >= $junk_score ) {
+ $is_junk = 1;
+ $r->{junk_hits}++;
+ if ($correlations) {
+ ProcessRelated( \@symbols, $r->{symbols_met_junk}, $sym_name );
+ }
+ }
+ else {
+ if ($correlations) {
+ ProcessRelated( \@symbols, $r->{symbols_met_ham}, $sym_name );
+ }
+ }
+
+ if ( $sym_score != 0 ) {
+ my $score_without = $score - $sym_score;
+
+ if ( $sym_score > 0 ) {
+ if ( $is_spam && $score_without < $reject_score ) {
+ $r->{spam_change}++;
+ }
+ if ( $is_junk && $score_without < $junk_score ) {
+ $r->{junk_change}++;
+ }
+ }
+ else {
+ if ( !$is_spam && $score_without >= $reject_score ) {
+ $r->{spam_change}++;
+ }
+ if ( !$is_junk && $score_without >= $junk_score ) {
+ $r->{junk_change}++;
+ }
+ }
+ }
+ } # End foreach symbols selected
+ }
+ }
+
+ if ($correlations) {
+ foreach my $sym (@sym_names) {
+ next if IsIgnored($sym);
+ my $r = $sym_res{$sym};
+
+ foreach my $corr_sym (@sym_names) {
+ if ( $corr_sym ne $sym ) {
+ if ( $r->{'corr'}->{$corr_sym} ) {
+ $r->{'corr'}->{$corr_sym}++;
+ }
+ else {
+ $r->{'corr'}->{$corr_sym} = 1;
+ }
+ }
+ }
+ } # End of correlations check
+ }
+ }
+ }
+}
+
+sub JsonObjectElt() {
+ my ( $k, $v ) = @_;
+ my $f = defined $_[2] ? $_[2] : '%s';
+
+ if ( $f eq "%s" ) {
+ $f = "\"%s\"";
+ }
+
+ printf "\"%s\":$f", $k, $v;
+}
+
+sub GetLogfilesList {
+ my ($dir) = @_;
+ opendir( my $fh, $dir ) or die $!;
+
+ my $pattern = join( '|', keys %decompressor );
+ my $re = qr/\.[0-9]+(?:\.(?:$pattern))?/;
+
+ # Add unnumbered logs first
+ my @logs =
+ grep { -f "$dir/$_" && !/$re/ } readdir($fh);
+
+ # Add numbered logs
+ rewinddir($fh);
+ push( @logs, ( sort numeric ( grep { -f "$dir/$_" && /$re/ } readdir($fh) ) ) );
+
+ closedir($fh);
+
+ # Select required logs and revers their order
+ @logs =
+ reverse splice( @logs, $exclude_logs, $num_logs ||= @logs - $exclude_logs );
+
+ # Loop through array printing out filenames
+ print { interactive(*STDERR) } "\nLog files to process:\n";
+ foreach my $file (@logs) {
+ print { interactive(*STDERR) } " $file\n";
+ }
+ print { interactive(*STDERR) } "\n";
+
+ return @logs;
+}
+
+sub log_time_format {
+ my $fh = shift;
+ my ( $format, $line );
+ while (<$fh>) {
+ $line = $_;
+
+ # 2017-08-08 00:00:01 #66984(
+ # 2017-08-08 00:00:01.001 #66984(
+ if (/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(\.\d{3,5})? #\d+\(/) {
+ $format = 'rspamd';
+ last;
+ }
+
+ # Aug 8 00:02:50 #66986(
+ elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d #\d+\(/) {
+ $format = 'syslog';
+ last;
+ }
+
+ # Aug 8 00:02:50 hostname rspamd[66986]
+ elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d \S+ rspamd\[\d+\]/) {
+ $format = 'syslog';
+ last;
+ }
+
+ # 2018-04-16T06:25:46.012590+02:00 rspamd rspamd[12968]
+ elsif (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?(Z|[-+]\d{2}:\d{2}) \S+ rspamd\[\d+\]/) {
+ $format = 'syslog5424';
+ last;
+ }
+
+ # Skip newsyslog messages
+ # Aug 8 00:00:00 hostname newsyslog[63284]: logfile turned over
+ elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d\ \S+ newsyslog\[\d+\]: logfile turned over$/) {
+ next;
+ }
+
+ # Skip journalctl messages
+ # -- Logs begin at Mon 2018-01-15 11:16:24 MSK, end at Fri 2018-04-27 09:10:30 MSK. --
+ elsif (
+/^-- Logs begin at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}, end at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}\. --$/
+ )
+ {
+ next;
+ }
+ else {
+ print "Unknown log format\n";
+ exit 1;
+ }
+ }
+ return ( $format, $line );
+}
+
+sub normalized_time {
+ return
+ if !defined( $_ = shift );
+
+ /^\d\d(?::\d\d){0,2}$/
+ ? sprintf '%04d-%02d-%02d %s', 1900 + (localtime)[5], 1 + (localtime)[4], (localtime)[3], $_
+ : $_;
+}
+
+sub numeric {
+ $a =~ /\.(\d+)\./;
+ my $a_num = $1;
+ $b =~ /\.(\d+)\./;
+ my $b_num = $1;
+
+ $a_num <=> $b_num;
+}
+
+sub spinner {
+ my @spinner = qw{/ - \ |};
+ return
+ if ( ( time - $spinner_update_time ) < 1 );
+ $spinner_update_time = time;
+ printf { interactive(*STDERR) } "%s\r", $spinner[ $spinner_update_time % @spinner ];
+ select()->flush();
+}
+
+# Convert syslog timestamp to "ISO 8601 like" format
+# using current year as syslog does not record the year (nor the timezone)
+# or the last year if the guessed time is in the future.
+sub syslog2iso {
+ my %month_map;
+ @month_map{qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)} = 0 .. 11;
+
+ my ( $month, @t ) = $_[0] =~ m/^(\w{3}) \s\s? (\d\d?) \s (\d\d):(\d\d):(\d\d)/x;
+ my $epoch =
+ timelocal( ( reverse @t ), $month_map{$month}, 1900 + (localtime)[5] );
+ sprintf '%04d-%02d-%02d %02d:%02d:%02d', 1900 + (localtime)[5] - ( $epoch > time ), $month_map{$month} + 1, @t;
+}
+
+### Imported from IO::Interactive 1.022 Perl module
+sub is_interactive {
+ ## no critic (ProhibitInteractiveTest)
+
+ my ($out_handle) = ( @_, select ); # Default to default output handle
+
+ # Not interactive if output is not to terminal...
+ return 0 if not -t $out_handle;
+
+ # If *ARGV is opened, we're interactive if...
+ if ( tied(*ARGV) or defined( fileno(ARGV) ) ) { # this is what 'Scalar::Util::openhandle *ARGV' boils down to
+
+ # ...it's currently opened to the magic '-' file
+ return -t *STDIN if defined $ARGV && $ARGV eq '-';
+
+ # ...it's at end-of-file and the next file is the magic '-' file
+ return @ARGV > 0 && $ARGV[0] eq '-' && -t *STDIN if eof *ARGV;
+
+ # ...it's directly attached to the terminal
+ return -t *ARGV;
+ }
+
+ # If *ARGV isn't opened, it will be interactive if *STDIN is attached
+ # to a terminal.
+ else {
+ return -t *STDIN;
+ }
+}
+
+### Imported from IO::Interactive 1.022 Perl module
+local ( *DEV_NULL, *DEV_NULL2 );
+my $dev_null;
+
+BEGIN {
+ pipe *DEV_NULL, *DEV_NULL2
+ or die "Internal error: can't create null filehandle";
+ $dev_null = \*DEV_NULL;
+}
+
+### Imported from IO::Interactive 1.022 Perl module
+sub interactive {
+ my ($out_handle) = ( @_, \*STDOUT ); # Default to STDOUT
+ return &is_interactive ? $out_handle : $dev_null;
+}
+
+__END__
+
+=head1 NAME
+
+rspamd_stats - analyze Rspamd rules by parsing log files
+
+=head1 SYNOPSIS
+
+rspamd_stats [options] [--symbol=SYM1 [--symbol=SYM2...]] [--log file]
+
+ Options:
+ --log=file log file or directory to read (stdin by default)
+ --reject-score=score set reject threshold (15 by default)
+ --junk-score=score set junk score (6.0 by default)
+ --symbol=sym check specified symbol (perl regexps, '.*' by default)
+ --alpha-score=score set ignore score for symbols (0.1 by default)
+ --correlations enable correlations report
+ --nrelated=integer show that amount of related symbols (10 by default)
+ --search-pattern do not process input unless the desired pattern is found
+ --start starting time (oldest) for log parsing
+ --end ending time (newest) for log parsing
+ --num-logs=integer number of recent logfiles to analyze (all files in the directory by default)
+ --exclude-logs=integer number of latest logs to exclude (0 by default)
+ --json print json output instead of human readable
+ --help brief help message
+ --mult=sym=number multiply symbol score
+ --man full documentation
+
+=head1 OPTIONS
+
+=over 8
+
+=item B<--log>
+
+Specifies log file or directory to read data from. If a directory is specified B<rspamd_stats> analyses files in the
+directory including known compressed file types. Number of log files can be limited using B<--num-logs> and
+B<--exclude-logs> options. This assumes that files in the log directory have B<newsyslog(8)>- or B<logrotate(8)>-like
+name format with numeric indexes. Files without indexes (generally it is merely one file) are considered the most
+recent and files with lower indexes are considered newer.
+
+=item B<--reject-score>
+
+Specifies the reject (spam) threshold.
+
+=item B<--junk-score>
+
+Specifies the junk (add header or rewrite subject) threshold.
+
+=item B<--alpha-score>
+
+Specifies the minimum score for a symbol to be considered by this script.
+
+=item B<--symbol>
+
+Add symbol or pattern (pcre format) to analyze.
+
+=item B<--num-logs>
+
+If set, limits number of analyzed logfiles in the directory to the specified value.
+
+=item B<--exclude-logs>
+
+Number of latest logs to exclude (0 by default).
+
+=item B<--correlations>
+
+Additionally print correlation rate for each symbol displayed. This routine calculates merely paired correlations
+between symbols.
+
+=item B<--search-pattern>
+
+Do not process input unless finding the specified regular expression. Useful to skip logs to a certain position.
+
+=item B<--exclude>
+
+Exclude log lines if certain symbols are fired (e.g. GTUBE). You may specify this option multiple time to skip multiple
+symbols.
+
+=item B<--start>
+
+Select log entries after this time. Format: C<YYYY-MM-DD HH:MM:SS> (can be truncated to any desired accuracy). If used
+with B<--end> select entries between B<--start> and B<--end>. The omitted date defaults to the current date if you
+supply the time.
+
+=item B<--end>
+
+Select log entries before this time. Format: C<YYYY-MM-DD HH:MM:SS> (can be truncated to any desired accuracy). If used
+with B<--start> select entries between B<--start> and B<--end>. The omitted date defaults to the current date if you
+supply the time.
+
+=item B<--mult=symbol=number>
+
+Multiplies score for the named symbol by the provided multiplier.
+
+=item B<--help>
+
+Print a brief help message and exits.
+
+=item B<--man>
+
+Prints the manual page and exits.
+
+=back
+
+=head1 DESCRIPTION
+
+B<rspamd_stats> will read the given log file (or standard input) and provide statistics for the specified symbols:
+
+ Symbol: BAYES_SPAM (weight 3.763) (381985 hits, 26.827%)
+ Ham hits: 184557 (48.315%), total ham: 1095487 (ham with BAYES_SPAM: 16.847%)
+ Spam hits: 15134 (3.962%), total spam: 16688 (spam with BAYES_SPAM: 90.688%)
+ Junk hits: 182294 (47.723%), total junk: 311699 (junk with BAYES_SPAM: 58.484%)
+ Spam changes (ham/junk -> spam): 7026 (1.839%), total percentage (changes / spam hits): 42.102%
+ Junk changes (ham -> junk): 95192 (24.920%), total percentage (changes / junk hits): 30.540%
+
+Where there are the following attributes:
+
+=over 4
+
+=item *
+
+B<Weight>: average score for a symbols
+
+=item *
+
+B<Total hits>: total number of hits and percentage of symbol hits divided by total number of messages
+
+=item *
+
+B<HAM hits>: provides the following information about B<HAM> messages with the specified symbol (from left to right):
+
+=over 4
+
+=item 1.
+
+B<total symbol hits>: number of messages that has this symbol and are B<HAM>
+
+=item 2.
+
+B<ham percentage>: number of symbol hits divided by overall B<HAM> messages count
+
+=item 3.
+
+B<total ham hits>: overall number of B<HAM> messages
+
+=item 4.
+
+B<ham with symbol percentage>: percentage of number of hits with specified symbol in B<HAM> messages divided by total
+number of B<HAM> messages.
+
+=back
+
+=item *
+
+B<SPAM hits>: provides the following information about B<SPAM> messages - same as previous but for B<SPAM> class.
+
+=item *
+
+B<Junk hits>: provides the following information about B<Junk> messages - same as previous but for B<JUNK> class.
+
+=item *
+
+B<Spam changes>: displays data about how much messages switched their class because of the specific symbol weight.
+
+=item *
+
+B<Junk changes>: displays data about how much messages switched their class because of the specific symbol weight.
+
+=back
+
+=cut
diff --git a/utils/sa_trivial_convert.lua b/utils/sa_trivial_convert.lua
new file mode 100644
index 0000000..2ea53be
--- /dev/null
+++ b/utils/sa_trivial_convert.lua
@@ -0,0 +1,443 @@
+local fun = require "fun"
+local rspamd_logger = require "rspamd_logger"
+local util = require "rspamd_util"
+local lua_util = require "lua_util"
+local rspamd_regexp = require "rspamd_regexp"
+local ucl = require "ucl"
+
+local complicated = {}
+local rules = {}
+local scores = {}
+
+local function words_to_re(words, start)
+ return table.concat(fun.totable(fun.drop_n(start, words)), " ");
+end
+
+local function split(str, delim)
+ local result = {}
+
+ if not delim then
+ delim = '[^%s]+'
+ end
+
+ for token in string.gmatch(str, delim) do
+ table.insert(result, token)
+ end
+
+ return result
+end
+
+local function handle_header_def(hline, cur_rule)
+ --Now check for modifiers inside header's name
+ local hdrs = split(hline, '[^|]+')
+ local hdr_params = {}
+ local cur_param = {}
+ -- Check if an re is an ordinary re
+ local ordinary = true
+
+ for _,h in ipairs(hdrs) do
+ if h == 'ALL' or h == 'ALL:raw' then
+ ordinary = false
+ else
+ local args = split(h, '[^:]+')
+ cur_param['strong'] = false
+ cur_param['raw'] = false
+ cur_param['header'] = args[1]
+
+ if args[2] then
+ -- We have some ops that are required for the header, so it's not ordinary
+ ordinary = false
+ end
+
+ fun.each(function(func)
+ if func == 'addr' then
+ cur_param['function'] = function(str)
+ local addr_parsed = util.parse_mail_address(str)
+ local ret = {}
+ if addr_parsed then
+ for _,elt in ipairs(addr_parsed) do
+ if elt['addr'] then
+ table.insert(ret, elt['addr'])
+ end
+ end
+ end
+
+ return ret
+ end
+ elseif func == 'name' then
+ cur_param['function'] = function(str)
+ local addr_parsed = util.parse_mail_address(str)
+ local ret = {}
+ if addr_parsed then
+ for _,elt in ipairs(addr_parsed) do
+ if elt['name'] then
+ table.insert(ret, elt['name'])
+ end
+ end
+ end
+
+ return ret
+ end
+ elseif func == 'raw' then
+ cur_param['raw'] = true
+ elseif func == 'case' then
+ cur_param['strong'] = true
+ else
+ rspamd_logger.warnx(rspamd_config, 'Function %1 is not supported in %2',
+ func, cur_rule['symbol'])
+ end
+ end, fun.tail(args))
+
+ -- Some header rules require splitting to check of multiple headers
+ if cur_param['header'] == 'MESSAGEID' then
+ -- Special case for spamassassin
+ ordinary = false
+ elseif cur_param['header'] == 'ToCc' then
+ ordinary = false
+ else
+ table.insert(hdr_params, cur_param)
+ end
+ end
+
+ cur_rule['ordinary'] = ordinary and #hdr_params <= 1
+ cur_rule['header'] = hdr_params
+ end
+end
+
+local function process_sa_conf(f)
+ local cur_rule = {}
+ local valid_rule = false
+
+ local function insert_cur_rule()
+ if not rules[cur_rule.type] then
+ rules[cur_rule.type] = {}
+ end
+
+ local target = rules[cur_rule.type]
+
+ if cur_rule.type == 'header' then
+ if not cur_rule.header[1].header then
+ rspamd_logger.errx(rspamd_config, 'bad rule definition: %1', cur_rule)
+ return
+ end
+ if not target[cur_rule.header[1].header] then
+ target[cur_rule.header[1].header] = {}
+ end
+ target = target[cur_rule.header[1].header]
+ end
+
+ if not cur_rule['symbol'] then
+ rspamd_logger.errx(rspamd_config, 'bad rule definition: %1', cur_rule)
+ return
+ end
+ target[cur_rule['symbol']] = cur_rule
+ cur_rule = {}
+ valid_rule = false
+ end
+
+ local function parse_score(words)
+ if #words == 3 then
+ -- score rule <x>
+ return tonumber(words[3])
+ elseif #words == 6 then
+ -- score rule <x1> <x2> <x3> <x4>
+ -- we assume here that bayes and network are enabled and select <x4>
+ return tonumber(words[6])
+ else
+ rspamd_logger.errx(rspamd_config, 'invalid score for %1', words[2])
+ end
+
+ return 0
+ end
+
+ local skip_to_endif = false
+ local if_nested = 0
+ for l in f:lines() do
+ (function ()
+ l = lua_util.rspamd_str_trim(l)
+ -- Replace bla=~/re/ with bla =~ /re/ (#2372)
+ l = l:gsub('([^%s])%s*([=!]~)%s*([^%s])', '%1 %2 %3')
+
+ if string.len(l) == 0 or string.sub(l, 1, 1) == '#' then
+ return
+ end
+
+ -- Unbalanced if/endif
+ if if_nested < 0 then if_nested = 0 end
+ if skip_to_endif then
+ if string.match(l, '^endif') then
+ if_nested = if_nested - 1
+
+ if if_nested == 0 then
+ skip_to_endif = false
+ end
+ elseif string.match(l, '^if') then
+ if_nested = if_nested + 1
+ elseif string.match(l, '^else') then
+ -- Else counterpart for if
+ skip_to_endif = false
+ end
+ table.insert(complicated, l)
+ return
+ else
+ if string.match(l, '^ifplugin') then
+ skip_to_endif = true
+ if_nested = if_nested + 1
+ table.insert(complicated, l)
+ elseif string.match(l, '^if !plugin%(') then
+ skip_to_endif = true
+ if_nested = if_nested + 1
+ table.insert(complicated, l)
+ elseif string.match(l, '^if') then
+ -- Unknown if
+ skip_to_endif = true
+ if_nested = if_nested + 1
+ table.insert(complicated, l)
+ elseif string.match(l, '^else') then
+ -- Else counterpart for if
+ skip_to_endif = true
+ table.insert(complicated, l)
+ elseif string.match(l, '^endif') then
+ if_nested = if_nested - 1
+ table.insert(complicated, l)
+ end
+ end
+
+ -- Skip comments
+ local words = fun.totable(fun.take_while(
+ function(w) return string.sub(w, 1, 1) ~= '#' end,
+ fun.filter(function(w)
+ return w ~= "" end,
+ fun.iter(split(l)))))
+
+ if words[1] == "header" then
+ -- header SYMBOL Header ~= /regexp/
+ if valid_rule then
+ insert_cur_rule()
+ end
+ if words[4] and (words[4] == '=~' or words[4] == '!~') then
+ cur_rule['type'] = 'header'
+ cur_rule['symbol'] = words[2]
+
+ if words[4] == '!~' then
+ table.insert(complicated, l)
+ return
+ end
+
+ cur_rule['re_expr'] = words_to_re(words, 4)
+ local unset_comp = string.find(cur_rule['re_expr'], '%s+%[if%-unset:')
+ if unset_comp then
+ table.insert(complicated, l)
+ return
+ end
+
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+
+ if not cur_rule['re'] then
+ rspamd_logger.warnx(rspamd_config, "Cannot parse regexp '%1' for %2",
+ cur_rule['re_expr'], cur_rule['symbol'])
+ table.insert(complicated, l)
+ return
+ else
+ handle_header_def(words[3], cur_rule)
+ if not cur_rule['ordinary'] then
+ table.insert(complicated, l)
+ return
+ end
+ end
+
+ valid_rule = true
+ else
+ table.insert(complicated, l)
+ return
+ end
+ elseif words[1] == "body" then
+ -- body SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
+ end
+
+ cur_rule['symbol'] = words[2]
+ if words[3] and (string.sub(words[3], 1, 1) == '/'
+ or string.sub(words[3], 1, 1) == 'm') then
+ cur_rule['type'] = 'sabody'
+ cur_rule['re_expr'] = words_to_re(words, 2)
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ if cur_rule['re'] then
+
+ valid_rule = true
+ end
+ else
+ -- might be function
+ table.insert(complicated, l)
+ return
+ end
+ elseif words[1] == "rawbody" then
+ -- body SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
+ end
+
+ cur_rule['symbol'] = words[2]
+ if words[3] and (string.sub(words[3], 1, 1) == '/'
+ or string.sub(words[3], 1, 1) == 'm') then
+ cur_rule['type'] = 'sarawbody'
+ cur_rule['re_expr'] = words_to_re(words, 2)
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ if cur_rule['re'] then
+ valid_rule = true
+ end
+ else
+ table.insert(complicated, l)
+ return
+ end
+ elseif words[1] == "full" then
+ -- body SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
+ end
+
+ cur_rule['symbol'] = words[2]
+
+ if words[3] and (string.sub(words[3], 1, 1) == '/'
+ or string.sub(words[3], 1, 1) == 'm') then
+ cur_rule['type'] = 'message'
+ cur_rule['re_expr'] = words_to_re(words, 2)
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ cur_rule['raw'] = true
+ if cur_rule['re'] then
+ valid_rule = true
+ end
+ else
+ table.insert(complicated, l)
+ return
+ end
+ elseif words[1] == "uri" then
+ -- uri SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
+ end
+ cur_rule['type'] = 'uri'
+ cur_rule['symbol'] = words[2]
+ cur_rule['re_expr'] = words_to_re(words, 2)
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ if cur_rule['re'] and cur_rule['symbol'] then
+ valid_rule = true
+ else
+ table.insert(complicated, l)
+ return
+ end
+ elseif words[1] == "meta" then
+ -- meta SYMBOL expression
+ if valid_rule then
+ insert_cur_rule()
+ end
+ table.insert(complicated, l)
+ return
+ elseif words[1] == "describe" and valid_rule then
+ cur_rule['description'] = words_to_re(words, 2)
+ elseif words[1] == "score" then
+ scores[words[2]] = parse_score(words)
+ else
+ table.insert(complicated, l)
+ return
+ end
+ end)()
+ end
+ if valid_rule then
+ insert_cur_rule()
+ end
+end
+
+for _,matched in ipairs(arg) do
+ local f = io.open(matched, "r")
+ if f then
+ rspamd_logger.messagex(rspamd_config, 'loading SA rules from %s', matched)
+ process_sa_conf(f)
+ else
+ rspamd_logger.errx(rspamd_config, "cannot open %1", matched)
+ end
+end
+
+local multimap_conf = {}
+
+local function handle_rule(what, syms, hdr)
+ local mtype
+ local filter
+ local fname
+ local header
+ local sym = what:upper()
+ if what == 'sabody' then
+ mtype = 'content'
+ fname = 'body_re.map'
+ filter = 'oneline'
+ elseif what == 'sarawbody' then
+ fname = 'raw_body_re.map'
+ mtype = 'content'
+ filter = 'rawtext'
+ elseif what == 'full' then
+ fname = 'full_re.map'
+ mtype = 'content'
+ filter = 'full'
+ elseif what == 'uri' then
+ fname = 'uri_re.map'
+ mtype = 'url'
+ filter = 'full'
+ elseif what == 'header' then
+ fname = ('hdr_' .. hdr .. '_re.map'):lower()
+ mtype = 'header'
+ header = hdr
+ sym = sym .. '_' .. hdr:upper()
+ else
+ rspamd_logger.errx('unknown type: %s', what)
+ return
+ end
+ local conf = {
+ type = mtype,
+ filter = filter,
+ symbol = 'SA_MAP_AUTO_' .. sym,
+ regexp = true,
+ map = fname,
+ header = header,
+ symbols = {}
+ }
+ local re_file = io.open(fname, 'w')
+
+ for k,r in pairs(syms) do
+ local score = 0.0
+ if scores[k] then
+ score = scores[k]
+ end
+ re_file:write(string.format('/%s/ %s:%f\n', tostring(r.re), k, score))
+ table.insert(conf.symbols, k)
+ end
+
+ re_file:close()
+
+ multimap_conf[sym:lower()] = conf
+ rspamd_logger.messagex('stored %s regexp in %s', sym:lower(), fname)
+end
+
+for k,v in pairs(rules) do
+ if k == 'header' then
+ for h,r in pairs(v) do
+ handle_rule(k, r, h)
+ end
+ else
+ handle_rule(k, v)
+ end
+end
+
+local out = ucl.to_format(multimap_conf, 'ucl')
+local mmap_conf = io.open('auto_multimap.conf', 'w')
+mmap_conf:write(out)
+mmap_conf:close()
+rspamd_logger.messagex('stored multimap conf in %s', 'auto_multimap.conf')
+
+local sa_remain = io.open('auto_sa.conf', 'w')
+fun.each(function(l)
+ sa_remain:write(l)
+ sa_remain:write('\n')
+end, fun.filter(function(l) return not string.match(l, '^%s+$') end, complicated))
+sa_remain:close()
+rspamd_logger.messagex('stored sa remains conf in %s', 'auto_sa.conf')